From 3715fd128eaa5dc6d027789470a5966b8d880253 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 15 Jun 2018 16:24:00 -0400
Subject: [PATCH 0001/1550] Relax pytest constraint in appveyor tests (#2060)

Previously appveyor was failing because pytest was pinned to a version that
made the pytest.timeout package unhappy.  Lets relax this constraint for now.
---
 continuous_integration/setup_conda_environment.cmd | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/continuous_integration/setup_conda_environment.cmd b/continuous_integration/setup_conda_environment.cmd
index 786868fca9f..4f2674dd65b 100644
--- a/continuous_integration/setup_conda_environment.cmd
+++ b/continuous_integration/setup_conda_environment.cmd
@@ -31,7 +31,7 @@ call deactivate
     jupyter_client ^
     mock ^
     psutil ^
-    pytest=3.1 ^
+    pytest ^
     python=%PYTHON% ^
     requests ^
     toolz ^

From 1eb486dae117fc93ef12f80d240117eb4a4f5fd6 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 20 Jun 2018 16:45:16 -0400
Subject: [PATCH 0002/1550] Pull data outside of while loop in gather (#2059)

See https://github.com/dask/distributed/issues/2025
---
 distributed/client.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 87255dd3499..525ccfe205d 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -1391,6 +1391,7 @@ def _gather(self, futures, errors='raise', direct=None, local_worker=None):
         futures2, keys = unpack_remotedata(futures, byte_keys=True)
         keys = [tokey(key) for key in keys]
         bad_data = dict()
+        data = {}
 
         if direct is None:
             try:
@@ -1445,8 +1446,6 @@ def wait(k):
 
             keys = [k for k in keys if k not in bad_keys]
 
-            data = {}
-
             if local_worker:  # look inside local worker
                 data.update({k: local_worker.data[k]
                              for k in keys

From 45bff01259c822986f4ff612b165b3fcdbb954cf Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 15 Jun 2018 16:22:20 -0400
Subject: [PATCH 0003/1550] Avoid reference cycle in str_graph

This caused an intermittent failure in

    distributed/tests/test_batched.py::test_dont_hold_on_to_large_messages
---
 distributed/tests/test_client.py |  2 +-
 distributed/utils.py             | 33 ++++++++++++++++----------------
 2 files changed, 18 insertions(+), 17 deletions(-)

diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index e015db9f99c..554bd451e04 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -3706,7 +3706,7 @@ def start_worker(sleep, duration, repeat=1):
             sleep(1)
 
             for i in range(count):
-                done.acquire()
+                done.acquire(timeout=20)
                 gc.collect()
                 if not running:
                     break
diff --git a/distributed/utils.py b/distributed/utils.py
index f9026976f05..4c02860becc 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -34,7 +34,7 @@
 
 import dask
 from dask import istask
-from toolz import memoize, valmap
+from toolz import memoize
 import tornado
 from tornado import gen
 from tornado.ioloop import IOLoop, PollIOLoop
@@ -798,22 +798,23 @@ def _maybe_complex(task):
             type(task) is dict and any(map(_maybe_complex, task.values())))
 
 
-def str_graph(dsk, extra_values=()):
-    def convert(task):
-        if type(task) is list:
-            return [convert(v) for v in task]
-        if type(task) is dict:
-            return valmap(convert, task)
-        if istask(task):
-            return (task[0],) + tuple(map(convert, task[1:]))
-        try:
-            if task in dsk or task in extra_values:
-                return tokey(task)
-        except TypeError:
-            pass
-        return task
+def convert(task, dsk, extra_values):
+    if type(task) is list:
+        return [convert(v, dsk, extra_values) for v in task]
+    if type(task) is dict:
+        return {k: convert(v, dsk, extra_values) for k, v in task.items()}
+    if istask(task):
+        return (task[0],) + tuple(convert(x, dsk, extra_values) for x in task[1:])
+    try:
+        if task in dsk or task in extra_values:
+            return tokey(task)
+    except TypeError:
+        pass
+    return task
 
-    return {tokey(k): convert(v) for k, v in dsk.items()}
+
+def str_graph(dsk, extra_values=()):
+    return {tokey(k): convert(v, dsk, extra_values) for k, v in dsk.items()}
 
 
 def seek_delimiter(file, delimiter, blocksize):

From a40fc080b909f3db32220030882bf081129a84da Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 18 Jun 2018 09:12:09 -0400
Subject: [PATCH 0004/1550] Use ConnectionPool for Worker.scheduler

---
 distributed/client.py            |  4 ++--
 distributed/core.py              |  4 ++++
 distributed/tests/test_client.py |  2 +-
 distributed/worker.py            | 11 ++++++-----
 4 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 525ccfe205d..7f474171835 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -46,7 +46,7 @@
                          scatter_to_workers, gather_from_workers)
 from .cfexecutor import ClientExecutor
 from .compatibility import Queue as pyQueue, Empty, isqueue, html_escape
-from .core import connect, rpc, clean_exception, CommClosedError
+from .core import connect, rpc, clean_exception, CommClosedError, PooledRPCCall
 from .metrics import time
 from .node import Node
 from .protocol import to_serialize
@@ -575,7 +575,7 @@ def __init__(self, address=None, loop=None, timeout=no_default,
                 logger.info("Config value `scheduler-address` found: %s",
                             address)
 
-        if isinstance(address, rpc):
+        if isinstance(address, (rpc, PooledRPCCall)):
             self.scheduler = address
         elif hasattr(address, "scheduler_address"):
             # It's a LocalCluster or LocalCluster-compatible object
diff --git a/distributed/core.py b/distributed/core.py
index b152a58cf6d..7bd6c12960c 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -598,6 +598,10 @@ def __init__(self, addr, pool, serializers=None, deserializers=None):
         self.serializers = serializers
         self.deserializers = deserializers if deserializers is not None else serializers
 
+    @property
+    def address(self):
+        return self.addr
+
     def __getattr__(self, key):
         @gen.coroutine
         def send_recv_from_rpc(**kwargs):
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 554bd451e04..e015db9f99c 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -3706,7 +3706,7 @@ def start_worker(sleep, duration, repeat=1):
             sleep(1)
 
             for i in range(count):
-                done.acquire(timeout=20)
+                done.acquire()
                 gc.collect()
                 if not running:
                     break
diff --git a/distributed/worker.py b/distributed/worker.py
index 74dbc0949db..52bf3d9af78 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -31,7 +31,7 @@
 from .comm.utils import offload
 from .compatibility import unicode, get_thread_identity, finalize
 from .core import (error_message, CommClosedError,
-                   rpc, pingpong, coerce_to_address)
+                   pingpong, coerce_to_address)
 from .diskutils import WorkSpace
 from .metrics import time
 from .node import ServerNode
@@ -165,14 +165,10 @@ def __init__(self, scheduler_ip=None, scheduler_port=None,
         self._closed = Event()
         self.reconnect = reconnect
         self.executor = executor or ThreadPoolExecutor(self.ncores)
-        self.scheduler = rpc(scheduler_addr, connection_args=self.connection_args)
         self.name = name
         self.scheduler_delay = 0
         self.stream_comms = dict()
         self.heartbeat_active = False
-        self.execution_state = {'scheduler': self.scheduler.address,
-                                'ioloop': self.loop,
-                                'worker': self}
         self._ipython_kernel = None
 
         if self.local_dir not in sys.path:
@@ -216,6 +212,11 @@ def __init__(self, scheduler_ip=None, scheduler_port=None,
                 connection_args=self.connection_args,
                 **kwargs)
 
+        self.scheduler = self.rpc(scheduler_addr)
+        self.execution_state = {'scheduler': self.scheduler.address,
+                                'ioloop': self.loop,
+                                'worker': self}
+
         pc = PeriodicCallback(self.heartbeat, 1000, io_loop=self.io_loop)
         self.periodic_callbacks['heartbeat'] = pc
         self._address = contact_address

From 83682686a52a6b29cba9688ec4ad3c77b52fe671 Mon Sep 17 00:00:00 2001
From: Tom Augspurger <TomAugspurger@users.noreply.github.com>
Date: Thu, 21 Jun 2018 08:00:13 -0500
Subject: [PATCH 0005/1550] BUG: Normalize address before comparison (#2066)

Fixes https://github.com/dask/distributed/issues/2058
---
 distributed/tests/test_worker_client.py | 16 ++++++++++++++++
 distributed/worker.py                   | 22 +++++++++++++++++++---
 2 files changed, 35 insertions(+), 3 deletions(-)

diff --git a/distributed/tests/test_worker_client.py b/distributed/tests/test_worker_client.py
index c4e3c775a3a..2b96ae59c4f 100644
--- a/distributed/tests/test_worker_client.py
+++ b/distributed/tests/test_worker_client.py
@@ -297,3 +297,19 @@ def f():
 
     result = yield c.submit(f)
     assert result
+
+
+@gen_cluster()
+def test_submit_different_names(s, a, b):
+    # https://github.com/dask/distributed/issues/2058
+    da = pytest.importorskip('dask.array')
+    c = yield Client('localhost:' + s.address.split(":")[-1], loop=s.loop,
+                     asynchronous=True)
+    try:
+        X = c.persist(da.random.uniform(size=(100, 10), chunks=50))
+        yield wait(X)
+
+        fut = yield c.submit(lambda x: x.sum().compute(), X)
+        assert fut > 0
+    finally:
+        yield c.close()
diff --git a/distributed/worker.py b/distributed/worker.py
index 52bf3d9af78..7bf212baa6f 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -25,7 +25,7 @@
 from tornado.ioloop import IOLoop
 from tornado.locks import Event
 
-from . import profile
+from . import profile, comm
 from .batched import BatchedSend
 from .comm import get_address_host, get_local_address_for, connect
 from .comm.utils import offload
@@ -2597,11 +2597,25 @@ def get_worker():
         raise ValueError("No workers found")
 
 
-def get_client(address=None, timeout=3):
-    """ Get a client while within a task
+def get_client(address=None, timeout=3, resolve_address=True):
+    """Get a client while within a task.
 
     This client connects to the same scheduler to which the worker is connected
 
+    Parameters
+    ----------
+    address : str, optional
+        The address of the scheduler to connect to. Defaults to the scheduler
+        the worker is connected to.
+    timeout : int, default 3
+        Timeout (in seconds) for getting the Client
+    resolve_address : bool, default True
+        Whether to resolve `address` to its canonical form.
+
+    Returns
+    -------
+    Client
+
     Examples
     --------
     >>> def f():
@@ -2620,6 +2634,8 @@ def get_client(address=None, timeout=3):
     worker_client
     secede
     """
+    if address and resolve_address:
+        address = comm.resolve_address(address)
     try:
         worker = get_worker()
     except ValueError:  # could not find worker

From cdec12c5da4b201a37b9aeca9411930f82df1f77 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 21 Jun 2018 18:47:11 -0400
Subject: [PATCH 0006/1550] Add asynchronous parameter to docstring of
 LocalCluster

---
 distributed/deploy/local.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index 399a4746b51..b88966a2a02 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -47,6 +47,9 @@ class LocalCluster(Cluster):
         to choose a random port, ``None`` to disable it, or an
         :samp:`({ip}:{port})` tuple to listen on a different IP address than
         the scheduler.
+    asynchronous: bool (False by default)
+        Set to True if using this cluster within async/await functions or within
+        Tornado gen.coroutines.  This should remain False for normal use.
     kwargs: dict
         Extra worker arguments, will be passed to the Worker constructor.
     service_kwargs: Dict[str, Dict]

From deaa0b3bfeb0076f458eabe748a10e86b40e1155 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sat, 23 Jun 2018 14:36:08 -0400
Subject: [PATCH 0007/1550] Support async def functions in Client.sync (#2070)

Remove support for using sync for synchronous functions
---
 distributed/tests/py3_test_client.py | 14 +++++++++++++-
 distributed/tests/test_utils.py      |  6 ------
 distributed/utils.py                 | 12 ++++--------
 3 files changed, 17 insertions(+), 15 deletions(-)

diff --git a/distributed/tests/py3_test_client.py b/distributed/tests/py3_test_client.py
index 8abd568d385..9c7a1fdf3ef 100644
--- a/distributed/tests/py3_test_client.py
+++ b/distributed/tests/py3_test_client.py
@@ -3,8 +3,9 @@
 import pytest
 from tornado import gen
 
-from distributed.utils_test import div, gen_cluster, inc, loop
+from distributed.utils_test import div, gen_cluster, inc, loop, cluster
 from distributed import as_completed, Client, Lock
+from distributed.utils import sync
 
 
 @gen_cluster(client=True)
@@ -111,3 +112,14 @@ async def f():
                 assert result is False
 
     loop.run_sync(f)
+
+
+def test_client_sync_with_async_def(loop):
+    async def ff():
+        await gen.sleep(0.01)
+        return 1
+
+    with cluster() as (s, [a, b]):
+        with Client(s['address'], loop=loop) as c:
+            assert sync(loop, ff) == 1
+            assert c.sync(ff) == 1
diff --git a/distributed/tests/test_utils.py b/distributed/tests/test_utils.py
index e53cbdbc7a1..60b5105a78d 100644
--- a/distributed/tests/test_utils.py
+++ b/distributed/tests/test_utils.py
@@ -59,12 +59,6 @@ def f():
     loop.run_sync(f)
 
 
-def test_sync(loop_in_thread):
-    loop = loop_in_thread
-    result = sync(loop, inc, 1)
-    assert result == 2
-
-
 def test_sync_error(loop_in_thread):
     loop = loop_in_thread
     try:
diff --git a/distributed/utils.py b/distributed/utils.py
index 4c02860becc..e10c64e6344 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -238,13 +238,6 @@ def sync(loop, func, *args, **kwargs):
 
     timeout = kwargs.pop('callback_timeout', None)
 
-    def make_coro():
-        coro = gen.maybe_future(func(*args, **kwargs))
-        if timeout is None:
-            return coro
-        else:
-            return gen.with_timeout(timedelta(seconds=timeout), coro)
-
     e = threading.Event()
     main_tid = get_thread_identity()
     result = [None]
@@ -257,7 +250,10 @@ def f():
                 raise RuntimeError("sync() called from thread of running loop")
             yield gen.moment
             thread_state.asynchronous = True
-            result[0] = yield make_coro()
+            future = func(*args, **kwargs)
+            if timeout is not None:
+                future = gen.with_timeout(timedelta(seconds=timeout), future)
+            result[0] = yield future
         except Exception as exc:
             error[0] = sys.exc_info()
         finally:

From 84246ff3943513a58241e39f44d5f02241d6b809 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sat, 23 Jun 2018 20:28:08 -0400
Subject: [PATCH 0008/1550] Have worker data transfer wait until recipient
 acknowledges (#2052)

Previously when a worker sent data it would think it was finished as soon as the data was dumped to the socket. Now we wait until we hear an acknowledgement from the recipient that the transfer is complete. This helps with our diagnostics a bit may help avoid backing up a bunch of memory on the OS level, and also assists in future GPU work, where the sending side wants to wait until deserialization on the recipient side has finished.
---
 distributed/tests/test_utils_comm.py |  2 ++
 distributed/tests/test_worker.py     | 18 +++++++++++++
 distributed/utils_comm.py            |  6 ++---
 distributed/worker.py                | 39 ++++++++++++++++++++++++----
 4 files changed, 56 insertions(+), 9 deletions(-)

diff --git a/distributed/tests/test_utils_comm.py b/distributed/tests/test_utils_comm.py
index 6d633a09db5..1e69eef6a03 100644
--- a/distributed/tests/test_utils_comm.py
+++ b/distributed/tests/test_utils_comm.py
@@ -1,5 +1,6 @@
 from __future__ import print_function, division, absolute_import
 
+import pytest
 
 from distributed.core import rpc
 from distributed.utils_test import gen_cluster
@@ -13,6 +14,7 @@ def test_pack_data():
     assert pack_data({'a': ['x'], 'b': 'y'}, data) == {'a': [1], 'b': 'y'}
 
 
+@pytest.mark.xfail(reason='rpc now needs to be a connection pool')
 @gen_cluster(client=True)
 def test_gather_from_workers_permissive(c, s, a, b):
     x = yield c.scatter({'x': 1}, workers=a.address)
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index afe95c0a69c..e8878029e8e 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -1162,3 +1162,21 @@ def test_scheduler_address_config(c, s):
         yield worker._start()
         assert worker.scheduler.address == s.address
     yield worker._close()
+
+
+@slow
+@gen_cluster(client=True)
+def test_wait_for_outgoing(c, s, a, b):
+    np = pytest.importorskip('numpy')
+    x = np.random.random(10000000)
+    future = yield c.scatter(x, workers=a.address)
+
+    y = c.submit(inc, future, workers=b.address)
+    yield wait(y)
+
+    assert len(b.incoming_transfer_log) == len(a.outgoing_transfer_log) == 1
+    bb = b.incoming_transfer_log[0]['duration']
+    aa = a.outgoing_transfer_log[0]['duration']
+    ratio = aa / bb
+
+    assert 1 / 3 < ratio < 3
diff --git a/distributed/utils_comm.py b/distributed/utils_comm.py
index 0f66def37ff..8e4d2ac1300 100644
--- a/distributed/utils_comm.py
+++ b/distributed/utils_comm.py
@@ -30,6 +30,7 @@ def gather_from_workers(who_has, rpc, close=True, serializers=None):
     gather
     _gather
     """
+    from .worker import get_data_from_worker
     bad_addresses = set()
     missing_workers = set()
     original_who_has = who_has
@@ -55,10 +56,7 @@ def gather_from_workers(who_has, rpc, close=True, serializers=None):
 
         rpcs = {addr: rpc(addr) for addr in d}
         try:
-            coroutines = {address: rpcs[address].get_data(
-                                    keys=keys,
-                                    close=close,
-                                    serializers=serializers)
+            coroutines = {address: get_data_from_worker(rpc, keys, address)
                           for address, keys in d.items()}
             response = {}
             for worker, c in coroutines.items():
diff --git a/distributed/worker.py b/distributed/worker.py
index 7bf212baa6f..ff7628b7346 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -30,7 +30,7 @@
 from .comm import get_address_host, get_local_address_for, connect
 from .comm.utils import offload
 from .compatibility import unicode, get_thread_identity, finalize
-from .core import (error_message, CommClosedError,
+from .core import (error_message, CommClosedError, send_recv,
                    pingpong, coerce_to_address)
 from .diskutils import WorkSpace
 from .metrics import time
@@ -604,14 +604,17 @@ def delete_data(self, comm=None, keys=None, report=True):
     def get_data(self, comm, keys=None, who=None, serializers=None):
         start = time()
 
-        msg = {k: to_serialize(self.data[k]) for k in keys if k in self.data}
-        nbytes = {k: self.nbytes.get(k) for k in keys if k in self.data}
+        data = {k: self.data[k] for k in keys if k in self.data}
+        msg = {k: to_serialize(v) for k, v in data.items()}
+        nbytes = {k: self.nbytes.get(k) for k in data}
         stop = time()
         if self.digests is not None:
             self.digests['get-data-load-duration'].add(stop - start)
         start = time()
         try:
             compressed = yield comm.write(msg, serializers=serializers)
+            response = yield comm.read(deserializers=serializers)
+            assert response == 'OK', response
         except EnvironmentError:
             logger.exception('failed during get data with %s -> %s',
                              self.address, who, exc_info=True)
@@ -1771,8 +1774,7 @@ def gather_dep(self, worker, dep, deps, total_nbytes, cause=None):
                 logger.debug("Request %d keys", len(deps))
 
                 start = time() + self.scheduler_delay
-                response = yield self.rpc(worker).get_data(keys=deps,
-                                                           who=self.address)
+                response = yield get_data_from_worker(self.rpc, deps, worker, self.address)
                 stop = time() + self.scheduler_delay
 
                 if cause:
@@ -2714,3 +2716,30 @@ def parse_memory_limit(memory_limit, ncores):
         return parse_bytes(memory_limit)
     else:
         return int(memory_limit)
+
+
+@gen.coroutine
+def get_data_from_worker(rpc, keys, worker, who=None):
+    """ Get keys from worker
+
+    The worker has a two step handshake to acknowledge when data has been fully
+    delivered.  This function implements that handshake.
+
+    See Also
+    --------
+    Worker.get_data
+    Worker.gather_deps
+    utils_comm.gather_data_from_workers
+    """
+    comm = yield rpc.connect(worker)
+    try:
+        response = yield send_recv(comm,
+                                   serializers=rpc.serializers,
+                                   deserializers=rpc.deserializers,
+                                   deserialize=rpc.deserialize,
+                                   op='get_data', keys=keys, who=who)
+        yield comm.write('OK')
+    finally:
+        rpc.reuse(worker, comm)
+
+    raise gen.Return(response)

From db758d0f8609dd0fa041b212ecc89a088b57291e Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sat, 23 Jun 2018 20:30:13 -0400
Subject: [PATCH 0009/1550] Allow adaptive to exist without a cluster (#2064)

This allows the Adaptive object to exist on the scheduler and make
recommendations with an external route without being attached to an
explicit cluster object.

For motivation see conversation in https://github.com/dask/dask-yarn/issues/1
---
 distributed/deploy/adaptive.py            | 82 ++++++++++++++---------
 distributed/deploy/tests/test_adaptive.py | 16 ++++-
 2 files changed, 66 insertions(+), 32 deletions(-)

diff --git a/distributed/deploy/adaptive.py b/distributed/deploy/adaptive.py
index d7ae6ea5997..014373ac7a8 100644
--- a/distributed/deploy/adaptive.py
+++ b/distributed/deploy/adaptive.py
@@ -4,6 +4,7 @@
 import logging
 import math
 
+import toolz
 from tornado import gen
 
 from ..metrics import time
@@ -85,7 +86,7 @@ class Adaptive(object):
     the cluster's ``scale_up`` method.
     '''
 
-    def __init__(self, scheduler, cluster, interval='1s', startup_cost='1s',
+    def __init__(self, scheduler, cluster=None, interval='1s', startup_cost='1s',
                  scale_factor=2, minimum=0, maximum=None, wait_count=3,
                  target_duration='5s', worker_key=lambda x: x, **kwargs):
         interval = parse_timedelta(interval, default='ms')
@@ -94,9 +95,10 @@ def __init__(self, scheduler, cluster, interval='1s', startup_cost='1s',
         self.cluster = cluster
         self.startup_cost = parse_timedelta(startup_cost, default='s')
         self.scale_factor = scale_factor
-        self._adapt_callback = PeriodicCallback(self._adapt, interval * 1000,
-                                                io_loop=scheduler.loop)
-        self.scheduler.loop.add_callback(self._adapt_callback.start)
+        if self.cluster:
+            self._adapt_callback = PeriodicCallback(self._adapt, interval * 1000,
+                                                    io_loop=scheduler.loop)
+            self.scheduler.loop.add_callback(self._adapt_callback.start)
         self._adapting = False
         self._workers_to_close_kwargs = kwargs
         self.minimum = minimum
@@ -106,10 +108,13 @@ def __init__(self, scheduler, cluster, interval='1s', startup_cost='1s',
         self.wait_count = wait_count
         self.target_duration = parse_timedelta(target_duration)
 
+        self.scheduler.handlers['adaptive_recommendations'] = self.recommendations
+
     def stop(self):
-        self._adapt_callback.stop()
-        self._adapt_callback = None
-        del self._adapt_callback
+        if self.cluster:
+            self._adapt_callback.stop()
+            self._adapt_callback = None
+            del self._adapt_callback
 
     def needs_cpu(self):
         """
@@ -272,27 +277,21 @@ def get_scale_up_kwargs(self):
         logger.info("Scaling up to %d workers", instances)
         return {'n': instances}
 
-    @gen.coroutine
-    def _adapt(self):
-        if self._adapting:  # Semaphore to avoid overlapping adapt calls
-            return
-
-        self._adapting = True
-        try:
-            should_scale_up = self.should_scale_up()
-            workers = set(self.workers_to_close(key=self.worker_key,
-                                                minimum=self.minimum))
-            if should_scale_up and workers:
-                logger.info("Attempting to scale up and scale down simultaneously.")
-                return
-
-            if should_scale_up:
-                kwargs = self.get_scale_up_kwargs()
-                f = self.cluster.scale_up(**kwargs)
-                self.log.append((time(), 'up', kwargs))
-                if gen.is_future(f):
-                    yield f
-
+    def recommendations(self, comm=None):
+        should_scale_up = self.should_scale_up()
+        workers = set(self.workers_to_close(key=self.worker_key,
+                                            minimum=self.minimum))
+        if should_scale_up and workers:
+            logger.info("Attempting to scale up and scale down simultaneously.")
+            self.close_counts.clear()
+            return {'status': 'error',
+                    'msg': 'Trying to scale up and down simultaneously'}
+
+        elif should_scale_up:
+            self.close_counts.clear()
+            return toolz.merge({'status': 'up'}, self.get_scale_up_kwargs())
+
+        elif workers:
             d = {}
             to_close = []
             for w, c in self.close_counts.items():
@@ -308,8 +307,31 @@ def _adapt(self):
             self.close_counts = d
 
             if to_close:
-                self.log.append((time(), 'down', workers))
-                workers = yield self._retire_workers(workers=to_close)
+                return {'status': 'down', 'workers': to_close}
+        else:
+            self.close_counts.clear()
+            return None
+
+    @gen.coroutine
+    def _adapt(self):
+        if self._adapting:  # Semaphore to avoid overlapping adapt calls
+            return
+
+        self._adapting = True
+        try:
+            recommendations = self.recommendations()
+            if not recommendations:
+                return
+            status = recommendations.pop('status')
+            if status == 'up':
+                f = self.cluster.scale_up(**recommendations)
+                self.log.append((time(), 'up', recommendations))
+                if gen.is_future(f):
+                    yield f
+
+            elif status == 'down':
+                self.log.append((time(), 'down', recommendations['workers']))
+                workers = yield self._retire_workers(workers=recommendations['workers'])
         finally:
             self._adapting = False
 
diff --git a/distributed/deploy/tests/test_adaptive.py b/distributed/deploy/tests/test_adaptive.py
index 9b73756bcfb..3014defa74d 100644
--- a/distributed/deploy/tests/test_adaptive.py
+++ b/distributed/deploy/tests/test_adaptive.py
@@ -8,7 +8,7 @@
 from tornado.ioloop import IOLoop
 
 from distributed import Client, wait, Adaptive, LocalCluster
-from distributed.utils_test import gen_cluster, gen_test, slowinc
+from distributed.utils_test import gen_cluster, gen_test, slowinc, inc
 from distributed.utils_test import loop, nodebug  # noqa: F401
 from distributed.metrics import time
 
@@ -215,7 +215,7 @@ def test_avoid_churn():
                                  diagnostics_port=None)
     client = yield Client(cluster, asynchronous=True)
     try:
-        adapt = Adaptive(cluster.scheduler, cluster, interval=20, wait_count=5)
+        adapt = Adaptive(cluster.scheduler, cluster, interval='20 ms', wait_count=5)
 
         for i in range(10):
             yield client.submit(slowinc, i, delay=0.040)
@@ -392,3 +392,15 @@ def key(ws):
         assert names == {'a-1', 'a-2'} or names == {'b-1', 'b-2'}
     finally:
         yield cluster._close()
+
+
+@gen_cluster(client=True, ncores=[])
+def test_without_cluster(c, s):
+    adapt = Adaptive(scheduler=s)
+
+    future = c.submit(inc, 1)
+    while not s.tasks:
+        yield gen.sleep(0.01)
+
+    response = yield c.scheduler.adaptive_recommendations()
+    assert response['status'] == 'up'

From 53e3770e01f98ce5ae08d191a35fc2b06d8b8269 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 25 Jun 2018 09:12:18 -0400
Subject: [PATCH 0010/1550] Add test for as_completed for loops in Python 2
 (#2071)

---
 distributed/client.py                  |  3 ++-
 distributed/compatibility.py           |  3 +++
 distributed/tests/test_as_completed.py | 20 ++++++++++++++++++--
 3 files changed, 23 insertions(+), 3 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 7f474171835..72c08978ee5 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -45,7 +45,8 @@
 from .utils_comm import (WrappedKey, unpack_remotedata, pack_data,
                          scatter_to_workers, gather_from_workers)
 from .cfexecutor import ClientExecutor
-from .compatibility import Queue as pyQueue, Empty, isqueue, html_escape
+from .compatibility import (Queue as pyQueue, Empty, isqueue, html_escape,
+        StopAsyncIteration)
 from .core import connect, rpc, clean_exception, CommClosedError, PooledRPCCall
 from .metrics import time
 from .node import Node
diff --git a/distributed/compatibility.py b/distributed/compatibility.py
index 2bf4ceb0f4f..ef5e7040586 100644
--- a/distributed/compatibility.py
+++ b/distributed/compatibility.py
@@ -18,6 +18,8 @@
     PY3 = False
     ConnectionRefusedError = OSError
     FileExistsError = OSError
+    class StopAsyncIteration(Exception):
+        pass
 
     import gzip
 
@@ -71,6 +73,7 @@ def iscoroutinefunction(func):
     from gzip import compress as gzip_compress
     ConnectionRefusedError = ConnectionRefusedError
     FileExistsError = FileExistsError
+    StopAsyncIteration = StopAsyncIteration
 
     def isqueue(o):
         return isinstance(o, Queue)
diff --git a/distributed/tests/test_as_completed.py b/distributed/tests/test_as_completed.py
index 42e906f68f6..d9c2636a178 100644
--- a/distributed/tests/test_as_completed.py
+++ b/distributed/tests/test_as_completed.py
@@ -8,10 +8,9 @@
 
 from distributed import Client
 from distributed.client import _as_completed, as_completed, _first_completed
-from distributed.compatibility import Empty
+from distributed.compatibility import Empty, StopAsyncIteration, Queue
 from distributed.utils_test import cluster, gen_cluster, inc
 from distributed.utils_test import loop  # noqa: F401
-from distributed.compatibility import Queue
 
 
 @gen_cluster(client=True)
@@ -152,3 +151,20 @@ def _():
             result = list(ac)
 
             assert result == [x]
+
+
+@gen_cluster(client=True)
+def test_async_for_py2_equivalent(c, s, a, b):
+    futures = c.map(sleep, [0.01] * 3, pure=False)
+    seq = as_completed(futures)
+    x = yield seq.__anext__()
+    y = yield seq.__anext__()
+    z = yield seq.__anext__()
+
+    assert x.done()
+    assert y.done()
+    assert z.done()
+    assert x.key != y.key
+
+    with pytest.raises(StopAsyncIteration):
+        yield seq.__anext__()

From 40e27ea577b25dc643c1166556b3ba273477b053 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 25 Jun 2018 12:29:49 -0400
Subject: [PATCH 0011/1550] support TB and PB in format bytes (#2072)

---
 distributed/utils.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/distributed/utils.py b/distributed/utils.py
index e10c64e6344..c3a3ac9cb62 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -1081,7 +1081,15 @@ def format_bytes(n):
     '12.35 MB'
     >>> format_bytes(1234567890)
     '1.23 GB'
+    >>> format_bytes(1234567890000)
+    '1.23 TB'
+    >>> format_bytes(1234567890000000)
+    '1.23 PB'
     """
+    if n > 1e15:
+        return '%0.2f PB' % (n / 1e15)
+    if n > 1e12:
+        return '%0.2f TB' % (n / 1e12)
     if n > 1e9:
         return '%0.2f GB' % (n / 1e9)
     if n > 1e6:

From 0dedc514caa08d7a177a13c8cc745fd22aaabb20 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 26 Jun 2018 11:39:14 -0400
Subject: [PATCH 0012/1550] Avoid accessing Worker.scheduler_delay around yield
 point (#2074)

If this changed during the transfer then this could cause negative
durations
---
 distributed/worker.py | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/distributed/worker.py b/distributed/worker.py
index ff7628b7346..702fc7d9eb7 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -1773,19 +1773,23 @@ def gather_dep(self, worker, dep, deps, total_nbytes, cause=None):
                 self.log.append(('request-dep', dep, worker, deps))
                 logger.debug("Request %d keys", len(deps))
 
-                start = time() + self.scheduler_delay
+                start = time()
                 response = yield get_data_from_worker(self.rpc, deps, worker, self.address)
-                stop = time() + self.scheduler_delay
+                stop = time()
 
                 if cause:
-                    self.startstops[cause].append(('transfer', start, stop))
+                    self.startstops[cause].append((
+                        'transfer',
+                        start + self.scheduler_delay,
+                        stop + self.scheduler_delay
+                    ))
 
                 total_bytes = sum(self.nbytes.get(dep, 0) for dep in response)
                 duration = (stop - start) or 0.5
                 self.incoming_transfer_log.append({
-                    'start': start,
-                    'stop': stop,
-                    'middle': (start + stop) / 2.0,
+                    'start': start + self.scheduler_delay,
+                    'stop': stop + self.scheduler_delay,
+                    'middle': (start + stop) / 2.0 + self.scheduler_delay,
                     'duration': duration,
                     'keys': {dep: self.nbytes.get(dep, None) for dep in response},
                     'total': total_bytes,

From 41e1a014603c5b63b22f5c83874f8f3b914a0ae2 Mon Sep 17 00:00:00 2001
From: Marius van Niekerk <marius.v.niekerk@gmail.com>
Date: Wed, 27 Jun 2018 12:01:08 -0400
Subject: [PATCH 0013/1550] Allow `name` to be explicitly passed in
 publish_dataset (#1995)

Added to the function signature for publish_dataset.  We can now make
datasets that don't have string names
---
 distributed/client.py             | 32 ++++++++++++++++++-----
 distributed/publish.py            | 10 +++----
 distributed/tests/test_publish.py | 43 ++++++++++++++++++++++++-------
 3 files changed, 64 insertions(+), 21 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 72c08978ee5..ca8123e1e34 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -1799,19 +1799,34 @@ def cancel(self, futures, asynchronous=None, force=False):
                          force=force)
 
     @gen.coroutine
-    def _publish_dataset(self, **kwargs):
+    def _publish_dataset(self, *args, **kwargs):
         with log_errors():
             coroutines = []
-            for name, data in kwargs.items():
+
+            def add_coro(name, data):
                 keys = [tokey(f.key) for f in futures_of(data)]
                 coroutines.append(self.scheduler.publish_put(keys=keys,
-                                                             name=tokey(name),
+                                                             name=name,
                                                              data=to_serialize(data),
                                                              client=self.id))
 
+            name = kwargs.pop('name', None)
+            if name:
+                if len(args) == 0:
+                    raise ValueError(
+                        "If name is provided, expecting call signature like"
+                        " publish_dataset(df, name='ds')")
+                # in case this is a singleton, collapse it
+                elif len(args) == 1:
+                    args = args[0]
+                add_coro(name, args)
+
+            for name, data in kwargs.items():
+                add_coro(name, data)
+
             yield coroutines
 
-    def publish_dataset(self, **kwargs):
+    def publish_dataset(self, *args, **kwargs):
         """
         Publish named datasets to scheduler
 
@@ -1824,6 +1839,8 @@ def publish_dataset(self, **kwargs):
 
         Parameters
         ----------
+        args : list of objects to publish as name
+        name : optional name of the dataset to publish
         kwargs: dict
             named collections to publish on the scheduler
 
@@ -1835,6 +1852,9 @@ def publish_dataset(self, **kwargs):
         >>> df = c.persist(df) # doctest: +SKIP
         >>> c.publish_dataset(my_dataset=df)  # doctest: +SKIP
 
+        Alternative invocation
+        >>> c.publish_dataset(df, name='my_dataset')
+
         Receiving client:
 
         >>> c.list_datasets()  # doctest: +SKIP
@@ -1852,7 +1872,7 @@ def publish_dataset(self, **kwargs):
         Client.unpublish_dataset
         Client.persist
         """
-        return self.sync(self._publish_dataset, **kwargs)
+        return self.sync(self._publish_dataset, *args, **kwargs)
 
     def unpublish_dataset(self, name, **kwargs):
         """
@@ -1902,7 +1922,7 @@ def get_dataset(self, name, **kwargs):
         Client.publish_dataset
         Client.list_datasets
         """
-        return self.sync(self._get_dataset, tokey(name), **kwargs)
+        return self.sync(self._get_dataset, name, **kwargs)
 
     @gen.coroutine
     def _run_on_scheduler(self, function, *args, **kwargs):
diff --git a/distributed/publish.py b/distributed/publish.py
index a275cfeff08..3260e99b0e4 100644
--- a/distributed/publish.py
+++ b/distributed/publish.py
@@ -1,5 +1,5 @@
 from collections import MutableMapping
-from distributed.utils import log_errors
+from distributed.utils import log_errors, tokey
 
 
 class PublishExtension(object):
@@ -27,18 +27,18 @@ def put(self, stream=None, keys=None, data=None, name=None, client=None):
         with log_errors():
             if name in self.datasets:
                 raise KeyError("Dataset %s already exists" % name)
-            self.scheduler.client_desires_keys(keys, 'published-%s' % name)
+            self.scheduler.client_desires_keys(keys, 'published-%s' % tokey(name))
             self.datasets[name] = {'data': data, 'keys': keys}
             return {'status':  'OK', 'name': name}
 
     def delete(self, stream=None, name=None):
         with log_errors():
             out = self.datasets.pop(name, {'keys': []})
-            self.scheduler.client_releases_keys(out['keys'], 'published-%s' % name)
+            self.scheduler.client_releases_keys(out['keys'], 'published-%s' % tokey(name))
 
     def list(self, *args):
         with log_errors():
-            return list(sorted(self.datasets.keys()))
+            return list(sorted(self.datasets.keys(), key=str))
 
     def get(self, stream, name=None, client=None):
         with log_errors():
@@ -60,7 +60,7 @@ def __getitem__(self, key):
         return self.__client.get_dataset(key)
 
     def __setitem__(self, key, value):
-        self.__client.publish_dataset(**{key: value})
+        self.__client.publish_dataset(value, name=key)
 
     def __delitem__(self, key):
         self.__client.unpublish_dataset(key)
diff --git a/distributed/tests/test_publish.py b/distributed/tests/test_publish.py
index a331b7b957a..a67bfce9887 100644
--- a/distributed/tests/test_publish.py
+++ b/distributed/tests/test_publish.py
@@ -37,6 +37,26 @@ def test_publish_simple(s, a, b):
     yield f.close()
 
 
+@gen_cluster(client=False)
+def test_publish_non_string_key(s, a, b):
+    c = yield Client((s.ip, s.port), asynchronous=True)
+    f = yield Client((s.ip, s.port), asynchronous=True)
+
+    try:
+        for name in [('a', 'b'), 9.0, 8]:
+            data = yield c.scatter(range(3))
+            out = yield c.publish_dataset(data, name=name)
+            assert name in s.extensions['publish'].datasets
+            assert isinstance(s.extensions['publish'].datasets[name]['data'], Serialized)
+
+            datasets = yield c.scheduler.publish_list()
+            assert name in datasets
+
+    finally:
+        c.close()
+        f.close()
+
+
 @gen_cluster(client=False)
 def test_publish_roundtrip(s, a, b):
     c = yield Client((s.ip, s.port), asynchronous=True)
@@ -167,26 +187,29 @@ def test_publish_bag(s, a, b):
 def test_datasets_setitem(loop):
     with cluster() as (s, _):
         with Client(s['address'], loop=loop) as client:
-            key, value = 'key', 'value'
-            client.datasets[key] = value
-            assert client.get_dataset('key') == value
+            for key in ['key', ('key', 'key'), 1]:
+                value = 'value'
+                client.datasets[key] = value
+                assert client.get_dataset(key) == value
 
 
 def test_datasets_getitem(loop):
     with cluster() as (s, _):
         with Client(s['address'], loop=loop) as client:
-            key, value = 'key', 'value'
-            client.publish_dataset(key=value)
-            assert client.datasets[key] == value
+            for key in ['key', ('key', 'key'), 1]:
+                value = 'value'
+                client.publish_dataset(value, name=key)
+                assert client.datasets[key] == value
 
 
 def test_datasets_delitem(loop):
     with cluster() as (s, _):
         with Client(s['address'], loop=loop) as client:
-            key, value = 'key', 'value'
-            client.publish_dataset(key=value)
-            del client.datasets[key]
-            assert key not in client.list_datasets()
+            for key in ['key', ('key', 'key'), 1]:
+                value = 'value'
+                client.publish_dataset(value, name=key)
+                del client.datasets[key]
+                assert key not in client.list_datasets()
 
 
 def test_datasets_keys(loop):

From a352a406ea7ad7cc4a8dab83c096bfbd04491d54 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lo=C3=AFc=20Est=C3=A8ve?= <loic.esteve@ymail.com>
Date: Wed, 27 Jun 2018 20:32:23 +0200
Subject: [PATCH 0014/1550] Install msgpack-python with conda on AppVeyor.
 (#2075)

This is a cleanup of the mspack-python to msgpack renaming on PyPI.
---
 continuous_integration/setup_conda_environment.cmd | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/continuous_integration/setup_conda_environment.cmd b/continuous_integration/setup_conda_environment.cmd
index 4f2674dd65b..f03441c336e 100644
--- a/continuous_integration/setup_conda_environment.cmd
+++ b/continuous_integration/setup_conda_environment.cmd
@@ -30,6 +30,7 @@ call deactivate
     joblib ^
     jupyter_client ^
     mock ^
+    msgpack-python ^
     psutil ^
     pytest ^
     python=%PYTHON% ^
@@ -48,7 +49,6 @@ call activate %CONDA_ENV%
 %PIP_INSTALL% git+https://github.com/dask/zict --upgrade
 
 %PIP_INSTALL% pytest-repeat pytest-timeout pytest-faulthandler sortedcollections
-%PIP_INSTALL% msgpack
 
 @rem Display final environment (for reproducing)
 %CONDA% list

From 1167cd0d7c6887645ce7fbf4d0e13fd3a01ebf7f Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 28 Jun 2018 14:35:13 -0400
Subject: [PATCH 0015/1550] Provide communication context to serialization
 functions (#2054)

* collect dask-specific type-based serialization to bottom of serialize.py

* Generalize has_keyword function and move to utils.py

* Provide context to serialization functions

This enables comms to provide a context of information to serialization
functions *if* they provide a ``context=`` keyword for it.
---
 distributed/comm/tcp.py                      |   4 +-
 distributed/comm/utils.py                    |   5 +-
 distributed/core.py                          |  17 +-
 distributed/node.py                          |   7 +-
 distributed/protocol/__init__.py             |   1 +
 distributed/protocol/core.py                 |   5 +-
 distributed/protocol/serialize.py            | 216 ++++++++++---------
 distributed/protocol/tests/test_serialize.py |  93 +++++++-
 distributed/utils.py                         |  11 +
 docs/source/serialization.rst                |  32 ++-
 10 files changed, 267 insertions(+), 124 deletions(-)

diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py
index 9a8fe7bf087..d1dcab7569c 100644
--- a/distributed/comm/tcp.py
+++ b/distributed/comm/tcp.py
@@ -216,7 +216,9 @@ def write(self, msg, serializers=None, on_error='message'):
 
         frames = yield to_frames(msg,
                                  serializers=serializers,
-                                 on_error=on_error)
+                                 on_error=on_error,
+                                 context={'sender': self._local_addr,
+                                          'recipient': self._peer_addr})
 
         try:
             lengths = ([struct.pack('Q', len(frames))] +
diff --git a/distributed/comm/utils.py b/distributed/comm/utils.py
index 32347e3952b..02677b9faba 100644
--- a/distributed/comm/utils.py
+++ b/distributed/comm/utils.py
@@ -29,7 +29,7 @@ def offload(fn, *args, **kwargs):
 
 
 @gen.coroutine
-def to_frames(msg, serializers=None, on_error='message'):
+def to_frames(msg, serializers=None, on_error='message', context=None):
     """
     Serialize a message into a list of Distributed protocol frames.
     """
@@ -37,7 +37,8 @@ def _to_frames():
         try:
             return list(protocol.dumps(msg,
                                        serializers=serializers,
-                                       on_error=on_error))
+                                       on_error=on_error,
+                                       context=context))
         except Exception as e:
             logger.info("Unserializable Message: %s", msg)
             logger.exception(e)
diff --git a/distributed/core.py b/distributed/core.py
index 7bd6c12960c..337c7a626ef 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -3,7 +3,6 @@
 from collections import defaultdict, deque
 from concurrent.futures import CancelledError
 from functools import partial
-import inspect
 import logging
 import six
 import traceback
@@ -17,14 +16,14 @@
 from tornado.ioloop import IOLoop
 from tornado.locks import Event
 
-from .compatibility import PY3, get_thread_identity
+from .compatibility import get_thread_identity
 from .comm import (connect, listen, CommClosedError,
                    normalize_address,
                    unparse_host_port, get_address_host_port)
 from .metrics import time
 from .system_monitor import SystemMonitor
 from .utils import (get_traceback, truncate_exception, ignoring, shutting_down,
-                    PeriodicCallback, parse_timedelta)
+                    PeriodicCallback, parse_timedelta, has_keyword)
 from . import protocol
 
 
@@ -310,7 +309,7 @@ def handle_comm(self, comm, shutting_down=shutting_down):
                     logger.warning("No handler %s found in %s", op,
                                    type(self).__name__, exc_info=True)
                 else:
-                    if serializers is not None and has_serializers_keyword(handler):
+                    if serializers is not None and has_keyword(handler, 'serializers'):
                         msg['serializers'] = serializers  # add back in
 
                     logger.debug("Calling into handler %s", handler.__name__)
@@ -852,13 +851,3 @@ def clean_exception(exception, traceback, **kwargs):
     elif isinstance(traceback, string_types):
         traceback = None  # happens if the traceback failed serializing
     return type(exception), exception, traceback
-
-
-def has_serializers_keyword(func):
-    if PY3:
-        return 'serializers' in inspect.signature(func).parameters
-    else:
-        # https://stackoverflow.com/questions/50100498/determine-keywords-of-a-tornado-coroutine
-        if gen.is_coroutine_function(func):
-            func = func.__wrapped__
-        return 'serializers' in inspect.getargspec(func).args
diff --git a/distributed/node.py b/distributed/node.py
index e7fe00484b6..8373c07709c 100644
--- a/distributed/node.py
+++ b/distributed/node.py
@@ -33,11 +33,14 @@ class ServerNode(Node, Server):
 
     def __init__(self, handlers=None, stream_handlers=None,
                  connection_limit=512, deserialize=True,
-                 connection_args=None, io_loop=None):
+                 connection_args=None, io_loop=None, serializers=None,
+                 deserializers=None):
         Node.__init__(self, deserialize=deserialize,
                       connection_limit=connection_limit,
                       connection_args=connection_args,
-                      io_loop=io_loop)
+                      io_loop=io_loop,
+                      serializers=serializers,
+                      deserializers=deserializers)
         Server.__init__(self, handlers=handlers,
                         stream_handlers=stream_handlers,
                         connection_limit=connection_limit,
diff --git a/distributed/protocol/__init__.py b/distributed/protocol/__init__.py
index 38c8ce05d95..a6a9afaf324 100644
--- a/distributed/protocol/__init__.py
+++ b/distributed/protocol/__init__.py
@@ -8,6 +8,7 @@
     serialize, deserialize, nested_deserialize, Serialize, Serialized,
     to_serialize, register_serialization, register_serialization_lazy,
     serialize_bytes, deserialize_bytes, serialize_bytelist,
+    register_serialization_family,
 )
 
 from ..utils import ignoring
diff --git a/distributed/protocol/core.py b/distributed/protocol/core.py
index 9209aa06184..4033c9be1a9 100644
--- a/distributed/protocol/core.py
+++ b/distributed/protocol/core.py
@@ -22,7 +22,7 @@
 logger = logging.getLogger(__name__)
 
 
-def dumps(msg, serializers=None, on_error='message'):
+def dumps(msg, serializers=None, on_error='message', context=None):
     """ Transform Python message to bytestream suitable for communication """
     try:
         data = {}
@@ -40,7 +40,8 @@ def dumps(msg, serializers=None, on_error='message'):
 
         data = {key: serialize(value.data,
                                serializers=serializers,
-                               on_error=on_error)
+                               on_error=on_error,
+                               context=context)
                 for key, value in data.items()
                 if type(value) is Serialize}
 
diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index 773b08d7887..c2a1274afe2 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -12,6 +12,7 @@
 
 from . import pickle
 from ..compatibility import PY2
+from ..utils import has_keyword
 from .compression import maybe_compress, decompress
 from .utils import unpack_frames, pack_frames_prelude, frame_split_size
 
@@ -21,12 +22,15 @@
 lazy_registrations = {}
 
 
-def dask_dumps(x):
+def dask_dumps(x, context=None):
     """Serialise object using the class-based registry"""
     typ = typename(type(x))
     if typ in class_serializers:
-        dumps, loads = class_serializers[typ]
-        header, frames = dumps(x)
+        dumps, loads, has_context = class_serializers[typ]
+        if has_context:
+            header, frames = dumps(x, context=context)
+        else:
+            header, frames = dumps(x)
         header['type'] = typ
         header['serializer'] = 'dask'
         return header, frames
@@ -43,7 +47,7 @@ def dask_loads(header, frames):
         _find_lazy_registration(typ)
 
     try:
-        dumps, loads = class_serializers[typ]
+        dumps, loads, _ = class_serializers[typ]
     except KeyError:
         raise TypeError("Serialization for type %s not found" % typ)
     else:
@@ -71,82 +75,20 @@ def serialization_error_loads(header, frames):
     raise TypeError(msg)
 
 
-families = {
- 'dask': (dask_dumps, dask_loads),
- 'pickle': (pickle_dumps, pickle_loads),
- 'msgpack': (msgpack_dumps, msgpack_loads),
- 'error': (None, serialization_error_loads),
-}
+families = {}
 
 
-def register_serialization(cls, serialize, deserialize):
-    """ Register a new class for dask-custom serialization
+def register_serialization_family(name, dumps, loads):
+    families[name] = (dumps, loads, dumps and has_keyword(dumps, 'context'))
 
-    Parameters
-    ----------
-    cls: type
-    serialize: function
-    deserialize: function
 
-    Examples
-    --------
-    >>> class Human(object):
-    ...     def __init__(self, name):
-    ...         self.name = name
-
-    >>> def serialize(human):
-    ...     header = {}
-    ...     frames = [human.name.encode()]
-    ...     return header, frames
+register_serialization_family('dask', dask_dumps, dask_loads)
+register_serialization_family('pickle', pickle_dumps, pickle_loads)
+register_serialization_family('msgpack', msgpack_dumps, msgpack_loads)
+register_serialization_family('error', None, serialization_error_loads)
 
-    >>> def deserialize(header, frames):
-    ...     return Human(frames[0].decode())
-
-    >>> register_serialization(Human, serialize, deserialize)
-    >>> serialize(Human('Alice'))
-    ({}, [b'Alice'])
-
-    See Also
-    --------
-    serialize
-    deserialize
-    """
-    if isinstance(cls, type):
-        name = typename(cls)
-    elif isinstance(cls, str):
-        name = cls
-    class_serializers[name] = (serialize, deserialize)
-
-
-def register_serialization_lazy(toplevel, func):
-    """Register a registration function to be called if *toplevel*
-    module is ever loaded.
-    """
-    lazy_registrations[toplevel] = func
-
-
-def typename(typ):
-    """ Return name of type
 
-    Examples
-    --------
-    >>> from distributed import Scheduler
-    >>> typename(Scheduler)
-    'distributed.scheduler.Scheduler'
-    """
-    return typ.__module__ + '.' + typ.__name__
-
-
-def _find_lazy_registration(typename):
-    toplevel, _, _ = typename.partition('.')
-    if toplevel in lazy_registrations:
-        lazy_registrations.pop(toplevel)()
-        return True
-    else:
-        return False
-
-
-def serialize(x, serializers=None, on_error='message'):
+def serialize(x, serializers=None, on_error='message', context=None):
     r"""
     Convert object to a header and list of bytestrings
 
@@ -191,9 +133,9 @@ def serialize(x, serializers=None, on_error='message'):
     tb = ''
 
     for name in serializers:
-        dumps, loads = families[name]
+        dumps, loads, wants_context = families[name]
         try:
-            header, frames = dumps(x)
+            header, frames = dumps(x, context=context) if wants_context else dumps(x)
             header['serializer'] = name
             return header, frames
         except NotImplementedError:
@@ -232,7 +174,7 @@ def deserialize(header, frames, deserializers=None):
     if deserializers is not None and name not in deserializers:
         raise TypeError("Data serialized with %s but only able to deserialize "
                         "data with %s" % (name, str(list(deserializers))))
-    dumps, loads = families[name]
+    dumps, loads, wants_context = families[name]
     return loads(header, frames)
 
 
@@ -394,29 +336,6 @@ def replace_inner(x):
     return replace_inner(x)
 
 
-@partial(normalize_token.register, Serialized)
-def normalize_Serialized(o):
-    return [o.header] + o.frames  # for dask.base.tokenize
-
-
-# Teach serialize how to handle bytestrings
-def _serialize_bytes(obj):
-    header = {}  # no special metadata
-    frames = [obj]
-    return header, frames
-
-
-def _deserialize_bytes(header, frames):
-    return frames[0]
-
-
-# NOTE: using the same exact serialization means a bytes object may be
-# deserialized as bytearray or vice-versa...  Not sure this is a problem
-# in practice.
-register_serialization(bytes, _serialize_bytes, _deserialize_bytes)
-register_serialization(bytearray, _serialize_bytes, _deserialize_bytes)
-
-
 def serialize_bytelist(x, **kwargs):
     header, frames = serialize(x, **kwargs)
     frames = frame_split_size(frames)
@@ -448,3 +367,100 @@ def deserialize_bytes(b):
         header = {}
     frames = decompress(header, frames)
     return deserialize(header, frames)
+
+
+################################
+# Class specific serialization #
+################################
+
+
+def register_serialization(cls, serialize, deserialize):
+    """ Register a new class for dask-custom serialization
+
+    Parameters
+    ----------
+    cls: type
+    serialize: function
+    deserialize: function
+
+    Examples
+    --------
+    >>> class Human(object):
+    ...     def __init__(self, name):
+    ...         self.name = name
+
+    >>> def serialize(human):
+    ...     header = {}
+    ...     frames = [human.name.encode()]
+    ...     return header, frames
+
+    >>> def deserialize(header, frames):
+    ...     return Human(frames[0].decode())
+
+    >>> register_serialization(Human, serialize, deserialize)
+    >>> serialize(Human('Alice'))
+    ({}, [b'Alice'])
+
+    See Also
+    --------
+    serialize
+    deserialize
+    """
+    if isinstance(cls, type):
+        name = typename(cls)
+    elif isinstance(cls, str):
+        name = cls
+    class_serializers[name] = (serialize,
+                               deserialize,
+                               has_keyword(serialize, 'context'))
+
+
+def register_serialization_lazy(toplevel, func):
+    """Register a registration function to be called if *toplevel*
+    module is ever loaded.
+    """
+    lazy_registrations[toplevel] = func
+
+
+def typename(typ):
+    """ Return name of type
+
+    Examples
+    --------
+    >>> from distributed import Scheduler
+    >>> typename(Scheduler)
+    'distributed.scheduler.Scheduler'
+    """
+    return typ.__module__ + '.' + typ.__name__
+
+
+def _find_lazy_registration(typename):
+    toplevel, _, _ = typename.partition('.')
+    if toplevel in lazy_registrations:
+        lazy_registrations.pop(toplevel)()
+        return True
+    else:
+        return False
+
+
+@partial(normalize_token.register, Serialized)
+def normalize_Serialized(o):
+    return [o.header] + o.frames  # for dask.base.tokenize
+
+
+# Teach serialize how to handle bytestrings
+def _serialize_bytes(obj):
+    header = {}  # no special metadata
+    frames = [obj]
+    return header, frames
+
+
+def _deserialize_bytes(header, frames):
+    return frames[0]
+
+
+# NOTE: using the same exact serialization means a bytes object may be
+# deserialized as bytearray or vice-versa...  Not sure this is a problem
+# in practice.
+register_serialization(bytes, _serialize_bytes, _deserialize_bytes)
+register_serialization(bytearray, _serialize_bytes, _deserialize_bytes)
diff --git a/distributed/protocol/tests/test_serialize.py b/distributed/protocol/tests/test_serialize.py
index a1cedf5f2f7..dc7377385ea 100644
--- a/distributed/protocol/tests/test_serialize.py
+++ b/distributed/protocol/tests/test_serialize.py
@@ -3,14 +3,17 @@
 import copy
 import pickle
 
+import msgpack
 import numpy as np
 import pytest
 from toolz import identity
 
+from distributed import wait
 from distributed.protocol import (register_serialization, serialize,
                                   deserialize, nested_deserialize, Serialize,
                                   Serialized, to_serialize, serialize_bytes,
-                                  deserialize_bytes, serialize_bytelist,)
+                                  deserialize_bytes, serialize_bytelist,
+                                  register_serialization_family)
 from distributed.utils import nbytes
 from distributed.utils_test import inc, gen_test
 from distributed.comm.utils import to_frames, from_frames
@@ -245,3 +248,91 @@ def test_err_on_bad_deserializer():
 
     with pytest.raises(TypeError) as info:
         yield from_frames(frames, deserializers=['msgpack'])
+
+
+class MyObject(object):
+    def __init__(self, **kwargs):
+        self.__dict__.update(kwargs)
+
+
+def my_dumps(obj, context=None):
+    if type(obj).__name__ == 'MyObject':
+        header = {'serializer': 'my-ser'}
+        frames = [msgpack.dumps(obj.__dict__, use_bin_type=True),
+                  msgpack.dumps(context, use_bin_type=True)]
+        return header, frames
+    else:
+        raise NotImplementedError()
+
+
+def my_loads(header, frames):
+    obj = MyObject(**msgpack.loads(frames[0], encoding='utf8'))
+
+    # to provide something to test against, lets just attach the context to
+    # the object itself
+    obj.context = msgpack.loads(frames[1], encoding='utf8')
+    return obj
+
+
+@gen_cluster(client=True,
+             client_kwargs={'serializers': ['my-ser', 'pickle']},
+             worker_kwargs={'serializers': ['my-ser', 'pickle']})
+def test_context_specific_serialization(c, s, a, b):
+    register_serialization_family('my-ser', my_dumps, my_loads)
+
+    try:
+        # Create the object on A, force communication to B
+        x = c.submit(MyObject, x=1, y=2, workers=a.address)
+        y = c.submit(lambda x: x, x, workers=b.address)
+
+        yield wait(y)
+
+        key = y.key
+
+        def check(dask_worker):
+            # Get the context from the object stored on B
+            my_obj = dask_worker.data[key]
+            return my_obj.context
+
+        result = yield c.run(check, workers=[b.address])
+        expected = {'sender': a.address, 'recipient': b.address}
+        assert result[b.address]['sender'] == a.address  # see origin worker
+
+        z = yield y  # bring object to local process
+
+        assert z.x == 1 and z.y == 2
+        assert z.context['sender'] == b.address
+    finally:
+        from distributed.protocol.serialize import families
+        del families['my-ser']
+
+
+@gen_cluster(client=True)
+def test_context_specific_serialization_class(c, s, a, b):
+    register_serialization(MyObject, my_dumps, my_loads)
+
+    try:
+        # Create the object on A, force communication to B
+        x = c.submit(MyObject, x=1, y=2, workers=a.address)
+        y = c.submit(lambda x: x, x, workers=b.address)
+
+        yield wait(y)
+
+        key = y.key
+
+        def check(dask_worker):
+            # Get the context from the object stored on B
+            my_obj = dask_worker.data[key]
+            return my_obj.context
+
+        result = yield c.run(check, workers=[b.address])
+        expected = {'sender': a.address, 'recipient': b.address}
+        assert result[b.address]['sender'] == a.address  # see origin worker
+
+        z = yield y  # bring object to local process
+
+        assert z.x == 1 and z.y == 2
+        assert z.context['sender'] == b.address
+    finally:
+        from distributed.protocol.serialize import class_serializers, typename
+        del class_serializers[typename(MyObject)]
diff --git a/distributed/utils.py b/distributed/utils.py
index c3a3ac9cb62..53495ba60b4 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -5,6 +5,7 @@
 from contextlib import contextmanager
 from datetime import timedelta
 import functools
+import inspect
 import json
 import logging
 import multiprocessing
@@ -1385,3 +1386,13 @@ def reset_logger_locks():
 # Only bother if asyncio has been loaded by Tornado
 if 'asyncio' in sys.modules:
     fix_asyncio_event_loop_policy(sys.modules['asyncio'])
+
+
+def has_keyword(func, keyword):
+    if PY3:
+        return keyword in inspect.signature(func).parameters
+    else:
+        # https://stackoverflow.com/questions/50100498/determine-keywords-of-a-tornado-coroutine
+        if gen.is_coroutine_function(func):
+            func = func.__wrapped__
+        return keyword in inspect.getargspec(func).args
diff --git a/docs/source/serialization.rst b/docs/source/serialization.rst
index 01eb1044dd6..e457681a662 100644
--- a/docs/source/serialization.rst
+++ b/docs/source/serialization.rst
@@ -86,13 +86,41 @@ dictionary with an appropriate name.  Here is the definition of
            frame = frames[0]
        return pickle.loads(frame)
 
-   from distributed.protocol.serialize import families
-   families['pickle'] = (pickle_dumps, pickle_loads)
+   from distributed.protocol.serialize import register_serialization_family
+   register_serialization_family('pickle', pickle_dumps, pickle_loads)
 
 After this the name ``'pickle'`` can be used in the ``serializers=`` and
 ``deserializers=`` keywords in ``Client`` and other parts of Dask.
 
 
+Communication Context
++++++++++++++++++++++
+
+.. note:: This is an experimental feature and may change without notice
+
+Dask :doc:`Comms <communications>` can provide additional context to
+serialization family functions if they provide a ``context=`` keyword.
+This allows serialization to behave differently according to how it is being
+used.
+
+.. code-block:: python
+
+   def my_dumps(x, context=None):
+       if context and 'recipient' in context:
+           # check if we're sending to the same host or not
+
+The context depends on the kind of communication.  For example when sending
+over TCP, the address of the sender (us) and the recipient are available in a
+dictionary.
+
+.. code-block:: python
+
+   >>> context
+   {'sender': 'tcp://127.0.0.1:1234', 'recipient': 'tcp://127.0.0.1:5678'}
+
+Other comms may provide other information.
+
+
 Dask Serialization Family
 -------------------------
 

From 94f076c92bd3e2ab9340a0876e10eab5b278bbd6 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sun, 1 Jul 2018 20:28:19 -0400
Subject: [PATCH 0016/1550] Use default pygments styling

---
 docs/source/conf.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/source/conf.py b/docs/source/conf.py
index 3bb5a13ae7c..8ba681f0a73 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -104,7 +104,7 @@
 #show_authors = False
 
 # The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
+pygments_style = 'default'
 
 # A list of ignored prefixes for module index sorting.
 #modindex_common_prefix = []

From f7c8f339412764367bcd42e01347a80fb25b9822 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lo=C3=AFc=20Est=C3=A8ve?= <loic.esteve@ymail.com>
Date: Mon, 2 Jul 2018 17:29:32 +0200
Subject: [PATCH 0017/1550] Fix typo in docstring (#2087)

---
 distributed/client.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/client.py b/distributed/client.py
index ca8123e1e34..b066d5cc05d 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -1165,7 +1165,7 @@ def submit(self, func, *args, **kwargs):
         key: str
             Unique identifier for the task.  Defaults to function-name and hash
         allow_other_workers: bool (defaults to False)
-            Used with `workers`. Inidicates whether or not the computations
+            Used with `workers`. Indicates whether or not the computations
             may be performed on workers that are not in the `workers` set(s).
         retries: int (default to 0)
             Number of allowed automatic retries if the task fails

From fc1312facff8df81f766562fb069ea9865ebe10d Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 2 Jul 2018 17:47:01 -0400
Subject: [PATCH 0018/1550] use https to get stylesheet in docs

---
 docs/source/conf.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/source/conf.py b/docs/source/conf.py
index 8ba681f0a73..3869ec0367b 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -385,4 +385,4 @@
     }
 
 def setup(app):
-    app.add_stylesheet("http://dask.pydata.org/en/latest/_static/style.css")
+    app.add_stylesheet("https://dask.pydata.org/en/latest/_static/style.css")

From e9d527ab7cf840e9ff40d2f9c3833f3999a198e9 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 4 Jul 2018 07:54:15 -0400
Subject: [PATCH 0019/1550] Handle exceptions on deserialized comm with text
 error (#2093)

---
 distributed/core.py            | 14 ++++++++++----
 distributed/tests/test_core.py | 16 +++++++++++++++-
 distributed/worker.py          |  1 -
 3 files changed, 25 insertions(+), 6 deletions(-)

diff --git a/distributed/core.py b/distributed/core.py
index 337c7a626ef..05b901cf818 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -408,8 +408,8 @@ def pingpong(comm):
 
 
 @gen.coroutine
-def send_recv(comm, reply=True, deserialize=True, serializers=None,
-              deserializers=None, **kwargs):
+def send_recv(comm, reply=True, serializers=None, deserializers=None,
+              **kwargs):
     """ Send and recv with a Comm.
 
     Keyword arguments turn into the message
@@ -442,7 +442,10 @@ def send_recv(comm, reply=True, deserialize=True, serializers=None,
             comm.abort()
 
     if isinstance(response, dict) and response.get('status') == 'uncaught-error':
-        six.reraise(*clean_exception(**response))
+        if comm.deserialize:
+            six.reraise(*clean_exception(**response))
+        else:
+            raise Exception(response['text'])
     raise gen.Return(response)
 
 
@@ -834,7 +837,10 @@ def error_message(e, status='error'):
     else:
         tb_result = protocol.to_serialize(tb)
 
-    return {'status': status, 'exception': e4, 'traceback': tb_result}
+    return {'status': status,
+            'exception': e4,
+            'traceback': tb_result,
+            'text': str(e2)}
 
 
 def clean_exception(exception, traceback, **kwargs):
diff --git a/distributed/tests/test_core.py b/distributed/tests/test_core.py
index bf7fa2aa42c..ce67817e6a6 100644
--- a/distributed/tests/test_core.py
+++ b/distributed/tests/test_core.py
@@ -23,7 +23,7 @@
     assert_can_connect_from_everywhere_4,
     assert_can_connect_from_everywhere_4_6, assert_can_connect_from_everywhere_6,
     assert_can_connect_locally_4, assert_can_connect_locally_6,
-    tls_security, captured_logger, inc)
+    tls_security, captured_logger, inc, throws)
 from distributed.utils_test import loop  # noqa F401
 
 
@@ -658,3 +658,17 @@ def f():
 @gen_cluster()
 def test_thread_id(s, a, b):
     assert s.thread_id == a.thread_id == b.thread_id == get_thread_identity()
+
+
+@gen_test()
+def test_deserialize_error():
+    server = Server({'throws': throws})
+    server.listen(0)
+
+    comm = yield connect(server.address, deserialize=False)
+    with pytest.raises(Exception) as info:
+        yield send_recv(comm, op='throws')
+
+    assert type(info.value) == Exception
+    for c in str(info.value):
+        assert c.isalpha() or c in "(',!)"  # no crazy bytestrings
diff --git a/distributed/worker.py b/distributed/worker.py
index 702fc7d9eb7..ab4f1d81570 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -2740,7 +2740,6 @@ def get_data_from_worker(rpc, keys, worker, who=None):
         response = yield send_recv(comm,
                                    serializers=rpc.serializers,
                                    deserializers=rpc.deserializers,
-                                   deserialize=rpc.deserialize,
                                    op='get_data', keys=keys, who=who)
         yield comm.write('OK')
     finally:

From b71e8250b475216a58154d449d4305cc2f1f7f6a Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 4 Jul 2018 07:55:01 -0400
Subject: [PATCH 0020/1550] Prefer gathering data from same host (#2090)

* Prefer gathering data from same host

* add who= to all get_data calls

* relax test_retire_many_workers
---
 distributed/tests/test_client.py |  4 +++-
 distributed/tests/test_worker.py | 12 ++++++++++++
 distributed/utils_comm.py        |  5 +++--
 distributed/worker.py            | 12 +++++++++---
 4 files changed, 27 insertions(+), 6 deletions(-)

diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index e015db9f99c..75558352111 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -4082,8 +4082,10 @@ def test_retire_many_workers(c, s, *workers):
     assert results == list(range(100))
 
     assert len(s.has_what) == len(s.ncores) == 3
+    assert all(future.done() for future in futures)
+    assert all(s.tasks[future.key].state == 'memory' for future in futures)
     for w, keys in s.has_what.items():
-        assert 20 < len(keys) < 50
+        assert 15 < len(keys) < 50
 
 
 @gen_cluster(client=True,
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index e8878029e8e..185278e7e94 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -1180,3 +1180,15 @@ def test_wait_for_outgoing(c, s, a, b):
     ratio = aa / bb
 
     assert 1 / 3 < ratio < 3
+
+
+@gen_cluster(ncores=[('127.0.0.1', 1), ('127.0.0.1', 1), ('127.0.0.2', 1)],
+             client=True)
+def test_prefer_gather_from_local_address(c, s, w1, w2, w3):
+    x = yield c.scatter(123, workers=[w1.address, w3.address], broadcast=True)
+
+    y = c.submit(inc, x, workers=[w2.address])
+    yield wait(y)
+
+    assert any(d['who'] == w2.address for d in w1.outgoing_transfer_log)
+    assert not any(d['who'] == w2.address for d in w3.outgoing_transfer_log)
diff --git a/distributed/utils_comm.py b/distributed/utils_comm.py
index 8e4d2ac1300..43dbe49fb0c 100644
--- a/distributed/utils_comm.py
+++ b/distributed/utils_comm.py
@@ -14,7 +14,7 @@
 
 
 @gen.coroutine
-def gather_from_workers(who_has, rpc, close=True, serializers=None):
+def gather_from_workers(who_has, rpc, close=True, serializers=None, who=None):
     """ Gather data directly from peers
 
     Parameters
@@ -56,7 +56,8 @@ def gather_from_workers(who_has, rpc, close=True, serializers=None):
 
         rpcs = {addr: rpc(addr) for addr in d}
         try:
-            coroutines = {address: get_data_from_worker(rpc, keys, address)
+            coroutines = {address: get_data_from_worker(rpc, keys, address,
+                                                        who=who)
                           for address, keys in d.items()}
             response = {}
             for worker, c in coroutines.items():
diff --git a/distributed/worker.py b/distributed/worker.py
index ab4f1d81570..df8cd6ad8a9 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -704,7 +704,7 @@ def gather(self, comm=None, who_has=None):
                    for k, v in who_has.items()
                    if k not in self.data}
         result, missing_keys, missing_workers = yield gather_from_workers(
-            who_has, rpc=self.rpc)
+            who_has, rpc=self.rpc, who=self.address)
         if missing_keys:
             logger.warning("Could not find data: %s on workers: %s (who_has: %s)",
                            missing_keys, missing_workers, who_has)
@@ -1662,7 +1662,12 @@ def ensure_communicating(self):
                     if not workers:
                         in_flight = True
                         continue
-                    worker = random.choice(list(workers))
+                    host = get_address_host(self.address)
+                    local = [w for w in workers if get_address_host(w) == host]
+                    if local:
+                        worker = random.choice(local)
+                    else:
+                        worker = random.choice(list(workers))
                     to_gather, total_nbytes = self.select_keys_for_gather(worker, dep)
                     self.comm_nbytes += total_nbytes
                     self.in_flight_workers[worker] = to_gather
@@ -1774,7 +1779,8 @@ def gather_dep(self, worker, dep, deps, total_nbytes, cause=None):
                 logger.debug("Request %d keys", len(deps))
 
                 start = time()
-                response = yield get_data_from_worker(self.rpc, deps, worker, self.address)
+                response = yield get_data_from_worker(self.rpc, deps, worker,
+                                                      who=self.address)
                 stop = time()
 
                 if cause:

From 89992d0403287b6b29398474af166ca43b0717c9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aur=C3=A9lien=20Ponte?= <aurelien.junk@gmail.com>
Date: Wed, 4 Jul 2018 14:25:34 +0200
Subject: [PATCH 0021/1550] Adjust worker doc after change in config file
 location and treatment (#2094)

---
 docs/source/worker.rst | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/docs/source/worker.rst b/docs/source/worker.rst
index ddaf5c334c4..deaa0243913 100644
--- a/docs/source/worker.rst
+++ b/docs/source/worker.rst
@@ -156,16 +156,19 @@ Workers use a few different policies to keep memory use beneath this limit:
 3.  At 80% of memory load, stop accepting new work on local thread pool
 4.  At 95% of memory load, terminate and restart the worker
 
-These values can be configured by modifying the ``~/.dask/config.yaml`` file
+These values can be configured by modifying the ``~/.config/dask/distributed.yaml`` file
 
 .. code-block:: yaml
 
-   # Fractions of worker memory at which we take action to avoid memory blowup
-   # Set any of the lower three values to False to turn off the behavior entirely
-   worker-memory-target: 0.60  # target fraction to stay below
-   worker-memory-spill: 0.70  # fraction at which we spill to disk
-   worker-memory-pause: 0.80  # fraction at which we pause worker threads
-   worker-memory-terminate: 0.95  # fraction at which we terminate the worker
+   distributed:
+     worker:
+       # Fractions of worker memory at which we take action to avoid memory blowup
+       # Set any of the lower three values to False to turn off the behavior entirely
+       memory:
+         target: 0.60  # target fraction to stay below
+         spill: 0.70  # fraction at which we spill to disk
+         pause: 0.80  # fraction at which we pause worker threads
+         terminate: 0.95  # fraction at which we terminate the worker
 
 
 Spill data to Disk

From 5bbbb3c4370ed05ba226925f47b023960966987e Mon Sep 17 00:00:00 2001
From: Bartosz Marcinkowski <bm371613@gmail.com>
Date: Thu, 5 Jul 2018 16:48:06 +0200
Subject: [PATCH 0022/1550] removed hardcoded value of memory terminate
 fraction from a log message (#2096)

---
 distributed/nanny.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/distributed/nanny.py b/distributed/nanny.py
index afe02431438..6b0c0ec9620 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -249,7 +249,7 @@ def _():
             raise gen.Return('OK')
 
     def memory_monitor(self):
-        """ Track worker's memory.  Restart if it goes above 95% """
+        """ Track worker's memory.  Restart if it goes above terminate fraction """
         if self.status != 'running':
             return
         process = self.process.process
@@ -262,7 +262,8 @@ def memory_monitor(self):
         memory = proc.memory_info().rss
         frac = memory / self.memory_limit
         if self.memory_terminate_fraction and frac > self.memory_terminate_fraction:
-            logger.warning("Worker exceeded 95% memory budget.  Restarting")
+            logger.warning("Worker exceeded %d%% memory budget. Restarting",
+                           100 * self.memory_terminate_fraction)
             process.terminate()
 
     def is_alive(self):

From 1c1c72e79ad47c3209c87ba1e66806d8c1fbb6c2 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Thu, 5 Jul 2018 12:38:18 -0400
Subject: [PATCH 0023/1550] Update example for stopping a worker (#2088)

The `remove_worker` syntax is outdated. Update it to use `stop_worker`,
which is the current syntax.
---
 distributed/deploy/local.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index b88966a2a02..be063366b6d 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -70,7 +70,7 @@ class LocalCluster(Cluster):
 
     Shut down the extra worker
 
-    >>> c.remove_worker(w)  # doctest: +SKIP
+    >>> c.stop_worker(w)  # doctest: +SKIP
 
     Pass extra keyword arguments to Bokeh
 

From 95d55ab2ca2b050b45c08737d902386775bc0940 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sat, 7 Jul 2018 14:08:43 -0400
Subject: [PATCH 0024/1550] Don't forget released keys (#2098)

Previously we would allow forgetting keys if a dependency of a forgotten
key had no active waiting tasks.  Now we properly check dependents, not
active waiters.
---
 distributed/scheduler.py            |  9 +++++----
 distributed/tests/test_scheduler.py | 15 +++++++++++++++
 2 files changed, 20 insertions(+), 4 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 33f8263b93d..bbcf0dad08d 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -3456,7 +3456,8 @@ def transition_processing_released(self, key):
                 recommendations[key] = 'forgotten'
             elif ts.waiters or ts.who_wants:
                 recommendations[key] = 'waiting'
-            else:
+
+            if recommendations.get(key) != 'waiting':
                 for dts in ts.dependencies:
                     if dts.state != 'released':
                         s = dts.waiters
@@ -3590,7 +3591,7 @@ def _propagate_forgotten(self, ts, recommendations):
             dts.dependents.remove(ts)
             s = dts.waiters
             s.discard(ts)
-            if not s and not dts.who_wants:
+            if not dts.dependents and not dts.who_wants:
                 # Task not needed anymore
                 assert dts is not ts
                 recommendations[dts.key] = 'forgotten'
@@ -3621,7 +3622,7 @@ def transition_memory_forgotten(self, key):
                 elif ts.has_lost_dependencies:
                     # It's ok to forget a task with forgotten dependencies
                     pass
-                elif not ts.who_wants and not ts.waiters:
+                elif not ts.who_wants and not ts.waiters and not ts.dependents:
                     # It's ok to forget a task that nobody needs
                     pass
                 else:
@@ -3656,7 +3657,7 @@ def transition_released_forgotten(self, key):
                 elif ts.has_lost_dependencies:
                     # It's ok to forget a task with forgotten dependencies
                     pass
-                elif not ts.who_wants and not ts.waiters:
+                elif not ts.who_wants and not ts.waiters and not ts.dependents:
                     # It's ok to forget a task that nobody needs
                     pass
                 else:
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 4b0cfca25ae..4f6bea05e1c 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -1300,6 +1300,21 @@ def test_dont_recompute_if_persisted_4(c, s, a, b):
     assert len(new) > len(old)
 
 
+@gen_cluster(client=True)
+def test_dont_forget_released_keys(c, s, a, b):
+    x = c.submit(inc, 1, key='x')
+    y = c.submit(inc, x, key='y')
+    z = c.submit(dec, x, key='z')
+    del x
+    yield wait([y, z])
+    del z
+
+    while 'z' in s.tasks:
+        yield gen.sleep(0.01)
+
+    assert 'x' in s.tasks
+
+
 @gen_cluster(client=True)
 def test_dont_recompute_if_erred(c, s, a, b):
     x = delayed(inc)(1, dask_key_name='x')

From 7a9fa83266cb05382094f45f204d12a69e904144 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sun, 8 Jul 2018 12:17:55 -0400
Subject: [PATCH 0025/1550] Allow worker to refuse data requests with busy
 signal (#2092)

This allows workers to say "I'm too busy right now" when presented with
a request for data from another worker.  That worker then waits a bit,
queries the scheduler to see if anyone else has that data, and then
tries again.  The wait time is an exponential backoff.

Pragmatically this means that when single pieces of data are in high
demand that the cluster will informally do a tree scattering.  Some workers
will get the data directly while others wait on the busy signal.  Then other
workers will get from them, etc..  We used to ask users to do this explicitly
with the following:

    client.replicate(future)
    or
    client.scatter(data, broadcast=True)

And now the replicate/broadcast step is no longer strictly necessary. (though
some scattering of local data still is).

Machines on the same host are given some preference, and so should be able to
sneak in more easily.
---
 distributed/bokeh/worker.py      |   2 +-
 distributed/distributed.yaml     |   3 +
 distributed/tests/test_worker.py |  21 +++++-
 distributed/utils_comm.py        |   5 +-
 distributed/worker.py            | 109 ++++++++++++++++++++++---------
 5 files changed, 104 insertions(+), 36 deletions(-)

diff --git a/distributed/bokeh/worker.py b/distributed/bokeh/worker.py
index fcc4ae91995..7e577979a30 100644
--- a/distributed/bokeh/worker.py
+++ b/distributed/bokeh/worker.py
@@ -159,7 +159,7 @@ def __init__(self, worker, **kwargs):
 
         fig = figure(title="Communication History",
                      x_axis_type='datetime',
-                     y_range=[-0.1, worker.total_connections + 0.5],
+                     y_range=[-0.1, worker.total_out_connections + 0.5],
                      height=150, tools='', x_range=x_range, **kwargs)
         fig.line(source=self.source, x='x', y='in', color='red')
         fig.line(source=self.source, x='x', y='out', color='blue')
diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index 35766471e00..31bd73e9663 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -19,6 +19,9 @@ distributed:
   worker:
     multiprocessing-method: forkserver
     use-file-locking: True
+    connections:            # Maximum concurrent connections for data
+      outgoing: 50          # This helps to control network saturation
+      incoming: 10
 
     profile:
       interval: 10ms        # Time between statistical profiling queries
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 185278e7e94..1210d5213d0 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -564,7 +564,7 @@ def test_clean_nbytes(c, s, a, b):
 
 @gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 20)
 def test_gather_many_small(c, s, a, *workers):
-    a.total_connections = 2
+    a.total_out_connections = 2
     futures = yield c._scatter(list(range(100)))
 
     assert all(w.data for w in workers)
@@ -1192,3 +1192,22 @@ def test_prefer_gather_from_local_address(c, s, w1, w2, w3):
 
     assert any(d['who'] == w2.address for d in w1.outgoing_transfer_log)
     assert not any(d['who'] == w2.address for d in w3.outgoing_transfer_log)
+
+
+@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 20, timeout=30,
+             config={'distributed.worker.connections.incoming': 1})
+def test_avoid_oversubscription(c, s, *workers):
+    np = pytest.importorskip('numpy')
+    x = c.submit(np.random.random, 1000000, workers=[workers[0].address])
+    yield wait(x)
+
+    futures = [c.submit(len, x, pure=False, workers=[w.address])
+               for w in workers[1:]]
+
+    yield wait(futures)
+
+    # Original worker not responsible for all transfers
+    assert len(workers[0].outgoing_transfer_log) < len(workers) - 2
+
+    # Some other workers did some work
+    assert len([w for w in workers if len(w.outgoing_transfer_log) > 0]) >= 3
diff --git a/distributed/utils_comm.py b/distributed/utils_comm.py
index 43dbe49fb0c..46724973996 100644
--- a/distributed/utils_comm.py
+++ b/distributed/utils_comm.py
@@ -57,7 +57,8 @@ def gather_from_workers(who_has, rpc, close=True, serializers=None, who=None):
         rpcs = {addr: rpc(addr) for addr in d}
         try:
             coroutines = {address: get_data_from_worker(rpc, keys, address,
-                                                        who=who)
+                                                        who=who,
+                                                        max_connections=False)
                           for address, keys in d.items()}
             response = {}
             for worker, c in coroutines.items():
@@ -66,7 +67,7 @@ def gather_from_workers(who_has, rpc, close=True, serializers=None, who=None):
                 except EnvironmentError:
                     missing_workers.add(worker)
                 else:
-                    response.update(r)
+                    response.update(r['data'])
         finally:
             for r in rpcs.values():
                 r.close_rpc()
diff --git a/distributed/worker.py b/distributed/worker.py
index df8cd6ad8a9..5d4bf44e3a0 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -601,11 +601,24 @@ def delete_data(self, comm=None, keys=None, report=True):
         raise Return('OK')
 
     @gen.coroutine
-    def get_data(self, comm, keys=None, who=None, serializers=None):
+    def get_data(self, comm, keys=None, who=None, serializers=None,
+                 max_connections=None):
         start = time()
 
+        if max_connections is None:
+            max_connections = self.total_in_connections
+
+        # Allow same-host connections more liberally
+        if max_connections and comm and get_address_host(comm.peer_address) == get_address_host(self.address):
+            max_connections = max_connections * 2
+
+        if max_connections is not False and self.outgoing_current_count > max_connections:
+            raise gen.Return({'status': 'busy'})
+
+        self.outgoing_current_count += 1
         data = {k: self.data[k] for k in keys if k in self.data}
-        msg = {k: to_serialize(v) for k, v in data.items()}
+        msg = {'status': 'OK',
+               'data': {k: to_serialize(v) for k, v in data.items()}}
         nbytes = {k: self.nbytes.get(k) for k in data}
         stop = time()
         if self.digests is not None:
@@ -620,6 +633,8 @@ def get_data(self, comm, keys=None, who=None, serializers=None):
                              self.address, who, exc_info=True)
             comm.abort()
             raise
+        finally:
+            self.outgoing_current_count -= 1
         stop = time()
         if self.digests is not None:
             self.digests['get-data-send-duration'].add(stop - start)
@@ -1002,8 +1017,10 @@ class Worker(WorkerBase):
     * **services:** ``{str: Server}``:
         Auxiliary web servers running on this worker
     * **service_ports:** ``{str: port}``:
-    * **total_connections**: ``int``
-        The maximum number of concurrent connections we want to see
+    * **total_out_connections**: ``int``
+        The maximum number of concurrent outgoing requests for data
+    * **total_in_connections**: ``int``
+        The maximum number of concurrent incoming requests for data
     * **total_comm_nbytes**: ``int``
     * **batched_stream**: ``BatchedSend``
         A batched stream along which we communicate to the scheduler
@@ -1146,7 +1163,8 @@ def __init__(self, *args, **kwargs):
 
         self.in_flight_tasks = dict()
         self.in_flight_workers = dict()
-        self.total_connections = 50
+        self.total_out_connections = dask.config.get('distributed.worker.connections.outgoing')
+        self.total_in_connections = dask.config.get('distributed.worker.connections.incoming')
         self.total_comm_nbytes = 10e6
         self.comm_nbytes = 0
         self.suspicious_deps = defaultdict(lambda: 0)
@@ -1211,6 +1229,8 @@ def __init__(self, *args, **kwargs):
         self.incoming_count = 0
         self.outgoing_transfer_log = deque(maxlen=(100000))
         self.outgoing_count = 0
+        self.outgoing_current_count = 0
+        self.repetitively_busy = 0
         self._client = None
 
         profile_cycle_interval = kwargs.pop('profile_cycle_interval',
@@ -1381,20 +1401,21 @@ def transition_dep_waiting_flight(self, dep, worker=None):
                 pdb.set_trace()
             raise
 
-    def transition_dep_flight_waiting(self, dep, worker=None):
+    def transition_dep_flight_waiting(self, dep, worker=None, remove=True):
         try:
             if self.validate:
                 assert dep in self.in_flight_tasks
 
             del self.in_flight_tasks[dep]
-            try:
-                self.who_has[dep].remove(worker)
-            except KeyError:
-                pass
-            try:
-                self.has_what[worker].remove(dep)
-            except KeyError:
-                pass
+            if remove:
+                try:
+                    self.who_has[dep].remove(worker)
+                except KeyError:
+                    pass
+                try:
+                    self.has_what[worker].remove(dep)
+                except KeyError:
+                    pass
 
             if not self.who_has.get(dep):
                 if dep not in self._missing_dep_flight:
@@ -1402,7 +1423,10 @@ def transition_dep_flight_waiting(self, dep, worker=None):
                     self.loop.add_callback(self.handle_missing_dep, dep)
             for key in self.dependents.get(dep, ()):
                 if self.task_state[key] == 'waiting':
-                    self.data_needed.appendleft(key)
+                    if remove:  # try a new worker immediately
+                        self.data_needed.appendleft(key)
+                    else:  # worker was probably busy, wait a while
+                        self.data_needed.append(key)
 
             if not self.dependents[dep]:
                 self.release_dep(dep)
@@ -1608,12 +1632,12 @@ def maybe_transition_long_running(self, key, compute_duration=None):
     def ensure_communicating(self):
         changed = True
         try:
-            while changed and self.data_needed and len(self.in_flight_workers) < self.total_connections:
+            while changed and self.data_needed and len(self.in_flight_workers) < self.total_out_connections:
                 changed = False
                 logger.debug("Ensure communicating.  Pending: %d.  Connections: %d/%d",
                              len(self.data_needed),
                              len(self.in_flight_workers),
-                             self.total_connections)
+                             self.total_out_connections)
 
                 key = self.data_needed[0]
 
@@ -1650,7 +1674,7 @@ def ensure_communicating(self):
 
                 in_flight = False
 
-                while deps and (len(self.in_flight_workers) < self.total_connections
+                while deps and (len(self.in_flight_workers) < self.total_out_connections
                                 or self.comm_nbytes < self.total_comm_nbytes):
                     dep = deps.pop()
                     if self.dep_state[dep] != 'waiting':
@@ -1783,6 +1807,12 @@ def gather_dep(self, worker, dep, deps, total_nbytes, cause=None):
                                                       who=self.address)
                 stop = time()
 
+                if response['status'] == 'busy':
+                    self.log.append(('busy-gather', worker, deps))
+                    for dep in deps:
+                        self.transition_dep(dep, 'waiting')
+                    return
+
                 if cause:
                     self.startstops[cause].append((
                         'transfer',
@@ -1790,14 +1820,14 @@ def gather_dep(self, worker, dep, deps, total_nbytes, cause=None):
                         stop + self.scheduler_delay
                     ))
 
-                total_bytes = sum(self.nbytes.get(dep, 0) for dep in response)
+                total_bytes = sum(self.nbytes.get(dep, 0) for dep in response['data'])
                 duration = (stop - start) or 0.5
                 self.incoming_transfer_log.append({
                     'start': start + self.scheduler_delay,
                     'stop': stop + self.scheduler_delay,
                     'middle': (start + stop) / 2.0 + self.scheduler_delay,
                     'duration': duration,
-                    'keys': {dep: self.nbytes.get(dep, None) for dep in response},
+                    'keys': {dep: self.nbytes.get(dep, None) for dep in response['data']},
                     'total': total_bytes,
                     'bandwidth': total_bytes / duration,
                     'who': worker
@@ -1805,14 +1835,14 @@ def gather_dep(self, worker, dep, deps, total_nbytes, cause=None):
                 if self.digests is not None:
                     self.digests['transfer-bandwidth'].add(total_bytes / duration)
                     self.digests['transfer-duration'].add(duration)
-                self.counters['transfer-count'].add(len(response))
+                self.counters['transfer-count'].add(len(response['data']))
                 self.incoming_count += 1
 
-                self.log.append(('receive-dep', worker, list(response)))
+                self.log.append(('receive-dep', worker, list(response['data'])))
 
-                if response:
+                if response['data']:
                     self.batched_stream.send({'op': 'add-keys',
-                                              'keys': list(response)})
+                                              'keys': list(response['data'])})
             except EnvironmentError as e:
                 logger.exception("Worker stream died during communication: %s",
                                  worker)
@@ -1830,14 +1860,16 @@ def gather_dep(self, worker, dep, deps, total_nbytes, cause=None):
                 raise
             finally:
                 self.comm_nbytes -= total_nbytes
+                busy = response['status'] == 'busy'
 
                 for d in self.in_flight_workers.pop(worker):
-                    if d in response:
-                        self.transition_dep(d, 'memory', value=response[d])
+                    if not busy and d in response['data']:
+                        self.transition_dep(d, 'memory', value=response['data'][d])
                     elif self.dep_state.get(d) != 'memory':
-                        self.transition_dep(d, 'waiting', worker=worker)
+                        self.transition_dep(d, 'waiting', worker=worker,
+                                            remove=not busy)
 
-                    if d not in response and d in self.dependents:
+                    if not busy and d not in response['data'] and d in self.dependents:
                         self.log.append(('missing-dep', d))
                         self.batched_stream.send({'op': 'missing-data',
                                                   'errant_worker': worker,
@@ -1847,7 +1879,18 @@ def gather_dep(self, worker, dep, deps, total_nbytes, cause=None):
                     self.validate_state()
 
                 self.ensure_computing()
-                self.ensure_communicating()
+
+                if not busy:
+                    self.repetitively_busy = 0
+                    self.ensure_communicating()
+                else:
+                    # Exponential backoff to avoid hammering scheduler/worker
+                    self.repetitively_busy += 1
+                    yield gen.sleep(0.100 * 1.5 ** self.repetitively_busy)
+
+                    # See if anyone new has the data
+                    yield self.query_who_has(dep)
+                    self.ensure_communicating()
 
     def bad_dep(self, dep):
         exc = ValueError("Could not find dependent %s.  Check worker logs" % str(dep))
@@ -2729,7 +2772,7 @@ def parse_memory_limit(memory_limit, ncores):
 
 
 @gen.coroutine
-def get_data_from_worker(rpc, keys, worker, who=None):
+def get_data_from_worker(rpc, keys, worker, who=None, max_connections=None):
     """ Get keys from worker
 
     The worker has a two step handshake to acknowledge when data has been fully
@@ -2746,8 +2789,10 @@ def get_data_from_worker(rpc, keys, worker, who=None):
         response = yield send_recv(comm,
                                    serializers=rpc.serializers,
                                    deserializers=rpc.deserializers,
-                                   op='get_data', keys=keys, who=who)
-        yield comm.write('OK')
+                                   op='get_data', keys=keys, who=who,
+                                   max_connections=max_connections)
+        if response['status'] == 'OK':
+            yield comm.write('OK')
     finally:
         rpc.reuse(worker, comm)
 

From 86260008d5799c89466f633c8e403608152b00b2 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 9 Jul 2018 14:37:31 -0400
Subject: [PATCH 0026/1550] Retire workers from scale (#2104)

---
 distributed/deploy/cluster.py          |  6 ++++-
 distributed/deploy/tests/test_local.py | 32 ++++++++++++++++++++++++++
 2 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
index 319b969d513..4265a151945 100644
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -93,7 +93,11 @@ def scale(self, n):
             if n >= len(self.scheduler.workers):
                 self.scheduler.loop.add_callback(self.scale_up, n)
             else:
-                to_close = self.scheduler.workers_to_close(n=len(self.scheduler.workers) - n)
+                to_close = self.scheduler.retire_workers(
+                    remove=False,
+                    close_workers=True,
+                    n=len(self.scheduler.workers) - n
+                )
                 logger.debug("Closing workers: %s", to_close)
                 self.scheduler.loop.add_callback(self.scale_down, to_close)
 
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index 988711c41fb..7f2f5874d41 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -467,5 +467,37 @@ def test_local_tls(loop):
              )
 
 
+@gen_test()
+def test_scale_retires_workers():
+    class MyCluster(LocalCluster):
+        def scale_down(self, *args, **kwargs):
+            pass
+
+    loop = IOLoop.current()
+    cluster = yield MyCluster(0, scheduler_port=0, processes=False,
+                              silence_logs=False, diagnostics_port=None,
+                              loop=loop, asynchronous=True)
+    c = yield Client(cluster, loop=loop, asynchronous=True)
+
+    assert not cluster.workers
+
+    yield cluster.scale(2)
+
+    start = time()
+    while len(cluster.scheduler.workers) != 2:
+        yield gen.sleep(0.01)
+        assert time() < start + 3
+
+    yield cluster.scale(1)
+
+    start = time()
+    while len(cluster.scheduler.workers) != 1:
+        yield gen.sleep(0.01)
+        assert time() < start + 3
+
+    yield c._close()
+    yield cluster._close()
+
+
 if sys.version_info >= (3, 5):
     from distributed.deploy.tests.py3_test_deploy import *  # noqa F401

From fd8ca5ebc3a781d5f6625d3daa4d09ce806938a3 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 9 Jul 2018 18:08:06 -0400
Subject: [PATCH 0027/1550] Be robust to empty response in gather_dep (#2105)

---
 distributed/worker.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/worker.py b/distributed/worker.py
index 5d4bf44e3a0..91100373e46 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -1860,7 +1860,7 @@ def gather_dep(self, worker, dep, deps, total_nbytes, cause=None):
                 raise
             finally:
                 self.comm_nbytes -= total_nbytes
-                busy = response['status'] == 'busy'
+                busy = response.get('status', '') == 'busy'
 
                 for d in self.in_flight_workers.pop(worker):
                     if not busy and d in response['data']:

From 696030682305b0ef5bb31583b547ffb434c04b04 Mon Sep 17 00:00:00 2001
From: Phil Tooley <32297355+ptooley@users.noreply.github.com>
Date: Thu, 12 Jul 2018 22:08:13 +0100
Subject: [PATCH 0028/1550] insert newline by default after TextProgressBar
 (#1976)

---
 distributed/diagnostics/progressbar.py            | 4 ++++
 distributed/diagnostics/tests/test_progressbar.py | 3 ++-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/distributed/diagnostics/progressbar.py b/distributed/diagnostics/progressbar.py
index d79f91f5e7d..3263503b2a2 100644
--- a/distributed/diagnostics/progressbar.py
+++ b/distributed/diagnostics/progressbar.py
@@ -124,6 +124,10 @@ def _draw_bar(self, remaining, all, **kwargs):
             sys.stdout.write(msg)
             sys.stdout.flush()
 
+    def _draw_stop(self, **kwargs):
+        sys.stdout.write('\r')
+        sys.stdout.flush()
+
 
 class ProgressWidget(ProgressBar):
     """ ProgressBar that uses an IPython ProgressBar widget for the notebook
diff --git a/distributed/diagnostics/tests/test_progressbar.py b/distributed/diagnostics/tests/test_progressbar.py
index 3c25a71b645..16eeeab0464 100644
--- a/distributed/diagnostics/tests/test_progressbar.py
+++ b/distributed/diagnostics/tests/test_progressbar.py
@@ -73,7 +73,8 @@ def f():
 
 def check_bar_completed(capsys, width=40):
     out, err = capsys.readouterr()
-    bar, percent, time = [i.strip() for i in out.split('\r')[-1].split('|')]
+    # trailing newline so grab next to last line for final state of bar
+    bar, percent, time = [i.strip() for i in out.split('\r')[-2].split('|')]
     assert bar == '[' + '#' * width + ']'
     assert percent == '100% Completed'
 

From 05a046b1d8aedee4e9fdd338acf5f5314ad9ead9 Mon Sep 17 00:00:00 2001
From: Tom Augspurger <TomAugspurger@users.noreply.github.com>
Date: Thu, 12 Jul 2018 20:31:41 -0500
Subject: [PATCH 0029/1550] TST: Added another nested parallelism test (#1710)

---
 distributed/tests/test_joblib.py | 37 ++++++++++++++++++++++++++++++++
 1 file changed, 37 insertions(+)

diff --git a/distributed/tests/test_joblib.py b/distributed/tests/test_joblib.py
index 6ff06a922ff..aa81a45b5f3 100644
--- a/distributed/tests/test_joblib.py
+++ b/distributed/tests/test_joblib.py
@@ -182,6 +182,43 @@ def test_errors(loop, joblib):
     assert "create a dask client" in str(info.value).lower()
 
 
+def test_correct_nested_backend(loop, joblib):
+    if LooseVersion(joblib.__version__) <= LooseVersion("0.11.0"):
+        pytest.skip("Requires nested parallelism")
+
+    with cluster() as (s, [a, b]):
+        with Client(s['address'], loop=loop) as client:
+            # No requirement, should be us
+            with joblib.parallel_backend('dask') as (ba, _):
+                result = joblib.Parallel(n_jobs=2)(joblib.delayed(outer)(
+                    joblib, nested_require=None) for _ in range(1))
+                assert isinstance(result[0][0][0],
+                                  distributed_joblib.DaskDistributedBackend)
+
+            # Require threads, should be threading
+            with joblib.parallel_backend('dask') as (ba, _):
+                result = joblib.Parallel(n_jobs=2)(joblib.delayed(outer)(
+                    joblib, nested_require='sharedmem') for _ in range(1))
+                assert isinstance(result[0][0][0],
+                                  joblib.parallel.ThreadingBackend)
+
+
+def outer(joblib, nested_require):
+    return joblib.Parallel(n_jobs=2, prefer='threads')(
+        joblib.delayed(middle)(joblib, nested_require) for _ in range(1)
+    )
+
+
+def middle(joblib, require):
+    return joblib.Parallel(n_jobs=2, require=require)(
+        joblib.delayed(inner)(joblib) for _ in range(1)
+    )
+
+
+def inner(joblib):
+    return joblib.parallel.Parallel()._backend
+
+
 def test_secede_with_no_processes(loop, joblib):
     # https://github.com/dask/distributed/issues/1775
 

From 67239aa40c1e939a2744bd210e4dd15e4b1624b6 Mon Sep 17 00:00:00 2001
From: Dave Hirschfeld <dave.hirschfeld@gmail.com>
Date: Fri, 13 Jul 2018 22:34:25 +1000
Subject: [PATCH 0030/1550] Use type hints to further emphasize the custom
 serialization api (#2116)

---
 distributed/protocol/serialize.py | 4 ++--
 docs/source/serialization.rst     | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index c2a1274afe2..cb3b802c504 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -380,8 +380,8 @@ def register_serialization(cls, serialize, deserialize):
     Parameters
     ----------
     cls: type
-    serialize: function
-    deserialize: function
+    serialize: callable(cls) -> Tuple[Dict, List[bytes]]
+    deserialize: callable(header: Dict, frames: List[bytes]) -> cls
 
     Examples
     --------
diff --git a/docs/source/serialization.rst b/docs/source/serialization.rst
index e457681a662..cdb765c380a 100644
--- a/docs/source/serialization.rst
+++ b/docs/source/serialization.rst
@@ -150,12 +150,12 @@ register them with Dask.
         def __init__(self, name):
             self.name = name
 
-    def serialize(human):
+    def serialize(human: Human) -> Tuple[Dict, List[bytes]]:
         header = {}
         frames = [human.name.encode()]
         return header, frames
 
-    def deserialize(header, frames):
+    def deserialize(header: Dict, frames: List[bytes]) -> Human:
         return Human(frames[0].decode())
 
     from distributed.protocol.serialize import register_serialization

From 81a0f9547e61701a9ef65f40436b18d318bd5f2c Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sun, 15 Jul 2018 17:19:33 -0500
Subject: [PATCH 0031/1550] Fix cleanup with empty response in gather dep
 (#2112)

---
 distributed/worker.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/distributed/worker.py b/distributed/worker.py
index 91100373e46..d6a0398efe3 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -1861,15 +1861,16 @@ def gather_dep(self, worker, dep, deps, total_nbytes, cause=None):
             finally:
                 self.comm_nbytes -= total_nbytes
                 busy = response.get('status', '') == 'busy'
+                data = response.get('data', {})
 
                 for d in self.in_flight_workers.pop(worker):
-                    if not busy and d in response['data']:
-                        self.transition_dep(d, 'memory', value=response['data'][d])
+                    if not busy and d in data:
+                        self.transition_dep(d, 'memory', value=data[d])
                     elif self.dep_state.get(d) != 'memory':
                         self.transition_dep(d, 'waiting', worker=worker,
                                             remove=not busy)
 
-                    if not busy and d not in response['data'] and d in self.dependents:
+                    if not busy and d not in data and d in self.dependents:
                         self.log.append(('missing-dep', d))
                         self.batched_stream.send({'op': 'missing-data',
                                                   'errant_worker': worker,

From 808afe23d7bea8e4c4a42f6b91fe49f4da013cb6 Mon Sep 17 00:00:00 2001
From: Matt Nicolls <2540582+nicolls1@users.noreply.github.com>
Date: Mon, 16 Jul 2018 22:38:07 +0200
Subject: [PATCH 0032/1550] Update dask-scheduler cli help text for preload
 (#2120)

---
 distributed/cli/dask_scheduler.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/distributed/cli/dask_scheduler.py b/distributed/cli/dask_scheduler.py
index 35eb72bb6b8..b719bbda12b 100755
--- a/distributed/cli/dask_scheduler.py
+++ b/distributed/cli/dask_scheduler.py
@@ -59,8 +59,8 @@
 @click.option('--local-directory', default='', type=str,
               help="Directory to place scheduler files")
 @click.option('--preload', type=str, multiple=True, is_eager=True,
-              help='Module that should be loaded by each worker process '
-                   'like "foo.bar" or "/path/to/foo.py"')
+              help='Module that should be loaded by the scheduler process  '
+                   'like "foo.bar" or "/path/to/foo.py".')
 @click.argument('preload_argv', nargs=-1,
                 type=click.UNPROCESSED, callback=validate_preload_argv)
 def main(host, port, bokeh_port, show, _bokeh, bokeh_whitelist, bokeh_prefix,

From 82d51e1974c4b11b14b628689ce9651b01065aec Mon Sep 17 00:00:00 2001
From: Dave Hirschfeld <dave.hirschfeld@gmail.com>
Date: Wed, 18 Jul 2018 02:37:35 +1000
Subject: [PATCH 0033/1550]  Add custom serialization support for pyarrow 
 (#2115)

---
 .gitignore                               |  2 +
 distributed/protocol/__init__.py         |  5 +++
 distributed/protocol/arrow.py            | 53 ++++++++++++++++++++++++
 distributed/protocol/tests/test_arrow.py | 44 ++++++++++++++++++++
 distributed/utils.py                     |  5 ++-
 5 files changed, 108 insertions(+), 1 deletion(-)
 create mode 100644 distributed/protocol/arrow.py
 create mode 100644 distributed/protocol/tests/test_arrow.py

diff --git a/.gitignore b/.gitignore
index 7e110237d4b..7510e74bcbf 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,3 +6,5 @@ docs/build
 continuous_integration/hdfs-initialized
 .cache
 .#*
+.idea/
+.pytest_cache/
diff --git a/distributed/protocol/__init__.py b/distributed/protocol/__init__.py
index a6a9afaf324..01ac7e8464a 100644
--- a/distributed/protocol/__init__.py
+++ b/distributed/protocol/__init__.py
@@ -37,3 +37,8 @@ def _register_keras():
 @partial(register_serialization_lazy, "sparse")
 def _register_sparse():
     from . import sparse
+
+
+@partial(register_serialization_lazy, "pyarrow")
+def _register_arrow():
+    from . import arrow
diff --git a/distributed/protocol/arrow.py b/distributed/protocol/arrow.py
new file mode 100644
index 00000000000..87c5d05c99f
--- /dev/null
+++ b/distributed/protocol/arrow.py
@@ -0,0 +1,53 @@
+from __future__ import print_function, division, absolute_import
+
+from .serialize import register_serialization
+
+
+def serialize_batch(batch):
+    import pyarrow as pa
+    sink = pa.BufferOutputStream()
+    writer = pa.RecordBatchStreamWriter(sink, batch.schema)
+    writer.write_batch(batch)
+    writer.close()
+    buf = sink.get_result()
+    header = {}
+    frames = [buf]
+    return header, frames
+
+
+def deserialize_batch(header, frames):
+    import pyarrow as pa
+    blob = frames[0]
+    reader = pa.RecordBatchStreamReader(pa.BufferReader(blob))
+    return reader.read_next_batch()
+
+
+def serialize_table(tbl):
+    import pyarrow as pa
+    sink = pa.BufferOutputStream()
+    writer = pa.RecordBatchStreamWriter(sink, tbl.schema)
+    writer.write_table(tbl)
+    writer.close()
+    buf = sink.get_result()
+    header = {}
+    frames = [buf]
+    return header, frames
+
+
+def deserialize_table(header, frames):
+    import pyarrow as pa
+    blob = frames[0]
+    reader = pa.RecordBatchStreamReader(pa.BufferReader(blob))
+    return reader.read_all()
+
+
+register_serialization(
+    'pyarrow.lib.RecordBatch',
+    serialize_batch,
+    deserialize_batch
+)
+register_serialization(
+    'pyarrow.lib.Table',
+    serialize_table,
+    deserialize_table
+)
diff --git a/distributed/protocol/tests/test_arrow.py b/distributed/protocol/tests/test_arrow.py
new file mode 100644
index 00000000000..6f014bae323
--- /dev/null
+++ b/distributed/protocol/tests/test_arrow.py
@@ -0,0 +1,44 @@
+import pandas as pd
+import pytest
+
+pa = pytest.importorskip('pyarrow')
+
+from distributed.utils_test import gen_cluster
+from distributed.protocol import deserialize, serialize
+from distributed.protocol.serialize import class_serializers, typename
+
+
+df = pd.DataFrame({'A': list('abc'), 'B': [1,2,3]})
+tbl = pa.Table.from_pandas(df, preserve_index=False)
+batch = pa.RecordBatch.from_pandas(df, preserve_index=False)
+
+
+@pytest.mark.parametrize('obj', [batch, tbl], ids=["RecordBatch", "Table"])
+def test_roundtrip(obj):
+    # Test that the serialize/deserialize functions actually
+    # work independent of distributed
+    header, frames = serialize(obj)
+    new_obj = deserialize(header, frames)
+    assert obj.equals(new_obj)
+
+
+@pytest.mark.parametrize('obj', [batch, tbl], ids=["RecordBatch", "Table"])
+def test_typename(obj):
+    # The typename used to register the custom serialization is hardcoded
+    # ensure that the typename hasn't changed
+    assert typename(type(obj)) in class_serializers
+
+
+def echo(arg):
+    return arg
+
+
+@pytest.mark.parametrize('obj', [batch, tbl], ids=["RecordBatch", "Table"])
+def test_scatter(obj):
+    @gen_cluster(client=True)
+    def run_test(client, scheduler, worker1, worker2):
+        obj_fut = yield client.scatter(obj)
+        fut = client.submit(echo, obj_fut)
+        result = yield fut
+        assert obj.equals(result)
+    run_test()
diff --git a/distributed/utils.py b/distributed/utils.py
index 53495ba60b4..666bb9dd26d 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -1266,7 +1266,10 @@ def nbytes(frame, _bytes_like=(bytes, bytearray)):
         if isinstance(frame, _bytes_like):
             return len(frame)
         else:
-            return frame.nbytes
+            try:
+                return frame.nbytes
+            except AttributeError:
+                return len(frame)
 
 
 def PeriodicCallback(callback, callback_time, io_loop=None):

From 1283d415879c065391e736b9361e9d357588edd9 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 19 Jul 2018 07:20:50 -0400
Subject: [PATCH 0034/1550] XFail test_open_close_many_workers (#2125)

We should fix this, but don't have the time right now.
Intermittent failures on this test are interrupting development flow.
---
 distributed/tests/test_client.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 75558352111..c854e2bcb27 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -3663,16 +3663,16 @@ def test_reconnect(loop):
     c.close()
 
 
-# On Python 2, heavy process spawning can deadlock (e.g. on a logging IO lock)
-_params = ([(Worker, 100, 5), (Nanny, 10, 20)]
-           if sys.version_info >= (3,)
-           else [(Worker, 100, 5)])
-
-
 @slow
 @pytest.mark.skipif(sys.platform.startswith('win'),
                     reason="num_fds not supported on windows")
-@pytest.mark.parametrize("worker,count,repeat", _params)
+@pytest.mark.skipif(sys.version_info[0] == 2,
+                    reason="Semaphore.acquire doesn't support timeout option")
+@pytest.mark.xfail(reason='TODO: intermittent failures')
+@pytest.mark.parametrize("worker,count,repeat", [
+    (Worker, 100, 5),
+    (Nanny, 10, 20)
+])
 def test_open_close_many_workers(loop, worker, count, repeat):
     psutil = pytest.importorskip('psutil')
     proc = psutil.Process()
@@ -3706,7 +3706,7 @@ def start_worker(sleep, duration, repeat=1):
             sleep(1)
 
             for i in range(count):
-                done.acquire()
+                done.acquire(timeout=5)
                 gc.collect()
                 if not running:
                     break

From ff2602adb26139dc9c0addcb5499466e982827cf Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 19 Jul 2018 14:58:39 -0400
Subject: [PATCH 0035/1550] Expand resources in graph_to_futures (#2131)

Previously this was handled only in a few of the submission functions.
Now we lower this logic to the core graph_to_futures method, applying
it more uniformly across computation.
---
 distributed/client.py               | 29 ++++++-------------
 distributed/tests/test_resources.py | 43 +++++++++++++++++++++++++++--
 2 files changed, 50 insertions(+), 22 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index b066d5cc05d..065a62c65a6 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -2069,6 +2069,15 @@ def _graph_to_futures(self, dsk, keys, restrictions=None,
                           user_priority=0, resources=None, retries=None,
                           fifo_timeout=0):
         with self._lock:
+            if resources:
+                resources = self._expand_resources(resources,
+                                                   all_keys=itertools.chain(dsk, keys))
+
+            if retries:
+                retries = self._expand_retries(retries,
+                                               all_keys=itertools.chain(dsk, keys))
+
+            print(resources)
             keyset = set(keys)
             flatkeys = list(map(tokey, keys))
             futures = {key: Future(key, self, inform=False) for key in keyset}
@@ -2327,16 +2336,6 @@ def compute(self, collections, sync=False, optimize_graph=True,
         restrictions, loose_restrictions = self.get_restrictions(collections,
                                                                  workers, allow_other_workers)
 
-        if resources:
-            resources = self._expand_resources(resources,
-                                               all_keys=itertools.chain(dsk, dsk2))
-
-        if retries:
-            retries = self._expand_retries(retries,
-                                           all_keys=itertools.chain(dsk, dsk2))
-        else:
-            retries = None
-
         if not isinstance(priority, Number):
             priority = {k: p for c, p in priority.items()
                              for k in self._expand_key(c)}
@@ -2429,16 +2428,6 @@ def persist(self, collections, optimize_graph=True, workers=None,
         restrictions, loose_restrictions = self.get_restrictions(collections,
                                                                  workers, allow_other_workers)
 
-        if resources:
-            resources = self._expand_resources(resources,
-                                               all_keys=itertools.chain(dsk, names))
-
-        if retries:
-            retries = self._expand_retries(retries,
-                                           all_keys=itertools.chain(dsk, names))
-        else:
-            retries = None
-
         if not isinstance(priority, Number):
             priority = {k: p for c, p in priority.items()
                              for k in self._expand_key(c)}
diff --git a/distributed/tests/test_resources.py b/distributed/tests/test_resources.py
index 55e178eb89d..46e22d0f530 100644
--- a/distributed/tests/test_resources.py
+++ b/distributed/tests/test_resources.py
@@ -6,10 +6,10 @@
 import pytest
 from tornado import gen
 
-from distributed import Worker
+from distributed import Worker, Client
 from distributed.client import wait
 from distributed.utils import tokey
-from distributed.utils_test import (inc, gen_cluster,
+from distributed.utils_test import (inc, gen_cluster, cluster,
                                     slowinc, slowadd)
 from distributed.utils_test import loop # noqa: F401
 
@@ -260,3 +260,42 @@ def test_dont_optimize_out(c, s, a, b):
 
     for key in map(tokey, y.__dask_keys__()):
         assert 'executing' in str(a.story(key))
+
+
+@gen_cluster(client=True, ncores=[('127.0.0.1', 1, {'resources': {'A': 1}}),
+                                  ('127.0.0.1', 1, {'resources': {'B': 1}})])
+def test_full_collections(c, s, a, b):
+    dd = pytest.importorskip('dask.dataframe')
+    df = dd.demo.make_timeseries(freq='60s', partition_freq='1d',
+            start='2000-01-01', end='2000-01-31')
+    z = df.x + df.y  # some extra nodes in the graph
+
+    yield c.compute(z, resources={tuple(z.dask): {'A': 1}})
+    assert a.log
+    assert not b.log
+
+
+@pytest.mark.parametrize('optimize_graph', [
+    pytest.mark.xfail(True, reason="don't track resources through optimization"),
+    False
+])
+def test_collections_get(loop, optimize_graph):
+    da = pytest.importorskip('dask.array')
+    with cluster() as (s, [a, b]):
+        with Client(s['address'], loop=loop) as c:
+            def f(dask_worker):
+                dask_worker.set_resources(**{'A': 1})
+
+            c.run(f, workers=[a['address']])
+
+            x = da.random.random(100, chunks=(10,)) + 1
+
+            x.compute(resources={tuple(x.dask): {'A': 1}},
+                      optimize_graph=optimize_graph)
+
+            def g(dask_worker):
+                return len(dask_worker.log)
+
+            logs = c.run(g)
+            assert logs[a['address']]
+            assert not logs[b['address']]

From ca70550daa6765bb45c846cfdc069f70151f1634 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 19 Jul 2018 14:59:25 -0400
Subject: [PATCH 0036/1550] Test that worker restrictions are cleared after
 cancellation (#2107)

---
 distributed/tests/test_scheduler.py | 20 ++++++++++++++++++++
 distributed/worker.py               |  3 +++
 2 files changed, 23 insertions(+)

diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 4f6bea05e1c..5a80faf8f83 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -6,6 +6,7 @@
 import json
 from operator import add, mul
 import sys
+from time import sleep
 
 import dask
 from dask import delayed
@@ -1340,3 +1341,22 @@ def test_closing_scheduler_closes_workers(s, a, b):
     while a.status != 'closed' or b.status != 'closed':
         yield gen.sleep(0.01)
         assert time() < start + 2
+
+
+@gen_cluster(client=True, ncores=[('127.0.0.1', 1)],
+             worker_kwargs={'resources': {'A': 1}})
+def test_resources_reset_after_cancelled_task(c, s, w):
+    future = c.submit(sleep, 0.2, resources={'A': 1})
+
+    while not w.executing:
+        yield gen.sleep(0.01)
+
+    yield future.cancel()
+
+    while w.executing:
+        yield gen.sleep(0.01)
+
+    assert not s.workers[w.address].used_resources['A']
+    assert w.available_resources == {'A': 1}
+
+    yield c.submit(inc, 1, resources={'A': 1})
diff --git a/distributed/worker.py b/distributed/worker.py
index d6a0398efe3..feb67ce5a92 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -2034,6 +2034,9 @@ def release_key(self, key, cause=None, reason=None, report=True):
                 self.executing.remove(key)
 
             if key in self.resource_restrictions:
+                if state == 'executing':
+                    for resource, quantity in self.resource_restrictions[key].items():
+                        self.available_resources[resource] += quantity
                 del self.resource_restrictions[key]
 
             if report and state in PROCESSING:  # not finished

From 525549938b6765900b6c77a444d43796b73dfa78 Mon Sep 17 00:00:00 2001
From: Dave Hirschfeld <dave.hirschfeld@gmail.com>
Date: Sat, 21 Jul 2018 01:52:20 +1000
Subject: [PATCH 0037/1550] Update .gitignore (#2135)

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index 7510e74bcbf..7888407d33f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,3 +8,4 @@ continuous_integration/hdfs-initialized
 .#*
 .idea/
 .pytest_cache/
+dask-worker-space/

From 8a78ec2dc12f9770758409ae7482872ab389086b Mon Sep 17 00:00:00 2001
From: Dror Birkman <dror.birkman@lightcyber.com>
Date: Sun, 22 Jul 2018 06:33:00 +0300
Subject: [PATCH 0038/1550] Use PID and counter in thread names (#2084) (#2128)

---
 distributed/tests/test_threadpoolexecutor.py | 6 ++++++
 distributed/threadpoolexecutor.py            | 7 ++++++-
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/distributed/tests/test_threadpoolexecutor.py b/distributed/tests/test_threadpoolexecutor.py
index f63dfe0244c..8777e574282 100644
--- a/distributed/tests/test_threadpoolexecutor.py
+++ b/distributed/tests/test_threadpoolexecutor.py
@@ -119,3 +119,9 @@ def f():
 
         future = e.submit(f)
         result = future.result()
+
+
+def test_thread_name():
+    with ThreadPoolExecutor(2) as e:
+        e.map(id, range(10))
+        assert len({thread.name for thread in e._threads}) == 2
diff --git a/distributed/threadpoolexecutor.py b/distributed/threadpoolexecutor.py
index 1b31ce125d7..8e9f10cadc8 100644
--- a/distributed/threadpoolexecutor.py
+++ b/distributed/threadpoolexecutor.py
@@ -23,8 +23,10 @@
 from __future__ import print_function, division, absolute_import
 
 from . import _concurrent_futures_thread as thread
+import os
 import logging
 import threading
+import itertools
 
 from .metrics import time
 
@@ -62,6 +64,9 @@ def _worker(executor, work_queue):
 
 
 class ThreadPoolExecutor(thread.ThreadPoolExecutor):
+    # Used to assign unique thread names
+    _counter = itertools.count()
+
     def __init__(self, *args, **kwargs):
         super(ThreadPoolExecutor, self).__init__(*args, **kwargs)
         self._rejoin_list = []
@@ -70,7 +75,7 @@ def __init__(self, *args, **kwargs):
     def _adjust_thread_count(self):
         if len(self._threads) < self._max_workers:
             t = threading.Thread(target=_worker,
-                                 name="ThreadPool worker %d" % len(self._threads,),
+                                 name="ThreadPoolExecutor-%d-%d" % (os.getpid(), next(self._counter)),
                                  args=(self, self._work_queue))
             t.daemon = True
             self._threads.add(t)

From 9a7016de0db2f5ecada1e1290eb0ee9fd91d65a2 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 23 Jul 2018 07:52:36 -0400
Subject: [PATCH 0039/1550] Allow client to query the task stream plot (#2122)

* Move TaskStreamPlugin to diagnostics

* Add Client.get_task_stream function
---
 distributed/__init__.py                       |   2 +-
 distributed/bokeh/scheduler.py                |   2 +-
 distributed/bokeh/tests/test_task_stream.py   |  40 -------
 distributed/client.py                         |  96 ++++++++++++++++
 .../{bokeh => diagnostics}/task_stream.py     |  62 +++++++++--
 .../diagnostics/tests/test_task_stream.py     | 104 ++++++++++++++++++
 distributed/scheduler.py                      |  17 ++-
 distributed/utils_test.py                     |   7 +-
 docs/source/api.rst                           |   2 +
 9 files changed, 277 insertions(+), 55 deletions(-)
 delete mode 100644 distributed/bokeh/tests/test_task_stream.py
 rename distributed/{bokeh => diagnostics}/task_stream.py (64%)
 create mode 100644 distributed/diagnostics/tests/test_task_stream.py

diff --git a/distributed/__init__.py b/distributed/__init__.py
index 3cde15ef52a..71e71a79143 100644
--- a/distributed/__init__.py
+++ b/distributed/__init__.py
@@ -7,7 +7,7 @@
 from .diagnostics import progress
 from .client import (Client, Executor, CompatibleExecutor,
                      wait, as_completed, default_client, fire_and_forget,
-                     Future, futures_of)
+                     Future, futures_of, get_task_stream)
 from .lock import Lock
 from .nanny import Nanny
 from .pubsub import Pub, Sub
diff --git a/distributed/bokeh/scheduler.py b/distributed/bokeh/scheduler.py
index 6533eb620ba..cd677e455c2 100644
--- a/distributed/bokeh/scheduler.py
+++ b/distributed/bokeh/scheduler.py
@@ -39,7 +39,7 @@
 from ..diagnostics.progress_stream import color_of, progress_quads, nbytes_bar
 from ..diagnostics.progress import AllProgress
 from ..diagnostics.graph_layout import GraphLayout
-from .task_stream import TaskStreamPlugin
+from ..diagnostics.task_stream import TaskStreamPlugin
 
 try:
     from cytoolz.curried import map, concat, groupby, valmap, first
diff --git a/distributed/bokeh/tests/test_task_stream.py b/distributed/bokeh/tests/test_task_stream.py
deleted file mode 100644
index 4b578a89c47..00000000000
--- a/distributed/bokeh/tests/test_task_stream.py
+++ /dev/null
@@ -1,40 +0,0 @@
-from __future__ import absolute_import, division, print_function
-
-import pytest
-pytest.importorskip('bokeh')
-
-from toolz import frequencies
-
-from distributed.utils_test import gen_cluster, div
-from distributed.client import wait
-from distributed.bokeh.task_stream import TaskStreamPlugin
-
-
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 3)
-def test_TaskStreamPlugin(c, s, *workers):
-    es = TaskStreamPlugin(s)
-    assert not es.buffer
-
-    futures = c.map(div, [1] * 10, range(10))
-    total = c.submit(sum, futures[1:])
-    yield wait(total)
-
-    assert len(es.buffer) == 11
-
-    workers = dict()
-
-    rects = es.rectangles(0, 10, workers)
-    assert all(n == 'div' for n in rects['name'])
-    assert all(d > 0 for d in rects['duration'])
-    counts = frequencies(rects['color'])
-    assert counts['black'] == 1
-    assert set(counts.values()) == {9, 1}
-    assert len(set(rects['y'])) == 3
-
-    rects = es.rectangles(2, 5, workers)
-    assert all(len(L) == 3 for L in rects.values())
-
-    starts = sorted(rects['start'])
-    rects = es.rectangles(2, 5, workers=workers,
-                          start_boundary=(starts[0] + starts[1]) / 2000)
-    assert set(rects['start']).issubset(set(starts[1:]))
diff --git a/distributed/client.py b/distributed/client.py
index 065a62c65a6..de4183138a6 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -3322,6 +3322,59 @@ def get_restrictions(cls, collections, workers, allow_other_workers):
     def collections_to_dsk(collections, *args, **kwargs):
         return collections_to_dsk(collections, *args, **kwargs)
 
+    def get_task_stream(self, start=None, stop=None, count=None):
+        """ Get task stream data from scheduler
+
+        This collects the data present in the diagnostic "Task Stream" plot on
+        the dashboard.  It includes the start, stop, transfer, and
+        deserialization time of every task for a particular duration.
+
+        Note that the task stream diagnostic does not run by default.  You may
+        wish to call this function once before you start work to ensure that
+        things start recording, and then again after you have completed.
+
+        Parameters
+        ----------
+        start: Number or string
+            When you want to start recording
+            If a number it should be the result of calling time()
+            If a string then it should be a time difference before now,
+            like '60s' or '500 ms'
+        stop: Number or string
+            When you want to stop recording
+        count: int
+            The number of desired records, ignored if both start and stop are
+            specified
+
+        Examples
+        --------
+        >>> client.get_task_stream()  # prime plugin if not already connected
+        >>> x.compute()  # do some work
+        >>> client.get_task_stream()
+        [{'task': ...,
+          'type': ...,
+          'thread': ...,
+          ...}]
+
+        Alternatively consider the context manager
+
+        >>> from dask.distributed import get_task_stream
+        >>> with get_task_stream() as ts:
+        ...     x.compute()
+        >>> ts.data
+        [...]
+
+        Returns
+        -------
+        L: List[Dict]
+
+        See Also
+        --------
+        get_task_stream: a dontext manager version of this method
+        """
+        return self.sync(self.scheduler.get_task_stream, start=start,
+                         stop=stop, count=count)
+
 
 class Executor(Client):
     """ Deprecated: see Client """
@@ -3691,6 +3744,49 @@ def fire_and_forget(obj):
                                           'client': 'fire-and-forget'})
 
 
+class get_task_stream(object):
+    """
+    Collect task stream within a context block
+
+    This provides diagnostic information about every task that was run during
+    the time when this block was active.
+
+    This must be used as a context manager.
+
+    Examples
+    --------
+    >>> with get_task_stream() as ts:
+    ...     x.compute()
+    >>> ts.data
+    [...]
+
+    See Also
+    --------
+    Client.get_task_stream: Function version of this context manager
+    """
+    def __init__(self, client=None):
+        self.data = []
+        self.client = client or default_client()
+        self.client.get_task_stream(start=0, stop=0)  # ensure plugin
+
+    def __enter__(self):
+        self.start = time()
+        return self
+
+    def __exit__(self, typ, value, traceback):
+        L = self.client.get_task_stream(start=self.start)
+        self.data.extend(L)
+
+    @gen.coroutine
+    def __aenter__(self):
+        raise gen.Return(self)
+
+    @gen.coroutine
+    def __aexit__(self, typ, value, traceback):
+        L = yield self.client.get_task_stream(start=self.start)
+        self.data.extend(L)
+
+
 @contextmanager
 def temp_default_client(c):
     """ Set the default client for the duration of the context
diff --git a/distributed/bokeh/task_stream.py b/distributed/diagnostics/task_stream.py
similarity index 64%
rename from distributed/bokeh/task_stream.py
rename to distributed/diagnostics/task_stream.py
index efa378c3dbf..7cabcf96311 100644
--- a/distributed/bokeh/task_stream.py
+++ b/distributed/diagnostics/task_stream.py
@@ -1,22 +1,23 @@
 from __future__ import print_function, division, absolute_import
 
+from collections import deque
 import logging
 
-from ..diagnostics.progress_stream import color_of
-from ..diagnostics.plugin import SchedulerPlugin
-from ..utils import key_split, format_time
+from .progress_stream import color_of
+from .plugin import SchedulerPlugin
+from ..utils import key_split, format_time, parse_timedelta
+from ..metrics import time
 
 
 logger = logging.getLogger(__name__)
 
 
 class TaskStreamPlugin(SchedulerPlugin):
-    def __init__(self, scheduler):
-        self.buffer = []
+    def __init__(self, scheduler, maxlen=100000):
+        self.buffer = deque(maxlen=maxlen)
         self.scheduler = scheduler
         scheduler.add_plugin(self)
         self.index = 0
-        self.maxlen = 100000
 
     def transition(self, key, start, finish, *args, **kwargs):
         if start == 'processing':
@@ -26,8 +27,48 @@ def transition(self, key, start, finish, *args, **kwargs):
             if finish == 'memory' or finish == 'erred':
                 self.buffer.append(kwargs)
                 self.index += 1
-                if len(self.buffer) > self.maxlen:
-                    self.buffer = self.buffer[len(self.buffer):]
+
+    def collect(self, start=None, stop=None, count=None):
+        def bisect(target, left, right):
+            if left == right:
+                return left
+
+            mid = (left + right) // 2
+            value = max(stop for _, start, stop in self.buffer[mid]['startstops'])
+
+            if value < target:
+                return bisect(target, mid + 1, right)
+            else:
+                return bisect(target, left, mid)
+
+        if isinstance(start, str):
+            start = time() - parse_timedelta(start)
+        if start is not None:
+            start = bisect(start, 0, len(self.buffer))
+
+        if isinstance(stop, str):
+            stop = time() - parse_timedelta(stop)
+        if stop is not None:
+            stop = bisect(stop, 0, len(self.buffer))
+
+        if count is not None:
+            if start is None and stop is None:
+                stop = len(self.buffer)
+                start = stop - count
+            elif start is None and stop is not None:
+                start = stop - count
+            elif start is not None and stop is None:
+                stop = start + count
+
+        if stop is None:
+            stop = len(self.buffer)
+        if start is None:
+            start = 0
+
+        start = max(0, start)
+        stop = min(stop, len(self.buffer))
+
+        return [self.buffer[i] for i in range(start, stop)]
 
     def rectangles(self, istart, istop=None, workers=None, start_boundary=0):
         L_start = []
@@ -42,7 +83,10 @@ def rectangles(self, istart, istop=None, workers=None, start_boundary=0):
         L_y = []
 
         diff = self.index - len(self.buffer)
-        for msg in self.buffer[istart - diff: istop - diff if istop else istop]:
+        if istop is None:
+            istop = len(self.buffer)
+        for i in range((istart or 0) - diff, istop - diff if istop else istop):
+            msg = self.buffer[i]
             key = msg['key']
             name = key_split(key)
             startstops = msg.get('startstops', [])
diff --git a/distributed/diagnostics/tests/test_task_stream.py b/distributed/diagnostics/tests/test_task_stream.py
new file mode 100644
index 00000000000..eccb0a9db8e
--- /dev/null
+++ b/distributed/diagnostics/tests/test_task_stream.py
@@ -0,0 +1,104 @@
+from __future__ import absolute_import, division, print_function
+
+from time import sleep
+
+from toolz import frequencies
+
+from distributed import Client, get_task_stream
+from distributed.utils_test import gen_cluster, div, inc, slowinc, cluster
+from distributed.utils_test import loop  # noqa F401
+from distributed.client import wait
+from distributed.diagnostics.task_stream import TaskStreamPlugin
+from distributed.metrics import time
+
+
+@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 3)
+def test_TaskStreamPlugin(c, s, *workers):
+    es = TaskStreamPlugin(s)
+    assert not es.buffer
+
+    futures = c.map(div, [1] * 10, range(10))
+    total = c.submit(sum, futures[1:])
+    yield wait(total)
+
+    assert len(es.buffer) == 11
+
+    workers = dict()
+
+    rects = es.rectangles(0, 10, workers)
+    assert all(n == 'div' for n in rects['name'])
+    assert all(d > 0 for d in rects['duration'])
+    counts = frequencies(rects['color'])
+    assert counts['black'] == 1
+    assert set(counts.values()) == {9, 1}
+    assert len(set(rects['y'])) == 3
+
+    rects = es.rectangles(2, 5, workers)
+    assert all(len(L) == 3 for L in rects.values())
+
+    starts = sorted(rects['start'])
+    rects = es.rectangles(2, 5, workers=workers,
+                          start_boundary=(starts[0] + starts[1]) / 2000)
+    assert set(rects['start']).issubset(set(starts[1:]))
+
+
+@gen_cluster(client=True)
+def test_maxlen(c, s, a, b):
+    tasks = TaskStreamPlugin(s, maxlen=5)
+    futures = c.map(inc, range(10))
+    yield wait(futures)
+    assert len(tasks.buffer) == 5
+
+
+@gen_cluster(client=True)
+def test_collect(c, s, a, b):
+    tasks = TaskStreamPlugin(s)
+    start = time()
+    futures = c.map(slowinc, range(10), delay=0.1)
+    yield wait(futures)
+
+    L = tasks.collect()
+    assert len(L) == len(futures)
+    L = tasks.collect(start=start)
+    assert len(L) == len(futures)
+
+    L = tasks.collect(start=start + 0.2)
+    assert 4 <= len(L) <= len(futures)
+
+    L = tasks.collect(start='20 s')
+    assert len(L) == len(futures)
+
+    L = tasks.collect(start='500ms')
+    assert 0 < len(L) <= len(futures)
+
+    L = tasks.collect(count=3)
+    assert len(L) == 3
+    assert L == list(tasks.buffer)[-3:]
+
+    assert tasks.collect(stop=start + 100, count=3) == tasks.collect(count=3)
+    assert tasks.collect(start=start, count=3) == list(tasks.buffer)[:3]
+
+
+@gen_cluster(client=True)
+def test_client(c, s, a, b):
+    L = yield c.get_task_stream()
+    assert L == ()
+
+    futures = c.map(slowinc, range(10), delay=0.1)
+    yield wait(futures)
+
+    tasks = [p for p in s.plugins if isinstance(p, TaskStreamPlugin)][0]
+    L = yield c.get_task_stream()
+    assert L == tuple(tasks.buffer)
+
+
+def test_client_sync(loop):
+    with cluster() as (s, [a, b]):
+        with Client(s['address'], loop=loop) as c:
+            with get_task_stream(client=c) as ts:
+                sleep(0.1)  # to smooth over time differences on the scheduler
+                # to smooth over time differences on the scheduler
+                futures = c.map(inc, range(10))
+                wait(futures)
+
+            assert len(ts.data) == 10
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index bbcf0dad08d..3fd802b117d 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -924,7 +924,8 @@ def __init__(
             'get_metadata': self.get_metadata,
             'set_metadata': self.set_metadata,
             'heartbeat_worker': self.heartbeat_worker,
-            'get_task_status': self.get_task_status
+            'get_task_status': self.get_task_status,
+            'get_task_stream': self.get_task_stream,
         }
 
         self._transitions = {
@@ -2100,12 +2101,18 @@ def handle_worker(self, comm=None, worker=None):
                 worker_comm.abort()
                 self.remove_worker(address=worker)
 
-    def add_plugin(self, plugin):
+    def add_plugin(self, plugin=None, idempotent=True, **kwargs):
         """
         Add external plugin to scheduler
 
         See https://distributed.readthedocs.io/en/latest/plugins.html
         """
+        if isinstance(plugin, type):
+            plugin = plugin(self, **kwargs)
+
+        if idempotent and any(isinstance(p, type(plugin)) for p in self.plugins):
+            return
+
         self.plugins.append(plugin)
 
     def remove_plugin(self, plugin):
@@ -2960,6 +2967,12 @@ def get_task_status(self, stream=None, keys=None):
                       if key in self.tasks else None)
                 for key in keys}
 
+    def get_task_stream(self, comm=None, start=None, stop=None, count=None):
+        from distributed.diagnostics.task_stream import TaskStreamPlugin
+        self.add_plugin(TaskStreamPlugin, idempotent=True)
+        ts = [p for p in self.plugins if isinstance(p, TaskStreamPlugin)][0]
+        return ts.collect(start=start, stop=stop, count=count)
+
     #####################
     # State Transitions #
     #####################
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index d6f713cd430..1e65c4e94cb 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -765,6 +765,7 @@ def test_func():
                     @gen.coroutine
                     def coro():
                         with dask.config.set(config):
+                            s = False
                             for i in range(5):
                                 try:
                                     s, ws = yield start_cluster(
@@ -774,9 +775,11 @@ def coro():
                                 except Exception as e:
                                     logger.error("Failed to start gen_cluster, retryng", exc_info=True)
                                 else:
+                                    workers[:] = ws
+                                    args = [s] + workers
                                     break
-                            workers[:] = ws
-                            args = [s] + workers
+                            if s is False:
+                                raise Exception("Could not start cluster")
                             if client:
                                 c = yield Client(s.address, loop=loop, security=security,
                                                  asynchronous=True, **client_kwargs)
diff --git a/docs/source/api.rst b/docs/source/api.rst
index eff1b5c3409..687b87bebb3 100644
--- a/docs/source/api.rst
+++ b/docs/source/api.rst
@@ -19,6 +19,7 @@ API
    Client.get_executor
    Client.get_metadata
    Client.get_scheduler_logs
+   Client.get_task_stream
    Client.get_worker_logs
    Client.has_what
    Client.list_datasets
@@ -92,6 +93,7 @@ API
    wait
    fire_and_forget
    futures_of
+   get_task_stream
 
 
 Asynchronous methods

From 9206f32bbc18128e7a3aaaaa8aec91ac85e43cbc Mon Sep 17 00:00:00 2001
From: Dror Birkman <dror.birkman@lightcyber.com>
Date: Tue, 24 Jul 2018 16:31:08 +0300
Subject: [PATCH 0040/1550] Remove extra print (#2141)

---
 distributed/client.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/distributed/client.py b/distributed/client.py
index de4183138a6..a0c3028c1b0 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -2077,7 +2077,6 @@ def _graph_to_futures(self, dsk, keys, restrictions=None,
                 retries = self._expand_retries(retries,
                                                all_keys=itertools.chain(dsk, keys))
 
-            print(resources)
             keyset = set(keys)
             flatkeys = list(map(tokey, keys))
             futures = {key: Future(key, self, inform=False) for key in keyset}

From 4979df6aa4acb16c2e8c694fa9d1478d6fe48b07 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sun, 29 Jul 2018 08:57:48 -0700
Subject: [PATCH 0041/1550] Make bokeh coloring deterministic using hash
 function (#2143)

Previously we would assign colors based on the order of tasks arrival in
the scheduler.  Now we use the has so that this is consistent across
sessions.
---
 distributed/diagnostics/progress_stream.py    | 26 +++----------------
 .../diagnostics/tests/test_progress_stream.py |  4 +--
 distributed/profile.py                        | 21 ++-------------
 distributed/utils.py                          | 21 ++++++++++++---
 4 files changed, 24 insertions(+), 48 deletions(-)

diff --git a/distributed/diagnostics/progress_stream.py b/distributed/diagnostics/progress_stream.py
index 84d87f4c157..60704a5670a 100644
--- a/distributed/diagnostics/progress_stream.py
+++ b/distributed/diagnostics/progress_stream.py
@@ -1,26 +1,20 @@
 from __future__ import print_function, division, absolute_import
 
-import itertools
 import logging
-import random
 
-from bokeh.palettes import viridis
-from toolz import valmap, merge, memoize
+from toolz import valmap, merge
 from tornado import gen
 
 from .progress import AllProgress
 
 from ..core import connect, coerce_to_address
 from ..scheduler import Scheduler
-from ..utils import key_split
+from ..utils import key_split, color_of
 from ..worker import dumps_function
 
 
 logger = logging.getLogger(__name__)
 
-task_stream_palette = list(viridis(25))
-random.shuffle(task_stream_palette)
-
 
 def counts(scheduler, allprogress):
     return merge({'all': valmap(len, allprogress.all),
@@ -29,20 +23,6 @@ def counts(scheduler, allprogress):
                      for state in ['memory', 'erred', 'released', 'processing']})
 
 
-counter = itertools.count()
-
-_incrementing_index_cache = dict()
-
-
-@memoize(cache=_incrementing_index_cache)
-def incrementing_index(o):
-    return next(counter)
-
-
-def color_of(o, palette=task_stream_palette):
-    return palette[incrementing_index(o) % len(palette)]
-
-
 @gen.coroutine
 def progress_stream(address, interval):
     """ Open a TCP connection to scheduler, receive progress messages
@@ -206,7 +186,7 @@ def color_of_message(msg):
           'compute': ''}
 
 
-def task_stream_append(lists, msg, workers, palette=task_stream_palette):
+def task_stream_append(lists, msg, workers):
     key = msg['key']
     name = key_split(key)
     startstops = msg.get('startstops', [])
diff --git a/distributed/diagnostics/tests/test_progress_stream.py b/distributed/diagnostics/tests/test_progress_stream.py
index 34bc6f2c9c1..ce21bb34193 100644
--- a/distributed/diagnostics/tests/test_progress_stream.py
+++ b/distributed/diagnostics/tests/test_progress_stream.py
@@ -7,7 +7,7 @@
 from dask import delayed
 from distributed.client import wait
 from distributed.diagnostics.progress_stream import (progress_quads,
-                                                     nbytes_bar, progress_stream, _incrementing_index_cache)
+                                                     nbytes_bar, progress_stream)
 from distributed.utils_test import div, gen_cluster, inc
 
 
@@ -18,8 +18,6 @@ def test_progress_quads():
            'released': {'inc': 1, 'dec': 0, 'add': 1},
            'processing': {'inc': 1, 'dec': 0, 'add': 2}}
 
-    _incrementing_index_cache.clear()
-
     d = progress_quads(msg, nrows=2)
     color = d.pop('color')
     assert len(set(color)) == 3
diff --git a/distributed/profile.py b/distributed/profile.py
index 3cb1c838d9b..46a4e441ebb 100644
--- a/distributed/profile.py
+++ b/distributed/profile.py
@@ -28,10 +28,8 @@
 
 from collections import defaultdict
 import linecache
-import itertools
-import toolz
 
-from .utils import format_time
+from .utils import format_time, color_of
 
 
 def identifier(frame):
@@ -190,7 +188,7 @@ def traverse(state, start, stop, height):
         try:
             colors.append(color_of(desc['filename']))
         except IndexError:
-            colors.append(palette[-1])
+            colors.append('gray')
 
         delta = (stop - start) / state['count']
 
@@ -216,18 +214,3 @@ def traverse(state, start, stop, height):
             'name': names,
             'time': times,
             'percentage': percentages}
-
-
-try:
-    from bokeh.palettes import viridis
-except ImportError:
-    palette = ['red', 'green', 'blue', 'yellow']
-else:
-    palette = viridis(10)
-
-counter = itertools.count()
-
-
-@toolz.memoize
-def color_of(x):
-    return palette[next(counter) % len(palette)]
diff --git a/distributed/utils.py b/distributed/utils.py
index 666bb9dd26d..81ea0d67d92 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -5,6 +5,7 @@
 from contextlib import contextmanager
 from datetime import timedelta
 import functools
+from hashlib import md5
 import inspect
 import json
 import logging
@@ -35,7 +36,7 @@
 
 import dask
 from dask import istask
-from toolz import memoize
+import toolz
 import tornado
 from tornado import gen
 from tornado.ioloop import IOLoop, PollIOLoop
@@ -113,7 +114,7 @@ def get_fileno_limit():
         return 512
 
 
-@memoize
+@toolz.memoize
 def _get_ip(host, port, family, default):
     # By using a UDP socket, we don't actually try to connect but
     # simply select the local address through which *host* is reachable.
@@ -670,7 +671,7 @@ def silence_logging(level, root='distributed'):
     return old
 
 
-@memoize
+@toolz.memoize
 def ensure_ip(hostname):
     """ Ensure that address is an IP address
 
@@ -1399,3 +1400,17 @@ def has_keyword(func, keyword):
         if gen.is_coroutine_function(func):
             func = func.__wrapped__
         return keyword in inspect.getargspec(func).args
+
+
+# from bokeh.palettes import viridis
+# palette = viridis(18)
+palette = ['#440154', '#471669', '#472A79', '#433C84', '#3C4D8A', '#355D8C',
+           '#2E6C8E', '#287A8E', '#23898D', '#1E978A', '#20A585', '#2EB27C',
+           '#45BF6F', '#64CB5D', '#88D547', '#AFDC2E', '#D7E219', '#FDE724']
+
+
+@toolz.memoize
+def color_of(x, palette=palette):
+    h = md5(str(x).encode())
+    n = int(h.hexdigest()[:8], 16)
+    return palette[n % len(palette)]

From da7eb711992b3a715f1d7a697ab825c088525a58 Mon Sep 17 00:00:00 2001
From: Ray Bell <rjbell1987@gmail.com>
Date: Tue, 31 Jul 2018 16:58:24 -0400
Subject: [PATCH 0042/1550] DOC: typos (#2148)

---
 docs/source/quickstart.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/source/quickstart.rst b/docs/source/quickstart.rst
index 7f5157fdb25..3d1e326f528 100644
--- a/docs/source/quickstart.rst
+++ b/docs/source/quickstart.rst
@@ -14,7 +14,7 @@ See :doc:`installation <install>` document for more information.
 Setup Dask.distributed the Easy Way
 -----------------------------------
 
-If you create an client without providing an address it will start up a local
+If you create a client without providing an address it will start up a local
 scheduler and worker for you.
 
 .. code-block:: python
@@ -41,7 +41,7 @@ Set up scheduler and worker processes on your local computer::
 .. note:: At least one ``dask-worker`` must be running after launching a
           scheduler.
 
-Launch an Client and point it to the IP/port of the scheduler.
+Launch a Client and point it to the IP/port of the scheduler.
 
 .. code-block:: python
 

From e4827100a814f3747a3d0aa01831da5053303bef Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 1 Aug 2018 10:11:04 -0700
Subject: [PATCH 0043/1550] Cleanup intermittent failures (#2146)

* make synchronous clients awaitable

* clean up handling of bare ioloop in tests

* test that the test suite doesn't leak threads

* reuse memory monitor's psutil.Process

Profiling showed that creating a new one each cycle generated a nontrivial
amount of overhead

* Be robust to missing data file

this happens during cleanup

* don't ask workers to report closed when removing them from scheduler

This is unnecessary and causes delays if the scheudler is going down as well.

* move client/worker cleanup within test function

Previously we did this at decorator call time, rather than testing time

* test that no new threads are created during test

* allow process watching threads to leak

* xfail test_quiet_client_close

* Don't test threads on python 2

* cleanup bokeh test

* improve reporting

* improve reporting

* don't check threads on windows

* clean up stealing test

* log del data error
---
 .../bokeh/tests/test_scheduler_bokeh.py       |  3 +-
 distributed/client.py                         |  8 ++-
 distributed/scheduler.py                      |  4 +-
 distributed/tests/test_client.py              | 23 +++---
 distributed/tests/test_steal.py               |  6 +-
 distributed/tests/test_worker.py              |  2 +-
 distributed/threadpoolexecutor.py             |  6 +-
 distributed/utils_test.py                     | 70 ++++++++++++-------
 distributed/worker.py                         |  9 ++-
 9 files changed, 87 insertions(+), 44 deletions(-)

diff --git a/distributed/bokeh/tests/test_scheduler_bokeh.py b/distributed/bokeh/tests/test_scheduler_bokeh.py
index a9cc0208801..235d85d0c5c 100644
--- a/distributed/bokeh/tests/test_scheduler_bokeh.py
+++ b/distributed/bokeh/tests/test_scheduler_bokeh.py
@@ -258,7 +258,8 @@ def test_ProcessingHistogram(c, s, a, b):
     assert (ph.source.data['top'] != 0).sum() == 1
 
     futures = c.map(slowinc, range(10), delay=0.050)
-    yield gen.sleep(0.100)
+    while not s.tasks:
+        yield gen.sleep(0.01)
 
     ph.update()
     assert ph.source.data['right'][-1] > 2
diff --git a/distributed/client.py b/distributed/client.py
index a0c3028c1b0..6b5de8ae3d4 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -739,7 +739,13 @@ def start(self, **kwargs):
             sync(self.loop, self._start, **kwargs)
 
     def __await__(self):
-        return self._started.__await__()
+        if hasattr(self, '_started'):
+            return self._started.__await__()
+        else:
+            @gen.coroutine
+            def _():
+                raise gen.Return(self)
+            return _().__await__()
 
     def _send_to_scheduler_safe(self, msg):
         if self.status in ('running', 'closing'):
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 3fd802b117d..a2412068c7b 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1160,7 +1160,7 @@ def close_worker(self, stream=None, worker=None, safe=None):
             nanny_addr = self.get_worker_service_addr(worker, 'nanny')
             address = nanny_addr or worker
 
-            self.worker_send(worker, {'op': 'close'})
+            self.worker_send(worker, {'op': 'close', 'report': False})
             self.remove_worker(address=worker, safe=safe)
 
     def _setup_logging(self):
@@ -1613,7 +1613,7 @@ def remove_worker(self, comm=None, address=None, safe=False, close=True):
             logger.info("Remove worker %s", address)
             if close:
                 with ignoring(AttributeError, CommClosedError):
-                    self.stream_comms[address].send({'op': 'close'})
+                    self.stream_comms[address].send({'op': 'close', 'report': False})
 
             self.remove_resources(address)
 
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index c854e2bcb27..57b9c3aca01 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -46,7 +46,7 @@
                                     randominc, inc, dec, div, throws, geninc, asyncinc,
                                     gen_cluster, gen_test, double, deep, popen,
                                     captured_logger, varying, map_varying,
-                                    wait_for, async_wait_for)
+                                    wait_for, async_wait_for, pristine_loop)
 from distributed.utils_test import loop, loop_in_thread, nodebug  # noqa F401
 
 
@@ -2084,7 +2084,6 @@ def test_multi_client(s, a, b):
 
 
 def long_running_client_connection(address):
-    from distributed.utils_test import pristine_loop
     with pristine_loop():
         c = Client(address)
         x = c.submit(lambda x: x + 1, 10)
@@ -3425,7 +3424,7 @@ def test_get_foo_lost_keys(c, s, u, v, w):
 
 
 @slow
-@gen_cluster(client=True, Worker=Nanny)
+@gen_cluster(client=True, Worker=Nanny, check_new_threads=False)
 def test_bad_tasks_fail(c, s, a, b):
     f = c.submit(sys.exit, 1)
     with pytest.raises(KilledWorker):
@@ -4128,7 +4127,7 @@ def f(x, y=0):
     assert len(b.data) > 2 * len(a.data)
 
 
-@gen_cluster(client=True)
+@gen_cluster(client=True, check_new_threads=False)
 def test_add_done_callback(c, s, a, b):
     S = set()
 
@@ -4628,6 +4627,7 @@ def test_fire_and_forget_err(c, s, a, b):
         assert time() < start + 1
 
 
+@pytest.mark.xfail(reason='Other tests bleed into the logs of this one')
 def test_quiet_client_close(loop):
     with captured_logger(logging.getLogger('distributed')) as logger:
         with Client(loop=loop, processes=False, threads_per_worker=4) as c:
@@ -5103,12 +5103,10 @@ def test_future_auto_inform(c, s, a, b):
 
 
 def test_client_async_before_loop_starts():
-    loop = IOLoop()
-    client = Client(asynchronous=True, loop=loop)
-    assert client.asynchronous
-    client.close()
-    # Avoid long wait for cluster close at shutdown
-    loop.close()
+    with pristine_loop() as loop:
+        client = Client(asynchronous=True, loop=loop)
+        assert client.asynchronous
+        client.close()
 
 
 @slow
@@ -5433,5 +5431,10 @@ def bad_fn(x):
     assert y.status == 'error'  # not cancelled
 
 
+def test_no_threads_lingering():
+    active = dict(threading._active)
+    assert threading.active_count() < 30, list(active.values())
+
+
 if sys.version_info >= (3, 5):
     from distributed.tests.py3_test_client import *  # noqa F401
diff --git a/distributed/tests/test_steal.py b/distributed/tests/test_steal.py
index 3309efc7240..01193b61e20 100644
--- a/distributed/tests/test_steal.py
+++ b/distributed/tests/test_steal.py
@@ -330,9 +330,11 @@ def test_steal_when_more_tasks(c, s, a, *rest):
 
     futures = [c.submit(slowidentity, x, pure=False, delay=0.2)
                for i in range(20)]
-    yield gen.sleep(0.1)
 
-    assert any(w.task_state for w in rest)
+    start = time()
+    while not any(w.task_state for w in rest):
+        yield gen.sleep(0.01)
+        assert time() < start + 1
 
 
 @gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 10)
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 1210d5213d0..8c6f0ee752b 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -181,7 +181,7 @@ def g():
     assert not os.path.exists(os.path.join(a.local_dir, 'foobar.py'))
 
 
-@pytest.mark.xfail(reason="don't yet support uploading pyc files")
+@pytest.mark.skip(reason="don't yet support uploading pyc files")
 @gen_cluster(client=True, ncores=[('127.0.0.1', 1)])
 def test_upload_file_pyc(c, s, w):
     with tmpfile() as dirname:
diff --git a/distributed/threadpoolexecutor.py b/distributed/threadpoolexecutor.py
index 8e9f10cadc8..c5c953ce0b9 100644
--- a/distributed/threadpoolexecutor.py
+++ b/distributed/threadpoolexecutor.py
@@ -23,6 +23,7 @@
 from __future__ import print_function, division, absolute_import
 
 from . import _concurrent_futures_thread as thread
+from .compatibility import Empty
 import os
 import logging
 import threading
@@ -48,7 +49,10 @@ def _worker(executor, work_queue):
                     executor._threads.remove(threading.current_thread())
                     rejoin_event.set()
                     break
-            task = work_queue.get()
+            try:
+                task = work_queue.get(timeout=1)
+            except Empty:
+                continue
             if task is not None:  # sentinel
                 task.run()
                 del task
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index 1e65c4e94cb..be4b241a9b7 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -39,8 +39,8 @@
 from tornado.gen import TimeoutError
 from tornado.ioloop import IOLoop
 
-from .client import default_client
-from .compatibility import PY3, iscoroutinefunction, Empty
+from .client import default_client, _global_clients
+from .compatibility import PY3, iscoroutinefunction, Empty, WINDOWS
 from .config import initialize_logging
 from .core import connect, rpc, CommClosedError
 from .metrics import time
@@ -97,6 +97,7 @@ def cleanup_global_workers():
 @pytest.fixture
 def loop():
     del _global_workers[:]
+    _global_clients.clear()
     with pristine_loop() as loop:
         # Monkey-patch IOLoop.start to wait for loop stop
         orig_start = loop.start
@@ -125,6 +126,7 @@ def start():
         else:
             is_stopped.wait()
     del _global_workers[:]
+    _global_clients.clear()
 
 
 @pytest.fixture
@@ -727,7 +729,7 @@ def gen_cluster(ncores=[('127.0.0.1', 1), ('127.0.0.1', 2)],
                 scheduler='127.0.0.1', timeout=10, security=None,
                 Worker=Worker, client=False, scheduler_kwargs={},
                 worker_kwargs={}, client_kwargs={}, active_rpc_timeout=1,
-                config={}):
+                config={}, check_new_threads=True):
     from distributed import Client
     """ Coroutine test with small cluster
 
@@ -739,11 +741,6 @@ def test_foo(scheduler, worker1, worker2):
         start
         end
     """
-    del _global_workers[:]
-
-    reset_config()
-
-    dask.config.set({'distributed.comm.timeouts.connect': '5s'})
     worker_kwargs = merge({'memory_limit': TOTAL_MEMORY, 'death_timeout': 5},
                           worker_kwargs)
 
@@ -752,6 +749,13 @@ def _(func):
             func = gen.coroutine(func)
 
         def test_func():
+            del _global_workers[:]
+            _global_clients.clear()
+            active_threads_start = set(threading._active)
+
+            reset_config()
+
+            dask.config.set({'distributed.comm.timeouts.connect': '5s'})
             # Restore default logging levels
             # XXX use pytest hooks/fixtures instead?
             for name, level in logging_levels.items():
@@ -810,22 +814,40 @@ def coro():
 
                     result = loop.run_sync(coro, timeout=timeout * 2 if timeout else timeout)
 
-            for w in workers:
-                if getattr(w, 'data', None):
-                    try:
-                        w.data.clear()
-                    except EnvironmentError:
-                        # zict backends can fail if their storage directory
-                        # was already removed
-                        pass
-                    del w.data
-            DequeHandler.clear_all_instances()
-            for w in _global_workers:
-                w = w()
-                w._close(report=False, executor_wait=False)
-                if w.status == 'running':
-                    w.close()
-            del _global_workers[:]
+                for w in workers:
+                    if getattr(w, 'data', None):
+                        try:
+                            w.data.clear()
+                        except EnvironmentError:
+                            # zict backends can fail if their storage directory
+                            # was already removed
+                            pass
+                        del w.data
+                DequeHandler.clear_all_instances()
+                for w in _global_workers:
+                    w = w()
+                    w._close(report=False, executor_wait=False)
+                    if w.status == 'running':
+                        w.close()
+                del _global_workers[:]
+
+            if PY3 and not WINDOWS and check_new_threads:
+                start = time()
+                while True:
+                    bad = [t for t, v in threading._active.items()
+                           if t not in active_threads_start and
+                          "Threaded" not in v.name and
+                          "watch message queue" not in v.name]
+                    if not bad:
+                        break
+                    else:
+                        sleep(0.01)
+                    if time() > start + 2:
+                        from distributed import profile
+                        tid = bad[0]
+                        thread = threading._active[tid]
+                        call_stacks = profile.call_stack(sys._current_frames()[tid])
+                        assert False, (thread, call_stacks)
             return result
 
         return test_func
diff --git a/distributed/worker.py b/distributed/worker.py
index feb67ce5a92..891b413c302 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -478,6 +478,7 @@ def _close(self, report=True, timeout=10, nanny=True, executor_wait=True):
                                            self.scheduler.unregister(address=self.contact_address))
             self.scheduler.close_rpc()
             if isinstance(self.executor, ThreadPoolExecutor):
+                self.executor._work_queue.queue.clear()
                 self.executor.shutdown(wait=executor_wait, timeout=timeout)
             else:
                 self.executor.shutdown(wait=False)
@@ -2004,7 +2005,11 @@ def release_key(self, key, cause=None, reason=None, report=True):
                 self.log.append((key, 'release-key'))
             del self.tasks[key]
             if key in self.data and key not in self.dep_state:
-                del self.data[key]
+                try:
+                    del self.data[key]
+                except FileNotFoundError:
+                    logger.error("Tried to delete %s but no file found",
+                                 exc_info=True)
                 del self.nbytes[key]
                 del self.types[key]
 
@@ -2266,7 +2271,7 @@ def memory_monitor(self):
         self._memory_monitoring = True
         total = 0
 
-        proc = psutil.Process()
+        proc = self.monitor.proc
         memory = proc.memory_info().rss
         frac = memory / self.memory_limit
 

From 444e7bb8eb7fb258628ff9bdeecca9babf9e71fe Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 1 Aug 2018 13:14:18 -0700
Subject: [PATCH 0044/1550] Include serializers in Scheduler.gather calls
 (#2151)

---
 distributed/tests/test_client.py | 74 +++++++++++++++++---------------
 distributed/utils_comm.py        |  1 +
 distributed/worker.py            | 12 ++++--
 3 files changed, 49 insertions(+), 38 deletions(-)

diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 57b9c3aca01..6ab903ffdee 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -5317,41 +5317,45 @@ def test_client_active_bad_port():
     http_server.stop()
 
 
-@gen_cluster()
-def test_turn_off_pickle(s, a, b):
-    import numpy as np
-    c = yield Client(s.address, asynchronous=True,
-                     serializers=['dask', 'msgpack'])
-    try:
-        assert (yield c.submit(inc, 1)) == 2
-        yield c.submit(np.ones, 5)
-        yield c.scatter(1)
-
-        # Can't send complex data
-        with pytest.raises(TypeError):
-            future = yield c.scatter(inc)
-
-        # can send complex tasks (this uses pickle regardless)
-        future = c.submit(lambda x: x, inc)
-        yield wait(future)
-
-        # but can't receive complex results
-        with pytest.raises(TypeError):
-            yield future
-
-        # Run works
-        result = yield c.run(lambda: 1)
-        assert list(result.values()) == [1, 1]
-        result = yield c.run_on_scheduler(lambda: 1)
-        assert result == 1
-
-        # But not with complex return values
-        with pytest.raises(TypeError):
-            yield c.run(lambda: inc)
-        with pytest.raises(TypeError):
-            yield c.run_on_scheduler(lambda: inc)
-    finally:
-        yield c._close()
+@pytest.mark.parametrize('direct', [True, False])
+def test_turn_off_pickle(direct):
+    @gen_cluster()
+    def test(s, a, b):
+        import numpy as np
+        c = yield Client(s.address, asynchronous=True,
+                         serializers=['dask', 'msgpack'])
+        try:
+            assert (yield c.submit(inc, 1)) == 2
+            yield c.submit(np.ones, 5)
+            yield c.scatter(1)
+
+            # Can't send complex data
+            with pytest.raises(TypeError):
+                future = yield c.scatter(inc)
+
+            # can send complex tasks (this uses pickle regardless)
+            future = c.submit(lambda x: x, inc)
+            yield wait(future)
+
+            # but can't receive complex results
+            with pytest.raises(TypeError):
+                yield c.gather(future, direct=direct)
+
+            # Run works
+            result = yield c.run(lambda: 1)
+            assert list(result.values()) == [1, 1]
+            result = yield c.run_on_scheduler(lambda: 1)
+            assert result == 1
+
+            # But not with complex return values
+            with pytest.raises(TypeError):
+                yield c.run(lambda: inc)
+            with pytest.raises(TypeError):
+                yield c.run_on_scheduler(lambda: inc)
+        finally:
+            yield c._close()
+
+    test()
 
 
 @gen_cluster()
diff --git a/distributed/utils_comm.py b/distributed/utils_comm.py
index 46724973996..7e8702e40a2 100644
--- a/distributed/utils_comm.py
+++ b/distributed/utils_comm.py
@@ -58,6 +58,7 @@ def gather_from_workers(who_has, rpc, close=True, serializers=None, who=None):
         try:
             coroutines = {address: get_data_from_worker(rpc, keys, address,
                                                         who=who,
+                                                        serializers=serializers,
                                                         max_connections=False)
                           for address, keys in d.items()}
             response = {}
diff --git a/distributed/worker.py b/distributed/worker.py
index 891b413c302..48e4bb8a2e8 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -2781,7 +2781,8 @@ def parse_memory_limit(memory_limit, ncores):
 
 
 @gen.coroutine
-def get_data_from_worker(rpc, keys, worker, who=None, max_connections=None):
+def get_data_from_worker(rpc, keys, worker, who=None, max_connections=None,
+                         serializers=None, deserializers=None):
     """ Get keys from worker
 
     The worker has a two step handshake to acknowledge when data has been fully
@@ -2793,11 +2794,16 @@ def get_data_from_worker(rpc, keys, worker, who=None, max_connections=None):
     Worker.gather_deps
     utils_comm.gather_data_from_workers
     """
+    if serializers is None:
+        serializers = rpc.serializers
+    if deserializers is None:
+        deserializers = rpc.deserializers
+
     comm = yield rpc.connect(worker)
     try:
         response = yield send_recv(comm,
-                                   serializers=rpc.serializers,
-                                   deserializers=rpc.deserializers,
+                                   serializers=serializers,
+                                   deserializers=deserializers,
                                    op='get_data', keys=keys, who=who,
                                    max_connections=max_connections)
         if response['status'] == 'OK':

From 9919543ca62f1c7ac80c97559bfa2b245899b634 Mon Sep 17 00:00:00 2001
From: Olivier Grisel <olivier.grisel@ensta.org>
Date: Thu, 2 Aug 2018 15:55:39 +0200
Subject: [PATCH 0045/1550] Fix msgpack PendingDeprecationWarning for
 encoding='utf-8' (#2153)

---
 distributed/protocol/core.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/distributed/protocol/core.py b/distributed/protocol/core.py
index 4033c9be1a9..f7df6597752 100644
--- a/distributed/protocol/core.py
+++ b/distributed/protocol/core.py
@@ -19,6 +19,14 @@
 _deserialize = deserialize
 
 
+try:
+    msgpack.loads(msgpack.dumps(''), raw=False)
+    msgpack_raw_false = {'raw': False}
+except TypeError:
+    # Backward compat with old msgpack (prior to 0.5.2)
+    msgpack_raw_false = {'encoding': 'utf-8'}
+
+
 logger = logging.getLogger(__name__)
 
 
@@ -102,7 +110,7 @@ def loads(frames, deserialize=True, deserializers=None):
             return msg
 
         header = frames.pop()
-        header = msgpack.loads(header, encoding='utf8', use_list=False)
+        header = msgpack.loads(header, use_list=False, **msgpack_raw_false)
         keys = header['keys']
         headers = header['headers']
         bytestrings = set(header['bytestrings'])
@@ -174,7 +182,7 @@ def loads_msgpack(header, payload):
         dumps_msgpack
     """
     if header:
-        header = msgpack.loads(header, encoding='utf8', use_list=False)
+        header = msgpack.loads(header, use_list=False, **msgpack_raw_false)
     else:
         header = {}
 
@@ -186,4 +194,4 @@ def loads_msgpack(header, payload):
             raise ValueError("Data is compressed as %s but we don't have this"
                              " installed" % str(header['compression']))
 
-    return msgpack.loads(payload, encoding='utf8', use_list=False)
+    return msgpack.loads(payload, use_list=False, **msgpack_raw_false)

From 0c9ee7328656b175bdfe4f310c5ceeb7897b692a Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 2 Aug 2018 16:40:02 -0400
Subject: [PATCH 0046/1550] Cleanup recent cleanup PR (#2152)

* Cleanup recent cleanup PR

Some fallout from the recent intermittent testing PR

* extend timeout

* xfail test_diskutils

Raised at https://github.com/dask/distributed/issues/2155
---
 distributed/cli/tests/test_dask_scheduler.py | 2 +-
 distributed/tests/test_diskutils.py          | 5 ++++-
 distributed/utils_test.py                    | 2 +-
 3 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/distributed/cli/tests/test_dask_scheduler.py b/distributed/cli/tests/test_dask_scheduler.py
index 88507429cdd..456f44c55c4 100644
--- a/distributed/cli/tests/test_dask_scheduler.py
+++ b/distributed/cli/tests/test_dask_scheduler.py
@@ -30,7 +30,7 @@ def test_defaults(loop):
         def f():
             # Default behaviour is to listen on all addresses
             yield [
-                assert_can_connect_from_everywhere_4_6(8786, 2.0),  # main port
+                assert_can_connect_from_everywhere_4_6(8786, 5.0),  # main port
             ]
 
         loop.run_sync(f)
diff --git a/distributed/tests/test_diskutils.py b/distributed/tests/test_diskutils.py
index 598c2506b0d..d7079ca2039 100644
--- a/distributed/tests/test_diskutils.py
+++ b/distributed/tests/test_diskutils.py
@@ -9,9 +9,10 @@
 from time import sleep
 
 import mock
+import pytest
 
 import dask
-from distributed.compatibility import Empty
+from distributed.compatibility import Empty, WINDOWS
 from distributed.diskutils import WorkSpace
 from distributed.metrics import time
 from distributed.utils import mp_context
@@ -257,6 +258,8 @@ def _test_workspace_concurrency(tmpdir, timeout, max_procs):
 
 
 def test_workspace_concurrency(tmpdir):
+    if WINDOWS:
+        raise pytest.xfail.Exception('TODO: unknown failure on windows')
     _test_workspace_concurrency(tmpdir, 2.0, 6)
 
 
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index be4b241a9b7..cce9ad75dda 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -837,7 +837,7 @@ def coro():
                     bad = [t for t, v in threading._active.items()
                            if t not in active_threads_start and
                           "Threaded" not in v.name and
-                          "watch message queue" not in v.name]
+                          "watch message" not in v.name]
                     if not bad:
                         break
                     else:

From 770b4afa51bbc8e5fbe77d6f6b5d2fbe1f69a7bc Mon Sep 17 00:00:00 2001
From: Martin Durant <martindurant@users.noreply.github.com>
Date: Fri, 3 Aug 2018 10:07:15 -0400
Subject: [PATCH 0047/1550] Worker class (#2147)

* Start: make worker class an attribute of Nanny

* Make Nanny subclass test

* as keyword argument

* fix tests
---
 distributed/nanny.py            | 17 +++++++++--------
 distributed/tests/test_nanny.py | 22 +++++++++++++++++++++-
 2 files changed, 30 insertions(+), 9 deletions(-)

diff --git a/distributed/nanny.py b/distributed/nanny.py
index 6b0c0ec9620..c973985db78 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -23,8 +23,7 @@
 from .security import Security
 from .utils import (get_ip, mp_context, silence_logging, json_load_robust,
         PeriodicCallback)
-from .worker import _ncores, run, parse_memory_limit
-
+from .worker import _ncores, run, parse_memory_limit, Worker
 
 logger = logging.getLogger(__name__)
 
@@ -44,7 +43,8 @@ def __init__(self, scheduler_ip=None, scheduler_port=None,
                  name=None, memory_limit='auto', reconnect=True,
                  validate=False, quiet=False, resources=None, silence_logs=None,
                  death_timeout=None, preload=(), preload_argv=[], security=None,
-                 contact_address=None, listen_address=None, **kwargs):
+                 contact_address=None, listen_address=None, worker_class=None,
+                 **kwargs):
         if scheduler_file:
             cfg = json_load_robust(scheduler_file)
             self.scheduler_addr = cfg['address']
@@ -62,6 +62,7 @@ def __init__(self, scheduler_ip=None, scheduler_port=None,
         self.death_timeout = death_timeout
         self.preload = preload
         self.preload_argv = preload_argv
+        self.Worker = Worker if worker_class is None else worker_class
 
         self.contact_address = contact_address
         self.memory_terminate_fraction = dask.config.get('distributed.worker.memory.terminate')
@@ -214,6 +215,7 @@ def instantiate(self, comm=None):
                 worker_start_args=(start_arg,),
                 silence_logs=self.silence_logs,
                 on_exit=self._on_exit,
+                worker=self.Worker
             )
 
         self.auto_restart = True
@@ -320,7 +322,7 @@ def _close(self, comm=None, timeout=5, report=None):
 class WorkerProcess(object):
 
     def __init__(self, worker_args, worker_kwargs, worker_start_args,
-                 silence_logs, on_exit):
+                 silence_logs, on_exit, worker):
         self.status = 'init'
         self.silence_logs = silence_logs
         self.worker_args = worker_args
@@ -328,6 +330,7 @@ def __init__(self, worker_args, worker_kwargs, worker_start_args,
         self.worker_start_args = worker_start_args
         self.on_exit = on_exit
         self.process = None
+        self.Worker = worker
 
         # Initialized when worker is ready
         self.worker_dir = None
@@ -357,7 +360,7 @@ def start(self):
                         silence_logs=self.silence_logs,
                         init_result_q=self.init_result_q,
                         child_stop_q=self.child_stop_q,
-                        uid=uid),
+                        uid=uid, Worker=self.Worker),
         )
         self.process.daemon = True
         self.process.set_exit_callback(self._on_exit)
@@ -485,9 +488,7 @@ def _wait_until_connected(self, uid):
 
     @classmethod
     def _run(cls, worker_args, worker_kwargs, worker_start_args,
-             silence_logs, init_result_q, child_stop_q, uid):  # pragma: no cover
-        from distributed import Worker
-
+             silence_logs, init_result_q, child_stop_q, uid, Worker):  # pragma: no cover
         try:
             from dask.multiprocessing import initialize_worker_process
         except ImportError:   # old Dask version
diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
index 7f0e703b6c4..9a9feabf114 100644
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -13,7 +13,7 @@
 from tornado import gen
 
 import dask
-from distributed import Nanny, rpc, Scheduler
+from distributed import Nanny, rpc, Scheduler, Worker
 from distributed.core import CommClosedError
 from distributed.metrics import time
 from distributed.protocol.pickle import dumps
@@ -145,6 +145,26 @@ def test_close_on_disconnect(s, w):
         assert time() < start + 9
 
 
+class Something(Worker):
+    # a subclass of Worker which is not Worker
+    pass
+
+
+@gen_cluster(client=True, Worker=Nanny)
+def test_nanny_worker_class(c, s, w1, w2):
+    out = yield c._run(lambda dask_worker=None: str(dask_worker.__class__))
+    assert 'Worker' in list(out.values())[0]
+    assert w1.Worker is Worker
+
+
+@gen_cluster(client=True, Worker=Nanny,
+             worker_kwargs={'worker_class': Something})
+def test_nanny_alt_worker_class(c, s, w1, w2):
+    out = yield c._run(lambda dask_worker=None: str(dask_worker.__class__))
+    assert 'Something' in list(out.values())[0]
+    assert w1.Worker is Something
+
+
 @slow
 @gen_cluster(client=False, ncores=[])
 def test_nanny_death_timeout(s):

From 4bb66acae20f0939e221f441b9bf00410358a79b Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 3 Aug 2018 18:56:52 -0400
Subject: [PATCH 0048/1550] Support lack of PollIOLoop in Tornado

---
 distributed/utils.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/distributed/utils.py b/distributed/utils.py
index 81ea0d67d92..9fb7480290f 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -39,7 +39,11 @@
 import toolz
 import tornado
 from tornado import gen
-from tornado.ioloop import IOLoop, PollIOLoop
+from tornado.ioloop import IOLoop
+try:
+    from tornado.ioloop import PollIOLoop
+except ImportError:
+    PollIOLoop = None  # dropped in tornado 6.0
 
 from .compatibility import Queue, PY3, PY2, get_thread_identity, unicode
 from .metrics import time
@@ -234,7 +238,7 @@ def sync(loop, func, *args, **kwargs):
     Run coroutine in loop running in separate thread.
     """
     # Tornado's PollIOLoop doesn't raise when using closed, do it ourselves
-    if ((isinstance(loop, PollIOLoop) and getattr(loop, '_closing', False)) or
+    if PollIOLoop and ((isinstance(loop, PollIOLoop) and getattr(loop, '_closing', False)) or
         (hasattr(loop, 'asyncio_loop') and loop.asyncio_loop._closed)):
         raise RuntimeError("IOLoop is closed")
 

From 3b8f67498adb9cf2965a8b1163b5bbeaa8af4964 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 3 Aug 2018 12:50:24 -0400
Subject: [PATCH 0049/1550] bump version to 1.22.1

---
 distributed/tests/test_worker.py |  2 ++
 distributed/utils.py             |  2 +-
 docs/source/changelog.rst        | 48 ++++++++++++++++++++++++++++++--
 3 files changed, 49 insertions(+), 3 deletions(-)

diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 8c6f0ee752b..77730fac85c 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -1182,6 +1182,8 @@ def test_wait_for_outgoing(c, s, a, b):
     assert 1 / 3 < ratio < 3
 
 
+@pytest.mark.skipif(not sys.platform.startswith('linux'),
+                    reason="Need 127.0.0.2 to mean localhost")
 @gen_cluster(ncores=[('127.0.0.1', 1), ('127.0.0.1', 1), ('127.0.0.2', 1)],
              client=True)
 def test_prefer_gather_from_local_address(c, s, w1, w2, w3):
diff --git a/distributed/utils.py b/distributed/utils.py
index 9fb7480290f..6257a9ba83b 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -239,7 +239,7 @@ def sync(loop, func, *args, **kwargs):
     """
     # Tornado's PollIOLoop doesn't raise when using closed, do it ourselves
     if PollIOLoop and ((isinstance(loop, PollIOLoop) and getattr(loop, '_closing', False)) or
-        (hasattr(loop, 'asyncio_loop') and loop.asyncio_loop._closed)):
+            (hasattr(loop, 'asyncio_loop') and loop.asyncio_loop._closed)):
         raise RuntimeError("IOLoop is closed")
 
     timeout = kwargs.pop('callback_timeout', None)
diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index 5dd96ede6e2..13e0f537294 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,10 +1,48 @@
 Changelog
 =========
 
-1.21.1 - 2018-XX-XX
+X.XX.X - 2018-XX-XX
 -------------------
 
--
+
+1.22.1 - 2018-08-03
+-------------------
+
+-  Add worker_class= keyword to Nanny to support different worker types (:pr:`2147`) `Martin Durant`_
+-  Cleanup intermittent worker failures (:pr:`2152`) (:pr:`2146`) `Matthew Rocklin`_
+-  Fix msgpack PendingDeprecationWarning for encoding='utf-8' (:pr:`2153`) `Olivier Grisel`_
+-  Make bokeh coloring deterministic using hash function (:pr:`2143`) `Matthew Rocklin`_
+-  Allow client to query the task stream plot (:pr:`2122`) `Matthew Rocklin`_
+-  Use PID and counter in thread names (:pr:`2084`) (:pr:`2128`) `Dror Birkman`_
+-  Test that worker restrictions are cleared after cancellation (:pr:`2107`) `Matthew Rocklin`_
+-  Expand resources in graph_to_futures (:pr:`2131`) `Matthew Rocklin`_
+-  Add custom serialization support for pyarrow  (:pr:`2115`) `Dave Hirschfeld`_
+-  Update dask-scheduler cli help text for preload (:pr:`2120`) `Matt Nicolls`_
+-  Added another nested parallelism test (:pr:`1710`) `Tom Augspurger`_
+-  insert newline by default after TextProgressBar (:pr:`1976`) `Phil Tooley`_
+-  Retire workers from scale (:pr:`2104`) `Matthew Rocklin`_
+-  Allow worker to refuse data requests with busy signal (:pr:`2092`) `Matthew Rocklin`_
+-  Don't forget released keys (:pr:`2098`) `Matthew Rocklin`_
+-  Update example for stopping a worker (:pr:`2088`) `John A Kirkham`_
+-  removed hardcoded value of memory terminate fraction from a log message (:pr:`2096`) `Bartosz Marcinkowski`_
+-  Adjust worker doc after change in config file location and treatment (:pr:`2094`) `Aurélien Ponte`_
+-  Prefer gathering data from same host (:pr:`2090`) `Matthew Rocklin`_
+-  Handle exceptions on deserialized comm with text error (:pr:`2093`) `Matthew Rocklin`_
+-  Fix typo in docstring (:pr:`2087`) `Loïc Estève`_
+-  Provide communication context to serialization functions (:pr:`2054`) `Matthew Rocklin`_
+-  Allow `name` to be explicitly passed in publish_dataset (:pr:`1995`) `Marius van Niekerk`_
+-  Avoid accessing Worker.scheduler_delay around yield point (:pr:`2074`) `Matthew Rocklin`_
+-  Support TB and PB in format bytes (:pr:`2072`) `Matthew Rocklin`_
+-  Add test for as_completed for loops in Python 2 (:pr:`2071`) `Matthew Rocklin`_
+-  Allow adaptive to exist without a cluster (:pr:`2064`) `Matthew Rocklin`_
+-  Have worker data transfer wait until recipient acknowledges (:pr:`2052`) `Matthew Rocklin`_
+-  Support async def functions in Client.sync (:pr:`2070`) `Matthew Rocklin`_
+-  Add asynchronous parameter to docstring of LocalCluster `Matthew Rocklin`_
+-  Normalize address before comparison (:pr:`2066`) `Tom Augspurger`_
+-  Use ConnectionPool for Worker.scheduler `Matthew Rocklin`_
+-  Avoid reference cycle in str_graph `Matthew Rocklin`_
+-  Pull data outside of while loop in gather (:pr:`2059`) `Matthew Rocklin`_
+
 
 1.22.0 - 2018-06-14
 -------------------
@@ -683,3 +721,9 @@ significantly without many new features.
 .. _`@bmaisson`: https://github.com/bmaisson
 .. _`Martin Durant`: https://github.com/martindurant
 .. _`Grant Jenks`: https://github.com/grantjenks
+.. _`Dror Birkman`: https://github.com/Dror-LightCyber
+.. _`Dave Hirschfeld`: https://github.com/dhirschfeld
+.. _`Matt Nicolls`: https://github.com/nicolls1
+.. _`Phil Tooley`: https://github.com/ptooley
+.. _`Bartosz Marcinkowski`: https://github.com/bm371613
+.. _`Aurélien Ponte`: https://github.com/apatlpo

From 257b4402275911a0498576aec105dc3e3fa5714c Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 19 Jul 2018 09:05:14 -0400
Subject: [PATCH 0050/1550] add direct_to_workers to Client

---
 distributed/client.py            | 10 +++++++++-
 distributed/tests/test_client.py | 22 ++++++++++++++++++++++
 distributed/worker.py            |  1 +
 3 files changed, 32 insertions(+), 1 deletion(-)

diff --git a/distributed/client.py b/distributed/client.py
index 6b5de8ae3d4..25dd2925538 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -482,6 +482,9 @@ class resembles executors in ``concurrent.futures`` but also allows
     name: string (optional)
         Gives the client a name that will be included in logs generated on
         the scheduler for matters relating to this client
+    direct_to_workers: bool (optional)
+        Can this client connect directly to workers or should it proxy through
+        the scheduler?
     heartbeat_interval: int
         Time in milliseconds between heartbeats to scheduler
 
@@ -514,7 +517,7 @@ def __init__(self, address=None, loop=None, timeout=no_default,
                  security=None, asynchronous=False,
                  name=None, heartbeat_interval=None,
                  serializers=None, deserializers=None,
-                 extensions=DEFAULT_EXTENSIONS,
+                 extensions=DEFAULT_EXTENSIONS, direct_to_workers=False,
                  **kwargs):
         if timeout == no_default:
             timeout = dask.config.get('distributed.comm.timeouts.connect')
@@ -544,6 +547,7 @@ def __init__(self, address=None, loop=None, timeout=no_default,
         if deserializers is None:
             deserializers = serializers
         self._deserializers = deserializers
+        self.direct_to_workers = direct_to_workers
 
         # Communication
         self.security = security or Security()
@@ -1408,6 +1412,8 @@ def _gather(self, futures, errors='raise', direct=None, local_worker=None):
             else:
                 if w.scheduler.address == self.scheduler.address:
                     direct = True
+        if direct is None:
+            direct = self.direct_to_workers
 
         @gen.coroutine
         def wait(k):
@@ -1610,6 +1616,8 @@ def _scatter(self, data, workers=None, broadcast=False, direct=None,
             else:
                 if w.scheduler.address == self.scheduler.address:
                     direct = True
+        if direct is None:
+            direct = self.direct_to_workers
 
         if local_worker:  # running within task
             local_worker.update_data(data=data, report=False)
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 6ab903ffdee..83ce26ec734 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -5440,5 +5440,27 @@ def test_no_threads_lingering():
     assert threading.active_count() < 30, list(active.values())
 
 
+@gen_cluster()
+def test_direct_async(s, a, b):
+    c = yield Client(s.address, asynchronous=True, direct_to_workers=True)
+    assert c.direct_to_workers
+    yield c.close()
+
+    c = yield Client(s.address, asynchronous=True, direct_to_workers=False)
+    assert not c.direct_to_workers
+    yield c.close()
+
+
+def test_direct_sync(loop):
+    with cluster() as (s, [a, b]):
+        with Client(s['address'], loop=loop) as c:
+            assert not c.direct_to_workers
+
+            def f():
+                return get_client().direct_to_workers
+
+            assert c.submit(f).result()
+
+
 if sys.version_info >= (3, 5):
     from distributed.tests.py3_test_client import *  # noqa F401
diff --git a/distributed/worker.py b/distributed/worker.py
index 48e4bb8a2e8..ee0a6628046 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -2605,6 +2605,7 @@ def _get_client(self, timeout=3):
                                   security=self.security,
                                   set_as_default=True,
                                   asynchronous=asynchronous,
+                                  direct_to_workers=True,
                                   name='worker',
                                   timeout=timeout)
             if not asynchronous:

From 0fc888cfa63920c8782423f905a713de96ed4324 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 19 Jul 2018 09:10:24 -0400
Subject: [PATCH 0051/1550] add Scheduler.proxy to workers

---
 distributed/core.py              | 2 +-
 distributed/scheduler.py         | 8 ++++++++
 distributed/tests/test_client.py | 6 ++++++
 distributed/worker.py            | 2 +-
 4 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/distributed/core.py b/distributed/core.py
index 05b901cf818..56360c7ce4d 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -233,7 +233,7 @@ def port(self):
             _, self._port = get_address_host_port(self.address)
         return self._port
 
-    def identity(self, comm):
+    def identity(self, comm=None):
         return {'type': type(self).__name__, 'id': self.id}
 
     def listen(self, port_or_addr=None, listen_args=None):
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index a2412068c7b..4ef146d06c6 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -903,6 +903,7 @@ def __init__(
             'feed': self.feed,
             'terminate': self.close,
             'broadcast': self.broadcast,
+            'proxy': self.proxy,
             'ncores': self.get_ncores,
             'has_what': self.get_has_what,
             'who_has': self.get_who_has,
@@ -2323,6 +2324,13 @@ def send_message(addr):
 
         raise Return(dict(zip(workers, results)))
 
+    @gen.coroutine
+    def proxy(self, comm=None, msg=None, worker=None, serializers=None):
+        """ Proxy a communication through the scheduler to some other worker """
+        d = yield self.broadcast(comm=comm, msg=msg, workers=[worker],
+                                 serializers=serializers)
+        raise gen.Return(d[worker])
+
     @gen.coroutine
     def rebalance(self, comm=None, keys=None, workers=None):
         """ Rebalance keys so that each worker stores roughly equal bytes
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 83ce26ec734..0e099ccb678 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -2200,6 +2200,12 @@ def test_broadcast(loop):
                 b['address']: {x.key, y.key}}
 
 
+@gen_cluster(client=True)
+def test_proxy(c, s, a, b):
+    msg = yield c.scheduler.proxy(msg={'op': 'identity'}, worker=a.address)
+    assert msg['id'] == a.identity()['id']
+
+
 @gen_cluster(client=True)
 def test__cancel(c, s, a, b):
     x = c.submit(slowinc, 1)
diff --git a/distributed/worker.py b/distributed/worker.py
index ee0a6628046..6d98ebc4659 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -450,7 +450,7 @@ def _start(self, addr_or_port=0):
     def start(self, port=0):
         self.loop.add_callback(self._start, port)
 
-    def identity(self, comm):
+    def identity(self, comm=None):
         return {'type': type(self).__name__,
                 'id': self.id,
                 'scheduler': self.scheduler.address,

From b16ee25506fd20ec5daa7d77e338e2725e778562 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 18 Jul 2018 15:02:25 -0400
Subject: [PATCH 0052/1550] Implement Actors

---
 distributed/__init__.py          |   1 +
 distributed/actor.py             | 209 +++++++++++++
 distributed/client.py            |  45 ++-
 distributed/core.py              |  12 +-
 distributed/scheduler.py         |  76 ++++-
 distributed/tests/test_actor.py  | 521 +++++++++++++++++++++++++++++++
 distributed/tests/test_worker.py |  15 -
 distributed/utils.py             |   8 +
 distributed/utils_test.py        |  11 +-
 distributed/worker.py            | 150 +++++++--
 docs/source/actors.rst           | 235 ++++++++++++++
 docs/source/index.rst            |   1 +
 12 files changed, 1198 insertions(+), 86 deletions(-)
 create mode 100644 distributed/actor.py
 create mode 100644 distributed/tests/test_actor.py
 create mode 100644 docs/source/actors.rst

diff --git a/distributed/__init__.py b/distributed/__init__.py
index 71e71a79143..ac324592dd2 100644
--- a/distributed/__init__.py
+++ b/distributed/__init__.py
@@ -2,6 +2,7 @@
 
 from . import config
 from dask.config import config
+from .actor import Actor, ActorFuture
 from .core import connect, rpc
 from .deploy import LocalCluster, Adaptive
 from .diagnostics import progress
diff --git a/distributed/actor.py b/distributed/actor.py
new file mode 100644
index 00000000000..b97c79a6041
--- /dev/null
+++ b/distributed/actor.py
@@ -0,0 +1,209 @@
+from tornado import gen
+import functools
+
+from .client import Future, default_client
+from .compatibility import get_thread_identity, Queue
+from .protocol import to_serialize
+from .utils import sync
+from .utils_comm import WrappedKey
+from .worker import get_worker
+
+
+class Actor(WrappedKey):
+    """ Controls an object on a remote worker
+
+    An actor allows remote control of a stateful object living on a remote
+    worker.  Method calls on this object trigger operations on the remote
+    object and return ActorFutures on which we can block to get results.
+
+    Examples
+    --------
+    >>> class Counter:
+    ...    def __init__(self):
+    ...        self.n = 0
+    ...    def increment(self):
+    ...        self.n += 1
+    ...        return self.n
+
+    >>> from dask.distributed import Client
+    >>> client = Client()
+
+    You can create an actor by submitting a class with the keyword
+    ``actor=True``.
+
+    >>> future = client.submit(Counter, actor=True)
+    >>> counter = future.result()
+    >>> counter
+    <Actor: Counter, key=Counter-1234abcd>
+
+    Calling methods on this object immediately returns deferred ``ActorFuture``
+    objects.  You can call ``.result()`` on these objects to block and get the
+    result of the function call.
+
+    >>> future = counter.increment()
+    >>> future.result()
+    1
+    >>> future = counter.increment()
+    >>> future.result()
+    2
+    """
+    def __init__(self, cls, address, key, worker=None):
+        self._cls = cls
+        self._address = address
+        self.key = key
+        self._future = None
+        if worker:
+            self._worker = worker
+            self._client = None
+        else:
+            try:
+                self._worker = get_worker()
+            except ValueError:
+                self._worker = None
+            try:
+                self._client = default_client()
+                self._future = Future(key)
+            except ValueError:
+                self._client = None
+
+    def __repr__(self):
+        return '<Actor: %s, key=%s>' % (self._cls.__name__, self.key)
+
+    def __reduce__(self):
+        return (Actor, (self._cls, self._address, self.key))
+
+    @property
+    def _io_loop(self):
+        if self._worker:
+            return self._worker.io_loop
+        else:
+            return self._client.io_loop
+
+    @property
+    def _scheduler_rpc(self):
+        if self._worker:
+            return self._worker.scheduler
+        else:
+            return self._client.scheduler
+
+    @property
+    def _worker_rpc(self):
+        if self._worker:
+            return self._worker.rpc(self._address)
+        else:
+            if self._client.direct_to_workers:
+                return self._client.rpc(self._address)
+            else:
+                return ProxyRPC(self._client.scheduler, self._address)
+
+    @property
+    def _asynchronous(self):
+        if self._client:
+            return self._client.asynchronous
+        else:
+            return get_thread_identity() == self._worker.thread_id
+
+    def _sync(self, func, *args, **kwargs):
+        if self._client:
+            return self._client.sync(func, *args, **kwargs)
+        else:
+            # TODO support sync operation by checking against thread ident of loop
+            return sync(self._worker.loop, func, *args, **kwargs)
+
+    def __dir__(self):
+        o = set(dir(type(self)))
+        o.update(attr for attr in dir(self._cls) if not attr.startswith('_'))
+        return sorted(o)
+
+    def __getattr__(self, key):
+        attr = getattr(self._cls, key)
+
+        if self._future and not self._future.status == 'finished':
+            raise ValueError("Worker holding Actor was lost")
+
+        if callable(attr):
+            @functools.wraps(attr)
+            def func(*args, **kwargs):
+                @gen.coroutine
+                def run_actor_function_on_worker():
+                    try:
+                        result = yield self._worker_rpc.actor_execute(
+                            function=key,
+                            actor=self.key,
+                            args=[to_serialize(arg) for arg in args],
+                            kwargs={k: to_serialize(v) for k, v in kwargs.items()},
+                        )
+                    except OSError:
+                        if self._future:
+                            yield self._future
+                        else:
+                            raise OSError("Unable to contact Actor's worker")
+                    raise gen.Return(result['result'])
+
+                if self._asynchronous:
+                    return run_actor_function_on_worker()
+                else:
+                    # TODO: this mechanism is error prone
+                    # we should endeavor to make dask's standard code work here
+                    q = Queue()
+
+                    @gen.coroutine
+                    def wait_then_add_to_queue():
+                        x = yield run_actor_function_on_worker()
+                        q.put(x)
+                    self._io_loop.add_callback(wait_then_add_to_queue)
+
+                    return ActorFuture(q, self._io_loop)
+            return func
+
+        else:
+            @gen.coroutine
+            def get_actor_attribute_from_worker():
+                x = yield self._worker_rpc.actor_attribute(attribute=key, actor=self.key)
+                raise gen.Return(x['result'])
+
+            return self._sync(get_actor_attribute_from_worker)
+
+
+class ProxyRPC(object):
+    """
+    An rpc-like object that uses the scheduler's rpc to connect to a worker
+    """
+    def __init__(self, rpc, address):
+        self.rpc = rpc
+        self._address = address
+
+    def __getattr__(self, key):
+        @gen.coroutine
+        def func(**msg):
+            msg['op'] = key
+            result = yield self.rpc.proxy(worker=self._address, msg=msg)
+            raise gen.Return(result)
+
+        return func
+
+
+class ActorFuture(object):
+    """ Future to an actor's method call
+
+    Whenever you call a method on an Actor you get an ActorFuture immediately
+    while the computation happens in the background.  You can call ``.result``
+    to block and collect the full result
+
+    See Also
+    --------
+    Actor
+    """
+    def __init__(self, q, io_loop):
+        self.q = q
+        self.io_loop = io_loop
+
+    def result(self, timeout=None):
+        try:
+            return self._cached_result
+        except AttributeError:
+            self._cached_result = self.q.get(timeout=timeout)
+            return self._cached_result
+
+    def __repr__(self):
+        return '<ActorFuture>'
diff --git a/distributed/client.py b/distributed/client.py
index 25dd2925538..f136b91bcd6 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -401,19 +401,7 @@ def lose(self):
         self._get_event().clear()
 
     def set_error(self, exception, traceback):
-        if isinstance(exception, bytes):
-            try:
-                exception = loads(exception)
-            except TypeError:
-                exception = Exception("Undeserializable exception", exception)
-        if traceback:
-            if isinstance(traceback, bytes):
-                try:
-                    traceback = loads(traceback)
-                except (TypeError, AttributeError):
-                    traceback = None
-        else:
-            traceback = None
+        _, exception, traceback = clean_exception(exception, traceback)
 
         self.status = 'error'
         self.exception = exception
@@ -1201,13 +1189,14 @@ def submit(self, func, *args, **kwargs):
             raise TypeError("First input to submit must be a callable function")
 
         key = kwargs.pop('key', None)
-        pure = kwargs.pop('pure', True)
         workers = kwargs.pop('workers', None)
         resources = kwargs.pop('resources', None)
         retries = kwargs.pop('retries', None)
         priority = kwargs.pop('priority', 0)
         fifo_timeout = kwargs.pop('fifo_timeout', '100ms')
         allow_other_workers = kwargs.pop('allow_other_workers', False)
+        actor = kwargs.pop('actor', kwargs.pop('actors', False))
+        pure = kwargs.pop('pure', not actor)
 
         if allow_other_workers not in (True, False, None):
             raise TypeError("allow_other_workers= must be True or False")
@@ -1246,7 +1235,8 @@ def submit(self, func, *args, **kwargs):
                                          user_priority=priority,
                                          resources={skey: resources} if resources else None,
                                          retries=retries,
-                                         fifo_timeout=fifo_timeout)
+                                         fifo_timeout=fifo_timeout,
+                                         actors=actor)
 
         logger.debug("Submit %s(...), %s", funcname(func), key)
 
@@ -1328,13 +1318,14 @@ def map(self, func, *iterables, **kwargs):
 
         key = kwargs.pop('key', None)
         key = key or funcname(func)
-        pure = kwargs.pop('pure', True)
         workers = kwargs.pop('workers', None)
         retries = kwargs.pop('retries', None)
         resources = kwargs.pop('resources', None)
         user_priority = kwargs.pop('priority', 0)
         allow_other_workers = kwargs.pop('allow_other_workers', False)
         fifo_timeout = kwargs.pop('fifo_timeout', '100ms')
+        actor = kwargs.pop('actor', kwargs.pop('actors', False))
+        pure = kwargs.pop('pure', not actor)
 
         if allow_other_workers and workers is None:
             raise ValueError("Only use allow_other_workers= if using workers=")
@@ -1392,7 +1383,8 @@ def map(self, func, *iterables, **kwargs):
                                          resources=resources,
                                          retries=retries,
                                          user_priority=user_priority,
-                                         fifo_timeout=fifo_timeout)
+                                         fifo_timeout=fifo_timeout,
+                                         actors=actor)
         logger.debug("map(%s, ...)", funcname(func))
 
         return [futures[tokey(k)] for k in keys]
@@ -2081,7 +2073,7 @@ def run_coroutine(self, function, *args, **kwargs):
     def _graph_to_futures(self, dsk, keys, restrictions=None,
                           loose_restrictions=None, priority=None,
                           user_priority=0, resources=None, retries=None,
-                          fifo_timeout=0):
+                          fifo_timeout=0, actors=None):
         with self._lock:
             if resources:
                 resources = self._expand_resources(resources,
@@ -2091,6 +2083,9 @@ def _graph_to_futures(self, dsk, keys, restrictions=None,
                 retries = self._expand_retries(retries,
                                                all_keys=itertools.chain(dsk, keys))
 
+            if actors is not None and actors is not True and actors is not False:
+                actors = list(self._expand_key(actors))
+
             keyset = set(keys)
             flatkeys = list(map(tokey, keys))
             futures = {key: Future(key, self, inform=False) for key in keyset}
@@ -2145,7 +2140,8 @@ def _graph_to_futures(self, dsk, keys, restrictions=None,
                                      'resources': resources,
                                      'submitting_task': getattr(thread_state, 'key', None),
                                      'retries': retries,
-                                     'fifo_timeout': fifo_timeout})
+                                     'fifo_timeout': fifo_timeout,
+                                     'actors': actors})
             return futures
 
     def get(self, dsk, keys, restrictions=None, loose_restrictions=None,
@@ -2265,7 +2261,8 @@ def normalize_collection(self, collection):
 
     def compute(self, collections, sync=False, optimize_graph=True,
                 workers=None, allow_other_workers=False, resources=None,
-                retries=0, priority=0, fifo_timeout='60s', **kwargs):
+                retries=0, priority=0, fifo_timeout='60s', actors=None,
+                **kwargs):
         """ Compute dask collections on cluster
 
         Parameters
@@ -2358,7 +2355,8 @@ def compute(self, collections, sync=False, optimize_graph=True,
                                               resources=resources,
                                               retries=retries,
                                               user_priority=priority,
-                                              fifo_timeout=fifo_timeout)
+                                              fifo_timeout=fifo_timeout,
+                                              actors=actors)
 
         i = 0
         futures = []
@@ -2381,7 +2379,7 @@ def compute(self, collections, sync=False, optimize_graph=True,
 
     def persist(self, collections, optimize_graph=True, workers=None,
                 allow_other_workers=None, resources=None, retries=None,
-                priority=0, fifo_timeout='60s', **kwargs):
+                priority=0, fifo_timeout='60s', actors=None, **kwargs):
         """ Persist dask collections on cluster
 
         Starts computation of the collection on the cluster in the background.
@@ -2450,7 +2448,8 @@ def persist(self, collections, optimize_graph=True, workers=None,
                                          resources=resources,
                                          retries=retries,
                                          user_priority=priority,
-                                         fifo_timeout=fifo_timeout)
+                                         fifo_timeout=fifo_timeout,
+                                         actors=actors)
 
         postpersists = [c.__dask_postpersist__() for c in collections]
         result = [func({k: futures[k] for k in flatten(c.__dask_keys__())}, *args)
diff --git a/distributed/core.py b/distributed/core.py
index 56360c7ce4d..aa8b77984d7 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -851,9 +851,17 @@ def clean_exception(exception, traceback, **kwargs):
     error_message: create and serialize errors into message
     """
     if isinstance(exception, bytes):
-        exception = protocol.pickle.loads(exception)
+        try:
+            exception = protocol.pickle.loads(exception)
+        except Exception:
+            exception = Exception(exception)
+    elif isinstance(exception, str):
+        exception = Exception(exception)
     if isinstance(traceback, bytes):
-        traceback = protocol.pickle.loads(traceback)
+        try:
+            traceback = protocol.pickle.loads(traceback)
+        except (TypeError, AttributeError):
+            traceback = None
     elif isinstance(traceback, string_types):
         traceback = None  # happens if the traceback failed serializing
     return type(exception), exception, traceback
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 4ef146d06c6..034ea3eeadb 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -182,6 +182,12 @@ class WorkerState(object):
        The last time we received a heartbeat from this worker, in local
        scheduler time.
 
+    .. attribute:: actors: {TaskState}
+
+       A set of all TaskStates on this worker that are actors.  This only
+       includes those actors whose state actually lives on this worker, not
+       actors to which this worker has a reference.
+
     """
     # XXX need a state field to signal active/removed?
 
@@ -200,6 +206,7 @@ class WorkerState(object):
         'used_resources',
         'status',
         'last_seen',
+        'actors',
     )
 
     def __init__(self, worker, ncores, memory_limit, name=None):
@@ -214,6 +221,7 @@ def __init__(self, worker, ncores, memory_limit, name=None):
         self.resources = {}
         self.used_resources = {}
         self.last_seen = 0
+        self.actors = set()
 
         self.info = {'name': name,
                      'memory_limit': memory_limit,
@@ -453,10 +461,13 @@ class TaskState(object):
        into the "processing" state and be sent for execution to another
        connected worker.
 
-    """
+    .. attribute: actor: bool
 
+       Whether or not this task is an Actor.
+    """
     __slots__ = (
         # === General description ===
+        'actor',
         # Key name
         'key',
         # Key prefix (see key_split())
@@ -518,6 +529,7 @@ def __init__(self, key, run_spec):
         self.worker_restrictions = None
         self.resource_restrictions = None
         self.loose_restrictions = False
+        self.actor = None
 
     def get_nbytes(self):
         nbytes = self.nbytes
@@ -1301,7 +1313,7 @@ def update_graph(self, client=None, tasks=None, keys=None,
                      dependencies=None, restrictions=None, priority=None,
                      loose_restrictions=None, resources=None,
                      submitting_task=None, retries=None, user_priority=0,
-                     fifo_timeout=0):
+                     actors=None, fifo_timeout=0):
         """
         Add new computations to the internal dask graph
 
@@ -1401,6 +1413,12 @@ def update_graph(self, client=None, tasks=None, keys=None,
         if isinstance(user_priority, Number):
             user_priority = {k: user_priority for k in tasks}
 
+        # Add actors
+        if actors is True:
+            actors = list(keys)
+        for actor in actors or []:
+            self.tasks[actor].actor = True
+
         priority = priority or dask.order.order(tasks)  # TODO: define order wrt old graph
 
         if submitting_task:  # sub-tasks get better priority than parent tasks
@@ -1965,6 +1983,8 @@ def send_task_to_worker(self, worker, key):
                    'duration': self.get_task_duration(ts)}
             if ts.resource_restrictions:
                 msg['resource_restrictions'] = ts.resource_restrictions
+            if ts.actor:
+                msg['actor'] = True
 
             deps = ts.dependencies
             if deps:
@@ -3206,6 +3226,9 @@ def transition_waiting_processing(self, key):
             self.check_idle_saturated(ws)
             self.n_tasks += 1
 
+            if ts.actor:
+                ws.actors.add(ts)
+
             # logger.debug("Send job to worker: %s, %s", worker, key)
 
             self.send_task_to_worker(worker, key)
@@ -3346,6 +3369,14 @@ def transition_memory_released(self, key, safe=False):
                 if safe:
                     assert not ts.waiters
 
+            if ts.actor:
+                for ws in ts.who_has:
+                    ws.actors.discard(ts)
+                if ts.who_wants:
+                    ts.exception_blame = ts
+                    ts.exception = "Worker holding Actor was lost"
+                    return {ts.key: 'erred'}  # don't try to recreate
+
             recommendations = OrderedDict()
 
             for dts in ts.waiters:
@@ -3509,6 +3540,10 @@ def transition_processing_erred(self, key, cause=None, exception=None,
                 assert not ts.who_has
                 assert not ts.waiting_on
 
+            if ts.actor:
+                ws = ts.processing_on
+                ws.actors.remove(ts)
+
             self._remove_from_processing(ts)
 
             if exception is not None:
@@ -3650,6 +3685,11 @@ def transition_memory_forgotten(self, key):
                     assert 0, (ts,)
 
             recommendations = {}
+
+            if ts.actor:
+                for ws in ts.who_has:
+                    ws.actors.discard(ts)
+
             self._propagate_forgotten(ts, recommendations)
 
             self.report_on_key(ts=ts)
@@ -3991,7 +4031,11 @@ def worker_objective(self, ts, ws):
                           if ws not in dts.who_has])
         stack_time = ws.occupancy / ws.ncores
         start_time = comm_bytes / BANDWIDTH + stack_time
-        return (start_time, ws.nbytes)
+
+        if ts.actor:
+            return (len(ws.actors), start_time, ws.nbytes)
+        else:
+            return (start_time, ws.nbytes)
 
     @gen.coroutine
     def get_profile(self, comm=None, workers=None, merge_workers=True,
@@ -4161,8 +4205,11 @@ def decide_worker(ts, all_workers, valid_workers, objective):
     """
     deps = ts.dependencies
     assert all(dts.who_has for dts in deps)
-    candidates = frequencies([ws for dts in deps
-                              for ws in dts.who_has])
+    if ts.actor:
+        candidates = all_workers
+    else:
+        candidates = frequencies([ws for dts in deps
+                                  for ws in dts.who_has])
     if valid_workers is True:
         if not candidates:
             candidates = all_workers
@@ -4239,6 +4286,21 @@ def validate_task_state(ts):
             assert ts in cs.wants_what, \
                 ("not in who_wants' wants_what", str(ts), str(cs), str(cs.wants_what))
 
+    if ts.actor:
+        if ts.state == 'memory':
+            assert sum([ts in ws.actors for ws in ts.who_has]) == 1
+        if ts.state == 'processing':
+            assert ts in ts.processing_on.actors
+
+
+def validate_worker_state(ws):
+    for ts in ws.has_what:
+        assert ws in ts.who_has, \
+            ("not in has_what' who_has", str(ws), str(ts), str(ts.who_has))
+
+    for ts in ws.actors:
+        assert ts.state in ('memory', 'processing')
+
 
 def validate_state(tasks, workers, clients):
     """
@@ -4251,9 +4313,7 @@ def validate_state(tasks, workers, clients):
         validate_task_state(ts)
 
     for ws in workers.values():
-        for ts in ws.has_what:
-            assert ws in ts.who_has, \
-                ("not in has_what' who_has", str(ws), str(ts), str(ts.who_has))
+        validate_worker_state(ws)
 
     for cs in clients.values():
         for ts in cs.wants_what:
diff --git a/distributed/tests/test_actor.py b/distributed/tests/test_actor.py
new file mode 100644
index 00000000000..580b884b554
--- /dev/null
+++ b/distributed/tests/test_actor.py
@@ -0,0 +1,521 @@
+import operator
+from time import sleep
+from tornado import gen
+
+import pytest
+
+import dask
+from distributed import Actor, ActorFuture, Client, Future, wait, Nanny
+from distributed.utils_test import gen_cluster, cluster
+from distributed.utils_test import loop  # noqa: F401
+from distributed.metrics import time
+
+
+class Counter(object):
+    n = 0
+
+    def __init__(self):
+        self.n = 0
+
+    def increment(self):
+        self.n += 1
+        return self.n
+
+    def add(self, x):
+        self.n += x
+        return self.n
+
+
+class List(object):
+    L = []
+
+    def __init__(self, dummy=None):
+        self.L = []
+
+    def append(self, x):
+        self.L.append(x)
+
+
+class ParameterServer(object):
+    def __init__(self):
+        self.data = {}
+
+    def put(self, key, value):
+        self.data[key] = value
+
+    def get(self, key):
+        return self.data[key]
+
+
+@pytest.mark.parametrize('direct_to_workers', [True, False])
+def test_client_actions(direct_to_workers):
+
+    @gen_cluster(client=True)
+    def test(c, s, a, b):
+        c = yield Client(s.address, asynchronous=True,
+                         direct_to_workers=direct_to_workers)
+
+        counter = c.submit(Counter, workers=[a.address], actor=True)
+        assert isinstance(counter, Future)
+        counter = yield counter
+        assert counter._address
+        assert hasattr(counter, 'increment')
+        assert hasattr(counter, 'add')
+        assert hasattr(counter, 'n')
+
+        n = yield counter.n
+        assert n == 0
+
+        assert counter._address == a.address
+
+        assert isinstance(a.actors[counter.key], Counter)
+        assert s.tasks[counter.key].actor
+
+        yield [counter.increment(), counter.increment()]
+
+        n = yield counter.n
+        assert n == 2
+
+        counter.add(10)
+        while (yield counter.n) != 10 + 2:
+            n = yield counter.n
+            yield gen.sleep(0.01)
+
+        yield c.close()
+
+    test()
+
+
+@pytest.mark.parametrize('separate_thread', [False, True])
+def test_worker_actions(separate_thread):
+
+    @gen_cluster(client=True)
+    def test(c, s, a, b):
+        counter = c.submit(Counter, workers=[a.address], actor=True)
+        a_address = a.address
+
+        def f(counter):
+            start = counter.n
+
+            assert type(counter) is Actor
+            assert counter._address == a_address
+
+            future = counter.increment(separate_thread=separate_thread)
+            assert isinstance(future, ActorFuture)
+            assert "Future" in type(future).__name__
+            end = future.result(timeout=1)
+            assert end > start
+
+        futures = [c.submit(f, counter, pure=False) for _ in range(10)]
+        yield futures
+
+        counter = yield counter
+        assert (yield counter.n) == 10
+
+    test()
+
+
+@gen_cluster(client=True)
+def test_Actor(c, s, a, b):
+    counter = yield c.submit(Counter, actor=True)
+
+    assert counter._cls == Counter
+
+    assert hasattr(counter, 'n')
+    assert hasattr(counter, 'increment')
+    assert hasattr(counter, 'add')
+
+    assert not hasattr(counter, 'abc')
+
+
+@pytest.mark.xfail(reason="Tornado can pass things out of order" +
+        "Should rely on sending small messages rather than rpc")
+@gen_cluster(client=True)
+def test_linear_access(c, s, a, b):
+    start = time()
+    future = c.submit(sleep, 0.2)
+    actor = c.submit(List, actor=True, dummy=future)
+    actor = yield actor
+
+    for i in range(100):
+        actor.append(i)
+
+    while True:
+        yield gen.sleep(0.1)
+        L = yield actor.L
+        if len(L) == 100:
+            break
+
+    L = yield actor.L
+    stop = time()
+    assert L == tuple(range(100))
+
+    assert stop - start > 0.2
+
+
+@gen_cluster(client=True)
+def test_exceptions_create(c, s, a, b):
+    class Foo(object):
+        x = 0
+
+        def __init__(self):
+            raise ValueError('bar')
+
+    with pytest.raises(ValueError) as info:
+        future = yield c.submit(Foo, actor=True)
+
+    assert "bar" in str(info.value)
+
+
+@gen_cluster(client=True)
+def test_exceptions_method(c, s, a, b):
+    class Foo(object):
+        def throw(self):
+            1 / 0
+
+    foo = yield c.submit(Foo, actor=True)
+    with pytest.raises(ZeroDivisionError):
+        yield foo.throw()
+
+
+@gen_cluster(client=True)
+def test_gc(c, s, a, b):
+    actor = c.submit(Counter, actor=True)
+    yield wait(actor)
+    del actor
+
+    while a.actors or b.actors:
+        yield gen.sleep(0.01)
+
+
+@gen_cluster(client=True)
+def test_track_dependencies(c, s, a, b):
+    actor = c.submit(Counter, actor=True)
+    yield wait(actor)
+    x = c.submit(sleep, 0.5)
+    y = c.submit(lambda x, y: x, x, actor)
+    del actor
+
+    yield gen.sleep(0.3)
+
+    assert a.actors or b.actors
+
+
+@gen_cluster(client=True)
+def test_future(c, s, a, b):
+    counter = c.submit(Counter, actor=True, workers=[a.address])
+    assert isinstance(counter, Future)
+    yield wait(counter)
+    assert isinstance(a.actors[counter.key], Counter)
+
+    counter = yield counter
+    assert isinstance(counter, Actor)
+    assert counter._address
+
+    yield gen.sleep(0.1)
+    assert counter.key in c.futures  # don't lose future
+
+
+@gen_cluster(client=True)
+def test_future_dependencies(c, s, a, b):
+    counter = c.submit(Counter, actor=True, workers=[a.address])
+
+    def f(a):
+        assert isinstance(a, Actor)
+        assert a._cls == Counter
+
+    x = c.submit(f, counter, workers=[b.address])
+    yield x
+
+    assert {ts.key for ts in s.tasks[x.key].dependencies} == {counter.key}
+    assert {ts.key for ts in s.tasks[counter.key].dependents} == {x.key}
+
+    y = c.submit(f, counter, workers=[a.address], pure=False)
+    yield y
+
+    assert {ts.key for ts in s.tasks[y.key].dependencies} == {counter.key}
+    assert {ts.key for ts in s.tasks[counter.key].dependents} == {x.key, y.key}
+
+
+def test_sync(loop):
+    with cluster() as (s, [a, b]):
+        with Client(s['address'], loop=loop) as c:
+            counter = c.submit(Counter, actor=True)
+            counter = counter.result()
+
+            assert counter.n == 0
+
+            future = counter.increment()
+            n = future.result()
+            assert n == 1
+            assert counter.n == 1
+
+            assert future.result() == future.result()
+
+            assert 'ActorFuture' in repr(future)
+            assert 'distributed.actor' not in repr(future)
+
+
+@gen_cluster(client=True, config={'distributed.comm.timeouts.connect': '1s'})
+def test_failed_worker(c, s, a, b):
+    future = c.submit(Counter, actor=True, workers=[a.address])
+    yield wait(future)
+    counter = yield future
+
+    yield a._close()
+
+    with pytest.raises(Exception) as info:
+        yield counter.increment()
+
+    assert "actor" in str(info.value).lower()
+    assert "worker" in str(info.value).lower()
+    assert "lost" in str(info.value).lower()
+
+
+@gen_cluster(client=True)
+def bench(c, s, a, b):
+    counter = yield c.submit(Counter, actor=True)
+
+    for i in range(1000):
+        yield counter.increment()
+
+
+@gen_cluster(client=True)
+def test_numpy_roundtrip(c, s, a, b):
+    np = pytest.importorskip('numpy')
+
+    server = yield c.submit(ParameterServer, actor=True)
+
+    x = np.random.random(1000)
+    yield server.put('x', x)
+
+    y = yield server.get('x')
+
+    assert (x == y).all()
+
+
+@gen_cluster(client=True)
+def test_numpy_roundtrip_getattr(c, s, a, b):
+    np = pytest.importorskip('numpy')
+
+    counter = yield c.submit(Counter, actor=True)
+
+    x = np.random.random(1000)
+
+    yield counter.add(x)
+
+    y = yield counter.n
+
+    assert (x == y).all()
+
+
+@gen_cluster(client=True)
+def test_repr(c, s, a, b):
+    counter = yield c.submit(Counter, actor=True)
+
+    assert 'Counter' in repr(counter)
+    assert 'Actor' in repr(counter)
+    assert counter.key in repr(counter)
+    assert 'distributed.actor' not in repr(counter)
+
+
+@gen_cluster(client=True)
+def test_dir(c, s, a, b):
+    counter = yield c.submit(Counter, actor=True)
+
+    d = set(dir(counter))
+
+    for attr in dir(Counter):
+        if not attr.startswith('_'):
+            assert attr in d
+
+
+@gen_cluster(client=True)
+def test_many_computations(c, s, a, b):
+    counter = yield c.submit(Counter, actor=True)
+
+    def add(n, counter):
+        for i in range(n):
+            counter.increment().result()
+
+    futures = c.map(add, range(10), counter=counter)
+    done = c.submit(lambda x: None, futures)
+
+    while not done.done():
+        assert len(s.processing) <= a.ncores + b.ncores
+        yield gen.sleep(0.01)
+
+    yield done
+
+
+@gen_cluster(client=True, ncores=[('127.0.0.1', 5)] * 2)
+def test_thread_safety(c, s, a, b):
+    class Unsafe(object):
+        def __init__(self):
+            self.n = 0
+
+        def f(self):
+            assert self.n == 0
+            self.n += 1
+
+            for i in range(20):
+                sleep(0.002)
+                assert self.n == 1
+            self.n = 0
+
+    unsafe = yield c.submit(Unsafe, actor=True)
+
+    futures = [unsafe.f() for i in range(10)]
+    yield futures
+
+
+@gen_cluster(client=True)
+def test_Actors_create_dependencies(c, s, a, b):
+    counter = yield c.submit(Counter, actor=True)
+    future = c.submit(lambda x: None, counter)
+    yield wait(future)
+    assert s.tasks[future.key].dependencies == {s.tasks[counter.key]}
+
+
+@gen_cluster(client=True)
+def test_load_balance(c, s, a, b):
+    class Foo(object):
+        def __init__(self, x):
+            pass
+
+    b = c.submit(operator.mul, 'b', 1000000)
+    yield wait(b)
+    [ws] = s.tasks[b.key].who_has
+
+    x = yield c.submit(Foo, b, actor=True)
+    y = yield c.submit(Foo, b, actor=True)
+    assert x.key != y.key  # actors assumed not pure
+
+    assert s.tasks[x.key].who_has == {ws}  # first went to best match
+    assert s.tasks[x.key].who_has != s.tasks[y.key].who_has  # second load balanced
+
+
+@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 5)
+def test_load_balance_map(c, s, *workers):
+    class Foo(object):
+        def __init__(self, x, y=None):
+            pass
+
+    b = c.submit(operator.mul, 'b', 1000000)
+    yield wait(b)
+
+    actors = c.map(Foo, range(10), y=b, actor=True)
+    yield wait(actors)
+
+    assert all(len(w.actors) == 2 for w in workers)
+
+
+@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 4, Worker=Nanny)
+def bench_param_server(c, s, *workers):
+    import dask.array as da
+    import numpy as np
+    x = da.random.random((500000, 1000), chunks=(1000, 1000))
+    x = x.persist()
+    yield wait(x)
+
+    class ParameterServer:
+        data = None
+
+        def __init__(self, n):
+            self.data = np.random.random(n)
+
+        def update(self, x):
+            self.data += x
+            self.data /= 2
+
+        def get_data(self):
+            return self.data
+
+    def f(block, ps=None):
+        start = time()
+        params = ps.get_data(separate_thread=False).result()
+        stop = time()
+        update = (block - params).mean(axis=0)
+        ps.update(update, separate_thread=False)
+        print(format_time(stop - start))
+        return np.array([[stop - start]])
+
+    from distributed.utils import format_time
+    start = time()
+    ps = yield c.submit(ParameterServer, x.shape[1], actor=True)
+    y = x.map_blocks(f, ps=ps, dtype=x.dtype)
+    # result = yield c.compute(y.mean())
+    yield wait(y.persist())
+    end = time()
+    print(format_time(end - start))
+
+
+@gen_cluster(client=True)
+def test_compute(c, s, a, b):
+
+    @dask.delayed
+    def f(n, counter):
+        assert isinstance(counter, Actor)
+        for i in range(n):
+            counter.increment().result()
+
+    @dask.delayed
+    def check(counter, blanks):
+        return counter.n
+
+    counter = dask.delayed(Counter)()
+    values = [f(i, counter) for i in range(5)]
+    final = check(counter, values)
+
+    result = yield c.compute(final, actors=counter)
+    assert result == 0 + 1 + 2 + 3 + 4
+
+    start = time()
+    while a.data or b.data or a.actors or b.actors:
+        yield gen.sleep(0.01)
+        assert time() < start + 2
+
+
+@gen_cluster(client=True, ncores=[('127.0.0.1', 1)],
+             config={'distributed.worker.profile.interval': '1ms'})
+def test_actors_in_profile(c, s, a):
+    class Sleeper(object):
+        def sleep(self, time):
+            sleep(time)
+
+    sleeper = yield c.submit(Sleeper, actor=True)
+
+    for i in range(5):
+        yield sleeper.sleep(0.200)
+        if (list(a.profile_recent['children'])[0].startswith('sleep') or
+                'Sleeper.sleep' in a.profile_keys):
+            return
+    assert False, list(a.profile_keys)
+
+
+@gen_cluster(client=True)
+def test_waiter(c, s, a, b):
+    from tornado.locks import Event
+
+    class Waiter(object):
+        def __init__(self):
+            self.event = Event()
+
+        @gen.coroutine
+        def set(self):
+            self.event.set()
+
+        @gen.coroutine
+        def wait(self):
+            yield self.event.wait()
+
+    waiter = yield c.submit(Waiter, actor=True)
+
+    futures = [waiter.wait() for i in range(5)]  # way more than we have actor threads
+
+    yield gen.sleep(0.1)
+    assert not any(future.done() for future in futures)
+
+    yield waiter.set()
+
+    yield futures
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 77730fac85c..4ebfd9869c9 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -705,21 +705,6 @@ def test_stop_doing_unnecessary_work(c, s, a, b):
 
 @gen_cluster(client=True, ncores=[('127.0.0.1', 1)])
 def test_priorities(c, s, w):
-    a = delayed(slowinc)(1, dask_key_name='a', delay=0.05)
-    b = delayed(slowinc)(2, dask_key_name='b', delay=0.05)
-    a1 = delayed(slowinc)(a, dask_key_name='a1', delay=0.05)
-    a2 = delayed(slowinc)(a1, dask_key_name='a2', delay=0.05)
-    b1 = delayed(slowinc)(b, dask_key_name='b1', delay=0.05)
-
-    z = delayed(add)(a2, b1)
-    future = yield c.compute(z)
-
-    log = [t for t in w.log if t[1] == 'executing' and t[2] == 'memory']
-    assert [t[0] for t in log[:5]] == ['a', 'b', 'a1', 'b1', 'a2']
-
-
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)])
-def test_priorities_2(c, s, w):
     values = []
     for i in range(10):
         a = delayed(slowinc)(i, dask_key_name='a-%d' % i, delay=0.01)
diff --git a/distributed/utils.py b/distributed/utils.py
index 6257a9ba83b..193005dbc83 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -1418,3 +1418,11 @@ def color_of(x, palette=palette):
     h = md5(str(x).encode())
     n = int(h.hexdigest()[:8], 16)
     return palette[n % len(palette)]
+
+
+def iscoroutinefunction(f):
+    if gen.is_coroutine_function(f):
+        return True
+    if sys.version_info >= (3, 5) and inspect.iscoroutinefunction(f):
+        return True
+    return False
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index cce9ad75dda..10e5329f4d4 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -8,7 +8,6 @@
 import gc
 from glob import glob
 import itertools
-import inspect
 import logging
 import logging.config
 import os
@@ -40,14 +39,14 @@
 from tornado.ioloop import IOLoop
 
 from .client import default_client, _global_clients
-from .compatibility import PY3, iscoroutinefunction, Empty, WINDOWS
+from .compatibility import PY3, Empty, WINDOWS
 from .config import initialize_logging
 from .core import connect, rpc, CommClosedError
 from .metrics import time
 from .proctitle import enable_proctitle_on_children
 from .security import Security
 from .utils import (ignoring, log_errors, mp_context, get_ip, get_ipv6,
-                    DequeHandler, reset_logger_locks, sync)
+                    DequeHandler, reset_logger_locks, sync, iscoroutinefunction)
 from .worker import Worker, TOTAL_MEMORY, _global_workers
 
 try:
@@ -719,12 +718,6 @@ def end_worker(w):
     s.stop()
 
 
-def iscoroutinefunction(f):
-    if sys.version_info >= (3, 5) and inspect.iscoroutinefunction(f):
-        return True
-    return False
-
-
 def gen_cluster(ncores=[('127.0.0.1', 1), ('127.0.0.1', 2)],
                 scheduler='127.0.0.1', timeout=10, security=None,
                 Worker=Worker, client=False, scheduler_kwargs={},
diff --git a/distributed/worker.py b/distributed/worker.py
index 6d98ebc4659..f614c8e0b34 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -47,7 +47,7 @@
                     ignoring, mp_context, import_file,
                     silence_logging, thread_state, json_load_robust, key_split,
                     format_bytes, DequeHandler, PeriodicCallback,
-                    parse_bytes, parse_timedelta)
+                    parse_bytes, parse_timedelta, iscoroutinefunction)
 from .utils_comm import pack_data, gather_from_workers
 from .utils_perf import ThrottledGC, enable_gc_diagnosis, disable_gc_diagnosis
 
@@ -160,11 +160,13 @@ def __init__(self, scheduler_ip=None, scheduler_port=None,
             self.data = Buffer({}, storage, target, weight)
         else:
             self.data = dict()
+        self.actors = {}
         self.loop = loop or IOLoop.current()
         self.status = None
         self._closed = Event()
         self.reconnect = reconnect
         self.executor = executor or ThreadPoolExecutor(self.ncores)
+        self.actor_executor = ThreadPoolExecutor(1)
         self.name = name
         self.scheduler_delay = 0
         self.stream_comms = dict()
@@ -195,6 +197,8 @@ def __init__(self, scheduler_ip=None, scheduler_port=None,
             'get_logs': self.get_logs,
             'keys': self.keys,
             'versions': self.versions,
+            'actor_execute': self.actor_execute,
+            'actor_attribute': self.actor_attribute,
         }
 
         stream_handlers = {
@@ -477,11 +481,13 @@ def _close(self, report=True, timeout=10, nanny=True, executor_wait=True):
                     yield gen.with_timeout(timedelta(seconds=timeout),
                                            self.scheduler.unregister(address=self.contact_address))
             self.scheduler.close_rpc()
+            self.actor_executor._work_queue.queue.clear()
             if isinstance(self.executor, ThreadPoolExecutor):
                 self.executor._work_queue.queue.clear()
                 self.executor.shutdown(wait=executor_wait, timeout=timeout)
             else:
                 self.executor.shutdown(wait=False)
+            self.actor_executor.shutdown(wait=executor_wait, timeout=timeout)
             self._workdir.release()
 
             for k, v in self.services.items():
@@ -527,7 +533,8 @@ def wait_until_closed(self):
         assert self.status == 'closed'
 
     @gen.coroutine
-    def executor_submit(self, key, function, *args, **kwargs):
+    def executor_submit(self, key, function, args=(), kwargs=None,
+                        executor=None):
         """ Safely run function in thread pool executor
 
         We've run into issues running concurrent.future futures within
@@ -535,9 +542,11 @@ def executor_submit(self, key, function, *args, **kwargs):
         callbacks to ensure things run smoothly.  This can get tricky, so we
         pull it off into an separate method.
         """
+        executor = executor or self.executor
         job_counter[0] += 1
         # logger.info("%s:%d Starts job %d, %s", self.ip, self.port, i, key)
-        future = self.executor.submit(function, *args, **kwargs)
+        kwargs = kwargs or {}
+        future = executor.submit(function, *args, **kwargs)
         pc = PeriodicCallback(lambda: logger.debug("future state: %s - %s",
                                                    key, future._state), 1000)
         pc.start()
@@ -558,6 +567,33 @@ def run_coroutine(self, comm, function, args=(), kwargs={}, wait=True):
         return run(self, comm, function=function, args=args, kwargs=kwargs,
                    is_coro=True, wait=wait)
 
+    @gen.coroutine
+    def actor_execute(self, comm=None, actor=None, function=None, args=(), kwargs={}):
+        separate_thread = kwargs.pop('separate_thread', True)
+        key = actor
+        actor = self.actors[key]
+        func = getattr(actor, function)
+        name = key_split(key) + '.' + function
+
+        if iscoroutinefunction(func):
+            result = yield func(*args, **kwargs)
+        elif separate_thread:
+            result = yield self.executor_submit(name,
+                                                apply_function_actor,
+                                                args=(func, args, kwargs,
+                                                      self.execution_state,
+                                                      name,
+                                                      self.active_threads,
+                                                      self.active_threads_lock),
+                                                executor=self.actor_executor)
+        else:
+            result = func(*args, **kwargs)
+        raise gen.Return({'status': 'OK', 'result': to_serialize(result)})
+
+    def actor_attribute(self, comm=None, actor=None, attribute=None):
+        value = getattr(self.actors[actor], attribute)
+        return {'status': 'OK', 'result': to_serialize(value)}
+
     def update_data(self, comm=None, data=None, report=True, serializers=None):
         for key, value in data.items():
             if key in self.task_state:
@@ -618,6 +654,13 @@ def get_data(self, comm, keys=None, who=None, serializers=None,
 
         self.outgoing_current_count += 1
         data = {k: self.data[k] for k in keys if k in self.data}
+
+        if len(data) < len(keys):
+            for k in set(keys) - set(data):
+                if k in self.actors:
+                    from .actor import Actor
+                    data[k] = Actor(type(self.actors[k]), self.address, k)
+
         msg = {'status': 'OK',
                'data': {k: to_serialize(v) for k, v in data.items()}}
         nbytes = {k: self.nbytes.get(k) for k in data}
@@ -625,6 +668,7 @@ def get_data(self, comm, keys=None, who=None, serializers=None,
         if self.digests is not None:
             self.digests['get-data-load-duration'].add(stop - start)
         start = time()
+
         try:
             compressed = yield comm.write(msg, serializers=serializers)
             response = yield comm.read(deserializers=serializers)
@@ -878,6 +922,30 @@ def apply_function(function, args, kwargs, execution_state, key,
     return msg
 
 
+def apply_function_actor(function, args, kwargs, execution_state, key,
+                         active_threads, active_threads_lock):
+    """ Run a function, collect information
+
+    Returns
+    -------
+    msg: dictionary with status, result/error, timings, etc..
+    """
+    ident = get_thread_identity()
+
+    with active_threads_lock:
+        active_threads[ident] = key
+
+    thread_state.execution_state = execution_state
+    thread_state.key = key
+
+    result = function(*args, **kwargs)
+
+    with active_threads_lock:
+        del active_threads[ident]
+
+    return result
+
+
 def get_msg_safe_str(msg):
     """ Make a worker msg, which contains args and kwargs, safe to cast to str:
     allowing for some arguments to raise exceptions during conversion and
@@ -1267,13 +1335,13 @@ def __repr__(self):
 
     def add_task(self, key, function=None, args=None, kwargs=None, task=None,
                  who_has=None, nbytes=None, priority=None, duration=None,
-                 resource_restrictions=None, **kwargs2):
+                 resource_restrictions=None, actor=False, **kwargs2):
         try:
             if key in self.tasks:
                 state = self.task_state[key]
                 if state in ('memory', 'error'):
                     if state == 'memory':
-                        assert key in self.data
+                        assert key in self.data or key in self.actors
                     logger.debug("Asked to compute pre-existing result: %s: %s",
                                  key, state)
                     self.send_task_state_to_scheduler(key)
@@ -1298,6 +1366,8 @@ def add_task(self, key, function=None, args=None, kwargs=None, task=None,
             try:
                 start = time()
                 self.tasks[key] = _deserialize(function, args, kwargs, task)
+                if actor:
+                    self.actors[key] = None
                 stop = time()
 
                 if stop - start > 0.010:
@@ -1490,7 +1560,7 @@ def transition_waiting_ready(self, key):
                 assert self.task_state[key] == 'waiting'
                 assert key in self.waiting_for_data
                 assert not self.waiting_for_data[key]
-                assert all(dep in self.data for dep in self.dependencies[key])
+                assert all(dep in self.data or dep in self.actors for dep in self.dependencies[key])
                 assert key not in self.executing
                 assert key not in self.ready
 
@@ -1532,7 +1602,7 @@ def transition_ready_executing(self, key):
                 # assert key not in self.data
                 assert self.task_state[key] in READY
                 assert key not in self.ready
-                assert all(dep in self.data for dep in self.dependencies[key])
+                assert all(dep in self.data or dep in self.actors for dep in self.dependencies[key])
 
             self.executing.add(key)
             self.loop.add_callback(self.execute, key)
@@ -1712,9 +1782,14 @@ def ensure_communicating(self):
             raise
 
     def send_task_state_to_scheduler(self, key):
-        if key in self.data:
-            nbytes = self.nbytes[key] or sizeof(self.data[key])
-            typ = self.types.get(key) or type(self.data[key])
+        if key in self.data or self.actors.get(key):
+            try:
+                value = self.data[key]
+            except KeyError:
+                value = self.actors[key]
+            nbytes = self.nbytes[key] or sizeof(value)
+            typ = self.types.get(key) or type(value)
+            del value
             try:
                 typ = dumps_function(typ)
             except PicklingError:
@@ -1747,11 +1822,15 @@ def put_key_in_memory(self, key, value, transition=True):
         if key in self.data:
             return
 
-        start = time()
-        self.data[key] = value
-        stop = time()
-        if stop - start > 0.020:
-            self.startstops[key].append(('disk-write', start, stop))
+        if key in self.actors:
+            self.actors[key] = value
+
+        else:
+            start = time()
+            self.data[key] = value
+            stop = time()
+            if stop - start > 0.020:
+                self.startstops[key].append(('disk-write', start, stop))
 
         if key not in self.nbytes:
             self.nbytes[key] = sizeof(value)
@@ -2012,6 +2091,10 @@ def release_key(self, key, cause=None, reason=None, report=True):
                                  exc_info=True)
                 del self.nbytes[key]
                 del self.types[key]
+            if key in self.actors and key not in self.dep_state:
+                del self.actors[key]
+                del self.nbytes[key]
+                del self.types[key]
 
             if key in self.waiting_for_data:
                 del self.waiting_for_data[key]
@@ -2075,6 +2158,9 @@ def release_dep(self, dep, report=False):
                 if dep in self.data:
                     del self.data[dep]
                     del self.types[dep]
+                if dep in self.actors:
+                    del self.actors[dep]
+                    del self.types[dep]
                 del self.nbytes[dep]
 
             if dep in self.in_flight_tasks:
@@ -2176,7 +2262,13 @@ def execute(self, key, report=False):
             function, args, kwargs = self.tasks[key]
 
             start = time()
-            data = {k: self.data[k] for k in self.dependencies[key]}
+            data = {}
+            for k in self.dependencies[key]:
+                try:
+                    data[k] = self.data[k]
+                except KeyError:
+                    from .actor import Actor  # TODO: create local actor
+                    data[k] = Actor(type(self.actors[k]), self.address, k, self)
             args2 = pack_data(args, data, key_types=(bytes, unicode))
             kwargs2 = pack_data(kwargs, data, key_types=(bytes, unicode))
             stop = time()
@@ -2187,12 +2279,12 @@ def execute(self, key, report=False):
 
             logger.debug("Execute key: %s worker: %s", key, self.address)  # TODO: comment out?
             try:
-                result = yield self.executor_submit(key, apply_function, function,
-                                                    args2, kwargs2,
-                                                    self.execution_state, key,
-                                                    self.active_threads,
-                                                    self.active_threads_lock,
-                                                    self.scheduler_delay)
+                result = yield self.executor_submit(key, apply_function,
+                                                    args=(function, args2, kwargs2,
+                                                          self.execution_state, key,
+                                                          self.active_threads,
+                                                          self.active_threads_lock,
+                                                          self.scheduler_delay))
             except RuntimeError as e:
                 executor_error = e
                 raise
@@ -2353,9 +2445,9 @@ def trigger_profile(self):
             if frame is not None:
                 key = key_split(active_threads[ident])
                 profile.process(frame, None, self.profile_recent,
-                                stop='_concurrent_futures_thread.py')
+                                stop='distributed/worker.py')
                 profile.process(frame, None, self.profile_keys[key],
-                                stop='_concurrent_futures_thread.py')
+                                stop='distributed/worker.py')
         stop = time()
         if self.digests is not None:
             self.digests['profile-duration'].add(stop - start)
@@ -2438,7 +2530,7 @@ def get_logs(self, comm=None, n=None):
         return [(msg.levelname, deque_handler.format(msg)) for msg in L]
 
     def validate_key_memory(self, key):
-        assert key in self.data
+        assert key in self.data or key in self.actors
         assert key in self.nbytes
         assert key not in self.waiting_for_data
         assert key not in self.executing
@@ -2450,14 +2542,14 @@ def validate_key_executing(self, key):
         assert key in self.executing
         assert key not in self.data
         assert key not in self.waiting_for_data
-        assert all(dep in self.data for dep in self.dependencies[key])
+        assert all(dep in self.data or dep in self.actors for dep in self.dependencies[key])
 
     def validate_key_ready(self, key):
         assert key in pluck(1, self.ready)
         assert key not in self.data
         assert key not in self.executing
         assert key not in self.waiting_for_data
-        assert all(dep in self.data for dep in self.dependencies[key])
+        assert all(dep in self.data or dep in self.actors for dep in self.dependencies[key])
 
     def validate_key_waiting(self, key):
         assert key not in self.data
@@ -2495,7 +2587,7 @@ def validate_dep_flight(self, dep):
         assert dep in self.in_flight_workers[peer]
 
     def validate_dep_memory(self, dep):
-        assert dep in self.data
+        assert dep in self.data or dep in self.actors
         assert dep in self.nbytes
         assert dep in self.types
         if dep in self.task_state:
@@ -2548,7 +2640,7 @@ def validate_state(self):
                 if self.task_state[key] == 'memory':
                     assert isinstance(self.nbytes[key], int)
                     assert key not in self.waiting_for_data
-                    assert key in self.data
+                    assert key in self.data or key in self.actors
 
         except Exception as e:
             logger.exception(e)
diff --git a/docs/source/actors.rst b/docs/source/actors.rst
new file mode 100644
index 00000000000..109b9d907aa
--- /dev/null
+++ b/docs/source/actors.rst
@@ -0,0 +1,235 @@
+Actors
+======
+
+.. note:: This is an experimental feature and is subject to change without notice
+.. note:: This is an advanced feature and may not be suitable for beginning users.
+   It is rarely necessary for common workloads.
+
+Actors enable stateful computations within a Dask workflow.  They are useful
+for some rare algorithms that require additional performance and are willing to
+sacrifice resilience.
+
+An actor is a pointer to a user-defined-object living on a remote worker.
+Anyone with that actor can call methods on that remote object.
+
+Example
+-------
+
+Here we create a simple ``Counter`` class, instantiate that class on one worker,
+and then call methods on that class remotely.
+
+.. code-block:: python
+
+   class Counter:
+       """ A simple class to manage an incrementing counter """
+       n = 0
+
+       def __init__(self):
+           self.n = 0
+
+       def increment(self):
+           self.n += 1
+           return self.n
+
+       def add(self, x):
+           self.n += x
+           return self.n
+
+   from dask.distributed import Client          # Start a Dask Client
+   client = Client()
+
+   future = client.submit(Counter, actor=True)  # Create a Counter on a worker
+   counter = future.result()                    # Get back a pointer to that object
+
+   counter
+   # <Actor: Counter, key=Counter-1234abcd>
+
+   future = counter.increment()                 # Call remote method
+   future.result()                              # Get back result
+   # 1
+
+   future = counter.add(10)                     # Call remote method
+   future.result()                              # Get back result
+   # 11
+
+Motivation
+----------
+
+Actors are motivated by some of the challenges of using pure task graphs.
+
+Normal Dask computations are composed of a graph of functions.
+This approach has a few limitations that are good for resilience, but can
+negatively affect performance:
+
+1.  **State**: The functions should not mutate their inputs in-place or rely on
+    global state.  They  should instead operate in a pure-functional manner,
+    consuming inputs and producing separate outputs.
+2.  **Central Overhead**: The execution location and order is determined by the
+    centralized scheduler.  Because the scheduler is involved in every decision
+    it can sometimes create a central bottleneck.
+
+Some workloads may need to update state directly, or may involve more tiny
+tasks than the scheduler can handle (the scheduler can coordinate about 4000
+tasks per second).
+
+Actors side-step both of these limitations:
+
+1.  **State**: Actors can hold on to and mutate state.  They are allowed to
+    update their state in-place.
+2.  **Overhead**: Operations on actors do not inform the central scheduler, and
+    so do not contribute to the 4000 task/second overhead.  They also avoid an
+    extra network hop and so have lower latencies.
+
+Create an Actor
+---------------
+
+You create an actor by submitting a Class to run on a worker using normal Dask
+computation functions like ``submit``, ``map``, ``compute``, or ``persist``,
+and using the ``actors=`` keyword (or ``actor=`` on ``submit``).
+
+.. code-block:: python
+
+   future = client.submit(Counter, actors=True)
+
+You can use all other keywords to these functions like ``workers=``,
+``resources=``, and so on to control where this actor ends up.
+
+This creates a normal Dask future on which you can call ``.result()`` to get
+the Actor once it has successfully run on a worker.
+
+.. code-block:: python
+
+   >>> counter = future.result()
+   >>> counter
+   <Actor: Counter, key=...>
+
+A ``Counter`` object has been instantiated on one of the workers, and this
+``Actor`` object serves as our proxy to that remote object.  It has the same
+methods and attributes.
+
+.. code-block:: python
+
+   >>> dir(counter)
+   ['add', 'increment', 'n']
+
+Call Remote Methods
+-------------------
+
+However accessing an attribute or calling a method will trigger a communication
+to the remote worker, run the method on the remote worker in a separate thread
+pool, and then communicate the result back to the calling side.  For attribute
+access these operations block and return when finished, for method calls they
+return an ``ActorFuture`` immediately.
+
+.. code-block:: python
+
+   >>> future = counter.increment()  # Immediately returns an ActorFuture
+   >>> future.result()               # Block until finished and result arrives
+   1
+
+``ActorFuture`` are similar to normal Dask ``Future`` objects, but not as fully
+featured.  They curently *only* support the ``result`` method and nothing else.
+They don't currently work with any other Dask functions that expect futures,
+like ``as_completed``, ``wait``, or ``client.gather``.  They can't be placed
+into additional submit or map calls to form dependencies.  They communicate
+their results immediately (rather than waiting for result to be called) and
+cache the result on the future itself.
+
+Access Attributes
+-----------------
+
+If you define an attribute at the class level then that attribute will be
+accessible to the actor.
+
+.. code-block:: python
+
+   class Counter:
+       n = 0   # Recall that we defined our class with `n` as a class variable
+
+       ...
+
+   >>> counter.n                     # Blocks until finished
+   1
+
+Attribute access blocks automatically.  It's as though you called ``.result()``.
+
+
+Execution on the Worker
+-----------------------
+
+When you call a method on an actor, your arguments get serialized and sent
+to the worker that owns the actor's object.  If you do this from a worker this
+communication is direct.  If you do this from a Client then this will be direct
+if the Client has direct access to the workers (create a client with
+``Client(..., direct_to_workers=True)`` if direct connections are possible) or
+by proxying through the scheduler if direct connections from the client to the
+workers are not possible.
+
+The appropriate method of the Actor's object is then called in a separate
+thread, the result captured, and then sent back to the calling side.  Currently
+workers have only a single thread for actors, but this may change in the
+future.
+
+The result is sent back immediately to the calling side, and is not stored on
+the worker with the actor.  It is cached on the ``ActorFuture`` object.
+
+
+Calling from coroutines and async/await
+--------------------------
+
+If you use actors within a coroutine or async/await function then actor methods
+and attrbute access will return Tornado futures
+
+.. code-block:: python
+
+   async def f():
+       counter = await client.submit(Counter, actor=True)
+
+       await counter.increment()
+       n = await counter.n
+
+
+Coroutines and async/await on the Actor
+---------------------------------------
+
+If you define an ``async def`` function on the actor class then that method
+will run on the Worker's event loop thread rather than a separate thread.
+
+.. code-block:: python
+
+   def Waiter(object):
+       def __init__(self):
+           self.event = tornado.locks.Event()
+
+       async def set(self):
+           self.event.set()
+
+       async def wait(self):
+           await self.event.wait()
+
+   waiter = client.submit(Waiter, actor=True).result()
+   waiter.wait().result()  # waits until set, without consuming a worker thread
+
+
+Performance
+-----------
+
+Worker operations currently have about 1ms of latency, on top of any network
+latency that may exist.  However other activity in a worker may easily increase
+these latencies if enough other activities are present.
+
+
+Limitations
+-----------
+
+Actors offer advanced capabilities, but with some cost:
+
+1.  **No Resilience:** No effort is made to make actor workloads resilient to
+    worker failure.  If the worker dies while holding an actor that actor is
+    lost forever.
+2.  **No Diagnostics:** Because the scheduler is not informed about actor
+    computations no diagnostics are available about these computations.
+3.  **No Load balancing:** Actors are allocated onto workers evenly, without
+    serious consideration given to avoiding communication.
+4.  **Experimental:** Actors are a new feature and subject to change without
+    warning
diff --git a/docs/source/index.rst b/docs/source/index.rst
index 2d1b6328aa6..211e94a4c3b 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -104,6 +104,7 @@ Contents
    :maxdepth: 1
    :caption: Additional Features
 
+   actors
    adaptive
    asynchronous
    configuration

From c5a0b2359cd1e35ae9f42bf35a811dd1f5e1cb0a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lo=C3=AFc=20Est=C3=A8ve?= <loic.esteve@ymail.com>
Date: Wed, 8 Aug 2018 16:27:15 +0200
Subject: [PATCH 0053/1550] Fix tooltip (#2168)

---
 distributed/bokeh/scheduler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/bokeh/scheduler.py b/distributed/bokeh/scheduler.py
index cd677e455c2..52e693c75d5 100644
--- a/distributed/bokeh/scheduler.py
+++ b/distributed/bokeh/scheduler.py
@@ -971,7 +971,7 @@ def __init__(self, scheduler, width=800, **kwargs):
             point_policy="follow_mouse",
             tooltips="""
                 <div>
-                  <span style="font-size: 10px; font-family: Monaco, monospace;">@host: </span>
+                  <span style="font-size: 10px; font-family: Monaco, monospace;">@worker: </span>
                   <span style="font-size: 10px; font-family: Monaco, monospace;">@memory_percent</span>
                 </div>
                 """

From 8ae684816916e1b0e8131f12d1f54b89f7f4d07f Mon Sep 17 00:00:00 2001
From: Joe Hamman <jhamman@ucar.edu>
Date: Wed, 8 Aug 2018 16:47:37 -0600
Subject: [PATCH 0054/1550] fix scale /  avoid returning coroutines (#2171)

---
 distributed/deploy/cluster.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
index 4265a151945..0c647d7adc9 100644
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -93,12 +93,10 @@ def scale(self, n):
             if n >= len(self.scheduler.workers):
                 self.scheduler.loop.add_callback(self.scale_up, n)
             else:
-                to_close = self.scheduler.retire_workers(
-                    remove=False,
-                    close_workers=True,
-                    n=len(self.scheduler.workers) - n
-                )
+                to_close = self.scheduler.workers_to_close(
+                    n=len(self.scheduler.workers) - n)
                 logger.debug("Closing workers: %s", to_close)
+                self.scheduler.loop.add_callback(self.scheduler.retire_workers, workers=to_close)
                 self.scheduler.loop.add_callback(self.scale_down, to_close)
 
     def _widget_status(self):

From 2cfaa2d179c14165eb77cc2abd1469d92acdb6bc Mon Sep 17 00:00:00 2001
From: Yu Feng <rainwoodman@gmail.com>
Date: Wed, 8 Aug 2018 17:12:27 -0700
Subject: [PATCH 0055/1550] Clarify dask-worker --nprocs (#2173)

---
 distributed/cli/dask_worker.py |  4 ++--
 docs/source/worker.rst         | 22 +++++++++++++---------
 2 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
index 05531a6f4a1..2c0676361ae 100755
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -62,13 +62,13 @@
 @click.option('--nthreads', type=int, default=0,
               help="Number of threads per process.")
 @click.option('--nprocs', type=int, default=1,
-              help="Number of worker processes.  Defaults to one.")
+              help="Number of worker processes to launch.  Defaults to one.")
 @click.option('--name', type=str, default='',
               help="A unique name for this worker like 'worker-1'. "
                    "If used with --nprocs then the process number "
                    "will be appended like name-0, name-1, name-2, ...")
 @click.option('--memory-limit', default='auto',
-              help="Bytes of memory that the worker can use. "
+              help="Bytes of memory per process that the worker can use. "
                    "This can be an integer (bytes), "
                    "float (fraction of total system memory), "
                    "string (like 5GB or 5000M), "
diff --git a/docs/source/worker.rst b/docs/source/worker.rst
index deaa0243913..5ac95c197c9 100644
--- a/docs/source/worker.rst
+++ b/docs/source/worker.rst
@@ -65,15 +65,18 @@ communication costs and generally simplifies deployment.
 
 If your computations are mostly Python code and don't release the GIL then it
 is advisable to run ``dask-worker`` processes with many processes and one
-thread per core::
+thread per process::
 
-   $ dask-worker scheduler:8786 --nprocs 8
+   $ dask-worker scheduler:8786 --nprocs 8 --nthreads 1
+
+This will launch 8 worker processes each of which has its own
+ThreadPoolExecutor of size 1. 
 
 If your computations are external to Python and long-running and don't release
 the GIL then beware that while the computation is running the worker process
 will not be able to communicate to other workers or to the scheduler.  This
 situation should be avoided.  If you don't link in your own custom C/Fortran
-code then this topic probably doesn't apply to you.
+code then this topic probably doesn't apply.
 
 Command Line tool
 -----------------
@@ -93,9 +96,9 @@ are the available options::
                             hopefully be visible from the scheduler network.
      --nthreads INTEGER     Number of threads per process. Defaults to number of
                             cores
-     --nprocs INTEGER       Number of worker processes.  Defaults to one.
+     --nprocs INTEGER       Number of worker processes to launch.  Defaults to one.
      --name TEXT            Alias
-     --memory-limit TEXT     Number of bytes before spilling data to disk
+     --memory-limit TEXT    Number of bytes (per worker process) before spilling data to disk
      --no-nanny
      --help                 Show this message and exit.
 
@@ -143,12 +146,13 @@ Memory Management
 
 Workers are given a target memory limit to stay under with the
 command line ``--memory-limit`` keyword or the ``memory_limit=`` Python
-keyword argument.::
+keyword argument, which sets the memory limit per worker processes launched
+by dask-workder ::
 
-    $ dask-worker tcp://scheduler:port --memory-limit=auto  # total available RAM
-    $ dask-worker tcp://scheduler:port --memory-limit=4e9  # four gigabytes
+    $ dask-worker tcp://scheduler:port --memory-limit=auto  # total available RAM on the machine
+    $ dask-worker tcp://scheduler:port --memory-limit=4e9  # four gigabytes per worker process.
 
-Workers use a few different policies to keep memory use beneath this limit:
+Workers use a few different heuristics to keep memory use beneath this limit:
 
 1.  At 60% of memory load (as estimated by ``sizeof``), spill least recently used data to disk
 2.  At 70% of memory load, spill least recently used data to disk regardless of

From a1d4a9d698d760496e06c0790cf4a617065ccbb3 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 9 Aug 2018 08:42:19 -0400
Subject: [PATCH 0056/1550] Concatenate all bytes of small messages in TCP
 comms (#2172)

Previously we would write lengths to a socket, and then follow up with
frames.  This causes additional socket.send calls, which can be costly.

Now for small messages we just bundle everything together and suffer a
memory copy, but avoid the extra socket.send calls.
---
 distributed/comm/tcp.py   | 35 ++++++++++++++++++++---------------
 distributed/utils_test.py |  4 ++--
 2 files changed, 22 insertions(+), 17 deletions(-)

diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py
index d1dcab7569c..a1785d907eb 100644
--- a/distributed/comm/tcp.py
+++ b/distributed/comm/tcp.py
@@ -221,21 +221,26 @@ def write(self, msg, serializers=None, on_error='message'):
                                           'recipient': self._peer_addr})
 
         try:
-            lengths = ([struct.pack('Q', len(frames))] +
-                       [struct.pack('Q', nbytes(frame)) for frame in frames])
-            stream.write(b''.join(lengths))
-
-            for frame in frames:
-                # Can't wait for the write() Future as it may be lost
-                # ("If write is called again before that Future has resolved,
-                #   the previous future will be orphaned and will never resolve")
-                if not self._iostream_allows_memoryview:
-                    frame = ensure_bytes(frame)
-                future = stream.write(frame)
-                bytes_since_last_yield += nbytes(frame)
-                if bytes_since_last_yield > 32e6:
-                    yield future
-                    bytes_since_last_yield = 0
+            lengths = [nbytes(frame) for frame in frames]
+            length_bytes = ([struct.pack('Q', len(frames))] +
+                            [struct.pack('Q', x) for x in lengths])
+            if PY3 and sum(lengths) < 2**17:  # 128kiB
+                b = b''.join(length_bytes + frames)  # small enough, send in one go
+                stream.write(b)
+            else:
+                stream.write(b''.join(length_bytes))  # avoid large memcpy, send in many
+
+                for frame in frames:
+                    # Can't wait for the write() Future as it may be lost
+                    # ("If write is called again before that Future has resolved,
+                    #   the previous future will be orphaned and will never resolve")
+                    if not self._iostream_allows_memoryview:
+                        frame = ensure_bytes(frame)
+                    future = stream.write(frame)
+                    bytes_since_last_yield += nbytes(frame)
+                    if bytes_since_last_yield > 32e6:
+                        yield future
+                        bytes_since_last_yield = 0
         except StreamClosedError as e:
             stream = None
             convert_stream_closed_error(self, e)
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index 10e5329f4d4..289987b2641 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -985,7 +985,7 @@ def assert_can_connect(addr, timeout=None, connection_args=None):
     within the given *timeout*.
     """
     if timeout is None:
-        timeout = 0.2
+        timeout = 0.5
     comm = yield connect(addr, timeout=timeout,
                          connection_args=connection_args)
     comm.abort()
@@ -998,7 +998,7 @@ def assert_cannot_connect(addr, timeout=None, connection_args=None, exception_cl
     within the given *timeout*.
     """
     if timeout is None:
-        timeout = 0.2
+        timeout = 0.5
     with pytest.raises(exception_class):
         comm = yield connect(addr, timeout=timeout,
                              connection_args=connection_args)

From 03341fd025276231423ed9d25fa302101a429b50 Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Thu, 9 Aug 2018 17:05:09 -0600
Subject: [PATCH 0057/1550] Add dashboard_link property (#2176)

---
 distributed/deploy/cluster.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
index 0c647d7adc9..3bc2b2d9124 100644
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -72,6 +72,13 @@ def adapt(self, **kwargs):
     def scheduler_address(self):
         return self.scheduler.address
 
+    @property
+    def dashboard_link(self):
+        template = dask.config.get('distributed.dashboard.link')
+        host = self.scheduler.address.split('://')[1].split(':')[0]
+        port = self.scheduler.services['bokeh'].port
+        return template.format(host=host, port=port, **os.environ)
+
     def scale(self, n):
         """ Scale cluster to n workers
 
@@ -140,11 +147,7 @@ def _widget(self):
         layout = Layout(width='150px')
 
         if 'bokeh' in self.scheduler.services:
-            template = dask.config.get('distributed.dashboard.link')
-
-            host = self.scheduler.address.split('://')[1].split(':')[0]
-            port = self.scheduler.services['bokeh'].port
-            link = template.format(host=host, port=port, **os.environ)
+            link = self.dashboard_link
             link = '<p><b>Dashboard: </b><a href="%s" target="_blank">%s</a></p>\n' % (link, link)
         else:
             link = ''

From cb10d6b10ad7800648b44ab3b194220d1e6fc5f8 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 10 Aug 2018 07:55:21 -0400
Subject: [PATCH 0058/1550] always offload to_frames (#2170)

---
 distributed/bokeh/tests/test_components.py |  4 ++--
 distributed/comm/utils.py                  | 11 +++++------
 distributed/tests/test_asyncprocess.py     |  2 +-
 distributed/utils_test.py                  |  4 ++++
 4 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/distributed/bokeh/tests/test_components.py b/distributed/bokeh/tests/test_components.py
index 89606eaf553..741c90c8d49 100644
--- a/distributed/bokeh/tests/test_components.py
+++ b/distributed/bokeh/tests/test_components.py
@@ -25,7 +25,7 @@ def test_basic(Component):
     c.update(messages)
 
 
-@gen_cluster(client=True)
+@gen_cluster(client=True, check_new_threads=False)
 def test_profile_plot(c, s, a, b):
     p = ProfilePlot()
     assert len(p.source.data['left']) <= 1
@@ -34,7 +34,7 @@ def test_profile_plot(c, s, a, b):
     assert len(p.source.data['left']) > 1
 
 
-@gen_cluster(client=True)
+@gen_cluster(client=True, check_new_threads=False)
 def test_profile_time_plot(c, s, a, b):
     from bokeh.io import curdoc
     sp = ProfileTimePlot(s, doc=curdoc())
diff --git a/distributed/comm/utils.py b/distributed/comm/utils.py
index 02677b9faba..d3e758a0741 100644
--- a/distributed/comm/utils.py
+++ b/distributed/comm/utils.py
@@ -7,9 +7,8 @@
 from tornado import gen
 
 from .. import protocol
-from ..compatibility import finalize
-from ..sizeof import sizeof
-from ..utils import get_ip, get_ipv6, mp_context, nbytes
+from ..compatibility import finalize, PY3
+from ..utils import get_ip, get_ipv6, nbytes
 
 
 logger = logging.getLogger(__name__)
@@ -20,7 +19,7 @@
 
 FRAME_OFFLOAD_THRESHOLD = 10 * 1024 ** 2   # 10 MB
 
-_offload_executor = ThreadPoolExecutor(max_workers=min(4, mp_context.cpu_count()))
+_offload_executor = ThreadPoolExecutor(max_workers=1)
 finalize(_offload_executor, _offload_executor.shutdown)
 
 
@@ -44,9 +43,9 @@ def _to_frames():
             logger.exception(e)
             raise
 
-    if sizeof(msg) > FRAME_OFFLOAD_THRESHOLD:
+    if PY3:
         res = yield offload(_to_frames)
-    else:
+    else:  # distributed/deploy/tests/test_adaptive.py::test_get_scale_up_kwargs fails on Py27.  Don't know why
         res = _to_frames()
 
     raise gen.Return(res)
diff --git a/distributed/tests/test_asyncprocess.py b/distributed/tests/test_asyncprocess.py
index 9c8a8da2531..af5a07acea0 100644
--- a/distributed/tests/test_asyncprocess.py
+++ b/distributed/tests/test_asyncprocess.py
@@ -267,7 +267,7 @@ def test_child_main_thread():
     yield proc.join()
     n_threads = q.get()
     main_name = q.get()
-    assert n_threads == 2
+    assert n_threads <= 3
     assert main_name == "MainThread"
     q.close()
     q._reader.close()
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index 289987b2641..93ce281bd4f 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -40,6 +40,7 @@
 
 from .client import default_client, _global_clients
 from .compatibility import PY3, Empty, WINDOWS
+from .comm.utils import offload
 from .config import initialize_logging
 from .core import connect, rpc, CommClosedError
 from .metrics import time
@@ -63,6 +64,9 @@
                   if isinstance(logger, logging.Logger)}
 
 
+offload(lambda: None).result()  # create thread during import
+
+
 @pytest.fixture(scope='session')
 def valid_python_script(tmpdir_factory):
     local_file = tmpdir_factory.mktemp('data').join('file.py')

From 88a1c0a69eadd302388dde984a4e6e45851eee22 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sun, 19 Aug 2018 11:52:52 -0400
Subject: [PATCH 0059/1550] Warn if desired port is already in use (#2191)

* Warn if desired port is already in use

* remove diagnostics_port from LocalCluster

* cleanup processes during tests
---
 distributed/bokeh/core.py                    | 12 +++++++--
 distributed/bokeh/tests/test_worker_bokeh.py | 17 ------------
 distributed/cli/dask_worker.py               |  4 +--
 distributed/deploy/local.py                  |  6 +++--
 distributed/deploy/tests/test_local.py       | 27 +++++++++++++++-----
 distributed/deploy/utils_test.py             |  7 ++---
 distributed/scheduler.py                     |  6 +++--
 distributed/tests/py3_test_asyncio.py        |  2 +-
 distributed/tests/test_core.py               |  2 +-
 distributed/utils_test.py                    | 17 +++++++++++-
 10 files changed, 62 insertions(+), 38 deletions(-)

diff --git a/distributed/bokeh/core.py b/distributed/bokeh/core.py
index 1cd8ed58a25..351901a3386 100644
--- a/distributed/bokeh/core.py
+++ b/distributed/bokeh/core.py
@@ -39,8 +39,16 @@ def listen(self, addr):
                 self.server._tornado.add_handlers(r'.*', handlers)
 
                 return
-            except (SystemExit, EnvironmentError):
-                port = 0
+            except (SystemExit, EnvironmentError) as exc:
+                if port != 0:
+                    if ("already in use" in str(exc) or  # Unix/Mac
+                            "Only one usage of" in str(exc)):  # Windows
+                        msg = ("Port %d is already in use. "
+                               "Perhaps you already have a cluster running?"
+                               % port)
+                    else:
+                        msg = "Failed to start diagnostics server on port %d. " % port + str(exc)
+                    raise type(exc)(msg)
                 if i == 4:
                     raise
 
diff --git a/distributed/bokeh/tests/test_worker_bokeh.py b/distributed/bokeh/tests/test_worker_bokeh.py
index cdb97689778..01242b1c0b3 100644
--- a/distributed/bokeh/tests/test_worker_bokeh.py
+++ b/distributed/bokeh/tests/test_worker_bokeh.py
@@ -118,20 +118,3 @@ def test_CommunicatingStream(c, s, a, b):
             len(first(bb.outgoing.data.values())))
     assert (len(first(aa.incoming.data.values())) and
             len(first(bb.incoming.data.values())))
-
-
-@pytest.mark.skipif(sys.version_info[0] == 2,
-                    reason='https://github.com/bokeh/bokeh/issues/5494')
-@gen_cluster(client=True)
-def test_port_overlap(c, s, a, b):
-    # When the given port is unavailable, another one is chosen automatically
-    sa = BokehWorker(a)
-    sa.listen(57384)
-    sb = BokehWorker(b)
-    sb.listen(57384)
-    assert sa.port
-    assert sb.port
-    assert sa.port != sb.port
-
-    sa.stop()
-    sb.stop()
diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
index 2c0676361ae..645869fc0eb 100755
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -40,8 +40,8 @@
               help="Serving computation port, defaults to random")
 @click.option('--nanny-port', type=int, default=0,
               help="Serving nanny port, defaults to random")
-@click.option('--bokeh-port', type=int, default=8789,
-              help="Bokeh port, defaults to 8789")
+@click.option('--bokeh-port', type=int, default=0,
+              help="Bokeh port, defaults to random port")
 @click.option('--bokeh/--no-bokeh', 'bokeh', default=True, show_default=True,
               required=False, help="Launch Bokeh Web UI")
 @click.option('--listen-address', type=str, default=None,
diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index be063366b6d..a277f6d53da 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -79,7 +79,7 @@ class LocalCluster(Cluster):
     def __init__(self, n_workers=None, threads_per_worker=None, processes=True,
                  loop=None, start=None, ip=None, scheduler_port=0,
                  silence_logs=logging.WARN, diagnostics_port=8787,
-                 services={}, worker_services={}, service_kwargs=None,
+                 services=None, worker_services=None, service_kwargs=None,
                  asynchronous=False, security=None, **worker_kwargs):
         if start is not None:
             msg = ("The start= parameter is deprecated. "
@@ -93,6 +93,8 @@ def __init__(self, n_workers=None, threads_per_worker=None, processes=True,
         self.silence_logs = silence_logs
         self._asynchronous = asynchronous
         self.security = security
+        services = services or {}
+        worker_services = worker_services or {}
         if silence_logs:
             self._old_logging_level = silence_logging(level=silence_logs)
         if n_workers is None and threads_per_worker is None:
@@ -116,7 +118,7 @@ def __init__(self, n_workers=None, threads_per_worker=None, processes=True,
         self._loop_runner = LoopRunner(loop=loop, asynchronous=asynchronous)
         self.loop = self._loop_runner.loop
 
-        if diagnostics_port is not None:
+        if diagnostics_port is not False and diagnostics_port is not None:
             try:
                 from distributed.bokeh.scheduler import BokehScheduler
                 from distributed.bokeh.worker import BokehWorker
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index 7f2f5874d41..25536222ce0 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -125,6 +125,7 @@ def test_transports():
 @pytest.mark.skipif('sys.version_info[0] == 2', reason='')
 class LocalTest(ClusterTest, unittest.TestCase):
     Cluster = partial(LocalCluster, silence_logs=False, diagnostics_port=None)
+    kwargs = {'diagnostics_port': None}
 
 
 @pytest.mark.skipif('sys.version_info[0] == 2', reason='')
@@ -142,6 +143,18 @@ def test_Client_solo(loop):
     assert c.cluster.status == 'closed'
 
 
+@gen_test()
+def test_duplicate_clients():
+    c1 = yield Client(processes=False, silence_logs=False, diagnostics_port=9876)
+    with pytest.warns(Exception) as info:
+        yield Client(processes=False, silence_logs=False, diagnostics_port=9876)
+
+    assert any(all(word in str(msg.message).lower()
+                   for word in ['9876', 'running', 'already in use'])
+               for msg in info.list)
+    yield c1.close()
+
+
 def test_Client_kwargs(loop):
     with Client(loop=loop, processes=False, n_workers=2, silence_logs=False) as c:
         assert len(c.cluster.workers) == 2
@@ -150,8 +163,8 @@ def test_Client_kwargs(loop):
 
 
 def test_Client_twice(loop):
-    with Client(loop=loop, silence_logs=False) as c:
-        with Client(loop=loop, silence_logs=False) as f:
+    with Client(loop=loop, silence_logs=False, diagnostics_port=None) as c:
+        with Client(loop=loop, silence_logs=False, diagnostics_port=None) as f:
             assert c.cluster.scheduler.port != f.cluster.scheduler.port
 
 
@@ -367,7 +380,7 @@ def test_logging():
 def test_ipywidgets(loop):
     ipywidgets = pytest.importorskip('ipywidgets')
     with LocalCluster(scheduler_port=0, silence_logs=False, loop=loop,
-                      diagnostics_port=0, processes=False) as cluster:
+                      diagnostics_port=False, processes=False) as cluster:
         cluster._ipython_display_()
         box = cluster._cached_widget
         assert isinstance(box, ipywidgets.Widget)
@@ -376,7 +389,7 @@ def test_ipywidgets(loop):
 def test_scale(loop):
     """ Directly calling scale both up and down works as expected """
     with LocalCluster(scheduler_port=0, silence_logs=False, loop=loop,
-                      diagnostics_port=0, processes=False, n_workers=0) as cluster:
+                      diagnostics_port=False, processes=False, n_workers=0) as cluster:
         assert not cluster.scheduler.workers
         cluster.scale(3)
 
@@ -397,7 +410,7 @@ def test_scale(loop):
 
 def test_adapt(loop):
     with LocalCluster(scheduler_port=0, silence_logs=False, loop=loop,
-                      diagnostics_port=0, processes=False, n_workers=0) as cluster:
+                      diagnostics_port=False, processes=False, n_workers=0) as cluster:
         cluster.adapt(minimum=0, maximum=2, interval='10ms')
         assert cluster._adaptive.minimum == 0
         assert cluster._adaptive.maximum == 2
@@ -423,7 +436,7 @@ def test_adapt(loop):
 def test_adapt_then_manual(loop):
     """ We can revert from adaptive, back to manual """
     with LocalCluster(scheduler_port=0, silence_logs=False, loop=loop,
-                      diagnostics_port=0, processes=False, n_workers=8) as cluster:
+                      diagnostics_port=False, processes=False, n_workers=8) as cluster:
         sleep(0.1)
         cluster.adapt(minimum=0, maximum=4, interval='10ms')
 
@@ -454,7 +467,7 @@ def test_local_tls(loop):
     from distributed.utils_test import tls_only_security
     security = tls_only_security()
     with LocalCluster(scheduler_port=8786, silence_logs=False, security=security,
-                      diagnostics_port=0, ip='tls://0.0.0.0', loop=loop) as c:
+                      diagnostics_port=False, ip='tls://0.0.0.0', loop=loop) as c:
         sync(loop, assert_can_connect_from_everywhere_4, c.scheduler.port,
              connection_args=security.get_connection_args('client'),
              protocol='tls', timeout=3)
diff --git a/distributed/deploy/utils_test.py b/distributed/deploy/utils_test.py
index 375612edc26..9bc8cacccad 100644
--- a/distributed/deploy/utils_test.py
+++ b/distributed/deploy/utils_test.py
@@ -3,9 +3,10 @@
 
 class ClusterTest(object):
     Cluster = None
+    kwargs = {}
 
     def setUp(self):
-        self.cluster = self.Cluster(2, scheduler_port=0)
+        self.cluster = self.Cluster(2, scheduler_port=0, **self.kwargs)
         self.client = Client(self.cluster.scheduler_address)
 
     def tearDown(self):
@@ -33,10 +34,10 @@ def test_start_worker(self):
         assert c == a
 
     def test_context_manager(self):
-        with self.Cluster() as c:
+        with self.Cluster(**self.kwargs) as c:
             with Client(c) as e:
                 assert e.ncores()
 
     def test_no_workers(self):
-        with self.Cluster(0, scheduler_port=0):
+        with self.Cluster(0, scheduler_port=0, **self.kwargs):
             pass
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 034ea3eeadb..e5396486761 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -12,6 +12,7 @@
 import pickle
 import random
 import six
+import warnings
 
 import psutil
 import sortedcontainers
@@ -1033,8 +1034,9 @@ def start_services(self, listen_ip):
                     service.listen((listen_ip, port))
                 self.services[k] = service
             except Exception as e:
-                logger.info("Could not launch service: %r", (k, port),
-                            exc_info=True)
+                warnings.warn("\nCould not launch service '%s' on port %d. " % (k, port) +
+                              "Got the following message:\n\n" + str(e),
+                              stacklevel=3)
 
     def stop_services(self):
         for service in self.services.values():
diff --git a/distributed/tests/py3_test_asyncio.py b/distributed/tests/py3_test_asyncio.py
index a57e1bd7f12..cf60b945b80 100644
--- a/distributed/tests/py3_test_asyncio.py
+++ b/distributed/tests/py3_test_asyncio.py
@@ -49,7 +49,7 @@ async def test_coro_test():
 
 @coro_test
 async def test_asyncio_start_close():
-    async with AioClient(processes=False) as c:
+    async with AioClient(processes=False, diagnostics_port=False) as c:
         assert c.status == 'running'
         # AioClient has installed its AioLoop shim.
         assert isinstance(IOLoop.current(instance=False), BaseAsyncIOLoop)
diff --git a/distributed/tests/test_core.py b/distributed/tests/test_core.py
index ce67817e6a6..66bd7d20058 100644
--- a/distributed/tests/test_core.py
+++ b/distributed/tests/test_core.py
@@ -401,7 +401,7 @@ def test_identity_inproc():
 
 
 def test_ports(loop):
-    port = 9876
+    port = 9877
     server = Server({}, io_loop=loop)
     server.listen(port)
     try:
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index 93ce281bd4f..a23f821f929 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -11,6 +11,7 @@
 import logging
 import logging.config
 import os
+import psutil
 import re
 import shutil
 import signal
@@ -39,11 +40,12 @@
 from tornado.ioloop import IOLoop
 
 from .client import default_client, _global_clients
-from .compatibility import PY3, Empty, WINDOWS
+from .compatibility import PY3, Empty, WINDOWS, PY2
 from .comm.utils import offload
 from .config import initialize_logging
 from .core import connect, rpc, CommClosedError
 from .metrics import time
+from .process import _cleanup_dangling
 from .proctitle import enable_proctitle_on_children
 from .security import Security
 from .utils import (ignoring, log_errors, mp_context, get_ip, get_ipv6,
@@ -129,6 +131,18 @@ def start():
         else:
             is_stopped.wait()
     del _global_workers[:]
+
+    start = time()
+    while set(_global_clients):
+        sleep(0.1)
+        assert time() < start + 5
+
+    _cleanup_dangling()
+
+    if PY2:  # no forkserver, so no extra procs
+        for child in psutil.Process().children(recursive=True):
+            child.terminate()
+
     _global_clients.clear()
 
 
@@ -845,6 +859,7 @@ def coro():
                         thread = threading._active[tid]
                         call_stacks = profile.call_stack(sys._current_frames()[tid])
                         assert False, (thread, call_stacks)
+            _cleanup_dangling()
             return result
 
         return test_func

From dbb529ba6c0acb03fb915eec4c1ff846bb2c8c72 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sun, 19 Aug 2018 18:29:31 -0400
Subject: [PATCH 0060/1550] Add profile page for event loop thread (#2144)

This adds a thread to each event loop that periodically polls the state
of the event loop and maintains a time series of profile information.

This data is served on the profile-server route of the bokeh servers
---
 .../setup_conda_environment.cmd               |   2 +-
 distributed/actor.py                          |   2 +-
 distributed/bokeh/components.py               | 134 ++++++++++++++++++
 distributed/bokeh/scheduler.py                |  16 ++-
 .../bokeh/tests/test_scheduler_bokeh.py       |  13 +-
 distributed/bokeh/worker.py                   |  18 ++-
 distributed/cli/tests/test_dask_scheduler.py  |   2 +-
 distributed/client.py                         |   5 +-
 distributed/comm/tests/test_comms.py          |   7 +-
 distributed/core.py                           |  20 +++
 distributed/deploy/local.py                   |   5 +-
 distributed/process.py                        |   5 +-
 distributed/profile.py                        |  95 ++++++++++++-
 distributed/scheduler.py                      |  10 +-
 distributed/tests/py3_test_pubsub.py          |   4 +-
 distributed/tests/test_client.py              |   6 +-
 distributed/tests/test_metrics.py             |   2 +-
 distributed/tests/test_profile.py             |  33 ++++-
 distributed/tests/test_queues.py              |   8 +-
 distributed/tests/test_scheduler.py           |  12 +-
 distributed/tests/test_utils.py               |   6 +-
 distributed/utils.py                          |   5 +-
 distributed/utils_test.py                     |   5 +-
 23 files changed, 369 insertions(+), 46 deletions(-)

diff --git a/continuous_integration/setup_conda_environment.cmd b/continuous_integration/setup_conda_environment.cmd
index f03441c336e..d8fc2445d27 100644
--- a/continuous_integration/setup_conda_environment.cmd
+++ b/continuous_integration/setup_conda_environment.cmd
@@ -37,7 +37,7 @@ call deactivate
     requests ^
     toolz ^
     tblib ^
-    tornado=4.5 ^
+    tornado=5 ^
     zict ^
     -c conda-forge
 
diff --git a/distributed/actor.py b/distributed/actor.py
index b97c79a6041..85f08dd9efd 100644
--- a/distributed/actor.py
+++ b/distributed/actor.py
@@ -119,7 +119,7 @@ def __getattr__(self, key):
         attr = getattr(self._cls, key)
 
         if self._future and not self._future.status == 'finished':
-            raise ValueError("Worker holding Actor was lost")
+            raise ValueError("Worker holding Actor was lost.  Status: " + self._future.status)
 
         if callable(attr):
             @functools.wraps(attr)
diff --git a/distributed/bokeh/components.py b/distributed/bokeh/components.py
index fe82d52ce80..c2f37844e6e 100644
--- a/distributed/bokeh/components.py
+++ b/distributed/bokeh/components.py
@@ -13,6 +13,7 @@
 from bokeh.plotting import figure
 import dask
 from tornado import gen
+import toolz
 
 from ..diagnostics.progress_stream import nbytes_bar
 from .. import profile
@@ -437,6 +438,7 @@ def cb(attr, old, new):
         self.ts_source = ColumnDataSource({'time': [], 'count': []})
         self.ts_plot = figure(title='Activity over time', height=100,
                               x_axis_type='datetime', active_drag='xbox_select',
+                              y_range=[0, 1 / profile_interval],
                               tools='xpan,xwheel_zoom,xbox_select,reset',
                               **kwargs)
         self.ts_plot.line('time', 'count', source=self.ts_source)
@@ -511,3 +513,135 @@ def cb():
                 self.doc().add_next_tick_callback(lambda: self.update(prof, metadata))
 
         self.server.loop.add_callback(cb)
+
+
+class ProfileServer(DashboardComponent):
+    """ Time plots of the current resource usage on the cluster
+
+    This is two plots, one for CPU and Memory and another for Network I/O
+    """
+
+    def __init__(self, server, doc=None, **kwargs):
+        if doc is not None:
+            self.doc = weakref.ref(doc)
+        self.server = server
+        self.log = self.server.io_loop.profile
+        self.start = None
+        self.stop = None
+        self.ts = {'count': [], 'time': []}
+        self.state = profile.get_profile(self.log)
+        data = profile.plot_data(self.state, profile_interval)
+        self.states = data.pop('states')
+        self.source = ColumnDataSource(data=data)
+
+        changing = [False]  # avoid repeated changes from within callback
+
+        def cb(attr, old, new):
+            if changing[0]:
+                return
+            with log_errors():
+                try:
+                    ind = new['1d']['indices'][0]
+                except IndexError:
+                    return
+                data = profile.plot_data(self.states[ind], profile_interval)
+                del self.states[:]
+                self.states.extend(data.pop('states'))
+                changing[0] = True  # don't recursively trigger callback
+                self.source.data.update(data)
+                self.source.selected = old
+                changing[0] = False
+
+        self.source.on_change('selected', cb)
+
+        self.profile_plot = figure(tools='tap', height=400, **kwargs)
+        r = self.profile_plot.quad('left', 'right', 'top', 'bottom', color='color',
+                                   line_color='black', source=self.source)
+        r.selection_glyph = None
+        r.nonselection_glyph = None
+
+        hover = HoverTool(
+            point_policy="follow_mouse",
+            tooltips="""
+                <div>
+                    <span style="font-size: 14px; font-weight: bold;">Name:</span>&nbsp;
+                    <span style="font-size: 10px; font-family: Monaco, monospace;">@name</span>
+                </div>
+                <div>
+                    <span style="font-size: 14px; font-weight: bold;">Filename:</span>&nbsp;
+                    <span style="font-size: 10px; font-family: Monaco, monospace;">@filename</span>
+                </div>
+                <div>
+                    <span style="font-size: 14px; font-weight: bold;">Line number:</span>&nbsp;
+                    <span style="font-size: 10px; font-family: Monaco, monospace;">@line_number</span>
+                </div>
+                <div>
+                    <span style="font-size: 14px; font-weight: bold;">Line:</span>&nbsp;
+                    <span style="font-size: 10px; font-family: Monaco, monospace;">@line</span>
+                </div>
+                <div>
+                    <span style="font-size: 14px; font-weight: bold;">Time:</span>&nbsp;
+                    <span style="font-size: 10px; font-family: Monaco, monospace;">@time</span>
+                </div>
+                <div>
+                    <span style="font-size: 14px; font-weight: bold;">Percentage:</span>&nbsp;
+                    <span style="font-size: 10px; font-family: Monaco, monospace;">@percentage</span>
+                </div>
+                """
+        )
+        self.profile_plot.add_tools(hover)
+
+        self.profile_plot.xaxis.visible = False
+        self.profile_plot.yaxis.visible = False
+        self.profile_plot.grid.visible = False
+
+        self.ts_source = ColumnDataSource({'time': [], 'count': []})
+        self.ts_plot = figure(title='Activity over time', height=100,
+                              x_axis_type='datetime', active_drag='xbox_select',
+                              y_range=[0, 1 / profile_interval],
+                              tools='xpan,xwheel_zoom,xbox_select,reset',
+                              **kwargs)
+        self.ts_plot.line('time', 'count', source=self.ts_source)
+        self.ts_plot.circle('time', 'count', source=self.ts_source, color=None,
+                            selection_color='orange')
+        self.ts_plot.yaxis.visible = False
+        self.ts_plot.grid.visible = False
+
+        def ts_change(attr, old, new):
+            with log_errors():
+                selected = self.ts_source.selected['1d']['indices']
+                if selected:
+                    start = self.ts_source.data['time'][min(selected)] / 1000
+                    stop = self.ts_source.data['time'][max(selected)] / 1000
+                    self.start, self.stop = min(start, stop), max(start, stop)
+                else:
+                    self.start = self.stop = None
+                self.trigger_update()
+
+        self.ts_source.on_change('selected', ts_change)
+
+        self.reset_button = Button(label="Reset", button_type="success")
+        self.reset_button.on_click(lambda: self.update(self.state))
+
+        self.update_button = Button(label="Update", button_type="success")
+        self.update_button.on_click(self.trigger_update)
+
+        self.root = column(row(self.reset_button, self.update_button,
+                               sizing_mode='scale_width'),
+                           self.profile_plot, self.ts_plot, **kwargs)
+
+    def update(self, state):
+        with log_errors():
+            self.state = state
+            data = profile.plot_data(self.state, profile_interval)
+            self.states = data.pop('states')
+            self.source.data.update(data)
+
+    def trigger_update(self):
+        self.state = profile.get_profile(self.log, start=self.start, stop=self.stop)
+        data = profile.plot_data(self.state, profile_interval)
+        self.states = data.pop('states')
+        self.source.data.update(data)
+        times = [t * 1000 for t, _ in self.log]
+        counts = list(toolz.pluck('count', toolz.pluck(1, self.log)))
+        self.ts_source.data.update({'time': times, 'count': counts})
diff --git a/distributed/bokeh/scheduler.py b/distributed/bokeh/scheduler.py
index 52e693c75d5..27c39674871 100644
--- a/distributed/bokeh/scheduler.py
+++ b/distributed/bokeh/scheduler.py
@@ -30,7 +30,7 @@
     np = False
 
 from . import components
-from .components import DashboardComponent, ProfileTimePlot
+from .components import (DashboardComponent, ProfileTimePlot, ProfileServer)
 from .core import BokehServer
 from .worker import SystemMonitor, counters_doc
 from .utils import transpose
@@ -1164,6 +1164,18 @@ def profile_doc(scheduler, extra, doc):
         prof.trigger_update()
 
 
+def profile_server_doc(scheduler, extra, doc):
+    with log_errors():
+        doc.title = "Dask: Profile of Event Loop"
+        prof = ProfileServer(scheduler, sizing_mode='scale_width', doc=doc)
+        doc.add_root(prof.root)
+        doc.template = template
+        # doc.template_variables['active_page'] = 'profile'
+        doc.template_variables.update(extra)
+
+        prof.trigger_update()
+
+
 class BokehScheduler(BokehServer):
     def __init__(self, scheduler, io_loop=None, prefix='', **kwargs):
         self.scheduler = scheduler
@@ -1184,6 +1196,7 @@ def __init__(self, scheduler, io_loop=None, prefix='', **kwargs):
         tasks = Application(FunctionHandler(partial(tasks_doc, scheduler, self.extra)))
         status = Application(FunctionHandler(partial(status_doc, scheduler, self.extra)))
         profile = Application(FunctionHandler(partial(profile_doc, scheduler, self.extra)))
+        profile_server = Application(FunctionHandler(partial(profile_server_doc, scheduler, self.extra)))
         graph = Application(FunctionHandler(partial(graph_doc, scheduler, self.extra)))
 
         self.apps = {
@@ -1195,6 +1208,7 @@ def __init__(self, scheduler, io_loop=None, prefix='', **kwargs):
             '/tasks': tasks,
             '/status': status,
             '/profile': profile,
+            '/profile-server': profile_server,
             '/graph': graph,
         }
 
diff --git a/distributed/bokeh/tests/test_scheduler_bokeh.py b/distributed/bokeh/tests/test_scheduler_bokeh.py
index 235d85d0c5c..bcd7eae476b 100644
--- a/distributed/bokeh/tests/test_scheduler_bokeh.py
+++ b/distributed/bokeh/tests/test_scheduler_bokeh.py
@@ -22,7 +22,7 @@
                                          MemoryUse, CurrentLoad,
                                          ProcessingHistogram,
                                          NBytesHistogram, WorkerTable,
-                                         GraphPlot)
+                                         GraphPlot, ProfileServer)
 
 from distributed.bokeh import scheduler
 
@@ -389,3 +389,14 @@ def test_GraphPlot_order(c, s, a, b):
     gp.update()
 
     assert gp.node_source.data['state'][gp.layout.index[y.key]] == 'erred'
+
+
+@gen_cluster(client=True,
+             config={'distributed.worker.profile.interval': '10ms',
+                     'distributed.worker.profile.cycle': '50ms'})
+def test_profile_server(c, s, a, b):
+    ptp = ProfileServer(s)
+    ptp.trigger_update()
+    yield gen.sleep(0.200)
+    ptp.trigger_update()
+    assert 2 < len(ptp.ts_source.data['time']) < 20
diff --git a/distributed/bokeh/worker.py b/distributed/bokeh/worker.py
index 7e577979a30..687da327d79 100644
--- a/distributed/bokeh/worker.py
+++ b/distributed/bokeh/worker.py
@@ -15,7 +15,7 @@
 from bokeh.palettes import RdBu
 from toolz import merge, partition_all
 
-from .components import DashboardComponent, ProfileTimePlot
+from .components import DashboardComponent, ProfileTimePlot, ProfileServer
 from .core import BokehServer
 from .utils import transpose
 from ..compatibility import WINDOWS
@@ -616,6 +616,18 @@ def profile_doc(server, extra, doc):
         doc.template_variables.update(extra)
 
 
+def profile_server_doc(server, extra, doc):
+    with log_errors():
+        doc.title = "Dask: Profile of Event Loop"
+        prof = ProfileServer(server, sizing_mode='scale_width', doc=doc)
+        doc.add_root(prof.root)
+        doc.template = template
+        # doc.template_variables['active_page'] = ''
+        doc.template_variables.update(extra)
+
+        prof.trigger_update()
+
+
 class BokehWorker(BokehServer):
     def __init__(self, worker, io_loop=None, prefix='', **kwargs):
         self.worker = worker
@@ -636,12 +648,14 @@ def __init__(self, worker, io_loop=None, prefix='', **kwargs):
         systemmonitor = Application(FunctionHandler(partial(systemmonitor_doc, worker, extra)))
         counters = Application(FunctionHandler(partial(counters_doc, worker, extra)))
         profile = Application(FunctionHandler(partial(profile_doc, worker, extra)))
+        profile_server = Application(FunctionHandler(partial(profile_server_doc, worker, extra)))
 
         self.apps = {'/main': main,
                      '/counters': counters,
                      '/crossfilter': crossfilter,
                      '/system': systemmonitor,
-                     '/profile': profile}
+                     '/profile': profile,
+                     '/profile-server': profile_server}
 
         self.loop = io_loop or worker.loop
         self.server = None
diff --git a/distributed/cli/tests/test_dask_scheduler.py b/distributed/cli/tests/test_dask_scheduler.py
index 456f44c55c4..ac6934f3dc9 100644
--- a/distributed/cli/tests/test_dask_scheduler.py
+++ b/distributed/cli/tests/test_dask_scheduler.py
@@ -50,7 +50,7 @@ def test_hostport(loop):
         def f():
             yield [
                 # The scheduler's main port can't be contacted from the outside
-                assert_can_connect_locally_4(8978, 2.0),
+                assert_can_connect_locally_4(8978, 5.0),
             ]
 
         loop.run_sync(f)
diff --git a/distributed/client.py b/distributed/client.py
index f136b91bcd6..2ee1ffea537 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -1478,7 +1478,10 @@ def wait(k):
                     self._send_to_scheduler({'op': 'report-key',
                                              'key': key})
                 for key in response['keys']:
-                    self.futures[key].reset()
+                    try:
+                        self.futures[key].reset()
+                    except KeyError:  # TODO: verify that this is safe
+                        pass
             else:
                 break
 
diff --git a/distributed/comm/tests/test_comms.py b/distributed/comm/tests/test_comms.py
index fc7a316c575..9eca015ebee 100644
--- a/distributed/comm/tests/test_comms.py
+++ b/distributed/comm/tests/test_comms.py
@@ -733,8 +733,11 @@ def handle_comm(comm):
 
     comm = yield connect(contact_addr)
     comm.write("foo")
-    yield gen.sleep(0.01)
-    assert comm.closed()
+
+    start = time()
+    while not comm.closed():
+        yield gen.sleep(0.01)
+        assert time() < start + 2
 
     comm.close()
     comm.close()
diff --git a/distributed/core.py b/distributed/core.py
index aa8b77984d7..e1543555ce5 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -21,6 +21,7 @@
                    normalize_address,
                    unparse_host_port, get_address_host_port)
 from .metrics import time
+from . import profile
 from .system_monitor import SystemMonitor
 from .utils import (get_traceback, truncate_exception, ignoring, shutting_down,
                     PeriodicCallback, parse_timedelta, has_keyword)
@@ -115,6 +116,25 @@ def __init__(self, handlers, stream_handlers=None, connection_limit=512,
         self.io_loop = io_loop or IOLoop.current()
         self.loop = self.io_loop
 
+        if not hasattr(self.io_loop, 'profile'):
+            ref = weakref.ref(self.io_loop)
+
+            if hasattr(self.io_loop, 'closing'):
+                def stop():
+                    loop = ref()
+                    return loop is None or loop.closing
+            else:
+                def stop():
+                    loop = ref()
+                    return loop is None or loop._closing
+
+            self.io_loop.profile = profile.watch(
+                    omit=('profile.py', 'selectors.py'),
+                    interval=dask.config.get('distributed.worker.profile.interval'),
+                    cycle=dask.config.get('distributed.worker.profile.cycle'),
+                    stop=stop,
+            )
+
         # Statistics counters for various events
         with ignoring(ImportError):
             from .counter import Digest
diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index a277f6d53da..b4e837a2a47 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -287,7 +287,10 @@ def close(self, timeout=20):
                 else:
                     sleep(0.01)
             del self.workers[:]
-            self._loop_runner.run_sync(self._close, callback_timeout=timeout)
+            try:
+                self._loop_runner.run_sync(self._close, callback_timeout=timeout)
+            except RuntimeError:  # IOLoop is closed
+                pass
             self._loop_runner.stop()
         finally:
             self.status = 'closed'
diff --git a/distributed/process.py b/distributed/process.py
index e3a1e2ecbb4..38e3af62c3b 100644
--- a/distributed/process.py
+++ b/distributed/process.py
@@ -5,7 +5,6 @@
 import logging
 import os
 import re
-import sys
 import threading
 import weakref
 
@@ -34,10 +33,10 @@ def _loop_add_callback(loop, func, *args):
 def _call_and_set_future(loop, future, func, *args, **kwargs):
     try:
         res = func(*args, **kwargs)
-    except Exception:
+    except Exception as exc:
         # Tornado futures are not thread-safe, need to
         # set_result() / set_exc_info() from the loop's thread
-        _loop_add_callback(loop, future.set_exc_info, sys.exc_info())
+        _loop_add_callback(loop, future.set_exception, exc)
     else:
         _loop_add_callback(loop, future.set_result, res)
 
diff --git a/distributed/profile.py b/distributed/profile.py
index 46a4e441ebb..71ff6d18205 100644
--- a/distributed/profile.py
+++ b/distributed/profile.py
@@ -24,12 +24,18 @@
                    'children': {...}}}
     }
 """
-
-
-from collections import defaultdict
+import bisect
+from collections import defaultdict, deque
 import linecache
+import sys
+import threading
+from time import sleep
 
-from .utils import format_time, color_of
+import toolz
+
+from .metrics import time
+from .utils import format_time, color_of, parse_timedelta
+from .compatibility import get_thread_identity
 
 
 def identifier(frame):
@@ -64,7 +70,7 @@ def info_frame(frame):
             'line': line}
 
 
-def process(frame, child, state, stop=None):
+def process(frame, child, state, stop=None, omit=None):
     """ Add counts from a frame stack onto existing state
 
     This recursively adds counts to the existing state dictionary and creates
@@ -84,9 +90,14 @@ def process(frame, child, state, stop=None):
      'description': 'root',
      'children': {'...'}}
     """
+    if omit is not None and any(frame.f_code.co_filename.endswith(o) for o in omit):
+        return False
+
     prev = frame.f_back
     if prev is not None and (stop is None or not prev.f_code.co_filename.endswith(stop)):
         state = process(prev, frame, state, stop=stop)
+        if state is False:
+            return False
 
     ident = identifier(frame)
 
@@ -214,3 +225,77 @@ def traverse(state, start, stop, height):
             'name': names,
             'time': times,
             'percentage': percentages}
+
+
+def _watch(thread_id, log, interval='20ms', cycle='2s', omit=None,
+           stop=lambda: False):
+    interval = parse_timedelta(interval)
+    cycle = parse_timedelta(cycle)
+
+    recent = create()
+    last = time()
+
+    while not stop():
+        if time() > last + cycle:
+            log.append((time(), recent))
+            recent = create()
+            last = time()
+        try:
+            frame = sys._current_frames()[thread_id]
+        except KeyError:
+            return
+
+        process(frame, None, recent, omit=omit)
+        sleep(interval)
+
+
+def watch(thread_id=None, interval='20ms', cycle='2s', maxlen=1000, omit=None,
+          stop=lambda: False):
+    if thread_id is None:
+        thread_id = get_thread_identity()
+
+    log = deque(maxlen=maxlen)
+
+    thread = threading.Thread(target=_watch,
+                              name='Profile',
+                              kwargs={'thread_id': thread_id,
+                                      'interval': interval,
+                                      'cycle': cycle,
+                                      'log': log,
+                                      'omit': omit,
+                                      'stop': stop})
+    thread.daemon = True
+    thread.start()
+
+    return log
+
+
+def get_profile(history, recent=None, start=None, stop=None, key=None):
+    now = time()
+    if start is None:
+        istart = 0
+    else:
+        istart = bisect.bisect_left(history, (start,))
+
+    if stop is None:
+        istop = None
+    else:
+        istop = bisect.bisect_right(history, (stop,)) + 1
+        if istop >= len(history):
+            istop = None  # include end
+
+    if istart == 0 and istop is None:
+        history = list(history)
+    else:
+        iistop = len(history) if istop is None else istop
+        history = [history[i] for i in range(istart, iistop)]
+
+    prof = merge(*toolz.pluck(1, history))
+
+    if not history:
+        return create()
+
+    if recent:
+        prof = merge(prof, recent)
+
+    return prof
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index e5396486761..f3a18612a25 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1376,7 +1376,7 @@ def update_graph(self, client=None, tasks=None, keys=None,
                             stack.append(dep)
 
             for d in done:
-                del tasks[d]
+                tasks.pop(d, None)
                 del dependencies[d]
 
         # Get or create task states
@@ -2804,7 +2804,13 @@ def update_data(self, comm=None, who_has=None, nbytes=None, client=None,
     def report_on_key(self, key=None, ts=None, client=None):
         assert (key is None) + (ts is None) == 1, (key, ts)
         if ts is None:
-            ts = self.tasks[key]
+            try:
+                ts = self.tasks[key]
+            except KeyError:
+                self.report({'op': 'cancelled-key',
+                             'key': key},
+                            client=client)
+                return
         else:
             key = ts.key
         if ts.state == 'forgotten':
diff --git a/distributed/tests/py3_test_pubsub.py b/distributed/tests/py3_test_pubsub.py
index 33d9477e92e..ede8023801b 100644
--- a/distributed/tests/py3_test_pubsub.py
+++ b/distributed/tests/py3_test_pubsub.py
@@ -13,7 +13,7 @@ async def publish():
         i = 0
         while True:
             await gen.sleep(0.01)
-            pub.put(i)
+            pub._put(i)
             i += 1
 
     def f(_):
@@ -32,4 +32,4 @@ def f(_):
         # assert r == [x, x + 1, x + 2, x + 3, x + 4]
 
         assert len(r) == 5
-        assert all(r[i] < r[i + 1] for i in range(0, 4))
+        assert all(r[i] < r[i + 1] for i in range(0, 4)), r
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 0e099ccb678..dcdfafc884d 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -2049,7 +2049,6 @@ def test_waiting_data(c, s, a, b):
 @gen_cluster()
 def test_multi_client(s, a, b):
     c = yield Client((s.ip, s.port), asynchronous=True)
-
     f = yield Client((s.ip, s.port), asynchronous=True)
 
     assert set(s.client_comms) == {c.id, f.id}
@@ -2080,7 +2079,10 @@ def test_multi_client(s, a, b):
 
     yield f.close()
 
-    assert not s.tasks
+    start = time()
+    while s.tasks:
+        yield gen.sleep(0.01)
+        assert time() < start + 2, s.tasks
 
 
 def long_running_client_connection(address):
diff --git a/distributed/tests/test_metrics.py b/distributed/tests/test_metrics.py
index 290ac8d8a23..84b7c180993 100644
--- a/distributed/tests/test_metrics.py
+++ b/distributed/tests/test_metrics.py
@@ -35,7 +35,7 @@ def test_process_time():
     t.start()
     t.join()
     dt = metrics.process_time() - start
-    assert dt >= 0.08
+    assert dt >= 0.05
 
     if PY3:
         # Sleep time not counted
diff --git a/distributed/tests/test_profile.py b/distributed/tests/test_profile.py
index 48aa5527823..2101d2a1669 100644
--- a/distributed/tests/test_profile.py
+++ b/distributed/tests/test_profile.py
@@ -1,11 +1,12 @@
 import sys
 import time
 from toolz import first
-from threading import Thread
+import threading
 
-from distributed.profile import (process, merge, create, call_stack,
-        identifier)
 from distributed.compatibility import get_thread_identity
+from distributed import metrics
+from distributed.profile import (process, merge, create, call_stack,
+        identifier, watch)
 
 
 def test_basic():
@@ -20,7 +21,7 @@ def test_f():
             test_g()
             test_h()
 
-    thread = Thread(target=test_f)
+    thread = threading.Thread(target=test_f)
     thread.daemon = True
     thread.start()
 
@@ -113,3 +114,27 @@ def test_identifier():
     frame = sys._current_frames()[get_thread_identity()]
     assert identifier(frame) == identifier(frame)
     assert identifier(None) == identifier(None)
+
+
+def test_watch():
+    start = metrics.time()
+
+    def stop():
+        return metrics.time() > start + 0.500
+
+    start_threads = threading.active_count()
+
+    log = watch(interval='10ms', cycle='50ms', stop=stop)
+
+    start = metrics.time()  # wait until thread starts up
+    while threading.active_count() <= start_threads:
+        assert metrics.time() < start + 2
+        time.sleep(0.01)
+
+    time.sleep(0.5)
+    assert 1 < len(log) < 10
+
+    start = metrics.time()
+    while threading.active_count() > start_threads:
+        assert metrics.time() < start + 2
+        time.sleep(0.01)
diff --git a/distributed/tests/test_queues.py b/distributed/tests/test_queues.py
index a5a8d63d8fa..913434d6909 100644
--- a/distributed/tests/test_queues.py
+++ b/distributed/tests/test_queues.py
@@ -262,14 +262,14 @@ def test_timeout(c, s, a, b):
 
     start = time()
     with pytest.raises(gen.TimeoutError):
-        yield q.get(timeout=0.1)
+        yield q.get(timeout=0.3)
     stop = time()
-    assert 0.1 < stop - start < 2.0
+    assert 0.2 < stop - start < 2.0
 
     yield q.put(1)
 
     start = time()
     with pytest.raises(gen.TimeoutError):
-        yield q.put(2, timeout=0.1)
+        yield q.put(2, timeout=0.3)
     stop = time()
-    assert 0.05 < stop - start < 2.0
+    assert 0.1 < stop - start < 2.0
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 5a80faf8f83..259ca100f19 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -732,26 +732,28 @@ def test_file_descriptors(c, s):
         assert time() < start + 3
 
 
+@slow
 @nodebug
 @gen_cluster(client=True)
 def test_learn_occupancy(c, s, a, b):
-    futures = c.map(slowinc, range(1000), delay=0.01)
+    futures = c.map(slowinc, range(1000), delay=0.2)
     while sum(len(ts.who_has) for ts in s.tasks.values()) < 10:
         yield gen.sleep(0.01)
 
-    assert 1 < s.total_occupancy < 40
+    assert 100 < s.total_occupancy < 1000
     for w in [a, b]:
-        assert 1 < s.workers[w.address].occupancy < 20
+        assert 50 < s.workers[w.address].occupancy < 700
 
 
+@slow
 @nodebug
 @gen_cluster(client=True)
 def test_learn_occupancy_2(c, s, a, b):
-    future = c.map(slowinc, range(1000), delay=0.1)
+    future = c.map(slowinc, range(1000), delay=0.2)
     while not any(ts.who_has for ts in s.tasks.values()):
         yield gen.sleep(0.01)
 
-    assert 50 < s.total_occupancy < 200
+    assert 100 < s.total_occupancy < 1000
 
 
 @gen_cluster(client=True)
diff --git a/distributed/tests/test_utils.py b/distributed/tests/test_utils.py
index 60b5105a78d..b0f75e37d5c 100644
--- a/distributed/tests/test_utils.py
+++ b/distributed/tests/test_utils.py
@@ -514,16 +514,16 @@ def test_parse_timedelta():
 def test_all_exceptions_logging():
     @gen.coroutine
     def throws():
-        raise Exception('foo')
+        raise Exception('foo1234')
 
     with captured_logger('') as sio:
         try:
             yield All([throws() for _ in range(5)],
-                    quiet_exceptions=Exception)
+                      quiet_exceptions=Exception)
         except Exception:
             pass
 
         import gc; gc.collect()
         yield gen.sleep(0.1)
 
-    assert not sio.getvalue()
+    assert 'foo1234' not in sio.getvalue()
diff --git a/distributed/utils.py b/distributed/utils.py
index 193005dbc83..1e9b9d69ae0 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -358,12 +358,11 @@ def run_loop(loop=self._loop):
             finally:
                 done_evt.set()
 
-        thread = threading.Thread(target=run_loop,
-                                  name="IO loop")
+        thread = threading.Thread(target=run_loop, name="IO loop")
         thread.daemon = True
         thread.start()
 
-        loop_evt.wait(timeout=1000)
+        loop_evt.wait(timeout=10)
         self._started = True
 
         actual_thread = in_thread[0]
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index a23f821f929..d65b1abcb80 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -637,7 +637,10 @@ def cluster(nworkers=2, nanny=False, worker_kwargs={}, active_rpc_timeout=1,
             else:
                 client.close()
 
-    assert not ws
+    start = time()
+    while list(ws):
+        sleep(0.01)
+        assert time() < start + 1, 'Workers still around after one second'
 
 
 @gen.coroutine

From 177dfb891089cc872bab34fb8369d75cae13bc0a Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sun, 19 Aug 2018 18:37:43 -0400
Subject: [PATCH 0061/1550] Use dispatch for dask serialization, also add
 sklearn, pytorch (#2175)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Use dispatch for dask serialization

This allows people to register serialization functions using decorator syntax

```python
@dask_serialize.register(np.ndarray)
def serialize_array(x):
    ...

@dask_deserialize.register(np.ndarray)
def serialize_array(header, frames):
    ...
```

This also means that inheritance turns on by default
(which is both good and bad)

* add torch serialization

```
In [1]: import torchvision

In [2]: from distributed.protocol import serialize, deserialize

In [3]: import pickle

In [4]: model = torchvision.models.resnet50()

In [5]: %time header, frames = serialize(model)
CPU times: user 46.9 ms, sys: 0 ns, total: 46.9 ms
Wall time: 46.3 ms

In [6]: %timeit serialize(model)
19.1 ms ± 1.14 ms per loop (mean ± std. dev. of 7 runs, 100 loops each)

In [7]: %timeit deserialize(header, frames)  # most of this seems to be
torch.Tensor(numpy_array)
64.2 ms ± 1.11 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)

In [8]: %time b = pickle.dumps(model)
CPU times: user 77.1 ms, sys: 68.2 ms, total: 145 ms
Wall time: 142 ms

In [9]: %timeit pickle.dumps(model)
108 ms ± 583 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)

In [10]: %timeit pickle.loads(b)
111 ms ± 1.9 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)
```
---
 .travis.yml                                  |   2 +-
 distributed/protocol/__init__.py             |  37 +++-
 distributed/protocol/arrow.py                |  36 ++--
 distributed/protocol/h5py.py                 |  22 +--
 distributed/protocol/keras.py                |  12 +-
 distributed/protocol/netcdf4.py              |  32 ++--
 distributed/protocol/numpy.py                |   7 +-
 distributed/protocol/serialize.py            | 175 ++++++++++++++-----
 distributed/protocol/sparse.py               |  11 +-
 distributed/protocol/tests/test_arrow.py     |   8 -
 distributed/protocol/tests/test_serialize.py |  34 ++--
 distributed/protocol/tests/test_sklearn.py   |  19 ++
 distributed/protocol/tests/test_torch.py     |  33 ++++
 distributed/protocol/torch.py                |  56 ++++++
 distributed/tests/test_client.py             |   6 +-
 distributed/tests/test_nanny.py              |   3 +-
 distributed/utils.py                         |   5 +
 distributed/utils_test.py                    |   2 +-
 docs/source/serialization.rst                |  37 +++-
 19 files changed, 364 insertions(+), 173 deletions(-)
 create mode 100644 distributed/protocol/tests/test_sklearn.py
 create mode 100644 distributed/protocol/tests/test_torch.py
 create mode 100644 distributed/protocol/torch.py

diff --git a/.travis.yml b/.travis.yml
index def0659e6a7..2dd96098c50 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -8,7 +8,7 @@ env:
   matrix:
     - PYTHON=2.7 TESTS=true PACKAGES="python-blosc futures faulthandler"
     - PYTHON=3.5.4 TESTS=true COVERAGE=true PACKAGES=python-blosc CRICK=true
-    - PYTHON=3.6 TESTS=true
+    - PYTHON=3.6 TESTS=true PACKAGES="scikit-learn"
 
 matrix:
   fast_finish: true
diff --git a/distributed/protocol/__init__.py b/distributed/protocol/__init__.py
index 01ac7e8464a..bd8f7331c8e 100644
--- a/distributed/protocol/__init__.py
+++ b/distributed/protocol/__init__.py
@@ -6,39 +6,60 @@
 from .core import (dumps, loads, maybe_compress, decompress, msgpack)
 from .serialize import (
     serialize, deserialize, nested_deserialize, Serialize, Serialized,
-    to_serialize, register_serialization, register_serialization_lazy,
+    to_serialize, register_serialization, dask_serialize, dask_deserialize,
     serialize_bytes, deserialize_bytes, serialize_bytelist,
-    register_serialization_family,
+    register_serialization_family, register_generic,
 )
 
 from ..utils import ignoring
 
 
-@partial(register_serialization_lazy, "numpy")
+@dask_serialize.register_lazy("numpy")
+@dask_deserialize.register_lazy("numpy")
 def _register_numpy():
     from . import numpy
 
 
-@partial(register_serialization_lazy, "h5py")
+@dask_serialize.register_lazy("h5py")
+@dask_deserialize.register_lazy("h5py")
 def _register_h5py():
     from . import h5py
 
 
-@partial(register_serialization_lazy, "netCDF4")
+@dask_serialize.register_lazy("netCDF4")
+@dask_deserialize.register_lazy("netCDF4")
 def _register_netcdf4():
     from . import netcdf4
 
 
-@partial(register_serialization_lazy, "keras")
+@dask_serialize.register_lazy("keras")
+@dask_deserialize.register_lazy("keras")
 def _register_keras():
     from . import keras
 
 
-@partial(register_serialization_lazy, "sparse")
+@dask_serialize.register_lazy("sparse")
+@dask_deserialize.register_lazy("sparse")
 def _register_sparse():
     from . import sparse
 
 
-@partial(register_serialization_lazy, "pyarrow")
+@dask_serialize.register_lazy("pyarrow")
+@dask_deserialize.register_lazy("pyarrow")
 def _register_arrow():
     from . import arrow
+
+
+@dask_serialize.register_lazy("sklearn")
+@dask_deserialize.register_lazy("sklearn")
+def _register_sklearn():
+    import sklearn.base
+    register_generic(sklearn.base.BaseEstimator)
+
+
+@dask_serialize.register_lazy("torch")
+@dask_deserialize.register_lazy("torch")
+@dask_serialize.register_lazy("torchvision")
+@dask_deserialize.register_lazy("torchvision")
+def _register_torch():
+    from . import torch
diff --git a/distributed/protocol/arrow.py b/distributed/protocol/arrow.py
index 87c5d05c99f..c90ba190cfc 100644
--- a/distributed/protocol/arrow.py
+++ b/distributed/protocol/arrow.py
@@ -1,12 +1,14 @@
 from __future__ import print_function, division, absolute_import
 
-from .serialize import register_serialization
+from .serialize import dask_serialize, dask_deserialize
 
+import pyarrow
 
+
+@dask_serialize.register(pyarrow.RecordBatch)
 def serialize_batch(batch):
-    import pyarrow as pa
-    sink = pa.BufferOutputStream()
-    writer = pa.RecordBatchStreamWriter(sink, batch.schema)
+    sink = pyarrow.BufferOutputStream()
+    writer = pyarrow.RecordBatchStreamWriter(sink, batch.schema)
     writer.write_batch(batch)
     writer.close()
     buf = sink.get_result()
@@ -15,17 +17,17 @@ def serialize_batch(batch):
     return header, frames
 
 
+@dask_deserialize.register(pyarrow.RecordBatch)
 def deserialize_batch(header, frames):
-    import pyarrow as pa
     blob = frames[0]
-    reader = pa.RecordBatchStreamReader(pa.BufferReader(blob))
+    reader = pyarrow.RecordBatchStreamReader(pyarrow.BufferReader(blob))
     return reader.read_next_batch()
 
 
+@dask_serialize.register(pyarrow.Table)
 def serialize_table(tbl):
-    import pyarrow as pa
-    sink = pa.BufferOutputStream()
-    writer = pa.RecordBatchStreamWriter(sink, tbl.schema)
+    sink = pyarrow.BufferOutputStream()
+    writer = pyarrow.RecordBatchStreamWriter(sink, tbl.schema)
     writer.write_table(tbl)
     writer.close()
     buf = sink.get_result()
@@ -34,20 +36,8 @@ def serialize_table(tbl):
     return header, frames
 
 
+@dask_deserialize.register(pyarrow.Table)
 def deserialize_table(header, frames):
-    import pyarrow as pa
     blob = frames[0]
-    reader = pa.RecordBatchStreamReader(pa.BufferReader(blob))
+    reader = pyarrow.RecordBatchStreamReader(pyarrow.BufferReader(blob))
     return reader.read_all()
-
-
-register_serialization(
-    'pyarrow.lib.RecordBatch',
-    serialize_batch,
-    deserialize_batch
-)
-register_serialization(
-    'pyarrow.lib.Table',
-    serialize_table,
-    deserialize_table
-)
diff --git a/distributed/protocol/h5py.py b/distributed/protocol/h5py.py
index 81a83cffcea..9936920a759 100644
--- a/distributed/protocol/h5py.py
+++ b/distributed/protocol/h5py.py
@@ -1,39 +1,31 @@
 from __future__ import print_function, division, absolute_import
 
-from .serialize import register_serialization
+from .serialize import dask_serialize, dask_deserialize
 
+import h5py
 
+
+@dask_serialize.register(h5py.File)
 def serialize_h5py_file(f):
     if f.mode != 'r':
         raise ValueError("Can only serialize read-only h5py files")
     return {'filename': f.filename}, []
 
 
+@dask_deserialize.register(h5py.File)
 def deserialize_h5py_file(header, frames):
     import h5py
     return h5py.File(header['filename'], mode='r')
 
 
-register_serialization('h5py._hl.files.File',
-                       serialize_h5py_file,
-                       deserialize_h5py_file)
-
-
+@dask_serialize.register((h5py.Group, h5py.Dataset))
 def serialize_h5py_dataset(x):
     header, _ = serialize_h5py_file(x.file)
     header['name'] = x.name
     return header, []
 
 
+@dask_deserialize.register((h5py.Group, h5py.Dataset))
 def deserialize_h5py_dataset(header, frames):
     file = deserialize_h5py_file(header, frames)
     return file[header['name']]
-
-
-register_serialization('h5py._hl.dataset.Dataset',
-                       serialize_h5py_dataset,
-                       deserialize_h5py_dataset)
-
-register_serialization('h5py._hl.group.Group',
-                       serialize_h5py_dataset,
-                       deserialize_h5py_dataset)
diff --git a/distributed/protocol/keras.py b/distributed/protocol/keras.py
index 2217380fc80..a5437f60e18 100644
--- a/distributed/protocol/keras.py
+++ b/distributed/protocol/keras.py
@@ -1,8 +1,11 @@
 from __future__ import print_function, division, absolute_import
 
-from .serialize import register_serialization, serialize, deserialize
+from .serialize import dask_serialize, dask_deserialize, serialize, deserialize
 
+import keras
 
+
+@dask_serialize.register(keras.Model)
 def serialize_keras_model(model):
     import keras
     if keras.__version__ < '1.2.0':
@@ -18,6 +21,7 @@ def serialize_keras_model(model):
     return header, frames
 
 
+@dask_deserialize.register(keras.Model)
 def deserialize_keras_model(header, frames):
     from keras.models import model_from_config
     n = 0
@@ -29,9 +33,3 @@ def deserialize_keras_model(header, frames):
     model = model_from_config(header)
     model.set_weights(weights)
     return model
-
-
-for module in ['keras', 'tensorflow.contrib.keras.python.keras']:
-    for name in ['engine.training.Model', 'models.Model', 'models.Sequential']:
-        register_serialization('.'.join([module, name]), serialize_keras_model,
-                               deserialize_keras_model)
diff --git a/distributed/protocol/netcdf4.py b/distributed/protocol/netcdf4.py
index 2154358e866..06711ad03cb 100644
--- a/distributed/protocol/netcdf4.py
+++ b/distributed/protocol/netcdf4.py
@@ -1,47 +1,39 @@
 from __future__ import print_function, division, absolute_import
 
-from .serialize import register_serialization, serialize, deserialize
+from .serialize import dask_serialize, dask_deserialize, serialize, deserialize
 
-try:
-    import netCDF4
-    HAS_NETCDF4 = True
-except ImportError:
-    HAS_NETCDF4 = False
+import netCDF4
 
 
+@dask_serialize.register(netCDF4.Dataset)
 def serialize_netcdf4_dataset(ds):
     # assume mode is read-only
     return {'filename': ds.filepath()}, []
 
 
+@dask_deserialize.register(netCDF4.Dataset)
 def deserialize_netcdf4_dataset(header, frames):
-    import netCDF4
     return netCDF4.Dataset(header['filename'], mode='r')
 
 
-if HAS_NETCDF4:
-    register_serialization(netCDF4.Dataset, serialize_netcdf4_dataset,
-                           deserialize_netcdf4_dataset)
-
-
+@dask_serialize.register(netCDF4.Variable)
 def serialize_netcdf4_variable(x):
     header, _ = serialize(x.group())
     header['parent-type'] = header['type']
+    header['parent-type-serialized'] = header['type-serialized']
     header['name'] = x.name
     return header, []
 
 
+@dask_deserialize.register(netCDF4.Variable)
 def deserialize_netcdf4_variable(header, frames):
     header['type'] = header['parent-type']
+    header['type-serialized'] = header['parent-type-serialized']
     parent = deserialize(header, frames)
     return parent.variables[header['name']]
 
 
-if HAS_NETCDF4:
-    register_serialization(netCDF4.Variable, serialize_netcdf4_variable,
-                           deserialize_netcdf4_variable)
-
-
+@dask_serialize.register(netCDF4.Group)
 def serialize_netcdf4_group(g):
     parent = g
     while parent.parent:
@@ -51,11 +43,7 @@ def serialize_netcdf4_group(g):
     return header, []
 
 
+@dask_deserialize.register(netCDF4.Group)
 def deserialize_netcdf4_group(header, frames):
     file = deserialize_netcdf4_dataset(header, frames)
     return file[header['path']]
-
-
-if HAS_NETCDF4:
-    register_serialization(netCDF4.Group, serialize_netcdf4_group,
-                           deserialize_netcdf4_group)
diff --git a/distributed/protocol/numpy.py b/distributed/protocol/numpy.py
index 5998294fd51..d6fc52a4e4d 100644
--- a/distributed/protocol/numpy.py
+++ b/distributed/protocol/numpy.py
@@ -11,7 +11,7 @@
     blosc = False
 
 from .utils import frame_split_size, merge_frames
-from .serialize import register_serialization
+from .serialize import dask_serialize, dask_deserialize
 from . import pickle
 
 from ..utils import log_errors
@@ -28,6 +28,7 @@ def itemsize(dt):
     return result
 
 
+@dask_serialize.register(np.ndarray)
 def serialize_numpy_ndarray(x):
     if x.dtype.hasobject:
         header = {'pickle': True}
@@ -88,6 +89,7 @@ def serialize_numpy_ndarray(x):
     return header, frames
 
 
+@dask_deserialize.register(np.ndarray)
 def deserialize_numpy_ndarray(header, frames):
     with log_errors():
         if len(frames) > 1:
@@ -106,6 +108,3 @@ def deserialize_numpy_ndarray(header, frames):
                        strides=header['strides'])
 
         return x
-
-
-register_serialization(np.ndarray, serialize_numpy_ndarray, deserialize_numpy_ndarray)
diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index cb3b802c504..3f0da622995 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -2,6 +2,7 @@
 from functools import partial
 import traceback
 
+import dask
 from dask.base import normalize_token
 try:
     from cytoolz import valmap, get_in
@@ -17,41 +18,35 @@
 from .utils import unpack_frames, pack_frames_prelude, frame_split_size
 
 
-class_serializers = {}
-
 lazy_registrations = {}
 
 
+dask_serialize = dask.utils.Dispatch('dask_serialize')
+dask_deserialize = dask.utils.Dispatch('dask_deserialize')
+
+
 def dask_dumps(x, context=None):
     """Serialise object using the class-based registry"""
-    typ = typename(type(x))
-    if typ in class_serializers:
-        dumps, loads, has_context = class_serializers[typ]
-        if has_context:
-            header, frames = dumps(x, context=context)
-        else:
-            header, frames = dumps(x)
-        header['type'] = typ
-        header['serializer'] = 'dask'
-        return header, frames
-    elif _find_lazy_registration(typ):
-        return dask_dumps(x)  # recurse
+    type_name = typename(type(x))
+    try:
+        dumps = dask_serialize.dispatch(type(x))
+    except TypeError:
+        raise NotImplementedError(type_name)
+    if has_keyword(dumps, 'context'):
+        header, frames = dumps(x, context=context)
     else:
-        raise NotImplementedError(typ)
-
+        header, frames = dumps(x)
 
-def dask_loads(header, frames):
-    typ = header['type']
+    header['type'] = type_name
+    header['type-serialized'] = pickle.dumps(type(x))
+    header['serializer'] = 'dask'
+    return header, frames
 
-    if typ not in class_serializers:
-        _find_lazy_registration(typ)
 
-    try:
-        dumps, loads, _ = class_serializers[typ]
-    except KeyError:
-        raise TypeError("Serialization for type %s not found" % typ)
-    else:
-        return loads(header, frames)
+def dask_loads(header, frames):
+    typ = pickle.loads(header['type-serialized'])
+    loads = dask_deserialize.dispatch(typ)
+    return loads(header, frames)
 
 
 def pickle_dumps(x):
@@ -406,20 +401,20 @@ def register_serialization(cls, serialize, deserialize):
     serialize
     deserialize
     """
-    if isinstance(cls, type):
-        name = typename(cls)
-    elif isinstance(cls, str):
-        name = cls
-    class_serializers[name] = (serialize,
-                               deserialize,
-                               has_keyword(serialize, 'context'))
+    if isinstance(cls, str):
+        raise TypeError(
+            "Strings are no longer accepted for type registration. "
+            "Use dask_serialize.register_lazy instead"
+        )
+    dask_serialize.register(cls)(serialize)
+    dask_deserialize.register(cls)(deserialize)
 
 
 def register_serialization_lazy(toplevel, func):
     """Register a registration function to be called if *toplevel*
     module is ever loaded.
     """
-    lazy_registrations[toplevel] = func
+    raise Exception("Serialization registration has changed. See documentation")
 
 
 def typename(typ):
@@ -434,33 +429,117 @@ def typename(typ):
     return typ.__module__ + '.' + typ.__name__
 
 
-def _find_lazy_registration(typename):
-    toplevel, _, _ = typename.partition('.')
-    if toplevel in lazy_registrations:
-        lazy_registrations.pop(toplevel)()
-        return True
-    else:
-        return False
-
-
 @partial(normalize_token.register, Serialized)
 def normalize_Serialized(o):
     return [o.header] + o.frames  # for dask.base.tokenize
 
 
 # Teach serialize how to handle bytestrings
+@dask_serialize.register((bytes, bytearray))
 def _serialize_bytes(obj):
     header = {}  # no special metadata
     frames = [obj]
     return header, frames
 
 
+@dask_deserialize.register((bytes, bytearray))
 def _deserialize_bytes(header, frames):
     return frames[0]
 
 
-# NOTE: using the same exact serialization means a bytes object may be
-# deserialized as bytearray or vice-versa...  Not sure this is a problem
-# in practice.
-register_serialization(bytes, _serialize_bytes, _deserialize_bytes)
-register_serialization(bytearray, _serialize_bytes, _deserialize_bytes)
+#########################
+# Descend into __dict__ #
+#########################
+
+
+def _is_msgpack_serializable(v):
+    typ = type(v)
+    return (typ is str or typ is int or typ is float or
+            isinstance(v, dict) and all(map(_is_msgpack_serializable, v.values()))
+                                and all(typ is str for x in v.keys()) or
+            isinstance(v, (list, tuple)) and all(map(_is_msgpack_serializable, v)))
+
+
+def serialize_object_with_dict(est):
+    header = {
+        'serializer': 'dask',
+        'type-serialized': pickle.dumps(type(est)),
+        'simple': {},
+        'complex': {}
+    }
+    frames = []
+
+    if isinstance(est, dict):
+        d = est
+    else:
+        d = est.__dict__
+
+    for k, v in d.items():
+        if _is_msgpack_serializable(v):
+            header['simple'][k] = v
+        else:
+            if isinstance(v, dict):
+                h, f = serialize_object_with_dict(v)
+            else:
+                h, f = serialize(v)
+            header['complex'][k] = {'header': h,
+                                    'start': len(frames),
+                                    'stop': len(frames) + len(f)}
+            frames += f
+    return header, frames
+
+
+def deserialize_object_with_dict(header, frames):
+    cls = pickle.loads(header['type-serialized'])
+    if issubclass(cls, dict):
+        dd = obj = {}
+    else:
+        obj = object.__new__(cls)
+        dd = obj.__dict__
+    dd.update(header['simple'])
+    for k, d in header['complex'].items():
+        h = d['header']
+        f = frames[d['start']: d['stop']]
+        v = deserialize(h, f)
+        dd[k] = v
+
+    return obj
+
+
+dask_deserialize.register(dict)(deserialize_object_with_dict)
+
+
+def register_generic(cls):
+    """ Register dask_(de)serialize to traverse through __dict__
+
+    Normally when registering new classes for Dask's custom serialization you
+    need to manage headers and frames, which can be tedious.  If all you want
+    to do is traverse through your object and apply serialize to all of your
+    object's attributes then this function may provide an easier path.
+
+    This registers a class for the custom Dask serialization family.  It
+    serializes it by traversing through its __dict__ of attributes and applying
+    ``serialize`` and ``deserialize`` recursively.  It collects a set of frames
+    and keeps small attributes in the header.  Deserialization reverses this
+    process.
+
+    This is a good idea if the following hold:
+
+    1.  Most of the bytes of your object are composed of data types that Dask's
+        custom serializtion already handles well, like Numpy arrays.
+    2.  Your object doesn't require any special constructor logic, other than
+        object.__new__(cls)
+
+    Examples
+    --------
+    >>> import sklearn.base
+    >>> from distributed.protocol import register_generic
+    >>> register_generic(sklearn.base.BaseEstimator)
+
+    See Also
+    --------
+    dask_serialize
+    dask_deserialize
+    """
+    dask_serialize.register(cls)(serialize_object_with_dict)
+    dask_deserialize.register(cls)(deserialize_object_with_dict)
diff --git a/distributed/protocol/sparse.py b/distributed/protocol/sparse.py
index d8b7a42c2f8..ca0c6f38a79 100644
--- a/distributed/protocol/sparse.py
+++ b/distributed/protocol/sparse.py
@@ -1,8 +1,11 @@
 from __future__ import print_function, division, absolute_import
 
-from .serialize import register_serialization, serialize, deserialize
+from .serialize import dask_serialize, dask_deserialize, serialize, deserialize
 
+import sparse
 
+
+@dask_serialize.register(sparse.COO)
 def serialize_sparse(x):
     coords_header, coords_frames = serialize(x.coords)
     data_header, data_frames = serialize(x.data)
@@ -14,8 +17,8 @@ def serialize_sparse(x):
     return header, coords_frames + data_frames
 
 
+@dask_deserialize.register(sparse.COO)
 def deserialize_sparse(header, frames):
-    import sparse
 
     coords_frames = frames[:header['nframes'][0]]
     data_frames = frames[header['nframes'][0]:]
@@ -26,7 +29,3 @@ def deserialize_sparse(header, frames):
     shape = header['shape']
 
     return sparse.COO(coords, data, shape=shape)
-
-
-register_serialization('sparse.core.COO', serialize_sparse, deserialize_sparse)  # version 0.1
-register_serialization('sparse.coo.COO', serialize_sparse, deserialize_sparse)  # version 0.2
diff --git a/distributed/protocol/tests/test_arrow.py b/distributed/protocol/tests/test_arrow.py
index 6f014bae323..eca8de9f1a3 100644
--- a/distributed/protocol/tests/test_arrow.py
+++ b/distributed/protocol/tests/test_arrow.py
@@ -5,7 +5,6 @@
 
 from distributed.utils_test import gen_cluster
 from distributed.protocol import deserialize, serialize
-from distributed.protocol.serialize import class_serializers, typename
 
 
 df = pd.DataFrame({'A': list('abc'), 'B': [1,2,3]})
@@ -22,13 +21,6 @@ def test_roundtrip(obj):
     assert obj.equals(new_obj)
 
 
-@pytest.mark.parametrize('obj', [batch, tbl], ids=["RecordBatch", "Table"])
-def test_typename(obj):
-    # The typename used to register the custom serialization is hardcoded
-    # ensure that the typename hasn't changed
-    assert typename(type(obj)) in class_serializers
-
-
 def echo(arg):
     return arg
 
diff --git a/distributed/protocol/tests/test_serialize.py b/distributed/protocol/tests/test_serialize.py
index dc7377385ea..4e9062cd044 100644
--- a/distributed/protocol/tests/test_serialize.py
+++ b/distributed/protocol/tests/test_serialize.py
@@ -311,28 +311,24 @@ def check(dask_worker):
 def test_context_specific_serialization_class(c, s, a, b):
     register_serialization(MyObject, my_dumps, my_loads)
 
-    try:
-        # Create the object on A, force communication to B
-        x = c.submit(MyObject, x=1, y=2, workers=a.address)
-        y = c.submit(lambda x: x, x, workers=b.address)
+    # Create the object on A, force communication to B
+    x = c.submit(MyObject, x=1, y=2, workers=a.address)
+    y = c.submit(lambda x: x, x, workers=b.address)
 
-        yield wait(y)
+    yield wait(y)
 
-        key = y.key
+    key = y.key
 
-        def check(dask_worker):
-            # Get the context from the object stored on B
-            my_obj = dask_worker.data[key]
-            return my_obj.context
+    def check(dask_worker):
+        # Get the context from the object stored on B
+        my_obj = dask_worker.data[key]
+        return my_obj.context
 
-        result = yield c.run(check, workers=[b.address])
-        expected = {'sender': a.address, 'recipient': b.address}
-        assert result[b.address]['sender'] == a.address  # see origin worker
+    result = yield c.run(check, workers=[b.address])
+    expected = {'sender': a.address, 'recipient': b.address}
+    assert result[b.address]['sender'] == a.address  # see origin worker
 
-        z = yield y  # bring object to local process
+    z = yield y  # bring object to local process
 
-        assert z.x == 1 and z.y == 2
-        assert z.context['sender'] == b.address
-    finally:
-        from distributed.protocol.serialize import class_serializers, typename
-        del class_serializers[typename(MyObject)]
+    assert z.x == 1 and z.y == 2
+    assert z.context['sender'] == b.address
diff --git a/distributed/protocol/tests/test_sklearn.py b/distributed/protocol/tests/test_sklearn.py
new file mode 100644
index 00000000000..4fa8aeb5369
--- /dev/null
+++ b/distributed/protocol/tests/test_sklearn.py
@@ -0,0 +1,19 @@
+import pytest
+pytest.importorskip('sklearn')
+
+import sklearn.linear_model
+
+from distributed.protocol import serialize, deserialize
+
+
+def test_basic():
+    est = sklearn.linear_model.LinearRegression()
+    est.fit([[0, 0], [1, 1], [2, 2]], [0, 1, 2])
+
+    header, frames = serialize(est)
+    assert header['serializer'] == 'dask'
+
+    est2 = deserialize(header, frames)
+
+    inp = [[2, 3], [-1, 3]]
+    assert (est.predict(inp) == est2.predict(inp)).all()
diff --git a/distributed/protocol/tests/test_torch.py b/distributed/protocol/tests/test_torch.py
new file mode 100644
index 00000000000..d5d93b16f03
--- /dev/null
+++ b/distributed/protocol/tests/test_torch.py
@@ -0,0 +1,33 @@
+from distributed.protocol import serialize, deserialize
+import pytest
+
+np = pytest.importorskip('numpy')
+torch = pytest.importorskip('torch')
+
+
+def test_tensor():
+    x = np.arange(10)
+    t = torch.Tensor(x)
+    header, frames = serialize(t)
+    assert header['serializer'] == 'dask'
+    t2 = deserialize(header, frames)
+    assert (x == t2.numpy()).all()
+
+
+def test_grad():
+    x = np.arange(10)
+    t = torch.Tensor(x)
+    t.grad = torch.zeros_like(t) + 1
+
+    t2 = deserialize(*serialize(t))
+    assert (t2.numpy() == x).all()
+    assert (t2.grad.numpy() == 1).all()
+
+
+def test_resnet():
+    torchvision = pytest.importorskip('torchvision')
+    model = torchvision.models.resnet.resnet18()
+
+    header, frames = serialize(model)
+    model2 = deserialize(header, frames)
+    assert str(model) == str(model2)
diff --git a/distributed/protocol/torch.py b/distributed/protocol/torch.py
new file mode 100644
index 00000000000..c25b1549004
--- /dev/null
+++ b/distributed/protocol/torch.py
@@ -0,0 +1,56 @@
+from .serialize import (serialize, dask_serialize, dask_deserialize,
+        register_generic)
+
+import torch
+import numpy as np
+
+
+@dask_serialize.register(torch.Tensor)
+def serialize_torch_Tensor(t):
+    header, frames = serialize(t.numpy())
+    if t.grad is not None:
+        grad_header, grad_frames = serialize(t.grad.numpy())
+        header['grad'] = {'header': grad_header, 'start': len(frames)}
+        frames += grad_frames
+    header['requires_grad'] = t.requires_grad
+    header['device'] = t.device.type
+    return header, frames
+
+
+@dask_deserialize.register(torch.Tensor)
+def deserialize_torch_Tensor(header, frames):
+    if header.get('grad', False):
+        i = header['grad']['start']
+        frames, grad_frames = frames[:i], frames[i:]
+        grad = dask_deserialize.dispatch(np.ndarray)(header['grad']['header'], grad_frames)
+    else:
+        grad = None
+
+    x = dask_deserialize.dispatch(np.ndarray)(header, frames)
+    if header['device'] == 'cpu':
+        t = torch.from_numpy(x)
+        if header['requires_grad']:
+            t = t.requires_grad_(True)
+    else:
+        t = torch.tensor(data=x,
+                         device=header['device'],
+                         requires_grad=header['requires_grad'])
+    if grad is not None:
+        t.grad = torch.from_numpy(grad)
+    return t
+
+
+@dask_serialize.register(torch.nn.Parameter)
+def serialize_torch_Parameters(p):
+    header, frames = serialize(p.detach())
+    header['requires_grad'] = p.requires_grad
+    return header, frames
+
+
+@dask_deserialize.register(torch.nn.Parameter)
+def deserialize_torch_Parameters(header, frames):
+    t = dask_deserialize.dispatch(torch.Tensor)(header, frames)
+    return torch.nn.Parameter(data=t, requires_grad=header['requires_grad'])
+
+
+register_generic(torch.nn.Module)
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index dcdfafc884d..70e93b4534f 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -3860,13 +3860,13 @@ def test_scatter_compute_lose(c, s, a, b):
 
     yield a._close()
 
+    with pytest.raises(CancelledError):
+        yield wait(z)
+
     assert x.status == 'cancelled'
     assert y.status == 'finished'
     assert z.status == 'cancelled'
 
-    with pytest.raises(CancelledError):
-        yield wait(z)
-
 
 @gen_cluster(client=True)
 def test_scatter_compute_store_lose(c, s, a, b):
diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
index 9a9feabf114..4e192408f7b 100644
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -265,7 +265,8 @@ def test_nanny_timeout(c, s, a):
 
 
 @gen_cluster(ncores=[('127.0.0.1', 1)], client=True, Worker=Nanny,
-             worker_kwargs={'memory_limit': 1e8}, timeout=20)
+             worker_kwargs={'memory_limit': 1e8}, timeout=20,
+             check_new_threads=False)
 def test_nanny_terminate(c, s, a):
     from time import sleep
 
diff --git a/distributed/utils.py b/distributed/utils.py
index 1e9b9d69ae0..3bebc65b24f 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -548,6 +548,7 @@ def key_split(s):
 try:
     from functools import lru_cache
 except ImportError:
+    lru_cache = False
     pass
 else:
     key_split = lru_cache(100000)(key_split)
@@ -1405,6 +1406,10 @@ def has_keyword(func, keyword):
         return keyword in inspect.getargspec(func).args
 
 
+if lru_cache:
+    has_keyword = lru_cache(1000)(has_keyword)
+
+
 # from bokeh.palettes import viridis
 # palette = viridis(18)
 palette = ['#440154', '#471669', '#472A79', '#433C84', '#3C4D8A', '#355D8C',
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index d65b1abcb80..900abab06ed 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -856,7 +856,7 @@ def coro():
                         break
                     else:
                         sleep(0.01)
-                    if time() > start + 2:
+                    if time() > start + 5:
                         from distributed import profile
                         tid = bad[0]
                         thread = threading._active[tid]
diff --git a/docs/source/serialization.rst b/docs/source/serialization.rst
index cdb765c380a..5cfd1515684 100644
--- a/docs/source/serialization.rst
+++ b/docs/source/serialization.rst
@@ -1,6 +1,8 @@
 Serialization
 =============
 
+.. currentmodule:: distributed.protocol.serialize
+
 When we communicate data between computers we first convert that data into a
 sequence of bytes that can be communicated across the network.  Choices made in
 serialization can affect performance and security.
@@ -138,6 +140,10 @@ pickle internally in some cases.  It should not be considered more secure.
 Extend
 ++++++
 
+.. autosummary::
+   dask_serialize
+   dask_deserialize
+
 As with serialization families in general, the Dask family in particular is
 *also* extensible.  This is a good way to support custom serialization of a
 single type of object.  The method is similar, you create serialize and
@@ -150,26 +156,43 @@ register them with Dask.
         def __init__(self, name):
             self.name = name
 
+    from distributed.protocol import dask_serialize, dask_deserialize
+
+    @dask_serialize.register(Human)
     def serialize(human: Human) -> Tuple[Dict, List[bytes]]:
         header = {}
         frames = [human.name.encode()]
         return header, frames
 
+    @dask_deserialize.register(Human)
     def deserialize(header: Dict, frames: List[bytes]) -> Human:
         return Human(frames[0].decode())
 
-    from distributed.protocol.serialize import register_serialization
-    register_serialization(Human, serialize, deserialize)
+
+Traverse attributes
++++++++++++++++++++
+
+.. autosummary::
+   register_generic
+
+A common case is that your object just wraps Numpy arrays or other objects that
+Dask already serializes well.  For example, Scikit-Learn estimators mostly
+surround Numpy arrays with a bit of extra metadata.  In these cases you can
+register your class for custom Dask serialization with the
+``register_generic``
+function.
 
 API
 ---
 
-.. currentmodule:: distributed.protocol.serialize
-
-.. autosummary:: register_serialization
-                 serialize
+.. autosummary:: serialize
                  deserialize
+                 dask_serialize
+                 dask_deserialize
+                 register_generic
 
-.. autofunction:: register_serialization
 .. autofunction:: serialize
 .. autofunction:: deserialize
+.. autofunction:: dask_serialize
+.. autofunction:: dask_deserialize
+.. autofunction:: register_generic

From cdae7ca0130563a0a448e23203d763d12dbf10a5 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 20 Aug 2018 07:55:47 -0400
Subject: [PATCH 0062/1550] Handle corner cases with busy signal (#2182)

There are a couple cases where in-flight dependencies can come in in atypical
ways:

1.  A request can be made and in-flight, for some reason the dependency state
    changes (it gets cancelled and then re-requested) then when the request
    comes in we try to transition to a memory state, buy may not have an
    established route to do so
2.  A request can be made and in-flight, for some reason the dependency state
    changes, then the request comes in with a busy signal and we try to
    transition down to waiting, but may not have an established route to do so

This currently doesn't have any tests, and transition_dep_waiting_memory is
entirely uncovered.
---
 distributed/worker.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/distributed/worker.py b/distributed/worker.py
index f614c8e0b34..a554307e42b 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -1542,6 +1542,8 @@ def transition_dep_waiting_memory(self, dep, value=None):
                 import pdb
                 pdb.set_trace()
             raise
+        if value is not no_value and dep not in self.data:
+            self.put_key_in_memory(dep, value, transition=False)
 
     def transition(self, key, finish, **kwargs):
         start = self.task_state[key]
@@ -1890,7 +1892,8 @@ def gather_dep(self, worker, dep, deps, total_nbytes, cause=None):
                 if response['status'] == 'busy':
                     self.log.append(('busy-gather', worker, deps))
                     for dep in deps:
-                        self.transition_dep(dep, 'waiting')
+                        if self.dep_state[dep] == 'flight':
+                            self.transition_dep(dep, 'waiting')
                     return
 
                 if cause:

From 12b8d22bd029cdcebbd25b7caaac61e2a43ee7d9 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 20 Aug 2018 15:09:58 -0400
Subject: [PATCH 0063/1550] Check self.dependencies when looking at tasks in
 memory (#2196)

Previously during a check to see if tasks and their dependencies were already
in memory we assumed that the dependencies were known to the client when it
sent in the computation.  However if the inputs were futures then this was not
the case, and a KeyError was raised because only the scheduler knew where
things were.

Now we properly check both the input dependencies and self.dependencies and are
robust to the information being in either place.

Fixes #2187
---
 distributed/scheduler.py            |  8 ++++++--
 distributed/tests/test_scheduler.py | 22 ++++++++++++++++++++++
 2 files changed, 28 insertions(+), 2 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index f3a18612a25..afb51ff4312 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1369,7 +1369,11 @@ def update_graph(self, client=None, tasks=None, keys=None,
             while stack:  # remove unnecessary dependencies
                 key = stack.pop()
                 ts = self.tasks[key]
-                for dep in dependencies[key]:
+                try:
+                    deps = dependencies[key]
+                except KeyError:
+                    deps = self.dependencies[key]
+                for dep in deps:
                     if all(d in done for d in dependents[dep]):
                         if dep in self.tasks:
                             done.add(dep)
@@ -1377,7 +1381,7 @@ def update_graph(self, client=None, tasks=None, keys=None,
 
             for d in done:
                 tasks.pop(d, None)
-                del dependencies[d]
+                dependencies.pop(d, None)
 
         # Get or create task states
         stack = list(keys)
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 259ca100f19..816283537e5 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -1362,3 +1362,25 @@ def test_resources_reset_after_cancelled_task(c, s, w):
     assert w.available_resources == {'A': 1}
 
     yield c.submit(inc, 1, resources={'A': 1})
+
+
+@gen_cluster(client=True)
+def test_gh2187(c, s, a, b):
+    def foo():
+        return 'foo'
+
+    def bar(x):
+        return x + 'bar'
+
+    def baz(x):
+        sleep(0.1)
+        return x + 'baz'
+
+    x = c.submit(foo, key='x')
+    y = c.submit(bar, x, key='y')
+    yield y
+    z = c.submit(baz, y, key='z')
+    del y
+    yield gen.sleep(0.1)
+    f = c.submit(bar, x, key='y')
+    yield f

From 2ec428ae5652f7d068baeb18223fb8a04ab8804e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lo=C3=AFc=20Est=C3=A8ve?= <loic.esteve@ymail.com>
Date: Tue, 21 Aug 2018 13:52:13 +0200
Subject: [PATCH 0064/1550] Add ability to log additional custom metrics from
 each worker (#2169)

* Add ability to log additional custom metrics from each worker

* Refactor ws.info to ws.metrics

This removes old state from the info attribute of a WorkerState
and renames it to metrics.

We also remove the legacy worker_info dictionary from code and tests
---
 distributed/bokeh/scheduler.py                |  45 ++++--
 distributed/bokeh/templates/worker-table.html |   9 +-
 .../bokeh/tests/test_scheduler_bokeh.py       | 142 ++++++++++++++++++
 distributed/bokeh/tests/test_worker_bokeh.py  |   6 +-
 distributed/deploy/adaptive.py                |   4 +-
 distributed/deploy/local.py                   |   2 +-
 distributed/scheduler.py                      |  58 ++++---
 distributed/tests/test_client.py              |   4 +-
 distributed/tests/test_nanny.py               |   8 +-
 distributed/tests/test_scheduler.py           |   4 +-
 distributed/tests/test_worker.py              |  11 +-
 distributed/worker.py                         |  54 ++++---
 docs/source/web.rst                           |  24 +++
 13 files changed, 295 insertions(+), 76 deletions(-)

diff --git a/distributed/bokeh/scheduler.py b/distributed/bokeh/scheduler.py
index 27c39674871..a282e012574 100644
--- a/distributed/bokeh/scheduler.py
+++ b/distributed/bokeh/scheduler.py
@@ -328,13 +328,13 @@ def update(self):
                 else:
                     processing_color.append('blue')
 
-            nbytes = [ws.info['memory'] for ws in workers]
+            nbytes = [ws.metrics['memory'] for ws in workers]
             nbytes_text = [format_bytes(nb) for nb in nbytes]
             nbytes_color = []
             max_limit = 0
             for ws, nb in zip(workers, nbytes):
                 try:
-                    limit = self.scheduler.worker_info[ws.address]['memory_limit']
+                    limit = self.scheduler.workers[ws.address].memory_limit
                 except KeyError:
                     limit = 16e9
                 if limit > max_limit:
@@ -927,12 +927,17 @@ class WorkerTable(DashboardComponent):
     This is two plots, a text-based table for each host and a thin horizontal
     plot laying out hosts by their current memory use.
     """
+    excluded_names = {'executing', 'in_flight', 'in_memory', 'ready', 'time'}
 
     def __init__(self, scheduler, width=800, **kwargs):
         self.scheduler = scheduler
         self.names = ['worker', 'ncores', 'cpu', 'memory', 'memory_limit',
                       'memory_percent', 'num_fds', 'read_bytes', 'write_bytes',
                       'cpu_fraction']
+        workers = self.scheduler.workers.values()
+        self.extra_names = sorted({m for ws in workers
+                                   for m in ws.metrics
+                                   if m not in self.names} - self.excluded_names)
 
         table_names = ['worker', 'ncores', 'cpu', 'memory', 'memory_limit',
                        'memory_percent', 'num_fds', 'read_bytes',
@@ -967,6 +972,17 @@ def __init__(self, scheduler, width=800, **kwargs):
             if name in formatters:
                 table.columns[table_names.index(name)].formatter = formatters[name]
 
+        extra_names = ['worker'] + self.extra_names
+        extra_columns = {name: TableColumn(field=name,
+                                           title=name.replace('_percent', '%'))
+                         for name in extra_names}
+
+        extra_table = DataTable(
+            source=self.source,
+            columns=[extra_columns[n] for n in extra_names],
+            reorderable=True, sortable=True, width=width, **dt_kwargs
+        )
+
         hover = HoverTool(
             point_policy="follow_mouse",
             tooltips="""
@@ -1015,20 +1031,25 @@ def __init__(self, scheduler, width=800, **kwargs):
         else:
             sizing_mode = {}
 
-        self.root = column(cpu_plot, mem_plot, table, id='bk-worker-table', **sizing_mode)
+        components = [cpu_plot, mem_plot, table]
+        if self.extra_names:
+            components.append(extra_table)
+
+        self.root = column(*components, id='bk-worker-table', **sizing_mode)
 
     def update(self):
-        data = {name: [] for name in self.names}
-        for worker, info in sorted(self.scheduler.worker_info.items()):
-            for name in self.names:
-                data[name].append(info.get(name, None))
-            data['worker'][-1] = worker
-            if info['memory_limit']:
-                data['memory_percent'][-1] = info['memory'] / info['memory_limit']
+        data = {name: [] for name in self.names + self.extra_names}
+        for addr, ws in sorted(self.scheduler.workers.items()):
+            for name in self.names + self.extra_names:
+                data[name].append(ws.metrics.get(name, None))
+            data['worker'][-1] = ws.address
+            if ws.memory_limit:
+                data['memory_percent'][-1] = ws.metrics['memory'] / ws.memory_limit
             else:
                 data['memory_percent'][-1] = ''
-            data['cpu'][-1] = info['cpu'] / 100.0
-            data['cpu_fraction'][-1] = info['cpu'] / 100.0 / info['ncores']
+            data['memory_limit'][-1] = ws.memory_limit
+            data['cpu'][-1] = ws.metrics['cpu'] / 100.0
+            data['cpu_fraction'][-1] = ws.metrics['cpu'] / 100.0 / ws.ncores
 
         self.source.data.update(data)
 
diff --git a/distributed/bokeh/templates/worker-table.html b/distributed/bokeh/templates/worker-table.html
index 1d1768cec11..90b59c08c54 100644
--- a/distributed/bokeh/templates/worker-table.html
+++ b/distributed/bokeh/templates/worker-table.html
@@ -11,17 +11,16 @@
         <th> Logs </th>
     </tr>
     {% for ws in worker_list %}
-    {% set wi = worker_info[ws.address] %}
     <tr>
         <td><a href="../worker/{{ url_escape(ws.address) }}.html">{{ws.address}}</a></td>
         <td> {{ ws.ncores }} </td>
-        <td> {{ format_bytes(wi['memory_limit']) }} </td>
-        <td> <progress class="progress" value="{{ wi['memory'] }}" max="{{ wi['memory_limit'] }}"></progress> </td>
+        <td> {{ format_bytes(ws.memory_limit) }} </td>
+        <td> <progress class="progress" value="{{ ws.metrics['memory'] }}" max="{{ ws.memory_limit }}"></progress> </td>
         <td> {{ format_time(ws.occupancy) }} </td>
         <td> {{ len(ws.processing) }} </td>
         <td> {{ len(ws.has_what) }} </td>
-        {% if 'bokeh' in wi['services'] %}
-        <td> <a href="http://{{ wi['host'] }}:{{ wi['services']['bokeh'] }}">bokeh</a> </td>
+        {% if 'bokeh' in ws.services %}
+        <td> <a href="http://{{ ws.host }}:{{ ws.services['bokeh'] }}">bokeh</a> </td>
         {% else %}
         <td> </td>
         {% end %}
diff --git a/distributed/bokeh/tests/test_scheduler_bokeh.py b/distributed/bokeh/tests/test_scheduler_bokeh.py
index bcd7eae476b..262a5e8a67f 100644
--- a/distributed/bokeh/tests/test_scheduler_bokeh.py
+++ b/distributed/bokeh/tests/test_scheduler_bokeh.py
@@ -286,6 +286,148 @@ def test_WorkerTable(c, s, a, b):
     assert all(len(v) == 2 for v in wt.source.data.values())
 
 
+@gen_cluster(client=True)
+def test_WorkerTable_custom_metrics(c, s, a, b):
+    def metric_port(worker):
+        return worker.port
+
+    def metric_address(worker):
+        return worker.address
+
+    metrics = {'metric_port': metric_port,
+               'metric_address': metric_address}
+
+    for w in [a, b]:
+        for name, func in metrics.items():
+            w.metrics[name] = func
+
+    while not all('metric_port' in s.workers[w.address].metrics for w in [a, b]):
+        yield gen.sleep(0.01)
+
+    for w in [a, b]:
+        assert s.workers[w.address].metrics['metric_port'] == w.port
+        assert s.workers[w.address].metrics['metric_address'] == w.address
+
+    wt = WorkerTable(s)
+    wt.update()
+    data = wt.source.data
+
+    for name in metrics:
+        assert name in data
+
+    assert all(data.values())
+    assert all(len(v) == 2 for v in data.values())
+    my_index = data['worker'].index(a.address), data['worker'].index(b.address)
+    assert [data['metric_port'][i] for i in my_index] == [a.port, b.port]
+    assert [data['metric_address'][i] for i in my_index] == [a.address, b.address]
+
+
+@gen_cluster(client=True)
+def test_WorkerTable_different_metrics(c, s, a, b):
+    def metric_port(worker):
+        return worker.port
+
+    a.metrics['metric_a'] = metric_port
+    b.metrics['metric_b'] = metric_port
+
+    while not ('metric_a' in s.workers[a.address].metrics and
+               'metric_b' in s.workers[b.address].metrics):
+        yield gen.sleep(0.01)
+
+    assert s.workers[a.address].metrics['metric_a'] == a.port
+    assert s.workers[b.address].metrics['metric_b'] == b.port
+
+    wt = WorkerTable(s)
+    wt.update()
+    data = wt.source.data
+
+    assert 'metric_a' in data
+    assert 'metric_b' in data
+    assert all(data.values())
+    assert all(len(v) == 2 for v in data.values())
+    my_index = data['worker'].index(a.address), data['worker'].index(b.address)
+    assert [data['metric_a'][i] for i in my_index] == [a.port, None]
+    assert [data['metric_b'][i] for i in my_index] == [None, b.port]
+
+
+@gen_cluster(client=True)
+def test_WorkerTable_metrics_with_different_metric_2(c, s, a, b):
+    def metric_port(worker):
+        return worker.port
+
+    a.metrics['metric_a'] = metric_port
+
+    while 'metric_a' not in s.workers[a.address].metrics:
+        yield gen.sleep(0.01)
+
+    wt = WorkerTable(s)
+    wt.update()
+    data = wt.source.data
+
+    assert 'metric_a' in data
+    assert all(data.values())
+    assert all(len(v) == 2 for v in data.values())
+    my_index = data['worker'].index(a.address), data['worker'].index(b.address)
+    assert [data['metric_a'][i] for i in my_index] == [a.port, None]
+
+
+@gen_cluster(client=True, worker_kwargs={'metrics': {'my_port': lambda w: w.port}})
+def test_WorkerTable_add_and_remove_metrics(c, s, a, b):
+    def metric_port(worker):
+        return worker.port
+
+    a.metrics['metric_a'] = metric_port
+    a.metrics['metric_b'] = metric_port
+
+    while not ('metric_a' in s.workers[a.address].metrics and
+               'metric_b' in s.workers[b.address].metrics):
+        yield gen.sleep(0.01)
+
+    assert s.workers[a.address].metrics['metric_a'] == a.port
+    assert s.workers[b.address].metrics['metric_b'] == b.port
+
+    wt = WorkerTable(s)
+    wt.update()
+    assert 'metric_a' in wt.source.data
+    assert 'metric_b' in wt.source.data
+
+    # Remove 'metric_b' from worker b
+    del b.metrics['metric_b']
+
+    while 'metric_b' in s.workers[b.address].metrics:
+        yield gen.sleep(0.01)
+
+    wt = WorkerTable(s)
+    wt.update()
+    assert 'metric_a' in wt.source.data
+
+    del a.metrics['metric_a']
+
+    while 'metric_a' in s.workers[a.address].metrics:
+        yield gen.sleep(0.01)
+
+    wt = WorkerTable(s)
+    wt.update()
+    assert 'metric_a' not in wt.source.data
+
+
+@gen_cluster(client=True)
+def test_WorkerTable_custom_metric_overlap_with_core_metric(c, s, a, b):
+    def metric(worker):
+        return -999
+
+    a.metrics['executing'] = metric
+    a.metrics['cpu'] = metric
+    a.metrics['metric'] = metric
+
+    while 'metric' not in s.workers[a.address].metrics:
+        yield gen.sleep(0.01)
+
+    assert s.workers[a.address].metrics['executing'] != -999
+    assert s.workers[a.address].metrics['cpu'] != -999
+    assert s.workers[a.address].metrics['metric'] == -999
+
+
 @gen_cluster(client=True)
 def test_GraphPlot(c, s, a, b):
     gp = GraphPlot(s)
diff --git a/distributed/bokeh/tests/test_worker_bokeh.py b/distributed/bokeh/tests/test_worker_bokeh.py
index 01242b1c0b3..e991e53e369 100644
--- a/distributed/bokeh/tests/test_worker_bokeh.py
+++ b/distributed/bokeh/tests/test_worker_bokeh.py
@@ -23,8 +23,8 @@
 @gen_cluster(client=True,
              worker_kwargs={'services': {('bokeh', 0):  BokehWorker}})
 def test_simple(c, s, a, b):
-    assert s.worker_info[a.address]['services'] == {'bokeh': a.services['bokeh'].port}
-    assert s.worker_info[b.address]['services'] == {'bokeh': b.services['bokeh'].port}
+    assert s.workers[a.address].services == {'bokeh': a.services['bokeh'].port}
+    assert s.workers[b.address].services == {'bokeh': b.services['bokeh'].port}
 
     future = c.submit(sleep, 1)
     yield gen.sleep(0.1)
@@ -39,7 +39,7 @@ def test_simple(c, s, a, b):
 @gen_cluster(client=True,
              worker_kwargs={'services': {('bokeh', 0):  (BokehWorker, {})}})
 def test_services_kwargs(c, s, a, b):
-    assert s.worker_info[a.address]['services'] == {'bokeh': a.services['bokeh'].port}
+    assert s.workers[a.address].services == {'bokeh': a.services['bokeh'].port}
     assert isinstance(a.services['bokeh'], BokehWorker)
 
 
diff --git a/distributed/deploy/adaptive.py b/distributed/deploy/adaptive.py
index 014373ac7a8..62d308c6e22 100644
--- a/distributed/deploy/adaptive.py
+++ b/distributed/deploy/adaptive.py
@@ -144,8 +144,8 @@ def needs_memory(self):
         Returns ``True`` if  the required bytes in distributed memory is some
         factor larger than the actual distributed memory available.
         """
-        limit_bytes = {w: self.scheduler.worker_info[w]['memory_limit']
-                        for w in self.scheduler.worker_info}
+        limit_bytes = {addr: ws.memory_limit
+                       for addr, ws in self.scheduler.workers.items()}
         worker_bytes = [ws.nbytes for ws in self.scheduler.workers.values()]
 
         limit = sum(limit_bytes.values())
diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index b4e837a2a47..d3feedb578f 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -209,7 +209,7 @@ def _start_worker(self, death_timeout=60, **kwargs):
 
         self.workers.append(w)
 
-        while w.status != 'closed' and w.worker_address not in self.scheduler.worker_info:
+        while w.status != 'closed' and w.worker_address not in self.scheduler.workers:
             yield gen.sleep(0.01)
 
         if w.status == 'closed' and self.scheduler.status == 'running':
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index afb51ff4312..9f2d961b2ff 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -196,15 +196,19 @@ class WorkerState(object):
         'address',
         'has_what',
         'info',
+        'local_directory',
         'memory_limit',
+        'metrics',
         'name',
         'nbytes',
         'ncores',
         'occupancy',
+        'pid',
         'processing',
         'resources',
         'time_delay',
         'used_resources',
+        'services',
         'status',
         'last_seen',
         'actors',
@@ -218,17 +222,14 @@ def __init__(self, worker, ncores, memory_limit, name=None):
         self.nbytes = 0
         self.ncores = ncores
         self.occupancy = 0
+        self.pid = 0
         self.processing = dict()
         self.resources = {}
         self.used_resources = {}
         self.last_seen = 0
+        self.services = {}
         self.actors = set()
-
-        self.info = {'name': name,
-                     'memory_limit': memory_limit,
-                     'host': self.host,
-                     'resources': self.resources,
-                     'ncores': self.ncores}  # for backwards compatibility
+        self.metrics = {}
 
     @property
     def host(self):
@@ -241,6 +242,21 @@ def __repr__(self):
     def __str__(self):
         return self.address
 
+    def identity(self):
+        return {
+            'type': 'Worker',
+            'id': self.name,
+            'host': self.host,
+            'resources': self.resources,
+            'local_directory': self.local_directory,
+            'name': self.name,
+            'ncores': self.ncores,
+            'memory_limit': self.memory_limit,
+            'last_seen': self.last_seen,
+            'services': self.services,
+            'metrics': self.metrics
+        }
+
 
 class TaskState(object):
     """
@@ -856,7 +872,7 @@ def __init__(
                 ('worker_resources', 'resources', None),
                 ('used_resources', 'used_resources', None),
                 ('occupancy', 'occupancy', None),
-                ('worker_info', 'info', None),
+                ('worker_info', 'metrics', None),
                 ('processing', 'processing', _legacy_task_key_dict),
                 ('has_what', 'has_what', _legacy_task_key_set)]:
             func = operator.attrgetter(new_attr)
@@ -996,7 +1012,8 @@ def identity(self, comm=None):
              'id': str(self.id),
              'address': self.address,
              'services': {key: v.port for (key, v) in self.services.items()},
-             'workers': dict(self.worker_info)}
+             'workers': {worker.address: worker.identity()
+                         for worker in self.workers.values()}}
         return d
 
     def get_worker_service_addr(self, worker, service_name):
@@ -1005,11 +1022,11 @@ def get_worker_service_addr(self, worker, service_name):
         Returns None if the service doesn't exist.
         """
         ws = self.workers[worker]
-        port = ws.info['services'].get(service_name)
+        port = ws.services.get(service_name)
         if port is None:
             return None
         else:
-            return ws.info['host'], port
+            return ws.host, port
 
     def start_services(self, listen_ip):
         for k, v in self.service_specs.items():
@@ -1190,14 +1207,14 @@ def _setup_logging(self):
 
     @gen.coroutine
     def heartbeat_worker(self, comm=None, address=None, resolve_address=True,
-                          now=None, resources=None, host_info=None, **info):
+                          now=None, resources=None, host_info=None, metrics=None):
             address = self.coerce_address(address, resolve_address)
             address = normalize_address(address)
             host = get_address_host(address)
 
             local_now = time()
             now = now or time()
-            info = info or {}
+            metrics = metrics or {}
             host_info = host_info or {}
 
             self.host_info[host]['last-seen'] = local_now
@@ -1208,8 +1225,8 @@ def heartbeat_worker(self, comm=None, address=None, resolve_address=True,
 
             ws.last_seen = time()
 
-            if info:
-                ws.info.update(info)
+            if metrics:
+                ws.metrics = metrics
 
             if host_info:
                 self.host_info[host].update(host_info)
@@ -1220,7 +1237,7 @@ def heartbeat_worker(self, comm=None, address=None, resolve_address=True,
             if resources:
                 self.add_resources(worker=address, resources=resources)
 
-            self.log_event(address, merge({'action': 'heartbeat'}, info))
+            self.log_event(address, merge({'action': 'heartbeat'}, metrics))
 
             return {'status': 'OK',
                     'time': time(),
@@ -1229,7 +1246,8 @@ def heartbeat_worker(self, comm=None, address=None, resolve_address=True,
     @gen.coroutine
     def add_worker(self, comm=None, address=None, keys=(), ncores=None,
                    name=None, resolve_address=True, nbytes=None, now=None,
-                   resources=None, host_info=None, memory_limit=None, **info):
+                   resources=None, host_info=None, memory_limit=None,
+                   metrics=None, pid=0, services=None, local_directory=None):
         """ Add a new worker to the cluster """
         with log_errors():
             address = self.coerce_address(address, resolve_address)
@@ -1260,11 +1278,15 @@ def add_worker(self, comm=None, address=None, keys=(), ncores=None,
             self.total_ncores += ncores
             self.aliases[name] = address
             ws.name = name
+            ws.pid = pid
+            ws.services = services
+            ws.local_directory = local_directory
 
             response = self.heartbeat_worker(address=address,
                                              resolve_address=resolve_address,
                                              now=now, resources=resources,
-                                             host_info=host_info, **info)
+                                             host_info=host_info,
+                                             metrics=metrics)
 
             # Do not need to adjust self.total_occupancy as self.occupancy[ws] cannot exist before this.
             self.check_idle_saturated(ws)
@@ -2732,7 +2754,7 @@ def retire_workers(self, comm=None, workers=None, remove=True,
                 else:
                     raise gen.Return([])
 
-            worker_keys = {ws.address: ws.info for ws in workers}
+            worker_keys = {ws.address: ws.identity() for ws in workers}
             if close_workers and worker_keys:
                 yield [self.close_worker(worker=w, safe=True)
                        for w in worker_keys]
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 70e93b4534f..af64e27dca5 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -897,7 +897,7 @@ def test_remove_worker(c, s, a, b):
 
     yield b._close()
 
-    assert b.address not in s.worker_info
+    assert b.address not in s.workers
 
     result = yield c.gather(L)
     assert result == list(map(inc, range(20)))
@@ -4076,7 +4076,7 @@ def test_retire_workers_2(c, s, a, b):
     assert s.who_has == {x.key: {b.address}}
     assert s.has_what == {b.address: {x.key}}
 
-    assert a.address not in s.worker_info
+    assert a.address not in s.workers
 
 
 @gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 10)
diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
index 4e192408f7b..03f29f9b884 100644
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -30,22 +30,22 @@ def test_nanny(s):
     with rpc(n.address) as nn:
         assert n.is_alive()
         assert s.ncores[n.worker_address] == 2
-        assert s.worker_info[n.worker_address]['services']['nanny'] > 1024
+        assert s.workers[n.worker_address].services['nanny'] > 1024
 
         yield nn.kill()
         assert not n.is_alive()
         assert n.worker_address not in s.ncores
-        assert n.worker_address not in s.worker_info
+        assert n.worker_address not in s.workers
 
         yield nn.kill()
         assert not n.is_alive()
         assert n.worker_address not in s.ncores
-        assert n.worker_address not in s.worker_info
+        assert n.worker_address not in s.workers
 
         yield nn.instantiate()
         assert n.is_alive()
         assert s.ncores[n.worker_address] == 2
-        assert s.worker_info[n.worker_address]['services']['nanny'] > 1024
+        assert s.workers[n.worker_address].services['nanny'] > 1024
 
         yield nn.terminate()
         assert not n.is_alive()
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 816283537e5..542e4d91cd2 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -442,7 +442,7 @@ def test_worker_name():
     s.start(0)
     w = Worker(s.ip, s.port, name='alice')
     yield w._start()
-    assert s.worker_info[w.address]['name'] == 'alice'
+    assert s.workers[w.address].name == 'alice'
     assert s.aliases['alice'] == w.address
 
     with pytest.raises(ValueError):
@@ -575,7 +575,7 @@ def test_scheduler_sees_memory_limits(s):
     w = Worker(s.ip, s.port, ncores=3, memory_limit=12345)
     yield w._start(0)
 
-    assert s.worker_info[w.address]['memory_limit'] == 12345
+    assert s.workers[w.address].memory_limit == 12345
     yield w._close()
 
 
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 4ebfd9869c9..d8a3d5f2371 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -740,8 +740,7 @@ def test_worker_dir(worker):
     with tmpfile() as fn:
         @gen_cluster(client=True, worker_kwargs={'local_dir': fn})
         def test_worker_dir(c, s, a, b):
-            directories = [info['local_directory']
-                           for info in s.worker_info.values()]
+            directories = [w.local_directory for w in s.workers.values()]
             assert all(d.startswith(fn) for d in directories)
             assert len(set(directories)) == 2  # distinct
 
@@ -817,7 +816,7 @@ def __sizeof__(self):
 
 @gen_cluster()
 def test_pid(s, a, b):
-    assert s.worker_info[a.address]['pid'] == os.getpid()
+    assert s.workers[a.address].pid == os.getpid()
 
 
 @gen_cluster(client=True)
@@ -1198,3 +1197,9 @@ def test_avoid_oversubscription(c, s, *workers):
 
     # Some other workers did some work
     assert len([w for w in workers if len(w.outgoing_transfer_log) > 0]) >= 3
+
+
+@gen_cluster(client=True, worker_kwargs={'metrics': {'my_port': lambda w: w.port}})
+def test_custom_metrics(c, s, a, b):
+    assert s.workers[a.address].metrics['my_port'] == a.port
+    assert s.workers[b.address].metrics['my_port'] == b.port
diff --git a/distributed/worker.py b/distributed/worker.py
index a554307e42b..eae46e8ef43 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -17,9 +17,9 @@
 from dask.core import istask
 from dask.compatibility import apply
 try:
-    from cytoolz import pluck, partial
+    from cytoolz import pluck, partial, merge
 except ImportError:
-    from toolz import pluck, partial
+    from toolz import pluck, partial, merge
 from tornado.gen import Return
 from tornado import gen
 from tornado.ioloop import IOLoop
@@ -89,7 +89,7 @@ def __init__(self, scheduler_ip=None, scheduler_port=None,
                  executor=None, resources=None, silence_logs=None,
                  death_timeout=None, preload=(), preload_argv=[], security=None,
                  contact_address=None, memory_monitor_interval='200ms',
-                 extensions=None, **kwargs):
+                 extensions=None, metrics=None, **kwargs):
 
         self._setup_logging()
 
@@ -179,6 +179,7 @@ def __init__(self, scheduler_ip=None, scheduler_port=None,
         self.services = {}
         self.service_ports = service_ports or {}
         self.service_specs = services or {}
+        self.metrics = metrics or {}
 
         handlers = {
             'gather': self.gather,
@@ -261,16 +262,12 @@ def heartbeat(self):
                 start = time()
                 response = yield self.scheduler.heartbeat_worker(
                     address=self.contact_address,
-                    name=self.name,
                     now=time(),
-                    memory_limit=self.memory_limit,
-                    executing=len(self.executing),
-                    in_memory=len(self.data),
-                    ready=len(self.ready),
-                    in_flight=len(self.in_flight_tasks),
-                    **self.monitor.recent())
+                    metrics=self.get_metrics()
+                )
                 end = time()
                 middle = (start + end) / 2
+
                 if response['status'] == 'missing':
                     yield self._register_with_scheduler()
                     return
@@ -283,6 +280,15 @@ def heartbeat(self):
         else:
             logger.debug("Heartbeat skipped: channel busy")
 
+    def get_metrics(self):
+        core = dict(executing=len(self.executing),
+                    in_memory=len(self.data),
+                    ready=len(self.ready),
+                    in_flight=len(self.in_flight_tasks))
+        custom = {k: metric(self) for k, metric in self.metrics.items()}
+
+        return merge(custom, self.monitor.recent(), core)
+
     @gen.coroutine
     def _register_with_scheduler(self):
         self.periodic_callbacks['heartbeat'].stop()
@@ -301,20 +307,20 @@ def _register_with_scheduler(self):
                 comm = yield connect(self.scheduler.address,
                                      connection_args=self.connection_args)
                 yield comm.write(dict(op='register-worker',
-                                         ncores=self.ncores,
-                                         address=self.contact_address,
-                                         keys=list(self.data),
-                                         name=self.name,
-                                         nbytes=self.nbytes,
-                                         now=time(),
-                                         services=self.service_ports,
-                                         memory_limit=self.memory_limit,
-                                         local_directory=self.local_dir,
-                                         resources=self.total_resources,
-                                         pid=os.getpid(),
-                                         reply=False,
-                                         **self.monitor.recent()),
-                                         serializers=['msgpack'])
+                                      reply=False,
+                                      address=self.contact_address,
+                                      keys=list(self.data),
+                                      ncores=self.ncores,
+                                      name=self.name,
+                                      nbytes=self.nbytes,
+                                      now=time(),
+                                      resources=self.total_resources,
+                                      memory_limit=self.memory_limit,
+                                      local_directory=self.local_dir,
+                                      services=self.service_ports,
+                                      pid=os.getpid(),
+                                      metrics=self.get_metrics()),
+                                 serializers=['msgpack'])
                 future = comm.read(deserializers=['msgpack'])
                 if self.death_timeout:
                     diff = self.death_timeout - (time() - start)
diff --git a/docs/source/web.rst b/docs/source/web.rst
index 97dca02cb29..7d81fe7d935 100644
--- a/docs/source/web.rst
+++ b/docs/source/web.rst
@@ -161,6 +161,30 @@ available in the ``workers/`` page.
 .. image:: https://raw.githubusercontent.com/dask/dask-org/master/images/bokeh-resources.gif
    :alt: Resources view of Dask web interface
 
+Per-worker resources
+~~~~~~~~~~~~~~~~~~~~
+
+The ``workers/`` page shows per-worker resources, the main ones being CPU and
+memory use. Custom metrics can be registered and displayed in this page. Here
+is an example showing how to display GPU utilization and GPU memory use:
+
+.. code-block:: python
+
+   import subprocess
+
+   def nvidia_data(name):
+       def dask_function(dask_worker):
+           cmd = 'nvidia-smi --query-gpu={} --format=csv,noheader'.format(name)
+           result = subprocess.check_output(cmd.split())
+           return result.strip().decode()
+       return dask_function
+
+   def register_metrics(dask_worker):
+       for name in ['utilization.gpu', 'utilization.memory']:
+           dask_worker.metrics[name] = nvidia_data(name)
+
+   client.run(register_metrics)
+
 Connecting to Web Interface
 ---------------------------
 

From 1b3bf8c47e26b0d4756ea233afd1640068906e9f Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 22 Aug 2018 09:37:52 -0400
Subject: [PATCH 0065/1550] Clean up metrics tests (#2205)

We were getting intermittent testing failures with the WorkerTable
custom metrics tests.

This makes two changes that will hopefully resolve the situation:

1.  It explicitly calls heartbeat to accelerate the tests
2.  We make a copy of the metrics dict in order to avoid sharing the same
    metrics across multiple workers in tests
---
 .../bokeh/tests/test_scheduler_bokeh.py       | 31 +++++--------------
 distributed/worker.py                         |  2 +-
 2 files changed, 9 insertions(+), 24 deletions(-)

diff --git a/distributed/bokeh/tests/test_scheduler_bokeh.py b/distributed/bokeh/tests/test_scheduler_bokeh.py
index 262a5e8a67f..1c07b437f21 100644
--- a/distributed/bokeh/tests/test_scheduler_bokeh.py
+++ b/distributed/bokeh/tests/test_scheduler_bokeh.py
@@ -301,8 +301,7 @@ def metric_address(worker):
         for name, func in metrics.items():
             w.metrics[name] = func
 
-    while not all('metric_port' in s.workers[w.address].metrics for w in [a, b]):
-        yield gen.sleep(0.01)
+    yield [a.heartbeat(), b.heartbeat()]
 
     for w in [a, b]:
         assert s.workers[w.address].metrics['metric_port'] == w.port
@@ -329,10 +328,7 @@ def metric_port(worker):
 
     a.metrics['metric_a'] = metric_port
     b.metrics['metric_b'] = metric_port
-
-    while not ('metric_a' in s.workers[a.address].metrics and
-               'metric_b' in s.workers[b.address].metrics):
-        yield gen.sleep(0.01)
+    yield [a.heartbeat(), b.heartbeat()]
 
     assert s.workers[a.address].metrics['metric_a'] == a.port
     assert s.workers[b.address].metrics['metric_b'] == b.port
@@ -356,9 +352,7 @@ def metric_port(worker):
         return worker.port
 
     a.metrics['metric_a'] = metric_port
-
-    while 'metric_a' not in s.workers[a.address].metrics:
-        yield gen.sleep(0.01)
+    yield [a.heartbeat(), b.heartbeat()]
 
     wt = WorkerTable(s)
     wt.update()
@@ -377,11 +371,8 @@ def metric_port(worker):
         return worker.port
 
     a.metrics['metric_a'] = metric_port
-    a.metrics['metric_b'] = metric_port
-
-    while not ('metric_a' in s.workers[a.address].metrics and
-               'metric_b' in s.workers[b.address].metrics):
-        yield gen.sleep(0.01)
+    b.metrics['metric_b'] = metric_port
+    yield [a.heartbeat(), b.heartbeat()]
 
     assert s.workers[a.address].metrics['metric_a'] == a.port
     assert s.workers[b.address].metrics['metric_b'] == b.port
@@ -393,18 +384,14 @@ def metric_port(worker):
 
     # Remove 'metric_b' from worker b
     del b.metrics['metric_b']
-
-    while 'metric_b' in s.workers[b.address].metrics:
-        yield gen.sleep(0.01)
+    yield [a.heartbeat(), b.heartbeat()]
 
     wt = WorkerTable(s)
     wt.update()
     assert 'metric_a' in wt.source.data
 
     del a.metrics['metric_a']
-
-    while 'metric_a' in s.workers[a.address].metrics:
-        yield gen.sleep(0.01)
+    yield [a.heartbeat(), b.heartbeat()]
 
     wt = WorkerTable(s)
     wt.update()
@@ -419,9 +406,7 @@ def metric(worker):
     a.metrics['executing'] = metric
     a.metrics['cpu'] = metric
     a.metrics['metric'] = metric
-
-    while 'metric' not in s.workers[a.address].metrics:
-        yield gen.sleep(0.01)
+    yield [a.heartbeat(), b.heartbeat()]
 
     assert s.workers[a.address].metrics['executing'] != -999
     assert s.workers[a.address].metrics['cpu'] != -999
diff --git a/distributed/worker.py b/distributed/worker.py
index eae46e8ef43..bd9f6040e5f 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -179,7 +179,7 @@ def __init__(self, scheduler_ip=None, scheduler_port=None,
         self.services = {}
         self.service_ports = service_ports or {}
         self.service_specs = services or {}
-        self.metrics = metrics or {}
+        self.metrics = dict(metrics) if metrics else {}
 
         handlers = {
             'gather': self.gather,

From 5b7658c1fe05b9dad405e2c56a9587727b9513c3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lo=C3=AFc=20Est=C3=A8ve?= <loic.esteve@ymail.com>
Date: Wed, 22 Aug 2018 15:38:22 +0200
Subject: [PATCH 0066/1550] Fix formatting when port is a tuple (#2204)

---
 distributed/scheduler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 9f2d961b2ff..86c75d1872d 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1051,7 +1051,7 @@ def start_services(self, listen_ip):
                     service.listen((listen_ip, port))
                 self.services[k] = service
             except Exception as e:
-                warnings.warn("\nCould not launch service '%s' on port %d. " % (k, port) +
+                warnings.warn("\nCould not launch service '%s' on port %s. " % (k, port) +
                               "Got the following message:\n\n" + str(e),
                               stacklevel=3)
 

From 58de8b6d21d285591a5a2a63eb218df843a70d07 Mon Sep 17 00:00:00 2001
From: Mike DePalatis <mike@depalatis.net>
Date: Sat, 25 Aug 2018 16:01:05 -0400
Subject: [PATCH 0067/1550] Describe what ZeroMQ is (#2211)

---
 docs/source/related-work.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/source/related-work.rst b/docs/source/related-work.rst
index 413f8cd86b9..e2b13e458d2 100644
--- a/docs/source/related-work.rst
+++ b/docs/source/related-work.rst
@@ -61,7 +61,7 @@ Direct Communication
 
 *   MPI4Py_: Wraps the Message Passing Interface popular in high performance
     computing.
-*   PyZMQ_: Wraps ZeroMQ, the gentleman's socket.
+*   PyZMQ_: Wraps ZeroMQ, the high-performance asynchronous messaging library.
 
 Venerable
 ~~~~~~~~~

From f701a538eb8c24b5e6234db32b90de1de4193a31 Mon Sep 17 00:00:00 2001
From: Anderson Banihirwe <axbanihirwe@ualr.edu>
Date: Tue, 28 Aug 2018 09:25:48 -0400
Subject: [PATCH 0068/1550] Tiny typo fix (#2214)

---
 docs/source/work-stealing.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/source/work-stealing.rst b/docs/source/work-stealing.rst
index acb2d699423..cf5a4bc48c1 100644
--- a/docs/source/work-stealing.rst
+++ b/docs/source/work-stealing.rst
@@ -118,7 +118,7 @@ sends a request to the busy worker.  The worker inspects its current state of
 the task and sends a response to the scheduler:
 
 1.  If the task is not yet running, then the worker cancels the task and
-    informs the scheduler that it can reroute the ask elsewhere.
+    informs the scheduler that it can reroute the task elsewhere.
 2.  If the task is already running or complete then the worker tells the
     scheduler that it should not replicate the task elsewhere.
 

From de340007509b019b3a94b67d2379985f47bf4a69 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 28 Aug 2018 10:22:35 -0400
Subject: [PATCH 0069/1550] Add Python 3.7 to travis.yml (#2203)

* Add Python 3.7 to travis.yml

* bump pytest to 3.7

* skip test that uses yield/StopIteration

This is no longer valid for Python 3.7

The solution of using yield then return works just fine, but raises a
SyntaxError in Python 2

* add compression libraries
---
 .travis.yml                              | 7 ++++---
 continuous_integration/travis/install.sh | 3 +--
 distributed/tests/test_client.py         | 2 ++
 3 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 2dd96098c50..5963855c59b 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -6,9 +6,10 @@ dist: trusty
 
 env:
   matrix:
-    - PYTHON=2.7 TESTS=true PACKAGES="python-blosc futures faulthandler"
-    - PYTHON=3.5.4 TESTS=true COVERAGE=true PACKAGES=python-blosc CRICK=true
-    - PYTHON=3.6 TESTS=true PACKAGES="scikit-learn"
+    - PYTHON=2.7 TESTS=true PACKAGES="python-blosc futures faulthandler lz4"
+    - PYTHON=3.5.4 TESTS=true COVERAGE=true PACKAGES="python-blosc lz4" CRICK=true
+    - PYTHON=3.6 TESTS=true PACKAGES="scikit-learn lz4"
+    - PYTHON=3.7 TESTS=true PACKAGES="scikit-learn python-snappy python-blosc"
 
 matrix:
   fast_finish: true
diff --git a/continuous_integration/travis/install.sh b/continuous_integration/travis/install.sh
index 72f13e20b07..4954f582f1c 100644
--- a/continuous_integration/travis/install.sh
+++ b/continuous_integration/travis/install.sh
@@ -39,12 +39,11 @@ conda install -q \
     ipywidgets \
     joblib \
     jupyter_client \
-    lz4 \
     mock \
     netcdf4 \
     paramiko \
     psutil \
-    pytest=3.1 \
+    pytest=3.7 \
     pytest-timeout \
     python=$PYTHON \
     requests \
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index af64e27dca5..6ed9e0faf8c 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -2347,6 +2347,8 @@ def test_map_queue(c, s, a, b):
     assert result == (1 + 1) * 2
 
 
+@pytest.mark.skipif(sys.version_info >= (3, 7),
+                    reason="replace StopIteration with return")
 @gen_cluster(client=True)
 def test_map_iterator_with_return(c, s, a, b):
     def g():

From a1aeff71096a08906ce073c710f80a3a9656c8c1 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 28 Aug 2018 13:27:31 -0400
Subject: [PATCH 0070/1550] Downgrade exception to warning when reusing port
 (#2199)

* Downgrade exception to warning when reusing port

* change warning message
---
 distributed/bokeh/core.py              | 7 +++++--
 distributed/deploy/tests/test_local.py | 5 ++++-
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/distributed/bokeh/core.py b/distributed/bokeh/core.py
index 351901a3386..f7409c043a0 100644
--- a/distributed/bokeh/core.py
+++ b/distributed/bokeh/core.py
@@ -2,6 +2,7 @@
 
 from distutils.version import LooseVersion
 import os
+import warnings
 
 import bokeh
 from bokeh.server.server import Server
@@ -44,11 +45,13 @@ def listen(self, addr):
                     if ("already in use" in str(exc) or  # Unix/Mac
                             "Only one usage of" in str(exc)):  # Windows
                         msg = ("Port %d is already in use. "
-                               "Perhaps you already have a cluster running?"
+                               "\nPerhaps you already have a cluster running?"
+                               "\nHosting the diagnostics dashboard on a random port instead."
                                % port)
                     else:
                         msg = "Failed to start diagnostics server on port %d. " % port + str(exc)
-                    raise type(exc)(msg)
+                    warnings.warn('\n' + msg)
+                    port = 0
                 if i == 4:
                     raise
 
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index 25536222ce0..b2c226c565c 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -147,7 +147,10 @@ def test_Client_solo(loop):
 def test_duplicate_clients():
     c1 = yield Client(processes=False, silence_logs=False, diagnostics_port=9876)
     with pytest.warns(Exception) as info:
-        yield Client(processes=False, silence_logs=False, diagnostics_port=9876)
+        c2 = yield Client(processes=False, silence_logs=False, diagnostics_port=9876)
+
+    assert 'bokeh' in c1.cluster.scheduler.services
+    assert 'bokeh' in c2.cluster.scheduler.services
 
     assert any(all(word in str(msg.message).lower()
                    for word in ['9876', 'running', 'already in use'])

From 15743f6505e12785f5f151789f04fa11f9ab4589 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 28 Aug 2018 15:48:52 -0400
Subject: [PATCH 0071/1550] Add plot= keyword to get_task_stream (#2198)

* Add plot= keyword to get_task_stream

```python
In [1]: from dask.distributed import Client, get_task_stream

In [2]: import dask.array as da

In [3]: client = Client()

In [4]: with get_task_stream(plot='save', filename='foo.html') as ts:
   ...:     da.random.random((10000, 10000), chunks=(1000, 1000)).std().compute()

In [5]: !ls foo.html
foo.html

In [6]: type(ts.figure)
Out[6]: bokeh.plotting.figure.Figure
```

* add default filename
---
 distributed/bokeh/components.py               |  85 +++++++------
 distributed/client.py                         |  65 +++++++++-
 distributed/diagnostics/task_stream.py        | 114 ++++++++++--------
 .../diagnostics/tests/test_task_stream.py     |  30 +++++
 4 files changed, 199 insertions(+), 95 deletions(-)

diff --git a/distributed/bokeh/components.py b/distributed/bokeh/components.py
index c2f37844e6e..f3d1ca4c3d4 100644
--- a/distributed/bokeh/components.py
+++ b/distributed/bokeh/components.py
@@ -59,12 +59,48 @@ def __init__(self, n_rectangles=1000, clear_interval='20s', **kwargs):
         """
         kwargs are applied to the bokeh.models.plots.Plot constructor
         """
-        clear_interval = parse_timedelta(clear_interval, default='ms')
         self.n_rectangles = n_rectangles
+        clear_interval = parse_timedelta(clear_interval, default='ms')
         self.clear_interval = clear_interval
         self.last = 0
 
-        self.source = ColumnDataSource(data=dict(
+        self.source, self.root = task_stream_figure(clear_interval, **kwargs)
+
+        # Required for update callback
+        self.task_stream_index = [0]
+
+    def update(self, messages):
+        with log_errors():
+            index = messages['task-events']['index']
+            rectangles = messages['task-events']['rectangles']
+
+            if not index or index[-1] == self.task_stream_index[0]:
+                return
+
+            ind = bisect(index, self.task_stream_index[0])
+            rectangles = {k: [v[i] for i in range(ind, len(index))]
+                          for k, v in rectangles.items()}
+            self.task_stream_index[0] = index[-1]
+
+            # If there has been a significant delay then clear old rectangles
+            if rectangles['start']:
+                m = min(map(add, rectangles['start'], rectangles['duration']))
+                if m > self.last:
+                    self.last, last = m, self.last
+                    if m > last + self.clear_interval:
+                        self.source.data.update(rectangles)
+                        return
+
+            self.source.stream(rectangles, self.n_rectangles)
+
+
+def task_stream_figure(clear_interval='20s', **kwargs):
+        """
+        kwargs are applied to the bokeh.models.plots.Plot constructor
+        """
+        clear_interval = parse_timedelta(clear_interval, default='ms')
+
+        source = ColumnDataSource(data=dict(
             start=[time() - clear_interval], duration=[0.1], key=['start'],
             name=['start'], color=['white'], duration_text=['100 ms'],
             worker=['foo'], y=[0], worker_thread=[1], alpha=[0.0])
@@ -73,21 +109,21 @@ def __init__(self, n_rectangles=1000, clear_interval='20s', **kwargs):
         x_range = DataRange1d(range_padding=0)
         y_range = DataRange1d(range_padding=0)
 
-        self.root = figure(
+        root = figure(
             title="Task Stream", id='bk-task-stream-plot',
             x_range=x_range, y_range=y_range, toolbar_location="above",
             x_axis_type='datetime', min_border_right=35, tools='', **kwargs)
-        self.root.yaxis.axis_label = 'Worker Core'
+        root.yaxis.axis_label = 'Worker Core'
 
-        rect = self.root.rect(source=self.source, x="start", y="y",
+        rect = root.rect(source=source, x="start", y="y",
             width="duration", height=0.4, fill_color="color",
             line_color="color", line_alpha=0.6, fill_alpha="alpha",
             line_width=3)
         rect.nonselection_glyph = None
 
-        self.root.yaxis.major_label_text_alpha = 0
-        self.root.yaxis.minor_tick_line_alpha = 0
-        self.root.xgrid.visible = False
+        root.yaxis.major_label_text_alpha = 0
+        root.yaxis.minor_tick_line_alpha = 0
+        root.xgrid.visible = False
 
         hover = HoverTool(
             point_policy="follow_mouse",
@@ -101,7 +137,7 @@ def __init__(self, n_rectangles=1000, clear_interval='20s', **kwargs):
 
         tap = TapTool(callback=OpenURL(url='/profile?key=@name'))
 
-        self.root.add_tools(
+        root.add_tools(
             hover, tap,
             BoxZoomTool(),
             ResetTool(),
@@ -110,35 +146,10 @@ def __init__(self, n_rectangles=1000, clear_interval='20s', **kwargs):
         )
         if ExportTool:
             export = ExportTool()
-            export.register_plot(self.root)
-            self.root.add_tools(export)
-
-        # Required for update callback
-        self.task_stream_index = [0]
-
-    def update(self, messages):
-        with log_errors():
-            index = messages['task-events']['index']
-            rectangles = messages['task-events']['rectangles']
-
-            if not index or index[-1] == self.task_stream_index[0]:
-                return
+            export.register_plot(root)
+            root.add_tools(export)
 
-            ind = bisect(index, self.task_stream_index[0])
-            rectangles = {k: [v[i] for i in range(ind, len(index))]
-                          for k, v in rectangles.items()}
-            self.task_stream_index[0] = index[-1]
-
-            # If there has been a significant delay then clear old rectangles
-            if rectangles['start']:
-                m = min(map(add, rectangles['start'], rectangles['duration']))
-                if m > self.last:
-                    self.last, last = m, self.last
-                    if m > last + self.clear_interval:
-                        self.source.data.update(rectangles)
-                        return
-
-            self.source.stream(rectangles, self.n_rectangles)
+        return source, root
 
 
 class MemoryUsage(DashboardComponent):
diff --git a/distributed/client.py b/distributed/client.py
index 2ee1ffea537..7e97df7dbdf 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -3337,7 +3337,8 @@ def get_restrictions(cls, collections, workers, allow_other_workers):
     def collections_to_dsk(collections, *args, **kwargs):
         return collections_to_dsk(collections, *args, **kwargs)
 
-    def get_task_stream(self, start=None, stop=None, count=None):
+    def get_task_stream(self, start=None, stop=None, count=None, plot=False,
+                        filename='task-stream.html'):
         """ Get task stream data from scheduler
 
         This collects the data present in the diagnostic "Task Stream" plot on
@@ -3360,6 +3361,11 @@ def get_task_stream(self, start=None, stop=None, count=None):
         count: int
             The number of desired records, ignored if both start and stop are
             specified
+        plot: boolean, str
+            If true then also return a Bokeh figure
+            If plot == 'save' then save the figure to a file
+        filename: str (optional)
+            The filename to save to if you set ``plot='save'``
 
         Examples
         --------
@@ -3371,6 +3377,11 @@ def get_task_stream(self, start=None, stop=None, count=None):
           'thread': ...,
           ...}]
 
+        Pass the ``plot=True`` or ``plot='save'`` keywords to get back a Bokeh
+        figure
+
+        >>> data, figure = client.get_task_stream(plot='save', filename='myfile.html')
+
         Alternatively consider the context manager
 
         >>> from dask.distributed import get_task_stream
@@ -3385,10 +3396,28 @@ def get_task_stream(self, start=None, stop=None, count=None):
 
         See Also
         --------
-        get_task_stream: a dontext manager version of this method
+        get_task_stream: a context manager version of this method
         """
-        return self.sync(self.scheduler.get_task_stream, start=start,
+        return self.sync(self._get_task_stream, start=start, stop=stop,
+                         count=count, plot=plot, filename=filename)
+
+    @gen.coroutine
+    def _get_task_stream(self, start=None, stop=None, count=None, plot=False,
+                         filename='task-stream.html'):
+        msgs = yield self.scheduler.get_task_stream(start=start,
                          stop=stop, count=count)
+        if plot:
+            from .diagnostics.task_stream import rectangles
+            rects = rectangles(msgs)
+            from .bokeh.components import task_stream_figure
+            source, figure = task_stream_figure(sizing_mode='stretch_both')
+            source.data.update(rects)
+            if plot == 'save':
+                from bokeh.plotting import save
+                save(figure, title='Dask Task Stream', filename=filename)
+            raise gen.Return((msgs, figure))
+        else:
+            raise gen.Return(msgs)
 
 
 class Executor(Client):
@@ -3768,6 +3797,14 @@ class get_task_stream(object):
 
     This must be used as a context manager.
 
+    Parameters
+    ----------
+    plot: boolean, str
+        If true then also return a Bokeh figure
+        If plot == 'save' then save the figure to a file
+    filename: str (optional)
+        The filename to save to if you set ``plot='save'``
+
     Examples
     --------
     >>> with get_task_stream() as ts:
@@ -3775,12 +3812,22 @@ class get_task_stream(object):
     >>> ts.data
     [...]
 
+    Get back a Bokeh figure and optionally save to a file
+
+    >>> with get_task_stream(plot='save', filename='myfile.html') as ts:
+    ...    x.compute()
+    >>> ts.figure
+    <Bokeh Figure>
+
     See Also
     --------
     Client.get_task_stream: Function version of this context manager
     """
-    def __init__(self, client=None):
+    def __init__(self, client=None, plot=False, filename='task-stream.html'):
         self.data = []
+        self._plot = plot
+        self._filename = filename
+        self.figure = None
         self.client = client or default_client()
         self.client.get_task_stream(start=0, stop=0)  # ensure plugin
 
@@ -3789,7 +3836,10 @@ def __enter__(self):
         return self
 
     def __exit__(self, typ, value, traceback):
-        L = self.client.get_task_stream(start=self.start)
+        L = self.client.get_task_stream(start=self.start, plot=self._plot,
+                                        filename=self._filename)
+        if self._plot:
+            L, self.figure = L
         self.data.extend(L)
 
     @gen.coroutine
@@ -3798,7 +3848,10 @@ def __aenter__(self):
 
     @gen.coroutine
     def __aexit__(self, typ, value, traceback):
-        L = yield self.client.get_task_stream(start=self.start)
+        L = yield self.client.get_task_stream(start=self.start, plot=self._plot,
+                                              filename=self._filename)
+        if self._plot:
+            L, self.figure = L
         self.data.extend(L)
 
 
diff --git a/distributed/diagnostics/task_stream.py b/distributed/diagnostics/task_stream.py
index 7cabcf96311..f3fd169d8d6 100644
--- a/distributed/diagnostics/task_stream.py
+++ b/distributed/diagnostics/task_stream.py
@@ -71,64 +71,74 @@ def bisect(target, left, right):
         return [self.buffer[i] for i in range(start, stop)]
 
     def rectangles(self, istart, istop=None, workers=None, start_boundary=0):
-        L_start = []
-        L_duration = []
-        L_duration_text = []
-        L_key = []
-        L_name = []
-        L_color = []
-        L_alpha = []
-        L_worker = []
-        L_worker_thread = []
-        L_y = []
-
+        msgs = []
         diff = self.index - len(self.buffer)
         if istop is None:
             istop = len(self.buffer)
         for i in range((istart or 0) - diff, istop - diff if istop else istop):
             msg = self.buffer[i]
-            key = msg['key']
-            name = key_split(key)
-            startstops = msg.get('startstops', [])
-            try:
-                worker_thread = '%s-%d' % (msg['worker'], msg['thread'])
-            except Exception:
+            msgs.append(msg)
+
+        return rectangles(msgs, workers=workers, start_boundary=start_boundary)
+
+
+def rectangles(msgs, workers=None, start_boundary=0):
+    workers = workers or {}
+
+    L_start = []
+    L_duration = []
+    L_duration_text = []
+    L_key = []
+    L_name = []
+    L_color = []
+    L_alpha = []
+    L_worker = []
+    L_worker_thread = []
+    L_y = []
+
+    for msg in msgs:
+        key = msg['key']
+        name = key_split(key)
+        startstops = msg.get('startstops', [])
+        try:
+            worker_thread = '%s-%d' % (msg['worker'], msg['thread'])
+        except Exception:
+            continue
+            logger.warning("Message contained bad information: %s", msg,
+                           exc_info=True)
+            worker_thread = ''
+
+        if worker_thread not in workers:
+            workers[worker_thread] = len(workers) / 2
+
+        for action, start, stop in startstops:
+            if start < start_boundary:
                 continue
-                logger.warning("Message contained bad information: %s", msg,
-                               exc_info=True)
-                worker_thread = ''
-
-            if worker_thread not in workers:
-                workers[worker_thread] = len(workers) / 2
-
-            for action, start, stop in startstops:
-                if start < start_boundary:
-                    continue
-                color = colors[action]
-                if type(color) is not str:
-                    color = color(msg)
-
-                L_start.append((start + stop) / 2 * 1000)
-                L_duration.append(1000 * (stop - start))
-                L_duration_text.append(format_time(stop - start))
-                L_key.append(key)
-                L_name.append(prefix[action] + name)
-                L_color.append(color)
-                L_alpha.append(alphas[action])
-                L_worker.append(msg['worker'])
-                L_worker_thread.append(worker_thread)
-                L_y.append(workers[worker_thread])
-
-        return {'start': L_start,
-                'duration': L_duration,
-                'duration_text': L_duration_text,
-                'key': L_key,
-                'name': L_name,
-                'color': L_color,
-                'alpha': L_alpha,
-                'worker': L_worker,
-                'worker_thread': L_worker_thread,
-                'y': L_y}
+            color = colors[action]
+            if type(color) is not str:
+                color = color(msg)
+
+            L_start.append((start + stop) / 2 * 1000)
+            L_duration.append(1000 * (stop - start))
+            L_duration_text.append(format_time(stop - start))
+            L_key.append(key)
+            L_name.append(prefix[action] + name)
+            L_color.append(color)
+            L_alpha.append(alphas[action])
+            L_worker.append(msg['worker'])
+            L_worker_thread.append(worker_thread)
+            L_y.append(workers[worker_thread])
+
+    return {'start': L_start,
+            'duration': L_duration,
+            'duration_text': L_duration_text,
+            'key': L_key,
+            'name': L_name,
+            'color': L_color,
+            'alpha': L_alpha,
+            'worker': L_worker,
+            'worker_thread': L_worker_thread,
+            'y': L_y}
 
 
 def color_of_message(msg):
diff --git a/distributed/diagnostics/tests/test_task_stream.py b/distributed/diagnostics/tests/test_task_stream.py
index eccb0a9db8e..f4354b74e6b 100644
--- a/distributed/diagnostics/tests/test_task_stream.py
+++ b/distributed/diagnostics/tests/test_task_stream.py
@@ -1,7 +1,9 @@
 from __future__ import absolute_import, division, print_function
 
+import os
 from time import sleep
 
+import pytest
 from toolz import frequencies
 
 from distributed import Client, get_task_stream
@@ -102,3 +104,31 @@ def test_client_sync(loop):
                 wait(futures)
 
             assert len(ts.data) == 10
+
+
+@gen_cluster(client=True)
+def test_get_task_stream_plot(c, s, a, b):
+    bokeh = pytest.importorskip('bokeh')
+    yield c.get_task_stream()
+
+    futures = c.map(slowinc, range(10), delay=0.1)
+    yield wait(futures)
+
+    data, figure = yield c.get_task_stream(plot=True)
+    assert isinstance(figure, bokeh.plotting.Figure)
+
+
+def test_get_task_stream_save(loop, tmpdir):
+    bokeh = pytest.importorskip('bokeh')
+    tmpdir = str(tmpdir)
+    fn = os.path.join(tmpdir, 'foo.html')
+    with cluster() as (s, [a, b]):
+        with Client(s['address'], loop=loop) as c:
+            with get_task_stream(plot='save', filename=fn) as ts:
+                wait(c.map(inc, range(10)))
+            with open(fn) as f:
+                data = f.read()
+            assert 'inc' in data
+            assert 'bokeh' in data
+
+            assert isinstance(ts.figure, bokeh.plotting.Figure)

From 1b9f9f4a6ec0d101869d9a92983ad1c7fe7c0241 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 29 Aug 2018 09:28:36 -0400
Subject: [PATCH 0072/1550] Add support for optional versions in
 Client.get_versions (#2216)

* add dask-ml to get_versions

* Add support for optional packages in get_versions
---
 distributed/client.py            | 14 +++++++++----
 distributed/node.py              |  4 ++--
 distributed/tests/test_client.py |  4 ++++
 distributed/versions.py          | 36 +++++++++++++++++++++++++++-----
 4 files changed, 47 insertions(+), 11 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 7e97df7dbdf..6461065dfe9 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -3039,7 +3039,7 @@ def set_metadata(self, key, value):
             key = (key,)
         return self.sync(self.scheduler.set_metadata, keys=key, value=value)
 
-    def get_versions(self, check=False):
+    def get_versions(self, check=False, packages=[]):
         """ Return version info for the scheduler, all workers and myself
 
         Parameters
@@ -3047,18 +3047,24 @@ def get_versions(self, check=False):
         check : boolean, default False
             raise ValueError if all required & optional packages
             do not match
+        packages : List[str]
+            Extra package names to check
 
         Examples
         --------
         >>> c.get_versions()  # doctest: +SKIP
+
+        >>> c.get_versions(packages=['sklearn', 'geopandas'])  # doctest: +SKIP
         """
-        client = get_versions()
+        client = get_versions(packages=packages)
         try:
-            scheduler = sync(self.loop, self.scheduler.versions)
+            scheduler = sync(self.loop, self.scheduler.versions,
+                             packages=packages)
         except KeyError:
             scheduler = None
 
-        workers = sync(self.loop, self.scheduler.broadcast, msg={'op': 'versions'})
+        workers = sync(self.loop, self.scheduler.broadcast,
+                       msg={'op': 'versions', 'packages': packages})
         result = {'scheduler': scheduler, 'workers': workers, 'client': client}
 
         if check:
diff --git a/distributed/node.py b/distributed/node.py
index 8373c07709c..654d67f376c 100644
--- a/distributed/node.py
+++ b/distributed/node.py
@@ -46,5 +46,5 @@ def __init__(self, handlers=None, stream_handlers=None,
                         connection_limit=connection_limit,
                         deserialize=deserialize, io_loop=self.io_loop)
 
-    def versions(self, comm=None):
-        return get_versions()
+    def versions(self, comm=None, packages=None):
+        return get_versions(packages=packages)
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 6ed9e0faf8c..7f09a2efce7 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -3814,6 +3814,10 @@ def test_get_versions(loop):
             # smoke test for versions
             # that this does not raise
 
+            v = c.get_versions(packages=['requests'])
+            import requests
+            assert dict(v['client']['packages']['optional'])['requests'] == requests.__version__
+
 
 def test_threaded_get_within_distributed(loop):
     with cluster() as (s, [a, b]):
diff --git a/distributed/versions.py b/distributed/versions.py
index 9f95591c880..fa7bbc0835a 100644
--- a/distributed/versions.py
+++ b/distributed/versions.py
@@ -9,6 +9,8 @@
 import locale
 import importlib
 
+from .utils import ignoring
+
 
 required_packages = [('dask', lambda p: p.__version__),
                      ('distributed', lambda p: p.__version__),
@@ -21,16 +23,20 @@
                      ('pandas', lambda p: p.__version__),
                      ('bokeh', lambda p: p.__version__),
                      ('lz4', lambda p: p.__version__),
+                     ('dask_ml', lambda p: p.__version__),
                      ('blosc', lambda p: p.__version__)]
 
 
-def get_versions():
-    """ Return basic information on our software installation,
-    and out installed versions of packages. """
+def get_versions(packages=None):
+    """
+    Return basic information on our software installation, and out installed versions of packages.
+    """
+    if packages is None:
+        packages = []
 
     d = {'host': get_system_info(),
          'packages': {'required': get_package_info(required_packages),
-                      'optional': get_package_info(optional_packages)}
+                      'optional': get_package_info(optional_packages + list(packages))}
          }
     return d
 
@@ -53,11 +59,31 @@ def get_system_info():
     return host
 
 
+def version_of_package(pkg):
+    """ Try a variety of common ways to get the version of a package """
+    with ignoring(AttributeError):
+        return pkg.__version__
+    with ignoring(AttributeError):
+        return str(pkg.version)
+    with ignoring(AttributeError):
+        return '.'.join(map(str, pkg.version_info))
+    return None
+
+
 def get_package_info(pkgs):
     """ get package versions for the passed required & optional packages """
 
     pversions = []
-    for (modname, ver_f) in pkgs:
+    for pkg in pkgs:
+        if isinstance(pkg, (tuple, list)):
+            modname, ver_f = pkg
+        else:
+            modname = pkg
+            ver_f = version_of_package
+
+        if ver_f is None:
+            ver_f = version_of_package
+
         try:
             mod = importlib.import_module(modname)
             ver = ver_f(mod)

From ee86eef4dfa292edafe193248f789dc30c7da414 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 29 Aug 2018 11:39:41 -0400
Subject: [PATCH 0073/1550] Add routes for solo bokeh figures in dashboard
 (#2185)

This adds new routes for individual plots.

This is to help some JLab extension work.
---
 distributed/bokeh/scheduler.py                | 70 +++++++++++++++++++
 distributed/bokeh/scheduler_html.py           | 11 +++
 .../bokeh/tests/test_scheduler_bokeh.py       | 15 ++--
 .../bokeh/tests/test_scheduler_bokeh_html.py  |  1 +
 4 files changed, 93 insertions(+), 4 deletions(-)

diff --git a/distributed/bokeh/scheduler.py b/distributed/bokeh/scheduler.py
index a282e012574..508d35eb037 100644
--- a/distributed/bokeh/scheduler.py
+++ b/distributed/bokeh/scheduler.py
@@ -1173,6 +1173,58 @@ def status_doc(scheduler, extra, doc):
         doc.template_variables.update(extra)
 
 
+def individual_task_stream_doc(scheduler, extra, doc):
+    task_stream = TaskStream(scheduler, n_rectangles=1000,
+                             clear_interval='10s', sizing_mode='stretch_both')
+    task_stream.update()
+    doc.add_periodic_callback(task_stream.update, 100)
+    doc.add_root(task_stream.root)
+
+
+def individual_load_doc(scheduler, extra, doc):
+    current_load = CurrentLoad(scheduler, height=160, sizing_mode='stretch_both')
+    current_load.update()
+    doc.add_periodic_callback(current_load.update, 100)
+    doc.add_root(current_load.root)
+
+
+def individual_progress_doc(scheduler, extra, doc):
+    task_progress = TaskProgress(scheduler, height=160, sizing_mode='stretch_both')
+    task_progress.update()
+    doc.add_periodic_callback(task_progress.update, 100)
+    doc.add_root(task_progress.root)
+
+
+def individual_graph_doc(scheduler, extra, doc):
+    with log_errors():
+        graph = GraphPlot(scheduler, sizing_mode='stretch_both')
+        graph.update()
+        doc.add_periodic_callback(graph.update, 200)
+        doc.add_root(graph.root)
+
+
+def individual_profile_doc(scheduler, extra, doc):
+    with log_errors():
+        prof = ProfileTimePlot(scheduler, sizing_mode='scale_width', doc=doc)
+        doc.add_root(prof.root)
+        prof.trigger_update()
+
+
+def individual_profile_server_doc(scheduler, extra, doc):
+    with log_errors():
+        prof = ProfileServer(scheduler, sizing_mode='scale_width', doc=doc)
+        doc.add_root(prof.root)
+        prof.trigger_update()
+
+
+def individual_workers_doc(scheduler, extra, doc):
+    with log_errors():
+        table = WorkerTable(scheduler)
+        table.update()
+        doc.add_periodic_callback(table.update, 500)
+        doc.add_root(table.root)
+
+
 def profile_doc(scheduler, extra, doc):
     with log_errors():
         doc.title = "Dask: Profile"
@@ -1220,6 +1272,16 @@ def __init__(self, scheduler, io_loop=None, prefix='', **kwargs):
         profile_server = Application(FunctionHandler(partial(profile_server_doc, scheduler, self.extra)))
         graph = Application(FunctionHandler(partial(graph_doc, scheduler, self.extra)))
 
+        individual_task_stream = Application(FunctionHandler(partial(
+            individual_task_stream_doc, scheduler, self.extra)))
+        individual_progress = Application(FunctionHandler(partial(individual_progress_doc, scheduler, self.extra)))
+        individual_graph = Application(FunctionHandler(partial(individual_graph_doc, scheduler, self.extra)))
+        individual_profile = Application(FunctionHandler(partial(individual_profile_doc, scheduler, self.extra)))
+        individual_profile_server = Application(FunctionHandler(partial(
+            individual_profile_server_doc, scheduler, self.extra)))
+        individual_load = Application(FunctionHandler(partial(individual_load_doc, scheduler, self.extra)))
+        individual_workers = Application(FunctionHandler(partial(individual_workers_doc, scheduler, self.extra)))
+
         self.apps = {
             '/system': systemmonitor,
             '/stealing': stealing,
@@ -1231,6 +1293,14 @@ def __init__(self, scheduler, io_loop=None, prefix='', **kwargs):
             '/profile': profile,
             '/profile-server': profile_server,
             '/graph': graph,
+
+            '/individual-task-stream': individual_task_stream,
+            '/individual-progress': individual_progress,
+            '/individual-graph': individual_graph,
+            '/individual-profile': individual_profile,
+            '/individual-profile-server': individual_profile_server,
+            '/individual-load': individual_load,
+            '/individual-workers': individual_workers,
         }
 
         self.loop = io_loop or scheduler.loop
diff --git a/distributed/bokeh/scheduler_html.py b/distributed/bokeh/scheduler_html.py
index bab032c7be7..086155be957 100644
--- a/distributed/bokeh/scheduler_html.py
+++ b/distributed/bokeh/scheduler_html.py
@@ -152,6 +152,16 @@ def get(self):
             self.render('json-index.html', routes=r, title='Index of JSON routes', **self.extra)
 
 
+class IndividualPlots(RequestHandler):
+    def get(self):
+        bokeh_server = self.server.services['bokeh']
+        result = {uri.strip('/').replace('-', ' ').title(): uri
+                  for uri in bokeh_server.apps
+                  if uri.lstrip('/').startswith('individual-')
+                  and not uri.endswith('.json')}
+        self.write(result)
+
+
 routes = [
         (r'info/main/workers.html', Workers),
         (r'info/worker/(.*).html', Worker),
@@ -163,6 +173,7 @@ def get(self):
         (r'json/counts.json', CountsJSON),
         (r'json/identity.json', IdentityJSON),
         (r'json/index.html', IndexJSON),
+        (r'individual-plots.json', IndividualPlots),
 ]
 
 
diff --git a/distributed/bokeh/tests/test_scheduler_bokeh.py b/distributed/bokeh/tests/test_scheduler_bokeh.py
index 1c07b437f21..ef8599bb6bd 100644
--- a/distributed/bokeh/tests/test_scheduler_bokeh.py
+++ b/distributed/bokeh/tests/test_scheduler_bokeh.py
@@ -1,10 +1,11 @@
 from __future__ import print_function, division, absolute_import
 
+import json
+import sys
 from time import sleep
 
 import pytest
 pytest.importorskip('bokeh')
-import sys
 from toolz import first
 from tornado import gen
 from tornado.httpclient import AsyncHTTPClient
@@ -35,17 +36,23 @@
              scheduler_kwargs={'services': {('bokeh', 0):  BokehScheduler}})
 def test_simple(c, s, a, b):
     assert isinstance(s.services['bokeh'], BokehScheduler)
+    port = s.services['bokeh'].port
 
     future = c.submit(sleep, 1)
     yield gen.sleep(0.1)
 
     http_client = AsyncHTTPClient()
     for suffix in ['system', 'counters', 'workers', 'status', 'tasks',
-                   'stealing', 'graph']:
-        response = yield http_client.fetch('http://localhost:%d/%s'
-                                           % (s.services['bokeh'].port, suffix))
+                   'stealing', 'graph', 'individual-task-stream', 'individual-progress',
+                   'individual-graph', 'individual-load',
+                   'individual-profile']:
+        response = yield http_client.fetch('http://localhost:%d/%s' % (port, suffix))
         assert 'bokeh' in response.body.decode().lower()
 
+    response = yield http_client.fetch('http://localhost:%d/individual-plots.json' % port)
+    response = json.loads(response.body.decode())
+    assert response
+
 
 @gen_cluster(client=True, worker_kwargs=dict(services={'bokeh': BokehWorker}))
 def test_basic(c, s, a, b):
diff --git a/distributed/bokeh/tests/test_scheduler_bokeh_html.py b/distributed/bokeh/tests/test_scheduler_bokeh_html.py
index 89fe9c47ca6..52eb65c803f 100644
--- a/distributed/bokeh/tests/test_scheduler_bokeh_html.py
+++ b/distributed/bokeh/tests/test_scheduler_bokeh_html.py
@@ -30,6 +30,7 @@ def test_connect(c, s, a, b):
                    'json/counts.json',
                    'json/identity.json',
                    'json/index.html',
+                   'individual-plots.json',
                    ]:
         response = yield http_client.fetch('http://localhost:%d/%s'
                                            % (s.services['bokeh'].port, suffix))

From 32c79ecba86ca1673a4d40a395d429d92e643202 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 29 Aug 2018 12:09:14 -0400
Subject: [PATCH 0074/1550] Fix intermittent failure for
 test_dont_steal_long_running_tasks (#2218)

---
 distributed/tests/test_steal.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/distributed/tests/test_steal.py b/distributed/tests/test_steal.py
index 01193b61e20..35315d531ca 100644
--- a/distributed/tests/test_steal.py
+++ b/distributed/tests/test_steal.py
@@ -587,10 +587,12 @@ def long(delay):
 
     yield gen.sleep(0.2)
 
-    assert sum(1 for k in s.processing[b.address] if k.startswith('long')) <= nb
-
     yield wait(long_tasks)
 
+    for t in long_tasks:
+        assert (sum(log[1] == 'executing' for log in a.story(t)) +
+                sum(log[1] == 'executing' for log in b.story(t))) <= 1
+
 
 @gen_cluster(client=True, ncores=[('127.0.0.1', 5)] * 2)
 def test_cleanup_repeated_tasks(c, s, a, b):

From b391a4f09a860fa5e4584ff1b7a008179e141225 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 29 Aug 2018 12:12:11 -0400
Subject: [PATCH 0075/1550] Be resilient to missing dep after busy signal
 (#2217)

---
 distributed/worker.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/worker.py b/distributed/worker.py
index bd9f6040e5f..66535e4517d 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -1898,7 +1898,7 @@ def gather_dep(self, worker, dep, deps, total_nbytes, cause=None):
                 if response['status'] == 'busy':
                     self.log.append(('busy-gather', worker, deps))
                     for dep in deps:
-                        if self.dep_state[dep] == 'flight':
+                        if self.dep_state.get(dep, None) == 'flight':
                             self.transition_dep(dep, 'waiting')
                     return
 

From 182a60ef404e7e73a1860f641737e65c9751fd78 Mon Sep 17 00:00:00 2001
From: Derek Ludwig <derek.s.ludwig@gmail.com>
Date: Wed, 29 Aug 2018 13:29:39 -0700
Subject: [PATCH 0076/1550] Use CSS Grid to layout status page on the dashboard
 (#2213)

This uses CSS Grid and the new Bokeh templates to layout the status page in a way that is more responsive to wide and narrow screens.

This removes the div around single-plot pages, which resolves a resizing issue that arose in Bokeh 0.13.0

This sets the minimum bokeh version to 0.13

Fixes https://github.com/dask/distributed/issues/2194
---
 distributed/bokeh/components.py               |  34 +++--
 distributed/bokeh/core.py                     |   6 +-
 distributed/bokeh/scheduler.py                | 137 ++++++++---------
 distributed/bokeh/static/dask_horizontal.svg  |  28 ----
 distributed/bokeh/static/dask_icon_no_pad.svg |  35 +++++
 distributed/bokeh/template.html               | 123 ---------------
 distributed/bokeh/templates/base.html         | 141 ++++++++++++++++++
 distributed/bokeh/templates/simple.html       |   6 +
 distributed/bokeh/templates/status.html       |  79 ++++++++++
 .../bokeh/tests/test_scheduler_bokeh.py       |   3 +-
 distributed/bokeh/worker.py                   |  20 +--
 11 files changed, 364 insertions(+), 248 deletions(-)
 delete mode 100644 distributed/bokeh/static/dask_horizontal.svg
 create mode 100644 distributed/bokeh/static/dask_icon_no_pad.svg
 delete mode 100644 distributed/bokeh/template.html
 create mode 100644 distributed/bokeh/templates/base.html
 create mode 100644 distributed/bokeh/templates/simple.html
 create mode 100644 distributed/bokeh/templates/status.html

diff --git a/distributed/bokeh/components.py b/distributed/bokeh/components.py
index f3d1ca4c3d4..5626977483f 100644
--- a/distributed/bokeh/components.py
+++ b/distributed/bokeh/components.py
@@ -110,19 +110,35 @@ def task_stream_figure(clear_interval='20s', **kwargs):
         y_range = DataRange1d(range_padding=0)
 
         root = figure(
-            title="Task Stream", id='bk-task-stream-plot',
-            x_range=x_range, y_range=y_range, toolbar_location="above",
-            x_axis_type='datetime', min_border_right=35, tools='', **kwargs)
-        root.yaxis.axis_label = 'Worker Core'
-
-        rect = root.rect(source=source, x="start", y="y",
-            width="duration", height=0.4, fill_color="color",
-            line_color="color", line_alpha=0.6, fill_alpha="alpha",
-            line_width=3)
+            name='task_stream',
+            title="Task Stream",
+            id='bk-task-stream-plot',
+            x_range=x_range,
+            y_range=y_range,
+            toolbar_location="above",
+            x_axis_type='datetime',
+            min_border_right=35,
+            tools='',
+            **kwargs
+        )
+
+        rect = root.rect(
+            source=source,
+            x="start",
+            y="y",
+            width="duration",
+            height=0.4,
+            fill_color="color",
+            line_color="color",
+            line_alpha=0.6,
+            fill_alpha="alpha",
+            line_width=3
+        )
         rect.nonselection_glyph = None
 
         root.yaxis.major_label_text_alpha = 0
         root.yaxis.minor_tick_line_alpha = 0
+        root.yaxis.major_tick_line_alpha = 0
         root.xgrid.visible = False
 
         hover = HoverTool(
diff --git a/distributed/bokeh/core.py b/distributed/bokeh/core.py
index f7409c043a0..9e8540f0037 100644
--- a/distributed/bokeh/core.py
+++ b/distributed/bokeh/core.py
@@ -9,8 +9,10 @@
 from tornado import web
 
 
-if LooseVersion(bokeh.__version__) < LooseVersion('0.12.6'):
-    raise ImportError("Dask needs bokeh >= 0.12.6")
+if LooseVersion(bokeh.__version__) < LooseVersion('0.13.0'):
+    warnings.warn("\nDask needs bokeh >= 0.13.0 for the dashboard."
+                  "\nContinuing without the dashboard.")
+    raise ImportError("Dask needs bokeh >= 0.13.0")
 
 
 class BokehServer(object):
diff --git a/distributed/bokeh/scheduler.py b/distributed/bokeh/scheduler.py
index 508d35eb037..dbef4064905 100644
--- a/distributed/bokeh/scheduler.py
+++ b/distributed/bokeh/scheduler.py
@@ -10,8 +10,6 @@
 import os
 
 import bokeh
-from bokeh.application import Application
-from bokeh.application.handlers.function import FunctionHandler
 from bokeh.layouts import column, row
 from bokeh.models import (ColumnDataSource, DataRange1d, HoverTool, ResetTool,
                           PanTool, WheelZoomTool, TapTool, OpenURL, Range1d, Plot, Quad,
@@ -51,12 +49,8 @@
 
 PROFILING = False
 
-import jinja2
-
-with open(os.path.join(os.path.dirname(__file__), 'template.html')) as f:
-    template_source = f.read()
-
-template = jinja2.Template(template_source)
+from jinja2 import Environment, FileSystemLoader
+env = Environment(loader=FileSystemLoader(os.path.join(os.path.dirname(__file__), 'templates')))
 
 template_variables = {'pages': ['status', 'workers', 'tasks', 'system', 'profile', 'graph']}
 
@@ -183,6 +177,7 @@ def __init__(self, scheduler, **kwargs):
 
             self.root = figure(title='Tasks Processing',
                                id='bk-nprocessing-histogram-plot',
+                               name='processing_hist',
                                **kwargs)
 
             self.root.xaxis.minor_tick_line_alpha = 0
@@ -215,6 +210,7 @@ def __init__(self, scheduler, **kwargs):
                                             'top': [0, 0]})
 
             self.root = figure(title='Bytes Stored',
+                               name='nbytes_hist',
                                id='bk-nbytes-histogram-plot',
                                **kwargs)
             self.root.xaxis[0].formatter = NumeralTickFormatter(format='0.0 b')
@@ -258,7 +254,7 @@ def __init__(self, scheduler, width=600, **kwargs):
                                             'bokeh_address': ['', '']})
 
             processing = figure(title='Tasks Processing', tools='', id='bk-nprocessing-plot',
-                                width=int(width / 2), **kwargs)
+                                name='processing_hist', width=int(width / 2), **kwargs)
             rect = processing.rect(source=self.source,
                                    x='nprocessing-half', y='y',
                                    width='nprocessing', height=1,
@@ -268,7 +264,7 @@ def __init__(self, scheduler, width=600, **kwargs):
 
             nbytes = figure(title='Bytes stored', tools='',
                             id='bk-nbytes-worker-plot', width=int(width / 2),
-                            **kwargs)
+                            name='nbytes_hist',  **kwargs)
             rect = nbytes.rect(source=self.source,
                                x='nbytes-half', y='y',
                                width='nbytes', height=1,
@@ -306,7 +302,6 @@ def __init__(self, scheduler, width=600, **kwargs):
             self.nbytes_figure = nbytes
 
             processing.y_range = nbytes.y_range
-            self.root = row(nbytes, processing, sizing_mode='scale_width')
 
     def update(self):
         with log_errors():
@@ -763,7 +758,7 @@ def __init__(self, scheduler, **kwargs):
         y_range = Range1d(-8, 0)
 
         self.root = figure(
-            id='bk-task-progress-plot', title='Progress',
+            id='bk-task-progress-plot', title='Progress', name='task_progress',
             x_range=x_range, y_range=y_range, toolbar_location=None, **kwargs
         )
         self.root.line(  # just to define early ranges
@@ -1061,7 +1056,7 @@ def systemmonitor_doc(scheduler, extra, doc):
         doc.add_periodic_callback(sysmon.update, 500)
 
         doc.add_root(column(sysmon.root, sizing_mode='scale_width'))
-        doc.template = template
+        doc.template = env.get_template('simple.html')
         doc.template_variables['active_page'] = 'system'
         doc.template_variables.update(extra)
 
@@ -1081,7 +1076,7 @@ def stealing_doc(scheduler, extra, doc):
                             stealing_events.root,
                             sizing_mode='scale_width'))
 
-        doc.template = template
+        doc.template = env.get_template('simple.html')
         doc.template_variables['active_page'] = 'stealing'
         doc.template_variables.update(extra)
 
@@ -1093,7 +1088,7 @@ def events_doc(scheduler, extra, doc):
         doc.add_periodic_callback(events.update, 500)
         doc.title = "Dask: Scheduler Events"
         doc.add_root(column(events.root, sizing_mode='scale_width'))
-        doc.template = template
+        doc.template = env.get_template('simple.html')
         doc.template_variables['active_page'] = 'events'
         doc.template_variables.update(extra)
 
@@ -1105,7 +1100,7 @@ def workers_doc(scheduler, extra, doc):
         doc.add_periodic_callback(table.update, 500)
         doc.title = "Dask: Workers"
         doc.add_root(table.root)
-        doc.template = template
+        doc.template = env.get_template('simple.html')
         doc.template_variables['active_page'] = 'workers'
         doc.template_variables.update(extra)
 
@@ -1118,7 +1113,7 @@ def tasks_doc(scheduler, extra, doc):
         doc.add_periodic_callback(ts.update, 5000)
         doc.title = "Dask: Task Stream"
         doc.add_root(ts.root)
-        doc.template = template
+        doc.template = env.get_template('simple.html')
         doc.template_variables['active_page'] = 'tasks'
         doc.template_variables.update(extra)
 
@@ -1131,7 +1126,7 @@ def graph_doc(scheduler, extra, doc):
         doc.add_periodic_callback(graph.update, 200)
         doc.add_root(graph.root)
 
-        doc.template = template
+        doc.template = env.get_template('simple.html')
         doc.template_variables['active_page'] = 'graph'
         doc.template_variables.update(extra)
 
@@ -1139,36 +1134,38 @@ def graph_doc(scheduler, extra, doc):
 def status_doc(scheduler, extra, doc):
     with log_errors():
         task_stream = TaskStream(scheduler, n_rectangles=1000,
-                                 clear_interval='10s', height=350)
+                                 clear_interval='10s', sizing_mode='stretch_both')
         task_stream.update()
         doc.add_periodic_callback(task_stream.update, 100)
 
-        task_progress = TaskProgress(scheduler, height=160)
+        task_progress = TaskProgress(scheduler, sizing_mode='stretch_both')
         task_progress.update()
         doc.add_periodic_callback(task_progress.update, 100)
 
         if len(scheduler.workers) < 50:
-            current_load = CurrentLoad(scheduler, height=160)
+            current_load = CurrentLoad(scheduler, sizing_mode='stretch_both')
             current_load.update()
             doc.add_periodic_callback(current_load.update, 100)
-            current_load_fig = current_load.root
+            doc.add_root(current_load.nbytes_figure)
+            doc.add_root(current_load.processing_figure)
         else:
-            nbytes_hist = NBytesHistogram(scheduler, width=300, height=160)
+            nbytes_hist = NBytesHistogram(scheduler, sizing_mode='stretch_both')
             nbytes_hist.update()
-            processing_hist = ProcessingHistogram(scheduler, width=300,
-                                                  height=160)
+            processing_hist = ProcessingHistogram(scheduler, sizing_mode='stretch_both')
             processing_hist.update()
             doc.add_periodic_callback(nbytes_hist.update, 100)
             doc.add_periodic_callback(processing_hist.update, 100)
             current_load_fig = row(nbytes_hist.root, processing_hist.root,
-                                   sizing_mode='scale_width')
+                                   sizing_mode='stretch_both')
+
+            doc.add_root(nbytes_hist.root)
+            doc.add_root(processing_hist.root)
 
         doc.title = "Dask: Status"
-        doc.add_root(column(current_load_fig,
-                            task_stream.root,
-                            task_progress.root,
-                            sizing_mode='scale_width'))
-        doc.template = template
+        doc.add_root(task_progress.root)
+        doc.add_root(task_stream.root)
+
+        doc.template = env.get_template('status.html')
         doc.template_variables['active_page'] = 'status'
         doc.template_variables.update(extra)
 
@@ -1181,11 +1178,18 @@ def individual_task_stream_doc(scheduler, extra, doc):
     doc.add_root(task_stream.root)
 
 
-def individual_load_doc(scheduler, extra, doc):
-    current_load = CurrentLoad(scheduler, height=160, sizing_mode='stretch_both')
+def individual_nbytes_doc(scheduler, extra, doc):
+    current_load = CurrentLoad(scheduler, sizing_mode='stretch_both')
     current_load.update()
     doc.add_periodic_callback(current_load.update, 100)
-    doc.add_root(current_load.root)
+    doc.add_root(current_load.nbytes_figure)
+
+
+def individual_nprocessing_doc(scheduler, extra, doc):
+    current_load = CurrentLoad(scheduler, sizing_mode='stretch_both')
+    current_load.update()
+    doc.add_periodic_callback(current_load.update, 100)
+    doc.add_root(current_load.processing_figure)
 
 
 def individual_progress_doc(scheduler, extra, doc):
@@ -1230,7 +1234,7 @@ def profile_doc(scheduler, extra, doc):
         doc.title = "Dask: Profile"
         prof = ProfileTimePlot(scheduler, sizing_mode='scale_width', doc=doc)
         doc.add_root(prof.root)
-        doc.template = template
+        doc.template = env.get_template('simple.html')
         doc.template_variables['active_page'] = 'profile'
         doc.template_variables.update(extra)
 
@@ -1242,7 +1246,7 @@ def profile_server_doc(scheduler, extra, doc):
         doc.title = "Dask: Profile of Event Loop"
         prof = ProfileServer(scheduler, sizing_mode='scale_width', doc=doc)
         doc.add_root(prof.root)
-        doc.template = template
+        doc.template = env.get_template('simple.html')
         # doc.template_variables['active_page'] = 'profile'
         doc.template_variables.update(extra)
 
@@ -1261,48 +1265,31 @@ def __init__(self, scheduler, io_loop=None, prefix='', **kwargs):
         self.server_kwargs = kwargs
         self.server_kwargs['prefix'] = prefix or None
 
-        systemmonitor = Application(FunctionHandler(partial(systemmonitor_doc, scheduler, self.extra)))
-        workers = Application(FunctionHandler(partial(workers_doc, scheduler, self.extra)))
-        stealing = Application(FunctionHandler(partial(stealing_doc, scheduler, self.extra)))
-        counters = Application(FunctionHandler(partial(counters_doc, scheduler, self.extra)))
-        events = Application(FunctionHandler(partial(events_doc, scheduler, self.extra)))
-        tasks = Application(FunctionHandler(partial(tasks_doc, scheduler, self.extra)))
-        status = Application(FunctionHandler(partial(status_doc, scheduler, self.extra)))
-        profile = Application(FunctionHandler(partial(profile_doc, scheduler, self.extra)))
-        profile_server = Application(FunctionHandler(partial(profile_server_doc, scheduler, self.extra)))
-        graph = Application(FunctionHandler(partial(graph_doc, scheduler, self.extra)))
-
-        individual_task_stream = Application(FunctionHandler(partial(
-            individual_task_stream_doc, scheduler, self.extra)))
-        individual_progress = Application(FunctionHandler(partial(individual_progress_doc, scheduler, self.extra)))
-        individual_graph = Application(FunctionHandler(partial(individual_graph_doc, scheduler, self.extra)))
-        individual_profile = Application(FunctionHandler(partial(individual_profile_doc, scheduler, self.extra)))
-        individual_profile_server = Application(FunctionHandler(partial(
-            individual_profile_server_doc, scheduler, self.extra)))
-        individual_load = Application(FunctionHandler(partial(individual_load_doc, scheduler, self.extra)))
-        individual_workers = Application(FunctionHandler(partial(individual_workers_doc, scheduler, self.extra)))
-
         self.apps = {
-            '/system': systemmonitor,
-            '/stealing': stealing,
-            '/workers': workers,
-            '/events': events,
-            '/counters': counters,
-            '/tasks': tasks,
-            '/status': status,
-            '/profile': profile,
-            '/profile-server': profile_server,
-            '/graph': graph,
-
-            '/individual-task-stream': individual_task_stream,
-            '/individual-progress': individual_progress,
-            '/individual-graph': individual_graph,
-            '/individual-profile': individual_profile,
-            '/individual-profile-server': individual_profile_server,
-            '/individual-load': individual_load,
-            '/individual-workers': individual_workers,
+            '/system': systemmonitor_doc,
+            '/stealing': stealing_doc,
+            '/workers': workers_doc,
+            '/events': events_doc,
+            '/counters': counters_doc,
+            '/tasks': tasks_doc,
+            '/status': status_doc,
+            '/profile': profile_doc,
+            '/profile-server': profile_server_doc,
+            '/graph': graph_doc,
+
+            '/individual-task-stream': individual_task_stream_doc,
+            '/individual-progress': individual_progress_doc,
+            '/individual-graph': individual_graph_doc,
+            '/individual-profile': individual_profile_doc,
+            '/individual-profile-server': individual_profile_server_doc,
+            '/individual-nbytes': individual_nbytes_doc,
+            '/individual-nprocessing': individual_nprocessing_doc,
+            '/individual-workers': individual_workers_doc,
         }
 
+        self.apps = {k: partial(v, scheduler, self.extra)
+                     for k, v in self.apps.items()}
+
         self.loop = io_loop or scheduler.loop
         self.server = None
 
diff --git a/distributed/bokeh/static/dask_horizontal.svg b/distributed/bokeh/static/dask_horizontal.svg
deleted file mode 100644
index bfce8ca6b67..00000000000
--- a/distributed/bokeh/static/dask_horizontal.svg
+++ /dev/null
@@ -1,28 +0,0 @@
-<svg id="Layer_1" 
-     data-name="Layer 1" 
-     xmlns="http://www.w3.org/2000/svg" 
-     xmlns:xlink="http://www.w3.org/1999/xlink" 
-     viewBox="40 30 470 190">
-  <defs>
-    <linearGradient id="linear-gradient" x1="154.55" y1="173.33" x2="242.36" y2="173.33" gradientTransform="translate(-26.62 -73.73) rotate(7.91)" gradientUnits="userSpaceOnUse">
-      <stop offset="0.01" stop-color="#c7422f" />
-      <stop offset="0.37" stop-color="#d46e43" />
-      <stop offset="1" stop-color="#eeb575" />
-    </linearGradient>
-    <linearGradient id="linear-gradient-2" x1="181.83" y1="171.07" x2="221.39" y2="171.07" gradientTransform="translate(-26.62 -73.73) rotate(7.91)" gradientUnits="userSpaceOnUse">
-      <stop offset="0.21" stop-color="#cf603b" />
-      <stop offset="1" stop-color="#eeb575" />
-    </linearGradient>
-    <linearGradient id="linear-gradient-3" x1="107.2" y1="175.53" x2="204.37" y2="175.53" xlink:href="#linear-gradient-2" />
-  </defs>
-
-  <title>Dask</title>
-
-  <path d="M214.33,85.8h36.1c24.73,0.28,30.29,21.36,30.29,40,0,39.54-20.35,39.14-30.29,39.14h-36.1V85.8Zm34.74,64.52c12.94,0,15.6-11.66,15.6-24.55,0-18.13-6.31-25.38-15.6-25.38h-18.7v49.93h18.7Z" style="fill:#101011" />
-  <path d="M311.7,86.08h18.46l27.18,78.84h-17L333.87,147H306.31l-6.92,17.88h-17Zm18.18,47.34-9.09-26.87-10,26.87h19Z" style="fill:#101011" />
-  <path d="M364.71,106.58c0-11.68,4.7-20.91,20.54-20.91,7.19,0,27.91,1.13,38.24,3v11.76s-21.49-.8-35.82-0.8c-5.45,0-6.91,3.45-6.91,7.19v5.89c0,6.07,2.67,6.19,6.91,6.19h20.61c14,0,18.95,8.92,18.95,20v7.75c0,15.95-9.55,19.64-18.95,19.64-6.58,0-35.16-.8-40.53-3.24V151.58s23.47,0.61,37.14.61c4.9,0,6.29-5.82,6.29-5.82v-7c0-4-1.56-5.91-6.29-5.91H384.76c-14.5,0-20.05-6.54-20.05-19.81v-7.1Z" style="fill:#101011" />
-  <path d="M438.85,86.08h15.56v33.19h8.64l22.75-33.19h18.67l-27.35,40.11,27.35,38.87H485.81l-23-31.67h-8.37v31.67H438.85v-79Z" style="fill:#101011" />
-  <path d="M192.41,110.26q0.17-1.83.29-3.66a119.55,119.55,0,0,0-12.24-60.92L173.64,32l-3.16,15a109,109,0,0,1-64.2,77.79l-4.69,2,1.78,4.78a107.9,107.9,0,0,1,6.31,48.3A109.44,109.44,0,0,1,104,205.75L100.36,216l10.28-3.38A119.71,119.71,0,0,0,192.41,110.26ZM122.68,196l-5.48,2.79,1.2-6a120.35,120.35,0,0,0,1.75-12,118.87,118.87,0,0,0-4.52-45.89l-0.73-2.41,2.25-1.12A120.24,120.24,0,0,0,173.4,72.22l3.51-8L179,72.7a107.63,107.63,0,0,1,2.69,36.54A108.48,108.48,0,0,1,122.68,196Z" style="fill:url(#linear-gradient)" />
-  <path d="M166.91,116.14c4.13-9.12,8.42-31.77,8.15-33.46A126,126,0,0,1,160,105.07c-0.85,2.24-1.74,4.47-2.74,6.67h0a108.87,108.87,0,0,1-31.62,40.47q0.85,6.14,1.07,12.36A119.4,119.4,0,0,0,166.91,116.14Z" style="fill:url(#linear-gradient-2)" />
-  <path d="M104.08,165.48a109,109,0,0,1-30.87,9.17l-6.08.86,3.08-5.31a120.74,120.74,0,0,0,5.54-10.74,118.79,118.79,0,0,0,10.62-44.87l0.09-2.51,2.49-.33a120.18,120.18,0,0,0,54-21.47A102.79,102.79,0,0,0,161.57,56.9a109,109,0,0,1-80.84,45l-5.08.35,0.13,5.1a107.92,107.92,0,0,1-9.72,47.73,109.43,109.43,0,0,1-13.83,22.67l-6.71,8.49,10.82,0.14a119.25,119.25,0,0,0,47.52-9.26A104.21,104.21,0,0,0,104.08,165.48Z" style="fill:url(#linear-gradient-3)" />
-</svg>
diff --git a/distributed/bokeh/static/dask_icon_no_pad.svg b/distributed/bokeh/static/dask_icon_no_pad.svg
new file mode 100644
index 00000000000..8999ed4a720
--- /dev/null
+++ b/distributed/bokeh/static/dask_icon_no_pad.svg
@@ -0,0 +1,35 @@
+<svg id="Layer_1" 
+     data-name="Layer 1"  
+     xmlns="http://www.w3.org/2000/svg"
+     xmlns:xlink="http://www.w3.org/1999/xlink" 
+     viewBox="80 55 300 380">
+  <defs>
+    <linearGradient id="linear-gradient" 
+                    x1="244.57" y1="186.17" x2="415.51" y2="186.17" 
+                    gradientTransform="translate(-18.34 16.11) rotate(7.91)" 
+                    gradientUnits="userSpaceOnUse">
+      <stop offset="0.01" stop-color="#c7422f"/>
+      <stop offset="0.37" stop-color="#d46e43"/>
+      <stop offset="1.00" stop-color="#eeb575"/>
+    </linearGradient>
+    <linearGradient id="linear-gradient-2" 
+                    x1="297.68" y1="181.79" x2="374.68" y2="181.79"
+                    gradientTransform="translate(-18.34 16.11) rotate(7.91)"
+                    gradientUnits="userSpaceOnUse">
+      <stop offset="0.21" stop-color="#cf603b"/>
+      <stop offset="1" stop-color="#eeb575"/>
+    </linearGradient>
+    <linearGradient id="linear-gradient-3" 
+                    x1="152.4" y1="190.46" x2="341.55" y2="190.46"
+                    xlink:href="#linear-gradient-2"/>
+  </defs>
+
+  <title>Dask</title>
+
+  <path d="M373.09,216.74q0.33-3.56.56-7.12A232.73,232.73,0,0,0,349.81,91L336.54,64.38l-6.15,29.14a212.21,212.21,0,0,1-125,151.43l-9.13,3.86,3.46,9.3a210,210,0,0,1,12.29,94,213,213,0,0,1-11.13,50.49l-7,19.87,20-6.58A233,233,0,0,0,373.09,216.74ZM237.35,383.64l-10.66,5.43L229,377.34a234.27,234.27,0,0,0,3.41-23.27,231.39,231.39,0,0,0-8.81-89.33l-1.41-4.69,4.37-2.18A234.07,234.07,0,0,0,336.06,142.69l6.83-15.58,4.12,16.5a209.52,209.52,0,0,1,5.23,71.12A211.17,211.17,0,0,1,237.35,383.64Z"
+        style="fill:url(#linear-gradient)"/>
+  <path d="M323.44,228.19c8-17.75,16.39-61.85,15.86-65.14a245.35,245.35,0,0,1-29.41,43.59c-1.65,4.37-3.39,8.7-5.33,13h0A211.92,211.92,0,0,1,243,298.41q1.65,11.94,2.09,24.05A232.43,232.43,0,0,0,323.44,228.19Z"
+        style="fill:url(#linear-gradient-2)"/>
+  <path d="M201.14,324.23a212.21,212.21,0,0,1-60.09,17.84l-11.83,1.67,6-10.33A235,235,0,0,0,146,312.5a231.24,231.24,0,0,0,20.68-87.35l0.18-4.89,4.85-.65a234,234,0,0,0,105.16-41.79,200.09,200.09,0,0,0,36.17-65,212.27,212.27,0,0,1-157.36,87.56l-9.89.69L146,211a210.08,210.08,0,0,1-18.92,92.92,213,213,0,0,1-26.92,44.14L87.13,364.6l21.07,0.28a232.13,232.13,0,0,0,92.5-18A202.86,202.86,0,0,0,201.14,324.23Z"
+        style="fill:url(#linear-gradient-3)"/>
+</svg>
diff --git a/distributed/bokeh/template.html b/distributed/bokeh/template.html
deleted file mode 100644
index e0b34e00448..00000000000
--- a/distributed/bokeh/template.html
+++ /dev/null
@@ -1,123 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-      <meta charset="utf-8"></meta>
-    <title>Dask - Status dashboard</title>
-    {{ bokeh_css }}
-    {{ bokeh_js }}
-    <style>
-      html {
-        width: 100%;
-        height: 100%;
-        background: #FAFAFA;
-      }
-      body {
-        min-height: 100%;
-        margin: 0 4%;
-        padding: 0px 20px;
-        border-left:  1px solid #F0F0F0;
-        border-right: 1px solid #F0F0F0;
-        background: #FFF;
-      }
-      @media only screen and (min-width: 90.063em) and (max-width: 120em) {
-        /* min-width 1441px and max-width 1920px */
-        body {
-          margin: 0 20%;
-        }
-      }
-      @media only screen and (min-width: 120.063em) {
-        /* min-width 1921px */
-        body {
-          margin: 0 20%;
-        }
-      }
-
-      body {
-        font-family: Helvetica, Arial, sans-serif;
-      }
-      .dashboard {
-        clear: both;
-      }
-      .bk-plot-layout:after {
-        clear: both;
-        display: block;
-        height: 0;
-        overflow: hidden;
-        visibility: hidden;
-        content: ".";
-        font-size: 0;
-      }
-      .navbar img {
-        height: 36px;
-        float: right;
-        margin: 3px;
-        padding: 0px 25px;
-      }
-      .navbar ul {
-          list-style-type: none;
-          margin: 0;
-          padding: 0;
-          overflow: hidden;
-          background-color: #EEE;
-      }
-
-      .navbar li {
-          float: left;
-          font-size: 17px;
-          transition: .3s background-color;
-      }
-
-      .navbar li a {
-          display: block;
-          color: black;
-          text-align: center;
-          padding: 11px 16px;
-          text-decoration: none;
-      }
-
-      .navbar li:hover {
-          background-color: #eaaa6d;
-      }
-
-      .active {
-          background-color: rgba(234, 170, 109, 0.7);
-      }
-
-      a {
-        color: #1f5396;
-        text-decoration: none;
-      }
-      a:hover,
-      a:focus {
-        color: #123057;
-        text-decoration: underline;
-      }
-      a:focus {
-        outline: 5px auto -webkit-focus-ring-color;
-        outline-offset: -2px;
-      } .bk-root .bk-toolbar-box .bk-toolbar-right {
-        top: 4px;
-        right: 4px;
-      }
-    </style>
-          <style>
          .commit-tease,
          .user-profile-mini-avatar,
          .avatar,
          .vcard-details,
          .signup-prompt-bg {
            display: none !IMPORTANT;
          }
        </style>
         <script>
          document.addEventListener('DOMContentLoaded', function() {
            this.querySelectorAll('a').forEach(anchor => {
              anchor.addEventListener('click', e => {
                e.preventDefault();

                const redact = new URLSearchParams(window.location.search).get('redact');
                const hasExistingParams = anchor.href.includes('?');
                window.location.href = anchor.href + (hasExistingParams ? `&redact=${redact}` : `?redact=${redact}`);
              });
            });
          });
        </script>
 </head>
-  <body>
-    <div class="navbar">
-      <ul>
-          <li>
-            <a href="http://dask.pydata.org/en/latest/" style="padding: 0px 0px">
-              <img src="statics/dask_horizontal.svg"></img>
-            </a>
-          </li>
-          {% for page in pages %}
-          <li{% if page == active_page %} class="active"{% endif %}><a href="{{ page }}">{{ page|title }}</a></li>
-          {% endfor %}
-          <li><a href="info/main/workers.html">Info</a></li>
-      </ul>
-    </div>
-    <div class="dashboard">
-      {{ plot_div }}
-    </div>
-    {{ plot_script }}
-  </body>
-</html>
diff --git a/distributed/bokeh/templates/base.html b/distributed/bokeh/templates/base.html
new file mode 100644
index 00000000000..ad9ffd152b4
--- /dev/null
+++ b/distributed/bokeh/templates/base.html
@@ -0,0 +1,141 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta charset="utf-8"/>
+    <title>Dask Diagnostic UI</title>
+    <meta name='viewport' content='width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no' />
+    <style>
+      html {
+        width: 100%;
+        height: 100%;
+        background: #FAFAFA;
+      }
+      body {
+        height: 100%;
+        width: 100%;
+        margin: 0;
+        padding: 0px 10px;
+        border-left:  1px solid #F0F0F0;
+        border-right: 1px solid #F0F0F0;
+        background: #FFF;
+      }
+      @media only screen and (min-width: 90.063em) and (max-width: 120em) {
+        /* min-width 1441px and max-width 1920px */
+        body {
+          margin: 0;
+        }
+      }
+      @media only screen and (min-width: 120.063em) {
+        /* min-width 1921px */
+        body {
+          margin: 0;
+        }
+      }
+      body {
+        font-family: Helvetica, Arial, sans-serif;
+      }
+      .dashboard {
+        clear: both;
+      }
+      .bk-plot-layout:after {
+        clear: both;
+        display: block;
+        height: 0;
+        overflow: hidden;
+        visibility: hidden;
+        content: ".";
+        font-size: 0;
+      }
+      .navbar img {
+        height: 36px;
+        float: right;
+        margin: 3px;
+        padding: 0px 25px;
+      }
+      .navbar ul {
+        list-style-type: none;
+        margin: 0;
+        padding: 0;
+        overflow: hidden;
+        background-color: #EEE;
+      }
+      .navbar li {
+        float: left;
+        font-size: 17px;
+        transition: .3s background-color;
+      }
+      .navbar li a {
+        display: block;
+        color: black;
+        text-align: center;
+        padding: 11px 16px;
+        text-decoration: none;
+      }
+      .navbar li:hover {
+        background-color: #eaaa6d;
+      }
+      .active {
+        background-color: rgba(234, 170, 109, 0.7);
+      }
+      a {
+        color: #1f5396;
+        text-decoration: none;
+      }
+      a:hover,
+      a:focus {
+        color: #123057;
+        text-decoration: underline;
+      }
+      a:focus {
+        outline: 5px auto -webkit-focus-ring-color;
+        outline-offset: -2px;
+        } .bk-root .bk-toolbar-box .bk-toolbar-right {
+        top: 4px;
+        right: 4px;
+      }
+      .navbar {
+        position: fixed;
+        top: 0;
+        left: 0;
+        right: 0;
+        z-index: 1;
+      }
+      body {
+        padding-top: 3rem;
+        padding-bottom: 1rem;
+        box-sizing: border-box;
+      }
+      .content {
+        width: 100%;
+        height: 100%;
+      }
+    </style>
+    {% block resources %}
+    {% block js_resources %}
+    {{ bokeh_css | indent(8) if bokeh_css }}
+    {% endblock %}
+    {% block css_resources%}
+    {{ bokeh_js | indent(8) if bokeh_js }}
+    {% endblock %}
+    {% endblock %}
+          <style>
          .commit-tease,
          .user-profile-mini-avatar,
          .avatar,
          .vcard-details,
          .signup-prompt-bg {
            display: none !IMPORTANT;
          }
        </style>
         <script>
          document.addEventListener('DOMContentLoaded', function() {
            this.querySelectorAll('a').forEach(anchor => {
              anchor.addEventListener('click', e => {
                e.preventDefault();

                const redact = new URLSearchParams(window.location.search).get('redact');
                const hasExistingParams = anchor.href.includes('?');
                window.location.href = anchor.href + (hasExistingParams ? `&redact=${redact}` : `?redact=${redact}`);
              });
            });
          });
        </script>
 </head>
+  <body>
+    <div class="navbar">
+      <ul>
+        <li>
+          <a href="http://dask.pydata.org/en/latest/" style="padding: 0px 0px">
+            <img src="statics/dask_icon_no_pad.svg"></img>
+          </a>
+        </li>
+        {% for page in pages %}
+        <li{% if page == active_page %} class="active"{% endif %}><a href="{{ page }}">{{ page|title }}</a></li>
+        {% endfor %}
+        <li><a href="info/main/workers.html">Info</a></li>
+      </ul>
+    </div>
+    <div class="content">
+      {% block content %}
+      {% endblock %}
+    </div>
+  </body>
+</html>
diff --git a/distributed/bokeh/templates/simple.html b/distributed/bokeh/templates/simple.html
new file mode 100644
index 00000000000..6f982b44f1c
--- /dev/null
+++ b/distributed/bokeh/templates/simple.html
@@ -0,0 +1,6 @@
+{% extends "base.html" %}
+
+{% block content %}
+{{ plot_div }}
+{{ plot_script }}
+{% endblock %}
diff --git a/distributed/bokeh/templates/status.html b/distributed/bokeh/templates/status.html
new file mode 100644
index 00000000000..face484386f
--- /dev/null
+++ b/distributed/bokeh/templates/status.html
@@ -0,0 +1,79 @@
+{% extends "base.html" %}
+
+{% block content %}
+{% from macros import embed %}
+<style type="text/css" media="screen">
+
+  #status-fluid {
+      display: grid;
+      height: 100%;
+  }
+
+  @media (min-width: 0px) {
+    #status-fluid {
+      grid-template-columns: 1fr 1fr;
+      grid-template-rows: 1fr 3fr 1fr;
+    }
+    #status-history {
+      grid-column: 1;
+      grid-row: 1;
+    }
+    #status-processing {
+      grid-column: 2;
+      grid-row: 1;
+    }
+    #status-tasks {
+      grid-column: 1 / span 2;
+      grid-row: 2;
+    }
+    #status-progress {
+      grid-column: 1 / span 2;
+      grid-row: 3;
+    }
+  }
+
+  @media (min-width: 992px) {
+    #status-fluid {
+      grid-template-columns: 1fr 3fr;
+      grid-template-rows: 1fr 1fr 1fr 1fr 1fr 1fr;
+    }
+    #status-history {
+      grid-column: 1;
+      grid-row: 1 / span 3;
+    }
+    #status-processing {
+      grid-column: 1;
+      grid-row: 4 / span 3;
+    }
+    #status-tasks {
+      grid-column: 2;
+      grid-row: 1 / span 4;
+    }
+    #status-progress {
+      grid-column: 2;
+      grid-row: 5 / span 2;
+    }
+  }
+</style>
+<div id="status-fluid">
+
+  <div id="status-history">
+    {{ embed(roots.nbytes_hist) }}
+  </div>
+
+  <div id="status-processing">
+    {{ embed(roots.processing_hist) }}
+  </div>
+
+  <div id="status-tasks">
+    {{ embed(roots.task_stream) }}
+  </div>
+
+  <div id="status-progress">
+    {{ embed(roots.task_progress) }}
+  </div>
+
+</div>
+{{ plot_script }}
+
+{% endblock %}
diff --git a/distributed/bokeh/tests/test_scheduler_bokeh.py b/distributed/bokeh/tests/test_scheduler_bokeh.py
index ef8599bb6bd..bf0eca90221 100644
--- a/distributed/bokeh/tests/test_scheduler_bokeh.py
+++ b/distributed/bokeh/tests/test_scheduler_bokeh.py
@@ -44,7 +44,8 @@ def test_simple(c, s, a, b):
     http_client = AsyncHTTPClient()
     for suffix in ['system', 'counters', 'workers', 'status', 'tasks',
                    'stealing', 'graph', 'individual-task-stream', 'individual-progress',
-                   'individual-graph', 'individual-load',
+                   'individual-graph', 'individual-nbytes',
+                   'individual-nprocessing',
                    'individual-profile']:
         response = yield http_client.fetch('http://localhost:%d/%s' % (port, suffix))
         assert 'bokeh' in response.body.decode().lower()
diff --git a/distributed/bokeh/worker.py b/distributed/bokeh/worker.py
index 687da327d79..257eab7284c 100644
--- a/distributed/bokeh/worker.py
+++ b/distributed/bokeh/worker.py
@@ -26,12 +26,12 @@
 
 logger = logging.getLogger(__name__)
 
-import jinja2
-
-with open(os.path.join(os.path.dirname(__file__), 'template.html')) as f:
+with open(os.path.join(os.path.dirname(__file__), 'templates', 'base.html')) as f:
     template_source = f.read()
 
-template = jinja2.Template(template_source)
+from jinja2 import Environment, FileSystemLoader
+env = Environment(loader=FileSystemLoader(os.path.join(os.path.dirname(__file__), 'templates')))
+
 template_variables = {'pages': ['main', 'system', 'profile', 'crossfilter']}
 
 
@@ -560,7 +560,7 @@ def main_doc(worker, extra, doc):
                             communicating_ts.root,
                             communicating_stream.root,
                             sizing_mode='scale_width'))
-        doc.template = template
+        doc.template = env.get_template('simple.html')
         doc.template_variables['active_page'] = 'main'
         doc.template_variables.update(extra)
 
@@ -575,7 +575,7 @@ def crossfilter_doc(worker, extra, doc):
         doc.add_periodic_callback(crossfilter.update, 500)
 
         doc.add_root(column(statetable.root, crossfilter.root))
-        doc.template = template
+        doc.template = env.get_template('simple.html')
         doc.template_variables['active_page'] = 'crossfilter'
         doc.template_variables.update(extra)
 
@@ -587,7 +587,7 @@ def systemmonitor_doc(worker, extra, doc):
         doc.add_periodic_callback(sysmon.update, 500)
 
         doc.add_root(sysmon.root)
-        doc.template = template
+        doc.template = env.get_template('simple.html')
         doc.template_variables['active_page'] = 'system'
         doc.template_variables.update(extra)
 
@@ -599,7 +599,7 @@ def counters_doc(server, extra, doc):
         doc.add_periodic_callback(counter.update, 500)
 
         doc.add_root(counter.root)
-        doc.template = template
+        doc.template = env.get_template('simple.html')
         doc.template_variables['active_page'] = 'counters'
         doc.template_variables.update(extra)
 
@@ -611,7 +611,7 @@ def profile_doc(server, extra, doc):
         profile.trigger_update()
 
         doc.add_root(profile.root)
-        doc.template = template
+        doc.template = env.get_template('simple.html')
         doc.template_variables['active_page'] = 'profile'
         doc.template_variables.update(extra)
 
@@ -621,7 +621,7 @@ def profile_server_doc(server, extra, doc):
         doc.title = "Dask: Profile of Event Loop"
         prof = ProfileServer(server, sizing_mode='scale_width', doc=doc)
         doc.add_root(prof.root)
-        doc.template = template
+        doc.template = env.get_template('simple.html')
         # doc.template_variables['active_page'] = ''
         doc.template_variables.update(extra)
 

From 5c023259ee9f91d2ba00f5448c8a73f72f91b250 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 29 Aug 2018 21:08:25 -0400
Subject: [PATCH 0077/1550] Fix deserialization of queues on main ioloop thread
 (#2221)

* Fix deserialization of queues on main ioloop thread

Previously we had a difficult time determining that we were on the IOLoop
thread and should act asynchronously.  This adds a new thread local,
`on_event_loop_thread` to verify this explicitly

Fixes https://github.com/dask/distributed/issues/2220

* clear thread_state.on_event_loop_thread in gen_cluster
---
 distributed/queues.py            |  6 +++---
 distributed/tests/test_queues.py | 16 ++++++++++++++++
 distributed/utils_test.py        |  5 ++++-
 distributed/variable.py          |  2 +-
 distributed/worker.py            |  1 +
 5 files changed, 25 insertions(+), 5 deletions(-)

diff --git a/distributed/queues.py b/distributed/queues.py
index 803985a30be..fda6daaae5c 100644
--- a/distributed/queues.py
+++ b/distributed/queues.py
@@ -9,7 +9,7 @@
 import tornado.queues
 
 from .client import Future, _get_global_client, Client
-from .utils import tokey, sync
+from .utils import tokey, sync, thread_state
 from .worker import get_client
 
 logger = logging.getLogger(__name__)
@@ -165,7 +165,7 @@ class Queue(object):
     def __init__(self, name=None, client=None, maxsize=0):
         self.client = client or _get_global_client()
         self.name = name or 'queue-' + uuid.uuid4().hex
-        if self.client.asynchronous:
+        if self.client.asynchronous or getattr(thread_state, 'on_event_loop_thread', False):
             self._started = self.client.scheduler.queue_create(name=self.name,
                                                                maxsize=maxsize)
         else:
@@ -258,7 +258,7 @@ def __setstate__(self, state):
         name, address = state
         try:
             client = get_client(address)
-            assert client.address == address
+            assert client.scheduler.address == address
         except (AttributeError, AssertionError):
             client = Client(address, set_as_default=False)
         self.__init__(name=name, client=client)
diff --git a/distributed/tests/test_queues.py b/distributed/tests/test_queues.py
index 913434d6909..faa8707cdf0 100644
--- a/distributed/tests/test_queues.py
+++ b/distributed/tests/test_queues.py
@@ -273,3 +273,19 @@ def test_timeout(c, s, a, b):
         yield q.put(2, timeout=0.3)
     stop = time()
     assert 0.1 < stop - start < 2.0
+
+
+@gen_cluster(client=True)
+def test_2220(c, s, a, b):
+    q = Queue()
+
+    def put():
+        q.put(55)
+
+    def get():
+        print(q.get())
+
+    fut = c.submit(put)
+    res = c.submit(get)
+
+    yield [res, fut]
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index 900abab06ed..0898be211b2 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -49,7 +49,8 @@
 from .proctitle import enable_proctitle_on_children
 from .security import Security
 from .utils import (ignoring, log_errors, mp_context, get_ip, get_ipv6,
-                    DequeHandler, reset_logger_locks, sync, iscoroutinefunction)
+                    DequeHandler, reset_logger_locks, sync,
+                    iscoroutinefunction, thread_state)
 from .worker import Worker, TOTAL_MEMORY, _global_workers
 
 try:
@@ -863,6 +864,8 @@ def coro():
                         call_stacks = profile.call_stack(sys._current_frames()[tid])
                         assert False, (thread, call_stacks)
             _cleanup_dangling()
+            with ignoring(AttributeError):
+                del thread_state.on_event_loop_thread
             return result
 
         return test_func
diff --git a/distributed/variable.py b/distributed/variable.py
index b21a047ce8c..5d905358a9e 100644
--- a/distributed/variable.py
+++ b/distributed/variable.py
@@ -207,7 +207,7 @@ def __setstate__(self, state):
         name, address = state
         try:
             client = get_client(address)
-            assert client.address == address
+            assert client.scheduler.address == address
         except (AttributeError, AssertionError):
             client = Client(address, set_as_default=False)
         self.__init__(name=name, client=client)
diff --git a/distributed/worker.py b/distributed/worker.py
index 66535e4517d..45ef87d739c 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -402,6 +402,7 @@ def _start(self, addr_or_port=0):
         assert self.status is None
 
         enable_gc_diagnosis()
+        thread_state.on_event_loop_thread = True
 
         # XXX Factor this out
         if not addr_or_port:

From ab83d8566cbab8b745ae6c62c45300c0927debde Mon Sep 17 00:00:00 2001
From: Guillaume EB <g.eynard.bontemps@gmail.com>
Date: Thu, 30 Aug 2018 09:10:07 -0700
Subject: [PATCH 0078/1550] Add a worker initialization function (#2201)

---
 distributed/client.py            | 31 ++++++++++++++
 distributed/scheduler.py         | 16 ++++++-
 distributed/tests/test_worker.py | 73 ++++++++++++++++++++++++++++++++
 distributed/worker.py            | 14 +++++-
 4 files changed, 131 insertions(+), 3 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 6461065dfe9..079b3994087 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -3425,6 +3425,37 @@ def _get_task_stream(self, start=None, stop=None, count=None, plot=False,
         else:
             raise gen.Return(msgs)
 
+    @gen.coroutine
+    def _register_worker_callbacks(self, setup=None):
+        responses = yield self.scheduler.register_worker_callbacks(setup=dumps(setup))
+        results = {}
+        for key, resp in responses.items():
+            if resp['status'] == 'OK':
+                results[key] = resp['result']
+            elif resp['status'] == 'error':
+                six.reraise(*clean_exception(**resp))
+        raise gen.Return(results)
+
+    def register_worker_callbacks(self, setup=None):
+        """
+        Registers a setup callback function for all current and future workers.
+
+        This registers a new setup function for workers in this cluster. The
+        function will run immediately on all currently connected workers. It
+        will also be run upon connection by any workers that are added in the
+        future. Multiple setup functions can be registered - these will be
+        called in the order they were added.
+
+        If the function takes an input argument named ``dask_worker`` then
+        that variable will be populated with the worker itself.
+
+        Parameters
+        ----------
+        setup : callable(dask_worker: Worker) -> None
+            Function to register and run on all workers
+        """
+        return self.sync(self._register_worker_callbacks, setup=setup)
+
 
 class Executor(Client):
     """ Deprecated: see Client """
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 86c75d1872d..dd72a03ca7c 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -899,6 +899,7 @@ def __init__(
         self.plugins = []
         self.transition_log = deque(maxlen=dask.config.get('distributed.scheduler.transition-log-length'))
         self.log = deque(maxlen=dask.config.get('distributed.scheduler.transition-log-length'))
+        self.worker_setups = []
 
         worker_handlers = {
             'task-finished': self.handle_task_finished,
@@ -956,6 +957,7 @@ def __init__(
             'heartbeat_worker': self.heartbeat_worker,
             'get_task_status': self.get_task_status,
             'get_task_stream': self.get_task_stream,
+            'register_worker_callbacks': self.register_worker_callbacks
         }
 
         self._transitions = {
@@ -1330,7 +1332,8 @@ def add_worker(self, comm=None, address=None, keys=(), ncores=None,
 
             yield comm.write({'status': 'OK',
                               'time': time(),
-                              'heartbeat-interval': heartbeat_interval(len(self.workers))})
+                              'heartbeat-interval': heartbeat_interval(len(self.workers)),
+                              'worker-setups': self.worker_setups})
             yield self.handle_worker(comm=comm, worker=address)
 
     def update_graph(self, client=None, tasks=None, keys=None,
@@ -3035,6 +3038,17 @@ def get_task_stream(self, comm=None, start=None, stop=None, count=None):
         ts = [p for p in self.plugins if isinstance(p, TaskStreamPlugin)][0]
         return ts.collect(start=start, stop=stop, count=count)
 
+    @gen.coroutine
+    def register_worker_callbacks(self, comm, setup=None):
+        """ Registers a setup function, and call it on every worker """
+        if setup is None:
+            raise gen.Return({})
+
+        self.worker_setups.append(setup)
+
+        responses = yield self.broadcast(msg=dict(op='run', function=setup))
+        raise gen.Return(responses)
+
     #####################
     # State Transitions #
     #####################
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index d8a3d5f2371..f85638d3656 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -1203,3 +1203,76 @@ def test_avoid_oversubscription(c, s, *workers):
 def test_custom_metrics(c, s, a, b):
     assert s.workers[a.address].metrics['my_port'] == a.port
     assert s.workers[b.address].metrics['my_port'] == b.port
+
+
+@gen_cluster(client=True)
+def test_register_worker_callbacks(c, s, a, b):
+    #preload function to run
+    def mystartup(dask_worker):
+        dask_worker.init_variable = 1
+
+    def mystartup2():
+        import os
+        os.environ['MY_ENV_VALUE'] = 'WORKER_ENV_VALUE'
+        return "Env set."
+
+    #Check that preload function has been run
+    def test_import(dask_worker):
+        return hasattr(dask_worker, 'init_variable')
+        #       and dask_worker.init_variable == 1
+
+    def test_startup2():
+        import os
+        return os.getenv('MY_ENV_VALUE', None) == 'WORKER_ENV_VALUE'
+
+    # Nothing has been run yet
+    assert len(s.worker_setups) == 0
+    result = yield c.run(test_import)
+    assert list(result.values()) == [False] * 2
+    result = yield c.run(test_startup2)
+    assert list(result.values()) == [False] * 2
+
+    # Start a worker and check that startup is not run
+    worker = Worker(s.address, loop=s.loop)
+    yield worker._start()
+    result = yield c.run(test_import, workers=[worker.address])
+    assert list(result.values()) == [False]
+    yield worker._close()
+
+    # Add a preload function
+    response = yield c.register_worker_callbacks(setup=mystartup)
+    assert len(response) == 2
+    assert len(s.worker_setups) == 1
+
+    # Check it has been ran on existing worker
+    result = yield c.run(test_import)
+    assert list(result.values()) == [True] * 2
+
+    # Start a worker and check it is ran on it
+    worker = Worker(s.address, loop=s.loop)
+    yield worker._start()
+    result = yield c.run(test_import, workers=[worker.address])
+    assert list(result.values()) == [True]
+    yield worker._close()
+
+    # Register another preload function
+    response = yield c.register_worker_callbacks(setup=mystartup2)
+    assert len(response) == 2
+    assert len(s.worker_setups) == 2
+
+    # Check it has been run
+    result = yield c.run(test_startup2)
+    assert list(result.values()) == [True] * 2
+
+    # Start a worker and check it is ran on it
+    worker = Worker(s.address, loop=s.loop)
+    yield worker._start()
+    result = yield c.run(test_import, workers=[worker.address])
+    assert list(result.values()) == [True]
+    result = yield c.run(test_startup2, workers=[worker.address])
+    assert list(result.values()) == [True]
+    yield worker._close()
+
+    # Final exception test
+    with pytest.raises(ZeroDivisionError):
+        yield c.register_worker_callbacks(setup=lambda: 1 / 0)
diff --git a/distributed/worker.py b/distributed/worker.py
index 45ef87d739c..c0ce3259276 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -342,6 +342,15 @@ def _register_with_scheduler(self):
             raise ValueError("Unexpected response from register: %r" %
                              (response,))
         else:
+            # Retrieve eventual init functions and run them
+            for function_bytes in response['worker-setups']:
+                setup_function = pickle.loads(function_bytes)
+                if has_arg(setup_function, 'dask_worker'):
+                    result = setup_function(dask_worker=self)
+                else:
+                    result = setup_function()
+                logger.info('Init function %s ran: output=%s' % (setup_function, result))
+
             logger.info('        Registered to: %26s', self.scheduler.address)
             logger.info('-' * 49)
 
@@ -567,10 +576,11 @@ def executor_submit(self, key, function, args=(), kwargs=None,
         # logger.info("Finish job %d, %s", i, key)
         raise gen.Return(result)
 
-    def run(self, comm, function, args=(), kwargs={}):
+    def run(self, comm, function, args=(), kwargs=None):
+        kwargs = kwargs or {}
         return run(self, comm, function=function, args=args, kwargs=kwargs)
 
-    def run_coroutine(self, comm, function, args=(), kwargs={}, wait=True):
+    def run_coroutine(self, comm, function, args=(), kwargs=None, wait=True):
         return run(self, comm, function=function, args=args, kwargs=kwargs,
                    is_coro=True, wait=wait)
 

From 6f583f440e8a6c1635779d0618fcc60f568bdec4 Mon Sep 17 00:00:00 2001
From: Luke Canavan <canavandl@gmail.com>
Date: Thu, 30 Aug 2018 11:27:00 -0500
Subject: [PATCH 0079/1550] Canavandl/collapse navbar (#2223)

* Move CSS into css file and add collapsing navbar

* Move navbar JS into js file

* Add bokeh theme to make status doc backgrounds clear

* Move active page highlight logic to JS

* Move status css to standalone file

* use self-closing tag

* [skip ci] add css to MANIFEST.in

* add bokeh theme to all plot
---
 MANIFEST.in                                   |   1 +
 distributed/bokeh/scheduler.py                |  30 ++--
 distributed/bokeh/static/css/base.css         | 108 +++++++++++++
 distributed/bokeh/static/css/status.css       |  50 ++++++
 .../dask-logo.svg}                            |   0
 distributed/bokeh/static/images/fa-bars.svg   |   1 +
 distributed/bokeh/templates/base.html         | 151 +++++-------------
 distributed/bokeh/templates/status.html       |  57 +------
 distributed/bokeh/theme.yaml                  |   5 +
 distributed/bokeh/worker.py                   |   9 ++
 10 files changed, 235 insertions(+), 177 deletions(-)
 create mode 100644 distributed/bokeh/static/css/base.css
 create mode 100644 distributed/bokeh/static/css/status.css
 rename distributed/bokeh/static/{dask_icon_no_pad.svg => images/dask-logo.svg} (100%)
 create mode 100644 distributed/bokeh/static/images/fa-bars.svg
 create mode 100644 distributed/bokeh/theme.yaml

diff --git a/MANIFEST.in b/MANIFEST.in
index ac05efbf329..a6c03274f24 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -2,6 +2,7 @@ recursive-include distributed *.py
 recursive-include distributed *.js
 recursive-include distributed *.coffee
 recursive-include distributed *.html
+recursive-include distributed *.css
 recursive-include distributed *.svg
 recursive-include distributed *.yaml
 recursive-include docs *.rst
diff --git a/distributed/bokeh/scheduler.py b/distributed/bokeh/scheduler.py
index dbef4064905..0c34f68d651 100644
--- a/distributed/bokeh/scheduler.py
+++ b/distributed/bokeh/scheduler.py
@@ -18,6 +18,7 @@
 from bokeh.models.widgets import DataTable, TableColumn
 from bokeh.plotting import figure
 from bokeh.palettes import Viridis11
+from bokeh.themes import Theme
 from bokeh.transform import factor_cmap
 from bokeh.io import curdoc
 from toolz import pipe, merge
@@ -54,6 +55,7 @@
 
 template_variables = {'pages': ['status', 'workers', 'tasks', 'system', 'profile', 'graph']}
 
+BOKEH_THEME = Theme(os.path.join(os.path.dirname(__file__), 'theme.yaml'))
 
 nan = float('nan')
 
@@ -1057,8 +1059,8 @@ def systemmonitor_doc(scheduler, extra, doc):
 
         doc.add_root(column(sysmon.root, sizing_mode='scale_width'))
         doc.template = env.get_template('simple.html')
-        doc.template_variables['active_page'] = 'system'
         doc.template_variables.update(extra)
+        doc.theme = BOKEH_THEME
 
 
 def stealing_doc(scheduler, extra, doc):
@@ -1077,8 +1079,8 @@ def stealing_doc(scheduler, extra, doc):
                             sizing_mode='scale_width'))
 
         doc.template = env.get_template('simple.html')
-        doc.template_variables['active_page'] = 'stealing'
         doc.template_variables.update(extra)
+        doc.theme = BOKEH_THEME
 
 
 def events_doc(scheduler, extra, doc):
@@ -1089,8 +1091,8 @@ def events_doc(scheduler, extra, doc):
         doc.title = "Dask: Scheduler Events"
         doc.add_root(column(events.root, sizing_mode='scale_width'))
         doc.template = env.get_template('simple.html')
-        doc.template_variables['active_page'] = 'events'
         doc.template_variables.update(extra)
+        doc.theme = BOKEH_THEME
 
 
 def workers_doc(scheduler, extra, doc):
@@ -1101,8 +1103,8 @@ def workers_doc(scheduler, extra, doc):
         doc.title = "Dask: Workers"
         doc.add_root(table.root)
         doc.template = env.get_template('simple.html')
-        doc.template_variables['active_page'] = 'workers'
         doc.template_variables.update(extra)
+        doc.theme = BOKEH_THEME
 
 
 def tasks_doc(scheduler, extra, doc):
@@ -1114,8 +1116,8 @@ def tasks_doc(scheduler, extra, doc):
         doc.title = "Dask: Task Stream"
         doc.add_root(ts.root)
         doc.template = env.get_template('simple.html')
-        doc.template_variables['active_page'] = 'tasks'
         doc.template_variables.update(extra)
+        doc.theme = BOKEH_THEME
 
 
 def graph_doc(scheduler, extra, doc):
@@ -1127,8 +1129,8 @@ def graph_doc(scheduler, extra, doc):
         doc.add_root(graph.root)
 
         doc.template = env.get_template('simple.html')
-        doc.template_variables['active_page'] = 'graph'
         doc.template_variables.update(extra)
+        doc.theme = BOKEH_THEME
 
 
 def status_doc(scheduler, extra, doc):
@@ -1164,10 +1166,10 @@ def status_doc(scheduler, extra, doc):
         doc.title = "Dask: Status"
         doc.add_root(task_progress.root)
         doc.add_root(task_stream.root)
-
+        doc.theme = BOKEH_THEME
         doc.template = env.get_template('status.html')
-        doc.template_variables['active_page'] = 'status'
         doc.template_variables.update(extra)
+        doc.theme = BOKEH_THEME
 
 
 def individual_task_stream_doc(scheduler, extra, doc):
@@ -1176,6 +1178,7 @@ def individual_task_stream_doc(scheduler, extra, doc):
     task_stream.update()
     doc.add_periodic_callback(task_stream.update, 100)
     doc.add_root(task_stream.root)
+    doc.theme = BOKEH_THEME
 
 
 def individual_nbytes_doc(scheduler, extra, doc):
@@ -1183,6 +1186,7 @@ def individual_nbytes_doc(scheduler, extra, doc):
     current_load.update()
     doc.add_periodic_callback(current_load.update, 100)
     doc.add_root(current_load.nbytes_figure)
+    doc.theme = BOKEH_THEME
 
 
 def individual_nprocessing_doc(scheduler, extra, doc):
@@ -1190,6 +1194,7 @@ def individual_nprocessing_doc(scheduler, extra, doc):
     current_load.update()
     doc.add_periodic_callback(current_load.update, 100)
     doc.add_root(current_load.processing_figure)
+    doc.theme = BOKEH_THEME
 
 
 def individual_progress_doc(scheduler, extra, doc):
@@ -1197,6 +1202,7 @@ def individual_progress_doc(scheduler, extra, doc):
     task_progress.update()
     doc.add_periodic_callback(task_progress.update, 100)
     doc.add_root(task_progress.root)
+    doc.theme = BOKEH_THEME
 
 
 def individual_graph_doc(scheduler, extra, doc):
@@ -1205,6 +1211,7 @@ def individual_graph_doc(scheduler, extra, doc):
         graph.update()
         doc.add_periodic_callback(graph.update, 200)
         doc.add_root(graph.root)
+        doc.theme = BOKEH_THEME
 
 
 def individual_profile_doc(scheduler, extra, doc):
@@ -1212,6 +1219,7 @@ def individual_profile_doc(scheduler, extra, doc):
         prof = ProfileTimePlot(scheduler, sizing_mode='scale_width', doc=doc)
         doc.add_root(prof.root)
         prof.trigger_update()
+        doc.theme = BOKEH_THEME
 
 
 def individual_profile_server_doc(scheduler, extra, doc):
@@ -1219,6 +1227,7 @@ def individual_profile_server_doc(scheduler, extra, doc):
         prof = ProfileServer(scheduler, sizing_mode='scale_width', doc=doc)
         doc.add_root(prof.root)
         prof.trigger_update()
+        doc.theme = BOKEH_THEME
 
 
 def individual_workers_doc(scheduler, extra, doc):
@@ -1227,6 +1236,7 @@ def individual_workers_doc(scheduler, extra, doc):
         table.update()
         doc.add_periodic_callback(table.update, 500)
         doc.add_root(table.root)
+        doc.theme = BOKEH_THEME
 
 
 def profile_doc(scheduler, extra, doc):
@@ -1235,8 +1245,8 @@ def profile_doc(scheduler, extra, doc):
         prof = ProfileTimePlot(scheduler, sizing_mode='scale_width', doc=doc)
         doc.add_root(prof.root)
         doc.template = env.get_template('simple.html')
-        doc.template_variables['active_page'] = 'profile'
         doc.template_variables.update(extra)
+        doc.theme = BOKEH_THEME
 
         prof.trigger_update()
 
@@ -1247,8 +1257,8 @@ def profile_server_doc(scheduler, extra, doc):
         prof = ProfileServer(scheduler, sizing_mode='scale_width', doc=doc)
         doc.add_root(prof.root)
         doc.template = env.get_template('simple.html')
-        # doc.template_variables['active_page'] = 'profile'
         doc.template_variables.update(extra)
+        doc.theme = BOKEH_THEME
 
         prof.trigger_update()
 
diff --git a/distributed/bokeh/static/css/base.css b/distributed/bokeh/static/css/base.css
new file mode 100644
index 00000000000..4731f93973b
--- /dev/null
+++ b/distributed/bokeh/static/css/base.css
@@ -0,0 +1,108 @@
+html {
+  width: 100%;
+  height: 100%;
+  background: #FAFAFA;
+}
+
+body {
+  height: 100%;
+  width: 100%;
+  margin: 0;
+  box-sizing: border-box;
+  font-family: Helvetica, Arial, sans-serif;
+  padding: 0px 10px;
+  padding-top: 3rem;
+  padding-bottom: 1rem;
+}
+
+.navbar {
+  position: fixed;
+  top: 0;
+  left: 0;
+  right: 0;
+  z-index: 1;
+}
+
+.navbar ul {
+  list-style-type: none;
+  margin: 0;
+  padding: 0;
+  overflow: hidden;
+  background-color: #EEE;
+}
+
+.navbar li {
+  float: left;
+  font-size: 17px;
+  transition: .3s background-color;
+}
+
+.navbar li.active {
+  background-color: rgba(234, 170, 109, 0.7);
+}
+
+.navbar li a {
+  display: block;
+  color: black;
+  padding: 11px 16px;
+  text-decoration: none;
+}
+
+.navbar li:hover {
+  background-color: #eaaa6d;
+}
+
+#dask-logo img {
+  height: 28px;
+  padding: 5px 15px;
+}
+
+#dask-logo a {
+  padding: 0px;
+}
+
+#navbar-toggle-icon {
+  float: right;
+}
+
+#navbar-toggle-icon a {
+  display: none;
+}
+
+#navbar-toggle-icon img {
+  height: 22px;
+}
+
+@media screen and (max-width: 650px) {
+  .navbar li:not(#dask-logo):not(#navbar-toggle-icon) a {
+    display: none;
+  }
+  #navbar-toggle-icon a {
+    display: block;
+  }
+}
+
+@media screen and (max-width: 650px) {
+  .navbar.responsive li:not(#navbar-toggle-icon) {
+    float: none;
+  }
+  .navbar.responsive li:not(#dask-logo):not(#navbar-toggle-icon) a {
+    display: block;
+    text-align: left;
+  }
+  .navbar.responsive #navbar-toggle-icon a {
+    position: absolute;
+    right: 0;
+    top: 0;
+  }
+}
+
+.bk-root .bk-toolbar-box .bk-toolbar-right {
+  top: 4px;
+  right: 4px;
+}
+
+.content {
+  width: 100%;
+  height: 100%;
+}
diff --git a/distributed/bokeh/static/css/status.css b/distributed/bokeh/static/css/status.css
new file mode 100644
index 00000000000..9de0d01b353
--- /dev/null
+++ b/distributed/bokeh/static/css/status.css
@@ -0,0 +1,50 @@
+#status-fluid {
+  display: grid;
+  height: 100%;
+}
+
+@media (min-width: 0px) {
+  #status-fluid {
+    grid-template-columns: 1fr 1fr;
+    grid-template-rows: 1fr 3fr 1fr;
+  }
+  #status-history {
+    grid-column: 1;
+    grid-row: 1;
+  }
+  #status-processing {
+    grid-column: 2;
+    grid-row: 1;
+  }
+  #status-tasks {
+    grid-column: 1 / span 2;
+    grid-row: 2;
+  }
+  #status-progress {
+    grid-column: 1 / span 2;
+    grid-row: 3;
+  }
+}
+
+@media (min-width: 992px) {
+  #status-fluid {
+    grid-template-columns: 1fr 3fr;
+    grid-template-rows: 1fr 1fr 1fr 1fr 1fr 1fr;
+  }
+  #status-history {
+    grid-column: 1;
+    grid-row: 1 / span 3;
+  }
+  #status-processing {
+    grid-column: 1;
+    grid-row: 4 / span 3;
+  }
+  #status-tasks {
+    grid-column: 2;
+    grid-row: 1 / span 4;
+  }
+  #status-progress {
+    grid-column: 2;
+    grid-row: 5 / span 2;
+  }
+}
diff --git a/distributed/bokeh/static/dask_icon_no_pad.svg b/distributed/bokeh/static/images/dask-logo.svg
similarity index 100%
rename from distributed/bokeh/static/dask_icon_no_pad.svg
rename to distributed/bokeh/static/images/dask-logo.svg
diff --git a/distributed/bokeh/static/images/fa-bars.svg b/distributed/bokeh/static/images/fa-bars.svg
new file mode 100644
index 00000000000..06e78c1c3a5
--- /dev/null
+++ b/distributed/bokeh/static/images/fa-bars.svg
@@ -0,0 +1 @@
+<svg aria-hidden="true" data-prefix="fas" data-icon="bars" class="svg-inline--fa fa-bars fa-w-14" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path fill="currentColor" d="M16 132h416c8.837 0 16-7.163 16-16V76c0-8.837-7.163-16-16-16H16C7.163 60 0 67.163 0 76v40c0 8.837 7.163 16 16 16zm0 160h416c8.837 0 16-7.163 16-16v-40c0-8.837-7.163-16-16-16H16c-8.837 0-16 7.163-16 16v40c0 8.837 7.163 16 16 16zm0 160h416c8.837 0 16-7.163 16-16v-40c0-8.837-7.163-16-16-16H16c-8.837 0-16 7.163-16 16v40c0 8.837 7.163 16 16 16z"></path></svg>
diff --git a/distributed/bokeh/templates/base.html b/distributed/bokeh/templates/base.html
index ad9ffd152b4..ac8a855d1fd 100644
--- a/distributed/bokeh/templates/base.html
+++ b/distributed/bokeh/templates/base.html
@@ -4,138 +4,61 @@
     <meta charset="utf-8"/>
     <title>Dask Diagnostic UI</title>
     <meta name='viewport' content='width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no' />
-    <style>
-      html {
-        width: 100%;
-        height: 100%;
-        background: #FAFAFA;
-      }
-      body {
-        height: 100%;
-        width: 100%;
-        margin: 0;
-        padding: 0px 10px;
-        border-left:  1px solid #F0F0F0;
-        border-right: 1px solid #F0F0F0;
-        background: #FFF;
-      }
-      @media only screen and (min-width: 90.063em) and (max-width: 120em) {
-        /* min-width 1441px and max-width 1920px */
-        body {
-          margin: 0;
-        }
-      }
-      @media only screen and (min-width: 120.063em) {
-        /* min-width 1921px */
-        body {
-          margin: 0;
-        }
-      }
-      body {
-        font-family: Helvetica, Arial, sans-serif;
-      }
-      .dashboard {
-        clear: both;
-      }
-      .bk-plot-layout:after {
-        clear: both;
-        display: block;
-        height: 0;
-        overflow: hidden;
-        visibility: hidden;
-        content: ".";
-        font-size: 0;
-      }
-      .navbar img {
-        height: 36px;
-        float: right;
-        margin: 3px;
-        padding: 0px 25px;
-      }
-      .navbar ul {
-        list-style-type: none;
-        margin: 0;
-        padding: 0;
-        overflow: hidden;
-        background-color: #EEE;
-      }
-      .navbar li {
-        float: left;
-        font-size: 17px;
-        transition: .3s background-color;
-      }
-      .navbar li a {
-        display: block;
-        color: black;
-        text-align: center;
-        padding: 11px 16px;
-        text-decoration: none;
-      }
-      .navbar li:hover {
-        background-color: #eaaa6d;
-      }
-      .active {
-        background-color: rgba(234, 170, 109, 0.7);
-      }
-      a {
-        color: #1f5396;
-        text-decoration: none;
-      }
-      a:hover,
-      a:focus {
-        color: #123057;
-        text-decoration: underline;
-      }
-      a:focus {
-        outline: 5px auto -webkit-focus-ring-color;
-        outline-offset: -2px;
-        } .bk-root .bk-toolbar-box .bk-toolbar-right {
-        top: 4px;
-        right: 4px;
-      }
-      .navbar {
-        position: fixed;
-        top: 0;
-        left: 0;
-        right: 0;
-        z-index: 1;
-      }
-      body {
-        padding-top: 3rem;
-        padding-bottom: 1rem;
-        box-sizing: border-box;
-      }
-      .content {
-        width: 100%;
-        height: 100%;
-      }
-    </style>
+    <link rel="stylesheet" href="statics/css/base.css"/>
     {% block resources %}
-    {% block js_resources %}
+    {% block css_resources %}
     {{ bokeh_css | indent(8) if bokeh_css }}
     {% endblock %}
-    {% block css_resources%}
+    {% block js_resources%}
     {{ bokeh_js | indent(8) if bokeh_js }}
     {% endblock %}
+    {% block extra_resources %}
+    {% endblock %}
     {% endblock %}
           <style>
          .commit-tease,
          .user-profile-mini-avatar,
          .avatar,
          .vcard-details,
          .signup-prompt-bg {
            display: none !IMPORTANT;
          }
        </style>
         <script>
          document.addEventListener('DOMContentLoaded', function() {
            this.querySelectorAll('a').forEach(anchor => {
              anchor.addEventListener('click', e => {
                e.preventDefault();

                const redact = new URLSearchParams(window.location.search).get('redact');
                const hasExistingParams = anchor.href.includes('?');
                window.location.href = anchor.href + (hasExistingParams ? `&redact=${redact}` : `?redact=${redact}`);
              });
            });
          });
        </script>
 </head>
   <body>
-    <div class="navbar">
+    <div class="navbar" id="myTopnav">
       <ul>
-        <li>
-          <a href="http://dask.pydata.org/en/latest/" style="padding: 0px 0px">
-            <img src="statics/dask_icon_no_pad.svg"></img>
+        <li id="dask-logo">
+          <a href="http://dask.pydata.org/en/latest/">
+            <img src="statics/images/dask-logo.svg"></img>
           </a>
         </li>
         {% for page in pages %}
-        <li{% if page == active_page %} class="active"{% endif %}><a href="{{ page }}">{{ page|title }}</a></li>
+        <li>
+          <a href="/{{ page }}">{{ page|title }}</a>
+        </li>
         {% endfor %}
-        <li><a href="info/main/workers.html">Info</a></li>
+        <li>
+          <a href="info/main/workers.html">Info</a>
+        </li>
+        <li id="navbar-toggle-icon">
+          <a href="javascript:void(0);" onclick="myFunction()">
+            <img src="statics/images/fa-bars.svg"></img>
+          </a>
+        </li>
       </ul>
     </div>
     <div class="content">
       {% block content %}
       {% endblock %}
     </div>
+    <script>
+      /* Add the "active" class to the current navbar li element */
+      var active = document.querySelectorAll("a[href='" + location.pathname + "']");
+      if (active.length > 0) {
+        active[0].closest('li').classList.add('active')
+      }
+
+      /* Toggle between adding and removing the "responsive" class to navbar when the user clicks on the icon */
+      function myFunction() {
+        var x = document.getElementById("myTopnav");
+        if (x.className === "navbar") {
+          x.className += " responsive";
+        } else {
+          x.className = "navbar";
+        }
+      }
+    </script>
   </body>
 </html>
diff --git a/distributed/bokeh/templates/status.html b/distributed/bokeh/templates/status.html
index face484386f..26e867f62c7 100644
--- a/distributed/bokeh/templates/status.html
+++ b/distributed/bokeh/templates/status.html
@@ -1,60 +1,11 @@
 {% extends "base.html" %}
 
+{% block extra_resources %}
+<link rel="stylesheet" href="statics/css/status.css">
+{% endblock %}
+
 {% block content %}
 {% from macros import embed %}
-<style type="text/css" media="screen">
-
-  #status-fluid {
-      display: grid;
-      height: 100%;
-  }
-
-  @media (min-width: 0px) {
-    #status-fluid {
-      grid-template-columns: 1fr 1fr;
-      grid-template-rows: 1fr 3fr 1fr;
-    }
-    #status-history {
-      grid-column: 1;
-      grid-row: 1;
-    }
-    #status-processing {
-      grid-column: 2;
-      grid-row: 1;
-    }
-    #status-tasks {
-      grid-column: 1 / span 2;
-      grid-row: 2;
-    }
-    #status-progress {
-      grid-column: 1 / span 2;
-      grid-row: 3;
-    }
-  }
-
-  @media (min-width: 992px) {
-    #status-fluid {
-      grid-template-columns: 1fr 3fr;
-      grid-template-rows: 1fr 1fr 1fr 1fr 1fr 1fr;
-    }
-    #status-history {
-      grid-column: 1;
-      grid-row: 1 / span 3;
-    }
-    #status-processing {
-      grid-column: 1;
-      grid-row: 4 / span 3;
-    }
-    #status-tasks {
-      grid-column: 2;
-      grid-row: 1 / span 4;
-    }
-    #status-progress {
-      grid-column: 2;
-      grid-row: 5 / span 2;
-    }
-  }
-</style>
 <div id="status-fluid">
 
   <div id="status-history">
diff --git a/distributed/bokeh/theme.yaml b/distributed/bokeh/theme.yaml
new file mode 100644
index 00000000000..5483a49d325
--- /dev/null
+++ b/distributed/bokeh/theme.yaml
@@ -0,0 +1,5 @@
+### contents of theme.yaml
+attrs:
+    Plot:
+        background_fill_color: null
+        border_fill_color: null
diff --git a/distributed/bokeh/worker.py b/distributed/bokeh/worker.py
index 257eab7284c..ee94ca4b1c0 100644
--- a/distributed/bokeh/worker.py
+++ b/distributed/bokeh/worker.py
@@ -13,6 +13,7 @@
 from bokeh.models.widgets import DataTable, TableColumn
 from bokeh.plotting import figure
 from bokeh.palettes import RdBu
+from bokeh.themes import Theme
 from toolz import merge, partition_all
 
 from .components import DashboardComponent, ProfileTimePlot, ProfileServer
@@ -32,6 +33,8 @@
 from jinja2 import Environment, FileSystemLoader
 env = Environment(loader=FileSystemLoader(os.path.join(os.path.dirname(__file__), 'templates')))
 
+BOKEH_THEME = Theme(os.path.join(os.path.dirname(__file__), 'theme.yaml'))
+
 template_variables = {'pages': ['main', 'system', 'profile', 'crossfilter']}
 
 
@@ -563,6 +566,7 @@ def main_doc(worker, extra, doc):
         doc.template = env.get_template('simple.html')
         doc.template_variables['active_page'] = 'main'
         doc.template_variables.update(extra)
+        doc.theme = BOKEH_THEME
 
 
 def crossfilter_doc(worker, extra, doc):
@@ -578,6 +582,7 @@ def crossfilter_doc(worker, extra, doc):
         doc.template = env.get_template('simple.html')
         doc.template_variables['active_page'] = 'crossfilter'
         doc.template_variables.update(extra)
+        doc.theme = BOKEH_THEME
 
 
 def systemmonitor_doc(worker, extra, doc):
@@ -590,6 +595,7 @@ def systemmonitor_doc(worker, extra, doc):
         doc.template = env.get_template('simple.html')
         doc.template_variables['active_page'] = 'system'
         doc.template_variables.update(extra)
+        doc.theme = BOKEH_THEME
 
 
 def counters_doc(server, extra, doc):
@@ -602,6 +608,7 @@ def counters_doc(server, extra, doc):
         doc.template = env.get_template('simple.html')
         doc.template_variables['active_page'] = 'counters'
         doc.template_variables.update(extra)
+        doc.theme = BOKEH_THEME
 
 
 def profile_doc(server, extra, doc):
@@ -614,6 +621,7 @@ def profile_doc(server, extra, doc):
         doc.template = env.get_template('simple.html')
         doc.template_variables['active_page'] = 'profile'
         doc.template_variables.update(extra)
+        doc.theme = BOKEH_THEME
 
 
 def profile_server_doc(server, extra, doc):
@@ -624,6 +632,7 @@ def profile_server_doc(server, extra, doc):
         doc.template = env.get_template('simple.html')
         # doc.template_variables['active_page'] = ''
         doc.template_variables.update(extra)
+        doc.theme = BOKEH_THEME
 
         prof.trigger_update()
 

From 9432f446abd5d09e96b8a799897d7917986dd9c7 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 29 Aug 2018 16:35:06 -0400
Subject: [PATCH 0080/1550] bump version to 1.23.0

---
 docs/source/changelog.rst | 40 ++++++++++++++++++++++++++++++++++++++-
 setup.py                  |  3 ++-
 2 files changed, 41 insertions(+), 2 deletions(-)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index 13e0f537294..421054011ef 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,9 +1,42 @@
 Changelog
 =========
 
-X.XX.X - 2018-XX-XX
+1.23.1 - YYYY-MM-DD
 -------------------
 
+-
+
+1.23.0 - 2018-08-30
+-------------------
+
+-  Add direct_to_workers to Client `Matthew Rocklin`_
+-  Add Scheduler.proxy to workers `Matthew Rocklin`_
+-  Implement Actors `Matthew Rocklin`_
+-  Fix tooltip (#2168) `Loïc Estève`_
+-  Fix scale /  avoid returning coroutines (#2171) `Joe Hamman`_
+-  Clarify dask-worker --nprocs (#2173) `Yu Feng`_
+-  Concatenate all bytes of small messages in TCP comms (#2172) `Matthew Rocklin`_
+-  Add dashboard_link property (#2176) `Jacob Tomlinson`_
+-  Always offload to_frames (#2170) `Matthew Rocklin`_
+-  Warn if desired port is already in use (#2191) (#2199) `Matthew Rocklin`_
+-  Add profile page for event loop thread (#2144) `Matthew Rocklin`_
+-  Use dispatch for dask serialization, also add sklearn, pytorch (#2175) `Matthew Rocklin`_
+-  Handle corner cases with busy signal (#2182) `Matthew Rocklin`_
+-  Check self.dependencies when looking at tasks in memory (#2196) `Matthew Rocklin`_
+-  Add ability to log additional custom metrics from each worker (#2169) `Loïc Estève`_
+-  Fix formatting when port is a tuple (#2204) `Loïc Estève`_
+-  Describe what ZeroMQ is (#2211) `Mike DePalatis`_
+-  Tiny typo fix (#2214) `Anderson Banihirwe`_
+-  Add Python 3.7 to travis.yml (#2203) `Matthew Rocklin`_
+-  Add plot= keyword to get_task_stream (#2198) `Matthew Rocklin`_
+-  Add support for optional versions in Client.get_versions (#2216) `Matthew Rocklin`_
+-  Add routes for solo bokeh figures in dashboard (#2185) `Matthew Rocklin`_
+-  Be resilient to missing dep after busy signal (#2217) `Matthew Rocklin`_
+-  Use CSS Grid to layout status page on the dashboard (#2213) `Derek Ludwig`_ and `Luke Canavan`_
+-  Fix deserialization of queues on main ioloop thread (#2221) `Matthew Rocklin`_
+-  Add a worker initialization function (#2201) `Guillaume EB`_
+-  Collapse navbar in dashboard (#2223) `Luke Canavan`_
+
 
 1.22.1 - 2018-08-03
 -------------------
@@ -727,3 +760,8 @@ significantly without many new features.
 .. _`Phil Tooley`: https://github.com/ptooley
 .. _`Bartosz Marcinkowski`: https://github.com/bm371613
 .. _`Aurélien Ponte`: https://github.com/apatlpo
+.. _`Luke Canavan`: https://github.com/canavandl
+.. _`Derek Ludwig`: https://github.com/dsludwig
+.. _`Anderson Banihirwe`: https://github.com/andersy005
+.. _`Yu Feng`: https://github.com/rainwoodman
+.. _`Guillaume EB`: https://github.com/guillaumeeb
diff --git a/setup.py b/setup.py
index 4d2e6e218a2..3ef26a047dc 100755
--- a/setup.py
+++ b/setup.py
@@ -22,7 +22,7 @@
 setup(name='distributed',
       version=versioneer.get_version(),
       cmdclass=versioneer.get_cmdclass(),
-      description='Distributed computing',
+      description='Distributed scheduler for Dask',
       url='https://distributed.readthedocs.io/en/latest/',
       maintainer='Matthew Rocklin',
       maintainer_email='mrocklin@gmail.com',
@@ -51,6 +51,7 @@
         "Programming Language :: Python :: 2.7",
         "Programming Language :: Python :: 3.5",
         "Programming Language :: Python :: 3.6",
+        "Programming Language :: Python :: 3.7",
         "Topic :: Scientific/Engineering",
         "Topic :: System :: Distributed Computing",
       ],

From c02ea63ca2d0225ef52382e69165343d02f197cd Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 5 Sep 2018 08:20:58 -0400
Subject: [PATCH 0081/1550] Raise informative error when mixing futures between
 clients (#2227)

Previously when a future was used within a different client the client
would just unpack the key and use it directly.  Now we explicitly check
that the future is being used with the correct client, and raise an
informative error if this is not the case.
---
 distributed/actor.py             |  4 ++++
 distributed/client.py            | 20 +++++++++++++-------
 distributed/tests/test_client.py | 12 ++++++++++++
 distributed/utils_comm.py        | 18 ++++++++++--------
 4 files changed, 39 insertions(+), 15 deletions(-)

diff --git a/distributed/actor.py b/distributed/actor.py
index 85f08dd9efd..47bf09ab1bd 100644
--- a/distributed/actor.py
+++ b/distributed/actor.py
@@ -164,6 +164,10 @@ def get_actor_attribute_from_worker():
 
             return self._sync(get_actor_attribute_from_worker)
 
+    @property
+    def client(self):
+        return self._future.client
+
 
 class ProxyRPC(object):
     """
diff --git a/distributed/client.py b/distributed/client.py
index 079b3994087..b094909844d 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -1391,8 +1391,8 @@ def map(self, func, *iterables, **kwargs):
 
     @gen.coroutine
     def _gather(self, futures, errors='raise', direct=None, local_worker=None):
-        futures2, keys = unpack_remotedata(futures, byte_keys=True)
-        keys = [tokey(key) for key in keys]
+        unpacked, future_set = unpack_remotedata(futures, byte_keys=True)
+        keys = [tokey(future.key) for future in future_set]
         bad_data = dict()
         data = {}
 
@@ -1485,11 +1485,11 @@ def wait(k):
             else:
                 break
 
-        if bad_data and errors == 'skip' and isinstance(futures2, list):
-            futures2 = [f for f in futures2 if f not in bad_data]
+        if bad_data and errors == 'skip' and isinstance(unpacked, list):
+            unpacked = [f for f in unpacked if f not in bad_data]
 
         data.update(response['data'])
-        result = pack_data(futures2, merge(data, bad_data))
+        result = pack_data(unpacked, merge(data, bad_data))
         raise gen.Return(result)
 
     def _threaded_gather(self, qin, qout, **kwargs):
@@ -2099,9 +2099,15 @@ def _graph_to_futures(self, dsk, keys, restrictions=None,
                 dsk = dask.optimization.inline(dsk, keys=values)
 
             d = {k: unpack_remotedata(v) for k, v in dsk.items()}
-            extra_keys = set.union(*[v[1] for v in d.values()]) if d else set()
+            extra_futures = set.union(*[v[1] for v in d.values()]) if d else set()
+            extra_keys = {tokey(future.key) for future in extra_futures}
             dsk2 = str_graph({k: v[0] for k, v in d.items()}, extra_keys)
             dsk3 = {k: v for k, v in dsk2.items() if k is not v}
+            for future in extra_futures:
+                if future.client is not self:
+                    msg = ("Inputs contain futures that were created by "
+                           "another client.")
+                    raise ValueError(msg)
 
             if restrictions:
                 restrictions = keymap(tokey, restrictions)
@@ -2110,7 +2116,7 @@ def _graph_to_futures(self, dsk, keys, restrictions=None,
             if loose_restrictions is not None:
                 loose_restrictions = list(map(tokey, loose_restrictions))
 
-            future_dependencies = {tokey(k): set(map(tokey, v[1])) for k, v in d.items()}
+            future_dependencies = {tokey(k): {tokey(f.key) for f in v[1]} for k, v in d.items()}
 
             for s in future_dependencies.values():
                 for v in s:
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 7f09a2efce7..9efdd18495f 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -5476,5 +5476,17 @@ def f():
             assert c.submit(f).result()
 
 
+@gen_cluster()
+def test_mixing_clients(s, a, b):
+    c1 = yield Client(s.address, asynchronous=True)
+    c2 = yield Client(s.address, asynchronous=True)
+
+    future = c1.submit(inc, 1)
+    with pytest.raises(ValueError):
+        c2.submit(inc, future)
+    yield c1.close()
+    yield c2.close()
+
+
 if sys.version_info >= (3, 5):
     from distributed.tests.py3_test_client import *  # noqa F401
diff --git a/distributed/utils_comm.py b/distributed/utils_comm.py
index 7e8702e40a2..5151d802142 100644
--- a/distributed/utils_comm.py
+++ b/distributed/utils_comm.py
@@ -90,10 +90,12 @@ class WrappedKey(object):
     only be accessed in a certain way.  Schedulers may have particular needs
     that can only be addressed by additional metadata.
     """
-
     def __init__(self, key):
         self.key = key
 
+    def __repr__(self):
+        return "%s('%s')" % (type(self).__name__, self.key)
+
 
 _round_robin_counter = [0]
 
@@ -144,7 +146,7 @@ def scatter_to_workers(ncores, data, rpc=rpc, report=True, serializers=None):
 def unpack_remotedata(o, byte_keys=False, myset=None):
     """ Unpack WrappedKey objects from collection
 
-    Returns original collection and set of all found keys
+    Returns original collection and set of all found WrappedKey objects
 
     Examples
     --------
@@ -154,19 +156,19 @@ def unpack_remotedata(o, byte_keys=False, myset=None):
     >>> unpack_remotedata(())
     ((), set())
     >>> unpack_remotedata(rd)
-    ('mykey', {'mykey'})
+    ('mykey', {WrappedKey('mykey')})
     >>> unpack_remotedata([1, rd])
-    ([1, 'mykey'], {'mykey'})
+    ([1, 'mykey'], {WrappedKey('mykey')})
     >>> unpack_remotedata({1: rd})
-    ({1: 'mykey'}, {'mykey'})
+    ({1: 'mykey'}, {WrappedKey('mykey')})
     >>> unpack_remotedata({1: [rd]})
-    ({1: ['mykey']}, {'mykey'})
+    ({1: ['mykey']}, {WrappedKey('mykey')})
 
     Use the ``byte_keys=True`` keyword to force string keys
 
     >>> rd = WrappedKey(('x', 1))
     >>> unpack_remotedata(rd, byte_keys=True)
-    ("('x', 1)", {"('x', 1)"})
+    ("('x', 1)", {WrappedKey('('x', 1)')})
     """
     if myset is None:
         myset = set()
@@ -190,7 +192,7 @@ def unpack_remotedata(o, byte_keys=False, myset=None):
         k = o.key
         if byte_keys:
             k = tokey(k)
-        myset.add(k)
+        myset.add(o)
         return k
     else:
         return o

From de068fd5a2609bdfa50b239c5f370cf21c34fd47 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 5 Sep 2018 11:08:21 -0400
Subject: [PATCH 0082/1550] add byte_keys to unpack_remotedata call (#2232)

---
 distributed/client.py            | 2 +-
 distributed/tests/test_client.py | 8 ++++++++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/distributed/client.py b/distributed/client.py
index b094909844d..91f58d2dcd5 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -2098,7 +2098,7 @@ def _graph_to_futures(self, dsk, keys, restrictions=None,
             if values:
                 dsk = dask.optimization.inline(dsk, keys=values)
 
-            d = {k: unpack_remotedata(v) for k, v in dsk.items()}
+            d = {k: unpack_remotedata(v, byte_keys=True) for k, v in dsk.items()}
             extra_futures = set.union(*[v[1] for v in d.values()]) if d else set()
             extra_keys = {tokey(future.key) for future in extra_futures}
             dsk2 = str_graph({k: v[0] for k, v in d.items()}, extra_keys)
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 9efdd18495f..f7befba1a98 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -5488,5 +5488,13 @@ def test_mixing_clients(s, a, b):
     yield c2.close()
 
 
+@gen_cluster(client=True)
+def test_tuple_keys(c, s, a, b):
+    x = dask.delayed(inc)(1, dask_key_name=('x', 1))
+    y = dask.delayed(inc)(x, dask_key_name=('y', 1))
+    future = c.compute(y)
+    assert (yield future) == 3
+
+
 if sys.version_info >= (3, 5):
     from distributed.tests.py3_test_client import *  # noqa F401

From a966f013668f5870183f77ba4aca4c69df2475cd Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 5 Sep 2018 18:46:21 -0400
Subject: [PATCH 0083/1550] Add documentation for gist/rawgit for
 get_task_stream (#2236)

---
 distributed/client.py | 15 ++++++++++++++-
 docs/source/api.rst   |  7 ++++---
 2 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 91f58d2dcd5..3b19df8bf42 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -3857,11 +3857,24 @@ class get_task_stream(object):
 
     Get back a Bokeh figure and optionally save to a file
 
-    >>> with get_task_stream(plot='save', filename='myfile.html') as ts:
+    >>> with get_task_stream(plot='save', filename='task-stream.html') as ts:
     ...    x.compute()
     >>> ts.figure
     <Bokeh Figure>
 
+    To share this file with others you may wish to upload and serve it online.
+    A common way to do this is to upload the file as a gist, and then serve it
+    on https://rawgit.com ::
+
+       $ pip install gist
+       $ gist task-stream.html
+       https://gist.github.com/8a5b3c74b10b413f612bb5e250856ceb
+
+    You can then navigate to that site, click the "Raw" button to the right of
+    the ``task-stream.html`` file, and then provide that URL to
+    https://rawgit.com .  This process should provide a sharable link that
+    others can use to see your task stream plot.
+
     See Also
     --------
     Client.get_task_stream: Function version of this context manager
diff --git a/docs/source/api.rst b/docs/source/api.rst
index 687b87bebb3..2e8b78bc4b7 100644
--- a/docs/source/api.rst
+++ b/docs/source/api.rst
@@ -3,7 +3,7 @@
 API
 ===
 
-.. currentmodule:: distributed.client
+.. currentmodule:: distributed
 
 **Client**
 
@@ -60,7 +60,7 @@ API
    ReplayExceptionClient.get_futures_error
    ReplayExceptionClient.recreate_error_locally
 
-.. currentmodule:: distributed.client
+.. currentmodule:: distributed
 
 
 **Future**
@@ -131,7 +131,7 @@ The asynchronous variants must be run within a Tornado coroutine.  See the
 Client
 ------
 
-.. currentmodule:: distributed.client
+.. currentmodule:: distributed
 
 .. autoclass:: Client
    :members:
@@ -163,6 +163,7 @@ Other
 .. autofunction:: distributed.get_client
 .. autofunction:: distributed.secede
 .. autofunction:: distributed.rejoin
+.. autoclass:: get_task_stream
 
 .. autoclass:: Lock
    :members:

From e2d12d7135a494a78f4186ca11da594ba84ee741 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 6 Sep 2018 07:17:34 -0400
Subject: [PATCH 0084/1550] Quiet Client.close by waiting for scheduler stop
 signal (#2237)

Fixes https://github.com/dask/distributed/issues/1969
---
 distributed/client.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/distributed/client.py b/distributed/client.py
index 3b19df8bf42..5f58e600d81 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -821,7 +821,8 @@ def _start(self, timeout=no_default, **kwargs):
         for pc in self._periodic_callbacks.values():
             pc.start()
 
-        self.coroutines.append(self._handle_report())
+        self._handle_scheduler_coroutine = self._handle_report()
+        self.coroutines.append(self._handle_scheduler_coroutine)
 
         raise gen.Return(self)
 
@@ -1044,9 +1045,18 @@ def _close(self, fast=False):
                 del dask.config.config['get']
             if self.status == 'closed':
                 raise gen.Return()
+
             if self.scheduler_comm and self.scheduler_comm.comm and not self.scheduler_comm.comm.closed():
                 self._send_to_scheduler({'op': 'close-client'})
                 self._send_to_scheduler({'op': 'close-stream'})
+
+            # Give the scheduler 'stream-closed' message 100ms to come through
+            # This makes the shutdown slightly smoother and quieter
+            with ignoring(AttributeError, gen.TimeoutError):
+                yield gen.with_timeout(timedelta(milliseconds=100),
+                                       self._handle_scheduler_coroutine)
+
+            if self.scheduler_comm and self.scheduler_comm.comm and not self.scheduler_comm.comm.closed():
                 yield self.scheduler_comm.close()
             for key in list(self.futures):
                 self._release_key(key=key)

From 10f2a90d27dc9c920cfd9525b2d1f0c4ac502e3b Mon Sep 17 00:00:00 2001
From: Derek Ludwig <derek.s.ludwig@gmail.com>
Date: Thu, 6 Sep 2018 05:06:31 -0700
Subject: [PATCH 0085/1550] Display system graphs nicely on different screen
 sizes (#2239)

---
 distributed/bokeh/scheduler.py          |  7 ++++---
 distributed/bokeh/static/css/system.css | 26 +++++++++++++++++++++++++
 distributed/bokeh/templates/system.html | 18 +++++++++++++++++
 3 files changed, 48 insertions(+), 3 deletions(-)
 create mode 100644 distributed/bokeh/static/css/system.css
 create mode 100644 distributed/bokeh/templates/system.html

diff --git a/distributed/bokeh/scheduler.py b/distributed/bokeh/scheduler.py
index 0c34f68d651..8a7ffbf5f93 100644
--- a/distributed/bokeh/scheduler.py
+++ b/distributed/bokeh/scheduler.py
@@ -1053,12 +1053,13 @@ def update(self):
 
 def systemmonitor_doc(scheduler, extra, doc):
     with log_errors():
-        sysmon = SystemMonitor(scheduler, sizing_mode='scale_width')
+        sysmon = SystemMonitor(scheduler, sizing_mode='stretch_both')
         doc.title = "Dask: Scheduler System Monitor"
         doc.add_periodic_callback(sysmon.update, 500)
 
-        doc.add_root(column(sysmon.root, sizing_mode='scale_width'))
-        doc.template = env.get_template('simple.html')
+        for subdoc in sysmon.root.children:
+            doc.add_root(subdoc)
+        doc.template = env.get_template('system.html')
         doc.template_variables.update(extra)
         doc.theme = BOKEH_THEME
 
diff --git a/distributed/bokeh/static/css/system.css b/distributed/bokeh/static/css/system.css
new file mode 100644
index 00000000000..988897aacc0
--- /dev/null
+++ b/distributed/bokeh/static/css/system.css
@@ -0,0 +1,26 @@
+#system-fluid {
+    display: flex;
+    flex-wrap: wrap;
+    height: 100%;
+}
+
+/* Small layout: stack all graphs on top of each other, space split equally */
+@media (min-width: 0px) {
+    #system-fluid {
+        flex-direction: column;
+    }
+    #system-fluid .system-item {
+        flex: 1;
+    }
+}
+
+/* Large layout: as many rows as necessary, each item consuming at least
+ * half of the width */
+@media (min-width: 992px) {
+    #system-fluid {
+        flex-direction: row;
+    }
+    #system-fluid .system-item {
+        flex: 1 50%;
+    }
+}
diff --git a/distributed/bokeh/templates/system.html b/distributed/bokeh/templates/system.html
new file mode 100644
index 00000000000..dfa8b898451
--- /dev/null
+++ b/distributed/bokeh/templates/system.html
@@ -0,0 +1,18 @@
+{% extends "base.html" %}
+
+{% block extra_resources %}
+<link rel="stylesheet" href="statics/css/system.css"/>
+{% endblock %}
+
+{% block content %}
+{% from macros import embed %}
+<div id="system-fluid">
+  {% for plot in roots %}
+    <div class="system-item">
+      {{ embed(plot) }}
+    </div>
+  {% endfor %}
+</div>
+{{ plot_script }}
+{% endblock %}
+

From 7b6ec836a262c4c651037cd194370d5fe7b6c445 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 6 Sep 2018 08:07:08 -0400
Subject: [PATCH 0086/1550] Mutate passed in workers dict in
 TaskStreamPlugin.rectangles (#2238)

Fixes https://github.com/dask/distributed/issues/2225
---
 distributed/diagnostics/task_stream.py            | 3 ++-
 distributed/diagnostics/tests/test_task_stream.py | 1 +
 distributed/tests/test_client.py                  | 7 +++++--
 3 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/distributed/diagnostics/task_stream.py b/distributed/diagnostics/task_stream.py
index f3fd169d8d6..fafcbebd5cb 100644
--- a/distributed/diagnostics/task_stream.py
+++ b/distributed/diagnostics/task_stream.py
@@ -83,7 +83,8 @@ def rectangles(self, istart, istop=None, workers=None, start_boundary=0):
 
 
 def rectangles(msgs, workers=None, start_boundary=0):
-    workers = workers or {}
+    if workers is None:
+        workers = {}
 
     L_start = []
     L_duration = []
diff --git a/distributed/diagnostics/tests/test_task_stream.py b/distributed/diagnostics/tests/test_task_stream.py
index f4354b74e6b..f7ab71f68fe 100644
--- a/distributed/diagnostics/tests/test_task_stream.py
+++ b/distributed/diagnostics/tests/test_task_stream.py
@@ -28,6 +28,7 @@ def test_TaskStreamPlugin(c, s, *workers):
     workers = dict()
 
     rects = es.rectangles(0, 10, workers)
+    assert workers
     assert all(n == 'div' for n in rects['name'])
     assert all(d > 0 for d in rects['duration'])
     counts = frequencies(rects['color'])
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index f7befba1a98..d33dcc0f482 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -1121,8 +1121,11 @@ def test_get_releases_data(c, s, a, b):
     [x] = yield c.get({'x': (inc, 1)}, ['x'], sync=False)
     import gc
     gc.collect()
-    yield gen.moment
-    assert c.refcount['x'] == 0
+
+    start = time()
+    while c.refcount['x']:
+        yield gen.sleep(0.01)
+        assert time() < start + 2
 
 
 def test_Current():

From 8bd288a7b7e087202ed74d67bc7a438221d0b2bf Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 6 Sep 2018 08:10:35 -0400
Subject: [PATCH 0087/1550] bump version to 1.23.1

---
 docs/source/actors.rst    |  2 +-
 docs/source/changelog.rst | 15 +++++++++++++--
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/docs/source/actors.rst b/docs/source/actors.rst
index 109b9d907aa..b8bbebc743a 100644
--- a/docs/source/actors.rst
+++ b/docs/source/actors.rst
@@ -175,7 +175,7 @@ the worker with the actor.  It is cached on the ``ActorFuture`` object.
 
 
 Calling from coroutines and async/await
---------------------------
+---------------------------------------
 
 If you use actors within a coroutine or async/await function then actor methods
 and attrbute access will return Tornado futures
diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index 421054011ef..c03d290b245 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,11 +1,22 @@
 Changelog
 =========
 
-1.23.1 - YYYY-MM-DD
+1.23.2 - YYYY-MM-DD
 -------------------
 
 -
 
+1.23.1 - 2018-09-06
+-------------------
+
+-  Raise informative error when mixing futures between clients (#2227) `Matthew Rocklin`_
+-  add byte_keys to unpack_remotedata call (#2232) `Matthew Rocklin`_
+-  Add documentation for gist/rawgit for get_task_stream (#2236) `Matthew Rocklin`_
+-  Quiet Client.close by waiting for scheduler stop signal (#2237) `Matthew Rocklin`_
+-  Display system graphs nicely on different screen sizes (#2239) `Derek Ludwig`_
+-  Mutate passed in workers dict in TaskStreamPlugin.rectangles (#2238) `Matthew Rocklin`_
+
+
 1.23.0 - 2018-08-30
 -------------------
 
@@ -56,7 +67,7 @@ Changelog
 -  Retire workers from scale (:pr:`2104`) `Matthew Rocklin`_
 -  Allow worker to refuse data requests with busy signal (:pr:`2092`) `Matthew Rocklin`_
 -  Don't forget released keys (:pr:`2098`) `Matthew Rocklin`_
--  Update example for stopping a worker (:pr:`2088`) `John A Kirkham`_
+-  Update example for stopping a worker (:pr:`2088`) `John Kirkham`_
 -  removed hardcoded value of memory terminate fraction from a log message (:pr:`2096`) `Bartosz Marcinkowski`_
 -  Adjust worker doc after change in config file location and treatment (:pr:`2094`) `Aurélien Ponte`_
 -  Prefer gathering data from same host (:pr:`2090`) `Matthew Rocklin`_

From 07949bd05ad8b2de37bc6ca7d608542be7fb1386 Mon Sep 17 00:00:00 2001
From: Chris White <white.cdw@gmail.com>
Date: Thu, 6 Sep 2018 08:30:14 -0700
Subject: [PATCH 0088/1550] Return cancelled futures in as_completed (#2233)

as_completed raises exceptions without hanging and optionally returns them
---
 distributed/client.py                  |  34 +++++--
 distributed/tests/py3_test_client.py   |   2 +-
 distributed/tests/test_as_completed.py | 125 +++++++++++++++++++++++--
 3 files changed, 144 insertions(+), 17 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 5f58e600d81..f7d223a80f3 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -3567,6 +3567,18 @@ class as_completed(object):
     Additionally, you can also add more futures to this object during
     computation with the ``.add`` method
 
+    Parameters
+    ----------
+    futures: Collection of futures
+        A list of Future objects to be iterated over in the order in which they
+        complete
+    with_results: bool (False)
+        Whether to wait and include results of futures as well;
+        in this case `as_completed` yields a tuple of (future, result)
+    raise_errors: bool (True)
+        Whether we should raise when the result of a future raises an exception;
+        only affects behavior when `with_results=True`.
+
     Examples
     --------
     >>> x, y, z = client.map(inc, [1, 2, 3])  # doctest: +SKIP
@@ -3602,7 +3614,7 @@ class as_completed(object):
     3
     """
 
-    def __init__(self, futures=None, loop=None, with_results=False):
+    def __init__(self, futures=None, loop=None, with_results=False, raise_errors=True):
         if futures is None:
             futures = []
         self.futures = defaultdict(lambda: 0)
@@ -3612,6 +3624,7 @@ def __init__(self, futures=None, loop=None, with_results=False):
         self.condition = Condition()
         self.thread_condition = threading.Condition()
         self.with_results = with_results
+        self.raise_errors = raise_errors
 
         if futures:
             self.update(futures)
@@ -3626,12 +3639,9 @@ def track_future(self, future):
         try:
             yield _wait(future)
         except CancelledError:
-            del self.futures[future]
-            if not self.futures:
-                self._notify()
-            return
+            pass
         if self.with_results:
-            result = yield future._result()
+            result = yield future._result(raiseit=False)
         with self.lock:
             self.futures[future] -= 1
             if not self.futures[future]:
@@ -3680,13 +3690,21 @@ def __iter__(self):
     def __aiter__(self):
         return self
 
+    def _get_and_raise(self):
+        res = self.queue.get()
+        if self.with_results:
+            future, result = res
+            if self.raise_errors and future.status == 'error':
+                six.reraise(*result)
+        return res
+
     def __next__(self):
         while self.queue.empty():
             if self.is_empty():
                 raise StopIteration()
             with self.thread_condition:
                 self.thread_condition.wait(timeout=0.100)
-        return self.queue.get()
+        return self._get_and_raise()
 
     @gen.coroutine
     def __anext__(self):
@@ -3697,7 +3715,7 @@ def __anext__(self):
                 raise StopAsyncIteration
             yield self.condition.wait()
 
-        raise gen.Return(self.queue.get())
+        raise gen.Return(self._get_and_raise())
 
     next = __next__
 
diff --git a/distributed/tests/py3_test_client.py b/distributed/tests/py3_test_client.py
index 9c7a1fdf3ef..aab55fe3991 100644
--- a/distributed/tests/py3_test_client.py
+++ b/distributed/tests/py3_test_client.py
@@ -79,7 +79,7 @@ async def f():
 
     yield f()
 
-    assert L == [x]
+    assert L == [x, y]
 
 
 def test_async_with(loop):
diff --git a/distributed/tests/test_as_completed.py b/distributed/tests/test_as_completed.py
index d9c2636a178..03e8ed9a630 100644
--- a/distributed/tests/test_as_completed.py
+++ b/distributed/tests/test_as_completed.py
@@ -1,4 +1,5 @@
 from collections import Iterator
+from concurrent.futures._base import CancelledError
 from operator import add
 import random
 from time import sleep
@@ -9,7 +10,7 @@
 from distributed import Client
 from distributed.client import _as_completed, as_completed, _first_completed
 from distributed.compatibility import Empty, StopAsyncIteration, Queue
-from distributed.utils_test import cluster, gen_cluster, inc
+from distributed.utils_test import cluster, gen_cluster, inc, throws
 from distributed.utils_test import loop  # noqa: F401
 
 
@@ -118,26 +119,30 @@ def test_as_completed_is_empty(loop):
 def test_as_completed_cancel(loop):
     with cluster() as (s, [a, b]):
         with Client(s['address'], loop=loop) as c:
-            x = c.submit(sleep, 1)
+            x = c.submit(inc, 1)
             y = c.submit(inc, 1)
 
             ac = as_completed([x, y])
             x.cancel()
 
-            assert next(ac) is y
+            assert next(ac) is x or y
+            assert next(ac) is y or x
 
             with pytest.raises(Empty):
                 ac.queue.get(timeout=0.1)
 
-            assert list(as_completed([x, y, x])) == [y]
+            res = list(as_completed([x, y, x]))
+            assert len(res) == 3
+            assert set(res) == {x, y}
+            assert res.count(x) == 2
 
 
 def test_as_completed_cancel_last(loop):
     with cluster() as (s, [a, b]):
         with Client(s['address'], loop=loop) as c:
-            w = c.submit(sleep, 0.3)
+            w = c.submit(inc, 0.3)
             x = c.submit(inc, 1)
-            y = c.submit(sleep, 0.3)
+            y = c.submit(inc, 0.3)
 
             @gen.coroutine
             def _():
@@ -148,9 +153,9 @@ def _():
             loop.add_callback(_)
 
             ac = as_completed([x, y])
-            result = list(ac)
+            result = set(ac)
 
-            assert result == [x]
+            assert result == {x, y}
 
 
 @gen_cluster(client=True)
@@ -168,3 +173,107 @@ def test_async_for_py2_equivalent(c, s, a, b):
 
     with pytest.raises(StopAsyncIteration):
         yield seq.__anext__()
+
+
+@gen_cluster(client=True)
+def test_as_completed_error_async(c, s, a, b):
+    x = c.submit(throws, 1)
+    y = c.submit(inc, 1)
+
+    ac = as_completed([x, y])
+    first = yield ac.__anext__()
+    second = yield ac.__anext__()
+    result = {first, second}
+
+    assert result == {x, y}
+    assert x.status == 'error'
+    assert y.status == 'finished'
+
+
+def test_as_completed_error(loop):
+    with cluster() as (s, [a, b]):
+        with Client(s['address'], loop=loop) as c:
+            x = c.submit(throws, 1)
+            y = c.submit(inc, 1)
+
+            ac = as_completed([x, y])
+            result = set(ac)
+
+            assert result == {x, y}
+            assert x.status == 'error'
+            assert y.status == 'finished'
+
+
+def test_as_completed_with_results(loop):
+    with cluster() as (s, [a, b]):
+        with Client(s['address'], loop=loop) as c:
+            x = c.submit(throws, 1)
+            y = c.submit(inc, 5)
+            z = c.submit(inc, 1)
+
+            ac = as_completed([x, y, z], with_results=True)
+            y.cancel()
+            with pytest.raises(RuntimeError) as exc:
+                res = list(ac)
+            assert str(exc.value) == 'hello!'
+
+
+@gen_cluster(client=True)
+def test_as_completed_with_results_async(c, s, a, b):
+    x = c.submit(throws, 1)
+    y = c.submit(inc, 5)
+    z = c.submit(inc, 1)
+
+    ac = as_completed([x, y, z], with_results=True)
+    y.cancel()
+    with pytest.raises(RuntimeError) as exc:
+        first = yield ac.__anext__()
+        second = yield ac.__anext__()
+        third = yield ac.__anext__()
+    assert str(exc.value) == 'hello!'
+
+
+def test_as_completed_with_results_no_raise(loop):
+    with cluster() as (s, [a, b]):
+        with Client(s['address'], loop=loop) as c:
+            x = c.submit(throws, 1)
+            y = c.submit(inc, 5)
+            z = c.submit(inc, 1)
+
+            ac = as_completed([x, y, z], with_results=True, raise_errors=False)
+            y.cancel()
+            res = list(ac)
+
+            dd = {r[0]: r[1:] for r in res}
+            assert set(dd.keys()) == {y, x, z}
+            assert x.status == 'error'
+            assert y.status == 'cancelled'
+            assert z.status == 'finished'
+
+            assert isinstance(dd[y][0], CancelledError)
+            assert isinstance(dd[x][0][1], RuntimeError)
+            assert dd[z][0] == 2
+
+
+@gen_cluster(client=True)
+def test_as_completed_with_results_no_raise_async(c, s, a, b):
+    x = c.submit(throws, 1)
+    y = c.submit(inc, 5)
+    z = c.submit(inc, 1)
+
+    ac = as_completed([x, y, z], with_results=True, raise_errors=False)
+    y.cancel()
+    first = yield ac.__anext__()
+    second = yield ac.__anext__()
+    third = yield ac.__anext__()
+    res = [first, second, third]
+
+    dd = {r[0]: r[1:] for r in res}
+    assert set(dd.keys()) == {y, x, z}
+    assert x.status == 'error'
+    assert y.status == 'cancelled'
+    assert z.status == 'finished'
+
+    assert isinstance(dd[y][0], CancelledError)
+    assert isinstance(dd[x][0][1], RuntimeError)
+    assert dd[z][0] == 2

From cdbbc5d606ceb8b957696d5effc52d5091ccbaa4 Mon Sep 17 00:00:00 2001
From: Vladyslav Moisieienkov <vladyslav.moisieienkov@gmail.com>
Date: Fri, 7 Sep 2018 02:22:16 +0200
Subject: [PATCH 0089/1550] docs: Fix broken AWS link in setup.rst file (#2240)

---
 docs/source/setup.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/source/setup.rst b/docs/source/setup.rst
index 8ab08ca4d98..f93f4c03f92 100644
--- a/docs/source/setup.rst
+++ b/docs/source/setup.rst
@@ -200,7 +200,7 @@ Using AWS
 See `Cloud Deployments`_ for the latest information on deploying to Amazon
 cloud.
 
-.. _cloud deployments: _https://dask.pydata.org/en/latest/setup/cloud.html
+.. _`Cloud Deployments`: https://dask.pydata.org/en/latest/setup/cloud.html
 
 
 Using Google Cloud

From 202a94fbfa7d8fe14a89f3ac9084aa979d842548 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 7 Sep 2018 11:09:41 -0400
Subject: [PATCH 0090/1550] Revert change to keep link relative (#2242)

* Revert change to keep link relative

* Avoid GC interruption in get_global_client

https://stackoverflow.com/questions/12428026/safely-iterating-over-weakkeydictionary-and-weakvaluedictionary

* test for absolute links in bokeh pages
---
 distributed/bokeh/templates/base.html                | 2 +-
 distributed/bokeh/tests/test_scheduler_bokeh.py      | 5 ++++-
 distributed/bokeh/tests/test_scheduler_bokeh_html.py | 2 ++
 distributed/client.py                                | 4 +++-
 4 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/distributed/bokeh/templates/base.html b/distributed/bokeh/templates/base.html
index ac8a855d1fd..a3a01ef3314 100644
--- a/distributed/bokeh/templates/base.html
+++ b/distributed/bokeh/templates/base.html
@@ -26,7 +26,7 @@
         </li>
         {% for page in pages %}
         <li>
-          <a href="/{{ page }}">{{ page|title }}</a>
+          <a href="{{ page }}">{{ page|title }}</a>
         </li>
         {% endfor %}
         <li>
diff --git a/distributed/bokeh/tests/test_scheduler_bokeh.py b/distributed/bokeh/tests/test_scheduler_bokeh.py
index bf0eca90221..f8f1fe0a0a7 100644
--- a/distributed/bokeh/tests/test_scheduler_bokeh.py
+++ b/distributed/bokeh/tests/test_scheduler_bokeh.py
@@ -1,6 +1,7 @@
 from __future__ import print_function, division, absolute_import
 
 import json
+import re
 import sys
 from time import sleep
 
@@ -48,7 +49,9 @@ def test_simple(c, s, a, b):
                    'individual-nprocessing',
                    'individual-profile']:
         response = yield http_client.fetch('http://localhost:%d/%s' % (port, suffix))
-        assert 'bokeh' in response.body.decode().lower()
+        body = response.body.decode()
+        assert 'bokeh' in body.lower()
+        assert not re.search("href=./", body)  # no absolute links
 
     response = yield http_client.fetch('http://localhost:%d/individual-plots.json' % port)
     response = json.loads(response.body.decode())
diff --git a/distributed/bokeh/tests/test_scheduler_bokeh_html.py b/distributed/bokeh/tests/test_scheduler_bokeh_html.py
index 52eb65c803f..2ff822f1305 100644
--- a/distributed/bokeh/tests/test_scheduler_bokeh_html.py
+++ b/distributed/bokeh/tests/test_scheduler_bokeh_html.py
@@ -1,6 +1,7 @@
 from __future__ import print_function, division, absolute_import
 
 import json
+import re
 import xml.etree.ElementTree
 
 import pytest
@@ -40,6 +41,7 @@ def test_connect(c, s, a, b):
             json.loads(body)
         else:
             assert xml.etree.ElementTree.fromstring(body) is not None
+            assert not re.search("href=./", body)  # no absolute links
 
 
 @gen_cluster(client=True,
diff --git a/distributed/client.py b/distributed/client.py
index f7d223a80f3..1d73baa6612 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -77,12 +77,14 @@
 
 
 def _get_global_client():
-    for k in sorted(_global_clients, reverse=True):
+    L = sorted(list(_global_clients), reverse=True)
+    for k in L:
         c = _global_clients[k]
         if c.status != 'closed':
             return c
         else:
             del _global_clients[k]
+    del L
     return None
 
 

From 1e597d87351340aedd961e6213941dfb54d9b81f Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Fri, 7 Sep 2018 12:45:03 -0400
Subject: [PATCH 0091/1550] Drop the Bokeh index page (#2241)

As the links on the Bokeh index page are sometimes incorrect, all the
other apps are accessible from each other, and conventional wisdom is to
direct users to the status page app, the index page is not particularly
useful and can be a bit confusing for the reasons cited. So simply
disable it.
---
 distributed/bokeh/core.py | 1 +
 distributed/utils_test.py | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/distributed/bokeh/core.py b/distributed/bokeh/core.py
index 9e8540f0037..deeccadb6cb 100644
--- a/distributed/bokeh/core.py
+++ b/distributed/bokeh/core.py
@@ -32,6 +32,7 @@ def listen(self, addr):
                                      port=port, address=ip,
                                      check_unused_sessions_milliseconds=500,
                                      allow_websocket_origin=["*"],
+                                     use_index=False,
                                      **self.server_kwargs)
                 self.server.start()
 
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index 0898be211b2..b250f808e5d 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -142,7 +142,8 @@ def start():
 
     if PY2:  # no forkserver, so no extra procs
         for child in psutil.Process().children(recursive=True):
-            child.terminate()
+            with ignoring(psutil.NoSuchProcess):
+                child.terminate()
 
     _global_clients.clear()
 

From 2fdb48fafad6a392ed6f247c6e21ac7634e74700 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 7 Sep 2018 13:18:31 -0400
Subject: [PATCH 0092/1550] Use dask_sphinx_theme

---
 docs/requirements.txt |  2 +-
 docs/source/conf.py   | 14 +++-----------
 2 files changed, 4 insertions(+), 12 deletions(-)

diff --git a/docs/requirements.txt b/docs/requirements.txt
index 2255dc97aa0..329b7d7d23b 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -4,4 +4,4 @@ cloudpickle
 dask
 numpydoc
 sphinx
-sphinx_rtd_theme
+dask_sphinx_theme
diff --git a/docs/source/conf.py b/docs/source/conf.py
index 3869ec0367b..35fee88716b 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -118,14 +118,9 @@
 
 # -- Options for HTML output ----------------------------------------------
 
-# Taken from docs.readthedocs.io:
-# on_rtd is whether we are on readthedocs.io
-on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
-
-if not on_rtd:  # only import and set the theme if we're building docs locally
-    import sphinx_rtd_theme
-    html_theme = 'sphinx_rtd_theme'
-    html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
+import dask_sphinx_theme
+html_theme = 'dask_sphinx_theme'
+html_theme_path = [dask_sphinx_theme.get_html_theme_path()]
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
@@ -383,6 +378,3 @@
     'python': ('https://docs.python.org/3', None),
     'numpy': ('http://docs.scipy.org/doc/numpy', None),
     }
-
-def setup(app):
-    app.add_stylesheet("https://dask.pydata.org/en/latest/_static/style.css")

From ae964e70ce2965d9478dcc94abb90a8ff0a0f643 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 14 Sep 2018 14:08:10 -0400
Subject: [PATCH 0093/1550] Discard dependent rather than remove (#2250)

There are some cases where Worker dependencies and dependents can get out of
sync, especially when we lose dependencies but the dependents are still around.

To reflect this we replace a set.remove with set.discard

Fixes https://github.com/dask/distributed/issues/2126
---
 distributed/worker.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/worker.py b/distributed/worker.py
index c0ce3259276..06b38a246c0 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -2121,7 +2121,7 @@ def release_key(self, key, cause=None, reason=None, report=True):
 
             for dep in self.dependencies.pop(key, ()):
                 if dep in self.dependents:
-                    self.dependents[dep].remove(key)
+                    self.dependents[dep].discard(key)
                     if not self.dependents[dep] and self.dep_state[dep] in ('waiting', 'flight'):
                         self.release_dep(dep)
 

From 57aeb70b96396021b2b996a49e6625e0a547609f Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 17 Sep 2018 09:56:32 -0400
Subject: [PATCH 0094/1550] bump version to 1.23.2

---
 docs/source/changelog.rst | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index c03d290b245..34cad1580c6 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,10 +1,16 @@
 Changelog
 =========
 
-1.23.2 - YYYY-MM-DD
+1.23.2 - 2018-09-17
 -------------------
 
--
+-  Discard dependent rather than remove (#2250) `Matthew Rocklin`_
+-  Use dask_sphinx_theme `Matthew Rocklin`_
+-  Drop the Bokeh index page (#2241) `John Kirkham`_
+-  Revert change to keep link relative (#2242) `Matthew Rocklin`_
+-  docs: Fix broken AWS link in setup.rst file (#2240) `Vladyslav Moisieienkov`_
+-  Return cancelled futures in as_completed (#2233) `Chris White`_
+
 
 1.23.1 - 2018-09-06
 -------------------
@@ -776,3 +782,5 @@ significantly without many new features.
 .. _`Anderson Banihirwe`: https://github.com/andersy005
 .. _`Yu Feng`: https://github.com/rainwoodman
 .. _`Guillaume EB`: https://github.com/guillaumeeb
+.. _`Vladyslav Moisieienkov`: https://github.com/VMois
+.. _`Chris White`: https://github.com/cicdw

From ca3edb398e3d9ae5f2032735be3f33ebdbc8a228 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 18 Sep 2018 10:17:55 -0400
Subject: [PATCH 0095/1550] Err in dask serialization if not a
 NotImplementedError (#2251)

Fixes https://github.com/dask/distributed/issues/2245
---
 distributed/protocol/serialize.py            | 11 ++++++++---
 distributed/protocol/tests/test_serialize.py | 16 +++++++++++++++-
 2 files changed, 23 insertions(+), 4 deletions(-)

diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index 3f0da622995..f2a93babbf7 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -58,7 +58,12 @@ def pickle_loads(header, frames):
 
 
 def msgpack_dumps(x):
-    return {'serializer': 'msgpack'}, [msgpack.dumps(x, use_bin_type=True)]
+    try:
+        frame = msgpack.dumps(x, use_bin_type=True)
+    except Exception:
+        raise NotImplementedError()
+    else:
+        return {'serializer': 'msgpack'}, [frame]
 
 
 def msgpack_loads(header, frames):
@@ -135,9 +140,9 @@ def serialize(x, serializers=None, on_error='message', context=None):
             return header, frames
         except NotImplementedError:
             continue
-        except Exception:
+        except Exception as e:
             tb = traceback.format_exc()
-            continue
+            break
 
     msg = "Could not serialize object of type %s" % type(x).__name__
     if on_error == 'message':
diff --git a/distributed/protocol/tests/test_serialize.py b/distributed/protocol/tests/test_serialize.py
index 4e9062cd044..81b4bf8ea0c 100644
--- a/distributed/protocol/tests/test_serialize.py
+++ b/distributed/protocol/tests/test_serialize.py
@@ -13,7 +13,7 @@
                                   deserialize, nested_deserialize, Serialize,
                                   Serialized, to_serialize, serialize_bytes,
                                   deserialize_bytes, serialize_bytelist,
-                                  register_serialization_family)
+                                  register_serialization_family, dask_serialize)
 from distributed.utils import nbytes
 from distributed.utils_test import inc, gen_test
 from distributed.comm.utils import to_frames, from_frames
@@ -332,3 +332,17 @@ def check(dask_worker):
 
     assert z.x == 1 and z.y == 2
     assert z.context['sender'] == b.address
+
+
+def test_serialize_raises():
+    class Foo(object):
+        pass
+
+    @dask_serialize.register(Foo)
+    def dumps(f):
+        raise Exception("Hello-123")
+
+    with pytest.raises(Exception) as info:
+        deserialize(*serialize(Foo()))
+
+    assert 'Hello-123' in str(info.value)

From 9b8e09247cf2f7a3bde75bb147004b14ed32550f Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 19 Sep 2018 14:03:04 -0400
Subject: [PATCH 0096/1550] Protect against key missing from priority in
 GraphLayout (#2259)

Fixes https://github.com/dask/distributed/issues/2193
---
 distributed/diagnostics/graph_layout.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/diagnostics/graph_layout.py b/distributed/diagnostics/graph_layout.py
index 02565f9027a..5c29fe28b83 100644
--- a/distributed/diagnostics/graph_layout.py
+++ b/distributed/diagnostics/graph_layout.py
@@ -39,7 +39,7 @@ def __init__(self, scheduler):
 
     def update_graph(self, scheduler, dependencies=None, priority=None,
                      **kwargs):
-        stack = sorted(dependencies, key=priority.get, reverse=True)
+        stack = sorted(dependencies, key=lambda k: priority.get(k, 0), reverse=True)
         while stack:
             key = stack.pop()
             if key in self.x or key not in scheduler.tasks:

From 10c63bfca7457cac0898db4443fe3d470cdb24ed Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 19 Sep 2018 19:27:41 -0400
Subject: [PATCH 0097/1550] relax test_dont_hold_on_to_large_messages (#2264)

I suspect that this mail fail sometimes because we refresh the reference to x
every 50ms, which may be too short for GC?  Just a guess.
---
 distributed/tests/test_batched.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/tests/test_batched.py b/distributed/tests/test_batched.py
index 1fcf026b68b..49f9e22633b 100644
--- a/distributed/tests/test_batched.py
+++ b/distributed/tests/test_batched.py
@@ -269,7 +269,7 @@ def test_dont_hold_on_to_large_messages(c, s, a, b):
                           f.f_code.co_name, f.f_code.co_filename, sorted(f.f_locals))
             pytest.fail("array should have been destroyed")
 
-        yield gen.sleep(0.05)
+        yield gen.sleep(0.200)
 
 
 @gen_test()

From 6804dea3b2655c467fceb1f2a010d44258af80bb Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 20 Sep 2018 12:43:26 -0400
Subject: [PATCH 0098/1550] Cleanup bokeh callbacks  (#2261)

* Avoid reference cycles in periodic callbacks in Bokeh dashboard

Fixes https://github.com/dask/distributed/issues/2195

* set add_plugin idempotent default to false

* flake8
---
 distributed/bokeh/components.py | 18 ++++++++++++++
 distributed/bokeh/scheduler.py  | 42 +++++++++++++++++----------------
 distributed/bokeh/worker.py     | 19 ++++++++-------
 distributed/scheduler.py        |  2 +-
 4 files changed, 51 insertions(+), 30 deletions(-)

diff --git a/distributed/bokeh/components.py b/distributed/bokeh/components.py
index 5626977483f..8cca28af7ff 100644
--- a/distributed/bokeh/components.py
+++ b/distributed/bokeh/components.py
@@ -672,3 +672,21 @@ def trigger_update(self):
         times = [t * 1000 for t, _ in self.log]
         counts = list(toolz.pluck('count', toolz.pluck(1, self.log)))
         self.ts_source.data.update({'time': times, 'count': counts})
+
+
+def add_periodic_callback(doc, component, interval):
+    """ Add periodic callback to doc in a way that avoids reference cycles
+
+    If we instead use ``doc.add_periodic_callback(component.update, 100)`` then
+    the component stays in memory as a reference cycle because its method is
+    still around.  This way we avoid that and let things clean up a bit more
+    nicely.
+    """
+    ref = weakref.ref(component)
+
+    def update():
+        component = ref()
+        if component is not None:
+            component.update()
+
+    doc.add_periodic_callback(update, interval)
diff --git a/distributed/bokeh/scheduler.py b/distributed/bokeh/scheduler.py
index 8a7ffbf5f93..a39eef848a2 100644
--- a/distributed/bokeh/scheduler.py
+++ b/distributed/bokeh/scheduler.py
@@ -29,7 +29,8 @@
     np = False
 
 from . import components
-from .components import (DashboardComponent, ProfileTimePlot, ProfileServer)
+from .components import (DashboardComponent, ProfileTimePlot, ProfileServer,
+                         add_periodic_callback)
 from .core import BokehServer
 from .worker import SystemMonitor, counters_doc
 from .utils import transpose
@@ -1055,7 +1056,7 @@ def systemmonitor_doc(scheduler, extra, doc):
     with log_errors():
         sysmon = SystemMonitor(scheduler, sizing_mode='stretch_both')
         doc.title = "Dask: Scheduler System Monitor"
-        doc.add_periodic_callback(sysmon.update, 500)
+        add_periodic_callback(doc, sysmon, 500)
 
         for subdoc in sysmon.root.children:
             doc.add_root(subdoc)
@@ -1071,9 +1072,9 @@ def stealing_doc(scheduler, extra, doc):
         stealing_events = StealingEvents(scheduler, sizing_mode='scale_width')
         stealing_events.root.x_range = stealing_ts.root.x_range
         doc.title = "Dask: Work Stealing"
-        doc.add_periodic_callback(occupancy.update, 500)
-        doc.add_periodic_callback(stealing_ts.update, 500)
-        doc.add_periodic_callback(stealing_events.update, 500)
+        add_periodic_callback(doc, occupancy, 500)
+        add_periodic_callback(doc, stealing_ts, 500)
+        add_periodic_callback(doc, stealing_events, 500)
 
         doc.add_root(column(occupancy.root, stealing_ts.root,
                             stealing_events.root,
@@ -1088,7 +1089,7 @@ def events_doc(scheduler, extra, doc):
     with log_errors():
         events = Events(scheduler, 'all', height=250)
         events.update()
-        doc.add_periodic_callback(events.update, 500)
+        add_periodic_callback(doc, events, 500)
         doc.title = "Dask: Scheduler Events"
         doc.add_root(column(events.root, sizing_mode='scale_width'))
         doc.template = env.get_template('simple.html')
@@ -1100,7 +1101,7 @@ def workers_doc(scheduler, extra, doc):
     with log_errors():
         table = WorkerTable(scheduler)
         table.update()
-        doc.add_periodic_callback(table.update, 500)
+        add_periodic_callback(doc, table, 500)
         doc.title = "Dask: Workers"
         doc.add_root(table.root)
         doc.template = env.get_template('simple.html')
@@ -1113,7 +1114,7 @@ def tasks_doc(scheduler, extra, doc):
         ts = TaskStream(scheduler, n_rectangles=100000, clear_interval='60s',
                         sizing_mode='stretch_both')
         ts.update()
-        doc.add_periodic_callback(ts.update, 5000)
+        add_periodic_callback(doc, ts, 5000)
         doc.title = "Dask: Task Stream"
         doc.add_root(ts.root)
         doc.template = env.get_template('simple.html')
@@ -1126,7 +1127,7 @@ def graph_doc(scheduler, extra, doc):
         graph = GraphPlot(scheduler, sizing_mode='stretch_both')
         doc.title = "Dask: Task Graph"
         graph.update()
-        doc.add_periodic_callback(graph.update, 200)
+        add_periodic_callback(doc, graph, 200)
         doc.add_root(graph.root)
 
         doc.template = env.get_template('simple.html')
@@ -1139,16 +1140,16 @@ def status_doc(scheduler, extra, doc):
         task_stream = TaskStream(scheduler, n_rectangles=1000,
                                  clear_interval='10s', sizing_mode='stretch_both')
         task_stream.update()
-        doc.add_periodic_callback(task_stream.update, 100)
+        add_periodic_callback(doc, task_stream, 100)
 
         task_progress = TaskProgress(scheduler, sizing_mode='stretch_both')
         task_progress.update()
-        doc.add_periodic_callback(task_progress.update, 100)
+        add_periodic_callback(doc, task_progress, 100)
 
         if len(scheduler.workers) < 50:
             current_load = CurrentLoad(scheduler, sizing_mode='stretch_both')
             current_load.update()
-            doc.add_periodic_callback(current_load.update, 100)
+            add_periodic_callback(doc, current_load, 100)
             doc.add_root(current_load.nbytes_figure)
             doc.add_root(current_load.processing_figure)
         else:
@@ -1156,8 +1157,8 @@ def status_doc(scheduler, extra, doc):
             nbytes_hist.update()
             processing_hist = ProcessingHistogram(scheduler, sizing_mode='stretch_both')
             processing_hist.update()
-            doc.add_periodic_callback(nbytes_hist.update, 100)
-            doc.add_periodic_callback(processing_hist.update, 100)
+            add_periodic_callback(doc, nbytes_hist, 100)
+            add_periodic_callback(doc, processing_hist, 100)
             current_load_fig = row(nbytes_hist.root, processing_hist.root,
                                    sizing_mode='stretch_both')
 
@@ -1177,7 +1178,7 @@ def individual_task_stream_doc(scheduler, extra, doc):
     task_stream = TaskStream(scheduler, n_rectangles=1000,
                              clear_interval='10s', sizing_mode='stretch_both')
     task_stream.update()
-    doc.add_periodic_callback(task_stream.update, 100)
+    add_periodic_callback(doc, task_stream, 100)
     doc.add_root(task_stream.root)
     doc.theme = BOKEH_THEME
 
@@ -1185,7 +1186,7 @@ def individual_task_stream_doc(scheduler, extra, doc):
 def individual_nbytes_doc(scheduler, extra, doc):
     current_load = CurrentLoad(scheduler, sizing_mode='stretch_both')
     current_load.update()
-    doc.add_periodic_callback(current_load.update, 100)
+    add_periodic_callback(doc, current_load, 100)
     doc.add_root(current_load.nbytes_figure)
     doc.theme = BOKEH_THEME
 
@@ -1193,7 +1194,7 @@ def individual_nbytes_doc(scheduler, extra, doc):
 def individual_nprocessing_doc(scheduler, extra, doc):
     current_load = CurrentLoad(scheduler, sizing_mode='stretch_both')
     current_load.update()
-    doc.add_periodic_callback(current_load.update, 100)
+    add_periodic_callback(doc, current_load, 100)
     doc.add_root(current_load.processing_figure)
     doc.theme = BOKEH_THEME
 
@@ -1201,7 +1202,7 @@ def individual_nprocessing_doc(scheduler, extra, doc):
 def individual_progress_doc(scheduler, extra, doc):
     task_progress = TaskProgress(scheduler, height=160, sizing_mode='stretch_both')
     task_progress.update()
-    doc.add_periodic_callback(task_progress.update, 100)
+    add_periodic_callback(doc, task_progress, 100)
     doc.add_root(task_progress.root)
     doc.theme = BOKEH_THEME
 
@@ -1210,7 +1211,8 @@ def individual_graph_doc(scheduler, extra, doc):
     with log_errors():
         graph = GraphPlot(scheduler, sizing_mode='stretch_both')
         graph.update()
-        doc.add_periodic_callback(graph.update, 200)
+
+        add_periodic_callback(doc, graph, 200)
         doc.add_root(graph.root)
         doc.theme = BOKEH_THEME
 
@@ -1235,7 +1237,7 @@ def individual_workers_doc(scheduler, extra, doc):
     with log_errors():
         table = WorkerTable(scheduler)
         table.update()
-        doc.add_periodic_callback(table.update, 500)
+        add_periodic_callback(doc, table, 500)
         doc.add_root(table.root)
         doc.theme = BOKEH_THEME
 
diff --git a/distributed/bokeh/worker.py b/distributed/bokeh/worker.py
index ee94ca4b1c0..4d7a924a6c8 100644
--- a/distributed/bokeh/worker.py
+++ b/distributed/bokeh/worker.py
@@ -16,7 +16,8 @@
 from bokeh.themes import Theme
 from toolz import merge, partition_all
 
-from .components import DashboardComponent, ProfileTimePlot, ProfileServer
+from .components import (DashboardComponent, ProfileTimePlot, ProfileServer,
+                         add_periodic_callback)
 from .core import BokehServer
 from .utils import transpose
 from ..compatibility import WINDOWS
@@ -554,10 +555,10 @@ def main_doc(worker, extra, doc):
         communicating_stream.root.x_range = xr
 
         doc.title = "Dask Worker Internal Monitor"
-        doc.add_periodic_callback(statetable.update, 200)
-        doc.add_periodic_callback(executing_ts.update, 200)
-        doc.add_periodic_callback(communicating_ts.update, 200)
-        doc.add_periodic_callback(communicating_stream.update, 200)
+        add_periodic_callback(doc, statetable, 200)
+        add_periodic_callback(doc, executing_ts, 200)
+        add_periodic_callback(doc, communicating_ts, 200)
+        add_periodic_callback(doc, communicating_stream, 200)
         doc.add_root(column(statetable.root,
                             executing_ts.root,
                             communicating_ts.root,
@@ -575,8 +576,8 @@ def crossfilter_doc(worker, extra, doc):
         crossfilter = CrossFilter(worker)
 
         doc.title = "Dask Worker Cross-filter"
-        doc.add_periodic_callback(statetable.update, 500)
-        doc.add_periodic_callback(crossfilter.update, 500)
+        add_periodic_callback(doc, statetable, 500)
+        add_periodic_callback(doc, crossfilter, 500)
 
         doc.add_root(column(statetable.root, crossfilter.root))
         doc.template = env.get_template('simple.html')
@@ -589,7 +590,7 @@ def systemmonitor_doc(worker, extra, doc):
     with log_errors():
         sysmon = SystemMonitor(worker, sizing_mode='scale_width')
         doc.title = "Dask Worker Monitor"
-        doc.add_periodic_callback(sysmon.update, 500)
+        add_periodic_callback(doc, sysmon, 500)
 
         doc.add_root(sysmon.root)
         doc.template = env.get_template('simple.html')
@@ -602,7 +603,7 @@ def counters_doc(server, extra, doc):
     with log_errors():
         doc.title = "Dask Worker Counters"
         counter = Counters(server, sizing_mode='stretch_both')
-        doc.add_periodic_callback(counter.update, 500)
+        add_periodic_callback(doc, counter, 500)
 
         doc.add_root(counter.root)
         doc.template = env.get_template('simple.html')
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index dd72a03ca7c..316913db81d 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -2153,7 +2153,7 @@ def handle_worker(self, comm=None, worker=None):
                 worker_comm.abort()
                 self.remove_worker(address=worker)
 
-    def add_plugin(self, plugin=None, idempotent=True, **kwargs):
+    def add_plugin(self, plugin=None, idempotent=False, **kwargs):
         """
         Add external plugin to scheduler
 

From 1e609dcc02e25985b4c2e20384e400bbcf5b675b Mon Sep 17 00:00:00 2001
From: Adam Klein <akleintek@gmail.com>
Date: Thu, 20 Sep 2018 13:34:39 -0400
Subject: [PATCH 0099/1550] Do not pull data twice in Client.gather (#2263)

Fixes #2025
---
 distributed/client.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/client.py b/distributed/client.py
index 1d73baa6612..69693f3a46e 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -1461,7 +1461,7 @@ def wait(k):
                     else:
                         raise ValueError("Bad value, `errors=%s`" % errors)
 
-            keys = [k for k in keys if k not in bad_keys]
+            keys = [k for k in keys if k not in bad_keys and k not in data]
 
             if local_worker:  # look inside local worker
                 data.update({k: local_worker.data[k]

From e74ba24fa623f7ec0823751fcccb378a49816a9a Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 21 Sep 2018 16:05:32 -0400
Subject: [PATCH 0100/1550] Add pytest fixture for cluster tests (#2262)

This replaces the ``with cluster(...)`` style tests for syncrhonous API
with a pytest fixture.  This reduces the nesting and boiler plate for
many of our tests.

The old system is still available for backwards compatibility.

Fixes https://github.com/dask/distributed/issues/1966

* relax heartbeat test
---
 .../diagnostics/tests/test_progressbar.py     |   50 +-
 .../diagnostics/tests/test_task_stream.py     |   43 +-
 distributed/diagnostics/tests/test_widgets.py |  133 +-
 distributed/tests/test_actor.py               |   28 +-
 distributed/tests/test_as_completed.py        |  311 ++---
 distributed/tests/test_client.py              | 1226 ++++++++---------
 distributed/tests/test_client_executor.py     |  453 +++---
 distributed/tests/test_collections.py         |   86 +-
 distributed/tests/test_joblib.py              |  127 +-
 distributed/tests/test_locks.py               |   25 +-
 distributed/tests/test_preload.py             |    1 +
 distributed/tests/test_publish.py             |  146 +-
 distributed/tests/test_queues.py              |   38 +-
 distributed/tests/test_resources.py           |   34 +-
 distributed/tests/test_variable.py            |   68 +-
 distributed/tests/test_worker.py              |   73 +-
 distributed/tests/test_worker_client.py       |   20 +-
 distributed/utils_test.py                     |   40 +-
 docs/source/develop.rst                       |   43 +-
 19 files changed, 1372 insertions(+), 1573 deletions(-)

diff --git a/distributed/diagnostics/tests/test_progressbar.py b/distributed/diagnostics/tests/test_progressbar.py
index 16eeeab0464..7d75b52eeef 100644
--- a/distributed/diagnostics/tests/test_progressbar.py
+++ b/distributed/diagnostics/tests/test_progressbar.py
@@ -4,30 +4,28 @@
 
 from tornado import gen
 
-from distributed import Client, Scheduler, Worker
+from distributed import Scheduler, Worker
 from distributed.diagnostics.progressbar import TextProgressBar, progress
 from distributed.metrics import time
-from distributed.utils_test import (cluster, inc, div, gen_cluster)
-from distributed.utils_test import loop  # noqa: F401
+from distributed.utils_test import (inc, div, gen_cluster)
+from distributed.utils_test import client, loop, cluster_fixture  # noqa: F401
 
 
-def test_text_progressbar(capsys, loop):
-    with cluster(nanny=True) as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            futures = c.map(inc, range(10))
-            p = TextProgressBar(futures, interval=0.01, complete=True)
-            c.gather(futures)
+def test_text_progressbar(capsys, client):
+    futures = client.map(inc, range(10))
+    p = TextProgressBar(futures, interval=0.01, complete=True)
+    client.gather(futures)
 
-            start = time()
-            while p.status != 'finished':
-                sleep(0.01)
-                assert time() - start < 5
+    start = time()
+    while p.status != 'finished':
+        sleep(0.01)
+        assert time() - start < 5
 
-            check_bar_completed(capsys)
-            assert p._last_response == {'all': 10,
-                                        'remaining': 0,
-                                        'status': 'finished'}
-            assert p.comm.closed()
+    check_bar_completed(capsys)
+    assert p._last_response == {'all': 10,
+                                'remaining': 0,
+                                'status': 'finished'}
+    assert p.comm.closed()
 
 
 @gen_cluster(client=True)
@@ -79,14 +77,12 @@ def check_bar_completed(capsys, width=40):
     assert percent == '100% Completed'
 
 
-def test_progress_function(loop, capsys):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            f = c.submit(lambda: 1)
-            g = c.submit(lambda: 2)
+def test_progress_function(client, capsys):
+    f = client.submit(lambda: 1)
+    g = client.submit(lambda: 2)
 
-            progress([[f], [[g]]], notebook=False)
-            check_bar_completed(capsys)
+    progress([[f], [[g]]], notebook=False)
+    check_bar_completed(capsys)
 
-            progress(f)
-            check_bar_completed(capsys)
+    progress(f)
+    check_bar_completed(capsys)
diff --git a/distributed/diagnostics/tests/test_task_stream.py b/distributed/diagnostics/tests/test_task_stream.py
index f7ab71f68fe..51bbc9e1021 100644
--- a/distributed/diagnostics/tests/test_task_stream.py
+++ b/distributed/diagnostics/tests/test_task_stream.py
@@ -6,9 +6,9 @@
 import pytest
 from toolz import frequencies
 
-from distributed import Client, get_task_stream
-from distributed.utils_test import gen_cluster, div, inc, slowinc, cluster
-from distributed.utils_test import loop  # noqa F401
+from distributed import get_task_stream
+from distributed.utils_test import gen_cluster, div, inc, slowinc
+from distributed.utils_test import client, loop, cluster_fixture  # noqa: F401
 from distributed.client import wait
 from distributed.diagnostics.task_stream import TaskStreamPlugin
 from distributed.metrics import time
@@ -95,16 +95,14 @@ def test_client(c, s, a, b):
     assert L == tuple(tasks.buffer)
 
 
-def test_client_sync(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            with get_task_stream(client=c) as ts:
-                sleep(0.1)  # to smooth over time differences on the scheduler
-                # to smooth over time differences on the scheduler
-                futures = c.map(inc, range(10))
-                wait(futures)
+def test_client_sync(client):
+    with get_task_stream(client=client) as ts:
+        sleep(0.1)  # to smooth over time differences on the scheduler
+        # to smooth over time differences on the scheduler
+        futures = client.map(inc, range(10))
+        wait(futures)
 
-            assert len(ts.data) == 10
+    assert len(ts.data) == 10
 
 
 @gen_cluster(client=True)
@@ -119,17 +117,16 @@ def test_get_task_stream_plot(c, s, a, b):
     assert isinstance(figure, bokeh.plotting.Figure)
 
 
-def test_get_task_stream_save(loop, tmpdir):
+def test_get_task_stream_save(client, tmpdir):
     bokeh = pytest.importorskip('bokeh')
     tmpdir = str(tmpdir)
     fn = os.path.join(tmpdir, 'foo.html')
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            with get_task_stream(plot='save', filename=fn) as ts:
-                wait(c.map(inc, range(10)))
-            with open(fn) as f:
-                data = f.read()
-            assert 'inc' in data
-            assert 'bokeh' in data
-
-            assert isinstance(ts.figure, bokeh.plotting.Figure)
+
+    with get_task_stream(plot='save', filename=fn) as ts:
+        wait(client.map(inc, range(10)))
+    with open(fn) as f:
+        data = f.read()
+    assert 'inc' in data
+    assert 'bokeh' in data
+
+    assert isinstance(ts.figure, bokeh.plotting.Figure)
diff --git a/distributed/diagnostics/tests/test_widgets.py b/distributed/diagnostics/tests/test_widgets.py
index e9a31c18d14..6ae29161a9c 100644
--- a/distributed/diagnostics/tests/test_widgets.py
+++ b/distributed/diagnostics/tests/test_widgets.py
@@ -76,12 +76,11 @@ def record_display(*args):
 
 from toolz import valmap
 
-from distributed.client import Client, wait
+from distributed.client import wait
 from distributed.worker import dumps_task
-from distributed.utils_test import (cluster, inc, dec, throws, gen_cluster,
+from distributed.utils_test import (inc, dec, throws, gen_cluster,
         gen_tls_cluster)
-from distributed.utils_test import loop  # noqa: F401
-from distributed.utils import sync
+from distributed.utils_test import client, loop, cluster_fixture  # noqa: F401
 from distributed.diagnostics.progressbar import (ProgressWidget,
                                                  MultiProgressWidget, progress)
 
@@ -162,64 +161,58 @@ def test_multi_progressbar_widget_after_close(s, a, b):
     assert 'x' in p.bars
 
 
-def test_values(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            L = [c.submit(inc, i) for i in range(5)]
-            wait(L)
-            p = MultiProgressWidget(L)
-            sync(loop, p.listen)
-            assert set(p.bars) == {'inc'}
-            assert p.status == 'finished'
-            assert p.comm.closed()
-            assert '5 / 5' in p.bar_texts['inc'].value
-            assert p.bars['inc'].value == 1.0
-
-            x = c.submit(throws, 1)
-            p = MultiProgressWidget([x])
-            sync(loop, p.listen)
-            assert p.status == 'error'
-
-
-def test_progressbar_done(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            L = [c.submit(inc, i) for i in range(5)]
-            wait(L)
-            p = ProgressWidget(L)
-            sync(loop, p.listen)
-            assert p.status == 'finished'
-            assert p.bar.value == 1.0
-            assert p.bar.bar_style == 'success'
-            assert 'Finished' in p.elapsed_time.value
-
-            f = c.submit(throws, L)
-            wait([f])
-
-            p = ProgressWidget([f])
-            sync(loop, p.listen)
-            assert p.status == 'error'
-            assert p.bar.value == 0.0
-            assert p.bar.bar_style == 'danger'
-            assert 'Exception' in p.elapsed_time.value
-
-            try:
-                throws(1)
-            except Exception as e:
-                assert repr(e) in p.elapsed_time.value
-
-
-def test_progressbar_cancel(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            import time
-            L = [c.submit(lambda: time.sleep(0.3), i) for i in range(5)]
-            p = ProgressWidget(L)
-            sync(loop, p.listen)
-            L[-1].cancel()
-            wait(L[:-1])
-            assert p.status == 'error'
-            assert p.bar.value == 0  # no tasks finish before cancel is called
+def test_values(client):
+    L = [client.submit(inc, i) for i in range(5)]
+    wait(L)
+    p = MultiProgressWidget(L)
+    client.sync(p.listen)
+    assert set(p.bars) == {'inc'}
+    assert p.status == 'finished'
+    assert p.comm.closed()
+    assert '5 / 5' in p.bar_texts['inc'].value
+    assert p.bars['inc'].value == 1.0
+
+    x = client.submit(throws, 1)
+    p = MultiProgressWidget([x])
+    client.sync(p.listen)
+    assert p.status == 'error'
+
+
+def test_progressbar_done(client):
+    L = [client.submit(inc, i) for i in range(5)]
+    wait(L)
+    p = ProgressWidget(L)
+    client.sync(p.listen)
+    assert p.status == 'finished'
+    assert p.bar.value == 1.0
+    assert p.bar.bar_style == 'success'
+    assert 'Finished' in p.elapsed_time.value
+
+    f = client.submit(throws, L)
+    wait([f])
+
+    p = ProgressWidget([f])
+    client.sync(p.listen)
+    assert p.status == 'error'
+    assert p.bar.value == 0.0
+    assert p.bar.bar_style == 'danger'
+    assert 'Exception' in p.elapsed_time.value
+
+    try:
+        throws(1)
+    except Exception as e:
+        assert repr(e) in p.elapsed_time.value
+
+
+def test_progressbar_cancel(client):
+    import time
+    L = [client.submit(lambda: time.sleep(0.3), i) for i in range(5)]
+    p = ProgressWidget(L)
+    client.sync(p.listen)
+    L[-1].cancel()
+    wait(L[:-1])
+    assert p.status == 'error'
+    assert p.bar.value == 0  # no tasks finish before cancel is called
 
 
 @gen_cluster()
@@ -245,15 +238,13 @@ def test_multibar_complete(s, a, b):
     assert '2 / 2' in p.bar_texts['y'].value
 
 
-def test_fast(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            L = c.map(inc, range(100))
-            L2 = c.map(dec, L)
-            L3 = c.map(add, L, L2)
-            p = progress(L3, multi=True, complete=True, notebook=True)
-            sync(loop, p.listen)
-            assert set(p._last_response['all']) == {'inc', 'dec', 'add'}
+def test_fast(client):
+    L = client.map(inc, range(100))
+    L2 = client.map(dec, L)
+    L3 = client.map(add, L, L2)
+    p = progress(L3, multi=True, complete=True, notebook=True)
+    client.sync(p.listen)
+    assert set(p._last_response['all']) == {'inc', 'dec', 'add'}
 
 
 @gen_cluster(client=True, client_kwargs={'serializers': ['msgpack']})
diff --git a/distributed/tests/test_actor.py b/distributed/tests/test_actor.py
index 580b884b554..8c4c6544dbe 100644
--- a/distributed/tests/test_actor.py
+++ b/distributed/tests/test_actor.py
@@ -6,8 +6,8 @@
 
 import dask
 from distributed import Actor, ActorFuture, Client, Future, wait, Nanny
-from distributed.utils_test import gen_cluster, cluster
-from distributed.utils_test import loop  # noqa: F401
+from distributed.utils_test import gen_cluster
+from distributed.utils_test import client, cluster_fixture, loop  # noqa: F401
 from distributed.metrics import time
 
 
@@ -237,23 +237,21 @@ def f(a):
     assert {ts.key for ts in s.tasks[counter.key].dependents} == {x.key, y.key}
 
 
-def test_sync(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            counter = c.submit(Counter, actor=True)
-            counter = counter.result()
+def test_sync(client):
+    counter = client.submit(Counter, actor=True)
+    counter = counter.result()
 
-            assert counter.n == 0
+    assert counter.n == 0
 
-            future = counter.increment()
-            n = future.result()
-            assert n == 1
-            assert counter.n == 1
+    future = counter.increment()
+    n = future.result()
+    assert n == 1
+    assert counter.n == 1
 
-            assert future.result() == future.result()
+    assert future.result() == future.result()
 
-            assert 'ActorFuture' in repr(future)
-            assert 'distributed.actor' not in repr(future)
+    assert 'ActorFuture' in repr(future)
+    assert 'distributed.actor' not in repr(future)
 
 
 @gen_cluster(client=True, config={'distributed.comm.timeouts.connect': '1s'})
diff --git a/distributed/tests/test_as_completed.py b/distributed/tests/test_as_completed.py
index 03e8ed9a630..256d962c6cc 100644
--- a/distributed/tests/test_as_completed.py
+++ b/distributed/tests/test_as_completed.py
@@ -7,11 +7,10 @@
 import pytest
 from tornado import gen
 
-from distributed import Client
 from distributed.client import _as_completed, as_completed, _first_completed
 from distributed.compatibility import Empty, StopAsyncIteration, Queue
-from distributed.utils_test import cluster, gen_cluster, inc, throws
-from distributed.utils_test import loop  # noqa: F401
+from distributed.utils_test import gen_cluster, inc, throws
+from distributed.utils_test import client, cluster_fixture, loop  # noqa: F401
 
 
 @gen_cluster(client=True)
@@ -30,132 +29,116 @@ def test__as_completed(c, s, a, b):
     assert result in [x, y, z]
 
 
-def test_as_completed(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            x = c.submit(inc, 1)
-            y = c.submit(inc, 2)
-            z = c.submit(inc, 1)
-
-            seq = as_completed([x, y, z])
-            assert seq.count() == 3
-            assert isinstance(seq, Iterator)
-            assert set(seq) == {x, y, z}
-            assert seq.count() == 0
-
-            assert list(as_completed([])) == []
-
-
-def test_as_completed_with_non_futures(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop):
-            with pytest.raises(TypeError):
-                list(as_completed([1, 2, 3]))
-
-
-def test_as_completed_add(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            total = 0
-            expected = sum(map(inc, range(10)))
-            futures = c.map(inc, range(10))
-            ac = as_completed(futures)
-            for future in ac:
-                result = future.result()
-                total += result
-                if random.random() < 0.5:
-                    future = c.submit(add, future, 10)
-                    ac.add(future)
-                    expected += result + 10
-            assert total == expected
-
-
-def test_as_completed_update(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            total = 0
-            todo = list(range(10))
-            expected = sum(map(inc, todo))
-            ac = as_completed([])
-            while todo or not ac.is_empty():
-                if todo:
-                    work, todo = todo[:4], todo[4:]
-                    ac.update(c.map(inc, work))
-                batch = ac.next_batch(block=True)
-                total += sum(r.result() for r in batch)
-            assert total == expected
-
-
-def test_as_completed_repeats(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            ac = as_completed()
-            x = c.submit(inc, 1)
-            ac.add(x)
-            ac.add(x)
-
-            assert next(ac) is x
-            assert next(ac) is x
-
-            with pytest.raises(StopIteration):
-                next(ac)
-
-            ac.add(x)
-            assert next(ac) is x
-
-
-def test_as_completed_is_empty(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            ac = as_completed()
-            assert ac.is_empty()
-            x = c.submit(inc, 1)
-            ac.add(x)
-            assert not ac.is_empty()
-            assert next(ac) is x
-            assert ac.is_empty()
-
-
-def test_as_completed_cancel(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            x = c.submit(inc, 1)
-            y = c.submit(inc, 1)
-
-            ac = as_completed([x, y])
-            x.cancel()
-
-            assert next(ac) is x or y
-            assert next(ac) is y or x
-
-            with pytest.raises(Empty):
-                ac.queue.get(timeout=0.1)
-
-            res = list(as_completed([x, y, x]))
-            assert len(res) == 3
-            assert set(res) == {x, y}
-            assert res.count(x) == 2
-
-
-def test_as_completed_cancel_last(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            w = c.submit(inc, 0.3)
-            x = c.submit(inc, 1)
-            y = c.submit(inc, 0.3)
-
-            @gen.coroutine
-            def _():
-                yield gen.sleep(0.1)
-                yield w.cancel(asynchronous=True)
-                yield y.cancel(asynchronous=True)
-
-            loop.add_callback(_)
-
-            ac = as_completed([x, y])
-            result = set(ac)
-
-            assert result == {x, y}
+def test_as_completed(client):
+    x = client.submit(inc, 1)
+    y = client.submit(inc, 2)
+    z = client.submit(inc, 1)
+
+    seq = as_completed([x, y, z])
+    assert seq.count() == 3
+    assert isinstance(seq, Iterator)
+    assert set(seq) == {x, y, z}
+    assert seq.count() == 0
+
+    assert list(as_completed([])) == []
+
+
+def test_as_completed_with_non_futures(client):
+    with pytest.raises(TypeError):
+        list(as_completed([1, 2, 3]))
+
+
+def test_as_completed_add(client):
+    total = 0
+    expected = sum(map(inc, range(10)))
+    futures = client.map(inc, range(10))
+    ac = as_completed(futures)
+    for future in ac:
+        result = future.result()
+        total += result
+        if random.random() < 0.5:
+            future = client.submit(add, future, 10)
+            ac.add(future)
+            expected += result + 10
+    assert total == expected
+
+
+def test_as_completed_update(client):
+    total = 0
+    todo = list(range(10))
+    expected = sum(map(inc, todo))
+    ac = as_completed([])
+    while todo or not ac.is_empty():
+        if todo:
+            work, todo = todo[:4], todo[4:]
+            ac.update(client.map(inc, work))
+        batch = ac.next_batch(block=True)
+        total += sum(r.result() for r in batch)
+    assert total == expected
+
+
+def test_as_completed_repeats(client):
+    ac = as_completed()
+    x = client.submit(inc, 1)
+    ac.add(x)
+    ac.add(x)
+
+    assert next(ac) is x
+    assert next(ac) is x
+
+    with pytest.raises(StopIteration):
+        next(ac)
+
+    ac.add(x)
+    assert next(ac) is x
+
+
+def test_as_completed_is_empty(client):
+    ac = as_completed()
+    assert ac.is_empty()
+    x = client.submit(inc, 1)
+    ac.add(x)
+    assert not ac.is_empty()
+    assert next(ac) is x
+    assert ac.is_empty()
+
+
+def test_as_completed_cancel(client):
+    x = client.submit(inc, 1)
+    y = client.submit(inc, 1)
+
+    ac = as_completed([x, y])
+    x.cancel()
+
+    assert next(ac) is x or y
+    assert next(ac) is y or x
+
+    with pytest.raises(Empty):
+        ac.queue.get(timeout=0.1)
+
+    res = list(as_completed([x, y, x]))
+    assert len(res) == 3
+    assert set(res) == {x, y}
+    assert res.count(x) == 2
+
+
+def test_as_completed_cancel_last(client):
+    w = client.submit(inc, 0.3)
+    x = client.submit(inc, 1)
+    y = client.submit(inc, 0.3)
+
+    @gen.coroutine
+    def _():
+        yield gen.sleep(0.1)
+        yield w.cancel(asynchronous=True)
+        yield y.cancel(asynchronous=True)
+
+    client.loop.add_callback(_)
+
+    ac = as_completed([x, y])
+    result = set(ac)
+
+    assert result == {x, y}
 
 
 @gen_cluster(client=True)
@@ -190,32 +173,28 @@ def test_as_completed_error_async(c, s, a, b):
     assert y.status == 'finished'
 
 
-def test_as_completed_error(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            x = c.submit(throws, 1)
-            y = c.submit(inc, 1)
+def test_as_completed_error(client):
+    x = client.submit(throws, 1)
+    y = client.submit(inc, 1)
 
-            ac = as_completed([x, y])
-            result = set(ac)
+    ac = as_completed([x, y])
+    result = set(ac)
 
-            assert result == {x, y}
-            assert x.status == 'error'
-            assert y.status == 'finished'
+    assert result == {x, y}
+    assert x.status == 'error'
+    assert y.status == 'finished'
 
 
-def test_as_completed_with_results(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            x = c.submit(throws, 1)
-            y = c.submit(inc, 5)
-            z = c.submit(inc, 1)
+def test_as_completed_with_results(client):
+    x = client.submit(throws, 1)
+    y = client.submit(inc, 5)
+    z = client.submit(inc, 1)
 
-            ac = as_completed([x, y, z], with_results=True)
-            y.cancel()
-            with pytest.raises(RuntimeError) as exc:
-                res = list(ac)
-            assert str(exc.value) == 'hello!'
+    ac = as_completed([x, y, z], with_results=True)
+    y.cancel()
+    with pytest.raises(RuntimeError) as exc:
+        res = list(ac)
+    assert str(exc.value) == 'hello!'
 
 
 @gen_cluster(client=True)
@@ -233,26 +212,24 @@ def test_as_completed_with_results_async(c, s, a, b):
     assert str(exc.value) == 'hello!'
 
 
-def test_as_completed_with_results_no_raise(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            x = c.submit(throws, 1)
-            y = c.submit(inc, 5)
-            z = c.submit(inc, 1)
+def test_as_completed_with_results_no_raise(client):
+    x = client.submit(throws, 1)
+    y = client.submit(inc, 5)
+    z = client.submit(inc, 1)
 
-            ac = as_completed([x, y, z], with_results=True, raise_errors=False)
-            y.cancel()
-            res = list(ac)
+    ac = as_completed([x, y, z], with_results=True, raise_errors=False)
+    y.cancel()
+    res = list(ac)
 
-            dd = {r[0]: r[1:] for r in res}
-            assert set(dd.keys()) == {y, x, z}
-            assert x.status == 'error'
-            assert y.status == 'cancelled'
-            assert z.status == 'finished'
+    dd = {r[0]: r[1:] for r in res}
+    assert set(dd.keys()) == {y, x, z}
+    assert x.status == 'error'
+    assert y.status == 'cancelled'
+    assert z.status == 'finished'
 
-            assert isinstance(dd[y][0], CancelledError)
-            assert isinstance(dd[x][0][1], RuntimeError)
-            assert dd[z][0] == 2
+    assert isinstance(dd[y][0], CancelledError)
+    assert isinstance(dd[x][0][1], RuntimeError)
+    assert dd[z][0] == 2
 
 
 @gen_cluster(client=True)
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index d33dcc0f482..d2f0a18bdca 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -27,6 +27,7 @@
 
 import dask
 from dask import delayed
+import dask.bag as db
 from distributed import (Worker, Nanny, fire_and_forget, LocalCluster,
                          get_client, secede, get_worker, Executor, profile,
                          TimeoutError)
@@ -47,7 +48,9 @@
                                     gen_cluster, gen_test, double, deep, popen,
                                     captured_logger, varying, map_varying,
                                     wait_for, async_wait_for, pristine_loop)
-from distributed.utils_test import loop, loop_in_thread, nodebug  # noqa F401
+from distributed.utils_test import (client as c, client_secondary as c2,# noqa F401
+                                    cluster_fixture, loop, loop_in_thread,# noqa F401
+                                    nodebug, s, a, b)  # noqa F401
 
 
 @gen_cluster(client=True, timeout=None)
@@ -212,17 +215,15 @@ def test_compute_retries(c, s, a, b):
     assert (yield z) == 80
 
 
-def test_retries_get(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            args = [ZeroDivisionError("one"), ZeroDivisionError("two"), 3]
-            x = delayed(varying(args))()
-            assert x.compute(retries=5) == 3
+def test_retries_get(c):
+    args = [ZeroDivisionError("one"), ZeroDivisionError("two"), 3]
+    x = delayed(varying(args))()
+    assert x.compute(retries=5) == 3
 
-            args = [ZeroDivisionError("one"), ZeroDivisionError("two"), 3]
-            x = delayed(varying(args))()
-            with pytest.raises(ZeroDivisionError):
-                x.compute()
+    args = [ZeroDivisionError("one"), ZeroDivisionError("two"), 3]
+    x = delayed(varying(args))()
+    with pytest.raises(ZeroDivisionError):
+        x.compute()
 
 
 @gen_cluster(client=True)
@@ -322,14 +323,12 @@ def test_Future_exception(c, s, a, b):
     assert result is None
 
 
-def test_Future_exception_sync(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            x = c.submit(div, 1, 0)
-            assert isinstance(x.exception(), ZeroDivisionError)
+def test_Future_exception_sync(c):
+    x = c.submit(div, 1, 0)
+    assert isinstance(x.exception(), ZeroDivisionError)
 
-            x = c.submit(div, 1, 1)
-            assert x.exception() is None
+    x = c.submit(div, 1, 1)
+    assert x.exception() is None
 
 
 @gen_cluster(client=True)
@@ -353,42 +352,38 @@ def test_Future_release(c, s, a, b):
     assert not c.futures
 
 
-def test_Future_release_sync(loop):
+def test_Future_release_sync(c):
     # Released Futures should be removed timely from the Client
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            x = c.submit(div, 1, 1)
-            x.result()
-            x.release()
-            wait_for(lambda: not c.futures, timeout=0.3)
+    x = c.submit(div, 1, 1)
+    x.result()
+    x.release()
+    wait_for(lambda: not c.futures, timeout=0.3)
 
-            x = c.submit(slowinc, 1, delay=0.8)
-            x.release()
-            wait_for(lambda: not c.futures, timeout=0.3)
+    x = c.submit(slowinc, 1, delay=0.8)
+    x.release()
+    wait_for(lambda: not c.futures, timeout=0.3)
 
-            x = c.submit(div, 1, 0)
-            x.exception()
-            x.release()
-            wait_for(lambda: not c.futures, timeout=0.3)
+    x = c.submit(div, 1, 0)
+    x.exception()
+    x.release()
+    wait_for(lambda: not c.futures, timeout=0.3)
 
 
-def test_short_tracebacks(loop):
+def test_short_tracebacks(loop, c):
     tblib = pytest.importorskip('tblib')
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            future = c.submit(div, 1, 0)
-            try:
-                future.result()
-            except Exception:
-                _, _, tb = sys.exc_info()
-            tb = tblib.Traceback(tb).to_dict()
-            n = 0
+    future = c.submit(div, 1, 0)
+    try:
+        future.result()
+    except Exception:
+        _, _, tb = sys.exc_info()
+    tb = tblib.Traceback(tb).to_dict()
+    n = 0
 
-            while tb is not None:
-                n += 1
-                tb = tb['tb_next']
+    while tb is not None:
+        n += 1
+        tb = tb['tb_next']
 
-            assert n < 5
+    assert n < 5
 
 
 @gen_cluster(client=True)
@@ -444,33 +439,29 @@ def test_gc(s, a, b):
     yield c.close()
 
 
-def test_thread(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            x = c.submit(inc, 1)
-            assert x.result() == 2
+def test_thread(c):
+    x = c.submit(inc, 1)
+    assert x.result() == 2
 
-            x = c.submit(slowinc, 1, delay=0.3)
-            with pytest.raises(gen.TimeoutError):
-                x.result(timeout=0.01)
-            assert x.result() == 2
+    x = c.submit(slowinc, 1, delay=0.3)
+    with pytest.raises(gen.TimeoutError):
+        x.result(timeout=0.01)
+    assert x.result() == 2
 
 
-def test_sync_exceptions(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            x = c.submit(div, 10, 2)
-            assert x.result() == 5
+def test_sync_exceptions(c):
+    x = c.submit(div, 10, 2)
+    assert x.result() == 5
 
-            y = c.submit(div, 10, 0)
-            try:
-                y.result()
-                assert False
-            except ZeroDivisionError:
-                pass
+    y = c.submit(div, 10, 0)
+    try:
+        y.result()
+        assert False
+    except ZeroDivisionError:
+        pass
 
-            z = c.submit(div, 10, 5)
-            assert z.result() == 2
+    z = c.submit(div, 10, 5)
+    assert z.result() == 2
 
 
 @gen_cluster(client=True)
@@ -497,19 +488,17 @@ def test_gather_lost(c, s, a, b):
         res = yield c.gather([x, y])
 
 
-def test_gather_sync(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            x = c.submit(inc, 1)
-            assert c.gather(x) == 2
+def test_gather_sync(c):
+    x = c.submit(inc, 1)
+    assert c.gather(x) == 2
 
-            y = c.submit(div, 1, 0)
+    y = c.submit(div, 1, 0)
 
-            with pytest.raises(ZeroDivisionError):
-                c.gather([x, y])
+    with pytest.raises(ZeroDivisionError):
+        c.gather([x, y])
 
-            [xx] = c.gather([x, y], errors='skip')
-            assert xx == 2
+    [xx] = c.gather([x, y], errors='skip')
+    assert xx == 2
 
 
 @gen_cluster(client=True)
@@ -558,35 +547,27 @@ def test_get(c, s, a, b):
     assert result == 3
 
 
-def test_get_sync(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            assert c.get({'x': (inc, 1)}, 'x') == 2
+def test_get_sync(c):
+    assert c.get({'x': (inc, 1)}, 'x') == 2
 
 
-def test_no_future_references(loop):
+def test_no_future_references(c):
     from weakref import WeakSet
     ws = WeakSet()
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            futures = c.map(inc, range(10))
-            ws.update(futures)
-            del futures
-            import gc
-            gc.collect()
-            start = time()
-            while list(ws):
-                sleep(0.01)
-                assert time() < start + 2
+    futures = c.map(inc, range(10))
+    ws.update(futures)
+    del futures
+    import gc
+    gc.collect()
+    start = time()
+    while list(ws):
+        sleep(0.01)
+        assert time() < start + 2
 
 
-def test_get_sync_optimize_graph_passes_through(loop):
-    import dask.bag as db
-    import dask
+def test_get_sync_optimize_graph_passes_through(c):
     bag = db.range(10, npartitions=3).map(inc)
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            dask.compute(bag.sum(), optimize_graph=False)
+    dask.compute(bag.sum(), optimize_graph=False)
 
 
 @gen_cluster(client=True)
@@ -627,33 +608,29 @@ def test_wait_timeout(c, s, a, b):
         yield wait(future, timeout=0.01)
 
 
-def test_wait_sync(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            x = c.submit(inc, 1)
-            y = c.submit(inc, 2)
+def test_wait_sync(c):
+    x = c.submit(inc, 1)
+    y = c.submit(inc, 2)
 
-            done, not_done = wait([x, y])
-            assert done == {x, y}
-            assert not_done == set()
-            assert x.status == y.status == 'finished'
+    done, not_done = wait([x, y])
+    assert done == {x, y}
+    assert not_done == set()
+    assert x.status == y.status == 'finished'
 
-            future = c.submit(sleep, 0.3)
-            with pytest.raises(gen.TimeoutError):
-                wait(future, timeout=0.01)
+    future = c.submit(sleep, 0.3)
+    with pytest.raises(gen.TimeoutError):
+        wait(future, timeout=0.01)
 
 
-def test_wait_informative_error_for_timeouts(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            x = c.submit(inc, 1)
-            y = c.submit(inc, 2)
+def test_wait_informative_error_for_timeouts(c):
+    x = c.submit(inc, 1)
+    y = c.submit(inc, 2)
 
-            try:
-                wait(x, y)
-            except Exception as e:
-                assert "timeout" in str(e)
-                assert "list" in str(e)
+    try:
+        wait(x, y)
+    except Exception as e:
+        assert "timeout" in str(e)
+        assert "list" in str(e)
 
 
 @gen_cluster(client=True)
@@ -1128,14 +1105,13 @@ def test_get_releases_data(c, s, a, b):
         assert time() < start + 2
 
 
-def test_Current():
-    with cluster() as (s, [a, b]):
-        with Client(s['address']) as c:
-            assert Client.current() is c
-        with pytest.raises(ValueError):
-            Client.current()
-        with Client(s['address']) as c:
-            assert Client.current() is c
+def test_Current(s, a, b):
+    with Client(s['address']) as c:
+        assert Client.current() is c
+    with pytest.raises(ValueError):
+        Client.current()
+    with Client(s['address']) as c:
+        assert Client.current() is c
 
 
 def test_global_clients(loop):
@@ -1243,12 +1219,10 @@ def test_get_with_error(c, s, a, b):
         yield c.get(dsk, 'y', sync=False)
 
 
-def test_get_with_error_sync(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            dsk = {'x': (div, 1, 0), 'y': (inc, 'x')}
-            with pytest.raises(ZeroDivisionError):
-                c.get(dsk, 'y')
+def test_get_with_error_sync(c):
+    dsk = {'x': (div, 1, 0), 'y': (inc, 'x')}
+    with pytest.raises(ZeroDivisionError):
+        c.get(dsk, 'y')
 
 
 @gen_cluster(client=True)
@@ -1261,119 +1235,106 @@ def test_directed_scatter(c, s, a, b):
     assert len(b.data) == 2
 
 
-def test_directed_scatter_sync(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            futures = c.scatter([1, 2, 3], workers=[b['address']])
-            has_what = sync(loop, c.scheduler.has_what)
-            assert len(has_what[b['address']]) == len(futures)
-            assert len(has_what[a['address']]) == 0
+def test_directed_scatter_sync(c, s, a, b, loop):
+    futures = c.scatter([1, 2, 3], workers=[b['address']])
+    has_what = sync(loop, c.scheduler.has_what)
+    assert len(has_what[b['address']]) == len(futures)
+    assert len(has_what[a['address']]) == 0
 
 
-def test_iterator_scatter(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            aa = c.scatter([1, 2, 3])
-            assert [1, 2, 3] == c.gather(aa)
+def test_iterator_scatter(c):
+    aa = c.scatter([1, 2, 3])
+    assert [1, 2, 3] == c.gather(aa)
 
-            g = (i for i in range(10))
-            futures = c.scatter(g)
-            assert isinstance(futures, Iterator)
+    g = (i for i in range(10))
+    futures = c.scatter(g)
+    assert isinstance(futures, Iterator)
 
-            a = next(futures)
-            assert c.gather(a) == 0
+    a = next(futures)
+    assert c.gather(a) == 0
 
-            futures = list(futures)
-            assert len(futures) == 9
-            assert c.gather(futures) == [1, 2, 3, 4, 5, 6, 7, 8, 9]
+    futures = list(futures)
+    assert len(futures) == 9
+    assert c.gather(futures) == [1, 2, 3, 4, 5, 6, 7, 8, 9]
 
 
-def test_queue_scatter(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as ee:
-            from distributed.compatibility import Queue
-            q = Queue()
-            for d in range(10):
-                q.put(d)
+def test_queue_scatter(c):
+    from distributed.compatibility import Queue
+    q = Queue()
+    for d in range(10):
+        q.put(d)
 
-            futures = ee.scatter(q)
-            assert isinstance(futures, Queue)
-            a = futures.get()
-            assert ee.gather(a) == 0
+    futures = c.scatter(q)
+    assert isinstance(futures, Queue)
+    a = futures.get()
+    assert c.gather(a) == 0
 
 
-def test_queue_scatter_gather_maxsize(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            from distributed.compatibility import Queue
-            q = Queue(maxsize=3)
-            out = c.scatter(q, maxsize=10)
-            assert out.maxsize == 10
-            local = c.gather(q)
-            assert not local.maxsize
+def test_queue_scatter_gather_maxsize(c):
+    from distributed.compatibility import Queue
+    q = Queue(maxsize=3)
+    out = c.scatter(q, maxsize=10)
+    assert out.maxsize == 10
+    local = c.gather(q)
+    assert not local.maxsize
 
-            q = Queue()
-            out = c.scatter(q)
-            assert not out.maxsize
-            local = c.gather(out, maxsize=10)
-            assert local.maxsize == 10
+    q = Queue()
+    out = c.scatter(q)
+    assert not out.maxsize
+    local = c.gather(out, maxsize=10)
+    assert local.maxsize == 10
 
-            q = Queue(maxsize=3)
-            out = c.scatter(q)
-            assert not out.maxsize
+    q = Queue(maxsize=3)
+    out = c.scatter(q)
+    assert not out.maxsize
 
 
-def test_queue_gather(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as ee:
-            from distributed.compatibility import Queue
-            q = Queue()
+def test_queue_gather(c):
+    from distributed.compatibility import Queue
+    q = Queue()
 
-            qin = list(range(10))
-            for d in qin:
-                q.put(d)
+    qin = list(range(10))
+    for d in qin:
+        q.put(d)
 
-            futures = ee.scatter(q)
-            assert isinstance(futures, Queue)
+    futures = c.scatter(q)
+    assert isinstance(futures, Queue)
 
-            ff = ee.gather(futures)
-            assert isinstance(ff, Queue)
+    ff = c.gather(futures)
+    assert isinstance(ff, Queue)
 
-            qout = []
-            for f in range(10):
-                qout.append(ff.get())
-            assert qout == qin
+    qout = []
+    for f in range(10):
+        qout.append(ff.get())
+    assert qout == qin
 
 
 @pytest.mark.skip(reason="intermittent blocking failures")
-def test_iterator_gather(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as ee:
-
-            i_in = list(range(10))
+def test_iterator_gather(c, c2):
+    i_in = list(range(10))
 
-            g = (d for d in i_in)
-            futures = ee.scatter(g)
-            assert isinstance(futures, Iterator)
+    g = (d for d in i_in)
+    futures = c.scatter(g)
+    assert isinstance(futures, Iterator)
 
-            ff = ee.gather(futures)
-            assert isinstance(ff, Iterator)
+    ff = c.gather(futures)
+    assert isinstance(ff, Iterator)
 
-            i_out = list(ff)
-            assert i_out == i_in
+    i_out = list(ff)
+    assert i_out == i_in
 
-            i_in = ['a', 'b', 'c', StopIteration('f'), StopIteration, 'd', 'c']
+    i_in = ['a', 'b', 'c', StopIteration('f'), StopIteration, 'd', 'c']
 
-            g = (d for d in i_in)
-            futures = ee.scatter(g)
+    g = (d for d in i_in)
+    futures = c.scatter(g)
 
-            ff = ee.gather(futures)
-            i_out = list(ff)
-            assert i_out[:3] == i_in[:3]
-            # This is because StopIteration('f') != StopIteration('f')
-            assert isinstance(i_out[3], StopIteration)
-            assert i_out[3].args == i_in[3].args
-            assert i_out[4:] == i_in[4:]
+    ff = c.gather(futures)
+    i_out = list(ff)
+    assert i_out[:3] == i_in[:3]
+    # This is because StopIteration('f') != StopIteration('f')
+    assert isinstance(i_out[3], StopIteration)
+    assert i_out[3].args == i_in[3].args
+    assert i_out[4:] == i_in[4:]
 
 
 @gen_cluster(client=True)
@@ -1453,14 +1414,12 @@ def test_scatter_direct_spread_evenly(c, s, *workers):
 
 @pytest.mark.parametrize('direct', [True, False])
 @pytest.mark.parametrize('broadcast', [True, False])
-def test_scatter_gather_sync(loop, direct, broadcast):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            futures = c.scatter([1, 2, 3], direct=direct, broadcast=broadcast)
-            results = c.gather(futures, direct=direct)
-            assert results == [1, 2, 3]
+def test_scatter_gather_sync(c, direct, broadcast):
+    futures = c.scatter([1, 2, 3], direct=direct, broadcast=broadcast)
+    results = c.gather(futures, direct=direct)
+    assert results == [1, 2, 3]
 
-            delayed(inc)(1).compute(direct=direct)
+    delayed(inc)(1).compute(direct=direct)
 
 
 @gen_cluster(client=True)
@@ -1510,25 +1469,23 @@ def test_gather_traceback(c, s, a, b):
         assert any('x / y' in line for line in L)
 
 
-def test_traceback_sync(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            x = c.submit(div, 1, 0)
-            tb = x.traceback()
-            if sys.version_info[0] >= 3:
-                assert any('x / y' in line
-                           for line in concat(traceback.extract_tb(tb))
-                           if isinstance(line, str))
+def test_traceback_sync(c):
+    x = c.submit(div, 1, 0)
+    tb = x.traceback()
+    if sys.version_info[0] >= 3:
+        assert any('x / y' in line
+                   for line in concat(traceback.extract_tb(tb))
+                   if isinstance(line, str))
 
-            y = c.submit(inc, x)
-            tb2 = y.traceback()
+    y = c.submit(inc, x)
+    tb2 = y.traceback()
 
-            assert set(pluck(3, traceback.extract_tb(tb2))).issuperset(
-                set(pluck(3, traceback.extract_tb(tb))))
+    assert set(pluck(3, traceback.extract_tb(tb2))).issuperset(
+        set(pluck(3, traceback.extract_tb(tb))))
 
-            z = c.submit(div, 1, 2)
-            tb = z.traceback()
-            assert tb is None
+    z = c.submit(div, 1, 2)
+    tb = z.traceback()
+    assert tb is None
 
 
 @gen_cluster(client=True)
@@ -1597,17 +1554,15 @@ def test_upload_large_file(c, s, a, b):
                     assert f.read() == 'def'
 
 
-def test_upload_file_sync(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            def g():
-                import myfile
-                return myfile.x
+def test_upload_file_sync(c):
+    def g():
+        import myfile
+        return myfile.x
 
-            with tmp_text('myfile.py', 'x = 123') as fn:
-                c.upload_file(fn)
-                x = c.submit(g)
-                assert x.result() == 123
+    with tmp_text('myfile.py', 'x = 123') as fn:
+        c.upload_file(fn)
+        x = c.submit(g)
+        assert x.result() == 123
 
 
 @gen_cluster(client=True)
@@ -1617,12 +1572,10 @@ def test_upload_file_exception(c, s, a, b):
             yield c.upload_file(fn)
 
 
-def test_upload_file_exception_sync(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            with tmp_text('myfile.py', 'syntax-error!') as fn:
-                with pytest.raises(SyntaxError):
-                    c.upload_file(fn)
+def test_upload_file_exception_sync(c):
+    with tmp_text('myfile.py', 'syntax-error!') as fn:
+        with pytest.raises(SyntaxError):
+            c.upload_file(fn)
 
 
 @pytest.mark.skip
@@ -1680,15 +1633,13 @@ def test_async_compute_with_scatter(c, s, a, b):
     assert result == 2 + 3
 
 
-def test_sync_compute(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            x = delayed(1)
-            y = delayed(inc)(x)
-            z = delayed(dec)(x)
+def test_sync_compute(c):
+    x = delayed(1)
+    y = delayed(inc)(x)
+    z = delayed(dec)(x)
 
-            yy, zz = c.compute([y, z], sync=True)
-            assert (yy, zz) == (2, 0)
+    yy, zz = c.compute([y, z], sync=True)
+    assert (yy, zz) == (2, 0)
 
 
 @gen_cluster(client=True)
@@ -1711,15 +1662,13 @@ def test_remote_submit_on_Future(c, s, a, b):
     assert result == 3
 
 
-def test_start_is_idempotent(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            c.start()
-            c.start()
-            c.start()
+def test_start_is_idempotent(c):
+    c.start()
+    c.start()
+    c.start()
 
-            x = c.submit(inc, 1)
-            assert x.result() == 2
+    x = c.submit(inc, 1)
+    assert x.result() == 2
 
 
 @gen_cluster(client=True)
@@ -1866,20 +1815,18 @@ def test_badly_serialized_input(c, s, a, b):
 
 
 @pytest.mark.skipif('True', reason="")
-def test_badly_serialized_input_stderr(capsys, loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            o = BadlySerializedObject()
-            future = c.submit(inc, o)
+def test_badly_serialized_input_stderr(capsys, c):
+    o = BadlySerializedObject()
+    future = c.submit(inc, o)
 
-            start = time()
-            while True:
-                sleep(0.01)
-                out, err = capsys.readouterr()
-                if 'hello!' in err:
-                    break
-                assert time() - start < 20
-            assert future.status == 'error'
+    start = time()
+    while True:
+        sleep(0.01)
+        out, err = capsys.readouterr()
+        if 'hello!' in err:
+            break
+        assert time() - start < 20
+    assert future.status == 'error'
 
 
 def test_repr(loop):
@@ -2026,15 +1973,13 @@ def test_forget_errors(c, s, a, b):
     assert z.key not in s.exceptions_blame
 
 
-def test_repr_sync(loop):
-    with cluster(nworkers=3) as (s, [a, b, c]):
-        with Client(s['address'], loop=loop) as c:
-            s = str(c)
-            r = repr(c)
-            assert c.scheduler.address in s
-            assert c.scheduler.address in r
-            assert str(3) in s  # nworkers
-            assert 'cores' in s
+def test_repr_sync(c):
+    s = str(c)
+    r = repr(c)
+    assert c.scheduler.address in s
+    assert c.scheduler.address in r
+    assert str(2) in s  # nworkers
+    assert 'cores' in s
 
 
 @gen_cluster(client=True)
@@ -2186,23 +2131,21 @@ def test__broadcast_dict(c, s, a, b):
     assert a.data == b.data == {'x': 1}
 
 
-def test_broadcast(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            x, y = c.scatter([1, 2], broadcast=True)
+def test_broadcast(c, s, a, b):
+    x, y = c.scatter([1, 2], broadcast=True)
 
-            has_what = sync(c.loop, c.scheduler.has_what)
+    has_what = sync(c.loop, c.scheduler.has_what)
 
-            assert {k: set(v) for k, v in has_what.items()} == {
-                a['address']: {x.key, y.key},
-                b['address']: {x.key, y.key}}
+    assert {k: set(v) for k, v in has_what.items()} == {
+        a['address']: {x.key, y.key},
+        b['address']: {x.key, y.key}}
 
-            [z] = c.scatter([3], broadcast=True, workers=[a['address']])
+    [z] = c.scatter([3], broadcast=True, workers=[a['address']])
 
-            has_what = sync(c.loop, c.scheduler.has_what)
-            assert {k: set(v) for k, v in has_what.items()} == {
-                a['address']: {x.key, y.key, z.key},
-                b['address']: {x.key, y.key}}
+    has_what = sync(c.loop, c.scheduler.has_what)
+    assert {k: set(v) for k, v in has_what.items()} == {
+        a['address']: {x.key, y.key, z.key},
+        b['address']: {x.key, y.key}}
 
 
 @gen_cluster(client=True)
@@ -2276,8 +2219,6 @@ def test__cancel_multi_client(s, a, b):
 
 @gen_cluster(client=True)
 def test__cancel_collection(c, s, a, b):
-    import dask.bag as db
-
     L = c.map(double, [[1], [2], [3]])
     x = db.Bag({('b', i): f for i, f in enumerate(L)}, 'b', 3)
 
@@ -2287,24 +2228,22 @@ def test__cancel_collection(c, s, a, b):
     assert not s.tasks
 
 
-def test_cancel(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            x = c.submit(slowinc, 1, key='x')
-            y = c.submit(slowinc, x, key='y')
-            z = c.submit(slowinc, y, key='z')
+def test_cancel(c):
+    x = c.submit(slowinc, 1, key='x')
+    y = c.submit(slowinc, x, key='y')
+    z = c.submit(slowinc, y, key='z')
 
-            c.cancel([y])
+    c.cancel([y])
 
-            start = time()
-            while not z.cancelled():
-                sleep(0.01)
-                assert time() < start + 5
+    start = time()
+    while not z.cancelled():
+        sleep(0.01)
+        assert time() < start + 5
 
-            assert x.result() == 2
+    assert x.result() == 2
 
-            z.cancel()
-            assert z.cancelled()
+    z.cancel()
+    assert z.cancelled()
 
 
 @gen_cluster(client=True)
@@ -2423,12 +2362,10 @@ def test_map_infinite_iterators(c, s, a, b):
     assert len(futures) == 2
 
 
-def test_map_iterator_sync(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            items = enumerate(range(10))
-            futures = c.map(lambda x: x, items)
-            next(futures).result() == (0, 0)
+def test_map_iterator_sync(c):
+    items = enumerate(range(10))
+    futures = c.map(lambda x: x, items)
+    next(futures).result() == (0, 0)
 
 
 @gen_cluster(client=True)
@@ -2500,23 +2437,21 @@ def test__persist(c, s, a, b):
     assert (gg == hh).all()
 
 
-def test_persist(loop):
+def test_persist(c):
     pytest.importorskip('dask.array')
     import dask.array as da
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            x = da.ones((10, 10), chunks=(5, 10))
-            y = 2 * (x + 1)
-            assert len(y.dask) == 6
-            yy = c.persist(y)
-            assert len(y.dask) == 6
-            assert len(yy.dask) == 2
-            assert all(isinstance(v, Future) for v in yy.dask.values())
-            assert yy.__dask_keys__() == y.__dask_keys__()
+    x = da.ones((10, 10), chunks=(5, 10))
+    y = 2 * (x + 1)
+    assert len(y.dask) == 6
+    yy = c.persist(y)
+    assert len(y.dask) == 6
+    assert len(yy.dask) == 2
+    assert all(isinstance(v, Future) for v in yy.dask.values())
+    assert yy.__dask_keys__() == y.__dask_keys__()
 
-            zz = yy.compute()
-            z = y.compute()
-            assert (zz == z).all()
+    zz = yy.compute()
+    z = y.compute()
+    assert (zz == z).all()
 
 
 @pytest.mark.avoid_travis  # This hangs intermittently.  We don't know why.
@@ -2538,8 +2473,6 @@ def test_long_traceback(c, s, a, b):
 
 @gen_cluster(client=True)
 def test_wait_on_collections(c, s, a, b):
-    import dask.bag as db
-
     L = c.map(double, [[1], [2], [3]])
     x = db.Bag({('b', i): f for i, f in enumerate(L)}, 'b', 3)
 
@@ -2557,7 +2490,6 @@ def test_futures_of(c, s, a, b):
     assert set(futures_of([x, [y], [[z]]])) == {x, y, z}
     assert set(futures_of({'x': x, 'y': [y]})) == {x, y}
 
-    import dask.bag as db
     b = db.Bag({('b', i): f for i, f in enumerate([x, y, z])}, 'b', 3)
     assert set(futures_of(b)) == {x, y, z}
 
@@ -2658,18 +2590,16 @@ def func():
     assert results == {w.address: func() for w in [a, b]}
 
 
-def test_run_sync(loop):
+def test_run_sync(c, s, a, b):
     def func(x, y=10):
         return x + y
 
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            result = c.run(func, 1, y=2)
-            assert result == {a['address']: 3,
-                              b['address']: 3}
+    result = c.run(func, 1, y=2)
+    assert result == {a['address']: 3,
+                      b['address']: 3}
 
-            result = c.run(func, 1, y=2, workers=[a['address']])
-            assert result == {a['address']: 3}
+    result = c.run(func, 1, y=2, workers=[a['address']])
+    assert result == {a['address']: 3}
 
 
 @gen_cluster(client=True)
@@ -2692,34 +2622,30 @@ def test_run_coroutine(c, s, a, b):
         assert results == {a.address: 3, b.address: 3}
 
 
-def test_run_coroutine_sync(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            result = c.run_coroutine(geninc, 2, delay=0.01)
-            assert result == {a['address']: 3,
-                              b['address']: 3}
+def test_run_coroutine_sync(c, s, a, b):
+    result = c.run_coroutine(geninc, 2, delay=0.01)
+    assert result == {a['address']: 3,
+                      b['address']: 3}
 
-            result = c.run_coroutine(geninc, 2,
-                                     workers=[a['address']])
-            assert result == {a['address']: 3}
+    result = c.run_coroutine(geninc, 2,
+                             workers=[a['address']])
+    assert result == {a['address']: 3}
 
-            t1 = time()
-            result = c.run_coroutine(geninc, 2, delay=10, wait=False)
-            t2 = time()
-            assert result is None
-            assert t2 - t1 <= 1.0
+    t1 = time()
+    result = c.run_coroutine(geninc, 2, delay=10, wait=False)
+    t2 = time()
+    assert result is None
+    assert t2 - t1 <= 1.0
 
 
-def test_run_exception(loop):
+def test_run_exception(c):
     def raise_exception(exc_type, exc_msg):
         raise exc_type(exc_msg)
 
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            for exc_type in [ValueError, RuntimeError]:
-                with pytest.raises(exc_type) as excinfo:
-                    c.run(raise_exception, exc_type, 'informative message')
-                assert 'informative message' in str(excinfo.value)
+    for exc_type in [ValueError, RuntimeError]:
+        with pytest.raises(exc_type) as excinfo:
+            c.run(raise_exception, exc_type, 'informative message')
+        assert 'informative message' in str(excinfo.value)
 
 
 def test_diagnostic_ui(loop):
@@ -2763,17 +2689,15 @@ def test_diagnostic_ui(loop):
             assert set(d) == {a_addr}
 
 
-def test_diagnostic_nbytes_sync(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            incs = c.map(inc, [1, 2, 3])
-            doubles = c.map(double, [1, 2, 3])
-            wait(incs + doubles)
+def test_diagnostic_nbytes_sync(c):
+    incs = c.map(inc, [1, 2, 3])
+    doubles = c.map(double, [1, 2, 3])
+    wait(incs + doubles)
 
-            assert c.nbytes(summary=False) == {k.key: sizeof(1)
-                                               for k in incs + doubles}
-            assert c.nbytes(summary=True) == {'inc': sizeof(1) * 3,
-                                              'double': sizeof(1) * 3}
+    assert c.nbytes(summary=False) == {k.key: sizeof(1)
+                                       for k in incs + doubles}
+    assert c.nbytes(summary=True) == {'inc': sizeof(1) * 3,
+                                      'double': sizeof(1) * 3}
 
 
 @gen_cluster(client=True)
@@ -2815,19 +2739,17 @@ def test_worker_aliases():
     yield s.close()
 
 
-def test_persist_get_sync(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            dadd = delayed(add)
-            x, y = delayed(1), delayed(2)
-            xx = delayed(add)(x, x)
-            yy = delayed(add)(y, y)
-            xxyy = delayed(add)(xx, yy)
+def test_persist_get_sync(c):
+    dadd = delayed(add)
+    x, y = delayed(1), delayed(2)
+    xx = delayed(add)(x, x)
+    yy = delayed(add)(y, y)
+    xxyy = delayed(add)(xx, yy)
 
-            xxyy2 = c.persist(xxyy)
-            xxyy3 = delayed(add)(xxyy2, 10)
+    xxyy2 = c.persist(xxyy)
+    xxyy3 = delayed(add)(xxyy2, 10)
 
-            assert xxyy3.compute() == ((1 + 1) + (2 + 2)) + 10
+    assert xxyy3.compute() == ((1 + 1) + (2 + 2)) + 10
 
 
 @gen_cluster(client=True)
@@ -2966,15 +2888,13 @@ def test_rebalance_execution(c, s, a, b):
     s.validate_state()
 
 
-def test_rebalance_sync(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            futures = c.map(inc, range(10), workers=[a['address']])
-            c.rebalance(futures)
+def test_rebalance_sync(c, s, a, b):
+    futures = c.map(inc, range(10), workers=[a['address']])
+    c.rebalance(futures)
 
-            has_what = c.has_what()
-            assert len(has_what) == 2
-            assert list(valmap(len, has_what).values()) == [5, 5]
+    has_what = c.has_what()
+    assert len(has_what) == 2
+    assert list(valmap(len, has_what).values()) == [5, 5]
 
 
 @gen_cluster(client=True)
@@ -3191,20 +3111,18 @@ def test_client_replicate_host(client, s, a, b, c):
     assert s.tasks[x.key].who_has == {aws, bws, cws}
 
 
-def test_client_replicate_sync(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            x = c.submit(inc, 1)
-            y = c.submit(inc, 2)
-            c.replicate([x, y], n=2)
+def test_client_replicate_sync(c):
+    x = c.submit(inc, 1)
+    y = c.submit(inc, 2)
+    c.replicate([x, y], n=2)
 
-            who_has = c.who_has()
-            assert len(who_has[x.key]) == len(who_has[y.key]) == 2
+    who_has = c.who_has()
+    assert len(who_has[x.key]) == len(who_has[y.key]) == 2
 
-            with pytest.raises(ValueError):
-                c.replicate([x], n=0)
+    with pytest.raises(ValueError):
+        c.replicate([x], n=0)
 
-            assert y.result() == 3
+    assert y.result() == 3
 
 
 @pytest.mark.skipif(sys.platform.startswith('win'),
@@ -3444,55 +3362,49 @@ def test_bad_tasks_fail(c, s, a, b):
         yield f
 
 
-def test_get_processing_sync(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            processing = c.processing()
-            assert not any(v for v in processing.values())
+def test_get_processing_sync(c, s, a, b):
+    processing = c.processing()
+    assert not any(v for v in processing.values())
 
-            futures = c.map(slowinc, range(10), delay=0.1,
-                            workers=[a['address']],
-                            allow_other_workers=False)
+    futures = c.map(slowinc, range(10), delay=0.1,
+                    workers=[a['address']],
+                    allow_other_workers=False)
 
-            sleep(0.2)
+    sleep(0.2)
 
-            aa = a['address']
-            bb = b['address']
-            processing = c.processing()
+    aa = a['address']
+    bb = b['address']
+    processing = c.processing()
 
-            assert set(c.processing(aa)) == {aa}
-            assert set(c.processing([aa])) == {aa}
+    assert set(c.processing(aa)) == {aa}
+    assert set(c.processing([aa])) == {aa}
 
-            c.cancel(futures)
+    c.cancel(futures)
 
 
-def test_close_idempotent(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            c.close()
-            c.close()
-            c.close()
+def test_close_idempotent(c):
+    c.close()
+    c.close()
+    c.close()
 
 
 @nodebug
-def test_get_returns_early(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            start = time()
-            with ignoring(RuntimeError):
-                result = c.get({'x': (throws, 1), 'y': (sleep, 1)}, ['x', 'y'])
-            assert time() < start + 0.5
-            # Futures should be released and forgotten
-            wait_for(lambda: not c.futures, timeout=0.1)
+def test_get_returns_early(c):
+    start = time()
+    with ignoring(RuntimeError):
+        result = c.get({'x': (throws, 1), 'y': (sleep, 1)}, ['x', 'y'])
+    assert time() < start + 0.5
+    # Futures should be released and forgotten
+    wait_for(lambda: not c.futures, timeout=0.1)
 
-            wait_for(lambda: not any(c.processing().values()), timeout=3)
+    wait_for(lambda: not any(c.processing().values()), timeout=3)
 
-            x = c.submit(inc, 1)
-            x.result()
+    x = c.submit(inc, 1)
+    x.result()
 
-            with ignoring(RuntimeError):
-                result = c.get({'x': (throws, 1), x.key: (inc, 1)}, ['x', x.key])
-            assert x.key in c.futures
+    with ignoring(RuntimeError):
+        result = c.get({'x': (throws, 1), x.key: (inc, 1)}, ['x', x.key])
+    assert x.key in c.futures
 
 
 @slow
@@ -3513,61 +3425,51 @@ def test_Client_clears_references_after_restart(c, s, a, b):
     assert key not in c.refcount
 
 
-def test_get_stops_work_after_error(loop):
-    with cluster(active_rpc_timeout=10) as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            with pytest.raises(RuntimeError):
-                c.get({'x': (throws, 1), 'y': (sleep, 1.5)}, ['x', 'y'])
+def test_get_stops_work_after_error(c):
+    with pytest.raises(RuntimeError):
+        c.get({'x': (throws, 1), 'y': (sleep, 1.5)}, ['x', 'y'])
 
-            start = time()
-            while any(c.processing().values()):
-                sleep(0.01)
-                assert time() < start + 0.5
+    start = time()
+    while any(c.processing().values()):
+        sleep(0.01)
+        assert time() < start + 0.5
 
 
-def test_as_completed_list(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            seq = c.map(inc, iter(range(5)))
-            seq2 = list(as_completed(seq))
-            assert set(c.gather(seq2)) == {1, 2, 3, 4, 5}
+def test_as_completed_list(c):
+    seq = c.map(inc, iter(range(5)))
+    seq2 = list(as_completed(seq))
+    assert set(c.gather(seq2)) == {1, 2, 3, 4, 5}
 
 
-def test_as_completed_results(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            seq = c.map(inc, range(5))
-            seq2 = list(as_completed(seq, with_results=True))
-            assert set(pluck(1, seq2)) == {1, 2, 3, 4, 5}
-            assert set(pluck(0, seq2)) == set(seq)
+def test_as_completed_results(c):
+    seq = c.map(inc, range(5))
+    seq2 = list(as_completed(seq, with_results=True))
+    assert set(pluck(1, seq2)) == {1, 2, 3, 4, 5}
+    assert set(pluck(0, seq2)) == set(seq)
 
 
 @pytest.mark.parametrize('with_results', [True, False])
-def test_as_completed_batches(loop, with_results):
+def test_as_completed_batches(c, with_results):
     n = 50
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            futures = c.map(slowinc, range(n), delay=0.01)
-            out = []
-            for batch in as_completed(futures, with_results=with_results).batches():
-                assert isinstance(batch, (tuple, list))
-                sleep(0.05)
-                out.extend(batch)
-
-            assert len(out) == n
-            if with_results:
-                assert set(pluck(1, out)) == set(range(1, n + 1))
-            else:
-                assert set(out) == set(futures)
+    futures = c.map(slowinc, range(n), delay=0.01)
+    out = []
+    for batch in as_completed(futures, with_results=with_results).batches():
+        assert isinstance(batch, (tuple, list))
+        sleep(0.05)
+        out.extend(batch)
+
+    assert len(out) == n
+    if with_results:
+        assert set(pluck(1, out)) == set(range(1, n + 1))
+    else:
+        assert set(out) == set(futures)
 
 
-def test_as_completed_next_batch(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            futures = c.map(slowinc, range(2), delay=0.1)
-            ac = as_completed(futures)
-            assert ac.next_batch(block=False) == []
-            assert set(ac.next_batch(block=True)).issubset(futures)
+def test_as_completed_next_batch(c):
+    futures = c.map(slowinc, range(2), delay=0.1)
+    ac = as_completed(futures)
+    assert ac.next_batch(block=False) == []
+    assert set(ac.next_batch(block=True)).issubset(futures)
 
 
 @gen_test()
@@ -3588,7 +3490,6 @@ def test_status():
 @gen_cluster(client=True)
 def test_persist_optimize_graph(c, s, a, b):
     i = 10
-    import dask.bag as db
     for method in [c.persist, c.compute]:
         b = db.range(i, npartitions=2)
         i += 1
@@ -3779,61 +3680,53 @@ def test_idempotence(s, a, b):
     yield f.close()
 
 
-def test_scheduler_info(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            info = c.scheduler_info()
-            assert isinstance(info, dict)
-            assert len(info['workers']) == 2
+def test_scheduler_info(c):
+    info = c.scheduler_info()
+    assert isinstance(info, dict)
+    assert len(info['workers']) == 2
 
 
-def test_write_scheduler_file(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            info = c.scheduler_info()
-            with tmpfile('json') as scheduler_file:
-                c.write_scheduler_file(scheduler_file)
-                with Client(scheduler_file=scheduler_file) as c2:
-                    info2 = c2.scheduler_info()
-                    assert c.scheduler.address == c2.scheduler.address
+def test_write_scheduler_file(c):
+    info = c.scheduler_info()
+    with tmpfile('json') as scheduler_file:
+        c.write_scheduler_file(scheduler_file)
+        with Client(scheduler_file=scheduler_file) as c2:
+            info2 = c2.scheduler_info()
+            assert c.scheduler.address == c2.scheduler.address
 
-                # test that a ValueError is raised if the scheduler_file
-                # attribute is already set
-                with pytest.raises(ValueError):
-                    c.write_scheduler_file(scheduler_file)
+        # test that a ValueError is raised if the scheduler_file
+        # attribute is already set
+        with pytest.raises(ValueError):
+            c.write_scheduler_file(scheduler_file)
 
 
-def test_get_versions(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            v = c.get_versions()
-            assert v['scheduler'] is not None
-            assert v['client'] is not None
-            assert len(v['workers']) == 2
-            for k, v in v['workers'].items():
-                assert v is not None
+def test_get_versions(c):
+    v = c.get_versions()
+    assert v['scheduler'] is not None
+    assert v['client'] is not None
+    assert len(v['workers']) == 2
+    for k, v in v['workers'].items():
+        assert v is not None
 
-            c.get_versions(check=True)
-            # smoke test for versions
-            # that this does not raise
+    c.get_versions(check=True)
+    # smoke test for versions
+    # that this does not raise
 
-            v = c.get_versions(packages=['requests'])
-            import requests
-            assert dict(v['client']['packages']['optional'])['requests'] == requests.__version__
+    v = c.get_versions(packages=['requests'])
+    import requests
+    assert dict(v['client']['packages']['optional'])['requests'] == requests.__version__
 
 
-def test_threaded_get_within_distributed(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            import dask.multiprocessing
-            for get in [dask.local.get_sync,
-                        dask.multiprocessing.get,
-                        dask.threaded.get]:
-                def f():
-                    return get({'x': (lambda: 1,)}, 'x')
+def test_threaded_get_within_distributed(c):
+    import dask.multiprocessing
+    for get in [dask.local.get_sync,
+                dask.multiprocessing.get,
+                dask.threaded.get]:
+        def f():
+            return get({'x': (lambda: 1,)}, 'x')
 
-                future = c.submit(f)
-                assert future.result() == 1
+        future = c.submit(f)
+        assert future.result() == 1
 
 
 @gen_cluster(client=True)
@@ -4238,22 +4131,20 @@ def test_auto_normalize_collection(c, s, a, b):
         assert end - start < 1
 
 
-def test_auto_normalize_collection_sync(loop):
+def test_auto_normalize_collection_sync(c):
     da = pytest.importorskip('dask.array')
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            x = da.ones(10, chunks=5)
+    x = da.ones(10, chunks=5)
 
-            y = x.map_blocks(slowinc, delay=1, dtype=x.dtype)
-            yy = c.persist(y)
+    y = x.map_blocks(slowinc, delay=1, dtype=x.dtype)
+    yy = c.persist(y)
 
-            wait(yy)
+    wait(yy)
 
-            with dask.config.set(optimizations=[c._optimize_insert_futures]):
-                start = time()
-                y.sum().compute()
-                end = time()
-                assert end - start < 1
+    with dask.config.set(optimizations=[c._optimize_insert_futures]):
+        start = time()
+        y.sum().compute()
+        end = time()
+        assert end - start < 1
 
 
 def assert_no_data_loss(scheduler):
@@ -4463,7 +4354,6 @@ def test_recreate_error_futures(c, s, a, b):
 
 @gen_cluster(client=True)
 def test_recreate_error_collection(c, s, a, b):
-    import dask.bag as db
     b = db.range(10, npartitions=4)
     b = b.map(lambda x: 1 / x)
     b = b.persist()
@@ -4505,28 +4395,24 @@ def test_recreate_error_array(c, s, a, b):
     assert '0.,0.,0.' in str(args).replace(' ', '')  # args contain actual arrays
 
 
-def test_recreate_error_sync(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            x0 = c.submit(dec, 2)
-            y0 = c.submit(dec, 1)
-            x = c.submit(div, 1, x0)
-            y = c.submit(div, 1, y0)
-            tot = c.submit(sum, x, y)
-            f = c.compute(tot)
+def test_recreate_error_sync(c):
+    x0 = c.submit(dec, 2)
+    y0 = c.submit(dec, 1)
+    x = c.submit(div, 1, x0)
+    y = c.submit(div, 1, y0)
+    tot = c.submit(sum, x, y)
+    f = c.compute(tot)
 
-            with pytest.raises(ZeroDivisionError) as e:
-                c.recreate_error_locally(f)
-            assert f.status == 'error'
+    with pytest.raises(ZeroDivisionError) as e:
+        c.recreate_error_locally(f)
+    assert f.status == 'error'
 
 
-def test_recreate_error_not_error(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            f = c.submit(dec, 2)
-            with pytest.raises(ValueError) as e:
-                c.recreate_error_locally(f)
-            assert "No errored futures passed" in str(e)
+def test_recreate_error_not_error(c):
+    f = c.submit(dec, 2)
+    with pytest.raises(ValueError) as e:
+        c.recreate_error_locally(f)
+    assert "No errored futures passed" in str(e)
 
 
 @gen_cluster(client=True)
@@ -4674,63 +4560,59 @@ def test_close(s, a, b):
         assert time() < start + 5
 
 
-def test_threadsafe(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            def f(_):
-                d = deque(maxlen=50)
-                for i in range(100):
-                    future = c.submit(inc, random.randint(0, 100))
-                    d.append(future)
-                    sleep(0.001)
-                c.gather(list(d))
-                total = c.submit(sum, list(d))
-                return total.result()
-
-            from concurrent.futures import ThreadPoolExecutor
-            with ThreadPoolExecutor(20) as e:
-                results = list(e.map(f, range(20)))
-                assert results and all(results)
-                del results
+def test_threadsafe(c):
+    def f(_):
+        d = deque(maxlen=50)
+        for i in range(100):
+            future = c.submit(inc, random.randint(0, 100))
+            d.append(future)
+            sleep(0.001)
+        c.gather(list(d))
+        total = c.submit(sum, list(d))
+        return total.result()
+
+    from concurrent.futures import ThreadPoolExecutor
+    with ThreadPoolExecutor(20) as e:
+        results = list(e.map(f, range(20)))
+        assert results and all(results)
+        del results
 
 
 @slow
-def test_threadsafe_get(loop):
+def test_threadsafe_get(c):
     da = pytest.importorskip('dask.array')
     x = da.arange(100, chunks=(10,))
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            def f(_):
-                total = 0
-                for i in range(20):
-                    total += (x + random.randint(0, 20)).sum().compute()
-                    sleep(0.001)
-                return total
 
-            from concurrent.futures import ThreadPoolExecutor
-            e = ThreadPoolExecutor(30)
-            results = list(e.map(f, range(30)))
-            assert results and all(results)
+    def f(_):
+        total = 0
+        for i in range(20):
+            total += (x + random.randint(0, 20)).sum().compute()
+            sleep(0.001)
+        return total
+
+    from concurrent.futures import ThreadPoolExecutor
+    e = ThreadPoolExecutor(30)
+    results = list(e.map(f, range(30)))
+    assert results and all(results)
 
 
 @slow
-def test_threadsafe_compute(loop):
+def test_threadsafe_compute(c):
     da = pytest.importorskip('dask.array')
     x = da.arange(100, chunks=(10,))
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            def f(_):
-                total = 0
-                for i in range(20):
-                    future = c.compute((x + random.randint(0, 20)).sum())
-                    total += future.result()
-                    sleep(0.001)
-                return total
 
-            from concurrent.futures import ThreadPoolExecutor
-            e = ThreadPoolExecutor(30)
-            results = list(e.map(f, range(30)))
-            assert results and all(results)
+    def f(_):
+        total = 0
+        for i in range(20):
+            future = c.compute((x + random.randint(0, 20)).sum())
+            total += future.result()
+            sleep(0.001)
+        return total
+
+    from concurrent.futures import ThreadPoolExecutor
+    e = ThreadPoolExecutor(30)
+    results = list(e.map(f, range(30)))
+    assert results and all(results)
 
 
 @gen_cluster(client=True)
@@ -4839,14 +4721,12 @@ def f():
         assert s.priorities['f'] > s.priorities['slowinc']  # lower values schedule first
 
 
-def test_get_client_sync(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            results = c.run(lambda: get_worker().scheduler.address)
-            assert results == {w['address']: s['address'] for w in [a, b]}
+def test_get_client_sync(c, s, a, b):
+    results = c.run(lambda: get_worker().scheduler.address)
+    assert results == {w['address']: s['address'] for w in [a, b]}
 
-            results = c.run(lambda: get_client().scheduler.address)
-            assert results == {w['address']: s['address'] for w in [a, b]}
+    results = c.run(lambda: get_client().scheduler.address)
+    assert results == {w['address']: s['address'] for w in [a, b]}
 
 
 @gen_cluster(client=True)
@@ -4865,22 +4745,20 @@ def test_serialize_collections_of_futures(c, s, a, b):
     assert_eq(df, df2)
 
 
-def test_serialize_collections_of_futures_sync(loop):
+def test_serialize_collections_of_futures_sync(c):
     pd = pytest.importorskip('pandas')
     dd = pytest.importorskip('dask.dataframe')
     from dask.dataframe.utils import assert_eq
 
     df = pd.DataFrame({'x': [1, 2, 3]})
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            ddf = dd.from_pandas(df, npartitions=2).persist()
-            future = c.scatter(ddf)
+    ddf = dd.from_pandas(df, npartitions=2).persist()
+    future = c.scatter(ddf)
 
-            result = future.result()
-            assert_eq(result.compute(), df)
+    result = future.result()
+    assert_eq(result.compute(), df)
 
-            assert future.type == dd.DataFrame
-            assert c.submit(lambda x, y:  assert_eq(x.compute(), y), future, df).result()
+    assert future.type == dd.DataFrame
+    assert c.submit(lambda x, y:  assert_eq(x.compute(), y), future, df).result()
 
 
 def _dynamic_workload(x, delay=0.01):
@@ -4898,20 +4776,18 @@ def _dynamic_workload(x, delay=0.01):
     return total.result()
 
 
-def _test_dynamic_workloads_sync(loop, delay):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            future = c.submit(_dynamic_workload, 0, delay=delay)
-            assert future.result(timeout=40) == 52
+def _test_dynamic_workloads_sync(c, delay):
+    future = c.submit(_dynamic_workload, 0, delay=delay)
+    assert future.result(timeout=40) == 52
 
 
-def test_dynamic_workloads_sync(loop):
-    _test_dynamic_workloads_sync(loop, delay=0.02)
+def test_dynamic_workloads_sync(c):
+    _test_dynamic_workloads_sync(c, delay=0.02)
 
 
 @slow
-def test_dynamic_workloads_sync_random(loop):
-    _test_dynamic_workloads_sync(loop, delay='random')
+def test_dynamic_workloads_sync_random(c):
+    _test_dynamic_workloads_sync(c, delay='random')
 
 
 @gen_cluster(client=True)
@@ -4959,18 +4835,16 @@ def test_unicode_keys(c, s, a, b):
     assert result3 == 123
 
 
-def test_use_synchronous_client_in_async_context(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            @gen.coroutine
-            def f():
-                x = yield c.scatter(123)
-                y = c.submit(inc, x)
-                z = yield c.gather(y)
-                raise gen.Return(z)
+def test_use_synchronous_client_in_async_context(loop, c):
+    @gen.coroutine
+    def f():
+        x = yield c.scatter(123)
+        y = c.submit(inc, x)
+        z = yield c.gather(y)
+        raise gen.Return(z)
 
-            z = sync(loop, f)
-            assert z == 124
+    z = sync(loop, f)
+    assert z == 124
 
 
 def test_quiet_quit_when_cluster_leaves(loop_in_thread):
@@ -4988,13 +4862,12 @@ def test_quiet_quit_when_cluster_leaves(loop_in_thread):
         assert not text
 
 
-def test_warn_executor(loop):
-    with cluster() as (s, [a, b]):
-        with warnings.catch_warnings(record=True) as record:
-            with Executor(s['address'], loop=loop) as c:
-                pass
+def test_warn_executor(loop, s, a, b):
+    with warnings.catch_warnings(record=True) as record:
+        with Executor(s['address'], loop=loop) as c:
+            pass
 
-        assert any('Client' in str(r.message) for r in record)
+    assert any('Client' in str(r.message) for r in record)
 
 
 @gen_cluster([('127.0.0.1', 4)] * 2, client=True)
@@ -5273,12 +5146,11 @@ def test_client_name(s, a, b):
     yield c._close()
 
 
-def test_client_doesnt_close_given_loop(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            assert c.submit(inc, 1).result() == 2
-        with Client(s['address'], loop=loop) as c:
-            assert c.submit(inc, 2).result() == 3
+def test_client_doesnt_close_given_loop(loop, s, a, b):
+    with Client(s['address'], loop=loop) as c:
+        assert c.submit(inc, 1).result() == 2
+    with Client(s['address'], loop=loop) as c:
+        assert c.submit(inc, 2).result() == 3
 
 
 @gen_cluster(client=True, ncores=[])
@@ -5468,15 +5340,13 @@ def test_direct_async(s, a, b):
     yield c.close()
 
 
-def test_direct_sync(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            assert not c.direct_to_workers
+def test_direct_sync(c):
+    assert not c.direct_to_workers
 
-            def f():
-                return get_client().direct_to_workers
+    def f():
+        return get_client().direct_to_workers
 
-            assert c.submit(f).result()
+    assert c.submit(f).result()
 
 
 @gen_cluster()
diff --git a/distributed/tests/test_client_executor.py b/distributed/tests/test_client_executor.py
index c4cbb8be05c..117c9c31dc3 100644
--- a/distributed/tests/test_client_executor.py
+++ b/distributed/tests/test_client_executor.py
@@ -10,255 +10,234 @@
 import pytest
 from toolz import take
 
-from distributed.client import Client
 from distributed.utils_test import (slowinc, slowadd, slowdec,
-                                    inc, throws, cluster, varying)
-from distributed.utils_test import loop  # noqa: F401
+                                    inc, throws, varying)
+from distributed.utils_test import client, cluster_fixture, loop, s, a, b  # noqa: F401
 
 
 def number_of_processing_tasks(client):
     return sum(len(v) for k, v in client.processing().items())
 
 
-def test_submit(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            with c.get_executor() as e:
-                f1 = e.submit(slowadd, 1, 2)
-                assert isinstance(f1, Future)
-                f2 = e.submit(slowadd, 3, y=4)
-                f3 = e.submit(throws, "foo")
-                f4 = e.submit(slowadd, x=5, y=6)
-                assert f1.result() == 3
-                assert f2.result() == 7
-                with pytest.raises(RuntimeError):
-                    f3.result()
-                assert f4.result() == 11
-
-
-def test_as_completed(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            with c.get_executor() as e:
-                N = 10
-                fs = [e.submit(slowinc, i, delay=0.02) for i in range(N)]
-                expected = set(range(1, N + 1))
-
-                for f in as_completed(fs):
-                    res = f.result()
-                    assert res in expected
-                    expected.remove(res)
-
-                assert not expected
-
-
-def test_wait(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            with c.get_executor(pure=False) as e:
-                N = 10
-                fs = [e.submit(slowinc, i, delay=0.05) for i in range(N)]
-                res = wait(fs, timeout=0.01)
-                assert len(res.not_done) > 0
-                res = wait(fs)
-                assert len(res.not_done) == 0
-                assert res.done == set(fs)
-
-                fs = [e.submit(slowinc, i, delay=0.05) for i in range(N)]
-                res = wait(fs, return_when=FIRST_COMPLETED)
-                assert len(res.not_done) > 0
-                assert len(res.done) >= 1
-                res = wait(fs)
-                assert len(res.not_done) == 0
-                assert res.done == set(fs)
-
-                fs = [e.submit(slowinc, i, delay=0.05) for i in range(N)]
-                fs += [e.submit(throws, None)]
-                fs += [e.submit(slowdec, i, delay=0.05) for i in range(N)]
-                res = wait(fs, return_when=FIRST_EXCEPTION)
-                assert any(f.exception() for f in res.done)
-                assert res.not_done
-
-                errors = []
-                for fs in res.done:
-                    try:
-                        fs.result()
-                    except RuntimeError as e:
-                        errors.append(e)
-
-                assert len(errors) == 1
-                assert "hello" in str(errors[0])
-
-
-def test_cancellation(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            with c.get_executor(pure=False) as e:
-                fut = e.submit(time.sleep, 2.0)
-                start = time.time()
-                while number_of_processing_tasks(c) == 0:
-                    assert time.time() < start + 1
-                    time.sleep(0.01)
-                assert not fut.done()
-
-                fut.cancel()
-                assert fut.cancelled()
-                start = time.time()
-                while number_of_processing_tasks(c) != 0:
-                    assert time.time() < start + 1
-                    time.sleep(0.01)
-
-                with pytest.raises(CancelledError):
-                    fut.result()
-
-            # With wait()
-            with c.get_executor(pure=False) as e:
-                N = 10
-                fs = [e.submit(slowinc, i, delay=0.02) for i in range(N)]
-                fs[3].cancel()
-                res = wait(fs, return_when=FIRST_COMPLETED)
-                assert len(res.not_done) > 0
-                assert len(res.done) >= 1
-
-                assert fs[3] in res.done
-                assert fs[3].cancelled()
-
-            # With as_completed()
-            with c.get_executor(pure=False) as e:
-                N = 10
-                fs = [e.submit(slowinc, i, delay=0.02) for i in range(N)]
-                fs[3].cancel()
-                fs[8].cancel()
-
-                n_cancelled = sum(f.cancelled() for f in as_completed(fs))
-                assert n_cancelled == 2
-
-
-def test_map(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            with c.get_executor() as e:
-                N = 10
-                it = e.map(inc, range(N))
-                expected = set(range(1, N + 1))
-                for x in it:
-                    expected.remove(x)
-                assert not expected
-
-            with c.get_executor(pure=False) as e:
-                N = 10
-                it = e.map(slowinc, range(N), [0.1] * N, timeout=0.4)
-                results = []
-                with pytest.raises(TimeoutError):
-                    for x in it:
-                        results.append(x)
-                assert 2 <= len(results) < 7
-
-            with c.get_executor(pure=False) as e:
-                N = 10
-                # Not consuming the iterator will cancel remaining tasks
-                it = e.map(slowinc, range(N), [0.1] * N)
-                for x in take(2, it):
-                    pass
-                # Some tasks still processing
-                assert number_of_processing_tasks(c) > 0
-                # Garbage collect the iterator => remaining tasks are cancelled
-                del it
-                assert number_of_processing_tasks(c) == 0
+def test_submit(client):
+    with client.get_executor() as e:
+        f1 = e.submit(slowadd, 1, 2)
+        assert isinstance(f1, Future)
+        f2 = e.submit(slowadd, 3, y=4)
+        f3 = e.submit(throws, "foo")
+        f4 = e.submit(slowadd, x=5, y=6)
+        assert f1.result() == 3
+        assert f2.result() == 7
+        with pytest.raises(RuntimeError):
+            f3.result()
+        assert f4.result() == 11
+
+
+def test_as_completed(client):
+    with client.get_executor() as e:
+        N = 10
+        fs = [e.submit(slowinc, i, delay=0.02) for i in range(N)]
+        expected = set(range(1, N + 1))
+
+        for f in as_completed(fs):
+            res = f.result()
+            assert res in expected
+            expected.remove(res)
+
+        assert not expected
+
+
+def test_wait(client):
+    with client.get_executor(pure=False) as e:
+        N = 10
+        fs = [e.submit(slowinc, i, delay=0.05) for i in range(N)]
+        res = wait(fs, timeout=0.01)
+        assert len(res.not_done) > 0
+        res = wait(fs)
+        assert len(res.not_done) == 0
+        assert res.done == set(fs)
+
+        fs = [e.submit(slowinc, i, delay=0.05) for i in range(N)]
+        res = wait(fs, return_when=FIRST_COMPLETED)
+        assert len(res.not_done) > 0
+        assert len(res.done) >= 1
+        res = wait(fs)
+        assert len(res.not_done) == 0
+        assert res.done == set(fs)
+
+        fs = [e.submit(slowinc, i, delay=0.05) for i in range(N)]
+        fs += [e.submit(throws, None)]
+        fs += [e.submit(slowdec, i, delay=0.05) for i in range(N)]
+        res = wait(fs, return_when=FIRST_EXCEPTION)
+        assert any(f.exception() for f in res.done)
+        assert res.not_done
+
+        errors = []
+        for fs in res.done:
+            try:
+                fs.result()
+            except RuntimeError as e:
+                errors.append(e)
+
+        assert len(errors) == 1
+        assert "hello" in str(errors[0])
+
+
+def test_cancellation(client):
+    with client.get_executor(pure=False) as e:
+        fut = e.submit(time.sleep, 2.0)
+        start = time.time()
+        while number_of_processing_tasks(client) == 0:
+            assert time.time() < start + 1
+            time.sleep(0.01)
+        assert not fut.done()
+
+        fut.cancel()
+        assert fut.cancelled()
+        start = time.time()
+        while number_of_processing_tasks(client) != 0:
+            assert time.time() < start + 1
+            time.sleep(0.01)
+
+        with pytest.raises(CancelledError):
+            fut.result()
+
+    # With wait()
+    with client.get_executor(pure=False) as e:
+        N = 10
+        fs = [e.submit(slowinc, i, delay=0.02) for i in range(N)]
+        fs[3].cancel()
+        res = wait(fs, return_when=FIRST_COMPLETED)
+        assert len(res.not_done) > 0
+        assert len(res.done) >= 1
+
+        assert fs[3] in res.done
+        assert fs[3].cancelled()
+
+    # With as_completed()
+    with client.get_executor(pure=False) as e:
+        N = 10
+        fs = [e.submit(slowinc, i, delay=0.02) for i in range(N)]
+        fs[3].cancel()
+        fs[8].cancel()
+
+        n_cancelled = sum(f.cancelled() for f in as_completed(fs))
+        assert n_cancelled == 2
+
+
+def test_map(client):
+    with client.get_executor() as e:
+        N = 10
+        it = e.map(inc, range(N))
+        expected = set(range(1, N + 1))
+        for x in it:
+            expected.remove(x)
+        assert not expected
+
+    with client.get_executor(pure=False) as e:
+        N = 10
+        it = e.map(slowinc, range(N), [0.1] * N, timeout=0.4)
+        results = []
+        with pytest.raises(TimeoutError):
+            for x in it:
+                results.append(x)
+        assert 2 <= len(results) < 7
+
+    with client.get_executor(pure=False) as e:
+        N = 10
+        # Not consuming the iterator will cancel remaining tasks
+        it = e.map(slowinc, range(N), [0.1] * N)
+        for x in take(2, it):
+            pass
+        # Some tasks still processing
+        assert number_of_processing_tasks(client) > 0
+        # Garbage collect the iterator => remaining tasks are cancelled
+        del it
+        assert number_of_processing_tasks(client) == 0
 
 
 def get_random():
     return random.random()
 
 
-def test_pure(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            N = 10
-            with c.get_executor() as e:
-                fs = [e.submit(get_random) for i in range(N)]
-                res = [fut.result() for fut in as_completed(fs)]
-                assert len(set(res)) < len(res)
-            with c.get_executor(pure=False) as e:
-                fs = [e.submit(get_random) for i in range(N)]
-                res = [fut.result() for fut in as_completed(fs)]
-                assert len(set(res)) == len(res)
-
-
-def test_workers(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            N = 10
-            with c.get_executor(workers=[b['address']]) as e:
-                fs = [e.submit(slowinc, i) for i in range(N)]
-                wait(fs)
-                has_what = c.has_what()
-                assert not has_what.get(a['address'])
-                assert len(has_what[b['address']]) == N
-
-
-def test_unsupported_arguments(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            with pytest.raises(TypeError) as excinfo:
-                c.get_executor(workers=[b['address']], foo=1, bar=2)
-            assert ("unsupported arguments to ClientExecutor: ['bar', 'foo']"
-                    in str(excinfo.value))
-
-
-def test_retries(loop):
+def test_pure(client):
+    N = 10
+    with client.get_executor() as e:
+        fs = [e.submit(get_random) for i in range(N)]
+        res = [fut.result() for fut in as_completed(fs)]
+        assert len(set(res)) < len(res)
+    with client.get_executor(pure=False) as e:
+        fs = [e.submit(get_random) for i in range(N)]
+        res = [fut.result() for fut in as_completed(fs)]
+        assert len(set(res)) == len(res)
+
+
+def test_workers(client, s, a, b):
+    N = 10
+    with client.get_executor(workers=[b['address']]) as e:
+        fs = [e.submit(slowinc, i) for i in range(N)]
+        wait(fs)
+        has_what = client.has_what()
+        assert not has_what.get(a['address'])
+        assert len(has_what[b['address']]) == N
+
+
+def test_unsupported_arguments(client, s, a, b):
+    with pytest.raises(TypeError) as excinfo:
+        client.get_executor(workers=[b['address']], foo=1, bar=2)
+    assert ("unsupported arguments to ClientExecutor: ['bar', 'foo']"
+            in str(excinfo.value))
+
+
+def test_retries(client):
     args = [ZeroDivisionError("one"), ZeroDivisionError("two"), 42]
 
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            with c.get_executor(retries=3, pure=False) as e:
-                future = e.submit(varying(args))
-                assert future.result() == 42
-
-            with c.get_executor(retries=2) as e:
-                future = e.submit(varying(args))
-                result = future.result()
-                assert result == 42
-
-            with c.get_executor(retries=1) as e:
-                future = e.submit(varying(args))
-                with pytest.raises(ZeroDivisionError) as exc_info:
-                    res = future.result()
-                exc_info.match("two")
-
-            with c.get_executor(retries=0) as e:
-                future = e.submit(varying(args))
-                with pytest.raises(ZeroDivisionError) as exc_info:
-                    res = future.result()
-                exc_info.match("one")
-
-
-def test_shutdown(loop):
-    with cluster(active_rpc_timeout=10) as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            # shutdown(wait=True) waits for pending tasks to finish
-            e = c.get_executor()
-            fut = e.submit(time.sleep, 1.0)
-            t1 = time.time()
-            e.shutdown()
-            dt = time.time() - t1
-            assert 0.5 <= dt <= 2.0
-            time.sleep(0.1)  # wait for future outcome to propagate
-            assert fut.done()
-            fut.result()  # doesn't raise
-
-            with pytest.raises(RuntimeError):
-                e.submit(time.sleep, 1.0)
-
-            # shutdown(wait=False) cancels pending tasks
-            e = c.get_executor()
-            fut = e.submit(time.sleep, 2.0)
-            t1 = time.time()
-            e.shutdown(wait=False)
-            dt = time.time() - t1
-            assert dt < 0.5
-            time.sleep(0.1)  # wait for future outcome to propagate
-            assert fut.cancelled()
-
-            with pytest.raises(RuntimeError):
-                e.submit(time.sleep, 1.0)
+    with client.get_executor(retries=3, pure=False) as e:
+        future = e.submit(varying(args))
+        assert future.result() == 42
+
+    with client.get_executor(retries=2) as e:
+        future = e.submit(varying(args))
+        result = future.result()
+        assert result == 42
+
+    with client.get_executor(retries=1) as e:
+        future = e.submit(varying(args))
+        with pytest.raises(ZeroDivisionError) as exc_info:
+            res = future.result()
+        exc_info.match("two")
+
+    with client.get_executor(retries=0) as e:
+        future = e.submit(varying(args))
+        with pytest.raises(ZeroDivisionError) as exc_info:
+            res = future.result()
+        exc_info.match("one")
+
+
+def test_shutdown(client):
+    # shutdown(wait=True) waits for pending tasks to finish
+    e = client.get_executor()
+    fut = e.submit(time.sleep, 1.0)
+    t1 = time.time()
+    e.shutdown()
+    dt = time.time() - t1
+    assert 0.5 <= dt <= 2.0
+    time.sleep(0.1)  # wait for future outcome to propagate
+    assert fut.done()
+    fut.result()  # doesn't raise
+
+    with pytest.raises(RuntimeError):
+        e.submit(time.sleep, 1.0)
+
+    # shutdown(wait=False) cancels pending tasks
+    e = client.get_executor()
+    fut = e.submit(time.sleep, 2.0)
+    t1 = time.time()
+    e.shutdown(wait=False)
+    dt = time.time() - t1
+    assert dt < 0.5
+    time.sleep(0.1)  # wait for future outcome to propagate
+    assert fut.cancelled()
+
+    with pytest.raises(RuntimeError):
+        e.submit(time.sleep, 1.0)
diff --git a/distributed/tests/test_collections.py b/distributed/tests/test_collections.py
index 370dbecfab9..cb51d62c5fd 100644
--- a/distributed/tests/test_collections.py
+++ b/distributed/tests/test_collections.py
@@ -7,10 +7,9 @@
 
 import dask.dataframe as dd
 import dask.bag as db
-from distributed import Client
 from distributed.client import wait
-from distributed.utils_test import cluster, gen_cluster
-from distributed.utils_test import loop # noqa F401
+from distributed.utils_test import gen_cluster
+from distributed.utils_test import client, cluster_fixture, loop # noqa F401
 import numpy as np
 import pandas as pd
 import pandas.util.testing as tm
@@ -104,35 +103,29 @@ def test_bag_groupby_tasks_default(c, s, a, b):
 
 
 @pytest.mark.parametrize('wait', [wait, lambda x: None])
-def test_dataframe_set_index_sync(loop, wait):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            df = dd.demo.make_timeseries('2000', '2001',
-                                         {'value': float, 'name': str, 'id': int},
-                                         freq='2H', partition_freq='1M', seed=1)
-            df = c.persist(df)
-            wait(df)
+def test_dataframe_set_index_sync(wait, client):
+    df = dd.demo.make_timeseries('2000', '2001',
+                                 {'value': float, 'name': str, 'id': int},
+                                 freq='2H', partition_freq='1M', seed=1)
+    df = client.persist(df)
+    wait(df)
 
-            df2 = df.set_index('name', shuffle='tasks')
-            df2 = c.persist(df2)
+    df2 = df.set_index('name', shuffle='tasks')
+    df2 = client.persist(df2)
 
-            assert len(df2)
+    assert len(df2)
 
 
-def test_loc_sync(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            df = pd.util.testing.makeTimeDataFrame()
-            ddf = dd.from_pandas(df, npartitions=10)
-            ddf.loc['2000-01-17':'2000-01-24'].compute()
+def test_loc_sync(client):
+    df = pd.util.testing.makeTimeDataFrame()
+    ddf = dd.from_pandas(df, npartitions=10)
+    ddf.loc['2000-01-17':'2000-01-24'].compute()
 
 
-def test_rolling_sync(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            df = pd.util.testing.makeTimeDataFrame()
-            ddf = dd.from_pandas(df, npartitions=10)
-            ddf.A.rolling(2).mean().compute()
+def test_rolling_sync(client):
+    df = pd.util.testing.makeTimeDataFrame()
+    ddf = dd.from_pandas(df, npartitions=10)
+    ddf.A.rolling(2).mean().compute()
 
 
 @gen_cluster(client=True)
@@ -143,31 +136,30 @@ def test_loc(c, s, a, b):
     yield future
 
 
-def test_dataframe_groupby_tasks(loop):
+def test_dataframe_groupby_tasks(client):
     df = pd.util.testing.makeTimeDataFrame()
     df['A'] = df.A // 0.1
     df['B'] = df.B // 0.1
     ddf = dd.from_pandas(df, npartitions=10)
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            for ind in [lambda x: 'A', lambda x: x.A]:
-                a = df.groupby(ind(df)).apply(len)
-                b = ddf.groupby(ind(ddf)).apply(len, meta=int)
-                assert_equal(a, b.compute(scheduler='sync').sort_index())
-                assert not any('partd' in k[0] for k in b.dask)
-
-                a = df.groupby(ind(df)).B.apply(len)
-                b = ddf.groupby(ind(ddf)).B.apply(len, meta=('B', int))
-                assert_equal(a, b.compute(scheduler='sync').sort_index())
-                assert not any('partd' in k[0] for k in b.dask)
-
-            with pytest.raises((NotImplementedError, ValueError)):
-                ddf.groupby(ddf[['A', 'B']]).apply(len, meta=int)
-
-            a = df.groupby(['A', 'B']).apply(len)
-            b = ddf.groupby(['A', 'B']).apply(len, meta=int)
-
-            assert_equal(a, b.compute(scheduler='sync').sort_index())
+
+    for ind in [lambda x: 'A', lambda x: x.A]:
+        a = df.groupby(ind(df)).apply(len)
+        b = ddf.groupby(ind(ddf)).apply(len, meta=int)
+        assert_equal(a, b.compute(scheduler='sync').sort_index())
+        assert not any('partd' in k[0] for k in b.dask)
+
+        a = df.groupby(ind(df)).B.apply(len)
+        b = ddf.groupby(ind(ddf)).B.apply(len, meta=('B', int))
+        assert_equal(a, b.compute(scheduler='sync').sort_index())
+        assert not any('partd' in k[0] for k in b.dask)
+
+    with pytest.raises((NotImplementedError, ValueError)):
+        ddf.groupby(ddf[['A', 'B']]).apply(len, meta=int)
+
+    a = df.groupby(['A', 'B']).apply(len)
+    b = ddf.groupby(['A', 'B']).apply(len, meta=int)
+
+    assert_equal(a, b.compute(scheduler='sync').sort_index())
 
 
 @gen_cluster(client=True)
diff --git a/distributed/tests/test_joblib.py b/distributed/tests/test_joblib.py
index aa81a45b5f3..8eec12899d6 100644
--- a/distributed/tests/test_joblib.py
+++ b/distributed/tests/test_joblib.py
@@ -10,7 +10,7 @@
 from distributed import Client
 from distributed.metrics import time
 from distributed.utils_test import cluster, inc
-from distributed.utils_test import loop # noqa F401
+from distributed.utils_test import loop, client, cluster_fixture, s, a, b  # noqa F401
 from toolz import identity
 
 distributed_joblib = pytest.importorskip('distributed.joblib')
@@ -39,49 +39,46 @@ def slow_raise_value_error(condition, duration=0.05):
         raise ValueError("condition evaluated to True")
 
 
-def test_simple(loop, joblib):
+def test_simple(client, joblib):
     Parallel = joblib.Parallel
     delayed = joblib.delayed
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as client:
-            with joblib.parallel_backend('dask') as (ba, _):
-                seq = Parallel()(delayed(inc)(i) for i in range(10))
-                assert seq == [inc(i) for i in range(10)]
 
-                with pytest.raises(ValueError):
-                    Parallel()(delayed(slow_raise_value_error)(i == 3)
-                               for i in range(10))
+    with joblib.parallel_backend('dask') as (ba, _):
+        seq = Parallel()(delayed(inc)(i) for i in range(10))
+        assert seq == [inc(i) for i in range(10)]
 
-                seq = Parallel()(delayed(inc)(i) for i in range(10))
-                assert seq == [inc(i) for i in range(10)]
+        with pytest.raises(ValueError):
+            Parallel()(delayed(slow_raise_value_error)(i == 3)
+                       for i in range(10))
+
+        seq = Parallel()(delayed(inc)(i) for i in range(10))
+        assert seq == [inc(i) for i in range(10)]
 
 
 def random2():
     return random()
 
 
-def test_dont_assume_function_purity(loop, joblib):
+def test_dont_assume_function_purity(client, joblib):
     Parallel = joblib.Parallel
     delayed = joblib.delayed
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as client:
-            with joblib.parallel_backend('dask') as (ba, _):
-                x, y = Parallel()(delayed(random2)() for i in range(2))
-                assert x != y
+
+    with joblib.parallel_backend('dask') as (ba, _):
+        x, y = Parallel()(delayed(random2)() for i in range(2))
+        assert x != y
 
 
-def test_joblib_funcname(loop, joblib):
+def test_joblib_funcname(client, joblib):
     Parallel = joblib.Parallel
     delayed = joblib.delayed
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as client:
-            with joblib.parallel_backend('dask') as (ba, _):
-                x, y = Parallel()(delayed(inc)(i) for i in range(2))
 
-            def f(dask_scheduler):
-                return list(dask_scheduler.transition_log)
-            log = client.run_on_scheduler(f)
-            assert all(tup[0].startswith('inc-batch') for tup in log)
+    with joblib.parallel_backend('dask') as (ba, _):
+        x, y = Parallel()(delayed(inc)(i) for i in range(2))
+
+    def f(dask_scheduler):
+        return list(dask_scheduler.transition_log)
+    log = client.run_on_scheduler(f)
+    assert all(tup[0].startswith('inc-batch') for tup in log)
 
 
 def test_joblib_backend_subclass(joblib):
@@ -108,7 +105,7 @@ def __reduce__(self):
         return (CountSerialized, (self.x,))
 
 
-def test_joblib_scatter(loop, joblib):
+def test_joblib_scatter(client, joblib):
     Parallel = joblib.Parallel
     delayed = joblib.delayed
 
@@ -116,22 +113,20 @@ def test_joblib_scatter(loop, joblib):
     y = CountSerialized(2)
     z = CountSerialized(3)
 
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as client:
-            with joblib.parallel_backend('dask', scatter=[x, y]) as (ba, _):
-                f = delayed(add5)
-                tasks = [f(x, y, z, d=4, e=5),
-                         f(x, z, y, d=5, e=4),
-                         f(y, x, z, d=x, e=5),
-                         f(z, z, x, d=z, e=y)]
-                sols = [func(*args, **kwargs) for func, args, kwargs in tasks]
-                results = Parallel()(tasks)
-
-            # Scatter must take a list/tuple
-            with pytest.raises(TypeError):
-                with joblib.parallel_backend('dask', loop=loop,
-                                             scatter=1):
-                    pass
+    with joblib.parallel_backend('dask', scatter=[x, y]) as (ba, _):
+        f = delayed(add5)
+        tasks = [f(x, y, z, d=4, e=5),
+                 f(x, z, y, d=5, e=4),
+                 f(y, x, z, d=x, e=5),
+                 f(z, z, x, d=z, e=y)]
+        sols = [func(*args, **kwargs) for func, args, kwargs in tasks]
+        results = Parallel()(tasks)
+
+    # Scatter must take a list/tuple
+    with pytest.raises(TypeError):
+        with joblib.parallel_backend('dask', loop=loop,
+                                     scatter=1):
+            pass
 
     for l, r in zip(sols, results):
         assert l == r
@@ -182,25 +177,23 @@ def test_errors(loop, joblib):
     assert "create a dask client" in str(info.value).lower()
 
 
-def test_correct_nested_backend(loop, joblib):
+def test_correct_nested_backend(client, joblib):
     if LooseVersion(joblib.__version__) <= LooseVersion("0.11.0"):
         pytest.skip("Requires nested parallelism")
 
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as client:
-            # No requirement, should be us
-            with joblib.parallel_backend('dask') as (ba, _):
-                result = joblib.Parallel(n_jobs=2)(joblib.delayed(outer)(
-                    joblib, nested_require=None) for _ in range(1))
-                assert isinstance(result[0][0][0],
-                                  distributed_joblib.DaskDistributedBackend)
+    # No requirement, should be us
+    with joblib.parallel_backend('dask') as (ba, _):
+        result = joblib.Parallel(n_jobs=2)(joblib.delayed(outer)(
+            joblib, nested_require=None) for _ in range(1))
+        assert isinstance(result[0][0][0],
+                          distributed_joblib.DaskDistributedBackend)
 
-            # Require threads, should be threading
-            with joblib.parallel_backend('dask') as (ba, _):
-                result = joblib.Parallel(n_jobs=2)(joblib.delayed(outer)(
-                    joblib, nested_require='sharedmem') for _ in range(1))
-                assert isinstance(result[0][0][0],
-                                  joblib.parallel.ThreadingBackend)
+    # Require threads, should be threading
+    with joblib.parallel_backend('dask') as (ba, _):
+        result = joblib.Parallel(n_jobs=2)(joblib.delayed(outer)(
+            joblib, nested_require='sharedmem') for _ in range(1))
+        assert isinstance(result[0][0][0],
+                          joblib.parallel.ThreadingBackend)
 
 
 def outer(joblib, nested_require):
@@ -232,16 +225,14 @@ def _test_keywords_f(_):
     return get_worker().address
 
 
-def test_keywords(loop, joblib):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as client:
-            with joblib.parallel_backend('dask', workers=a['address']) as (ba, _):
-                seq = joblib.Parallel()(joblib.delayed(_test_keywords_f)(i) for i in range(10))
-                assert seq == [a['address']] * 10
+def test_keywords(client, joblib, s, a, b):
+    with joblib.parallel_backend('dask', workers=a['address']) as (ba, _):
+        seq = joblib.Parallel()(joblib.delayed(_test_keywords_f)(i) for i in range(10))
+        assert seq == [a['address']] * 10
 
-            with joblib.parallel_backend('dask', workers=b['address']) as (ba, _):
-                seq = joblib.Parallel()(joblib.delayed(_test_keywords_f)(i) for i in range(10))
-                assert seq == [b['address']] * 10
+    with joblib.parallel_backend('dask', workers=b['address']) as (ba, _):
+        seq = joblib.Parallel()(joblib.delayed(_test_keywords_f)(i) for i in range(10))
+        assert seq == [b['address']] * 10
 
 
 def test_cleanup(loop, joblib):
diff --git a/distributed/tests/test_locks.py b/distributed/tests/test_locks.py
index 5abb8adee7a..5799396604e 100644
--- a/distributed/tests/test_locks.py
+++ b/distributed/tests/test_locks.py
@@ -5,10 +5,10 @@
 
 import pytest
 
-from distributed import Client, Lock, get_client
+from distributed import Lock, get_client
 from distributed.metrics import time
-from distributed.utils_test import gen_cluster, cluster
-from distributed.utils_test import loop  # noqa F401
+from distributed.utils_test import gen_cluster
+from distributed.utils_test import client, cluster_fixture, loop  # noqa F401
 
 
 @gen_cluster(client=True, ncores=[('127.0.0.1', 8)] * 2)
@@ -65,11 +65,9 @@ def test_acquires_with_zero_timeout(c, s, a, b):
     yield lock.release()
 
 
-def test_timeout_sync(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            with Lock('x') as lock:
-                assert Lock('x').acquire(timeout=0.1) is False
+def test_timeout_sync(client):
+    with Lock('x') as lock:
+        assert Lock('x').acquire(timeout=0.1) is False
 
 
 @gen_cluster(client=True)
@@ -79,7 +77,7 @@ def test_errors(c, s, a, b):
         yield lock.release()
 
 
-def test_lock_sync(loop):
+def test_lock_sync(client):
     def f(x):
         with Lock('x') as lock:
             client = get_client()
@@ -88,11 +86,10 @@ def f(x):
             sleep(0.05)
             assert client.get_metadata('locked') is True
             client.set_metadata('locked', False)
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            c.set_metadata('locked', False)
-            futures = c.map(f, range(10))
-            c.gather(futures)
+
+    client.set_metadata('locked', False)
+    futures = client.map(f, range(10))
+    client.gather(futures)
 
 
 @gen_cluster(client=True)
diff --git a/distributed/tests/test_preload.py b/distributed/tests/test_preload.py
index 1e93e4452dc..6239fdd8c0b 100644
--- a/distributed/tests/test_preload.py
+++ b/distributed/tests/test_preload.py
@@ -30,6 +30,7 @@ def check_worker():
         path = os.path.join(tmpdir, 'worker_info.py')
         with open(path, 'w') as f:
             f.write(PRELOAD_TEXT)
+
         with cluster(worker_kwargs={'preload': [path]}) as (s, workers), \
                 Client(s['address'], loop=loop) as c:
 
diff --git a/distributed/tests/test_publish.py b/distributed/tests/test_publish.py
index a67bfce9887..c6a899374c8 100644
--- a/distributed/tests/test_publish.py
+++ b/distributed/tests/test_publish.py
@@ -5,8 +5,8 @@
 from distributed import Client
 from distributed.client import futures_of
 from distributed.metrics import time
-from distributed.utils_test import gen_cluster, inc, cluster
-from distributed.utils_test import loop  # noqa F401
+from distributed.utils_test import gen_cluster, inc
+from distributed.utils_test import client, cluster_fixture, loop  # noqa F401
 from distributed.protocol import Serialized
 from tornado import gen
 
@@ -106,18 +106,16 @@ def test_unpublish(c, s, a, b):
     assert "data" in str(exc_info.value)
 
 
-def test_unpublish_sync(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address']) as c:
-            data = c.scatter([0, 1, 2])
-            c.publish_dataset(data=data)
-            c.unpublish_dataset(name='data')
+def test_unpublish_sync(client):
+    data = client.scatter([0, 1, 2])
+    client.publish_dataset(data=data)
+    client.unpublish_dataset(name='data')
 
-            with pytest.raises(KeyError) as exc_info:
-                result = c.get_dataset(name='data')
+    with pytest.raises(KeyError) as exc_info:
+        result = client.get_dataset(name='data')
 
-            assert "not found" in str(exc_info.value)
-            assert "data" in str(exc_info.value)
+    assert "not found" in str(exc_info.value)
+    assert "data" in str(exc_info.value)
 
 
 @gen_cluster(client=True)
@@ -130,30 +128,28 @@ def test_publish_multiple_datasets(c, s, a, b):
     assert set(datasets) == {'x', 'y'}
 
 
-def test_unpublish_multiple_datasets_sync(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address']) as c:
-            x = delayed(inc)(1)
-            y = delayed(inc)(2)
-            c.publish_dataset(x=x, y=y)
-            c.unpublish_dataset(name='x')
+def test_unpublish_multiple_datasets_sync(client):
+    x = delayed(inc)(1)
+    y = delayed(inc)(2)
+    client.publish_dataset(x=x, y=y)
+    client.unpublish_dataset(name='x')
 
-            with pytest.raises(KeyError) as exc_info:
-                result = c.get_dataset(name='x')
+    with pytest.raises(KeyError) as exc_info:
+        result = client.get_dataset(name='x')
 
-            datasets = c.list_datasets()
-            assert set(datasets) == {'y'}
+    datasets = client.list_datasets()
+    assert set(datasets) == {'y'}
 
-            assert "not found" in str(exc_info.value)
-            assert "x" in str(exc_info.value)
+    assert "not found" in str(exc_info.value)
+    assert "x" in str(exc_info.value)
 
-            c.unpublish_dataset(name='y')
+    client.unpublish_dataset(name='y')
 
-            with pytest.raises(KeyError) as exc_info:
-                result = c.get_dataset(name='y')
+    with pytest.raises(KeyError) as exc_info:
+        result = client.get_dataset(name='y')
 
-            assert "not found" in str(exc_info.value)
-            assert "y" in str(exc_info.value)
+    assert "not found" in str(exc_info.value)
+    assert "y" in str(exc_info.value)
 
 
 @gen_cluster(client=False)
@@ -184,57 +180,45 @@ def test_publish_bag(s, a, b):
     yield f.close()
 
 
-def test_datasets_setitem(loop):
-    with cluster() as (s, _):
-        with Client(s['address'], loop=loop) as client:
-            for key in ['key', ('key', 'key'), 1]:
-                value = 'value'
-                client.datasets[key] = value
-                assert client.get_dataset(key) == value
-
-
-def test_datasets_getitem(loop):
-    with cluster() as (s, _):
-        with Client(s['address'], loop=loop) as client:
-            for key in ['key', ('key', 'key'), 1]:
-                value = 'value'
-                client.publish_dataset(value, name=key)
-                assert client.datasets[key] == value
-
-
-def test_datasets_delitem(loop):
-    with cluster() as (s, _):
-        with Client(s['address'], loop=loop) as client:
-            for key in ['key', ('key', 'key'), 1]:
-                value = 'value'
-                client.publish_dataset(value, name=key)
-                del client.datasets[key]
-                assert key not in client.list_datasets()
-
-
-def test_datasets_keys(loop):
-    with cluster() as (s, _):
-        with Client(s['address'], loop=loop) as client:
-            client.publish_dataset(**{str(n): n for n in range(10)})
-            keys = list(client.datasets.keys())
-            assert keys == [str(n) for n in range(10)]
-
-
-def test_datasets_contains(loop):
-    with cluster() as (s, _):
-        with Client(s['address'], loop=loop) as client:
-            key, value = 'key', 'value'
-            client.publish_dataset(key=value)
-            assert key in client.datasets
-
-
-def test_datasets_iter(loop):
-    with cluster() as (s, _):
-        with Client(s['address'], loop=loop) as client:
-            keys = [n for n in range(10)]
-            client.publish_dataset(**{str(key): key for key in keys})
-            for n, key in enumerate(client.datasets):
-                assert key == str(n)
+def test_datasets_setitem(client):
+    for key in ['key', ('key', 'key'), 1]:
+        value = 'value'
+        client.datasets[key] = value
+        assert client.get_dataset(key) == value
+
+
+def test_datasets_getitem(client):
+    for key in ['key', ('key', 'key'), 1]:
+        value = 'value'
+        client.publish_dataset(value, name=key)
+        assert client.datasets[key] == value
+
+
+def test_datasets_delitem(client):
+    for key in ['key', ('key', 'key'), 1]:
+        value = 'value'
+        client.publish_dataset(value, name=key)
+        del client.datasets[key]
+        assert key not in client.list_datasets()
+
+
+def test_datasets_keys(client):
+    client.publish_dataset(**{str(n): n for n in range(10)})
+    keys = list(client.datasets.keys())
+    assert keys == [str(n) for n in range(10)]
+
+
+def test_datasets_contains(client):
+    key, value = 'key', 'value'
+    client.publish_dataset(key=value)
+    assert key in client.datasets
+
+
+def test_datasets_iter(client):
+    keys = [n for n in range(10)]
+    client.publish_dataset(**{str(key): key for key in keys})
+    for n, key in enumerate(client.datasets):
+        assert key == str(n)
 
 
 @gen_cluster(client=True)
diff --git a/distributed/tests/test_queues.py b/distributed/tests/test_queues.py
index faa8707cdf0..6d6306ca6f9 100644
--- a/distributed/tests/test_queues.py
+++ b/distributed/tests/test_queues.py
@@ -9,8 +9,8 @@
 
 from distributed import Client, Queue, Nanny, worker_client, wait
 from distributed.metrics import time
-from distributed.utils_test import (gen_cluster, inc, cluster, slow, div)
-from distributed.utils_test import loop # noqa: F401
+from distributed.utils_test import (gen_cluster, inc, slow, div)
+from distributed.utils_test import client, cluster_fixture, loop # noqa: F401
 
 
 @gen_cluster(client=True)
@@ -57,18 +57,16 @@ def test_queue_with_data(c, s, a, b):
         yield x.get(timeout=0.1)
 
 
-def test_sync(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address']) as c:
-            future = c.submit(lambda x: x + 1, 10)
-            x = Queue('x')
-            xx = Queue('x')
-            x.put(future)
-            assert x.qsize() == 1
-            assert xx.qsize() == 1
-            future2 = xx.get()
+def test_sync(client):
+    future = client.submit(lambda x: x + 1, 10)
+    x = Queue('x')
+    xx = Queue('x')
+    x.put(future)
+    assert x.qsize() == 1
+    assert xx.qsize() == 1
+    future2 = xx.get()
 
-            assert future2.result() == 11
+    assert future2.result() == 11
 
 
 @gen_cluster()
@@ -104,17 +102,15 @@ def f(x):
     assert result == 11
 
 
-def test_picklability_sync(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address']) as c:
-            q = Queue()
+def test_picklability_sync(client):
+    q = Queue()
 
-            def f(x):
-                q.put(x + 1)
+    def f(x):
+        q.put(x + 1)
 
-            c.submit(f, 10).result()
+    client.submit(f, 10).result()
 
-            assert q.get() == 11
+    assert q.get() == 11
 
 
 @pytest.mark.skipif(sys.version_info[0] == 2, reason='Multi-client issues')
diff --git a/distributed/tests/test_resources.py b/distributed/tests/test_resources.py
index 46e22d0f530..2d5b1dab8b7 100644
--- a/distributed/tests/test_resources.py
+++ b/distributed/tests/test_resources.py
@@ -6,12 +6,11 @@
 import pytest
 from tornado import gen
 
-from distributed import Worker, Client
+from distributed import Worker
 from distributed.client import wait
 from distributed.utils import tokey
-from distributed.utils_test import (inc, gen_cluster, cluster,
-                                    slowinc, slowadd)
-from distributed.utils_test import loop # noqa: F401
+from distributed.utils_test import (inc, gen_cluster, slowinc, slowadd)
+from distributed.utils_test import client, cluster_fixture, loop, s, a, b  # noqa: F401
 
 
 @gen_cluster(client=True, ncores=[])
@@ -279,23 +278,22 @@ def test_full_collections(c, s, a, b):
     pytest.mark.xfail(True, reason="don't track resources through optimization"),
     False
 ])
-def test_collections_get(loop, optimize_graph):
+def test_collections_get(client, optimize_graph, s, a, b):
     da = pytest.importorskip('dask.array')
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            def f(dask_worker):
-                dask_worker.set_resources(**{'A': 1})
 
-            c.run(f, workers=[a['address']])
+    def f(dask_worker):
+        dask_worker.set_resources(**{'A': 1})
 
-            x = da.random.random(100, chunks=(10,)) + 1
+    client.run(f, workers=[a['address']])
 
-            x.compute(resources={tuple(x.dask): {'A': 1}},
-                      optimize_graph=optimize_graph)
+    x = da.random.random(100, chunks=(10,)) + 1
 
-            def g(dask_worker):
-                return len(dask_worker.log)
+    x.compute(resources={tuple(x.dask): {'A': 1}},
+              optimize_graph=optimize_graph)
 
-            logs = c.run(g)
-            assert logs[a['address']]
-            assert not logs[b['address']]
+    def g(dask_worker):
+        return len(dask_worker.log)
+
+    logs = client.run(g)
+    assert logs[a['address']]
+    assert not logs[b['address']]
diff --git a/distributed/tests/test_variable.py b/distributed/tests/test_variable.py
index 6debaa80719..0d0898923c4 100644
--- a/distributed/tests/test_variable.py
+++ b/distributed/tests/test_variable.py
@@ -9,8 +9,8 @@
 
 from distributed import Client, Variable, worker_client, Nanny, wait
 from distributed.metrics import time
-from distributed.utils_test import (gen_cluster, inc, cluster, slow, div)
-from distributed.utils_test import loop # noqa: F401
+from distributed.utils_test import (gen_cluster, inc, slow, div)
+from distributed.utils_test import client, cluster_fixture, loop # noqa: F401
 
 
 @gen_cluster(client=True)
@@ -50,16 +50,14 @@ def test_queue_with_data(c, s, a, b):
     assert data == (1, 'hello')
 
 
-def test_sync(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address']) as c:
-            future = c.submit(lambda x: x + 1, 10)
-            x = Variable('x')
-            xx = Variable('x')
-            x.set(future)
-            future2 = xx.get()
+def test_sync(client):
+    future = client.submit(lambda x: x + 1, 10)
+    x = Variable('x')
+    xx = Variable('x')
+    x.set(future)
+    future2 = xx.get()
 
-            assert future2.result() == 11
+    assert future2.result() == 11
 
 
 @gen_cluster()
@@ -93,15 +91,13 @@ def test_timeout(c, s, a, b):
     assert 0.1 < stop - start < 2.0
 
 
-def test_timeout_sync(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address']) as c:
-            v = Variable('v')
-            start = time()
-            with pytest.raises(gen.TimeoutError):
-                v.get(timeout=0.1)
-            stop = time()
-            assert 0.1 < stop - start < 2.0
+def test_timeout_sync(client):
+    v = Variable('v')
+    start = time()
+    with pytest.raises(gen.TimeoutError):
+        v.get(timeout=0.1)
+    stop = time()
+    assert 0.1 < stop - start < 2.0
 
 
 @gen_cluster(client=True)
@@ -127,16 +123,14 @@ def test_cleanup(c, s, a, b):
     assert result == 11
 
 
-def test_pickleable(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address']) as c:
-            v = Variable('v')
+def test_pickleable(client):
+    v = Variable('v')
 
-            def f(x):
-                v.set(x + 1)
+    def f(x):
+        v.set(x + 1)
 
-            c.submit(f, 10).result()
-            assert v.get() == 11
+    client.submit(f, 10).result()
+    assert v.get() == 11
 
 
 @gen_cluster(client=True)
@@ -233,16 +227,14 @@ def test_erred_future(c, s, a, b):
     assert isinstance(exc, ZeroDivisionError)
 
 
-def test_future_erred_sync(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address']) as c:
-            future = c.submit(div, 1, 0)
-            var = Variable()
-            var.set(future)
+def test_future_erred_sync(client):
+    future = client.submit(div, 1, 0)
+    var = Variable()
+    var.set(future)
 
-            sleep(0.1)
+    sleep(0.1)
 
-            future2 = var.get()
+    future2 = var.get()
 
-            with pytest.raises(ZeroDivisionError):
-                future2.result()
+    with pytest.raises(ZeroDivisionError):
+        future2.result()
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index f85638d3656..2ee1a988b7b 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -19,7 +19,7 @@
 from tornado import gen
 from tornado.ioloop import TimeoutError
 
-from distributed import (Nanny, Client, get_client, wait, default_client,
+from distributed import (Nanny, get_client, wait, default_client,
         get_worker, Reschedule)
 from distributed.compatibility import WINDOWS, cache_from_source
 from distributed.core import rpc
@@ -29,9 +29,8 @@
 from distributed.worker import Worker, error_message, logger, TOTAL_MEMORY
 from distributed.utils import tmpfile, format_bytes
 from distributed.utils_test import (inc, mul, gen_cluster, div, dec,
-                                    slow, slowinc, gen_test, cluster,
-                                    captured_logger)
-from distributed.utils_test import loop, nodebug # noqa: F401
+                                    slow, slowinc, gen_test, captured_logger)
+from distributed.utils_test import client, loop, nodebug, cluster_fixture, s, a, b  # noqa: F401
 
 
 def test_worker_ncores():
@@ -729,9 +728,11 @@ def test_priorities(c, s, w):
 @gen_cluster(client=True)
 def test_heartbeats(c, s, a, b):
     x = s.workers[a.address].last_seen
+    start = time()
     yield gen.sleep(a.periodic_callbacks['heartbeat'].callback_time / 1000 + 0.1)
-    y = s.workers[a.address].last_seen
-    assert x != y
+    while s.workers[a.address].last_seen == x:
+        yield gen.sleep(0.01)
+        assert time() < start + 2
     assert a.periodic_callbacks['heartbeat'].callback_time < 1000
 
 
@@ -846,16 +847,14 @@ def f(x):
     assert a._client is a_client
 
 
-def test_get_client_sync(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            def f(x):
-                cc = get_client()
-                future = cc.submit(inc, x)
-                return future.result()
+def test_get_client_sync(client):
+    def f(x):
+        cc = get_client()
+        future = cc.submit(inc, x)
+        return future.result()
 
-            future = c.submit(f, 10)
-            assert future.result() == 11
+    future = client.submit(f, 10)
+    assert future.result() == 11
 
 
 @gen_cluster(client=True)
@@ -872,19 +871,17 @@ def f():
                        b.address: 11}
 
 
-def test_get_client_coroutine_sync(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            @gen.coroutine
-            def f():
-                client = yield get_client()
-                future = client.submit(inc, 10)
-                result = yield future
-                raise gen.Return(result)
+def test_get_client_coroutine_sync(client, s, a, b):
+    @gen.coroutine
+    def f():
+        client = yield get_client()
+        future = client.submit(inc, 10)
+        result = yield future
+        raise gen.Return(result)
 
-            results = c.run_coroutine(f)
-            assert results == {a['address']: 11,
-                               b['address']: 11}
+    results = client.run_coroutine(f)
+    assert results == {a['address']: 11,
+                       b['address']: 11}
 
 
 @gen_cluster()
@@ -1116,21 +1113,19 @@ def test_dict_data_if_no_spill_to_disk(s, w):
     assert type(w.data) is dict
 
 
-def test_get_worker_name(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            def f():
-                get_client().submit(inc, 1).result()
+def test_get_worker_name(client):
+    def f():
+        get_client().submit(inc, 1).result()
 
-            c.run(f)
+    client.run(f)
 
-            def func(dask_scheduler):
-                return list(dask_scheduler.clients)
+    def func(dask_scheduler):
+        return list(dask_scheduler.clients)
 
-            start = time()
-            while not any('worker' in n for n in c.run_on_scheduler(func)):
-                sleep(0.1)
-                assert time() < start + 10
+    start = time()
+    while not any('worker' in n for n in client.run_on_scheduler(func)):
+        sleep(0.1)
+        assert time() < start + 10
 
 
 @gen_cluster(ncores=[('127.0.0.1', 1)],
diff --git a/distributed/tests/test_worker_client.py b/distributed/tests/test_worker_client.py
index 2b96ae59c4f..b5d96edb238 100644
--- a/distributed/tests/test_worker_client.py
+++ b/distributed/tests/test_worker_client.py
@@ -13,8 +13,8 @@
 from distributed import (worker_client, Client, as_completed, get_worker, wait,
                          get_client)
 from distributed.metrics import time
-from distributed.utils_test import cluster, double, gen_cluster, inc
-from distributed.utils_test import loop # noqa: F401
+from distributed.utils_test import double, gen_cluster, inc
+from distributed.utils_test import client, cluster_fixture, loop  # noqa: F401
 
 
 @gen_cluster(client=True)
@@ -122,7 +122,7 @@ def f():
     assert result
 
 
-def test_sync(loop):
+def test_sync(client):
     def mysum():
         result = 0
         sub_tasks = [delayed(double)(i) for i in range(100)]
@@ -133,9 +133,7 @@ def mysum():
                 result += f.result()
         return result
 
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            assert delayed(mysum)().compute() == 9900
+    assert delayed(mysum)().compute() == 9900
 
 
 @gen_cluster(client=True)
@@ -235,16 +233,14 @@ def func(x):
     assert c.status == 'running'
 
 
-def test_timeout(loop):
+def test_timeout(client):
     def func():
         with worker_client(timeout=0) as wc:
             print('hello')
 
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            future = c.submit(func)
-            with pytest.raises(EnvironmentError):
-                result = future.result()
+    future = client.submit(func)
+    with pytest.raises(EnvironmentError):
+        result = future.result()
 
 
 def test_secede_without_stealing_issue_1262():
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index b250f808e5d..3ef7aa189d9 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -39,7 +39,7 @@
 from tornado.gen import TimeoutError
 from tornado.ioloop import IOLoop
 
-from .client import default_client, _global_clients
+from .client import default_client, _global_clients, Client
 from .compatibility import PY3, Empty, WINDOWS, PY2
 from .comm.utils import offload
 from .config import initialize_logging
@@ -534,6 +534,44 @@ def wait():
     loop.run_sync(wait)
 
 
+@pytest.fixture
+def cluster_fixture(loop):
+    with cluster() as (scheduler, workers):
+        yield (scheduler, workers)
+
+
+@pytest.fixture
+def s(cluster_fixture):
+    scheduler, workers = cluster_fixture
+    return scheduler
+
+
+@pytest.fixture
+def a(cluster_fixture):
+    scheduler, workers = cluster_fixture
+    return workers[0]
+
+
+@pytest.fixture
+def b(cluster_fixture):
+    scheduler, workers = cluster_fixture
+    return workers[1]
+
+
+@pytest.fixture
+def client(loop, cluster_fixture):
+    scheduler, workers = cluster_fixture
+    with Client(scheduler['address'], loop=loop) as client:
+        yield client
+
+
+@pytest.fixture
+def client_secondary(loop, cluster_fixture):
+    scheduler, workers = cluster_fixture
+    with Client(scheduler['address'], loop=loop) as client:
+        yield client
+
+
 @contextmanager
 def cluster(nworkers=2, nanny=False, worker_kwargs={}, active_rpc_timeout=1,
             scheduler_kwargs={}):
diff --git a/docs/source/develop.rst b/docs/source/develop.rst
index 70f91b1afa9..78225c6d36f 100644
--- a/docs/source/develop.rst
+++ b/docs/source/develop.rst
@@ -71,7 +71,7 @@ The test suite contains three kinds of tests
     same event loop in the main thread.  These are good for testing complex
     logic and inspecting the state of the system directly.  They are also
     easier to debug and cause the fewest problems with shutdowns.
-2.  ``with cluster()``: Tests with multiple processes forked from the master
+2.  ``def test_foo(client)``: Tests with multiple processes forked from the master
     process.  These are good for testing the synchronous (normal user) API and
     when triggering hard failures for resilience tests.
 3.  ``popen``: Tests that call out to the command line to start the system.
@@ -82,6 +82,8 @@ using the ``@gen_cluster`` style of test
 
 .. code-block:: python
 
+   from distributed.utils_test import gen_cluster
+
    @gen_cluster(client=True)
    def test_submit(c, s, a, b):
        assert isinstance(c, Client)
@@ -107,25 +109,34 @@ instead you need to use the coroutine API, where all blocking functions are
 prepended with an underscore (``_``).  Beware, it is a common mistake to use
 the blocking interface within these tests.
 
-If you want to test the normal synchronous API you can use a ``with cluster``
-style test, which sets up a scheduler and workers for you in different forked
-processes:
+If you want to test the normal synchronous API you can use the ``client``
+pytest fixture style test, which sets up a scheduler and workers for you in
+different forked processes:
+
+.. code-block:: python
+
+   from distributed.utils_test import client
+
+   def test_submit(client):
+       future = client.submit(inc, 10)
+       assert future.result() == 11
+
+Additionally, if you want access to the scheduler and worker processes you can
+also add the ``s, a, b`` fixtures as well.
+
 
 .. code-block:: python
 
-   def test_submit_sync(loop):
-       with cluster() as (s, [a, b]):
-           with Client(s['address'], loop=loop) as c:
-               future = c.submit(inc, 1)
-               assert future.key in c.futures
+   from distributed.utils_test import client
 
-               result = future.result()  # use the synchronous/blocking API here
-               assert result == 2
+   def test_submit(client, s, a, b):
+       future = client.submit(inc, 10)
+       assert future.result() == 11  # use the synchronous/blocking API here
 
-               a['proc'].terminate()  # kill one of the workers
+       a['proc'].terminate()  # kill one of the workers
 
-               result = future.result()  # test that future remains valid
-               assert result == 2
+       result = future.result()  # test that future remains valid
+       assert result == 2
 
 In this style of test you do not have access to the scheduler or workers.  The
 variables ``s, a, b`` are now dictionaries holding a
@@ -134,8 +145,8 @@ use the normal synchronous API (never use yield in this style of test) and you
 can close processes easily by terminating them.
 
 Typically for most user-facing functions you will find both kinds of tests.
-The ``@gen_cluster`` tests test particular logic while the ``with cluster``
-tests test basic interface and resilience.
+The ``@gen_cluster`` tests test particular logic while the ``client`` pytest
+fixture tests test basic interface and resilience.
 
 You should avoid ``popen`` style tests unless absolutely necessary, such as if
 you need to test the command line interface.

From acc4b9076e034574533a960323e0d774b4f15e1e Mon Sep 17 00:00:00 2001
From: Brett Naul <brettnaul@gmail.com>
Date: Tue, 25 Sep 2018 08:47:22 -0700
Subject: [PATCH 0101/1550] Fix bokeh error for `memory_limit=None` (#2255)

* Fix bokeh error for `memory_limit=None`

Diagnostics page currently breaks if `memory_limit` is set to `None`.
```
distributed.utils - ERROR - '>' not supported between instances of 'NoneType' and 'int'
Traceback (most recent call last):
  File "/Users/brettnaul/venvs/model37/lib/python3.7/site-packages/distributed/utils.py", line 646, in log_errors
    yield
  File "/Users/brettnaul/venvs/model37/lib/python3.7/site-packages/distributed/bokeh/scheduler.py", line 337, in update
    if limit > max_limit:
TypeError: '>' not supported between instances of 'NoneType' and 'int'
```
Maybe there's a more elegant way to do this? I'm not sure how to check if the `memory_limit` attr exists, I assume that's why it's currently done using a try/except

* Update scheduler.py

* support python 2
---
 distributed/bokeh/scheduler.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/distributed/bokeh/scheduler.py b/distributed/bokeh/scheduler.py
index a39eef848a2..bfec2418a9f 100644
--- a/distributed/bokeh/scheduler.py
+++ b/distributed/bokeh/scheduler.py
@@ -4,7 +4,6 @@
 from functools import partial
 import logging
 import math
-from math import sqrt
 from numbers import Number
 from operator import add
 import os
@@ -59,6 +58,7 @@
 BOKEH_THEME = Theme(os.path.join(os.path.dirname(__file__), 'theme.yaml'))
 
 nan = float('nan')
+inf = float('inf')
 
 
 def update(source, data):
@@ -331,10 +331,8 @@ def update(self):
             nbytes_color = []
             max_limit = 0
             for ws, nb in zip(workers, nbytes):
-                try:
-                    limit = self.scheduler.workers[ws.address].memory_limit
-                except KeyError:
-                    limit = 16e9
+                limit = getattr(self.scheduler.workers[ws.address], 'memory_limit', inf) or inf
+
                 if limit > max_limit:
                     max_limit = limit
 
@@ -445,7 +443,7 @@ def convert(self, msgs):
         except (KeyError, IndexError):
             color = 'black'
 
-        radius = sqrt(min(total_duration, 10)) * 30 + 2
+        radius = math.sqrt(min(total_duration, 10)) * 30 + 2
 
         d = {'time': time * 1000, 'level': level, 'count': len(msgs),
              'color': color, 'duration': total_duration, 'radius': radius,

From b092bc38bcccf1ab890776d68cfd10904d27ec0c Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 2 Oct 2018 10:15:51 -0400
Subject: [PATCH 0102/1550] Revert reference cycle fix in bokeh plots (#2278)

* Revert reference cycle fix in bokeh plots

in https://github.com/dask/distributed/pull/2261 we used weakrefs to avoid
reference cycles that kept around Bokeh documents forever, causing increased
load from plugins.

Unfortunately we needed some reference from the Doc to the Plot Component,
otherwise the component would disappear.  We add that back in, but
unfortunately this causes a reference cycle.  Attempts to avoid this reference
cycle have proved difficult.

* pip install pip ugrade

* Protect exception in intermittent testing failure

This was failing somewhat frequently in CI.
Lets add a check to give us more information when it does.
---
 .../setup_conda_environment.cmd               |  1 +
 distributed/bokeh/components.py               | 22 ++++++++++++++-----
 distributed/utils.py                          |  2 ++
 3 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/continuous_integration/setup_conda_environment.cmd b/continuous_integration/setup_conda_environment.cmd
index d8fc2445d27..af8c71728e7 100644
--- a/continuous_integration/setup_conda_environment.cmd
+++ b/continuous_integration/setup_conda_environment.cmd
@@ -44,6 +44,7 @@ call deactivate
 call activate %CONDA_ENV%
 
 %CONDA% uninstall -q -y --force dask joblib zict
+%PIP_INSTALL% pip --upgrade
 %PIP_INSTALL% git+https://github.com/dask/dask --upgrade
 %PIP_INSTALL% git+https://github.com/joblib/joblib.git --upgrade
 %PIP_INSTALL% git+https://github.com/dask/zict --upgrade
diff --git a/distributed/bokeh/components.py b/distributed/bokeh/components.py
index 8cca28af7ff..a5c4a831b49 100644
--- a/distributed/bokeh/components.py
+++ b/distributed/bokeh/components.py
@@ -681,12 +681,24 @@ def add_periodic_callback(doc, component, interval):
     the component stays in memory as a reference cycle because its method is
     still around.  This way we avoid that and let things clean up a bit more
     nicely.
+
+    TODO: we still have reference cycles.  Docs seem to be referred to by their
+    add_periodic_callback methods.
     """
     ref = weakref.ref(component)
 
-    def update():
-        component = ref()
-        if component is not None:
-            component.update()
+    doc.add_periodic_callback(lambda: update(ref), interval)
+    _attach(doc, component)
+
+
+def update(ref):
+    comp = ref()
+    if comp is not None:
+        comp.update()
+
+
+def _attach(doc, component):
+    if not hasattr(doc, 'components'):
+        doc.components = set()
 
-    doc.add_periodic_callback(update, interval)
+    doc.components.add(component)
diff --git a/distributed/utils.py b/distributed/utils.py
index 3bebc65b24f..430bb7163d1 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -370,6 +370,8 @@ def run_loop(loop=self._loop):
             # Loop already running in other thread (user-launched)
             done_evt.wait(5)
             if not isinstance(start_exc[0], RuntimeError):
+                if not isinstance(start_exc[0]):  # track down infrequent error
+                    raise TypeError("not an exception", start_exc[0])
                 raise start_exc[0]
             self._all_loops[self._loop] = count + 1, None
         else:

From 169a8d966d6ff42a9440b4f43a65f56cd69ad221 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 2 Oct 2018 11:30:33 -0400
Subject: [PATCH 0103/1550] Place large keywords into task graph in Client.map
 (#2281)

Previously these would be included in every task, resulting in excessive
transmission of serialized data.
---
 distributed/client.py            | 16 ++++++++++++++--
 distributed/tests/test_client.py | 12 ++++++++++++
 2 files changed, 26 insertions(+), 2 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 69693f3a46e..f9cf7f94985 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -1296,6 +1296,9 @@ def map(self, func, *iterables, **kwargs):
             Higher priorities take precedence
         fifo_timeout: str timedelta (default '100ms')
             Allowed amount of time between calls to consider the same priority
+        **kwargs: dict
+            Extra keywords to send to the function.
+            Large values will be included explicitly in the task graph.
 
         Examples
         --------
@@ -1358,8 +1361,17 @@ def map(self, func, *iterables, **kwargs):
             dsk = {key: (func,) + args
                    for key, args in zip(keys, zip(*iterables))}
         else:
-            dsk = {key: (apply, func, (tuple, list(args)), kwargs)
-                   for key, args in zip(keys, zip(*iterables))}
+            kwargs2 = {}
+            dsk = {}
+            for k, v in kwargs.items():
+                if sizeof(v) > 1e5:
+                    vv = dask.delayed(v)
+                    kwargs2[k] = vv._key
+                    dsk.update(vv.dask)
+                else:
+                    kwargs2[k] = v
+            dsk.update({key: (apply, func, (tuple, list(args)), kwargs2)
+                        for key, args in zip(keys, zip(*iterables))})
 
         if isinstance(workers, six.string_types + (Number,)):
             workers = [workers]
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index d2f0a18bdca..6b2749478d1 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -5369,5 +5369,17 @@ def test_tuple_keys(c, s, a, b):
     assert (yield future) == 3
 
 
+@gen_cluster(client=True)
+def test_map_large_kwargs_in_graph(c, s, a, b):
+    np = pytest.importorskip('numpy')
+    x = np.random.random(100000)
+    futures = c.map(lambda a, b: a + b, range(100), b=x)
+    while not s.tasks:
+        yield gen.sleep(0.01)
+
+    assert len(s.tasks) == 101
+    assert any(k.startswith('ndarray') for k in s.tasks)
+
+
 if sys.version_info >= (3, 5):
     from distributed.tests.py3_test_client import *  # noqa F401

From cd1b7530fd07dfcb3d2113fe9ba1589f86390271 Mon Sep 17 00:00:00 2001
From: MikeG <michael.gevaert@epfl.ch>
Date: Wed, 3 Oct 2018 23:09:53 +0200
Subject: [PATCH 0104/1550] remove blosc threading from protocol.numpy (#2284)

* remove blosc threading from protocol.numpy

 * add importskip for "test_compression_takes_advantage_of_itemsize" if
 lz4 is missing
---
 distributed/protocol/numpy.py            | 6 ------
 distributed/protocol/tests/test_numpy.py | 1 +
 2 files changed, 1 insertion(+), 6 deletions(-)

diff --git a/distributed/protocol/numpy.py b/distributed/protocol/numpy.py
index d6fc52a4e4d..5b18e8ebcb7 100644
--- a/distributed/protocol/numpy.py
+++ b/distributed/protocol/numpy.py
@@ -4,12 +4,6 @@
 
 import numpy as np
 
-try:
-    import blosc
-    n = blosc.set_nthreads(2)
-except ImportError:
-    blosc = False
-
 from .utils import frame_split_size, merge_frames
 from .serialize import dask_serialize, dask_deserialize
 from . import pickle
diff --git a/distributed/protocol/tests/test_numpy.py b/distributed/protocol/tests/test_numpy.py
index 6718806fa13..89c441f34db 100644
--- a/distributed/protocol/tests/test_numpy.py
+++ b/distributed/protocol/tests/test_numpy.py
@@ -192,6 +192,7 @@ def test_dumps_large_blosc(c, s, a, b):
 @pytest.mark.skipif(sys.version_info[0] < 3,
                     reason='numpy doesnt use memoryviews')
 def test_compression_takes_advantage_of_itemsize():
+    pytest.importorskip('lz4')
     blosc = pytest.importorskip('blosc')
     x = np.arange(1000000, dtype='i8')
 

From ee582a0ab21241f207897cac8ab3adaa41f23ab0 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 5 Oct 2018 07:39:39 -0400
Subject: [PATCH 0105/1550] dask.pydata.org -> dask.org

---
 distributed/bokeh/templates/base.html |  2 +-
 docs/source/client.rst                | 10 +++++-----
 docs/source/develop.rst               |  4 ++--
 docs/source/efficiency.rst            |  2 +-
 docs/source/index.rst                 |  6 +++---
 docs/source/related-work.rst          |  4 ++--
 docs/source/setup.rst                 |  2 +-
 docs/source/task-launch.rst           |  2 +-
 8 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/distributed/bokeh/templates/base.html b/distributed/bokeh/templates/base.html
index a3a01ef3314..da15df28b69 100644
--- a/distributed/bokeh/templates/base.html
+++ b/distributed/bokeh/templates/base.html
@@ -20,7 +20,7 @@
     <div class="navbar" id="myTopnav">
       <ul>
         <li id="dask-logo">
-          <a href="http://dask.pydata.org/en/latest/">
+          <a href="https://dask.org/">
             <img src="statics/images/dask-logo.svg"></img>
           </a>
         </li>
diff --git a/docs/source/client.rst b/docs/source/client.rst
index 3c7f46846ec..b2b520d9614 100644
--- a/docs/source/client.rst
+++ b/docs/source/client.rst
@@ -196,8 +196,8 @@ following pages:
 
 .. _concurrent.futures:  https://docs.python.org/3/library/concurrent.futures.html
 .. _PEP-3148: https://www.python.org/dev/peps/pep-3148/
-.. _dask.array: http://dask.pydata.org/en/latest/array.html
-.. _dask.bag: http://dask.pydata.org/en/latest/bag.html
-.. _dask.dataframe: http://dask.pydata.org/en/latest/dataframe.html
-.. _dask.delayed: http://dask.pydata.org/en/latest/delayed.html
-.. _Dask: http://dask.pydata.org/en/latest/
+.. _dask.array: https://docs.dask.org/en/latest/array.html
+.. _dask.bag: https://docs.dask.org/en/latest/bag.html
+.. _dask.dataframe: https://docs.dask.org/en/latest/dataframe.html
+.. _dask.delayed: https://docs.dask.org/en/latest/delayed.html
+.. _Dask: https://dask.org
diff --git a/docs/source/develop.rst b/docs/source/develop.rst
index 78225c6d36f..9b3b70afb15 100644
--- a/docs/source/develop.rst
+++ b/docs/source/develop.rst
@@ -6,8 +6,8 @@ including where to ask for help, a layout of repositories, testing practices,
 and documentation and style standards are available at the `Dask developer
 guidelines`_ in the main documentation.
 
-.. _Dask: http://dask.pydata.org/en/latest/
-.. _`Dask developer guidelines`: http://dask.pydata.org/en/latest/develop.html
+.. _Dask: http://dask.org
+.. _`Dask developer guidelines`: http://docs.dask.org/en/latest/develop.html
 
 Install
 -------
diff --git a/docs/source/efficiency.rst b/docs/source/efficiency.rst
index a69d02d8e51..39ca16d93d5 100644
--- a/docs/source/efficiency.rst
+++ b/docs/source/efficiency.rst
@@ -105,5 +105,5 @@ C/Fortran/Numba code might be enough to give you the 10x speed boost that
 you're looking for.  Parallelism and distributed computing are expensive ways
 to accelerate your application.
 
-.. _dask: http://dask.pydata.org/en/latest/
+.. _dask: https://dask.org
 .. _concurrent.futures: https://docs.python.org/3/library/concurrent.futures.html
diff --git a/docs/source/index.rst b/docs/source/index.rst
index 211e94a4c3b..5b821f3115c 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -31,7 +31,7 @@ In particular it meets the following needs:
     and easy to :doc:`set up <setup>` on your own cluster.
 
 .. _`concurrent.futures`: https://www.python.org/dev/peps/pep-3148/
-.. _`dask`: http://dask.pydata.org/en/latest/
+.. _`dask`: https://dask.org
 
 Architecture
 ------------
@@ -65,8 +65,8 @@ and Pandas, while the simple ``client.submit`` interface provides users with
 custom control when they want to break out of canned "big data" abstractions
 and submit fully custom workloads.
 
-.. _dask.array: http://dask.pydata.org/en/latest/array.html
-.. _dask.dataframe: http://dask.pydata.org/en/latest/dataframe.html
+.. _dask.array: https://dask.docs.org/en/latest/array.html
+.. _dask.dataframe: https://dask.docs.org/en/latest/dataframe.html
 
 Contents
 --------
diff --git a/docs/source/related-work.rst b/docs/source/related-work.rst
index e2b13e458d2..4de458bad66 100644
--- a/docs/source/related-work.rst
+++ b/docs/source/related-work.rst
@@ -7,7 +7,7 @@ down below.
 
 You may also find the `dask comparison with spark`_ of interest.
 
-.. _`dask comparison with spark`: http://dask.pydata.org/en/latest/spark.html
+.. _`dask comparison with spark`: http://docs.dask.org/en/latest/spark.html
 
 
 Big Data World
@@ -191,7 +191,7 @@ IPython Parallel has the following advantages over ``distributed``
     decorators, etc..
 
 .. _`a recipe`: https://ipython.org/ipython-doc/3/parallel/dag_dependencies.html#dag-dependencies
-.. _dask: http://dask.pydata.org/en/latest/
+.. _dask: https://dask.org/
 
 
 concurrent.futures
diff --git a/docs/source/setup.rst b/docs/source/setup.rst
index f93f4c03f92..d872c7e04c0 100644
--- a/docs/source/setup.rst
+++ b/docs/source/setup.rst
@@ -200,7 +200,7 @@ Using AWS
 See `Cloud Deployments`_ for the latest information on deploying to Amazon
 cloud.
 
-.. _`Cloud Deployments`: https://dask.pydata.org/en/latest/setup/cloud.html
+.. _`Cloud Deployments`: https://docs.dask.org/en/latest/setup/cloud.html
 
 
 Using Google Cloud
diff --git a/docs/source/task-launch.rst b/docs/source/task-launch.rst
index 0c393514420..291c87ba6c1 100644
--- a/docs/source/task-launch.rst
+++ b/docs/source/task-launch.rst
@@ -142,7 +142,7 @@ detail, see `dask.delayed`_.
         result = fib(10).compute()
         print(result)  # prints "55"
 
-.. _dask.delayed: https://dask.pydata.org/en/latest/delayed.html
+.. _dask.delayed: https://docs.dask.org/en/latest/delayed.html
 
 Getting the client on a worker
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

From ba7b062acc72ab43d549236f71ec15c7a273bdda Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 5 Oct 2018 10:25:54 -0400
Subject: [PATCH 0106/1550] Add ncores to workertable (#2289)

Fixes https://github.com/dask/distributed/issues/2283
---
 distributed/bokeh/scheduler.py                  | 1 +
 distributed/bokeh/tests/test_scheduler_bokeh.py | 3 +++
 2 files changed, 4 insertions(+)

diff --git a/distributed/bokeh/scheduler.py b/distributed/bokeh/scheduler.py
index bfec2418a9f..4693b232b10 100644
--- a/distributed/bokeh/scheduler.py
+++ b/distributed/bokeh/scheduler.py
@@ -1046,6 +1046,7 @@ def update(self):
             data['memory_limit'][-1] = ws.memory_limit
             data['cpu'][-1] = ws.metrics['cpu'] / 100.0
             data['cpu_fraction'][-1] = ws.metrics['cpu'] / 100.0 / ws.ncores
+            data['ncores'][-1] = ws.ncores
 
         self.source.data.update(data)
 
diff --git a/distributed/bokeh/tests/test_scheduler_bokeh.py b/distributed/bokeh/tests/test_scheduler_bokeh.py
index f8f1fe0a0a7..cec663c9b65 100644
--- a/distributed/bokeh/tests/test_scheduler_bokeh.py
+++ b/distributed/bokeh/tests/test_scheduler_bokeh.py
@@ -296,6 +296,9 @@ def test_WorkerTable(c, s, a, b):
     assert all(wt.source.data.values())
     assert all(len(v) == 2 for v in wt.source.data.values())
 
+    ncores = wt.source.data['ncores']
+    assert all(ncores)
+
 
 @gen_cluster(client=True)
 def test_WorkerTable_custom_metrics(c, s, a, b):

From b16b85d1306df62a8b5a37b00bca24e2009a04a1 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 5 Oct 2018 10:26:09 -0400
Subject: [PATCH 0107/1550] Support upload_file on files with no extension
 (#2290)

This was due to a typo where we missed the trailing comma from a tuple

Fixes https://github.com/dask/distributed/issues/2276
---
 distributed/tests/test_client.py | 6 ++++++
 distributed/utils.py             | 2 +-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 6b2749478d1..f9f47362eb4 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -1508,6 +1508,12 @@ def g():
             del sys.modules['myfile']
 
 
+@gen_cluster(client=True)
+def test_upload_file_no_extension(c, s, a, b):
+    with tmp_text('myfile', '') as fn:
+        yield c.upload_file(fn)
+
+
 @gen_cluster(client=True)
 def test_upload_file_zip(c, s, a, b):
     def g():
diff --git a/distributed/utils.py b/distributed/utils.py
index 430bb7163d1..9f42037090a 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -1020,7 +1020,7 @@ def import_file(path):
     names_to_import = []
     tmp_python_path = None
 
-    if ext in ('.py'):  # , '.pyc'):
+    if ext in ('.py',):  # , '.pyc'):
         if directory not in sys.path:
             tmp_python_path = directory
         names_to_import.append(name)

From 3eaca221e39391073d2946b4f8afc8081f886a87 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 5 Oct 2018 10:41:00 -0400
Subject: [PATCH 0108/1550] bump version to 1.23.3

---
 docs/source/changelog.rst | 87 +++++++++++++++++++++++----------------
 1 file changed, 52 insertions(+), 35 deletions(-)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index 34cad1580c6..9cab7dde998 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,26 +1,41 @@
 Changelog
 =========
 
+1.23.3 - 2018-10-05
+-------------------
+
+-  Err in dask serialization if not a NotImplementedError (:pr:`2251`) `Matthew Rocklin`_
+-  Protect against key missing from priority in GraphLayout (:pr:`2259`) `Matthew Rocklin`_
+-  Do not pull data twice in Client.gather (:pr:`2263`) `Adam Klein`_
+-  Add pytest fixture for cluster tests (:pr:`2262`) `Matthew Rocklin`_
+-  Cleanup bokeh callbacks  (:pr:`2261`) (:pr:`2278`) `Matthew Rocklin`_
+-  Fix bokeh error for `memory_limit=None` (:pr:`2255`) `Brett Naul`_
+-  Place large keywords into task graph in Client.map (:pr:`2281`) `Matthew Rocklin`_
+-  Remove redundant blosc threading code from protocol.numpy (:pr:`2284`) `Mike Gevaert`_
+-  Add ncores to workertable (:pr:`2289`) `Matthew Rocklin`_
+-  Support upload_file on files with no extension (:pr:`2290`) `Matthew Rocklin`_
+
+
 1.23.2 - 2018-09-17
 -------------------
 
--  Discard dependent rather than remove (#2250) `Matthew Rocklin`_
+-  Discard dependent rather than remove (:pr:`2250`) `Matthew Rocklin`_
 -  Use dask_sphinx_theme `Matthew Rocklin`_
--  Drop the Bokeh index page (#2241) `John Kirkham`_
--  Revert change to keep link relative (#2242) `Matthew Rocklin`_
--  docs: Fix broken AWS link in setup.rst file (#2240) `Vladyslav Moisieienkov`_
--  Return cancelled futures in as_completed (#2233) `Chris White`_
+-  Drop the Bokeh index page (:pr:`2241`) `John Kirkham`_
+-  Revert change to keep link relative (:pr:`2242`) `Matthew Rocklin`_
+-  docs: Fix broken AWS link in setup.rst file (:pr:`2240`) `Vladyslav Moisieienkov`_
+-  Return cancelled futures in as_completed (:pr:`2233`) `Chris White`_
 
 
 1.23.1 - 2018-09-06
 -------------------
 
--  Raise informative error when mixing futures between clients (#2227) `Matthew Rocklin`_
--  add byte_keys to unpack_remotedata call (#2232) `Matthew Rocklin`_
--  Add documentation for gist/rawgit for get_task_stream (#2236) `Matthew Rocklin`_
--  Quiet Client.close by waiting for scheduler stop signal (#2237) `Matthew Rocklin`_
--  Display system graphs nicely on different screen sizes (#2239) `Derek Ludwig`_
--  Mutate passed in workers dict in TaskStreamPlugin.rectangles (#2238) `Matthew Rocklin`_
+-  Raise informative error when mixing futures between clients (:pr:`2227`) `Matthew Rocklin`_
+-  add byte_keys to unpack_remotedata call (:pr:`2232`) `Matthew Rocklin`_
+-  Add documentation for gist/rawgit for get_task_stream (:pr:`2236`) `Matthew Rocklin`_
+-  Quiet Client.close by waiting for scheduler stop signal (:pr:`2237`) `Matthew Rocklin`_
+-  Display system graphs nicely on different screen sizes (:pr:`2239`) `Derek Ludwig`_
+-  Mutate passed in workers dict in TaskStreamPlugin.rectangles (:pr:`2238`) `Matthew Rocklin`_
 
 
 1.23.0 - 2018-08-30
@@ -29,30 +44,30 @@ Changelog
 -  Add direct_to_workers to Client `Matthew Rocklin`_
 -  Add Scheduler.proxy to workers `Matthew Rocklin`_
 -  Implement Actors `Matthew Rocklin`_
--  Fix tooltip (#2168) `Loïc Estève`_
--  Fix scale /  avoid returning coroutines (#2171) `Joe Hamman`_
--  Clarify dask-worker --nprocs (#2173) `Yu Feng`_
--  Concatenate all bytes of small messages in TCP comms (#2172) `Matthew Rocklin`_
--  Add dashboard_link property (#2176) `Jacob Tomlinson`_
--  Always offload to_frames (#2170) `Matthew Rocklin`_
--  Warn if desired port is already in use (#2191) (#2199) `Matthew Rocklin`_
--  Add profile page for event loop thread (#2144) `Matthew Rocklin`_
--  Use dispatch for dask serialization, also add sklearn, pytorch (#2175) `Matthew Rocklin`_
--  Handle corner cases with busy signal (#2182) `Matthew Rocklin`_
--  Check self.dependencies when looking at tasks in memory (#2196) `Matthew Rocklin`_
--  Add ability to log additional custom metrics from each worker (#2169) `Loïc Estève`_
--  Fix formatting when port is a tuple (#2204) `Loïc Estève`_
--  Describe what ZeroMQ is (#2211) `Mike DePalatis`_
--  Tiny typo fix (#2214) `Anderson Banihirwe`_
--  Add Python 3.7 to travis.yml (#2203) `Matthew Rocklin`_
--  Add plot= keyword to get_task_stream (#2198) `Matthew Rocklin`_
--  Add support for optional versions in Client.get_versions (#2216) `Matthew Rocklin`_
--  Add routes for solo bokeh figures in dashboard (#2185) `Matthew Rocklin`_
--  Be resilient to missing dep after busy signal (#2217) `Matthew Rocklin`_
--  Use CSS Grid to layout status page on the dashboard (#2213) `Derek Ludwig`_ and `Luke Canavan`_
--  Fix deserialization of queues on main ioloop thread (#2221) `Matthew Rocklin`_
--  Add a worker initialization function (#2201) `Guillaume EB`_
--  Collapse navbar in dashboard (#2223) `Luke Canavan`_
+-  Fix tooltip (:pr:`2168`) `Loïc Estève`_
+-  Fix scale /  avoid returning coroutines (:pr:`2171`) `Joe Hamman`_
+-  Clarify dask-worker --nprocs (:pr:`2173`) `Yu Feng`_
+-  Concatenate all bytes of small messages in TCP comms (:pr:`2172`) `Matthew Rocklin`_
+-  Add dashboard_link property (:pr:`2176`) `Jacob Tomlinson`_
+-  Always offload to_frames (:pr:`2170`) `Matthew Rocklin`_
+-  Warn if desired port is already in use (:pr:`2191`) (:pr:`2199`) `Matthew Rocklin`_
+-  Add profile page for event loop thread (:pr:`2144`) `Matthew Rocklin`_
+-  Use dispatch for dask serialization, also add sklearn, pytorch (:pr:`2175`) `Matthew Rocklin`_
+-  Handle corner cases with busy signal (:pr:`2182`) `Matthew Rocklin`_
+-  Check self.dependencies when looking at tasks in memory (:pr:`2196`) `Matthew Rocklin`_
+-  Add ability to log additional custom metrics from each worker (:pr:`2169`) `Loïc Estève`_
+-  Fix formatting when port is a tuple (:pr:`2204`) `Loïc Estève`_
+-  Describe what ZeroMQ is (:pr:`2211`) `Mike DePalatis`_
+-  Tiny typo fix (:pr:`2214`) `Anderson Banihirwe`_
+-  Add Python 3.7 to travis.yml (:pr:`2203`) `Matthew Rocklin`_
+-  Add plot= keyword to get_task_stream (:pr:`2198`) `Matthew Rocklin`_
+-  Add support for optional versions in Client.get_versions (:pr:`2216`) `Matthew Rocklin`_
+-  Add routes for solo bokeh figures in dashboard (:pr:`2185`) `Matthew Rocklin`_
+-  Be resilient to missing dep after busy signal (:pr:`2217`) `Matthew Rocklin`_
+-  Use CSS Grid to layout status page on the dashboard (:pr:`2213`) `Derek Ludwig`_ and `Luke Canavan`_
+-  Fix deserialization of queues on main ioloop thread (:pr:`2221`) `Matthew Rocklin`_
+-  Add a worker initialization function (:pr:`2201`) `Guillaume EB`_
+-  Collapse navbar in dashboard (:pr:`2223`) `Luke Canavan`_
 
 
 1.22.1 - 2018-08-03
@@ -784,3 +799,5 @@ significantly without many new features.
 .. _`Guillaume EB`: https://github.com/guillaumeeb
 .. _`Vladyslav Moisieienkov`: https://github.com/VMois
 .. _`Chris White`: https://github.com/cicdw
+.. _`Adam Klein`: https://github.com/adamklein
+.. _`Mike Gevaert`: https://github.com/mgeplf

From a2d3fff94cb9dcf1b998cde572b00369245ab744 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 8 Oct 2018 09:37:18 -0400
Subject: [PATCH 0109/1550] Remove Joblib Dask Backend from codebase (#2298)

* Remove Joblib Dask Backend from codebase

This code has moved to joblib itself

This commits removes all relevant code and tests, and raises an
informative error message instead.

* err nicely on response['status'] keyerror

* skip intermittent test on windows
---
 distributed/joblib.py               | 301 +---------------------------
 distributed/tests/test_joblib.py    | 295 ---------------------------
 distributed/tests/test_resources.py |   3 +-
 distributed/worker.py               |   9 +-
 4 files changed, 20 insertions(+), 588 deletions(-)
 delete mode 100644 distributed/tests/test_joblib.py

diff --git a/distributed/joblib.py b/distributed/joblib.py
index 517b82ebc28..fd81a8b078e 100644
--- a/distributed/joblib.py
+++ b/distributed/joblib.py
@@ -1,298 +1,19 @@
-from __future__ import print_function, division, absolute_import
+msg = """ It is no longer necessary to `import dask_ml.joblib` or
+`import distributed.joblib`.
 
-import contextlib
+This functionality has moved into the core Joblib codebase.
 
-from distutils.version import LooseVersion
-from uuid import uuid4
-import weakref
+To use Joblib's Dask backend with Scikit-Learn >= 0.20.0
 
-from tornado import gen
+    from dask.distributed import Client
+    client = Client()
 
-from .client import Client, _wait
-from .utils import ignoring, funcname, itemgetter
-from . import get_client, secede, rejoin
-from .worker import thread_state
-from .sizeof import sizeof
+    from sklearn.externals import joblib
 
-# A user could have installed joblib, sklearn, both, or neither. Further, only
-# joblib >= 0.10.0 supports backends, so we also need to check for that. This
-# bit of logic is to ensure that we create and register the backend for all
-# viable installations of joblib.
-joblib = sk_joblib = None
-with ignoring(ImportError):
-    import joblib
-    if LooseVersion(joblib.__version__) < '0.10.2':
-        joblib = None
-with ignoring(ImportError):
-    import sklearn.externals.joblib as sk_joblib
-    if LooseVersion(sk_joblib.__version__) < '0.10.2':
-        sk_joblib = None
+    with joblib.parallel_backend('dask'):
+        # your scikit-learn code
 
-_bases = []
-if joblib:
-    from joblib.parallel import AutoBatchingMixin, ParallelBackendBase
-    _bases.append(ParallelBackendBase)
-if sk_joblib:
-    from sklearn.externals.joblib.parallel import (AutoBatchingMixin,  # noqa
-            ParallelBackendBase)
-    _bases.append(ParallelBackendBase)
-if not _bases:
-    raise RuntimeError("Joblib backend requires either `joblib` >= '0.10.2' "
-                       " or `sklearn` > '0.17.1'. Please install or upgrade")
+See http://ml.dask.org/joblib.html for more information."""
 
 
-def is_weakrefable(obj):
-    try:
-        weakref.ref(obj)
-        return True
-    except TypeError:
-        return False
-
-
-class _WeakKeyDictionary:
-    """A variant of weakref.WeakKeyDictionary for unhashable objects.
-
-    This datastructure is used to store futures for broadcasted data objects
-    such as large numpy arrays or pandas dataframes that are not hashable and
-    therefore cannot be used as keys of traditional python dicts.
-
-    Futhermore using a dict with id(array) as key is not safe because the
-    Python is likely to reuse id of recently collected arrays.
-    """
-
-    def __init__(self):
-        self._data = {}
-
-    def __getitem__(self, obj):
-        ref, val = self._data[id(obj)]
-        if ref() is not obj:
-            # In case of a race condition with on_destroy.
-            raise KeyError(obj)
-        return val
-
-    def __setitem__(self, obj, value):
-        key = id(obj)
-        try:
-            ref, _ = self._data[key]
-            if ref() is not obj:
-                # In case of race condition with on_destroy.
-                raise KeyError(obj)
-        except KeyError:
-            # Insert the new entry in the mapping along with a weakref
-            # callback to automatically delete the entry from the mapping
-            # as soon as the object used as key is garbage collected.
-            def on_destroy(_):
-                del self._data[key]
-            ref = weakref.ref(obj, on_destroy)
-        self._data[key] = ref, value
-
-    def __len__(self):
-        return len(self._data)
-
-    def clear(self):
-        self._data.clear()
-
-
-def joblib_funcname(x):
-    try:
-        # Can't do isinstance, since joblib is often bundled in packages, and
-        # separate installs will have non-equivalent types.
-        if type(x).__name__ == 'BatchedCalls':
-            x = x.items[0][0]
-    except Exception:
-        pass
-    return funcname(x)
-
-
-class Batch(object):
-    def __init__(self, tasks):
-        self.tasks = tasks
-
-    def __call__(self, *data):
-        results = []
-        for func, args, kwargs in self.tasks:
-            args = [a(data) if isinstance(a, itemgetter) else a
-                    for a in args]
-            kwargs = {k: v(data) if isinstance(v, itemgetter) else v
-                      for (k, v) in kwargs.items()}
-            results.append(func(*args, **kwargs))
-        return results
-
-    def __reduce__(self):
-        return (Batch, (self.tasks,))
-
-
-class DaskDistributedBackend(ParallelBackendBase, AutoBatchingMixin):
-    MIN_IDEAL_BATCH_DURATION = 0.2
-    MAX_IDEAL_BATCH_DURATION = 1.0
-
-    def __init__(self, scheduler_host=None, scatter=None,
-                 client=None, loop=None, **submit_kwargs):
-        if client is None:
-            if scheduler_host:
-                client = Client(scheduler_host, loop=loop, set_as_default=False)
-            else:
-                try:
-                    client = get_client()
-                except ValueError:
-                    msg = ("To use Joblib with Dask first create a Dask Client"
-                           "\n\n"
-                           "    from dask.distributed import Client\n"
-                           "    client = Client()\n"
-                           "or\n"
-                           "    client = Client('scheduler-address:8786')")
-                    raise ValueError(msg)
-
-        self.client = client
-
-        if scatter is not None and not isinstance(scatter, (list, tuple)):
-            raise TypeError("scatter must be a list/tuple, got "
-                            "`%s`" % type(scatter).__name__)
-
-        if scatter is not None and len(scatter) > 0:
-            # Keep a reference to the scattered data to keep the ids the same
-            self._scatter = list(scatter)
-            scattered = self.client.scatter(scatter, broadcast=True)
-            self.data_futures = {id(x): f for x, f in zip(scatter, scattered)}
-        else:
-            self._scatter = []
-            self.data_futures = {}
-        self.task_futures = set()
-        self.submit_kwargs = submit_kwargs
-
-    def __reduce__(self):
-        return (DaskDistributedBackend, ())
-
-    def get_nested_backend(self):
-        return DaskDistributedBackend()
-
-    def configure(self, n_jobs=1, parallel=None, **backend_args):
-        return self.effective_n_jobs(n_jobs)
-
-    def start_call(self):
-        self.call_data_futures = _WeakKeyDictionary()
-
-    def stop_call(self):
-        # The explicit call to clear is required to break a cycling reference
-        # to the futures.
-        self.call_data_futures.clear()
-
-    def effective_n_jobs(self, n_jobs):
-        return sum(self.client.ncores().values())
-
-    def _to_func_args(self, func):
-        collected_futures = []
-        itemgetters = dict()
-
-        # Futures that are dynamically generated during a single call to
-        # Parallel.__call__.
-        call_data_futures = getattr(self, 'call_data_futures', None)
-
-        def maybe_to_futures(args):
-            for arg in args:
-                arg_id = id(arg)
-                if arg_id in itemgetters:
-                    yield itemgetters[arg_id]
-                    continue
-
-                f = None
-                if f is None and arg_id in self.data_futures:
-                    f = self.data_futures[arg_id]
-
-                elif f is None and call_data_futures is not None:
-                    try:
-                        f = call_data_futures[arg]
-                    except KeyError:
-                        if is_weakrefable(arg) and sizeof(arg) > 1e6:
-                            # Automatically scatter large objects to some of
-                            # the workers to avoid duplicated data transfers.
-                            # Rely on automated inter-worker data stealing if
-                            # more workers need to reuse this data concurrently
-                            # beyond the initial broadcast arity.
-                            [f] = self.client.scatter([arg], broadcast=3)
-                            call_data_futures[arg] = f
-
-                if f is not None:
-                    getter = itemgetter(len(collected_futures))
-                    collected_futures.append(f)
-                    itemgetters[arg_id] = getter
-                    arg = getter
-                yield arg
-
-        tasks = []
-        for f, args, kwargs in func.items:
-            args = list(maybe_to_futures(args))
-            kwargs = dict(zip(kwargs.keys(), maybe_to_futures(kwargs.values())))
-            tasks.append((f, args, kwargs))
-
-        if not collected_futures:
-            return func, ()
-        return Batch(tasks), collected_futures
-
-    def apply_async(self, func, callback=None):
-        key = '%s-batch-%s' % (joblib_funcname(func), uuid4().hex)
-        func, args = self._to_func_args(func)
-
-        future = self.client.submit(func, *args, key=key, **self.submit_kwargs)
-        self.task_futures.add(future)
-
-        @gen.coroutine
-        def callback_wrapper():
-            result = yield _wait([future])
-            self.task_futures.remove(future)
-            if callback is not None:
-                callback(result)  # gets called in separate thread
-
-        self.client.loop.add_callback(callback_wrapper)
-
-        ref = weakref.ref(future)  # avoid reference cycle
-
-        def get():
-            return ref().result()
-
-        future.get = get # monkey patch to achieve AsyncResult API
-        return future
-
-    def abort_everything(self, ensure_ready=True):
-        """ Tell the client to cancel any task submitted via this instance
-
-        joblib.Parallel will never access those results
-        """
-        self.client.cancel(self.task_futures)
-        self.task_futures.clear()
-
-    @contextlib.contextmanager
-    def retrieval_context(self):
-        """Override ParallelBackendBase.retrieval_context to avoid deadlocks.
-
-        This removes thread from the worker's thread pool (using 'secede').
-        Seceding avoids deadlock in nested parallelism settings.
-        """
-        # See 'joblib.Parallel.__call__' and 'joblib.Parallel.retrieve' for how
-        # this is used.
-        if hasattr(thread_state, 'execution_state'):
-            # we are in a worker. Secede to avoid deadlock.
-            secede()
-
-        yield
-
-        if hasattr(thread_state, 'execution_state'):
-            rejoin()
-
-
-for base in _bases:
-    base.register(DaskDistributedBackend)
-
-
-DistributedBackend = DaskDistributedBackend
-
-
-# Register the backend with any available versions of joblib
-if joblib:
-    joblib.register_parallel_backend('dask', DaskDistributedBackend)
-    joblib.register_parallel_backend('distributed', DaskDistributedBackend)
-    joblib.register_parallel_backend('dask.distributed', DaskDistributedBackend)
-if sk_joblib:
-    sk_joblib.register_parallel_backend('dask', DaskDistributedBackend)
-    sk_joblib.register_parallel_backend('distributed', DaskDistributedBackend)
-    sk_joblib.register_parallel_backend('dask.distributed', DaskDistributedBackend)
+raise ImportError(msg)
diff --git a/distributed/tests/test_joblib.py b/distributed/tests/test_joblib.py
deleted file mode 100644
index 8eec12899d6..00000000000
--- a/distributed/tests/test_joblib.py
+++ /dev/null
@@ -1,295 +0,0 @@
-from __future__ import print_function, division, absolute_import
-import os
-import importlib
-from distutils.version import LooseVersion
-
-import pytest
-from random import random
-from time import sleep
-
-from distributed import Client
-from distributed.metrics import time
-from distributed.utils_test import cluster, inc
-from distributed.utils_test import loop, client, cluster_fixture, s, a, b  # noqa F401
-from toolz import identity
-
-distributed_joblib = pytest.importorskip('distributed.joblib')
-joblib_funcname = distributed_joblib.joblib_funcname
-
-
-@pytest.fixture(params=['joblib', 'sk_joblib'])
-def joblib(request):
-    if request.param == 'joblib':
-        try:
-            this_joblib = importlib.import_module('joblib')
-        except ImportError:
-            pytest.skip("joblib not available")
-    else:
-        try:
-            this_joblib = importlib.import_module("sklearn.externals.joblib")
-        except ImportError:
-            pytest.skip("sklearn.externals.joblib not available")
-
-    return this_joblib
-
-
-def slow_raise_value_error(condition, duration=0.05):
-    sleep(duration)
-    if condition:
-        raise ValueError("condition evaluated to True")
-
-
-def test_simple(client, joblib):
-    Parallel = joblib.Parallel
-    delayed = joblib.delayed
-
-    with joblib.parallel_backend('dask') as (ba, _):
-        seq = Parallel()(delayed(inc)(i) for i in range(10))
-        assert seq == [inc(i) for i in range(10)]
-
-        with pytest.raises(ValueError):
-            Parallel()(delayed(slow_raise_value_error)(i == 3)
-                       for i in range(10))
-
-        seq = Parallel()(delayed(inc)(i) for i in range(10))
-        assert seq == [inc(i) for i in range(10)]
-
-
-def random2():
-    return random()
-
-
-def test_dont_assume_function_purity(client, joblib):
-    Parallel = joblib.Parallel
-    delayed = joblib.delayed
-
-    with joblib.parallel_backend('dask') as (ba, _):
-        x, y = Parallel()(delayed(random2)() for i in range(2))
-        assert x != y
-
-
-def test_joblib_funcname(client, joblib):
-    Parallel = joblib.Parallel
-    delayed = joblib.delayed
-
-    with joblib.parallel_backend('dask') as (ba, _):
-        x, y = Parallel()(delayed(inc)(i) for i in range(2))
-
-    def f(dask_scheduler):
-        return list(dask_scheduler.transition_log)
-    log = client.run_on_scheduler(f)
-    assert all(tup[0].startswith('inc-batch') for tup in log)
-
-
-def test_joblib_backend_subclass(joblib):
-    assert issubclass(distributed_joblib.DaskDistributedBackend,
-                      joblib.parallel.ParallelBackendBase)
-
-
-def add5(a, b, c, d=0, e=0):
-    return a + b + c + d + e
-
-
-class CountSerialized(object):
-    def __init__(self, x):
-        self.x = x
-        self.count = 0
-
-    def __add__(self, other):
-        return self.x + getattr(other, 'x', other)
-
-    __radd__ = __add__
-
-    def __reduce__(self):
-        self.count += 1
-        return (CountSerialized, (self.x,))
-
-
-def test_joblib_scatter(client, joblib):
-    Parallel = joblib.Parallel
-    delayed = joblib.delayed
-
-    x = CountSerialized(1)
-    y = CountSerialized(2)
-    z = CountSerialized(3)
-
-    with joblib.parallel_backend('dask', scatter=[x, y]) as (ba, _):
-        f = delayed(add5)
-        tasks = [f(x, y, z, d=4, e=5),
-                 f(x, z, y, d=5, e=4),
-                 f(y, x, z, d=x, e=5),
-                 f(z, z, x, d=z, e=y)]
-        sols = [func(*args, **kwargs) for func, args, kwargs in tasks]
-        results = Parallel()(tasks)
-
-    # Scatter must take a list/tuple
-    with pytest.raises(TypeError):
-        with joblib.parallel_backend('dask', loop=loop,
-                                     scatter=1):
-            pass
-
-    for l, r in zip(sols, results):
-        assert l == r
-
-    # Scattered variables only serialized once
-    assert x.count == 1
-    assert y.count == 1
-    assert z.count == 4
-
-
-def test_nested_backend_context_manager(loop, joblib):
-    if LooseVersion(joblib.__version__) <= "0.11.0":
-        pytest.skip("Joblib >= 0.11.1 required for nested parallelism.")
-    Parallel = joblib.Parallel
-    delayed = joblib.delayed
-
-    def get_nested_pids():
-        pids = set(Parallel(n_jobs=2)(delayed(os.getpid)() for _ in range(2)))
-        pids |= set(Parallel(n_jobs=2)(delayed(os.getpid)() for _ in range(2)))
-        return pids
-
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as client:
-            with joblib.parallel_backend('dask') as (ba, _):
-                pid_groups = Parallel(n_jobs=2)(
-                    delayed(get_nested_pids, check_pickle=False)()
-                    for _ in range(10)
-                )
-                for pid_group in pid_groups:
-                    assert len(set(pid_group)) <= 2
-
-        # No deadlocks
-        with Client(s['address'], loop=loop) as client:
-            with joblib.parallel_backend('dask') as (ba, _):
-                pid_groups = Parallel(n_jobs=2)(
-                    delayed(get_nested_pids, check_pickle=False)()
-                    for _ in range(10)
-                )
-                for pid_group in pid_groups:
-                    assert len(set(pid_group)) <= 2
-
-
-def test_errors(loop, joblib):
-    with pytest.raises(ValueError) as info:
-        with joblib.parallel_backend('dask'):
-            pass
-
-    assert "create a dask client" in str(info.value).lower()
-
-
-def test_correct_nested_backend(client, joblib):
-    if LooseVersion(joblib.__version__) <= LooseVersion("0.11.0"):
-        pytest.skip("Requires nested parallelism")
-
-    # No requirement, should be us
-    with joblib.parallel_backend('dask') as (ba, _):
-        result = joblib.Parallel(n_jobs=2)(joblib.delayed(outer)(
-            joblib, nested_require=None) for _ in range(1))
-        assert isinstance(result[0][0][0],
-                          distributed_joblib.DaskDistributedBackend)
-
-    # Require threads, should be threading
-    with joblib.parallel_backend('dask') as (ba, _):
-        result = joblib.Parallel(n_jobs=2)(joblib.delayed(outer)(
-            joblib, nested_require='sharedmem') for _ in range(1))
-        assert isinstance(result[0][0][0],
-                          joblib.parallel.ThreadingBackend)
-
-
-def outer(joblib, nested_require):
-    return joblib.Parallel(n_jobs=2, prefer='threads')(
-        joblib.delayed(middle)(joblib, nested_require) for _ in range(1)
-    )
-
-
-def middle(joblib, require):
-    return joblib.Parallel(n_jobs=2, require=require)(
-        joblib.delayed(inner)(joblib) for _ in range(1)
-    )
-
-
-def inner(joblib):
-    return joblib.parallel.Parallel()._backend
-
-
-def test_secede_with_no_processes(loop, joblib):
-    # https://github.com/dask/distributed/issues/1775
-
-    with Client(loop=loop, processes=False, set_as_default=True):
-        with joblib.parallel_backend('dask'):
-            joblib.Parallel(n_jobs=4)(joblib.delayed(identity)(i) for i in range(2))
-
-
-def _test_keywords_f(_):
-    from distributed import get_worker
-    return get_worker().address
-
-
-def test_keywords(client, joblib, s, a, b):
-    with joblib.parallel_backend('dask', workers=a['address']) as (ba, _):
-        seq = joblib.Parallel()(joblib.delayed(_test_keywords_f)(i) for i in range(10))
-        assert seq == [a['address']] * 10
-
-    with joblib.parallel_backend('dask', workers=b['address']) as (ba, _):
-        seq = joblib.Parallel()(joblib.delayed(_test_keywords_f)(i) for i in range(10))
-        assert seq == [b['address']] * 10
-
-
-def test_cleanup(loop, joblib):
-    with Client(processes=False, loop=loop) as client:
-        with joblib.parallel_backend('dask'):
-            joblib.Parallel()(joblib.delayed(inc)(i) for i in range(10))
-
-        start = time()
-        while client.cluster.scheduler.tasks:
-            sleep(0.01)
-            assert time() < start + 5
-
-        assert not client.futures
-
-
-def test_auto_scatter(loop, joblib):
-    base_type = joblib._parallel_backends.ParallelBackendBase
-    if not hasattr(base_type, 'start_call'):
-        raise pytest.skip('joblib version does not support backend callbacks')
-
-    np = pytest.importorskip('numpy')
-    data = np.ones(int(1e7), dtype=np.uint8)
-
-    Parallel = joblib.Parallel
-    delayed = joblib.delayed
-
-    def noop(*args, **kwargs):
-        pass
-
-    def count_events(event_name, client):
-        worker_events = client.run(lambda dask_worker: dask_worker.log)
-        event_counts = {}
-        for w, events in worker_events.items():
-            event_counts[w] = len([event for event in list(events)
-                                   if event[1] == event_name])
-        return event_counts
-
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as client:
-            with joblib.parallel_backend('dask') as (ba, _):
-                # Passing the same data as arg and kwarg triggers a single
-                # scatter operation whose result is reused.
-                Parallel()(delayed(noop)(data, data, i, opt=data)
-                           for i in range(5))
-            # By default large array are automatically scattered with
-            # broadcast=3 which means that each worker can directly receive
-            # the data from the scatter operation once.
-            counts = count_events('receive-from-scatter', client)
-            assert counts[a['address']] == 1
-            assert counts[b['address']] == 1
-
-    with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as client:
-            with joblib.parallel_backend('dask') as (ba, _):
-                Parallel()(delayed(noop)(data[:3], i) for i in range(5))
-            # Small arrays are passed within the task definition without going
-            # through a scatter operation.
-            counts = count_events('receive-from-scatter', client)
-            assert counts[a['address']] == 0
-            assert counts[b['address']] == 0
diff --git a/distributed/tests/test_resources.py b/distributed/tests/test_resources.py
index 2d5b1dab8b7..5fa8b1c8a34 100644
--- a/distributed/tests/test_resources.py
+++ b/distributed/tests/test_resources.py
@@ -8,6 +8,7 @@
 
 from distributed import Worker
 from distributed.client import wait
+from distributed.compatibility import WINDOWS
 from distributed.utils import tokey
 from distributed.utils_test import (inc, gen_cluster, slowinc, slowadd)
 from distributed.utils_test import client, cluster_fixture, loop, s, a, b  # noqa: F401
@@ -276,7 +277,7 @@ def test_full_collections(c, s, a, b):
 
 @pytest.mark.parametrize('optimize_graph', [
     pytest.mark.xfail(True, reason="don't track resources through optimization"),
-    False
+    pytest.mark.skipif(WINDOWS, False, reason="intermittent failure"),
 ])
 def test_collections_get(client, optimize_graph, s, a, b):
     da = pytest.importorskip('dask.array')
diff --git a/distributed/worker.py b/distributed/worker.py
index 06b38a246c0..9154a268de2 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -2919,8 +2919,13 @@ def get_data_from_worker(rpc, keys, worker, who=None, max_connections=None,
                                    deserializers=deserializers,
                                    op='get_data', keys=keys, who=who,
                                    max_connections=max_connections)
-        if response['status'] == 'OK':
-            yield comm.write('OK')
+        try:
+            status = response['status']
+        except KeyError:
+            raise ValueError("Unexpected response", response)
+        else:
+            if status == 'OK':
+                yield comm.write('OK')
     finally:
         rpc.reuse(worker, comm)
 

From d7374979ec92efb5e85d7114062d450dcdc3771c Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 8 Oct 2018 16:08:44 -0400
Subject: [PATCH 0110/1550] Include worker tls protocol in Scheduler.restart
 (#2295)

Fixes https://github.com/dask/distributed/issues/2282

Previously we would try to connect to the Nanny using only the host and
port information, which would default to using the tcp:// protocol and
so dropped all of the security information.  Now we reuse the same
protocol that we use for the worker.
---
 distributed/deploy/tests/test_local.py | 15 +++++++++++++++
 distributed/scheduler.py               | 25 ++++++++++++++++++++-----
 2 files changed, 35 insertions(+), 5 deletions(-)

diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index b2c226c565c..10e4a3514e1 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -515,5 +515,20 @@ def scale_down(self, *args, **kwargs):
     yield cluster._close()
 
 
+def test_local_tls_restart(loop):
+    from distributed.utils_test import tls_only_security
+    security = tls_only_security()
+    with LocalCluster(n_workers=1, scheduler_port=8786, silence_logs=False, security=security,
+                      diagnostics_port=False, ip='tls://0.0.0.0', loop=loop) as c:
+        with Client(c.scheduler.address, loop=loop, security=security) as client:
+            print(c.workers, c.workers[0].address)
+            workers_before = set(client.scheduler_info()['workers'])
+            assert client.submit(inc, 1).result() == 2
+            client.restart()
+            workers_after = set(client.scheduler_info()['workers'])
+            assert client.submit(inc, 2).result() == 3
+            assert workers_before != workers_after
+
+
 if sys.version_info >= (3, 5):
     from distributed.deploy.tests.py3_test_deploy import *  # noqa F401
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 316913db81d..d0903860896 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1018,15 +1018,30 @@ def identity(self, comm=None):
                          for worker in self.workers.values()}}
         return d
 
-    def get_worker_service_addr(self, worker, service_name):
+    def get_worker_service_addr(self, worker, service_name, protocol=False):
         """
         Get the (host, port) address of the named service on the *worker*.
         Returns None if the service doesn't exist.
+
+        Parameters
+        ----------
+        worker : address
+        service_name : str
+            Common services include 'bokeh' and 'nanny'
+        protocol : boolean
+            Whether or not to include a full address with protocol (True)
+            or just a (host, port) pair
         """
         ws = self.workers[worker]
         port = ws.services.get(service_name)
         if port is None:
             return None
+        elif protocol:
+            return '%(protocol)s://%(host)s:%(port)d' % {
+                'protocol': ws.address.split('://')[0],
+                'host': ws.host,
+                'port': port
+            }
         else:
             return ws.host, port
 
@@ -1191,7 +1206,7 @@ def close_worker(self, stream=None, worker=None, safe=None):
         logger.info("Closing worker %s", worker)
         with log_errors():
             self.log_event(worker, {'action': 'close-worker'})
-            nanny_addr = self.get_worker_service_addr(worker, 'nanny')
+            nanny_addr = self.get_worker_service_addr(worker, 'nanny', protocol=True)
             address = nanny_addr or worker
 
             self.worker_send(worker, {'op': 'close', 'report': False})
@@ -2290,7 +2305,7 @@ def restart(self, client=None, timeout=3):
                 self.client_releases_keys(keys=[ts.key for ts in cs.wants_what],
                                           client=cs.client_key)
 
-            nannies = {addr: self.get_worker_service_addr(addr, 'nanny')
+            nannies = {addr: self.get_worker_service_addr(addr, 'nanny', protocol=True)
                        for addr in self.workers}
 
             for addr in list(self.workers):
@@ -2357,7 +2372,7 @@ def broadcast(self, comm=None, msg=None, workers=None, hosts=None,
         # TODO replace with worker_list
 
         if nanny:
-            addresses = [self.get_worker_service_addr(w, 'nanny')
+            addresses = [self.get_worker_service_addr(w, 'nanny', protocol=True)
                          for w in workers]
         else:
             addresses = workers
@@ -2369,7 +2384,7 @@ def send_message(addr):
             resp = yield send_recv(comm, close=True, serializers=serializers, **msg)
             raise gen.Return(resp)
 
-        results = yield All([send_message(self.coerce_address(address))
+        results = yield All([send_message(address)
                              for address in addresses
                              if address is not None])
 

From bdad55d2514fa65bd7d69f7b98e927c160d08986 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 8 Oct 2018 16:28:56 -0400
Subject: [PATCH 0111/1550] Adapt to new Bokeh selection for 1.0 (#2292)

Adapt to new Bokeh selection for 1.0
---
 distributed/bokeh/components.py      | 10 ++++++++--
 distributed/tests/py3_test_pubsub.py |  2 ++
 distributed/tests/test_profile.py    |  2 +-
 3 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/distributed/bokeh/components.py b/distributed/bokeh/components.py
index a5c4a831b49..8141936a800 100644
--- a/distributed/bokeh/components.py
+++ b/distributed/bokeh/components.py
@@ -476,7 +476,10 @@ def cb(attr, old, new):
 
         def ts_change(attr, old, new):
             with log_errors():
-                selected = self.ts_source.selected['1d']['indices']
+                try:
+                    selected = self.ts_source.selected.indices
+                except AttributeError:
+                    selected = self.ts_source.selected['1d']['indices']
                 if selected:
                     start = self.ts_source.data['time'][min(selected)] / 1000
                     stop = self.ts_source.data['time'][max(selected)] / 1000
@@ -636,7 +639,10 @@ def cb(attr, old, new):
 
         def ts_change(attr, old, new):
             with log_errors():
-                selected = self.ts_source.selected['1d']['indices']
+                try:
+                    selected = self.ts_source.selected.indices
+                except AttributeError:
+                    selected = self.ts_source.selected['1d']['indices']
                 if selected:
                     start = self.ts_source.data['time'][min(selected)] / 1000
                     stop = self.ts_source.data['time'][max(selected)] / 1000
diff --git a/distributed/tests/py3_test_pubsub.py b/distributed/tests/py3_test_pubsub.py
index ede8023801b..b7cde193d37 100644
--- a/distributed/tests/py3_test_pubsub.py
+++ b/distributed/tests/py3_test_pubsub.py
@@ -3,8 +3,10 @@
 
 import toolz
 from tornado import gen
+import pytest
 
 
+@pytest.mark.xfail(reason="out of order execution")
 @gen_cluster(client=True)
 def test_basic(c, s, a, b):
     async def publish():
diff --git a/distributed/tests/test_profile.py b/distributed/tests/test_profile.py
index 2101d2a1669..b7c717e1b61 100644
--- a/distributed/tests/test_profile.py
+++ b/distributed/tests/test_profile.py
@@ -43,7 +43,7 @@ def test_f():
     h = [c for c in d['children'].values() if 'test_h' in str(c['description'])][0]
 
     assert g['count'] < h['count']
-    assert g['count'] + h['count'] == 100
+    assert 95 < g['count'] + h['count'] <= 100
 
 
 def test_merge():

From 3d7115b6f49e683dae36b0dcf21fc506b4ac07ca Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 8 Oct 2018 17:43:52 -0400
Subject: [PATCH 0112/1550] Add explicit retry method to Future and Client
 (#2299)

This allows for the explicit retrying of failed tasks

Fixes #2272
---
 distributed/client.py            | 37 +++++++++++++++++++
 distributed/scheduler.py         | 62 ++++++++++++++++++++++++++++++++
 distributed/tests/test_client.py | 55 ++++++++++++++++++++++++++++
 distributed/tests/test_core.py   |  4 +++
 distributed/utils.py             |  2 +-
 distributed/worker.py            |  8 +++--
 docs/source/api.rst              |  2 ++
 7 files changed, 166 insertions(+), 4 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index f9cf7f94985..8e513466bfe 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -271,6 +271,15 @@ def cancel(self, **kwargs):
         """
         return self.client.cancel([self], **kwargs)
 
+    def retry(self, **kwargs):
+        """ Retry this future if it has failed
+
+        See Also
+        --------
+        Client.retry
+        """
+        return self.client.retry([self], **kwargs)
+
     def cancelled(self):
         """ Returns True if the future has been cancelled """
         return self._state.status == 'cancelled'
@@ -402,6 +411,10 @@ def lose(self):
         self.status = 'lost'
         self._get_event().clear()
 
+    def retry(self):
+        self.status = 'pending'
+        self._get_event().clear()
+
     def set_error(self, exception, traceback):
         _, exception, traceback = clean_exception(exception, traceback)
 
@@ -588,6 +601,7 @@ def __init__(self, address=None, loop=None, timeout=no_default,
             'key-in-memory': self._handle_key_in_memory,
             'lost-data': self._handle_lost_data,
             'cancelled-key': self._handle_cancelled_key,
+            'task-retried': self._handle_retried_key,
             'task-erred': self._handle_task_erred,
             'restart': self._handle_restart,
             'error': self._handle_error
@@ -1012,6 +1026,11 @@ def _handle_cancelled_key(self, key=None):
         if state is not None:
             state.cancel()
 
+    def _handle_retried_key(self, key=None):
+        state = self.futures.get(key)
+        if state is not None:
+            state.retry()
+
     def _handle_task_erred(self, key=None, exception=None, traceback=None):
         state = self.futures.get(key)
         if state is not None:
@@ -1831,6 +1850,24 @@ def cancel(self, futures, asynchronous=None, force=False):
         return self.sync(self._cancel, futures, asynchronous=asynchronous,
                          force=force)
 
+    @gen.coroutine
+    def _retry(self, futures):
+        keys = list({tokey(f.key) for f in futures_of(futures)})
+        response = yield self.scheduler.retry(keys=keys, client=self.id)
+        for key in response:
+            st = self.futures[key]
+            st.retry()
+
+    def retry(self, futures, asynchronous=None):
+        """
+        Retry failed futures
+
+        Parameters
+        ----------
+        futures: list of Futures
+        """
+        return self.sync(self._retry, futures, asynchronous=asynchronous)
+
     @gen.coroutine
     def _publish_dataset(self, *args, **kwargs):
         with log_errors():
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index d0903860896..49e2f3a8cd4 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -930,6 +930,7 @@ def __init__(
             'unregister': self.remove_worker,
             'gather': self.gather,
             'cancel': self.stimulus_cancel,
+            'retry': self.stimulus_retry,
             'feed': self.feed,
             'terminate': self.close,
             'broadcast': self.broadcast,
@@ -973,6 +974,7 @@ def __init__(
             ('released', 'forgotten'): self.transition_released_forgotten,
             ('memory', 'forgotten'): self.transition_memory_forgotten,
             ('erred', 'forgotten'): self.transition_released_forgotten,
+            ('erred', 'released'): self.transition_erred_released,
             ('memory', 'released'): self.transition_memory_released,
             ('released', 'erred'): self.transition_released_erred
         }
@@ -1653,6 +1655,33 @@ def stimulus_missing_data(self, cause=None, key=None, worker=None,
 
             return {}
 
+    def stimulus_retry(self, comm=None, keys=None, client=None):
+        logger.info("Client %s requests to retry %d keys", client, len(keys))
+        if client:
+            self.log_event(client, {'action': 'retry', 'count': len(keys)})
+
+        stack = list(keys)
+        seen = set()
+        roots = []
+        while stack:
+            key = stack.pop()
+            seen.add(key)
+            erred_deps = [dts.key for dts in self.tasks[key].dependencies
+                          if dts.state == 'erred']
+            if erred_deps:
+                stack.extend(erred_deps)
+            else:
+                roots.append(key)
+
+        recommendations = {key: 'waiting' for key in roots}
+        self.transitions(recommendations)
+
+        if self.validate:
+            for key in seen:
+                assert not self.tasks[key].exception_blame
+
+        return tuple(seen)
+
     def remove_worker(self, comm=None, address=None, safe=False, close=True):
         """
         Remove worker from cluster
@@ -3515,6 +3544,39 @@ def transition_released_erred(self, key):
                 pdb.set_trace()
             raise
 
+    def transition_erred_released(self, key):
+        try:
+            ts = self.tasks[key]
+
+            if self.validate:
+                with log_errors(pdb=LOG_PDB):
+                    assert all(dts.state != 'erred' for dts in ts.dependencies)
+                    assert ts.exception_blame
+                    assert not ts.who_has
+                    assert not ts.waiting_on
+                    assert not ts.waiters
+
+            recommendations = OrderedDict()
+
+            ts.exception = None
+            ts.exception_blame = None
+            ts.traceback = None
+
+            for dep in ts.dependents:
+                if dep.state == 'erred':
+                    recommendations[dep.key] = 'waiting'
+
+            self.report({'op': 'task-retried', 'key': key})
+            ts.state = 'released'
+
+            return recommendations
+        except Exception as e:
+            logger.exception(e)
+            if LOG_PDB:
+                import pdb
+                pdb.set_trace()
+            raise
+
     def transition_waiting_released(self, key):
         try:
             ts = self.tasks[key]
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index f9f47362eb4..d8c58b6c361 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -5387,5 +5387,60 @@ def test_map_large_kwargs_in_graph(c, s, a, b):
     assert any(k.startswith('ndarray') for k in s.tasks)
 
 
+@gen_cluster(client=True)
+def test_retry(c, s, a, b):
+    def f():
+        assert dask.config.get('foo')
+
+    with dask.config.set(foo=False):
+        future = c.submit(f)
+        with pytest.raises(AssertionError):
+            yield future
+
+    with dask.config.set(foo=True):
+        yield future.retry()
+        yield future
+
+
+@gen_cluster(client=True)
+def test_retry_dependencies(c, s, a, b):
+    def f():
+        return dask.config.get('foo')
+
+    x = c.submit(f)
+    y = c.submit(inc, x)
+
+    with pytest.raises(KeyError):
+        yield y
+
+    with dask.config.set(foo=100):
+        yield y.retry()
+        result = yield y
+        assert result == 101
+
+        yield y.retry()
+        yield x.retry()
+        result = yield y
+        assert result == 101
+
+
+@gen_cluster(client=True)
+def test_released_dependencies(c, s, a, b):
+    def f(x):
+        return dask.config.get('foo') + 1
+
+    x = c.submit(inc, 1, key='x')
+    y = c.submit(f, x, key='y')
+    del x
+
+    with pytest.raises(KeyError):
+        yield y
+
+    with dask.config.set(foo=100):
+        yield y.retry()
+        result = yield y
+        assert result == 101
+
+
 if sys.version_info >= (3, 5):
     from distributed.tests.py3_test_client import *  # noqa F401
diff --git a/distributed/tests/test_core.py b/distributed/tests/test_core.py
index 66bd7d20058..41e816c1e4a 100644
--- a/distributed/tests/test_core.py
+++ b/distributed/tests/test_core.py
@@ -301,6 +301,8 @@ def check_rpc_message_lifetime(*listen_args):
         # If additional instances were created, they were deleted as well
         assert CountedObject.n_instances == 0
 
+    server.stop()
+
 
 @gen_test()
 def test_rpc_message_lifetime_default():
@@ -652,6 +654,8 @@ def f():
             result = yield r.echo(x=to_serialize(inc))
             assert result == {'result': inc}
 
+        server.stop()
+
     loop.run_sync(f)
 
 
diff --git a/distributed/utils.py b/distributed/utils.py
index 9f42037090a..df7561aad3e 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -370,7 +370,7 @@ def run_loop(loop=self._loop):
             # Loop already running in other thread (user-launched)
             done_evt.wait(5)
             if not isinstance(start_exc[0], RuntimeError):
-                if not isinstance(start_exc[0]):  # track down infrequent error
+                if not isinstance(start_exc[0], Exception):  # track down infrequent error
                     raise TypeError("not an exception", start_exc[0])
                 raise start_exc[0]
             self._all_loops[self._loop] = count + 1, None
diff --git a/distributed/worker.py b/distributed/worker.py
index 9154a268de2..43b3ee13ed2 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -1356,15 +1356,17 @@ def add_task(self, key, function=None, args=None, kwargs=None, task=None,
         try:
             if key in self.tasks:
                 state = self.task_state[key]
-                if state in ('memory', 'error'):
-                    if state == 'memory':
-                        assert key in self.data or key in self.actors
+                if state == 'memory':
+                    assert key in self.data or key in self.actors
                     logger.debug("Asked to compute pre-existing result: %s: %s",
                                  key, state)
                     self.send_task_state_to_scheduler(key)
                     return
                 if state in IN_PLAY:
                     return
+                if state == 'erred':
+                    del self.exceptions[key]
+                    del self.tracebacks[key]
 
             if priority is not None:
                 priority = tuple(priority) + (self.generation,)
diff --git a/docs/source/api.rst b/docs/source/api.rst
index 2e8b78bc4b7..33634bff6c7 100644
--- a/docs/source/api.rst
+++ b/docs/source/api.rst
@@ -31,6 +31,7 @@ API
    Client.rebalance
    Client.replicate
    Client.restart
+   Client.retry
    Client.run
    Client.run_on_scheduler
    Client.scatter
@@ -73,6 +74,7 @@ API
    Future.done
    Future.exception
    Future.result
+   Future.retry
    Future.traceback
 
 **Client Coordination**

From f0a07956e168c8d1c7251da609a0972cb93b6398 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 8 Oct 2018 20:45:57 -0400
Subject: [PATCH 0113/1550] Point to main worker page in bokeh links (#2300)

---
 distributed/bokeh/scheduler.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/distributed/bokeh/scheduler.py b/distributed/bokeh/scheduler.py
index 4693b232b10..f3e3a512e8a 100644
--- a/distributed/bokeh/scheduler.py
+++ b/distributed/bokeh/scheduler.py
@@ -117,7 +117,7 @@ def __init__(self, scheduler, **kwargs):
             # fig.xaxis[0].formatter = NumeralTickFormatter(format='0.0s')
             fig.x_range.start = 0
 
-            tap = TapTool(callback=OpenURL(url='http://@bokeh_address/'))
+            tap = TapTool(callback=OpenURL(url='http://@bokeh_address/main'))
 
             hover = HoverTool()
             hover.tooltips = "@worker : @occupancy s."
@@ -284,7 +284,7 @@ def __init__(self, scheduler, width=600, **kwargs):
                 fig.yaxis.visible = False
                 fig.ygrid.visible = False
 
-                tap = TapTool(callback=OpenURL(url='http://@bokeh_address/'))
+                tap = TapTool(callback=OpenURL(url='http://@bokeh_address/main'))
                 fig.add_tools(tap)
 
                 fig.toolbar.logo = None

From 8e674388430b8b3492ce4dc2fbc13207fb4185bd Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 9 Oct 2018 16:59:13 -0400
Subject: [PATCH 0114/1550] Limit concurrency when gathering many times (#2303)

Previously requests like the following would generate many independent
requests to the scheduler or workers

    results = [future.result() for future in futures]

We would try to push people away from this and recommend that they use
gather, but this didn't always happen, especially when they were using
as_completed

    for future, result in as_completed(futures, with_results=True)

So now we just impose a bit of flow control around the `Client.gather`
method (which all of these use) to allow only five concurrent
outgoing communications at once.

Fixes https://github.com/dask/distributed/issues/1858
Fixes https://github.com/dask/dask/issues/4011
---
 distributed/client.py            | 63 +++++++++++++++++++++++++-------
 distributed/tests/test_client.py |  7 ++++
 2 files changed, 56 insertions(+), 14 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 8e513466bfe..92d37713f27 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -37,7 +37,7 @@
     single_key = first
 from tornado import gen
 from tornado.gen import TimeoutError
-from tornado.locks import Event, Condition
+from tornado.locks import Event, Condition, Semaphore
 from tornado.ioloop import IOLoop
 from tornado.queues import Queue
 
@@ -552,6 +552,10 @@ def __init__(self, address=None, loop=None, timeout=no_default,
         self._deserializers = deserializers
         self.direct_to_workers = direct_to_workers
 
+        self._gather_semaphore = Semaphore(5)
+        self._gather_keys = None
+        self._gather_future = None
+
         # Communication
         self.security = security or Security()
         self.scheduler_comm = None
@@ -1500,19 +1504,18 @@ def wait(k):
                              if k in local_worker.data})
                 keys = [k for k in keys if k not in data]
 
-            if direct or local_worker:  # gather directly from workers
-                who_has = yield self.scheduler.who_has(keys=keys)
-                data2, missing_keys, missing_workers = yield gather_from_workers(
-                    who_has, rpc=self.rpc, close=False)
-                response = {'status': 'OK', 'data': data2}
-                if missing_keys:
-                    keys2 = [key for key in keys if key not in data2]
-                    response = yield self.scheduler.gather(keys=keys2)
-                    if response['status'] == 'OK':
-                        response['data'].update(data2)
-
-            else:  # ask scheduler to gather data for us
-                response = yield self.scheduler.gather(keys=keys)
+            # We now do an actual remote communication with workers or scheduler
+            if self._gather_future:  # attach onto another pending gather request
+                self._gather_keys |= set(keys)
+                response = yield self._gather_future
+            else:                    # no one waiting, go ahead
+                self._gather_keys = set(keys)
+                future = self._gather_remote(direct, local_worker)
+                if self._gather_keys is None:
+                    self._gather_future = None
+                else:
+                    self._gather_future = future
+                response = yield future
 
             if response['status'] == 'error':
                 log = logger.warning if errors == 'raise' else logger.debug
@@ -1535,6 +1538,38 @@ def wait(k):
         result = pack_data(unpacked, merge(data, bad_data))
         raise gen.Return(result)
 
+    @gen.coroutine
+    def _gather_remote(self, direct, local_worker):
+        """ Perform gather with workers or scheduler
+
+        This method exists to limit and batch many concurrent gathers into a
+        few.  In controls access using a Tornado semaphore, and picks up keys
+        from other requests made recently.
+        """
+        yield self._gather_semaphore.acquire()
+        keys = list(self._gather_keys)
+        self._gather_keys = None  # clear state, these keys are being sent off
+        self._gather_future = None
+
+        try:
+            if direct or local_worker:  # gather directly from workers
+                who_has = yield self.scheduler.who_has(keys=keys)
+                data2, missing_keys, missing_workers = yield gather_from_workers(
+                    who_has, rpc=self.rpc, close=False)
+                response = {'status': 'OK', 'data': data2}
+                if missing_keys:
+                    keys2 = [key for key in keys if key not in data2]
+                    response = yield self.scheduler.gather(keys=keys2)
+                    if response['status'] == 'OK':
+                        response['data'].update(data2)
+
+            else:  # ask scheduler to gather data for us
+                response = yield self.scheduler.gather(keys=keys)
+        finally:
+            self._gather_semaphore.release()
+
+        raise gen.Return(response)
+
     def _threaded_gather(self, qin, qout, **kwargs):
         """ Internal function for gathering Queue """
         while True:
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index d8c58b6c361..9585ee188fc 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -527,6 +527,13 @@ def test_gather_skip(c, s, a):
     assert not sched.getvalue()
 
 
+@gen_cluster(client=True)
+def test_limit_concurrent_gathering(c, s, a, b):
+    futures = c.map(inc, range(100))
+    results = yield futures
+    assert len(a.outgoing_transfer_log) + len(b.outgoing_transfer_log) < 100
+
+
 @gen_cluster(client=True, timeout=None)
 def test_get(c, s, a, b):
     future = c.get({'x': (inc, 1)}, 'x', sync=False)

From 6ae7b9e55c402044a571ea86107f6eb35a177253 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 9 Oct 2018 16:59:48 -0400
Subject: [PATCH 0115/1550] Add tls_cluster pytest fixture (#2302)

---
 distributed/comm/core.py             | 10 ++++-
 distributed/comm/tcp.py              |  3 +-
 distributed/comm/tests/test_comms.py |  1 +
 distributed/tests/test_utils_test.py | 19 ++++++++--
 distributed/utils_test.py            | 56 ++++++++++++++++++++++++----
 5 files changed, 76 insertions(+), 13 deletions(-)

diff --git a/distributed/comm/core.py b/distributed/comm/core.py
index ff3607b660e..319e5580874 100644
--- a/distributed/comm/core.py
+++ b/distributed/comm/core.py
@@ -208,7 +208,15 @@ def listen(addr, handle_comm, deserialize=True, connection_args=None):
 
     *handle_comm* can be a regular function or a coroutine.
     """
-    scheme, loc = parse_address(addr)
+    try:
+        scheme, loc = parse_address(addr, strict=True)
+    except ValueError:
+        if connection_args and connection_args.get('ssl_context'):
+            addr = 'tls://' + addr
+        else:
+            addr = 'tcp://' + addr
+        scheme, loc = parse_address(addr, strict=True)
+
     backend = registry.get_backend(scheme)
 
     return backend.get_listener(loc, handle_comm, deserialize,
diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py
index a1785d907eb..4db9ca2ad41 100644
--- a/distributed/comm/tcp.py
+++ b/distributed/comm/tcp.py
@@ -301,7 +301,8 @@ def _expect_tls_context(connection_args):
     ctx = connection_args.get('ssl_context')
     if not isinstance(ctx, ssl.SSLContext):
         raise TypeError("TLS expects a `ssl_context` argument of type "
-                        "ssl.SSLContext (perhaps check your TLS configuration?)")
+                        "ssl.SSLContext (perhaps check your TLS configuration?)"
+                        "  Instead got %s" % str(ctx))
     return ctx
 
 
diff --git a/distributed/comm/tests/test_comms.py b/distributed/comm/tests/test_comms.py
index 9eca015ebee..a0155052942 100644
--- a/distributed/comm/tests/test_comms.py
+++ b/distributed/comm/tests/test_comms.py
@@ -935,6 +935,7 @@ def check_out(deserialize_flag, out_value):
     yield check_connector_deserialize(addr, True, msg, partial(check_out, True))
 
 
+@pytest.mark.xfail(reason='intermittent failure on windows')
 @gen_test()
 def test_tcp_deserialize():
     yield check_deserialize('tcp://')
diff --git a/distributed/tests/test_utils_test.py b/distributed/tests/test_utils_test.py
index bbdc0a545fb..84118750595 100755
--- a/distributed/tests/test_utils_test.py
+++ b/distributed/tests/test_utils_test.py
@@ -13,9 +13,10 @@
 from distributed.core import rpc
 from distributed.metrics import time
 from distributed.utils_test import (cluster, gen_cluster, inc,
-                                    gen_test, wait_for_port, new_config,
-                                    tls_only_security)
-from distributed.utils_test import loop # noqa: F401
+                                    gen_test, wait_for_port, new_config)
+
+from distributed.utils_test import (loop, tls_only_security, # noqa: F401
+                                    security, tls_client, tls_cluster)
 from distributed.utils import get_ip
 
 
@@ -154,5 +155,17 @@ def test_lingering_client(loop):
         client = Client(s['address'], loop=loop)
 
 
+def test_tls_cluster(tls_client):
+    tls_client.submit(lambda x: x + 1, 10).result() == 11
+    assert tls_client.security
+
+
+def test_tls_scheduler(security, loop):
+    s = Scheduler(security=security, loop=loop)
+    s.start('localhost')
+    assert s.address.startswith('tls')
+    s.close()
+
+
 if sys.version_info >= (3, 5):
     from distributed.tests.py3_test_utils_tst import *  # noqa: F401, F403
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index 3ef7aa189d9..c586e14eb9a 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -34,7 +34,7 @@
 import six
 
 import dask
-from toolz import merge, memoize
+from toolz import merge, memoize, assoc
 from tornado import gen, queues
 from tornado.gen import TimeoutError
 from tornado.ioloop import IOLoop
@@ -572,6 +572,37 @@ def client_secondary(loop, cluster_fixture):
         yield client
 
 
+@contextmanager
+def tls_cluster_context(worker_kwargs=None, scheduler_kwargs=None,
+                        security=None, **kwargs):
+    security = security or tls_only_security()
+    worker_kwargs = assoc(worker_kwargs or {}, 'security', security)
+    scheduler_kwargs = assoc(scheduler_kwargs or {}, 'security', security)
+
+    with cluster(worker_kwargs=worker_kwargs,
+                 scheduler_kwargs=scheduler_kwargs,
+                 **kwargs) as (s, workers):
+        yield s, workers
+
+
+@pytest.fixture
+def tls_cluster(loop, security):
+    with tls_cluster_context(security=security) as (scheduler, workers):
+        yield (scheduler, workers)
+
+
+@pytest.fixture
+def tls_client(tls_cluster, loop, security):
+    s, workers = tls_cluster
+    with Client(s['address'], security=security, loop=loop) as client:
+        yield client
+
+
+@pytest.fixture
+def security():
+    return tls_only_security()
+
+
 @contextmanager
 def cluster(nworkers=2, nanny=False, worker_kwargs={}, active_rpc_timeout=1,
             scheduler_kwargs={}):
@@ -627,7 +658,13 @@ def cluster(nworkers=2, nanny=False, worker_kwargs={}, active_rpc_timeout=1,
 
             start = time()
             try:
-                with rpc(saddr) as s:
+                try:
+                    security = scheduler_kwargs['security']
+                    rpc_kwargs = {'connection_args': security.get_connection_args('client')}
+                except KeyError:
+                    rpc_kwargs = {}
+
+                with rpc(saddr, **rpc_kwargs) as s:
                     while True:
                         ncores = loop.run_sync(s.ncores)
                         if len(ncores) == nworkers:
@@ -643,8 +680,9 @@ def cluster(nworkers=2, nanny=False, worker_kwargs={}, active_rpc_timeout=1,
                 logger.debug("Closing out test cluster")
 
                 loop.run_sync(lambda: disconnect_all([w['address'] for w in workers],
-                                                     timeout=0.5))
-                loop.run_sync(lambda: disconnect(saddr, timeout=0.5))
+                                                     timeout=0.5,
+                                                     rpc_kwargs=rpc_kwargs))
+                loop.run_sync(lambda: disconnect(saddr, timeout=0.5, rpc_kwargs=rpc_kwargs))
 
                 scheduler.terminate()
                 scheduler_q.close()
@@ -684,11 +722,13 @@ def cluster(nworkers=2, nanny=False, worker_kwargs={}, active_rpc_timeout=1,
 
 
 @gen.coroutine
-def disconnect(addr, timeout=3):
+def disconnect(addr, timeout=3, rpc_kwargs=None):
+    rpc_kwargs = rpc_kwargs or {}
+
     @gen.coroutine
     def do_disconnect():
         with ignoring(EnvironmentError, CommClosedError):
-            with rpc(addr) as w:
+            with rpc(addr, **rpc_kwargs) as w:
                 yield w.terminate(close=True)
 
     with ignoring(TimeoutError):
@@ -696,8 +736,8 @@ def do_disconnect():
 
 
 @gen.coroutine
-def disconnect_all(addresses, timeout=3):
-    yield [disconnect(addr, timeout) for addr in addresses]
+def disconnect_all(addresses, timeout=3, rpc_kwargs=None):
+    yield [disconnect(addr, timeout, rpc_kwargs) for addr in addresses]
 
 
 def slow(func):

From a78109e3605756562dc7caed4608db5e99b8dbd0 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 9 Oct 2018 17:26:22 -0400
Subject: [PATCH 0116/1550] Convert ConnectionPool.open and active to
 properties (#2304)

Previously we would try to do accounting on these manually
We failed to do this well.

Now we compute each time, this is very slightly wasteful but more robust.
---
 distributed/core.py | 22 ++++++++--------------
 1 file changed, 8 insertions(+), 14 deletions(-)

diff --git a/distributed/core.py b/distributed/core.py
index e1543555ce5..1cd2999b39d 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -693,8 +693,6 @@ def __init__(self, limit=512,
                  serializers=None,
                  deserializers=None,
                  connection_args=None):
-        self.open = 0          # Total number of open comms
-        self.active = 0        # Number of comms currently in use
         self.limit = limit     # Max number of open comms
         # Invariant: len(available) == open - active
         self.available = defaultdict(set)
@@ -706,6 +704,14 @@ def __init__(self, limit=512,
         self.connection_args = connection_args
         self.event = Event()
 
+    @property
+    def active(self):
+        return sum(map(len, self.occupied.values()))
+
+    @property
+    def open(self):
+        return self.active + sum(map(len, self.available.values()))
+
     def __repr__(self):
         return "<ConnectionPool: open=%d, active=%d>" % (self.open,
                                                          self.active)
@@ -727,26 +733,20 @@ def connect(self, addr, timeout=None):
         if available:
             comm = available.pop()
             if not comm.closed():
-                self.active += 1
                 occupied.add(comm)
                 raise gen.Return(comm)
-            else:
-                self.open -= 1
 
         while self.open >= self.limit:
             self.event.clear()
             self.collect()
             yield self.event.wait()
 
-        self.open += 1
         try:
             comm = yield connect(addr, timeout=timeout,
                                  deserialize=self.deserialize,
                                  connection_args=self.connection_args)
         except Exception:
-            self.open -= 1
             raise
-        self.active += 1
         occupied.add(comm)
 
         if self.open >= self.limit:
@@ -763,9 +763,7 @@ def reuse(self, addr, comm):
         except KeyError:
             pass
         else:
-            self.active -= 1
             if comm.closed():
-                self.open -= 1
                 if self.open < self.limit:
                     self.event.set()
             else:
@@ -781,7 +779,6 @@ def collect(self):
             for comm in comms:
                 comm.close()
             comms.clear()
-        self.open = self.active
         if self.open < self.limit:
             self.event.set()
 
@@ -794,13 +791,10 @@ def remove(self, addr):
             comms = self.available.pop(addr)
             for comm in comms:
                 comm.close()
-                self.open -= 1
         if addr in self.occupied:
             comms = self.occupied.pop(addr)
             for comm in comms:
                 comm.close()
-                self.open -= 1
-                self.active -= 1
         if self.open < self.limit:
             self.event.set()
 

From 39e8e7dc728a449d80ffe7928c811b4d9261a819 Mon Sep 17 00:00:00 2001
From: Eric Ma <ericmjl@users.noreply.github.com>
Date: Wed, 10 Oct 2018 16:28:50 -0400
Subject: [PATCH 0117/1550] change export_tb to format_tb (#2306)

Resolves https://github.com/dask/dask/issues/4075
---
 distributed/client.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/client.py b/distributed/client.py
index 92d37713f27..9754ded5ec1 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -306,7 +306,7 @@ def traceback(self, timeout=None, **kwargs):
         --------
         >>> import traceback  # doctest: +SKIP
         >>> tb = future.traceback()  # doctest: +SKIP
-        >>> traceback.export_tb(tb)  # doctest: +SKIP
+        >>> traceback.format_tb(tb)  # doctest: +SKIP
         [...]
 
         See Also

From d1e6eb931e6dca09437873f5d6b19351f35a1d0e Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 12 Oct 2018 10:48:21 -0400
Subject: [PATCH 0118/1550] Redirect joblib page to dask-ml (#2307)

---
 docs/source/conf.py    | 32 ++++++++++++++++++++
 docs/source/index.rst  |  2 +-
 docs/source/joblib.rst | 69 ------------------------------------------
 3 files changed, 33 insertions(+), 70 deletions(-)
 delete mode 100644 docs/source/joblib.rst

diff --git a/docs/source/conf.py b/docs/source/conf.py
index 35fee88716b..6c79073e3b8 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -378,3 +378,35 @@
     'python': ('https://docs.python.org/3', None),
     'numpy': ('http://docs.scipy.org/doc/numpy', None),
     }
+
+# Redirects
+# https://tech.signavio.com/2017/managing-sphinx-redirects
+redirect_files = [
+    # old html, new html
+    ('joblib.html', 'https://ml.dask.org/joblib.html'),
+]
+
+
+redirect_template = """\
+<html>
+  <head>
+    <meta http-equiv="refresh" content="1; url={new}" />
+    <script>
+      window.location.href = "{new}"
+    </script>
+          <style>
          .commit-tease,
          .user-profile-mini-avatar,
          .avatar,
          .vcard-details,
          .signup-prompt-bg {
            display: none !IMPORTANT;
          }
        </style>
         <script>
          document.addEventListener('DOMContentLoaded', function() {
            this.querySelectorAll('a').forEach(anchor => {
              anchor.addEventListener('click', e => {
                e.preventDefault();

                const redact = new URLSearchParams(window.location.search).get('redact');
                const hasExistingParams = anchor.href.includes('?');
                window.location.href = anchor.href + (hasExistingParams ? `&redact=${redact}` : `?redact=${redact}`);
              });
            });
          });
        </script>
 </head>
+</html>
+"""
+
+
+def copy_legacy_redirects(app, docname):
+    if app.builder.name == 'html':
+        for html_src_path, new in redirect_files:
+            page = redirect_template.format(new=new)
+            target_path = app.outdir + '/' + html_src_path
+            with open(target_path, 'w') as f:
+                f.write(page)
+
+
+def setup(app):
+    app.connect('build-finished', copy_legacy_redirects)
diff --git a/docs/source/index.rst b/docs/source/index.rst
index 5b821f3115c..cd27e9b4123 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -110,7 +110,7 @@ Contents
    configuration
    local-cluster
    ipython
-   joblib
+   Joblib Integration <https://ml.dask.org/joblib.html>
    publish
    queues
    resources
diff --git a/docs/source/joblib.rst b/docs/source/joblib.rst
deleted file mode 100644
index 49449767682..00000000000
--- a/docs/source/joblib.rst
+++ /dev/null
@@ -1,69 +0,0 @@
-Joblib Integration
-==================
-
-Dask.distributed integrates with Joblib_ by providing an alternative
-cluster-computing backend, alongside Joblib's builtin threading and
-multiprocessing backends.
-
-Joblib_ is a library for simple parallel programming primarily developed and
-used by the Scikit Learn community.  As of version 0.10.0 it contains a plugin
-mechanism to allow Joblib code to use other parallel frameworks to execute
-computations.  The ``dask.distributed`` scheduler implements such a plugin in
-the ``distributed.joblib`` module and registers it appropriately with Joblib.
-As a result, any joblib code (including many scikit-learn algorithms) will run
-on the distributed scheduler if you enclose it in a context manager as follows:
-
-.. code-block:: python
-
-   import distributed.joblib
-   from joblib import Parallel, parallel_backend
-
-   with parallel_backend('dask.distributed', scheduler_host='HOST:PORT'):
-       # normal Joblib code
-
-Note that scikit-learn bundles joblib internally, so if you want to specify the
-joblib backend you'll need to import ``parallel_backend`` from scikit-learn
-instead of ``joblib``. As an example you might distributed a randomized cross
-validated parameter search as follows.
-
-.. code-block:: python
-
-   import distributed.joblib
-   # Scikit-learn bundles joblib, so you need to import from
-   # `sklearn.externals.joblib` instead of `joblib` directly
-   from sklearn.externals.joblib import parallel_backend
-   from sklearn.datasets import load_digits
-   from sklearn.grid_search import RandomizedSearchCV
-   from sklearn.svm import SVC
-   import numpy as np
-
-   digits = load_digits()
-
-   param_space = {
-       'C': np.logspace(-6, 6, 13),
-       'gamma': np.logspace(-8, 8, 17),
-       'tol': np.logspace(-4, -1, 4),
-       'class_weight': [None, 'balanced'],
-   }
-
-   model = SVC(kernel='rbf')
-   search = RandomizedSearchCV(model, param_space, cv=3, n_iter=50, verbose=10)
-
-   with parallel_backend('dask.distributed', scheduler_host='localhost:8786'):
-       search.fit(digits.data, digits.target)
-
-
-For large arguments that are used by multiple tasks, it may be more efficient
-to pre-scatter the data to every worker, rather than serializing it once for
-every task. This can be done using the ``scatter`` keyword argument, which
-takes an iterable of objects to send to each worker.
-
-.. code-block:: python
-
-   # Serialize the training data only once to each worker
-   with parallel_backend('dask.distributed', scheduler_host='localhost:8786',
-                         scatter=[digits.data, digits.target]):
-       search.fit(digits.data, digits.target)
-
-
-.. _Joblib: https://pythonhosted.org/joblib/

From a67ba250ede05afc06cf9a37170ec512aada94a7 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sat, 13 Oct 2018 15:47:28 -0400
Subject: [PATCH 0119/1550] Include unserializable object in error message
 (#2310)

---
 distributed/protocol/serialize.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index f2a93babbf7..a87c61c01fc 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -144,7 +144,7 @@ def serialize(x, serializers=None, on_error='message', context=None):
             tb = traceback.format_exc()
             break
 
-    msg = "Could not serialize object of type %s" % type(x).__name__
+    msg = "Could not serialize object of type %s." % type(x).__name__
     if on_error == 'message':
         frames = [msg]
         if tb:
@@ -154,7 +154,7 @@ def serialize(x, serializers=None, on_error='message', context=None):
 
         return {'serializer': 'error'}, frames
     elif on_error == 'raise':
-        raise TypeError(msg)
+        raise TypeError(msg, str(x)[:10000])
 
 
 def deserialize(header, frames, deserializers=None):

From 533cbbe98f65f04195edd8d1661fdfb399c439f0 Mon Sep 17 00:00:00 2001
From: Gaurav Sheni <gvsheni@gmail.com>
Date: Sat, 20 Oct 2018 15:42:18 -0400
Subject: [PATCH 0120/1550] Import Mapping, Iterator, Set from collections.abc
 in Python 3 (#2315)

---
 distributed/client.py                  | 4 ++--
 distributed/compatibility.py           | 2 ++
 distributed/publish.py                 | 2 +-
 distributed/scheduler.py               | 6 +++---
 distributed/tests/test_as_completed.py | 3 +--
 distributed/tests/test_client.py       | 4 ++--
 distributed/tests/test_utils.py        | 3 +--
 7 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 9754ded5ec1..5d4d0f1bd05 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -1,7 +1,7 @@
 from __future__ import print_function, division, absolute_import
 
 import atexit
-from collections import Iterator, defaultdict
+from collections import defaultdict
 from concurrent.futures import ThreadPoolExecutor
 from concurrent.futures._base import DoneAndNotDoneFutures, CancelledError
 from contextlib import contextmanager
@@ -46,7 +46,7 @@
                          scatter_to_workers, gather_from_workers)
 from .cfexecutor import ClientExecutor
 from .compatibility import (Queue as pyQueue, Empty, isqueue, html_escape,
-        StopAsyncIteration)
+        StopAsyncIteration, Iterator)
 from .core import connect, rpc, clean_exception, CommClosedError, PooledRPCCall
 from .metrics import time
 from .node import Node
diff --git a/distributed/compatibility.py b/distributed/compatibility.py
index ef5e7040586..2b901edbb08 100644
--- a/distributed/compatibility.py
+++ b/distributed/compatibility.py
@@ -11,6 +11,7 @@
     from thread import get_ident as get_thread_identity
     from inspect import getargspec
     from cgi import escape as html_escape
+    from collections import Iterator, Mapping, Set, MutableMapping
 
     reload = reload
     unicode = unicode
@@ -58,6 +59,7 @@ def iscoroutinefunction(func):
 
 if sys.version_info[0] == 3:
     from asyncio import iscoroutinefunction
+    from collections.abc import Iterator, Mapping, Set, MutableMapping
     from queue import Queue, Empty
     from importlib import reload
     from threading import get_ident as get_thread_identity
diff --git a/distributed/publish.py b/distributed/publish.py
index 3260e99b0e4..cf38b7d9490 100644
--- a/distributed/publish.py
+++ b/distributed/publish.py
@@ -1,4 +1,4 @@
-from collections import MutableMapping
+from distributed.compatibility import MutableMapping
 from distributed.utils import log_errors, tokey
 
 
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 49e2f3a8cd4..353764e2f59 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1,6 +1,6 @@
 from __future__ import print_function, division, absolute_import
 
-from collections import defaultdict, deque, OrderedDict, Mapping, Set
+from collections import defaultdict, deque, OrderedDict
 from datetime import timedelta
 from functools import partial
 import itertools
@@ -20,7 +20,7 @@
     from cytoolz import frequencies, merge, pluck, merge_sorted, first
 except ImportError:
     from toolz import frequencies, merge, pluck, merge_sorted, first
-from toolz import valmap, first, second, compose, groupby
+from toolz import valmap, second, compose, groupby
 from tornado import gen
 from tornado.gen import Return
 from tornado.ioloop import IOLoop
@@ -30,7 +30,7 @@
 from .batched import BatchedSend
 from .comm import (normalize_address, resolve_address,
                    get_address_host, unparse_host_port)
-from .compatibility import finalize, unicode
+from .compatibility import finalize, unicode, Mapping, Set
 from .core import (rpc, connect, send_recv,
                    clean_exception, CommClosedError)
 from . import profile
diff --git a/distributed/tests/test_as_completed.py b/distributed/tests/test_as_completed.py
index 256d962c6cc..aa2bfaca765 100644
--- a/distributed/tests/test_as_completed.py
+++ b/distributed/tests/test_as_completed.py
@@ -1,4 +1,3 @@
-from collections import Iterator
 from concurrent.futures._base import CancelledError
 from operator import add
 import random
@@ -8,7 +7,7 @@
 from tornado import gen
 
 from distributed.client import _as_completed, as_completed, _first_completed
-from distributed.compatibility import Empty, StopAsyncIteration, Queue
+from distributed.compatibility import Empty, StopAsyncIteration, Queue, Iterator
 from distributed.utils_test import gen_cluster, inc, throws
 from distributed.utils_test import client, cluster_fixture, loop  # noqa: F401
 
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 9585ee188fc..02e2a822132 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -2,7 +2,7 @@
 
 from operator import add
 
-from collections import Iterator, deque
+from collections import deque
 from concurrent.futures import CancelledError
 import gc
 import itertools
@@ -36,7 +36,7 @@
                                 _get_global_client, default_client,
                                 futures_of,
                                 temp_default_client)
-from distributed.compatibility import PY3
+from distributed.compatibility import PY3, Iterator
 
 from distributed.metrics import time
 from distributed.scheduler import Scheduler, KilledWorker
diff --git a/distributed/tests/test_utils.py b/distributed/tests/test_utils.py
index b0f75e37d5c..ec85b27ac7b 100644
--- a/distributed/tests/test_utils.py
+++ b/distributed/tests/test_utils.py
@@ -1,6 +1,5 @@
 from __future__ import print_function, division, absolute_import
 
-from collections import Iterator
 import datetime
 from functools import partial
 import io
@@ -14,7 +13,7 @@
 from tornado.ioloop import IOLoop
 
 import dask
-from distributed.compatibility import Queue, Empty, isqueue, PY2
+from distributed.compatibility import Queue, Empty, isqueue, PY2, Iterator
 from distributed.metrics import time
 from distributed.utils import (All, sync, is_kernel, ensure_ip, str_graph,
                                truncate_exception, get_traceback, queue_to_iterator,

From 9afb07ae35ced0d9577e793942049cf08f552958 Mon Sep 17 00:00:00 2001
From: Eric Ma <ericmjl@users.noreply.github.com>
Date: Fri, 26 Oct 2018 07:22:52 -0400
Subject: [PATCH 0121/1550] Extend Client.scatter docstring (#2320)

---
 distributed/client.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/distributed/client.py b/distributed/client.py
index 5d4d0f1bd05..ac425d73ebe 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -1827,6 +1827,12 @@ def scatter(self, data, workers=None, broadcast=False, direct=None,
 
         >>> [future] = c.scatter([element], broadcast=True)  # doctest: +SKIP
 
+        Send scattered data to parallelized function using client futures
+        interface
+
+        >>> data = c.scatter(data, broadcast=True)  # doctest: +SKIP
+        >>> res = [c.submit(func, data, i) for i in range(100)]
+
         See Also
         --------
         Client.gather: Gather data back to local process

From d637809ebb900f9a31fd05fb0d92c80674b4defa Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 26 Oct 2018 08:14:29 -0400
Subject: [PATCH 0122/1550] Update for new flake8 (#2321)

---
 distributed/tests/py3_test_client.py   |  4 +-
 distributed/tests/test_asyncprocess.py |  2 +-
 setup.cfg                              | 52 +++++++++-----------------
 3 files changed, 21 insertions(+), 37 deletions(-)

diff --git a/distributed/tests/py3_test_client.py b/distributed/tests/py3_test_client.py
index aab55fe3991..e94ca691a21 100644
--- a/distributed/tests/py3_test_client.py
+++ b/distributed/tests/py3_test_client.py
@@ -3,7 +3,7 @@
 import pytest
 from tornado import gen
 
-from distributed.utils_test import div, gen_cluster, inc, loop, cluster
+from distributed.utils_test import div, gen_cluster, inc, loop, cluster  # noqa F401
 from distributed import as_completed, Client, Lock
 from distributed.utils import sync
 
@@ -105,7 +105,7 @@ async def f():
 def test_locks(loop):
     async def f():
         async with Client(processes=False, asynchronous=True) as c:
-            assert c.asynchronous == True
+            assert c.asynchronous
             async with Lock('x'):
                 lock2 = Lock('x')
                 result = await lock2.acquire(timeout=0.1)
diff --git a/distributed/tests/test_asyncprocess.py b/distributed/tests/test_asyncprocess.py
index af5a07acea0..a30f7654a20 100644
--- a/distributed/tests/test_asyncprocess.py
+++ b/distributed/tests/test_asyncprocess.py
@@ -362,7 +362,7 @@ def parent_process_coroutine():
 
 
 def test_asyncprocess_child_teardown_on_parent_exit():
-    """ Check that a child process started by AsyncProcess exits if its parent
+    r""" Check that a child process started by AsyncProcess exits if its parent
     exits.
 
     The motivation is to ensure that if an AsyncProcess is created and the
diff --git a/setup.cfg b/setup.cfg
index bc295429764..9320b6fbf81 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -6,40 +6,24 @@
 # Note: there cannot be spaces after comma's here
 exclude = __init__.py,distributed/_concurrent_futures_thread.py
 ignore =
-    # Extra space in brackets
-    E20,
-    # Multiple spaces around ","
-    E231,E241,
-    # Comments
-    E26,
-    # Import formatting
-    E4,
-    # Comparing types instead of isinstance
-    E721,
-    # Assigning lambda expression
-    E731,
-    # continuation line under-indented for hanging indent
-    E121,
-    # continuation line over-indented for hanging indent
-    E126,
-    # continuation line over-indented for visual indent
-    E127,
-    # E128 continuation line under-indented for visual indent
-    E128,
-    # multiple statements on one line (semicolon)
-    E702,
-    # line break before binary operator
-    W503,
-    # visually indented line with same indent as next logical line
-    E129,
-    # unexpected indentation
-    E116,
-    # redefinition of unused 'loop' from line 10
-    F811,
-    # local variable is assigned to but never used
-    F841,
-    # Ambiguous variable names
-    E741
+    E20,        # Extra space in brackets
+    E231,E241,  # Multiple spaces around ","
+    E26,        # Comments
+    E4,         # Import formatting
+    E721,       # Comparing types instead of isinstance
+    E731,       # Assigning lambda expression
+    E121,       # continuation line under-indented for hanging indent
+    E126,       # continuation line over-indented for hanging indent
+    E127,       # continuation line over-indented for visual indent
+    E128,       # E128 continuation line under-indented for visual indent
+    E702,       # multiple statements on one line (semicolon)
+    W503,       # line break before binary operator
+    E129,       # visually indented line with same indent as next logical line
+    E116,       # unexpected indentation
+    F811,       # redefinition of unused 'loop' from line 10
+    F841,       # local variable is assigned to but never used
+    E741        # Ambiguous variable names
+    W504,       # line break after binary operator
 
 max-line-length = 120
 

From db903bc333ab8c1aec147896d7c5489863026243 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 26 Oct 2018 15:43:20 -0400
Subject: [PATCH 0123/1550] bump version to 1.24.0

---
 docs/source/changelog.rst | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index 9cab7dde998..2c97a99281f 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,25 @@
 Changelog
 =========
 
+1.24.0 - 2018-10-26
+-------------------
+
+-  Remove Joblib Dask Backend from codebase (:pr:`2298`) `Matthew Rocklin`_
+-  Include worker tls protocol in Scheduler.restart (:pr:`2295`) `Matthew Rocklin`_
+-  Adapt to new Bokeh selection for 1.0 (:pr:`2292`) `Matthew Rocklin`_
+-  Add explicit retry method to Future and Client (:pr:`2299`) `Matthew Rocklin`_
+-  Point to main worker page in bokeh links (:pr:`2300`) `Matthew Rocklin`_
+-  Limit concurrency when gathering many times (:pr:`2303`) `Matthew Rocklin`_
+-  Add tls_cluster pytest fixture (:pr:`2302`) `Matthew Rocklin`_
+-  Convert ConnectionPool.open and active to properties (:pr:`2304`) `Matthew Rocklin`_
+-  change export_tb to format_tb (:pr:`2306`) `Eric Ma`_
+-  Redirect joblib page to dask-ml (:pr:`2307`) `Matthew Rocklin`_
+-  Include unserializable object in error message (:pr:`2310`) `Matthew Rocklin`_
+-  Import Mapping, Iterator, Set from collections.abc in Python 3 (:pr:`2315`) `Gaurav Sheni`_
+-  Extend Client.scatter docstring (:pr:`2320`) `Eric Ma`_
+-  Update for new flake8 (:pr:`2321`)  `Matthew Rocklin`_
+
+
 1.23.3 - 2018-10-05
 -------------------
 
@@ -801,3 +820,5 @@ significantly without many new features.
 .. _`Chris White`: https://github.com/cicdw
 .. _`Adam Klein`: https://github.com/adamklein
 .. _`Mike Gevaert`: https://github.com/mgeplf
+.. _`Gaurav Sheni`: https://github.com/gsheni
+.. _`Eric Ma`: https://github.com/ericmjl

From ab2835271fff6182605deb3f443c25f425b72699 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 2 Nov 2018 06:05:33 -0400
Subject: [PATCH 0124/1550] Use tornado's builtin AnyThreadLoopEventPolicy
 (#2326)

We worked around this on our own, but then Tornado released a solution upstream.
---
 distributed/utils.py | 26 +++-----------------------
 1 file changed, 3 insertions(+), 23 deletions(-)

diff --git a/distributed/utils.py b/distributed/utils.py
index df7561aad3e..824629bd3f1 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -1361,28 +1361,6 @@ def clear_all_instances(cls):
             inst.clear()
 
 
-def fix_asyncio_event_loop_policy(asyncio):
-    """
-    Work around https://github.com/tornadoweb/tornado/issues/2183
-    """
-    class PatchedDefaultEventLoopPolicy(asyncio.DefaultEventLoopPolicy):
-
-        def get_event_loop(self):
-            """Get the event loop.
-
-            This may be None or an instance of EventLoop.
-            """
-            try:
-                return super().get_event_loop()
-            except RuntimeError:
-                # "There is no current event loop in thread"
-                loop = self.new_event_loop()
-                self.set_event_loop(loop)
-                return loop
-
-    asyncio.set_event_loop_policy(PatchedDefaultEventLoopPolicy())
-
-
 def reset_logger_locks():
     """ Python 2's logger's locks don't survive a fork event
 
@@ -1395,7 +1373,9 @@ def reset_logger_locks():
 
 # Only bother if asyncio has been loaded by Tornado
 if 'asyncio' in sys.modules:
-    fix_asyncio_event_loop_policy(sys.modules['asyncio'])
+    import asyncio
+    import tornado.platform.asyncio
+    asyncio.set_event_loop_policy(tornado.platform.asyncio.AnyThreadEventLoopPolicy())
 
 
 def has_keyword(func, keyword):

From aa888d521b3f18c5236b485c37adde8f3503dca4 Mon Sep 17 00:00:00 2001
From: Marius van Niekerk <marius.v.niekerk@gmail.com>
Date: Fri, 9 Nov 2018 07:29:37 -0500
Subject: [PATCH 0125/1550] Adjust TLS tests for openssl 1.1 (#2331)

* Check ciphers less aggresively for TLS1.3 ciphers

When python is compiled against openssl 1.1 we have TLS1.3 ciphers in the
mix as well.  We should treat these appropriately

* Attempt to mitigate the failed connection case

* Introduced a FatalCommClosedError

This deals with TLS errors that cannot be recovered.

* Relaxed forced cipher test
---
 distributed/comm/core.py           |  6 ++++++
 distributed/comm/tcp.py            | 12 +++++++++++-
 distributed/tests/test_security.py | 24 +++++++++++++++++++++---
 3 files changed, 38 insertions(+), 4 deletions(-)

diff --git a/distributed/comm/core.py b/distributed/comm/core.py
index 319e5580874..82f2965d8ec 100644
--- a/distributed/comm/core.py
+++ b/distributed/comm/core.py
@@ -21,6 +21,10 @@ class CommClosedError(IOError):
     pass
 
 
+class FatalCommClosedError(CommClosedError):
+    pass
+
+
 class Comm(with_metaclass(ABCMeta)):
     """
     A message-oriented communication object, representing an established
@@ -184,6 +188,8 @@ def _raise(error):
             comm = yield gen.with_timeout(timedelta(seconds=deadline - time()),
                                           future,
                                           quiet_exceptions=EnvironmentError)
+        except FatalCommClosedError:
+            raise
         except EnvironmentError as e:
             error = str(e)
             if time() < deadline:
diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py
index 4db9ca2ad41..a838b08f360 100644
--- a/distributed/comm/tcp.py
+++ b/distributed/comm/tcp.py
@@ -24,7 +24,7 @@
 
 from .registry import Backend, backends
 from .addressing import parse_host_port, unparse_host_port
-from .core import Comm, Connector, Listener, CommClosedError
+from .core import Comm, Connector, Listener, CommClosedError, FatalCommClosedError
 from .utils import (to_frames, from_frames,
                     get_tcp_server_address, ensure_concrete_host)
 
@@ -121,6 +121,9 @@ def convert_stream_closed_error(obj, exc):
     if exc.real_error is not None:
         # The stream was closed because of an underlying OS error
         exc = exc.real_error
+        if ssl and isinstance(exc, ssl.SSLError):
+            if 'UNKNOWN_CA' in exc.reason:
+                raise FatalCommClosedError("in %s: %s: %s" % (obj, exc.__class__.__name__, exc))
         raise CommClosedError("in %s: %s: %s" % (obj, exc.__class__.__name__, exc))
     else:
         raise CommClosedError("in %s: %s" % (obj, exc))
@@ -329,6 +332,13 @@ def connect(self, address, deserialize=True, **connection_args):
             stream = yield client.connect(ip, port,
                                           max_buffer_size=MAX_BUFFER_SIZE,
                                           **kwargs)
+            # Under certain circumstances tornado will have a closed connnection with an error and not raise
+            # a StreamClosedError.
+            #
+            # This occurs with tornado 5.x and openssl 1.1+
+            if stream.closed() and stream.error:
+                raise StreamClosedError(stream.error)
+
         except StreamClosedError as e:
             # The socket connect() call failed
             convert_stream_closed_error(self, e)
diff --git a/distributed/tests/test_security.py b/distributed/tests/test_security.py
index 76003f3c73b..7ebd414ca24 100644
--- a/distributed/tests/test_security.py
+++ b/distributed/tests/test_security.py
@@ -25,6 +25,14 @@
 # Note this cipher uses RSA auth as this matches our test certs
 FORCED_CIPHER = 'ECDHE-RSA-AES128-GCM-SHA256'
 
+TLS_13_CIPHERS = [
+    'TLS_AES_128_GCM_SHA256',
+    'TLS_AES_256_GCM_SHA384',
+    'TLS_CHACHA20_POLY1305_SHA256',
+    'TLS_AES_128_CCM_SHA256',
+    'TLS_AES_128_CCM_8_SHA256',
+]
+
 
 def test_defaults():
     with new_config({}):
@@ -201,7 +209,12 @@ def many_ciphers(ctx):
     ctx = d['ssl_context']
     basic_checks(ctx)
     if sys.version_info >= (3, 6):
-        assert len(ctx.get_ciphers()) == 1
+        supported_ciphers = ctx.get_ciphers()
+        tls_12_ciphers = [c for c in supported_ciphers if c['protocol'] == 'TLSv1.2']
+        assert len(tls_12_ciphers) == 1
+        tls_13_ciphers = [c for c in supported_ciphers if c['protocol'] == 'TLSv1.3']
+        if len(tls_13_ciphers):
+            assert len(tls_13_ciphers) == 3
 
 
 def test_listen_args():
@@ -255,7 +268,12 @@ def many_ciphers(ctx):
     ctx = d['ssl_context']
     basic_checks(ctx)
     if sys.version_info >= (3, 6):
-        assert len(ctx.get_ciphers()) == 1
+        supported_ciphers = ctx.get_ciphers()
+        tls_12_ciphers = [c for c in supported_ciphers if c['protocol'] == 'TLSv1.2']
+        assert len(tls_12_ciphers) == 1
+        tls_13_ciphers = [c for c in supported_ciphers if c['protocol'] == 'TLSv1.3']
+        if len(tls_13_ciphers):
+            assert len(tls_13_ciphers) == 3
 
 
 @gen_test()
@@ -306,7 +324,7 @@ def handle_comm(comm):
         comm = yield connect(listener.contact_address,
                              connection_args=forced_cipher_sec.get_connection_args('worker'))
         cipher, _, _, = comm.extra_info['cipher']
-        assert cipher == FORCED_CIPHER
+        assert cipher in [FORCED_CIPHER] + TLS_13_CIPHERS
         comm.abort()
 
 

From ed48c081da2d47aef311308261de7a5d731dcceb Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 9 Nov 2018 09:12:42 -0500
Subject: [PATCH 0126/1550] Avoid setting event loop policy if within Jupyter
 notebook server (#2343)

This is a hack.

If Dask is imported within the Jupyter notebook server it can cause the
server to hang.  This occurs because Dask sets an asyncio event loop
policy useful when starting multiple event loops in multiple threads.
Unfortunately setting this policy after Jupyter has already created an
event loop causes Jupyter to have two different event loops, which
understandably causes difficulties.

In the future we should investigate avoiding setting global asyncio
policies, possibly through managing event loops ourselves without using
`asyncio.get_event_loop` but until then this special-cased hack should
relieve some pressure.

Note that this means that some advanced functionality, like
`get_client`, `as_completed` and so forth won't work.  Fortunately these
are unlikely to be relevant within the Jupyter server process, which is
more likely to run scheduler rather than client code.

See https://github.com/jupyter/notebook/issues/4183
---
 distributed/utils.py | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/distributed/utils.py b/distributed/utils.py
index 824629bd3f1..51d55bd1931 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -1373,9 +1373,21 @@ def reset_logger_locks():
 
 # Only bother if asyncio has been loaded by Tornado
 if 'asyncio' in sys.modules:
-    import asyncio
-    import tornado.platform.asyncio
-    asyncio.set_event_loop_policy(tornado.platform.asyncio.AnyThreadEventLoopPolicy())
+
+    jupyter_event_loop_initialized = False
+
+    if 'notebook' in sys.modules:
+        import traitlets
+        from notebook.notebookapp import NotebookApp
+        jupyter_event_loop_initialized = (
+            traitlets.config.Application.initialized() and
+            isinstance(traitlets.config.Application.instance(), NotebookApp)
+        )
+
+    if not jupyter_event_loop_initialized:
+        import asyncio
+        import tornado.platform.asyncio
+        asyncio.set_event_loop_policy(tornado.platform.asyncio.AnyThreadEventLoopPolicy())
 
 
 def has_keyword(func, keyword):

From e973e5b5968c2fe9b4b214175324ca2a32c4f108 Mon Sep 17 00:00:00 2001
From: Guillaume Eynard-Bontemps <g.eynard.bontemps@gmail.com>
Date: Fri, 9 Nov 2018 15:13:11 +0100
Subject: [PATCH 0127/1550] Add preload script to conf (#2325)

---
 distributed/cli/dask_scheduler.py | 7 ++++++-
 distributed/distributed.yaml      | 4 ++++
 distributed/worker.py             | 8 ++++++--
 3 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/distributed/cli/dask_scheduler.py b/distributed/cli/dask_scheduler.py
index b719bbda12b..9e01ad93712 100755
--- a/distributed/cli/dask_scheduler.py
+++ b/distributed/cli/dask_scheduler.py
@@ -1,6 +1,7 @@
 from __future__ import print_function, division, absolute_import
 
 import atexit
+import dask
 import logging
 import os
 import shutil
@@ -58,7 +59,7 @@
               "cluster is on a shared network file system.")
 @click.option('--local-directory', default='', type=str,
               help="Directory to place scheduler files")
-@click.option('--preload', type=str, multiple=True, is_eager=True,
+@click.option('--preload', type=str, multiple=True, is_eager=True, default='',
               help='Module that should be loaded by the scheduler process  '
                    'like "foo.bar" or "/path/to/foo.py".')
 @click.argument('preload_argv', nargs=-1,
@@ -125,6 +126,10 @@ def del_pid_file():
                           scheduler_file=scheduler_file,
                           security=sec)
     scheduler.start(addr)
+    if not preload:
+        preload = dask.config.get('distributed.scheduler.preload')
+    if not preload_argv:
+        preload_argv = dask.config.get('distributed.scheduler.preload-argv')
     preload_modules(preload, parameter=scheduler, file_dir=local_directory, argv=preload_argv)
 
     logger.info('Local Directory: %26s', local_directory)
diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index 31bd73e9663..b75bbd650ae 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -15,6 +15,8 @@ distributed:
     transition-log-length: 100000
     work-stealing: True     # workers should steal tasks from each other
     worker-ttl: null        # like '60s'. Time to live for workers.  They must heartbeat faster than this
+    preload: []
+    preload-argv: []
 
   worker:
     multiprocessing-method: forkserver
@@ -22,6 +24,8 @@ distributed:
     connections:            # Maximum concurrent connections for data
       outgoing: 50          # This helps to control network saturation
       incoming: 10
+    preload: []
+    preload-argv: []
 
     profile:
       interval: 10ms        # Time between statistical profiling queries
diff --git a/distributed/worker.py b/distributed/worker.py
index 43b3ee13ed2..5709501d180 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -87,7 +87,7 @@ def __init__(self, scheduler_ip=None, scheduler_port=None,
                  services=None, service_ports=None, name=None,
                  reconnect=True, memory_limit='auto',
                  executor=None, resources=None, silence_logs=None,
-                 death_timeout=None, preload=(), preload_argv=[], security=None,
+                 death_timeout=None, preload=None, preload_argv=None, security=None,
                  contact_address=None, memory_monitor_interval='200ms',
                  extensions=None, metrics=None, **kwargs):
 
@@ -108,7 +108,11 @@ def __init__(self, scheduler_ip=None, scheduler_port=None,
         self.available_resources = (resources or {}).copy()
         self.death_timeout = death_timeout
         self.preload = preload
-        self.preload_argv = preload_argv,
+        if self.preload is None:
+            self.preload = dask.config.get('distributed.worker.preload')
+        self.preload_argv = preload_argv
+        if self.preload_argv is None:
+            self.preload_argv = dask.config.get('distributed.worker.preload-argv')
         self.contact_address = contact_address
         self.memory_monitor_interval = parse_timedelta(memory_monitor_interval, default='ms')
         self.extensions = dict()

From 22f013dd31f7397c8ccb165de72317082bf35f82 Mon Sep 17 00:00:00 2001
From: Peter Killick <peter.killick@metoffice.gov.uk>
Date: Fri, 9 Nov 2018 14:25:48 +0000
Subject: [PATCH 0128/1550] Add serializer for Numpy masked arrays (#2335)

---
 distributed/protocol/numpy.py            | 34 +++++++++
 distributed/protocol/tests/test_numpy.py | 94 +++++++++++++++---------
 2 files changed, 92 insertions(+), 36 deletions(-)

diff --git a/distributed/protocol/numpy.py b/distributed/protocol/numpy.py
index 5b18e8ebcb7..d04a1860e28 100644
--- a/distributed/protocol/numpy.py
+++ b/distributed/protocol/numpy.py
@@ -102,3 +102,37 @@ def deserialize_numpy_ndarray(header, frames):
                        strides=header['strides'])
 
         return x
+
+
+@dask_serialize.register(np.ma.core.MaskedArray)
+def serialize_numpy_maskedarray(x):
+    # Separate elements of the masked array that we need to deal with discretely.
+    data = x.data
+    mask = x.mask
+    fill_value = x.fill_value
+
+    # Make use of existing numpy serialization for the two ndarray elements of
+    # the masked array.
+    data_header, data_frames = serialize_numpy_ndarray(data)
+    mask_header, mask_frames = serialize_numpy_ndarray(mask)
+
+    header = {"data-header": data_header,
+              "mask-header": mask_header,
+              "fill_value": fill_value,
+              "nframes": len(data_frames)}
+    return header, data_frames + mask_frames
+
+
+@dask_deserialize.register(np.ma.core.MaskedArray)
+def deserialize_numpy_maskedarray(header, frames):
+    data_frames = frames[:header["nframes"]]
+    mask_frames = frames[header["nframes"]:]
+    data_header = header["data-header"]
+    mask_header = header["mask-header"]
+
+    # Get the individual elements of the masked array in order to reconstruct.
+    data = deserialize_numpy_ndarray(data_header, data_frames)
+    mask = deserialize_numpy_ndarray(mask_header, mask_frames)
+    fill_value = header["fill_value"]
+
+    return np.ma.masked_array(data, mask=mask, fill_value=fill_value)
diff --git a/distributed/protocol/tests/test_numpy.py b/distributed/protocol/tests/test_numpy.py
index 89c441f34db..d757644edb4 100644
--- a/distributed/protocol/tests/test_numpy.py
+++ b/distributed/protocol/tests/test_numpy.py
@@ -6,6 +6,7 @@
 import numpy as np
 import pytest
 
+from distributed.compatibility import PY2
 from distributed.protocol import (serialize, deserialize, decompress, dumps,
                                   loads, to_serialize, msgpack)
 from distributed.protocol.utils import BIG_BYTES_SHARD_SIZE
@@ -27,46 +28,53 @@ def test_serialize():
     assert (result == x).all()
 
 
-@pytest.mark.parametrize('x',
-                         [np.ones(5),
-                          np.array(5),
-                             np.random.random((5, 5)),
-                             np.random.random((5, 5))[::2, :],
-                             np.random.random((5, 5))[:, ::2],
-                             np.asfortranarray(np.random.random((5, 5))),
-                             np.asfortranarray(np.random.random((5, 5)))[::2, :],
-                             np.asfortranarray(np.random.random((5, 5)))[:, ::2],
-                             np.random.random(5).astype('f4'),
-                             np.random.random(5).astype('>i8'),
-                             np.random.random(5).astype('<i8'),
-                             np.arange(5).astype('M8[us]'),
-                             np.arange(5).astype('M8[ms]'),
-                             np.arange(5).astype('m8'),
-                             np.arange(5).astype('m8[s]'),
-                             np.arange(5).astype('c16'),
-                             np.arange(5).astype('c8'),
-                             np.array([True, False, True]),
-                             np.ones(shape=5, dtype=[('a', 'i4'), ('b', 'M8[us]')]),
-                             np.array(['abc'], dtype='S3'),
-                             np.array(['abc'], dtype='U3'),
-                             np.array(['abc'], dtype=object),
-                             np.ones(shape=(5,), dtype=('f8', 32)),
-                             np.ones(shape=(5,), dtype=[('x', 'f8', 32)]),
-                             np.ones(shape=(5,), dtype=np.dtype([('a', 'i1'), ('b', 'f8')], align=False)),
-                             np.ones(shape=(5,), dtype=np.dtype([('a', 'i1'), ('b', 'f8')], align=True)),
-                             np.ones(shape=(5,), dtype=np.dtype([('a', 'm8[us]')], align=False)),
-                             # this dtype fails unpickling
-                             np.ones(shape=(5,), dtype=np.dtype([('a', 'm8')], align=False)),
-                             np.array([(1, 'abc')], dtype=[('x', 'i4'), ('s', object)]),
-                             np.zeros(5000, dtype=[('x%d' % i, '<f8') for i in range(4)]),
-                             np.zeros(5000, dtype='S32'),
-                             np.zeros((1, 1000, 1000)),
-                             np.arange(12)[::2],  # non-contiguous array
-                             np.ones(shape=(5, 6)).astype(dtype=[('total', '<f8'), ('n', '<f8')])])
+@pytest.mark.parametrize('x', [
+    np.ones(5),
+    np.array(5),
+    np.random.random((5, 5)),
+    np.random.random((5, 5))[::2, :],
+    np.random.random((5, 5))[:, ::2],
+    np.asfortranarray(np.random.random((5, 5))),
+    np.asfortranarray(np.random.random((5, 5)))[::2, :],
+    np.asfortranarray(np.random.random((5, 5)))[:, ::2],
+    np.random.random(5).astype('f4'),
+    np.random.random(5).astype('>i8'),
+    np.random.random(5).astype('<i8'),
+    np.arange(5).astype('M8[us]'),
+    np.arange(5).astype('M8[ms]'),
+    np.arange(5).astype('m8'),
+    np.arange(5).astype('m8[s]'),
+    np.arange(5).astype('c16'),
+    np.arange(5).astype('c8'),
+    np.array([True, False, True]),
+    np.ones(shape=5, dtype=[('a', 'i4'), ('b', 'M8[us]')]),
+    np.array(['abc'], dtype='S3'),
+    np.array(['abc'], dtype='U3'),
+    np.array(['abc'], dtype=object),
+    np.ones(shape=(5,), dtype=('f8', 32)),
+    np.ones(shape=(5,), dtype=[('x', 'f8', 32)]),
+    np.ones(shape=(5,), dtype=np.dtype([('a', 'i1'), ('b', 'f8')], align=False)),
+    np.ones(shape=(5,), dtype=np.dtype([('a', 'i1'), ('b', 'f8')], align=True)),
+    np.ones(shape=(5,), dtype=np.dtype([('a', 'm8[us]')], align=False)),
+    # this dtype fails unpickling
+    np.ones(shape=(5,), dtype=np.dtype([('a', 'm8')], align=False)),
+    np.array([(1, 'abc')], dtype=[('x', 'i4'), ('s', object)]),
+    np.zeros(5000, dtype=[('x%d' % i, '<f8') for i in range(4)]),
+    np.zeros(5000, dtype='S32'),
+    np.zeros((1, 1000, 1000)),
+    np.arange(12)[::2],  # non-contiguous array
+    np.ones(shape=(5, 6)).astype(dtype=[('total', '<f8'), ('n', '<f8')]),
+    np.ma.masked_array((5, 6), mask=[True, False]),  # int array
+    np.ma.masked_array((5., 6.), mask=[True, False]),  # float array (different default fill_value)
+    np.ma.masked_array((5., 6.), mask=[True, False], fill_value=np.nan),
+])
 def test_dumps_serialize_numpy(x):
     header, frames = serialize(x)
     if 'compression' in header:
         frames = decompress(header, frames)
+    buffer_interface = buffer if PY2 else memoryview  # noqa: F821
+    for frame in frames:
+        assert isinstance(frame, (bytes, buffer_interface))
     y = deserialize(header, frames)
 
     np.testing.assert_equal(x, y)
@@ -74,6 +82,20 @@ def test_dumps_serialize_numpy(x):
         assert x.strides == y.strides
 
 
+def test_masked_array_serialize():
+    data = (5, 6)
+    mask = [True, False]
+    fill_value = 999
+    x = np.ma.masked_array(data, mask=mask, fill_value=fill_value)
+    header, frames = serialize(x)
+    y = deserialize(header, frames)
+
+    # Explicitly test the particular elements of the masked array.
+    np.testing.assert_equal(data, y.data)
+    np.testing.assert_equal(mask, y.mask)
+    assert fill_value == y.fill_value
+
+
 def test_dumps_serialize_numpy_custom_dtype():
     from six.moves import builtins
     test_rational = pytest.importorskip('numpy.core.test_rational')

From 85d2daede03a1a441568e2a25459ae9225c87b7f Mon Sep 17 00:00:00 2001
From: NotSqrt <notsqrt@gmail.com>
Date: Fri, 9 Nov 2018 15:52:21 +0100
Subject: [PATCH 0129/1550] Use psutil.Process.oneshot (#2339)

---
 distributed/system_monitor.py | 5 +++--
 requirements.txt              | 2 +-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/distributed/system_monitor.py b/distributed/system_monitor.py
index 735217fe2fc..3c68e114b7f 100644
--- a/distributed/system_monitor.py
+++ b/distributed/system_monitor.py
@@ -46,8 +46,9 @@ def recent(self):
             return {k: None for k, v in self.quantities.items()}
 
     def update(self):
-        cpu = self.proc.cpu_percent()
-        memory = self.proc.memory_info().rss
+        with self.proc.oneshot():
+            cpu = self.proc.cpu_percent()
+            memory = self.proc.memory_info().rss
         now = time()
 
         self.cpu.append(cpu)
diff --git a/requirements.txt b/requirements.txt
index fd3d84f508f..212e97df92d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,7 +2,7 @@ click >= 6.6
 cloudpickle >= 0.2.2
 dask >= 0.18.0
 msgpack
-psutil
+psutil >= 5.0
 six
 sortedcontainers !=2.0.0, !=2.0.1
 tblib

From 357bca686b47293062584e12543f2ce22bd8f16f Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 9 Nov 2018 09:52:56 -0500
Subject: [PATCH 0130/1550] Improve error message on missing op key (#2313)

---
 distributed/core.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/distributed/core.py b/distributed/core.py
index 1cd2999b39d..bf08abebc8c 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -310,7 +310,13 @@ def handle_comm(self, comm, shutting_down=shutting_down):
                     raise TypeError("Bad message type.  Expected dict, got\n  "
                                     + str(msg))
 
-                op = msg.pop('op')
+                try:
+                    op = msg.pop('op')
+                except KeyError:
+                    raise ValueError(
+                        "Received unexpected message without 'op' key: " %
+                        str(msg)
+                    )
                 if self.counters is not None:
                     self.counters['op'].add(op)
                 self._comms[comm] = op

From dfa54ebc617675a8863e54d2fb138f6eba6ddbc2 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 9 Nov 2018 11:37:58 -0500
Subject: [PATCH 0131/1550] use worker SSL context when getting client from
 worker. (#2301)

---
 distributed/client.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/distributed/client.py b/distributed/client.py
index ac425d73ebe..91dcb82e201 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -560,7 +560,12 @@ def __init__(self, address=None, loop=None, timeout=no_default,
         self.security = security or Security()
         self.scheduler_comm = None
         assert isinstance(self.security, Security)
-        self.connection_args = self.security.get_connection_args('client')
+
+        if name == 'worker':
+            self.connection_args = self.security.get_connection_args('worker')
+        else:
+            self.connection_args = self.security.get_connection_args('client')
+
         self._connecting_to_scheduler = False
         self._asynchronous = asynchronous
         self._should_close_loop = not loop

From 32341216c9f62e37c2e01c898e26b117e2b872b3 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 9 Nov 2018 13:46:26 -0500
Subject: [PATCH 0132/1550] bump version to 1.24.1

---
 docs/source/changelog.rst | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index 2c97a99281f..ee978f4ad0e 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,18 @@
 Changelog
 =========
 
+1.24.1 - 2018-11-09
+-------------------
+
+-  Use tornado's builtin AnyThreadLoopEventPolicy (:pr:`2326`) `Matthew Rocklin`_
+-  Adjust TLS tests for openssl 1.1 (:pr:`2331`) `Marius van Niekerk`_
+-  Avoid setting event loop policy if within Jupyter notebook server (:pr:`2343`) `Matthew Rocklin`_
+-  Add preload script to conf (:pr:`2325`) `Guillaume Eynard-Bontemps`_
+-  Add serializer for Numpy masked arrays (:pr:`2335`) `Peter Killick`_
+-  Use psutil.Process.oneshot (:pr:`2339`) `NotSqrt`_
+-  Use worker SSL context when getting client from worker. (:pr:`2301`) Anonymous
+
+
 1.24.0 - 2018-10-26
 -------------------
 
@@ -85,7 +97,7 @@ Changelog
 -  Be resilient to missing dep after busy signal (:pr:`2217`) `Matthew Rocklin`_
 -  Use CSS Grid to layout status page on the dashboard (:pr:`2213`) `Derek Ludwig`_ and `Luke Canavan`_
 -  Fix deserialization of queues on main ioloop thread (:pr:`2221`) `Matthew Rocklin`_
--  Add a worker initialization function (:pr:`2201`) `Guillaume EB`_
+-  Add a worker initialization function (:pr:`2201`) `Guillaume Eynard-Bontemps`_
 -  Collapse navbar in dashboard (:pr:`2223`) `Luke Canavan`_
 
 
@@ -815,10 +827,12 @@ significantly without many new features.
 .. _`Derek Ludwig`: https://github.com/dsludwig
 .. _`Anderson Banihirwe`: https://github.com/andersy005
 .. _`Yu Feng`: https://github.com/rainwoodman
-.. _`Guillaume EB`: https://github.com/guillaumeeb
+.. _`Guillaume Eynard-Bontemps`: https://github.com/guillaumeeb
 .. _`Vladyslav Moisieienkov`: https://github.com/VMois
 .. _`Chris White`: https://github.com/cicdw
 .. _`Adam Klein`: https://github.com/adamklein
 .. _`Mike Gevaert`: https://github.com/mgeplf
 .. _`Gaurav Sheni`: https://github.com/gsheni
 .. _`Eric Ma`: https://github.com/ericmjl
+.. _`Peter Killick`: https://github.com/dkillick
+.. _`NotSqrt`: https://github.com/NotSqrt

From e6048a692685a536da5eafe1389f42a213c25440 Mon Sep 17 00:00:00 2001
From: Tom Rochette <roctom@gmail.com>
Date: Mon, 12 Nov 2018 08:01:51 -0500
Subject: [PATCH 0133/1550] Fix typo in worker documentation (#2349)

---
 docs/source/worker.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/source/worker.rst b/docs/source/worker.rst
index 5ac95c197c9..ce888a388f8 100644
--- a/docs/source/worker.rst
+++ b/docs/source/worker.rst
@@ -147,7 +147,7 @@ Memory Management
 Workers are given a target memory limit to stay under with the
 command line ``--memory-limit`` keyword or the ``memory_limit=`` Python
 keyword argument, which sets the memory limit per worker processes launched
-by dask-workder ::
+by dask-worker ::
 
     $ dask-worker tcp://scheduler:port --memory-limit=auto  # total available RAM on the machine
     $ dask-worker tcp://scheduler:port --memory-limit=4e9  # four gigabytes per worker process.

From 21cba865ee3c9e6b269d37b8dd9b10ece9a2f41b Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 14 Nov 2018 08:15:37 -0500
Subject: [PATCH 0134/1550] Use attribute access on Bokeh Selection objects
 (#2348)

This happened in the change to 1.0
---
 distributed/bokeh/components.py | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/distributed/bokeh/components.py b/distributed/bokeh/components.py
index 8141936a800..eeae4282c4a 100644
--- a/distributed/bokeh/components.py
+++ b/distributed/bokeh/components.py
@@ -312,7 +312,11 @@ def __init__(self, **kwargs):
         def cb(attr, old, new):
             with log_errors():
                 try:
-                    ind = new['1d']['indices'][0]
+                    selected = new.indices
+                except AttributeError:
+                    selected = new['1d']['indices']
+                try:
+                    ind = selected[0]
                 except IndexError:
                     return
                 data = profile.plot_data(self.states[ind], profile_interval)
@@ -408,7 +412,11 @@ def cb(attr, old, new):
                 return
             with log_errors():
                 try:
-                    ind = new['1d']['indices'][0]
+                    selected = new.indices
+                except AttributeError:
+                    selected = new['1d']['indices']
+                try:
+                    ind = selected[0]
                 except IndexError:
                     return
                 data = profile.plot_data(self.states[ind], profile_interval)
@@ -571,7 +579,11 @@ def cb(attr, old, new):
                 return
             with log_errors():
                 try:
-                    ind = new['1d']['indices'][0]
+                    selected = new.indices
+                except AttributeError:
+                    selected = new['1d']['indices']
+                try:
+                    ind = selected[0]
                 except IndexError:
                     return
                 data = profile.plot_data(self.states[ind], profile_interval)

From 9caa13cf8110d8c015b477b880ac33dc6c93a2bf Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 14 Nov 2018 09:12:53 -0500
Subject: [PATCH 0135/1550] Replace logger.warn with logger.warning (#2355)

This is showing up as a deprecation warning in appveyor logs
---
 distributed/scheduler.py  | 4 ++--
 distributed/utils_perf.py | 9 +++++----
 distributed/worker.py     | 2 +-
 3 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 353764e2f59..d0ef7606a5c 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -4307,8 +4307,8 @@ def check_worker_ttl(self):
         now = time()
         for ws in self.workers.values():
             if ws.last_seen < now - self.worker_ttl:
-                logger.warn("Worker failed to heartbeat within %s seconds. "
-                            "Closing: %s", self.worker_ttl, ws)
+                logger.warning("Worker failed to heartbeat within %s seconds. "
+                               "Closing: %s", self.worker_ttl, ws)
                 self.remove_worker(address=ws.address)
 
 
diff --git a/distributed/utils_perf.py b/distributed/utils_perf.py
index 6402ef50eb0..c6c1d37b107 100644
--- a/distributed/utils_perf.py
+++ b/distributed/utils_perf.py
@@ -192,8 +192,9 @@ def _gc_callback(self, phase, info):
         self._fractional_timer.stop_timing()
         frac = self._fractional_timer.running_fraction
         if frac is not None and frac >= self._warn_over_frac:
-            logger.warn("full garbage collections took %d%% CPU time recently "
-                        "(threshold: %d%%)", 100 * frac, 100 * self._warn_over_frac)
+            logger.warning("full garbage collections took %d%% CPU time "
+                           "recently (threshold: %d%%)",
+                           100 * frac, 100 * self._warn_over_frac)
         rss_saved = self._gc_rss_before - rss
         if rss_saved >= self._info_over_rss_win:
             logger.info("full garbage collection released %s "
@@ -202,8 +203,8 @@ def _gc_callback(self, phase, info):
                         format_bytes(self._info_over_rss_win))
         if info['uncollectable'] > 0:
             # This should ideally never happen on Python 3, but who knows?
-            logger.warn("garbage collector couldn't collect %d objects, "
-                        "please look in gc.garbage", info['uncollectable'])
+            logger.warning("garbage collector couldn't collect %d objects, "
+                           "please look in gc.garbage", info['uncollectable'])
 
 
 _gc_diagnosis = GCDiagnosis()
diff --git a/distributed/worker.py b/distributed/worker.py
index 5709501d180..c435c4dd2bc 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -278,7 +278,7 @@ def heartbeat(self):
                 self.scheduler_delay = response['time'] - middle
                 self.periodic_callbacks['heartbeat'].callback_time = response['heartbeat-interval'] * 1000
             except CommClosedError:
-                logger.warn("Heartbeat to scheduler failed")
+                logger.warning("Heartbeat to scheduler failed")
             finally:
                 self.heartbeat_active = False
         else:

From a056b985ea40569c2ad7f233441d2cbb87ec1cc8 Mon Sep 17 00:00:00 2001
From: Roy Wedge <roy.wedge@gmail.com>
Date: Wed, 14 Nov 2018 21:06:42 -0500
Subject: [PATCH 0136/1550] set loop policy if tornado version 5 or above
 (#2353)

Fixes #2351
---
 distributed/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/utils.py b/distributed/utils.py
index 51d55bd1931..c5227ade1cb 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -1372,7 +1372,7 @@ def reset_logger_locks():
 
 
 # Only bother if asyncio has been loaded by Tornado
-if 'asyncio' in sys.modules:
+if 'asyncio' in sys.modules and tornado.version_info[0] >= 5:
 
     jupyter_event_loop_initialized = False
 

From 75bae45fb4e2104df156b4606de72daf8396fa17 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 15 Nov 2018 07:27:49 -0500
Subject: [PATCH 0137/1550] Replace deprecated use with gcd, msgpack,
 ipywidgets (#2357)

* add gcd to compatibility
* avoid msgpack deprecation warning
* remove deprecated height/width values in ipywidgets
---
 distributed/compatibility.py             | 2 ++
 distributed/diagnostics/progressbar.py   | 9 ++++-----
 distributed/protocol/numpy.py            | 7 +++----
 distributed/protocol/tests/test_numpy.py | 2 +-
 4 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/distributed/compatibility.py b/distributed/compatibility.py
index 2b901edbb08..fbf8f86df5d 100644
--- a/distributed/compatibility.py
+++ b/distributed/compatibility.py
@@ -12,6 +12,7 @@
     from inspect import getargspec
     from cgi import escape as html_escape
     from collections import Iterator, Mapping, Set, MutableMapping
+    from fractions import gcd
 
     reload = reload
     unicode = unicode
@@ -67,6 +68,7 @@ def iscoroutinefunction(func):
     from importlib.util import cache_from_source
     from inspect import getfullargspec as getargspec
     from html import escape as html_escape
+    from math import gcd
 
     PY2 = False
     PY3 = True
diff --git a/distributed/diagnostics/progressbar.py b/distributed/diagnostics/progressbar.py
index 3263503b2a2..38f784e9cf6 100644
--- a/distributed/diagnostics/progressbar.py
+++ b/distributed/diagnostics/progressbar.py
@@ -145,8 +145,8 @@ def __init__(self, keys, scheduler=None, interval='100ms',
 
         from ipywidgets import FloatProgress, HBox, VBox, HTML
         self.elapsed_time = HTML('')
-        self.bar = FloatProgress(min=0, max=1, description='', height='10px')
-        self.bar_text = HTML('', width="140px")
+        self.bar = FloatProgress(min=0, max=1, description='')
+        self.bar_text = HTML('')
 
         self.bar_widget = HBox([self.bar_text, self.bar])
         self.widget = VBox([self.elapsed_time, self.bar_widget])
@@ -269,10 +269,9 @@ def __init__(self, keys, scheduler=None, minimum=0, interval=0.1, func=key_split
     def make_widget(self, all):
         from ipywidgets import FloatProgress, HBox, VBox, HTML
         self.elapsed_time = HTML('')
-        self.bars = {key: FloatProgress(min=0, max=1, description='',
-                                        height='10px')
+        self.bars = {key: FloatProgress(min=0, max=1, description='')
                      for key in all}
-        self.bar_texts = {key: HTML('', width="140px") for key in all}
+        self.bar_texts = {key: HTML('') for key in all}
         self.bar_labels = {key: HTML('<div style=\"padding: 0px 10px 0px 10px;'
                                      ' text-align:left; word-wrap: '
                                      'break-word;\">' +
diff --git a/distributed/protocol/numpy.py b/distributed/protocol/numpy.py
index d04a1860e28..fc12e2ef2d0 100644
--- a/distributed/protocol/numpy.py
+++ b/distributed/protocol/numpy.py
@@ -1,13 +1,12 @@
 from __future__ import print_function, division, absolute_import
 
-import fractions
-
 import numpy as np
 
 from .utils import frame_split_size, merge_frames
 from .serialize import dask_serialize, dask_deserialize
 from . import pickle
 
+from ..compatibility import gcd
 from ..utils import log_errors
 
 
@@ -61,13 +60,13 @@ def serialize_numpy_ndarray(x):
         data = x.ravel()
 
     if data.dtype.fields or data.dtype.itemsize > 8:
-        data = data.view('u%d' % fractions.gcd(x.dtype.itemsize, 8))
+        data = data.view('u%d' % gcd(x.dtype.itemsize, 8))
 
     try:
         data = data.data
     except ValueError:
         # "ValueError: cannot include dtype 'M' in a buffer"
-        data = data.view('u%d' % fractions.gcd(x.dtype.itemsize, 8)).data
+        data = data.view('u%d' % gcd(x.dtype.itemsize, 8)).data
 
     header = {'dtype': dt,
               'shape': x.shape,
diff --git a/distributed/protocol/tests/test_numpy.py b/distributed/protocol/tests/test_numpy.py
index d757644edb4..eae76623519 100644
--- a/distributed/protocol/tests/test_numpy.py
+++ b/distributed/protocol/tests/test_numpy.py
@@ -168,7 +168,7 @@ def test_compress_numpy():
     frames = dumps({'x': to_serialize(x)})
     assert sum(map(nbytes, frames)) < x.nbytes
 
-    header = msgpack.loads(frames[2], encoding='utf8', use_list=False)
+    header = msgpack.loads(frames[2], raw=False, use_list=False)
     try:
         import blosc  # noqa: F401
     except ImportError:

From a887d221c8f04e5e5987a96c022b7e8b2300bf0a Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 15 Nov 2018 07:28:05 -0500
Subject: [PATCH 0138/1550] Update on_change behavior for Bokeh 1.0 (#2356)

---
 distributed/bokeh/components.py | 48 ++++++++++++++++++++++++---------
 1 file changed, 35 insertions(+), 13 deletions(-)

diff --git a/distributed/bokeh/components.py b/distributed/bokeh/components.py
index eeae4282c4a..14aec7308d2 100644
--- a/distributed/bokeh/components.py
+++ b/distributed/bokeh/components.py
@@ -5,6 +5,7 @@
 from time import time
 import weakref
 
+import bokeh
 from bokeh.layouts import row, column
 from bokeh.models import ( ColumnDataSource, Plot, DataRange1d, LinearAxis,
         HoverTool, BoxZoomTool, ResetTool, PanTool, WheelZoomTool, Range1d,
@@ -325,7 +326,10 @@ def cb(attr, old, new):
                 self.source.data.update(data)
                 self.source.selected = old
 
-        self.source.on_change('selected', cb)
+        if bokeh.__version__ >= '1.0':
+            self.source.selected.on_change('indices', cb)
+        else:
+            self.source.on_change('selected', cb)
 
         self.root = figure(tools='tap', **kwargs)
         self.root.quad('left', 'right', 'top', 'bottom', color='color',
@@ -411,9 +415,9 @@ def cb(attr, old, new):
             if changing[0]:
                 return
             with log_errors():
-                try:
-                    selected = new.indices
-                except AttributeError:
+                if isinstance(new, list):  # bokeh >= 1.0
+                    selected = new
+                else:
                     selected = new['1d']['indices']
                 try:
                     ind = selected[0]
@@ -424,10 +428,16 @@ def cb(attr, old, new):
                 self.states.extend(data.pop('states'))
                 changing[0] = True  # don't recursively trigger callback
                 self.source.data.update(data)
-                self.source.selected = old
+                if isinstance(new, list):  # bokeh >= 1.0
+                    self.source.selected.indices = old
+                else:
+                    self.source.selected = old
                 changing[0] = False
 
-        self.source.on_change('selected', cb)
+        if bokeh.__version__ >= '1.0':
+            self.source.selected.on_change('indices', cb)
+        else:
+            self.source.on_change('selected', cb)
 
         self.profile_plot = figure(tools='tap', height=400, **kwargs)
         r = self.profile_plot.quad('left', 'right', 'top', 'bottom', color='color',
@@ -496,7 +506,10 @@ def ts_change(attr, old, new):
                     self.start = self.stop = None
                 self.trigger_update(update_metadata=False)
 
-        self.ts_source.on_change('selected', ts_change)
+        if bokeh.__version__ >= '1.0':
+            self.ts_source.selected.on_change('indices', ts_change)
+        else:
+            self.ts_source.on_change('selected', ts_change)
 
         self.reset_button = Button(label="Reset", button_type="success")
         self.reset_button.on_click(lambda: self.update(self.state) )
@@ -578,9 +591,9 @@ def cb(attr, old, new):
             if changing[0]:
                 return
             with log_errors():
-                try:
-                    selected = new.indices
-                except AttributeError:
+                if isinstance(new, list):  # bokeh >= 1.0
+                    selected = new
+                else:
                     selected = new['1d']['indices']
                 try:
                     ind = selected[0]
@@ -591,10 +604,16 @@ def cb(attr, old, new):
                 self.states.extend(data.pop('states'))
                 changing[0] = True  # don't recursively trigger callback
                 self.source.data.update(data)
-                self.source.selected = old
+                if isinstance(new, list):  # bokeh >= 1.0
+                    self.source.selected.indices = old
+                else:
+                    self.source.selected = old
                 changing[0] = False
 
-        self.source.on_change('selected', cb)
+        if bokeh.__version__ >= '1.0':
+            self.source.selected.on_change('indices', cb)
+        else:
+            self.source.on_change('selected', cb)
 
         self.profile_plot = figure(tools='tap', height=400, **kwargs)
         r = self.profile_plot.quad('left', 'right', 'top', 'bottom', color='color',
@@ -663,7 +682,10 @@ def ts_change(attr, old, new):
                     self.start = self.stop = None
                 self.trigger_update()
 
-        self.ts_source.on_change('selected', ts_change)
+        if bokeh.__version__ >= '1.0':
+            self.ts_source.selected.on_change('indices', ts_change)
+        else:
+            self.ts_source.on_change('selected', ts_change)
 
         self.reset_button = Button(label="Reset", button_type="success")
         self.reset_button.on_click(lambda: self.update(self.state))

From 0e33bb295f03ec6a971deea817462534c4cf2c76 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 15 Nov 2018 07:46:27 -0500
Subject: [PATCH 0139/1550] bump version to 1.24.2

---
 docs/source/changelog.rst | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index ee978f4ad0e..b9f246d0600 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,14 @@
 Changelog
 =========
 
+1.24.2 - 2018-11-15
+-------------------
+
+-  Add support for Bokeh 1.0 (:pr:`2348`) (:pr:`2356`) `Matthew Rocklin`_
+-  Fix regression that dropped support for Tornado 4 (:pr:`2353`) `Roy Wedge`_
+-  Avoid deprecation warnings (:pr:`2355`) (:pr:`2357`) `Matthew Rocklin`_
+-  Fix typo in worker documentation (:pr:`2349`) `Tom Rochette`_
+
 1.24.1 - 2018-11-09
 -------------------
 
@@ -836,3 +844,5 @@ significantly without many new features.
 .. _`Eric Ma`: https://github.com/ericmjl
 .. _`Peter Killick`: https://github.com/dkillick
 .. _`NotSqrt`: https://github.com/NotSqrt
+.. _`Tom Rochette`: https://github.com/tomzx
+.. _`Roy Wedge`: https://github.com/rwedge

From 964a917c45e8c11b16086cf05949d3529cc0986a Mon Sep 17 00:00:00 2001
From: Michael Wheeler <Wheeler.m@husky.neu.edu>
Date: Fri, 16 Nov 2018 09:42:17 -0500
Subject: [PATCH 0140/1550] Fixed the 404 error on the Scheduler Dashboard
 homepage (#2361)

* Fix: Added "extra_patterns" redirect to display status app as homepage for dashboard.

* Test: Checks that hitting scheduler dashboard at / redirects to /status
---
 distributed/bokeh/core.py                       | 1 +
 distributed/bokeh/tests/test_scheduler_bokeh.py | 9 +++++++++
 2 files changed, 10 insertions(+)

diff --git a/distributed/bokeh/core.py b/distributed/bokeh/core.py
index deeccadb6cb..9263a701cdc 100644
--- a/distributed/bokeh/core.py
+++ b/distributed/bokeh/core.py
@@ -33,6 +33,7 @@ def listen(self, addr):
                                      check_unused_sessions_milliseconds=500,
                                      allow_websocket_origin=["*"],
                                      use_index=False,
+                                     extra_patterns=[(r'/', web.RedirectHandler, {'url': '/status'})],
                                      **self.server_kwargs)
                 self.server.start()
 
diff --git a/distributed/bokeh/tests/test_scheduler_bokeh.py b/distributed/bokeh/tests/test_scheduler_bokeh.py
index cec663c9b65..acbf54bc102 100644
--- a/distributed/bokeh/tests/test_scheduler_bokeh.py
+++ b/distributed/bokeh/tests/test_scheduler_bokeh.py
@@ -541,3 +541,12 @@ def test_profile_server(c, s, a, b):
     yield gen.sleep(0.200)
     ptp.trigger_update()
     assert 2 < len(ptp.ts_source.data['time']) < 20
+
+
+@gen_cluster(client=True,
+             scheduler_kwargs={'services': {('bokeh', 0): BokehScheduler}})
+def test_root_redirect(c, s, a, b):
+    http_client = AsyncHTTPClient()
+    response = yield http_client.fetch('http://localhost:%d/' % s.services['bokeh'].port)
+    assert response.code == 200
+    assert "/status" in response.effective_url

From 19a32f4d255ce56aa3984020aba4256b9a217699 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 19 Nov 2018 09:06:58 -0500
Subject: [PATCH 0141/1550] Consolidate two Worker classes into one (#2363)

These were separate for historical reasons.
We merge them together and organize methods by theme.
---
 distributed/nanny.py  |   14 +-
 distributed/worker.py | 2811 +++++++++++++++++++++--------------------
 2 files changed, 1418 insertions(+), 1407 deletions(-)

diff --git a/distributed/nanny.py b/distributed/nanny.py
index c973985db78..27b56baccc0 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -38,13 +38,13 @@ class Nanny(ServerNode):
     status = None
 
     def __init__(self, scheduler_ip=None, scheduler_port=None,
-                 scheduler_file=None, worker_port=0,
-                 ncores=None, loop=None, local_dir=None, services=None,
-                 name=None, memory_limit='auto', reconnect=True,
-                 validate=False, quiet=False, resources=None, silence_logs=None,
-                 death_timeout=None, preload=(), preload_argv=[], security=None,
-                 contact_address=None, listen_address=None, worker_class=None,
-                 **kwargs):
+            scheduler_file=None, worker_port=0, ncores=None, loop=None,
+            local_dir='dask-worker-space', services=None, name=None,
+            memory_limit='auto', reconnect=True, validate=False, quiet=False,
+            resources=None, silence_logs=None, death_timeout=None, preload=(),
+            preload_argv=[], security=None, contact_address=None,
+            listen_address=None, worker_class=None, **kwargs):
+
         if scheduler_file:
             cfg = json_load_robust(scheduler_file)
             self.scheduler_addr = cfg['address']
diff --git a/distributed/worker.py b/distributed/worker.py
index c435c4dd2bc..4fb41a4aecb 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -81,1278 +81,965 @@
 _global_workers = []
 
 
-class WorkerBase(ServerNode):
-    def __init__(self, scheduler_ip=None, scheduler_port=None,
-                 scheduler_file=None, ncores=None, loop=None, local_dir=None,
-                 services=None, service_ports=None, name=None,
-                 reconnect=True, memory_limit='auto',
-                 executor=None, resources=None, silence_logs=None,
-                 death_timeout=None, preload=None, preload_argv=None, security=None,
-                 contact_address=None, memory_monitor_interval='200ms',
-                 extensions=None, metrics=None, **kwargs):
-
-        self._setup_logging()
-
-        if scheduler_file:
-            cfg = json_load_robust(scheduler_file)
-            scheduler_addr = cfg['address']
-        elif scheduler_ip is None and dask.config.get('scheduler-address', None):
-            scheduler_addr = dask.config.get('scheduler-address')
-        elif scheduler_port is None:
-            scheduler_addr = coerce_to_address(scheduler_ip)
-        else:
-            scheduler_addr = coerce_to_address((scheduler_ip, scheduler_port))
-        self._port = 0
-        self.ncores = ncores or _ncores
-        self.total_resources = resources or {}
-        self.available_resources = (resources or {}).copy()
-        self.death_timeout = death_timeout
-        self.preload = preload
-        if self.preload is None:
-            self.preload = dask.config.get('distributed.worker.preload')
-        self.preload_argv = preload_argv
-        if self.preload_argv is None:
-            self.preload_argv = dask.config.get('distributed.worker.preload-argv')
-        self.contact_address = contact_address
-        self.memory_monitor_interval = parse_timedelta(memory_monitor_interval, default='ms')
-        self.extensions = dict()
-        if silence_logs:
-            silence_logging(level=silence_logs)
-
-        if local_dir:
-            local_dir = os.path.abspath(local_dir)
-        else:
-            local_dir = 'dask-worker-space'
-        self._workspace = WorkSpace(local_dir)
-        self._workdir = self._workspace.new_work_dir(prefix='worker-')
-        self.local_dir = self._workdir.dir_path
-
-        self.security = security or Security()
-        assert isinstance(self.security, Security)
-        self.connection_args = self.security.get_connection_args('worker')
-        self.listen_args = self.security.get_listen_args('worker')
-
-        self.memory_limit = parse_memory_limit(memory_limit, self.ncores)
-
-        self.paused = False
-
-        if 'memory_target_fraction' in kwargs:
-            self.memory_target_fraction = kwargs.pop('memory_target_fraction')
-        else:
-            self.memory_target_fraction = dask.config.get('distributed.worker.memory.target')
-        if 'memory_spill_fraction' in kwargs:
-            self.memory_spill_fraction = kwargs.pop('memory_spill_fraction')
-        else:
-            self.memory_spill_fraction = dask.config.get('distributed.worker.memory.spill')
-        if 'memory_pause_fraction' in kwargs:
-            self.memory_pause_fraction = kwargs.pop('memory_pause_fraction')
-        else:
-            self.memory_pause_fraction = dask.config.get('distributed.worker.memory.pause')
-
-        if (self.memory_limit and
-                (self.memory_target_fraction or
-                 self.memory_spill_fraction)):
-            try:
-                from zict import Buffer, File, Func
-            except ImportError:
-                raise ImportError("Please `pip install zict` for spill-to-disk workers")
-            path = os.path.join(self.local_dir, 'storage')
-            storage = Func(partial(serialize_bytelist, on_error='raise'),
-                           deserialize_bytes,
-                           File(path))
-            target = int(float(self.memory_limit) * self.memory_target_fraction)
-            self.data = Buffer({}, storage, target, weight)
-        else:
-            self.data = dict()
-        self.actors = {}
-        self.loop = loop or IOLoop.current()
-        self.status = None
-        self._closed = Event()
-        self.reconnect = reconnect
-        self.executor = executor or ThreadPoolExecutor(self.ncores)
-        self.actor_executor = ThreadPoolExecutor(1)
-        self.name = name
-        self.scheduler_delay = 0
-        self.stream_comms = dict()
-        self.heartbeat_active = False
-        self._ipython_kernel = None
-
-        if self.local_dir not in sys.path:
-            sys.path.insert(0, self.local_dir)
-
-        self.services = {}
-        self.service_ports = service_ports or {}
-        self.service_specs = services or {}
-        self.metrics = dict(metrics) if metrics else {}
-
-        handlers = {
-            'gather': self.gather,
-            'run': self.run,
-            'run_coroutine': self.run_coroutine,
-            'get_data': self.get_data,
-            'update_data': self.update_data,
-            'delete_data': self.delete_data,
-            'terminate': self.terminate,
-            'ping': pingpong,
-            'upload_file': self.upload_file,
-            'start_ipython': self.start_ipython,
-            'call_stack': self.get_call_stack,
-            'profile': self.get_profile,
-            'profile_metadata': self.get_profile_metadata,
-            'get_logs': self.get_logs,
-            'keys': self.keys,
-            'versions': self.versions,
-            'actor_execute': self.actor_execute,
-            'actor_attribute': self.actor_attribute,
-        }
-
-        stream_handlers = {
-            'close': self._close,
-            'compute-task': self.add_task,
-            'release-task': partial(self.release_key, report=False),
-            'delete-data': self.delete_data,
-            'steal-request': self.steal_request,
-        }
-
-        super(WorkerBase, self).__init__(
-                handlers=handlers,
-                stream_handlers=stream_handlers,
-                io_loop=self.loop,
-                connection_args=self.connection_args,
-                **kwargs)
-
-        self.scheduler = self.rpc(scheduler_addr)
-        self.execution_state = {'scheduler': self.scheduler.address,
-                                'ioloop': self.loop,
-                                'worker': self}
-
-        pc = PeriodicCallback(self.heartbeat, 1000, io_loop=self.io_loop)
-        self.periodic_callbacks['heartbeat'] = pc
-        self._address = contact_address
-
-        if self.memory_limit:
-            self._memory_monitoring = False
-            pc = PeriodicCallback(self.memory_monitor,
-                                  self.memory_monitor_interval * 1000,
-                                  io_loop=self.io_loop)
-            self.periodic_callbacks['memory'] = pc
-
-        if extensions is None:
-            extensions = DEFAULT_EXTENSIONS
-        for ext in extensions:
-            ext(self)
-
-        self._throttled_gc = ThrottledGC(logger=logger)
-
-        setproctitle("dask-worker [not started]")
-
-    def _setup_logging(self):
-        self._deque_handler = DequeHandler(n=dask.config.get('distributed.admin.log-length'))
-        self._deque_handler.setFormatter(logging.Formatter(dask.config.get('distributed.admin.log-format')))
-        logger.addHandler(self._deque_handler)
-        finalize(self, logger.removeHandler, self._deque_handler)
-
-    @property
-    def worker_address(self):
-        """ For API compatibility with Nanny """
-        return self.address
-
-    @gen.coroutine
-    def heartbeat(self):
-        if not self.heartbeat_active:
-            self.heartbeat_active = True
-            logger.debug("Heartbeat: %s" % self.address)
-            try:
-                start = time()
-                response = yield self.scheduler.heartbeat_worker(
-                    address=self.contact_address,
-                    now=time(),
-                    metrics=self.get_metrics()
-                )
-                end = time()
-                middle = (start + end) / 2
-
-                if response['status'] == 'missing':
-                    yield self._register_with_scheduler()
-                    return
-                self.scheduler_delay = response['time'] - middle
-                self.periodic_callbacks['heartbeat'].callback_time = response['heartbeat-interval'] * 1000
-            except CommClosedError:
-                logger.warning("Heartbeat to scheduler failed")
-            finally:
-                self.heartbeat_active = False
-        else:
-            logger.debug("Heartbeat skipped: channel busy")
-
-    def get_metrics(self):
-        core = dict(executing=len(self.executing),
-                    in_memory=len(self.data),
-                    ready=len(self.ready),
-                    in_flight=len(self.in_flight_tasks))
-        custom = {k: metric(self) for k, metric in self.metrics.items()}
-
-        return merge(custom, self.monitor.recent(), core)
-
-    @gen.coroutine
-    def _register_with_scheduler(self):
-        self.periodic_callbacks['heartbeat'].stop()
-        start = time()
-        if self.contact_address is None:
-            self.contact_address = self.address
-        logger.info('-' * 49)
-        while True:
-            if self.death_timeout and time() > start + self.death_timeout:
-                yield self._close(timeout=1)
-                return
-            if self.status in ('closed', 'closing'):
-                raise gen.Return
-            try:
-                _start = time()
-                comm = yield connect(self.scheduler.address,
-                                     connection_args=self.connection_args)
-                yield comm.write(dict(op='register-worker',
-                                      reply=False,
-                                      address=self.contact_address,
-                                      keys=list(self.data),
-                                      ncores=self.ncores,
-                                      name=self.name,
-                                      nbytes=self.nbytes,
-                                      now=time(),
-                                      resources=self.total_resources,
-                                      memory_limit=self.memory_limit,
-                                      local_directory=self.local_dir,
-                                      services=self.service_ports,
-                                      pid=os.getpid(),
-                                      metrics=self.get_metrics()),
-                                 serializers=['msgpack'])
-                future = comm.read(deserializers=['msgpack'])
-                if self.death_timeout:
-                    diff = self.death_timeout - (time() - start)
-                    if diff < 0:
-                        continue
-                    future = gen.with_timeout(timedelta(seconds=diff), future)
-                response = yield future
-                _end = time()
-                middle = (_start + _end) / 2
-                self.scheduler_delay = response['time'] - middle
-                self.status = 'running'
-                break
-            except EnvironmentError:
-                logger.info('Waiting to connect to: %26s', self.scheduler.address)
-                yield gen.sleep(0.1)
-            except gen.TimeoutError:
-                logger.info("Timed out when connecting to scheduler")
-        if response['status'] != 'OK':
-            raise ValueError("Unexpected response from register: %r" %
-                             (response,))
-        else:
-            # Retrieve eventual init functions and run them
-            for function_bytes in response['worker-setups']:
-                setup_function = pickle.loads(function_bytes)
-                if has_arg(setup_function, 'dask_worker'):
-                    result = setup_function(dask_worker=self)
-                else:
-                    result = setup_function()
-                logger.info('Init function %s ran: output=%s' % (setup_function, result))
-
-            logger.info('        Registered to: %26s', self.scheduler.address)
-            logger.info('-' * 49)
-
-        self.batched_stream = BatchedSend(interval='2ms', loop=self.loop)
-        self.batched_stream.start(comm)
-        self.periodic_callbacks['heartbeat'].start()
-        self.loop.add_callback(self.handle_scheduler, comm)
-
-    @gen.coroutine
-    def handle_scheduler(self, comm):
-        try:
-            yield self.handle_stream(comm, every_cycle=[self.ensure_communicating,
-                                                        self.ensure_computing])
-        except Exception as e:
-            logger.exception(e)
-            raise
-        finally:
-            if self.reconnect:
-                logger.info("Connection to scheduler broken.  Reconnecting...")
-                self.loop.add_callback(self._register_with_scheduler)
-            else:
-                yield self._close(report=False)
-
-    def start_services(self, listen_ip=''):
-        for k, v in self.service_specs.items():
-            if isinstance(k, tuple):
-                k, port = k
-            else:
-                port = 0
-
-            if isinstance(v, tuple):
-                v, kwargs = v
-            else:
-                v, kwargs = v, {}
-            self.services[k] = v(self, io_loop=self.loop, **kwargs)
-            self.services[k].listen((listen_ip, port))
-            self.service_ports[k] = self.services[k].port
-
-    def send_to_worker(self, address, msg):
-        if address not in self.stream_comms:
-            bcomm = BatchedSend(interval='1ms', loop=self.loop)
-            self.stream_comms[address] = bcomm
-
-            @gen.coroutine
-            def batched_send_connect():
-                comm = yield connect(address,  # TODO, serialization
-                                     connection_args=self.connection_args)
-                yield comm.write({'op': 'connection_stream'})
-
-                bcomm.start(comm)
-
-            self.loop.add_callback(batched_send_connect)
-
-        self.stream_comms[address].send(msg)
-
-    @gen.coroutine
-    def _start(self, addr_or_port=0):
-        assert self.status is None
-
-        enable_gc_diagnosis()
-        thread_state.on_event_loop_thread = True
-
-        # XXX Factor this out
-        if not addr_or_port:
-            # Default address is the required one to reach the scheduler
-            listen_host = get_address_host(self.scheduler.address)
-            self.listen(get_local_address_for(self.scheduler.address),
-                        listen_args=self.listen_args)
-            self.ip = get_address_host(self.address)
-        elif isinstance(addr_or_port, int):
-            # addr_or_port is an integer => assume TCP
-            listen_host = self.ip = get_ip(
-                get_address_host(self.scheduler.address)
-            )
-            self.listen((listen_host, addr_or_port),
-                        listen_args=self.listen_args)
-        else:
-            self.listen(addr_or_port, listen_args=self.listen_args)
-            self.ip = get_address_host(self.address)
-            try:
-                listen_host = get_address_host(addr_or_port)
-            except ValueError:
-                listen_host = addr_or_port
-
-        if '://' in listen_host:
-            protocol, listen_host = listen_host.split('://')
-
-        self.name = self.name or self.address
-        preload_modules(self.preload, parameter=self, file_dir=self.local_dir, argv=self.preload_argv)
-        # Services listen on all addresses
-        # Note Nanny is not a "real" service, just some metadata
-        # passed in service_ports...
-        self.start_services(listen_host)
-
-        try:
-            listening_address = '%s%s:%d' % (self.listener.prefix, listen_host, self.port)
-        except Exception:
-            listening_address = '%s%s' % (self.listener.prefix, listen_host)
-
-        logger.info('      Start worker at: %26s', self.address)
-        logger.info('         Listening to: %26s', listening_address)
-        for k, v in self.service_ports.items():
-            logger.info('  %16s at: %26s' % (k, listen_host + ':' + str(v)))
-        logger.info('Waiting to connect to: %26s', self.scheduler.address)
-        logger.info('-' * 49)
-        logger.info('              Threads: %26d', self.ncores)
-        if self.memory_limit:
-            logger.info('               Memory: %26s', format_bytes(self.memory_limit))
-        logger.info('      Local Directory: %26s', self.local_dir)
-
-        setproctitle("dask-worker [%s]" % self.address)
-
-        yield self._register_with_scheduler()
-
-        self.start_periodic_callbacks()
-
-    def start(self, port=0):
-        self.loop.add_callback(self._start, port)
-
-    def identity(self, comm=None):
-        return {'type': type(self).__name__,
-                'id': self.id,
-                'scheduler': self.scheduler.address,
-                'ncores': self.ncores,
-                'memory_limit': self.memory_limit}
-
-    @gen.coroutine
-    def _close(self, report=True, timeout=10, nanny=True, executor_wait=True):
-        with log_errors():
-            if self.status in ('closed', 'closing'):
-                return
-
-            disable_gc_diagnosis()
-
-            logger.info("Stopping worker at %s", self.address)
-            self.status = 'closing'
-            setproctitle("dask-worker [closing]")
-
-            self.stop()
-            for pc in self.periodic_callbacks.values():
-                pc.stop()
-            with ignoring(EnvironmentError, gen.TimeoutError):
-                if report:
-                    yield gen.with_timeout(timedelta(seconds=timeout),
-                                           self.scheduler.unregister(address=self.contact_address))
-            self.scheduler.close_rpc()
-            self.actor_executor._work_queue.queue.clear()
-            if isinstance(self.executor, ThreadPoolExecutor):
-                self.executor._work_queue.queue.clear()
-                self.executor.shutdown(wait=executor_wait, timeout=timeout)
-            else:
-                self.executor.shutdown(wait=False)
-            self.actor_executor.shutdown(wait=executor_wait, timeout=timeout)
-            self._workdir.release()
-
-            for k, v in self.services.items():
-                v.stop()
-
-            self.status = 'closed'
-
-            if nanny and 'nanny' in self.service_ports:
-                with self.rpc((self.ip, self.service_ports['nanny'])) as r:
-                    yield r.terminate()
-
-            if self.batched_stream and not self.batched_stream.comm.closed():
-                self.batched_stream.send({'op': 'close-stream'})
-
-            if self.batched_stream:
-                self.batched_stream.close()
-
-            self.rpc.close()
-            self._closed.set()
-            self._remove_from_global_workers()
-            yield super(WorkerBase, self).close()
-
-            setproctitle("dask-worker [closed]")
+class Worker(ServerNode):
+    """ Worker node in a Dask distributed cluster
 
-    def __del__(self):
-        self._remove_from_global_workers()
+    Workers perform two functions:
 
-    def _remove_from_global_workers(self):
-        for ref in list(_global_workers):
-            if ref() is self:
-                _global_workers.remove(ref)
-            if ref() is None:
-                _global_workers.remove(ref)
+    1.  **Serve data** from a local dictionary
+    2.  **Perform computation** on that data and on data from peers
 
-    @gen.coroutine
-    def terminate(self, comm, report=True):
-        yield self._close(report=report)
-        raise Return('OK')
+    Workers keep the scheduler informed of their data and use that scheduler to
+    gather data from other workers when necessary to perform a computation.
 
-    @gen.coroutine
-    def wait_until_closed(self):
-        yield self._closed.wait()
-        assert self.status == 'closed'
+    You can start a worker with the ``dask-worker`` command line application::
 
-    @gen.coroutine
-    def executor_submit(self, key, function, args=(), kwargs=None,
-                        executor=None):
-        """ Safely run function in thread pool executor
+        $ dask-worker scheduler-ip:port
 
-        We've run into issues running concurrent.future futures within
-        tornado.  Apparently it's advantageous to use timeouts and periodic
-        callbacks to ensure things run smoothly.  This can get tricky, so we
-        pull it off into an separate method.
-        """
-        executor = executor or self.executor
-        job_counter[0] += 1
-        # logger.info("%s:%d Starts job %d, %s", self.ip, self.port, i, key)
-        kwargs = kwargs or {}
-        future = executor.submit(function, *args, **kwargs)
-        pc = PeriodicCallback(lambda: logger.debug("future state: %s - %s",
-                                                   key, future._state), 1000)
-        pc.start()
-        try:
-            yield future
-        finally:
-            pc.stop()
+    Use the ``--help`` flag to see more options
 
-        result = future.result()
+        $ dask-worker --help
 
-        # logger.info("Finish job %d, %s", i, key)
-        raise gen.Return(result)
+    The rest of this docstring is about the internal state the the worker uses
+    to manage and track internal computations.
 
-    def run(self, comm, function, args=(), kwargs=None):
-        kwargs = kwargs or {}
-        return run(self, comm, function=function, args=args, kwargs=kwargs)
+    **State**
 
-    def run_coroutine(self, comm, function, args=(), kwargs=None, wait=True):
-        return run(self, comm, function=function, args=args, kwargs=kwargs,
-                   is_coro=True, wait=wait)
+    **Informational State**
 
-    @gen.coroutine
-    def actor_execute(self, comm=None, actor=None, function=None, args=(), kwargs={}):
-        separate_thread = kwargs.pop('separate_thread', True)
-        key = actor
-        actor = self.actors[key]
-        func = getattr(actor, function)
-        name = key_split(key) + '.' + function
+    These attributes don't change significantly during execution.
 
-        if iscoroutinefunction(func):
-            result = yield func(*args, **kwargs)
-        elif separate_thread:
-            result = yield self.executor_submit(name,
-                                                apply_function_actor,
-                                                args=(func, args, kwargs,
-                                                      self.execution_state,
-                                                      name,
-                                                      self.active_threads,
-                                                      self.active_threads_lock),
-                                                executor=self.actor_executor)
-        else:
-            result = func(*args, **kwargs)
-        raise gen.Return({'status': 'OK', 'result': to_serialize(result)})
+    * **ncores:** ``int``:
+        Number of cores used by this worker process
+    * **executor:** ``concurrent.futures.ThreadPoolExecutor``:
+        Executor used to perform computation
+    * **local_dir:** ``path``:
+        Path on local machine to store temporary files
+    * **scheduler:** ``rpc``:
+        Location of scheduler.  See ``.ip/.port`` attributes.
+    * **name:** ``string``:
+        Alias
+    * **services:** ``{str: Server}``:
+        Auxiliary web servers running on this worker
+    * **service_ports:** ``{str: port}``:
+    * **total_out_connections**: ``int``
+        The maximum number of concurrent outgoing requests for data
+    * **total_in_connections**: ``int``
+        The maximum number of concurrent incoming requests for data
+    * **total_comm_nbytes**: ``int``
+    * **batched_stream**: ``BatchedSend``
+        A batched stream along which we communicate to the scheduler
+    * **log**: ``[(message)]``
+        A structured and queryable log.  See ``Worker.story``
 
-    def actor_attribute(self, comm=None, actor=None, attribute=None):
-        value = getattr(self.actors[actor], attribute)
-        return {'status': 'OK', 'result': to_serialize(value)}
+    **Volatile State**
 
-    def update_data(self, comm=None, data=None, report=True, serializers=None):
-        for key, value in data.items():
-            if key in self.task_state:
-                self.transition(key, 'memory', value=value)
-            else:
-                self.put_key_in_memory(key, value)
-                self.task_state[key] = 'memory'
-                self.tasks[key] = None
-                self.priorities[key] = None
-                self.durations[key] = None
-                self.dependencies[key] = set()
+    This attributes track the progress of tasks that this worker is trying to
+    complete.  In the descriptions below a ``key`` is the name of a task that
+    we want to compute and ``dep`` is the name of a piece of dependent data
+    that we want to collect from others.
 
-            if key in self.dep_state:
-                self.transition_dep(key, 'memory', value=value)
+    * **data:** ``{key: object}``:
+        Dictionary mapping keys to actual values
+    * **task_state**: ``{key: string}``:
+        The state of all tasks that the scheduler has asked us to compute.
+        Valid states include waiting, constrained, exeucuting, memory, erred
+    * **tasks**: ``{key: dict}``
+        The function, args, kwargs of a task.  We run this when appropriate
+    * **dependencies**: ``{key: {deps}}``
+        The data needed by this key to run
+    * **dependents**: ``{dep: {keys}}``
+        The keys that use this dependency
+    * **data_needed**: deque(keys)
+        The keys whose data we still lack, arranged in a deque
+    * **waiting_for_data**: ``{kep: {deps}}``
+        A dynamic verion of dependencies.  All dependencies that we still don't
+        have for a particular key.
+    * **ready**: [keys]
+        Keys that are ready to run.  Stored in a LIFO stack
+    * **constrained**: [keys]
+        Keys for which we have the data to run, but are waiting on abstract
+        resources like GPUs.  Stored in a FIFO deque
+    * **executing**: {keys}
+        Keys that are currently executing
+    * **executed_count**: int
+        A number of tasks that this worker has run in its lifetime
+    * **long_running**: {keys}
+        A set of keys of tasks that are running and have started their own
+        long-running clients.
 
-            self.log.append((key, 'receive-from-scatter'))
+    * **dep_state**: ``{dep: string}``:
+        The state of all dependencies required by our tasks
+        Valid states include waiting, flight, and memory
+    * **who_has**: ``{dep: {worker}}``
+        Workers that we believe have this data
+    * **has_what**: ``{worker: {deps}}``
+        The data that we care about that we think a worker has
+    * **pending_data_per_worker**: ``{worker: [dep]}``
+        The data on each worker that we still want, prioritized as a deque
+    * **in_flight_tasks**: ``{task: worker}``
+        All dependencies that are coming to us in current peer-to-peer
+        connections and the workers from which they are coming.
+    * **in_flight_workers**: ``{worker: {task}}``
+        The workers from which we are currently gathering data and the
+        dependencies we expect from those connections
+    * **comm_bytes**: ``int``
+        The total number of bytes in flight
+    * **suspicious_deps**: ``{dep: int}``
+        The number of times a dependency has not been where we expected it
 
-        if report:
-            self.batched_stream.send({'op': 'add-keys',
-                                      'keys': list(data)})
-        info = {'nbytes': {k: sizeof(v) for k, v in data.items()},
-                'status': 'OK'}
-        return info
+    * **nbytes**: ``{key: int}``
+        The size of a particular piece of data
+    * **types**: ``{key: type}``
+        The type of a particular piece of data
+    * **threads**: ``{key: int}``
+        The ID of the thread on which the task ran
+    * **active_threads**: ``{int: key}``
+        The keys currently running on active threads
+    * **exceptions**: ``{key: exception}``
+        The exception caused by running a task if it erred
+    * **tracebacks**: ``{key: traceback}``
+        The exception caused by running a task if it erred
+    * **startstops**: ``{key: [(str, float, float)]}``
+        Log of transfer, load, and compute times for a task
 
-    @gen.coroutine
-    def delete_data(self, comm=None, keys=None, report=True):
-        if keys:
-            for key in list(keys):
-                self.log.append((key, 'delete'))
-                if key in self.task_state:
-                    self.release_key(key)
+    * **priorities**: ``{key: tuple}``
+        The priority of a key given by the scheduler.  Determines run order.
+    * **durations**: ``{key: float}``
+        Expected duration of a task
+    * **resource_restrictions**: ``{key: {str: number}}``
+        Abstract resources required to run a task
 
-                if key in self.dep_state:
-                    self.release_dep(key)
+    Parameters
+    ----------
+    scheduler_ip: str
+    scheduler_port: int
+    ip: str, optional
+    ncores: int, optional
+    loop: tornado.ioloop.IOLoop
+    local_dir: str, optional
+        Directory where we place local resources
+    name: str, optional
+    memory_limit: int, float, string
+        Number of bytes of memory that this worker should use.
+        Set to zero for no limit.  Set to 'auto' for 60% of memory use.
+        Use strings or numbers like 5GB or 5e9
+    memory_target_fraction: float
+        Fraction of memory to try to stay beneath
+    memory_spill_fraction: float
+        Fraction of memory at which we start spilling to disk
+    memory_pause_fraction: float
+        Fraction of memory at which we stop running new tasks
+    executor: concurrent.futures.Executor
+    resources: dict
+        Resources that thiw worker has like ``{'GPU': 2}``
 
-            logger.debug("Deleted %d keys", len(keys))
-            if report:
-                logger.debug("Reporting loss of keys to scheduler")
-                # TODO: this route seems to not exist?
-                yield self.scheduler.remove_keys(address=self.contact_address,
-                                                 keys=list(keys))
-        raise Return('OK')
+    Examples
+    --------
 
-    @gen.coroutine
-    def get_data(self, comm, keys=None, who=None, serializers=None,
-                 max_connections=None):
-        start = time()
+    Use the command line to start a worker::
 
-        if max_connections is None:
-            max_connections = self.total_in_connections
+        $ dask-scheduler
+        Start scheduler at 127.0.0.1:8786
 
-        # Allow same-host connections more liberally
-        if max_connections and comm and get_address_host(comm.peer_address) == get_address_host(self.address):
-            max_connections = max_connections * 2
+        $ dask-worker 127.0.0.1:8786
+        Start worker at:               127.0.0.1:1234
+        Registered with scheduler at:  127.0.0.1:8786
 
-        if max_connections is not False and self.outgoing_current_count > max_connections:
-            raise gen.Return({'status': 'busy'})
+    See Also
+    --------
+    distributed.scheduler.Scheduler
+    distributed.nanny.Nanny
+    """
+
+    def __init__(self, scheduler_ip=None, scheduler_port=None,
+                 scheduler_file=None, ncores=None, loop=None, local_dir='dask-worker-space',
+                 services=None, service_ports=None, name=None,
+                 reconnect=True, memory_limit='auto',
+                 executor=None, resources=None, silence_logs=None,
+                 death_timeout=None, preload=None, preload_argv=None, security=None,
+                 contact_address=None, memory_monitor_interval='200ms',
+                 extensions=None, metrics=None, **kwargs):
+        self.tasks = dict()
+        self.task_state = dict()
+        self.dep_state = dict()
+        self.dependencies = dict()
+        self.dependents = dict()
+        self.waiting_for_data = dict()
+        self.who_has = dict()
+        self.has_what = defaultdict(set)
+        self.pending_data_per_worker = defaultdict(deque)
+        self._lock = threading.Lock()
 
-        self.outgoing_current_count += 1
-        data = {k: self.data[k] for k in keys if k in self.data}
+        self.data_needed = deque()  # TODO: replace with heap?
 
-        if len(data) < len(keys):
-            for k in set(keys) - set(data):
-                if k in self.actors:
-                    from .actor import Actor
-                    data[k] = Actor(type(self.actors[k]), self.address, k)
+        self.in_flight_tasks = dict()
+        self.in_flight_workers = dict()
+        self.total_out_connections = dask.config.get('distributed.worker.connections.outgoing')
+        self.total_in_connections = dask.config.get('distributed.worker.connections.incoming')
+        self.total_comm_nbytes = 10e6
+        self.comm_nbytes = 0
+        self.suspicious_deps = defaultdict(lambda: 0)
+        self._missing_dep_flight = set()
 
-        msg = {'status': 'OK',
-               'data': {k: to_serialize(v) for k, v in data.items()}}
-        nbytes = {k: self.nbytes.get(k) for k in data}
-        stop = time()
-        if self.digests is not None:
-            self.digests['get-data-load-duration'].add(stop - start)
-        start = time()
+        self.nbytes = dict()
+        self.types = dict()
+        self.threads = dict()
+        self.exceptions = dict()
+        self.tracebacks = dict()
 
-        try:
-            compressed = yield comm.write(msg, serializers=serializers)
-            response = yield comm.read(deserializers=serializers)
-            assert response == 'OK', response
-        except EnvironmentError:
-            logger.exception('failed during get data with %s -> %s',
-                             self.address, who, exc_info=True)
-            comm.abort()
-            raise
-        finally:
-            self.outgoing_current_count -= 1
-        stop = time()
-        if self.digests is not None:
-            self.digests['get-data-send-duration'].add(stop - start)
+        self.active_threads_lock = threading.Lock()
+        self.active_threads = dict()
+        self.profile_keys = defaultdict(profile.create)
+        self.profile_keys_history = deque(maxlen=3600)
+        self.profile_recent = profile.create()
+        self.profile_history = deque(maxlen=3600)
 
-        total_bytes = sum(filter(None, nbytes.values()))
+        self.priorities = dict()
+        self.generation = 0
+        self.durations = dict()
+        self.startstops = defaultdict(list)
+        self.resource_restrictions = dict()
 
-        self.outgoing_count += 1
-        duration = (stop - start) or 0.5  # windows
-        self.outgoing_transfer_log.append({
-            'start': start + self.scheduler_delay,
-            'stop': stop + self.scheduler_delay,
-            'middle': (start + stop) / 2,
-            'duration': duration,
-            'who': who,
-            'keys': nbytes,
-            'total': total_bytes,
-            'compressed': compressed,
-            'bandwidth': total_bytes / duration
-        })
+        self.ready = list()
+        self.constrained = deque()
+        self.executing = set()
+        self.executed_count = 0
+        self.long_running = set()
 
-        raise gen.Return('dont-reply')
+        self.batched_stream = None
+        self.recent_messages_log = deque(maxlen=dask.config.get('distributed.comm.recent-messages-log-length'))
+        self.target_message_size = 50e6  # 50 MB
 
-    @gen.coroutine
-    def set_resources(self, **resources):
-        for r, quantity in resources.items():
-            if r in self.total_resources:
-                self.available_resources[r] += quantity - self.total_resources[r]
-            else:
-                self.available_resources[r] = quantity
-            self.total_resources[r] = quantity
+        self.log = deque(maxlen=100000)
+        self.validate = kwargs.pop('validate', False)
 
-        yield self.scheduler.set_resources(resources=self.total_resources,
-                                           worker=self.contact_address)
+        self._transitions = {
+            ('waiting', 'ready'): self.transition_waiting_ready,
+            ('waiting', 'memory'): self.transition_waiting_done,
+            ('waiting', 'error'): self.transition_waiting_done,
+            ('ready', 'executing'): self.transition_ready_executing,
+            ('ready', 'memory'): self.transition_ready_memory,
+            ('constrained', 'executing'): self.transition_constrained_executing,
+            ('executing', 'memory'): self.transition_executing_done,
+            ('executing', 'error'): self.transition_executing_done,
+            ('executing', 'rescheduled'): self.transition_executing_done,
+            ('executing', 'long-running'): self.transition_executing_long_running,
+            ('long-running', 'error'): self.transition_executing_done,
+            ('long-running', 'memory'): self.transition_executing_done,
+            ('long-running', 'rescheduled'): self.transition_executing_done,
+        }
 
-    def start_ipython(self, comm):
-        """Start an IPython kernel
+        self._dep_transitions = {
+            ('waiting', 'flight'): self.transition_dep_waiting_flight,
+            ('waiting', 'memory'): self.transition_dep_waiting_memory,
+            ('flight', 'waiting'): self.transition_dep_flight_waiting,
+            ('flight', 'memory'): self.transition_dep_flight_memory,
+        }
 
-        Returns Jupyter connection info dictionary.
-        """
-        from ._ipython_utils import start_ipython
-        if self._ipython_kernel is None:
-            self._ipython_kernel = start_ipython(
-                ip=self.ip,
-                ns={'worker': self},
-                log=logger,
-            )
-        return self._ipython_kernel.get_connection_info()
+        self.incoming_transfer_log = deque(maxlen=(100000))
+        self.incoming_count = 0
+        self.outgoing_transfer_log = deque(maxlen=(100000))
+        self.outgoing_count = 0
+        self.outgoing_current_count = 0
+        self.repetitively_busy = 0
+        self._client = None
 
-    @gen.coroutine
-    def upload_file(self, comm, filename=None, data=None, load=True):
-        out_filename = os.path.join(self.local_dir, filename)
+        profile_cycle_interval = kwargs.pop('profile_cycle_interval',
+                                            dask.config.get('distributed.worker.profile.cycle'))
+        profile_cycle_interval = parse_timedelta(profile_cycle_interval, default='ms')
 
-        def func(data):
-            if isinstance(data, unicode):
-                data = data.encode()
-            with open(out_filename, 'wb') as f:
-                f.write(data)
-                f.flush()
-            return data
+        self._setup_logging()
 
-        if len(data) < 10000:
-            data = func(data)
+        if scheduler_file:
+            cfg = json_load_robust(scheduler_file)
+            scheduler_addr = cfg['address']
+        elif scheduler_ip is None and dask.config.get('scheduler-address', None):
+            scheduler_addr = dask.config.get('scheduler-address')
+        elif scheduler_port is None:
+            scheduler_addr = coerce_to_address(scheduler_ip)
         else:
-            data = yield offload(func, data)
-
-        if load:
-            try:
-                import_file(out_filename)
-            except Exception as e:
-                logger.exception(e)
-                raise gen.Return({'status': 'error',
-                                  'exception': to_serialize(e)})
-
-        raise gen.Return({'status': 'OK', 'nbytes': len(data)})
+            scheduler_addr = coerce_to_address((scheduler_ip, scheduler_port))
+        self._port = 0
+        self.ncores = ncores or _ncores
+        self.total_resources = resources or {}
+        self.available_resources = (resources or {}).copy()
+        self.death_timeout = death_timeout
+        self.preload = preload
+        if self.preload is None:
+            self.preload = dask.config.get('distributed.worker.preload')
+        self.preload_argv = preload_argv
+        if self.preload_argv is None:
+            self.preload_argv = dask.config.get('distributed.worker.preload-argv')
+        self.contact_address = contact_address
+        self.memory_monitor_interval = parse_timedelta(memory_monitor_interval, default='ms')
+        self.extensions = dict()
+        if silence_logs:
+            silence_logging(level=silence_logs)
 
-    def keys(self, comm=None):
-        return list(self.data)
+        self._workspace = WorkSpace(os.path.abspath(local_dir))
+        self._workdir = self._workspace.new_work_dir(prefix='worker-')
+        self.local_dir = self._workdir.dir_path
 
-    @gen.coroutine
-    def gather(self, comm=None, who_has=None):
-        who_has = {k: [coerce_to_address(addr) for addr in v]
-                   for k, v in who_has.items()
-                   if k not in self.data}
-        result, missing_keys, missing_workers = yield gather_from_workers(
-            who_has, rpc=self.rpc, who=self.address)
-        if missing_keys:
-            logger.warning("Could not find data: %s on workers: %s (who_has: %s)",
-                           missing_keys, missing_workers, who_has)
-            raise Return({'status': 'missing-data',
-                          'keys': missing_keys})
-        else:
-            self.update_data(data=result, report=False)
-            raise Return({'status': 'OK'})
+        self.security = security or Security()
+        assert isinstance(self.security, Security)
+        self.connection_args = self.security.get_connection_args('worker')
+        self.listen_args = self.security.get_listen_args('worker')
 
+        self.memory_limit = parse_memory_limit(memory_limit, self.ncores)
 
-job_counter = [0]
+        self.paused = False
 
+        if 'memory_target_fraction' in kwargs:
+            self.memory_target_fraction = kwargs.pop('memory_target_fraction')
+        else:
+            self.memory_target_fraction = dask.config.get('distributed.worker.memory.target')
+        if 'memory_spill_fraction' in kwargs:
+            self.memory_spill_fraction = kwargs.pop('memory_spill_fraction')
+        else:
+            self.memory_spill_fraction = dask.config.get('distributed.worker.memory.spill')
+        if 'memory_pause_fraction' in kwargs:
+            self.memory_pause_fraction = kwargs.pop('memory_pause_fraction')
+        else:
+            self.memory_pause_fraction = dask.config.get('distributed.worker.memory.pause')
 
-def _deserialize(function=None, args=None, kwargs=None, task=None):
-    """ Deserialize task inputs and regularize to func, args, kwargs """
-    if function is not None:
-        function = pickle.loads(function)
-    if args:
-        args = pickle.loads(args)
-    if kwargs:
-        kwargs = pickle.loads(kwargs)
+        if (self.memory_limit and
+                (self.memory_target_fraction or
+                 self.memory_spill_fraction)):
+            try:
+                from zict import Buffer, File, Func
+            except ImportError:
+                raise ImportError("Please `pip install zict` for spill-to-disk workers")
+            path = os.path.join(self.local_dir, 'storage')
+            storage = Func(partial(serialize_bytelist, on_error='raise'),
+                           deserialize_bytes,
+                           File(path))
+            target = int(float(self.memory_limit) * self.memory_target_fraction)
+            self.data = Buffer({}, storage, target, weight)
+        else:
+            self.data = dict()
 
-    if task is not None:
-        assert not function and not args and not kwargs
-        function = execute_task
-        args = (task,)
+        self.actors = {}
+        self.loop = loop or IOLoop.current()
+        self.status = None
+        self._closed = Event()
+        self.reconnect = reconnect
+        self.executor = executor or ThreadPoolExecutor(self.ncores)
+        self.actor_executor = ThreadPoolExecutor(1)
+        self.name = name
+        self.scheduler_delay = 0
+        self.stream_comms = dict()
+        self.heartbeat_active = False
+        self._ipython_kernel = None
 
-    return function, args or (), kwargs or {}
+        if self.local_dir not in sys.path:
+            sys.path.insert(0, self.local_dir)
 
+        self.services = {}
+        self.service_ports = service_ports or {}
+        self.service_specs = services or {}
+        self.metrics = dict(metrics) if metrics else {}
 
-def execute_task(task):
-    """ Evaluate a nested task
+        handlers = {
+            'gather': self.gather,
+            'run': self.run,
+            'run_coroutine': self.run_coroutine,
+            'get_data': self.get_data,
+            'update_data': self.update_data,
+            'delete_data': self.delete_data,
+            'terminate': self.terminate,
+            'ping': pingpong,
+            'upload_file': self.upload_file,
+            'start_ipython': self.start_ipython,
+            'call_stack': self.get_call_stack,
+            'profile': self.get_profile,
+            'profile_metadata': self.get_profile_metadata,
+            'get_logs': self.get_logs,
+            'keys': self.keys,
+            'versions': self.versions,
+            'actor_execute': self.actor_execute,
+            'actor_attribute': self.actor_attribute,
+        }
 
-    >>> inc = lambda x: x + 1
-    >>> execute_task((inc, 1))
-    2
-    >>> execute_task((sum, [1, 2, (inc, 3)]))
-    7
-    """
-    if istask(task):
-        func, args = task[0], task[1:]
-        return func(*map(execute_task, args))
-    elif isinstance(task, list):
-        return list(map(execute_task, task))
-    else:
-        return task
+        stream_handlers = {
+            'close': self._close,
+            'compute-task': self.add_task,
+            'release-task': partial(self.release_key, report=False),
+            'delete-data': self.delete_data,
+            'steal-request': self.steal_request,
+        }
 
+        super(Worker, self).__init__(
+                handlers=handlers,
+                stream_handlers=stream_handlers,
+                io_loop=self.loop,
+                connection_args=self.connection_args,
+                **kwargs)
 
-cache = dict()
+        self.scheduler = self.rpc(scheduler_addr)
+        self.execution_state = {'scheduler': self.scheduler.address,
+                                'ioloop': self.loop,
+                                'worker': self}
 
+        pc = PeriodicCallback(self.heartbeat, 1000, io_loop=self.io_loop)
+        self.periodic_callbacks['heartbeat'] = pc
+        self._address = contact_address
 
-def dumps_function(func):
-    """ Dump a function to bytes, cache functions """
-    try:
-        result = cache[func]
-    except KeyError:
-        result = pickle.dumps(func)
-        if len(result) < 100000:
-            cache[func] = result
-    except TypeError:
-        result = pickle.dumps(func)
-    return result
+        if self.memory_limit:
+            self._memory_monitoring = False
+            pc = PeriodicCallback(self.memory_monitor,
+                                  self.memory_monitor_interval * 1000,
+                                  io_loop=self.io_loop)
+            self.periodic_callbacks['memory'] = pc
 
+        if extensions is None:
+            extensions = DEFAULT_EXTENSIONS
+        for ext in extensions:
+            ext(self)
 
-def dumps_task(task):
-    """ Serialize a dask task
+        self._throttled_gc = ThrottledGC(logger=logger)
 
-    Returns a dict of bytestrings that can each be loaded with ``loads``
+        setproctitle("dask-worker [not started]")
 
-    Examples
-    --------
-    Either returns a task as a function, args, kwargs dict
+        pc = PeriodicCallback(
+                self.trigger_profile,
+                parse_timedelta(dask.config.get('distributed.worker.profile.interval'), default='ms') * 1000,
+                io_loop=self.io_loop
+        )
+        self.periodic_callbacks['profile'] = pc
 
-    >>> from operator import add
-    >>> dumps_task((add, 1))  # doctest: +SKIP
-    {'function': b'\x80\x04\x95\x00\x8c\t_operator\x94\x8c\x03add\x94\x93\x94.'
-     'args': b'\x80\x04\x95\x07\x00\x00\x00K\x01K\x02\x86\x94.'}
+        pc = PeriodicCallback(self.cycle_profile,
+                              profile_cycle_interval * 1000,
+                              io_loop=self.io_loop)
+        self.periodic_callbacks['profile-cycle'] = pc
 
-    Or as a single task blob if it can't easily decompose the result.  This
-    happens either if the task is highly nested, or if it isn't a task at all
+        _global_workers.append(weakref.ref(self))
 
-    >>> dumps_task(1)  # doctest: +SKIP
-    {'task': b'\x80\x04\x95\x03\x00\x00\x00\x00\x00\x00\x00K\x01.'}
-    """
-    if istask(task):
-        if task[0] is apply and not any(map(_maybe_complex, task[2:])):
-            d = {'function': dumps_function(task[1]),
-                 'args': warn_dumps(task[2])}
-            if len(task) == 4:
-                d['kwargs'] = warn_dumps(task[3])
-            return d
-        elif not any(map(_maybe_complex, task[1:])):
-            return {'function': dumps_function(task[0]),
-                    'args': warn_dumps(task[1:])}
-    return to_serialize(task)
+    ##################
+    # Administrative #
+    ##################
+
+    def __repr__(self):
+        return "<%s: %s, %s, stored: %d, running: %d/%d, ready: %d, comm: %d, waiting: %d>" % (
+            self.__class__.__name__, self.address, self.status,
+            len(self.data), len(self.executing), self.ncores,
+            len(self.ready), len(self.in_flight_tasks),
+            len(self.waiting_for_data))
 
+    def _setup_logging(self):
+        self._deque_handler = DequeHandler(n=dask.config.get('distributed.admin.log-length'))
+        self._deque_handler.setFormatter(logging.Formatter(dask.config.get('distributed.admin.log-format')))
+        logger.addHandler(self._deque_handler)
+        finalize(self, logger.removeHandler, self._deque_handler)
 
-_warn_dumps_warned = [False]
+    @property
+    def worker_address(self):
+        """ For API compatibility with Nanny """
+        return self.address
 
+    def get_metrics(self):
+        core = dict(executing=len(self.executing),
+                    in_memory=len(self.data),
+                    ready=len(self.ready),
+                    in_flight=len(self.in_flight_tasks))
+        custom = {k: metric(self) for k, metric in self.metrics.items()}
 
-def warn_dumps(obj, dumps=pickle.dumps, limit=1e6):
-    """ Dump an object to bytes, warn if those bytes are large """
-    b = dumps(obj)
-    if not _warn_dumps_warned[0] and len(b) > limit:
-        _warn_dumps_warned[0] = True
-        s = str(obj)
-        if len(s) > 70:
-            s = s[:50] + ' ... ' + s[-15:]
-        warnings.warn("Large object of size %s detected in task graph: \n"
-                      "  %s\n"
-                      "Consider scattering large objects ahead of time\n"
-                      "with client.scatter to reduce scheduler burden and \n"
-                      "keep data on workers\n\n"
-                      "    future = client.submit(func, big_data)    # bad\n\n"
-                      "    big_future = client.scatter(big_data)     # good\n"
-                      "    future = client.submit(func, big_future)  # good"
-                      % (format_bytes(len(b)), s))
-    return b
+        return merge(custom, self.monitor.recent(), core)
 
+    def identity(self, comm=None):
+        return {'type': type(self).__name__,
+                'id': self.id,
+                'scheduler': self.scheduler.address,
+                'ncores': self.ncores,
+                'memory_limit': self.memory_limit}
 
-def apply_function(function, args, kwargs, execution_state, key,
-                   active_threads, active_threads_lock, time_delay):
-    """ Run a function, collect information
+    #####################
+    # External Services #
+    #####################
 
-    Returns
-    -------
-    msg: dictionary with status, result/error, timings, etc..
-    """
-    ident = get_thread_identity()
-    with active_threads_lock:
-        active_threads[ident] = key
-    thread_state.start_time = time()
-    thread_state.execution_state = execution_state
-    thread_state.key = key
-    start = time()
-    try:
-        result = function(*args, **kwargs)
-    except Exception as e:
-        msg = error_message(e)
-        msg['op'] = 'task-erred'
-        msg['actual-exception'] = e
-    else:
-        msg = {'op': 'task-finished',
-               'status': 'OK',
-               'result': result,
-               'nbytes': sizeof(result),
-               'type': type(result) if result is not None else None}
-    finally:
-        end = time()
-    msg['start'] = start + time_delay
-    msg['stop'] = end + time_delay
-    msg['thread'] = ident
-    with active_threads_lock:
-        del active_threads[ident]
-    return msg
+    @gen.coroutine
+    def _register_with_scheduler(self):
+        self.periodic_callbacks['heartbeat'].stop()
+        start = time()
+        if self.contact_address is None:
+            self.contact_address = self.address
+        logger.info('-' * 49)
+        while True:
+            if self.death_timeout and time() > start + self.death_timeout:
+                yield self._close(timeout=1)
+                return
+            if self.status in ('closed', 'closing'):
+                raise gen.Return
+            try:
+                _start = time()
+                comm = yield connect(self.scheduler.address,
+                                     connection_args=self.connection_args)
+                yield comm.write(dict(op='register-worker',
+                                      reply=False,
+                                      address=self.contact_address,
+                                      keys=list(self.data),
+                                      ncores=self.ncores,
+                                      name=self.name,
+                                      nbytes=self.nbytes,
+                                      now=time(),
+                                      resources=self.total_resources,
+                                      memory_limit=self.memory_limit,
+                                      local_directory=self.local_dir,
+                                      services=self.service_ports,
+                                      pid=os.getpid(),
+                                      metrics=self.get_metrics()),
+                                 serializers=['msgpack'])
+                future = comm.read(deserializers=['msgpack'])
+                if self.death_timeout:
+                    diff = self.death_timeout - (time() - start)
+                    if diff < 0:
+                        continue
+                    future = gen.with_timeout(timedelta(seconds=diff), future)
+                response = yield future
+                _end = time()
+                middle = (_start + _end) / 2
+                self.scheduler_delay = response['time'] - middle
+                self.status = 'running'
+                break
+            except EnvironmentError:
+                logger.info('Waiting to connect to: %26s', self.scheduler.address)
+                yield gen.sleep(0.1)
+            except gen.TimeoutError:
+                logger.info("Timed out when connecting to scheduler")
+        if response['status'] != 'OK':
+            raise ValueError("Unexpected response from register: %r" %
+                             (response,))
+        else:
+            # Retrieve eventual init functions and run them
+            for function_bytes in response['worker-setups']:
+                setup_function = pickle.loads(function_bytes)
+                if has_arg(setup_function, 'dask_worker'):
+                    result = setup_function(dask_worker=self)
+                else:
+                    result = setup_function()
+                logger.info('Init function %s ran: output=%s' % (setup_function, result))
 
+            logger.info('        Registered to: %26s', self.scheduler.address)
+            logger.info('-' * 49)
 
-def apply_function_actor(function, args, kwargs, execution_state, key,
-                         active_threads, active_threads_lock):
-    """ Run a function, collect information
+        self.batched_stream = BatchedSend(interval='2ms', loop=self.loop)
+        self.batched_stream.start(comm)
+        self.periodic_callbacks['heartbeat'].start()
+        self.loop.add_callback(self.handle_scheduler, comm)
 
-    Returns
-    -------
-    msg: dictionary with status, result/error, timings, etc..
-    """
-    ident = get_thread_identity()
+    @gen.coroutine
+    def heartbeat(self):
+        if not self.heartbeat_active:
+            self.heartbeat_active = True
+            logger.debug("Heartbeat: %s" % self.address)
+            try:
+                start = time()
+                response = yield self.scheduler.heartbeat_worker(
+                    address=self.contact_address,
+                    now=time(),
+                    metrics=self.get_metrics()
+                )
+                end = time()
+                middle = (start + end) / 2
 
-    with active_threads_lock:
-        active_threads[ident] = key
+                if response['status'] == 'missing':
+                    yield self._register_with_scheduler()
+                    return
+                self.scheduler_delay = response['time'] - middle
+                self.periodic_callbacks['heartbeat'].callback_time = response['heartbeat-interval'] * 1000
+            except CommClosedError:
+                logger.warning("Heartbeat to scheduler failed")
+            finally:
+                self.heartbeat_active = False
+        else:
+            logger.debug("Heartbeat skipped: channel busy")
 
-    thread_state.execution_state = execution_state
-    thread_state.key = key
+    @gen.coroutine
+    def handle_scheduler(self, comm):
+        try:
+            yield self.handle_stream(comm, every_cycle=[self.ensure_communicating,
+                                                        self.ensure_computing])
+        except Exception as e:
+            logger.exception(e)
+            raise
+        finally:
+            if self.reconnect:
+                logger.info("Connection to scheduler broken.  Reconnecting...")
+                self.loop.add_callback(self._register_with_scheduler)
+            else:
+                yield self._close(report=False)
 
-    result = function(*args, **kwargs)
+    def start_ipython(self, comm):
+        """Start an IPython kernel
 
-    with active_threads_lock:
-        del active_threads[ident]
+        Returns Jupyter connection info dictionary.
+        """
+        from ._ipython_utils import start_ipython
+        if self._ipython_kernel is None:
+            self._ipython_kernel = start_ipython(
+                ip=self.ip,
+                ns={'worker': self},
+                log=logger,
+            )
+        return self._ipython_kernel.get_connection_info()
 
-    return result
+    @gen.coroutine
+    def upload_file(self, comm, filename=None, data=None, load=True):
+        out_filename = os.path.join(self.local_dir, filename)
 
+        def func(data):
+            if isinstance(data, unicode):
+                data = data.encode()
+            with open(out_filename, 'wb') as f:
+                f.write(data)
+                f.flush()
+            return data
 
-def get_msg_safe_str(msg):
-    """ Make a worker msg, which contains args and kwargs, safe to cast to str:
-    allowing for some arguments to raise exceptions during conversion and
-    ignoring them.
-    """
-    class Repr(object):
-        def __init__(self, f, val):
-            self._f = f
-            self._val = val
+        if len(data) < 10000:
+            data = func(data)
+        else:
+            data = yield offload(func, data)
 
-        def __repr__(self):
-            return self._f(self._val)
+        if load:
+            try:
+                import_file(out_filename)
+            except Exception as e:
+                logger.exception(e)
+                raise gen.Return({'status': 'error',
+                                  'exception': to_serialize(e)})
 
-    msg = msg.copy()
-    if "args" in msg:
-        msg["args"] = Repr(convert_args_to_str, msg["args"])
-    if "kwargs" in msg:
-        msg["kwargs"] = Repr(convert_kwargs_to_str, msg["kwargs"])
-    return msg
+        raise gen.Return({'status': 'OK', 'nbytes': len(data)})
 
+    def keys(self, comm=None):
+        return list(self.data)
 
-def convert_args_to_str(args, max_len=None):
-    """ Convert args to a string, allowing for some arguments to raise
-    exceptions during conversion and ignoring them.
-    """
-    length = 0
-    strs = ["" for i in range(len(args))]
-    for i, arg in enumerate(args):
-        try:
-            sarg = repr(arg)
-        except Exception:
-            sarg = "< could not convert arg to str >"
-        strs[i] = sarg
-        length += len(sarg) + 2
-        if max_len is not None and length > max_len:
-            return "({}".format(", ".join(strs[:i + 1]))[:max_len]
-    else:
-        return "({})".format(", ".join(strs))
+    @gen.coroutine
+    def gather(self, comm=None, who_has=None):
+        who_has = {k: [coerce_to_address(addr) for addr in v]
+                   for k, v in who_has.items()
+                   if k not in self.data}
+        result, missing_keys, missing_workers = yield gather_from_workers(
+            who_has, rpc=self.rpc, who=self.address)
+        if missing_keys:
+            logger.warning("Could not find data: %s on workers: %s (who_has: %s)",
+                           missing_keys, missing_workers, who_has)
+            raise Return({'status': 'missing-data',
+                          'keys': missing_keys})
+        else:
+            self.update_data(data=result, report=False)
+            raise Return({'status': 'OK'})
 
+    def get_logs(self, comm=None, n=None):
+        deque_handler = self._deque_handler
+        if n is None:
+            L = list(deque_handler.deque)
+        else:
+            L = deque_handler.deque
+            L = [L[-i] for i in range(min(n, len(L)))]
+        return [(msg.levelname, deque_handler.format(msg)) for msg in L]
 
-def convert_kwargs_to_str(kwargs, max_len=None):
-    """ Convert kwargs to a string, allowing for some arguments to raise
-    exceptions during conversion and ignoring them.
-    """
-    length = 0
-    strs = ["" for i in range(len(kwargs))]
-    for i, (argname, arg) in enumerate(kwargs.items()):
-        try:
-            sarg = repr(arg)
-        except Exception:
-            sarg = "< could not convert arg to str >"
-        skwarg = repr(argname) + ": " + sarg
-        strs[i] = skwarg
-        length += len(skwarg) + 2
-        if max_len is not None and length > max_len:
-            return "{{{}".format(", ".join(strs[:i + 1]))[:max_len]
-    else:
-        return "{{{}}}".format(", ".join(strs))
+    #############
+    # Lifecycle #
+    #############
 
+    def start_services(self, listen_ip=''):
+        for k, v in self.service_specs.items():
+            if isinstance(k, tuple):
+                k, port = k
+            else:
+                port = 0
 
-def weight(k, v):
-    return sizeof(v)
+            if isinstance(v, tuple):
+                v, kwargs = v
+            else:
+                v, kwargs = v, {}
+            self.services[k] = v(self, io_loop=self.loop, **kwargs)
+            self.services[k].listen((listen_ip, port))
+            self.service_ports[k] = self.services[k].port
 
+    @gen.coroutine
+    def _start(self, addr_or_port=0):
+        assert self.status is None
 
-@gen.coroutine
-def run(server, comm, function, args=(), kwargs={}, is_coro=False, wait=True):
-    assert wait or is_coro, "Combination not supported"
-    function = pickle.loads(function)
-    if args:
-        args = pickle.loads(args)
-    if kwargs:
-        kwargs = pickle.loads(kwargs)
-    if has_arg(function, 'dask_worker'):
-        kwargs['dask_worker'] = server
-    if has_arg(function, 'dask_scheduler'):
-        kwargs['dask_scheduler'] = server
-    logger.info("Run out-of-band function %r", funcname(function))
-    try:
-        result = function(*args, **kwargs)
-        if is_coro:
-            result = (yield result) if wait else None
-    except Exception as e:
-        logger.warning(" Run Failed\n"
-                       "Function: %s\n"
-                       "args:     %s\n"
-                       "kwargs:   %s\n",
-                       str(funcname(function))[:1000],
-                       convert_args_to_str(args, max_len=1000),
-                       convert_kwargs_to_str(kwargs, max_len=1000), exc_info=True)
+        enable_gc_diagnosis()
+        thread_state.on_event_loop_thread = True
+
+        # XXX Factor this out
+        if not addr_or_port:
+            # Default address is the required one to reach the scheduler
+            listen_host = get_address_host(self.scheduler.address)
+            self.listen(get_local_address_for(self.scheduler.address),
+                        listen_args=self.listen_args)
+            self.ip = get_address_host(self.address)
+        elif isinstance(addr_or_port, int):
+            # addr_or_port is an integer => assume TCP
+            listen_host = self.ip = get_ip(
+                get_address_host(self.scheduler.address)
+            )
+            self.listen((listen_host, addr_or_port),
+                        listen_args=self.listen_args)
+        else:
+            self.listen(addr_or_port, listen_args=self.listen_args)
+            self.ip = get_address_host(self.address)
+            try:
+                listen_host = get_address_host(addr_or_port)
+            except ValueError:
+                listen_host = addr_or_port
+
+        if '://' in listen_host:
+            protocol, listen_host = listen_host.split('://')
+
+        self.name = self.name or self.address
+        preload_modules(self.preload, parameter=self, file_dir=self.local_dir, argv=self.preload_argv)
+        # Services listen on all addresses
+        # Note Nanny is not a "real" service, just some metadata
+        # passed in service_ports...
+        self.start_services(listen_host)
 
-        response = error_message(e)
-    else:
-        response = {
-            'status': 'OK',
-            'result': to_serialize(result),
-        }
-    raise Return(response)
+        try:
+            listening_address = '%s%s:%d' % (self.listener.prefix, listen_host, self.port)
+        except Exception:
+            listening_address = '%s%s' % (self.listener.prefix, listen_host)
 
+        logger.info('      Start worker at: %26s', self.address)
+        logger.info('         Listening to: %26s', listening_address)
+        for k, v in self.service_ports.items():
+            logger.info('  %16s at: %26s' % (k, listen_host + ':' + str(v)))
+        logger.info('Waiting to connect to: %26s', self.scheduler.address)
+        logger.info('-' * 49)
+        logger.info('              Threads: %26d', self.ncores)
+        if self.memory_limit:
+            logger.info('               Memory: %26s', format_bytes(self.memory_limit))
+        logger.info('      Local Directory: %26s', self.local_dir)
 
-class Worker(WorkerBase):
-    """ Worker node in a Dask distributed cluster
+        setproctitle("dask-worker [%s]" % self.address)
 
-    Workers perform two functions:
+        yield self._register_with_scheduler()
 
-    1.  **Serve data** from a local dictionary
-    2.  **Perform computation** on that data and on data from peers
+        self.start_periodic_callbacks()
 
-    Workers keep the scheduler informed of their data and use that scheduler to
-    gather data from other workers when necessary to perform a computation.
+    def start(self, port=0):
+        self.loop.add_callback(self._start, port)
 
-    You can start a worker with the ``dask-worker`` command line application::
+    @gen.coroutine
+    def _close(self, report=True, timeout=10, nanny=True, executor_wait=True):
+        with log_errors():
+            if self.status in ('closed', 'closing'):
+                return
 
-        $ dask-worker scheduler-ip:port
+            disable_gc_diagnosis()
 
-    Use the ``--help`` flag to see more options
+            logger.info("Stopping worker at %s", self.address)
+            self.status = 'closing'
+            setproctitle("dask-worker [closing]")
 
-        $ dask-worker --help
+            self.stop()
+            for pc in self.periodic_callbacks.values():
+                pc.stop()
+            with ignoring(EnvironmentError, gen.TimeoutError):
+                if report:
+                    yield gen.with_timeout(timedelta(seconds=timeout),
+                                           self.scheduler.unregister(address=self.contact_address))
+            self.scheduler.close_rpc()
+            self.actor_executor._work_queue.queue.clear()
+            if isinstance(self.executor, ThreadPoolExecutor):
+                self.executor._work_queue.queue.clear()
+                self.executor.shutdown(wait=executor_wait, timeout=timeout)
+            else:
+                self.executor.shutdown(wait=False)
+            self.actor_executor.shutdown(wait=executor_wait, timeout=timeout)
+            self._workdir.release()
 
-    The rest of this docstring is about the internal state the the worker uses
-    to manage and track internal computations.
+            for k, v in self.services.items():
+                v.stop()
 
-    **State**
+            self.status = 'closed'
 
-    **Informational State**
+            if nanny and 'nanny' in self.service_ports:
+                with self.rpc((self.ip, self.service_ports['nanny'])) as r:
+                    yield r.terminate()
 
-    These attributes don't change significantly during execution.
+            if self.batched_stream and not self.batched_stream.comm.closed():
+                self.batched_stream.send({'op': 'close-stream'})
 
-    * **ncores:** ``int``:
-        Number of cores used by this worker process
-    * **executor:** ``concurrent.futures.ThreadPoolExecutor``:
-        Executor used to perform computation
-    * **local_dir:** ``path``:
-        Path on local machine to store temporary files
-    * **scheduler:** ``rpc``:
-        Location of scheduler.  See ``.ip/.port`` attributes.
-    * **name:** ``string``:
-        Alias
-    * **services:** ``{str: Server}``:
-        Auxiliary web servers running on this worker
-    * **service_ports:** ``{str: port}``:
-    * **total_out_connections**: ``int``
-        The maximum number of concurrent outgoing requests for data
-    * **total_in_connections**: ``int``
-        The maximum number of concurrent incoming requests for data
-    * **total_comm_nbytes**: ``int``
-    * **batched_stream**: ``BatchedSend``
-        A batched stream along which we communicate to the scheduler
-    * **log**: ``[(message)]``
-        A structured and queryable log.  See ``Worker.story``
+            if self.batched_stream:
+                self.batched_stream.close()
 
-    **Volatile State**
+            self.rpc.close()
+            self._closed.set()
+            self._remove_from_global_workers()
+            yield self.close()
 
-    This attributes track the progress of tasks that this worker is trying to
-    complete.  In the descriptions below a ``key`` is the name of a task that
-    we want to compute and ``dep`` is the name of a piece of dependent data
-    that we want to collect from others.
+            setproctitle("dask-worker [closed]")
 
-    * **data:** ``{key: object}``:
-        Dictionary mapping keys to actual values
-    * **task_state**: ``{key: string}``:
-        The state of all tasks that the scheduler has asked us to compute.
-        Valid states include waiting, constrained, exeucuting, memory, erred
-    * **tasks**: ``{key: dict}``
-        The function, args, kwargs of a task.  We run this when appropriate
-    * **dependencies**: ``{key: {deps}}``
-        The data needed by this key to run
-    * **dependents**: ``{dep: {keys}}``
-        The keys that use this dependency
-    * **data_needed**: deque(keys)
-        The keys whose data we still lack, arranged in a deque
-    * **waiting_for_data**: ``{kep: {deps}}``
-        A dynamic verion of dependencies.  All dependencies that we still don't
-        have for a particular key.
-    * **ready**: [keys]
-        Keys that are ready to run.  Stored in a LIFO stack
-    * **constrained**: [keys]
-        Keys for which we have the data to run, but are waiting on abstract
-        resources like GPUs.  Stored in a FIFO deque
-    * **executing**: {keys}
-        Keys that are currently executing
-    * **executed_count**: int
-        A number of tasks that this worker has run in its lifetime
-    * **long_running**: {keys}
-        A set of keys of tasks that are running and have started their own
-        long-running clients.
+    def __del__(self):
+        self._remove_from_global_workers()
 
-    * **dep_state**: ``{dep: string}``:
-        The state of all dependencies required by our tasks
-        Valid states include waiting, flight, and memory
-    * **who_has**: ``{dep: {worker}}``
-        Workers that we believe have this data
-    * **has_what**: ``{worker: {deps}}``
-        The data that we care about that we think a worker has
-    * **pending_data_per_worker**: ``{worker: [dep]}``
-        The data on each worker that we still want, prioritized as a deque
-    * **in_flight_tasks**: ``{task: worker}``
-        All dependencies that are coming to us in current peer-to-peer
-        connections and the workers from which they are coming.
-    * **in_flight_workers**: ``{worker: {task}}``
-        The workers from which we are currently gathering data and the
-        dependencies we expect from those connections
-    * **comm_bytes**: ``int``
-        The total number of bytes in flight
-    * **suspicious_deps**: ``{dep: int}``
-        The number of times a dependency has not been where we expected it
+    def _remove_from_global_workers(self):
+        for ref in list(_global_workers):
+            if ref() is self:
+                _global_workers.remove(ref)
+            if ref() is None:
+                _global_workers.remove(ref)
 
-    * **nbytes**: ``{key: int}``
-        The size of a particular piece of data
-    * **types**: ``{key: type}``
-        The type of a particular piece of data
-    * **threads**: ``{key: int}``
-        The ID of the thread on which the task ran
-    * **active_threads**: ``{int: key}``
-        The keys currently running on active threads
-    * **exceptions**: ``{key: exception}``
-        The exception caused by running a task if it erred
-    * **tracebacks**: ``{key: traceback}``
-        The exception caused by running a task if it erred
-    * **startstops**: ``{key: [(str, float, float)]}``
-        Log of transfer, load, and compute times for a task
+    @gen.coroutine
+    def terminate(self, comm, report=True):
+        yield self._close(report=report)
+        raise Return('OK')
 
-    * **priorities**: ``{key: tuple}``
-        The priority of a key given by the scheduler.  Determines run order.
-    * **durations**: ``{key: float}``
-        Expected duration of a task
-    * **resource_restrictions**: ``{key: {str: number}}``
-        Abstract resources required to run a task
+    @gen.coroutine
+    def wait_until_closed(self):
+        yield self._closed.wait()
+        assert self.status == 'closed'
 
-    Parameters
-    ----------
-    scheduler_ip: str
-    scheduler_port: int
-    ip: str, optional
-    ncores: int, optional
-    loop: tornado.ioloop.IOLoop
-    local_dir: str, optional
-        Directory where we place local resources
-    name: str, optional
-    memory_limit: int, float, string
-        Number of bytes of memory that this worker should use.
-        Set to zero for no limit.  Set to 'auto' for 60% of memory use.
-        Use strings or numbers like 5GB or 5e9
-    memory_target_fraction: float
-        Fraction of memory to try to stay beneath
-    memory_spill_fraction: float
-        Fraction of memory at which we start spilling to disk
-    memory_pause_fraction: float
-        Fraction of memory at which we stop running new tasks
-    executor: concurrent.futures.Executor
-    resources: dict
-        Resources that thiw worker has like ``{'GPU': 2}``
+    ################
+    # Worker Peers #
+    ################
 
-    Examples
-    --------
+    def send_to_worker(self, address, msg):
+        if address not in self.stream_comms:
+            bcomm = BatchedSend(interval='1ms', loop=self.loop)
+            self.stream_comms[address] = bcomm
 
-    Use the command line to start a worker::
+            @gen.coroutine
+            def batched_send_connect():
+                comm = yield connect(address,  # TODO, serialization
+                                     connection_args=self.connection_args)
+                yield comm.write({'op': 'connection_stream'})
 
-        $ dask-scheduler
-        Start scheduler at 127.0.0.1:8786
+                bcomm.start(comm)
 
-        $ dask-worker 127.0.0.1:8786
-        Start worker at:               127.0.0.1:1234
-        Registered with scheduler at:  127.0.0.1:8786
+            self.loop.add_callback(batched_send_connect)
 
-    See Also
-    --------
-    distributed.scheduler.Scheduler
-    distributed.nanny.Nanny
-    """
+        self.stream_comms[address].send(msg)
+
+    @gen.coroutine
+    def get_data(self, comm, keys=None, who=None, serializers=None,
+                 max_connections=None):
+        start = time()
+
+        if max_connections is None:
+            max_connections = self.total_in_connections
+
+        # Allow same-host connections more liberally
+        if max_connections and comm and get_address_host(comm.peer_address) == get_address_host(self.address):
+            max_connections = max_connections * 2
 
-    def __init__(self, *args, **kwargs):
-        self.tasks = dict()
-        self.task_state = dict()
-        self.dep_state = dict()
-        self.dependencies = dict()
-        self.dependents = dict()
-        self.waiting_for_data = dict()
-        self.who_has = dict()
-        self.has_what = defaultdict(set)
-        self.pending_data_per_worker = defaultdict(deque)
-        self._lock = threading.Lock()
+        if max_connections is not False and self.outgoing_current_count > max_connections:
+            raise gen.Return({'status': 'busy'})
 
-        self.data_needed = deque()  # TODO: replace with heap?
+        self.outgoing_current_count += 1
+        data = {k: self.data[k] for k in keys if k in self.data}
 
-        self.in_flight_tasks = dict()
-        self.in_flight_workers = dict()
-        self.total_out_connections = dask.config.get('distributed.worker.connections.outgoing')
-        self.total_in_connections = dask.config.get('distributed.worker.connections.incoming')
-        self.total_comm_nbytes = 10e6
-        self.comm_nbytes = 0
-        self.suspicious_deps = defaultdict(lambda: 0)
-        self._missing_dep_flight = set()
+        if len(data) < len(keys):
+            for k in set(keys) - set(data):
+                if k in self.actors:
+                    from .actor import Actor
+                    data[k] = Actor(type(self.actors[k]), self.address, k)
 
-        self.nbytes = dict()
-        self.types = dict()
-        self.threads = dict()
-        self.exceptions = dict()
-        self.tracebacks = dict()
+        msg = {'status': 'OK',
+               'data': {k: to_serialize(v) for k, v in data.items()}}
+        nbytes = {k: self.nbytes.get(k) for k in data}
+        stop = time()
+        if self.digests is not None:
+            self.digests['get-data-load-duration'].add(stop - start)
+        start = time()
 
-        self.active_threads_lock = threading.Lock()
-        self.active_threads = dict()
-        self.profile_keys = defaultdict(profile.create)
-        self.profile_keys_history = deque(maxlen=3600)
-        self.profile_recent = profile.create()
-        self.profile_history = deque(maxlen=3600)
+        try:
+            compressed = yield comm.write(msg, serializers=serializers)
+            response = yield comm.read(deserializers=serializers)
+            assert response == 'OK', response
+        except EnvironmentError:
+            logger.exception('failed during get data with %s -> %s',
+                             self.address, who, exc_info=True)
+            comm.abort()
+            raise
+        finally:
+            self.outgoing_current_count -= 1
+        stop = time()
+        if self.digests is not None:
+            self.digests['get-data-send-duration'].add(stop - start)
 
-        self.priorities = dict()
-        self.generation = 0
-        self.durations = dict()
-        self.startstops = defaultdict(list)
-        self.resource_restrictions = dict()
+        total_bytes = sum(filter(None, nbytes.values()))
 
-        self.ready = list()
-        self.constrained = deque()
-        self.executing = set()
-        self.executed_count = 0
-        self.long_running = set()
+        self.outgoing_count += 1
+        duration = (stop - start) or 0.5  # windows
+        self.outgoing_transfer_log.append({
+            'start': start + self.scheduler_delay,
+            'stop': stop + self.scheduler_delay,
+            'middle': (start + stop) / 2,
+            'duration': duration,
+            'who': who,
+            'keys': nbytes,
+            'total': total_bytes,
+            'compressed': compressed,
+            'bandwidth': total_bytes / duration
+        })
 
-        self.batched_stream = None
-        self.recent_messages_log = deque(maxlen=dask.config.get('distributed.comm.recent-messages-log-length'))
-        self.target_message_size = 50e6  # 50 MB
+        raise gen.Return('dont-reply')
 
-        self.log = deque(maxlen=100000)
-        self.validate = kwargs.pop('validate', False)
+    ###################
+    # Local Execution #
+    ###################
 
-        self._transitions = {
-            ('waiting', 'ready'): self.transition_waiting_ready,
-            ('waiting', 'memory'): self.transition_waiting_done,
-            ('waiting', 'error'): self.transition_waiting_done,
-            ('ready', 'executing'): self.transition_ready_executing,
-            ('ready', 'memory'): self.transition_ready_memory,
-            ('constrained', 'executing'): self.transition_constrained_executing,
-            ('executing', 'memory'): self.transition_executing_done,
-            ('executing', 'error'): self.transition_executing_done,
-            ('executing', 'rescheduled'): self.transition_executing_done,
-            ('executing', 'long-running'): self.transition_executing_long_running,
-            ('long-running', 'error'): self.transition_executing_done,
-            ('long-running', 'memory'): self.transition_executing_done,
-            ('long-running', 'rescheduled'): self.transition_executing_done,
-        }
+    def update_data(self, comm=None, data=None, report=True, serializers=None):
+        for key, value in data.items():
+            if key in self.task_state:
+                self.transition(key, 'memory', value=value)
+            else:
+                self.put_key_in_memory(key, value)
+                self.task_state[key] = 'memory'
+                self.tasks[key] = None
+                self.priorities[key] = None
+                self.durations[key] = None
+                self.dependencies[key] = set()
 
-        self._dep_transitions = {
-            ('waiting', 'flight'): self.transition_dep_waiting_flight,
-            ('waiting', 'memory'): self.transition_dep_waiting_memory,
-            ('flight', 'waiting'): self.transition_dep_flight_waiting,
-            ('flight', 'memory'): self.transition_dep_flight_memory,
-        }
+            if key in self.dep_state:
+                self.transition_dep(key, 'memory', value=value)
 
-        self.incoming_transfer_log = deque(maxlen=(100000))
-        self.incoming_count = 0
-        self.outgoing_transfer_log = deque(maxlen=(100000))
-        self.outgoing_count = 0
-        self.outgoing_current_count = 0
-        self.repetitively_busy = 0
-        self._client = None
+            self.log.append((key, 'receive-from-scatter'))
 
-        profile_cycle_interval = kwargs.pop('profile_cycle_interval',
-                                            dask.config.get('distributed.worker.profile.cycle'))
-        profile_cycle_interval = parse_timedelta(profile_cycle_interval, default='ms')
+        if report:
+            self.batched_stream.send({'op': 'add-keys',
+                                      'keys': list(data)})
+        info = {'nbytes': {k: sizeof(v) for k, v in data.items()},
+                'status': 'OK'}
+        return info
 
-        WorkerBase.__init__(self, *args, **kwargs)
+    @gen.coroutine
+    def delete_data(self, comm=None, keys=None, report=True):
+        if keys:
+            for key in list(keys):
+                self.log.append((key, 'delete'))
+                if key in self.task_state:
+                    self.release_key(key)
 
-        pc = PeriodicCallback(
-                self.trigger_profile,
-                parse_timedelta(dask.config.get('distributed.worker.profile.interval'), default='ms') * 1000,
-                io_loop=self.io_loop
-        )
-        self.periodic_callbacks['profile'] = pc
+                if key in self.dep_state:
+                    self.release_dep(key)
 
-        pc = PeriodicCallback(self.cycle_profile,
-                              profile_cycle_interval * 1000,
-                              io_loop=self.io_loop)
-        self.periodic_callbacks['profile-cycle'] = pc
+            logger.debug("Deleted %d keys", len(keys))
+            if report:
+                logger.debug("Reporting loss of keys to scheduler")
+                # TODO: this route seems to not exist?
+                yield self.scheduler.remove_keys(address=self.contact_address,
+                                                 keys=list(keys))
+        raise Return('OK')
 
-        _global_workers.append(weakref.ref(self))
+    @gen.coroutine
+    def set_resources(self, **resources):
+        for r, quantity in resources.items():
+            if r in self.total_resources:
+                self.available_resources[r] += quantity - self.total_resources[r]
+            else:
+                self.available_resources[r] = quantity
+            self.total_resources[r] = quantity
 
-    def __repr__(self):
-        return "<%s: %s, %s, stored: %d, running: %d/%d, ready: %d, comm: %d, waiting: %d>" % (
-            self.__class__.__name__, self.address, self.status,
-            len(self.data), len(self.executing), self.ncores,
-            len(self.ready), len(self.in_flight_tasks),
-            len(self.waiting_for_data))
+        yield self.scheduler.set_resources(resources=self.total_resources,
+                                           worker=self.contact_address)
 
-    ################
-    # Update Graph #
-    ################
+    ###################
+    # Task Management #
+    ###################
 
     def add_task(self, key, function=None, args=None, kwargs=None, task=None,
                  who_has=None, nbytes=None, priority=None, duration=None,
@@ -1462,10 +1149,6 @@ def add_task(self, key, function=None, args=None, kwargs=None, task=None,
                 pdb.set_trace()
             raise
 
-    ###############
-    # Transitions #
-    ###############
-
     def transition_dep(self, dep, finish, **kwargs):
         try:
             start = self.dep_state[dep]
@@ -1721,9 +1404,19 @@ def maybe_transition_long_running(self, key, compute_duration=None):
         if self.task_state.get(key) == 'executing':
             self.transition(key, 'long-running', compute_duration=compute_duration)
 
-    ##########################
-    # Gather Data from Peers #
-    ##########################
+    def stateof(self, key):
+        return {'executing': key in self.executing,
+                'waiting_for_data': key in self.waiting_for_data,
+                'heap': key in pluck(1, self.ready),
+                'data': key in self.data}
+
+    def story(self, *keys):
+        return [msg for msg in self.log
+                if any(key in msg for key in keys)
+                or any(key in c
+                       for key in keys
+                       for c in msg
+                       if isinstance(c, (tuple, list, set)))]
 
     def ensure_communicating(self):
         changed = True
@@ -2221,23 +1914,86 @@ def rescind_key(self, key):
                 if not self.dependents[dep]:
                     del self.dependents[dep]
 
-            if key not in self.dependents:
-                # if key in self.nbytes:
-                #     del self.nbytes[key]
-                if key in self.priorities:
-                    del self.priorities[key]
-                if key in self.durations:
-                    del self.durations[key]
-        except Exception as e:
-            logger.exception(e)
-            if LOG_PDB:
-                import pdb
-                pdb.set_trace()
-            raise
+            if key not in self.dependents:
+                # if key in self.nbytes:
+                #     del self.nbytes[key]
+                if key in self.priorities:
+                    del self.priorities[key]
+                if key in self.durations:
+                    del self.durations[key]
+        except Exception as e:
+            logger.exception(e)
+            if LOG_PDB:
+                import pdb
+                pdb.set_trace()
+            raise
+
+    ################
+    # Execute Task #
+    ################
+
+    @gen.coroutine
+    def executor_submit(self, key, function, args=(), kwargs=None,
+                        executor=None):
+        """ Safely run function in thread pool executor
+
+        We've run into issues running concurrent.future futures within
+        tornado.  Apparently it's advantageous to use timeouts and periodic
+        callbacks to ensure things run smoothly.  This can get tricky, so we
+        pull it off into an separate method.
+        """
+        executor = executor or self.executor
+        job_counter[0] += 1
+        # logger.info("%s:%d Starts job %d, %s", self.ip, self.port, i, key)
+        kwargs = kwargs or {}
+        future = executor.submit(function, *args, **kwargs)
+        pc = PeriodicCallback(lambda: logger.debug("future state: %s - %s",
+                                                   key, future._state), 1000)
+        pc.start()
+        try:
+            yield future
+        finally:
+            pc.stop()
+
+        result = future.result()
+
+        # logger.info("Finish job %d, %s", i, key)
+        raise gen.Return(result)
+
+    def run(self, comm, function, args=(), kwargs=None):
+        kwargs = kwargs or {}
+        return run(self, comm, function=function, args=args, kwargs=kwargs)
+
+    def run_coroutine(self, comm, function, args=(), kwargs=None, wait=True):
+        return run(self, comm, function=function, args=args, kwargs=kwargs,
+                   is_coro=True, wait=wait)
+
+    @gen.coroutine
+    def actor_execute(self, comm=None, actor=None, function=None, args=(), kwargs={}):
+        separate_thread = kwargs.pop('separate_thread', True)
+        key = actor
+        actor = self.actors[key]
+        func = getattr(actor, function)
+        name = key_split(key) + '.' + function
 
-    ################
-    # Execute Task #
-    ################
+        if iscoroutinefunction(func):
+            result = yield func(*args, **kwargs)
+        elif separate_thread:
+            result = yield self.executor_submit(name,
+                                                apply_function_actor,
+                                                args=(func, args, kwargs,
+                                                      self.execution_state,
+                                                      name,
+                                                      self.active_threads,
+                                                      self.active_threads_lock),
+                                                executor=self.actor_executor)
+        else:
+            result = func(*args, **kwargs)
+        raise gen.Return({'status': 'OK', 'result': to_serialize(result)})
+
+    def actor_attribute(self, comm=None, actor=None, attribute=None):
+        value = getattr(self.actors[actor], attribute)
+        return {'status': 'OK', 'result': to_serialize(value)}
 
     def meets_resource_constraints(self, key):
         if key not in self.resource_restrictions:
@@ -2546,14 +2302,9 @@ def get_call_stack(self, comm=None, keys=None):
         result = {k: profile.call_stack(frame) for k, frame in frames.items()}
         return result
 
-    def get_logs(self, comm=None, n=None):
-        deque_handler = self._deque_handler
-        if n is None:
-            L = list(deque_handler.deque)
-        else:
-            L = deque_handler.deque
-            L = [L[-i] for i in range(min(n, len(L)))]
-        return [(msg.levelname, deque_handler.format(msg)) for msg in L]
+    ##############
+    # Validation #
+    ##############
 
     def validate_key_memory(self, key):
         assert key in self.data or key in self.actors
@@ -2645,294 +2396,554 @@ def validate_state(self):
                 for w in workers:
                     assert key in self.has_what[w]
 
-            for worker, keys in self.has_what.items():
-                for k in keys:
-                    assert worker in self.who_has[k]
+            for worker, keys in self.has_what.items():
+                for k in keys:
+                    assert worker in self.who_has[k]
+
+            for key in self.task_state:
+                self.validate_key(key)
+
+            for dep in self.dep_state:
+                self.validate_dep(dep)
+
+            for key, deps in self.waiting_for_data.items():
+                if key not in self.data_needed:
+                    for dep in deps:
+                        assert (dep in self.in_flight_tasks or
+                                dep in self._missing_dep_flight or
+                                self.who_has[dep].issubset(self.in_flight_workers))
+
+            for key in self.tasks:
+                if self.task_state[key] == 'memory':
+                    assert isinstance(self.nbytes[key], int)
+                    assert key not in self.waiting_for_data
+                    assert key in self.data or key in self.actors
+
+        except Exception as e:
+            logger.exception(e)
+            if LOG_PDB:
+                import pdb
+                pdb.set_trace()
+            raise
+
+    #######################################
+    # Worker Clients (advanced workloads) #
+    #######################################
+
+    @property
+    def client(self):
+        with self._lock:
+            if self._client:
+                return self._client
+            else:
+                return self._get_client()
+
+    def _get_client(self, timeout=3):
+        """ Get local client attached to this worker
+
+        If no such client exists, create one
+
+        See Also
+        --------
+        get_client
+        """
+        try:
+            from .client import default_client
+            client = default_client()
+        except ValueError:  # no clients found, need to make a new one
+            pass
+        else:
+            if (client.scheduler and client.scheduler.address == self.scheduler.address
+                    or client._start_arg == self.scheduler.address):
+                self._client = client
+
+        if not self._client:
+            from .client import Client
+            asynchronous = self.loop is IOLoop.current()
+            self._client = Client(self.scheduler, loop=self.loop,
+                                  security=self.security,
+                                  set_as_default=True,
+                                  asynchronous=asynchronous,
+                                  direct_to_workers=True,
+                                  name='worker',
+                                  timeout=timeout)
+            if not asynchronous:
+                assert self._client.status == 'running'
+        return self._client
+
+    def get_current_task(self):
+        """ Get the key of the task we are currently running
+
+        This only makes sense to run within a task
+
+        Examples
+        --------
+        >>> from dask.distributed import get_worker
+        >>> def f():
+        ...     return get_worker().get_current_task()
+
+        >>> future = client.submit(f)  # doctest: +SKIP
+        >>> future.result()  # doctest: +SKIP
+        'f-1234'
+
+        See Also
+        --------
+        get_worker
+        """
+        return self.active_threads[get_thread_identity()]
+
+
+def get_worker():
+    """ Get the worker currently running this task
+
+    Examples
+    --------
+    >>> def f():
+    ...     worker = get_worker()  # The worker on which this task is running
+    ...     return worker.address
+
+    >>> future = client.submit(f)  # doctest: +SKIP
+    >>> future.result()  # doctest: +SKIP
+    'tcp://127.0.0.1:47373'
+
+    See Also
+    --------
+    get_client
+    worker_client
+    """
+    try:
+        return thread_state.execution_state['worker']
+    except AttributeError:
+        for ref in _global_workers[::-1]:
+            worker = ref()
+            if worker:
+                return worker
+        raise ValueError("No workers found")
+
+
+def get_client(address=None, timeout=3, resolve_address=True):
+    """Get a client while within a task.
+
+    This client connects to the same scheduler to which the worker is connected
+
+    Parameters
+    ----------
+    address : str, optional
+        The address of the scheduler to connect to. Defaults to the scheduler
+        the worker is connected to.
+    timeout : int, default 3
+        Timeout (in seconds) for getting the Client
+    resolve_address : bool, default True
+        Whether to resolve `address` to its canonical form.
+
+    Returns
+    -------
+    Client
+
+    Examples
+    --------
+    >>> def f():
+    ...     client = get_client()
+    ...     futures = client.map(lambda x: x + 1, range(10))  # spawn many tasks
+    ...     results = client.gather(futures)
+    ...     return sum(results)
+
+    >>> future = client.submit(f)  # doctest: +SKIP
+    >>> future.result()  # doctest: +SKIP
+    55
+
+    See Also
+    --------
+    get_worker
+    worker_client
+    secede
+    """
+    if address and resolve_address:
+        address = comm.resolve_address(address)
+    try:
+        worker = get_worker()
+    except ValueError:  # could not find worker
+        pass
+    else:
+        if not address or worker.scheduler.address == address:
+            return worker._get_client(timeout=timeout)
+
+    from .client import _get_global_client
+    client = _get_global_client()  # TODO: assumes the same scheduler
+    if client and (not address or client.scheduler.address == address):
+        return client
+    elif address:
+        from .client import Client
+        return Client(address, timeout=timeout)
+    else:
+        raise ValueError("No global client found and no address provided")
+
+
+def secede():
+    """
+    Have this task secede from the worker's thread pool
+
+    This opens up a new scheduling slot and a new thread for a new task. This
+    enables the client to schedule tasks on this node, which is
+    especially useful while waiting for other jobs to finish (e.g., with
+    ``client.gather``).
+
+    Examples
+    --------
+    >>> def mytask(x):
+    ...     # do some work
+    ...     client = get_client()
+    ...     futures = client.map(...)  # do some remote work
+    ...     secede()  # while that work happens, remove ourself from the pool
+    ...     return client.gather(futures)  # return gathered results
+
+    See Also
+    --------
+    get_client
+    get_worker
+    """
+    worker = get_worker()
+    tpe_secede()  # have this thread secede from the thread pool
+    duration = time() - thread_state.start_time
+    worker.loop.add_callback(worker.maybe_transition_long_running,
+                             thread_state.key, compute_duration=duration)
+
 
-            for key in self.task_state:
-                self.validate_key(key)
+class Reschedule(Exception):
+    """ Reschedule this task
 
-            for dep in self.dep_state:
-                self.validate_dep(dep)
+    Raising this exception will stop the current execution of the task and ask
+    the scheduler to reschedule this task, possibly on a different machine.
 
-            for key, deps in self.waiting_for_data.items():
-                if key not in self.data_needed:
-                    for dep in deps:
-                        assert (dep in self.in_flight_tasks or
-                                dep in self._missing_dep_flight or
-                                self.who_has[dep].issubset(self.in_flight_workers))
+    This does not guarantee that the task will move onto a different machine.
+    The scheduler will proceed through its normal heuristics to determine the
+    optimal machine to accept this task.  The machine will likely change if the
+    load across the cluster has significantly changed since first scheduling
+    the task.
+    """
+    pass
 
-            for key in self.tasks:
-                if self.task_state[key] == 'memory':
-                    assert isinstance(self.nbytes[key], int)
-                    assert key not in self.waiting_for_data
-                    assert key in self.data or key in self.actors
 
-        except Exception as e:
-            logger.exception(e)
-            if LOG_PDB:
-                import pdb
-                pdb.set_trace()
-            raise
+def parse_memory_limit(memory_limit, ncores):
+    if memory_limit is None:
+        return None
+    if memory_limit == 'auto':
+        memory_limit = int(TOTAL_MEMORY * min(1, ncores / _ncores))
+    with ignoring(ValueError, TypeError):
+        x = float(memory_limit)
+        if isinstance(x, float) and x <= 1:
+            return int(x * TOTAL_MEMORY)
 
-    def stateof(self, key):
-        return {'executing': key in self.executing,
-                'waiting_for_data': key in self.waiting_for_data,
-                'heap': key in pluck(1, self.ready),
-                'data': key in self.data}
+    if isinstance(memory_limit, (unicode, str)):
+        return parse_bytes(memory_limit)
+    else:
+        return int(memory_limit)
 
-    def story(self, *keys):
-        return [msg for msg in self.log
-                if any(key in msg for key in keys)
-                or any(key in c
-                       for key in keys
-                       for c in msg
-                       if isinstance(c, (tuple, list, set)))]
 
-    @property
-    def client(self):
-        with self._lock:
-            if self._client:
-                return self._client
-            else:
-                return self._get_client()
+@gen.coroutine
+def get_data_from_worker(rpc, keys, worker, who=None, max_connections=None,
+                         serializers=None, deserializers=None):
+    """ Get keys from worker
 
-    def _get_client(self, timeout=3):
-        """ Get local client attached to this worker
+    The worker has a two step handshake to acknowledge when data has been fully
+    delivered.  This function implements that handshake.
 
-        If no such client exists, create one
+    See Also
+    --------
+    Worker.get_data
+    Worker.gather_deps
+    utils_comm.gather_data_from_workers
+    """
+    if serializers is None:
+        serializers = rpc.serializers
+    if deserializers is None:
+        deserializers = rpc.deserializers
 
-        See Also
-        --------
-        get_client
-        """
+    comm = yield rpc.connect(worker)
+    try:
+        response = yield send_recv(comm,
+                                   serializers=serializers,
+                                   deserializers=deserializers,
+                                   op='get_data', keys=keys, who=who,
+                                   max_connections=max_connections)
         try:
-            from .client import default_client
-            client = default_client()
-        except ValueError:  # no clients found, need to make a new one
-            pass
+            status = response['status']
+        except KeyError:
+            raise ValueError("Unexpected response", response)
         else:
-            if (client.scheduler and client.scheduler.address == self.scheduler.address
-                    or client._start_arg == self.scheduler.address):
-                self._client = client
+            if status == 'OK':
+                yield comm.write('OK')
+    finally:
+        rpc.reuse(worker, comm)
 
-        if not self._client:
-            from .client import Client
-            asynchronous = self.loop is IOLoop.current()
-            self._client = Client(self.scheduler, loop=self.loop,
-                                  security=self.security,
-                                  set_as_default=True,
-                                  asynchronous=asynchronous,
-                                  direct_to_workers=True,
-                                  name='worker',
-                                  timeout=timeout)
-            if not asynchronous:
-                assert self._client.status == 'running'
-        return self._client
+    raise gen.Return(response)
 
-    def get_current_task(self):
-        """ Get the key of the task we are currently running
 
-        This only makes sense to run within a task
+job_counter = [0]
 
-        Examples
-        --------
-        >>> from dask.distributed import get_worker
-        >>> def f():
-        ...     return get_worker().get_current_task()
 
-        >>> future = client.submit(f)  # doctest: +SKIP
-        >>> future.result()  # doctest: +SKIP
-        'f-1234'
+def _deserialize(function=None, args=None, kwargs=None, task=None):
+    """ Deserialize task inputs and regularize to func, args, kwargs """
+    if function is not None:
+        function = pickle.loads(function)
+    if args:
+        args = pickle.loads(args)
+    if kwargs:
+        kwargs = pickle.loads(kwargs)
 
-        See Also
-        --------
-        get_worker
-        """
-        return self.active_threads[get_thread_identity()]
+    if task is not None:
+        assert not function and not args and not kwargs
+        function = execute_task
+        args = (task,)
 
+    return function, args or (), kwargs or {}
 
-def get_worker():
-    """ Get the worker currently running this task
+
+def execute_task(task):
+    """ Evaluate a nested task
+
+    >>> inc = lambda x: x + 1
+    >>> execute_task((inc, 1))
+    2
+    >>> execute_task((sum, [1, 2, (inc, 3)]))
+    7
+    """
+    if istask(task):
+        func, args = task[0], task[1:]
+        return func(*map(execute_task, args))
+    elif isinstance(task, list):
+        return list(map(execute_task, task))
+    else:
+        return task
+
+
+cache = dict()
+
+
+def dumps_function(func):
+    """ Dump a function to bytes, cache functions """
+    try:
+        result = cache[func]
+    except KeyError:
+        result = pickle.dumps(func)
+        if len(result) < 100000:
+            cache[func] = result
+    except TypeError:
+        result = pickle.dumps(func)
+    return result
+
+
+def dumps_task(task):
+    """ Serialize a dask task
+
+    Returns a dict of bytestrings that can each be loaded with ``loads``
 
     Examples
     --------
-    >>> def f():
-    ...     worker = get_worker()  # The worker on which this task is running
-    ...     return worker.address
+    Either returns a task as a function, args, kwargs dict
 
-    >>> future = client.submit(f)  # doctest: +SKIP
-    >>> future.result()  # doctest: +SKIP
-    'tcp://127.0.0.1:47373'
+    >>> from operator import add
+    >>> dumps_task((add, 1))  # doctest: +SKIP
+    {'function': b'\x80\x04\x95\x00\x8c\t_operator\x94\x8c\x03add\x94\x93\x94.'
+     'args': b'\x80\x04\x95\x07\x00\x00\x00K\x01K\x02\x86\x94.'}
 
-    See Also
-    --------
-    get_client
-    worker_client
+    Or as a single task blob if it can't easily decompose the result.  This
+    happens either if the task is highly nested, or if it isn't a task at all
+
+    >>> dumps_task(1)  # doctest: +SKIP
+    {'task': b'\x80\x04\x95\x03\x00\x00\x00\x00\x00\x00\x00K\x01.'}
     """
-    try:
-        return thread_state.execution_state['worker']
-    except AttributeError:
-        for ref in _global_workers[::-1]:
-            worker = ref()
-            if worker:
-                return worker
-        raise ValueError("No workers found")
+    if istask(task):
+        if task[0] is apply and not any(map(_maybe_complex, task[2:])):
+            d = {'function': dumps_function(task[1]),
+                 'args': warn_dumps(task[2])}
+            if len(task) == 4:
+                d['kwargs'] = warn_dumps(task[3])
+            return d
+        elif not any(map(_maybe_complex, task[1:])):
+            return {'function': dumps_function(task[0]),
+                    'args': warn_dumps(task[1:])}
+    return to_serialize(task)
 
 
-def get_client(address=None, timeout=3, resolve_address=True):
-    """Get a client while within a task.
+_warn_dumps_warned = [False]
+
+
+def warn_dumps(obj, dumps=pickle.dumps, limit=1e6):
+    """ Dump an object to bytes, warn if those bytes are large """
+    b = dumps(obj)
+    if not _warn_dumps_warned[0] and len(b) > limit:
+        _warn_dumps_warned[0] = True
+        s = str(obj)
+        if len(s) > 70:
+            s = s[:50] + ' ... ' + s[-15:]
+        warnings.warn("Large object of size %s detected in task graph: \n"
+                      "  %s\n"
+                      "Consider scattering large objects ahead of time\n"
+                      "with client.scatter to reduce scheduler burden and \n"
+                      "keep data on workers\n\n"
+                      "    future = client.submit(func, big_data)    # bad\n\n"
+                      "    big_future = client.scatter(big_data)     # good\n"
+                      "    future = client.submit(func, big_future)  # good"
+                      % (format_bytes(len(b)), s))
+    return b
 
-    This client connects to the same scheduler to which the worker is connected
 
-    Parameters
-    ----------
-    address : str, optional
-        The address of the scheduler to connect to. Defaults to the scheduler
-        the worker is connected to.
-    timeout : int, default 3
-        Timeout (in seconds) for getting the Client
-    resolve_address : bool, default True
-        Whether to resolve `address` to its canonical form.
+def apply_function(function, args, kwargs, execution_state, key,
+                   active_threads, active_threads_lock, time_delay):
+    """ Run a function, collect information
 
     Returns
     -------
-    Client
-
-    Examples
-    --------
-    >>> def f():
-    ...     client = get_client()
-    ...     futures = client.map(lambda x: x + 1, range(10))  # spawn many tasks
-    ...     results = client.gather(futures)
-    ...     return sum(results)
-
-    >>> future = client.submit(f)  # doctest: +SKIP
-    >>> future.result()  # doctest: +SKIP
-    55
-
-    See Also
-    --------
-    get_worker
-    worker_client
-    secede
+    msg: dictionary with status, result/error, timings, etc..
     """
-    if address and resolve_address:
-        address = comm.resolve_address(address)
+    ident = get_thread_identity()
+    with active_threads_lock:
+        active_threads[ident] = key
+    thread_state.start_time = time()
+    thread_state.execution_state = execution_state
+    thread_state.key = key
+    start = time()
     try:
-        worker = get_worker()
-    except ValueError:  # could not find worker
-        pass
+        result = function(*args, **kwargs)
+    except Exception as e:
+        msg = error_message(e)
+        msg['op'] = 'task-erred'
+        msg['actual-exception'] = e
     else:
-        if not address or worker.scheduler.address == address:
-            return worker._get_client(timeout=timeout)
+        msg = {'op': 'task-finished',
+               'status': 'OK',
+               'result': result,
+               'nbytes': sizeof(result),
+               'type': type(result) if result is not None else None}
+    finally:
+        end = time()
+    msg['start'] = start + time_delay
+    msg['stop'] = end + time_delay
+    msg['thread'] = ident
+    with active_threads_lock:
+        del active_threads[ident]
+    return msg
 
-    from .client import _get_global_client
-    client = _get_global_client()  # TODO: assumes the same scheduler
-    if client and (not address or client.scheduler.address == address):
-        return client
-    elif address:
-        from .client import Client
-        return Client(address, timeout=timeout)
-    else:
-        raise ValueError("No global client found and no address provided")
 
+def apply_function_actor(function, args, kwargs, execution_state, key,
+                         active_threads, active_threads_lock):
+    """ Run a function, collect information
 
-def secede():
+    Returns
+    -------
+    msg: dictionary with status, result/error, timings, etc..
     """
-    Have this task secede from the worker's thread pool
+    ident = get_thread_identity()
 
-    This opens up a new scheduling slot and a new thread for a new task. This
-    enables the client to schedule tasks on this node, which is
-    especially useful while waiting for other jobs to finish (e.g., with
-    ``client.gather``).
+    with active_threads_lock:
+        active_threads[ident] = key
 
-    Examples
-    --------
-    >>> def mytask(x):
-    ...     # do some work
-    ...     client = get_client()
-    ...     futures = client.map(...)  # do some remote work
-    ...     secede()  # while that work happens, remove ourself from the pool
-    ...     return client.gather(futures)  # return gathered results
+    thread_state.execution_state = execution_state
+    thread_state.key = key
 
-    See Also
-    --------
-    get_client
-    get_worker
-    """
-    worker = get_worker()
-    tpe_secede()  # have this thread secede from the thread pool
-    duration = time() - thread_state.start_time
-    worker.loop.add_callback(worker.maybe_transition_long_running,
-                             thread_state.key, compute_duration=duration)
+    result = function(*args, **kwargs)
 
+    with active_threads_lock:
+        del active_threads[ident]
 
-class Reschedule(Exception):
-    """ Reschedule this task
+    return result
 
-    Raising this exception will stop the current execution of the task and ask
-    the scheduler to reschedule this task, possibly on a different machine.
 
-    This does not guarantee that the task will move onto a different machine.
-    The scheduler will proceed through its normal heuristics to determine the
-    optimal machine to accept this task.  The machine will likely change if the
-    load across the cluster has significantly changed since first scheduling
-    the task.
+def get_msg_safe_str(msg):
+    """ Make a worker msg, which contains args and kwargs, safe to cast to str:
+    allowing for some arguments to raise exceptions during conversion and
+    ignoring them.
     """
-    pass
+    class Repr(object):
+        def __init__(self, f, val):
+            self._f = f
+            self._val = val
 
+        def __repr__(self):
+            return self._f(self._val)
 
-def parse_memory_limit(memory_limit, ncores):
-    if memory_limit is None:
-        return None
-    if memory_limit == 'auto':
-        memory_limit = int(TOTAL_MEMORY * min(1, ncores / _ncores))
-    with ignoring(ValueError, TypeError):
-        x = float(memory_limit)
-        if isinstance(x, float) and x <= 1:
-            return int(x * TOTAL_MEMORY)
+    msg = msg.copy()
+    if "args" in msg:
+        msg["args"] = Repr(convert_args_to_str, msg["args"])
+    if "kwargs" in msg:
+        msg["kwargs"] = Repr(convert_kwargs_to_str, msg["kwargs"])
+    return msg
 
-    if isinstance(memory_limit, (unicode, str)):
-        return parse_bytes(memory_limit)
+
+def convert_args_to_str(args, max_len=None):
+    """ Convert args to a string, allowing for some arguments to raise
+    exceptions during conversion and ignoring them.
+    """
+    length = 0
+    strs = ["" for i in range(len(args))]
+    for i, arg in enumerate(args):
+        try:
+            sarg = repr(arg)
+        except Exception:
+            sarg = "< could not convert arg to str >"
+        strs[i] = sarg
+        length += len(sarg) + 2
+        if max_len is not None and length > max_len:
+            return "({}".format(", ".join(strs[:i + 1]))[:max_len]
     else:
-        return int(memory_limit)
+        return "({})".format(", ".join(strs))
 
 
-@gen.coroutine
-def get_data_from_worker(rpc, keys, worker, who=None, max_connections=None,
-                         serializers=None, deserializers=None):
-    """ Get keys from worker
+def convert_kwargs_to_str(kwargs, max_len=None):
+    """ Convert kwargs to a string, allowing for some arguments to raise
+    exceptions during conversion and ignoring them.
+    """
+    length = 0
+    strs = ["" for i in range(len(kwargs))]
+    for i, (argname, arg) in enumerate(kwargs.items()):
+        try:
+            sarg = repr(arg)
+        except Exception:
+            sarg = "< could not convert arg to str >"
+        skwarg = repr(argname) + ": " + sarg
+        strs[i] = skwarg
+        length += len(skwarg) + 2
+        if max_len is not None and length > max_len:
+            return "{{{}".format(", ".join(strs[:i + 1]))[:max_len]
+    else:
+        return "{{{}}}".format(", ".join(strs))
 
-    The worker has a two step handshake to acknowledge when data has been fully
-    delivered.  This function implements that handshake.
 
-    See Also
-    --------
-    Worker.get_data
-    Worker.gather_deps
-    utils_comm.gather_data_from_workers
-    """
-    if serializers is None:
-        serializers = rpc.serializers
-    if deserializers is None:
-        deserializers = rpc.deserializers
+def weight(k, v):
+    return sizeof(v)
 
-    comm = yield rpc.connect(worker)
+
+@gen.coroutine
+def run(server, comm, function, args=(), kwargs={}, is_coro=False, wait=True):
+    assert wait or is_coro, "Combination not supported"
+    function = pickle.loads(function)
+    if args:
+        args = pickle.loads(args)
+    if kwargs:
+        kwargs = pickle.loads(kwargs)
+    if has_arg(function, 'dask_worker'):
+        kwargs['dask_worker'] = server
+    if has_arg(function, 'dask_scheduler'):
+        kwargs['dask_scheduler'] = server
+    logger.info("Run out-of-band function %r", funcname(function))
     try:
-        response = yield send_recv(comm,
-                                   serializers=serializers,
-                                   deserializers=deserializers,
-                                   op='get_data', keys=keys, who=who,
-                                   max_connections=max_connections)
-        try:
-            status = response['status']
-        except KeyError:
-            raise ValueError("Unexpected response", response)
-        else:
-            if status == 'OK':
-                yield comm.write('OK')
-    finally:
-        rpc.reuse(worker, comm)
+        result = function(*args, **kwargs)
+        if is_coro:
+            result = (yield result) if wait else None
+    except Exception as e:
+        logger.warning(" Run Failed\n"
+                       "Function: %s\n"
+                       "args:     %s\n"
+                       "kwargs:   %s\n",
+                       str(funcname(function))[:1000],
+                       convert_args_to_str(args, max_len=1000),
+                       convert_kwargs_to_str(kwargs, max_len=1000), exc_info=True)
 
-    raise gen.Return(response)
+        response = error_message(e)
+    else:
+        response = {
+            'status': 'OK',
+            'result': to_serialize(result),
+        }
+    raise Return(response)

From 8fd95ede2d3672c40e38f86503a2e38a46a50fb4 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 21 Nov 2018 10:19:18 -0500
Subject: [PATCH 0142/1550] Avoid warnings in pyarrow and msgpack (#2364)

---
 distributed/comm/tests/test_comms.py         | 5 ++++-
 distributed/protocol/arrow.py                | 7 +++++--
 distributed/protocol/serialize.py            | 2 +-
 distributed/protocol/tests/test_serialize.py | 4 ++--
 setup.cfg                                    | 9 +++++++++
 5 files changed, 21 insertions(+), 6 deletions(-)

diff --git a/distributed/comm/tests/test_comms.py b/distributed/comm/tests/test_comms.py
index a0155052942..e7927593f40 100644
--- a/distributed/comm/tests/test_comms.py
+++ b/distributed/comm/tests/test_comms.py
@@ -4,6 +4,7 @@
 import os
 import sys
 import threading
+import warnings
 
 import pytest
 
@@ -28,7 +29,9 @@
 
 EXTERNAL_IP4 = get_ip()
 if has_ipv6():
-    EXTERNAL_IP6 = get_ipv6()
+    with warnings.catch_warnings(record=True):
+        warnings.simplefilter('always')
+        EXTERNAL_IP6 = get_ipv6()
 
 
 ca_file = get_cert('tls-ca-cert.pem')
diff --git a/distributed/protocol/arrow.py b/distributed/protocol/arrow.py
index c90ba190cfc..012a91e6afc 100644
--- a/distributed/protocol/arrow.py
+++ b/distributed/protocol/arrow.py
@@ -3,6 +3,9 @@
 from .serialize import dask_serialize, dask_deserialize
 
 import pyarrow
+if pyarrow.__version__ < '0.10':
+    raise ImportError("Need pyarrow >= 0.10 . "
+                      "See https://arrow.apache.org/docs/python/install.html")
 
 
 @dask_serialize.register(pyarrow.RecordBatch)
@@ -11,7 +14,7 @@ def serialize_batch(batch):
     writer = pyarrow.RecordBatchStreamWriter(sink, batch.schema)
     writer.write_batch(batch)
     writer.close()
-    buf = sink.get_result()
+    buf = sink.getvalue()
     header = {}
     frames = [buf]
     return header, frames
@@ -30,7 +33,7 @@ def serialize_table(tbl):
     writer = pyarrow.RecordBatchStreamWriter(sink, tbl.schema)
     writer.write_table(tbl)
     writer.close()
-    buf = sink.get_result()
+    buf = sink.getvalue()
     header = {}
     frames = [buf]
     return header, frames
diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index a87c61c01fc..434e235f584 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -362,7 +362,7 @@ def deserialize_bytes(b):
     frames = unpack_frames(b)
     header, frames = frames[0], frames[1:]
     if header:
-        header = msgpack.loads(header, encoding='utf8', use_list=False)
+        header = msgpack.loads(header, raw=False, use_list=False)
     else:
         header = {}
     frames = decompress(header, frames)
diff --git a/distributed/protocol/tests/test_serialize.py b/distributed/protocol/tests/test_serialize.py
index 81b4bf8ea0c..64d7adc8d41 100644
--- a/distributed/protocol/tests/test_serialize.py
+++ b/distributed/protocol/tests/test_serialize.py
@@ -266,11 +266,11 @@ def my_dumps(obj, context=None):
 
 
 def my_loads(header, frames):
-    obj = MyObject(**msgpack.loads(frames[0], encoding='utf8'))
+    obj = MyObject(**msgpack.loads(frames[0], raw=False))
 
     # to provide something to test against, lets just attach the context to
     # the object itself
-    obj.context = msgpack.loads(frames[1], encoding='utf8')
+    obj.context = msgpack.loads(frames[1], raw=False)
     return obj
 
 
diff --git a/setup.cfg b/setup.cfg
index 9320b6fbf81..434b1fd258c 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -37,3 +37,12 @@ parentdir_prefix = distributed-
 
 [bdist_wheel]
 universal=1
+
+[tool:pytest]
+addopts = -rsx -v --durations=10
+minversion = 3.2
+# filterwarnings =
+#     error
+#     ignore::UserWarning
+#     ignore::ImportWarning
+#     ignore::PendingDeprecationWarning

From 3296ecd0d3ee63b09940ae3eee2e44e7577da56d Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 26 Nov 2018 16:38:49 -0500
Subject: [PATCH 0143/1550] Avoid race condition in Actor's Future (#2374)

Previously we would err if an actor's future had been created, and the
task finished, but the worker holding the ActorFuture hadn't yet been
informed that the task had finished.  Now we allow the status to be
"pending".

We also move logic from an async test to a sync test because previously
the worker-client would be the same as the main client, which created a
reference cycle and so the future wouldn't get cleaned up.
---
 distributed/actor.py            |  2 +-
 distributed/client.py           | 21 +++++++++++++--------
 distributed/tests/test_actor.py | 29 ++++++++++++++++++++++++++++-
 3 files changed, 42 insertions(+), 10 deletions(-)

diff --git a/distributed/actor.py b/distributed/actor.py
index 47bf09ab1bd..4f2dc5e5abd 100644
--- a/distributed/actor.py
+++ b/distributed/actor.py
@@ -118,7 +118,7 @@ def __dir__(self):
     def __getattr__(self, key):
         attr = getattr(self._cls, key)
 
-        if self._future and not self._future.status == 'finished':
+        if self._future and self._future.status not in ('finished', 'pending'):
             raise ValueError("Worker holding Actor was lost.  Status: " + self._future.status)
 
         if callable(attr):
diff --git a/distributed/client.py b/distributed/client.py
index 91dcb82e201..e758be8206c 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -2262,7 +2262,8 @@ def _graph_to_futures(self, dsk, keys, restrictions=None,
 
     def get(self, dsk, keys, restrictions=None, loose_restrictions=None,
             resources=None, sync=True, asynchronous=None, direct=None,
-            retries=None, priority=0, fifo_timeout='60s', **kwargs):
+            retries=None, priority=0, fifo_timeout='60s', actors=None,
+            **kwargs):
         """ Compute dask graph
 
         Parameters
@@ -2293,13 +2294,17 @@ def get(self, dsk, keys, restrictions=None, loose_restrictions=None,
         --------
         Client.compute: Compute asynchronous collections
         """
-        futures = self._graph_to_futures(dsk, set(flatten([keys])),
-                                         restrictions, loose_restrictions,
-                                         resources=resources,
-                                         fifo_timeout=fifo_timeout,
-                                         retries=retries,
-                                         user_priority=priority,
-                                         )
+        futures = self._graph_to_futures(
+            dsk,
+            keys=set(flatten([keys])),
+            restrictions=restrictions,
+            loose_restrictions=loose_restrictions,
+            resources=resources,
+            fifo_timeout=fifo_timeout,
+            retries=retries,
+            user_priority=priority,
+            actors=actors,
+        )
         packed = pack_data(keys, futures)
         if sync:
             if getattr(thread_state, 'key', False):
diff --git a/distributed/tests/test_actor.py b/distributed/tests/test_actor.py
index 8c4c6544dbe..463b6585af8 100644
--- a/distributed/tests/test_actor.py
+++ b/distributed/tests/test_actor.py
@@ -469,11 +469,38 @@ def check(counter, blanks):
     assert result == 0 + 1 + 2 + 3 + 4
 
     start = time()
-    while a.data or b.data or a.actors or b.actors:
+    while a.data or b.data:
         yield gen.sleep(0.01)
         assert time() < start + 2
 
 
+def test_compute_sync(client):
+    @dask.delayed
+    def f(n, counter):
+        assert isinstance(counter, Actor), type(counter)
+        for i in range(n):
+            counter.increment().result()
+
+    @dask.delayed
+    def check(counter, blanks):
+        return counter.n
+
+    counter = dask.delayed(Counter)()
+    values = [f(i, counter) for i in range(5)]
+    final = check(counter, values)
+
+    result = final.compute(actors=counter)
+    assert result == 0 + 1 + 2 + 3 + 4
+
+    def check(dask_worker):
+        return len(dask_worker.data) + len(dask_worker.actors)
+
+    start = time()
+    while any(client.run(check).values()):
+        sleep(0.01)
+        assert time() < start + 2
+
+
 @gen_cluster(client=True, ncores=[('127.0.0.1', 1)],
              config={'distributed.worker.profile.interval': '1ms'})
 def test_actors_in_profile(c, s, a):

From 9cd56a267477747025253a2a7ed05cf76500a9c2 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 28 Nov 2018 09:00:13 -0500
Subject: [PATCH 0144/1550] Support missing packages keyword in
 Client.get_versions (#2379)

Previously users would get a non-informative error if their scheduler
was too old to support the packages= keyword.  Now we try again without
the keyword.  This is unfortunate because it drops the packages=
keyword, but is good because it will give a more informative error.
---
 distributed/client.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/distributed/client.py b/distributed/client.py
index e758be8206c..04e0e4ccf73 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -3180,6 +3180,8 @@ def get_versions(self, check=False, packages=[]):
                              packages=packages)
         except KeyError:
             scheduler = None
+        except TypeError:  # packages keyword not supported
+            scheduler = sync(self.loop, self.scheduler.versions)  # this raises
 
         workers = sync(self.loop, self.scheduler.broadcast,
                        msg={'op': 'versions', 'packages': packages})

From 5741dad65f8d141d15379a385ae297e7ce19672e Mon Sep 17 00:00:00 2001
From: Jim Crist <jcrist@users.noreply.github.com>
Date: Wed, 28 Nov 2018 09:22:02 -0600
Subject: [PATCH 0145/1550] Fixup serializing masked arrays (#2373)

A few fixups for serializing masked arrays:

- Properly serialize `fill_value` across all dtypes
- Be more efficient when serializing masked arrays with nomask
- Improve test coverage across dtypes
---
 distributed/protocol/numpy.py            | 50 ++++++++++++++----------
 distributed/protocol/tests/test_numpy.py | 29 +++++++-------
 2 files changed, 43 insertions(+), 36 deletions(-)

diff --git a/distributed/protocol/numpy.py b/distributed/protocol/numpy.py
index fc12e2ef2d0..e596cbbb3cf 100644
--- a/distributed/protocol/numpy.py
+++ b/distributed/protocol/numpy.py
@@ -105,33 +105,41 @@ def deserialize_numpy_ndarray(header, frames):
 
 @dask_serialize.register(np.ma.core.MaskedArray)
 def serialize_numpy_maskedarray(x):
-    # Separate elements of the masked array that we need to deal with discretely.
-    data = x.data
-    mask = x.mask
-    fill_value = x.fill_value
-
-    # Make use of existing numpy serialization for the two ndarray elements of
-    # the masked array.
-    data_header, data_frames = serialize_numpy_ndarray(data)
-    mask_header, mask_frames = serialize_numpy_ndarray(mask)
+    data_header, frames = serialize_numpy_ndarray(x.data)
+    header = {'data-header': data_header,
+              'nframes': len(frames)}
+
+    # Serialize mask if present
+    if x.mask is not np.ma.nomask:
+        mask_header, mask_frames = serialize_numpy_ndarray(x.mask)
+        header['mask-header'] = mask_header
+        frames += mask_frames
+
+    # Only a few dtypes have python equivalents msgpack can serialize
+    if isinstance(x.fill_value, (np.integer, np.floating, np.bool_)):
+        serialized_fill_value = (False, x.fill_value.item())
+    else:
+        serialized_fill_value = (True, pickle.dumps(x.fill_value))
+    header['fill-value'] = serialized_fill_value
 
-    header = {"data-header": data_header,
-              "mask-header": mask_header,
-              "fill_value": fill_value,
-              "nframes": len(data_frames)}
-    return header, data_frames + mask_frames
+    return header, frames
 
 
 @dask_deserialize.register(np.ma.core.MaskedArray)
 def deserialize_numpy_maskedarray(header, frames):
-    data_frames = frames[:header["nframes"]]
-    mask_frames = frames[header["nframes"]:]
     data_header = header["data-header"]
-    mask_header = header["mask-header"]
-
-    # Get the individual elements of the masked array in order to reconstruct.
+    data_frames = frames[:header["nframes"]]
     data = deserialize_numpy_ndarray(data_header, data_frames)
-    mask = deserialize_numpy_ndarray(mask_header, mask_frames)
-    fill_value = header["fill_value"]
+
+    if 'mask-header' in header:
+        mask_header = header["mask-header"]
+        mask_frames = frames[header["nframes"]:]
+        mask = deserialize_numpy_ndarray(mask_header, mask_frames)
+    else:
+        mask = np.ma.nomask
+
+    pickled_fv, fill_value = header["fill-value"]
+    if pickled_fv:
+        fill_value = pickle.loads(fill_value)
 
     return np.ma.masked_array(data, mask=mask, fill_value=fill_value)
diff --git a/distributed/protocol/tests/test_numpy.py b/distributed/protocol/tests/test_numpy.py
index eae76623519..549b1cb3650 100644
--- a/distributed/protocol/tests/test_numpy.py
+++ b/distributed/protocol/tests/test_numpy.py
@@ -64,9 +64,6 @@ def test_serialize():
     np.zeros((1, 1000, 1000)),
     np.arange(12)[::2],  # non-contiguous array
     np.ones(shape=(5, 6)).astype(dtype=[('total', '<f8'), ('n', '<f8')]),
-    np.ma.masked_array((5, 6), mask=[True, False]),  # int array
-    np.ma.masked_array((5., 6.), mask=[True, False]),  # float array (different default fill_value)
-    np.ma.masked_array((5., 6.), mask=[True, False], fill_value=np.nan),
 ])
 def test_dumps_serialize_numpy(x):
     header, frames = serialize(x)
@@ -82,18 +79,20 @@ def test_dumps_serialize_numpy(x):
         assert x.strides == y.strides
 
 
-def test_masked_array_serialize():
-    data = (5, 6)
-    mask = [True, False]
-    fill_value = 999
-    x = np.ma.masked_array(data, mask=mask, fill_value=fill_value)
-    header, frames = serialize(x)
-    y = deserialize(header, frames)
-
-    # Explicitly test the particular elements of the masked array.
-    np.testing.assert_equal(data, y.data)
-    np.testing.assert_equal(mask, y.mask)
-    assert fill_value == y.fill_value
+@pytest.mark.parametrize('x', [
+    np.ma.masked_array([5, 6], mask=[True, False], fill_value=10, dtype='i4'),
+    np.ma.masked_array([5., 6.], mask=[True, False], fill_value=10, dtype='f4'),
+    np.ma.masked_array([5., 6.], mask=[True, False], fill_value=np.nan, dtype='f8'),
+    np.ma.masked_array([5., 6.], mask=np.ma.nomask, fill_value=np.nan, dtype='f8'),
+    np.ma.masked_array([True, False], mask=np.ma.nomask, fill_value=True, dtype='bool'),
+    np.ma.masked_array(['a', 'b'], mask=[True, False], fill_value='c', dtype='O')
+])
+def test_masked_array_serialize(x):
+    y, = loads(dumps([to_serialize(x)]))
+    assert x.data.dtype == y.data.dtype
+    np.testing.assert_equal(x.data, y.data)
+    np.testing.assert_equal(x.mask, y.mask)
+    np.testing.assert_equal(x.fill_value, y.fill_value)
 
 
 def test_dumps_serialize_numpy_custom_dtype():

From e6b1ef5027c4914170af12882ab4528dce2e2ba3 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 28 Nov 2018 10:47:33 -0500
Subject: [PATCH 0146/1550] bump version to 1.25.0

---
 docs/source/changelog.rst | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index b9f246d0600..bb76c3c92ca 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,17 @@
 Changelog
 =========
 
+1.25.0 - 2018-11-28
+-------------------
+
+-  Fixed the 404 error on the Scheduler Dashboard homepage (:pr:`2361`) `Michael Wheeler`_
+-  Consolidate two Worker classes into one (:pr:`2363`) `Matthew Rocklin`_
+-  Avoid warnings in pyarrow and msgpack (:pr:`2364`) `Matthew Rocklin`_
+-  Avoid race condition in Actor's Future (:pr:`2374`) `Matthew Rocklin`_
+-  Support missing packages keyword in Client.get_versions (:pr:`2379`) `Matthew Rocklin`_
+-  Fixup serializing masked arrays (:pr:`2373`) `Jim Crist`_
+
+
 1.24.2 - 2018-11-15
 -------------------
 
@@ -9,6 +20,7 @@ Changelog
 -  Avoid deprecation warnings (:pr:`2355`) (:pr:`2357`) `Matthew Rocklin`_
 -  Fix typo in worker documentation (:pr:`2349`) `Tom Rochette`_
 
+
 1.24.1 - 2018-11-09
 -------------------
 
@@ -846,3 +858,4 @@ significantly without many new features.
 .. _`NotSqrt`: https://github.com/NotSqrt
 .. _`Tom Rochette`: https://github.com/tomzx
 .. _`Roy Wedge`: https://github.com/rwedge
+.. _`Michael Wheeler`: https://github.com/mikewheel

From 1d8b8313f54a30a53edae4f03ca82b70d4927c99 Mon Sep 17 00:00:00 2001
From: Jim Crist <jcrist@users.noreply.github.com>
Date: Fri, 30 Nov 2018 10:09:55 -0600
Subject: [PATCH 0147/1550] Serialize numpy.ma.masked objects properly (#2384)

`numpy.ma.masked` is a singleton of type `np.ma.core.MaskedConstant`.
Since this singleton is a subclass of `np.ma.core.MaskedArray`, it was
erroneously being picked up by the serializer for `MaskedArray`s,
leading to errors. We now register a serializer/deserializer for
`MaskedConstant` directly, fixing the issue.
---
 distributed/protocol/numpy.py            | 10 ++++++++++
 distributed/protocol/tests/test_numpy.py |  7 ++++++-
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/distributed/protocol/numpy.py b/distributed/protocol/numpy.py
index e596cbbb3cf..3227a4bbaec 100644
--- a/distributed/protocol/numpy.py
+++ b/distributed/protocol/numpy.py
@@ -103,6 +103,16 @@ def deserialize_numpy_ndarray(header, frames):
         return x
 
 
+@dask_serialize.register(np.ma.core.MaskedConstant)
+def serialize_numpy_ma_masked(x):
+    return {}, []
+
+
+@dask_deserialize.register(np.ma.core.MaskedConstant)
+def deserialize_numpy_ma_masked(header, frames):
+    return np.ma.masked
+
+
 @dask_serialize.register(np.ma.core.MaskedArray)
 def serialize_numpy_maskedarray(x):
     data_header, frames = serialize_numpy_ndarray(x.data)
diff --git a/distributed/protocol/tests/test_numpy.py b/distributed/protocol/tests/test_numpy.py
index 549b1cb3650..c86a5f03199 100644
--- a/distributed/protocol/tests/test_numpy.py
+++ b/distributed/protocol/tests/test_numpy.py
@@ -87,7 +87,7 @@ def test_dumps_serialize_numpy(x):
     np.ma.masked_array([True, False], mask=np.ma.nomask, fill_value=True, dtype='bool'),
     np.ma.masked_array(['a', 'b'], mask=[True, False], fill_value='c', dtype='O')
 ])
-def test_masked_array_serialize(x):
+def test_serialize_numpy_ma_masked_array(x):
     y, = loads(dumps([to_serialize(x)]))
     assert x.data.dtype == y.data.dtype
     np.testing.assert_equal(x.data, y.data)
@@ -95,6 +95,11 @@ def test_masked_array_serialize(x):
     np.testing.assert_equal(x.fill_value, y.fill_value)
 
 
+def test_serialize_numpy_ma_masked():
+    y, = loads(dumps([to_serialize(np.ma.masked)]))
+    assert y is np.ma.masked
+
+
 def test_dumps_serialize_numpy_custom_dtype():
     from six.moves import builtins
     test_rational = pytest.importorskip('numpy.core.test_rational')

From 22c6f174644da7ff3e7861b5b02c36623e9690cc Mon Sep 17 00:00:00 2001
From: Jim Crist <jcrist@users.noreply.github.com>
Date: Fri, 30 Nov 2018 14:29:13 -0600
Subject: [PATCH 0148/1550] Turn off bokeh property validation in dashboard
 (#2387)

Bokeh 1.0.0 added a method to turn off property validation contextually,
improving performance. Here we make use that method to remove validation
in the dashboard update methods. For older versions of bokeh this PR is
a no-op.
---
 distributed/bokeh/components.py | 23 +++++++++++++++++------
 distributed/bokeh/scheduler.py  | 21 +++++++++++++++++----
 distributed/bokeh/utils.py      | 16 ++++++++++++++++
 distributed/bokeh/worker.py     | 10 +++++++++-
 4 files changed, 59 insertions(+), 11 deletions(-)

diff --git a/distributed/bokeh/components.py b/distributed/bokeh/components.py
index 14aec7308d2..a9a9e09fc55 100644
--- a/distributed/bokeh/components.py
+++ b/distributed/bokeh/components.py
@@ -5,7 +5,6 @@
 from time import time
 import weakref
 
-import bokeh
 from bokeh.layouts import row, column
 from bokeh.models import ( ColumnDataSource, Plot, DataRange1d, LinearAxis,
         HoverTool, BoxZoomTool, ResetTool, PanTool, WheelZoomTool, Range1d,
@@ -16,6 +15,7 @@
 from tornado import gen
 import toolz
 
+from .utils import without_property_validation, BOKEH_VERSION
 from ..diagnostics.progress_stream import nbytes_bar
 from .. import profile
 from ..utils import log_errors, parse_timedelta
@@ -70,6 +70,7 @@ def __init__(self, n_rectangles=1000, clear_interval='20s', **kwargs):
         # Required for update callback
         self.task_stream_index = [0]
 
+    @without_property_validation
     def update(self, messages):
         with log_errors():
             index = messages['task-events']['index']
@@ -211,6 +212,7 @@ def __init__(self, **kwargs):
         )
         self.root.add_tools(hover)
 
+    @without_property_validation
     def update(self, messages):
         with log_errors():
             msg = messages['progress']
@@ -260,6 +262,7 @@ def __init__(self, **kwargs):
 
         self.root = fig
 
+    @without_property_validation
     def update(self, messages):
         with log_errors():
             msg = messages['processing']
@@ -310,6 +313,7 @@ def __init__(self, **kwargs):
         self.states = data.pop('states')
         self.source = ColumnDataSource(data=data)
 
+        @without_property_validation
         def cb(attr, old, new):
             with log_errors():
                 try:
@@ -326,7 +330,7 @@ def cb(attr, old, new):
                 self.source.data.update(data)
                 self.source.selected = old
 
-        if bokeh.__version__ >= '1.0':
+        if BOKEH_VERSION >= '1.0.0':
             self.source.selected.on_change('indices', cb)
         else:
             self.source.on_change('selected', cb)
@@ -370,6 +374,7 @@ def cb(attr, old, new):
         self.root.yaxis.visible = False
         self.root.grid.visible = False
 
+    @without_property_validation
     def update(self, state):
         with log_errors():
             self.state = state
@@ -411,6 +416,7 @@ def __init__(self, server, doc=None, **kwargs):
 
         changing = [False]  # avoid repeated changes from within callback
 
+        @without_property_validation
         def cb(attr, old, new):
             if changing[0]:
                 return
@@ -434,7 +440,7 @@ def cb(attr, old, new):
                     self.source.selected = old
                 changing[0] = False
 
-        if bokeh.__version__ >= '1.0':
+        if BOKEH_VERSION >= '1.0.0':
             self.source.selected.on_change('indices', cb)
         else:
             self.source.on_change('selected', cb)
@@ -506,7 +512,7 @@ def ts_change(attr, old, new):
                     self.start = self.stop = None
                 self.trigger_update(update_metadata=False)
 
-        if bokeh.__version__ >= '1.0':
+        if BOKEH_VERSION >= '1.0.0':
             self.ts_source.selected.on_change('indices', ts_change)
         else:
             self.ts_source.on_change('selected', ts_change)
@@ -531,6 +537,7 @@ def select_cb(attr, old, new):
                                self.update_button, sizing_mode='scale_width'),
                            self.profile_plot, self.ts_plot, **kwargs)
 
+    @without_property_validation
     def update(self, state, metadata=None):
         with log_errors():
             self.state = state
@@ -550,6 +557,7 @@ def update(self, state, metadata=None):
 
                 self.ts_source.data.update(self.ts)
 
+    @without_property_validation
     def trigger_update(self, update_metadata=True):
         @gen.coroutine
         def cb():
@@ -587,6 +595,7 @@ def __init__(self, server, doc=None, **kwargs):
 
         changing = [False]  # avoid repeated changes from within callback
 
+        @without_property_validation
         def cb(attr, old, new):
             if changing[0]:
                 return
@@ -610,7 +619,7 @@ def cb(attr, old, new):
                     self.source.selected = old
                 changing[0] = False
 
-        if bokeh.__version__ >= '1.0':
+        if BOKEH_VERSION >= '1.0.0':
             self.source.selected.on_change('indices', cb)
         else:
             self.source.on_change('selected', cb)
@@ -682,7 +691,7 @@ def ts_change(attr, old, new):
                     self.start = self.stop = None
                 self.trigger_update()
 
-        if bokeh.__version__ >= '1.0':
+        if BOKEH_VERSION >= '1.0.0':
             self.ts_source.selected.on_change('indices', ts_change)
         else:
             self.ts_source.on_change('selected', ts_change)
@@ -697,6 +706,7 @@ def ts_change(attr, old, new):
                                sizing_mode='scale_width'),
                            self.profile_plot, self.ts_plot, **kwargs)
 
+    @without_property_validation
     def update(self, state):
         with log_errors():
             self.state = state
@@ -704,6 +714,7 @@ def update(self, state):
             self.states = data.pop('states')
             self.source.data.update(data)
 
+    @without_property_validation
     def trigger_update(self):
         self.state = profile.get_profile(self.log, start=self.start, stop=self.stop)
         data = profile.plot_data(self.state, profile_interval)
diff --git a/distributed/bokeh/scheduler.py b/distributed/bokeh/scheduler.py
index f3e3a512e8a..d8823701aab 100644
--- a/distributed/bokeh/scheduler.py
+++ b/distributed/bokeh/scheduler.py
@@ -1,6 +1,5 @@
 from __future__ import print_function, division, absolute_import
 
-from distutils.version import LooseVersion
 from functools import partial
 import logging
 import math
@@ -8,7 +7,6 @@
 from operator import add
 import os
 
-import bokeh
 from bokeh.layouts import column, row
 from bokeh.models import (ColumnDataSource, DataRange1d, HoverTool, ResetTool,
                           PanTool, WheelZoomTool, TapTool, OpenURL, Range1d, Plot, Quad,
@@ -32,7 +30,7 @@
                          add_periodic_callback)
 from .core import BokehServer
 from .worker import SystemMonitor, counters_doc
-from .utils import transpose
+from .utils import transpose, BOKEH_VERSION, without_property_validation
 from ..metrics import time
 from ..utils import log_errors, format_bytes, format_time
 from ..diagnostics.progress_stream import color_of, progress_quads, nbytes_bar
@@ -61,6 +59,7 @@
 inf = float('inf')
 
 
+@without_property_validation
 def update(source, data):
     """ Update source with data
 
@@ -126,6 +125,7 @@ def __init__(self, scheduler, **kwargs):
 
             self.root = fig
 
+    @without_property_validation
     def update(self):
         with log_errors():
             workers = list(self.scheduler.workers.values())
@@ -193,6 +193,7 @@ def __init__(self, scheduler, **kwargs):
                            left='left', right='right', bottom=0, top='top',
                            color='blue')
 
+    @without_property_validation
     def update(self):
         L = [len(ws.processing) for ws in self.scheduler.workers.values()]
         counts, x = np.histogram(L, bins=40)
@@ -229,6 +230,7 @@ def __init__(self, scheduler, **kwargs):
                            left='left', right='right', bottom=0, top='top',
                            color='blue')
 
+    @without_property_validation
     def update(self):
         nbytes = np.asarray([ws.nbytes for ws in self.scheduler.workers.values()])
         counts, x = np.histogram(nbytes, bins=40)
@@ -306,6 +308,7 @@ def __init__(self, scheduler, width=600, **kwargs):
 
             processing.y_range = nbytes.y_range
 
+    @without_property_validation
     def update(self):
         with log_errors():
             workers = list(self.scheduler.workers.values())
@@ -386,6 +389,7 @@ def __init__(self, scheduler, **kwargs):
 
         self.root = fig
 
+    @without_property_validation
     def update(self):
         with log_errors():
             result = {'time': [time() * 1000],
@@ -451,6 +455,7 @@ def convert(self, msgs):
 
         return d
 
+    @without_property_validation
     def update(self):
         with log_errors():
             log = self.steal.log
@@ -500,6 +505,7 @@ def __init__(self, scheduler, name, height=150, **kwargs):
 
         self.root = fig
 
+    @without_property_validation
     def update(self):
         with log_errors():
             log = self.scheduler.events[self.name]
@@ -553,6 +559,7 @@ def __init__(self, scheduler, n_rectangles=1000, clear_interval='20s', **kwargs)
         components.TaskStream.__init__(self, n_rectangles=n_rectangles,
                                        clear_interval=clear_interval, **kwargs)
 
+    @without_property_validation
     def update(self):
         if self.index == self.plugin.index:
             return
@@ -640,6 +647,7 @@ def __init__(self, scheduler, **kwargs):
         rect.nonselection_glyph = None
         self.root.add_tools(hover, tap)
 
+    @without_property_validation
     def update(self):
         with log_errors():
             # occasionally reset the column data source to remove old nodes
@@ -658,6 +666,7 @@ def update(self):
 
             self.patch_updates()
 
+    @without_property_validation
     def add_new_nodes_edges(self, new, new_edges, update=False):
         if new or update:
             node_key = []
@@ -710,6 +719,7 @@ def add_new_nodes_edges(self, new, new_edges, update=False):
                 self.node_source.stream(node)
                 self.edge_source.stream(edge)
 
+    @without_property_validation
     def patch_updates(self):
         """
         Small updates like color changes or lost nodes from task transitions
@@ -837,6 +847,7 @@ def __init__(self, scheduler, **kwargs):
         )
         self.root.add_tools(hover)
 
+    @without_property_validation
     def update(self):
         with log_errors():
             state = {'all': valmap(len, self.plugin.all),
@@ -909,6 +920,7 @@ def __init__(self, scheduler, **kwargs):
         )
         self.root.add_tools(hover)
 
+    @without_property_validation
     def update(self):
         with log_errors():
             nb = nbytes_bar(self.plugin.nbytes)
@@ -954,7 +966,7 @@ def __init__(self, scheduler, width=800, **kwargs):
                       'num_fds': NumberFormatter(format='0'),
                       'ncores': NumberFormatter(format='0')}
 
-        if LooseVersion(bokeh.__version__) < '0.12.15':
+        if BOKEH_VERSION < '0.12.15':
             dt_kwargs = {'row_headers': False}
         else:
             dt_kwargs = {'index_position': None}
@@ -1033,6 +1045,7 @@ def __init__(self, scheduler, width=800, **kwargs):
 
         self.root = column(*components, id='bk-worker-table', **sizing_mode)
 
+    @without_property_validation
     def update(self):
         data = {name: [] for name in self.names + self.extra_names}
         for addr, ws in sorted(self.scheduler.workers.items()):
diff --git a/distributed/bokeh/utils.py b/distributed/bokeh/utils.py
index 4ca4b464ade..3bfada9402e 100644
--- a/distributed/bokeh/utils.py
+++ b/distributed/bokeh/utils.py
@@ -1,7 +1,23 @@
 from __future__ import print_function, division, absolute_import
 
+from distutils.version import LooseVersion
+
+import bokeh
 from toolz import partition
 
+from ..compatibility import PY2
+
+BOKEH_VERSION = LooseVersion(bokeh.__version__)
+
+
+if BOKEH_VERSION >= '1.0.0' and not PY2:
+    # This decorator is only available in bokeh >= 1.0.0, and doesn't work for
+    # callbacks in Python 2, since the signature introspection won't line up.
+    from bokeh.core.properties import without_property_validation
+else:
+    def without_property_validation(f):
+        return f
+
 
 def parse_args(args):
     options = dict(partition(2, args))
diff --git a/distributed/bokeh/worker.py b/distributed/bokeh/worker.py
index 4d7a924a6c8..898226279f3 100644
--- a/distributed/bokeh/worker.py
+++ b/distributed/bokeh/worker.py
@@ -19,7 +19,7 @@
 from .components import (DashboardComponent, ProfileTimePlot, ProfileServer,
                          add_periodic_callback)
 from .core import BokehServer
-from .utils import transpose
+from .utils import transpose, without_property_validation
 from ..compatibility import WINDOWS
 from ..diagnostics.progress_stream import color_of
 from ..metrics import time
@@ -57,6 +57,7 @@ def __init__(self, worker):
         )
         self.root = table
 
+    @without_property_validation
     def update(self):
         with log_errors():
             w = self.worker
@@ -108,6 +109,7 @@ def __init__(self, worker, height=300, **kwargs):
             self.last_outgoing = 0
             self.who = dict()
 
+    @without_property_validation
     def update(self):
         with log_errors():
             outgoing = self.worker.outgoing_transfer_log
@@ -176,6 +178,7 @@ def __init__(self, worker, **kwargs):
 
         self.root = fig
 
+    @without_property_validation
     def update(self):
         with log_errors():
             self.source.stream({'x': [time() * 1000],
@@ -204,6 +207,7 @@ def __init__(self, worker, **kwargs):
 
         self.root = fig
 
+    @without_property_validation
     def update(self):
         with log_errors():
             self.source.stream({'x': [time() * 1000],
@@ -264,6 +268,7 @@ def __init__(self, worker, **kwargs):
 
             self.root = self.layout
 
+    @without_property_validation
     def update(self):
         with log_errors():
             outgoing = self.worker.outgoing_transfer_log
@@ -323,6 +328,7 @@ def create_figure(self, **kwargs):
             )
             return fig
 
+    @without_property_validation
     def update_figure(self, attr, old, new):
         with log_errors():
             fig = self.create_figure(**self.kwargs)
@@ -418,6 +424,7 @@ def get_data(self):
         self.last = self.worker.monitor.count
         return d
 
+    @without_property_validation
     def update(self):
         with log_errors():
             self.source.stream(self.get_data(), 1000)
@@ -505,6 +512,7 @@ def add_counter_figure(self, name):
             self.counter_figures[name] = fig
             return fig
 
+    @without_property_validation
     def update(self):
         with log_errors():
             for name, fig in self.digest_figures.items():

From bb3934373d75a333c51f0b28183ab718ded0fa4d Mon Sep 17 00:00:00 2001
From: Jim Crist <jcrist@users.noreply.github.com>
Date: Fri, 30 Nov 2018 17:01:31 -0600
Subject: [PATCH 0149/1550] Fully initialize WorkerState objects (#2388)

Previously these weren't fully initialized in the constructor, leading
to potential codepaths accessing missing attributes. We now fully
initialize in the constructor.
---
 distributed/scheduler.py         | 53 ++++++++++++++++++--------------
 distributed/tests/test_client.py |  3 +-
 2 files changed, 32 insertions(+), 24 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index d0ef7606a5c..f6b71a455d4 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -193,9 +193,10 @@ class WorkerState(object):
     # XXX need a state field to signal active/removed?
 
     __slots__ = (
+        'actors',
         'address',
         'has_what',
-        'info',
+        'last_seen',
         'local_directory',
         'memory_limit',
         'metrics',
@@ -206,30 +207,34 @@ class WorkerState(object):
         'pid',
         'processing',
         'resources',
-        'time_delay',
-        'used_resources',
         'services',
         'status',
-        'last_seen',
-        'actors',
+        'time_delay',
+        'used_resources',
     )
 
-    def __init__(self, worker, ncores, memory_limit, name=None):
-        self.address = worker
-        self.has_what = set()
-        self.memory_limit = memory_limit
+    def __init__(self, address=None, pid=0, name=None, ncores=0, memory_limit=0,
+                 local_directory=None, services=None):
+        self.address = address
+        self.pid = pid
         self.name = name
-        self.nbytes = 0
         self.ncores = ncores
+        self.memory_limit = memory_limit
+        self.local_directory = local_directory
+        self.services = services or {}
+
+        self.status = 'running'
+        self.nbytes = 0
         self.occupancy = 0
-        self.pid = 0
-        self.processing = dict()
-        self.resources = {}
-        self.used_resources = {}
+        self.metrics = {}
         self.last_seen = 0
-        self.services = {}
+        self.time_delay = 0
+
         self.actors = set()
-        self.metrics = {}
+        self.has_what = set()
+        self.processing = {}
+        self.resources = {}
+        self.used_resources = {}
 
     @property
     def host(self):
@@ -1277,9 +1282,15 @@ def add_worker(self, comm=None, address=None, keys=(), ncores=None,
             if ws is not None:
                 raise ValueError("Worker already exists %s" % address)
 
-            ws = WorkerState(address, ncores, memory_limit, name)
-            ws.status = 'running'
-            self.workers[address] = ws
+            self.workers[address] = ws = WorkerState(
+                    address=address,
+                    pid=pid,
+                    ncores=ncores,
+                    memory_limit=memory_limit,
+                    name=name,
+                    local_directory=local_directory,
+                    services=services
+            )
 
             if name in self.aliases:
                 msg = {'status': 'error',
@@ -1296,10 +1307,6 @@ def add_worker(self, comm=None, address=None, keys=(), ncores=None,
 
             self.total_ncores += ncores
             self.aliases[name] = address
-            ws.name = name
-            ws.pid = pid
-            ws.services = services
-            ws.local_directory = local_directory
 
             response = self.heartbeat_worker(address=address,
                                              resolve_address=resolve_address,
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 02e2a822132..e15002828b4 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -3714,6 +3714,8 @@ def test_write_scheduler_file(c):
 
 
 def test_get_versions(c):
+    requests = pytest.importorskip('requests')
+
     v = c.get_versions()
     assert v['scheduler'] is not None
     assert v['client'] is not None
@@ -3726,7 +3728,6 @@ def test_get_versions(c):
     # that this does not raise
 
     v = c.get_versions(packages=['requests'])
-    import requests
     assert dict(v['client']['packages']['optional'])['requests'] == requests.__version__
 
 

From 1fd7a2e8a5d38ec369d5f5e36a1884d737b9d2db Mon Sep 17 00:00:00 2001
From: Russ Bubley <russ.bubley@googlemail.com>
Date: Mon, 3 Dec 2018 15:12:18 +0000
Subject: [PATCH 0150/1550] Fix typo in scheduler docstring (#2393)

---
 distributed/scheduler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index f6b71a455d4..f94c532ac5b 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -743,7 +743,7 @@ class Scheduler(ServerNode):
         Information about each worker host
 
     * **clients:** ``{client key: ClientState}``
-        Workers currently connected to the scheduler
+        Clients currently connected to the scheduler
 
     * **services:** ``{str: port}``:
         Other services running on this scheduler, like Bokeh

From f75fe8c45892494b1144d542bff710414f5c09e8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lo=C3=AFc=20Est=C3=A8ve?= <loic.esteve@ymail.com>
Date: Mon, 3 Dec 2018 17:52:09 +0100
Subject: [PATCH 0151/1550] DOC: fix typo in distributed.worker.Worker
 docstring (#2395)

---
 distributed/worker.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/worker.py b/distributed/worker.py
index 4fb41a4aecb..024d39c9d34 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -143,7 +143,7 @@ class Worker(ServerNode):
         Dictionary mapping keys to actual values
     * **task_state**: ``{key: string}``:
         The state of all tasks that the scheduler has asked us to compute.
-        Valid states include waiting, constrained, exeucuting, memory, erred
+        Valid states include waiting, constrained, executing, memory, erred
     * **tasks**: ``{key: dict}``
         The function, args, kwargs of a task.  We run this when appropriate
     * **dependencies**: ``{key: {deps}}``

From 0cea2324497e8e9bd6dd0fdd729bf6d36c08cde3 Mon Sep 17 00:00:00 2001
From: fjetter <fjetter@users.noreply.github.com>
Date: Tue, 4 Dec 2018 16:27:46 +0100
Subject: [PATCH 0152/1550] Remove clients and workers from event log after
 removal (#2394)

Fixes #2371
---
 distributed/distributed.yaml        |  3 +++
 distributed/scheduler.py            | 24 +++++++++++++++++++
 distributed/tests/test_scheduler.py | 37 +++++++++++++++++++++++++++++
 3 files changed, 64 insertions(+)

diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index b75bbd650ae..f6bf5e6fda8 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -12,6 +12,9 @@ distributed:
     allowed-failures: 3     # number of retries before a task is considered bad
     bandwidth: 100000000    # 100 MB/s estimated worker-worker bandwidth
     default-data-size: 1000
+    # Number of seconds to wait until workers or clients are removed from the events log
+    # after they have been removed from the scheduler
+    events-cleanup-delay: 1h
     transition-log-length: 100000
     work-stealing: True     # workers should steal tasks from each other
     worker-ttl: null        # like '60s'. Time to live for workers.  They must heartbeat faster than this
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index f94c532ac5b..2ec02119376 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1767,7 +1767,19 @@ def remove_worker(self, comm=None, address=None, safe=False, close=True):
             if not self.workers:
                 logger.info("Lost all workers")
 
+            @gen.coroutine
+            def remove_worker_from_events():
+                # If the worker isn't registered anymore after the delay, remove from events
+                if address not in self.workers and address in self.events:
+                    del self.events[address]
+
+            cleanup_delay = parse_timedelta(dask.config.get('distributed.scheduler.events-cleanup-delay'))
+            self.loop.call_later(
+                cleanup_delay,
+                remove_worker_from_events
+            )
             logger.debug("Removed worker %s", address)
+
         return 'OK'
 
     def stimulus_cancel(self, comm, keys=None, client=None, force=False):
@@ -2054,6 +2066,18 @@ def remove_client(self, client=None):
                                       client=cs.client_key)
             del self.clients[client]
 
+        @gen.coroutine
+        def remove_client_from_events():
+            # If the client isn't registered anymore after the delay, remove from events
+            if client not in self.clients and client in self.events:
+                del self.events[client]
+
+        cleanup_delay = parse_timedelta(dask.config.get('distributed.scheduler.events-cleanup-delay'))
+        self.loop.call_later(
+            cleanup_delay,
+            remove_client_from_events
+        )
+
     def send_task_to_worker(self, worker, key):
         """ Send a single computational task to a worker """
         try:
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 542e4d91cd2..94f95745ef6 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -194,6 +194,43 @@ def test_remove_worker_from_scheduler(s, a, b):
     s.validate_state()
 
 
+@gen_cluster(config={"distributed.scheduler.events-cleanup-delay": "10 ms"})
+def test_clear_events_worker_removal(s, a, b):
+    assert a.address in s.events
+    assert a.address in s.ncores
+    assert b.address in s.events
+    assert b.address in s.ncores
+
+    s.remove_worker(address=a.address)
+    # Shortly after removal, the events should still be there
+    assert a.address in s.events
+    assert a.address not in s.ncores
+    s.validate_state()
+
+    start = time()
+    while a.address in s.events:
+        yield gen.sleep(0.01)
+        assert time() < start + 2
+    assert b.address in s.events
+
+
+@gen_cluster(config={"distributed.scheduler.events-cleanup-delay": "10 ms"}, client=True)
+def test_clear_events_client_removal(c, s, a, b):
+    assert c.id in s.events
+    s.remove_client(c.id)
+
+    assert c.id in s.events
+    assert c.id not in s.clients
+    assert c not in s.clients
+
+    s.remove_client(c.id)
+    # If it doesn't reconnect after a given time, the events log should be cleared
+    start = time()
+    while c.id in s.events:
+        yield gen.sleep(0.01)
+        assert time() < start + 2
+
+
 @gen_cluster()
 def test_add_worker(s, a, b):
     w = Worker(s.ip, s.port, ncores=3)

From 1a171baf157d2cae19ff899a49f7f94296601e5f Mon Sep 17 00:00:00 2001
From: tjb900 <ozburgess@gmail.com>
Date: Sat, 8 Dec 2018 01:02:03 +0800
Subject: [PATCH 0153/1550] Support msgpack 0.6.0 by providing length keywords
 (#2399)

Fixes #2392
---
 distributed/protocol/core.py                | 15 ++++++++-------
 distributed/protocol/serialize.py           |  8 +++++++-
 distributed/protocol/tests/test_protocol.py | 12 ++++++++++++
 3 files changed, 27 insertions(+), 8 deletions(-)

diff --git a/distributed/protocol/core.py b/distributed/protocol/core.py
index f7df6597752..8d9e1b2b127 100644
--- a/distributed/protocol/core.py
+++ b/distributed/protocol/core.py
@@ -12,7 +12,7 @@
 
 from .compression import compressions, maybe_compress, decompress
 from .serialize import (serialize, deserialize, Serialize, Serialized,
-                        extract_serialize)
+                        extract_serialize, msgpack_len_opts)
 from .utils import frame_split_size, merge_frames
 from ..utils import nbytes
 
@@ -20,11 +20,12 @@
 
 
 try:
-    msgpack.loads(msgpack.dumps(''), raw=False)
-    msgpack_raw_false = {'raw': False}
+    msgpack.loads(msgpack.dumps(''), raw=False, **msgpack_len_opts)
+    msgpack_opts = {'raw': False}
+    msgpack_opts.update(msgpack_len_opts)
 except TypeError:
     # Backward compat with old msgpack (prior to 0.5.2)
-    msgpack_raw_false = {'encoding': 'utf-8'}
+    msgpack_opts = {'encoding': 'utf-8'}
 
 
 logger = logging.getLogger(__name__)
@@ -110,7 +111,7 @@ def loads(frames, deserialize=True, deserializers=None):
             return msg
 
         header = frames.pop()
-        header = msgpack.loads(header, use_list=False, **msgpack_raw_false)
+        header = msgpack.loads(header, use_list=False, **msgpack_opts)
         keys = header['keys']
         headers = header['headers']
         bytestrings = set(header['bytestrings'])
@@ -182,7 +183,7 @@ def loads_msgpack(header, payload):
         dumps_msgpack
     """
     if header:
-        header = msgpack.loads(header, use_list=False, **msgpack_raw_false)
+        header = msgpack.loads(header, use_list=False, **msgpack_opts)
     else:
         header = {}
 
@@ -194,4 +195,4 @@ def loads_msgpack(header, payload):
             raise ValueError("Data is compressed as %s but we don't have this"
                              " installed" % str(header['compression']))
 
-    return msgpack.loads(payload, use_list=False, **msgpack_raw_false)
+    return msgpack.loads(payload, use_list=False, **msgpack_opts)
diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index 434e235f584..417aa43484f 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -57,6 +57,11 @@ def pickle_loads(header, frames):
     return pickle.loads(b''.join(frames))
 
 
+msgpack_len_opts = {
+    ('max_%s_len' % x): 2**31 - 1
+    for x in ['str', 'bin', 'array', 'map', 'ext']}
+
+
 def msgpack_dumps(x):
     try:
         frame = msgpack.dumps(x, use_bin_type=True)
@@ -67,7 +72,8 @@ def msgpack_dumps(x):
 
 
 def msgpack_loads(header, frames):
-    return msgpack.loads(b''.join(frames), encoding='utf8', use_list=False)
+    return msgpack.loads(b''.join(frames), encoding='utf8', use_list=False,
+                         **msgpack_len_opts)
 
 
 def serialization_error_loads(header, frames):
diff --git a/distributed/protocol/tests/test_protocol.py b/distributed/protocol/tests/test_protocol.py
index 509640e47fa..9701cf7768c 100644
--- a/distributed/protocol/tests/test_protocol.py
+++ b/distributed/protocol/tests/test_protocol.py
@@ -137,6 +137,18 @@ def test_large_messages():
     assert (msg['y']['a'].data == msg2['y']['a']).all()
 
 
+def test_large_messages_map():
+    import psutil
+    if psutil.virtual_memory().total < 8e9:
+        pytest.skip("insufficient memory")
+
+    x = {i: 'mystring_%d' % i for i in range(100000)}
+
+    b = dumps(x)
+    x2 = loads(b)
+    assert x == x2
+
+
 def test_loads_deserialize_False():
     frames = dumps({'data': Serialize(123), 'status': 'OK'})
     msg = loads(frames)

From ec48674254a0fec2d8c405638d4105b8340ada16 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sun, 9 Dec 2018 17:26:09 -0500
Subject: [PATCH 0154/1550] Use async-await on large messages test (#2404)

This resolves an intermittent error.  Previously the variable x would
hang around.

After tracking down the reference it seems like the generator structure
held onto a frame that held onto `x` sometimes.  Now we use async-await
in Python 3 and this seems to go away.
---
 distributed/tests/py3_test_client.py | 35 +++++++++++++++++++++++++
 distributed/tests/test_batched.py    | 38 +---------------------------
 2 files changed, 36 insertions(+), 37 deletions(-)

diff --git a/distributed/tests/py3_test_client.py b/distributed/tests/py3_test_client.py
index e94ca691a21..28ee964b495 100644
--- a/distributed/tests/py3_test_client.py
+++ b/distributed/tests/py3_test_client.py
@@ -1,10 +1,14 @@
+import gc
+import sys
 from time import sleep
+import weakref
 
 import pytest
 from tornado import gen
 
 from distributed.utils_test import div, gen_cluster, inc, loop, cluster  # noqa F401
 from distributed import as_completed, Client, Lock
+from distributed.metrics import time
 from distributed.utils import sync
 
 
@@ -123,3 +127,34 @@ async def ff():
         with Client(s['address'], loop=loop) as c:
             assert sync(loop, ff) == 1
             assert c.sync(ff) == 1
+
+
+@gen_cluster(client=True)
+async def test_dont_hold_on_to_large_messages(c, s, a, b):
+    np = pytest.importorskip('numpy')
+    da = pytest.importorskip('dask.array')
+    x = np.random.random(1000000)
+    xr = weakref.ref(x)
+
+    d = da.from_array(x, chunks=(100000,))
+    d = d.persist()
+    del x
+
+    start = time()
+    while xr() is not None:
+        if time() > start + 5:
+            # Help diagnosing
+            from types import FrameType
+            x = xr()
+            if x is not None:
+                del x
+                rc = sys.getrefcount(xr())
+                refs = gc.get_referrers(xr())
+                print("refs to x:", rc, refs, gc.isenabled())
+                frames = [r for r in refs if isinstance(r, FrameType)]
+                for i, f in enumerate(frames):
+                    print("frames #%d:" % i,
+                          f.f_code.co_name, f.f_code.co_filename, sorted(f.f_locals))
+            pytest.fail("array should have been destroyed")
+
+        await gen.sleep(0.200)
diff --git a/distributed/tests/test_batched.py b/distributed/tests/test_batched.py
index 49f9e22633b..386de5957cb 100644
--- a/distributed/tests/test_batched.py
+++ b/distributed/tests/test_batched.py
@@ -1,10 +1,6 @@
-
 from contextlib import contextmanager
 from datetime import timedelta
-import gc
 import random
-import sys
-import weakref
 
 import pytest
 from toolz import assoc
@@ -14,7 +10,7 @@
 from distributed.core import listen, connect, CommClosedError
 from distributed.metrics import time
 from distributed.utils import All
-from distributed.utils_test import gen_test, slow, gen_cluster, captured_logger
+from distributed.utils_test import gen_test, slow, captured_logger
 from distributed.protocol import to_serialize
 
 
@@ -240,38 +236,6 @@ def test_large_traffic_jam():
     yield run_traffic_jam(500, 1500000)
 
 
-@pytest.mark.skipif(sys.version_info[0] < 3, reason="intermittent failure")
-@gen_cluster(client=True)
-def test_dont_hold_on_to_large_messages(c, s, a, b):
-    np = pytest.importorskip('numpy')
-    da = pytest.importorskip('dask.array')
-    x = np.random.random(1000000)
-    xr = weakref.ref(x)
-
-    d = da.from_array(x, chunks=(100000,))
-    d = d.persist()
-    del x
-
-    start = time()
-    while xr() is not None:
-        if time() > start + 5:
-            # Help diagnosing
-            from types import FrameType
-            x = xr()
-            if x is not None:
-                del x
-                rc = sys.getrefcount(xr())
-                refs = gc.get_referrers(xr())
-                print("refs to x:", rc, refs, gc.isenabled())
-                frames = [r for r in refs if isinstance(r, FrameType)]
-                for i, f in enumerate(frames):
-                    print("frames #%d:" % i,
-                          f.f_code.co_name, f.f_code.co_filename, sorted(f.f_locals))
-            pytest.fail("array should have been destroyed")
-
-        yield gen.sleep(0.200)
-
-
 @gen_test()
 def test_serializers():
     with echo_server() as e:

From d0df3d563929061bedad1326c458fc44f6aaa57d Mon Sep 17 00:00:00 2001
From: Jim Crist <jcrist@users.noreply.github.com>
Date: Sun, 9 Dec 2018 16:28:10 -0600
Subject: [PATCH 0155/1550] Fix race condition in normalize_collection (#2386)

This cleans up the use of locks in `Client`, and fixes a race condition
between `Client.normalize_collection` and `Future.release`. We now have
single reentrant-lock on the client, that guards the refcounts on all
futures managed by that client. Any operation that changes this state,
or requires consistency throughout its operation, should acquire this
lock before proceeding. We then guard the critical section of
`normalize_collection` preventing a race condition with
`Future.release`, where previously futures would be decrefed and freed
and then still used in the collection later. A test is added to confirm
the race condition is fixed.
---
 distributed/client.py            | 40 +++++++++++++++++---------------
 distributed/tests/test_client.py | 18 ++++++++++++++
 2 files changed, 39 insertions(+), 19 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 04e0e4ccf73..c26e76d4ec7 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -543,8 +543,11 @@ def __init__(self, address=None, loop=None, timeout=no_default,
         self.cluster = None
         self.scheduler = None
         self._scheduler_identity = {}
-        self._lock = threading.Lock()
-        self._refcount_lock = threading.Lock()
+        # A reentrant-lock on the refcounts for futures associated with this
+        # client. Should be held by individual operations modifying refcounts,
+        # or any bulk operation that needs to ensure the set of futures doesn't
+        # change during operation.
+        self._refcount_lock = threading.RLock()
         self.datasets = Datasets(self)
         self._serializers = serializers
         if deserializers is None:
@@ -1249,7 +1252,7 @@ def submit(self, func, *args, **kwargs):
 
         skey = tokey(key)
 
-        with self._lock:
+        with self._refcount_lock:
             if skey in self.futures:
                 return Future(key, self, inform=False)
 
@@ -2184,7 +2187,7 @@ def _graph_to_futures(self, dsk, keys, restrictions=None,
                           loose_restrictions=None, priority=None,
                           user_priority=0, resources=None, retries=None,
                           fifo_timeout=0, actors=None):
-        with self._lock:
+        with self._refcount_lock:
             if resources:
                 resources = self._expand_resources(resources,
                                                    all_keys=itertools.chain(dsk, keys))
@@ -2334,13 +2337,14 @@ def _optimize_insert_futures(self, dsk, keys):
         This returns the same graph if unchanged but a new graph if any changes
         were necessary.
         """
-        changed = False
-        for key in list(dsk):
-            if tokey(key) in self.futures:
-                if not changed:
-                    changed = True
-                    dsk = dict(dsk)
-                dsk[key] = Future(key, self, inform=False)
+        with self._refcount_lock:
+            changed = False
+            for key in list(dsk):
+                if tokey(key) in self.futures:
+                    if not changed:
+                        changed = True
+                        dsk = dict(dsk)
+                    dsk[key] = Future(key, self, inform=False)
 
         if changed:
             dsk, _ = dask.optimization.cull(dsk, keys)
@@ -2370,15 +2374,13 @@ def normalize_collection(self, collection):
         --------
         Client.persist: trigger computation of collection's tasks
         """
-        with self._lock:
-            dsk = self._optimize_insert_futures(
-                    collection.__dask_graph__(),
-                    collection.__dask_keys__())
+        dsk_orig = collection.__dask_graph__()
+        dsk = self._optimize_insert_futures(dsk_orig, collection.__dask_keys__())
 
-            if dsk is collection.__dask_graph__():
-                return collection
-            else:
-                return redict_collection(collection, dsk)
+        if dsk is dsk_orig:
+            return collection
+        else:
+            return redict_collection(collection, dsk)
 
     def compute(self, collections, sync=False, optimize_graph=True,
                 workers=None, allow_other_workers=False, resources=None,
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index e15002828b4..a53f212f419 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -4119,6 +4119,24 @@ def test_normalize_collection_dask_array(c, s, a, b):
     assert result1 == result2
 
 
+@slow
+def test_normalize_collection_with_released_futures(c):
+    da = pytest.importorskip('dask.array')
+
+    x = da.arange(2**20, chunks=2**10)
+    y = x.persist()
+    wait(y)
+    sol = y.sum().compute()
+    # Start releasing futures
+    del y
+    # Try to reuse futures. Previously this was a race condition,
+    # and the call to `.compute()` would error out due to missing
+    # futures on the scheduler at compute time.
+    normalized = c.normalize_collection(x)
+    res = normalized.sum().compute()
+    assert res == sol
+
+
 @gen_cluster(client=True)
 def test_auto_normalize_collection(c, s, a, b):
     da = pytest.importorskip('dask.array')

From c8584016edda6e0cfeb197ffda568293aa48a5c3 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 12 Dec 2018 20:00:58 -0500
Subject: [PATCH 0156/1550] Fix redict collection after HighLevelGraph fix
 upstream (#2413)

---
 distributed/client.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/client.py b/distributed/client.py
index c26e76d4ec7..ab004139874 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -3905,7 +3905,7 @@ def ensure_default_get(client):
 def redict_collection(c, dsk):
     from dask.delayed import Delayed
     if isinstance(c, Delayed):
-        return Delayed(c.key, [dsk])
+        return Delayed(c.key, dsk)
     else:
         cc = copy.copy(c)
         cc.dask = dsk

From 897c559214081cfe34846e7ecc9dc67f6bf684a3 Mon Sep 17 00:00:00 2001
From: Stephan Hoyer <shoyer@gmail.com>
Date: Thu, 13 Dec 2018 06:02:32 -0800
Subject: [PATCH 0157/1550] Add a blocking argument to Lock.acquire() (#2412)

---
 distributed/lock.py             | 13 +++++++++++--
 distributed/tests/test_locks.py | 12 ++++++++++++
 2 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/distributed/lock.py b/distributed/lock.py
index 905be0baa3f..9f1c4390653 100644
--- a/distributed/lock.py
+++ b/distributed/lock.py
@@ -98,14 +98,17 @@ def __init__(self, name=None, client=None):
         self.id = uuid.uuid4().hex
         self._locked = False
 
-    def acquire(self, timeout=None):
+    def acquire(self, blocking=True, timeout=None):
         """ Acquire the lock
 
         Parameters
         ----------
-        timeout: number
+        blocking : bool, optional
+            If false, don't wait on the lock in the scheduler at all.
+        timeout : number, optional
             Seconds to wait on the lock in the scheduler.  This does not
             include local coroutine time, network transfer time, etc..
+            It is forbidden to specify a timeout when blocking is false.
 
         Examples
         --------
@@ -116,6 +119,12 @@ def acquire(self, timeout=None):
         -------
         True or False whether or not it sucessfully acquired the lock
         """
+        if not blocking:
+            if timeout is not None:
+                raise ValueError(
+                    "can't specify a timeout for a non-blocking call")
+            timeout = 0
+
         result = self.client.sync(self.client.scheduler.lock_acquire,
                                   name=self.name, id=self.id, timeout=timeout)
         self._locked = True
diff --git a/distributed/tests/test_locks.py b/distributed/tests/test_locks.py
index 5799396604e..b35d9d6268f 100644
--- a/distributed/tests/test_locks.py
+++ b/distributed/tests/test_locks.py
@@ -65,6 +65,18 @@ def test_acquires_with_zero_timeout(c, s, a, b):
     yield lock.release()
 
 
+@gen_cluster(client=True)
+def test_acquires_blocking(c, s, a, b):
+    lock = Lock('x')
+    yield lock.acquire(blocking=False)
+    assert lock.locked()
+    yield lock.release()
+    assert not lock.locked()
+
+    with pytest.raises(ValueError):
+        lock.acquire(blocking=False, timeout=1)
+
+
 def test_timeout_sync(client):
     with Lock('x') as lock:
         assert Lock('x').acquire(timeout=0.1) is False

From 2fa8e796e99d75fde7b179776a82a539bc02e563 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sat, 15 Dec 2018 11:13:31 -0500
Subject: [PATCH 0158/1550] fix long traceback test (#2417)

---
 distributed/deploy/tests/test_local.py |  2 +-
 distributed/tests/test_client.py       | 21 ++++++++++-----------
 2 files changed, 11 insertions(+), 12 deletions(-)

diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index 10e4a3514e1..c8b6bbc7665 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -245,7 +245,7 @@ def test_repeated():
 @pytest.mark.parametrize('processes', [True, False])
 def test_bokeh(loop, processes):
     pytest.importorskip('bokeh')
-    import requests
+    requests = pytest.importorskip('requests')
     with LocalCluster(scheduler_port=0, silence_logs=False, loop=loop,
                       processes=processes, diagnostics_port=0) as c:
         bokeh_port = c.scheduler.services['bokeh'].port
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index a53f212f419..2a8f5d502dc 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -45,7 +45,7 @@
                                tmpfile)
 from distributed.utils_test import (cluster, slow, slowinc, slowadd, slowdec,
                                     randominc, inc, dec, div, throws, geninc, asyncinc,
-                                    gen_cluster, gen_test, double, deep, popen,
+                                    gen_cluster, gen_test, double, popen,
                                     captured_logger, varying, map_varying,
                                     wait_for, async_wait_for, pristine_loop)
 from distributed.utils_test import (client as c, client_secondary as c2,# noqa F401
@@ -2467,21 +2467,20 @@ def test_persist(c):
     assert (zz == z).all()
 
 
-@pytest.mark.avoid_travis  # This hangs intermittently.  We don't know why.
 @gen_cluster(timeout=60, client=True)
 def test_long_traceback(c, s, a, b):
     from distributed.protocol.pickle import dumps
 
-    n = sys.getrecursionlimit()
-    sys.setrecursionlimit(500)
+    def deep(n):
+        if n == 0:
+            1 / 0
+        else:
+            return deep(n - 1)
 
-    try:
-        x = c.submit(deep, 1000)
-        yield wait([x])
-        assert len(dumps(c.futures[x.key].traceback)) < 10000
-        assert isinstance(c.futures[x.key].exception, RuntimeError)
-    finally:
-        sys.setrecursionlimit(n)
+    x = c.submit(deep, 200)
+    yield wait([x])
+    assert len(dumps(c.futures[x.key].traceback)) < 10000
+    assert isinstance(c.futures[x.key].exception, ZeroDivisionError)
 
 
 @gen_cluster(client=True)

From f242303fb920b1efa3788dc42b0b3380bfe1e3da Mon Sep 17 00:00:00 2001
From: Diane Trout <diane@caltech.edu>
Date: Sat, 15 Dec 2018 08:13:51 -0800
Subject: [PATCH 0159/1550] Update x509 certificates to current OpenSSL
 standards. (#2418)

OpenSSL updated their TLS security standards to require RSA keys of
2048 bits and to use SHA256 for certificate signatures.

make_tls_certs was updated to meet the requirements and then used to
generate new x509 certificates.
---
 distributed/tests/make_tls_certs.py        |   4 +-
 distributed/tests/tls-ca-cert.pem          | 118 +++++------
 distributed/tests/tls-ca-key.pem           |  52 ++---
 distributed/tests/tls-cert-chain.pem       | 218 +++++++++++----------
 distributed/tests/tls-cert.pem             | 100 +++++-----
 distributed/tests/tls-key-cert.pem         | 140 +++++++------
 distributed/tests/tls-key.pem              |  40 ++--
 distributed/tests/tls-self-signed-cert.pem |  30 +--
 distributed/tests/tls-self-signed-key.pem  |  40 ++--
 9 files changed, 410 insertions(+), 332 deletions(-)

diff --git a/distributed/tests/make_tls_certs.py b/distributed/tests/make_tls_certs.py
index 0392a1783e9..8ffd62e876d 100644
--- a/distributed/tests/make_tls_certs.py
+++ b/distributed/tests/make_tls_certs.py
@@ -33,7 +33,7 @@
     dir = cadir
     database  = $dir/index.txt
     crlnumber = $dir/crl.txt
-    default_md = sha1
+    default_md = sha256
     default_days = 3600
     default_crl_days = 3600
     certificate = tls-ca-cert.pem
@@ -81,7 +81,7 @@ def make_cert_key(hostname, sign=False):
         with open(req_file, 'w') as f:
             f.write(req)
         args = ['req', '-new', '-days', '3650', '-nodes',
-                '-newkey', 'rsa:1024', '-keyout', key_file,
+                '-newkey', 'rsa:2048', '-keyout', key_file,
                 '-config', req_file]
         if sign:
             with tempfile.NamedTemporaryFile(delete=False) as f:
diff --git a/distributed/tests/tls-ca-cert.pem b/distributed/tests/tls-ca-cert.pem
index a6c39bae145..5441bacbc78 100644
--- a/distributed/tests/tls-ca-cert.pem
+++ b/distributed/tests/tls-ca-cert.pem
@@ -2,77 +2,77 @@ Certificate:
     Data:
         Version: 3 (0x2)
         Serial Number:
-            a4:da:fb:14:c7:46:fe:d2
-    Signature Algorithm: sha1WithRSAEncryption
+            3f:d8:8a:ed:26:a6:7e:6f:53:9f:fe:85:a4:a3:62:44:65:9f:ff:46
+        Signature Algorithm: sha256WithRSAEncryption
         Issuer: C=XY, O=Dask CA, CN=our-ca-server
         Validity
-            Not Before: Apr  5 14:36:25 2017 GMT
-            Not After : Apr  3 14:36:25 2027 GMT
+            Not Before: Dec 13 02:52:10 2018 GMT
+            Not After : Dec 10 02:52:10 2028 GMT
         Subject: C=XY, O=Dask CA, CN=our-ca-server
         Subject Public Key Info:
             Public Key Algorithm: rsaEncryption
-                Public-Key: (2048 bit)
+                RSA Public-Key: (2048 bit)
                 Modulus:
-                    00:d2:ab:69:b2:9e:76:de:6e:85:7d:18:6c:8b:d4:
-                    68:fd:4a:96:e2:03:2f:8e:dd:ab:46:53:c4:46:af:
-                    8a:2d:5a:c8:39:7f:7b:ce:39:69:ae:f0:b1:b3:5d:
-                    84:b1:81:b0:94:bd:b6:de:22:38:bd:7a:cc:c9:d6:
-                    ed:e0:ba:9c:a6:59:67:bb:9f:ca:01:c3:11:2c:15:
-                    05:24:9a:a3:31:1e:4f:f5:40:ad:97:54:24:f4:ca:
-                    e6:68:ad:ef:33:fd:d1:91:3f:bc:36:7b:45:99:53:
-                    fd:3c:50:2a:2c:84:38:b7:83:81:bf:72:b4:d0:eb:
-                    e3:43:be:8c:cb:91:92:bc:c6:76:98:7a:28:14:cc:
-                    a1:29:1d:ff:3f:22:5f:8d:21:e4:88:4c:b1:81:93:
-                    4c:d8:ca:1c:55:cf:c9:a1:4d:0f:de:7d:b9:87:38:
-                    0f:d4:a8:2f:ec:35:d4:76:0f:1b:7f:20:9b:c9:2e:
-                    50:4d:89:d0:a5:61:a9:ae:8e:73:48:50:1a:33:1e:
-                    28:55:c4:31:80:95:11:03:27:97:83:8a:49:c0:99:
-                    6d:93:32:80:c1:5d:60:2b:52:0a:ca:ca:f7:54:32:
-                    10:3c:62:8c:73:14:a6:25:21:42:1a:45:a2:42:6a:
-                    c7:fe:c9:04:c3:fe:ef:ee:4a:34:61:3c:c5:80:a9:
-                    48:91
+                    00:d0:49:58:02:6d:4c:db:00:c2:bf:90:d7:95:f6:
+                    87:c0:8d:9c:1e:3b:c4:ed:2c:33:02:2d:be:58:82:
+                    59:2b:7e:4c:91:23:8b:73:2c:97:89:07:2d:ac:93:
+                    5e:39:11:25:75:11:7a:1a:d0:d6:e3:80:e6:e3:5b:
+                    93:db:3d:97:be:bb:96:3f:ff:1d:26:c8:fe:e2:a8:
+                    cb:28:cb:3b:5f:88:05:df:27:81:b1:38:29:d5:77:
+                    6d:f7:e5:9a:19:60:4b:f8:a9:53:00:90:c8:a3:32:
+                    0e:73:0a:3b:51:a9:ab:92:1e:6b:2f:28:82:01:18:
+                    6e:13:fd:69:1a:79:bd:a9:40:cb:70:b7:92:75:1e:
+                    19:0c:b4:3c:c1:7e:55:80:05:be:8d:f0:80:d9:25:
+                    89:74:1a:92:1d:ba:26:a1:65:39:96:50:46:95:e4:
+                    be:29:47:8b:77:0d:11:eb:a1:7a:87:19:14:bf:82:
+                    0a:4d:ca:d4:6d:40:66:4a:5f:12:cd:a3:16:14:19:
+                    a2:8d:7a:07:80:36:04:56:e6:f3:7d:38:c6:75:97:
+                    c9:7c:95:9a:f9:ff:79:6d:c7:40:7a:c2:92:2d:ef:
+                    7a:52:07:de:6b:03:9f:af:17:72:63:53:31:88:57:
+                    76:d5:45:e1:2f:c5:aa:ed:93:d6:46:7e:07:00:cf:
+                    49:eb
                 Exponent: 65537 (0x10001)
         X509v3 extensions:
             X509v3 Subject Key Identifier: 
-                92:E3:52:19:13:1F:38:4E:31:EC:D4:66:CA:3C:E6:74:ED:9D:2E:3B
+                59:66:3D:F8:65:B7:98:7E:C1:E4:31:19:34:E5:A3:0D:2E:3D:F9:38
             X509v3 Authority Key Identifier: 
-                keyid:92:E3:52:19:13:1F:38:4E:31:EC:D4:66:CA:3C:E6:74:ED:9D:2E:3B
+                keyid:59:66:3D:F8:65:B7:98:7E:C1:E4:31:19:34:E5:A3:0D:2E:3D:F9:38
 
             X509v3 Basic Constraints: 
                 CA:TRUE
-    Signature Algorithm: sha1WithRSAEncryption
-         41:dc:da:b5:ba:a6:1b:81:9d:d8:72:9e:c9:14:ab:b6:b1:e2:
-         36:7b:6f:d4:b7:7e:8a:c6:ad:f5:15:45:ac:1f:f9:38:4d:cd:
-         de:05:00:4f:c2:74:ba:fb:4d:19:25:0f:c5:8e:d7:02:e2:e6:
-         a9:69:a2:b4:36:4a:6c:10:49:00:09:0d:9b:3a:5e:4a:14:57:
-         7b:5a:63:bf:83:33:6c:af:88:64:e0:a5:e1:d5:56:97:d1:4d:
-         79:db:8e:b3:a9:b7:a9:aa:41:e8:26:2d:36:05:9c:fc:9d:60:
-         c4:23:a2:0c:92:48:b1:cf:3a:c6:e9:67:9e:71:b2:d6:97:1f:
-         be:fa:e0:ff:72:52:a7:78:50:87:26:0a:28:e6:cc:46:e8:88:
-         3f:a0:aa:86:0e:b8:20:b1:b8:fd:45:0c:94:4c:67:e2:05:99:
-         25:65:92:21:16:85:df:91:1a:0c:02:36:3b:7a:f1:3d:08:29:
-         5a:47:83:45:0b:2e:ff:ce:cb:97:64:8f:fa:d8:99:1d:eb:c1:
-         20:2e:ca:61:11:e9:08:f9:c7:ce:9f:14:a5:c6:55:e5:ed:07:
-         4a:eb:2f:0a:18:90:fb:5d:23:b4:19:f7:29:1f:66:6e:32:e4:
-         61:1c:7f:78:73:1d:56:24:ef:1a:ab:4c:85:72:a8:0a:63:ef:
-         67:f7:03:6f
+    Signature Algorithm: sha256WithRSAEncryption
+         a2:bd:c7:e4:23:24:4e:da:db:d3:bd:7f:33:87:5f:1f:3b:e8:
+         fa:19:54:17:eb:60:86:ac:d6:18:47:35:90:88:47:d6:f1:5a:
+         31:ed:e1:44:b4:16:6a:04:c3:6f:9b:ba:cf:99:14:0a:c8:32:
+         0a:69:fd:41:c3:67:25:3d:f7:84:31:81:8b:2e:b0:9c:04:74:
+         2c:0b:87:b2:a8:48:a2:c7:42:bb:90:31:82:a5:51:38:03:df:
+         e7:aa:76:f7:1a:ae:b9:69:5c:94:62:7f:19:3e:5c:6b:97:99:
+         9e:4a:9e:4b:34:f5:39:15:fe:12:72:88:7b:1f:be:db:3a:3b:
+         7f:14:06:7b:ef:b2:fc:c4:ad:82:51:4a:95:6c:23:a5:bb:65:
+         38:0e:55:70:e1:96:88:bc:29:82:38:48:63:ac:5a:73:20:3c:
+         d7:39:25:e9:35:46:9b:6b:b1:a0:77:56:a0:a6:30:0f:7e:0e:
+         12:10:11:41:28:6a:f4:ca:b5:c3:a9:fd:fe:9f:cf:29:5e:26:
+         d7:a8:4c:28:a5:31:10:c2:07:25:c2:0a:5d:35:73:f7:db:3c:
+         6b:5d:2f:1a:f1:b0:e7:81:26:22:fe:38:63:2f:3e:5b:a7:d5:
+         b5:a8:ef:fe:87:b4:6f:2d:5e:7e:48:6c:c0:b9:ca:7f:ab:28:
+         c3:31:ae:93
 -----BEGIN CERTIFICATE-----
-MIIDQTCCAimgAwIBAgIJAKTa+xTHRv7SMA0GCSqGSIb3DQEBBQUAMDcxCzAJBgNV
-BAYTAlhZMRAwDgYDVQQKDAdEYXNrIENBMRYwFAYDVQQDDA1vdXItY2Etc2VydmVy
-MB4XDTE3MDQwNTE0MzYyNVoXDTI3MDQwMzE0MzYyNVowNzELMAkGA1UEBhMCWFkx
-EDAOBgNVBAoMB0Rhc2sgQ0ExFjAUBgNVBAMMDW91ci1jYS1zZXJ2ZXIwggEiMA0G
-CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDSq2mynnbeboV9GGyL1Gj9SpbiAy+O
-3atGU8RGr4otWsg5f3vOOWmu8LGzXYSxgbCUvbbeIji9eszJ1u3gupymWWe7n8oB
-wxEsFQUkmqMxHk/1QK2XVCT0yuZore8z/dGRP7w2e0WZU/08UCoshDi3g4G/crTQ
-6+NDvozLkZK8xnaYeigUzKEpHf8/Il+NIeSITLGBk0zYyhxVz8mhTQ/efbmHOA/U
-qC/sNdR2Dxt/IJvJLlBNidClYamujnNIUBozHihVxDGAlREDJ5eDiknAmW2TMoDB
-XWArUgrKyvdUMhA8YoxzFKYlIUIaRaJCasf+yQTD/u/uSjRhPMWAqUiRAgMBAAGj
-UDBOMB0GA1UdDgQWBBSS41IZEx84TjHs1GbKPOZ07Z0uOzAfBgNVHSMEGDAWgBSS
-41IZEx84TjHs1GbKPOZ07Z0uOzAMBgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBBQUA
-A4IBAQBB3Nq1uqYbgZ3Ycp7JFKu2seI2e2/Ut36Kxq31FUWsH/k4Tc3eBQBPwnS6
-+00ZJQ/FjtcC4uapaaK0NkpsEEkACQ2bOl5KFFd7WmO/gzNsr4hk4KXh1VaX0U15
-246zqbepqkHoJi02BZz8nWDEI6IMkkixzzrG6WeecbLWlx+++uD/clKneFCHJgoo
-5sxG6Ig/oKqGDrggsbj9RQyUTGfiBZklZZIhFoXfkRoMAjY7evE9CClaR4NFCy7/
-zsuXZI/62Jkd68EgLsphEekI+cfOnxSlxlXl7QdK6y8KGJD7XSO0GfcpH2ZuMuRh
-HH94cx1WJO8aq0yFcqgKY+9n9wNv
+MIIDTDCCAjSgAwIBAgIUP9iK7Samfm9Tn/6FpKNiRGWf/0YwDQYJKoZIhvcNAQEL
+BQAwNzELMAkGA1UEBhMCWFkxEDAOBgNVBAoMB0Rhc2sgQ0ExFjAUBgNVBAMMDW91
+ci1jYS1zZXJ2ZXIwHhcNMTgxMjEzMDI1MjEwWhcNMjgxMjEwMDI1MjEwWjA3MQsw
+CQYDVQQGEwJYWTEQMA4GA1UECgwHRGFzayBDQTEWMBQGA1UEAwwNb3VyLWNhLXNl
+cnZlcjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANBJWAJtTNsAwr+Q
+15X2h8CNnB47xO0sMwItvliCWSt+TJEji3Msl4kHLayTXjkRJXURehrQ1uOA5uNb
+k9s9l767lj//HSbI/uKoyyjLO1+IBd8ngbE4KdV3bfflmhlgS/ipUwCQyKMyDnMK
+O1Gpq5Ieay8oggEYbhP9aRp5valAy3C3knUeGQy0PMF+VYAFvo3wgNkliXQakh26
+JqFlOZZQRpXkvilHi3cNEeuheocZFL+CCk3K1G1AZkpfEs2jFhQZoo16B4A2BFbm
+8304xnWXyXyVmvn/eW3HQHrCki3velIH3msDn68XcmNTMYhXdtVF4S/Fqu2T1kZ+
+BwDPSesCAwEAAaNQME4wHQYDVR0OBBYEFFlmPfhlt5h+weQxGTTlow0uPfk4MB8G
+A1UdIwQYMBaAFFlmPfhlt5h+weQxGTTlow0uPfk4MAwGA1UdEwQFMAMBAf8wDQYJ
+KoZIhvcNAQELBQADggEBAKK9x+QjJE7a29O9fzOHXx876PoZVBfrYIas1hhHNZCI
+R9bxWjHt4US0FmoEw2+bus+ZFArIMgpp/UHDZyU994QxgYsusJwEdCwLh7KoSKLH
+QruQMYKlUTgD3+eqdvcarrlpXJRifxk+XGuXmZ5Knks09TkV/hJyiHsfvts6O38U
+BnvvsvzErYJRSpVsI6W7ZTgOVXDhloi8KYI4SGOsWnMgPNc5Jek1RptrsaB3VqCm
+MA9+DhIQEUEoavTKtcOp/f6fzyleJteoTCilMRDCByXCCl01c/fbPGtdLxrxsOeB
+JiL+OGMvPlun1bWo7/6HtG8tXn5IbMC5yn+rKMMxrpM=
 -----END CERTIFICATE-----
diff --git a/distributed/tests/tls-ca-key.pem b/distributed/tests/tls-ca-key.pem
index 9471db65c5b..a5807f29c14 100644
--- a/distributed/tests/tls-ca-key.pem
+++ b/distributed/tests/tls-ca-key.pem
@@ -1,28 +1,28 @@
 -----BEGIN PRIVATE KEY-----
-MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDSq2mynnbeboV9
-GGyL1Gj9SpbiAy+O3atGU8RGr4otWsg5f3vOOWmu8LGzXYSxgbCUvbbeIji9eszJ
-1u3gupymWWe7n8oBwxEsFQUkmqMxHk/1QK2XVCT0yuZore8z/dGRP7w2e0WZU/08
-UCoshDi3g4G/crTQ6+NDvozLkZK8xnaYeigUzKEpHf8/Il+NIeSITLGBk0zYyhxV
-z8mhTQ/efbmHOA/UqC/sNdR2Dxt/IJvJLlBNidClYamujnNIUBozHihVxDGAlRED
-J5eDiknAmW2TMoDBXWArUgrKyvdUMhA8YoxzFKYlIUIaRaJCasf+yQTD/u/uSjRh
-PMWAqUiRAgMBAAECggEBAKWJred+HzePLvNqQrjpcoo9lywHhtJDGreLUPqeKvyS
-jiyoIPeeI44kce3YOVFl8AQgwtXvnBg/qzWa3pas7DTGof1LhOc3iYHTDpSBsmQB
-RqEXbpAvvU5hNQXiTsjS1F7G7zT8Tmc6GQ3QjCMozoxdGJIAJ1bXVGBCFc5MGDqe
-ZlVRImZKLopvc8N7brvqjLZ2WWg4aeXUCtV2AsBQzlU7SzjLUKfKi+QtwKIUX//K
-Iv5Qtx0yR+ZgxQ8ONlz89PCUsDKfMG+2opINyII91t5Mk9FUJBU+xgtarwIzAy5q
-uG/v16y3nz0slyKMe92RYzQJQGc67Prz65EU5P+m0VUCgYEA+ek2oZDYmkbXoR9L
-hv205lp6pWu2b+QWgDwRMzJV9bUCuapDfbsqpXWgSojzPbYPWL8jI4gIPtADixTk
-Ql/UI+r6T0ypkcAbqpwL5ivsvwTL84vy8K3I55B06y6fsgKQX9+PAophN8PFNDwQ
-r/nG361DEbOF6ZoEmm5S/lZxM9MCgYEA181vuHLcBuaUdrJ5ARQW+9FP0sFPIniu
-aqUaFLHFGp6DcXa0rx2gGhs50l3o07fOj1RVi9xAkPXSMGXb5Ufk6JvBwL/4ppuJ
-wjoHJOWVypUhNyB6JqBRvJ1ir9gNMsTyusOUMW81i+9U9ri4ntuAiKlFntSfj2UL
-imJYMS1fJ4sCgYAY5pXc71PFu31MvpOF9c2KjkVeKidZar6bNGD6Wu+Ztdl5qm9j
-Hb0YE2pPQuoJohRRYWoc5jb8GznfJ8u+mVHH6mfacucKcLvQ+g9tdb73IYE42R7x
-wD8eY5RiwpiC4iKEiv/y3trk9Yq2s61t+1NHgjoGj36uUfxXbtqyIObSpQKBgBwZ
-BoPYiKdpXEk8IZ8vU3LEJQWG39vYJLWD+6pYHxbBLtIWJ55pWBAuHVAzo/30/7MP
-wver3MhDCXP9vyzSOI7EwOxs0l43AC/GyGlYbzFqS2HXG7BtzVamk1Pb6JQEqqoA
-jwoEtc8bgU+Ca59baxfgGk18162e9Wq2NU9p3erTAoGBAMXtJsVWvrT0pY+nmMFo
-oV2cdDaJ1U/BBNtc2+cmxyHURIgoGgJygs/ioV6Lwcj+1zQHzkNeIVcEhX3hH4MX
-f1ibf0NNTMLuCRydEHBS5dTRdqePe1WJoTQFBKiuW2pFfZh13MAcYu6CbqG+VwFB
-sjuaJuu2e8mRKoaHWsjLopUs
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDQSVgCbUzbAMK/
+kNeV9ofAjZweO8TtLDMCLb5YglkrfkyRI4tzLJeJBy2sk145ESV1EXoa0NbjgObj
+W5PbPZe+u5Y//x0myP7iqMsoyztfiAXfJ4GxOCnVd2335ZoZYEv4qVMAkMijMg5z
+CjtRqauSHmsvKIIBGG4T/Wkaeb2pQMtwt5J1HhkMtDzBflWABb6N8IDZJYl0GpId
+uiahZTmWUEaV5L4pR4t3DRHroXqHGRS/ggpNytRtQGZKXxLNoxYUGaKNegeANgRW
+5vN9OMZ1l8l8lZr5/3ltx0B6wpIt73pSB95rA5+vF3JjUzGIV3bVReEvxartk9ZG
+fgcAz0nrAgMBAAECggEBAIve1PU8RQJEroNKVxvjpCIQdqhu5r2GkvKBmXOChTMJ
+640D25+KcHCam3dqtzmHZCoLhH4BhGzeoS5Izy/JWt/AxRNQ1Fi5XUEErPlGEScE
+Bz1mIFfMKXuqafSxARzNZ/cibhTRJYtCECX81vzt/cNwHfZbM4Jcz38pC62110GS
+hSQOAhZv5TFskzlNrmNEVF0vo48v5CS17H2V6qTqQ1bWfLIl8KgMVdvExxrqeIcf
+0wq6QcDZ2vuuTnz8fBMl8q87SoB5oBSqQvkRi67Wv4+8XqmJH06IN4i54IJAG3MG
+ROcvjma+v0zJJsskngE0FtB51axo2gg478A/qpehkjECgYEA8LyC4XEJHL1FykRP
+uzu9HnH50clAUBTGFVGmavY7yytWiEO2bUIES77fZjMv2sWh38emSo3efjCp+s8I
+WYhfrgN4hGLHeHd58tU4wzuQC9kj+8vhuwXVLP7ppVNjDQtuwKV3cHb4Nz5kQ5Iw
+2SOsWGZ5GK3rSvP/Rha4esqi768CgYEA3X4gKDldUIfeMmw88uMLfgxrwuGSuL8Z
+Me/iICTqQjz9DNVS+xfW3qWnYK9NXW8qqu5PqiOyPTEk5aPUgFZXat+a4jsxgK0p
+5I7J4k7uP+5F0CVAIERY+/r8T9kWFAB8yyOz0lm9Z1N8K2Id8+P8wKloJpN5nVwc
+v2mhKgbwfIUCgYAW2u7aFG83CUks8NtY/UzxLQT4w9CVScOclonkMchve4fVnAgf
+y1q5DkALZeSvMoAxzLSe82UiOeCJbUpvpaDf1niEqRQi/cTYJyIidu8wEMd3yl19
+8gnRIcpw5b/Lp0A2jf8O8FyUchn2yFRW6SodomzwyOJD1XJw5wtQrRhXBQKBgDOq
+MugPEXQybSMH2mOxkEGztgM8SNQtRzw9v1pOIhKXVVOEAxos4mMG8vb7jSlJQAOl
+GRIPx4i8iOXEBFJ3UWbNh7xToGVqfwYgRHk0fiCNfN1jgwTDzJniXq5lbd/2iSV9
+S3m6kxKiB6m69k+ok5GBMpkk9048aTxA8GX0Pm5dAoGBAN+yHAq72nXT9ZrjI5gA
+Fhqn7lnZJddLr31/2y6vO4h4a3Uf/pUFqsu5ku/BVwglwJxHx3AeUqlWrHliLqkC
+VQC5hUkaG7jtKumLgZKXuLoAFWsqu2pImNLTYOiEcf3gUXkopa3kbIxfHgspLYXV
+XrYnjJSpTav4WIESxs1SqICB
 -----END PRIVATE KEY-----
diff --git a/distributed/tests/tls-cert-chain.pem b/distributed/tests/tls-cert-chain.pem
index d3418c6fd6f..d967b0c25ed 100644
--- a/distributed/tests/tls-cert-chain.pem
+++ b/distributed/tests/tls-cert-chain.pem
@@ -2,134 +2,146 @@ Certificate:
     Data:
         Version: 1 (0x0)
         Serial Number:
-            a4:da:fb:14:c7:46:fe:d3
-    Signature Algorithm: sha1WithRSAEncryption
+            3f:d8:8a:ed:26:a6:7e:6f:53:9f:fe:85:a4:a3:62:44:65:9f:ff:47
+        Signature Algorithm: sha256WithRSAEncryption
         Issuer: C=XY, O=Dask CA, CN=our-ca-server
         Validity
-            Not Before: Apr  5 14:36:25 2017 GMT
-            Not After : Feb 12 14:36:25 2027 GMT
+            Not Before: Dec 13 02:52:10 2018 GMT
+            Not After : Oct 21 02:52:10 2028 GMT
         Subject: C=XY, L=Dask-distributed, O=Dask, CN=localhost
         Subject Public Key Info:
             Public Key Algorithm: rsaEncryption
-                Public-Key: (1024 bit)
+                RSA Public-Key: (2048 bit)
                 Modulus:
-                    00:d0:de:a3:22:83:df:a6:f8:73:f1:7c:f7:2b:33:
-                    5a:d4:0d:c8:c4:e6:c3:b3:12:13:fd:b0:9c:b2:eb:
-                    77:f7:d4:c9:f2:01:52:85:e5:3c:f0:5b:fd:8d:21:
-                    f9:92:6c:35:9c:c4:c9:75:32:85:bd:5b:46:10:e1:
-                    8b:ad:1b:00:93:f1:d8:df:11:af:08:fc:c7:ba:61:
-                    fa:e2:ce:85:22:ed:d1:5c:f1:20:82:5f:8d:fe:9e:
-                    9d:8b:fc:8f:3c:3e:e2:e3:ce:d6:c0:05:ce:14:58:
-                    32:f7:c7:a0:53:3b:5b:63:7e:5f:59:da:14:fd:b4:
-                    86:17:cd:3a:11:1b:bf:36:95
+                    00:e5:73:ee:b0:34:96:b9:f2:12:3e:15:8a:e9:76:
+                    c7:11:29:e0:23:78:84:03:ec:eb:a8:1e:eb:b6:02:
+                    bf:34:95:43:29:1e:fb:27:41:fc:7d:55:f8:74:ea:
+                    dc:e2:da:88:d1:8d:4e:ee:78:bf:ca:4c:da:5a:01:
+                    24:e7:4e:95:c6:b2:8d:55:e0:b1:6d:70:a4:c7:0e:
+                    a5:35:77:91:75:06:42:74:6a:a4:2c:7f:8f:d4:0a:
+                    f0:9e:af:b1:97:1d:0e:37:4e:fa:e8:ba:c1:a9:59:
+                    38:f5:5d:67:d1:1f:61:37:e5:ca:72:89:6f:3b:61:
+                    49:45:7f:fa:e7:b4:17:76:8e:4a:77:ef:6a:2e:6b:
+                    b4:ba:24:27:7c:45:d7:af:d4:20:53:e2:ec:da:4f:
+                    bd:c2:e0:fc:a2:61:8a:d8:96:f0:68:67:41:21:7e:
+                    79:25:17:7b:96:bb:f8:59:c0:9c:e5:e5:03:58:26:
+                    2d:e4:b3:61:27:94:38:b8:55:ff:65:fb:22:10:e8:
+                    63:56:2b:1e:26:63:cc:7d:43:16:1e:1b:49:56:38:
+                    1b:79:f7:03:e8:b2:f5:02:fa:ed:15:06:d3:fc:af:
+                    84:d8:34:a6:df:27:15:08:59:83:c7:83:34:65:ca:
+                    c6:d5:8c:86:9a:f0:30:83:20:24:41:10:18:6c:20:
+                    06:5d
                 Exponent: 65537 (0x10001)
-    Signature Algorithm: sha1WithRSAEncryption
-         15:c0:ca:1b:ec:f1:3b:ba:08:65:fb:6a:f6:60:6f:29:15:48:
-         f6:4c:ba:f0:56:56:5e:2a:6c:3c:68:94:d6:6b:eb:23:d7:ec:
-         09:2f:72:2b:21:91:fa:4a:e8:b6:f3:79:28:db:00:8e:66:8d:
-         2d:63:69:fa:ac:e5:3f:e0:71:d5:ae:ea:d6:f2:26:7e:a0:e6:
-         04:db:f0:98:03:d1:0a:67:fd:c5:53:8a:6d:46:7a:7a:8e:48:
-         57:42:76:0d:11:f2:d0:34:ee:e4:a5:ca:25:22:09:d5:74:f9:
-         30:6a:03:30:5b:2a:ec:d0:2e:09:ef:8e:f1:f8:af:1b:3f:bf:
-         29:08:89:cf:a2:58:f1:4f:01:60:82:59:9e:6c:33:e1:9b:4a:
-         cd:ec:c1:e7:73:c9:eb:68:34:e0:d3:08:f8:7c:41:08:d7:1f:
-         b1:00:01:d4:e0:0b:5b:52:39:be:1c:d0:75:d2:05:55:7d:14:
-         82:4c:8c:16:9f:30:ff:03:c1:ac:09:53:19:76:71:e5:f1:d9:
-         07:4b:db:d3:99:71:a8:31:7c:1f:99:27:3f:1c:6f:c9:02:e9:
-         0c:a8:d3:62:fd:a8:b9:57:28:c4:8f:f8:33:9c:c1:f1:a9:47:
-         fb:bb:c6:60:31:b3:06:1e:ab:89:eb:e4:23:32:09:17:a3:5e:
-         04:8e:88:c4
+    Signature Algorithm: sha256WithRSAEncryption
+         6f:78:94:3b:66:82:b5:6f:c5:59:91:fe:54:91:da:5d:04:28:
+         37:63:53:d3:76:16:21:b4:4f:18:57:c6:ee:11:ef:c1:5f:d1:
+         9a:a4:a0:15:60:33:94:52:2f:be:3d:60:9c:9f:51:79:aa:36:
+         28:02:cf:0d:60:a9:e9:54:b8:a5:d4:49:d1:10:e6:f4:a6:a2:
+         1d:27:55:c7:28:63:9e:48:99:ec:d2:88:d7:da:34:03:b6:d9:
+         32:a0:4f:0f:6d:b5:a3:36:ac:54:cc:60:fd:6a:e8:86:c8:57:
+         6e:61:de:66:b3:44:ab:18:95:1c:7f:2d:6e:53:a3:66:17:6e:
+         8c:c8:0c:e1:3c:da:82:d6:c6:28:f8:4c:fa:7f:ca:71:0a:9d:
+         de:7e:19:40:17:e9:40:2f:b3:18:aa:e8:31:bc:0f:5e:61:b5:
+         50:e6:14:3b:e2:7c:dd:70:b3:dc:b9:d0:18:af:87:e4:db:29:
+         1b:f1:a1:6d:2a:44:e5:bf:d7:8d:18:77:dc:f5:25:6a:b3:8a:
+         19:00:3e:15:2b:28:ed:40:c1:45:ad:66:db:74:a0:53:e8:0c:
+         2f:fb:ae:d6:35:9c:3f:c8:81:f9:13:dd:49:06:77:26:82:7d:
+         6d:66:a1:87:a9:9d:ca:4b:2f:c5:19:91:a6:3c:4a:f2:b7:bf:
+         af:8c:97:90
 -----BEGIN CERTIFICATE-----
-MIICejCCAWICCQCk2vsUx0b+0zANBgkqhkiG9w0BAQUFADA3MQswCQYDVQQGEwJY
-WTEQMA4GA1UECgwHRGFzayBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0x
-NzA0MDUxNDM2MjVaFw0yNzAyMTIxNDM2MjVaMEsxCzAJBgNVBAYTAlhZMRkwFwYD
-VQQHDBBEYXNrLWRpc3RyaWJ1dGVkMQ0wCwYDVQQKDAREYXNrMRIwEAYDVQQDDAls
-b2NhbGhvc3QwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANDeoyKD36b4c/F8
-9yszWtQNyMTmw7MSE/2wnLLrd/fUyfIBUoXlPPBb/Y0h+ZJsNZzEyXUyhb1bRhDh
-i60bAJPx2N8Rrwj8x7ph+uLOhSLt0VzxIIJfjf6enYv8jzw+4uPO1sAFzhRYMvfH
-oFM7W2N+X1naFP20hhfNOhEbvzaVAgMBAAEwDQYJKoZIhvcNAQEFBQADggEBABXA
-yhvs8Tu6CGX7avZgbykVSPZMuvBWVl4qbDxolNZr6yPX7AkvcishkfpK6LbzeSjb
-AI5mjS1jafqs5T/gcdWu6tbyJn6g5gTb8JgD0Qpn/cVTim1GenqOSFdCdg0R8tA0
-7uSlyiUiCdV0+TBqAzBbKuzQLgnvjvH4rxs/vykIic+iWPFPAWCCWZ5sM+GbSs3s
-wedzyetoNODTCPh8QQjXH7EAAdTgC1tSOb4c0HXSBVV9FIJMjBafMP8DwawJUxl2
-ceXx2QdL29OZcagxfB+ZJz8cb8kC6Qyo02L9qLlXKMSP+DOcwfGpR/u7xmAxswYe
-q4nr5CMyCRejXgSOiMQ=
+MIIDCTCCAfECFD/Yiu0mpn5vU5/+haSjYkRln/9HMA0GCSqGSIb3DQEBCwUAMDcx
+CzAJBgNVBAYTAlhZMRAwDgYDVQQKDAdEYXNrIENBMRYwFAYDVQQDDA1vdXItY2Et
+c2VydmVyMB4XDTE4MTIxMzAyNTIxMFoXDTI4MTAyMTAyNTIxMFowSzELMAkGA1UE
+BhMCWFkxGTAXBgNVBAcMEERhc2stZGlzdHJpYnV0ZWQxDTALBgNVBAoMBERhc2sx
+EjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC
+ggEBAOVz7rA0lrnyEj4Viul2xxEp4CN4hAPs66ge67YCvzSVQyke+ydB/H1V+HTq
+3OLaiNGNTu54v8pM2loBJOdOlcayjVXgsW1wpMcOpTV3kXUGQnRqpCx/j9QK8J6v
+sZcdDjdO+ui6walZOPVdZ9EfYTflynKJbzthSUV/+ue0F3aOSnfvai5rtLokJ3xF
+16/UIFPi7NpPvcLg/KJhitiW8GhnQSF+eSUXe5a7+FnAnOXlA1gmLeSzYSeUOLhV
+/2X7IhDoY1YrHiZjzH1DFh4bSVY4G3n3A+iy9QL67RUG0/yvhNg0pt8nFQhZg8eD
+NGXKxtWMhprwMIMgJEEQGGwgBl0CAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAb3iU
+O2aCtW/FWZH+VJHaXQQoN2NT03YWIbRPGFfG7hHvwV/RmqSgFWAzlFIvvj1gnJ9R
+eao2KALPDWCp6VS4pdRJ0RDm9KaiHSdVxyhjnkiZ7NKI19o0A7bZMqBPD221ozas
+VMxg/WrohshXbmHeZrNEqxiVHH8tblOjZhdujMgM4TzagtbGKPhM+n/KcQqd3n4Z
+QBfpQC+zGKroMbwPXmG1UOYUO+J83XCz3LnQGK+H5NspG/GhbSpE5b/XjRh33PUl
+arOKGQA+FSso7UDBRa1m23SgU+gML/uu1jWcP8iB+RPdSQZ3JoJ9bWahh6mdyksv
+xRmRpjxK8re/r4yXkA==
 -----END CERTIFICATE-----
 Certificate:
     Data:
         Version: 3 (0x2)
         Serial Number:
-            a4:da:fb:14:c7:46:fe:d2
-    Signature Algorithm: sha1WithRSAEncryption
+            3f:d8:8a:ed:26:a6:7e:6f:53:9f:fe:85:a4:a3:62:44:65:9f:ff:46
+        Signature Algorithm: sha256WithRSAEncryption
         Issuer: C=XY, O=Dask CA, CN=our-ca-server
         Validity
-            Not Before: Apr  5 14:36:25 2017 GMT
-            Not After : Apr  3 14:36:25 2027 GMT
+            Not Before: Dec 13 02:52:10 2018 GMT
+            Not After : Dec 10 02:52:10 2028 GMT
         Subject: C=XY, O=Dask CA, CN=our-ca-server
         Subject Public Key Info:
             Public Key Algorithm: rsaEncryption
-                Public-Key: (2048 bit)
+                RSA Public-Key: (2048 bit)
                 Modulus:
-                    00:d2:ab:69:b2:9e:76:de:6e:85:7d:18:6c:8b:d4:
-                    68:fd:4a:96:e2:03:2f:8e:dd:ab:46:53:c4:46:af:
-                    8a:2d:5a:c8:39:7f:7b:ce:39:69:ae:f0:b1:b3:5d:
-                    84:b1:81:b0:94:bd:b6:de:22:38:bd:7a:cc:c9:d6:
-                    ed:e0:ba:9c:a6:59:67:bb:9f:ca:01:c3:11:2c:15:
-                    05:24:9a:a3:31:1e:4f:f5:40:ad:97:54:24:f4:ca:
-                    e6:68:ad:ef:33:fd:d1:91:3f:bc:36:7b:45:99:53:
-                    fd:3c:50:2a:2c:84:38:b7:83:81:bf:72:b4:d0:eb:
-                    e3:43:be:8c:cb:91:92:bc:c6:76:98:7a:28:14:cc:
-                    a1:29:1d:ff:3f:22:5f:8d:21:e4:88:4c:b1:81:93:
-                    4c:d8:ca:1c:55:cf:c9:a1:4d:0f:de:7d:b9:87:38:
-                    0f:d4:a8:2f:ec:35:d4:76:0f:1b:7f:20:9b:c9:2e:
-                    50:4d:89:d0:a5:61:a9:ae:8e:73:48:50:1a:33:1e:
-                    28:55:c4:31:80:95:11:03:27:97:83:8a:49:c0:99:
-                    6d:93:32:80:c1:5d:60:2b:52:0a:ca:ca:f7:54:32:
-                    10:3c:62:8c:73:14:a6:25:21:42:1a:45:a2:42:6a:
-                    c7:fe:c9:04:c3:fe:ef:ee:4a:34:61:3c:c5:80:a9:
-                    48:91
+                    00:d0:49:58:02:6d:4c:db:00:c2:bf:90:d7:95:f6:
+                    87:c0:8d:9c:1e:3b:c4:ed:2c:33:02:2d:be:58:82:
+                    59:2b:7e:4c:91:23:8b:73:2c:97:89:07:2d:ac:93:
+                    5e:39:11:25:75:11:7a:1a:d0:d6:e3:80:e6:e3:5b:
+                    93:db:3d:97:be:bb:96:3f:ff:1d:26:c8:fe:e2:a8:
+                    cb:28:cb:3b:5f:88:05:df:27:81:b1:38:29:d5:77:
+                    6d:f7:e5:9a:19:60:4b:f8:a9:53:00:90:c8:a3:32:
+                    0e:73:0a:3b:51:a9:ab:92:1e:6b:2f:28:82:01:18:
+                    6e:13:fd:69:1a:79:bd:a9:40:cb:70:b7:92:75:1e:
+                    19:0c:b4:3c:c1:7e:55:80:05:be:8d:f0:80:d9:25:
+                    89:74:1a:92:1d:ba:26:a1:65:39:96:50:46:95:e4:
+                    be:29:47:8b:77:0d:11:eb:a1:7a:87:19:14:bf:82:
+                    0a:4d:ca:d4:6d:40:66:4a:5f:12:cd:a3:16:14:19:
+                    a2:8d:7a:07:80:36:04:56:e6:f3:7d:38:c6:75:97:
+                    c9:7c:95:9a:f9:ff:79:6d:c7:40:7a:c2:92:2d:ef:
+                    7a:52:07:de:6b:03:9f:af:17:72:63:53:31:88:57:
+                    76:d5:45:e1:2f:c5:aa:ed:93:d6:46:7e:07:00:cf:
+                    49:eb
                 Exponent: 65537 (0x10001)
         X509v3 extensions:
             X509v3 Subject Key Identifier: 
-                92:E3:52:19:13:1F:38:4E:31:EC:D4:66:CA:3C:E6:74:ED:9D:2E:3B
+                59:66:3D:F8:65:B7:98:7E:C1:E4:31:19:34:E5:A3:0D:2E:3D:F9:38
             X509v3 Authority Key Identifier: 
-                keyid:92:E3:52:19:13:1F:38:4E:31:EC:D4:66:CA:3C:E6:74:ED:9D:2E:3B
+                keyid:59:66:3D:F8:65:B7:98:7E:C1:E4:31:19:34:E5:A3:0D:2E:3D:F9:38
 
             X509v3 Basic Constraints: 
                 CA:TRUE
-    Signature Algorithm: sha1WithRSAEncryption
-         41:dc:da:b5:ba:a6:1b:81:9d:d8:72:9e:c9:14:ab:b6:b1:e2:
-         36:7b:6f:d4:b7:7e:8a:c6:ad:f5:15:45:ac:1f:f9:38:4d:cd:
-         de:05:00:4f:c2:74:ba:fb:4d:19:25:0f:c5:8e:d7:02:e2:e6:
-         a9:69:a2:b4:36:4a:6c:10:49:00:09:0d:9b:3a:5e:4a:14:57:
-         7b:5a:63:bf:83:33:6c:af:88:64:e0:a5:e1:d5:56:97:d1:4d:
-         79:db:8e:b3:a9:b7:a9:aa:41:e8:26:2d:36:05:9c:fc:9d:60:
-         c4:23:a2:0c:92:48:b1:cf:3a:c6:e9:67:9e:71:b2:d6:97:1f:
-         be:fa:e0:ff:72:52:a7:78:50:87:26:0a:28:e6:cc:46:e8:88:
-         3f:a0:aa:86:0e:b8:20:b1:b8:fd:45:0c:94:4c:67:e2:05:99:
-         25:65:92:21:16:85:df:91:1a:0c:02:36:3b:7a:f1:3d:08:29:
-         5a:47:83:45:0b:2e:ff:ce:cb:97:64:8f:fa:d8:99:1d:eb:c1:
-         20:2e:ca:61:11:e9:08:f9:c7:ce:9f:14:a5:c6:55:e5:ed:07:
-         4a:eb:2f:0a:18:90:fb:5d:23:b4:19:f7:29:1f:66:6e:32:e4:
-         61:1c:7f:78:73:1d:56:24:ef:1a:ab:4c:85:72:a8:0a:63:ef:
-         67:f7:03:6f
+    Signature Algorithm: sha256WithRSAEncryption
+         a2:bd:c7:e4:23:24:4e:da:db:d3:bd:7f:33:87:5f:1f:3b:e8:
+         fa:19:54:17:eb:60:86:ac:d6:18:47:35:90:88:47:d6:f1:5a:
+         31:ed:e1:44:b4:16:6a:04:c3:6f:9b:ba:cf:99:14:0a:c8:32:
+         0a:69:fd:41:c3:67:25:3d:f7:84:31:81:8b:2e:b0:9c:04:74:
+         2c:0b:87:b2:a8:48:a2:c7:42:bb:90:31:82:a5:51:38:03:df:
+         e7:aa:76:f7:1a:ae:b9:69:5c:94:62:7f:19:3e:5c:6b:97:99:
+         9e:4a:9e:4b:34:f5:39:15:fe:12:72:88:7b:1f:be:db:3a:3b:
+         7f:14:06:7b:ef:b2:fc:c4:ad:82:51:4a:95:6c:23:a5:bb:65:
+         38:0e:55:70:e1:96:88:bc:29:82:38:48:63:ac:5a:73:20:3c:
+         d7:39:25:e9:35:46:9b:6b:b1:a0:77:56:a0:a6:30:0f:7e:0e:
+         12:10:11:41:28:6a:f4:ca:b5:c3:a9:fd:fe:9f:cf:29:5e:26:
+         d7:a8:4c:28:a5:31:10:c2:07:25:c2:0a:5d:35:73:f7:db:3c:
+         6b:5d:2f:1a:f1:b0:e7:81:26:22:fe:38:63:2f:3e:5b:a7:d5:
+         b5:a8:ef:fe:87:b4:6f:2d:5e:7e:48:6c:c0:b9:ca:7f:ab:28:
+         c3:31:ae:93
 -----BEGIN CERTIFICATE-----
-MIIDQTCCAimgAwIBAgIJAKTa+xTHRv7SMA0GCSqGSIb3DQEBBQUAMDcxCzAJBgNV
-BAYTAlhZMRAwDgYDVQQKDAdEYXNrIENBMRYwFAYDVQQDDA1vdXItY2Etc2VydmVy
-MB4XDTE3MDQwNTE0MzYyNVoXDTI3MDQwMzE0MzYyNVowNzELMAkGA1UEBhMCWFkx
-EDAOBgNVBAoMB0Rhc2sgQ0ExFjAUBgNVBAMMDW91ci1jYS1zZXJ2ZXIwggEiMA0G
-CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDSq2mynnbeboV9GGyL1Gj9SpbiAy+O
-3atGU8RGr4otWsg5f3vOOWmu8LGzXYSxgbCUvbbeIji9eszJ1u3gupymWWe7n8oB
-wxEsFQUkmqMxHk/1QK2XVCT0yuZore8z/dGRP7w2e0WZU/08UCoshDi3g4G/crTQ
-6+NDvozLkZK8xnaYeigUzKEpHf8/Il+NIeSITLGBk0zYyhxVz8mhTQ/efbmHOA/U
-qC/sNdR2Dxt/IJvJLlBNidClYamujnNIUBozHihVxDGAlREDJ5eDiknAmW2TMoDB
-XWArUgrKyvdUMhA8YoxzFKYlIUIaRaJCasf+yQTD/u/uSjRhPMWAqUiRAgMBAAGj
-UDBOMB0GA1UdDgQWBBSS41IZEx84TjHs1GbKPOZ07Z0uOzAfBgNVHSMEGDAWgBSS
-41IZEx84TjHs1GbKPOZ07Z0uOzAMBgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBBQUA
-A4IBAQBB3Nq1uqYbgZ3Ycp7JFKu2seI2e2/Ut36Kxq31FUWsH/k4Tc3eBQBPwnS6
-+00ZJQ/FjtcC4uapaaK0NkpsEEkACQ2bOl5KFFd7WmO/gzNsr4hk4KXh1VaX0U15
-246zqbepqkHoJi02BZz8nWDEI6IMkkixzzrG6WeecbLWlx+++uD/clKneFCHJgoo
-5sxG6Ig/oKqGDrggsbj9RQyUTGfiBZklZZIhFoXfkRoMAjY7evE9CClaR4NFCy7/
-zsuXZI/62Jkd68EgLsphEekI+cfOnxSlxlXl7QdK6y8KGJD7XSO0GfcpH2ZuMuRh
-HH94cx1WJO8aq0yFcqgKY+9n9wNv
+MIIDTDCCAjSgAwIBAgIUP9iK7Samfm9Tn/6FpKNiRGWf/0YwDQYJKoZIhvcNAQEL
+BQAwNzELMAkGA1UEBhMCWFkxEDAOBgNVBAoMB0Rhc2sgQ0ExFjAUBgNVBAMMDW91
+ci1jYS1zZXJ2ZXIwHhcNMTgxMjEzMDI1MjEwWhcNMjgxMjEwMDI1MjEwWjA3MQsw
+CQYDVQQGEwJYWTEQMA4GA1UECgwHRGFzayBDQTEWMBQGA1UEAwwNb3VyLWNhLXNl
+cnZlcjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANBJWAJtTNsAwr+Q
+15X2h8CNnB47xO0sMwItvliCWSt+TJEji3Msl4kHLayTXjkRJXURehrQ1uOA5uNb
+k9s9l767lj//HSbI/uKoyyjLO1+IBd8ngbE4KdV3bfflmhlgS/ipUwCQyKMyDnMK
+O1Gpq5Ieay8oggEYbhP9aRp5valAy3C3knUeGQy0PMF+VYAFvo3wgNkliXQakh26
+JqFlOZZQRpXkvilHi3cNEeuheocZFL+CCk3K1G1AZkpfEs2jFhQZoo16B4A2BFbm
+8304xnWXyXyVmvn/eW3HQHrCki3velIH3msDn68XcmNTMYhXdtVF4S/Fqu2T1kZ+
+BwDPSesCAwEAAaNQME4wHQYDVR0OBBYEFFlmPfhlt5h+weQxGTTlow0uPfk4MB8G
+A1UdIwQYMBaAFFlmPfhlt5h+weQxGTTlow0uPfk4MAwGA1UdEwQFMAMBAf8wDQYJ
+KoZIhvcNAQELBQADggEBAKK9x+QjJE7a29O9fzOHXx876PoZVBfrYIas1hhHNZCI
+R9bxWjHt4US0FmoEw2+bus+ZFArIMgpp/UHDZyU994QxgYsusJwEdCwLh7KoSKLH
+QruQMYKlUTgD3+eqdvcarrlpXJRifxk+XGuXmZ5Knks09TkV/hJyiHsfvts6O38U
+BnvvsvzErYJRSpVsI6W7ZTgOVXDhloi8KYI4SGOsWnMgPNc5Jek1RptrsaB3VqCm
+MA9+DhIQEUEoavTKtcOp/f6fzyleJteoTCilMRDCByXCCl01c/fbPGtdLxrxsOeB
+JiL+OGMvPlun1bWo7/6HtG8tXn5IbMC5yn+rKMMxrpM=
 -----END CERTIFICATE-----
diff --git a/distributed/tests/tls-cert.pem b/distributed/tests/tls-cert.pem
index a5dd70d46d0..eb510c57061 100644
--- a/distributed/tests/tls-cert.pem
+++ b/distributed/tests/tls-cert.pem
@@ -2,56 +2,68 @@ Certificate:
     Data:
         Version: 1 (0x0)
         Serial Number:
-            a4:da:fb:14:c7:46:fe:d3
-    Signature Algorithm: sha1WithRSAEncryption
+            3f:d8:8a:ed:26:a6:7e:6f:53:9f:fe:85:a4:a3:62:44:65:9f:ff:47
+        Signature Algorithm: sha256WithRSAEncryption
         Issuer: C=XY, O=Dask CA, CN=our-ca-server
         Validity
-            Not Before: Apr  5 14:36:25 2017 GMT
-            Not After : Feb 12 14:36:25 2027 GMT
+            Not Before: Dec 13 02:52:10 2018 GMT
+            Not After : Oct 21 02:52:10 2028 GMT
         Subject: C=XY, L=Dask-distributed, O=Dask, CN=localhost
         Subject Public Key Info:
             Public Key Algorithm: rsaEncryption
-                Public-Key: (1024 bit)
+                RSA Public-Key: (2048 bit)
                 Modulus:
-                    00:d0:de:a3:22:83:df:a6:f8:73:f1:7c:f7:2b:33:
-                    5a:d4:0d:c8:c4:e6:c3:b3:12:13:fd:b0:9c:b2:eb:
-                    77:f7:d4:c9:f2:01:52:85:e5:3c:f0:5b:fd:8d:21:
-                    f9:92:6c:35:9c:c4:c9:75:32:85:bd:5b:46:10:e1:
-                    8b:ad:1b:00:93:f1:d8:df:11:af:08:fc:c7:ba:61:
-                    fa:e2:ce:85:22:ed:d1:5c:f1:20:82:5f:8d:fe:9e:
-                    9d:8b:fc:8f:3c:3e:e2:e3:ce:d6:c0:05:ce:14:58:
-                    32:f7:c7:a0:53:3b:5b:63:7e:5f:59:da:14:fd:b4:
-                    86:17:cd:3a:11:1b:bf:36:95
+                    00:e5:73:ee:b0:34:96:b9:f2:12:3e:15:8a:e9:76:
+                    c7:11:29:e0:23:78:84:03:ec:eb:a8:1e:eb:b6:02:
+                    bf:34:95:43:29:1e:fb:27:41:fc:7d:55:f8:74:ea:
+                    dc:e2:da:88:d1:8d:4e:ee:78:bf:ca:4c:da:5a:01:
+                    24:e7:4e:95:c6:b2:8d:55:e0:b1:6d:70:a4:c7:0e:
+                    a5:35:77:91:75:06:42:74:6a:a4:2c:7f:8f:d4:0a:
+                    f0:9e:af:b1:97:1d:0e:37:4e:fa:e8:ba:c1:a9:59:
+                    38:f5:5d:67:d1:1f:61:37:e5:ca:72:89:6f:3b:61:
+                    49:45:7f:fa:e7:b4:17:76:8e:4a:77:ef:6a:2e:6b:
+                    b4:ba:24:27:7c:45:d7:af:d4:20:53:e2:ec:da:4f:
+                    bd:c2:e0:fc:a2:61:8a:d8:96:f0:68:67:41:21:7e:
+                    79:25:17:7b:96:bb:f8:59:c0:9c:e5:e5:03:58:26:
+                    2d:e4:b3:61:27:94:38:b8:55:ff:65:fb:22:10:e8:
+                    63:56:2b:1e:26:63:cc:7d:43:16:1e:1b:49:56:38:
+                    1b:79:f7:03:e8:b2:f5:02:fa:ed:15:06:d3:fc:af:
+                    84:d8:34:a6:df:27:15:08:59:83:c7:83:34:65:ca:
+                    c6:d5:8c:86:9a:f0:30:83:20:24:41:10:18:6c:20:
+                    06:5d
                 Exponent: 65537 (0x10001)
-    Signature Algorithm: sha1WithRSAEncryption
-         15:c0:ca:1b:ec:f1:3b:ba:08:65:fb:6a:f6:60:6f:29:15:48:
-         f6:4c:ba:f0:56:56:5e:2a:6c:3c:68:94:d6:6b:eb:23:d7:ec:
-         09:2f:72:2b:21:91:fa:4a:e8:b6:f3:79:28:db:00:8e:66:8d:
-         2d:63:69:fa:ac:e5:3f:e0:71:d5:ae:ea:d6:f2:26:7e:a0:e6:
-         04:db:f0:98:03:d1:0a:67:fd:c5:53:8a:6d:46:7a:7a:8e:48:
-         57:42:76:0d:11:f2:d0:34:ee:e4:a5:ca:25:22:09:d5:74:f9:
-         30:6a:03:30:5b:2a:ec:d0:2e:09:ef:8e:f1:f8:af:1b:3f:bf:
-         29:08:89:cf:a2:58:f1:4f:01:60:82:59:9e:6c:33:e1:9b:4a:
-         cd:ec:c1:e7:73:c9:eb:68:34:e0:d3:08:f8:7c:41:08:d7:1f:
-         b1:00:01:d4:e0:0b:5b:52:39:be:1c:d0:75:d2:05:55:7d:14:
-         82:4c:8c:16:9f:30:ff:03:c1:ac:09:53:19:76:71:e5:f1:d9:
-         07:4b:db:d3:99:71:a8:31:7c:1f:99:27:3f:1c:6f:c9:02:e9:
-         0c:a8:d3:62:fd:a8:b9:57:28:c4:8f:f8:33:9c:c1:f1:a9:47:
-         fb:bb:c6:60:31:b3:06:1e:ab:89:eb:e4:23:32:09:17:a3:5e:
-         04:8e:88:c4
+    Signature Algorithm: sha256WithRSAEncryption
+         6f:78:94:3b:66:82:b5:6f:c5:59:91:fe:54:91:da:5d:04:28:
+         37:63:53:d3:76:16:21:b4:4f:18:57:c6:ee:11:ef:c1:5f:d1:
+         9a:a4:a0:15:60:33:94:52:2f:be:3d:60:9c:9f:51:79:aa:36:
+         28:02:cf:0d:60:a9:e9:54:b8:a5:d4:49:d1:10:e6:f4:a6:a2:
+         1d:27:55:c7:28:63:9e:48:99:ec:d2:88:d7:da:34:03:b6:d9:
+         32:a0:4f:0f:6d:b5:a3:36:ac:54:cc:60:fd:6a:e8:86:c8:57:
+         6e:61:de:66:b3:44:ab:18:95:1c:7f:2d:6e:53:a3:66:17:6e:
+         8c:c8:0c:e1:3c:da:82:d6:c6:28:f8:4c:fa:7f:ca:71:0a:9d:
+         de:7e:19:40:17:e9:40:2f:b3:18:aa:e8:31:bc:0f:5e:61:b5:
+         50:e6:14:3b:e2:7c:dd:70:b3:dc:b9:d0:18:af:87:e4:db:29:
+         1b:f1:a1:6d:2a:44:e5:bf:d7:8d:18:77:dc:f5:25:6a:b3:8a:
+         19:00:3e:15:2b:28:ed:40:c1:45:ad:66:db:74:a0:53:e8:0c:
+         2f:fb:ae:d6:35:9c:3f:c8:81:f9:13:dd:49:06:77:26:82:7d:
+         6d:66:a1:87:a9:9d:ca:4b:2f:c5:19:91:a6:3c:4a:f2:b7:bf:
+         af:8c:97:90
 -----BEGIN CERTIFICATE-----
-MIICejCCAWICCQCk2vsUx0b+0zANBgkqhkiG9w0BAQUFADA3MQswCQYDVQQGEwJY
-WTEQMA4GA1UECgwHRGFzayBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0x
-NzA0MDUxNDM2MjVaFw0yNzAyMTIxNDM2MjVaMEsxCzAJBgNVBAYTAlhZMRkwFwYD
-VQQHDBBEYXNrLWRpc3RyaWJ1dGVkMQ0wCwYDVQQKDAREYXNrMRIwEAYDVQQDDAls
-b2NhbGhvc3QwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANDeoyKD36b4c/F8
-9yszWtQNyMTmw7MSE/2wnLLrd/fUyfIBUoXlPPBb/Y0h+ZJsNZzEyXUyhb1bRhDh
-i60bAJPx2N8Rrwj8x7ph+uLOhSLt0VzxIIJfjf6enYv8jzw+4uPO1sAFzhRYMvfH
-oFM7W2N+X1naFP20hhfNOhEbvzaVAgMBAAEwDQYJKoZIhvcNAQEFBQADggEBABXA
-yhvs8Tu6CGX7avZgbykVSPZMuvBWVl4qbDxolNZr6yPX7AkvcishkfpK6LbzeSjb
-AI5mjS1jafqs5T/gcdWu6tbyJn6g5gTb8JgD0Qpn/cVTim1GenqOSFdCdg0R8tA0
-7uSlyiUiCdV0+TBqAzBbKuzQLgnvjvH4rxs/vykIic+iWPFPAWCCWZ5sM+GbSs3s
-wedzyetoNODTCPh8QQjXH7EAAdTgC1tSOb4c0HXSBVV9FIJMjBafMP8DwawJUxl2
-ceXx2QdL29OZcagxfB+ZJz8cb8kC6Qyo02L9qLlXKMSP+DOcwfGpR/u7xmAxswYe
-q4nr5CMyCRejXgSOiMQ=
+MIIDCTCCAfECFD/Yiu0mpn5vU5/+haSjYkRln/9HMA0GCSqGSIb3DQEBCwUAMDcx
+CzAJBgNVBAYTAlhZMRAwDgYDVQQKDAdEYXNrIENBMRYwFAYDVQQDDA1vdXItY2Et
+c2VydmVyMB4XDTE4MTIxMzAyNTIxMFoXDTI4MTAyMTAyNTIxMFowSzELMAkGA1UE
+BhMCWFkxGTAXBgNVBAcMEERhc2stZGlzdHJpYnV0ZWQxDTALBgNVBAoMBERhc2sx
+EjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC
+ggEBAOVz7rA0lrnyEj4Viul2xxEp4CN4hAPs66ge67YCvzSVQyke+ydB/H1V+HTq
+3OLaiNGNTu54v8pM2loBJOdOlcayjVXgsW1wpMcOpTV3kXUGQnRqpCx/j9QK8J6v
+sZcdDjdO+ui6walZOPVdZ9EfYTflynKJbzthSUV/+ue0F3aOSnfvai5rtLokJ3xF
+16/UIFPi7NpPvcLg/KJhitiW8GhnQSF+eSUXe5a7+FnAnOXlA1gmLeSzYSeUOLhV
+/2X7IhDoY1YrHiZjzH1DFh4bSVY4G3n3A+iy9QL67RUG0/yvhNg0pt8nFQhZg8eD
+NGXKxtWMhprwMIMgJEEQGGwgBl0CAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAb3iU
+O2aCtW/FWZH+VJHaXQQoN2NT03YWIbRPGFfG7hHvwV/RmqSgFWAzlFIvvj1gnJ9R
+eao2KALPDWCp6VS4pdRJ0RDm9KaiHSdVxyhjnkiZ7NKI19o0A7bZMqBPD221ozas
+VMxg/WrohshXbmHeZrNEqxiVHH8tblOjZhdujMgM4TzagtbGKPhM+n/KcQqd3n4Z
+QBfpQC+zGKroMbwPXmG1UOYUO+J83XCz3LnQGK+H5NspG/GhbSpE5b/XjRh33PUl
+arOKGQA+FSso7UDBRa1m23SgU+gML/uu1jWcP8iB+RPdSQZ3JoJ9bWahh6mdyksv
+xRmRpjxK8re/r4yXkA==
 -----END CERTIFICATE-----
diff --git a/distributed/tests/tls-key-cert.pem b/distributed/tests/tls-key-cert.pem
index d835f07b9f3..3b52d8d8b40 100644
--- a/distributed/tests/tls-key-cert.pem
+++ b/distributed/tests/tls-key-cert.pem
@@ -1,73 +1,97 @@
 -----BEGIN PRIVATE KEY-----
-MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBANDeoyKD36b4c/F8
-9yszWtQNyMTmw7MSE/2wnLLrd/fUyfIBUoXlPPBb/Y0h+ZJsNZzEyXUyhb1bRhDh
-i60bAJPx2N8Rrwj8x7ph+uLOhSLt0VzxIIJfjf6enYv8jzw+4uPO1sAFzhRYMvfH
-oFM7W2N+X1naFP20hhfNOhEbvzaVAgMBAAECgYAxFJ/w9FwRuEBGdr79/8tQGQ0c
-d710zsKz0lSAFqTSL8Zkf7sMi5mOQY+LIniq15Khi/xq9bxHirMkNFvGCZpQvM6m
-DuKAHf/0zFQOfM3QtXxEb64uupS8UTsiTkiq84Zwb3g6uWYXB+3zvY+FDbi5SkXp
-P1c6c58wu1WSSqBh4QJBAOijVjTxu1FtvrVN/e4J+lfuXKYs7cO9tbC7L2cAhB0A
-6lAD2w1NRbIoY9IXwFc0E+vbyNqp18wYWuUNi+dYTEcCQQDl2ENZ3vc4LvtWWkMm
-fmQ95ks7WTWCnxkWdFKWLEnRAOUM18dJnJgoJxvySaCifrMhOObz+UgJx/a23Sqj
-IsBDAkBRXshJbn/xhGxfKC94cj8BQJtheHeVrqZ9cYusAG81YduTLzYQWgXVwfDg
-THR3S/zn05FWgm+7sI0ibvGPcsADAkEAjr/dPLDk/Aj54qzWrc6r9WHVcbu7JzAm
-xbJjJI/HHL6gg3R0enx+Z9m6WqVt1d445LZRgk9ddW2zBemgtIc9vwJBAJkuovHW
-wq/hmDkBuPqbe2KX5id4sQZqCr7V+fAs12lxjEIwk/fe2Oop0E1dsLGXQaCzqKmp
-GXaAzcrQXi7JtZc=
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDlc+6wNJa58hI+
+FYrpdscRKeAjeIQD7OuoHuu2Ar80lUMpHvsnQfx9Vfh06tzi2ojRjU7ueL/KTNpa
+ASTnTpXGso1V4LFtcKTHDqU1d5F1BkJ0aqQsf4/UCvCer7GXHQ43TvrousGpWTj1
+XWfRH2E35cpyiW87YUlFf/rntBd2jkp372oua7S6JCd8Rdev1CBT4uzaT73C4Pyi
+YYrYlvBoZ0EhfnklF3uWu/hZwJzl5QNYJi3ks2EnlDi4Vf9l+yIQ6GNWKx4mY8x9
+QxYeG0lWOBt59wPosvUC+u0VBtP8r4TYNKbfJxUIWYPHgzRlysbVjIaa8DCDICRB
+EBhsIAZdAgMBAAECggEBAJlC8bjSt6bmUVcUzp+Yzejh/qvSsONqWhIG7Uy60vqN
+hLmvzvRu9/bD+NwnkHkanW7TI1jYYMsAwIyD3SF/dT2uQAePuROvvPCVELcA8bCo
+CLV4EccDQY/R6t0j38xfBQLeh2vyaq7yQMBrro+ZlTdgDvSoZhrUpbt9e5tOyKlP
+p8EwVW74251XSfONrbsbEvTq5GsNSgerAJT9pC05FhpXp92WPYrDFV0906hlt6z2
+vyLw/PVaQR3YLeNcZ7Fp/AEOTht3M4vfaMq1ibEOi5DueBV0RSGcFzKaP1FFnqsD
+a5x3NGpY5q8ngCcwlWWEsBLnVzDIjI7FJWOnX5F87GUCgYEA/dWuKo09GD6g+oeC
+R6LsSxZdqLl5jopUFgKMpdFWs07rZD1HoH+8Dpl0bYSvSBnXza7d5BmE5HwpA3ts
+Y9OinVu9UCgqToO3PZbbyqQ9AhqLW7YlVjZ2G+U8BM6Z1j0+C3aZ8jRXRRXQj9V1
+pUDuDjCK7WEp3Z3fyuqFpMLy+msCgYEA52kB539F7hRkxoob0gMidC3+qiMDjC6t
+uwbiovyWnO38t+ggQR27AZHfVHqW+iXSt2oiTjZ8pNhrBzpWcLZLvOVXzlTeJL+B
+YL2I/rjnRlplBDAfoUyLRCfQQ9FpaBvvoxooySNeRAIZ+9GgeiqTRYz91/CZX5h/
+OIgNV6Y5xFcCgYAo6ny8ly+17MJ+H8G8UmLMZ7qJK/yQIXQUaCzLvca/+ISxD7+v
++2cQ42OFE9I9SQ0kNn/F+OcW/XyO9yItpTudKeCGPzMynMC4yMx9CaRb5nN4QRAb
+qYO4U2ctTzA0/II3ztJZ3F4E4NNRnBLupFy7JNAAG51n/77HMq6ANHx+KQKBgQDi
+d4CWlpbQVYL+TiDF2a15N54X0SZipTf+ZdGcokefksSVEoWmVDs0YRbH7SUd77AV
+MbgbBPXcubOcojoAGldqqKPS2HmCSyyJiVOtvTURQ9vCSnpZ97f/XltSU3mtmlFe
+NHafyY4yTkAxkkhT5qC0GzQ9hUgEJxCyOWudUJ8OBwKBgGd8n3UchJ+Ll3calcHO
+GKlWdqNPI6yLtY0wTqZGCZIWGjskm6YoYj4kEyBten/ERBArSuUeucj5qCxij97Y
+ue27ZP2xDX7ONdd5D36UPck3mfqBiW+BlUI9vGPhBDlA9sRhT2wIkDuu8W9MkyKd
+7Y9YAmBiswDTAj5TYYeWqWcM
 -----END PRIVATE KEY-----
 Certificate:
     Data:
         Version: 1 (0x0)
         Serial Number:
-            a4:da:fb:14:c7:46:fe:d3
-    Signature Algorithm: sha1WithRSAEncryption
+            3f:d8:8a:ed:26:a6:7e:6f:53:9f:fe:85:a4:a3:62:44:65:9f:ff:47
+        Signature Algorithm: sha256WithRSAEncryption
         Issuer: C=XY, O=Dask CA, CN=our-ca-server
         Validity
-            Not Before: Apr  5 14:36:25 2017 GMT
-            Not After : Feb 12 14:36:25 2027 GMT
+            Not Before: Dec 13 02:52:10 2018 GMT
+            Not After : Oct 21 02:52:10 2028 GMT
         Subject: C=XY, L=Dask-distributed, O=Dask, CN=localhost
         Subject Public Key Info:
             Public Key Algorithm: rsaEncryption
-                Public-Key: (1024 bit)
+                RSA Public-Key: (2048 bit)
                 Modulus:
-                    00:d0:de:a3:22:83:df:a6:f8:73:f1:7c:f7:2b:33:
-                    5a:d4:0d:c8:c4:e6:c3:b3:12:13:fd:b0:9c:b2:eb:
-                    77:f7:d4:c9:f2:01:52:85:e5:3c:f0:5b:fd:8d:21:
-                    f9:92:6c:35:9c:c4:c9:75:32:85:bd:5b:46:10:e1:
-                    8b:ad:1b:00:93:f1:d8:df:11:af:08:fc:c7:ba:61:
-                    fa:e2:ce:85:22:ed:d1:5c:f1:20:82:5f:8d:fe:9e:
-                    9d:8b:fc:8f:3c:3e:e2:e3:ce:d6:c0:05:ce:14:58:
-                    32:f7:c7:a0:53:3b:5b:63:7e:5f:59:da:14:fd:b4:
-                    86:17:cd:3a:11:1b:bf:36:95
+                    00:e5:73:ee:b0:34:96:b9:f2:12:3e:15:8a:e9:76:
+                    c7:11:29:e0:23:78:84:03:ec:eb:a8:1e:eb:b6:02:
+                    bf:34:95:43:29:1e:fb:27:41:fc:7d:55:f8:74:ea:
+                    dc:e2:da:88:d1:8d:4e:ee:78:bf:ca:4c:da:5a:01:
+                    24:e7:4e:95:c6:b2:8d:55:e0:b1:6d:70:a4:c7:0e:
+                    a5:35:77:91:75:06:42:74:6a:a4:2c:7f:8f:d4:0a:
+                    f0:9e:af:b1:97:1d:0e:37:4e:fa:e8:ba:c1:a9:59:
+                    38:f5:5d:67:d1:1f:61:37:e5:ca:72:89:6f:3b:61:
+                    49:45:7f:fa:e7:b4:17:76:8e:4a:77:ef:6a:2e:6b:
+                    b4:ba:24:27:7c:45:d7:af:d4:20:53:e2:ec:da:4f:
+                    bd:c2:e0:fc:a2:61:8a:d8:96:f0:68:67:41:21:7e:
+                    79:25:17:7b:96:bb:f8:59:c0:9c:e5:e5:03:58:26:
+                    2d:e4:b3:61:27:94:38:b8:55:ff:65:fb:22:10:e8:
+                    63:56:2b:1e:26:63:cc:7d:43:16:1e:1b:49:56:38:
+                    1b:79:f7:03:e8:b2:f5:02:fa:ed:15:06:d3:fc:af:
+                    84:d8:34:a6:df:27:15:08:59:83:c7:83:34:65:ca:
+                    c6:d5:8c:86:9a:f0:30:83:20:24:41:10:18:6c:20:
+                    06:5d
                 Exponent: 65537 (0x10001)
-    Signature Algorithm: sha1WithRSAEncryption
-         15:c0:ca:1b:ec:f1:3b:ba:08:65:fb:6a:f6:60:6f:29:15:48:
-         f6:4c:ba:f0:56:56:5e:2a:6c:3c:68:94:d6:6b:eb:23:d7:ec:
-         09:2f:72:2b:21:91:fa:4a:e8:b6:f3:79:28:db:00:8e:66:8d:
-         2d:63:69:fa:ac:e5:3f:e0:71:d5:ae:ea:d6:f2:26:7e:a0:e6:
-         04:db:f0:98:03:d1:0a:67:fd:c5:53:8a:6d:46:7a:7a:8e:48:
-         57:42:76:0d:11:f2:d0:34:ee:e4:a5:ca:25:22:09:d5:74:f9:
-         30:6a:03:30:5b:2a:ec:d0:2e:09:ef:8e:f1:f8:af:1b:3f:bf:
-         29:08:89:cf:a2:58:f1:4f:01:60:82:59:9e:6c:33:e1:9b:4a:
-         cd:ec:c1:e7:73:c9:eb:68:34:e0:d3:08:f8:7c:41:08:d7:1f:
-         b1:00:01:d4:e0:0b:5b:52:39:be:1c:d0:75:d2:05:55:7d:14:
-         82:4c:8c:16:9f:30:ff:03:c1:ac:09:53:19:76:71:e5:f1:d9:
-         07:4b:db:d3:99:71:a8:31:7c:1f:99:27:3f:1c:6f:c9:02:e9:
-         0c:a8:d3:62:fd:a8:b9:57:28:c4:8f:f8:33:9c:c1:f1:a9:47:
-         fb:bb:c6:60:31:b3:06:1e:ab:89:eb:e4:23:32:09:17:a3:5e:
-         04:8e:88:c4
+    Signature Algorithm: sha256WithRSAEncryption
+         6f:78:94:3b:66:82:b5:6f:c5:59:91:fe:54:91:da:5d:04:28:
+         37:63:53:d3:76:16:21:b4:4f:18:57:c6:ee:11:ef:c1:5f:d1:
+         9a:a4:a0:15:60:33:94:52:2f:be:3d:60:9c:9f:51:79:aa:36:
+         28:02:cf:0d:60:a9:e9:54:b8:a5:d4:49:d1:10:e6:f4:a6:a2:
+         1d:27:55:c7:28:63:9e:48:99:ec:d2:88:d7:da:34:03:b6:d9:
+         32:a0:4f:0f:6d:b5:a3:36:ac:54:cc:60:fd:6a:e8:86:c8:57:
+         6e:61:de:66:b3:44:ab:18:95:1c:7f:2d:6e:53:a3:66:17:6e:
+         8c:c8:0c:e1:3c:da:82:d6:c6:28:f8:4c:fa:7f:ca:71:0a:9d:
+         de:7e:19:40:17:e9:40:2f:b3:18:aa:e8:31:bc:0f:5e:61:b5:
+         50:e6:14:3b:e2:7c:dd:70:b3:dc:b9:d0:18:af:87:e4:db:29:
+         1b:f1:a1:6d:2a:44:e5:bf:d7:8d:18:77:dc:f5:25:6a:b3:8a:
+         19:00:3e:15:2b:28:ed:40:c1:45:ad:66:db:74:a0:53:e8:0c:
+         2f:fb:ae:d6:35:9c:3f:c8:81:f9:13:dd:49:06:77:26:82:7d:
+         6d:66:a1:87:a9:9d:ca:4b:2f:c5:19:91:a6:3c:4a:f2:b7:bf:
+         af:8c:97:90
 -----BEGIN CERTIFICATE-----
-MIICejCCAWICCQCk2vsUx0b+0zANBgkqhkiG9w0BAQUFADA3MQswCQYDVQQGEwJY
-WTEQMA4GA1UECgwHRGFzayBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0x
-NzA0MDUxNDM2MjVaFw0yNzAyMTIxNDM2MjVaMEsxCzAJBgNVBAYTAlhZMRkwFwYD
-VQQHDBBEYXNrLWRpc3RyaWJ1dGVkMQ0wCwYDVQQKDAREYXNrMRIwEAYDVQQDDAls
-b2NhbGhvc3QwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANDeoyKD36b4c/F8
-9yszWtQNyMTmw7MSE/2wnLLrd/fUyfIBUoXlPPBb/Y0h+ZJsNZzEyXUyhb1bRhDh
-i60bAJPx2N8Rrwj8x7ph+uLOhSLt0VzxIIJfjf6enYv8jzw+4uPO1sAFzhRYMvfH
-oFM7W2N+X1naFP20hhfNOhEbvzaVAgMBAAEwDQYJKoZIhvcNAQEFBQADggEBABXA
-yhvs8Tu6CGX7avZgbykVSPZMuvBWVl4qbDxolNZr6yPX7AkvcishkfpK6LbzeSjb
-AI5mjS1jafqs5T/gcdWu6tbyJn6g5gTb8JgD0Qpn/cVTim1GenqOSFdCdg0R8tA0
-7uSlyiUiCdV0+TBqAzBbKuzQLgnvjvH4rxs/vykIic+iWPFPAWCCWZ5sM+GbSs3s
-wedzyetoNODTCPh8QQjXH7EAAdTgC1tSOb4c0HXSBVV9FIJMjBafMP8DwawJUxl2
-ceXx2QdL29OZcagxfB+ZJz8cb8kC6Qyo02L9qLlXKMSP+DOcwfGpR/u7xmAxswYe
-q4nr5CMyCRejXgSOiMQ=
+MIIDCTCCAfECFD/Yiu0mpn5vU5/+haSjYkRln/9HMA0GCSqGSIb3DQEBCwUAMDcx
+CzAJBgNVBAYTAlhZMRAwDgYDVQQKDAdEYXNrIENBMRYwFAYDVQQDDA1vdXItY2Et
+c2VydmVyMB4XDTE4MTIxMzAyNTIxMFoXDTI4MTAyMTAyNTIxMFowSzELMAkGA1UE
+BhMCWFkxGTAXBgNVBAcMEERhc2stZGlzdHJpYnV0ZWQxDTALBgNVBAoMBERhc2sx
+EjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC
+ggEBAOVz7rA0lrnyEj4Viul2xxEp4CN4hAPs66ge67YCvzSVQyke+ydB/H1V+HTq
+3OLaiNGNTu54v8pM2loBJOdOlcayjVXgsW1wpMcOpTV3kXUGQnRqpCx/j9QK8J6v
+sZcdDjdO+ui6walZOPVdZ9EfYTflynKJbzthSUV/+ue0F3aOSnfvai5rtLokJ3xF
+16/UIFPi7NpPvcLg/KJhitiW8GhnQSF+eSUXe5a7+FnAnOXlA1gmLeSzYSeUOLhV
+/2X7IhDoY1YrHiZjzH1DFh4bSVY4G3n3A+iy9QL67RUG0/yvhNg0pt8nFQhZg8eD
+NGXKxtWMhprwMIMgJEEQGGwgBl0CAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAb3iU
+O2aCtW/FWZH+VJHaXQQoN2NT03YWIbRPGFfG7hHvwV/RmqSgFWAzlFIvvj1gnJ9R
+eao2KALPDWCp6VS4pdRJ0RDm9KaiHSdVxyhjnkiZ7NKI19o0A7bZMqBPD221ozas
+VMxg/WrohshXbmHeZrNEqxiVHH8tblOjZhdujMgM4TzagtbGKPhM+n/KcQqd3n4Z
+QBfpQC+zGKroMbwPXmG1UOYUO+J83XCz3LnQGK+H5NspG/GhbSpE5b/XjRh33PUl
+arOKGQA+FSso7UDBRa1m23SgU+gML/uu1jWcP8iB+RPdSQZ3JoJ9bWahh6mdyksv
+xRmRpjxK8re/r4yXkA==
 -----END CERTIFICATE-----
diff --git a/distributed/tests/tls-key.pem b/distributed/tests/tls-key.pem
index da13f9ce1b8..ffd84eac31f 100644
--- a/distributed/tests/tls-key.pem
+++ b/distributed/tests/tls-key.pem
@@ -1,16 +1,28 @@
 -----BEGIN PRIVATE KEY-----
-MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBANDeoyKD36b4c/F8
-9yszWtQNyMTmw7MSE/2wnLLrd/fUyfIBUoXlPPBb/Y0h+ZJsNZzEyXUyhb1bRhDh
-i60bAJPx2N8Rrwj8x7ph+uLOhSLt0VzxIIJfjf6enYv8jzw+4uPO1sAFzhRYMvfH
-oFM7W2N+X1naFP20hhfNOhEbvzaVAgMBAAECgYAxFJ/w9FwRuEBGdr79/8tQGQ0c
-d710zsKz0lSAFqTSL8Zkf7sMi5mOQY+LIniq15Khi/xq9bxHirMkNFvGCZpQvM6m
-DuKAHf/0zFQOfM3QtXxEb64uupS8UTsiTkiq84Zwb3g6uWYXB+3zvY+FDbi5SkXp
-P1c6c58wu1WSSqBh4QJBAOijVjTxu1FtvrVN/e4J+lfuXKYs7cO9tbC7L2cAhB0A
-6lAD2w1NRbIoY9IXwFc0E+vbyNqp18wYWuUNi+dYTEcCQQDl2ENZ3vc4LvtWWkMm
-fmQ95ks7WTWCnxkWdFKWLEnRAOUM18dJnJgoJxvySaCifrMhOObz+UgJx/a23Sqj
-IsBDAkBRXshJbn/xhGxfKC94cj8BQJtheHeVrqZ9cYusAG81YduTLzYQWgXVwfDg
-THR3S/zn05FWgm+7sI0ibvGPcsADAkEAjr/dPLDk/Aj54qzWrc6r9WHVcbu7JzAm
-xbJjJI/HHL6gg3R0enx+Z9m6WqVt1d445LZRgk9ddW2zBemgtIc9vwJBAJkuovHW
-wq/hmDkBuPqbe2KX5id4sQZqCr7V+fAs12lxjEIwk/fe2Oop0E1dsLGXQaCzqKmp
-GXaAzcrQXi7JtZc=
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDlc+6wNJa58hI+
+FYrpdscRKeAjeIQD7OuoHuu2Ar80lUMpHvsnQfx9Vfh06tzi2ojRjU7ueL/KTNpa
+ASTnTpXGso1V4LFtcKTHDqU1d5F1BkJ0aqQsf4/UCvCer7GXHQ43TvrousGpWTj1
+XWfRH2E35cpyiW87YUlFf/rntBd2jkp372oua7S6JCd8Rdev1CBT4uzaT73C4Pyi
+YYrYlvBoZ0EhfnklF3uWu/hZwJzl5QNYJi3ks2EnlDi4Vf9l+yIQ6GNWKx4mY8x9
+QxYeG0lWOBt59wPosvUC+u0VBtP8r4TYNKbfJxUIWYPHgzRlysbVjIaa8DCDICRB
+EBhsIAZdAgMBAAECggEBAJlC8bjSt6bmUVcUzp+Yzejh/qvSsONqWhIG7Uy60vqN
+hLmvzvRu9/bD+NwnkHkanW7TI1jYYMsAwIyD3SF/dT2uQAePuROvvPCVELcA8bCo
+CLV4EccDQY/R6t0j38xfBQLeh2vyaq7yQMBrro+ZlTdgDvSoZhrUpbt9e5tOyKlP
+p8EwVW74251XSfONrbsbEvTq5GsNSgerAJT9pC05FhpXp92WPYrDFV0906hlt6z2
+vyLw/PVaQR3YLeNcZ7Fp/AEOTht3M4vfaMq1ibEOi5DueBV0RSGcFzKaP1FFnqsD
+a5x3NGpY5q8ngCcwlWWEsBLnVzDIjI7FJWOnX5F87GUCgYEA/dWuKo09GD6g+oeC
+R6LsSxZdqLl5jopUFgKMpdFWs07rZD1HoH+8Dpl0bYSvSBnXza7d5BmE5HwpA3ts
+Y9OinVu9UCgqToO3PZbbyqQ9AhqLW7YlVjZ2G+U8BM6Z1j0+C3aZ8jRXRRXQj9V1
+pUDuDjCK7WEp3Z3fyuqFpMLy+msCgYEA52kB539F7hRkxoob0gMidC3+qiMDjC6t
+uwbiovyWnO38t+ggQR27AZHfVHqW+iXSt2oiTjZ8pNhrBzpWcLZLvOVXzlTeJL+B
+YL2I/rjnRlplBDAfoUyLRCfQQ9FpaBvvoxooySNeRAIZ+9GgeiqTRYz91/CZX5h/
+OIgNV6Y5xFcCgYAo6ny8ly+17MJ+H8G8UmLMZ7qJK/yQIXQUaCzLvca/+ISxD7+v
++2cQ42OFE9I9SQ0kNn/F+OcW/XyO9yItpTudKeCGPzMynMC4yMx9CaRb5nN4QRAb
+qYO4U2ctTzA0/II3ztJZ3F4E4NNRnBLupFy7JNAAG51n/77HMq6ANHx+KQKBgQDi
+d4CWlpbQVYL+TiDF2a15N54X0SZipTf+ZdGcokefksSVEoWmVDs0YRbH7SUd77AV
+MbgbBPXcubOcojoAGldqqKPS2HmCSyyJiVOtvTURQ9vCSnpZ97f/XltSU3mtmlFe
+NHafyY4yTkAxkkhT5qC0GzQ9hUgEJxCyOWudUJ8OBwKBgGd8n3UchJ+Ll3calcHO
+GKlWdqNPI6yLtY0wTqZGCZIWGjskm6YoYj4kEyBten/ERBArSuUeucj5qCxij97Y
+ue27ZP2xDX7ONdd5D36UPck3mfqBiW+BlUI9vGPhBDlA9sRhT2wIkDuu8W9MkyKd
+7Y9YAmBiswDTAj5TYYeWqWcM
 -----END PRIVATE KEY-----
diff --git a/distributed/tests/tls-self-signed-cert.pem b/distributed/tests/tls-self-signed-cert.pem
index 69729e857c2..9348cd3ed76 100644
--- a/distributed/tests/tls-self-signed-cert.pem
+++ b/distributed/tests/tls-self-signed-cert.pem
@@ -1,14 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIICLDCCAZWgAwIBAgIJAMA9Gen+zrxJMA0GCSqGSIb3DQEBCwUAMEsxCzAJBgNV
-BAYTAlhZMRkwFwYDVQQHDBBEYXNrLWRpc3RyaWJ1dGVkMQ0wCwYDVQQKDAREYXNr
-MRIwEAYDVQQDDAlsb2NhbGhvc3QwHhcNMTcwNDA1MTQzNjI1WhcNMjcwNDAzMTQz
-NjI1WjBLMQswCQYDVQQGEwJYWTEZMBcGA1UEBwwQRGFzay1kaXN0cmlidXRlZDEN
-MAsGA1UECgwERGFzazESMBAGA1UEAwwJbG9jYWxob3N0MIGfMA0GCSqGSIb3DQEB
-AQUAA4GNADCBiQKBgQDE7N1D4bg1tmhg5xAa+H85TPZCG+JOGqzZ8dpxjl5OdKw4
-0x3c4KTySAXUY7TtOCG7Woc0/Vam7PJTdU3ZAcU3xzBdYBCze2umY2zsV2lNFQmC
-XuJ1Dnv90DEgljcFNOD4vzfb6kyGUcZeacId6+7D3bdq7r4zEeqCwSBdiIbo3wID
-AQABoxgwFjAUBgNVHREEDTALgglsb2NhbGhvc3QwDQYJKoZIhvcNAQELBQADgYEA
-Mzek0ecKBNJe0d7bD+cilK4S+Gu9NxnXiFw3AUGBhJGmQWrrRaRWuwVZWwrW4qoj
-NnxyB/U0VRiTH2jOMzV/ejfhRp9YPXJj2J0EMEvbnxsWqcSnCg5Y0Px/VdXDHx5P
-V5XJHrcfYAiR/KeDclLoKytf5lTMAZ/2fXUoWa2oPm0=
+MIIDPDCCAiSgAwIBAgIURCSGaBiclxOXPJYUmnDRNUjo1ggwDQYJKoZIhvcNAQEL
+BQAwSzELMAkGA1UEBhMCWFkxGTAXBgNVBAcMEERhc2stZGlzdHJpYnV0ZWQxDTAL
+BgNVBAoMBERhc2sxEjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0xODEyMTMwMjUyMTBa
+Fw0yODEyMTAwMjUyMTBaMEsxCzAJBgNVBAYTAlhZMRkwFwYDVQQHDBBEYXNrLWRp
+c3RyaWJ1dGVkMQ0wCwYDVQQKDAREYXNrMRIwEAYDVQQDDAlsb2NhbGhvc3QwggEi
+MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDD5bauoB8+u6oFQwPjQ8OAgMGr
+qUV6jkvfnhBpjUVhX/Y6CjZAGwFUye6twYzGXrwMrrwFtV4ApfcefAv4naCLvs1F
+yhAM8H8Sgqt7dChg/ejAnEvbxJW7vFnnpVmaiKfvkv36uLyLNjncumZ6e/jvtGgT
+Wr2rgrkjjT9zwRXt+be7zONht2Hx2P3iJQU/d4CTvU2c0DfnJqRI4B4vQ1sCeHHn
+nnKMjd+qYqQ3T5bmfH2gqqAESRbwfaRdf1k3S+zLpqWbqg7zW8cY9kysNZIvyfCg
+brHFzgvjyhw4enJWQiRxEHSfiHEza9TWqwm3gaETdhxryK+1KlQ2Q/fLuBJXAgMB
+AAGjGDAWMBQGA1UdEQQNMAuCCWxvY2FsaG9zdDANBgkqhkiG9w0BAQsFAAOCAQEA
+Ai9eW0p8XK26NF14jYZi830Qs0sUQ2uqO5glZbBGJz2U9L7kID68JDU+W2DsspYo
+xfj7aSFNo/1bYFMcWR/iJq0SzHX7ROwJ2GBpQwHWN/dMvmFVBreBPN2ggrV+wOPD
+iWHkitYEZGdViRQkMoyPeIBbkKcq3BPwNDcNzWQLhwbcxJNUk9PeEUG1c98Zcbvl
+H1YQtGh6LO/wPL8b+zqVxG0klYfe77uZHVFt92RiQA+KnTXAi+W+iWpQgDIdC3YA
+f3Jd/17CirqWtD+9u9kLV7vknL8TUCRO28as/2fLOjv7LjKliu23L2mfWp5Vhn0G
+A2ql2SoZUyYUWt/ZgWX0JA==
 -----END CERTIFICATE-----
diff --git a/distributed/tests/tls-self-signed-key.pem b/distributed/tests/tls-self-signed-key.pem
index 189d83ac3b0..78c5705552e 100644
--- a/distributed/tests/tls-self-signed-key.pem
+++ b/distributed/tests/tls-self-signed-key.pem
@@ -1,16 +1,28 @@
 -----BEGIN PRIVATE KEY-----
-MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBAMTs3UPhuDW2aGDn
-EBr4fzlM9kIb4k4arNnx2nGOXk50rDjTHdzgpPJIBdRjtO04IbtahzT9Vqbs8lN1
-TdkBxTfHMF1gELN7a6ZjbOxXaU0VCYJe4nUOe/3QMSCWNwU04Pi/N9vqTIZRxl5p
-wh3r7sPdt2ruvjMR6oLBIF2IhujfAgMBAAECgYAHh68mS1lJ5IG1QuCgx2PGAyVf
-fcV3ixmU8+SQdwswINH9EkWQDp8ADLT8XBus/np7uYh2BEHt/B2AB3pElAbAfBQU
-nTLcdwTvS5lVgvBtTzw9mPCaISlZ/YUmnRvFemXu5VP+EwLvCHim7XkLz3t9ldgY
-rjk8FlypH6Xx1cdlYQJBAOmgYqOu6GtwE6toCTGbP+xeNyb0kMqcjoV+B6sBAeCy
-cNyyqR6M2Mgm38+GvcEZ2fT2JbbFb6KRFdq7e1+MhSkCQQDXyLTvg1WYuilx4Ggg
-Fys6FNY7liIx8lFfoH3SPvFVDyOk7q7p4jOgIRaYmaK34HNF8+K1bTArPN/lDyBM
-ZvbHAkEArKAm4n12HRTWY9velzn41MWVCQLDDgL6SGckcVWwNoCDZn/Y+Mrnz5Tx
-4RzVjx5z704F9fQTFdMQi+vmMGbbaQJAXTBpq5KNkeK76VkfvtQir/e1uZAR/n4G
-1ISMMhNKyBA8tYr68TjmUL5ZctfO5RhC0DAOdYnNJq8ohlTs/vHsmwJAJL5R/NBo
-CeffVkMArufyG9yXhPHPhWQCH4Q1h40A+5H9VCeiW7uGs2hWeVRDAqWEAbo7tj2+
-0BZ5fzdy3VCleA==
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDD5bauoB8+u6oF
+QwPjQ8OAgMGrqUV6jkvfnhBpjUVhX/Y6CjZAGwFUye6twYzGXrwMrrwFtV4Apfce
+fAv4naCLvs1FyhAM8H8Sgqt7dChg/ejAnEvbxJW7vFnnpVmaiKfvkv36uLyLNjnc
+umZ6e/jvtGgTWr2rgrkjjT9zwRXt+be7zONht2Hx2P3iJQU/d4CTvU2c0DfnJqRI
+4B4vQ1sCeHHnnnKMjd+qYqQ3T5bmfH2gqqAESRbwfaRdf1k3S+zLpqWbqg7zW8cY
+9kysNZIvyfCgbrHFzgvjyhw4enJWQiRxEHSfiHEza9TWqwm3gaETdhxryK+1KlQ2
+Q/fLuBJXAgMBAAECggEAYcPL1vyce09m+7dFQm94KzKW9RfGT/KiFRiqcjMJ1B3z
+IWYlSvrvQCSDFH5KxllTGjU+Ks8n+cTaPoGD6/CmxfdUNKcyOXLwyAkkkbkXycgO
+F8oo5aYHYCkQ6VWREVlwDlVaoXz+w68tU2ZHbItgYiiz5CzeCbF0ujRs3VUYn6mo
+CqGk979FS6Q+q48aoGYtrWqslP9RT9oB4wsrSawxiRTnSFdjqpmmMLpVheC5f0u9
+szF0P9Uv/gkX+TZXGU4h099LPav7jRDhDX8jQ7RJbDG7QdUT0HWFiix+DexdPN5T
+ZBqoSTIdncSmLF0DlcHxe45sXkrAQ1KsH+9G9KM4WQKBgQDn3OJKibhnp2cPbO6F
+kT2r46V681xZ1YMPioTD3vPquOtBUbzyIqdAZcAGn9IjtksQXv5riq/Y2yzxHYja
+ZmouBhu8BgZyxQujpAHeduKuE1xYWK9uGrFKt3t3YNtUUum7bISPyzgSzldNIaqG
+fTjCZ+fC/2OC4w6k0arTePOR9QKBgQDYSlps8EoT/t7Vu9KZs57Kph5dyGjSL6M9
+Gn0lwZR0wyJA5Snw64nFcBJETIM5MIMaMDeUAo2i5upqfKzNEuK8+q3+E3khfb8i
+HZBnBDZiRZbnECobSswuxyZfMZko2guZ5PrA4XnWiJDpGWP6u7F8C45Ji2W191GM
+KpiyVJ0HmwKBgG+d8L3LMJ+Ngdp2KRjZg2PE8fuPPcztMOuBVkpogOlk3bOTIq85
+qWQh12AY/fM9JUObR0UD/YUadbcY7bGdjc3cGOvZxq7nVqQPJusuNg9wuIq7HrCV
+XvXGAETGtq8QPv5/zgze2n6LIJG7/DhD/dfwZyyC/yeaIImNeXvWBQMlAoGBAImK
+3u4f8Du+/F3WzxpHC1e7Q9xHscDX4mrdCXCVJZ8K6CF0mbNHVZlkp3kfXuNjYjup
+Je7NoNxTFAcbuZe/xsh+cV79ay5NFNrRCh84oozjzA6sTNqGOkoVAuXOWKON5Yq7
+/mnh+OTge6aAKwArk7ZacuV56/hHimgkPm5JOIK5AoGBAKDmXI2DeziBvKu4GoXf
+ZjsAULxm09vnVqJTD+R8a//HTGBH2WTp3/tiEilOtDQ3eWhfhwqS5VFsqYSw/asS
+HTahl1q4aYQbYM+B83zlrBglV89GpN7ZBlFXXbasDAKh8mkg3nRad6Zy2Vz99Nts
+m1W0vRoaqKJIcxM25x9Vc16f
 -----END PRIVATE KEY-----

From 19a393b5ea2b8dc10bd6d18a3a38c2f1d67655d0 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sat, 15 Dec 2018 11:58:40 -0500
Subject: [PATCH 0160/1550] bump version to 1.25.1

---
 docs/source/changelog.rst | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index bb76c3c92ca..ed82e098e18 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,24 @@
 Changelog
 =========
 
+1.25.1 - 2018-12-15
+-------------------
+
+-  Serialize numpy.ma.masked objects properly (:pr:`2384`) `Jim Crist`_
+-  Turn off bokeh property validation in dashboard (:pr:`2387`) `Jim Crist`_
+-  Fully initialize WorkerState objects (:pr:`2388`) `Jim Crist`_
+-  Fix typo in scheduler docstring (:pr:`2393`) `Russ Bubley`_
+-  DOC: fix typo in distributed.worker.Worker docstring (:pr:`2395`) `Loïc Estève`_
+-  Remove clients and workers from event log after removal (:pr:`2394`) `tjb900`_
+-  Support msgpack 0.6.0 by providing length keywords (:pr:`2399`) `tjb900`_
+-  Use async-await on large messages test (:pr:`2404`) `Matthew Rocklin`_
+-  Fix race condition in normalize_collection (:pr:`2386`) `Jim Crist`_
+-  Fix redict collection after HighLevelGraph fix upstream (:pr:`2413`) `Matthew Rocklin`_
+-  Add a blocking argument to Lock.acquire(`) (:pr:`2412`) `Stephan Hoyer`_
+-  Fix long traceback test (:pr:`2417`) `Matthew Rocklin`_
+-  Update x509 certificates to current OpenSSL standards. (:pr:`2418`) `Diane Trout`_
+
+
 1.25.0 - 2018-11-28
 -------------------
 
@@ -859,3 +877,6 @@ significantly without many new features.
 .. _`Tom Rochette`: https://github.com/tomzx
 .. _`Roy Wedge`: https://github.com/rwedge
 .. _`Michael Wheeler`: https://github.com/mikewheel
+.. _`Diane Trout`: https://github.com/detrout
+.. _`tjb900`: https://github.com/tjb900
+.. _`Stephan Hoyer`: https://github.com/shoyer

From 879cdb35c52135cd53ff89c36c485311b7801004 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sat, 15 Dec 2018 16:59:37 -0500
Subject: [PATCH 0161/1550] Name threads in ThreadPoolExecutors (#2408)

This helps to improve debugging
---
 distributed/client.py             | 5 ++++-
 distributed/comm/utils.py         | 5 ++++-
 distributed/threadpoolexecutor.py | 3 ++-
 distributed/worker.py             | 4 ++--
 4 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index ab004139874..6d61442b7db 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -250,7 +250,10 @@ def add_done_callback(self, fn):
         """
         cls = Future
         if cls._cb_executor is None or cls._cb_executor_pid != os.getpid():
-            cls._cb_executor = ThreadPoolExecutor(1)
+            try:
+                cls._cb_executor = ThreadPoolExecutor(1, thread_name_prefix="Dask-Callback-Thread")
+            except TypeError:
+                cls._cb_executor = ThreadPoolExecutor(1)
             cls._cb_executor_pid = os.getpid()
 
         def execute_callback(fut):
diff --git a/distributed/comm/utils.py b/distributed/comm/utils.py
index d3e758a0741..6c9a99b8a8d 100644
--- a/distributed/comm/utils.py
+++ b/distributed/comm/utils.py
@@ -19,7 +19,10 @@
 
 FRAME_OFFLOAD_THRESHOLD = 10 * 1024 ** 2   # 10 MB
 
-_offload_executor = ThreadPoolExecutor(max_workers=1)
+try:
+    _offload_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix='Dask-Offload')
+except TypeError:
+    _offload_executor = ThreadPoolExecutor(max_workers=1)
 finalize(_offload_executor, _offload_executor.shutdown)
 
 
diff --git a/distributed/threadpoolexecutor.py b/distributed/threadpoolexecutor.py
index c5c953ce0b9..84c08f447da 100644
--- a/distributed/threadpoolexecutor.py
+++ b/distributed/threadpoolexecutor.py
@@ -75,11 +75,12 @@ def __init__(self, *args, **kwargs):
         super(ThreadPoolExecutor, self).__init__(*args, **kwargs)
         self._rejoin_list = []
         self._rejoin_lock = threading.Lock()
+        self._thread_name_prefix = kwargs.get('thread_name_prefix', 'DaskThreadPoolExecutor')
 
     def _adjust_thread_count(self):
         if len(self._threads) < self._max_workers:
             t = threading.Thread(target=_worker,
-                                 name="ThreadPoolExecutor-%d-%d" % (os.getpid(), next(self._counter)),
+                                 name=self._thread_name_prefix + "-%d-%d" % (os.getpid(), next(self._counter)),
                                  args=(self, self._work_queue))
             t.daemon = True
             self._threads.add(t)
diff --git a/distributed/worker.py b/distributed/worker.py
index 024d39c9d34..1291c5e5322 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -424,8 +424,8 @@ def __init__(self, scheduler_ip=None, scheduler_port=None,
         self.status = None
         self._closed = Event()
         self.reconnect = reconnect
-        self.executor = executor or ThreadPoolExecutor(self.ncores)
-        self.actor_executor = ThreadPoolExecutor(1)
+        self.executor = executor or ThreadPoolExecutor(self.ncores, thread_name_prefix="Dask-Worker-Threads'")
+        self.actor_executor = ThreadPoolExecutor(1, thread_name_prefix="Dask-Actor-Threads")
         self.name = name
         self.scheduler_delay = 0
         self.stream_comms = dict()

From f2139b368d403bc1348ce0767c4fcbe595261f9f Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 19 Dec 2018 09:10:37 -0500
Subject: [PATCH 0162/1550] Support coroutines and async-def functions in
 run/run_scheduler (#2427)

---
 distributed/client.py                | 35 +++++++---------------
 distributed/scheduler.py             |  4 +--
 distributed/tests/py3_test_client.py | 44 ++++++++++++++++++++++++++++
 distributed/tests/test_client.py     | 17 +++++------
 distributed/tests/test_worker.py     |  6 ++--
 distributed/worker.py                | 28 +++++++++++++-----
 6 files changed, 88 insertions(+), 46 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 6d61442b7db..b1e6108cb13 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -2048,9 +2048,11 @@ def get_dataset(self, name, **kwargs):
 
     @gen.coroutine
     def _run_on_scheduler(self, function, *args, **kwargs):
+        wait = kwargs.pop('wait', True)
         response = yield self.scheduler.run_function(function=dumps(function),
                                                      args=dumps(args),
-                                                     kwargs=dumps(kwargs))
+                                                     kwargs=dumps(kwargs),
+                                                     wait=wait)
         if response['status'] == 'error':
             six.reraise(*clean_exception(**response))
         else:
@@ -2084,9 +2086,11 @@ def run_on_scheduler(self, function, *args, **kwargs):
     def _run(self, function, *args, **kwargs):
         nanny = kwargs.pop('nanny', False)
         workers = kwargs.pop('workers', None)
+        wait = kwargs.pop('wait', True)
         responses = yield self.scheduler.broadcast(msg=dict(op='run',
                                                             function=dumps(function),
                                                             args=dumps(args),
+                                                            wait=wait,
                                                             kwargs=dumps(kwargs)),
                                                    workers=workers, nanny=nanny)
         results = {}
@@ -2095,7 +2099,8 @@ def _run(self, function, *args, **kwargs):
                 results[key] = resp['result']
             elif resp['status'] == 'error':
                 six.reraise(*clean_exception(**resp))
-        raise gen.Return(results)
+        if wait:
+            raise gen.Return(results)
 
     def run(self, function, *args, **kwargs):
         """
@@ -2142,27 +2147,6 @@ def run(self, function, *args, **kwargs):
         """
         return self.sync(self._run, function, *args, **kwargs)
 
-    @gen.coroutine
-    def _run_coroutine(self, function, *args, **kwargs):
-        workers = kwargs.pop('workers', None)
-        wait = kwargs.pop('wait', True)
-        responses = yield self.scheduler.broadcast(msg=dict(op='run_coroutine',
-                                                            function=dumps(function),
-                                                            args=dumps(args),
-                                                            kwargs=dumps(kwargs),
-                                                            wait=wait),
-                                                   workers=workers)
-        if not wait:
-            raise gen.Return(None)
-        else:
-            results = {}
-            for key, resp in responses.items():
-                if resp['status'] == 'OK':
-                    results[key] = resp['result']
-                elif resp['status'] == 'error':
-                    six.reraise(*clean_exception(**resp))
-            raise gen.Return(results)
-
     def run_coroutine(self, function, *args, **kwargs):
         """
         Spawn a coroutine on all workers.
@@ -2184,7 +2168,10 @@ def run_coroutine(self, function, *args, **kwargs):
             Workers on which to run the function. Defaults to all known workers.
 
         """
-        return self.sync(self._run_coroutine, function, *args, **kwargs)
+        warnings.warn("This method has been deprecated. "
+                      "Instead use Client.run which detects async functions "
+                      "automatically")
+        return self.run(function, *args, **kwargs)
 
     def _graph_to_futures(self, dsk, keys, restrictions=None,
                           loose_restrictions=None, priority=None,
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 2ec02119376..ee24e0182de 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -3068,7 +3068,7 @@ def get_task_duration(self, ts, default=0.5):
             self.unknown_durations[prefix].add(ts)
             return default
 
-    def run_function(self, stream, function, args=(), kwargs={}):
+    def run_function(self, stream, function, args=(), kwargs={}, wait=True):
         """ Run a function within this process
 
         See Also
@@ -3077,7 +3077,7 @@ def run_function(self, stream, function, args=(), kwargs={}):
         """
         from .worker import run
         self.log_event('all', {'action': 'run-function', 'function': function})
-        return run(self, stream, function=function, args=args, kwargs=kwargs)
+        return run(self, stream, function=function, args=args, kwargs=kwargs, wait=wait)
 
     def set_metadata(self, stream=None, keys=None, value=None):
         try:
diff --git a/distributed/tests/py3_test_client.py b/distributed/tests/py3_test_client.py
index 28ee964b495..cad9fb67fb0 100644
--- a/distributed/tests/py3_test_client.py
+++ b/distributed/tests/py3_test_client.py
@@ -158,3 +158,47 @@ async def test_dont_hold_on_to_large_messages(c, s, a, b):
             pytest.fail("array should have been destroyed")
 
         await gen.sleep(0.200)
+
+
+@gen_cluster(client=True)
+async def test_run_scheduler_async_def(c, s, a, b):
+    async def f(dask_scheduler):
+        await gen.sleep(0.01)
+        dask_scheduler.foo = 'bar'
+
+    await c.run_on_scheduler(f)
+
+    assert s.foo == 'bar'
+
+    async def f(dask_worker):
+        await gen.sleep(0.01)
+        dask_worker.foo = 'bar'
+
+    await c.run(f)
+    assert a.foo == 'bar'
+    assert b.foo == 'bar'
+
+
+@gen_cluster(client=True)
+async def test_run_scheduler_async_def_wait(c, s, a, b):
+    async def f(dask_scheduler):
+        await gen.sleep(0.01)
+        dask_scheduler.foo = 'bar'
+
+    await c.run_on_scheduler(f, wait=False)
+
+    while not hasattr(s, 'foo'):
+        await gen.sleep(0.01)
+    assert s.foo == 'bar'
+
+    async def f(dask_worker):
+        await gen.sleep(0.01)
+        dask_worker.foo = 'bar'
+
+    await c.run(f, wait=False)
+
+    while not hasattr(a, 'foo') or not hasattr(b, 'foo'):
+        await gen.sleep(0.01)
+
+    assert a.foo == 'bar'
+    assert b.foo == 'bar'
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 2a8f5d502dc..35af16fd4d9 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -2616,35 +2616,34 @@ def func(x, y=10):
 
 @gen_cluster(client=True)
 def test_run_coroutine(c, s, a, b):
-    results = yield c.run_coroutine(geninc, 1, delay=0.05)
+    results = yield c.run(geninc, 1, delay=0.05)
     assert results == {a.address: 2, b.address: 2}
 
-    results = yield c.run_coroutine(geninc, 1, delay=0.05, workers=[a.address])
+    results = yield c.run(geninc, 1, delay=0.05, workers=[a.address])
     assert results == {a.address: 2}
 
-    results = yield c.run_coroutine(geninc, 1, workers=[])
+    results = yield c.run(geninc, 1, workers=[])
     assert results == {}
 
     with pytest.raises(RuntimeError) as exc_info:
-        yield c.run_coroutine(throws, 1)
+        yield c.run(throws, 1)
     assert "hello" in str(exc_info)
 
     if sys.version_info >= (3, 5):
-        results = yield c.run_coroutine(asyncinc, 2, delay=0.01)
+        results = yield c.run(asyncinc, 2, delay=0.01)
         assert results == {a.address: 3, b.address: 3}
 
 
 def test_run_coroutine_sync(c, s, a, b):
-    result = c.run_coroutine(geninc, 2, delay=0.01)
+    result = c.run(geninc, 2, delay=0.01)
     assert result == {a['address']: 3,
                       b['address']: 3}
 
-    result = c.run_coroutine(geninc, 2,
-                             workers=[a['address']])
+    result = c.run(geninc, 2, workers=[a['address']])
     assert result == {a['address']: 3}
 
     t1 = time()
-    result = c.run_coroutine(geninc, 2, delay=10, wait=False)
+    result = c.run(geninc, 2, delay=10, wait=False)
     t2 = time()
     assert result is None
     assert t2 - t1 <= 1.0
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 2ee1a988b7b..dcdf4b570eb 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -399,7 +399,7 @@ def f(dask_worker=None):
         yield gen.sleep(0.001)
         raise gen.Return(dask_worker.id)
 
-    response = yield c._run_coroutine(f)
+    response = yield c.run(f)
     assert response == {a.address: a.id, b.address: b.id}
 
 
@@ -866,7 +866,7 @@ def f():
         result = yield future
         raise gen.Return(result)
 
-    results = yield c.run_coroutine(f)
+    results = yield c.run(f)
     assert results == {a.address: 11,
                        b.address: 11}
 
@@ -879,7 +879,7 @@ def f():
         result = yield future
         raise gen.Return(result)
 
-    results = client.run_coroutine(f)
+    results = client.run(f)
     assert results == {a['address']: 11,
                        b['address']: 11}
 
diff --git a/distributed/worker.py b/distributed/worker.py
index 1291c5e5322..6f05a0260a6 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -1960,13 +1960,14 @@ def executor_submit(self, key, function, args=(), kwargs=None,
         # logger.info("Finish job %d, %s", i, key)
         raise gen.Return(result)
 
-    def run(self, comm, function, args=(), kwargs=None):
+    def run(self, comm, function, args=(), wait=True, kwargs=None):
         kwargs = kwargs or {}
-        return run(self, comm, function=function, args=args, kwargs=kwargs)
+        return run(self, comm, function=function, args=args, kwargs=kwargs,
+                   wait=wait)
 
     def run_coroutine(self, comm, function, args=(), kwargs=None, wait=True):
         return run(self, comm, function=function, args=args, kwargs=kwargs,
-                   is_coro=True, wait=wait)
+                   wait=wait)
 
     @gen.coroutine
     def actor_execute(self, comm=None, actor=None, function=None, args=(), kwargs={}):
@@ -2915,9 +2916,14 @@ def weight(k, v):
 
 
 @gen.coroutine
-def run(server, comm, function, args=(), kwargs={}, is_coro=False, wait=True):
-    assert wait or is_coro, "Combination not supported"
+def run(server, comm, function, args=(), kwargs={}, is_coro=None, wait=True):
     function = pickle.loads(function)
+    if is_coro is None:
+        is_coro = iscoroutinefunction(function)
+    else:
+        warnings.warn("The is_coro= parameter is deprecated. "
+                      "We now automatically detect coroutines/async functions")
+    assert wait or is_coro, "Combination not supported"
     if args:
         args = pickle.loads(args)
     if kwargs:
@@ -2928,9 +2934,15 @@ def run(server, comm, function, args=(), kwargs={}, is_coro=False, wait=True):
         kwargs['dask_scheduler'] = server
     logger.info("Run out-of-band function %r", funcname(function))
     try:
-        result = function(*args, **kwargs)
-        if is_coro:
-            result = (yield result) if wait else None
+        if not is_coro:
+            result = function(*args, **kwargs)
+        else:
+            if wait:
+                result = yield function(*args, **kwargs)
+            else:
+                server.loop.add_callback(function, *args, **kwargs)
+                result = None
+
     except Exception as e:
         logger.warning(" Run Failed\n"
                        "Function: %s\n"

From 9a327fb166dc0adaf51cd362d126800e1a553dc7 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 19 Dec 2018 14:25:36 -0500
Subject: [PATCH 0163/1550] Add parameters to Client.run docstring (#2429)

---
 distributed/client.py | 27 ++++++++++++++++++++++++---
 1 file changed, 24 insertions(+), 3 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index b1e6108cb13..17485f0444b 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -2074,6 +2074,15 @@ def run_on_scheduler(self, function, *args, **kwargs):
         >>> client.run_on_scheduler(get_number_of_tasks)  # doctest: +SKIP
         100
 
+        Run asynchronous functions in the background:
+
+        >>> async def print_state(dask_scheduler):  # doctest: +SKIP
+        ...    while True:
+        ...        print(dask_scheduler.status)
+        ...        await gen.sleep(1)
+
+        >>> c.run(print_state, wait=False)  # doctest: +SKIP
+
         See Also
         --------
         Client.run: Run a function on all workers
@@ -2122,6 +2131,9 @@ def run(self, function, *args, **kwargs):
         **kwargs: keyword arguments for remote function
         workers: list
             Workers on which to run the function. Defaults to all known workers.
+        wait: boolean (optional)
+            If the function is asynchronous whether or not to wait until that
+            function finishes.
 
         Examples
         --------
@@ -2144,6 +2156,15 @@ def run(self, function, *args, **kwargs):
         >>> c.run(get_hostname)  # doctest: +SKIP
         {'192.168.0.100:9000': 'running',
          '192.168.0.101:9000': 'running}
+
+        Run asynchronous functions in the background:
+
+        >>> async def print_state(dask_worker):  # doctest: +SKIP
+        ...    while True:
+        ...        print(dask_worker.status)
+        ...        await gen.sleep(1)
+
+        >>> c.run(print_state, wait=False)  # doctest: +SKIP
         """
         return self.sync(self._run, function, *args, **kwargs)
 
@@ -3013,10 +3034,10 @@ def write_scheduler_file(self, scheduler_file):
 
         Examples
         --------
-        >>> client = Client()
-        >>> client.write_scheduler_file('scheduler.json')
+        >>> client = Client()  # doctest: +SKIP
+        >>> client.write_scheduler_file('scheduler.json')  # doctest: +SKIP
         # connect to previous client's scheduler
-        >>> client2 = Client(scheduler_file='scheduler.json')
+        >>> client2 = Client(scheduler_file='scheduler.json')  # doctest: +SKIP
         """
         if self.scheduler_file:
             raise ValueError('Scheduler file already set')

From 0f06178a26d9e768c60f35c8c5c5ba31de90172a Mon Sep 17 00:00:00 2001
From: tjb900 <ozburgess@gmail.com>
Date: Thu, 20 Dec 2018 03:28:00 +0800
Subject: [PATCH 0164/1550] gather_dep: don't request dependencies we already
 found out we don't want (#2428)

---
 distributed/worker.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/distributed/worker.py b/distributed/worker.py
index 6f05a0260a6..12b41ca12af 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -1594,8 +1594,10 @@ def gather_dep(self, worker, dep, deps, total_nbytes, cause=None):
                 if self.validate:
                     self.validate_state()
 
+                # dep states may have changed before gather_dep runs
+                # if a dep is no longer in-flight then don't fetch it
                 deps = tuple(dep for dep in deps
-                                 if self.dep_state.get(dep) in ('waiting', 'flight'))
+                                 if self.dep_state.get(dep) == 'flight')
 
                 self.log.append(('request-dep', dep, worker, deps))
                 logger.debug("Request %d keys", len(deps))

From 538767b4977d1bd14679ae555b7705088a7e5a16 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sun, 23 Dec 2018 08:44:01 -0800
Subject: [PATCH 0165/1550] Make LocalCluster.close async friendly (#2437)

---
 distributed/deploy/local.py               | 55 +++++++++++++----------
 distributed/deploy/tests/test_adaptive.py | 32 ++++++-------
 distributed/deploy/tests/test_local.py    |  9 ++--
 distributed/tests/test_client.py          |  2 +-
 4 files changed, 52 insertions(+), 46 deletions(-)

diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index d3feedb578f..4f6ebe12fd3 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -4,7 +4,7 @@
 from datetime import timedelta
 import logging
 import math
-from time import sleep
+import warnings
 import weakref
 import toolz
 
@@ -13,7 +13,7 @@
 from .cluster import Cluster
 from ..core import CommClosedError
 from ..utils import (sync, ignoring, All, silence_logging, LoopRunner,
-        log_errors, thread_state)
+        log_errors, thread_state, parse_timedelta)
 from ..nanny import Nanny
 from ..scheduler import Scheduler
 from ..worker import Worker, _ncores
@@ -151,9 +151,12 @@ def __repr__(self):
     def __await__(self):
         return self._started.__await__()
 
+    @property
+    def asynchronous(self):
+        return self._asynchronous or getattr(thread_state, 'asynchronous', False)
+
     def sync(self, func, *args, **kwargs):
-        asynchronous = kwargs.pop('asynchronous', None)
-        if asynchronous or self._asynchronous or getattr(thread_state, 'asynchronous', False):
+        if kwargs.pop('asynchronous', None) or self.asynchronous:
             callback_timeout = kwargs.pop('callback_timeout', None)
             future = func(*args, **kwargs)
             if callback_timeout is not None:
@@ -196,6 +199,10 @@ def _start(self, ip=None, n_workers=0):
 
     @gen.coroutine
     def _start_worker(self, death_timeout=60, **kwargs):
+        if self.status and self.status.startswith('clos'):
+            warnings.warn("Tried to start a worker while status=='%s'" % self.status)
+            return
+
         if self.processes:
             W = Nanny
             kwargs['quiet'] = True
@@ -257,14 +264,22 @@ def stop_worker(self, w):
         self.sync(self._stop_worker, w)
 
     @gen.coroutine
-    def _close(self):
+    def _close(self, timeout='2s'):
         # Can be 'closing' as we're called by close() below
         if self.status == 'closed':
             return
+        self.status = 'closing'
+
+        self.scheduler.clear_task_state()
+
+        with ignoring(gen.TimeoutError):
+            yield gen.with_timeout(
+                timedelta(seconds=parse_timedelta(timeout)),
+                All([self._stop_worker(w) for w in self.workers]),
+            )
+        del self.workers[:]
 
         try:
-            with ignoring(gen.TimeoutError, CommClosedError, OSError):
-                yield All([w._close() for w in self.workers])
             with ignoring(gen.TimeoutError, CommClosedError, OSError):
                 yield self.scheduler.close(fast=True)
             del self.workers[:]
@@ -277,26 +292,18 @@ def close(self, timeout=20):
             return
 
         try:
-            self.scheduler.clear_task_state()
-
-            for w in self.workers:
-                self.loop.add_callback(self._stop_worker, w)
-            for i in range(10):
-                if not self.workers:
-                    break
-                else:
-                    sleep(0.01)
-            del self.workers[:]
-            try:
-                self._loop_runner.run_sync(self._close, callback_timeout=timeout)
-            except RuntimeError:  # IOLoop is closed
-                pass
-            self._loop_runner.stop()
-        finally:
-            self.status = 'closed'
+            result = self.sync(self._close, callback_timeout=timeout)
+        except RuntimeError:  # IOLoop is closed
+            pass
+
         with ignoring(AttributeError):
             silence_logging(self._old_logging_level)
 
+        if not self.asynchronous:
+            self._loop_runner.stop()
+
+        return result
+
     @gen.coroutine
     def scale_up(self, n, **kwargs):
         """ Bring the total count of workers up to ``n``
diff --git a/distributed/deploy/tests/test_adaptive.py b/distributed/deploy/tests/test_adaptive.py
index 3014defa74d..c3a40b23c21 100644
--- a/distributed/deploy/tests/test_adaptive.py
+++ b/distributed/deploy/tests/test_adaptive.py
@@ -91,7 +91,7 @@ def test_adaptive_local_cluster_multi_workers():
                                  asynchronous=True)
     try:
         cluster.scheduler.allowed_failures = 1000
-        alc = Adaptive(cluster.scheduler, cluster, interval=100)
+        alc = cluster.adapt(interval=100)
         c = yield Client(cluster, asynchronous=True)
 
         futures = c.map(slowinc, range(100), delay=0.01)
@@ -120,8 +120,8 @@ def test_adaptive_local_cluster_multi_workers():
         yield c.gather(futures)
 
     finally:
-        yield c._close()
-        yield cluster._close()
+        yield c.close()
+        yield cluster.close()
 
 
 @gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 10, active_rpc_timeout=10)
@@ -199,8 +199,8 @@ def test_min_max():
             assert time() < start + 2
         assert frequencies(pluck(1, adapt.log)) == {'up': 2, 'down': 1}
     finally:
-        yield c._close()
-        yield cluster._close()
+        yield c.close()
+        yield cluster.close()
 
 
 @gen_test()
@@ -223,8 +223,8 @@ def test_avoid_churn():
 
         assert frequencies(pluck(1, adapt.log)) == {'up': 1}
     finally:
-        yield client._close()
-        yield cluster._close()
+        yield client.close()
+        yield cluster.close()
 
 
 @gen_test(timeout=None)
@@ -270,8 +270,8 @@ def test_adapt_quickly():
         yield gen.sleep(0.1)
         assert len(cluster.scheduler.workers) == 1
     finally:
-        yield client._close()
-        yield cluster._close()
+        yield client.close()
+        yield cluster.close()
 
 
 @gen_test(timeout=None)
@@ -295,8 +295,8 @@ def test_adapt_down():
             yield gen.sleep(0.1)
             assert time() < start + 1
     finally:
-        yield client._close()
-        yield cluster._close()
+        yield client.close()
+        yield cluster.close()
 
 
 @pytest.mark.xfail(reason="we currently only judge occupancy, not ntasks")
@@ -317,8 +317,8 @@ def test_no_more_workers_than_tasks():
 
         assert len(cluster.scheduler.workers) <= 1
     finally:
-        yield client._close()
-        yield cluster._close()
+        yield client.close()
+        yield cluster.close()
 
 
 def test_basic_no_loop():
@@ -358,8 +358,8 @@ def test_target_duration():
         assert adaptive.log[1][1:] == ('up', {'n': 20})
 
     finally:
-        yield client._close()
-        yield cluster._close()
+        yield client.close()
+        yield cluster.close()
 
 
 @gen_test(timeout=None)
@@ -391,7 +391,7 @@ def key(ws):
         names = {ws.name for ws in cluster.scheduler.workers.values()}
         assert names == {'a-1', 'a-2'} or names == {'b-1', 'b-2'}
     finally:
-        yield cluster._close()
+        yield cluster.close()
 
 
 @gen_cluster(client=True, ncores=[])
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index c8b6bbc7665..ee586c5be4a 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -50,7 +50,6 @@ def test_close_twice():
             cluster.close()
             sleep(0.5)
         log = log.getvalue()
-        print(log)
         assert not log
 
 
@@ -291,8 +290,8 @@ def test_scale_up_and_down():
     assert len(cluster.workers) == 1
     assert addr not in cluster.scheduler.ncores
 
-    yield c._close()
-    yield cluster._close()
+    yield c.close()
+    yield cluster.close()
 
 
 def test_silent_startup():
@@ -511,8 +510,8 @@ def scale_down(self, *args, **kwargs):
         yield gen.sleep(0.01)
         assert time() < start + 3
 
-    yield c._close()
-    yield cluster._close()
+    yield c.close()
+    yield cluster.close()
 
 
 def test_local_tls_restart(loop):
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 35af16fd4d9..f9aaf304c91 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -1871,7 +1871,7 @@ def test_repr_localcluster():
         assert cluster.scheduler.address in text
     finally:
         yield client.close()
-        yield cluster._close()
+        yield cluster.close()
 
 
 @gen_cluster(client=True)

From 9adcfdcc45d4119234216aefb1e81d4dba8b22ff Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 28 Dec 2018 10:38:31 -0800
Subject: [PATCH 0166/1550] Make the direct keyword docstring entries uniform
 [skip ci] (#2441)

---
 distributed/client.py | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 17485f0444b..018d8ca6113 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -489,8 +489,8 @@ class resembles executors in ``concurrent.futures`` but also allows
         Gives the client a name that will be included in logs generated on
         the scheduler for matters relating to this client
     direct_to_workers: bool (optional)
-        Can this client connect directly to workers or should it proxy through
-        the scheduler?
+        Whether or not to connect directly to the workers, or to ask
+        the scheduler to serve as intermediary.
     heartbeat_interval: int
         Time in milliseconds between heartbeats to scheduler
 
@@ -1609,6 +1609,10 @@ def gather(self, futures, errors='raise', maxsize=0, direct=None,
         errors: string
             Either 'raise' or 'skip' if we should raise if a future has erred
             or skip its inclusion in the output collection
+        direct: boolean
+            Whether or not to connect directly to the workers, or to ask
+            the scheduler to serve as intermediary.  This can also be set when
+            creating the Client.
         maxsize: int
             If the input is a queue then this produces an output queue with a
             maximum size.
@@ -1795,9 +1799,9 @@ def scatter(self, data, workers=None, broadcast=False, direct=None,
             Whether to send each data element to all workers.
             By default we round-robin based on number of cores.
         direct: bool (defaults to automatically check)
-            Send data directly to workers, bypassing the central scheduler
-            This avoids burdening the scheduler but assumes that the client is
-            able to talk directly with the workers.
+            Whether or not to connect directly to the workers, or to ask
+            the scheduler to serve as intermediary.  This can also be set when
+            creating the Client.
         maxsize: int (optional)
             Maximum size of queue if using queues, 0 implies infinite
         hash: bool (optional)
@@ -2295,7 +2299,9 @@ def get(self, dsk, keys, restrictions=None, loose_restrictions=None,
         sync: bool (optional)
             Returns Futures if False or concrete values if True (default).
         direct: bool
-            Gather results directly from workers
+            Whether or not to connect directly to the workers, or to ask
+            the scheduler to serve as intermediary.  This can also be set when
+            creating the Client.
 
         Examples
         --------

From 5655381ac570065ab1c742b88dae82f73b980ece Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 28 Dec 2018 10:39:20 -0800
Subject: [PATCH 0167/1550] Add optional environment variables to Nanny (#2431)

---
 distributed/nanny.py            | 16 +++++++++++-----
 distributed/tests/test_nanny.py | 10 ++++++++++
 2 files changed, 21 insertions(+), 5 deletions(-)

diff --git a/distributed/nanny.py b/distributed/nanny.py
index 27b56baccc0..b5769a25ffb 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -43,7 +43,7 @@ def __init__(self, scheduler_ip=None, scheduler_port=None,
             memory_limit='auto', reconnect=True, validate=False, quiet=False,
             resources=None, silence_logs=None, death_timeout=None, preload=(),
             preload_argv=[], security=None, contact_address=None,
-            listen_address=None, worker_class=None, **kwargs):
+            listen_address=None, worker_class=None, env=None, **kwargs):
 
         if scheduler_file:
             cfg = json_load_robust(scheduler_file)
@@ -63,6 +63,7 @@ def __init__(self, scheduler_ip=None, scheduler_port=None,
         self.preload = preload
         self.preload_argv = preload_argv
         self.Worker = Worker if worker_class is None else worker_class
+        self.env = env or {}
 
         self.contact_address = contact_address
         self.memory_terminate_fraction = dask.config.get('distributed.worker.memory.terminate')
@@ -215,7 +216,8 @@ def instantiate(self, comm=None):
                 worker_start_args=(start_arg,),
                 silence_logs=self.silence_logs,
                 on_exit=self._on_exit,
-                worker=self.Worker
+                worker=self.Worker,
+                env=self.env,
             )
 
         self.auto_restart = True
@@ -322,7 +324,7 @@ def _close(self, comm=None, timeout=5, report=None):
 class WorkerProcess(object):
 
     def __init__(self, worker_args, worker_kwargs, worker_start_args,
-                 silence_logs, on_exit, worker):
+                 silence_logs, on_exit, worker, env):
         self.status = 'init'
         self.silence_logs = silence_logs
         self.worker_args = worker_args
@@ -331,6 +333,7 @@ def __init__(self, worker_args, worker_kwargs, worker_start_args,
         self.on_exit = on_exit
         self.process = None
         self.Worker = worker
+        self.env = env
 
         # Initialized when worker is ready
         self.worker_dir = None
@@ -360,7 +363,9 @@ def start(self):
                         silence_logs=self.silence_logs,
                         init_result_q=self.init_result_q,
                         child_stop_q=self.child_stop_q,
-                        uid=uid, Worker=self.Worker),
+                        uid=uid,
+                        Worker=self.Worker,
+                        env=self.env),
         )
         self.process.daemon = True
         self.process.set_exit_callback(self._on_exit)
@@ -488,7 +493,8 @@ def _wait_until_connected(self, uid):
 
     @classmethod
     def _run(cls, worker_args, worker_kwargs, worker_start_args,
-             silence_logs, init_result_q, child_stop_q, uid, Worker):  # pragma: no cover
+             silence_logs, init_result_q, child_stop_q, uid, env, Worker):  # pragma: no cover
+        os.environ.update(env)
         try:
             from dask.multiprocessing import initialize_worker_process
         except ImportError:   # old Dask version
diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
index 03f29f9b884..c37bf056d34 100644
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -333,3 +333,13 @@ def test_wait_for_scheduler():
     log = log.getvalue()
     assert 'error' not in log.lower(), log
     assert 'restart' not in log.lower(), log
+
+
+@gen_cluster(ncores=[], client=True)
+def test_environment_variable(c, s):
+    a = Nanny(s.address, loop=s.loop, memory_limit=0, env={"FOO": "123"})
+    b = Nanny(s.address, loop=s.loop, memory_limit=0, env={"FOO": "456"})
+    yield [a._start(), b._start()]
+    results = yield c.run(lambda: os.environ['FOO'])
+    assert results == {a.worker_address: "123", b.worker_address: "456"}
+    yield [a._close(), b._close()]

From bcb4ffac2cd91742b748e730c99591dfb630efd8 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 31 Dec 2018 19:29:16 -0800
Subject: [PATCH 0168/1550] xfail test with resources and collections (#2445)

---
 distributed/tests/test_resources.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/distributed/tests/test_resources.py b/distributed/tests/test_resources.py
index 5fa8b1c8a34..f54dc602f14 100644
--- a/distributed/tests/test_resources.py
+++ b/distributed/tests/test_resources.py
@@ -262,6 +262,7 @@ def test_dont_optimize_out(c, s, a, b):
         assert 'executing' in str(a.story(key))
 
 
+@pytest.mark.xfail(reason="atop fusion seemed to break this")
 @gen_cluster(client=True, ncores=[('127.0.0.1', 1, {'resources': {'A': 1}}),
                                   ('127.0.0.1', 1, {'resources': {'B': 1}})])
 def test_full_collections(c, s, a, b):

From 54e17754ff671ac14dd2c00b09d6e434bb0c3e21 Mon Sep 17 00:00:00 2001
From: Dirk Petersen <dirkpetersen@users.noreply.github.com>
Date: Tue, 1 Jan 2019 16:36:20 -0800
Subject: [PATCH 0169/1550] Add short error message if bokeh cannot be imported
 (#2444)

Fixes https://github.com/dask/distributed/issues/2443
---
 distributed/cli/dask_scheduler.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/distributed/cli/dask_scheduler.py b/distributed/cli/dask_scheduler.py
index 9e01ad93712..f8ed240f16b 100755
--- a/distributed/cli/dask_scheduler.py
+++ b/distributed/cli/dask_scheduler.py
@@ -14,7 +14,7 @@
 
 from distributed import Scheduler
 from distributed.security import Security
-from distributed.utils import get_ip_interface, ignoring
+from distributed.utils import get_ip_interface
 from distributed.cli.utils import (check_python_3, install_signal_handlers,
                                    uri_from_host_port)
 from distributed.preloading import preload_modules, validate_preload_argv
@@ -118,10 +118,16 @@ def del_pid_file():
 
     services = {}
     if _bokeh:
-        with ignoring(ImportError):
+        try:
             from distributed.bokeh.scheduler import BokehScheduler
             services[('bokeh', bokeh_port)] = (BokehScheduler,
                                                {'prefix': bokeh_prefix})
+        except ImportError as error:
+            if str(error).startswith('No module named'):
+                logger.info('Web dashboard not loaded.  Unable to import bokeh')
+            else:
+                logger.info('Unable to import bokeh: %s' % str(error))
+
     scheduler = Scheduler(loop=loop, services=services,
                           scheduler_file=scheduler_file,
                           security=sec)

From a0d0ed21a87622ea35bfe1e686b52a8f221c5bca Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 3 Jan 2019 08:45:07 -0800
Subject: [PATCH 0170/1550] Clean up LocalCluster logging better in async mode
 (#2448)

---
 distributed/deploy/local.py      | 9 ++++++---
 distributed/tests/test_client.py | 9 ++++++---
 2 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index 4f6ebe12fd3..6d6b4b4199a 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -294,10 +294,13 @@ def close(self, timeout=20):
         try:
             result = self.sync(self._close, callback_timeout=timeout)
         except RuntimeError:  # IOLoop is closed
-            pass
+            result = None
 
-        with ignoring(AttributeError):
-            silence_logging(self._old_logging_level)
+        if hasattr(self, '_old_logging_level'):
+            if self.asynchronous:
+                result.add_done_callback(lambda _: silence_logging(self._old_logging_level))
+            else:
+                silence_logging(self._old_logging_level)
 
         if not self.asynchronous:
             self._loop_runner.stop()
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index f9aaf304c91..23a95eec74d 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -4560,7 +4560,6 @@ def test_fire_and_forget_err(c, s, a, b):
         assert time() < start + 1
 
 
-@pytest.mark.xfail(reason='Other tests bleed into the logs of this one')
 def test_quiet_client_close(loop):
     with captured_logger(logging.getLogger('distributed')) as logger:
         with Client(loop=loop, processes=False, threads_per_worker=4) as c:
@@ -4572,8 +4571,12 @@ def test_quiet_client_close(loop):
         lines = out.strip().split('\n')
         assert len(lines) <= 2
         for line in lines:
-            assert not line or 'Reconnecting' in line or set(line) == {'-'}
-        # assert not out
+            assert (
+                not line or
+                'Reconnecting' in line or
+                'garbage' in line or
+                set(line) == {'-'}
+            ), line
 
 
 @gen_cluster()

From 4e38022ed91b7d90ffe54703e9975d94a37fb9c3 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 4 Jan 2019 14:14:00 -0800
Subject: [PATCH 0171/1550] bump version to 1.25.2

---
 docs/source/changelog.rst | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index ed82e098e18..cc029317e5e 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,21 @@
 Changelog
 =========
 
+1.25.2 - 2019-01-04
+-------------------
+
+-  Clean up LocalCluster logging better in async mode (:pr:`2448`) `Matthew Rocklin`_
+-  Add short error message if bokeh cannot be imported (:pr:`2444`) `Dirk Petersen`_
+-  Add optional environment variables to Nanny (:pr:`2431`) `Matthew Rocklin`_
+-  Make the direct keyword docstring entries uniform (:pr:`2441`) `Matthew Rocklin`_
+-  Make LocalCluster.close async friendly (:pr:`2437`) `Matthew Rocklin`_
+-  gather_dep: don't request dependencies we already found out we don't want (:pr:`2428`) `tjb900`_
+-  Add parameters to Client.run docstring (:pr:`2429`) `Matthew Rocklin`_
+-  Support coroutines and async-def functions in run/run_scheduler (:pr:`2427`) `Matthew Rocklin`_
+-  Name threads in ThreadPoolExecutors (:pr:`2408`) `Matthew Rocklin`_
+
+
+
 1.25.1 - 2018-12-15
 -------------------
 
@@ -880,3 +895,5 @@ significantly without many new features.
 .. _`Diane Trout`: https://github.com/detrout
 .. _`tjb900`: https://github.com/tjb900
 .. _`Stephan Hoyer`: https://github.com/shoyer
+.. _`tjb900`: https://github.com/tjb900
+.. _`Dirk Petersen`: https://github.com/dirkpetersen

From 70c5129428e0352d322c74dcc83a0c09fcd9ce39 Mon Sep 17 00:00:00 2001
From: Daniel Farrell <danpf@uw.edu>
Date: Tue, 8 Jan 2019 11:48:32 -0800
Subject: [PATCH 0172/1550] Fix excess threading on missing connections (#2403)

---
 distributed/comm/core.py             |  1 +
 distributed/comm/tcp.py              | 13 +++++++--
 distributed/comm/tests/test_comms.py | 41 ++++++++++++++++++++++++++++
 3 files changed, 53 insertions(+), 2 deletions(-)

diff --git a/distributed/comm/core.py b/distributed/comm/core.py
index 82f2965d8ec..31f35cfc6f1 100644
--- a/distributed/comm/core.py
+++ b/distributed/comm/core.py
@@ -181,6 +181,7 @@ def _raise(error):
                % (addr, timeout, error))
         raise IOError(msg)
 
+    # This starts a thread
     while True:
         try:
             future = connector.connect(loc, deserialize=deserialize,
diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py
index a838b08f360..fb0aad15106 100644
--- a/distributed/comm/tcp.py
+++ b/distributed/comm/tcp.py
@@ -11,6 +11,8 @@
 except ImportError:
     ssl = None
 
+from concurrent.futures import ThreadPoolExecutor
+
 import dask
 import tornado
 from tornado import gen, netutil
@@ -320,6 +322,13 @@ def _check_encryption(self, address, connection_args):
 
 
 class BaseTCPConnector(Connector, RequireEncryptionMixin):
+    if PY3:  # see github PR #2403 discussion for more info
+        _executor = ThreadPoolExecutor(2)
+        _resolver = netutil.ExecutorResolver(close_executor=False,
+                                             executor=_executor)
+    else:
+        _resolver = None
+    client = TCPClient(resolver=_resolver)
 
     @gen.coroutine
     def connect(self, address, deserialize=True, **connection_args):
@@ -327,11 +336,11 @@ def connect(self, address, deserialize=True, **connection_args):
         ip, port = parse_host_port(address)
         kwargs = self._get_connect_args(**connection_args)
 
-        client = TCPClient()
         try:
-            stream = yield client.connect(ip, port,
+            stream = yield BaseTCPConnector.client.connect(ip, port,
                                           max_buffer_size=MAX_BUFFER_SIZE,
                                           **kwargs)
+
             # Under certain circumstances tornado will have a closed connnection with an error and not raise
             # a StreamClosedError.
             #
diff --git a/distributed/comm/tests/test_comms.py b/distributed/comm/tests/test_comms.py
index e7927593f40..0cfeb7a63a8 100644
--- a/distributed/comm/tests/test_comms.py
+++ b/distributed/comm/tests/test_comms.py
@@ -11,6 +11,7 @@
 from tornado import gen, ioloop, locks, queues
 from tornado.concurrent import Future
 
+from distributed.compatibility import PY3
 from distributed.metrics import time
 from distributed.utils import get_ip, get_ipv6
 from distributed.utils_test import (gen_test, requires_ipv6, has_ipv6,
@@ -289,6 +290,46 @@ def client_communicate(key, delay=0):
     assert set(l) == {1234} | set(range(N))
 
 
+@gen_test()
+def test_comm_failure_threading():
+    """
+    When we fail to connect, make sure we don't make a lot
+    of threads.
+
+    We only assert for PY3, because the thread limit only is
+    set for python 3.  See github PR #2403 discussion for info.
+    """
+
+    @gen.coroutine
+    def sleep_for_60ms():
+        max_thread_count = 0
+        for x in range(60):
+            yield gen.sleep(0.001)
+            thread_count = threading.active_count()
+            if thread_count > max_thread_count:
+                max_thread_count = thread_count
+        raise gen.Return(max_thread_count)
+    original_thread_count = threading.active_count()
+
+    # tcp.TCPConnector()
+    sleep_future = sleep_for_60ms()
+    with pytest.raises(IOError):
+        yield connect("tcp://localhost:28400", 0.052)
+    max_thread_count = yield sleep_future
+    # 2 is the number set by BaseTCPConnector.executor (ThreadPoolExecutor)
+    if PY3:
+        assert max_thread_count <= 2 + original_thread_count
+
+    # tcp.TLSConnector()
+    sleep_future = sleep_for_60ms()
+    with pytest.raises(IOError):
+        yield connect("tls://localhost:28400", 0.052,
+                                 connection_args={'ssl_context': get_client_ssl_context()})
+    max_thread_count = yield sleep_future
+    if PY3:
+        assert max_thread_count <= 2 + original_thread_count
+
+
 @gen.coroutine
 def check_inproc_specific(run_client):
     """

From cc1746457497d5f4ad4bd34709a976a360552ee3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lo=C3=AFc=20Est=C3=A8ve?= <loic.esteve@ymail.com>
Date: Fri, 11 Jan 2019 16:20:27 +0100
Subject: [PATCH 0173/1550] Fix typo in doc (#2457)

---
 docs/source/resources.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/source/resources.rst b/docs/source/resources.rst
index 5d501ce4bd2..87040ef0873 100644
--- a/docs/source/resources.rst
+++ b/docs/source/resources.rst
@@ -71,7 +71,7 @@ resource requirements during compute or persist calls.
 
     x = dd.read_csv(...)
     y = x.map_partitions(func1)
-    z = y.map_parititons(func2)
+    z = y.map_partitions(func2)
 
     z.compute(resources={tuple(y.__dask_keys__()): {'GPU': 1})
 

From dfad76f7277e0c732883c36e630d627149392420 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 14 Jan 2019 12:15:16 -0800
Subject: [PATCH 0174/1550] Start fewer but larger workers with LocalCluster
 (#2452)

When we have a large number of available cores we should set the default number
of threads per process to be higher than one.

This implements a policy that aims for a number of processes equal to the
square root of the number of cores, at least above a certain amount of cores.

Partially addresses https://github.com/dask/distributed/issues/2450
---
 distributed/deploy/local.py            | 32 ++++++++++++++++++++++++--
 distributed/deploy/tests/test_local.py | 15 +++++++++++-
 2 files changed, 44 insertions(+), 3 deletions(-)

diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index 6d6b4b4199a..c467acea84e 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -8,6 +8,7 @@
 import weakref
 import toolz
 
+from dask.utils import factors
 from tornado import gen
 
 from .cluster import Cluster
@@ -99,8 +100,7 @@ def __init__(self, n_workers=None, threads_per_worker=None, processes=True,
             self._old_logging_level = silence_logging(level=silence_logs)
         if n_workers is None and threads_per_worker is None:
             if processes:
-                n_workers = _ncores
-                threads_per_worker = 1
+                n_workers, threads_per_worker = nprocesses_nthreads(_ncores)
             else:
                 n_workers = 1
                 threads_per_worker = _ncores
@@ -372,6 +372,34 @@ def scheduler_address(self):
             return '<unstarted>'
 
 
+def nprocesses_nthreads(n):
+    """
+    The default breakdown of processes and threads for a given number of cores
+
+    Parameters
+    ----------
+    n: int
+        Number of available cores
+
+    Examples
+    --------
+    >>> nprocesses_nthreads(4)
+    (4, 1)
+    >>> nprocesses_nthreads(32)
+    (8, 4)
+
+    Returns
+    -------
+    nprocesses, nthreads
+    """
+    if n <= 4:
+        processes = n
+    else:
+        processes = min(f for f in factors(n) if f >= math.sqrt(n))
+    threads = n // processes
+    return (processes, threads)
+
+
 clusters_to_close = weakref.WeakSet()
 
 
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index ee586c5be4a..224906961eb 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -14,7 +14,7 @@
 import pytest
 
 from distributed import Client, Worker, Nanny
-from distributed.deploy.local import LocalCluster
+from distributed.deploy.local import LocalCluster, nprocesses_nthreads
 from distributed.metrics import time
 from distributed.utils_test import (inc, gen_test, slowinc,
                                     assert_cannot_connect,
@@ -529,5 +529,18 @@ def test_local_tls_restart(loop):
             assert workers_before != workers_after
 
 
+def test_default_process_thread_breakdown():
+    assert nprocesses_nthreads(1) == (1, 1)
+    assert nprocesses_nthreads(4) == (4, 1)
+    assert nprocesses_nthreads(5) == (5, 1)
+    assert nprocesses_nthreads(8) == (4, 2)
+    assert nprocesses_nthreads(12) in ((6, 2), (4, 3))
+    assert nprocesses_nthreads(20) == (5, 4)
+    assert nprocesses_nthreads(24) in ((6, 4), (8, 3))
+    assert nprocesses_nthreads(32) == (8, 4)
+    assert nprocesses_nthreads(40) in ((8, 5), (10, 4))
+    assert nprocesses_nthreads(80) in ((10, 8), (16, 5))
+
+
 if sys.version_info >= (3, 5):
     from distributed.deploy.tests.py3_test_deploy import *  # noqa F401

From 4fef1adb47c7f79afcd26e3bf91727be310fd7f5 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Wed, 16 Jan 2019 10:03:09 -0500
Subject: [PATCH 0175/1550] Check for non-zero `length` first in `read` loop
 (#2465)

Since both the Python 2 and Python 3 cases need to handle the case of
zero and non-zero length separately, pull this check outside of the
Python 2 or 3 version checks to simplify the code a bit.
---
 distributed/comm/tcp.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py
index fb0aad15106..5af63b164a1 100644
--- a/distributed/comm/tcp.py
+++ b/distributed/comm/tcp.py
@@ -186,16 +186,15 @@ def read(self, deserializers=None):
 
             frames = []
             for length in lengths:
-                if PY3 and self._iostream_has_read_into:
-                    frame = bytearray(length)
-                    if length:
+                if length:
+                    if PY3 and self._iostream_has_read_into:
+                        frame = bytearray(length)
                         n = yield stream.read_into(frame)
                         assert n == length, (n, length)
-                else:
-                    if length:
-                        frame = yield stream.read_bytes(length)
                     else:
-                        frame = b''
+                        frame = yield stream.read_bytes(length)
+                else:
+                    frame = b''
                 frames.append(frame)
         except StreamClosedError as e:
             self.stream = None

From b818d788deea53b9d6a5fff332d92dcda3084b76 Mon Sep 17 00:00:00 2001
From: Peter Killick <peter.killick@informaticslab.co.uk>
Date: Wed, 16 Jan 2019 15:05:58 +0000
Subject: [PATCH 0176/1550] DOC: Use of local cluster in script (#2462)

---
 docs/source/local-cluster.rst | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/docs/source/local-cluster.rst b/docs/source/local-cluster.rst
index 56d5c6e0b4e..c415bbfbeba 100644
--- a/docs/source/local-cluster.rst
+++ b/docs/source/local-cluster.rst
@@ -28,6 +28,18 @@ an ``Client`` with no arguments:
    >>> client
    <Client: scheduler=127.0.0.1:8786 processes=8 cores=8>
 
+.. note::
+
+   Within a Python script you need to start a local cluster in the
+   ``if __name__ == '__main__'`` block:
+
+   .. code-block:: python
+
+      if __name__ == '__main__':
+          cluster = LocalCluster()
+          client = Client(cluster)
+          # Your code follows here
+
 API
 ---
 

From 90758dcd79182ff7ce0cce4b4c29d46316349836 Mon Sep 17 00:00:00 2001
From: Tom Augspurger <TomAugspurger@users.noreply.github.com>
Date: Mon, 21 Jan 2019 11:59:02 -0600
Subject: [PATCH 0177/1550] DOC/API: Signature for base class write / read
 (#2472)

https://github.com/dask/distributed/blob/b818d788deea53b9d6a5fff332d92dcda3084b76/distributed/core.py#L455-L459
will unconditionally pass these arguments, so they should likely be
present in the base class's signature. I think they're generally good to
support if possible.
---
 distributed/comm/core.py          | 17 +++++++++++++++--
 distributed/protocol/serialize.py |  3 +++
 docs/source/serialization.rst     |  2 ++
 3 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/distributed/comm/core.py b/distributed/comm/core.py
index 31f35cfc6f1..a7aaf7217db 100644
--- a/distributed/comm/core.py
+++ b/distributed/comm/core.py
@@ -40,19 +40,32 @@ class Comm(with_metaclass(ABCMeta)):
     # XXX add set_close_callback()?
 
     @abstractmethod
-    def read(self):
+    def read(self, deserializers=None):
         """
         Read and return a message (a Python object).
 
         This method is a coroutine.
+
+        Parameters
+        ----------
+        deserializers : Optional[Dict[str, Tuple[Callable, Callable, bool]]]
+            An optional dict appropriate for distributed.protocol.deserialize.
+            See :ref:`serialization` for more.
         """
 
     @abstractmethod
-    def write(self, msg):
+    def write(self, msg, on_error=None):
         """
         Write a message (a Python object).
 
         This method is a coroutine.
+
+        Parameters
+        ----------
+        msg :
+        on_error : Optional[str]
+            The behavior when serialization fails. See
+            ``distributed.protocol.core.dumps`` for valid values.
         """
 
     @abstractmethod
diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index 417aa43484f..d81a307a0f9 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -171,6 +171,9 @@ def deserialize(header, frames, deserializers=None):
     ----------
     header: dict
     frames: list of bytes
+    deserializers : Optional[Dict[str, Tuple[Callable, Callable, bool]]]
+        An optional dict mapping a name to a (de)serializer.
+        See `dask_serialize` and `dask_deserialize` for more.
 
     See Also
     --------
diff --git a/docs/source/serialization.rst b/docs/source/serialization.rst
index 5cfd1515684..25b2ae49476 100644
--- a/docs/source/serialization.rst
+++ b/docs/source/serialization.rst
@@ -1,3 +1,5 @@
+.. _serialization:
+
 Serialization
 =============
 

From f434cc49b4629ec077b670c34ec5edcf12b958f7 Mon Sep 17 00:00:00 2001
From: Adam Beberg <beberg@mithral.com>
Date: Thu, 24 Jan 2019 14:23:47 -0800
Subject: [PATCH 0178/1550] Support Pytest 4 in Tests (#2478)

---
 .gitignore                               | 4 ++++
 continuous_integration/travis/install.sh | 2 +-
 distributed/tests/test_counter.py        | 8 +++++---
 distributed/tests/test_resources.py      | 6 ++++--
 distributed/tests/test_steal.py          | 9 +++------
 5 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/.gitignore b/.gitignore
index 7888407d33f..a3a40e19289 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,11 +1,15 @@
 *.pyc
 *.py~
 build/
+dist/
 *.egg-info
 docs/build
 continuous_integration/hdfs-initialized
+.DS_Store
 .cache
+*.lock
 .#*
 .idea/
 .pytest_cache/
 dask-worker-space/
+.vscode/
diff --git a/continuous_integration/travis/install.sh b/continuous_integration/travis/install.sh
index 4954f582f1c..dbd3d8e043b 100644
--- a/continuous_integration/travis/install.sh
+++ b/continuous_integration/travis/install.sh
@@ -43,7 +43,7 @@ conda install -q \
     netcdf4 \
     paramiko \
     psutil \
-    pytest=3.7 \
+    pytest \
     pytest-timeout \
     python=$PYTHON \
     requests \
diff --git a/distributed/tests/test_counter.py b/distributed/tests/test_counter.py
index 66aab4d766f..43b5e4d022c 100644
--- a/distributed/tests/test_counter.py
+++ b/distributed/tests/test_counter.py
@@ -11,9 +11,11 @@
     Digest = None
 
 
-@pytest.mark.parametrize('CD,size', [(Counter, lambda d: sum(d.values())),
-                                     pytest.mark.skipif(not Digest, reason="no crick library")(
-                                     (Digest, lambda x: x.size()))])
+@pytest.mark.parametrize('CD,size', [
+    (Counter, lambda d: sum(d.values())),
+    pytest.param(Digest, lambda x: x.size(),
+                 marks=pytest.mark.skipif(not Digest, reason="no crick library"))
+])
 def test_digest(loop, CD, size):
     c = CD(loop=loop)
     c.add(1)
diff --git a/distributed/tests/test_resources.py b/distributed/tests/test_resources.py
index f54dc602f14..bcec6d1604c 100644
--- a/distributed/tests/test_resources.py
+++ b/distributed/tests/test_resources.py
@@ -277,8 +277,10 @@ def test_full_collections(c, s, a, b):
 
 
 @pytest.mark.parametrize('optimize_graph', [
-    pytest.mark.xfail(True, reason="don't track resources through optimization"),
-    pytest.mark.skipif(WINDOWS, False, reason="intermittent failure"),
+    pytest.param(True,
+        marks=pytest.mark.xfail(reason="don't track resources through optimization")),
+    pytest.param(False,
+        marks=pytest.mark.skipif(WINDOWS, reason="intermittent failure"))
 ])
 def test_collections_get(client, optimize_graph, s, a, b):
     da = pytest.importorskip('dask.array')
diff --git a/distributed/tests/test_steal.py b/distributed/tests/test_steal.py
index 35315d531ca..cd76908ead2 100644
--- a/distributed/tests/test_steal.py
+++ b/distributed/tests/test_steal.py
@@ -468,12 +468,9 @@ def assert_balanced(inp, expected, c, s, *workers):
       [1],
       [1]]),
 
-    pytest.mark.xfail(([[1, 1, 1, 1, 1, 1, 1],
-      [1, 1], [1, 1], [1, 1],
-      []],
-     [[1, 1, 1, 1, 1],
-      [1, 1], [1, 1], [1, 1],
-      [1, 1]]), reason="Some uncertainty based on executing stolen task")
+    pytest.param([[1, 1, 1, 1, 1, 1, 1], [1, 1], [1, 1], [1, 1], []],
+                 [[1, 1, 1, 1, 1], [1, 1], [1, 1], [1, 1], [1, 1]],
+                 marks=pytest.mark.xfail(reason="Some uncertainty based on executing stolen task"))
 ])
 def test_balance(inp, expected):
     test = lambda *args, **kwargs: assert_balanced(inp, expected, *args, **kwargs)

From b18a6219b7d1ef8c1933b12858541a968d0d3684 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sun, 27 Jan 2019 17:58:04 -0800
Subject: [PATCH 0179/1550] Ensure async behavior in event loop with
 LocalCluster (#2484)

---
 distributed/deploy/local.py            |  7 ++++++-
 distributed/deploy/tests/test_local.py | 11 +++++++++++
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index c467acea84e..72a12cfaced 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -12,6 +12,7 @@
 from tornado import gen
 
 from .cluster import Cluster
+from ..compatibility import get_thread_identity
 from ..core import CommClosedError
 from ..utils import (sync, ignoring, All, silence_logging, LoopRunner,
         log_errors, thread_state, parse_timedelta)
@@ -153,7 +154,11 @@ def __await__(self):
 
     @property
     def asynchronous(self):
-        return self._asynchronous or getattr(thread_state, 'asynchronous', False)
+        return (
+            self._asynchronous or
+            getattr(thread_state, 'asynchronous', False) or
+            hasattr(self.loop, '_thread_identity') and self.loop._thread_identity == get_thread_identity()
+        )
 
     def sync(self, func, *args, **kwargs):
         if kwargs.pop('asynchronous', None) or self.asynchronous:
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index 224906961eb..758b27b334f 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -542,5 +542,16 @@ def test_default_process_thread_breakdown():
     assert nprocesses_nthreads(80) in ((10, 8), (16, 5))
 
 
+def test_asynchronous_property(loop):
+    with LocalCluster(4, scheduler_port=0, processes=False, silence_logs=False,
+                      diagnostics_port=None, loop=loop) as cluster:
+
+        @gen.coroutine
+        def _():
+            assert cluster.asynchronous
+
+        cluster.sync(_)
+
+
 if sys.version_info >= (3, 5):
     from distributed.deploy.tests.py3_test_deploy import *  # noqa F401

From 51f5c6f68f8c8bc7c3987bdbe1b363a0170c6b54 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lo=C3=AFc=20Est=C3=A8ve?= <loic.esteve@ymail.com>
Date: Mon, 28 Jan 2019 17:49:29 +0100
Subject: [PATCH 0180/1550] Fix spurious CancelledError (#2485)

---
 distributed/client.py            |  3 ++-
 distributed/tests/test_client.py | 16 ++++++++++++++++
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/distributed/client.py b/distributed/client.py
index 018d8ca6113..061bffe59ca 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -1090,7 +1090,8 @@ def _close(self, fast=False):
             # This makes the shutdown slightly smoother and quieter
             with ignoring(AttributeError, gen.TimeoutError):
                 yield gen.with_timeout(timedelta(milliseconds=100),
-                                       self._handle_scheduler_coroutine)
+                                       self._handle_scheduler_coroutine,
+                                       quiet_exceptions=(CancelledError,))
 
             if self.scheduler_comm and self.scheduler_comm.comm and not self.scheduler_comm.comm.closed():
                 yield self.scheduler_comm.close()
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 23a95eec74d..ea8dba50951 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -4579,6 +4579,22 @@ def test_quiet_client_close(loop):
             ), line
 
 
+def test_quiet_client_close_when_cluster_is_closed_before_client(loop):
+    n_attempts = 5
+    # Trying a few times to reduce the flakiness of the test. Without the bug
+    # fix in #2477 and with 5 attempts, this test passes by chance in about 10%
+    # of the cases.
+    for _ in range(n_attempts):
+        with captured_logger(logging.getLogger('tornado.application')) as logger:
+            cluster = LocalCluster(loop=loop)
+            client = Client(cluster, loop=loop)
+            cluster.close()
+            client.close()
+
+        out = logger.getvalue()
+        assert 'CancelledError' not in out
+
+
 @gen_cluster()
 def test_close(s, a, b):
     c = yield Client(s.address, asynchronous=True)

From b08622b53a699ad39e6efa7d5111dada1598d4fa Mon Sep 17 00:00:00 2001
From: George Sakkis <george.sakkis@gmail.com>
Date: Mon, 28 Jan 2019 19:11:57 +0200
Subject: [PATCH 0181/1550] Properly reset dask.config scheduler and shuffle
 when closing the client (#2475)

---
 distributed/client.py            | 20 +++++---------------
 distributed/tests/test_client.py |  8 ++++----
 2 files changed, 9 insertions(+), 19 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 061bffe59ca..c24790028c2 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -606,11 +606,8 @@ def __init__(self, address=None, loop=None, timeout=no_default,
 
         self._start_arg = address
         if set_as_default:
-            self._previous_scheduler = dask.config.get('scheduler', None)
-            dask.config.set(scheduler='dask.distributed')
-
-            self._previous_shuffle = dask.config.get('shuffle', None)
-            dask.config.set(shuffle='tasks')
+            self._set_config = dask.config.set(scheduler='dask.distributed',
+                                               shuffle='tasks')
 
         self._stream_handlers = {
             'key-in-memory': self._handle_key_in_memory,
@@ -1074,9 +1071,9 @@ def _close(self, fast=False):
                 pc.stop()
             self._scheduler_identity = {}
             with ignoring(AttributeError):
-                dask.config.set(scheduler=self._previous_scheduler)
-            with ignoring(AttributeError):
-                dask.config.set(shuffle=self._previous_shuffle)
+                # clear the dask.config set keys
+                with self._set_config:
+                    pass
             if self.get == dask.config.get('get', None):
                 del dask.config.config['get']
             if self.status == 'closed':
@@ -1160,13 +1157,6 @@ def close(self, timeout=no_default):
         if self._should_close_loop and not shutting_down():
             self._loop_runner.stop()
 
-        with ignoring(AttributeError):
-            dask.config.set(scheduler=self._previous_scheduler)
-        with ignoring(AttributeError):
-            dask.config.set(shuffle=self._previous_shuffle)
-        if self.get == dask.config.get('get', None):
-            del dask.config.config['get']
-
     def shutdown(self, *args, **kwargs):
         """ Deprecated, see close instead
 
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index ea8dba50951..96ae9aa85fb 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -3243,17 +3243,17 @@ def test_cancel_clears_processing(c, s, *workers):
 def test_default_get():
     with cluster() as (s, [a, b]):
         pre_get = dask.base.get_scheduler()
-        pre_shuffle = dask.config.get('shuffle', None)
+        pytest.raises(KeyError, dask.config.get, 'shuffle')
         with Client(s['address'], set_as_default=True) as c:
             assert dask.base.get_scheduler() == c.get
             assert dask.config.get('shuffle') == 'tasks'
 
         assert dask.base.get_scheduler() == pre_get
-        assert dask.config.get('shuffle') == pre_shuffle
+        pytest.raises(KeyError, dask.config.get, 'shuffle')
 
         c = Client(s['address'], set_as_default=False)
         assert dask.base.get_scheduler() == pre_get
-        assert dask.config.get('shuffle') == pre_shuffle
+        pytest.raises(KeyError, dask.config.get, 'shuffle')
         c.close()
 
         c = Client(s['address'], set_as_default=True)
@@ -3261,7 +3261,7 @@ def test_default_get():
         assert dask.base.get_scheduler() == c.get
         c.close()
         assert dask.base.get_scheduler() == pre_get
-        assert dask.config.get('shuffle') == pre_shuffle
+        pytest.raises(KeyError, dask.config.get, 'shuffle')
 
         with Client(s['address']) as c:
             assert dask.base.get_scheduler() == c.get

From 1c6b2e84e7751cf1d65e3761969a9c7ffa3feb69 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lo=C3=AFc=20Est=C3=A8ve?= <loic.esteve@ymail.com>
Date: Mon, 28 Jan 2019 18:12:53 +0100
Subject: [PATCH 0182/1550] Make it more explict that resources are per worker.
 (#2470)

---
 distributed/cli/dask_worker.py |  4 +++-
 docs/source/resources.rst      | 28 ++++++++++++++++++++++++++++
 2 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
index 645869fc0eb..aa6c5f8a087 100755
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -82,7 +82,9 @@
 @click.option('--local-directory', default='', type=str,
               help="Directory to place worker files")
 @click.option('--resources', type=str, default='',
-              help='Resources for task constraints like "GPU=2 MEM=10e9"')
+              help='Resources for task constraints like "GPU=2 MEM=10e9". '
+                   'Resources are applied separately to each worker process '
+                   "(only relevant when starting multiple worker processes with '--nprocs').")
 @click.option('--scheduler-file', type=str, default='',
               help='Filename to JSON encoded scheduler information. '
                    'Use with dask-scheduler --scheduler-file')
diff --git a/docs/source/resources.rst b/docs/source/resources.rst
index 87040ef0873..bd4d8b5e81c 100644
--- a/docs/source/resources.rst
+++ b/docs/source/resources.rst
@@ -48,6 +48,34 @@ When we submit tasks to the cluster we specify constraints per task
    final = client.submit(aggregate, processed, resources={'MEMORY': 70e9})
 
 
+Resources are applied separately to each worker process
+-------------------------------------------------------
+
+If you are using ``dask-worker --nprocs <nprocs>`` the resource will be applied
+separately to each of the ``nprocs`` worker processes. Suppose you have 2 GPUs
+on your machine, if you want to use two worker processes, you have 1 GPU per
+worker process so you need to do something like this::
+
+   dask-worker scheduler:8786 --nprocs 2 --resources "GPU=1"
+
+Here is an example that illustrates how to use resources to ensure each task is
+run inside a separate process, which is useful to execute non thread-safe tasks
+or tasks that uses multithreading internally::
+
+   dask-worker scheduler:8786 --nprocs 3 --nthreads 2 --resources "process=1"
+
+With the code below, there will be at most 3 tasks running concurrently and
+each task will run in a separate process:
+
+.. code-block:: python
+
+   from distributed import Client
+   client = Client('scheduler:8786')
+
+   futures = [client.submit(non_thread_safe_function, arg,
+                            resources={'process': 1}) for arg in args]
+
+
 Resources are Abstract
 ----------------------
 

From 08f253d5f9c8f83e10e8613892f36be60a91e2f8 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 29 Jan 2019 15:02:12 -0800
Subject: [PATCH 0183/1550] Remove references to center (#2488)

---
 distributed/scheduler.py |  1 -
 docs/source/setup.rst    | 12 ++++++------
 2 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index ee24e0182de..ea46aa7e00c 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -764,7 +764,6 @@ class Scheduler(ServerNode):
 
     def __init__(
             self,
-            center=None,
             loop=None,
             delete_interval='500ms',
             synchronize_worker_interval='60s',
diff --git a/docs/source/setup.rst b/docs/source/setup.rst
index d872c7e04c0..f1158712901 100644
--- a/docs/source/setup.rst
+++ b/docs/source/setup.rst
@@ -21,16 +21,16 @@ Then launch ``dask-worker`` on the rest of the nodes, providing the address to t
 node that hosts ``dask-scheduler``::
 
    $ dask-worker 192.168.0.1:8786
-   Start worker at:            192.168.0.2:12345
-   Registered with center at:  192.168.0.1:8786
+   Start worker at:               192.168.0.2:12345
+   Registered with Scheduler at:  192.168.0.1:8786
 
    $ dask-worker 192.168.0.1:8786
-   Start worker at:            192.168.0.3:12346
-   Registered with center at:  192.168.0.1:8786
+   Start worker at:               192.168.0.3:12346
+   Registered with Scheduler at:  192.168.0.1:8786
 
    $ dask-worker 192.168.0.1:8786
-   Start worker at:            192.168.0.4:12347
-   Registered with center at:  192.168.0.1:8786
+   Start worker at:               192.168.0.4:12347
+   Registered with Scheduler at:  192.168.0.1:8786
 
 There are various mechanisms to deploy these executables on a cluster, ranging
 from manualy SSH-ing into all of the nodes to more automated systems like

From 87fd71675dd7b76cd0d2b0052372e0b2afd064f0 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 29 Jan 2019 21:11:17 -0800
Subject: [PATCH 0184/1550] Expand client clearing timeout to 10s in testing
 (#2493)

Appveyor can be very very slow sometimes
---
 distributed/utils_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index c586e14eb9a..1f1abd8b1f1 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -136,7 +136,7 @@ def start():
     start = time()
     while set(_global_clients):
         sleep(0.1)
-        assert time() < start + 5
+        assert time() < start + 10
 
     _cleanup_dangling()
 

From 45eb7c1d15cfd160349937deb17759a79aa753d1 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 29 Jan 2019 21:11:25 -0800
Subject: [PATCH 0185/1550] Propagate key keyword in progressbar (#2492)

---
 distributed/diagnostics/progress.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/diagnostics/progress.py b/distributed/diagnostics/progress.py
index 999f859b855..5a9fe7f083e 100644
--- a/distributed/diagnostics/progress.py
+++ b/distributed/diagnostics/progress.py
@@ -116,7 +116,7 @@ def transition(self, key, start, finish, *args, **kwargs):
 
         if key in self.keys and finish == 'forgotten':
             logger.debug("A task was cancelled (%s), stopping progress", key)
-            self.stop(exception=True)
+            self.stop(exception=True, key=key)
 
     def restart(self, scheduler):
         self.stop()

From bb5bac0a4955d1ded068763d97cc4af2e1e121bf Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 29 Jan 2019 21:49:02 -0800
Subject: [PATCH 0186/1550] Use provided cluster's IOLoop if present in Client
 (#2494)

---
 distributed/client.py                  | 26 ++++++++++++++------------
 distributed/deploy/tests/test_local.py | 10 ++++++----
 2 files changed, 20 insertions(+), 16 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index c24790028c2..e4a5cb3759b 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -572,6 +572,20 @@ def __init__(self, address=None, loop=None, timeout=no_default,
         else:
             self.connection_args = self.security.get_connection_args('client')
 
+        if address is None:
+            address = dask.config.get('scheduler-address', None)
+            if address:
+                logger.info("Config value `scheduler-address` found: %s",
+                            address)
+
+        if isinstance(address, (rpc, PooledRPCCall)):
+            self.scheduler = address
+        elif hasattr(address, "scheduler_address"):
+            # It's a LocalCluster or LocalCluster-compatible object
+            self.cluster = address
+            with ignoring(AttributeError):
+                loop = address.loop
+
         self._connecting_to_scheduler = False
         self._asynchronous = asynchronous
         self._should_close_loop = not loop
@@ -592,18 +606,6 @@ def __init__(self, address=None, loop=None, timeout=no_default,
                 io_loop=self.loop
         )
 
-        if address is None:
-            address = dask.config.get('scheduler-address', None)
-            if address:
-                logger.info("Config value `scheduler-address` found: %s",
-                            address)
-
-        if isinstance(address, (rpc, PooledRPCCall)):
-            self.scheduler = address
-        elif hasattr(address, "scheduler_address"):
-            # It's a LocalCluster or LocalCluster-compatible object
-            self.cluster = address
-
         self._start_arg = address
         if set_as_default:
             self._set_config = dask.config.set(scheduler='dask.distributed',
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index 758b27b334f..26d3d28784a 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -32,12 +32,14 @@
 def test_simple(loop):
     with LocalCluster(4, scheduler_port=0, processes=False, silence_logs=False,
                       diagnostics_port=None, loop=loop) as c:
-        with Client(c.scheduler_address, loop=loop) as e:
+        with Client(c) as e:
             x = e.submit(inc, 1)
             x.result()
             assert x.key in c.scheduler.tasks
             assert any(w.data == {x.key: 2} for w in c.workers)
 
+            assert e.loop is c.loop
+
 
 @pytest.mark.skipif('sys.version_info[0] == 2', reason='fork issues')
 def test_close_twice():
@@ -131,7 +133,7 @@ class LocalTest(ClusterTest, unittest.TestCase):
 def test_Client_with_local(loop):
     with LocalCluster(1, scheduler_port=0, silence_logs=False,
                       diagnostics_port=None, loop=loop) as c:
-        with Client(c, loop=loop) as e:
+        with Client(c) as e:
             assert len(e.ncores()) == len(c.workers)
             assert c.scheduler_address in repr(c)
 
@@ -276,7 +278,7 @@ def test_scale_up_and_down():
     cluster = yield LocalCluster(0, scheduler_port=0, processes=False,
                                  silence_logs=False, diagnostics_port=None,
                                  loop=loop, asynchronous=True)
-    c = yield Client(cluster, loop=loop, asynchronous=True)
+    c = yield Client(cluster, asynchronous=True)
 
     assert not cluster.workers
 
@@ -492,7 +494,7 @@ def scale_down(self, *args, **kwargs):
     cluster = yield MyCluster(0, scheduler_port=0, processes=False,
                               silence_logs=False, diagnostics_port=None,
                               loop=loop, asynchronous=True)
-    c = yield Client(cluster, loop=loop, asynchronous=True)
+    c = yield Client(cluster, asynchronous=True)
 
     assert not cluster.workers
 

From 163731b1eb2cf4131943722f513a6087ccc8da5e Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 31 Jan 2019 07:43:41 -0800
Subject: [PATCH 0187/1550] Add client.profile(filename='myfile.html') option
 (#2482)

This allows users to save profile results directly to a sharable file
---
 distributed/bokeh/components.py        | 127 +------------------------
 distributed/client.py                  |  40 +++++++-
 distributed/profile.py                 |  56 +++++++++++
 distributed/tests/test_client.py       |  13 +++
 docs/source/diagnosing-performance.rst |   4 +-
 5 files changed, 112 insertions(+), 128 deletions(-)

diff --git a/distributed/bokeh/components.py b/distributed/bokeh/components.py
index a9a9e09fc55..0554b3b1e58 100644
--- a/distributed/bokeh/components.py
+++ b/distributed/bokeh/components.py
@@ -311,7 +311,7 @@ def __init__(self, **kwargs):
         state = profile.create()
         data = profile.plot_data(state, profile_interval)
         self.states = data.pop('states')
-        self.source = ColumnDataSource(data=data)
+        self.root, self.source = profile.plot_figure(data, **kwargs)
 
         @without_property_validation
         def cb(attr, old, new):
@@ -335,45 +335,6 @@ def cb(attr, old, new):
         else:
             self.source.on_change('selected', cb)
 
-        self.root = figure(tools='tap', **kwargs)
-        self.root.quad('left', 'right', 'top', 'bottom', color='color',
-                      line_color='black', line_width=2, source=self.source)
-
-        hover = HoverTool(
-            point_policy="follow_mouse",
-            tooltips="""
-                <div>
-                    <span style="font-size: 14px; font-weight: bold;">Name:</span>&nbsp;
-                    <span style="font-size: 10px; font-family: Monaco, monospace;">@name</span>
-                </div>
-                <div>
-                    <span style="font-size: 14px; font-weight: bold;">Filename:</span>&nbsp;
-                    <span style="font-size: 10px; font-family: Monaco, monospace;">@filename</span>
-                </div>
-                <div>
-                    <span style="font-size: 14px; font-weight: bold;">Line number:</span>&nbsp;
-                    <span style="font-size: 10px; font-family: Monaco, monospace;">@line_number</span>
-                </div>
-                <div>
-                    <span style="font-size: 14px; font-weight: bold;">Line:</span>&nbsp;
-                    <span style="font-size: 10px; font-family: Monaco, monospace;">@line</span>
-                </div>
-                <div>
-                    <span style="font-size: 14px; font-weight: bold;">Time:</span>&nbsp;
-                    <span style="font-size: 10px; font-family: Monaco, monospace;">@time</span>
-                </div>
-                <div>
-                    <span style="font-size: 14px; font-weight: bold;">Percentage:</span>&nbsp;
-                    <span style="font-size: 10px; font-family: Monaco, monospace;">@width</span>
-                </div>
-                """
-        )
-        self.root.add_tools(hover)
-
-        self.root.xaxis.visible = False
-        self.root.yaxis.visible = False
-        self.root.grid.visible = False
-
     @without_property_validation
     def update(self, state):
         with log_errors():
@@ -412,7 +373,7 @@ def __init__(self, server, doc=None, **kwargs):
         self.state = profile.create()
         data = profile.plot_data(self.state, profile_interval)
         self.states = data.pop('states')
-        self.source = ColumnDataSource(data=data)
+        self.profile_plot, self.source = profile.plot_figure(data, **kwargs)
 
         changing = [False]  # avoid repeated changes from within callback
 
@@ -445,47 +406,6 @@ def cb(attr, old, new):
         else:
             self.source.on_change('selected', cb)
 
-        self.profile_plot = figure(tools='tap', height=400, **kwargs)
-        r = self.profile_plot.quad('left', 'right', 'top', 'bottom', color='color',
-                                   line_color='black', source=self.source)
-        r.selection_glyph = None
-        r.nonselection_glyph = None
-
-        hover = HoverTool(
-            point_policy="follow_mouse",
-            tooltips="""
-                <div>
-                    <span style="font-size: 14px; font-weight: bold;">Name:</span>&nbsp;
-                    <span style="font-size: 10px; font-family: Monaco, monospace;">@name</span>
-                </div>
-                <div>
-                    <span style="font-size: 14px; font-weight: bold;">Filename:</span>&nbsp;
-                    <span style="font-size: 10px; font-family: Monaco, monospace;">@filename</span>
-                </div>
-                <div>
-                    <span style="font-size: 14px; font-weight: bold;">Line number:</span>&nbsp;
-                    <span style="font-size: 10px; font-family: Monaco, monospace;">@line_number</span>
-                </div>
-                <div>
-                    <span style="font-size: 14px; font-weight: bold;">Line:</span>&nbsp;
-                    <span style="font-size: 10px; font-family: Monaco, monospace;">@line</span>
-                </div>
-                <div>
-                    <span style="font-size: 14px; font-weight: bold;">Time:</span>&nbsp;
-                    <span style="font-size: 10px; font-family: Monaco, monospace;">@time</span>
-                </div>
-                <div>
-                    <span style="font-size: 14px; font-weight: bold;">Percentage:</span>&nbsp;
-                    <span style="font-size: 10px; font-family: Monaco, monospace;">@percentage</span>
-                </div>
-                """
-        )
-        self.profile_plot.add_tools(hover)
-
-        self.profile_plot.xaxis.visible = False
-        self.profile_plot.yaxis.visible = False
-        self.profile_plot.grid.visible = False
-
         self.ts_source = ColumnDataSource({'time': [], 'count': []})
         self.ts_plot = figure(title='Activity over time', height=100,
                               x_axis_type='datetime', active_drag='xbox_select',
@@ -591,7 +511,7 @@ def __init__(self, server, doc=None, **kwargs):
         self.state = profile.get_profile(self.log)
         data = profile.plot_data(self.state, profile_interval)
         self.states = data.pop('states')
-        self.source = ColumnDataSource(data=data)
+        self.profile_plot, self.source = profile.plot_figure(data, **kwargs)
 
         changing = [False]  # avoid repeated changes from within callback
 
@@ -624,47 +544,6 @@ def cb(attr, old, new):
         else:
             self.source.on_change('selected', cb)
 
-        self.profile_plot = figure(tools='tap', height=400, **kwargs)
-        r = self.profile_plot.quad('left', 'right', 'top', 'bottom', color='color',
-                                   line_color='black', source=self.source)
-        r.selection_glyph = None
-        r.nonselection_glyph = None
-
-        hover = HoverTool(
-            point_policy="follow_mouse",
-            tooltips="""
-                <div>
-                    <span style="font-size: 14px; font-weight: bold;">Name:</span>&nbsp;
-                    <span style="font-size: 10px; font-family: Monaco, monospace;">@name</span>
-                </div>
-                <div>
-                    <span style="font-size: 14px; font-weight: bold;">Filename:</span>&nbsp;
-                    <span style="font-size: 10px; font-family: Monaco, monospace;">@filename</span>
-                </div>
-                <div>
-                    <span style="font-size: 14px; font-weight: bold;">Line number:</span>&nbsp;
-                    <span style="font-size: 10px; font-family: Monaco, monospace;">@line_number</span>
-                </div>
-                <div>
-                    <span style="font-size: 14px; font-weight: bold;">Line:</span>&nbsp;
-                    <span style="font-size: 10px; font-family: Monaco, monospace;">@line</span>
-                </div>
-                <div>
-                    <span style="font-size: 14px; font-weight: bold;">Time:</span>&nbsp;
-                    <span style="font-size: 10px; font-family: Monaco, monospace;">@time</span>
-                </div>
-                <div>
-                    <span style="font-size: 14px; font-weight: bold;">Percentage:</span>&nbsp;
-                    <span style="font-size: 10px; font-family: Monaco, monospace;">@percentage</span>
-                </div>
-                """
-        )
-        self.profile_plot.add_tools(hover)
-
-        self.profile_plot.xaxis.visible = False
-        self.profile_plot.yaxis.visible = False
-        self.profile_plot.grid.visible = False
-
         self.ts_source = ColumnDataSource({'time': [], 'count': []})
         self.ts_plot = figure(title='Activity over time', height=100,
                               x_axis_type='datetime', active_drag='xbox_select',
diff --git a/distributed/client.py b/distributed/client.py
index e4a5cb3759b..6f30c38934e 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -2977,7 +2977,8 @@ def call_stack(self, futures=None, keys=None):
             keys += list(map(tokey, {f.key for f in futures}))
         return self.sync(self.scheduler.call_stack, keys=keys or None)
 
-    def profile(self, key=None, start=None, stop=None, workers=None, merge_workers=True):
+    def profile(self, key=None, start=None, stop=None, workers=None,
+                merge_workers=True, plot=False, filename=None):
         """ Collect statistical profiling information about recent work
 
         Parameters
@@ -2989,16 +2990,49 @@ def profile(self, key=None, start=None, stop=None, workers=None, merge_workers=T
         stop: time
         workers: list
             List of workers to restrict profile information
+        plot: boolean or string
+            Whether or not to return a plot object
+        filename: str
+            Filename to save the plot
 
         Examples
         --------
         >>> client.profile()  # call on collections
+        >>> client.profile(filename='dask-profile.html')  # save to html file
         """
         if isinstance(workers, six.string_types + (Number,)):
             workers = [workers]
 
-        return self.sync(self.scheduler.profile, key=key, workers=workers,
-                         merge_workers=merge_workers, start=start, stop=stop)
+        return self.sync(self._profile, key=key, workers=workers,
+                         merge_workers=merge_workers, start=start, stop=stop,
+                         plot=plot, filename=filename)
+
+    @gen.coroutine
+    def _profile(self, key=None, start=None, stop=None, workers=None,
+                 merge_workers=True, plot=False, filename=None):
+        if isinstance(workers, six.string_types + (Number,)):
+            workers = [workers]
+
+        state = yield self.scheduler.profile(key=key, workers=workers,
+                merge_workers=merge_workers, start=start, stop=stop)
+
+        if filename:
+            plot = True
+
+        if plot:
+            from . import profile
+            data = profile.plot_data(state)
+            figure, source = profile.plot_figure(data, sizing_mode='stretch_both')
+
+            if plot == 'save' and not filename:
+                filename = 'dask-profile.html'
+
+            from bokeh.plotting import save
+            save(figure, title='Dask Profile', filename=filename)
+            raise gen.Return((state, figure))
+
+        else:
+            raise gen.Return(state)
 
     def scheduler_info(self, **kwargs):
         """ Basic information about the workers in the cluster
diff --git a/distributed/profile.py b/distributed/profile.py
index 71ff6d18205..54e62c288c4 100644
--- a/distributed/profile.py
+++ b/distributed/profile.py
@@ -24,6 +24,8 @@
                    'children': {...}}}
     }
 """
+from __future__ import print_function, division, absolute_import
+
 import bisect
 from collections import defaultdict, deque
 import linecache
@@ -299,3 +301,57 @@ def get_profile(history, recent=None, start=None, stop=None, key=None):
         prof = merge(prof, recent)
 
     return prof
+
+
+def plot_figure(data, **kwargs):
+    from bokeh.plotting import ColumnDataSource, figure
+    from bokeh.models import HoverTool
+
+    if 'states' in data:
+        data = toolz.dissoc(data, 'states')
+
+    source = ColumnDataSource(data=data)
+
+    fig = figure(tools='tap', **kwargs)
+    r = fig.quad('left', 'right', 'top', 'bottom', color='color',
+             line_color='black', line_width=2, source=source)
+
+    r.selection_glyph = None
+    r.nonselection_glyph = None
+
+    hover = HoverTool(
+        point_policy="follow_mouse",
+        tooltips="""
+            <div>
+                <span style="font-size: 14px; font-weight: bold;">Name:</span>&nbsp;
+                <span style="font-size: 10px; font-family: Monaco, monospace;">@name</span>
+            </div>
+            <div>
+                <span style="font-size: 14px; font-weight: bold;">Filename:</span>&nbsp;
+                <span style="font-size: 10px; font-family: Monaco, monospace;">@filename</span>
+            </div>
+            <div>
+                <span style="font-size: 14px; font-weight: bold;">Line number:</span>&nbsp;
+                <span style="font-size: 10px; font-family: Monaco, monospace;">@line_number</span>
+            </div>
+            <div>
+                <span style="font-size: 14px; font-weight: bold;">Line:</span>&nbsp;
+                <span style="font-size: 10px; font-family: Monaco, monospace;">@line</span>
+            </div>
+            <div>
+                <span style="font-size: 14px; font-weight: bold;">Time:</span>&nbsp;
+                <span style="font-size: 10px; font-family: Monaco, monospace;">@time</span>
+            </div>
+            <div>
+                <span style="font-size: 14px; font-weight: bold;">Percentage:</span>&nbsp;
+                <span style="font-size: 10px; font-family: Monaco, monospace;">@width</span>
+            </div>
+            """
+    )
+    fig.add_tools(hover)
+
+    fig.xaxis.visible = False
+    fig.yaxis.visible = False
+    fig.grid.visible = False
+
+    return fig, source
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 96ae9aa85fb..8c074bd5881 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -5485,5 +5485,18 @@ def f(x):
         assert result == 101
 
 
+@gen_cluster(client=True, check_new_threads=False)
+def test_profile_bokeh(c, s, a, b):
+    pytest.importorskip('bokeh.plotting')
+    from bokeh.model import Model
+    yield c.map(slowinc, range(10), delay=0.2)
+    state, figure = yield c.profile(plot=True)
+    assert isinstance(figure, Model)
+
+    with tmpfile('html') as fn:
+        yield c.profile(filename=fn)
+        assert os.path.exists(fn)
+
+
 if sys.version_info >= (3, 5):
     from distributed.tests.py3_test_client import *  # noqa F401
diff --git a/docs/source/diagnosing-performance.rst b/docs/source/diagnosing-performance.rst
index c326801bc12..28d7d9aba44 100644
--- a/docs/source/diagnosing-performance.rst
+++ b/docs/source/diagnosing-performance.rst
@@ -84,7 +84,9 @@ plot on the ``/status`` page.
 
 Users can also query this data directly using the :doc:`Client.profile <api>`
 function.  This will deliver the raw data structure used to produce these
-plots.
+plots.  They can also pass a filename to save the plot as an HTML file
+directly.  Note that this file will have to be served from a webserver like
+``python -m http.server`` to be visible.
 
 The 10ms and 1s parameters can be controlled by the ``profile-interval`` and
 ``profile-cycle-interval`` entries in the config.yaml file.

From c5af1e2e4cd18cc482e57700fe4d50fd7759f40b Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 31 Jan 2019 07:43:57 -0800
Subject: [PATCH 0188/1550] Add scheduler address to workers page (#2483)

Fixes https://github.com/dask/dask/issues/4424
---
 distributed/bokeh/scheduler_html.py      | 1 +
 distributed/bokeh/templates/workers.html | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/distributed/bokeh/scheduler_html.py b/distributed/bokeh/scheduler_html.py
index 086155be957..86d5a043fb4 100644
--- a/distributed/bokeh/scheduler_html.py
+++ b/distributed/bokeh/scheduler_html.py
@@ -27,6 +27,7 @@ def get(self):
         with log_errors():
             self.render('workers.html',
                         title='Workers',
+                        scheduler=self.server,
                         **toolz.merge(self.server.__dict__, ns, self.extra))
 
 
diff --git a/distributed/bokeh/templates/workers.html b/distributed/bokeh/templates/workers.html
index 03d750a7040..6a2b7fc9345 100644
--- a/distributed/bokeh/templates/workers.html
+++ b/distributed/bokeh/templates/workers.html
@@ -1,6 +1,8 @@
 {% extends main.html %}
 {% block content %}
 
+  <h1 class="title"> Scheduler {{scheduler.address}} </h1>
+
   <a class="button is-primary" href="logs.html">Logs</a>
   <a class="button is-primary" href="../../status">Bokeh</a>
 

From f7abbd68b824dc03c8535b57f9e914bddd1d447c Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 31 Jan 2019 12:16:55 -0800
Subject: [PATCH 0189/1550] bump version to 1.25.3

---
 docs/source/changelog.rst | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index cc029317e5e..6e823f1027c 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,26 @@
 Changelog
 =========
 
+1.25.3 - 2019-01-31
+-------------------
+
+-  Fix excess threading on missing connections (:pr:`2403`) `Daniel Farrell`_
+-  Fix typo in doc (:pr:`2457`) `Loïc Estève`_
+-  Start fewer but larger workers with LocalCluster (:pr:`2452`) `Matthew Rocklin`_
+-  Check for non-zero ``length`` first in ``read`` loop (:pr:`2465`) `John Kirkham`_
+-  DOC: Use of local cluster in script (:pr:`2462`) `Peter Killick`_
+-  DOC/API: Signature for base class write / read (:pr:`2472`) `Tom Augspurger`_
+-  Support Pytest 4 in Tests (:pr:`2478`) `Adam Beberg`_
+-  Ensure async behavior in event loop with LocalCluster (:pr:`2484`) `Matthew Rocklin`_
+-  Fix spurious CancelledError (:pr:`2485`) `Loïc Estève`_
+-  Properly reset dask.config scheduler and shuffle when closing the client (:pr:`2475`) `George Sakkis`_
+-  Make it more explict that resources are per worker. (:pr:`2470`) `Loïc Estève`_
+-  Remove references to center (:pr:`2488`)  `Matthew Rocklin`_
+-  Expand client clearing timeout to 10s in testing (:pr:`2493`) `Matthew Rocklin`_
+-  Propagate key keyword in progressbar (:pr:`2492`) `Matthew Rocklin`_
+-  Use provided cluster's IOLoop if present in Client (:pr:`2494`) `Matthew Rocklin`_
+
+
 1.25.2 - 2019-01-04
 -------------------
 
@@ -897,3 +917,6 @@ significantly without many new features.
 .. _`Stephan Hoyer`: https://github.com/shoyer
 .. _`tjb900`: https://github.com/tjb900
 .. _`Dirk Petersen`: https://github.com/dirkpetersen
+.. _`Daniel Farrell`: https://github.com/danpf
+.. _`George Sakkis`: https://github.com/gsakkis
+.. _`Adam Beberg`: https://github.com/beberg

From 1b1b925cee845ecc1872f264a303be9bc05a77fa Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 1 Feb 2019 18:48:52 -0800
Subject: [PATCH 0190/1550] Update style to fix recent flake8 update (#2500)

---
 distributed/bokeh/components.py | 142 ++++++++++++++++----------------
 distributed/scheduler.py        |  49 ++++++-----
 2 files changed, 95 insertions(+), 96 deletions(-)

diff --git a/distributed/bokeh/components.py b/distributed/bokeh/components.py
index 0554b3b1e58..351dc168f86 100644
--- a/distributed/bokeh/components.py
+++ b/distributed/bokeh/components.py
@@ -97,77 +97,77 @@ def update(self, messages):
 
 
 def task_stream_figure(clear_interval='20s', **kwargs):
-        """
-        kwargs are applied to the bokeh.models.plots.Plot constructor
-        """
-        clear_interval = parse_timedelta(clear_interval, default='ms')
-
-        source = ColumnDataSource(data=dict(
-            start=[time() - clear_interval], duration=[0.1], key=['start'],
-            name=['start'], color=['white'], duration_text=['100 ms'],
-            worker=['foo'], y=[0], worker_thread=[1], alpha=[0.0])
-        )
-
-        x_range = DataRange1d(range_padding=0)
-        y_range = DataRange1d(range_padding=0)
-
-        root = figure(
-            name='task_stream',
-            title="Task Stream",
-            id='bk-task-stream-plot',
-            x_range=x_range,
-            y_range=y_range,
-            toolbar_location="above",
-            x_axis_type='datetime',
-            min_border_right=35,
-            tools='',
-            **kwargs
-        )
-
-        rect = root.rect(
-            source=source,
-            x="start",
-            y="y",
-            width="duration",
-            height=0.4,
-            fill_color="color",
-            line_color="color",
-            line_alpha=0.6,
-            fill_alpha="alpha",
-            line_width=3
-        )
-        rect.nonselection_glyph = None
-
-        root.yaxis.major_label_text_alpha = 0
-        root.yaxis.minor_tick_line_alpha = 0
-        root.yaxis.major_tick_line_alpha = 0
-        root.xgrid.visible = False
-
-        hover = HoverTool(
-            point_policy="follow_mouse",
-            tooltips="""
-                <div>
-                    <span style="font-size: 12px; font-weight: bold;">@name:</span>&nbsp;
-                    <span style="font-size: 10px; font-family: Monaco, monospace;">@duration_text</span>
-                </div>
-                """
-        )
-
-        tap = TapTool(callback=OpenURL(url='/profile?key=@name'))
-
-        root.add_tools(
-            hover, tap,
-            BoxZoomTool(),
-            ResetTool(),
-            PanTool(dimensions="width"),
-            WheelZoomTool(dimensions="width")
-        )
-        if ExportTool:
-            export = ExportTool()
-            export.register_plot(root)
-            root.add_tools(export)
-
-        return source, root
+    """
+    kwargs are applied to the bokeh.models.plots.Plot constructor
+    """
+    clear_interval = parse_timedelta(clear_interval, default='ms')
+
+    source = ColumnDataSource(data=dict(
+        start=[time() - clear_interval], duration=[0.1], key=['start'],
+        name=['start'], color=['white'], duration_text=['100 ms'],
+        worker=['foo'], y=[0], worker_thread=[1], alpha=[0.0])
+    )
+
+    x_range = DataRange1d(range_padding=0)
+    y_range = DataRange1d(range_padding=0)
+
+    root = figure(
+        name='task_stream',
+        title="Task Stream",
+        id='bk-task-stream-plot',
+        x_range=x_range,
+        y_range=y_range,
+        toolbar_location="above",
+        x_axis_type='datetime',
+        min_border_right=35,
+        tools='',
+        **kwargs
+    )
+
+    rect = root.rect(
+        source=source,
+        x="start",
+        y="y",
+        width="duration",
+        height=0.4,
+        fill_color="color",
+        line_color="color",
+        line_alpha=0.6,
+        fill_alpha="alpha",
+        line_width=3
+    )
+    rect.nonselection_glyph = None
+
+    root.yaxis.major_label_text_alpha = 0
+    root.yaxis.minor_tick_line_alpha = 0
+    root.yaxis.major_tick_line_alpha = 0
+    root.xgrid.visible = False
+
+    hover = HoverTool(
+        point_policy="follow_mouse",
+        tooltips="""
+            <div>
+                <span style="font-size: 12px; font-weight: bold;">@name:</span>&nbsp;
+                <span style="font-size: 10px; font-family: Monaco, monospace;">@duration_text</span>
+            </div>
+            """
+    )
+
+    tap = TapTool(callback=OpenURL(url='/profile?key=@name'))
+
+    root.add_tools(
+        hover, tap,
+        BoxZoomTool(),
+        ResetTool(),
+        PanTool(dimensions="width"),
+        WheelZoomTool(dimensions="width")
+    )
+    if ExportTool:
+        export = ExportTool()
+        export.register_plot(root)
+        root.add_tools(export)
+
+    return source, root
 
 
 class MemoryUsage(DashboardComponent):
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index ea46aa7e00c..0ad8e12ef2d 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -97,7 +97,6 @@ class ClientState(object):
        collection) gets garbage-collected.
 
     """
-
     __slots__ = (
         'client_key',
         'wants_what',
@@ -1231,40 +1230,40 @@ def _setup_logging(self):
     @gen.coroutine
     def heartbeat_worker(self, comm=None, address=None, resolve_address=True,
                           now=None, resources=None, host_info=None, metrics=None):
-            address = self.coerce_address(address, resolve_address)
-            address = normalize_address(address)
-            host = get_address_host(address)
+        address = self.coerce_address(address, resolve_address)
+        address = normalize_address(address)
+        host = get_address_host(address)
 
-            local_now = time()
-            now = now or time()
-            metrics = metrics or {}
-            host_info = host_info or {}
+        local_now = time()
+        now = now or time()
+        metrics = metrics or {}
+        host_info = host_info or {}
 
-            self.host_info[host]['last-seen'] = local_now
+        self.host_info[host]['last-seen'] = local_now
 
-            ws = self.workers.get(address)
-            if not ws:
-                return {'status': 'missing'}
+        ws = self.workers.get(address)
+        if not ws:
+            return {'status': 'missing'}
 
-            ws.last_seen = time()
+        ws.last_seen = time()
 
-            if metrics:
-                ws.metrics = metrics
+        if metrics:
+            ws.metrics = metrics
 
-            if host_info:
-                self.host_info[host].update(host_info)
+        if host_info:
+            self.host_info[host].update(host_info)
 
-            delay = time() - now
-            ws.time_delay = delay
+        delay = time() - now
+        ws.time_delay = delay
 
-            if resources:
-                self.add_resources(worker=address, resources=resources)
+        if resources:
+            self.add_resources(worker=address, resources=resources)
 
-            self.log_event(address, merge({'action': 'heartbeat'}, metrics))
+        self.log_event(address, merge({'action': 'heartbeat'}, metrics))
 
-            return {'status': 'OK',
-                    'time': time(),
-                    'heartbeat-interval': heartbeat_interval(len(self.workers))}
+        return {'status': 'OK',
+                'time': time(),
+                'heartbeat-interval': heartbeat_interval(len(self.workers))}
 
     @gen.coroutine
     def add_worker(self, comm=None, address=None, keys=(), ncores=None,

From 597d3e5f980bc976d6306b51d2a7f325eaa3a3cc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lo=C3=AFc=20Est=C3=A8ve?= <loic.esteve@ymail.com>
Date: Tue, 5 Feb 2019 00:00:26 +0100
Subject: [PATCH 0191/1550] Fix typo in gen_cluster log message (#2503)

---
 distributed/utils_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index 1f1abd8b1f1..2801869cfb5 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -871,7 +871,7 @@ def coro():
                                         Worker=Worker, scheduler_kwargs=scheduler_kwargs,
                                         worker_kwargs=worker_kwargs)
                                 except Exception as e:
-                                    logger.error("Failed to start gen_cluster, retryng", exc_info=True)
+                                    logger.error("Failed to start gen_cluster, retrying", exc_info=True)
                                 else:
                                     workers[:] = ws
                                     args = [s] + workers

From 7d6c66eccc7296c11c1d4689159a99e96da98835 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 6 Feb 2019 10:30:27 -0600
Subject: [PATCH 0192/1550] Fix over-indented code highlighted by flake8 update
 (#2509)


From 76d4b8b2646db9a16ffb6620a426517704b0e906 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 6 Feb 2019 11:22:19 -0600
Subject: [PATCH 0193/1550] Allow KeyError when closing event loop (#2498)

In some shutdown situations we can close the event loop after it has already
left an internal system.
---
 distributed/utils.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/distributed/utils.py b/distributed/utils.py
index c5227ade1cb..812fec91db4 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -408,7 +408,8 @@ def _real_stop(self, timeout):
             try:
                 self._loop.add_callback(self._loop.stop)
                 self._loop_thread.join(timeout=timeout)
-                self._loop.close()
+                with ignoring(KeyError):  # IOLoop can be missing
+                    self._loop.close()
             finally:
                 self._loop_thread = None
 

From 33cc87ffac8a3a51165a4f83d179d0feba472376 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 7 Feb 2019 13:13:37 -0600
Subject: [PATCH 0194/1550] Avoid thread testing for TCP ThreadPoolExecutor
 (#2510)

* Increase size of data in test_pause_executor
* Don't count TCP executor in excess thread testing
* xfail known intermittent failure
* use our threadpoolexecutor to get thread_name_prefix
* remove spilling in test_pause_executor
* add memory pause fraction in config
* xfail test_actor.py::test_compute()
* add debug info to test_pause_executor
---
 distributed/comm/tcp.py                       |  5 ++--
 .../diagnostics/tests/test_progress.py        |  4 +--
 distributed/tests/py3_test_client.py          |  1 +
 distributed/tests/test_actor.py               |  3 ++-
 distributed/tests/test_client.py              |  2 +-
 distributed/tests/test_worker.py              | 25 ++++++++++++-------
 distributed/utils_test.py                     |  3 ++-
 7 files changed, 26 insertions(+), 17 deletions(-)

diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py
index 5af63b164a1..f538ea81c72 100644
--- a/distributed/comm/tcp.py
+++ b/distributed/comm/tcp.py
@@ -11,8 +11,6 @@
 except ImportError:
     ssl = None
 
-from concurrent.futures import ThreadPoolExecutor
-
 import dask
 import tornado
 from tornado import gen, netutil
@@ -21,6 +19,7 @@
 from tornado.tcpserver import TCPServer
 
 from ..compatibility import finalize, PY3
+from ..threadpoolexecutor import ThreadPoolExecutor
 from ..utils import (ensure_bytes, ensure_ip, get_ip, get_ipv6, nbytes,
                      parse_timedelta, shutting_down)
 
@@ -322,7 +321,7 @@ def _check_encryption(self, address, connection_args):
 
 class BaseTCPConnector(Connector, RequireEncryptionMixin):
     if PY3:  # see github PR #2403 discussion for more info
-        _executor = ThreadPoolExecutor(2)
+        _executor = ThreadPoolExecutor(2, thread_name_prefix="TCP-Executor")
         _resolver = netutil.ExecutorResolver(close_executor=False,
                                              executor=_executor)
     else:
diff --git a/distributed/diagnostics/tests/test_progress.py b/distributed/diagnostics/tests/test_progress.py
index 0da8f1121d9..2d88054a34b 100644
--- a/distributed/diagnostics/tests/test_progress.py
+++ b/distributed/diagnostics/tests/test_progress.py
@@ -39,7 +39,7 @@ def test_many_Progress(c, s, a, b):
     start = time()
     while not all(b.status == 'finished' for b in bars):
         yield gen.sleep(0.1)
-        assert time() < start + 2
+        assert time() < start + 5
 
 
 @gen_cluster(client=True)
@@ -182,7 +182,7 @@ def test_AllProgress_lost_key(c, s, a, b, timeout=None):
     start = time()
     while len(p.state['memory']['inc']) > 0:
         yield gen.sleep(0.1)
-        assert time() < start + 2
+        assert time() < start + 5
 
 
 @gen_cluster(client=True)
diff --git a/distributed/tests/py3_test_client.py b/distributed/tests/py3_test_client.py
index cad9fb67fb0..d75b2bd0801 100644
--- a/distributed/tests/py3_test_client.py
+++ b/distributed/tests/py3_test_client.py
@@ -129,6 +129,7 @@ async def ff():
             assert c.sync(ff) == 1
 
 
+@pytest.mark.xfail(reason="known intermittent failure")
 @gen_cluster(client=True)
 async def test_dont_hold_on_to_large_messages(c, s, a, b):
     np = pytest.importorskip('numpy')
diff --git a/distributed/tests/test_actor.py b/distributed/tests/test_actor.py
index 463b6585af8..942d3f1e761 100644
--- a/distributed/tests/test_actor.py
+++ b/distributed/tests/test_actor.py
@@ -448,6 +448,7 @@ def f(block, ps=None):
     print(format_time(end - start))
 
 
+@pytest.mark.xfail(reason='unknown')
 @gen_cluster(client=True)
 def test_compute(c, s, a, b):
 
@@ -471,7 +472,7 @@ def check(counter, blanks):
     start = time()
     while a.data or b.data:
         yield gen.sleep(0.01)
-        assert time() < start + 2
+        assert time() < start + 5
 
 
 def test_compute_sync(client):
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 8c074bd5881..c7c5fcc1f80 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -5375,7 +5375,7 @@ def bad_fn(x):
 
 def test_no_threads_lingering():
     active = dict(threading._active)
-    assert threading.active_count() < 30, list(active.values())
+    assert threading.active_count() < 40, list(active.values())
 
 
 @gen_cluster()
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index dcdf4b570eb..77e7d040a8d 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -1005,25 +1005,32 @@ def f(n):
 
 
 @pytest.mark.slow
-@gen_cluster(ncores=[('127.0.0.1', 2)], client=True,
-             worker_kwargs={'memory_monitor_interval': 10},
+@gen_cluster(ncores=[('127.0.0.1', 2)],
+             client=True,
+             worker_kwargs={'memory_monitor_interval': 10,
+                            'memory_spill_fraction': False,  # don't spill
+                            'memory_target_fraction': False,
+                            'memory_pause_fraction': 0.5},
              timeout=20)
 def test_pause_executor(c, s, a):
     memory = psutil.Process().memory_info().rss
-    a.memory_limit = memory / 0.8 + 200e6
+    a.memory_limit = memory / 0.5 + 200e6
     np = pytest.importorskip('numpy')
 
     def f():
-        x = np.ones(int(300e6), dtype='u1')
+        x = np.ones(int(400e6), dtype='u1')
         sleep(1)
 
     with captured_logger(logging.getLogger('distributed.worker')) as logger:
         future = c.submit(f)
-        futures = c.map(slowinc, range(10), delay=0.1)
-
-        yield gen.sleep(0.3)
-        assert a.paused, (format_bytes(psutil.Process().memory_info().rss),
-                          format_bytes(a.memory_limit))
+        futures = c.map(slowinc, range(30), delay=0.1)
+
+        start = time()
+        while not a.paused:
+            yield gen.sleep(0.01)
+            assert time() < start + 4,  (format_bytes(psutil.Process().memory_info().rss),
+                                         format_bytes(a.memory_limit),
+                                         len(a.data))
         out = logger.getvalue()
         assert 'memory' in out.lower()
         assert 'pausing' in out.lower()
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index 2801869cfb5..6763eb18027 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -931,7 +931,8 @@ def coro():
                     bad = [t for t, v in threading._active.items()
                            if t not in active_threads_start and
                           "Threaded" not in v.name and
-                          "watch message" not in v.name]
+                          "watch message" not in v.name and
+                          "TCP-Executor" not in v.name]
                     if not bad:
                         break
                     else:

From 0a6260ea55e4b1dcb16b0ebc7fa92622736fb01b Mon Sep 17 00:00:00 2001
From: Jim Crist <jcrist@users.noreply.github.com>
Date: Fri, 8 Feb 2019 16:25:35 -0600
Subject: [PATCH 0195/1550] Find Futures inside SubgraphCallable (#2505)

---
 distributed/client.py            |  9 ++++--
 distributed/tests/test_client.py | 50 +++++++++++++++++++++++++++++++-
 distributed/utils_comm.py        | 22 +++++++++++++-
 3 files changed, 76 insertions(+), 5 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 6f30c38934e..a5047b916a7 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -26,6 +26,7 @@
 import dask
 from dask.base import tokenize, normalize_token, collections_to_dsk
 from dask.core import flatten, get_dependencies
+from dask.optimization import SubgraphCallable
 from dask.compatibility import apply, unicode
 try:
     from cytoolz import first, groupby, merge, valmap, keymap
@@ -3964,11 +3965,13 @@ def futures_of(o, client=None):
         x = stack.pop()
         if type(x) in (tuple, set, list):
             stack.extend(x)
-        if type(x) is dict:
+        elif type(x) is dict:
             stack.extend(x.values())
-        if isinstance(x, Future):
+        elif type(x) is SubgraphCallable:
+            stack.extend(x.dsk.values())
+        elif isinstance(x, Future):
             futures.add(x)
-        if dask.is_dask_collection(x):
+        elif dask.is_dask_collection(x):
             stack.extend(x.__dask_graph__().values())
 
     if client is not None:
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index c7c5fcc1f80..5968ea045b8 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -27,6 +27,7 @@
 
 import dask
 from dask import delayed
+from dask.optimization import SubgraphCallable
 import dask.bag as db
 from distributed import (Worker, Nanny, fire_and_forget, LocalCluster,
                          get_client, secede, get_worker, Executor, profile,
@@ -2493,7 +2494,7 @@ def test_wait_on_collections(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_futures_of(c, s, a, b):
+def test_futures_of_get(c, s, a, b):
     x, y, z = c.map(inc, [1, 2, 3])
 
     assert set(futures_of(0)) == set()
@@ -2505,6 +2506,11 @@ def test_futures_of(c, s, a, b):
     b = db.Bag({('b', i): f for i, f in enumerate([x, y, z])}, 'b', 3)
     assert set(futures_of(b)) == {x, y, z}
 
+    sg = SubgraphCallable({'x': x, 'y': y, 'z': z,
+                           'out': (add, (add, (add, x, y), z), 'in')},
+                          'out', ('in',))
+    assert set(futures_of(sg)) == {x, y, z}
+
 
 def test_futures_of_class():
     da = pytest.importorskip('dask.array')
@@ -5498,5 +5504,47 @@ def test_profile_bokeh(c, s, a, b):
         assert os.path.exists(fn)
 
 
+@gen_cluster(client=True)
+def test_get_mix_futures_and_SubgraphCallable(c, s, a, b):
+    future = c.submit(add, 1, 2)
+
+    subgraph = SubgraphCallable({'_2': (add, '_0', '_1'),
+                                 '_3': (add, future, '_2')},
+                                '_3', ('_0', '_1'))
+    dsk = {'a': 1,
+           'b': 2,
+           'c': (subgraph, 'a', 'b'),
+           'd': (subgraph, 'c', 'b')}
+
+    future2 = c.get(dsk, 'd', sync=False)
+    result = yield future2
+    assert result == 11
+
+    # Nested subgraphs
+    subgraph2 = SubgraphCallable({'_2': (subgraph, '_0', '_1'),
+                                  '_3': (subgraph, '_2', '_1'),
+                                  '_4': (add, '_3', future2)},
+                                 '_4', ('_0', '_1'))
+
+    dsk2 = {'e': 1, 'f': 2, 'g': (subgraph2, 'e', 'f')}
+
+    result = yield c.get(dsk2, 'g', sync=False)
+    assert result == 22
+
+
+@gen_cluster(client=True)
+def test_get_mix_futures_and_SubgraphCallable_dask_dataframe(c, s, a, b):
+    dd = pytest.importorskip('dask.dataframe')
+    import pandas as pd
+    df = pd.DataFrame({'x': range(1, 11)})
+    ddf = dd.from_pandas(df, npartitions=2).persist()
+    ddf = ddf.map_partitions(lambda x: x)
+    ddf['x'] = ddf['x'].astype('f8')
+    ddf = ddf.map_partitions(lambda x: x)
+    ddf['x'] = ddf['x'].astype('f8')
+    result = yield c.compute(ddf)
+    assert result.equals(df.astype('f8'))
+
+
 if sys.version_info >= (3, 5):
     from distributed.tests.py3_test_client import *  # noqa F401
diff --git a/distributed/utils_comm.py b/distributed/utils_comm.py
index 5151d802142..63622044291 100644
--- a/distributed/utils_comm.py
+++ b/distributed/utils_comm.py
@@ -7,6 +7,7 @@
 from tornado import gen
 from tornado.gen import Return
 
+from dask.optimization import SubgraphCallable
 from toolz import merge, concat, groupby, drop
 
 from .core import rpc
@@ -177,11 +178,30 @@ def unpack_remotedata(o, byte_keys=False, myset=None):
 
     typ = type(o)
 
+    if typ is tuple:
+        if not o:
+            return o
+        if type(o[0]) is SubgraphCallable:
+            sc = o[0]
+            futures = set()
+            dsk = {k: unpack_remotedata(v, byte_keys, futures)
+                   for k, v in sc.dsk.items()}
+            args = tuple(unpack_remotedata(i, byte_keys, futures) for i in o[1:])
+            if futures:
+                myset.update(futures)
+                futures = (tuple(tokey(f.key) for f in futures)
+                           if byte_keys else tuple(f.key for f in futures))
+                inkeys = sc.inkeys + futures
+                return (SubgraphCallable(dsk, sc.outkey, inkeys, sc.name),) + args + futures
+            else:
+                return o
+        else:
+            return tuple(unpack_remotedata(item, byte_keys, myset) for item in o)
     if typ in collection_types:
         if not o:
             return o
         outs = [unpack_remotedata(item, byte_keys, myset) for item in o]
-        return type(o)(outs)
+        return typ(outs)
     elif typ is dict:
         if o:
             values = [unpack_remotedata(v, byte_keys, myset) for v in o.values()]

From 0a5b8da4041414e1e621c1b8d613730720c6e684 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 8 Feb 2019 16:08:47 -0800
Subject: [PATCH 0196/1550] Avoid AttributeError when closing and sending a
 message (#2514)

---
 distributed/client.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/client.py b/distributed/client.py
index a5047b916a7..65e034ff5fd 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -774,7 +774,7 @@ def _send_to_scheduler_safe(self, msg):
         if self.status in ('running', 'closing'):
             try:
                 self.scheduler_comm.send(msg)
-            except CommClosedError:
+            except (CommClosedError, AttributeError):
                 if self.status == 'running':
                     raise
         elif self.status in ('connecting', 'newly-created'):

From 40e859a63daad1973c871d9e143936b5fd67edde Mon Sep 17 00:00:00 2001
From: jukent <46687291+jukent@users.noreply.github.com>
Date: Tue, 12 Feb 2019 12:19:36 -0700
Subject: [PATCH 0197/1550] Add deprecation warning to dask_mpi.py (#2522)

---
 distributed/cli/dask_mpi.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/distributed/cli/dask_mpi.py b/distributed/cli/dask_mpi.py
index f567e07f637..e26e0bc91e1 100644
--- a/distributed/cli/dask_mpi.py
+++ b/distributed/cli/dask_mpi.py
@@ -4,6 +4,7 @@
 from mpi4py import MPI
 from tornado.ioloop import IOLoop
 from tornado import gen
+from warnings import warn
 
 from distributed import Scheduler, Nanny, Worker
 from distributed.bokeh.worker import BokehWorker
@@ -98,6 +99,10 @@ def close():
 
 def go():
     check_python_3()
+    warn("The dask-mpi command line utility in the `distributed` "
+         "package is deprecated.  "
+         "Please install the `dask-mpi` package instead. "
+         "More information is available at https://mpi.dask.org")
     main()
 
 

From c94cde1f7f5bc06ae57a303f7345635e323dad21 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 12 Feb 2019 22:25:51 -0800
Subject: [PATCH 0198/1550] Relax statistical profiling test (#2527)

---
 distributed/tests/test_worker.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 77e7d040a8d..4de655d6858 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -1051,8 +1051,12 @@ def test_statistical_profiling_cycle(c, s, a, b):
     x = a.get_profile(start=time() + 10, stop=time() + 20)
     assert not x['count']
 
-    x = a.get_profile(start=0, stop=time())
-    assert x['count'] == sum(p['count'] for _, p in a.profile_history) + a.profile_recent['count']
+    for i in range(5):  # there is a chance that this will be slightly off. Try a few times
+        x = a.get_profile(start=0, stop=time())
+        if x['count'] == sum(p['count'] for _, p in a.profile_history) + a.profile_recent['count']:
+            break
+    else:
+        raise Exception(x['count'], sum(p['count'] for _, p in a.profile_history) + a.profile_recent['count'])
 
     y = a.get_profile(start=end - 0.300, stop=time())
     assert 0 < y['count'] <= x['count']

From d6ba683a55816ece13f66a68f7e2ef7668775800 Mon Sep 17 00:00:00 2001
From: Adam Beberg <beberg@mithral.com>
Date: Thu, 14 Feb 2019 12:44:03 -0800
Subject: [PATCH 0199/1550] Support alternative --remote-dask-worker
 SSHCluster() and dask-ssh CLI (#2526)

---
 distributed/cli/dask_ssh.py |  6 ++++--
 distributed/deploy/ssh.py   | 13 +++++++++----
 2 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/distributed/cli/dask_ssh.py b/distributed/cli/dask_ssh.py
index 99aed59baed..acb87d21642 100755
--- a/distributed/cli/dask_ssh.py
+++ b/distributed/cli/dask_ssh.py
@@ -47,10 +47,12 @@
               help="Serving computation port, defaults to random")
 @click.option('--nanny-port', type=int, default=0,
               help="Serving nanny port, defaults to random")
+@click.option('--remote-dask-worker', default=None, type=str,
+              help="Worker to run. Defaults to distributed.cli.dask_worker")
 @click.pass_context
 def main(ctx, scheduler, scheduler_port, hostnames, hostfile, nthreads, nprocs,
          ssh_username, ssh_port, ssh_private_key, nohost, log_directory, remote_python,
-         memory_limit, worker_port, nanny_port):
+         memory_limit, worker_port, nanny_port, remote_dask_worker):
     try:
         hostnames = list(hostnames)
         if hostfile:
@@ -67,7 +69,7 @@ def main(ctx, scheduler, scheduler_port, hostnames, hostfile, nthreads, nprocs,
 
     c = SSHCluster(scheduler, scheduler_port, hostnames, nthreads, nprocs,
                    ssh_username, ssh_port, ssh_private_key, nohost, log_directory, remote_python,
-                   memory_limit, worker_port, nanny_port)
+                   memory_limit, worker_port, nanny_port, remote_dask_worker)
 
     import distributed
     print('\n---------------------------------------------------------------')
diff --git a/distributed/deploy/ssh.py b/distributed/deploy/ssh.py
index c10cedd881a..b37fd1b9aa8 100644
--- a/distributed/deploy/ssh.py
+++ b/distributed/deploy/ssh.py
@@ -214,9 +214,10 @@ def start_worker(logdir, scheduler_addr, scheduler_port, worker_addr, nthreads,
                  memory_limit,
                  worker_port,
                  nanny_port,
-                 remote_python=None):
+                 remote_python=None,
+                 remote_dask_worker='distributed.cli.dask_worker'):
 
-    cmd = ('{python} -m distributed.cli.dask_worker '
+    cmd = ('{python} -m {remote_dask_worker} '
            '{scheduler_addr}:{scheduler_port} '
            '--nthreads {nthreads} --nprocs {nprocs} ')
 
@@ -234,6 +235,7 @@ def start_worker(logdir, scheduler_addr, scheduler_port, worker_addr, nthreads,
 
     cmd = cmd.format(
         python=remote_python or sys.executable,
+        remote_dask_worker=remote_dask_worker,
         scheduler_addr=scheduler_addr,
         scheduler_port=scheduler_port,
         worker_addr=worker_addr,
@@ -273,7 +275,8 @@ class SSHCluster(object):
     def __init__(self, scheduler_addr, scheduler_port, worker_addrs, nthreads=0, nprocs=1,
                  ssh_username=None, ssh_port=22, ssh_private_key=None,
                  nohost=False, logdir=None, remote_python=None,
-                 memory_limit=None, worker_port=None, nanny_port=None):
+                 memory_limit=None, worker_port=None, nanny_port=None,
+                 remote_dask_worker='distributed.cli.dask_worker'):
 
         self.scheduler_addr = scheduler_addr
         self.scheduler_port = scheduler_port
@@ -291,6 +294,7 @@ def __init__(self, scheduler_addr, scheduler_port, worker_addrs, nthreads=0, npr
         self.memory_limit = memory_limit
         self.worker_port = worker_port
         self.nanny_port = nanny_port
+        self.remote_dask_worker = remote_dask_worker
 
         # Generate a universal timestamp to use for log files
         import datetime
@@ -351,7 +355,8 @@ def add_worker(self, address):
                                          self.memory_limit,
                                          self.worker_port,
                                          self.nanny_port,
-                                         self.remote_python))
+                                         self.remote_python,
+                                         self.remote_dask_worker))
 
     def shutdown(self):
         all_processes = [self.scheduler] + self.workers

From 5a87506cd9e01c7e6dc7870ec899969f83f92c5b Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 14 Feb 2019 12:44:58 -0800
Subject: [PATCH 0200/1550] Iterate over full list of plugins in transition
 (#2518)

The progress plugin removes itself, which can screw with iteration semantics
This was causing a a failure in test_many_Progress
---
 distributed/scheduler.py         | 2 +-
 distributed/tests/test_worker.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 0ad8e12ef2d..4f3c8520063 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -3954,7 +3954,7 @@ def transition(self, key, finish, *args, **kwargs):
                     except KeyError:
                         pass
                     self.tasks[ts.key] = ts
-                for plugin in self.plugins:
+                for plugin in list(self.plugins):
                     try:
                         plugin.transition(key, start, finish2, *args, **kwargs)
                     except Exception:
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 4de655d6858..5a72f8d21cc 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -1040,7 +1040,7 @@ def f():
     yield wait(futures)
 
 
-@gen_cluster(client=True, worker_kwargs={'profile_cycle_interval': 100})
+@gen_cluster(client=True, worker_kwargs={'profile_cycle_interval': '10 ms'})
 def test_statistical_profiling_cycle(c, s, a, b):
     futures = c.map(slowinc, range(20), delay=0.05)
     yield wait(futures)

From b5e1293a11d7a8e56d838b3a06d9408a6d517947 Mon Sep 17 00:00:00 2001
From: Adam Beberg <beberg@mithral.com>
Date: Thu, 14 Feb 2019 12:48:11 -0800
Subject: [PATCH 0201/1550] Create Prometheus Endpoint (#2499)

---
 .../setup_conda_environment.cmd               |  1 +
 continuous_integration/travis/install.sh      |  1 +
 distributed/bokeh/scheduler_html.py           | 21 ++++++++++
 .../bokeh/tests/test_scheduler_bokeh_html.py  | 10 +++++
 distributed/bokeh/tests/test_worker_bokeh.py  | 13 ++++++
 distributed/bokeh/worker.py                   | 13 ++++++
 distributed/bokeh/worker_html.py              | 41 +++++++++++++++++++
 7 files changed, 100 insertions(+)
 create mode 100644 distributed/bokeh/worker_html.py

diff --git a/continuous_integration/setup_conda_environment.cmd b/continuous_integration/setup_conda_environment.cmd
index af8c71728e7..cd201ff46d5 100644
--- a/continuous_integration/setup_conda_environment.cmd
+++ b/continuous_integration/setup_conda_environment.cmd
@@ -31,6 +31,7 @@ call deactivate
     jupyter_client ^
     mock ^
     msgpack-python ^
+    prometheus_client ^
     psutil ^
     pytest ^
     python=%PYTHON% ^
diff --git a/continuous_integration/travis/install.sh b/continuous_integration/travis/install.sh
index dbd3d8e043b..a1ce6560e9c 100644
--- a/continuous_integration/travis/install.sh
+++ b/continuous_integration/travis/install.sh
@@ -42,6 +42,7 @@ conda install -q \
     mock \
     netcdf4 \
     paramiko \
+    prometheus_client \
     psutil \
     pytest \
     pytest-timeout \
diff --git a/distributed/bokeh/scheduler_html.py b/distributed/bokeh/scheduler_html.py
index 86d5a043fb4..10e84bd4b31 100644
--- a/distributed/bokeh/scheduler_html.py
+++ b/distributed/bokeh/scheduler_html.py
@@ -163,6 +163,26 @@ def get(self):
         self.write(result)
 
 
+class PrometheusHandler(RequestHandler):
+    def __init__(self, *args, **kwargs):
+        import prometheus_client # keep out of global namespace
+        self.prometheus_client = prometheus_client
+
+        super(PrometheusHandler, self).__init__(*args, **kwargs)
+        self.workers = self.prometheus_client.Gauge('workers_total',
+            'Total number of workers.',
+            namespace='scheduler')
+        self.clients = self.prometheus_client.Gauge('clients_total',
+            'Total number of clients.',
+            namespace='scheduler')
+
+    def get(self):
+        self.workers.set(len(self.server.workers))
+        self.clients.set(len(self.server.clients))
+
+        self.write(self.prometheus_client.generate_latest())
+
+
 routes = [
         (r'info/main/workers.html', Workers),
         (r'info/worker/(.*).html', Worker),
@@ -175,6 +195,7 @@ def get(self):
         (r'json/identity.json', IdentityJSON),
         (r'json/index.html', IndexJSON),
         (r'individual-plots.json', IndividualPlots),
+        (r'metrics', PrometheusHandler),
 ]
 
 
diff --git a/distributed/bokeh/tests/test_scheduler_bokeh_html.py b/distributed/bokeh/tests/test_scheduler_bokeh_html.py
index 2ff822f1305..a372b42b9a7 100644
--- a/distributed/bokeh/tests/test_scheduler_bokeh_html.py
+++ b/distributed/bokeh/tests/test_scheduler_bokeh_html.py
@@ -60,3 +60,13 @@ def test_prefix(c, s, a, b):
             json.loads(body)
         else:
             assert xml.etree.ElementTree.fromstring(body) is not None
+
+
+@gen_cluster(client=True,
+             check_new_threads=False,
+             scheduler_kwargs={'services': {('bokeh', 0):  BokehScheduler}})
+def test_prometheus(c, s, a, b):
+    http_client = AsyncHTTPClient()
+    response = yield http_client.fetch('http://localhost:%d/metrics'
+                                       % s.services['bokeh'].port)
+    assert response.code == 200
diff --git a/distributed/bokeh/tests/test_worker_bokeh.py b/distributed/bokeh/tests/test_worker_bokeh.py
index e991e53e369..ccc0ba22cba 100644
--- a/distributed/bokeh/tests/test_worker_bokeh.py
+++ b/distributed/bokeh/tests/test_worker_bokeh.py
@@ -118,3 +118,16 @@ def test_CommunicatingStream(c, s, a, b):
             len(first(bb.outgoing.data.values())))
     assert (len(first(aa.incoming.data.values())) and
             len(first(bb.incoming.data.values())))
+
+
+@gen_cluster(client=True,
+             check_new_threads=False,
+             worker_kwargs={'services': {('bokeh', 0):  BokehWorker}})
+def test_prometheus(c, s, a, b):
+    assert s.workers[a.address].services == {'bokeh': a.services['bokeh'].port}
+
+    http_client = AsyncHTTPClient()
+    for suffix in ['metrics']:
+        response = yield http_client.fetch('http://localhost:%d/%s'
+                                           % (a.services['bokeh'].port, suffix))
+        assert response.code == 200
diff --git a/distributed/bokeh/worker.py b/distributed/bokeh/worker.py
index 898226279f3..ef6c27e0404 100644
--- a/distributed/bokeh/worker.py
+++ b/distributed/bokeh/worker.py
@@ -678,6 +678,19 @@ def __init__(self, worker, io_loop=None, prefix='', **kwargs):
         self.loop = io_loop or worker.loop
         self.server = None
 
+    @property
+    def extra(self):
+        return merge({'prefix': self.prefix}, template_variables)
+
     @property
     def my_server(self):
         return self.worker
+
+    def listen(self, *args, **kwargs):
+        super(BokehWorker, self).listen(*args, **kwargs)
+
+        from .worker_html import routes
+        handlers = [(self.prefix + '/' + url, cls, {'server': self.my_server, 'extra': self.extra})
+                    for url, cls in routes]
+
+        self.server._tornado.add_handlers(r'.*', handlers)
diff --git a/distributed/bokeh/worker_html.py b/distributed/bokeh/worker_html.py
new file mode 100644
index 00000000000..4428d19957c
--- /dev/null
+++ b/distributed/bokeh/worker_html.py
@@ -0,0 +1,41 @@
+import os
+
+from tornado import web
+
+dirname = os.path.dirname(__file__)
+
+
+class RequestHandler(web.RequestHandler):
+    def initialize(self, server=None, extra=None):
+        self.server = server
+        self.extra = extra or {}
+
+    def get_template_path(self):
+        return os.path.join(dirname, 'templates')
+
+
+class PrometheusHandler(RequestHandler):
+    def __init__(self, *args, **kwargs):
+        import prometheus_client # keep out of global namespace
+        self.prometheus_client = prometheus_client
+
+        super(PrometheusHandler, self).__init__(*args, **kwargs)
+        # Add metrics like this:
+        # self.workers = self.prometheus_client.Gauge('memory_bytes',
+        #    'Total memory.',
+        #    namespace='worker')
+
+    def get(self):
+        # Example metric update
+        # self.workers.set(0.)
+
+        self.write(self.prometheus_client.generate_latest())
+
+
+routes = [
+        (r'metrics', PrometheusHandler),
+]
+
+
+def get_handlers(server):
+    return [(url, cls, {'server': server}) for url, cls in routes]

From be472ca7d6860fde92e383154af859f6d4e29e69 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sat, 16 Feb 2019 16:20:47 -0800
Subject: [PATCH 0202/1550] Use pytest.importorskip for prometheus test (#2533)

See https://github.com/cloudpipe/cloudpickle/pull/246#issuecomment-464365951
---
 distributed/bokeh/tests/test_scheduler_bokeh_html.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/distributed/bokeh/tests/test_scheduler_bokeh_html.py b/distributed/bokeh/tests/test_scheduler_bokeh_html.py
index a372b42b9a7..c2e5d3220ed 100644
--- a/distributed/bokeh/tests/test_scheduler_bokeh_html.py
+++ b/distributed/bokeh/tests/test_scheduler_bokeh_html.py
@@ -66,6 +66,7 @@ def test_prefix(c, s, a, b):
              check_new_threads=False,
              scheduler_kwargs={'services': {('bokeh', 0):  BokehScheduler}})
 def test_prometheus(c, s, a, b):
+    pytest.importorskip('prometheus_client')
     http_client = AsyncHTTPClient()
     response = yield http_client.fetch('http://localhost:%d/metrics'
                                        % s.services['bokeh'].port)

From 435a634841c6f432076b7c01457615037ccb317a Mon Sep 17 00:00:00 2001
From: Olivier Grisel <olivier.grisel@ensta.org>
Date: Sun, 17 Feb 2019 18:43:24 +0100
Subject: [PATCH 0203/1550] MAINT skip prometheus test when no installed
 (#2534)

---
 distributed/bokeh/tests/test_worker_bokeh.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/distributed/bokeh/tests/test_worker_bokeh.py b/distributed/bokeh/tests/test_worker_bokeh.py
index ccc0ba22cba..32c14d4fa50 100644
--- a/distributed/bokeh/tests/test_worker_bokeh.py
+++ b/distributed/bokeh/tests/test_worker_bokeh.py
@@ -124,6 +124,7 @@ def test_CommunicatingStream(c, s, a, b):
              check_new_threads=False,
              worker_kwargs={'services': {('bokeh', 0):  BokehWorker}})
 def test_prometheus(c, s, a, b):
+    pytest.importorskip('prometheus_client')
     assert s.workers[a.address].services == {'bokeh': a.services['bokeh'].port}
 
     http_client = AsyncHTTPClient()

From 53dd9e441fbbb0bab2ab5f9d18f35293d9a8bbda Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sun, 17 Feb 2019 18:44:49 -0800
Subject: [PATCH 0204/1550] Fix intermittent testing failures (#2535)

* Fix statistical profiling test

We bound the time by before/after rather than test equality

* print rather than err on gc.callbacks
---
 distributed/tests/test_utils_perf.py |  3 ++-
 distributed/tests/test_worker.py     | 12 +++++-------
 2 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/distributed/tests/test_utils_perf.py b/distributed/tests/test_utils_perf.py
index fea35482638..e8e4cae9e37 100644
--- a/distributed/tests/test_utils_perf.py
+++ b/distributed/tests/test_utils_perf.py
@@ -70,7 +70,8 @@ def check_fraction(timer, ft):
 @contextlib.contextmanager
 def enable_gc_diagnosis_and_log(diag, level='INFO'):
     disable_gc_diagnosis(force=True)  # just in case
-    assert not gc.callbacks
+    if gc.callbacks:
+        print("Unexpected gc.callbacks", gc.callbacks)
 
     with captured_logger('distributed.utils_perf', level=level,
                          propagate=False) as sio:
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 5a72f8d21cc..662c666cb4e 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -1040,7 +1040,7 @@ def f():
     yield wait(futures)
 
 
-@gen_cluster(client=True, worker_kwargs={'profile_cycle_interval': '10 ms'})
+@gen_cluster(client=True, worker_kwargs={'profile_cycle_interval': '50 ms'})
 def test_statistical_profiling_cycle(c, s, a, b):
     futures = c.map(slowinc, range(20), delay=0.05)
     yield wait(futures)
@@ -1051,12 +1051,10 @@ def test_statistical_profiling_cycle(c, s, a, b):
     x = a.get_profile(start=time() + 10, stop=time() + 20)
     assert not x['count']
 
-    for i in range(5):  # there is a chance that this will be slightly off. Try a few times
-        x = a.get_profile(start=0, stop=time())
-        if x['count'] == sum(p['count'] for _, p in a.profile_history) + a.profile_recent['count']:
-            break
-    else:
-        raise Exception(x['count'], sum(p['count'] for _, p in a.profile_history) + a.profile_recent['count'])
+    x = a.get_profile(start=0, stop=time())
+    actual = sum(p['count'] for _, p in a.profile_history) + a.profile_recent['count']
+    x2 = a.get_profile(start=0, stop=time())
+    assert x['count'] <= actual <= x2['count']
 
     y = a.get_profile(start=end - 0.300, stop=time())
     assert 0 < y['count'] <= x['count']

From 78a1a347b84ce6835e5b49e4b0a83236cf35a20a Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 18 Feb 2019 09:01:03 -0800
Subject: [PATCH 0205/1550] Avoid using nprocs keyword in dask-ssh if set to
 one (#2531)

It's complex and gets in the way.  It also assumes that the underlying
dask-worker command supports it (which may not be universally true as we allow
more dask-worker commands).

Concretely my motivation for this is for dask-cuda-worker
---
 distributed/deploy/ssh.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/distributed/deploy/ssh.py b/distributed/deploy/ssh.py
index b37fd1b9aa8..d0d6813083d 100644
--- a/distributed/deploy/ssh.py
+++ b/distributed/deploy/ssh.py
@@ -219,7 +219,8 @@ def start_worker(logdir, scheduler_addr, scheduler_port, worker_addr, nthreads,
 
     cmd = ('{python} -m {remote_dask_worker} '
            '{scheduler_addr}:{scheduler_port} '
-           '--nthreads {nthreads} --nprocs {nprocs} ')
+           '--nthreads {nthreads}'
+           + ('--nprocs {nprocs}' if nprocs != 1 else ''))
 
     if not nohost:
         cmd += ' --host {worker_addr} '

From 92f403a92ee4682177143291e337186aa668ab47 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 25 Feb 2019 07:55:09 -0800
Subject: [PATCH 0206/1550] bump version to 1.26.0

---
 docs/source/changelog.rst | 21 +++++++++++++++++++++
 requirements.txt          |  2 +-
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index 6e823f1027c..e5256806b02 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,27 @@
 Changelog
 =========
 
+1.26.0 - 2019-02-25
+-------------------
+
+-  Update style to fix recent flake8 update (#2500) (#2509) `Matthew Rocklin`_
+-  Fix typo in gen_cluster log message (#2503) `Loïc Estève`_
+-  Allow KeyError when closing event loop (#2498) `Matthew Rocklin`_
+-  Avoid thread testing for TCP ThreadPoolExecutor (#2510) `Matthew Rocklin`_
+-  Find Futures inside SubgraphCallable (#2505) `Jim Crist`_
+-  Avoid AttributeError when closing and sending a message (#2514) `Matthew Rocklin`_
+-  Add deprecation warning to dask_mpi.py (#2522) `jukent`_
+-  Relax statistical profiling test (#2527) `Matthew Rocklin`_
+-  Support alternative --remote-dask-worker SSHCluster() and dask-ssh CLI (#2526) `Adam Beberg`_
+-  Iterate over full list of plugins in transition (#2518) `Matthew Rocklin`_
+-  Create Prometheus Endpoint (#2499) `Adam Beberg`_
+-  Use pytest.importorskip for prometheus test (#2533) `Matthew Rocklin`_
+-  MAINT skip prometheus test when no installed (#2534) `Olivier Grisel`_
+-  Fix intermittent testing failures (#2535) `Matthew Rocklin`_
+-  Avoid using nprocs keyword in dask-ssh if set to one (#2531)  `Matthew Rocklin`_
+-  Bump minimum Tornado version to 5.0
+
+
 1.25.3 - 2019-01-31
 -------------------
 
diff --git a/requirements.txt b/requirements.txt
index 212e97df92d..a6c6b0f62f6 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -7,7 +7,7 @@ six
 sortedcontainers !=2.0.0, !=2.0.1
 tblib
 toolz >= 0.7.4
-tornado >= 4.5.1
+tornado >= 5
 zict >= 0.1.3
 # Compatibility packages
 futures; python_version < '3.0'

From 8662e34265df70689cff365760bf78e726ed2def Mon Sep 17 00:00:00 2001
From: George Sakkis <george.sakkis@gmail.com>
Date: Mon, 25 Feb 2019 18:02:09 +0200
Subject: [PATCH 0207/1550] Fix LocalCluster to not overallocate memory when
 overcommitting threads per worker (#2541)

Determine memory_limit in LocalCluster to avoid overallocation when overcommitting threads per worker
---
 distributed/deploy/local.py            |  4 +++-
 distributed/deploy/tests/test_local.py | 14 ++++++++------
 distributed/worker.py                  |  4 ++--
 3 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index 72a12cfaced..95cd7016011 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -18,7 +18,7 @@
         log_errors, thread_state, parse_timedelta)
 from ..nanny import Nanny
 from ..scheduler import Scheduler
-from ..worker import Worker, _ncores
+from ..worker import Worker, parse_memory_limit, _ncores
 
 logger = logging.getLogger(__name__)
 
@@ -110,6 +110,8 @@ def __init__(self, n_workers=None, threads_per_worker=None, processes=True,
         if n_workers and threads_per_worker is None:
             # Overcommit threads per worker, rather than undercommit
             threads_per_worker = max(1, int(math.ceil(_ncores / n_workers)))
+        if n_workers and 'memory_limit' not in worker_kwargs:
+            worker_kwargs['memory_limit'] = parse_memory_limit('auto', 1, n_workers)
 
         worker_kwargs.update({
             'ncores': threads_per_worker,
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index 26d3d28784a..d873aec5190 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -328,15 +328,17 @@ def test_remote_access(loop):
         sync(loop, assert_can_connect_from_everywhere_4_6, c.scheduler.port)
 
 
-def test_memory(loop):
-    with LocalCluster(scheduler_port=0, processes=False, silence_logs=False,
-                      diagnostics_port=None, loop=loop) as cluster:
+@pytest.mark.parametrize('n_workers', [None, 3])
+def test_memory(loop, n_workers):
+    with LocalCluster(n_workers=n_workers, scheduler_port=0, processes=False,
+                      silence_logs=False, diagnostics_port=None, loop=loop) as cluster:
         assert sum(w.memory_limit for w in cluster.workers) <= TOTAL_MEMORY
 
 
-def test_memory_nanny(loop):
-    with LocalCluster(scheduler_port=0, processes=True, silence_logs=False,
-                      diagnostics_port=None, loop=loop) as cluster:
+@pytest.mark.parametrize('n_workers', [None, 3])
+def test_memory_nanny(loop, n_workers):
+    with LocalCluster(n_workers=n_workers, scheduler_port=0, processes=True,
+                      silence_logs=False, diagnostics_port=None, loop=loop) as cluster:
         with Client(cluster.scheduler_address, loop=loop) as c:
             info = c.scheduler_info()
             assert (sum(w['memory_limit'] for w in info['workers'].values())
diff --git a/distributed/worker.py b/distributed/worker.py
index 12b41ca12af..1aba8b5c783 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -2627,11 +2627,11 @@ class Reschedule(Exception):
     pass
 
 
-def parse_memory_limit(memory_limit, ncores):
+def parse_memory_limit(memory_limit, ncores, total_cores=_ncores):
     if memory_limit is None:
         return None
     if memory_limit == 'auto':
-        memory_limit = int(TOTAL_MEMORY * min(1, ncores / _ncores))
+        memory_limit = int(TOTAL_MEMORY * min(1, ncores / total_cores))
     with ignoring(ValueError, TypeError):
         x = float(memory_limit)
         if isinstance(x, float) and x <= 1:

From 716142e843fb5c196c6fa7382dbb3d287799537b Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 25 Feb 2019 11:19:57 -0800
Subject: [PATCH 0208/1550] Make closing resilient to lacking an address
 (#2542)

Previously we were getting this intermittent error

    File "/home/travis/build/dask/distributed/distributed/worker.py", line 832, in _close
        logger.info("Stopping worker at %s", self.address)
    File "/home/travis/build/dask/distributed/distributed/core.py", line 228, in address
        raise ValueError("cannot get address of non-running Server")
---
 distributed/worker.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/distributed/worker.py b/distributed/worker.py
index 1aba8b5c783..46114ffc95c 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -829,7 +829,10 @@ def _close(self, report=True, timeout=10, nanny=True, executor_wait=True):
 
             disable_gc_diagnosis()
 
-            logger.info("Stopping worker at %s", self.address)
+            try:
+                logger.info("Stopping worker at %s", self.address)
+            except ValueError:  # address not available if already closed
+                logger.info("Stopping worker")
             self.status = 'closing'
             setproctitle("dask-worker [closing]")
 

From df4b70cdcf667960db5d28166c392145c7aa26c8 Mon Sep 17 00:00:00 2001
From: Brett Jurman <i.be.brett@gmail.com>
Date: Mon, 4 Mar 2019 09:16:18 -0500
Subject: [PATCH 0209/1550] fix typo in comment (#2546)

---
 distributed/worker.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/worker.py b/distributed/worker.py
index 46114ffc95c..645ef40c16e 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -232,7 +232,7 @@ class Worker(ServerNode):
         Fraction of memory at which we stop running new tasks
     executor: concurrent.futures.Executor
     resources: dict
-        Resources that thiw worker has like ``{'GPU': 2}``
+        Resources that this worker has like ``{'GPU': 2}``
 
     Examples
     --------

From b800a35e2a203315d6ead65750ed4ba12c747336 Mon Sep 17 00:00:00 2001
From: Marco Neumann <marco@crepererum.net>
Date: Mon, 4 Mar 2019 18:09:16 +0000
Subject: [PATCH 0210/1550] Fix double init of prometheus metrics (#2544)

Tornado handlers are initaliazed multiple times within the same
python interpreter process (once per request). Adding new prometheus
gauges/counters every time leads to `ValueError: Duplicated timeseries`
errors because `prometheus_client` has one single, global registry to
track all known metrics. Instead, use the collector mechanism which is
the recommended way to generate metrics.

Also, enable prometheus tests for worker and add `prometheus_client` to
the official dev requirements.
---
 dev-requirements.txt                          |  1 +
 distributed/bokeh/scheduler_html.py           | 47 +++++++++++++++----
 .../bokeh/tests/test_scheduler_bokeh_html.py  | 20 ++++++--
 .../bokeh/tests/test_worker_bokeh_html.py     | 30 ++++++++++++
 distributed/bokeh/worker_html.py              | 47 +++++++++++++++----
 5 files changed, 124 insertions(+), 21 deletions(-)
 create mode 100644 distributed/bokeh/tests/test_worker_bokeh_html.py

diff --git a/dev-requirements.txt b/dev-requirements.txt
index 295fed1f221..7d684343ca7 100644
--- a/dev-requirements.txt
+++ b/dev-requirements.txt
@@ -9,3 +9,4 @@ ipython >= 5.0.0
 jupyter_client >= 4.4.0
 ipykernel >= 4.5.2
 pytest >= 3.0.5
+prometheus_client >= 0.6.0
diff --git a/distributed/bokeh/scheduler_html.py b/distributed/bokeh/scheduler_html.py
index 10e84bd4b31..e8b030156dd 100644
--- a/distributed/bokeh/scheduler_html.py
+++ b/distributed/bokeh/scheduler_html.py
@@ -163,24 +163,51 @@ def get(self):
         self.write(result)
 
 
+class _PrometheusCollector(object):
+    def __init__(self, server, prometheus_client):
+        self.server = server
+        self.prometheus_client = prometheus_client
+
+    def collect(self):
+        yield self.prometheus_client.core.GaugeMetricFamily(
+            'dask_scheduler_workers',
+            'Number of workers.',
+            value=len(self.server.workers),
+        )
+        yield self.prometheus_client.core.GaugeMetricFamily(
+            'dask_scheduler_clients',
+            'Number of clients.',
+            value=len(self.server.clients),
+        )
+
+
 class PrometheusHandler(RequestHandler):
+    _initialized = False
+
     def __init__(self, *args, **kwargs):
-        import prometheus_client # keep out of global namespace
+        import prometheus_client  # keep out of global namespace
         self.prometheus_client = prometheus_client
 
         super(PrometheusHandler, self).__init__(*args, **kwargs)
-        self.workers = self.prometheus_client.Gauge('workers_total',
-            'Total number of workers.',
-            namespace='scheduler')
-        self.clients = self.prometheus_client.Gauge('clients_total',
-            'Total number of clients.',
-            namespace='scheduler')
 
-    def get(self):
-        self.workers.set(len(self.server.workers))
-        self.clients.set(len(self.server.clients))
+        self._init()
+
+    def _init(self):
+        if PrometheusHandler._initialized:
+            return
+
+        self.prometheus_client.REGISTRY.register(
+            _PrometheusCollector(
+                self.server,
+                self.prometheus_client,
+            )
+        )
 
+        PrometheusHandler._initialized = True
+
+    def get(self):
         self.write(self.prometheus_client.generate_latest())
+        self.set_header('Content-Type', 'text/plain; version=0.0.4')
 
 
 routes = [
diff --git a/distributed/bokeh/tests/test_scheduler_bokeh_html.py b/distributed/bokeh/tests/test_scheduler_bokeh_html.py
index c2e5d3220ed..5f8b55a531d 100644
--- a/distributed/bokeh/tests/test_scheduler_bokeh_html.py
+++ b/distributed/bokeh/tests/test_scheduler_bokeh_html.py
@@ -67,7 +67,21 @@ def test_prefix(c, s, a, b):
              scheduler_kwargs={'services': {('bokeh', 0):  BokehScheduler}})
 def test_prometheus(c, s, a, b):
     pytest.importorskip('prometheus_client')
+    from prometheus_client.parser import text_string_to_metric_families
+
     http_client = AsyncHTTPClient()
-    response = yield http_client.fetch('http://localhost:%d/metrics'
-                                       % s.services['bokeh'].port)
-    assert response.code == 200
+
+    # request data twice since there once was a case where metrics got registered multiple times resulting in
+    # prometheus_client errors
+    for _ in range(2):
+        response = yield http_client.fetch('http://localhost:%d/metrics'
+                                           % s.services['bokeh'].port)
+        assert response.code == 200
+        assert response.headers['Content-Type'] == 'text/plain; version=0.0.4'
+
+        txt = response.body.decode('utf8')
+        families = {
+            familiy.name
+            for familiy in text_string_to_metric_families(txt)
+        }
+        assert 'dask_scheduler_workers' in families
diff --git a/distributed/bokeh/tests/test_worker_bokeh_html.py b/distributed/bokeh/tests/test_worker_bokeh_html.py
new file mode 100644
index 00000000000..0a56e97e055
--- /dev/null
+++ b/distributed/bokeh/tests/test_worker_bokeh_html.py
@@ -0,0 +1,30 @@
+import pytest
+pytest.importorskip('bokeh')
+
+from tornado.httpclient import AsyncHTTPClient
+from distributed.utils_test import gen_cluster
+from distributed.bokeh.worker import BokehWorker
+
+
+@gen_cluster(client=True,
+             worker_kwargs={'services': {('bokeh', 0):  BokehWorker}})
+def test_prometheus(c, s, a, b):
+    pytest.importorskip('prometheus_client')
+    from prometheus_client.parser import text_string_to_metric_families
+
+    http_client = AsyncHTTPClient()
+
+    # request data twice since there once was a case where metrics got registered multiple times resulting in
+    # prometheus_client errors
+    for _ in range(2):
+        response = yield http_client.fetch('http://localhost:%d/metrics'
+                                           % a.services['bokeh'].port)
+        assert response.code == 200
+        assert response.headers['Content-Type'] == 'text/plain; version=0.0.4'
+
+        txt = response.body.decode('utf8')
+        families = {
+            familiy.name
+            for familiy in text_string_to_metric_families(txt)
+        }
+        assert len(families) > 0
diff --git a/distributed/bokeh/worker_html.py b/distributed/bokeh/worker_html.py
index 4428d19957c..a3c24b085fc 100644
--- a/distributed/bokeh/worker_html.py
+++ b/distributed/bokeh/worker_html.py
@@ -14,22 +14,53 @@ def get_template_path(self):
         return os.path.join(dirname, 'templates')
 
 
+class _PrometheusCollector(object):
+    def __init__(self, server, prometheus_client):
+        self.server = server
+        self.prometheus_client = prometheus_client
+
+    def collect(self):
+        # add your metrics here:
+        #
+        # 1. remove the following lines
+        while False:
+            yield None
+        #
+        # 2. yield your metrics
+        #     yield self.prometheus_client.core.GaugeMetricFamily(
+        #         'dask_worker_connections',
+        #         'Number of connections currently open.',
+        #         value=???,
+        #     )
+
+
 class PrometheusHandler(RequestHandler):
+    _initialized = False
+
     def __init__(self, *args, **kwargs):
-        import prometheus_client # keep out of global namespace
+        import prometheus_client  # keep out of global namespace
         self.prometheus_client = prometheus_client
 
         super(PrometheusHandler, self).__init__(*args, **kwargs)
-        # Add metrics like this:
-        # self.workers = self.prometheus_client.Gauge('memory_bytes',
-        #    'Total memory.',
-        #    namespace='worker')
 
-    def get(self):
-        # Example metric update
-        # self.workers.set(0.)
+        self._init()
+
+    def _init(self):
+        if PrometheusHandler._initialized:
+            return
+
+        self.prometheus_client.REGISTRY.register(
+            _PrometheusCollector(
+                self.server,
+                self.prometheus_client,
+            )
+        )
 
+        PrometheusHandler._initialized = True
+
+    def get(self):
         self.write(self.prometheus_client.generate_latest())
+        self.set_header('Content-Type', 'text/plain; version=0.0.4')
 
 
 routes = [

From ae18f6597619c2c365c46bc0ff5671444b713c7e Mon Sep 17 00:00:00 2001
From: Elliott Sales de Andrade <quantum.analyst@gmail.com>
Date: Thu, 7 Mar 2019 09:12:43 -0500
Subject: [PATCH 0211/1550] TST: Skip test_duplicate_clients without bokeh.
 (#2553)

Just like the other tests in this file that use bokeh.
---
 distributed/deploy/tests/test_local.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index d873aec5190..3edd283b5ad 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -146,6 +146,7 @@ def test_Client_solo(loop):
 
 @gen_test()
 def test_duplicate_clients():
+    pytest.importorskip('bokeh')
     c1 = yield Client(processes=False, silence_logs=False, diagnostics_port=9876)
     with pytest.warns(Exception) as info:
         c2 = yield Client(processes=False, silence_logs=False, diagnostics_port=9876)

From 8e843cdc2c3ebc896f6ba158e1a47b1d6264fb4a Mon Sep 17 00:00:00 2001
From: Chris White <white.cdw@gmail.com>
Date: Mon, 11 Mar 2019 08:57:02 -0700
Subject: [PATCH 0212/1550] Add blocked_handlers to servers (#2556)

This lets users opt out of handlers, which is particularly useful for security concerns
---
 distributed/core.py                    | 19 +++++++++++++++++--
 distributed/deploy/local.py            |  8 ++++++--
 distributed/deploy/tests/test_local.py |  9 +++++++++
 distributed/distributed.yaml           |  2 ++
 distributed/node.py                    |  3 ++-
 distributed/tests/test_core.py         | 20 ++++++++++++++++++++
 distributed/tests/test_scheduler.py    | 25 +++++++++++++++++++++++++
 7 files changed, 81 insertions(+), 5 deletions(-)

diff --git a/distributed/core.py b/distributed/core.py
index bf08abebc8c..a9883801549 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -43,6 +43,12 @@ def get_total_physical_memory():
         return 2e9
 
 
+def raise_later(exc):
+    def _raise(*args, **kwargs):
+        raise exc
+    return _raise
+
+
 MAX_BUFFER_SIZE = get_total_physical_memory()
 
 tick_maximum_delay = parse_timedelta(dask.config.get('distributed.admin.tick.limit'), default='ms')
@@ -89,13 +95,16 @@ class Server(object):
     default_ip = ''
     default_port = 0
 
-    def __init__(self, handlers, stream_handlers=None, connection_limit=512,
+    def __init__(self, handlers, blocked_handlers=None, stream_handlers=None, connection_limit=512,
                  deserialize=True, io_loop=None):
         self.handlers = {
             'identity': self.identity,
             'connection_stream': self.handle_stream,
         }
         self.handlers.update(handlers)
+        if blocked_handlers is None:
+            blocked_handlers = dask.config.get('distributed.%s.blocked-handlers' % type(self).__name__.lower(), [])
+        self.blocked_handlers = blocked_handlers
         self.stream_handlers = {}
         self.stream_handlers.update(stream_handlers or {})
 
@@ -330,7 +339,13 @@ def handle_comm(self, comm, shutting_down=shutting_down):
 
                 result = None
                 try:
-                    handler = self.handlers[op]
+                    if op in self.blocked_handlers:
+                        _msg = ("The '{op}' handler has been explicitly disallowed "
+                                "in {obj}, possibly due to security concerns.")
+                        exc = ValueError(_msg.format(op=op, obj=type(self).__name__))
+                        handler = raise_later(exc)
+                    else:
+                        handler = self.handlers[op]
                 except KeyError:
                     logger.warning("No handler %s found in %s", op,
                                    type(self).__name__, exc_info=True)
diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index 95cd7016011..6d749be9ba5 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -54,6 +54,9 @@ class LocalCluster(Cluster):
         Tornado gen.coroutines.  This should remain False for normal use.
     kwargs: dict
         Extra worker arguments, will be passed to the Worker constructor.
+    blocked_handlers: List[str]
+        A list of strings specifying a blacklist of handlers to disallow on the Scheduler,
+        like ``['feed', 'run_function']``
     service_kwargs: Dict[str, Dict]
         Extra keywords to hand to the running services
     security : Security
@@ -82,7 +85,7 @@ def __init__(self, n_workers=None, threads_per_worker=None, processes=True,
                  loop=None, start=None, ip=None, scheduler_port=0,
                  silence_logs=logging.WARN, diagnostics_port=8787,
                  services=None, worker_services=None, service_kwargs=None,
-                 asynchronous=False, security=None, **worker_kwargs):
+                 asynchronous=False, security=None, blocked_handlers=None, **worker_kwargs):
         if start is not None:
             msg = ("The start= parameter is deprecated. "
                    "LocalCluster always starts. "
@@ -133,7 +136,8 @@ def __init__(self, n_workers=None, threads_per_worker=None, processes=True,
 
         self.scheduler = Scheduler(loop=self.loop,
                                    services=services,
-                                   security=security)
+                                   security=security,
+                                   blocked_handlers=blocked_handlers)
         self.scheduler_port = scheduler_port
 
         self.workers = []
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index 3edd283b5ad..c5934148a03 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -41,6 +41,15 @@ def test_simple(loop):
             assert e.loop is c.loop
 
 
+def test_local_cluster_supports_blocked_handlers(loop):
+    with LocalCluster(blocked_handlers=['run_function'], loop=loop) as c:
+        with Client(c) as client:
+            with pytest.raises(ValueError) as exc:
+                client.run_on_scheduler(lambda x: x, 42)
+
+    assert "'run_function' handler has been explicitly disallowed in Scheduler" in str(exc.value)
+
+
 @pytest.mark.skipif('sys.version_info[0] == 2', reason='fork issues')
 def test_close_twice():
     with LocalCluster() as cluster:
diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index f6bf5e6fda8..156c90a127e 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -11,6 +11,7 @@ distributed:
   scheduler:
     allowed-failures: 3     # number of retries before a task is considered bad
     bandwidth: 100000000    # 100 MB/s estimated worker-worker bandwidth
+    blocked-handlers: []
     default-data-size: 1000
     # Number of seconds to wait until workers or clients are removed from the events log
     # after they have been removed from the scheduler
@@ -22,6 +23,7 @@ distributed:
     preload-argv: []
 
   worker:
+    blocked-handlers: []
     multiprocessing-method: forkserver
     use-file-locking: True
     connections:            # Maximum concurrent connections for data
diff --git a/distributed/node.py b/distributed/node.py
index 654d67f376c..4123617620b 100644
--- a/distributed/node.py
+++ b/distributed/node.py
@@ -31,7 +31,7 @@ class ServerNode(Node, Server):
     # XXX avoid inheriting from Server? there is some large potential for confusion
     # between base and derived attribute namespaces...
 
-    def __init__(self, handlers=None, stream_handlers=None,
+    def __init__(self, handlers=None, blocked_handlers=None, stream_handlers=None,
                  connection_limit=512, deserialize=True,
                  connection_args=None, io_loop=None, serializers=None,
                  deserializers=None):
@@ -42,6 +42,7 @@ def __init__(self, handlers=None, stream_handlers=None,
                       serializers=serializers,
                       deserializers=deserializers)
         Server.__init__(self, handlers=handlers,
+                        blocked_handlers=blocked_handlers,
                         stream_handlers=stream_handlers,
                         connection_limit=connection_limit,
                         deserialize=deserialize, io_loop=self.io_loop)
diff --git a/distributed/tests/test_core.py b/distributed/tests/test_core.py
index 41e816c1e4a..07cb4e2214a 100644
--- a/distributed/tests/test_core.py
+++ b/distributed/tests/test_core.py
@@ -96,6 +96,26 @@ def f():
     loop.run_sync(f)
 
 
+def test_server_raises_on_blocked_handlers(loop):
+    @gen.coroutine
+    def f():
+        server = Server({'ping': pingpong}, blocked_handlers=['ping'])
+        server.listen(8881)
+
+        comm = yield connect(server.address)
+        yield comm.write({'op': 'ping'})
+        msg = yield comm.read()
+
+        assert 'exception' in msg
+        assert isinstance(msg['exception'], ValueError)
+        assert "'ping' handler has been explicitly disallowed" in repr(msg['exception'])
+
+        comm.close()
+        server.stop()
+
+    res = loop.run_sync(f)
+
+
 class MyServer(Server):
     default_port = 8756
 
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 94f95745ef6..a739e43a310 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -252,6 +252,31 @@ def test_add_worker(s, a, b):
     yield w._close()
 
 
+@gen_cluster(scheduler_kwargs={'blocked_handlers': ['feed']})
+def test_blocked_handlers_are_respected(s, a, b):
+    def func(scheduler):
+        return dumps(dict(scheduler.worker_info))
+
+    comm = yield connect(s.address)
+    yield comm.write({'op': 'feed',
+                      'function': dumps(func),
+                      'interval': 0.01})
+
+    response = yield comm.read()
+
+    assert 'exception' in response
+    assert isinstance(response['exception'], ValueError)
+    assert "'feed' handler has been explicitly disallowed" in repr(response['exception'])
+
+    yield comm.close()
+
+
+def test_scheduler_init_pulls_blocked_handlers_from_config():
+    with dask.config.set({'distributed.scheduler.blocked-handlers': ['test-handler']}):
+        s = Scheduler()
+    assert s.blocked_handlers == ['test-handler']
+
+
 @gen_cluster()
 def test_feed(s, a, b):
     def func(scheduler):

From fb30c33562862f30864456766424b44a3e91aa5b Mon Sep 17 00:00:00 2001
From: Tom Augspurger <TomAugspurger@users.noreply.github.com>
Date: Mon, 11 Mar 2019 17:52:19 -0500
Subject: [PATCH 0213/1550] Always yield Server.handle_comm coroutine (#2559)

Without this change, I see the following when running [this](https://github.com/TomAugspurger/dask-perf/blob/master/bench_comm.py)
comm benchmark.

```
(ucx-dev) an.taugspurger@dgx05:~/ucx-dev-env/dask-perf$ python bench_comm.py
/home/nfs/an.taugspurger/ucx-dev-env/distributed/distributed/comm/tcp.py:436: RuntimeWarning: coroutine 'server_handle_comm' was never awaited
  self.comm_handler(comm)
```
---
 distributed/comm/tcp.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py
index f538ea81c72..f10af0bc167 100644
--- a/distributed/comm/tcp.py
+++ b/distributed/comm/tcp.py
@@ -27,7 +27,7 @@
 from .addressing import parse_host_port, unparse_host_port
 from .core import Comm, Connector, Listener, CommClosedError, FatalCommClosedError
 from .utils import (to_frames, from_frames,
-                    get_tcp_server_address, ensure_concrete_host)
+                    get_tcp_server_address, ensure_concrete_host,)
 
 
 logger = logging.getLogger(__name__)
@@ -433,7 +433,7 @@ def _handle_stream(self, stream, address):
                      address, self.contact_address)
         local_address = self.prefix + get_stream_address(stream)
         comm = self.comm_class(stream, local_address, address, self.deserialize)
-        self.comm_handler(comm)
+        yield self.comm_handler(comm)
 
     def get_host_port(self):
         """

From 09d70f682e8785762e4e6eaac2f9ebcbfe991467 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 18 Mar 2019 12:13:42 -0700
Subject: [PATCH 0214/1550] Use yaml.safe_load (#2566)

---
 distributed/config.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/config.py b/distributed/config.py
index a1fc7444e1b..d2b27397393 100644
--- a/distributed/config.py
+++ b/distributed/config.py
@@ -17,7 +17,7 @@
 dask.config.ensure_file(source=fn)
 
 with open(fn) as f:
-    defaults = yaml.load(f)
+    defaults = yaml.safe_load(f)
 
 dask.config.update_defaults(defaults)
 

From 393cdf1c24e5caf9926b9f6f8e76042d481914e8 Mon Sep 17 00:00:00 2001
From: Elliott Sales de Andrade <quantum.analyst@gmail.com>
Date: Mon, 18 Mar 2019 22:21:42 -0400
Subject: [PATCH 0215/1550] TST: Fetch executables from build root. (#2551)

---
 distributed/utils_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index 6763eb18027..11f4df047e9 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -992,7 +992,7 @@ def popen(args, **kwargs):
     if sys.platform.startswith('win'):
         args[0] = os.path.join(sys.prefix, 'Scripts', args[0])
     else:
-        args[0] = os.path.join(sys.prefix, 'bin', args[0])
+        args[0] = os.path.join(os.environ.get('DESTDIR', '') + sys.prefix, 'bin', args[0])
     proc = subprocess.Popen(args, **kwargs)
     try:
         yield proc

From 68f3a6b9d1cd2cc04427306091813247c665c3e5 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 18 Mar 2019 21:08:48 -0700
Subject: [PATCH 0216/1550] Fix Torando 6 test failures (#2570)

* Some tests were faulty when the IOLoop had been run before being used a second time.

* include write in test_tls_reject_certificate

* Pin tornado in travis-ci
---
 .travis.yml                                  | 6 +++---
 continuous_integration/travis/install.sh     | 6 +-----
 distributed/cli/tests/test_dask_scheduler.py | 7 ++-----
 distributed/comm/tests/test_comms.py         | 3 ++-
 4 files changed, 8 insertions(+), 14 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 5963855c59b..c8420a9c1bf 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -7,9 +7,9 @@ dist: trusty
 env:
   matrix:
     - PYTHON=2.7 TESTS=true PACKAGES="python-blosc futures faulthandler lz4"
-    - PYTHON=3.5.4 TESTS=true COVERAGE=true PACKAGES="python-blosc lz4" CRICK=true
-    - PYTHON=3.6 TESTS=true PACKAGES="scikit-learn lz4"
-    - PYTHON=3.7 TESTS=true PACKAGES="scikit-learn python-snappy python-blosc"
+    - PYTHON=3.5.4 TESTS=true COVERAGE=true PACKAGES="python-blosc lz4" CRICK=true TORNADO=6
+    - PYTHON=3.6 TESTS=true PACKAGES="scikit-learn lz4" TORNADO=5
+    - PYTHON=3.7 TESTS=true PACKAGES="scikit-learn python-snappy python-blosc" TORNADO=6
 
 matrix:
   fast_finish: true
diff --git a/continuous_integration/travis/install.sh b/continuous_integration/travis/install.sh
index a1ce6560e9c..9b66feb3a31 100644
--- a/continuous_integration/travis/install.sh
+++ b/continuous_integration/travis/install.sh
@@ -51,7 +51,7 @@ conda install -q \
     scipy \
     tblib \
     toolz \
-    tornado \
+    tornado=${TORNADO:-5}\
     $PACKAGES
 
 pip install -q pytest-repeat pytest-faulthandler
@@ -71,10 +71,6 @@ fi;
 # Install distributed
 pip install --no-deps -e .
 
-if [[ ! -z $TORNADO ]]; then
-    pip install -U tornado==$TORNADO
-fi
-
 # For debugging
 echo -e "--\n--Conda Environment\n--"
 conda list
diff --git a/distributed/cli/tests/test_dask_scheduler.py b/distributed/cli/tests/test_dask_scheduler.py
index ac6934f3dc9..e893f0c2ba2 100644
--- a/distributed/cli/tests/test_dask_scheduler.py
+++ b/distributed/cli/tests/test_dask_scheduler.py
@@ -33,10 +33,8 @@ def f():
                 assert_can_connect_from_everywhere_4_6(8786, 5.0),  # main port
             ]
 
-        loop.run_sync(f)
-
         with Client('127.0.0.1:%d' % Scheduler.default_port, loop=loop) as c:
-            pass
+            c.sync(f)
 
     with pytest.raises(Exception):
         requests.get('http://127.0.0.1:8787/status/')
@@ -53,10 +51,9 @@ def f():
                 assert_can_connect_locally_4(8978, 5.0),
             ]
 
-        loop.run_sync(f)
-
         with Client('127.0.0.1:8978', loop=loop) as c:
             assert len(c.ncores()) == 0
+            c.sync(f)
 
 
 def test_no_bokeh(loop):
diff --git a/distributed/comm/tests/test_comms.py b/distributed/comm/tests/test_comms.py
index 0cfeb7a63a8..e4aee5805db 100644
--- a/distributed/comm/tests/test_comms.py
+++ b/distributed/comm/tests/test_comms.py
@@ -624,8 +624,9 @@ def handle_comm(comm):
     listener.start()
 
     with pytest.raises(EnvironmentError) as excinfo:
-        yield connect(listener.contact_address, timeout=0.5,
+        comm = yield connect(listener.contact_address, timeout=0.5,
                       connection_args={'ssl_context': bad_cli_ctx})
+        yield comm.write({'x': 'foo'})  # TODO: why is this necessary in Tornado 6 ?
 
     # The wrong error is reported on Python 2, see https://github.com/tornadoweb/tornado/pull/2028
     if sys.version_info >= (3,) and os.name != 'nt':

From b36e87d14e75c0cbdd0e3292ac65db2a836f74bd Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 19 Mar 2019 15:10:09 -0700
Subject: [PATCH 0217/1550] Fix test_sync_closed_loop (#2572)

* Fix test_sync_closed_loop

* relax tornado constraint in Python 3.5
---
 .travis.yml                              | 2 +-
 continuous_integration/travis/install.sh | 2 +-
 distributed/utils.py                     | 5 +++++
 3 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index c8420a9c1bf..a91f35bf037 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -7,7 +7,7 @@ dist: trusty
 env:
   matrix:
     - PYTHON=2.7 TESTS=true PACKAGES="python-blosc futures faulthandler lz4"
-    - PYTHON=3.5.4 TESTS=true COVERAGE=true PACKAGES="python-blosc lz4" CRICK=true TORNADO=6
+    - PYTHON=3.5.4 TESTS=true COVERAGE=true PACKAGES="python-blosc lz4" CRICK=true
     - PYTHON=3.6 TESTS=true PACKAGES="scikit-learn lz4" TORNADO=5
     - PYTHON=3.7 TESTS=true PACKAGES="scikit-learn python-snappy python-blosc" TORNADO=6
 
diff --git a/continuous_integration/travis/install.sh b/continuous_integration/travis/install.sh
index 9b66feb3a31..8e9d1ff7f12 100644
--- a/continuous_integration/travis/install.sh
+++ b/continuous_integration/travis/install.sh
@@ -51,7 +51,7 @@ conda install -q \
     scipy \
     tblib \
     toolz \
-    tornado=${TORNADO:-5}\
+    tornado=$TORNADO \
     $PACKAGES
 
 pip install -q pytest-repeat pytest-faulthandler
diff --git a/distributed/utils.py b/distributed/utils.py
index 812fec91db4..306106bb0a9 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -241,6 +241,11 @@ def sync(loop, func, *args, **kwargs):
     if PollIOLoop and ((isinstance(loop, PollIOLoop) and getattr(loop, '_closing', False)) or
             (hasattr(loop, 'asyncio_loop') and loop.asyncio_loop._closed)):
         raise RuntimeError("IOLoop is closed")
+    try:
+        if loop.asyncio_loop.is_closed():  # tornado 6
+            raise RuntimeError("IOLoop is closed")
+    except AttributeError:
+        pass
 
     timeout = kwargs.pop('callback_timeout', None)
 

From ef62ea42dae9471b974c834f7eb200ef52dc10f4 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 29 Mar 2019 10:41:52 -0700
Subject: [PATCH 0218/1550] bump version to 1.26.1

---
 docs/source/changelog.rst | 53 ++++++++++++++++++++++++++-------------
 1 file changed, 36 insertions(+), 17 deletions(-)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index e5256806b02..db53bd25bbf 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,24 +1,39 @@
 Changelog
 =========
 
+1.26.1 - 2019-03-29
+-------------------
+
+-  Fix LocalCluster to not overallocate memory when overcommitting threads per worker (:pr:`2541`) `George Sakkis`_
+-  Make closing resilient to lacking an address (:pr:`2542`) `Matthew Rocklin`_
+-  fix typo in comment (:pr:`2546`) `Brett Jurman`_
+-  Fix double init of prometheus metrics (:pr:`2544`) `Marco Neumann`_
+-  Skip test_duplicate_clients without bokeh. (:pr:`2553`) `Elliott Sales de Andrade`_
+-  Add blocked_handlers to servers (:pr:`2556`) `Chris White`_
+-  Always yield Server.handle_comm coroutine (:pr:`2559`) `Tom Augspurger`_
+-  Use yaml.safe_load (:pr:`2566`) `Matthew Rocklin`_
+-  Fetch executables from build root. (:pr:`2551`) `Elliott Sales de Andrade`_
+-  Fix Torando 6 test failures (:pr:`2570`) `Matthew Rocklin`_
+-  Fix test_sync_closed_loop (:pr:`2572`) `Matthew Rocklin`_
+
 1.26.0 - 2019-02-25
 -------------------
 
--  Update style to fix recent flake8 update (#2500) (#2509) `Matthew Rocklin`_
--  Fix typo in gen_cluster log message (#2503) `Loïc Estève`_
--  Allow KeyError when closing event loop (#2498) `Matthew Rocklin`_
--  Avoid thread testing for TCP ThreadPoolExecutor (#2510) `Matthew Rocklin`_
--  Find Futures inside SubgraphCallable (#2505) `Jim Crist`_
--  Avoid AttributeError when closing and sending a message (#2514) `Matthew Rocklin`_
--  Add deprecation warning to dask_mpi.py (#2522) `jukent`_
--  Relax statistical profiling test (#2527) `Matthew Rocklin`_
--  Support alternative --remote-dask-worker SSHCluster() and dask-ssh CLI (#2526) `Adam Beberg`_
--  Iterate over full list of plugins in transition (#2518) `Matthew Rocklin`_
--  Create Prometheus Endpoint (#2499) `Adam Beberg`_
--  Use pytest.importorskip for prometheus test (#2533) `Matthew Rocklin`_
--  MAINT skip prometheus test when no installed (#2534) `Olivier Grisel`_
--  Fix intermittent testing failures (#2535) `Matthew Rocklin`_
--  Avoid using nprocs keyword in dask-ssh if set to one (#2531)  `Matthew Rocklin`_
+-  Update style to fix recent flake8 update (:pr:`2500`) (:pr:`2509`) `Matthew Rocklin`_
+-  Fix typo in gen_cluster log message (:pr:`2503`) `Loïc Estève`_
+-  Allow KeyError when closing event loop (:pr:`2498`) `Matthew Rocklin`_
+-  Avoid thread testing for TCP ThreadPoolExecutor (:pr:`2510`) `Matthew Rocklin`_
+-  Find Futures inside SubgraphCallable (:pr:`2505`) `Jim Crist`_
+-  Avoid AttributeError when closing and sending a message (:pr:`2514`) `Matthew Rocklin`_
+-  Add deprecation warning to dask_mpi.py (:pr:`2522`) `Julia Kent`_
+-  Relax statistical profiling test (:pr:`2527`) `Matthew Rocklin`_
+-  Support alternative --remote-dask-worker SSHCluster() and dask-ssh CLI (:pr:`2526`) `Adam Beberg`_
+-  Iterate over full list of plugins in transition (:pr:`2518`) `Matthew Rocklin`_
+-  Create Prometheus Endpoint (:pr:`2499`) `Adam Beberg`_
+-  Use pytest.importorskip for prometheus test (:pr:`2533`) `Matthew Rocklin`_
+-  MAINT skip prometheus test when no installed (:pr:`2534`) `Olivier Grisel`_
+-  Fix intermittent testing failures (:pr:`2535`) `Matthew Rocklin`_
+-  Avoid using nprocs keyword in dask-ssh if set to one (:pr:`2531`)  `Matthew Rocklin`_
 -  Bump minimum Tornado version to 5.0
 
 
@@ -70,7 +85,7 @@ Changelog
 -  Use async-await on large messages test (:pr:`2404`) `Matthew Rocklin`_
 -  Fix race condition in normalize_collection (:pr:`2386`) `Jim Crist`_
 -  Fix redict collection after HighLevelGraph fix upstream (:pr:`2413`) `Matthew Rocklin`_
--  Add a blocking argument to Lock.acquire(`) (:pr:`2412`) `Stephan Hoyer`_
+-  Add a blocking argument to Lock.acquire() (:pr:`2412`) `Stephan Hoyer`_
 -  Fix long traceback test (:pr:`2417`) `Matthew Rocklin`_
 -  Update x509 certificates to current OpenSSL standards. (:pr:`2418`) `Diane Trout`_
 
@@ -301,7 +316,7 @@ Changelog
 -  Fix bug where queues would not clean up cleanly (:pr:`1922`) `Matthew Rocklin`_
 -  Delete cached file safely in upload file (:pr:`1921`) `Matthew Rocklin`_
 -  Accept KeyError when closing tornado IOLoop in tests (:pr:`1937`) `Matthew Rocklin`_
--  Quiet the client and scheduler when gather(..., errors='skip'`) (:pr:`1936`) `Matthew Rocklin`_
+-  Quiet the client and scheduler when gather(..., errors='skip') (:pr:`1936`) `Matthew Rocklin`_
 -  Clarify couldn't gather keys warning (:pr:`1942`) `Kenneth Koski`_
 -  Support submit keywords in joblib (:pr:`1947`) `Matthew Rocklin`_
 -  Avoid use of external resources in bokeh server (:pr:`1934`) `Matthew Rocklin`_
@@ -941,3 +956,7 @@ significantly without many new features.
 .. _`Daniel Farrell`: https://github.com/danpf
 .. _`George Sakkis`: https://github.com/gsakkis
 .. _`Adam Beberg`: https://github.com/beberg
+.. _`Marco Neumann`: https://github.com/crepererum
+.. _`Elliott Sales de Andrade`: https://github.com/QuLogic
+.. _`Brett Jurman`: https://github.com/ibebrett
+.. _`Julia Kent`: https://github.com/jukent

From 50dd21f71d763edd10146da4144a55f0ad48ca23 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 2 Apr 2019 09:29:07 -0700
Subject: [PATCH 0219/1550] Specify protocol in LocalCluster (#2489)

---
 distributed/deploy/local.py            | 40 ++++++++++++++++++++------
 distributed/deploy/tests/test_local.py | 23 +++++++++++----
 2 files changed, 49 insertions(+), 14 deletions(-)

diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index 6d749be9ba5..c9f307bb421 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -60,6 +60,10 @@ class LocalCluster(Cluster):
     service_kwargs: Dict[str, Dict]
         Extra keywords to hand to the running services
     security : Security
+    protocol: str (optional)
+        Protocol to use like ``tcp://``, ``tls://``, ``inproc://``
+        This defaults to sensible choice given other keyword arguments like
+        ``processes`` and ``security``
 
     Examples
     --------
@@ -85,7 +89,8 @@ def __init__(self, n_workers=None, threads_per_worker=None, processes=True,
                  loop=None, start=None, ip=None, scheduler_port=0,
                  silence_logs=logging.WARN, diagnostics_port=8787,
                  services=None, worker_services=None, service_kwargs=None,
-                 asynchronous=False, security=None, blocked_handlers=None, **worker_kwargs):
+                 asynchronous=False, security=None, protocol=None,
+                 blocked_handlers=None, **worker_kwargs):
         if start is not None:
             msg = ("The start= parameter is deprecated. "
                    "LocalCluster always starts. "
@@ -95,6 +100,20 @@ def __init__(self, n_workers=None, threads_per_worker=None, processes=True,
 
         self.status = None
         self.processes = processes
+
+        if protocol is None:
+            if ip and '://' in ip:
+                protocol = ip.split('://')[0]
+            elif security:
+                protocol = 'tls://'
+            elif not self.processes and not scheduler_port:
+                protocol = 'inproc://'
+            else:
+                protocol = 'tcp://'
+        if not protocol.endswith('://'):
+            protocol = protocol + '://'
+        self.protocol = protocol
+
         self.silence_logs = silence_logs
         self._asynchronous = asynchronous
         self.security = security
@@ -191,16 +210,21 @@ def _start(self, ip=None, n_workers=0):
         """
         if self.status == 'running':
             return
-        if (ip is None) and (not self.scheduler_port) and (not self.processes):
-            # Use inproc transport for optimization
-            scheduler_address = 'inproc://'
-        elif ip is not None and ip.startswith('tls://'):
-            scheduler_address = ('%s:%d' % (ip, self.scheduler_port))
+
+        if self.protocol == 'inproc://':
+            address = self.protocol
         else:
             if ip is None:
                 ip = '127.0.0.1'
-            scheduler_address = (ip, self.scheduler_port)
-        self.scheduler.start(scheduler_address)
+
+            if '://' in ip:
+                address = ip
+            else:
+                address = self.protocol + ip
+            if self.scheduler_port:
+                address += ':' + str(self.scheduler_port)
+
+        self.scheduler.start(address)
 
         yield [self._start_worker(**self.worker_kwargs) for i in range(n_workers)]
 
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index c5934148a03..b21c9e60ba2 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -186,12 +186,6 @@ def test_Client_twice(loop):
 def test_defaults():
     from distributed.worker import _ncores
 
-    with LocalCluster(scheduler_port=0, silence_logs=False,
-                      diagnostics_port=None) as c:
-        assert sum(w.ncores for w in c.workers) == _ncores
-        assert all(isinstance(w, Nanny) for w in c.workers)
-        assert all(w.ncores == 1 for w in c.workers)
-
     with LocalCluster(processes=False, scheduler_port=0, silence_logs=False,
                       diagnostics_port=None) as c:
         assert sum(w.ncores for w in c.workers) == _ncores
@@ -567,5 +561,22 @@ def _():
         cluster.sync(_)
 
 
+def test_protocol_inproc(loop):
+    with LocalCluster(protocol='inproc://', loop=loop, processes=False) as cluster:
+        assert cluster.scheduler.address.startswith('inproc://')
+
+
+def test_protocol_tcp(loop):
+    with LocalCluster(protocol='tcp', loop=loop, processes=False) as cluster:
+        assert cluster.scheduler.address.startswith('tcp://')
+
+
+@pytest.mark.skipif(not sys.platform.startswith('linux'),
+                    reason="Need 127.0.0.2 to mean localhost")
+def test_protocol_ip(loop):
+    with LocalCluster(ip='tcp://127.0.0.2', loop=loop, processes=False) as cluster:
+        assert cluster.scheduler.address.startswith('tcp://127.0.0.2')
+
+
 if sys.version_info >= (3, 5):
     from distributed.deploy.tests.py3_test_deploy import *  # noqa F401

From 785d1dcaa7630d1d710b90dbd9fa1a1db689aed0 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Wed, 3 Apr 2019 22:29:43 -0500
Subject: [PATCH 0220/1550] Use ensure_dict instead of dict (#2594)

---
 distributed/client.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/distributed/client.py b/distributed/client.py
index 65e034ff5fd..02ec82422ee 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -28,6 +28,7 @@
 from dask.core import flatten, get_dependencies
 from dask.optimization import SubgraphCallable
 from dask.compatibility import apply, unicode
+from dask.utils import ensure_dict
 try:
     from cytoolz import first, groupby, merge, valmap, keymap
 except ImportError:
@@ -2354,7 +2355,7 @@ def _optimize_insert_futures(self, dsk, keys):
                 if tokey(key) in self.futures:
                     if not changed:
                         changed = True
-                        dsk = dict(dsk)
+                        dsk = ensure_dict(dsk)
                     dsk[key] = Future(key, self, inform=False)
 
         if changed:

From b18f32a99acf8a69c2722a2b9ba8c17501b644fc Mon Sep 17 00:00:00 2001
From: Brian Chu <bc@brianchu.com>
Date: Sat, 6 Apr 2019 08:27:02 -0700
Subject: [PATCH 0221/1550] Forward kwargs through Nanny to Worker (#2596)

---
 distributed/nanny.py | 41 ++++++++++++++++++++++-------------------
 1 file changed, 22 insertions(+), 19 deletions(-)

diff --git a/distributed/nanny.py b/distributed/nanny.py
index b5769a25ffb..e2577f5846a 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -43,7 +43,7 @@ def __init__(self, scheduler_ip=None, scheduler_port=None,
             memory_limit='auto', reconnect=True, validate=False, quiet=False,
             resources=None, silence_logs=None, death_timeout=None, preload=(),
             preload_argv=[], security=None, contact_address=None,
-            listen_address=None, worker_class=None, env=None, **kwargs):
+            listen_address=None, worker_class=None, env=None, **worker_kwargs):
 
         if scheduler_file:
             cfg = json_load_robust(scheduler_file)
@@ -64,6 +64,7 @@ def __init__(self, scheduler_ip=None, scheduler_port=None,
         self.preload_argv = preload_argv
         self.Worker = Worker if worker_class is None else worker_class
         self.env = env or {}
+        self.worker_kwargs = worker_kwargs
 
         self.contact_address = contact_address
         self.memory_terminate_fraction = dask.config.get('distributed.worker.memory.terminate')
@@ -96,8 +97,7 @@ def __init__(self, scheduler_ip=None, scheduler_port=None,
                     'run': self.run}
 
         super(Nanny, self).__init__(handlers, io_loop=self.loop,
-                                    connection_args=self.connection_args,
-                                    **kwargs)
+                                    connection_args=self.connection_args)
 
         if self.memory_limit:
             pc = PeriodicCallback(self.memory_monitor, 100, io_loop=self.loop)
@@ -196,23 +196,26 @@ def instantiate(self, comm=None):
                     self._given_worker_port)
 
         if self.process is None:
+            worker_kwargs = dict(scheduler_ip=self.scheduler_addr,
+                                 ncores=self.ncores,
+                                 local_dir=self.local_dir,
+                                 services=self.services,
+                                 service_ports={'nanny': self.port},
+                                 name=self.name,
+                                 memory_limit=self.memory_limit,
+                                 reconnect=self.reconnect,
+                                 resources=self.resources,
+                                 validate=self.validate,
+                                 silence_logs=self.silence_logs,
+                                 death_timeout=self.death_timeout,
+                                 preload=self.preload,
+                                 preload_argv=self.preload_argv,
+                                 security=self.security,
+                                 contact_address=self.contact_address)
+            worker_kwargs.update(self.worker_kwargs)
             self.process = WorkerProcess(
-                worker_args=(self.scheduler_addr,),
-                worker_kwargs=dict(ncores=self.ncores,
-                                   local_dir=self.local_dir,
-                                   services=self.services,
-                                   service_ports={'nanny': self.port},
-                                   name=self.name,
-                                   memory_limit=self.memory_limit,
-                                   reconnect=self.reconnect,
-                                   resources=self.resources,
-                                   validate=self.validate,
-                                   silence_logs=self.silence_logs,
-                                   death_timeout=self.death_timeout,
-                                   preload=self.preload,
-                                   preload_argv=self.preload_argv,
-                                   security=self.security,
-                                   contact_address=self.contact_address),
+                worker_args=tuple(),
+                worker_kwargs=worker_kwargs,
                 worker_start_args=(start_arg,),
                 silence_logs=self.silence_logs,
                 on_exit=self._on_exit,

From 769c4f04c8729e8f58d17f2b911e7f676755e9f6 Mon Sep 17 00:00:00 2001
From: Nikos Tsaousis <tsanikgr@users.noreply.github.com>
Date: Mon, 8 Apr 2019 14:26:35 +0100
Subject: [PATCH 0222/1550] Allow return_when='FIRST_COMPLETED' in wait (#2598)

---
 distributed/client.py            | 42 +++++++++++++++++++-------------
 distributed/tests/test_client.py | 15 ++++++++++++
 distributed/utils.py             | 39 +++++++++++++++++++++++++++++
 3 files changed, 79 insertions(+), 17 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 02ec82422ee..47bccc7fc82 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -63,7 +63,7 @@
 from .utils import (All, sync, funcname, ignoring, queue_to_iterator,
                     tokey, log_errors, str_graph, key_split, format_bytes, asciitable,
                     thread_state, no_default, PeriodicCallback, LoopRunner,
-                    parse_timedelta, shutting_down)
+                    parse_timedelta, shutting_down, Any)
 from .versions import get_versions
 
 
@@ -3637,35 +3637,43 @@ def CompatibleExecutor(*args, **kwargs):
     raise Exception("This has been moved to the Client.get_executor() method")
 
 
+ALL_COMPLETED = 'ALL_COMPLETED'
+FIRST_COMPLETED = 'FIRST_COMPLETED'
+
+
 @gen.coroutine
-def _wait(fs, timeout=None, return_when='ALL_COMPLETED'):
+def _wait(fs, timeout=None, return_when=ALL_COMPLETED):
     if timeout is not None and not isinstance(timeout, Number):
         raise TypeError("timeout= keyword received a non-numeric value.\n"
                         "Beware that wait expects a list of values\n"
                         "  Bad:  wait(x, y, z)\n"
                         "  Good: wait([x, y, z])")
     fs = futures_of(fs)
-    if return_when == 'ALL_COMPLETED':
-        future = All({f._state.wait() for f in fs})
-        if timeout is not None:
-            future = gen.with_timeout(timedelta(seconds=timeout), future)
-        yield future
-        done, not_done = set(fs), set()
-        cancelled = [f.key for f in done
-                     if f.status == 'cancelled']
-        if cancelled:
-            raise CancelledError(cancelled)
+    if return_when == ALL_COMPLETED:
+        wait_for = All
+    elif return_when == FIRST_COMPLETED:
+        wait_for = Any
     else:
-        raise NotImplementedError("Only return_when='ALL_COMPLETED' supported")
+        raise NotImplementedError("Only return_when='ALL_COMPLETED' and 'FIRST_COMPLETED' are "
+                                  "supported")
 
-    raise gen.Return(DoneAndNotDoneFutures(done, not_done))
+    future = wait_for({f._state.wait() for f in fs})
+    if timeout is not None:
+        future = gen.with_timeout(timedelta(seconds=timeout), future)
+    yield future
 
+    done, not_done = ({fu for fu in fs if fu.status != 'pending'},
+                      {fu for fu in fs if fu.status == 'pending'})
+    cancelled = [f.key for f in done
+                 if f.status == 'cancelled']
+    if cancelled:
+        raise CancelledError(cancelled)
 
-ALL_COMPLETED = 'ALL_COMPLETED'
+    raise gen.Return(DoneAndNotDoneFutures(done, not_done))
 
 
-def wait(fs, timeout=None, return_when='ALL_COMPLETED'):
-    """ Wait until all futures are complete
+def wait(fs, timeout=None, return_when=ALL_COMPLETED):
+    """ Wait until all/any futures are finished
 
     Parameters
     ----------
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 5968ea045b8..8f55760e05a 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -609,6 +609,21 @@ def test_wait(c, s, a, b):
     assert x.status == y.status == 'finished'
 
 
+@gen_cluster(client=True)
+def test_wait_first_completed(c, s, a, b):
+    x = c.submit(slowinc, 1)
+    y = c.submit(slowinc, 1)
+    z = c.submit(inc, 2)
+
+    done, not_done = yield wait([x, y, z], return_when='FIRST_COMPLETED')
+
+    assert done == {z}
+    assert not_done == {x, y}
+    assert z.status == 'finished'
+    assert x.status == 'pending'
+    assert y.status == 'pending'
+
+
 @gen_cluster(client=True, timeout=2)
 def test_wait_timeout(c, s, a, b):
     future = c.submit(sleep, 0.3)
diff --git a/distributed/utils.py b/distributed/utils.py
index 306106bb0a9..521c211b6af 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -233,6 +233,45 @@ def quiet():
     raise gen.Return(results)
 
 
+@gen.coroutine
+def Any(args, quiet_exceptions=()):
+    """ Wait on many tasks at the same time and return when any is finished
+
+    Err once any of the tasks err.
+
+    Parameters
+    ----------
+    args: futures to wait for
+    quiet_exceptions: tuple, Exception
+        Exception types to avoid logging if they fail
+    """
+    tasks = gen.WaitIterator(*args)
+    results = [None for _ in args]
+    while not tasks.done():
+        try:
+            result = yield tasks.next()
+        except Exception:
+            @gen.coroutine
+            def quiet():
+                """ Watch unfinished tasks
+
+                Otherwise if they err they get logged in a way that is hard to
+                control.  They need some other task to watch them so that they
+                are not orphaned
+                """
+                for task in list(tasks._unfinished):
+                    try:
+                        yield task
+                    except quiet_exceptions:
+                        pass
+            quiet()
+            raise
+
+        results[tasks.current_index] = result
+        break
+    raise gen.Return(results)
+
+
 def sync(loop, func, *args, **kwargs):
     """
     Run coroutine in loop running in separate thread.

From 4c7baa4e3ce6fa604e4b52f0e44130b3cd90bf21 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lo=C3=AFc=20Est=C3=A8ve?= <loic.esteve@ymail.com>
Date: Mon, 8 Apr 2019 15:30:13 +0200
Subject: [PATCH 0223/1550] Fix typo in pub-sub doc. (#2599)

---
 distributed/pubsub.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/pubsub.py b/distributed/pubsub.py
index b17d154e177..cdbe9e95a7a 100644
--- a/distributed/pubsub.py
+++ b/distributed/pubsub.py
@@ -218,7 +218,7 @@ class Pub(object):
     transfers.  For context, most data transfer first checks with the scheduler to find which
     workers should participate, and then does direct worker-to-worker
     transfers.  This checking in with the scheduler provides some stability
-    guarnatees, but also adds in a few extra network hops.  PubSub doesn't do
+    guarantees, but also adds in a few extra network hops.  PubSub doesn't do
     this, and so is faster, but also can easily drop messages if Pubs or Subs
     disappear without notice.
 

From f50b239b8e6420fb87646f7183edaafb4b8e20be Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 8 Apr 2019 10:17:40 -0500
Subject: [PATCH 0224/1550] Add warning if creating scratch directories takes a
 long time (#2561)

This adds a context manager `warn_on_duration` which warns if an action
takes a surprisingly long time.

We then use the context manager when creating a worker's local
directory.

Fixes https://github.com/rapidsai/dask-cuda/issues/11
See also https://github.com/dask/dask-jobqueue/issues/193
---
 distributed/tests/test_utils.py | 17 ++++++++++++++++-
 distributed/utils.py            |  9 +++++++++
 distributed/worker.py           | 16 ++++++++++++----
 3 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/distributed/tests/test_utils.py b/distributed/tests/test_utils.py
index ec85b27ac7b..45de450fc44 100644
--- a/distributed/tests/test_utils.py
+++ b/distributed/tests/test_utils.py
@@ -5,6 +5,7 @@
 import io
 import socket
 import sys
+from time import sleep
 import traceback
 
 import numpy as np
@@ -20,7 +21,7 @@
                                iterator_to_queue, _maybe_complex, read_block, seek_delimiter,
                                funcname, ensure_bytes, open_port, get_ip_interface, nbytes,
                                set_thread_state, thread_state, LoopRunner,
-                               parse_bytes, parse_timedelta)
+                               parse_bytes, parse_timedelta, warn_on_duration)
 from distributed.utils_test import loop, loop_in_thread  # noqa: F401
 from distributed.utils_test import (div, has_ipv6, inc, throws, gen_test,
         captured_logger)
@@ -526,3 +527,17 @@ def throws():
         yield gen.sleep(0.1)
 
     assert 'foo1234' not in sio.getvalue()
+
+
+def test_warn_on_duration():
+    with pytest.warns(None) as record:
+        with warn_on_duration('10s', 'foo'):
+            pass
+    assert not record
+
+    with pytest.warns(None) as record:
+        with warn_on_duration('1ms', 'foo'):
+            sleep(0.100)
+
+    assert record
+    assert any('foo' in str(rec.message) for rec in record)
diff --git a/distributed/utils.py b/distributed/utils.py
index 521c211b6af..dbc27251758 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -1469,3 +1469,12 @@ def iscoroutinefunction(f):
     if sys.version_info >= (3, 5) and inspect.iscoroutinefunction(f):
         return True
     return False
+
+
+@contextmanager
+def warn_on_duration(duration, msg):
+    start = time()
+    yield
+    stop = time()
+    if stop - start > parse_timedelta(duration):
+        warnings.warn(msg, stacklevel=2)
diff --git a/distributed/worker.py b/distributed/worker.py
index 645ef40c16e..46d5ae61001 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -47,7 +47,8 @@
                     ignoring, mp_context, import_file,
                     silence_logging, thread_state, json_load_robust, key_split,
                     format_bytes, DequeHandler, PeriodicCallback,
-                    parse_bytes, parse_timedelta, iscoroutinefunction)
+                    parse_bytes, parse_timedelta, iscoroutinefunction,
+                    warn_on_duration)
 from .utils_comm import pack_data, gather_from_workers
 from .utils_perf import ThrottledGC, enable_gc_diagnosis, disable_gc_diagnosis
 
@@ -377,9 +378,16 @@ def __init__(self, scheduler_ip=None, scheduler_port=None,
         if silence_logs:
             silence_logging(level=silence_logs)
 
-        self._workspace = WorkSpace(os.path.abspath(local_dir))
-        self._workdir = self._workspace.new_work_dir(prefix='worker-')
-        self.local_dir = self._workdir.dir_path
+        with warn_on_duration(
+            '1s',
+            "Creating scratch directories is taking a surprisingly long time. "
+            "This is often due to running workers on a network file system. "
+            "Consider specifying a local-directory to point workers to write "
+            "scratch data to a local disk."
+        ):
+            self._workspace = WorkSpace(os.path.abspath(local_dir))
+            self._workdir = self._workspace.new_work_dir(prefix='worker-')
+            self.local_dir = self._workdir.dir_path
 
         self.security = security or Security()
         assert isinstance(self.security, Security)

From af1cbdb9c1b8dd56770b0e494f28eb808cdcf977 Mon Sep 17 00:00:00 2001
From: Muammar El Khatib <muammarelkhatib@gmail.com>
Date: Mon, 8 Apr 2019 10:01:22 -0700
Subject: [PATCH 0225/1550] .detach_() pytorch tensor to serialize data as
 numpy array. (#2586)

This commit is related to #2581. Tensors with requires_grad=True cannot
be converted to numpy unless they are detached from the computational
graph first. To fix that, the tensor has to be detached as a leaf using
the .detach_() method. For more information see:

https://pytorch.org/docs/stable/autograd.html#torch.Tensor.detach_
---
 distributed/protocol/tests/test_torch.py |  8 ++++++++
 distributed/protocol/torch.py            | 10 ++++++----
 2 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/distributed/protocol/tests/test_torch.py b/distributed/protocol/tests/test_torch.py
index d5d93b16f03..cac8fa05d66 100644
--- a/distributed/protocol/tests/test_torch.py
+++ b/distributed/protocol/tests/test_torch.py
@@ -31,3 +31,11 @@ def test_resnet():
     header, frames = serialize(model)
     model2 = deserialize(header, frames)
     assert str(model) == str(model2)
+
+
+def test_deserialize_grad():
+    a = np.random.rand(8, 1)
+    t = torch.tensor(a, requires_grad=True, dtype=torch.float)
+    t2 = deserialize(*serialize(t))
+    assert t2.requires_grad
+    assert np.allclose(a, t2.detach_().numpy())
diff --git a/distributed/protocol/torch.py b/distributed/protocol/torch.py
index c25b1549004..9a171b6d84f 100644
--- a/distributed/protocol/torch.py
+++ b/distributed/protocol/torch.py
@@ -1,5 +1,5 @@
 from .serialize import (serialize, dask_serialize, dask_deserialize,
-        register_generic)
+                        register_generic)
 
 import torch
 import numpy as np
@@ -7,12 +7,13 @@
 
 @dask_serialize.register(torch.Tensor)
 def serialize_torch_Tensor(t):
-    header, frames = serialize(t.numpy())
+    requires_grad_ = t.requires_grad
+    header, frames = serialize(t.detach_().numpy())
     if t.grad is not None:
         grad_header, grad_frames = serialize(t.grad.numpy())
         header['grad'] = {'header': grad_header, 'start': len(frames)}
         frames += grad_frames
-    header['requires_grad'] = t.requires_grad
+    header['requires_grad'] = requires_grad_
     header['device'] = t.device.type
     return header, frames
 
@@ -22,7 +23,8 @@ def deserialize_torch_Tensor(header, frames):
     if header.get('grad', False):
         i = header['grad']['start']
         frames, grad_frames = frames[:i], frames[i:]
-        grad = dask_deserialize.dispatch(np.ndarray)(header['grad']['header'], grad_frames)
+        grad = dask_deserialize.dispatch(np.ndarray)(header['grad']['header'],
+                                                     grad_frames)
     else:
         grad = None
 

From e363904af067de1beac98313c0a6f8362980d406 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 9 Apr 2019 08:24:33 -0500
Subject: [PATCH 0226/1550] Change bokeh port keywords to dashboard_address
 (#2589)

---
 distributed/cli/dask_scheduler.py            | 21 +++--
 distributed/cli/dask_worker.py               | 18 +++-
 distributed/cli/tests/test_dask_scheduler.py |  4 +-
 distributed/cli/tests/test_dask_worker.py    | 23 +++++
 distributed/deploy/local.py                  | 24 ++++--
 distributed/deploy/tests/test_adaptive.py    | 22 ++---
 distributed/deploy/tests/test_local.py       | 91 +++++++++++---------
 distributed/scheduler.py                     | 20 +++--
 distributed/tests/py3_test_asyncio.py        |  2 +-
 distributed/tests/test_client.py             |  4 +-
 distributed/tests/test_scheduler.py          | 46 ++++++----
 distributed/tests/test_worker.py             | 40 +++++++--
 distributed/worker.py                        | 17 +++-
 docs/source/web.rst                          |  2 +-
 14 files changed, 219 insertions(+), 115 deletions(-)

diff --git a/distributed/cli/dask_scheduler.py b/distributed/cli/dask_scheduler.py
index f8ed240f16b..f48e8b4e26d 100755
--- a/distributed/cli/dask_scheduler.py
+++ b/distributed/cli/dask_scheduler.py
@@ -7,6 +7,7 @@
 import shutil
 import sys
 import tempfile
+import warnings
 
 import click
 
@@ -40,8 +41,10 @@
 @click.option('--tls-key', type=pem_file_option_type, default=None,
               help="private key file for TLS (in PEM format)")
 # XXX default port (or URI) values should be centralized somewhere
-@click.option('--bokeh-port', type=int, default=8787,
-              help="Bokeh port for visual diagnostics")
+@click.option('--bokeh-port', type=int, default=None,
+              help="Deprecated.  See --dashboard-address")
+@click.option('--dashboard-address', type=str, default=':8787',
+              help="Address on which to listen for diagnostics dashboard")
 @click.option('--bokeh/--no-bokeh', '_bokeh', default=True, show_default=True,
               required=False, help="Launch Bokeh Web UI")
 @click.option('--show/--no-show', default=False, help="Show web UI")
@@ -66,11 +69,19 @@
                 type=click.UNPROCESSED, callback=validate_preload_argv)
 def main(host, port, bokeh_port, show, _bokeh, bokeh_whitelist, bokeh_prefix,
         use_xheaders, pid_file, scheduler_file, interface,
-        local_directory, preload, preload_argv, tls_ca_file, tls_cert, tls_key):
+        local_directory, preload, preload_argv, tls_ca_file, tls_cert, tls_key,
+        dashboard_address):
 
     enable_proctitle_on_current()
     enable_proctitle_on_children()
 
+    if bokeh_port is not None:
+        warnings.warn(
+            "The --bokeh-port flag has been renamed to --dashboard-address. "
+            "Consider adding ``--dashboard-address :%d`` " % bokeh_port
+        )
+        dashboard_address = bokeh_port
+
     sec = Security(tls_ca_file=tls_ca_file,
                    tls_scheduler_cert=tls_cert,
                    tls_scheduler_key=tls_key,
@@ -120,8 +131,8 @@ def del_pid_file():
     if _bokeh:
         try:
             from distributed.bokeh.scheduler import BokehScheduler
-            services[('bokeh', bokeh_port)] = (BokehScheduler,
-                                               {'prefix': bokeh_prefix})
+            services[('bokeh', dashboard_address)] = (BokehScheduler,
+                                                      {'prefix': bokeh_prefix})
         except ImportError as error:
             if str(error).startswith('No module named'):
                 logger.info('Web dashboard not loaded.  Unable to import bokeh')
diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
index aa6c5f8a087..ebc0fb441b6 100755
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -4,6 +4,7 @@
 import logging
 import os
 from sys import exit
+import warnings
 
 import click
 from distributed import Nanny, Worker
@@ -40,8 +41,10 @@
               help="Serving computation port, defaults to random")
 @click.option('--nanny-port', type=int, default=0,
               help="Serving nanny port, defaults to random")
-@click.option('--bokeh-port', type=int, default=0,
-              help="Bokeh port, defaults to random port")
+@click.option('--bokeh-port', type=int, default=None,
+              help="Deprecated.  See --dashboard-address")
+@click.option('--dashboard-address', type=str, default=':0',
+              help="Address on which to listen for diagnostics dashboard")
 @click.option('--bokeh/--no-bokeh', 'bokeh', default=True, show_default=True,
               required=False, help="Launch Bokeh Web UI")
 @click.option('--listen-address', type=str, default=None,
@@ -102,10 +105,17 @@ def main(scheduler, host, worker_port, listen_address, contact_address,
          memory_limit, pid_file, reconnect, resources, bokeh,
          bokeh_port, local_directory, scheduler_file, interface,
          death_timeout, preload, preload_argv, bokeh_prefix, tls_ca_file,
-         tls_cert, tls_key):
+         tls_cert, tls_key, dashboard_address):
     enable_proctitle_on_current()
     enable_proctitle_on_children()
 
+    if bokeh_port is not None:
+        warnings.warn(
+            "The --bokeh-port flag has been renamed to --dashboard-address. "
+            "Consider adding ``--dashboard-address :%d`` " % bokeh_port
+        )
+        dashboard_address = bokeh_port
+
     sec = Security(tls_ca_file=tls_ca_file,
                    tls_worker_cert=tls_cert,
                    tls_worker_key=tls_key,
@@ -177,7 +187,7 @@ def del_pid_file():
                 result = (BokehWorker, {'prefix': bokeh_prefix})
             else:
                 result = BokehWorker
-            services[('bokeh', bokeh_port)] = result
+            services[('bokeh', dashboard_address)] = result
 
     if resources:
         resources = resources.replace(',', ' ').split()
diff --git a/distributed/cli/tests/test_dask_scheduler.py b/distributed/cli/tests/test_dask_scheduler.py
index e893f0c2ba2..7564c3f05bd 100644
--- a/distributed/cli/tests/test_dask_scheduler.py
+++ b/distributed/cli/tests/test_dask_scheduler.py
@@ -101,7 +101,7 @@ def test_bokeh_non_standard_ports(loop):
 
     with popen(['dask-scheduler',
                 '--port', '3448',
-                '--bokeh-port', '4832']) as proc:
+                '--dashboard-address', ':4832']) as proc:
         with Client('127.0.0.1:3448', loop=loop) as c:
             pass
 
@@ -226,7 +226,7 @@ def test_scheduler_port_zero(loop):
 def test_bokeh_port_zero(loop):
     pytest.importorskip('bokeh')
     with tmpfile() as fn:
-        with popen(['dask-scheduler', '--bokeh-port', '0']) as proc:
+        with popen(['dask-scheduler', '--dashboard-address', ':0']) as proc:
             count = 0
             while count < 1:
                 line = proc.stderr.readline()
diff --git a/distributed/cli/tests/test_dask_worker.py b/distributed/cli/tests/test_dask_worker.py
index f4f9e7fa98c..5797905c7cd 100644
--- a/distributed/cli/tests/test_dask_worker.py
+++ b/distributed/cli/tests/test_dask_worker.py
@@ -3,6 +3,7 @@
 import pytest
 pytest.importorskip('requests')
 
+import requests
 import sys
 from time import sleep
 from toolz import first
@@ -183,3 +184,25 @@ def func(dask_worker):
 
                 listen_addresses = client.run(func)
                 assert all(host in v for v in listen_addresses.values())
+
+
+def test_bokeh_non_standard_ports(loop):
+    pytest.importorskip('bokeh')
+
+    with popen(['dask-scheduler', '--port', '3449', '--no-bokeh']):
+        with popen(['dask-worker', 'tcp://127.0.0.1:3449',
+                    '--dashboard-address', ':4833']) as proc:
+            with Client('127.0.0.1:3449', loop=loop) as c:
+                pass
+
+            start = time()
+            while True:
+                try:
+                    response = requests.get('http://127.0.0.1:4833/main')
+                    assert response.ok
+                    break
+                except Exception:
+                    sleep(0.5)
+                    assert time() < start + 20
+        with pytest.raises(Exception):
+            requests.get('http://localhost:4833/status/')
diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index c9f307bb421..8ba03f27217 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -44,11 +44,13 @@ class LocalCluster(Cluster):
         Use a falsey value like False or None for no change.
     ip: string
         IP address on which the scheduler will listen, defaults to only localhost
+    dashboard_address: str
+        Address on which to listen for the Bokeh diagnostics server like
+        'localhost:8787' or '0.0.0.0:8787'.  Defaults to ':8787'.
+        Set to ``None`` to disable the dashboard.
+        Use port 0 for a random port.
     diagnostics_port: int
-        Port on which the :doc:`web` will be provided.  8787 by default, use 0
-        to choose a random port, ``None`` to disable it, or an
-        :samp:`({ip}:{port})` tuple to listen on a different IP address than
-        the scheduler.
+        Deprecated.  See dashboard_address.
     asynchronous: bool (False by default)
         Set to True if using this cluster within async/await functions or within
         Tornado gen.coroutines.  This should remain False for normal use.
@@ -87,7 +89,8 @@ class LocalCluster(Cluster):
     """
     def __init__(self, n_workers=None, threads_per_worker=None, processes=True,
                  loop=None, start=None, ip=None, scheduler_port=0,
-                 silence_logs=logging.WARN, diagnostics_port=8787,
+                 silence_logs=logging.WARN, dashboard_address=':8787',
+                 diagnostics_port=None,
                  services=None, worker_services=None, service_kwargs=None,
                  asynchronous=False, security=None, protocol=None,
                  blocked_handlers=None, **worker_kwargs):
@@ -98,6 +101,13 @@ def __init__(self, n_workers=None, threads_per_worker=None, processes=True,
                    "  cluster = yield LocalCluster(asynchronous=True)")
             raise ValueError(msg)
 
+        if diagnostics_port is not None:
+            warnings.warn(
+                "diagnostics_port has been deprecated. "
+                "Please use `dashboard_address=` instead"
+            )
+            dashboard_address = diagnostics_port
+
         self.status = None
         self.processes = processes
 
@@ -143,14 +153,14 @@ def __init__(self, n_workers=None, threads_per_worker=None, processes=True,
         self._loop_runner = LoopRunner(loop=loop, asynchronous=asynchronous)
         self.loop = self._loop_runner.loop
 
-        if diagnostics_port is not False and diagnostics_port is not None:
+        if dashboard_address is not False and dashboard_address is not None:
             try:
                 from distributed.bokeh.scheduler import BokehScheduler
                 from distributed.bokeh.worker import BokehWorker
             except ImportError:
                 logger.debug("To start diagnostics web server please install Bokeh")
             else:
-                services[('bokeh', diagnostics_port)] = (BokehScheduler, (service_kwargs or {}).get('bokeh', {}))
+                services[('bokeh', dashboard_address)] = (BokehScheduler, (service_kwargs or {}).get('bokeh', {}))
                 worker_services[('bokeh', 0)] = BokehWorker
 
         self.scheduler = Scheduler(loop=self.loop,
diff --git a/distributed/deploy/tests/test_adaptive.py b/distributed/deploy/tests/test_adaptive.py
index c3a40b23c21..dca8653a752 100644
--- a/distributed/deploy/tests/test_adaptive.py
+++ b/distributed/deploy/tests/test_adaptive.py
@@ -15,7 +15,7 @@
 
 def test_get_scale_up_kwargs(loop):
     with LocalCluster(0, scheduler_port=0, silence_logs=False,
-                      diagnostics_port=None, loop=loop) as cluster:
+                      dashboard_address=None, loop=loop) as cluster:
 
         alc = Adaptive(cluster.scheduler, cluster, interval=100,
                        scale_factor=3)
@@ -62,7 +62,7 @@ def scale_down(self, workers):
 
 def test_adaptive_local_cluster(loop):
     with LocalCluster(0, scheduler_port=0, silence_logs=False,
-                      diagnostics_port=None, loop=loop) as cluster:
+                      dashboard_address=None, loop=loop) as cluster:
         alc = Adaptive(cluster.scheduler, cluster, interval=100)
         with Client(cluster, loop=loop) as c:
             assert not c.ncores()
@@ -87,7 +87,7 @@ def test_adaptive_local_cluster(loop):
 @gen_test(timeout=30)
 def test_adaptive_local_cluster_multi_workers():
     cluster = yield LocalCluster(0, scheduler_port=0, silence_logs=False,
-                                 processes=False, diagnostics_port=None,
+                                 processes=False, dashboard_address=None,
                                  asynchronous=True)
     try:
         cluster.scheduler.allowed_failures = 1000
@@ -161,7 +161,7 @@ def scale_down(self, workers):
 def test_min_max():
     loop = IOLoop.current()
     cluster = yield LocalCluster(0, scheduler_port=0, silence_logs=False,
-                                 processes=False, diagnostics_port=None,
+                                 processes=False, dashboard_address=None,
                                  loop=loop, asynchronous=True)
     yield cluster._start()
     try:
@@ -212,7 +212,7 @@ def test_avoid_churn():
     """
     cluster = yield LocalCluster(0, asynchronous=True, processes=False,
                                  scheduler_port=0, silence_logs=False,
-                                 diagnostics_port=None)
+                                 dashboard_address=None)
     client = yield Client(cluster, asynchronous=True)
     try:
         adapt = Adaptive(cluster.scheduler, cluster, interval='20 ms', wait_count=5)
@@ -236,7 +236,7 @@ def test_adapt_quickly():
     """
     cluster = yield LocalCluster(0, asynchronous=True, processes=False,
                                  scheduler_port=0, silence_logs=False,
-                                 diagnostics_port=None)
+                                 dashboard_address=None)
     client = yield Client(cluster, asynchronous=True)
     adapt = Adaptive(cluster.scheduler, cluster, interval=20, wait_count=5,
                      maximum=10)
@@ -279,7 +279,7 @@ def test_adapt_down():
     """ Ensure that redefining adapt with a lower maximum removes workers """
     cluster = yield LocalCluster(0, asynchronous=True, processes=False,
                                  scheduler_port=0, silence_logs=False,
-                                 diagnostics_port=None)
+                                 dashboard_address=None)
     client = yield Client(cluster, asynchronous=True)
     cluster.adapt(interval='20ms', maximum=5)
 
@@ -304,7 +304,7 @@ def test_adapt_down():
 def test_no_more_workers_than_tasks():
     loop = IOLoop.current()
     cluster = yield LocalCluster(0, scheduler_port=0, silence_logs=False,
-                                 processes=False, diagnostics_port=None,
+                                 processes=False, dashboard_address=None,
                                  loop=loop, asynchronous=True)
     yield cluster._start()
     try:
@@ -324,7 +324,7 @@ def test_no_more_workers_than_tasks():
 def test_basic_no_loop():
     try:
         with LocalCluster(0, scheduler_port=0, silence_logs=False,
-                          diagnostics_port=None) as cluster:
+                          dashboard_address=None) as cluster:
             with Client(cluster) as client:
                 cluster.adapt()
                 future = client.submit(lambda x: x + 1, 1)
@@ -339,7 +339,7 @@ def test_target_duration():
     """ Ensure that redefining adapt with a lower maximum removes workers """
     cluster = yield LocalCluster(0, asynchronous=True, processes=False,
                                  scheduler_port=0, silence_logs=False,
-                                 diagnostics_port=None)
+                                 dashboard_address=None)
     client = yield Client(cluster, asynchronous=True)
     adaptive = cluster.adapt(interval='20ms', minimum=2, target_duration='5s')
 
@@ -367,7 +367,7 @@ def test_worker_keys():
     """ Ensure that redefining adapt with a lower maximum removes workers """
     cluster = yield LocalCluster(0, asynchronous=True, processes=False,
                                  scheduler_port=0, silence_logs=False,
-                                 diagnostics_port=None)
+                                 dashboard_address=None)
 
     try:
         yield [cluster.start_worker(name='a-1'),
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index b21c9e60ba2..95b4bb42fd4 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -31,7 +31,7 @@
 
 def test_simple(loop):
     with LocalCluster(4, scheduler_port=0, processes=False, silence_logs=False,
-                      diagnostics_port=None, loop=loop) as c:
+                      dashboard_address=None, loop=loop) as c:
         with Client(c) as e:
             x = e.submit(inc, 1)
             x.result()
@@ -67,7 +67,7 @@ def test_close_twice():
 @pytest.mark.skipif('sys.version_info[0] == 2', reason='multi-loop')
 def test_procs():
     with LocalCluster(2, scheduler_port=0, processes=False, threads_per_worker=3,
-                      diagnostics_port=None, silence_logs=False) as c:
+                      dashboard_address=None, silence_logs=False) as c:
         assert len(c.workers) == 2
         assert all(isinstance(w, Worker) for w in c.workers)
         with Client(c.scheduler.address) as e:
@@ -76,7 +76,7 @@ def test_procs():
         repr(c)
 
     with LocalCluster(2, scheduler_port=0, processes=True, threads_per_worker=3,
-                      diagnostics_port=None, silence_logs=False) as c:
+                      dashboard_address=None, silence_logs=False) as c:
         assert len(c.workers) == 2
         assert all(isinstance(w, Nanny) for w in c.workers)
         with Client(c.scheduler.address) as e:
@@ -93,7 +93,7 @@ def test_move_unserializable_data():
     transports.
     """
     with LocalCluster(processes=False, silence_logs=False,
-                      diagnostics_port=None) as cluster:
+                      dashboard_address=None) as cluster:
         assert cluster.scheduler_address.startswith('inproc://')
         assert cluster.workers[0].address.startswith('inproc://')
         with Client(cluster) as client:
@@ -108,7 +108,7 @@ def test_transports():
     Test the transport chosen by LocalCluster depending on arguments.
     """
     with LocalCluster(1, processes=False, silence_logs=False,
-                      diagnostics_port=None) as c:
+                      dashboard_address=None) as c:
         assert c.scheduler_address.startswith('inproc://')
         assert c.workers[0].address.startswith('inproc://')
         with Client(c.scheduler.address) as e:
@@ -116,7 +116,7 @@ def test_transports():
 
     # Have nannies => need TCP
     with LocalCluster(1, processes=True, silence_logs=False,
-                      diagnostics_port=None) as c:
+                      dashboard_address=None) as c:
         assert c.scheduler_address.startswith('tcp://')
         assert c.workers[0].address.startswith('tcp://')
         with Client(c.scheduler.address) as e:
@@ -124,7 +124,7 @@ def test_transports():
 
     # Scheduler port specified => need TCP
     with LocalCluster(1, processes=False, scheduler_port=8786, silence_logs=False,
-                      diagnostics_port=None) as c:
+                      dashboard_address=None) as c:
 
         assert c.scheduler_address == 'tcp://127.0.0.1:8786'
         assert c.workers[0].address.startswith('tcp://')
@@ -134,14 +134,14 @@ def test_transports():
 
 @pytest.mark.skipif('sys.version_info[0] == 2', reason='')
 class LocalTest(ClusterTest, unittest.TestCase):
-    Cluster = partial(LocalCluster, silence_logs=False, diagnostics_port=None)
-    kwargs = {'diagnostics_port': None}
+    Cluster = partial(LocalCluster, silence_logs=False, dashboard_address=None)
+    kwargs = {'dashboard_address': None}
 
 
 @pytest.mark.skipif('sys.version_info[0] == 2', reason='')
 def test_Client_with_local(loop):
     with LocalCluster(1, scheduler_port=0, silence_logs=False,
-                      diagnostics_port=None, loop=loop) as c:
+                      dashboard_address=None, loop=loop) as c:
         with Client(c) as e:
             assert len(e.ncores()) == len(c.workers)
             assert c.scheduler_address in repr(c)
@@ -156,9 +156,9 @@ def test_Client_solo(loop):
 @gen_test()
 def test_duplicate_clients():
     pytest.importorskip('bokeh')
-    c1 = yield Client(processes=False, silence_logs=False, diagnostics_port=9876)
+    c1 = yield Client(processes=False, silence_logs=False, dashboard_address=9876)
     with pytest.warns(Exception) as info:
-        c2 = yield Client(processes=False, silence_logs=False, diagnostics_port=9876)
+        c2 = yield Client(processes=False, silence_logs=False, dashboard_address=9876)
 
     assert 'bokeh' in c1.cluster.scheduler.services
     assert 'bokeh' in c2.cluster.scheduler.services
@@ -177,8 +177,8 @@ def test_Client_kwargs(loop):
 
 
 def test_Client_twice(loop):
-    with Client(loop=loop, silence_logs=False, diagnostics_port=None) as c:
-        with Client(loop=loop, silence_logs=False, diagnostics_port=None) as f:
+    with Client(loop=loop, silence_logs=False, dashboard_address=None) as c:
+        with Client(loop=loop, silence_logs=False, dashboard_address=None) as f:
             assert c.cluster.scheduler.port != f.cluster.scheduler.port
 
 
@@ -186,14 +186,19 @@ def test_Client_twice(loop):
 def test_defaults():
     from distributed.worker import _ncores
 
+    with LocalCluster(scheduler_port=0, silence_logs=False,
+                      dashboard_address=None) as c:
+        assert sum(w.ncores for w in c.workers) == _ncores
+        assert all(isinstance(w, Nanny) for w in c.workers)
+
     with LocalCluster(processes=False, scheduler_port=0, silence_logs=False,
-                      diagnostics_port=None) as c:
+                      dashboard_address=None) as c:
         assert sum(w.ncores for w in c.workers) == _ncores
         assert all(isinstance(w, Worker) for w in c.workers)
         assert len(c.workers) == 1
 
     with LocalCluster(n_workers=2, scheduler_port=0, silence_logs=False,
-                      diagnostics_port=None) as c:
+                      dashboard_address=None) as c:
         if _ncores % 2 == 0:
             expected_total_threads = max(2, _ncores)
         else:
@@ -202,27 +207,27 @@ def test_defaults():
         assert sum(w.ncores for w in c.workers) == expected_total_threads
 
     with LocalCluster(threads_per_worker=_ncores * 2, scheduler_port=0,
-                      silence_logs=False, diagnostics_port=None) as c:
+                      silence_logs=False, dashboard_address=None) as c:
         assert len(c.workers) == 1
 
     with LocalCluster(n_workers=_ncores * 2, scheduler_port=0,
-                      silence_logs=False, diagnostics_port=None) as c:
+                      silence_logs=False, dashboard_address=None) as c:
         assert all(w.ncores == 1 for w in c.workers)
     with LocalCluster(threads_per_worker=2, n_workers=3, scheduler_port=0,
-                      silence_logs=False, diagnostics_port=None) as c:
+                      silence_logs=False, dashboard_address=None) as c:
         assert len(c.workers) == 3
         assert all(w.ncores == 2 for w in c.workers)
 
 
 def test_worker_params():
     with LocalCluster(n_workers=2, scheduler_port=0, silence_logs=False,
-                      diagnostics_port=None, memory_limit=500) as c:
+                      dashboard_address=None, memory_limit=500) as c:
         assert [w.memory_limit for w in c.workers] == [500] * 2
 
 
 def test_memory_limit_none():
     with LocalCluster(n_workers=2, scheduler_port=0, silence_logs=False,
-                      processes=False,  diagnostics_port=None, memory_limit=None) as c:
+                      processes=False,  dashboard_address=None, memory_limit=None) as c:
         w = c.workers[0]
         assert type(w.data) is dict
         assert w.memory_limit is None
@@ -230,20 +235,20 @@ def test_memory_limit_none():
 
 def test_cleanup():
     c = LocalCluster(2, scheduler_port=0, silence_logs=False,
-                     diagnostics_port=None)
+                     dashboard_address=None)
     port = c.scheduler.port
     c.close()
     c2 = LocalCluster(2, scheduler_port=port, silence_logs=False,
-                      diagnostics_port=None)
+                      dashboard_address=None)
     c.close()
 
 
 def test_repeated():
     with LocalCluster(scheduler_port=8448, silence_logs=False,
-                      diagnostics_port=None) as c:
+                      dashboard_address=None) as c:
         pass
     with LocalCluster(scheduler_port=8448, silence_logs=False,
-                      diagnostics_port=None) as c:
+                      dashboard_address=None) as c:
         pass
 
 
@@ -252,7 +257,7 @@ def test_bokeh(loop, processes):
     pytest.importorskip('bokeh')
     requests = pytest.importorskip('requests')
     with LocalCluster(scheduler_port=0, silence_logs=False, loop=loop,
-                      processes=processes, diagnostics_port=0) as c:
+                      processes=processes, dashboard_address=0) as c:
         bokeh_port = c.scheduler.services['bokeh'].port
         url = 'http://127.0.0.1:%d/status/' % bokeh_port
         start = time()
@@ -280,7 +285,7 @@ def test_blocks_until_full(loop):
 def test_scale_up_and_down():
     loop = IOLoop.current()
     cluster = yield LocalCluster(0, scheduler_port=0, processes=False,
-                                 silence_logs=False, diagnostics_port=None,
+                                 silence_logs=False, dashboard_address=None,
                                  loop=loop, asynchronous=True)
     c = yield Client(cluster, asynchronous=True)
 
@@ -305,7 +310,7 @@ def test_silent_startup():
         from time import sleep
         from distributed import LocalCluster
 
-        with LocalCluster(1, diagnostics_port=None, scheduler_port=0):
+        with LocalCluster(1, dashboard_address=None, scheduler_port=0):
             sleep(1.5)
         """
 
@@ -322,27 +327,27 @@ def test_silent_startup():
 
 def test_only_local_access(loop):
     with LocalCluster(scheduler_port=0, silence_logs=False,
-                      diagnostics_port=None, loop=loop) as c:
+                      dashboard_address=None, loop=loop) as c:
         sync(loop, assert_can_connect_locally_4, c.scheduler.port)
 
 
 def test_remote_access(loop):
     with LocalCluster(scheduler_port=0, silence_logs=False,
-                      diagnostics_port=None, ip='', loop=loop) as c:
+                      dashboard_address=None, ip='', loop=loop) as c:
         sync(loop, assert_can_connect_from_everywhere_4_6, c.scheduler.port)
 
 
 @pytest.mark.parametrize('n_workers', [None, 3])
 def test_memory(loop, n_workers):
     with LocalCluster(n_workers=n_workers, scheduler_port=0, processes=False,
-                      silence_logs=False, diagnostics_port=None, loop=loop) as cluster:
+                      silence_logs=False, dashboard_address=None, loop=loop) as cluster:
         assert sum(w.memory_limit for w in cluster.workers) <= TOTAL_MEMORY
 
 
 @pytest.mark.parametrize('n_workers', [None, 3])
 def test_memory_nanny(loop, n_workers):
     with LocalCluster(n_workers=n_workers, scheduler_port=0, processes=True,
-                      silence_logs=False, diagnostics_port=None, loop=loop) as cluster:
+                      silence_logs=False, dashboard_address=None, loop=loop) as cluster:
         with Client(cluster.scheduler_address, loop=loop) as c:
             info = c.scheduler_info()
             assert (sum(w['memory_limit'] for w in info['workers'].values())
@@ -352,7 +357,7 @@ def test_memory_nanny(loop, n_workers):
 def test_death_timeout_raises(loop):
     with pytest.raises(gen.TimeoutError):
         with LocalCluster(scheduler_port=0, silence_logs=False,
-                          death_timeout=1e-10, diagnostics_port=None,
+                          death_timeout=1e-10, dashboard_address=None,
                           loop=loop) as cluster:
             pass
 
@@ -361,7 +366,7 @@ def test_death_timeout_raises(loop):
 def test_bokeh_kwargs(loop):
     pytest.importorskip('bokeh')
     with LocalCluster(scheduler_port=0, silence_logs=False, loop=loop,
-                      diagnostics_port=0,
+                      dashboard_address=0,
                       service_kwargs={'bokeh': {'prefix': '/foo'}}) as c:
 
         bs = c.scheduler.services['bokeh']
@@ -382,7 +387,7 @@ def test_logging():
     """
     Workers and scheduler have logs even when silenced
     """
-    with LocalCluster(1, processes=False, diagnostics_port=None) as c:
+    with LocalCluster(1, processes=False, dashboard_address=None) as c:
         assert c.scheduler._deque_handler.deque
         assert c.workers[0]._deque_handler.deque
 
@@ -390,7 +395,7 @@ def test_logging():
 def test_ipywidgets(loop):
     ipywidgets = pytest.importorskip('ipywidgets')
     with LocalCluster(scheduler_port=0, silence_logs=False, loop=loop,
-                      diagnostics_port=False, processes=False) as cluster:
+                      dashboard_address=False, processes=False) as cluster:
         cluster._ipython_display_()
         box = cluster._cached_widget
         assert isinstance(box, ipywidgets.Widget)
@@ -399,7 +404,7 @@ def test_ipywidgets(loop):
 def test_scale(loop):
     """ Directly calling scale both up and down works as expected """
     with LocalCluster(scheduler_port=0, silence_logs=False, loop=loop,
-                      diagnostics_port=False, processes=False, n_workers=0) as cluster:
+                      dashboard_address=False, processes=False, n_workers=0) as cluster:
         assert not cluster.scheduler.workers
         cluster.scale(3)
 
@@ -420,7 +425,7 @@ def test_scale(loop):
 
 def test_adapt(loop):
     with LocalCluster(scheduler_port=0, silence_logs=False, loop=loop,
-                      diagnostics_port=False, processes=False, n_workers=0) as cluster:
+                      dashboard_address=False, processes=False, n_workers=0) as cluster:
         cluster.adapt(minimum=0, maximum=2, interval='10ms')
         assert cluster._adaptive.minimum == 0
         assert cluster._adaptive.maximum == 2
@@ -446,7 +451,7 @@ def test_adapt(loop):
 def test_adapt_then_manual(loop):
     """ We can revert from adaptive, back to manual """
     with LocalCluster(scheduler_port=0, silence_logs=False, loop=loop,
-                      diagnostics_port=False, processes=False, n_workers=8) as cluster:
+                      dashboard_address=False, processes=False, n_workers=8) as cluster:
         sleep(0.1)
         cluster.adapt(minimum=0, maximum=4, interval='10ms')
 
@@ -477,7 +482,7 @@ def test_local_tls(loop):
     from distributed.utils_test import tls_only_security
     security = tls_only_security()
     with LocalCluster(scheduler_port=8786, silence_logs=False, security=security,
-                      diagnostics_port=False, ip='tls://0.0.0.0', loop=loop) as c:
+                      dashboard_address=False, ip='tls://0.0.0.0', loop=loop) as c:
         sync(loop, assert_can_connect_from_everywhere_4, c.scheduler.port,
              connection_args=security.get_connection_args('client'),
              protocol='tls', timeout=3)
@@ -498,7 +503,7 @@ def scale_down(self, *args, **kwargs):
 
     loop = IOLoop.current()
     cluster = yield MyCluster(0, scheduler_port=0, processes=False,
-                              silence_logs=False, diagnostics_port=None,
+                              silence_logs=False, dashboard_address=None,
                               loop=loop, asynchronous=True)
     c = yield Client(cluster, asynchronous=True)
 
@@ -526,7 +531,7 @@ def test_local_tls_restart(loop):
     from distributed.utils_test import tls_only_security
     security = tls_only_security()
     with LocalCluster(n_workers=1, scheduler_port=8786, silence_logs=False, security=security,
-                      diagnostics_port=False, ip='tls://0.0.0.0', loop=loop) as c:
+                      dashboard_address=False, ip='tls://0.0.0.0', loop=loop) as c:
         with Client(c.scheduler.address, loop=loop, security=security) as client:
             print(c.workers, c.workers[0].address)
             workers_before = set(client.scheduler_info()['workers'])
@@ -552,7 +557,7 @@ def test_default_process_thread_breakdown():
 
 def test_asynchronous_property(loop):
     with LocalCluster(4, scheduler_port=0, processes=False, silence_logs=False,
-                      diagnostics_port=None, loop=loop) as cluster:
+                      dashboard_address=None, loop=loop) as cluster:
 
         @gen.coroutine
         def _():
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 4f3c8520063..c4ecbf9e8a9 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1050,27 +1050,31 @@ def get_worker_service_addr(self, worker, service_name, protocol=False):
         else:
             return ws.host, port
 
-    def start_services(self, listen_ip):
+    def start_services(self, default_listen_ip):
+        if default_listen_ip == '0.0.0.0':
+            default_listen_ip = ''  # for IPV6
+
         for k, v in self.service_specs.items():
+            listen_ip = None
             if isinstance(k, tuple):
                 k, port = k
             else:
                 port = 0
 
+            if isinstance(port, (str, unicode)):
+                port = port.split(':')
+
+            if isinstance(port, (tuple, list)):
+                listen_ip, port = (port[0], int(port[1]))
+
             if isinstance(v, tuple):
                 v, kwargs = v
             else:
                 kwargs = {}
 
-            if listen_ip == '0.0.0.0':
-                listen_ip = ''  # for IPv6
-
             try:
                 service = v(self, io_loop=self.loop, **kwargs)
-                if isinstance(port, tuple):
-                    service.listen(port)
-                else:
-                    service.listen((listen_ip, port))
+                service.listen((listen_ip if listen_ip is not None else default_listen_ip, port))
                 self.services[k] = service
             except Exception as e:
                 warnings.warn("\nCould not launch service '%s' on port %s. " % (k, port) +
diff --git a/distributed/tests/py3_test_asyncio.py b/distributed/tests/py3_test_asyncio.py
index cf60b945b80..3c8629c2ba3 100644
--- a/distributed/tests/py3_test_asyncio.py
+++ b/distributed/tests/py3_test_asyncio.py
@@ -49,7 +49,7 @@ async def test_coro_test():
 
 @coro_test
 async def test_asyncio_start_close():
-    async with AioClient(processes=False, diagnostics_port=False) as c:
+    async with AioClient(processes=False, dashboard_address=False) as c:
         assert c.status == 'running'
         # AioClient has installed its AioLoop shim.
         assert isinstance(IOLoop.current(instance=False), BaseAsyncIOLoop)
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 8f55760e05a..8d80808695d 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -1879,7 +1879,7 @@ def test_repr_async(c, s, a, b):
 
 @gen_test()
 def test_repr_localcluster():
-    cluster = yield LocalCluster(processes=False, diagnostics_port=None,
+    cluster = yield LocalCluster(processes=False, dashboard_address=None,
                                  asynchronous=True)
     client = yield Client(cluster, asynchronous=True)
     try:
@@ -4919,7 +4919,7 @@ def f():
 
 def test_quiet_quit_when_cluster_leaves(loop_in_thread):
     loop = loop_in_thread
-    with LocalCluster(loop=loop, scheduler_port=0, diagnostics_port=None,
+    with LocalCluster(loop=loop, scheduler_port=0, dashboard_address=None,
                       silence_logs=False) as cluster:
         with captured_logger('distributed.comm') as sio:
             with Client(cluster, loop=loop) as client:
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index a739e43a310..95bcebfef26 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -1132,30 +1132,38 @@ def test_correct_bad_time_estimate(c, s, *workers):
     assert all(w.data for w in workers), [sorted(w.data) for w in workers]
 
 
-@pytest.mark.skipif(not sys.platform.startswith('linux'),
-                    reason="Need 127.0.0.* to mean localhost")
 @gen_test()
 def test_service_hosts():
     pytest.importorskip('bokeh')
     from distributed.bokeh.scheduler import BokehScheduler
 
-    for port in [0, ('127.0.0.3', 0)]:
-        for url, expected in [('tcp://0.0.0.0', ('::', '0.0.0.0')),
-                              ('tcp://127.0.0.2', '127.0.0.2'),
-                              ('tcp://127.0.0.2:38275', '127.0.0.2')]:
-            services = {('bokeh', port): BokehScheduler}
-
-            s = Scheduler(services=services)
-            yield s.start(url)
-
-            sock = first(s.services['bokeh'].server._http._sockets.values())
-            if isinstance(port, tuple):    # host explicitly overridden
-                assert sock.getsockname()[0] == port[0]
-            elif isinstance(expected, tuple):
-                assert sock.getsockname()[0] in expected
-            else:
-                assert sock.getsockname()[0] == expected
-            yield s.close()
+    port = 0
+    for url, expected in [
+            ('tcp://0.0.0.0', ('::', '0.0.0.0')),
+            ('tcp://127.0.0.1', '127.0.0.1'),
+            ('tcp://127.0.0.1:38275', '127.0.0.1')]:
+        services = {('bokeh', port): BokehScheduler}
+
+        s = Scheduler(services=services)
+        yield s.start(url)
+
+        sock = first(s.services['bokeh'].server._http._sockets.values())
+        if isinstance(expected, tuple):
+            assert sock.getsockname()[0] in expected
+        else:
+            assert sock.getsockname()[0] == expected
+        yield s.close()
+
+    port = ('127.0.0.1', 0)
+    for url in ['tcp://0.0.0.0', 'tcp://127.0.0.1', 'tcp://127.0.0.1:38275']:
+        services = {('bokeh', port): BokehScheduler}
+
+        s = Scheduler(services=services)
+        yield s.start(url)
+
+        sock = first(s.services['bokeh'].server._http._sockets.values())
+        assert sock.getsockname()[0] == '127.0.0.1'
+        yield s.close()
 
 
 @gen_cluster(client=True, worker_kwargs={'profile_cycle_interval': 100})
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 662c666cb4e..805d150e555 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -919,20 +919,42 @@ def test_worker_fds(s):
         assert time() < start + 0.5
 
 
-@pytest.mark.skipif(not sys.platform.startswith('linux'),
-                    reason="Need 127.0.0.2 to mean localhost")
 @gen_cluster(ncores=[])
 def test_service_hosts_match_worker(s):
     pytest.importorskip('bokeh')
     from distributed.bokeh.worker import BokehWorker
-    services = {('bokeh', 0): BokehWorker}
-    for host in ['tcp://0.0.0.0', 'tcp://127.0.0.2']:
-        w = Worker(s.address, services=services)
-        yield w._start(host)
+    services = {('bokeh', ':0'): BokehWorker}
 
-        sock = first(w.services['bokeh'].server._http._sockets.values())
-        assert sock.getsockname()[0] == host.split('://')[1]
-        yield w._close()
+    w = Worker(s.address, services={('bokeh', ':0'): BokehWorker})
+    yield w._start('tcp://0.0.0.0')
+    sock = first(w.services['bokeh'].server._http._sockets.values())
+    assert sock.getsockname()[0] in ('::', '0.0.0.0')
+    yield w._close()
+
+    w = Worker(s.address, services={('bokeh', ':0'): BokehWorker})
+    yield w._start('tcp://127.0.0.1')
+    sock = first(w.services['bokeh'].server._http._sockets.values())
+    assert sock.getsockname()[0] in ('::', '0.0.0.0')
+    yield w._close()
+
+    w = Worker(s.address, services={('bokeh', 0): BokehWorker})
+    yield w._start('tcp://127.0.0.1')
+    sock = first(w.services['bokeh'].server._http._sockets.values())
+    assert sock.getsockname()[0] == '127.0.0.1'
+    yield w._close()
+
+
+@gen_cluster(ncores=[])
+def test_start_services(s):
+    pytest.importorskip('bokeh')
+    from distributed.bokeh.worker import BokehWorker
+    services = {('bokeh', ':1234'): BokehWorker}
+
+    w = Worker(s.address, services=services)
+    yield w._start()
+
+    assert w.services['bokeh'].server.port == 1234
+    yield w._close()
 
 
 @gen_test()
diff --git a/distributed/worker.py b/distributed/worker.py
index 46d5ae61001..2ef303735c5 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -750,19 +750,30 @@ def get_logs(self, comm=None, n=None):
     # Lifecycle #
     #############
 
-    def start_services(self, listen_ip=''):
+    def start_services(self, default_listen_ip):
+        if default_listen_ip == '0.0.0.0':
+            default_listen_ip = ''  # for IPV6
+
         for k, v in self.service_specs.items():
+            listen_ip = None
             if isinstance(k, tuple):
                 k, port = k
             else:
                 port = 0
 
+            if isinstance(port, (str, unicode)):
+                port = port.split(':')
+
+            if isinstance(port, (tuple, list)):
+                listen_ip, port = (port[0], int(port[1]))
+
             if isinstance(v, tuple):
                 v, kwargs = v
             else:
-                v, kwargs = v, {}
+                kwargs = {}
+
             self.services[k] = v(self, io_loop=self.loop, **kwargs)
-            self.services[k].listen((listen_ip, port))
+            self.services[k].listen((listen_ip if listen_ip is not None else default_listen_ip, port))
             self.service_ports[k] = self.services[k].port
 
     @gen.coroutine
diff --git a/docs/source/web.rst b/docs/source/web.rst
index 7d81fe7d935..c888bb30763 100644
--- a/docs/source/web.rst
+++ b/docs/source/web.rst
@@ -210,7 +210,7 @@ a few ways to handle this:
 1.  Open port ``8787`` to the outside world.  Often this involves asking your
     cluster administrator.
 2.  Use a different port that is publicly accessible using the
-    ``--bokeh-port PORT`` option on the ``dask-scheduler`` command.
+    ``--dashboard-address :8787`` option on the ``dask-scheduler`` command.
 3.  Use fancier techniques, like `Port Forwarding`_
 
 Running distributed on a remote machine can cause issues with viewing the web

From d97b49812f489f4fce220281e386278f9578a159 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 9 Apr 2019 10:32:41 -0500
Subject: [PATCH 0227/1550] Specify data storage explicitly from Worker
 constructor (#2600)

* Support providing data= keyword to worker directly

* Use Worker/Nanny.__await__ more extensively
---
 distributed/deploy/local.py                  |  3 +-
 distributed/diagnostics/tests/test_plugin.py |  7 ++-
 distributed/nanny.py                         |  5 ++
 distributed/tests/test_client.py             |  7 ++-
 distributed/tests/test_nanny.py              | 40 +++++++-------
 distributed/tests/test_resources.py          |  5 +-
 distributed/tests/test_scheduler.py          | 26 ++++-----
 distributed/tests/test_steal.py              | 10 ++--
 distributed/tests/test_worker.py             | 57 ++++++++++++--------
 distributed/worker.py                        | 19 +++++--
 10 files changed, 97 insertions(+), 82 deletions(-)

diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index 8ba03f27217..ececc9fe12c 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -254,10 +254,9 @@ def _start_worker(self, death_timeout=60, **kwargs):
         else:
             W = Worker
 
-        w = W(self.scheduler.address, loop=self.loop,
+        w = yield W(self.scheduler.address, loop=self.loop,
               death_timeout=death_timeout,
               silence_logs=self.silence_logs, **kwargs)
-        yield w._start()
 
         self.workers.append(w)
 
diff --git a/distributed/diagnostics/tests/test_plugin.py b/distributed/diagnostics/tests/test_plugin.py
index 14a68f66929..afc1f4987d5 100644
--- a/distributed/diagnostics/tests/test_plugin.py
+++ b/distributed/diagnostics/tests/test_plugin.py
@@ -54,8 +54,8 @@ def remove_worker(self, worker, scheduler):
 
     a = Worker(s.address)
     b = Worker(s.address)
-    yield a._start()
-    yield b._start()
+    yield a
+    yield b
     yield a._close()
     yield b._close()
 
@@ -67,7 +67,6 @@ def remove_worker(self, worker, scheduler):
 
     events[:] = []
     s.remove_plugin(plugin)
-    a = Worker(s.address)
-    yield a._start()
+    a = yield Worker(s.address)
     yield a._close()
     assert events == []
diff --git a/distributed/nanny.py b/distributed/nanny.py
index e2577f5846a..c1965759685 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -164,6 +164,11 @@ def _start(self, addr_or_port=0):
 
         self.start_periodic_callbacks()
 
+        raise gen.Return(self)
+
+    def __await__(self):
+        return self._start().__await__()
+
     def start(self, addr_or_port=0):
         self.loop.add_callback(self._start, addr_or_port)
 
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 8d80808695d..f9594a672b9 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -2751,7 +2751,7 @@ def test_worker_aliases():
     a = Worker(s.ip, s.port, name='alice')
     b = Worker(s.ip, s.port, name='bob')
     w = Worker(s.ip, s.port, name=3)
-    yield [a._start(), b._start(), w._start()]
+    yield [a, b, w]
 
     c = yield Client((s.ip, s.port), asynchronous=True)
 
@@ -2965,8 +2965,7 @@ def test_unrunnable_task_runs(c, s, a, b):
     assert s.tasks[x.key] in s.unrunnable
     assert s.get_task_status(keys=[x.key]) == {x.key: 'no-worker'}
 
-    w = Worker(s.ip, s.port, loop=s.loop)
-    yield w._start()
+    w = yield Worker(s.ip, s.port, loop=s.loop)
 
     start = time()
     while x.status != 'finished':
@@ -3634,7 +3633,7 @@ def start_worker(sleep, duration, repeat=1):
                 yield gen.sleep(sleep)
                 w = worker(s['address'], loop=loop)
                 running[w] = None
-                yield w._start()
+                yield w
                 addr = w.worker_address
                 running[w] = addr
                 yield gen.sleep(duration)
diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
index c37bf056d34..23e40c4c8ef 100644
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -24,9 +24,8 @@
 
 @gen_cluster(ncores=[])
 def test_nanny(s):
-    n = Nanny(s.ip, s.port, ncores=2, loop=s.loop)
+    n = yield Nanny(s.ip, s.port, ncores=2, loop=s.loop)
 
-    yield n._start(0)
     with rpc(n.address) as nn:
         assert n.is_alive()
         assert s.ncores[n.worker_address] == 2
@@ -55,8 +54,7 @@ def test_nanny(s):
 
 @gen_cluster(ncores=[])
 def test_many_kills(s):
-    n = Nanny(s.address, ncores=2, loop=s.loop)
-    yield n._start(0)
+    n = yield Nanny(s.address, ncores=2, loop=s.loop)
     assert n.is_alive()
     yield [n.kill() for i in range(5)]
     yield [n.kill() for i in range(5)]
@@ -73,8 +71,7 @@ def test_str(s, a, b):
 
 @gen_cluster(ncores=[], timeout=20, client=True)
 def test_nanny_process_failure(c, s):
-    n = Nanny(s.ip, s.port, ncores=2, loop=s.loop)
-    yield n._start()
+    n = yield Nanny(s.ip, s.port, ncores=2, loop=s.loop)
     first_dir = n.worker_dir
 
     assert os.path.exists(first_dir)
@@ -121,8 +118,7 @@ def test_nanny_no_port():
 @gen_cluster(ncores=[])
 def test_run(s):
     pytest.importorskip('psutil')
-    n = Nanny(s.ip, s.port, ncores=2, loop=s.loop)
-    yield n._start()
+    n = yield Nanny(s.ip, s.port, ncores=2, loop=s.loop)
 
     with rpc(n.address) as nn:
         response = yield nn.run(function=dumps(lambda: 1))
@@ -169,8 +165,7 @@ def test_nanny_alt_worker_class(c, s, w1, w2):
 @gen_cluster(client=False, ncores=[])
 def test_nanny_death_timeout(s):
     yield s.close()
-    w = Nanny(s.address, death_timeout=1)
-    yield w._start()
+    w = yield Nanny(s.address, death_timeout=1)
 
     yield gen.sleep(3)
     assert w.status == 'closed'
@@ -199,8 +194,7 @@ def test_num_fds(s):
     proc = psutil.Process()
 
     # Warm up
-    w = Nanny(s.address)
-    yield w._start()
+    w = yield Nanny(s.address)
     yield w._close()
     del w
     gc.collect()
@@ -208,8 +202,7 @@ def test_num_fds(s):
     before = proc.num_fds()
 
     for i in range(3):
-        w = Nanny(s.address)
-        yield w._start()
+        w = yield Nanny(s.address)
         yield gen.sleep(0.1)
         yield w._close()
 
@@ -241,8 +234,7 @@ def test_scheduler_file():
     with tmpfile() as fn:
         s = Scheduler(scheduler_file=fn)
         s.start(8008)
-        w = Nanny(scheduler_file=fn)
-        yield w._start()
+        w = yield Nanny(scheduler_file=fn)
         assert set(s.workers) == {w.worker_address}
         yield w._close()
         s.stop()
@@ -290,8 +282,7 @@ def leak():
 
 @gen_cluster(ncores=[], client=True)
 def test_avoid_memory_monitor_if_zero_limit(c, s):
-    nanny = Nanny(s.address, loop=s.loop, memory_limit=0)
-    yield nanny._start()
+    nanny = yield Nanny(s.address, loop=s.loop, memory_limit=0)
     typ = yield c.run(lambda dask_worker: type(dask_worker.data))
     assert typ == {nanny.worker_address: dict}
     pcs = yield c.run(lambda dask_worker: list(dask_worker.periodic_callbacks))
@@ -310,8 +301,7 @@ def test_avoid_memory_monitor_if_zero_limit(c, s):
 @gen_cluster(ncores=[], client=True)
 def test_scheduler_address_config(c, s):
     with dask.config.set({'scheduler-address': s.address}):
-        nanny = Nanny(loop=s.loop)
-        yield nanny._start()
+        nanny = yield Nanny(loop=s.loop)
         assert nanny.scheduler.address == s.address
 
         start = time()
@@ -339,7 +329,15 @@ def test_wait_for_scheduler():
 def test_environment_variable(c, s):
     a = Nanny(s.address, loop=s.loop, memory_limit=0, env={"FOO": "123"})
     b = Nanny(s.address, loop=s.loop, memory_limit=0, env={"FOO": "456"})
-    yield [a._start(), b._start()]
+    yield [a, b]
     results = yield c.run(lambda: os.environ['FOO'])
     assert results == {a.worker_address: "123", b.worker_address: "456"}
     yield [a._close(), b._close()]
+
+
+@gen_cluster(ncores=[], client=True)
+def test_data_types(c, s):
+    w = yield Nanny(s.address, data=dict)
+    r = yield c.run(lambda dask_worker: type(dask_worker.data))
+    assert r[w.worker_address] == dict
+    yield w._close()
diff --git a/distributed/tests/test_resources.py b/distributed/tests/test_resources.py
index bcec6d1604c..4b1e9e2a80c 100644
--- a/distributed/tests/test_resources.py
+++ b/distributed/tests/test_resources.py
@@ -22,7 +22,7 @@ def test_resources(c, s):
     a = Worker(s.ip, s.port, loop=s.loop, resources={'GPU': 2})
     b = Worker(s.ip, s.port, loop=s.loop, resources={'GPU': 1, 'DB': 1})
 
-    yield [a._start(), b._start()]
+    yield [a, b]
 
     assert s.resources == {'GPU': {a.address: 2, b.address: 1},
                            'DB': {b.address: 1}}
@@ -52,8 +52,7 @@ def test_resource_submit(c, s, a, b):
 
     assert s.get_task_status(keys=[z.key]) == {z.key: 'no-worker'}
 
-    d = Worker(s.ip, s.port, loop=s.loop, resources={'C': 10})
-    yield d._start()
+    d = yield Worker(s.ip, s.port, loop=s.loop, resources={'C': 10})
 
     yield wait(z)
     assert z.key in d.data
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 95bcebfef26..1d101cd5224 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -236,7 +236,7 @@ def test_add_worker(s, a, b):
     w = Worker(s.ip, s.port, ncores=3)
     w.data['x-5'] = 6
     w.data['y'] = 1
-    yield w._start(0)
+    yield w
 
     dsk = {('x-%d' % i): (inc, i) for i in range(10)}
     s.update_graph(tasks=valmap(dumps_task, dsk), keys=list(dsk), client='client',
@@ -502,14 +502,12 @@ def test_broadcast_nanny(s, a, b):
 def test_worker_name():
     s = Scheduler(validate=True)
     s.start(0)
-    w = Worker(s.ip, s.port, name='alice')
-    yield w._start()
+    w = yield Worker(s.ip, s.port, name='alice')
     assert s.workers[w.address].name == 'alice'
     assert s.aliases['alice'] == w.address
 
     with pytest.raises(ValueError):
-        w2 = Worker(s.ip, s.port, name='alice')
-        yield w2._start()
+        w2 = yield Worker(s.ip, s.port, name='alice')
         yield w2._close()
 
     yield s.close()
@@ -525,7 +523,7 @@ def test_coerce_address():
         a = Worker(s.ip, s.port, name='alice')
         b = Worker(s.ip, s.port, name=123)
         c = Worker('127.0.0.1', s.port, name='charlie')
-        yield [a._start(), b._start(), c._start()]
+        yield [a, b, c]
 
         assert s.coerce_address('127.0.0.1:8000') == 'tcp://127.0.0.1:8000'
         assert s.coerce_address('[::1]:8000') == 'tcp://[::1]:8000'
@@ -559,9 +557,7 @@ def test_file_descriptors_dont_leak(s):
     proc = psutil.Process()
     before = proc.num_fds()
 
-    w = Worker(s.ip, s.port)
-
-    yield w._start(0)
+    w = yield Worker(s.ip, s.port)
     yield w._close()
 
     during = proc.num_fds()
@@ -634,8 +630,7 @@ def test_scatter_no_workers(c, s):
 
 @gen_cluster(ncores=[])
 def test_scheduler_sees_memory_limits(s):
-    w = Worker(s.ip, s.port, ncores=3, memory_limit=12345)
-    yield w._start(0)
+    w = yield Worker(s.ip, s.port, ncores=3, memory_limit=12345)
 
     assert s.workers[w.address].memory_limit == 12345
     yield w._close()
@@ -751,8 +746,7 @@ def test_file_descriptors(c, s):
     num_fds_1 = proc.num_fds()
 
     N = 20
-    nannies = [Nanny(s.ip, s.port, loop=s.loop) for i in range(N)]
-    yield [n._start() for n in nannies]
+    nannies = yield [Nanny(s.ip, s.port, loop=s.loop) for i in range(N)]
 
     while len(s.ncores) < N:
         yield gen.sleep(0.1)
@@ -894,7 +888,7 @@ def test_worker_arrives_with_processing_data(c, s, a, b):
     w = Worker(s.ip, s.port, ncores=1)
     w.put_key_in_memory(y.key, 3)
 
-    yield w._start()
+    yield w
 
     start = time()
 
@@ -945,7 +939,7 @@ def test_no_workers_to_memory(c, s):
     w = Worker(s.ip, s.port, ncores=1)
     w.put_key_in_memory(y.key, 3)
 
-    yield w._start()
+    yield w
 
     start = time()
 
@@ -975,7 +969,7 @@ def test_no_worker_to_memory_restrictions(c, s, a, b):
     w = Worker(s.ip, s.port, ncores=1, name='alice')
     w.put_key_in_memory(y.key, 3)
 
-    yield w._start()
+    yield w
 
     while len(s.workers) < 3:
         yield gen.sleep(0.01)
diff --git a/distributed/tests/test_steal.py b/distributed/tests/test_steal.py
index cd76908ead2..caaa939b665 100644
--- a/distributed/tests/test_steal.py
+++ b/distributed/tests/test_steal.py
@@ -162,8 +162,7 @@ def test_new_worker_steals(c, s, a):
     while len(a.task_state) < 10:
         yield gen.sleep(0.01)
 
-    b = Worker(s.ip, s.port, loop=s.loop, ncores=1, memory_limit=TOTAL_MEMORY)
-    yield b._start()
+    b = yield Worker(s.ip, s.port, loop=s.loop, ncores=1, memory_limit=TOTAL_MEMORY)
 
     result = yield total
     assert result == sum(map(inc, range(100)))
@@ -266,8 +265,7 @@ def test_steal_resource_restrictions(c, s, a):
         yield gen.sleep(0.01)
     assert len(a.task_state) == 101
 
-    b = Worker(s.ip, s.port, loop=s.loop, ncores=1, resources={'A': 4})
-    yield b._start()
+    b = yield Worker(s.ip, s.port, loop=s.loop, ncores=1, resources={'A': 4})
 
     start = time()
     while not b.task_state or len(a.task_state) == 101:
@@ -527,8 +525,8 @@ def test_steal_twice(c, s, a, b):
     while len(s.tasks) < 100:  # tasks are all allocated
         yield gen.sleep(0.01)
 
-    workers = [Worker(s.ip, s.port, loop=s.loop) for _ in range(20)]
-    yield [w._start() for w in workers]  # army of new workers arrives to help
+    # Army of new workers arrives to help
+    workers = yield [Worker(s.ip, s.port, loop=s.loop) for _ in range(20)]
 
     yield wait(futures)
 
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 805d150e555..3c9b51f2baa 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -281,8 +281,7 @@ def test_broadcast(s, a, b):
 def test_worker_with_port_zero():
     s = Scheduler()
     s.start(8007)
-    w = Worker(s.address)
-    yield w._start()
+    w = yield Worker(s.address)
     assert isinstance(w.port, int)
     assert w.port > 1024
 
@@ -293,8 +292,7 @@ def test_worker_with_port_zero():
 def test_worker_waits_for_center_to_come_up(loop):
     @gen.coroutine
     def f():
-        w = Worker('127.0.0.1', 8007)
-        yield w._start()
+        w = yield Worker('127.0.0.1', 8007)
 
     try:
         loop.run_sync(f, timeout=4)
@@ -345,9 +343,8 @@ def test_io_loop(loop):
 @gen_cluster(client=True, ncores=[])
 def test_spill_to_disk(c, s):
     np = pytest.importorskip('numpy')
-    w = Worker(s.address, loop=s.loop, memory_limit=1200 / 0.6,
+    w = yield Worker(s.address, loop=s.loop, memory_limit=1200 / 0.6,
                memory_pause_fraction=None, memory_spill_fraction=None)
-    yield w._start()
 
     x = c.submit(np.random.randint, 0, 255, size=500, dtype='u1', key='x')
     yield wait(x)
@@ -408,7 +405,7 @@ def test_Executor(c, s):
     with ThreadPoolExecutor(2) as e:
         w = Worker(s.ip, s.port, executor=e)
         assert w.executor is e
-        yield w._start()
+        w = yield w
 
         future = c.submit(inc, 1)
         result = yield future
@@ -682,8 +679,7 @@ def test_hold_onto_dependents(c, s, a, b):
 def test_worker_death_timeout(s):
     with dask.config.set({'distributed.comm.timeouts.connect': '1s'}):
         yield s.close()
-        w = Worker(s.address, death_timeout=1)
-        yield w._start()
+        w = yield Worker(s.address, death_timeout=1)
 
     yield gen.sleep(2)
     assert w.status == 'closed'
@@ -902,8 +898,7 @@ def test_worker_fds(s):
     yield gen.sleep(0.05)
     start = psutil.Process().num_fds()
 
-    worker = Worker(s.address, loop=s.loop)
-    yield worker._start()
+    worker = yield Worker(s.address, loop=s.loop)
     yield gen.sleep(0.1)
     middle = psutil.Process().num_fds()
     start = time()
@@ -962,8 +957,7 @@ def test_scheduler_file():
     with tmpfile() as fn:
         s = Scheduler(scheduler_file=fn)
         s.start(8009)
-        w = Worker(scheduler_file=fn)
-        yield w._start()
+        w = yield Worker(scheduler_file=fn)
         assert set(s.workers) == {w.address}
         yield w._close()
         s.stop()
@@ -1122,9 +1116,8 @@ def test_deque_handler():
 
 @gen_cluster(ncores=[], client=True)
 def test_avoid_memory_monitor_if_zero_limit(c, s):
-    worker = Worker(s.address, loop=s.loop, memory_limit=0,
+    worker = yield Worker(s.address, loop=s.loop, memory_limit=0,
                     memory_monitor_interval=10)
-    yield worker._start()
     assert type(worker.data) is dict
     assert 'memory' not in worker.periodic_callbacks
 
@@ -1168,8 +1161,7 @@ def test_parse_memory_limit(s, w):
 @gen_cluster(ncores=[], client=True)
 def test_scheduler_address_config(c, s):
     with dask.config.set({'scheduler-address': s.address}):
-        worker = Worker(loop=s.loop)
-        yield worker._start()
+        worker = yield Worker(loop=s.loop)
         assert worker.scheduler.address == s.address
     yield worker._close()
 
@@ -1259,8 +1251,7 @@ def test_startup2():
     assert list(result.values()) == [False] * 2
 
     # Start a worker and check that startup is not run
-    worker = Worker(s.address, loop=s.loop)
-    yield worker._start()
+    worker = yield Worker(s.address, loop=s.loop)
     result = yield c.run(test_import, workers=[worker.address])
     assert list(result.values()) == [False]
     yield worker._close()
@@ -1275,8 +1266,7 @@ def test_startup2():
     assert list(result.values()) == [True] * 2
 
     # Start a worker and check it is ran on it
-    worker = Worker(s.address, loop=s.loop)
-    yield worker._start()
+    worker = yield Worker(s.address, loop=s.loop)
     result = yield c.run(test_import, workers=[worker.address])
     assert list(result.values()) == [True]
     yield worker._close()
@@ -1291,8 +1281,7 @@ def test_startup2():
     assert list(result.values()) == [True] * 2
 
     # Start a worker and check it is ran on it
-    worker = Worker(s.address, loop=s.loop)
-    yield worker._start()
+    worker = yield Worker(s.address, loop=s.loop)
     result = yield c.run(test_import, workers=[worker.address])
     assert list(result.values()) == [True]
     result = yield c.run(test_startup2, workers=[worker.address])
@@ -1302,3 +1291,25 @@ def test_startup2():
     # Final exception test
     with pytest.raises(ZeroDivisionError):
         yield c.register_worker_callbacks(setup=lambda: 1 / 0)
+
+
+@gen_cluster(ncores=[])
+def test_data_types(s):
+    w = yield Worker(s.address, data=dict)
+    assert isinstance(w.data, dict)
+    yield w._close()
+
+    data = dict()
+    w = yield Worker(s.address, data=data)
+    assert w.data is data
+    yield w._close()
+
+    class Data(dict):
+        def __init__(self, x, y):
+            self.x = x
+            self.y = y
+
+    w = yield Worker(s.address, data=(Data, {'x': 123, 'y': 456}))
+    assert w.data.x == 123
+    assert w.data.y == 456
+    yield w._close()
diff --git a/distributed/worker.py b/distributed/worker.py
index 2ef303735c5..2ed696214a3 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -29,7 +29,8 @@
 from .batched import BatchedSend
 from .comm import get_address_host, get_local_address_for, connect
 from .comm.utils import offload
-from .compatibility import unicode, get_thread_identity, finalize
+from .compatibility import (unicode, get_thread_identity, finalize,
+        MutableMapping)
 from .core import (error_message, CommClosedError, send_recv,
                    pingpong, coerce_to_address)
 from .diskutils import WorkSpace
@@ -216,6 +217,8 @@ class Worker(ServerNode):
     scheduler_ip: str
     scheduler_port: int
     ip: str, optional
+    data: MutableMapping, type, None
+        The object to use for storage, builds a disk-backed LRU dict by default
     ncores: int, optional
     loop: tornado.ioloop.IOLoop
     local_dir: str, optional
@@ -260,7 +263,7 @@ def __init__(self, scheduler_ip=None, scheduler_port=None,
                  executor=None, resources=None, silence_logs=None,
                  death_timeout=None, preload=None, preload_argv=None, security=None,
                  contact_address=None, memory_monitor_interval='200ms',
-                 extensions=None, metrics=None, **kwargs):
+                 extensions=None, metrics=None, data=None, **kwargs):
         self.tasks = dict()
         self.task_state = dict()
         self.dep_state = dict()
@@ -411,7 +414,13 @@ def __init__(self, scheduler_ip=None, scheduler_port=None,
         else:
             self.memory_pause_fraction = dask.config.get('distributed.worker.memory.pause')
 
-        if (self.memory_limit and
+        if isinstance(data, MutableMapping):
+            self.data = data
+        elif callable(data):
+            self.data = data()
+        elif isinstance(data, tuple):
+            self.data = data[0](**data[1])
+        elif (self.memory_limit and
                 (self.memory_target_fraction or
                  self.memory_spill_fraction)):
             try:
@@ -836,6 +845,10 @@ def _start(self, addr_or_port=0):
         yield self._register_with_scheduler()
 
         self.start_periodic_callbacks()
+        raise gen.Return(self)
+
+    def __await__(self):
+        return self._start().__await__()
 
     def start(self, port=0):
         self.loop.add_callback(self._start, port)

From 34905e0a90b327530133e153f130ee10c3429fc7 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 9 Apr 2019 10:33:18 -0500
Subject: [PATCH 0228/1550] Use ensure_bytes in serialize_error (#2588)

---
 distributed/protocol/serialize.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index d81a307a0f9..a6cfbd6d042 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -15,7 +15,8 @@
 from ..compatibility import PY2
 from ..utils import has_keyword
 from .compression import maybe_compress, decompress
-from .utils import unpack_frames, pack_frames_prelude, frame_split_size
+from .utils import (unpack_frames, pack_frames_prelude, frame_split_size,
+        ensure_bytes)
 
 
 lazy_registrations = {}
@@ -77,7 +78,7 @@ def msgpack_loads(header, frames):
 
 
 def serialization_error_loads(header, frames):
-    msg = '\n'.join([frame.decode('utf8') for frame in frames])
+    msg = '\n'.join([ensure_bytes(frame).decode('utf8') for frame in frames])
     raise TypeError(msg)
 
 

From 61be8a65e4bd6e211dbcdb77dafd94ccd5b35904 Mon Sep 17 00:00:00 2001
From: Matt Nicolls <2540582+nicolls1@users.noreply.github.com>
Date: Wed, 10 Apr 2019 16:10:00 +0200
Subject: [PATCH 0229/1550] add resources to scheduler update_graph plugin
 (#2603)

---
 distributed/scheduler.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index c4ecbf9e8a9..569d097690e 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1567,7 +1567,8 @@ def update_graph(self, client=None, tasks=None, keys=None,
                                     keys=keys, restrictions=restrictions or {},
                                     dependencies=dependencies,
                                     priority=priority,
-                                    loose_restrictions=loose_restrictions)
+                                    loose_restrictions=loose_restrictions,
+                                    resources=resources)
             except Exception as e:
                 logger.exception(e)
 

From 3b67416f0b8ea2c402ca419b8a72407d12d22f66 Mon Sep 17 00:00:00 2001
From: Matt Nicolls <2540582+nicolls1@users.noreply.github.com>
Date: Wed, 10 Apr 2019 19:36:21 +0200
Subject: [PATCH 0230/1550] Add RabbitMQ SchedulerPlugin example (#2604)

---
 docs/source/plugins.rst | 46 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 46 insertions(+)

diff --git a/docs/source/plugins.rst b/docs/source/plugins.rst
index 2014a3d5758..edf64362c56 100644
--- a/docs/source/plugins.rst
+++ b/docs/source/plugins.rst
@@ -3,3 +3,49 @@ Scheduler Plugins
 
 .. autoclass:: distributed.diagnostics.plugin.SchedulerPlugin
    :members:
+
+
+RabbitMQ Example
+----------------
+
+RabbitMQ is a distributed messaging queue that we can use to post updates about
+task transitions. By posting transitions to RabbitMQ, we allow other machines
+to do the processing of transitions and keep scheduler processing to a minimum.
+See the
+`RabbitMQ tutorial <https://www.rabbitmq.com/tutorials/tutorial-two-python.html>`_
+for more information on RabbitMQ and how to consume the messages.
+
+.. code-block:: python
+
+   import json
+   from distributed.diagnostics.plugin import SchedulerPlugin
+   import pika
+
+   class RabbitMQPlugin(SchedulerPlugin):
+       def __init__(self):
+           # Update host to be your RabbitMQ host
+           self.connection = pika.BlockingConnection(
+               pika.ConnectionParameters(host='localhost'))
+           self.channel = self.connection.channel()
+           self.channel.queue_declare(queue='dask_task_status', durable=True)
+
+       def transition(self, key, start, finish, *args, **kwargs):
+           message = dict(
+               key=key,
+               start=start,
+               finish=finish,
+           )
+           self.channel.basic_publish(
+               exchange='',
+               routing_key='dask_task_status',
+               body=json.dumps(message),
+               properties=pika.BasicProperties(
+                   delivery_mode=2,  # make message persistent
+               ))
+
+   @click.command()
+   def dask_setup(scheduler):
+       plugin = RabbitMQPlugin()
+       scheduler.add_plugin(plugin)
+
+Run with: ``dask-scheduler --preload <filename.py>``

From 2c37805ed7dac34470cc64f6916e355edb42fe74 Mon Sep 17 00:00:00 2001
From: deepthirajagopalan7
 <49467685+deepthirajagopalan7@users.noreply.github.com>
Date: Wed, 10 Apr 2019 18:56:08 -0700
Subject: [PATCH 0231/1550] Check self.dependencies when looking at dependent
 tasks in memory (#2606)

---
 distributed/scheduler.py            |  6 +++++-
 distributed/tests/test_scheduler.py | 12 ++++++++----
 2 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 569d097690e..cf26d0c955d 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1425,7 +1425,11 @@ def update_graph(self, client=None, tasks=None, keys=None,
                 except KeyError:
                     deps = self.dependencies[key]
                 for dep in deps:
-                    if all(d in done for d in dependents[dep]):
+                    if dep in dependents:
+                        child_deps = dependents[dep]
+                    else:
+                        child_deps = self.dependencies[dep]
+                    if all(d in done for d in child_deps):
                         if dep in self.tasks:
                             done.add(dep)
                             stack.append(dep)
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 1d101cd5224..edf4a4eaece 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -1437,13 +1437,17 @@ def bar(x):
         return x + 'bar'
 
     def baz(x):
-        sleep(0.1)
         return x + 'baz'
 
-    x = c.submit(foo, key='x')
-    y = c.submit(bar, x, key='y')
+    def qux(x):
+        sleep(0.1)
+        return x + 'qux'
+
+    w = c.submit(foo, key='w')
+    x = c.submit(bar, w, key='x')
+    y = c.submit(baz, x, key='y')
     yield y
-    z = c.submit(baz, y, key='z')
+    z = c.submit(qux, y, key='z')
     del y
     yield gen.sleep(0.1)
     f = c.submit(bar, x, key='y')

From af677b7dd1c43c7d28a3bc8a709bb0b802fbe5d3 Mon Sep 17 00:00:00 2001
From: Brett Randall <javabrett@gmail.com>
Date: Thu, 11 Apr 2019 11:59:49 +1000
Subject: [PATCH 0232/1550] Improved description accuracy of --memory-limit
 option. (#2601)

Signed-off-by: Brett Randall <javabrett@gmail.com>
---
 distributed/worker.py  | 3 ++-
 docs/source/worker.rst | 8 +++++---
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/distributed/worker.py b/distributed/worker.py
index 2ed696214a3..c7720888f4f 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -226,7 +226,8 @@ class Worker(ServerNode):
     name: str, optional
     memory_limit: int, float, string
         Number of bytes of memory that this worker should use.
-        Set to zero for no limit.  Set to 'auto' for 60% of memory use.
+        Set to zero for no limit.  Set to 'auto' to calculate
+        as TOTAL_MEMORY * min(1, ncores / total_cores)
         Use strings or numbers like 5GB or 5e9
     memory_target_fraction: float
         Fraction of memory to try to stay beneath
diff --git a/docs/source/worker.rst b/docs/source/worker.rst
index ce888a388f8..4b835d7ba67 100644
--- a/docs/source/worker.rst
+++ b/docs/source/worker.rst
@@ -70,7 +70,7 @@ thread per process::
    $ dask-worker scheduler:8786 --nprocs 8 --nthreads 1
 
 This will launch 8 worker processes each of which has its own
-ThreadPoolExecutor of size 1. 
+ThreadPoolExecutor of size 1.
 
 If your computations are external to Python and long-running and don't release
 the GIL then beware that while the computation is running the worker process
@@ -98,7 +98,9 @@ are the available options::
                             cores
      --nprocs INTEGER       Number of worker processes to launch.  Defaults to one.
      --name TEXT            Alias
-     --memory-limit TEXT    Number of bytes (per worker process) before spilling data to disk
+     --memory-limit TEXT    Maximum bytes of memory that this worker should use.
+                            Use 0 for unlimited, or 'auto' for
+                            TOTAL_MEMORY * min(1, ncores / total_cores)
      --no-nanny
      --help                 Show this message and exit.
 
@@ -149,7 +151,7 @@ command line ``--memory-limit`` keyword or the ``memory_limit=`` Python
 keyword argument, which sets the memory limit per worker processes launched
 by dask-worker ::
 
-    $ dask-worker tcp://scheduler:port --memory-limit=auto  # total available RAM on the machine
+    $ dask-worker tcp://scheduler:port --memory-limit=auto  # TOTAL_MEMORY * min(1, ncores / total_cores)
     $ dask-worker tcp://scheduler:port --memory-limit=4e9  # four gigabytes per worker process.
 
 Workers use a few different heuristics to keep memory use beneath this limit:

From 01c47334e22e2060a64482fd598d645b36749539 Mon Sep 17 00:00:00 2001
From: amerkel2 <andreas.merkel@blue-yonder.com>
Date: Fri, 12 Apr 2019 18:17:20 +0200
Subject: [PATCH 0233/1550] Add basic health endpoints to scheduler and worker
 bokeh. (#2607)

---
 distributed/bokeh/scheduler_html.py               |  7 +++++++
 .../bokeh/tests/test_scheduler_bokeh_html.py      | 15 +++++++++++++++
 distributed/bokeh/tests/test_worker_bokeh_html.py | 14 ++++++++++++++
 distributed/bokeh/worker_html.py                  |  7 +++++++
 docs/source/web.rst                               |  1 +
 5 files changed, 44 insertions(+)

diff --git a/distributed/bokeh/scheduler_html.py b/distributed/bokeh/scheduler_html.py
index e8b030156dd..e8050e4a9fa 100644
--- a/distributed/bokeh/scheduler_html.py
+++ b/distributed/bokeh/scheduler_html.py
@@ -210,6 +210,12 @@ def get(self):
         self.set_header('Content-Type', 'text/plain; version=0.0.4')
 
 
+class HealthHandler(RequestHandler):
+    def get(self):
+        self.write('ok')
+        self.set_header('Content-Type', 'text/plain')
+
+
 routes = [
         (r'info/main/workers.html', Workers),
         (r'info/worker/(.*).html', Worker),
@@ -223,6 +229,7 @@ def get(self):
         (r'json/index.html', IndexJSON),
         (r'individual-plots.json', IndividualPlots),
         (r'metrics', PrometheusHandler),
+        (r'health', HealthHandler),
 ]
 
 
diff --git a/distributed/bokeh/tests/test_scheduler_bokeh_html.py b/distributed/bokeh/tests/test_scheduler_bokeh_html.py
index 5f8b55a531d..96dc71c6e67 100644
--- a/distributed/bokeh/tests/test_scheduler_bokeh_html.py
+++ b/distributed/bokeh/tests/test_scheduler_bokeh_html.py
@@ -85,3 +85,18 @@ def test_prometheus(c, s, a, b):
             for familiy in text_string_to_metric_families(txt)
         }
         assert 'dask_scheduler_workers' in families
+
+
+@gen_cluster(client=True,
+    check_new_threads=False,
+    scheduler_kwargs={'services': {('bokeh', 0):  BokehScheduler}})
+def test_health(c, s, a, b):
+    http_client = AsyncHTTPClient()
+
+    response = yield http_client.fetch('http://localhost:%d/health'
+                                       % s.services['bokeh'].port)
+    assert response.code == 200
+    assert response.headers['Content-Type'] == 'text/plain'
+
+    txt = response.body.decode('utf8')
+    assert txt == 'ok'
diff --git a/distributed/bokeh/tests/test_worker_bokeh_html.py b/distributed/bokeh/tests/test_worker_bokeh_html.py
index 0a56e97e055..80819972050 100644
--- a/distributed/bokeh/tests/test_worker_bokeh_html.py
+++ b/distributed/bokeh/tests/test_worker_bokeh_html.py
@@ -28,3 +28,17 @@ def test_prometheus(c, s, a, b):
             for familiy in text_string_to_metric_families(txt)
         }
         assert len(families) > 0
+
+
+@gen_cluster(client=True,
+    worker_kwargs={'services': {('bokeh', 0):  BokehWorker}})
+def test_health(c, s, a, b):
+    http_client = AsyncHTTPClient()
+
+    response = yield http_client.fetch('http://localhost:%d/health'
+                                       % a.services['bokeh'].port)
+    assert response.code == 200
+    assert response.headers['Content-Type'] == 'text/plain'
+
+    txt = response.body.decode('utf8')
+    assert txt == 'ok'
diff --git a/distributed/bokeh/worker_html.py b/distributed/bokeh/worker_html.py
index a3c24b085fc..5a956231c6a 100644
--- a/distributed/bokeh/worker_html.py
+++ b/distributed/bokeh/worker_html.py
@@ -63,8 +63,15 @@ def get(self):
         self.set_header('Content-Type', 'text/plain; version=0.0.4')
 
 
+class HealthHandler(RequestHandler):
+    def get(self):
+        self.write('ok')
+        self.set_header('Content-Type', 'text/plain')
+
+
 routes = [
         (r'metrics', PrometheusHandler),
+        (r'health', HealthHandler),
 ]
 
 
diff --git a/docs/source/web.rst b/docs/source/web.rst
index c888bb30763..c73838c13dd 100644
--- a/docs/source/web.rst
+++ b/docs/source/web.rst
@@ -50,6 +50,7 @@ The available pages are ``http://scheduler-address:8787/<page>/`` where ``<page>
 - ``status``: a stream of recently run tasks, progress bars, resource use
 - ``tasks``: a larger stream of the last 100k tasks
 - ``workers``: basic information about workers and their current load
+- ``health``: basic health check, returns ``ok`` if service is running
 
 .. _Bokeh: http://bokeh.pydata.org/en/latest/
 

From 8c07c8783c777d3bb6615a0e7607f7f0e562b1d3 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 12 Apr 2019 16:29:55 -0500
Subject: [PATCH 0234/1550] bump version to 1.27.0

---
 docs/source/changelog.rst | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index db53bd25bbf..affad66a759 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,25 @@
 Changelog
 =========
 
+1.27.0 - 2019-04-12
+-------------------
+
+   Add basic health endpoints to scheduler and worker bokeh. (#2607) `amerkel2`_
+-  Improved description accuracy of --memory-limit option. (:pr:`2601`) `Brett Randall`_
+-  Check self.dependencies when looking at dependent tasks in memory (:pr:`2606`) `deepthirajagopalan7`_
+-  Add RabbitMQ SchedulerPlugin example (:pr:`2604`) `Matt Nicolls`_
+-  add resources to scheduler update_graph plugin (:pr:`2603`) `Matt Nicolls`_
+-  Use ensure_bytes in serialize_error (:pr:`2588`) `Matthew Rocklin`_
+-  Specify data storage explicitly from Worker constructor (:pr:`2600`) `Matthew Rocklin`_
+-  Change bokeh port keywords to dashboard_address (:pr:`2589`) `Matthew Rocklin`_
+-  .detach_(`) pytorch tensor to serialize data as numpy array. (:pr:`2586`) `Muammar El Khatib`_
+-  Add warning if creating scratch directories takes a long time (:pr:`2561`) `Matthew Rocklin`_
+-  Fix typo in pub-sub doc. (:pr:`2599`) `Loïc Estève`_
+-  Allow return_when='FIRST_COMPLETED' in wait (:pr:`2598`) `Nikos Tsaousis`_
+-  Forward kwargs through Nanny to Worker (:pr:`2596`) `Brian Chu`_
+-  Use ensure_dict instead of dict (:pr:`2594`) `James Bourbeau`_
+-  Specify protocol in LocalCluster (:pr:`2489`) `Matthew Rocklin`_
+
 1.26.1 - 2019-03-29
 -------------------
 
@@ -960,3 +979,10 @@ significantly without many new features.
 .. _`Elliott Sales de Andrade`: https://github.com/QuLogic
 .. _`Brett Jurman`: https://github.com/ibebrett
 .. _`Julia Kent`: https://github.com/jukent
+.. _`Brett Randall`: https://github.com/javabrett
+.. _`deepthirajagopalan7`: https://github.com/deepthirajagopalan7
+.. _`Muammar El Khatib`: https://github.com/muammar
+.. _`Nikos Tsaousis`: https://github.com/tsanikgr
+.. _`Brian Chu`: https://github.com/bchu
+.. _`James Bourbeau`: https://github.com/jrbourbeau
+.. _`amerkel2`: https://github.com/amerkel2

From 1b32bd30201ef6ced5029180143d2c37b393b586 Mon Sep 17 00:00:00 2001
From: condoratberlin <49398997+condoratberlin@users.noreply.github.com>
Date: Mon, 15 Apr 2019 14:50:06 +0200
Subject: [PATCH 0235/1550] added missing whitespace to start_worker cmd
 (#2613)

A whitspace was missing in front of the optional nprocs option. Aditionally changed the following optional options to a more readable way, where the separating whitspace is always in front of the additional command.
---
 distributed/deploy/ssh.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/distributed/deploy/ssh.py b/distributed/deploy/ssh.py
index d0d6813083d..20888471ff8 100644
--- a/distributed/deploy/ssh.py
+++ b/distributed/deploy/ssh.py
@@ -220,19 +220,19 @@ def start_worker(logdir, scheduler_addr, scheduler_port, worker_addr, nthreads,
     cmd = ('{python} -m {remote_dask_worker} '
            '{scheduler_addr}:{scheduler_port} '
            '--nthreads {nthreads}'
-           + ('--nprocs {nprocs}' if nprocs != 1 else ''))
+           + (' --nprocs {nprocs}' if nprocs != 1 else ''))
 
     if not nohost:
-        cmd += ' --host {worker_addr} '
+        cmd += ' --host {worker_addr}'
 
     if memory_limit:
-        cmd += '--memory-limit {memory_limit} '
+        cmd += ' --memory-limit {memory_limit}'
 
     if worker_port:
-        cmd += '--worker-port {worker_port} '
+        cmd += ' --worker-port {worker_port}'
 
     if nanny_port:
-        cmd += '--nanny-port {nanny_port} '
+        cmd += ' --nanny-port {nanny_port}'
 
     cmd = cmd.format(
         python=remote_python or sys.executable,

From 04ae4212cf6d1e513a4ea83666719c79b7e5867a Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 15 Apr 2019 14:52:54 -0500
Subject: [PATCH 0236/1550] Apply Black to standardize code styling (#2614)

This uses the [black](https://github.com/ambv/black) code styler to manage our style.

1.  Apply `black` to the entire codebase
2.  Add `black` to our Travis CI
3.  Add a pre-commit hook to make this automatic,
    should people use `pre-commit install` at the base of this repository

The only concern I have about this is that it screws with `git blame`.

To future maintainers that come across this commit,
I recommend heading back one more commit `HEAD^`
and then using `git blame` again.
---
 .pre-commit-config.yaml                       |   10 +
 .travis.yml                                   |    1 +
 distributed/__init__.py                       |   20 +-
 distributed/_concurrent_futures_thread.py     |   28 +-
 distributed/_ipython_utils.py                 |   53 +-
 distributed/_version.py                       |  154 +-
 distributed/actor.py                          |   29 +-
 distributed/asyncio.py                        |    3 +
 distributed/batched.py                        |   28 +-
 distributed/bokeh/__init__.py                 |   54 +-
 distributed/bokeh/components.py               |  364 +--
 distributed/bokeh/core.py                     |   69 +-
 distributed/bokeh/export_tool.py              |    4 +-
 distributed/bokeh/scheduler.py                | 1089 +++++----
 distributed/bokeh/scheduler_html.py           |  161 +-
 distributed/bokeh/tests/test_components.py    |   23 +-
 .../bokeh/tests/test_scheduler_bokeh.py       |  245 +-
 .../bokeh/tests/test_scheduler_bokeh_html.py  |   99 +-
 distributed/bokeh/tests/test_worker_bokeh.py  |  111 +-
 .../bokeh/tests/test_worker_bokeh_html.py     |   36 +-
 distributed/bokeh/utils.py                    |    3 +-
 distributed/bokeh/worker.py                   |  619 +++--
 distributed/bokeh/worker_html.py              |   21 +-
 distributed/cfexecutor.py                     |   27 +-
 distributed/cli/dask_mpi.py                   |  123 +-
 distributed/cli/dask_remote.py                |    7 +-
 distributed/cli/dask_scheduler.py             |  212 +-
 distributed/cli/dask_ssh.py                   |  192 +-
 distributed/cli/dask_submit.py                |    6 +-
 distributed/cli/dask_worker.py                |  357 ++-
 distributed/cli/tests/test_cli_utils.py       |   59 +-
 distributed/cli/tests/test_dask_mpi.py        |   58 +-
 distributed/cli/tests/test_dask_remote.py     |    4 +-
 distributed/cli/tests/test_dask_scheduler.py  |  232 +-
 distributed/cli/tests/test_dask_submit.py     |    4 +-
 distributed/cli/tests/test_dask_worker.py     |  226 +-
 distributed/cli/tests/test_tls_cli.py         |   63 +-
 distributed/cli/utils.py                      |   23 +-
 distributed/client.py                         | 1481 +++++++-----
 distributed/comm/__init__.py                  |   17 +-
 distributed/comm/addressing.py                |   35 +-
 distributed/comm/core.py                      |   44 +-
 distributed/comm/inproc.py                    |   77 +-
 distributed/comm/registry.py                  |    5 +-
 distributed/comm/tcp.py                       |  174 +-
 distributed/comm/tests/test_comms.py          |  493 ++--
 distributed/comm/utils.py                     |   31 +-
 distributed/compatibility.py                  |   31 +-
 distributed/config.py                         |  102 +-
 distributed/core.py                           |  322 ++-
 distributed/counter.py                        |    1 +
 distributed/deploy/__init__.py                |    1 +
 distributed/deploy/adaptive.py                |   90 +-
 distributed/deploy/cluster.py                 |   65 +-
 distributed/deploy/local.py                   |  176 +-
 distributed/deploy/ssh.py                     |  340 ++-
 distributed/deploy/tests/test_adaptive.py     |  193 +-
 distributed/deploy/tests/test_local.py        |  453 ++--
 distributed/deploy/tests/test_ssh.py          |   13 +-
 distributed/diagnostics/__init__.py           |    1 +
 distributed/diagnostics/eventstream.py        |   20 +-
 distributed/diagnostics/graph_layout.py       |   44 +-
 distributed/diagnostics/plugin.py             |    3 +-
 distributed/diagnostics/progress.py           |   68 +-
 distributed/diagnostics/progress_stream.py    |  175 +-
 distributed/diagnostics/progressbar.py        |  267 ++-
 distributed/diagnostics/task_stream.py        |   97 +-
 .../diagnostics/tests/test_eventstream.py     |   17 +-
 .../diagnostics/tests/test_graph_layout.py    |    6 +-
 distributed/diagnostics/tests/test_plugin.py  |   18 +-
 .../diagnostics/tests/test_progress.py        |   85 +-
 .../diagnostics/tests/test_progress_stream.py |  126 +-
 .../diagnostics/tests/test_progressbar.py     |   35 +-
 .../diagnostics/tests/test_task_stream.py     |   37 +-
 distributed/diagnostics/tests/test_widgets.py |  168 +-
 distributed/diskutils.py                      |   55 +-
 distributed/lock.py                           |   26 +-
 distributed/locket.py                         |   20 +-
 distributed/metrics.py                        |    7 +-
 distributed/nanny.py                          |  327 +--
 distributed/node.py                           |   70 +-
 distributed/preloading.py                     |   21 +-
 distributed/process.py                        |   54 +-
 distributed/proctitle.py                      |    4 +-
 distributed/profile.py                        |  187 +-
 distributed/protocol/__init__.py              |   21 +-
 distributed/protocol/arrow.py                 |    9 +-
 distributed/protocol/compression.py           |   70 +-
 distributed/protocol/core.py                  |  100 +-
 distributed/protocol/h5py.py                  |   11 +-
 distributed/protocol/keras.py                 |   17 +-
 distributed/protocol/netcdf4.py               |   20 +-
 distributed/protocol/numpy.py                 |   38 +-
 distributed/protocol/pickle.py                |   14 +-
 distributed/protocol/serialize.py             |  146 +-
 distributed/protocol/sparse.py                |   20 +-
 distributed/protocol/tests/test_arrow.py      |    9 +-
 distributed/protocol/tests/test_h5py.py       |   53 +-
 distributed/protocol/tests/test_keras.py      |    9 +-
 distributed/protocol/tests/test_netcdf4.py    |   54 +-
 distributed/protocol/tests/test_numpy.py      |  215 +-
 distributed/protocol/tests/test_pandas.py     |   32 +-
 distributed/protocol/tests/test_pickle.py     |    5 +-
 distributed/protocol/tests/test_protocol.py   |  140 +-
 .../protocol/tests/test_protocol_utils.py     |   14 +-
 distributed/protocol/tests/test_serialize.py  |  119 +-
 distributed/protocol/tests/test_sklearn.py    |    5 +-
 distributed/protocol/tests/test_sparse.py     |    5 +-
 distributed/protocol/tests/test_torch.py      |    8 +-
 distributed/protocol/torch.py                 |   32 +-
 distributed/protocol/utils.py                 |   23 +-
 distributed/publish.py                        |   25 +-
 distributed/pubsub.py                         |  136 +-
 distributed/pytest_resourceleaks.py           |  166 +-
 distributed/queues.py                         |  124 +-
 distributed/recreate_exceptions.py            |   28 +-
 distributed/scheduler.py                      | 2006 ++++++++++-------
 distributed/security.py                       |   69 +-
 distributed/sizeof.py                         |    3 +-
 distributed/stealing.py                       |  156 +-
 distributed/submit.py                         |   39 +-
 distributed/system_monitor.py                 |   29 +-
 distributed/tests/make_tls_certs.py           |  137 +-
 distributed/tests/py3_test_asyncio.py         |   31 +-
 distributed/tests/py3_test_client.py          |   49 +-
 distributed/tests/py3_test_pubsub.py          |    4 +-
 distributed/tests/test_actor.py               |   83 +-
 distributed/tests/test_as_completed.py        |   24 +-
 distributed/tests/test_asyncprocess.py        |   34 +-
 distributed/tests/test_batched.py             |   67 +-
 distributed/tests/test_client.py              | 1508 +++++++------
 distributed/tests/test_client_executor.py     |   26 +-
 distributed/tests/test_collections.py         |  121 +-
 distributed/tests/test_compatibility.py       |    5 +-
 distributed/tests/test_config.py              |   92 +-
 distributed/tests/test_core.py                |  285 +--
 distributed/tests/test_counter.py             |   16 +-
 distributed/tests/test_diskutils.py           |  112 +-
 distributed/tests/test_failed_workers.py      |   77 +-
 distributed/tests/test_ipython.py             |   85 +-
 distributed/tests/test_locks.py               |   60 +-
 distributed/tests/test_metrics.py             |    2 +-
 distributed/tests/test_nanny.py               |   98 +-
 distributed/tests/test_preload.py             |   29 +-
 distributed/tests/test_priorities.py          |   52 +-
 distributed/tests/test_profile.py             |  125 +-
 distributed/tests/test_publish.py             |   70 +-
 distributed/tests/test_pubsub.py              |   48 +-
 distributed/tests/test_queues.py              |   68 +-
 distributed/tests/test_resources.py           |  261 ++-
 distributed/tests/test_scheduler.py           |  616 ++---
 distributed/tests/test_security.py            |  328 ++-
 distributed/tests/test_steal.py               |  359 +--
 distributed/tests/test_stress.py              |  118 +-
 distributed/tests/test_submit_cli.py          |   36 +-
 .../tests/test_submit_remote_client.py        |   36 +-
 distributed/tests/test_system_monitor.py      |    2 +-
 distributed/tests/test_threadpoolexecutor.py  |    1 +
 distributed/tests/test_tls_functional.py      |   39 +-
 distributed/tests/test_utils.py               |  255 ++-
 distributed/tests/test_utils_comm.py          |   21 +-
 distributed/tests/test_utils_perf.py          |   32 +-
 distributed/tests/test_utils_test.py          |   59 +-
 distributed/tests/test_variable.py            |   57 +-
 distributed/tests/test_worker.py              |  489 ++--
 distributed/tests/test_worker_client.py       |   42 +-
 distributed/threadpoolexecutor.py             |   15 +-
 distributed/utils.py                          |  344 +--
 distributed/utils_comm.py                     |   54 +-
 distributed/utils_perf.py                     |   67 +-
 distributed/utils_test.py                     |  475 ++--
 distributed/variable.py                       |   85 +-
 distributed/versions.py                       |   69 +-
 distributed/worker.py                         | 1529 +++++++------
 distributed/worker_client.py                  |    2 +-
 175 files changed, 13699 insertions(+), 9869 deletions(-)
 create mode 100644 .pre-commit-config.yaml

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
new file mode 100644
index 00000000000..5939ad63655
--- /dev/null
+++ b/.pre-commit-config.yaml
@@ -0,0 +1,10 @@
+repos:
+-   repo: https://github.com/ambv/black
+    rev: stable
+    hooks:
+    - id: black
+      language_version: python3.7
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v1.2.3
+    hooks:
+    - id: flake8
diff --git a/.travis.yml b/.travis.yml
index a91f35bf037..96331468b55 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -32,6 +32,7 @@ install:
 script:
   - if [[ $TESTS == true ]]; then source continuous_integration/travis/run_tests.sh ; fi
   - if [[ $LINT == true ]]; then pip install flake8 ; flake8 distributed ; fi
+  - if [[ $LINT == true ]]; then pip install black; black distributed ; fi
 
 after_success:
   - if [[ $COVERAGE == true ]]; then coverage report; pip install -q coveralls ; coveralls ; fi
diff --git a/distributed/__init__.py b/distributed/__init__.py
index ac324592dd2..7b2bc4ab082 100644
--- a/distributed/__init__.py
+++ b/distributed/__init__.py
@@ -6,9 +6,18 @@
 from .core import connect, rpc
 from .deploy import LocalCluster, Adaptive
 from .diagnostics import progress
-from .client import (Client, Executor, CompatibleExecutor,
-                     wait, as_completed, default_client, fire_and_forget,
-                     Future, futures_of, get_task_stream)
+from .client import (
+    Client,
+    Executor,
+    CompatibleExecutor,
+    wait,
+    as_completed,
+    default_client,
+    fire_and_forget,
+    Future,
+    futures_of,
+    get_task_stream,
+)
 from .lock import Lock
 from .nanny import Nanny
 from .pubsub import Pub, Sub
@@ -23,7 +32,8 @@
 from tornado.gen import TimeoutError
 
 from ._version import get_versions
+
 versions = get_versions()
-__version__ = versions['version']
-__git_revision__ = versions['full-revisionid']
+__version__ = versions["version"]
+__git_revision__ = versions["full-revisionid"]
 del get_versions, versions
diff --git a/distributed/_concurrent_futures_thread.py b/distributed/_concurrent_futures_thread.py
index 06aa810e1af..02ff7c649aa 100644
--- a/distributed/_concurrent_futures_thread.py
+++ b/distributed/_concurrent_futures_thread.py
@@ -5,11 +5,12 @@
 
 """Implements ThreadPoolExecutor."""
 
-__author__ = 'Brian Quinlan (brian@sweetapp.com)'
+__author__ = "Brian Quinlan (brian@sweetapp.com)"
 
 import atexit
 from concurrent.futures import _base
 import itertools
+
 try:
     import queue
 except ImportError:
@@ -88,7 +89,7 @@ def _worker(executor_reference, work_queue):
                 return
             del executor
     except BaseException:
-        _base.LOGGER.critical('Exception in worker', exc_info=True)
+        _base.LOGGER.critical("Exception in worker", exc_info=True)
 
 
 class ThreadPoolExecutor(_base.Executor):
@@ -96,7 +97,7 @@ class ThreadPoolExecutor(_base.Executor):
     # Used to assign unique thread names when thread_name_prefix is not supplied.
     _counter = itertools.count()
 
-    def __init__(self, max_workers=None, thread_name_prefix=''):
+    def __init__(self, max_workers=None, thread_name_prefix=""):
         """Initializes a new ThreadPoolExecutor instance.
 
         Args:
@@ -116,13 +117,14 @@ def __init__(self, max_workers=None, thread_name_prefix=''):
         self._threads = set()
         self._shutdown = False
         self._shutdown_lock = threading.Lock()
-        self._thread_name_prefix = (thread_name_prefix or
-                                    ("ThreadPoolExecutor-%d" % next(self._counter)))
+        self._thread_name_prefix = thread_name_prefix or (
+            "ThreadPoolExecutor-%d" % next(self._counter)
+        )
 
     def submit(self, fn, *args, **kwargs):
         with self._shutdown_lock:
             if self._shutdown:
-                raise RuntimeError('cannot schedule new futures after shutdown')
+                raise RuntimeError("cannot schedule new futures after shutdown")
 
             f = _base.Future()
             w = _WorkItem(f, fn, args, kwargs)
@@ -130,6 +132,7 @@ def submit(self, fn, *args, **kwargs):
             self._work_queue.put(w)
             self._adjust_thread_count()
             return f
+
     submit.__doc__ = _base.Executor.submit.__doc__
 
     def _adjust_thread_count(self):
@@ -137,15 +140,17 @@ def _adjust_thread_count(self):
         # the worker threads.
         def weakref_cb(_, q=self._work_queue):
             q.put(None)
+
         # TODO(bquinlan): Should avoid creating new threads if there are more
         # idle threads than items in the work queue.
         num_threads = len(self._threads)
         if num_threads < self._max_workers:
-            thread_name = '%s_%d' % (self._thread_name_prefix or self,
-                                     num_threads)
-            t = threading.Thread(name=thread_name, target=_worker,
-                                 args=(weakref.ref(self, weakref_cb),
-                                       self._work_queue))
+            thread_name = "%s_%d" % (self._thread_name_prefix or self, num_threads)
+            t = threading.Thread(
+                name=thread_name,
+                target=_worker,
+                args=(weakref.ref(self, weakref_cb), self._work_queue),
+            )
             t.daemon = True
             t.start()
             self._threads.add(t)
@@ -158,4 +163,5 @@ def shutdown(self, wait=True):
         if wait:
             for t in self._threads:
                 t.join()
+
     shutdown.__doc__ = _base.Executor.shutdown.__doc__
diff --git a/distributed/_ipython_utils.py b/distributed/_ipython_utils.py
index 6f880872a62..512f8911588 100644
--- a/distributed/_ipython_utils.py
+++ b/distributed/_ipython_utils.py
@@ -8,6 +8,7 @@
 
 import atexit
 import os
+
 try:
     import queue
 except ImportError:
@@ -37,7 +38,7 @@ def run_cell_remote(ip, kc, cell):
     """
     msg_id = kc.execute(cell)
 
-    in_kernel = getattr(ip, 'kernel', False)
+    in_kernel = getattr(ip, "kernel", False)
     if in_kernel:
         socket = ip.display_pub.pub_socket
         session = ip.display_pub.session
@@ -49,30 +50,30 @@ def run_cell_remote(ip, kc, cell):
         except queue.Empty:
             raise TimeoutError("Timeout waiting for IPython output")
 
-        if msg['parent_header'].get('msg_id') != msg_id:
+        if msg["parent_header"].get("msg_id") != msg_id:
             continue
-        msg_type = msg['header']['msg_type']
-        content = msg['content']
-        if msg_type == 'status':
-            if content['execution_state'] == 'idle':
+        msg_type = msg["header"]["msg_type"]
+        content = msg["content"]
+        if msg_type == "status":
+            if content["execution_state"] == "idle":
                 # idle means output is done
                 break
-        elif msg_type == 'stream':
-            stream = getattr(sys, content['name'])
-            stream.write(content['text'])
-        elif msg_type in ('display_data', 'execute_result', 'error'):
+        elif msg_type == "stream":
+            stream = getattr(sys, content["name"])
+            stream.write(content["text"])
+        elif msg_type in ("display_data", "execute_result", "error"):
             if in_kernel:
                 session.send(socket, msg_type, content, parent=parent_header)
             else:
-                if msg_type == 'error':
-                    print('\n'.join(content['traceback']), file=sys.stderr)
+                if msg_type == "error":
+                    print("\n".join(content["traceback"]), file=sys.stderr)
                 else:
-                    sys.stdout.write(content['data'].get('text/plain', ''))
+                    sys.stdout.write(content["data"].get("text/plain", ""))
         else:
             pass
 
 
-def register_worker_magic(connection_info, magic_name='worker'):
+def register_worker_magic(connection_info, magic_name="worker"):
     """Register a %worker magic, given connection_info.
 
     Both a line and cell magic are registered,
@@ -80,7 +81,7 @@ def register_worker_magic(connection_info, magic_name='worker'):
     """
     ip = get_ipython()
     info = dict(connection_info)  # copy
-    key = info.pop('key')
+    key = info.pop("key")
     kc = BlockingKernelClient(**connection_info)
     kc.session.key = key
     kc.start_channels()
@@ -93,8 +94,8 @@ def remote(line, cell=None):
         run_cell_remote(ip, kc, cell)
 
     remote.client = kc  # preserve reference on kc, largely for mocking
-    ip.register_magic_function(remote, magic_kind='line', magic_name=magic_name)
-    ip.register_magic_function(remote, magic_kind='cell', magic_name=magic_name)
+    ip.register_magic_function(remote, magic_kind="line", magic_name=magic_name)
+    ip.register_magic_function(remote, magic_kind="cell", magic_name=magic_name)
 
 
 def remote_magic(line, cell=None):
@@ -124,8 +125,8 @@ def remote_magic(line, cell=None):
         cell = split_line[1]
 
     # turn info dict to hashable str for use as lookup key in _clients cache
-    key = ','.join(map(str, sorted(connection_info.items())))
-    session_key = connection_info.pop('key')
+    key = ",".join(map(str, sorted(connection_info.items())))
+    session_key = connection_info.pop("key")
 
     if key in remote_magic._clients:
         kc = remote_magic._clients[key]
@@ -144,7 +145,7 @@ def remote_magic(line, cell=None):
 remote_magic._clients = {}
 
 
-def register_remote_magic(magic_name='remote'):
+def register_remote_magic(magic_name="remote"):
     """Define the parameterized %remote magic
 
     See remote_magic above for details.
@@ -152,8 +153,8 @@ def register_remote_magic(magic_name='remote'):
     ip = get_ipython()
     if ip is None:
         return  # do nothing if IPython's not running
-    ip.register_magic_function(remote_magic, magic_kind='line', magic_name=magic_name)
-    ip.register_magic_function(remote_magic, magic_kind='cell', magic_name=magic_name)
+    ip.register_magic_function(remote_magic, magic_kind="line", magic_name=magic_name)
+    ip.register_magic_function(remote_magic, magic_kind="cell", magic_name=magic_name)
 
 
 def connect_qtconsole(connection_info, name=None, extra_args=None):
@@ -167,9 +168,9 @@ def connect_qtconsole(connection_info, name=None, extra_args=None):
     if name is None:
         name = uuid4().hex
 
-    path = os.path.join(runtime_dir, name + '.json')
+    path = os.path.join(runtime_dir, name + ".json")
     write_connection_file(path, **connection_info)
-    cmd = ['jupyter', 'qtconsole', '--existing', path]
+    cmd = ["jupyter", "qtconsole", "--existing", path]
     if extra_args:
         cmd.extend(extra_args)
     Popen(cmd)
@@ -197,11 +198,13 @@ def start_ipython(ip=None, ns=None, log=None):
         Hook up IPython's logging to an existing logger instead of the default.
     """
     from IPython import get_ipython
+
     if get_ipython() is not None:
         raise RuntimeError("Cannot start IPython, it's already running.")
 
     from zmq.eventloop.ioloop import ZMQIOLoop
     from ipykernel.kernelapp import IPKernelApp
+
     # save the global IOLoop instance
     # since IPython relies on it, but we are going to put it in a thread.
     save_inst = IOLoop.instance()
@@ -212,7 +215,7 @@ def start_ipython(ip=None, ns=None, log=None):
     # start IPython, disabling its signal handlers that won't work due to running in a thread:
     app = IPKernelApp.instance(log=log)
     # Don't connect to the history database
-    app.config.HistoryManager.hist_file = ':memory:'
+    app.config.HistoryManager.hist_file = ":memory:"
     # listen on all interfaces, so remote clients can connect:
     if ip:
         app.ip = ip
diff --git a/distributed/_version.py b/distributed/_version.py
index 1482ff1efe6..f48634810c3 100644
--- a/distributed/_version.py
+++ b/distributed/_version.py
@@ -1,4 +1,3 @@
-
 # This file helps to compute a version number in source trees obtained from
 # git-archive tarball (such as those provided by githubs download-from-tag
 # feature). Distribution tarballs (built by setup.py sdist) and build
@@ -58,17 +57,18 @@ class NotThisMethod(Exception):
 
 def register_vcs_handler(vcs, method):  # decorator
     """Decorator to mark a method as the handler for a particular VCS."""
+
     def decorate(f):
         """Store f in HANDLERS[vcs][method]."""
         if vcs not in HANDLERS:
             HANDLERS[vcs] = {}
         HANDLERS[vcs][method] = f
         return f
+
     return decorate
 
 
-def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False,
-                env=None):
+def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env=None):
     """Call the given command(s)."""
     assert isinstance(commands, list)
     p = None
@@ -76,10 +76,13 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False,
         try:
             dispcmd = str([c] + args)
             # remember shell=False, so use git.cmd on windows, not just git
-            p = subprocess.Popen([c] + args, cwd=cwd, env=env,
-                                 stdout=subprocess.PIPE,
-                                 stderr=(subprocess.PIPE if hide_stderr
-                                         else None))
+            p = subprocess.Popen(
+                [c] + args,
+                cwd=cwd,
+                env=env,
+                stdout=subprocess.PIPE,
+                stderr=(subprocess.PIPE if hide_stderr else None),
+            )
             break
         except EnvironmentError:
             e = sys.exc_info()[1]
@@ -116,16 +119,22 @@ def versions_from_parentdir(parentdir_prefix, root, verbose):
     for i in range(3):
         dirname = os.path.basename(root)
         if dirname.startswith(parentdir_prefix):
-            return {"version": dirname[len(parentdir_prefix):],
-                    "full-revisionid": None,
-                    "dirty": False, "error": None, "date": None}
+            return {
+                "version": dirname[len(parentdir_prefix) :],
+                "full-revisionid": None,
+                "dirty": False,
+                "error": None,
+                "date": None,
+            }
         else:
             rootdirs.append(root)
             root = os.path.dirname(root)  # up a level
 
     if verbose:
-        print("Tried directories %s but none started with prefix %s" %
-              (str(rootdirs), parentdir_prefix))
+        print(
+            "Tried directories %s but none started with prefix %s"
+            % (str(rootdirs), parentdir_prefix)
+        )
     raise NotThisMethod("rootdir doesn't start with parentdir_prefix")
 
 
@@ -181,7 +190,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose):
     # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of
     # just "foo-1.0". If we see a "tag: " prefix, prefer those.
     TAG = "tag: "
-    tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)])
+    tags = set([r[len(TAG) :] for r in refs if r.startswith(TAG)])
     if not tags:
         # Either we're using git < 1.8.3, or there really are no tags. We use
         # a heuristic: assume all version tags have a digit. The old git %d
@@ -190,7 +199,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose):
         # between branches and tags. By ignoring refnames without digits, we
         # filter out many common branch names like "release" and
         # "stabilization", as well as "HEAD" and "master".
-        tags = set([r for r in refs if re.search(r'\d', r)])
+        tags = set([r for r in refs if re.search(r"\d", r)])
         if verbose:
             print("discarding '%s', no digits" % ",".join(refs - tags))
     if verbose:
@@ -198,19 +207,26 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose):
     for ref in sorted(tags):
         # sorting will prefer e.g. "2.0" over "2.0rc1"
         if ref.startswith(tag_prefix):
-            r = ref[len(tag_prefix):]
+            r = ref[len(tag_prefix) :]
             if verbose:
                 print("picking %s" % r)
-            return {"version": r,
-                    "full-revisionid": keywords["full"].strip(),
-                    "dirty": False, "error": None,
-                    "date": date}
+            return {
+                "version": r,
+                "full-revisionid": keywords["full"].strip(),
+                "dirty": False,
+                "error": None,
+                "date": date,
+            }
     # no suitable tags, so version is "0+unknown", but full hex is still there
     if verbose:
         print("no suitable tags, using unknown + full revision id")
-    return {"version": "0+unknown",
-            "full-revisionid": keywords["full"].strip(),
-            "dirty": False, "error": "no suitable tags", "date": None}
+    return {
+        "version": "0+unknown",
+        "full-revisionid": keywords["full"].strip(),
+        "dirty": False,
+        "error": "no suitable tags",
+        "date": None,
+    }
 
 
 @register_vcs_handler("git", "pieces_from_vcs")
@@ -225,8 +241,7 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
     if sys.platform == "win32":
         GITS = ["git.cmd", "git.exe"]
 
-    out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root,
-                          hide_stderr=True)
+    out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=True)
     if rc != 0:
         if verbose:
             print("Directory %s not under git control" % root)
@@ -234,10 +249,19 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
 
     # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty]
     # if there isn't one, this yields HEX[-dirty] (no NUM)
-    describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty",
-                                          "--always", "--long",
-                                          "--match", "%s*" % tag_prefix],
-                                   cwd=root)
+    describe_out, rc = run_command(
+        GITS,
+        [
+            "describe",
+            "--tags",
+            "--dirty",
+            "--always",
+            "--long",
+            "--match",
+            "%s*" % tag_prefix,
+        ],
+        cwd=root,
+    )
     # --long was added in git-1.5.5
     if describe_out is None:
         raise NotThisMethod("'git describe' failed")
@@ -260,17 +284,16 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
     dirty = git_describe.endswith("-dirty")
     pieces["dirty"] = dirty
     if dirty:
-        git_describe = git_describe[:git_describe.rindex("-dirty")]
+        git_describe = git_describe[: git_describe.rindex("-dirty")]
 
     # now we have TAG-NUM-gHEX or HEX
 
     if "-" in git_describe:
         # TAG-NUM-gHEX
-        mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe)
+        mo = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", git_describe)
         if not mo:
             # unparseable. Maybe git-describe is misbehaving?
-            pieces["error"] = ("unable to parse git-describe output: '%s'"
-                               % describe_out)
+            pieces["error"] = "unable to parse git-describe output: '%s'" % describe_out
             return pieces
 
         # tag
@@ -279,10 +302,12 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
             if verbose:
                 fmt = "tag '%s' doesn't start with prefix '%s'"
                 print(fmt % (full_tag, tag_prefix))
-            pieces["error"] = ("tag '%s' doesn't start with prefix '%s'"
-                               % (full_tag, tag_prefix))
+            pieces["error"] = "tag '%s' doesn't start with prefix '%s'" % (
+                full_tag,
+                tag_prefix,
+            )
             return pieces
-        pieces["closest-tag"] = full_tag[len(tag_prefix):]
+        pieces["closest-tag"] = full_tag[len(tag_prefix) :]
 
         # distance: number of commits since tag
         pieces["distance"] = int(mo.group(2))
@@ -293,13 +318,13 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
     else:
         # HEX: no tags
         pieces["closest-tag"] = None
-        count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"],
-                                    cwd=root)
+        count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], cwd=root)
         pieces["distance"] = int(count_out)  # total number of commits
 
     # commit date: see ISO-8601 comment in git_versions_from_keywords()
-    date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"],
-                       cwd=root)[0].strip()
+    date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[
+        0
+    ].strip()
     pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1)
 
     return pieces
@@ -330,8 +355,7 @@ def render_pep440(pieces):
                 rendered += ".dirty"
     else:
         # exception #1
-        rendered = "0+untagged.%d.g%s" % (pieces["distance"],
-                                          pieces["short"])
+        rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"])
         if pieces["dirty"]:
             rendered += ".dirty"
     return rendered
@@ -445,11 +469,13 @@ def render_git_describe_long(pieces):
 def render(pieces, style):
     """Render the given version pieces into the requested style."""
     if pieces["error"]:
-        return {"version": "unknown",
-                "full-revisionid": pieces.get("long"),
-                "dirty": None,
-                "error": pieces["error"],
-                "date": None}
+        return {
+            "version": "unknown",
+            "full-revisionid": pieces.get("long"),
+            "dirty": None,
+            "error": pieces["error"],
+            "date": None,
+        }
 
     if not style or style == "default":
         style = "pep440"  # the default
@@ -469,9 +495,13 @@ def render(pieces, style):
     else:
         raise ValueError("unknown style '%s'" % style)
 
-    return {"version": rendered, "full-revisionid": pieces["long"],
-            "dirty": pieces["dirty"], "error": None,
-            "date": pieces.get("date")}
+    return {
+        "version": rendered,
+        "full-revisionid": pieces["long"],
+        "dirty": pieces["dirty"],
+        "error": None,
+        "date": pieces.get("date"),
+    }
 
 
 def get_versions():
@@ -485,8 +515,7 @@ def get_versions():
     verbose = cfg.verbose
 
     try:
-        return git_versions_from_keywords(get_keywords(), cfg.tag_prefix,
-                                          verbose)
+        return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, verbose)
     except NotThisMethod:
         pass
 
@@ -495,13 +524,16 @@ def get_versions():
         # versionfile_source is the relative path from the top of the source
         # tree (where the .git directory might live) to this file. Invert
         # this to find the root from __file__.
-        for i in cfg.versionfile_source.split('/'):
+        for i in cfg.versionfile_source.split("/"):
             root = os.path.dirname(root)
     except NameError:
-        return {"version": "0+unknown", "full-revisionid": None,
-                "dirty": None,
-                "error": "unable to find root of source tree",
-                "date": None}
+        return {
+            "version": "0+unknown",
+            "full-revisionid": None,
+            "dirty": None,
+            "error": "unable to find root of source tree",
+            "date": None,
+        }
 
     try:
         pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose)
@@ -515,6 +547,10 @@ def get_versions():
     except NotThisMethod:
         pass
 
-    return {"version": "0+unknown", "full-revisionid": None,
-            "dirty": None,
-            "error": "unable to compute version", "date": None}
+    return {
+        "version": "0+unknown",
+        "full-revisionid": None,
+        "dirty": None,
+        "error": "unable to compute version",
+        "date": None,
+    }
diff --git a/distributed/actor.py b/distributed/actor.py
index 4f2dc5e5abd..4bbe6faf78d 100644
--- a/distributed/actor.py
+++ b/distributed/actor.py
@@ -47,6 +47,7 @@ class Actor(WrappedKey):
     >>> future.result()
     2
     """
+
     def __init__(self, cls, address, key, worker=None):
         self._cls = cls
         self._address = address
@@ -67,7 +68,7 @@ def __init__(self, cls, address, key, worker=None):
                 self._client = None
 
     def __repr__(self):
-        return '<Actor: %s, key=%s>' % (self._cls.__name__, self.key)
+        return "<Actor: %s, key=%s>" % (self._cls.__name__, self.key)
 
     def __reduce__(self):
         return (Actor, (self._cls, self._address, self.key))
@@ -112,16 +113,19 @@ def _sync(self, func, *args, **kwargs):
 
     def __dir__(self):
         o = set(dir(type(self)))
-        o.update(attr for attr in dir(self._cls) if not attr.startswith('_'))
+        o.update(attr for attr in dir(self._cls) if not attr.startswith("_"))
         return sorted(o)
 
     def __getattr__(self, key):
         attr = getattr(self._cls, key)
 
-        if self._future and self._future.status not in ('finished', 'pending'):
-            raise ValueError("Worker holding Actor was lost.  Status: " + self._future.status)
+        if self._future and self._future.status not in ("finished", "pending"):
+            raise ValueError(
+                "Worker holding Actor was lost.  Status: " + self._future.status
+            )
 
         if callable(attr):
+
             @functools.wraps(attr)
             def func(*args, **kwargs):
                 @gen.coroutine
@@ -138,7 +142,7 @@ def run_actor_function_on_worker():
                             yield self._future
                         else:
                             raise OSError("Unable to contact Actor's worker")
-                    raise gen.Return(result['result'])
+                    raise gen.Return(result["result"])
 
                 if self._asynchronous:
                     return run_actor_function_on_worker()
@@ -151,16 +155,21 @@ def run_actor_function_on_worker():
                     def wait_then_add_to_queue():
                         x = yield run_actor_function_on_worker()
                         q.put(x)
+
                     self._io_loop.add_callback(wait_then_add_to_queue)
 
                     return ActorFuture(q, self._io_loop)
+
             return func
 
         else:
+
             @gen.coroutine
             def get_actor_attribute_from_worker():
-                x = yield self._worker_rpc.actor_attribute(attribute=key, actor=self.key)
-                raise gen.Return(x['result'])
+                x = yield self._worker_rpc.actor_attribute(
+                    attribute=key, actor=self.key
+                )
+                raise gen.Return(x["result"])
 
             return self._sync(get_actor_attribute_from_worker)
 
@@ -173,6 +182,7 @@ class ProxyRPC(object):
     """
     An rpc-like object that uses the scheduler's rpc to connect to a worker
     """
+
     def __init__(self, rpc, address):
         self.rpc = rpc
         self._address = address
@@ -180,7 +190,7 @@ def __init__(self, rpc, address):
     def __getattr__(self, key):
         @gen.coroutine
         def func(**msg):
-            msg['op'] = key
+            msg["op"] = key
             result = yield self.rpc.proxy(worker=self._address, msg=msg)
             raise gen.Return(result)
 
@@ -198,6 +208,7 @@ class ActorFuture(object):
     --------
     Actor
     """
+
     def __init__(self, q, io_loop):
         self.q = q
         self.io_loop = io_loop
@@ -210,4 +221,4 @@ def result(self, timeout=None):
             return self._cached_result
 
     def __repr__(self):
-        return '<ActorFuture>'
+        return "<ActorFuture>"
diff --git a/distributed/asyncio.py b/distributed/asyncio.py
index 3d680a57789..b75bf2a1130 100644
--- a/distributed/asyncio.py
+++ b/distributed/asyncio.py
@@ -18,11 +18,13 @@
 
 def to_asyncio(fn, **default_kwargs):
     """Converts Tornado gen.coroutines and futures to asyncio ones"""
+
     @wraps(fn)
     def convert(*args, **kwargs):
         if default_kwargs:
             kwargs = merge(default_kwargs, kwargs)
         return to_asyncio_future(fn(*args, **kwargs))
+
     return convert
 
 
@@ -92,6 +94,7 @@ async def gather_some_results():
     distributed.client.Client: Blocking Client
     distributed.scheduler.Scheduler: Internal scheduler
     """
+
     def __init__(self, *args, **kwargs):
         loop = asyncio.get_event_loop()
         ioloop = BaseAsyncIOLoop(loop)
diff --git a/distributed/batched.py b/distributed/batched.py
index a13707b3da0..bc77cc7fda2 100644
--- a/distributed/batched.py
+++ b/distributed/batched.py
@@ -36,12 +36,13 @@ class BatchedSend(object):
 
         ['Hello,', 'world!']
     """
+
     # XXX why doesn't BatchedSend follow either the IOStream or Comm API?
 
     def __init__(self, interval, loop=None, serializers=None):
         # XXX is the loop arg useful?
         self.loop = loop or IOLoop.current()
-        self.interval = parse_timedelta(interval, default='ms')
+        self.interval = parse_timedelta(interval, default="ms")
         self.waker = locks.Event()
         self.stopped = locks.Event()
         self.please_stop = False
@@ -51,7 +52,9 @@ def __init__(self, interval, loop=None, serializers=None):
         self.batch_count = 0
         self.byte_count = 0
         self.next_deadline = None
-        self.recent_message_log = deque(maxlen=dask.config.get('distributed.comm.recent-messages-log-length'))
+        self.recent_message_log = deque(
+            maxlen=dask.config.get("distributed.comm.recent-messages-log-length")
+        )
         self.serializers = serializers
 
     def start(self, comm):
@@ -63,9 +66,9 @@ def closed(self):
 
     def __repr__(self):
         if self.closed():
-            return '<BatchedSend: closed>'
+            return "<BatchedSend: closed>"
         else:
-            return '<BatchedSend: %d in buffer>' % len(self.buffer)
+            return "<BatchedSend: %d in buffer>" % len(self.buffer)
 
     __str__ = __repr__
 
@@ -81,21 +84,20 @@ def _background_send(self):
                 # Nothing to send
                 self.next_deadline = None
                 continue
-            if (self.next_deadline is not None and
-                    self.loop.time() < self.next_deadline):
+            if self.next_deadline is not None and self.loop.time() < self.next_deadline:
                 # Send interval not expired yet
                 continue
             payload, self.buffer = self.buffer, []
             self.batch_count += 1
             self.next_deadline = self.loop.time() + self.interval
             try:
-                nbytes = yield self.comm.write(payload,
-                                               serializers=self.serializers,
-                                               on_error='raise')
+                nbytes = yield self.comm.write(
+                    payload, serializers=self.serializers, on_error="raise"
+                )
                 if nbytes < 1e6:
                     self.recent_message_log.append(payload)
                 else:
-                    self.recent_message_log.append('large-message')
+                    self.recent_message_log.append("large-message")
                 self.byte_count += nbytes
             except CommClosedError as e:
                 logger.info("Batched Comm Closed: %s", e)
@@ -134,9 +136,9 @@ def close(self):
             try:
                 if self.buffer:
                     self.buffer, payload = [], self.buffer
-                    yield self.comm.write(payload,
-                                          serializers=self.serializers,
-                                          on_error='raise')
+                    yield self.comm.write(
+                        payload, serializers=self.serializers, on_error="raise"
+                    )
             except CommClosedError:
                 pass
             yield self.comm.close()
diff --git a/distributed/bokeh/__init__.py b/distributed/bokeh/__init__.py
index 12dd9bae52b..24e082fa8e0 100644
--- a/distributed/bokeh/__init__.py
+++ b/distributed/bokeh/__init__.py
@@ -6,30 +6,32 @@
 m = 100000
 
 messages = {
-    'workers': {'interval': 1000,
-                'deque': deque(maxlen=n),
-                'times': deque(maxlen=n),
-                'index': deque(maxlen=n),
-                'plot-data': {'time': deque(maxlen=n),
-                              'cpu': deque(maxlen=n),
-                              'memory_percent': deque(maxlen=n),
-                              'network-send': deque(maxlen=n),
-                              'network-recv': deque(maxlen=n)}},
-
-    'tasks': {'interval': 150,
-              'deque': deque(maxlen=100),
-              'times': deque(maxlen=100)},
-
-    'progress': {},
-
-    'processing': {'processing': {}, 'memory': 0, 'waiting': 0},
-
-    'task-events': {'interval': 200,
-                    'deque': deque(maxlen=m),
-                    'times': deque(maxlen=m),
-                    'index': deque(maxlen=m),
-                    'rectangles': {name: deque(maxlen=m) for name in
-                                   'start duration key name color worker worker_thread y alpha'.split()},
-                    'workers': dict(),
-                    'last_seen': [time()]}
+    "workers": {
+        "interval": 1000,
+        "deque": deque(maxlen=n),
+        "times": deque(maxlen=n),
+        "index": deque(maxlen=n),
+        "plot-data": {
+            "time": deque(maxlen=n),
+            "cpu": deque(maxlen=n),
+            "memory_percent": deque(maxlen=n),
+            "network-send": deque(maxlen=n),
+            "network-recv": deque(maxlen=n),
+        },
+    },
+    "tasks": {"interval": 150, "deque": deque(maxlen=100), "times": deque(maxlen=100)},
+    "progress": {},
+    "processing": {"processing": {}, "memory": 0, "waiting": 0},
+    "task-events": {
+        "interval": 200,
+        "deque": deque(maxlen=m),
+        "times": deque(maxlen=m),
+        "index": deque(maxlen=m),
+        "rectangles": {
+            name: deque(maxlen=m)
+            for name in "start duration key name color worker worker_thread y alpha".split()
+        },
+        "workers": dict(),
+        "last_seen": [time()],
+    },
 }
diff --git a/distributed/bokeh/components.py b/distributed/bokeh/components.py
index 351dc168f86..16efa1d2eb0 100644
--- a/distributed/bokeh/components.py
+++ b/distributed/bokeh/components.py
@@ -6,9 +6,23 @@
 import weakref
 
 from bokeh.layouts import row, column
-from bokeh.models import ( ColumnDataSource, Plot, DataRange1d, LinearAxis,
-        HoverTool, BoxZoomTool, ResetTool, PanTool, WheelZoomTool, Range1d,
-        Quad, TapTool, OpenURL, Button, Select)
+from bokeh.models import (
+    ColumnDataSource,
+    Plot,
+    DataRange1d,
+    LinearAxis,
+    HoverTool,
+    BoxZoomTool,
+    ResetTool,
+    PanTool,
+    WheelZoomTool,
+    Range1d,
+    Quad,
+    TapTool,
+    OpenURL,
+    Button,
+    Select,
+)
 from bokeh.palettes import Spectral9
 from bokeh.plotting import figure
 import dask
@@ -20,14 +34,14 @@
 from .. import profile
 from ..utils import log_errors, parse_timedelta
 
-if dask.config.get('distributed.dashboard.export-tool'):
+if dask.config.get("distributed.dashboard.export-tool"):
     from .export_tool import ExportTool
 else:
     ExportTool = None
 
 
-profile_interval = dask.config.get('distributed.worker.profile.interval')
-profile_interval = parse_timedelta(profile_interval, default='ms')
+profile_interval = dask.config.get("distributed.worker.profile.interval")
+profile_interval = parse_timedelta(profile_interval, default="ms")
 
 
 class DashboardComponent(object):
@@ -56,12 +70,12 @@ class TaskStream(DashboardComponent):
     The start and stop time of tasks as they occur on each core of the cluster.
     """
 
-    def __init__(self, n_rectangles=1000, clear_interval='20s', **kwargs):
+    def __init__(self, n_rectangles=1000, clear_interval="20s", **kwargs):
         """
         kwargs are applied to the bokeh.models.plots.Plot constructor
         """
         self.n_rectangles = n_rectangles
-        clear_interval = parse_timedelta(clear_interval, default='ms')
+        clear_interval = parse_timedelta(clear_interval, default="ms")
         self.clear_interval = clear_interval
         self.last = 0
 
@@ -73,20 +87,21 @@ def __init__(self, n_rectangles=1000, clear_interval='20s', **kwargs):
     @without_property_validation
     def update(self, messages):
         with log_errors():
-            index = messages['task-events']['index']
-            rectangles = messages['task-events']['rectangles']
+            index = messages["task-events"]["index"]
+            rectangles = messages["task-events"]["rectangles"]
 
             if not index or index[-1] == self.task_stream_index[0]:
                 return
 
             ind = bisect(index, self.task_stream_index[0])
-            rectangles = {k: [v[i] for i in range(ind, len(index))]
-                          for k, v in rectangles.items()}
+            rectangles = {
+                k: [v[i] for i in range(ind, len(index))] for k, v in rectangles.items()
+            }
             self.task_stream_index[0] = index[-1]
 
             # If there has been a significant delay then clear old rectangles
-            if rectangles['start']:
-                m = min(map(add, rectangles['start'], rectangles['duration']))
+            if rectangles["start"]:
+                m = min(map(add, rectangles["start"], rectangles["duration"]))
                 if m > self.last:
                     self.last, last = m, self.last
                     if m > last + self.clear_interval:
@@ -96,31 +111,40 @@ def update(self, messages):
             self.source.stream(rectangles, self.n_rectangles)
 
 
-def task_stream_figure(clear_interval='20s', **kwargs):
+def task_stream_figure(clear_interval="20s", **kwargs):
     """
     kwargs are applied to the bokeh.models.plots.Plot constructor
     """
-    clear_interval = parse_timedelta(clear_interval, default='ms')
-
-    source = ColumnDataSource(data=dict(
-        start=[time() - clear_interval], duration=[0.1], key=['start'],
-        name=['start'], color=['white'], duration_text=['100 ms'],
-        worker=['foo'], y=[0], worker_thread=[1], alpha=[0.0])
+    clear_interval = parse_timedelta(clear_interval, default="ms")
+
+    source = ColumnDataSource(
+        data=dict(
+            start=[time() - clear_interval],
+            duration=[0.1],
+            key=["start"],
+            name=["start"],
+            color=["white"],
+            duration_text=["100 ms"],
+            worker=["foo"],
+            y=[0],
+            worker_thread=[1],
+            alpha=[0.0],
+        )
     )
 
     x_range = DataRange1d(range_padding=0)
     y_range = DataRange1d(range_padding=0)
 
     root = figure(
-        name='task_stream',
+        name="task_stream",
         title="Task Stream",
-        id='bk-task-stream-plot',
+        id="bk-task-stream-plot",
         x_range=x_range,
         y_range=y_range,
         toolbar_location="above",
-        x_axis_type='datetime',
+        x_axis_type="datetime",
         min_border_right=35,
-        tools='',
+        tools="",
         **kwargs
     )
 
@@ -134,7 +158,7 @@ def task_stream_figure(clear_interval='20s', **kwargs):
         line_color="color",
         line_alpha=0.6,
         fill_alpha="alpha",
-        line_width=3
+        line_width=3,
     )
     rect.nonselection_glyph = None
 
@@ -150,17 +174,18 @@ def task_stream_figure(clear_interval='20s', **kwargs):
                 <span style="font-size: 12px; font-weight: bold;">@name:</span>&nbsp;
                 <span style="font-size: 10px; font-family: Monaco, monospace;">@duration_text</span>
             </div>
-            """
+            """,
     )
 
-    tap = TapTool(callback=OpenURL(url='/profile?key=@name'))
+    tap = TapTool(callback=OpenURL(url="/profile?key=@name"))
 
     root.add_tools(
-        hover, tap,
+        hover,
+        tap,
         BoxZoomTool(),
         ResetTool(),
         PanTool(dimensions="width"),
-        WheelZoomTool(dimensions="width")
+        WheelZoomTool(dimensions="width"),
     )
     if ExportTool:
         export = ExportTool()
@@ -174,24 +199,42 @@ class MemoryUsage(DashboardComponent):
     """ The memory usage across the cluster, grouped by task type """
 
     def __init__(self, **kwargs):
-        self.source = ColumnDataSource(data=dict(
-            name=[], left=[], right=[], center=[], color=[],
-            percent=[], MB=[], text=[])
+        self.source = ColumnDataSource(
+            data=dict(
+                name=[],
+                left=[],
+                right=[],
+                center=[],
+                color=[],
+                percent=[],
+                MB=[],
+                text=[],
+            )
         )
 
         self.root = Plot(
-            id='bk-nbytes-plot', x_range=DataRange1d(), y_range=DataRange1d(),
-            toolbar_location=None, outline_line_color=None, **kwargs
+            id="bk-nbytes-plot",
+            x_range=DataRange1d(),
+            y_range=DataRange1d(),
+            toolbar_location=None,
+            outline_line_color=None,
+            **kwargs
         )
 
         self.root.add_glyph(
             self.source,
-            Quad(top=1, bottom=0, left='left', right='right',
-                 fill_color='color', fill_alpha=1)
+            Quad(
+                top=1,
+                bottom=0,
+                left="left",
+                right="right",
+                fill_color="color",
+                fill_alpha=1,
+            ),
         )
 
-        self.root.add_layout(LinearAxis(), 'left')
-        self.root.add_layout(LinearAxis(), 'below')
+        self.root.add_layout(LinearAxis(), "left")
+        self.root.add_layout(LinearAxis(), "below")
 
         hover = HoverTool(
             point_policy="follow_mouse",
@@ -208,20 +251,21 @@ def __init__(self, **kwargs):
                     <span style="font-size: 14px; font-weight: bold;">MB:</span>&nbsp;
                     <span style="font-size: 10px; font-family: Monaco, monospace;">@MB</span>
                 </div>
-                """
+                """,
         )
         self.root.add_tools(hover)
 
     @without_property_validation
     def update(self, messages):
         with log_errors():
-            msg = messages['progress']
+            msg = messages["progress"]
             if not msg:
                 return
-            nb = nbytes_bar(msg['nbytes'])
+            nb = nbytes_bar(msg["nbytes"])
             self.source.data.update(nb)
-            self.root.title.text = \
-                "Memory Use: %0.2f MB" % (sum(msg['nbytes'].values()) / 1e6)
+            self.root.title.text = "Memory Use: %0.2f MB" % (
+                sum(msg["nbytes"].values()) / 1e6
+            )
 
 
 class Processing(DashboardComponent):
@@ -232,14 +276,25 @@ class Processing(DashboardComponent):
     """
 
     def __init__(self, **kwargs):
-        data = self.processing_update({'processing': {}, 'ncores': {}})
+        data = self.processing_update({"processing": {}, "ncores": {}})
         self.source = ColumnDataSource(data)
 
         x_range = Range1d(-1, 1)
-        fig = figure(title='Processing and Pending', tools='',
-                     x_range=x_range, id='bk-processing-stacks-plot', **kwargs)
-        fig.quad(source=self.source, left=0, right='right', color=Spectral9[0],
-                 top='top', bottom='bottom')
+        fig = figure(
+            title="Processing and Pending",
+            tools="",
+            x_range=x_range,
+            id="bk-processing-stacks-plot",
+            **kwargs
+        )
+        fig.quad(
+            source=self.source,
+            left=0,
+            right="right",
+            color=Spectral9[0],
+            top="top",
+            bottom="bottom",
+        )
 
         fig.xaxis.minor_tick_line_alpha = 0
         fig.yaxis.visible = False
@@ -258,20 +313,20 @@ def __init__(self, **kwargs):
             <span style="font-size: 10px; font-family: Monaco, monospace;">@processing</span>
         </div>
         """
-        hover.point_policy = 'follow_mouse'
+        hover.point_policy = "follow_mouse"
 
         self.root = fig
 
     @without_property_validation
     def update(self, messages):
         with log_errors():
-            msg = messages['processing']
-            if not msg.get('ncores'):
+            msg = messages["processing"]
+            if not msg.get("ncores"):
                 return
             data = self.processing_update(msg)
             x_range = self.root.x_range
-            max_right = max(data['right'])
-            cores = max(data['ncores'])
+            max_right = max(data["right"])
+            cores = max(data["ncores"])
             if x_range.end < max_right:
                 x_range.end = max_right + 2
             elif x_range.end > 2 * max_right + cores:  # way out there, walk back
@@ -282,21 +337,23 @@ def update(self, messages):
     @staticmethod
     def processing_update(msg):
         with log_errors():
-            names = sorted(msg['processing'])
+            names = sorted(msg["processing"])
             names = sorted(names)
-            processing = msg['processing']
+            processing = msg["processing"]
             processing = [processing[name] for name in names]
-            ncores = msg['ncores']
+            ncores = msg["ncores"]
             ncores = [ncores[name] for name in names]
             n = len(names)
-            d = {'name': list(names),
-                 'processing': processing,
-                 'right': list(processing),
-                 'top': list(range(n, 0, -1)),
-                 'bottom': list(range(n - 1, -1, -1)),
-                 'ncores': ncores}
+            d = {
+                "name": list(names),
+                "processing": processing,
+                "right": list(processing),
+                "top": list(range(n, 0, -1)),
+                "bottom": list(range(n - 1, -1, -1)),
+                "ncores": ncores,
+            }
 
-            d['alpha'] = [0.7] * n
+            d["alpha"] = [0.7] * n
 
             return d
 
@@ -310,7 +367,7 @@ class ProfilePlot(DashboardComponent):
     def __init__(self, **kwargs):
         state = profile.create()
         data = profile.plot_data(state, profile_interval)
-        self.states = data.pop('states')
+        self.states = data.pop("states")
         self.root, self.source = profile.plot_figure(data, **kwargs)
 
         @without_property_validation
@@ -319,28 +376,28 @@ def cb(attr, old, new):
                 try:
                     selected = new.indices
                 except AttributeError:
-                    selected = new['1d']['indices']
+                    selected = new["1d"]["indices"]
                 try:
                     ind = selected[0]
                 except IndexError:
                     return
                 data = profile.plot_data(self.states[ind], profile_interval)
                 del self.states[:]
-                self.states.extend(data.pop('states'))
+                self.states.extend(data.pop("states"))
                 self.source.data.update(data)
                 self.source.selected = old
 
-        if BOKEH_VERSION >= '1.0.0':
-            self.source.selected.on_change('indices', cb)
+        if BOKEH_VERSION >= "1.0.0":
+            self.source.selected.on_change("indices", cb)
         else:
-            self.source.on_change('selected', cb)
+            self.source.on_change("selected", cb)
 
     @without_property_validation
     def update(self, state):
         with log_errors():
             self.state = state
             data = profile.plot_data(self.state, profile_interval)
-            self.states = data.pop('states')
+            self.states = data.pop("states")
             self.source.data.update(data)
 
 
@@ -354,25 +411,25 @@ def __init__(self, server, doc=None, **kwargs):
         if doc is not None:
             self.doc = weakref.ref(doc)
             try:
-                self.key = doc.session_context.request.arguments.get('key', None)
+                self.key = doc.session_context.request.arguments.get("key", None)
             except AttributeError:
                 self.key = None
             if isinstance(self.key, list):
                 self.key = self.key[0]
             if isinstance(self.key, bytes):
                 self.key = self.key.decode()
-            self.task_names = ['All', self.key]
+            self.task_names = ["All", self.key]
         else:
             self.key = None
-            self.task_names = ['All']
+            self.task_names = ["All"]
 
         self.server = server
         self.start = None
         self.stop = None
-        self.ts = {'count': [], 'time': []}
+        self.ts = {"count": [], "time": []}
         self.state = profile.create()
         data = profile.plot_data(self.state, profile_interval)
-        self.states = data.pop('states')
+        self.states = data.pop("states")
         self.profile_plot, self.source = profile.plot_figure(data, **kwargs)
 
         changing = [False]  # avoid repeated changes from within callback
@@ -385,14 +442,14 @@ def cb(attr, old, new):
                 if isinstance(new, list):  # bokeh >= 1.0
                     selected = new
                 else:
-                    selected = new['1d']['indices']
+                    selected = new["1d"]["indices"]
                 try:
                     ind = selected[0]
                 except IndexError:
                     return
                 data = profile.plot_data(self.states[ind], profile_interval)
                 del self.states[:]
-                self.states.extend(data.pop('states'))
+                self.states.extend(data.pop("states"))
                 changing[0] = True  # don't recursively trigger callback
                 self.source.data.update(data)
                 if isinstance(new, list):  # bokeh >= 1.0
@@ -401,20 +458,25 @@ def cb(attr, old, new):
                     self.source.selected = old
                 changing[0] = False
 
-        if BOKEH_VERSION >= '1.0.0':
-            self.source.selected.on_change('indices', cb)
+        if BOKEH_VERSION >= "1.0.0":
+            self.source.selected.on_change("indices", cb)
         else:
-            self.source.on_change('selected', cb)
-
-        self.ts_source = ColumnDataSource({'time': [], 'count': []})
-        self.ts_plot = figure(title='Activity over time', height=100,
-                              x_axis_type='datetime', active_drag='xbox_select',
-                              y_range=[0, 1 / profile_interval],
-                              tools='xpan,xwheel_zoom,xbox_select,reset',
-                              **kwargs)
-        self.ts_plot.line('time', 'count', source=self.ts_source)
-        self.ts_plot.circle('time', 'count', source=self.ts_source, color=None,
-                            selection_color='orange')
+            self.source.on_change("selected", cb)
+
+        self.ts_source = ColumnDataSource({"time": [], "count": []})
+        self.ts_plot = figure(
+            title="Activity over time",
+            height=100,
+            x_axis_type="datetime",
+            active_drag="xbox_select",
+            y_range=[0, 1 / profile_interval],
+            tools="xpan,xwheel_zoom,xbox_select,reset",
+            **kwargs
+        )
+        self.ts_plot.line("time", "count", source=self.ts_source)
+        self.ts_plot.circle(
+            "time", "count", source=self.ts_source, color=None, selection_color="orange"
+        )
         self.ts_plot.yaxis.visible = False
         self.ts_plot.grid.visible = False
 
@@ -423,22 +485,22 @@ def ts_change(attr, old, new):
                 try:
                     selected = self.ts_source.selected.indices
                 except AttributeError:
-                    selected = self.ts_source.selected['1d']['indices']
+                    selected = self.ts_source.selected["1d"]["indices"]
                 if selected:
-                    start = self.ts_source.data['time'][min(selected)] / 1000
-                    stop = self.ts_source.data['time'][max(selected)] / 1000
+                    start = self.ts_source.data["time"][min(selected)] / 1000
+                    stop = self.ts_source.data["time"][max(selected)] / 1000
                     self.start, self.stop = min(start, stop), max(start, stop)
                 else:
                     self.start = self.stop = None
                 self.trigger_update(update_metadata=False)
 
-        if BOKEH_VERSION >= '1.0.0':
-            self.ts_source.selected.on_change('indices', ts_change)
+        if BOKEH_VERSION >= "1.0.0":
+            self.ts_source.selected.on_change("indices", ts_change)
         else:
-            self.ts_source.on_change('selected', ts_change)
+            self.ts_source.on_change("selected", ts_change)
 
         self.reset_button = Button(label="Reset", button_type="success")
-        self.reset_button.on_click(lambda: self.update(self.state) )
+        self.reset_button.on_click(lambda: self.update(self.state))
 
         self.update_button = Button(label="Update", button_type="success")
         self.update_button.on_click(self.trigger_update)
@@ -446,34 +508,42 @@ def ts_change(attr, old, new):
         self.select = Select(value=self.task_names[-1], options=self.task_names)
 
         def select_cb(attr, old, new):
-            if new == 'All':
+            if new == "All":
                 new = None
             self.key = new
             self.trigger_update(update_metadata=False)
 
-        self.select.on_change('value', select_cb)
-
-        self.root = column(row(self.select, self.reset_button,
-                               self.update_button, sizing_mode='scale_width'),
-                           self.profile_plot, self.ts_plot, **kwargs)
+        self.select.on_change("value", select_cb)
+
+        self.root = column(
+            row(
+                self.select,
+                self.reset_button,
+                self.update_button,
+                sizing_mode="scale_width",
+            ),
+            self.profile_plot,
+            self.ts_plot,
+            **kwargs
+        )
 
     @without_property_validation
     def update(self, state, metadata=None):
         with log_errors():
             self.state = state
             data = profile.plot_data(self.state, profile_interval)
-            self.states = data.pop('states')
+            self.states = data.pop("states")
             self.source.data.update(data)
 
-            if metadata is not None and metadata['counts']:
-                self.task_names = ['All'] + sorted(metadata['keys'])
+            if metadata is not None and metadata["counts"]:
+                self.task_names = ["All"] + sorted(metadata["keys"])
                 self.select.options = self.task_names
                 if self.key:
-                    ts = metadata['keys'][self.key]
+                    ts = metadata["keys"][self.key]
                 else:
-                    ts = metadata['counts']
+                    ts = metadata["counts"]
                 times, counts = zip(*ts)
-                self.ts = {'count': counts, 'time': [t * 1000 for t in times]}
+                self.ts = {"count": counts, "time": [t * 1000 for t in times]}
 
                 self.ts_source.data.update(self.ts)
 
@@ -482,7 +552,9 @@ def trigger_update(self, update_metadata=True):
         @gen.coroutine
         def cb():
             with log_errors():
-                prof = self.server.get_profile(key=self.key, start=self.start, stop=self.stop)
+                prof = self.server.get_profile(
+                    key=self.key, start=self.start, stop=self.stop
+                )
                 if update_metadata:
                     metadata = self.server.get_profile_metadata()
                 else:
@@ -507,10 +579,10 @@ def __init__(self, server, doc=None, **kwargs):
         self.log = self.server.io_loop.profile
         self.start = None
         self.stop = None
-        self.ts = {'count': [], 'time': []}
+        self.ts = {"count": [], "time": []}
         self.state = profile.get_profile(self.log)
         data = profile.plot_data(self.state, profile_interval)
-        self.states = data.pop('states')
+        self.states = data.pop("states")
         self.profile_plot, self.source = profile.plot_figure(data, **kwargs)
 
         changing = [False]  # avoid repeated changes from within callback
@@ -523,14 +595,14 @@ def cb(attr, old, new):
                 if isinstance(new, list):  # bokeh >= 1.0
                     selected = new
                 else:
-                    selected = new['1d']['indices']
+                    selected = new["1d"]["indices"]
                 try:
                     ind = selected[0]
                 except IndexError:
                     return
                 data = profile.plot_data(self.states[ind], profile_interval)
                 del self.states[:]
-                self.states.extend(data.pop('states'))
+                self.states.extend(data.pop("states"))
                 changing[0] = True  # don't recursively trigger callback
                 self.source.data.update(data)
                 if isinstance(new, list):  # bokeh >= 1.0
@@ -539,20 +611,25 @@ def cb(attr, old, new):
                     self.source.selected = old
                 changing[0] = False
 
-        if BOKEH_VERSION >= '1.0.0':
-            self.source.selected.on_change('indices', cb)
+        if BOKEH_VERSION >= "1.0.0":
+            self.source.selected.on_change("indices", cb)
         else:
-            self.source.on_change('selected', cb)
-
-        self.ts_source = ColumnDataSource({'time': [], 'count': []})
-        self.ts_plot = figure(title='Activity over time', height=100,
-                              x_axis_type='datetime', active_drag='xbox_select',
-                              y_range=[0, 1 / profile_interval],
-                              tools='xpan,xwheel_zoom,xbox_select,reset',
-                              **kwargs)
-        self.ts_plot.line('time', 'count', source=self.ts_source)
-        self.ts_plot.circle('time', 'count', source=self.ts_source, color=None,
-                            selection_color='orange')
+            self.source.on_change("selected", cb)
+
+        self.ts_source = ColumnDataSource({"time": [], "count": []})
+        self.ts_plot = figure(
+            title="Activity over time",
+            height=100,
+            x_axis_type="datetime",
+            active_drag="xbox_select",
+            y_range=[0, 1 / profile_interval],
+            tools="xpan,xwheel_zoom,xbox_select,reset",
+            **kwargs
+        )
+        self.ts_plot.line("time", "count", source=self.ts_source)
+        self.ts_plot.circle(
+            "time", "count", source=self.ts_source, color=None, selection_color="orange"
+        )
         self.ts_plot.yaxis.visible = False
         self.ts_plot.grid.visible = False
 
@@ -561,19 +638,19 @@ def ts_change(attr, old, new):
                 try:
                     selected = self.ts_source.selected.indices
                 except AttributeError:
-                    selected = self.ts_source.selected['1d']['indices']
+                    selected = self.ts_source.selected["1d"]["indices"]
                 if selected:
-                    start = self.ts_source.data['time'][min(selected)] / 1000
-                    stop = self.ts_source.data['time'][max(selected)] / 1000
+                    start = self.ts_source.data["time"][min(selected)] / 1000
+                    stop = self.ts_source.data["time"][max(selected)] / 1000
                     self.start, self.stop = min(start, stop), max(start, stop)
                 else:
                     self.start = self.stop = None
                 self.trigger_update()
 
-        if BOKEH_VERSION >= '1.0.0':
-            self.ts_source.selected.on_change('indices', ts_change)
+        if BOKEH_VERSION >= "1.0.0":
+            self.ts_source.selected.on_change("indices", ts_change)
         else:
-            self.ts_source.on_change('selected', ts_change)
+            self.ts_source.on_change("selected", ts_change)
 
         self.reset_button = Button(label="Reset", button_type="success")
         self.reset_button.on_click(lambda: self.update(self.state))
@@ -581,27 +658,30 @@ def ts_change(attr, old, new):
         self.update_button = Button(label="Update", button_type="success")
         self.update_button.on_click(self.trigger_update)
 
-        self.root = column(row(self.reset_button, self.update_button,
-                               sizing_mode='scale_width'),
-                           self.profile_plot, self.ts_plot, **kwargs)
+        self.root = column(
+            row(self.reset_button, self.update_button, sizing_mode="scale_width"),
+            self.profile_plot,
+            self.ts_plot,
+            **kwargs
+        )
 
     @without_property_validation
     def update(self, state):
         with log_errors():
             self.state = state
             data = profile.plot_data(self.state, profile_interval)
-            self.states = data.pop('states')
+            self.states = data.pop("states")
             self.source.data.update(data)
 
     @without_property_validation
     def trigger_update(self):
         self.state = profile.get_profile(self.log, start=self.start, stop=self.stop)
         data = profile.plot_data(self.state, profile_interval)
-        self.states = data.pop('states')
+        self.states = data.pop("states")
         self.source.data.update(data)
         times = [t * 1000 for t, _ in self.log]
-        counts = list(toolz.pluck('count', toolz.pluck(1, self.log)))
-        self.ts_source.data.update({'time': times, 'count': counts})
+        counts = list(toolz.pluck("count", toolz.pluck(1, self.log)))
+        self.ts_source.data.update({"time": times, "count": counts})
 
 
 def add_periodic_callback(doc, component, interval):
@@ -628,7 +708,7 @@ def update(ref):
 
 
 def _attach(doc, component):
-    if not hasattr(doc, 'components'):
+    if not hasattr(doc, "components"):
         doc.components = set()
 
     doc.components.add(component)
diff --git a/distributed/bokeh/core.py b/distributed/bokeh/core.py
index 9263a701cdc..a85efb3233c 100644
--- a/distributed/bokeh/core.py
+++ b/distributed/bokeh/core.py
@@ -9,9 +9,11 @@
 from tornado import web
 
 
-if LooseVersion(bokeh.__version__) < LooseVersion('0.13.0'):
-    warnings.warn("\nDask needs bokeh >= 0.13.0 for the dashboard."
-                  "\nContinuing without the dashboard.")
+if LooseVersion(bokeh.__version__) < LooseVersion("0.13.0"):
+    warnings.warn(
+        "\nDask needs bokeh >= 0.13.0 for the dashboard."
+        "\nContinuing without the dashboard."
+    )
     raise ImportError("Dask needs bokeh >= 0.13.0")
 
 
@@ -28,41 +30,58 @@ def listen(self, addr):
             ip = None
         for i in range(5):
             try:
-                self.server = Server(self.apps,
-                                     port=port, address=ip,
-                                     check_unused_sessions_milliseconds=500,
-                                     allow_websocket_origin=["*"],
-                                     use_index=False,
-                                     extra_patterns=[(r'/', web.RedirectHandler, {'url': '/status'})],
-                                     **self.server_kwargs)
+                self.server = Server(
+                    self.apps,
+                    port=port,
+                    address=ip,
+                    check_unused_sessions_milliseconds=500,
+                    allow_websocket_origin=["*"],
+                    use_index=False,
+                    extra_patterns=[(r"/", web.RedirectHandler, {"url": "/status"})],
+                    **self.server_kwargs
+                )
                 self.server.start()
 
-                handlers = [(self.prefix + r'/statics/(.*)',
-                             web.StaticFileHandler,
-                             {'path': os.path.join(os.path.dirname(__file__), 'static')})]
+                handlers = [
+                    (
+                        self.prefix + r"/statics/(.*)",
+                        web.StaticFileHandler,
+                        {"path": os.path.join(os.path.dirname(__file__), "static")},
+                    )
+                ]
 
-                self.server._tornado.add_handlers(r'.*', handlers)
+                self.server._tornado.add_handlers(r".*", handlers)
 
                 return
             except (SystemExit, EnvironmentError) as exc:
                 if port != 0:
-                    if ("already in use" in str(exc) or  # Unix/Mac
-                            "Only one usage of" in str(exc)):  # Windows
-                        msg = ("Port %d is already in use. "
-                               "\nPerhaps you already have a cluster running?"
-                               "\nHosting the diagnostics dashboard on a random port instead."
-                               % port)
+                    if "already in use" in str(
+                        exc
+                    ) or "Only one usage of" in str(  # Unix/Mac
+                        exc
+                    ):  # Windows
+                        msg = (
+                            "Port %d is already in use. "
+                            "\nPerhaps you already have a cluster running?"
+                            "\nHosting the diagnostics dashboard on a random port instead."
+                            % port
+                        )
                     else:
-                        msg = "Failed to start diagnostics server on port %d. " % port + str(exc)
-                    warnings.warn('\n' + msg)
+                        msg = (
+                            "Failed to start diagnostics server on port %d. " % port
+                            + str(exc)
+                        )
+                    warnings.warn("\n" + msg)
                     port = 0
                 if i == 4:
                     raise
 
     @property
     def port(self):
-        return (self.server.port or
-                list(self.server._http._sockets.values())[0].getsockname()[1])
+        return (
+            self.server.port
+            or list(self.server._http._sockets.values())[0].getsockname()[1]
+        )
 
     def stop(self):
         for context in self.server._tornado._applications.values():
@@ -74,5 +93,5 @@ def stop(self):
             self.server._tornado._ping_job.stop()
 
         # https://github.com/bokeh/bokeh/issues/5494
-        if LooseVersion(bokeh.__version__) >= '0.12.4':
+        if LooseVersion(bokeh.__version__) >= "0.12.4":
             self.server.stop()
diff --git a/distributed/bokeh/export_tool.py b/distributed/bokeh/export_tool.py
index bd2e01e52b5..5d8f1c067ae 100644
--- a/distributed/bokeh/export_tool.py
+++ b/distributed/bokeh/export_tool.py
@@ -10,7 +10,7 @@
 
 
 fn = __file__
-fn = os.path.join(os.path.dirname(fn), 'export_tool.js')
+fn = os.path.join(os.path.dirname(fn), "export_tool.js")
 with open(fn) as f:
     JS_CODE = f.read()
 
@@ -27,4 +27,4 @@ def export_callback(attr, old, new):
             html = file_html(plot, CDN, "Task Stream")
             self.content = html
 
-        self.on_change('event', export_callback)
+        self.on_change("event", export_callback)
diff --git a/distributed/bokeh/scheduler.py b/distributed/bokeh/scheduler.py
index d8823701aab..a6f07351730 100644
--- a/distributed/bokeh/scheduler.py
+++ b/distributed/bokeh/scheduler.py
@@ -8,10 +8,27 @@
 import os
 
 from bokeh.layouts import column, row
-from bokeh.models import (ColumnDataSource, DataRange1d, HoverTool, ResetTool,
-                          PanTool, WheelZoomTool, TapTool, OpenURL, Range1d, Plot, Quad,
-                          value, LinearAxis, NumeralTickFormatter, BasicTicker, NumberFormatter,
-                          BoxSelectTool, GroupFilter, CDSView)
+from bokeh.models import (
+    ColumnDataSource,
+    DataRange1d,
+    HoverTool,
+    ResetTool,
+    PanTool,
+    WheelZoomTool,
+    TapTool,
+    OpenURL,
+    Range1d,
+    Plot,
+    Quad,
+    value,
+    LinearAxis,
+    NumeralTickFormatter,
+    BasicTicker,
+    NumberFormatter,
+    BoxSelectTool,
+    GroupFilter,
+    CDSView,
+)
 from bokeh.models.widgets import DataTable, TableColumn
 from bokeh.plotting import figure
 from bokeh.palettes import Viridis11
@@ -20,14 +37,19 @@
 from bokeh.io import curdoc
 from toolz import pipe, merge
 from tornado import escape
+
 try:
     import numpy as np
 except ImportError:
     np = False
 
 from . import components
-from .components import (DashboardComponent, ProfileTimePlot, ProfileServer,
-                         add_periodic_callback)
+from .components import (
+    DashboardComponent,
+    ProfileTimePlot,
+    ProfileServer,
+    add_periodic_callback,
+)
 from .core import BokehServer
 from .worker import SystemMonitor, counters_doc
 from .utils import transpose, BOKEH_VERSION, without_property_validation
@@ -49,14 +71,19 @@
 PROFILING = False
 
 from jinja2 import Environment, FileSystemLoader
-env = Environment(loader=FileSystemLoader(os.path.join(os.path.dirname(__file__), 'templates')))
 
-template_variables = {'pages': ['status', 'workers', 'tasks', 'system', 'profile', 'graph']}
+env = Environment(
+    loader=FileSystemLoader(os.path.join(os.path.dirname(__file__), "templates"))
+)
+
+template_variables = {
+    "pages": ["status", "workers", "tasks", "system", "profile", "graph"]
+}
 
-BOKEH_THEME = Theme(os.path.join(os.path.dirname(__file__), 'theme.yaml'))
+BOKEH_THEME = Theme(os.path.join(os.path.dirname(__file__), "theme.yaml"))
 
-nan = float('nan')
-inf = float('inf')
+nan = float("nan")
+inf = float("inf")
 
 
 @without_property_validation
@@ -70,8 +97,7 @@ def update(source, data):
         arrays
     3.  If profiling then perform the update in another callback
     """
-    if (not np or not any(isinstance(v, np.ndarray)
-                          for v in source.data.values())):
+    if not np or not any(isinstance(v, np.ndarray) for v in source.data.values()):
         if source.data == data:
             return
     if np and len(data[first(data)]) > 10:
@@ -96,18 +122,28 @@ class Occupancy(DashboardComponent):
     def __init__(self, scheduler, **kwargs):
         with log_errors():
             self.scheduler = scheduler
-            self.source = ColumnDataSource({'occupancy': [0, 0],
-                                            'worker': ['a', 'b'],
-                                            'x': [0.0, 0.1],
-                                            'y': [1, 2],
-                                            'ms': [1, 2],
-                                            'color': ['red', 'blue'],
-                                            'bokeh_address': ['', '']})
-
-            fig = figure(title='Occupancy', tools='', id='bk-occupancy-plot',
-                         x_axis_type='datetime', **kwargs)
-            rect = fig.rect(source=self.source, x='x', width='ms', y='y', height=1,
-                            color='color')
+            self.source = ColumnDataSource(
+                {
+                    "occupancy": [0, 0],
+                    "worker": ["a", "b"],
+                    "x": [0.0, 0.1],
+                    "y": [1, 2],
+                    "ms": [1, 2],
+                    "color": ["red", "blue"],
+                    "bokeh_address": ["", ""],
+                }
+            )
+
+            fig = figure(
+                title="Occupancy",
+                tools="",
+                id="bk-occupancy-plot",
+                x_axis_type="datetime",
+                **kwargs
+            )
+            rect = fig.rect(
+                source=self.source, x="x", width="ms", y="y", height=1, color="color"
+            )
             rect.nonselection_glyph = None
 
             fig.xaxis.minor_tick_line_alpha = 0
@@ -116,11 +152,11 @@ def __init__(self, scheduler, **kwargs):
             # fig.xaxis[0].formatter = NumeralTickFormatter(format='0.0s')
             fig.x_range.start = 0
 
-            tap = TapTool(callback=OpenURL(url='http://@bokeh_address/main'))
+            tap = TapTool(callback=OpenURL(url="http://@bokeh_address/main"))
 
             hover = HoverTool()
             hover.tooltips = "@worker : @occupancy s."
-            hover.point_policy = 'follow_mouse'
+            hover.point_policy = "follow_mouse"
             fig.add_tools(hover, tap)
 
             self.root = fig
@@ -132,8 +168,8 @@ def update(self):
 
             bokeh_addresses = []
             for ws in workers:
-                addr = self.scheduler.get_worker_service_addr(ws.address, 'bokeh')
-                bokeh_addresses.append('%s:%d' % addr if addr is not None else '')
+                addr = self.scheduler.get_worker_service_addr(ws.address, "bokeh")
+                bokeh_addresses.append("%s:%d" % addr if addr is not None else "")
 
             y = list(range(len(workers)))
             occupancy = [ws.occupancy for ws in workers]
@@ -143,26 +179,30 @@ def update(self):
             color = []
             for ws in workers:
                 if ws in self.scheduler.idle:
-                    color.append('red')
+                    color.append("red")
                 elif ws in self.scheduler.saturated:
-                    color.append('green')
+                    color.append("green")
                 else:
-                    color.append('blue')
+                    color.append("blue")
 
             if total:
-                self.root.title.text = ('Occupancy -- total time: %s  wall time: %s' %
-                                        (format_time(total),
-                                         format_time(total / self.scheduler.total_ncores)))
+                self.root.title.text = "Occupancy -- total time: %s  wall time: %s" % (
+                    format_time(total),
+                    format_time(total / self.scheduler.total_ncores),
+                )
             else:
-                self.root.title.text = 'Occupancy'
+                self.root.title.text = "Occupancy"
 
             if occupancy:
-                result = {'occupancy': occupancy,
-                          'worker': [ws.address for ws in workers],
-                          'ms': ms,
-                          'color': color,
-                          'bokeh_address': bokeh_addresses,
-                          'x': x, 'y': y}
+                result = {
+                    "occupancy": occupancy,
+                    "worker": [ws.address for ws in workers],
+                    "ms": ms,
+                    "color": color,
+                    "bokeh_address": bokeh_addresses,
+                    "x": x,
+                    "y": y,
+                }
 
                 update(self.source, result)
 
@@ -174,14 +214,16 @@ def __init__(self, scheduler, **kwargs):
         with log_errors():
             self.last = 0
             self.scheduler = scheduler
-            self.source = ColumnDataSource({'left': [1, 2],
-                                            'right': [10, 10],
-                                            'top': [0, 0]})
+            self.source = ColumnDataSource(
+                {"left": [1, 2], "right": [10, 10], "top": [0, 0]}
+            )
 
-            self.root = figure(title='Tasks Processing',
-                               id='bk-nprocessing-histogram-plot',
-                               name='processing_hist',
-                               **kwargs)
+            self.root = figure(
+                title="Tasks Processing",
+                id="bk-nprocessing-histogram-plot",
+                name="processing_hist",
+                **kwargs
+            )
 
             self.root.xaxis.minor_tick_line_alpha = 0
             self.root.ygrid.visible = False
@@ -189,17 +231,20 @@ def __init__(self, scheduler, **kwargs):
             self.root.toolbar.logo = None
             self.root.toolbar_location = None
 
-            self.root.quad(source=self.source,
-                           left='left', right='right', bottom=0, top='top',
-                           color='blue')
+            self.root.quad(
+                source=self.source,
+                left="left",
+                right="right",
+                bottom=0,
+                top="top",
+                color="blue",
+            )
 
     @without_property_validation
     def update(self):
         L = [len(ws.processing) for ws in self.scheduler.workers.values()]
         counts, x = np.histogram(L, bins=40)
-        self.source.data.update({'left': x[:-1],
-                                 'right': x[1:],
-                                 'top': counts})
+        self.source.data.update({"left": x[:-1], "right": x[1:], "top": counts})
 
 
 class NBytesHistogram(DashboardComponent):
@@ -209,15 +254,17 @@ def __init__(self, scheduler, **kwargs):
         with log_errors():
             self.last = 0
             self.scheduler = scheduler
-            self.source = ColumnDataSource({'left': [1, 2],
-                                            'right': [10, 10],
-                                            'top': [0, 0]})
-
-            self.root = figure(title='Bytes Stored',
-                               name='nbytes_hist',
-                               id='bk-nbytes-histogram-plot',
-                               **kwargs)
-            self.root.xaxis[0].formatter = NumeralTickFormatter(format='0.0 b')
+            self.source = ColumnDataSource(
+                {"left": [1, 2], "right": [10, 10], "top": [0, 0]}
+            )
+
+            self.root = figure(
+                title="Bytes Stored",
+                name="nbytes_hist",
+                id="bk-nbytes-histogram-plot",
+                **kwargs
+            )
+            self.root.xaxis[0].formatter = NumeralTickFormatter(format="0.0 b")
             self.root.xaxis.major_label_orientation = -math.pi / 12
 
             self.root.xaxis.minor_tick_line_alpha = 0
@@ -226,18 +273,23 @@ def __init__(self, scheduler, **kwargs):
             self.root.toolbar.logo = None
             self.root.toolbar_location = None
 
-            self.root.quad(source=self.source,
-                           left='left', right='right', bottom=0, top='top',
-                           color='blue')
+            self.root.quad(
+                source=self.source,
+                left="left",
+                right="right",
+                bottom=0,
+                top="top",
+                color="blue",
+            )
 
     @without_property_validation
     def update(self):
         nbytes = np.asarray([ws.nbytes for ws in self.scheduler.workers.values()])
         counts, x = np.histogram(nbytes, bins=40)
-        d = {'left': x[:-1], 'right': x[1:], 'top': counts}
+        d = {"left": x[:-1], "right": x[1:], "top": counts}
         self.source.data.update(d)
 
-        self.root.title.text = 'Bytes stored: ' + format_bytes(nbytes.sum())
+        self.root.title.text = "Bytes stored: " + format_bytes(nbytes.sum())
 
 
 class CurrentLoad(DashboardComponent):
@@ -247,37 +299,60 @@ def __init__(self, scheduler, width=600, **kwargs):
         with log_errors():
             self.last = 0
             self.scheduler = scheduler
-            self.source = ColumnDataSource({'nprocessing': [1, 2],
-                                            'nprocessing-half': [0.5, 1],
-                                            'nprocessing-color': ['red', 'blue'],
-                                            'nbytes': [1, 2],
-                                            'nbytes-half': [0.5, 1],
-                                            'nbytes_text': ['1B', '2B'],
-                                            'worker': ['a', 'b'],
-                                            'y': [1, 2],
-                                            'nbytes-color': ['blue', 'blue'],
-                                            'bokeh_address': ['', '']})
-
-            processing = figure(title='Tasks Processing', tools='', id='bk-nprocessing-plot',
-                                name='processing_hist', width=int(width / 2), **kwargs)
-            rect = processing.rect(source=self.source,
-                                   x='nprocessing-half', y='y',
-                                   width='nprocessing', height=1,
-                                   color='nprocessing-color')
+            self.source = ColumnDataSource(
+                {
+                    "nprocessing": [1, 2],
+                    "nprocessing-half": [0.5, 1],
+                    "nprocessing-color": ["red", "blue"],
+                    "nbytes": [1, 2],
+                    "nbytes-half": [0.5, 1],
+                    "nbytes_text": ["1B", "2B"],
+                    "worker": ["a", "b"],
+                    "y": [1, 2],
+                    "nbytes-color": ["blue", "blue"],
+                    "bokeh_address": ["", ""],
+                }
+            )
+
+            processing = figure(
+                title="Tasks Processing",
+                tools="",
+                id="bk-nprocessing-plot",
+                name="processing_hist",
+                width=int(width / 2),
+                **kwargs
+            )
+            rect = processing.rect(
+                source=self.source,
+                x="nprocessing-half",
+                y="y",
+                width="nprocessing",
+                height=1,
+                color="nprocessing-color",
+            )
             processing.x_range.start = 0
             rect.nonselection_glyph = None
 
-            nbytes = figure(title='Bytes stored', tools='',
-                            id='bk-nbytes-worker-plot', width=int(width / 2),
-                            name='nbytes_hist',  **kwargs)
-            rect = nbytes.rect(source=self.source,
-                               x='nbytes-half', y='y',
-                               width='nbytes', height=1,
-                               color='nbytes-color')
+            nbytes = figure(
+                title="Bytes stored",
+                tools="",
+                id="bk-nbytes-worker-plot",
+                width=int(width / 2),
+                name="nbytes_hist",
+                **kwargs
+            )
+            rect = nbytes.rect(
+                source=self.source,
+                x="nbytes-half",
+                y="y",
+                width="nbytes",
+                height=1,
+                color="nbytes-color",
+            )
             rect.nonselection_glyph = None
 
             nbytes.axis[0].ticker = BasicTicker(mantissas=[1, 256, 512], base=1024)
-            nbytes.xaxis[0].formatter = NumeralTickFormatter(format='0.0 b')
+            nbytes.xaxis[0].formatter = NumeralTickFormatter(format="0.0 b")
             nbytes.xaxis.major_label_orientation = -math.pi / 12
             nbytes.x_range.start = 0
 
@@ -286,7 +361,7 @@ def __init__(self, scheduler, width=600, **kwargs):
                 fig.yaxis.visible = False
                 fig.ygrid.visible = False
 
-                tap = TapTool(callback=OpenURL(url='http://@bokeh_address/main'))
+                tap = TapTool(callback=OpenURL(url="http://@bokeh_address/main"))
                 fig.add_tools(tap)
 
                 fig.toolbar.logo = None
@@ -295,12 +370,12 @@ def __init__(self, scheduler, width=600, **kwargs):
 
             hover = HoverTool()
             hover.tooltips = "@worker : @nprocessing tasks"
-            hover.point_policy = 'follow_mouse'
+            hover.point_policy = "follow_mouse"
             processing.add_tools(hover)
 
             hover = HoverTool()
             hover.tooltips = "@worker : @nbytes_text"
-            hover.point_policy = 'follow_mouse'
+            hover.point_policy = "follow_mouse"
             nbytes.add_tools(hover)
 
             self.processing_figure = processing
@@ -315,52 +390,59 @@ def update(self):
 
             bokeh_addresses = []
             for ws in workers:
-                addr = self.scheduler.get_worker_service_addr(ws.address, 'bokeh')
-                bokeh_addresses.append('%s:%d' % addr if addr is not None else '')
+                addr = self.scheduler.get_worker_service_addr(ws.address, "bokeh")
+                bokeh_addresses.append("%s:%d" % addr if addr is not None else "")
 
             y = list(range(len(workers)))
             nprocessing = [len(ws.processing) for ws in workers]
             processing_color = []
             for ws in workers:
                 if ws in self.scheduler.idle:
-                    processing_color.append('red')
+                    processing_color.append("red")
                 elif ws in self.scheduler.saturated:
-                    processing_color.append('green')
+                    processing_color.append("green")
                 else:
-                    processing_color.append('blue')
+                    processing_color.append("blue")
 
-            nbytes = [ws.metrics['memory'] for ws in workers]
+            nbytes = [ws.metrics["memory"] for ws in workers]
             nbytes_text = [format_bytes(nb) for nb in nbytes]
             nbytes_color = []
             max_limit = 0
             for ws, nb in zip(workers, nbytes):
-                limit = getattr(self.scheduler.workers[ws.address], 'memory_limit', inf) or inf
+                limit = (
+                    getattr(self.scheduler.workers[ws.address], "memory_limit", inf)
+                    or inf
+                )
 
                 if limit > max_limit:
                     max_limit = limit
 
                 if nb > limit:
-                    nbytes_color.append('red')
+                    nbytes_color.append("red")
                 elif nb > limit / 2:
-                    nbytes_color.append('orange')
+                    nbytes_color.append("orange")
                 else:
-                    nbytes_color.append('blue')
+                    nbytes_color.append("blue")
 
             now = time()
             if any(nprocessing) or self.last + 1 < now:
                 self.last = now
-                result = {'nprocessing': nprocessing,
-                          'nprocessing-half': [np / 2 for np in nprocessing],
-                          'nprocessing-color': processing_color,
-                          'nbytes': nbytes,
-                          'nbytes-half': [nb / 2 for nb in nbytes],
-                          'nbytes-color': nbytes_color,
-                          'nbytes_text': nbytes_text,
-                          'bokeh_address': bokeh_addresses,
-                          'worker': [ws.address for ws in workers],
-                          'y': y}
-
-                self.nbytes_figure.title.text = 'Bytes stored: ' + format_bytes(sum(nbytes))
+                result = {
+                    "nprocessing": nprocessing,
+                    "nprocessing-half": [np / 2 for np in nprocessing],
+                    "nprocessing-color": processing_color,
+                    "nbytes": nbytes,
+                    "nbytes-half": [nb / 2 for nb in nbytes],
+                    "nbytes-color": nbytes_color,
+                    "nbytes_text": nbytes_text,
+                    "bokeh_address": bokeh_addresses,
+                    "worker": [ws.address for ws in workers],
+                    "y": y,
+                }
+
+                self.nbytes_figure.title.text = "Bytes stored: " + format_bytes(
+                    sum(nbytes)
+                )
 
                 update(self.source, result)
 
@@ -368,23 +450,27 @@ def update(self):
 class StealingTimeSeries(DashboardComponent):
     def __init__(self, scheduler, **kwargs):
         self.scheduler = scheduler
-        self.source = ColumnDataSource({'time': [time(), time() + 1],
-                                        'idle': [0, 0.1],
-                                        'saturated': [0, 0.1]})
+        self.source = ColumnDataSource(
+            {"time": [time(), time() + 1], "idle": [0, 0.1], "saturated": [0, 0.1]}
+        )
 
-        x_range = DataRange1d(follow='end', follow_interval=20000, range_padding=0)
+        x_range = DataRange1d(follow="end", follow_interval=20000, range_padding=0)
 
-        fig = figure(title="Idle and Saturated Workers Over Time",
-                     x_axis_type='datetime', y_range=[-0.1, len(scheduler.workers) + 0.1],
-                     height=150, tools='', x_range=x_range, **kwargs)
-        fig.line(source=self.source, x='time', y='idle', color='red')
-        fig.line(source=self.source, x='time', y='saturated', color='green')
+        fig = figure(
+            title="Idle and Saturated Workers Over Time",
+            x_axis_type="datetime",
+            y_range=[-0.1, len(scheduler.workers) + 0.1],
+            height=150,
+            tools="",
+            x_range=x_range,
+            **kwargs
+        )
+        fig.line(source=self.source, x="time", y="idle", color="red")
+        fig.line(source=self.source, x="time", y="saturated", color="green")
         fig.yaxis.minor_tick_line_color = None
 
         fig.add_tools(
-            ResetTool(),
-            PanTool(dimensions="width"),
-            WheelZoomTool(dimensions="width")
+            ResetTool(), PanTool(dimensions="width"), WheelZoomTool(dimensions="width")
         )
 
         self.root = fig
@@ -392,11 +478,15 @@ def __init__(self, scheduler, **kwargs):
     @without_property_validation
     def update(self):
         with log_errors():
-            result = {'time': [time() * 1000],
-                      'idle': [len(self.scheduler.idle)],
-                      'saturated': [len(self.scheduler.saturated)]}
+            result = {
+                "time": [time() * 1000],
+                "idle": [len(self.scheduler.idle)],
+                "saturated": [len(self.scheduler.saturated)],
+            }
             if PROFILING:
-                curdoc().add_next_tick_callback(lambda: self.source.stream(result, 10000))
+                curdoc().add_next_tick_callback(
+                    lambda: self.source.stream(result, 10000)
+                )
             else:
                 self.source.stream(result, 10000)
 
@@ -404,33 +494,51 @@ def update(self):
 class StealingEvents(DashboardComponent):
     def __init__(self, scheduler, **kwargs):
         self.scheduler = scheduler
-        self.steal = scheduler.extensions['stealing']
+        self.steal = scheduler.extensions["stealing"]
         self.last = 0
-        self.source = ColumnDataSource({'time': [time() - 20, time()],
-                                        'level': [0, 15],
-                                        'color': ['white', 'white'],
-                                        'duration': [0, 0], 'radius': [1, 1],
-                                        'cost_factor': [0, 10], 'count': [1, 1]})
+        self.source = ColumnDataSource(
+            {
+                "time": [time() - 20, time()],
+                "level": [0, 15],
+                "color": ["white", "white"],
+                "duration": [0, 0],
+                "radius": [1, 1],
+                "cost_factor": [0, 10],
+                "count": [1, 1],
+            }
+        )
 
-        x_range = DataRange1d(follow='end', follow_interval=20000, range_padding=0)
+        x_range = DataRange1d(follow="end", follow_interval=20000, range_padding=0)
 
-        fig = figure(title="Stealing Events",
-                     x_axis_type='datetime', y_axis_type='log',
-                     height=250, tools='', x_range=x_range, **kwargs)
+        fig = figure(
+            title="Stealing Events",
+            x_axis_type="datetime",
+            y_axis_type="log",
+            height=250,
+            tools="",
+            x_range=x_range,
+            **kwargs
+        )
 
-        fig.circle(source=self.source, x='time', y='cost_factor', color='color',
-                   size='radius', alpha=0.5)
+        fig.circle(
+            source=self.source,
+            x="time",
+            y="cost_factor",
+            color="color",
+            size="radius",
+            alpha=0.5,
+        )
         fig.yaxis.axis_label = "Cost Multiplier"
 
         hover = HoverTool()
         hover.tooltips = "Level: @level, Duration: @duration, Count: @count, Cost factor: @cost_factor"
-        hover.point_policy = 'follow_mouse'
+        hover.point_policy = "follow_mouse"
 
         fig.add_tools(
             hover,
             ResetTool(),
             PanTool(dimensions="width"),
-            WheelZoomTool(dimensions="width")
+            WheelZoomTool(dimensions="width"),
         )
 
         self.root = fig
@@ -445,13 +553,19 @@ def convert(self, msgs):
         try:
             color = Viridis11[level]
         except (KeyError, IndexError):
-            color = 'black'
+            color = "black"
 
         radius = math.sqrt(min(total_duration, 10)) * 30 + 2
 
-        d = {'time': time * 1000, 'level': level, 'count': len(msgs),
-             'color': color, 'duration': total_duration, 'radius': radius,
-             'cost_factor': min(10, self.steal.cost_multipliers[level])}
+        d = {
+            "time": time * 1000,
+            "level": level,
+            "count": len(msgs),
+            "color": color,
+            "duration": total_duration,
+            "radius": radius,
+            "cost_factor": min(10, self.steal.cost_multipliers[level]),
+        }
 
         return d
 
@@ -464,11 +578,19 @@ def update(self):
             self.last = self.steal.count
 
             if log:
-                new = pipe(log, map(groupby(1)), map(dict.values), concat,
-                           map(self.convert), list, transpose)
+                new = pipe(
+                    log,
+                    map(groupby(1)),
+                    map(dict.values),
+                    concat,
+                    map(self.convert),
+                    list,
+                    transpose,
+                )
                 if PROFILING:
                     curdoc().add_next_tick_callback(
-                        lambda: self.source.stream(new, 10000))
+                        lambda: self.source.stream(new, 10000)
+                    )
                 else:
                     self.source.stream(new, 10000)
 
@@ -479,28 +601,42 @@ def __init__(self, scheduler, name, height=150, **kwargs):
         self.action_ys = dict()
         self.last = 0
         self.name = name
-        self.source = ColumnDataSource({'time': [], 'action': [], 'hover': [],
-                                        'y': [], 'color': []})
+        self.source = ColumnDataSource(
+            {"time": [], "action": [], "hover": [], "y": [], "color": []}
+        )
 
-        x_range = DataRange1d(follow='end', follow_interval=200000)
+        x_range = DataRange1d(follow="end", follow_interval=200000)
 
-        fig = figure(title=name, x_axis_type='datetime',
-                     height=height, tools='', x_range=x_range, **kwargs)
+        fig = figure(
+            title=name,
+            x_axis_type="datetime",
+            height=height,
+            tools="",
+            x_range=x_range,
+            **kwargs
+        )
 
-        fig.circle(source=self.source, x='time', y='y', color='color',
-                   size=50, alpha=0.5, legend='action')
+        fig.circle(
+            source=self.source,
+            x="time",
+            y="y",
+            color="color",
+            size=50,
+            alpha=0.5,
+            legend="action",
+        )
         fig.yaxis.axis_label = "Action"
-        fig.legend.location = 'top_left'
+        fig.legend.location = "top_left"
 
         hover = HoverTool()
         hover.tooltips = "@action<br>@hover"
-        hover.point_policy = 'follow_mouse'
+        hover.point_policy = "follow_mouse"
 
         fig.add_tools(
             hover,
             ResetTool(),
             PanTool(dimensions="width"),
-            WheelZoomTool(dimensions="width")
+            WheelZoomTool(dimensions="width"),
         )
 
         self.root = fig
@@ -521,8 +657,8 @@ def update(self):
                 ys = []
                 colors = []
                 for msg in log:
-                    times.append(msg['time'] * 1000)
-                    action = msg['action']
+                    times.append(msg["time"] * 1000)
+                    action = msg["action"]
                     actions.append(action)
                     try:
                         ys.append(self.action_ys[action])
@@ -530,22 +666,26 @@ def update(self):
                         self.action_ys[action] = len(self.action_ys)
                         ys.append(self.action_ys[action])
                     colors.append(color_of(action))
-                    hovers.append('TODO')
+                    hovers.append("TODO")
 
-                new = {'time': times,
-                       'action': actions,
-                       'hover': hovers,
-                       'y': ys,
-                       'color': colors}
+                new = {
+                    "time": times,
+                    "action": actions,
+                    "hover": hovers,
+                    "y": ys,
+                    "color": colors,
+                }
 
                 if PROFILING:
-                    curdoc().add_next_tick_callback(lambda: self.source.stream(new, 10000))
+                    curdoc().add_next_tick_callback(
+                        lambda: self.source.stream(new, 10000)
+                    )
                 else:
                     self.source.stream(new, 10000)
 
 
 class TaskStream(components.TaskStream):
-    def __init__(self, scheduler, n_rectangles=1000, clear_interval='20s', **kwargs):
+    def __init__(self, scheduler, n_rectangles=1000, clear_interval="20s", **kwargs):
         self.scheduler = scheduler
         self.offset = 0
         es = [p for p in self.scheduler.plugins if isinstance(p, TaskStreamPlugin)]
@@ -556,39 +696,40 @@ def __init__(self, scheduler, n_rectangles=1000, clear_interval='20s', **kwargs)
         self.index = max(0, self.plugin.index - n_rectangles)
         self.workers = dict()
 
-        components.TaskStream.__init__(self, n_rectangles=n_rectangles,
-                                       clear_interval=clear_interval, **kwargs)
+        components.TaskStream.__init__(
+            self, n_rectangles=n_rectangles, clear_interval=clear_interval, **kwargs
+        )
 
     @without_property_validation
     def update(self):
         if self.index == self.plugin.index:
             return
         with log_errors():
-            if self.index and len(self.source.data['start']):
-                start = min(self.source.data['start'])
-                duration = max(self.source.data['duration'])
+            if self.index and len(self.source.data["start"]):
+                start = min(self.source.data["start"])
+                duration = max(self.source.data["duration"])
                 boundary = (self.offset + start - duration) / 1000
             else:
                 boundary = self.offset
-            rectangles = self.plugin.rectangles(istart=self.index,
-                                                workers=self.workers,
-                                                start_boundary=boundary)
-            n = len(rectangles['name'])
+            rectangles = self.plugin.rectangles(
+                istart=self.index, workers=self.workers, start_boundary=boundary
+            )
+            n = len(rectangles["name"])
             self.index = self.plugin.index
 
-            if not rectangles['start']:
+            if not rectangles["start"]:
                 return
 
             # If there has been a significant delay then clear old rectangles
-            first_end = min(map(add, rectangles['start'], rectangles['duration']))
+            first_end = min(map(add, rectangles["start"], rectangles["duration"]))
             if first_end > self.last:
                 last = self.last
                 self.last = first_end
                 if first_end > last + self.clear_interval * 1000:
-                    self.offset = min(rectangles['start'])
+                    self.offset = min(rectangles["start"])
                     self.source.data.update({k: [] for k in rectangles})
 
-            rectangles['start'] = [x - self.offset for x in rectangles['start']]
+            rectangles["start"] = [x - self.offset for x in rectangles["start"]]
 
             # Convert to numpy for serialization speed
             if n >= 10 and np:
@@ -597,8 +738,9 @@ def update(self):
                         rectangles[k] = np.array(v)
 
             if PROFILING:
-                curdoc().add_next_tick_callback(lambda:
-                                                self.source.stream(rectangles, self.n_rectangles))
+                curdoc().add_next_tick_callback(
+                    lambda: self.source.stream(rectangles, self.n_rectangles)
+                )
             else:
                 self.source.stream(rectangles, self.n_rectangles)
 
@@ -610,40 +752,60 @@ class GraphPlot(DashboardComponent):
     See also the GraphLayout diagnostic at
     distributed/diagnostics/graph_layout.py
     """
+
     def __init__(self, scheduler, **kwargs):
         self.scheduler = scheduler
         self.layout = GraphLayout(scheduler)
         self.invisible_count = 0  # number of invisible nodes
 
-        self.node_source = ColumnDataSource({'x': [], 'y': [], 'name': [],
-                                             'state': [], 'visible': [],
-                                             'key': []})
-        self.edge_source = ColumnDataSource({'x': [], 'y': [], 'visible': []})
+        self.node_source = ColumnDataSource(
+            {"x": [], "y": [], "name": [], "state": [], "visible": [], "key": []}
+        )
+        self.edge_source = ColumnDataSource({"x": [], "y": [], "visible": []})
 
-        node_view = CDSView(source=self.node_source,
-                            filters=[GroupFilter(column_name='visible', group='True')])
-        edge_view = CDSView(source=self.edge_source,
-                            filters=[GroupFilter(column_name='visible', group='True')])
+        node_view = CDSView(
+            source=self.node_source,
+            filters=[GroupFilter(column_name="visible", group="True")],
+        )
+        edge_view = CDSView(
+            source=self.edge_source,
+            filters=[GroupFilter(column_name="visible", group="True")],
+        )
 
-        node_colors = factor_cmap('state',
-                factors=['waiting', 'processing', 'memory', 'released', 'erred'],
-                palette=['gray', 'green', 'red', 'blue', 'black']
+        node_colors = factor_cmap(
+            "state",
+            factors=["waiting", "processing", "memory", "released", "erred"],
+            palette=["gray", "green", "red", "blue", "black"],
         )
 
-        self.root = figure(title='Task Graph', **kwargs)
-        self.root.multi_line(xs='x', ys='y', source=self.edge_source,
-                             line_width=1, view=edge_view, color='black',
-                             alpha=0.3)
-        rect = self.root.square(x='x', y='y', size=10, color=node_colors,
-                                source=self.node_source, view=node_view,
-                                legend='state')
+        self.root = figure(title="Task Graph", **kwargs)
+        self.root.multi_line(
+            xs="x",
+            ys="y",
+            source=self.edge_source,
+            line_width=1,
+            view=edge_view,
+            color="black",
+            alpha=0.3,
+        )
+        rect = self.root.square(
+            x="x",
+            y="y",
+            size=10,
+            color=node_colors,
+            source=self.node_source,
+            view=node_view,
+            legend="state",
+        )
         self.root.xgrid.grid_line_color = None
         self.root.ygrid.grid_line_color = None
 
-        hover = HoverTool(point_policy="follow_mouse", tooltips="<b>@name</b>: @state",
-                          renderers=[rect])
-        tap = TapTool(callback=OpenURL(url='info/task/@key.html'),
-                      renderers=[rect])
+        hover = HoverTool(
+            point_policy="follow_mouse",
+            tooltips="<b>@name</b>: @state",
+            renderers=[rect],
+        )
+        tap = TapTool(callback=OpenURL(url="info/task/@key.html"), renderers=[rect])
         rect.nonselection_glyph = None
         self.root.add_tools(hover, tap)
 
@@ -651,7 +813,7 @@ def __init__(self, scheduler, **kwargs):
     def update(self):
         with log_errors():
             # occasionally reset the column data source to remove old nodes
-            if self.invisible_count > len(self.node_source.data['x']) / 2:
+            if self.invisible_count > len(self.node_source.data["x"]) / 2:
                 self.layout.reset_index()
                 self.invisible_count = 0
                 update = True
@@ -701,17 +863,17 @@ def add_new_nodes_edges(self, new, new_edges, update=False):
                 except KeyError:
                     pass
 
-            node = {'x': node_x,
-                    'y': node_y,
-                    'state': node_state,
-                    'name': node_name,
-                    'key': node_key,
-                    'visible': ['True'] * len(node_x)}
-            edge = {'x': edge_x,
-                    'y': edge_y,
-                    'visible': ['True'] * len(edge_x)}
-
-            if update or not len(self.node_source.data['x']):
+            node = {
+                "x": node_x,
+                "y": node_y,
+                "state": node_state,
+                "name": node_name,
+                "key": node_key,
+                "visible": ["True"] * len(node_x),
+            }
+            edge = {"x": edge_x, "y": edge_y, "visible": ["True"] * len(edge_x)}
+
+            if update or not len(self.node_source.data["x"]):
                 # see https://github.com/bokeh/bokeh/issues/7523
                 self.node_source.data.update(node)
                 self.edge_source.data.update(edge)
@@ -724,27 +886,27 @@ def patch_updates(self):
         """
         Small updates like color changes or lost nodes from task transitions
         """
-        n = len(self.node_source.data['x'])
-        m = len(self.edge_source.data['x'])
+        n = len(self.node_source.data["x"])
+        m = len(self.edge_source.data["x"])
 
         if self.layout.state_updates:
             state_updates = self.layout.state_updates
             self.layout.state_updates = []
             updates = [(i, c) for i, c in state_updates if i < n]
-            self.node_source.patch({'state': updates})
+            self.node_source.patch({"state": updates})
 
         if self.layout.visible_updates:
             updates = self.layout.visible_updates
             updates = [(i, c) for i, c in updates if i < n]
             self.visible_updates = []
-            self.node_source.patch({'visible': updates})
+            self.node_source.patch({"visible": updates})
             self.invisible_count += len(updates)
 
         if self.layout.visible_edge_updates:
             updates = self.layout.visible_edge_updates
             updates = [(i, c) for i, c in updates if i < m]
             self.visible_updates = []
-            self.edge_source.patch({'visible': updates})
+            self.edge_source.patch({"visible": updates})
 
     def __del__(self):
         self.scheduler.remove_plugin(self.layout)
@@ -761,57 +923,93 @@ def __init__(self, scheduler, **kwargs):
         else:
             self.plugin = AllProgress(scheduler)
 
-        data = progress_quads(dict(all={}, memory={}, erred={}, released={},
-                                   processing={}))
+        data = progress_quads(
+            dict(all={}, memory={}, erred={}, released={}, processing={})
+        )
         self.source = ColumnDataSource(data=data)
 
         x_range = DataRange1d(range_padding=0)
         y_range = Range1d(-8, 0)
 
         self.root = figure(
-            id='bk-task-progress-plot', title='Progress', name='task_progress',
-            x_range=x_range, y_range=y_range, toolbar_location=None, **kwargs
+            id="bk-task-progress-plot",
+            title="Progress",
+            name="task_progress",
+            x_range=x_range,
+            y_range=y_range,
+            toolbar_location=None,
+            **kwargs
         )
         self.root.line(  # just to define early ranges
-            x=[0, 0.9], y=[-1, 0], line_color="#FFFFFF", alpha=0.0)
+            x=[0, 0.9], y=[-1, 0], line_color="#FFFFFF", alpha=0.0
+        )
         self.root.quad(
             source=self.source,
-            top='top', bottom='bottom', left='left', right='right',
-            fill_color="#aaaaaa", line_color='#aaaaaa', fill_alpha=0.1,
+            top="top",
+            bottom="bottom",
+            left="left",
+            right="right",
+            fill_color="#aaaaaa",
+            line_color="#aaaaaa",
+            fill_alpha=0.1,
             line_alpha=0.3,
         )
         self.root.quad(
             source=self.source,
-            top='top', bottom='bottom', left='left', right='released-loc',
-            fill_color="color", line_color="color", fill_alpha=0.6
+            top="top",
+            bottom="bottom",
+            left="left",
+            right="released-loc",
+            fill_color="color",
+            line_color="color",
+            fill_alpha=0.6,
         )
         self.root.quad(
             source=self.source,
-            top='top', bottom='bottom', left='released-loc',
-            right='memory-loc', fill_color="color", line_color="color",
-            fill_alpha=1.0
+            top="top",
+            bottom="bottom",
+            left="released-loc",
+            right="memory-loc",
+            fill_color="color",
+            line_color="color",
+            fill_alpha=1.0,
         )
         self.root.quad(
             source=self.source,
-            top='top', bottom='bottom', left='memory-loc',
-            right='erred-loc', fill_color='black',
-            fill_alpha=0.5, line_alpha=0,
+            top="top",
+            bottom="bottom",
+            left="memory-loc",
+            right="erred-loc",
+            fill_color="black",
+            fill_alpha=0.5,
+            line_alpha=0,
         )
         self.root.quad(
             source=self.source,
-            top='top', bottom='bottom', left='erred-loc',
-            right='processing-loc', fill_color='gray',
-            fill_alpha=0.35, line_alpha=0,
+            top="top",
+            bottom="bottom",
+            left="erred-loc",
+            right="processing-loc",
+            fill_color="gray",
+            fill_alpha=0.35,
+            line_alpha=0,
         )
         self.root.text(
             source=self.source,
-            text='show-name', y='bottom', x='left', x_offset=5,
-            text_font_size=value('10pt')
+            text="show-name",
+            y="bottom",
+            x="left",
+            x_offset=5,
+            text_font_size=value("10pt"),
         )
         self.root.text(
             source=self.source,
-            text='done', y='bottom', x='right', x_offset=-5,
-            text_align='right', text_font_size=value('10pt')
+            text="done",
+            y="bottom",
+            x="right",
+            x_offset=-5,
+            text_align="right",
+            text_font_size=value("10pt"),
         )
         self.root.ygrid.visible = False
         self.root.yaxis.minor_tick_line_alpha = 0
@@ -843,32 +1041,36 @@ def __init__(self, scheduler, **kwargs):
                     <span style="font-size: 14px; font-weight: bold;">Ready:</span>&nbsp;
                     <span style="font-size: 10px; font-family: Monaco, monospace;">@processing</span>
                 </div>
-                """
+                """,
         )
         self.root.add_tools(hover)
 
     @without_property_validation
     def update(self):
         with log_errors():
-            state = {'all': valmap(len, self.plugin.all),
-                     'nbytes': self.plugin.nbytes}
-            for k in ['memory', 'erred', 'released', 'processing']:
+            state = {"all": valmap(len, self.plugin.all), "nbytes": self.plugin.nbytes}
+            for k in ["memory", "erred", "released", "processing"]:
                 state[k] = valmap(len, self.plugin.state[k])
-            if not state['all'] and not len(self.source.data['all']):
+            if not state["all"] and not len(self.source.data["all"]):
                 return
 
             d = progress_quads(state)
 
             update(self.source, d)
 
-            totals = {k: sum(state[k].values())
-                      for k in ['all', 'memory', 'erred', 'released']}
-            totals['processing'] = totals['all'] - sum(v for k, v in
-                                                       totals.items() if k != 'all')
+            totals = {
+                k: sum(state[k].values())
+                for k in ["all", "memory", "erred", "released"]
+            }
+            totals["processing"] = totals["all"] - sum(
+                v for k, v in totals.items() if k != "all"
+            )
 
-            self.root.title.text = ("Progress -- total: %(all)s, "
-                                    "in-memory: %(memory)s, processing: %(processing)s, "
-                                    "erred: %(erred)s" % totals)
+            self.root.title.text = (
+                "Progress -- total: %(all)s, "
+                "in-memory: %(memory)s, processing: %(processing)s, "
+                "erred: %(erred)s" % totals
+            )
 
 
 class MemoryUse(DashboardComponent):
@@ -882,24 +1084,42 @@ def __init__(self, scheduler, **kwargs):
         else:
             self.plugin = AllProgress(scheduler)
 
-        self.source = ColumnDataSource(data=dict(
-            name=[], left=[], right=[], center=[], color=[],
-            percent=[], MB=[], text=[])
+        self.source = ColumnDataSource(
+            data=dict(
+                name=[],
+                left=[],
+                right=[],
+                center=[],
+                color=[],
+                percent=[],
+                MB=[],
+                text=[],
+            )
         )
 
         self.root = Plot(
-            id='bk-nbytes-plot', x_range=DataRange1d(), y_range=DataRange1d(),
-            toolbar_location=None, outline_line_color=None, **kwargs
+            id="bk-nbytes-plot",
+            x_range=DataRange1d(),
+            y_range=DataRange1d(),
+            toolbar_location=None,
+            outline_line_color=None,
+            **kwargs
         )
 
         self.root.add_glyph(
             self.source,
-            Quad(top=1, bottom=0, left='left', right='right',
-                 fill_color='color', fill_alpha=1)
+            Quad(
+                top=1,
+                bottom=0,
+                left="left",
+                right="right",
+                fill_color="color",
+                fill_alpha=1,
+            ),
         )
 
-        self.root.add_layout(LinearAxis(), 'left')
-        self.root.add_layout(LinearAxis(), 'below')
+        self.root.add_layout(LinearAxis(), "left")
+        self.root.add_layout(LinearAxis(), "below")
 
         hover = HoverTool(
             point_policy="follow_mouse",
@@ -916,7 +1136,7 @@ def __init__(self, scheduler, **kwargs):
                     <span style="font-size: 14px; font-weight: bold;">MB:</span>&nbsp;
                     <span style="font-size: 10px; font-family: Monaco, monospace;">@MB</span>
                 </div>
-                """
+                """,
         )
         self.root.add_tools(hover)
 
@@ -925,8 +1145,9 @@ def update(self):
         with log_errors():
             nb = nbytes_bar(self.plugin.nbytes)
             update(self.source, nb)
-            self.root.title.text = \
-                "Memory Use: %0.2f MB" % (sum(self.plugin.nbytes.values()) / 1e6)
+            self.root.title.text = "Memory Use: %0.2f MB" % (
+                sum(self.plugin.nbytes.values()) / 1e6
+            )
 
 
 class WorkerTable(DashboardComponent):
@@ -935,60 +1156,90 @@ class WorkerTable(DashboardComponent):
     This is two plots, a text-based table for each host and a thin horizontal
     plot laying out hosts by their current memory use.
     """
-    excluded_names = {'executing', 'in_flight', 'in_memory', 'ready', 'time'}
+
+    excluded_names = {"executing", "in_flight", "in_memory", "ready", "time"}
 
     def __init__(self, scheduler, width=800, **kwargs):
         self.scheduler = scheduler
-        self.names = ['worker', 'ncores', 'cpu', 'memory', 'memory_limit',
-                      'memory_percent', 'num_fds', 'read_bytes', 'write_bytes',
-                      'cpu_fraction']
+        self.names = [
+            "worker",
+            "ncores",
+            "cpu",
+            "memory",
+            "memory_limit",
+            "memory_percent",
+            "num_fds",
+            "read_bytes",
+            "write_bytes",
+            "cpu_fraction",
+        ]
         workers = self.scheduler.workers.values()
-        self.extra_names = sorted({m for ws in workers
-                                   for m in ws.metrics
-                                   if m not in self.names} - self.excluded_names)
+        self.extra_names = sorted(
+            {m for ws in workers for m in ws.metrics if m not in self.names}
+            - self.excluded_names
+        )
 
-        table_names = ['worker', 'ncores', 'cpu', 'memory', 'memory_limit',
-                       'memory_percent', 'num_fds', 'read_bytes',
-                       'write_bytes']
+        table_names = [
+            "worker",
+            "ncores",
+            "cpu",
+            "memory",
+            "memory_limit",
+            "memory_percent",
+            "num_fds",
+            "read_bytes",
+            "write_bytes",
+        ]
 
         self.source = ColumnDataSource({k: [] for k in self.names})
 
-        columns = {name: TableColumn(field=name,
-                                     title=name.replace('_percent', ' %'))
-                   for name in table_names}
-
-        formatters = {'cpu': NumberFormatter(format='0.0 %'),
-                      'memory_percent': NumberFormatter(format='0.0 %'),
-                      'memory': NumberFormatter(format='0 b'),
-                      'memory_limit': NumberFormatter(format='0 b'),
-                      'read_bytes': NumberFormatter(format='0 b'),
-                      'write_bytes': NumberFormatter(format='0 b'),
-                      'num_fds': NumberFormatter(format='0'),
-                      'ncores': NumberFormatter(format='0')}
-
-        if BOKEH_VERSION < '0.12.15':
-            dt_kwargs = {'row_headers': False}
+        columns = {
+            name: TableColumn(field=name, title=name.replace("_percent", " %"))
+            for name in table_names
+        }
+
+        formatters = {
+            "cpu": NumberFormatter(format="0.0 %"),
+            "memory_percent": NumberFormatter(format="0.0 %"),
+            "memory": NumberFormatter(format="0 b"),
+            "memory_limit": NumberFormatter(format="0 b"),
+            "read_bytes": NumberFormatter(format="0 b"),
+            "write_bytes": NumberFormatter(format="0 b"),
+            "num_fds": NumberFormatter(format="0"),
+            "ncores": NumberFormatter(format="0"),
+        }
+
+        if BOKEH_VERSION < "0.12.15":
+            dt_kwargs = {"row_headers": False}
         else:
-            dt_kwargs = {'index_position': None}
+            dt_kwargs = {"index_position": None}
 
         table = DataTable(
-            source=self.source, columns=[columns[n] for n in table_names],
-            reorderable=True, sortable=True, width=width, **dt_kwargs
+            source=self.source,
+            columns=[columns[n] for n in table_names],
+            reorderable=True,
+            sortable=True,
+            width=width,
+            **dt_kwargs
         )
 
         for name in table_names:
             if name in formatters:
                 table.columns[table_names.index(name)].formatter = formatters[name]
 
-        extra_names = ['worker'] + self.extra_names
-        extra_columns = {name: TableColumn(field=name,
-                                           title=name.replace('_percent', '%'))
-                         for name in extra_names}
+        extra_names = ["worker"] + self.extra_names
+        extra_columns = {
+            name: TableColumn(field=name, title=name.replace("_percent", "%"))
+            for name in extra_names
+        }
 
         extra_table = DataTable(
             source=self.source,
             columns=[extra_columns[n] for n in extra_names],
-            reorderable=True, sortable=True, width=width, **dt_kwargs
+            reorderable=True,
+            sortable=True,
+            width=width,
+            **dt_kwargs
         )
 
         hover = HoverTool(
@@ -998,14 +1249,22 @@ def __init__(self, scheduler, width=800, **kwargs):
                   <span style="font-size: 10px; font-family: Monaco, monospace;">@worker: </span>
                   <span style="font-size: 10px; font-family: Monaco, monospace;">@memory_percent</span>
                 </div>
-                """
+                """,
         )
 
-        mem_plot = figure(title='Memory Use (%)', toolbar_location=None,
-                          x_range=(0, 1), y_range=(-0.1, 0.1), height=60,
-                          width=width, tools='', **kwargs)
-        mem_plot.circle(source=self.source, x='memory_percent', y=0,
-                        size=10, fill_alpha=0.5)
+        mem_plot = figure(
+            title="Memory Use (%)",
+            toolbar_location=None,
+            x_range=(0, 1),
+            y_range=(-0.1, 0.1),
+            height=60,
+            width=width,
+            tools="",
+            **kwargs
+        )
+        mem_plot.circle(
+            source=self.source, x="memory_percent", y=0, size=10, fill_alpha=0.5
+        )
         mem_plot.ygrid.visible = False
         mem_plot.yaxis.minor_tick_line_alpha = 0
         mem_plot.xaxis.visible = False
@@ -1019,14 +1278,22 @@ def __init__(self, scheduler, width=800, **kwargs):
                   <span style="font-size: 10px; font-family: Monaco, monospace;">@worker: </span>
                   <span style="font-size: 10px; font-family: Monaco, monospace;">@cpu</span>
                 </div>
-                """
+                """,
         )
 
-        cpu_plot = figure(title='CPU Use (%)', toolbar_location=None,
-                          x_range=(0, 1), y_range=(-0.1, 0.1), height=60,
-                          width=width, tools='', **kwargs)
-        cpu_plot.circle(source=self.source, x='cpu_fraction', y=0,
-                        size=10, fill_alpha=0.5)
+        cpu_plot = figure(
+            title="CPU Use (%)",
+            toolbar_location=None,
+            x_range=(0, 1),
+            y_range=(-0.1, 0.1),
+            height=60,
+            width=width,
+            tools="",
+            **kwargs
+        )
+        cpu_plot.circle(
+            source=self.source, x="cpu_fraction", y=0, size=10, fill_alpha=0.5
+        )
         cpu_plot.ygrid.visible = False
         cpu_plot.yaxis.minor_tick_line_alpha = 0
         cpu_plot.xaxis.visible = False
@@ -1034,8 +1301,8 @@ def __init__(self, scheduler, width=800, **kwargs):
         cpu_plot.add_tools(hover, BoxSelectTool())
         self.cpu_plot = cpu_plot
 
-        if 'sizing_mode' in kwargs:
-            sizing_mode = {'sizing_mode': kwargs['sizing_mode']}
+        if "sizing_mode" in kwargs:
+            sizing_mode = {"sizing_mode": kwargs["sizing_mode"]}
         else:
             sizing_mode = {}
 
@@ -1043,7 +1310,7 @@ def __init__(self, scheduler, width=800, **kwargs):
         if self.extra_names:
             components.append(extra_table)
 
-        self.root = column(*components, id='bk-worker-table', **sizing_mode)
+        self.root = column(*components, id="bk-worker-table", **sizing_mode)
 
     @without_property_validation
     def update(self):
@@ -1051,60 +1318,65 @@ def update(self):
         for addr, ws in sorted(self.scheduler.workers.items()):
             for name in self.names + self.extra_names:
                 data[name].append(ws.metrics.get(name, None))
-            data['worker'][-1] = ws.address
+            data["worker"][-1] = ws.address
             if ws.memory_limit:
-                data['memory_percent'][-1] = ws.metrics['memory'] / ws.memory_limit
+                data["memory_percent"][-1] = ws.metrics["memory"] / ws.memory_limit
             else:
-                data['memory_percent'][-1] = ''
-            data['memory_limit'][-1] = ws.memory_limit
-            data['cpu'][-1] = ws.metrics['cpu'] / 100.0
-            data['cpu_fraction'][-1] = ws.metrics['cpu'] / 100.0 / ws.ncores
-            data['ncores'][-1] = ws.ncores
+                data["memory_percent"][-1] = ""
+            data["memory_limit"][-1] = ws.memory_limit
+            data["cpu"][-1] = ws.metrics["cpu"] / 100.0
+            data["cpu_fraction"][-1] = ws.metrics["cpu"] / 100.0 / ws.ncores
+            data["ncores"][-1] = ws.ncores
 
         self.source.data.update(data)
 
 
 def systemmonitor_doc(scheduler, extra, doc):
     with log_errors():
-        sysmon = SystemMonitor(scheduler, sizing_mode='stretch_both')
+        sysmon = SystemMonitor(scheduler, sizing_mode="stretch_both")
         doc.title = "Dask: Scheduler System Monitor"
         add_periodic_callback(doc, sysmon, 500)
 
         for subdoc in sysmon.root.children:
             doc.add_root(subdoc)
-        doc.template = env.get_template('system.html')
+        doc.template = env.get_template("system.html")
         doc.template_variables.update(extra)
         doc.theme = BOKEH_THEME
 
 
 def stealing_doc(scheduler, extra, doc):
     with log_errors():
-        occupancy = Occupancy(scheduler, height=200, sizing_mode='scale_width')
-        stealing_ts = StealingTimeSeries(scheduler, sizing_mode='scale_width')
-        stealing_events = StealingEvents(scheduler, sizing_mode='scale_width')
+        occupancy = Occupancy(scheduler, height=200, sizing_mode="scale_width")
+        stealing_ts = StealingTimeSeries(scheduler, sizing_mode="scale_width")
+        stealing_events = StealingEvents(scheduler, sizing_mode="scale_width")
         stealing_events.root.x_range = stealing_ts.root.x_range
         doc.title = "Dask: Work Stealing"
         add_periodic_callback(doc, occupancy, 500)
         add_periodic_callback(doc, stealing_ts, 500)
         add_periodic_callback(doc, stealing_events, 500)
 
-        doc.add_root(column(occupancy.root, stealing_ts.root,
-                            stealing_events.root,
-                            sizing_mode='scale_width'))
+        doc.add_root(
+            column(
+                occupancy.root,
+                stealing_ts.root,
+                stealing_events.root,
+                sizing_mode="scale_width",
+            )
+        )
 
-        doc.template = env.get_template('simple.html')
+        doc.template = env.get_template("simple.html")
         doc.template_variables.update(extra)
         doc.theme = BOKEH_THEME
 
 
 def events_doc(scheduler, extra, doc):
     with log_errors():
-        events = Events(scheduler, 'all', height=250)
+        events = Events(scheduler, "all", height=250)
         events.update()
         add_periodic_callback(doc, events, 500)
         doc.title = "Dask: Scheduler Events"
-        doc.add_root(column(events.root, sizing_mode='scale_width'))
-        doc.template = env.get_template('simple.html')
+        doc.add_root(column(events.root, sizing_mode="scale_width"))
+        doc.template = env.get_template("simple.html")
         doc.template_variables.update(extra)
         doc.theme = BOKEH_THEME
 
@@ -1116,63 +1388,72 @@ def workers_doc(scheduler, extra, doc):
         add_periodic_callback(doc, table, 500)
         doc.title = "Dask: Workers"
         doc.add_root(table.root)
-        doc.template = env.get_template('simple.html')
+        doc.template = env.get_template("simple.html")
         doc.template_variables.update(extra)
         doc.theme = BOKEH_THEME
 
 
 def tasks_doc(scheduler, extra, doc):
     with log_errors():
-        ts = TaskStream(scheduler, n_rectangles=100000, clear_interval='60s',
-                        sizing_mode='stretch_both')
+        ts = TaskStream(
+            scheduler,
+            n_rectangles=100000,
+            clear_interval="60s",
+            sizing_mode="stretch_both",
+        )
         ts.update()
         add_periodic_callback(doc, ts, 5000)
         doc.title = "Dask: Task Stream"
         doc.add_root(ts.root)
-        doc.template = env.get_template('simple.html')
+        doc.template = env.get_template("simple.html")
         doc.template_variables.update(extra)
         doc.theme = BOKEH_THEME
 
 
 def graph_doc(scheduler, extra, doc):
     with log_errors():
-        graph = GraphPlot(scheduler, sizing_mode='stretch_both')
+        graph = GraphPlot(scheduler, sizing_mode="stretch_both")
         doc.title = "Dask: Task Graph"
         graph.update()
         add_periodic_callback(doc, graph, 200)
         doc.add_root(graph.root)
 
-        doc.template = env.get_template('simple.html')
+        doc.template = env.get_template("simple.html")
         doc.template_variables.update(extra)
         doc.theme = BOKEH_THEME
 
 
 def status_doc(scheduler, extra, doc):
     with log_errors():
-        task_stream = TaskStream(scheduler, n_rectangles=1000,
-                                 clear_interval='10s', sizing_mode='stretch_both')
+        task_stream = TaskStream(
+            scheduler,
+            n_rectangles=1000,
+            clear_interval="10s",
+            sizing_mode="stretch_both",
+        )
         task_stream.update()
         add_periodic_callback(doc, task_stream, 100)
 
-        task_progress = TaskProgress(scheduler, sizing_mode='stretch_both')
+        task_progress = TaskProgress(scheduler, sizing_mode="stretch_both")
         task_progress.update()
         add_periodic_callback(doc, task_progress, 100)
 
         if len(scheduler.workers) < 50:
-            current_load = CurrentLoad(scheduler, sizing_mode='stretch_both')
+            current_load = CurrentLoad(scheduler, sizing_mode="stretch_both")
             current_load.update()
             add_periodic_callback(doc, current_load, 100)
             doc.add_root(current_load.nbytes_figure)
             doc.add_root(current_load.processing_figure)
         else:
-            nbytes_hist = NBytesHistogram(scheduler, sizing_mode='stretch_both')
+            nbytes_hist = NBytesHistogram(scheduler, sizing_mode="stretch_both")
             nbytes_hist.update()
-            processing_hist = ProcessingHistogram(scheduler, sizing_mode='stretch_both')
+            processing_hist = ProcessingHistogram(scheduler, sizing_mode="stretch_both")
             processing_hist.update()
             add_periodic_callback(doc, nbytes_hist, 100)
             add_periodic_callback(doc, processing_hist, 100)
-            current_load_fig = row(nbytes_hist.root, processing_hist.root,
-                                   sizing_mode='stretch_both')
+            current_load_fig = row(
+                nbytes_hist.root, processing_hist.root, sizing_mode="stretch_both"
+            )
 
             doc.add_root(nbytes_hist.root)
             doc.add_root(processing_hist.root)
@@ -1181,14 +1462,15 @@ def status_doc(scheduler, extra, doc):
         doc.add_root(task_progress.root)
         doc.add_root(task_stream.root)
         doc.theme = BOKEH_THEME
-        doc.template = env.get_template('status.html')
+        doc.template = env.get_template("status.html")
         doc.template_variables.update(extra)
         doc.theme = BOKEH_THEME
 
 
 def individual_task_stream_doc(scheduler, extra, doc):
-    task_stream = TaskStream(scheduler, n_rectangles=1000,
-                             clear_interval='10s', sizing_mode='stretch_both')
+    task_stream = TaskStream(
+        scheduler, n_rectangles=1000, clear_interval="10s", sizing_mode="stretch_both"
+    )
     task_stream.update()
     add_periodic_callback(doc, task_stream, 100)
     doc.add_root(task_stream.root)
@@ -1196,7 +1478,7 @@ def individual_task_stream_doc(scheduler, extra, doc):
 
 
 def individual_nbytes_doc(scheduler, extra, doc):
-    current_load = CurrentLoad(scheduler, sizing_mode='stretch_both')
+    current_load = CurrentLoad(scheduler, sizing_mode="stretch_both")
     current_load.update()
     add_periodic_callback(doc, current_load, 100)
     doc.add_root(current_load.nbytes_figure)
@@ -1204,7 +1486,7 @@ def individual_nbytes_doc(scheduler, extra, doc):
 
 
 def individual_nprocessing_doc(scheduler, extra, doc):
-    current_load = CurrentLoad(scheduler, sizing_mode='stretch_both')
+    current_load = CurrentLoad(scheduler, sizing_mode="stretch_both")
     current_load.update()
     add_periodic_callback(doc, current_load, 100)
     doc.add_root(current_load.processing_figure)
@@ -1212,7 +1494,7 @@ def individual_nprocessing_doc(scheduler, extra, doc):
 
 
 def individual_progress_doc(scheduler, extra, doc):
-    task_progress = TaskProgress(scheduler, height=160, sizing_mode='stretch_both')
+    task_progress = TaskProgress(scheduler, height=160, sizing_mode="stretch_both")
     task_progress.update()
     add_periodic_callback(doc, task_progress, 100)
     doc.add_root(task_progress.root)
@@ -1221,7 +1503,7 @@ def individual_progress_doc(scheduler, extra, doc):
 
 def individual_graph_doc(scheduler, extra, doc):
     with log_errors():
-        graph = GraphPlot(scheduler, sizing_mode='stretch_both')
+        graph = GraphPlot(scheduler, sizing_mode="stretch_both")
         graph.update()
 
         add_periodic_callback(doc, graph, 200)
@@ -1231,7 +1513,7 @@ def individual_graph_doc(scheduler, extra, doc):
 
 def individual_profile_doc(scheduler, extra, doc):
     with log_errors():
-        prof = ProfileTimePlot(scheduler, sizing_mode='scale_width', doc=doc)
+        prof = ProfileTimePlot(scheduler, sizing_mode="scale_width", doc=doc)
         doc.add_root(prof.root)
         prof.trigger_update()
         doc.theme = BOKEH_THEME
@@ -1239,7 +1521,7 @@ def individual_profile_doc(scheduler, extra, doc):
 
 def individual_profile_server_doc(scheduler, extra, doc):
     with log_errors():
-        prof = ProfileServer(scheduler, sizing_mode='scale_width', doc=doc)
+        prof = ProfileServer(scheduler, sizing_mode="scale_width", doc=doc)
         doc.add_root(prof.root)
         prof.trigger_update()
         doc.theme = BOKEH_THEME
@@ -1257,9 +1539,9 @@ def individual_workers_doc(scheduler, extra, doc):
 def profile_doc(scheduler, extra, doc):
     with log_errors():
         doc.title = "Dask: Profile"
-        prof = ProfileTimePlot(scheduler, sizing_mode='scale_width', doc=doc)
+        prof = ProfileTimePlot(scheduler, sizing_mode="scale_width", doc=doc)
         doc.add_root(prof.root)
-        doc.template = env.get_template('simple.html')
+        doc.template = env.get_template("simple.html")
         doc.template_variables.update(extra)
         doc.theme = BOKEH_THEME
 
@@ -1269,9 +1551,9 @@ def profile_doc(scheduler, extra, doc):
 def profile_server_doc(scheduler, extra, doc):
     with log_errors():
         doc.title = "Dask: Profile of Event Loop"
-        prof = ProfileServer(scheduler, sizing_mode='scale_width', doc=doc)
+        prof = ProfileServer(scheduler, sizing_mode="scale_width", doc=doc)
         doc.add_root(prof.root)
-        doc.template = env.get_template('simple.html')
+        doc.template = env.get_template("simple.html")
         doc.template_variables.update(extra)
         doc.theme = BOKEH_THEME
 
@@ -1279,48 +1561,46 @@ def profile_server_doc(scheduler, extra, doc):
 
 
 class BokehScheduler(BokehServer):
-    def __init__(self, scheduler, io_loop=None, prefix='', **kwargs):
+    def __init__(self, scheduler, io_loop=None, prefix="", **kwargs):
         self.scheduler = scheduler
-        prefix = prefix or ''
-        prefix = prefix.rstrip('/')
-        if prefix and not prefix.startswith('/'):
-            prefix = '/' + prefix
+        prefix = prefix or ""
+        prefix = prefix.rstrip("/")
+        if prefix and not prefix.startswith("/"):
+            prefix = "/" + prefix
         self.prefix = prefix
 
         self.server_kwargs = kwargs
-        self.server_kwargs['prefix'] = prefix or None
+        self.server_kwargs["prefix"] = prefix or None
 
         self.apps = {
-            '/system': systemmonitor_doc,
-            '/stealing': stealing_doc,
-            '/workers': workers_doc,
-            '/events': events_doc,
-            '/counters': counters_doc,
-            '/tasks': tasks_doc,
-            '/status': status_doc,
-            '/profile': profile_doc,
-            '/profile-server': profile_server_doc,
-            '/graph': graph_doc,
-
-            '/individual-task-stream': individual_task_stream_doc,
-            '/individual-progress': individual_progress_doc,
-            '/individual-graph': individual_graph_doc,
-            '/individual-profile': individual_profile_doc,
-            '/individual-profile-server': individual_profile_server_doc,
-            '/individual-nbytes': individual_nbytes_doc,
-            '/individual-nprocessing': individual_nprocessing_doc,
-            '/individual-workers': individual_workers_doc,
+            "/system": systemmonitor_doc,
+            "/stealing": stealing_doc,
+            "/workers": workers_doc,
+            "/events": events_doc,
+            "/counters": counters_doc,
+            "/tasks": tasks_doc,
+            "/status": status_doc,
+            "/profile": profile_doc,
+            "/profile-server": profile_server_doc,
+            "/graph": graph_doc,
+            "/individual-task-stream": individual_task_stream_doc,
+            "/individual-progress": individual_progress_doc,
+            "/individual-graph": individual_graph_doc,
+            "/individual-profile": individual_profile_doc,
+            "/individual-profile-server": individual_profile_server_doc,
+            "/individual-nbytes": individual_nbytes_doc,
+            "/individual-nprocessing": individual_nprocessing_doc,
+            "/individual-workers": individual_workers_doc,
         }
 
-        self.apps = {k: partial(v, scheduler, self.extra)
-                     for k, v in self.apps.items()}
+        self.apps = {k: partial(v, scheduler, self.extra) for k, v in self.apps.items()}
 
         self.loop = io_loop or scheduler.loop
         self.server = None
 
     @property
     def extra(self):
-        return merge({'prefix': self.prefix}, template_variables)
+        return merge({"prefix": self.prefix}, template_variables)
 
     @property
     def my_server(self):
@@ -1330,7 +1610,14 @@ def listen(self, *args, **kwargs):
         super(BokehScheduler, self).listen(*args, **kwargs)
 
         from .scheduler_html import routes
-        handlers = [(self.prefix + '/' + url, cls, {'server': self.my_server, 'extra': self.extra})
-                    for url, cls in routes]
 
-        self.server._tornado.add_handlers(r'.*', handlers)
+        handlers = [
+            (
+                self.prefix + "/" + url,
+                cls,
+                {"server": self.my_server, "extra": self.extra},
+            )
+            for url, cls in routes
+        ]
+
+        self.server._tornado.add_handlers(r".*", handlers)
diff --git a/distributed/bokeh/scheduler_html.py b/distributed/bokeh/scheduler_html.py
index e8050e4a9fa..d1ba2646ed6 100644
--- a/distributed/bokeh/scheduler_html.py
+++ b/distributed/bokeh/scheduler_html.py
@@ -10,7 +10,9 @@
 
 dirname = os.path.dirname(__file__)
 
-ns = {func.__name__: func for func in [format_bytes, format_time, datetime.fromtimestamp]}
+ns = {
+    func.__name__: func for func in [format_bytes, format_time, datetime.fromtimestamp]
+}
 
 
 class RequestHandler(web.RequestHandler):
@@ -19,43 +21,50 @@ def initialize(self, server=None, extra=None):
         self.extra = extra or {}
 
     def get_template_path(self):
-        return os.path.join(dirname, 'templates')
+        return os.path.join(dirname, "templates")
 
 
 class Workers(RequestHandler):
     def get(self):
         with log_errors():
-            self.render('workers.html',
-                        title='Workers',
-                        scheduler=self.server,
-                        **toolz.merge(self.server.__dict__, ns, self.extra))
+            self.render(
+                "workers.html",
+                title="Workers",
+                scheduler=self.server,
+                **toolz.merge(self.server.__dict__, ns, self.extra)
+            )
 
 
 class Worker(RequestHandler):
     def get(self, worker):
         worker = escape.url_unescape(worker)
         with log_errors():
-            self.render('worker.html',
-                        title='Worker: ' + worker, Worker=worker,
-                        **toolz.merge(self.server.__dict__, ns, self.extra))
+            self.render(
+                "worker.html",
+                title="Worker: " + worker,
+                Worker=worker,
+                **toolz.merge(self.server.__dict__, ns, self.extra)
+            )
 
 
 class Task(RequestHandler):
     def get(self, task):
         task = escape.url_unescape(task)
         with log_errors():
-            self.render('task.html',
-                        title='Task: ' + task,
-                        Task=task,
-                        server=self.server,
-                        **toolz.merge(self.server.__dict__, ns, self.extra))
+            self.render(
+                "task.html",
+                title="Task: " + task,
+                Task=task,
+                server=self.server,
+                **toolz.merge(self.server.__dict__, ns, self.extra)
+            )
 
 
 class Logs(RequestHandler):
     def get(self):
         with log_errors():
             logs = self.server.get_logs()
-            self.render('logs.html', title="Logs", logs=logs, **self.extra)
+            self.render("logs.html", title="Logs", logs=logs, **self.extra)
 
 
 class WorkerLogs(RequestHandler):
@@ -65,8 +74,7 @@ def get(self, worker):
             worker = escape.url_unescape(worker)
             logs = yield self.server.get_worker_logs(workers=[worker])
             logs = logs[worker]
-            self.render('logs.html', title="Logs: " + worker, logs=logs,
-                        **self.extra)
+            self.render("logs.html", title="Logs: " + worker, logs=logs, **self.extra)
 
 
 class WorkerCallStacks(RequestHandler):
@@ -76,8 +84,12 @@ def get(self, worker):
             worker = escape.url_unescape(worker)
             keys = self.server.processing[worker]
             call_stack = yield self.server.get_call_stack(keys=keys)
-            self.render('call-stack.html', title="Call Stacks: " + worker,
-                        call_stack=call_stack, **self.extra)
+            self.render(
+                "call-stack.html",
+                title="Call Stacks: " + worker,
+                call_stack=call_stack,
+                **self.extra
+            )
 
 
 class TaskCallStack(RequestHandler):
@@ -87,11 +99,17 @@ def get(self, key):
             key = escape.url_unescape(key)
             call_stack = yield self.server.get_call_stack(keys=[key])
             if not call_stack:
-                self.write("<p>Task not actively running. "
-                           "It may be finished or not yet started</p>")
+                self.write(
+                    "<p>Task not actively running. "
+                    "It may be finished or not yet started</p>"
+                )
             else:
-                self.render('call-stack.html', title="Call Stack: " + key,
-                            call_stack=call_stack, **self.extra)
+                self.render(
+                    "call-stack.html",
+                    title="Call Stack: " + key,
+                    call_stack=call_stack,
+                    **self.extra
+                )
 
 
 class CountsJSON(RequestHandler):
@@ -109,7 +127,7 @@ def get(self):
         for ts in scheduler.tasks.values():
             if ts.exception_blame is not None:
                 erred += 1
-            elif ts.state == 'released':
+            elif ts.state == "released":
                 released += 1
             if ts.waiting_on:
                 waiting += 1
@@ -122,21 +140,21 @@ def get(self):
             processing += len(ws.processing)
 
         response = {
-            'bytes': nbytes,
-            'clients': len(scheduler.clients),
-            'cores': ncores,
-            'erred': erred,
-            'hosts': len(scheduler.host_info),
-            'idle': len(scheduler.idle),
-            'memory': memory,
-            'processing': processing,
-            'released': released,
-            'saturated': len(scheduler.saturated),
-            'tasks': len(scheduler.tasks),
-            'unrunnable': len(scheduler.unrunnable),
-            'waiting': waiting,
-            'waiting_data': waiting_data,
-            'workers': len(scheduler.workers),
+            "bytes": nbytes,
+            "clients": len(scheduler.clients),
+            "cores": ncores,
+            "erred": erred,
+            "hosts": len(scheduler.host_info),
+            "idle": len(scheduler.idle),
+            "memory": memory,
+            "processing": processing,
+            "released": released,
+            "saturated": len(scheduler.saturated),
+            "tasks": len(scheduler.tasks),
+            "unrunnable": len(scheduler.unrunnable),
+            "waiting": waiting,
+            "waiting_data": waiting_data,
+            "workers": len(scheduler.workers),
         }
         self.write(response)
 
@@ -149,17 +167,20 @@ def get(self):
 class IndexJSON(RequestHandler):
     def get(self):
         with log_errors():
-            r = [url for url, _ in routes if url.endswith('.json')]
-            self.render('json-index.html', routes=r, title='Index of JSON routes', **self.extra)
+            r = [url for url, _ in routes if url.endswith(".json")]
+            self.render(
+                "json-index.html", routes=r, title="Index of JSON routes", **self.extra
+            )
 
 
 class IndividualPlots(RequestHandler):
     def get(self):
-        bokeh_server = self.server.services['bokeh']
-        result = {uri.strip('/').replace('-', ' ').title(): uri
-                  for uri in bokeh_server.apps
-                  if uri.lstrip('/').startswith('individual-')
-                  and not uri.endswith('.json')}
+        bokeh_server = self.server.services["bokeh"]
+        result = {
+            uri.strip("/").replace("-", " ").title(): uri
+            for uri in bokeh_server.apps
+            if uri.lstrip("/").startswith("individual-") and not uri.endswith(".json")
+        }
         self.write(result)
 
 
@@ -170,13 +191,13 @@ def __init__(self, server, prometheus_client):
 
     def collect(self):
         yield self.prometheus_client.core.GaugeMetricFamily(
-            'dask_scheduler_workers',
-            'Number of workers.',
+            "dask_scheduler_workers",
+            "Number of workers.",
             value=len(self.server.workers),
         )
         yield self.prometheus_client.core.GaugeMetricFamily(
-            'dask_scheduler_clients',
-            'Number of clients.',
+            "dask_scheduler_clients",
+            "Number of clients.",
             value=len(self.server.clients),
         )
 
@@ -186,6 +207,7 @@ class PrometheusHandler(RequestHandler):
 
     def __init__(self, *args, **kwargs):
         import prometheus_client  # keep out of global namespace
+
         self.prometheus_client = prometheus_client
 
         super(PrometheusHandler, self).__init__(*args, **kwargs)
@@ -197,41 +219,38 @@ def _init(self):
             return
 
         self.prometheus_client.REGISTRY.register(
-            _PrometheusCollector(
-                self.server,
-                self.prometheus_client,
-            )
+            _PrometheusCollector(self.server, self.prometheus_client)
         )
 
         PrometheusHandler._initialized = True
 
     def get(self):
         self.write(self.prometheus_client.generate_latest())
-        self.set_header('Content-Type', 'text/plain; version=0.0.4')
+        self.set_header("Content-Type", "text/plain; version=0.0.4")
 
 
 class HealthHandler(RequestHandler):
     def get(self):
-        self.write('ok')
-        self.set_header('Content-Type', 'text/plain')
+        self.write("ok")
+        self.set_header("Content-Type", "text/plain")
 
 
 routes = [
-        (r'info/main/workers.html', Workers),
-        (r'info/worker/(.*).html', Worker),
-        (r'info/task/(.*).html', Task),
-        (r'info/main/logs.html', Logs),
-        (r'info/call-stacks/(.*).html', WorkerCallStacks),
-        (r'info/call-stack/(.*).html', TaskCallStack),
-        (r'info/logs/(.*).html', WorkerLogs),
-        (r'json/counts.json', CountsJSON),
-        (r'json/identity.json', IdentityJSON),
-        (r'json/index.html', IndexJSON),
-        (r'individual-plots.json', IndividualPlots),
-        (r'metrics', PrometheusHandler),
-        (r'health', HealthHandler),
+    (r"info/main/workers.html", Workers),
+    (r"info/worker/(.*).html", Worker),
+    (r"info/task/(.*).html", Task),
+    (r"info/main/logs.html", Logs),
+    (r"info/call-stacks/(.*).html", WorkerCallStacks),
+    (r"info/call-stack/(.*).html", TaskCallStack),
+    (r"info/logs/(.*).html", WorkerLogs),
+    (r"json/counts.json", CountsJSON),
+    (r"json/identity.json", IdentityJSON),
+    (r"json/index.html", IndexJSON),
+    (r"individual-plots.json", IndividualPlots),
+    (r"metrics", PrometheusHandler),
+    (r"health", HealthHandler),
 ]
 
 
 def get_handlers(server):
-    return [(url, cls, {'server': server}) for url, cls in routes]
+    return [(url, cls, {"server": server}) for url, cls in routes]
diff --git a/distributed/bokeh/tests/test_components.py b/distributed/bokeh/tests/test_components.py
index 741c90c8d49..4f4df92f6cd 100644
--- a/distributed/bokeh/tests/test_components.py
+++ b/distributed/bokeh/tests/test_components.py
@@ -1,7 +1,8 @@
 from __future__ import print_function, division, absolute_import
 
 import pytest
-pytest.importorskip('bokeh')
+
+pytest.importorskip("bokeh")
 
 from bokeh.models import ColumnDataSource, Model
 from tornado import gen
@@ -10,14 +11,15 @@
 from distributed.utils_test import slowinc, gen_cluster
 
 from distributed.bokeh.components import (
-    TaskStream, MemoryUsage,
-    Processing, ProfilePlot, ProfileTimePlot
+    TaskStream,
+    MemoryUsage,
+    Processing,
+    ProfilePlot,
+    ProfileTimePlot,
 )
 
 
-@pytest.mark.parametrize('Component', [TaskStream,
-                                       MemoryUsage,
-                                       Processing])
+@pytest.mark.parametrize("Component", [TaskStream, MemoryUsage, Processing])
 def test_basic(Component):
     c = Component()
     assert isinstance(c.source, ColumnDataSource)
@@ -28,23 +30,24 @@ def test_basic(Component):
 @gen_cluster(client=True, check_new_threads=False)
 def test_profile_plot(c, s, a, b):
     p = ProfilePlot()
-    assert len(p.source.data['left']) <= 1
+    assert len(p.source.data["left"]) <= 1
     yield c.map(slowinc, range(10), delay=0.05)
     p.update(a.profile_recent)
-    assert len(p.source.data['left']) > 1
+    assert len(p.source.data["left"]) > 1
 
 
 @gen_cluster(client=True, check_new_threads=False)
 def test_profile_time_plot(c, s, a, b):
     from bokeh.io import curdoc
+
     sp = ProfileTimePlot(s, doc=curdoc())
     sp.trigger_update()
 
     ap = ProfileTimePlot(a, doc=curdoc())
     ap.trigger_update()
 
-    assert len(sp.source.data['left']) <= 1
-    assert len(ap.source.data['left']) <= 1
+    assert len(sp.source.data["left"]) <= 1
+    assert len(ap.source.data["left"]) <= 1
 
     yield c.map(slowinc, range(10), delay=0.05)
     ap.trigger_update()
diff --git a/distributed/bokeh/tests/test_scheduler_bokeh.py b/distributed/bokeh/tests/test_scheduler_bokeh.py
index acbf54bc102..380dff104e2 100644
--- a/distributed/bokeh/tests/test_scheduler_bokeh.py
+++ b/distributed/bokeh/tests/test_scheduler_bokeh.py
@@ -6,7 +6,8 @@
 from time import sleep
 
 import pytest
-pytest.importorskip('bokeh')
+
+pytest.importorskip("bokeh")
 from toolz import first
 from tornado import gen
 from tornado.httpclient import AsyncHTTPClient
@@ -17,48 +18,69 @@
 from distributed.metrics import time
 from distributed.utils_test import gen_cluster, inc, dec, slowinc, div
 from distributed.bokeh.worker import Counters, BokehWorker
-from distributed.bokeh.scheduler import (BokehScheduler, SystemMonitor,
-                                         Occupancy, StealingTimeSeries,
-                                         StealingEvents, Events,
-                                         TaskStream, TaskProgress,
-                                         MemoryUse, CurrentLoad,
-                                         ProcessingHistogram,
-                                         NBytesHistogram, WorkerTable,
-                                         GraphPlot, ProfileServer)
+from distributed.bokeh.scheduler import (
+    BokehScheduler,
+    SystemMonitor,
+    Occupancy,
+    StealingTimeSeries,
+    StealingEvents,
+    Events,
+    TaskStream,
+    TaskProgress,
+    MemoryUse,
+    CurrentLoad,
+    ProcessingHistogram,
+    NBytesHistogram,
+    WorkerTable,
+    GraphPlot,
+    ProfileServer,
+)
 
 from distributed.bokeh import scheduler
 
 scheduler.PROFILING = False
 
 
-@pytest.mark.skipif(sys.version_info[0] == 2,
-                    reason='https://github.com/bokeh/bokeh/issues/5494')
-@gen_cluster(client=True,
-             scheduler_kwargs={'services': {('bokeh', 0):  BokehScheduler}})
+@pytest.mark.skipif(
+    sys.version_info[0] == 2, reason="https://github.com/bokeh/bokeh/issues/5494"
+)
+@gen_cluster(client=True, scheduler_kwargs={"services": {("bokeh", 0): BokehScheduler}})
 def test_simple(c, s, a, b):
-    assert isinstance(s.services['bokeh'], BokehScheduler)
-    port = s.services['bokeh'].port
+    assert isinstance(s.services["bokeh"], BokehScheduler)
+    port = s.services["bokeh"].port
 
     future = c.submit(sleep, 1)
     yield gen.sleep(0.1)
 
     http_client = AsyncHTTPClient()
-    for suffix in ['system', 'counters', 'workers', 'status', 'tasks',
-                   'stealing', 'graph', 'individual-task-stream', 'individual-progress',
-                   'individual-graph', 'individual-nbytes',
-                   'individual-nprocessing',
-                   'individual-profile']:
-        response = yield http_client.fetch('http://localhost:%d/%s' % (port, suffix))
+    for suffix in [
+        "system",
+        "counters",
+        "workers",
+        "status",
+        "tasks",
+        "stealing",
+        "graph",
+        "individual-task-stream",
+        "individual-progress",
+        "individual-graph",
+        "individual-nbytes",
+        "individual-nprocessing",
+        "individual-profile",
+    ]:
+        response = yield http_client.fetch("http://localhost:%d/%s" % (port, suffix))
         body = response.body.decode()
-        assert 'bokeh' in body.lower()
+        assert "bokeh" in body.lower()
         assert not re.search("href=./", body)  # no absolute links
 
-    response = yield http_client.fetch('http://localhost:%d/individual-plots.json' % port)
+    response = yield http_client.fetch(
+        "http://localhost:%d/individual-plots.json" % port
+    )
     response = json.loads(response.body.decode())
     assert response
 
 
-@gen_cluster(client=True, worker_kwargs=dict(services={'bokeh': BokehWorker}))
+@gen_cluster(client=True, worker_kwargs=dict(services={"bokeh": BokehWorker}))
 def test_basic(c, s, a, b):
     for component in [SystemMonitor, Occupancy, StealingTimeSeries]:
         ss = component(s)
@@ -67,14 +89,13 @@ def test_basic(c, s, a, b):
         data = ss.source.data
         assert len(first(data.values()))
         if component is Occupancy:
-            assert all(addr.startswith('127.0.0.1:')
-                       for addr in data['bokeh_address'])
+            assert all(addr.startswith("127.0.0.1:") for addr in data["bokeh_address"])
 
 
 @gen_cluster(client=True)
 def test_counters(c, s, a, b):
-    pytest.importorskip('crick')
-    while 'tick-duration' not in s.digests:
+    pytest.importorskip("crick")
+    while "tick-duration" not in s.digests:
         yield gen.sleep(0.01)
     ss = Counters(s)
 
@@ -83,7 +104,7 @@ def test_counters(c, s, a, b):
     ss.update()
 
     start = time()
-    while not len(ss.digest_sources['tick-duration'][0].data['x']):
+    while not len(ss.digest_sources["tick-duration"][0].data["x"]):
         yield gen.sleep(1)
         assert time() < start + 5
 
@@ -92,8 +113,9 @@ def test_counters(c, s, a, b):
 def test_stealing_events(c, s, a, b):
     se = StealingEvents(s)
 
-    futures = c.map(slowinc, range(100), delay=0.1, workers=a.address,
-                    allow_other_workers=True)
+    futures = c.map(
+        slowinc, range(100), delay=0.1, workers=a.address, allow_other_workers=True
+    )
 
     while not b.task_state:  # will steal soon
         yield gen.sleep(0.01)
@@ -105,17 +127,18 @@ def test_stealing_events(c, s, a, b):
 
 @gen_cluster(client=True)
 def test_events(c, s, a, b):
-    e = Events(s, 'all')
+    e = Events(s, "all")
 
-    futures = c.map(slowinc, range(100), delay=0.1, workers=a.address,
-                    allow_other_workers=True)
+    futures = c.map(
+        slowinc, range(100), delay=0.1, workers=a.address, allow_other_workers=True
+    )
 
     while not b.task_state:
         yield gen.sleep(0.01)
 
     e.update()
     d = dict(e.source.data)
-    assert sum(a == 'add-worker' for a in d['action']) == 2
+    assert sum(a == "add-worker" for a in d["action"]) == 2
 
 
 @gen_cluster(client=True)
@@ -130,7 +153,7 @@ def test_task_stream(c, s, a, b):
     d = dict(ts.source.data)
 
     assert all(len(L) == 10 for L in d.values())
-    assert min(d['start']) == 0  # zero based
+    assert min(d["start"]) == 0  # zero based
 
     ts.update()
     d = dict(ts.source.data)
@@ -151,7 +174,7 @@ def test_task_stream_n_rectangles(c, s, a, b):
     yield wait(futures)
     ts.update()
 
-    assert len(ts.source.data['start']) == 10
+    assert len(ts.source.data["start"]) == 10
 
 
 @gen_cluster(client=True)
@@ -177,16 +200,16 @@ def test_task_stream_clear_interval(c, s, a, b):
     ts.update()
 
     assert len(set(map(len, ts.source.data.values()))) == 1
-    assert ts.source.data['name'].count('inc') == 10
-    assert ts.source.data['name'].count('dec') == 10
+    assert ts.source.data["name"].count("inc") == 10
+    assert ts.source.data["name"].count("dec") == 10
 
     yield gen.sleep(0.300)
     yield wait(c.map(inc, range(10, 20)))
     ts.update()
 
     assert len(set(map(len, ts.source.data.values()))) == 1
-    assert ts.source.data['name'].count('inc') == 10
-    assert ts.source.data['name'].count('dec') == 0
+    assert ts.source.data["name"].count("inc") == 10
+    assert ts.source.data["name"].count("dec") == 0
 
 
 @gen_cluster(client=True)
@@ -199,7 +222,7 @@ def test_TaskProgress(c, s, a, b):
     tp.update()
     d = dict(tp.source.data)
     assert all(len(L) == 1 for L in d.values())
-    assert d['name'] == ['slowinc']
+    assert d["name"] == ["slowinc"]
 
     futures2 = c.map(dec, range(5))
     yield wait(futures2)
@@ -207,7 +230,7 @@ def test_TaskProgress(c, s, a, b):
     tp.update()
     d = dict(tp.source.data)
     assert all(len(L) == 2 for L in d.values())
-    assert d['name'] == ['slowinc', 'dec']
+    assert d["name"] == ["slowinc", "dec"]
 
     del futures, futures2
 
@@ -215,7 +238,7 @@ def test_TaskProgress(c, s, a, b):
         yield gen.sleep(0.01)
 
     tp.update()
-    assert not tp.source.data['all']
+    assert not tp.source.data["all"]
 
 
 @gen_cluster(client=True)
@@ -223,7 +246,7 @@ def test_TaskProgress_empty(c, s, a, b):
     tp = TaskProgress(s)
     tp.update()
 
-    futures = [c.submit(inc, i, key='f-' + 'a' * i) for i in range(20)]
+    futures = [c.submit(inc, i, key="f-" + "a" * i) for i in range(20)]
     yield wait(futures)
     tp.update()
 
@@ -245,7 +268,7 @@ def test_MemoryUse(c, s, a, b):
     mu.update()
     d = dict(mu.source.data)
     assert all(len(L) == 1 for L in d.values())
-    assert d['name'] == ['slowinc']
+    assert d["name"] == ["slowinc"]
 
 
 @gen_cluster(client=True)
@@ -259,34 +282,34 @@ def test_CurrentLoad(c, s, a, b):
     d = dict(cl.source.data)
 
     assert all(len(L) == 2 for L in d.values())
-    assert all(d['nbytes'])
+    assert all(d["nbytes"])
 
 
 @gen_cluster(client=True)
 def test_ProcessingHistogram(c, s, a, b):
     ph = ProcessingHistogram(s)
     ph.update()
-    assert (ph.source.data['top'] != 0).sum() == 1
+    assert (ph.source.data["top"] != 0).sum() == 1
 
     futures = c.map(slowinc, range(10), delay=0.050)
     while not s.tasks:
         yield gen.sleep(0.01)
 
     ph.update()
-    assert ph.source.data['right'][-1] > 2
+    assert ph.source.data["right"][-1] > 2
 
 
 @gen_cluster(client=True)
 def test_NBytesHistogram(c, s, a, b):
     nh = NBytesHistogram(s)
     nh.update()
-    assert (nh.source.data['top'] != 0).sum() == 1
+    assert (nh.source.data["top"] != 0).sum() == 1
 
     futures = c.map(inc, range(10))
     yield wait(futures)
 
     nh.update()
-    assert nh.source.data['right'][-1] > 5 * 20
+    assert nh.source.data["right"][-1] > 5 * 20
 
 
 @gen_cluster(client=True)
@@ -296,7 +319,7 @@ def test_WorkerTable(c, s, a, b):
     assert all(wt.source.data.values())
     assert all(len(v) == 2 for v in wt.source.data.values())
 
-    ncores = wt.source.data['ncores']
+    ncores = wt.source.data["ncores"]
     assert all(ncores)
 
 
@@ -308,8 +331,7 @@ def metric_port(worker):
     def metric_address(worker):
         return worker.address
 
-    metrics = {'metric_port': metric_port,
-               'metric_address': metric_address}
+    metrics = {"metric_port": metric_port, "metric_address": metric_address}
 
     for w in [a, b]:
         for name, func in metrics.items():
@@ -318,8 +340,8 @@ def metric_address(worker):
     yield [a.heartbeat(), b.heartbeat()]
 
     for w in [a, b]:
-        assert s.workers[w.address].metrics['metric_port'] == w.port
-        assert s.workers[w.address].metrics['metric_address'] == w.address
+        assert s.workers[w.address].metrics["metric_port"] == w.port
+        assert s.workers[w.address].metrics["metric_address"] == w.address
 
     wt = WorkerTable(s)
     wt.update()
@@ -330,9 +352,9 @@ def metric_address(worker):
 
     assert all(data.values())
     assert all(len(v) == 2 for v in data.values())
-    my_index = data['worker'].index(a.address), data['worker'].index(b.address)
-    assert [data['metric_port'][i] for i in my_index] == [a.port, b.port]
-    assert [data['metric_address'][i] for i in my_index] == [a.address, b.address]
+    my_index = data["worker"].index(a.address), data["worker"].index(b.address)
+    assert [data["metric_port"][i] for i in my_index] == [a.port, b.port]
+    assert [data["metric_address"][i] for i in my_index] == [a.address, b.address]
 
 
 @gen_cluster(client=True)
@@ -340,24 +362,24 @@ def test_WorkerTable_different_metrics(c, s, a, b):
     def metric_port(worker):
         return worker.port
 
-    a.metrics['metric_a'] = metric_port
-    b.metrics['metric_b'] = metric_port
+    a.metrics["metric_a"] = metric_port
+    b.metrics["metric_b"] = metric_port
     yield [a.heartbeat(), b.heartbeat()]
 
-    assert s.workers[a.address].metrics['metric_a'] == a.port
-    assert s.workers[b.address].metrics['metric_b'] == b.port
+    assert s.workers[a.address].metrics["metric_a"] == a.port
+    assert s.workers[b.address].metrics["metric_b"] == b.port
 
     wt = WorkerTable(s)
     wt.update()
     data = wt.source.data
 
-    assert 'metric_a' in data
-    assert 'metric_b' in data
+    assert "metric_a" in data
+    assert "metric_b" in data
     assert all(data.values())
     assert all(len(v) == 2 for v in data.values())
-    my_index = data['worker'].index(a.address), data['worker'].index(b.address)
-    assert [data['metric_a'][i] for i in my_index] == [a.port, None]
-    assert [data['metric_b'][i] for i in my_index] == [None, b.port]
+    my_index = data["worker"].index(a.address), data["worker"].index(b.address)
+    assert [data["metric_a"][i] for i in my_index] == [a.port, None]
+    assert [data["metric_b"][i] for i in my_index] == [None, b.port]
 
 
 @gen_cluster(client=True)
@@ -365,51 +387,51 @@ def test_WorkerTable_metrics_with_different_metric_2(c, s, a, b):
     def metric_port(worker):
         return worker.port
 
-    a.metrics['metric_a'] = metric_port
+    a.metrics["metric_a"] = metric_port
     yield [a.heartbeat(), b.heartbeat()]
 
     wt = WorkerTable(s)
     wt.update()
     data = wt.source.data
 
-    assert 'metric_a' in data
+    assert "metric_a" in data
     assert all(data.values())
     assert all(len(v) == 2 for v in data.values())
-    my_index = data['worker'].index(a.address), data['worker'].index(b.address)
-    assert [data['metric_a'][i] for i in my_index] == [a.port, None]
+    my_index = data["worker"].index(a.address), data["worker"].index(b.address)
+    assert [data["metric_a"][i] for i in my_index] == [a.port, None]
 
 
-@gen_cluster(client=True, worker_kwargs={'metrics': {'my_port': lambda w: w.port}})
+@gen_cluster(client=True, worker_kwargs={"metrics": {"my_port": lambda w: w.port}})
 def test_WorkerTable_add_and_remove_metrics(c, s, a, b):
     def metric_port(worker):
         return worker.port
 
-    a.metrics['metric_a'] = metric_port
-    b.metrics['metric_b'] = metric_port
+    a.metrics["metric_a"] = metric_port
+    b.metrics["metric_b"] = metric_port
     yield [a.heartbeat(), b.heartbeat()]
 
-    assert s.workers[a.address].metrics['metric_a'] == a.port
-    assert s.workers[b.address].metrics['metric_b'] == b.port
+    assert s.workers[a.address].metrics["metric_a"] == a.port
+    assert s.workers[b.address].metrics["metric_b"] == b.port
 
     wt = WorkerTable(s)
     wt.update()
-    assert 'metric_a' in wt.source.data
-    assert 'metric_b' in wt.source.data
+    assert "metric_a" in wt.source.data
+    assert "metric_b" in wt.source.data
 
     # Remove 'metric_b' from worker b
-    del b.metrics['metric_b']
+    del b.metrics["metric_b"]
     yield [a.heartbeat(), b.heartbeat()]
 
     wt = WorkerTable(s)
     wt.update()
-    assert 'metric_a' in wt.source.data
+    assert "metric_a" in wt.source.data
 
-    del a.metrics['metric_a']
+    del a.metrics["metric_a"]
     yield [a.heartbeat(), b.heartbeat()]
 
     wt = WorkerTable(s)
     wt.update()
-    assert 'metric_a' not in wt.source.data
+    assert "metric_a" not in wt.source.data
 
 
 @gen_cluster(client=True)
@@ -417,14 +439,14 @@ def test_WorkerTable_custom_metric_overlap_with_core_metric(c, s, a, b):
     def metric(worker):
         return -999
 
-    a.metrics['executing'] = metric
-    a.metrics['cpu'] = metric
-    a.metrics['metric'] = metric
+    a.metrics["executing"] = metric
+    a.metrics["cpu"] = metric
+    a.metrics["metric"] = metric
     yield [a.heartbeat(), b.heartbeat()]
 
-    assert s.workers[a.address].metrics['executing'] != -999
-    assert s.workers[a.address].metrics['cpu'] != -999
-    assert s.workers[a.address].metrics['metric'] == -999
+    assert s.workers[a.address].metrics["executing"] != -999
+    assert s.workers[a.address].metrics["cpu"] != -999
+    assert s.workers[a.address].metrics["metric"] == -999
 
 
 @gen_cluster(client=True)
@@ -438,7 +460,7 @@ def test_GraphPlot(c, s, a, b):
     assert set(map(len, gp.node_source.data.values())) == {6}
     assert set(map(len, gp.edge_source.data.values())) == {5}
 
-    da = pytest.importorskip('dask.array')
+    da = pytest.importorskip("dask.array")
     x = da.random.random((20, 20), chunks=(10, 10)).persist()
     y = (x + x.T) - x.mean(axis=0)
     y = y.persist()
@@ -459,12 +481,12 @@ def test_GraphPlot(c, s, a, b):
     while key in s.tasks:
         yield gen.sleep(0.01)
 
-    assert 'memory' in gp.node_source.data['state']
+    assert "memory" in gp.node_source.data["state"]
 
     gp.update()
     gp.update()
 
-    assert not all(x == 'False' for x in gp.edge_source.data['visible'])
+    assert not all(x == "False" for x in gp.edge_source.data["visible"])
 
 
 @gen_cluster(client=True)
@@ -493,31 +515,33 @@ def test_GraphPlot_clear(c, s, a, b):
 
 @gen_cluster(client=True, timeout=30)
 def test_GraphPlot_complex(c, s, a, b):
-    da = pytest.importorskip('dask.array')
+    da = pytest.importorskip("dask.array")
     gp = GraphPlot(s)
     x = da.random.random((2000, 2000), chunks=(1000, 1000))
     y = ((x + x.T) - x.mean(axis=0)).persist()
     yield wait(y)
     gp.update()
-    assert len(gp.layout.index) == len(gp.node_source.data['x'])
+    assert len(gp.layout.index) == len(gp.node_source.data["x"])
     assert len(gp.layout.index) == len(s.tasks)
     z = (x - y).sum().persist()
     yield wait(z)
     gp.update()
-    assert len(gp.layout.index) == len(gp.node_source.data['x'])
+    assert len(gp.layout.index) == len(gp.node_source.data["x"])
     assert len(gp.layout.index) == len(s.tasks)
     del z
     yield gen.sleep(0.2)
     gp.update()
-    assert len(gp.layout.index) == sum(v == 'True' for v in gp.node_source.data['visible'])
+    assert len(gp.layout.index) == sum(
+        v == "True" for v in gp.node_source.data["visible"]
+    )
     assert len(gp.layout.index) == len(s.tasks)
-    assert max(gp.layout.index.values()) < len(gp.node_source.data['visible'])
-    assert gp.layout.next_index == len(gp.node_source.data['visible'])
+    assert max(gp.layout.index.values()) < len(gp.node_source.data["visible"])
+    assert gp.layout.next_index == len(gp.node_source.data["visible"])
     gp.update()
     assert set(gp.layout.index.values()) == set(range(len(gp.layout.index)))
-    visible = gp.node_source.data['visible']
+    visible = gp.node_source.data["visible"]
     keys = list(map(tokey, flatten(y.__dask_keys__())))
-    assert all(visible[gp.layout.index[key]] == 'True' for key in keys)
+    assert all(visible[gp.layout.index[key]] == "True" for key in keys)
 
 
 @gen_cluster(client=True)
@@ -529,24 +553,29 @@ def test_GraphPlot_order(c, s, a, b):
     gp = GraphPlot(s)
     gp.update()
 
-    assert gp.node_source.data['state'][gp.layout.index[y.key]] == 'erred'
+    assert gp.node_source.data["state"][gp.layout.index[y.key]] == "erred"
 
 
-@gen_cluster(client=True,
-             config={'distributed.worker.profile.interval': '10ms',
-                     'distributed.worker.profile.cycle': '50ms'})
+@gen_cluster(
+    client=True,
+    config={
+        "distributed.worker.profile.interval": "10ms",
+        "distributed.worker.profile.cycle": "50ms",
+    },
+)
 def test_profile_server(c, s, a, b):
     ptp = ProfileServer(s)
     ptp.trigger_update()
     yield gen.sleep(0.200)
     ptp.trigger_update()
-    assert 2 < len(ptp.ts_source.data['time']) < 20
+    assert 2 < len(ptp.ts_source.data["time"]) < 20
 
 
-@gen_cluster(client=True,
-             scheduler_kwargs={'services': {('bokeh', 0): BokehScheduler}})
+@gen_cluster(client=True, scheduler_kwargs={"services": {("bokeh", 0): BokehScheduler}})
 def test_root_redirect(c, s, a, b):
     http_client = AsyncHTTPClient()
-    response = yield http_client.fetch('http://localhost:%d/' % s.services['bokeh'].port)
+    response = yield http_client.fetch(
+        "http://localhost:%d/" % s.services["bokeh"].port
+    )
     assert response.code == 200
     assert "/status" in response.effective_url
diff --git a/distributed/bokeh/tests/test_scheduler_bokeh_html.py b/distributed/bokeh/tests/test_scheduler_bokeh_html.py
index 96dc71c6e67..d5ca1ee7f05 100644
--- a/distributed/bokeh/tests/test_scheduler_bokeh_html.py
+++ b/distributed/bokeh/tests/test_scheduler_bokeh_html.py
@@ -5,7 +5,8 @@
 import xml.etree.ElementTree
 
 import pytest
-pytest.importorskip('bokeh')
+
+pytest.importorskip("bokeh")
 
 from tornado.escape import url_escape
 from tornado.httpclient import AsyncHTTPClient
@@ -14,59 +15,62 @@
 from distributed.bokeh.scheduler import BokehScheduler
 
 
-@gen_cluster(client=True,
-             scheduler_kwargs={'services': {('bokeh', 0):  BokehScheduler}})
+@gen_cluster(client=True, scheduler_kwargs={"services": {("bokeh", 0): BokehScheduler}})
 def test_connect(c, s, a, b):
     future = c.submit(lambda x: x + 1, 1)
     x = c.submit(slowinc, 1, delay=1, retries=5)
     yield future
     http_client = AsyncHTTPClient()
-    for suffix in ['info/main/workers.html',
-                   'info/worker/' + url_escape(a.address) + '.html',
-                   'info/task/' + url_escape(future.key) + '.html',
-                   'info/main/logs.html',
-                   'info/logs/' + url_escape(a.address) + '.html',
-                   'info/call-stack/' + url_escape(x.key) + '.html',
-                   'info/call-stacks/' + url_escape(a.address) + '.html',
-                   'json/counts.json',
-                   'json/identity.json',
-                   'json/index.html',
-                   'individual-plots.json',
-                   ]:
-        response = yield http_client.fetch('http://localhost:%d/%s'
-                                           % (s.services['bokeh'].port, suffix))
+    for suffix in [
+        "info/main/workers.html",
+        "info/worker/" + url_escape(a.address) + ".html",
+        "info/task/" + url_escape(future.key) + ".html",
+        "info/main/logs.html",
+        "info/logs/" + url_escape(a.address) + ".html",
+        "info/call-stack/" + url_escape(x.key) + ".html",
+        "info/call-stacks/" + url_escape(a.address) + ".html",
+        "json/counts.json",
+        "json/identity.json",
+        "json/index.html",
+        "individual-plots.json",
+    ]:
+        response = yield http_client.fetch(
+            "http://localhost:%d/%s" % (s.services["bokeh"].port, suffix)
+        )
         assert response.code == 200
         body = response.body.decode()
-        if suffix.endswith('.json'):
+        if suffix.endswith(".json"):
             json.loads(body)
         else:
             assert xml.etree.ElementTree.fromstring(body) is not None
             assert not re.search("href=./", body)  # no absolute links
 
 
-@gen_cluster(client=True,
-             scheduler_kwargs={'services': {('bokeh', 0):  (BokehScheduler,
-                 {'prefix': '/foo'})}})
+@gen_cluster(
+    client=True,
+    scheduler_kwargs={"services": {("bokeh", 0): (BokehScheduler, {"prefix": "/foo"})}},
+)
 def test_prefix(c, s, a, b):
     http_client = AsyncHTTPClient()
-    for suffix in ['foo/info/main/workers.html',
-                   'foo/json/index.html',
-                   'foo/system']:
-        response = yield http_client.fetch('http://localhost:%d/%s'
-                                           % (s.services['bokeh'].port, suffix))
+    for suffix in ["foo/info/main/workers.html", "foo/json/index.html", "foo/system"]:
+        response = yield http_client.fetch(
+            "http://localhost:%d/%s" % (s.services["bokeh"].port, suffix)
+        )
         assert response.code == 200
         body = response.body.decode()
-        if suffix.endswith('.json'):
+        if suffix.endswith(".json"):
             json.loads(body)
         else:
             assert xml.etree.ElementTree.fromstring(body) is not None
 
 
-@gen_cluster(client=True,
-             check_new_threads=False,
-             scheduler_kwargs={'services': {('bokeh', 0):  BokehScheduler}})
+@gen_cluster(
+    client=True,
+    check_new_threads=False,
+    scheduler_kwargs={"services": {("bokeh", 0): BokehScheduler}},
+)
 def test_prometheus(c, s, a, b):
-    pytest.importorskip('prometheus_client')
+    pytest.importorskip("prometheus_client")
     from prometheus_client.parser import text_string_to_metric_families
 
     http_client = AsyncHTTPClient()
@@ -74,29 +78,30 @@ def test_prometheus(c, s, a, b):
     # request data twice since there once was a case where metrics got registered multiple times resulting in
     # prometheus_client errors
     for _ in range(2):
-        response = yield http_client.fetch('http://localhost:%d/metrics'
-                                           % s.services['bokeh'].port)
+        response = yield http_client.fetch(
+            "http://localhost:%d/metrics" % s.services["bokeh"].port
+        )
         assert response.code == 200
-        assert response.headers['Content-Type'] == 'text/plain; version=0.0.4'
+        assert response.headers["Content-Type"] == "text/plain; version=0.0.4"
 
-        txt = response.body.decode('utf8')
-        families = {
-            familiy.name
-            for familiy in text_string_to_metric_families(txt)
-        }
-        assert 'dask_scheduler_workers' in families
+        txt = response.body.decode("utf8")
+        families = {familiy.name for familiy in text_string_to_metric_families(txt)}
+        assert "dask_scheduler_workers" in families
 
 
-@gen_cluster(client=True,
+@gen_cluster(
+    client=True,
     check_new_threads=False,
-    scheduler_kwargs={'services': {('bokeh', 0):  BokehScheduler}})
+    scheduler_kwargs={"services": {("bokeh", 0): BokehScheduler}},
+)
 def test_health(c, s, a, b):
     http_client = AsyncHTTPClient()
 
-    response = yield http_client.fetch('http://localhost:%d/health'
-                                       % s.services['bokeh'].port)
+    response = yield http_client.fetch(
+        "http://localhost:%d/health" % s.services["bokeh"].port
+    )
     assert response.code == 200
-    assert response.headers['Content-Type'] == 'text/plain'
+    assert response.headers["Content-Type"] == "text/plain"
 
-    txt = response.body.decode('utf8')
-    assert txt == 'ok'
+    txt = response.body.decode("utf8")
+    assert txt == "ok"
diff --git a/distributed/bokeh/tests/test_worker_bokeh.py b/distributed/bokeh/tests/test_worker_bokeh.py
index 32c14d4fa50..03a7ed3861b 100644
--- a/distributed/bokeh/tests/test_worker_bokeh.py
+++ b/distributed/bokeh/tests/test_worker_bokeh.py
@@ -4,7 +4,8 @@
 from time import sleep
 
 import pytest
-pytest.importorskip('bokeh')
+
+pytest.importorskip("bokeh")
 import sys
 from toolz import first
 from tornado import gen
@@ -13,40 +14,52 @@
 from distributed.client import wait
 from distributed.metrics import time
 from distributed.utils_test import gen_cluster, inc, dec
-from distributed.bokeh.worker import (BokehWorker, StateTable, CrossFilter,
-                                      CommunicatingStream, ExecutingTimeSeries, CommunicatingTimeSeries,
-                                      SystemMonitor, Counters)
-
-
-@pytest.mark.skipif(sys.version_info[0] == 2,
-                    reason='https://github.com/bokeh/bokeh/issues/5494')
-@gen_cluster(client=True,
-             worker_kwargs={'services': {('bokeh', 0):  BokehWorker}})
+from distributed.bokeh.worker import (
+    BokehWorker,
+    StateTable,
+    CrossFilter,
+    CommunicatingStream,
+    ExecutingTimeSeries,
+    CommunicatingTimeSeries,
+    SystemMonitor,
+    Counters,
+)
+
+
+@pytest.mark.skipif(
+    sys.version_info[0] == 2, reason="https://github.com/bokeh/bokeh/issues/5494"
+)
+@gen_cluster(client=True, worker_kwargs={"services": {("bokeh", 0): BokehWorker}})
 def test_simple(c, s, a, b):
-    assert s.workers[a.address].services == {'bokeh': a.services['bokeh'].port}
-    assert s.workers[b.address].services == {'bokeh': b.services['bokeh'].port}
+    assert s.workers[a.address].services == {"bokeh": a.services["bokeh"].port}
+    assert s.workers[b.address].services == {"bokeh": b.services["bokeh"].port}
 
     future = c.submit(sleep, 1)
     yield gen.sleep(0.1)
 
     http_client = AsyncHTTPClient()
-    for suffix in ['main', 'crossfilter', 'system']:
-        response = yield http_client.fetch('http://localhost:%d/%s'
-                                           % (a.services['bokeh'].port, suffix))
-        assert 'bokeh' in response.body.decode().lower()
+    for suffix in ["main", "crossfilter", "system"]:
+        response = yield http_client.fetch(
+            "http://localhost:%d/%s" % (a.services["bokeh"].port, suffix)
+        )
+        assert "bokeh" in response.body.decode().lower()
 
 
-@gen_cluster(client=True,
-             worker_kwargs={'services': {('bokeh', 0):  (BokehWorker, {})}})
+@gen_cluster(client=True, worker_kwargs={"services": {("bokeh", 0): (BokehWorker, {})}})
 def test_services_kwargs(c, s, a, b):
-    assert s.workers[a.address].services == {'bokeh': a.services['bokeh'].port}
-    assert isinstance(a.services['bokeh'], BokehWorker)
+    assert s.workers[a.address].services == {"bokeh": a.services["bokeh"].port}
+    assert isinstance(a.services["bokeh"], BokehWorker)
 
 
 @gen_cluster(client=True)
 def test_basic(c, s, a, b):
-    for component in [StateTable, ExecutingTimeSeries,
-                      CommunicatingTimeSeries, CrossFilter, SystemMonitor]:
+    for component in [
+        StateTable,
+        ExecutingTimeSeries,
+        CommunicatingTimeSeries,
+        CrossFilter,
+        SystemMonitor,
+    ]:
 
         aa = component(a)
         bb = component(b)
@@ -64,14 +77,15 @@ def slowall(*args):
         aa.update()
         bb.update()
 
-        assert (len(first(aa.source.data.values())) and
-                len(first(bb.source.data.values())))
+        assert len(first(aa.source.data.values())) and len(
+            first(bb.source.data.values())
+        )
 
 
 @gen_cluster(client=True)
 def test_counters(c, s, a, b):
-    pytest.importorskip('crick')
-    while 'tick-duration' not in a.digests:
+    pytest.importorskip("crick")
+    while "tick-duration" not in a.digests:
         yield gen.sleep(0.01)
     aa = Counters(a)
 
@@ -80,18 +94,18 @@ def test_counters(c, s, a, b):
     aa.update()
 
     start = time()
-    while not len(aa.digest_sources['tick-duration'][0].data['x']):
+    while not len(aa.digest_sources["tick-duration"][0].data["x"]):
         yield gen.sleep(1)
         assert time() < start + 5
 
-    a.digests['foo'].add(1)
-    a.digests['foo'].add(2)
-    aa.add_digest_figure('foo')
+    a.digests["foo"].add(1)
+    a.digests["foo"].add(2)
+    aa.add_digest_figure("foo")
 
-    a.counters['bar'].add(1)
-    a.counters['bar'].add(2)
-    a.counters['bar'].add(2)
-    aa.add_counter_figure('bar')
+    a.counters["bar"].add(1)
+    a.counters["bar"].add(2)
+    a.counters["bar"].add(2)
+    aa.add_counter_figure("bar")
 
     for x in [aa.counter_sources.values(), aa.digest_sources.values()]:
         for y in x:
@@ -114,21 +128,26 @@ def test_CommunicatingStream(c, s, a, b):
     aa.update()
     bb.update()
 
-    assert (len(first(aa.outgoing.data.values())) and
-            len(first(bb.outgoing.data.values())))
-    assert (len(first(aa.incoming.data.values())) and
-            len(first(bb.incoming.data.values())))
+    assert len(first(aa.outgoing.data.values())) and len(
+        first(bb.outgoing.data.values())
+    )
+    assert len(first(aa.incoming.data.values())) and len(
+        first(bb.incoming.data.values())
+    )
 
 
-@gen_cluster(client=True,
-             check_new_threads=False,
-             worker_kwargs={'services': {('bokeh', 0):  BokehWorker}})
+@gen_cluster(
+    client=True,
+    check_new_threads=False,
+    worker_kwargs={"services": {("bokeh", 0): BokehWorker}},
+)
 def test_prometheus(c, s, a, b):
-    pytest.importorskip('prometheus_client')
-    assert s.workers[a.address].services == {'bokeh': a.services['bokeh'].port}
+    pytest.importorskip("prometheus_client")
+    assert s.workers[a.address].services == {"bokeh": a.services["bokeh"].port}
 
     http_client = AsyncHTTPClient()
-    for suffix in ['metrics']:
-        response = yield http_client.fetch('http://localhost:%d/%s'
-                                           % (a.services['bokeh'].port, suffix))
+    for suffix in ["metrics"]:
+        response = yield http_client.fetch(
+            "http://localhost:%d/%s" % (a.services["bokeh"].port, suffix)
+        )
         assert response.code == 200
diff --git a/distributed/bokeh/tests/test_worker_bokeh_html.py b/distributed/bokeh/tests/test_worker_bokeh_html.py
index 80819972050..d59fec8d2d8 100644
--- a/distributed/bokeh/tests/test_worker_bokeh_html.py
+++ b/distributed/bokeh/tests/test_worker_bokeh_html.py
@@ -1,15 +1,15 @@
 import pytest
-pytest.importorskip('bokeh')
+
+pytest.importorskip("bokeh")
 
 from tornado.httpclient import AsyncHTTPClient
 from distributed.utils_test import gen_cluster
 from distributed.bokeh.worker import BokehWorker
 
 
-@gen_cluster(client=True,
-             worker_kwargs={'services': {('bokeh', 0):  BokehWorker}})
+@gen_cluster(client=True, worker_kwargs={"services": {("bokeh", 0): BokehWorker}})
 def test_prometheus(c, s, a, b):
-    pytest.importorskip('prometheus_client')
+    pytest.importorskip("prometheus_client")
     from prometheus_client.parser import text_string_to_metric_families
 
     http_client = AsyncHTTPClient()
@@ -17,28 +17,26 @@ def test_prometheus(c, s, a, b):
     # request data twice since there once was a case where metrics got registered multiple times resulting in
     # prometheus_client errors
     for _ in range(2):
-        response = yield http_client.fetch('http://localhost:%d/metrics'
-                                           % a.services['bokeh'].port)
+        response = yield http_client.fetch(
+            "http://localhost:%d/metrics" % a.services["bokeh"].port
+        )
         assert response.code == 200
-        assert response.headers['Content-Type'] == 'text/plain; version=0.0.4'
+        assert response.headers["Content-Type"] == "text/plain; version=0.0.4"
 
-        txt = response.body.decode('utf8')
-        families = {
-            familiy.name
-            for familiy in text_string_to_metric_families(txt)
-        }
+        txt = response.body.decode("utf8")
+        families = {familiy.name for familiy in text_string_to_metric_families(txt)}
         assert len(families) > 0
 
 
-@gen_cluster(client=True,
-    worker_kwargs={'services': {('bokeh', 0):  BokehWorker}})
+@gen_cluster(client=True, worker_kwargs={"services": {("bokeh", 0): BokehWorker}})
 def test_health(c, s, a, b):
     http_client = AsyncHTTPClient()
 
-    response = yield http_client.fetch('http://localhost:%d/health'
-                                       % a.services['bokeh'].port)
+    response = yield http_client.fetch(
+        "http://localhost:%d/health" % a.services["bokeh"].port
+    )
     assert response.code == 200
-    assert response.headers['Content-Type'] == 'text/plain'
+    assert response.headers["Content-Type"] == "text/plain"
 
-    txt = response.body.decode('utf8')
-    assert txt == 'ok'
+    txt = response.body.decode("utf8")
+    assert txt == "ok"
diff --git a/distributed/bokeh/utils.py b/distributed/bokeh/utils.py
index 3bfada9402e..516ca5bfb88 100644
--- a/distributed/bokeh/utils.py
+++ b/distributed/bokeh/utils.py
@@ -10,11 +10,12 @@
 BOKEH_VERSION = LooseVersion(bokeh.__version__)
 
 
-if BOKEH_VERSION >= '1.0.0' and not PY2:
+if BOKEH_VERSION >= "1.0.0" and not PY2:
     # This decorator is only available in bokeh >= 1.0.0, and doesn't work for
     # callbacks in Python 2, since the signature introspection won't line up.
     from bokeh.core.properties import without_property_validation
 else:
+
     def without_property_validation(f):
         return f
 
diff --git a/distributed/bokeh/worker.py b/distributed/bokeh/worker.py
index ef6c27e0404..c7ced4d90fc 100644
--- a/distributed/bokeh/worker.py
+++ b/distributed/bokeh/worker.py
@@ -6,9 +6,17 @@
 import os
 
 from bokeh.layouts import row, column, widgetbox
-from bokeh.models import (ColumnDataSource, DataRange1d, HoverTool,
-                          BoxZoomTool, ResetTool, PanTool, WheelZoomTool, NumeralTickFormatter,
-                          Select)
+from bokeh.models import (
+    ColumnDataSource,
+    DataRange1d,
+    HoverTool,
+    BoxZoomTool,
+    ResetTool,
+    PanTool,
+    WheelZoomTool,
+    NumeralTickFormatter,
+    Select,
+)
 
 from bokeh.models.widgets import DataTable, TableColumn
 from bokeh.plotting import figure
@@ -16,27 +24,34 @@
 from bokeh.themes import Theme
 from toolz import merge, partition_all
 
-from .components import (DashboardComponent, ProfileTimePlot, ProfileServer,
-                         add_periodic_callback)
+from .components import (
+    DashboardComponent,
+    ProfileTimePlot,
+    ProfileServer,
+    add_periodic_callback,
+)
 from .core import BokehServer
 from .utils import transpose, without_property_validation
 from ..compatibility import WINDOWS
 from ..diagnostics.progress_stream import color_of
 from ..metrics import time
-from ..utils import (log_errors, key_split, format_bytes, format_time)
+from ..utils import log_errors, key_split, format_bytes, format_time
 
 
 logger = logging.getLogger(__name__)
 
-with open(os.path.join(os.path.dirname(__file__), 'templates', 'base.html')) as f:
+with open(os.path.join(os.path.dirname(__file__), "templates", "base.html")) as f:
     template_source = f.read()
 
 from jinja2 import Environment, FileSystemLoader
-env = Environment(loader=FileSystemLoader(os.path.join(os.path.dirname(__file__), 'templates')))
 
-BOKEH_THEME = Theme(os.path.join(os.path.dirname(__file__), 'theme.yaml'))
+env = Environment(
+    loader=FileSystemLoader(os.path.join(os.path.dirname(__file__), "templates"))
+)
 
-template_variables = {'pages': ['main', 'system', 'profile', 'crossfilter']}
+BOKEH_THEME = Theme(os.path.join(os.path.dirname(__file__), "theme.yaml"))
+
+template_variables = {"pages": ["main", "system", "profile", "crossfilter"]}
 
 
 class StateTable(DashboardComponent):
@@ -45,15 +60,13 @@ class StateTable(DashboardComponent):
     def __init__(self, worker):
         self.worker = worker
 
-        names = ['Stored', 'Executing', 'Ready', 'Waiting', 'Connections', 'Serving']
+        names = ["Stored", "Executing", "Ready", "Waiting", "Connections", "Serving"]
         self.source = ColumnDataSource({name: [] for name in names})
 
-        columns = {name: TableColumn(field=name, title=name)
-                   for name in names}
+        columns = {name: TableColumn(field=name, title=name) for name in names}
 
         table = DataTable(
-            source=self.source, columns=[columns[n] for n in names],
-            height=70,
+            source=self.source, columns=[columns[n] for n in names], height=70
         )
         self.root = table
 
@@ -61,12 +74,14 @@ def __init__(self, worker):
     def update(self):
         with log_errors():
             w = self.worker
-            d = {'Stored': [len(w.data)],
-                 'Executing': ['%d / %d' % (len(w.executing), w.ncores)],
-                 'Ready': [len(w.ready)],
-                 'Waiting': [len(w.waiting_for_data)],
-                 'Connections': [len(w.in_flight_workers)],
-                 'Serving': [len(w._comms)]}
+            d = {
+                "Stored": [len(w.data)],
+                "Executing": ["%d / %d" % (len(w.executing), w.ncores)],
+                "Ready": [len(w.ready)],
+                "Waiting": [len(w.waiting_for_data)],
+                "Connections": [len(w.in_flight_workers)],
+                "Serving": [len(w._comms)],
+            }
             self.source.data.update(d)
 
 
@@ -74,8 +89,18 @@ class CommunicatingStream(DashboardComponent):
     def __init__(self, worker, height=300, **kwargs):
         with log_errors():
             self.worker = worker
-            names = ['start', 'stop', 'middle', 'duration', 'who', 'y',
-                     'hover', 'alpha', 'bandwidth', 'total']
+            names = [
+                "start",
+                "stop",
+                "middle",
+                "duration",
+                "who",
+                "y",
+                "hover",
+                "alpha",
+                "bandwidth",
+                "total",
+            ]
 
             self.incoming = ColumnDataSource({name: [] for name in names})
             self.outgoing = ColumnDataSource({name: [] for name in names})
@@ -83,24 +108,41 @@ def __init__(self, worker, height=300, **kwargs):
             x_range = DataRange1d(range_padding=0)
             y_range = DataRange1d(range_padding=0)
 
-            fig = figure(title='Peer Communications',
-                         x_axis_type='datetime', x_range=x_range, y_range=y_range,
-                         height=height, tools='', **kwargs)
-
-            fig.rect(source=self.incoming, x='middle', y='y', width='duration',
-                     height=0.9, color='red', alpha='alpha')
-            fig.rect(source=self.outgoing, x='middle', y='y', width='duration',
-                     height=0.9, color='blue', alpha='alpha')
+            fig = figure(
+                title="Peer Communications",
+                x_axis_type="datetime",
+                x_range=x_range,
+                y_range=y_range,
+                height=height,
+                tools="",
+                **kwargs
+            )
 
-            hover = HoverTool(
-                point_policy="follow_mouse",
-                tooltips="""@hover"""
+            fig.rect(
+                source=self.incoming,
+                x="middle",
+                y="y",
+                width="duration",
+                height=0.9,
+                color="red",
+                alpha="alpha",
+            )
+            fig.rect(
+                source=self.outgoing,
+                x="middle",
+                y="y",
+                width="duration",
+                height=0.9,
+                color="blue",
+                alpha="alpha",
             )
+
+            hover = HoverTool(point_policy="follow_mouse", tooltips="""@hover""")
             fig.add_tools(
                 hover,
                 ResetTool(),
                 PanTool(dimensions="width"),
-                WheelZoomTool(dimensions="width")
+                WheelZoomTool(dimensions="width"),
             )
 
             self.root = fig
@@ -122,35 +164,40 @@ def update(self):
             incoming = [incoming[-i].copy() for i in range(1, n + 1)]
             self.last_incoming = self.worker.incoming_count
 
-            for [msgs, source] in [[incoming, self.incoming],
-                                   [outgoing, self.outgoing]]:
+            for [msgs, source] in [
+                [incoming, self.incoming],
+                [outgoing, self.outgoing],
+            ]:
 
                 for msg in msgs:
-                    if 'compressed' in msg:
-                        del msg['compressed']
-                    del msg['keys']
+                    if "compressed" in msg:
+                        del msg["compressed"]
+                    del msg["keys"]
 
-                    bandwidth = msg['total'] / (msg['duration'] or 0.5)
+                    bandwidth = msg["total"] / (msg["duration"] or 0.5)
                     bw = max(min(bandwidth / 500e6, 1), 0.3)
-                    msg['alpha'] = bw
+                    msg["alpha"] = bw
                     try:
-                        msg['y'] = self.who[msg['who']]
+                        msg["y"] = self.who[msg["who"]]
                     except KeyError:
-                        self.who[msg['who']] = len(self.who)
-                        msg['y'] = self.who[msg['who']]
+                        self.who[msg["who"]] = len(self.who)
+                        msg["y"] = self.who[msg["who"]]
 
-                    msg['hover'] = '%s / %s = %s/s' % (
-                        format_bytes(msg['total']),
-                        format_time(msg['duration']),
-                        format_bytes(msg['total'] / msg['duration']))
+                    msg["hover"] = "%s / %s = %s/s" % (
+                        format_bytes(msg["total"]),
+                        format_time(msg["duration"]),
+                        format_bytes(msg["total"] / msg["duration"]),
+                    )
 
-                    for k in ['middle', 'duration', 'start', 'stop']:
+                    for k in ["middle", "duration", "start", "stop"]:
                         msg[k] = msg[k] * 1000
 
                 if msgs:
                     msgs = transpose(msgs)
-                    if (len(source.data['stop']) and
-                            min(msgs['start']) > source.data['stop'][-1] + 10000):
+                    if (
+                        len(source.data["stop"])
+                        and min(msgs["start"]) > source.data["stop"][-1] + 10000
+                    ):
                         source.data.update(msgs)
                     else:
                         source.stream(msgs, rollover=10000)
@@ -159,21 +206,24 @@ def update(self):
 class CommunicatingTimeSeries(DashboardComponent):
     def __init__(self, worker, **kwargs):
         self.worker = worker
-        self.source = ColumnDataSource({'x': [], 'in': [], 'out': []})
-
-        x_range = DataRange1d(follow='end', follow_interval=20000, range_padding=0)
-
-        fig = figure(title="Communication History",
-                     x_axis_type='datetime',
-                     y_range=[-0.1, worker.total_out_connections + 0.5],
-                     height=150, tools='', x_range=x_range, **kwargs)
-        fig.line(source=self.source, x='x', y='in', color='red')
-        fig.line(source=self.source, x='x', y='out', color='blue')
+        self.source = ColumnDataSource({"x": [], "in": [], "out": []})
+
+        x_range = DataRange1d(follow="end", follow_interval=20000, range_padding=0)
+
+        fig = figure(
+            title="Communication History",
+            x_axis_type="datetime",
+            y_range=[-0.1, worker.total_out_connections + 0.5],
+            height=150,
+            tools="",
+            x_range=x_range,
+            **kwargs
+        )
+        fig.line(source=self.source, x="x", y="in", color="red")
+        fig.line(source=self.source, x="x", y="out", color="blue")
 
         fig.add_tools(
-            ResetTool(),
-            PanTool(dimensions="width"),
-            WheelZoomTool(dimensions="width")
+            ResetTool(), PanTool(dimensions="width"), WheelZoomTool(dimensions="width")
         )
 
         self.root = fig
@@ -181,28 +231,36 @@ def __init__(self, worker, **kwargs):
     @without_property_validation
     def update(self):
         with log_errors():
-            self.source.stream({'x': [time() * 1000],
-                                'out': [len(self.worker._comms)],
-                                'in': [len(self.worker.in_flight_workers)]},
-                               10000)
+            self.source.stream(
+                {
+                    "x": [time() * 1000],
+                    "out": [len(self.worker._comms)],
+                    "in": [len(self.worker.in_flight_workers)],
+                },
+                10000,
+            )
 
 
 class ExecutingTimeSeries(DashboardComponent):
     def __init__(self, worker, **kwargs):
         self.worker = worker
-        self.source = ColumnDataSource({'x': [], 'y': []})
-
-        x_range = DataRange1d(follow='end', follow_interval=20000, range_padding=0)
-
-        fig = figure(title="Executing History",
-                     x_axis_type='datetime', y_range=[-0.1, worker.ncores + 0.1],
-                     height=150, tools='', x_range=x_range, **kwargs)
-        fig.line(source=self.source, x='x', y='y')
+        self.source = ColumnDataSource({"x": [], "y": []})
+
+        x_range = DataRange1d(follow="end", follow_interval=20000, range_padding=0)
+
+        fig = figure(
+            title="Executing History",
+            x_axis_type="datetime",
+            y_range=[-0.1, worker.ncores + 0.1],
+            height=150,
+            tools="",
+            x_range=x_range,
+            **kwargs
+        )
+        fig.line(source=self.source, x="x", y="y")
 
         fig.add_tools(
-            ResetTool(),
-            PanTool(dimensions="width"),
-            WheelZoomTool(dimensions="width")
+            ResetTool(), PanTool(dimensions="width"), WheelZoomTool(dimensions="width")
         )
 
         self.root = fig
@@ -210,8 +268,9 @@ def __init__(self, worker, **kwargs):
     @without_property_validation
     def update(self):
         with log_errors():
-            self.source.stream({'x': [time() * 1000],
-                                'y': [len(self.worker.executing)]}, 1000)
+            self.source.stream(
+                {"x": [time() * 1000], "y": [len(self.worker.executing)]}, 1000
+            )
 
 
 class CrossFilter(DashboardComponent):
@@ -219,52 +278,55 @@ def __init__(self, worker, **kwargs):
         with log_errors():
             self.worker = worker
 
-            quantities = ['nbytes', 'duration', 'bandwidth', 'count',
-                          'start', 'stop']
-            colors = ['inout-color', 'type-color', 'key-color']
+            quantities = ["nbytes", "duration", "bandwidth", "count", "start", "stop"]
+            colors = ["inout-color", "type-color", "key-color"]
 
             # self.source = ColumnDataSource({name: [] for name in names})
-            self.source = ColumnDataSource({
-                'nbytes': [1, 2],
-                'duration': [0.01, 0.02],
-                'bandwidth': [0.01, 0.02],
-                'count': [1, 2],
-                'type': ['int', 'str'],
-                'inout-color': ['blue', 'red'],
-                'type-color': ['blue', 'red'],
-                'key': ['add', 'inc'],
-                'start': [1, 2],
-                'stop': [1, 2]
-            })
-
-            self.x = Select(title='X-Axis', value='nbytes', options=quantities)
-            self.x.on_change('value', self.update_figure)
-
-            self.y = Select(title='Y-Axis', value='bandwidth', options=quantities)
-            self.y.on_change('value', self.update_figure)
-
-            self.size = Select(title='Size', value='None',
-                               options=['None'] + quantities)
-            self.size.on_change('value', self.update_figure)
-
-            self.color = Select(title='Color', value='inout-color',
-                                options=['black'] + colors)
-            self.color.on_change('value', self.update_figure)
-
-            if 'sizing_mode' in kwargs:
-                kw = {'sizing_mode': kwargs['sizing_mode']}
+            self.source = ColumnDataSource(
+                {
+                    "nbytes": [1, 2],
+                    "duration": [0.01, 0.02],
+                    "bandwidth": [0.01, 0.02],
+                    "count": [1, 2],
+                    "type": ["int", "str"],
+                    "inout-color": ["blue", "red"],
+                    "type-color": ["blue", "red"],
+                    "key": ["add", "inc"],
+                    "start": [1, 2],
+                    "stop": [1, 2],
+                }
+            )
+
+            self.x = Select(title="X-Axis", value="nbytes", options=quantities)
+            self.x.on_change("value", self.update_figure)
+
+            self.y = Select(title="Y-Axis", value="bandwidth", options=quantities)
+            self.y.on_change("value", self.update_figure)
+
+            self.size = Select(
+                title="Size", value="None", options=["None"] + quantities
+            )
+            self.size.on_change("value", self.update_figure)
+
+            self.color = Select(
+                title="Color", value="inout-color", options=["black"] + colors
+            )
+            self.color.on_change("value", self.update_figure)
+
+            if "sizing_mode" in kwargs:
+                kw = {"sizing_mode": kwargs["sizing_mode"]}
             else:
                 kw = {}
 
-            self.control = widgetbox([self.x, self.y, self.size, self.color],
-                                     width=200, **kw)
+            self.control = widgetbox(
+                [self.x, self.y, self.size, self.color], width=200, **kw
+            )
 
             self.last_outgoing = 0
             self.last_incoming = 0
             self.kwargs = kwargs
 
-            self.layout = row(self.control, self.create_figure(**self.kwargs),
-                              **kw)
+            self.layout = row(self.control, self.create_figure(**self.kwargs), **kw)
 
             self.root = self.layout
 
@@ -286,36 +348,44 @@ def update(self):
             out = []
 
             for msg in incoming:
-                if msg['keys']:
+                if msg["keys"]:
                     d = self.process_msg(msg)
-                    d['inout-color'] = 'red'
+                    d["inout-color"] = "red"
                     out.append(d)
 
             for msg in outgoing:
-                if msg['keys']:
+                if msg["keys"]:
                     d = self.process_msg(msg)
-                    d['inout-color'] = 'blue'
+                    d["inout-color"] = "blue"
                     out.append(d)
 
             if out:
                 out = transpose(out)
-                if (len(self.source.data['stop']) and
-                        min(out['start']) > self.source.data['stop'][-1] + 10):
+                if (
+                    len(self.source.data["stop"])
+                    and min(out["start"]) > self.source.data["stop"][-1] + 10
+                ):
                     self.source.data.update(out)
                 else:
                     self.source.stream(out, rollover=1000)
 
     def create_figure(self, **kwargs):
         with log_errors():
-            fig = figure(title='', tools='', **kwargs)
+            fig = figure(title="", tools="", **kwargs)
 
             size = self.size.value
-            if size == 'None':
+            if size == "None":
                 size = 1
 
-            fig.circle(source=self.source, x=self.x.value, y=self.y.value,
-                       color=self.color.value, size=10, alpha=0.5,
-                       hover_alpha=1)
+            fig.circle(
+                source=self.source,
+                x=self.x.value,
+                y=self.y.value,
+                color=self.color.value,
+                size=10,
+                alpha=0.5,
+                hover_alpha=1,
+            )
             fig.xaxis.axis_label = self.x.value
             fig.yaxis.axis_label = self.y.value
 
@@ -336,22 +406,24 @@ def update_figure(self, attr, old, new):
 
     def process_msg(self, msg):
         try:
+
             def func(k):
-                return msg['keys'].get(k, 0)
-            main_key = max(msg['keys'], key=func)
+                return msg["keys"].get(k, 0)
+
+            main_key = max(msg["keys"], key=func)
             typ = self.worker.types.get(main_key, object).__name__
             keyname = key_split(main_key)
             d = {
-                'nbytes': msg['total'],
-                'duration': msg['duration'],
-                'bandwidth': msg['bandwidth'],
-                'count': len(msg['keys']),
-                'type': typ,
-                'type-color': color_of(typ),
-                'key': keyname,
-                'key-color': color_of(keyname),
-                'start': msg['start'],
-                'stop': msg['stop']
+                "nbytes": msg["total"],
+                "duration": msg["duration"],
+                "bandwidth": msg["bandwidth"],
+                "count": len(msg["keys"]),
+                "type": typ,
+                "type-color": color_of(typ),
+                "key": keyname,
+                "key-color": color_of(keyname),
+                "start": msg["start"],
+                "stop": msg["stop"],
             }
             return d
         except Exception as e:
@@ -368,44 +440,63 @@ def __init__(self, worker, height=150, **kwargs):
         self.source = ColumnDataSource({name: [] for name in names})
         self.source.data.update(self.get_data())
 
-        x_range = DataRange1d(follow='end', follow_interval=20000,
-                              range_padding=0)
-
-        tools = 'reset,xpan,xwheel_zoom'
-
-        self.cpu = figure(title="CPU", x_axis_type='datetime',
-                          height=height, tools=tools, x_range=x_range, **kwargs)
-        self.cpu.line(source=self.source, x='time', y='cpu')
-        self.cpu.yaxis.axis_label = 'Percentage'
-        self.mem = figure(title="Memory", x_axis_type='datetime',
-                          height=height, tools=tools, x_range=x_range, **kwargs)
-        self.mem.line(source=self.source, x='time', y='memory')
-        self.mem.yaxis.axis_label = 'Bytes'
-        self.bandwidth = figure(title='Bandwidth', x_axis_type='datetime',
-                                height=height,
-                                x_range=x_range, tools=tools, **kwargs)
-        self.bandwidth.line(source=self.source, x='time', y='read_bytes',
-                            color='red')
-        self.bandwidth.line(source=self.source, x='time', y='write_bytes',
-                            color='blue')
-        self.bandwidth.yaxis.axis_label = 'Bytes / second'
+        x_range = DataRange1d(follow="end", follow_interval=20000, range_padding=0)
+
+        tools = "reset,xpan,xwheel_zoom"
+
+        self.cpu = figure(
+            title="CPU",
+            x_axis_type="datetime",
+            height=height,
+            tools=tools,
+            x_range=x_range,
+            **kwargs
+        )
+        self.cpu.line(source=self.source, x="time", y="cpu")
+        self.cpu.yaxis.axis_label = "Percentage"
+        self.mem = figure(
+            title="Memory",
+            x_axis_type="datetime",
+            height=height,
+            tools=tools,
+            x_range=x_range,
+            **kwargs
+        )
+        self.mem.line(source=self.source, x="time", y="memory")
+        self.mem.yaxis.axis_label = "Bytes"
+        self.bandwidth = figure(
+            title="Bandwidth",
+            x_axis_type="datetime",
+            height=height,
+            x_range=x_range,
+            tools=tools,
+            **kwargs
+        )
+        self.bandwidth.line(source=self.source, x="time", y="read_bytes", color="red")
+        self.bandwidth.line(source=self.source, x="time", y="write_bytes", color="blue")
+        self.bandwidth.yaxis.axis_label = "Bytes / second"
 
         # self.cpu.yaxis[0].formatter = NumeralTickFormatter(format='0%')
-        self.bandwidth.yaxis[0].formatter = NumeralTickFormatter(format='0.0b')
-        self.mem.yaxis[0].formatter = NumeralTickFormatter(format='0.0b')
+        self.bandwidth.yaxis[0].formatter = NumeralTickFormatter(format="0.0b")
+        self.mem.yaxis[0].formatter = NumeralTickFormatter(format="0.0b")
 
         plots = [self.cpu, self.mem, self.bandwidth]
 
         if not WINDOWS:
-            self.num_fds = figure(title='Number of File Descriptors',
-                                  x_axis_type='datetime', height=height,
-                                  x_range=x_range, tools=tools, **kwargs)
+            self.num_fds = figure(
+                title="Number of File Descriptors",
+                x_axis_type="datetime",
+                height=height,
+                x_range=x_range,
+                tools=tools,
+                **kwargs
+            )
 
-            self.num_fds.line(source=self.source, x='time', y='num_fds')
+            self.num_fds.line(source=self.source, x="time", y="num_fds")
             plots.append(self.num_fds)
 
-        if 'sizing_mode' in kwargs:
-            kw = {'sizing_mode': kwargs['sizing_mode']}
+        if "sizing_mode" in kwargs:
+            kw = {"sizing_mode": kwargs["sizing_mode"]}
         else:
             kw = {}
 
@@ -420,7 +511,7 @@ def __init__(self, worker, height=150, **kwargs):
 
     def get_data(self):
         d = self.worker.monitor.range_query(start=self.last)
-        d['time'] = [x * 1000 for x in d['time']]
+        d["time"] = [x * 1000 for x in d["time"]]
         self.last = self.worker.monitor.count
         return d
 
@@ -431,7 +522,7 @@ def update(self):
 
 
 class Counters(DashboardComponent):
-    def __init__(self, server, sizing_mode='stretch_both', **kwargs):
+    def __init__(self, server, sizing_mode="stretch_both", **kwargs):
         self.server = server
         self.counter_figures = {}
         self.counter_sources = {}
@@ -451,31 +542,40 @@ def __init__(self, server, sizing_mode='stretch_both', **kwargs):
         if len(figures) <= 5:
             self.root = column(figures, sizing_mode=sizing_mode)
         else:
-            self.root = column(*[row(*pair, sizing_mode=sizing_mode)
-                                 for pair in partition_all(2, figures)],
-                               sizing_mode=sizing_mode)
+            self.root = column(
+                *[
+                    row(*pair, sizing_mode=sizing_mode)
+                    for pair in partition_all(2, figures)
+                ],
+                sizing_mode=sizing_mode
+            )
 
     def add_digest_figure(self, name):
         with log_errors():
             n = len(self.server.digests[name].intervals)
-            sources = {i: ColumnDataSource({'x': [], 'y': []})
-                       for i in range(n)}
+            sources = {i: ColumnDataSource({"x": [], "y": []}) for i in range(n)}
 
             kwargs = {}
-            if name.endswith('duration'):
-                kwargs['x_axis_type'] = 'datetime'
+            if name.endswith("duration"):
+                kwargs["x_axis_type"] = "datetime"
 
-            fig = figure(title=name, tools='', height=150,
-                         sizing_mode=self.sizing_mode, **kwargs)
+            fig = figure(
+                title=name, tools="", height=150, sizing_mode=self.sizing_mode, **kwargs
+            )
             fig.yaxis.visible = False
             fig.ygrid.visible = False
-            if name.endswith('bandwidth') or name.endswith('bytes'):
-                fig.xaxis[0].formatter = NumeralTickFormatter(format='0.0b')
+            if name.endswith("bandwidth") or name.endswith("bytes"):
+                fig.xaxis[0].formatter = NumeralTickFormatter(format="0.0b")
 
             for i in range(n):
                 alpha = 0.3 + 0.3 * (n - i) / n
-                fig.line(source=sources[i], x='x', y='y',
-                         alpha=alpha, color=RdBu[max(n, 3)][-i])
+                fig.line(
+                    source=sources[i],
+                    x="x",
+                    y="y",
+                    alpha=alpha,
+                    color=RdBu[max(n, 3)][-i],
+                )
 
             fig.xaxis.major_label_orientation = math.pi / 12
             fig.toolbar.logo = None
@@ -486,22 +586,33 @@ def add_digest_figure(self, name):
     def add_counter_figure(self, name):
         with log_errors():
             n = len(self.server.counters[name].intervals)
-            sources = {i: ColumnDataSource({'x': [], 'y': [],
-                                            'y-center': [], 'counts': []})
-                       for i in range(n)}
+            sources = {
+                i: ColumnDataSource({"x": [], "y": [], "y-center": [], "counts": []})
+                for i in range(n)
+            }
 
-            fig = figure(title=name, tools='', height=150,
-                         sizing_mode=self.sizing_mode,
-                         x_range=sorted(map(str, self.server.counters[name].components[0])))
+            fig = figure(
+                title=name,
+                tools="",
+                height=150,
+                sizing_mode=self.sizing_mode,
+                x_range=sorted(map(str, self.server.counters[name].components[0])),
+            )
             fig.ygrid.visible = False
 
             for i in range(n):
                 width = 0.5 + 0.4 * i / n
-                fig.rect(source=sources[i], x='x', y='y-center', width=width,
-                         height='y', alpha=0.3, color=RdBu[max(n, 3)][-i])
+                fig.rect(
+                    source=sources[i],
+                    x="x",
+                    y="y-center",
+                    width=width,
+                    height="y",
+                    alpha=0.3,
+                    color=RdBu[max(n, 3)][-i],
+                )
                 hover = HoverTool(
-                    point_policy="follow_mouse",
-                    tooltips="""@x : @counts"""
+                    point_policy="follow_mouse", tooltips="""@x : @counts"""
                 )
                 fig.add_tools(hover)
                 fig.xaxis.major_label_orientation = math.pi / 12
@@ -522,10 +633,10 @@ def update(self):
                     if d.size():
                         ys, xs = d.histogram(100)
                         xs = xs[1:]
-                        if name.endswith('duration'):
+                        if name.endswith("duration"):
                             xs *= 1000
-                        self.digest_sources[name][i].data.update({'x': xs, 'y': ys})
-                fig.title.text = '%s: %d' % (name, digest.size())
+                        self.digest_sources[name][i].data.update({"x": xs, "y": ys})
+                fig.title.text = "%s: %d" % (name, digest.size())
 
             for name, fig in self.counter_figures.items():
                 counter = self.server.counters[name]
@@ -538,10 +649,9 @@ def update(self):
                         ys = [factor * c for c in counts]
                         y_centers = [y / 2 for y in ys]
                         xs = list(map(str, xs))
-                        d = {'x': xs, 'y': ys, 'y-center': y_centers,
-                             'counts': counts}
+                        d = {"x": xs, "y": ys, "y-center": y_centers, "counts": counts}
                         self.counter_sources[name][i].data.update(d)
-                    fig.title.text = '%s: %d' % (name, counter.size())
+                    fig.title.text = "%s: %d" % (name, counter.size())
                     fig.x_range.factors = list(map(str, xs))
 
 
@@ -552,11 +662,9 @@ def update(self):
 def main_doc(worker, extra, doc):
     with log_errors():
         statetable = StateTable(worker)
-        executing_ts = ExecutingTimeSeries(worker, sizing_mode='scale_width')
-        communicating_ts = CommunicatingTimeSeries(worker,
-                                                   sizing_mode='scale_width')
-        communicating_stream = CommunicatingStream(worker,
-                                                   sizing_mode='scale_width')
+        executing_ts = ExecutingTimeSeries(worker, sizing_mode="scale_width")
+        communicating_ts = CommunicatingTimeSeries(worker, sizing_mode="scale_width")
+        communicating_stream = CommunicatingStream(worker, sizing_mode="scale_width")
 
         xr = executing_ts.root.x_range
         communicating_ts.root.x_range = xr
@@ -567,13 +675,17 @@ def main_doc(worker, extra, doc):
         add_periodic_callback(doc, executing_ts, 200)
         add_periodic_callback(doc, communicating_ts, 200)
         add_periodic_callback(doc, communicating_stream, 200)
-        doc.add_root(column(statetable.root,
-                            executing_ts.root,
-                            communicating_ts.root,
-                            communicating_stream.root,
-                            sizing_mode='scale_width'))
-        doc.template = env.get_template('simple.html')
-        doc.template_variables['active_page'] = 'main'
+        doc.add_root(
+            column(
+                statetable.root,
+                executing_ts.root,
+                communicating_ts.root,
+                communicating_stream.root,
+                sizing_mode="scale_width",
+            )
+        )
+        doc.template = env.get_template("simple.html")
+        doc.template_variables["active_page"] = "main"
         doc.template_variables.update(extra)
         doc.theme = BOKEH_THEME
 
@@ -588,21 +700,21 @@ def crossfilter_doc(worker, extra, doc):
         add_periodic_callback(doc, crossfilter, 500)
 
         doc.add_root(column(statetable.root, crossfilter.root))
-        doc.template = env.get_template('simple.html')
-        doc.template_variables['active_page'] = 'crossfilter'
+        doc.template = env.get_template("simple.html")
+        doc.template_variables["active_page"] = "crossfilter"
         doc.template_variables.update(extra)
         doc.theme = BOKEH_THEME
 
 
 def systemmonitor_doc(worker, extra, doc):
     with log_errors():
-        sysmon = SystemMonitor(worker, sizing_mode='scale_width')
+        sysmon = SystemMonitor(worker, sizing_mode="scale_width")
         doc.title = "Dask Worker Monitor"
         add_periodic_callback(doc, sysmon, 500)
 
         doc.add_root(sysmon.root)
-        doc.template = env.get_template('simple.html')
-        doc.template_variables['active_page'] = 'system'
+        doc.template = env.get_template("simple.html")
+        doc.template_variables["active_page"] = "system"
         doc.template_variables.update(extra)
         doc.theme = BOKEH_THEME
 
@@ -610,12 +722,12 @@ def systemmonitor_doc(worker, extra, doc):
 def counters_doc(server, extra, doc):
     with log_errors():
         doc.title = "Dask Worker Counters"
-        counter = Counters(server, sizing_mode='stretch_both')
+        counter = Counters(server, sizing_mode="stretch_both")
         add_periodic_callback(doc, counter, 500)
 
         doc.add_root(counter.root)
-        doc.template = env.get_template('simple.html')
-        doc.template_variables['active_page'] = 'counters'
+        doc.template = env.get_template("simple.html")
+        doc.template_variables["active_page"] = "counters"
         doc.template_variables.update(extra)
         doc.theme = BOKEH_THEME
 
@@ -623,12 +735,12 @@ def counters_doc(server, extra, doc):
 def profile_doc(server, extra, doc):
     with log_errors():
         doc.title = "Dask Worker Profile"
-        profile = ProfileTimePlot(server, sizing_mode='scale_width')
+        profile = ProfileTimePlot(server, sizing_mode="scale_width")
         profile.trigger_update()
 
         doc.add_root(profile.root)
-        doc.template = env.get_template('simple.html')
-        doc.template_variables['active_page'] = 'profile'
+        doc.template = env.get_template("simple.html")
+        doc.template_variables["active_page"] = "profile"
         doc.template_variables.update(extra)
         doc.theme = BOKEH_THEME
 
@@ -636,9 +748,9 @@ def profile_doc(server, extra, doc):
 def profile_server_doc(server, extra, doc):
     with log_errors():
         doc.title = "Dask: Profile of Event Loop"
-        prof = ProfileServer(server, sizing_mode='scale_width', doc=doc)
+        prof = ProfileServer(server, sizing_mode="scale_width", doc=doc)
         doc.add_root(prof.root)
-        doc.template = env.get_template('simple.html')
+        doc.template = env.get_template("simple.html")
         # doc.template_variables['active_page'] = ''
         doc.template_variables.update(extra)
         doc.theme = BOKEH_THEME
@@ -647,40 +759,48 @@ def profile_server_doc(server, extra, doc):
 
 
 class BokehWorker(BokehServer):
-    def __init__(self, worker, io_loop=None, prefix='', **kwargs):
+    def __init__(self, worker, io_loop=None, prefix="", **kwargs):
         self.worker = worker
         self.server_kwargs = kwargs
-        self.server_kwargs['prefix'] = prefix or None
-        prefix = prefix or ''
-        prefix = prefix.rstrip('/')
-        if prefix and not prefix.startswith('/'):
-            prefix = '/' + prefix
+        self.server_kwargs["prefix"] = prefix or None
+        prefix = prefix or ""
+        prefix = prefix.rstrip("/")
+        if prefix and not prefix.startswith("/"):
+            prefix = "/" + prefix
         self.prefix = prefix
 
-        extra = {'prefix': prefix}
+        extra = {"prefix": prefix}
 
         extra.update(template_variables)
 
         main = Application(FunctionHandler(partial(main_doc, worker, extra)))
-        crossfilter = Application(FunctionHandler(partial(crossfilter_doc, worker, extra)))
-        systemmonitor = Application(FunctionHandler(partial(systemmonitor_doc, worker, extra)))
+        crossfilter = Application(
+            FunctionHandler(partial(crossfilter_doc, worker, extra))
+        )
+        systemmonitor = Application(
+            FunctionHandler(partial(systemmonitor_doc, worker, extra))
+        )
         counters = Application(FunctionHandler(partial(counters_doc, worker, extra)))
         profile = Application(FunctionHandler(partial(profile_doc, worker, extra)))
-        profile_server = Application(FunctionHandler(partial(profile_server_doc, worker, extra)))
+        profile_server = Application(
+            FunctionHandler(partial(profile_server_doc, worker, extra))
+        )
 
-        self.apps = {'/main': main,
-                     '/counters': counters,
-                     '/crossfilter': crossfilter,
-                     '/system': systemmonitor,
-                     '/profile': profile,
-                     '/profile-server': profile_server}
+        self.apps = {
+            "/main": main,
+            "/counters": counters,
+            "/crossfilter": crossfilter,
+            "/system": systemmonitor,
+            "/profile": profile,
+            "/profile-server": profile_server,
+        }
 
         self.loop = io_loop or worker.loop
         self.server = None
 
     @property
     def extra(self):
-        return merge({'prefix': self.prefix}, template_variables)
+        return merge({"prefix": self.prefix}, template_variables)
 
     @property
     def my_server(self):
@@ -690,7 +810,14 @@ def listen(self, *args, **kwargs):
         super(BokehWorker, self).listen(*args, **kwargs)
 
         from .worker_html import routes
-        handlers = [(self.prefix + '/' + url, cls, {'server': self.my_server, 'extra': self.extra})
-                    for url, cls in routes]
 
-        self.server._tornado.add_handlers(r'.*', handlers)
+        handlers = [
+            (
+                self.prefix + "/" + url,
+                cls,
+                {"server": self.my_server, "extra": self.extra},
+            )
+            for url, cls in routes
+        ]
+
+        self.server._tornado.add_handlers(r".*", handlers)
diff --git a/distributed/bokeh/worker_html.py b/distributed/bokeh/worker_html.py
index 5a956231c6a..3ddf9490c4d 100644
--- a/distributed/bokeh/worker_html.py
+++ b/distributed/bokeh/worker_html.py
@@ -11,7 +11,7 @@ def initialize(self, server=None, extra=None):
         self.extra = extra or {}
 
     def get_template_path(self):
-        return os.path.join(dirname, 'templates')
+        return os.path.join(dirname, "templates")
 
 
 class _PrometheusCollector(object):
@@ -39,6 +39,7 @@ class PrometheusHandler(RequestHandler):
 
     def __init__(self, *args, **kwargs):
         import prometheus_client  # keep out of global namespace
+
         self.prometheus_client = prometheus_client
 
         super(PrometheusHandler, self).__init__(*args, **kwargs)
@@ -50,30 +51,24 @@ def _init(self):
             return
 
         self.prometheus_client.REGISTRY.register(
-            _PrometheusCollector(
-                self.server,
-                self.prometheus_client,
-            )
+            _PrometheusCollector(self.server, self.prometheus_client)
         )
 
         PrometheusHandler._initialized = True
 
     def get(self):
         self.write(self.prometheus_client.generate_latest())
-        self.set_header('Content-Type', 'text/plain; version=0.0.4')
+        self.set_header("Content-Type", "text/plain; version=0.0.4")
 
 
 class HealthHandler(RequestHandler):
     def get(self):
-        self.write('ok')
-        self.set_header('Content-Type', 'text/plain')
+        self.write("ok")
+        self.set_header("Content-Type", "text/plain")
 
 
-routes = [
-        (r'metrics', PrometheusHandler),
-        (r'health', HealthHandler),
-]
+routes = [(r"metrics", PrometheusHandler), (r"health", HealthHandler)]
 
 
 def get_handlers(server):
-    return [(url, cls, {'server': server}) for url, cls in routes]
+    return [(url, cls, {"server": server}) for url, cls in routes]
diff --git a/distributed/cfexecutor.py b/distributed/cfexecutor.py
index b22f36783c4..eb7bbf05646 100644
--- a/distributed/cfexecutor.py
+++ b/distributed/cfexecutor.py
@@ -21,9 +21,9 @@ def _cascade_future(future, cf_future):
     """
     result = yield future._result(raiseit=False)
     status = future.status
-    if status == 'finished':
+    if status == "finished":
         cf_future.set_result(result)
-    elif status == 'cancelled':
+    elif status == "cancelled":
         cf_future.cancel()
         # Necessary for wait() and as_completed() to wake up
         cf_future.set_running_or_notify_cancel()
@@ -48,13 +48,17 @@ class ClientExecutor(cf.Executor):
     A concurrent.futures Executor that executes tasks on a dask.distributed Client.
     """
 
-    _allowed_kwargs = frozenset(['pure', 'workers', 'resources', 'allow_other_workers', 'retries'])
+    _allowed_kwargs = frozenset(
+        ["pure", "workers", "resources", "allow_other_workers", "retries"]
+    )
 
     def __init__(self, client, **kwargs):
         sk = set(kwargs)
         if not sk <= self._allowed_kwargs:
-            raise TypeError("unsupported arguments to ClientExecutor: %s"
-                            % sorted(sk - self._allowed_kwargs))
+            raise TypeError(
+                "unsupported arguments to ClientExecutor: %s"
+                % sorted(sk - self._allowed_kwargs)
+            )
         self._client = client
         self._futures = weakref.WeakSet()
         self._shutdown = False
@@ -68,7 +72,7 @@ def _wrap_future(self, future):
 
         # Support cancelling task through .cancel() on c.f.Future
         def cf_callback(cf_future):
-            if cf_future.cancelled() and future.status != 'cancelled':
+            if cf_future.cancelled() and future.status != "cancelled":
                 future.cancel()
 
         cf_future.add_done_callback(cf_callback)
@@ -87,7 +91,7 @@ def submit(self, fn, *args, **kwargs):
         A Future representing the given call.
         """
         if self._shutdown:
-            raise RuntimeError('cannot schedule new futures after shutdown')
+            raise RuntimeError("cannot schedule new futures after shutdown")
         future = self._client.submit(fn, *args, **merge(self._kwargs, kwargs))
         self._futures.add(future)
         return self._wrap_future(future)
@@ -115,14 +119,13 @@ def map(self, fn, *iterables, **kwargs):
             before the given timeout.
         Exception: If ``fn(*args)`` raises for any values.
         """
-        timeout = kwargs.pop('timeout', None)
+        timeout = kwargs.pop("timeout", None)
         if timeout is not None:
             end_time = timeout + time()
-        if 'chunksize' in kwargs:
-            del kwargs['chunksize']
+        if "chunksize" in kwargs:
+            del kwargs["chunksize"]
         if kwargs:
-            raise TypeError("unexpected arguments to map(): %s"
-                            % sorted(kwargs))
+            raise TypeError("unexpected arguments to map(): %s" % sorted(kwargs))
 
         fs = self._client.map(fn, *iterables, **self._kwargs)
 
diff --git a/distributed/cli/dask_mpi.py b/distributed/cli/dask_mpi.py
index e26e0bc91e1..ef7dd0c59fa 100644
--- a/distributed/cli/dask_mpi.py
+++ b/distributed/cli/dask_mpi.py
@@ -18,32 +18,62 @@
 
 
 @click.command()
-@click.option('--scheduler-file', type=str, default='scheduler.json',
-              help='Filename to JSON encoded scheduler information. ')
-@click.option('--interface', type=str, default=None,
-              help="Network interface like 'eth0' or 'ib0'")
-@click.option('--nthreads', type=int, default=0,
-              help="Number of threads per worker.")
-@click.option('--memory-limit', default='auto',
-              help="Number of bytes before spilling data to disk. "
-                   "This can be an integer (nbytes) "
-                   "float (fraction of total memory) "
-                   "or 'auto'")
-@click.option('--local-directory', default='', type=str,
-              help="Directory to place worker files")
-@click.option('--scheduler/--no-scheduler', default=True,
-              help=("Whether or not to include a scheduler. "
-                    "Use --no-scheduler to increase an existing dask cluster"))
-@click.option('--nanny/--no-nanny', default=True,
-              help="Start workers in nanny process for management")
-@click.option('--bokeh-port', type=int, default=8787,
-              help="Bokeh port for visual diagnostics")
-@click.option('--bokeh-worker-port', type=int, default=8789,
-              help="Worker's Bokeh port for visual diagnostics")
-@click.option('--bokeh-prefix', type=str, default=None,
-              help="Prefix for the bokeh app")
-def main(scheduler_file, interface, nthreads, local_directory, memory_limit,
-         scheduler, bokeh_port, bokeh_prefix, nanny, bokeh_worker_port):
+@click.option(
+    "--scheduler-file",
+    type=str,
+    default="scheduler.json",
+    help="Filename to JSON encoded scheduler information. ",
+)
+@click.option(
+    "--interface", type=str, default=None, help="Network interface like 'eth0' or 'ib0'"
+)
+@click.option("--nthreads", type=int, default=0, help="Number of threads per worker.")
+@click.option(
+    "--memory-limit",
+    default="auto",
+    help="Number of bytes before spilling data to disk. "
+    "This can be an integer (nbytes) "
+    "float (fraction of total memory) "
+    "or 'auto'",
+)
+@click.option(
+    "--local-directory", default="", type=str, help="Directory to place worker files"
+)
+@click.option(
+    "--scheduler/--no-scheduler",
+    default=True,
+    help=(
+        "Whether or not to include a scheduler. "
+        "Use --no-scheduler to increase an existing dask cluster"
+    ),
+)
+@click.option(
+    "--nanny/--no-nanny",
+    default=True,
+    help="Start workers in nanny process for management",
+)
+@click.option(
+    "--bokeh-port", type=int, default=8787, help="Bokeh port for visual diagnostics"
+)
+@click.option(
+    "--bokeh-worker-port",
+    type=int,
+    default=8789,
+    help="Worker's Bokeh port for visual diagnostics",
+)
+@click.option("--bokeh-prefix", type=str, default=None, help="Prefix for the bokeh app")
+def main(
+    scheduler_file,
+    interface,
+    nthreads,
+    local_directory,
+    memory_limit,
+    scheduler,
+    bokeh_port,
+    bokeh_prefix,
+    nanny,
+    bokeh_worker_port,
+):
     if interface:
         host = get_ip_interface(interface)
     else:
@@ -55,11 +85,12 @@ def main(scheduler_file, interface, nthreads, local_directory, memory_limit,
         except ImportError:
             services = {}
         else:
-            services = {('bokeh',  bokeh_port): partial(BokehScheduler,
-                                                        prefix=bokeh_prefix)}
-        scheduler = Scheduler(scheduler_file=scheduler_file,
-                              loop=loop,
-                              services=services)
+            services = {
+                ("bokeh", bokeh_port): partial(BokehScheduler, prefix=bokeh_prefix)
+            }
+        scheduler = Scheduler(
+            scheduler_file=scheduler_file, loop=loop, services=services
+        )
         addr = uri_from_host_port(host, None, 8786)
         scheduler.start(addr)
         try:
@@ -69,19 +100,21 @@ def main(scheduler_file, interface, nthreads, local_directory, memory_limit,
             scheduler.stop()
     else:
         W = Nanny if nanny else Worker
-        worker = W(scheduler_file=scheduler_file,
-                   loop=loop,
-                   name=rank if scheduler else None,
-                   ncores=nthreads,
-                   local_dir=local_directory,
-                   services={('bokeh', bokeh_worker_port): BokehWorker},
-                   memory_limit=memory_limit)
+        worker = W(
+            scheduler_file=scheduler_file,
+            loop=loop,
+            name=rank if scheduler else None,
+            ncores=nthreads,
+            local_dir=local_directory,
+            services={("bokeh", bokeh_worker_port): BokehWorker},
+            memory_limit=memory_limit,
+        )
         addr = uri_from_host_port(host, None, 0)
 
         @gen.coroutine
         def run():
             yield worker._start(addr)
-            while worker.status != 'closed':
+            while worker.status != "closed":
                 yield gen.sleep(0.2)
 
         try:
@@ -99,12 +132,14 @@ def close():
 
 def go():
     check_python_3()
-    warn("The dask-mpi command line utility in the `distributed` "
-         "package is deprecated.  "
-         "Please install the `dask-mpi` package instead. "
-         "More information is available at https://mpi.dask.org")
+    warn(
+        "The dask-mpi command line utility in the `distributed` "
+        "package is deprecated.  "
+        "Please install the `dask-mpi` package instead. "
+        "More information is available at https://mpi.dask.org"
+    )
     main()
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     go()
diff --git a/distributed/cli/dask_remote.py b/distributed/cli/dask_remote.py
index 2d94d0e1142..933d8d318b0 100644
--- a/distributed/cli/dask_remote.py
+++ b/distributed/cli/dask_remote.py
@@ -6,9 +6,8 @@
 
 
 @click.command()
-@click.option('--host', type=str, default=None,
-              help="IP or hostname of this server")
-@click.option('--port', type=int, default=8788, help="Remote Client Port")
+@click.option("--host", type=str, default=None, help="IP or hostname of this server")
+@click.option("--port", type=int, default=8788, help="Remote Client Port")
 def main(host, port):
     _remote(host, port)
 
@@ -19,5 +18,5 @@ def go():
     main()
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     go()
diff --git a/distributed/cli/dask_scheduler.py b/distributed/cli/dask_scheduler.py
index f48e8b4e26d..0e8415ac132 100755
--- a/distributed/cli/dask_scheduler.py
+++ b/distributed/cli/dask_scheduler.py
@@ -16,61 +16,127 @@
 from distributed import Scheduler
 from distributed.security import Security
 from distributed.utils import get_ip_interface
-from distributed.cli.utils import (check_python_3, install_signal_handlers,
-                                   uri_from_host_port)
+from distributed.cli.utils import (
+    check_python_3,
+    install_signal_handlers,
+    uri_from_host_port,
+)
 from distributed.preloading import preload_modules, validate_preload_argv
-from distributed.proctitle import (enable_proctitle_on_children,
-                                   enable_proctitle_on_current)
+from distributed.proctitle import (
+    enable_proctitle_on_children,
+    enable_proctitle_on_current,
+)
 
-logger = logging.getLogger('distributed.scheduler')
+logger = logging.getLogger("distributed.scheduler")
 
 
 pem_file_option_type = click.Path(exists=True, resolve_path=True)
 
 
 @click.command(context_settings=dict(ignore_unknown_options=True))
-@click.option('--host', type=str, default='',
-              help="URI, IP or hostname of this server")
-@click.option('--port', type=int, default=None, help="Serving port")
-@click.option('--interface', type=str, default=None,
-              help="Preferred network interface like 'eth0' or 'ib0'")
-@click.option('--tls-ca-file', type=pem_file_option_type, default=None,
-              help="CA cert(s) file for TLS (in PEM format)")
-@click.option('--tls-cert', type=pem_file_option_type, default=None,
-              help="certificate file for TLS (in PEM format)")
-@click.option('--tls-key', type=pem_file_option_type, default=None,
-              help="private key file for TLS (in PEM format)")
+@click.option("--host", type=str, default="", help="URI, IP or hostname of this server")
+@click.option("--port", type=int, default=None, help="Serving port")
+@click.option(
+    "--interface",
+    type=str,
+    default=None,
+    help="Preferred network interface like 'eth0' or 'ib0'",
+)
+@click.option(
+    "--tls-ca-file",
+    type=pem_file_option_type,
+    default=None,
+    help="CA cert(s) file for TLS (in PEM format)",
+)
+@click.option(
+    "--tls-cert",
+    type=pem_file_option_type,
+    default=None,
+    help="certificate file for TLS (in PEM format)",
+)
+@click.option(
+    "--tls-key",
+    type=pem_file_option_type,
+    default=None,
+    help="private key file for TLS (in PEM format)",
+)
 # XXX default port (or URI) values should be centralized somewhere
-@click.option('--bokeh-port', type=int, default=None,
-              help="Deprecated.  See --dashboard-address")
-@click.option('--dashboard-address', type=str, default=':8787',
-              help="Address on which to listen for diagnostics dashboard")
-@click.option('--bokeh/--no-bokeh', '_bokeh', default=True, show_default=True,
-              required=False, help="Launch Bokeh Web UI")
-@click.option('--show/--no-show', default=False, help="Show web UI")
-@click.option('--bokeh-whitelist', default=None, multiple=True,
-              help="IP addresses to whitelist for bokeh.")
-@click.option('--bokeh-prefix', type=str, default=None,
-              help="Prefix for the bokeh app")
-@click.option('--use-xheaders', type=bool, default=False, show_default=True,
-              help="User xheaders in bokeh app for ssl termination in header")
-@click.option('--pid-file', type=str, default='',
-              help="File to write the process PID")
-@click.option('--scheduler-file', type=str, default='',
-              help="File to write connection information. "
-              "This may be a good way to share connection information if your "
-              "cluster is on a shared network file system.")
-@click.option('--local-directory', default='', type=str,
-              help="Directory to place scheduler files")
-@click.option('--preload', type=str, multiple=True, is_eager=True, default='',
-              help='Module that should be loaded by the scheduler process  '
-                   'like "foo.bar" or "/path/to/foo.py".')
-@click.argument('preload_argv', nargs=-1,
-                type=click.UNPROCESSED, callback=validate_preload_argv)
-def main(host, port, bokeh_port, show, _bokeh, bokeh_whitelist, bokeh_prefix,
-        use_xheaders, pid_file, scheduler_file, interface,
-        local_directory, preload, preload_argv, tls_ca_file, tls_cert, tls_key,
-        dashboard_address):
+@click.option(
+    "--bokeh-port", type=int, default=None, help="Deprecated.  See --dashboard-address"
+)
+@click.option(
+    "--dashboard-address",
+    type=str,
+    default=":8787",
+    help="Address on which to listen for diagnostics dashboard",
+)
+@click.option(
+    "--bokeh/--no-bokeh",
+    "_bokeh",
+    default=True,
+    show_default=True,
+    required=False,
+    help="Launch Bokeh Web UI",
+)
+@click.option("--show/--no-show", default=False, help="Show web UI")
+@click.option(
+    "--bokeh-whitelist",
+    default=None,
+    multiple=True,
+    help="IP addresses to whitelist for bokeh.",
+)
+@click.option("--bokeh-prefix", type=str, default=None, help="Prefix for the bokeh app")
+@click.option(
+    "--use-xheaders",
+    type=bool,
+    default=False,
+    show_default=True,
+    help="User xheaders in bokeh app for ssl termination in header",
+)
+@click.option("--pid-file", type=str, default="", help="File to write the process PID")
+@click.option(
+    "--scheduler-file",
+    type=str,
+    default="",
+    help="File to write connection information. "
+    "This may be a good way to share connection information if your "
+    "cluster is on a shared network file system.",
+)
+@click.option(
+    "--local-directory", default="", type=str, help="Directory to place scheduler files"
+)
+@click.option(
+    "--preload",
+    type=str,
+    multiple=True,
+    is_eager=True,
+    default="",
+    help="Module that should be loaded by the scheduler process  "
+    'like "foo.bar" or "/path/to/foo.py".',
+)
+@click.argument(
+    "preload_argv", nargs=-1, type=click.UNPROCESSED, callback=validate_preload_argv
+)
+def main(
+    host,
+    port,
+    bokeh_port,
+    show,
+    _bokeh,
+    bokeh_whitelist,
+    bokeh_prefix,
+    use_xheaders,
+    pid_file,
+    scheduler_file,
+    interface,
+    local_directory,
+    preload,
+    preload_argv,
+    tls_ca_file,
+    tls_cert,
+    tls_key,
+    dashboard_address,
+):
 
     enable_proctitle_on_current()
     enable_proctitle_on_children()
@@ -82,21 +148,21 @@ def main(host, port, bokeh_port, show, _bokeh, bokeh_whitelist, bokeh_prefix,
         )
         dashboard_address = bokeh_port
 
-    sec = Security(tls_ca_file=tls_ca_file,
-                   tls_scheduler_cert=tls_cert,
-                   tls_scheduler_key=tls_key,
-                   )
+    sec = Security(
+        tls_ca_file=tls_ca_file, tls_scheduler_cert=tls_cert, tls_scheduler_key=tls_key
+    )
 
     if not host and (tls_ca_file or tls_cert or tls_key):
-        host = 'tls://'
+        host = "tls://"
 
     if pid_file:
-        with open(pid_file, 'w') as f:
+        with open(pid_file, "w") as f:
             f.write(str(os.getpid()))
 
         def del_pid_file():
             if os.path.exists(pid_file):
                 os.remove(pid_file)
+
         atexit.register(del_pid_file)
 
     local_directory_created = False
@@ -105,13 +171,14 @@ def del_pid_file():
             os.mkdir(local_directory)
             local_directory_created = True
     else:
-        local_directory = tempfile.mkdtemp(prefix='scheduler-')
+        local_directory = tempfile.mkdtemp(prefix="scheduler-")
         local_directory_created = True
     if local_directory not in sys.path:
         sys.path.insert(0, local_directory)
 
-    if sys.platform.startswith('linux'):
-        import resource   # module fails importing on Windows
+    if sys.platform.startswith("linux"):
+        import resource  # module fails importing on Windows
+
         soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
         limit = max(soft, hard // 2)
         resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard))
@@ -125,32 +192,37 @@ def del_pid_file():
     addr = uri_from_host_port(host, port, 8786)
 
     loop = IOLoop.current()
-    logger.info('-' * 47)
+    logger.info("-" * 47)
 
     services = {}
     if _bokeh:
         try:
             from distributed.bokeh.scheduler import BokehScheduler
-            services[('bokeh', dashboard_address)] = (BokehScheduler,
-                                                      {'prefix': bokeh_prefix})
+
+            services[("bokeh", dashboard_address)] = (
+                BokehScheduler,
+                {"prefix": bokeh_prefix},
+            )
         except ImportError as error:
-            if str(error).startswith('No module named'):
-                logger.info('Web dashboard not loaded.  Unable to import bokeh')
+            if str(error).startswith("No module named"):
+                logger.info("Web dashboard not loaded.  Unable to import bokeh")
             else:
-                logger.info('Unable to import bokeh: %s' % str(error))
+                logger.info("Unable to import bokeh: %s" % str(error))
 
-    scheduler = Scheduler(loop=loop, services=services,
-                          scheduler_file=scheduler_file,
-                          security=sec)
+    scheduler = Scheduler(
+        loop=loop, services=services, scheduler_file=scheduler_file, security=sec
+    )
     scheduler.start(addr)
     if not preload:
-        preload = dask.config.get('distributed.scheduler.preload')
+        preload = dask.config.get("distributed.scheduler.preload")
     if not preload_argv:
-        preload_argv = dask.config.get('distributed.scheduler.preload-argv')
-    preload_modules(preload, parameter=scheduler, file_dir=local_directory, argv=preload_argv)
+        preload_argv = dask.config.get("distributed.scheduler.preload-argv")
+    preload_modules(
+        preload, parameter=scheduler, file_dir=local_directory, argv=preload_argv
+    )
 
-    logger.info('Local Directory: %26s', local_directory)
-    logger.info('-' * 47)
+    logger.info("Local Directory: %26s", local_directory)
+    logger.info("-" * 47)
 
     install_signal_handlers(loop)
 
@@ -170,5 +242,5 @@ def go():
     main()
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     go()
diff --git a/distributed/cli/dask_ssh.py b/distributed/cli/dask_ssh.py
index acb87d21642..df2b1c6fe94 100755
--- a/distributed/cli/dask_ssh.py
+++ b/distributed/cli/dask_ssh.py
@@ -6,53 +6,120 @@
 from distributed.cli.utils import check_python_3
 
 
-@click.command(help="""Launch a distributed cluster over SSH. A 'dask-scheduler' process will run on the
+@click.command(
+    help="""Launch a distributed cluster over SSH. A 'dask-scheduler' process will run on the
                          first host specified in [HOSTNAMES] or in the hostfile (unless --scheduler is specified
                          explicitly). One or more 'dask-worker' processes will be run each host in [HOSTNAMES] or
                          in the hostfile. Use command line flags to adjust how many dask-worker process are run on
-                         each host (--nprocs) and how many cpus are used by each dask-worker process (--nthreads).""")
-@click.option('--scheduler', default=None, type=str,
-              help="Specify scheduler node.  Defaults to first address.")
-@click.option('--scheduler-port', default=8786, type=int,
-              help="Specify scheduler port number.  Defaults to port 8786.")
-@click.option('--nthreads', default=0, type=int,
-              help=("Number of threads per worker process. "
-                    "Defaults to number of cores divided by the number of "
-                    "processes per host."))
-@click.option('--nprocs', default=1, type=int,
-              help="Number of worker processes per host.  Defaults to one.")
-@click.argument('hostnames', nargs=-1, type=str)
-@click.option('--hostfile', default=None, type=click.Path(exists=True),
-              help="Textfile with hostnames/IP addresses")
-@click.option('--ssh-username', default=None, type=str,
-              help="Username to use when establishing SSH connections.")
-@click.option('--ssh-port', default=22, type=int,
-              help="Port to use for SSH connections.")
-@click.option('--ssh-private-key', default=None, type=str,
-              help="Private key file to use for SSH connections.")
-@click.option('--nohost', is_flag=True,
-              help="Do not pass the hostname to the worker.")
-@click.option('--log-directory', default=None, type=click.Path(exists=True),
-              help=("Directory to use on all cluster nodes for the output of "
-                    "dask-scheduler and dask-worker commands."))
-@click.option('--remote-python', default=None, type=str,
-              help="Path to Python on remote nodes.")
-@click.option('--memory-limit', default='auto',
-              help="Bytes of memory that the worker can use. "
-                   "This can be an integer (bytes), "
-                   "float (fraction of total system memory), "
-                   "string (like 5GB or 5000M), "
-                   "'auto', or zero for no memory management")
-@click.option('--worker-port', type=int, default=0,
-              help="Serving computation port, defaults to random")
-@click.option('--nanny-port', type=int, default=0,
-              help="Serving nanny port, defaults to random")
-@click.option('--remote-dask-worker', default=None, type=str,
-              help="Worker to run. Defaults to distributed.cli.dask_worker")
+                         each host (--nprocs) and how many cpus are used by each dask-worker process (--nthreads)."""
+)
+@click.option(
+    "--scheduler",
+    default=None,
+    type=str,
+    help="Specify scheduler node.  Defaults to first address.",
+)
+@click.option(
+    "--scheduler-port",
+    default=8786,
+    type=int,
+    help="Specify scheduler port number.  Defaults to port 8786.",
+)
+@click.option(
+    "--nthreads",
+    default=0,
+    type=int,
+    help=(
+        "Number of threads per worker process. "
+        "Defaults to number of cores divided by the number of "
+        "processes per host."
+    ),
+)
+@click.option(
+    "--nprocs",
+    default=1,
+    type=int,
+    help="Number of worker processes per host.  Defaults to one.",
+)
+@click.argument("hostnames", nargs=-1, type=str)
+@click.option(
+    "--hostfile",
+    default=None,
+    type=click.Path(exists=True),
+    help="Textfile with hostnames/IP addresses",
+)
+@click.option(
+    "--ssh-username",
+    default=None,
+    type=str,
+    help="Username to use when establishing SSH connections.",
+)
+@click.option(
+    "--ssh-port", default=22, type=int, help="Port to use for SSH connections."
+)
+@click.option(
+    "--ssh-private-key",
+    default=None,
+    type=str,
+    help="Private key file to use for SSH connections.",
+)
+@click.option("--nohost", is_flag=True, help="Do not pass the hostname to the worker.")
+@click.option(
+    "--log-directory",
+    default=None,
+    type=click.Path(exists=True),
+    help=(
+        "Directory to use on all cluster nodes for the output of "
+        "dask-scheduler and dask-worker commands."
+    ),
+)
+@click.option(
+    "--remote-python", default=None, type=str, help="Path to Python on remote nodes."
+)
+@click.option(
+    "--memory-limit",
+    default="auto",
+    help="Bytes of memory that the worker can use. "
+    "This can be an integer (bytes), "
+    "float (fraction of total system memory), "
+    "string (like 5GB or 5000M), "
+    "'auto', or zero for no memory management",
+)
+@click.option(
+    "--worker-port",
+    type=int,
+    default=0,
+    help="Serving computation port, defaults to random",
+)
+@click.option(
+    "--nanny-port", type=int, default=0, help="Serving nanny port, defaults to random"
+)
+@click.option(
+    "--remote-dask-worker",
+    default=None,
+    type=str,
+    help="Worker to run. Defaults to distributed.cli.dask_worker",
+)
 @click.pass_context
-def main(ctx, scheduler, scheduler_port, hostnames, hostfile, nthreads, nprocs,
-         ssh_username, ssh_port, ssh_private_key, nohost, log_directory, remote_python,
-         memory_limit, worker_port, nanny_port, remote_dask_worker):
+def main(
+    ctx,
+    scheduler,
+    scheduler_port,
+    hostnames,
+    hostfile,
+    nthreads,
+    nprocs,
+    ssh_username,
+    ssh_port,
+    ssh_private_key,
+    nohost,
+    log_directory,
+    remote_python,
+    memory_limit,
+    worker_port,
+    nanny_port,
+    remote_dask_worker,
+):
     try:
         hostnames = list(hostnames)
         if hostfile:
@@ -67,18 +134,37 @@ def main(ctx, scheduler, scheduler_port, hostnames, hostfile, nthreads, nprocs,
         print(ctx.get_help())
         exit(1)
 
-    c = SSHCluster(scheduler, scheduler_port, hostnames, nthreads, nprocs,
-                   ssh_username, ssh_port, ssh_private_key, nohost, log_directory, remote_python,
-                   memory_limit, worker_port, nanny_port, remote_dask_worker)
+    c = SSHCluster(
+        scheduler,
+        scheduler_port,
+        hostnames,
+        nthreads,
+        nprocs,
+        ssh_username,
+        ssh_port,
+        ssh_private_key,
+        nohost,
+        log_directory,
+        remote_python,
+        memory_limit,
+        worker_port,
+        nanny_port,
+        remote_dask_worker,
+    )
 
     import distributed
-    print('\n---------------------------------------------------------------')
-    print('                 Dask.distributed v{version}\n'.format(version=distributed.__version__))
-    print('Worker nodes:'.format(n=len(hostnames)))
+
+    print("\n---------------------------------------------------------------")
+    print(
+        "                 Dask.distributed v{version}\n".format(
+            version=distributed.__version__
+        )
+    )
+    print("Worker nodes:".format(n=len(hostnames)))
     for i, host in enumerate(hostnames):
-        print('  {num}: {host}'.format(num=i, host=host))
-    print('\nscheduler node: {addr}:{port}'.format(addr=scheduler, port=scheduler_port))
-    print('---------------------------------------------------------------\n\n')
+        print("  {num}: {host}".format(num=i, host=host))
+    print("\nscheduler node: {addr}:{port}".format(addr=scheduler, port=scheduler_port))
+    print("---------------------------------------------------------------\n\n")
 
     # Monitor the output of remote processes.  This blocks until the user issues a KeyboardInterrupt.
     c.monitor_remote_processes()
@@ -94,5 +180,5 @@ def go():
     main()
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     go()
diff --git a/distributed/cli/dask_submit.py b/distributed/cli/dask_submit.py
index 1bb507dd9fe..1ef759407c6 100644
--- a/distributed/cli/dask_submit.py
+++ b/distributed/cli/dask_submit.py
@@ -7,8 +7,8 @@
 
 
 @click.command()
-@click.argument('remote_client_address', type=str, required=True)
-@click.argument('filepath', type=str, required=True)
+@click.argument("remote_client_address", type=str, required=True)
+@click.argument("filepath", type=str, required=True)
 def main(remote_client_address, filepath):
     @gen.coroutine
     def f():
@@ -27,5 +27,5 @@ def go():
     main()
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     go()
diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
index ebc0fb441b6..0eb5a7973fb 100755
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -12,100 +12,201 @@
 from distributed.utils import get_ip_interface, parse_timedelta
 from distributed.worker import _ncores
 from distributed.security import Security
-from distributed.cli.utils import (check_python_3, uri_from_host_port,
-                                   install_signal_handlers)
+from distributed.cli.utils import (
+    check_python_3,
+    uri_from_host_port,
+    install_signal_handlers,
+)
 from distributed.comm import get_address_host_port
 from distributed.preloading import validate_preload_argv
-from distributed.proctitle import (enable_proctitle_on_children,
-                                   enable_proctitle_on_current)
+from distributed.proctitle import (
+    enable_proctitle_on_children,
+    enable_proctitle_on_current,
+)
 
 from toolz import valmap
 from tornado.ioloop import IOLoop, TimeoutError
 from tornado import gen
 
-logger = logging.getLogger('distributed.dask_worker')
+logger = logging.getLogger("distributed.dask_worker")
 
 
 pem_file_option_type = click.Path(exists=True, resolve_path=True)
 
 
 @click.command(context_settings=dict(ignore_unknown_options=True))
-@click.argument('scheduler', type=str, required=False)
-@click.option('--tls-ca-file', type=pem_file_option_type, default=None,
-              help="CA cert(s) file for TLS (in PEM format)")
-@click.option('--tls-cert', type=pem_file_option_type, default=None,
-              help="certificate file for TLS (in PEM format)")
-@click.option('--tls-key', type=pem_file_option_type, default=None,
-              help="private key file for TLS (in PEM format)")
-@click.option('--worker-port', type=int, default=0,
-              help="Serving computation port, defaults to random")
-@click.option('--nanny-port', type=int, default=0,
-              help="Serving nanny port, defaults to random")
-@click.option('--bokeh-port', type=int, default=None,
-              help="Deprecated.  See --dashboard-address")
-@click.option('--dashboard-address', type=str, default=':0',
-              help="Address on which to listen for diagnostics dashboard")
-@click.option('--bokeh/--no-bokeh', 'bokeh', default=True, show_default=True,
-              required=False, help="Launch Bokeh Web UI")
-@click.option('--listen-address', type=str, default=None,
-        help="The address to which the worker binds. "
-             "Example: tcp://0.0.0.0:9000")
-@click.option('--contact-address', type=str, default=None,
-        help="The address the worker advertises to the scheduler for "
-             "communication with it and other workers. "
-             "Example: tcp://127.0.0.1:9000")
-@click.option('--host', type=str, default=None,
-              help="Serving host. Should be an ip address that is"
-                   " visible to the scheduler and other workers. "
-                   "See --listen-address and --contact-address if you "
-                   "need different listen and contact addresses. "
-                   "See --interface.")
-@click.option('--interface', type=str, default=None,
-              help="Network interface like 'eth0' or 'ib0'")
-@click.option('--nthreads', type=int, default=0,
-              help="Number of threads per process.")
-@click.option('--nprocs', type=int, default=1,
-              help="Number of worker processes to launch.  Defaults to one.")
-@click.option('--name', type=str, default='',
-              help="A unique name for this worker like 'worker-1'. "
-                   "If used with --nprocs then the process number "
-                   "will be appended like name-0, name-1, name-2, ...")
-@click.option('--memory-limit', default='auto',
-              help="Bytes of memory per process that the worker can use. "
-                   "This can be an integer (bytes), "
-                   "float (fraction of total system memory), "
-                   "string (like 5GB or 5000M), "
-                   "'auto', or zero for no memory management")
-@click.option('--reconnect/--no-reconnect', default=True,
-              help="Reconnect to scheduler if disconnected")
-@click.option('--nanny/--no-nanny', default=True,
-              help="Start workers in nanny process for management")
-@click.option('--pid-file', type=str, default='',
-              help="File to write the process PID")
-@click.option('--local-directory', default='', type=str,
-              help="Directory to place worker files")
-@click.option('--resources', type=str, default='',
-              help='Resources for task constraints like "GPU=2 MEM=10e9". '
-                   'Resources are applied separately to each worker process '
-                   "(only relevant when starting multiple worker processes with '--nprocs').")
-@click.option('--scheduler-file', type=str, default='',
-              help='Filename to JSON encoded scheduler information. '
-                   'Use with dask-scheduler --scheduler-file')
-@click.option('--death-timeout', type=str, default=None,
-              help="Seconds to wait for a scheduler before closing")
-@click.option('--bokeh-prefix', type=str, default=None,
-              help="Prefix for the bokeh app")
-@click.option('--preload', type=str, multiple=True, is_eager=True,
-              help='Module that should be loaded by each worker process '
-                   'like "foo.bar" or "/path/to/foo.py"')
-@click.argument('preload_argv', nargs=-1,
-                type=click.UNPROCESSED, callback=validate_preload_argv)
-def main(scheduler, host, worker_port, listen_address, contact_address,
-         nanny_port, nthreads, nprocs, nanny, name,
-         memory_limit, pid_file, reconnect, resources, bokeh,
-         bokeh_port, local_directory, scheduler_file, interface,
-         death_timeout, preload, preload_argv, bokeh_prefix, tls_ca_file,
-         tls_cert, tls_key, dashboard_address):
+@click.argument("scheduler", type=str, required=False)
+@click.option(
+    "--tls-ca-file",
+    type=pem_file_option_type,
+    default=None,
+    help="CA cert(s) file for TLS (in PEM format)",
+)
+@click.option(
+    "--tls-cert",
+    type=pem_file_option_type,
+    default=None,
+    help="certificate file for TLS (in PEM format)",
+)
+@click.option(
+    "--tls-key",
+    type=pem_file_option_type,
+    default=None,
+    help="private key file for TLS (in PEM format)",
+)
+@click.option(
+    "--worker-port",
+    type=int,
+    default=0,
+    help="Serving computation port, defaults to random",
+)
+@click.option(
+    "--nanny-port", type=int, default=0, help="Serving nanny port, defaults to random"
+)
+@click.option(
+    "--bokeh-port", type=int, default=None, help="Deprecated.  See --dashboard-address"
+)
+@click.option(
+    "--dashboard-address",
+    type=str,
+    default=":0",
+    help="Address on which to listen for diagnostics dashboard",
+)
+@click.option(
+    "--bokeh/--no-bokeh",
+    "bokeh",
+    default=True,
+    show_default=True,
+    required=False,
+    help="Launch Bokeh Web UI",
+)
+@click.option(
+    "--listen-address",
+    type=str,
+    default=None,
+    help="The address to which the worker binds. " "Example: tcp://0.0.0.0:9000",
+)
+@click.option(
+    "--contact-address",
+    type=str,
+    default=None,
+    help="The address the worker advertises to the scheduler for "
+    "communication with it and other workers. "
+    "Example: tcp://127.0.0.1:9000",
+)
+@click.option(
+    "--host",
+    type=str,
+    default=None,
+    help="Serving host. Should be an ip address that is"
+    " visible to the scheduler and other workers. "
+    "See --listen-address and --contact-address if you "
+    "need different listen and contact addresses. "
+    "See --interface.",
+)
+@click.option(
+    "--interface", type=str, default=None, help="Network interface like 'eth0' or 'ib0'"
+)
+@click.option("--nthreads", type=int, default=0, help="Number of threads per process.")
+@click.option(
+    "--nprocs",
+    type=int,
+    default=1,
+    help="Number of worker processes to launch.  Defaults to one.",
+)
+@click.option(
+    "--name",
+    type=str,
+    default="",
+    help="A unique name for this worker like 'worker-1'. "
+    "If used with --nprocs then the process number "
+    "will be appended like name-0, name-1, name-2, ...",
+)
+@click.option(
+    "--memory-limit",
+    default="auto",
+    help="Bytes of memory per process that the worker can use. "
+    "This can be an integer (bytes), "
+    "float (fraction of total system memory), "
+    "string (like 5GB or 5000M), "
+    "'auto', or zero for no memory management",
+)
+@click.option(
+    "--reconnect/--no-reconnect",
+    default=True,
+    help="Reconnect to scheduler if disconnected",
+)
+@click.option(
+    "--nanny/--no-nanny",
+    default=True,
+    help="Start workers in nanny process for management",
+)
+@click.option("--pid-file", type=str, default="", help="File to write the process PID")
+@click.option(
+    "--local-directory", default="", type=str, help="Directory to place worker files"
+)
+@click.option(
+    "--resources",
+    type=str,
+    default="",
+    help='Resources for task constraints like "GPU=2 MEM=10e9". '
+    "Resources are applied separately to each worker process "
+    "(only relevant when starting multiple worker processes with '--nprocs').",
+)
+@click.option(
+    "--scheduler-file",
+    type=str,
+    default="",
+    help="Filename to JSON encoded scheduler information. "
+    "Use with dask-scheduler --scheduler-file",
+)
+@click.option(
+    "--death-timeout",
+    type=str,
+    default=None,
+    help="Seconds to wait for a scheduler before closing",
+)
+@click.option("--bokeh-prefix", type=str, default=None, help="Prefix for the bokeh app")
+@click.option(
+    "--preload",
+    type=str,
+    multiple=True,
+    is_eager=True,
+    help="Module that should be loaded by each worker process "
+    'like "foo.bar" or "/path/to/foo.py"',
+)
+@click.argument(
+    "preload_argv", nargs=-1, type=click.UNPROCESSED, callback=validate_preload_argv
+)
+def main(
+    scheduler,
+    host,
+    worker_port,
+    listen_address,
+    contact_address,
+    nanny_port,
+    nthreads,
+    nprocs,
+    nanny,
+    name,
+    memory_limit,
+    pid_file,
+    reconnect,
+    resources,
+    bokeh,
+    bokeh_port,
+    local_directory,
+    scheduler_file,
+    interface,
+    death_timeout,
+    preload,
+    preload_argv,
+    bokeh_prefix,
+    tls_ca_file,
+    tls_cert,
+    tls_key,
+    dashboard_address,
+):
     enable_proctitle_on_current()
     enable_proctitle_on_children()
 
@@ -116,32 +217,41 @@ def main(scheduler, host, worker_port, listen_address, contact_address,
         )
         dashboard_address = bokeh_port
 
-    sec = Security(tls_ca_file=tls_ca_file,
-                   tls_worker_cert=tls_cert,
-                   tls_worker_key=tls_key,
-                   )
+    sec = Security(
+        tls_ca_file=tls_ca_file, tls_worker_cert=tls_cert, tls_worker_key=tls_key
+    )
 
     if nprocs > 1 and worker_port != 0:
-        logger.error("Failed to launch worker.  You cannot use the --port argument when nprocs > 1.")
+        logger.error(
+            "Failed to launch worker.  You cannot use the --port argument when nprocs > 1."
+        )
         exit(1)
 
     if nprocs > 1 and not nanny:
-        logger.error("Failed to launch worker.  You cannot use the --no-nanny argument when nprocs > 1.")
+        logger.error(
+            "Failed to launch worker.  You cannot use the --no-nanny argument when nprocs > 1."
+        )
         exit(1)
 
     if contact_address and not listen_address:
-        logger.error("Failed to launch worker. "
-                     "Must specify --listen-address when --contact-address is given")
+        logger.error(
+            "Failed to launch worker. "
+            "Must specify --listen-address when --contact-address is given"
+        )
         exit(1)
 
     if nprocs > 1 and listen_address:
-        logger.error("Failed to launch worker. "
-                     "You cannot specify --listen-address when nprocs > 1.")
+        logger.error(
+            "Failed to launch worker. "
+            "You cannot specify --listen-address when nprocs > 1."
+        )
         exit(1)
 
     if (worker_port or host) and listen_address:
-        logger.error("Failed to launch worker. "
-                     "You cannot specify --listen-address when --worker-port or --host is given.")
+        logger.error(
+            "Failed to launch worker. "
+            "You cannot specify --listen-address when --worker-port or --host is given."
+        )
         exit(1)
 
     try:
@@ -167,12 +277,13 @@ def main(scheduler, host, worker_port, listen_address, contact_address,
         nthreads = _ncores // nprocs
 
     if pid_file:
-        with open(pid_file, 'w') as f:
+        with open(pid_file, "w") as f:
             f.write(str(os.getpid()))
 
         def del_pid_file():
             if os.path.exists(pid_file):
                 os.remove(pid_file)
+
         atexit.register(del_pid_file)
 
     services = {}
@@ -184,14 +295,14 @@ def del_pid_file():
             pass
         else:
             if bokeh_prefix:
-                result = (BokehWorker, {'prefix': bokeh_prefix})
+                result = (BokehWorker, {"prefix": bokeh_prefix})
             else:
                 result = BokehWorker
-            services[('bokeh', dashboard_address)] = result
+            services[("bokeh", dashboard_address)] = result
 
     if resources:
-        resources = resources.replace(',', ' ').split()
-        resources = dict(pair.split('=') for pair in resources)
+        resources = resources.replace(",", " ").split()
+        resources = dict(pair.split("=") for pair in resources)
         resources = valmap(float, resources)
     else:
         resources = None
@@ -199,17 +310,19 @@ def del_pid_file():
     loop = IOLoop.current()
 
     if nanny:
-        kwargs = {'worker_port': worker_port, 'listen_address': listen_address}
+        kwargs = {"worker_port": worker_port, "listen_address": listen_address}
         t = Nanny
     else:
         kwargs = {}
         if nanny_port:
-            kwargs['service_ports'] = {'nanny': nanny_port}
+            kwargs["service_ports"] = {"nanny": nanny_port}
         t = Worker
 
-    if not scheduler and not scheduler_file and 'scheduler-address' not in config:
-        raise ValueError("Need to provide scheduler address like\n"
-                         "dask-worker SCHEDULER_ADDRESS:8786")
+    if not scheduler and not scheduler_file and "scheduler-address" not in config:
+        raise ValueError(
+            "Need to provide scheduler address like\n"
+            "dask-worker SCHEDULER_ADDRESS:8786"
+        )
 
     if interface:
         if host:
@@ -224,17 +337,29 @@ def del_pid_file():
         addr = None
 
     if death_timeout is not None:
-        death_timeout = parse_timedelta(death_timeout, 's')
-
-    nannies = [t(scheduler, scheduler_file=scheduler_file, ncores=nthreads,
-                 services=services, loop=loop, resources=resources,
-                 memory_limit=memory_limit, reconnect=reconnect,
-                 local_dir=local_directory, death_timeout=death_timeout,
-                 preload=preload, preload_argv=preload_argv,
-                 security=sec, contact_address=contact_address,
-                 name=name if nprocs == 1 or not name else name + '-' + str(i),
-                 **kwargs)
-               for i in range(nprocs)]
+        death_timeout = parse_timedelta(death_timeout, "s")
+
+    nannies = [
+        t(
+            scheduler,
+            scheduler_file=scheduler_file,
+            ncores=nthreads,
+            services=services,
+            loop=loop,
+            resources=resources,
+            memory_limit=memory_limit,
+            reconnect=reconnect,
+            local_dir=local_directory,
+            death_timeout=death_timeout,
+            preload=preload,
+            preload_argv=preload_argv,
+            security=sec,
+            contact_address=contact_address,
+            name=name if nprocs == 1 or not name else name + "-" + str(i),
+            **kwargs
+        )
+        for i in range(nprocs)
+    ]
 
     @gen.coroutine
     def close_all():
@@ -249,7 +374,7 @@ def on_signal(signum):
     @gen.coroutine
     def run():
         yield [n._start(addr) for n in nannies]
-        while all(n.status != 'closed' for n in nannies):
+        while all(n.status != "closed" for n in nannies):
             yield gen.sleep(0.2)
 
     install_signal_handlers(loop, cleanup=on_signal)
@@ -267,5 +392,5 @@ def go():
     main()
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     go()
diff --git a/distributed/cli/tests/test_cli_utils.py b/distributed/cli/tests/test_cli_utils.py
index fcc3507b028..4f07f699de5 100644
--- a/distributed/cli/tests/test_cli_utils.py
+++ b/distributed/cli/tests/test_cli_utils.py
@@ -1,7 +1,8 @@
 from __future__ import print_function, division, absolute_import
 
 import pytest
-pytest.importorskip('requests')
+
+pytest.importorskip("requests")
 
 from distributed.cli.utils import uri_from_host_port
 from distributed.utils import get_ip
@@ -13,41 +14,41 @@
 def test_uri_from_host_port():
     f = uri_from_host_port
 
-    assert f('', 456, None) == 'tcp://:456'
-    assert f('', 456, 123) == 'tcp://:456'
-    assert f('', None, 123) == 'tcp://:123'
-    assert f('', None, 0) == 'tcp://'
-    assert f('', 0, 123) == 'tcp://'
+    assert f("", 456, None) == "tcp://:456"
+    assert f("", 456, 123) == "tcp://:456"
+    assert f("", None, 123) == "tcp://:123"
+    assert f("", None, 0) == "tcp://"
+    assert f("", 0, 123) == "tcp://"
 
-    assert f('localhost', 456, None) == 'tcp://localhost:456'
-    assert f('localhost', 456, 123) == 'tcp://localhost:456'
-    assert f('localhost', None, 123) == 'tcp://localhost:123'
-    assert f('localhost', None, 0) == 'tcp://localhost'
+    assert f("localhost", 456, None) == "tcp://localhost:456"
+    assert f("localhost", 456, 123) == "tcp://localhost:456"
+    assert f("localhost", None, 123) == "tcp://localhost:123"
+    assert f("localhost", None, 0) == "tcp://localhost"
 
-    assert f('192.168.1.2', 456, None) == 'tcp://192.168.1.2:456'
-    assert f('192.168.1.2', 456, 123) == 'tcp://192.168.1.2:456'
-    assert f('192.168.1.2', None, 123) == 'tcp://192.168.1.2:123'
-    assert f('192.168.1.2', None, 0) == 'tcp://192.168.1.2'
+    assert f("192.168.1.2", 456, None) == "tcp://192.168.1.2:456"
+    assert f("192.168.1.2", 456, 123) == "tcp://192.168.1.2:456"
+    assert f("192.168.1.2", None, 123) == "tcp://192.168.1.2:123"
+    assert f("192.168.1.2", None, 0) == "tcp://192.168.1.2"
 
-    assert f('tcp://192.168.1.2', 456, None) == 'tcp://192.168.1.2:456'
-    assert f('tcp://192.168.1.2', 456, 123) == 'tcp://192.168.1.2:456'
-    assert f('tcp://192.168.1.2', None, 123) == 'tcp://192.168.1.2:123'
-    assert f('tcp://192.168.1.2', None, 0) == 'tcp://192.168.1.2'
+    assert f("tcp://192.168.1.2", 456, None) == "tcp://192.168.1.2:456"
+    assert f("tcp://192.168.1.2", 456, 123) == "tcp://192.168.1.2:456"
+    assert f("tcp://192.168.1.2", None, 123) == "tcp://192.168.1.2:123"
+    assert f("tcp://192.168.1.2", None, 0) == "tcp://192.168.1.2"
 
-    assert f('tcp://192.168.1.2:456', None, None) == 'tcp://192.168.1.2:456'
-    assert f('tcp://192.168.1.2:456', 0, 0) == 'tcp://192.168.1.2:456'
-    assert f('tcp://192.168.1.2:456', 0, 123) == 'tcp://192.168.1.2:456'
-    assert f('tcp://192.168.1.2:456', 456, 123) == 'tcp://192.168.1.2:456'
+    assert f("tcp://192.168.1.2:456", None, None) == "tcp://192.168.1.2:456"
+    assert f("tcp://192.168.1.2:456", 0, 0) == "tcp://192.168.1.2:456"
+    assert f("tcp://192.168.1.2:456", 0, 123) == "tcp://192.168.1.2:456"
+    assert f("tcp://192.168.1.2:456", 456, 123) == "tcp://192.168.1.2:456"
 
     with pytest.raises(ValueError):
         # Two incompatible port values
-        f('tcp://192.168.1.2:456', 123, None)
+        f("tcp://192.168.1.2:456", 123, None)
 
-    assert f('tls://192.168.1.2:456', None, None) == 'tls://192.168.1.2:456'
-    assert f('tls://192.168.1.2:456', 0, 0) == 'tls://192.168.1.2:456'
-    assert f('tls://192.168.1.2:456', 0, 123) == 'tls://192.168.1.2:456'
-    assert f('tls://192.168.1.2:456', 456, 123) == 'tls://192.168.1.2:456'
+    assert f("tls://192.168.1.2:456", None, None) == "tls://192.168.1.2:456"
+    assert f("tls://192.168.1.2:456", 0, 0) == "tls://192.168.1.2:456"
+    assert f("tls://192.168.1.2:456", 0, 123) == "tls://192.168.1.2:456"
+    assert f("tls://192.168.1.2:456", 456, 123) == "tls://192.168.1.2:456"
 
-    assert f('tcp://[::1]:456', None, None) == 'tcp://[::1]:456'
+    assert f("tcp://[::1]:456", None, None) == "tcp://[::1]:456"
 
-    assert f('tls://[::1]:456', None, None) == 'tls://[::1]:456'
+    assert f("tls://[::1]:456", None, None) == "tls://[::1]:456"
diff --git a/distributed/cli/tests/test_dask_mpi.py b/distributed/cli/tests/test_dask_mpi.py
index 36298ca5a83..8bc8dddca2e 100644
--- a/distributed/cli/tests/test_dask_mpi.py
+++ b/distributed/cli/tests/test_dask_mpi.py
@@ -4,7 +4,8 @@
 from time import sleep
 
 import pytest
-pytest.importorskip('mpi4py')
+
+pytest.importorskip("mpi4py")
 
 import requests
 
@@ -15,15 +16,17 @@
 from distributed.utils_test import loop  # noqa: F401
 
 
-@pytest.mark.parametrize('nanny', ['--nanny', '--no-nanny'])
+@pytest.mark.parametrize("nanny", ["--nanny", "--no-nanny"])
 def test_basic(loop, nanny):
     with tmpfile() as fn:
-        with popen(['mpirun', '--np', '4', 'dask-mpi', '--scheduler-file', fn, nanny],
-                   stdin=subprocess.DEVNULL):
+        with popen(
+            ["mpirun", "--np", "4", "dask-mpi", "--scheduler-file", fn, nanny],
+            stdin=subprocess.DEVNULL,
+        ):
             with Client(scheduler_file=fn) as c:
 
                 start = time()
-                while len(c.scheduler_info()['workers']) != 3:
+                while len(c.scheduler_info()["workers"]) != 3:
                     assert time() < start + 10
                     sleep(0.2)
 
@@ -32,36 +35,59 @@ def test_basic(loop, nanny):
 
 def test_no_scheduler(loop):
     with tmpfile() as fn:
-        with popen(['mpirun', '--np', '2', 'dask-mpi', '--scheduler-file', fn],
-                   stdin=subprocess.DEVNULL):
+        with popen(
+            ["mpirun", "--np", "2", "dask-mpi", "--scheduler-file", fn],
+            stdin=subprocess.DEVNULL,
+        ):
             with Client(scheduler_file=fn) as c:
 
                 start = time()
-                while len(c.scheduler_info()['workers']) != 1:
+                while len(c.scheduler_info()["workers"]) != 1:
                     assert time() < start + 10
                     sleep(0.2)
 
                 assert c.submit(lambda x: x + 1, 10).result() == 11
-                with popen(['mpirun', '--np', '1', 'dask-mpi',
-                            '--scheduler-file', fn, '--no-scheduler']):
+                with popen(
+                    [
+                        "mpirun",
+                        "--np",
+                        "1",
+                        "dask-mpi",
+                        "--scheduler-file",
+                        fn,
+                        "--no-scheduler",
+                    ]
+                ):
 
                     start = time()
-                    while len(c.scheduler_info()['workers']) != 2:
+                    while len(c.scheduler_info()["workers"]) != 2:
                         assert time() < start + 10
                         sleep(0.2)
 
 
 def test_bokeh(loop):
     with tmpfile() as fn:
-        with popen(['mpirun', '--np', '2', 'dask-mpi', '--scheduler-file', fn,
-                    '--bokeh-port', '59583', '--bokeh-worker-port', '59584'],
-                   stdin=subprocess.DEVNULL):
+        with popen(
+            [
+                "mpirun",
+                "--np",
+                "2",
+                "dask-mpi",
+                "--scheduler-file",
+                fn,
+                "--bokeh-port",
+                "59583",
+                "--bokeh-worker-port",
+                "59584",
+            ],
+            stdin=subprocess.DEVNULL,
+        ):
 
             for port in [59853, 59584]:
                 start = time()
                 while True:
                     try:
-                        response = requests.get('http://localhost:%d/status/' % port)
+                        response = requests.get("http://localhost:%d/status/" % port)
                         assert response.ok
                         break
                     except Exception:
@@ -69,4 +95,4 @@ def test_bokeh(loop):
                         assert time() < start + 20
 
     with pytest.raises(Exception):
-        requests.get('http://localhost:59583/status/')
+        requests.get("http://localhost:59583/status/")
diff --git a/distributed/cli/tests/test_dask_remote.py b/distributed/cli/tests/test_dask_remote.py
index 66ac4006963..04d04d62ecf 100644
--- a/distributed/cli/tests/test_dask_remote.py
+++ b/distributed/cli/tests/test_dask_remote.py
@@ -4,6 +4,6 @@
 
 def test_dask_remote():
     runner = CliRunner()
-    result = runner.invoke(main, ['--help'])
-    assert '--host TEXT     IP or hostname of this server' in result.output
+    result = runner.invoke(main, ["--help"])
+    assert "--host TEXT     IP or hostname of this server" in result.output
     assert result.exit_code == 0
diff --git a/distributed/cli/tests/test_dask_scheduler.py b/distributed/cli/tests/test_dask_scheduler.py
index 7564c3f05bd..26fe607b901 100644
--- a/distributed/cli/tests/test_dask_scheduler.py
+++ b/distributed/cli/tests/test_dask_scheduler.py
@@ -1,7 +1,8 @@
 from __future__ import print_function, division, absolute_import
 
 import pytest
-pytest.importorskip('requests')
+
+pytest.importorskip("requests")
 
 import os
 import requests
@@ -15,67 +16,67 @@
 
 from distributed import Scheduler, Client
 from distributed.utils import get_ip, get_ip_interface, tmpfile
-from distributed.utils_test import (popen,
-                                    assert_can_connect_from_everywhere_4_6,
-                                    assert_can_connect_locally_4,
-                                    )
+from distributed.utils_test import (
+    popen,
+    assert_can_connect_from_everywhere_4_6,
+    assert_can_connect_locally_4,
+)
 from distributed.utils_test import loop  # noqa: F401
 from distributed.metrics import time
 
 
 def test_defaults(loop):
-    with popen(['dask-scheduler', '--no-bokeh']) as proc:
+    with popen(["dask-scheduler", "--no-bokeh"]) as proc:
 
         @gen.coroutine
         def f():
             # Default behaviour is to listen on all addresses
-            yield [
-                assert_can_connect_from_everywhere_4_6(8786, 5.0),  # main port
-            ]
+            yield [assert_can_connect_from_everywhere_4_6(8786, 5.0)]  # main port
 
-        with Client('127.0.0.1:%d' % Scheduler.default_port, loop=loop) as c:
+        with Client("127.0.0.1:%d" % Scheduler.default_port, loop=loop) as c:
             c.sync(f)
 
     with pytest.raises(Exception):
-        requests.get('http://127.0.0.1:8787/status/')
+        requests.get("http://127.0.0.1:8787/status/")
     with pytest.raises(Exception):
-        response = requests.get('http://127.0.0.1:9786/info.json')
+        response = requests.get("http://127.0.0.1:9786/info.json")
 
 
 def test_hostport(loop):
-    with popen(['dask-scheduler', '--no-bokeh', '--host', '127.0.0.1:8978']):
+    with popen(["dask-scheduler", "--no-bokeh", "--host", "127.0.0.1:8978"]):
+
         @gen.coroutine
         def f():
             yield [
                 # The scheduler's main port can't be contacted from the outside
-                assert_can_connect_locally_4(8978, 5.0),
+                assert_can_connect_locally_4(8978, 5.0)
             ]
 
-        with Client('127.0.0.1:8978', loop=loop) as c:
+        with Client("127.0.0.1:8978", loop=loop) as c:
             assert len(c.ncores()) == 0
             c.sync(f)
 
 
 def test_no_bokeh(loop):
-    pytest.importorskip('bokeh')
-    with popen(['dask-scheduler', '--no-bokeh']) as proc:
-        with Client('127.0.0.1:%d' % Scheduler.default_port, loop=loop) as c:
+    pytest.importorskip("bokeh")
+    with popen(["dask-scheduler", "--no-bokeh"]) as proc:
+        with Client("127.0.0.1:%d" % Scheduler.default_port, loop=loop) as c:
             for i in range(3):
                 line = proc.stderr.readline()
-                assert b'bokeh' not in line.lower()
+                assert b"bokeh" not in line.lower()
             with pytest.raises(Exception):
-                requests.get('http://127.0.0.1:8787/status/')
+                requests.get("http://127.0.0.1:8787/status/")
 
 
 def test_bokeh(loop):
-    pytest.importorskip('bokeh')
+    pytest.importorskip("bokeh")
 
-    with popen(['dask-scheduler']) as proc:
-        with Client('127.0.0.1:%d' % Scheduler.default_port, loop=loop) as c:
+    with popen(["dask-scheduler"]) as proc:
+        with Client("127.0.0.1:%d" % Scheduler.default_port, loop=loop) as c:
             pass
 
-        names = ['localhost', '127.0.0.1', get_ip()]
-        if 'linux' in sys.platform:
+        names = ["localhost", "127.0.0.1", get_ip()]
+        if "linux" in sys.platform:
             names.append(socket.gethostname())
 
         start = time()
@@ -83,58 +84,66 @@ def test_bokeh(loop):
             try:
                 # All addresses should respond
                 for name in names:
-                    uri = 'http://%s:8787/status/' % name
+                    uri = "http://%s:8787/status/" % name
                     response = requests.get(uri)
                     assert response.ok
                 break
             except Exception as f:
-                print('got error on %r: %s' % (uri, f))
+                print("got error on %r: %s" % (uri, f))
                 sleep(0.1)
                 assert time() < start + 10
 
     with pytest.raises(Exception):
-        requests.get('http://127.0.0.1:8787/status/')
+        requests.get("http://127.0.0.1:8787/status/")
 
 
 def test_bokeh_non_standard_ports(loop):
-    pytest.importorskip('bokeh')
+    pytest.importorskip("bokeh")
 
-    with popen(['dask-scheduler',
-                '--port', '3448',
-                '--dashboard-address', ':4832']) as proc:
-        with Client('127.0.0.1:3448', loop=loop) as c:
+    with popen(
+        ["dask-scheduler", "--port", "3448", "--dashboard-address", ":4832"]
+    ) as proc:
+        with Client("127.0.0.1:3448", loop=loop) as c:
             pass
 
         start = time()
         while True:
             try:
-                response = requests.get('http://localhost:4832/status/')
+                response = requests.get("http://localhost:4832/status/")
                 assert response.ok
                 break
             except Exception:
                 sleep(0.1)
                 assert time() < start + 20
     with pytest.raises(Exception):
-        requests.get('http://localhost:4832/status/')
+        requests.get("http://localhost:4832/status/")
 
 
-@pytest.mark.skipif(not sys.platform.startswith('linux'),
-                    reason="Need 127.0.0.2 to mean localhost")
+@pytest.mark.skipif(
+    not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
+)
 def test_bokeh_whitelist(loop):
-    pytest.importorskip('bokeh')
+    pytest.importorskip("bokeh")
     with pytest.raises(Exception):
-        requests.get('http://localhost:8787/status/').ok
-
-    with popen(['dask-scheduler', '--bokeh-whitelist', '127.0.0.2:8787',
-                                  '--bokeh-whitelist', '127.0.0.3:8787']) as proc:
-        with Client('127.0.0.1:%d' % Scheduler.default_port, loop=loop) as c:
+        requests.get("http://localhost:8787/status/").ok
+
+    with popen(
+        [
+            "dask-scheduler",
+            "--bokeh-whitelist",
+            "127.0.0.2:8787",
+            "--bokeh-whitelist",
+            "127.0.0.3:8787",
+        ]
+    ) as proc:
+        with Client("127.0.0.1:%d" % Scheduler.default_port, loop=loop) as c:
             pass
 
         start = time()
         while True:
             try:
-                for name in ['127.0.0.2', '127.0.0.3']:
-                    response = requests.get('http://%s:8787/status/' % name)
+                for name in ["127.0.0.2", "127.0.0.3"]:
+                    response = requests.get("http://%s:8787/status/" % name)
                     assert response.ok
                 break
             except Exception as f:
@@ -144,10 +153,10 @@ def test_bokeh_whitelist(loop):
 
 
 def test_multiple_workers(loop):
-    with popen(['dask-scheduler', '--no-bokeh']) as s:
-        with popen(['dask-worker', 'localhost:8786', '--no-bokeh']) as a:
-            with popen(['dask-worker', 'localhost:8786', '--no-bokeh']) as b:
-                with Client('127.0.0.1:%d' % Scheduler.default_port, loop=loop) as c:
+    with popen(["dask-scheduler", "--no-bokeh"]) as s:
+        with popen(["dask-worker", "localhost:8786", "--no-bokeh"]) as a:
+            with popen(["dask-worker", "localhost:8786", "--no-bokeh"]) as b:
+                with Client("127.0.0.1:%d" % Scheduler.default_port, loop=loop) as c:
                     start = time()
                     while len(c.ncores()) < 2:
                         sleep(0.1)
@@ -155,7 +164,7 @@ def test_multiple_workers(loop):
 
 
 def test_interface(loop):
-    psutil = pytest.importorskip('psutil')
+    psutil = pytest.importorskip("psutil")
     if_names = sorted(psutil.net_if_addrs())
     for if_name in if_names:
         try:
@@ -163,23 +172,26 @@ def test_interface(loop):
         except ValueError:
             pass
         else:
-            if ipv4_addr == '127.0.0.1':
+            if ipv4_addr == "127.0.0.1":
                 break
     else:
-        pytest.skip("Could not find loopback interface. "
-                    "Available interfaces are: %s." % (if_names,))
-
-    with popen(['dask-scheduler', '--no-bokeh', '--interface', if_name]) as s:
-        with popen(['dask-worker', '127.0.0.1:8786', '--no-bokeh', '--interface', if_name]) as a:
-            with Client('tcp://127.0.0.1:%d' % Scheduler.default_port, loop=loop) as c:
+        pytest.skip(
+            "Could not find loopback interface. "
+            "Available interfaces are: %s." % (if_names,)
+        )
+
+    with popen(["dask-scheduler", "--no-bokeh", "--interface", if_name]) as s:
+        with popen(
+            ["dask-worker", "127.0.0.1:8786", "--no-bokeh", "--interface", if_name]
+        ) as a:
+            with Client("tcp://127.0.0.1:%d" % Scheduler.default_port, loop=loop) as c:
                 start = time()
                 while not len(c.ncores()):
                     sleep(0.1)
                     assert time() - start < 5
                 info = c.scheduler_info()
-                assert 'tcp://127.0.0.1' in info['address']
-                assert all('127.0.0.1' == d['host']
-                           for d in info['workers'].values())
+                assert "tcp://127.0.0.1" in info["address"]
+                assert all("127.0.0.1" == d["host"] for d in info["workers"].values())
 
 
 def test_pid_file(loop):
@@ -197,7 +209,7 @@ def check_pidfile(proc, pidfile):
             with open(pidfile) as f:
                 text = f.read()
         pid = int(text)
-        if sys.platform.startswith('win'):
+        if sys.platform.startswith("win"):
             # On Windows, `dask-XXX` invokes the dask-XXX.exe
             # shim, but the PID is written out by the child Python process
             assert pid
@@ -205,35 +217,37 @@ def check_pidfile(proc, pidfile):
             assert proc.pid == pid
 
     with tmpfile() as s:
-        with popen(['dask-scheduler', '--pid-file', s, '--no-bokeh']) as sched:
+        with popen(["dask-scheduler", "--pid-file", s, "--no-bokeh"]) as sched:
             check_pidfile(sched, s)
 
         with tmpfile() as w:
-            with popen(['dask-worker', '127.0.0.1:8786', '--pid-file', w,
-                        '--no-bokeh']) as worker:
+            with popen(
+                ["dask-worker", "127.0.0.1:8786", "--pid-file", w, "--no-bokeh"]
+            ) as worker:
                 check_pidfile(worker, w)
 
 
 def test_scheduler_port_zero(loop):
     with tmpfile() as fn:
-        with popen(['dask-scheduler', '--no-bokeh', '--scheduler-file', fn,
-                    '--port', '0']) as sched:
+        with popen(
+            ["dask-scheduler", "--no-bokeh", "--scheduler-file", fn, "--port", "0"]
+        ) as sched:
             with Client(scheduler_file=fn, loop=loop) as c:
                 assert c.scheduler.port
                 assert c.scheduler.port != 8786
 
 
 def test_bokeh_port_zero(loop):
-    pytest.importorskip('bokeh')
+    pytest.importorskip("bokeh")
     with tmpfile() as fn:
-        with popen(['dask-scheduler', '--dashboard-address', ':0']) as proc:
+        with popen(["dask-scheduler", "--dashboard-address", ":0"]) as proc:
             count = 0
             while count < 1:
                 line = proc.stderr.readline()
-                if b'bokeh' in line.lower() or b'web' in line.lower():
+                if b"bokeh" in line.lower() or b"web" in line.lower():
                     sleep(0.01)
                     count += 1
-                    assert b':0' not in line
+                    assert b":0" not in line
 
 
 PRELOAD_TEXT = """
@@ -248,49 +262,53 @@ def get_scheduler_address():
 
 
 def test_preload_file(loop):
-
     def check_scheduler():
         import scheduler_info
+
         return scheduler_info.get_scheduler_address()
 
     tmpdir = tempfile.mkdtemp()
     try:
-        path = os.path.join(tmpdir, 'scheduler_info.py')
-        with open(path, 'w') as f:
+        path = os.path.join(tmpdir, "scheduler_info.py")
+        with open(path, "w") as f:
             f.write(PRELOAD_TEXT)
         with tmpfile() as fn:
-            with popen(['dask-scheduler', '--scheduler-file', fn,
-                        '--preload', path]):
+            with popen(["dask-scheduler", "--scheduler-file", fn, "--preload", path]):
                 with Client(scheduler_file=fn, loop=loop) as c:
-                    assert c.run_on_scheduler(check_scheduler) == \
-                        c.scheduler.address
+                    assert c.run_on_scheduler(check_scheduler) == c.scheduler.address
     finally:
         shutil.rmtree(tmpdir)
 
 
 def test_preload_module(loop):
-
     def check_scheduler():
         import scheduler_info
+
         return scheduler_info.get_scheduler_address()
 
     tmpdir = tempfile.mkdtemp()
     try:
-        path = os.path.join(tmpdir, 'scheduler_info.py')
-        with open(path, 'w') as f:
+        path = os.path.join(tmpdir, "scheduler_info.py")
+        with open(path, "w") as f:
             f.write(PRELOAD_TEXT)
         env = os.environ.copy()
-        if 'PYTHONPATH' in env:
-            env['PYTHONPATH'] = tmpdir + ':' + env['PYTHONPATH']
+        if "PYTHONPATH" in env:
+            env["PYTHONPATH"] = tmpdir + ":" + env["PYTHONPATH"]
         else:
-            env['PYTHONPATH'] = tmpdir
+            env["PYTHONPATH"] = tmpdir
         with tmpfile() as fn:
-            with popen(['dask-scheduler', '--scheduler-file', fn,
-                        '--preload', 'scheduler_info'],
-                       env=env):
+            with popen(
+                [
+                    "dask-scheduler",
+                    "--scheduler-file",
+                    fn,
+                    "--preload",
+                    "scheduler_info",
+                ],
+                env=env,
+            ):
                 with Client(scheduler_file=fn, loop=loop) as c:
-                    assert c.run_on_scheduler(check_scheduler) == \
-                        c.scheduler.address
+                    assert c.run_on_scheduler(check_scheduler) == c.scheduler.address
     finally:
         shutil.rmtree(tmpdir)
 
@@ -310,47 +328,57 @@ def get_passthrough():
 
 
 def test_preload_command(loop):
-
     def check_passthrough():
         import passthrough_info
+
         return passthrough_info.get_passthrough()
 
     tmpdir = tempfile.mkdtemp()
     try:
-        path = os.path.join(tmpdir, 'passthrough_info.py')
-        with open(path, 'w') as f:
+        path = os.path.join(tmpdir, "passthrough_info.py")
+        with open(path, "w") as f:
             f.write(PRELOAD_COMMAND_TEXT)
 
         with tmpfile() as fn:
             print(fn)
-            with popen(['dask-scheduler', '--scheduler-file', fn,
-                        '--preload', path, "--passthrough", "foobar"]):
+            with popen(
+                [
+                    "dask-scheduler",
+                    "--scheduler-file",
+                    fn,
+                    "--preload",
+                    path,
+                    "--passthrough",
+                    "foobar",
+                ]
+            ):
                 with Client(scheduler_file=fn, loop=loop) as c:
-                    assert c.run_on_scheduler(check_passthrough) == \
-                        "foobar"
+                    assert c.run_on_scheduler(check_passthrough) == "foobar"
     finally:
         shutil.rmtree(tmpdir)
 
 
 def test_preload_command_default(loop):
-
     def check_passthrough():
         import passthrough_info
+
         return passthrough_info.get_passthrough()
 
     tmpdir = tempfile.mkdtemp()
     try:
-        path = os.path.join(tmpdir, 'passthrough_info.py')
-        with open(path, 'w') as f:
+        path = os.path.join(tmpdir, "passthrough_info.py")
+        with open(path, "w") as f:
             f.write(PRELOAD_COMMAND_TEXT)
 
         with tmpfile() as fn2:
             print(fn2)
-            with popen(['dask-scheduler', '--scheduler-file', fn2,
-                        '--preload', path], stdout=sys.stdout, stderr=sys.stderr):
+            with popen(
+                ["dask-scheduler", "--scheduler-file", fn2, "--preload", path],
+                stdout=sys.stdout,
+                stderr=sys.stderr,
+            ):
                 with Client(scheduler_file=fn2, loop=loop) as c:
-                    assert c.run_on_scheduler(check_passthrough) == \
-                        "default"
+                    assert c.run_on_scheduler(check_passthrough) == "default"
 
     finally:
         shutil.rmtree(tmpdir)
diff --git a/distributed/cli/tests/test_dask_submit.py b/distributed/cli/tests/test_dask_submit.py
index 9f3aef0ed34..83c7c1067fa 100644
--- a/distributed/cli/tests/test_dask_submit.py
+++ b/distributed/cli/tests/test_dask_submit.py
@@ -4,6 +4,6 @@
 
 def test_submit_runs_as_a_cli():
     runner = CliRunner()
-    result = runner.invoke(main, ['--help'])
+    result = runner.invoke(main, ["--help"])
     assert result.exit_code == 0
-    assert 'Usage: main [OPTIONS] REMOTE_CLIENT_ADDRESS FILEPATH' in result.output
+    assert "Usage: main [OPTIONS] REMOTE_CLIENT_ADDRESS FILEPATH" in result.output
diff --git a/distributed/cli/tests/test_dask_worker.py b/distributed/cli/tests/test_dask_worker.py
index 5797905c7cd..72084e53141 100644
--- a/distributed/cli/tests/test_dask_worker.py
+++ b/distributed/cli/tests/test_dask_worker.py
@@ -1,7 +1,8 @@
 from __future__ import print_function, division, absolute_import
 
 import pytest
-pytest.importorskip('requests')
+
+pytest.importorskip("requests")
 
 import requests
 import sys
@@ -11,56 +12,73 @@
 from distributed import Client
 from distributed.metrics import time
 from distributed.utils import sync, tmpfile
-from distributed.utils_test import (popen, slow, terminate_process,
-                                    wait_for_port)
+from distributed.utils_test import popen, slow, terminate_process, wait_for_port
 from distributed.utils_test import loop  # noqa: F401
 
 
 def test_nanny_worker_ports(loop):
-    with popen(['dask-scheduler', '--port', '9359', '--no-bokeh']) as sched:
-        with popen(['dask-worker', '127.0.0.1:9359', '--host', '127.0.0.1',
-                    '--worker-port', '9684', '--nanny-port', '5273',
-                    '--no-bokeh']) as worker:
-            with Client('127.0.0.1:9359', loop=loop) as c:
+    with popen(["dask-scheduler", "--port", "9359", "--no-bokeh"]) as sched:
+        with popen(
+            [
+                "dask-worker",
+                "127.0.0.1:9359",
+                "--host",
+                "127.0.0.1",
+                "--worker-port",
+                "9684",
+                "--nanny-port",
+                "5273",
+                "--no-bokeh",
+            ]
+        ) as worker:
+            with Client("127.0.0.1:9359", loop=loop) as c:
                 start = time()
                 while True:
                     d = sync(c.loop, c.scheduler.identity)
-                    if d['workers']:
+                    if d["workers"]:
                         break
                     else:
                         assert time() - start < 5
                         sleep(0.1)
-                assert d['workers']['tcp://127.0.0.1:9684']['services']['nanny'] == 5273
+                assert d["workers"]["tcp://127.0.0.1:9684"]["services"]["nanny"] == 5273
 
 
 def test_memory_limit(loop):
-    with popen(['dask-scheduler', '--no-bokeh']) as sched:
-        with popen(['dask-worker', '127.0.0.1:8786', '--memory-limit', '2e3MB',
-                    '--no-bokeh']) as worker:
-            with Client('127.0.0.1:8786', loop=loop) as c:
+    with popen(["dask-scheduler", "--no-bokeh"]) as sched:
+        with popen(
+            ["dask-worker", "127.0.0.1:8786", "--memory-limit", "2e3MB", "--no-bokeh"]
+        ) as worker:
+            with Client("127.0.0.1:8786", loop=loop) as c:
                 while not c.ncores():
                     sleep(0.1)
                 info = c.scheduler_info()
-                d = first(info['workers'].values())
-                assert isinstance(d['memory_limit'], int)
-                assert d['memory_limit'] == 2e9
+                d = first(info["workers"].values())
+                assert isinstance(d["memory_limit"], int)
+                assert d["memory_limit"] == 2e9
 
 
 def test_no_nanny(loop):
-    with popen(['dask-scheduler', '--no-bokeh']) as sched:
-        with popen(['dask-worker', '127.0.0.1:8786', '--no-nanny',
-                    '--no-bokeh']) as worker:
-            assert any(b'Registered' in worker.stderr.readline()
-                       for i in range(15))
+    with popen(["dask-scheduler", "--no-bokeh"]) as sched:
+        with popen(
+            ["dask-worker", "127.0.0.1:8786", "--no-nanny", "--no-bokeh"]
+        ) as worker:
+            assert any(b"Registered" in worker.stderr.readline() for i in range(15))
 
 
 @slow
-@pytest.mark.parametrize('nanny', ['--nanny', '--no-nanny'])
+@pytest.mark.parametrize("nanny", ["--nanny", "--no-nanny"])
 def test_no_reconnect(nanny, loop):
-    with popen(['dask-scheduler', '--no-bokeh']) as sched:
-        wait_for_port(('127.0.0.1', 8786))
-        with popen(['dask-worker', 'tcp://127.0.0.1:8786', '--no-reconnect', nanny,
-                    '--no-bokeh']) as worker:
+    with popen(["dask-scheduler", "--no-bokeh"]) as sched:
+        wait_for_port(("127.0.0.1", 8786))
+        with popen(
+            [
+                "dask-worker",
+                "tcp://127.0.0.1:8786",
+                "--no-reconnect",
+                nanny,
+                "--no-bokeh",
+            ]
+        ) as worker:
             sleep(2)
             terminate_process(sched)
         start = time()
@@ -70,88 +88,116 @@ def test_no_reconnect(nanny, loop):
 
 
 def test_resources(loop):
-    with popen(['dask-scheduler', '--no-bokeh']) as sched:
-        with popen(['dask-worker', 'tcp://127.0.0.1:8786', '--no-bokeh',
-                    '--resources', 'A=1 B=2,C=3']) as worker:
-            with Client('127.0.0.1:8786', loop=loop) as c:
-                while not c.scheduler_info()['workers']:
+    with popen(["dask-scheduler", "--no-bokeh"]) as sched:
+        with popen(
+            [
+                "dask-worker",
+                "tcp://127.0.0.1:8786",
+                "--no-bokeh",
+                "--resources",
+                "A=1 B=2,C=3",
+            ]
+        ) as worker:
+            with Client("127.0.0.1:8786", loop=loop) as c:
+                while not c.scheduler_info()["workers"]:
                     sleep(0.1)
                 info = c.scheduler_info()
-                worker = list(info['workers'].values())[0]
-                assert worker['resources'] == {'A': 1, 'B': 2, 'C': 3}
+                worker = list(info["workers"].values())[0]
+                assert worker["resources"] == {"A": 1, "B": 2, "C": 3}
 
 
-@pytest.mark.parametrize('nanny', ['--nanny', '--no-nanny'])
+@pytest.mark.parametrize("nanny", ["--nanny", "--no-nanny"])
 def test_local_directory(loop, nanny):
     with tmpfile() as fn:
-        with popen(['dask-scheduler', '--no-bokeh']) as sched:
-            with popen(['dask-worker', '127.0.0.1:8786', nanny,
-                        '--no-bokeh', '--local-directory', fn]) as worker:
-                with Client('127.0.0.1:8786', loop=loop, timeout=10) as c:
+        with popen(["dask-scheduler", "--no-bokeh"]) as sched:
+            with popen(
+                [
+                    "dask-worker",
+                    "127.0.0.1:8786",
+                    nanny,
+                    "--no-bokeh",
+                    "--local-directory",
+                    fn,
+                ]
+            ) as worker:
+                with Client("127.0.0.1:8786", loop=loop, timeout=10) as c:
                     start = time()
-                    while not c.scheduler_info()['workers']:
+                    while not c.scheduler_info()["workers"]:
                         sleep(0.1)
                         assert time() < start + 8
                     info = c.scheduler_info()
-                    worker = list(info['workers'].values())[0]
-                    assert worker['local_directory'].startswith(fn)
+                    worker = list(info["workers"].values())[0]
+                    assert worker["local_directory"].startswith(fn)
 
 
-@pytest.mark.parametrize('nanny', ['--nanny', '--no-nanny'])
+@pytest.mark.parametrize("nanny", ["--nanny", "--no-nanny"])
 def test_scheduler_file(loop, nanny):
     with tmpfile() as fn:
-        with popen(['dask-scheduler', '--no-bokeh', '--scheduler-file', fn]) as sched:
-            with popen(['dask-worker', '--scheduler-file', fn, nanny, '--no-bokeh']):
+        with popen(["dask-scheduler", "--no-bokeh", "--scheduler-file", fn]) as sched:
+            with popen(["dask-worker", "--scheduler-file", fn, nanny, "--no-bokeh"]):
                 with Client(scheduler_file=fn, loop=loop) as c:
                     start = time()
-                    while not c.scheduler_info()['workers']:
+                    while not c.scheduler_info()["workers"]:
                         sleep(0.1)
                         assert time() < start + 10
 
 
 def test_nprocs_requires_nanny(loop):
-    with popen(['dask-scheduler', '--no-bokeh']) as sched:
-        with popen(['dask-worker', '127.0.0.1:8786', '--nprocs=2',
-                    '--no-nanny']) as worker:
-            assert any(b'Failed to launch worker' in worker.stderr.readline()
-                       for i in range(15))
+    with popen(["dask-scheduler", "--no-bokeh"]) as sched:
+        with popen(
+            ["dask-worker", "127.0.0.1:8786", "--nprocs=2", "--no-nanny"]
+        ) as worker:
+            assert any(
+                b"Failed to launch worker" in worker.stderr.readline()
+                for i in range(15)
+            )
 
 
 def test_nprocs_expands_name(loop):
-    with popen(['dask-scheduler', '--no-bokeh']) as sched:
-        with popen(['dask-worker', '127.0.0.1:8786', '--nprocs', '2',
-                    '--name', 'foo']) as worker:
-            with popen(['dask-worker', '127.0.0.1:8786', '--nprocs', '2']) as worker:
-                with Client('tcp://127.0.0.1:8786', loop=loop) as c:
+    with popen(["dask-scheduler", "--no-bokeh"]) as sched:
+        with popen(
+            ["dask-worker", "127.0.0.1:8786", "--nprocs", "2", "--name", "foo"]
+        ) as worker:
+            with popen(["dask-worker", "127.0.0.1:8786", "--nprocs", "2"]) as worker:
+                with Client("tcp://127.0.0.1:8786", loop=loop) as c:
                     start = time()
-                    while len(c.scheduler_info()['workers']) < 4:
+                    while len(c.scheduler_info()["workers"]) < 4:
                         sleep(0.2)
                         assert time() < start + 10
 
                     info = c.scheduler_info()
-                    names = [d['name'] for d in info['workers'].values()]
-                    foos = [n for n in names if n.startswith('foo')]
+                    names = [d["name"] for d in info["workers"].values()]
+                    foos = [n for n in names if n.startswith("foo")]
                     assert len(foos) == 2
                     assert len(set(names)) == 4
 
 
-@pytest.mark.skipif(not sys.platform.startswith('linux'),
-                    reason="Need 127.0.0.2 to mean localhost")
-@pytest.mark.parametrize('nanny', ['--nanny', '--no-nanny'])
-@pytest.mark.parametrize('listen_address', [
-    'tcp://0.0.0.0:39837',
-    'tcp://127.0.0.2:39837'])
+@pytest.mark.skipif(
+    not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
+)
+@pytest.mark.parametrize("nanny", ["--nanny", "--no-nanny"])
+@pytest.mark.parametrize(
+    "listen_address", ["tcp://0.0.0.0:39837", "tcp://127.0.0.2:39837"]
+)
 def test_contact_listen_address(loop, nanny, listen_address):
-    with popen(['dask-scheduler', '--no-bokeh']) as sched:
-        with popen(['dask-worker', '127.0.0.1:8786',
-                    nanny, '--no-bokeh',
-                    '--contact-address', 'tcp://127.0.0.2:39837',
-                    '--listen-address', listen_address]) as worker:
-            with Client('127.0.0.1:8786') as client:
+    with popen(["dask-scheduler", "--no-bokeh"]) as sched:
+        with popen(
+            [
+                "dask-worker",
+                "127.0.0.1:8786",
+                nanny,
+                "--no-bokeh",
+                "--contact-address",
+                "tcp://127.0.0.2:39837",
+                "--listen-address",
+                listen_address,
+            ]
+        ) as worker:
+            with Client("127.0.0.1:8786") as client:
                 while not client.ncores():
                     sleep(0.1)
                 info = client.scheduler_info()
-                assert 'tcp://127.0.0.2:39837' in info['workers']
+                assert "tcp://127.0.0.2:39837" in info["workers"]
 
                 # roundtrip works
                 assert client.submit(lambda x: x + 1, 10).result() == 11
@@ -159,19 +205,20 @@ def test_contact_listen_address(loop, nanny, listen_address):
                 def func(dask_worker):
                     return dask_worker.listener.listen_address
 
-                assert client.run(func) == {'tcp://127.0.0.2:39837': listen_address}
+                assert client.run(func) == {"tcp://127.0.0.2:39837": listen_address}
 
 
-@pytest.mark.skipif(not sys.platform.startswith('linux'),
-                    reason="Need 127.0.0.2 to mean localhost")
-@pytest.mark.parametrize('nanny', ['--nanny', '--no-nanny'])
-@pytest.mark.parametrize('host', ['127.0.0.2', '0.0.0.0'])
+@pytest.mark.skipif(
+    not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
+)
+@pytest.mark.parametrize("nanny", ["--nanny", "--no-nanny"])
+@pytest.mark.parametrize("host", ["127.0.0.2", "0.0.0.0"])
 def test_respect_host_listen_address(loop, nanny, host):
-    with popen(['dask-scheduler', '--no-bokeh']) as sched:
-        with popen(['dask-worker', '127.0.0.1:8786',
-                    nanny, '--no-bokeh',
-                    '--host', host]) as worker:
-            with Client('127.0.0.1:8786') as client:
+    with popen(["dask-scheduler", "--no-bokeh"]) as sched:
+        with popen(
+            ["dask-worker", "127.0.0.1:8786", nanny, "--no-bokeh", "--host", host]
+        ) as worker:
+            with Client("127.0.0.1:8786") as client:
                 while not client.ncores():
                     sleep(0.1)
                 info = client.scheduler_info()
@@ -187,22 +234,23 @@ def func(dask_worker):
 
 
 def test_bokeh_non_standard_ports(loop):
-    pytest.importorskip('bokeh')
+    pytest.importorskip("bokeh")
 
-    with popen(['dask-scheduler', '--port', '3449', '--no-bokeh']):
-        with popen(['dask-worker', 'tcp://127.0.0.1:3449',
-                    '--dashboard-address', ':4833']) as proc:
-            with Client('127.0.0.1:3449', loop=loop) as c:
+    with popen(["dask-scheduler", "--port", "3449", "--no-bokeh"]):
+        with popen(
+            ["dask-worker", "tcp://127.0.0.1:3449", "--dashboard-address", ":4833"]
+        ) as proc:
+            with Client("127.0.0.1:3449", loop=loop) as c:
                 pass
 
             start = time()
             while True:
                 try:
-                    response = requests.get('http://127.0.0.1:4833/main')
+                    response = requests.get("http://127.0.0.1:4833/main")
                     assert response.ok
                     break
                 except Exception:
                     sleep(0.5)
                     assert time() < start + 20
         with pytest.raises(Exception):
-            requests.get('http://localhost:4833/status/')
+            requests.get("http://localhost:4833/status/")
diff --git a/distributed/cli/tests/test_tls_cli.py b/distributed/cli/tests/test_tls_cli.py
index 017a982d39e..d983039c962 100644
--- a/distributed/cli/tests/test_tls_cli.py
+++ b/distributed/cli/tests/test_tls_cli.py
@@ -4,20 +4,25 @@
 
 
 from distributed import Client
-from distributed.utils_test import (popen, get_cert, new_config_file,
-                                    tls_security, tls_only_config)
+from distributed.utils_test import (
+    popen,
+    get_cert,
+    new_config_file,
+    tls_security,
+    tls_only_config,
+)
 from distributed.utils_test import loop  # noqa: F401
 from distributed.metrics import time
 
 
-ca_file = get_cert('tls-ca-cert.pem')
-cert = get_cert('tls-cert.pem')
-key = get_cert('tls-key.pem')
-keycert = get_cert('tls-key-cert.pem')
+ca_file = get_cert("tls-ca-cert.pem")
+cert = get_cert("tls-cert.pem")
+key = get_cert("tls-key.pem")
+keycert = get_cert("tls-key-cert.pem")
 
 
-tls_args = ['--tls-ca-file', ca_file, '--tls-cert', keycert]
-tls_args_2 = ['--tls-ca-file', ca_file, '--tls-cert', cert, '--tls-key', key]
+tls_args = ["--tls-ca-file", ca_file, "--tls-cert", keycert]
+tls_args_2 = ["--tls-ca-file", ca_file, "--tls-cert", cert, "--tls-key", key]
 
 
 def wait_for_cores(c, ncores=1):
@@ -28,33 +33,43 @@ def wait_for_cores(c, ncores=1):
 
 
 def test_basic(loop):
-    with popen(['dask-scheduler', '--no-bokeh'] + tls_args) as s:
-        with popen(['dask-worker', '--no-bokeh', 'tls://127.0.0.1:8786'] + tls_args) as w:
-            with Client('tls://127.0.0.1:8786', loop=loop,
-                        security=tls_security()) as c:
+    with popen(["dask-scheduler", "--no-bokeh"] + tls_args) as s:
+        with popen(
+            ["dask-worker", "--no-bokeh", "tls://127.0.0.1:8786"] + tls_args
+        ) as w:
+            with Client(
+                "tls://127.0.0.1:8786", loop=loop, security=tls_security()
+            ) as c:
                 wait_for_cores(c)
 
 
 def test_nanny(loop):
-    with popen(['dask-scheduler', '--no-bokeh'] + tls_args) as s:
-        with popen(['dask-worker', '--no-bokeh', '--nanny', 'tls://127.0.0.1:8786'] + tls_args) as w:
-            with Client('tls://127.0.0.1:8786', loop=loop,
-                        security=tls_security()) as c:
+    with popen(["dask-scheduler", "--no-bokeh"] + tls_args) as s:
+        with popen(
+            ["dask-worker", "--no-bokeh", "--nanny", "tls://127.0.0.1:8786"] + tls_args
+        ) as w:
+            with Client(
+                "tls://127.0.0.1:8786", loop=loop, security=tls_security()
+            ) as c:
                 wait_for_cores(c)
 
 
 def test_separate_key_cert(loop):
-    with popen(['dask-scheduler', '--no-bokeh'] + tls_args_2) as s:
-        with popen(['dask-worker', '--no-bokeh', 'tls://127.0.0.1:8786'] + tls_args_2) as w:
-            with Client('tls://127.0.0.1:8786', loop=loop,
-                        security=tls_security()) as c:
+    with popen(["dask-scheduler", "--no-bokeh"] + tls_args_2) as s:
+        with popen(
+            ["dask-worker", "--no-bokeh", "tls://127.0.0.1:8786"] + tls_args_2
+        ) as w:
+            with Client(
+                "tls://127.0.0.1:8786", loop=loop, security=tls_security()
+            ) as c:
                 wait_for_cores(c)
 
 
 def test_use_config_file(loop):
     with new_config_file(tls_only_config()):
-        with popen(['dask-scheduler', '--no-bokeh', '--host', 'tls://']) as s:
-            with popen(['dask-worker', '--no-bokeh', 'tls://127.0.0.1:8786']) as w:
-                with Client('tls://127.0.0.1:8786', loop=loop,
-                            security=tls_security()) as c:
+        with popen(["dask-scheduler", "--no-bokeh", "--host", "tls://"]) as s:
+            with popen(["dask-worker", "--no-bokeh", "tls://127.0.0.1:8786"]) as w:
+                with Client(
+                    "tls://127.0.0.1:8786", loop=loop, security=tls_security()
+                ) as c:
                     wait_for_cores(c)
diff --git a/distributed/cli/utils.py b/distributed/cli/utils.py
index 86250fd21ae..4ce1d845821 100644
--- a/distributed/cli/utils.py
+++ b/distributed/cli/utils.py
@@ -3,8 +3,12 @@
 from tornado import gen
 from tornado.ioloop import IOLoop
 
-from distributed.comm import (parse_address, unparse_address,
-                              parse_host_port, unparse_host_port)
+from distributed.comm import (
+    parse_address,
+    unparse_address,
+    parse_host_port,
+    unparse_host_port,
+)
 
 
 py3_err_msg = """
@@ -30,13 +34,16 @@ def check_python_3():
     """Ensures that the environment is good for unicode on Python 3."""
     # https://github.com/pallets/click/issues/448#issuecomment-246029304
     import click.core
+
     click.core._verify_python3_env = lambda: None
 
     try:
         from click import _unicodefun
+
         _unicodefun._verify_python3_env()
     except (TypeError, RuntimeError) as e:
         import click
+
         click.echo(py3_err_msg, err=True)
 
 
@@ -78,16 +85,20 @@ def uri_from_host_port(host_arg, port_arg, default_port):
     # Much of distributed depends on a well-known IP being assigned to
     # each entity (Worker, Scheduler, etc.), so avoid "universal" addresses
     # like '' which would listen on all registered IPs and interfaces.
-    scheme, loc = parse_address(host_arg or '')
+    scheme, loc = parse_address(host_arg or "")
 
-    host, port = parse_host_port(loc, port_arg if port_arg is not None else default_port)
+    host, port = parse_host_port(
+        loc, port_arg if port_arg is not None else default_port
+    )
 
     if port is None and port_arg is None:
         port_arg = default_port
 
     if port and port_arg and port != port_arg:
-        raise ValueError("port number given twice in options: "
-                         "host %r and port %r" % (host_arg, port_arg))
+        raise ValueError(
+            "port number given twice in options: "
+            "host %r and port %r" % (host_arg, port_arg)
+        )
     if port is None and port_arg is not None:
         port = port_arg
     # Note `port = 0` means "choose a random port"
diff --git a/distributed/client.py b/distributed/client.py
index 47bccc7fc82..96d20a7ece2 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -29,6 +29,7 @@
 from dask.optimization import SubgraphCallable
 from dask.compatibility import apply, unicode
 from dask.utils import ensure_dict
+
 try:
     from cytoolz import first, groupby, merge, valmap, keymap
 except ImportError:
@@ -44,11 +45,22 @@
 from tornado.queues import Queue
 
 from .batched import BatchedSend
-from .utils_comm import (WrappedKey, unpack_remotedata, pack_data,
-                         scatter_to_workers, gather_from_workers)
+from .utils_comm import (
+    WrappedKey,
+    unpack_remotedata,
+    pack_data,
+    scatter_to_workers,
+    gather_from_workers,
+)
 from .cfexecutor import ClientExecutor
-from .compatibility import (Queue as pyQueue, Empty, isqueue, html_escape,
-        StopAsyncIteration, Iterator)
+from .compatibility import (
+    Queue as pyQueue,
+    Empty,
+    isqueue,
+    html_escape,
+    StopAsyncIteration,
+    Iterator,
+)
 from .core import connect, rpc, clean_exception, CommClosedError, PooledRPCCall
 from .metrics import time
 from .node import Node
@@ -60,10 +72,26 @@
 from .sizeof import sizeof
 from .threadpoolexecutor import rejoin
 from .worker import dumps_task, get_client, get_worker, secede
-from .utils import (All, sync, funcname, ignoring, queue_to_iterator,
-                    tokey, log_errors, str_graph, key_split, format_bytes, asciitable,
-                    thread_state, no_default, PeriodicCallback, LoopRunner,
-                    parse_timedelta, shutting_down, Any)
+from .utils import (
+    All,
+    sync,
+    funcname,
+    ignoring,
+    queue_to_iterator,
+    tokey,
+    log_errors,
+    str_graph,
+    key_split,
+    format_bytes,
+    asciitable,
+    thread_state,
+    no_default,
+    PeriodicCallback,
+    LoopRunner,
+    parse_timedelta,
+    shutting_down,
+    Any,
+)
 from .versions import get_versions
 
 
@@ -73,16 +101,14 @@
 _global_client_index = [0]
 
 
-DEFAULT_EXTENSIONS = [
-        PubSubClientExtension,
-]
+DEFAULT_EXTENSIONS = [PubSubClientExtension]
 
 
 def _get_global_client():
     L = sorted(list(_global_clients), reverse=True)
     for k in L:
         c = _global_clients[k]
-        if c.status != 'closed':
+        if c.status != "closed":
             return c
         else:
             del _global_clients[k]
@@ -140,6 +166,7 @@ class Future(WrappedKey):
     --------
     Client:  Creates futures
     """
+
     _cb_executor = None
     _cb_executor_pid = None
 
@@ -157,9 +184,13 @@ def __init__(self, key, client=None, inform=True, state=None):
             self._state = self.client.futures[tkey] = FutureState()
 
         if inform:
-            self.client._send_to_scheduler({'op': 'client-desires-keys',
-                                            'keys': [tokey(key)],
-                                            'client': self.client.id})
+            self.client._send_to_scheduler(
+                {
+                    "op": "client-desires-keys",
+                    "keys": [tokey(key)],
+                    "client": self.client.id,
+                }
+            )
 
         if state is not None:
             try:
@@ -191,11 +222,10 @@ def result(self, timeout=None):
             return self.client.sync(self._result, callback_timeout=timeout)
 
         # shorten error traceback
-        result = self.client.sync(self._result, callback_timeout=timeout,
-                                  raiseit=False)
-        if self.status == 'error':
+        result = self.client.sync(self._result, callback_timeout=timeout, raiseit=False)
+        if self.status == "error":
             six.reraise(*result)
-        elif self.status == 'cancelled':
+        elif self.status == "cancelled":
             raise result
         else:
             return result
@@ -203,14 +233,13 @@ def result(self, timeout=None):
     @gen.coroutine
     def _result(self, raiseit=True):
         yield self._state.wait()
-        if self.status == 'error':
-            exc = clean_exception(self._state.exception,
-                                  self._state.traceback)
+        if self.status == "error":
+            exc = clean_exception(self._state.exception, self._state.traceback)
             if raiseit:
                 six.reraise(*exc)
             else:
                 raise gen.Return(exc)
-        elif self.status == 'cancelled':
+        elif self.status == "cancelled":
             exception = CancelledError(self.key)
             if raiseit:
                 raise exception
@@ -223,7 +252,7 @@ def _result(self, raiseit=True):
     @gen.coroutine
     def _exception(self):
         yield self._state.wait()
-        if self.status == 'error':
+        if self.status == "error":
             raise gen.Return(self._state.exception)
         else:
             raise gen.Return(None)
@@ -238,8 +267,7 @@ def exception(self, timeout=None, **kwargs):
         --------
         Future.traceback
         """
-        return self.client.sync(self._exception, callback_timeout=timeout,
-                                **kwargs)
+        return self.client.sync(self._exception, callback_timeout=timeout, **kwargs)
 
     def add_done_callback(self, fn):
         """ Call callback on future when callback has finished
@@ -253,7 +281,9 @@ def add_done_callback(self, fn):
         cls = Future
         if cls._cb_executor is None or cls._cb_executor_pid != os.getpid():
             try:
-                cls._cb_executor = ThreadPoolExecutor(1, thread_name_prefix="Dask-Callback-Thread")
+                cls._cb_executor = ThreadPoolExecutor(
+                    1, thread_name_prefix="Dask-Callback-Thread"
+                )
             except TypeError:
                 cls._cb_executor = ThreadPoolExecutor(1)
             cls._cb_executor_pid = os.getpid()
@@ -264,8 +294,9 @@ def execute_callback(fut):
             except BaseException:
                 logger.exception("Error in callback %s of %s:", fn, fut)
 
-        self.client.loop.add_callback(done_callback, self,
-                                      partial(cls._cb_executor.submit, execute_callback))
+        self.client.loop.add_callback(
+            done_callback, self, partial(cls._cb_executor.submit, execute_callback)
+        )
 
     def cancel(self, **kwargs):
         """ Cancel request to run this future
@@ -287,12 +318,12 @@ def retry(self, **kwargs):
 
     def cancelled(self):
         """ Returns True if the future has been cancelled """
-        return self._state.status == 'cancelled'
+        return self._state.status == "cancelled"
 
     @gen.coroutine
     def _traceback(self):
         yield self._state.wait()
-        if self.status == 'error':
+        if self.status == "error":
             raise gen.Return(self._state.traceback)
         else:
             raise gen.Return(None)
@@ -318,8 +349,7 @@ def traceback(self, timeout=None, **kwargs):
         --------
         Future.exception
         """
-        return self.client.sync(self._traceback, callback_timeout=timeout,
-                                **kwargs)
+        return self.client.sync(self._traceback, callback_timeout=timeout, **kwargs)
 
     @property
     def type(self):
@@ -342,8 +372,14 @@ def __setstate__(self, state):
         key, address = state
         c = get_client(address)
         Future.__init__(self, key, c)
-        c._send_to_scheduler({'op': 'update-graph', 'tasks': {},
-                              'keys': [tokey(self.key)], 'client': c.id})
+        c._send_to_scheduler(
+            {
+                "op": "update-graph",
+                "tasks": {},
+                "keys": [tokey(self.key)],
+                "client": c.id,
+            }
+        )
 
     def __del__(self):
         try:
@@ -357,17 +393,23 @@ def __repr__(self):
                 typ = self.type.__name__
             except AttributeError:
                 typ = str(self.type)
-            return '<Future: status: %s, type: %s, key: %s>' % (self.status,
-                                                                typ, self.key)
+            return "<Future: status: %s, type: %s, key: %s>" % (
+                self.status,
+                typ,
+                self.key,
+            )
         else:
-            return '<Future: status: %s, key: %s>' % (self.status, self.key)
+            return "<Future: status: %s, key: %s>" % (self.status, self.key)
 
     def _repr_html_(self):
-        text = '<b>Future: %s</b> ' % html_escape(key_split(self.key))
-        text += ('<font color="gray">status: </font>'
-                 '<font color="%(color)s">%(status)s</font>, ') % {
-            'status': self.status,
-            'color': 'red' if self.status == 'error' else 'black'}
+        text = "<b>Future: %s</b> " % html_escape(key_split(self.key))
+        text += (
+            '<font color="gray">status: </font>'
+            '<font color="%(color)s">%(status)s</font>, '
+        ) % {
+            "status": self.status,
+            "color": "red" if self.status == "error" else "black",
+        }
         if self.type:
             try:
                 typ = self.type.__name__
@@ -386,11 +428,12 @@ class FutureState(object):
 
     This is shared between all Futures with the same key and client.
     """
-    __slots__ = ('_event', 'status', 'type', 'exception', 'traceback')
+
+    __slots__ = ("_event", "status", "type", "exception", "traceback")
 
     def __init__(self):
         self._event = None
-        self.status = 'pending'
+        self.status = "pending"
         self.type = None
 
     def _get_event(self):
@@ -403,27 +446,27 @@ def _get_event(self):
         return event
 
     def cancel(self):
-        self.status = 'cancelled'
+        self.status = "cancelled"
         self._get_event().set()
 
     def finish(self, type=None):
-        self.status = 'finished'
+        self.status = "finished"
         self._get_event().set()
         if type is not None:
             self.type = type
 
     def lose(self):
-        self.status = 'lost'
+        self.status = "lost"
         self._get_event().clear()
 
     def retry(self):
-        self.status = 'pending'
+        self.status = "pending"
         self._get_event().clear()
 
     def set_error(self, exception, traceback):
         _, exception, traceback = clean_exception(exception, traceback)
 
-        self.status = 'error'
+        self.status = "error"
         self.exception = exception
         self.traceback = traceback
         self._get_event().set()
@@ -432,7 +475,7 @@ def done(self):
         return self._event is not None and self._event.is_set()
 
     def reset(self):
-        self.status = 'pending'
+        self.status = "pending"
         if self._event is not None:
             self._event.clear()
 
@@ -441,13 +484,13 @@ def wait(self, timeout=None):
         yield self._get_event().wait(timeout)
 
     def __repr__(self):
-        return '<%s: %s>' % (self.__class__.__name__, self.status)
+        return "<%s: %s>" % (self.__class__.__name__, self.status)
 
 
 @gen.coroutine
 def done_callback(future, callback):
     """ Coroutine that waits on future, then calls callback """
-    while future.status == 'pending':
+    while future.status == "pending":
         yield future._state.wait()
     callback(future)
 
@@ -520,27 +563,42 @@ class resembles executors in ``concurrent.futures`` but also allows
     --------
     distributed.scheduler.Scheduler: Internal scheduler
     """
-    def __init__(self, address=None, loop=None, timeout=no_default,
-                 set_as_default=True, scheduler_file=None,
-                 security=None, asynchronous=False,
-                 name=None, heartbeat_interval=None,
-                 serializers=None, deserializers=None,
-                 extensions=DEFAULT_EXTENSIONS, direct_to_workers=False,
-                 **kwargs):
+
+    def __init__(
+        self,
+        address=None,
+        loop=None,
+        timeout=no_default,
+        set_as_default=True,
+        scheduler_file=None,
+        security=None,
+        asynchronous=False,
+        name=None,
+        heartbeat_interval=None,
+        serializers=None,
+        deserializers=None,
+        extensions=DEFAULT_EXTENSIONS,
+        direct_to_workers=False,
+        **kwargs
+    ):
         if timeout == no_default:
-            timeout = dask.config.get('distributed.comm.timeouts.connect')
+            timeout = dask.config.get("distributed.comm.timeouts.connect")
         if timeout is not None:
-            timeout = parse_timedelta(timeout, 's')
+            timeout = parse_timedelta(timeout, "s")
         self._timeout = timeout
 
         self.futures = dict()
         self.refcount = defaultdict(lambda: 0)
         self.coroutines = []
         if name is None:
-            name = dask.config.get('client-name', None)
-        self.id = type(self).__name__ + ('-' + name + '-' if name else '-') + str(uuid.uuid1(clock_seq=os.getpid()))
+            name = dask.config.get("client-name", None)
+        self.id = (
+            type(self).__name__
+            + ("-" + name + "-" if name else "-")
+            + str(uuid.uuid1(clock_seq=os.getpid()))
+        )
         self.generation = 0
-        self.status = 'newly-created'
+        self.status = "newly-created"
         self._pending_msg_buffer = []
         self.extensions = {}
         self.scheduler_file = scheduler_file
@@ -569,16 +627,15 @@ def __init__(self, address=None, loop=None, timeout=no_default,
         self.scheduler_comm = None
         assert isinstance(self.security, Security)
 
-        if name == 'worker':
-            self.connection_args = self.security.get_connection_args('worker')
+        if name == "worker":
+            self.connection_args = self.security.get_connection_args("worker")
         else:
-            self.connection_args = self.security.get_connection_args('client')
+            self.connection_args = self.security.get_connection_args("client")
 
         if address is None:
-            address = dask.config.get('scheduler-address', None)
+            address = dask.config.get("scheduler-address", None)
             if address:
-                logger.info("Config value `scheduler-address` found: %s",
-                            address)
+                logger.info("Config value `scheduler-address` found: %s", address)
 
         if isinstance(address, (rpc, PooledRPCCall)):
             self.scheduler = address
@@ -595,44 +652,45 @@ def __init__(self, address=None, loop=None, timeout=no_default,
         self.loop = self._loop_runner.loop
 
         if heartbeat_interval is None:
-            heartbeat_interval = dask.config.get('distributed.client.heartbeat')
-        heartbeat_interval = parse_timedelta(heartbeat_interval, default='ms')
+            heartbeat_interval = dask.config.get("distributed.client.heartbeat")
+        heartbeat_interval = parse_timedelta(heartbeat_interval, default="ms")
 
         self._periodic_callbacks = dict()
-        self._periodic_callbacks['scheduler-info'] = PeriodicCallback(
-                self._update_scheduler_info, 2000, io_loop=self.loop
+        self._periodic_callbacks["scheduler-info"] = PeriodicCallback(
+            self._update_scheduler_info, 2000, io_loop=self.loop
         )
-        self._periodic_callbacks['heartbeat'] = PeriodicCallback(
-                self._heartbeat,
-                heartbeat_interval * 1000,
-                io_loop=self.loop
+        self._periodic_callbacks["heartbeat"] = PeriodicCallback(
+            self._heartbeat, heartbeat_interval * 1000, io_loop=self.loop
         )
 
         self._start_arg = address
         if set_as_default:
-            self._set_config = dask.config.set(scheduler='dask.distributed',
-                                               shuffle='tasks')
+            self._set_config = dask.config.set(
+                scheduler="dask.distributed", shuffle="tasks"
+            )
 
         self._stream_handlers = {
-            'key-in-memory': self._handle_key_in_memory,
-            'lost-data': self._handle_lost_data,
-            'cancelled-key': self._handle_cancelled_key,
-            'task-retried': self._handle_retried_key,
-            'task-erred': self._handle_task_erred,
-            'restart': self._handle_restart,
-            'error': self._handle_error
+            "key-in-memory": self._handle_key_in_memory,
+            "lost-data": self._handle_lost_data,
+            "cancelled-key": self._handle_cancelled_key,
+            "task-retried": self._handle_retried_key,
+            "task-erred": self._handle_task_erred,
+            "restart": self._handle_restart,
+            "error": self._handle_error,
         }
 
         self._state_handlers = {
-            'memory': self._handle_key_in_memory,
-            'lost': self._handle_lost_data,
-            'erred': self._handle_task_erred
+            "memory": self._handle_key_in_memory,
+            "lost": self._handle_lost_data,
+            "erred": self._handle_task_erred,
         }
 
-        super(Client, self).__init__(connection_args=self.connection_args,
-                                     io_loop=self.loop,
-                                     serializers=serializers,
-                                     deserializers=deserializers)
+        super(Client, self).__init__(
+            connection_args=self.connection_args,
+            io_loop=self.loop,
+            serializers=serializers,
+            deserializers=deserializers,
+        )
 
         for ext in extensions:
             ext(self)
@@ -640,6 +698,7 @@ def __init__(self, address=None, loop=None, timeout=no_default,
         self.start(timeout=timeout)
 
         from distributed.recreate_exceptions import ReplayExceptionClient
+
         ReplayExceptionClient(self)
 
     @classmethod
@@ -665,13 +724,16 @@ def asynchronous(self):
         return self._asynchronous and self.loop is IOLoop.current()
 
     def sync(self, func, *args, **kwargs):
-        asynchronous = kwargs.pop('asynchronous', None)
-        if asynchronous or self.asynchronous or getattr(thread_state, 'asynchronous', False):
-            callback_timeout = kwargs.pop('callback_timeout', None)
+        asynchronous = kwargs.pop("asynchronous", None)
+        if (
+            asynchronous
+            or self.asynchronous
+            or getattr(thread_state, "asynchronous", False)
+        ):
+            callback_timeout = kwargs.pop("callback_timeout", None)
             future = func(*args, **kwargs)
             if callback_timeout is not None:
-                future = gen.with_timeout(timedelta(seconds=callback_timeout),
-                                          future)
+                future = gen.with_timeout(timedelta(seconds=callback_timeout), future)
             return future
         else:
             return sync(self.loop, func, *args, **kwargs)
@@ -679,26 +741,38 @@ def sync(self, func, *args, **kwargs):
     def __repr__(self):
         # Note: avoid doing I/O here...
         info = self._scheduler_identity
-        addr = info.get('address')
+        addr = info.get("address")
         if addr:
-            workers = info.get('workers', {})
+            workers = info.get("workers", {})
             nworkers = len(workers)
-            ncores = sum(w['ncores'] for w in workers.values())
-            return '<%s: scheduler=%r processes=%d cores=%d>' % (
-                self.__class__.__name__, addr, nworkers, ncores)
+            ncores = sum(w["ncores"] for w in workers.values())
+            return "<%s: scheduler=%r processes=%d cores=%d>" % (
+                self.__class__.__name__,
+                addr,
+                nworkers,
+                ncores,
+            )
         elif self.scheduler is not None:
-            return '<%s: scheduler=%r>' % (
-                self.__class__.__name__, self.scheduler.address)
+            return "<%s: scheduler=%r>" % (
+                self.__class__.__name__,
+                self.scheduler.address,
+            )
         else:
-            return '<%s: not connected>' % (self.__class__.__name__,)
+            return "<%s: not connected>" % (self.__class__.__name__,)
 
     def _repr_html_(self):
-        if self.cluster and hasattr(self.cluster, 'scheduler') and self.cluster.scheduler:
+        if (
+            self.cluster
+            and hasattr(self.cluster, "scheduler")
+            and self.cluster.scheduler
+        ):
             info = self.cluster.scheduler.identity()
             scheduler = self.cluster.scheduler
-        elif (self._loop_runner.is_started() and
-                self.scheduler and
-                not (self.asynchronous and self.loop is IOLoop.current())):
+        elif (
+            self._loop_runner.is_started()
+            and self.scheduler
+            and not (self.asynchronous and self.loop is IOLoop.current())
+        ):
             info = sync(self.loop, self.scheduler.identity)
             scheduler = self.scheduler
         else:
@@ -706,56 +780,63 @@ def _repr_html_(self):
             scheduler = self.scheduler
 
         if scheduler is not None:
-            text = ("<h3>Client</h3>\n"
-                    "<ul>\n"
-                    "  <li><b>Scheduler: </b>%s\n") % scheduler.address
+            text = (
+                "<h3>Client</h3>\n" "<ul>\n" "  <li><b>Scheduler: </b>%s\n"
+            ) % scheduler.address
         else:
-            text = ("<h3>Client</h3>\n"
-                    "<ul>\n"
-                    "  <li><b>Scheduler: not connected</b>\n")
-        if info and 'bokeh' in info['services']:
-            protocol, rest = scheduler.address.split('://')
-            port = info['services']['bokeh']
-            if protocol == 'inproc':
-                host = 'localhost'
+            text = (
+                "<h3>Client</h3>\n" "<ul>\n" "  <li><b>Scheduler: not connected</b>\n"
+            )
+        if info and "bokeh" in info["services"]:
+            protocol, rest = scheduler.address.split("://")
+            port = info["services"]["bokeh"]
+            if protocol == "inproc":
+                host = "localhost"
             else:
-                host = rest.split(':')[0]
-            template = dask.config.get('distributed.dashboard.link')
+                host = rest.split(":")[0]
+            template = dask.config.get("distributed.dashboard.link")
             address = template.format(host=host, port=port, **os.environ)
-            text += "  <li><b>Dashboard: </b><a href='%(web)s' target='_blank'>%(web)s</a>\n" % {'web': address}
+            text += (
+                "  <li><b>Dashboard: </b><a href='%(web)s' target='_blank'>%(web)s</a>\n"
+                % {"web": address}
+            )
 
         text += "</ul>\n"
 
         if info:
-            workers = len(info['workers'])
-            cores = sum(w['ncores'] for w in info['workers'].values())
-            memory = sum(w['memory_limit'] for w in info['workers'].values())
+            workers = len(info["workers"])
+            cores = sum(w["ncores"] for w in info["workers"].values())
+            memory = sum(w["memory_limit"] for w in info["workers"].values())
             memory = format_bytes(memory)
-            text2 = ("<h3>Cluster</h3>\n"
-                     "<ul>\n"
-                     "  <li><b>Workers: </b>%d</li>\n"
-                     "  <li><b>Cores: </b>%d</li>\n"
-                     "  <li><b>Memory: </b>%s</li>\n"
-                     "</ul>\n") % (workers, cores, memory)
-
-            return ('<table style="border: 2px solid white;">\n'
-                    '<tr>\n'
-                    '<td style="vertical-align: top; border: 0px solid white">\n%s</td>\n'
-                    '<td style="vertical-align: top; border: 0px solid white">\n%s</td>\n'
-                    '</tr>\n</table>') % (text, text2)
+            text2 = (
+                "<h3>Cluster</h3>\n"
+                "<ul>\n"
+                "  <li><b>Workers: </b>%d</li>\n"
+                "  <li><b>Cores: </b>%d</li>\n"
+                "  <li><b>Memory: </b>%s</li>\n"
+                "</ul>\n"
+            ) % (workers, cores, memory)
+
+            return (
+                '<table style="border: 2px solid white;">\n'
+                "<tr>\n"
+                '<td style="vertical-align: top; border: 0px solid white">\n%s</td>\n'
+                '<td style="vertical-align: top; border: 0px solid white">\n%s</td>\n'
+                "</tr>\n</table>"
+            ) % (text, text2)
 
         else:
             return text
 
     def start(self, **kwargs):
         """ Start scheduler running in separate thread """
-        if self.status != 'newly-created':
+        if self.status != "newly-created":
             return
 
         self._loop_runner.start()
 
         _set_global_client(self)
-        self.status = 'connecting'
+        self.status = "connecting"
 
         if self.asynchronous:
             self._started = self._start(**kwargs)
@@ -763,37 +844,41 @@ def start(self, **kwargs):
             sync(self.loop, self._start, **kwargs)
 
     def __await__(self):
-        if hasattr(self, '_started'):
+        if hasattr(self, "_started"):
             return self._started.__await__()
         else:
+
             @gen.coroutine
             def _():
                 raise gen.Return(self)
+
             return _().__await__()
 
     def _send_to_scheduler_safe(self, msg):
-        if self.status in ('running', 'closing'):
+        if self.status in ("running", "closing"):
             try:
                 self.scheduler_comm.send(msg)
             except (CommClosedError, AttributeError):
-                if self.status == 'running':
+                if self.status == "running":
                     raise
-        elif self.status in ('connecting', 'newly-created'):
+        elif self.status in ("connecting", "newly-created"):
             self._pending_msg_buffer.append(msg)
 
     def _send_to_scheduler(self, msg):
-        if self.status in ('running', 'closing', 'connecting', 'newly-created'):
+        if self.status in ("running", "closing", "connecting", "newly-created"):
             self.loop.add_callback(self._send_to_scheduler_safe, msg)
         else:
-            raise Exception("Tried sending message after closing.  Status: %s\n"
-                            "Message: %s" % (self.status, msg))
+            raise Exception(
+                "Tried sending message after closing.  Status: %s\n"
+                "Message: %s" % (self.status, msg)
+            )
 
     @gen.coroutine
     def _start(self, timeout=no_default, **kwargs):
         if timeout == no_default:
             timeout = self._timeout
         if timeout is not None:
-            timeout = parse_timedelta(timeout, 's')
+            timeout = parse_timedelta(timeout, "s")
 
         address = self._start_arg
         if self.cluster is not None:
@@ -803,8 +888,10 @@ def _start(self, timeout=no_default, **kwargs):
             except AttributeError:  # Some clusters don't have this method
                 pass
             except Exception:
-                logger.info("Tried to start cluster and received an error. "
-                            "Proceeding.", exc_info=True)
+                logger.info(
+                    "Tried to start cluster and received an error. " "Proceeding.",
+                    exc_info=True,
+                )
             address = self.cluster.scheduler_address
         elif self.scheduler_file is not None:
             while not os.path.exists(self.scheduler_file):
@@ -813,7 +900,7 @@ def _start(self, timeout=no_default, **kwargs):
                 try:
                     with open(self.scheduler_file) as f:
                         cfg = json.load(f)
-                    address = cfg['address']
+                    address = cfg["address"]
                     break
                 except (ValueError, KeyError):  # JSON file not yet flushed
                     yield gen.sleep(0.01)
@@ -821,31 +908,39 @@ def _start(self, timeout=no_default, **kwargs):
             from .deploy import LocalCluster
 
             try:
-                self.cluster = LocalCluster(loop=self.loop, asynchronous=True,
-                                            **self._startup_kwargs)
+                self.cluster = LocalCluster(
+                    loop=self.loop, asynchronous=True, **self._startup_kwargs
+                )
                 yield self.cluster
             except (OSError, socket.error) as e:
                 if e.errno != errno.EADDRINUSE:
                     raise
                 # The default port was taken, use a random one
-                self.cluster = LocalCluster(scheduler_port=0, loop=self.loop,
-                                            asynchronous=True,
-                                            **self._startup_kwargs)
+                self.cluster = LocalCluster(
+                    scheduler_port=0,
+                    loop=self.loop,
+                    asynchronous=True,
+                    **self._startup_kwargs
+                )
                 yield self.cluster
 
             # Wait for all workers to be ready
             # XXX should be a LocalCluster method instead
-            while (not self.cluster.workers or
-                   len(self.cluster.scheduler.workers) < len(self.cluster.workers)):
+            while not self.cluster.workers or len(self.cluster.scheduler.workers) < len(
+                self.cluster.workers
+            ):
                 yield gen.sleep(0.01)
 
             address = self.cluster.scheduler_address
 
         if self.scheduler is None:
-            self.scheduler = rpc(address, timeout=timeout,
-                                 connection_args=self.connection_args,
-                                 serializers=self._serializers,
-                                 deserializers=self._deserializers)
+            self.scheduler = rpc(
+                address,
+                timeout=timeout,
+                connection_args=self.connection_args,
+                serializers=self._serializers,
+                deserializers=self._deserializers,
+            )
         self.scheduler_comm = None
 
         yield self._ensure_connected(timeout=timeout)
@@ -862,14 +957,14 @@ def _start(self, timeout=no_default, **kwargs):
     def _reconnect(self, timeout=0.1):
         with log_errors():
             assert self.scheduler_comm.comm.closed()
-            self.status = 'connecting'
+            self.status = "connecting"
             self.scheduler_comm = None
 
             for st in self.futures.values():
                 st.cancel()
             self.futures.clear()
 
-            while self.status == 'connecting':
+            while self.status == "connecting":
                 try:
                     yield self._ensure_connected()
                     break
@@ -878,39 +973,46 @@ def _reconnect(self, timeout=0.1):
 
     @gen.coroutine
     def _ensure_connected(self, timeout=None):
-        if (self.scheduler_comm and not self.scheduler_comm.closed() or
-                self._connecting_to_scheduler or self.scheduler is None):
+        if (
+            self.scheduler_comm
+            and not self.scheduler_comm.closed()
+            or self._connecting_to_scheduler
+            or self.scheduler is None
+        ):
             return
 
         self._connecting_to_scheduler = True
 
         try:
-            comm = yield connect(self.scheduler.address, timeout=timeout,
-                                 connection_args=self.connection_args)
+            comm = yield connect(
+                self.scheduler.address,
+                timeout=timeout,
+                connection_args=self.connection_args,
+            )
             if timeout is not None:
-                yield gen.with_timeout(timedelta(seconds=timeout),
-                                       self._update_scheduler_info())
+                yield gen.with_timeout(
+                    timedelta(seconds=timeout), self._update_scheduler_info()
+                )
             else:
                 yield self._update_scheduler_info()
-            yield comm.write({'op': 'register-client',
-                              'client': self.id,
-                              'reply': False})
+            yield comm.write(
+                {"op": "register-client", "client": self.id, "reply": False}
+            )
         finally:
             self._connecting_to_scheduler = False
         if timeout is not None:
-            msg = yield gen.with_timeout(timedelta(seconds=timeout),
-                                         comm.read())
+            msg = yield gen.with_timeout(timedelta(seconds=timeout), comm.read())
         else:
             msg = yield comm.read()
         assert len(msg) == 1
-        assert msg[0]['op'] == 'stream-start'
+        assert msg[0]["op"] == "stream-start"
 
-        bcomm = BatchedSend(interval='10ms', loop=self.loop)
+        bcomm = BatchedSend(interval="10ms", loop=self.loop)
         bcomm.start(comm)
         self.scheduler_comm = bcomm
 
         _set_global_client(self)
-        self.status = 'running'
+        self.status = "running"
 
         for msg in self._pending_msg_buffer:
             self._send_to_scheduler(msg)
@@ -920,7 +1022,7 @@ def _ensure_connected(self, timeout=None):
 
     @gen.coroutine
     def _update_scheduler_info(self):
-        if self.status not in ('running', 'connecting'):
+        if self.status not in ("running", "connecting"):
             return
         try:
             self._scheduler_identity = yield self.scheduler.identity()
@@ -929,7 +1031,7 @@ def _update_scheduler_info(self):
 
     def _heartbeat(self):
         if self.scheduler_comm:
-            self.scheduler_comm.send({'op': 'heartbeat-client'})
+            self.scheduler_comm.send({"op": "heartbeat-client"})
 
     def __enter__(self):
         if not self._loop_runner.is_started():
@@ -968,10 +1070,10 @@ def _release_key(self, key):
         st = self.futures.pop(key, None)
         if st is not None:
             st.cancel()
-        if self.status != 'closed':
-            self._send_to_scheduler({'op': 'client-releases-keys',
-                                     'keys': [key],
-                                     'client': self.id})
+        if self.status != "closed":
+            self._send_to_scheduler(
+                {"op": "client-releases-keys", "keys": [key], "client": self.id}
+            )
 
     @gen.coroutine
     def _handle_report(self):
@@ -984,10 +1086,10 @@ def _handle_report(self):
                     try:
                         msgs = yield self.scheduler_comm.comm.read()
                     except CommClosedError:
-                        if self.status == 'running':
+                        if self.status == "running":
                             logger.info("Client report stream closed to scheduler")
                             logger.info("Reconnecting...")
-                            self.status = 'connecting'
+                            self.status = "connecting"
                             yield self._reconnect()
                             continue
                         else:
@@ -999,12 +1101,12 @@ def _handle_report(self):
                     for msg in msgs:
                         logger.debug("Client receives message %s", msg)
 
-                        if 'status' in msg and 'error' in msg['status']:
+                        if "status" in msg and "error" in msg["status"]:
                             six.reraise(*clean_exception(**msg))
 
-                        op = msg.pop('op')
+                        op = msg.pop("op")
 
-                        if op == 'close' or op == 'stream-closed':
+                        if op == "close" or op == "stream-closed":
                             breakout = True
                             break
 
@@ -1067,7 +1169,7 @@ def _handle_error(self, exception=None):
     @gen.coroutine
     def _close(self, fast=False):
         """ Send close signal and wait until scheduler completes """
-        self.status = 'closing'
+        self.status = "closing"
 
         with log_errors():
             _del_global_client(self)
@@ -1078,30 +1180,40 @@ def _close(self, fast=False):
                 # clear the dask.config set keys
                 with self._set_config:
                     pass
-            if self.get == dask.config.get('get', None):
-                del dask.config.config['get']
-            if self.status == 'closed':
+            if self.get == dask.config.get("get", None):
+                del dask.config.config["get"]
+            if self.status == "closed":
                 raise gen.Return()
 
-            if self.scheduler_comm and self.scheduler_comm.comm and not self.scheduler_comm.comm.closed():
-                self._send_to_scheduler({'op': 'close-client'})
-                self._send_to_scheduler({'op': 'close-stream'})
+            if (
+                self.scheduler_comm
+                and self.scheduler_comm.comm
+                and not self.scheduler_comm.comm.closed()
+            ):
+                self._send_to_scheduler({"op": "close-client"})
+                self._send_to_scheduler({"op": "close-stream"})
 
             # Give the scheduler 'stream-closed' message 100ms to come through
             # This makes the shutdown slightly smoother and quieter
             with ignoring(AttributeError, gen.TimeoutError):
-                yield gen.with_timeout(timedelta(milliseconds=100),
-                                       self._handle_scheduler_coroutine,
-                                       quiet_exceptions=(CancelledError,))
-
-            if self.scheduler_comm and self.scheduler_comm.comm and not self.scheduler_comm.comm.closed():
+                yield gen.with_timeout(
+                    timedelta(milliseconds=100),
+                    self._handle_scheduler_coroutine,
+                    quiet_exceptions=(CancelledError,),
+                )
+
+            if (
+                self.scheduler_comm
+                and self.scheduler_comm.comm
+                and not self.scheduler_comm.comm.closed()
+            ):
                 yield self.scheduler_comm.close()
             for key in list(self.futures):
                 self._release_key(key=key)
             if self._start_arg is None:
                 with ignoring(AttributeError):
                     yield self.cluster._close()
-            self.status = 'closed'
+            self.status = "closed"
             if _get_global_client() is self:
                 _set_global_client(None)
             coroutines = set(self.coroutines)
@@ -1115,13 +1227,12 @@ def _close(self, fast=False):
             del self.coroutines[:]
             if not fast:
                 with ignoring(TimeoutError):
-                    yield gen.with_timeout(timedelta(seconds=2),
-                                           list(coroutines))
+                    yield gen.with_timeout(timedelta(seconds=2), list(coroutines))
             with ignoring(AttributeError):
                 self.scheduler.close_rpc()
             self.scheduler = None
 
-        self.status = 'closed'
+        self.status = "closed"
 
     _shutdown = _close
 
@@ -1140,9 +1251,9 @@ def close(self, timeout=no_default):
         if timeout == no_default:
             timeout = self._timeout * 2
         # XXX handling of self.status here is not thread-safe
-        if self.status == 'closed':
+        if self.status == "closed":
             return
-        self.status = 'closing'
+        self.status = "closing"
 
         if self.asynchronous:
             future = self._close()
@@ -1156,7 +1267,7 @@ def close(self, timeout=no_default):
 
         sync(self.loop, self._close, fast=True)
 
-        assert self.status == 'closed'
+        assert self.status == "closed"
 
         if self._should_close_loop and not shutting_down():
             self._loop_runner.stop()
@@ -1229,24 +1340,24 @@ def submit(self, func, *args, **kwargs):
         if not callable(func):
             raise TypeError("First input to submit must be a callable function")
 
-        key = kwargs.pop('key', None)
-        workers = kwargs.pop('workers', None)
-        resources = kwargs.pop('resources', None)
-        retries = kwargs.pop('retries', None)
-        priority = kwargs.pop('priority', 0)
-        fifo_timeout = kwargs.pop('fifo_timeout', '100ms')
-        allow_other_workers = kwargs.pop('allow_other_workers', False)
-        actor = kwargs.pop('actor', kwargs.pop('actors', False))
-        pure = kwargs.pop('pure', not actor)
+        key = kwargs.pop("key", None)
+        workers = kwargs.pop("workers", None)
+        resources = kwargs.pop("resources", None)
+        retries = kwargs.pop("retries", None)
+        priority = kwargs.pop("priority", 0)
+        fifo_timeout = kwargs.pop("fifo_timeout", "100ms")
+        allow_other_workers = kwargs.pop("allow_other_workers", False)
+        actor = kwargs.pop("actor", kwargs.pop("actors", False))
+        pure = kwargs.pop("pure", not actor)
 
         if allow_other_workers not in (True, False, None):
             raise TypeError("allow_other_workers= must be True or False")
 
         if key is None:
             if pure:
-                key = funcname(func) + '-' + tokenize(func, kwargs, *args)
+                key = funcname(func) + "-" + tokenize(func, kwargs, *args)
             else:
-                key = funcname(func) + '-' + str(uuid.uuid4())
+                key = funcname(func) + "-" + str(uuid.uuid4())
 
         skey = tokey(key)
 
@@ -1271,13 +1382,18 @@ def submit(self, func, *args, **kwargs):
         else:
             dsk = {skey: (func,) + tuple(args)}
 
-        futures = self._graph_to_futures(dsk, [skey], restrictions,
-                                         loose_restrictions, priority={skey: 0},
-                                         user_priority=priority,
-                                         resources={skey: resources} if resources else None,
-                                         retries=retries,
-                                         fifo_timeout=fifo_timeout,
-                                         actors=actor)
+        futures = self._graph_to_futures(
+            dsk,
+            [skey],
+            restrictions,
+            loose_restrictions,
+            priority={skey: 0},
+            user_priority=priority,
+            resources={skey: resources} if resources else None,
+            retries=retries,
+            fifo_timeout=fifo_timeout,
+            actors=actor,
+        )
 
         logger.debug("Submit %s(...), %s", funcname(func), key)
 
@@ -1345,14 +1461,17 @@ def map(self, func, *iterables, **kwargs):
         if not callable(func):
             raise TypeError("First input to map must be a callable function")
 
-        if (all(map(isqueue, iterables)) or
-                all(isinstance(i, Iterator) for i in iterables)):
-            maxsize = kwargs.pop('maxsize', 0)
+        if all(map(isqueue, iterables)) or all(
+            isinstance(i, Iterator) for i in iterables
+        ):
+            maxsize = kwargs.pop("maxsize", 0)
             q_out = pyQueue(maxsize=maxsize)
-            t = threading.Thread(target=self._threaded_map,
-                                 name="Threaded map()",
-                                 args=(q_out, func, iterables),
-                                 kwargs=kwargs)
+            t = threading.Thread(
+                target=self._threaded_map,
+                name="Threaded map()",
+                args=(q_out, func, iterables),
+                kwargs=kwargs,
+            )
             t.daemon = True
             t.start()
             if isqueue(iterables[0]):
@@ -1360,16 +1479,16 @@ def map(self, func, *iterables, **kwargs):
             else:
                 return queue_to_iterator(q_out)
 
-        key = kwargs.pop('key', None)
+        key = kwargs.pop("key", None)
         key = key or funcname(func)
-        workers = kwargs.pop('workers', None)
-        retries = kwargs.pop('retries', None)
-        resources = kwargs.pop('resources', None)
-        user_priority = kwargs.pop('priority', 0)
-        allow_other_workers = kwargs.pop('allow_other_workers', False)
-        fifo_timeout = kwargs.pop('fifo_timeout', '100ms')
-        actor = kwargs.pop('actor', kwargs.pop('actors', False))
-        pure = kwargs.pop('pure', not actor)
+        workers = kwargs.pop("workers", None)
+        retries = kwargs.pop("retries", None)
+        resources = kwargs.pop("resources", None)
+        user_priority = kwargs.pop("priority", 0)
+        allow_other_workers = kwargs.pop("allow_other_workers", False)
+        fifo_timeout = kwargs.pop("fifo_timeout", "100ms")
+        actor = kwargs.pop("actor", kwargs.pop("actors", False))
+        pure = kwargs.pop("pure", not actor)
 
         if allow_other_workers and workers is None:
             raise ValueError("Only use allow_other_workers= if using workers=")
@@ -1379,16 +1498,23 @@ def map(self, func, *iterables, **kwargs):
             keys = key
         else:
             if pure:
-                keys = [key + '-' + tokenize(func, kwargs, *args)
-                        for args in zip(*iterables)]
+                keys = [
+                    key + "-" + tokenize(func, kwargs, *args)
+                    for args in zip(*iterables)
+                ]
             else:
                 uid = str(uuid.uuid4())
-                keys = [key + '-' + uid + '-' + str(i)
-                        for i in range(min(map(len, iterables)))] if iterables else []
+                keys = (
+                    [
+                        key + "-" + uid + "-" + str(i)
+                        for i in range(min(map(len, iterables)))
+                    ]
+                    if iterables
+                    else []
+                )
 
         if not kwargs:
-            dsk = {key: (func,) + args
-                   for key, args in zip(keys, zip(*iterables))}
+            dsk = {key: (func,) + args for key, args in zip(keys, zip(*iterables))}
         else:
             kwargs2 = {}
             dsk = {}
@@ -1399,16 +1525,22 @@ def map(self, func, *iterables, **kwargs):
                     dsk.update(vv.dask)
                 else:
                     kwargs2[k] = v
-            dsk.update({key: (apply, func, (tuple, list(args)), kwargs2)
-                        for key, args in zip(keys, zip(*iterables))})
+            dsk.update(
+                {
+                    key: (apply, func, (tuple, list(args)), kwargs2)
+                    for key, args in zip(keys, zip(*iterables))
+                }
+            )
 
         if isinstance(workers, six.string_types + (Number,)):
             workers = [workers]
         if isinstance(workers, (list, set)):
             if workers and isinstance(first(workers), (list, set)):
                 if len(workers) != len(keys):
-                    raise ValueError("You only provided %d worker restrictions"
-                                     " for a sequence of length %d" % (len(workers), len(keys)))
+                    raise ValueError(
+                        "You only provided %d worker restrictions"
+                        " for a sequence of length %d" % (len(workers), len(keys))
+                    )
                 restrictions = dict(zip(keys, workers))
             else:
                 restrictions = {k: workers for k in keys}
@@ -1430,20 +1562,24 @@ def map(self, func, *iterables, **kwargs):
         else:
             resources = None
 
-        futures = self._graph_to_futures(dsk, keys, restrictions,
-                                         loose_restrictions,
-                                         priority=priority,
-                                         resources=resources,
-                                         retries=retries,
-                                         user_priority=user_priority,
-                                         fifo_timeout=fifo_timeout,
-                                         actors=actor)
+        futures = self._graph_to_futures(
+            dsk,
+            keys,
+            restrictions,
+            loose_restrictions,
+            priority=priority,
+            resources=resources,
+            retries=retries,
+            user_priority=user_priority,
+            fifo_timeout=fifo_timeout,
+            actors=actor,
+        )
         logger.debug("map(%s, ...)", funcname(func))
 
         return [futures[tokey(k)] for k in keys]
 
     @gen.coroutine
-    def _gather(self, futures, errors='raise', direct=None, local_worker=None):
+    def _gather(self, futures, errors="raise", direct=None, local_worker=None):
         unpacked, future_set = unpack_remotedata(futures, byte_keys=True)
         keys = [tokey(future.key) for future in future_set]
         bad_data = dict()
@@ -1465,38 +1601,35 @@ def wait(k):
             """ Want to stop the All(...) early if we find an error """
             st = self.futures[k]
             yield st.wait()
-            if st.status != 'finished' and errors == 'raise' :
+            if st.status != "finished" and errors == "raise":
                 raise AllExit()
 
         while True:
             logger.debug("Waiting on futures to clear before gather")
 
             with ignoring(AllExit):
-                yield All([wait(key) for key in keys if key in self.futures],
-                           quiet_exceptions=AllExit)
+                yield All(
+                    [wait(key) for key in keys if key in self.futures],
+                    quiet_exceptions=AllExit,
+                )
 
-            failed = ('error', 'cancelled')
+            failed = ("error", "cancelled")
 
             exceptions = set()
             bad_keys = set()
             for key in keys:
-                if (key not in self.futures or
-                        self.futures[key].status in failed):
+                if key not in self.futures or self.futures[key].status in failed:
                     exceptions.add(key)
-                    if errors == 'raise':
+                    if errors == "raise":
                         try:
                             st = self.futures[key]
                             exception = st.exception
                             traceback = st.traceback
                         except (AttributeError, KeyError):
-                            six.reraise(CancelledError,
-                                        CancelledError(key),
-                                        None)
+                            six.reraise(CancelledError, CancelledError(key), None)
                         else:
-                            six.reraise(type(exception),
-                                        exception,
-                                        traceback)
-                    if errors == 'skip':
+                            six.reraise(type(exception), exception, traceback)
+                    if errors == "skip":
                         bad_keys.add(key)
                         bad_data[key] = None
                     else:
@@ -1505,16 +1638,16 @@ def wait(k):
             keys = [k for k in keys if k not in bad_keys and k not in data]
 
             if local_worker:  # look inside local worker
-                data.update({k: local_worker.data[k]
-                             for k in keys
-                             if k in local_worker.data})
+                data.update(
+                    {k: local_worker.data[k] for k in keys if k in local_worker.data}
+                )
                 keys = [k for k in keys if k not in data]
 
             # We now do an actual remote communication with workers or scheduler
             if self._gather_future:  # attach onto another pending gather request
                 self._gather_keys |= set(keys)
                 response = yield self._gather_future
-            else:                    # no one waiting, go ahead
+            else:  # no one waiting, go ahead
                 self._gather_keys = set(keys)
                 future = self._gather_remote(direct, local_worker)
                 if self._gather_keys is None:
@@ -1523,13 +1656,16 @@ def wait(k):
                     self._gather_future = future
                 response = yield future
 
-            if response['status'] == 'error':
-                log = logger.warning if errors == 'raise' else logger.debug
-                log("Couldn't gather %s keys, rescheduling %s", len(response['keys']), response['keys'])
-                for key in response['keys']:
-                    self._send_to_scheduler({'op': 'report-key',
-                                             'key': key})
-                for key in response['keys']:
+            if response["status"] == "error":
+                log = logger.warning if errors == "raise" else logger.debug
+                log(
+                    "Couldn't gather %s keys, rescheduling %s",
+                    len(response["keys"]),
+                    response["keys"],
+                )
+                for key in response["keys"]:
+                    self._send_to_scheduler({"op": "report-key", "key": key})
+                for key in response["keys"]:
                     try:
                         self.futures[key].reset()
                     except KeyError:  # TODO: verify that this is safe
@@ -1537,10 +1673,10 @@ def wait(k):
             else:
                 break
 
-        if bad_data and errors == 'skip' and isinstance(unpacked, list):
+        if bad_data and errors == "skip" and isinstance(unpacked, list):
             unpacked = [f for f in unpacked if f not in bad_data]
 
-        data.update(response['data'])
+        data.update(response["data"])
         result = pack_data(unpacked, merge(data, bad_data))
         raise gen.Return(result)
 
@@ -1561,13 +1697,14 @@ def _gather_remote(self, direct, local_worker):
             if direct or local_worker:  # gather directly from workers
                 who_has = yield self.scheduler.who_has(keys=keys)
                 data2, missing_keys, missing_workers = yield gather_from_workers(
-                    who_has, rpc=self.rpc, close=False)
-                response = {'status': 'OK', 'data': data2}
+                    who_has, rpc=self.rpc, close=False
+                )
+                response = {"status": "OK", "data": data2}
                 if missing_keys:
                     keys2 = [key for key in keys if key not in data2]
                     response = yield self.scheduler.gather(keys=keys2)
-                    if response['status'] == 'OK':
-                        response['data'].update(data2)
+                    if response["status"] == "OK":
+                        response["data"].update(data2)
 
             else:  # ask scheduler to gather data for us
                 response = yield self.scheduler.gather(keys=keys)
@@ -1589,8 +1726,9 @@ def _threaded_gather(self, qin, qout, **kwargs):
             for item in results:
                 qout.put(item)
 
-    def gather(self, futures, errors='raise', maxsize=0, direct=None,
-               asynchronous=None):
+    def gather(
+        self, futures, errors="raise", maxsize=0, direct=None, asynchronous=None
+    ):
         """ Gather futures from distributed memory
 
         Accepts a future, nested container of futures, iterator, or queue.
@@ -1637,34 +1775,49 @@ def gather(self, futures, errors='raise', maxsize=0, direct=None,
         """
         if isqueue(futures):
             qout = pyQueue(maxsize=maxsize)
-            t = threading.Thread(target=self._threaded_gather,
-                                 name="Threaded gather()",
-                                 args=(futures, qout),
-                                 kwargs={'errors': errors, 'direct': direct})
+            t = threading.Thread(
+                target=self._threaded_gather,
+                name="Threaded gather()",
+                args=(futures, qout),
+                kwargs={"errors": errors, "direct": direct},
+            )
             t.daemon = True
             t.start()
             return qout
         elif isinstance(futures, Iterator):
-            return (self.gather(f, errors=errors, direct=direct)
-                    for f in futures)
+            return (self.gather(f, errors=errors, direct=direct) for f in futures)
         else:
-            if hasattr(thread_state, 'execution_state'):  # within worker task
-                local_worker = thread_state.execution_state['worker']
+            if hasattr(thread_state, "execution_state"):  # within worker task
+                local_worker = thread_state.execution_state["worker"]
             else:
                 local_worker = None
-            return self.sync(self._gather, futures, errors=errors,
-                             direct=direct, local_worker=local_worker,
-                             asynchronous=asynchronous)
+            return self.sync(
+                self._gather,
+                futures,
+                errors=errors,
+                direct=direct,
+                local_worker=local_worker,
+                asynchronous=asynchronous,
+            )
 
     @gen.coroutine
-    def _scatter(self, data, workers=None, broadcast=False, direct=None,
-                 local_worker=None, timeout=no_default, hash=True):
+    def _scatter(
+        self,
+        data,
+        workers=None,
+        broadcast=False,
+        direct=None,
+        local_worker=None,
+        timeout=no_default,
+        hash=True,
+    ):
         if timeout == no_default:
             timeout = self._timeout
         if isinstance(workers, six.string_types + (Number,)):
             workers = [workers]
-        if isinstance(data, dict) and not all(isinstance(k, (bytes, unicode))
-                                              for k in data):
+        if isinstance(data, dict) and not all(
+            isinstance(k, (bytes, unicode)) for k in data
+        ):
             d = yield self._scatter(keymap(tokey, data), workers, broadcast)
             raise gen.Return({k: d[tokey(k)] for k in data})
 
@@ -1682,9 +1835,9 @@ def _scatter(self, data, workers=None, broadcast=False, direct=None,
             data = [data]
         if isinstance(data, (list, tuple)):
             if hash:
-                names = [type(x).__name__ + '-' + tokenize(x) for x in data]
+                names = [type(x).__name__ + "-" + tokenize(x) for x in data]
             else:
-                names = [type(x).__name__ + '-' + uuid.uuid4().hex for x in data]
+                names = [type(x).__name__ + "-" + uuid.uuid4().hex for x in data]
             data = dict(zip(names, data))
 
         assert isinstance(data, dict)
@@ -1708,7 +1861,8 @@ def _scatter(self, data, workers=None, broadcast=False, direct=None,
             yield self.scheduler.update_data(
                 who_has={key: [local_worker.address] for key in data},
                 nbytes=valmap(sizeof, data),
-                client=self.id)
+                client=self.id,
+            )
 
         else:
             data2 = valmap(to_serialize, data)
@@ -1724,18 +1878,21 @@ def _scatter(self, data, workers=None, broadcast=False, direct=None,
                 if not ncores:
                     raise ValueError("No valid workers")
 
-                _, who_has, nbytes = yield scatter_to_workers(ncores, data2,
-                                                              report=False,
-                                                              rpc=self.rpc)
+                _, who_has, nbytes = yield scatter_to_workers(
+                    ncores, data2, report=False, rpc=self.rpc
+                )
 
-                yield self.scheduler.update_data(who_has=who_has,
-                                                 nbytes=nbytes,
-                                                 client=self.id)
+                yield self.scheduler.update_data(
+                    who_has=who_has, nbytes=nbytes, client=self.id
+                )
             else:
-                yield self.scheduler.scatter(data=data2, workers=workers,
-                                             client=self.id,
-                                             broadcast=broadcast,
-                                             timeout=timeout)
+                yield self.scheduler.scatter(
+                    data=data2,
+                    workers=workers,
+                    client=self.id,
+                    broadcast=broadcast,
+                    timeout=timeout,
+                )
 
         out = {k: Future(k, self, inform=False) for k in data}
         for key, typ in types.items():
@@ -1774,8 +1931,17 @@ def _threaded_scatter(self, q_or_i, qout, **kwargs):
             for future in futures:
                 qout.put(future)
 
-    def scatter(self, data, workers=None, broadcast=False, direct=None,
-                hash=True, maxsize=0, timeout=no_default, asynchronous=None):
+    def scatter(
+        self,
+        data,
+        workers=None,
+        broadcast=False,
+        direct=None,
+        hash=True,
+        maxsize=0,
+        timeout=no_default,
+        asynchronous=None,
+    ):
         """ Scatter data into distributed memory
 
         This moves data from the local client process into the workers of the
@@ -1853,11 +2019,12 @@ def scatter(self, data, workers=None, broadcast=False, direct=None,
             logger.debug("Starting thread for streaming data")
             qout = pyQueue(maxsize=maxsize)
 
-            t = threading.Thread(target=self._threaded_scatter,
-                                 name="Threaded scatter()",
-                                 args=(data, qout),
-                                 kwargs={'workers': workers,
-                                         'broadcast': broadcast})
+            t = threading.Thread(
+                target=self._threaded_scatter,
+                name="Threaded scatter()",
+                args=(data, qout),
+                kwargs={"workers": workers, "broadcast": broadcast},
+            )
             t.daemon = True
             t.start()
 
@@ -1866,14 +2033,21 @@ def scatter(self, data, workers=None, broadcast=False, direct=None,
             else:
                 return queue_to_iterator(qout)
         else:
-            if hasattr(thread_state, 'execution_state'):  # within worker task
-                local_worker = thread_state.execution_state['worker']
+            if hasattr(thread_state, "execution_state"):  # within worker task
+                local_worker = thread_state.execution_state["worker"]
             else:
                 local_worker = None
-            return self.sync(self._scatter, data, workers=workers,
-                             broadcast=broadcast, direct=direct,
-                             local_worker=local_worker, timeout=timeout,
-                             asynchronous=asynchronous, hash=hash)
+            return self.sync(
+                self._scatter,
+                data,
+                workers=workers,
+                broadcast=broadcast,
+                direct=direct,
+                local_worker=local_worker,
+                timeout=timeout,
+                asynchronous=asynchronous,
+                hash=hash,
+            )
 
     @gen.coroutine
     def _cancel(self, futures, force=False):
@@ -1898,8 +2072,7 @@ def cancel(self, futures, asynchronous=None, force=False):
         force: boolean (False)
             Cancel this future even if other clients desire it
         """
-        return self.sync(self._cancel, futures, asynchronous=asynchronous,
-                         force=force)
+        return self.sync(self._cancel, futures, asynchronous=asynchronous, force=force)
 
     @gen.coroutine
     def _retry(self, futures):
@@ -1926,17 +2099,19 @@ def _publish_dataset(self, *args, **kwargs):
 
             def add_coro(name, data):
                 keys = [tokey(f.key) for f in futures_of(data)]
-                coroutines.append(self.scheduler.publish_put(keys=keys,
-                                                             name=name,
-                                                             data=to_serialize(data),
-                                                             client=self.id))
+                coroutines.append(
+                    self.scheduler.publish_put(
+                        keys=keys, name=name, data=to_serialize(data), client=self.id
+                    )
+                )
 
-            name = kwargs.pop('name', None)
+            name = kwargs.pop("name", None)
             if name:
                 if len(args) == 0:
                     raise ValueError(
                         "If name is provided, expecting call signature like"
-                        " publish_dataset(df, name='ds')")
+                        " publish_dataset(df, name='ds')"
+                    )
                 # in case this is a singleton, collapse it
                 elif len(args) == 1:
                     args = args[0]
@@ -2031,7 +2206,7 @@ def _get_dataset(self, name):
             raise KeyError("Dataset '%s' not found" % name)
 
         with temp_default_client(self):
-            data = out['data']
+            data = out["data"]
         raise gen.Return(data)
 
     def get_dataset(self, name, **kwargs):
@@ -2047,15 +2222,14 @@ def get_dataset(self, name, **kwargs):
 
     @gen.coroutine
     def _run_on_scheduler(self, function, *args, **kwargs):
-        wait = kwargs.pop('wait', True)
-        response = yield self.scheduler.run_function(function=dumps(function),
-                                                     args=dumps(args),
-                                                     kwargs=dumps(kwargs),
-                                                     wait=wait)
-        if response['status'] == 'error':
+        wait = kwargs.pop("wait", True)
+        response = yield self.scheduler.run_function(
+            function=dumps(function), args=dumps(args), kwargs=dumps(kwargs), wait=wait
+        )
+        if response["status"] == "error":
             six.reraise(*clean_exception(**response))
         else:
-            raise gen.Return(response['result'])
+            raise gen.Return(response["result"])
 
     def run_on_scheduler(self, function, *args, **kwargs):
         """ Run a function on the scheduler process
@@ -2087,25 +2261,29 @@ def run_on_scheduler(self, function, *args, **kwargs):
         Client.run: Run a function on all workers
         Client.start_ipython_scheduler: Start an IPython session on scheduler
         """
-        return self.sync(self._run_on_scheduler, function, *args,
-                         **kwargs)
+        return self.sync(self._run_on_scheduler, function, *args, **kwargs)
 
     @gen.coroutine
     def _run(self, function, *args, **kwargs):
-        nanny = kwargs.pop('nanny', False)
-        workers = kwargs.pop('workers', None)
-        wait = kwargs.pop('wait', True)
-        responses = yield self.scheduler.broadcast(msg=dict(op='run',
-                                                            function=dumps(function),
-                                                            args=dumps(args),
-                                                            wait=wait,
-                                                            kwargs=dumps(kwargs)),
-                                                   workers=workers, nanny=nanny)
+        nanny = kwargs.pop("nanny", False)
+        workers = kwargs.pop("workers", None)
+        wait = kwargs.pop("wait", True)
+        responses = yield self.scheduler.broadcast(
+            msg=dict(
+                op="run",
+                function=dumps(function),
+                args=dumps(args),
+                wait=wait,
+                kwargs=dumps(kwargs),
+            ),
+            workers=workers,
+            nanny=nanny,
+        )
         results = {}
         for key, resp in responses.items():
-            if resp['status'] == 'OK':
-                results[key] = resp['result']
-            elif resp['status'] == 'error':
+            if resp["status"] == "OK":
+                results[key] = resp["result"]
+            elif resp["status"] == "error":
                 six.reraise(*clean_exception(**resp))
         if wait:
             raise gen.Return(results)
@@ -2188,23 +2366,36 @@ def run_coroutine(self, function, *args, **kwargs):
             Workers on which to run the function. Defaults to all known workers.
 
         """
-        warnings.warn("This method has been deprecated. "
-                      "Instead use Client.run which detects async functions "
-                      "automatically")
+        warnings.warn(
+            "This method has been deprecated. "
+            "Instead use Client.run which detects async functions "
+            "automatically"
+        )
         return self.run(function, *args, **kwargs)
 
-    def _graph_to_futures(self, dsk, keys, restrictions=None,
-                          loose_restrictions=None, priority=None,
-                          user_priority=0, resources=None, retries=None,
-                          fifo_timeout=0, actors=None):
+    def _graph_to_futures(
+        self,
+        dsk,
+        keys,
+        restrictions=None,
+        loose_restrictions=None,
+        priority=None,
+        user_priority=0,
+        resources=None,
+        retries=None,
+        fifo_timeout=0,
+        actors=None,
+    ):
         with self._refcount_lock:
             if resources:
-                resources = self._expand_resources(resources,
-                                                   all_keys=itertools.chain(dsk, keys))
+                resources = self._expand_resources(
+                    resources, all_keys=itertools.chain(dsk, keys)
+                )
 
             if retries:
-                retries = self._expand_retries(retries,
-                                               all_keys=itertools.chain(dsk, keys))
+                retries = self._expand_retries(
+                    retries, all_keys=itertools.chain(dsk, keys)
+                )
 
             if actors is not None and actors is not True and actors is not False:
                 actors = list(self._expand_key(actors))
@@ -2213,8 +2404,9 @@ def _graph_to_futures(self, dsk, keys, restrictions=None,
             flatkeys = list(map(tokey, keys))
             futures = {key: Future(key, self, inform=False) for key in keyset}
 
-            values = {k for k, v in dsk.items() if isinstance(v, Future)
-                      and k not in keyset}
+            values = {
+                k for k, v in dsk.items() if isinstance(v, Future) and k not in keyset
+            }
             if values:
                 dsk = dask.optimization.inline(dsk, keys=values)
 
@@ -2225,8 +2417,9 @@ def _graph_to_futures(self, dsk, keys, restrictions=None,
             dsk3 = {k: v for k, v in dsk2.items() if k is not v}
             for future in extra_futures:
                 if future.client is not self:
-                    msg = ("Inputs contain futures that were created by "
-                           "another client.")
+                    msg = (
+                        "Inputs contain futures that were created by " "another client."
+                    )
                     raise ValueError(msg)
 
             if restrictions:
@@ -2236,7 +2429,9 @@ def _graph_to_futures(self, dsk, keys, restrictions=None,
             if loose_restrictions is not None:
                 loose_restrictions = list(map(tokey, loose_restrictions))
 
-            future_dependencies = {tokey(k): {tokey(f.key) for f in v[1]} for k, v in d.items()}
+            future_dependencies = {
+                tokey(k): {tokey(f.key) for f in v[1]} for k, v in d.items()
+            }
 
             for s in future_dependencies.values():
                 for v in s:
@@ -2249,8 +2444,10 @@ def _graph_to_futures(self, dsk, keys, restrictions=None,
                 priority = dask.order.order(dsk, dependencies=dependencies)
                 priority = keymap(tokey, priority)
 
-            dependencies = {tokey(k): [tokey(dep) for dep in deps]
-                            for k, deps in dependencies.items()}
+            dependencies = {
+                tokey(k): [tokey(dep) for dep in deps]
+                for k, deps in dependencies.items()
+            }
             for k, deps in future_dependencies.items():
                 if deps:
                     dependencies[k] = list(set(dependencies.get(k, ())) | deps)
@@ -2258,25 +2455,41 @@ def _graph_to_futures(self, dsk, keys, restrictions=None,
             if isinstance(retries, Number) and retries > 0:
                 retries = {k: retries for k in dsk3}
 
-            self._send_to_scheduler({'op': 'update-graph',
-                                     'tasks': valmap(dumps_task, dsk3),
-                                     'dependencies': dependencies,
-                                     'keys': list(flatkeys),
-                                     'restrictions': restrictions or {},
-                                     'loose_restrictions': loose_restrictions,
-                                     'priority': priority,
-                                     'user_priority': user_priority,
-                                     'resources': resources,
-                                     'submitting_task': getattr(thread_state, 'key', None),
-                                     'retries': retries,
-                                     'fifo_timeout': fifo_timeout,
-                                     'actors': actors})
+            self._send_to_scheduler(
+                {
+                    "op": "update-graph",
+                    "tasks": valmap(dumps_task, dsk3),
+                    "dependencies": dependencies,
+                    "keys": list(flatkeys),
+                    "restrictions": restrictions or {},
+                    "loose_restrictions": loose_restrictions,
+                    "priority": priority,
+                    "user_priority": user_priority,
+                    "resources": resources,
+                    "submitting_task": getattr(thread_state, "key", None),
+                    "retries": retries,
+                    "fifo_timeout": fifo_timeout,
+                    "actors": actors,
+                }
+            )
             return futures
 
-    def get(self, dsk, keys, restrictions=None, loose_restrictions=None,
-            resources=None, sync=True, asynchronous=None, direct=None,
-            retries=None, priority=0, fifo_timeout='60s', actors=None,
-            **kwargs):
+    def get(
+        self,
+        dsk,
+        keys,
+        restrictions=None,
+        loose_restrictions=None,
+        resources=None,
+        sync=True,
+        asynchronous=None,
+        direct=None,
+        retries=None,
+        priority=0,
+        fifo_timeout="60s",
+        actors=None,
+        **kwargs
+    ):
         """ Compute dask graph
 
         Parameters
@@ -2322,19 +2535,18 @@ def get(self, dsk, keys, restrictions=None, loose_restrictions=None,
         )
         packed = pack_data(keys, futures)
         if sync:
-            if getattr(thread_state, 'key', False):
+            if getattr(thread_state, "key", False):
                 try:
                     secede()
                     should_rejoin = True
                 except Exception:
                     should_rejoin = False
             try:
-                results = self.gather(packed, asynchronous=asynchronous,
-                                      direct=direct)
+                results = self.gather(packed, asynchronous=asynchronous, direct=direct)
             finally:
                 for f in futures.values():
                     f.release()
-                if getattr(thread_state, 'key', False) and should_rejoin:
+                if getattr(thread_state, "key", False) and should_rejoin:
                     rejoin()
             return results
         return packed
@@ -2394,10 +2606,20 @@ def normalize_collection(self, collection):
         else:
             return redict_collection(collection, dsk)
 
-    def compute(self, collections, sync=False, optimize_graph=True,
-                workers=None, allow_other_workers=False, resources=None,
-                retries=0, priority=0, fifo_timeout='60s', actors=None,
-                **kwargs):
+    def compute(
+        self,
+        collections,
+        sync=False,
+        optimize_graph=True,
+        workers=None,
+        allow_other_workers=False,
+        resources=None,
+        retries=0,
+        priority=0,
+        fifo_timeout="60s",
+        actors=None,
+        **kwargs
+    ):
         """ Compute dask collections on cluster
 
         Parameters
@@ -2459,16 +2681,19 @@ def compute(self, collections, sync=False, optimize_graph=True,
             collections = [collections]
             singleton = True
 
-        traverse = kwargs.pop('traverse', True)
+        traverse = kwargs.pop("traverse", True)
         if traverse:
-            collections = tuple(dask.delayed(a)
-                                if isinstance(a, (list, set, tuple, dict, Iterator))
-                                else a for a in collections)
+            collections = tuple(
+                dask.delayed(a)
+                if isinstance(a, (list, set, tuple, dict, Iterator))
+                else a
+                for a in collections
+            )
 
         variables = [a for a in collections if dask.is_dask_collection(a)]
 
         dsk = self.collections_to_dsk(variables, optimize_graph, **kwargs)
-        names = ['finalize-%s' % tokenize(v) for v in variables]
+        names = ["finalize-%s" % tokenize(v) for v in variables]
         dsk2 = {}
         for i, (name, v) in enumerate(zip(names, variables)):
             func, extra_args = v.__dask_postcompute__()
@@ -2478,20 +2703,24 @@ def compute(self, collections, sync=False, optimize_graph=True,
             else:
                 dsk2[name] = (func, keys) + extra_args
 
-        restrictions, loose_restrictions = self.get_restrictions(collections,
-                                                                 workers, allow_other_workers)
+        restrictions, loose_restrictions = self.get_restrictions(
+            collections, workers, allow_other_workers
+        )
 
         if not isinstance(priority, Number):
-            priority = {k: p for c, p in priority.items()
-                             for k in self._expand_key(c)}
+            priority = {k: p for c, p in priority.items() for k in self._expand_key(c)}
 
-        futures_dict = self._graph_to_futures(merge(dsk2, dsk), names,
-                                              restrictions, loose_restrictions,
-                                              resources=resources,
-                                              retries=retries,
-                                              user_priority=priority,
-                                              fifo_timeout=fifo_timeout,
-                                              actors=actors)
+        futures_dict = self._graph_to_futures(
+            merge(dsk2, dsk),
+            names,
+            restrictions,
+            loose_restrictions,
+            resources=resources,
+            retries=retries,
+            user_priority=priority,
+            fifo_timeout=fifo_timeout,
+            actors=actors,
+        )
 
         i = 0
         futures = []
@@ -2512,9 +2741,19 @@ def compute(self, collections, sync=False, optimize_graph=True,
         else:
             return result
 
-    def persist(self, collections, optimize_graph=True, workers=None,
-                allow_other_workers=None, resources=None, retries=None,
-                priority=0, fifo_timeout='60s', actors=None, **kwargs):
+    def persist(
+        self,
+        collections,
+        optimize_graph=True,
+        workers=None,
+        allow_other_workers=None,
+        resources=None,
+        retries=None,
+        priority=0,
+        fifo_timeout="60s",
+        actors=None,
+        **kwargs
+    ):
         """ Persist dask collections on cluster
 
         Starts computation of the collection on the cluster in the background.
@@ -2571,24 +2810,30 @@ def persist(self, collections, optimize_graph=True, workers=None,
 
         names = {k for c in collections for k in flatten(c.__dask_keys__())}
 
-        restrictions, loose_restrictions = self.get_restrictions(collections,
-                                                                 workers, allow_other_workers)
+        restrictions, loose_restrictions = self.get_restrictions(
+            collections, workers, allow_other_workers
+        )
 
         if not isinstance(priority, Number):
-            priority = {k: p for c, p in priority.items()
-                             for k in self._expand_key(c)}
+            priority = {k: p for c, p in priority.items() for k in self._expand_key(c)}
 
-        futures = self._graph_to_futures(dsk, names, restrictions,
-                                         loose_restrictions,
-                                         resources=resources,
-                                         retries=retries,
-                                         user_priority=priority,
-                                         fifo_timeout=fifo_timeout,
-                                         actors=actors)
+        futures = self._graph_to_futures(
+            dsk,
+            names,
+            restrictions,
+            loose_restrictions,
+            resources=resources,
+            retries=retries,
+            user_priority=priority,
+            fifo_timeout=fifo_timeout,
+            actors=actors,
+        )
 
         postpersists = [c.__dask_postpersist__() for c in collections]
-        result = [func({k: futures[k] for k in flatten(c.__dask_keys__())}, *args)
-                  for (func, args), c in zip(postpersists, collections)]
+        result = [
+            func({k: futures[k] for k in flatten(c.__dask_keys__())}, *args)
+            for (func, args), c in zip(postpersists, collections)
+        ]
 
         if singleton:
             return first(result)
@@ -2604,6 +2849,7 @@ def unzip(dask_worker=None):
             from distributed.utils import log_errors
             import zipfile
             import shutil
+
             with log_errors():
                 a = os.path.join(dask_worker.worker_dir, name)
                 b = os.path.join(dask_worker.local_dir, name)
@@ -2613,7 +2859,7 @@ def unzip(dask_worker=None):
                 with zipfile.ZipFile(b) as f:
                     f.extractall(path=c)
 
-                for fn in glob(os.path.join(c, name[:-4], 'bin', '*')):
+                for fn in glob(os.path.join(c, name[:-4], "bin", "*")):
                     st = os.stat(fn)
                     os.chmod(fn, st.st_mode | 64)  # chmod u+x fn
 
@@ -2630,7 +2876,7 @@ def upload_environment(self, name, zipfile):
     def _restart(self, timeout=no_default):
         if timeout == no_default:
             timeout = self._timeout * 2
-        self._send_to_scheduler({'op': 'restart', 'timeout': timeout})
+        self._send_to_scheduler({"op": "restart", "timeout": timeout})
         self._restart_event = Event()
         try:
             yield self._restart_event.wait(self.loop.time() + timeout)
@@ -2653,29 +2899,28 @@ def restart(self, **kwargs):
 
     @gen.coroutine
     def _upload_file(self, filename, raise_on_error=True):
-        with open(filename, 'rb') as f:
+        with open(filename, "rb") as f:
             data = f.read()
         _, fn = os.path.split(filename)
-        d = yield self.scheduler.broadcast(msg={'op': 'upload_file',
-                                                'filename': fn,
-                                                'data': to_serialize(data)})
+        d = yield self.scheduler.broadcast(
+            msg={"op": "upload_file", "filename": fn, "data": to_serialize(data)}
+        )
 
-        if any(v['status'] == 'error' for v in d.values()):
-            exceptions = [v['exception'] for v in d.values()
-                          if v['status'] == 'error']
+        if any(v["status"] == "error" for v in d.values()):
+            exceptions = [v["exception"] for v in d.values() if v["status"] == "error"]
             if raise_on_error:
                 raise exceptions[0]
             else:
                 raise gen.Return(exceptions[0])
 
-        assert all(len(data) == v['nbytes'] for v in d.values())
+        assert all(len(data) == v["nbytes"] for v in d.values())
 
     @gen.coroutine
     def _upload_large_file(self, local_filename, remote_filename=None):
         if remote_filename is None:
             remote_filename = os.path.split(local_filename)[1]
 
-        with open(local_filename, 'rb') as f:
+        with open(local_filename, "rb") as f:
             data = f.read()
 
         [future] = yield self._scatter([data])
@@ -2687,7 +2932,7 @@ def dump_to_file(dask_worker=None):
                 fn = os.path.join(dask_worker.local_dir, remote_filename)
             else:
                 fn = remote_filename
-            with open(fn, 'wb') as f:
+            with open(fn, "wb") as f:
                 f.write(dask_worker.data[key])
 
             return len(dask_worker.data[key])
@@ -2714,8 +2959,9 @@ def upload_file(self, filename, **kwargs):
         >>> from mylibrary import myfunc  # doctest: +SKIP
         >>> L = c.map(myfunc, seq)  # doctest: +SKIP
         """
-        result = self.sync(self._upload_file, filename,
-                           raise_on_error=self.asynchronous, **kwargs)
+        result = self.sync(
+            self._upload_file, filename, raise_on_error=self.asynchronous, **kwargs
+        )
         if isinstance(result, Exception):
             raise result
         else:
@@ -2726,7 +2972,7 @@ def _rebalance(self, futures=None, workers=None):
         yield _wait(futures)
         keys = list({tokey(f.key) for f in self.futures_of(futures)})
         result = yield self.scheduler.rebalance(keys=keys, workers=workers)
-        assert result['status'] == 'OK'
+        assert result["status"] == "OK"
 
     def rebalance(self, futures=None, workers=None, **kwargs):
         """ Rebalance data within network
@@ -2753,11 +2999,11 @@ def _replicate(self, futures, n=None, workers=None, branching_factor=2):
         futures = self.futures_of(futures)
         yield _wait(futures)
         keys = {tokey(f.key) for f in futures}
-        yield self.scheduler.replicate(keys=list(keys), n=n, workers=workers,
-                                       branching_factor=branching_factor)
+        yield self.scheduler.replicate(
+            keys=list(keys), n=n, workers=workers, branching_factor=branching_factor
+        )
 
-    def replicate(self, futures, n=None, workers=None, branching_factor=2,
-                  **kwargs):
+    def replicate(self, futures, n=None, workers=None, branching_factor=2, **kwargs):
         """ Set replication of futures within network
 
         Copy data onto many workers.  This helps to broadcast frequently
@@ -2793,8 +3039,14 @@ def replicate(self, futures, n=None, workers=None, branching_factor=2,
         --------
         Client.rebalance
         """
-        return self.sync(self._replicate, futures, n=n, workers=workers,
-                         branching_factor=branching_factor, **kwargs)
+        return self.sync(
+            self._replicate,
+            futures,
+            n=n,
+            workers=workers,
+            branching_factor=branching_factor,
+            **kwargs
+        )
 
     def ncores(self, workers=None, **kwargs):
         """ The number of threads/cores available on each worker node
@@ -2818,8 +3070,9 @@ def ncores(self, workers=None, **kwargs):
         Client.who_has
         Client.has_what
         """
-        if (isinstance(workers, tuple)
-                and all(isinstance(i, (str, tuple)) for i in workers)):
+        if isinstance(workers, tuple) and all(
+            isinstance(i, (str, tuple)) for i in workers
+        ):
             workers = list(workers)
         if workers is not None and not isinstance(workers, (tuple, list, set)):
             workers = [workers]
@@ -2884,8 +3137,9 @@ def has_what(self, workers=None, **kwargs):
         Client.ncores
         Client.processing
         """
-        if (isinstance(workers, tuple)
-                and all(isinstance(i, (str, tuple)) for i in workers)):
+        if isinstance(workers, tuple) and all(
+            isinstance(i, (str, tuple)) for i in workers
+        ):
             workers = list(workers)
         if workers is not None and not isinstance(workers, (tuple, list, set)):
             workers = [workers]
@@ -2913,8 +3167,9 @@ def processing(self, workers=None):
         Client.has_what
         Client.ncores
         """
-        if (isinstance(workers, tuple)
-                and all(isinstance(i, (str, tuple)) for i in workers)):
+        if isinstance(workers, tuple) and all(
+            isinstance(i, (str, tuple)) for i in workers
+        ):
             workers = list(workers)
         if workers is not None and not isinstance(workers, (tuple, list, set)):
             workers = [workers]
@@ -2948,8 +3203,7 @@ def nbytes(self, keys=None, summary=True, **kwargs):
         --------
         Client.who_has
         """
-        return self.sync(self.scheduler.nbytes, keys=keys,
-                         summary=summary, **kwargs)
+        return self.sync(self.scheduler.nbytes, keys=keys, summary=summary, **kwargs)
 
     def call_stack(self, futures=None, keys=None):
         """ The actively running call stack of all relevant keys
@@ -2979,8 +3233,16 @@ def call_stack(self, futures=None, keys=None):
             keys += list(map(tokey, {f.key for f in futures}))
         return self.sync(self.scheduler.call_stack, keys=keys or None)
 
-    def profile(self, key=None, start=None, stop=None, workers=None,
-                merge_workers=True, plot=False, filename=None):
+    def profile(
+        self,
+        key=None,
+        start=None,
+        stop=None,
+        workers=None,
+        merge_workers=True,
+        plot=False,
+        filename=None,
+    ):
         """ Collect statistical profiling information about recent work
 
         Parameters
@@ -3005,32 +3267,54 @@ def profile(self, key=None, start=None, stop=None, workers=None,
         if isinstance(workers, six.string_types + (Number,)):
             workers = [workers]
 
-        return self.sync(self._profile, key=key, workers=workers,
-                         merge_workers=merge_workers, start=start, stop=stop,
-                         plot=plot, filename=filename)
+        return self.sync(
+            self._profile,
+            key=key,
+            workers=workers,
+            merge_workers=merge_workers,
+            start=start,
+            stop=stop,
+            plot=plot,
+            filename=filename,
+        )
 
     @gen.coroutine
-    def _profile(self, key=None, start=None, stop=None, workers=None,
-                 merge_workers=True, plot=False, filename=None):
+    def _profile(
+        self,
+        key=None,
+        start=None,
+        stop=None,
+        workers=None,
+        merge_workers=True,
+        plot=False,
+        filename=None,
+    ):
         if isinstance(workers, six.string_types + (Number,)):
             workers = [workers]
 
-        state = yield self.scheduler.profile(key=key, workers=workers,
-                merge_workers=merge_workers, start=start, stop=stop)
+        state = yield self.scheduler.profile(
+            key=key,
+            workers=workers,
+            merge_workers=merge_workers,
+            start=start,
+            stop=stop,
+        )
 
         if filename:
             plot = True
 
         if plot:
             from . import profile
+
             data = profile.plot_data(state)
-            figure, source = profile.plot_figure(data, sizing_mode='stretch_both')
+            figure, source = profile.plot_figure(data, sizing_mode="stretch_both")
 
-            if plot == 'save' and not filename:
-                filename = 'dask-profile.html'
+            if plot == "save" and not filename:
+                filename = "dask-profile.html"
 
             from bokeh.plotting import save
-            save(figure, title='Dask Profile', filename=filename)
+
+            save(figure, title="Dask Profile", filename=filename)
             raise gen.Return((state, figure))
 
         else:
@@ -3075,11 +3359,11 @@ def write_scheduler_file(self, scheduler_file):
         >>> client2 = Client(scheduler_file='scheduler.json')  # doctest: +SKIP
         """
         if self.scheduler_file:
-            raise ValueError('Scheduler file already set')
+            raise ValueError("Scheduler file already set")
         else:
             self.scheduler_file = scheduler_file
 
-        with open(self.scheduler_file, 'w') as f:
+        with open(self.scheduler_file, "w") as f:
             json.dump(self.scheduler_info(), f, indent=2)
 
     def get_metadata(self, keys, default=no_default):
@@ -3102,8 +3386,7 @@ def get_metadata(self, keys, default=no_default):
         """
         if not isinstance(keys, (list, tuple)):
             keys = (keys,)
-        return self.sync(self.scheduler.get_metadata, keys=keys,
-                         default=default)
+        return self.sync(self.scheduler.get_metadata, keys=keys, default=default)
 
     def get_scheduler_logs(self, n=None):
         """ Get logs from scheduler
@@ -3155,8 +3438,12 @@ def retire_workers(self, workers=None, close_workers=True, **kwargs):
         --------
         dask.distributed.Scheduler.retire_workers
         """
-        return self.sync(self.scheduler.retire_workers, workers=workers,
-                         close_workers=close_workers, **kwargs)
+        return self.sync(
+            self.scheduler.retire_workers,
+            workers=workers,
+            close_workers=close_workers,
+            **kwargs
+        )
 
     def set_metadata(self, key, value):
         """ Set arbitrary metadata in the scheduler
@@ -3224,44 +3511,46 @@ def get_versions(self, check=False, packages=[]):
         """
         client = get_versions(packages=packages)
         try:
-            scheduler = sync(self.loop, self.scheduler.versions,
-                             packages=packages)
+            scheduler = sync(self.loop, self.scheduler.versions, packages=packages)
         except KeyError:
             scheduler = None
         except TypeError:  # packages keyword not supported
             scheduler = sync(self.loop, self.scheduler.versions)  # this raises
 
-        workers = sync(self.loop, self.scheduler.broadcast,
-                       msg={'op': 'versions', 'packages': packages})
-        result = {'scheduler': scheduler, 'workers': workers, 'client': client}
+        workers = sync(
+            self.loop,
+            self.scheduler.broadcast,
+            msg={"op": "versions", "packages": packages},
+        )
+        result = {"scheduler": scheduler, "workers": workers, "client": client}
 
         if check:
             # we care about the required & optional packages matching
             def to_packages(d):
-                L = list(d['packages'].values())
+                L = list(d["packages"].values())
                 return dict(sum(L, type(L[0])()))
-            client_versions = to_packages(result['client'])
-            versions = [('scheduler', to_packages(result['scheduler']))]
-            versions.extend((w, to_packages(d))
-                            for w, d in sorted(workers.items()))
+
+            client_versions = to_packages(result["client"])
+            versions = [("scheduler", to_packages(result["scheduler"]))]
+            versions.extend((w, to_packages(d)) for w, d in sorted(workers.items()))
 
             mismatched = defaultdict(list)
             for name, vers in versions:
                 for pkg, cv in client_versions.items():
-                    v = vers.get(pkg, 'MISSING')
+                    v = vers.get(pkg, "MISSING")
                     if cv != v:
                         mismatched[pkg].append((name, v))
 
             if mismatched:
                 errs = []
                 for pkg, versions in sorted(mismatched.items()):
-                    rows = [('client', client_versions[pkg])]
+                    rows = [("client", client_versions[pkg])]
                     rows.extend(versions)
-                    errs.append("%s\n%s" % (pkg, asciitable(['', 'version'], rows)))
+                    errs.append("%s\n%s" % (pkg, asciitable(["", "version"], rows)))
 
-                raise ValueError("Mismatched versions found\n"
-                                 "\n"
-                                 "%s" % ('\n\n'.join(errs)))
+                raise ValueError(
+                    "Mismatched versions found\n" "\n" "%s" % ("\n\n".join(errs))
+                )
 
         return result
 
@@ -3277,12 +3566,13 @@ def _start_ipython_workers(self, workers):
             workers = yield self.scheduler.ncores()
 
         responses = yield self.scheduler.broadcast(
-            msg=dict(op='start_ipython'), workers=workers,
+            msg=dict(op="start_ipython"), workers=workers
         )
         raise gen.Return((workers, responses))
 
-    def start_ipython_workers(self, workers=None, magic_names=False,
-                              qtconsole=False, qtconsole_args=None):
+    def start_ipython_workers(
+        self, workers=None, magic_names=False, qtconsole=False, qtconsole_args=None
+    ):
         """ Start IPython kernels on workers
 
         Parameters
@@ -3337,31 +3627,34 @@ def start_ipython_workers(self, workers=None, magic_names=False,
         (workers, info_dict) = sync(self.loop, self._start_ipython_workers, workers)
 
         if magic_names and isinstance(magic_names, six.string_types):
-            if '*' in magic_names:
-                magic_names = [magic_names.replace('*', str(i))
-                               for i in range(len(workers))]
+            if "*" in magic_names:
+                magic_names = [
+                    magic_names.replace("*", str(i)) for i in range(len(workers))
+                ]
             else:
                 magic_names = [magic_names]
 
-        if 'IPython' in sys.modules:
+        if "IPython" in sys.modules:
             from ._ipython_utils import register_remote_magic
+
             register_remote_magic()
         if magic_names:
             from ._ipython_utils import register_worker_magic
+
             for worker, magic_name in zip(workers, magic_names):
                 connection_info = info_dict[worker]
                 register_worker_magic(connection_info, magic_name)
         if qtconsole:
             from ._ipython_utils import connect_qtconsole
+
             for worker, connection_info in info_dict.items():
-                name = 'dask-' + worker.replace(':', '-').replace('/', '-')
-                connect_qtconsole(connection_info, name=name,
-                                  extra_args=qtconsole_args,
-                                  )
+                name = "dask-" + worker.replace(":", "-").replace("/", "-")
+                connect_qtconsole(connection_info, name=name, extra_args=qtconsole_args)
         return info_dict
 
-    def start_ipython_scheduler(self, magic_name='scheduler_if_ipython',
-                                qtconsole=False, qtconsole_args=None):
+    def start_ipython_scheduler(
+        self, magic_name="scheduler_if_ipython", qtconsole=False, qtconsole_args=None
+    ):
         """ Start IPython kernel on the scheduler
 
         Parameters
@@ -3397,23 +3690,25 @@ def start_ipython_scheduler(self, magic_name='scheduler_if_ipython',
         Client.start_ipython_workers: Start IPython on the workers
         """
         info = sync(self.loop, self.scheduler.start_ipython)
-        if magic_name == 'scheduler_if_ipython':
+        if magic_name == "scheduler_if_ipython":
             # default to %scheduler if in IPython, no magic otherwise
             in_ipython = False
-            if 'IPython' in sys.modules:
+            if "IPython" in sys.modules:
                 from IPython import get_ipython
+
                 in_ipython = bool(get_ipython())
             if in_ipython:
-                magic_name = 'scheduler'
+                magic_name = "scheduler"
             else:
                 magic_name = None
         if magic_name:
             from ._ipython_utils import register_worker_magic
+
             register_worker_magic(info, magic_name)
         if qtconsole:
             from ._ipython_utils import connect_qtconsole
-            connect_qtconsole(info, name='dask-scheduler',
-                              extra_args=qtconsole_args,)
+
+            connect_qtconsole(info, name="dask-scheduler", extra_args=qtconsole_args)
         return info
 
     @classmethod
@@ -3438,15 +3733,18 @@ def _expand_retries(cls, retries, all_keys):
         to a {task key: Integral} dictionary.
         """
         if retries and isinstance(retries, dict):
-            result = {name: value
-                      for key, value in retries.items()
-                      for name in cls._expand_key(key)}
+            result = {
+                name: value
+                for key, value in retries.items()
+                for name in cls._expand_key(key)
+            }
         elif isinstance(retries, Integral):
             # Each task unit may potentially fail, allow retrying all of them
             result = {name: retries for name in all_keys}
         else:
-            raise TypeError("`retries` should be an integer or dict, got %r"
-                            % (type(retries,)))
+            raise TypeError(
+                "`retries` should be an integer or dict, got %r" % (type(retries))
+            )
         return keymap(tokey, result)
 
     def _expand_resources(cls, resources, all_keys):
@@ -3459,8 +3757,7 @@ def _expand_resources(cls, resources, all_keys):
         # such as {'x': {'GPU': 1}, 'y': {'SSD': 4}} indicating
         # per-key requirements
         if not isinstance(resources, dict):
-            raise TypeError("`resources` should be a dict, got %r"
-                            % (type(resources,)))
+            raise TypeError("`resources` should be a dict, got %r" % (type(resources)))
 
         per_key_reqs = {}
         global_reqs = {}
@@ -3474,8 +3771,10 @@ def _expand_resources(cls, resources, all_keys):
                 global_reqs.update((kk, {k: v}) for kk in all_keys)
 
         if global_reqs and per_key_reqs:
-            raise ValueError("cannot have both per-key and all-key requirements "
-                             "in resources dict %r" % (resources,))
+            raise ValueError(
+                "cannot have both per-key and all-key requirements "
+                "in resources dict %r" % (resources,)
+            )
         return global_reqs or per_key_reqs
 
     @classmethod
@@ -3491,8 +3790,9 @@ def get_restrictions(cls, collections, workers, allow_other_workers):
                 if dask.is_dask_collection(colls):
                     keys = flatten(colls.__dask_keys__())
                 else:
-                    keys = list({k for c in flatten(colls)
-                                 for k in flatten(c.__dask_keys__())})
+                    keys = list(
+                        {k for c in flatten(colls) for k in flatten(c.__dask_keys__())}
+                    )
                 restrictions.update({k: ws for k in keys})
         else:
             restrictions = {}
@@ -3500,8 +3800,9 @@ def get_restrictions(cls, collections, workers, allow_other_workers):
         if allow_other_workers is True:
             loose_restrictions = list(restrictions)
         elif allow_other_workers:
-            loose_restrictions = list({k for c in flatten(allow_other_workers)
-                                       for k in c.__dask_keys__()})
+            loose_restrictions = list(
+                {k for c in flatten(allow_other_workers) for k in c.__dask_keys__()}
+            )
         else:
             loose_restrictions = []
 
@@ -3511,8 +3812,9 @@ def get_restrictions(cls, collections, workers, allow_other_workers):
     def collections_to_dsk(collections, *args, **kwargs):
         return collections_to_dsk(collections, *args, **kwargs)
 
-    def get_task_stream(self, start=None, stop=None, count=None, plot=False,
-                        filename='task-stream.html'):
+    def get_task_stream(
+        self, start=None, stop=None, count=None, plot=False, filename="task-stream.html"
+    ):
         """ Get task stream data from scheduler
 
         This collects the data present in the diagnostic "Task Stream" plot on
@@ -3572,23 +3874,32 @@ def get_task_stream(self, start=None, stop=None, count=None, plot=False,
         --------
         get_task_stream: a context manager version of this method
         """
-        return self.sync(self._get_task_stream, start=start, stop=stop,
-                         count=count, plot=plot, filename=filename)
+        return self.sync(
+            self._get_task_stream,
+            start=start,
+            stop=stop,
+            count=count,
+            plot=plot,
+            filename=filename,
+        )
 
     @gen.coroutine
-    def _get_task_stream(self, start=None, stop=None, count=None, plot=False,
-                         filename='task-stream.html'):
-        msgs = yield self.scheduler.get_task_stream(start=start,
-                         stop=stop, count=count)
+    def _get_task_stream(
+        self, start=None, stop=None, count=None, plot=False, filename="task-stream.html"
+    ):
+        msgs = yield self.scheduler.get_task_stream(start=start, stop=stop, count=count)
         if plot:
             from .diagnostics.task_stream import rectangles
+
             rects = rectangles(msgs)
             from .bokeh.components import task_stream_figure
-            source, figure = task_stream_figure(sizing_mode='stretch_both')
+
+            source, figure = task_stream_figure(sizing_mode="stretch_both")
             source.data.update(rects)
-            if plot == 'save':
+            if plot == "save":
                 from bokeh.plotting import save
-                save(figure, title='Dask Task Stream', filename=filename)
+
+                save(figure, title="Dask Task Stream", filename=filename)
             raise gen.Return((msgs, figure))
         else:
             raise gen.Return(msgs)
@@ -3598,9 +3909,9 @@ def _register_worker_callbacks(self, setup=None):
         responses = yield self.scheduler.register_worker_callbacks(setup=dumps(setup))
         results = {}
         for key, resp in responses.items():
-            if resp['status'] == 'OK':
-                results[key] = resp['result']
-            elif resp['status'] == 'error':
+            if resp["status"] == "OK":
+                results[key] = resp["result"]
+            elif resp["status"] == "error":
                 six.reraise(*clean_exception(**resp))
         raise gen.Return(results)
 
@@ -3637,35 +3948,39 @@ def CompatibleExecutor(*args, **kwargs):
     raise Exception("This has been moved to the Client.get_executor() method")
 
 
-ALL_COMPLETED = 'ALL_COMPLETED'
-FIRST_COMPLETED = 'FIRST_COMPLETED'
+ALL_COMPLETED = "ALL_COMPLETED"
+FIRST_COMPLETED = "FIRST_COMPLETED"
 
 
 @gen.coroutine
 def _wait(fs, timeout=None, return_when=ALL_COMPLETED):
     if timeout is not None and not isinstance(timeout, Number):
-        raise TypeError("timeout= keyword received a non-numeric value.\n"
-                        "Beware that wait expects a list of values\n"
-                        "  Bad:  wait(x, y, z)\n"
-                        "  Good: wait([x, y, z])")
+        raise TypeError(
+            "timeout= keyword received a non-numeric value.\n"
+            "Beware that wait expects a list of values\n"
+            "  Bad:  wait(x, y, z)\n"
+            "  Good: wait([x, y, z])"
+        )
     fs = futures_of(fs)
     if return_when == ALL_COMPLETED:
         wait_for = All
     elif return_when == FIRST_COMPLETED:
         wait_for = Any
     else:
-        raise NotImplementedError("Only return_when='ALL_COMPLETED' and 'FIRST_COMPLETED' are "
-                                  "supported")
+        raise NotImplementedError(
+            "Only return_when='ALL_COMPLETED' and 'FIRST_COMPLETED' are " "supported"
+        )
 
     future = wait_for({f._state.wait() for f in fs})
     if timeout is not None:
         future = gen.with_timeout(timedelta(seconds=timeout), future)
     yield future
 
-    done, not_done = ({fu for fu in fs if fu.status != 'pending'},
-                      {fu for fu in fs if fu.status == 'pending'})
-    cancelled = [f.key for f in done
-                 if f.status == 'cancelled']
+    done, not_done = (
+        {fu for fu in fs if fu.status != "pending"},
+        {fu for fu in fs if fu.status == "pending"},
+    )
+    cancelled = [f.key for f in done if f.status == "cancelled"]
     if cancelled:
         raise CancelledError(cancelled)
 
@@ -3854,7 +4169,7 @@ def _get_and_raise(self):
         res = self.queue.get()
         if self.with_results:
             future, result = res
-            if self.raise_errors and future.status == 'error':
+            if self.raise_errors and future.status == "error":
                 six.reraise(*result)
         return res
 
@@ -3945,19 +4260,22 @@ def default_client(c=None):
     if c:
         return c
     else:
-        raise ValueError("No clients found\n"
-                         "Start an client and point it to the scheduler address\n"
-                         "  from distributed import Client\n"
-                         "  client = Client('ip-addr-of-scheduler:8786')\n")
+        raise ValueError(
+            "No clients found\n"
+            "Start an client and point it to the scheduler address\n"
+            "  from distributed import Client\n"
+            "  client = Client('ip-addr-of-scheduler:8786')\n"
+        )
 
 
 def ensure_default_get(client):
-    dask.config.set(scheduler='dask.distributed')
+    dask.config.set(scheduler="dask.distributed")
     _set_global_client(client)
 
 
 def redict_collection(c, dsk):
     from dask.delayed import Delayed
+
     if isinstance(c, Delayed):
         return Delayed(c.key, dsk)
     else:
@@ -4016,9 +4334,13 @@ def fire_and_forget(obj):
     """
     futures = futures_of(obj)
     for future in futures:
-        future.client._send_to_scheduler({'op': 'client-desires-keys',
-                                          'keys': [tokey(future.key)],
-                                          'client': 'fire-and-forget'})
+        future.client._send_to_scheduler(
+            {
+                "op": "client-desires-keys",
+                "keys": [tokey(future.key)],
+                "client": "fire-and-forget",
+            }
+        )
 
 
 class get_task_stream(object):
@@ -4069,7 +4391,8 @@ class get_task_stream(object):
     --------
     Client.get_task_stream: Function version of this context manager
     """
-    def __init__(self, client=None, plot=False, filename='task-stream.html'):
+
+    def __init__(self, client=None, plot=False, filename="task-stream.html"):
         self.data = []
         self._plot = plot
         self._filename = filename
@@ -4082,8 +4405,9 @@ def __enter__(self):
         return self
 
     def __exit__(self, typ, value, traceback):
-        L = self.client.get_task_stream(start=self.start, plot=self._plot,
-                                        filename=self._filename)
+        L = self.client.get_task_stream(
+            start=self.start, plot=self._plot, filename=self._filename
+        )
         if self._plot:
             L, self.figure = L
         self.data.extend(L)
@@ -4094,8 +4418,9 @@ def __aenter__(self):
 
     @gen.coroutine
     def __aexit__(self, typ, value, traceback):
-        L = yield self.client.get_task_stream(start=self.start, plot=self._plot,
-                                              filename=self._filename)
+        L = yield self.client.get_task_stream(
+            start=self.start, plot=self._plot, filename=self._filename
+        )
         if self._plot:
             L, self.figure = L
         self.data.extend(L)
diff --git a/distributed/comm/__init__.py b/distributed/comm/__init__.py
index dfda0459a54..0f7c701847d 100644
--- a/distributed/comm/__init__.py
+++ b/distributed/comm/__init__.py
@@ -1,11 +1,16 @@
 from __future__ import print_function, division, absolute_import
 
-from .addressing import (parse_address, unparse_address,
-                         normalize_address, parse_host_port,
-                         unparse_host_port, resolve_address,
-                         get_address_host_port, get_address_host,
-                         get_local_address_for,
-                         )
+from .addressing import (
+    parse_address,
+    unparse_address,
+    normalize_address,
+    parse_host_port,
+    unparse_host_port,
+    resolve_address,
+    get_address_host_port,
+    get_address_host,
+    get_local_address_for,
+)
 from .core import connect, listen, Comm, CommClosedError
 
 
diff --git a/distributed/comm/addressing.py b/distributed/comm/addressing.py
index 8ff401475b9..20ddb2c863f 100644
--- a/distributed/comm/addressing.py
+++ b/distributed/comm/addressing.py
@@ -7,7 +7,7 @@
 from . import registry
 
 
-DEFAULT_SCHEME = dask.config.get('distributed.comm.default-scheme')
+DEFAULT_SCHEME = dask.config.get("distributed.comm.default-scheme")
 
 
 def parse_address(addr, strict=False):
@@ -21,11 +21,13 @@ def parse_address(addr, strict=False):
     """
     if not isinstance(addr, six.string_types):
         raise TypeError("expected str, got %r" % addr.__class__.__name__)
-    scheme, sep, loc = addr.rpartition('://')
+    scheme, sep, loc = addr.rpartition("://")
     if strict and not sep:
-        msg = ("Invalid url scheme. "
-               "Must include protocol like tcp://localhost:8000. "
-               "Got %s" % addr)
+        msg = (
+            "Invalid url scheme. "
+            "Must include protocol like tcp://localhost:8000. "
+            "Got %s" % addr
+        )
         raise ValueError(msg)
     if not sep:
         scheme = DEFAULT_SCHEME
@@ -39,7 +41,7 @@ def unparse_address(scheme, loc):
     >>> unparse_address('tcp', '127.0.0.1')
     'tcp://127.0.0.1'
     """
-    return '%s://%s' % (scheme, loc)
+    return "%s://%s" % (scheme, loc)
 
 
 def normalize_address(addr):
@@ -69,24 +71,24 @@ def _default():
             raise ValueError("missing port number in address %r" % (address,))
         return default_port
 
-    if address.startswith('['):
+    if address.startswith("["):
         # IPv6 notation: '[addr]:port' or '[addr]'.
         # The address may contain multiple colons.
-        host, sep, tail = address[1:].partition(']')
+        host, sep, tail = address[1:].partition("]")
         if not sep:
             _fail()
         if not tail:
             port = _default()
         else:
-            if not tail.startswith(':'):
+            if not tail.startswith(":"):
                 _fail()
             port = tail[1:]
     else:
         # Generic notation: 'addr:port' or 'addr'.
-        host, sep, port = address.partition(':')
+        host, sep, port = address.partition(":")
         if not sep:
             port = _default()
-        elif ':' in host:
+        elif ":" in host:
             _fail()
 
     return host, int(port)
@@ -96,10 +98,10 @@ def unparse_host_port(host, port=None):
     """
     Undo parse_host_port().
     """
-    if ':' in host and not host.startswith('['):
-        host = '[%s]' % host
+    if ":" in host and not host.startswith("["):
+        host = "[%s]" % host
     if port:
-        return '%s:%s' % (host, port)
+        return "%s:%s" % (host, port)
     else:
         return host
 
@@ -119,8 +121,9 @@ def get_address_host_port(addr, strict=False):
     try:
         return backend.get_address_host_port(loc)
     except NotImplementedError:
-        raise ValueError("don't know how to extract host and port "
-                         "for address %r" % (addr,))
+        raise ValueError(
+            "don't know how to extract host and port " "for address %r" % (addr,)
+        )
 
 
 def get_address_host(addr):
diff --git a/distributed/comm/core.py b/distributed/comm/core.py
index a7aaf7217db..b66be0b6dc4 100644
--- a/distributed/comm/core.py
+++ b/distributed/comm/core.py
@@ -116,12 +116,14 @@ def __repr__(self):
         if self.closed():
             return "<closed %s>" % (clsname,)
         else:
-            return ("<%s local=%s remote=%s>"
-                    % (clsname, self.local_address, self.peer_address))
+            return "<%s local=%s remote=%s>" % (
+                clsname,
+                self.local_address,
+                self.peer_address,
+            )
 
 
 class Listener(with_metaclass(ABCMeta)):
-
     @abstractmethod
     def start(self):
         """
@@ -158,7 +160,6 @@ def __exit__(self, *exc):
 
 
 class Connector(with_metaclass(ABCMeta)):
-
     @abstractmethod
     def connect(self, address, deserialize=True):
         """
@@ -177,8 +178,8 @@ def connect(addr, timeout=None, deserialize=True, connection_args=None):
     retried until the *timeout* is expired.
     """
     if timeout is None:
-        timeout = dask.config.get('distributed.comm.timeouts.connect')
-    timeout = parse_timedelta(timeout, default='seconds')
+        timeout = dask.config.get("distributed.comm.timeouts.connect")
+    timeout = parse_timedelta(timeout, default="seconds")
 
     scheme, loc = parse_address(addr)
     backend = registry.get_backend(scheme)
@@ -190,18 +191,24 @@ def connect(addr, timeout=None, deserialize=True, connection_args=None):
 
     def _raise(error):
         error = error or "connect() didn't finish in time"
-        msg = ("Timed out trying to connect to %r after %s s: %s"
-               % (addr, timeout, error))
+        msg = "Timed out trying to connect to %r after %s s: %s" % (
+            addr,
+            timeout,
+            error,
+        )
         raise IOError(msg)
 
     # This starts a thread
     while True:
         try:
-            future = connector.connect(loc, deserialize=deserialize,
-                                       **(connection_args or {}))
-            comm = yield gen.with_timeout(timedelta(seconds=deadline - time()),
-                                          future,
-                                          quiet_exceptions=EnvironmentError)
+            future = connector.connect(
+                loc, deserialize=deserialize, **(connection_args or {})
+            )
+            comm = yield gen.with_timeout(
+                timedelta(seconds=deadline - time()),
+                future,
+                quiet_exceptions=EnvironmentError,
+            )
         except FatalCommClosedError:
             raise
         except EnvironmentError as e:
@@ -231,13 +238,14 @@ def listen(addr, handle_comm, deserialize=True, connection_args=None):
     try:
         scheme, loc = parse_address(addr, strict=True)
     except ValueError:
-        if connection_args and connection_args.get('ssl_context'):
-            addr = 'tls://' + addr
+        if connection_args and connection_args.get("ssl_context"):
+            addr = "tls://" + addr
         else:
-            addr = 'tcp://' + addr
+            addr = "tcp://" + addr
         scheme, loc = parse_address(addr, strict=True)
 
     backend = registry.get_backend(scheme)
 
-    return backend.get_listener(loc, handle_comm, deserialize,
-                                **(connection_args or {}))
+    return backend.get_listener(
+        loc, handle_comm, deserialize, **(connection_args or {})
+    )
diff --git a/distributed/comm/inproc.py b/distributed/comm/inproc.py
index f5d9adf7de2..8721a3df8ac 100644
--- a/distributed/comm/inproc.py
+++ b/distributed/comm/inproc.py
@@ -21,9 +21,9 @@
 
 logger = logging.getLogger(__name__)
 
-ConnectionRequest = namedtuple('ConnectionRequest',
-                               ('c2s_q', 's2c_q', 'c_loop', 'c_addr',
-                                'conn_event'))
+ConnectionRequest = namedtuple(
+    "ConnectionRequest", ("c2s_q", "s2c_q", "c_loop", "c_addr", "conn_event")
+)
 
 
 class Manager(object):
@@ -62,10 +62,12 @@ def validate_address(self, addr):
         """
         Validate the address' IP and pid.
         """
-        ip, pid, suffix = addr.split('/')
+        ip, pid, suffix = addr.split("/")
         if ip != self.ip or int(pid) != os.getpid():
-            raise ValueError("inproc address %r does not match host (%r) or pid (%r)"
-                             % (addr, self.ip, os.getpid()))
+            raise ValueError(
+                "inproc address %r does not match host (%r) or pid (%r)"
+                % (addr, self.ip, os.getpid())
+            )
 
 
 global_manager = Manager()
@@ -75,7 +77,7 @@ def new_address():
     """
     Generate a new address.
     """
-    return 'inproc://' + global_manager.new_address()
+    return "inproc://" + global_manager.new_address()
 
 
 class QueueEmpty(Exception):
@@ -144,10 +146,12 @@ class InProc(Comm):
     Reminder: a Comm must always be used from a single thread.
     Its peer Comm can be running in any thread.
     """
+
     _initialized = False
 
-    def __init__(self, local_addr, peer_addr, read_q, write_q, write_loop,
-                 deserialize=True):
+    def __init__(
+        self, local_addr, peer_addr, read_q, write_q, write_loop, deserialize=True
+    ):
         self._local_addr = local_addr
         self._peer_addr = peer_addr
         self.deserialize = deserialize
@@ -161,8 +165,7 @@ def __init__(self, local_addr, peer_addr, read_q, write_q, write_loop,
         self._initialized = True
 
     def _get_finalizer(self):
-        def finalize(write_q=self._write_q, write_loop=self._write_loop,
-                     r=repr(self)):
+        def finalize(write_q=self._write_q, write_loop=self._write_loop, r=repr(self)):
             logger.warning("Closing dangling queue in %s" % (r,))
             write_loop.add_callback(write_q.put_nowait, _EOF)
 
@@ -177,7 +180,7 @@ def peer_address(self):
         return self._peer_addr
 
     @gen.coroutine
-    def read(self, deserializers='ignored'):
+    def read(self, deserializers="ignored"):
         if self._closed:
             raise CommClosedError
 
@@ -233,7 +236,7 @@ def closed(self):
 
 
 class InProcListener(Listener):
-    prefix = 'inproc'
+    prefix = "inproc"
 
     def __init__(self, address, comm_handler, deserialize=True):
         self.manager = global_manager
@@ -248,12 +251,14 @@ def _listen(self):
             conn_req = yield self.listen_q.get()
             if conn_req is None:
                 break
-            comm = InProc(local_addr='inproc://' + self.address,
-                          peer_addr='inproc://' + conn_req.c_addr,
-                          read_q=conn_req.c2s_q,
-                          write_q=conn_req.s2c_q,
-                          write_loop=conn_req.c_loop,
-                          deserialize=self.deserialize)
+            comm = InProc(
+                local_addr="inproc://" + self.address,
+                peer_addr="inproc://" + conn_req.c_addr,
+                read_q=conn_req.c2s_q,
+                write_q=conn_req.s2c_q,
+                write_loop=conn_req.c_loop,
+                deserialize=self.deserialize,
+            )
             # Notify connector
             conn_req.c_loop.add_callback(conn_req.conn_event.set)
             self.comm_handler(comm)
@@ -272,15 +277,14 @@ def stop(self):
 
     @property
     def listen_address(self):
-        return 'inproc://' + self.address
+        return "inproc://" + self.address
 
     @property
     def contact_address(self):
-        return 'inproc://' + self.address
+        return "inproc://" + self.address
 
 
 class InProcConnector(Connector):
-
     def __init__(self, manager):
         self.manager = manager
 
@@ -290,24 +294,27 @@ def connect(self, address, deserialize=True, **connection_args):
         if listener is None:
             raise IOError("no endpoint for inproc address %r" % (address,))
 
-        conn_req = ConnectionRequest(c2s_q=Queue(),
-                                     s2c_q=Queue(),
-                                     c_loop=IOLoop.current(),
-                                     c_addr=self.manager.new_address(),
-                                     conn_event=locks.Event(),
-                                     )
+        conn_req = ConnectionRequest(
+            c2s_q=Queue(),
+            s2c_q=Queue(),
+            c_loop=IOLoop.current(),
+            c_addr=self.manager.new_address(),
+            conn_event=locks.Event(),
+        )
         listener.connect_threadsafe(conn_req)
         # Wait for connection acknowledgement
         # (do not pretend we're connected if the other comm never gets
         #  created, for example if the listener was stopped in the meantime)
         yield conn_req.conn_event.wait()
 
-        comm = InProc(local_addr='inproc://' + conn_req.c_addr,
-                      peer_addr='inproc://' + address,
-                      read_q=conn_req.s2c_q,
-                      write_q=conn_req.c2s_q,
-                      write_loop=listener.loop,
-                      deserialize=deserialize)
+        comm = InProc(
+            local_addr="inproc://" + conn_req.c_addr,
+            peer_addr="inproc://" + address,
+            read_q=conn_req.s2c_q,
+            write_q=conn_req.c2s_q,
+            write_loop=listener.loop,
+            deserialize=deserialize,
+        )
         raise gen.Return(comm)
 
 
@@ -336,4 +343,4 @@ def get_local_address_for(self, loc):
         return self.manager.new_address()
 
 
-backends['inproc'] = InProcBackend()
+backends["inproc"] = InProcBackend()
diff --git a/distributed/comm/registry.py b/distributed/comm/registry.py
index a47b0f7435d..a646b4d71b9 100644
--- a/distributed/comm/registry.py
+++ b/distributed/comm/registry.py
@@ -67,6 +67,7 @@ def get_backend(scheme):
     """
     backend = backends.get(scheme)
     if backend is None:
-        raise ValueError("unknown address scheme %r (known schemes: %s)"
-                         % (scheme, sorted(backends)))
+        raise ValueError(
+            "unknown address scheme %r (known schemes: %s)" % (scheme, sorted(backends))
+        )
     return backend
diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py
index f10af0bc167..6d90a7bc9c7 100644
--- a/distributed/comm/tcp.py
+++ b/distributed/comm/tcp.py
@@ -20,14 +20,20 @@
 
 from ..compatibility import finalize, PY3
 from ..threadpoolexecutor import ThreadPoolExecutor
-from ..utils import (ensure_bytes, ensure_ip, get_ip, get_ipv6, nbytes,
-                     parse_timedelta, shutting_down)
+from ..utils import (
+    ensure_bytes,
+    ensure_ip,
+    get_ip,
+    get_ipv6,
+    nbytes,
+    parse_timedelta,
+    shutting_down,
+)
 
 from .registry import Backend, backends
 from .addressing import parse_host_port, unparse_host_port
 from .core import Comm, Connector, Listener, CommClosedError, FatalCommClosedError
-from .utils import (to_frames, from_frames,
-                    get_tcp_server_address, ensure_concrete_host,)
+from .utils import to_frames, from_frames, get_tcp_server_address, ensure_concrete_host
 
 
 logger = logging.getLogger(__name__)
@@ -36,6 +42,7 @@
 def get_total_physical_memory():
     try:
         import psutil
+
         return psutil.virtual_memory().total / 2
     except ImportError:
         return 2e9
@@ -51,8 +58,8 @@ def set_tcp_timeout(stream):
     if stream.closed():
         return
 
-    timeout = dask.config.get('distributed.comm.timeouts.tcp')
-    timeout = int(parse_timedelta(timeout, default='seconds'))
+    timeout = dask.config.get("distributed.comm.timeouts.tcp")
+    timeout = int(parse_timedelta(timeout, default="seconds"))
 
     sock = stream.socket
 
@@ -68,8 +75,7 @@ def set_tcp_timeout(stream):
 
     try:
         if sys.platform.startswith("win"):
-            logger.debug("Setting TCP keepalive: idle=%d, interval=%d",
-                         idle, interval)
+            logger.debug("Setting TCP keepalive: idle=%d, interval=%d", idle, interval)
             sock.ioctl(socket.SIO_KEEPALIVE_VALS, (1, idle * 1000, interval * 1000))
         else:
             sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
@@ -86,15 +92,18 @@ def set_tcp_timeout(stream):
                     TCP_KEEPIDLE = None
 
             if TCP_KEEPIDLE is not None:
-                logger.debug("Setting TCP keepalive: nprobes=%d, idle=%d, interval=%d",
-                             nprobes, idle, interval)
+                logger.debug(
+                    "Setting TCP keepalive: nprobes=%d, idle=%d, interval=%d",
+                    nprobes,
+                    idle,
+                    interval,
+                )
                 sock.setsockopt(socket.SOL_TCP, TCP_KEEPCNT, nprobes)
                 sock.setsockopt(socket.SOL_TCP, TCP_KEEPIDLE, idle)
                 sock.setsockopt(socket.SOL_TCP, TCP_KEEPINTVL, interval)
 
         if sys.platform.startswith("linux"):
-            logger.debug("Setting TCP user timeout: %d ms",
-                         timeout * 1000)
+            logger.debug("Setting TCP user timeout: %d ms", timeout * 1000)
             TCP_USER_TIMEOUT = 18  # since Linux 2.6.37
             sock.setsockopt(socket.SOL_TCP, TCP_USER_TIMEOUT, timeout * 1000)
     except EnvironmentError as e:
@@ -123,8 +132,10 @@ def convert_stream_closed_error(obj, exc):
         # The stream was closed because of an underlying OS error
         exc = exc.real_error
         if ssl and isinstance(exc, ssl.SSLError):
-            if 'UNKNOWN_CA' in exc.reason:
-                raise FatalCommClosedError("in %s: %s: %s" % (obj, exc.__class__.__name__, exc))
+            if "UNKNOWN_CA" in exc.reason:
+                raise FatalCommClosedError(
+                    "in %s: %s: %s" % (obj, exc.__class__.__name__, exc)
+                )
         raise CommClosedError("in %s: %s: %s" % (obj, exc.__class__.__name__, exc))
     else:
         raise CommClosedError("in %s: %s" % (obj, exc))
@@ -134,6 +145,7 @@ class TCP(Comm):
     """
     An established communication based on an underlying Tornado IOStream.
     """
+
     _iostream_allows_memoryview = tornado.version_info >= (4, 5)
     # IOStream.read_into() currently proposed in
     # https://github.com/tornadoweb/tornado/pull/2193
@@ -179,9 +191,9 @@ def read(self, deserializers=None):
 
         try:
             n_frames = yield stream.read_bytes(8)
-            n_frames = struct.unpack('Q', n_frames)[0]
+            n_frames = struct.unpack("Q", n_frames)[0]
             lengths = yield stream.read_bytes(8 * n_frames)
-            lengths = struct.unpack('Q' * n_frames, lengths)
+            lengths = struct.unpack("Q" * n_frames, lengths)
 
             frames = []
             for length in lengths:
@@ -193,7 +205,7 @@ def read(self, deserializers=None):
                     else:
                         frame = yield stream.read_bytes(length)
                 else:
-                    frame = b''
+                    frame = b""
                 frames.append(frame)
         except StreamClosedError as e:
             self.stream = None
@@ -201,9 +213,9 @@ def read(self, deserializers=None):
                 convert_stream_closed_error(self, e)
         else:
             try:
-                msg = yield from_frames(frames,
-                                        deserialize=self.deserialize,
-                                        deserializers=deserializers)
+                msg = yield from_frames(
+                    frames, deserialize=self.deserialize, deserializers=deserializers
+                )
             except EOFError:
                 # Frames possibly garbled or truncated by communication error
                 self.abort()
@@ -211,27 +223,29 @@ def read(self, deserializers=None):
             raise gen.Return(msg)
 
     @gen.coroutine
-    def write(self, msg, serializers=None, on_error='message'):
+    def write(self, msg, serializers=None, on_error="message"):
         stream = self.stream
         bytes_since_last_yield = 0
         if stream is None:
             raise CommClosedError
 
-        frames = yield to_frames(msg,
-                                 serializers=serializers,
-                                 on_error=on_error,
-                                 context={'sender': self._local_addr,
-                                          'recipient': self._peer_addr})
+        frames = yield to_frames(
+            msg,
+            serializers=serializers,
+            on_error=on_error,
+            context={"sender": self._local_addr, "recipient": self._peer_addr},
+        )
 
         try:
             lengths = [nbytes(frame) for frame in frames]
-            length_bytes = ([struct.pack('Q', len(frames))] +
-                            [struct.pack('Q', x) for x in lengths])
-            if PY3 and sum(lengths) < 2**17:  # 128kiB
-                b = b''.join(length_bytes + frames)  # small enough, send in one go
+            length_bytes = [struct.pack("Q", len(frames))] + [
+                struct.pack("Q", x) for x in lengths
+            ]
+            if PY3 and sum(lengths) < 2 ** 17:  # 128kiB
+                b = b"".join(length_bytes + frames)  # small enough, send in one go
                 stream.write(b)
             else:
-                stream.write(b''.join(length_bytes))  # avoid large memcpy, send in many
+                stream.write(b"".join(length_bytes))  # avoid large memcpy, send in many
 
                 for frame in frames:
                     # Can't wait for the write() Future as it may be lost
@@ -262,7 +276,7 @@ def close(self):
             try:
                 # Flush the stream's write buffer by waiting for a last write.
                 if stream.writing():
-                    yield stream.write(b'')
+                    yield stream.write(b"")
                 stream.socket.shutdown(socket.SHUT_RDWR)
             except EnvironmentError:
                 pass
@@ -293,37 +307,42 @@ def _read_extra(self):
         TCP._read_extra(self)
         sock = self.stream.socket
         if sock is not None:
-            self._extra.update(peercert=sock.getpeercert(),
-                               cipher=sock.cipher())
-            cipher, proto, bits = self._extra['cipher']
-            logger.debug("TLS connection with %r: protocol=%s, cipher=%s, bits=%d",
-                         self._peer_addr, proto, cipher, bits)
+            self._extra.update(peercert=sock.getpeercert(), cipher=sock.cipher())
+            cipher, proto, bits = self._extra["cipher"]
+            logger.debug(
+                "TLS connection with %r: protocol=%s, cipher=%s, bits=%d",
+                self._peer_addr,
+                proto,
+                cipher,
+                bits,
+            )
 
 
 def _expect_tls_context(connection_args):
-    ctx = connection_args.get('ssl_context')
+    ctx = connection_args.get("ssl_context")
     if not isinstance(ctx, ssl.SSLContext):
-        raise TypeError("TLS expects a `ssl_context` argument of type "
-                        "ssl.SSLContext (perhaps check your TLS configuration?)"
-                        "  Instead got %s" % str(ctx))
+        raise TypeError(
+            "TLS expects a `ssl_context` argument of type "
+            "ssl.SSLContext (perhaps check your TLS configuration?)"
+            "  Instead got %s" % str(ctx)
+        )
     return ctx
 
 
 class RequireEncryptionMixin(object):
-
     def _check_encryption(self, address, connection_args):
-        if not self.encrypted and connection_args.get('require_encryption'):
+        if not self.encrypted and connection_args.get("require_encryption"):
             # XXX Should we have a dedicated SecurityError class?
-            raise RuntimeError("encryption required by Dask configuration, "
-                               "refusing communication from/to %r"
-                               % (self.prefix + address,))
+            raise RuntimeError(
+                "encryption required by Dask configuration, "
+                "refusing communication from/to %r" % (self.prefix + address,)
+            )
 
 
 class BaseTCPConnector(Connector, RequireEncryptionMixin):
     if PY3:  # see github PR #2403 discussion for more info
         _executor = ThreadPoolExecutor(2, thread_name_prefix="TCP-Executor")
-        _resolver = netutil.ExecutorResolver(close_executor=False,
-                                             executor=_executor)
+        _resolver = netutil.ExecutorResolver(close_executor=False, executor=_executor)
     else:
         _resolver = None
     client = TCPClient(resolver=_resolver)
@@ -335,9 +354,9 @@ def connect(self, address, deserialize=True, **connection_args):
         kwargs = self._get_connect_args(**connection_args)
 
         try:
-            stream = yield BaseTCPConnector.client.connect(ip, port,
-                                          max_buffer_size=MAX_BUFFER_SIZE,
-                                          **kwargs)
+            stream = yield BaseTCPConnector.client.connect(
+                ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs
+            )
 
             # Under certain circumstances tornado will have a closed connnection with an error and not raise
             # a StreamClosedError.
@@ -351,14 +370,13 @@ def connect(self, address, deserialize=True, **connection_args):
             convert_stream_closed_error(self, e)
 
         local_address = self.prefix + get_stream_address(stream)
-        raise gen.Return(self.comm_class(stream,
-                                         local_address,
-                                         self.prefix + address,
-                                         deserialize))
+        raise gen.Return(
+            self.comm_class(stream, local_address, self.prefix + address, deserialize)
+        )
 
 
 class TCPConnector(BaseTCPConnector):
-    prefix = 'tcp://'
+    prefix = "tcp://"
     comm_class = TCP
     encrypted = False
 
@@ -367,19 +385,19 @@ def _get_connect_args(self, **connection_args):
 
 
 class TLSConnector(BaseTCPConnector):
-    prefix = 'tls://'
+    prefix = "tls://"
     comm_class = TLS
     encrypted = True
 
     def _get_connect_args(self, **connection_args):
         ctx = _expect_tls_context(connection_args)
-        return {'ssl_options': ctx}
+        return {"ssl_options": ctx}
 
 
 class BaseTCPListener(Listener, RequireEncryptionMixin):
-
-    def __init__(self, address, comm_handler, deserialize=True,
-                 default_port=0, **connection_args):
+    def __init__(
+        self, address, comm_handler, deserialize=True, default_port=0, **connection_args
+    ):
         self._check_encryption(address, connection_args)
         self.ip, self.port = parse_host_port(address, default_port)
         self.comm_handler = comm_handler
@@ -389,18 +407,18 @@ def __init__(self, address, comm_handler, deserialize=True,
         self.bound_address = None
 
     def start(self):
-        self.tcp_server = TCPServer(max_buffer_size=MAX_BUFFER_SIZE,
-                                    **self.server_args)
+        self.tcp_server = TCPServer(max_buffer_size=MAX_BUFFER_SIZE, **self.server_args)
         self.tcp_server.handle_stream = self._handle_stream
-        backlog = int(dask.config.get('distributed.comm.socket-backlog'))
+        backlog = int(dask.config.get("distributed.comm.socket-backlog"))
         for i in range(5):
             try:
                 # When shuffling data between workers, there can
                 # really be O(cluster size) connection requests
                 # on a single worker socket, make sure the backlog
                 # is large enough not to lose any.
-                sockets = netutil.bind_sockets(self.port, address=self.ip,
-                                               backlog=backlog)
+                sockets = netutil.bind_sockets(
+                    self.port, address=self.ip, backlog=backlog
+                )
             except EnvironmentError as e:
                 # EADDRINUSE can happen sporadically when trying to bind
                 # to an ephemeral port
@@ -429,8 +447,7 @@ def _handle_stream(self, stream, address):
         if stream is None:
             # Preparation failed
             return
-        logger.debug("Incoming connection from %r to %r",
-                     address, self.contact_address)
+        logger.debug("Incoming connection from %r to %r", address, self.contact_address)
         local_address = self.prefix + get_stream_address(stream)
         comm = self.comm_class(stream, local_address, address, self.deserialize)
         yield self.comm_handler(comm)
@@ -464,7 +481,7 @@ def contact_address(self):
 
 
 class TCPListener(BaseTCPListener):
-    prefix = 'tcp://'
+    prefix = "tcp://"
     comm_class = TCP
     encrypted = False
 
@@ -477,13 +494,13 @@ def _prepare_stream(self, stream, address):
 
 
 class TLSListener(BaseTCPListener):
-    prefix = 'tls://'
+    prefix = "tls://"
     comm_class = TLS
     encrypted = True
 
     def _get_server_args(self, **connection_args):
         ctx = _expect_tls_context(connection_args)
-        return {'ssl_options': ctx}
+        return {"ssl_options": ctx}
 
     @gen.coroutine
     def _prepare_stream(self, stream, address):
@@ -491,9 +508,12 @@ def _prepare_stream(self, stream, address):
             yield stream.wait_for_handshake()
         except EnvironmentError as e:
             # The handshake went wrong, log and ignore
-            logger.warning("Listener on %r: TLS handshake failed with remote %r: %s",
-                           self.listen_address, address,
-                           getattr(e, "real_error", None) or e)
+            logger.warning(
+                "Listener on %r: TLS handshake failed with remote %r: %s",
+                self.listen_address,
+                address,
+                getattr(e, "real_error", None) or e,
+            )
         else:
             raise gen.Return(stream)
 
@@ -523,7 +543,7 @@ def resolve_address(self, loc):
     def get_local_address_for(self, loc):
         host, port = parse_host_port(loc)
         host = ensure_ip(host)
-        if ':' in host:
+        if ":" in host:
             local_host = get_ipv6(host)
         else:
             local_host = get_ip(host)
@@ -540,5 +560,5 @@ class TLSBackend(BaseTCPBackend):
     _listener_class = TLSListener
 
 
-backends['tcp'] = TCPBackend()
-backends['tls'] = TLSBackend()
+backends["tcp"] = TCPBackend()
+backends["tls"] = TLSBackend()
diff --git a/distributed/comm/tests/test_comms.py b/distributed/comm/tests/test_comms.py
index e4aee5805db..0e8782718a0 100644
--- a/distributed/comm/tests/test_comms.py
+++ b/distributed/comm/tests/test_comms.py
@@ -14,82 +14,96 @@
 from distributed.compatibility import PY3
 from distributed.metrics import time
 from distributed.utils import get_ip, get_ipv6
-from distributed.utils_test import (gen_test, requires_ipv6, has_ipv6,
-                                    get_cert, get_server_ssl_context,
-                                    get_client_ssl_context)
+from distributed.utils_test import (
+    gen_test,
+    requires_ipv6,
+    has_ipv6,
+    get_cert,
+    get_server_ssl_context,
+    get_client_ssl_context,
+)
 from distributed.utils_test import loop  # noqa: F401
 
-from distributed.protocol import (to_serialize, Serialized, serialize,
-                                  deserialize)
-
-from distributed.comm import (tcp, inproc, connect, listen, CommClosedError,
-                              parse_address, parse_host_port,
-                              unparse_host_port, resolve_address,
-                              get_address_host, get_local_address_for)
+from distributed.protocol import to_serialize, Serialized, serialize, deserialize
+
+from distributed.comm import (
+    tcp,
+    inproc,
+    connect,
+    listen,
+    CommClosedError,
+    parse_address,
+    parse_host_port,
+    unparse_host_port,
+    resolve_address,
+    get_address_host,
+    get_local_address_for,
+)
 
 
 EXTERNAL_IP4 = get_ip()
 if has_ipv6():
     with warnings.catch_warnings(record=True):
-        warnings.simplefilter('always')
+        warnings.simplefilter("always")
         EXTERNAL_IP6 = get_ipv6()
 
 
-ca_file = get_cert('tls-ca-cert.pem')
+ca_file = get_cert("tls-ca-cert.pem")
 
 # The Subject field of our test certs
 cert_subject = (
-    (('countryName', 'XY'),),
-    (('localityName', 'Dask-distributed'),),
-    (('organizationName', 'Dask'),),
-    (('commonName', 'localhost'),)
+    (("countryName", "XY"),),
+    (("localityName", "Dask-distributed"),),
+    (("organizationName", "Dask"),),
+    (("commonName", "localhost"),),
 )
 
 
 def check_tls_extra(info):
     assert isinstance(info, dict)
-    assert info['peercert']['subject'] == cert_subject
-    assert 'cipher' in info
-    cipher_name, proto_name, secret_bits = info['cipher']
+    assert info["peercert"]["subject"] == cert_subject
+    assert "cipher" in info
+    cipher_name, proto_name, secret_bits = info["cipher"]
     # Most likely
-    assert 'AES' in cipher_name
-    assert 'TLS' in proto_name
+    assert "AES" in cipher_name
+    assert "TLS" in proto_name
     assert secret_bits >= 128
 
 
-tls_kwargs = dict(listen_args={'ssl_context': get_server_ssl_context()},
-                  connect_args={'ssl_context': get_client_ssl_context()})
+tls_kwargs = dict(
+    listen_args={"ssl_context": get_server_ssl_context()},
+    connect_args={"ssl_context": get_client_ssl_context()},
+)
 
 
 @gen.coroutine
-def get_comm_pair(listen_addr, listen_args=None, connect_args=None,
-                  **kwargs):
+def get_comm_pair(listen_addr, listen_args=None, connect_args=None, **kwargs):
     q = queues.Queue()
 
     def handle_comm(comm):
         q.put(comm)
 
-    listener = listen(listen_addr, handle_comm,
-                      connection_args=listen_args, **kwargs)
+    listener = listen(listen_addr, handle_comm, connection_args=listen_args, **kwargs)
     listener.start()
 
-    comm = yield connect(listener.contact_address,
-                         connection_args=connect_args, **kwargs)
+    comm = yield connect(
+        listener.contact_address, connection_args=connect_args, **kwargs
+    )
     serv_comm = yield q.get()
     raise gen.Return((comm, serv_comm))
 
 
 def get_tcp_comm_pair(**kwargs):
-    return get_comm_pair('tcp://', **kwargs)
+    return get_comm_pair("tcp://", **kwargs)
 
 
 def get_tls_comm_pair(**kwargs):
     kwargs.update(tls_kwargs)
-    return get_comm_pair('tls://', **kwargs)
+    return get_comm_pair("tls://", **kwargs)
 
 
 def get_inproc_comm_pair(**kwargs):
-    return get_comm_pair('inproc://', **kwargs)
+    return get_comm_pair("inproc://", **kwargs)
 
 
 @gen.coroutine
@@ -99,7 +113,7 @@ def debug_loop():
     """
     while True:
         loop = ioloop.IOLoop.current()
-        print('.', loop, loop._handlers)
+        print(".", loop, loop._handlers)
         yield gen.sleep(0.50)
 
 
@@ -107,86 +121,86 @@ def debug_loop():
 # Test utility functions
 #
 
+
 def test_parse_host_port():
     f = parse_host_port
 
-    assert f('localhost:123') == ('localhost', 123)
-    assert f('127.0.0.1:456') == ('127.0.0.1', 456)
-    assert f('localhost:123', 80) == ('localhost', 123)
-    assert f('localhost', 80) == ('localhost', 80)
+    assert f("localhost:123") == ("localhost", 123)
+    assert f("127.0.0.1:456") == ("127.0.0.1", 456)
+    assert f("localhost:123", 80) == ("localhost", 123)
+    assert f("localhost", 80) == ("localhost", 80)
 
     with pytest.raises(ValueError):
-        f('localhost')
+        f("localhost")
 
-    assert f('[::1]:123') == ('::1', 123)
-    assert f('[fe80::1]:123', 80) == ('fe80::1', 123)
-    assert f('[::1]', 80) == ('::1', 80)
+    assert f("[::1]:123") == ("::1", 123)
+    assert f("[fe80::1]:123", 80) == ("fe80::1", 123)
+    assert f("[::1]", 80) == ("::1", 80)
 
     with pytest.raises(ValueError):
-        f('[::1]')
+        f("[::1]")
     with pytest.raises(ValueError):
-        f('::1:123')
+        f("::1:123")
     with pytest.raises(ValueError):
-        f('::1')
+        f("::1")
 
 
 def test_unparse_host_port():
     f = unparse_host_port
 
-    assert f('localhost', 123) == 'localhost:123'
-    assert f('127.0.0.1', 123) == '127.0.0.1:123'
-    assert f('::1', 123) == '[::1]:123'
-    assert f('[::1]', 123) == '[::1]:123'
+    assert f("localhost", 123) == "localhost:123"
+    assert f("127.0.0.1", 123) == "127.0.0.1:123"
+    assert f("::1", 123) == "[::1]:123"
+    assert f("[::1]", 123) == "[::1]:123"
 
-    assert f('127.0.0.1') == '127.0.0.1'
-    assert f('127.0.0.1', 0) == '127.0.0.1'
-    assert f('127.0.0.1', None) == '127.0.0.1'
-    assert f('127.0.0.1', '*') == '127.0.0.1:*'
+    assert f("127.0.0.1") == "127.0.0.1"
+    assert f("127.0.0.1", 0) == "127.0.0.1"
+    assert f("127.0.0.1", None) == "127.0.0.1"
+    assert f("127.0.0.1", "*") == "127.0.0.1:*"
 
-    assert f('::1') == '[::1]'
-    assert f('[::1]') == '[::1]'
-    assert f('::1', '*') == '[::1]:*'
+    assert f("::1") == "[::1]"
+    assert f("[::1]") == "[::1]"
+    assert f("::1", "*") == "[::1]:*"
 
 
 def test_get_address_host():
     f = get_address_host
 
-    assert f('tcp://127.0.0.1:123') == '127.0.0.1'
-    assert f('inproc://%s/%d/123' % (get_ip(), os.getpid())) == get_ip()
+    assert f("tcp://127.0.0.1:123") == "127.0.0.1"
+    assert f("inproc://%s/%d/123" % (get_ip(), os.getpid())) == get_ip()
 
 
 def test_resolve_address():
     f = resolve_address
 
-    assert f('tcp://127.0.0.1:123') == 'tcp://127.0.0.1:123'
-    assert f('127.0.0.2:789') == 'tcp://127.0.0.2:789'
-    assert f('tcp://0.0.0.0:456') == 'tcp://0.0.0.0:456'
-    assert f('tcp://0.0.0.0:456') == 'tcp://0.0.0.0:456'
+    assert f("tcp://127.0.0.1:123") == "tcp://127.0.0.1:123"
+    assert f("127.0.0.2:789") == "tcp://127.0.0.2:789"
+    assert f("tcp://0.0.0.0:456") == "tcp://0.0.0.0:456"
+    assert f("tcp://0.0.0.0:456") == "tcp://0.0.0.0:456"
 
     if has_ipv6():
-        assert f('tcp://[::1]:123') == 'tcp://[::1]:123'
-        assert f('tls://[::1]:123') == 'tls://[::1]:123'
+        assert f("tcp://[::1]:123") == "tcp://[::1]:123"
+        assert f("tls://[::1]:123") == "tls://[::1]:123"
         # OS X returns '::0.0.0.2' as canonical representation
-        assert f('[::2]:789') in ('tcp://[::2]:789',
-                                  'tcp://[::0.0.0.2]:789')
-        assert f('tcp://[::]:123') == 'tcp://[::]:123'
+        assert f("[::2]:789") in ("tcp://[::2]:789", "tcp://[::0.0.0.2]:789")
+        assert f("tcp://[::]:123") == "tcp://[::]:123"
 
-    assert f('localhost:123') == 'tcp://127.0.0.1:123'
-    assert f('tcp://localhost:456') == 'tcp://127.0.0.1:456'
-    assert f('tls://localhost:456') == 'tls://127.0.0.1:456'
+    assert f("localhost:123") == "tcp://127.0.0.1:123"
+    assert f("tcp://localhost:456") == "tcp://127.0.0.1:456"
+    assert f("tls://localhost:456") == "tls://127.0.0.1:456"
 
 
 def test_get_local_address_for():
     f = get_local_address_for
 
-    assert f('tcp://127.0.0.1:80') == 'tcp://127.0.0.1'
-    assert f('tcp://8.8.8.8:4444') == 'tcp://' + get_ip()
+    assert f("tcp://127.0.0.1:80") == "tcp://127.0.0.1"
+    assert f("tcp://8.8.8.8:4444") == "tcp://" + get_ip()
     if has_ipv6():
-        assert f('tcp://[::1]:123') == 'tcp://[::1]'
+        assert f("tcp://[::1]:123") == "tcp://[::1]"
 
-    inproc_arg = 'inproc://%s/%d/444' % (get_ip(), os.getpid())
+    inproc_arg = "inproc://%s/%d/444" % (get_ip(), os.getpid())
     inproc_res = f(inproc_arg)
-    assert inproc_res.startswith('inproc://')
+    assert inproc_res.startswith("inproc://")
     assert inproc_res != inproc_arg
 
 
@@ -194,24 +208,26 @@ def test_get_local_address_for():
 # Test concrete transport APIs
 #
 
+
 @gen_test()
 def test_tcp_specific():
     """
     Test concrete TCP API.
     """
+
     @gen.coroutine
     def handle_comm(comm):
-        assert comm.peer_address.startswith('tcp://' + host)
+        assert comm.peer_address.startswith("tcp://" + host)
         assert comm.extra_info == {}
         msg = yield comm.read()
-        msg['op'] = 'pong'
+        msg["op"] = "pong"
         yield comm.write(msg)
         yield comm.close()
 
-    listener = tcp.TCPListener('localhost', handle_comm)
+    listener = tcp.TCPListener("localhost", handle_comm)
     listener.start()
     host, port = listener.get_host_port()
-    assert host in ('localhost', '127.0.0.1', '::1')
+    assert host in ("localhost", "127.0.0.1", "::1")
     assert port > 0
 
     connector = tcp.TCPConnector()
@@ -219,15 +235,15 @@ def handle_comm(comm):
 
     @gen.coroutine
     def client_communicate(key, delay=0):
-        addr = '%s:%d' % (host, port)
+        addr = "%s:%d" % (host, port)
         comm = yield connector.connect(addr)
-        assert comm.peer_address == 'tcp://' + addr
+        assert comm.peer_address == "tcp://" + addr
         assert comm.extra_info == {}
-        yield comm.write({'op': 'ping', 'data': key})
+        yield comm.write({"op": "ping", "data": key})
         if delay:
             yield gen.sleep(delay)
         msg = yield comm.read()
-        assert msg == {'op': 'pong', 'data': key}
+        assert msg == {"op": "pong", "data": key}
         l.append(key)
         yield comm.close()
 
@@ -245,23 +261,23 @@ def test_tls_specific():
     """
     Test concrete TLS API.
     """
+
     @gen.coroutine
     def handle_comm(comm):
-        assert comm.peer_address.startswith('tls://' + host)
+        assert comm.peer_address.startswith("tls://" + host)
         check_tls_extra(comm.extra_info)
         msg = yield comm.read()
-        msg['op'] = 'pong'
+        msg["op"] = "pong"
         yield comm.write(msg)
         yield comm.close()
 
     server_ctx = get_server_ssl_context()
     client_ctx = get_client_ssl_context()
 
-    listener = tcp.TLSListener('localhost', handle_comm,
-                               ssl_context=server_ctx)
+    listener = tcp.TLSListener("localhost", handle_comm, ssl_context=server_ctx)
     listener.start()
     host, port = listener.get_host_port()
-    assert host in ('localhost', '127.0.0.1', '::1')
+    assert host in ("localhost", "127.0.0.1", "::1")
     assert port > 0
 
     connector = tcp.TLSConnector()
@@ -269,15 +285,15 @@ def handle_comm(comm):
 
     @gen.coroutine
     def client_communicate(key, delay=0):
-        addr = '%s:%d' % (host, port)
+        addr = "%s:%d" % (host, port)
         comm = yield connector.connect(addr, ssl_context=client_ctx)
-        assert comm.peer_address == 'tls://' + addr
+        assert comm.peer_address == "tls://" + addr
         check_tls_extra(comm.extra_info)
-        yield comm.write({'op': 'ping', 'data': key})
+        yield comm.write({"op": "ping", "data": key})
         if delay:
             yield gen.sleep(delay)
         msg = yield comm.read()
-        assert msg == {'op': 'pong', 'data': key}
+        assert msg == {"op": "pong", "data": key}
         l.append(key)
         yield comm.close()
 
@@ -309,6 +325,7 @@ def sleep_for_60ms():
             if thread_count > max_thread_count:
                 max_thread_count = thread_count
         raise gen.Return(max_thread_count)
+
     original_thread_count = threading.active_count()
 
     # tcp.TCPConnector()
@@ -323,8 +340,11 @@ def sleep_for_60ms():
     # tcp.TLSConnector()
     sleep_future = sleep_for_60ms()
     with pytest.raises(IOError):
-        yield connect("tls://localhost:28400", 0.052,
-                                 connection_args={'ssl_context': get_client_ssl_context()})
+        yield connect(
+            "tls://localhost:28400",
+            0.052,
+            connection_args={"ssl_context": get_client_ssl_context()},
+        )
     max_thread_count = yield sleep_future
     if PY3:
         assert max_thread_count <= 2 + original_thread_count
@@ -336,7 +356,7 @@ def check_inproc_specific(run_client):
     Test concrete InProc API.
     """
     listener_addr = inproc.global_manager.new_address()
-    addr_head = listener_addr.rpartition('/')[0]
+    addr_head = listener_addr.rpartition("/")[0]
 
     client_addresses = set()
 
@@ -344,17 +364,21 @@ def check_inproc_specific(run_client):
 
     @gen.coroutine
     def handle_comm(comm):
-        assert comm.peer_address.startswith('inproc://' + addr_head)
+        assert comm.peer_address.startswith("inproc://" + addr_head)
         client_addresses.add(comm.peer_address)
         for i in range(N_MSGS):
             msg = yield comm.read()
-            msg['op'] = 'pong'
+            msg["op"] = "pong"
             yield comm.write(msg)
         yield comm.close()
 
     listener = inproc.InProcListener(listener_addr, handle_comm)
     listener.start()
-    assert listener.listen_address == listener.contact_address == 'inproc://' + listener_addr
+    assert (
+        listener.listen_address
+        == listener.contact_address
+        == "inproc://" + listener_addr
+    )
 
     connector = inproc.InProcConnector(inproc.global_manager)
     l = []
@@ -362,13 +386,13 @@ def handle_comm(comm):
     @gen.coroutine
     def client_communicate(key, delay=0):
         comm = yield connector.connect(listener_addr)
-        assert comm.peer_address == 'inproc://' + listener_addr
+        assert comm.peer_address == "inproc://" + listener_addr
         for i in range(N_MSGS):
-            yield comm.write({'op': 'ping', 'data': key})
+            yield comm.write({"op": "ping", "data": key})
             if delay:
                 yield gen.sleep(delay)
             msg = yield comm.read()
-        assert msg == {'op': 'pong', 'data': key}
+        assert msg == {"op": "pong", "data": key}
         l.append(key)
         with pytest.raises(CommClosedError):
             yield comm.read()
@@ -399,8 +423,7 @@ def run_coro_in_thread(func, *args, **kwargs):
     def run():
         thread_loop = ioloop.IOLoop()  # need fresh IO loop for run_sync()
         try:
-            res = thread_loop.run_sync(partial(func, *args, **kwargs),
-                                       timeout=10)
+            res = thread_loop.run_sync(partial(func, *args, **kwargs), timeout=10)
         except Exception:
             main_loop.add_callback(fut.set_exc_info, sys.exc_info())
         else:
@@ -427,30 +450,37 @@ def test_inproc_specific_different_threads():
 # Test communications through the abstract API
 #
 
+
 @gen.coroutine
-def check_client_server(addr, check_listen_addr=None, check_contact_addr=None,
-                        listen_args=None, connect_args=None):
+def check_client_server(
+    addr,
+    check_listen_addr=None,
+    check_contact_addr=None,
+    listen_args=None,
+    connect_args=None,
+):
     """
     Abstract client / server test.
     """
+
     @gen.coroutine
     def handle_comm(comm):
         scheme, loc = parse_address(comm.peer_address)
         assert scheme == bound_scheme
 
         msg = yield comm.read()
-        assert msg['op'] == 'ping'
-        msg['op'] = 'pong'
+        assert msg["op"] == "ping"
+        msg["op"] = "pong"
         yield comm.write(msg)
 
         msg = yield comm.read()
-        assert msg['op'] == 'foobar'
+        assert msg["op"] == "foobar"
 
         yield comm.close()
 
     # Arbitrary connection args should be ignored
-    listen_args = listen_args or {'xxx': 'bar'}
-    connect_args = connect_args or {'xxx': 'foo'}
+    listen_args = listen_args or {"xxx": "bar"}
+    connect_args = connect_args or {"xxx": "foo"}
 
     listener = listen(addr, handle_comm, connection_args=listen_args)
     listener.start()
@@ -458,7 +488,7 @@ def handle_comm(comm):
     # Check listener properties
     bound_addr = listener.listen_address
     bound_scheme, bound_loc = parse_address(bound_addr)
-    assert bound_scheme in ('inproc', 'tcp', 'tls')
+    assert bound_scheme in ("inproc", "tcp", "tls")
     assert bound_scheme == parse_address(addr)[0]
 
     if check_listen_addr is not None:
@@ -478,16 +508,15 @@ def handle_comm(comm):
 
     @gen.coroutine
     def client_communicate(key, delay=0):
-        comm = yield connect(listener.contact_address,
-                             connection_args=connect_args)
+        comm = yield connect(listener.contact_address, connection_args=connect_args)
         assert comm.peer_address == listener.contact_address
 
-        yield comm.write({'op': 'ping', 'data': key})
-        yield comm.write({'op': 'foobar'})
+        yield comm.write({"op": "ping", "data": key})
+        yield comm.write({"op": "foobar"})
         if delay:
             yield gen.sleep(delay)
         msg = yield comm.read()
-        assert msg == {'op': 'pong', 'data': key}
+        assert msg == {"op": "pong", "data": key}
         l.append(key)
         yield comm.close()
 
@@ -521,7 +550,7 @@ def inproc_check():
     expected_pid = os.getpid()
 
     def checker(loc):
-        ip, pid, suffix = loc.split('/')
+        ip, pid, suffix = loc.split("/")
         assert ip == expected_ip
         assert int(pid) == expected_pid
 
@@ -531,70 +560,75 @@ def checker(loc):
 @gen_test()
 def test_default_client_server_ipv4():
     # Default scheme is (currently) TCP
-    yield check_client_server('127.0.0.1', tcp_eq('127.0.0.1'))
-    yield check_client_server('127.0.0.1:3201', tcp_eq('127.0.0.1', 3201))
-    yield check_client_server('0.0.0.0',
-                              tcp_eq('0.0.0.0'), tcp_eq(EXTERNAL_IP4))
-    yield check_client_server('0.0.0.0:3202',
-                              tcp_eq('0.0.0.0', 3202), tcp_eq(EXTERNAL_IP4, 3202))
+    yield check_client_server("127.0.0.1", tcp_eq("127.0.0.1"))
+    yield check_client_server("127.0.0.1:3201", tcp_eq("127.0.0.1", 3201))
+    yield check_client_server("0.0.0.0", tcp_eq("0.0.0.0"), tcp_eq(EXTERNAL_IP4))
+    yield check_client_server(
+        "0.0.0.0:3202", tcp_eq("0.0.0.0", 3202), tcp_eq(EXTERNAL_IP4, 3202)
+    )
     # IPv4 is preferred for the bound address
-    yield check_client_server('',
-                              tcp_eq('0.0.0.0'), tcp_eq(EXTERNAL_IP4))
-    yield check_client_server(':3203',
-                              tcp_eq('0.0.0.0', 3203), tcp_eq(EXTERNAL_IP4, 3203))
+    yield check_client_server("", tcp_eq("0.0.0.0"), tcp_eq(EXTERNAL_IP4))
+    yield check_client_server(
+        ":3203", tcp_eq("0.0.0.0", 3203), tcp_eq(EXTERNAL_IP4, 3203)
+    )
 
 
 @requires_ipv6
 @gen_test()
 def test_default_client_server_ipv6():
-    yield check_client_server('[::1]', tcp_eq('::1'))
-    yield check_client_server('[::1]:3211', tcp_eq('::1', 3211))
-    yield check_client_server('[::]', tcp_eq('::'), tcp_eq(EXTERNAL_IP6))
-    yield check_client_server('[::]:3212', tcp_eq('::', 3212), tcp_eq(EXTERNAL_IP6, 3212))
+    yield check_client_server("[::1]", tcp_eq("::1"))
+    yield check_client_server("[::1]:3211", tcp_eq("::1", 3211))
+    yield check_client_server("[::]", tcp_eq("::"), tcp_eq(EXTERNAL_IP6))
+    yield check_client_server(
+        "[::]:3212", tcp_eq("::", 3212), tcp_eq(EXTERNAL_IP6, 3212)
+    )
 
 
 @gen_test()
 def test_tcp_client_server_ipv4():
-    yield check_client_server('tcp://127.0.0.1', tcp_eq('127.0.0.1'))
-    yield check_client_server('tcp://127.0.0.1:3221', tcp_eq('127.0.0.1', 3221))
-    yield check_client_server('tcp://0.0.0.0',
-                              tcp_eq('0.0.0.0'), tcp_eq(EXTERNAL_IP4))
-    yield check_client_server('tcp://0.0.0.0:3222',
-                              tcp_eq('0.0.0.0', 3222), tcp_eq(EXTERNAL_IP4, 3222))
-    yield check_client_server('tcp://',
-                              tcp_eq('0.0.0.0'), tcp_eq(EXTERNAL_IP4))
-    yield check_client_server('tcp://:3223',
-                              tcp_eq('0.0.0.0', 3223), tcp_eq(EXTERNAL_IP4, 3223))
+    yield check_client_server("tcp://127.0.0.1", tcp_eq("127.0.0.1"))
+    yield check_client_server("tcp://127.0.0.1:3221", tcp_eq("127.0.0.1", 3221))
+    yield check_client_server("tcp://0.0.0.0", tcp_eq("0.0.0.0"), tcp_eq(EXTERNAL_IP4))
+    yield check_client_server(
+        "tcp://0.0.0.0:3222", tcp_eq("0.0.0.0", 3222), tcp_eq(EXTERNAL_IP4, 3222)
+    )
+    yield check_client_server("tcp://", tcp_eq("0.0.0.0"), tcp_eq(EXTERNAL_IP4))
+    yield check_client_server(
+        "tcp://:3223", tcp_eq("0.0.0.0", 3223), tcp_eq(EXTERNAL_IP4, 3223)
+    )
 
 
 @requires_ipv6
 @gen_test()
 def test_tcp_client_server_ipv6():
-    yield check_client_server('tcp://[::1]', tcp_eq('::1'))
-    yield check_client_server('tcp://[::1]:3231', tcp_eq('::1', 3231))
-    yield check_client_server('tcp://[::]',
-                              tcp_eq('::'), tcp_eq(EXTERNAL_IP6))
-    yield check_client_server('tcp://[::]:3232',
-                              tcp_eq('::', 3232), tcp_eq(EXTERNAL_IP6, 3232))
+    yield check_client_server("tcp://[::1]", tcp_eq("::1"))
+    yield check_client_server("tcp://[::1]:3231", tcp_eq("::1", 3231))
+    yield check_client_server("tcp://[::]", tcp_eq("::"), tcp_eq(EXTERNAL_IP6))
+    yield check_client_server(
+        "tcp://[::]:3232", tcp_eq("::", 3232), tcp_eq(EXTERNAL_IP6, 3232)
+    )
 
 
 @gen_test()
 def test_tls_client_server_ipv4():
-    yield check_client_server('tls://127.0.0.1', tls_eq('127.0.0.1'), **tls_kwargs)
-    yield check_client_server('tls://127.0.0.1:3221', tls_eq('127.0.0.1', 3221), **tls_kwargs)
-    yield check_client_server('tls://', tls_eq('0.0.0.0'),
-                              tls_eq(EXTERNAL_IP4), **tls_kwargs)
+    yield check_client_server("tls://127.0.0.1", tls_eq("127.0.0.1"), **tls_kwargs)
+    yield check_client_server(
+        "tls://127.0.0.1:3221", tls_eq("127.0.0.1", 3221), **tls_kwargs
+    )
+    yield check_client_server(
+        "tls://", tls_eq("0.0.0.0"), tls_eq(EXTERNAL_IP4), **tls_kwargs
+    )
 
 
 @requires_ipv6
 @gen_test()
 def test_tls_client_server_ipv6():
-    yield check_client_server('tls://[::1]', tls_eq('::1'), **tls_kwargs)
+    yield check_client_server("tls://[::1]", tls_eq("::1"), **tls_kwargs)
 
 
 @gen_test()
 def test_inproc_client_server():
-    yield check_client_server('inproc://', inproc_check())
+    yield check_client_server("inproc://", inproc_check())
     yield check_client_server(inproc.new_address(), inproc_check())
 
 
@@ -602,56 +636,66 @@ def test_inproc_client_server():
 # TLS certificate handling
 #
 
+
 @gen_test()
 def test_tls_reject_certificate():
     cli_ctx = get_client_ssl_context()
     serv_ctx = get_server_ssl_context()
 
     # These certs are not signed by our test CA
-    bad_cert_key = ('tls-self-signed-cert.pem', 'tls-self-signed-key.pem')
+    bad_cert_key = ("tls-self-signed-cert.pem", "tls-self-signed-key.pem")
     bad_cli_ctx = get_client_ssl_context(*bad_cert_key)
     bad_serv_ctx = get_server_ssl_context(*bad_cert_key)
 
     @gen.coroutine
     def handle_comm(comm):
         scheme, loc = parse_address(comm.peer_address)
-        assert scheme == 'tls'
+        assert scheme == "tls"
         yield comm.close()
 
     # Listener refuses a connector not signed by the CA
-    listener = listen('tls://', handle_comm,
-                      connection_args={'ssl_context': serv_ctx})
+    listener = listen("tls://", handle_comm, connection_args={"ssl_context": serv_ctx})
     listener.start()
 
     with pytest.raises(EnvironmentError) as excinfo:
-        comm = yield connect(listener.contact_address, timeout=0.5,
-                      connection_args={'ssl_context': bad_cli_ctx})
-        yield comm.write({'x': 'foo'})  # TODO: why is this necessary in Tornado 6 ?
+        comm = yield connect(
+            listener.contact_address,
+            timeout=0.5,
+            connection_args={"ssl_context": bad_cli_ctx},
+        )
+        yield comm.write({"x": "foo"})  # TODO: why is this necessary in Tornado 6 ?
 
     # The wrong error is reported on Python 2, see https://github.com/tornadoweb/tornado/pull/2028
-    if sys.version_info >= (3,) and os.name != 'nt':
+    if sys.version_info >= (3,) and os.name != "nt":
         try:
             # See https://serverfault.com/questions/793260/what-does-tlsv1-alert-unknown-ca-mean
             assert "unknown ca" in str(excinfo.value)
         except AssertionError:
-            if os.name == 'nt':
-                assert "An existing connection was forcibly closed" in str(excinfo.value)
+            if os.name == "nt":
+                assert "An existing connection was forcibly closed" in str(
+                    excinfo.value
+                )
             else:
                 raise
 
     # Sanity check
-    comm = yield connect(listener.contact_address, timeout=0.5,
-                         connection_args={'ssl_context': cli_ctx})
+    comm = yield connect(
+        listener.contact_address, timeout=0.5, connection_args={"ssl_context": cli_ctx}
+    )
     yield comm.close()
 
     # Connector refuses a listener not signed by the CA
-    listener = listen('tls://', handle_comm,
-                      connection_args={'ssl_context': bad_serv_ctx})
+    listener = listen(
+        "tls://", handle_comm, connection_args={"ssl_context": bad_serv_ctx}
+    )
     listener.start()
 
     with pytest.raises(EnvironmentError) as excinfo:
-        yield connect(listener.contact_address, timeout=0.5,
-                      connection_args={'ssl_context': cli_ctx})
+        yield connect(
+            listener.contact_address,
+            timeout=0.5,
+            connection_args={"ssl_context": cli_ctx},
+        )
     # The wrong error is reported on Python 2, see https://github.com/tornadoweb/tornado/pull/2028
     if sys.version_info >= (3,):
         assert "certificate verify failed" in str(excinfo.value)
@@ -661,9 +705,9 @@ def handle_comm(comm):
 # Test communication closing
 #
 
+
 @gen.coroutine
-def check_comm_closed_implicit(addr, delay=None, listen_args=None,
-                               connect_args=None):
+def check_comm_closed_implicit(addr, delay=None, listen_args=None, connect_args=None):
     @gen.coroutine
     def handle_comm(comm):
         yield comm.close()
@@ -683,12 +727,12 @@ def handle_comm(comm):
 
 @gen_test()
 def test_tcp_comm_closed_implicit():
-    yield check_comm_closed_implicit('tcp://127.0.0.1')
+    yield check_comm_closed_implicit("tcp://127.0.0.1")
 
 
 @gen_test()
 def test_tls_comm_closed_implicit():
-    yield check_comm_closed_implicit('tls://127.0.0.1', **tls_kwargs)
+    yield check_comm_closed_implicit("tls://127.0.0.1", **tls_kwargs)
 
 
 @gen_test()
@@ -722,12 +766,12 @@ def check_comm_closed_explicit(addr, listen_args=None, connect_args=None):
 
 @gen_test()
 def test_tcp_comm_closed_explicit():
-    yield check_comm_closed_explicit('tcp://127.0.0.1')
+    yield check_comm_closed_explicit("tcp://127.0.0.1")
 
 
 @gen_test()
 def test_tls_comm_closed_explicit():
-    yield check_comm_closed_explicit('tls://127.0.0.1', **tls_kwargs)
+    yield check_comm_closed_explicit("tls://127.0.0.1", **tls_kwargs)
 
 
 @gen_test()
@@ -750,7 +794,7 @@ def handle_comm(comm):
         else:
             comm.close()
 
-    listener = listen('inproc://', handle_comm)
+    listener = listen("inproc://", handle_comm)
     listener.start()
     contact_addr = listener.contact_address
 
@@ -792,6 +836,7 @@ def handle_comm(comm):
 # Various stress tests
 #
 
+
 @gen.coroutine
 def check_connect_timeout(addr):
     t1 = time()
@@ -803,7 +848,7 @@ def check_connect_timeout(addr):
 
 @gen_test()
 def test_tcp_connect_timeout():
-    yield check_connect_timeout('tcp://127.0.0.1:44444')
+    yield check_connect_timeout("tcp://127.0.0.1:44444")
 
 
 @gen_test()
@@ -833,20 +878,21 @@ def handle_comm(comm):
 
 @gen_test()
 def test_tcp_many_listeners():
-    check_many_listeners('tcp://127.0.0.1')
-    check_many_listeners('tcp://0.0.0.0')
-    check_many_listeners('tcp://')
+    check_many_listeners("tcp://127.0.0.1")
+    check_many_listeners("tcp://0.0.0.0")
+    check_many_listeners("tcp://")
 
 
 @gen_test()
 def test_inproc_many_listeners():
-    check_many_listeners('inproc://')
+    check_many_listeners("inproc://")
 
 
 #
 # Test deserialization
 #
 
+
 @gen.coroutine
 def check_listener_deserialize(addr, deserialize, in_value, check_out):
     q = queues.Queue()
@@ -893,21 +939,22 @@ def check_deserialize(addr):
     """
     # Test with Serialize and Serialized objects
 
-    msg = {'op': 'update',
-           'x': b'abc',
-           'to_ser': [to_serialize(123)],
-           'ser': Serialized(*serialize(456)),
-           }
+    msg = {
+        "op": "update",
+        "x": b"abc",
+        "to_ser": [to_serialize(123)],
+        "ser": Serialized(*serialize(456)),
+    }
     msg_orig = msg.copy()
 
     def check_out_false(out_value):
         # Check output with deserialize=False
         out_value = out_value.copy()  # in case transport passed the object as-is
-        to_ser = out_value.pop('to_ser')
-        ser = out_value.pop('ser')
+        to_ser = out_value.pop("to_ser")
+        ser = out_value.pop("ser")
         expected_msg = msg_orig.copy()
-        del expected_msg['ser']
-        del expected_msg['to_ser']
+        del expected_msg["ser"]
+        del expected_msg["to_ser"]
         assert out_value == expected_msg
 
         assert isinstance(ser, Serialized)
@@ -925,8 +972,8 @@ def check_out_false(out_value):
     def check_out_true(out_value):
         # Check output with deserialize=True
         expected_msg = msg.copy()
-        expected_msg['ser'] = 456
-        expected_msg['to_ser'] = [123]
+        expected_msg["ser"] = 456
+        expected_msg["to_ser"] = [123]
         assert out_value == expected_msg
 
     yield check_listener_deserialize(addr, False, msg, check_out_false)
@@ -940,22 +987,23 @@ def check_out_true(out_value):
 
     _uncompressible = os.urandom(1024 ** 2) * 4  # end size: 8 MB
 
-    msg = {'op': 'update',
-           'x': _uncompressible,
-           'to_ser': [to_serialize(_uncompressible)],
-           'ser': Serialized(*serialize(_uncompressible)),
-           }
+    msg = {
+        "op": "update",
+        "x": _uncompressible,
+        "to_ser": [to_serialize(_uncompressible)],
+        "ser": Serialized(*serialize(_uncompressible)),
+    }
     msg_orig = msg.copy()
 
     def check_out(deserialize_flag, out_value):
         # Check output with deserialize=False
         assert sorted(out_value) == sorted(msg_orig)
         out_value = out_value.copy()  # in case transport passed the object as-is
-        to_ser = out_value.pop('to_ser')
-        ser = out_value.pop('ser')
+        to_ser = out_value.pop("to_ser")
+        ser = out_value.pop("ser")
         expected_msg = msg_orig.copy()
-        del expected_msg['ser']
-        del expected_msg['to_ser']
+        del expected_msg["ser"]
+        del expected_msg["to_ser"]
         assert out_value == expected_msg
 
         if deserialize_flag:
@@ -980,15 +1028,15 @@ def check_out(deserialize_flag, out_value):
     yield check_connector_deserialize(addr, True, msg, partial(check_out, True))
 
 
-@pytest.mark.xfail(reason='intermittent failure on windows')
+@pytest.mark.xfail(reason="intermittent failure on windows")
 @gen_test()
 def test_tcp_deserialize():
-    yield check_deserialize('tcp://')
+    yield check_deserialize("tcp://")
 
 
 @gen_test()
 def test_inproc_deserialize():
-    yield check_deserialize('inproc://')
+    yield check_deserialize("inproc://")
 
 
 @gen.coroutine
@@ -1000,11 +1048,12 @@ def check_deserialize_roundtrip(addr):
     # as a separate payload
     _uncompressible = os.urandom(1024 ** 2) * 4  # end size: 4 MB
 
-    msg = {'op': 'update',
-           'x': _uncompressible,
-           'to_ser': [to_serialize(_uncompressible)],
-           'ser': Serialized(*serialize(_uncompressible)),
-           }
+    msg = {
+        "op": "update",
+        "x": _uncompressible,
+        "to_ser": [to_serialize(_uncompressible)],
+        "ser": Serialized(*serialize(_uncompressible)),
+    }
 
     for should_deserialize in (True, False):
         a, b = yield get_comm_pair(addr, deserialize=should_deserialize)
@@ -1014,24 +1063,24 @@ def check_deserialize_roundtrip(addr):
         got = yield a.read()
 
         assert sorted(got) == sorted(msg)
-        for k in ('op', 'x'):
+        for k in ("op", "x"):
             assert got[k] == msg[k]
         if should_deserialize:
-            assert isinstance(got['to_ser'][0], (bytes, bytearray))
-            assert isinstance(got['ser'], (bytes, bytearray))
+            assert isinstance(got["to_ser"][0], (bytes, bytearray))
+            assert isinstance(got["ser"], (bytes, bytearray))
         else:
-            assert isinstance(got['to_ser'][0], (to_serialize, Serialized))
-            assert isinstance(got['ser'], Serialized)
+            assert isinstance(got["to_ser"][0], (to_serialize, Serialized))
+            assert isinstance(got["ser"], Serialized)
 
 
 @gen_test()
 def test_inproc_deserialize_roundtrip():
-    yield check_deserialize_roundtrip('inproc://')
+    yield check_deserialize_roundtrip("inproc://")
 
 
 @gen_test()
 def test_tcp_deserialize_roundtrip():
-    yield check_deserialize_roundtrip('tcp://')
+    yield check_deserialize_roundtrip("tcp://")
 
 
 def _raise_eoferror():
@@ -1048,9 +1097,10 @@ def check_deserialize_eoferror(addr):
     """
     EOFError when deserializing should close the comm.
     """
+
     @gen.coroutine
     def handle_comm(comm):
-        yield comm.write({'data': to_serialize(_EOFRaising())})
+        yield comm.write({"data": to_serialize(_EOFRaising())})
         with pytest.raises(CommClosedError):
             yield comm.read()
 
@@ -1062,21 +1112,22 @@ def handle_comm(comm):
 
 @gen_test()
 def test_tcp_deserialize_eoferror():
-    yield check_deserialize_eoferror('tcp://')
+    yield check_deserialize_eoferror("tcp://")
 
 
 #
 # Test various properties
 #
 
+
 @gen.coroutine
 def check_repr(a, b):
-    assert 'closed' not in repr(a)
-    assert 'closed' not in repr(b)
+    assert "closed" not in repr(a)
+    assert "closed" not in repr(b)
     yield a.close()
-    assert 'closed' in repr(a)
+    assert "closed" in repr(a)
     yield b.close()
-    assert 'closed' in repr(b)
+    assert "closed" in repr(b)
 
 
 @gen_test()
diff --git a/distributed/comm/utils.py b/distributed/comm/utils.py
index 6c9a99b8a8d..bb6621e2021 100644
--- a/distributed/comm/utils.py
+++ b/distributed/comm/utils.py
@@ -17,10 +17,12 @@
 # Offload (de)serializing large frames to improve event loop responsiveness.
 # We use at most 4 threads to allow for parallel processing of large messages.
 
-FRAME_OFFLOAD_THRESHOLD = 10 * 1024 ** 2   # 10 MB
+FRAME_OFFLOAD_THRESHOLD = 10 * 1024 ** 2  # 10 MB
 
 try:
-    _offload_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix='Dask-Offload')
+    _offload_executor = ThreadPoolExecutor(
+        max_workers=1, thread_name_prefix="Dask-Offload"
+    )
 except TypeError:
     _offload_executor = ThreadPoolExecutor(max_workers=1)
 finalize(_offload_executor, _offload_executor.shutdown)
@@ -31,16 +33,18 @@ def offload(fn, *args, **kwargs):
 
 
 @gen.coroutine
-def to_frames(msg, serializers=None, on_error='message', context=None):
+def to_frames(msg, serializers=None, on_error="message", context=None):
     """
     Serialize a message into a list of Distributed protocol frames.
     """
+
     def _to_frames():
         try:
-            return list(protocol.dumps(msg,
-                                       serializers=serializers,
-                                       on_error=on_error,
-                                       context=context))
+            return list(
+                protocol.dumps(
+                    msg, serializers=serializers, on_error=on_error, context=context
+                )
+            )
         except Exception as e:
             logger.info("Unserializable Message: %s", msg)
             logger.exception(e)
@@ -63,17 +67,16 @@ def from_frames(frames, deserialize=True, deserializers=None):
 
     def _from_frames():
         try:
-            return protocol.loads(frames,
-                                  deserialize=deserialize,
-                                  deserializers=deserializers)
+            return protocol.loads(
+                frames, deserialize=deserialize, deserializers=deserializers
+            )
         except EOFError:
             if size > 1000:
                 datastr = "[too large to display]"
             else:
                 datastr = frames
             # Aid diagnosing
-            logger.error("truncated data stream (%d bytes): %s", size,
-                         datastr)
+            logger.error("truncated data stream (%d bytes): %s", size, datastr)
             raise
 
     if deserialize and size > FRAME_OFFLOAD_THRESHOLD:
@@ -114,9 +117,9 @@ def ensure_concrete_host(host):
     Ensure the given host string (or IP) denotes a concrete host, not a
     wildcard listening address.
     """
-    if host in ('0.0.0.0', ''):
+    if host in ("0.0.0.0", ""):
         return get_ip()
-    elif host == '::':
+    elif host == "::":
         return get_ipv6()
     else:
         return host
diff --git a/distributed/compatibility.py b/distributed/compatibility.py
index fbf8f86df5d..f3a85973802 100644
--- a/distributed/compatibility.py
+++ b/distributed/compatibility.py
@@ -20,6 +20,7 @@
     PY3 = False
     ConnectionRefusedError = OSError
     FileExistsError = OSError
+
     class StopAsyncIteration(Exception):
         pass
 
@@ -33,7 +34,7 @@ def gzip_decompress(b):
 
     def gzip_compress(b):
         bio = BytesIO()
-        f = gzip.GzipFile(fileobj=bio, mode='w')
+        f = gzip.GzipFile(fileobj=bio, mode="w")
         f.write(b)
         f.close()
         bio.seek(0)
@@ -41,23 +42,25 @@ def gzip_compress(b):
         return result
 
     def isqueue(o):
-        return (hasattr(o, 'queue') and
-                hasattr(o, '__module__') and
-                o.__module__ == 'Queue')
+        return (
+            hasattr(o, "queue") and hasattr(o, "__module__") and o.__module__ == "Queue"
+        )
 
     def invalidate_caches():
         pass
 
     def cache_from_source(path):
         import os
+
         name, ext = os.path.splitext(path)
-        return name + '.pyc'
+        return name + ".pyc"
 
     logging_names = logging._levelNames
 
     def iscoroutinefunction(func):
         return False
 
+
 if sys.version_info[0] == 3:
     from asyncio import iscoroutinefunction
     from collections.abc import Iterator, Mapping, Set, MutableMapping
@@ -75,6 +78,7 @@ def iscoroutinefunction(func):
     unicode = str
     from gzip import decompress as gzip_decompress
     from gzip import compress as gzip_compress
+
     ConnectionRefusedError = ConnectionRefusedError
     FileExistsError = FileExistsError
     StopAsyncIteration = StopAsyncIteration
@@ -87,8 +91,9 @@ def isqueue(o):
 
 
 import platform
-PYPY = platform.python_implementation().lower() == 'pypy'
-WINDOWS = sys.platform.startswith('win')
+
+PYPY = platform.python_implementation().lower() == "pypy"
+WINDOWS = sys.platform.startswith("win")
 
 
 try:
@@ -141,6 +146,7 @@ def __init__(self, obj, func, *args, **kwargs):
                 # We may register the exit function more than once because
                 # of a thread race, but that is harmless
                 import atexit
+
                 atexit.register(self._exitfunc)
                 finalize._registered_with_atexit = True
             info = self._Info()
@@ -197,10 +203,14 @@ def __repr__(self):
             info = self._registry.get(self)
             obj = info and info.weakref()
             if obj is None:
-                return '<%s object at %#x; dead>' % (type(self).__name__, id(self))
+                return "<%s object at %#x; dead>" % (type(self).__name__, id(self))
             else:
-                return '<%s object at %#x; for %r at %#x>' % \
-                    (type(self).__name__, id(self), type(obj).__name__, id(obj))
+                return "<%s object at %#x; for %r at %#x>" % (
+                    type(self).__name__,
+                    id(self),
+                    type(obj).__name__,
+                    id(obj),
+                )
 
         @classmethod
         def _select_for_exit(cls):
@@ -218,6 +228,7 @@ def _exitfunc(cls):
             try:
                 if cls._registry:
                     import gc
+
                     if gc.isenabled():
                         reenable_gc = True
                         gc.disable()
diff --git a/distributed/config.py b/distributed/config.py
index d2b27397393..4b7b589d58f 100644
--- a/distributed/config.py
+++ b/distributed/config.py
@@ -13,7 +13,7 @@
 config = dask.config.config
 
 
-fn = os.path.join(os.path.dirname(__file__), 'distributed.yaml')
+fn = os.path.join(os.path.dirname(__file__), "distributed.yaml")
 dask.config.ensure_file(source=fn)
 
 with open(fn) as f:
@@ -22,39 +22,34 @@
 dask.config.update_defaults(defaults)
 
 aliases = {
-    'allowed-failures': 'distributed.scheduler.allowed-failures',
-    'bandwidth': 'distributed.scheduler.bandwidth',
-    'default-data-size': 'distributed.scheduler.default-data-size',
-    'transition-log-length': 'distributed.scheduler.transition-log-length',
-    'work-stealing': 'distributed.scheduler.work-stealing',
-    'worker-ttl': 'distributed.scheduler.worker-ttl',
-
-    'multiprocessing-method': 'distributed.worker.multiprocessing-method',
-    'use-file-locking': 'distributed.worker.use-file-locking',
-    'profile-interval': 'distributed.worker.profile.interval',
-    'profile-cycle-interval': 'distributed.worker.profile.cycle',
-    'worker-memory-target': 'distributed.worker.memory.target',
-    'worker-memory-spill': 'distributed.worker.memory.spill',
-    'worker-memory-pause': 'distributed.worker.memory.pause',
-    'worker-memory-terminate': 'distributed.worker.memory.terminate',
-
-    'heartbeat-interval': 'distributed.client.heartbeat',
-
-    'compression': 'distributed.comm.compression',
-    'connect-timeout': 'distributed.comm.timeouts.connect',
-    'tcp-timeout': 'distributed.comm.timeouts.tcp',
-    'default-scheme': 'distributed.comm.default-scheme',
-    'socket-backlog': 'distributed.comm.socket-backlog',
-    'recent-messages-log-length': 'distributed.comm.recent-messages-log-length',
-
-    'diagnostics-link': 'distributed.dashboard.link',
-    'bokeh-export-tool': 'distributed.dashboard.export-tool',
-
-    'tick-time': 'distributed.admin.tick.interval',
-    'tick-maximum-delay': 'distributed.admin.tick.limit',
-    'log-length': 'distributed.admin.log-length',
-    'log-format': 'distributed.admin.log-format',
-    'pdb-on-err': 'distributed.admin.pdb-on-err',
+    "allowed-failures": "distributed.scheduler.allowed-failures",
+    "bandwidth": "distributed.scheduler.bandwidth",
+    "default-data-size": "distributed.scheduler.default-data-size",
+    "transition-log-length": "distributed.scheduler.transition-log-length",
+    "work-stealing": "distributed.scheduler.work-stealing",
+    "worker-ttl": "distributed.scheduler.worker-ttl",
+    "multiprocessing-method": "distributed.worker.multiprocessing-method",
+    "use-file-locking": "distributed.worker.use-file-locking",
+    "profile-interval": "distributed.worker.profile.interval",
+    "profile-cycle-interval": "distributed.worker.profile.cycle",
+    "worker-memory-target": "distributed.worker.memory.target",
+    "worker-memory-spill": "distributed.worker.memory.spill",
+    "worker-memory-pause": "distributed.worker.memory.pause",
+    "worker-memory-terminate": "distributed.worker.memory.terminate",
+    "heartbeat-interval": "distributed.client.heartbeat",
+    "compression": "distributed.comm.compression",
+    "connect-timeout": "distributed.comm.timeouts.connect",
+    "tcp-timeout": "distributed.comm.timeouts.tcp",
+    "default-scheme": "distributed.comm.default-scheme",
+    "socket-backlog": "distributed.comm.socket-backlog",
+    "recent-messages-log-length": "distributed.comm.recent-messages-log-length",
+    "diagnostics-link": "distributed.dashboard.link",
+    "bokeh-export-tool": "distributed.dashboard.export-tool",
+    "tick-time": "distributed.admin.tick.interval",
+    "tick-maximum-delay": "distributed.admin.tick.limit",
+    "log-length": "distributed.admin.log-length",
+    "log-format": "distributed.admin.log-format",
+    "pdb-on-err": "distributed.admin.pdb-on-err",
 }
 
 dask.config.rename(aliases)
@@ -81,17 +76,20 @@ def _initialize_logging_old_style(config):
         }
     """
     loggers = {  # default values
-        'distributed': 'info',
-        'distributed.client': 'warning',
-        'bokeh': 'critical',
-        'tornado': 'critical',
-        'tornado.application': 'error',
+        "distributed": "info",
+        "distributed.client": "warning",
+        "bokeh": "critical",
+        "tornado": "critical",
+        "tornado.application": "error",
     }
-    loggers.update(config.get('logging', {}))
+    loggers.update(config.get("logging", {}))
 
     handler = logging.StreamHandler(sys.stderr)
-    handler.setFormatter(logging.Formatter(dask.config.get('distributed.admin.log-format',
-                                                           config=config)))
+    handler.setFormatter(
+        logging.Formatter(
+            dask.config.get("distributed.admin.log-format", config=config)
+        )
+    )
     for name, level in loggers.items():
         if isinstance(level, str):
             level = logging_names[level.upper()]
@@ -107,7 +105,7 @@ def _initialize_logging_new_style(config):
     Initialize logging using logging's "Configuration dictionary schema".
     (ref.: https://docs.python.org/2/library/logging.config.html#logging-config-dictschema)
     """
-    logging.config.dictConfig(config.get('logging'))
+    logging.config.dictConfig(config.get("logging"))
 
 
 def _initialize_logging_file_config(config):
@@ -115,19 +113,23 @@ def _initialize_logging_file_config(config):
     Initialize logging using logging's "Configuration file format".
     (ref.: https://docs.python.org/2/library/logging.config.html#configuration-file-format)
     """
-    logging.config.fileConfig(config.get('logging-file-config'), disable_existing_loggers=False)
+    logging.config.fileConfig(
+        config.get("logging-file-config"), disable_existing_loggers=False
+    )
 
 
 def initialize_logging(config):
-    if 'logging-file-config' in config:
-        if 'logging' in config:
-            raise RuntimeError("Config options 'logging-file-config' and 'logging' are mutually exclusive.")
+    if "logging-file-config" in config:
+        if "logging" in config:
+            raise RuntimeError(
+                "Config options 'logging-file-config' and 'logging' are mutually exclusive."
+            )
         _initialize_logging_file_config(config)
     else:
-        log_config = config.get('logging', {})
-        if 'version' in log_config:
+        log_config = config.get("logging", {})
+        if "version" in log_config:
             # logging module mandates version to be an int
-            log_config['version'] = int(log_config['version'])
+            log_config["version"] = int(log_config["version"])
             _initialize_logging_new_style(config)
         else:
             _initialize_logging_old_style(config)
diff --git a/distributed/core.py b/distributed/core.py
index a9883801549..e074fa68148 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -17,14 +17,26 @@
 from tornado.locks import Event
 
 from .compatibility import get_thread_identity
-from .comm import (connect, listen, CommClosedError,
-                   normalize_address,
-                   unparse_host_port, get_address_host_port)
+from .comm import (
+    connect,
+    listen,
+    CommClosedError,
+    normalize_address,
+    unparse_host_port,
+    get_address_host_port,
+)
 from .metrics import time
 from . import profile
 from .system_monitor import SystemMonitor
-from .utils import (get_traceback, truncate_exception, ignoring, shutting_down,
-                    PeriodicCallback, parse_timedelta, has_keyword)
+from .utils import (
+    get_traceback,
+    truncate_exception,
+    ignoring,
+    shutting_down,
+    PeriodicCallback,
+    parse_timedelta,
+    has_keyword,
+)
 from . import protocol
 
 
@@ -38,6 +50,7 @@ class RPCClosed(IOError):
 def get_total_physical_memory():
     try:
         import psutil
+
         return psutil.virtual_memory().total / 2
     except ImportError:
         return 2e9
@@ -46,14 +59,17 @@ def get_total_physical_memory():
 def raise_later(exc):
     def _raise(*args, **kwargs):
         raise exc
+
     return _raise
 
 
 MAX_BUFFER_SIZE = get_total_physical_memory()
 
-tick_maximum_delay = parse_timedelta(dask.config.get('distributed.admin.tick.limit'), default='ms')
+tick_maximum_delay = parse_timedelta(
+    dask.config.get("distributed.admin.tick.limit"), default="ms"
+)
 
-LOG_PDB = dask.config.get('distributed.admin.pdb-on-err')
+LOG_PDB = dask.config.get("distributed.admin.pdb-on-err")
 
 
 class Server(object):
@@ -92,23 +108,33 @@ class Server(object):
     *  ``{'op': 'add', 'x': 10, 'y': 20}``
 
     """
-    default_ip = ''
+
+    default_ip = ""
     default_port = 0
 
-    def __init__(self, handlers, blocked_handlers=None, stream_handlers=None, connection_limit=512,
-                 deserialize=True, io_loop=None):
+    def __init__(
+        self,
+        handlers,
+        blocked_handlers=None,
+        stream_handlers=None,
+        connection_limit=512,
+        deserialize=True,
+        io_loop=None,
+    ):
         self.handlers = {
-            'identity': self.identity,
-            'connection_stream': self.handle_stream,
+            "identity": self.identity,
+            "connection_stream": self.handle_stream,
         }
         self.handlers.update(handlers)
         if blocked_handlers is None:
-            blocked_handlers = dask.config.get('distributed.%s.blocked-handlers' % type(self).__name__.lower(), [])
+            blocked_handlers = dask.config.get(
+                "distributed.%s.blocked-handlers" % type(self).__name__.lower(), []
+            )
         self.blocked_handlers = blocked_handlers
         self.stream_handlers = {}
         self.stream_handlers.update(stream_handlers or {})
 
-        self.id = type(self).__name__ + '-' + str(uuid.uuid4())
+        self.id = type(self).__name__ + "-" + str(uuid.uuid4())
         self._address = None
         self._listen_address = None
         self._port = None
@@ -125,31 +151,36 @@ def __init__(self, handlers, blocked_handlers=None, stream_handlers=None, connec
         self.io_loop = io_loop or IOLoop.current()
         self.loop = self.io_loop
 
-        if not hasattr(self.io_loop, 'profile'):
+        if not hasattr(self.io_loop, "profile"):
             ref = weakref.ref(self.io_loop)
 
-            if hasattr(self.io_loop, 'closing'):
+            if hasattr(self.io_loop, "closing"):
+
                 def stop():
                     loop = ref()
                     return loop is None or loop.closing
+
             else:
+
                 def stop():
                     loop = ref()
                     return loop is None or loop._closing
 
             self.io_loop.profile = profile.watch(
-                    omit=('profile.py', 'selectors.py'),
-                    interval=dask.config.get('distributed.worker.profile.interval'),
-                    cycle=dask.config.get('distributed.worker.profile.cycle'),
-                    stop=stop,
+                omit=("profile.py", "selectors.py"),
+                interval=dask.config.get("distributed.worker.profile.interval"),
+                cycle=dask.config.get("distributed.worker.profile.cycle"),
+                stop=stop,
             )
 
         # Statistics counters for various events
         with ignoring(ImportError):
             from .counter import Digest
+
             self.digests = defaultdict(partial(Digest, loop=self.io_loop))
 
         from .counter import Counter
+
         self.counters = defaultdict(partial(Counter, loop=self.io_loop))
         self.events = defaultdict(lambda: deque(maxlen=10000))
         self.event_counts = defaultdict(lambda: 0)
@@ -157,15 +188,18 @@ def stop():
         self.periodic_callbacks = dict()
 
         pc = PeriodicCallback(self.monitor.update, 500, io_loop=self.io_loop)
-        self.periodic_callbacks['monitor'] = pc
+        self.periodic_callbacks["monitor"] = pc
 
         self._last_tick = time()
         pc = PeriodicCallback(
-                self._measure_tick,
-                parse_timedelta(dask.config.get('distributed.admin.tick.interval'), default='ms') * 1000,
-                io_loop=self.io_loop
+            self._measure_tick,
+            parse_timedelta(
+                dask.config.get("distributed.admin.tick.interval"), default="ms"
+            )
+            * 1000,
+            io_loop=self.io_loop,
         )
-        self.periodic_callbacks['tick'] = pc
+        self.periodic_callbacks["tick"] = pc
 
         self.thread_id = 0
 
@@ -189,6 +223,7 @@ def start_pcs():
             for pc in self.periodic_callbacks.values():
                 if not pc.is_running():
                     pc.start()
+
         self.io_loop.add_callback(start_pcs)
 
     def stop(self):
@@ -209,16 +244,19 @@ def _measure_tick(self):
         diff = now - self._last_tick
         self._last_tick = now
         if diff > tick_maximum_delay:
-            logger.info("Event loop was unresponsive in %s for %.2fs.  "
-                        "This is often caused by long-running GIL-holding "
-                        "functions or moving large chunks of data. "
-                        "This can cause timeouts and instability.",
-                        type(self).__name__, diff)
+            logger.info(
+                "Event loop was unresponsive in %s for %.2fs.  "
+                "This is often caused by long-running GIL-holding "
+                "functions or moving large chunks of data. "
+                "This can cause timeouts and instability.",
+                type(self).__name__,
+                diff,
+            )
         if self.digests is not None:
-            self.digests['tick-duration'].add(diff)
+            self.digests["tick-duration"].add(diff)
 
     def log_event(self, name, msg):
-        msg['time'] = time()
+        msg["time"] = time()
         if isinstance(name, list):
             for n in name:
                 self.events[n].append(msg)
@@ -263,7 +301,7 @@ def port(self):
         return self._port
 
     def identity(self, comm=None):
-        return {'type': type(self).__name__, 'id': self.id}
+        return {"type": type(self).__name__, "id": self.id}
 
     def listen(self, port_or_addr=None, listen_args=None):
         if port_or_addr is None:
@@ -275,9 +313,12 @@ def listen(self, port_or_addr=None, listen_args=None):
         else:
             addr = port_or_addr
             assert isinstance(addr, string_types)
-        self.listener = listen(addr, self.handle_comm,
-                               deserialize=self.deserialize,
-                               connection_args=listen_args)
+        self.listener = listen(
+            addr,
+            self.handle_comm,
+            deserialize=self.deserialize,
+            connection_args=listen_args,
+        )
         self.listener.start()
 
     @gen.coroutine
@@ -307,51 +348,61 @@ def handle_comm(self, comm, shutting_down=shutting_down):
                     logger.debug("Message from %r: %s", address, msg)
                 except EnvironmentError as e:
                     if not shutting_down():
-                        logger.debug("Lost connection to %r while reading message: %s."
-                                     " Last operation: %s",
-                                     address, e, op)
+                        logger.debug(
+                            "Lost connection to %r while reading message: %s."
+                            " Last operation: %s",
+                            address,
+                            e,
+                            op,
+                        )
                     break
                 except Exception as e:
                     logger.exception(e)
-                    yield comm.write(error_message(e, status='uncaught-error'))
+                    yield comm.write(error_message(e, status="uncaught-error"))
                     continue
                 if not isinstance(msg, dict):
-                    raise TypeError("Bad message type.  Expected dict, got\n  "
-                                    + str(msg))
+                    raise TypeError(
+                        "Bad message type.  Expected dict, got\n  " + str(msg)
+                    )
 
                 try:
-                    op = msg.pop('op')
+                    op = msg.pop("op")
                 except KeyError:
                     raise ValueError(
-                        "Received unexpected message without 'op' key: " %
-                        str(msg)
+                        "Received unexpected message without 'op' key: " % str(msg)
                     )
                 if self.counters is not None:
-                    self.counters['op'].add(op)
+                    self.counters["op"].add(op)
                 self._comms[comm] = op
-                serializers = msg.pop('serializers', None)
-                close_desired = msg.pop('close', False)
-                reply = msg.pop('reply', True)
-                if op == 'close':
+                serializers = msg.pop("serializers", None)
+                close_desired = msg.pop("close", False)
+                reply = msg.pop("reply", True)
+                if op == "close":
                     if reply:
-                        yield comm.write('OK')
+                        yield comm.write("OK")
                     break
 
                 result = None
                 try:
                     if op in self.blocked_handlers:
-                        _msg = ("The '{op}' handler has been explicitly disallowed "
-                                "in {obj}, possibly due to security concerns.")
+                        _msg = (
+                            "The '{op}' handler has been explicitly disallowed "
+                            "in {obj}, possibly due to security concerns."
+                        )
                         exc = ValueError(_msg.format(op=op, obj=type(self).__name__))
                         handler = raise_later(exc)
                     else:
                         handler = self.handlers[op]
                 except KeyError:
-                    logger.warning("No handler %s found in %s", op,
-                                   type(self).__name__, exc_info=True)
+                    logger.warning(
+                        "No handler %s found in %s",
+                        op,
+                        type(self).__name__,
+                        exc_info=True,
+                    )
                 else:
-                    if serializers is not None and has_keyword(handler, 'serializers'):
-                        msg['serializers'] = serializers  # add back in
+                    if serializers is not None and has_keyword(handler, "serializers"):
+                        msg["serializers"] = serializers  # add back in
 
                     logger.debug("Calling into handler %s", handler.__name__)
                     try:
@@ -360,19 +411,23 @@ def handle_comm(self, comm, shutting_down=shutting_down):
                             self._ongoing_coroutines.add(result)
                             result = yield result
                     except (CommClosedError, CancelledError) as e:
-                        if self.status == 'running':
+                        if self.status == "running":
                             logger.info("Lost connection to %r: %s", address, e)
                         break
                     except Exception as e:
                         logger.exception(e)
-                        result = error_message(e, status='uncaught-error')
+                        result = error_message(e, status="uncaught-error")
 
-                if reply and result != 'dont-reply':
+                if reply and result != "dont-reply":
                     try:
                         yield comm.write(result, serializers=serializers)
                     except (EnvironmentError, TypeError) as e:
-                        logger.debug("Lost connection to %r while sending result for op %r: %s",
-                                     address, op, e)
+                        logger.debug(
+                            "Lost connection to %r while sending result for op %r: %s",
+                            address,
+                            op,
+                            e,
+                        )
                         break
                 msg = result = None
                 if close_desired:
@@ -386,8 +441,9 @@ def handle_comm(self, comm, shutting_down=shutting_down):
                 try:
                     comm.abort()
                 except Exception as e:
-                    logger.error("Failed while closing connection to %r: %s",
-                                 address, e)
+                    logger.error(
+                        "Failed while closing connection to %r: %s", address, e
+                    )
 
     @gen.coroutine
     def handle_stream(self, comm, extra=None, every_cycle=[]):
@@ -404,11 +460,11 @@ def handle_stream(self, comm, extra=None, every_cycle=[]):
 
                 if not comm.closed():
                     for msg in msgs:
-                        if msg == 'OK':  # from close
+                        if msg == "OK":  # from close
                             break
-                        op = msg.pop('op')
+                        op = msg.pop("op")
                         if op:
-                            if op == 'close-stream':
+                            if op == "close-stream":
                                 closed = True
                                 break
                             handler = self.stream_handlers[op]
@@ -424,6 +480,7 @@ def handle_stream(self, comm, extra=None, every_cycle=[]):
             logger.exception(e)
             if LOG_PDB:
                 import pdb
+
                 pdb.set_trace()
             raise
         finally:
@@ -445,12 +502,11 @@ def close(self):
 
 
 def pingpong(comm):
-    return b'pong'
+    return b"pong"
 
 
 @gen.coroutine
-def send_recv(comm, reply=True, serializers=None, deserializers=None,
-              **kwargs):
+def send_recv(comm, reply=True, serializers=None, deserializers=None, **kwargs):
     """ Send and recv with a Comm.
 
     Keyword arguments turn into the message
@@ -458,16 +514,16 @@ def send_recv(comm, reply=True, serializers=None, deserializers=None,
     response = yield send_recv(comm, op='ping', reply=True)
     """
     msg = kwargs
-    msg['reply'] = reply
-    please_close = kwargs.get('close')
+    msg["reply"] = reply
+    please_close = kwargs.get("close")
     force_close = False
     if deserializers is None:
         deserializers = serializers
     if deserializers is not None:
-        msg['serializers'] = deserializers
+        msg["serializers"] = deserializers
 
     try:
-        yield comm.write(msg, serializers=serializers, on_error='raise')
+        yield comm.write(msg, serializers=serializers, on_error="raise")
         if reply:
             response = yield comm.read(deserializers=deserializers)
         else:
@@ -482,11 +538,11 @@ def send_recv(comm, reply=True, serializers=None, deserializers=None,
         elif force_close:
             comm.abort()
 
-    if isinstance(response, dict) and response.get('status') == 'uncaught-error':
+    if isinstance(response, dict) and response.get("status") == "uncaught-error":
         if comm.deserialize:
             six.reraise(*clean_exception(**response))
         else:
-            raise Exception(response['text'])
+            raise Exception(response["text"])
     raise gen.Return(response)
 
 
@@ -514,16 +570,25 @@ class rpc(object):
 
     >>> remote.close_comms()  # doctest: +SKIP
     """
+
     active = weakref.WeakSet()
     comms = ()
     address = None
 
-    def __init__(self, arg=None, comm=None, deserialize=True, timeout=None,
-                 connection_args=None, serializers=None, deserializers=None):
+    def __init__(
+        self,
+        arg=None,
+        comm=None,
+        deserialize=True,
+        timeout=None,
+        connection_args=None,
+        serializers=None,
+        deserializers=None,
+    ):
         self.comms = {}
         self.address = coerce_to_address(arg)
         self.timeout = timeout
-        self.status = 'running'
+        self.status = "running"
         self.deserialize = deserialize
         self.serializers = serializers
         self.deserializers = deserializers if deserializers is not None else serializers
@@ -549,7 +614,7 @@ def live_comm(self):
 
         As is done in __getattr__ below.
         """
-        if self.status == 'closed':
+        if self.status == "closed":
             raise RPCClosed("RPC Closed")
         to_clear = set()
         open = False
@@ -561,19 +626,21 @@ def live_comm(self):
         for s in to_clear:
             del self.comms[s]
         if not open or comm.closed():
-            comm = yield connect(self.address, self.timeout,
-                                 deserialize=self.deserialize,
-                                 connection_args=self.connection_args)
-        self.comms[comm] = False     # mark as taken
+            comm = yield connect(
+                self.address,
+                self.timeout,
+                deserialize=self.deserialize,
+                connection_args=self.connection_args,
+            )
+        self.comms[comm] = False  # mark as taken
         raise gen.Return(comm)
 
     def close_comms(self):
-
         @gen.coroutine
         def _close_comm(comm):
             # Make sure we tell the peer to close
             try:
-                yield comm.write({'op': 'close', 'reply': False})
+                yield comm.write({"op": "close", "reply": False})
                 yield comm.close()
             except EnvironmentError:
                 comm.abort()
@@ -586,25 +653,27 @@ def _close_comm(comm):
     def __getattr__(self, key):
         @gen.coroutine
         def send_recv_from_rpc(**kwargs):
-            if self.serializers is not None and kwargs.get('serializers') is None:
-                kwargs['serializers'] = self.serializers
-            if self.deserializers is not None and kwargs.get('deserializers') is None:
-                kwargs['deserializers'] = self.deserializers
+            if self.serializers is not None and kwargs.get("serializers") is None:
+                kwargs["serializers"] = self.serializers
+            if self.deserializers is not None and kwargs.get("deserializers") is None:
+                kwargs["deserializers"] = self.deserializers
             try:
                 comm = yield self.live_comm()
                 result = yield send_recv(comm=comm, op=key, **kwargs)
             except (RPCClosed, CommClosedError) as e:
-                raise e.__class__("%s: while trying to call remote method %r"
-                                  % (e, key,))
+                raise e.__class__(
+                    "%s: while trying to call remote method %r" % (e, key)
+                )
 
             self.comms[comm] = True  # mark as open
             raise gen.Return(result)
+
         return send_recv_from_rpc
 
     def close_rpc(self):
-        if self.status != 'closed':
+        if self.status != "closed":
             rpc.active.discard(self)
-        self.status = 'closed'
+        self.status = "closed"
         self.close_comms()
 
     def __enter__(self):
@@ -614,13 +683,14 @@ def __exit__(self, *args):
         self.close_rpc()
 
     def __del__(self):
-        if self.status != 'closed':
+        if self.status != "closed":
             rpc.active.discard(self)
-            self.status = 'closed'
+            self.status = "closed"
             still_open = [comm for comm in self.comms if not comm.closed()]
             if still_open:
-                logger.warning("rpc object %s deleted with %d open comms",
-                               self, len(still_open))
+                logger.warning(
+                    "rpc object %s deleted with %d open comms", self, len(still_open)
+                )
                 for comm in still_open:
                     comm.abort()
 
@@ -648,10 +718,10 @@ def address(self):
     def __getattr__(self, key):
         @gen.coroutine
         def send_recv_from_rpc(**kwargs):
-            if self.serializers is not None and kwargs.get('serializers') is None:
-                kwargs['serializers'] = self.serializers
-            if self.deserializers is not None and kwargs.get('deserializers') is None:
-                kwargs['deserializers'] = self.deserializers
+            if self.serializers is not None and kwargs.get("serializers") is None:
+                kwargs["serializers"] = self.serializers
+            if self.deserializers is not None and kwargs.get("deserializers") is None:
+                kwargs["deserializers"] = self.deserializers
             comm = yield self.pool.connect(self.addr)
             try:
                 result = yield send_recv(comm=comm, op=key, **kwargs)
@@ -659,6 +729,7 @@ def send_recv_from_rpc(**kwargs):
                 self.pool.reuse(self.addr, comm)
 
             raise gen.Return(result)
+
         return send_recv_from_rpc
 
     def close_rpc(self):
@@ -709,12 +780,15 @@ class ConnectionPool(object):
         Whether or not to deserialize data by default or pass it through
     """
 
-    def __init__(self, limit=512,
-                 deserialize=True,
-                 serializers=None,
-                 deserializers=None,
-                 connection_args=None):
-        self.limit = limit     # Max number of open comms
+    def __init__(
+        self,
+        limit=512,
+        deserialize=True,
+        serializers=None,
+        deserializers=None,
+        connection_args=None,
+    ):
+        self.limit = limit  # Max number of open comms
         # Invariant: len(available) == open - active
         self.available = defaultdict(set)
         # Invariant: len(occupied) == active
@@ -734,15 +808,14 @@ def open(self):
         return self.active + sum(map(len, self.available.values()))
 
     def __repr__(self):
-        return "<ConnectionPool: open=%d, active=%d>" % (self.open,
-                                                         self.active)
+        return "<ConnectionPool: open=%d, active=%d>" % (self.open, self.active)
 
     def __call__(self, addr=None, ip=None, port=None):
         """ Cached rpc objects """
         addr = addr_from_args(addr=addr, ip=ip, port=port)
-        return PooledRPCCall(addr, self,
-                             serializers=self.serializers,
-                             deserializers=self.deserializers)
+        return PooledRPCCall(
+            addr, self, serializers=self.serializers, deserializers=self.deserializers
+        )
 
     @gen.coroutine
     def connect(self, addr, timeout=None):
@@ -763,9 +836,12 @@ def connect(self, addr, timeout=None):
             yield self.event.wait()
 
         try:
-            comm = yield connect(addr, timeout=timeout,
-                                 deserialize=self.deserialize,
-                                 connection_args=self.connection_args)
+            comm = yield connect(
+                addr,
+                timeout=timeout,
+                deserialize=self.deserialize,
+                connection_args=self.connection_args,
+            )
         except Exception:
             raise
         occupied.add(comm)
@@ -794,8 +870,9 @@ def collect(self):
         """
         Collect open but unused communications, to allow opening other ones.
         """
-        logger.info("Collecting unused comms.  open: %d, active: %d",
-                    self.open, self.active)
+        logger.info(
+            "Collecting unused comms.  open: %d, active: %d", self.open, self.active
+        )
         for addr, comms in self.available.items():
             for comm in comms:
                 comm.close()
@@ -838,7 +915,7 @@ def coerce_to_address(o):
     return normalize_address(o)
 
 
-def error_message(e, status='error'):
+def error_message(e, status="error"):
     """ Produce message to send back given an exception has occurred
 
     This does the following:
@@ -865,17 +942,14 @@ def error_message(e, status='error'):
     try:
         tb2 = protocol.pickle.dumps(tb)
     except Exception:
-        tb = tb2 = ''.join(traceback.format_tb(tb))
+        tb = tb2 = "".join(traceback.format_tb(tb))
 
     if len(tb2) > 10000:
         tb_result = None
     else:
         tb_result = protocol.to_serialize(tb)
 
-    return {'status': status,
-            'exception': e4,
-            'traceback': tb_result,
-            'text': str(e2)}
+    return {"status": status, "exception": e4, "traceback": tb_result, "text": str(e2)}
 
 
 def clean_exception(exception, traceback, **kwargs):
diff --git a/distributed/counter.py b/distributed/counter.py
index 8d76def7189..d5a3181b112 100644
--- a/distributed/counter.py
+++ b/distributed/counter.py
@@ -12,6 +12,7 @@
 except ImportError:
     pass
 else:
+
     class Digest(object):
         def __init__(self, loop=None, intervals=(5, 60, 3600)):
             self.intervals = intervals
diff --git a/distributed/deploy/__init__.py b/distributed/deploy/__init__.py
index c283cde8394..35abf0a6439 100644
--- a/distributed/deploy/__init__.py
+++ b/distributed/deploy/__init__.py
@@ -5,5 +5,6 @@
 from .cluster import Cluster
 from .local import LocalCluster
 from .adaptive import Adaptive
+
 with ignoring(ImportError):
     from .ssh import SSHCluster
diff --git a/distributed/deploy/adaptive.py b/distributed/deploy/adaptive.py
index 62d308c6e22..890e30c027f 100644
--- a/distributed/deploy/adaptive.py
+++ b/distributed/deploy/adaptive.py
@@ -86,18 +86,30 @@ class Adaptive(object):
     the cluster's ``scale_up`` method.
     '''
 
-    def __init__(self, scheduler, cluster=None, interval='1s', startup_cost='1s',
-                 scale_factor=2, minimum=0, maximum=None, wait_count=3,
-                 target_duration='5s', worker_key=lambda x: x, **kwargs):
-        interval = parse_timedelta(interval, default='ms')
+    def __init__(
+        self,
+        scheduler,
+        cluster=None,
+        interval="1s",
+        startup_cost="1s",
+        scale_factor=2,
+        minimum=0,
+        maximum=None,
+        wait_count=3,
+        target_duration="5s",
+        worker_key=lambda x: x,
+        **kwargs
+    ):
+        interval = parse_timedelta(interval, default="ms")
         self.worker_key = worker_key
         self.scheduler = scheduler
         self.cluster = cluster
-        self.startup_cost = parse_timedelta(startup_cost, default='s')
+        self.startup_cost = parse_timedelta(startup_cost, default="s")
         self.scale_factor = scale_factor
         if self.cluster:
-            self._adapt_callback = PeriodicCallback(self._adapt, interval * 1000,
-                                                    io_loop=scheduler.loop)
+            self._adapt_callback = PeriodicCallback(
+                self._adapt, interval * 1000, io_loop=scheduler.loop
+            )
             self.scheduler.loop.add_callback(self._adapt_callback.start)
         self._adapting = False
         self._workers_to_close_kwargs = kwargs
@@ -108,7 +120,7 @@ def __init__(self, scheduler, cluster=None, interval='1s', startup_cost='1s',
         self.wait_count = wait_count
         self.target_duration = parse_timedelta(target_duration)
 
-        self.scheduler.handlers['adaptive_recommendations'] = self.recommendations
+        self.scheduler.handlers["adaptive_recommendations"] = self.recommendations
 
     def stop(self):
         if self.cluster:
@@ -129,8 +141,11 @@ def needs_cpu(self):
         total_cores = sum([ws.ncores for ws in self.scheduler.workers.values()])
 
         if total_occupancy / (total_cores + 1e-9) > self.startup_cost * 2:
-            logger.info("CPU limit exceeded [%d occupancy / %d cores]",
-                        total_occupancy, total_cores)
+            logger.info(
+                "CPU limit exceeded [%d occupancy / %d cores]",
+                total_occupancy,
+                total_cores,
+            )
             return True
         else:
             return False
@@ -144,8 +159,9 @@ def needs_memory(self):
         Returns ``True`` if  the required bytes in distributed memory is some
         factor larger than the actual distributed memory available.
         """
-        limit_bytes = {addr: ws.memory_limit
-                       for addr, ws in self.scheduler.workers.items()}
+        limit_bytes = {
+            addr: ws.memory_limit for addr, ws in self.scheduler.workers.items()
+        }
         worker_bytes = [ws.nbytes for ws in self.scheduler.workers.values()]
 
         limit = sum(limit_bytes.values())
@@ -221,25 +237,24 @@ def workers_to_close(self, **kwargs):
         kw.update(kwargs)
 
         if self.maximum is not None and len(self.scheduler.workers) > self.maximum:
-            kw['n'] = len(self.scheduler.workers) - self.maximum
+            kw["n"] = len(self.scheduler.workers) - self.maximum
 
         L = self.scheduler.workers_to_close(**kw)
         if len(self.scheduler.workers) - len(L) < self.minimum:
-            L = L[:len(self.scheduler.workers) - self.minimum]
+            L = L[: len(self.scheduler.workers) - self.minimum]
 
         return L
 
     @gen.coroutine
     def _retire_workers(self, workers=None):
         if workers is None:
-            workers = self.workers_to_close(key=self.worker_key,
-                                            minimum=self.minimum)
+            workers = self.workers_to_close(key=self.worker_key, minimum=self.minimum)
         if not workers:
             raise gen.Return(workers)
         with log_errors():
-            yield self.scheduler.retire_workers(workers=workers,
-                                                remove=True,
-                                                close_workers=True)
+            yield self.scheduler.retire_workers(
+                workers=workers, remove=True, close_workers=True
+            )
 
             logger.info("Retiring workers %s", workers)
             f = self.cluster.scale_down(workers)
@@ -263,33 +278,32 @@ def get_scale_up_kwargs(self):
         --------
         LocalCluster.scale_up
         """
-        target = math.ceil(self.scheduler.total_occupancy /
-                           self.target_duration)
-        instances = max(1,
-                        len(self.scheduler.workers) * self.scale_factor,
-                        target,
-                        self.minimum)
+        target = math.ceil(self.scheduler.total_occupancy / self.target_duration)
+        instances = max(
+            1, len(self.scheduler.workers) * self.scale_factor, target, self.minimum
+        )
 
         if self.maximum:
             instances = min(self.maximum, instances)
 
         instances = int(instances)
         logger.info("Scaling up to %d workers", instances)
-        return {'n': instances}
+        return {"n": instances}
 
     def recommendations(self, comm=None):
         should_scale_up = self.should_scale_up()
-        workers = set(self.workers_to_close(key=self.worker_key,
-                                            minimum=self.minimum))
+        workers = set(self.workers_to_close(key=self.worker_key, minimum=self.minimum))
         if should_scale_up and workers:
             logger.info("Attempting to scale up and scale down simultaneously.")
             self.close_counts.clear()
-            return {'status': 'error',
-                    'msg': 'Trying to scale up and down simultaneously'}
+            return {
+                "status": "error",
+                "msg": "Trying to scale up and down simultaneously",
+            }
 
         elif should_scale_up:
             self.close_counts.clear()
-            return toolz.merge({'status': 'up'}, self.get_scale_up_kwargs())
+            return toolz.merge({"status": "up"}, self.get_scale_up_kwargs())
 
         elif workers:
             d = {}
@@ -307,7 +321,7 @@ def recommendations(self, comm=None):
             self.close_counts = d
 
             if to_close:
-                return {'status': 'down', 'workers': to_close}
+                return {"status": "down", "workers": to_close}
         else:
             self.close_counts.clear()
             return None
@@ -322,16 +336,16 @@ def _adapt(self):
             recommendations = self.recommendations()
             if not recommendations:
                 return
-            status = recommendations.pop('status')
-            if status == 'up':
+            status = recommendations.pop("status")
+            if status == "up":
                 f = self.cluster.scale_up(**recommendations)
-                self.log.append((time(), 'up', recommendations))
+                self.log.append((time(), "up", recommendations))
                 if gen.is_future(f):
                     yield f
 
-            elif status == 'down':
-                self.log.append((time(), 'down', recommendations['workers']))
-                workers = yield self._retire_workers(workers=recommendations['workers'])
+            elif status == "down":
+                self.log.append((time(), "down", recommendations["workers"]))
+                workers = yield self._retire_workers(workers=recommendations["workers"])
         finally:
             self._adapting = False
 
diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
index 3bc2b2d9124..f170d4ea5ad 100644
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -51,6 +51,7 @@ def scale_down(self, workers: List[str]):
     --------
     LocalCluster: a simple implementation with local workers
     """
+
     def adapt(self, **kwargs):
         """ Turn on adaptivity
 
@@ -62,7 +63,7 @@ def adapt(self, **kwargs):
         """
         with ignoring(AttributeError):
             self._adaptive.stop()
-        if not hasattr(self, '_adaptive_options'):
+        if not hasattr(self, "_adaptive_options"):
             self._adaptive_options = {}
         self._adaptive_options.update(kwargs)
         self._adaptive = Adaptive(self.scheduler, self, **self._adaptive_options)
@@ -74,9 +75,9 @@ def scheduler_address(self):
 
     @property
     def dashboard_link(self):
-        template = dask.config.get('distributed.dashboard.link')
-        host = self.scheduler.address.split('://')[1].split(':')[0]
-        port = self.scheduler.services['bokeh'].port
+        template = dask.config.get("distributed.dashboard.link")
+        host = self.scheduler.address.split("://")[1].split(":")[0]
+        port = self.scheduler.services["bokeh"].port
         return template.format(host=host, port=port, **os.environ)
 
     def scale(self, n):
@@ -101,9 +102,12 @@ def scale(self, n):
                 self.scheduler.loop.add_callback(self.scale_up, n)
             else:
                 to_close = self.scheduler.workers_to_close(
-                    n=len(self.scheduler.workers) - n)
+                    n=len(self.scheduler.workers) - n
+                )
                 logger.debug("Closing workers: %s", to_close)
-                self.scheduler.loop.add_callback(self.scheduler.retire_workers, workers=to_close)
+                self.scheduler.loop.add_callback(
+                    self.scheduler.retire_workers, workers=to_close
+                )
                 self.scheduler.loop.add_callback(self.scale_down, to_close)
 
     def _widget_status(self):
@@ -132,7 +136,11 @@ def _widget_status(self):
     <tr><th>Memory</th> <td>%s</td></tr>
   </table>
 </div>
-""" % (workers, cores, memory)
+""" % (
+            workers,
+            cores,
+            memory,
+        )
         return text
 
     def _widget(self):
@@ -144,38 +152,39 @@ def _widget(self):
 
         from ipywidgets import Layout, VBox, HBox, IntText, Button, HTML, Accordion
 
-        layout = Layout(width='150px')
+        layout = Layout(width="150px")
 
-        if 'bokeh' in self.scheduler.services:
+        if "bokeh" in self.scheduler.services:
             link = self.dashboard_link
-            link = '<p><b>Dashboard: </b><a href="%s" target="_blank">%s</a></p>\n' % (link, link)
+            link = '<p><b>Dashboard: </b><a href="%s" target="_blank">%s</a></p>\n' % (
+                link,
+                link,
+            )
         else:
-            link = ''
+            link = ""
 
-        title = '<h2>%s</h2>' % type(self).__name__
+        title = "<h2>%s</h2>" % type(self).__name__
         title = HTML(title)
         dashboard = HTML(link)
 
-        status = HTML(self._widget_status(), layout=Layout(min_width='150px'))
+        status = HTML(self._widget_status(), layout=Layout(min_width="150px"))
 
-        request = IntText(0, description='Workers', layout=layout)
-        scale = Button(description='Scale', layout=layout)
+        request = IntText(0, description="Workers", layout=layout)
+        scale = Button(description="Scale", layout=layout)
 
-        minimum = IntText(0, description='Minimum', layout=layout)
-        maximum = IntText(0, description='Maximum', layout=layout)
-        adapt = Button(description='Adapt', layout=layout)
+        minimum = IntText(0, description="Minimum", layout=layout)
+        maximum = IntText(0, description="Maximum", layout=layout)
+        adapt = Button(description="Adapt", layout=layout)
 
-        accordion = Accordion([HBox([request, scale]),
-                               HBox([minimum, maximum, adapt])],
-                               layout=Layout(min_width='500px'))
+        accordion = Accordion(
+            [HBox([request, scale]), HBox([minimum, maximum, adapt])],
+            layout=Layout(min_width="500px"),
+        )
         accordion.selected_index = None
-        accordion.set_title(0, 'Manual Scaling')
-        accordion.set_title(1, 'Adaptive Scaling')
+        accordion.set_title(0, "Manual Scaling")
+        accordion.set_title(1, "Adaptive Scaling")
 
-        box = VBox([title,
-                    HBox([status,
-                          accordion]),
-                    dashboard])
+        box = VBox([title, HBox([status, accordion]), dashboard])
 
         self._cached_widget = box
 
@@ -199,7 +208,7 @@ def update():
             status.value = self._widget_status()
 
         pc = PeriodicCallback(update, 500, io_loop=self.scheduler.loop)
-        self.scheduler.periodic_callbacks['cluster-repr'] = pc
+        self.scheduler.periodic_callbacks["cluster-repr"] = pc
         pc.start()
 
         return box
diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index ececc9fe12c..22a796bbb78 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -14,8 +14,16 @@
 from .cluster import Cluster
 from ..compatibility import get_thread_identity
 from ..core import CommClosedError
-from ..utils import (sync, ignoring, All, silence_logging, LoopRunner,
-        log_errors, thread_state, parse_timedelta)
+from ..utils import (
+    sync,
+    ignoring,
+    All,
+    silence_logging,
+    LoopRunner,
+    log_errors,
+    thread_state,
+    parse_timedelta,
+)
 from ..nanny import Nanny
 from ..scheduler import Scheduler
 from ..worker import Worker, parse_memory_limit, _ncores
@@ -87,18 +95,35 @@ class LocalCluster(Cluster):
 
     >>> LocalCluster(service_kwargs={'bokeh': {'prefix': '/foo'}})  # doctest: +SKIP
     """
-    def __init__(self, n_workers=None, threads_per_worker=None, processes=True,
-                 loop=None, start=None, ip=None, scheduler_port=0,
-                 silence_logs=logging.WARN, dashboard_address=':8787',
-                 diagnostics_port=None,
-                 services=None, worker_services=None, service_kwargs=None,
-                 asynchronous=False, security=None, protocol=None,
-                 blocked_handlers=None, **worker_kwargs):
+
+    def __init__(
+        self,
+        n_workers=None,
+        threads_per_worker=None,
+        processes=True,
+        loop=None,
+        start=None,
+        ip=None,
+        scheduler_port=0,
+        silence_logs=logging.WARN,
+        dashboard_address=":8787",
+        diagnostics_port=None,
+        services=None,
+        worker_services=None,
+        service_kwargs=None,
+        asynchronous=False,
+        security=None,
+        protocol=None,
+        blocked_handlers=None,
+        **worker_kwargs
+    ):
         if start is not None:
-            msg = ("The start= parameter is deprecated. "
-                   "LocalCluster always starts. "
-                   "For asynchronous operation use the following: \n\n"
-                   "  cluster = yield LocalCluster(asynchronous=True)")
+            msg = (
+                "The start= parameter is deprecated. "
+                "LocalCluster always starts. "
+                "For asynchronous operation use the following: \n\n"
+                "  cluster = yield LocalCluster(asynchronous=True)"
+            )
             raise ValueError(msg)
 
         if diagnostics_port is not None:
@@ -112,16 +137,16 @@ def __init__(self, n_workers=None, threads_per_worker=None, processes=True,
         self.processes = processes
 
         if protocol is None:
-            if ip and '://' in ip:
-                protocol = ip.split('://')[0]
+            if ip and "://" in ip:
+                protocol = ip.split("://")[0]
             elif security:
-                protocol = 'tls://'
+                protocol = "tls://"
             elif not self.processes and not scheduler_port:
-                protocol = 'inproc://'
+                protocol = "inproc://"
             else:
-                protocol = 'tcp://'
-        if not protocol.endswith('://'):
-            protocol = protocol + '://'
+                protocol = "tcp://"
+        if not protocol.endswith("://"):
+            protocol = protocol + "://"
         self.protocol = protocol
 
         self.silence_logs = silence_logs
@@ -142,13 +167,12 @@ def __init__(self, n_workers=None, threads_per_worker=None, processes=True,
         if n_workers and threads_per_worker is None:
             # Overcommit threads per worker, rather than undercommit
             threads_per_worker = max(1, int(math.ceil(_ncores / n_workers)))
-        if n_workers and 'memory_limit' not in worker_kwargs:
-            worker_kwargs['memory_limit'] = parse_memory_limit('auto', 1, n_workers)
+        if n_workers and "memory_limit" not in worker_kwargs:
+            worker_kwargs["memory_limit"] = parse_memory_limit("auto", 1, n_workers)
 
-        worker_kwargs.update({
-            'ncores': threads_per_worker,
-            'services': worker_services,
-        })
+        worker_kwargs.update(
+            {"ncores": threads_per_worker, "services": worker_services}
+        )
 
         self._loop_runner = LoopRunner(loop=loop, asynchronous=asynchronous)
         self.loop = self._loop_runner.loop
@@ -160,29 +184,35 @@ def __init__(self, n_workers=None, threads_per_worker=None, processes=True,
             except ImportError:
                 logger.debug("To start diagnostics web server please install Bokeh")
             else:
-                services[('bokeh', dashboard_address)] = (BokehScheduler, (service_kwargs or {}).get('bokeh', {}))
-                worker_services[('bokeh', 0)] = BokehWorker
+                services[("bokeh", dashboard_address)] = (
+                    BokehScheduler,
+                    (service_kwargs or {}).get("bokeh", {}),
+                )
+                worker_services[("bokeh", 0)] = BokehWorker
 
-        self.scheduler = Scheduler(loop=self.loop,
-                                   services=services,
-                                   security=security,
-                                   blocked_handlers=blocked_handlers)
+        self.scheduler = Scheduler(
+            loop=self.loop,
+            services=services,
+            security=security,
+            blocked_handlers=blocked_handlers,
+        )
         self.scheduler_port = scheduler_port
 
         self.workers = []
         self.worker_kwargs = worker_kwargs
         if security:
-            self.worker_kwargs['security'] = security
+            self.worker_kwargs["security"] = security
 
         self.start(ip=ip, n_workers=n_workers)
 
         clusters_to_close.add(self)
 
     def __repr__(self):
-        return ('LocalCluster(%r, workers=%d, ncores=%d)' %
-                (self.scheduler_address, len(self.workers),
-                 sum(w.ncores for w in self.workers))
-                )
+        return "LocalCluster(%r, workers=%d, ncores=%d)" % (
+            self.scheduler_address,
+            len(self.workers),
+            sum(w.ncores for w in self.workers),
+        )
 
     def __await__(self):
         return self._started.__await__()
@@ -190,18 +220,18 @@ def __await__(self):
     @property
     def asynchronous(self):
         return (
-            self._asynchronous or
-            getattr(thread_state, 'asynchronous', False) or
-            hasattr(self.loop, '_thread_identity') and self.loop._thread_identity == get_thread_identity()
+            self._asynchronous
+            or getattr(thread_state, "asynchronous", False)
+            or hasattr(self.loop, "_thread_identity")
+            and self.loop._thread_identity == get_thread_identity()
         )
 
     def sync(self, func, *args, **kwargs):
-        if kwargs.pop('asynchronous', None) or self.asynchronous:
-            callback_timeout = kwargs.pop('callback_timeout', None)
+        if kwargs.pop("asynchronous", None) or self.asynchronous:
+            callback_timeout = kwargs.pop("callback_timeout", None)
             future = func(*args, **kwargs)
             if callback_timeout is not None:
-                future = gen.with_timeout(timedelta(seconds=callback_timeout),
-                                          future)
+                future = gen.with_timeout(timedelta(seconds=callback_timeout), future)
             return future
         else:
             return sync(self.loop, func, *args, **kwargs)
@@ -218,52 +248,56 @@ def _start(self, ip=None, n_workers=0):
         """
         Start all cluster services.
         """
-        if self.status == 'running':
+        if self.status == "running":
             return
 
-        if self.protocol == 'inproc://':
+        if self.protocol == "inproc://":
             address = self.protocol
         else:
             if ip is None:
-                ip = '127.0.0.1'
+                ip = "127.0.0.1"
 
-            if '://' in ip:
+            if "://" in ip:
                 address = ip
             else:
                 address = self.protocol + ip
             if self.scheduler_port:
-                address += ':' + str(self.scheduler_port)
+                address += ":" + str(self.scheduler_port)
 
         self.scheduler.start(address)
 
         yield [self._start_worker(**self.worker_kwargs) for i in range(n_workers)]
 
-        self.status = 'running'
+        self.status = "running"
 
         raise gen.Return(self)
 
     @gen.coroutine
     def _start_worker(self, death_timeout=60, **kwargs):
-        if self.status and self.status.startswith('clos'):
+        if self.status and self.status.startswith("clos"):
             warnings.warn("Tried to start a worker while status=='%s'" % self.status)
             return
 
         if self.processes:
             W = Nanny
-            kwargs['quiet'] = True
+            kwargs["quiet"] = True
         else:
             W = Worker
 
-        w = yield W(self.scheduler.address, loop=self.loop,
-              death_timeout=death_timeout,
-              silence_logs=self.silence_logs, **kwargs)
+        w = yield W(
+            self.scheduler.address,
+            loop=self.loop,
+            death_timeout=death_timeout,
+            silence_logs=self.silence_logs,
+            **kwargs
+        )
 
         self.workers.append(w)
 
-        while w.status != 'closed' and w.worker_address not in self.scheduler.workers:
+        while w.status != "closed" and w.worker_address not in self.scheduler.workers:
             yield gen.sleep(0.01)
 
-        if w.status == 'closed' and self.scheduler.status == 'running':
+        if w.status == "closed" and self.scheduler.status == "running":
             self.workers.remove(w)
             raise gen.TimeoutError("Worker failed to start")
 
@@ -308,11 +342,11 @@ def stop_worker(self, w):
         self.sync(self._stop_worker, w)
 
     @gen.coroutine
-    def _close(self, timeout='2s'):
+    def _close(self, timeout="2s"):
         # Can be 'closing' as we're called by close() below
-        if self.status == 'closed':
+        if self.status == "closed":
             return
-        self.status = 'closing'
+        self.status = "closing"
 
         self.scheduler.clear_task_state()
 
@@ -328,11 +362,11 @@ def _close(self, timeout='2s'):
                 yield self.scheduler.close(fast=True)
             del self.workers[:]
         finally:
-            self.status = 'closed'
+            self.status = "closed"
 
     def close(self, timeout=20):
         """ Close the cluster """
-        if self.status == 'closed':
+        if self.status == "closed":
             return
 
         try:
@@ -340,9 +374,11 @@ def close(self, timeout=20):
         except RuntimeError:  # IOLoop is closed
             result = None
 
-        if hasattr(self, '_old_logging_level'):
+        if hasattr(self, "_old_logging_level"):
             if self.asynchronous:
-                result.add_done_callback(lambda _: silence_logging(self._old_logging_level))
+                result.add_done_callback(
+                    lambda _: silence_logging(self._old_logging_level)
+                )
             else:
                 silence_logging(self._old_logging_level)
 
@@ -362,11 +398,13 @@ def scale_up(self, n, **kwargs):
         """
         with log_errors():
             kwargs2 = toolz.merge(self.worker_kwargs, kwargs)
-            yield [self._start_worker(**kwargs2)
-                   for i in range(n - len(self.scheduler.workers))]
+            yield [
+                self._start_worker(**kwargs2)
+                for i in range(n - len(self.scheduler.workers))
+            ]
 
             # clean up any closed worker
-            self.workers = [w for w in self.workers if w.status != 'closed']
+            self.workers = [w for w in self.workers if w.status != "closed"]
 
     @gen.coroutine
     def scale_down(self, workers):
@@ -380,7 +418,7 @@ def scale_down(self, workers):
         """
         with log_errors():
             # clean up any closed worker
-            self.workers = [w for w in self.workers if w.status != 'closed']
+            self.workers = [w for w in self.workers if w.status != "closed"]
             workers = set(workers)
 
             # we might be given addresses
@@ -413,7 +451,7 @@ def scheduler_address(self):
         try:
             return self.scheduler.address
         except ValueError:
-            return '<unstarted>'
+            return "<unstarted>"
 
 
 def nprocesses_nthreads(n):
diff --git a/distributed/deploy/ssh.py b/distributed/deploy/ssh.py
index 20888471ff8..ba8ed01d1c7 100644
--- a/distributed/deploy/ssh.py
+++ b/distributed/deploy/ssh.py
@@ -25,20 +25,21 @@
 # These are handy for creating colorful terminal output to enhance readability
 # of the output generated by dask-ssh.
 class bcolors:
-    HEADER = '\033[95m'
-    OKBLUE = '\033[94m'
-    OKGREEN = '\033[92m'
-    WARNING = '\033[93m'
-    FAIL = '\033[91m'
-    ENDC = '\033[0m'
-    BOLD = '\033[1m'
-    UNDERLINE = '\033[4m'
+    HEADER = "\033[95m"
+    OKBLUE = "\033[94m"
+    OKGREEN = "\033[92m"
+    WARNING = "\033[93m"
+    FAIL = "\033[91m"
+    ENDC = "\033[0m"
+    BOLD = "\033[1m"
+    UNDERLINE = "\033[4m"
 
 
 def async_ssh(cmd_dict):
     import paramiko
     from paramiko.buffered_pipe import PipeTimeout
-    from paramiko.ssh_exception import (SSHException, PasswordRequiredException)
+    from paramiko.ssh_exception import SSHException, PasswordRequiredException
+
     ssh = paramiko.SSHClient()
     ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
 
@@ -46,28 +47,40 @@ def async_ssh(cmd_dict):
     while True:  # Be robust to transient SSH failures.
         try:
             # Set paramiko logging to WARN or higher to squelch INFO messages.
-            logging.getLogger('paramiko').setLevel(logging.WARN)
-
-            ssh.connect(hostname=cmd_dict['address'],
-                        username=cmd_dict['ssh_username'],
-                        port=cmd_dict['ssh_port'],
-                        key_filename=cmd_dict['ssh_private_key'],
-                        compress=True,
-                        timeout=20,
-                        banner_timeout=20)  # Helps prevent timeouts when many concurrent ssh connections are opened.
+            logging.getLogger("paramiko").setLevel(logging.WARN)
+
+            ssh.connect(
+                hostname=cmd_dict["address"],
+                username=cmd_dict["ssh_username"],
+                port=cmd_dict["ssh_port"],
+                key_filename=cmd_dict["ssh_private_key"],
+                compress=True,
+                timeout=20,
+                banner_timeout=20,
+            )  # Helps prevent timeouts when many concurrent ssh connections are opened.
             # Connection successful, break out of while loop
             break
 
-        except (SSHException,
-                PasswordRequiredException) as e:
-
-            print('[ dask-ssh ] : ' + bcolors.FAIL +
-                  'SSH connection error when connecting to {addr}:{port}'
-                  'to run \'{cmd}\''.format(addr=cmd_dict['address'],
-                                            port=cmd_dict['ssh_port'],
-                                            cmd=cmd_dict['cmd']) + bcolors.ENDC)
-
-            print(bcolors.FAIL + '               SSH reported this exception: ' + str(e) + bcolors.ENDC)
+        except (SSHException, PasswordRequiredException) as e:
+
+            print(
+                "[ dask-ssh ] : "
+                + bcolors.FAIL
+                + "SSH connection error when connecting to {addr}:{port}"
+                "to run '{cmd}'".format(
+                    addr=cmd_dict["address"],
+                    port=cmd_dict["ssh_port"],
+                    cmd=cmd_dict["cmd"],
+                )
+                + bcolors.ENDC
+            )
+
+            print(
+                bcolors.FAIL
+                + "               SSH reported this exception: "
+                + str(e)
+                + bcolors.ENDC
+            )
 
             # Print an exception traceback
             traceback.print_exc()
@@ -77,18 +90,23 @@ def async_ssh(cmd_dict):
             # attempts to retry.
             retries += 1
             if retries >= 3:
-                print('[ dask-ssh ] : '
-                      + bcolors.FAIL
-                      + 'SSH connection failed after 3 retries. Exiting.'
-                      + bcolors.ENDC)
+                print(
+                    "[ dask-ssh ] : "
+                    + bcolors.FAIL
+                    + "SSH connection failed after 3 retries. Exiting."
+                    + bcolors.ENDC
+                )
 
                 # Connection failed after multiple attempts.  Terminate this thread.
                 os._exit(1)
 
             # Wait a moment before retrying
-            print('               ' + bcolors.FAIL +
-                  'Retrying... (attempt {n}/{total})'.format(n=retries, total=3) +
-                  bcolors.ENDC)
+            print(
+                "               "
+                + bcolors.FAIL
+                + "Retrying... (attempt {n}/{total})".format(n=retries, total=3)
+                + bcolors.ENDC
+            )
 
             time.sleep(1)
 
@@ -99,9 +117,10 @@ def async_ssh(cmd_dict):
     # before the command is run. This should help to ensure that important
     # aspects of the environment like PATH and PYTHONPATH are configured.
 
-    print('[ {label} ] : {cmd}'.format(label=cmd_dict['label'],
-                                       cmd=cmd_dict['cmd']))
-    stdin, stdout, stderr = ssh.exec_command('$SHELL -i -c \'' + cmd_dict['cmd'] + '\'', get_pty=True)
+    print("[ {label} ] : {cmd}".format(label=cmd_dict["label"], cmd=cmd_dict["cmd"]))
+    stdin, stdout, stderr = ssh.exec_command(
+        "$SHELL -i -c '" + cmd_dict["cmd"] + "'", get_pty=True
+    )
 
     # Set up channel timeout (which we rely on below to make readline() non-blocking)
     channel = stdout.channel
@@ -113,11 +132,14 @@ def read_from_stdout():
         """
         try:
             line = stdout.readline()
-            while len(line) > 0:    # Loops until a timeout exception occurs
+            while len(line) > 0:  # Loops until a timeout exception occurs
                 line = line.rstrip()
-                logger.debug('stdout from ssh channel: %s', line)
-                cmd_dict['output_queue'].put('[ {label} ] : {output}'.format(label=cmd_dict['label'],
-                                                                             output=line))
+                logger.debug("stdout from ssh channel: %s", line)
+                cmd_dict["output_queue"].put(
+                    "[ {label} ] : {output}".format(
+                        label=cmd_dict["label"], output=line
+                    )
+                )
                 line = stdout.readline()
         except (PipeTimeout, socket.timeout):
             pass
@@ -130,9 +152,13 @@ def read_from_stderr():
             line = stderr.readline()
             while len(line) > 0:
                 line = line.rstrip()
-                logger.debug('stderr from ssh channel: %s', line)
-                cmd_dict['output_queue'].put('[ {label} ] : '.format(label=cmd_dict['label']) +
-                                             bcolors.FAIL + '{output}'.format(output=line) + bcolors.ENDC)
+                logger.debug("stderr from ssh channel: %s", line)
+                cmd_dict["output_queue"].put(
+                    "[ {label} ] : ".format(label=cmd_dict["label"])
+                    + bcolors.FAIL
+                    + "{output}".format(output=line)
+                    + bcolors.ENDC
+                )
                 line = stderr.readline()
         except (PipeTimeout, socket.timeout):
             pass
@@ -149,15 +175,18 @@ def communicate():
         # terminate.
         if channel.exit_status_ready():
             exit_status = channel.recv_exit_status()
-            cmd_dict['output_queue'].put('[ {label} ] : '.format(label=cmd_dict['label']) +
-                                         bcolors.FAIL +
-                                         "remote process exited with exit status " +
-                                         str(exit_status) + bcolors.ENDC)
+            cmd_dict["output_queue"].put(
+                "[ {label} ] : ".format(label=cmd_dict["label"])
+                + bcolors.FAIL
+                + "remote process exited with exit status "
+                + str(exit_status)
+                + bcolors.ENDC
+            )
             return True
 
     # Wait for a message on the input_queue. Any message received signals this
     # thread to shut itself down.
-    while cmd_dict['input_queue'].empty():
+    while cmd_dict["input_queue"].empty():
         # Kill some time so that this thread does not hog the CPU.
         time.sleep(1.0)
         if communicate():
@@ -166,7 +195,7 @@ def communicate():
     # Ctrl-C the executing command and wait a bit for command to end cleanly
     start = time.time()
     while time.time() < start + 5.0:
-        channel.send(b'\x03')  # Ctrl-C
+        channel.send(b"\x03")  # Ctrl-C
         if communicate():
             break
         time.sleep(1.0)
@@ -176,63 +205,87 @@ def communicate():
     ssh.close()
 
 
-def start_scheduler(logdir, addr, port, ssh_username, ssh_port, ssh_private_key, remote_python=None):
-    cmd = '{python} -m distributed.cli.dask_scheduler --port {port}'.format(
-        python=remote_python or sys.executable, port=port, logdir=logdir)
+def start_scheduler(
+    logdir, addr, port, ssh_username, ssh_port, ssh_private_key, remote_python=None
+):
+    cmd = "{python} -m distributed.cli.dask_scheduler --port {port}".format(
+        python=remote_python or sys.executable, port=port, logdir=logdir
+    )
 
     # Optionally re-direct stdout and stderr to a logfile
     if logdir is not None:
-        cmd = 'mkdir -p {logdir} && '.format(logdir=logdir) + cmd
-        cmd += '&> {logdir}/dask_scheduler_{addr}:{port}.log'.format(addr=addr,
-                                                                     port=port, logdir=logdir)
+        cmd = "mkdir -p {logdir} && ".format(logdir=logdir) + cmd
+        cmd += "&> {logdir}/dask_scheduler_{addr}:{port}.log".format(
+            addr=addr, port=port, logdir=logdir
+        )
 
     # Format output labels we can prepend to each line of output, and create
     # a 'status' key to keep track of jobs that terminate prematurely.
-    label = (bcolors.BOLD +
-             'scheduler {addr}:{port}'.format(addr=addr, port=port) +
-             bcolors.ENDC)
+    label = (
+        bcolors.BOLD
+        + "scheduler {addr}:{port}".format(addr=addr, port=port)
+        + bcolors.ENDC
+    )
 
     # Create a command dictionary, which contains everything we need to run and
     # interact with this command.
     input_queue = Queue()
     output_queue = Queue()
-    cmd_dict = {'cmd': cmd, 'label': label, 'address': addr, 'port': port,
-                'input_queue': input_queue, 'output_queue': output_queue,
-                'ssh_username': ssh_username, 'ssh_port': ssh_port,
-                'ssh_private_key': ssh_private_key}
+    cmd_dict = {
+        "cmd": cmd,
+        "label": label,
+        "address": addr,
+        "port": port,
+        "input_queue": input_queue,
+        "output_queue": output_queue,
+        "ssh_username": ssh_username,
+        "ssh_port": ssh_port,
+        "ssh_private_key": ssh_private_key,
+    }
 
     # Start the thread
     thread = Thread(target=async_ssh, args=[cmd_dict])
     thread.daemon = True
     thread.start()
 
-    return merge(cmd_dict, {'thread': thread})
-
-
-def start_worker(logdir, scheduler_addr, scheduler_port, worker_addr, nthreads, nprocs,
-                 ssh_username, ssh_port, ssh_private_key, nohost,
-                 memory_limit,
-                 worker_port,
-                 nanny_port,
-                 remote_python=None,
-                 remote_dask_worker='distributed.cli.dask_worker'):
-
-    cmd = ('{python} -m {remote_dask_worker} '
-           '{scheduler_addr}:{scheduler_port} '
-           '--nthreads {nthreads}'
-           + (' --nprocs {nprocs}' if nprocs != 1 else ''))
+    return merge(cmd_dict, {"thread": thread})
+
+
+def start_worker(
+    logdir,
+    scheduler_addr,
+    scheduler_port,
+    worker_addr,
+    nthreads,
+    nprocs,
+    ssh_username,
+    ssh_port,
+    ssh_private_key,
+    nohost,
+    memory_limit,
+    worker_port,
+    nanny_port,
+    remote_python=None,
+    remote_dask_worker="distributed.cli.dask_worker",
+):
+
+    cmd = (
+        "{python} -m {remote_dask_worker} "
+        "{scheduler_addr}:{scheduler_port} "
+        "--nthreads {nthreads}" + (" --nprocs {nprocs}" if nprocs != 1 else "")
+    )
 
     if not nohost:
-        cmd += ' --host {worker_addr}'
+        cmd += " --host {worker_addr}"
 
     if memory_limit:
-        cmd += ' --memory-limit {memory_limit}'
+        cmd += " --memory-limit {memory_limit}"
 
     if worker_port:
-        cmd += ' --worker-port {worker_port}'
+        cmd += " --worker-port {worker_port}"
 
     if nanny_port:
-        cmd += ' --nanny-port {nanny_port}'
+        cmd += " --nanny-port {nanny_port}"
 
     cmd = cmd.format(
         python=remote_python or sys.executable,
@@ -244,40 +297,60 @@ def start_worker(logdir, scheduler_addr, scheduler_port, worker_addr, nthreads,
         nprocs=nprocs,
         memory_limit=memory_limit,
         worker_port=worker_port,
-        nanny_port=nanny_port)
+        nanny_port=nanny_port,
+    )
 
     # Optionally redirect stdout and stderr to a logfile
     if logdir is not None:
-        cmd = 'mkdir -p {logdir} && '.format(logdir=logdir) + cmd
-        cmd += '&> {logdir}/dask_scheduler_{addr}.log'.format(
-            addr=worker_addr, logdir=logdir)
+        cmd = "mkdir -p {logdir} && ".format(logdir=logdir) + cmd
+        cmd += "&> {logdir}/dask_scheduler_{addr}.log".format(
+            addr=worker_addr, logdir=logdir
+        )
 
-    label = 'worker {addr}'.format(addr=worker_addr)
+    label = "worker {addr}".format(addr=worker_addr)
 
     # Create a command dictionary, which contains everything we need to run and
     # interact with this command.
     input_queue = Queue()
     output_queue = Queue()
-    cmd_dict = {'cmd': cmd, 'label': label, 'address': worker_addr,
-                'input_queue': input_queue, 'output_queue': output_queue,
-                'ssh_username': ssh_username, 'ssh_port': ssh_port,
-                'ssh_private_key': ssh_private_key}
+    cmd_dict = {
+        "cmd": cmd,
+        "label": label,
+        "address": worker_addr,
+        "input_queue": input_queue,
+        "output_queue": output_queue,
+        "ssh_username": ssh_username,
+        "ssh_port": ssh_port,
+        "ssh_private_key": ssh_private_key,
+    }
 
     # Start the thread
     thread = Thread(target=async_ssh, args=[cmd_dict])
     thread.daemon = True
     thread.start()
 
-    return merge(cmd_dict, {'thread': thread})
+    return merge(cmd_dict, {"thread": thread})
 
 
 class SSHCluster(object):
-
-    def __init__(self, scheduler_addr, scheduler_port, worker_addrs, nthreads=0, nprocs=1,
-                 ssh_username=None, ssh_port=22, ssh_private_key=None,
-                 nohost=False, logdir=None, remote_python=None,
-                 memory_limit=None, worker_port=None, nanny_port=None,
-                 remote_dask_worker='distributed.cli.dask_worker'):
+    def __init__(
+        self,
+        scheduler_addr,
+        scheduler_port,
+        worker_addrs,
+        nthreads=0,
+        nprocs=1,
+        ssh_username=None,
+        ssh_port=22,
+        ssh_private_key=None,
+        nohost=False,
+        logdir=None,
+        remote_python=None,
+        memory_limit=None,
+        worker_port=None,
+        nanny_port=None,
+        remote_dask_worker="distributed.cli.dask_worker",
+    ):
 
         self.scheduler_addr = scheduler_addr
         self.scheduler_port = scheduler_port
@@ -299,20 +372,34 @@ def __init__(self, scheduler_addr, scheduler_port, worker_addrs, nthreads=0, npr
 
         # Generate a universal timestamp to use for log files
         import datetime
+
         if logdir is not None:
-            logdir = os.path.join(logdir, "dask-ssh_" + datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S"))
-            print(bcolors.WARNING + 'Output will be redirected to logfiles '
-                  'stored locally on individual worker nodes under "{logdir}".'.format(logdir=logdir)
-                  + bcolors.ENDC)
+            logdir = os.path.join(
+                logdir,
+                "dask-ssh_" + datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S"),
+            )
+            print(
+                bcolors.WARNING + "Output will be redirected to logfiles "
+                'stored locally on individual worker nodes under "{logdir}".'.format(
+                    logdir=logdir
+                )
+                + bcolors.ENDC
+            )
         self.logdir = logdir
 
         # Keep track of all running threads
         self.threads = []
 
         # Start the scheduler node
-        self.scheduler = start_scheduler(logdir, scheduler_addr,
-                                         scheduler_port, ssh_username, ssh_port,
-                                         ssh_private_key, remote_python)
+        self.scheduler = start_scheduler(
+            logdir,
+            scheduler_addr,
+            scheduler_port,
+            ssh_username,
+            ssh_port,
+            ssh_private_key,
+            remote_python,
+        )
 
         # Start worker nodes
         self.workers = []
@@ -325,7 +412,7 @@ def _start(self):
 
     @property
     def scheduler_address(self):
-        return '%s:%d' % (self.scheduler_addr, self.scheduler_port)
+        return "%s:%d" % (self.scheduler_addr, self.scheduler_port)
 
     def monitor_remote_processes(self):
 
@@ -335,8 +422,8 @@ def monitor_remote_processes(self):
         try:
             while True:
                 for process in all_processes:
-                    while not process['output_queue'].empty():
-                        print(process['output_queue'].get())
+                    while not process["output_queue"].empty():
+                        print(process["output_queue"].get())
 
                 # Kill some time and free up CPU before starting the next sweep
                 # through the processes.
@@ -345,26 +432,35 @@ def monitor_remote_processes(self):
             # end while true
 
         except KeyboardInterrupt:
-            pass   # Return execution to the calling process
+            pass  # Return execution to the calling process
 
     def add_worker(self, address):
-        self.workers.append(start_worker(self.logdir, self.scheduler_addr,
-                                         self.scheduler_port, address,
-                                         self.nthreads, self.nprocs,
-                                         self.ssh_username, self.ssh_port,
-                                         self.ssh_private_key, self.nohost,
-                                         self.memory_limit,
-                                         self.worker_port,
-                                         self.nanny_port,
-                                         self.remote_python,
-                                         self.remote_dask_worker))
+        self.workers.append(
+            start_worker(
+                self.logdir,
+                self.scheduler_addr,
+                self.scheduler_port,
+                address,
+                self.nthreads,
+                self.nprocs,
+                self.ssh_username,
+                self.ssh_port,
+                self.ssh_private_key,
+                self.nohost,
+                self.memory_limit,
+                self.worker_port,
+                self.nanny_port,
+                self.remote_python,
+                self.remote_dask_worker,
+            )
+        )
 
     def shutdown(self):
         all_processes = [self.scheduler] + self.workers
 
         for process in all_processes:
-            process['input_queue'].put('shutdown')
-            process['thread'].join()
+            process["input_queue"].put("shutdown")
+            process["thread"].join()
 
     def __enter__(self):
         return self
diff --git a/distributed/deploy/tests/test_adaptive.py b/distributed/deploy/tests/test_adaptive.py
index dca8653a752..1d8a48bf7fc 100644
--- a/distributed/deploy/tests/test_adaptive.py
+++ b/distributed/deploy/tests/test_adaptive.py
@@ -14,21 +14,21 @@
 
 
 def test_get_scale_up_kwargs(loop):
-    with LocalCluster(0, scheduler_port=0, silence_logs=False,
-                      dashboard_address=None, loop=loop) as cluster:
+    with LocalCluster(
+        0, scheduler_port=0, silence_logs=False, dashboard_address=None, loop=loop
+    ) as cluster:
 
-        alc = Adaptive(cluster.scheduler, cluster, interval=100,
-                       scale_factor=3)
-        assert alc.get_scale_up_kwargs() == {'n': 1}
+        alc = Adaptive(cluster.scheduler, cluster, interval=100, scale_factor=3)
+        assert alc.get_scale_up_kwargs() == {"n": 1}
 
         with Client(cluster, loop=loop) as c:
             future = c.submit(lambda x: x + 1, 1)
             assert future.result() == 2
             assert c.ncores()
-            assert alc.get_scale_up_kwargs() == {'n': 3}
+            assert alc.get_scale_up_kwargs() == {"n": 3}
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 4)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 4)
 def test_simultaneous_scale_up_and_down(c, s, *workers):
     class TestAdaptive(Adaptive):
         def get_scale_up_kwargs(self):
@@ -46,11 +46,11 @@ def scale_down(self, workers):
 
     cluster = TestCluster()
 
-    s.task_duration['a'] = 4
-    s.task_duration['b'] = 4
-    s.task_duration['c'] = 1
+    s.task_duration["a"] = 4
+    s.task_duration["b"] = 4
+    s.task_duration["c"] = 1
 
-    future = c.map(slowinc, [1, 1, 1], key=['a-4', 'b-4', 'c-1'])
+    future = c.map(slowinc, [1, 1, 1], key=["a-4", "b-4", "c-1"])
 
     while len(s.rprocessing) < 3:
         yield gen.sleep(0.001)
@@ -61,8 +61,9 @@ def scale_down(self, workers):
 
 
 def test_adaptive_local_cluster(loop):
-    with LocalCluster(0, scheduler_port=0, silence_logs=False,
-                      dashboard_address=None, loop=loop) as cluster:
+    with LocalCluster(
+        0, scheduler_port=0, silence_logs=False, dashboard_address=None, loop=loop
+    ) as cluster:
         alc = Adaptive(cluster.scheduler, cluster, interval=100)
         with Client(cluster, loop=loop) as c:
             assert not c.ncores()
@@ -86,9 +87,14 @@ def test_adaptive_local_cluster(loop):
 @nodebug
 @gen_test(timeout=30)
 def test_adaptive_local_cluster_multi_workers():
-    cluster = yield LocalCluster(0, scheduler_port=0, silence_logs=False,
-                                 processes=False, dashboard_address=None,
-                                 asynchronous=True)
+    cluster = yield LocalCluster(
+        0,
+        scheduler_port=0,
+        silence_logs=False,
+        processes=False,
+        dashboard_address=None,
+        asynchronous=True,
+    )
     try:
         cluster.scheduler.allowed_failures = 1000
         alc = cluster.adapt(interval=100)
@@ -124,7 +130,7 @@ def test_adaptive_local_cluster_multi_workers():
         yield cluster.close()
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 10, active_rpc_timeout=10)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 10, active_rpc_timeout=10)
 def test_adaptive_scale_down_override(c, s, *workers):
     class TestAdaptive(Adaptive):
         def __init__(self, *args, **kwargs):
@@ -135,7 +141,7 @@ def workers_to_close(self, **kwargs):
             num_workers = len(self.scheduler.workers)
             to_close = self.scheduler.workers_to_close(**kwargs)
             if num_workers - len(to_close) < self.min_size:
-                to_close = to_close[:num_workers - self.min_size]
+                to_close = to_close[: num_workers - self.min_size]
 
             return to_close
 
@@ -151,7 +157,7 @@ def scale_down(self, workers):
     # Assert that adaptive cycle does not reduce cluster below minimum size
     # as determined via override.
     cluster = TestCluster()
-    ta = TestAdaptive(s, cluster, min_size=2, interval=.1, scale_factor=2)
+    ta = TestAdaptive(s, cluster, min_size=2, interval=0.1, scale_factor=2)
     yield gen.sleep(0.3)
 
     assert len(s.workers) == 2
@@ -160,13 +166,25 @@ def scale_down(self, workers):
 @gen_test(timeout=30)
 def test_min_max():
     loop = IOLoop.current()
-    cluster = yield LocalCluster(0, scheduler_port=0, silence_logs=False,
-                                 processes=False, dashboard_address=None,
-                                 loop=loop, asynchronous=True)
+    cluster = yield LocalCluster(
+        0,
+        scheduler_port=0,
+        silence_logs=False,
+        processes=False,
+        dashboard_address=None,
+        loop=loop,
+        asynchronous=True,
+    )
     yield cluster._start()
     try:
-        adapt = Adaptive(cluster.scheduler, cluster, minimum=1, maximum=2,
-                         interval='20 ms', wait_count=10)
+        adapt = Adaptive(
+            cluster.scheduler,
+            cluster,
+            minimum=1,
+            maximum=2,
+            interval="20 ms",
+            wait_count=10,
+        )
         c = yield Client(cluster, asynchronous=True, loop=loop)
 
         start = time()
@@ -176,7 +194,7 @@ def test_min_max():
 
         yield gen.sleep(0.2)
         assert len(cluster.scheduler.workers) == 1
-        assert frequencies(pluck(1, adapt.log)) == {'up': 1}
+        assert frequencies(pluck(1, adapt.log)) == {"up": 1}
 
         futures = c.map(slowinc, range(100), delay=0.1)
 
@@ -189,7 +207,7 @@ def test_min_max():
         yield gen.sleep(0.5)
         assert len(cluster.scheduler.workers) == 2
         assert len(cluster.workers) == 2
-        assert frequencies(pluck(1, adapt.log)) == {'up': 2}
+        assert frequencies(pluck(1, adapt.log)) == {"up": 2}
 
         del futures
 
@@ -197,7 +215,7 @@ def test_min_max():
         while len(cluster.scheduler.workers) != 1:
             yield gen.sleep(0.01)
             assert time() < start + 2
-        assert frequencies(pluck(1, adapt.log)) == {'up': 2, 'down': 1}
+        assert frequencies(pluck(1, adapt.log)) == {"up": 2, "down": 1}
     finally:
         yield c.close()
         yield cluster.close()
@@ -210,18 +228,23 @@ def test_avoid_churn():
     Instead we want to wait a few beats before removing a worker in case the
     user is taking a brief pause between work
     """
-    cluster = yield LocalCluster(0, asynchronous=True, processes=False,
-                                 scheduler_port=0, silence_logs=False,
-                                 dashboard_address=None)
+    cluster = yield LocalCluster(
+        0,
+        asynchronous=True,
+        processes=False,
+        scheduler_port=0,
+        silence_logs=False,
+        dashboard_address=None,
+    )
     client = yield Client(cluster, asynchronous=True)
     try:
-        adapt = Adaptive(cluster.scheduler, cluster, interval='20 ms', wait_count=5)
+        adapt = Adaptive(cluster.scheduler, cluster, interval="20 ms", wait_count=5)
 
         for i in range(10):
             yield client.submit(slowinc, i, delay=0.040)
             yield gen.sleep(0.040)
 
-        assert frequencies(pluck(1, adapt.log)) == {'up': 1}
+        assert frequencies(pluck(1, adapt.log)) == {"up": 1}
     finally:
         yield client.close()
         yield cluster.close()
@@ -234,12 +257,16 @@ def test_adapt_quickly():
     Instead we want to wait a few beats before removing a worker in case the
     user is taking a brief pause between work
     """
-    cluster = yield LocalCluster(0, asynchronous=True, processes=False,
-                                 scheduler_port=0, silence_logs=False,
-                                 dashboard_address=None)
+    cluster = yield LocalCluster(
+        0,
+        asynchronous=True,
+        processes=False,
+        scheduler_port=0,
+        silence_logs=False,
+        dashboard_address=None,
+    )
     client = yield Client(cluster, asynchronous=True)
-    adapt = Adaptive(cluster.scheduler, cluster, interval=20, wait_count=5,
-                     maximum=10)
+    adapt = Adaptive(cluster.scheduler, cluster, interval=20, wait_count=5, maximum=10)
     try:
         future = client.submit(slowinc, 1, delay=0.100)
         yield wait(future)
@@ -247,12 +274,12 @@ def test_adapt_quickly():
 
         # Scale up when there is plenty of available work
         futures = client.map(slowinc, range(1000), delay=0.100)
-        while frequencies(pluck(1, adapt.log)) == {'up': 1}:
+        while frequencies(pluck(1, adapt.log)) == {"up": 1}:
             yield gen.sleep(0.01)
         assert len(adapt.log) == 2
-        assert 'up' in adapt.log[-1]
+        assert "up" in adapt.log[-1]
         d = [x for x in adapt.log[-1] if isinstance(x, dict)][0]
-        assert 2 < d['n'] <= adapt.maximum
+        assert 2 < d["n"] <= adapt.maximum
 
         while len(cluster.scheduler.workers) < adapt.maximum:
             yield gen.sleep(0.01)
@@ -277,11 +304,16 @@ def test_adapt_quickly():
 @gen_test(timeout=None)
 def test_adapt_down():
     """ Ensure that redefining adapt with a lower maximum removes workers """
-    cluster = yield LocalCluster(0, asynchronous=True, processes=False,
-                                 scheduler_port=0, silence_logs=False,
-                                 dashboard_address=None)
+    cluster = yield LocalCluster(
+        0,
+        asynchronous=True,
+        processes=False,
+        scheduler_port=0,
+        silence_logs=False,
+        dashboard_address=None,
+    )
     client = yield Client(cluster, asynchronous=True)
-    cluster.adapt(interval='20ms', maximum=5)
+    cluster.adapt(interval="20ms", maximum=5)
 
     try:
         futures = client.map(slowinc, range(1000), delay=0.1)
@@ -303,15 +335,22 @@ def test_adapt_down():
 @gen_test(timeout=30)
 def test_no_more_workers_than_tasks():
     loop = IOLoop.current()
-    cluster = yield LocalCluster(0, scheduler_port=0, silence_logs=False,
-                                 processes=False, dashboard_address=None,
-                                 loop=loop, asynchronous=True)
+    cluster = yield LocalCluster(
+        0,
+        scheduler_port=0,
+        silence_logs=False,
+        processes=False,
+        dashboard_address=None,
+        loop=loop,
+        asynchronous=True,
+    )
     yield cluster._start()
     try:
-        adapt = Adaptive(cluster.scheduler, cluster, minimum=0, maximum=4,
-                         interval='10 ms')
+        adapt = Adaptive(
+            cluster.scheduler, cluster, minimum=0, maximum=4, interval="10 ms"
+        )
         client = yield Client(cluster, asynchronous=True, loop=loop)
-        cluster.scheduler.task_duration['slowinc'] = 1000
+        cluster.scheduler.task_duration["slowinc"] = 1000
 
         yield client.submit(slowinc, 1, delay=0.100)
 
@@ -323,8 +362,9 @@ def test_no_more_workers_than_tasks():
 
 def test_basic_no_loop():
     try:
-        with LocalCluster(0, scheduler_port=0, silence_logs=False,
-                          dashboard_address=None) as cluster:
+        with LocalCluster(
+            0, scheduler_port=0, silence_logs=False, dashboard_address=None
+        ) as cluster:
             with Client(cluster) as client:
                 cluster.adapt()
                 future = client.submit(lambda x: x + 1, 1)
@@ -337,13 +377,18 @@ def test_basic_no_loop():
 @gen_test(timeout=None)
 def test_target_duration():
     """ Ensure that redefining adapt with a lower maximum removes workers """
-    cluster = yield LocalCluster(0, asynchronous=True, processes=False,
-                                 scheduler_port=0, silence_logs=False,
-                                 dashboard_address=None)
+    cluster = yield LocalCluster(
+        0,
+        asynchronous=True,
+        processes=False,
+        scheduler_port=0,
+        silence_logs=False,
+        dashboard_address=None,
+    )
     client = yield Client(cluster, asynchronous=True)
-    adaptive = cluster.adapt(interval='20ms', minimum=2, target_duration='5s')
+    adaptive = cluster.adapt(interval="20ms", minimum=2, target_duration="5s")
 
-    cluster.scheduler.task_duration['slowinc'] = 1
+    cluster.scheduler.task_duration["slowinc"] = 1
 
     try:
         while len(cluster.scheduler.workers) < 2:
@@ -354,8 +399,8 @@ def test_target_duration():
         while len(adaptive.log) < 2:
             yield gen.sleep(0.01)
 
-        assert adaptive.log[0][1:] == ('up', {'n': 2})
-        assert adaptive.log[1][1:] == ('up', {'n': 20})
+        assert adaptive.log[0][1:] == ("up", {"n": 2})
+        assert adaptive.log[1][1:] == ("up", {"n": 20})
 
     finally:
         yield client.close()
@@ -365,22 +410,30 @@ def test_target_duration():
 @gen_test(timeout=None)
 def test_worker_keys():
     """ Ensure that redefining adapt with a lower maximum removes workers """
-    cluster = yield LocalCluster(0, asynchronous=True, processes=False,
-                                 scheduler_port=0, silence_logs=False,
-                                 dashboard_address=None)
+    cluster = yield LocalCluster(
+        0,
+        asynchronous=True,
+        processes=False,
+        scheduler_port=0,
+        silence_logs=False,
+        dashboard_address=None,
+    )
 
     try:
-        yield [cluster.start_worker(name='a-1'),
-               cluster.start_worker(name='a-2'),
-               cluster.start_worker(name='b-1'),
-               cluster.start_worker(name='b-2')]
+        yield [
+            cluster.start_worker(name="a-1"),
+            cluster.start_worker(name="a-2"),
+            cluster.start_worker(name="b-1"),
+            cluster.start_worker(name="b-2"),
+        ]
 
         while len(cluster.scheduler.workers) != 4:
             yield gen.sleep(0.01)
 
         def key(ws):
-            return ws.name.split('-')[0]
-        cluster._adaptive_options = {'worker_key': key}
+            return ws.name.split("-")[0]
+
+        cluster._adaptive_options = {"worker_key": key}
 
         adaptive = cluster.adapt(minimum=1)
         yield adaptive._adapt()
@@ -389,7 +442,7 @@ def key(ws):
             yield gen.sleep(0.01)
 
         names = {ws.name for ws in cluster.scheduler.workers.values()}
-        assert names == {'a-1', 'a-2'} or names == {'b-1', 'b-2'}
+        assert names == {"a-1", "a-2"} or names == {"b-1", "b-2"}
     finally:
         yield cluster.close()
 
@@ -403,4 +456,4 @@ def test_without_cluster(c, s):
         yield gen.sleep(0.01)
 
     response = yield c.scheduler.adaptive_recommendations()
-    assert response['status'] == 'up'
+    assert response["status"] == "up"
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index 95b4bb42fd4..ee2d48c2df3 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -16,12 +16,16 @@
 from distributed import Client, Worker, Nanny
 from distributed.deploy.local import LocalCluster, nprocesses_nthreads
 from distributed.metrics import time
-from distributed.utils_test import (inc, gen_test, slowinc,
-                                    assert_cannot_connect,
-                                    assert_can_connect_locally_4,
-                                    assert_can_connect_from_everywhere_4,
-                                    assert_can_connect_from_everywhere_4_6,
-                                    captured_logger)
+from distributed.utils_test import (
+    inc,
+    gen_test,
+    slowinc,
+    assert_cannot_connect,
+    assert_can_connect_locally_4,
+    assert_can_connect_from_everywhere_4,
+    assert_can_connect_from_everywhere_4_6,
+    captured_logger,
+)
 from distributed.utils_test import loop  # noqa: F401
 from distributed.utils import sync
 from distributed.worker import TOTAL_MEMORY
@@ -30,8 +34,14 @@
 
 
 def test_simple(loop):
-    with LocalCluster(4, scheduler_port=0, processes=False, silence_logs=False,
-                      dashboard_address=None, loop=loop) as c:
+    with LocalCluster(
+        4,
+        scheduler_port=0,
+        processes=False,
+        silence_logs=False,
+        dashboard_address=None,
+        loop=loop,
+    ) as c:
         with Client(c) as e:
             x = e.submit(inc, 1)
             x.result()
@@ -42,21 +52,23 @@ def test_simple(loop):
 
 
 def test_local_cluster_supports_blocked_handlers(loop):
-    with LocalCluster(blocked_handlers=['run_function'], loop=loop) as c:
+    with LocalCluster(blocked_handlers=["run_function"], loop=loop) as c:
         with Client(c) as client:
             with pytest.raises(ValueError) as exc:
                 client.run_on_scheduler(lambda x: x, 42)
 
-    assert "'run_function' handler has been explicitly disallowed in Scheduler" in str(exc.value)
+    assert "'run_function' handler has been explicitly disallowed in Scheduler" in str(
+        exc.value
+    )
 
 
-@pytest.mark.skipif('sys.version_info[0] == 2', reason='fork issues')
+@pytest.mark.skipif("sys.version_info[0] == 2", reason="fork issues")
 def test_close_twice():
     with LocalCluster() as cluster:
         with Client(cluster.scheduler_address) as client:
             f = client.map(inc, range(100))
             client.gather(f)
-        with captured_logger('tornado.application') as log:
+        with captured_logger("tornado.application") as log:
             cluster.close()
             cluster.close()
             sleep(0.5)
@@ -64,10 +76,16 @@ def test_close_twice():
         assert not log
 
 
-@pytest.mark.skipif('sys.version_info[0] == 2', reason='multi-loop')
+@pytest.mark.skipif("sys.version_info[0] == 2", reason="multi-loop")
 def test_procs():
-    with LocalCluster(2, scheduler_port=0, processes=False, threads_per_worker=3,
-                      dashboard_address=None, silence_logs=False) as c:
+    with LocalCluster(
+        2,
+        scheduler_port=0,
+        processes=False,
+        threads_per_worker=3,
+        dashboard_address=None,
+        silence_logs=False,
+    ) as c:
         assert len(c.workers) == 2
         assert all(isinstance(w, Worker) for w in c.workers)
         with Client(c.scheduler.address) as e:
@@ -75,8 +93,14 @@ def test_procs():
             assert all(isinstance(w, Worker) for w in c.workers)
         repr(c)
 
-    with LocalCluster(2, scheduler_port=0, processes=True, threads_per_worker=3,
-                      dashboard_address=None, silence_logs=False) as c:
+    with LocalCluster(
+        2,
+        scheduler_port=0,
+        processes=True,
+        threads_per_worker=3,
+        dashboard_address=None,
+        silence_logs=False,
+    ) as c:
         assert len(c.workers) == 2
         assert all(isinstance(w, Nanny) for w in c.workers)
         with Client(c.scheduler.address) as e:
@@ -92,10 +116,11 @@ def test_move_unserializable_data():
     Test that unserializable data is still fine to transfer over inproc
     transports.
     """
-    with LocalCluster(processes=False, silence_logs=False,
-                      dashboard_address=None) as cluster:
-        assert cluster.scheduler_address.startswith('inproc://')
-        assert cluster.workers[0].address.startswith('inproc://')
+    with LocalCluster(
+        processes=False, silence_logs=False, dashboard_address=None
+    ) as cluster:
+        assert cluster.scheduler_address.startswith("inproc://")
+        assert cluster.workers[0].address.startswith("inproc://")
         with Client(cluster) as client:
             lock = Lock()
             x = client.scatter(lock)
@@ -107,41 +132,49 @@ def test_transports():
     """
     Test the transport chosen by LocalCluster depending on arguments.
     """
-    with LocalCluster(1, processes=False, silence_logs=False,
-                      dashboard_address=None) as c:
-        assert c.scheduler_address.startswith('inproc://')
-        assert c.workers[0].address.startswith('inproc://')
+    with LocalCluster(
+        1, processes=False, silence_logs=False, dashboard_address=None
+    ) as c:
+        assert c.scheduler_address.startswith("inproc://")
+        assert c.workers[0].address.startswith("inproc://")
         with Client(c.scheduler.address) as e:
             assert e.submit(inc, 4).result() == 5
 
     # Have nannies => need TCP
-    with LocalCluster(1, processes=True, silence_logs=False,
-                      dashboard_address=None) as c:
-        assert c.scheduler_address.startswith('tcp://')
-        assert c.workers[0].address.startswith('tcp://')
+    with LocalCluster(
+        1, processes=True, silence_logs=False, dashboard_address=None
+    ) as c:
+        assert c.scheduler_address.startswith("tcp://")
+        assert c.workers[0].address.startswith("tcp://")
         with Client(c.scheduler.address) as e:
             assert e.submit(inc, 4).result() == 5
 
     # Scheduler port specified => need TCP
-    with LocalCluster(1, processes=False, scheduler_port=8786, silence_logs=False,
-                      dashboard_address=None) as c:
-
-        assert c.scheduler_address == 'tcp://127.0.0.1:8786'
-        assert c.workers[0].address.startswith('tcp://')
+    with LocalCluster(
+        1,
+        processes=False,
+        scheduler_port=8786,
+        silence_logs=False,
+        dashboard_address=None,
+    ) as c:
+
+        assert c.scheduler_address == "tcp://127.0.0.1:8786"
+        assert c.workers[0].address.startswith("tcp://")
         with Client(c.scheduler.address) as e:
             assert e.submit(inc, 4).result() == 5
 
 
-@pytest.mark.skipif('sys.version_info[0] == 2', reason='')
+@pytest.mark.skipif("sys.version_info[0] == 2", reason="")
 class LocalTest(ClusterTest, unittest.TestCase):
     Cluster = partial(LocalCluster, silence_logs=False, dashboard_address=None)
-    kwargs = {'dashboard_address': None}
+    kwargs = {"dashboard_address": None}
 
 
-@pytest.mark.skipif('sys.version_info[0] == 2', reason='')
+@pytest.mark.skipif("sys.version_info[0] == 2", reason="")
 def test_Client_with_local(loop):
-    with LocalCluster(1, scheduler_port=0, silence_logs=False,
-                      dashboard_address=None, loop=loop) as c:
+    with LocalCluster(
+        1, scheduler_port=0, silence_logs=False, dashboard_address=None, loop=loop
+    ) as c:
         with Client(c) as e:
             assert len(e.ncores()) == len(c.workers)
             assert c.scheduler_address in repr(c)
@@ -150,22 +183,26 @@ def test_Client_with_local(loop):
 def test_Client_solo(loop):
     with Client(loop=loop, silence_logs=False) as c:
         pass
-    assert c.cluster.status == 'closed'
+    assert c.cluster.status == "closed"
 
 
 @gen_test()
 def test_duplicate_clients():
-    pytest.importorskip('bokeh')
+    pytest.importorskip("bokeh")
     c1 = yield Client(processes=False, silence_logs=False, dashboard_address=9876)
     with pytest.warns(Exception) as info:
         c2 = yield Client(processes=False, silence_logs=False, dashboard_address=9876)
 
-    assert 'bokeh' in c1.cluster.scheduler.services
-    assert 'bokeh' in c2.cluster.scheduler.services
+    assert "bokeh" in c1.cluster.scheduler.services
+    assert "bokeh" in c2.cluster.scheduler.services
 
-    assert any(all(word in str(msg.message).lower()
-                   for word in ['9876', 'running', 'already in use'])
-               for msg in info.list)
+    assert any(
+        all(
+            word in str(msg.message).lower()
+            for word in ["9876", "running", "already in use"]
+        )
+        for msg in info.list
+    )
     yield c1.close()
 
 
@@ -173,7 +210,7 @@ def test_Client_kwargs(loop):
     with Client(loop=loop, processes=False, n_workers=2, silence_logs=False) as c:
         assert len(c.cluster.workers) == 2
         assert all(isinstance(w, Worker) for w in c.cluster.workers)
-    assert c.cluster.status == 'closed'
+    assert c.cluster.status == "closed"
 
 
 def test_Client_twice(loop):
@@ -182,23 +219,26 @@ def test_Client_twice(loop):
             assert c.cluster.scheduler.port != f.cluster.scheduler.port
 
 
-@pytest.mark.skipif('sys.version_info[0] == 2', reason='fork issues')
+@pytest.mark.skipif("sys.version_info[0] == 2", reason="fork issues")
 def test_defaults():
     from distributed.worker import _ncores
 
-    with LocalCluster(scheduler_port=0, silence_logs=False,
-                      dashboard_address=None) as c:
+    with LocalCluster(
+        scheduler_port=0, silence_logs=False, dashboard_address=None
+    ) as c:
         assert sum(w.ncores for w in c.workers) == _ncores
         assert all(isinstance(w, Nanny) for w in c.workers)
 
-    with LocalCluster(processes=False, scheduler_port=0, silence_logs=False,
-                      dashboard_address=None) as c:
+    with LocalCluster(
+        processes=False, scheduler_port=0, silence_logs=False, dashboard_address=None
+    ) as c:
         assert sum(w.ncores for w in c.workers) == _ncores
         assert all(isinstance(w, Worker) for w in c.workers)
         assert len(c.workers) == 1
 
-    with LocalCluster(n_workers=2, scheduler_port=0, silence_logs=False,
-                      dashboard_address=None) as c:
+    with LocalCluster(
+        n_workers=2, scheduler_port=0, silence_logs=False, dashboard_address=None
+    ) as c:
         if _ncores % 2 == 0:
             expected_total_threads = max(2, _ncores)
         else:
@@ -206,60 +246,91 @@ def test_defaults():
             expected_total_threads = max(2, _ncores + 1)
         assert sum(w.ncores for w in c.workers) == expected_total_threads
 
-    with LocalCluster(threads_per_worker=_ncores * 2, scheduler_port=0,
-                      silence_logs=False, dashboard_address=None) as c:
+    with LocalCluster(
+        threads_per_worker=_ncores * 2,
+        scheduler_port=0,
+        silence_logs=False,
+        dashboard_address=None,
+    ) as c:
         assert len(c.workers) == 1
 
-    with LocalCluster(n_workers=_ncores * 2, scheduler_port=0,
-                      silence_logs=False, dashboard_address=None) as c:
+    with LocalCluster(
+        n_workers=_ncores * 2,
+        scheduler_port=0,
+        silence_logs=False,
+        dashboard_address=None,
+    ) as c:
         assert all(w.ncores == 1 for w in c.workers)
-    with LocalCluster(threads_per_worker=2, n_workers=3, scheduler_port=0,
-                      silence_logs=False, dashboard_address=None) as c:
+    with LocalCluster(
+        threads_per_worker=2,
+        n_workers=3,
+        scheduler_port=0,
+        silence_logs=False,
+        dashboard_address=None,
+    ) as c:
         assert len(c.workers) == 3
         assert all(w.ncores == 2 for w in c.workers)
 
 
 def test_worker_params():
-    with LocalCluster(n_workers=2, scheduler_port=0, silence_logs=False,
-                      dashboard_address=None, memory_limit=500) as c:
+    with LocalCluster(
+        n_workers=2,
+        scheduler_port=0,
+        silence_logs=False,
+        dashboard_address=None,
+        memory_limit=500,
+    ) as c:
         assert [w.memory_limit for w in c.workers] == [500] * 2
 
 
 def test_memory_limit_none():
-    with LocalCluster(n_workers=2, scheduler_port=0, silence_logs=False,
-                      processes=False,  dashboard_address=None, memory_limit=None) as c:
+    with LocalCluster(
+        n_workers=2,
+        scheduler_port=0,
+        silence_logs=False,
+        processes=False,
+        dashboard_address=None,
+        memory_limit=None,
+    ) as c:
         w = c.workers[0]
         assert type(w.data) is dict
         assert w.memory_limit is None
 
 
 def test_cleanup():
-    c = LocalCluster(2, scheduler_port=0, silence_logs=False,
-                     dashboard_address=None)
+    c = LocalCluster(2, scheduler_port=0, silence_logs=False, dashboard_address=None)
     port = c.scheduler.port
     c.close()
-    c2 = LocalCluster(2, scheduler_port=port, silence_logs=False,
-                      dashboard_address=None)
+    c2 = LocalCluster(
+        2, scheduler_port=port, silence_logs=False, dashboard_address=None
+    )
     c.close()
 
 
 def test_repeated():
-    with LocalCluster(scheduler_port=8448, silence_logs=False,
-                      dashboard_address=None) as c:
+    with LocalCluster(
+        scheduler_port=8448, silence_logs=False, dashboard_address=None
+    ) as c:
         pass
-    with LocalCluster(scheduler_port=8448, silence_logs=False,
-                      dashboard_address=None) as c:
+    with LocalCluster(
+        scheduler_port=8448, silence_logs=False, dashboard_address=None
+    ) as c:
         pass
 
 
-@pytest.mark.parametrize('processes', [True, False])
+@pytest.mark.parametrize("processes", [True, False])
 def test_bokeh(loop, processes):
-    pytest.importorskip('bokeh')
-    requests = pytest.importorskip('requests')
-    with LocalCluster(scheduler_port=0, silence_logs=False, loop=loop,
-                      processes=processes, dashboard_address=0) as c:
-        bokeh_port = c.scheduler.services['bokeh'].port
-        url = 'http://127.0.0.1:%d/status/' % bokeh_port
+    pytest.importorskip("bokeh")
+    requests = pytest.importorskip("requests")
+    with LocalCluster(
+        scheduler_port=0,
+        silence_logs=False,
+        loop=loop,
+        processes=processes,
+        dashboard_address=0,
+    ) as c:
+        bokeh_port = c.scheduler.services["bokeh"].port
+        url = "http://127.0.0.1:%d/status/" % bokeh_port
         start = time()
         while True:
             response = requests.get(url)
@@ -268,14 +339,14 @@ def test_bokeh(loop, processes):
             assert time() < start + 20
             sleep(0.01)
         # 'localhost' also works
-        response = requests.get('http://localhost:%d/status/' % bokeh_port)
+        response = requests.get("http://localhost:%d/status/" % bokeh_port)
         assert response.ok
 
     with pytest.raises(requests.RequestException):
         requests.get(url, timeout=0.2)
 
 
-@pytest.mark.skipif(sys.version_info < (3, 6), reason='Unknown')
+@pytest.mark.skipif(sys.version_info < (3, 6), reason="Unknown")
 def test_blocks_until_full(loop):
     with Client(loop=loop) as c:
         assert len(c.ncores()) > 0
@@ -284,9 +355,15 @@ def test_blocks_until_full(loop):
 @gen_test()
 def test_scale_up_and_down():
     loop = IOLoop.current()
-    cluster = yield LocalCluster(0, scheduler_port=0, processes=False,
-                                 silence_logs=False, dashboard_address=None,
-                                 loop=loop, asynchronous=True)
+    cluster = yield LocalCluster(
+        0,
+        scheduler_port=0,
+        processes=False,
+        silence_logs=False,
+        dashboard_address=None,
+        loop=loop,
+        asynchronous=True,
+    )
     c = yield Client(cluster, asynchronous=True)
 
     assert not cluster.workers
@@ -314,8 +391,9 @@ def test_silent_startup():
             sleep(1.5)
         """
 
-    out = subprocess.check_output([sys.executable, "-Wi", "-c", code],
-                                  stderr=subprocess.STDOUT)
+    out = subprocess.check_output(
+        [sys.executable, "-Wi", "-c", code], stderr=subprocess.STDOUT
+    )
     out = out.decode()
     try:
         assert not out
@@ -326,51 +404,74 @@ def test_silent_startup():
 
 
 def test_only_local_access(loop):
-    with LocalCluster(scheduler_port=0, silence_logs=False,
-                      dashboard_address=None, loop=loop) as c:
+    with LocalCluster(
+        scheduler_port=0, silence_logs=False, dashboard_address=None, loop=loop
+    ) as c:
         sync(loop, assert_can_connect_locally_4, c.scheduler.port)
 
 
 def test_remote_access(loop):
-    with LocalCluster(scheduler_port=0, silence_logs=False,
-                      dashboard_address=None, ip='', loop=loop) as c:
+    with LocalCluster(
+        scheduler_port=0, silence_logs=False, dashboard_address=None, ip="", loop=loop
+    ) as c:
         sync(loop, assert_can_connect_from_everywhere_4_6, c.scheduler.port)
 
 
-@pytest.mark.parametrize('n_workers', [None, 3])
+@pytest.mark.parametrize("n_workers", [None, 3])
 def test_memory(loop, n_workers):
-    with LocalCluster(n_workers=n_workers, scheduler_port=0, processes=False,
-                      silence_logs=False, dashboard_address=None, loop=loop) as cluster:
+    with LocalCluster(
+        n_workers=n_workers,
+        scheduler_port=0,
+        processes=False,
+        silence_logs=False,
+        dashboard_address=None,
+        loop=loop,
+    ) as cluster:
         assert sum(w.memory_limit for w in cluster.workers) <= TOTAL_MEMORY
 
 
-@pytest.mark.parametrize('n_workers', [None, 3])
+@pytest.mark.parametrize("n_workers", [None, 3])
 def test_memory_nanny(loop, n_workers):
-    with LocalCluster(n_workers=n_workers, scheduler_port=0, processes=True,
-                      silence_logs=False, dashboard_address=None, loop=loop) as cluster:
+    with LocalCluster(
+        n_workers=n_workers,
+        scheduler_port=0,
+        processes=True,
+        silence_logs=False,
+        dashboard_address=None,
+        loop=loop,
+    ) as cluster:
         with Client(cluster.scheduler_address, loop=loop) as c:
             info = c.scheduler_info()
-            assert (sum(w['memory_limit'] for w in info['workers'].values())
-                    <= TOTAL_MEMORY)
+            assert (
+                sum(w["memory_limit"] for w in info["workers"].values()) <= TOTAL_MEMORY
+            )
 
 
 def test_death_timeout_raises(loop):
     with pytest.raises(gen.TimeoutError):
-        with LocalCluster(scheduler_port=0, silence_logs=False,
-                          death_timeout=1e-10, dashboard_address=None,
-                          loop=loop) as cluster:
+        with LocalCluster(
+            scheduler_port=0,
+            silence_logs=False,
+            death_timeout=1e-10,
+            dashboard_address=None,
+            loop=loop,
+        ) as cluster:
             pass
 
 
-@pytest.mark.skipif(sys.version_info < (3, 6), reason='Unknown')
+@pytest.mark.skipif(sys.version_info < (3, 6), reason="Unknown")
 def test_bokeh_kwargs(loop):
-    pytest.importorskip('bokeh')
-    with LocalCluster(scheduler_port=0, silence_logs=False, loop=loop,
-                      dashboard_address=0,
-                      service_kwargs={'bokeh': {'prefix': '/foo'}}) as c:
+    pytest.importorskip("bokeh")
+    with LocalCluster(
+        scheduler_port=0,
+        silence_logs=False,
+        loop=loop,
+        dashboard_address=0,
+        service_kwargs={"bokeh": {"prefix": "/foo"}},
+    ) as c:
 
-        bs = c.scheduler.services['bokeh']
-        assert bs.prefix == '/foo'
+        bs = c.scheduler.services["bokeh"]
+        assert bs.prefix == "/foo"
 
 
 def test_io_loop_periodic_callbacks(loop):
@@ -393,9 +494,14 @@ def test_logging():
 
 
 def test_ipywidgets(loop):
-    ipywidgets = pytest.importorskip('ipywidgets')
-    with LocalCluster(scheduler_port=0, silence_logs=False, loop=loop,
-                      dashboard_address=False, processes=False) as cluster:
+    ipywidgets = pytest.importorskip("ipywidgets")
+    with LocalCluster(
+        scheduler_port=0,
+        silence_logs=False,
+        loop=loop,
+        dashboard_address=False,
+        processes=False,
+    ) as cluster:
         cluster._ipython_display_()
         box = cluster._cached_widget
         assert isinstance(box, ipywidgets.Widget)
@@ -403,8 +509,14 @@ def test_ipywidgets(loop):
 
 def test_scale(loop):
     """ Directly calling scale both up and down works as expected """
-    with LocalCluster(scheduler_port=0, silence_logs=False, loop=loop,
-                      dashboard_address=False, processes=False, n_workers=0) as cluster:
+    with LocalCluster(
+        scheduler_port=0,
+        silence_logs=False,
+        loop=loop,
+        dashboard_address=False,
+        processes=False,
+        n_workers=0,
+    ) as cluster:
         assert not cluster.scheduler.workers
         cluster.scale(3)
 
@@ -424,14 +536,20 @@ def test_scale(loop):
 
 
 def test_adapt(loop):
-    with LocalCluster(scheduler_port=0, silence_logs=False, loop=loop,
-                      dashboard_address=False, processes=False, n_workers=0) as cluster:
-        cluster.adapt(minimum=0, maximum=2, interval='10ms')
+    with LocalCluster(
+        scheduler_port=0,
+        silence_logs=False,
+        loop=loop,
+        dashboard_address=False,
+        processes=False,
+        n_workers=0,
+    ) as cluster:
+        cluster.adapt(minimum=0, maximum=2, interval="10ms")
         assert cluster._adaptive.minimum == 0
         assert cluster._adaptive.maximum == 2
         ref = weakref.ref(cluster._adaptive)
 
-        cluster.adapt(minimum=1, maximum=2, interval='10ms')
+        cluster.adapt(minimum=1, maximum=2, interval="10ms")
         assert cluster._adaptive.minimum == 1
         gc.collect()
 
@@ -450,10 +568,16 @@ def test_adapt(loop):
 
 def test_adapt_then_manual(loop):
     """ We can revert from adaptive, back to manual """
-    with LocalCluster(scheduler_port=0, silence_logs=False, loop=loop,
-                      dashboard_address=False, processes=False, n_workers=8) as cluster:
+    with LocalCluster(
+        scheduler_port=0,
+        silence_logs=False,
+        loop=loop,
+        dashboard_address=False,
+        processes=False,
+        n_workers=8,
+    ) as cluster:
         sleep(0.1)
-        cluster.adapt(minimum=0, maximum=4, interval='10ms')
+        cluster.adapt(minimum=0, maximum=4, interval="10ms")
 
         start = time()
         while cluster.scheduler.workers or cluster.workers:
@@ -480,19 +604,33 @@ def test_adapt_then_manual(loop):
 
 def test_local_tls(loop):
     from distributed.utils_test import tls_only_security
+
     security = tls_only_security()
-    with LocalCluster(scheduler_port=8786, silence_logs=False, security=security,
-                      dashboard_address=False, ip='tls://0.0.0.0', loop=loop) as c:
-        sync(loop, assert_can_connect_from_everywhere_4, c.scheduler.port,
-             connection_args=security.get_connection_args('client'),
-             protocol='tls', timeout=3)
+    with LocalCluster(
+        scheduler_port=8786,
+        silence_logs=False,
+        security=security,
+        dashboard_address=False,
+        ip="tls://0.0.0.0",
+        loop=loop,
+    ) as c:
+        sync(
+            loop,
+            assert_can_connect_from_everywhere_4,
+            c.scheduler.port,
+            connection_args=security.get_connection_args("client"),
+            protocol="tls",
+            timeout=3,
+        )
 
         # If we connect to a TLS localculster without ssl information we should fail
-        sync(loop, assert_cannot_connect,
-             addr='tcp://127.0.0.1:%d' % c.scheduler.port,
-             connection_args=security.get_connection_args('client'),
-             exception_class=RuntimeError,
-             )
+        sync(
+            loop,
+            assert_cannot_connect,
+            addr="tcp://127.0.0.1:%d" % c.scheduler.port,
+            connection_args=security.get_connection_args("client"),
+            exception_class=RuntimeError,
+        )
 
 
 @gen_test()
@@ -502,9 +640,15 @@ def scale_down(self, *args, **kwargs):
             pass
 
     loop = IOLoop.current()
-    cluster = yield MyCluster(0, scheduler_port=0, processes=False,
-                              silence_logs=False, dashboard_address=None,
-                              loop=loop, asynchronous=True)
+    cluster = yield MyCluster(
+        0,
+        scheduler_port=0,
+        processes=False,
+        silence_logs=False,
+        dashboard_address=None,
+        loop=loop,
+        asynchronous=True,
+    )
     c = yield Client(cluster, asynchronous=True)
 
     assert not cluster.workers
@@ -529,15 +673,23 @@ def scale_down(self, *args, **kwargs):
 
 def test_local_tls_restart(loop):
     from distributed.utils_test import tls_only_security
+
     security = tls_only_security()
-    with LocalCluster(n_workers=1, scheduler_port=8786, silence_logs=False, security=security,
-                      dashboard_address=False, ip='tls://0.0.0.0', loop=loop) as c:
+    with LocalCluster(
+        n_workers=1,
+        scheduler_port=8786,
+        silence_logs=False,
+        security=security,
+        dashboard_address=False,
+        ip="tls://0.0.0.0",
+        loop=loop,
+    ) as c:
         with Client(c.scheduler.address, loop=loop, security=security) as client:
             print(c.workers, c.workers[0].address)
-            workers_before = set(client.scheduler_info()['workers'])
+            workers_before = set(client.scheduler_info()["workers"])
             assert client.submit(inc, 1).result() == 2
             client.restart()
-            workers_after = set(client.scheduler_info()['workers'])
+            workers_after = set(client.scheduler_info()["workers"])
             assert client.submit(inc, 2).result() == 3
             assert workers_before != workers_after
 
@@ -556,8 +708,14 @@ def test_default_process_thread_breakdown():
 
 
 def test_asynchronous_property(loop):
-    with LocalCluster(4, scheduler_port=0, processes=False, silence_logs=False,
-                      dashboard_address=None, loop=loop) as cluster:
+    with LocalCluster(
+        4,
+        scheduler_port=0,
+        processes=False,
+        silence_logs=False,
+        dashboard_address=None,
+        loop=loop,
+    ) as cluster:
 
         @gen.coroutine
         def _():
@@ -567,20 +725,21 @@ def _():
 
 
 def test_protocol_inproc(loop):
-    with LocalCluster(protocol='inproc://', loop=loop, processes=False) as cluster:
-        assert cluster.scheduler.address.startswith('inproc://')
+    with LocalCluster(protocol="inproc://", loop=loop, processes=False) as cluster:
+        assert cluster.scheduler.address.startswith("inproc://")
 
 
 def test_protocol_tcp(loop):
-    with LocalCluster(protocol='tcp', loop=loop, processes=False) as cluster:
-        assert cluster.scheduler.address.startswith('tcp://')
+    with LocalCluster(protocol="tcp", loop=loop, processes=False) as cluster:
+        assert cluster.scheduler.address.startswith("tcp://")
 
 
-@pytest.mark.skipif(not sys.platform.startswith('linux'),
-                    reason="Need 127.0.0.2 to mean localhost")
+@pytest.mark.skipif(
+    not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
+)
 def test_protocol_ip(loop):
-    with LocalCluster(ip='tcp://127.0.0.2', loop=loop, processes=False) as cluster:
-        assert cluster.scheduler.address.startswith('tcp://127.0.0.2')
+    with LocalCluster(ip="tcp://127.0.0.2", loop=loop, processes=False) as cluster:
+        assert cluster.scheduler.address.startswith("tcp://127.0.0.2")
 
 
 if sys.version_info >= (3, 5):
diff --git a/distributed/deploy/tests/test_ssh.py b/distributed/deploy/tests/test_ssh.py
index 5c6c76ea3c8..a86a8ddd280 100644
--- a/distributed/deploy/tests/test_ssh.py
+++ b/distributed/deploy/tests/test_ssh.py
@@ -3,7 +3,8 @@
 from time import sleep
 
 import pytest
-pytest.importorskip('paramiko')
+
+pytest.importorskip("paramiko")
 
 from distributed import Client
 from distributed.deploy.ssh import SSHCluster
@@ -13,16 +14,18 @@
 
 @pytest.mark.avoid_travis
 def test_cluster(loop):
-    with SSHCluster(scheduler_addr='127.0.0.1',
-                    scheduler_port=7437,
-                    worker_addrs=['127.0.0.1', '127.0.0.1']) as c:
+    with SSHCluster(
+        scheduler_addr="127.0.0.1",
+        scheduler_port=7437,
+        worker_addrs=["127.0.0.1", "127.0.0.1"],
+    ) as c:
         with Client(c, loop=loop) as e:
             start = time()
             while len(e.ncores()) != 2:
                 sleep(0.01)
                 assert time() < start + 5
 
-            c.add_worker('127.0.0.1')
+            c.add_worker("127.0.0.1")
 
             start = time()
             while len(e.ncores()) != 3:
diff --git a/distributed/diagnostics/__init__.py b/distributed/diagnostics/__init__.py
index ab9f6d7a9dd..9469c3855d1 100644
--- a/distributed/diagnostics/__init__.py
+++ b/distributed/diagnostics/__init__.py
@@ -2,6 +2,7 @@
 
 from ..utils import ignoring
 from .graph_layout import GraphLayout
+
 with ignoring(ImportError):
     from .progressbar import progress
 with ignoring(ImportError):
diff --git a/distributed/diagnostics/eventstream.py b/distributed/diagnostics/eventstream.py
index 1eabf0ea4dc..a4eb0830534 100644
--- a/distributed/diagnostics/eventstream.py
+++ b/distributed/diagnostics/eventstream.py
@@ -22,9 +22,9 @@ def __init__(self, scheduler=None):
             scheduler.add_plugin(self)
 
     def transition(self, key, start, finish, *args, **kwargs):
-        if start == 'processing':
-            kwargs['key'] = key
-            if finish == 'memory' or finish == 'erred':
+        if start == "processing":
+            kwargs["key"] = key
+            if finish == "memory" or finish == "erred":
                 self.buffer.append(kwargs)
 
 
@@ -66,9 +66,13 @@ def eventstream(address, interval):
     """
     address = coerce_to_address(address)
     comm = yield connect(address)
-    yield comm.write({'op': 'feed',
-                      'setup': dumps_function(EventStream),
-                      'function': dumps_function(swap_buffer),
-                      'interval': interval,
-                      'teardown': dumps_function(teardown)})
+    yield comm.write(
+        {
+            "op": "feed",
+            "setup": dumps_function(EventStream),
+            "function": dumps_function(swap_buffer),
+            "interval": interval,
+            "teardown": dumps_function(teardown),
+        }
+    )
     raise gen.Return(comm)
diff --git a/distributed/diagnostics/graph_layout.py b/distributed/diagnostics/graph_layout.py
index 5c29fe28b83..62e115a9ad4 100644
--- a/distributed/diagnostics/graph_layout.py
+++ b/distributed/diagnostics/graph_layout.py
@@ -12,6 +12,7 @@ class GraphLayout(SchedulerPlugin):
     It is commonly used with distributed/bokeh/scheduler.py::GraphPlot, which
     is rendered at /graph on the diagnostic dashboard.
     """
+
     def __init__(self, scheduler):
         self.x = {}
         self.y = {}
@@ -31,14 +32,16 @@ def __init__(self, scheduler):
         scheduler.add_plugin(self)
 
         if self.scheduler.tasks:
-            dependencies = {k: [ds.key for ds in ts.dependencies]
-                            for k, ts in scheduler.tasks.items()}
+            dependencies = {
+                k: [ds.key for ds in ts.dependencies]
+                for k, ts in scheduler.tasks.items()
+            }
             priority = {k: ts.priority for k, ts in scheduler.tasks.items()}
-            self.update_graph(self.scheduler, dependencies=dependencies,
-                    priority=priority)
+            self.update_graph(
+                self.scheduler, dependencies=dependencies, priority=priority
+            )
 
-    def update_graph(self, scheduler, dependencies=None, priority=None,
-                     **kwargs):
+    def update_graph(self, scheduler, dependencies=None, priority=None, **kwargs):
         stack = sorted(dependencies, key=lambda k: priority.get(k, 0), reverse=True)
         while stack:
             key = stack.pop()
@@ -48,15 +51,18 @@ def update_graph(self, scheduler, dependencies=None, priority=None,
             if deps:
                 if not all(dep in self.y for dep in deps):
                     stack.append(key)
-                    stack.extend(sorted(deps, key=lambda k: priority.get(k, 0),
-                                        reverse=True))
+                    stack.extend(
+                        sorted(deps, key=lambda k: priority.get(k, 0), reverse=True)
+                    )
                     continue
                 else:
-                    total_deps = sum(len(scheduler.tasks[dep].dependents)
-                                     for dep in deps)
-                    y = sum(self.y[dep] * len(scheduler.tasks[dep].dependents)
-                                          / total_deps
-                            for dep in deps)
+                    total_deps = sum(
+                        len(scheduler.tasks[dep].dependents) for dep in deps
+                    )
+                    y = sum(
+                        self.y[dep] * len(scheduler.tasks[dep].dependents) / total_deps
+                        for dep in deps
+                    )
                     x = max(self.x[dep] for dep in deps) + 1
             else:
                 x = 0
@@ -83,16 +89,20 @@ def update_graph(self, scheduler, dependencies=None, priority=None,
                 self.new_edges.append(edge)
 
     def transition(self, key, start, finish, *args, **kwargs):
-        if finish != 'forgotten':
+        if finish != "forgotten":
             self.state_updates.append((self.index[key], finish))
         else:
-            self.visible_updates.append((self.index[key], 'False'))
+            self.visible_updates.append((self.index[key], "False"))
             task = self.scheduler.tasks[key]
             for dep in task.dependents:
                 edge = (key, dep.key)
-                self.visible_edge_updates.append((self.index_edge.pop((key, dep.key)), 'False'))
+                self.visible_edge_updates.append(
+                    (self.index_edge.pop((key, dep.key)), "False")
+                )
             for dep in task.dependencies:
-                self.visible_edge_updates.append((self.index_edge.pop((dep.key, key)), 'False'))
+                self.visible_edge_updates.append(
+                    (self.index_edge.pop((dep.key, key)), "False")
+                )
 
             try:
                 del self.collision[(self.x[key], self.y[key])]
diff --git a/distributed/diagnostics/plugin.py b/distributed/diagnostics/plugin.py
index f5898e7f4e0..e1da4378fd4 100644
--- a/distributed/diagnostics/plugin.py
+++ b/distributed/diagnostics/plugin.py
@@ -39,8 +39,7 @@ class SchedulerPlugin(object):
     >>> scheduler.add_plugin(c)  # doctest: +SKIP
     """
 
-    def update_graph(self, scheduler, dsk=None, keys=None,
-                     restrictions=None, **kwargs):
+    def update_graph(self, scheduler, dsk=None, keys=None, restrictions=None, **kwargs):
         """ Run when a new graph / tasks enter the scheduler """
 
     def restart(self, scheduler, **kwargs):
diff --git a/distributed/diagnostics/progress.py b/distributed/diagnostics/progress.py
index 5a9fe7f083e..38638a248dd 100644
--- a/distributed/diagnostics/progress.py
+++ b/distributed/diagnostics/progress.py
@@ -62,7 +62,7 @@ class Progress(SchedulerPlugin):
     """
 
     def __init__(self, keys, scheduler, minimum=0, dt=0.1, complete=False):
-        self.keys = {k.key if hasattr(k, 'key') else k for k in keys}
+        self.keys = {k.key if hasattr(k, "key") else k for k in keys}
         self.keys = {tokey(k) for k in self.keys}
         self.scheduler = scheduler
         self.complete = complete
@@ -100,21 +100,21 @@ def setup(self):
         logger.debug("Set up Progress keys")
 
         for k in errors:
-            self.transition(k, None, 'erred', exception=True)
+            self.transition(k, None, "erred", exception=True)
 
     def transition(self, key, start, finish, *args, **kwargs):
-        if key in self.keys and start == 'processing' and finish == 'memory':
+        if key in self.keys and start == "processing" and finish == "memory":
             logger.debug("Progress sees key %s", key)
             self.keys.remove(key)
 
             if not self.keys:
                 self.stop()
 
-        if key in self.all_keys and finish == 'erred':
+        if key in self.all_keys and finish == "erred":
             logger.debug("Progress sees task erred")
-            self.stop(exception=kwargs['exception'], key=key)
+            self.stop(exception=kwargs["exception"], key=key)
 
-        if key in self.keys and finish == 'forgotten':
+        if key in self.keys and finish == "forgotten":
             logger.debug("A task was cancelled (%s), stopping progress", key)
             self.stop(exception=True, key=key)
 
@@ -125,11 +125,10 @@ def stop(self, exception=None, key=None):
         if self in self.scheduler.plugins:
             self.scheduler.plugins.remove(self)
         if exception:
-            self.status = 'error'
-            self.extra.update({'exception': self.scheduler.exceptions[key],
-                               'key': key})
+            self.status = "error"
+            self.extra.update({"exception": self.scheduler.exceptions[key], "key": key})
         else:
-            self.status = 'finished'
+            self.status = "finished"
         logger.debug("Remove Progress plugin")
 
 
@@ -156,11 +155,13 @@ class MultiProgress(Progress):
      'y': {'y-1', 'y-2'}}
     """
 
-    def __init__(self, keys, scheduler=None, func=key_split, minimum=0, dt=0.1,
-                 complete=False):
+    def __init__(
+        self, keys, scheduler=None, func=key_split, minimum=0, dt=0.1, complete=False
+    ):
         self.func = func
-        Progress.__init__(self, keys, scheduler, minimum=minimum, dt=dt,
-                          complete=complete)
+        Progress.__init__(
+            self, keys, scheduler, minimum=minimum, dt=dt, complete=complete
+        )
 
     @gen.coroutine
     def setup(self):
@@ -193,11 +194,11 @@ def setup(self):
                 self.keys[k] = set()
 
         for k in errors:
-            self.transition(k, None, 'erred', exception=True)
+            self.transition(k, None, "erred", exception=True)
         logger.debug("Set up Progress keys")
 
     def transition(self, key, start, finish, *args, **kwargs):
-        if start == 'processing' and finish == 'memory':
+        if start == "processing" and finish == "memory":
             s = self.keys.get(self.func(key), None)
             if s and key in s:
                 s.remove(key)
@@ -205,13 +206,13 @@ def transition(self, key, start, finish, *args, **kwargs):
             if not self.keys or not any(self.keys.values()):
                 self.stop()
 
-        if finish == 'erred':
+        if finish == "erred":
             logger.debug("Progress sees task erred")
             k = self.func(key)
-            if (k in self.all_keys and key in self.all_keys[k]):
-                self.stop(exception=kwargs.get('exception'), key=key)
+            if k in self.all_keys and key in self.all_keys[k]:
+                self.stop(exception=kwargs.get("exception"), key=key)
 
-        if finish == 'forgotten':
+        if finish == "forgotten":
             k = self.func(key)
             if k in self.all_keys and key in self.all_keys[k]:
                 logger.debug("A task was cancelled (%s), stopping progress", key)
@@ -231,11 +232,11 @@ def format_time(t):
     m, s = divmod(t, 60)
     h, m = divmod(m, 60)
     if h:
-        return '{0:2.0f}hr {1:2.0f}min {2:4.1f}s'.format(h, m, s)
+        return "{0:2.0f}hr {1:2.0f}min {2:4.1f}s".format(h, m, s)
     elif m:
-        return '{0:2.0f}min {1:4.1f}s'.format(m, s)
+        return "{0:2.0f}min {1:4.1f}s".format(m, s)
     else:
-        return '{0:4.1f}s'.format(s)
+        return "{0:4.1f}s".format(s)
 
 
 class AllProgress(SchedulerPlugin):
@@ -266,13 +267,13 @@ def transition(self, key, start, finish, *args, **kwargs):
         except KeyError:  # TODO: remove me once we have a new or clean state
             pass
 
-        if start == 'memory':
+        if start == "memory":
             # XXX why not respect DEFAULT_DATA_SIZE?
             self.nbytes[prefix] -= ts.nbytes or 0
-        if finish == 'memory':
+        if finish == "memory":
             self.nbytes[prefix] += ts.nbytes or 0
 
-        if finish != 'forgotten':
+        if finish != "forgotten":
             self.state[finish][prefix].add(key)
         else:
             s = self.all[prefix]
@@ -290,6 +291,7 @@ def restart(self, scheduler):
 
 class GroupProgress(SchedulerPlugin):
     """ Keep track of all keys, grouped by key_split """
+
     def __init__(self, scheduler):
         self.scheduler = scheduler
         self.keys = dict()
@@ -305,7 +307,7 @@ def __init__(self, scheduler):
                 self.create(key, k)
             self.keys[k].add(key)
             self.groups[k][ts.state] += 1
-            if ts.state == 'memory' and ts.nbytes is not None:
+            if ts.state == "memory" and ts.nbytes is not None:
                 self.nbytes[k] += ts.nbytes
 
         scheduler.add_plugin(self)
@@ -313,14 +315,12 @@ def __init__(self, scheduler):
     def create(self, key, k):
         with log_errors():
             ts = self.scheduler.tasks[key]
-            g = {'memory': 0, 'erred': 0, 'waiting': 0,
-                 'released': 0, 'processing': 0}
+            g = {"memory": 0, "erred": 0, "waiting": 0, "released": 0, "processing": 0}
             self.keys[k] = set()
             self.groups[k] = g
             self.nbytes[k] = 0
             self.durations[k] = 0
-            self.dependents[k] = {key_split_group(dts.key)
-                                  for dts in ts.dependents}
+            self.dependents[k] = {key_split_group(dts.key) for dts in ts.dependents}
             for dts in ts.dependencies:
                 d = key_split_group(dts.key)
                 self.dependents[d].add(k)
@@ -340,7 +340,7 @@ def transition(self, key, start, finish, *args, **kwargs):
             else:
                 g[start] -= 1
 
-            if finish != 'forgotten':
+            if finish != "forgotten":
                 g[finish] += 1
             else:
                 self.keys[k].remove(key)
@@ -350,9 +350,9 @@ def transition(self, key, start, finish, *args, **kwargs):
                     for dep in self.dependencies.pop(k):
                         self.dependents[key_split_group(dep)].remove(k)
 
-            if start == 'memory' and ts.nbytes is not None:
+            if start == "memory" and ts.nbytes is not None:
                 self.nbytes[k] -= ts.nbytes
-            if finish == 'memory' and ts.nbytes is not None:
+            if finish == "memory" and ts.nbytes is not None:
                 self.nbytes[k] += ts.nbytes
 
     def restart(self, scheduler):
diff --git a/distributed/diagnostics/progress_stream.py b/distributed/diagnostics/progress_stream.py
index 60704a5670a..1630251658a 100644
--- a/distributed/diagnostics/progress_stream.py
+++ b/distributed/diagnostics/progress_stream.py
@@ -17,10 +17,13 @@
 
 
 def counts(scheduler, allprogress):
-    return merge({'all': valmap(len, allprogress.all),
-                  'nbytes': allprogress.nbytes},
-                 {state: valmap(len, allprogress.state[state])
-                     for state in ['memory', 'erred', 'released', 'processing']})
+    return merge(
+        {"all": valmap(len, allprogress.all), "nbytes": allprogress.nbytes},
+        {
+            state: valmap(len, allprogress.state[state])
+            for state in ["memory", "erred", "released", "processing"]
+        },
+    )
 
 
 @gen.coroutine
@@ -44,11 +47,15 @@ def progress_stream(address, interval):
     """
     address = coerce_to_address(address)
     comm = yield connect(address)
-    yield comm.write({'op': 'feed',
-                      'setup': dumps_function(AllProgress),
-                      'function': dumps_function(counts),
-                      'interval': interval,
-                      'teardown': dumps_function(Scheduler.remove_plugin)})
+    yield comm.write(
+        {
+            "op": "feed",
+            "setup": dumps_function(AllProgress),
+            "function": dumps_function(counts),
+            "interval": interval,
+            "teardown": dumps_function(Scheduler.remove_plugin),
+        }
+    )
     raise gen.Return(comm)
 
 
@@ -64,14 +71,16 @@ def nbytes_bar(nbytes):
     total = sum(nbytes.values())
     names = sorted(nbytes)
 
-    d = {'name': [],
-         'text': [],
-         'left': [],
-         'right': [],
-         'center': [],
-         'color': [],
-         'percent': [],
-         'MB': []}
+    d = {
+        "name": [],
+        "text": [],
+        "left": [],
+        "right": [],
+        "center": [],
+        "color": [],
+        "percent": [],
+        "MB": [],
+    }
 
     if not total:
         return d
@@ -81,17 +90,17 @@ def nbytes_bar(nbytes):
         left = right
         right = nbytes[name] / total + left
         center = (right + left) / 2
-        d['MB'].append(nbytes[name] / 1000000)
-        d['percent'].append(round(nbytes[name] / total * 100, 2))
-        d['left'].append(left)
-        d['right'].append(right)
-        d['center'].append(center)
-        d['color'].append(color_of(name))
-        d['name'].append(name)
+        d["MB"].append(nbytes[name] / 1000000)
+        d["percent"].append(round(nbytes[name] / total * 100, 2))
+        d["left"].append(left)
+        d["right"].append(right)
+        d["center"].append(center)
+        d["color"].append(color_of(name))
+        d["name"].append(name)
         if right - left > 0.1:
-            d['text'].append(name)
+            d["text"].append(name)
         else:
-            d['text'].append('')
+            d["text"].append("")
 
     return d
 
@@ -122,92 +131,98 @@ def progress_quads(msg, nrows=8, ncols=3):
      'processing-loc': [4 / 5, 1 / 1, 1]}}
     """
     width = 0.9
-    names = sorted(msg['all'], key=msg['all'].get, reverse=True)
-    names = names[:nrows * ncols]
+    names = sorted(msg["all"], key=msg["all"].get, reverse=True)
+    names = names[: nrows * ncols]
     n = len(names)
     d = {k: [v.get(name, 0) for name in names] for k, v in msg.items()}
 
-    d['name'] = names
-    d['show-name'] = [name if len(name) <= 15 else name[:12] + '...'
-                      for name in names]
-    d['left'] = [i // nrows for i in range(n)]
-    d['right'] = [i // nrows + width for i in range(n)]
-    d['top'] = [-(i % nrows) for i in range(n)]
-    d['bottom'] = [-(i % nrows) - 0.8 for i in range(n)]
-    d['color'] = [color_of(name) for name in names]
-
-    d['released-loc'] = []
-    d['memory-loc'] = []
-    d['erred-loc'] = []
-    d['processing-loc'] = []
-    d['done'] = []
-    for r, m, e, p, a, l in zip(d['released'], d['memory'], d['erred'],
-                                d['processing'], d['all'], d['left']):
+    d["name"] = names
+    d["show-name"] = [name if len(name) <= 15 else name[:12] + "..." for name in names]
+    d["left"] = [i // nrows for i in range(n)]
+    d["right"] = [i // nrows + width for i in range(n)]
+    d["top"] = [-(i % nrows) for i in range(n)]
+    d["bottom"] = [-(i % nrows) - 0.8 for i in range(n)]
+    d["color"] = [color_of(name) for name in names]
+
+    d["released-loc"] = []
+    d["memory-loc"] = []
+    d["erred-loc"] = []
+    d["processing-loc"] = []
+    d["done"] = []
+    for r, m, e, p, a, l in zip(
+        d["released"], d["memory"], d["erred"], d["processing"], d["all"], d["left"]
+    ):
         rl = width * r / a + l
         ml = width * (r + m) / a + l
         el = width * (r + m + e) / a + l
         pl = width * (p + r + m + e) / a + l
-        done = '%d / %d' % (r + m + e, a)
-        d['released-loc'].append(rl)
-        d['memory-loc'].append(ml)
-        d['erred-loc'].append(el)
-        d['processing-loc'].append(pl)
-        d['done'].append(done)
+        done = "%d / %d" % (r + m + e, a)
+        d["released-loc"].append(rl)
+        d["memory-loc"].append(ml)
+        d["erred-loc"].append(el)
+        d["processing-loc"].append(pl)
+        d["done"].append(done)
 
     return d
 
 
 def color_of_message(msg):
-    if msg['status'] == 'OK':
-        split = key_split(msg['key'])
+    if msg["status"] == "OK":
+        split = key_split(msg["key"])
         return color_of(split)
     else:
-        return 'black'
+        return "black"
 
 
-colors = {'transfer': 'red',
-          'disk-write': 'orange',
-          'disk-read': 'orange',
-          'deserialize': 'gray',
-          'compute': color_of_message}
+colors = {
+    "transfer": "red",
+    "disk-write": "orange",
+    "disk-read": "orange",
+    "deserialize": "gray",
+    "compute": color_of_message,
+}
 
 
-alphas = {'transfer': 0.4,
-          'compute': 1,
-          'deserialize': 0.4,
-          'disk-write': 0.4,
-          'disk-read': 0.4}
+alphas = {
+    "transfer": 0.4,
+    "compute": 1,
+    "deserialize": 0.4,
+    "disk-write": 0.4,
+    "disk-read": 0.4,
+}
 
 
-prefix = {'transfer': 'transfer-',
-          'disk-write': 'disk-write-',
-          'disk-read': 'disk-read-',
-          'deserialize': 'deserialize-',
-          'compute': ''}
+prefix = {
+    "transfer": "transfer-",
+    "disk-write": "disk-write-",
+    "disk-read": "disk-read-",
+    "deserialize": "deserialize-",
+    "compute": "",
+}
 
 
 def task_stream_append(lists, msg, workers):
-    key = msg['key']
+    key = msg["key"]
     name = key_split(key)
-    startstops = msg.get('startstops', [])
+    startstops = msg.get("startstops", [])
 
     for action, start, stop in startstops:
         color = colors[action]
         if type(color) is not str:
             color = color(msg)
 
-        lists['start'].append((start + stop) / 2 * 1000)
-        lists['duration'].append(1000 * (stop - start))
-        lists['key'].append(key)
-        lists['name'].append(prefix[action] + name)
-        lists['color'].append(color)
-        lists['alpha'].append(alphas[action])
-        lists['worker'].append(msg['worker'])
+        lists["start"].append((start + stop) / 2 * 1000)
+        lists["duration"].append(1000 * (stop - start))
+        lists["key"].append(key)
+        lists["name"].append(prefix[action] + name)
+        lists["color"].append(color)
+        lists["alpha"].append(alphas[action])
+        lists["worker"].append(msg["worker"])
 
-        worker_thread = '%s-%d' % (msg['worker'], msg['thread'])
-        lists['worker_thread'].append(worker_thread)
+        worker_thread = "%s-%d" % (msg["worker"], msg["thread"])
+        lists["worker_thread"].append(worker_thread)
         if worker_thread not in workers:
             workers[worker_thread] = len(workers) / 2
-        lists['y'].append(workers[worker_thread])
+        lists["y"].append(workers[worker_thread])
 
     return len(startstops)
diff --git a/distributed/diagnostics/progressbar.py b/distributed/diagnostics/progressbar.py
index 38f784e9cf6..08ba8f7da63 100644
--- a/distributed/diagnostics/progressbar.py
+++ b/distributed/diagnostics/progressbar.py
@@ -15,8 +15,7 @@
 from ..core import connect, coerce_to_address, CommClosedError
 from ..client import default_client, futures_of
 from ..protocol.pickle import dumps
-from ..utils import (ignoring, key_split, is_kernel, LoopRunner,
-        parse_timedelta)
+from ..utils import ignoring, key_split, is_kernel, LoopRunner, parse_timedelta
 
 
 logger = logging.getLogger(__name__)
@@ -29,17 +28,17 @@ def get_scheduler(scheduler):
 
 
 class ProgressBar(object):
-    def __init__(self, keys, scheduler=None, interval='100ms', complete=True):
+    def __init__(self, keys, scheduler=None, interval="100ms", complete=True):
         self.scheduler = get_scheduler(scheduler)
 
         self.client = None
         for key in keys:
-            if hasattr(key, 'client'):
+            if hasattr(key, "client"):
                 self.client = weakref.ref(key.client)
                 break
 
-        self.keys = {k.key if hasattr(k, 'key') else k for k in keys}
-        self.interval = parse_timedelta(interval, default='s')
+        self.keys = {k.key if hasattr(k, "key") else k for k in keys}
+        self.interval = parse_timedelta(interval, default="s")
         self.complete = complete
         self._start_time = default_timer()
 
@@ -59,34 +58,42 @@ def setup(scheduler):
             raise gen.Return(p)
 
         def function(scheduler, p):
-            result = {'all': len(p.all_keys),
-                      'remaining': len(p.keys),
-                      'status': p.status}
-            if p.status == 'error':
+            result = {
+                "all": len(p.all_keys),
+                "remaining": len(p.keys),
+                "status": p.status,
+            }
+            if p.status == "error":
                 result.update(p.extra)
             return result
 
-        self.comm = yield connect(self.scheduler,
-                                  connection_args=self.client().connection_args
-                                  if self.client else None)
+        self.comm = yield connect(
+            self.scheduler,
+            connection_args=self.client().connection_args if self.client else None,
+        )
         logger.debug("Progressbar Connected to scheduler")
 
-        yield self.comm.write({'op': 'feed',
-                               'setup': dumps(setup),
-                               'function': dumps(function),
-                               'interval': self.interval},
-                              serializers=self.client()._serializers if self.client else None)
+        yield self.comm.write(
+            {
+                "op": "feed",
+                "setup": dumps(setup),
+                "function": dumps(function),
+                "interval": self.interval,
+            },
+            serializers=self.client()._serializers if self.client else None,
+        )
 
         while True:
             try:
-                response = yield self.comm.read(deserializers=self.client()._deserializers
-                                                if self.client else None)
+                response = yield self.comm.read(
+                    deserializers=self.client()._deserializers if self.client else None
+                )
             except CommClosedError:
                 break
             self._last_response = response
-            self.status = response['status']
+            self.status = response["status"]
             self._draw_bar(**response)
-            if response['status'] in ('error', 'finished'):
+            if response["status"] in ("error", "finished"):
                 yield self.comm.close()
                 self._draw_stop(**response)
                 break
@@ -102,10 +109,17 @@ def __del__(self):
 
 
 class TextProgressBar(ProgressBar):
-    def __init__(self, keys, scheduler=None, interval='100ms', width=40,
-                 loop=None, complete=True, start=True):
-        super(TextProgressBar, self).__init__(keys, scheduler, interval,
-                                              complete)
+    def __init__(
+        self,
+        keys,
+        scheduler=None,
+        interval="100ms",
+        width=40,
+        loop=None,
+        complete=True,
+        start=True,
+    ):
+        super(TextProgressBar, self).__init__(keys, scheduler, interval, complete)
         self.width = width
         self.loop = loop or IOLoop()
 
@@ -115,17 +129,18 @@ def __init__(self, keys, scheduler=None, interval='100ms', width=40,
 
     def _draw_bar(self, remaining, all, **kwargs):
         frac = (1 - remaining / all) if all else 1.0
-        bar = '#' * int(self.width * frac)
+        bar = "#" * int(self.width * frac)
         percent = int(100 * frac)
         elapsed = format_time(self.elapsed)
-        msg = '\r[{0:<{1}}] | {2}% Completed | {3}'.format(bar, self.width,
-                                                           percent, elapsed)
+        msg = "\r[{0:<{1}}] | {2}% Completed | {3}".format(
+            bar, self.width, percent, elapsed
+        )
         with ignoring(ValueError):
             sys.stdout.write(msg)
             sys.stdout.flush()
 
     def _draw_stop(self, **kwargs):
-        sys.stdout.write('\r')
+        sys.stdout.write("\r")
         sys.stdout.flush()
 
 
@@ -138,15 +153,16 @@ class ProgressWidget(ProgressBar):
     TextProgressBar: Text version suitable for the console
     """
 
-    def __init__(self, keys, scheduler=None, interval='100ms',
-                 complete=False, loop=None):
-        super(ProgressWidget, self).__init__(keys, scheduler, interval,
-                                             complete)
+    def __init__(
+        self, keys, scheduler=None, interval="100ms", complete=False, loop=None
+    ):
+        super(ProgressWidget, self).__init__(keys, scheduler, interval, complete)
 
         from ipywidgets import FloatProgress, HBox, VBox, HTML
-        self.elapsed_time = HTML('')
-        self.bar = FloatProgress(min=0, max=1, description='')
-        self.bar_text = HTML('')
+
+        self.elapsed_time = HTML("")
+        self.bar = FloatProgress(min=0, max=1, description="")
+        self.bar_text = HTML("")
 
         self.bar_widget = HBox([self.bar_text, self.bar])
         self.widget = VBox([self.elapsed_time, self.bar_widget])
@@ -156,38 +172,52 @@ def _ipython_display_(self, **kwargs):
         return self.widget._ipython_display_(**kwargs)
 
     def _draw_stop(self, remaining, status, exception=None, **kwargs):
-        if status == 'error':
-            self.bar.bar_style = 'danger'
+        if status == "error":
+            self.bar.bar_style = "danger"
             self.elapsed_time.value = (
-                    '<div style="padding: 0px 10px 5px 10px"><b>Exception</b> '
-                    '<tt>' + repr(exception) + '</tt>:' +
-                    format_time(self.elapsed) + ' ' +
-                    '</div>'
+                '<div style="padding: 0px 10px 5px 10px"><b>Exception</b> '
+                "<tt>"
+                + repr(exception)
+                + "</tt>:"
+                + format_time(self.elapsed)
+                + " "
+                + "</div>"
             )
         elif not remaining:
-            self.bar.bar_style = 'success'
-            self.elapsed_time.value = '<div style="padding: 0px 10px 5px 10px"><b>Finished:</b> ' + \
-                format_time(self.elapsed) + '</div>'
+            self.bar.bar_style = "success"
+            self.elapsed_time.value = (
+                '<div style="padding: 0px 10px 5px 10px"><b>Finished:</b> '
+                + format_time(self.elapsed)
+                + "</div>"
+            )
 
     def _draw_bar(self, remaining, all, **kwargs):
         ndone = all - remaining
-        self.elapsed_time.value = '<div style=\"padding: 0px 10px 5px 10px\"><b>Computing:</b> ' + \
-            format_time(self.elapsed) + '</div>'
+        self.elapsed_time.value = (
+            '<div style="padding: 0px 10px 5px 10px"><b>Computing:</b> '
+            + format_time(self.elapsed)
+            + "</div>"
+        )
         self.bar.value = ndone / all if all else 1.0
-        self.bar_text.value = '<div style="padding: 0px 10px 0px 10px; text-align:right;">%d / %d</div>' % (ndone, all)
+        self.bar_text.value = (
+            '<div style="padding: 0px 10px 0px 10px; text-align:right;">%d / %d</div>'
+            % (ndone, all)
+        )
 
 
 class MultiProgressBar(object):
-    def __init__(self, keys, scheduler=None, func=key_split, interval='100ms', complete=False):
+    def __init__(
+        self, keys, scheduler=None, func=key_split, interval="100ms", complete=False
+    ):
         self.scheduler = get_scheduler(scheduler)
 
         self.client = None
         for key in keys:
-            if hasattr(key, 'client'):
+            if hasattr(key, "client"):
                 self.client = weakref.ref(key.client)
                 break
 
-        self.keys = {k.key if hasattr(k, 'key') else k for k in keys}
+        self.keys = {k.key if hasattr(k, "key") else k for k in keys}
         self.func = func
         self.interval = interval
         self.complete = complete
@@ -210,30 +240,38 @@ def setup(scheduler):
             raise gen.Return(p)
 
         def function(scheduler, p):
-            result = {'all': valmap(len, p.all_keys),
-                      'remaining': valmap(len, p.keys),
-                      'status': p.status}
-            if p.status == 'error':
+            result = {
+                "all": valmap(len, p.all_keys),
+                "remaining": valmap(len, p.keys),
+                "status": p.status,
+            }
+            if p.status == "error":
                 result.update(p.extra)
             return result
 
-        self.comm = yield connect(self.scheduler,
-                                  connection_args=self.client().connection_args
-                                  if self.client else None)
+        self.comm = yield connect(
+            self.scheduler,
+            connection_args=self.client().connection_args if self.client else None,
+        )
         logger.debug("Progressbar Connected to scheduler")
 
-        yield self.comm.write({'op': 'feed',
-                               'setup': dumps(setup),
-                               'function': dumps(function),
-                               'interval': self.interval})
+        yield self.comm.write(
+            {
+                "op": "feed",
+                "setup": dumps(setup),
+                "function": dumps(function),
+                "interval": self.interval,
+            }
+        )
 
         while True:
-            response = yield self.comm.read(deserializers=self.client()._deserializers if
-                                            self.client else None)
+            response = yield self.comm.read(
+                deserializers=self.client()._deserializers if self.client else None
+            )
             self._last_response = response
-            self.status = response['status']
+            self.status = response["status"]
             self._draw_bar(**response)
-            if response['status'] in ('error', 'finished'):
+            if response["status"] in ("error", "finished"):
                 yield self.comm.close()
                 self._draw_stop(**response)
                 break
@@ -260,26 +298,38 @@ class MultiProgressWidget(MultiProgressBar):
     ProgressWidget: Single progress bar widget
     """
 
-    def __init__(self, keys, scheduler=None, minimum=0, interval=0.1, func=key_split,
-                 complete=False):
-        super(MultiProgressWidget, self).__init__(keys, scheduler, func, interval, complete)
+    def __init__(
+        self,
+        keys,
+        scheduler=None,
+        minimum=0,
+        interval=0.1,
+        func=key_split,
+        complete=False,
+    ):
+        super(MultiProgressWidget, self).__init__(
+            keys, scheduler, func, interval, complete
+        )
         from ipywidgets import VBox
+
         self.widget = VBox([])
 
     def make_widget(self, all):
         from ipywidgets import FloatProgress, HBox, VBox, HTML
-        self.elapsed_time = HTML('')
-        self.bars = {key: FloatProgress(min=0, max=1, description='')
-                     for key in all}
-        self.bar_texts = {key: HTML('') for key in all}
-        self.bar_labels = {key: HTML('<div style=\"padding: 0px 10px 0px 10px;'
-                                     ' text-align:left; word-wrap: '
-                                     'break-word;\">' +
-                                     html_escape(key.decode()
-                                                 if isinstance(key, bytes)
-                                                 else key) +
-                                     '</div>')
-                           for key in all}
+
+        self.elapsed_time = HTML("")
+        self.bars = {key: FloatProgress(min=0, max=1, description="") for key in all}
+        self.bar_texts = {key: HTML("") for key in all}
+        self.bar_labels = {
+            key: HTML(
+                '<div style="padding: 0px 10px 0px 10px;'
+                " text-align:left; word-wrap: "
+                'break-word;">'
+                + html_escape(key.decode() if isinstance(key, bytes) else key)
+                + "</div>"
+            )
+            for key in all
+        }
 
         def keyfunc(kv):
             """ Order keys by most numerous, then by string name """
@@ -287,10 +337,12 @@ def keyfunc(kv):
 
         key_order = [k for k, v in sorted(all.items(), key=keyfunc, reverse=True)]
 
-        self.bar_widgets = VBox([HBox([self.bar_texts[key],
-                                       self.bars[key],
-                                       self.bar_labels[key]])
-                                 for key in key_order])
+        self.bar_widgets = VBox(
+            [
+                HBox([self.bar_texts[key], self.bars[key], self.bar_labels[key]])
+                for key in key_order
+            ]
+        )
         self.widget.children = (self.elapsed_time, self.bar_widgets)
 
     def _ipython_display_(self, **kwargs):
@@ -300,32 +352,43 @@ def _ipython_display_(self, **kwargs):
     def _draw_stop(self, remaining, status, exception=None, key=None, **kwargs):
         for k, v in remaining.items():
             if not v:
-                self.bars[k].bar_style = 'success'
+                self.bars[k].bar_style = "success"
             else:
-                self.bars[k].bar_style = 'danger'
+                self.bars[k].bar_style = "danger"
 
-        if status == 'error':
+        if status == "error":
             # self.bars[self.func(key)].bar_style = 'danger'  # TODO
             self.elapsed_time.value = (
-                '<div style="padding: 0px 10px 5px 10px"><b>Exception</b> ' +
-                '<tt>' + repr(exception) + '</tt>:' +
-                format_time(self.elapsed) + ' ' +
-                '</div>'
+                '<div style="padding: 0px 10px 5px 10px"><b>Exception</b> '
+                + "<tt>"
+                + repr(exception)
+                + "</tt>:"
+                + format_time(self.elapsed)
+                + " "
+                + "</div>"
             )
         else:
-            self.elapsed_time.value = '<div style="padding: 0px 10px 5px 10px"><b>Finished:</b> ' + \
-                format_time(self.elapsed) + '</div>'
+            self.elapsed_time.value = (
+                '<div style="padding: 0px 10px 5px 10px"><b>Finished:</b> '
+                + format_time(self.elapsed)
+                + "</div>"
+            )
 
     def _draw_bar(self, remaining, all, status, **kwargs):
         if self.keys and not self.widget.children:
             self.make_widget(all)
         for k, ntasks in all.items():
             ndone = ntasks - remaining[k]
-            self.elapsed_time.value = '<div style="padding: 0px 10px 5px 10px"><b>Computing:</b> ' + \
-                format_time(self.elapsed) + '</div>'
+            self.elapsed_time.value = (
+                '<div style="padding: 0px 10px 5px 10px"><b>Computing:</b> '
+                + format_time(self.elapsed)
+                + "</div>"
+            )
             self.bars[k].value = ndone / ntasks if ntasks else 1.0
-            self.bar_texts[k].value = '<div style="padding: 0px 10px 0px 10px; text-align: right">%d / %d</div>' % (
-                ndone, ntasks)
+            self.bar_texts[k].value = (
+                '<div style="padding: 0px 10px 0px 10px; text-align: right">%d / %d</div>'
+                % (ndone, ntasks)
+            )
 
 
 def progress(*futures, **kwargs):
@@ -359,9 +422,9 @@ def progress(*futures, **kwargs):
     >>> progress(futures)  # doctest: +SKIP
     [########################################] | 100% Completed |  1.7s
     """
-    notebook = kwargs.pop('notebook', None)
-    multi = kwargs.pop('multi', True)
-    complete = kwargs.pop('complete', True)
+    notebook = kwargs.pop("notebook", None)
+    multi = kwargs.pop("multi", True)
+    complete = kwargs.pop("complete", True)
     assert not kwargs
 
     futures = futures_of(futures)
diff --git a/distributed/diagnostics/task_stream.py b/distributed/diagnostics/task_stream.py
index fafcbebd5cb..89cacb67c97 100644
--- a/distributed/diagnostics/task_stream.py
+++ b/distributed/diagnostics/task_stream.py
@@ -20,11 +20,11 @@ def __init__(self, scheduler, maxlen=100000):
         self.index = 0
 
     def transition(self, key, start, finish, *args, **kwargs):
-        if start == 'processing':
+        if start == "processing":
             if key not in self.scheduler.tasks:
                 return
-            kwargs['key'] = key
-            if finish == 'memory' or finish == 'erred':
+            kwargs["key"] = key
+            if finish == "memory" or finish == "erred":
                 self.buffer.append(kwargs)
                 self.index += 1
 
@@ -34,7 +34,7 @@ def bisect(target, left, right):
                 return left
 
             mid = (left + right) // 2
-            value = max(stop for _, start, stop in self.buffer[mid]['startstops'])
+            value = max(stop for _, start, stop in self.buffer[mid]["startstops"])
 
             if value < target:
                 return bisect(target, mid + 1, right)
@@ -98,16 +98,15 @@ def rectangles(msgs, workers=None, start_boundary=0):
     L_y = []
 
     for msg in msgs:
-        key = msg['key']
+        key = msg["key"]
         name = key_split(key)
-        startstops = msg.get('startstops', [])
+        startstops = msg.get("startstops", [])
         try:
-            worker_thread = '%s-%d' % (msg['worker'], msg['thread'])
+            worker_thread = "%s-%d" % (msg["worker"], msg["thread"])
         except Exception:
             continue
-            logger.warning("Message contained bad information: %s", msg,
-                           exc_info=True)
-            worker_thread = ''
+            logger.warning("Message contained bad information: %s", msg, exc_info=True)
+            worker_thread = ""
 
         if worker_thread not in workers:
             workers[worker_thread] = len(workers) / 2
@@ -126,46 +125,54 @@ def rectangles(msgs, workers=None, start_boundary=0):
             L_name.append(prefix[action] + name)
             L_color.append(color)
             L_alpha.append(alphas[action])
-            L_worker.append(msg['worker'])
+            L_worker.append(msg["worker"])
             L_worker_thread.append(worker_thread)
             L_y.append(workers[worker_thread])
 
-    return {'start': L_start,
-            'duration': L_duration,
-            'duration_text': L_duration_text,
-            'key': L_key,
-            'name': L_name,
-            'color': L_color,
-            'alpha': L_alpha,
-            'worker': L_worker,
-            'worker_thread': L_worker_thread,
-            'y': L_y}
+    return {
+        "start": L_start,
+        "duration": L_duration,
+        "duration_text": L_duration_text,
+        "key": L_key,
+        "name": L_name,
+        "color": L_color,
+        "alpha": L_alpha,
+        "worker": L_worker,
+        "worker_thread": L_worker_thread,
+        "y": L_y,
+    }
 
 
 def color_of_message(msg):
-    if msg['status'] == 'OK':
-        split = key_split(msg['key'])
+    if msg["status"] == "OK":
+        split = key_split(msg["key"])
         return color_of(split)
     else:
-        return 'black'
-
-
-colors = {'transfer': 'red',
-          'disk-write': 'orange',
-          'disk-read': 'orange',
-          'deserialize': 'gray',
-          'compute': color_of_message}
-
-
-alphas = {'transfer': 0.4,
-          'compute': 1,
-          'deserialize': 0.4,
-          'disk-write': 0.4,
-          'disk-read': 0.4}
-
-
-prefix = {'transfer': 'transfer-',
-          'disk-write': 'disk-write-',
-          'disk-read': 'disk-read-',
-          'deserialize': 'deserialize-',
-          'compute': ''}
+        return "black"
+
+
+colors = {
+    "transfer": "red",
+    "disk-write": "orange",
+    "disk-read": "orange",
+    "deserialize": "gray",
+    "compute": color_of_message,
+}
+
+
+alphas = {
+    "transfer": 0.4,
+    "compute": 1,
+    "deserialize": 0.4,
+    "disk-write": 0.4,
+    "disk-read": 0.4,
+}
+
+
+prefix = {
+    "transfer": "transfer-",
+    "disk-write": "disk-write-",
+    "disk-read": "disk-read-",
+    "deserialize": "deserialize-",
+    "compute": "",
+}
diff --git a/distributed/diagnostics/tests/test_eventstream.py b/distributed/diagnostics/tests/test_eventstream.py
index 7504a59846b..0995d80db26 100644
--- a/distributed/diagnostics/tests/test_eventstream.py
+++ b/distributed/diagnostics/tests/test_eventstream.py
@@ -11,9 +11,9 @@
 from distributed.utils_test import div, gen_cluster
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 3)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 3)
 def test_eventstream(c, s, *workers):
-    pytest.importorskip('bokeh')
+    pytest.importorskip("bokeh")
 
     es = EventStream()
     s.add_plugin(es)
@@ -28,17 +28,18 @@ def test_eventstream(c, s, *workers):
 
     from distributed.bokeh import messages
     from distributed.diagnostics.progress_stream import task_stream_append
-    lists = deepcopy(messages['task-events']['rectangles'])
+
+    lists = deepcopy(messages["task-events"]["rectangles"])
     workers = dict()
     for msg in es.buffer:
         task_stream_append(lists, msg, workers)
 
-    assert len([n for n in lists['name'] if n.startswith('transfer')]) == 2
-    for name, color in zip(lists['name'], lists['color']):
-        if name == 'transfer':
-            assert color == 'red'
+    assert len([n for n in lists["name"] if n.startswith("transfer")]) == 2
+    for name, color in zip(lists["name"], lists["color"]):
+        if name == "transfer":
+            assert color == "red"
 
-    assert any(c == 'black' for c in lists['color'])
+    assert any(c == "black" for c in lists["color"])
 
 
 @gen_cluster(client=True)
diff --git a/distributed/diagnostics/tests/test_graph_layout.py b/distributed/diagnostics/tests/test_graph_layout.py
index 63ecb0c7008..fc8fba8d028 100644
--- a/distributed/diagnostics/tests/test_graph_layout.py
+++ b/distributed/diagnostics/tests/test_graph_layout.py
@@ -45,9 +45,9 @@ def test_states(c, s, a, b):
     yield total
 
     updates = {state for idx, state in gl.state_updates}
-    assert 'memory' in updates
-    assert 'processing' in updates
-    assert 'released' in updates
+    assert "memory" in updates
+    assert "processing" in updates
+    assert "released" in updates
 
 
 @gen_cluster(client=True)
diff --git a/distributed/diagnostics/tests/test_plugin.py b/distributed/diagnostics/tests/test_plugin.py
index afc1f4987d5..b1d5406e052 100644
--- a/distributed/diagnostics/tests/test_plugin.py
+++ b/distributed/diagnostics/tests/test_plugin.py
@@ -7,7 +7,6 @@
 
 @gen_cluster(client=True)
 def test_simple(c, s, a, b):
-
     class Counter(SchedulerPlugin):
         def start(self, scheduler):
             self.scheduler = scheduler
@@ -15,7 +14,7 @@ def start(self, scheduler):
             self.count = 0
 
         def transition(self, key, start, finish, *args, **kwargs):
-            if start == 'processing' and finish == 'memory':
+            if start == "processing" and finish == "memory":
                 self.count += 1
 
     counter = Counter()
@@ -42,11 +41,11 @@ def test_add_remove_worker(s):
     class MyPlugin(SchedulerPlugin):
         def add_worker(self, worker, scheduler):
             assert scheduler is s
-            events.append(('add_worker', worker))
+            events.append(("add_worker", worker))
 
         def remove_worker(self, worker, scheduler):
             assert scheduler is s
-            events.append(('remove_worker', worker))
+            events.append(("remove_worker", worker))
 
     plugin = MyPlugin()
     s.add_plugin(plugin)
@@ -59,11 +58,12 @@ def remove_worker(self, worker, scheduler):
     yield a._close()
     yield b._close()
 
-    assert events == [('add_worker', a.address),
-                      ('add_worker', b.address),
-                      ('remove_worker', a.address),
-                      ('remove_worker', b.address),
-                      ]
+    assert events == [
+        ("add_worker", a.address),
+        ("add_worker", b.address),
+        ("remove_worker", a.address),
+        ("remove_worker", b.address),
+    ]
 
     events[:] = []
     s.remove_plugin(plugin)
diff --git a/distributed/diagnostics/tests/test_progress.py b/distributed/diagnostics/tests/test_progress.py
index 2d88054a34b..d8435cc7ff0 100644
--- a/distributed/diagnostics/tests/test_progress.py
+++ b/distributed/diagnostics/tests/test_progress.py
@@ -8,8 +8,13 @@
 from distributed.client import wait
 from distributed.metrics import time
 from distributed.utils_test import gen_cluster, inc, dec, div, nodebug
-from distributed.diagnostics.progress import (Progress, SchedulerPlugin,
-                                              AllProgress, GroupProgress, MultiProgress)
+from distributed.diagnostics.progress import (
+    Progress,
+    SchedulerPlugin,
+    AllProgress,
+    GroupProgress,
+    MultiProgress,
+)
 
 
 def f(*args):
@@ -37,7 +42,7 @@ def test_many_Progress(c, s, a, b):
     yield z
 
     start = time()
-    while not all(b.status == 'finished' for b in bars):
+    while not all(b.status == "finished" for b in bars):
         yield gen.sleep(0.1)
         assert time() < start + 5
 
@@ -53,19 +58,20 @@ def test_multiprogress(c, s, a, b):
     p = MultiProgress([y2], scheduler=s, complete=True)
     yield p.setup()
 
-    assert p.all_keys == {'f': {f.key for f in [x1, x2, x3]},
-                          'g': {f.key for f in [y1, y2]}}
+    assert p.all_keys == {
+        "f": {f.key for f in [x1, x2, x3]},
+        "g": {f.key for f in [y1, y2]},
+    }
 
     yield x3
 
-    assert p.keys['f'] == set()
+    assert p.keys["f"] == set()
 
     yield y2
 
-    assert p.keys == {'f': set(),
-                      'g': set()}
+    assert p.keys == {"f": set(), "g": set()}
 
-    assert p.status == 'finished'
+    assert p.status == "finished"
 
 
 @gen_cluster(client=True)
@@ -85,9 +91,9 @@ def transition(self, key, start, finish, **kwargs):
 
 def check_bar_completed(capsys, width=40):
     out, err = capsys.readouterr()
-    bar, percent, time = [i.strip() for i in out.split('\r')[-1].split('|')]
-    assert bar == '[' + '#' * width + ']'
-    assert percent == '100% Completed'
+    bar, percent, time = [i.strip() for i in out.split("\r")[-1].split("|")]
+    assert bar == "[" + "#" * width + "]"
+    assert percent == "100% Completed"
 
 
 @gen_cluster(client=True, Worker=Nanny, timeout=None)
@@ -97,20 +103,20 @@ def test_AllProgress(c, s, a, b):
 
     yield wait([x, y, z])
     p = AllProgress(s)
-    assert p.all['inc'] == {x.key, y.key, z.key}
-    assert p.state['memory']['inc'] == {x.key, y.key, z.key}
-    assert p.state['released'] == {}
-    assert p.state['erred'] == {}
-    assert 'inc' in p.nbytes
-    assert isinstance(p.nbytes['inc'], int)
-    assert p.nbytes['inc'] > 0
+    assert p.all["inc"] == {x.key, y.key, z.key}
+    assert p.state["memory"]["inc"] == {x.key, y.key, z.key}
+    assert p.state["released"] == {}
+    assert p.state["erred"] == {}
+    assert "inc" in p.nbytes
+    assert isinstance(p.nbytes["inc"], int)
+    assert p.nbytes["inc"] > 0
 
     yield wait([xx, yy, zz])
-    assert p.all['dec'] == {xx.key, yy.key, zz.key}
-    assert p.state['memory']['dec'] == {xx.key, yy.key, zz.key}
-    assert p.state['released'] == {}
-    assert p.state['erred'] == {}
-    assert p.nbytes['inc'] == p.nbytes['dec']
+    assert p.all["dec"] == {xx.key, yy.key, zz.key}
+    assert p.state["memory"]["dec"] == {xx.key, yy.key, zz.key}
+    assert p.state["released"] == {}
+    assert p.state["erred"] == {}
+    assert p.nbytes["inc"] == p.nbytes["dec"]
 
     t = c.submit(sum, [x, y, z])
     yield t
@@ -118,32 +124,34 @@ def test_AllProgress(c, s, a, b):
     keys = {x.key, y.key, z.key}
     del x, y, z
     import gc
+
     gc.collect()
 
     while any(k in s.who_has for k in keys):
         yield gen.sleep(0.01)
 
-    assert p.state['released']['inc'] == keys
-    assert p.all['inc'] == keys
-    assert p.all['dec'] == {xx.key, yy.key, zz.key}
-    if 'inc' in p.nbytes:
-        assert p.nbytes['inc'] == 0
+    assert p.state["released"]["inc"] == keys
+    assert p.all["inc"] == keys
+    assert p.all["dec"] == {xx.key, yy.key, zz.key}
+    if "inc" in p.nbytes:
+        assert p.nbytes["inc"] == 0
 
     xxx = c.submit(div, 1, 0)
     yield wait([xxx])
-    assert p.state['erred'] == {'div': {xxx.key}}
+    assert p.state["erred"] == {"div": {xxx.key}}
 
     tkey = t.key
     del xx, yy, zz, t
     import gc
+
     gc.collect()
 
     while tkey in s.tasks:
         yield gen.sleep(0.01)
 
     for coll in [p.all, p.nbytes] + list(p.state.values()):
-        assert 'inc' not in coll
-        assert 'dec' not in coll
+        assert "inc" not in coll
+        assert "dec" not in coll
 
     def f(x):
         return x
@@ -151,12 +159,13 @@ def f(x):
     for i in range(4):
         future = c.submit(f, i)
     import gc
+
     gc.collect()
 
     yield gen.sleep(1)
 
     yield wait([future])
-    assert p.state['memory'] == {'f': {future.key}}
+    assert p.state["memory"] == {"f": {future.key}}
 
     yield c._restart()
 
@@ -165,8 +174,8 @@ def f(x):
 
     x = c.submit(div, 1, 2)
     yield wait([x])
-    assert set(p.all) == {'div'}
-    assert all(set(d) == {'div'} for d in p.state.values())
+    assert set(p.all) == {"div"}
+    assert all(set(d) == {"div"} for d in p.state.values())
 
 
 @gen_cluster(client=True, Worker=Nanny)
@@ -174,20 +183,20 @@ def test_AllProgress_lost_key(c, s, a, b, timeout=None):
     p = AllProgress(s)
     futures = c.map(inc, range(5))
     yield wait(futures)
-    assert len(p.state['memory']['inc']) == 5
+    assert len(p.state["memory"]["inc"]) == 5
 
     yield a._close()
     yield b._close()
 
     start = time()
-    while len(p.state['memory']['inc']) > 0:
+    while len(p.state["memory"]["inc"]) > 0:
         yield gen.sleep(0.1)
         assert time() < start + 5
 
 
 @gen_cluster(client=True)
 def test_GroupProgress(c, s, a, b):
-    da = pytest.importorskip('dask.array')
+    da = pytest.importorskip("dask.array")
     fp = GroupProgress(s)
     x = da.ones(100, chunks=10)
     y = x + 1
diff --git a/distributed/diagnostics/tests/test_progress_stream.py b/distributed/diagnostics/tests/test_progress_stream.py
index ce21bb34193..9cf89817f34 100644
--- a/distributed/diagnostics/tests/test_progress_stream.py
+++ b/distributed/diagnostics/tests/test_progress_stream.py
@@ -2,54 +2,64 @@
 
 
 import pytest
-pytest.importorskip('bokeh')
+
+pytest.importorskip("bokeh")
 
 from dask import delayed
 from distributed.client import wait
-from distributed.diagnostics.progress_stream import (progress_quads,
-                                                     nbytes_bar, progress_stream)
+from distributed.diagnostics.progress_stream import (
+    progress_quads,
+    nbytes_bar,
+    progress_stream,
+)
 from distributed.utils_test import div, gen_cluster, inc
 
 
 def test_progress_quads():
-    msg = {'all': {'inc': 5, 'dec': 1, 'add': 4},
-           'memory': {'inc': 2, 'dec': 0, 'add': 1},
-           'erred': {'inc': 0, 'dec': 1, 'add': 0},
-           'released': {'inc': 1, 'dec': 0, 'add': 1},
-           'processing': {'inc': 1, 'dec': 0, 'add': 2}}
+    msg = {
+        "all": {"inc": 5, "dec": 1, "add": 4},
+        "memory": {"inc": 2, "dec": 0, "add": 1},
+        "erred": {"inc": 0, "dec": 1, "add": 0},
+        "released": {"inc": 1, "dec": 0, "add": 1},
+        "processing": {"inc": 1, "dec": 0, "add": 2},
+    }
 
     d = progress_quads(msg, nrows=2)
-    color = d.pop('color')
+    color = d.pop("color")
     assert len(set(color)) == 3
-    expected = {'name': ['inc', 'add', 'dec'],
-                'show-name': ['inc', 'add', 'dec'],
-                'left': [0, 0, 1],
-                'right': [0.9, 0.9, 1.9],
-                'top': [0, -1, 0],
-                'bottom': [-.8, -1.8, -.8],
-                'all': [5, 4, 1],
-                'released': [1, 1, 0],
-                'memory': [2, 1, 0],
-                'erred': [0, 0, 1],
-                'processing': [1, 2, 0],
-                'done': ['3 / 5', '2 / 4', '1 / 1'],
-                'released-loc': [.9 * 1 / 5, .25 * 0.9, 1.0],
-                'memory-loc': [.9 * 3 / 5, .5 * 0.9, 1.0],
-                'erred-loc': [.9 * 3 / 5, .5 * 0.9, 1.9],
-                'processing-loc': [.9 * 4 / 5, 1 * 0.9, 1 * 0.9 + 1]}
+    expected = {
+        "name": ["inc", "add", "dec"],
+        "show-name": ["inc", "add", "dec"],
+        "left": [0, 0, 1],
+        "right": [0.9, 0.9, 1.9],
+        "top": [0, -1, 0],
+        "bottom": [-0.8, -1.8, -0.8],
+        "all": [5, 4, 1],
+        "released": [1, 1, 0],
+        "memory": [2, 1, 0],
+        "erred": [0, 0, 1],
+        "processing": [1, 2, 0],
+        "done": ["3 / 5", "2 / 4", "1 / 1"],
+        "released-loc": [0.9 * 1 / 5, 0.25 * 0.9, 1.0],
+        "memory-loc": [0.9 * 3 / 5, 0.5 * 0.9, 1.0],
+        "erred-loc": [0.9 * 3 / 5, 0.5 * 0.9, 1.9],
+        "processing-loc": [0.9 * 4 / 5, 1 * 0.9, 1 * 0.9 + 1],
+    }
     assert d == expected
 
 
 def test_progress_quads_too_many():
-    keys = ['x-%d' % i for i in range(1000)]
-    msg = {'all': {k: 1 for k in keys},
-           'memory': {k: 0 for k in keys},
-           'erred': {k: 0 for k in keys},
-           'released': {k: 0 for k in keys},
-           'processing': {k: 0 for k in keys}}
+    keys = ["x-%d" % i for i in range(1000)]
+    msg = {
+        "all": {k: 1 for k in keys},
+        "memory": {k: 0 for k in keys},
+        "erred": {k: 0 for k in keys},
+        "released": {k: 0 for k in keys},
+        "processing": {k: 0 for k in keys},
+    }
 
     d = progress_quads(msg, nrows=6, ncols=3)
-    assert len(d['name']) == 6 * 3
+    assert len(d["name"]) == 6 * 3
 
 
 @gen_cluster(client=True)
@@ -65,13 +75,15 @@ def test_progress_stream(c, s, a, b):
 
     comm = yield progress_stream(s.address, interval=0.010)
     msg = yield comm.read()
-    nbytes = msg.pop('nbytes')
-    assert msg == {'all': {'div': 10, 'inc': 5},
-                   'erred': {'div': 1},
-                   'memory': {'div': 9, 'inc': 1},
-                   'released': {'inc': 4},
-                   'processing': {}}
-    assert set(nbytes) == set(msg['all'])
+    nbytes = msg.pop("nbytes")
+    assert msg == {
+        "all": {"div": 10, "inc": 5},
+        "erred": {"div": 1},
+        "memory": {"div": 9, "inc": 1},
+        "released": {"inc": 4},
+        "processing": {},
+    }
+    assert set(nbytes) == set(msg["all"])
     assert all(v > 0 for v in nbytes.values())
 
     assert progress_quads(msg)
@@ -80,29 +92,33 @@ def test_progress_stream(c, s, a, b):
 
 
 def test_nbytes_bar():
-    nbytes = {'inc': 1000, 'dec': 3000}
-    expected = {'name': ['dec', 'inc'],
-                'left': [0, 0.75],
-                'center': [0.375, 0.875],
-                'right': [0.75, 1.0],
-                'percent': [75, 25],
-                'MB': [0.003, 0.001],
-                'text': ['dec', 'inc']}
+    nbytes = {"inc": 1000, "dec": 3000}
+    expected = {
+        "name": ["dec", "inc"],
+        "left": [0, 0.75],
+        "center": [0.375, 0.875],
+        "right": [0.75, 1.0],
+        "percent": [75, 25],
+        "MB": [0.003, 0.001],
+        "text": ["dec", "inc"],
+    }
 
     result = nbytes_bar(nbytes)
-    color = result.pop('color')
+    color = result.pop("color")
     assert len(set(color)) == 2
     assert result == expected
 
 
 def test_progress_quads_many_functions():
-    funcnames = ['fn%d' % i for i in range(1000)]
-    msg = {'all': {fn: 1 for fn in funcnames},
-           'memory': {fn: 1 for fn in funcnames},
-           'erred': {fn: 0 for fn in funcnames},
-           'released': {fn: 0 for fn in funcnames},
-           'processing': {fn: 0 for fn in funcnames}}
+    funcnames = ["fn%d" % i for i in range(1000)]
+    msg = {
+        "all": {fn: 1 for fn in funcnames},
+        "memory": {fn: 1 for fn in funcnames},
+        "erred": {fn: 0 for fn in funcnames},
+        "released": {fn: 0 for fn in funcnames},
+        "processing": {fn: 0 for fn in funcnames},
+    }
 
     d = progress_quads(msg, nrows=2)
-    color = d.pop('color')
+    color = d.pop("color")
     assert len(set(color)) < 100
diff --git a/distributed/diagnostics/tests/test_progressbar.py b/distributed/diagnostics/tests/test_progressbar.py
index 7d75b52eeef..8738cb60e22 100644
--- a/distributed/diagnostics/tests/test_progressbar.py
+++ b/distributed/diagnostics/tests/test_progressbar.py
@@ -7,7 +7,7 @@
 from distributed import Scheduler, Worker
 from distributed.diagnostics.progressbar import TextProgressBar, progress
 from distributed.metrics import time
-from distributed.utils_test import (inc, div, gen_cluster)
+from distributed.utils_test import inc, div, gen_cluster
 from distributed.utils_test import client, loop, cluster_fixture  # noqa: F401
 
 
@@ -17,14 +17,12 @@ def test_text_progressbar(capsys, client):
     client.gather(futures)
 
     start = time()
-    while p.status != 'finished':
+    while p.status != "finished":
         sleep(0.01)
         assert time() - start < 5
 
     check_bar_completed(capsys)
-    assert p._last_response == {'all': 10,
-                                'remaining': 0,
-                                'status': 'finished'}
+    assert p._last_response == {"all": 10, "remaining": 0, "status": "finished"}
     assert p.comm.closed()
 
 
@@ -32,17 +30,19 @@ def test_text_progressbar(capsys, client):
 def test_TextProgressBar_error(c, s, a, b):
     x = c.submit(div, 1, 0)
 
-    progress = TextProgressBar([x.key], scheduler=(s.ip, s.port),
-                               start=False, interval=0.01)
+    progress = TextProgressBar(
+        [x.key], scheduler=(s.ip, s.port), start=False, interval=0.01
+    )
     yield progress.listen()
 
-    assert progress.status == 'error'
+    assert progress.status == "error"
     assert progress.comm.closed()
 
-    progress = TextProgressBar([x.key], scheduler=(s.ip, s.port),
-                               start=False, interval=0.01)
+    progress = TextProgressBar(
+        [x.key], scheduler=(s.ip, s.port), start=False, interval=0.01
+    )
     yield progress.listen()
-    assert progress.status == 'error'
+    assert progress.status == "error"
     assert progress.comm.closed()
 
 
@@ -55,11 +55,12 @@ def f():
         b = Worker(s.ip, s.port, loop=loop, ncores=1)
         yield [a._start(0), b._start(0)]
 
-        progress = TextProgressBar([], scheduler=(s.ip, s.port), start=False,
-                                   interval=0.01)
+        progress = TextProgressBar(
+            [], scheduler=(s.ip, s.port), start=False, interval=0.01
+        )
         yield progress.listen()
 
-        assert progress.status == 'finished'
+        assert progress.status == "finished"
         check_bar_completed(capsys)
 
         yield [a._close(), b._close()]
@@ -72,9 +73,9 @@ def f():
 def check_bar_completed(capsys, width=40):
     out, err = capsys.readouterr()
     # trailing newline so grab next to last line for final state of bar
-    bar, percent, time = [i.strip() for i in out.split('\r')[-2].split('|')]
-    assert bar == '[' + '#' * width + ']'
-    assert percent == '100% Completed'
+    bar, percent, time = [i.strip() for i in out.split("\r")[-2].split("|")]
+    assert bar == "[" + "#" * width + "]"
+    assert percent == "100% Completed"
 
 
 def test_progress_function(client, capsys):
diff --git a/distributed/diagnostics/tests/test_task_stream.py b/distributed/diagnostics/tests/test_task_stream.py
index 51bbc9e1021..366de8d79d5 100644
--- a/distributed/diagnostics/tests/test_task_stream.py
+++ b/distributed/diagnostics/tests/test_task_stream.py
@@ -14,7 +14,7 @@
 from distributed.metrics import time
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 3)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 3)
 def test_TaskStreamPlugin(c, s, *workers):
     es = TaskStreamPlugin(s)
     assert not es.buffer
@@ -29,20 +29,21 @@ def test_TaskStreamPlugin(c, s, *workers):
 
     rects = es.rectangles(0, 10, workers)
     assert workers
-    assert all(n == 'div' for n in rects['name'])
-    assert all(d > 0 for d in rects['duration'])
-    counts = frequencies(rects['color'])
-    assert counts['black'] == 1
+    assert all(n == "div" for n in rects["name"])
+    assert all(d > 0 for d in rects["duration"])
+    counts = frequencies(rects["color"])
+    assert counts["black"] == 1
     assert set(counts.values()) == {9, 1}
-    assert len(set(rects['y'])) == 3
+    assert len(set(rects["y"])) == 3
 
     rects = es.rectangles(2, 5, workers)
     assert all(len(L) == 3 for L in rects.values())
 
-    starts = sorted(rects['start'])
-    rects = es.rectangles(2, 5, workers=workers,
-                          start_boundary=(starts[0] + starts[1]) / 2000)
-    assert set(rects['start']).issubset(set(starts[1:]))
+    starts = sorted(rects["start"])
+    rects = es.rectangles(
+        2, 5, workers=workers, start_boundary=(starts[0] + starts[1]) / 2000
+    )
+    assert set(rects["start"]).issubset(set(starts[1:]))
 
 
 @gen_cluster(client=True)
@@ -68,10 +69,10 @@ def test_collect(c, s, a, b):
     L = tasks.collect(start=start + 0.2)
     assert 4 <= len(L) <= len(futures)
 
-    L = tasks.collect(start='20 s')
+    L = tasks.collect(start="20 s")
     assert len(L) == len(futures)
 
-    L = tasks.collect(start='500ms')
+    L = tasks.collect(start="500ms")
     assert 0 < len(L) <= len(futures)
 
     L = tasks.collect(count=3)
@@ -107,7 +108,7 @@ def test_client_sync(client):
 
 @gen_cluster(client=True)
 def test_get_task_stream_plot(c, s, a, b):
-    bokeh = pytest.importorskip('bokeh')
+    bokeh = pytest.importorskip("bokeh")
     yield c.get_task_stream()
 
     futures = c.map(slowinc, range(10), delay=0.1)
@@ -118,15 +119,15 @@ def test_get_task_stream_plot(c, s, a, b):
 
 
 def test_get_task_stream_save(client, tmpdir):
-    bokeh = pytest.importorskip('bokeh')
+    bokeh = pytest.importorskip("bokeh")
     tmpdir = str(tmpdir)
-    fn = os.path.join(tmpdir, 'foo.html')
+    fn = os.path.join(tmpdir, "foo.html")
 
-    with get_task_stream(plot='save', filename=fn) as ts:
+    with get_task_stream(plot="save", filename=fn) as ts:
         wait(client.map(inc, range(10)))
     with open(fn) as f:
         data = f.read()
-    assert 'inc' in data
-    assert 'bokeh' in data
+    assert "inc" in data
+    assert "bokeh" in data
 
     assert isinstance(ts.figure, bokeh.plotting.Figure)
diff --git a/distributed/diagnostics/tests/test_widgets.py b/distributed/diagnostics/tests/test_widgets.py
index 6ae29161a9c..033d49251cb 100644
--- a/distributed/diagnostics/tests/test_widgets.py
+++ b/distributed/diagnostics/tests/test_widgets.py
@@ -1,7 +1,8 @@
 from __future__ import print_function, division, absolute_import
 
 import pytest
-pytest.importorskip('ipywidgets')
+
+pytest.importorskip("ipywidgets")
 
 from ipykernel.comm import Comm
 from ipywidgets import Widget
@@ -17,7 +18,7 @@
 
 
 class DummyComm(Comm):
-    comm_id = 'a-b-c-d'
+    comm_id = "a-b-c-d"
 
     def open(self, *args, **kwargs):
         pass
@@ -35,12 +36,13 @@ def close(self, *args, **kwargs):
 
 
 def setup():
-    _widget_attrs['_comm_default'] = getattr(Widget, '_comm_default', undefined)
+    _widget_attrs["_comm_default"] = getattr(Widget, "_comm_default", undefined)
     Widget._comm_default = lambda self: DummyComm()
-    _widget_attrs['_ipython_display_'] = Widget._ipython_display_
+    _widget_attrs["_ipython_display_"] = Widget._ipython_display_
 
     def raise_not_implemented(*args, **kwargs):
         raise NotImplementedError()
+
     Widget._ipython_display_ = raise_not_implemented
 
 
@@ -78,11 +80,13 @@ def record_display(*args):
 
 from distributed.client import wait
 from distributed.worker import dumps_task
-from distributed.utils_test import (inc, dec, throws, gen_cluster,
-        gen_tls_cluster)
+from distributed.utils_test import inc, dec, throws, gen_cluster, gen_tls_cluster
 from distributed.utils_test import client, loop, cluster_fixture  # noqa: F401
-from distributed.diagnostics.progressbar import (ProgressWidget,
-                                                 MultiProgressWidget, progress)
+from distributed.diagnostics.progressbar import (
+    ProgressWidget,
+    MultiProgressWidget,
+    progress,
+)
 
 
 @gen_cluster(client=True)
@@ -96,7 +100,7 @@ def test_progressbar_widget(c, s, a, b):
     yield progress.listen()
 
     assert progress.bar.value == 1.0
-    assert '3 / 3' in progress.bar_text.value
+    assert "3 / 3" in progress.bar_text.value
 
     progress = ProgressWidget([z.key], scheduler=s.address)
     yield progress.listen()
@@ -116,49 +120,61 @@ def test_multi_progressbar_widget(c, s, a, b):
     p = MultiProgressWidget([e.key], scheduler=s.address, complete=True)
     yield p.listen()
 
-    assert p.bars['inc'].value == 1.0
-    assert p.bars['dec'].value == 1.0
-    assert p.bars['throws'].value == 0.0
-    assert '3 / 3' in p.bar_texts['inc'].value
-    assert '2 / 2' in p.bar_texts['dec'].value
-    assert '0 / 1' in p.bar_texts['throws'].value
+    assert p.bars["inc"].value == 1.0
+    assert p.bars["dec"].value == 1.0
+    assert p.bars["throws"].value == 0.0
+    assert "3 / 3" in p.bar_texts["inc"].value
+    assert "2 / 2" in p.bar_texts["dec"].value
+    assert "0 / 1" in p.bar_texts["throws"].value
 
-    assert p.bars['inc'].bar_style == 'success'
-    assert p.bars['dec'].bar_style == 'success'
-    assert p.bars['throws'].bar_style == 'danger'
+    assert p.bars["inc"].bar_style == "success"
+    assert p.bars["dec"].bar_style == "success"
+    assert p.bars["throws"].bar_style == "danger"
 
-    assert p.status == 'error'
-    assert 'Exception' in p.elapsed_time.value
+    assert p.status == "error"
+    assert "Exception" in p.elapsed_time.value
 
     try:
         throws(1)
     except Exception as e:
         assert repr(e) in p.elapsed_time.value
 
-    capacities = [int(re.search(r'\d+ / \d+', row.children[0].value)
-                      .group().split(' / ')[1])
-                  for row in p.bar_widgets.children]
+    capacities = [
+        int(re.search(r"\d+ / \d+", row.children[0].value).group().split(" / ")[1])
+        for row in p.bar_widgets.children
+    ]
     assert sorted(capacities, reverse=True) == capacities
 
 
 @gen_cluster()
 def test_multi_progressbar_widget_after_close(s, a, b):
-    s.update_graph(tasks=valmap(dumps_task, {'x-1': (inc, 1),
-                                             'x-2': (inc, 'x-1'),
-                                             'x-3': (inc, 'x-2'),
-                                             'y-1': (dec, 'x-3'),
-                                             'y-2': (dec, 'y-1'),
-                                             'e': (throws, 'y-2'),
-                                             'other': (inc, 123)}),
-                   keys=['e'],
-                   dependencies={'x-2': {'x-1'}, 'x-3': {'x-2'},
-                                 'y-1': {'x-3'}, 'y-2': {'y-1'},
-                                 'e': {'y-2'}})
-
-    p = MultiProgressWidget(['x-1', 'x-2', 'x-3'], scheduler=s.address)
+    s.update_graph(
+        tasks=valmap(
+            dumps_task,
+            {
+                "x-1": (inc, 1),
+                "x-2": (inc, "x-1"),
+                "x-3": (inc, "x-2"),
+                "y-1": (dec, "x-3"),
+                "y-2": (dec, "y-1"),
+                "e": (throws, "y-2"),
+                "other": (inc, 123),
+            },
+        ),
+        keys=["e"],
+        dependencies={
+            "x-2": {"x-1"},
+            "x-3": {"x-2"},
+            "y-1": {"x-3"},
+            "y-2": {"y-1"},
+            "e": {"y-2"},
+        },
+    )
+
+    p = MultiProgressWidget(["x-1", "x-2", "x-3"], scheduler=s.address)
     yield p.listen()
 
-    assert 'x' in p.bars
+    assert "x" in p.bars
 
 
 def test_values(client):
@@ -166,16 +182,16 @@ def test_values(client):
     wait(L)
     p = MultiProgressWidget(L)
     client.sync(p.listen)
-    assert set(p.bars) == {'inc'}
-    assert p.status == 'finished'
+    assert set(p.bars) == {"inc"}
+    assert p.status == "finished"
     assert p.comm.closed()
-    assert '5 / 5' in p.bar_texts['inc'].value
-    assert p.bars['inc'].value == 1.0
+    assert "5 / 5" in p.bar_texts["inc"].value
+    assert p.bars["inc"].value == 1.0
 
     x = client.submit(throws, 1)
     p = MultiProgressWidget([x])
     client.sync(p.listen)
-    assert p.status == 'error'
+    assert p.status == "error"
 
 
 def test_progressbar_done(client):
@@ -183,20 +199,20 @@ def test_progressbar_done(client):
     wait(L)
     p = ProgressWidget(L)
     client.sync(p.listen)
-    assert p.status == 'finished'
+    assert p.status == "finished"
     assert p.bar.value == 1.0
-    assert p.bar.bar_style == 'success'
-    assert 'Finished' in p.elapsed_time.value
+    assert p.bar.bar_style == "success"
+    assert "Finished" in p.elapsed_time.value
 
     f = client.submit(throws, L)
     wait([f])
 
     p = ProgressWidget([f])
     client.sync(p.listen)
-    assert p.status == 'error'
+    assert p.status == "error"
     assert p.bar.value == 0.0
-    assert p.bar.bar_style == 'danger'
-    assert 'Exception' in p.elapsed_time.value
+    assert p.bar.bar_style == "danger"
+    assert "Exception" in p.elapsed_time.value
 
     try:
         throws(1)
@@ -206,36 +222,48 @@ def test_progressbar_done(client):
 
 def test_progressbar_cancel(client):
     import time
+
     L = [client.submit(lambda: time.sleep(0.3), i) for i in range(5)]
     p = ProgressWidget(L)
     client.sync(p.listen)
     L[-1].cancel()
     wait(L[:-1])
-    assert p.status == 'error'
+    assert p.status == "error"
     assert p.bar.value == 0  # no tasks finish before cancel is called
 
 
 @gen_cluster()
 def test_multibar_complete(s, a, b):
-    s.update_graph(tasks=valmap(dumps_task, {'x-1': (inc, 1),
-                                             'x-2': (inc, 'x-1'),
-                                             'x-3': (inc, 'x-2'),
-                                             'y-1': (dec, 'x-3'),
-                                             'y-2': (dec, 'y-1'),
-                                             'e': (throws, 'y-2'),
-                                             'other': (inc, 123)}),
-                   keys=['e'],
-                   dependencies={'x-2': {'x-1'}, 'x-3': {'x-2'},
-                                 'y-1': {'x-3'}, 'y-2': {'y-1'},
-                                 'e': {'y-2'}})
-
-    p = MultiProgressWidget(['e'], scheduler=s.address, complete=True)
+    s.update_graph(
+        tasks=valmap(
+            dumps_task,
+            {
+                "x-1": (inc, 1),
+                "x-2": (inc, "x-1"),
+                "x-3": (inc, "x-2"),
+                "y-1": (dec, "x-3"),
+                "y-2": (dec, "y-1"),
+                "e": (throws, "y-2"),
+                "other": (inc, 123),
+            },
+        ),
+        keys=["e"],
+        dependencies={
+            "x-2": {"x-1"},
+            "x-3": {"x-2"},
+            "y-1": {"x-3"},
+            "y-2": {"y-1"},
+            "e": {"y-2"},
+        },
+    )
+
+    p = MultiProgressWidget(["e"], scheduler=s.address, complete=True)
     yield p.listen()
 
-    assert p._last_response['all'] == {'x': 3, 'y': 2, 'e': 1}
-    assert all(b.value == 1.0 for k, b in p.bars.items() if k != 'e')
-    assert '3 / 3' in p.bar_texts['x'].value
-    assert '2 / 2' in p.bar_texts['y'].value
+    assert p._last_response["all"] == {"x": 3, "y": 2, "e": 1}
+    assert all(b.value == 1.0 for k, b in p.bars.items() if k != "e")
+    assert "3 / 3" in p.bar_texts["x"].value
+    assert "2 / 2" in p.bar_texts["y"].value
 
 
 def test_fast(client):
@@ -244,10 +272,10 @@ def test_fast(client):
     L3 = client.map(add, L, L2)
     p = progress(L3, multi=True, complete=True, notebook=True)
     client.sync(p.listen)
-    assert set(p._last_response['all']) == {'inc', 'dec', 'add'}
+    assert set(p._last_response["all"]) == {"inc", "dec", "add"}
 
 
-@gen_cluster(client=True, client_kwargs={'serializers': ['msgpack']})
+@gen_cluster(client=True, client_kwargs={"serializers": ["msgpack"]})
 def test_serializers(c, s, a, b):
     x = c.submit(inc, 1)
     y = c.submit(inc, x)
@@ -258,7 +286,7 @@ def test_serializers(c, s, a, b):
     yield progress.listen()
 
     assert progress.bar.value == 1.0
-    assert '3 / 3' in progress.bar_text.value
+    assert "3 / 3" in progress.bar_text.value
 
 
 @gen_tls_cluster(client=True)
@@ -272,4 +300,4 @@ def test_tls(c, s, a, b):
     yield progress.listen()
 
     assert progress.bar.value == 1.0
-    assert '3 / 3' in progress.bar_text.value
+    assert "3 / 3" in progress.bar_text.value
diff --git a/distributed/diskutils.py b/distributed/diskutils.py
index ccc3096c038..395f7828505 100644
--- a/distributed/diskutils.py
+++ b/distributed/diskutils.py
@@ -16,11 +16,11 @@
 
 logger = logging.getLogger(__name__)
 
-DIR_LOCK_EXT = '.dirlock'
+DIR_LOCK_EXT = ".dirlock"
 
 
 def is_locking_enabled():
-    return dask.config.get('distributed.worker.use-file-locking')
+    return dask.config.get("distributed.worker.use-file-locking")
 
 
 def safe_unlink(path):
@@ -58,24 +58,33 @@ def __init__(self, workspace, name=None, prefix=None):
                         self._lock_file = locket.lock_file(self._lock_path)
                         self._lock_file.acquire()
                 except OSError as e:
-                    logger.exception("Could not acquire workspace lock on "
-                                     "path: %s ."
-                                     "Continuing without lock. "
-                                     "This may result in workspaces not being "
-                                     "cleaned up", self._lock_path,
-                                     exc_info=True)
+                    logger.exception(
+                        "Could not acquire workspace lock on "
+                        "path: %s ."
+                        "Continuing without lock. "
+                        "This may result in workspaces not being "
+                        "cleaned up",
+                        self._lock_path,
+                        exc_info=True,
+                    )
                     self._lock_file = None
             except Exception:
                 shutil.rmtree(self.dir_path, ignore_errors=True)
                 raise
             workspace._known_locks.add(self._lock_path)
 
-            self._finalizer = finalize(self, self._finalize,
-                                       workspace, self._lock_path,
-                                       self._lock_file, self.dir_path)
+            self._finalizer = finalize(
+                self,
+                self._finalize,
+                workspace,
+                self._lock_path,
+                self._lock_file,
+                self.dir_path,
+            )
         else:
-            self._finalizer = finalize(self, self._finalize,
-                                       workspace, None, None, self.dir_path)
+            self._finalizer = finalize(
+                self, self._finalize, workspace, None, None, self.dir_path
+            )
 
     def release(self):
         """
@@ -109,8 +118,8 @@ class WorkSpace(object):
     def __init__(self, base_dir):
         self.base_dir = os.path.abspath(base_dir)
         self._init_workspace()
-        self._global_lock_path = os.path.join(self.base_dir, 'global.lock')
-        self._purge_lock_path = os.path.join(self.base_dir, 'purge.lock')
+        self._global_lock_path = os.path.join(self.base_dir, "global.lock")
+        self._purge_lock_path = os.path.join(self.base_dir, "purge.lock")
 
     def _init_workspace(self):
         try:
@@ -165,7 +174,7 @@ def _purge_leftovers(self):
         return purged
 
     def _list_unknown_locks(self):
-        for p in glob.glob(os.path.join(self.base_dir, '*' + DIR_LOCK_EXT)):
+        for p in glob.glob(os.path.join(self.base_dir, "*" + DIR_LOCK_EXT)):
             try:
                 st = os.stat(p)
             except EnvironmentError:
@@ -199,10 +208,9 @@ def _check_lock_or_purge(self, lock_path):
             return False
         try:
             # Lock file is stale, therefore purge corresponding directory
-            dir_path = lock_path[:-len(DIR_LOCK_EXT)]
+            dir_path = lock_path[: -len(DIR_LOCK_EXT)]
             if os.path.exists(dir_path):
-                logger.info("Found stale lock file and directory %r, purging",
-                            dir_path)
+                logger.info("Found stale lock file and directory %r, purging", dir_path)
                 self._purge_directory(dir_path)
         finally:
             lock.release()
@@ -212,8 +220,7 @@ def _check_lock_or_purge(self, lock_path):
 
     def _on_remove_error(self, func, path, exc_info):
         typ, exc, tb = exc_info
-        logger.error("Failed to remove %r (failed in %r): %s",
-                     path, func, str(exc))
+        logger.error("Failed to remove %r (failed in %r): %s", path, func, str(exc))
 
     def new_work_dir(self, **kwargs):
         """
@@ -231,6 +238,8 @@ def new_work_dir(self, **kwargs):
         try:
             self._purge_leftovers()
         except OSError:
-            logger.error("Failed to clean up lingering worker directories "
-                         "in path: %s ", exc_info=True)
+            logger.error(
+                "Failed to clean up lingering worker directories " "in path: %s ",
+                exc_info=True,
+            )
         return WorkDir(self, **kwargs)
diff --git a/distributed/lock.py b/distributed/lock.py
index 9f1c4390653..d12b1c41e15 100644
--- a/distributed/lock.py
+++ b/distributed/lock.py
@@ -29,10 +29,11 @@ def __init__(self, scheduler):
         self.events = defaultdict(deque)
         self.ids = dict()
 
-        self.scheduler.handlers.update({'lock_acquire': self.acquire,
-                                        'lock_release': self.release})
+        self.scheduler.handlers.update(
+            {"lock_acquire": self.acquire, "lock_release": self.release}
+        )
 
-        self.scheduler.extensions['locks'] = self
+        self.scheduler.extensions["locks"] = self
 
     @gen.coroutine
     def acquire(self, stream=None, name=None, id=None, timeout=None):
@@ -92,9 +93,10 @@ class Lock(object):
     >>> # do things with protected resource
     >>> lock.release()  # doctest: +SKIP
     """
+
     def __init__(self, name=None, client=None):
         self.client = client or _get_global_client() or get_worker().client
-        self.name = name or 'lock-' + uuid.uuid4().hex
+        self.name = name or "lock-" + uuid.uuid4().hex
         self.id = uuid.uuid4().hex
         self._locked = False
 
@@ -121,12 +123,15 @@ def acquire(self, blocking=True, timeout=None):
         """
         if not blocking:
             if timeout is not None:
-                raise ValueError(
-                    "can't specify a timeout for a non-blocking call")
+                raise ValueError("can't specify a timeout for a non-blocking call")
             timeout = 0
 
-        result = self.client.sync(self.client.scheduler.lock_acquire,
-                                  name=self.name, id=self.id, timeout=timeout)
+        result = self.client.sync(
+            self.client.scheduler.lock_acquire,
+            name=self.name,
+            id=self.id,
+            timeout=timeout,
+        )
         self._locked = True
         return result
 
@@ -134,8 +139,9 @@ def release(self):
         """ Release the lock if already acquired """
         if not self.locked():
             raise ValueError("Lock is not yet acquired")
-        result = self.client.sync(self.client.scheduler.lock_release,
-                                  name=self.name, id=self.id)
+        result = self.client.sync(
+            self.client.scheduler.lock_release, name=self.name, id=self.id
+        )
         self._locked = False
         return result
 
diff --git a/distributed/locket.py b/distributed/locket.py
index 84f4af2ca1f..1ed7b023085 100644
--- a/distributed/locket.py
+++ b/distributed/locket.py
@@ -19,16 +19,22 @@
         import ctypes.wintypes
         import msvcrt
     except ImportError:
-        raise ImportError("Platform not supported (failed to import fcntl, ctypes, msvcrt)")
+        raise ImportError(
+            "Platform not supported (failed to import fcntl, ctypes, msvcrt)"
+        )
     else:
-        _kernel32 = ctypes.WinDLL('kernel32', use_last_error=True)
+        _kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
         _WinAPI_LockFile = _kernel32.LockFile
         _WinAPI_LockFile.restype = ctypes.wintypes.BOOL
-        _WinAPI_LockFile.argtypes = [ctypes.wintypes.HANDLE] + [ctypes.wintypes.DWORD] * 4
+        _WinAPI_LockFile.argtypes = [ctypes.wintypes.HANDLE] + [
+            ctypes.wintypes.DWORD
+        ] * 4
 
         _WinAPI_UnlockFile = _kernel32.UnlockFile
         _WinAPI_UnlockFile.restype = ctypes.wintypes.BOOL
-        _WinAPI_UnlockFile.argtypes = [ctypes.wintypes.HANDLE] + [ctypes.wintypes.DWORD] * 4
+        _WinAPI_UnlockFile.argtypes = [ctypes.wintypes.HANDLE] + [
+            ctypes.wintypes.DWORD
+        ] * 4
 
         _lock_file_blocking_available = False
 
@@ -46,8 +52,10 @@ def _lock_file_non_blocking(file_):
         def _unlock_file(file_):
             _WinAPI_UnlockFile(msvcrt.get_osfhandle(file_.fileno()), 0, 0, 1, 0)
 
+
 else:
     _lock_file_blocking_available = True
+
     def _lock_file_blocking(file_):
         fcntl.flock(file_.fileno(), fcntl.LOCK_EX)
 
@@ -100,8 +108,7 @@ def _acquire_non_blocking(acquire, timeout, retry_period, path):
         success = acquire()
         if success:
             return
-        elif (timeout is not None and
-                time.time() - start_time > timeout):
+        elif timeout is not None and time.time() - start_time > timeout:
             raise LockError("Couldn't lock {0}".format(path))
         else:
             time.sleep(retry_period)
@@ -179,6 +186,7 @@ class _Locker(object):
     A lock wrapper to always apply the given *timeout* and *retry_period*
     to acquire() calls.
     """
+
     def __init__(self, lock, timeout=None, retry_period=None):
         self._lock = lock
         self._timeout = timeout
diff --git a/distributed/metrics.py b/distributed/metrics.py
index fb047faec79..6c0bdb4dc7e 100755
--- a/distributed/metrics.py
+++ b/distributed/metrics.py
@@ -83,7 +83,7 @@ def resync(self):
 
 
 # A high-resolution wall clock timer measuring the seconds since Unix epoch
-if sys.platform.startswith('win'):
+if sys.platform.startswith("win"):
     time = _WindowsTime().time
 else:
     # Under modern Unices, time.time() should be good enough
@@ -97,7 +97,7 @@ def _native_thread_time():
 
 def _linux_thread_time():
     # Use hardcoded CLOCK_THREAD_CPUTIME_ID on Python 3 <= 3.6
-    if sys.platform != 'linux':
+    if sys.platform != "linux":
         raise OSError
     return timemod.clock_gettime(3)
 
@@ -134,8 +134,7 @@ def _detect_thread_time():
     Return a per-thread CPU timer function if possible, otherwise
     a per-process CPU timer function, or at worse a wall-clock timer.
     """
-    for func in [_native_thread_time, _linux_thread_time,
-                 _native_process_time]:
+    for func in [_native_thread_time, _linux_thread_time, _native_process_time]:
         try:
             func()
             return func
diff --git a/distributed/nanny.py b/distributed/nanny.py
index c1965759685..356ebc3168d 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -21,8 +21,13 @@
 from .process import AsyncProcess
 from .proctitle import enable_proctitle_on_children
 from .security import Security
-from .utils import (get_ip, mp_context, silence_logging, json_load_robust,
-        PeriodicCallback)
+from .utils import (
+    get_ip,
+    mp_context,
+    silence_logging,
+    json_load_robust,
+    PeriodicCallback,
+)
 from .worker import _ncores, run, parse_memory_limit, Worker
 
 logger = logging.getLogger(__name__)
@@ -34,22 +39,43 @@ class Nanny(ServerNode):
     The nanny spins up Worker processes, watches then, and kills or restarts
     them as necessary.
     """
+
     process = None
     status = None
 
-    def __init__(self, scheduler_ip=None, scheduler_port=None,
-            scheduler_file=None, worker_port=0, ncores=None, loop=None,
-            local_dir='dask-worker-space', services=None, name=None,
-            memory_limit='auto', reconnect=True, validate=False, quiet=False,
-            resources=None, silence_logs=None, death_timeout=None, preload=(),
-            preload_argv=[], security=None, contact_address=None,
-            listen_address=None, worker_class=None, env=None, **worker_kwargs):
+    def __init__(
+        self,
+        scheduler_ip=None,
+        scheduler_port=None,
+        scheduler_file=None,
+        worker_port=0,
+        ncores=None,
+        loop=None,
+        local_dir="dask-worker-space",
+        services=None,
+        name=None,
+        memory_limit="auto",
+        reconnect=True,
+        validate=False,
+        quiet=False,
+        resources=None,
+        silence_logs=None,
+        death_timeout=None,
+        preload=(),
+        preload_argv=[],
+        security=None,
+        contact_address=None,
+        listen_address=None,
+        worker_class=None,
+        env=None,
+        **worker_kwargs
+    ):
 
         if scheduler_file:
             cfg = json_load_robust(scheduler_file)
-            self.scheduler_addr = cfg['address']
-        elif scheduler_ip is None and dask.config.get('scheduler-address'):
-            self.scheduler_addr = dask.config.get('scheduler-address')
+            self.scheduler_addr = cfg["address"]
+        elif scheduler_ip is None and dask.config.get("scheduler-address"):
+            self.scheduler_addr = dask.config.get("scheduler-address")
         elif scheduler_port is None:
             self.scheduler_addr = coerce_to_address(scheduler_ip)
         else:
@@ -67,12 +93,14 @@ def __init__(self, scheduler_ip=None, scheduler_port=None,
         self.worker_kwargs = worker_kwargs
 
         self.contact_address = contact_address
-        self.memory_terminate_fraction = dask.config.get('distributed.worker.memory.terminate')
+        self.memory_terminate_fraction = dask.config.get(
+            "distributed.worker.memory.terminate"
+        )
 
         self.security = security or Security()
         assert isinstance(self.security, Security)
-        self.connection_args = self.security.get_connection_args('worker')
-        self.listen_args = self.security.get_listen_args('worker')
+        self.connection_args = self.security.get_connection_args("worker")
+        self.listen_args = self.security.get_listen_args("worker")
 
         self.local_dir = local_dir
 
@@ -89,22 +117,25 @@ def __init__(self, scheduler_ip=None, scheduler_port=None,
             silence_logging(level=silence_logs)
         self.silence_logs = silence_logs
 
-        handlers = {'instantiate': self.instantiate,
-                    'kill': self.kill,
-                    'restart': self.restart,
-                    # cannot call it 'close' on the rpc side for naming conflict
-                    'terminate': self._close,
-                    'run': self.run}
-
-        super(Nanny, self).__init__(handlers, io_loop=self.loop,
-                                    connection_args=self.connection_args)
+        handlers = {
+            "instantiate": self.instantiate,
+            "kill": self.kill,
+            "restart": self.restart,
+            # cannot call it 'close' on the rpc side for naming conflict
+            "terminate": self._close,
+            "run": self.run,
+        }
+
+        super(Nanny, self).__init__(
+            handlers, io_loop=self.loop, connection_args=self.connection_args
+        )
 
         if self.memory_limit:
             pc = PeriodicCallback(self.memory_monitor, 100, io_loop=self.loop)
-            self.periodic_callbacks['memory'] = pc
+            self.periodic_callbacks["memory"] = pc
 
         self._listen_address = listen_address
-        self.status = 'init'
+        self.status = "init"
 
     def __repr__(self):
         return "<Nanny: %s, threads: %d>" % (self.worker_address, self.ncores)
@@ -117,11 +148,18 @@ def _unregister(self, timeout=10):
         if worker_address is None:
             return
 
-        allowed_errors = (gen.TimeoutError, CommClosedError, EnvironmentError, RPCClosed)
+        allowed_errors = (
+            gen.TimeoutError,
+            CommClosedError,
+            EnvironmentError,
+            RPCClosed,
+        )
         try:
-            yield gen.with_timeout(timedelta(seconds=timeout),
-                                   self.scheduler.unregister(address=self.worker_address),
-                                   quiet_exceptions=allowed_errors)
+            yield gen.with_timeout(
+                timedelta(seconds=timeout),
+                self.scheduler.unregister(address=self.worker_address),
+                quiet_exceptions=allowed_errors,
+            )
         except allowed_errors:
             pass
 
@@ -140,25 +178,24 @@ def _start(self, addr_or_port=0):
         # XXX Factor this out
         if not addr_or_port:
             # Default address is the required one to reach the scheduler
-            self.listen(get_local_address_for(self.scheduler.address),
-                        listen_args=self.listen_args)
+            self.listen(
+                get_local_address_for(self.scheduler.address),
+                listen_args=self.listen_args,
+            )
             self.ip = get_address_host(self.address)
         elif isinstance(addr_or_port, int):
             # addr_or_port is an integer => assume TCP
-            self.ip = get_ip(
-                get_address_host(self.scheduler.address)
-            )
-            self.listen((self.ip, addr_or_port),
-                        listen_args=self.listen_args)
+            self.ip = get_ip(get_address_host(self.scheduler.address))
+            self.listen((self.ip, addr_or_port), listen_args=self.listen_args)
         else:
             self.listen(addr_or_port, listen_args=self.listen_args)
             self.ip = get_address_host(self.address)
 
-        logger.info('        Start Nanny at: %r', self.address)
+        logger.info("        Start Nanny at: %r", self.address)
         response = yield self.instantiate()
-        if response == 'running':
+        if response == "running":
             assert self.worker_address
-            self.status = 'running'
+            self.status = "running"
         else:
             yield self._close()
 
@@ -181,7 +218,7 @@ def kill(self, comm=None, timeout=2):
         """
         self.auto_restart = False
         if self.process is None:
-            raise gen.Return('OK')
+            raise gen.Return("OK")
 
         deadline = self.loop.time() + timeout
         yield self.process.kill(timeout=0.8 * (deadline - self.loop.time()))
@@ -197,26 +234,29 @@ def instantiate(self, comm=None):
             start_arg = self._listen_address
         else:
             host = self.listener.bound_address[0]
-            start_arg = self.listener.prefix + unparse_host_port(host,
-                    self._given_worker_port)
+            start_arg = self.listener.prefix + unparse_host_port(
+                host, self._given_worker_port
+            )
 
         if self.process is None:
-            worker_kwargs = dict(scheduler_ip=self.scheduler_addr,
-                                 ncores=self.ncores,
-                                 local_dir=self.local_dir,
-                                 services=self.services,
-                                 service_ports={'nanny': self.port},
-                                 name=self.name,
-                                 memory_limit=self.memory_limit,
-                                 reconnect=self.reconnect,
-                                 resources=self.resources,
-                                 validate=self.validate,
-                                 silence_logs=self.silence_logs,
-                                 death_timeout=self.death_timeout,
-                                 preload=self.preload,
-                                 preload_argv=self.preload_argv,
-                                 security=self.security,
-                                 contact_address=self.contact_address)
+            worker_kwargs = dict(
+                scheduler_ip=self.scheduler_addr,
+                ncores=self.ncores,
+                local_dir=self.local_dir,
+                services=self.services,
+                service_ports={"nanny": self.port},
+                name=self.name,
+                memory_limit=self.memory_limit,
+                reconnect=self.reconnect,
+                resources=self.resources,
+                validate=self.validate,
+                silence_logs=self.silence_logs,
+                death_timeout=self.death_timeout,
+                preload=self.preload,
+                preload_argv=self.preload_argv,
+                security=self.security,
+                contact_address=self.contact_address,
+            )
             worker_kwargs.update(self.worker_kwargs)
             self.process = WorkerProcess(
                 worker_args=tuple(),
@@ -232,12 +272,11 @@ def instantiate(self, comm=None):
         if self.death_timeout:
             try:
                 result = yield gen.with_timeout(
-                        timedelta(seconds=self.death_timeout),
-                        self.process.start()
+                    timedelta(seconds=self.death_timeout), self.process.start()
                 )
             except gen.TimeoutError:
                 yield self._close(timeout=self.death_timeout)
-                raise gen.Return('timed out')
+                raise gen.Return("timed out")
         else:
             result = yield self.process.start()
         raise gen.Return(result)
@@ -256,13 +295,13 @@ def _():
             yield gen.with_timeout(timedelta(seconds=timeout), _())
         except gen.TimeoutError:
             logger.error("Restart timed out, returning before finished")
-            raise gen.Return('timed out')
+            raise gen.Return("timed out")
         else:
-            raise gen.Return('OK')
+            raise gen.Return("OK")
 
     def memory_monitor(self):
         """ Track worker's memory.  Restart if it goes above terminate fraction """
-        if self.status != 'running':
+        if self.status != "running":
             return
         process = self.process.process
         if process is None:
@@ -274,19 +313,21 @@ def memory_monitor(self):
         memory = proc.memory_info().rss
         frac = memory / self.memory_limit
         if self.memory_terminate_fraction and frac > self.memory_terminate_fraction:
-            logger.warning("Worker exceeded %d%% memory budget. Restarting",
-                           100 * self.memory_terminate_fraction)
+            logger.warning(
+                "Worker exceeded %d%% memory budget. Restarting",
+                100 * self.memory_terminate_fraction,
+            )
             process.terminate()
 
     def is_alive(self):
-        return self.process is not None and self.process.status == 'running'
+        return self.process is not None and self.process.status == "running"
 
     def run(self, *args, **kwargs):
         return run(self, *args, **kwargs)
 
     @gen.coroutine
     def _on_exit(self, exitcode):
-        if self.status not in ('closing', 'closed'):
+        if self.status not in ("closing", "closed"):
             try:
                 yield self.scheduler.unregister(address=self.worker_address)
             except (EnvironmentError, CommClosedError):
@@ -295,13 +336,14 @@ def _on_exit(self, exitcode):
                     return
 
             try:
-                if self.status not in ('closing', 'closed'):
+                if self.status not in ("closing", "closed"):
                     if self.auto_restart:
                         logger.warning("Restarting worker")
                         yield self.instantiate()
             except Exception:
-                logger.error("Failed to restart worker after its process exited",
-                             exc_info=True)
+                logger.error(
+                    "Failed to restart worker after its process exited", exc_info=True
+                )
 
     @property
     def pid(self):
@@ -312,9 +354,9 @@ def _close(self, comm=None, timeout=5, report=None):
         """
         Close the worker process, stop all comms.
         """
-        if self.status in ('closing', 'closed'):
-            raise gen.Return('OK')
-        self.status = 'closing'
+        if self.status in ("closing", "closed"):
+            raise gen.Return("OK")
+        self.status = "closing"
         logger.info("Closing Nanny at %r", self.address)
         self.stop()
         try:
@@ -325,15 +367,22 @@ def _close(self, comm=None, timeout=5, report=None):
         self.process = None
         self.rpc.close()
         self.scheduler.close_rpc()
-        self.status = 'closed'
-        raise gen.Return('OK')
+        self.status = "closed"
+        raise gen.Return("OK")
 
 
 class WorkerProcess(object):
-
-    def __init__(self, worker_args, worker_kwargs, worker_start_args,
-                 silence_logs, on_exit, worker, env):
-        self.status = 'init'
+    def __init__(
+        self,
+        worker_args,
+        worker_kwargs,
+        worker_start_args,
+        silence_logs,
+        on_exit,
+        worker,
+        env,
+    ):
+        self.status = "init"
         self.silence_logs = silence_logs
         self.worker_args = worker_args
         self.worker_kwargs = worker_kwargs
@@ -353,9 +402,9 @@ def start(self):
         Ensure the worker process is started.
         """
         enable_proctitle_on_children()
-        if self.status == 'running':
+        if self.status == "running":
             raise gen.Return(self.status)
-        if self.status == 'starting':
+        if self.status == "starting":
             yield self.running.wait()
             raise gen.Return(self.status)
 
@@ -365,29 +414,31 @@ def start(self):
 
         self.process = AsyncProcess(
             target=self._run,
-            kwargs=dict(worker_args=self.worker_args,
-                        worker_kwargs=self.worker_kwargs,
-                        worker_start_args=self.worker_start_args,
-                        silence_logs=self.silence_logs,
-                        init_result_q=self.init_result_q,
-                        child_stop_q=self.child_stop_q,
-                        uid=uid,
-                        Worker=self.Worker,
-                        env=self.env),
+            kwargs=dict(
+                worker_args=self.worker_args,
+                worker_kwargs=self.worker_kwargs,
+                worker_start_args=self.worker_start_args,
+                silence_logs=self.silence_logs,
+                init_result_q=self.init_result_q,
+                child_stop_q=self.child_stop_q,
+                uid=uid,
+                Worker=self.Worker,
+                env=self.env,
+            ),
         )
         self.process.daemon = True
         self.process.set_exit_callback(self._on_exit)
         self.running = Event()
         self.stopped = Event()
-        self.status = 'starting'
+        self.status = "starting"
         yield self.process.start()
         msg = yield self._wait_until_connected(uid)
         if not msg:
             raise gen.Return(self.status)
-        self.worker_address = msg['address']
-        self.worker_dir = msg['dir']
+        self.worker_address = msg["address"]
+        self.worker_dir = msg["dir"]
         assert self.worker_address
-        self.status = 'running'
+        self.status = "running"
         self.running.set()
 
         init_q.close()
@@ -405,27 +456,25 @@ def _death_message(self, pid, exitcode):
         if exitcode == 255:
             return "Worker process %d was killed by unknown signal" % (pid,)
         elif exitcode >= 0:
-            return "Worker process %d exited with status %d" % (pid, exitcode,)
+            return "Worker process %d exited with status %d" % (pid, exitcode)
         else:
-            return "Worker process %d was killed by signal %d" % (pid, -exitcode,)
+            return "Worker process %d was killed by signal %d" % (pid, -exitcode)
 
     def is_alive(self):
         return self.process is not None and self.process.is_alive()
 
     @property
     def pid(self):
-        return (self.process.pid
-                if self.process and self.process.is_alive()
-                else None)
+        return self.process.pid if self.process and self.process.is_alive() else None
 
     def mark_stopped(self):
-        if self.status != 'stopped':
+        if self.status != "stopped":
             r = self.process.exitcode
             assert r is not None
             if r != 0:
                 msg = self._death_message(self.process.pid, r)
                 logger.warning(msg)
-            self.status = 'stopped'
+            self.status = "stopped"
             self.stopped.set()
             # Release resources
             self.process.close()
@@ -449,28 +498,31 @@ def kill(self, timeout=2, executor_wait=True):
         loop = IOLoop.current()
         deadline = loop.time() + timeout
 
-        if self.status == 'stopped':
+        if self.status == "stopped":
             return
-        if self.status == 'stopping':
+        if self.status == "stopping":
             yield self.stopped.wait()
             return
-        assert self.status in ('starting', 'running')
-        self.status = 'stopping'
+        assert self.status in ("starting", "running")
+        self.status = "stopping"
 
         process = self.process
-        self.child_stop_q.put({
-            'op': 'stop',
-            'timeout': max(0, deadline - loop.time()) * 0.8,
-            'executor_wait': executor_wait,
-        })
+        self.child_stop_q.put(
+            {
+                "op": "stop",
+                "timeout": max(0, deadline - loop.time()) * 0.8,
+                "executor_wait": executor_wait,
+            }
+        )
         self.child_stop_q.close()
 
         while process.is_alive() and loop.time() < deadline:
             yield gen.sleep(0.05)
 
         if process.is_alive():
-            logger.warning("Worker process still alive after %d seconds, killing",
-                           timeout)
+            logger.warning(
+                "Worker process still alive after %d seconds, killing", timeout
+            )
             try:
                 yield process.terminate()
             except Exception as e:
@@ -480,7 +532,7 @@ def kill(self, timeout=2, executor_wait=True):
     def _wait_until_connected(self, uid):
         delay = 0.05
         while True:
-            if self.status != 'starting':
+            if self.status != "starting":
                 return
             try:
                 msg = self.init_result_q.get_nowait()
@@ -488,24 +540,35 @@ def _wait_until_connected(self, uid):
                 yield gen.sleep(delay)
                 continue
 
-            if msg['uid'] != uid:  # ensure that we didn't cross queues
+            if msg["uid"] != uid:  # ensure that we didn't cross queues
                 continue
 
-            if 'exception' in msg:
-                logger.error("Failed while trying to start worker process: %s",
-                             msg['exception'])
+            if "exception" in msg:
+                logger.error(
+                    "Failed while trying to start worker process: %s", msg["exception"]
+                )
                 yield self.process.join()
                 raise msg
             else:
                 raise gen.Return(msg)
 
     @classmethod
-    def _run(cls, worker_args, worker_kwargs, worker_start_args,
-             silence_logs, init_result_q, child_stop_q, uid, env, Worker):  # pragma: no cover
+    def _run(
+        cls,
+        worker_args,
+        worker_kwargs,
+        worker_start_args,
+        silence_logs,
+        init_result_q,
+        child_stop_q,
+        uid,
+        env,
+        Worker,
+    ):  # pragma: no cover
         os.environ.update(env)
         try:
             from dask.multiprocessing import initialize_worker_process
-        except ImportError:   # old Dask version
+        except ImportError:  # old Dask version
             pass
         else:
             initialize_worker_process()
@@ -521,10 +584,12 @@ def _run(cls, worker_args, worker_kwargs, worker_start_args,
         @gen.coroutine
         def do_stop(timeout=5, executor_wait=True):
             try:
-                yield worker._close(report=False,
-                                    nanny=False,
-                                    executor_wait=executor_wait,
-                                    timeout=timeout)
+                yield worker._close(
+                    report=False,
+                    nanny=False,
+                    executor_wait=executor_wait,
+                    timeout=timeout,
+                )
             finally:
                 loop.stop()
 
@@ -540,7 +605,7 @@ def watch_stop_q():
                     pass
                 else:
                     child_stop_q.close()
-                    assert msg.pop('op') == 'stop'
+                    assert msg.pop("op") == "stop"
                     loop.add_callback(do_stop, **msg)
                     break
 
@@ -557,13 +622,13 @@ def run():
                 yield worker._start(*worker_start_args)
             except Exception as e:
                 logger.exception("Failed to start worker")
-                init_result_q.put({'uid': uid, 'exception': e})
+                init_result_q.put({"uid": uid, "exception": e})
                 init_result_q.close()
             else:
                 assert worker.address
-                init_result_q.put({'address': worker.address,
-                                   'dir': worker.local_dir,
-                                   'uid': uid})
+                init_result_q.put(
+                    {"address": worker.address, "dir": worker.local_dir, "uid": uid}
+                )
                 init_result_q.close()
                 yield worker.wait_until_closed()
                 logger.info("Worker closed")
diff --git a/distributed/node.py b/distributed/node.py
index 4123617620b..8a0b8c12195 100644
--- a/distributed/node.py
+++ b/distributed/node.py
@@ -11,41 +11,65 @@ class Node(object):
     Base class for nodes in a distributed cluster.
     """
 
-    def __init__(self, connection_limit=512, deserialize=True,
-                 connection_args=None, io_loop=None,
-                 serializers=None, deserializers=None):
+    def __init__(
+        self,
+        connection_limit=512,
+        deserialize=True,
+        connection_args=None,
+        io_loop=None,
+        serializers=None,
+        deserializers=None,
+    ):
         self.io_loop = io_loop or IOLoop.current()
-        self.rpc = ConnectionPool(limit=connection_limit,
-                                  deserialize=deserialize,
-                                  serializers=serializers,
-                                  deserializers=deserializers,
-                                  connection_args=connection_args)
+        self.rpc = ConnectionPool(
+            limit=connection_limit,
+            deserialize=deserialize,
+            serializers=serializers,
+            deserializers=deserializers,
+            connection_args=connection_args,
+        )
 
 
 class ServerNode(Node, Server):
     """
     Base class for server nodes in a distributed cluster.
     """
+
     # TODO factor out security, listening, services, etc. here
 
     # XXX avoid inheriting from Server? there is some large potential for confusion
     # between base and derived attribute namespaces...
 
-    def __init__(self, handlers=None, blocked_handlers=None, stream_handlers=None,
-                 connection_limit=512, deserialize=True,
-                 connection_args=None, io_loop=None, serializers=None,
-                 deserializers=None):
-        Node.__init__(self, deserialize=deserialize,
-                      connection_limit=connection_limit,
-                      connection_args=connection_args,
-                      io_loop=io_loop,
-                      serializers=serializers,
-                      deserializers=deserializers)
-        Server.__init__(self, handlers=handlers,
-                        blocked_handlers=blocked_handlers,
-                        stream_handlers=stream_handlers,
-                        connection_limit=connection_limit,
-                        deserialize=deserialize, io_loop=self.io_loop)
+    def __init__(
+        self,
+        handlers=None,
+        blocked_handlers=None,
+        stream_handlers=None,
+        connection_limit=512,
+        deserialize=True,
+        connection_args=None,
+        io_loop=None,
+        serializers=None,
+        deserializers=None,
+    ):
+        Node.__init__(
+            self,
+            deserialize=deserialize,
+            connection_limit=connection_limit,
+            connection_args=connection_args,
+            io_loop=io_loop,
+            serializers=serializers,
+            deserializers=deserializers,
+        )
+        Server.__init__(
+            self,
+            handlers=handlers,
+            blocked_handlers=blocked_handlers,
+            stream_handlers=stream_handlers,
+            connection_limit=connection_limit,
+            deserialize=deserialize,
+            io_loop=self.io_loop,
+        )
 
     def versions(self, comm=None, packages=None):
         return get_versions(packages=packages)
diff --git a/distributed/preloading.py b/distributed/preloading.py
index 00fa4eaeae2..0f08f60f71c 100644
--- a/distributed/preloading.py
+++ b/distributed/preloading.py
@@ -25,25 +25,28 @@ def validate_preload_argv(ctx, param, value):
         for a in unexpected_args:
             raise click.NoSuchOption(a)
         raise click.UsageError(
-            "Got unexpected extra argument%s: (%s)" %
-            ("s" if len(value) > 1 else "", " ".join(value))
+            "Got unexpected extra argument%s: (%s)"
+            % ("s" if len(value) > 1 else "", " ".join(value))
         )
 
     preload_modules = _import_modules(ctx.params.get("preload"))
 
     preload_commands = [
-        m["dask_setup"] for m in preload_modules.values()
+        m["dask_setup"]
+        for m in preload_modules.values()
         if isinstance(m["dask_setup"], click.Command)
     ]
 
     if len(preload_commands) > 1:
         raise click.UsageError(
-            "Multiple --preload modules with click-configurable setup: %s" %
-            list(preload_modules.keys()))
+            "Multiple --preload modules with click-configurable setup: %s"
+            % list(preload_modules.keys())
+        )
 
     if value and not preload_commands:
         raise click.UsageError(
-            "Unknown argument specified: %r Was click-configurable --preload target provided?")
+            "Unknown argument specified: %r Was click-configurable --preload target provided?"
+        )
     if not preload_commands:
         return value
     else:
@@ -98,7 +101,7 @@ def _import_modules(names, file_dir=None):
             module = sys.modules[name]
 
         result_modules[name] = {
-            attrname : getattr(module, attrname, None)
+            attrname: getattr(module, attrname, None)
             for attrname in ("dask_setup", "dask_teardown")
         }
 
@@ -128,7 +131,9 @@ def preload_modules(names, parameter=None, file_dir=None, argv=None):
 
         if dask_setup:
             if isinstance(dask_setup, click.Command):
-                context = dask_setup.make_context("dask_setup", list(argv), allow_extra_args=False)
+                context = dask_setup.make_context(
+                    "dask_setup", list(argv), allow_extra_args=False
+                )
                 dask_setup.callback(parameter, *context.args, **context.params)
             else:
                 dask_setup(parameter)
diff --git a/distributed/process.py b/distributed/process.py
index 38e3af62c3b..5dd9368fdc1 100644
--- a/distributed/process.py
+++ b/distributed/process.py
@@ -55,8 +55,7 @@ class AsyncProcess(object):
 
     def __init__(self, loop=None, target=None, name=None, args=(), kwargs={}):
         if not callable(target):
-            raise TypeError("`target` needs to be callable, not %r"
-                            % (type(target),))
+            raise TypeError("`target` needs to be callable, not %r" % (type(target),))
         self._state = _ProcessState()
         self._loop = loop or IOLoop.current(instance=False)
 
@@ -71,10 +70,11 @@ def __init__(self, loop=None, target=None, name=None, args=(), kwargs={}):
         # for the assignment here.
         parent_alive_pipe, self._keep_child_alive = mp_context.Pipe(duplex=False)
 
-        self._process = mp_context.Process(target=self._run, name=name,
-                                           args=(target, args, kwargs,
-                                                 parent_alive_pipe,
-                                                 self._keep_child_alive))
+        self._process = mp_context.Process(
+            target=self._run,
+            name=name,
+            args=(target, args, kwargs, parent_alive_pipe, self._keep_child_alive),
+        )
         _dangling.add(self._process)
         self._name = self._process.name
         self._watch_q = PyQueue()
@@ -95,13 +95,20 @@ def _start_threads(self):
         self._watch_message_thread = threading.Thread(
             target=self._watch_message_queue,
             name="AsyncProcess %s watch message queue" % self.name,
-            args=(weakref.ref(self), self._process, self._loop,
-                  self._state, self._watch_q, self._exit_future,))
+            args=(
+                weakref.ref(self),
+                self._process,
+                self._loop,
+                self._state,
+                self._watch_q,
+                self._exit_future,
+            ),
+        )
         self._watch_message_thread.daemon = True
         self._watch_message_thread.start()
 
         def stop_thread(q):
-            q.put_nowait({'op': 'stop'})
+            q.put_nowait({"op": "stop"})
             # We don't join the thread here as a finalizer can be called
             # asynchronously from anywhere
 
@@ -120,6 +127,7 @@ def _immediate_exit_when_closed(cls, parent_alive_pipe):
         """
         Immediately exit the process when parent_alive_pipe is closed.
         """
+
         def monitor_parent():
             try:
                 # The parent_alive_pipe should be held open as long as the
@@ -186,7 +194,8 @@ def _start():
             thread = threading.Thread(
                 target=AsyncProcess._watch_process,
                 name="AsyncProcess %s watch process join" % name,
-                args=(selfref, process, state, q))
+                args=(selfref, process, state, q),
+            )
             thread.daemon = True
             thread.start()
 
@@ -197,12 +206,12 @@ def _start():
         while True:
             msg = q.get()
             logger.debug("[%s] got message %r" % (r, msg))
-            op = msg['op']
-            if op == 'start':
-                _call_and_set_future(loop, msg['future'], _start)
-            elif op == 'terminate':
-                _call_and_set_future(loop, msg['future'], process.terminate)
-            elif op == 'stop':
+            op = msg["op"]
+            if op == "start":
+                _call_and_set_future(loop, msg["future"], _start)
+            elif op == "terminate":
+                _call_and_set_future(loop, msg["future"], process.terminate)
+            elif op == "stop":
                 break
             else:
                 assert 0, msg
@@ -213,8 +222,7 @@ def _watch_process(cls, selfref, process, state, q):
         process.join()
         exitcode = process.exitcode
         assert exitcode is not None
-        logger.debug("[%s] process %r exited with code %r",
-                     r, state.pid, exitcode)
+        logger.debug("[%s] process %r exited with code %r", r, state.pid, exitcode)
         state.is_alive = False
         state.exitcode = exitcode
         # Make sure the process is removed from the global list
@@ -235,7 +243,7 @@ def start(self):
         """
         self._check_closed()
         fut = Future()
-        self._watch_q.put_nowait({'op': 'start', 'future': fut})
+        self._watch_q.put_nowait({"op": "start", "future": fut})
         return fut
 
     def terminate(self):
@@ -246,7 +254,7 @@ def terminate(self):
         """
         self._check_closed()
         fut = Future()
-        self._watch_q.put_nowait({'op': 'terminate', 'future': fut})
+        self._watch_q.put_nowait({"op": "terminate", "future": fut})
         return fut
 
     @gen.coroutine
@@ -257,7 +265,7 @@ def join(self, timeout=None):
         This method is a coroutine.
         """
         self._check_closed()
-        assert self._state.pid is not None, 'can only join a started process'
+        assert self._state.pid is not None, "can only join a started process"
         if self._state.exitcode is not None:
             return
         if timeout is None:
@@ -287,7 +295,9 @@ def set_exit_callback(self, func):
         """
         # XXX should this be a property instead?
         assert callable(func), "exit callback should be callable"
-        assert self._state.pid is None, "cannot set exit callback when process already started"
+        assert (
+            self._state.pid is None
+        ), "cannot set exit callback when process already started"
         self._exit_callback = func
 
     def is_alive(self):
diff --git a/distributed/proctitle.py b/distributed/proctitle.py
index bdaf8bed5d6..50c9859e17e 100644
--- a/distributed/proctitle.py
+++ b/distributed/proctitle.py
@@ -16,7 +16,7 @@ def enable_proctitle_on_children():
     Enable setting the process title on this process' children and
     grandchildren.
     """
-    os.environ['DASK_PARENT'] = str(os.getpid())
+    os.environ["DASK_PARENT"] = str(os.getpid())
 
 
 def enable_proctitle_on_current():
@@ -37,7 +37,7 @@ def setproctitle(title):
     enabled = _enabled
     if not enabled:
         try:
-            enabled = int(os.environ.get('DASK_PARENT', '')) != os.getpid()
+            enabled = int(os.environ.get("DASK_PARENT", "")) != os.getpid()
         except ValueError:
             pass
     if enabled:
diff --git a/distributed/profile.py b/distributed/profile.py
index 54e62c288c4..385c7449e75 100644
--- a/distributed/profile.py
+++ b/distributed/profile.py
@@ -46,30 +46,34 @@ def identifier(frame):
     Strings are cheaper to use as indexes into dicts than tuples or dicts
     """
     if frame is None:
-        return 'None'
+        return "None"
     else:
-        return ';'.join((frame.f_code.co_name,
-                         frame.f_code.co_filename,
-                         str(frame.f_code.co_firstlineno)))
+        return ";".join(
+            (
+                frame.f_code.co_name,
+                frame.f_code.co_filename,
+                str(frame.f_code.co_firstlineno),
+            )
+        )
 
 
 def repr_frame(frame):
     """ Render a frame as a line for inclusion into a text traceback """
     co = frame.f_code
-    text = '  File "%s", line %s, in %s' % (co.co_filename,
-                                            frame.f_lineno,
-                                            co.co_name)
+    text = '  File "%s", line %s, in %s' % (co.co_filename, frame.f_lineno, co.co_name)
     line = linecache.getline(co.co_filename, frame.f_lineno, frame.f_globals).lstrip()
-    return text + '\n\t' + line
+    return text + "\n\t" + line
 
 
 def info_frame(frame):
     co = frame.f_code
     line = linecache.getline(co.co_filename, frame.f_lineno, frame.f_globals).lstrip()
-    return {'filename': co.co_filename,
-            'name': co.co_name,
-            'line_number': frame.f_lineno,
-            'line': line}
+    return {
+        "filename": co.co_filename,
+        "name": co.co_name,
+        "line_number": frame.f_lineno,
+        "line": line,
+    }
 
 
 def process(frame, child, state, stop=None, omit=None):
@@ -96,7 +100,9 @@ def process(frame, child, state, stop=None, omit=None):
         return False
 
     prev = frame.f_back
-    if prev is not None and (stop is None or not prev.f_code.co_filename.endswith(stop)):
+    if prev is not None and (
+        stop is None or not prev.f_code.co_filename.endswith(stop)
+    ):
         state = process(prev, frame, state, stop=stop)
         if state is False:
             return False
@@ -104,45 +110,53 @@ def process(frame, child, state, stop=None, omit=None):
     ident = identifier(frame)
 
     try:
-        d = state['children'][ident]
+        d = state["children"][ident]
     except KeyError:
-        d = {'count': 0,
-             'description': info_frame(frame),
-             'children': {},
-             'identifier': ident}
-        state['children'][ident] = d
+        d = {
+            "count": 0,
+            "description": info_frame(frame),
+            "children": {},
+            "identifier": ident,
+        }
+        state["children"][ident] = d
 
-    state['count'] += 1
+    state["count"] += 1
 
     if child is not None:
         return d
     else:
-        d['count'] += 1
+        d["count"] += 1
 
 
 def merge(*args):
     """ Merge multiple frame states together """
     if not args:
         return create()
-    s = {arg['identifier'] for arg in args}
+    s = {arg["identifier"] for arg in args}
     if len(s) != 1:
         raise ValueError("Expected identifiers, got %s" % str(s))
     children = defaultdict(list)
     for arg in args:
-        for child in arg['children']:
-            children[child].append(arg['children'][child])
+        for child in arg["children"]:
+            children[child].append(arg["children"][child])
 
     children = {k: merge(*v) for k, v in children.items()}
-    count = sum(arg['count'] for arg in args)
-    return {'description': args[0]['description'],
-            'children': dict(children),
-            'count': count,
-            'identifier': args[0]['identifier']}
+    count = sum(arg["count"] for arg in args)
+    return {
+        "description": args[0]["description"],
+        "children": dict(children),
+        "count": count,
+        "identifier": args[0]["identifier"],
+    }
 
 
 def create():
-    return {'count': 0, 'children': {}, 'identifier': 'root', 'description':
-            {'filename': '', 'name': '', 'line_number': 0, 'line': ''}}
+    return {
+        "count": 0,
+        "children": {},
+        "identifier": "root",
+        "description": {"filename": "", "name": "", "line_number": 0, "line": ""},
+    }
 
 
 def call_stack(frame):
@@ -180,7 +194,7 @@ def plot_data(state, profile_interval=0.010):
     names = []
 
     def traverse(state, start, stop, height):
-        if not state['count']:
+        if not state["count"]:
             return
         starts.append(start)
         stops.append(stop)
@@ -188,49 +202,50 @@ def traverse(state, start, stop, height):
         width = stop - start
         widths.append(width)
         states.append(state)
-        times.append(format_time(state['count'] * profile_interval))
+        times.append(format_time(state["count"] * profile_interval))
 
-        desc = state['description']
-        filenames.append(desc['filename'])
-        lines.append(desc['line'])
-        line_numbers.append(desc['line_number'])
-        names.append(desc['name'])
+        desc = state["description"]
+        filenames.append(desc["filename"])
+        lines.append(desc["line"])
+        line_numbers.append(desc["line_number"])
+        names.append(desc["name"])
 
-        ident = state['identifier']
+        ident = state["identifier"]
 
         try:
-            colors.append(color_of(desc['filename']))
+            colors.append(color_of(desc["filename"]))
         except IndexError:
-            colors.append('gray')
+            colors.append("gray")
 
-        delta = (stop - start) / state['count']
+        delta = (stop - start) / state["count"]
 
         x = start
 
-        for name, child in state['children'].items():
-            width = child['count'] * delta
+        for name, child in state["children"].items():
+            width = child["count"] * delta
             traverse(child, x, x + width, height + 1)
             x += width
 
     traverse(state, 0, 1, 0)
     percentages = ["{:.2f}%".format(100 * w) for w in widths]
-    return {'left': starts,
-            'right': stops,
-            'bottom': heights,
-            'width': widths,
-            'top': [x + 1 for x in heights],
-            'color': colors,
-            'states': states,
-            'filename': filenames,
-            'line': lines,
-            'line_number': line_numbers,
-            'name': names,
-            'time': times,
-            'percentage': percentages}
-
-
-def _watch(thread_id, log, interval='20ms', cycle='2s', omit=None,
-           stop=lambda: False):
+    return {
+        "left": starts,
+        "right": stops,
+        "bottom": heights,
+        "width": widths,
+        "top": [x + 1 for x in heights],
+        "color": colors,
+        "states": states,
+        "filename": filenames,
+        "line": lines,
+        "line_number": line_numbers,
+        "name": names,
+        "time": times,
+        "percentage": percentages,
+    }
+
+
+def _watch(thread_id, log, interval="20ms", cycle="2s", omit=None, stop=lambda: False):
     interval = parse_timedelta(interval)
     cycle = parse_timedelta(cycle)
 
@@ -251,21 +266,31 @@ def _watch(thread_id, log, interval='20ms', cycle='2s', omit=None,
         sleep(interval)
 
 
-def watch(thread_id=None, interval='20ms', cycle='2s', maxlen=1000, omit=None,
-          stop=lambda: False):
+def watch(
+    thread_id=None,
+    interval="20ms",
+    cycle="2s",
+    maxlen=1000,
+    omit=None,
+    stop=lambda: False,
+):
     if thread_id is None:
         thread_id = get_thread_identity()
 
     log = deque(maxlen=maxlen)
 
-    thread = threading.Thread(target=_watch,
-                              name='Profile',
-                              kwargs={'thread_id': thread_id,
-                                      'interval': interval,
-                                      'cycle': cycle,
-                                      'log': log,
-                                      'omit': omit,
-                                      'stop': stop})
+    thread = threading.Thread(
+        target=_watch,
+        name="Profile",
+        kwargs={
+            "thread_id": thread_id,
+            "interval": interval,
+            "cycle": cycle,
+            "log": log,
+            "omit": omit,
+            "stop": stop,
+        },
+    )
     thread.daemon = True
     thread.start()
 
@@ -307,14 +332,22 @@ def plot_figure(data, **kwargs):
     from bokeh.plotting import ColumnDataSource, figure
     from bokeh.models import HoverTool
 
-    if 'states' in data:
-        data = toolz.dissoc(data, 'states')
+    if "states" in data:
+        data = toolz.dissoc(data, "states")
 
     source = ColumnDataSource(data=data)
 
-    fig = figure(tools='tap', **kwargs)
-    r = fig.quad('left', 'right', 'top', 'bottom', color='color',
-             line_color='black', line_width=2, source=source)
+    fig = figure(tools="tap", **kwargs)
+    r = fig.quad(
+        "left",
+        "right",
+        "top",
+        "bottom",
+        color="color",
+        line_color="black",
+        line_width=2,
+        source=source,
+    )
 
     r.selection_glyph = None
     r.nonselection_glyph = None
@@ -346,7 +379,7 @@ def plot_figure(data, **kwargs):
                 <span style="font-size: 14px; font-weight: bold;">Percentage:</span>&nbsp;
                 <span style="font-size: 10px; font-family: Monaco, monospace;">@width</span>
             </div>
-            """
+            """,
     )
     fig.add_tools(hover)
 
diff --git a/distributed/protocol/__init__.py b/distributed/protocol/__init__.py
index bd8f7331c8e..cf1a3df8994 100644
--- a/distributed/protocol/__init__.py
+++ b/distributed/protocol/__init__.py
@@ -3,12 +3,22 @@
 from functools import partial
 
 from .compression import compressions, default_compression
-from .core import (dumps, loads, maybe_compress, decompress, msgpack)
+from .core import dumps, loads, maybe_compress, decompress, msgpack
 from .serialize import (
-    serialize, deserialize, nested_deserialize, Serialize, Serialized,
-    to_serialize, register_serialization, dask_serialize, dask_deserialize,
-    serialize_bytes, deserialize_bytes, serialize_bytelist,
-    register_serialization_family, register_generic,
+    serialize,
+    deserialize,
+    nested_deserialize,
+    Serialize,
+    Serialized,
+    to_serialize,
+    register_serialization,
+    dask_serialize,
+    dask_deserialize,
+    serialize_bytes,
+    deserialize_bytes,
+    serialize_bytelist,
+    register_serialization_family,
+    register_generic,
 )
 
 from ..utils import ignoring
@@ -54,6 +64,7 @@ def _register_arrow():
 @dask_deserialize.register_lazy("sklearn")
 def _register_sklearn():
     import sklearn.base
+
     register_generic(sklearn.base.BaseEstimator)
 
 
diff --git a/distributed/protocol/arrow.py b/distributed/protocol/arrow.py
index 012a91e6afc..cac146a575c 100644
--- a/distributed/protocol/arrow.py
+++ b/distributed/protocol/arrow.py
@@ -3,9 +3,12 @@
 from .serialize import dask_serialize, dask_deserialize
 
 import pyarrow
-if pyarrow.__version__ < '0.10':
-    raise ImportError("Need pyarrow >= 0.10 . "
-                      "See https://arrow.apache.org/docs/python/install.html")
+
+if pyarrow.__version__ < "0.10":
+    raise ImportError(
+        "Need pyarrow >= 0.10 . "
+        "See https://arrow.apache.org/docs/python/install.html"
+    )
 
 
 @dask_serialize.register(pyarrow.RecordBatch)
diff --git a/distributed/protocol/compression.py b/distributed/protocol/compression.py
index 2a6de6bfca1..f729748acc8 100644
--- a/distributed/protocol/compression.py
+++ b/distributed/protocol/compression.py
@@ -13,8 +13,9 @@
 
 try:
     import blosc
+
     n = blosc.set_nthreads(2)
-    if hasattr('blosc', 'releasegil'):
+    if hasattr("blosc", "releasegil"):
         blosc.set_releasegil(True)
 except ImportError:
     blosc = False
@@ -22,8 +23,7 @@
 from ..utils import ignoring, ensure_bytes
 
 
-compressions = {None: {'compress': identity,
-                       'decompress': identity}}
+compressions = {None: {"compress": identity, "decompress": identity}}
 
 compressions[False] = compressions[None]  # alias
 
@@ -36,8 +36,8 @@
 
 with ignoring(ImportError):
     import zlib
-    compressions['zlib'] = {'compress': zlib.compress,
-                            'decompress': zlib.decompress}
+
+    compressions["zlib"] = {"compress": zlib.compress, "decompress": zlib.decompress}
 
 with ignoring(ImportError):
     import snappy
@@ -48,9 +48,11 @@ def _fixed_snappy_decompress(data):
             data = bytes(data)
         return snappy.decompress(data)
 
-    compressions['snappy'] = {'compress': snappy.compress,
-                              'decompress': _fixed_snappy_decompress}
-    default_compression = 'snappy'
+    compressions["snappy"] = {
+        "compress": snappy.compress,
+        "decompress": _fixed_snappy_decompress,
+    }
+    default_compression = "snappy"
 
 with ignoring(ImportError):
     import lz4
@@ -58,6 +60,7 @@ def _fixed_snappy_decompress(data):
     try:
         # try using the new lz4 API
         import lz4.block
+
         lz4_compress = lz4.block.compress
         lz4_decompress = lz4.block.decompress
     except ImportError:
@@ -86,25 +89,31 @@ def _fixed_lz4_decompress(data):
             else:
                 raise
 
-    compressions['lz4'] = {'compress': _fixed_lz4_compress,
-                           'decompress': _fixed_lz4_decompress}
-    default_compression = 'lz4'
+    compressions["lz4"] = {
+        "compress": _fixed_lz4_compress,
+        "decompress": _fixed_lz4_decompress,
+    }
+    default_compression = "lz4"
 
 with ignoring(ImportError):
     import blosc
-    compressions['blosc'] = {'compress': partial(blosc.compress, clevel=5,
-                                                 cname='lz4'),
-                             'decompress': blosc.decompress}
+
+    compressions["blosc"] = {
+        "compress": partial(blosc.compress, clevel=5, cname="lz4"),
+        "decompress": blosc.decompress,
+    }
 
 
-default = dask.config.get('distributed.comm.compression')
-if default != 'auto':
+default = dask.config.get("distributed.comm.compression")
+if default != "auto":
     if default in compressions:
         default_compression = default
     else:
-        raise ValueError("Default compression '%s' not found.\n"
-                         "Choices include auto, %s" % (
-                             default, ', '.join(sorted(map(str, compressions)))))
+        raise ValueError(
+            "Default compression '%s' not found.\n"
+            "Choices include auto, %s"
+            % (default, ", ".join(sorted(map(str, compressions))))
+        )
 
 
 def byte_sample(b, size, n):
@@ -125,7 +134,7 @@ def byte_sample(b, size, n):
     ends.append(starts[-1] + size)
 
     parts = [b[start:end] for start, end in zip(starts, ends)]
-    return b''.join(map(ensure_bytes, parts))
+    return b"".join(map(ensure_bytes, parts))
 
 
 def maybe_compress(payload, min_size=1e4, sample_size=1e4, nsamples=5):
@@ -139,21 +148,21 @@ def maybe_compress(payload, min_size=1e4, sample_size=1e4, nsamples=5):
         return the original
     4.  We return the compressed result
     """
-    compression = dask.config.get('distributed.comm.compression')
-    if compression == 'auto':
+    compression = dask.config.get("distributed.comm.compression")
+    if compression == "auto":
         compression = default_compression
 
     if not compression:
         return None, payload
     if len(payload) < min_size:
         return None, payload
-    if len(payload) > 2**31:  # Too large, compression libraries often fail
+    if len(payload) > 2 ** 31:  # Too large, compression libraries often fail
         return None, payload
 
     min_size = int(min_size)
     sample_size = int(sample_size)
 
-    compress = compressions[compression]['compress']
+    compress = compressions[compression]["compress"]
 
     # Compress a sample, return original if not very compressed
     sample = byte_sample(payload, sample_size, nsamples)
@@ -167,9 +176,10 @@ def maybe_compress(payload, min_size=1e4, sample_size=1e4, nsamples=5):
 
     if default_compression and blosc and type(payload) is memoryview:
         # Blosc does itemsize-aware shuffling, resulting in better compression
-        compressed = blosc.compress(payload, typesize=payload.itemsize,
-                                    cname='lz4', clevel=5)
-        compression = 'blosc'
+        compressed = blosc.compress(
+            payload, typesize=payload.itemsize, cname="lz4", clevel=5
+        )
+        compression = "blosc"
     else:
         compressed = compress(ensure_bytes(payload))
 
@@ -181,5 +191,7 @@ def maybe_compress(payload, min_size=1e4, sample_size=1e4, nsamples=5):
 
 def decompress(header, frames):
     """ Decompress frames according to information in the header """
-    return [compressions[c]['decompress'](frame)
-            for c, frame in zip(header['compression'], frames)]
+    return [
+        compressions[c]["decompress"](frame)
+        for c, frame in zip(header["compression"], frames)
+    ]
diff --git a/distributed/protocol/core.py b/distributed/protocol/core.py
index 8d9e1b2b127..0b5f7eb0fea 100644
--- a/distributed/protocol/core.py
+++ b/distributed/protocol/core.py
@@ -11,8 +11,14 @@
     from toolz import reduce
 
 from .compression import compressions, maybe_compress, decompress
-from .serialize import (serialize, deserialize, Serialize, Serialized,
-                        extract_serialize, msgpack_len_opts)
+from .serialize import (
+    serialize,
+    deserialize,
+    Serialize,
+    Serialized,
+    extract_serialize,
+    msgpack_len_opts,
+)
 from .utils import frame_split_size, merge_frames
 from ..utils import nbytes
 
@@ -20,18 +26,18 @@
 
 
 try:
-    msgpack.loads(msgpack.dumps(''), raw=False, **msgpack_len_opts)
-    msgpack_opts = {'raw': False}
+    msgpack.loads(msgpack.dumps(""), raw=False, **msgpack_len_opts)
+    msgpack_opts = {"raw": False}
     msgpack_opts.update(msgpack_len_opts)
 except TypeError:
     # Backward compat with old msgpack (prior to 0.5.2)
-    msgpack_opts = {'encoding': 'utf-8'}
+    msgpack_opts = {"encoding": "utf-8"}
 
 
 logger = logging.getLogger(__name__)
 
 
-def dumps(msg, serializers=None, on_error='message', context=None):
+def dumps(msg, serializers=None, on_error="message", context=None):
     """ Transform Python message to bytestream suitable for communication """
     try:
         data = {}
@@ -43,56 +49,60 @@ def dumps(msg, serializers=None, on_error='message', context=None):
         if not data:  # fast path without serialized data
             return small_header, small_payload
 
-        pre = {key: (value.header, value.frames)
-               for key, value in data.items()
-               if type(value) is Serialized}
+        pre = {
+            key: (value.header, value.frames)
+            for key, value in data.items()
+            if type(value) is Serialized
+        }
 
-        data = {key: serialize(value.data,
-                               serializers=serializers,
-                               on_error=on_error,
-                               context=context)
-                for key, value in data.items()
-                if type(value) is Serialize}
+        data = {
+            key: serialize(
+                value.data, serializers=serializers, on_error=on_error, context=context
+            )
+            for key, value in data.items()
+            if type(value) is Serialize
+        }
 
-        header = {'headers': {},
-                  'keys': [],
-                  'bytestrings': list(bytestrings)}
+        header = {"headers": {}, "keys": [], "bytestrings": list(bytestrings)}
 
         out_frames = []
 
         for key, (head, frames) in data.items():
-            if 'lengths' not in head:
-                head['lengths'] = tuple(map(nbytes, frames))
-            if 'compression' not in head:
+            if "lengths" not in head:
+                head["lengths"] = tuple(map(nbytes, frames))
+            if "compression" not in head:
                 frames = frame_split_size(frames)
                 if frames:
                     compression, frames = zip(*map(maybe_compress, frames))
                 else:
                     compression = []
-                head['compression'] = compression
-            head['count'] = len(frames)
-            header['headers'][key] = head
-            header['keys'].append(key)
+                head["compression"] = compression
+            head["count"] = len(frames)
+            header["headers"][key] = head
+            header["keys"].append(key)
             out_frames.extend(frames)
 
         for key, (head, frames) in pre.items():
-            if 'lengths' not in head:
-                head['lengths'] = tuple(map(nbytes, frames))
-            head['count'] = len(frames)
-            header['headers'][key] = head
-            header['keys'].append(key)
+            if "lengths" not in head:
+                head["lengths"] = tuple(map(nbytes, frames))
+            head["count"] = len(frames)
+            header["headers"][key] = head
+            header["keys"].append(key)
             out_frames.extend(frames)
 
         for i, frame in enumerate(out_frames):
             if type(frame) is memoryview and frame.strides != (1,):
                 try:
-                    frame = frame.cast('b')
+                    frame = frame.cast("b")
                 except TypeError:
                     frame = frame.tobytes()
                 out_frames[i] = frame
 
-        return [small_header, small_payload,
-                msgpack.dumps(header, use_bin_type=True)] + out_frames
+        return [
+            small_header,
+            small_payload,
+            msgpack.dumps(header, use_bin_type=True),
+        ] + out_frames
     except Exception:
         logger.critical("Failed to Serialize", exc_info=True)
         raise
@@ -112,13 +122,13 @@ def loads(frames, deserialize=True, deserializers=None):
 
         header = frames.pop()
         header = msgpack.loads(header, use_list=False, **msgpack_opts)
-        keys = header['keys']
-        headers = header['headers']
-        bytestrings = set(header['bytestrings'])
+        keys = header["keys"]
+        headers = header["headers"]
+        bytestrings = set(header["bytestrings"])
 
         for key in keys:
             head = headers[key]
-            count = head['count']
+            count = head["count"]
             if count:
                 fs = frames[-count::][::-1]
                 del frames[-count:]
@@ -126,7 +136,7 @@ def loads(frames, deserialize=True, deserializers=None):
                 fs = []
 
             if deserialize or key in bytestrings:
-                if 'compression' in head:
+                if "compression" in head:
                     fs = decompress(head, fs)
                 fs = merge_frames(head, fs)
                 value = _deserialize(head, fs, deserializers=deserializers)
@@ -166,12 +176,12 @@ def dumps_msgpack(msg):
 
     fmt, payload = maybe_compress(payload)
     if fmt:
-        header['compression'] = fmt
+        header["compression"] = fmt
 
     if header:
         header_bytes = msgpack.dumps(header, use_bin_type=True)
     else:
-        header_bytes = b''
+        header_bytes = b""
 
     return [header_bytes, payload]
 
@@ -187,12 +197,14 @@ def loads_msgpack(header, payload):
     else:
         header = {}
 
-    if header.get('compression'):
+    if header.get("compression"):
         try:
-            decompress = compressions[header['compression']]['decompress']
+            decompress = compressions[header["compression"]]["decompress"]
             payload = decompress(payload)
         except KeyError:
-            raise ValueError("Data is compressed as %s but we don't have this"
-                             " installed" % str(header['compression']))
+            raise ValueError(
+                "Data is compressed as %s but we don't have this"
+                " installed" % str(header["compression"])
+            )
 
     return msgpack.loads(payload, use_list=False, **msgpack_opts)
diff --git a/distributed/protocol/h5py.py b/distributed/protocol/h5py.py
index 9936920a759..cf08719e259 100644
--- a/distributed/protocol/h5py.py
+++ b/distributed/protocol/h5py.py
@@ -7,25 +7,26 @@
 
 @dask_serialize.register(h5py.File)
 def serialize_h5py_file(f):
-    if f.mode != 'r':
+    if f.mode != "r":
         raise ValueError("Can only serialize read-only h5py files")
-    return {'filename': f.filename}, []
+    return {"filename": f.filename}, []
 
 
 @dask_deserialize.register(h5py.File)
 def deserialize_h5py_file(header, frames):
     import h5py
-    return h5py.File(header['filename'], mode='r')
+
+    return h5py.File(header["filename"], mode="r")
 
 
 @dask_serialize.register((h5py.Group, h5py.Dataset))
 def serialize_h5py_dataset(x):
     header, _ = serialize_h5py_file(x.file)
-    header['name'] = x.name
+    header["name"] = x.name
     return header, []
 
 
 @dask_deserialize.register((h5py.Group, h5py.Dataset))
 def deserialize_h5py_dataset(header, frames):
     file = deserialize_h5py_file(header, frames)
-    return file[header['name']]
+    return file[header["name"]]
diff --git a/distributed/protocol/keras.py b/distributed/protocol/keras.py
index a5437f60e18..4c6fc4b4d0a 100644
--- a/distributed/protocol/keras.py
+++ b/distributed/protocol/keras.py
@@ -8,15 +8,17 @@
 @dask_serialize.register(keras.Model)
 def serialize_keras_model(model):
     import keras
-    if keras.__version__ < '1.2.0':
-        raise ImportError("Need Keras >= 1.2.0. "
-                          "Try pip install keras --upgrade --no-deps")
+
+    if keras.__version__ < "1.2.0":
+        raise ImportError(
+            "Need Keras >= 1.2.0. " "Try pip install keras --upgrade --no-deps"
+        )
 
     header = model._updated_config()
     weights = model.get_weights()
     headers, frames = list(zip(*map(serialize, weights)))
-    header['headers'] = headers
-    header['nframes'] = [len(L) for L in frames]
+    header["headers"] = headers
+    header["nframes"] = [len(L) for L in frames]
     frames = [frame for L in frames for frame in L]
     return header, frames
 
@@ -24,10 +26,11 @@ def serialize_keras_model(model):
 @dask_deserialize.register(keras.Model)
 def deserialize_keras_model(header, frames):
     from keras.models import model_from_config
+
     n = 0
     weights = []
-    for head, length in zip(header['headers'], header['nframes']):
-        x = deserialize(head, frames[n: n + length])
+    for head, length in zip(header["headers"], header["nframes"]):
+        x = deserialize(head, frames[n : n + length])
         weights.append(x)
         n += length
     model = model_from_config(header)
diff --git a/distributed/protocol/netcdf4.py b/distributed/protocol/netcdf4.py
index 06711ad03cb..e04864d2b73 100644
--- a/distributed/protocol/netcdf4.py
+++ b/distributed/protocol/netcdf4.py
@@ -8,29 +8,29 @@
 @dask_serialize.register(netCDF4.Dataset)
 def serialize_netcdf4_dataset(ds):
     # assume mode is read-only
-    return {'filename': ds.filepath()}, []
+    return {"filename": ds.filepath()}, []
 
 
 @dask_deserialize.register(netCDF4.Dataset)
 def deserialize_netcdf4_dataset(header, frames):
-    return netCDF4.Dataset(header['filename'], mode='r')
+    return netCDF4.Dataset(header["filename"], mode="r")
 
 
 @dask_serialize.register(netCDF4.Variable)
 def serialize_netcdf4_variable(x):
     header, _ = serialize(x.group())
-    header['parent-type'] = header['type']
-    header['parent-type-serialized'] = header['type-serialized']
-    header['name'] = x.name
+    header["parent-type"] = header["type"]
+    header["parent-type-serialized"] = header["type-serialized"]
+    header["name"] = x.name
     return header, []
 
 
 @dask_deserialize.register(netCDF4.Variable)
 def deserialize_netcdf4_variable(header, frames):
-    header['type'] = header['parent-type']
-    header['type-serialized'] = header['parent-type-serialized']
+    header["type"] = header["parent-type"]
+    header["type-serialized"] = header["parent-type-serialized"]
     parent = deserialize(header, frames)
-    return parent.variables[header['name']]
+    return parent.variables[header["name"]]
 
 
 @dask_serialize.register(netCDF4.Group)
@@ -39,11 +39,11 @@ def serialize_netcdf4_group(g):
     while parent.parent:
         parent = parent.parent
     header, _ = serialize_netcdf4_dataset(parent)
-    header['path'] = g.path
+    header["path"] = g.path
     return header, []
 
 
 @dask_deserialize.register(netCDF4.Group)
 def deserialize_netcdf4_group(header, frames):
     file = deserialize_netcdf4_dataset(header, frames)
-    return file[header['path']]
+    return file[header["path"]]
diff --git a/distributed/protocol/numpy.py b/distributed/protocol/numpy.py
index 3227a4bbaec..d8da4f204e4 100644
--- a/distributed/protocol/numpy.py
+++ b/distributed/protocol/numpy.py
@@ -24,13 +24,13 @@ def itemsize(dt):
 @dask_serialize.register(np.ndarray)
 def serialize_numpy_ndarray(x):
     if x.dtype.hasobject:
-        header = {'pickle': True}
+        header = {"pickle": True}
         frames = [pickle.dumps(x)]
         return header, frames
 
     # We cannot blindly pickle the dtype as some may fail pickling,
     # so we have a mixture of strategies.
-    if x.dtype.kind == 'V':
+    if x.dtype.kind == "V":
         # Preserving all the information works best when pickling
         try:
             # Only use stdlib pickle as cloudpickle is slow when failing
@@ -53,31 +53,29 @@ def serialize_numpy_ndarray(x):
     elif x.flags.c_contiguous or x.flags.f_contiguous:
         # Avoid a copy and respect order when unserializing
         strides = x.strides
-        data = x.ravel(order='K')
+        data = x.ravel(order="K")
     else:
         x = np.ascontiguousarray(x)
         strides = x.strides
         data = x.ravel()
 
     if data.dtype.fields or data.dtype.itemsize > 8:
-        data = data.view('u%d' % gcd(x.dtype.itemsize, 8))
+        data = data.view("u%d" % gcd(x.dtype.itemsize, 8))
 
     try:
         data = data.data
     except ValueError:
         # "ValueError: cannot include dtype 'M' in a buffer"
-        data = data.view('u%d' % gcd(x.dtype.itemsize, 8)).data
+        data = data.view("u%d" % gcd(x.dtype.itemsize, 8)).data
 
-    header = {'dtype': dt,
-              'shape': x.shape,
-              'strides': strides}
+    header = {"dtype": dt, "shape": x.shape, "strides": strides}
 
     if x.nbytes > 1e5:
         frames = frame_split_size([data])
     else:
         frames = [data]
 
-    header['lengths'] = [x.nbytes]
+    header["lengths"] = [x.nbytes]
 
     return header, frames
 
@@ -88,17 +86,18 @@ def deserialize_numpy_ndarray(header, frames):
         if len(frames) > 1:
             frames = merge_frames(header, frames)
 
-        if header.get('pickle'):
+        if header.get("pickle"):
             return pickle.loads(frames[0])
 
-        is_custom, dt = header['dtype']
+        is_custom, dt = header["dtype"]
         if is_custom:
             dt = pickle.loads(dt)
         else:
             dt = np.dtype(dt)
 
-        x = np.ndarray(header['shape'], dtype=dt, buffer=frames[0],
-                       strides=header['strides'])
+        x = np.ndarray(
+            header["shape"], dtype=dt, buffer=frames[0], strides=header["strides"]
+        )
 
         return x
 
@@ -116,13 +115,12 @@ def deserialize_numpy_ma_masked(header, frames):
 @dask_serialize.register(np.ma.core.MaskedArray)
 def serialize_numpy_maskedarray(x):
     data_header, frames = serialize_numpy_ndarray(x.data)
-    header = {'data-header': data_header,
-              'nframes': len(frames)}
+    header = {"data-header": data_header, "nframes": len(frames)}
 
     # Serialize mask if present
     if x.mask is not np.ma.nomask:
         mask_header, mask_frames = serialize_numpy_ndarray(x.mask)
-        header['mask-header'] = mask_header
+        header["mask-header"] = mask_header
         frames += mask_frames
 
     # Only a few dtypes have python equivalents msgpack can serialize
@@ -130,7 +128,7 @@ def serialize_numpy_maskedarray(x):
         serialized_fill_value = (False, x.fill_value.item())
     else:
         serialized_fill_value = (True, pickle.dumps(x.fill_value))
-    header['fill-value'] = serialized_fill_value
+    header["fill-value"] = serialized_fill_value
 
     return header, frames
 
@@ -138,12 +136,12 @@ def serialize_numpy_maskedarray(x):
 @dask_deserialize.register(np.ma.core.MaskedArray)
 def deserialize_numpy_maskedarray(header, frames):
     data_header = header["data-header"]
-    data_frames = frames[:header["nframes"]]
+    data_frames = frames[: header["nframes"]]
     data = deserialize_numpy_ndarray(data_header, data_frames)
 
-    if 'mask-header' in header:
+    if "mask-header" in header:
         mask_header = header["mask-header"]
-        mask_frames = frames[header["nframes"]:]
+        mask_frames = frames[header["nframes"] :]
         mask = deserialize_numpy_ndarray(mask_header, mask_frames)
     else:
         mask = np.ma.nomask
diff --git a/distributed/protocol/pickle.py b/distributed/protocol/pickle.py
index 8419541687f..080bb9037db 100644
--- a/distributed/protocol/pickle.py
+++ b/distributed/protocol/pickle.py
@@ -14,14 +14,16 @@
 
 
 def _always_use_pickle_for(x):
-    mod, _, _ = x.__class__.__module__.partition('.')
-    if mod == 'numpy':
+    mod, _, _ = x.__class__.__module__.partition(".")
+    if mod == "numpy":
         import numpy as np
+
         return isinstance(x, np.ndarray)
-    elif mod == 'pandas':
+    elif mod == "pandas":
         import pandas as pd
+
         return isinstance(x, pd.core.generic.NDFrame)
-    elif mod == 'builtins':
+    elif mod == "builtins":
         return isinstance(x, (str, bytes))
     else:
         return False
@@ -37,12 +39,12 @@ def dumps(x):
     try:
         result = pickle.dumps(x, protocol=pickle.HIGHEST_PROTOCOL)
         if len(result) < 1000:
-            if b'__main__' in result:
+            if b"__main__" in result:
                 return cloudpickle.dumps(x, protocol=pickle.HIGHEST_PROTOCOL)
             else:
                 return result
         else:
-            if _always_use_pickle_for(x) or b'__main__' not in result:
+            if _always_use_pickle_for(x) or b"__main__" not in result:
                 return result
             else:
                 return cloudpickle.dumps(x, protocol=pickle.HIGHEST_PROTOCOL)
diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index a6cfbd6d042..3b0a45c8a6f 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -4,6 +4,7 @@
 
 import dask
 from dask.base import normalize_token
+
 try:
     from cytoolz import valmap, get_in
 except ImportError:
@@ -15,15 +16,14 @@
 from ..compatibility import PY2
 from ..utils import has_keyword
 from .compression import maybe_compress, decompress
-from .utils import (unpack_frames, pack_frames_prelude, frame_split_size,
-        ensure_bytes)
+from .utils import unpack_frames, pack_frames_prelude, frame_split_size, ensure_bytes
 
 
 lazy_registrations = {}
 
 
-dask_serialize = dask.utils.Dispatch('dask_serialize')
-dask_deserialize = dask.utils.Dispatch('dask_deserialize')
+dask_serialize = dask.utils.Dispatch("dask_serialize")
+dask_deserialize = dask.utils.Dispatch("dask_deserialize")
 
 
 def dask_dumps(x, context=None):
@@ -33,34 +33,34 @@ def dask_dumps(x, context=None):
         dumps = dask_serialize.dispatch(type(x))
     except TypeError:
         raise NotImplementedError(type_name)
-    if has_keyword(dumps, 'context'):
+    if has_keyword(dumps, "context"):
         header, frames = dumps(x, context=context)
     else:
         header, frames = dumps(x)
 
-    header['type'] = type_name
-    header['type-serialized'] = pickle.dumps(type(x))
-    header['serializer'] = 'dask'
+    header["type"] = type_name
+    header["type-serialized"] = pickle.dumps(type(x))
+    header["serializer"] = "dask"
     return header, frames
 
 
 def dask_loads(header, frames):
-    typ = pickle.loads(header['type-serialized'])
+    typ = pickle.loads(header["type-serialized"])
     loads = dask_deserialize.dispatch(typ)
     return loads(header, frames)
 
 
 def pickle_dumps(x):
-    return {'serializer': 'pickle'}, [pickle.dumps(x)]
+    return {"serializer": "pickle"}, [pickle.dumps(x)]
 
 
 def pickle_loads(header, frames):
-    return pickle.loads(b''.join(frames))
+    return pickle.loads(b"".join(frames))
 
 
 msgpack_len_opts = {
-    ('max_%s_len' % x): 2**31 - 1
-    for x in ['str', 'bin', 'array', 'map', 'ext']}
+    ("max_%s_len" % x): 2 ** 31 - 1 for x in ["str", "bin", "array", "map", "ext"]
+}
 
 
 def msgpack_dumps(x):
@@ -69,16 +69,17 @@ def msgpack_dumps(x):
     except Exception:
         raise NotImplementedError()
     else:
-        return {'serializer': 'msgpack'}, [frame]
+        return {"serializer": "msgpack"}, [frame]
 
 
 def msgpack_loads(header, frames):
-    return msgpack.loads(b''.join(frames), encoding='utf8', use_list=False,
-                         **msgpack_len_opts)
+    return msgpack.loads(
+        b"".join(frames), encoding="utf8", use_list=False, **msgpack_len_opts
+    )
 
 
 def serialization_error_loads(header, frames):
-    msg = '\n'.join([ensure_bytes(frame).decode('utf8') for frame in frames])
+    msg = "\n".join([ensure_bytes(frame).decode("utf8") for frame in frames])
     raise TypeError(msg)
 
 
@@ -86,16 +87,16 @@ def serialization_error_loads(header, frames):
 
 
 def register_serialization_family(name, dumps, loads):
-    families[name] = (dumps, loads, dumps and has_keyword(dumps, 'context'))
+    families[name] = (dumps, loads, dumps and has_keyword(dumps, "context"))
 
 
-register_serialization_family('dask', dask_dumps, dask_loads)
-register_serialization_family('pickle', pickle_dumps, pickle_loads)
-register_serialization_family('msgpack', msgpack_dumps, msgpack_loads)
-register_serialization_family('error', None, serialization_error_loads)
+register_serialization_family("dask", dask_dumps, dask_loads)
+register_serialization_family("pickle", pickle_dumps, pickle_loads)
+register_serialization_family("msgpack", msgpack_dumps, msgpack_loads)
+register_serialization_family("error", None, serialization_error_loads)
 
 
-def serialize(x, serializers=None, on_error='message', context=None):
+def serialize(x, serializers=None, on_error="message", context=None):
     r"""
     Convert object to a header and list of bytestrings
 
@@ -132,18 +133,18 @@ def serialize(x, serializers=None, on_error='message', context=None):
     register_serialization: Register custom serialization functions
     """
     if serializers is None:
-        serializers = ('dask', 'pickle')  # TODO: get from configuration
+        serializers = ("dask", "pickle")  # TODO: get from configuration
 
     if isinstance(x, Serialized):
         return x.header, x.frames
 
-    tb = ''
+    tb = ""
 
     for name in serializers:
         dumps, loads, wants_context = families[name]
         try:
             header, frames = dumps(x, context=context) if wants_context else dumps(x)
-            header['serializer'] = name
+            header["serializer"] = name
             return header, frames
         except NotImplementedError:
             continue
@@ -152,15 +153,15 @@ def serialize(x, serializers=None, on_error='message', context=None):
             break
 
     msg = "Could not serialize object of type %s." % type(x).__name__
-    if on_error == 'message':
+    if on_error == "message":
         frames = [msg]
         if tb:
             frames.append(tb[:100000])
 
         frames = [frame.encode() for frame in frames]
 
-        return {'serializer': 'error'}, frames
-    elif on_error == 'raise':
+        return {"serializer": "error"}, frames
+    elif on_error == "raise":
         raise TypeError(msg, str(x)[:10000])
 
 
@@ -180,10 +181,12 @@ def deserialize(header, frames, deserializers=None):
     --------
     serialize
     """
-    name = header.get('serializer')
+    name = header.get("serializer")
     if deserializers is not None and name not in deserializers:
-        raise TypeError("Data serialized with %s but only able to deserialize "
-                        "data with %s" % (name, str(list(deserializers))))
+        raise TypeError(
+            "Data serialized with %s but only able to deserialize "
+            "data with %s" % (name, str(list(deserializers)))
+        )
     dumps, loads, wants_context = families[name]
     return loads(header, frames)
 
@@ -209,8 +212,7 @@ def __repr__(self):
         return "<Serialize: %s>" % str(self.data)
 
     def __eq__(self, other):
-        return (isinstance(other, Serialize) and
-                other.data == self.data)
+        return isinstance(other, Serialize) and other.data == self.data
 
     def __ne__(self, other):
         return not (self == other)
@@ -237,13 +239,16 @@ def __init__(self, header, frames):
 
     def deserialize(self):
         from .core import decompress
+
         frames = decompress(self.header, self.frames)
         return deserialize(self.header, frames)
 
     def __eq__(self, other):
-        return (isinstance(other, Serialized) and
-                other.header == self.header and
-                other.frames == self.frames)
+        return (
+            isinstance(other, Serialized)
+            and other.header == self.header
+            and other.frames == self.frames
+        )
 
     def __ne__(self, other):
         return not (self == other)
@@ -296,16 +301,24 @@ def _extract_serialize(x, ser, path=()):
             typ = type(v)
             if typ is list or typ is dict:
                 _extract_serialize(v, ser, path + (k,))
-            elif (typ is Serialize or typ is Serialized
-                  or typ in (bytes, bytearray) and len(v) > 2**16):
+            elif (
+                typ is Serialize
+                or typ is Serialized
+                or typ in (bytes, bytearray)
+                and len(v) > 2 ** 16
+            ):
                 ser[path + (k,)] = v
     elif type(x) is list:
         for k, v in enumerate(x):
             typ = type(v)
             if typ is list or typ is dict:
                 _extract_serialize(v, ser, path + (k,))
-            elif (typ is Serialize or typ is Serialized
-                  or typ in (bytes, bytearray) and len(v) > 2**16):
+            elif (
+                typ is Serialize
+                or typ is Serialized
+                or typ in (bytes, bytearray)
+                and len(v) > 2 ** 16
+            ):
                 ser[path + (k,)] = v
 
 
@@ -318,6 +331,7 @@ def nested_deserialize(x):
     >>> nested_deserialize(msg)
     {'op': 'update', 'data': 123}
     """
+
     def replace_inner(x):
         if type(x) is dict:
             x = x.copy()
@@ -353,8 +367,8 @@ def serialize_bytelist(x, **kwargs):
         compression, frames = zip(*map(maybe_compress, frames))
     else:
         compression = []
-    header['compression'] = compression
-    header['count'] = len(frames)
+    header["compression"] = compression
+    header["count"] = len(frames)
 
     header = msgpack.dumps(header, use_bin_type=True)
     frames2 = [header] + list(frames)
@@ -365,7 +379,7 @@ def serialize_bytes(x, **kwargs):
     L = serialize_bytelist(x, **kwargs)
     if PY2:
         L = [bytes(y) for y in L]
-    return b''.join(L)
+    return b"".join(L)
 
 
 def deserialize_bytes(b):
@@ -441,7 +455,7 @@ def typename(typ):
     >>> typename(Scheduler)
     'distributed.scheduler.Scheduler'
     """
-    return typ.__module__ + '.' + typ.__name__
+    return typ.__module__ + "." + typ.__name__
 
 
 @partial(normalize_token.register, Serialized)
@@ -469,18 +483,24 @@ def _deserialize_bytes(header, frames):
 
 def _is_msgpack_serializable(v):
     typ = type(v)
-    return (typ is str or typ is int or typ is float or
-            isinstance(v, dict) and all(map(_is_msgpack_serializable, v.values()))
-                                and all(typ is str for x in v.keys()) or
-            isinstance(v, (list, tuple)) and all(map(_is_msgpack_serializable, v)))
+    return (
+        typ is str
+        or typ is int
+        or typ is float
+        or isinstance(v, dict)
+        and all(map(_is_msgpack_serializable, v.values()))
+        and all(typ is str for x in v.keys())
+        or isinstance(v, (list, tuple))
+        and all(map(_is_msgpack_serializable, v))
+    )
 
 
 def serialize_object_with_dict(est):
     header = {
-        'serializer': 'dask',
-        'type-serialized': pickle.dumps(type(est)),
-        'simple': {},
-        'complex': {}
+        "serializer": "dask",
+        "type-serialized": pickle.dumps(type(est)),
+        "simple": {},
+        "complex": {},
     }
     frames = []
 
@@ -491,30 +511,32 @@ def serialize_object_with_dict(est):
 
     for k, v in d.items():
         if _is_msgpack_serializable(v):
-            header['simple'][k] = v
+            header["simple"][k] = v
         else:
             if isinstance(v, dict):
                 h, f = serialize_object_with_dict(v)
             else:
                 h, f = serialize(v)
-            header['complex'][k] = {'header': h,
-                                    'start': len(frames),
-                                    'stop': len(frames) + len(f)}
+            header["complex"][k] = {
+                "header": h,
+                "start": len(frames),
+                "stop": len(frames) + len(f),
+            }
             frames += f
     return header, frames
 
 
 def deserialize_object_with_dict(header, frames):
-    cls = pickle.loads(header['type-serialized'])
+    cls = pickle.loads(header["type-serialized"])
     if issubclass(cls, dict):
         dd = obj = {}
     else:
         obj = object.__new__(cls)
         dd = obj.__dict__
-    dd.update(header['simple'])
-    for k, d in header['complex'].items():
-        h = d['header']
-        f = frames[d['start']: d['stop']]
+    dd.update(header["simple"])
+    for k, d in header["complex"].items():
+        h = d["header"]
+        f = frames[d["start"] : d["stop"]]
         v = deserialize(h, f)
         dd[k] = v
 
diff --git a/distributed/protocol/sparse.py b/distributed/protocol/sparse.py
index ca0c6f38a79..b5a437a32a4 100644
--- a/distributed/protocol/sparse.py
+++ b/distributed/protocol/sparse.py
@@ -10,22 +10,24 @@ def serialize_sparse(x):
     coords_header, coords_frames = serialize(x.coords)
     data_header, data_frames = serialize(x.data)
 
-    header = {'coords-header': coords_header,
-              'data-header': data_header,
-              'shape': x.shape,
-              'nframes': [len(coords_frames), len(data_frames)]}
+    header = {
+        "coords-header": coords_header,
+        "data-header": data_header,
+        "shape": x.shape,
+        "nframes": [len(coords_frames), len(data_frames)],
+    }
     return header, coords_frames + data_frames
 
 
 @dask_deserialize.register(sparse.COO)
 def deserialize_sparse(header, frames):
 
-    coords_frames = frames[:header['nframes'][0]]
-    data_frames = frames[header['nframes'][0]:]
+    coords_frames = frames[: header["nframes"][0]]
+    data_frames = frames[header["nframes"][0] :]
 
-    coords = deserialize(header['coords-header'], coords_frames)
-    data = deserialize(header['data-header'], data_frames)
+    coords = deserialize(header["coords-header"], coords_frames)
+    data = deserialize(header["data-header"], data_frames)
 
-    shape = header['shape']
+    shape = header["shape"]
 
     return sparse.COO(coords, data, shape=shape)
diff --git a/distributed/protocol/tests/test_arrow.py b/distributed/protocol/tests/test_arrow.py
index eca8de9f1a3..a363ee9511e 100644
--- a/distributed/protocol/tests/test_arrow.py
+++ b/distributed/protocol/tests/test_arrow.py
@@ -1,18 +1,18 @@
 import pandas as pd
 import pytest
 
-pa = pytest.importorskip('pyarrow')
+pa = pytest.importorskip("pyarrow")
 
 from distributed.utils_test import gen_cluster
 from distributed.protocol import deserialize, serialize
 
 
-df = pd.DataFrame({'A': list('abc'), 'B': [1,2,3]})
+df = pd.DataFrame({"A": list("abc"), "B": [1, 2, 3]})
 tbl = pa.Table.from_pandas(df, preserve_index=False)
 batch = pa.RecordBatch.from_pandas(df, preserve_index=False)
 
 
-@pytest.mark.parametrize('obj', [batch, tbl], ids=["RecordBatch", "Table"])
+@pytest.mark.parametrize("obj", [batch, tbl], ids=["RecordBatch", "Table"])
 def test_roundtrip(obj):
     # Test that the serialize/deserialize functions actually
     # work independent of distributed
@@ -25,7 +25,7 @@ def echo(arg):
     return arg
 
 
-@pytest.mark.parametrize('obj', [batch, tbl], ids=["RecordBatch", "Table"])
+@pytest.mark.parametrize("obj", [batch, tbl], ids=["RecordBatch", "Table"])
 def test_scatter(obj):
     @gen_cluster(client=True)
     def run_test(client, scheduler, worker1, worker2):
@@ -33,4 +33,5 @@ def run_test(client, scheduler, worker1, worker2):
         fut = client.submit(echo, obj_fut)
         result = yield fut
         assert obj.equals(result)
+
     run_test()
diff --git a/distributed/protocol/tests/test_h5py.py b/distributed/protocol/tests/test_h5py.py
index 3fffa9fecd4..f2f9a6625cb 100644
--- a/distributed/protocol/tests/test_h5py.py
+++ b/distributed/protocol/tests/test_h5py.py
@@ -3,7 +3,7 @@
 
 import pytest
 
-h5py = pytest.importorskip('h5py')
+h5py = pytest.importorskip("h5py")
 
 from distributed.protocol import deserialize, serialize
 
@@ -33,39 +33,39 @@ def wrapper():
 @silence_h5py_issue775
 def test_serialize_deserialize_file():
     with tmpfile() as fn:
-        with h5py.File(fn, mode='a') as f:
-            f.create_dataset('/x', shape=(2, 2), dtype='i4')
-        with h5py.File(fn, mode='r') as f:
+        with h5py.File(fn, mode="a") as f:
+            f.create_dataset("/x", shape=(2, 2), dtype="i4")
+        with h5py.File(fn, mode="r") as f:
             g = deserialize(*serialize(f))
             assert f.filename == g.filename
             assert isinstance(g, h5py.File)
             assert f.mode == g.mode
 
-            assert g['x'].shape == (2, 2)
+            assert g["x"].shape == (2, 2)
 
 
 @silence_h5py_issue775
 def test_serialize_deserialize_group():
     with tmpfile() as fn:
-        with h5py.File(fn, mode='a') as f:
-            f.create_dataset('/group1/group2/x', shape=(2, 2), dtype='i4')
-        with h5py.File(fn, mode='r') as f:
-            group = f['/group1/group2']
+        with h5py.File(fn, mode="a") as f:
+            f.create_dataset("/group1/group2/x", shape=(2, 2), dtype="i4")
+        with h5py.File(fn, mode="r") as f:
+            group = f["/group1/group2"]
             group2 = deserialize(*serialize(group))
 
             assert isinstance(group2, h5py.Group)
             assert group.file.filename == group2.file.filename
 
-            assert group2['x'].shape == (2, 2)
+            assert group2["x"].shape == (2, 2)
 
 
 @silence_h5py_issue775
 def test_serialize_deserialize_dataset():
     with tmpfile() as fn:
-        with h5py.File(fn, mode='a') as f:
-            x = f.create_dataset('/group1/group2/x', shape=(2, 2), dtype='i4')
-        with h5py.File(fn, mode='r') as f:
-            x = f['group1/group2/x']
+        with h5py.File(fn, mode="a") as f:
+            x = f.create_dataset("/group1/group2/x", shape=(2, 2), dtype="i4")
+        with h5py.File(fn, mode="r") as f:
+            x = f["group1/group2/x"]
             y = deserialize(*serialize(x))
             assert isinstance(y, h5py.Dataset)
             assert x.name == y.name
@@ -76,8 +76,8 @@ def test_serialize_deserialize_dataset():
 @silence_h5py_issue775
 def test_raise_error_on_serialize_write_permissions():
     with tmpfile() as fn:
-        with h5py.File(fn, mode='a') as f:
-            x = f.create_dataset('/x', shape=(2, 2), dtype='i4')
+        with h5py.File(fn, mode="a") as f:
+            x = f.create_dataset("/x", shape=(2, 2), dtype="i4")
             f.flush()
             with pytest.raises(TypeError):
                 deserialize(*serialize(x))
@@ -95,14 +95,14 @@ def test_raise_error_on_serialize_write_permissions():
 @gen_cluster(client=True)
 def test_h5py_serialize(c, s, a, b):
     from dask.utils import SerializableLock
-    lock = SerializableLock('hdf5')
+
+    lock = SerializableLock("hdf5")
     with tmpfile() as fn:
-        with h5py.File(fn, mode='a') as f:
-            x = f.create_dataset('/group/x', shape=(4,), dtype='i4',
-                                 chunks=(2,))
+        with h5py.File(fn, mode="a") as f:
+            x = f.create_dataset("/group/x", shape=(4,), dtype="i4", chunks=(2,))
             x[:] = [1, 2, 3, 4]
-        with h5py.File(fn, mode='r') as f:
-            dset = f['/group/x']
+        with h5py.File(fn, mode="r") as f:
+            dset = f["/group/x"]
             x = da.from_array(dset, chunks=dset.chunks, lock=lock)
             y = c.compute(x)
             y = yield y
@@ -112,12 +112,11 @@ def test_h5py_serialize(c, s, a, b):
 @gen_cluster(client=True)
 def test_h5py_serialize_2(c, s, a, b):
     with tmpfile() as fn:
-        with h5py.File(fn, mode='a') as f:
-            x = f.create_dataset('/group/x', shape=(12,), dtype='i4',
-                                 chunks=(4,))
+        with h5py.File(fn, mode="a") as f:
+            x = f.create_dataset("/group/x", shape=(12,), dtype="i4", chunks=(4,))
             x[:] = [1, 2, 3, 4] * 3
-        with h5py.File(fn, mode='r') as f:
-            dset = f['/group/x']
+        with h5py.File(fn, mode="r") as f:
+            dset = f["/group/x"]
             x = da.from_array(dset, chunks=(3,))
             y = c.compute(x.sum())
             y = yield y
diff --git a/distributed/protocol/tests/test_keras.py b/distributed/protocol/tests/test_keras.py
index b246b33bfe8..da8cdf6374a 100644
--- a/distributed/protocol/tests/test_keras.py
+++ b/distributed/protocol/tests/test_keras.py
@@ -1,9 +1,8 @@
-
 import numpy as np
 from numpy.testing import assert_allclose
 import pytest
 
-keras = pytest.importorskip('keras')
+keras = pytest.importorskip("keras")
 
 from distributed.protocol import serialize, deserialize, dumps, loads, to_serialize
 
@@ -12,7 +11,7 @@ def test_serialize_deserialize_model():
     model = keras.models.Sequential()
     model.add(keras.layers.Dense(5, input_dim=3))
     model.add(keras.layers.Dense(2))
-    model.compile(optimizer='sgd', loss='mse')
+    model.compile(optimizer="sgd", loss="mse")
     x = np.random.random((1, 3))
     y = np.random.random((1, 2))
     model.train_on_batch(x, y)
@@ -20,7 +19,7 @@ def test_serialize_deserialize_model():
     loaded = deserialize(*serialize(model))
     assert_allclose(loaded.predict(x), model.predict(x))
 
-    data = {'model': to_serialize(model)}
+    data = {"model": to_serialize(model)}
     frames = dumps(data)
     result = loads(frames)
-    assert_allclose(result['model'].predict(x), model.predict(x))
+    assert_allclose(result["model"].predict(x), model.predict(x))
diff --git a/distributed/protocol/tests/test_netcdf4.py b/distributed/protocol/tests/test_netcdf4.py
index 381f6468182..f1ddcead3ef 100644
--- a/distributed/protocol/tests/test_netcdf4.py
+++ b/distributed/protocol/tests/test_netcdf4.py
@@ -1,7 +1,7 @@
 import pytest
 
-netCDF4 = pytest.importorskip('netCDF4')
-np = pytest.importorskip('numpy')
+netCDF4 = pytest.importorskip("netCDF4")
+np = pytest.importorskip("numpy")
 
 from distributed.protocol import deserialize, serialize
 
@@ -9,51 +9,51 @@
 
 
 def create_test_dataset(fn):
-    with netCDF4.Dataset(fn, mode='w') as ds:
-        ds.createDimension('x', 3)
-        v = ds.createVariable('x', np.int32, ('x',))
+    with netCDF4.Dataset(fn, mode="w") as ds:
+        ds.createDimension("x", 3)
+        v = ds.createVariable("x", np.int32, ("x",))
         v[:] = np.arange(3)
 
-        g = ds.createGroup('group')
-        g2 = ds.createGroup('group/group1')
+        g = ds.createGroup("group")
+        g2 = ds.createGroup("group/group1")
 
-        v2 = ds.createVariable('group/y', np.int32, ('x',))
+        v2 = ds.createVariable("group/y", np.int32, ("x",))
         v2[:] = np.arange(3) + 1
 
-        v3 = ds.createVariable('group/group1/z', np.int32, ('x',))
+        v3 = ds.createVariable("group/group1/z", np.int32, ("x",))
         v3[:] = np.arange(3) + 2
 
 
 def test_serialize_deserialize_dataset():
     with tmpfile() as fn:
         create_test_dataset(fn)
-        with netCDF4.Dataset(fn, mode='r') as f:
+        with netCDF4.Dataset(fn, mode="r") as f:
             g = deserialize(*serialize(f))
             assert f.filepath() == g.filepath()
             assert isinstance(g, netCDF4.Dataset)
 
-            assert g.variables['x'].dimensions == ('x',)
-            assert g.variables['x'].dtype == np.int32
-            assert (g.variables['x'][:] == np.arange(3)).all()
+            assert g.variables["x"].dimensions == ("x",)
+            assert g.variables["x"].dtype == np.int32
+            assert (g.variables["x"][:] == np.arange(3)).all()
 
 
 def test_serialize_deserialize_variable():
     with tmpfile() as fn:
         create_test_dataset(fn)
-        with netCDF4.Dataset(fn, mode='r') as f:
-            x = f.variables['x']
+        with netCDF4.Dataset(fn, mode="r") as f:
+            x = f.variables["x"]
             y = deserialize(*serialize(x))
             assert isinstance(y, netCDF4.Variable)
-            assert y.dimensions == ('x',)
-            assert (x.dtype == y.dtype)
+            assert y.dimensions == ("x",)
+            assert x.dtype == y.dtype
             assert (x[:] == y[:]).all()
 
 
 def test_serialize_deserialize_group():
     with tmpfile() as fn:
         create_test_dataset(fn)
-        with netCDF4.Dataset(fn, mode='r') as f:
-            for path in ['group', 'group/group1']:
+        with netCDF4.Dataset(fn, mode="r") as f:
+            for path in ["group", "group/group1"]:
                 g = f[path]
                 h = deserialize(*serialize(g))
                 assert isinstance(h, netCDF4.Group)
@@ -61,15 +61,17 @@ def test_serialize_deserialize_group():
                 assert list(g.groups) == list(h.groups)
                 assert list(g.variables) == list(h.variables)
 
-            vars = [f.variables['x'],
-                    f['group'].variables['y'],
-                    f['group/group1'].variables['z']]
+            vars = [
+                f.variables["x"],
+                f["group"].variables["y"],
+                f["group/group1"].variables["z"],
+            ]
 
             for x in vars:
                 y = deserialize(*serialize(x))
                 assert isinstance(y, netCDF4.Variable)
-                assert y.dimensions == ('x',)
-                assert (x.dtype == y.dtype)
+                assert y.dimensions == ("x",)
+                assert x.dtype == y.dtype
                 assert (x[:] == y[:]).all()
 
 
@@ -83,8 +85,8 @@ def test_serialize_deserialize_group():
 def test_netcdf4_serialize(c, s, a, b):
     with tmpfile() as fn:
         create_test_dataset(fn)
-        with netCDF4.Dataset(fn, mode='r') as f:
-            dset = f.variables['x']
+        with netCDF4.Dataset(fn, mode="r") as f:
+            dset = f.variables["x"]
             x = da.from_array(dset, chunks=2)
             y = c.compute(x)
             y = yield y
diff --git a/distributed/protocol/tests/test_numpy.py b/distributed/protocol/tests/test_numpy.py
index c86a5f03199..849c2964fd6 100644
--- a/distributed/protocol/tests/test_numpy.py
+++ b/distributed/protocol/tests/test_numpy.py
@@ -7,8 +7,15 @@
 import pytest
 
 from distributed.compatibility import PY2
-from distributed.protocol import (serialize, deserialize, decompress, dumps,
-                                  loads, to_serialize, msgpack)
+from distributed.protocol import (
+    serialize,
+    deserialize,
+    decompress,
+    dumps,
+    loads,
+    to_serialize,
+    msgpack,
+)
 from distributed.protocol.utils import BIG_BYTES_SHARD_SIZE
 from distributed.utils import tmpfile, nbytes
 from distributed.utils_test import slow, gen_cluster
@@ -19,55 +26,58 @@
 def test_serialize():
     x = np.ones((5, 5))
     header, frames = serialize(x)
-    assert header['type']
+    assert header["type"]
     assert len(frames) == 1
 
-    if 'compression' in header:
+    if "compression" in header:
         frames = decompress(header, frames)
     result = deserialize(header, frames)
     assert (result == x).all()
 
 
-@pytest.mark.parametrize('x', [
-    np.ones(5),
-    np.array(5),
-    np.random.random((5, 5)),
-    np.random.random((5, 5))[::2, :],
-    np.random.random((5, 5))[:, ::2],
-    np.asfortranarray(np.random.random((5, 5))),
-    np.asfortranarray(np.random.random((5, 5)))[::2, :],
-    np.asfortranarray(np.random.random((5, 5)))[:, ::2],
-    np.random.random(5).astype('f4'),
-    np.random.random(5).astype('>i8'),
-    np.random.random(5).astype('<i8'),
-    np.arange(5).astype('M8[us]'),
-    np.arange(5).astype('M8[ms]'),
-    np.arange(5).astype('m8'),
-    np.arange(5).astype('m8[s]'),
-    np.arange(5).astype('c16'),
-    np.arange(5).astype('c8'),
-    np.array([True, False, True]),
-    np.ones(shape=5, dtype=[('a', 'i4'), ('b', 'M8[us]')]),
-    np.array(['abc'], dtype='S3'),
-    np.array(['abc'], dtype='U3'),
-    np.array(['abc'], dtype=object),
-    np.ones(shape=(5,), dtype=('f8', 32)),
-    np.ones(shape=(5,), dtype=[('x', 'f8', 32)]),
-    np.ones(shape=(5,), dtype=np.dtype([('a', 'i1'), ('b', 'f8')], align=False)),
-    np.ones(shape=(5,), dtype=np.dtype([('a', 'i1'), ('b', 'f8')], align=True)),
-    np.ones(shape=(5,), dtype=np.dtype([('a', 'm8[us]')], align=False)),
-    # this dtype fails unpickling
-    np.ones(shape=(5,), dtype=np.dtype([('a', 'm8')], align=False)),
-    np.array([(1, 'abc')], dtype=[('x', 'i4'), ('s', object)]),
-    np.zeros(5000, dtype=[('x%d' % i, '<f8') for i in range(4)]),
-    np.zeros(5000, dtype='S32'),
-    np.zeros((1, 1000, 1000)),
-    np.arange(12)[::2],  # non-contiguous array
-    np.ones(shape=(5, 6)).astype(dtype=[('total', '<f8'), ('n', '<f8')]),
-])
+@pytest.mark.parametrize(
+    "x",
+    [
+        np.ones(5),
+        np.array(5),
+        np.random.random((5, 5)),
+        np.random.random((5, 5))[::2, :],
+        np.random.random((5, 5))[:, ::2],
+        np.asfortranarray(np.random.random((5, 5))),
+        np.asfortranarray(np.random.random((5, 5)))[::2, :],
+        np.asfortranarray(np.random.random((5, 5)))[:, ::2],
+        np.random.random(5).astype("f4"),
+        np.random.random(5).astype(">i8"),
+        np.random.random(5).astype("<i8"),
+        np.arange(5).astype("M8[us]"),
+        np.arange(5).astype("M8[ms]"),
+        np.arange(5).astype("m8"),
+        np.arange(5).astype("m8[s]"),
+        np.arange(5).astype("c16"),
+        np.arange(5).astype("c8"),
+        np.array([True, False, True]),
+        np.ones(shape=5, dtype=[("a", "i4"), ("b", "M8[us]")]),
+        np.array(["abc"], dtype="S3"),
+        np.array(["abc"], dtype="U3"),
+        np.array(["abc"], dtype=object),
+        np.ones(shape=(5,), dtype=("f8", 32)),
+        np.ones(shape=(5,), dtype=[("x", "f8", 32)]),
+        np.ones(shape=(5,), dtype=np.dtype([("a", "i1"), ("b", "f8")], align=False)),
+        np.ones(shape=(5,), dtype=np.dtype([("a", "i1"), ("b", "f8")], align=True)),
+        np.ones(shape=(5,), dtype=np.dtype([("a", "m8[us]")], align=False)),
+        # this dtype fails unpickling
+        np.ones(shape=(5,), dtype=np.dtype([("a", "m8")], align=False)),
+        np.array([(1, "abc")], dtype=[("x", "i4"), ("s", object)]),
+        np.zeros(5000, dtype=[("x%d" % i, "<f8") for i in range(4)]),
+        np.zeros(5000, dtype="S32"),
+        np.zeros((1, 1000, 1000)),
+        np.arange(12)[::2],  # non-contiguous array
+        np.ones(shape=(5, 6)).astype(dtype=[("total", "<f8"), ("n", "<f8")]),
+    ],
+)
 def test_dumps_serialize_numpy(x):
     header, frames = serialize(x)
-    if 'compression' in header:
+    if "compression" in header:
         frames = decompress(header, frames)
     buffer_interface = buffer if PY2 else memoryview  # noqa: F821
     for frame in frames:
@@ -79,14 +89,23 @@ def test_dumps_serialize_numpy(x):
         assert x.strides == y.strides
 
 
-@pytest.mark.parametrize('x', [
-    np.ma.masked_array([5, 6], mask=[True, False], fill_value=10, dtype='i4'),
-    np.ma.masked_array([5., 6.], mask=[True, False], fill_value=10, dtype='f4'),
-    np.ma.masked_array([5., 6.], mask=[True, False], fill_value=np.nan, dtype='f8'),
-    np.ma.masked_array([5., 6.], mask=np.ma.nomask, fill_value=np.nan, dtype='f8'),
-    np.ma.masked_array([True, False], mask=np.ma.nomask, fill_value=True, dtype='bool'),
-    np.ma.masked_array(['a', 'b'], mask=[True, False], fill_value='c', dtype='O')
-])
+@pytest.mark.parametrize(
+    "x",
+    [
+        np.ma.masked_array([5, 6], mask=[True, False], fill_value=10, dtype="i4"),
+        np.ma.masked_array([5.0, 6.0], mask=[True, False], fill_value=10, dtype="f4"),
+        np.ma.masked_array(
+            [5.0, 6.0], mask=[True, False], fill_value=np.nan, dtype="f8"
+        ),
+        np.ma.masked_array(
+            [5.0, 6.0], mask=np.ma.nomask, fill_value=np.nan, dtype="f8"
+        ),
+        np.ma.masked_array(
+            [True, False], mask=np.ma.nomask, fill_value=True, dtype="bool"
+        ),
+        np.ma.masked_array(["a", "b"], mask=[True, False], fill_value="c", dtype="O"),
+    ],
+)
 def test_serialize_numpy_ma_masked_array(x):
     y, = loads(dumps([to_serialize(x)]))
     assert x.data.dtype == y.data.dtype
@@ -102,10 +121,13 @@ def test_serialize_numpy_ma_masked():
 
 def test_dumps_serialize_numpy_custom_dtype():
     from six.moves import builtins
-    test_rational = pytest.importorskip('numpy.core.test_rational')
+
+    test_rational = pytest.importorskip("numpy.core.test_rational")
     rational = test_rational.rational
     try:
-        builtins.rational = rational  # Work around https://github.com/numpy/numpy/issues/9160
+        builtins.rational = (
+            rational
+        )  # Work around https://github.com/numpy/numpy/issues/9160
         x = np.array([1], dtype=rational)
         header, frames = serialize(x)
         y = deserialize(header, frames)
@@ -116,14 +138,14 @@ def test_dumps_serialize_numpy_custom_dtype():
 
 
 def test_memmap():
-    with tmpfile('npy') as fn:
-        with open(fn, 'wb') as f:  # touch file
+    with tmpfile("npy") as fn:
+        with open(fn, "wb") as f:  # touch file
             pass
-        x = np.memmap(fn, shape=(5, 5), dtype='i4', mode='readwrite')
+        x = np.memmap(fn, shape=(5, 5), dtype="i4", mode="readwrite")
         x[:] = 5
 
         header, frames = serialize(x)
-        if 'compression' in header:
+        if "compression" in header:
             frames = decompress(header, frames)
         y = deserialize(header, frames)
 
@@ -132,10 +154,10 @@ def test_memmap():
 
 @slow
 def test_dumps_serialize_numpy_large():
-    psutil = pytest.importorskip('psutil')
+    psutil = pytest.importorskip("psutil")
     if psutil.virtual_memory().total < 2e9:
         return
-    x = np.random.random(size=int(BIG_BYTES_SHARD_SIZE * 2 // 8)).view('u1')
+    x = np.random.random(size=int(BIG_BYTES_SHARD_SIZE * 2 // 8)).view("u1")
     assert x.nbytes == BIG_BYTES_SHARD_SIZE * 2
     frames = dumps([to_serialize(x)])
     dtype, shape = x.dtype, x.shape
@@ -147,29 +169,33 @@ def test_dumps_serialize_numpy_large():
     assert crc32(y) == checksum, "Arrays are unequal"
 
 
-@pytest.mark.parametrize('dt,size', [('f8', 8),
-                                     ('i4', 4),
-                                     ('c16', 16),
-                                     ('b', 1),
-                                     ('S3', 3),
-                                     ('M8[us]', 8),
-                                     ('M8[s]', 8),
-                                     ('U3', 12),
-                                     ([('a', 'i4'), ('b', 'f8')], 12),
-                                     (('i4', 100), 4),
-                                     ([('a', 'i4', 100)], 8),
-                                     ([('a', 'i4', 20), ('b', 'f8')], 20 * 4 + 8),
-                                     ([('a', 'i4', 200), ('b', 'f8')], 8)])
+@pytest.mark.parametrize(
+    "dt,size",
+    [
+        ("f8", 8),
+        ("i4", 4),
+        ("c16", 16),
+        ("b", 1),
+        ("S3", 3),
+        ("M8[us]", 8),
+        ("M8[s]", 8),
+        ("U3", 12),
+        ([("a", "i4"), ("b", "f8")], 12),
+        (("i4", 100), 4),
+        ([("a", "i4", 100)], 8),
+        ([("a", "i4", 20), ("b", "f8")], 20 * 4 + 8),
+        ([("a", "i4", 200), ("b", "f8")], 8),
+    ],
+)
 def test_itemsize(dt, size):
     assert itemsize(np.dtype(dt)) == size
 
 
-@pytest.mark.skipif(sys.version_info[0] < 3,
-                    reason='numpy doesnt use memoryviews')
+@pytest.mark.skipif(sys.version_info[0] < 3, reason="numpy doesnt use memoryviews")
 def test_compress_numpy():
-    pytest.importorskip('lz4')
-    x = np.ones(10000000, dtype='i4')
-    frames = dumps({'x': to_serialize(x)})
+    pytest.importorskip("lz4")
+    x = np.ones(10000000, dtype="i4")
+    frames = dumps({"x": to_serialize(x)})
     assert sum(map(nbytes, frames)) < x.nbytes
 
     header = msgpack.loads(frames[2], raw=False, use_list=False)
@@ -178,12 +204,11 @@ def test_compress_numpy():
     except ImportError:
         pass
     else:
-        assert all(c == 'blosc' for c in
-                   header['headers'][('x',)]['compression'])
+        assert all(c == "blosc" for c in header["headers"][("x",)]["compression"])
 
 
 def test_compress_memoryview():
-    mv = memoryview(b'0' * 1000000)
+    mv = memoryview(b"0" * 1000000)
     compression, compressed = maybe_compress(mv)
     if compression:
         assert len(compressed) < len(mv)
@@ -191,49 +216,49 @@ def test_compress_memoryview():
 
 @pytest.mark.skip
 def test_dont_compress_uncompressable_data():
-    blosc = pytest.importorskip('blosc')
-    x = np.random.randint(0, 255, size=100000).astype('uint8')
+    blosc = pytest.importorskip("blosc")
+    x = np.random.randint(0, 255, size=100000).astype("uint8")
     header, [data] = serialize(x)
-    assert 'compression' not in header
+    assert "compression" not in header
     assert data == x.data
 
     x = np.ones(1000000)
     header, [data] = serialize(x)
-    assert header['compression'] == ['blosc']
+    assert header["compression"] == ["blosc"]
     assert data != x.data
 
     x = np.ones(100)
     header, [data] = serialize(x)
-    assert 'compression' not in header
+    assert "compression" not in header
     if isinstance(data, memoryview):
         assert data.obj.ctypes.data == x.ctypes.data
 
 
 @gen_cluster(client=True, timeout=60)
 def test_dumps_large_blosc(c, s, a, b):
-    x = c.submit(np.ones, BIG_BYTES_SHARD_SIZE * 2, dtype='u1')
+    x = c.submit(np.ones, BIG_BYTES_SHARD_SIZE * 2, dtype="u1")
     result = yield x
 
 
-@pytest.mark.skipif(sys.version_info[0] < 3,
-                    reason='numpy doesnt use memoryviews')
+@pytest.mark.skipif(sys.version_info[0] < 3, reason="numpy doesnt use memoryviews")
 def test_compression_takes_advantage_of_itemsize():
-    pytest.importorskip('lz4')
-    blosc = pytest.importorskip('blosc')
-    x = np.arange(1000000, dtype='i8')
+    pytest.importorskip("lz4")
+    blosc = pytest.importorskip("blosc")
+    x = np.arange(1000000, dtype="i8")
 
-    assert (len(blosc.compress(x.data, typesize=8))
-            < len(blosc.compress(x.data, typesize=1)))
+    assert len(blosc.compress(x.data, typesize=8)) < len(
+        blosc.compress(x.data, typesize=1)
+    )
 
     _, a = serialize(x)
     aa = [maybe_compress(frame)[1] for frame in a]
-    _, b = serialize(x.view('u1'))
+    _, b = serialize(x.view("u1"))
     bb = [maybe_compress(frame)[1] for frame in b]
 
     assert sum(map(nbytes, aa)) < sum(map(nbytes, bb))
 
 
 def test_large_numpy_array():
-    x = np.ones((100000000,), dtype='u4')
+    x = np.ones((100000000,), dtype="u4")
     header, frames = serialize(x)
-    assert sum(header['lengths']) == sum(map(nbytes, frames))
+    assert sum(header["lengths"]) == sum(map(nbytes, frames))
diff --git a/distributed/protocol/tests/test_pandas.py b/distributed/protocol/tests/test_pandas.py
index ad74f04fd39..8f5827f7896 100644
--- a/distributed/protocol/tests/test_pandas.py
+++ b/distributed/protocol/tests/test_pandas.py
@@ -7,26 +7,24 @@
 
 from dask.dataframe.utils import assert_eq
 
-from distributed.protocol import (serialize, deserialize, decompress)
+from distributed.protocol import serialize, deserialize, decompress
 
 
 dfs = [
     pd.DataFrame({}),
-    pd.DataFrame({'x': [1, 2, 3]}),
-    pd.DataFrame({'x': [1., 2., 3.]}),
+    pd.DataFrame({"x": [1, 2, 3]}),
+    pd.DataFrame({"x": [1.0, 2.0, 3.0]}),
     pd.DataFrame({0: [1, 2, 3]}),
-    pd.DataFrame({'x': [1., 2., 3.], 'y': [4., 5., 6.]}),
-    pd.DataFrame({'x': [1., 2., 3.]}, index=pd.Index([4, 5, 6], name='bar')),
-    pd.Series([1., 2., 3.]),
-    pd.Series([1., 2., 3.], name='foo'),
-    pd.Series([1., 2., 3.], name='foo',
-              index=[4, 5, 6]),
-    pd.Series([1., 2., 3.], name='foo',
-              index=pd.Index([4, 5, 6], name='bar')),
-    pd.DataFrame({'x': ['a', 'b', 'c']}),
-    pd.DataFrame({'x': [b'a', b'b', b'c']}),
-    pd.DataFrame({'x': pd.Categorical(['a', 'b', 'a'], ordered=True)}),
-    pd.DataFrame({'x': pd.Categorical(['a', 'b', 'a'], ordered=False)}),
+    pd.DataFrame({"x": [1.0, 2.0, 3.0], "y": [4.0, 5.0, 6.0]}),
+    pd.DataFrame({"x": [1.0, 2.0, 3.0]}, index=pd.Index([4, 5, 6], name="bar")),
+    pd.Series([1.0, 2.0, 3.0]),
+    pd.Series([1.0, 2.0, 3.0], name="foo"),
+    pd.Series([1.0, 2.0, 3.0], name="foo", index=[4, 5, 6]),
+    pd.Series([1.0, 2.0, 3.0], name="foo", index=pd.Index([4, 5, 6], name="bar")),
+    pd.DataFrame({"x": ["a", "b", "c"]}),
+    pd.DataFrame({"x": [b"a", b"b", b"c"]}),
+    pd.DataFrame({"x": pd.Categorical(["a", "b", "a"], ordered=True)}),
+    pd.DataFrame({"x": pd.Categorical(["a", "b", "a"], ordered=False)}),
     tm.makeCategoricalIndex(),
     tm.makeCustomDataframe(5, 3),
     tm.makeDataFrame(),
@@ -42,10 +40,10 @@
 ]
 
 
-@pytest.mark.parametrize('df', dfs)
+@pytest.mark.parametrize("df", dfs)
 def test_dumps_serialize_numpy(df):
     header, frames = serialize(df)
-    if 'compression' in header:
+    if "compression" in header:
         frames = decompress(header, frames)
     df2 = deserialize(header, frames)
 
diff --git a/distributed/protocol/tests/test_pickle.py b/distributed/protocol/tests/test_pickle.py
index c330ecfc1d9..70d9cdaff22 100644
--- a/distributed/protocol/tests/test_pickle.py
+++ b/distributed/protocol/tests/test_pickle.py
@@ -8,13 +8,13 @@
 
 
 def test_pickle_data():
-    data = [1, b'123', '123', [123], {}, set()]
+    data = [1, b"123", "123", [123], {}, set()]
     for d in data:
         assert loads(dumps(d)) == d
 
 
 def test_pickle_numpy():
-    np = pytest.importorskip('numpy')
+    np = pytest.importorskip("numpy")
     x = np.ones(5)
     assert (loads(dumps(x)) == x).all()
 
@@ -28,6 +28,7 @@ def make_closure():
 
         def f(x):  # closure
             return x + value
+
         return f
 
     def funcs():
diff --git a/distributed/protocol/tests/test_protocol.py b/distributed/protocol/tests/test_protocol.py
index 9701cf7768c..f0dc1dc6c2f 100644
--- a/distributed/protocol/tests/test_protocol.py
+++ b/distributed/protocol/tests/test_protocol.py
@@ -5,58 +5,55 @@
 import dask
 import pytest
 
-from distributed.protocol import (loads, dumps, msgpack, maybe_compress,
-                                  to_serialize)
+from distributed.protocol import loads, dumps, msgpack, maybe_compress, to_serialize
 from distributed.protocol.compression import compressions
-from distributed.protocol.serialize import (Serialize, Serialized,
-                                            serialize, deserialize)
+from distributed.protocol.serialize import Serialize, Serialized, serialize, deserialize
 from distributed.utils import nbytes
 from distributed.utils_test import slow
 
 
 def test_protocol():
-    for msg in [1, 'a', b'a', {'x': 1}, {b'x': 1}, {'x': b''}, {}]:
+    for msg in [1, "a", b"a", {"x": 1}, {b"x": 1}, {"x": b""}, {}]:
         assert loads(dumps(msg)) == msg
 
 
 def test_compression_1():
-    pytest.importorskip('lz4')
-    np = pytest.importorskip('numpy')
+    pytest.importorskip("lz4")
+    np = pytest.importorskip("numpy")
     x = np.ones(1000000)
-    frames = dumps({'x': Serialize(x.tobytes())})
+    frames = dumps({"x": Serialize(x.tobytes())})
     assert sum(map(nbytes, frames)) < x.nbytes
     y = loads(frames)
-    assert {'x': x.tobytes()} == y
+    assert {"x": x.tobytes()} == y
 
 
 def test_compression_2():
-    pytest.importorskip('lz4')
-    np = pytest.importorskip('numpy')
+    pytest.importorskip("lz4")
+    np = pytest.importorskip("numpy")
     x = np.random.random(10000)
     header, payload = dumps(x.tobytes())
-    assert (not header or
-            not msgpack.loads(header, encoding='utf8').get('compression'))
+    assert not header or not msgpack.loads(header, encoding="utf8").get("compression")
 
 
 def test_compression_without_deserialization():
-    pytest.importorskip('lz4')
-    np = pytest.importorskip('numpy')
+    pytest.importorskip("lz4")
+    np = pytest.importorskip("numpy")
     x = np.ones(1000000)
 
-    frames = dumps({'x': Serialize(x)})
+    frames = dumps({"x": Serialize(x)})
     assert all(len(frame) < 1000000 for frame in frames)
 
     msg = loads(frames, deserialize=False)
-    assert all(len(frame) < 1000000 for frame in msg['x'].frames)
+    assert all(len(frame) < 1000000 for frame in msg["x"].frames)
 
 
 def test_small():
-    assert sum(map(nbytes, dumps(b''))) < 10
+    assert sum(map(nbytes, dumps(b""))) < 10
     assert sum(map(nbytes, dumps(1))) < 10
 
 
 def test_small_and_big():
-    d = {'x': (1, 2, 3), 'y': b'0' * 10000000}
+    d = {"x": (1, 2, 3), "y": b"0" * 10000000}
     L = dumps(d)
     assert loads(L) == d
     # assert loads([small_header, small]) == {'x': [1, 2, 3]}
@@ -67,11 +64,11 @@ def test_maybe_compress():
     pass
 
     try_converters = [bytes, memoryview]
-    try_compressions = ['zlib', 'lz4']
+    try_compressions = ["zlib", "lz4"]
 
-    payload = b'123'
+    payload = b"123"
 
-    with dask.config.set({'distributed.comm.compression': None}):
+    with dask.config.set({"distributed.comm.compression": None}):
         for f in try_converters:
             assert maybe_compress(f(payload)) == (None, payload)
 
@@ -81,22 +78,22 @@ def test_maybe_compress():
         except ImportError:
             continue
 
-        with dask.config.set({'distributed.comm.compression': compression}):
+        with dask.config.set({"distributed.comm.compression": compression}):
             for f in try_converters:
-                payload = b'123'
+                payload = b"123"
                 assert maybe_compress(f(payload)) == (None, payload)
 
-                payload = b'0' * 10000
+                payload = b"0" * 10000
                 rc, rd = maybe_compress(f(payload))
                 # For some reason compressing memoryviews can force blosc...
-                assert rc in (compression, 'blosc')
-                assert compressions[rc]['decompress'](rd) == payload
+                assert rc in (compression, "blosc")
+                assert compressions[rc]["decompress"](rd) == payload
 
 
 def test_maybe_compress_sample():
-    np = pytest.importorskip('numpy')
-    lz4 = pytest.importorskip('lz4')
-    payload = np.random.randint(0, 255, size=10000).astype('u1').tobytes()
+    np = pytest.importorskip("numpy")
+    lz4 = pytest.importorskip("lz4")
+    payload = np.random.randint(0, 255, size=10000).astype("u1").tobytes()
     fmt, compressed = maybe_compress(payload)
     assert fmt is None
     assert compressed == payload
@@ -104,7 +101,7 @@ def test_maybe_compress_sample():
 
 def test_large_bytes():
     for tp in (bytes, bytearray):
-        msg = {'x': tp(b'0' * 1000000), 'y': 1}
+        msg = {"x": tp(b"0" * 1000000), "y": 1}
         frames = dumps(msg)
         assert loads(frames) == msg
         assert len(frames[0]) < 1000
@@ -115,34 +112,37 @@ def test_large_bytes():
 
 @slow
 def test_large_messages():
-    np = pytest.importorskip('numpy')
-    psutil = pytest.importorskip('psutil')
-    pytest.importorskip('lz4')
+    np = pytest.importorskip("numpy")
+    psutil = pytest.importorskip("psutil")
+    pytest.importorskip("lz4")
     if psutil.virtual_memory().total < 8e9:
         return
 
     if sys.version_info.major == 2:
         return 2
 
-    x = np.random.randint(0, 255, size=200000000, dtype='u1')
+    x = np.random.randint(0, 255, size=200000000, dtype="u1")
 
-    msg = {'x': [Serialize(x), b'small_bytes'],
-           'y': {'a': Serialize(x), 'b': b'small_bytes'}}
+    msg = {
+        "x": [Serialize(x), b"small_bytes"],
+        "y": {"a": Serialize(x), "b": b"small_bytes"},
+    }
 
     b = dumps(msg)
     msg2 = loads(b)
-    assert msg['x'][1] == msg2['x'][1]
-    assert msg['y']['b'] == msg2['y']['b']
-    assert (msg['x'][0].data == msg2['x'][0]).all()
-    assert (msg['y']['a'].data == msg2['y']['a']).all()
+    assert msg["x"][1] == msg2["x"][1]
+    assert msg["y"]["b"] == msg2["y"]["b"]
+    assert (msg["x"][0].data == msg2["x"][0]).all()
+    assert (msg["y"]["a"].data == msg2["y"]["a"]).all()
 
 
 def test_large_messages_map():
     import psutil
+
     if psutil.virtual_memory().total < 8e9:
         pytest.skip("insufficient memory")
 
-    x = {i: 'mystring_%d' % i for i in range(100000)}
+    x = {i: "mystring_%d" % i for i in range(100000)}
 
     b = dumps(x)
     x2 = loads(b)
@@ -150,57 +150,54 @@ def test_large_messages_map():
 
 
 def test_loads_deserialize_False():
-    frames = dumps({'data': Serialize(123), 'status': 'OK'})
+    frames = dumps({"data": Serialize(123), "status": "OK"})
     msg = loads(frames)
-    assert msg == {'data': 123, 'status': 'OK'}
+    assert msg == {"data": 123, "status": "OK"}
 
     msg = loads(frames, deserialize=False)
-    assert msg['status'] == 'OK'
-    assert isinstance(msg['data'], Serialized)
+    assert msg["status"] == "OK"
+    assert isinstance(msg["data"], Serialized)
 
-    result = deserialize(msg['data'].header, msg['data'].frames)
+    result = deserialize(msg["data"].header, msg["data"].frames)
     assert result == 123
 
 
 def test_loads_without_deserialization_avoids_compression():
-    pytest.importorskip('lz4')
-    b = b'0' * 100000
+    pytest.importorskip("lz4")
+    b = b"0" * 100000
 
-    msg = {'x': 1, 'data': to_serialize(b)}
+    msg = {"x": 1, "data": to_serialize(b)}
     frames = dumps(msg)
 
     assert sum(map(nbytes, frames)) < 10000
 
     msg2 = loads(frames, deserialize=False)
-    assert sum(map(nbytes, msg2['data'].frames)) < 10000
+    assert sum(map(nbytes, msg2["data"].frames)) < 10000
 
     msg3 = dumps(msg2)
     msg4 = loads(msg3)
 
-    assert msg4 == {'x': 1, 'data': b'0' * 100000}
+    assert msg4 == {"x": 1, "data": b"0" * 100000}
 
 
 def eq_frames(a, b):
-    if b'headers' in a:
-        return (msgpack.loads(a, use_list=False)
-                == msgpack.loads(b, use_list=False))
+    if b"headers" in a:
+        return msgpack.loads(a, use_list=False) == msgpack.loads(b, use_list=False)
     else:
         return a == b
 
 
 def test_dumps_loads_Serialize():
-    msg = {'x': 1, 'data': Serialize(123)}
+    msg = {"x": 1, "data": Serialize(123)}
     frames = dumps(msg)
     assert len(frames) > 2
     result = loads(frames)
-    assert result == {'x': 1, 'data': 123}
+    assert result == {"x": 1, "data": 123}
 
     result2 = loads(frames, deserialize=False)
-    assert result2['x'] == 1
-    assert isinstance(result2['data'], Serialized)
-    assert any(a is b
-               for a in result2['data'].frames
-               for b in frames)
+    assert result2["x"] == 1
+    assert isinstance(result2["data"], Serialized)
+    assert any(a is b for a in result2["data"].frames for b in frames)
 
     frames2 = dumps(result2)
     assert all(map(eq_frames, frames, frames2))
@@ -210,13 +207,11 @@ def test_dumps_loads_Serialize():
 
 
 def test_dumps_loads_Serialized():
-    msg = {'x': 1,
-           'data': Serialized(*serialize(123)),
-           }
+    msg = {"x": 1, "data": Serialized(*serialize(123))}
     frames = dumps(msg)
     assert len(frames) > 2
     result = loads(frames)
-    assert result == {'x': 1, 'data': 123}
+    assert result == {"x": 1, "data": 123}
 
     result2 = loads(frames, deserialize=False)
     assert result2 == msg
@@ -228,18 +223,17 @@ def test_dumps_loads_Serialized():
     assert result == result3
 
 
-@pytest.mark.skipif(sys.version_info[0] < 3,
-                    reason='NumPy doesnt use memoryviews')
+@pytest.mark.skipif(sys.version_info[0] < 3, reason="NumPy doesnt use memoryviews")
 def test_maybe_compress_memoryviews():
-    np = pytest.importorskip('numpy')
-    pytest.importorskip('lz4')
-    x = np.arange(1000000, dtype='int64')
+    np = pytest.importorskip("numpy")
+    pytest.importorskip("lz4")
+    x = np.arange(1000000, dtype="int64")
     compression, payload = maybe_compress(x.data)
     try:
         import blosc  # noqa: F401
     except ImportError:
-        assert compression == 'lz4'
+        assert compression == "lz4"
         assert len(payload) < x.nbytes * 0.75
     else:
-        assert compression == 'blosc'
+        assert compression == "blosc"
         assert len(payload) < x.nbytes / 10
diff --git a/distributed/protocol/tests/test_protocol_utils.py b/distributed/protocol/tests/test_protocol_utils.py
index 0db79ccaafa..f4b98ab0e1d 100644
--- a/distributed/protocol/tests/test_protocol_utils.py
+++ b/distributed/protocol/tests/test_protocol_utils.py
@@ -5,20 +5,20 @@
 
 
 def test_merge_frames():
-    result = merge_frames({'lengths': [3, 4]}, [b'12', b'34', b'567'])
-    expected = [b'123', b'4567']
+    result = merge_frames({"lengths": [3, 4]}, [b"12", b"34", b"567"])
+    expected = [b"123", b"4567"]
 
     assert list(map(ensure_bytes, result)) == expected
 
-    b = b'123'
-    assert merge_frames({'lengths': [3]}, [b])[0] is b
+    b = b"123"
+    assert merge_frames({"lengths": [3]}, [b])[0] is b
 
-    L = [b'123', b'456']
-    assert merge_frames({'lengths': [3, 3]}, L) is L
+    L = [b"123", b"456"]
+    assert merge_frames({"lengths": [3, 3]}, L) is L
 
 
 def test_pack_frames():
-    frames = [b'123', b'asdf']
+    frames = [b"123", b"asdf"]
     b = pack_frames(frames)
     assert isinstance(b, bytes)
     frames2 = unpack_frames(b)
diff --git a/distributed/protocol/tests/test_serialize.py b/distributed/protocol/tests/test_serialize.py
index 64d7adc8d41..da43021d550 100644
--- a/distributed/protocol/tests/test_serialize.py
+++ b/distributed/protocol/tests/test_serialize.py
@@ -9,11 +9,20 @@
 from toolz import identity
 
 from distributed import wait
-from distributed.protocol import (register_serialization, serialize,
-                                  deserialize, nested_deserialize, Serialize,
-                                  Serialized, to_serialize, serialize_bytes,
-                                  deserialize_bytes, serialize_bytelist,
-                                  register_serialization_family, dask_serialize)
+from distributed.protocol import (
+    register_serialization,
+    serialize,
+    deserialize,
+    nested_deserialize,
+    Serialize,
+    Serialized,
+    to_serialize,
+    serialize_bytes,
+    deserialize_bytes,
+    serialize_bytelist,
+    register_serialization_family,
+    dask_serialize,
+)
 from distributed.utils import nbytes
 from distributed.utils_test import inc, gen_test
 from distributed.comm.utils import to_frames, from_frames
@@ -24,7 +33,7 @@ def __init__(self, data):
         self.data = data
 
     def __getstate__(self):
-        raise Exception('Not picklable')
+        raise Exception("Not picklable")
 
 
 def serialize_myobj(x):
@@ -41,7 +50,7 @@ def deserialize_myobj(header, frames):
 def test_dumps_serialize():
     for x in [123, [1, 2, 3]]:
         header, frames = serialize(x)
-        assert header['serializer'] == 'pickle'
+        assert header["serializer"] == "pickle"
         assert len(frames) == 1
 
         result = deserialize(header, frames)
@@ -49,7 +58,7 @@ def test_dumps_serialize():
 
     x = MyObj(123)
     header, frames = serialize(x)
-    assert header['type']
+    assert header["type"]
     assert len(frames) == 1
 
     result = deserialize(header, frames)
@@ -57,7 +66,7 @@ def test_dumps_serialize():
 
 
 def test_serialize_bytestrings():
-    for b in (b'123', bytearray(b'4567')):
+    for b in (b"123", bytearray(b"4567")):
         header, frames = serialize(b)
         assert frames[0] is b
         bb = deserialize(header, frames)
@@ -66,7 +75,7 @@ def test_serialize_bytestrings():
 
 def test_Serialize():
     s = Serialize(123)
-    assert '123' in str(s)
+    assert "123" in str(s)
     assert s.data == 123
 
     t = Serialize((1, 2))
@@ -92,18 +101,18 @@ def test_Serialized():
 
 
 def test_nested_deserialize():
-    x = {'op': 'update',
-         'x': [to_serialize(123), to_serialize(456), 789],
-         'y': {'a': ['abc', Serialized(*serialize('def'))],
-               'b': b'ghi'}
-         }
+    x = {
+        "op": "update",
+        "x": [to_serialize(123), to_serialize(456), 789],
+        "y": {"a": ["abc", Serialized(*serialize("def"))], "b": b"ghi"},
+    }
     x_orig = copy.deepcopy(x)
 
-    assert nested_deserialize(x) == {'op': 'update',
-                                     'x': [123, 456, 789],
-                                     'y': {'a': ['abc', 'def'],
-                                           'b': b'ghi'}
-                                     }
+    assert nested_deserialize(x) == {
+        "op": "update",
+        "x": [123, 456, 789],
+        "y": {"a": ["abc", "def"], "b": b"ghi"},
+    }
     assert x == x_orig  # x wasn't mutated
 
 
@@ -146,7 +155,7 @@ def test_inter_worker_comms(c, s, a, b):
 
 class Empty(object):
     def __getstate__(self):
-        raise Exception('Not picklable')
+        raise Exception("Not picklable")
 
 
 def serialize_empty(x):
@@ -168,6 +177,7 @@ def test_empty():
 
 def test_empty_loads():
     from distributed.protocol import loads, dumps
+
     e = Empty()
     e2 = loads(dumps([to_serialize(e)]))
     assert isinstance(e2[0], Empty)
@@ -175,13 +185,14 @@ def test_empty_loads():
 
 def test_empty_loads_deep():
     from distributed.protocol import loads, dumps
+
     e = Empty()
     e2 = loads(dumps([[[to_serialize(e)]]]))
     assert isinstance(e2[0][0][0], Empty)
 
 
 def test_serialize_bytes():
-    for x in [1, 'abc', np.arange(5)]:
+    for x in [1, "abc", np.arange(5)]:
         b = serialize_bytes(x)
         assert isinstance(b, bytes)
         y = deserialize_bytes(b)
@@ -189,12 +200,12 @@ def test_serialize_bytes():
 
 
 def test_serialize_list_compress():
-    pytest.importorskip('lz4')
+    pytest.importorskip("lz4")
     x = np.ones(1000000)
     L = serialize_bytelist(x)
     assert sum(map(nbytes, L)) < x.nbytes / 2
 
-    b = b''.join(L)
+    b = b"".join(L)
     y = deserialize_bytes(b)
     assert (x == y).all()
 
@@ -217,7 +228,7 @@ def __getstate__(self):
     assert "Sneaky" not in str(info.value)
     assert "MyClass" in str(info.value)
 
-    header, frames = serialize(obj, serializers=['pickle'])
+    header, frames = serialize(obj, serializers=["pickle"])
     with pytest.raises(Exception) as info:
         deserialize(header, frames)
 
@@ -226,28 +237,27 @@ def __getstate__(self):
 
 
 def test_errors():
-    msg = {'data': {'foo': to_serialize(inc)}}
+    msg = {"data": {"foo": to_serialize(inc)}}
 
-    header, frames = serialize(msg, serializers=['msgpack', 'pickle'])
-    assert header['serializer'] == 'pickle'
+    header, frames = serialize(msg, serializers=["msgpack", "pickle"])
+    assert header["serializer"] == "pickle"
 
-    header, frames = serialize(msg, serializers=['msgpack'])
-    assert header['serializer'] == 'error'
+    header, frames = serialize(msg, serializers=["msgpack"])
+    assert header["serializer"] == "error"
 
     with pytest.raises(TypeError):
-        serialize(msg, serializers=['msgpack'], on_error='raise')
+        serialize(msg, serializers=["msgpack"], on_error="raise")
 
 
 @gen_test()
 def test_err_on_bad_deserializer():
-    frames = yield to_frames({'x': to_serialize(1234)},
-                                     serializers=['pickle'])
+    frames = yield to_frames({"x": to_serialize(1234)}, serializers=["pickle"])
 
-    result = yield from_frames(frames, deserializers=['pickle', 'foo'])
-    assert result == {'x': 1234}
+    result = yield from_frames(frames, deserializers=["pickle", "foo"])
+    assert result == {"x": 1234}
 
     with pytest.raises(TypeError) as info:
-        yield from_frames(frames, deserializers=['msgpack'])
+        yield from_frames(frames, deserializers=["msgpack"])
 
 
 class MyObject(object):
@@ -256,10 +266,12 @@ def __init__(self, **kwargs):
 
 
 def my_dumps(obj, context=None):
-    if type(obj).__name__ == 'MyObject':
-        header = {'serializer': 'my-ser'}
-        frames = [msgpack.dumps(obj.__dict__, use_bin_type=True),
-                  msgpack.dumps(context, use_bin_type=True)]
+    if type(obj).__name__ == "MyObject":
+        header = {"serializer": "my-ser"}
+        frames = [
+            msgpack.dumps(obj.__dict__, use_bin_type=True),
+            msgpack.dumps(context, use_bin_type=True),
+        ]
         return header, frames
     else:
         raise NotImplementedError()
@@ -274,11 +286,13 @@ def my_loads(header, frames):
     return obj
 
 
-@gen_cluster(client=True,
-             client_kwargs={'serializers': ['my-ser', 'pickle']},
-             worker_kwargs={'serializers': ['my-ser', 'pickle']})
+@gen_cluster(
+    client=True,
+    client_kwargs={"serializers": ["my-ser", "pickle"]},
+    worker_kwargs={"serializers": ["my-ser", "pickle"]},
+)
 def test_context_specific_serialization(c, s, a, b):
-    register_serialization_family('my-ser', my_dumps, my_loads)
+    register_serialization_family("my-ser", my_dumps, my_loads)
 
     try:
         # Create the object on A, force communication to B
@@ -295,16 +309,17 @@ def check(dask_worker):
             return my_obj.context
 
         result = yield c.run(check, workers=[b.address])
-        expected = {'sender': a.address, 'recipient': b.address}
-        assert result[b.address]['sender'] == a.address  # see origin worker
+        expected = {"sender": a.address, "recipient": b.address}
+        assert result[b.address]["sender"] == a.address  # see origin worker
 
         z = yield y  # bring object to local process
 
         assert z.x == 1 and z.y == 2
-        assert z.context['sender'] == b.address
+        assert z.context["sender"] == b.address
     finally:
         from distributed.protocol.serialize import families
-        del families['my-ser']
+
+        del families["my-ser"]
 
 
 @gen_cluster(client=True)
@@ -325,13 +340,13 @@ def check(dask_worker):
         return my_obj.context
 
     result = yield c.run(check, workers=[b.address])
-    expected = {'sender': a.address, 'recipient': b.address}
-    assert result[b.address]['sender'] == a.address  # see origin worker
+    expected = {"sender": a.address, "recipient": b.address}
+    assert result[b.address]["sender"] == a.address  # see origin worker
 
     z = yield y  # bring object to local process
 
     assert z.x == 1 and z.y == 2
-    assert z.context['sender'] == b.address
+    assert z.context["sender"] == b.address
 
 
 def test_serialize_raises():
@@ -345,4 +360,4 @@ def dumps(f):
     with pytest.raises(Exception) as info:
         deserialize(*serialize(Foo()))
 
-    assert 'Hello-123' in str(info.value)
+    assert "Hello-123" in str(info.value)
diff --git a/distributed/protocol/tests/test_sklearn.py b/distributed/protocol/tests/test_sklearn.py
index 4fa8aeb5369..051a0440f3a 100644
--- a/distributed/protocol/tests/test_sklearn.py
+++ b/distributed/protocol/tests/test_sklearn.py
@@ -1,5 +1,6 @@
 import pytest
-pytest.importorskip('sklearn')
+
+pytest.importorskip("sklearn")
 
 import sklearn.linear_model
 
@@ -11,7 +12,7 @@ def test_basic():
     est.fit([[0, 0], [1, 1], [2, 2]], [0, 1, 2])
 
     header, frames = serialize(est)
-    assert header['serializer'] == 'dask'
+    assert header["serializer"] == "dask"
 
     est2 = deserialize(header, frames)
 
diff --git a/distributed/protocol/tests/test_sparse.py b/distributed/protocol/tests/test_sparse.py
index 2ff97c143a1..89f9da09bc2 100644
--- a/distributed/protocol/tests/test_sparse.py
+++ b/distributed/protocol/tests/test_sparse.py
@@ -1,9 +1,8 @@
-
 import numpy as np
 from numpy.testing import assert_allclose
 import pytest
 
-sparse = pytest.importorskip('sparse')
+sparse = pytest.importorskip("sparse")
 
 from distributed.protocol import deserialize, serialize
 
@@ -14,7 +13,7 @@ def test_serialize_deserialize_sparse():
 
     y = sparse.COO(x)
     header, frames = serialize(y)
-    assert 'sparse' in header['type']
+    assert "sparse" in header["type"]
     z = deserialize(*serialize(y))
 
     assert_allclose(y.data, z.data)
diff --git a/distributed/protocol/tests/test_torch.py b/distributed/protocol/tests/test_torch.py
index cac8fa05d66..6cc8bb20986 100644
--- a/distributed/protocol/tests/test_torch.py
+++ b/distributed/protocol/tests/test_torch.py
@@ -1,15 +1,15 @@
 from distributed.protocol import serialize, deserialize
 import pytest
 
-np = pytest.importorskip('numpy')
-torch = pytest.importorskip('torch')
+np = pytest.importorskip("numpy")
+torch = pytest.importorskip("torch")
 
 
 def test_tensor():
     x = np.arange(10)
     t = torch.Tensor(x)
     header, frames = serialize(t)
-    assert header['serializer'] == 'dask'
+    assert header["serializer"] == "dask"
     t2 = deserialize(header, frames)
     assert (x == t2.numpy()).all()
 
@@ -25,7 +25,7 @@ def test_grad():
 
 
 def test_resnet():
-    torchvision = pytest.importorskip('torchvision')
+    torchvision = pytest.importorskip("torchvision")
     model = torchvision.models.resnet.resnet18()
 
     header, frames = serialize(model)
diff --git a/distributed/protocol/torch.py b/distributed/protocol/torch.py
index 9a171b6d84f..e69be68b0c1 100644
--- a/distributed/protocol/torch.py
+++ b/distributed/protocol/torch.py
@@ -1,5 +1,4 @@
-from .serialize import (serialize, dask_serialize, dask_deserialize,
-                        register_generic)
+from .serialize import serialize, dask_serialize, dask_deserialize, register_generic
 
 import torch
 import numpy as np
@@ -11,32 +10,33 @@ def serialize_torch_Tensor(t):
     header, frames = serialize(t.detach_().numpy())
     if t.grad is not None:
         grad_header, grad_frames = serialize(t.grad.numpy())
-        header['grad'] = {'header': grad_header, 'start': len(frames)}
+        header["grad"] = {"header": grad_header, "start": len(frames)}
         frames += grad_frames
-    header['requires_grad'] = requires_grad_
-    header['device'] = t.device.type
+    header["requires_grad"] = requires_grad_
+    header["device"] = t.device.type
     return header, frames
 
 
 @dask_deserialize.register(torch.Tensor)
 def deserialize_torch_Tensor(header, frames):
-    if header.get('grad', False):
-        i = header['grad']['start']
+    if header.get("grad", False):
+        i = header["grad"]["start"]
         frames, grad_frames = frames[:i], frames[i:]
-        grad = dask_deserialize.dispatch(np.ndarray)(header['grad']['header'],
-                                                     grad_frames)
+        grad = dask_deserialize.dispatch(np.ndarray)(
+            header["grad"]["header"], grad_frames
+        )
     else:
         grad = None
 
     x = dask_deserialize.dispatch(np.ndarray)(header, frames)
-    if header['device'] == 'cpu':
+    if header["device"] == "cpu":
         t = torch.from_numpy(x)
-        if header['requires_grad']:
+        if header["requires_grad"]:
             t = t.requires_grad_(True)
     else:
-        t = torch.tensor(data=x,
-                         device=header['device'],
-                         requires_grad=header['requires_grad'])
+        t = torch.tensor(
+            data=x, device=header["device"], requires_grad=header["requires_grad"]
+        )
     if grad is not None:
         t.grad = torch.from_numpy(grad)
     return t
@@ -45,14 +45,14 @@ def deserialize_torch_Tensor(header, frames):
 @dask_serialize.register(torch.nn.Parameter)
 def serialize_torch_Parameters(p):
     header, frames = serialize(p.detach())
-    header['requires_grad'] = p.requires_grad
+    header["requires_grad"] = p.requires_grad
     return header, frames
 
 
 @dask_deserialize.register(torch.nn.Parameter)
 def deserialize_torch_Parameters(header, frames):
     t = dask_deserialize.dispatch(torch.Tensor)(header, frames)
-    return torch.nn.Parameter(data=t, requires_grad=header['requires_grad'])
+    return torch.nn.Parameter(data=t, requires_grad=header["requires_grad"])
 
 
 register_generic(torch.nn.Module)
diff --git a/distributed/protocol/utils.py b/distributed/protocol/utils.py
index 2d4258e9383..90d30342951 100644
--- a/distributed/protocol/utils.py
+++ b/distributed/protocol/utils.py
@@ -4,7 +4,7 @@
 
 from ..utils import ensure_bytes, nbytes
 
-BIG_BYTES_SHARD_SIZE = 2**26
+BIG_BYTES_SHARD_SIZE = 2 ** 26
 
 
 def frame_split_size(frames, n=BIG_BYTES_SHARD_SIZE):
@@ -34,7 +34,7 @@ def frame_split_size(frames, n=BIG_BYTES_SHARD_SIZE):
             except AttributeError:
                 itemsize = 1
             for i in range(0, nbytes(frame) // itemsize, n // itemsize):
-                out.append(frame[i: i + n // itemsize])
+                out.append(frame[i : i + n // itemsize])
         else:
             out.append(frame)
     return out
@@ -50,7 +50,7 @@ def merge_frames(header, frames):
     >>> merge_frames({'lengths': [6]}, [b'123', b'456'])
     [b'123456']
     """
-    lengths = list(header['lengths'])
+    lengths = list(header["lengths"])
 
     if not frames:
         return frames
@@ -77,15 +77,16 @@ def merge_frames(header, frames):
                 L.append(mv[:l])
                 frames.append(mv[l:])
                 l = 0
-        out.append(b''.join(map(ensure_bytes, L)))
+        out.append(b"".join(map(ensure_bytes, L)))
     return out
 
 
 def pack_frames_prelude(frames):
     lengths = [len(f) for f in frames]
-    lengths = ([struct.pack('Q', len(frames))] +
-               [struct.pack('Q', nbytes(frame)) for frame in frames])
-    return b''.join(lengths)
+    lengths = [struct.pack("Q", len(frames))] + [
+        struct.pack("Q", nbytes(frame)) for frame in frames
+    ]
+    return b"".join(lengths)
 
 
 def pack_frames(frames):
@@ -102,7 +103,7 @@ def pack_frames(frames):
     if not isinstance(frames, list):
         frames = list(frames)
 
-    return b''.join(prelude + frames)
+    return b"".join(prelude + frames)
 
 
 def unpack_frames(b):
@@ -115,13 +116,13 @@ def unpack_frames(b):
     --------
     pack_frames
     """
-    (n_frames,) = struct.unpack('Q', b[:8])
+    (n_frames,) = struct.unpack("Q", b[:8])
 
     frames = []
     start = 8 + n_frames * 8
     for i in range(n_frames):
-        (length,) = struct.unpack('Q', b[(i + 1) * 8: (i + 2) * 8])
-        frame = b[start: start + length]
+        (length,) = struct.unpack("Q", b[(i + 1) * 8 : (i + 2) * 8])
+        frame = b[start : start + length]
         frames.append(frame)
         start += length
 
diff --git a/distributed/publish.py b/distributed/publish.py
index cf38b7d9490..a21f5ef37ed 100644
--- a/distributed/publish.py
+++ b/distributed/publish.py
@@ -15,26 +15,30 @@ def __init__(self, scheduler):
         self.scheduler = scheduler
         self.datasets = dict()
 
-        handlers = {'publish_list': self.list,
-                    'publish_put': self.put,
-                    'publish_get': self.get,
-                    'publish_delete': self.delete}
+        handlers = {
+            "publish_list": self.list,
+            "publish_put": self.put,
+            "publish_get": self.get,
+            "publish_delete": self.delete,
+        }
 
         self.scheduler.handlers.update(handlers)
-        self.scheduler.extensions['publish'] = self
+        self.scheduler.extensions["publish"] = self
 
     def put(self, stream=None, keys=None, data=None, name=None, client=None):
         with log_errors():
             if name in self.datasets:
                 raise KeyError("Dataset %s already exists" % name)
-            self.scheduler.client_desires_keys(keys, 'published-%s' % tokey(name))
-            self.datasets[name] = {'data': data, 'keys': keys}
-            return {'status':  'OK', 'name': name}
+            self.scheduler.client_desires_keys(keys, "published-%s" % tokey(name))
+            self.datasets[name] = {"data": data, "keys": keys}
+            return {"status": "OK", "name": name}
 
     def delete(self, stream=None, name=None):
         with log_errors():
-            out = self.datasets.pop(name, {'keys': []})
-            self.scheduler.client_releases_keys(out['keys'], 'published-%s' % tokey(name))
+            out = self.datasets.pop(name, {"keys": []})
+            self.scheduler.client_releases_keys(
+                out["keys"], "published-%s" % tokey(name)
+            )
 
     def list(self, *args):
         with log_errors():
@@ -53,6 +57,7 @@ class Datasets(MutableMapping):
     client : distributed.client.Client
 
     """
+
     def __init__(self, client):
         self.__client = client
 
diff --git a/distributed/pubsub.py b/distributed/pubsub.py
index cdbe9e95a7a..5e086492923 100644
--- a/distributed/pubsub.py
+++ b/distributed/pubsub.py
@@ -16,46 +16,51 @@
 
 class PubSubSchedulerExtension(object):
     """ Extend Dask's scheduler with routes to handle PubSub machinery """
+
     def __init__(self, scheduler):
         self.scheduler = scheduler
         self.publishers = defaultdict(set)
         self.subscribers = defaultdict(set)
         self.client_subscribers = defaultdict(set)
 
-        self.scheduler.handlers.update({
-            'pubsub_add_publisher': self.add_publisher,
-        })
+        self.scheduler.handlers.update({"pubsub_add_publisher": self.add_publisher})
 
-        self.scheduler.stream_handlers.update({
-            'pubsub-add-subscriber': self.add_subscriber,
-            'pubsub-remove-publisher': self.remove_publisher,
-            'pubsub-remove-subscriber': self.remove_subscriber,
-            'pubsub-msg': self.handle_message,
-        })
+        self.scheduler.stream_handlers.update(
+            {
+                "pubsub-add-subscriber": self.add_subscriber,
+                "pubsub-remove-publisher": self.remove_publisher,
+                "pubsub-remove-subscriber": self.remove_subscriber,
+                "pubsub-msg": self.handle_message,
+            }
+        )
 
-        self.scheduler.extensions['pubsub'] = self
+        self.scheduler.extensions["pubsub"] = self
 
     def add_publisher(self, comm=None, name=None, worker=None):
         logger.debug("Add publisher: %s %s", name, worker)
         self.publishers[name].add(worker)
-        return {'subscribers': {addr: {} for addr in self.subscribers[name]},
-                'publish-scheduler': name in self.client_subscribers and
-                                     len(self.client_subscribers[name]) > 0}
+        return {
+            "subscribers": {addr: {} for addr in self.subscribers[name]},
+            "publish-scheduler": name in self.client_subscribers
+            and len(self.client_subscribers[name]) > 0,
+        }
 
     def add_subscriber(self, comm=None, name=None, worker=None, client=None):
         if worker:
             logger.debug("Add worker subscriber: %s %s", name, worker)
             self.subscribers[name].add(worker)
             for pub in self.publishers[name]:
-                self.scheduler.worker_send(pub, {'op': 'pubsub-add-subscriber',
-                                                 'address': worker,
-                                                 'name': name})
+                self.scheduler.worker_send(
+                    pub,
+                    {"op": "pubsub-add-subscriber", "address": worker, "name": name},
+                )
         elif client:
             logger.debug("Add client subscriber: %s %s", name, client)
             for pub in self.publishers[name]:
-                self.scheduler.worker_send(pub, {'op': 'pubsub-publish-scheduler',
-                                                 'name': name,
-                                                 'publish': True})
+                self.scheduler.worker_send(
+                    pub,
+                    {"op": "pubsub-publish-scheduler", "name": name, "publish": True},
+                )
             self.client_subscribers[name].add(client)
 
     def remove_publisher(self, comm=None, name=None, worker=None):
@@ -72,18 +77,24 @@ def remove_subscriber(self, comm=None, name=None, worker=None, client=None):
             logger.debug("Add worker subscriber: %s %s", name, worker)
             self.subscribers[name].remove(worker)
             for pub in self.publishers[name]:
-                self.scheduler.worker_send(pub, {'op': 'pubsub-remove-subscriber',
-                                                 'address': worker,
-                                                 'name': name})
+                self.scheduler.worker_send(
+                    pub,
+                    {"op": "pubsub-remove-subscriber", "address": worker, "name": name},
+                )
         elif client:
             logger.debug("Add client subscriber: %s %s", name, client)
             self.client_subscribers[name].remove(client)
             if not self.client_subscribers[name]:
                 del self.client_subscribers[name]
                 for pub in self.publishers[name]:
-                    self.scheduler.worker_send(pub, {'op': 'pubsub-publish-scheduler',
-                                                     'name': name,
-                                                     'publish': False})
+                    self.scheduler.worker_send(
+                        pub,
+                        {
+                            "op": "pubsub-publish-scheduler",
+                            "name": name,
+                            "publish": False,
+                        },
+                    )
 
         if not self.subscribers[name] and not self.publishers[name]:
             logger.debug("Remove PubSub topic %s", name)
@@ -93,35 +104,38 @@ def remove_subscriber(self, comm=None, name=None, worker=None, client=None):
     def handle_message(self, name=None, msg=None, worker=None, client=None):
         for c in list(self.client_subscribers[name]):
             try:
-                self.scheduler.client_comms[c].send({'op': 'pubsub-msg',
-                                                     'name': name,
-                                                     'msg': msg})
+                self.scheduler.client_comms[c].send(
+                    {"op": "pubsub-msg", "name": name, "msg": msg}
+                )
             except (KeyError, CommClosedError):
                 self.remove_subscriber(name=name, client=c)
 
         if client:
             for sub in self.subscribers[name]:
-                self.scheduler.worker_send(sub, {'op': 'pubsub-msg',
-                                                 'name': name,
-                                                 'msg': msg})
+                self.scheduler.worker_send(
+                    sub, {"op": "pubsub-msg", "name": name, "msg": msg}
+                )
 
 
 class PubSubWorkerExtension(object):
     """ Extend Dask's Worker with routes to handle PubSub machinery """
+
     def __init__(self, worker):
         self.worker = worker
-        self.worker.stream_handlers.update({
-            'pubsub-add-subscriber': self.add_subscriber,
-            'pubsub-remove-subscriber': self.remove_subscriber,
-            'pubsub-msg': self.handle_message,
-            'pubsub-publish-scheduler': self.publish_scheduler,
-        })
+        self.worker.stream_handlers.update(
+            {
+                "pubsub-add-subscriber": self.add_subscriber,
+                "pubsub-remove-subscriber": self.remove_subscriber,
+                "pubsub-msg": self.handle_message,
+                "pubsub-publish-scheduler": self.publish_scheduler,
+            }
+        )
 
         self.subscribers = defaultdict(weakref.WeakSet)
         self.publishers = defaultdict(weakref.WeakSet)
         self.publish_to_scheduler = defaultdict(lambda: False)
 
-        self.worker.extensions['pubsub'] = self  # circular reference
+        self.worker.extensions["pubsub"] = self  # circular reference
 
     def add_subscriber(self, name=None, address=None, **info):
         for pub in self.publishers[name]:
@@ -144,15 +158,13 @@ def trigger_cleanup(self):
     def cleanup(self):
         for name, s in dict(self.subscribers).items():
             if not len(s):
-                msg = {'op': 'pubsub-remove-subscriber',
-                       'name': name}
+                msg = {"op": "pubsub-remove-subscriber", "name": name}
                 self.worker.batched_stream.send(msg)
                 del self.subscribers[name]
 
         for name, p in dict(self.publishers).items():
             if not len(p):
-                msg = {'op': 'pubsub-remove-publisher',
-                       'name': name}
+                msg = {"op": "pubsub-remove-publisher", "name": name}
                 self.worker.batched_stream.send(msg)
                 del self.publishers[name]
                 del self.publish_to_scheduler[name]
@@ -160,22 +172,22 @@ def cleanup(self):
 
 class PubSubClientExtension(object):
     """ Extend Dask's Client with handlers to handle PubSub machinery """
+
     def __init__(self, client):
         self.client = client
-        self.client._stream_handlers.update({
-            'pubsub-msg': self.handle_message
-        })
+        self.client._stream_handlers.update({"pubsub-msg": self.handle_message})
 
         self.subscribers = defaultdict(weakref.WeakSet)
-        self.client.extensions['pubsub'] = self  # TODO: circular reference
+        self.client.extensions["pubsub"] = self  # TODO: circular reference
 
     def handle_message(self, name=None, msg=None):
         for sub in self.subscribers[name]:
             sub._put(msg)
 
         if not self.subscribers[name]:
-            self.client.scheduler_comm.send({'op': 'pubsub-remove-subscribers',
-                                             'name': name})
+            self.client.scheduler_comm.send(
+                {"op": "pubsub-remove-subscribers", "name": name}
+            )
 
     def trigger_cleanup(self):
         self.client.loop.add_callback(self.cleanup)
@@ -183,8 +195,7 @@ def trigger_cleanup(self):
     def cleanup(self):
         for name, s in self.subscribers.items():
             if not s:
-                msg = {'op': 'pubsub-remove-subscriber',
-                       'name': name}
+                msg = {"op": "pubsub-remove-subscriber", "name": name}
                 self.client.scheduler_comm.send(msg)
 
 
@@ -265,9 +276,11 @@ class Pub(object):
     --------
     Sub
     """
+
     def __init__(self, name, worker=None, client=None):
         if worker is None and client is None:
             from distributed import get_worker, get_client
+
             try:
                 worker = get_worker()
             except Exception:
@@ -291,7 +304,7 @@ def __init__(self, name, worker=None, client=None):
         self.loop.add_callback(self._start)
 
         if self.worker:
-            pubsub = self.worker.extensions['pubsub']
+            pubsub = self.worker.extensions["pubsub"]
             self.loop.add_callback(pubsub.publishers[name].add, self)
             finalize(self, pubsub.trigger_cleanup)
 
@@ -299,12 +312,11 @@ def __init__(self, name, worker=None, client=None):
     def _start(self):
         if self.worker:
             result = yield self.scheduler.pubsub_add_publisher(
-                    name=self.name,
-                    worker=self.worker.address
+                name=self.name, worker=self.worker.address
             )
-            pubsub = self.worker.extensions['pubsub']
-            self.subscribers.update(result['subscribers'])
-            pubsub.publish_to_scheduler[self.name] = result['publish-scheduler']
+            pubsub = self.worker.extensions["pubsub"]
+            self.subscribers.update(result["subscribers"])
+            pubsub.publish_to_scheduler[self.name] = result["publish-scheduler"]
 
         self._started = True
 
@@ -317,13 +329,13 @@ def _put(self, msg):
             self._buffer.append(msg)
             return
 
-        data = {'op': 'pubsub-msg', 'name': self.name, 'msg': to_serialize(msg)}
+        data = {"op": "pubsub-msg", "name": self.name, "msg": to_serialize(msg)}
 
         if self.worker:
             for sub in self.subscribers:
                 self.worker.send_to_worker(sub, data)
 
-            if self.worker.extensions['pubsub'].publish_to_scheduler[self.name]:
+            if self.worker.extensions["pubsub"].publish_to_scheduler[self.name]:
                 self.worker.batched_stream.send(data)
         elif self.client:
             self.client.scheduler_comm.send(data)
@@ -340,9 +352,11 @@ class Sub(object):
     --------
     Pub: for full docstring
     """
+
     def __init__(self, name, worker=None, client=None):
         if worker is None and client is None:
             from distributed.worker import get_worker, get_client
+
             try:
                 worker = get_worker()
             except Exception:
@@ -359,12 +373,12 @@ def __init__(self, name, worker=None, client=None):
         self.condition = tornado.locks.Condition()
 
         if self.worker:
-            pubsub = self.worker.extensions['pubsub']
+            pubsub = self.worker.extensions["pubsub"]
         elif self.client:
-            pubsub = self.client.extensions['pubsub']
+            pubsub = self.client.extensions["pubsub"]
         self.loop.add_callback(pubsub.subscribers[name].add, self)
 
-        msg = {'op': 'pubsub-add-subscriber', 'name': self.name}
+        msg = {"op": "pubsub-add-subscriber", "name": self.name}
         if self.worker:
             self.loop.add_callback(self.worker.batched_stream.send, msg)
         elif self.client:
diff --git a/distributed/pytest_resourceleaks.py b/distributed/pytest_resourceleaks.py
index 29331e1b404..bb62d3916d0 100644
--- a/distributed/pytest_resourceleaks.py
+++ b/distributed/pytest_resourceleaks.py
@@ -15,66 +15,70 @@
 
 
 def pytest_addoption(parser):
-    group = parser.getgroup('resource leaks')
+    group = parser.getgroup("resource leaks")
     group.addoption(
-        '-L', '--leaks',
-        action='store',
-        dest='leaks',
-        help='''\
+        "-L",
+        "--leaks",
+        action="store",
+        dest="leaks",
+        help="""\
 List of resources to monitor for leaks before and after each test.
 Can be 'all' or a comma-separated list of resource names
 (possible values: {known_checkers}).
-'''.format(known_checkers=', '.join(sorted("'%s'" % s for s in all_checkers)))
+""".format(
+            known_checkers=", ".join(sorted("'%s'" % s for s in all_checkers))
+        ),
     )
     group.addoption(
-        '--leaks-timeout',
-        action='store',
-        type='float',
-        dest='leaks_timeout',
+        "--leaks-timeout",
+        action="store",
+        type="float",
+        dest="leaks_timeout",
         default=0.5,
-        help='''\
+        help="""\
 Wait at most this number of seconds to mark a test leaking
 (default: %(default)s).
-'''
+""",
     )
     group.addoption(
-        '--leaks-fail',
-        action='store_true',
-        dest='leaks_mark_failed',
+        "--leaks-fail",
+        action="store_true",
+        dest="leaks_mark_failed",
         default=False,
-        help='''Mark leaked tests failed.'''
+        help="""Mark leaked tests failed.""",
     )
     group.addoption(
-        '--leak-retries',
-        action='store',
+        "--leak-retries",
+        action="store",
         type=int,
-        dest='leak_retries',
+        dest="leak_retries",
         default=1,
-        help='''\
+        help="""\
 Max number of times to retry a test when it leaks, to ignore
 warmup-related issues (default: 1).
-'''
+""",
     )
 
 
 def pytest_configure(config):
-    leaks = config.getvalue('leaks')
+    leaks = config.getvalue("leaks")
     if leaks:
-        if leaks == 'all':
+        if leaks == "all":
             leaks = sorted(all_checkers)
         else:
-            leaks = leaks.split(',')
+            leaks = leaks.split(",")
         unknown = sorted(set(leaks) - set(all_checkers))
         if unknown:
             raise ValueError("unknown resources: %r" % (unknown,))
 
         checkers = [all_checkers[leak]() for leak in leaks]
-        checker = LeakChecker(checkers=checkers,
-                              grace_delay=config.getvalue('leaks_timeout'),
-                              mark_failed=config.getvalue('leaks_mark_failed'),
-                              max_retries=config.getvalue('leak_retries'),
-                              )
-        config.pluginmanager.register(checker, 'leaks_checker')
+        checker = LeakChecker(
+            checkers=checkers,
+            grace_delay=config.getvalue("leaks_timeout"),
+            mark_failed=config.getvalue("leaks_mark_failed"),
+            max_retries=config.getvalue("leak_retries"),
+        )
+        config.pluginmanager.register(checker, "leaks_checker")
 
 
 all_checkers = {}
@@ -91,7 +95,6 @@ def decorate(cls):
 
 
 class ResourceChecker(object):
-
     def on_start_test(self):
         pass
 
@@ -111,12 +114,12 @@ def format(self, before, after):
         raise NotImplementedError
 
 
-@register_checker('fds')
+@register_checker("fds")
 class FDChecker(ResourceChecker):
-
     def measure(self):
-        if os.name == 'posix':
+        if os.name == "posix":
             import psutil
+
             return psutil.Process().num_fds()
         else:
             return 0
@@ -128,11 +131,11 @@ def format(self, before, after):
         return "leaked %d file descriptor(s)" % (after - before)
 
 
-@register_checker('memory')
+@register_checker("memory")
 class RSSMemoryChecker(ResourceChecker):
-
     def measure(self):
         import psutil
+
         return psutil.Process().memory_info().rss
 
     def has_leak(self, before, after):
@@ -142,9 +145,8 @@ def format(self, before, after):
         return "leaked %d MB of RSS memory" % ((after - before) / 1e6)
 
 
-@register_checker('threads')
+@register_checker("threads")
 class ActiveThreadsChecker(ResourceChecker):
-
     def measure(self):
         return set(threading.enumerate())
 
@@ -154,23 +156,22 @@ def has_leak(self, before, after):
     def format(self, before, after):
         leaked = after - before
         assert leaked
-        return ("leaked %d Python threads: %s"
-                % (len(leaked), sorted(leaked, key=str)))
-
+        return "leaked %d Python threads: %s" % (len(leaked), sorted(leaked, key=str))
 
-class _ChildProcess(collections.namedtuple('_ChildProcess',
-                                           ('pid', 'name', 'cmdline'))):
 
+class _ChildProcess(
+    collections.namedtuple("_ChildProcess", ("pid", "name", "cmdline"))
+):
     @classmethod
     def from_process(cls, p):
         return cls(p.pid, p.name(), p.cmdline())
 
 
-@register_checker('processes')
+@register_checker("processes")
 class ChildProcessesChecker(ResourceChecker):
-
     def measure(self):
         import psutil
+
         # We use pid and creation time as keys to disambiguate between
         # processes (and protect against pid reuse)
         # Other properties such as cmdline may change for a given process
@@ -181,12 +182,18 @@ def measure(self):
                 with c.oneshot():
                     if c.ppid() == p.pid and os.path.samefile(c.exe(), sys.executable):
                         cmdline = c.cmdline()
-                        if any(a.startswith('from multiprocessing.semaphore_tracker import main')
-                               for a in cmdline):
+                        if any(
+                            a.startswith(
+                                "from multiprocessing.semaphore_tracker import main"
+                            )
+                            for a in cmdline
+                        ):
                             # Skip multiprocessing semaphore tracker
                             continue
-                        if any(a.startswith('from multiprocessing.forkserver import main')
-                               for a in cmdline):
+                        if any(
+                            a.startswith("from multiprocessing.forkserver import main")
+                            for a in cmdline
+                        ):
                             # Skip forkserver process, the forkserver's children
                             # however will be recorded normally
                             continue
@@ -204,15 +211,14 @@ def format(self, before, after):
         formatted = []
         for key in sorted(leaked):
             p = after[key]
-            formatted.append('  - pid={p.pid}, name={p.name!r}, cmdline={p.cmdline!r}'
-                             .format(p=p))
-        return ("leaked %d processes:\n%s"
-                % (len(leaked), '\n'.join(formatted)))
+            formatted.append(
+                "  - pid={p.pid}, name={p.name!r}, cmdline={p.cmdline!r}".format(p=p)
+            )
+        return "leaked %d processes:\n%s" % (len(leaked), "\n".join(formatted))
 
 
-@register_checker('tracemalloc')
+@register_checker("tracemalloc")
 class TracemallocMemoryChecker(ResourceChecker):
-
     def __init__(self):
         global tracemalloc
         import tracemalloc
@@ -225,6 +231,7 @@ def on_stop_test(self):
 
     def measure(self):
         import tracemalloc
+
         current, peak = tracemalloc.get_traced_memory()
         snap = tracemalloc.take_snapshot()
         return current, snap
@@ -235,13 +242,15 @@ def has_leak(self, before, after):
     def format(self, before, after):
         bytes_before, snap_before = before
         bytes_after, snap_after = after
-        diff = snap_after.compare_to(snap_before, 'traceback')
+        diff = snap_after.compare_to(snap_before, "traceback")
         ndiff = 5
         min_size_diff = 2e5
 
         lines = []
-        lines += ["leaked %.1f MB of traced Python memory"
-                  % ((bytes_after - bytes_before) / 1e6)]
+        lines += [
+            "leaked %.1f MB of traced Python memory"
+            % ((bytes_after - bytes_before) / 1e6)
+        ]
         for stat in diff[:ndiff]:
             size_diff = stat.size_diff or stat.size
             if size_diff < min_size_diff:
@@ -276,8 +285,7 @@ def cleanup(self):
         gc.collect()
 
     def checks_for_item(self, nodeid):
-        return [c for c in self.checkers
-                if c not in self.skip_checkers.get(nodeid, ())]
+        return [c for c in self.checkers if c not in self.skip_checkers.get(nodeid, ())]
 
     def measure(self, nodeid):
         # Return items in order
@@ -293,7 +301,7 @@ def measure_before_test(self, nodeid):
     def measure_after_test(self, nodeid):
         outcomes = self.outcomes[nodeid]
         assert outcomes
-        if outcomes != {'passed'}:
+        if outcomes != {"passed"}:
             # Test failed or skipped
             return
 
@@ -337,6 +345,7 @@ def run_test_again():
             # This invokes our setup/teardown hooks again
             # Inspired by https://pypi.python.org/pypi/pytest-rerunfailures
             from _pytest.runner import runtestprotocol
+
             item._initrequest()  # Re-init fixtures
             reports = runtestprotocol(item, nextitem=nextitem, log=False)
 
@@ -350,6 +359,7 @@ def run_test_again():
             except Exception as e:
                 print("--- Exception when re-running test ---")
                 import traceback
+
                 traceback.print_exc()
             else:
                 leaks = self.leaks.get(nodeid)
@@ -376,15 +386,17 @@ def pytest_runtest_protocol(self, item, nextitem):
             assert nodeid not in self.counters
             self.counters[nodeid] = {c: [] for c in self.checkers}
 
-            leaking = item.get_marker('leaking')
+            leaking = item.get_marker("leaking")
             if leaking is not None:
                 unknown = sorted(set(leaking.args) - set(all_checkers))
                 if unknown:
-                    raise ValueError("pytest.mark.leaking: unknown resources %r"
-                                     % (unknown,))
+                    raise ValueError(
+                        "pytest.mark.leaking: unknown resources %r" % (unknown,)
+                    )
                 classes = tuple(all_checkers[a] for a in leaking.args)
-                self.skip_checkers[nodeid] = {c for c in self.checkers
-                                              if isinstance(c, classes)}
+                self.skip_checkers[nodeid] = {
+                    c for c in self.checkers if isinstance(c, classes)
+                }
 
         yield
 
@@ -410,29 +422,31 @@ def pytest_report_teststatus(self, report):
         outcomes.add(report.outcome)
         outcome = yield
         if not self._retrying:
-            if report.when == 'teardown':
+            if report.when == "teardown":
                 leaks = self.leaks.get(report.nodeid)
                 if leaks:
                     if self.mark_failed:
-                        outcome.force_result(('failed', 'L', 'LEAKED'))
-                        report.outcome = 'failed'
+                        outcome.force_result(("failed", "L", "LEAKED"))
+                        report.outcome = "failed"
                         report.longrepr = "\n".join(
-                            ["%s %s" % (nodeid, checker.format(before, after))
-                             for checker, before, after in leaks])
+                            [
+                                "%s %s" % (nodeid, checker.format(before, after))
+                                for checker, before, after in leaks
+                            ]
+                        )
                     else:
-                        outcome.force_result(('leaked', 'L', 'LEAKED'))
+                        outcome.force_result(("leaked", "L", "LEAKED"))
                 # XXX should we log retried tests
 
     @pytest.hookimpl
     def pytest_terminal_summary(self, terminalreporter, exitstatus):
         tr = terminalreporter
-        leaked = tr.getreports('leaked')
+        leaked = tr.getreports("leaked")
         if leaked:
             # If mark_failed is False, leaks are output as a separate
             # results section
-            tr.write_sep("=", 'RESOURCE LEAKS')
+            tr.write_sep("=", "RESOURCE LEAKS")
             for rep in leaked:
                 nodeid = rep.nodeid
                 for checker, before, after in self.leaks[nodeid]:
-                    tr.line("%s %s"
-                            % (rep.nodeid, checker.format(before, after)))
+                    tr.line("%s %s" % (rep.nodeid, checker.format(before, after)))
diff --git a/distributed/queues.py b/distributed/queues.py
index fda6daaae5c..72f0f9fe52c 100644
--- a/distributed/queues.py
+++ b/distributed/queues.py
@@ -33,19 +33,20 @@ def __init__(self, scheduler):
         self.client_refcount = dict()
         self.future_refcount = defaultdict(lambda: 0)
 
-        self.scheduler.handlers.update({
-            'queue_create': self.create,
-            'queue_put': self.put,
-            'queue_get': self.get,
-            'queue_qsize': self.qsize}
+        self.scheduler.handlers.update(
+            {
+                "queue_create": self.create,
+                "queue_put": self.put,
+                "queue_get": self.get,
+                "queue_qsize": self.qsize,
+            }
         )
 
-        self.scheduler.stream_handlers.update({
-            'queue-future-release': self.future_release,
-            'queue_release': self.release,
-        })
+        self.scheduler.stream_handlers.update(
+            {"queue-future-release": self.future_release, "queue_release": self.release}
+        )
 
-        self.scheduler.extensions['queues'] = self
+        self.scheduler.extensions["queues"] = self
 
     def create(self, stream=None, name=None, client=None, maxsize=0):
         if name not in self.queues:
@@ -64,18 +65,20 @@ def release(self, stream=None, name=None, client=None):
             futures = self.queues[name]._queue
             del self.queues[name]
             self.scheduler.client_releases_keys(
-                    keys=[d['value'] for d in futures if d['type'] == 'Future'],
-                    client='queue-%s' % name
+                keys=[d["value"] for d in futures if d["type"] == "Future"],
+                client="queue-%s" % name,
             )
 
     @gen.coroutine
-    def put(self, stream=None, name=None, key=None, data=None, client=None, timeout=None):
+    def put(
+        self, stream=None, name=None, key=None, data=None, client=None, timeout=None
+    ):
         if key is not None:
-            record = {'type': 'Future', 'value': key}
+            record = {"type": "Future", "value": key}
             self.future_refcount[name, key] += 1
-            self.scheduler.client_desires_keys(keys=[key], client='queue-%s' % name)
+            self.scheduler.client_desires_keys(keys=[key], client="queue-%s" % name)
         else:
-            record = {'type': 'msgpack', 'value': data}
+            record = {"type": "msgpack", "value": data}
         if timeout is not None:
             timeout = datetime.timedelta(seconds=(timeout))
         yield self.queues[name].put(record, timeout=timeout)
@@ -83,25 +86,23 @@ def put(self, stream=None, name=None, key=None, data=None, client=None, timeout=
     def future_release(self, name=None, key=None, client=None):
         self.future_refcount[name, key] -= 1
         if self.future_refcount[name, key] == 0:
-            self.scheduler.client_releases_keys(keys=[key],
-                                                client='queue-%s' % name)
+            self.scheduler.client_releases_keys(keys=[key], client="queue-%s" % name)
             del self.future_refcount[name, key]
 
     @gen.coroutine
-    def get(self, stream=None, name=None, client=None, timeout=None,
-            batch=False):
+    def get(self, stream=None, name=None, client=None, timeout=None, batch=False):
         def process(record):
             """ Add task status if known """
-            if record['type'] == 'Future':
+            if record["type"] == "Future":
                 record = record.copy()
-                key = record['value']
+                key = record["value"]
                 ts = self.scheduler.tasks.get(key)
-                state = ts.state if ts is not None else 'lost'
+                state = ts.state if ts is not None else "lost"
 
-                record['state'] = state
-                if state == 'erred':
-                    record['exception'] = ts.exception_blame.exception
-                    record['traceback'] = ts.exception_blame.traceback
+                record["state"] = state
+                if state == "erred":
+                    record["exception"] = ts.exception_blame.exception
+                    record["traceback"] = ts.exception_blame.traceback
 
             return record
 
@@ -114,8 +115,10 @@ def process(record):
                     out.append(record)
             else:
                 if timeout is not None:
-                    msg = ("Dask queues don't support simultaneous use of "
-                           "integer batch sizes and timeouts")
+                    msg = (
+                        "Dask queues don't support simultaneous use of "
+                        "integer batch sizes and timeouts"
+                    )
                     raise NotImplementedError(msg)
                 for i in range(batch):
                     record = yield q.get()
@@ -164,13 +167,20 @@ class Queue(object):
 
     def __init__(self, name=None, client=None, maxsize=0):
         self.client = client or _get_global_client()
-        self.name = name or 'queue-' + uuid.uuid4().hex
-        if self.client.asynchronous or getattr(thread_state, 'on_event_loop_thread', False):
-            self._started = self.client.scheduler.queue_create(name=self.name,
-                                                               maxsize=maxsize)
+        self.name = name or "queue-" + uuid.uuid4().hex
+        if self.client.asynchronous or getattr(
+            thread_state, "on_event_loop_thread", False
+        ):
+            self._started = self.client.scheduler.queue_create(
+                name=self.name, maxsize=maxsize
+            )
         else:
-            sync(self.client.loop, self.client.scheduler.queue_create,
-                 name=self.name, maxsize=maxsize)
+            sync(
+                self.client.loop,
+                self.client.scheduler.queue_create,
+                name=self.name,
+                maxsize=maxsize,
+            )
             self._started = gen.moment
 
     def __await__(self):
@@ -178,18 +188,19 @@ def __await__(self):
         def _():
             yield self._started
             raise gen.Return(self)
+
         return _().__await__()
 
     @gen.coroutine
     def _put(self, value, timeout=None):
         if isinstance(value, Future):
-            yield self.client.scheduler.queue_put(key=tokey(value.key),
-                                                  timeout=timeout,
-                                                  name=self.name)
+            yield self.client.scheduler.queue_put(
+                key=tokey(value.key), timeout=timeout, name=self.name
+            )
         else:
-            yield self.client.scheduler.queue_put(data=value,
-                                                  timeout=timeout,
-                                                  name=self.name)
+            yield self.client.scheduler.queue_put(
+                data=value, timeout=timeout, name=self.name
+            )
 
     def put(self, value, timeout=None, **kwargs):
         """ Put data into the queue """
@@ -207,8 +218,7 @@ def get(self, timeout=None, batch=False, **kwargs):
             If an integer than return that many elements from the queue
             If False (default) then return one item at a time
          """
-        return self.client.sync(self._get, timeout=timeout, batch=batch,
-                                **kwargs)
+        return self.client.sync(self._get, timeout=timeout, batch=batch, **kwargs)
 
     def qsize(self, **kwargs):
         """ Current number of elements in the queue """
@@ -216,21 +226,20 @@ def qsize(self, **kwargs):
 
     @gen.coroutine
     def _get(self, timeout=None, batch=False):
-        resp = yield self.client.scheduler.queue_get(timeout=timeout,
-                                                     name=self.name,
-                                                     batch=batch)
+        resp = yield self.client.scheduler.queue_get(
+            timeout=timeout, name=self.name, batch=batch
+        )
 
         def process(d):
-            if d['type'] == 'Future':
-                value = Future(d['value'], self.client, inform=True,
-                               state=d['state'])
-                if d['state'] == 'erred':
-                    value._state.set_error(d['exception'], d['traceback'])
-                self.client._send_to_scheduler({'op': 'queue-future-release',
-                                                'name': self.name,
-                                                'key': d['value']})
+            if d["type"] == "Future":
+                value = Future(d["value"], self.client, inform=True, state=d["state"])
+                if d["state"] == "erred":
+                    value._state.set_error(d["exception"], d["traceback"])
+                self.client._send_to_scheduler(
+                    {"op": "queue-future-release", "name": self.name, "key": d["value"]}
+                )
             else:
-                value = d['value']
+                value = d["value"]
 
             return value
 
@@ -247,9 +256,8 @@ def _qsize(self):
         raise gen.Return(result)
 
     def close(self):
-        if self.client.status == 'running':  # TODO: can leave zombie futures
-            self.client._send_to_scheduler({'op': 'queue_release',
-                                            'name': self.name})
+        if self.client.status == "running":  # TODO: can leave zombie futures
+            self.client._send_to_scheduler({"op": "queue_release", "name": self.name})
 
     def __getstate__(self):
         return (self.name, self.client.scheduler.address)
diff --git a/distributed/recreate_exceptions.py b/distributed/recreate_exceptions.py
index cd252af8ac6..d5351bb4d59 100644
--- a/distributed/recreate_exceptions.py
+++ b/distributed/recreate_exceptions.py
@@ -20,8 +20,8 @@ class ReplayExceptionScheduler(object):
 
     def __init__(self, scheduler):
         self.scheduler = scheduler
-        self.scheduler.handlers['cause_of_failure'] = self.cause_of_failure
-        self.scheduler.extensions['exceptions'] = self
+        self.scheduler.handlers["cause_of_failure"] = self.cause_of_failure
+        self.scheduler.extensions["exceptions"] = self
 
     def cause_of_failure(self, *args, **kwargs):
         """
@@ -39,7 +39,7 @@ def cause_of_failure(self, *args, **kwargs):
         deps: keys that the task depends on
         """
 
-        keys = kwargs.pop('keys', [])
+        keys = kwargs.pop("keys", [])
         for key in keys:
             if isinstance(key, list):
                 key = tuple(key)  # ensure not a list from msgpack
@@ -48,9 +48,11 @@ def cause_of_failure(self, *args, **kwargs):
             if ts is not None and ts.exception_blame is not None:
                 cause = ts.exception_blame
                 # NOTE: cannot serialize sets
-                return {'deps': [dts.key for dts in cause.dependencies],
-                        'cause': cause.key,
-                        'task': cause.run_spec}
+                return {
+                    "deps": [dts.key for dts in cause.dependencies],
+                    "cause": cause.key,
+                    "task": cause.run_spec,
+                }
 
 
 class ReplayExceptionClient(object):
@@ -66,7 +68,7 @@ class ReplayExceptionClient(object):
 
     def __init__(self, client):
         self.client = client
-        self.client.extensions['exceptions'] = self
+        self.client.extensions["exceptions"] = self
         # monkey patch
         self.client.recreate_error_locally = self.recreate_error_locally
         self.client._recreate_error_locally = self._recreate_error_locally
@@ -80,12 +82,11 @@ def scheduler(self):
     @gen.coroutine
     def _get_futures_error(self, future):
         # only get errors for futures that errored.
-        futures = [f for f in futures_of(future) if f.status == 'error']
+        futures = [f for f in futures_of(future) if f.status == "error"]
         if not futures:
             raise ValueError("No errored futures passed")
-        out = yield self.scheduler.cause_of_failure(
-            keys=[f.key for f in futures])
-        deps, task = out['deps'], out['task']
+        out = yield self.scheduler.cause_of_failure(keys=[f.key for f in futures])
+        deps, task = out["deps"], out["task"]
         if isinstance(task, dict):
             function, args, kwargs = _deserialize(**task)
             raise gen.Return((function, args, kwargs, deps))
@@ -177,6 +178,7 @@ def recreate_error_locally(self, future):
         Nothing; the function runs and should raise an exception, allowing
         the debugger to run.
         """
-        func, args, kwargs = sync(self.client.loop,
-                                  self._recreate_error_locally, future)
+        func, args, kwargs = sync(
+            self.client.loop, self._recreate_error_locally, future
+        )
         func(*args, **kwargs)
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index cf26d0c955d..5e5e2843c2c 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -16,6 +16,7 @@
 
 import psutil
 import sortedcontainers
+
 try:
     from cytoolz import frequencies, merge, pluck, merge_sorted, first
 except ImportError:
@@ -28,20 +29,34 @@
 import dask
 
 from .batched import BatchedSend
-from .comm import (normalize_address, resolve_address,
-                   get_address_host, unparse_host_port)
+from .comm import (
+    normalize_address,
+    resolve_address,
+    get_address_host,
+    unparse_host_port,
+)
 from .compatibility import finalize, unicode, Mapping, Set
-from .core import (rpc, connect, send_recv,
-                   clean_exception, CommClosedError)
+from .core import rpc, connect, send_recv, clean_exception, CommClosedError
 from . import profile
 from .metrics import time
 from .node import ServerNode
 from .proctitle import setproctitle
 from .security import Security
-from .utils import (All, ignoring, get_ip, get_fileno_limit, log_errors,
-                    key_split, validate_key, no_default, DequeHandler,
-                    parse_timedelta, PeriodicCallback, shutting_down)
-from .utils_comm import (scatter_to_workers, gather_from_workers)
+from .utils import (
+    All,
+    ignoring,
+    get_ip,
+    get_fileno_limit,
+    log_errors,
+    key_split,
+    validate_key,
+    no_default,
+    DequeHandler,
+    parse_timedelta,
+    PeriodicCallback,
+    shutting_down,
+)
+from .utils_comm import scatter_to_workers, gather_from_workers
 from .utils_perf import enable_gc_diagnosis, disable_gc_diagnosis
 
 from .publish import PublishExtension
@@ -56,11 +71,11 @@
 logger = logging.getLogger(__name__)
 
 
-BANDWIDTH = dask.config.get('distributed.scheduler.bandwidth')
-ALLOWED_FAILURES = dask.config.get('distributed.scheduler.allowed-failures')
+BANDWIDTH = dask.config.get("distributed.scheduler.bandwidth")
+ALLOWED_FAILURES = dask.config.get("distributed.scheduler.allowed-failures")
 
-LOG_PDB = dask.config.get('distributed.admin.pdb-on-err')
-DEFAULT_DATA_SIZE = dask.config.get('distributed.scheduler.default-data-size')
+LOG_PDB = dask.config.get("distributed.admin.pdb-on-err")
+DEFAULT_DATA_SIZE = dask.config.get("distributed.scheduler.default-data-size")
 
 DEFAULT_EXTENSIONS = [
     LockExtension,
@@ -71,10 +86,10 @@
     PubSubSchedulerExtension,
 ]
 
-if dask.config.get('distributed.scheduler.work-stealing'):
+if dask.config.get("distributed.scheduler.work-stealing"):
     DEFAULT_EXTENSIONS.append(WorkStealing)
 
-ALL_TASK_STATES = {'released', 'waiting', 'no-worker', 'processing', 'erred', 'memory'}
+ALL_TASK_STATES = {"released", "waiting", "no-worker", "processing", "erred", "memory"}
 
 
 class ClientState(object):
@@ -97,11 +112,8 @@ class ClientState(object):
        collection) gets garbage-collected.
 
     """
-    __slots__ = (
-        'client_key',
-        'wants_what',
-        'last_seen',
-    )
+
+    __slots__ = ("client_key", "wants_what", "last_seen")
 
     def __init__(self, client):
         self.client_key = client
@@ -189,31 +201,40 @@ class WorkerState(object):
        actors to which this worker has a reference.
 
     """
+
     # XXX need a state field to signal active/removed?
 
     __slots__ = (
-        'actors',
-        'address',
-        'has_what',
-        'last_seen',
-        'local_directory',
-        'memory_limit',
-        'metrics',
-        'name',
-        'nbytes',
-        'ncores',
-        'occupancy',
-        'pid',
-        'processing',
-        'resources',
-        'services',
-        'status',
-        'time_delay',
-        'used_resources',
+        "actors",
+        "address",
+        "has_what",
+        "last_seen",
+        "local_directory",
+        "memory_limit",
+        "metrics",
+        "name",
+        "nbytes",
+        "ncores",
+        "occupancy",
+        "pid",
+        "processing",
+        "resources",
+        "services",
+        "status",
+        "time_delay",
+        "used_resources",
     )
 
-    def __init__(self, address=None, pid=0, name=None, ncores=0, memory_limit=0,
-                 local_directory=None, services=None):
+    def __init__(
+        self,
+        address=None,
+        pid=0,
+        name=None,
+        ncores=0,
+        memory_limit=0,
+        local_directory=None,
+        services=None,
+    ):
         self.address = address
         self.pid = pid
         self.name = name
@@ -222,7 +243,7 @@ def __init__(self, address=None, pid=0, name=None, ncores=0, memory_limit=0,
         self.local_directory = local_directory
         self.services = services or {}
 
-        self.status = 'running'
+        self.status = "running"
         self.nbytes = 0
         self.occupancy = 0
         self.metrics = {}
@@ -240,25 +261,28 @@ def host(self):
         return get_address_host(self.address)
 
     def __repr__(self):
-        return "<Worker %r, memory: %d, processing: %d>" % (self.address,
-                len(self.has_what), len(self.processing))
+        return "<Worker %r, memory: %d, processing: %d>" % (
+            self.address,
+            len(self.has_what),
+            len(self.processing),
+        )
 
     def __str__(self):
         return self.address
 
     def identity(self):
         return {
-            'type': 'Worker',
-            'id': self.name,
-            'host': self.host,
-            'resources': self.resources,
-            'local_directory': self.local_directory,
-            'name': self.name,
-            'ncores': self.ncores,
-            'memory_limit': self.memory_limit,
-            'last_seen': self.last_seen,
-            'services': self.services,
-            'metrics': self.metrics
+            "type": "Worker",
+            "id": self.name,
+            "host": self.host,
+            "resources": self.resources,
+            "local_directory": self.local_directory,
+            "name": self.name,
+            "ncores": self.ncores,
+            "memory_limit": self.memory_limit,
+            "last_seen": self.last_seen,
+            "services": self.services,
+            "metrics": self.metrics,
         }
 
 
@@ -486,47 +510,48 @@ class TaskState(object):
 
        Whether or not this task is an Actor.
     """
+
     __slots__ = (
         # === General description ===
-        'actor',
+        "actor",
         # Key name
-        'key',
+        "key",
         # Key prefix (see key_split())
-        'prefix',
+        "prefix",
         # How to run the task (None if pure data)
-        'run_spec',
+        "run_spec",
         # Alive dependents and dependencies
-        'dependencies',
-        'dependents',
+        "dependencies",
+        "dependents",
         # Compute priority
-        'priority',
+        "priority",
         # Restrictions
-        'host_restrictions',
-        'worker_restrictions',  # not WorkerStates but addresses
-        'resource_restrictions',
-        'loose_restrictions',
+        "host_restrictions",
+        "worker_restrictions",  # not WorkerStates but addresses
+        "resource_restrictions",
+        "loose_restrictions",
         # === Task state ===
-        'state',
+        "state",
         # Whether some dependencies were forgotten
-        'has_lost_dependencies',
+        "has_lost_dependencies",
         # If in 'waiting' state, which tasks need to complete
         # before we can run
-        'waiting_on',
+        "waiting_on",
         # If in 'waiting' or 'processing' state, which tasks needs us
         # to complete before they can run
-        'waiters',
+        "waiters",
         # In in 'processing' state, which worker we are processing on
-        'processing_on',
+        "processing_on",
         # If in 'memory' state, Which workers have us
-        'who_has',
+        "who_has",
         # Which clients want us
-        'who_wants',
-        'exception',
-        'traceback',
-        'exception_blame',
-        'suspicious',
-        'retries',
-        'nbytes',
+        "who_wants",
+        "exception",
+        "traceback",
+        "exception_blame",
+        "suspicious",
+        "retries",
+        "nbytes",
     )
 
     def __init__(self, key, run_spec):
@@ -581,6 +606,7 @@ def validate(self):
             logger.exception(e)
             if LOG_PDB:
                 import pdb
+
                 pdb.set_trace()
 
 
@@ -611,6 +637,7 @@ class _OptionalStateLegacyMapping(_StateLegacyMapping):
     Similar to _StateLegacyMapping, but a false-y value is interpreted
     as a missing key.
     """
+
     # For tasks etc.
 
     def __iter__(self):
@@ -636,6 +663,7 @@ class _StateLegacySet(Set):
     Similar to _StateLegacyMapping, but exposes a set containing
     all values with a true value.
     """
+
     # For loose_restrictions
 
     def __init__(self, states, accessor):
@@ -759,21 +787,23 @@ class Scheduler(ServerNode):
     * **coroutines:** ``[Futures]``:
         A list of active futures that control operation
     """
+
     default_port = 8786
 
     def __init__(
-            self,
-            loop=None,
-            delete_interval='500ms',
-            synchronize_worker_interval='60s',
-            services=None,
-            allowed_failures=ALLOWED_FAILURES,
-            extensions=None,
-            validate=False,
-            scheduler_file=None,
-            security=None,
-            worker_ttl=None,
-            **kwargs):
+        self,
+        loop=None,
+        delete_interval="500ms",
+        synchronize_worker_interval="60s",
+        services=None,
+        allowed_failures=ALLOWED_FAILURES,
+        extensions=None,
+        validate=False,
+        scheduler_file=None,
+        security=None,
+        worker_ttl=None,
+        **kwargs
+    ):
 
         self._setup_logging()
 
@@ -782,19 +812,21 @@ def __init__(
         self.validate = validate
         self.status = None
         self.proc = psutil.Process()
-        self.delete_interval = parse_timedelta(delete_interval, default='ms')
-        self.synchronize_worker_interval = parse_timedelta(synchronize_worker_interval, default='ms')
+        self.delete_interval = parse_timedelta(delete_interval, default="ms")
+        self.synchronize_worker_interval = parse_timedelta(
+            synchronize_worker_interval, default="ms"
+        )
         self.digests = None
         self.service_specs = services or {}
         self.services = {}
         self.scheduler_file = scheduler_file
-        worker_ttl = worker_ttl or dask.config.get('distributed.scheduler.worker-ttl')
+        worker_ttl = worker_ttl or dask.config.get("distributed.scheduler.worker-ttl")
         self.worker_ttl = parse_timedelta(worker_ttl) if worker_ttl else None
 
         self.security = security or Security()
         assert isinstance(self.security, Security)
-        self.connection_args = self.security.get_connection_args('scheduler')
-        self.listen_args = self.security.get_listen_args('scheduler')
+        self.connection_args = self.security.get_connection_args("scheduler")
+        self.listen_args = self.security.get_listen_args("scheduler")
 
         # Communication state
         self.loop = loop or IOLoop.current()
@@ -807,42 +839,43 @@ def __init__(
         # Task state
         self.tasks = dict()
         for old_attr, new_attr, wrap in [
-                ('priority', 'priority', None),
-                ('dependencies', 'dependencies', _legacy_task_key_set),
-                ('dependents', 'dependents', _legacy_task_key_set),
-                ('retries', 'retries', None)]:
+            ("priority", "priority", None),
+            ("dependencies", "dependencies", _legacy_task_key_set),
+            ("dependents", "dependents", _legacy_task_key_set),
+            ("retries", "retries", None),
+        ]:
             func = operator.attrgetter(new_attr)
             if wrap is not None:
                 func = compose(wrap, func)
-            setattr(self, old_attr,
-                    _StateLegacyMapping(self.tasks, func))
+            setattr(self, old_attr, _StateLegacyMapping(self.tasks, func))
 
         for old_attr, new_attr, wrap in [
-                ('nbytes', 'nbytes', None),
-                ('who_wants', 'who_wants', _legacy_client_key_set),
-                ('who_has', 'who_has', _legacy_worker_key_set),
-                ('waiting', 'waiting_on', _legacy_task_key_set),
-                ('waiting_data', 'waiters', _legacy_task_key_set),
-                ('rprocessing', 'processing_on', None),
-                ('host_restrictions', 'host_restrictions', None),
-                ('worker_restrictions', 'worker_restrictions', None),
-                ('resource_restrictions', 'resource_restrictions', None),
-                ('suspicious_tasks', 'suspicious', None),
-                ('exceptions', 'exception', None),
-                ('tracebacks', 'traceback', None),
-                ('exceptions_blame', 'exception_blame', _task_key_or_none)]:
+            ("nbytes", "nbytes", None),
+            ("who_wants", "who_wants", _legacy_client_key_set),
+            ("who_has", "who_has", _legacy_worker_key_set),
+            ("waiting", "waiting_on", _legacy_task_key_set),
+            ("waiting_data", "waiters", _legacy_task_key_set),
+            ("rprocessing", "processing_on", None),
+            ("host_restrictions", "host_restrictions", None),
+            ("worker_restrictions", "worker_restrictions", None),
+            ("resource_restrictions", "resource_restrictions", None),
+            ("suspicious_tasks", "suspicious", None),
+            ("exceptions", "exception", None),
+            ("tracebacks", "traceback", None),
+            ("exceptions_blame", "exception_blame", _task_key_or_none),
+        ]:
             func = operator.attrgetter(new_attr)
             if wrap is not None:
                 func = compose(wrap, func)
-            setattr(self, old_attr,
-                    _OptionalStateLegacyMapping(self.tasks, func))
+            setattr(self, old_attr, _OptionalStateLegacyMapping(self.tasks, func))
 
-        for old_attr, new_attr, wrap in [('loose_restrictions', 'loose_restrictions', None)]:
+        for old_attr, new_attr, wrap in [
+            ("loose_restrictions", "loose_restrictions", None)
+        ]:
             func = operator.attrgetter(new_attr)
             if wrap is not None:
                 func = compose(wrap, func)
-            setattr(self, old_attr,
-                    _StateLegacySet(self.tasks, func))
+            setattr(self, old_attr, _StateLegacySet(self.tasks, func))
 
         self.generation = 0
         self._last_client = None
@@ -859,32 +892,33 @@ def __init__(
 
         # Client state
         self.clients = dict()
-        for old_attr, new_attr, wrap in [('wants_what', 'wants_what', _legacy_task_key_set)]:
+        for old_attr, new_attr, wrap in [
+            ("wants_what", "wants_what", _legacy_task_key_set)
+        ]:
             func = operator.attrgetter(new_attr)
             if wrap is not None:
                 func = compose(wrap, func)
-            setattr(self, old_attr,
-                    _StateLegacyMapping(self.clients, func))
-        self.clients['fire-and-forget'] = ClientState('fire-and-forget')
+            setattr(self, old_attr, _StateLegacyMapping(self.clients, func))
+        self.clients["fire-and-forget"] = ClientState("fire-and-forget")
 
         # Worker state
         self.workers = sortedcontainers.SortedDict()
         for old_attr, new_attr, wrap in [
-                ('ncores', 'ncores', None),
-                ('worker_bytes', 'nbytes', None),
-                ('worker_resources', 'resources', None),
-                ('used_resources', 'used_resources', None),
-                ('occupancy', 'occupancy', None),
-                ('worker_info', 'metrics', None),
-                ('processing', 'processing', _legacy_task_key_dict),
-                ('has_what', 'has_what', _legacy_task_key_set)]:
+            ("ncores", "ncores", None),
+            ("worker_bytes", "nbytes", None),
+            ("worker_resources", "resources", None),
+            ("used_resources", "used_resources", None),
+            ("occupancy", "occupancy", None),
+            ("worker_info", "metrics", None),
+            ("processing", "processing", _legacy_task_key_dict),
+            ("has_what", "has_what", _legacy_task_key_set),
+        ]:
             func = operator.attrgetter(new_attr)
             if wrap is not None:
                 func = compose(wrap, func)
-            setattr(self, old_attr,
-                    _StateLegacyMapping(self.workers, func))
+            setattr(self, old_attr, _StateLegacyMapping(self.workers, func))
 
-        self.idle = sortedcontainers.SortedSet(key=operator.attrgetter('address'))
+        self.idle = sortedcontainers.SortedSet(key=operator.attrgetter("address"))
         self.saturated = set()
 
         self.total_ncores = 0
@@ -895,91 +929,99 @@ def __init__(
 
         self._task_state_collections = [self.unrunnable]
 
-        self._worker_collections = [self.workers, self.host_info,
-                                    self.resources, self.aliases]
+        self._worker_collections = [
+            self.workers,
+            self.host_info,
+            self.resources,
+            self.aliases,
+        ]
 
         self.extensions = {}
         self.plugins = []
-        self.transition_log = deque(maxlen=dask.config.get('distributed.scheduler.transition-log-length'))
-        self.log = deque(maxlen=dask.config.get('distributed.scheduler.transition-log-length'))
+        self.transition_log = deque(
+            maxlen=dask.config.get("distributed.scheduler.transition-log-length")
+        )
+        self.log = deque(
+            maxlen=dask.config.get("distributed.scheduler.transition-log-length")
+        )
         self.worker_setups = []
 
         worker_handlers = {
-            'task-finished': self.handle_task_finished,
-            'task-erred': self.handle_task_erred,
-            'release': self.handle_release_data,
-            'release-worker-data': self.release_worker_data,
-            'add-keys': self.add_keys,
-            'missing-data': self.handle_missing_data,
-            'long-running': self.handle_long_running,
-            'reschedule': self.reschedule
+            "task-finished": self.handle_task_finished,
+            "task-erred": self.handle_task_erred,
+            "release": self.handle_release_data,
+            "release-worker-data": self.release_worker_data,
+            "add-keys": self.add_keys,
+            "missing-data": self.handle_missing_data,
+            "long-running": self.handle_long_running,
+            "reschedule": self.reschedule,
         }
 
         client_handlers = {
-            'update-graph': self.update_graph,
-            'client-desires-keys': self.client_desires_keys,
-            'update-data': self.update_data,
-            'report-key': self.report_on_key,
-            'client-releases-keys': self.client_releases_keys,
-            'heartbeat-client': self.client_heartbeat,
-            'close-client': self.remove_client,
-            'restart': self.restart
+            "update-graph": self.update_graph,
+            "client-desires-keys": self.client_desires_keys,
+            "update-data": self.update_data,
+            "report-key": self.report_on_key,
+            "client-releases-keys": self.client_releases_keys,
+            "heartbeat-client": self.client_heartbeat,
+            "close-client": self.remove_client,
+            "restart": self.restart,
         }
 
         self.handlers = {
-            'register-client': self.add_client,
-            'scatter': self.scatter,
-            'register-worker': self.add_worker,
-            'unregister': self.remove_worker,
-            'gather': self.gather,
-            'cancel': self.stimulus_cancel,
-            'retry': self.stimulus_retry,
-            'feed': self.feed,
-            'terminate': self.close,
-            'broadcast': self.broadcast,
-            'proxy': self.proxy,
-            'ncores': self.get_ncores,
-            'has_what': self.get_has_what,
-            'who_has': self.get_who_has,
-            'processing': self.get_processing,
-            'call_stack': self.get_call_stack,
-            'profile': self.get_profile,
-            'logs': self.get_logs,
-            'worker_logs': self.get_worker_logs,
-            'nbytes': self.get_nbytes,
-            'versions': self.versions,
-            'add_keys': self.add_keys,
-            'rebalance': self.rebalance,
-            'replicate': self.replicate,
-            'start_ipython': self.start_ipython,
-            'run_function': self.run_function,
-            'update_data': self.update_data,
-            'set_resources': self.add_resources,
-            'retire_workers': self.retire_workers,
-            'get_metadata': self.get_metadata,
-            'set_metadata': self.set_metadata,
-            'heartbeat_worker': self.heartbeat_worker,
-            'get_task_status': self.get_task_status,
-            'get_task_stream': self.get_task_stream,
-            'register_worker_callbacks': self.register_worker_callbacks
+            "register-client": self.add_client,
+            "scatter": self.scatter,
+            "register-worker": self.add_worker,
+            "unregister": self.remove_worker,
+            "gather": self.gather,
+            "cancel": self.stimulus_cancel,
+            "retry": self.stimulus_retry,
+            "feed": self.feed,
+            "terminate": self.close,
+            "broadcast": self.broadcast,
+            "proxy": self.proxy,
+            "ncores": self.get_ncores,
+            "has_what": self.get_has_what,
+            "who_has": self.get_who_has,
+            "processing": self.get_processing,
+            "call_stack": self.get_call_stack,
+            "profile": self.get_profile,
+            "logs": self.get_logs,
+            "worker_logs": self.get_worker_logs,
+            "nbytes": self.get_nbytes,
+            "versions": self.versions,
+            "add_keys": self.add_keys,
+            "rebalance": self.rebalance,
+            "replicate": self.replicate,
+            "start_ipython": self.start_ipython,
+            "run_function": self.run_function,
+            "update_data": self.update_data,
+            "set_resources": self.add_resources,
+            "retire_workers": self.retire_workers,
+            "get_metadata": self.get_metadata,
+            "set_metadata": self.set_metadata,
+            "heartbeat_worker": self.heartbeat_worker,
+            "get_task_status": self.get_task_status,
+            "get_task_stream": self.get_task_stream,
+            "register_worker_callbacks": self.register_worker_callbacks,
         }
 
         self._transitions = {
-            ('released', 'waiting'): self.transition_released_waiting,
-            ('waiting', 'released'): self.transition_waiting_released,
-            ('waiting', 'processing'): self.transition_waiting_processing,
-            ('waiting', 'memory'): self.transition_waiting_memory,
-            ('processing', 'released'): self.transition_processing_released,
-            ('processing', 'memory'): self.transition_processing_memory,
-            ('processing', 'erred'): self.transition_processing_erred,
-            ('no-worker', 'released'): self.transition_no_worker_released,
-            ('no-worker', 'waiting'): self.transition_no_worker_waiting,
-            ('released', 'forgotten'): self.transition_released_forgotten,
-            ('memory', 'forgotten'): self.transition_memory_forgotten,
-            ('erred', 'forgotten'): self.transition_released_forgotten,
-            ('erred', 'released'): self.transition_erred_released,
-            ('memory', 'released'): self.transition_memory_released,
-            ('released', 'erred'): self.transition_released_erred
+            ("released", "waiting"): self.transition_released_waiting,
+            ("waiting", "released"): self.transition_waiting_released,
+            ("waiting", "processing"): self.transition_waiting_processing,
+            ("waiting", "memory"): self.transition_waiting_memory,
+            ("processing", "released"): self.transition_processing_released,
+            ("processing", "memory"): self.transition_processing_memory,
+            ("processing", "erred"): self.transition_processing_erred,
+            ("no-worker", "released"): self.transition_no_worker_released,
+            ("no-worker", "waiting"): self.transition_no_worker_waiting,
+            ("released", "forgotten"): self.transition_released_forgotten,
+            ("memory", "forgotten"): self.transition_memory_forgotten,
+            ("erred", "forgotten"): self.transition_released_forgotten,
+            ("erred", "released"): self.transition_erred_released,
+            ("memory", "released"): self.transition_memory_released,
+            ("released", "erred"): self.transition_released_erred,
         }
 
         connection_limit = get_fileno_limit() / 2
@@ -988,15 +1030,15 @@ def __init__(
             handlers=self.handlers,
             stream_handlers=merge(worker_handlers, client_handlers),
             io_loop=self.loop,
-            connection_limit=connection_limit, deserialize=False,
+            connection_limit=connection_limit,
+            deserialize=False,
             connection_args=self.connection_args,
-            **kwargs)
+            **kwargs
+        )
 
         if self.worker_ttl:
-            pc = PeriodicCallback(self.check_worker_ttl,
-                                  self.worker_ttl,
-                                  io_loop=loop)
-            self.periodic_callbacks['worker-ttl'] = pc
+            pc = PeriodicCallback(self.check_worker_ttl, self.worker_ttl, io_loop=loop)
+            self.periodic_callbacks["worker-ttl"] = pc
 
         if extensions is None:
             extensions = DEFAULT_EXTENSIONS
@@ -1011,16 +1053,22 @@ def __init__(
 
     def __repr__(self):
         return '<Scheduler: "%s" processes: %d cores: %d>' % (
-            self.address, len(self.workers), self.total_ncores)
+            self.address,
+            len(self.workers),
+            self.total_ncores,
+        )
 
     def identity(self, comm=None):
         """ Basic information about ourselves and our cluster """
-        d = {'type': type(self).__name__,
-             'id': str(self.id),
-             'address': self.address,
-             'services': {key: v.port for (key, v) in self.services.items()},
-             'workers': {worker.address: worker.identity()
-                         for worker in self.workers.values()}}
+        d = {
+            "type": type(self).__name__,
+            "id": str(self.id),
+            "address": self.address,
+            "services": {key: v.port for (key, v) in self.services.items()},
+            "workers": {
+                worker.address: worker.identity() for worker in self.workers.values()
+            },
+        }
         return d
 
     def get_worker_service_addr(self, worker, service_name, protocol=False):
@@ -1042,17 +1090,17 @@ def get_worker_service_addr(self, worker, service_name, protocol=False):
         if port is None:
             return None
         elif protocol:
-            return '%(protocol)s://%(host)s:%(port)d' % {
-                'protocol': ws.address.split('://')[0],
-                'host': ws.host,
-                'port': port
+            return "%(protocol)s://%(host)s:%(port)d" % {
+                "protocol": ws.address.split("://")[0],
+                "host": ws.host,
+                "port": port,
             }
         else:
             return ws.host, port
 
     def start_services(self, default_listen_ip):
-        if default_listen_ip == '0.0.0.0':
-            default_listen_ip = ''  # for IPV6
+        if default_listen_ip == "0.0.0.0":
+            default_listen_ip = ""  # for IPV6
 
         for k, v in self.service_specs.items():
             listen_ip = None
@@ -1062,7 +1110,7 @@ def start_services(self, default_listen_ip):
                 port = 0
 
             if isinstance(port, (str, unicode)):
-                port = port.split(':')
+                port = port.split(":")
 
             if isinstance(port, (tuple, list)):
                 listen_ip, port = (port[0], int(port[1]))
@@ -1074,12 +1122,17 @@ def start_services(self, default_listen_ip):
 
             try:
                 service = v(self, io_loop=self.loop, **kwargs)
-                service.listen((listen_ip if listen_ip is not None else default_listen_ip, port))
+                service.listen(
+                    (listen_ip if listen_ip is not None else default_listen_ip, port)
+                )
                 self.services[k] = service
             except Exception as e:
-                warnings.warn("\nCould not launch service '%s' on port %s. " % (k, port) +
-                              "Got the following message:\n\n" + str(e),
-                              stacklevel=3)
+                warnings.warn(
+                    "\nCould not launch service '%s' on port %s. " % (k, port)
+                    + "Got the following message:\n\n"
+                    + str(e),
+                    stacklevel=3,
+                )
 
     def stop_services(self):
         for service in self.services.values():
@@ -1101,36 +1154,36 @@ def start(self, addr_or_port=8786, start_queues=True):
                 if exc:
                     raise exc
 
-        if self.status != 'running':
+        if self.status != "running":
             if isinstance(addr_or_port, int):
                 # Listen on all interfaces.  `get_ip()` is not suitable
                 # as it would prevent connecting via 127.0.0.1.
-                self.listen(('', addr_or_port), listen_args=self.listen_args)
+                self.listen(("", addr_or_port), listen_args=self.listen_args)
                 self.ip = get_ip()
-                listen_ip = ''
+                listen_ip = ""
             else:
                 self.listen(addr_or_port, listen_args=self.listen_args)
                 self.ip = get_address_host(self.listen_address)
                 listen_ip = self.ip
 
-            if listen_ip == '0.0.0.0':
-                listen_ip = ''
+            if listen_ip == "0.0.0.0":
+                listen_ip = ""
 
-            if isinstance(addr_or_port, str) and addr_or_port.startswith('inproc://'):
-                listen_ip = 'localhost'
+            if isinstance(addr_or_port, str) and addr_or_port.startswith("inproc://"):
+                listen_ip = "localhost"
 
             # Services listen on all addresses
             self.start_services(listen_ip)
 
-            self.status = 'running'
+            self.status = "running"
             logger.info("  Scheduler at: %25s", self.address)
             for k, v in self.services.items():
-                logger.info("%11s at: %25s", k, '%s:%d' % (listen_ip, v.port))
+                logger.info("%11s at: %25s", k, "%s:%d" % (listen_ip, v.port))
 
             self.loop.add_callback(self.reevaluate_occupancy)
 
         if self.scheduler_file:
-            with open(self.scheduler_file, 'w') as f:
+            with open(self.scheduler_file, "w") as f:
                 json.dump(self.identity(), f, indent=2)
 
             fn = self.scheduler_file  # remove file when we close the process
@@ -1161,9 +1214,9 @@ def close(self, comm=None, fast=False):
         --------
         Scheduler.cleanup
         """
-        if self.status.startswith('clos'):
+        if self.status.startswith("clos"):
             return
-        self.status = 'closing'
+        self.status = "closing"
 
         logger.info("Scheduler closing...")
         setproctitle("dask-scheduler [closing]")
@@ -1181,8 +1234,8 @@ def close(self, comm=None, fast=False):
         futures = []
         for w, comm in list(self.stream_comms.items()):
             if not comm.closed():
-                comm.send({'op': 'close', 'report': False})
-                comm.send({'op': 'close-stream'})
+                comm.send({"op": "close", "report": False})
+                comm.send({"op": "close-stream"})
             with ignoring(AttributeError):
                 futures.append(comm.close())
 
@@ -1197,7 +1250,7 @@ def close(self, comm=None, fast=False):
 
         self.rpc.close()
 
-        self.status = 'closed'
+        self.status = "closed"
         self.stop()
         yield super(Scheduler, self).close()
 
@@ -1214,16 +1267,20 @@ def close_worker(self, stream=None, worker=None, safe=None):
         """
         logger.info("Closing worker %s", worker)
         with log_errors():
-            self.log_event(worker, {'action': 'close-worker'})
-            nanny_addr = self.get_worker_service_addr(worker, 'nanny', protocol=True)
+            self.log_event(worker, {"action": "close-worker"})
+            nanny_addr = self.get_worker_service_addr(worker, "nanny", protocol=True)
             address = nanny_addr or worker
 
-            self.worker_send(worker, {'op': 'close', 'report': False})
+            self.worker_send(worker, {"op": "close", "report": False})
             self.remove_worker(address=worker, safe=safe)
 
     def _setup_logging(self):
-        self._deque_handler = DequeHandler(n=dask.config.get('distributed.admin.log-length'))
-        self._deque_handler.setFormatter(logging.Formatter(dask.config.get('distributed.admin.log-format')))
+        self._deque_handler = DequeHandler(
+            n=dask.config.get("distributed.admin.log-length")
+        )
+        self._deque_handler.setFormatter(
+            logging.Formatter(dask.config.get("distributed.admin.log-format"))
+        )
         logger.addHandler(self._deque_handler)
         finalize(self, logger.removeHandler, self._deque_handler)
 
@@ -1232,8 +1289,16 @@ def _setup_logging(self):
     ###########
 
     @gen.coroutine
-    def heartbeat_worker(self, comm=None, address=None, resolve_address=True,
-                          now=None, resources=None, host_info=None, metrics=None):
+    def heartbeat_worker(
+        self,
+        comm=None,
+        address=None,
+        resolve_address=True,
+        now=None,
+        resources=None,
+        host_info=None,
+        metrics=None,
+    ):
         address = self.coerce_address(address, resolve_address)
         address = normalize_address(address)
         host = get_address_host(address)
@@ -1243,11 +1308,11 @@ def heartbeat_worker(self, comm=None, address=None, resolve_address=True,
         metrics = metrics or {}
         host_info = host_info or {}
 
-        self.host_info[host]['last-seen'] = local_now
+        self.host_info[host]["last-seen"] = local_now
 
         ws = self.workers.get(address)
         if not ws:
-            return {'status': 'missing'}
+            return {"status": "missing"}
 
         ws.last_seen = time()
 
@@ -1263,17 +1328,33 @@ def heartbeat_worker(self, comm=None, address=None, resolve_address=True,
         if resources:
             self.add_resources(worker=address, resources=resources)
 
-        self.log_event(address, merge({'action': 'heartbeat'}, metrics))
+        self.log_event(address, merge({"action": "heartbeat"}, metrics))
 
-        return {'status': 'OK',
-                'time': time(),
-                'heartbeat-interval': heartbeat_interval(len(self.workers))}
+        return {
+            "status": "OK",
+            "time": time(),
+            "heartbeat-interval": heartbeat_interval(len(self.workers)),
+        }
 
     @gen.coroutine
-    def add_worker(self, comm=None, address=None, keys=(), ncores=None,
-                   name=None, resolve_address=True, nbytes=None, now=None,
-                   resources=None, host_info=None, memory_limit=None,
-                   metrics=None, pid=0, services=None, local_directory=None):
+    def add_worker(
+        self,
+        comm=None,
+        address=None,
+        keys=(),
+        ncores=None,
+        name=None,
+        resolve_address=True,
+        nbytes=None,
+        now=None,
+        resources=None,
+        host_info=None,
+        memory_limit=None,
+        metrics=None,
+        pid=0,
+        services=None,
+        local_directory=None,
+    ):
         """ Add a new worker to the cluster """
         with log_errors():
             address = self.coerce_address(address, resolve_address)
@@ -1285,36 +1366,41 @@ def add_worker(self, comm=None, address=None, keys=(), ncores=None,
                 raise ValueError("Worker already exists %s" % address)
 
             self.workers[address] = ws = WorkerState(
-                    address=address,
-                    pid=pid,
-                    ncores=ncores,
-                    memory_limit=memory_limit,
-                    name=name,
-                    local_directory=local_directory,
-                    services=services
+                address=address,
+                pid=pid,
+                ncores=ncores,
+                memory_limit=memory_limit,
+                name=name,
+                local_directory=local_directory,
+                services=services,
             )
 
             if name in self.aliases:
-                msg = {'status': 'error',
-                       'message': 'name taken, %s' % name,
-                       'time': time()}
+                msg = {
+                    "status": "error",
+                    "message": "name taken, %s" % name,
+                    "time": time(),
+                }
                 yield comm.write(msg)
                 return
 
-            if 'addresses' not in self.host_info[host]:
-                self.host_info[host].update({'addresses': set(), 'cores': 0})
+            if "addresses" not in self.host_info[host]:
+                self.host_info[host].update({"addresses": set(), "cores": 0})
 
-            self.host_info[host]['addresses'].add(address)
-            self.host_info[host]['cores'] += ncores
+            self.host_info[host]["addresses"].add(address)
+            self.host_info[host]["cores"] += ncores
 
             self.total_ncores += ncores
             self.aliases[name] = address
 
-            response = self.heartbeat_worker(address=address,
-                                             resolve_address=resolve_address,
-                                             now=now, resources=resources,
-                                             host_info=host_info,
-                                             metrics=metrics)
+            response = self.heartbeat_worker(
+                address=address,
+                resolve_address=resolve_address,
+                now=now,
+                resources=resources,
+                host_info=host_info,
+                metrics=metrics,
+            )
 
             # Do not need to adjust self.total_occupancy as self.occupancy[ws] cannot exist before this.
             self.check_idle_saturated(ws)
@@ -1322,7 +1408,7 @@ def add_worker(self, comm=None, address=None, keys=(), ncores=None,
             # for key in keys:  # TODO
             #     self.mark_key_in_memory(key, [address])
 
-            self.stream_comms[address] = BatchedSend(interval='5ms', loop=self.loop)
+            self.stream_comms[address] = BatchedSend(interval="5ms", loop=self.loop)
 
             if ws.ncores > len(ws.processing):
                 self.idle.add(ws)
@@ -1336,37 +1422,51 @@ def add_worker(self, comm=None, address=None, keys=(), ncores=None,
             if nbytes:
                 for key in nbytes:
                     ts = self.tasks.get(key)
-                    if ts is not None and ts.state in ('processing', 'waiting'):
-                        recommendations = self.transition(key, 'memory',
-                                                          worker=address,
-                                                          nbytes=nbytes[key])
+                    if ts is not None and ts.state in ("processing", "waiting"):
+                        recommendations = self.transition(
+                            key, "memory", worker=address, nbytes=nbytes[key]
+                        )
                         self.transitions(recommendations)
 
             recommendations = {}
             for ts in list(self.unrunnable):
                 valid = self.valid_workers(ts)
                 if valid is True or ws in valid:
-                    recommendations[ts.key] = 'waiting'
+                    recommendations[ts.key] = "waiting"
 
             if recommendations:
                 self.transitions(recommendations)
 
-            self.log_event(address, {'action': 'add-worker'})
-            self.log_event('all', {'action': 'add-worker',
-                                   'worker': address})
+            self.log_event(address, {"action": "add-worker"})
+            self.log_event("all", {"action": "add-worker", "worker": address})
             logger.info("Register %s", str(address))
 
-            yield comm.write({'status': 'OK',
-                              'time': time(),
-                              'heartbeat-interval': heartbeat_interval(len(self.workers)),
-                              'worker-setups': self.worker_setups})
+            yield comm.write(
+                {
+                    "status": "OK",
+                    "time": time(),
+                    "heartbeat-interval": heartbeat_interval(len(self.workers)),
+                    "worker-setups": self.worker_setups,
+                }
+            )
             yield self.handle_worker(comm=comm, worker=address)
 
-    def update_graph(self, client=None, tasks=None, keys=None,
-                     dependencies=None, restrictions=None, priority=None,
-                     loose_restrictions=None, resources=None,
-                     submitting_task=None, retries=None, user_priority=0,
-                     actors=None, fifo_timeout=0):
+    def update_graph(
+        self,
+        client=None,
+        tasks=None,
+        keys=None,
+        dependencies=None,
+        restrictions=None,
+        priority=None,
+        loose_restrictions=None,
+        resources=None,
+        submitting_task=None,
+        retries=None,
+        user_priority=0,
+        actors=None,
+        fifo_timeout=0,
+    ):
         """
         Add new computations to the internal dask graph
 
@@ -1376,8 +1476,9 @@ def update_graph(self, client=None, tasks=None, keys=None,
         fifo_timeout = parse_timedelta(fifo_timeout)
         keys = set(keys)
         if len(tasks) > 1:
-            self.log_event(['all', client], {'action': 'update_graph',
-                                             'count': len(tasks)})
+            self.log_event(
+                ["all", client], {"action": "update_graph", "count": len(tasks)}
+            )
 
         # Remove aliases
         for k in list(tasks):
@@ -1390,14 +1491,15 @@ def update_graph(self, client=None, tasks=None, keys=None,
         while len(tasks) != n:  # walk through new tasks, cancel any bad deps
             n = len(tasks)
             for k, deps in list(dependencies.items()):
-                if any(dep not in self.tasks and dep not in tasks
-                       for dep in deps):  # bad key
-                    logger.info('User asked for computation on lost data, %s', k)
+                if any(
+                    dep not in self.tasks and dep not in tasks for dep in deps
+                ):  # bad key
+                    logger.info("User asked for computation on lost data, %s", k)
                     del tasks[k]
                     del dependencies[k]
                     if k in keys:
                         keys.remove(k)
-                    self.report({'op': 'cancelled-key', 'key': k}, client=client)
+                    self.report({"op": "cancelled-key", "key": k}, client=client)
                     self.client_releases_keys(keys=[k], client=client)
 
         # Remove any self-dependencies (happens on test_publish_bag() and others)
@@ -1410,7 +1512,7 @@ def update_graph(self, client=None, tasks=None, keys=None,
         # Avoid computation that is already finished
         already_in_memory = set()  # tasks that are already done
         for k, v in dependencies.items():
-            if v and k in self.tasks and self.tasks[k].state in ('memory', 'erred'):
+            if v and k in self.tasks and self.tasks[k].state in ("memory", "erred"):
                 already_in_memory.add(k)
 
         if already_in_memory:
@@ -1450,7 +1552,7 @@ def update_graph(self, client=None, tasks=None, keys=None,
             ts = self.tasks.get(k)
             if ts is None:
                 ts = self.tasks[k] = TaskState(k, tasks.get(k))
-                ts.state = 'released'
+                ts.state = "released"
             elif not ts.run_spec:
                 ts.run_spec = tasks.get(k)
 
@@ -1480,7 +1582,9 @@ def update_graph(self, client=None, tasks=None, keys=None,
         for actor in actors or []:
             self.tasks[actor].actor = True
 
-        priority = priority or dask.order.order(tasks)  # TODO: define order wrt old graph
+        priority = priority or dask.order.order(
+            tasks
+        )  # TODO: define order wrt old graph
 
         if submitting_task:  # sub-tasks get better priority than parent tasks
             ts = self.tasks.get(submitting_task)
@@ -1501,8 +1605,7 @@ def update_graph(self, client=None, tasks=None, keys=None,
                 ts.priority = (-user_priority.get(key, 0), generation, priority[key])
 
         # Ensure all runnables have a priority
-        runnables = [ts for ts in touched_tasks
-                     if ts.run_spec]
+        runnables = [ts for ts in touched_tasks if ts.run_spec]
         for ts in runnables:
             if ts.priority is None and ts.run_spec:
                 ts.priority = (self.generation, 0)
@@ -1553,38 +1656,42 @@ def update_graph(self, client=None, tasks=None, keys=None,
         # Compute recommendations
         recommendations = OrderedDict()
 
-        for ts in sorted(runnables, key=operator.attrgetter('priority'),
-                reverse=True):
-            if ts.state == 'released' and ts.run_spec:
-                recommendations[ts.key] = 'waiting'
+        for ts in sorted(runnables, key=operator.attrgetter("priority"), reverse=True):
+            if ts.state == "released" and ts.run_spec:
+                recommendations[ts.key] = "waiting"
 
         for ts in touched_tasks:
             for dts in ts.dependencies:
                 if dts.exception_blame:
                     ts.exception_blame = dts.exception_blame
-                    recommendations[ts.key] = 'erred'
+                    recommendations[ts.key] = "erred"
                     break
 
         for plugin in self.plugins[:]:
             try:
-                plugin.update_graph(self, client=client, tasks=tasks,
-                                    keys=keys, restrictions=restrictions or {},
-                                    dependencies=dependencies,
-                                    priority=priority,
-                                    loose_restrictions=loose_restrictions,
-                                    resources=resources)
+                plugin.update_graph(
+                    self,
+                    client=client,
+                    tasks=tasks,
+                    keys=keys,
+                    restrictions=restrictions or {},
+                    dependencies=dependencies,
+                    priority=priority,
+                    loose_restrictions=loose_restrictions,
+                    resources=resources,
+                )
             except Exception as e:
                 logger.exception(e)
 
         self.transitions(recommendations)
 
         for ts in touched_tasks:
-            if ts.state in ('memory', 'erred'):
+            if ts.state in ("memory", "erred"):
                 self.report_on_key(ts.key, client=client)
 
         end = time()
         if self.digests is not None:
-            self.digests['update-graph-duration'].add(end - start)
+            self.digests["update-graph-duration"].add(end - start)
 
         # TODO: balance workers
 
@@ -1597,24 +1704,29 @@ def stimulus_task_finished(self, key=None, worker=None, **kwargs):
             return {}
         ws = self.workers[worker]
 
-        if ts.state == 'processing':
-            recommendations = self.transition(key, 'memory', worker=worker,
-                                              **kwargs)
+        if ts.state == "processing":
+            recommendations = self.transition(key, "memory", worker=worker, **kwargs)
 
-            if ts.state == 'memory':
+            if ts.state == "memory":
                 assert ws in ts.who_has
         else:
-            logger.debug("Received already computed task, worker: %s, state: %s"
-                         ", key: %s, who_has: %s",
-                         worker, ts.state, key, ts.who_has)
+            logger.debug(
+                "Received already computed task, worker: %s, state: %s"
+                ", key: %s, who_has: %s",
+                worker,
+                ts.state,
+                key,
+                ts.who_has,
+            )
             if ws not in ts.who_has:
-                self.worker_send(worker, {'op': 'release-task', 'key': key})
+                self.worker_send(worker, {"op": "release-task", "key": key})
             recommendations = {}
 
         return recommendations
 
-    def stimulus_task_erred(self, key=None, worker=None,
-                            exception=None, traceback=None, **kwargs):
+    def stimulus_task_erred(
+        self, key=None, worker=None, exception=None, traceback=None, **kwargs
+    ):
         """ Mark that a task has erred on a particular worker """
         logger.debug("Stimulus task erred %s, %s", key, worker)
 
@@ -1622,45 +1734,49 @@ def stimulus_task_erred(self, key=None, worker=None,
         if ts is None:
             return {}
 
-        if ts.state == 'processing':
+        if ts.state == "processing":
             retries = ts.retries
             if retries > 0:
                 ts.retries = retries - 1
-                recommendations = self.transition(key, 'waiting')
+                recommendations = self.transition(key, "waiting")
             else:
-                recommendations = self.transition(key, 'erred',
-                                                  cause=key,
-                                                  exception=exception,
-                                                  traceback=traceback,
-                                                  worker=worker,
-                                                  **kwargs)
+                recommendations = self.transition(
+                    key,
+                    "erred",
+                    cause=key,
+                    exception=exception,
+                    traceback=traceback,
+                    worker=worker,
+                    **kwargs
+                )
         else:
             recommendations = {}
 
         return recommendations
 
-    def stimulus_missing_data(self, cause=None, key=None, worker=None,
-                              ensure=True, **kwargs):
+    def stimulus_missing_data(
+        self, cause=None, key=None, worker=None, ensure=True, **kwargs
+    ):
         """ Mark that certain keys have gone missing.  Recover. """
         with log_errors():
             logger.debug("Stimulus missing data %s, %s", key, worker)
 
             ts = self.tasks.get(key)
-            if ts is None or ts.state == 'memory':
+            if ts is None or ts.state == "memory":
                 return {}
             cts = self.tasks.get(cause)
 
             recommendations = OrderedDict()
 
-            if cts is not None and cts.state == 'memory':  # couldn't find this
+            if cts is not None and cts.state == "memory":  # couldn't find this
                 for ws in cts.who_has:  # TODO: this behavior is extreme
                     ws.has_what.remove(cts)
                     ws.nbytes -= cts.get_nbytes()
                 cts.who_has.clear()
-                recommendations[cause] = 'released'
+                recommendations[cause] = "released"
 
             if key:
-                recommendations[key] = 'released'
+                recommendations[key] = "released"
 
             self.transitions(recommendations)
 
@@ -1672,7 +1788,7 @@ def stimulus_missing_data(self, cause=None, key=None, worker=None,
     def stimulus_retry(self, comm=None, keys=None, client=None):
         logger.info("Client %s requests to retry %d keys", client, len(keys))
         if client:
-            self.log_event(client, {'action': 'retry', 'count': len(keys)})
+            self.log_event(client, {"action": "retry", "count": len(keys)})
 
         stack = list(keys)
         seen = set()
@@ -1680,14 +1796,15 @@ def stimulus_retry(self, comm=None, keys=None, client=None):
         while stack:
             key = stack.pop()
             seen.add(key)
-            erred_deps = [dts.key for dts in self.tasks[key].dependencies
-                          if dts.state == 'erred']
+            erred_deps = [
+                dts.key for dts in self.tasks[key].dependencies if dts.state == "erred"
+            ]
             if erred_deps:
                 stack.extend(erred_deps)
             else:
                 roots.append(key)
 
-        recommendations = {key: 'waiting' for key in roots}
+        recommendations = {key: "waiting" for key in roots}
         self.transitions(recommendations)
 
         if self.validate:
@@ -1705,31 +1822,36 @@ def remove_worker(self, comm=None, address=None, safe=False, close=True):
         state.
         """
         with log_errors():
-            if self.status == 'closed':
+            if self.status == "closed":
                 return
             if address not in self.workers:
-                return 'already-removed'
+                return "already-removed"
 
             address = self.coerce_address(address)
             host = get_address_host(address)
 
             ws = self.workers[address]
 
-            self.log_event(['all', address], {'action': 'remove-worker',
-                                              'worker': address,
-                                              'processing-tasks': dict(ws.processing)})
+            self.log_event(
+                ["all", address],
+                {
+                    "action": "remove-worker",
+                    "worker": address,
+                    "processing-tasks": dict(ws.processing),
+                },
+            )
             logger.info("Remove worker %s", address)
             if close:
                 with ignoring(AttributeError, CommClosedError):
-                    self.stream_comms[address].send({'op': 'close', 'report': False})
+                    self.stream_comms[address].send({"op": "close", "report": False})
 
             self.remove_resources(address)
 
-            self.host_info[host]['cores'] -= ws.ncores
-            self.host_info[host]['addresses'].remove(address)
+            self.host_info[host]["cores"] -= ws.ncores
+            self.host_info[host]["addresses"].remove(address)
             self.total_ncores -= ws.ncores
 
-            if not self.host_info[host]['addresses']:
+            if not self.host_info[host]["addresses"]:
                 del self.host_info[host]
 
             self.rpc.remove(address)
@@ -1738,29 +1860,29 @@ def remove_worker(self, comm=None, address=None, safe=False, close=True):
             self.idle.discard(ws)
             self.saturated.discard(ws)
             del self.workers[address]
-            ws.status = 'closed'
+            ws.status = "closed"
             self.total_occupancy -= ws.occupancy
 
             recommendations = OrderedDict()
 
             for ts in list(ws.processing):
                 k = ts.key
-                recommendations[k] = 'released'
+                recommendations[k] = "released"
                 if not safe:
                     ts.suspicious += 1
                     if ts.suspicious > self.allowed_failures:
                         del recommendations[k]
                         e = pickle.dumps(KilledWorker(k, address))
-                        r = self.transition(k, 'erred', exception=e, cause=k)
+                        r = self.transition(k, "erred", exception=e, cause=k)
                         recommendations.update(r)
 
             for ts in ws.has_what:
                 ts.who_has.remove(ws)
                 if not ts.who_has:
                     if ts.run_spec:
-                        recommendations[ts.key] = 'released'
+                        recommendations[ts.key] = "released"
                     else:  # pure data
-                        recommendations[ts.key] = 'forgotten'
+                        recommendations[ts.key] = "forgotten"
             ws.has_what.clear()
 
             self.transitions(recommendations)
@@ -1780,21 +1902,21 @@ def remove_worker_from_events():
                 if address not in self.workers and address in self.events:
                     del self.events[address]
 
-            cleanup_delay = parse_timedelta(dask.config.get('distributed.scheduler.events-cleanup-delay'))
-            self.loop.call_later(
-                cleanup_delay,
-                remove_worker_from_events
+            cleanup_delay = parse_timedelta(
+                dask.config.get("distributed.scheduler.events-cleanup-delay")
             )
+            self.loop.call_later(cleanup_delay, remove_worker_from_events)
             logger.debug("Removed worker %s", address)
 
-        return 'OK'
+        return "OK"
 
     def stimulus_cancel(self, comm, keys=None, client=None, force=False):
         """ Stop execution on a list of keys """
         logger.info("Client %s requests to cancel %d keys", client, len(keys))
         if client:
-            self.log_event(client, {'action': 'cancel', 'count': len(keys),
-                                    'force': force})
+            self.log_event(
+                client, {"action": "cancel", "count": len(keys), "force": force}
+            )
         for key in keys:
             self.cancel_key(key, client, force=force)
 
@@ -1805,14 +1927,15 @@ def cancel_key(self, key, client, retries=5, force=False):
         cs = self.clients[client]
         if ts is None or not ts.who_wants:  # no key yet, lets try again in a moment
             if retries:
-                self.loop.add_future(gen.sleep(0.2),
-                                     lambda _: self.cancel_key(key, client, retries - 1))
+                self.loop.add_future(
+                    gen.sleep(0.2), lambda _: self.cancel_key(key, client, retries - 1)
+                )
             return
         if force or ts.who_wants == {cs}:  # no one else wants this key
             for dts in list(ts.dependents):
                 self.cancel_key(dts.key, client, force=force)
         logger.info("Scheduler cancels key %s.  Force=%s", key, force)
-        self.report({'op': 'cancelled-key', 'key': key})
+        self.report({"op": "cancelled-key", "key": key})
         clients = list(ts.who_wants) if force else [cs]
         for c in clients:
             self.client_releases_keys(keys=[key], client=c.client_key)
@@ -1827,11 +1950,11 @@ def client_desires_keys(self, keys=None, client=None):
             if ts is None:
                 # For publish, queues etc.
                 ts = self.tasks[k] = TaskState(k, None)
-                ts.state = 'released'
+                ts.state = "released"
             ts.who_wants.add(cs)
             cs.wants_what.add(ts)
 
-            if ts.state in ('memory', 'erred'):
+            if ts.state in ("memory", "erred"):
                 self.report_on_key(k, client=client)
 
     def client_releases_keys(self, keys=None, client=None):
@@ -1852,9 +1975,9 @@ def client_releases_keys(self, keys=None, client=None):
         for ts in tasks2:
             if not ts.dependents:
                 # No live dependents, can forget
-                recommendations[ts.key] = 'forgotten'
-            elif ts.state != 'erred' and not ts.waiters:
-                recommendations[ts.key] = 'released'
+                recommendations[ts.key] = "forgotten"
+            elif ts.state != "erred" and not ts.waiters:
+                recommendations[ts.key] = "released"
 
         self.transitions(recommendations)
 
@@ -1868,13 +1991,12 @@ def client_heartbeat(self, client=None):
 
     def validate_released(self, key):
         ts = self.tasks[key]
-        assert ts.state == 'released'
+        assert ts.state == "released"
         assert not ts.waiters
         assert not ts.waiting_on
         assert not ts.who_has
         assert not ts.processing_on
-        assert not any(ts in dts.waiters
-                       for dts in ts.dependencies)
+        assert not any(ts in dts.waiters for dts in ts.dependencies)
         assert ts not in self.unrunnable
 
     def validate_waiting(self, key):
@@ -1906,7 +2028,7 @@ def validate_memory(self, key):
         assert not ts.waiting_on
         assert ts not in self.unrunnable
         for dts in ts.dependents:
-            assert (dts in ts.waiters) == (dts.state in ('waiting', 'processing'))
+            assert (dts in ts.waiters) == (dts.state in ("waiting", "processing"))
             assert ts not in dts.waiting_on
 
     def validate_no_worker(self, key):
@@ -1933,16 +2055,18 @@ def validate_key(self, key, ts=None):
             else:
                 ts.validate()
                 try:
-                    func = getattr(self, 'validate_' + ts.state.replace('-', '_'))
+                    func = getattr(self, "validate_" + ts.state.replace("-", "_"))
                 except AttributeError:
-                    logger.error("self.validate_%s not found",
-                                 ts.state.replace('-', '_'))
+                    logger.error(
+                        "self.validate_%s not found", ts.state.replace("-", "_")
+                    )
                 else:
                     func(key)
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
                 import pdb
+
                 pdb.set_trace()
             raise
 
@@ -1972,8 +2096,10 @@ def validate_state(self, allow_overlap=False):
             assert cs.client_key == c
 
         a = {w: ws.nbytes for w, ws in self.workers.items()}
-        b = {w: sum(ts.get_nbytes() for ts in ws.has_what)
-             for w, ws in self.workers.items()}
+        b = {
+            w: sum(ts.get_nbytes() for ts in ws.has_what)
+            for w, ws in self.workers.items()
+        }
         assert a == b, (a, b)
 
         actual_total_occupancy = 0
@@ -1981,8 +2107,10 @@ def validate_state(self, allow_overlap=False):
             assert abs(sum(ws.processing.values()) - ws.occupancy) < 1e-8
             actual_total_occupancy += ws.occupancy
 
-        assert abs(actual_total_occupancy - self.total_occupancy) < 1e-8, \
-            (actual_total_occupancy, self.total_occupancy)
+        assert abs(actual_total_occupancy - self.total_occupancy) < 1e-8, (
+            actual_total_occupancy,
+            self.total_occupancy,
+        )
 
     ###################
     # Manage Messages #
@@ -2000,27 +2128,29 @@ def report(self, msg, ts=None, client=None):
                 comm = self.client_comms[client]
                 comm.send(msg)
             except CommClosedError:
-                if self.status == 'running':
+                if self.status == "running":
                     logger.critical("Tried writing to closed comm: %s", msg)
             except KeyError:
                 pass
 
-        if ts is None and 'key' in msg:
-            ts = self.tasks.get(msg['key'])
+        if ts is None and "key" in msg:
+            ts = self.tasks.get(msg["key"])
         if ts is None:
             # Notify all clients
             comms = self.client_comms.values()
         else:
             # Notify clients interested in key
-            comms = [self.client_comms[c.client_key]
-                     for c in ts.who_wants
-                     if c.client_key in self.client_comms]
+            comms = [
+                self.client_comms[c.client_key]
+                for c in ts.who_wants
+                if c.client_key in self.client_comms
+            ]
         for c in comms:
             try:
                 c.send(msg)
                 # logger.debug("Scheduler sends message to client %s", msg)
             except CommClosedError:
-                if self.status == 'running':
+                if self.status == "running":
                     logger.critical("Tried writing to closed comm: %s", msg)
 
     @gen.coroutine
@@ -2031,46 +2161,45 @@ def add_client(self, comm, client=None):
         """
         assert client is not None
         logger.info("Receive client connection: %s", client)
-        self.log_event(['all', client], {'action': 'add-client',
-                                         'client': client})
+        self.log_event(["all", client], {"action": "add-client", "client": client})
         self.clients[client] = ClientState(client)
         try:
-            bcomm = BatchedSend(interval='2ms', loop=self.loop)
+            bcomm = BatchedSend(interval="2ms", loop=self.loop)
             bcomm.start(comm)
             self.client_comms[client] = bcomm
-            bcomm.send({'op': 'stream-start'})
+            bcomm.send({"op": "stream-start"})
 
             try:
-                yield self.handle_stream(comm=comm, extra={'client': client})
+                yield self.handle_stream(comm=comm, extra={"client": client})
             finally:
                 self.remove_client(client=client)
-                logger.debug('Finished handling client %s', client)
+                logger.debug("Finished handling client %s", client)
         finally:
             if not comm.closed():
-                self.client_comms[client].send({'op': 'stream-closed'})
+                self.client_comms[client].send({"op": "stream-closed"})
             try:
                 if not shutting_down():
                     yield self.client_comms[client].close()
                     del self.client_comms[client]
-                    if self.status == 'running':
+                    if self.status == "running":
                         logger.info("Close client connection: %s", client)
             except TypeError:  # comm becomes None during GC
                 pass
 
     def remove_client(self, client=None):
         """ Remove client from network """
-        if self.status == 'running':
+        if self.status == "running":
             logger.info("Remove client %s", client)
-        self.log_event(['all', client], {'action': 'remove-client',
-                                         'client': client})
+        self.log_event(["all", client], {"action": "remove-client", "client": client})
         try:
             cs = self.clients[client]
         except KeyError:
             # XXX is this a legitimate condition?
             pass
         else:
-            self.client_releases_keys(keys=[ts.key for ts in cs.wants_what],
-                                      client=cs.client_key)
+            self.client_releases_keys(
+                keys=[ts.key for ts in cs.wants_what], client=cs.client_key
+            )
             del self.clients[client]
 
         @gen.coroutine
@@ -2079,46 +2208,49 @@ def remove_client_from_events():
             if client not in self.clients and client in self.events:
                 del self.events[client]
 
-        cleanup_delay = parse_timedelta(dask.config.get('distributed.scheduler.events-cleanup-delay'))
-        self.loop.call_later(
-            cleanup_delay,
-            remove_client_from_events
+        cleanup_delay = parse_timedelta(
+            dask.config.get("distributed.scheduler.events-cleanup-delay")
         )
+        self.loop.call_later(cleanup_delay, remove_client_from_events)
 
     def send_task_to_worker(self, worker, key):
         """ Send a single computational task to a worker """
         try:
             ts = self.tasks[key]
 
-            msg = {'op': 'compute-task',
-                   'key': key,
-                   'priority': ts.priority,
-                   'duration': self.get_task_duration(ts)}
+            msg = {
+                "op": "compute-task",
+                "key": key,
+                "priority": ts.priority,
+                "duration": self.get_task_duration(ts),
+            }
             if ts.resource_restrictions:
-                msg['resource_restrictions'] = ts.resource_restrictions
+                msg["resource_restrictions"] = ts.resource_restrictions
             if ts.actor:
-                msg['actor'] = True
+                msg["actor"] = True
 
             deps = ts.dependencies
             if deps:
-                msg['who_has'] = {dep.key: [ws.address for ws in dep.who_has]
-                                  for dep in deps}
-                msg['nbytes'] = {dep.key: dep.nbytes for dep in deps}
+                msg["who_has"] = {
+                    dep.key: [ws.address for ws in dep.who_has] for dep in deps
+                }
+                msg["nbytes"] = {dep.key: dep.nbytes for dep in deps}
 
             if self.validate and deps:
-                assert all(msg['who_has'].values())
+                assert all(msg["who_has"].values())
 
             task = ts.run_spec
             if type(task) is dict:
                 msg.update(task)
             else:
-                msg['task'] = task
+                msg["task"] = task
 
             self.worker_send(worker, msg)
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
                 import pdb
+
                 pdb.set_trace()
             raise
 
@@ -2148,7 +2280,7 @@ def handle_release_data(self, key=None, worker=None, client=None, **msg):
 
     def handle_missing_data(self, key=None, errant_worker=None, **kwargs):
         logger.debug("handle missing data key=%s worker=%s", key, errant_worker)
-        self.log.append(('missing', key, errant_worker))
+        self.log.append(("missing", key, errant_worker))
 
         ts = self.tasks.get(key)
         if ts is None or not ts.who_has:
@@ -2161,9 +2293,9 @@ def handle_missing_data(self, key=None, errant_worker=None, **kwargs):
                 ws.nbytes -= ts.get_nbytes()
         if not ts.who_has:
             if ts.run_spec:
-                self.transitions({key: 'released'})
+                self.transitions({key: "released"})
             else:
-                self.transitions({key: 'forgotten'})
+                self.transitions({key: "forgotten"})
 
     def release_worker_data(self, stream=None, keys=None, worker=None):
         ws = self.workers[worker]
@@ -2177,7 +2309,7 @@ def release_worker_data(self, stream=None, keys=None, worker=None):
             wh = ts.who_has
             wh.remove(ws)
             if not wh:
-                recommendations[ts.key] = 'released'
+                recommendations[ts.key] = "released"
         if recommendations:
             self.transitions(recommendations)
 
@@ -2188,13 +2320,14 @@ def handle_long_running(self, key=None, worker=None, compute_duration=None):
         duration accounting as if the task has stopped.
         """
         ts = self.tasks[key]
-        if 'stealing' in self.extensions:
-            self.extensions['stealing'].remove_key_from_stealable(ts)
+        if "stealing" in self.extensions:
+            self.extensions["stealing"].remove_key_from_stealable(ts)
 
         ws = ts.processing_on
         if ws is None:
-            logger.debug("Received long-running signal from duplicate task. "
-                         "Ignoring.")
+            logger.debug(
+                "Received long-running signal from duplicate task. " "Ignoring."
+            )
             return
 
         if compute_duration:
@@ -2204,8 +2337,7 @@ def handle_long_running(self, key=None, worker=None, compute_duration=None):
             if not old_duration:
                 avg_duration = new_duration
             else:
-                avg_duration = (0.5 * old_duration
-                                + 0.5 * new_duration)
+                avg_duration = 0.5 * old_duration + 0.5 * new_duration
 
             self.task_duration[prefix] = avg_duration
 
@@ -2229,7 +2361,7 @@ def handle_worker(self, comm=None, worker=None):
         worker_comm.start(comm)
         logger.info("Starting worker compute stream, %s", worker)
         try:
-            yield self.handle_stream(comm=comm, extra={'worker': worker})
+            yield self.handle_stream(comm=comm, extra={"worker": worker})
         finally:
             if worker in self.stream_comms:
                 worker_comm.abort()
@@ -2269,8 +2401,15 @@ def worker_send(self, worker, msg):
     ############################
 
     @gen.coroutine
-    def scatter(self, comm=None, data=None, workers=None, client=None,
-                broadcast=False, timeout=2):
+    def scatter(
+        self,
+        comm=None,
+        data=None,
+        workers=None,
+        client=None,
+        broadcast=False,
+        timeout=2,
+    ):
         """ Send data out to workers
 
         See also
@@ -2291,9 +2430,9 @@ def scatter(self, comm=None, data=None, workers=None, client=None,
 
         assert isinstance(data, dict)
 
-        keys, who_has, nbytes = yield scatter_to_workers(ncores, data,
-                                                         rpc=self.rpc,
-                                                         report=False)
+        keys, who_has, nbytes = yield scatter_to_workers(
+            ncores, data, rpc=self.rpc, report=False
+        )
 
         self.update_data(who_has=who_has, nbytes=nbytes, client=client)
 
@@ -2304,9 +2443,9 @@ def scatter(self, comm=None, data=None, workers=None, client=None,
                 n = broadcast
             yield self.replicate(keys=keys, workers=workers, n=n)
 
-        self.log_event([client, 'all'], {'action': 'scatter',
-                                         'client': client,
-                                         'count': len(data)})
+        self.log_event(
+            [client, "all"], {"action": "scatter", "client": client, "count": len(data)}
+        )
         raise gen.Return(keys)
 
     @gen.coroutine
@@ -2322,16 +2461,22 @@ def gather(self, comm=None, keys=None, serializers=None):
                 who_has[key] = []
 
         data, missing_keys, missing_workers = yield gather_from_workers(
-            who_has, rpc=self.rpc, close=False, serializers=serializers)
+            who_has, rpc=self.rpc, close=False, serializers=serializers
+        )
         if not missing_keys:
-            result = {'status': 'OK', 'data': data}
+            result = {"status": "OK", "data": data}
         else:
-            missing_states = [(self.tasks[key].state
-                               if key in self.tasks else None)
-                              for key in missing_keys]
-            logger.debug("Couldn't gather keys %s state: %s workers: %s",
-                         missing_keys, missing_states, missing_workers)
-            result = {'status': 'error', 'keys': missing_keys}
+            missing_states = [
+                (self.tasks[key].state if key in self.tasks else None)
+                for key in missing_keys
+            ]
+            logger.debug(
+                "Couldn't gather keys %s state: %s workers: %s",
+                missing_keys,
+                missing_states,
+                missing_workers,
+            )
+            result = {"status": "error", "keys": missing_keys}
             with log_errors():
                 for worker in missing_workers:
                     self.remove_worker(address=worker)  # this is extreme
@@ -2339,18 +2484,20 @@ def gather(self, comm=None, keys=None, serializers=None):
                     if not workers:
                         continue
                     ts = self.tasks[key]
-                    logger.exception("Workers don't have promised key: %s, %s",
-                                     str(workers), str(key))
+                    logger.exception(
+                        "Workers don't have promised key: %s, %s",
+                        str(workers),
+                        str(key),
+                    )
                     for worker in workers:
                         ws = self.workers.get(worker)
                         if ws is not None and ts in ws.has_what:
                             ws.has_what.remove(ts)
                             ts.who_has.remove(ws)
                             ws.nbytes -= ts.get_nbytes()
-                            self.transitions({key: 'released'})
+                            self.transitions({key: "released"})
 
-        self.log_event('all', {'action': 'gather',
-                               'count': len(keys)})
+        self.log_event("all", {"action": "gather", "count": len(keys)})
         raise gen.Return(result)
 
     def clear_task_state(self):
@@ -2369,11 +2516,14 @@ def restart(self, client=None, timeout=3):
 
             logger.info("Send lost future signal to clients")
             for cs in self.clients.values():
-                self.client_releases_keys(keys=[ts.key for ts in cs.wants_what],
-                                          client=cs.client_key)
+                self.client_releases_keys(
+                    keys=[ts.key for ts in cs.wants_what], client=cs.client_key
+                )
 
-            nannies = {addr: self.get_worker_service_addr(addr, 'nanny', protocol=True)
-                       for addr in self.workers}
+            nannies = {
+                addr: self.get_worker_service_addr(addr, "nanny", protocol=True)
+                for addr in self.workers
+            }
 
             for addr in list(self.workers):
                 try:
@@ -2381,8 +2531,9 @@ def restart(self, client=None, timeout=3):
                     # otherwise the nanny will kill it anyway
                     self.remove_worker(address=addr, close=addr not in nannies)
                 except Exception as e:
-                    logger.info("Exception while restarting.  This is normal",
-                                exc_info=True)
+                    logger.info(
+                        "Exception while restarting.  This is normal", exc_info=True
+                    )
 
             self.clear_task_state()
 
@@ -2394,38 +2545,54 @@ def restart(self, client=None, timeout=3):
 
             logger.debug("Send kill signal to nannies: %s", nannies)
 
-            nannies = [rpc(nanny_address, connection_args=self.connection_args)
-                       for nanny_address in nannies.values()
-                       if nanny_address is not None]
+            nannies = [
+                rpc(nanny_address, connection_args=self.connection_args)
+                for nanny_address in nannies.values()
+                if nanny_address is not None
+            ]
 
             try:
-                resps = All([nanny.restart(close=True, timeout=timeout * 0.8,
-                                           executor_wait=False)
-                             for nanny in nannies])
+                resps = All(
+                    [
+                        nanny.restart(
+                            close=True, timeout=timeout * 0.8, executor_wait=False
+                        )
+                        for nanny in nannies
+                    ]
+                )
                 resps = yield gen.with_timeout(timedelta(seconds=timeout), resps)
-                if not all(resp == 'OK' for resp in resps):
-                    logger.error("Not all workers responded positively: %s",
-                                 resps, exc_info=True)
+                if not all(resp == "OK" for resp in resps):
+                    logger.error(
+                        "Not all workers responded positively: %s", resps, exc_info=True
+                    )
             except gen.TimeoutError:
-                logger.error("Nannies didn't report back restarted within "
-                             "timeout.  Continuuing with restart process")
+                logger.error(
+                    "Nannies didn't report back restarted within "
+                    "timeout.  Continuuing with restart process"
+                )
             finally:
                 for nanny in nannies:
                     nanny.close_rpc()
 
             self.start()
 
-            self.log_event([client, 'all'], {'action': 'restart',
-                                             'client': client})
+            self.log_event([client, "all"], {"action": "restart", "client": client})
             start = time()
             while time() < start + 10 and len(self.workers) < n_workers:
                 yield gen.sleep(0.01)
 
-            self.report({'op': 'restart'})
+            self.report({"op": "restart"})
 
     @gen.coroutine
-    def broadcast(self, comm=None, msg=None, workers=None, hosts=None,
-                  nanny=False, serializers=None):
+    def broadcast(
+        self,
+        comm=None,
+        msg=None,
+        workers=None,
+        hosts=None,
+        nanny=False,
+        serializers=None,
+    ):
         """ Broadcast message to workers, return all results """
         if workers is None:
             if hosts is None:
@@ -2435,33 +2602,36 @@ def broadcast(self, comm=None, msg=None, workers=None, hosts=None,
         if hosts is not None:
             for host in hosts:
                 if host in self.host_info:
-                    workers.extend(self.host_info[host]['addresses'])
+                    workers.extend(self.host_info[host]["addresses"])
         # TODO replace with worker_list
 
         if nanny:
-            addresses = [self.get_worker_service_addr(w, 'nanny', protocol=True)
-                         for w in workers]
+            addresses = [
+                self.get_worker_service_addr(w, "nanny", protocol=True) for w in workers
+            ]
         else:
             addresses = workers
 
         @gen.coroutine
         def send_message(addr):
-            comm = yield connect(addr, deserialize=self.deserialize,
-                                 connection_args=self.connection_args)
+            comm = yield connect(
+                addr, deserialize=self.deserialize, connection_args=self.connection_args
+            )
             resp = yield send_recv(comm, close=True, serializers=serializers, **msg)
             raise gen.Return(resp)
 
-        results = yield All([send_message(address)
-                             for address in addresses
-                             if address is not None])
+        results = yield All(
+            [send_message(address) for address in addresses if address is not None]
+        )
 
         raise Return(dict(zip(workers, results)))
 
     @gen.coroutine
     def proxy(self, comm=None, msg=None, worker=None, serializers=None):
         """ Proxy a communication through the scheduler to some other worker """
-        d = yield self.broadcast(comm=comm, msg=msg, workers=[worker],
-                                 serializers=serializers)
+        d = yield self.broadcast(
+            comm=comm, msg=msg, workers=[worker], serializers=serializers
+        )
         raise gen.Return(d[worker])
 
     @gen.coroutine
@@ -2481,8 +2651,7 @@ def rebalance(self, comm=None, keys=None, workers=None):
                 tasks = {self.tasks[k] for k in keys}
                 missing_data = [ts.key for ts in tasks if not ts.who_has]
                 if missing_data:
-                    raise Return({'status': 'missing-data',
-                                  'keys': missing_data})
+                    raise Return({"status": "missing-data", "keys": missing_data})
             else:
                 tasks = set(self.tasks.values())
 
@@ -2499,27 +2668,31 @@ def rebalance(self, comm=None, keys=None, workers=None):
                 for vv in v:
                     tasks_by_worker[vv].add(k)
 
-            worker_bytes = {ws: sum(ts.get_nbytes() for ts in v)
-                            for ws, v in tasks_by_worker.items()}
+            worker_bytes = {
+                ws: sum(ts.get_nbytes() for ts in v)
+                for ws, v in tasks_by_worker.items()
+            }
 
             avg = sum(worker_bytes.values()) / len(worker_bytes)
 
-            sorted_workers = list(map(first, sorted(worker_bytes.items(),
-                                                    key=second, reverse=True)))
+            sorted_workers = list(
+                map(first, sorted(worker_bytes.items(), key=second, reverse=True))
+            )
 
             recipients = iter(reversed(sorted_workers))
             recipient = next(recipients)
             msgs = []  # (sender, recipient, key)
-            for sender in sorted_workers[:len(workers) // 2]:
-                sender_keys = {ts: ts.get_nbytes()
-                               for ts in tasks_by_worker[sender]}
-                sender_keys = iter(sorted(sender_keys.items(),
-                                          key=second, reverse=True))
+            for sender in sorted_workers[: len(workers) // 2]:
+                sender_keys = {ts: ts.get_nbytes() for ts in tasks_by_worker[sender]}
+                sender_keys = iter(
+                    sorted(sender_keys.items(), key=second, reverse=True)
+                )
 
                 try:
                     while worker_bytes[sender] > avg:
-                        while (worker_bytes[recipient] < avg and
-                               worker_bytes[sender] > avg):
+                        while (
+                            worker_bytes[recipient] < avg and worker_bytes[sender] > avg
+                        ):
                             ts, nb = next(sender_keys)
                             if ts not in tasks_by_worker[recipient]:
                                 tasks_by_worker[recipient].add(ts)
@@ -2538,44 +2711,62 @@ def rebalance(self, comm=None, keys=None, workers=None):
                 to_recipients[recipient.address][ts.key].append(sender.address)
                 to_senders[sender.address].append(ts.key)
 
-            result = yield {r: self.rpc(addr=r).gather(who_has=v)
-                            for r, v in to_recipients.items()}
+            result = yield {
+                r: self.rpc(addr=r).gather(who_has=v) for r, v in to_recipients.items()
+            }
             for r, v in to_recipients.items():
-                self.log_event(r, {'action': 'rebalance',
-                                   'who_has': v})
-
-            self.log_event('all', {'action': 'rebalance',
-                                   'total-keys': len(tasks),
-                                   'senders': valmap(len, to_senders),
-                                   'recipients': valmap(len, to_recipients),
-                                   'moved_keys': len(msgs)})
+                self.log_event(r, {"action": "rebalance", "who_has": v})
+
+            self.log_event(
+                "all",
+                {
+                    "action": "rebalance",
+                    "total-keys": len(tasks),
+                    "senders": valmap(len, to_senders),
+                    "recipients": valmap(len, to_recipients),
+                    "moved_keys": len(msgs),
+                },
+            )
 
-            if not all(r['status'] == 'OK' for r in result.values()):
-                raise Return({'status': 'missing-data',
-                              'keys': sum([r['keys'] for r in result
-                                           if 'keys' in r], [])})
+            if not all(r["status"] == "OK" for r in result.values()):
+                raise Return(
+                    {
+                        "status": "missing-data",
+                        "keys": sum([r["keys"] for r in result if "keys" in r], []),
+                    }
+                )
 
             for sender, recipient, ts in msgs:
-                assert ts.state == 'memory'
+                assert ts.state == "memory"
                 ts.who_has.add(recipient)
                 recipient.has_what.add(ts)
                 recipient.nbytes += ts.get_nbytes()
-                self.log.append(('rebalance', ts.key, time(),
-                                 sender.address, recipient.address))
+                self.log.append(
+                    ("rebalance", ts.key, time(), sender.address, recipient.address)
+                )
 
-            result = yield {r: self.rpc(addr=r).delete_data(keys=v, report=False)
-                            for r, v in to_senders.items()}
+            result = yield {
+                r: self.rpc(addr=r).delete_data(keys=v, report=False)
+                for r, v in to_senders.items()
+            }
 
             for sender, recipient, ts in msgs:
                 ts.who_has.remove(sender)
                 sender.has_what.remove(ts)
                 sender.nbytes -= ts.get_nbytes()
 
-            raise Return({'status': 'OK'})
+            raise Return({"status": "OK"})
 
     @gen.coroutine
-    def replicate(self, comm=None, keys=None, n=None, workers=None,
-                  branching_factor=2, delete=True):
+    def replicate(
+        self,
+        comm=None,
+        keys=None,
+        n=None,
+        workers=None,
+        branching_factor=2,
+        delete=True,
+    ):
         """ Replicate data throughout cluster
 
         This performs a tree copy of the data throughout the network
@@ -2610,8 +2801,7 @@ def replicate(self, comm=None, keys=None, n=None, workers=None,
         tasks = {self.tasks[k] for k in keys}
         missing_data = [ts.key for ts in tasks if not ts.who_has]
         if missing_data:
-            raise Return({'status': 'missing-data',
-                          'keys': missing_data})
+            raise Return({"status": "missing-data", "keys": missing_data})
 
         # Delete extraneous data
         if delete:
@@ -2619,22 +2809,25 @@ def replicate(self, comm=None, keys=None, n=None, workers=None,
             for ts in tasks:
                 del_candidates = ts.who_has & workers
                 if len(del_candidates) > n:
-                    for ws in random.sample(del_candidates,
-                                            len(del_candidates) - n):
+                    for ws in random.sample(del_candidates, len(del_candidates) - n):
                         del_worker_tasks[ws].add(ts)
 
-            yield [self.rpc(addr=ws.address)
-                       .delete_data(keys=[ts.key for ts in tasks], report=False)
-                   for ws, tasks in del_worker_tasks.items()]
+            yield [
+                self.rpc(addr=ws.address).delete_data(
+                    keys=[ts.key for ts in tasks], report=False
+                )
+                for ws, tasks in del_worker_tasks.items()
+            ]
 
             for ws, tasks in del_worker_tasks.items():
                 ws.has_what -= tasks
                 for ts in tasks:
                     ts.who_has.remove(ws)
                     ws.nbytes -= ts.get_nbytes()
-                self.log_event(ws.address,
-                               {'action': 'replicate-remove',
-                                'keys': [ts.key for ts in tasks]})
+                self.log_event(
+                    ws.address,
+                    {"action": "replicate-remove", "keys": [ts.key for ts in tasks]},
+                )
 
         # Copy not-yet-filled data
         while tasks:
@@ -2646,33 +2839,35 @@ def replicate(self, comm=None, keys=None, n=None, workers=None,
                     tasks.remove(ts)
                     continue
 
-                count = min(n_missing,
-                            branching_factor * len(ts.who_has))
+                count = min(n_missing, branching_factor * len(ts.who_has))
                 assert count > 0
 
                 for ws in random.sample(workers - ts.who_has, count):
-                    gathers[ws.address][ts.key] = [wws.address
-                                                      for wws in ts.who_has]
+                    gathers[ws.address][ts.key] = [wws.address for wws in ts.who_has]
 
-            results = yield {w: self.rpc(addr=w).gather(who_has=who_has)
-                             for w, who_has in gathers.items()}
+            results = yield {
+                w: self.rpc(addr=w).gather(who_has=who_has)
+                for w, who_has in gathers.items()
+            }
             for w, v in results.items():
-                if v['status'] == 'OK':
+                if v["status"] == "OK":
                     self.add_keys(worker=w, keys=list(gathers[w]))
                 else:
-                    logger.warning("Communication failed during replication: %s",
-                                   v)
-
-                self.log_event(w, {'action': 'replicate-add',
-                                   'keys': gathers[w]})
-
-        self.log_event('all', {'action': 'replicate',
-                               'workers': list(workers),
-                               'key-count': len(keys),
-                               'branching-factor': branching_factor})
+                    logger.warning("Communication failed during replication: %s", v)
+
+                self.log_event(w, {"action": "replicate-add", "keys": gathers[w]})
+
+        self.log_event(
+            "all",
+            {
+                "action": "replicate",
+                "workers": list(workers),
+                "key-count": len(keys),
+                "branching-factor": branching_factor,
+            },
+        )
 
-    def workers_to_close(self, memory_ratio=None, n=None, key=None,
-                         minimum=None):
+    def workers_to_close(self, memory_ratio=None, n=None, key=None, minimum=None):
         """
         Find workers that we can close with low cost
 
@@ -2737,10 +2932,10 @@ def workers_to_close(self, memory_ratio=None, n=None, key=None,
 
             groups = groupby(key, self.workers.values())
 
-            limit_bytes = {k: sum(ws.memory_limit for ws in v)
-                           for k, v in groups.items()}
-            group_bytes = {k: sum(ws.nbytes for ws in v)
-                           for k, v in groups.items()}
+            limit_bytes = {
+                k: sum(ws.memory_limit for ws in v) for k, v in groups.items()
+            }
+            group_bytes = {k: sum(ws.nbytes for ws in v) for k, v in groups.items()}
 
             limit = sum(limit_bytes.values())
             total = sum(group_bytes.values())
@@ -2765,8 +2960,9 @@ def key(group):
 
                 limit -= limit_bytes[group]
 
-                if ((n is not None and len(to_close) < n) or
-                    (memory_ratio is not None and limit >= memory_ratio * total)):
+                if (n is not None and len(to_close) < n) or (
+                    memory_ratio is not None and limit >= memory_ratio * total
+                ):
                     to_close.append(group)
                     n_remain -= len(groups[group])
 
@@ -2780,8 +2976,9 @@ def key(group):
             return result
 
     @gen.coroutine
-    def retire_workers(self, comm=None, workers=None, remove=True,
-                       close_workers=False, **kwargs):
+    def retire_workers(
+        self, comm=None, workers=None, remove=True, close_workers=False, **kwargs
+    ):
         """ Gracefully retire workers from cluster
 
         Parameters
@@ -2815,9 +3012,11 @@ def retire_workers(self, comm=None, workers=None, remove=True,
                     try:
                         workers = self.workers_to_close(**kwargs)
                         if workers:
-                            workers = yield self.retire_workers(workers=workers,
-                                                                remove=remove,
-                                                                close_workers=close_workers)
+                            workers = yield self.retire_workers(
+                                workers=workers,
+                                remove=remove,
+                                close_workers=close_workers,
+                            )
                         raise gen.Return(workers)
                     except KeyError:  # keys left during replicate
                         pass
@@ -2833,24 +3032,31 @@ def retire_workers(self, comm=None, workers=None, remove=True,
             other_workers = set(self.workers.values()) - workers
             if keys:
                 if other_workers:
-                    yield self.replicate(keys=keys,
-                                         workers=[ws.address for ws in other_workers],
-                                         n=1, delete=False)
+                    yield self.replicate(
+                        keys=keys,
+                        workers=[ws.address for ws in other_workers],
+                        n=1,
+                        delete=False,
+                    )
                 else:
                     raise gen.Return([])
 
             worker_keys = {ws.address: ws.identity() for ws in workers}
             if close_workers and worker_keys:
-                yield [self.close_worker(worker=w, safe=True)
-                       for w in worker_keys]
+                yield [self.close_worker(worker=w, safe=True) for w in worker_keys]
             if remove:
                 for w in worker_keys:
                     self.remove_worker(address=w, safe=True)
 
-            self.log_event('all', {'action': 'retire-workers',
-                                   'workers': worker_keys,
-                                   'moved-keys': len(keys)})
-            self.log_event(list(worker_keys), {'action': 'retired'})
+            self.log_event(
+                "all",
+                {
+                    "action": "retire-workers",
+                    "workers": worker_keys,
+                    "moved-keys": len(keys),
+                },
+            )
+            self.log_event(list(worker_keys), {"action": "retired"})
 
             raise gen.Return(worker_keys)
 
@@ -2862,24 +3068,25 @@ def add_keys(self, comm=None, worker=None, keys=()):
         reasons.  However, it is sent by workers from time to time.
         """
         if worker not in self.workers:
-            return 'not found'
+            return "not found"
         ws = self.workers[worker]
         for key in keys:
             ts = self.tasks.get(key)
-            if ts is not None and ts.state == 'memory':
+            if ts is not None and ts.state == "memory":
                 if ts not in ws.has_what:
                     ws.nbytes += ts.get_nbytes()
                     ws.has_what.add(ts)
                     ts.who_has.add(ws)
             else:
-                self.worker_send(worker, {'op': 'delete-data',
-                                          'keys': [key],
-                                          'report': False})
+                self.worker_send(
+                    worker, {"op": "delete-data", "keys": [key], "report": False}
+                )
 
-        return 'OK'
+        return "OK"
 
-    def update_data(self, comm=None, who_has=None, nbytes=None, client=None,
-                    serializers=None):
+    def update_data(
+        self, comm=None, who_has=None, nbytes=None, client=None, serializers=None
+    ):
         """
         Learn that new data has entered the network from an external source
 
@@ -2888,15 +3095,16 @@ def update_data(self, comm=None, who_has=None, nbytes=None, client=None,
         Scheduler.mark_key_in_memory
         """
         with log_errors():
-            who_has = {k: [self.coerce_address(vv) for vv in v]
-                       for k, v in who_has.items()}
+            who_has = {
+                k: [self.coerce_address(vv) for vv in v] for k, v in who_has.items()
+            }
             logger.debug("Update data %s", who_has)
 
             for key, workers in who_has.items():
                 ts = self.tasks.get(key)
                 if ts is None:
                     ts = self.tasks[key] = TaskState(key, None)
-                ts.state = 'memory'
+                ts.state = "memory"
                 if key in nbytes:
                     ts.set_nbytes(nbytes[key])
                 for w in workers:
@@ -2905,9 +3113,9 @@ def update_data(self, comm=None, who_has=None, nbytes=None, client=None,
                         ws.nbytes += ts.get_nbytes()
                         ws.has_what.add(ts)
                         ts.who_has.add(ws)
-                self.report({'op': 'key-in-memory',
-                             'key': key,
-                             'workers': list(workers)})
+                self.report(
+                    {"op": "key-in-memory", "key": key, "workers": list(workers)}
+                )
 
             if client:
                 self.client_desires_keys(keys=list(who_has), client=client)
@@ -2918,29 +3126,31 @@ def report_on_key(self, key=None, ts=None, client=None):
             try:
                 ts = self.tasks[key]
             except KeyError:
-                self.report({'op': 'cancelled-key',
-                             'key': key},
-                            client=client)
+                self.report({"op": "cancelled-key", "key": key}, client=client)
                 return
         else:
             key = ts.key
-        if ts.state == 'forgotten':
-            self.report({'op': 'cancelled-key',
-                         'key': key}, ts=ts, client=client)
-        elif ts.state == 'memory':
-            self.report({'op': 'key-in-memory',
-                         'key': key}, ts=ts, client=client)
-        elif ts.state == 'erred':
+        if ts.state == "forgotten":
+            self.report({"op": "cancelled-key", "key": key}, ts=ts, client=client)
+        elif ts.state == "memory":
+            self.report({"op": "key-in-memory", "key": key}, ts=ts, client=client)
+        elif ts.state == "erred":
             failing_ts = ts.exception_blame
-            self.report({'op': 'task-erred',
-                         'key': key,
-                         'exception': failing_ts.exception,
-                         'traceback': failing_ts.traceback},
-                        ts=ts, client=client)
+            self.report(
+                {
+                    "op": "task-erred",
+                    "key": key,
+                    "exception": failing_ts.exception,
+                    "traceback": failing_ts.traceback,
+                },
+                ts=ts,
+                client=client,
+            )
 
     @gen.coroutine
-    def feed(self, comm, function=None, setup=None, teardown=None,
-             interval='1s', **kwargs):
+    def feed(
+        self, comm, function=None, setup=None, teardown=None, interval="1s", **kwargs
+    ):
         """
         Provides a data Comm to external requester
 
@@ -2948,6 +3158,7 @@ def feed(self, comm, function=None, setup=None, teardown=None,
         eventually be phased out.  It is mostly used by diagnostics.
         """
         import pickle
+
         interval = parse_timedelta(interval)
         with log_errors():
             if function:
@@ -2960,7 +3171,7 @@ def feed(self, comm, function=None, setup=None, teardown=None,
             if isinstance(state, gen.Future):
                 state = yield state
             try:
-                while self.status == 'running':
+                while self.status == "running":
                     if state is None:
                         response = function(self)
                     else:
@@ -2976,36 +3187,41 @@ def feed(self, comm, function=None, setup=None, teardown=None,
     def get_processing(self, comm=None, workers=None):
         if workers is not None:
             workers = set(map(self.coerce_address, workers))
-            return {w: [ts.key for ts in self.workers[w].processing]
-                    for w in workers}
+            return {w: [ts.key for ts in self.workers[w].processing] for w in workers}
         else:
-            return {w: [ts.key for ts in ws.processing]
-                    for w, ws in self.workers.items()}
+            return {
+                w: [ts.key for ts in ws.processing] for w, ws in self.workers.items()
+            }
 
     def get_who_has(self, comm=None, keys=None):
         if keys is not None:
-            return {k: [ws.address for ws in self.tasks[k].who_has]
-                       if k in self.tasks else []
-                    for k in keys}
+            return {
+                k: [ws.address for ws in self.tasks[k].who_has]
+                if k in self.tasks
+                else []
+                for k in keys
+            }
         else:
-            return {key: [ws.address for ws in ts.who_has]
-                    for key, ts in self.tasks.items()}
+            return {
+                key: [ws.address for ws in ts.who_has] for key, ts in self.tasks.items()
+            }
 
     def get_has_what(self, comm=None, workers=None):
         if workers is not None:
             workers = map(self.coerce_address, workers)
-            return {w: [ts.key for ts in self.workers[w].has_what]
-                       if w in self.workers else []
-                    for w in workers}
+            return {
+                w: [ts.key for ts in self.workers[w].has_what]
+                if w in self.workers
+                else []
+                for w in workers
+            }
         else:
-            return {w: [ts.key for ts in ws.has_what]
-                    for w, ws in self.workers.items()}
+            return {w: [ts.key for ts in ws.has_what] for w, ws in self.workers.items()}
 
     def get_ncores(self, comm=None, workers=None):
         if workers is not None:
             workers = map(self.coerce_address, workers)
-            return {w: self.workers[w].ncores
-                    for w in workers if w in self.workers}
+            return {w: self.workers[w].ncores for w in workers if w in self.workers}
         else:
             return {w: ws.ncores for w, ws in self.workers.items()}
 
@@ -3017,9 +3233,9 @@ def get_call_stack(self, comm=None, keys=None):
             while stack:
                 key = stack.pop()
                 ts = self.tasks[key]
-                if ts.state == 'waiting':
+                if ts.state == "waiting":
                     stack.extend(dts.key for dts in ts.dependencies)
-                elif ts.state == 'processing':
+                elif ts.state == "processing":
                     processing.add(ts)
 
             workers = defaultdict(list)
@@ -3033,8 +3249,9 @@ def get_call_stack(self, comm=None, keys=None):
             raise gen.Return({})
 
         else:
-            response = yield {w: self.rpc(w).call_stack(keys=v)
-                              for w, v in workers.items()}
+            response = yield {
+                w: self.rpc(w).call_stack(keys=v) for w, v in workers.items()
+            }
             response = {k: v for k, v in response.items() if v}
             raise gen.Return(response)
 
@@ -3043,8 +3260,11 @@ def get_nbytes(self, comm=None, keys=None, summary=True):
             if keys is not None:
                 result = {k: self.tasks[k].nbytes for k in keys}
             else:
-                result = {k: ts.nbytes for k, ts in self.tasks.items()
-                          if ts.nbytes is not None}
+                result = {
+                    k: ts.nbytes
+                    for k, ts in self.tasks.items()
+                    if ts.nbytes is not None
+                }
 
             if summary:
                 out = defaultdict(lambda: 0)
@@ -3059,9 +3279,7 @@ def get_comm_cost(self, ts, ws):
         Get the estimated communication cost (in s.) to compute the task
         on the given worker.
         """
-        return (sum(dts.nbytes
-                    for dts in ts.dependencies - ws.has_what)
-                / BANDWIDTH)
+        return sum(dts.nbytes for dts in ts.dependencies - ws.has_what) / BANDWIDTH
 
     def get_task_duration(self, ts, default=0.5):
         """
@@ -3083,7 +3301,8 @@ def run_function(self, stream, function, args=(), kwargs={}, wait=True):
         Client.run_on_scheduler:
         """
         from .worker import run
-        self.log_event('all', {'action': 'run-function', 'function': function})
+
+        self.log_event("all", {"action": "run-function", "function": function})
         return run(self, stream, function=function, args=args, kwargs=kwargs, wait=wait)
 
     def set_metadata(self, stream=None, keys=None, value=None):
@@ -3095,7 +3314,9 @@ def set_metadata(self, stream=None, keys=None, value=None):
                 metadata = metadata[key]
             metadata[keys[-1]] = value
         except Exception as e:
-            import pdb; pdb.set_trace()
+            import pdb
+
+            pdb.set_trace()
 
     def get_metadata(self, stream=None, keys=None, default=no_default):
         metadata = self.task_metadata
@@ -3110,12 +3331,13 @@ def get_metadata(self, stream=None, keys=None, default=no_default):
                 raise
 
     def get_task_status(self, stream=None, keys=None):
-        return {key: (self.tasks[key].state
-                      if key in self.tasks else None)
-                for key in keys}
+        return {
+            key: (self.tasks[key].state if key in self.tasks else None) for key in keys
+        }
 
     def get_task_stream(self, comm=None, start=None, stop=None, count=None):
         from distributed.diagnostics.task_stream import TaskStreamPlugin
+
         self.add_plugin(TaskStreamPlugin, idempotent=True)
         ts = [p for p in self.plugins if isinstance(p, TaskStreamPlugin)][0]
         return ts.collect(start=start, stop=stop, count=count)
@@ -3128,7 +3350,7 @@ def register_worker_callbacks(self, comm, setup=None):
 
         self.worker_setups.append(setup)
 
-        responses = yield self.broadcast(msg=dict(op='run', function=setup))
+        responses = yield self.broadcast(msg=dict(op="run", function=setup))
         raise gen.Return(responses)
 
     #####################
@@ -3168,35 +3390,33 @@ def _add_to_memory(self, ts, ws, recommendations, type=None, **kwargs):
 
         deps = ts.dependents
         if len(deps) > 1:
-            deps = sorted(deps, key=operator.attrgetter('priority'),
-                          reverse=True)
+            deps = sorted(deps, key=operator.attrgetter("priority"), reverse=True)
         for dts in deps:
             s = dts.waiting_on
             if ts in s:
                 s.discard(ts)
                 if not s:  # new task ready to run
-                    recommendations[dts.key] = 'processing'
+                    recommendations[dts.key] = "processing"
 
         for dts in ts.dependencies:
             s = dts.waiters
             s.discard(ts)
             if not s and not dts.who_wants:
-                recommendations[dts.key] = 'released'
+                recommendations[dts.key] = "released"
 
         if not ts.waiters and not ts.who_wants:
-            recommendations[ts.key] = 'released'
+            recommendations[ts.key] = "released"
         else:
-            msg = {'op': 'key-in-memory',
-                   'key': ts.key}
+            msg = {"op": "key-in-memory", "key": ts.key}
             if type is not None:
-                msg['type'] = type
+                msg["type"] = type
             self.report(msg)
 
-        ts.state = 'memory'
+        ts.state = "memory"
 
-        cs = self.clients['fire-and-forget']
+        cs = self.clients["fire-and-forget"]
         if ts in cs.wants_what:
-            self.client_releases_keys(client='fire-and-forget', keys=[ts.key])
+            self.client_releases_keys(client="fire-and-forget", keys=[ts.key])
 
     def transition_released_waiting(self, key):
         try:
@@ -3207,45 +3427,45 @@ def transition_released_waiting(self, key):
                 assert not ts.waiting_on
                 assert not ts.who_has
                 assert not ts.processing_on
-                assert not any(dts.state == 'forgotten' for dts in ts.dependencies)
+                assert not any(dts.state == "forgotten" for dts in ts.dependencies)
 
             if ts.has_lost_dependencies:
-                return {key: 'forgotten'}
+                return {key: "forgotten"}
 
-            ts.state = 'waiting'
+            ts.state = "waiting"
 
             recommendations = OrderedDict()
 
             for dts in ts.dependencies:
                 if dts.exception_blame:
                     ts.exception_blame = dts.exception_blame
-                    recommendations[key] = 'erred'
+                    recommendations[key] = "erred"
                     return recommendations
 
             for dts in ts.dependencies:
                 dep = dts.key
                 if not dts.who_has:
                     ts.waiting_on.add(dts)
-                if dts.state == 'released':
-                    recommendations[dep] = 'waiting'
+                if dts.state == "released":
+                    recommendations[dep] = "waiting"
                 else:
                     dts.waiters.add(ts)
 
-            ts.waiters = {dts for dts in ts.dependents
-                          if dts.state == 'waiting'}
+            ts.waiters = {dts for dts in ts.dependents if dts.state == "waiting"}
 
             if not ts.waiting_on:
                 if self.workers:
-                    recommendations[key] = 'processing'
+                    recommendations[key] = "processing"
                 else:
                     self.unrunnable.add(ts)
-                    ts.state = 'no-worker'
+                    ts.state = "no-worker"
 
             return recommendations
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
                 import pdb
+
                 pdb.set_trace()
             raise
 
@@ -3262,7 +3482,7 @@ def transition_no_worker_waiting(self, key):
             self.unrunnable.remove(ts)
 
             if ts.has_lost_dependencies:
-                return {key: 'forgotten'}
+                return {key: "forgotten"}
 
             recommendations = OrderedDict()
 
@@ -3270,25 +3490,26 @@ def transition_no_worker_waiting(self, key):
                 dep = dts.key
                 if not dts.who_has:
                     ts.waiting_on.add(dep)
-                if dts.state == 'released':
-                    recommendations[dep] = 'waiting'
+                if dts.state == "released":
+                    recommendations[dep] = "waiting"
                 else:
                     dts.waiters.add(ts)
 
-            ts.state = 'waiting'
+            ts.state = "waiting"
 
             if not ts.waiting_on:
                 if self.workers:
-                    recommendations[key] = 'processing'
+                    recommendations[key] = "processing"
                 else:
                     self.unrunnable.add(ts)
-                    ts.state = 'no-worker'
+                    ts.state = "no-worker"
 
             return recommendations
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
                 import pdb
+
                 pdb.set_trace()
             raise
 
@@ -3300,27 +3521,34 @@ def decide_worker(self, ts):
 
         if not valid_workers and not ts.loose_restrictions and self.workers:
             self.unrunnable.add(ts)
-            ts.state = 'no-worker'
+            ts.state = "no-worker"
             return None
 
         if ts.dependencies or valid_workers is not True:
-            worker = decide_worker(ts, self.workers.values(), valid_workers,
-                                   partial(self.worker_objective, ts))
+            worker = decide_worker(
+                ts,
+                self.workers.values(),
+                valid_workers,
+                partial(self.worker_objective, ts),
+            )
         elif self.idle:
             if len(self.idle) < 20:  # smart but linear in small case
-                worker = min(self.idle,
-                             key=operator.attrgetter('occupancy'))
+                worker = min(self.idle, key=operator.attrgetter("occupancy"))
             else:  # dumb but fast in large case
                 worker = self.idle[self.n_tasks % len(self.idle)]
         else:
             if len(self.workers) < 20:  # smart but linear in small case
-                worker = min(self.workers.values(),
-                             key=operator.attrgetter('occupancy'))
+                worker = min(
+                    self.workers.values(), key=operator.attrgetter("occupancy")
+                )
             else:  # dumb but fast in large case
                 worker = self.workers.values()[self.n_tasks % len(self.workers)]
 
         if self.validate:
-            assert worker is None or isinstance(worker, WorkerState), (type(worker), worker)
+            assert worker is None or isinstance(worker, WorkerState), (
+                type(worker),
+                worker,
+            )
             assert worker.address in self.workers
 
         return worker
@@ -3336,8 +3564,7 @@ def transition_waiting_processing(self, key):
                 assert not ts.processing_on
                 assert not ts.has_lost_dependencies
                 assert ts not in self.unrunnable
-                assert all(dts.who_has
-                           for dts in ts.dependencies)
+                assert all(dts.who_has for dts in ts.dependencies)
 
             ws = self.decide_worker(ts)
             if ws is None:
@@ -3351,7 +3578,7 @@ def transition_waiting_processing(self, key):
             ts.processing_on = ws
             ws.occupancy += duration + comm
             self.total_occupancy += duration + comm
-            ts.state = 'processing'
+            ts.state = "processing"
             self.consume_resources(ts, ws)
             self.check_idle_saturated(ws)
             self.n_tasks += 1
@@ -3368,6 +3595,7 @@ def transition_waiting_processing(self, key):
             logger.exception(e)
             if LOG_PDB:
                 import pdb
+
                 pdb.set_trace()
             raise
 
@@ -3379,7 +3607,7 @@ def transition_waiting_memory(self, key, nbytes=None, worker=None, **kwargs):
             if self.validate:
                 assert not ts.processing_on
                 assert ts.waiting_on
-                assert ts.state == 'waiting'
+                assert ts.state == "waiting"
 
             ts.waiting_on.clear()
 
@@ -3402,11 +3630,13 @@ def transition_waiting_memory(self, key, nbytes=None, worker=None, **kwargs):
             logger.exception(e)
             if LOG_PDB:
                 import pdb
+
                 pdb.set_trace()
             raise
 
-    def transition_processing_memory(self, key, nbytes=None, type=None,
-                                     worker=None, startstops=None, **kwargs):
+    def transition_processing_memory(
+        self, key, nbytes=None, type=None, worker=None, startstops=None, **kwargs
+    ):
         try:
             ts = self.tasks[key]
             assert worker
@@ -3419,20 +3649,24 @@ def transition_processing_memory(self, key, nbytes=None, type=None,
                 assert not ts.waiting_on
                 assert not ts.who_has, (ts, ts.who_has)
                 assert not ts.exception_blame
-                assert ts.state == 'processing'
+                assert ts.state == "processing"
 
             ws = self.workers.get(worker)
             if ws is None:
-                return {key: 'released'}
+                return {key: "released"}
 
             if ws is not ts.processing_on:  # someone else has this task
-                logger.info("Unexpected worker completed task, likely due to"
-                            " work stealing.  Expected: %s, Got: %s, Key: %s",
-                            ts.processing_on, ws, key)
+                logger.info(
+                    "Unexpected worker completed task, likely due to"
+                    " work stealing.  Expected: %s, Got: %s, Key: %s",
+                    ts.processing_on,
+                    ws,
+                    key,
+                )
                 return {}
 
             if startstops:
-                L = [(b, c) for a, b, c in startstops if a == 'compute']
+                L = [(b, c) for a, b, c in startstops if a == "compute"]
                 if L:
                     compute_start, compute_stop = L[0]
                 else:  # This is very rare
@@ -3451,8 +3685,7 @@ def transition_processing_memory(self, key, nbytes=None, type=None,
                 if not old_duration:
                     avg_duration = new_duration
                 else:
-                    avg_duration = (0.5 * old_duration
-                                    + 0.5 * new_duration)
+                    avg_duration = 0.5 * old_duration + 0.5 * new_duration
 
                 self.task_duration[prefix] = avg_duration
 
@@ -3486,6 +3719,7 @@ def transition_processing_memory(self, key, nbytes=None, type=None,
             logger.exception(e)
             if LOG_PDB:
                 import pdb
+
                 pdb.set_trace()
             raise
 
@@ -3505,35 +3739,35 @@ def transition_memory_released(self, key, safe=False):
                 if ts.who_wants:
                     ts.exception_blame = ts
                     ts.exception = "Worker holding Actor was lost"
-                    return {ts.key: 'erred'}  # don't try to recreate
+                    return {ts.key: "erred"}  # don't try to recreate
 
             recommendations = OrderedDict()
 
             for dts in ts.waiters:
-                if dts.state in ('no-worker', 'processing'):
-                    recommendations[dts.key] = 'waiting'
-                elif dts.state == 'waiting':
+                if dts.state in ("no-worker", "processing"):
+                    recommendations[dts.key] = "waiting"
+                elif dts.state == "waiting":
                     dts.waiting_on.add(ts)
 
             # XXX factor this out?
             for ws in ts.who_has:
                 ws.has_what.remove(ts)
                 ws.nbytes -= ts.get_nbytes()
-                self.worker_send(ws.address, {'op': 'delete-data',
-                                              'keys': [key],
-                                              'report': False})
+                self.worker_send(
+                    ws.address, {"op": "delete-data", "keys": [key], "report": False}
+                )
             ts.who_has.clear()
 
-            ts.state = 'released'
+            ts.state = "released"
 
-            self.report({'op': 'lost-data', 'key': key})
+            self.report({"op": "lost-data", "key": key})
 
             if not ts.run_spec:  # pure data
-                recommendations[key] = 'forgotten'
+                recommendations[key] = "forgotten"
             elif ts.has_lost_dependencies:
-                recommendations[key] = 'forgotten'
+                recommendations[key] = "forgotten"
             elif ts.who_wants or ts.waiters:
-                recommendations[key] = 'waiting'
+                recommendations[key] = "waiting"
 
             if self.validate:
                 assert not ts.waiting_on
@@ -3543,6 +3777,7 @@ def transition_memory_released(self, key, safe=False):
             logger.exception(e)
             if LOG_PDB:
                 import pdb
+
                 pdb.set_trace()
             raise
 
@@ -3564,14 +3799,18 @@ def transition_released_erred(self, key):
             for dts in ts.dependents:
                 dts.exception_blame = failing_ts
                 if not dts.who_has:
-                    recommendations[dts.key] = 'erred'
-
-            self.report({'op': 'task-erred',
-                         'key': key,
-                         'exception': failing_ts.exception,
-                         'traceback': failing_ts.traceback})
+                    recommendations[dts.key] = "erred"
+
+            self.report(
+                {
+                    "op": "task-erred",
+                    "key": key,
+                    "exception": failing_ts.exception,
+                    "traceback": failing_ts.traceback,
+                }
+            )
 
-            ts.state = 'erred'
+            ts.state = "erred"
 
             # TODO: waiting data?
             return recommendations
@@ -3579,6 +3818,7 @@ def transition_released_erred(self, key):
             logger.exception(e)
             if LOG_PDB:
                 import pdb
+
                 pdb.set_trace()
             raise
 
@@ -3588,7 +3828,7 @@ def transition_erred_released(self, key):
 
             if self.validate:
                 with log_errors(pdb=LOG_PDB):
-                    assert all(dts.state != 'erred' for dts in ts.dependencies)
+                    assert all(dts.state != "erred" for dts in ts.dependencies)
                     assert ts.exception_blame
                     assert not ts.who_has
                     assert not ts.waiting_on
@@ -3601,17 +3841,18 @@ def transition_erred_released(self, key):
             ts.traceback = None
 
             for dep in ts.dependents:
-                if dep.state == 'erred':
-                    recommendations[dep.key] = 'waiting'
+                if dep.state == "erred":
+                    recommendations[dep.key] = "waiting"
 
-            self.report({'op': 'task-retried', 'key': key})
-            ts.state = 'released'
+            self.report({"op": "task-retried", "key": key})
+            ts.state = "released"
 
             return recommendations
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
                 import pdb
+
                 pdb.set_trace()
             raise
 
@@ -3630,15 +3871,15 @@ def transition_waiting_released(self, key):
                 if ts in s:
                     s.discard(ts)
                     if not s and not dts.who_wants:
-                        recommendations[dts.key] = 'released'
+                        recommendations[dts.key] = "released"
             ts.waiting_on.clear()
 
-            ts.state = 'released'
+            ts.state = "released"
 
             if ts.has_lost_dependencies:
-                recommendations[key] = 'forgotten'
+                recommendations[key] = "forgotten"
             elif not ts.exception_blame and (ts.who_wants or ts.waiters):
-                recommendations[key] = 'waiting'
+                recommendations[key] = "waiting"
             else:
                 ts.waiters.clear()
 
@@ -3647,6 +3888,7 @@ def transition_waiting_released(self, key):
             logger.exception(e)
             if LOG_PDB:
                 import pdb
+
                 pdb.set_trace()
             raise
 
@@ -3658,27 +3900,28 @@ def transition_processing_released(self, key):
                 assert ts.processing_on
                 assert not ts.who_has
                 assert not ts.waiting_on
-                assert self.tasks[key].state == 'processing'
+                assert self.tasks[key].state == "processing"
 
-            self._remove_from_processing(ts, send_worker_msg={'op': 'release-task',
-                                                              'key': key})
+            self._remove_from_processing(
+                ts, send_worker_msg={"op": "release-task", "key": key}
+            )
 
-            ts.state = 'released'
+            ts.state = "released"
 
             recommendations = OrderedDict()
 
             if ts.has_lost_dependencies:
-                recommendations[key] = 'forgotten'
+                recommendations[key] = "forgotten"
             elif ts.waiters or ts.who_wants:
-                recommendations[key] = 'waiting'
+                recommendations[key] = "waiting"
 
-            if recommendations.get(key) != 'waiting':
+            if recommendations.get(key) != "waiting":
                 for dts in ts.dependencies:
-                    if dts.state != 'released':
+                    if dts.state != "released":
                         s = dts.waiters
                         s.discard(ts)
                         if not s and not dts.who_wants:
-                            recommendations[dts.key] = 'released'
+                            recommendations[dts.key] = "released"
                 ts.waiters.clear()
 
             if self.validate:
@@ -3689,11 +3932,13 @@ def transition_processing_released(self, key):
             logger.exception(e)
             if LOG_PDB:
                 import pdb
+
                 pdb.set_trace()
             raise
 
-    def transition_processing_erred(self, key, cause=None, exception=None,
-                                    traceback=None, **kwargs):
+    def transition_processing_erred(
+        self, key, cause=None, exception=None, traceback=None, **kwargs
+    ):
         try:
             ts = self.tasks[key]
 
@@ -3723,26 +3968,30 @@ def transition_processing_erred(self, key, cause=None, exception=None,
 
             for dts in ts.dependents:
                 dts.exception_blame = failing_ts
-                recommendations[dts.key] = 'erred'
+                recommendations[dts.key] = "erred"
 
             for dts in ts.dependencies:
                 s = dts.waiters
                 s.discard(ts)
                 if not s and not dts.who_wants:
-                    recommendations[dts.key] = 'released'
+                    recommendations[dts.key] = "released"
 
             ts.waiters.clear()  # do anything with this?
 
-            ts.state = 'erred'
+            ts.state = "erred"
 
-            self.report({'op': 'task-erred',
-                         'key': key,
-                         'exception': failing_ts.exception,
-                         'traceback': failing_ts.traceback})
+            self.report(
+                {
+                    "op": "task-erred",
+                    "key": key,
+                    "exception": failing_ts.exception,
+                    "traceback": failing_ts.traceback,
+                }
+            )
 
-            cs = self.clients['fire-and-forget']
+            cs = self.clients["fire-and-forget"]
             if ts in cs.wants_what:
-                self.client_releases_keys(client='fire-and-forget', keys=[key])
+                self.client_releases_keys(client="fire-and-forget", keys=[key])
 
             if self.validate:
                 assert not ts.processing_on
@@ -3752,6 +4001,7 @@ def transition_processing_erred(self, key, cause=None, exception=None,
             logger.exception(e)
             if LOG_PDB:
                 import pdb
+
                 pdb.set_trace()
             raise
 
@@ -3760,12 +4010,12 @@ def transition_no_worker_released(self, key):
             ts = self.tasks[key]
 
             if self.validate:
-                assert self.tasks[key].state == 'no-worker'
+                assert self.tasks[key].state == "no-worker"
                 assert not ts.who_has
                 assert not ts.waiting_on
 
             self.unrunnable.remove(ts)
-            ts.state = 'released'
+            ts.state = "released"
 
             for dts in ts.dependencies:
                 dts.waiters.discard(ts)
@@ -3777,12 +4027,13 @@ def transition_no_worker_released(self, key):
             logger.exception(e)
             if LOG_PDB:
                 import pdb
+
                 pdb.set_trace()
             raise
 
     def remove_key(self, key):
         ts = self.tasks.pop(key)
-        assert ts.state == 'forgotten'
+        assert ts.state == "forgotten"
         self.unrunnable.discard(ts)
         for cs in ts.who_wants:
             cs.wants_what.remove(ts)
@@ -3794,15 +4045,15 @@ def remove_key(self, key):
             del self.task_metadata[key]
 
     def _propagate_forgotten(self, ts, recommendations):
-        ts.state = 'forgotten'
+        ts.state = "forgotten"
         key = ts.key
         for dts in ts.dependents:
             dts.has_lost_dependencies = True
             dts.dependencies.remove(ts)
             dts.waiting_on.discard(ts)
-            if dts.state not in ('memory', 'erred'):
+            if dts.state not in ("memory", "erred"):
                 # Cannot compute task anymore
-                recommendations[dts.key] = 'forgotten'
+                recommendations[dts.key] = "forgotten"
         ts.dependents.clear()
         ts.waiters.clear()
 
@@ -3813,7 +4064,7 @@ def _propagate_forgotten(self, ts, recommendations):
             if not dts.dependents and not dts.who_wants:
                 # Task not needed anymore
                 assert dts is not ts
-                recommendations[dts.key] = 'forgotten'
+                recommendations[dts.key] = "forgotten"
         ts.dependencies.clear()
         ts.waiting_on.clear()
 
@@ -3822,9 +4073,9 @@ def _propagate_forgotten(self, ts, recommendations):
             ws.nbytes -= ts.get_nbytes()
             w = ws.address
             if w in self.workers:  # in case worker has died
-                self.worker_send(w, {'op': 'delete-data',
-                                     'keys': [key],
-                                     'report': False})
+                self.worker_send(
+                    w, {"op": "delete-data", "keys": [key], "report": False}
+                )
         ts.who_has.clear()
 
     def transition_memory_forgotten(self, key):
@@ -3832,7 +4083,7 @@ def transition_memory_forgotten(self, key):
             ts = self.tasks[key]
 
             if self.validate:
-                assert ts.state == 'memory'
+                assert ts.state == "memory"
                 assert not ts.processing_on
                 assert not ts.waiting_on
                 if not ts.run_spec:
@@ -3863,6 +4114,7 @@ def transition_memory_forgotten(self, key):
             logger.exception(e)
             if LOG_PDB:
                 import pdb
+
                 pdb.set_trace()
             raise
 
@@ -3871,7 +4123,7 @@ def transition_released_forgotten(self, key):
             ts = self.tasks[key]
 
             if self.validate:
-                assert ts.state in ('released', 'erred')
+                assert ts.state in ("released", "erred")
                 assert not ts.who_has
                 assert not ts.processing_on
                 assert not ts.waiting_on, (ts, ts.waiting_on)
@@ -3898,6 +4150,7 @@ def transition_released_forgotten(self, key):
             logger.exception(e)
             if LOG_PDB:
                 import pdb
+
                 pdb.set_trace()
             raise
 
@@ -3933,30 +4186,36 @@ def transition(self, key, finish, *args, **kwargs):
             if (start, finish) in self._transitions:
                 func = self._transitions[start, finish]
                 recommendations = func(key, *args, **kwargs)
-            elif 'released' not in (start, finish):
-                func = self._transitions['released', finish]
+            elif "released" not in (start, finish):
+                func = self._transitions["released", finish]
                 assert not args and not kwargs
-                a = self.transition(key, 'released')
+                a = self.transition(key, "released")
                 if key in a:
-                    func = self._transitions['released', a[key]]
+                    func = self._transitions["released", a[key]]
                 b = func(key)
                 a = a.copy()
                 a.update(b)
                 recommendations = a
-                start = 'released'
+                start = "released"
             else:
-                raise RuntimeError("Impossible transition from %r to %r"
-                                   % (start, finish))
+                raise RuntimeError(
+                    "Impossible transition from %r to %r" % (start, finish)
+                )
 
             finish2 = ts.state
-            self.transition_log.append((key, start, finish2, recommendations,
-                                        time()))
+            self.transition_log.append((key, start, finish2, recommendations, time()))
             if self.validate:
-                logger.debug("Transitioned %r %s->%s (actual: %s).  Consequence: %s",
-                             key, start, finish2, ts.state, dict(recommendations))
+                logger.debug(
+                    "Transitioned %r %s->%s (actual: %s).  Consequence: %s",
+                    key,
+                    start,
+                    finish2,
+                    ts.state,
+                    dict(recommendations),
+                )
             if self.plugins:
                 # Temporarily put back forgotten key for plugin to retrieve it
-                if ts.state == 'forgotten':
+                if ts.state == "forgotten":
                     try:
                         ts.dependents = dependents
                         ts.dependencies = dependencies
@@ -3968,15 +4227,15 @@ def transition(self, key, finish, *args, **kwargs):
                         plugin.transition(key, start, finish2, *args, **kwargs)
                     except Exception:
                         logger.info("Plugin failed with exception", exc_info=True)
-                if ts.state == 'forgotten':
+                if ts.state == "forgotten":
                     del self.tasks[ts.key]
 
             return recommendations
         except Exception as e:
-            logger.exception("Error transitioning %r from %r to %r",
-                             key, start, finish)
+            logger.exception("Error transitioning %r from %r to %r", key, start, finish)
             if LOG_PDB:
                 import pdb
+
                 pdb.set_trace()
             raise
 
@@ -4001,8 +4260,9 @@ def transitions(self, recommendations):
     def story(self, *keys):
         """ Get all transitions that touch one of the input keys """
         keys = set(keys)
-        return [t for t in self.transition_log
-                if t[0] in keys or keys.intersection(t[3])]
+        return [
+            t for t in self.transition_log if t[0] in keys or keys.intersection(t[3])
+        ]
 
     transition_story = story
 
@@ -4013,18 +4273,18 @@ def reschedule(self, key=None, worker=None):
         elsewhere
         """
         ts = self.tasks[key]
-        if ts.state != 'processing':
+        if ts.state != "processing":
             return
         if worker and ts.processing_on.address != worker:
             return
-        self.transitions({key: 'released'})
+        self.transitions({key: "released"})
 
     ##############################
     # Assigning Tasks to Workers #
     ##############################
 
     def check_idle_saturated(self, ws, occ=None):
-        if self.total_ncores == 0 or ws.status == 'closed':
+        if self.total_ncores == 0 or ws.status == "closed":
             return
         if occ is None:
             occ = ws.occupancy
@@ -4065,8 +4325,7 @@ def valid_workers(self, ts):
             # may not be connected when host_restrictions is populated
             hr = [self.coerce_hostname(h) for h in ts.host_restrictions]
             # XXX need HostState?
-            ss = [self.host_info[h]['addresses']
-                  for h in hr if h in self.host_info]
+            ss = [self.host_info[h]["addresses"] for h in hr if h in self.host_info]
             ss = set.union(*ss) if ss else set()
             if s is True:
                 s = ss
@@ -4074,9 +4333,14 @@ def valid_workers(self, ts):
                 s |= ss
 
         if ts.resource_restrictions:
-            w = {resource: {w for w, supplied in self.resources[resource].items()
-                            if supplied >= required}
-                 for resource, required in ts.resource_restrictions.items()}
+            w = {
+                resource: {
+                    w
+                    for w, supplied in self.resources[resource].items()
+                    if supplied >= required
+                }
+                for resource, required in ts.resource_restrictions.items()
+            }
 
             ww = set.intersection(*w.values())
 
@@ -4112,7 +4376,7 @@ def add_resources(self, stream=None, worker=None, resources=None):
         for resource, quantity in ws.resources.items():
             ws.used_resources[resource] = 0
             self.resources[resource][worker] = quantity
-        return 'OK'
+        return "OK"
 
     def remove_resources(self, worker):
         ws = self.workers[worker]
@@ -4132,8 +4396,7 @@ def coerce_address(self, addr, resolve=True):
         if isinstance(addr, tuple):
             addr = unparse_host_port(*addr)
         if not isinstance(addr, six.string_types):
-            raise TypeError("addresses should be strings or tuples, got %r"
-                            % (addr,))
+            raise TypeError("addresses should be strings or tuples, got %r" % (addr,))
 
         if resolve:
             addr = resolve_address(addr)
@@ -4163,7 +4426,7 @@ def workers_list(self, workers):
 
         out = set()
         for w in workers:
-            if ':' in w:
+            if ":" in w:
                 out.add(w)
             else:
                 out.update({ww for ww in self.workers if w in ww})  # TODO: quadratic
@@ -4175,11 +4438,10 @@ def start_ipython(self, comm=None):
         Returns Jupyter connection info dictionary.
         """
         from ._ipython_utils import start_ipython
+
         if self._ipython_kernel is None:
             self._ipython_kernel = start_ipython(
-                ip=self.ip,
-                ns={'scheduler': self},
-                log=logger,
+                ip=self.ip, ns={"scheduler": self}, log=logger
             )
         return self._ipython_kernel.get_connection_info()
 
@@ -4189,9 +4451,9 @@ def worker_objective(self, ts, ws):
 
         Minimize expected start time.  If a tie then break with data storage.
         """
-        comm_bytes = sum([dts.get_nbytes()
-                          for dts in ts.dependencies
-                          if ws not in dts.who_has])
+        comm_bytes = sum(
+            [dts.get_nbytes() for dts in ts.dependencies if ws not in dts.who_has]
+        )
         stack_time = ws.occupancy / ws.ncores
         start_time = comm_bytes / BANDWIDTH + stack_time
 
@@ -4201,43 +4463,61 @@ def worker_objective(self, ts, ws):
             return (start_time, ws.nbytes)
 
     @gen.coroutine
-    def get_profile(self, comm=None, workers=None, merge_workers=True,
-                    start=None, stop=None, key=None):
+    def get_profile(
+        self,
+        comm=None,
+        workers=None,
+        merge_workers=True,
+        start=None,
+        stop=None,
+        key=None,
+    ):
         if workers is None:
             workers = self.workers
         else:
             workers = set(self.workers) & set(workers)
-        result = yield {w: self.rpc(w).profile(start=start, stop=stop, key=key)
-                        for w in workers}
+        result = yield {
+            w: self.rpc(w).profile(start=start, stop=stop, key=key) for w in workers
+        }
         if merge_workers:
             result = profile.merge(*result.values())
         raise gen.Return(result)
 
     @gen.coroutine
-    def get_profile_metadata(self, comm=None, workers=None, merge_workers=True,
-                             start=None, stop=None, profile_cycle_interval=None):
-        dt = profile_cycle_interval or dask.config.get('distributed.worker.profile.cycle')
-        dt = parse_timedelta(dt, default='ms')
+    def get_profile_metadata(
+        self,
+        comm=None,
+        workers=None,
+        merge_workers=True,
+        start=None,
+        stop=None,
+        profile_cycle_interval=None,
+    ):
+        dt = profile_cycle_interval or dask.config.get(
+            "distributed.worker.profile.cycle"
+        )
+        dt = parse_timedelta(dt, default="ms")
 
         if workers is None:
             workers = self.workers
         else:
             workers = set(self.workers) & set(workers)
-        result = yield {w: self.rpc(w).profile_metadata(start=start, stop=stop)
-                        for w in workers}
+        result = yield {
+            w: self.rpc(w).profile_metadata(start=start, stop=stop) for w in workers
+        }
 
-        counts = [v['counts'] for v in result.values()]
+        counts = [v["counts"] for v in result.values()]
         counts = itertools.groupby(merge_sorted(*counts), lambda t: t[0] // dt * dt)
         counts = [(time, sum(pluck(1, group))) for time, group in counts]
 
         keys = set()
         for v in result.values():
-            for t, d in v['keys']:
+            for t, d in v["keys"]:
                 for k in d:
                     keys.add(k)
         keys = {k: [] for k in keys}
 
-        groups1 = [v['keys'] for v in result.values()]
+        groups1 = [v["keys"] for v in result.values()]
         groups2 = list(merge_sorted(*groups1, key=first))
 
         last = 0
@@ -4250,7 +4530,7 @@ def get_profile_metadata(self, comm=None, workers=None, merge_workers=True,
             for k, v in d.items():
                 keys[k][-1][1] += v
 
-        raise gen.Return({'counts': counts, 'keys': keys})
+        raise gen.Return({"counts": counts, "keys": keys})
 
     def get_logs(self, comm=None, n=None):
         deque_handler = self._deque_handler
@@ -4263,8 +4543,7 @@ def get_logs(self, comm=None, n=None):
 
     @gen.coroutine
     def get_worker_logs(self, comm=None, n=None, workers=None):
-        results = yield self.broadcast(msg={'op': 'get_logs', 'n': n},
-                                       workers=workers)
+        results = yield self.broadcast(msg={"op": "get_logs", "n": n}, workers=workers)
         raise gen.Return(results)
 
     ###########
@@ -4288,7 +4567,7 @@ def reevaluate_occupancy(self, worker_index=0):
         """
         DELAY = 0.1
         try:
-            if self.status == 'closed':
+            if self.status == "closed":
                 return
 
             last = time()
@@ -4311,8 +4590,9 @@ def reevaluate_occupancy(self, worker_index=0):
                         next_time = timedelta(seconds=duration * 5)  # 25ms gap
                         break
 
-            self.loop.add_timeout(next_time, self.reevaluate_occupancy,
-                                  worker_index=worker_index)
+            self.loop.add_timeout(
+                next_time, self.reevaluate_occupancy, worker_index=worker_index
+            )
 
         except Exception:
             logger.error("Error in reevaluate occupancy", exc_info=True)
@@ -4335,8 +4615,8 @@ def _reevaluate_occupancy_worker(self, ws):
         self.check_idle_saturated(ws)
 
         # significant increase in duration
-        if (new > old * 1.3) and ('stealing' in self.extensions):
-            steal = self.extensions['stealing']
+        if (new > old * 1.3) and ("stealing" in self.extensions):
+            steal = self.extensions["stealing"]
             for ts in ws.processing:
                 steal.remove_key_from_stealable(ts)
                 steal.put_key_in_stealable(ts)
@@ -4345,8 +4625,11 @@ def check_worker_ttl(self):
         now = time()
         for ws in self.workers.values():
             if ws.last_seen < now - self.worker_ttl:
-                logger.warning("Worker failed to heartbeat within %s seconds. "
-                               "Closing: %s", self.worker_ttl, ws)
+                logger.warning(
+                    "Worker failed to heartbeat within %s seconds. " "Closing: %s",
+                    self.worker_ttl,
+                    ws,
+                )
                 self.remove_worker(address=ws.address)
 
 
@@ -4371,8 +4654,7 @@ def decide_worker(ts, all_workers, valid_workers, objective):
     if ts.actor:
         candidates = all_workers
     else:
-        candidates = frequencies([ws for dts in deps
-                                  for ws in dts.who_has])
+        candidates = frequencies([ws for dts in deps for ws in dts.who_has])
     if valid_workers is True:
         if not candidates:
             candidates = all_workers
@@ -4398,71 +4680,107 @@ def validate_task_state(ts):
     """
     Validate the given TaskState.
     """
-    assert ts.state in ALL_TASK_STATES or ts.state == 'forgotten', ts
+    assert ts.state in ALL_TASK_STATES or ts.state == "forgotten", ts
 
     if ts.waiting_on:
-        assert ts.waiting_on.issubset(ts.dependencies), \
-            ("waiting not subset of dependencies", str(ts.waiting_on), str(ts.dependencies))
+        assert ts.waiting_on.issubset(ts.dependencies), (
+            "waiting not subset of dependencies",
+            str(ts.waiting_on),
+            str(ts.dependencies),
+        )
     if ts.waiters:
-        assert ts.waiters.issubset(ts.dependents), \
-            ("waiters not subset of dependents", str(ts.waiters), str(ts.dependents))
+        assert ts.waiters.issubset(ts.dependents), (
+            "waiters not subset of dependents",
+            str(ts.waiters),
+            str(ts.dependents),
+        )
 
     for dts in ts.waiting_on:
-        assert not dts.who_has, \
-            ("waiting on in-memory dep", str(ts), str(dts))
-        assert dts.state != 'released', \
-            ("waiting on released dep", str(ts), str(dts))
+        assert not dts.who_has, ("waiting on in-memory dep", str(ts), str(dts))
+        assert dts.state != "released", ("waiting on released dep", str(ts), str(dts))
     for dts in ts.dependencies:
-        assert ts in dts.dependents, \
-            ("not in dependency's dependents", str(ts), str(dts), str(dts.dependents))
-        if ts.state in ('waiting', 'processing'):
-            assert dts in ts.waiting_on or dts.who_has, \
-                ("dep missing", str(ts), str(dts))
-        assert dts.state != 'forgotten'
+        assert ts in dts.dependents, (
+            "not in dependency's dependents",
+            str(ts),
+            str(dts),
+            str(dts.dependents),
+        )
+        if ts.state in ("waiting", "processing"):
+            assert dts in ts.waiting_on or dts.who_has, (
+                "dep missing",
+                str(ts),
+                str(dts),
+            )
+        assert dts.state != "forgotten"
 
     for dts in ts.waiters:
-        assert dts.state in ('waiting', 'processing'), \
-            ("waiter not in play", str(ts), str(dts))
+        assert dts.state in ("waiting", "processing"), (
+            "waiter not in play",
+            str(ts),
+            str(dts),
+        )
     for dts in ts.dependents:
-        assert ts in dts.dependencies, \
-            ("not in dependent's dependencies", str(ts), str(dts), str(dts.dependencies))
-        assert dts.state != 'forgotten'
+        assert ts in dts.dependencies, (
+            "not in dependent's dependencies",
+            str(ts),
+            str(dts),
+            str(dts.dependencies),
+        )
+        assert dts.state != "forgotten"
 
-    assert (ts.processing_on is not None) == (ts.state == 'processing')
-    assert bool(ts.who_has) == (ts.state == 'memory'), (ts, ts.who_has)
+    assert (ts.processing_on is not None) == (ts.state == "processing")
+    assert bool(ts.who_has) == (ts.state == "memory"), (ts, ts.who_has)
 
-    if ts.state == 'processing':
-        assert all(dts.who_has for dts in ts.dependencies), \
-            ("task processing without all deps", str(ts), str(ts.dependencies))
+    if ts.state == "processing":
+        assert all(dts.who_has for dts in ts.dependencies), (
+            "task processing without all deps",
+            str(ts),
+            str(ts.dependencies),
+        )
         assert not ts.waiting_on
 
     if ts.who_has:
-        assert ts.waiters or ts.who_wants, \
-            ("unneeded task in memory", str(ts), str(ts.who_has))
+        assert ts.waiters or ts.who_wants, (
+            "unneeded task in memory",
+            str(ts),
+            str(ts.who_has),
+        )
         assert not any(ts in dts.waiting_on for dts in ts.dependents)
         for ws in ts.who_has:
-            assert ts in ws.has_what, \
-                ("not in who_has' has_what", str(ts), str(ws), str(ws.has_what))
+            assert ts in ws.has_what, (
+                "not in who_has' has_what",
+                str(ts),
+                str(ws),
+                str(ws.has_what),
+            )
 
     if ts.who_wants:
         for cs in ts.who_wants:
-            assert ts in cs.wants_what, \
-                ("not in who_wants' wants_what", str(ts), str(cs), str(cs.wants_what))
+            assert ts in cs.wants_what, (
+                "not in who_wants' wants_what",
+                str(ts),
+                str(cs),
+                str(cs.wants_what),
+            )
 
     if ts.actor:
-        if ts.state == 'memory':
+        if ts.state == "memory":
             assert sum([ts in ws.actors for ws in ts.who_has]) == 1
-        if ts.state == 'processing':
+        if ts.state == "processing":
             assert ts in ts.processing_on.actors
 
 
 def validate_worker_state(ws):
     for ts in ws.has_what:
-        assert ws in ts.who_has, \
-            ("not in has_what' who_has", str(ws), str(ts), str(ts.who_has))
+        assert ws in ts.who_has, (
+            "not in has_what' who_has",
+            str(ws),
+            str(ts),
+            str(ts.who_has),
+        )
 
     for ts in ws.actors:
-        assert ts.state in ('memory', 'processing')
+        assert ts.state in ("memory", "processing")
 
 
 def validate_state(tasks, workers, clients):
@@ -4480,14 +4798,18 @@ def validate_state(tasks, workers, clients):
 
     for cs in clients.values():
         for ts in cs.wants_what:
-            assert cs in ts.who_wants, \
-                ("not in wants_what' who_wants", str(cs), str(ts), str(ts.who_wants))
+            assert cs in ts.who_wants, (
+                "not in wants_what' who_wants",
+                str(cs),
+                str(ts),
+                str(ts.who_wants),
+            )
 
 
 _round_robin = [0]
 
 
-fast_tasks = {'rechunk-split', 'shuffle-split'}
+fast_tasks = {"rechunk-split", "shuffle-split"}
 
 
 def heartbeat_interval(n):
diff --git a/distributed/security.py b/distributed/security.py
index 0a40396a54a..e86c0602860 100644
--- a/distributed/security.py
+++ b/distributed/security.py
@@ -8,24 +8,23 @@
 import dask
 
 
-_roles = ['client', 'scheduler', 'worker']
+_roles = ["client", "scheduler", "worker"]
 
-_tls_per_role_fields = ['key', 'cert']
+_tls_per_role_fields = ["key", "cert"]
 
-_tls_fields = ['ca_file', 'ciphers']
+_tls_fields = ["ca_file", "ciphers"]
 
-_misc_fields = ['require_encryption']
+_misc_fields = ["require_encryption"]
 
-_fields = set(_misc_fields +
-              ['tls_%s' % field for field in _tls_fields] +
-              ['tls_%s_%s' % (role, field)
-               for role in _roles
-               for field in _tls_per_role_fields]
-              )
+_fields = set(
+    _misc_fields
+    + ["tls_%s" % field for field in _tls_fields]
+    + ["tls_%s_%s" % (role, field) for role in _roles for field in _tls_per_role_fields]
+)
 
 
 def _field_to_config_key(field):
-    return field.replace('_', '-')
+    return field.replace("_", "-")
 
 
 class Security(object):
@@ -61,65 +60,65 @@ def _init_from_dict(self, d):
         """
         Initialize Security from nested dict.
         """
-        self._init_fields_from_dict(d, '', _misc_fields, {})
-        self._init_fields_from_dict(d, 'tls', _tls_fields, _tls_per_role_fields)
+        self._init_fields_from_dict(d, "", _misc_fields, {})
+        self._init_fields_from_dict(d, "tls", _tls_fields, _tls_per_role_fields)
 
-    def _init_fields_from_dict(self, d, category,
-                               fields, per_role_fields):
+    def _init_fields_from_dict(self, d, category, fields, per_role_fields):
         if category:
             d = d.get(category, {})
-            category_prefix = category + '_'
+            category_prefix = category + "_"
         else:
-            category_prefix = ''
+            category_prefix = ""
         for field in fields:
             k = _field_to_config_key(field)
             if k in d:
-                setattr(self, '%s%s' % (category_prefix, field), d[k])
+                setattr(self, "%s%s" % (category_prefix, field), d[k])
         for role in _roles:
             dd = d.get(role, {})
             for field in per_role_fields:
                 k = _field_to_config_key(field)
                 if k in dd:
-                    setattr(self, '%s%s_%s' % (category_prefix, role, field), dd[k])
+                    setattr(self, "%s%s_%s" % (category_prefix, role, field), dd[k])
 
     def __repr__(self):
         items = sorted((k, getattr(self, k)) for k in _fields)
-        return ("Security(" +
-                ", ".join("%s=%r" % (k, v) for k, v in items if v is not None) +
-                ")")
+        return (
+            "Security("
+            + ", ".join("%s=%r" % (k, v) for k, v in items if v is not None)
+            + ")"
+        )
 
     def get_tls_config_for_role(self, role):
         """
         Return the TLS configuration for the given role, as a flat dict.
         """
-        return self._get_config_for_role('tls', role, _tls_fields, _tls_per_role_fields)
+        return self._get_config_for_role("tls", role, _tls_fields, _tls_per_role_fields)
 
     def _get_config_for_role(self, category, role, fields, per_role_fields):
         if role not in _roles:
             raise ValueError("unknown role %r" % (role,))
         d = {}
         for field in fields:
-            k = '%s_%s' % (category, field)
+            k = "%s_%s" % (category, field)
             d[field] = getattr(self, k)
         for field in per_role_fields:
-            k = '%s_%s_%s' % (category, role, field)
+            k = "%s_%s_%s" % (category, role, field)
             d[field] = getattr(self, k)
         return d
 
     def _get_tls_context(self, tls, purpose):
-        if tls.get('ca_file') and tls.get('cert'):
+        if tls.get("ca_file") and tls.get("cert"):
             try:
-                ctx = ssl.create_default_context(purpose=purpose,
-                                                 cafile=tls['ca_file'])
+                ctx = ssl.create_default_context(purpose=purpose, cafile=tls["ca_file"])
             except AttributeError:
                 raise RuntimeError("TLS functionality requires Python 2.7.9+")
             ctx.verify_mode = ssl.CERT_REQUIRED
             # We expect a dedicated CA for the cluster and people using
             # IP addresses rather than hostnames
             ctx.check_hostname = False
-            ctx.load_cert_chain(tls['cert'], tls.get('key'))
-            if tls.get('ciphers'):
-                ctx.set_ciphers(tls.get('ciphers'))
+            ctx.load_cert_chain(tls["cert"], tls.get("key"))
+            if tls.get("ciphers"):
+                ctx.set_ciphers(tls.get("ciphers"))
             return ctx
 
     def get_connection_args(self, role):
@@ -131,8 +130,8 @@ def get_connection_args(self, role):
         tls = self.get_tls_config_for_role(role)
         # Ensure backwards compatibility (ssl.Purpose is Python 2.7.9+ only)
         purpose = ssl.Purpose.SERVER_AUTH if hasattr(ssl, "Purpose") else None
-        d['ssl_context'] = self._get_tls_context(tls, purpose)
-        d['require_encryption'] = self.require_encryption
+        d["ssl_context"] = self._get_tls_context(tls, purpose)
+        d["require_encryption"] = self.require_encryption
         return d
 
     def get_listen_args(self, role):
@@ -144,6 +143,6 @@ def get_listen_args(self, role):
         tls = self.get_tls_config_for_role(role)
         # Ensure backwards compatibility (ssl.Purpose is Python 2.7.9+ only)
         purpose = ssl.Purpose.CLIENT_AUTH if hasattr(ssl, "Purpose") else None
-        d['ssl_context'] = self._get_tls_context(tls, purpose)
-        d['require_encryption'] = self.require_encryption
+        d["ssl_context"] = self._get_tls_context(tls, purpose)
+        d["require_encryption"] = self.require_encryption
         return d
diff --git a/distributed/sizeof.py b/distributed/sizeof.py
index cbf65638fa7..0bc094e35a7 100644
--- a/distributed/sizeof.py
+++ b/distributed/sizeof.py
@@ -15,6 +15,5 @@ def safe_sizeof(obj, default_size=1e6):
     try:
         return sizeof(obj)
     except Exception:
-        logger.warning('Sizeof calculation failed.  Defaulting to 1MB',
-                       exc_info=True)
+        logger.warning("Sizeof calculation failed.  Defaulting to 1MB", exc_info=True)
         return int(default_size)
diff --git a/distributed/stealing.py b/distributed/stealing.py
index b7a73613a56..d361305b105 100644
--- a/distributed/stealing.py
+++ b/distributed/stealing.py
@@ -22,7 +22,7 @@
 logger = logging.getLogger(__name__)
 
 
-LOG_PDB = dask.config.get('distributed.admin.pdb-on-err')
+LOG_PDB = dask.config.get("distributed.admin.pdb-on-err")
 
 
 class WorkStealing(SchedulerPlugin):
@@ -43,25 +43,25 @@ def __init__(self, scheduler):
         for worker in scheduler.workers:
             self.add_worker(worker=worker)
 
-        pc = PeriodicCallback(callback=self.balance,
-                              callback_time=100,
-                              io_loop=self.scheduler.loop)
+        pc = PeriodicCallback(
+            callback=self.balance, callback_time=100, io_loop=self.scheduler.loop
+        )
         self._pc = pc
-        self.scheduler.periodic_callbacks['stealing'] = pc
+        self.scheduler.periodic_callbacks["stealing"] = pc
         self.scheduler.plugins.append(self)
-        self.scheduler.extensions['stealing'] = self
-        self.scheduler.events['stealing'] = deque(maxlen=100000)
+        self.scheduler.extensions["stealing"] = self
+        self.scheduler.events["stealing"] = deque(maxlen=100000)
         self.count = 0
         # { task state: <stealing info dict> }
         self.in_flight = dict()
         # { worker state: occupancy }
         self.in_flight_occupancy = defaultdict(lambda: 0)
 
-        self.scheduler.stream_handlers['steal-response'] = self.move_task_confirm
+        self.scheduler.stream_handlers["steal-response"] = self.move_task_confirm
 
     @property
     def log(self):
-        return self.scheduler.events['stealing']
+        return self.scheduler.events["stealing"]
 
     def add_worker(self, scheduler=None, worker=None):
         self.stealable[worker] = [set() for i in range(15)]
@@ -72,17 +72,18 @@ def remove_worker(self, scheduler=None, worker=None):
     def teardown(self):
         self._pc.stop()
 
-    def transition(self, key, start, finish, compute_start=None,
-                   compute_stop=None, *args, **kwargs):
+    def transition(
+        self, key, start, finish, compute_start=None, compute_stop=None, *args, **kwargs
+    ):
         ts = self.scheduler.tasks[key]
-        if finish == 'processing':
+        if finish == "processing":
             self.put_key_in_stealable(ts)
 
-        if start == 'processing':
+        if start == "processing":
             self.remove_key_from_stealable(ts)
-            if finish == 'memory':
+            if finish == "memory":
                 for tts in self.stealable_unknown_durations.pop(ts.prefix, ()):
-                    if tts not in self.in_flight and tts.state == 'processing':
+                    if tts not in self.in_flight and tts.state == "processing":
                         self.put_key_in_stealable(tts)
             else:
                 self.in_flight.pop(ts, None)
@@ -91,7 +92,7 @@ def put_key_in_stealable(self, ts):
         ws = ts.processing_on
         worker = ws.address
         cost_multiplier, level = self.steal_time_ratio(ts)
-        self.log.append(('add-stealable', ts.key, worker, level))
+        self.log.append(("add-stealable", ts.key, worker, level))
         if cost_multiplier is not None:
             self.stealable_all[level].add(ts)
             self.stealable[worker][level].add(ts)
@@ -103,7 +104,7 @@ def remove_key_from_stealable(self, ts):
             return
 
         worker, level = result
-        self.log.append(('remove-stealable', ts.key, worker, level))
+        self.log.append(("remove-stealable", ts.key, worker, level))
         try:
             self.stealable[worker][level].remove(ts)
         except KeyError:
@@ -123,9 +124,9 @@ def steal_time_ratio(self, ts):
         For example a result of zero implies a task without dependencies.
         level: The location within a stealable list to place this value
         """
-        if (not ts.loose_restrictions
-            and (ts.host_restrictions or ts.worker_restrictions
-                 or ts.resource_restrictions)):
+        if not ts.loose_restrictions and (
+            ts.host_restrictions or ts.worker_restrictions or ts.resource_restrictions
+        ):
             return None, None  # don't steal
 
         if not ts.dependencies:  # no dependencies fast path
@@ -158,28 +159,36 @@ def move_task_request(self, ts, victim, thief):
             if self.scheduler.validate:
                 if victim is not ts.processing_on:
                     import pdb
+
                     pdb.set_trace()
 
             key = ts.key
             self.remove_key_from_stealable(ts)
-            logger.debug("Request move %s, %s: %2f -> %s: %2f", key,
-                         victim, victim.occupancy,
-                         thief, thief.occupancy)
+            logger.debug(
+                "Request move %s, %s: %2f -> %s: %2f",
+                key,
+                victim,
+                victim.occupancy,
+                thief,
+                thief.occupancy,
+            )
 
             victim_duration = victim.processing[ts]
 
-            thief_duration = (
-                self.scheduler.get_task_duration(ts) +
-                self.scheduler.get_comm_cost(ts, thief)
-            )
+            thief_duration = self.scheduler.get_task_duration(
+                ts
+            ) + self.scheduler.get_comm_cost(ts, thief)
 
             self.scheduler.stream_comms[victim.address].send(
-                {'op': 'steal-request', 'key': key})
+                {"op": "steal-request", "key": key}
+            )
 
-            self.in_flight[ts] = {'victim': victim,
-                                  'thief': thief,
-                                  'victim_duration': victim_duration,
-                                  'thief_duration': thief_duration}
+            self.in_flight[ts] = {
+                "victim": victim,
+                "thief": thief,
+                "victim_duration": victim_duration,
+                "thief_duration": thief_duration,
+            }
 
             self.in_flight_occupancy[victim] -= victim_duration
             self.in_flight_occupancy[thief] += thief_duration
@@ -189,6 +198,7 @@ def move_task_request(self, ts, victim, thief):
             logger.exception(e)
             if LOG_PDB:
                 import pdb
+
                 pdb.set_trace()
             raise
 
@@ -203,42 +213,48 @@ def move_task_confirm(self, key=None, worker=None, state=None):
                 d = self.in_flight.pop(ts)
             except KeyError:
                 return
-            thief = d['thief']
-            victim = d['victim']
-            logger.debug("Confirm move %s, %s -> %s.  State: %s",
-                         key, victim, thief, state)
+            thief = d["thief"]
+            victim = d["victim"]
+            logger.debug(
+                "Confirm move %s, %s -> %s.  State: %s", key, victim, thief, state
+            )
 
-            self.in_flight_occupancy[thief] -= d['thief_duration']
-            self.in_flight_occupancy[victim] += d['victim_duration']
+            self.in_flight_occupancy[thief] -= d["thief_duration"]
+            self.in_flight_occupancy[victim] += d["victim_duration"]
 
             if not self.in_flight:
                 self.in_flight_occupancy = defaultdict(lambda: 0)
 
-            if ts.state != 'processing' or ts.processing_on is not victim:
+            if ts.state != "processing" or ts.processing_on is not victim:
                 old_thief = thief.occupancy
                 new_thief = sum(thief.processing.values())
                 old_victim = victim.occupancy
                 new_victim = sum(victim.processing.values())
                 thief.occupancy = new_thief
                 victim.occupancy = new_victim
-                self.scheduler.total_occupancy += new_thief - old_thief + new_victim - old_victim
+                self.scheduler.total_occupancy += (
+                    new_thief - old_thief + new_victim - old_victim
+                )
                 return
 
             # One of the pair has left, punt and reschedule
-            if (thief.address not in self.scheduler.workers or
-                victim.address not in self.scheduler.workers):
+            if (
+                thief.address not in self.scheduler.workers
+                or victim.address not in self.scheduler.workers
+            ):
                 self.scheduler.reschedule(key)
                 return
 
             # Victim had already started execution, reverse stealing
-            if state in ('memory', 'executing', 'long-running', None):
-                self.log.append(('already-computing',
-                                 key, victim.address, thief.address))
+            if state in ("memory", "executing", "long-running", None):
+                self.log.append(
+                    ("already-computing", key, victim.address, thief.address)
+                )
                 self.scheduler.check_idle_saturated(thief)
                 self.scheduler.check_idle_saturated(victim)
 
             # Victim was waiting, has given up task, enact steal
-            elif state in ('waiting', 'ready'):
+            elif state in ("waiting", "ready"):
                 self.remove_key_from_stealable(ts)
                 ts.processing_on = thief
                 duration = victim.processing.pop(ts)
@@ -247,23 +263,23 @@ def move_task_confirm(self, key=None, worker=None, state=None):
                 if not victim.processing:
                     self.scheduler.total_occupancy -= victim.occupancy
                     victim.occupancy = 0
-                thief.processing[ts] = d['thief_duration']
-                thief.occupancy += d['thief_duration']
-                self.scheduler.total_occupancy += d['thief_duration']
+                thief.processing[ts] = d["thief_duration"]
+                thief.occupancy += d["thief_duration"]
+                self.scheduler.total_occupancy += d["thief_duration"]
                 self.put_key_in_stealable(ts)
 
                 try:
                     self.scheduler.send_task_to_worker(thief.address, key)
                 except CommClosedError:
                     self.scheduler.remove_worker(thief.address)
-                self.log.append(('confirm',
-                                 key, victim.address, thief.address))
+                self.log.append(("confirm", key, victim.address, thief.address))
             else:
                 raise ValueError("Unexpected task state: %s" % state)
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
                 import pdb
+
                 pdb.set_trace()
             raise
         finally:
@@ -286,11 +302,20 @@ def maybe_move_task(level, ts, sat, idl, duration, cost_multiplier):
             occ_idl = combined_occupancy(idl)
             occ_sat = combined_occupancy(sat)
 
-            if (occ_idl + cost_multiplier * duration <= occ_sat - duration / 2):
+            if occ_idl + cost_multiplier * duration <= occ_sat - duration / 2:
                 self.move_task_request(ts, sat, idl)
-                log.append((start, level, ts.key, duration,
-                            sat.address, occ_sat,
-                            idl.address, occ_idl))
+                log.append(
+                    (
+                        start,
+                        level,
+                        ts.key,
+                        duration,
+                        sat.address,
+                        occ_sat,
+                        idl.address,
+                        occ_idl,
+                    )
+                )
                 s.check_idle_saturated(sat, occ=occ_sat)
                 s.check_idle_saturated(idl, occ=occ_idl)
 
@@ -306,9 +331,11 @@ def maybe_move_task(level, ts, sat, idl, duration, cost_multiplier):
 
             if not s.saturated:
                 saturated = topk(10, s.workers.values(), key=combined_occupancy)
-                saturated = [ws for ws in saturated
-                             if combined_occupancy(ws) > 0.2
-                             and len(ws.processing) > ws.ncores]
+                saturated = [
+                    ws
+                    for ws in saturated
+                    if combined_occupancy(ws) > 0.2 and len(ws.processing) > ws.ncores
+                ]
             elif len(s.saturated) < 20:
                 saturated = sorted(saturated, key=combined_occupancy, reverse=True)
             if len(idle) < 20:
@@ -323,8 +350,7 @@ def maybe_move_task(level, ts, sat, idl, duration, cost_multiplier):
                         continue
 
                     for ts in list(stealable):
-                        if (ts not in self.key_stealable or
-                                ts.processing_on is not sat):
+                        if ts not in self.key_stealable or ts.processing_on is not sat:
                             stealable.discard(ts)
                             continue
                         i += 1
@@ -337,8 +363,7 @@ def maybe_move_task(level, ts, sat, idl, duration, cost_multiplier):
                             stealable.discard(ts)
                             continue
 
-                        maybe_move_task(level, ts, sat, idl,
-                                        duration, cost_multiplier)
+                        maybe_move_task(level, ts, sat, idl, duration, cost_multiplier)
 
                 if self.cost_multipliers[level] < 20:  # don't steal from public at cost
                     stealable = self.stealable_all[level]
@@ -362,15 +387,14 @@ def maybe_move_task(level, ts, sat, idl, duration, cost_multiplier):
                         idl = idle[i % len(idle)]
                         duration = sat.processing[ts]
 
-                        maybe_move_task(level, ts, sat, idl,
-                                        duration, cost_multiplier)
+                        maybe_move_task(level, ts, sat, idl, duration, cost_multiplier)
 
             if log:
                 self.log.append(log)
                 self.count += 1
             stop = time()
             if s.digests:
-                s.digests['steal-duration'].add(stop - start)
+                s.digests["steal-duration"].add(stop - start)
 
     def restart(self, scheduler):
         for stealable in self.stealable.values():
@@ -394,4 +418,4 @@ def story(self, *keys):
         return out
 
 
-fast_tasks = {'shuffle-split'}
+fast_tasks = {"shuffle-split"}
diff --git a/distributed/submit.py b/distributed/submit.py
index 2d7d62ac1f9..bdbe3251a9d 100644
--- a/distributed/submit.py
+++ b/distributed/submit.py
@@ -18,20 +18,26 @@
 from distributed.utils import get_ip
 
 
-logger = logging.getLogger('distributed.remote')
+logger = logging.getLogger("distributed.remote")
 
 
 class RemoteClient(Server):
-    def __init__(self, ip=None, local_dir=tempfile.mkdtemp(prefix='client-'),
-                 loop=None, security=None, **kwargs):
+    def __init__(
+        self,
+        ip=None,
+        local_dir=tempfile.mkdtemp(prefix="client-"),
+        loop=None,
+        security=None,
+        **kwargs
+    ):
         self.ip = ip or get_ip()
         self.loop = loop or IOLoop.current()
         self.local_dir = local_dir
-        handlers = {'upload_file': self.upload_file, 'execute': self.execute}
+        handlers = {"upload_file": self.upload_file, "execute": self.execute}
 
         self.security = security or Security()
         assert isinstance(self.security, Security)
-        self.listen_args = self.security.get_listen_args('scheduler')
+        self.listen_args = self.security.get_listen_args("scheduler")
 
         super(RemoteClient, self).__init__(handlers, io_loop=self.loop, **kwargs)
 
@@ -46,22 +52,21 @@ def start(self, port=0):
     @gen.coroutine
     def execute(self, stream=None, filename=None):
         script_path = os.path.join(self.local_dir, filename)
-        cmd = '{0} {1}'.format(sys.executable, script_path)
-        process = subprocess.Popen(cmd, shell=True,
-                                   stdout=subprocess.PIPE,
-                                   stderr=subprocess.PIPE)
+        cmd = "{0} {1}".format(sys.executable, script_path)
+        process = subprocess.Popen(
+            cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE
+        )
         out, err = process.communicate()
         return_code = process.returncode
-        raise gen.Return({'stdout': out, 'stderr': err,
-                          'returncode': return_code})
+        raise gen.Return({"stdout": out, "stderr": err, "returncode": return_code})
 
     def upload_file(self, stream, filename=None, file_payload=None):
         out_filename = os.path.join(self.local_dir, filename)
         if isinstance(file_payload, unicode):
             file_payload = file_payload.encode()
-        with open(out_filename, 'wb') as f:
+        with open(out_filename, "wb") as f:
             f.write(file_payload)
-        return {'status': 'OK', 'nbytes': len(file_payload)}
+        return {"status": "OK", "nbytes": len(file_payload)}
 
     @gen.coroutine
     def _close(self):
@@ -70,8 +75,8 @@ def _close(self):
 
 def _remote(host, port, loop=IOLoop.current(), client=RemoteClient):
     host = host or get_ip()
-    if ':' in host and port == 8788:
-        host, port = host.rsplit(':', 1)
+    if ":" in host and port == 8788:
+        host, port = host.rsplit(":", 1)
         port = int(port)
     ip = socket.gethostbyname(host)
     remote_client = client(ip=ip, loop=loop)
@@ -86,8 +91,8 @@ def _remote(host, port, loop=IOLoop.current(), client=RemoteClient):
 def _submit(remote_client_address, filepath, connection_args=None):
     rc = rpc(remote_client_address, connection_args=connection_args)
     remote_file = os.path.basename(filepath)
-    with open(filepath, 'rb') as f:
+    with open(filepath, "rb") as f:
         bytes_read = f.read()
     yield rc.upload_file(filename=remote_file, file_payload=bytes_read)
     result = yield rc.execute(filename=remote_file)
-    raise gen.Return((result['stdout'], result['stderr']))
+    raise gen.Return((result["stdout"], result["stderr"]))
diff --git a/distributed/system_monitor.py b/distributed/system_monitor.py
index 3c68e114b7f..30efc3ceb87 100644
--- a/distributed/system_monitor.py
+++ b/distributed/system_monitor.py
@@ -16,9 +16,7 @@ def __init__(self, n=10000):
         self.memory = deque(maxlen=n)
         self.count = 0
 
-        self.quantities = {'cpu': self.cpu,
-                           'memory': self.memory,
-                           'time': self.time}
+        self.quantities = {"cpu": self.cpu, "memory": self.memory, "time": self.time}
 
         try:
             ioc = psutil.net_io_counters()
@@ -28,14 +26,14 @@ def __init__(self, n=10000):
             self.last_time = time()
             self.read_bytes = deque(maxlen=n)
             self.write_bytes = deque(maxlen=n)
-            self.quantities['read_bytes'] = self.read_bytes
-            self.quantities['write_bytes'] = self.write_bytes
+            self.quantities["read_bytes"] = self.read_bytes
+            self.quantities["write_bytes"] = self.write_bytes
             self._last_io_counters = ioc
             self._collect_net_io_counters = True
 
         if not WINDOWS:
             self.num_fds = deque(maxlen=n)
-            self.quantities['num_fds'] = self.num_fds
+            self.quantities["num_fds"] = self.num_fds
 
         self.update()
 
@@ -56,10 +54,7 @@ def update(self):
         self.time.append(now)
         self.count += 1
 
-        result = {'cpu': cpu,
-                  'memory': memory,
-                  'time': now,
-                  'count': self.count}
+        result = {"cpu": cpu, "memory": memory, "time": now, "count": self.count}
 
         if self._collect_net_io_counters:
             try:
@@ -75,20 +70,22 @@ def update(self):
                 self._last_io_counters = ioc
                 self.read_bytes.append(read_bytes)
                 self.write_bytes.append(write_bytes)
-                result['read_bytes'] = read_bytes
-                result['write_bytes'] = write_bytes
+                result["read_bytes"] = read_bytes
+                result["write_bytes"] = write_bytes
 
         if not WINDOWS:
             num_fds = self.proc.num_fds()
             self.num_fds.append(num_fds)
-            result['num_fds'] = num_fds
+            result["num_fds"] = num_fds
 
         return result
 
     def __repr__(self):
-        return '<SystemMonitor: cpu: %d memory: %d MB fds: %d>' % (
-            self.cpu[-1], self.memory[-1] / 1e6,
-            -1 if WINDOWS else self.num_fds[-1])
+        return "<SystemMonitor: cpu: %d memory: %d MB fds: %d>" % (
+            self.cpu[-1],
+            self.memory[-1] / 1e6,
+            -1 if WINDOWS else self.num_fds[-1],
+        )
 
     def range_query(self, start):
         if start == self.count:
diff --git a/distributed/tests/make_tls_certs.py b/distributed/tests/make_tls_certs.py
index 8ffd62e876d..0c1c5876134 100644
--- a/distributed/tests/make_tls_certs.py
+++ b/distributed/tests/make_tls_certs.py
@@ -78,29 +78,51 @@ def make_cert_key(hostname, sign=False):
     req_file, cert_file, key_file = tempnames
     try:
         req = req_template.format(hostname=hostname)
-        with open(req_file, 'w') as f:
+        with open(req_file, "w") as f:
             f.write(req)
-        args = ['req', '-new', '-days', '3650', '-nodes',
-                '-newkey', 'rsa:2048', '-keyout', key_file,
-                '-config', req_file]
+        args = [
+            "req",
+            "-new",
+            "-days",
+            "3650",
+            "-nodes",
+            "-newkey",
+            "rsa:2048",
+            "-keyout",
+            key_file,
+            "-config",
+            req_file,
+        ]
         if sign:
             with tempfile.NamedTemporaryFile(delete=False) as f:
                 tempnames.append(f.name)
                 reqfile = f.name
-            args += ['-out', reqfile]
+            args += ["-out", reqfile]
 
         else:
-            args += ['-x509', '-out', cert_file]
-        subprocess.check_call(['openssl'] + args)
+            args += ["-x509", "-out", cert_file]
+        subprocess.check_call(["openssl"] + args)
 
         if sign:
-            args = ['ca', '-config', req_file, '-out', cert_file, '-outdir', 'cadir',
-                    '-policy', 'policy_anything', '-batch', '-infiles', reqfile]
-            subprocess.check_call(['openssl'] + args)
-
-        with open(cert_file, 'r') as f:
+            args = [
+                "ca",
+                "-config",
+                req_file,
+                "-out",
+                cert_file,
+                "-outdir",
+                "cadir",
+                "-policy",
+                "policy_anything",
+                "-batch",
+                "-infiles",
+                reqfile,
+            ]
+            subprocess.check_call(["openssl"] + args)
+
+        with open(cert_file, "r") as f:
             cert = f.read()
-        with open(key_file, 'r') as f:
+        with open(key_file, "r") as f:
             key = f.read()
         return cert, key
     finally:
@@ -108,7 +130,7 @@ def make_cert_key(hostname, sign=False):
             os.remove(name)
 
 
-TMP_CADIR = 'cadir'
+TMP_CADIR = "cadir"
 
 
 def unmake_ca():
@@ -117,53 +139,82 @@ def unmake_ca():
 
 def make_ca():
     os.mkdir(TMP_CADIR)
-    with open(os.path.join('cadir', 'index.txt'), 'a+') as f:
+    with open(os.path.join("cadir", "index.txt"), "a+") as f:
         pass  # empty file
     # with open(os.path.join('cadir','crl.txt'),'a+') as f:
-        # f.write("00")
-    with open(os.path.join('cadir', 'index.txt.attr'), 'w+') as f:
-        f.write('unique_subject = no')
+    # f.write("00")
+    with open(os.path.join("cadir", "index.txt.attr"), "w+") as f:
+        f.write("unique_subject = no")
 
     with tempfile.NamedTemporaryFile("w") as t:
-        t.write(req_template.format(hostname='our-ca-server'))
+        t.write(req_template.format(hostname="our-ca-server"))
         t.flush()
         with tempfile.NamedTemporaryFile() as f:
-            args = ['req', '-new', '-days', '3650', '-extensions', 'v3_ca', '-nodes',
-                    '-newkey', 'rsa:2048', '-keyout', 'tls-ca-key.pem',
-                    '-out', f.name,
-                    '-subj', '/C=XY/L=Dask-distributed/O=Dask CA/CN=our-ca-server']
-            subprocess.check_call(['openssl'] + args)
-            args = ['ca', '-config', t.name, '-create_serial',
-                    '-out', 'tls-ca-cert.pem', '-batch', '-outdir', TMP_CADIR,
-                    '-keyfile', 'tls-ca-key.pem', '-days', '3650',
-                    '-selfsign', '-extensions', 'v3_ca', '-infiles', f.name]
-            subprocess.check_call(['openssl'] + args)
-            #args = ['ca', '-config', t.name, '-gencrl', '-out', 'revocation.crl']
-            #subprocess.check_call(['openssl'] + args)
-
-
-if __name__ == '__main__':
+            args = [
+                "req",
+                "-new",
+                "-days",
+                "3650",
+                "-extensions",
+                "v3_ca",
+                "-nodes",
+                "-newkey",
+                "rsa:2048",
+                "-keyout",
+                "tls-ca-key.pem",
+                "-out",
+                f.name,
+                "-subj",
+                "/C=XY/L=Dask-distributed/O=Dask CA/CN=our-ca-server",
+            ]
+            subprocess.check_call(["openssl"] + args)
+            args = [
+                "ca",
+                "-config",
+                t.name,
+                "-create_serial",
+                "-out",
+                "tls-ca-cert.pem",
+                "-batch",
+                "-outdir",
+                TMP_CADIR,
+                "-keyfile",
+                "tls-ca-key.pem",
+                "-days",
+                "3650",
+                "-selfsign",
+                "-extensions",
+                "v3_ca",
+                "-infiles",
+                f.name,
+            ]
+            subprocess.check_call(["openssl"] + args)
+            # args = ['ca', '-config', t.name, '-gencrl', '-out', 'revocation.crl']
+            # subprocess.check_call(['openssl'] + args)
+
+
+if __name__ == "__main__":
     os.chdir(here)
-    cert, key = make_cert_key('localhost')
-    with open('tls-self-signed-cert.pem', 'w') as f:
+    cert, key = make_cert_key("localhost")
+    with open("tls-self-signed-cert.pem", "w") as f:
         f.write(cert)
-    with open('tls-self-signed-key.pem', 'w') as f:
+    with open("tls-self-signed-key.pem", "w") as f:
         f.write(key)
 
     # For certificate matching tests
     make_ca()
-    with open('tls-ca-cert.pem', 'r') as f:
+    with open("tls-ca-cert.pem", "r") as f:
         ca_cert = f.read()
 
-    cert, key = make_cert_key('localhost', sign=True)
-    with open('tls-cert.pem', 'w') as f:
+    cert, key = make_cert_key("localhost", sign=True)
+    with open("tls-cert.pem", "w") as f:
         f.write(cert)
-    with open('tls-cert-chain.pem', 'w') as f:
+    with open("tls-cert-chain.pem", "w") as f:
         f.write(cert)
         f.write(ca_cert)
-    with open('tls-key.pem', 'w') as f:
+    with open("tls-key.pem", "w") as f:
         f.write(key)
-    with open('tls-key-cert.pem', 'w') as f:
+    with open("tls-key-cert.pem", "w") as f:
         f.write(key)
         f.write(cert)
 
diff --git a/distributed/tests/py3_test_asyncio.py b/distributed/tests/py3_test_asyncio.py
index 3c8629c2ba3..90e20268617 100644
--- a/distributed/tests/py3_test_asyncio.py
+++ b/distributed/tests/py3_test_asyncio.py
@@ -1,7 +1,7 @@
 # flake8: noqa
 import pytest
 
-asyncio = pytest.importorskip('asyncio')
+asyncio = pytest.importorskip("asyncio")
 
 import functools
 from time import time
@@ -50,7 +50,7 @@ async def test_coro_test():
 @coro_test
 async def test_asyncio_start_close():
     async with AioClient(processes=False, dashboard_address=False) as c:
-        assert c.status == 'running'
+        assert c.status == "running"
         # AioClient has installed its AioLoop shim.
         assert isinstance(IOLoop.current(instance=False), BaseAsyncIOLoop)
 
@@ -58,7 +58,7 @@ async def test_asyncio_start_close():
         assert result == 11
 
         await c.close()
-        assert c.status == 'closed'
+        assert c.status == "closed"
         # assert IOLoop.current(instance=False) is None
 
 
@@ -153,25 +153,23 @@ async def test_asyncio_gather():
         assert result == 11
         result = await c.gather([x])
         assert result == [11]
-        result = await c.gather({'x': x, 'y': [y]})
-        assert result == {'x': 11, 'y': [12]}
+        result = await c.gather({"x": x, "y": [y]})
+        assert result == {"x": 11, "y": [12]}
 
 
 @coro_test
 async def test_asyncio_get():
     async with AioClient(processes=False) as c:
-        result = await c.get({'x': (inc, 1)}, 'x')
+        result = await c.get({"x": (inc, 1)}, "x")
         assert result == 2
 
-        result = await c.get({'x': (inc, 1)}, ['x'])
+        result = await c.get({"x": (inc, 1)}, ["x"])
         assert result == [2]
 
         result = await c.get({}, [])
         assert result == []
 
-        result = await c.get({('x', 1): (inc, 1),
-                              ('x', 2): (inc, ('x', 1))},
-                             ('x', 2))
+        result = await c.get({("x", 1): (inc, 1), ("x", 2): (inc, ("x", 1))}, ("x", 2))
         assert result == 3
 
 
@@ -228,7 +226,7 @@ async def test_asyncio_cancel():
         await c.cancel([x])
 
         assert x.cancelled()
-        assert 'cancel' in str(x)
+        assert "cancel" in str(x)
         s.validate_state()
 
         start = time()
@@ -244,7 +242,7 @@ async def test_asyncio_cancel():
 @coro_test
 async def test_asyncio_cancel_tuple_key():
     async with AioClient(processes=False) as c:
-        x = c.submit(inc, 1, key=('x', 0, 1))
+        x = c.submit(inc, 1, key=("x", 0, 1))
         await x
         await c.cancel(x)
         with pytest.raises(CancelledError):
@@ -298,7 +296,7 @@ async def aioinc(x, delay=0.02):
 
     async def aiothrows(x, delay=0.02):
         await asyncio.sleep(delay)
-        raise RuntimeError('hello')
+        raise RuntimeError("hello")
 
     async with AioClient(processes=False) as c:
         results = await c.run_coroutine(aioinc, 1, delay=0.05)
@@ -317,7 +315,7 @@ async def aiothrows(x, delay=0.02):
 @coro_test
 async def test_asyncio_restart():
     async with AioClient(processes=False) as c:
-        assert c.status == 'running'
+        assert c.status == "running"
         x = c.submit(inc, 1)
         assert x.key in c.refcount
 
@@ -327,6 +325,7 @@ async def test_asyncio_restart():
         key = x.key
         del x
         import gc
+
         gc.collect()
 
         assert key not in c.refcount
@@ -343,8 +342,8 @@ async def test_asyncio_variable():
     async with AioClient(processes=False) as c:
         s = c.cluster.scheduler
 
-        x = Variable('x')
-        xx = Variable('x')
+        x = Variable("x")
+        xx = Variable("x")
         assert x.client is c
 
         future = c.submit(inc, 1)
diff --git a/distributed/tests/py3_test_client.py b/distributed/tests/py3_test_client.py
index d75b2bd0801..b5d10f8d553 100644
--- a/distributed/tests/py3_test_client.py
+++ b/distributed/tests/py3_test_client.py
@@ -60,7 +60,7 @@ async def f():
     yield f()
 
     assert set(results) == set(range(1, 11))
-    assert not s.counters['op'].components[0]['gather']
+    assert not s.counters["op"].components[0]["gather"]
 
 
 @gen_cluster(client=True)
@@ -102,16 +102,16 @@ async def f():
     loop.run_sync(f)
 
     assert result == 11
-    assert client.status == 'closed'
-    assert cluster.status == 'closed'
+    assert client.status == "closed"
+    assert cluster.status == "closed"
 
 
 def test_locks(loop):
     async def f():
         async with Client(processes=False, asynchronous=True) as c:
             assert c.asynchronous
-            async with Lock('x'):
-                lock2 = Lock('x')
+            async with Lock("x"):
+                lock2 = Lock("x")
                 result = await lock2.acquire(timeout=0.1)
                 assert result is False
 
@@ -124,7 +124,7 @@ async def ff():
         return 1
 
     with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
+        with Client(s["address"], loop=loop) as c:
             assert sync(loop, ff) == 1
             assert c.sync(ff) == 1
 
@@ -132,8 +132,8 @@ async def ff():
 @pytest.mark.xfail(reason="known intermittent failure")
 @gen_cluster(client=True)
 async def test_dont_hold_on_to_large_messages(c, s, a, b):
-    np = pytest.importorskip('numpy')
-    da = pytest.importorskip('dask.array')
+    np = pytest.importorskip("numpy")
+    da = pytest.importorskip("dask.array")
     x = np.random.random(1000000)
     xr = weakref.ref(x)
 
@@ -146,6 +146,7 @@ async def test_dont_hold_on_to_large_messages(c, s, a, b):
         if time() > start + 5:
             # Help diagnosing
             from types import FrameType
+
             x = xr()
             if x is not None:
                 del x
@@ -154,8 +155,12 @@ async def test_dont_hold_on_to_large_messages(c, s, a, b):
                 print("refs to x:", rc, refs, gc.isenabled())
                 frames = [r for r in refs if isinstance(r, FrameType)]
                 for i, f in enumerate(frames):
-                    print("frames #%d:" % i,
-                          f.f_code.co_name, f.f_code.co_filename, sorted(f.f_locals))
+                    print(
+                        "frames #%d:" % i,
+                        f.f_code.co_name,
+                        f.f_code.co_filename,
+                        sorted(f.f_locals),
+                    )
             pytest.fail("array should have been destroyed")
 
         await gen.sleep(0.200)
@@ -165,41 +170,41 @@ async def test_dont_hold_on_to_large_messages(c, s, a, b):
 async def test_run_scheduler_async_def(c, s, a, b):
     async def f(dask_scheduler):
         await gen.sleep(0.01)
-        dask_scheduler.foo = 'bar'
+        dask_scheduler.foo = "bar"
 
     await c.run_on_scheduler(f)
 
-    assert s.foo == 'bar'
+    assert s.foo == "bar"
 
     async def f(dask_worker):
         await gen.sleep(0.01)
-        dask_worker.foo = 'bar'
+        dask_worker.foo = "bar"
 
     await c.run(f)
-    assert a.foo == 'bar'
-    assert b.foo == 'bar'
+    assert a.foo == "bar"
+    assert b.foo == "bar"
 
 
 @gen_cluster(client=True)
 async def test_run_scheduler_async_def_wait(c, s, a, b):
     async def f(dask_scheduler):
         await gen.sleep(0.01)
-        dask_scheduler.foo = 'bar'
+        dask_scheduler.foo = "bar"
 
     await c.run_on_scheduler(f, wait=False)
 
-    while not hasattr(s, 'foo'):
+    while not hasattr(s, "foo"):
         await gen.sleep(0.01)
-    assert s.foo == 'bar'
+    assert s.foo == "bar"
 
     async def f(dask_worker):
         await gen.sleep(0.01)
-        dask_worker.foo = 'bar'
+        dask_worker.foo = "bar"
 
     await c.run(f, wait=False)
 
-    while not hasattr(a, 'foo') or not hasattr(b, 'foo'):
+    while not hasattr(a, "foo") or not hasattr(b, "foo"):
         await gen.sleep(0.01)
 
-    assert a.foo == 'bar'
-    assert b.foo == 'bar'
+    assert a.foo == "bar"
+    assert b.foo == "bar"
diff --git a/distributed/tests/py3_test_pubsub.py b/distributed/tests/py3_test_pubsub.py
index b7cde193d37..172c8734819 100644
--- a/distributed/tests/py3_test_pubsub.py
+++ b/distributed/tests/py3_test_pubsub.py
@@ -10,7 +10,7 @@
 @gen_cluster(client=True)
 def test_basic(c, s, a, b):
     async def publish():
-        pub = Pub('a')
+        pub = Pub("a")
 
         i = 0
         while True:
@@ -19,7 +19,7 @@ async def publish():
             i += 1
 
     def f(_):
-        sub = Sub('a')
+        sub = Sub("a")
         return list(toolz.take(5, sub))
 
     c.run_coroutine(publish, workers=[a.address])
diff --git a/distributed/tests/test_actor.py b/distributed/tests/test_actor.py
index 942d3f1e761..fba0f50cbfe 100644
--- a/distributed/tests/test_actor.py
+++ b/distributed/tests/test_actor.py
@@ -47,21 +47,21 @@ def get(self, key):
         return self.data[key]
 
 
-@pytest.mark.parametrize('direct_to_workers', [True, False])
+@pytest.mark.parametrize("direct_to_workers", [True, False])
 def test_client_actions(direct_to_workers):
-
     @gen_cluster(client=True)
     def test(c, s, a, b):
-        c = yield Client(s.address, asynchronous=True,
-                         direct_to_workers=direct_to_workers)
+        c = yield Client(
+            s.address, asynchronous=True, direct_to_workers=direct_to_workers
+        )
 
         counter = c.submit(Counter, workers=[a.address], actor=True)
         assert isinstance(counter, Future)
         counter = yield counter
         assert counter._address
-        assert hasattr(counter, 'increment')
-        assert hasattr(counter, 'add')
-        assert hasattr(counter, 'n')
+        assert hasattr(counter, "increment")
+        assert hasattr(counter, "add")
+        assert hasattr(counter, "n")
 
         n = yield counter.n
         assert n == 0
@@ -86,9 +86,8 @@ def test(c, s, a, b):
     test()
 
 
-@pytest.mark.parametrize('separate_thread', [False, True])
+@pytest.mark.parametrize("separate_thread", [False, True])
 def test_worker_actions(separate_thread):
-
     @gen_cluster(client=True)
     def test(c, s, a, b):
         counter = c.submit(Counter, workers=[a.address], actor=True)
@@ -121,15 +120,17 @@ def test_Actor(c, s, a, b):
 
     assert counter._cls == Counter
 
-    assert hasattr(counter, 'n')
-    assert hasattr(counter, 'increment')
-    assert hasattr(counter, 'add')
+    assert hasattr(counter, "n")
+    assert hasattr(counter, "increment")
+    assert hasattr(counter, "add")
 
-    assert not hasattr(counter, 'abc')
+    assert not hasattr(counter, "abc")
 
 
-@pytest.mark.xfail(reason="Tornado can pass things out of order" +
-        "Should rely on sending small messages rather than rpc")
+@pytest.mark.xfail(
+    reason="Tornado can pass things out of order"
+    + "Should rely on sending small messages rather than rpc"
+)
 @gen_cluster(client=True)
 def test_linear_access(c, s, a, b):
     start = time()
@@ -159,7 +160,7 @@ class Foo(object):
         x = 0
 
         def __init__(self):
-            raise ValueError('bar')
+            raise ValueError("bar")
 
     with pytest.raises(ValueError) as info:
         future = yield c.submit(Foo, actor=True)
@@ -250,11 +251,11 @@ def test_sync(client):
 
     assert future.result() == future.result()
 
-    assert 'ActorFuture' in repr(future)
-    assert 'distributed.actor' not in repr(future)
+    assert "ActorFuture" in repr(future)
+    assert "distributed.actor" not in repr(future)
 
 
-@gen_cluster(client=True, config={'distributed.comm.timeouts.connect': '1s'})
+@gen_cluster(client=True, config={"distributed.comm.timeouts.connect": "1s"})
 def test_failed_worker(c, s, a, b):
     future = c.submit(Counter, actor=True, workers=[a.address])
     yield wait(future)
@@ -280,21 +281,21 @@ def bench(c, s, a, b):
 
 @gen_cluster(client=True)
 def test_numpy_roundtrip(c, s, a, b):
-    np = pytest.importorskip('numpy')
+    np = pytest.importorskip("numpy")
 
     server = yield c.submit(ParameterServer, actor=True)
 
     x = np.random.random(1000)
-    yield server.put('x', x)
+    yield server.put("x", x)
 
-    y = yield server.get('x')
+    y = yield server.get("x")
 
     assert (x == y).all()
 
 
 @gen_cluster(client=True)
 def test_numpy_roundtrip_getattr(c, s, a, b):
-    np = pytest.importorskip('numpy')
+    np = pytest.importorskip("numpy")
 
     counter = yield c.submit(Counter, actor=True)
 
@@ -311,10 +312,10 @@ def test_numpy_roundtrip_getattr(c, s, a, b):
 def test_repr(c, s, a, b):
     counter = yield c.submit(Counter, actor=True)
 
-    assert 'Counter' in repr(counter)
-    assert 'Actor' in repr(counter)
+    assert "Counter" in repr(counter)
+    assert "Actor" in repr(counter)
     assert counter.key in repr(counter)
-    assert 'distributed.actor' not in repr(counter)
+    assert "distributed.actor" not in repr(counter)
 
 
 @gen_cluster(client=True)
@@ -324,7 +325,7 @@ def test_dir(c, s, a, b):
     d = set(dir(counter))
 
     for attr in dir(Counter):
-        if not attr.startswith('_'):
+        if not attr.startswith("_"):
             assert attr in d
 
 
@@ -346,7 +347,7 @@ def add(n, counter):
     yield done
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 5)] * 2)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 5)] * 2)
 def test_thread_safety(c, s, a, b):
     class Unsafe(object):
         def __init__(self):
@@ -381,7 +382,7 @@ class Foo(object):
         def __init__(self, x):
             pass
 
-    b = c.submit(operator.mul, 'b', 1000000)
+    b = c.submit(operator.mul, "b", 1000000)
     yield wait(b)
     [ws] = s.tasks[b.key].who_has
 
@@ -393,13 +394,13 @@ def __init__(self, x):
     assert s.tasks[x.key].who_has != s.tasks[y.key].who_has  # second load balanced
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 5)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 5)
 def test_load_balance_map(c, s, *workers):
     class Foo(object):
         def __init__(self, x, y=None):
             pass
 
-    b = c.submit(operator.mul, 'b', 1000000)
+    b = c.submit(operator.mul, "b", 1000000)
     yield wait(b)
 
     actors = c.map(Foo, range(10), y=b, actor=True)
@@ -408,10 +409,11 @@ def __init__(self, x, y=None):
     assert all(len(w.actors) == 2 for w in workers)
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 4, Worker=Nanny)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 4, Worker=Nanny)
 def bench_param_server(c, s, *workers):
     import dask.array as da
     import numpy as np
+
     x = da.random.random((500000, 1000), chunks=(1000, 1000))
     x = x.persist()
     yield wait(x)
@@ -439,6 +441,7 @@ def f(block, ps=None):
         return np.array([[stop - start]])
 
     from distributed.utils import format_time
+
     start = time()
     ps = yield c.submit(ParameterServer, x.shape[1], actor=True)
     y = x.map_blocks(f, ps=ps, dtype=x.dtype)
@@ -448,10 +451,9 @@ def f(block, ps=None):
     print(format_time(end - start))
 
 
-@pytest.mark.xfail(reason='unknown')
+@pytest.mark.xfail(reason="unknown")
 @gen_cluster(client=True)
 def test_compute(c, s, a, b):
-
     @dask.delayed
     def f(n, counter):
         assert isinstance(counter, Actor)
@@ -502,8 +504,11 @@ def check(dask_worker):
         assert time() < start + 2
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)],
-             config={'distributed.worker.profile.interval': '1ms'})
+@gen_cluster(
+    client=True,
+    ncores=[("127.0.0.1", 1)],
+    config={"distributed.worker.profile.interval": "1ms"},
+)
 def test_actors_in_profile(c, s, a):
     class Sleeper(object):
         def sleep(self, time):
@@ -513,8 +518,10 @@ def sleep(self, time):
 
     for i in range(5):
         yield sleeper.sleep(0.200)
-        if (list(a.profile_recent['children'])[0].startswith('sleep') or
-                'Sleeper.sleep' in a.profile_keys):
+        if (
+            list(a.profile_recent["children"])[0].startswith("sleep")
+            or "Sleeper.sleep" in a.profile_keys
+        ):
             return
     assert False, list(a.profile_keys)
 
diff --git a/distributed/tests/test_as_completed.py b/distributed/tests/test_as_completed.py
index aa2bfaca765..8e66b58dd4e 100644
--- a/distributed/tests/test_as_completed.py
+++ b/distributed/tests/test_as_completed.py
@@ -168,8 +168,8 @@ def test_as_completed_error_async(c, s, a, b):
     result = {first, second}
 
     assert result == {x, y}
-    assert x.status == 'error'
-    assert y.status == 'finished'
+    assert x.status == "error"
+    assert y.status == "finished"
 
 
 def test_as_completed_error(client):
@@ -180,8 +180,8 @@ def test_as_completed_error(client):
     result = set(ac)
 
     assert result == {x, y}
-    assert x.status == 'error'
-    assert y.status == 'finished'
+    assert x.status == "error"
+    assert y.status == "finished"
 
 
 def test_as_completed_with_results(client):
@@ -193,7 +193,7 @@ def test_as_completed_with_results(client):
     y.cancel()
     with pytest.raises(RuntimeError) as exc:
         res = list(ac)
-    assert str(exc.value) == 'hello!'
+    assert str(exc.value) == "hello!"
 
 
 @gen_cluster(client=True)
@@ -208,7 +208,7 @@ def test_as_completed_with_results_async(c, s, a, b):
         first = yield ac.__anext__()
         second = yield ac.__anext__()
         third = yield ac.__anext__()
-    assert str(exc.value) == 'hello!'
+    assert str(exc.value) == "hello!"
 
 
 def test_as_completed_with_results_no_raise(client):
@@ -222,9 +222,9 @@ def test_as_completed_with_results_no_raise(client):
 
     dd = {r[0]: r[1:] for r in res}
     assert set(dd.keys()) == {y, x, z}
-    assert x.status == 'error'
-    assert y.status == 'cancelled'
-    assert z.status == 'finished'
+    assert x.status == "error"
+    assert y.status == "cancelled"
+    assert z.status == "finished"
 
     assert isinstance(dd[y][0], CancelledError)
     assert isinstance(dd[x][0][1], RuntimeError)
@@ -246,9 +246,9 @@ def test_as_completed_with_results_no_raise_async(c, s, a, b):
 
     dd = {r[0]: r[1:] for r in res}
     assert set(dd.keys()) == {y, x, z}
-    assert x.status == 'error'
-    assert y.status == 'cancelled'
-    assert z.status == 'finished'
+    assert x.status == "error"
+    assert y.status == "cancelled"
+    assert z.status == "finished"
 
     assert isinstance(dd[y][0], CancelledError)
     assert isinstance(dd[x][0][1], RuntimeError)
diff --git a/distributed/tests/test_asyncprocess.py b/distributed/tests/test_asyncprocess.py
index a30f7654a20..1e7a5d2804f 100644
--- a/distributed/tests/test_asyncprocess.py
+++ b/distributed/tests/test_asyncprocess.py
@@ -116,6 +116,7 @@ def test_simple():
     if wr1() is not None:
         # Help diagnosing
         from types import FrameType
+
         p = wr1()
         if p is not None:
             rc = sys.getrefcount(p)
@@ -124,8 +125,12 @@ def test_simple():
             print("refs to proc:", rc, refs)
             frames = [r for r in refs if isinstance(r, FrameType)]
             for i, f in enumerate(frames):
-                print("frames #%d:" % i,
-                      f.f_code.co_name, f.f_code.co_filename, sorted(f.f_locals))
+                print(
+                    "frames #%d:" % i,
+                    f.f_code.co_name,
+                    f.f_code.co_filename,
+                    sorted(f.f_locals),
+                )
         pytest.fail("AsyncProcess should have been destroyed")
     t1 = time()
     while wr2() is not None:
@@ -139,7 +144,7 @@ def test_simple():
 def test_exitcode():
     q = mp_context.Queue()
 
-    proc = AsyncProcess(target=exit, kwargs={'q': q})
+    proc = AsyncProcess(target=exit, kwargs={"q": q})
     proc.daemon = True
     assert not proc.is_alive()
     assert proc.exitcode is None
@@ -154,7 +159,7 @@ def test_exitcode():
     assert proc.exitcode == 5
 
 
-@pytest.mark.skipif(os.name == 'nt', reason="POSIX only")
+@pytest.mark.skipif(os.name == "nt", reason="POSIX only")
 @gen_test()
 def test_signal():
     proc = AsyncProcess(target=exit_with_signal, args=(signal.SIGINT,))
@@ -274,11 +279,12 @@ def test_child_main_thread():
     q._writer.close()
 
 
-@pytest.mark.skipif(sys.platform.startswith('win'),
-                    reason="num_fds not supported on windows")
+@pytest.mark.skipif(
+    sys.platform.startswith("win"), reason="num_fds not supported on windows"
+)
 @gen_test()
 def test_num_fds():
-    psutil = pytest.importorskip('psutil')
+    psutil = pytest.importorskip("psutil")
 
     # Warm up
     proc = AsyncProcess(target=exit_now)
@@ -324,7 +330,7 @@ def _worker_process(worker_ready, child_pipe):
 
     # The parent exiting should cause this process to os._exit from a monitor
     # thread. This sleep should never return.
-    shorter_timeout = 2.5 # timeout shorter than that in the spawning test.
+    shorter_timeout = 2.5  # timeout shorter than that in the spawning test.
     sleep(shorter_timeout)
 
     # Unreachable if functioning correctly.
@@ -336,11 +342,11 @@ def _parent_process(child_pipe):
 
     The child_alive pipe is held open for as long as the child is alive, and can
     be used to determine if it exited correctly. """
+
     def parent_process_coroutine():
         worker_ready = mp_context.Event()
 
-        worker = AsyncProcess(target=_worker_process,
-                              args=(worker_ready, child_pipe))
+        worker = AsyncProcess(target=_worker_process, args=(worker_ready, child_pipe))
 
         yield worker.start()
 
@@ -394,7 +400,7 @@ def test_asyncprocess_child_teardown_on_parent_exit():
         # when the child is ready to enter the sleep, so all of the slow things
         # (process startup, etc) should have happened by now, even on a busy
         # system. A short timeout should therefore be appropriate.
-        short_timeout = 5.
+        short_timeout = 5.0
         # Poll is used to allow other tests to proceed after this one in case of
         # test failure.
         try:
@@ -402,7 +408,7 @@ def test_asyncprocess_child_teardown_on_parent_exit():
         except EnvironmentError:
             # Windows can raise BrokenPipeError. EnvironmentError is caught for
             # Python2/3 portability.
-            assert sys.platform.startswith('win'), "should only raise on windows"
+            assert sys.platform.startswith("win"), "should only raise on windows"
             # Broken pipe implies closed, which is readable.
             readable = True
 
@@ -415,11 +421,11 @@ def test_asyncprocess_child_teardown_on_parent_exit():
             # This won't block due to the above 'assert readable'.
             result = children_alive.recv()
         except EOFError:
-            pass # Test passes.
+            pass  # Test passes.
         except EnvironmentError:
             # Windows can raise BrokenPipeError. EnvironmentError is caught for
             # Python2/3 portability.
-            assert sys.platform.startswith('win'), "should only raise on windows"
+            assert sys.platform.startswith("win"), "should only raise on windows"
             # Test passes.
         else:
             # Oops, children_alive read something. It should be closed. If
diff --git a/distributed/tests/test_batched.py b/distributed/tests/test_batched.py
index 386de5957cb..2f22134f7ae 100644
--- a/distributed/tests/test_batched.py
+++ b/distributed/tests/test_batched.py
@@ -28,7 +28,7 @@ def handle_comm(self, comm):
                 return
 
     def listen(self):
-        listener = listen('', self.handle_comm)
+        listener = listen("", self.handle_comm)
         listener.start()
         self.address = listener.contact_address
         self.stop = listener.stop
@@ -57,17 +57,17 @@ def test_BatchedSend():
 
         yield gen.sleep(0.020)
 
-        b.send('hello')
-        b.send('hello')
-        b.send('world')
+        b.send("hello")
+        b.send("hello")
+        b.send("world")
         yield gen.sleep(0.020)
-        b.send('HELLO')
-        b.send('HELLO')
+        b.send("HELLO")
+        b.send("HELLO")
 
         result = yield comm.read()
-        assert result == ('hello', 'hello', 'world')
+        assert result == ("hello", "hello", "world")
         result = yield comm.read()
-        assert result == ('HELLO', 'HELLO')
+        assert result == ("HELLO", "HELLO")
 
         assert b.byte_count > 1
 
@@ -79,12 +79,12 @@ def test_send_before_start():
 
         b = BatchedSend(interval=10)
 
-        b.send('hello')
-        b.send('world')
+        b.send("hello")
+        b.send("world")
 
         b.start(comm)
         result = yield comm.read()
-        assert result == ('hello', 'world')
+        assert result == ("hello", "world")
 
 
 @gen_test()
@@ -95,12 +95,12 @@ def test_send_after_stream_start():
         b = BatchedSend(interval=10)
 
         b.start(comm)
-        b.send('hello')
-        b.send('world')
+        b.send("hello")
+        b.send("world")
         result = yield comm.read()
         if len(result) < 2:
             result += yield comm.read()
-        assert result == ('hello', 'world')
+        assert result == ("hello", "world")
 
 
 @gen_test()
@@ -112,8 +112,8 @@ def test_send_before_close():
         b.start(comm)
 
         cnt = int(e.count)
-        b.send('hello')
-        yield b.close()         # close immediately after sending
+        b.send("hello")
+        yield b.close()  # close immediately after sending
         assert not b.buffer
 
         start = time()
@@ -122,7 +122,7 @@ def test_send_before_close():
             assert time() < start + 5
 
         with pytest.raises(CommClosedError):
-            b.send('123')
+            b.send("123")
 
 
 @gen_test()
@@ -137,8 +137,8 @@ def test_close_closed():
         comm.close()  # external closing
 
         yield b.close()
-        assert 'closed' in repr(b)
-        assert 'closed' in str(b)
+        assert "closed" in repr(b)
+        assert "closed" in str(b)
 
 
 @gen_test()
@@ -191,18 +191,19 @@ def recv():
 @gen.coroutine
 def run_traffic_jam(nsends, nbytes):
     # This test eats `nsends * nbytes` bytes in RAM
-    np = pytest.importorskip('numpy')
+    np = pytest.importorskip("numpy")
     from distributed.protocol import to_serialize
-    data = bytes(np.random.randint(0, 255, size=(nbytes,)).astype('u1').data)
+
+    data = bytes(np.random.randint(0, 255, size=(nbytes,)).astype("u1").data)
     with echo_server() as e:
         comm = yield connect(e.address)
 
         b = BatchedSend(interval=0.01)
         b.start(comm)
 
-        msg = {'x': to_serialize(data)}
+        msg = {"x": to_serialize(data)}
         for i in range(nsends):
-            b.send(assoc(msg, 'i', i))
+            b.send(assoc(msg, "i", i))
             if np.random.random() > 0.5:
                 yield gen.sleep(0.001)
 
@@ -214,7 +215,7 @@ def run_traffic_jam(nsends, nbytes):
             # loses some of our messages
             L = yield gen.with_timeout(timedelta(seconds=5), comm.read())
             count += 1
-            results.extend(r['i'] for r in L)
+            results.extend(r["i"] for r in L)
 
         assert count == b.batch_count == e.count
         assert b.message_count == nsends
@@ -241,25 +242,25 @@ def test_serializers():
     with echo_server() as e:
         comm = yield connect(e.address)
 
-        b = BatchedSend(interval='10ms', serializers=['msgpack'])
+        b = BatchedSend(interval="10ms", serializers=["msgpack"])
         b.start(comm)
 
-        b.send({'x': to_serialize(123)})
-        b.send({'x': to_serialize('hello')})
+        b.send({"x": to_serialize(123)})
+        b.send({"x": to_serialize("hello")})
         yield gen.sleep(0.100)
 
-        b.send({'x': to_serialize(lambda x: x + 1)})
+        b.send({"x": to_serialize(lambda x: x + 1)})
 
-        with captured_logger('distributed.protocol') as sio:
+        with captured_logger("distributed.protocol") as sio:
             yield gen.sleep(0.100)
 
         value = sio.getvalue()
-        assert 'serialize' in value
-        assert 'type' in value
-        assert 'function' in value
+        assert "serialize" in value
+        assert "type" in value
+        assert "function" in value
 
         msg = yield comm.read()
-        assert list(msg) == [{'x': 123}, {'x': 'hello'}]
+        assert list(msg) == [{"x": 123}, {"x": "hello"}]
 
         with pytest.raises(gen.TimeoutError):
             msg = yield gen.with_timeout(timedelta(milliseconds=100), comm.read())
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index f9594a672b9..634834bf671 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -20,8 +20,7 @@
 import zipfile
 
 import pytest
-from toolz import (identity, isdistinct, concat, pluck, valmap,
-                   partial, first, merge)
+from toolz import identity, isdistinct, concat, pluck, valmap, partial, first, merge
 from tornado import gen
 from tornado.ioloop import IOLoop
 
@@ -29,29 +28,71 @@
 from dask import delayed
 from dask.optimization import SubgraphCallable
 import dask.bag as db
-from distributed import (Worker, Nanny, fire_and_forget, LocalCluster,
-                         get_client, secede, get_worker, Executor, profile,
-                         TimeoutError)
+from distributed import (
+    Worker,
+    Nanny,
+    fire_and_forget,
+    LocalCluster,
+    get_client,
+    secede,
+    get_worker,
+    Executor,
+    profile,
+    TimeoutError,
+)
 from distributed.comm import CommClosedError
-from distributed.client import (Client, Future, wait, as_completed, tokenize,
-                                _get_global_client, default_client,
-                                futures_of,
-                                temp_default_client)
+from distributed.client import (
+    Client,
+    Future,
+    wait,
+    as_completed,
+    tokenize,
+    _get_global_client,
+    default_client,
+    futures_of,
+    temp_default_client,
+)
 from distributed.compatibility import PY3, Iterator
 
 from distributed.metrics import time
 from distributed.scheduler import Scheduler, KilledWorker
 from distributed.sizeof import sizeof
-from distributed.utils import (ignoring, mp_context, sync, tmp_text, tokey,
-                               tmpfile)
-from distributed.utils_test import (cluster, slow, slowinc, slowadd, slowdec,
-                                    randominc, inc, dec, div, throws, geninc, asyncinc,
-                                    gen_cluster, gen_test, double, popen,
-                                    captured_logger, varying, map_varying,
-                                    wait_for, async_wait_for, pristine_loop)
-from distributed.utils_test import (client as c, client_secondary as c2,# noqa F401
-                                    cluster_fixture, loop, loop_in_thread,# noqa F401
-                                    nodebug, s, a, b)  # noqa F401
+from distributed.utils import ignoring, mp_context, sync, tmp_text, tokey, tmpfile
+from distributed.utils_test import (
+    cluster,
+    slow,
+    slowinc,
+    slowadd,
+    slowdec,
+    randominc,
+    inc,
+    dec,
+    div,
+    throws,
+    geninc,
+    asyncinc,
+    gen_cluster,
+    gen_test,
+    double,
+    popen,
+    captured_logger,
+    varying,
+    map_varying,
+    wait_for,
+    async_wait_for,
+    pristine_loop,
+)
+from distributed.utils_test import (  # noqa: F401
+    client as c,
+    client_secondary as c2,
+    cluster_fixture,
+    loop,
+    loop_in_thread,
+    nodebug,
+    s,
+    a,
+    b,
+)
 
 
 @gen_cluster(client=True, timeout=None)
@@ -129,23 +170,25 @@ def test_map_empty(c, s, a, b):
 
 @gen_cluster(client=True)
 def test_map_keynames(c, s, a, b):
-    futures = c.map(inc, range(4), key='INC')
-    assert all(f.key.startswith('INC') for f in futures)
+    futures = c.map(inc, range(4), key="INC")
+    assert all(f.key.startswith("INC") for f in futures)
     assert isdistinct(f.key for f in futures)
 
-    futures2 = c.map(inc, [5, 6, 7, 8], key='INC')
+    futures2 = c.map(inc, [5, 6, 7, 8], key="INC")
     assert [f.key for f in futures] != [f.key for f in futures2]
 
-    keys = ['inc-1', 'inc-2', 'inc-3', 'inc-4']
+    keys = ["inc-1", "inc-2", "inc-3", "inc-4"]
     futures = c.map(inc, range(4), key=keys)
     assert [f.key for f in futures] == keys
 
 
 @gen_cluster(client=True)
 def test_map_retries(c, s, a, b):
-    args = [[ZeroDivisionError("one"), 2, 3],
-            [4, 5, 6],
-            [ZeroDivisionError("seven"), ZeroDivisionError("eight"), 9]]
+    args = [
+        [ZeroDivisionError("one"), 2, 3],
+        [4, 5, 6],
+        [ZeroDivisionError("seven"), ZeroDivisionError("eight"), 9],
+    ]
 
     x, y, z = c.map(*map_varying(args), retries=2)
     assert (yield x) == 2
@@ -287,7 +330,7 @@ def test_persist_retries(c, s, a, b):
 
 @gen_cluster(client=True)
 def test_retries_dask_array(c, s, a, b):
-    da = pytest.importorskip('dask.array')
+    da = pytest.importorskip("dask.array")
     x = da.ones((10, 10), chunks=(3, 3))
     future = c.compute(x.sum(), retries=2)
     y = yield future
@@ -305,7 +348,7 @@ def test_future_repr(c, s, a, b):
 
 @gen_cluster(client=True)
 def test_future_tuple_repr(c, s, a, b):
-    da = pytest.importorskip('dask.array')
+    da = pytest.importorskip("dask.array")
     y = da.arange(10, chunks=(5,)).persist()
     f = futures_of(y)[0]
     for func in [repr, lambda x: x._repr_html_()]:
@@ -371,7 +414,7 @@ def test_Future_release_sync(c):
 
 
 def test_short_tracebacks(loop, c):
-    tblib = pytest.importorskip('tblib')
+    tblib = pytest.importorskip("tblib")
     future = c.submit(div, 1, 0)
     try:
         future.result()
@@ -382,7 +425,7 @@ def test_short_tracebacks(loop, c):
 
     while tb is not None:
         n += 1
-        tb = tb['tb_next']
+        tb = tb["tb_next"]
 
     assert n < 5
 
@@ -435,7 +478,9 @@ def test_gc(s, a, b):
     yield x
     assert s.tasks[x.key].who_has
     x.__del__()
-    yield async_wait_for(lambda: x.key not in s.tasks or not s.tasks[x.key].who_has, timeout=0.3)
+    yield async_wait_for(
+        lambda: x.key not in s.tasks or not s.tasks[x.key].who_has, timeout=0.3
+    )
 
     yield c.close()
 
@@ -474,8 +519,8 @@ def test_gather(c, s, a, b):
     assert result == 11
     result = yield c.gather([x])
     assert result == [11]
-    result = yield c.gather({'x': x, 'y': [y]})
-    assert result == {'x': 11, 'y': [12]}
+    result = yield c.gather({"x": x, "y": [y]})
+    assert result == {"x": 11, "y": [12]}
 
 
 @gen_cluster(client=True)
@@ -498,7 +543,7 @@ def test_gather_sync(c):
     with pytest.raises(ZeroDivisionError):
         c.gather([x, y])
 
-    [xx] = c.gather([x, y], errors='skip')
+    [xx] = c.gather([x, y], errors="skip")
     assert xx == 2
 
 
@@ -510,18 +555,18 @@ def test_gather_strict(c, s, a, b):
     with pytest.raises(ZeroDivisionError):
         yield c.gather([x, y])
 
-    [xx] = yield c.gather([x, y], errors='skip')
+    [xx] = yield c.gather([x, y], errors="skip")
     assert xx == 2
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)])
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)])
 def test_gather_skip(c, s, a):
     x = c.submit(div, 1, 0, priority=10)
     y = c.submit(slowinc, 1, delay=0.5)
 
-    with captured_logger(logging.getLogger('distributed.scheduler')) as sched:
-        with captured_logger(logging.getLogger('distributed.client')) as client:
-            L = yield c.gather([x, y], errors='skip')
+    with captured_logger(logging.getLogger("distributed.scheduler")) as sched:
+        with captured_logger(logging.getLogger("distributed.client")) as client:
+            L = yield c.gather([x, y], errors="skip")
             assert L == [2]
 
     assert not client.getvalue()
@@ -537,12 +582,12 @@ def test_limit_concurrent_gathering(c, s, a, b):
 
 @gen_cluster(client=True, timeout=None)
 def test_get(c, s, a, b):
-    future = c.get({'x': (inc, 1)}, 'x', sync=False)
+    future = c.get({"x": (inc, 1)}, "x", sync=False)
     assert isinstance(future, Future)
     result = yield future
     assert result == 2
 
-    futures = c.get({'x': (inc, 1)}, ['x'], sync=False)
+    futures = c.get({"x": (inc, 1)}, ["x"], sync=False)
     assert isinstance(futures[0], Future)
     result = yield futures
     assert result == [2]
@@ -550,22 +595,25 @@ def test_get(c, s, a, b):
     result = yield c.get({}, [], sync=False)
     assert result == []
 
-    result = yield c.get({('x', 1): (inc, 1), ('x', 2): (inc, ('x', 1))},
-                         ('x', 2), sync=False)
+    result = yield c.get(
+        {("x", 1): (inc, 1), ("x", 2): (inc, ("x", 1))}, ("x", 2), sync=False
+    )
     assert result == 3
 
 
 def test_get_sync(c):
-    assert c.get({'x': (inc, 1)}, 'x') == 2
+    assert c.get({"x": (inc, 1)}, "x") == 2
 
 
 def test_no_future_references(c):
     from weakref import WeakSet
+
     ws = WeakSet()
     futures = c.map(inc, range(10))
     ws.update(futures)
     del futures
     import gc
+
     gc.collect()
     start = time()
     while list(ws):
@@ -606,7 +654,7 @@ def test_wait(c, s, a, b):
 
     assert done == {x, y, z}
     assert not_done == set()
-    assert x.status == y.status == 'finished'
+    assert x.status == y.status == "finished"
 
 
 @gen_cluster(client=True)
@@ -615,13 +663,13 @@ def test_wait_first_completed(c, s, a, b):
     y = c.submit(slowinc, 1)
     z = c.submit(inc, 2)
 
-    done, not_done = yield wait([x, y, z], return_when='FIRST_COMPLETED')
+    done, not_done = yield wait([x, y, z], return_when="FIRST_COMPLETED")
 
     assert done == {z}
     assert not_done == {x, y}
-    assert z.status == 'finished'
-    assert x.status == 'pending'
-    assert y.status == 'pending'
+    assert z.status == "finished"
+    assert x.status == "pending"
+    assert y.status == "pending"
 
 
 @gen_cluster(client=True, timeout=2)
@@ -638,7 +686,7 @@ def test_wait_sync(c):
     done, not_done = wait([x, y])
     assert done == {x, y}
     assert not_done == set()
-    assert x.status == y.status == 'finished'
+    assert x.status == y.status == "finished"
 
     future = c.submit(sleep, 0.3)
     with pytest.raises(gen.TimeoutError):
@@ -683,7 +731,7 @@ def test_garbage_collection(c, s, a, b):
 def test_garbage_collection_with_scatter(c, s, a, b):
     [future] = yield c.scatter([1])
     assert future.key in c.futures
-    assert future.status == 'finished'
+    assert future.status == "finished"
     assert s.who_wants[future.key] == {c.id}
 
     key = future.key
@@ -708,6 +756,7 @@ def test_recompute_released_key(c, s, a, b):
     xkey = x.key
     del x
     import gc
+
     gc.collect()
     yield gen.moment
     assert c.refcount[xkey] == 0
@@ -726,6 +775,7 @@ def test_recompute_released_key(c, s, a, b):
 @gen_cluster(client=True)
 def test_long_tasks_dont_trigger_timeout(c, s, a, b):
     from time import sleep
+
     x = c.submit(sleep, 3)
     yield x
 
@@ -812,9 +862,10 @@ def test_tokenize_on_futures(c, s, a, b):
     assert tok == tokenize(y)
 
 
-@pytest.mark.skipif(not sys.platform.startswith('linux'),
-                    reason="Need 127.0.0.2 to mean localhost")
-@gen_cluster([('127.0.0.1', 1), ('127.0.0.2', 2)], client=True)
+@pytest.mark.skipif(
+    not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
+)
+@gen_cluster([("127.0.0.1", 1), ("127.0.0.2", 2)], client=True)
 def test_restrictions_submit(c, s, a, b):
     x = c.submit(inc, 1, workers={a.ip})
     y = c.submit(inc, x, workers={b.ip})
@@ -840,9 +891,10 @@ def test_restrictions_ip_port(c, s, a, b):
     assert y.key in b.data
 
 
-@pytest.mark.skipif(not sys.platform.startswith('linux'),
-                    reason="Need 127.0.0.2 to mean localhost")
-@gen_cluster([('127.0.0.1', 1), ('127.0.0.2', 2)], client=True)
+@pytest.mark.skipif(
+    not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
+)
+@gen_cluster([("127.0.0.1", 1), ("127.0.0.2", 2)], client=True)
 def test_restrictions_map(c, s, a, b):
     L = c.map(inc, range(5), workers={a.ip})
     yield wait(L)
@@ -852,9 +904,7 @@ def test_restrictions_map(c, s, a, b):
     for x in L:
         assert s.host_restrictions[x.key] == {a.ip}
 
-    L = c.map(inc, [10, 11, 12], workers=[{a.ip},
-                                          {a.ip, b.ip},
-                                          {b.ip}])
+    L = c.map(inc, [10, 11, 12], workers=[{a.ip}, {a.ip, b.ip}, {b.ip}])
     yield wait(L)
 
     assert s.host_restrictions[L[0].key] == {a.ip}
@@ -865,28 +915,29 @@ def test_restrictions_map(c, s, a, b):
         c.map(inc, [10, 11, 12], workers=[{a.ip}])
 
 
-@pytest.mark.skipif(not sys.platform.startswith('linux'),
-                    reason="Need 127.0.0.2 to mean localhost")
-@gen_cluster([('127.0.0.1', 1), ('127.0.0.2', 2)], client=True)
+@pytest.mark.skipif(
+    not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
+)
+@gen_cluster([("127.0.0.1", 1), ("127.0.0.2", 2)], client=True)
 def test_restrictions_get(c, s, a, b):
-    dsk = {'x': 1, 'y': (inc, 'x'), 'z': (inc, 'y')}
-    restrictions = {'y': {a.ip}, 'z': {b.ip}}
+    dsk = {"x": 1, "y": (inc, "x"), "z": (inc, "y")}
+    restrictions = {"y": {a.ip}, "z": {b.ip}}
 
-    futures = c.get(dsk, ['y', 'z'], restrictions, sync=False)
+    futures = c.get(dsk, ["y", "z"], restrictions, sync=False)
     result = yield futures
     assert result == [2, 3]
-    assert 'y' in a.data
-    assert 'z' in b.data
+    assert "y" in a.data
+    assert "z" in b.data
 
 
 @gen_cluster(client=True)
 def dont_test_bad_restrictions_raise_exception(c, s, a, b):
-    z = c.submit(inc, 2, workers={'bad-address'})
+    z = c.submit(inc, 2, workers={"bad-address"})
     try:
         yield z
         assert False
     except ValueError as e:
-        assert 'bad-address' in str(e)
+        assert "bad-address" in str(e)
         assert z.key in str(e)
 
 
@@ -903,15 +954,12 @@ def test_remove_worker(c, s, a, b):
     assert result == list(map(inc, range(20)))
 
 
-@gen_cluster(ncores=[('127.0.0.1', 1)], client=True)
+@gen_cluster(ncores=[("127.0.0.1", 1)], client=True)
 def test_errors_dont_block(c, s, w):
-    L = [c.submit(inc, 1),
-         c.submit(throws, 1),
-         c.submit(inc, 2),
-         c.submit(throws, 2)]
+    L = [c.submit(inc, 1), c.submit(throws, 1), c.submit(inc, 2), c.submit(throws, 2)]
 
     start = time()
-    while not (L[0].status == L[2].status == 'finished'):
+    while not (L[0].status == L[2].status == "finished"):
         assert time() < start + 5
         yield gen.sleep(0.01)
 
@@ -978,9 +1026,9 @@ def test_two_consecutive_clients_share_results(s, a, b):
 @gen_cluster(client=True)
 def test_submit_then_get_with_Future(c, s, a, b):
     x = c.submit(slowinc, 1)
-    dsk = {'y': (inc, x)}
+    dsk = {"y": (inc, x)}
 
-    result = yield c.get(dsk, 'y', sync=False)
+    result = yield c.get(dsk, "y", sync=False)
     assert result == 3
 
 
@@ -988,17 +1036,18 @@ def test_submit_then_get_with_Future(c, s, a, b):
 def test_aliases(c, s, a, b):
     x = c.submit(inc, 1)
 
-    dsk = {'y': x}
-    result = yield c.get(dsk, 'y', sync=False)
+    dsk = {"y": x}
+    result = yield c.get(dsk, "y", sync=False)
     assert result == 2
 
 
 @gen_cluster(client=True)
 def test_aliases_2(c, s, a, b):
     dsk_keys = [
-        ({'x': (inc, 1), 'y': 'x', 'z': 'x', 'w': (add, 'y', 'z')}, ['y', 'w']),
-        ({'x': 'y', 'y': 1}, ['x']),
-        ({'x': 1, 'y': 'x', 'z': 'y', 'w': (inc, 'z')}, ['w'])]
+        ({"x": (inc, 1), "y": "x", "z": "x", "w": (add, "y", "z")}, ["y", "w"]),
+        ({"x": "y", "y": 1}, ["x"]),
+        ({"x": 1, "y": "x", "z": "y", "w": (inc, "z")}, ["w"]),
+    ]
     for dsk, keys in dsk_keys:
         result = yield c.get(dsk, keys, sync=False)
         assert list(result) == list(dask.get(dsk, keys))
@@ -1007,14 +1056,13 @@ def test_aliases_2(c, s, a, b):
 
 @gen_cluster(client=True)
 def test__scatter(c, s, a, b):
-    d = yield c.scatter({'y': 20})
-    assert isinstance(d['y'], Future)
-    assert a.data.get('y') == 20 or b.data.get('y') == 20
-    y_who_has = s.get_who_has(keys=['y'])['y']
-    assert (a.address in y_who_has or
-            b.address in y_who_has)
-    assert s.get_nbytes(summary=False) == {'y': sizeof(20)}
-    yy = yield c.gather([d['y']])
+    d = yield c.scatter({"y": 20})
+    assert isinstance(d["y"], Future)
+    assert a.data.get("y") == 20 or b.data.get("y") == 20
+    y_who_has = s.get_who_has(keys=["y"])["y"]
+    assert a.address in y_who_has or b.address in y_who_has
+    assert s.get_nbytes(summary=False) == {"y": sizeof(20)}
+    yy = yield c.gather([d["y"]])
     assert yy == [20]
 
     [x] = yield c.scatter([10])
@@ -1023,12 +1071,14 @@ def test__scatter(c, s, a, b):
     xx = yield c.gather([x])
     x_who_has = s.get_who_has(keys=[x.key])[x.key]
     assert s.tasks[x.key].who_has
-    assert (s.workers[a.address] in s.tasks[x.key].who_has or
-            s.workers[b.address] in s.tasks[x.key].who_has)
-    assert s.get_nbytes(summary=False) == {'y': sizeof(20), x.key: sizeof(10)}
+    assert (
+        s.workers[a.address] in s.tasks[x.key].who_has
+        or s.workers[b.address] in s.tasks[x.key].who_has
+    )
+    assert s.get_nbytes(summary=False) == {"y": sizeof(20), x.key: sizeof(10)}
     assert xx == [10]
 
-    z = c.submit(add, x, d['y'])  # submit works on Future
+    z = c.submit(add, x, d["y"])  # submit works on Future
     result = yield z
     assert result == 10 + 20
     result = yield c.gather([z, x])
@@ -1037,9 +1087,9 @@ def test__scatter(c, s, a, b):
 
 @gen_cluster(client=True)
 def test__scatter_types(c, s, a, b):
-    d = yield c.scatter({'x': 1})
+    d = yield c.scatter({"x": 1})
     assert isinstance(d, dict)
-    assert list(d) == ['x']
+    assert list(d) == ["x"]
 
     for seq in [[1], (1,), {1}, frozenset([1])]:
         L = yield c.scatter(seq)
@@ -1082,7 +1132,7 @@ class MyObj(object):
     @normalize_token.register(MyObj)
     def f(x):
         L.append(x)
-        return 'x'
+        return "x"
 
     obj = MyObj()
 
@@ -1092,9 +1142,9 @@ def f(x):
 
 @gen_cluster(client=True)
 def test_scatter_singletons(c, s, a, b):
-    np = pytest.importorskip('numpy')
-    pd = pytest.importorskip('pandas')
-    for x in [1, np.ones(5), pd.DataFrame({'x': [1, 2, 3]})]:
+    np = pytest.importorskip("numpy")
+    pd = pytest.importorskip("pandas")
+    for x in [1, np.ones(5), pd.DataFrame({"x": [1, 2, 3]})]:
         future = yield c.scatter(x)
         result = yield future
         assert str(result) == str(x)
@@ -1103,7 +1153,7 @@ def test_scatter_singletons(c, s, a, b):
 @gen_cluster(client=True)
 def test_scatter_typename(c, s, a, b):
     future = yield c.scatter(123)
-    assert future.key.startswith('int')
+    assert future.key.startswith("int")
 
 
 @gen_cluster(client=True)
@@ -1118,22 +1168,23 @@ def test_scatter_hash(c, s, a, b):
 
 @gen_cluster(client=True)
 def test_get_releases_data(c, s, a, b):
-    [x] = yield c.get({'x': (inc, 1)}, ['x'], sync=False)
+    [x] = yield c.get({"x": (inc, 1)}, ["x"], sync=False)
     import gc
+
     gc.collect()
 
     start = time()
-    while c.refcount['x']:
+    while c.refcount["x"]:
         yield gen.sleep(0.01)
         assert time() < start + 2
 
 
 def test_Current(s, a, b):
-    with Client(s['address']) as c:
+    with Client(s["address"]) as c:
         assert Client.current() is c
     with pytest.raises(ValueError):
         Client.current()
-    with Client(s['address']) as c:
+    with Client(s["address"]) as c:
         assert Client.current() is c
 
 
@@ -1142,10 +1193,10 @@ def test_global_clients(loop):
     with pytest.raises(ValueError):
         default_client()
     with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
+        with Client(s["address"], loop=loop) as c:
             assert _get_global_client() is c
             assert default_client() is c
-            with Client(s['address'], loop=loop) as f:
+            with Client(s["address"], loop=loop) as f:
                 assert _get_global_client() is f
                 assert default_client() is f
                 assert default_client(c) is c
@@ -1176,13 +1227,13 @@ def test_get_nbytes(c, s, a, b):
     y = c.submit(inc, x)
     yield y
 
-    assert s.get_nbytes(summary=False) == {x.key: sizeof(1),
-                                           y.key: sizeof(2)}
+    assert s.get_nbytes(summary=False) == {x.key: sizeof(1), y.key: sizeof(2)}
 
 
-@pytest.mark.skipif(not sys.platform.startswith('linux'),
-                    reason="Need 127.0.0.2 to mean localhost")
-@gen_cluster([('127.0.0.1', 1), ('127.0.0.2', 2)], client=True)
+@pytest.mark.skipif(
+    not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
+)
+@gen_cluster([("127.0.0.1", 1), ("127.0.0.2", 2)], client=True)
 def test_nbytes_determines_worker(c, s, a, b):
     x = c.submit(identity, 1, workers=[a.ip])
     y = c.submit(identity, tuple(range(100)), workers=[b.ip])
@@ -1207,29 +1258,35 @@ def test_if_intermediates_clear_on_error(c, s, a, b):
 
 @gen_cluster(client=True)
 def test_pragmatic_move_small_data_to_large_data(c, s, a, b):
-    np = pytest.importorskip('numpy')
+    np = pytest.importorskip("numpy")
     lists = c.map(np.ones, [10000] * 10, pure=False)
     sums = c.map(np.sum, lists)
     total = c.submit(sum, sums)
 
     def f(x, y):
         return None
-    s.task_duration['f'] = 0.001
+
+    s.task_duration["f"] = 0.001
     results = c.map(f, lists, [total] * 10)
 
     yield wait([total])
 
     yield wait(results)
 
-    assert sum(s.tasks[r.key].who_has.issubset(s.tasks[l.key].who_has)
-               for l, r in zip(lists, results)) >= 9
+    assert (
+        sum(
+            s.tasks[r.key].who_has.issubset(s.tasks[l.key].who_has)
+            for l, r in zip(lists, results)
+        )
+        >= 9
+    )
 
 
 @gen_cluster(client=True)
 def test_get_with_non_list_key(c, s, a, b):
-    dsk = {('x', 0): (inc, 1), 5: (inc, 2)}
+    dsk = {("x", 0): (inc, 1), 5: (inc, 2)}
 
-    x = yield c.get(dsk, ('x', 0), sync=False)
+    x = yield c.get(dsk, ("x", 0), sync=False)
     y = yield c.get(dsk, 5, sync=False)
     assert x == 2
     assert y == 3
@@ -1237,15 +1294,15 @@ def test_get_with_non_list_key(c, s, a, b):
 
 @gen_cluster(client=True)
 def test_get_with_error(c, s, a, b):
-    dsk = {'x': (div, 1, 0), 'y': (inc, 'x')}
+    dsk = {"x": (div, 1, 0), "y": (inc, "x")}
     with pytest.raises(ZeroDivisionError):
-        yield c.get(dsk, 'y', sync=False)
+        yield c.get(dsk, "y", sync=False)
 
 
 def test_get_with_error_sync(c):
-    dsk = {'x': (div, 1, 0), 'y': (inc, 'x')}
+    dsk = {"x": (div, 1, 0), "y": (inc, "x")}
     with pytest.raises(ZeroDivisionError):
-        c.get(dsk, 'y')
+        c.get(dsk, "y")
 
 
 @gen_cluster(client=True)
@@ -1259,10 +1316,10 @@ def test_directed_scatter(c, s, a, b):
 
 
 def test_directed_scatter_sync(c, s, a, b, loop):
-    futures = c.scatter([1, 2, 3], workers=[b['address']])
+    futures = c.scatter([1, 2, 3], workers=[b["address"]])
     has_what = sync(loop, c.scheduler.has_what)
-    assert len(has_what[b['address']]) == len(futures)
-    assert len(has_what[a['address']]) == 0
+    assert len(has_what[b["address"]]) == len(futures)
+    assert len(has_what[a["address"]]) == 0
 
 
 def test_iterator_scatter(c):
@@ -1283,6 +1340,7 @@ def test_iterator_scatter(c):
 
 def test_queue_scatter(c):
     from distributed.compatibility import Queue
+
     q = Queue()
     for d in range(10):
         q.put(d)
@@ -1295,6 +1353,7 @@ def test_queue_scatter(c):
 
 def test_queue_scatter_gather_maxsize(c):
     from distributed.compatibility import Queue
+
     q = Queue(maxsize=3)
     out = c.scatter(q, maxsize=10)
     assert out.maxsize == 10
@@ -1314,6 +1373,7 @@ def test_queue_scatter_gather_maxsize(c):
 
 def test_queue_gather(c):
     from distributed.compatibility import Queue
+
     q = Queue()
 
     qin = list(range(10))
@@ -1346,7 +1406,7 @@ def test_iterator_gather(c, c2):
     i_out = list(ff)
     assert i_out == i_in
 
-    i_in = ['a', 'b', 'c', StopIteration('f'), StopIteration, 'd', 'c']
+    i_in = ["a", "b", "c", StopIteration("f"), StopIteration, "d", "c"]
 
     g = (d for d in i_in)
     futures = c.scatter(g)
@@ -1365,26 +1425,26 @@ def test_scatter_direct(c, s, a, b):
     future = yield c.scatter(123, direct=True)
     assert future.key in a.data or future.key in b.data
     assert s.tasks[future.key].who_has
-    assert future.status == 'finished'
+    assert future.status == "finished"
     result = yield future
     assert result == 123
-    assert not s.counters['op'].components[0]['scatter']
+    assert not s.counters["op"].components[0]["scatter"]
 
     result = yield future
-    assert not s.counters['op'].components[0]['gather']
+    assert not s.counters["op"].components[0]["gather"]
 
     result = yield c.gather(future)
-    assert not s.counters['op'].components[0]['gather']
+    assert not s.counters["op"].components[0]["gather"]
 
 
 @gen_cluster(client=True)
 def test_scatter_direct_numpy(c, s, a, b):
-    np = pytest.importorskip('numpy')
+    np = pytest.importorskip("numpy")
     x = np.ones(5)
     future = yield c.scatter(x, direct=True)
     result = yield future
     assert np.allclose(x, result)
-    assert not s.counters['op'].components[0]['scatter']
+    assert not s.counters["op"].components[0]["scatter"]
 
 
 @gen_cluster(client=True)
@@ -1392,31 +1452,35 @@ def test_scatter_direct_broadcast(c, s, a, b):
     future2 = yield c.scatter(456, direct=True, broadcast=True)
     assert future2.key in a.data
     assert future2.key in b.data
-    assert s.tasks[future2.key].who_has == {s.workers[a.address],
-                                            s.workers[b.address]}
+    assert s.tasks[future2.key].who_has == {s.workers[a.address], s.workers[b.address]}
     result = yield future2
     assert result == 456
-    assert not s.counters['op'].components[0]['scatter']
+    assert not s.counters["op"].components[0]["scatter"]
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 4)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 4)
 def test_scatter_direct_balanced(c, s, *workers):
     futures = yield c.scatter([1, 2, 3], direct=True)
     assert sorted([len(w.data) for w in workers]) == [0, 1, 1, 1]
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 4)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 4)
 def test_scatter_direct_broadcast_target(c, s, *workers):
-    futures = yield c.scatter([123, 456], direct=True,
-                              workers=workers[0].address)
+    futures = yield c.scatter([123, 456], direct=True, workers=workers[0].address)
     assert futures[0].key in workers[0].data
     assert futures[1].key in workers[0].data
 
-    futures = yield c.scatter([123, 456], direct=True, broadcast=True,
-                              workers=[w.address for w in workers[:3]])
-    assert (f.key in w.data and w.address in s.tasks[f.key].who_has
-            for f in futures
-            for w in workers[:3])
+    futures = yield c.scatter(
+        [123, 456],
+        direct=True,
+        broadcast=True,
+        workers=[w.address for w in workers[:3]],
+    )
+    assert (
+        f.key in w.data and w.address in s.tasks[f.key].who_has
+        for f in futures
+        for w in workers[:3]
+    )
 
 
 @gen_cluster(client=True, ncores=[])
@@ -1425,7 +1489,7 @@ def test_scatter_direct_empty(c, s):
         yield c.scatter(123, direct=True, timeout=0.1)
 
 
-@gen_cluster(client=True, timeout=None, ncores=[('127.0.0.1', 1)] * 5)
+@gen_cluster(client=True, timeout=None, ncores=[("127.0.0.1", 1)] * 5)
 def test_scatter_direct_spread_evenly(c, s, *workers):
     futures = []
     for i in range(10):
@@ -1435,8 +1499,8 @@ def test_scatter_direct_spread_evenly(c, s, *workers):
     assert all(w.data for w in workers)
 
 
-@pytest.mark.parametrize('direct', [True, False])
-@pytest.mark.parametrize('broadcast', [True, False])
+@pytest.mark.parametrize("direct", [True, False])
+@pytest.mark.parametrize("broadcast", [True, False])
 def test_scatter_gather_sync(c, direct, broadcast):
     futures = c.scatter([1, 2, 3], direct=direct, broadcast=broadcast)
     results = c.gather(futures, direct=direct)
@@ -1467,18 +1531,17 @@ def test_traceback(c, s, a, b):
     tb = yield x.traceback()
 
     if sys.version_info[0] >= 3:
-        assert any('x / y' in line
-                   for line in pluck(3, traceback.extract_tb(tb)))
+        assert any("x / y" in line for line in pluck(3, traceback.extract_tb(tb)))
 
 
 @gen_cluster(client=True)
 def test_get_traceback(c, s, a, b):
     try:
-        yield c.get({'x': (div, 1, 0)}, 'x', sync=False)
+        yield c.get({"x": (div, 1, 0)}, "x", sync=False)
     except ZeroDivisionError:
         exc_type, exc_value, exc_traceback = sys.exc_info()
         L = traceback.format_tb(exc_traceback)
-        assert any('x / y' in line for line in L)
+        assert any("x / y" in line for line in L)
 
 
 @gen_cluster(client=True)
@@ -1489,22 +1552,25 @@ def test_gather_traceback(c, s, a, b):
     except ZeroDivisionError:
         exc_type, exc_value, exc_traceback = sys.exc_info()
         L = traceback.format_tb(exc_traceback)
-        assert any('x / y' in line for line in L)
+        assert any("x / y" in line for line in L)
 
 
 def test_traceback_sync(c):
     x = c.submit(div, 1, 0)
     tb = x.traceback()
     if sys.version_info[0] >= 3:
-        assert any('x / y' in line
-                   for line in concat(traceback.extract_tb(tb))
-                   if isinstance(line, str))
+        assert any(
+            "x / y" in line
+            for line in concat(traceback.extract_tb(tb))
+            if isinstance(line, str)
+        )
 
     y = c.submit(inc, x)
     tb2 = y.traceback()
 
     assert set(pluck(3, traceback.extract_tb(tb2))).issuperset(
-        set(pluck(3, traceback.extract_tb(tb))))
+        set(pluck(3, traceback.extract_tb(tb)))
+    )
 
     z = c.submit(div, 1, 2)
     tb = z.traceback()
@@ -1515,11 +1581,12 @@ def test_traceback_sync(c):
 def test_upload_file(c, s, a, b):
     def g():
         import myfile
+
         return myfile.f()
 
     try:
         for value in [123, 456]:
-            with tmp_text('myfile.py', 'def f():\n    return {}'.format(value)) as fn:
+            with tmp_text("myfile.py", "def f():\n    return {}".format(value)) as fn:
                 yield c.upload_file(fn)
 
             x = c.submit(g, pure=False)
@@ -1527,13 +1594,13 @@ def g():
             assert result == value
     finally:
         # Ensure that this test won't impact the others
-        if 'myfile' in sys.modules:
-            del sys.modules['myfile']
+        if "myfile" in sys.modules:
+            del sys.modules["myfile"]
 
 
 @gen_cluster(client=True)
 def test_upload_file_no_extension(c, s, a, b):
-    with tmp_text('myfile', '') as fn:
+    with tmp_text("myfile", "") as fn:
         yield c.upload_file(fn)
 
 
@@ -1541,26 +1608,29 @@ def test_upload_file_no_extension(c, s, a, b):
 def test_upload_file_zip(c, s, a, b):
     def g():
         import myfile
+
         return myfile.f()
 
     try:
         for value in [123, 456]:
-            with tmp_text('myfile.py', 'def f():\n    return {}'.format(value)) as fn_my_file:
-                with zipfile.ZipFile('myfile.zip', 'w') as z:
+            with tmp_text(
+                "myfile.py", "def f():\n    return {}".format(value)
+            ) as fn_my_file:
+                with zipfile.ZipFile("myfile.zip", "w") as z:
                     z.write(fn_my_file, arcname=os.path.basename(fn_my_file))
-                yield c.upload_file('myfile.zip')
+                yield c.upload_file("myfile.zip")
 
                 x = c.submit(g, pure=False)
                 result = yield x
                 assert result == value
     finally:
         # Ensure that this test won't impact the others
-        if os.path.exists('myfile.zip'):
-            os.remove('myfile.zip')
-        if 'myfile' in sys.modules:
-            del sys.modules['myfile']
+        if os.path.exists("myfile.zip"):
+            os.remove("myfile.zip")
+        if "myfile" in sys.modules:
+            del sys.modules["myfile"]
         for path in sys.path:
-            if os.path.basename(path) == 'myfile.zip':
+            if os.path.basename(path) == "myfile.zip":
                 sys.path.remove(path)
                 break
 
@@ -1569,26 +1639,27 @@ def g():
 def test_upload_large_file(c, s, a, b):
     assert a.local_dir
     assert b.local_dir
-    with tmp_text('myfile', 'abc') as fn:
-        with tmp_text('myfile2', 'def') as fn2:
-            yield c._upload_large_file(fn, remote_filename='x')
+    with tmp_text("myfile", "abc") as fn:
+        with tmp_text("myfile2", "def") as fn2:
+            yield c._upload_large_file(fn, remote_filename="x")
             yield c._upload_large_file(fn2)
 
             for w in [a, b]:
-                assert os.path.exists(os.path.join(w.local_dir, 'x'))
-                assert os.path.exists(os.path.join(w.local_dir, 'myfile2'))
-                with open(os.path.join(w.local_dir, 'x')) as f:
-                    assert f.read() == 'abc'
-                with open(os.path.join(w.local_dir, 'myfile2')) as f:
-                    assert f.read() == 'def'
+                assert os.path.exists(os.path.join(w.local_dir, "x"))
+                assert os.path.exists(os.path.join(w.local_dir, "myfile2"))
+                with open(os.path.join(w.local_dir, "x")) as f:
+                    assert f.read() == "abc"
+                with open(os.path.join(w.local_dir, "myfile2")) as f:
+                    assert f.read() == "def"
 
 
 def test_upload_file_sync(c):
     def g():
         import myfile
+
         return myfile.x
 
-    with tmp_text('myfile.py', 'x = 123') as fn:
+    with tmp_text("myfile.py", "x = 123") as fn:
         c.upload_file(fn)
         x = c.submit(g)
         assert x.result() == 123
@@ -1596,13 +1667,13 @@ def g():
 
 @gen_cluster(client=True)
 def test_upload_file_exception(c, s, a, b):
-    with tmp_text('myfile.py', 'syntax-error!') as fn:
+    with tmp_text("myfile.py", "syntax-error!") as fn:
         with pytest.raises(SyntaxError):
             yield c.upload_file(fn)
 
 
 def test_upload_file_exception_sync(c):
-    with tmp_text('myfile.py', 'syntax-error!') as fn:
+    with tmp_text("myfile.py", "syntax-error!") as fn:
         with pytest.raises(SyntaxError):
             c.upload_file(fn)
 
@@ -1633,6 +1704,7 @@ def test_multiple_clients(s, a, b):
 @gen_cluster(client=True)
 def test_async_compute(c, s, a, b):
     from dask.delayed import delayed
+
     x = delayed(1)
     y = delayed(inc)(x)
     z = delayed(dec)(x)
@@ -1651,10 +1723,11 @@ def test_async_compute(c, s, a, b):
 
 @gen_cluster(client=True)
 def test_async_compute_with_scatter(c, s, a, b):
-    d = yield c.scatter({('x', 1): 1, ('y', 1): 2})
-    x, y = d[('x', 1)], d[('y', 1)]
+    d = yield c.scatter({("x", 1): 1, ("y", 1): 2})
+    x, y = d[("x", 1)], d[("y", 1)]
 
     from dask.delayed import delayed
+
     z = delayed(add)(delayed(inc)(x), delayed(inc)(y))
     zz = c.compute(z)
 
@@ -1716,13 +1789,14 @@ def test_client_with_scheduler(c, s, a, b):
     AA, BB, xx = yield c.gather([A, B, x])
     assert (AA, BB, xx) == (1, 2, 2)
 
-    result = yield c.get({'x': (inc, 1), 'y': (add, 'x', 10)}, 'y', sync=False)
+    result = yield c.get({"x": (inc, 1), "y": (add, "x", 10)}, "y", sync=False)
     assert result == 12
 
 
-@pytest.mark.skipif(not sys.platform.startswith('linux'),
-                    reason="Need 127.0.0.2 to mean localhost")
-@gen_cluster([('127.0.0.1', 1), ('127.0.0.2', 2)], client=True)
+@pytest.mark.skipif(
+    not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
+)
+@gen_cluster([("127.0.0.1", 1), ("127.0.0.2", 2)], client=True)
 def test_allow_restrictions(c, s, a, b):
     aws = s.workers[a.address]
     bws = s.workers[a.address]
@@ -1742,13 +1816,13 @@ def test_allow_restrictions(c, s, a, b):
     assert all(s.tasks[f.key].who_has == {aws} for f in L)
     assert {f.key for f in L}.issubset(s.loose_restrictions)
 
-    x = c.submit(inc, 15, workers='127.0.0.3', allow_other_workers=True)
+    x = c.submit(inc, 15, workers="127.0.0.3", allow_other_workers=True)
 
     yield x
     assert s.tasks[x.key].who_has
     assert x.key in s.loose_restrictions
 
-    L = c.map(inc, range(15, 25), workers='127.0.0.3', allow_other_workers=True)
+    L = c.map(inc, range(15, 25), workers="127.0.0.3", allow_other_workers=True)
     yield wait(L)
     assert all(s.tasks[f.key].who_has for f in L)
     assert {f.key for f in L}.issubset(s.loose_restrictions)
@@ -1760,21 +1834,21 @@ def test_allow_restrictions(c, s, a, b):
         c.map(inc, [1], allow_other_workers=True)
 
     with pytest.raises(TypeError):
-        c.submit(inc, 20, workers='127.0.0.1', allow_other_workers='Hello!')
+        c.submit(inc, 20, workers="127.0.0.1", allow_other_workers="Hello!")
 
     with pytest.raises(TypeError):
-        c.map(inc, [20], workers='127.0.0.1', allow_other_workers='Hello!')
+        c.map(inc, [20], workers="127.0.0.1", allow_other_workers="Hello!")
 
 
-@pytest.mark.skipif('True', reason='because')
+@pytest.mark.skipif("True", reason="because")
 def test_bad_address():
     try:
-        Client('123.123.123.123:1234', timeout=0.1)
+        Client("123.123.123.123:1234", timeout=0.1)
     except (IOError, gen.TimeoutError) as e:
         assert "connect" in str(e).lower()
 
     try:
-        Client('127.0.0.1:1234', timeout=0.1)
+        Client("127.0.0.1:1234", timeout=0.1)
     except (IOError, gen.TimeoutError) as e:
         assert "connect" in str(e).lower()
 
@@ -1782,7 +1856,7 @@ def test_bad_address():
 @gen_cluster(client=True)
 def test_long_error(c, s, a, b):
     def bad(x):
-        raise ValueError('a' * 100000)
+        raise ValueError("a" * 100000)
 
     x = c.submit(bad, 10)
 
@@ -1792,9 +1866,11 @@ def bad(x):
         assert len(str(e)) < 100000
 
     tb = yield x.traceback()
-    assert all(len(line) < 100000
-               for line in concat(traceback.extract_tb(tb))
-               if isinstance(line, str))
+    assert all(
+        len(line) < 100000
+        for line in concat(traceback.extract_tb(tb))
+        if isinstance(line, str)
+    )
 
 
 @gen_cluster(client=True)
@@ -1828,6 +1904,7 @@ def __getstate__(self):
     def __setstate__(self, state):
         print("This should never have been deserialized, closing")
         import sys
+
         sys.exit(0)
 
 
@@ -1840,10 +1917,10 @@ def test_badly_serialized_input(c, s, a, b):
 
     L = yield c.gather(futures)
     assert list(L) == list(map(inc, range(10)))
-    assert future.status == 'error'
+    assert future.status == "error"
 
 
-@pytest.mark.skipif('True', reason="")
+@pytest.mark.skipif("True", reason="")
 def test_badly_serialized_input_stderr(capsys, c):
     o = BadlySerializedObject()
     future = c.submit(inc, o)
@@ -1852,24 +1929,24 @@ def test_badly_serialized_input_stderr(capsys, c):
     while True:
         sleep(0.01)
         out, err = capsys.readouterr()
-        if 'hello!' in err:
+        if "hello!" in err:
             break
         assert time() - start < 20
-    assert future.status == 'error'
+    assert future.status == "error"
 
 
 def test_repr(loop):
     funcs = [str, repr, lambda x: x._repr_html_()]
     with cluster(nworkers=3) as (s, [a, b, c]):
-        with Client(s['address'], loop=loop) as c:
+        with Client(s["address"], loop=loop) as c:
             for func in funcs:
                 text = func(c)
                 assert c.scheduler.address in text
-                assert '3' in text
+                assert "3" in text
 
         for func in funcs:
             text = func(c)
-            assert 'not connected' in text
+            assert "not connected" in text
 
 
 @gen_cluster(client=True)
@@ -1879,8 +1956,9 @@ def test_repr_async(c, s, a, b):
 
 @gen_test()
 def test_repr_localcluster():
-    cluster = yield LocalCluster(processes=False, dashboard_address=None,
-                                 asynchronous=True)
+    cluster = yield LocalCluster(
+        processes=False, dashboard_address=None, asynchronous=True
+    )
     client = yield Client(cluster, asynchronous=True)
     try:
         text = client._repr_html_()
@@ -2008,7 +2086,7 @@ def test_repr_sync(c):
     assert c.scheduler.address in s
     assert c.scheduler.address in r
     assert str(2) in s  # nworkers
-    assert 'cores' in s
+    assert "cores" in s
 
 
 @gen_cluster(client=True)
@@ -2038,9 +2116,11 @@ def test_multi_client(s, a, b):
 
     yield wait([x, y])
 
-    assert s.wants_what == {c.id: {x.key, y.key},
-                            f.id: {y.key},
-                            'fire-and-forget': set()}
+    assert s.wants_what == {
+        c.id: {x.key, y.key},
+        f.id: {y.key},
+        "fire-and-forget": set(),
+    }
     assert s.who_wants == {x.key: {c.id}, y.key: {c.id, f.id}}
 
     yield c.close()
@@ -2072,8 +2152,7 @@ def long_running_client_connection(address):
 
 @gen_cluster()
 def test_cleanup_after_broken_client_connection(s, a, b):
-    proc = mp_context.Process(target=long_running_client_connection,
-                              args=(s.address,))
+    proc = mp_context.Process(target=long_running_client_connection, args=(s.address,))
     proc.daemon = True
     proc.start()
 
@@ -2110,9 +2189,7 @@ def test_multi_garbage_collection(s, a, b):
         yield gen.sleep(0.01)
         assert time() < start + 5
 
-    assert s.wants_what == {c.id: {y.key},
-                            f.id: {y.key},
-                            'fire-and-forget': set()}
+    assert s.wants_what == {c.id: {y.key}, f.id: {y.key}, "fire-and-forget": set()}
     assert s.who_wants == {y.key: {c.id, f.id}}
 
     y.__del__()
@@ -2123,9 +2200,7 @@ def test_multi_garbage_collection(s, a, b):
 
     yield gen.sleep(0.1)
     assert y.key in a.data or y.key in b.data
-    assert s.wants_what == {c.id: {y.key},
-                            f.id: set(),
-                            'fire-and-forget': set()}
+    assert s.wants_what == {c.id: {y.key}, f.id: set(), "fire-and-forget": set()}
     assert s.who_wants == {y.key: {c.id}}
 
     y2.__del__()
@@ -2147,7 +2222,7 @@ def test__broadcast(c, s, a, b):
     assert a.data == b.data == {x.key: 1, y.key: 2}
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 4)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 4)
 def test__broadcast_integer(c, s, *workers):
     x, y = yield c.scatter([1, 2], broadcast=2)
     assert len(s.tasks[x.key].who_has) == 2
@@ -2156,8 +2231,8 @@ def test__broadcast_integer(c, s, *workers):
 
 @gen_cluster(client=True)
 def test__broadcast_dict(c, s, a, b):
-    d = yield c.scatter({'x': 1}, broadcast=True)
-    assert a.data == b.data == {'x': 1}
+    d = yield c.scatter({"x": 1}, broadcast=True)
+    assert a.data == b.data == {"x": 1}
 
 
 def test_broadcast(c, s, a, b):
@@ -2166,21 +2241,23 @@ def test_broadcast(c, s, a, b):
     has_what = sync(c.loop, c.scheduler.has_what)
 
     assert {k: set(v) for k, v in has_what.items()} == {
-        a['address']: {x.key, y.key},
-        b['address']: {x.key, y.key}}
+        a["address"]: {x.key, y.key},
+        b["address"]: {x.key, y.key},
+    }
 
-    [z] = c.scatter([3], broadcast=True, workers=[a['address']])
+    [z] = c.scatter([3], broadcast=True, workers=[a["address"]])
 
     has_what = sync(c.loop, c.scheduler.has_what)
     assert {k: set(v) for k, v in has_what.items()} == {
-        a['address']: {x.key, y.key, z.key},
-        b['address']: {x.key, y.key}}
+        a["address"]: {x.key, y.key, z.key},
+        b["address"]: {x.key, y.key},
+    }
 
 
 @gen_cluster(client=True)
 def test_proxy(c, s, a, b):
-    msg = yield c.scheduler.proxy(msg={'op': 'identity'}, worker=a.address)
-    assert msg['id'] == a.identity()['id']
+    msg = yield c.scheduler.proxy(msg={"op": "identity"}, worker=a.address)
+    assert msg["id"] == a.identity()["id"]
 
 
 @gen_cluster(client=True)
@@ -2194,7 +2271,7 @@ def test__cancel(c, s, a, b):
     yield c.cancel([x])
 
     assert x.cancelled()
-    assert 'cancel' in str(x)
+    assert "cancel" in str(x)
     s.validate_state()
 
     start = time()
@@ -2208,7 +2285,7 @@ def test__cancel(c, s, a, b):
 
 @gen_cluster(client=True)
 def test__cancel_tuple_key(c, s, a, b):
-    x = c.submit(inc, 1, key=('x', 0, 1))
+    x = c.submit(inc, 1, key=("x", 0, 1))
 
     result = yield x
     yield c.cancel(x)
@@ -2249,7 +2326,7 @@ def test__cancel_multi_client(s, a, b):
 @gen_cluster(client=True)
 def test__cancel_collection(c, s, a, b):
     L = c.map(double, [[1], [2], [3]])
-    x = db.Bag({('b', i): f for i, f in enumerate(L)}, 'b', 3)
+    x = db.Bag({("b", i): f for i, f in enumerate(L)}, "b", 3)
 
     yield c.cancel(x)
     yield c.cancel([x])
@@ -2258,9 +2335,9 @@ def test__cancel_collection(c, s, a, b):
 
 
 def test_cancel(c):
-    x = c.submit(slowinc, 1, key='x')
-    y = c.submit(slowinc, x, key='y')
-    z = c.submit(slowinc, y, key='z')
+    x = c.submit(slowinc, 1, key="x")
+    y = c.submit(slowinc, x, key="y")
+    z = c.submit(slowinc, y, key="z")
 
     c.cancel([y])
 
@@ -2280,7 +2357,7 @@ def test_future_type(c, s, a, b):
     x = c.submit(inc, 1)
     yield wait([x])
     assert x.type == int
-    assert 'int' in str(x)
+    assert "int" in str(x)
 
 
 @gen_cluster(client=True)
@@ -2292,14 +2369,15 @@ def test_traceback_clean(c, s, a, b):
         f = e
         exc_type, exc_value, tb = sys.exc_info()
         while tb:
-            assert 'scheduler' not in tb.tb_frame.f_code.co_filename
-            assert 'worker' not in tb.tb_frame.f_code.co_filename
+            assert "scheduler" not in tb.tb_frame.f_code.co_filename
+            assert "worker" not in tb.tb_frame.f_code.co_filename
             tb = tb.tb_next
 
 
 @gen_cluster(client=True)
 def test_map_queue(c, s, a, b):
     from distributed.compatibility import Queue, isqueue
+
     q_1 = Queue(maxsize=2)
     q_2 = c.map(inc, q_1)
     assert isqueue(q_2)
@@ -2318,14 +2396,16 @@ def test_map_queue(c, s, a, b):
     assert result == (1 + 1) * 2
 
 
-@pytest.mark.skipif(sys.version_info >= (3, 7),
-                    reason="replace StopIteration with return")
+@pytest.mark.skipif(
+    sys.version_info >= (3, 7), reason="replace StopIteration with return"
+)
 @gen_cluster(client=True)
 def test_map_iterator_with_return(c, s, a, b):
     def g():
         yield 1
         yield 2
         raise StopIteration(3)  # py2.7 compat.
+
     f1 = c.map(lambda x: x, g())
     assert isinstance(f1, Iterator)
 
@@ -2404,11 +2484,11 @@ def test_map_differnet_lengths(c, s, a, b):
 
 def test_Future_exception_sync_2(loop, capsys):
     with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
+        with Client(s["address"], loop=loop) as c:
             assert dask.base.get_scheduler() == c.get
 
     out, err = capsys.readouterr()
-    assert len(out.strip().split('\n')) == 1
+    assert len(out.strip().split("\n")) == 1
 
     assert dask.base.get_scheduler() != c.get
 
@@ -2416,6 +2496,7 @@ def test_Future_exception_sync_2(loop, capsys):
 @gen_cluster(timeout=60, client=True)
 def test_async_persist(c, s, a, b):
     from dask.delayed import delayed, Delayed
+
     x = delayed(1)
     y = delayed(inc)(x)
     z = delayed(dec)(x)
@@ -2447,7 +2528,7 @@ def test_async_persist(c, s, a, b):
 
 @gen_cluster(client=True)
 def test__persist(c, s, a, b):
-    pytest.importorskip('dask.array')
+    pytest.importorskip("dask.array")
     import dask.array as da
 
     x = da.ones((10, 10), chunks=(5, 10))
@@ -2467,8 +2548,9 @@ def test__persist(c, s, a, b):
 
 
 def test_persist(c):
-    pytest.importorskip('dask.array')
+    pytest.importorskip("dask.array")
     import dask.array as da
+
     x = da.ones((10, 10), chunks=(5, 10))
     y = 2 * (x + 1)
     assert len(y.dask) == 6
@@ -2502,7 +2584,7 @@ def deep(n):
 @gen_cluster(client=True)
 def test_wait_on_collections(c, s, a, b):
     L = c.map(double, [[1], [2], [3]])
-    x = db.Bag({('b', i): f for i, f in enumerate(L)}, 'b', 3)
+    x = db.Bag({("b", i): f for i, f in enumerate(L)}, "b", 3)
 
     yield wait(x)
     assert all(f.key in a.data or f.key in b.data for f in L)
@@ -2516,19 +2598,21 @@ def test_futures_of_get(c, s, a, b):
     assert set(futures_of(x)) == {x}
     assert set(futures_of([x, y, z])) == {x, y, z}
     assert set(futures_of([x, [y], [[z]]])) == {x, y, z}
-    assert set(futures_of({'x': x, 'y': [y]})) == {x, y}
+    assert set(futures_of({"x": x, "y": [y]})) == {x, y}
 
-    b = db.Bag({('b', i): f for i, f in enumerate([x, y, z])}, 'b', 3)
+    b = db.Bag({("b", i): f for i, f in enumerate([x, y, z])}, "b", 3)
     assert set(futures_of(b)) == {x, y, z}
 
-    sg = SubgraphCallable({'x': x, 'y': y, 'z': z,
-                           'out': (add, (add, (add, x, y), z), 'in')},
-                          'out', ('in',))
+    sg = SubgraphCallable(
+        {"x": x, "y": y, "z": z, "out": (add, (add, (add, x, y), z), "in")},
+        "out",
+        ("in",),
+    )
     assert set(futures_of(sg)) == {x, y, z}
 
 
 def test_futures_of_class():
-    da = pytest.importorskip('dask.array')
+    da = pytest.importorskip("dask.array")
     assert futures_of([da.Array]) == []
 
 
@@ -2541,7 +2625,7 @@ def test_futures_of_cancelled_raises(c, s, a, b):
         yield x
 
     with pytest.raises(CancelledError):
-        yield c.get({'x': (inc, x), 'y': (inc, 2)}, ['x', 'y'], sync=False)
+        yield c.get({"x": (inc, x), "y": (inc, 2)}, ["x", "y"], sync=False)
 
     with pytest.raises(CancelledError):
         c.submit(inc, x)
@@ -2552,20 +2636,20 @@ def test_futures_of_cancelled_raises(c, s, a, b):
     with pytest.raises(CancelledError):
         c.map(add, [1], y=x)
 
-    assert 'y' not in s.tasks
+    assert "y" not in s.tasks
 
 
 @pytest.mark.skip
-@gen_cluster(ncores=[('127.0.0.1', 1)], client=True)
+@gen_cluster(ncores=[("127.0.0.1", 1)], client=True)
 def test_dont_delete_recomputed_results(c, s, w):
-    x = c.submit(inc, 1)                        # compute first time
+    x = c.submit(inc, 1)  # compute first time
     yield wait([x])
-    x.__del__()                                 # trigger garbage collection
+    x.__del__()  # trigger garbage collection
     yield gen.moment
-    xx = c.submit(inc, 1)                       # compute second time
+    xx = c.submit(inc, 1)  # compute second time
 
     start = time()
-    while xx.key not in w.data:                               # data shows up
+    while xx.key not in w.data:  # data shows up
         yield gen.sleep(0.01)
         assert time() < start + 1
 
@@ -2584,7 +2668,7 @@ def test_fatally_serialized_input(c, s):
         yield gen.sleep(0.01)
 
 
-@pytest.mark.skip(reason='Use fast random selection now')
+@pytest.mark.skip(reason="Use fast random selection now")
 @gen_cluster(client=True)
 def test_balance_tasks_by_stacks(c, s, a, b):
     x = c.submit(inc, 1)
@@ -2614,7 +2698,7 @@ def test_run_handles_picklable_data(c, s, a, b):
     yield wait(futures)
 
     def func():
-        return {}, set(), [], (), 1, 'hello', b'100'
+        return {}, set(), [], (), 1, "hello", b"100"
 
     results = yield c.run_on_scheduler(func)
     assert results == func()
@@ -2628,11 +2712,10 @@ def func(x, y=10):
         return x + y
 
     result = c.run(func, 1, y=2)
-    assert result == {a['address']: 3,
-                      b['address']: 3}
+    assert result == {a["address"]: 3, b["address"]: 3}
 
-    result = c.run(func, 1, y=2, workers=[a['address']])
-    assert result == {a['address']: 3}
+    result = c.run(func, 1, y=2, workers=[a["address"]])
+    assert result == {a["address"]: 3}
 
 
 @gen_cluster(client=True)
@@ -2657,11 +2740,10 @@ def test_run_coroutine(c, s, a, b):
 
 def test_run_coroutine_sync(c, s, a, b):
     result = c.run(geninc, 2, delay=0.01)
-    assert result == {a['address']: 3,
-                      b['address']: 3}
+    assert result == {a["address"]: 3, b["address"]: 3}
 
-    result = c.run(geninc, 2, workers=[a['address']])
-    assert result == {a['address']: 3}
+    result = c.run(geninc, 2, workers=[a["address"]])
+    assert result == {a["address"]: 3}
 
     t1 = time()
     result = c.run(geninc, 2, delay=10, wait=False)
@@ -2676,15 +2758,15 @@ def raise_exception(exc_type, exc_msg):
 
     for exc_type in [ValueError, RuntimeError]:
         with pytest.raises(exc_type) as excinfo:
-            c.run(raise_exception, exc_type, 'informative message')
-        assert 'informative message' in str(excinfo.value)
+            c.run(raise_exception, exc_type, "informative message")
+        assert "informative message" in str(excinfo.value)
 
 
 def test_diagnostic_ui(loop):
     with cluster() as (s, [a, b]):
-        a_addr = a['address']
-        b_addr = b['address']
-        with Client(s['address'], loop=loop) as c:
+        a_addr = a["address"]
+        b_addr = b["address"]
+        with Client(s["address"], loop=loop) as c:
             d = c.ncores()
             assert d == {a_addr: 1, b_addr: 1}
 
@@ -2692,7 +2774,7 @@ def test_diagnostic_ui(loop):
             assert d == {a_addr: 1}
             d = c.ncores(a_addr)
             assert d == {a_addr: 1}
-            d = c.ncores(a['address'])
+            d = c.ncores(a["address"])
             assert d == {a_addr: 1}
 
             x = c.submit(inc, 1)
@@ -2726,10 +2808,8 @@ def test_diagnostic_nbytes_sync(c):
     doubles = c.map(double, [1, 2, 3])
     wait(incs + doubles)
 
-    assert c.nbytes(summary=False) == {k.key: sizeof(1)
-                                       for k in incs + doubles}
-    assert c.nbytes(summary=True) == {'inc': sizeof(1) * 3,
-                                      'double': sizeof(1) * 3}
+    assert c.nbytes(summary=False) == {k.key: sizeof(1) for k in incs + doubles}
+    assert c.nbytes(summary=True) == {"inc": sizeof(1) * 3, "double": sizeof(1) * 3}
 
 
 @gen_cluster(client=True)
@@ -2738,31 +2818,29 @@ def test_diagnostic_nbytes(c, s, a, b):
     doubles = c.map(double, [1, 2, 3])
     yield wait(incs + doubles)
 
-    assert s.get_nbytes(summary=False) == {k.key: sizeof(1)
-                                           for k in incs + doubles}
-    assert s.get_nbytes(summary=True) == {'inc': sizeof(1) * 3,
-                                          'double': sizeof(1) * 3}
+    assert s.get_nbytes(summary=False) == {k.key: sizeof(1) for k in incs + doubles}
+    assert s.get_nbytes(summary=True) == {"inc": sizeof(1) * 3, "double": sizeof(1) * 3}
 
 
 @gen_test()
 def test_worker_aliases():
     s = Scheduler(validate=True)
     s.start(0)
-    a = Worker(s.ip, s.port, name='alice')
-    b = Worker(s.ip, s.port, name='bob')
+    a = Worker(s.ip, s.port, name="alice")
+    b = Worker(s.ip, s.port, name="bob")
     w = Worker(s.ip, s.port, name=3)
     yield [a, b, w]
 
     c = yield Client((s.ip, s.port), asynchronous=True)
 
-    L = c.map(inc, range(10), workers='alice')
+    L = c.map(inc, range(10), workers="alice")
     future = yield c.scatter(123, workers=3)
     yield wait(L)
     assert len(a.data) == 10
     assert len(b.data) == 0
     assert dict(w.data) == {future.key: 123}
 
-    for i, alias in enumerate([3, [3], 'alice']):
+    for i, alias in enumerate([3, [3], "alice"]):
         result = yield c.submit(lambda x: x + 1, i, workers=alias)
         assert result == i + 1
 
@@ -2809,16 +2887,17 @@ def test_persist_get(c, s, a, b):
     assert result == ((1 + 1) + (2 + 2)) + 10
 
 
-@pytest.mark.skipif(sys.platform.startswith('win'),
-                    reason="num_fds not supported on windows")
+@pytest.mark.skipif(
+    sys.platform.startswith("win"), reason="num_fds not supported on windows"
+)
 def test_client_num_fds(loop):
-    psutil = pytest.importorskip('psutil')
+    psutil = pytest.importorskip("psutil")
     with cluster() as (s, [a, b]):
         proc = psutil.Process()
-        with Client(s['address'], loop=loop) as c:  # first client to start loop
-            before = proc.num_fds()                 # measure
+        with Client(s["address"], loop=loop) as c:  # first client to start loop
+            before = proc.num_fds()  # measure
             for i in range(4):
-                with Client(s['address'], loop=loop):   # start more clients
+                with Client(s["address"], loop=loop):  # start more clients
                     pass
             start = time()
             while proc.num_fds() > before:
@@ -2837,14 +2916,14 @@ def test_startup_close_startup(s, a, b):
 
 def test_startup_close_startup_sync(loop):
     with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
+        with Client(s["address"], loop=loop) as c:
             sleep(0.1)
-        with Client(s['address']) as c:
+        with Client(s["address"]) as c:
             pass
-        with Client(s['address']) as c:
+        with Client(s["address"]) as c:
             pass
         sleep(0.1)
-        with Client(s['address']) as c:
+        with Client(s["address"]) as c:
             pass
 
 
@@ -2854,14 +2933,15 @@ def f():
         class BadlySerializedException(Exception):
             def __reduce__(self):
                 raise TypeError()
-        raise BadlySerializedException('hello world')
+
+        raise BadlySerializedException("hello world")
 
     x = c.submit(f)
 
     try:
         result = yield x
     except Exception as e:
-        assert 'hello world' in str(e)
+        assert "hello world" in str(e)
     else:
         assert False
 
@@ -2885,11 +2965,10 @@ def test_rebalance(c, s, a, b):
 
     assert len(a.data) == 1
     assert {ts.key for ts in aws.has_what} == set(a.data)
-    assert (aws not in s.tasks[x.key].who_has or
-            aws not in s.tasks[y.key].who_has)
+    assert aws not in s.tasks[x.key].who_has or aws not in s.tasks[y.key].who_has
 
 
-@gen_cluster(ncores=[('127.0.0.1', 1)] * 4, client=True)
+@gen_cluster(ncores=[("127.0.0.1", 1)] * 4, client=True)
 def test_rebalance_workers(e, s, a, b, c, d):
     w, x, y, z = yield e.scatter([1, 2, 3, 4], workers=[a.address])
     assert len(a.data) == 4
@@ -2921,7 +3000,7 @@ def test_rebalance_execution(c, s, a, b):
 
 
 def test_rebalance_sync(c, s, a, b):
-    futures = c.map(inc, range(10), workers=[a['address']])
+    futures = c.map(inc, range(10), workers=[a["address"]])
     c.rebalance(futures)
 
     has_what = c.has_what()
@@ -2944,31 +3023,32 @@ def test_receive_lost_key(c, s, a, b):
     yield a._close()
 
     start = time()
-    while x.status == 'finished':
+    while x.status == "finished":
         assert time() < start + 5
         yield gen.sleep(0.01)
 
 
-@pytest.mark.skipif(not sys.platform.startswith('linux'),
-                    reason="Need 127.0.0.2 to mean localhost")
-@gen_cluster([('127.0.0.1', 1), ('127.0.0.2', 2)], client=True)
+@pytest.mark.skipif(
+    not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
+)
+@gen_cluster([("127.0.0.1", 1), ("127.0.0.2", 2)], client=True)
 def test_unrunnable_task_runs(c, s, a, b):
     x = c.submit(inc, 1, workers=[a.ip])
     result = yield x
 
     yield a._close()
     start = time()
-    while x.status == 'finished':
+    while x.status == "finished":
         assert time() < start + 5
         yield gen.sleep(0.01)
 
     assert s.tasks[x.key] in s.unrunnable
-    assert s.get_task_status(keys=[x.key]) == {x.key: 'no-worker'}
+    assert s.get_task_status(keys=[x.key]) == {x.key: "no-worker"}
 
     w = yield Worker(s.ip, s.port, loop=s.loop)
 
     start = time()
-    while x.status != 'finished':
+    while x.status != "finished":
         assert time() < start + 2
         yield gen.sleep(0.01)
 
@@ -2990,16 +3070,16 @@ def test_add_worker_after_tasks(c, s):
     yield n._close()
 
 
-@pytest.mark.skipif(not sys.platform.startswith('linux'),
-                    reason="Need 127.0.0.2 to mean localhost")
-@gen_cluster([('127.0.0.1', 1), ('127.0.0.2', 2)], client=True)
+@pytest.mark.skipif(
+    not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
+)
+@gen_cluster([("127.0.0.1", 1), ("127.0.0.2", 2)], client=True)
 def test_workers_register_indirect_data(c, s, a, b):
     [x] = yield c.scatter([1], workers=a.address)
     y = c.submit(inc, x, workers=b.ip)
     yield y
     assert b.data[x.key] == 1
-    assert s.tasks[x.key].who_has == {s.workers[a.address],
-                                      s.workers[b.address]}
+    assert s.tasks[x.key].who_has == {s.workers[a.address], s.workers[b.address]}
     assert s.workers[b.address].has_what == {s.tasks[x.key], s.tasks[y.key]}
     s.validate_state()
 
@@ -3015,7 +3095,7 @@ def test_submit_on_cancelled_future(c, s, a, b):
         y = c.submit(inc, x)
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 10)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 10)
 def test_replicate(c, s, *workers):
     [a, b] = yield c.scatter([1, 2])
     yield s.replicate(keys=[a.key, b.key], n=5)
@@ -3030,7 +3110,7 @@ def test_replicate(c, s, *workers):
 
 @gen_cluster(client=True)
 def test_replicate_tuple_keys(c, s, a, b):
-    x = delayed(inc)(1, dask_key_name=('x', 1))
+    x = delayed(inc)(1, dask_key_name=("x", 1))
     f = c.persist(x)
     yield c.replicate(f, n=5)
     s.validate_state()
@@ -3040,12 +3120,13 @@ def test_replicate_tuple_keys(c, s, a, b):
     s.validate_state()
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 10)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 10)
 def test_replicate_workers(c, s, *workers):
 
     [a, b] = yield c.scatter([1, 2], workers=[workers[0].address])
-    yield s.replicate(keys=[a.key, b.key], n=5,
-                      workers=[w.address for w in workers[:5]])
+    yield s.replicate(
+        keys=[a.key, b.key], n=5, workers=[w.address for w in workers[:5]]
+    )
 
     assert len(s.tasks[a.key].who_has) == 5
     assert len(s.tasks[b.key].who_has) == 5
@@ -3069,8 +3150,9 @@ def test_replicate_workers(c, s, *workers):
     assert len(s.tasks[b.key].who_has) == 10
     s.validate_state()
 
-    yield s.replicate(keys=[a.key, b.key], n=1,
-                      workers=[w.address for w in workers[:5]])
+    yield s.replicate(
+        keys=[a.key, b.key], n=1, workers=[w.address for w in workers[:5]]
+    )
     assert sum(a.key in w.data for w in workers[:5]) == 1
     assert sum(b.key in w.data for w in workers[:5]) == 1
     assert sum(a.key in w.data for w in workers[5:]) == 5
@@ -3089,7 +3171,7 @@ def __getstate__(self):
         return self.n
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 10)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 10)
 def test_replicate_tree_branching(c, s, *workers):
     obj = CountSerialization()
     [future] = yield c.scatter([obj])
@@ -3099,7 +3181,7 @@ def test_replicate_tree_branching(c, s, *workers):
     assert max_count > 1
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 10)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 10)
 def test_client_replicate(c, s, *workers):
     x = c.submit(inc, 1)
     y = c.submit(inc, 2)
@@ -3120,25 +3202,27 @@ def test_client_replicate(c, s, *workers):
     assert len(s.tasks[y.key].who_has) == 10
 
 
-@pytest.mark.skipif(not sys.platform.startswith('linux'),
-                    reason="Need 127.0.0.2 to mean localhost")
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1),
-                                  ('127.0.0.2', 1),
-                                  ('127.0.0.2', 1)], timeout=None)
+@pytest.mark.skipif(
+    not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
+)
+@gen_cluster(
+    client=True,
+    ncores=[("127.0.0.1", 1), ("127.0.0.2", 1), ("127.0.0.2", 1)],
+    timeout=None,
+)
 def test_client_replicate_host(client, s, a, b, c):
     aws = s.workers[a.address]
     bws = s.workers[b.address]
     cws = s.workers[c.address]
 
-    x = client.submit(inc, 1, workers='127.0.0.2')
+    x = client.submit(inc, 1, workers="127.0.0.2")
     yield wait([x])
-    assert (s.tasks[x.key].who_has == {bws} or
-            s.tasks[x.key].who_has == {cws})
+    assert s.tasks[x.key].who_has == {bws} or s.tasks[x.key].who_has == {cws}
 
-    yield client.replicate([x], workers=['127.0.0.2'])
+    yield client.replicate([x], workers=["127.0.0.2"])
     assert s.tasks[x.key].who_has == {bws, cws}
 
-    yield client.replicate([x], workers=['127.0.0.1'])
+    yield client.replicate([x], workers=["127.0.0.1"])
     assert s.tasks[x.key].who_has == {aws, bws, cws}
 
 
@@ -3156,21 +3240,22 @@ def test_client_replicate_sync(c):
     assert y.result() == 3
 
 
-@pytest.mark.skipif(sys.platform.startswith('win'),
-                    reason="Windows timer too coarse-grained")
-@gen_cluster(client=True, ncores=[('127.0.0.1', 4)] * 1)
+@pytest.mark.skipif(
+    sys.platform.startswith("win"), reason="Windows timer too coarse-grained"
+)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 4)] * 1)
 def test_task_load_adapts_quickly(c, s, a):
     future = c.submit(slowinc, 1, delay=0.2)  # slow
     yield wait(future)
-    assert 0.15 < s.task_duration['slowinc'] < 0.4
+    assert 0.15 < s.task_duration["slowinc"] < 0.4
 
     futures = c.map(slowinc, range(10), delay=0)  # very fast
     yield wait(futures)
 
-    assert 0 < s.task_duration['slowinc'] < 0.1
+    assert 0 < s.task_duration["slowinc"] < 0.1
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 2)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 2)
 def test_even_load_after_fast_functions(c, s, a, b):
     x = c.submit(inc, 1, workers=a.address)  # very fast
     y = c.submit(inc, 2, workers=b.address)  # very fast
@@ -3184,7 +3269,7 @@ def test_even_load_after_fast_functions(c, s, a, b):
     # assert abs(len(a.data) - len(b.data)) <= 3
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 2)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 2)
 def test_even_load_on_startup(c, s, a, b):
     x, y = c.map(inc, [1, 2])
     yield wait([x, y])
@@ -3192,7 +3277,7 @@ def test_even_load_on_startup(c, s, a, b):
 
 
 @pytest.mark.skip
-@gen_cluster(client=True, ncores=[('127.0.0.1', 2)] * 2)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 2)] * 2)
 def test_contiguous_load(c, s, a, b):
     w, x, y, z = c.map(inc, [1, 2, 3, 4])
     yield wait([w, x, y, z])
@@ -3202,7 +3287,7 @@ def test_contiguous_load(c, s, a, b):
     assert {y.key, z.key} in groups
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 4)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 4)
 def test_balanced_with_submit(c, s, *workers):
     L = [c.submit(slowinc, i) for i in range(4)]
     yield wait(L)
@@ -3210,7 +3295,7 @@ def test_balanced_with_submit(c, s, *workers):
         assert len(w.data) == 1
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 4)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 4)
 def test_balanced_with_submit_and_resident_data(c, s, *workers):
     [x] = yield c.scatter([10], broadcast=True)
     L = [c.submit(slowinc, x, pure=False) for i in range(4)]
@@ -3219,34 +3304,38 @@ def test_balanced_with_submit_and_resident_data(c, s, *workers):
         assert len(w.data) == 2
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 20)] * 2)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 20)] * 2)
 def test_scheduler_saturates_cores(c, s, a, b):
     for delay in [0, 0.01, 0.1]:
         futures = c.map(slowinc, range(100), delay=delay)
         futures = c.map(slowinc, futures, delay=delay / 10)
         while not s.tasks:
             if s.tasks:
-                assert all(len(p) >= 20
-                           for w in s.workers.values()
-                           for p in w.processing.values())
+                assert all(
+                    len(p) >= 20
+                    for w in s.workers.values()
+                    for p in w.processing.values()
+                )
             yield gen.sleep(0.01)
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 20)] * 2)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 20)] * 2)
 def test_scheduler_saturates_cores_random(c, s, a, b):
     for delay in [0, 0.01, 0.1]:
         futures = c.map(randominc, range(100), scale=0.1)
         while not s.tasks:
             if s.tasks:
-                assert all(len(p) >= 20
-                           for w in s.workers.values()
-                           for p in w.processing.values())
+                assert all(
+                    len(p) >= 20
+                    for w in s.workers.values()
+                    for p in w.processing.values()
+                )
             yield gen.sleep(0.01)
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 4)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 4)
 def test_cancel_clears_processing(c, s, *workers):
-    da = pytest.importorskip('dask.array')
+    da = pytest.importorskip("dask.array")
     x = c.submit(slowinc, 1, delay=0.2)
     while not s.tasks:
         yield gen.sleep(0.01)
@@ -3263,36 +3352,36 @@ def test_cancel_clears_processing(c, s, *workers):
 def test_default_get():
     with cluster() as (s, [a, b]):
         pre_get = dask.base.get_scheduler()
-        pytest.raises(KeyError, dask.config.get, 'shuffle')
-        with Client(s['address'], set_as_default=True) as c:
+        pytest.raises(KeyError, dask.config.get, "shuffle")
+        with Client(s["address"], set_as_default=True) as c:
             assert dask.base.get_scheduler() == c.get
-            assert dask.config.get('shuffle') == 'tasks'
+            assert dask.config.get("shuffle") == "tasks"
 
         assert dask.base.get_scheduler() == pre_get
-        pytest.raises(KeyError, dask.config.get, 'shuffle')
+        pytest.raises(KeyError, dask.config.get, "shuffle")
 
-        c = Client(s['address'], set_as_default=False)
+        c = Client(s["address"], set_as_default=False)
         assert dask.base.get_scheduler() == pre_get
-        pytest.raises(KeyError, dask.config.get, 'shuffle')
+        pytest.raises(KeyError, dask.config.get, "shuffle")
         c.close()
 
-        c = Client(s['address'], set_as_default=True)
-        assert dask.config.get('shuffle') == 'tasks'
+        c = Client(s["address"], set_as_default=True)
+        assert dask.config.get("shuffle") == "tasks"
         assert dask.base.get_scheduler() == c.get
         c.close()
         assert dask.base.get_scheduler() == pre_get
-        pytest.raises(KeyError, dask.config.get, 'shuffle')
+        pytest.raises(KeyError, dask.config.get, "shuffle")
 
-        with Client(s['address']) as c:
+        with Client(s["address"]) as c:
             assert dask.base.get_scheduler() == c.get
 
-        with Client(s['address'], set_as_default=False) as c:
+        with Client(s["address"], set_as_default=False) as c:
             assert dask.base.get_scheduler() != c.get
         assert dask.base.get_scheduler() != c.get
 
-        with Client(s['address'], set_as_default=True) as c1:
+        with Client(s["address"], set_as_default=True) as c1:
             assert dask.base.get_scheduler() == c1.get
-            with Client(s['address'], set_as_default=True) as c2:
+            with Client(s["address"], set_as_default=True) as c2:
                 assert dask.base.get_scheduler() == c2.get
             assert dask.base.get_scheduler() == c1.get
         assert dask.base.get_scheduler() == pre_get
@@ -3303,8 +3392,9 @@ def test_get_processing(c, s, a, b):
     processing = yield c.processing()
     assert processing == valmap(tuple, s.processing)
 
-    futures = c.map(slowinc, range(10), delay=0.1, workers=[a.address],
-                    allow_other_workers=True)
+    futures = c.map(
+        slowinc, range(10), delay=0.1, workers=[a.address], allow_other_workers=True
+    )
 
     yield gen.sleep(0.2)
 
@@ -3353,7 +3443,7 @@ def assert_dict_key_equal(expected, actual):
         assert list(ev) == list(av)
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 3)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 3)
 def test_get_foo_lost_keys(c, s, u, v, w):
     x = c.submit(inc, 1, workers=[u.address])
     y = yield c.scatter(3, workers=[v.address])
@@ -3366,7 +3456,7 @@ def test_get_foo_lost_keys(c, s, u, v, w):
     d = yield c.scheduler.has_what(workers=[ua, va])
     assert_dict_key_equal(d, {ua: [x.key], va: [y.key]})
     d = yield c.scheduler.who_has()
-    assert_dict_key_equal(d,  {x.key: [ua], y.key: [va]})
+    assert_dict_key_equal(d, {x.key: [ua], y.key: [va]})
     d = yield c.scheduler.who_has(keys=[x.key, y.key])
     assert_dict_key_equal(d, {x.key: [ua], y.key: [va]})
 
@@ -3397,14 +3487,14 @@ def test_get_processing_sync(c, s, a, b):
     processing = c.processing()
     assert not any(v for v in processing.values())
 
-    futures = c.map(slowinc, range(10), delay=0.1,
-                    workers=[a['address']],
-                    allow_other_workers=False)
+    futures = c.map(
+        slowinc, range(10), delay=0.1, workers=[a["address"]], allow_other_workers=False
+    )
 
     sleep(0.2)
 
-    aa = a['address']
-    bb = b['address']
+    aa = a["address"]
+    bb = b["address"]
     processing = c.processing()
 
     assert set(c.processing(aa)) == {aa}
@@ -3423,7 +3513,7 @@ def test_close_idempotent(c):
 def test_get_returns_early(c):
     start = time()
     with ignoring(RuntimeError):
-        result = c.get({'x': (throws, 1), 'y': (sleep, 1)}, ['x', 'y'])
+        result = c.get({"x": (throws, 1), "y": (sleep, 1)}, ["x", "y"])
     assert time() < start + 0.5
     # Futures should be released and forgotten
     wait_for(lambda: not c.futures, timeout=0.1)
@@ -3434,7 +3524,7 @@ def test_get_returns_early(c):
     x.result()
 
     with ignoring(RuntimeError):
-        result = c.get({'x': (throws, 1), x.key: (inc, 1)}, ['x', x.key])
+        result = c.get({"x": (throws, 1), x.key: (inc, 1)}, ["x", x.key])
     assert x.key in c.futures
 
 
@@ -3450,6 +3540,7 @@ def test_Client_clears_references_after_restart(c, s, a, b):
     key = x.key
     del x
     import gc
+
     gc.collect()
     yield gen.moment
 
@@ -3458,7 +3549,7 @@ def test_Client_clears_references_after_restart(c, s, a, b):
 
 def test_get_stops_work_after_error(c):
     with pytest.raises(RuntimeError):
-        c.get({'x': (throws, 1), 'y': (sleep, 1.5)}, ['x', 'y'])
+        c.get({"x": (throws, 1), "y": (sleep, 1.5)}, ["x", "y"])
 
     start = time()
     while any(c.processing().values()):
@@ -3479,7 +3570,7 @@ def test_as_completed_results(c):
     assert set(pluck(0, seq2)) == set(seq)
 
 
-@pytest.mark.parametrize('with_results', [True, False])
+@pytest.mark.parametrize("with_results", [True, False])
 def test_as_completed_batches(c, with_results):
     n = 50
     futures = c.map(slowinc, range(n), delay=0.01)
@@ -3509,11 +3600,11 @@ def test_status():
     s.start(0)
 
     c = yield Client((s.ip, s.port), asynchronous=True)
-    assert c.status == 'running'
+    assert c.status == "running"
     x = c.submit(inc, 1)
 
     yield c.close()
-    assert c.status == 'closed'
+    assert c.status == "closed"
 
     yield s.close()
 
@@ -3551,13 +3642,19 @@ def test_scatter_raises_if_no_workers(c, s):
 
 @slow
 def test_reconnect(loop):
-    w = Worker('127.0.0.1', 9393, loop=loop)
+    w = Worker("127.0.0.1", 9393, loop=loop)
     w.start()
 
-    scheduler_cli = ['dask-scheduler', '--host', '127.0.0.1',
-                     '--port', '9393', '--no-bokeh']
+    scheduler_cli = [
+        "dask-scheduler",
+        "--host",
+        "127.0.0.1",
+        "--port",
+        "9393",
+        "--no-bokeh",
+    ]
     with popen(scheduler_cli) as s:
-        c = Client('127.0.0.1:9393', loop=loop)
+        c = Client("127.0.0.1:9393", loop=loop)
         start = time()
         while len(c.ncores()) != 1:
             sleep(0.1)
@@ -3567,20 +3664,20 @@ def test_reconnect(loop):
         assert x.result() == 2
 
     start = time()
-    while c.status != 'connecting':
+    while c.status != "connecting":
         assert time() < start + 5
         sleep(0.01)
 
     with pytest.raises(Exception):
         c.ncores()
 
-    assert x.status == 'cancelled'
+    assert x.status == "cancelled"
     with pytest.raises(CancelledError):
         x.result()
 
     with popen(scheduler_cli) as s:
         start = time()
-        while c.status != 'running':
+        while c.status != "running":
             sleep(0.1)
             assert time() < start + 5
         start = time()
@@ -3608,17 +3705,16 @@ def test_reconnect(loop):
 
 
 @slow
-@pytest.mark.skipif(sys.platform.startswith('win'),
-                    reason="num_fds not supported on windows")
-@pytest.mark.skipif(sys.version_info[0] == 2,
-                    reason="Semaphore.acquire doesn't support timeout option")
-@pytest.mark.xfail(reason='TODO: intermittent failures')
-@pytest.mark.parametrize("worker,count,repeat", [
-    (Worker, 100, 5),
-    (Nanny, 10, 20)
-])
+@pytest.mark.skipif(
+    sys.platform.startswith("win"), reason="num_fds not supported on windows"
+)
+@pytest.mark.skipif(
+    sys.version_info[0] == 2, reason="Semaphore.acquire doesn't support timeout option"
+)
+@pytest.mark.xfail(reason="TODO: intermittent failures")
+@pytest.mark.parametrize("worker,count,repeat", [(Worker, 100, 5), (Nanny, 10, 20)])
 def test_open_close_many_workers(loop, worker, count, repeat):
-    psutil = pytest.importorskip('psutil')
+    psutil = pytest.importorskip("psutil")
     proc = psutil.Process()
 
     with cluster(nworkers=0, active_rpc_timeout=20) as (s, _):
@@ -3631,7 +3727,7 @@ def test_open_close_many_workers(loop, worker, count, repeat):
         def start_worker(sleep, duration, repeat=1):
             for i in range(repeat):
                 yield gen.sleep(sleep)
-                w = worker(s['address'], loop=loop)
+                w = worker(s["address"], loop=loop)
                 running[w] = None
                 yield w
                 addr = w.worker_address
@@ -3643,10 +3739,11 @@ def start_worker(sleep, duration, repeat=1):
             done.release()
 
         for i in range(count):
-            loop.add_callback(start_worker, random.random() / 5, random.random() / 5,
-                              repeat=repeat)
+            loop.add_callback(
+                start_worker, random.random() / 5, random.random() / 5, repeat=repeat
+            )
 
-        with Client(s['address'], loop=loop) as c:
+        with Client(s["address"], loop=loop) as c:
             sleep(1)
 
             for i in range(count):
@@ -3689,7 +3786,7 @@ def test_idempotence(s, a, b):
     # Error
     a = c.submit(div, 1, 0)
     yield wait(a)
-    assert a.status == 'error'
+    assert a.status == "error"
     log = list(s.transition_log)
 
     b = f.submit(div, 1, 0)
@@ -3714,12 +3811,12 @@ def test_idempotence(s, a, b):
 def test_scheduler_info(c):
     info = c.scheduler_info()
     assert isinstance(info, dict)
-    assert len(info['workers']) == 2
+    assert len(info["workers"]) == 2
 
 
 def test_write_scheduler_file(c):
     info = c.scheduler_info()
-    with tmpfile('json') as scheduler_file:
+    with tmpfile("json") as scheduler_file:
         c.write_scheduler_file(scheduler_file)
         with Client(scheduler_file=scheduler_file) as c2:
             info2 = c2.scheduler_info()
@@ -3732,30 +3829,30 @@ def test_write_scheduler_file(c):
 
 
 def test_get_versions(c):
-    requests = pytest.importorskip('requests')
+    requests = pytest.importorskip("requests")
 
     v = c.get_versions()
-    assert v['scheduler'] is not None
-    assert v['client'] is not None
-    assert len(v['workers']) == 2
-    for k, v in v['workers'].items():
+    assert v["scheduler"] is not None
+    assert v["client"] is not None
+    assert len(v["workers"]) == 2
+    for k, v in v["workers"].items():
         assert v is not None
 
     c.get_versions(check=True)
     # smoke test for versions
     # that this does not raise
 
-    v = c.get_versions(packages=['requests'])
-    assert dict(v['client']['packages']['optional'])['requests'] == requests.__version__
+    v = c.get_versions(packages=["requests"])
+    assert dict(v["client"]["packages"]["optional"])["requests"] == requests.__version__
 
 
 def test_threaded_get_within_distributed(c):
     import dask.multiprocessing
-    for get in [dask.local.get_sync,
-                dask.multiprocessing.get,
-                dask.threaded.get]:
+
+    for get in [dask.local.get_sync, dask.multiprocessing.get, dask.threaded.get]:
+
         def f():
-            return get({'x': (lambda: 1,)}, 'x')
+            return get({"x": (lambda: 1,)}, "x")
 
         future = c.submit(f)
         assert future.result() == 1
@@ -3768,11 +3865,11 @@ def test_lose_scattered_data(c, s, a, b):
     yield a._close()
     yield gen.sleep(0.1)
 
-    assert x.status == 'cancelled'
+    assert x.status == "cancelled"
     assert x.key not in s.tasks
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 3)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 3)
 def test_partially_lose_scattered_data(e, s, a, b, c):
     [x] = yield e.scatter([1], workers=a.address)
     yield e.replicate(x, n=2)
@@ -3780,8 +3877,8 @@ def test_partially_lose_scattered_data(e, s, a, b, c):
     yield a._close()
     yield gen.sleep(0.1)
 
-    assert x.status == 'finished'
-    assert s.get_task_status(keys=[x.key]) == {x.key: 'memory'}
+    assert x.status == "finished"
+    assert s.get_task_status(keys=[x.key]) == {x.key: "memory"}
 
 
 @gen_cluster(client=True)
@@ -3797,9 +3894,9 @@ def test_scatter_compute_lose(c, s, a, b):
     with pytest.raises(CancelledError):
         yield wait(z)
 
-    assert x.status == 'cancelled'
-    assert y.status == 'finished'
-    assert z.status == 'cancelled'
+    assert x.status == "cancelled"
+    assert y.status == "finished"
+    assert z.status == "cancelled"
 
 
 @gen_cluster(client=True)
@@ -3821,13 +3918,13 @@ def test_scatter_compute_store_lose(c, s, a, b):
     yield a._close()
 
     start = time()
-    while x.status == 'finished':
+    while x.status == "finished":
         yield gen.sleep(0.01)
         assert time() < start + 2
 
     # assert xx.status == 'finished'
-    assert y.status == 'finished'
-    assert z.status == 'finished'
+    assert y.status == "finished"
+    assert z.status == "finished"
 
     zz = c.submit(inc, z)
     yield wait(zz)
@@ -3836,7 +3933,7 @@ def test_scatter_compute_store_lose(c, s, a, b):
     del z
 
     start = time()
-    while s.get_task_status(keys=[zkey]) != {zkey: 'released'}:
+    while s.get_task_status(keys=[zkey]) != {zkey: "released"}:
         yield gen.sleep(0.01)
         assert time() < start + 2
 
@@ -3844,9 +3941,7 @@ def test_scatter_compute_store_lose(c, s, a, b):
     del xx
 
     start = time()
-    while (x.key in s.tasks and
-           zkey not in s.tasks and
-           xxkey not in s.tasks):
+    while x.key in s.tasks and zkey not in s.tasks and xxkey not in s.tasks:
         yield gen.sleep(0.01)
         assert time() < start + 2
 
@@ -3868,12 +3963,12 @@ def test_scatter_compute_store_lose_processing(c, s, a, b):
     yield a._close()
 
     start = time()
-    while x.status == 'finished':
+    while x.status == "finished":
         yield gen.sleep(0.01)
         assert time() < start + 2
 
-    assert y.status == 'cancelled'
-    assert z.status == 'cancelled'
+    assert y.status == "cancelled"
+    assert z.status == "cancelled"
 
 
 @gen_cluster(client=False)
@@ -3913,19 +4008,23 @@ def test_temp_client(s, a, b):
 
 
 @nodebug  # test timing is fragile
-@gen_cluster(ncores=[('127.0.0.1', 1)] * 3, client=True)
+@gen_cluster(ncores=[("127.0.0.1", 1)] * 3, client=True)
 def test_persist_workers(e, s, a, b, c):
     L1 = [delayed(inc)(i) for i in range(4)]
     total = delayed(sum)(L1)
     L2 = [delayed(add)(i, total) for i in L1]
     total2 = delayed(sum)(L2)
 
-    out = e.persist(L1 + L2 + [total, total2],
-                    workers={tuple(L1): a.address,
-                             total: b.address,
-                             tuple(L2): [c.address],
-                             total2: b.address},
-                    allow_other_workers=L2 + [total2])
+    out = e.persist(
+        L1 + L2 + [total, total2],
+        workers={
+            tuple(L1): a.address,
+            total: b.address,
+            tuple(L2): [c.address],
+            total2: b.address,
+        },
+        allow_other_workers=L2 + [total2],
+    )
 
     yield wait(out)
     assert all(v.key in a.data for v in L1)
@@ -3934,17 +4033,17 @@ def test_persist_workers(e, s, a, b, c):
     assert s.loose_restrictions == {total2.key} | {v.key for v in L2}
 
 
-@gen_cluster(ncores=[('127.0.0.1', 1)] * 3, client=True)
+@gen_cluster(ncores=[("127.0.0.1", 1)] * 3, client=True)
 def test_compute_workers(e, s, a, b, c):
     L1 = [delayed(inc)(i) for i in range(4)]
     total = delayed(sum)(L1)
     L2 = [delayed(add)(i, total) for i in L1]
 
-    out = e.compute(L1 + L2 + [total],
-                    workers={tuple(L1): a.address,
-                             total: b.address,
-                             tuple(L2): [c.address]},
-                    allow_other_workers=L1 + [total])
+    out = e.compute(
+        L1 + L2 + [total],
+        workers={tuple(L1): a.address, total: b.address, tuple(L2): [c.address]},
+        allow_other_workers=L1 + [total],
+    )
 
     yield wait(out)
     for v in L1:
@@ -3958,16 +4057,16 @@ def test_compute_workers(e, s, a, b, c):
 
 @gen_cluster(client=True)
 def test_compute_nested_containers(c, s, a, b):
-    da = pytest.importorskip('dask.array')
-    np = pytest.importorskip('numpy')
+    da = pytest.importorskip("dask.array")
+    np = pytest.importorskip("numpy")
     x = da.ones(10, chunks=(5,)) + 1
 
-    future = c.compute({'x': [x], 'y': 123})
+    future = c.compute({"x": [x], "y": 123})
     result = yield future
 
     assert isinstance(result, dict)
-    assert (result['x'][0] == np.ones(10) + 1).all()
-    assert result['y'] == 123
+    assert (result["x"][0] == np.ones(10) + 1).all()
+    assert result["y"] == 123
 
 
 def test_get_restrictions():
@@ -3975,20 +4074,20 @@ def test_get_restrictions():
     total = delayed(sum)(L1)
     L2 = [delayed(add)(i, total) for i in L1]
 
-    r1, loose = Client.get_restrictions(L2, '127.0.0.1', False)
-    assert r1 == {d.key: ['127.0.0.1'] for d in L2}
+    r1, loose = Client.get_restrictions(L2, "127.0.0.1", False)
+    assert r1 == {d.key: ["127.0.0.1"] for d in L2}
     assert not loose
 
-    r1, loose = Client.get_restrictions(L2, ['127.0.0.1'], True)
-    assert r1 == {d.key: ['127.0.0.1'] for d in L2}
+    r1, loose = Client.get_restrictions(L2, ["127.0.0.1"], True)
+    assert r1 == {d.key: ["127.0.0.1"] for d in L2}
     assert set(loose) == {d.key for d in L2}
 
-    r1, loose = Client.get_restrictions(L2, {total: '127.0.0.1'}, True)
-    assert r1 == {total.key: ['127.0.0.1']}
+    r1, loose = Client.get_restrictions(L2, {total: "127.0.0.1"}, True)
+    assert r1 == {total.key: ["127.0.0.1"]}
     assert loose == [total.key]
 
-    r1, loose = Client.get_restrictions(L2, {(total,): '127.0.0.1'}, True)
-    assert r1 == {total.key: ['127.0.0.1']}
+    r1, loose = Client.get_restrictions(L2, {(total,): "127.0.0.1"}, True)
+    assert r1 == {total.key: ["127.0.0.1"]}
     assert loose == [total.key]
 
 
@@ -3997,8 +4096,8 @@ def test_scatter_type(c, s, a, b):
     [future] = yield c.scatter([1])
     assert future.type == int
 
-    d = yield c.scatter({'x': 1.0})
-    assert d['x'].type == float
+    d = yield c.scatter({"x": 1.0})
+    assert d["x"].type == float
 
 
 @gen_cluster(client=True)
@@ -4013,7 +4112,7 @@ def test_retire_workers_2(c, s, a, b):
     assert a.address not in s.workers
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 10)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 10)
 def test_retire_many_workers(c, s, *workers):
     futures = yield c.scatter(list(range(100)))
 
@@ -4024,16 +4123,15 @@ def test_retire_many_workers(c, s, *workers):
 
     assert len(s.has_what) == len(s.ncores) == 3
     assert all(future.done() for future in futures)
-    assert all(s.tasks[future.key].state == 'memory' for future in futures)
+    assert all(s.tasks[future.key].state == "memory" for future in futures)
     for w, keys in s.has_what.items():
         assert 15 < len(keys) < 50
 
 
-@gen_cluster(client=True,
-             ncores=[('127.0.0.1', 3)] * 2)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 3)] * 2)
 def test_weight_occupancy_against_data_movement(c, s, a, b):
-    s.extensions['stealing']._pc.callback_time = 1000000
-    s.task_duration['f'] = 0.01
+    s.extensions["stealing"]._pc.callback_time = 1000000
+    s.task_duration["f"] = 0.01
 
     def f(x, y=0, z=0):
         sleep(0.01)
@@ -4050,11 +4148,10 @@ def f(x, y=0, z=0):
     assert sum(f.key in b.data for f in futures) >= 1
 
 
-@gen_cluster(client=True,
-             ncores=[('127.0.0.1', 1), ('127.0.0.1', 10)])
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1), ("127.0.0.1", 10)])
 def test_distribute_tasks_by_ncores(c, s, a, b):
-    s.task_duration['f'] = 0.01
-    s.extensions['stealing']._pc.callback_time = 1000000
+    s.task_duration["f"] = 0.01
+    s.extensions["stealing"]._pc.callback_time = 1000000
 
     def f(x, y=0):
         sleep(0.01)
@@ -4079,10 +4176,10 @@ def f(future):
     def g(future):
         S.add((future.key, future.status))
 
-    u = c.submit(inc, 1, key='u')
-    v = c.submit(throws, "hello", key='v')
-    w = c.submit(slowinc, 2, delay=0.3, key='w')
-    x = c.submit(inc, 3, key='x')
+    u = c.submit(inc, 1, key="u")
+    v = c.submit(throws, "hello", key="v")
+    w = c.submit(slowinc, 2, delay=0.3, key="w")
+    x = c.submit(inc, 3, key="x")
     u.add_done_callback(f)
     v.add_done_callback(f)
     w.add_done_callback(f)
@@ -4115,7 +4212,7 @@ def test_normalize_collection(c, s, a, b):
 
 @gen_cluster(client=True)
 def test_normalize_collection_dask_array(c, s, a, b):
-    da = pytest.importorskip('dask.array')
+    da = pytest.importorskip("dask.array")
 
     x = da.ones(10, chunks=(5,))
     y = x + 1
@@ -4139,9 +4236,9 @@ def test_normalize_collection_dask_array(c, s, a, b):
 
 @slow
 def test_normalize_collection_with_released_futures(c):
-    da = pytest.importorskip('dask.array')
+    da = pytest.importorskip("dask.array")
 
-    x = da.arange(2**20, chunks=2**10)
+    x = da.arange(2 ** 20, chunks=2 ** 10)
     y = x.persist()
     wait(y)
     sol = y.sum().compute()
@@ -4157,7 +4254,7 @@ def test_normalize_collection_with_released_futures(c):
 
 @gen_cluster(client=True)
 def test_auto_normalize_collection(c, s, a, b):
-    da = pytest.importorskip('dask.array')
+    da = pytest.importorskip("dask.array")
 
     x = da.ones(10, chunks=5)
     assert len(x.dask) == 2
@@ -4182,7 +4279,7 @@ def test_auto_normalize_collection(c, s, a, b):
 
 
 def test_auto_normalize_collection_sync(c):
-    da = pytest.importorskip('dask.array')
+    da = pytest.importorskip("dask.array")
     x = da.ones(10, chunks=5)
 
     y = x.map_blocks(slowinc, delay=1, dtype=x.dtype)
@@ -4199,14 +4296,15 @@ def test_auto_normalize_collection_sync(c):
 
 def assert_no_data_loss(scheduler):
     for key, start, finish, recommendations, _ in scheduler.transition_log:
-        if start == 'memory' and finish == 'released':
+        if start == "memory" and finish == "released":
             for k, v in recommendations.items():
-                assert not (k == key and v == 'waiting')
+                assert not (k == key and v == "waiting")
 
 
 @gen_cluster(client=True, timeout=None)
 def test_interleave_computations(c, s, a, b):
     import distributed
+
     distributed.g = s
     xs = [delayed(slowinc)(i, delay=0.02) for i in range(30)]
     ys = [delayed(slowdec)(x, delay=0.02) for x in xs]
@@ -4216,7 +4314,7 @@ def test_interleave_computations(c, s, a, b):
 
     future = c.compute(total)
 
-    done = ('memory', 'released')
+    done = ("memory", "released")
 
     yield gen.sleep(0.1)
 
@@ -4226,12 +4324,9 @@ def test_interleave_computations(c, s, a, b):
 
     while not s.tasks or any(w.processing for w in s.workers.values()):
         yield gen.sleep(0.05)
-        x_done = sum(state in done
-                     for state in s.get_task_status(keys=x_keys).values())
-        y_done = sum(state in done
-                     for state in s.get_task_status(keys=y_keys).values())
-        z_done = sum(state in done
-                     for state in s.get_task_status(keys=z_keys).values())
+        x_done = sum(state in done for state in s.get_task_status(keys=x_keys).values())
+        y_done = sum(state in done for state in s.get_task_status(keys=y_keys).values())
+        z_done = sum(state in done for state in s.get_task_status(keys=z_keys).values())
 
         assert x_done >= y_done >= z_done
         assert x_done < y_done + 10
@@ -4247,7 +4342,7 @@ def test_interleave_computations_map(c, s, a, b):
     ys = c.map(slowdec, xs, delay=0.02)
     zs = c.map(slowadd, xs, ys, delay=0.02)
 
-    done = ('memory', 'released')
+    done = ("memory", "released")
 
     x_keys = [x.key for x in xs]
     y_keys = [y.key for y in ys]
@@ -4255,12 +4350,9 @@ def test_interleave_computations_map(c, s, a, b):
 
     while not s.tasks or any(w.processing for w in s.workers.values()):
         yield gen.sleep(0.05)
-        x_done = sum(state in done
-                     for state in s.get_task_status(keys=x_keys).values())
-        y_done = sum(state in done
-                     for state in s.get_task_status(keys=y_keys).values())
-        z_done = sum(state in done
-                     for state in s.get_task_status(keys=z_keys).values())
+        x_done = sum(state in done for state in s.get_task_status(keys=x_keys).values())
+        y_done = sum(state in done for state in s.get_task_status(keys=y_keys).values())
+        z_done = sum(state in done for state in s.get_task_status(keys=z_keys).values())
 
         assert x_done >= y_done >= z_done
         assert x_done < y_done + 10
@@ -4269,20 +4361,20 @@ def test_interleave_computations_map(c, s, a, b):
 
 @gen_cluster(client=True)
 def test_scatter_dict_workers(c, s, a, b):
-    yield c.scatter({'a': 10}, workers=[a.address, b.address])
-    assert 'a' in a.data or 'a' in b.data
+    yield c.scatter({"a": 10}, workers=[a.address, b.address])
+    assert "a" in a.data or "a" in b.data
 
 
 @slow
 @gen_test()
 def test_client_timeout():
     loop = IOLoop.current()
-    c = Client('127.0.0.1:57484', asynchronous=True)
+    c = Client("127.0.0.1:57484", asynchronous=True)
 
     s = Scheduler(loop=loop)
     yield gen.sleep(4)
     try:
-        s.start(('127.0.0.1', 57484))
+        s.start(("127.0.0.1", 57484))
     except EnvironmentError:  # port in use
         return
 
@@ -4337,29 +4429,29 @@ def test_dont_clear_waiting_data(c, s, a, b):
 def test_get_future_error_simple(c, s, a, b):
     f = c.submit(div, 1, 0)
     yield wait(f)
-    assert f.status == 'error'
+    assert f.status == "error"
 
     function, args, kwargs, deps = yield c._get_futures_error(f)
     # args contains only solid values, not keys
-    assert function.__name__ == 'div'
+    assert function.__name__ == "div"
     with pytest.raises(ZeroDivisionError):
         function(*args, **kwargs)
 
 
 @gen_cluster(client=True)
 def test_get_futures_error(c, s, a, b):
-    x0 = delayed(dec)(2, dask_key_name='x0')
-    y0 = delayed(dec)(1, dask_key_name='y0')
-    x = delayed(div)(1, x0, dask_key_name='x')
-    y = delayed(div)(1, y0, dask_key_name='y')
-    tot = delayed(sum)(x, y, dask_key_name='tot')
+    x0 = delayed(dec)(2, dask_key_name="x0")
+    y0 = delayed(dec)(1, dask_key_name="y0")
+    x = delayed(div)(1, x0, dask_key_name="x")
+    y = delayed(div)(1, y0, dask_key_name="y")
+    tot = delayed(sum)(x, y, dask_key_name="tot")
 
     f = c.compute(tot)
     yield wait(f)
-    assert f.status == 'error'
+    assert f.status == "error"
 
     function, args, kwargs, deps = yield c._get_futures_error(f)
-    assert function.__name__ == 'div'
+    assert function.__name__ == "div"
     assert args == (1, y0.key)
 
 
@@ -4373,11 +4465,11 @@ def test_recreate_error_delayed(c, s, a, b):
 
     f = c.compute(tot)
 
-    assert f.status == 'pending'
+    assert f.status == "pending"
 
     function, args, kwargs = yield c._recreate_error_locally(f)
-    assert f.status == 'error'
-    assert function.__name__ == 'div'
+    assert f.status == "error"
+    assert function.__name__ == "div"
     assert args == (1, 0)
     with pytest.raises(ZeroDivisionError):
         function(*args, **kwargs)
@@ -4392,11 +4484,11 @@ def test_recreate_error_futures(c, s, a, b):
     tot = c.submit(sum, x, y)
     f = c.compute(tot)
 
-    assert f.status == 'pending'
+    assert f.status == "pending"
 
     function, args, kwargs = yield c._recreate_error_locally(f)
-    assert f.status == 'error'
-    assert function.__name__ == 'div'
+    assert f.status == "error"
+    assert function.__name__ == "div"
     assert args == (1, 0)
     with pytest.raises(ZeroDivisionError):
         function(*args, **kwargs)
@@ -4413,15 +4505,17 @@ def test_recreate_error_collection(c, s, a, b):
     with pytest.raises(ZeroDivisionError):
         function(*args, **kwargs)
 
-    dd = pytest.importorskip('dask.dataframe')
+    dd = pytest.importorskip("dask.dataframe")
     import pandas as pd
-    df = dd.from_pandas(pd.DataFrame({'a': [0, 1, 2, 3, 4]}), chunksize=2)
+
+    df = dd.from_pandas(pd.DataFrame({"a": [0, 1, 2, 3, 4]}), chunksize=2)
 
     def make_err(x):
         # because pandas would happily work with NaN
         if x == 0:
             raise ValueError
         return x
+
     df2 = df.a.map(make_err)
     f = c.compute(df2)
     function, args, kwargs = yield c._recreate_error_locally(f)
@@ -4437,12 +4531,12 @@ def make_err(x):
 
 @gen_cluster(client=True)
 def test_recreate_error_array(c, s, a, b):
-    da = pytest.importorskip('dask.array')
-    pytest.importorskip('scipy')
+    da = pytest.importorskip("dask.array")
+    pytest.importorskip("scipy")
     z = (da.linalg.inv(da.zeros((10, 10), chunks=10)) + 1).sum()
     zz = z.persist()
     func, args, kwargs = yield c._recreate_error_locally(zz)
-    assert '0.,0.,0.' in str(args).replace(' ', '')  # args contain actual arrays
+    assert "0.,0.,0." in str(args).replace(" ", "")  # args contain actual arrays
 
 
 def test_recreate_error_sync(c):
@@ -4455,7 +4549,7 @@ def test_recreate_error_sync(c):
 
     with pytest.raises(ZeroDivisionError) as e:
         c.recreate_error_locally(f)
-    assert f.status == 'error'
+    assert f.status == "error"
 
 
 def test_recreate_error_not_error(c):
@@ -4472,7 +4566,7 @@ def test_retire_workers(c, s, a, b):
     assert set(s.workers) == {b.address}
 
     start = time()
-    while a.status != 'closed':
+    while a.status != "closed":
         yield gen.sleep(0.01)
         assert time() < start + 5
 
@@ -4504,7 +4598,7 @@ def __getstate__(self):
             return 1
 
         def __setstate__(self, state):
-            raise MyException('hello')
+            raise MyException("hello")
 
     future = c.submit(identity, Foo())
     with pytest.raises(MyException):
@@ -4524,7 +4618,7 @@ def __getstate__(self):
             return 1
 
         def __setstate__(self, state):
-            raise MyException('hello')
+            raise MyException("hello")
 
         def __call__(self, *args):
             return 1
@@ -4552,7 +4646,7 @@ def f(x):
         fire_and_forget(c.submit(f, future))
 
         start = time()
-        while not hasattr(distributed, 'foo'):
+        while not hasattr(distributed, "foo"):
             yield gen.sleep(0.01)
             assert time() < start + 2
         assert distributed.foo == 123
@@ -4581,21 +4675,21 @@ def test_fire_and_forget_err(c, s, a, b):
 
 
 def test_quiet_client_close(loop):
-    with captured_logger(logging.getLogger('distributed')) as logger:
+    with captured_logger(logging.getLogger("distributed")) as logger:
         with Client(loop=loop, processes=False, threads_per_worker=4) as c:
             futures = c.map(slowinc, range(1000), delay=0.01)
             sleep(0.200)  # stop part-way
-        sleep(.1)  # let things settle
+        sleep(0.1)  # let things settle
 
         out = logger.getvalue()
-        lines = out.strip().split('\n')
+        lines = out.strip().split("\n")
         assert len(lines) <= 2
         for line in lines:
             assert (
-                not line or
-                'Reconnecting' in line or
-                'garbage' in line or
-                set(line) == {'-'}
+                not line
+                or "Reconnecting" in line
+                or "garbage" in line
+                or set(line) == {"-"}
             ), line
 
 
@@ -4605,14 +4699,14 @@ def test_quiet_client_close_when_cluster_is_closed_before_client(loop):
     # fix in #2477 and with 5 attempts, this test passes by chance in about 10%
     # of the cases.
     for _ in range(n_attempts):
-        with captured_logger(logging.getLogger('tornado.application')) as logger:
+        with captured_logger(logging.getLogger("tornado.application")) as logger:
             cluster = LocalCluster(loop=loop)
             client = Client(cluster, loop=loop)
             cluster.close()
             client.close()
 
         out = logger.getvalue()
-        assert 'CancelledError' not in out
+        assert "CancelledError" not in out
 
 
 @gen_cluster()
@@ -4641,6 +4735,7 @@ def f(_):
         return total.result()
 
     from concurrent.futures import ThreadPoolExecutor
+
     with ThreadPoolExecutor(20) as e:
         results = list(e.map(f, range(20)))
         assert results and all(results)
@@ -4649,7 +4744,7 @@ def f(_):
 
 @slow
 def test_threadsafe_get(c):
-    da = pytest.importorskip('dask.array')
+    da = pytest.importorskip("dask.array")
     x = da.arange(100, chunks=(10,))
 
     def f(_):
@@ -4660,6 +4755,7 @@ def f(_):
         return total
 
     from concurrent.futures import ThreadPoolExecutor
+
     e = ThreadPoolExecutor(30)
     results = list(e.map(f, range(30)))
     assert results and all(results)
@@ -4667,7 +4763,7 @@ def f(_):
 
 @slow
 def test_threadsafe_compute(c):
-    da = pytest.importorskip('dask.array')
+    da = pytest.importorskip("dask.array")
     x = da.arange(100, chunks=(10,))
 
     def f(_):
@@ -4679,6 +4775,7 @@ def f(_):
         return total
 
     from concurrent.futures import ThreadPoolExecutor
+
     e = ThreadPoolExecutor(30)
     results = list(e.map(f, range(30)))
     assert results and all(results)
@@ -4686,13 +4783,13 @@ def f(_):
 
 @gen_cluster(client=True)
 def test_identity(c, s, a, b):
-    assert c.id.lower().startswith('client')
-    assert a.id.lower().startswith('worker')
-    assert b.id.lower().startswith('worker')
-    assert s.id.lower().startswith('scheduler')
+    assert c.id.lower().startswith("client")
+    assert a.id.lower().startswith("worker")
+    assert b.id.lower().startswith("worker")
+    assert s.id.lower().startswith("scheduler")
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 4)] * 2)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 4)] * 2)
 def test_get_client(c, s, a, b):
     assert get_client() is c
     assert c.asynchronous
@@ -4701,11 +4798,13 @@ def f(x):
         client = get_client()
         future = client.submit(inc, x)
         import distributed
+
         assert not client.asynchronous
         assert client is distributed.tmp_client
         return future.result()
 
     import distributed
+
     distributed.tmp_client = c
     try:
         futures = c.map(f, range(5))
@@ -4719,16 +4818,17 @@ def test_get_client_no_cluster():
     # Clean up any global workers added by other tests. This test requires that
     # there are no global workers.
     from distributed.worker import _global_workers
+
     del _global_workers[:]
 
-    msg = 'No global client found and no address provided'
-    with pytest.raises(ValueError, match=r'^{}$'.format(msg)):
+    msg = "No global client found and no address provided"
+    with pytest.raises(ValueError, match=r"^{}$".format(msg)):
         get_client()
 
 
 @gen_cluster(client=True)
 def test_serialize_collections(c, s, a, b):
-    da = pytest.importorskip('dask.array')
+    da = pytest.importorskip("dask.array")
     x = da.arange(10, chunks=(5,)).persist()
 
     def f(x):
@@ -4740,7 +4840,7 @@ def f(x):
     assert result == sum(range(10))
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 1, timeout=100)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 1, timeout=100)
 def test_secede_simple(c, s, a):
     def f():
         client = get_client()
@@ -4752,7 +4852,7 @@ def f():
 
 
 @slow
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 2, timeout=60)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 2, timeout=60)
 def test_secede_balances(c, s, a, b):
     count = threading.active_count()
 
@@ -4766,7 +4866,7 @@ def f(x):
 
     futures = c.map(f, range(100))
     start = time()
-    while not all(f.status == 'finished' for f in futures):
+    while not all(f.status == "finished" for f in futures):
         yield gen.sleep(0.01)
         assert threading.active_count() < count + 50
 
@@ -4782,29 +4882,31 @@ def f(x):
 def test_sub_submit_priority(c, s, a, b):
     def f():
         client = get_client()
-        client.submit(slowinc, 1, delay=0.2, key='slowinc')
+        client.submit(slowinc, 1, delay=0.2, key="slowinc")
 
-    future = c.submit(f, key='f')
+    future = c.submit(f, key="f")
     yield gen.sleep(0.1)
     if len(s.tasks) == 2:
-        assert s.priorities['f'] > s.priorities['slowinc']  # lower values schedule first
+        assert (
+            s.priorities["f"] > s.priorities["slowinc"]
+        )  # lower values schedule first
 
 
 def test_get_client_sync(c, s, a, b):
     results = c.run(lambda: get_worker().scheduler.address)
-    assert results == {w['address']: s['address'] for w in [a, b]}
+    assert results == {w["address"]: s["address"] for w in [a, b]}
 
     results = c.run(lambda: get_client().scheduler.address)
-    assert results == {w['address']: s['address'] for w in [a, b]}
+    assert results == {w["address"]: s["address"] for w in [a, b]}
 
 
 @gen_cluster(client=True)
 def test_serialize_collections_of_futures(c, s, a, b):
-    pd = pytest.importorskip('pandas')
-    dd = pytest.importorskip('dask.dataframe')
+    pd = pytest.importorskip("pandas")
+    dd = pytest.importorskip("dask.dataframe")
     from dask.dataframe.utils import assert_eq
 
-    df = pd.DataFrame({'x': [1, 2, 3]})
+    df = pd.DataFrame({"x": [1, 2, 3]})
     ddf = dd.from_pandas(df, npartitions=2).persist()
     future = yield c.scatter(ddf)
 
@@ -4815,11 +4917,11 @@ def test_serialize_collections_of_futures(c, s, a, b):
 
 
 def test_serialize_collections_of_futures_sync(c):
-    pd = pytest.importorskip('pandas')
-    dd = pytest.importorskip('dask.dataframe')
+    pd = pytest.importorskip("pandas")
+    dd = pytest.importorskip("dask.dataframe")
     from dask.dataframe.utils import assert_eq
 
-    df = pd.DataFrame({'x': [1, 2, 3]})
+    df = pd.DataFrame({"x": [1, 2, 3]})
     ddf = dd.from_pandas(df, npartitions=2).persist()
     future = c.scatter(ddf)
 
@@ -4827,11 +4929,11 @@ def test_serialize_collections_of_futures_sync(c):
     assert_eq(result.compute(), df)
 
     assert future.type == dd.DataFrame
-    assert c.submit(lambda x, y:  assert_eq(x.compute(), y), future, df).result()
+    assert c.submit(lambda x, y: assert_eq(x.compute(), y), future, df).result()
 
 
 def _dynamic_workload(x, delay=0.01):
-    if delay == 'random':
+    if delay == "random":
         sleep(random.random() / 2)
     else:
         sleep(delay)
@@ -4839,8 +4941,9 @@ def _dynamic_workload(x, delay=0.01):
         return 4
     secede()
     client = get_client()
-    futures = client.map(_dynamic_workload, [x + i + 1 for i in range(2)],
-                         pure=False, delay=delay)
+    futures = client.map(
+        _dynamic_workload, [x + i + 1 for i in range(2)], pure=False, delay=delay
+    )
     total = client.submit(sum, futures)
     return total.result()
 
@@ -4856,12 +4959,12 @@ def test_dynamic_workloads_sync(c):
 
 @slow
 def test_dynamic_workloads_sync_random(c):
-    _test_dynamic_workloads_sync(c, delay='random')
+    _test_dynamic_workloads_sync(c, delay="random")
 
 
 @gen_cluster(client=True)
 def test_bytes_keys(c, s, a, b):
-    key = b'inc-123'
+    key = b"inc-123"
     future = c.submit(inc, 1, key=key)
     result = yield future
     assert type(future.key) is bytes
@@ -4874,7 +4977,7 @@ def test_bytes_keys(c, s, a, b):
 def test_unicode_ascii_keys(c, s, a, b):
     # cross-version unicode type (py2: unicode, py3: str)
     uni_type = type(u"")
-    key = u'inc-123'
+    key = u"inc-123"
     future = c.submit(inc, 1, key=key)
     result = yield future
     assert type(future.key) is uni_type
@@ -4887,7 +4990,7 @@ def test_unicode_ascii_keys(c, s, a, b):
 def test_unicode_keys(c, s, a, b):
     # cross-version unicode type (py2: unicode, py3: str)
     uni_type = type(u"")
-    key = u'inc-123\u03bc'
+    key = u"inc-123\u03bc"
     future = c.submit(inc, 1, key=key)
     result = yield future
     assert type(future.key) is uni_type
@@ -4899,8 +5002,8 @@ def test_unicode_keys(c, s, a, b):
     result2 = yield future2
     assert result2 == 3
 
-    future3 = yield c.scatter({u'data-123': 123})
-    result3 = yield future3[u'data-123']
+    future3 = yield c.scatter({u"data-123": 123})
+    result3 = yield future3[u"data-123"]
     assert result3 == 123
 
 
@@ -4918,9 +5021,10 @@ def f():
 
 def test_quiet_quit_when_cluster_leaves(loop_in_thread):
     loop = loop_in_thread
-    with LocalCluster(loop=loop, scheduler_port=0, dashboard_address=None,
-                      silence_logs=False) as cluster:
-        with captured_logger('distributed.comm') as sio:
+    with LocalCluster(
+        loop=loop, scheduler_port=0, dashboard_address=None, silence_logs=False
+    ) as cluster:
+        with captured_logger("distributed.comm") as sio:
             with Client(cluster, loop=loop) as client:
                 futures = client.map(lambda x: x + 1, range(10))
                 sleep(0.05)
@@ -4933,13 +5037,13 @@ def test_quiet_quit_when_cluster_leaves(loop_in_thread):
 
 def test_warn_executor(loop, s, a, b):
     with warnings.catch_warnings(record=True) as record:
-        with Executor(s['address'], loop=loop) as c:
+        with Executor(s["address"], loop=loop) as c:
             pass
 
-    assert any('Client' in str(r.message) for r in record)
+    assert any("Client" in str(r.message) for r in record)
 
 
-@gen_cluster([('127.0.0.1', 4)] * 2, client=True)
+@gen_cluster([("127.0.0.1", 4)] * 2, client=True)
 def test_call_stack_future(c, s, a, b):
     x = c.submit(slowdec, 1, delay=0.5)
     future = c.submit(slowinc, 1, delay=0.5)
@@ -4951,11 +5055,11 @@ def test_call_stack_future(c, s, a, b):
     w = a if future.key in a.executing else b
     assert list(result) == [w.address]
     assert list(result[w.address]) == [future.key]
-    assert 'slowinc' in str(result)
-    assert 'slowdec' not in str(result)
+    assert "slowinc" in str(result)
+    assert "slowdec" not in str(result)
 
 
-@gen_cluster([('127.0.0.1', 4)] * 2, client=True)
+@gen_cluster([("127.0.0.1", 4)] * 2, client=True)
 def test_call_stack_all(c, s, a, b):
     future = c.submit(slowinc, 1, delay=0.5)
     yield gen.sleep(0.1)
@@ -4963,12 +5067,12 @@ def test_call_stack_all(c, s, a, b):
     w = a if a.executing else b
     assert list(result) == [w.address]
     assert list(result[w.address]) == [future.key]
-    assert 'slowinc' in str(result)
+    assert "slowinc" in str(result)
 
 
-@gen_cluster([('127.0.0.1', 4)] * 2, client=True)
+@gen_cluster([("127.0.0.1", 4)] * 2, client=True)
 def test_call_stack_collections(c, s, a, b):
-    da = pytest.importorskip('dask.array')
+    da = pytest.importorskip("dask.array")
     x = da.random.random(100, chunks=(10,)).map_blocks(slowinc, delay=0.5).persist()
     while not a.executing and not b.executing:
         yield gen.sleep(0.001)
@@ -4976,9 +5080,9 @@ def test_call_stack_collections(c, s, a, b):
     assert result
 
 
-@gen_cluster([('127.0.0.1', 4)] * 2, client=True)
+@gen_cluster([("127.0.0.1", 4)] * 2, client=True)
 def test_call_stack_collections_all(c, s, a, b):
-    da = pytest.importorskip('dask.array')
+    da = pytest.importorskip("dask.array")
     x = da.random.random(100, chunks=(10,)).map_blocks(slowinc, delay=0.5).persist()
     while not a.executing and not b.executing:
         yield gen.sleep(0.001)
@@ -4986,39 +5090,42 @@ def test_call_stack_collections_all(c, s, a, b):
     assert result
 
 
-@gen_cluster(client=True, worker_kwargs={'profile_cycle_interval': 100})
+@gen_cluster(client=True, worker_kwargs={"profile_cycle_interval": 100})
 def test_profile(c, s, a, b):
     futures = c.map(slowinc, range(10), delay=0.05, workers=a.address)
     yield wait(futures)
 
     x = yield c.profile(start=time() + 10, stop=time() + 20)
-    assert not x['count']
+    assert not x["count"]
 
     x = yield c.profile(start=0, stop=time())
-    assert x['count'] == sum(p['count'] for _, p in a.profile_history) + a.profile_recent['count']
+    assert (
+        x["count"]
+        == sum(p["count"] for _, p in a.profile_history) + a.profile_recent["count"]
+    )
 
     y = yield c.profile(start=time() - 0.300, stop=time())
-    assert 0 < y['count'] < x['count']
+    assert 0 < y["count"] < x["count"]
 
-    assert not any(p['count'] for _, p in b.profile_history)
+    assert not any(p["count"] for _, p in b.profile_history)
     result = yield c.profile(workers=b.address)
-    assert not result['count']
+    assert not result["count"]
 
 
-@gen_cluster(client=True, worker_kwargs={'profile_cycle_interval': 100})
+@gen_cluster(client=True, worker_kwargs={"profile_cycle_interval": 100})
 def test_profile_keys(c, s, a, b):
     x = c.map(slowinc, range(10), delay=0.05, workers=a.address)
     y = c.map(slowdec, range(10), delay=0.05, workers=a.address)
     yield wait(x + y)
 
-    xp = yield c.profile('slowinc')
-    yp = yield c.profile('slowdec')
+    xp = yield c.profile("slowinc")
+    yp = yield c.profile("slowdec")
     p = yield c.profile()
 
-    assert p['count'] == xp['count'] + yp['count']
+    assert p["count"] == xp["count"] + yp["count"]
 
-    with captured_logger(logging.getLogger('distributed')) as logger:
-        prof = yield c.profile('does-not-exist')
+    with captured_logger(logging.getLogger("distributed")) as logger:
+        prof = yield c.profile("does-not-exist")
         assert prof == profile.create()
     out = logger.getvalue()
     assert not out
@@ -5026,14 +5133,15 @@ def test_profile_keys(c, s, a, b):
 
 @gen_cluster()
 def test_client_with_name(s, a, b):
-    with captured_logger('distributed.scheduler') as sio:
-        client = yield Client(s.address, asynchronous=True, name='foo',
-                              silence_logs=False)
-        assert 'foo' in client.id
+    with captured_logger("distributed.scheduler") as sio:
+        client = yield Client(
+            s.address, asynchronous=True, name="foo", silence_logs=False
+        )
+        assert "foo" in client.id
         yield client.close()
 
     text = sio.getvalue()
-    assert 'foo' in text
+    assert "foo" in text
 
 
 @gen_cluster(client=True)
@@ -5054,7 +5162,7 @@ def test_future_auto_inform(c, s, a, b):
     future = Future(x.key, client)
 
     start = time()
-    while future.status != 'finished':
+    while future.status != "finished":
         yield gen.sleep(0.01)
         assert time() < start + 1
 
@@ -5069,8 +5177,12 @@ def test_client_async_before_loop_starts():
 
 
 @slow
-@gen_cluster(client=True, Worker=Nanny if PY3 else Worker, timeout=60,
-             ncores=[('127.0.0.1', 3)] * 2)
+@gen_cluster(
+    client=True,
+    Worker=Nanny if PY3 else Worker,
+    timeout=60,
+    ncores=[("127.0.0.1", 3)] * 2,
+)
 def test_nested_compute(c, s, a, b):
     def fib(x):
         assert get_worker().get_current_task()
@@ -5089,8 +5201,8 @@ def fib(x):
 
 @gen_cluster(client=True)
 def test_task_metadata(c, s, a, b):
-    yield c.set_metadata('x', 1)
-    result = yield c.get_metadata('x')
+    yield c.set_metadata("x", 1)
+    result = yield c.get_metadata("x")
     assert result == 1
 
     future = c.submit(inc, 1)
@@ -5111,18 +5223,18 @@ def test_task_metadata(c, s, a, b):
     result = yield c.get_metadata(key, None)
     assert result is None
 
-    yield c.set_metadata(['x', 'a'], 1)
-    result = yield c.get_metadata('x')
-    assert result == {'a': 1}
-    yield c.set_metadata(['x', 'b'], 2)
-    result = yield c.get_metadata('x')
-    assert result == {'a': 1, 'b': 2}
-    result = yield c.get_metadata(['x', 'a'])
+    yield c.set_metadata(["x", "a"], 1)
+    result = yield c.get_metadata("x")
+    assert result == {"a": 1}
+    yield c.set_metadata(["x", "b"], 2)
+    result = yield c.get_metadata("x")
+    assert result == {"a": 1, "b": 2}
+    result = yield c.get_metadata(["x", "a"])
     assert result == 1
 
-    yield c.set_metadata(['x', 'a', 'c', 'd'], 1)
-    result = yield c.get_metadata('x')
-    assert result == {'a': {'c': {'d': 1}}, 'b': 2}
+    yield c.set_metadata(["x", "a", "c", "d"], 1)
+    result = yield c.get_metadata("x")
+    assert result == {"a": {"c": {"d": 1}}, "b": 2}
 
 
 @gen_cluster(client=True)
@@ -5132,13 +5244,13 @@ def test_logs(c, s, a, b):
     assert logs
 
     for _, msg in logs:
-        assert 'distributed.scheduler' in msg
+        assert "distributed.scheduler" in msg
 
     w_logs = yield c.get_worker_logs(n=5)
     assert set(w_logs.keys()) == {a.address, b.address}
     for log in w_logs.values():
         for _, msg in log:
-            assert 'distributed.worker' in msg
+            assert "distributed.worker" in msg
 
 
 @gen_cluster(client=True)
@@ -5152,8 +5264,8 @@ def test_avoid_delayed_finalize(c, s, a, b):
 
 @gen_cluster()
 def test_config_scheduler_address(s, a, b):
-    with dask.config.set({'scheduler-address': s.address}):
-        with captured_logger('distributed.client') as sio:
+    with dask.config.set({"scheduler-address": s.address}):
+        with captured_logger("distributed.client") as sio:
             c = yield Client(asynchronous=True)
             assert c.scheduler.address == s.address
 
@@ -5166,18 +5278,18 @@ def test_config_scheduler_address(s, a, b):
 @gen_cluster(client=True)
 def test_warn_when_submitting_large_values(c, s, a, b):
     with warnings.catch_warnings(record=True) as record:
-        future = c.submit(lambda x: x + 1, b'0' * 2000000)
+        future = c.submit(lambda x: x + 1, b"0" * 2000000)
 
     text = str(record[0].message)
-    assert '2.00 MB' in text
-    assert 'large' in text
-    assert '...' in text
+    assert "2.00 MB" in text
+    assert "large" in text
+    assert "..." in text
     assert "'000" in text
     assert "000'" in text
     assert len(text) < 2000
 
     with warnings.catch_warnings(record=True) as record:
-        data = b'0' * 2000000
+        data = b"0" * 2000000
         for i in range(10):
             future = c.submit(lambda x, y: x, data, i)
 
@@ -5201,14 +5313,14 @@ def test_scatter_direct(s, a, b):
 @pytest.mark.skipif(sys.version_info[0] < 3, reason="cloudpickle Py27 issue")
 @gen_cluster(client=True)
 def test_unhashable_function(c, s, a, b):
-    d = {'a': 1}
-    result = yield c.submit(d.get, 'a')
+    d = {"a": 1}
+    result = yield c.submit(d.get, "a")
     assert result == 1
 
 
 @gen_cluster()
 def test_client_name(s, a, b):
-    with dask.config.set({'client-name': 'hello-world'}):
+    with dask.config.set({"client-name": "hello-world"}):
         c = yield Client(s.address, asynchronous=True)
         assert any("hello-world" in name for name in list(s.clients))
 
@@ -5216,39 +5328,45 @@ def test_client_name(s, a, b):
 
 
 def test_client_doesnt_close_given_loop(loop, s, a, b):
-    with Client(s['address'], loop=loop) as c:
+    with Client(s["address"], loop=loop) as c:
         assert c.submit(inc, 1).result() == 2
-    with Client(s['address'], loop=loop) as c:
+    with Client(s["address"], loop=loop) as c:
         assert c.submit(inc, 2).result() == 3
 
 
 @gen_cluster(client=True, ncores=[])
 def test_quiet_scheduler_loss(c, s):
-    c._periodic_callbacks['scheduler-info'].interval = 10
-    with captured_logger(logging.getLogger('distributed.client')) as logger:
+    c._periodic_callbacks["scheduler-info"].interval = 10
+    with captured_logger(logging.getLogger("distributed.client")) as logger:
         yield s.close()
         yield c._update_scheduler_info()
     text = logger.getvalue()
     assert "BrokenPipeError" not in text
 
 
-@pytest.mark.skipif('USER' not in os.environ, reason='no USER env variable')
+@pytest.mark.skipif("USER" not in os.environ, reason="no USER env variable")
 def test_diagnostics_link_env_variable(loop):
-    pytest.importorskip('bokeh')
+    pytest.importorskip("bokeh")
     from distributed.bokeh.scheduler import BokehScheduler
-    with cluster(scheduler_kwargs={'services': {('bokeh', 12355): BokehScheduler}}) as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
-            with dask.config.set({'distributed.dashboard.link': 'http://foo-{USER}:{port}/status'}):
+
+    with cluster(scheduler_kwargs={"services": {("bokeh", 12355): BokehScheduler}}) as (
+        s,
+        [a, b],
+    ):
+        with Client(s["address"], loop=loop) as c:
+            with dask.config.set(
+                {"distributed.dashboard.link": "http://foo-{USER}:{port}/status"}
+            ):
                 text = c._repr_html_()
-                link = 'http://foo-' + os.environ['USER'] + ':12355/status'
+                link = "http://foo-" + os.environ["USER"] + ":12355/status"
                 assert link in text
 
 
 @gen_test()
 def test_client_timeout_2():
-    with dask.config.set({'distributed.comm.timeouts.connect': '10ms'}):
+    with dask.config.set({"distributed.comm.timeouts.connect": "10ms"}):
         start = time()
-        c = Client('127.0.0.1:3755', asynchronous=True)
+        c = Client("127.0.0.1:3755", asynchronous=True)
         with pytest.raises((TimeoutError, IOError)):
             yield c
         stop = time()
@@ -5262,26 +5380,25 @@ def test_client_timeout_2():
 def test_client_active_bad_port():
     import tornado.web
     import tornado.httpserver
-    application = tornado.web.Application([
-        (r"/", tornado.web.RequestHandler),
-    ])
+
+    application = tornado.web.Application([(r"/", tornado.web.RequestHandler)])
     http_server = tornado.httpserver.HTTPServer(application)
     http_server.listen(8080)
-    with dask.config.set({'distributed.comm.timeouts.connect': '10ms'}):
-        c = Client('127.0.0.1:8080', asynchronous=True)
+    with dask.config.set({"distributed.comm.timeouts.connect": "10ms"}):
+        c = Client("127.0.0.1:8080", asynchronous=True)
         with pytest.raises((TimeoutError, IOError)):
             yield c
         yield c._close(fast=True)
     http_server.stop()
 
 
-@pytest.mark.parametrize('direct', [True, False])
+@pytest.mark.parametrize("direct", [True, False])
 def test_turn_off_pickle(direct):
     @gen_cluster()
     def test(s, a, b):
         import numpy as np
-        c = yield Client(s.address, asynchronous=True,
-                         serializers=['dask', 'msgpack'])
+
+        c = yield Client(s.address, asynchronous=True, serializers=["dask", "msgpack"])
         try:
             assert (yield c.submit(inc, 1)) == 2
             yield c.submit(np.ones, 5)
@@ -5319,9 +5436,13 @@ def test(s, a, b):
 @gen_cluster()
 def test_de_serialization(s, a, b):
     import numpy as np
-    c = yield Client(s.address, asynchronous=True,
-                     serializers=['msgpack', 'pickle'],
-                     deserializers=['msgpack'])
+
+    c = yield Client(
+        s.address,
+        asynchronous=True,
+        serializers=["msgpack", "pickle"],
+        deserializers=["msgpack"],
+    )
     try:
         # Can send complex data
         future = yield c.scatter(np.ones(5))
@@ -5336,8 +5457,8 @@ def test_de_serialization(s, a, b):
 @gen_cluster()
 def test_de_serialization_none(s, a, b):
     import numpy as np
-    c = yield Client(s.address, asynchronous=True,
-                     deserializers=['msgpack'])
+
+    c = yield Client(s.address, asynchronous=True, deserializers=["msgpack"])
     try:
         # Can send complex data
         future = yield c.scatter(np.ones(5))
@@ -5362,10 +5483,10 @@ def test_client_repr_closed_sync(loop):
         c._repr_html_()
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)])
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)])
 def test_nested_prioritization(c, s, w):
-    x = delayed(inc)(1, dask_key_name=('a', 2))
-    y = delayed(inc)(2, dask_key_name=('a', 10))
+    x = delayed(inc)(1, dask_key_name=("a", 2))
+    y = delayed(inc)(2, dask_key_name=("a", 10))
 
     o = dask.order.order(merge(x.__dask_graph__(), y.__dask_graph__()))
 
@@ -5373,24 +5494,25 @@ def test_nested_prioritization(c, s, w):
 
     yield wait([fx, fy])
 
-    assert ((o[x.key] < o[y.key]) ==
-            (s.tasks[tokey(fx.key)].priority < s.tasks[tokey(fy.key)].priority))
+    assert (o[x.key] < o[y.key]) == (
+        s.tasks[tokey(fx.key)].priority < s.tasks[tokey(fy.key)].priority
+    )
 
 
 @gen_cluster(client=True)
 def test_scatter_error_cancel(c, s, a, b):
     # https://github.com/dask/distributed/issues/2038
     def bad_fn(x):
-        raise Exception('lol')
+        raise Exception("lol")
 
     x = yield c.scatter(1)
     y = c.submit(bad_fn, x)
     del x
 
     yield wait(y)
-    assert y.status == 'error'
+    assert y.status == "error"
     yield gen.sleep(0.1)
-    assert y.status == 'error'  # not cancelled
+    assert y.status == "error"  # not cancelled
 
 
 def test_no_threads_lingering():
@@ -5432,28 +5554,28 @@ def test_mixing_clients(s, a, b):
 
 @gen_cluster(client=True)
 def test_tuple_keys(c, s, a, b):
-    x = dask.delayed(inc)(1, dask_key_name=('x', 1))
-    y = dask.delayed(inc)(x, dask_key_name=('y', 1))
+    x = dask.delayed(inc)(1, dask_key_name=("x", 1))
+    y = dask.delayed(inc)(x, dask_key_name=("y", 1))
     future = c.compute(y)
     assert (yield future) == 3
 
 
 @gen_cluster(client=True)
 def test_map_large_kwargs_in_graph(c, s, a, b):
-    np = pytest.importorskip('numpy')
+    np = pytest.importorskip("numpy")
     x = np.random.random(100000)
     futures = c.map(lambda a, b: a + b, range(100), b=x)
     while not s.tasks:
         yield gen.sleep(0.01)
 
     assert len(s.tasks) == 101
-    assert any(k.startswith('ndarray') for k in s.tasks)
+    assert any(k.startswith("ndarray") for k in s.tasks)
 
 
 @gen_cluster(client=True)
 def test_retry(c, s, a, b):
     def f():
-        assert dask.config.get('foo')
+        assert dask.config.get("foo")
 
     with dask.config.set(foo=False):
         future = c.submit(f)
@@ -5468,7 +5590,7 @@ def f():
 @gen_cluster(client=True)
 def test_retry_dependencies(c, s, a, b):
     def f():
-        return dask.config.get('foo')
+        return dask.config.get("foo")
 
     x = c.submit(f)
     y = c.submit(inc, x)
@@ -5490,10 +5612,10 @@ def f():
 @gen_cluster(client=True)
 def test_released_dependencies(c, s, a, b):
     def f(x):
-        return dask.config.get('foo') + 1
+        return dask.config.get("foo") + 1
 
-    x = c.submit(inc, 1, key='x')
-    y = c.submit(f, x, key='y')
+    x = c.submit(inc, 1, key="x")
+    y = c.submit(f, x, key="y")
     del x
 
     with pytest.raises(KeyError):
@@ -5507,13 +5629,14 @@ def f(x):
 
 @gen_cluster(client=True, check_new_threads=False)
 def test_profile_bokeh(c, s, a, b):
-    pytest.importorskip('bokeh.plotting')
+    pytest.importorskip("bokeh.plotting")
     from bokeh.model import Model
+
     yield c.map(slowinc, range(10), delay=0.2)
     state, figure = yield c.profile(plot=True)
     assert isinstance(figure, Model)
 
-    with tmpfile('html') as fn:
+    with tmpfile("html") as fn:
         yield c.profile(filename=fn)
         assert os.path.exists(fn)
 
@@ -5522,42 +5645,45 @@ def test_profile_bokeh(c, s, a, b):
 def test_get_mix_futures_and_SubgraphCallable(c, s, a, b):
     future = c.submit(add, 1, 2)
 
-    subgraph = SubgraphCallable({'_2': (add, '_0', '_1'),
-                                 '_3': (add, future, '_2')},
-                                '_3', ('_0', '_1'))
-    dsk = {'a': 1,
-           'b': 2,
-           'c': (subgraph, 'a', 'b'),
-           'd': (subgraph, 'c', 'b')}
+    subgraph = SubgraphCallable(
+        {"_2": (add, "_0", "_1"), "_3": (add, future, "_2")}, "_3", ("_0", "_1")
+    )
+    dsk = {"a": 1, "b": 2, "c": (subgraph, "a", "b"), "d": (subgraph, "c", "b")}
 
-    future2 = c.get(dsk, 'd', sync=False)
+    future2 = c.get(dsk, "d", sync=False)
     result = yield future2
     assert result == 11
 
     # Nested subgraphs
-    subgraph2 = SubgraphCallable({'_2': (subgraph, '_0', '_1'),
-                                  '_3': (subgraph, '_2', '_1'),
-                                  '_4': (add, '_3', future2)},
-                                 '_4', ('_0', '_1'))
-
-    dsk2 = {'e': 1, 'f': 2, 'g': (subgraph2, 'e', 'f')}
-
-    result = yield c.get(dsk2, 'g', sync=False)
+    subgraph2 = SubgraphCallable(
+        {
+            "_2": (subgraph, "_0", "_1"),
+            "_3": (subgraph, "_2", "_1"),
+            "_4": (add, "_3", future2),
+        },
+        "_4",
+        ("_0", "_1"),
+    )
+
+    dsk2 = {"e": 1, "f": 2, "g": (subgraph2, "e", "f")}
+
+    result = yield c.get(dsk2, "g", sync=False)
     assert result == 22
 
 
 @gen_cluster(client=True)
 def test_get_mix_futures_and_SubgraphCallable_dask_dataframe(c, s, a, b):
-    dd = pytest.importorskip('dask.dataframe')
+    dd = pytest.importorskip("dask.dataframe")
     import pandas as pd
-    df = pd.DataFrame({'x': range(1, 11)})
+
+    df = pd.DataFrame({"x": range(1, 11)})
     ddf = dd.from_pandas(df, npartitions=2).persist()
     ddf = ddf.map_partitions(lambda x: x)
-    ddf['x'] = ddf['x'].astype('f8')
+    ddf["x"] = ddf["x"].astype("f8")
     ddf = ddf.map_partitions(lambda x: x)
-    ddf['x'] = ddf['x'].astype('f8')
+    ddf["x"] = ddf["x"].astype("f8")
     result = yield c.compute(ddf)
-    assert result.equals(df.astype('f8'))
+    assert result.equals(df.astype("f8"))
 
 
 if sys.version_info >= (3, 5):
diff --git a/distributed/tests/test_client_executor.py b/distributed/tests/test_client_executor.py
index 117c9c31dc3..a7f10491efb 100644
--- a/distributed/tests/test_client_executor.py
+++ b/distributed/tests/test_client_executor.py
@@ -4,14 +4,19 @@
 import time
 
 from concurrent.futures import (
-    CancelledError, TimeoutError, Future, wait, as_completed,
-    FIRST_COMPLETED, FIRST_EXCEPTION)
+    CancelledError,
+    TimeoutError,
+    Future,
+    wait,
+    as_completed,
+    FIRST_COMPLETED,
+    FIRST_EXCEPTION,
+)
 
 import pytest
 from toolz import take
 
-from distributed.utils_test import (slowinc, slowadd, slowdec,
-                                    inc, throws, varying)
+from distributed.utils_test import slowinc, slowadd, slowdec, inc, throws, varying
 from distributed.utils_test import client, cluster_fixture, loop, s, a, b  # noqa: F401
 
 
@@ -174,19 +179,20 @@ def test_pure(client):
 
 def test_workers(client, s, a, b):
     N = 10
-    with client.get_executor(workers=[b['address']]) as e:
+    with client.get_executor(workers=[b["address"]]) as e:
         fs = [e.submit(slowinc, i) for i in range(N)]
         wait(fs)
         has_what = client.has_what()
-        assert not has_what.get(a['address'])
-        assert len(has_what[b['address']]) == N
+        assert not has_what.get(a["address"])
+        assert len(has_what[b["address"]]) == N
 
 
 def test_unsupported_arguments(client, s, a, b):
     with pytest.raises(TypeError) as excinfo:
-        client.get_executor(workers=[b['address']], foo=1, bar=2)
-    assert ("unsupported arguments to ClientExecutor: ['bar', 'foo']"
-            in str(excinfo.value))
+        client.get_executor(workers=[b["address"]], foo=1, bar=2)
+    assert "unsupported arguments to ClientExecutor: ['bar', 'foo']" in str(
+        excinfo.value
+    )
 
 
 def test_retries(client):
diff --git a/distributed/tests/test_collections.py b/distributed/tests/test_collections.py
index cb51d62c5fd..f640d2d21e0 100644
--- a/distributed/tests/test_collections.py
+++ b/distributed/tests/test_collections.py
@@ -2,22 +2,25 @@
 
 
 import pytest
-pytest.importorskip('numpy')
-pytest.importorskip('pandas')
+
+pytest.importorskip("numpy")
+pytest.importorskip("pandas")
 
 import dask.dataframe as dd
 import dask.bag as db
 from distributed.client import wait
 from distributed.utils_test import gen_cluster
-from distributed.utils_test import client, cluster_fixture, loop # noqa F401
+from distributed.utils_test import client, cluster_fixture, loop  # noqa F401
 import numpy as np
 import pandas as pd
 import pandas.util.testing as tm
 
 
-dfs = [pd.DataFrame({'x': [1, 2, 3]}, index=[0, 10, 20]),
-       pd.DataFrame({'x': [4, 5, 6]}, index=[30, 40, 50]),
-       pd.DataFrame({'x': [7, 8, 9]}, index=[60, 70, 80])]
+dfs = [
+    pd.DataFrame({"x": [1, 2, 3]}, index=[0, 10, 20]),
+    pd.DataFrame({"x": [4, 5, 6]}, index=[30, 40, 50]),
+    pd.DataFrame({"x": [7, 8, 9]}, index=[60, 70, 80]),
+]
 
 
 def assert_equal(a, b):
@@ -34,9 +37,10 @@ def assert_equal(a, b):
 
 @gen_cluster(timeout=240, client=True)
 def test_dataframes(c, s, a, b):
-    df = pd.DataFrame({'x': np.random.random(1000),
-                       'y': np.random.random(1000)},
-                      index=np.arange(1000))
+    df = pd.DataFrame(
+        {"x": np.random.random(1000), "y": np.random.random(1000)},
+        index=np.arange(1000),
+    )
     ldf = dd.from_pandas(df, npartitions=10)
 
     rdf = c.persist(ldf)
@@ -46,18 +50,20 @@ def test_dataframes(c, s, a, b):
     remote = c.compute(rdf)
     result = yield remote
 
-    tm.assert_frame_equal(result, ldf.compute(scheduler='sync'))
-
-    exprs = [lambda df: df.x.mean(),
-             lambda df: df.y.std(),
-             lambda df: df.assign(z=df.x + df.y).drop_duplicates(),
-             lambda df: df.index,
-             lambda df: df.x,
-             lambda df: df.x.cumsum(),
-             lambda df: df.groupby(['x', 'y']).count(),
-             lambda df: df.loc[50:75]]
+    tm.assert_frame_equal(result, ldf.compute(scheduler="sync"))
+
+    exprs = [
+        lambda df: df.x.mean(),
+        lambda df: df.y.std(),
+        lambda df: df.assign(z=df.x + df.y).drop_duplicates(),
+        lambda df: df.index,
+        lambda df: df.x,
+        lambda df: df.x.cumsum(),
+        lambda df: df.groupby(["x", "y"]).count(),
+        lambda df: df.loc[50:75],
+    ]
     for f in exprs:
-        local = f(ldf).compute(scheduler='sync')
+        local = f(ldf).compute(scheduler="sync")
         remote = c.compute(f(rdf))
         remote = yield remote
         assert_equal(local, remote)
@@ -67,27 +73,27 @@ def test_dataframes(c, s, a, b):
 def test__dask_array_collections(c, s, a, b):
     import dask.array as da
 
-    x_dsk = {('x', i, j): np.random.random((3, 3)) for i in range(3)
-             for j in range(2)}
-    y_dsk = {('y', i, j): np.random.random((3, 3)) for i in range(2)
-             for j in range(3)}
+    x_dsk = {("x", i, j): np.random.random((3, 3)) for i in range(3) for j in range(2)}
+    y_dsk = {("y", i, j): np.random.random((3, 3)) for i in range(2) for j in range(3)}
     x_futures = yield c._scatter(x_dsk)
     y_futures = yield c._scatter(y_dsk)
 
     dt = np.random.random(0).dtype
-    x_local = da.Array(x_dsk, 'x', ((3, 3, 3), (3, 3)), dt)
-    y_local = da.Array(y_dsk, 'y', ((3, 3), (3, 3, 3)), dt)
+    x_local = da.Array(x_dsk, "x", ((3, 3, 3), (3, 3)), dt)
+    y_local = da.Array(y_dsk, "y", ((3, 3), (3, 3, 3)), dt)
 
-    x_remote = da.Array(x_futures, 'x', ((3, 3, 3), (3, 3)), dt)
-    y_remote = da.Array(y_futures, 'y', ((3, 3), (3, 3, 3)), dt)
+    x_remote = da.Array(x_futures, "x", ((3, 3, 3), (3, 3)), dt)
+    y_remote = da.Array(y_futures, "y", ((3, 3), (3, 3, 3)), dt)
 
-    exprs = [lambda x, y: x.T + y,
-             lambda x, y: x.mean() + y.mean(),
-             lambda x, y: x.dot(y).std(axis=0),
-             lambda x, y: x - x.mean(axis=1)[:, None]]
+    exprs = [
+        lambda x, y: x.T + y,
+        lambda x, y: x.mean() + y.mean(),
+        lambda x, y: x.dot(y).std(axis=0),
+        lambda x, y: x - x.mean(axis=1)[:, None],
+    ]
 
     for expr in exprs:
-        local = expr(x_local, y_local).compute(scheduler='sync')
+        local = expr(x_local, y_local).compute(scheduler="sync")
 
         remote = c.compute(expr(x_remote, y_remote))
         remote = yield remote
@@ -99,18 +105,23 @@ def test__dask_array_collections(c, s, a, b):
 def test_bag_groupby_tasks_default(c, s, a, b):
     b = db.range(100, npartitions=10)
     b2 = b.groupby(lambda x: x % 13)
-    assert not any('partd' in k[0] for k in b2.dask)
+    assert not any("partd" in k[0] for k in b2.dask)
 
 
-@pytest.mark.parametrize('wait', [wait, lambda x: None])
+@pytest.mark.parametrize("wait", [wait, lambda x: None])
 def test_dataframe_set_index_sync(wait, client):
-    df = dd.demo.make_timeseries('2000', '2001',
-                                 {'value': float, 'name': str, 'id': int},
-                                 freq='2H', partition_freq='1M', seed=1)
+    df = dd.demo.make_timeseries(
+        "2000",
+        "2001",
+        {"value": float, "name": str, "id": int},
+        freq="2H",
+        partition_freq="1M",
+        seed=1,
+    )
     df = client.persist(df)
     wait(df)
 
-    df2 = df.set_index('name', shuffle='tasks')
+    df2 = df.set_index("name", shuffle="tasks")
     df2 = client.persist(df2)
 
     assert len(df2)
@@ -119,7 +130,7 @@ def test_dataframe_set_index_sync(wait, client):
 def test_loc_sync(client):
     df = pd.util.testing.makeTimeDataFrame()
     ddf = dd.from_pandas(df, npartitions=10)
-    ddf.loc['2000-01-17':'2000-01-24'].compute()
+    ddf.loc["2000-01-17":"2000-01-24"].compute()
 
 
 def test_rolling_sync(client):
@@ -132,40 +143,40 @@ def test_rolling_sync(client):
 def test_loc(c, s, a, b):
     df = pd.util.testing.makeTimeDataFrame()
     ddf = dd.from_pandas(df, npartitions=10)
-    future = c.compute(ddf.loc['2000-01-17':'2000-01-24'])
+    future = c.compute(ddf.loc["2000-01-17":"2000-01-24"])
     yield future
 
 
 def test_dataframe_groupby_tasks(client):
     df = pd.util.testing.makeTimeDataFrame()
-    df['A'] = df.A // 0.1
-    df['B'] = df.B // 0.1
+    df["A"] = df.A // 0.1
+    df["B"] = df.B // 0.1
     ddf = dd.from_pandas(df, npartitions=10)
 
-    for ind in [lambda x: 'A', lambda x: x.A]:
+    for ind in [lambda x: "A", lambda x: x.A]:
         a = df.groupby(ind(df)).apply(len)
         b = ddf.groupby(ind(ddf)).apply(len, meta=int)
-        assert_equal(a, b.compute(scheduler='sync').sort_index())
-        assert not any('partd' in k[0] for k in b.dask)
+        assert_equal(a, b.compute(scheduler="sync").sort_index())
+        assert not any("partd" in k[0] for k in b.dask)
 
         a = df.groupby(ind(df)).B.apply(len)
-        b = ddf.groupby(ind(ddf)).B.apply(len, meta=('B', int))
-        assert_equal(a, b.compute(scheduler='sync').sort_index())
-        assert not any('partd' in k[0] for k in b.dask)
+        b = ddf.groupby(ind(ddf)).B.apply(len, meta=("B", int))
+        assert_equal(a, b.compute(scheduler="sync").sort_index())
+        assert not any("partd" in k[0] for k in b.dask)
 
     with pytest.raises((NotImplementedError, ValueError)):
-        ddf.groupby(ddf[['A', 'B']]).apply(len, meta=int)
+        ddf.groupby(ddf[["A", "B"]]).apply(len, meta=int)
 
-    a = df.groupby(['A', 'B']).apply(len)
-    b = ddf.groupby(['A', 'B']).apply(len, meta=int)
+    a = df.groupby(["A", "B"]).apply(len)
+    b = ddf.groupby(["A", "B"]).apply(len, meta=int)
 
-    assert_equal(a, b.compute(scheduler='sync').sort_index())
+    assert_equal(a, b.compute(scheduler="sync").sort_index())
 
 
 @gen_cluster(client=True)
 def test_sparse_arrays(c, s, a, b):
-    sparse = pytest.importorskip('sparse')
-    da = pytest.importorskip('dask.array')
+    sparse = pytest.importorskip("sparse")
+    da = pytest.importorskip("dask.array")
 
     x = da.random.random((100, 10), chunks=(10, 10))
     x[x < 0.95] = 0
diff --git a/distributed/tests/test_compatibility.py b/distributed/tests/test_compatibility.py
index 54a39ad4a34..42eae448aa1 100644
--- a/distributed/tests/test_compatibility.py
+++ b/distributed/tests/test_compatibility.py
@@ -1,11 +1,10 @@
 from __future__ import print_function, division, absolute_import
 
-from distributed.compatibility import (
-    gzip_compress, gzip_decompress, finalize)
+from distributed.compatibility import gzip_compress, gzip_decompress, finalize
 
 
 def test_gzip():
-    b = b'Hello, world!'
+    b = b"Hello, world!"
     c = gzip_compress(b)
     d = gzip_decompress(c)
     assert b == d
diff --git a/distributed/tests/test_config.py b/distributed/tests/test_config.py
index f14f07308c2..cdd4070f7bb 100644
--- a/distributed/tests/test_config.py
+++ b/distributed/tests/test_config.py
@@ -8,8 +8,12 @@
 
 import pytest
 
-from distributed.utils_test import (captured_handler, captured_logger,
-                                    new_config, new_config_file)
+from distributed.utils_test import (
+    captured_handler,
+    captured_logger,
+    new_config,
+    new_config_file,
+)
 from distributed.config import initialize_logging
 
 
@@ -20,12 +24,15 @@ def dump_logger_list():
     print("== Loggers (name, level, effective level, propagate) ==")
 
     def logger_info(name, logger):
-        return (name, logging.getLevelName(logger.level),
-                logging.getLevelName(logger.getEffectiveLevel()),
-                logger.propagate)
+        return (
+            name,
+            logging.getLevelName(logger.level),
+            logging.getLevelName(logger.getEffectiveLevel()),
+            logger.propagate,
+        )
 
     infos = []
-    infos.append(logger_info('<root>', root))
+    infos.append(logger_info("<root>", root))
 
     for name, logger in sorted(loggers.items()):
         if not isinstance(logger, logging.Logger):
@@ -44,28 +51,28 @@ def test_logging_default():
     """
     Test default logging configuration.
     """
-    d = logging.getLogger('distributed')
+    d = logging.getLogger("distributed")
     assert len(d.handlers) == 1
     assert isinstance(d.handlers[0], logging.StreamHandler)
 
     # Work around Bokeh messing with the root logger level
     # https://github.com/bokeh/bokeh/issues/5793
-    root = logging.getLogger('')
+    root = logging.getLogger("")
     old_root_level = root.level
-    root.setLevel('WARN')
+    root.setLevel("WARN")
 
     for handler in d.handlers:
-        handler.setLevel('INFO')
+        handler.setLevel("INFO")
 
     try:
-        dfb = logging.getLogger('distributed.foo.bar')
-        f = logging.getLogger('foo')
-        fb = logging.getLogger('foo.bar')
+        dfb = logging.getLogger("distributed.foo.bar")
+        f = logging.getLogger("foo")
+        fb = logging.getLogger("foo.bar")
 
         with captured_handler(d.handlers[0]) as distributed_log:
             with captured_logger(root, level=logging.ERROR) as foreign_log:
                 h = logging.StreamHandler(foreign_log)
-                fmt = '[%(levelname)s in %(name)s] - %(message)s'
+                fmt = "[%(levelname)s in %(name)s] - %(message)s"
                 h.setFormatter(logging.Formatter(fmt))
                 fb.addHandler(h)
                 fb.propagate = False
@@ -92,10 +99,7 @@ def test_logging_default():
 
         # foreign logs should be unaffected by distributed's logging
         # configuration.  They get the default ERROR level from logging.
-        assert foreign_log == [
-            "[ERROR in foo.bar] - 5: error",
-            "7: error",
-        ]
+        assert foreign_log == ["[ERROR in foo.bar] - 5: error", "7: error"]
 
     finally:
         root.setLevel(old_root_level)
@@ -110,12 +114,7 @@ def test_logging_simple():
     """
     Test simple ("old-style") logging configuration.
     """
-    c = {
-        'logging': {
-            'distributed.foo': 'info',
-            'distributed.foo.bar': 'error',
-        }
-    }
+    c = {"logging": {"distributed.foo": "info", "distributed.foo.bar": "error"}}
     # Must test using a subprocess to avoid wrecking pre-existing configuration
     with new_config_file(c):
         code = """if 1:
@@ -151,35 +150,30 @@ def test_logging_extended():
     Test extended ("new-style") logging configuration.
     """
     c = {
-        'logging': {
-            'version': '1',
-            'formatters': {
-                'simple': {
-                    'format': '%(levelname)s: %(name)s: %(message)s',
-                },
+        "logging": {
+            "version": "1",
+            "formatters": {
+                "simple": {"format": "%(levelname)s: %(name)s: %(message)s"}
             },
-            'handlers': {
-                'console': {
-                    'class': 'logging.StreamHandler',
-                    'stream': 'ext://sys.stderr',
-                    'formatter': 'simple',
-                },
+            "handlers": {
+                "console": {
+                    "class": "logging.StreamHandler",
+                    "stream": "ext://sys.stderr",
+                    "formatter": "simple",
+                }
             },
-            'loggers': {
-                'distributed.foo': {
-                    'level': 'INFO',
+            "loggers": {
+                "distributed.foo": {
+                    "level": "INFO",
                     #'handlers': ['console'],
                 },
-                'distributed.foo.bar': {
-                    'level': 'ERROR',
+                "distributed.foo.bar": {
+                    "level": "ERROR",
                     #'handlers': ['console'],
                 },
             },
-            'root': {
-                'level': 'WARNING',
-                'handlers': ['console'],
-            },
-        },
+            "root": {"level": "WARNING", "handlers": ["console"]},
+        }
     }
     # Must test using a subprocess to avoid wrecking pre-existing configuration
     with new_config_file(c):
@@ -217,7 +211,7 @@ def test_logging_mutual_exclusive():
     """
     Ensure that 'logging-file-config' and 'logging' have to be mutual exclusive.
     """
-    config = {'logging': {'dask': 'warning'}, 'logging-file-config': '/path/to/config'}
+    config = {"logging": {"dask": "warning"}, "logging-file-config": "/path/to/config"}
     with pytest.raises(RuntimeError):
         initialize_logging(config)
 
@@ -259,9 +253,9 @@ def test_logging_file_config():
 handlers=console
 qualname=foo.bar
 """
-    with tempfile.NamedTemporaryFile(mode='w', delete=False) as logging_config:
+    with tempfile.NamedTemporaryFile(mode="w", delete=False) as logging_config:
         logging_config.write(logging_config_contents)
-    dask_config = {'logging-file-config': logging_config.name}
+    dask_config = {"logging-file-config": logging_config.name}
     with new_config_file(dask_config):
         code = """if 1:
             import logging
diff --git a/distributed/tests/test_core.py b/distributed/tests/test_core.py
index 07cb4e2214a..2c8f63de6ff 100644
--- a/distributed/tests/test_core.py
+++ b/distributed/tests/test_core.py
@@ -10,20 +10,37 @@
 
 import dask
 from distributed.compatibility import finalize, get_thread_identity
-from distributed.core import (pingpong, Server, rpc, connect, send_recv,
-                               coerce_to_address, ConnectionPool)
+from distributed.core import (
+    pingpong,
+    Server,
+    rpc,
+    connect,
+    send_recv,
+    coerce_to_address,
+    ConnectionPool,
+)
 from distributed.protocol.compression import compressions
 
 from distributed.metrics import time
 from distributed.protocol import to_serialize
 from distributed.utils import get_ip, get_ipv6
 from distributed.utils_test import (
-    slow, gen_test, gen_cluster, has_ipv6,
-    assert_can_connect, assert_cannot_connect,
+    slow,
+    gen_test,
+    gen_cluster,
+    has_ipv6,
+    assert_can_connect,
+    assert_cannot_connect,
     assert_can_connect_from_everywhere_4,
-    assert_can_connect_from_everywhere_4_6, assert_can_connect_from_everywhere_6,
-    assert_can_connect_locally_4, assert_can_connect_locally_6,
-    tls_security, captured_logger, inc, throws)
+    assert_can_connect_from_everywhere_4_6,
+    assert_can_connect_from_everywhere_6,
+    assert_can_connect_locally_4,
+    assert_can_connect_locally_6,
+    tls_security,
+    captured_logger,
+    inc,
+    throws,
+)
 from distributed.utils_test import loop  # noqa F401
 
 
@@ -40,6 +57,7 @@ class CountedObject(object):
     """
     A class which counts the number of live instances.
     """
+
     n_instances = 0
 
     # Use __new__, as __init__ can be bypassed by pickle.
@@ -55,39 +73,40 @@ def _finalize(cls, *args):
 
 
 def echo_serialize(comm, x):
-    return {'result': to_serialize(x)}
+    return {"result": to_serialize(x)}
 
 
 def echo_no_serialize(comm, x):
-    return {'result': x}
+    return {"result": x}
 
 
 def test_server(loop):
     """
     Simple Server test.
     """
+
     @gen.coroutine
     def f():
-        server = Server({'ping': pingpong})
+        server = Server({"ping": pingpong})
         with pytest.raises(ValueError):
             server.port
         server.listen(8881)
         assert server.port == 8881
-        assert server.address == ('tcp://%s:8881' % get_ip())
+        assert server.address == ("tcp://%s:8881" % get_ip())
 
-        for addr in ('127.0.0.1:8881', 'tcp://127.0.0.1:8881', server.address):
+        for addr in ("127.0.0.1:8881", "tcp://127.0.0.1:8881", server.address):
             comm = yield connect(addr)
 
-            n = yield comm.write({'op': 'ping'})
+            n = yield comm.write({"op": "ping"})
             assert isinstance(n, int)
             assert 4 <= n <= 1000
 
             response = yield comm.read()
-            assert response == b'pong'
+            assert response == b"pong"
 
-            yield comm.write({'op': 'ping', 'close': True})
+            yield comm.write({"op": "ping", "close": True})
             response = yield comm.read()
-            assert response == b'pong'
+            assert response == b"pong"
 
             yield comm.close()
 
@@ -99,16 +118,16 @@ def f():
 def test_server_raises_on_blocked_handlers(loop):
     @gen.coroutine
     def f():
-        server = Server({'ping': pingpong}, blocked_handlers=['ping'])
+        server = Server({"ping": pingpong}, blocked_handlers=["ping"])
         server.listen(8881)
 
         comm = yield connect(server.address)
-        yield comm.write({'op': 'ping'})
+        yield comm.write({"op": "ping"})
         msg = yield comm.read()
 
-        assert 'exception' in msg
-        assert isinstance(msg['exception'], ValueError)
-        assert "'ping' handler has been explicitly disallowed" in repr(msg['exception'])
+        assert "exception" in msg
+        assert isinstance(msg["exception"], ValueError)
+        assert "'ping' handler has been explicitly disallowed" in repr(msg["exception"])
 
         comm.close()
         server.stop()
@@ -139,94 +158,96 @@ def listen_on(cls, *args, **kwargs):
 
     with listen_on(Server, 7800) as server:
         assert server.port == 7800
-        assert server.address == 'tcp://%s:%d' % (EXTERNAL_IP4, server.port)
+        assert server.address == "tcp://%s:%d" % (EXTERNAL_IP4, server.port)
         yield assert_can_connect(server.address)
         yield assert_can_connect_from_everywhere_4_6(server.port)
 
     with listen_on(Server) as server:
         assert server.port > 0
-        assert server.address == 'tcp://%s:%d' % (EXTERNAL_IP4, server.port)
+        assert server.address == "tcp://%s:%d" % (EXTERNAL_IP4, server.port)
         yield assert_can_connect(server.address)
         yield assert_can_connect_from_everywhere_4_6(server.port)
 
     with listen_on(MyServer) as server:
         assert server.port == MyServer.default_port
-        assert server.address == 'tcp://%s:%d' % (EXTERNAL_IP4, server.port)
+        assert server.address == "tcp://%s:%d" % (EXTERNAL_IP4, server.port)
         yield assert_can_connect(server.address)
         yield assert_can_connect_from_everywhere_4_6(server.port)
 
-    with listen_on(Server, ('', 7801)) as server:
+    with listen_on(Server, ("", 7801)) as server:
         assert server.port == 7801
-        assert server.address == 'tcp://%s:%d' % (EXTERNAL_IP4, server.port)
+        assert server.address == "tcp://%s:%d" % (EXTERNAL_IP4, server.port)
         yield assert_can_connect(server.address)
         yield assert_can_connect_from_everywhere_4_6(server.port)
 
-    with listen_on(Server, 'tcp://:7802') as server:
+    with listen_on(Server, "tcp://:7802") as server:
         assert server.port == 7802
-        assert server.address == 'tcp://%s:%d' % (EXTERNAL_IP4, server.port)
+        assert server.address == "tcp://%s:%d" % (EXTERNAL_IP4, server.port)
         yield assert_can_connect(server.address)
         yield assert_can_connect_from_everywhere_4_6(server.port)
 
     # Only IPv4
 
-    with listen_on(Server, ('0.0.0.0', 7810)) as server:
+    with listen_on(Server, ("0.0.0.0", 7810)) as server:
         assert server.port == 7810
-        assert server.address == 'tcp://%s:%d' % (EXTERNAL_IP4, server.port)
+        assert server.address == "tcp://%s:%d" % (EXTERNAL_IP4, server.port)
         yield assert_can_connect(server.address)
         yield assert_can_connect_from_everywhere_4(server.port)
 
-    with listen_on(Server, ('127.0.0.1', 7811)) as server:
+    with listen_on(Server, ("127.0.0.1", 7811)) as server:
         assert server.port == 7811
-        assert server.address == 'tcp://127.0.0.1:%d' % server.port
+        assert server.address == "tcp://127.0.0.1:%d" % server.port
         yield assert_can_connect(server.address)
         yield assert_can_connect_locally_4(server.port)
 
-    with listen_on(Server, 'tcp://127.0.0.1:7812') as server:
+    with listen_on(Server, "tcp://127.0.0.1:7812") as server:
         assert server.port == 7812
-        assert server.address == 'tcp://127.0.0.1:%d' % server.port
+        assert server.address == "tcp://127.0.0.1:%d" % server.port
         yield assert_can_connect(server.address)
         yield assert_can_connect_locally_4(server.port)
 
     # Only IPv6
 
     if has_ipv6():
-        with listen_on(Server, ('::', 7813)) as server:
+        with listen_on(Server, ("::", 7813)) as server:
             assert server.port == 7813
-            assert server.address == 'tcp://[%s]:%d' % (EXTERNAL_IP6, server.port)
+            assert server.address == "tcp://[%s]:%d" % (EXTERNAL_IP6, server.port)
             yield assert_can_connect(server.address)
             yield assert_can_connect_from_everywhere_6(server.port)
 
-        with listen_on(Server, ('::1', 7814)) as server:
+        with listen_on(Server, ("::1", 7814)) as server:
             assert server.port == 7814
-            assert server.address == 'tcp://[::1]:%d' % server.port
+            assert server.address == "tcp://[::1]:%d" % server.port
             yield assert_can_connect(server.address)
             yield assert_can_connect_locally_6(server.port)
 
-        with listen_on(Server, 'tcp://[::1]:7815') as server:
+        with listen_on(Server, "tcp://[::1]:7815") as server:
             assert server.port == 7815
-            assert server.address == 'tcp://[::1]:%d' % server.port
+            assert server.address == "tcp://[::1]:%d" % server.port
             yield assert_can_connect(server.address)
             yield assert_can_connect_locally_6(server.port)
 
     # TLS
 
     sec = tls_security()
-    with listen_on(Server, 'tls://',
-                   listen_args=sec.get_listen_args('scheduler')) as server:
-        assert server.address.startswith('tls://')
-        yield assert_can_connect(server.address,
-                                 connection_args=sec.get_connection_args('client'))
+    with listen_on(
+        Server, "tls://", listen_args=sec.get_listen_args("scheduler")
+    ) as server:
+        assert server.address.startswith("tls://")
+        yield assert_can_connect(
+            server.address, connection_args=sec.get_connection_args("client")
+        )
 
     # InProc
 
-    with listen_on(Server, 'inproc://') as server:
+    with listen_on(Server, "inproc://") as server:
         inproc_addr1 = server.address
-        assert inproc_addr1.startswith('inproc://%s/%d/' % (get_ip(), os.getpid()))
+        assert inproc_addr1.startswith("inproc://%s/%d/" % (get_ip(), os.getpid()))
         yield assert_can_connect(inproc_addr1)
 
-        with listen_on(Server, 'inproc://') as server2:
+        with listen_on(Server, "inproc://") as server2:
             inproc_addr2 = server2.address
-            assert inproc_addr2.startswith('inproc://%s/%d/' % (get_ip(), os.getpid()))
+            assert inproc_addr2.startswith("inproc://%s/%d/" % (get_ip(), os.getpid()))
             yield assert_can_connect(inproc_addr2)
 
         yield assert_can_connect(inproc_addr1)
@@ -235,58 +256,59 @@ def listen_on(cls, *args, **kwargs):
 
 @gen.coroutine
 def check_rpc(listen_addr, rpc_addr=None, listen_args=None, connection_args=None):
-    server = Server({'ping': pingpong})
+    server = Server({"ping": pingpong})
     server.listen(listen_addr, listen_args=listen_args)
     if rpc_addr is None:
         rpc_addr = server.address
 
     with rpc(rpc_addr, connection_args=connection_args) as remote:
         response = yield remote.ping()
-        assert response == b'pong'
+        assert response == b"pong"
         assert remote.comms
 
         response = yield remote.ping(close=True)
-        assert response == b'pong'
+        assert response == b"pong"
         response = yield remote.ping()
-        assert response == b'pong'
+        assert response == b"pong"
 
     assert not remote.comms
-    assert remote.status == 'closed'
+    assert remote.status == "closed"
 
     server.stop()
 
 
 @gen_test()
 def test_rpc_default():
-    yield check_rpc(8883, '127.0.0.1:8883')
+    yield check_rpc(8883, "127.0.0.1:8883")
     yield check_rpc(8883)
 
 
 @gen_test()
 def test_rpc_tcp():
-    yield check_rpc('tcp://:8883', 'tcp://127.0.0.1:8883')
-    yield check_rpc('tcp://')
+    yield check_rpc("tcp://:8883", "tcp://127.0.0.1:8883")
+    yield check_rpc("tcp://")
 
 
 @gen_test()
 def test_rpc_tls():
     sec = tls_security()
-    yield check_rpc('tcp://', None, sec.get_listen_args('scheduler'),
-                    sec.get_connection_args('worker'))
+    yield check_rpc(
+        "tcp://",
+        None,
+        sec.get_listen_args("scheduler"),
+        sec.get_connection_args("worker"),
+    )
 
 
 @gen_test()
 def test_rpc_inproc():
-    yield check_rpc('inproc://', None)
+    yield check_rpc("inproc://", None)
 
 
 def test_rpc_inputs():
-    L = [rpc('127.0.0.1:8884'),
-         rpc(('127.0.0.1', 8884)),
-         rpc('tcp://127.0.0.1:8884'),
-         ]
+    L = [rpc("127.0.0.1:8884"), rpc(("127.0.0.1", 8884)), rpc("tcp://127.0.0.1:8884")]
 
-    assert all(r.address == 'tcp://127.0.0.1:8884' for r in L), L
+    assert all(r.address == "tcp://127.0.0.1:8884" for r in L), L
 
     for r in L:
         r.close_rpc()
@@ -296,7 +318,7 @@ def test_rpc_inputs():
 def check_rpc_message_lifetime(*listen_args):
     # Issue #956: rpc arguments and result shouldn't be kept alive longer
     # than necessary
-    server = Server({'echo': echo_serialize})
+    server = Server({"echo": echo_serialize})
     server.listen(*listen_args)
 
     # Sanity check
@@ -308,12 +330,12 @@ def check_rpc_message_lifetime(*listen_args):
     with rpc(server.address) as remote:
         obj = CountedObject()
         res = yield remote.echo(x=to_serialize(obj))
-        assert isinstance(res['result'], CountedObject)
+        assert isinstance(res["result"], CountedObject)
         # Make sure resource cleanup code in coroutines runs
         yield gen.sleep(0.05)
 
         w1 = weakref.ref(obj)
-        w2 = weakref.ref(res['result'])
+        w2 = weakref.ref(res["result"])
         del obj, res
 
         assert w1() is None
@@ -331,12 +353,12 @@ def test_rpc_message_lifetime_default():
 
 @gen_test()
 def test_rpc_message_lifetime_tcp():
-    yield check_rpc_message_lifetime('tcp://')
+    yield check_rpc_message_lifetime("tcp://")
 
 
 @gen_test()
 def test_rpc_message_lifetime_inproc():
-    yield check_rpc_message_lifetime('inproc://')
+    yield check_rpc_message_lifetime("inproc://")
 
 
 @gen.coroutine
@@ -346,7 +368,7 @@ def g():
         for i in range(10):
             yield remote.ping()
 
-    server = Server({'ping': pingpong})
+    server = Server({"ping": pingpong})
     server.listen(listen_arg)
 
     remote = rpc(server.address)
@@ -360,26 +382,26 @@ def g():
 
 @gen_test()
 def test_rpc_with_many_connections_tcp():
-    yield check_rpc_with_many_connections('tcp://')
+    yield check_rpc_with_many_connections("tcp://")
 
 
 @gen_test()
 def test_rpc_with_many_connections_inproc():
-    yield check_rpc_with_many_connections('inproc://')
+    yield check_rpc_with_many_connections("inproc://")
 
 
 @gen.coroutine
 def check_large_packets(listen_arg):
     """ tornado has a 100MB cap by default """
-    server = Server({'echo': echo})
+    server = Server({"echo": echo})
     server.listen(listen_arg)
 
-    data = b'0' * int(200e6)  # slightly more than 100MB
+    data = b"0" * int(200e6)  # slightly more than 100MB
     conn = rpc(server.address)
     result = yield conn.echo(x=data)
     assert result == data
 
-    d = {'x': data}
+    d = {"x": data}
     result = yield conn.echo(x=d)
     assert result == d
 
@@ -390,12 +412,12 @@ def check_large_packets(listen_arg):
 @slow
 @gen_test()
 def test_large_packets_tcp():
-    yield check_large_packets('tcp://')
+    yield check_large_packets("tcp://")
 
 
 @gen_test()
 def test_large_packets_inproc():
-    yield check_large_packets('inproc://')
+    yield check_large_packets("inproc://")
 
 
 @gen.coroutine
@@ -406,20 +428,20 @@ def check_identity(listen_arg):
     with rpc(server.address) as remote:
         a = yield remote.identity()
         b = yield remote.identity()
-        assert a['type'] == 'Server'
-        assert a['id'] == b['id']
+        assert a["type"] == "Server"
+        assert a["id"] == b["id"]
 
     server.stop()
 
 
 @gen_test()
 def test_identity_tcp():
-    yield check_identity('tcp://')
+    yield check_identity("tcp://")
 
 
 @gen_test()
 def test_identity_inproc():
-    yield check_identity('inproc://')
+    yield check_identity("inproc://")
 
 
 def test_ports(loop):
@@ -450,10 +472,10 @@ def stream_div(stream=None, x=None, y=None):
 
 @gen_test()
 def test_errors():
-    server = Server({'div': stream_div})
+    server = Server({"div": stream_div})
     server.listen(0)
 
-    with rpc(('127.0.0.1', server.port)) as r:
+    with rpc(("127.0.0.1", server.port)) as r:
         with pytest.raises(ZeroDivisionError):
             yield r.div(x=1, y=0)
 
@@ -461,67 +483,64 @@ def test_errors():
 @gen_test()
 def test_connect_raises():
     with pytest.raises((gen.TimeoutError, IOError)):
-        yield connect('127.0.0.1:58259', timeout=0.01)
+        yield connect("127.0.0.1:58259", timeout=0.01)
 
 
 @gen_test()
 def test_send_recv_args():
-    server = Server({'echo': echo})
+    server = Server({"echo": echo})
     server.listen(0)
 
     comm = yield connect(server.address)
-    result = yield send_recv(comm, op='echo', x=b'1')
-    assert result == b'1'
+    result = yield send_recv(comm, op="echo", x=b"1")
+    assert result == b"1"
     assert not comm.closed()
-    result = yield send_recv(comm, op='echo', x=b'2', reply=False)
+    result = yield send_recv(comm, op="echo", x=b"2", reply=False)
     assert result is None
     assert not comm.closed()
-    result = yield send_recv(comm, op='echo', x=b'3', close=True)
-    assert result == b'3'
+    result = yield send_recv(comm, op="echo", x=b"3", close=True)
+    assert result == b"3"
     assert comm.closed()
 
     server.stop()
 
 
 def test_coerce_to_address():
-    for arg in ['127.0.0.1:8786',
-                ('127.0.0.1', 8786),
-                ('127.0.0.1', '8786')]:
-        assert coerce_to_address(arg) == 'tcp://127.0.0.1:8786'
+    for arg in ["127.0.0.1:8786", ("127.0.0.1", 8786), ("127.0.0.1", "8786")]:
+        assert coerce_to_address(arg) == "tcp://127.0.0.1:8786"
 
 
 @gen_test()
 def test_connection_pool():
-
     @gen.coroutine
     def ping(comm, delay=0.1):
         yield gen.sleep(delay)
-        raise gen.Return('pong')
+        raise gen.Return("pong")
 
-    servers = [Server({'ping': ping}) for i in range(10)]
+    servers = [Server({"ping": ping}) for i in range(10)]
     for server in servers:
         server.listen(0)
 
     rpc = ConnectionPool(limit=5)
 
     # Reuse connections
-    yield [rpc(ip='127.0.0.1', port=s.port).ping() for s in servers[:5]]
+    yield [rpc(ip="127.0.0.1", port=s.port).ping() for s in servers[:5]]
     yield [rpc(s.address).ping() for s in servers[:5]]
-    yield [rpc('127.0.0.1:%d' % s.port).ping() for s in servers[:5]]
-    yield [rpc(ip='127.0.0.1', port=s.port).ping() for s in servers[:5]]
+    yield [rpc("127.0.0.1:%d" % s.port).ping() for s in servers[:5]]
+    yield [rpc(ip="127.0.0.1", port=s.port).ping() for s in servers[:5]]
     assert sum(map(len, rpc.available.values())) == 5
     assert sum(map(len, rpc.occupied.values())) == 0
     assert rpc.active == 0
     assert rpc.open == 5
 
     # Clear out connections to make room for more
-    yield [rpc(ip='127.0.0.1', port=s.port).ping() for s in servers[5:]]
+    yield [rpc(ip="127.0.0.1", port=s.port).ping() for s in servers[5:]]
     assert rpc.active == 0
     assert rpc.open == 5
 
     s = servers[0]
-    yield [rpc(ip='127.0.0.1', port=s.port).ping(delay=0.1) for i in range(3)]
-    assert len(rpc.available['tcp://127.0.0.1:%d' % s.port]) == 3
+    yield [rpc(ip="127.0.0.1", port=s.port).ping(delay=0.1) for i in range(3)]
+    assert len(rpc.available["tcp://127.0.0.1:%d" % s.port]) == 3
 
     # Explicitly clear out connections
     rpc.collect()
@@ -539,17 +558,17 @@ def test_connection_pool_tls():
     Make sure connection args are supported.
     """
     sec = tls_security()
-    connection_args = sec.get_connection_args('client')
-    listen_args = sec.get_listen_args('scheduler')
+    connection_args = sec.get_connection_args("client")
+    listen_args = sec.get_listen_args("scheduler")
 
     @gen.coroutine
     def ping(comm, delay=0.01):
         yield gen.sleep(delay)
-        raise gen.Return('pong')
+        raise gen.Return("pong")
 
-    servers = [Server({'ping': ping}) for i in range(10)]
+    servers = [Server({"ping": ping}) for i in range(10)]
     for server in servers:
-        server.listen('tls://', listen_args=listen_args)
+        server.listen("tls://", listen_args=listen_args)
 
     rpc = ConnectionPool(limit=5, connection_args=connection_args)
 
@@ -563,13 +582,12 @@ def ping(comm, delay=0.01):
 
 @gen_test()
 def test_connection_pool_remove():
-
     @gen.coroutine
     def ping(comm, delay=0.01):
         yield gen.sleep(delay)
-        raise gen.Return('pong')
+        raise gen.Return("pong")
 
-    servers = [Server({'ping': ping}) for i in range(5)]
+    servers = [Server({"ping": ping}) for i in range(5)]
     for server in servers:
         server.listen(0)
 
@@ -601,8 +619,8 @@ def ping(comm, delay=0.01):
 
 @gen_test()
 def test_counters():
-    server = Server({'div': stream_div})
-    server.listen('tcp://')
+    server = Server({"div": stream_div})
+    server.listen("tcp://")
 
     with rpc(server.address) as r:
         for i in range(2):
@@ -611,49 +629,50 @@ def test_counters():
             yield r.div(x=1, y=0)
 
         c = server.counters
-        assert c['op'].components[0] == {'identity': 2, 'div': 1}
+        assert c["op"].components[0] == {"identity": 2, "div": 1}
 
 
 @gen_cluster()
 def test_ticks(s, a, b):
-    pytest.importorskip('crick')
+    pytest.importorskip("crick")
     yield gen.sleep(0.1)
-    c = s.digests['tick-duration']
+    c = s.digests["tick-duration"]
     assert c.size()
     assert 0.01 < c.components[0].quantile(0.5) < 0.5
 
 
 @gen_cluster()
 def test_tick_logging(s, a, b):
-    pytest.importorskip('crick')
+    pytest.importorskip("crick")
     from distributed import core
+
     old = core.tick_maximum_delay
     core.tick_maximum_delay = 0.001
     try:
-        with captured_logger('distributed.core') as sio:
+        with captured_logger("distributed.core") as sio:
             yield gen.sleep(0.1)
 
         text = sio.getvalue()
         assert "unresponsive" in text
-        assert 'Scheduler' in text or 'Worker' in text
+        assert "Scheduler" in text or "Worker" in text
     finally:
         core.tick_maximum_delay = old
 
 
-@pytest.mark.parametrize('compression', list(compressions))
-@pytest.mark.parametrize('serialize', [echo_serialize, echo_no_serialize])
+@pytest.mark.parametrize("compression", list(compressions))
+@pytest.mark.parametrize("serialize", [echo_serialize, echo_no_serialize])
 def test_compression(compression, serialize, loop):
     with dask.config.set(compression=compression):
 
         @gen.coroutine
         def f():
-            server = Server({'echo': serialize})
-            server.listen('tcp://')
+            server = Server({"echo": serialize})
+            server.listen("tcp://")
 
             with rpc(server.address) as r:
-                data = b'1' * 1000000
+                data = b"1" * 1000000
                 result = yield r.echo(x=to_serialize(data))
-                assert result == {'result': data}
+                assert result == {"result": data}
 
             server.stop()
 
@@ -663,16 +682,16 @@ def f():
 def test_rpc_serialization(loop):
     @gen.coroutine
     def f():
-        server = Server({'echo': echo_serialize})
-        server.listen('tcp://')
+        server = Server({"echo": echo_serialize})
+        server.listen("tcp://")
 
-        with rpc(server.address, serializers=['msgpack']) as r:
+        with rpc(server.address, serializers=["msgpack"]) as r:
             with pytest.raises(TypeError):
                 yield r.echo(x=to_serialize(inc))
 
-        with rpc(server.address, serializers=['msgpack', 'pickle']) as r:
+        with rpc(server.address, serializers=["msgpack", "pickle"]) as r:
             result = yield r.echo(x=to_serialize(inc))
-            assert result == {'result': inc}
+            assert result == {"result": inc}
 
         server.stop()
 
@@ -686,12 +705,12 @@ def test_thread_id(s, a, b):
 
 @gen_test()
 def test_deserialize_error():
-    server = Server({'throws': throws})
+    server = Server({"throws": throws})
     server.listen(0)
 
     comm = yield connect(server.address, deserialize=False)
     with pytest.raises(Exception) as info:
-        yield send_recv(comm, op='throws')
+        yield send_recv(comm, op="throws")
 
     assert type(info.value) == Exception
     for c in str(info.value):
diff --git a/distributed/tests/test_counter.py b/distributed/tests/test_counter.py
index 43b5e4d022c..956a682920c 100644
--- a/distributed/tests/test_counter.py
+++ b/distributed/tests/test_counter.py
@@ -11,11 +11,17 @@
     Digest = None
 
 
-@pytest.mark.parametrize('CD,size', [
-    (Counter, lambda d: sum(d.values())),
-    pytest.param(Digest, lambda x: x.size(),
-                 marks=pytest.mark.skipif(not Digest, reason="no crick library"))
-])
+@pytest.mark.parametrize(
+    "CD,size",
+    [
+        (Counter, lambda d: sum(d.values())),
+        pytest.param(
+            Digest,
+            lambda x: x.size(),
+            marks=pytest.mark.skipif(not Digest, reason="no crick library"),
+        ),
+    ],
+)
 def test_digest(loop, CD, size):
     c = CD(loop=loop)
     c.add(1)
diff --git a/distributed/tests/test_diskutils.py b/distributed/tests/test_diskutils.py
index d7079ca2039..8bf4000178e 100644
--- a/distributed/tests/test_diskutils.py
+++ b/distributed/tests/test_diskutils.py
@@ -21,9 +21,11 @@
 
 def assert_directory_contents(dir_path, expected):
     expected = [os.path.join(dir_path, p) for p in expected]
-    actual = [os.path.join(dir_path, p)
-              for p in os.listdir(dir_path)
-              if p not in ('global.lock', 'purge.lock')]
+    actual = [
+        os.path.join(dir_path, p)
+        for p in os.listdir(dir_path)
+        if p not in ("global.lock", "purge.lock")
+    ]
     assert sorted(actual) == sorted(expected)
 
 
@@ -34,29 +36,29 @@ def test_workdir_simple(tmpdir):
 
     ws = WorkSpace(base_dir)
     assert_contents([])
-    a = ws.new_work_dir(name='aa')
-    assert_contents(['aa', 'aa.dirlock'])
-    b = ws.new_work_dir(name='bb')
-    assert_contents(['aa', 'aa.dirlock', 'bb', 'bb.dirlock'])
+    a = ws.new_work_dir(name="aa")
+    assert_contents(["aa", "aa.dirlock"])
+    b = ws.new_work_dir(name="bb")
+    assert_contents(["aa", "aa.dirlock", "bb", "bb.dirlock"])
     ws._purge_leftovers()
-    assert_contents(['aa', 'aa.dirlock', 'bb', 'bb.dirlock'])
+    assert_contents(["aa", "aa.dirlock", "bb", "bb.dirlock"])
 
     a.release()
-    assert_contents(['bb', 'bb.dirlock'])
+    assert_contents(["bb", "bb.dirlock"])
     del b
     gc.collect()
     assert_contents([])
 
     # Generated temporary name with a prefix
-    a = ws.new_work_dir(prefix='foo-')
-    b = ws.new_work_dir(prefix='bar-')
-    c = ws.new_work_dir(prefix='bar-')
-    assert_contents({a.dir_path, a._lock_path,
-                     b.dir_path, b._lock_path,
-                     c.dir_path, c._lock_path})
-    assert os.path.basename(a.dir_path).startswith('foo-')
-    assert os.path.basename(b.dir_path).startswith('bar-')
-    assert os.path.basename(c.dir_path).startswith('bar-')
+    a = ws.new_work_dir(prefix="foo-")
+    b = ws.new_work_dir(prefix="bar-")
+    c = ws.new_work_dir(prefix="bar-")
+    assert_contents(
+        {a.dir_path, a._lock_path, b.dir_path, b._lock_path, c.dir_path, c._lock_path}
+    )
+    assert os.path.basename(a.dir_path).startswith("foo-")
+    assert os.path.basename(b.dir_path).startswith("bar-")
+    assert os.path.basename(c.dir_path).startswith("bar-")
     assert b.dir_path != c.dir_path
 
 
@@ -68,19 +70,19 @@ def test_two_workspaces_in_same_directory(tmpdir):
 
     ws = WorkSpace(base_dir)
     assert_contents([])
-    a = ws.new_work_dir(name='aa')
-    assert_contents(['aa', 'aa.dirlock'])
+    a = ws.new_work_dir(name="aa")
+    assert_contents(["aa", "aa.dirlock"])
 
     ws2 = WorkSpace(base_dir)
     ws2._purge_leftovers()
-    assert_contents(['aa', 'aa.dirlock'])
-    b = ws.new_work_dir(name='bb')
-    assert_contents(['aa', 'aa.dirlock', 'bb', 'bb.dirlock'])
+    assert_contents(["aa", "aa.dirlock"])
+    b = ws.new_work_dir(name="bb")
+    assert_contents(["aa", "aa.dirlock", "bb", "bb.dirlock"])
 
     del ws
     del b
     gc.collect()
-    assert_contents(['aa', 'aa.dirlock'])
+    assert_contents(["aa", "aa.dirlock"])
     del a
     gc.collect()
     assert_contents([])
@@ -108,26 +110,31 @@ def test_workspace_process_crash(tmpdir):
         sys.stdout.flush()
 
         time.sleep(100)
-        """ % dict(base_dir=base_dir)
-
-    p = subprocess.Popen([sys.executable, '-c', code],
-                         stdin=subprocess.PIPE, stdout=subprocess.PIPE,
-                         universal_newlines=True)
+        """ % dict(
+        base_dir=base_dir
+    )
+
+    p = subprocess.Popen(
+        [sys.executable, "-c", code],
+        stdin=subprocess.PIPE,
+        stdout=subprocess.PIPE,
+        universal_newlines=True,
+    )
     line = p.stdout.readline()
     assert p.poll() is None
     a_path, b_path = eval(line)
-    assert_contents([a_path, a_path + '.dirlock', b_path, b_path + '.dirlock'])
+    assert_contents([a_path, a_path + ".dirlock", b_path, b_path + ".dirlock"])
 
     # The child process holds a lock so the work dirs shouldn't be removed
     ws._purge_leftovers()
-    assert_contents([a_path, a_path + '.dirlock', b_path, b_path + '.dirlock'])
+    assert_contents([a_path, a_path + ".dirlock", b_path, b_path + ".dirlock"])
 
     # Kill the process so it's unable to clear the work dirs itself
     p.kill()
     assert p.wait()  # process returned with non-zero code
-    assert_contents([a_path, a_path + '.dirlock', b_path, b_path + '.dirlock'])
+    assert_contents([a_path, a_path + ".dirlock", b_path, b_path + ".dirlock"])
 
-    with captured_logger('distributed.diskutils', 'INFO', propagate=False) as sio:
+    with captured_logger("distributed.diskutils", "INFO", propagate=False) as sio:
         ws._purge_leftovers()
     assert_contents([])
     # One log line per purged directory
@@ -141,9 +148,9 @@ def test_workspace_rmtree_failure(tmpdir):
     base_dir = str(tmpdir)
 
     ws = WorkSpace(base_dir)
-    a = ws.new_work_dir(name='aa')
+    a = ws.new_work_dir(name="aa")
     shutil.rmtree(a.dir_path)
-    with captured_logger('distributed.diskutils', 'ERROR', propagate=False) as sio:
+    with captured_logger("distributed.diskutils", "ERROR", propagate=False) as sio:
         a.release()
     lines = sio.getvalue().splitlines()
     # shutil.rmtree() may call its onerror callback several times
@@ -155,21 +162,21 @@ def test_workspace_rmtree_failure(tmpdir):
 def test_locking_disabled(tmpdir):
     base_dir = str(tmpdir)
 
-    with dask.config.set({'distributed.worker.use-file-locking': False}):
-        with mock.patch('distributed.diskutils.locket.lock_file') as lock_file:
+    with dask.config.set({"distributed.worker.use-file-locking": False}):
+        with mock.patch("distributed.diskutils.locket.lock_file") as lock_file:
             assert_contents = functools.partial(assert_directory_contents, base_dir)
 
             ws = WorkSpace(base_dir)
             assert_contents([])
-            a = ws.new_work_dir(name='aa')
-            assert_contents(['aa'])
-            b = ws.new_work_dir(name='bb')
-            assert_contents(['aa', 'bb'])
+            a = ws.new_work_dir(name="aa")
+            assert_contents(["aa"])
+            b = ws.new_work_dir(name="bb")
+            assert_contents(["aa", "bb"])
             ws._purge_leftovers()
-            assert_contents(['aa', 'bb'])
+            assert_contents(["aa", "bb"])
 
             a.release()
-            assert_contents(['bb'])
+            assert_contents(["bb"])
             del b
             gc.collect()
             assert_contents([])
@@ -180,7 +187,7 @@ def test_locking_disabled(tmpdir):
 def _workspace_concurrency(base_dir, purged_q, err_q, stop_evt):
     ws = WorkSpace(base_dir)
     n_purged = 0
-    with captured_logger('distributed.diskutils', 'ERROR') as sio:
+    with captured_logger("distributed.diskutils", "ERROR") as sio:
         while not stop_evt.is_set():
             # Add a bunch of locks, and simulate forgetting them
             try:
@@ -193,7 +200,7 @@ def _workspace_concurrency(base_dir, purged_q, err_q, stop_evt):
     lines = sio.getvalue().splitlines()
     if lines:
         try:
-            raise AssertionError("got %d logs, see stderr" % (len(lines,)))
+            raise AssertionError("got %d logs, see stderr" % (len(lines)))
         except Exception as e:
             err_q.put(e)
 
@@ -215,10 +222,13 @@ def _test_workspace_concurrency(tmpdir, timeout, max_procs):
     ws._purge_leftovers = lambda: None
 
     # Run a bunch of child processes that will try to purge concurrently
-    NPROCS = 2 if sys.platform == 'win32' else max_procs
-    processes = [mp_context.Process(target=_workspace_concurrency,
-                                    args=(base_dir, purged_q, err_q, stop_evt))
-                 for i in range(NPROCS)]
+    NPROCS = 2 if sys.platform == "win32" else max_procs
+    processes = [
+        mp_context.Process(
+            target=_workspace_concurrency, args=(base_dir, purged_q, err_q, stop_evt)
+        )
+        for i in range(NPROCS)
+    ]
     for p in processes:
         p.start()
 
@@ -230,7 +240,7 @@ def _test_workspace_concurrency(tmpdir, timeout, max_procs):
             # Add a bunch of locks, and simulate forgetting them.
             # The concurrent processes should try to purge them.
             for i in range(50):
-                d = ws.new_work_dir(prefix='workspace-concurrency-')
+                d = ws.new_work_dir(prefix="workspace-concurrency-")
                 d._finalizer.detach()
                 n_created += 1
             sleep(1e-2)
@@ -259,7 +269,7 @@ def _test_workspace_concurrency(tmpdir, timeout, max_procs):
 
 def test_workspace_concurrency(tmpdir):
     if WINDOWS:
-        raise pytest.xfail.Exception('TODO: unknown failure on windows')
+        raise pytest.xfail.Exception("TODO: unknown failure on windows")
     _test_workspace_concurrency(tmpdir, 2.0, 6)
 
 
diff --git a/distributed/tests/test_failed_workers.py b/distributed/tests/test_failed_workers.py
index 72811c6fc52..5bb1c61fb5b 100644
--- a/distributed/tests/test_failed_workers.py
+++ b/distributed/tests/test_failed_workers.py
@@ -15,17 +15,25 @@
 from distributed.client import wait
 from distributed.metrics import time
 from distributed.utils import sync, ignoring
-from distributed.utils_test import (gen_cluster, cluster, inc, slow, div,
-                                    slowinc, slowadd, captured_logger)
-from distributed.utils_test import loop # noqa: F401
+from distributed.utils_test import (
+    gen_cluster,
+    cluster,
+    inc,
+    slow,
+    div,
+    slowinc,
+    slowadd,
+    captured_logger,
+)
+from distributed.utils_test import loop  # noqa: F401
 
 
 def test_submit_after_failed_worker_sync(loop):
     with cluster(active_rpc_timeout=10) as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
+        with Client(s["address"], loop=loop) as c:
             L = c.map(inc, range(10))
             wait(L)
-            a['proc']().terminate()
+            a["proc"]().terminate()
             total = c.submit(sum, L)
             assert total.result() == sum(map(inc, range(10)))
 
@@ -61,16 +69,20 @@ def test_submit_after_failed_worker(c, s, a, b):
 
 def test_gather_after_failed_worker(loop):
     with cluster(active_rpc_timeout=10) as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
+        with Client(s["address"], loop=loop) as c:
             L = c.map(inc, range(10))
             wait(L)
-            a['proc']().terminate()
+            a["proc"]().terminate()
             result = c.gather(L)
             assert result == list(map(inc, range(10)))
 
 
-@gen_cluster(client=True, Worker=Nanny, ncores=[('127.0.0.1', 1)] * 4,
-             config={'distributed.comm.timeouts.connect': '1s'})
+@gen_cluster(
+    client=True,
+    Worker=Nanny,
+    ncores=[("127.0.0.1", 1)] * 4,
+    config={"distributed.comm.timeouts.connect": "1s"},
+)
 def test_gather_then_submit_after_failed_workers(c, s, w, x, y, z):
     L = c.map(inc, range(20))
     yield wait(L)
@@ -168,7 +180,7 @@ def test_restart_cleared(c, s, a, b):
 
 def test_restart_sync_no_center(loop):
     with cluster(nanny=True) as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
+        with Client(s["address"], loop=loop) as c:
             x = c.submit(inc, 1)
             c.restart()
             assert x.cancelled()
@@ -179,7 +191,7 @@ def test_restart_sync_no_center(loop):
 
 def test_restart_sync(loop):
     with cluster(nanny=True) as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
+        with Client(s["address"], loop=loop) as c:
             x = c.submit(div, 1, 2)
             x.result()
 
@@ -205,7 +217,7 @@ def test_restart_fast(c, s, a, b):
     assert time() - start < 10
     assert len(s.ncores) == 2
 
-    assert all(x.status == 'cancelled' for x in L)
+    assert all(x.status == "cancelled" for x in L)
 
     x = c.submit(inc, 1)
     result = yield x
@@ -214,7 +226,7 @@ def test_restart_fast(c, s, a, b):
 
 def test_worker_doesnt_await_task_completion(loop):
     with cluster(nanny=True, nworkers=1) as (s, [w]):
-        with Client(s['address'], loop=loop) as c:
+        with Client(s["address"], loop=loop) as c:
             future = c.submit(sleep, 100)
             sleep(0.1)
             start = time()
@@ -225,7 +237,7 @@ def test_worker_doesnt_await_task_completion(loop):
 
 def test_restart_fast_sync(loop):
     with cluster(nanny=True) as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
+        with Client(s["address"], loop=loop) as c:
             L = c.map(sleep, range(10))
 
             start = time()
@@ -233,7 +245,7 @@ def test_restart_fast_sync(loop):
             assert time() - start < 10
             assert len(c.ncores()) == 2
 
-            assert all(x.status == 'cancelled' for x in L)
+            assert all(x.status == "cancelled" for x in L)
 
             x = c.submit(inc, 1)
             assert x.result() == 2
@@ -247,7 +259,7 @@ def test_fast_kill(c, s, a, b):
     yield c._restart()
     assert time() - start < 10
 
-    assert all(x.status == 'cancelled' for x in L)
+    assert all(x.status == "cancelled" for x in L)
 
     x = c.submit(inc, 1)
     result = yield x
@@ -278,6 +290,7 @@ def test_multiple_clients_restart(s, a, b):
 @gen_cluster(Worker=Nanny, timeout=60)
 def test_restart_scheduler(s, a, b):
     import gc
+
     gc.collect()
     addrs = (a.worker_address, b.worker_address)
     yield s.restart()
@@ -294,6 +307,7 @@ def test_forgotten_futures_dont_clean_up_new_futures(c, s, a, b):
     y = c.submit(inc, 1)
     del x
     import gc
+
     gc.collect()
     yield gen.sleep(0.1)
     yield y
@@ -313,12 +327,14 @@ def test_broken_worker_during_computation(c, s, a, b):
     N = 256
     expected_result = N * (N + 1) // 2
     i = 0
-    L = c.map(inc, range(N),
-              key=['inc-%d-%d' % (i, j) for j in range(N)])
+    L = c.map(inc, range(N), key=["inc-%d-%d" % (i, j) for j in range(N)])
     while len(L) > 1:
         i += 1
-        L = c.map(slowadd, *zip(*partition_all(2, L)),
-                  key=['add-%d-%d' % (i, j) for j in range(len(L) // 2)])
+        L = c.map(
+            slowadd,
+            *zip(*partition_all(2, L)),
+            key=["add-%d-%d" % (i, j) for j in range(len(L) // 2)]
+        )
 
     yield gen.sleep(random.random() / 20)
     with ignoring(CommClosedError):  # comm will be closed abrupty
@@ -328,7 +344,9 @@ def test_broken_worker_during_computation(c, s, a, b):
     while len(s.workers) < 3:
         yield gen.sleep(0.01)
 
-    with ignoring(CommClosedError, EnvironmentError):  # perhaps new worker can't be contacted yet
+    with ignoring(
+        CommClosedError, EnvironmentError
+    ):  # perhaps new worker can't be contacted yet
         yield c._run(os._exit, 1, workers=[n.worker_address])
 
     [result] = yield c.gather(L)
@@ -365,8 +383,7 @@ def test_worker_who_has_clears_after_failed_connection(c, s, a, b):
         yield gen.sleep(0.01)
         assert time() < start + 5
 
-    futures = c.map(slowinc, range(20), delay=0.01,
-                    key=['f%d' % i for i in range(20)])
+    futures = c.map(slowinc, range(20), delay=0.01, key=["f%d" % i for i in range(20)])
     yield wait(futures)
 
     result = yield c.submit(sum, futures, workers=a.address)
@@ -390,20 +407,20 @@ def test_worker_who_has_clears_after_failed_connection(c, s, a, b):
 
 
 @slow
-@gen_cluster(client=True, timeout=60, Worker=Nanny, ncores=[('127.0.0.1', 1)])
+@gen_cluster(client=True, timeout=60, Worker=Nanny, ncores=[("127.0.0.1", 1)])
 def test_restart_timeout_on_long_running_task(c, s, a):
-    with captured_logger('distributed.scheduler') as sio:
+    with captured_logger("distributed.scheduler") as sio:
         future = c.submit(sleep, 3600)
         yield gen.sleep(0.1)
         yield c.restart(timeout=20)
 
     text = sio.getvalue()
-    assert 'timeout' not in text.lower()
+    assert "timeout" not in text.lower()
 
 
-@gen_cluster(client=True, scheduler_kwargs={'worker_ttl': '100ms'})
+@gen_cluster(client=True, scheduler_kwargs={"worker_ttl": "100ms"})
 def test_worker_time_to_live(c, s, a, b):
-    a.periodic_callbacks['heartbeat'].stop()
+    a.periodic_callbacks["heartbeat"].stop()
     yield gen.sleep(0.010)
     assert set(s.workers) == {a.address, b.address}
 
@@ -415,8 +432,8 @@ def test_worker_time_to_live(c, s, a, b):
     set(s.workers) == {b.address}
 
     start = time()
-    while b.status == 'running':
+    while b.status == "running":
         yield gen.sleep(0.050)
         assert time() < start + 1
 
-    assert b.status in ('closed', 'closing')
+    assert b.status in ("closed", "closing")
diff --git a/distributed/tests/test_ipython.py b/distributed/tests/test_ipython.py
index 4730e0c9515..8bb64bb4e0b 100644
--- a/distributed/tests/test_ipython.py
+++ b/distributed/tests/test_ipython.py
@@ -30,18 +30,18 @@ def test_start_ipython_workers(loop, zmq_ctx):
     from jupyter_client import BlockingKernelClient
 
     with cluster(1) as (s, [a]):
-        with Client(s['address'], loop=loop) as e:
+        with Client(s["address"], loop=loop) as e:
             info_dict = e.start_ipython_workers()
             info = first(info_dict.values())
-            key = info.pop('key')
+            key = info.pop("key")
             kc = BlockingKernelClient(**info)
             kc.session.key = key
             kc.start_channels()
             kc.wait_for_ready(timeout=10)
             msg_id = kc.execute("worker")
             reply = kc.get_shell_msg(timeout=10)
-            assert reply['parent_header']['msg_id'] == msg_id
-            assert reply['content']['status'] == 'ok'
+            assert reply["parent_header"]["msg_id"] == msg_id
+            assert reply["content"]["status"] == "ok"
             kc.stop_channels()
 
 
@@ -51,9 +51,9 @@ def test_start_ipython_scheduler(loop, zmq_ctx):
     from jupyter_client import BlockingKernelClient
 
     with cluster(1) as (s, [a]):
-        with Client(s['address'], loop=loop) as e:
+        with Client(s["address"], loop=loop) as e:
             info = e.start_ipython_scheduler()
-            key = info.pop('key')
+            key = info.pop("key")
             kc = BlockingKernelClient(**info)
             kc.session.key = key
             kc.start_channels()
@@ -66,15 +66,17 @@ def test_start_ipython_scheduler(loop, zmq_ctx):
 @need_functional_ipython
 def test_start_ipython_scheduler_magic(loop, zmq_ctx):
     with cluster(1) as (s, [a]):
-        with Client(s['address'], loop=loop) as e, mock_ipython() as ip:
+        with Client(s["address"], loop=loop) as e, mock_ipython() as ip:
             info = e.start_ipython_scheduler()
 
         expected = [
-            {'magic_kind': 'line', 'magic_name': 'scheduler'},
-            {'magic_kind': 'cell', 'magic_name': 'scheduler'},
+            {"magic_kind": "line", "magic_name": "scheduler"},
+            {"magic_kind": "cell", "magic_name": "scheduler"},
         ]
 
-        call_kwargs_list = [kwargs for (args, kwargs) in ip.register_magic_function.call_args_list]
+        call_kwargs_list = [
+            kwargs for (args, kwargs) in ip.register_magic_function.call_args_list
+        ]
         assert call_kwargs_list == expected
         magic = ip.register_magic_function.call_args_list[0][0][0]
         magic(line="", cell="scheduler")
@@ -85,20 +87,22 @@ def test_start_ipython_scheduler_magic(loop, zmq_ctx):
 def test_start_ipython_workers_magic(loop, zmq_ctx):
     with cluster(2) as (s, [a, b]):
 
-        with Client(s['address'], loop=loop) as e, mock_ipython() as ip:
+        with Client(s["address"], loop=loop) as e, mock_ipython() as ip:
             workers = list(e.ncores())[:2]
-            names = ['magic%i' % i for i in range(len(workers))]
+            names = ["magic%i" % i for i in range(len(workers))]
             info_dict = e.start_ipython_workers(workers, magic_names=names)
 
         expected = [
-            {'magic_kind': 'line', 'magic_name': 'remote'},
-            {'magic_kind': 'cell', 'magic_name': 'remote'},
-            {'magic_kind': 'line', 'magic_name': 'magic0'},
-            {'magic_kind': 'cell', 'magic_name': 'magic0'},
-            {'magic_kind': 'line', 'magic_name': 'magic1'},
-            {'magic_kind': 'cell', 'magic_name': 'magic1'},
+            {"magic_kind": "line", "magic_name": "remote"},
+            {"magic_kind": "cell", "magic_name": "remote"},
+            {"magic_kind": "line", "magic_name": "magic0"},
+            {"magic_kind": "cell", "magic_name": "magic0"},
+            {"magic_kind": "line", "magic_name": "magic1"},
+            {"magic_kind": "cell", "magic_name": "magic1"},
+        ]
+        call_kwargs_list = [
+            kwargs for (args, kwargs) in ip.register_magic_function.call_args_list
         ]
-        call_kwargs_list = [kwargs for (args, kwargs) in ip.register_magic_function.call_args_list]
         assert call_kwargs_list == expected
         assert ip.register_magic_function.call_count == 6
         magics = [args[0][0] for args in ip.register_magic_function.call_args_list[2:]]
@@ -111,19 +115,21 @@ def test_start_ipython_workers_magic(loop, zmq_ctx):
 def test_start_ipython_workers_magic_asterix(loop, zmq_ctx):
     with cluster(2) as (s, [a, b]):
 
-        with Client(s['address'], loop=loop) as e, mock_ipython() as ip:
+        with Client(s["address"], loop=loop) as e, mock_ipython() as ip:
             workers = list(e.ncores())[:2]
-            info_dict = e.start_ipython_workers(workers, magic_names='magic_*')
+            info_dict = e.start_ipython_workers(workers, magic_names="magic_*")
 
         expected = [
-            {'magic_kind': 'line', 'magic_name': 'remote'},
-            {'magic_kind': 'cell', 'magic_name': 'remote'},
-            {'magic_kind': 'line', 'magic_name': 'magic_0'},
-            {'magic_kind': 'cell', 'magic_name': 'magic_0'},
-            {'magic_kind': 'line', 'magic_name': 'magic_1'},
-            {'magic_kind': 'cell', 'magic_name': 'magic_1'},
+            {"magic_kind": "line", "magic_name": "remote"},
+            {"magic_kind": "cell", "magic_name": "remote"},
+            {"magic_kind": "line", "magic_name": "magic_0"},
+            {"magic_kind": "cell", "magic_name": "magic_0"},
+            {"magic_kind": "line", "magic_name": "magic_1"},
+            {"magic_kind": "cell", "magic_name": "magic_1"},
+        ]
+        call_kwargs_list = [
+            kwargs for (args, kwargs) in ip.register_magic_function.call_args_list
         ]
-        call_kwargs_list = [kwargs for (args, kwargs) in ip.register_magic_function.call_args_list]
         assert call_kwargs_list == expected
         assert ip.register_magic_function.call_count == 6
         magics = [args[0][0] for args in ip.register_magic_function.call_args_list[2:]]
@@ -135,16 +141,17 @@ def test_start_ipython_workers_magic_asterix(loop, zmq_ctx):
 @need_functional_ipython
 def test_start_ipython_remote(loop, zmq_ctx):
     from distributed._ipython_utils import remote_magic
+
     with cluster(1) as (s, [a]):
-        with Client(s['address'], loop=loop) as e, mock_ipython() as ip:
+        with Client(s["address"], loop=loop) as e, mock_ipython() as ip:
             worker = first(e.ncores())
-            ip.user_ns['info'] = e.start_ipython_workers(worker)[worker]
-            remote_magic('info 1')  # line magic
-            remote_magic('info', 'worker')  # cell magic
+            ip.user_ns["info"] = e.start_ipython_workers(worker)[worker]
+            remote_magic("info 1")  # line magic
+            remote_magic("info", "worker")  # cell magic
 
         expected = [
-            ((remote_magic,), {'magic_kind': 'line', 'magic_name': 'remote'}),
-            ((remote_magic,), {'magic_kind': 'cell', 'magic_name': 'remote'}),
+            ((remote_magic,), {"magic_kind": "line", "magic_name": "remote"}),
+            ((remote_magic,), {"magic_kind": "cell", "magic_name": "remote"}),
         ]
         assert ip.register_magic_function.call_args_list == expected
         assert ip.register_magic_function.call_count == 2
@@ -155,12 +162,14 @@ def test_start_ipython_remote(loop, zmq_ctx):
 def test_start_ipython_qtconsole(loop):
     Popen = mock.Mock()
     with cluster() as (s, [a, b]):
-        with mock.patch('distributed._ipython_utils.Popen', Popen), Client(s['address'], loop=loop) as e:
+        with mock.patch("distributed._ipython_utils.Popen", Popen), Client(
+            s["address"], loop=loop
+        ) as e:
             worker = first(e.ncores())
             e.start_ipython_workers(worker, qtconsole=True)
-            e.start_ipython_workers(worker, qtconsole=True, qtconsole_args=['--debug'])
+            e.start_ipython_workers(worker, qtconsole=True, qtconsole_args=["--debug"])
     assert Popen.call_count == 2
     (cmd,), kwargs = Popen.call_args_list[0]
-    assert cmd[:3] == ['jupyter', 'qtconsole', '--existing']
+    assert cmd[:3] == ["jupyter", "qtconsole", "--existing"]
     (cmd,), kwargs = Popen.call_args_list[1]
-    assert cmd[-1:] == ['--debug']
+    assert cmd[-1:] == ["--debug"]
diff --git a/distributed/tests/test_locks.py b/distributed/tests/test_locks.py
index b35d9d6268f..952d43ceb9b 100644
--- a/distributed/tests/test_locks.py
+++ b/distributed/tests/test_locks.py
@@ -11,34 +11,34 @@
 from distributed.utils_test import client, cluster_fixture, loop  # noqa F401
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 8)] * 2)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 8)] * 2)
 def test_lock(c, s, a, b):
-    c.set_metadata('locked', False)
+    c.set_metadata("locked", False)
 
     def f(x):
         client = get_client()
-        with Lock('x') as lock:
-            assert client.get_metadata('locked') is False
-            client.set_metadata('locked', True)
+        with Lock("x") as lock:
+            assert client.get_metadata("locked") is False
+            client.set_metadata("locked", True)
             sleep(0.05)
-            assert client.get_metadata('locked') is True
-            client.set_metadata('locked', False)
+            assert client.get_metadata("locked") is True
+            client.set_metadata("locked", False)
 
     futures = c.map(f, range(20))
     results = yield futures
-    assert not s.extensions['locks'].events
-    assert not s.extensions['locks'].ids
+    assert not s.extensions["locks"].events
+    assert not s.extensions["locks"].ids
 
 
 @gen_cluster(client=True)
 def test_timeout(c, s, a, b):
-    locks = s.extensions['locks']
-    lock = Lock('x')
+    locks = s.extensions["locks"]
+    lock = Lock("x")
     result = yield lock.acquire()
     assert result is True
-    assert locks.ids['x'] == lock.id
+    assert locks.ids["x"] == lock.id
 
-    lock2 = Lock('x')
+    lock2 = Lock("x")
     assert lock.id != lock2.id
 
     start = time()
@@ -46,15 +46,15 @@ def test_timeout(c, s, a, b):
     stop = time()
     assert stop - start < 0.3
     assert result is False
-    assert locks.ids['x'] == lock.id
-    assert not locks.events['x']
+    assert locks.ids["x"] == lock.id
+    assert not locks.events["x"]
 
     yield lock.release()
 
 
 @gen_cluster(client=True)
 def test_acquires_with_zero_timeout(c, s, a, b):
-    lock = Lock('x')
+    lock = Lock("x")
     yield lock.acquire(timeout=0)
     assert lock.locked()
     yield lock.release()
@@ -67,7 +67,7 @@ def test_acquires_with_zero_timeout(c, s, a, b):
 
 @gen_cluster(client=True)
 def test_acquires_blocking(c, s, a, b):
-    lock = Lock('x')
+    lock = Lock("x")
     yield lock.acquire(blocking=False)
     assert lock.locked()
     yield lock.release()
@@ -78,52 +78,52 @@ def test_acquires_blocking(c, s, a, b):
 
 
 def test_timeout_sync(client):
-    with Lock('x') as lock:
-        assert Lock('x').acquire(timeout=0.1) is False
+    with Lock("x") as lock:
+        assert Lock("x").acquire(timeout=0.1) is False
 
 
 @gen_cluster(client=True)
 def test_errors(c, s, a, b):
-    lock = Lock('x')
+    lock = Lock("x")
     with pytest.raises(ValueError):
         yield lock.release()
 
 
 def test_lock_sync(client):
     def f(x):
-        with Lock('x') as lock:
+        with Lock("x") as lock:
             client = get_client()
-            assert client.get_metadata('locked') is False
-            client.set_metadata('locked', True)
+            assert client.get_metadata("locked") is False
+            client.set_metadata("locked", True)
             sleep(0.05)
-            assert client.get_metadata('locked') is True
-            client.set_metadata('locked', False)
+            assert client.get_metadata("locked") is True
+            client.set_metadata("locked", False)
 
-    client.set_metadata('locked', False)
+    client.set_metadata("locked", False)
     futures = client.map(f, range(10))
     client.gather(futures)
 
 
 @gen_cluster(client=True)
 def test_lock_types(c, s, a, b):
-    for name in [1, ('a', 1), ['a', 1], b'123', '123']:
+    for name in [1, ("a", 1), ["a", 1], b"123", "123"]:
         lock = Lock(name)
         assert lock.name == name
 
         yield lock.acquire()
         yield lock.release()
 
-    assert not s.extensions['locks'].events
+    assert not s.extensions["locks"].events
 
 
 @gen_cluster(client=True)
 def test_serializable(c, s, a, b):
     def f(x, lock=None):
         with lock:
-            assert lock.name == 'x'
+            assert lock.name == "x"
             return x + 1
 
-    lock = Lock('x')
+    lock = Lock("x")
     futures = c.map(f, range(10), lock=lock)
     yield c.gather(futures)
 
diff --git a/distributed/tests/test_metrics.py b/distributed/tests/test_metrics.py
index 84b7c180993..d1eb4a1dad0 100644
--- a/distributed/tests/test_metrics.py
+++ b/distributed/tests/test_metrics.py
@@ -58,7 +58,7 @@ def test_thread_time():
         dt = metrics.thread_time() - start
         assert dt <= 0.05
 
-        if sys.platform == 'linux':
+        if sys.platform == "linux":
             # Always per-thread on Linux
             t = threading.Thread(target=run_for, args=(0.1,))
             start = metrics.thread_time()
diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
index 23e40c4c8ef..932419015f3 100644
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -18,8 +18,7 @@
 from distributed.metrics import time
 from distributed.protocol.pickle import dumps
 from distributed.utils import ignoring, tmpfile
-from distributed.utils_test import (gen_cluster, gen_test, slow, inc,
-        captured_logger)
+from distributed.utils_test import gen_cluster, gen_test, slow, inc, captured_logger
 
 
 @gen_cluster(ncores=[])
@@ -29,7 +28,7 @@ def test_nanny(s):
     with rpc(n.address) as nn:
         assert n.is_alive()
         assert s.ncores[n.worker_address] == 2
-        assert s.workers[n.worker_address].services['nanny'] > 1024
+        assert s.workers[n.worker_address].services["nanny"] > 1024
 
         yield nn.kill()
         assert not n.is_alive()
@@ -44,7 +43,7 @@ def test_nanny(s):
         yield nn.instantiate()
         assert n.is_alive()
         assert s.ncores[n.worker_address] == 2
-        assert s.workers[n.worker_address].services['nanny'] > 1024
+        assert s.workers[n.worker_address].services["nanny"] > 1024
 
         yield nn.terminate()
         assert not n.is_alive()
@@ -78,7 +77,7 @@ def test_nanny_process_failure(c, s):
 
     original_address = n.worker_address
     ww = rpc(n.worker_address)
-    yield ww.update_data(data=valmap(dumps, {'x': 1, 'y': 2}))
+    yield ww.update_data(data=valmap(dumps, {"x": 1, "y": 2}))
     pid = n.pid
     assert pid is not None
     with ignoring(CommClosedError):
@@ -112,31 +111,31 @@ def test_nanny_process_failure(c, s):
 
 
 def test_nanny_no_port():
-    _ = str(Nanny('127.0.0.1', 8786))
+    _ = str(Nanny("127.0.0.1", 8786))
 
 
 @gen_cluster(ncores=[])
 def test_run(s):
-    pytest.importorskip('psutil')
+    pytest.importorskip("psutil")
     n = yield Nanny(s.ip, s.port, ncores=2, loop=s.loop)
 
     with rpc(n.address) as nn:
         response = yield nn.run(function=dumps(lambda: 1))
-        assert response['status'] == 'OK'
-        assert response['result'] == 1
+        assert response["status"] == "OK"
+        assert response["result"] == 1
 
     yield n._close()
 
 
 @slow
-@gen_cluster(Worker=Nanny,
-             ncores=[('127.0.0.1', 1)],
-             worker_kwargs={'reconnect': False})
+@gen_cluster(
+    Worker=Nanny, ncores=[("127.0.0.1", 1)], worker_kwargs={"reconnect": False}
+)
 def test_close_on_disconnect(s, w):
     yield s.close()
 
     start = time()
-    while w.status != 'closed':
+    while w.status != "closed":
         yield gen.sleep(0.05)
         assert time() < start + 9
 
@@ -149,15 +148,14 @@ class Something(Worker):
 @gen_cluster(client=True, Worker=Nanny)
 def test_nanny_worker_class(c, s, w1, w2):
     out = yield c._run(lambda dask_worker=None: str(dask_worker.__class__))
-    assert 'Worker' in list(out.values())[0]
+    assert "Worker" in list(out.values())[0]
     assert w1.Worker is Worker
 
 
-@gen_cluster(client=True, Worker=Nanny,
-             worker_kwargs={'worker_class': Something})
+@gen_cluster(client=True, Worker=Nanny, worker_kwargs={"worker_class": Something})
 def test_nanny_alt_worker_class(c, s, w1, w2):
     out = yield c._run(lambda dask_worker=None: str(dask_worker.__class__))
-    assert 'Something' in list(out.values())[0]
+    assert "Something" in list(out.values())[0]
     assert w1.Worker is Something
 
 
@@ -168,15 +166,15 @@ def test_nanny_death_timeout(s):
     w = yield Nanny(s.address, death_timeout=1)
 
     yield gen.sleep(3)
-    assert w.status == 'closed'
+    assert w.status == "closed"
 
 
 @gen_cluster(client=True, Worker=Nanny)
 def test_random_seed(c, s, a, b):
     @gen.coroutine
     def check_func(func):
-        x = c.submit(func, 0, 2**31, pure=False, workers=a.worker_address)
-        y = c.submit(func, 0, 2**31, pure=False, workers=b.worker_address)
+        x = c.submit(func, 0, 2 ** 31, pure=False, workers=a.worker_address)
+        y = c.submit(func, 0, 2 ** 31, pure=False, workers=b.worker_address)
         assert x.key != y.key
         x = yield x
         y = yield y
@@ -186,11 +184,12 @@ def check_func(func):
     yield check_func(lambda a, b: np.random.randint(a, b))
 
 
-@pytest.mark.skipif(sys.platform.startswith('win'),
-                    reason="num_fds not supported on windows")
+@pytest.mark.skipif(
+    sys.platform.startswith("win"), reason="num_fds not supported on windows"
+)
 @gen_cluster(client=False, ncores=[])
 def test_num_fds(s):
-    psutil = pytest.importorskip('psutil')
+    psutil = pytest.importorskip("psutil")
     proc = psutil.Process()
 
     # Warm up
@@ -213,11 +212,12 @@ def test_num_fds(s):
         assert time() < start + 10
 
 
-@pytest.mark.skipif(not sys.platform.startswith('linux'),
-                    reason="Need 127.0.0.2 to mean localhost")
+@pytest.mark.skipif(
+    not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
+)
 @gen_cluster(client=True, ncores=[])
 def test_worker_uses_same_host_as_nanny(c, s):
-    for host in ['tcp://0.0.0.0', 'tcp://127.0.0.2']:
+    for host in ["tcp://0.0.0.0", "tcp://127.0.0.2"]:
         n = Nanny(s.address)
         yield n._start(host)
 
@@ -240,44 +240,50 @@ def test_scheduler_file():
         s.stop()
 
 
-@gen_cluster(client=True, Worker=Nanny, ncores=[('127.0.0.1', 2)])
+@gen_cluster(client=True, Worker=Nanny, ncores=[("127.0.0.1", 2)])
 def test_nanny_timeout(c, s, a):
     x = yield c.scatter(123)
-    with captured_logger(logging.getLogger('distributed.nanny'),
-                         level=logging.ERROR) as logger:
+    with captured_logger(
+        logging.getLogger("distributed.nanny"), level=logging.ERROR
+    ) as logger:
         response = yield a.restart(timeout=0.1)
 
     out = logger.getvalue()
-    assert 'timed out' in out.lower()
+    assert "timed out" in out.lower()
 
     start = time()
-    while x.status != 'cancelled':
+    while x.status != "cancelled":
         yield gen.sleep(0.1)
         assert time() < start + 7
 
 
-@gen_cluster(ncores=[('127.0.0.1', 1)], client=True, Worker=Nanny,
-             worker_kwargs={'memory_limit': 1e8}, timeout=20,
-             check_new_threads=False)
+@gen_cluster(
+    ncores=[("127.0.0.1", 1)],
+    client=True,
+    Worker=Nanny,
+    worker_kwargs={"memory_limit": 1e8},
+    timeout=20,
+    check_new_threads=False,
+)
 def test_nanny_terminate(c, s, a):
     from time import sleep
 
     def leak():
         L = []
         while True:
-            L.append(b'0' * 5000000)
+            L.append(b"0" * 5000000)
             sleep(0.01)
 
     proc = a.process.pid
-    with captured_logger(logging.getLogger('distributed.nanny')) as logger:
+    with captured_logger(logging.getLogger("distributed.nanny")) as logger:
         future = c.submit(leak)
         start = time()
         while a.process.pid == proc:
             yield gen.sleep(0.1)
             assert time() < start + 10
         out = logger.getvalue()
-        assert 'restart' in out.lower()
-        assert 'memory' in out.lower()
+        assert "restart" in out.lower()
+        assert "memory" in out.lower()
 
 
 @gen_cluster(ncores=[], client=True)
@@ -286,8 +292,8 @@ def test_avoid_memory_monitor_if_zero_limit(c, s):
     typ = yield c.run(lambda dask_worker: type(dask_worker.data))
     assert typ == {nanny.worker_address: dict}
     pcs = yield c.run(lambda dask_worker: list(dask_worker.periodic_callbacks))
-    assert 'memory' not in pcs
-    assert 'memory' not in nanny.periodic_callbacks
+    assert "memory" not in pcs
+    assert "memory" not in nanny.periodic_callbacks
 
     future = c.submit(inc, 1)
     assert (yield future) == 2
@@ -300,7 +306,7 @@ def test_avoid_memory_monitor_if_zero_limit(c, s):
 
 @gen_cluster(ncores=[], client=True)
 def test_scheduler_address_config(c, s):
-    with dask.config.set({'scheduler-address': s.address}):
+    with dask.config.set({"scheduler-address": s.address}):
         nanny = yield Nanny(loop=s.loop)
         assert nanny.scheduler.address == s.address
 
@@ -315,14 +321,14 @@ def test_scheduler_address_config(c, s):
 @slow
 @gen_test()
 def test_wait_for_scheduler():
-    with captured_logger('distributed') as log:
-        w = Nanny('127.0.0.1:44737')
+    with captured_logger("distributed") as log:
+        w = Nanny("127.0.0.1:44737")
         w._start()
         yield gen.sleep(6)
 
     log = log.getvalue()
-    assert 'error' not in log.lower(), log
-    assert 'restart' not in log.lower(), log
+    assert "error" not in log.lower(), log
+    assert "restart" not in log.lower(), log
 
 
 @gen_cluster(ncores=[], client=True)
@@ -330,7 +336,7 @@ def test_environment_variable(c, s):
     a = Nanny(s.address, loop=s.loop, memory_limit=0, env={"FOO": "123"})
     b = Nanny(s.address, loop=s.loop, memory_limit=0, env={"FOO": "456"})
     yield [a, b]
-    results = yield c.run(lambda: os.environ['FOO'])
+    results = yield c.run(lambda: os.environ["FOO"])
     assert results == {a.worker_address: "123", b.worker_address: "456"}
     yield [a._close(), b._close()]
 
diff --git a/distributed/tests/test_preload.py b/distributed/tests/test_preload.py
index 6239fdd8c0b..07ee56d85a6 100644
--- a/distributed/tests/test_preload.py
+++ b/distributed/tests/test_preload.py
@@ -20,47 +20,48 @@ def get_worker_address():
 
 
 def test_worker_preload_file(loop):
-
     def check_worker():
         import worker_info
+
         return worker_info.get_worker_address()
 
     tmpdir = tempfile.mkdtemp()
     try:
-        path = os.path.join(tmpdir, 'worker_info.py')
-        with open(path, 'w') as f:
+        path = os.path.join(tmpdir, "worker_info.py")
+        with open(path, "w") as f:
             f.write(PRELOAD_TEXT)
 
-        with cluster(worker_kwargs={'preload': [path]}) as (s, workers), \
-                Client(s['address'], loop=loop) as c:
+        with cluster(worker_kwargs={"preload": [path]}) as (s, workers), Client(
+            s["address"], loop=loop
+        ) as c:
 
             assert c.run(check_worker) == {
-                worker['address']: worker['address']
-                for worker in workers
+                worker["address"]: worker["address"] for worker in workers
             }
     finally:
         shutil.rmtree(tmpdir)
 
 
 def test_worker_preload_module(loop):
-
     def check_worker():
         import worker_info
+
         return worker_info.get_worker_address()
 
     tmpdir = tempfile.mkdtemp()
     sys.path.insert(0, tmpdir)
     try:
-        path = os.path.join(tmpdir, 'worker_info.py')
-        with open(path, 'w') as f:
+        path = os.path.join(tmpdir, "worker_info.py")
+        with open(path, "w") as f:
             f.write(PRELOAD_TEXT)
 
-        with cluster(worker_kwargs={'preload': ['worker_info']}) \
-                as (s, workers), Client(s['address'], loop=loop) as c:
+        with cluster(worker_kwargs={"preload": ["worker_info"]}) as (
+            s,
+            workers,
+        ), Client(s["address"], loop=loop) as c:
 
             assert c.run(check_worker) == {
-                worker['address']: worker['address']
-                for worker in workers
+                worker["address"]: worker["address"] for worker in workers
             }
     finally:
         sys.path.remove(tmpdir)
diff --git a/distributed/tests/test_priorities.py b/distributed/tests/test_priorities.py
index 0b18b1ba729..421bf7e3028 100644
--- a/distributed/tests/test_priorities.py
+++ b/distributed/tests/test_priorities.py
@@ -17,7 +17,7 @@ def test_submit(c, s, a, b):
     high = c.submit(inc, 2, priority=1)
     yield wait(high)
     assert all(s.processing.values())
-    assert s.tasks[low.key].state == 'processing'
+    assert s.tasks[low.key].state == "processing"
 
 
 @gen_cluster(client=True)
@@ -27,12 +27,12 @@ def test_map(c, s, a, b):
     high = c.map(inc, [4, 5, 6], priority=1)
     yield wait(high)
     assert all(s.processing.values())
-    assert s.tasks[low[0].key].state == 'processing'
+    assert s.tasks[low[0].key].state == "processing"
 
 
 @gen_cluster(client=True)
 def test_compute(c, s, a, b):
-    da = pytest.importorskip('dask.array')
+    da = pytest.importorskip("dask.array")
     x = da.random.random((10, 10), chunks=(5, 5))
     y = da.random.random((10, 10), chunks=(5, 5))
 
@@ -41,12 +41,12 @@ def test_compute(c, s, a, b):
     high = c.compute(y, priority=1)
     yield wait(high)
     assert all(s.processing.values())
-    assert s.tasks[tokey(low.key)].state in ('processing', 'waiting')
+    assert s.tasks[tokey(low.key)].state in ("processing", "waiting")
 
 
 @gen_cluster(client=True)
 def test_persist(c, s, a, b):
-    da = pytest.importorskip('dask.array')
+    da = pytest.importorskip("dask.array")
     x = da.random.random((10, 10), chunks=(5, 5))
     y = da.random.random((10, 10), chunks=(5, 5))
 
@@ -55,8 +55,10 @@ def test_persist(c, s, a, b):
     high = y.persist(priority=1)
     yield wait(high)
     assert all(s.processing.values())
-    assert all(s.tasks[tokey(k)].state in ('processing', 'waiting')
-               for k in flatten(low.__dask_keys__()))
+    assert all(
+        s.tasks[tokey(k)].state in ("processing", "waiting")
+        for k in flatten(low.__dask_keys__())
+    )
 
 
 @gen_cluster(client=True)
@@ -67,43 +69,51 @@ def test_expand_compute(c, s, a, b):
 
     low, many, high = c.compute([low, many, high], priority={low: -1, high: 1})
     yield wait(high)
-    assert s.tasks[low.key].state == 'processing'
+    assert s.tasks[low.key].state == "processing"
 
 
 @gen_cluster(client=True)
 def test_expand_persist(c, s, a, b):
-    low = delayed(inc)(1, dask_key_name='low')
+    low = delayed(inc)(1, dask_key_name="low")
     many = [delayed(slowinc)(i, delay=0.1) for i in range(4)]
-    high = delayed(inc)(2, dask_key_name='high')
+    high = delayed(inc)(2, dask_key_name="high")
 
     low, high, x, y, z, w = persist(low, high, *many, priority={low: -1, high: 1})
     yield wait(high)
-    assert s.tasks[low.key].state == 'processing'
+    assert s.tasks[low.key].state == "processing"
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)])
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)])
 def test_repeated_persists_same_priority(c, s, w):
-    xs = [delayed(slowinc)(i, delay=0.05, dask_key_name='x-%d' % i) for i in range(10)]
-    ys = [delayed(slowinc)(x, delay=0.05, dask_key_name='y-%d' % i) for i, x in enumerate(xs)]
-    zs = [delayed(slowdec)(x, delay=0.05, dask_key_name='z-%d' % i) for i, x in enumerate(xs)]
+    xs = [delayed(slowinc)(i, delay=0.05, dask_key_name="x-%d" % i) for i in range(10)]
+    ys = [
+        delayed(slowinc)(x, delay=0.05, dask_key_name="y-%d" % i)
+        for i, x in enumerate(xs)
+    ]
+    zs = [
+        delayed(slowdec)(x, delay=0.05, dask_key_name="z-%d" % i)
+        for i, x in enumerate(xs)
+    ]
 
     ys = dask.persist(*ys)
     zs = dask.persist(*zs)
 
-    while sum(t.state == 'memory' for t in s.tasks.values()) < 5:  # TODO: reduce this number
+    while (
+        sum(t.state == "memory" for t in s.tasks.values()) < 5
+    ):  # TODO: reduce this number
         yield gen.sleep(0.01)
 
-    assert any(s.tasks[y.key].state == 'memory' for y in ys)
-    assert any(s.tasks[z.key].state == 'memory' for z in zs)
+    assert any(s.tasks[y.key].state == "memory" for y in ys)
+    assert any(s.tasks[z.key].state == "memory" for z in zs)
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)])
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)])
 def test_last_in_first_out(c, s, w):
     xs = [c.submit(slowinc, i, delay=0.05) for i in range(5)]
     ys = [c.submit(slowinc, x, delay=0.05) for x in xs]
     zs = [c.submit(slowinc, y, delay=0.05) for y in ys]
 
-    while len(s.tasks) < 15 or not any(s.tasks[z.key].state == 'memory' for z in zs):
+    while len(s.tasks) < 15 or not any(s.tasks[z.key].state == "memory" for z in zs):
         yield gen.sleep(0.01)
 
-    assert not all(s.tasks[x.key].state == 'memory' for x in xs)
+    assert not all(s.tasks[x.key].state == "memory" for x in xs)
diff --git a/distributed/tests/test_profile.py b/distributed/tests/test_profile.py
index b7c717e1b61..57a7ca657e4 100644
--- a/distributed/tests/test_profile.py
+++ b/distributed/tests/test_profile.py
@@ -5,8 +5,7 @@
 
 from distributed.compatibility import get_thread_identity
 from distributed import metrics
-from distributed.profile import (process, merge, create, call_stack,
-        identifier, watch)
+from distributed.profile import process, merge, create, call_stack, identifier, watch
 
 
 def test_basic():
@@ -32,66 +31,86 @@ def test_f():
         frame = sys._current_frames()[thread.ident]
         process(frame, None, state)
 
-    assert state['count'] == 100
+    assert state["count"] == 100
     d = state
-    while len(d['children']) == 1:
-        d = first(d['children'].values())
+    while len(d["children"]) == 1:
+        d = first(d["children"].values())
 
-    assert d['count'] == 100
-    assert 'test_f' in str(d['description'])
-    g = [c for c in d['children'].values() if 'test_g' in str(c['description'])][0]
-    h = [c for c in d['children'].values() if 'test_h' in str(c['description'])][0]
+    assert d["count"] == 100
+    assert "test_f" in str(d["description"])
+    g = [c for c in d["children"].values() if "test_g" in str(c["description"])][0]
+    h = [c for c in d["children"].values() if "test_h" in str(c["description"])][0]
 
-    assert g['count'] < h['count']
-    assert 95 < g['count'] + h['count'] <= 100
+    assert g["count"] < h["count"]
+    assert 95 < g["count"] + h["count"] <= 100
 
 
 def test_merge():
     a1 = {
-         'count': 5,
-         'identifier': 'root',
-         'description': 'a',
-         'children': {
-             'b': {'count': 3,
-                   'description': 'b-func',
-                   'identifier': 'b',
-                   'children': {}},
-             'c': {'count': 2,
-                   'description': 'c-func',
-                   'identifier': 'c',
-                   'children': {}}}}
+        "count": 5,
+        "identifier": "root",
+        "description": "a",
+        "children": {
+            "b": {
+                "count": 3,
+                "description": "b-func",
+                "identifier": "b",
+                "children": {},
+            },
+            "c": {
+                "count": 2,
+                "description": "c-func",
+                "identifier": "c",
+                "children": {},
+            },
+        },
+    }
 
     a2 = {
-         'count': 4,
-         'description': 'a',
-         'identifier': 'root',
-         'children': {
-             'd': {'count': 2,
-                   'description': 'd-func',
-                   'children': {},
-                   'identifier': 'd'},
-             'c': {'count': 2,
-                   'description': 'c-func',
-                   'children': {},
-                   'identifier': 'c'}}}
+        "count": 4,
+        "description": "a",
+        "identifier": "root",
+        "children": {
+            "d": {
+                "count": 2,
+                "description": "d-func",
+                "children": {},
+                "identifier": "d",
+            },
+            "c": {
+                "count": 2,
+                "description": "c-func",
+                "children": {},
+                "identifier": "c",
+            },
+        },
+    }
 
     expected = {
-         'count': 9,
-         'identifier': 'root',
-         'description': 'a',
-         'children': {
-             'b': {'count': 3,
-                   'description': 'b-func',
-                   'identifier': 'b',
-                   'children': {}},
-             'd': {'count': 2,
-                   'description': 'd-func',
-                   'identifier': 'd',
-                   'children': {}},
-             'c': {'count': 4,
-                   'description': 'c-func',
-                   'identifier': 'c',
-                   'children': {}}}}
+        "count": 9,
+        "identifier": "root",
+        "description": "a",
+        "children": {
+            "b": {
+                "count": 3,
+                "description": "b-func",
+                "identifier": "b",
+                "children": {},
+            },
+            "d": {
+                "count": 2,
+                "description": "d-func",
+                "identifier": "d",
+                "children": {},
+            },
+            "c": {
+                "count": 4,
+                "description": "c-func",
+                "identifier": "c",
+                "children": {},
+            },
+        },
+    }
 
     assert merge(a1, a2) == expected
 
@@ -107,7 +126,7 @@ def test_call_stack():
     L = call_stack(frame)
     assert isinstance(L, list)
     assert all(isinstance(s, str) for s in L)
-    assert 'test_call_stack' in str(L[-1])
+    assert "test_call_stack" in str(L[-1])
 
 
 def test_identifier():
@@ -124,7 +143,7 @@ def stop():
 
     start_threads = threading.active_count()
 
-    log = watch(interval='10ms', cycle='50ms', stop=stop)
+    log = watch(interval="10ms", cycle="50ms", stop=stop)
 
     start = metrics.time()  # wait until thread starts up
     while threading.active_count() <= start_threads:
diff --git a/distributed/tests/test_publish.py b/distributed/tests/test_publish.py
index c6a899374c8..e4789589c48 100644
--- a/distributed/tests/test_publish.py
+++ b/distributed/tests/test_publish.py
@@ -1,4 +1,3 @@
-
 import pytest
 
 from dask import delayed
@@ -18,8 +17,8 @@ def test_publish_simple(s, a, b):
 
     data = yield c.scatter(range(3))
     out = yield c.publish_dataset(data=data)
-    assert 'data' in s.extensions['publish'].datasets
-    assert isinstance(s.extensions['publish'].datasets['data']['data'], Serialized)
+    assert "data" in s.extensions["publish"].datasets
+    assert isinstance(s.extensions["publish"].datasets["data"]["data"], Serialized)
 
     with pytest.raises(KeyError) as exc_info:
         out = yield c.publish_dataset(data=data)
@@ -28,10 +27,10 @@ def test_publish_simple(s, a, b):
     assert "data" in str(exc_info.value)
 
     result = yield c.scheduler.publish_list()
-    assert result == ('data',)
+    assert result == ("data",)
 
     result = yield f.scheduler.publish_list()
-    assert result == ('data',)
+    assert result == ("data",)
 
     yield c.close()
     yield f.close()
@@ -43,11 +42,13 @@ def test_publish_non_string_key(s, a, b):
     f = yield Client((s.ip, s.port), asynchronous=True)
 
     try:
-        for name in [('a', 'b'), 9.0, 8]:
+        for name in [("a", "b"), 9.0, 8]:
             data = yield c.scatter(range(3))
             out = yield c.publish_dataset(data, name=name)
-            assert name in s.extensions['publish'].datasets
-            assert isinstance(s.extensions['publish'].datasets[name]['data'], Serialized)
+            assert name in s.extensions["publish"].datasets
+            assert isinstance(
+                s.extensions["publish"].datasets[name]["data"], Serialized
+            )
 
             datasets = yield c.scheduler.publish_list()
             assert name in datasets
@@ -65,15 +66,15 @@ def test_publish_roundtrip(s, a, b):
     data = yield c.scatter([0, 1, 2])
     yield c.publish_dataset(data=data)
 
-    assert 'published-data' in s.who_wants[data[0].key]
-    result = yield f.get_dataset(name='data')
+    assert "published-data" in s.who_wants[data[0].key]
+    result = yield f.get_dataset(name="data")
 
     assert len(result) == len(data)
     out = yield f.gather(result)
     assert out == [0, 1, 2]
 
     with pytest.raises(KeyError) as exc_info:
-        result = yield f.get_dataset(name='nonexistent')
+        result = yield f.get_dataset(name="nonexistent")
 
     assert "not found" in str(exc_info.value)
     assert "nonexistent" in str(exc_info.value)
@@ -90,9 +91,9 @@ def test_unpublish(c, s, a, b):
     key = data[0].key
     del data
 
-    yield c.scheduler.publish_delete(name='data')
+    yield c.scheduler.publish_delete(name="data")
 
-    assert 'data' not in s.extensions['publish'].datasets
+    assert "data" not in s.extensions["publish"].datasets
 
     start = time()
     while key in s.who_wants:
@@ -100,7 +101,7 @@ def test_unpublish(c, s, a, b):
         assert time() < start + 5
 
     with pytest.raises(KeyError) as exc_info:
-        result = yield c.get_dataset(name='data')
+        result = yield c.get_dataset(name="data")
 
     assert "not found" in str(exc_info.value)
     assert "data" in str(exc_info.value)
@@ -109,10 +110,10 @@ def test_unpublish(c, s, a, b):
 def test_unpublish_sync(client):
     data = client.scatter([0, 1, 2])
     client.publish_dataset(data=data)
-    client.unpublish_dataset(name='data')
+    client.unpublish_dataset(name="data")
 
     with pytest.raises(KeyError) as exc_info:
-        result = client.get_dataset(name='data')
+        result = client.get_dataset(name="data")
 
     assert "not found" in str(exc_info.value)
     assert "data" in str(exc_info.value)
@@ -125,28 +126,28 @@ def test_publish_multiple_datasets(c, s, a, b):
 
     yield c.publish_dataset(x=x, y=y)
     datasets = yield c.scheduler.publish_list()
-    assert set(datasets) == {'x', 'y'}
+    assert set(datasets) == {"x", "y"}
 
 
 def test_unpublish_multiple_datasets_sync(client):
     x = delayed(inc)(1)
     y = delayed(inc)(2)
     client.publish_dataset(x=x, y=y)
-    client.unpublish_dataset(name='x')
+    client.unpublish_dataset(name="x")
 
     with pytest.raises(KeyError) as exc_info:
-        result = client.get_dataset(name='x')
+        result = client.get_dataset(name="x")
 
     datasets = client.list_datasets()
-    assert set(datasets) == {'y'}
+    assert set(datasets) == {"y"}
 
     assert "not found" in str(exc_info.value)
     assert "x" in str(exc_info.value)
 
-    client.unpublish_dataset(name='y')
+    client.unpublish_dataset(name="y")
 
     with pytest.raises(KeyError) as exc_info:
-        result = client.get_dataset(name='y')
+        result = client.get_dataset(name="y")
 
     assert "not found" in str(exc_info.value)
     assert "y" in str(exc_info.value)
@@ -154,7 +155,7 @@ def test_unpublish_multiple_datasets_sync(client):
 
 @gen_cluster(client=False)
 def test_publish_bag(s, a, b):
-    db = pytest.importorskip('dask.bag')
+    db = pytest.importorskip("dask.bag")
     c = yield Client((s.ip, s.port), asynchronous=True)
     f = yield Client((s.ip, s.port), asynchronous=True)
 
@@ -170,7 +171,7 @@ def test_publish_bag(s, a, b):
     # check that serialization didn't affect original bag's dask
     assert len(futures_of(bagp)) == 3
 
-    result = yield f.get_dataset('data')
+    result = yield f.get_dataset("data")
     assert set(result.dask.keys()) == set(bagp.dask.keys())
     assert {f.key for f in result.dask.values()} == {f.key for f in bagp.dask.values()}
 
@@ -181,22 +182,22 @@ def test_publish_bag(s, a, b):
 
 
 def test_datasets_setitem(client):
-    for key in ['key', ('key', 'key'), 1]:
-        value = 'value'
+    for key in ["key", ("key", "key"), 1]:
+        value = "value"
         client.datasets[key] = value
         assert client.get_dataset(key) == value
 
 
 def test_datasets_getitem(client):
-    for key in ['key', ('key', 'key'), 1]:
-        value = 'value'
+    for key in ["key", ("key", "key"), 1]:
+        value = "value"
         client.publish_dataset(value, name=key)
         assert client.datasets[key] == value
 
 
 def test_datasets_delitem(client):
-    for key in ['key', ('key', 'key'), 1]:
-        value = 'value'
+    for key in ["key", ("key", "key"), 1]:
+        value = "value"
         client.publish_dataset(value, name=key)
         del client.datasets[key]
         assert key not in client.list_datasets()
@@ -209,7 +210,7 @@ def test_datasets_keys(client):
 
 
 def test_datasets_contains(client):
-    key, value = 'key', 'value'
+    key, value = "key", "value"
     client.publish_dataset(key=value)
     assert key in client.datasets
 
@@ -223,11 +224,10 @@ def test_datasets_iter(client):
 
 @gen_cluster(client=True)
 def test_pickle_safe(c, s, a, b):
-    c2 = yield Client(s.address, asynchronous=True,
-                     serializers=['msgpack'])
+    c2 = yield Client(s.address, asynchronous=True, serializers=["msgpack"])
     try:
         yield c2.publish_dataset(x=[1, 2, 3])
-        result = yield c2.get_dataset('x')
+        result = yield c2.get_dataset("x")
         assert result == (1, 2, 3)
 
         with pytest.raises(TypeError):
@@ -236,6 +236,6 @@ def test_pickle_safe(c, s, a, b):
         yield c.publish_dataset(z=lambda x: x)  # this can use pickle
 
         with pytest.raises(TypeError):
-            yield c2.get_dataset('z')
+            yield c2.get_dataset("z")
     finally:
         yield c2.close()
diff --git a/distributed/tests/test_pubsub.py b/distributed/tests/test_pubsub.py
index ff299400e6e..c44637cf9fd 100644
--- a/distributed/tests/test_pubsub.py
+++ b/distributed/tests/test_pubsub.py
@@ -18,6 +18,7 @@ def test_speed(c, s, a, b):
 
     Interestingly this runs 10x slower on Python 2
     """
+
     def pingpong(a, b, start=False, n=1000, msg=1):
         sub = Sub(a)
         pub = Pub(b)
@@ -36,10 +37,11 @@ def pingpong(a, b, start=False, n=1000, msg=1):
         return n
 
     import numpy as np
+
     x = np.random.random(1000)
 
-    x = c.submit(pingpong, 'a', 'b', start=True, msg=x, n=100)
-    y = c.submit(pingpong, 'b', 'a', n=100)
+    x = c.submit(pingpong, "a", "b", start=True, msg=x, n=100)
+    y = c.submit(pingpong, "b", "a", n=100)
 
     start = time()
     yield c.gather([x, y])
@@ -51,14 +53,14 @@ def pingpong(a, b, start=False, n=1000, msg=1):
 def test_client(c, s):
     with pytest.raises(Exception):
         get_worker()
-    sub = Sub('a')
-    pub = Pub('a')
+    sub = Sub("a")
+    pub = Pub("a")
 
-    sps = s.extensions['pubsub']
-    cps = c.extensions['pubsub']
+    sps = s.extensions["pubsub"]
+    cps = c.extensions["pubsub"]
 
     start = time()
-    while not set(sps.client_subscribers['a']) == {c.id}:
+    while not set(sps.client_subscribers["a"]) == {c.id}:
         yield gen.sleep(0.01)
         assert time() < start + 3
 
@@ -70,10 +72,10 @@ def test_client(c, s):
 
 @gen_cluster(client=True)
 def test_client_worker(c, s, a, b):
-    sub = Sub('a', client=c, worker=None)
+    sub = Sub("a", client=c, worker=None)
 
     def f(x):
-        pub = Pub('a')
+        pub = Pub("a")
         pub.put(x)
 
     futures = c.map(f, range(10))
@@ -86,32 +88,36 @@ def f(x):
 
     assert set(L) == set(range(10))
 
-    sps = s.extensions['pubsub']
-    aps = a.extensions['pubsub']
-    bps = b.extensions['pubsub']
+    sps = s.extensions["pubsub"]
+    aps = a.extensions["pubsub"]
+    bps = b.extensions["pubsub"]
 
     start = time()
-    while (sps.publishers['a'] or
-           sps.subscribers['a'] or
-           aps.publishers['a'] or
-           bps.publishers['a'] or
-           len(sps.client_subscribers['a']) != 1):
+    while (
+        sps.publishers["a"]
+        or sps.subscribers["a"]
+        or aps.publishers["a"]
+        or bps.publishers["a"]
+        or len(sps.client_subscribers["a"]) != 1
+    ):
         yield gen.sleep(0.01)
         assert time() < start + 3
 
     del sub
 
     start = time()
-    while (sps.client_subscribers or
-           any(aps.publish_to_scheduler.values()) or
-           any(bps.publish_to_scheduler.values())):
+    while (
+        sps.client_subscribers
+        or any(aps.publish_to_scheduler.values())
+        or any(bps.publish_to_scheduler.values())
+    ):
         yield gen.sleep(0.01)
         assert time() < start + 3
 
 
 @gen_cluster(client=True)
 def test_timeouts(c, s, a, b):
-    sub = Sub('a', client=c, worker=None)
+    sub = Sub("a", client=c, worker=None)
     start = time()
     with pytest.raises(TimeoutError):
         yield sub.get(timeout=0.1)
diff --git a/distributed/tests/test_queues.py b/distributed/tests/test_queues.py
index 6d6306ca6f9..e82b893989b 100644
--- a/distributed/tests/test_queues.py
+++ b/distributed/tests/test_queues.py
@@ -9,15 +9,15 @@
 
 from distributed import Client, Queue, Nanny, worker_client, wait
 from distributed.metrics import time
-from distributed.utils_test import (gen_cluster, inc, slow, div)
-from distributed.utils_test import client, cluster_fixture, loop # noqa: F401
+from distributed.utils_test import gen_cluster, inc, slow, div
+from distributed.utils_test import client, cluster_fixture, loop  # noqa: F401
 
 
 @gen_cluster(client=True)
 def test_queue(c, s, a, b):
-    x = yield Queue('x')
-    y = yield Queue('y')
-    xx = yield Queue('x')
+    x = yield Queue("x")
+    y = yield Queue("y")
+    xx = yield Queue("x")
     assert x.client is c
 
     future = c.submit(inc, 1)
@@ -44,14 +44,14 @@ def test_queue(c, s, a, b):
 
 @gen_cluster(client=True)
 def test_queue_with_data(c, s, a, b):
-    x = yield Queue('x')
-    xx = yield Queue('x')
+    x = yield Queue("x")
+    xx = yield Queue("x")
     assert x.client is c
 
-    yield x.put((1, 'hello'))
+    yield x.put((1, "hello"))
     data = yield xx.get()
 
-    assert data == (1, 'hello')
+    assert data == (1, "hello")
 
     with pytest.raises(gen.TimeoutError):
         yield x.get(timeout=0.1)
@@ -59,8 +59,8 @@ def test_queue_with_data(c, s, a, b):
 
 def test_sync(client):
     future = client.submit(lambda x: x + 1, 10)
-    x = Queue('x')
-    xx = Queue('x')
+    x = Queue("x")
+    xx = Queue("x")
     x.put(future)
     assert x.qsize() == 1
     assert xx.qsize() == 1
@@ -73,7 +73,7 @@ def test_sync(client):
 def test_hold_futures(s, a, b):
     c1 = yield Client(s.address, asynchronous=True)
     future = c1.submit(lambda x: x + 1, 10)
-    q1 = yield Queue('q')
+    q1 = yield Queue("q")
     yield q1.put(future)
     del q1
     yield c1.close()
@@ -81,7 +81,7 @@ def test_hold_futures(s, a, b):
     yield gen.sleep(0.1)
 
     c2 = yield Client(s.address, asynchronous=True)
-    q2 = yield Queue('q')
+    q2 = yield Queue("q")
     future2 = yield q2.get()
     result = yield future2
 
@@ -89,7 +89,7 @@ def test_hold_futures(s, a, b):
     yield c2.close()
 
 
-@pytest.mark.skip(reason='getting same client from main thread')
+@pytest.mark.skip(reason="getting same client from main thread")
 @gen_cluster(client=True)
 def test_picklability(c, s, a, b):
     q = Queue()
@@ -113,14 +113,13 @@ def f(x):
     assert q.get() == 11
 
 
-@pytest.mark.skipif(sys.version_info[0] == 2, reason='Multi-client issues')
+@pytest.mark.skipif(sys.version_info[0] == 2, reason="Multi-client issues")
 @slow
-@gen_cluster(client=True, ncores=[('127.0.0.1', 2)] * 5, Worker=Nanny,
-             timeout=None)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 2)] * 5, Worker=Nanny, timeout=None)
 def test_race(c, s, *workers):
     def f(i):
         with worker_client() as c:
-            q = Queue('x', client=c)
+            q = Queue("x", client=c)
             for _ in range(100):
                 future = q.get()
                 x = future.result()
@@ -130,7 +129,7 @@ def f(i):
             result = q.get().result()
             return result
 
-    q = Queue('x', client=c)
+    q = Queue("x", client=c)
     L = yield c.scatter(range(5))
     for future in L:
         yield q.put(future)
@@ -145,32 +144,32 @@ def f(i):
 
 @gen_cluster(client=True)
 def test_same_futures(c, s, a, b):
-    q = Queue('x')
+    q = Queue("x")
     future = yield c.scatter(123)
 
     for i in range(5):
         yield q.put(future)
 
-    assert s.wants_what['queue-x'] == {future.key}
+    assert s.wants_what["queue-x"] == {future.key}
 
     for i in range(4):
         future2 = yield q.get()
-        assert s.wants_what['queue-x'] == {future.key}
+        assert s.wants_what["queue-x"] == {future.key}
         yield gen.sleep(0.05)
-        assert s.wants_what['queue-x'] == {future.key}
+        assert s.wants_what["queue-x"] == {future.key}
 
     yield q.get()
 
     start = time()
-    while s.wants_what['queue-x']:
+    while s.wants_what["queue-x"]:
         yield gen.sleep(0.01)
         assert time() - start < 2
 
 
 @gen_cluster(client=True)
 def test_get_many(c, s, a, b):
-    x = yield Queue('x')
-    xx = yield Queue('x')
+    x = yield Queue("x")
+    xx = yield Queue("x")
 
     yield x.put(1)
     yield x.put(2)
@@ -187,27 +186,26 @@ def test_get_many(c, s, a, b):
     assert data == [1, 2]
 
     with pytest.raises(gen.TimeoutError):
-        data = yield gen.with_timeout(timedelta(seconds=0.100),
-                                      xx.get(batch=2))
+        data = yield gen.with_timeout(timedelta(seconds=0.100), xx.get(batch=2))
 
 
 @gen_cluster(client=True)
 def test_Future_knows_status_immediately(c, s, a, b):
     x = yield c.scatter(123)
-    q = yield Queue('q')
+    q = yield Queue("q")
     yield q.put(x)
 
     c2 = yield Client(s.address, asynchronous=True)
-    q2 = yield Queue('q', client=c2)
+    q2 = yield Queue("q", client=c2)
     future = yield q2.get()
-    assert future.status == 'finished'
+    assert future.status == "finished"
 
     x = c.submit(div, 1, 0)
     yield wait(x)
     yield q.put(x)
 
     future2 = yield q2.get()
-    assert future2.status == 'error'
+    assert future2.status == "error"
     with pytest.raises(Exception):
         yield future2
 
@@ -242,19 +240,19 @@ def test_erred_future(c, s, a, b):
 def test_close(c, s, a, b):
     q = Queue()
 
-    while q.name not in s.extensions['queues'].queues:
+    while q.name not in s.extensions["queues"].queues:
         yield gen.sleep(0.01)
 
     q.close()
     q.close()
 
-    while q.name in s.extensions['queues'].queues:
+    while q.name in s.extensions["queues"].queues:
         yield gen.sleep(0.01)
 
 
 @gen_cluster(client=True)
 def test_timeout(c, s, a, b):
-    q = Queue('v', maxsize=1)
+    q = Queue("v", maxsize=1)
 
     start = time()
     with pytest.raises(gen.TimeoutError):
diff --git a/distributed/tests/test_resources.py b/distributed/tests/test_resources.py
index 4b1e9e2a80c..35f5e160969 100644
--- a/distributed/tests/test_resources.py
+++ b/distributed/tests/test_resources.py
@@ -10,7 +10,7 @@
 from distributed.client import wait
 from distributed.compatibility import WINDOWS
 from distributed.utils import tokey
-from distributed.utils_test import (inc, gen_cluster, slowinc, slowadd)
+from distributed.utils_test import inc, gen_cluster, slowinc, slowadd
 from distributed.utils_test import client, cluster_fixture, loop, s, a, b  # noqa: F401
 
 
@@ -19,30 +19,33 @@ def test_resources(c, s):
     assert not s.worker_resources
     assert not s.resources
 
-    a = Worker(s.ip, s.port, loop=s.loop, resources={'GPU': 2})
-    b = Worker(s.ip, s.port, loop=s.loop, resources={'GPU': 1, 'DB': 1})
+    a = Worker(s.ip, s.port, loop=s.loop, resources={"GPU": 2})
+    b = Worker(s.ip, s.port, loop=s.loop, resources={"GPU": 1, "DB": 1})
 
     yield [a, b]
 
-    assert s.resources == {'GPU': {a.address: 2, b.address: 1},
-                           'DB': {b.address: 1}}
-    assert s.worker_resources == {a.address: {'GPU': 2},
-                                  b.address: {'GPU': 1, 'DB': 1}}
+    assert s.resources == {"GPU": {a.address: 2, b.address: 1}, "DB": {b.address: 1}}
+    assert s.worker_resources == {a.address: {"GPU": 2}, b.address: {"GPU": 1, "DB": 1}}
 
     yield b._close()
 
-    assert s.resources == {'GPU': {a.address: 2}, 'DB': {}}
-    assert s.worker_resources == {a.address: {'GPU': 2}}
+    assert s.resources == {"GPU": {a.address: 2}, "DB": {}}
+    assert s.worker_resources == {a.address: {"GPU": 2}}
 
     yield a._close()
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1, {'resources': {'A': 5}}),
-                                  ('127.0.0.1', 1, {'resources': {'A': 1, 'B': 1}})])
+@gen_cluster(
+    client=True,
+    ncores=[
+        ("127.0.0.1", 1, {"resources": {"A": 5}}),
+        ("127.0.0.1", 1, {"resources": {"A": 1, "B": 1}}),
+    ],
+)
 def test_resource_submit(c, s, a, b):
-    x = c.submit(inc, 1, resources={'A': 3})
-    y = c.submit(inc, 2, resources={'B': 1})
-    z = c.submit(inc, 3, resources={'C': 2})
+    x = c.submit(inc, 1, resources={"A": 3})
+    y = c.submit(inc, 2, resources={"B": 1})
+    z = c.submit(inc, 3, resources={"C": 2})
 
     yield wait(x)
     assert x.key in a.data
@@ -50,9 +53,9 @@ def test_resource_submit(c, s, a, b):
     yield wait(y)
     assert y.key in b.data
 
-    assert s.get_task_status(keys=[z.key]) == {z.key: 'no-worker'}
+    assert s.get_task_status(keys=[z.key]) == {z.key: "no-worker"}
 
-    d = yield Worker(s.ip, s.port, loop=s.loop, resources={'C': 10})
+    d = yield Worker(s.ip, s.port, loop=s.loop, resources={"C": 10})
 
     yield wait(z)
     assert z.key in d.data
@@ -60,55 +63,81 @@ def test_resource_submit(c, s, a, b):
     yield d._close()
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1, {'resources': {'A': 1}}),
-                                  ('127.0.0.1', 1, {'resources': {'B': 1}})])
+@gen_cluster(
+    client=True,
+    ncores=[
+        ("127.0.0.1", 1, {"resources": {"A": 1}}),
+        ("127.0.0.1", 1, {"resources": {"B": 1}}),
+    ],
+)
 def test_submit_many_non_overlapping(c, s, a, b):
-    futures = [c.submit(inc, i, resources={'A': 1}) for i in range(5)]
+    futures = [c.submit(inc, i, resources={"A": 1}) for i in range(5)]
     yield wait(futures)
 
     assert len(a.data) == 5
     assert len(b.data) == 0
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1, {'resources': {'A': 1}}),
-                                  ('127.0.0.1', 1, {'resources': {'B': 1}})])
+@gen_cluster(
+    client=True,
+    ncores=[
+        ("127.0.0.1", 1, {"resources": {"A": 1}}),
+        ("127.0.0.1", 1, {"resources": {"B": 1}}),
+    ],
+)
 def test_move(c, s, a, b):
     [x] = yield c._scatter([1], workers=b.address)
 
-    future = c.submit(inc, x, resources={'A': 1})
+    future = c.submit(inc, x, resources={"A": 1})
 
     yield wait(future)
     assert a.data[future.key] == 2
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1, {'resources': {'A': 1}}),
-                                  ('127.0.0.1', 1, {'resources': {'B': 1}})])
+@gen_cluster(
+    client=True,
+    ncores=[
+        ("127.0.0.1", 1, {"resources": {"A": 1}}),
+        ("127.0.0.1", 1, {"resources": {"B": 1}}),
+    ],
+)
 def test_dont_work_steal(c, s, a, b):
     [x] = yield c._scatter([1], workers=a.address)
 
-    futures = [c.submit(slowadd, x, i, resources={'A': 1}, delay=0.05)
-               for i in range(10)]
+    futures = [
+        c.submit(slowadd, x, i, resources={"A": 1}, delay=0.05) for i in range(10)
+    ]
 
     yield wait(futures)
     assert all(f.key in a.data for f in futures)
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1, {'resources': {'A': 1}}),
-                                  ('127.0.0.1', 1, {'resources': {'B': 1}})])
+@gen_cluster(
+    client=True,
+    ncores=[
+        ("127.0.0.1", 1, {"resources": {"A": 1}}),
+        ("127.0.0.1", 1, {"resources": {"B": 1}}),
+    ],
+)
 def test_map(c, s, a, b):
-    futures = c.map(inc, range(10), resources={'B': 1})
+    futures = c.map(inc, range(10), resources={"B": 1})
     yield wait(futures)
     assert set(b.data) == {f.key for f in futures}
     assert not a.data
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1, {'resources': {'A': 1}}),
-                                  ('127.0.0.1', 1, {'resources': {'B': 1}})])
+@gen_cluster(
+    client=True,
+    ncores=[
+        ("127.0.0.1", 1, {"resources": {"A": 1}}),
+        ("127.0.0.1", 1, {"resources": {"B": 1}}),
+    ],
+)
 def test_persist(c, s, a, b):
     x = delayed(inc)(1)
     y = delayed(inc)(x)
 
-    xx, yy = c.persist([x, y], resources={x: {'A': 1}, y: {'B': 1}})
+    xx, yy = c.persist([x, y], resources={x: {"A": 1}, y: {"B": 1}})
 
     yield wait([xx, yy])
 
@@ -116,40 +145,55 @@ def test_persist(c, s, a, b):
     assert y.key in b.data
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1, {'resources': {'A': 1}}),
-                                  ('127.0.0.1', 1, {'resources': {'B': 11}})])
+@gen_cluster(
+    client=True,
+    ncores=[
+        ("127.0.0.1", 1, {"resources": {"A": 1}}),
+        ("127.0.0.1", 1, {"resources": {"B": 11}}),
+    ],
+)
 def test_compute(c, s, a, b):
     x = delayed(inc)(1)
     y = delayed(inc)(x)
 
-    yy = c.compute(y, resources={x: {'A': 1}, y: {'B': 1}})
+    yy = c.compute(y, resources={x: {"A": 1}, y: {"B": 1}})
     yield wait(yy)
 
     assert b.data
 
     xs = [delayed(inc)(i) for i in range(10, 20)]
-    xxs = c.compute(xs, resources={'B': 1})
+    xxs = c.compute(xs, resources={"B": 1})
     yield wait(xxs)
 
     assert len(b.data) > 10
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1, {'resources': {'A': 1}}),
-                                  ('127.0.0.1', 1, {'resources': {'B': 1}})])
+@gen_cluster(
+    client=True,
+    ncores=[
+        ("127.0.0.1", 1, {"resources": {"A": 1}}),
+        ("127.0.0.1", 1, {"resources": {"B": 1}}),
+    ],
+)
 def test_get(c, s, a, b):
-    dsk = {'x': (inc, 1), 'y': (inc, 'x')}
+    dsk = {"x": (inc, 1), "y": (inc, "x")}
 
-    result = yield c.get(dsk, 'y', resources={'y': {'A': 1}}, sync=False)
+    result = yield c.get(dsk, "y", resources={"y": {"A": 1}}, sync=False)
     assert result == 3
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1, {'resources': {'A': 1}}),
-                                  ('127.0.0.1', 1, {'resources': {'B': 1}})])
+@gen_cluster(
+    client=True,
+    ncores=[
+        ("127.0.0.1", 1, {"resources": {"A": 1}}),
+        ("127.0.0.1", 1, {"resources": {"B": 1}}),
+    ],
+)
 def test_persist_tuple(c, s, a, b):
     x = delayed(inc)(1)
     y = delayed(inc)(x)
 
-    xx, yy = c.persist([x, y], resources={(x, y): {'A': 1}})
+    xx, yy = c.persist([x, y], resources={(x, y): {"A": 1}})
 
     yield wait([xx, yy])
 
@@ -158,10 +202,15 @@ def test_persist_tuple(c, s, a, b):
     assert not b.data
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 4, {'resources': {'A': 2}}),
-                                  ('127.0.0.1', 4, {'resources': {'A': 1}})])
+@gen_cluster(
+    client=True,
+    ncores=[
+        ("127.0.0.1", 4, {"resources": {"A": 2}}),
+        ("127.0.0.1", 4, {"resources": {"A": 1}}),
+    ],
+)
 def test_submit_many_non_overlapping(c, s, a, b):
-    futures = c.map(slowinc, range(100), resources={'A': 1}, delay=0.02)
+    futures = c.map(slowinc, range(100), resources={"A": 1}, delay=0.02)
 
     while len(a.data) + len(b.data) < 100:
         yield gen.sleep(0.01)
@@ -173,9 +222,9 @@ def test_submit_many_non_overlapping(c, s, a, b):
     assert b.total_resources == b.available_resources
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 4, {'resources': {'A': 2, 'B': 1}})])
+@gen_cluster(client=True, ncores=[("127.0.0.1", 4, {"resources": {"A": 2, "B": 1}})])
 def test_minimum_resource(c, s, a):
-    futures = c.map(slowinc, range(30), resources={'A': 1, 'B': 1}, delay=0.02)
+    futures = c.map(slowinc, range(30), resources={"A": 1, "B": 1}, delay=0.02)
 
     while len(a.data) < 30:
         yield gen.sleep(0.01)
@@ -185,10 +234,10 @@ def test_minimum_resource(c, s, a):
     assert a.total_resources == a.available_resources
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 2, {'resources': {'A': 1}})])
+@gen_cluster(client=True, ncores=[("127.0.0.1", 2, {"resources": {"A": 1}})])
 def test_prefer_constrained(c, s, a):
     futures = c.map(slowinc, range(1000), delay=0.1)
-    constrained = c.map(inc, range(10), resources={'A': 1})
+    constrained = c.map(inc, range(10), resources={"A": 1})
 
     start = time()
     yield wait(constrained)
@@ -201,44 +250,54 @@ def test_prefer_constrained(c, s, a):
 
 
 @pytest.mark.skip(reason="")
-@gen_cluster(client=True, ncores=[('127.0.0.1', 2, {'resources': {'A': 1}}),
-                                  ('127.0.0.1', 2, {'resources': {'A': 1}})])
+@gen_cluster(
+    client=True,
+    ncores=[
+        ("127.0.0.1", 2, {"resources": {"A": 1}}),
+        ("127.0.0.1", 2, {"resources": {"A": 1}}),
+    ],
+)
 def test_balance_resources(c, s, a, b):
     futures = c.map(slowinc, range(100), delay=0.1, workers=a.address)
-    constrained = c.map(inc, range(2), resources={'A': 1})
+    constrained = c.map(inc, range(2), resources={"A": 1})
 
     yield wait(constrained)
     assert any(f.key in a.data for f in constrained)  # share
     assert any(f.key in b.data for f in constrained)
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 2)])
+@gen_cluster(client=True, ncores=[("127.0.0.1", 2)])
 def test_set_resources(c, s, a):
     yield a.set_resources(A=2)
-    assert a.total_resources['A'] == 2
-    assert a.available_resources['A'] == 2
-    assert s.worker_resources[a.address] == {'A': 2}
+    assert a.total_resources["A"] == 2
+    assert a.available_resources["A"] == 2
+    assert s.worker_resources[a.address] == {"A": 2}
 
-    future = c.submit(slowinc, 1, delay=1, resources={'A': 1})
-    while a.available_resources['A'] == 2:
+    future = c.submit(slowinc, 1, delay=1, resources={"A": 1})
+    while a.available_resources["A"] == 2:
         yield gen.sleep(0.01)
 
     yield a.set_resources(A=3)
-    assert a.total_resources['A'] == 3
-    assert a.available_resources['A'] == 2
-    assert s.worker_resources[a.address] == {'A': 3}
-
-
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1, {'resources': {'A': 1}}),
-                                  ('127.0.0.1', 1, {'resources': {'B': 1}})])
+    assert a.total_resources["A"] == 3
+    assert a.available_resources["A"] == 2
+    assert s.worker_resources[a.address] == {"A": 3}
+
+
+@gen_cluster(
+    client=True,
+    ncores=[
+        ("127.0.0.1", 1, {"resources": {"A": 1}}),
+        ("127.0.0.1", 1, {"resources": {"B": 1}}),
+    ],
+)
 def test_persist_collections(c, s, a, b):
-    da = pytest.importorskip('dask.array')
+    da = pytest.importorskip("dask.array")
     x = da.arange(10, chunks=(5,))
     y = x.map_blocks(lambda x: x + 1)
     z = y.map_blocks(lambda x: 2 * x)
     w = z.sum()
 
-    ww, yy = c.persist([w, y], resources={tuple(y.__dask_keys__()): {'A': 1}})
+    ww, yy = c.persist([w, y], resources={tuple(y.__dask_keys__()): {"A": 1}})
 
     yield wait([ww, yy])
 
@@ -246,57 +305,75 @@ def test_persist_collections(c, s, a, b):
 
 
 @pytest.mark.skip(reason="Should protect resource keys from optimization")
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1, {'resources': {'A': 1}}),
-                                  ('127.0.0.1', 1, {'resources': {'B': 1}})])
+@gen_cluster(
+    client=True,
+    ncores=[
+        ("127.0.0.1", 1, {"resources": {"A": 1}}),
+        ("127.0.0.1", 1, {"resources": {"B": 1}}),
+    ],
+)
 def test_dont_optimize_out(c, s, a, b):
-    da = pytest.importorskip('dask.array')
+    da = pytest.importorskip("dask.array")
     x = da.arange(10, chunks=(5,))
     y = x.map_blocks(lambda x: x + 1)
     z = y.map_blocks(lambda x: 2 * x)
     w = z.sum()
 
-    yield c.compute(w, resources={tuple(y.__dask_keys__()): {'A': 1}},)
+    yield c.compute(w, resources={tuple(y.__dask_keys__()): {"A": 1}})
 
     for key in map(tokey, y.__dask_keys__()):
-        assert 'executing' in str(a.story(key))
+        assert "executing" in str(a.story(key))
 
 
 @pytest.mark.xfail(reason="atop fusion seemed to break this")
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1, {'resources': {'A': 1}}),
-                                  ('127.0.0.1', 1, {'resources': {'B': 1}})])
+@gen_cluster(
+    client=True,
+    ncores=[
+        ("127.0.0.1", 1, {"resources": {"A": 1}}),
+        ("127.0.0.1", 1, {"resources": {"B": 1}}),
+    ],
+)
 def test_full_collections(c, s, a, b):
-    dd = pytest.importorskip('dask.dataframe')
-    df = dd.demo.make_timeseries(freq='60s', partition_freq='1d',
-            start='2000-01-01', end='2000-01-31')
+    dd = pytest.importorskip("dask.dataframe")
+    df = dd.demo.make_timeseries(
+        freq="60s", partition_freq="1d", start="2000-01-01", end="2000-01-31"
+    )
     z = df.x + df.y  # some extra nodes in the graph
 
-    yield c.compute(z, resources={tuple(z.dask): {'A': 1}})
+    yield c.compute(z, resources={tuple(z.dask): {"A": 1}})
     assert a.log
     assert not b.log
 
 
-@pytest.mark.parametrize('optimize_graph', [
-    pytest.param(True,
-        marks=pytest.mark.xfail(reason="don't track resources through optimization")),
-    pytest.param(False,
-        marks=pytest.mark.skipif(WINDOWS, reason="intermittent failure"))
-])
+@pytest.mark.parametrize(
+    "optimize_graph",
+    [
+        pytest.param(
+            True,
+            marks=pytest.mark.xfail(
+                reason="don't track resources through optimization"
+            ),
+        ),
+        pytest.param(
+            False, marks=pytest.mark.skipif(WINDOWS, reason="intermittent failure")
+        ),
+    ],
+)
 def test_collections_get(client, optimize_graph, s, a, b):
-    da = pytest.importorskip('dask.array')
+    da = pytest.importorskip("dask.array")
 
     def f(dask_worker):
-        dask_worker.set_resources(**{'A': 1})
+        dask_worker.set_resources(**{"A": 1})
 
-    client.run(f, workers=[a['address']])
+    client.run(f, workers=[a["address"]])
 
     x = da.random.random(100, chunks=(10,)) + 1
 
-    x.compute(resources={tuple(x.dask): {'A': 1}},
-              optimize_graph=optimize_graph)
+    x.compute(resources={tuple(x.dask): {"A": 1}}, optimize_graph=optimize_graph)
 
     def g(dask_worker):
         return len(dask_worker.log)
 
     logs = client.run(g)
-    assert logs[a['address']]
-    assert not logs[b['address']]
+    assert logs[a["address"]]
+    assert not logs[b["address"]]
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index edf4a4eaece..02f15e1e1a2 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -23,15 +23,25 @@
 from distributed.protocol.pickle import dumps
 from distributed.worker import dumps_function, dumps_task
 from distributed.utils import tmpfile
-from distributed.utils_test import (inc, dec, gen_cluster, gen_test,
-                                    slowinc, slowadd, slowdec, cluster, div,
-                                    varying, slow)
+from distributed.utils_test import (
+    inc,
+    dec,
+    gen_cluster,
+    gen_test,
+    slowinc,
+    slowadd,
+    slowdec,
+    cluster,
+    div,
+    varying,
+    slow,
+)
 from distributed.utils_test import loop, nodebug  # noqa: F401
 from dask.compatibility import apply
 
 
-alice = 'alice:1234'
-bob = 'bob:1234'
+alice = "alice:1234"
+bob = "bob:1234"
 
 occupancy = defaultdict(lambda: 0)
 
@@ -44,7 +54,7 @@ def test_administration(s, a, b):
     assert str(len(s.ncores)) in repr(s)
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)])
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)])
 def test_respect_data_in_memory(c, s, a):
     x = delayed(inc)(1)
     y = delayed(inc)(x)
@@ -79,28 +89,31 @@ def test_recompute_released_results(c, s, a, b):
 
 @gen_cluster(client=True)
 def test_decide_worker_with_many_independent_leaves(c, s, a, b):
-    xs = yield [c.scatter(list(range(0, 100, 2)), workers=a.address),
-                c.scatter(list(range(1, 100, 2)), workers=b.address)]
+    xs = yield [
+        c.scatter(list(range(0, 100, 2)), workers=a.address),
+        c.scatter(list(range(1, 100, 2)), workers=b.address),
+    ]
     xs = list(concat(zip(*xs)))
     ys = [delayed(inc)(x) for x in xs]
 
     y2s = c.persist(ys)
     yield wait(y2s)
 
-    nhits = (sum(y.key in a.data for y in y2s[::2]) +
-             sum(y.key in b.data for y in y2s[1::2]))
+    nhits = sum(y.key in a.data for y in y2s[::2]) + sum(
+        y.key in b.data for y in y2s[1::2]
+    )
 
     assert nhits > 80
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 3)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 3)
 def test_decide_worker_with_restrictions(client, s, a, b, c):
     x = client.submit(inc, 1, workers=[a.address, b.address])
     yield wait(x)
     assert x.key in a.data or x.key in b.data
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 3)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 3)
 def test_move_data_over_break_restrictions(client, s, a, b, c):
     [x] = yield client.scatter([1], workers=b.address)
     y = client.submit(inc, x, workers=[a.address, b.address])
@@ -108,19 +121,21 @@ def test_move_data_over_break_restrictions(client, s, a, b, c):
     assert y.key in a.data or y.key in b.data
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 3)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 3)
 def test_balance_with_restrictions(client, s, a, b, c):
-    [x], [y] = yield [client.scatter([[1, 2, 3]], workers=a.address),
-                      client.scatter([1], workers=c.address)]
+    [x], [y] = yield [
+        client.scatter([[1, 2, 3]], workers=a.address),
+        client.scatter([1], workers=c.address),
+    ]
     z = client.submit(inc, 1, workers=[a.address, c.address])
     yield wait(z)
 
     assert s.tasks[z.key].who_has == {s.workers[c.address]}
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 3)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 3)
 def test_no_valid_workers(client, s, a, b, c):
-    x = client.submit(inc, 1, workers='127.0.0.5:9999')
+    x = client.submit(inc, 1, workers="127.0.0.5:9999")
     while not s.tasks:
         yield gen.sleep(0.01)
 
@@ -130,10 +145,9 @@ def test_no_valid_workers(client, s, a, b, c):
         yield gen.with_timeout(timedelta(milliseconds=50), x)
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 3)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 3)
 def test_no_valid_workers_loose_restrictions(client, s, a, b, c):
-    x = client.submit(inc, 1, workers='127.0.0.5:9999',
-                      allow_other_workers=True)
+    x = client.submit(inc, 1, workers="127.0.0.5:9999", allow_other_workers=True)
 
     result = yield x
     assert result == 2
@@ -158,16 +172,17 @@ def test_retire_workers_empty(s):
 
 @gen_cluster()
 def test_remove_client(s, a, b):
-    s.update_graph(tasks={'x': dumps_task((inc, 1)),
-                          'y': dumps_task((inc, 'x'))},
-                   dependencies={'x': [], 'y': ['x']},
-                   keys=['y'],
-                   client='ident')
+    s.update_graph(
+        tasks={"x": dumps_task((inc, 1)), "y": dumps_task((inc, "x"))},
+        dependencies={"x": [], "y": ["x"]},
+        keys=["y"],
+        client="ident",
+    )
 
     assert s.tasks
     assert s.dependencies
 
-    s.remove_client(client='ident')
+    s.remove_client(client="ident")
 
     assert not s.tasks
     assert not s.dependencies
@@ -177,15 +192,18 @@ def test_remove_client(s, a, b):
 def test_server_listens_to_other_ops(s, a, b):
     with rpc(s.address) as r:
         ident = yield r.identity()
-        assert ident['type'] == 'Scheduler'
-        assert ident['id'].lower().startswith('scheduler')
+        assert ident["type"] == "Scheduler"
+        assert ident["id"].lower().startswith("scheduler")
 
 
 @gen_cluster()
 def test_remove_worker_from_scheduler(s, a, b):
-    dsk = {('x-%d' % i): (inc, i) for i in range(20)}
-    s.update_graph(tasks=valmap(dumps_task, dsk), keys=list(dsk),
-                   dependencies={k: set() for k in dsk})
+    dsk = {("x-%d" % i): (inc, i) for i in range(20)}
+    s.update_graph(
+        tasks=valmap(dumps_task, dsk),
+        keys=list(dsk),
+        dependencies={k: set() for k in dsk},
+    )
 
     assert a.address in s.stream_comms
     s.remove_worker(address=a.address)
@@ -214,7 +232,9 @@ def test_clear_events_worker_removal(s, a, b):
     assert b.address in s.events
 
 
-@gen_cluster(config={"distributed.scheduler.events-cleanup-delay": "10 ms"}, client=True)
+@gen_cluster(
+    config={"distributed.scheduler.events-cleanup-delay": "10 ms"}, client=True
+)
 def test_clear_events_client_removal(c, s, a, b):
     assert c.id in s.events
     s.remove_client(c.id)
@@ -234,47 +254,52 @@ def test_clear_events_client_removal(c, s, a, b):
 @gen_cluster()
 def test_add_worker(s, a, b):
     w = Worker(s.ip, s.port, ncores=3)
-    w.data['x-5'] = 6
-    w.data['y'] = 1
+    w.data["x-5"] = 6
+    w.data["y"] = 1
     yield w
 
-    dsk = {('x-%d' % i): (inc, i) for i in range(10)}
-    s.update_graph(tasks=valmap(dumps_task, dsk), keys=list(dsk), client='client',
-                   dependencies={k: set() for k in dsk})
+    dsk = {("x-%d" % i): (inc, i) for i in range(10)}
+    s.update_graph(
+        tasks=valmap(dumps_task, dsk),
+        keys=list(dsk),
+        client="client",
+        dependencies={k: set() for k in dsk},
+    )
 
-    s.add_worker(address=w.address, keys=list(w.data),
-                 ncores=w.ncores, services=s.services)
+    s.add_worker(
+        address=w.address, keys=list(w.data), ncores=w.ncores, services=s.services
+    )
 
     s.validate_state()
 
     assert w.ip in s.host_info
-    assert s.host_info[w.ip]['addresses'] == {a.address, b.address, w.address}
+    assert s.host_info[w.ip]["addresses"] == {a.address, b.address, w.address}
     yield w._close()
 
 
-@gen_cluster(scheduler_kwargs={'blocked_handlers': ['feed']})
+@gen_cluster(scheduler_kwargs={"blocked_handlers": ["feed"]})
 def test_blocked_handlers_are_respected(s, a, b):
     def func(scheduler):
         return dumps(dict(scheduler.worker_info))
 
     comm = yield connect(s.address)
-    yield comm.write({'op': 'feed',
-                      'function': dumps(func),
-                      'interval': 0.01})
+    yield comm.write({"op": "feed", "function": dumps(func), "interval": 0.01})
 
     response = yield comm.read()
 
-    assert 'exception' in response
-    assert isinstance(response['exception'], ValueError)
-    assert "'feed' handler has been explicitly disallowed" in repr(response['exception'])
+    assert "exception" in response
+    assert isinstance(response["exception"], ValueError)
+    assert "'feed' handler has been explicitly disallowed" in repr(
+        response["exception"]
+    )
 
     yield comm.close()
 
 
 def test_scheduler_init_pulls_blocked_handlers_from_config():
-    with dask.config.set({'distributed.scheduler.blocked-handlers': ['test-handler']}):
+    with dask.config.set({"distributed.scheduler.blocked-handlers": ["test-handler"]}):
         s = Scheduler()
-    assert s.blocked_handlers == ['test-handler']
+    assert s.blocked_handlers == ["test-handler"]
 
 
 @gen_cluster()
@@ -283,9 +308,7 @@ def func(scheduler):
         return dumps(dict(scheduler.worker_info))
 
     comm = yield connect(s.address)
-    yield comm.write({'op': 'feed',
-                      'function': dumps(func),
-                      'interval': 0.01})
+    yield comm.write({"op": "feed", "function": dumps(func), "interval": 0.01})
 
     for i in range(5):
         response = yield comm.read()
@@ -302,32 +325,36 @@ def setup(scheduler):
 
     def func(scheduler, state):
         assert state == 1
-        return 'OK'
+        return "OK"
 
     def teardown(scheduler, state):
-        scheduler.flag = 'done'
+        scheduler.flag = "done"
 
     comm = yield connect(s.address)
-    yield comm.write({'op': 'feed',
-                      'function': dumps(func),
-                      'setup': dumps(setup),
-                      'teardown': dumps(teardown),
-                      'interval': 0.01})
+    yield comm.write(
+        {
+            "op": "feed",
+            "function": dumps(func),
+            "setup": dumps(setup),
+            "teardown": dumps(teardown),
+            "interval": 0.01,
+        }
+    )
 
     for i in range(5):
         response = yield comm.read()
-        assert response == 'OK'
+        assert response == "OK"
 
     yield comm.close()
     start = time()
-    while not hasattr(s, 'flag'):
+    while not hasattr(s, "flag"):
         yield gen.sleep(0.01)
         assert time() - start < 5
 
 
 @gen_cluster()
 def test_feed_large_bytestring(s, a, b):
-    np = pytest.importorskip('numpy')
+    np = pytest.importorskip("numpy")
 
     x = np.ones(10000000)
 
@@ -336,9 +363,7 @@ def func(scheduler):
         return True
 
     comm = yield connect(s.address)
-    yield comm.write({'op': 'feed',
-                      'function': dumps(func),
-                      'interval': 0.05})
+    yield comm.write({"op": "feed", "function": dumps(func), "interval": 0.05})
 
     for i in range(5):
         response = yield comm.read()
@@ -349,22 +374,22 @@ def func(scheduler):
 
 @gen_cluster(client=True)
 def test_delete_data(c, s, a, b):
-    d = yield c.scatter({'x': 1, 'y': 2, 'z': 3})
+    d = yield c.scatter({"x": 1, "y": 2, "z": 3})
 
-    assert {ts.key for ts in s.tasks.values() if ts.who_has} == {'x', 'y', 'z'}
-    assert set(a.data) | set(b.data) == {'x', 'y', 'z'}
-    assert merge(a.data, b.data) == {'x': 1, 'y': 2, 'z': 3}
+    assert {ts.key for ts in s.tasks.values() if ts.who_has} == {"x", "y", "z"}
+    assert set(a.data) | set(b.data) == {"x", "y", "z"}
+    assert merge(a.data, b.data) == {"x": 1, "y": 2, "z": 3}
 
-    del d['x']
-    del d['y']
+    del d["x"]
+    del d["y"]
 
     start = time()
-    while set(a.data) | set(b.data) != {'z'}:
+    while set(a.data) | set(b.data) != {"z"}:
         yield gen.sleep(0.01)
         assert time() < start + 5
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)])
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)])
 def test_delete(c, s, a):
     x = c.submit(inc, 1)
     yield x
@@ -382,33 +407,39 @@ def test_delete(c, s, a):
 def test_filtered_communication(s, a, b):
     c = yield connect(s.address)
     f = yield connect(s.address)
-    yield c.write({'op': 'register-client', 'client': 'c'})
-    yield f.write({'op': 'register-client', 'client': 'f'})
+    yield c.write({"op": "register-client", "client": "c"})
+    yield f.write({"op": "register-client", "client": "f"})
     yield c.read()
     yield f.read()
 
-    assert set(s.client_comms) == {'c', 'f'}
-
-    yield c.write({'op': 'update-graph',
-                   'tasks': {'x': dumps_task((inc, 1)),
-                             'y': dumps_task((inc, 'x'))},
-                   'dependencies': {'x': [], 'y': ['x']},
-                   'client': 'c',
-                   'keys': ['y']})
-
-    yield f.write({'op': 'update-graph',
-                   'tasks': {'x': dumps_task((inc, 1)),
-                             'z': dumps_task((add, 'x', 10))},
-                   'dependencies': {'x': [], 'z': ['x']},
-                   'client': 'f',
-                   'keys': ['z']})
+    assert set(s.client_comms) == {"c", "f"}
+
+    yield c.write(
+        {
+            "op": "update-graph",
+            "tasks": {"x": dumps_task((inc, 1)), "y": dumps_task((inc, "x"))},
+            "dependencies": {"x": [], "y": ["x"]},
+            "client": "c",
+            "keys": ["y"],
+        }
+    )
+
+    yield f.write(
+        {
+            "op": "update-graph",
+            "tasks": {"x": dumps_task((inc, 1)), "z": dumps_task((add, "x", 10))},
+            "dependencies": {"x": [], "z": ["x"]},
+            "client": "f",
+            "keys": ["z"],
+        }
+    )
 
     msg, = yield c.read()
-    assert msg['op'] == 'key-in-memory'
-    assert msg['key'] == 'y'
+    assert msg["op"] == "key-in-memory"
+    assert msg["key"] == "y"
     msg, = yield f.read()
-    assert msg['op'] == 'key-in-memory'
-    assert msg['key'] == 'z'
+    assert msg["op"] == "key-in-memory"
+    assert msg["key"] == "z"
 
 
 def test_dumps_function():
@@ -424,26 +455,28 @@ def test_dumps_function():
 
 def test_dumps_task():
     d = dumps_task((inc, 1))
-    assert set(d) == {'function', 'args'}
+    assert set(d) == {"function", "args"}
 
     f = lambda x, y=2: x + y
-    d = dumps_task((apply, f, (1,), {'y': 10}))
-    assert cloudpickle.loads(d['function'])(1, 2) == 3
-    assert cloudpickle.loads(d['args']) == (1,)
-    assert cloudpickle.loads(d['kwargs']) == {'y': 10}
+    d = dumps_task((apply, f, (1,), {"y": 10}))
+    assert cloudpickle.loads(d["function"])(1, 2) == 3
+    assert cloudpickle.loads(d["args"]) == (1,)
+    assert cloudpickle.loads(d["kwargs"]) == {"y": 10}
 
     d = dumps_task((apply, f, (1,)))
-    assert cloudpickle.loads(d['function'])(1, 2) == 3
-    assert cloudpickle.loads(d['args']) == (1,)
-    assert set(d) == {'function', 'args'}
+    assert cloudpickle.loads(d["function"])(1, 2) == 3
+    assert cloudpickle.loads(d["args"]) == (1,)
+    assert set(d) == {"function", "args"}
 
 
 @gen_cluster()
 def test_ready_remove_worker(s, a, b):
-    s.update_graph(tasks={'x-%d' % i: dumps_task((inc, i)) for i in range(20)},
-                   keys=['x-%d' % i for i in range(20)],
-                   client='client',
-                   dependencies={'x-%d' % i: [] for i in range(20)})
+    s.update_graph(
+        tasks={"x-%d" % i: dumps_task((inc, i)) for i in range(20)},
+        keys=["x-%d" % i for i in range(20)],
+        client="client",
+        dependencies={"x-%d" % i: [] for i in range(20)},
+    )
 
     assert all(len(w.processing) > w.ncores for w in s.workers.values())
 
@@ -472,29 +505,28 @@ def test_restart(c, s, a, b):
 
 @gen_cluster()
 def test_broadcast(s, a, b):
-    result = yield s.broadcast(msg={'op': 'ping'})
-    assert result == {a.address: b'pong', b.address: b'pong'}
+    result = yield s.broadcast(msg={"op": "ping"})
+    assert result == {a.address: b"pong", b.address: b"pong"}
 
-    result = yield s.broadcast(msg={'op': 'ping'}, workers=[a.address])
-    assert result == {a.address: b'pong'}
+    result = yield s.broadcast(msg={"op": "ping"}, workers=[a.address])
+    assert result == {a.address: b"pong"}
 
-    result = yield s.broadcast(msg={'op': 'ping'}, hosts=[a.ip])
-    assert result == {a.address: b'pong', b.address: b'pong'}
+    result = yield s.broadcast(msg={"op": "ping"}, hosts=[a.ip])
+    assert result == {a.address: b"pong", b.address: b"pong"}
 
 
 @gen_cluster(Worker=Nanny)
 def test_broadcast_nanny(s, a, b):
-    result1 = yield s.broadcast(msg={'op': 'identity'}, nanny=True)
-    assert all(d['type'] == 'Nanny' for d in result1.values())
+    result1 = yield s.broadcast(msg={"op": "identity"}, nanny=True)
+    assert all(d["type"] == "Nanny" for d in result1.values())
 
-    result2 = yield s.broadcast(msg={'op': 'identity'},
-                                workers=[a.worker_address],
-                                nanny=True)
+    result2 = yield s.broadcast(
+        msg={"op": "identity"}, workers=[a.worker_address], nanny=True
+    )
     assert len(result2) == 1
-    assert first(result2.values())['id'] == a.id
+    assert first(result2.values())["id"] == a.id
 
-    result3 = yield s.broadcast(msg={'op': 'identity'}, hosts=[a.ip],
-                                nanny=True)
+    result3 = yield s.broadcast(msg={"op": "identity"}, hosts=[a.ip], nanny=True)
     assert result1 == result3
 
 
@@ -502,12 +534,12 @@ def test_broadcast_nanny(s, a, b):
 def test_worker_name():
     s = Scheduler(validate=True)
     s.start(0)
-    w = yield Worker(s.ip, s.port, name='alice')
-    assert s.workers[w.address].name == 'alice'
-    assert s.aliases['alice'] == w.address
+    w = yield Worker(s.ip, s.port, name="alice")
+    assert s.workers[w.address].name == "alice"
+    assert s.aliases["alice"] == w.address
 
     with pytest.raises(ValueError):
-        w2 = yield Worker(s.ip, s.port, name='alice')
+        w2 = yield Worker(s.ip, s.port, name="alice")
         yield w2._close()
 
     yield s.close()
@@ -516,44 +548,51 @@ def test_worker_name():
 
 @gen_test()
 def test_coerce_address():
-    with dask.config.set({'distributed.comm.timeouts.connect': '100ms'}):
+    with dask.config.set({"distributed.comm.timeouts.connect": "100ms"}):
         s = Scheduler(validate=True)
         s.start(0)
         print("scheduler:", s.address, s.listen_address)
-        a = Worker(s.ip, s.port, name='alice')
+        a = Worker(s.ip, s.port, name="alice")
         b = Worker(s.ip, s.port, name=123)
-        c = Worker('127.0.0.1', s.port, name='charlie')
+        c = Worker("127.0.0.1", s.port, name="charlie")
         yield [a, b, c]
 
-        assert s.coerce_address('127.0.0.1:8000') == 'tcp://127.0.0.1:8000'
-        assert s.coerce_address('[::1]:8000') == 'tcp://[::1]:8000'
-        assert s.coerce_address('tcp://127.0.0.1:8000') == 'tcp://127.0.0.1:8000'
-        assert s.coerce_address('tcp://[::1]:8000') == 'tcp://[::1]:8000'
-        assert s.coerce_address('localhost:8000') in ('tcp://127.0.0.1:8000', 'tcp://[::1]:8000')
-        assert s.coerce_address(u'localhost:8000') in ('tcp://127.0.0.1:8000', 'tcp://[::1]:8000')
+        assert s.coerce_address("127.0.0.1:8000") == "tcp://127.0.0.1:8000"
+        assert s.coerce_address("[::1]:8000") == "tcp://[::1]:8000"
+        assert s.coerce_address("tcp://127.0.0.1:8000") == "tcp://127.0.0.1:8000"
+        assert s.coerce_address("tcp://[::1]:8000") == "tcp://[::1]:8000"
+        assert s.coerce_address("localhost:8000") in (
+            "tcp://127.0.0.1:8000",
+            "tcp://[::1]:8000",
+        )
+        assert s.coerce_address(u"localhost:8000") in (
+            "tcp://127.0.0.1:8000",
+            "tcp://[::1]:8000",
+        )
         assert s.coerce_address(a.address) == a.address
         # Aliases
-        assert s.coerce_address('alice') == a.address
+        assert s.coerce_address("alice") == a.address
         assert s.coerce_address(123) == b.address
-        assert s.coerce_address('charlie') == c.address
+        assert s.coerce_address("charlie") == c.address
 
-        assert s.coerce_hostname('127.0.0.1') == '127.0.0.1'
-        assert s.coerce_hostname('alice') == a.ip
+        assert s.coerce_hostname("127.0.0.1") == "127.0.0.1"
+        assert s.coerce_hostname("alice") == a.ip
         assert s.coerce_hostname(123) == b.ip
-        assert s.coerce_hostname('charlie') == c.ip
-        assert s.coerce_hostname('jimmy') == 'jimmy'
+        assert s.coerce_hostname("charlie") == c.ip
+        assert s.coerce_hostname("jimmy") == "jimmy"
 
-        assert s.coerce_address('zzzt:8000', resolve=False) == 'tcp://zzzt:8000'
+        assert s.coerce_address("zzzt:8000", resolve=False) == "tcp://zzzt:8000"
 
         yield s.close()
         yield [w._close() for w in [a, b, c]]
 
 
-@pytest.mark.skipif(sys.platform.startswith('win'),
-                    reason="file descriptors not really a thing")
+@pytest.mark.skipif(
+    sys.platform.startswith("win"), reason="file descriptors not really a thing"
+)
 @gen_cluster(ncores=[])
 def test_file_descriptors_dont_leak(s):
-    psutil = pytest.importorskip('psutil')
+    psutil = pytest.importorskip("psutil")
     proc = psutil.Process()
     before = proc.num_fds()
 
@@ -570,14 +609,18 @@ def test_file_descriptors_dont_leak(s):
 
 @gen_cluster()
 def test_update_graph_culls(s, a, b):
-    s.update_graph(tasks={'x': dumps_task((inc, 1)),
-                          'y': dumps_task((inc, 'x')),
-                          'z': dumps_task((inc, 2))},
-                   keys=['y'],
-                   dependencies={'y': 'x', 'x': [], 'z': []},
-                   client='client')
-    assert 'z' not in s.tasks
-    assert 'z' not in s.dependencies
+    s.update_graph(
+        tasks={
+            "x": dumps_task((inc, 1)),
+            "y": dumps_task((inc, "x")),
+            "z": dumps_task((inc, 2)),
+        },
+        keys=["y"],
+        dependencies={"y": "x", "x": [], "z": []},
+        client="client",
+    )
+    assert "z" not in s.tasks
+    assert "z" not in s.dependencies
 
 
 @gen_cluster(ncores=[])
@@ -613,7 +656,7 @@ def test_story(c, s, a, b):
 @gen_cluster(ncores=[], client=True)
 def test_scatter_no_workers(c, s):
     with pytest.raises(gen.TimeoutError):
-        yield s.scatter(data={'x': 1}, client='alice', timeout=0.1)
+        yield s.scatter(data={"x": 1}, client="alice", timeout=0.1)
 
     start = time()
     with pytest.raises(gen.TimeoutError):
@@ -621,10 +664,9 @@ def test_scatter_no_workers(c, s):
     assert time() < start + 1.5
 
     w = Worker(s.ip, s.port, ncores=3)
-    yield [c.scatter(data={'y': 2}, timeout=5),
-           w._start()]
+    yield [c.scatter(data={"y": 2}, timeout=5), w._start()]
 
-    assert w.data['y'] == 2
+    assert w.data["y"] == 2
     yield w._close()
 
 
@@ -645,13 +687,12 @@ def test_retire_workers(c, s, a, b):
 
     workers = yield s.retire_workers()
     assert list(workers) == [a.address]
-    assert workers[a.address]['ncores'] == a.ncores
+    assert workers[a.address]["ncores"] == a.ncores
     assert list(s.ncores) == [b.address]
 
     assert s.workers_to_close() == []
 
-    assert s.workers[b.address].has_what == {s.tasks[x.key],
-                                             s.tasks[y.key]}
+    assert s.workers[b.address].has_what == {s.tasks[x.key], s.tasks[y.key]}
 
     workers = yield s.retire_workers()
     assert not workers
@@ -671,17 +712,17 @@ def test_retire_workers_n(c, s, a, b):
     yield s.retire_workers(n=0, close_workers=True)
     assert len(s.workers) == 0
 
-    while not (a.status.startswith('clos') and b.status.startswith('clos')):
+    while not (a.status.startswith("clos") and b.status.startswith("clos")):
         yield gen.sleep(0.01)
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 4)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 4)
 def test_workers_to_close(cl, s, *workers):
-    s.task_duration['a'] = 4
-    s.task_duration['b'] = 4
-    s.task_duration['c'] = 1
+    s.task_duration["a"] = 4
+    s.task_duration["b"] = 4
+    s.task_duration["c"] = 1
 
-    futures = cl.map(slowinc, [1, 1, 1], key=['a-4','b-4','c-1'])
+    futures = cl.map(slowinc, [1, 1, 1], key=["a-4", "b-4", "c-1"])
     while sum(len(w.processing) for w in s.workers.values()) < 3:
         yield gen.sleep(0.001)
 
@@ -690,24 +731,26 @@ def test_workers_to_close(cl, s, *workers):
     assert len(wtc) == 1
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 4)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 4)
 def test_workers_to_close_grouped(c, s, *workers):
-    groups = {workers[0].address: 'a', workers[1].address: 'a',
-              workers[2].address: 'b', workers[3].address: 'b'}
+    groups = {
+        workers[0].address: "a",
+        workers[1].address: "a",
+        workers[2].address: "b",
+        workers[3].address: "b",
+    }
 
     def key(ws):
         return groups[ws.address]
 
-    assert (set(s.workers_to_close(key=key))
-            == set(w.address for w in workers))
+    assert set(s.workers_to_close(key=key)) == set(w.address for w in workers)
 
     # Assert that job in one worker blocks closure of group
     future = c.submit(slowinc, 1, delay=0.2, workers=workers[0].address)
     while len(s.rprocessing) < 1:
         yield gen.sleep(0.001)
 
-    assert (set(s.workers_to_close(key=key))
-            == {workers[2].address, workers[3].address})
+    assert set(s.workers_to_close(key=key)) == {workers[2].address, workers[3].address}
 
     del future
 
@@ -719,13 +762,14 @@ def key(ws):
     bv = yield c.scatter("b" * 75, workers=workers[2].address)
     bv2 = yield c.scatter("b" * 75, workers=workers[3].address)
 
-    assert (set(s.workers_to_close(key=key))
-             == {workers[0].address, workers[1].address})
+    assert set(s.workers_to_close(key=key)) == {workers[0].address, workers[1].address}
 
 
 @gen_cluster(client=True)
 def test_retire_workers_no_suspicious_tasks(c, s, a, b):
-    future = c.submit(slowinc, 100, delay=0.5, workers=a.address, allow_other_workers=True)
+    future = c.submit(
+        slowinc, 100, delay=0.5, workers=a.address, allow_other_workers=True
+    )
     yield gen.sleep(0.2)
     yield s.retire_workers(workers=[a.address])
 
@@ -733,15 +777,15 @@ def test_retire_workers_no_suspicious_tasks(c, s, a, b):
 
 
 @slow
-@pytest.mark.skipif(sys.platform.startswith('win'),
-                    reason="file descriptors not really a thing")
-@pytest.mark.skipif(sys.version_info < (3, 6),
-                    reason="intermittent failure")
+@pytest.mark.skipif(
+    sys.platform.startswith("win"), reason="file descriptors not really a thing"
+)
+@pytest.mark.skipif(sys.version_info < (3, 6), reason="intermittent failure")
 @gen_cluster(client=True, ncores=[], timeout=240)
 def test_file_descriptors(c, s):
     yield gen.sleep(0.1)
-    psutil = pytest.importorskip('psutil')
-    da = pytest.importorskip('dask.array')
+    psutil = pytest.importorskip("psutil")
+    da = pytest.importorskip("dask.array")
     proc = psutil.Process()
     num_fds_1 = proc.num_fds()
 
@@ -828,7 +872,7 @@ def test_occupancy_cleardown(c, s, a, b):
 
 
 @nodebug
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 30)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 30)
 def test_balance_many_workers(c, s, *workers):
     futures = c.map(slowinc, range(20), delay=0.2)
     yield wait(futures)
@@ -836,9 +880,9 @@ def test_balance_many_workers(c, s, *workers):
 
 
 @nodebug
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 30)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 30)
 def test_balance_many_workers_2(c, s, *workers):
-    s.extensions['stealing']._pc.callback_time = 100000000
+    s.extensions["stealing"]._pc.callback_time = 100000000
     futures = c.map(slowinc, range(90), delay=0.2)
     yield wait(futures)
     assert {len(w.has_what) for w in s.workers.values()} == {3}
@@ -852,16 +896,15 @@ def test_learn_occupancy_multiple_workers(c, s, a, b):
 
     yield wait(x)
 
-    assert not any(v == 0.5 for w in s.workers.values()
-                   for v in w.processing.values())
+    assert not any(v == 0.5 for w in s.workers.values() for v in w.processing.values())
     s.validate_state()
 
 
 @gen_cluster(client=True)
 def test_include_communication_in_occupancy(c, s, a, b):
-    s.task_duration['slowadd'] = 0.001
-    x = c.submit(mul, b'0', int(BANDWIDTH), workers=a.address)
-    y = c.submit(mul, b'1', int(BANDWIDTH * 1.5), workers=b.address)
+    s.task_duration["slowadd"] = 0.001
+    x = c.submit(mul, b"0", int(BANDWIDTH), workers=a.address)
+    y = c.submit(mul, b"1", int(BANDWIDTH * 1.5), workers=b.address)
 
     z = c.submit(slowadd, x, y, delay=1)
     while z.key not in s.tasks or not s.tasks[z.key].processing_on:
@@ -896,15 +939,15 @@ def test_worker_arrives_with_processing_data(c, s, a, b):
         yield gen.sleep(0.01)
 
     assert s.get_task_status(keys={x.key, y.key, z.key}) == {
-        x.key: 'released',
-        y.key: 'memory',
-        z.key: 'processing',
+        x.key: "released",
+        y.key: "memory",
+        z.key: "processing",
     }
 
     yield w._close()
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)])
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)])
 def test_worker_breaks_and_returns(c, s, a):
     future = c.submit(slowinc, 1, delay=0.1)
     for i in range(10):
@@ -922,7 +965,7 @@ def test_worker_breaks_and_returns(c, s, a):
     assert end - start < 1
 
     states = frequencies(ts.state for ts in s.tasks.values())
-    assert states == {'memory': 1, 'released': 10}
+    assert states == {"memory": 1, "released": 10}
 
 
 @gen_cluster(client=True, ncores=[])
@@ -947,9 +990,9 @@ def test_no_workers_to_memory(c, s):
         yield gen.sleep(0.01)
 
     assert s.get_task_status(keys={x.key, y.key, z.key}) == {
-        x.key: 'released',
-        y.key: 'memory',
-        z.key: 'processing',
+        x.key: "released",
+        y.key: "memory",
+        z.key: "processing",
     }
 
     yield w._close()
@@ -961,12 +1004,12 @@ def test_no_worker_to_memory_restrictions(c, s, a, b):
     y = delayed(slowinc)(x, delay=0.4)
     z = delayed(slowinc)(y, delay=0.4)
 
-    yy, zz = c.persist([y, z], workers={(x, y, z): 'alice'})
+    yy, zz = c.persist([y, z], workers={(x, y, z): "alice"})
 
     while not s.tasks:
         yield gen.sleep(0.01)
 
-    w = Worker(s.ip, s.port, ncores=1, name='alice')
+    w = Worker(s.ip, s.port, ncores=1, name="alice")
     w.put_key_in_memory(y.key, 3)
 
     yield w
@@ -976,9 +1019,9 @@ def test_no_worker_to_memory_restrictions(c, s, a, b):
     yield gen.sleep(0.3)
 
     assert s.get_task_status(keys={x.key, y.key, z.key}) == {
-        x.key: 'released',
-        y.key: 'memory',
-        z.key: 'processing',
+        x.key: "released",
+        y.key: "memory",
+        z.key: "processing",
     }
 
     yield w._close()
@@ -989,9 +1032,9 @@ def f(dask_scheduler=None):
         return dask_scheduler.address
 
     with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
+        with Client(s["address"], loop=loop) as c:
             address = c.run_on_scheduler(f)
-            assert address == s['address']
+            assert address == s["address"]
 
             with pytest.raises(ZeroDivisionError):
                 c.run_on_scheduler(div, 1, 0)
@@ -1046,7 +1089,7 @@ def test_close_nanny(c, s, a, b):
         assert not a.is_alive()
         assert a.pid is None
 
-    while a.status != 'closed':
+    while a.status != "closed":
         yield gen.sleep(0.05)
         assert time() < start + 10
 
@@ -1055,7 +1098,7 @@ def test_close_nanny(c, s, a, b):
 def test_retire_workers_close(c, s, a, b):
     yield s.retire_workers(close_workers=True)
     assert not s.workers
-    while a.status != 'closed' and b.status != 'closed':
+    while a.status != "closed" and b.status != "closed":
         yield gen.sleep(0.01)
 
 
@@ -1067,7 +1110,7 @@ def test_retire_nannies_close(c, s, a, b):
 
     start = time()
 
-    while any(n.status != 'closed' for n in nannies):
+    while any(n.status != "closed" for n in nannies):
         yield gen.sleep(0.05)
         assert time() < start + 10
 
@@ -1075,16 +1118,15 @@ def test_retire_nannies_close(c, s, a, b):
     assert not s.workers
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 2)])
+@gen_cluster(client=True, ncores=[("127.0.0.1", 2)])
 def test_fifo_submission(c, s, w):
     futures = []
     for i in range(20):
-        future = c.submit(slowinc, i, delay=0.1, key='inc-%02d' % i,
-                          fifo_timeout=0.01)
+        future = c.submit(slowinc, i, delay=0.1, key="inc-%02d" % i, fifo_timeout=0.01)
         futures.append(future)
         yield gen.sleep(0.02)
     yield wait(futures[-1])
-    assert futures[10].status == 'finished'
+    assert futures[10].status == "finished"
 
 
 @gen_test()
@@ -1094,30 +1136,29 @@ def test_scheduler_file():
         s.start(0)
         with open(fn) as f:
             data = json.load(f)
-        assert data['address'] == s.address
+        assert data["address"] == s.address
 
         c = yield Client(scheduler_file=fn, loop=s.loop, asynchronous=True)
     yield s.close()
 
 
-@pytest.mark.xfail(reason='')
+@pytest.mark.xfail(reason="")
 @gen_cluster(client=True, ncores=[])
 def test_non_existent_worker(c, s):
-    with dask.config.set({'distributed.comm.timeouts.connect': '100ms'}):
-        s.add_worker(address='127.0.0.1:5738', ncores=2, nbytes={}, host_info={})
+    with dask.config.set({"distributed.comm.timeouts.connect": "100ms"}):
+        s.add_worker(address="127.0.0.1:5738", ncores=2, nbytes={}, host_info={})
         futures = c.map(inc, range(10))
         yield gen.sleep(0.300)
         assert not s.workers
-        assert all(ts.state == 'no-worker' for ts in s.tasks.values())
+        assert all(ts.state == "no-worker" for ts in s.tasks.values())
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 3)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 3)
 def test_correct_bad_time_estimate(c, s, *workers):
     future = c.submit(slowinc, 1, delay=0)
     yield wait(future)
 
-    futures = [c.submit(slowinc, future, delay=0.1, pure=False)
-               for i in range(20)]
+    futures = [c.submit(slowinc, future, delay=0.1, pure=False) for i in range(20)]
 
     yield gen.sleep(0.5)
 
@@ -1128,39 +1169,40 @@ def test_correct_bad_time_estimate(c, s, *workers):
 
 @gen_test()
 def test_service_hosts():
-    pytest.importorskip('bokeh')
+    pytest.importorskip("bokeh")
     from distributed.bokeh.scheduler import BokehScheduler
 
     port = 0
     for url, expected in [
-            ('tcp://0.0.0.0', ('::', '0.0.0.0')),
-            ('tcp://127.0.0.1', '127.0.0.1'),
-            ('tcp://127.0.0.1:38275', '127.0.0.1')]:
-        services = {('bokeh', port): BokehScheduler}
+        ("tcp://0.0.0.0", ("::", "0.0.0.0")),
+        ("tcp://127.0.0.1", "127.0.0.1"),
+        ("tcp://127.0.0.1:38275", "127.0.0.1"),
+    ]:
+        services = {("bokeh", port): BokehScheduler}
 
         s = Scheduler(services=services)
         yield s.start(url)
 
-        sock = first(s.services['bokeh'].server._http._sockets.values())
+        sock = first(s.services["bokeh"].server._http._sockets.values())
         if isinstance(expected, tuple):
             assert sock.getsockname()[0] in expected
         else:
             assert sock.getsockname()[0] == expected
         yield s.close()
 
-    port = ('127.0.0.1', 0)
-    for url in ['tcp://0.0.0.0', 'tcp://127.0.0.1', 'tcp://127.0.0.1:38275']:
-        services = {('bokeh', port): BokehScheduler}
+    port = ("127.0.0.1", 0)
+    for url in ["tcp://0.0.0.0", "tcp://127.0.0.1", "tcp://127.0.0.1:38275"]:
+        services = {("bokeh", port): BokehScheduler}
 
         s = Scheduler(services=services)
         yield s.start(url)
 
-        sock = first(s.services['bokeh'].server._http._sockets.values())
-        assert sock.getsockname()[0] == '127.0.0.1'
+        sock = first(s.services["bokeh"].server._http._sockets.values())
+        assert sock.getsockname()[0] == "127.0.0.1"
         yield s.close()
 
 
-@gen_cluster(client=True, worker_kwargs={'profile_cycle_interval': 100})
+@gen_cluster(client=True, worker_kwargs={"profile_cycle_interval": 100})
 def test_profile_metadata(c, s, a, b):
     start = time() - 1
     futures = c.map(slowinc, range(10), delay=0.05, workers=a.address)
@@ -1170,12 +1212,12 @@ def test_profile_metadata(c, s, a, b):
     meta = yield s.get_profile_metadata(profile_cycle_interval=0.100)
     now = time() + 1
     assert meta
-    assert all(start < t < now for t, count in meta['counts'])
-    assert all(0 <= count < 30 for t, count in meta['counts'][:4])
-    assert not meta['counts'][-1][1]
+    assert all(start < t < now for t, count in meta["counts"])
+    assert all(0 <= count < 30 for t, count in meta["counts"][:4])
+    assert not meta["counts"][-1][1]
 
 
-@gen_cluster(client=True, worker_kwargs={'profile_cycle_interval': 100})
+@gen_cluster(client=True, worker_kwargs={"profile_cycle_interval": 100})
 def test_profile_metadata_keys(c, s, a, b):
     start = time() - 1
     x = c.map(slowinc, range(10), delay=0.05)
@@ -1183,8 +1225,8 @@ def test_profile_metadata_keys(c, s, a, b):
     yield wait(x + y)
 
     meta = yield s.get_profile_metadata(profile_cycle_interval=0.100)
-    assert set(meta['keys']) == {'slowinc', 'slowdec'}
-    assert len(meta['counts']) == len(meta['keys']['slowinc'])
+    assert set(meta["keys"]) == {"slowinc", "slowdec"}
+    assert len(meta["counts"]) == len(meta["keys"]["slowinc"])
 
 
 @gen_cluster(client=True)
@@ -1198,7 +1240,7 @@ def test_cancel_fire_and_forget(c, s, a, b):
 
     yield gen.sleep(0.05)
     yield future.cancel(force=True)
-    assert future.status == 'cancelled'
+    assert future.status == "cancelled"
     assert not s.tasks
 
 
@@ -1206,10 +1248,10 @@ def test_cancel_fire_and_forget(c, s, a, b):
 def test_log_tasks_during_restart(c, s, a, b):
     future = c.submit(sys.exit, 0)
     yield wait(future)
-    assert 'exit' in str(s.events)
+    assert "exit" in str(s.events)
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 2)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 2)
 def test_reschedule(c, s, a, b):
     yield c.submit(slowinc, -1, delay=0.1)  # learn cost
     x = c.map(slowinc, range(4), delay=0.1)
@@ -1235,18 +1277,19 @@ def test_get_task_status(c, s, a, b):
     yield wait(future)
 
     result = yield a.scheduler.get_task_status(keys=[future.key])
-    assert result == {future.key: 'memory'}
+    assert result == {future.key: "memory"}
 
 
 def test_deque_handler():
     from distributed.scheduler import logger
+
     s = Scheduler()
     deque_handler = s._deque_handler
-    logger.info('foo123')
+    logger.info("foo123")
     assert len(deque_handler.deque) >= 1
     msg = deque_handler.deque[-1]
-    assert 'distributed.scheduler' in deque_handler.format(msg)
-    assert any(msg.msg == 'foo123' for msg in deque_handler.deque)
+    assert "distributed.scheduler" in deque_handler.format(msg)
+    assert any(msg.msg == "foo123" for msg in deque_handler.deque)
 
 
 @gen_cluster(client=True)
@@ -1277,10 +1320,10 @@ def test_retries(c, s, a, b):
 
 
 @pytest.mark.xfail(reason="second worker also errant for some reason")
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 3, timeout=5)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 3, timeout=5)
 def test_mising_data_errant_worker(c, s, w1, w2, w3):
-    with dask.config.set({'distributed.comm.timeouts.connect': '1s'}):
-        np = pytest.importorskip('numpy')
+    with dask.config.set({"distributed.comm.timeouts.connect": "1s"}):
+        np = pytest.importorskip("numpy")
 
         x = c.submit(np.random.random, 10000000, workers=w1.address)
         yield wait(x)
@@ -1295,8 +1338,8 @@ def test_mising_data_errant_worker(c, s, w1, w2, w3):
 
 @gen_cluster(client=True)
 def test_dont_recompute_if_persisted(c, s, a, b):
-    x = delayed(inc)(1, dask_key_name='x')
-    y = delayed(inc)(x, dask_key_name='y')
+    x = delayed(inc)(1, dask_key_name="x")
+    y = delayed(inc)(x, dask_key_name="y")
 
     yy = y.persist()
     yield wait(yy)
@@ -1312,28 +1355,28 @@ def test_dont_recompute_if_persisted(c, s, a, b):
 
 @gen_cluster(client=True)
 def test_dont_recompute_if_persisted_2(c, s, a, b):
-    x = delayed(inc)(1, dask_key_name='x')
-    y = delayed(inc)(x, dask_key_name='y')
-    z = delayed(inc)(y, dask_key_name='z')
+    x = delayed(inc)(1, dask_key_name="x")
+    y = delayed(inc)(x, dask_key_name="y")
+    z = delayed(inc)(y, dask_key_name="z")
 
     yy = y.persist()
     yield wait(yy)
 
-    old = s.story('x', 'y')
+    old = s.story("x", "y")
 
     zz = z.persist()
     yield wait(zz)
 
     yield gen.sleep(0.100)
-    assert s.story('x', 'y') == old
+    assert s.story("x", "y") == old
 
 
 @gen_cluster(client=True)
 def test_dont_recompute_if_persisted_3(c, s, a, b):
-    x = delayed(inc)(1, dask_key_name='x')
-    y = delayed(inc)(2, dask_key_name='y')
-    z = delayed(inc)(y, dask_key_name='z')
-    w = delayed(add)(x, z, dask_key_name='w')
+    x = delayed(inc)(1, dask_key_name="x")
+    y = delayed(inc)(2, dask_key_name="y")
+    z = delayed(inc)(y, dask_key_name="z")
+    w = delayed(add)(x, z, dask_key_name="w")
 
     ww = w.persist()
     yield wait(ww)
@@ -1348,44 +1391,44 @@ def test_dont_recompute_if_persisted_3(c, s, a, b):
 
 @gen_cluster(client=True)
 def test_dont_recompute_if_persisted_4(c, s, a, b):
-    x = delayed(inc)(1, dask_key_name='x')
-    y = delayed(inc)(x, dask_key_name='y')
-    z = delayed(inc)(x, dask_key_name='z')
+    x = delayed(inc)(1, dask_key_name="x")
+    y = delayed(inc)(x, dask_key_name="y")
+    z = delayed(inc)(x, dask_key_name="z")
 
     yy = y.persist()
     yield wait(yy)
 
-    old = s.story('x')
+    old = s.story("x")
 
-    while s.tasks['x'].state == 'memory':
+    while s.tasks["x"].state == "memory":
         yield gen.sleep(0.01)
 
     yyy, zzz = dask.persist(y, z)
     yield wait([yyy, zzz])
 
-    new = s.story('x')
+    new = s.story("x")
     assert len(new) > len(old)
 
 
 @gen_cluster(client=True)
 def test_dont_forget_released_keys(c, s, a, b):
-    x = c.submit(inc, 1, key='x')
-    y = c.submit(inc, x, key='y')
-    z = c.submit(dec, x, key='z')
+    x = c.submit(inc, 1, key="x")
+    y = c.submit(inc, x, key="y")
+    z = c.submit(dec, x, key="z")
     del x
     yield wait([y, z])
     del z
 
-    while 'z' in s.tasks:
+    while "z" in s.tasks:
         yield gen.sleep(0.01)
 
-    assert 'x' in s.tasks
+    assert "x" in s.tasks
 
 
 @gen_cluster(client=True)
 def test_dont_recompute_if_erred(c, s, a, b):
-    x = delayed(inc)(1, dask_key_name='x')
-    y = delayed(div)(x, 0, dask_key_name='y')
+    x = delayed(inc)(1, dask_key_name="x")
+    y = delayed(div)(x, 0, dask_key_name="y")
 
     yy = y.persist()
     yield wait(yy)
@@ -1404,15 +1447,16 @@ def test_closing_scheduler_closes_workers(s, a, b):
     yield s.close()
 
     start = time()
-    while a.status != 'closed' or b.status != 'closed':
+    while a.status != "closed" or b.status != "closed":
         yield gen.sleep(0.01)
         assert time() < start + 2
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)],
-             worker_kwargs={'resources': {'A': 1}})
+@gen_cluster(
+    client=True, ncores=[("127.0.0.1", 1)], worker_kwargs={"resources": {"A": 1}}
+)
 def test_resources_reset_after_cancelled_task(c, s, w):
-    future = c.submit(sleep, 0.2, resources={'A': 1})
+    future = c.submit(sleep, 0.2, resources={"A": 1})
 
     while not w.executing:
         yield gen.sleep(0.01)
@@ -1422,33 +1466,33 @@ def test_resources_reset_after_cancelled_task(c, s, w):
     while w.executing:
         yield gen.sleep(0.01)
 
-    assert not s.workers[w.address].used_resources['A']
-    assert w.available_resources == {'A': 1}
+    assert not s.workers[w.address].used_resources["A"]
+    assert w.available_resources == {"A": 1}
 
-    yield c.submit(inc, 1, resources={'A': 1})
+    yield c.submit(inc, 1, resources={"A": 1})
 
 
 @gen_cluster(client=True)
 def test_gh2187(c, s, a, b):
     def foo():
-        return 'foo'
+        return "foo"
 
     def bar(x):
-        return x + 'bar'
+        return x + "bar"
 
     def baz(x):
-        return x + 'baz'
+        return x + "baz"
 
     def qux(x):
         sleep(0.1)
-        return x + 'qux'
+        return x + "qux"
 
-    w = c.submit(foo, key='w')
-    x = c.submit(bar, w, key='x')
-    y = c.submit(baz, x, key='y')
+    w = c.submit(foo, key="w")
+    x = c.submit(bar, w, key="x")
+    y = c.submit(baz, x, key="y")
     yield y
-    z = c.submit(qux, y, key='z')
+    z = c.submit(qux, y, key="z")
     del y
     yield gen.sleep(0.1)
-    f = c.submit(bar, x, key='y')
+    f = c.submit(bar, x, key="y")
     yield f
diff --git a/distributed/tests/test_security.py b/distributed/tests/test_security.py
index 7ebd414ca24..8e82db1308e 100644
--- a/distributed/tests/test_security.py
+++ b/distributed/tests/test_security.py
@@ -16,21 +16,21 @@
 from distributed.utils_test import new_config, get_cert, gen_test
 
 
-ca_file = get_cert('tls-ca-cert.pem')
+ca_file = get_cert("tls-ca-cert.pem")
 
-cert1 = get_cert('tls-cert.pem')
-key1 = get_cert('tls-key.pem')
-keycert1 = get_cert('tls-key-cert.pem')
+cert1 = get_cert("tls-cert.pem")
+key1 = get_cert("tls-key.pem")
+keycert1 = get_cert("tls-key-cert.pem")
 
 # Note this cipher uses RSA auth as this matches our test certs
-FORCED_CIPHER = 'ECDHE-RSA-AES128-GCM-SHA256'
+FORCED_CIPHER = "ECDHE-RSA-AES128-GCM-SHA256"
 
 TLS_13_CIPHERS = [
-    'TLS_AES_128_GCM_SHA256',
-    'TLS_AES_256_GCM_SHA384',
-    'TLS_CHACHA20_POLY1305_SHA256',
-    'TLS_AES_128_CCM_SHA256',
-    'TLS_AES_128_CCM_8_SHA256',
+    "TLS_AES_128_GCM_SHA256",
+    "TLS_AES_256_GCM_SHA384",
+    "TLS_CHACHA20_POLY1305_SHA256",
+    "TLS_AES_128_CCM_SHA256",
+    "TLS_AES_128_CCM_8_SHA256",
 ]
 
 
@@ -50,7 +50,7 @@ def test_defaults():
 
 def test_attribute_error():
     sec = Security()
-    assert hasattr(sec, 'tls_ca_file')
+    assert hasattr(sec, "tls_ca_file")
     with pytest.raises(AttributeError):
         sec.tls_foobar
     with pytest.raises(AttributeError):
@@ -59,103 +59,93 @@ def test_attribute_error():
 
 def test_from_config():
     c = {
-        'tls': {
-            'ca-file': 'ca.pem',
-            'scheduler': {
-                'key': 'skey.pem',
-                'cert': 'scert.pem',
-            },
-            'worker': {
-                'cert': 'wcert.pem',
-            },
-            'ciphers': FORCED_CIPHER,
+        "tls": {
+            "ca-file": "ca.pem",
+            "scheduler": {"key": "skey.pem", "cert": "scert.pem"},
+            "worker": {"cert": "wcert.pem"},
+            "ciphers": FORCED_CIPHER,
         },
-        'require-encryption': True,
+        "require-encryption": True,
     }
     with new_config(c):
         sec = Security()
     assert sec.require_encryption is True
-    assert sec.tls_ca_file == 'ca.pem'
+    assert sec.tls_ca_file == "ca.pem"
     assert sec.tls_ciphers == FORCED_CIPHER
     assert sec.tls_client_key is None
     assert sec.tls_client_cert is None
-    assert sec.tls_scheduler_key == 'skey.pem'
-    assert sec.tls_scheduler_cert == 'scert.pem'
+    assert sec.tls_scheduler_key == "skey.pem"
+    assert sec.tls_scheduler_cert == "scert.pem"
     assert sec.tls_worker_key is None
-    assert sec.tls_worker_cert == 'wcert.pem'
+    assert sec.tls_worker_cert == "wcert.pem"
 
 
 def test_kwargs():
     c = {
-        'tls': {
-            'ca-file': 'ca.pem',
-            'scheduler': {
-                'key': 'skey.pem',
-                'cert': 'scert.pem',
-            },
-        },
+        "tls": {
+            "ca-file": "ca.pem",
+            "scheduler": {"key": "skey.pem", "cert": "scert.pem"},
+        }
     }
     with new_config(c):
-        sec = Security(tls_scheduler_cert='newcert.pem',
-                       require_encryption=True,
-                       tls_ca_file=None)
+        sec = Security(
+            tls_scheduler_cert="newcert.pem", require_encryption=True, tls_ca_file=None
+        )
     assert sec.require_encryption is True
     # None value didn't override default
-    assert sec.tls_ca_file == 'ca.pem'
+    assert sec.tls_ca_file == "ca.pem"
     assert sec.tls_ciphers is None
     assert sec.tls_client_key is None
     assert sec.tls_client_cert is None
-    assert sec.tls_scheduler_key == 'skey.pem'
-    assert sec.tls_scheduler_cert == 'newcert.pem'
+    assert sec.tls_scheduler_key == "skey.pem"
+    assert sec.tls_scheduler_cert == "newcert.pem"
     assert sec.tls_worker_key is None
     assert sec.tls_worker_cert is None
 
 
 def test_repr():
     with new_config({}):
-        sec = Security(tls_ca_file='ca.pem', tls_scheduler_cert='scert.pem')
-        assert repr(sec) == "Security(tls_ca_file='ca.pem', tls_scheduler_cert='scert.pem')"
+        sec = Security(tls_ca_file="ca.pem", tls_scheduler_cert="scert.pem")
+        assert (
+            repr(sec)
+            == "Security(tls_ca_file='ca.pem', tls_scheduler_cert='scert.pem')"
+        )
 
 
 def test_tls_config_for_role():
     c = {
-        'tls': {
-            'ca-file': 'ca.pem',
-            'scheduler': {
-                'key': 'skey.pem',
-                'cert': 'scert.pem',
-            },
-            'worker': {
-                'cert': 'wcert.pem',
-            },
-            'ciphers': FORCED_CIPHER,
-        },
+        "tls": {
+            "ca-file": "ca.pem",
+            "scheduler": {"key": "skey.pem", "cert": "scert.pem"},
+            "worker": {"cert": "wcert.pem"},
+            "ciphers": FORCED_CIPHER,
+        }
     }
     with new_config(c):
         sec = Security()
-    t = sec.get_tls_config_for_role('scheduler')
+    t = sec.get_tls_config_for_role("scheduler")
     assert t == {
-        'ca_file': 'ca.pem',
-        'key': 'skey.pem',
-        'cert': 'scert.pem',
-        'ciphers': FORCED_CIPHER,
+        "ca_file": "ca.pem",
+        "key": "skey.pem",
+        "cert": "scert.pem",
+        "ciphers": FORCED_CIPHER,
     }
-    t = sec.get_tls_config_for_role('worker')
+    t = sec.get_tls_config_for_role("worker")
     assert t == {
-        'ca_file': 'ca.pem',
-        'key': None,
-        'cert': 'wcert.pem',
-        'ciphers': FORCED_CIPHER,
+        "ca_file": "ca.pem",
+        "key": None,
+        "cert": "wcert.pem",
+        "ciphers": FORCED_CIPHER,
     }
-    t = sec.get_tls_config_for_role('client')
+    t = sec.get_tls_config_for_role("client")
     assert t == {
-        'ca_file': 'ca.pem',
-        'key': None,
-        'cert': None,
-        'ciphers': FORCED_CIPHER,
+        "ca_file": "ca.pem",
+        "key": None,
+        "cert": None,
+        "ciphers": FORCED_CIPHER,
     }
     with pytest.raises(ValueError):
-        sec.get_tls_config_for_role('supervisor')
+        sec.get_tls_config_for_role("supervisor")
 
 
 def test_connection_args():
@@ -168,51 +158,46 @@ def many_ciphers(ctx):
             assert len(ctx.get_ciphers()) > 2  # Most likely
 
     c = {
-        'tls': {
-            'ca-file': ca_file,
-            'scheduler': {
-                'key': key1,
-                'cert': cert1,
-            },
-            'worker': {
-                'cert': keycert1,
-            },
-        },
+        "tls": {
+            "ca-file": ca_file,
+            "scheduler": {"key": key1, "cert": cert1},
+            "worker": {"cert": keycert1},
+        }
     }
     with new_config(c):
         sec = Security()
 
-    d = sec.get_connection_args('scheduler')
-    assert not d['require_encryption']
-    ctx = d['ssl_context']
+    d = sec.get_connection_args("scheduler")
+    assert not d["require_encryption"]
+    ctx = d["ssl_context"]
     basic_checks(ctx)
     many_ciphers(ctx)
 
-    d = sec.get_connection_args('worker')
-    ctx = d['ssl_context']
+    d = sec.get_connection_args("worker")
+    ctx = d["ssl_context"]
     basic_checks(ctx)
     many_ciphers(ctx)
 
     # No cert defined => no TLS
-    d = sec.get_connection_args('client')
-    assert d.get('ssl_context') is None
+    d = sec.get_connection_args("client")
+    assert d.get("ssl_context") is None
 
     # With more settings
-    c['tls']['ciphers'] = FORCED_CIPHER
-    c['require-encryption'] = True
+    c["tls"]["ciphers"] = FORCED_CIPHER
+    c["require-encryption"] = True
 
     with new_config(c):
         sec = Security()
 
-    d = sec.get_listen_args('scheduler')
-    assert d['require_encryption']
-    ctx = d['ssl_context']
+    d = sec.get_listen_args("scheduler")
+    assert d["require_encryption"]
+    ctx = d["ssl_context"]
     basic_checks(ctx)
     if sys.version_info >= (3, 6):
         supported_ciphers = ctx.get_ciphers()
-        tls_12_ciphers = [c for c in supported_ciphers if c['protocol'] == 'TLSv1.2']
+        tls_12_ciphers = [c for c in supported_ciphers if c["protocol"] == "TLSv1.2"]
         assert len(tls_12_ciphers) == 1
-        tls_13_ciphers = [c for c in supported_ciphers if c['protocol'] == 'TLSv1.3']
+        tls_13_ciphers = [c for c in supported_ciphers if c["protocol"] == "TLSv1.3"]
         if len(tls_13_ciphers):
             assert len(tls_13_ciphers) == 3
 
@@ -227,51 +212,46 @@ def many_ciphers(ctx):
             assert len(ctx.get_ciphers()) > 2  # Most likely
 
     c = {
-        'tls': {
-            'ca-file': ca_file,
-            'scheduler': {
-                'key': key1,
-                'cert': cert1,
-            },
-            'worker': {
-                'cert': keycert1,
-            },
-        },
+        "tls": {
+            "ca-file": ca_file,
+            "scheduler": {"key": key1, "cert": cert1},
+            "worker": {"cert": keycert1},
+        }
     }
     with new_config(c):
         sec = Security()
 
-    d = sec.get_listen_args('scheduler')
-    assert not d['require_encryption']
-    ctx = d['ssl_context']
+    d = sec.get_listen_args("scheduler")
+    assert not d["require_encryption"]
+    ctx = d["ssl_context"]
     basic_checks(ctx)
     many_ciphers(ctx)
 
-    d = sec.get_listen_args('worker')
-    ctx = d['ssl_context']
+    d = sec.get_listen_args("worker")
+    ctx = d["ssl_context"]
     basic_checks(ctx)
     many_ciphers(ctx)
 
     # No cert defined => no TLS
-    d = sec.get_listen_args('client')
-    assert d.get('ssl_context') is None
+    d = sec.get_listen_args("client")
+    assert d.get("ssl_context") is None
 
     # With more settings
-    c['tls']['ciphers'] = FORCED_CIPHER
-    c['require-encryption'] = True
+    c["tls"]["ciphers"] = FORCED_CIPHER
+    c["require-encryption"] = True
 
     with new_config(c):
         sec = Security()
 
-    d = sec.get_listen_args('scheduler')
-    assert d['require_encryption']
-    ctx = d['ssl_context']
+    d = sec.get_listen_args("scheduler")
+    assert d["require_encryption"]
+    ctx = d["ssl_context"]
     basic_checks(ctx)
     if sys.version_info >= (3, 6):
         supported_ciphers = ctx.get_ciphers()
-        tls_12_ciphers = [c for c in supported_ciphers if c['protocol'] == 'TLSv1.2']
+        tls_12_ciphers = [c for c in supported_ciphers if c["protocol"] == "TLSv1.2"]
         assert len(tls_12_ciphers) == 1
-        tls_13_ciphers = [c for c in supported_ciphers if c['protocol'] == 'TLSv1.3']
+        tls_13_ciphers = [c for c in supported_ciphers if c["protocol"] == "TLSv1.3"]
         if len(tls_13_ciphers):
             assert len(tls_13_ciphers) == 3
 
@@ -281,49 +261,51 @@ def test_tls_listen_connect():
     """
     Functional test for TLS connection args.
     """
+
     @gen.coroutine
     def handle_comm(comm):
         peer_addr = comm.peer_address
-        assert peer_addr.startswith('tls://')
-        yield comm.write('hello')
+        assert peer_addr.startswith("tls://")
+        yield comm.write("hello")
         yield comm.close()
 
     c = {
-        'tls': {
-            'ca-file': ca_file,
-            'scheduler': {
-                'key': key1,
-                'cert': cert1,
-            },
-            'worker': {
-                'cert': keycert1,
-            },
-        },
+        "tls": {
+            "ca-file": ca_file,
+            "scheduler": {"key": key1, "cert": cert1},
+            "worker": {"cert": keycert1},
+        }
     }
     with new_config(c):
         sec = Security()
 
-    c['tls']['ciphers'] = FORCED_CIPHER
+    c["tls"]["ciphers"] = FORCED_CIPHER
     with new_config(c):
         forced_cipher_sec = Security()
 
-    with listen('tls://', handle_comm,
-                connection_args=sec.get_listen_args('scheduler')) as listener:
-        comm = yield connect(listener.contact_address,
-                             connection_args=sec.get_connection_args('worker'))
+    with listen(
+        "tls://", handle_comm, connection_args=sec.get_listen_args("scheduler")
+    ) as listener:
+        comm = yield connect(
+            listener.contact_address, connection_args=sec.get_connection_args("worker")
+        )
         msg = yield comm.read()
-        assert msg == 'hello'
+        assert msg == "hello"
         comm.abort()
 
         # No SSL context for client
         with pytest.raises(TypeError):
-            yield connect(listener.contact_address,
-                          connection_args=sec.get_connection_args('client'))
+            yield connect(
+                listener.contact_address,
+                connection_args=sec.get_connection_args("client"),
+            )
 
         # Check forced cipher
-        comm = yield connect(listener.contact_address,
-                             connection_args=forced_cipher_sec.get_connection_args('worker'))
-        cipher, _, _, = comm.extra_info['cipher']
+        comm = yield connect(
+            listener.contact_address,
+            connection_args=forced_cipher_sec.get_connection_args("worker"),
+        )
+        cipher, _, _, = comm.extra_info["cipher"]
         assert cipher in [FORCED_CIPHER] + TLS_13_CIPHERS
         comm.abort()
 
@@ -333,39 +315,41 @@ def test_require_encryption():
     """
     Functional test for "require_encryption" setting.
     """
+
     @gen.coroutine
     def handle_comm(comm):
         comm.abort()
 
     c = {
-        'tls': {
-            'ca-file': ca_file,
-            'scheduler': {
-                'key': key1,
-                'cert': cert1,
-            },
-            'worker': {
-                'cert': keycert1,
-            },
-        },
+        "tls": {
+            "ca-file": ca_file,
+            "scheduler": {"key": key1, "cert": cert1},
+            "worker": {"cert": keycert1},
+        }
     }
     with new_config(c):
         sec = Security()
-    c['require-encryption'] = True
+    c["require-encryption"] = True
     with new_config(c):
         sec2 = Security()
 
-    for listen_addr in ['inproc://', 'tls://']:
-        with listen(listen_addr, handle_comm,
-                    connection_args=sec.get_listen_args('scheduler')) as listener:
-            comm = yield connect(listener.contact_address,
-                                 connection_args=sec2.get_connection_args('worker'))
+    for listen_addr in ["inproc://", "tls://"]:
+        with listen(
+            listen_addr, handle_comm, connection_args=sec.get_listen_args("scheduler")
+        ) as listener:
+            comm = yield connect(
+                listener.contact_address,
+                connection_args=sec2.get_connection_args("worker"),
+            )
             comm.abort()
 
-        with listen(listen_addr, handle_comm,
-                    connection_args=sec2.get_listen_args('scheduler')) as listener:
-            comm = yield connect(listener.contact_address,
-                                 connection_args=sec2.get_connection_args('worker'))
+        with listen(
+            listen_addr, handle_comm, connection_args=sec2.get_listen_args("scheduler")
+        ) as listener:
+            comm = yield connect(
+                listener.contact_address,
+                connection_args=sec2.get_connection_args("worker"),
+            )
             comm.abort()
 
     @contextmanager
@@ -374,17 +358,25 @@ def check_encryption_error():
             yield
         assert "encryption required" in str(excinfo.value)
 
-    for listen_addr in ['tcp://']:
-        with listen(listen_addr, handle_comm,
-                    connection_args=sec.get_listen_args('scheduler')) as listener:
-            comm = yield connect(listener.contact_address,
-                                 connection_args=sec.get_connection_args('worker'))
+    for listen_addr in ["tcp://"]:
+        with listen(
+            listen_addr, handle_comm, connection_args=sec.get_listen_args("scheduler")
+        ) as listener:
+            comm = yield connect(
+                listener.contact_address,
+                connection_args=sec.get_connection_args("worker"),
+            )
             comm.abort()
 
             with pytest.raises(RuntimeError):
-                yield connect(listener.contact_address,
-                              connection_args=sec2.get_connection_args('worker'))
+                yield connect(
+                    listener.contact_address,
+                    connection_args=sec2.get_connection_args("worker"),
+                )
 
         with pytest.raises(RuntimeError):
-            listen(listen_addr, handle_comm,
-                   connection_args=sec2.get_listen_args('scheduler'))
+            listen(
+                listen_addr,
+                handle_comm,
+                connection_args=sec2.get_listen_args("scheduler"),
+            )
diff --git a/distributed/tests/test_steal.py b/distributed/tests/test_steal.py
index caaa939b665..f93022e6d81 100644
--- a/distributed/tests/test_steal.py
+++ b/distributed/tests/test_steal.py
@@ -15,10 +15,15 @@
 from distributed.config import config
 from distributed.metrics import time
 from distributed.scheduler import BANDWIDTH, key_split
-from distributed.utils_test import (slowinc, slowadd, inc, gen_cluster,
-                                    slowidentity, captured_logger)
-from distributed.utils_test import (nodebug_setup_module,
-                                    nodebug_teardown_module)
+from distributed.utils_test import (
+    slowinc,
+    slowadd,
+    inc,
+    gen_cluster,
+    slowidentity,
+    captured_logger,
+)
+from distributed.utils_test import nodebug_setup_module, nodebug_teardown_module
 from distributed.worker import TOTAL_MEMORY
 
 import pytest
@@ -29,10 +34,10 @@
 teardown_module = nodebug_teardown_module
 
 
-@pytest.mark.skipif(not sys.platform.startswith('linux'),
-                    reason="Need 127.0.0.2 to mean localhost")
-@gen_cluster(client=True, ncores=[('127.0.0.1', 2), ('127.0.0.2', 2)],
-             timeout=20)
+@pytest.mark.skipif(
+    not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
+)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 2), ("127.0.0.2", 2)], timeout=20)
 def test_work_stealing(c, s, a, b):
     [x] = yield c._scatter([1], workers=a.address)
     futures = c.map(slowadd, range(50), [x] * 50)
@@ -41,37 +46,40 @@ def test_work_stealing(c, s, a, b):
     assert len(b.data) > 10
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 2)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 2)
 def test_dont_steal_expensive_data_fast_computation(c, s, a, b):
-    np = pytest.importorskip('numpy')
+    np = pytest.importorskip("numpy")
     x = c.submit(np.arange, 1000000, workers=a.address)
     yield wait([x])
     future = c.submit(np.sum, [1], workers=a.address)  # learn that sum is fast
     yield wait([future])
 
-    cheap = [c.submit(np.sum, x, pure=False, workers=a.address,
-                      allow_other_workers=True) for i in range(10)]
+    cheap = [
+        c.submit(np.sum, x, pure=False, workers=a.address, allow_other_workers=True)
+        for i in range(10)
+    ]
     yield wait(cheap)
     assert len(s.who_has[x.key]) == 1
     assert len(b.data) == 0
     assert len(a.data) == 12
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 2)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 2)
 def test_steal_cheap_data_slow_computation(c, s, a, b):
     x = c.submit(slowinc, 100, delay=0.1)  # learn that slowinc is slow
     yield wait(x)
 
-    futures = c.map(slowinc, range(10), delay=0.1, workers=a.address,
-                    allow_other_workers=True)
+    futures = c.map(
+        slowinc, range(10), delay=0.1, workers=a.address, allow_other_workers=True
+    )
     yield wait(futures)
     assert abs(len(a.data) - len(b.data)) <= 5
 
 
 @pytest.mark.avoid_travis
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 2)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 2)
 def test_steal_expensive_data_slow_computation(c, s, a, b):
-    np = pytest.importorskip('numpy')
+    np = pytest.importorskip("numpy")
 
     x = c.submit(slowinc, 100, delay=0.2, workers=a.address)
     yield wait(x)  # learn that slowinc is slow
@@ -86,7 +94,7 @@ def test_steal_expensive_data_slow_computation(c, s, a, b):
     assert b.data  # not empty
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 10)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 10)
 def test_worksteal_many_thieves(c, s, *workers):
     x = c.submit(slowinc, -1, delay=0.1)
     yield x
@@ -102,7 +110,7 @@ def test_worksteal_many_thieves(c, s, *workers):
     assert sum(map(len, s.has_what.values())) < 150
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 2)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 2)
 def test_dont_steal_unknown_functions(c, s, a, b):
     futures = c.map(inc, [1, 2], workers=a.address, allow_other_workers=True)
     yield wait(futures)
@@ -110,20 +118,22 @@ def test_dont_steal_unknown_functions(c, s, a, b):
     assert len(b.data) == 0
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 2)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 2)
 def test_eventually_steal_unknown_functions(c, s, a, b):
-    futures = c.map(slowinc, range(10), delay=0.1,  workers=a.address,
-                    allow_other_workers=True)
+    futures = c.map(
+        slowinc, range(10), delay=0.1, workers=a.address, allow_other_workers=True
+    )
     yield wait(futures)
     assert len(a.data) >= 3
     assert len(b.data) >= 3
 
 
-@pytest.mark.skip(reason='')
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 3)
+@pytest.mark.skip(reason="")
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 3)
 def test_steal_related_tasks(e, s, a, b, c):
-    futures = e.map(slowinc, range(20), delay=0.05, workers=a.address,
-                    allow_other_workers=True)
+    futures = e.map(
+        slowinc, range(20), delay=0.05, workers=a.address, allow_other_workers=True
+    )
 
     yield wait(futures)
 
@@ -135,9 +145,9 @@ def test_steal_related_tasks(e, s, a, b, c):
     assert nearby > 10
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 10, timeout=1000)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 10, timeout=1000)
 def test_dont_steal_fast_tasks(c, s, *workers):
-    np = pytest.importorskip('numpy')
+    np = pytest.importorskip("numpy")
     x = c.submit(np.random.random, 10000000, workers=workers[0].address)
 
     def do_nothing(x, y=None):
@@ -153,7 +163,7 @@ def do_nothing(x, y=None):
     assert len(s.has_what[workers[0].address]) == 1001
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)], timeout=20)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)], timeout=20)
 def test_new_worker_steals(c, s, a):
     yield wait(c.submit(slowinc, 1, delay=0.01))
 
@@ -179,8 +189,9 @@ def test_new_worker_steals(c, s, a):
 def test_work_steal_no_kwargs(c, s, a, b):
     yield wait(c.submit(slowinc, 1, delay=0.05))
 
-    futures = c.map(slowinc, range(100), workers=a.address,
-                    allow_other_workers=True, delay=0.05)
+    futures = c.map(
+        slowinc, range(100), workers=a.address, allow_other_workers=True, delay=0.05
+    )
 
     yield wait(futures)
 
@@ -193,7 +204,7 @@ def test_work_steal_no_kwargs(c, s, a, b):
     assert result == sum(map(inc, range(100)))
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1), ('127.0.0.1', 2)])
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1), ("127.0.0.1", 2)])
 def test_dont_steal_worker_restrictions(c, s, a, b):
     future = c.submit(slowinc, 1, delay=0.10, workers=a.address)
     yield future
@@ -206,7 +217,7 @@ def test_dont_steal_worker_restrictions(c, s, a, b):
     assert len(a.task_state) == 100
     assert len(b.task_state) == 0
 
-    result = s.extensions['stealing'].balance()
+    result = s.extensions["stealing"].balance()
 
     yield gen.sleep(0.1)
 
@@ -214,58 +225,59 @@ def test_dont_steal_worker_restrictions(c, s, a, b):
     assert len(b.task_state) == 0
 
 
-@pytest.mark.skipif(not sys.platform.startswith('linux'),
-                    reason="Need 127.0.0.2 to mean localhost")
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1), ('127.0.0.2', 1)])
+@pytest.mark.skipif(
+    not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
+)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1), ("127.0.0.2", 1)])
 def test_dont_steal_host_restrictions(c, s, a, b):
     future = c.submit(slowinc, 1, delay=0.10, workers=a.address)
     yield future
 
-    futures = c.map(slowinc, range(100), delay=0.1, workers='127.0.0.1')
+    futures = c.map(slowinc, range(100), delay=0.1, workers="127.0.0.1")
     while len(a.task_state) < 10:
         yield gen.sleep(0.01)
     assert len(a.task_state) == 100
     assert len(b.task_state) == 0
 
-    result = s.extensions['stealing'].balance()
+    result = s.extensions["stealing"].balance()
 
     yield gen.sleep(0.1)
     assert len(a.task_state) == 100
     assert len(b.task_state) == 0
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1, {'resources': {'A': 2}}),
-                                  ('127.0.0.1', 1)])
+@gen_cluster(
+    client=True, ncores=[("127.0.0.1", 1, {"resources": {"A": 2}}), ("127.0.0.1", 1)]
+)
 def test_dont_steal_resource_restrictions(c, s, a, b):
     future = c.submit(slowinc, 1, delay=0.10, workers=a.address)
     yield future
 
-    futures = c.map(slowinc, range(100), delay=0.1, resources={'A': 1})
+    futures = c.map(slowinc, range(100), delay=0.1, resources={"A": 1})
     while len(a.task_state) < 10:
         yield gen.sleep(0.01)
     assert len(a.task_state) == 100
     assert len(b.task_state) == 0
 
-    result = s.extensions['stealing'].balance()
+    result = s.extensions["stealing"].balance()
 
     yield gen.sleep(0.1)
     assert len(a.task_state) == 100
     assert len(b.task_state) == 0
 
 
-@pytest.mark.skip(reason='no stealing of resources')
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1, {'resources': {'A': 2}})],
-             timeout=3)
+@pytest.mark.skip(reason="no stealing of resources")
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1, {"resources": {"A": 2}})], timeout=3)
 def test_steal_resource_restrictions(c, s, a):
     future = c.submit(slowinc, 1, delay=0.10, workers=a.address)
     yield future
 
-    futures = c.map(slowinc, range(100), delay=0.2, resources={'A': 1})
+    futures = c.map(slowinc, range(100), delay=0.2, resources={"A": 1})
     while len(a.task_state) < 101:
         yield gen.sleep(0.01)
     assert len(a.task_state) == 101
 
-    b = yield Worker(s.ip, s.port, loop=s.loop, ncores=1, resources={'A': 4})
+    b = yield Worker(s.ip, s.port, loop=s.loop, ncores=1, resources={"A": 4})
 
     start = time()
     while not b.task_state or len(a.task_state) == 101:
@@ -278,14 +290,15 @@ def test_steal_resource_restrictions(c, s, a):
     yield b._close()
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 5, timeout=20)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 5, timeout=20)
 def test_balance_without_dependencies(c, s, *workers):
-    s.extensions['stealing']._pc.callback_time = 20
+    s.extensions["stealing"]._pc.callback_time = 20
 
     def slow(x):
         y = random.random() * 0.1
         sleep(y)
         return y
+
     futures = c.map(slow, range(100))
     yield wait(futures)
 
@@ -293,22 +306,23 @@ def slow(x):
     assert max(durations) / min(durations) < 3
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 4)] * 2)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 4)] * 2)
 def test_dont_steal_executing_tasks(c, s, a, b):
-    futures = c.map(slowinc, range(4), delay=0.1, workers=a.address,
-                    allow_other_workers=True)
+    futures = c.map(
+        slowinc, range(4), delay=0.1, workers=a.address, allow_other_workers=True
+    )
 
     yield wait(futures)
     assert len(a.data) == 4
     assert len(b.data) == 0
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 10)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 10)
 def test_dont_steal_few_saturated_tasks_many_workers(c, s, a, *rest):
-    s.extensions['stealing']._pc.callback_time = 20
-    x = c.submit(mul, b'0', 100000000, workers=a.address)  # 100 MB
+    s.extensions["stealing"]._pc.callback_time = 20
+    x = c.submit(mul, b"0", 100000000, workers=a.address)  # 100 MB
     yield wait(x)
-    s.task_duration['slowidentity'] = 0.2
+    s.task_duration["slowidentity"] = 0.2
 
     futures = [c.submit(slowidentity, x, pure=False, delay=0.2) for i in range(2)]
 
@@ -318,16 +332,18 @@ def test_dont_steal_few_saturated_tasks_many_workers(c, s, a, *rest):
     assert not any(w.task_state for w in rest)
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 10,
-             worker_kwargs={'memory_limit': TOTAL_MEMORY})
+@gen_cluster(
+    client=True,
+    ncores=[("127.0.0.1", 1)] * 10,
+    worker_kwargs={"memory_limit": TOTAL_MEMORY},
+)
 def test_steal_when_more_tasks(c, s, a, *rest):
-    s.extensions['stealing']._pc.callback_time = 20
-    x = c.submit(mul, b'0', 50000000, workers=a.address)  # 50 MB
+    s.extensions["stealing"]._pc.callback_time = 20
+    x = c.submit(mul, b"0", 50000000, workers=a.address)  # 50 MB
     yield wait(x)
-    s.task_duration['slowidentity'] = 0.2
+    s.task_duration["slowidentity"] = 0.2
 
-    futures = [c.submit(slowidentity, x, pure=False, delay=0.2)
-               for i in range(20)]
+    futures = [c.submit(slowidentity, x, pure=False, delay=0.2) for i in range(20)]
 
     start = time()
     while not any(w.task_state for w in rest):
@@ -335,22 +351,20 @@ def test_steal_when_more_tasks(c, s, a, *rest):
         assert time() < start + 1
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 10)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 10)
 def test_steal_more_attractive_tasks(c, s, a, *rest):
-
     def slow2(x):
         sleep(1)
         return x
 
-    s.extensions['stealing']._pc.callback_time = 20
-    x = c.submit(mul, b'0', 100000000, workers=a.address)  # 100 MB
+    s.extensions["stealing"]._pc.callback_time = 20
+    x = c.submit(mul, b"0", 100000000, workers=a.address)  # 100 MB
     yield wait(x)
 
-    s.task_duration['slowidentity'] = 0.2
-    s.task_duration['slow2'] = 1
+    s.task_duration["slowidentity"] = 0.2
+    s.task_duration["slow2"] = 1
 
-    futures = [c.submit(slowidentity, x, pure=False, delay=0.2)
-               for i in range(10)]
+    futures = [c.submit(slowidentity, x, pure=False, delay=0.2) for i in range(10)]
     future = c.submit(slow2, x, priority=-1)
 
     while not any(w.task_state for w in rest):
@@ -365,7 +379,7 @@ def func(x):
 
 
 def assert_balanced(inp, expected, c, s, *workers):
-    steal = s.extensions['stealing']
+    steal = s.extensions["stealing"]
     steal._pc.stop()
 
     counter = itertools.count()
@@ -387,9 +401,15 @@ def assert_balanced(inp, expected, c, s, *workers):
                 dat = 123
             s.task_duration[str(int(t))] = 1
             i = next(counter)
-            f = c.submit(func, dat, key='%d-%d' % (int(t), i),
-                         workers=w.address, allow_other_workers=True,
-                         pure=False, priority=-i)
+            f = c.submit(
+                func,
+                dat,
+                key="%d-%d" % (int(t), i),
+                workers=w.address,
+                allow_other_workers=True,
+                pure=False,
+                priority=-i,
+            )
             futures.append(f)
 
     while len(s.rprocessing) < len(futures):
@@ -401,90 +421,71 @@ def assert_balanced(inp, expected, c, s, *workers):
         while steal.in_flight:
             yield gen.sleep(0.001)
 
-        result = [sorted([int(key_split(k)) for k in s.processing[w.address]],
-                         reverse=True)
-                  for w in workers]
+        result = [
+            sorted([int(key_split(k)) for k in s.processing[w.address]], reverse=True)
+            for w in workers
+        ]
 
         result2 = sorted(result, reverse=True)
         expected2 = sorted(expected, reverse=True)
 
-        if config.get('pdb-on-err'):
+        if config.get("pdb-on-err"):
             if result2 != expected2:
                 import pdb
+
                 pdb.set_trace()
 
         if result2 == expected2:
             return
-    raise Exception('Expected: {}; got: {}'.format(str(expected2), str(result2)))
-
-
-@pytest.mark.parametrize('inp,expected', [
-    ([[1], []],  # don't move unnecessarily
-     [[1], []]),
-
-    ([[0, 0], []],  # balance
-     [[0], [0]]),
-
-    ([[0.1, 0.1], []],  # balance even if results in even
-     [[0], [0]]),
-
-    ([[0, 0, 0], []],  # don't over balance
-     [[0, 0], [0]]),
-
-    ([[0, 0], [0, 0, 0], []],  # move from larger
-     [[0, 0], [0, 0], [0]]),
-
-    ([[0, 0, 0], [0], []],  # move to smaller
-     [[0, 0], [0], [0]]),
-
-    ([[0, 1], []],  # choose easier first
-     [[1], [0]]),
-
-    ([[0, 0, 0, 0], [], []],  # spread evenly
-     [[0, 0], [0], [0]]),
-
-    ([[1, 0, 2, 0], [], []],  # move easier
-     [[2, 1], [0], [0]]),
-
-    ([[1, 1, 1], []],  # be willing to move costly items
-     [[1, 1], [1]]),
-
-    ([[1, 1, 1, 1], []],  # but don't move too many
-     [[1, 1, 1], [1]]),
-
-    ([[0, 0], [0, 0], [0, 0], []],  # no one clearly saturated
-     [[0, 0], [0, 0], [0], [0]]),
-
-    ([[4, 2, 2, 2, 2, 1, 1],
-      [4, 2, 1, 1],
-      [],
-      [],
-      []],
-     [[4, 2, 2, 2, 2],
-      [4, 2, 1],
-      [1],
-      [1],
-      [1]]),
-
-    pytest.param([[1, 1, 1, 1, 1, 1, 1], [1, 1], [1, 1], [1, 1], []],
-                 [[1, 1, 1, 1, 1], [1, 1], [1, 1], [1, 1], [1, 1]],
-                 marks=pytest.mark.xfail(reason="Some uncertainty based on executing stolen task"))
-])
+    raise Exception("Expected: {}; got: {}".format(str(expected2), str(result2)))
+
+
+@pytest.mark.parametrize(
+    "inp,expected",
+    [
+        ([[1], []], [[1], []]),  # don't move unnecessarily
+        ([[0, 0], []], [[0], [0]]),  # balance
+        ([[0.1, 0.1], []], [[0], [0]]),  # balance even if results in even
+        ([[0, 0, 0], []], [[0, 0], [0]]),  # don't over balance
+        ([[0, 0], [0, 0, 0], []], [[0, 0], [0, 0], [0]]),  # move from larger
+        ([[0, 0, 0], [0], []], [[0, 0], [0], [0]]),  # move to smaller
+        ([[0, 1], []], [[1], [0]]),  # choose easier first
+        ([[0, 0, 0, 0], [], []], [[0, 0], [0], [0]]),  # spread evenly
+        ([[1, 0, 2, 0], [], []], [[2, 1], [0], [0]]),  # move easier
+        ([[1, 1, 1], []], [[1, 1], [1]]),  # be willing to move costly items
+        ([[1, 1, 1, 1], []], [[1, 1, 1], [1]]),  # but don't move too many
+        (
+            [[0, 0], [0, 0], [0, 0], []],  # no one clearly saturated
+            [[0, 0], [0, 0], [0], [0]],
+        ),
+        (
+            [[4, 2, 2, 2, 2, 1, 1], [4, 2, 1, 1], [], [], []],
+            [[4, 2, 2, 2, 2], [4, 2, 1], [1], [1], [1]],
+        ),
+        pytest.param(
+            [[1, 1, 1, 1, 1, 1, 1], [1, 1], [1, 1], [1, 1], []],
+            [[1, 1, 1, 1, 1], [1, 1], [1, 1], [1, 1], [1, 1]],
+            marks=pytest.mark.xfail(
+                reason="Some uncertainty based on executing stolen task"
+            ),
+        ),
+    ],
+)
 def test_balance(inp, expected):
     test = lambda *args, **kwargs: assert_balanced(inp, expected, *args, **kwargs)
-    test = gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * len(inp))(test)
+    test = gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * len(inp))(test)
     test()
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 2, Worker=Nanny,
-             timeout=20)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 2, Worker=Nanny, timeout=20)
 def test_restart(c, s, a, b):
-    futures = c.map(slowinc, range(100), delay=0.1, workers=a.address,
-                    allow_other_workers=True)
+    futures = c.map(
+        slowinc, range(100), delay=0.1, workers=a.address, allow_other_workers=True
+    )
     while not s.processing[b.worker_address]:
         yield gen.sleep(0.01)
 
-    steal = s.extensions['stealing']
+    steal = s.extensions["stealing"]
     assert any(st for st in steal.stealable_all)
     assert any(x for L in steal.stealable.values() for x in L)
 
@@ -496,14 +497,23 @@ def test_restart(c, s, a, b):
 
 @gen_cluster(client=True)
 def test_steal_communication_heavy_tasks(c, s, a, b):
-    steal = s.extensions['stealing']
-    s.task_duration['slowadd'] = 0.001
-    x = c.submit(mul, b'0', int(BANDWIDTH), workers=a.address)
-    y = c.submit(mul, b'1', int(BANDWIDTH), workers=b.address)
-
-    futures = [c.submit(slowadd, x, y, delay=1, pure=False, workers=a.address,
-                        allow_other_workers=True)
-               for i in range(10)]
+    steal = s.extensions["stealing"]
+    s.task_duration["slowadd"] = 0.001
+    x = c.submit(mul, b"0", int(BANDWIDTH), workers=a.address)
+    y = c.submit(mul, b"1", int(BANDWIDTH), workers=b.address)
+
+    futures = [
+        c.submit(
+            slowadd,
+            x,
+            y,
+            delay=1,
+            pure=False,
+            workers=a.address,
+            allow_other_workers=True,
+        )
+        for i in range(10)
+    ]
 
     while not any(f.key in s.rprocessing for f in futures):
         yield gen.sleep(0.01)
@@ -533,8 +543,10 @@ def test_steal_twice(c, s, a, b):
     has_what = dict(s.has_what)  # take snapshot
     empty_workers = [w for w, keys in has_what.items() if not len(keys)]
     if len(empty_workers) > 2:
-        pytest.fail("Too many workers without keys (%d out of %d)"
-                    % (len(empty_workers), len(has_what)))
+        pytest.fail(
+            "Too many workers without keys (%d out of %d)"
+            % (len(empty_workers), len(has_what))
+        )
     assert max(map(len, has_what.values())) < 30
 
     yield c._close()
@@ -543,20 +555,21 @@ def test_steal_twice(c, s, a, b):
 
 @gen_cluster(client=True)
 def test_dont_steal_executing_tasks(c, s, a, b):
-    steal = s.extensions['stealing']
+    steal = s.extensions["stealing"]
 
     future = c.submit(slowinc, 1, delay=0.5, workers=a.address)
     while not a.executing:
         yield gen.sleep(0.01)
 
-    steal.move_task_request(s.tasks[future.key],
-                            s.workers[a.address], s.workers[b.address])
+    steal.move_task_request(
+        s.tasks[future.key], s.workers[a.address], s.workers[b.address]
+    )
     yield gen.sleep(0.1)
     assert future.key in a.executing
     assert not b.executing
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 2)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 2)
 def test_dont_steal_long_running_tasks(c, s, a, b):
     def long(delay):
         with worker_client() as c:
@@ -565,13 +578,12 @@ def long(delay):
     yield c.submit(long, 0.1)  # learn duration
     yield c.submit(inc, 1)  # learn duration
 
-    long_tasks = c.map(long, [0.5, 0.6], workers=a.address,
-                       allow_other_workers=True)
+    long_tasks = c.map(long, [0.5, 0.6], workers=a.address, allow_other_workers=True)
     while sum(map(len, s.processing.values())) < 2:  # let them start
         yield gen.sleep(0.01)
 
     start = time()
-    while any(t.key in s.extensions['stealing'].key_stealable for t in long_tasks):
+    while any(t.key in s.extensions["stealing"].key_stealable for t in long_tasks):
         yield gen.sleep(0.01)
         assert time() < start + 1
 
@@ -585,21 +597,24 @@ def long(delay):
     yield wait(long_tasks)
 
     for t in long_tasks:
-        assert (sum(log[1] == 'executing' for log in a.story(t)) +
-                sum(log[1] == 'executing' for log in b.story(t))) <= 1
+        assert (
+            sum(log[1] == "executing" for log in a.story(t))
+            + sum(log[1] == "executing" for log in b.story(t))
+        ) <= 1
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 5)] * 2)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 5)] * 2)
 def test_cleanup_repeated_tasks(c, s, a, b):
     class Foo(object):
         pass
 
-    s.extensions['stealing']._pc.callback_time = 20
+    s.extensions["stealing"]._pc.callback_time = 20
     yield c.submit(slowidentity, -1, delay=0.1)
     objects = [c.submit(Foo, pure=False, workers=a.address) for _ in range(50)]
 
-    x = c.map(slowidentity, objects, workers=a.address, allow_other_workers=True,
-              delay=0.05)
+    x = c.map(
+        slowidentity, objects, workers=a.address, allow_other_workers=True, delay=0.05
+    )
     del objects
     yield wait(x)
     assert a.data and b.data
@@ -620,15 +635,21 @@ class Foo(object):
     assert not list(ws)
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 2)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 2)
 def test_lose_task(c, s, a, b):
-    with captured_logger('distributed.stealing') as log:
-        s.periodic_callbacks['stealing'].interval = 1
+    with captured_logger("distributed.stealing") as log:
+        s.periodic_callbacks["stealing"].interval = 1
         for i in range(100):
-            futures = c.map(slowinc, range(10), delay=0.01, pure=False,
-                            workers=a.address, allow_other_workers=True)
+            futures = c.map(
+                slowinc,
+                range(10),
+                delay=0.01,
+                pure=False,
+                workers=a.address,
+                allow_other_workers=True,
+            )
             yield gen.sleep(0.01)
             del futures
 
     out = log.getvalue()
-    assert 'Error' not in out
+    assert "Error" not in out
diff --git a/distributed/tests/test_stress.py b/distributed/tests/test_stress.py
index 50edba30d81..8a36b8b3b94 100644
--- a/distributed/tests/test_stress.py
+++ b/distributed/tests/test_stress.py
@@ -15,9 +15,21 @@
 from distributed.config import config
 from distributed.metrics import time
 from distributed.utils import All
-from distributed.utils_test import (gen_cluster, cluster, inc, slowinc,
-                                    slowadd, slow, slowsum, bump_rlimit)
-from distributed.utils_test import (loop, nodebug_setup_module, nodebug_teardown_module)  # noqa: F401
+from distributed.utils_test import (
+    gen_cluster,
+    cluster,
+    inc,
+    slowinc,
+    slowadd,
+    slow,
+    slowsum,
+    bump_rlimit,
+)
+from distributed.utils_test import (  # noqa: F401
+    loop,
+    nodebug_setup_module,
+    nodebug_teardown_module,
+)
 from distributed.client import wait
 from tornado import gen
 
@@ -29,21 +41,20 @@
 
 @gen_cluster(client=True)
 def test_stress_1(c, s, a, b):
-    n = 2**6
+    n = 2 ** 6
 
     seq = c.map(inc, range(n))
     while len(seq) > 1:
         yield gen.sleep(0.1)
-        seq = [c.submit(add, seq[i], seq[i + 1])
-               for i in range(0, len(seq), 2)]
+        seq = [c.submit(add, seq[i], seq[i + 1]) for i in range(0, len(seq), 2)]
     result = yield seq[0]
     assert result == sum(map(inc, range(n)))
 
 
-@pytest.mark.parametrize(('func', 'n'), [(slowinc, 100), (inc, 1000)])
+@pytest.mark.parametrize(("func", "n"), [(slowinc, 100), (inc, 1000)])
 def test_stress_gc(loop, func, n):
     with cluster() as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
+        with Client(s["address"], loop=loop) as c:
             x = c.submit(func, 1)
             for i in range(n):
                 x = c.submit(func, x)
@@ -51,11 +62,12 @@ def test_stress_gc(loop, func, n):
             assert x.result() == n + 2
 
 
-@pytest.mark.skipif(sys.platform.startswith('win'),
-                    reason="test can leave dangling RPC objects")
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 8, timeout=None)
+@pytest.mark.skipif(
+    sys.platform.startswith("win"), reason="test can leave dangling RPC objects"
+)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 8, timeout=None)
 def test_cancel_stress(c, s, *workers):
-    da = pytest.importorskip('dask.array')
+    da = pytest.importorskip("dask.array")
     x = da.random.random((50, 50), chunks=(2, 2))
     x = c.persist(x)
     yield wait([x])
@@ -69,10 +81,10 @@ def test_cancel_stress(c, s, *workers):
 
 
 def test_cancel_stress_sync(loop):
-    da = pytest.importorskip('dask.array')
+    da = pytest.importorskip("dask.array")
     x = da.random.random((50, 50), chunks=(2, 2))
     with cluster(active_rpc_timeout=10) as (s, [a, b]):
-        with Client(s['address'], loop=loop) as c:
+        with Client(s["address"], loop=loop) as c:
             x = c.persist(x)
             y = (x.sum(axis=0) + x.sum(axis=1) + 1).std()
             wait(x)
@@ -86,7 +98,7 @@ def test_cancel_stress_sync(loop):
 def test_stress_creation_and_deletion(c, s):
     # Assertions are handled by the validate mechanism in the scheduler
     s.allowed_failures = 100000
-    da = pytest.importorskip('dask.array')
+    da = pytest.importorskip("dask.array")
 
     x = da.random.random(size=(2000, 2000), chunks=(100, 100))
     y = (x + 1).T + (x * 2) - x.mean(axis=1)
@@ -105,28 +117,35 @@ def create_and_destroy_worker(delay):
             yield n._close()
             print("Killed nanny")
 
-    yield gen.with_timeout(timedelta(minutes=1),
-                           All([create_and_destroy_worker(0.1 * i) for i in
-                                range(20)]))
+    yield gen.with_timeout(
+        timedelta(minutes=1),
+        All([create_and_destroy_worker(0.1 * i) for i in range(20)]),
+    )
 
 
-@gen_cluster(ncores=[('127.0.0.1', 1)] * 10, client=True, timeout=60)
+@gen_cluster(ncores=[("127.0.0.1", 1)] * 10, client=True, timeout=60)
 def test_stress_scatter_death(c, s, *workers):
     import random
+
     s.allowed_failures = 1000
-    np = pytest.importorskip('numpy')
+    np = pytest.importorskip("numpy")
     L = yield c.scatter([np.random.random(10000) for i in range(len(workers))])
     yield c._replicate(L, n=2)
 
-    adds = [delayed(slowadd, pure=True)(random.choice(L),
-                                        random.choice(L),
-                                        delay=0.05,
-                                        dask_key_name='slowadd-1-%d' % i)
-            for i in range(50)]
-
-    adds = [delayed(slowadd, pure=True)(a, b, delay=0.02,
-                                        dask_key_name='slowadd-2-%d' % i)
-            for i, (a, b) in enumerate(sliding_window(2, adds))]
+    adds = [
+        delayed(slowadd, pure=True)(
+            random.choice(L),
+            random.choice(L),
+            delay=0.05,
+            dask_key_name="slowadd-1-%d" % i,
+        )
+        for i in range(50)
+    ]
+
+    adds = [
+        delayed(slowadd, pure=True)(a, b, delay=0.02, dask_key_name="slowadd-2-%d" % i)
+        for i, (a, b) in enumerate(sliding_window(2, adds))
+    ]
 
     futures = c.compute(adds)
     L = adds = None
@@ -141,8 +160,9 @@ def test_stress_scatter_death(c, s, *workers):
             s.validate_state()
         except Exception as c:
             logger.exception(c)
-            if config.get('log-on-err'):
+            if config.get("log-on-err"):
                 import pdb
+
                 pdb.set_trace()
             else:
                 raise
@@ -153,7 +173,7 @@ def test_stress_scatter_death(c, s, *workers):
     try:
         yield gen.with_timeout(timedelta(seconds=25), c._gather(futures))
     except gen.TimeoutError:
-        ws = {w.address: w for w in workers if w.status != 'closed'}
+        ws = {w.address: w for w in workers if w.status != "closed"}
         print(s.processing)
         print(ws)
         print(futures)
@@ -161,8 +181,9 @@ def test_stress_scatter_death(c, s, *workers):
             worker = [w for w in ws.values() if w.waiting_for_data][0]
         except Exception:
             pass
-        if config.get('log-on-err'):
+        if config.get("log-on-err"):
             import pdb
+
             pdb.set_trace()
         else:
             raise
@@ -178,18 +199,18 @@ def vsum(*args):
 
 @pytest.mark.avoid_travis
 @slow
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 80, timeout=1000)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 80, timeout=1000)
 def test_stress_communication(c, s, *workers):
     s.validate = False  # very slow otherwise
-    da = pytest.importorskip('dask.array')
+    da = pytest.importorskip("dask.array")
     # Test consumes many file descriptors and can hang if the limit is too low
-    resource = pytest.importorskip('resource')
+    resource = pytest.importorskip("resource")
     bump_rlimit(resource.RLIMIT_NOFILE, 8192)
 
     n = 20
     xs = [da.random.random((100, 100), chunks=(5, 5)) for i in range(n)]
     ys = [x + x.T for x in xs]
-    z = da.atop(vsum, 'ij', *concat(zip(ys, ['ij'] * n)), dtype='float64')
+    z = da.atop(vsum, "ij", *concat(zip(ys, ["ij"] * n)), dtype="float64")
 
     future = c.compute(z.sum())
 
@@ -198,7 +219,7 @@ def test_stress_communication(c, s, *workers):
 
 
 @pytest.mark.skip
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 10, timeout=60)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 10, timeout=60)
 def test_stress_steal(c, s, *workers):
     s.validate = False
     for w in workers:
@@ -207,13 +228,12 @@ def test_stress_steal(c, s, *workers):
     dinc = delayed(slowinc)
     L = [delayed(slowinc)(i, delay=0.005) for i in range(100)]
     for i in range(5):
-        L = [delayed(slowsum)(part, delay=0.005)
-             for part in sliding_window(5, L)]
+        L = [delayed(slowsum)(part, delay=0.005) for part in sliding_window(5, L)]
 
     total = delayed(sum)(L)
     future = c.compute(total)
 
-    while future.status != 'finished':
+    while future.status != "finished":
         yield gen.sleep(0.1)
         for i in range(3):
             a = random.choice(workers)
@@ -225,9 +245,9 @@ def test_stress_steal(c, s, *workers):
 
 
 @slow
-@gen_cluster(ncores=[('127.0.0.1', 1)] * 10, client=True, timeout=120)
+@gen_cluster(ncores=[("127.0.0.1", 1)] * 10, client=True, timeout=120)
 def test_close_connections(c, s, *workers):
-    da = pytest.importorskip('dask.array')
+    da = pytest.importorskip("dask.array")
     x = da.random.random(size=(1000, 1000), chunks=(1000, 1))
     for i in range(3):
         x = x.rechunk((1, 1000))
@@ -246,12 +266,14 @@ def test_close_connections(c, s, *workers):
     yield wait(future)
 
 
-@pytest.mark.xfail(reason="IOStream._handle_write blocks on large write_buffer"
-                          " https://github.com/tornadoweb/tornado/issues/2110")
-@gen_cluster(client=True, timeout=20, ncores=[('127.0.0.1', 1)])
+@pytest.mark.xfail(
+    reason="IOStream._handle_write blocks on large write_buffer"
+    " https://github.com/tornadoweb/tornado/issues/2110"
+)
+@gen_cluster(client=True, timeout=20, ncores=[("127.0.0.1", 1)])
 def test_no_delay_during_large_transfer(c, s, w):
-    pytest.importorskip('crick')
-    np = pytest.importorskip('numpy')
+    pytest.importorskip("crick")
+    np = pytest.importorskip("numpy")
     x = np.random.random(100000000)
     x_nbytes = x.nbytes
 
@@ -273,7 +295,7 @@ def test_no_delay_during_large_transfer(c, s, w):
     x = None  # lose ref
 
     for server in [s, w]:
-        assert server.digests['tick-duration'].components[0].max() < 0.5
+        assert server.digests["tick-duration"].components[0].max() < 0.5
 
     nbytes = np.array([t.mem for t in rprof.results])
     nbytes -= nbytes[0]
diff --git a/distributed/tests/test_submit_cli.py b/distributed/tests/test_submit_cli.py
index 7d84ce8cc12..04267a28e2b 100644
--- a/distributed/tests/test_submit_cli.py
+++ b/distributed/tests/test_submit_cli.py
@@ -4,34 +4,40 @@
 from tornado import gen
 from tornado.ioloop import IOLoop
 from distributed.submit import RemoteClient, _submit, _remote
-from distributed.utils_test import (valid_python_script, invalid_python_script, loop)  # noqa: F401
+from distributed.utils_test import (  # noqa: F401
+    valid_python_script,
+    invalid_python_script,
+    loop,
+)
 
 
-def test_dask_submit_cli_writes_result_to_stdout(loop, tmpdir,
-                                                 valid_python_script):
+def test_dask_submit_cli_writes_result_to_stdout(loop, tmpdir, valid_python_script):
     @gen.coroutine
     def test():
-        remote_client = RemoteClient(ip='127.0.0.1', local_dir=str(tmpdir))
+        remote_client = RemoteClient(ip="127.0.0.1", local_dir=str(tmpdir))
         yield remote_client._start()
 
-        out, err = yield _submit('127.0.0.1:{0}'.format(remote_client.port),
-                                 str(valid_python_script))
-        assert b'hello world!' in out
+        out, err = yield _submit(
+            "127.0.0.1:{0}".format(remote_client.port), str(valid_python_script)
+        )
+        assert b"hello world!" in out
         yield remote_client._close()
 
     loop.run_sync(test, timeout=5)
 
 
-def test_dask_submit_cli_writes_traceback_to_stdout(loop, tmpdir,
-                                                    invalid_python_script):
+def test_dask_submit_cli_writes_traceback_to_stdout(
+    loop, tmpdir, invalid_python_script
+):
     @gen.coroutine
     def test():
-        remote_client = RemoteClient(ip='127.0.0.1', local_dir=str(tmpdir))
+        remote_client = RemoteClient(ip="127.0.0.1", local_dir=str(tmpdir))
         yield remote_client._start()
 
-        out, err = yield _submit('127.0.0.1:{0}'.format(remote_client.port),
-                                 str(invalid_python_script))
-        assert b'Traceback' in err
+        out, err = yield _submit(
+            "127.0.0.1:{0}".format(remote_client.port), str(invalid_python_script)
+        )
+        assert b"Traceback" in err
         yield remote_client._close()
 
     loop.run_sync(test, timeout=5)
@@ -41,9 +47,9 @@ def test_cli_runs_remote_client():
     mock_remote_client = Mock(spec=RemoteClient)
     mock_ioloop = Mock(spec=IOLoop.current())
 
-    _remote('127.0.0.1:8799', 8788, loop=mock_ioloop, client=mock_remote_client)
+    _remote("127.0.0.1:8799", 8788, loop=mock_ioloop, client=mock_remote_client)
 
-    mock_remote_client.assert_called_once_with(ip='127.0.0.1', loop=mock_ioloop)
+    mock_remote_client.assert_called_once_with(ip="127.0.0.1", loop=mock_ioloop)
     mock_remote_client().start.assert_called_once_with(port=8799)
 
     assert mock_ioloop.start.called
diff --git a/distributed/tests/test_submit_remote_client.py b/distributed/tests/test_submit_remote_client.py
index d74c3952497..e6527d8319b 100644
--- a/distributed/tests/test_submit_remote_client.py
+++ b/distributed/tests/test_submit_remote_client.py
@@ -4,19 +4,25 @@
 
 from distributed import rpc
 from distributed.submit import RemoteClient
-from distributed.utils_test import (loop, valid_python_script, invalid_python_script)  # noqa: F401
+from distributed.utils_test import (  # noqa: F401
+    loop,
+    valid_python_script,
+    invalid_python_script,
+)
 
 
 def test_remote_client_uploads_a_file(loop, tmpdir):
     @gen.coroutine
     def test():
-        remote_client = RemoteClient(ip='127.0.0.1', local_dir=str(tmpdir))
+        remote_client = RemoteClient(ip="127.0.0.1", local_dir=str(tmpdir))
         yield remote_client._start(0)
         remote_process = rpc(remote_client.address)
-        upload = yield remote_process.upload_file(filename='script.py', file_payload='x=1')
+        upload = yield remote_process.upload_file(
+            filename="script.py", file_payload="x=1"
+        )
 
-        assert upload == {'status': 'OK', 'nbytes': 3}
-        assert tmpdir.join('script.py').read() == "x=1"
+        assert upload == {"status": "OK", "nbytes": 3}
+        assert tmpdir.join("script.py").read() == "x=1"
 
         yield remote_client._close()
 
@@ -26,14 +32,14 @@ def test():
 def test_remote_client_execution_outputs_to_stdout(loop, tmpdir):
     @gen.coroutine
     def test():
-        remote_client = RemoteClient(ip='127.0.0.1', local_dir=str(tmpdir))
+        remote_client = RemoteClient(ip="127.0.0.1", local_dir=str(tmpdir))
         yield remote_client._start(0)
         rr = rpc(remote_client.address)
-        yield rr.upload_file(filename='script.py', file_payload='print("hello world!")')
+        yield rr.upload_file(filename="script.py", file_payload='print("hello world!")')
 
-        message = yield rr.execute(filename='script.py')
-        assert message['stdout'] == b'hello world!' + os.linesep.encode()
-        assert message['returncode'] == 0
+        message = yield rr.execute(filename="script.py")
+        assert message["stdout"] == b"hello world!" + os.linesep.encode()
+        assert message["returncode"] == 0
 
         yield remote_client._close()
 
@@ -43,14 +49,14 @@ def test():
 def test_remote_client_execution_outputs_stderr(loop, tmpdir, invalid_python_script):
     @gen.coroutine
     def test():
-        remote_client = RemoteClient(ip='127.0.0.1', local_dir=str(tmpdir))
+        remote_client = RemoteClient(ip="127.0.0.1", local_dir=str(tmpdir))
         yield remote_client._start(0)
         rr = rpc(remote_client.address)
-        yield rr.upload_file(filename='script.py', file_payload='a+1')
+        yield rr.upload_file(filename="script.py", file_payload="a+1")
 
-        message = yield rr.execute(filename='script.py')
-        assert b'\'a\' is not defined' in message['stderr']
-        assert message['returncode'] == 1
+        message = yield rr.execute(filename="script.py")
+        assert b"'a' is not defined" in message["stderr"]
+        assert message["returncode"] == 1
 
         yield remote_client._close()
 
diff --git a/distributed/tests/test_system_monitor.py b/distributed/tests/test_system_monitor.py
index 9c3e284dd36..f42fb8e3e08 100644
--- a/distributed/tests/test_system_monitor.py
+++ b/distributed/tests/test_system_monitor.py
@@ -18,7 +18,7 @@ def test_SystemMonitor():
     assert all(wb >= 0 for wb in sm.write_bytes)
     assert all(len(q) == 3 for q in sm.quantities.values())
 
-    assert 'cpu' in repr(sm)
+    assert "cpu" in repr(sm)
 
 
 def test_count():
diff --git a/distributed/tests/test_threadpoolexecutor.py b/distributed/tests/test_threadpoolexecutor.py
index 8777e574282..8b807512168 100644
--- a/distributed/tests/test_threadpoolexecutor.py
+++ b/distributed/tests/test_threadpoolexecutor.py
@@ -111,6 +111,7 @@ def f():
 
 def test_rejoin_idempotent():
     with ThreadPoolExecutor(2) as e:
+
         def f():
             secede()
             for i in range(5):
diff --git a/distributed/tests/test_tls_functional.py b/distributed/tests/test_tls_functional.py
index 14f349545ba..6b71941257c 100644
--- a/distributed/tests/test_tls_functional.py
+++ b/distributed/tests/test_tls_functional.py
@@ -10,16 +10,15 @@
 
 from distributed import Nanny, worker_client, Queue
 from distributed.client import wait
-from distributed.utils_test import (gen_tls_cluster, inc, double, slowinc,
-        slowadd)
+from distributed.utils_test import gen_tls_cluster, inc, double, slowinc, slowadd
 
 
 @gen_tls_cluster(client=True)
 def test_Queue(c, s, a, b):
-    assert s.address.startswith('tls://')
+    assert s.address.startswith("tls://")
 
-    x = Queue('x')
-    y = Queue('y')
+    x = Queue("x")
+    y = Queue("y")
 
     size = yield x.qsize()
     assert size == 0
@@ -34,7 +33,7 @@ def test_Queue(c, s, a, b):
 
 @gen_tls_cluster(client=True, timeout=None)
 def test_client_submit(c, s, a, b):
-    assert s.address.startswith('tls://')
+    assert s.address.startswith("tls://")
 
     x = c.submit(inc, 10)
     result = yield x
@@ -49,7 +48,7 @@ def test_client_submit(c, s, a, b):
 
 @gen_tls_cluster(client=True)
 def test_gather(c, s, a, b):
-    assert s.address.startswith('tls://')
+    assert s.address.startswith("tls://")
 
     x = c.submit(inc, 10)
     y = c.submit(inc, x)
@@ -58,29 +57,29 @@ def test_gather(c, s, a, b):
     assert result == 11
     result = yield c._gather([x])
     assert result == [11]
-    result = yield c._gather({'x': x, 'y': [y]})
-    assert result == {'x': 11, 'y': [12]}
+    result = yield c._gather({"x": x, "y": [y]})
+    assert result == {"x": 11, "y": [12]}
 
 
 @gen_tls_cluster(client=True)
 def test_scatter(c, s, a, b):
-    assert s.address.startswith('tls://')
+    assert s.address.startswith("tls://")
 
-    d = yield c._scatter({'y': 20})
-    ts = s.tasks['y']
+    d = yield c._scatter({"y": 20})
+    ts = s.tasks["y"]
     assert ts.who_has
     assert ts.nbytes > 0
-    yy = yield c._gather([d['y']])
+    yy = yield c._gather([d["y"]])
     assert yy == [20]
 
 
 @gen_tls_cluster(client=True, Worker=Nanny)
 def test_nanny(c, s, a, b):
-    assert s.address.startswith('tls://')
+    assert s.address.startswith("tls://")
     for n in [a, b]:
         assert isinstance(n, Nanny)
-        assert n.address.startswith('tls://')
-        assert n.worker_address.startswith('tls://')
+        assert n.address.startswith("tls://")
+        assert n.worker_address.startswith("tls://")
     assert s.ncores == {n.worker_address: n.ncores for n in [a, b]}
 
     x = c.submit(inc, 10)
@@ -100,7 +99,7 @@ def test_rebalance(c, s, a, b):
     assert len(b.data) == 1
 
 
-@gen_tls_cluster(client=True, ncores=[('tls://127.0.0.1', 2)] * 2)
+@gen_tls_cluster(client=True, ncores=[("tls://127.0.0.1", 2)] * 2)
 def test_work_stealing(c, s, a, b):
     [x] = yield c._scatter([1], workers=a.address)
     futures = c.map(slowadd, range(50), [x] * 50, delay=0.1)
@@ -126,12 +125,12 @@ def func(x):
     assert yy == 20 + 1 + (20 + 1) * 2
 
 
-@gen_tls_cluster(client=True, ncores=[('tls://127.0.0.1', 1)] * 2)
+@gen_tls_cluster(client=True, ncores=[("tls://127.0.0.1", 1)] * 2)
 def test_worker_client_gather(c, s, a, b):
     a_address = a.address
     b_address = b.address
-    assert a_address.startswith('tls://')
-    assert b_address.startswith('tls://')
+    assert a_address.startswith("tls://")
+    assert b_address.startswith("tls://")
     assert a_address != b_address
 
     def func():
diff --git a/distributed/tests/test_utils.py b/distributed/tests/test_utils.py
index 45de450fc44..f4423d26e4a 100644
--- a/distributed/tests/test_utils.py
+++ b/distributed/tests/test_utils.py
@@ -16,15 +16,33 @@
 import dask
 from distributed.compatibility import Queue, Empty, isqueue, PY2, Iterator
 from distributed.metrics import time
-from distributed.utils import (All, sync, is_kernel, ensure_ip, str_graph,
-                               truncate_exception, get_traceback, queue_to_iterator,
-                               iterator_to_queue, _maybe_complex, read_block, seek_delimiter,
-                               funcname, ensure_bytes, open_port, get_ip_interface, nbytes,
-                               set_thread_state, thread_state, LoopRunner,
-                               parse_bytes, parse_timedelta, warn_on_duration)
+from distributed.utils import (
+    All,
+    sync,
+    is_kernel,
+    ensure_ip,
+    str_graph,
+    truncate_exception,
+    get_traceback,
+    queue_to_iterator,
+    iterator_to_queue,
+    _maybe_complex,
+    read_block,
+    seek_delimiter,
+    funcname,
+    ensure_bytes,
+    open_port,
+    get_ip_interface,
+    nbytes,
+    set_thread_state,
+    thread_state,
+    LoopRunner,
+    parse_bytes,
+    parse_timedelta,
+    warn_on_duration,
+)
 from distributed.utils_test import loop, loop_in_thread  # noqa: F401
-from distributed.utils_test import (div, has_ipv6, inc, throws, gen_test,
-        captured_logger)
+from distributed.utils_test import div, has_ipv6, inc, throws, gen_test, captured_logger
 
 
 def test_All(loop):
@@ -65,10 +83,10 @@ def test_sync_error(loop_in_thread):
         result = sync(loop, throws, 1)
     except Exception as exc:
         f = exc
-        assert 'hello' in str(exc)
+        assert "hello" in str(exc)
         tb = get_traceback()
         L = traceback.format_tb(tb)
-        assert any('throws' in line for line in L)
+        assert any("throws" in line for line in L)
 
     def function1(x):
         return function2(x)
@@ -79,11 +97,11 @@ def function2(x):
     try:
         result = sync(loop, function1, 1)
     except Exception as exc:
-        assert 'hello' in str(exc)
+        assert "hello" in str(exc)
         tb = get_traceback()
         L = traceback.format_tb(tb)
-        assert any('function1' in line for line in L)
-        assert any('function2' in line for line in L)
+        assert any("function1" in line for line in L)
+        assert any("function2" in line for line in L)
 
 
 def test_sync_timeout(loop_in_thread):
@@ -104,47 +122,47 @@ def test_sync_closed_loop():
 
 
 def test_is_kernel():
-    pytest.importorskip('IPython')
+    pytest.importorskip("IPython")
     assert is_kernel() is False
 
 
-#@pytest.mark.leaking('fds')
-#def test_zzz_leaks(l=[]):
-    #import os, subprocess
-    #l.append(b"x" * (17 * 1024**2))
-    #os.open(__file__, os.O_RDONLY)
-    #subprocess.Popen('sleep 100', shell=True, stdin=subprocess.DEVNULL)
+# @pytest.mark.leaking('fds')
+# def test_zzz_leaks(l=[]):
+# import os, subprocess
+# l.append(b"x" * (17 * 1024**2))
+# os.open(__file__, os.O_RDONLY)
+# subprocess.Popen('sleep 100', shell=True, stdin=subprocess.DEVNULL)
 
 
 def test_ensure_ip():
-    assert ensure_ip('localhost') in ('127.0.0.1', '::1')
-    assert ensure_ip('123.123.123.123') == '123.123.123.123'
-    assert ensure_ip('8.8.8.8') == '8.8.8.8'
+    assert ensure_ip("localhost") in ("127.0.0.1", "::1")
+    assert ensure_ip("123.123.123.123") == "123.123.123.123"
+    assert ensure_ip("8.8.8.8") == "8.8.8.8"
     if has_ipv6():
-        assert ensure_ip('2001:4860:4860::8888') == '2001:4860:4860::8888'
-        assert ensure_ip('::1') == '::1'
+        assert ensure_ip("2001:4860:4860::8888") == "2001:4860:4860::8888"
+        assert ensure_ip("::1") == "::1"
 
 
 def test_get_ip_interface():
-    if sys.platform == 'darwin':
-        assert get_ip_interface('lo0') == '127.0.0.1'
-    elif sys.platform.startswith('linux'):
-        assert get_ip_interface('lo') == '127.0.0.1'
+    if sys.platform == "darwin":
+        assert get_ip_interface("lo0") == "127.0.0.1"
+    elif sys.platform.startswith("linux"):
+        assert get_ip_interface("lo") == "127.0.0.1"
     else:
         pytest.skip("test needs to be enhanced for platform %r" % (sys.platform,))
     with pytest.raises(KeyError):
-        get_ip_interface('__non-existent-interface')
+        get_ip_interface("__non-existent-interface")
 
 
 def test_truncate_exception():
-    e = ValueError('a' * 1000)
+    e = ValueError("a" * 1000)
     assert len(str(e)) >= 1000
     f = truncate_exception(e, 100)
     assert type(f) == type(e)
     assert len(str(f)) < 200
-    assert 'aaaa' in str(f)
+    assert "aaaa" in str(f)
 
-    e = ValueError('a')
+    e = ValueError("a")
     assert truncate_exception(e) is e
 
 
@@ -162,7 +180,7 @@ def c(x):
         c(1)
     except Exception as e:
         tb = get_traceback()
-        assert type(tb).__name__ == 'traceback'
+        assert type(tb).__name__ == "traceback"
 
 
 def test_queue_to_iterator():
@@ -185,20 +203,23 @@ def test_iterator_to_queue():
 
 
 def test_str_graph():
-    dsk = {'x': 1}
+    dsk = {"x": 1}
     assert str_graph(dsk) == dsk
 
-    dsk = {('x', 1): (inc, 1)}
-    assert str_graph(dsk) == {str(('x', 1)): (inc, 1)}
+    dsk = {("x", 1): (inc, 1)}
+    assert str_graph(dsk) == {str(("x", 1)): (inc, 1)}
 
-    dsk = {('x', 1): (inc, 1), ('x', 2): (inc, ('x', 1))}
-    assert str_graph(dsk) == {str(('x', 1)): (inc, 1),
-                              str(('x', 2)): (inc, str(('x', 1)))}
+    dsk = {("x", 1): (inc, 1), ("x", 2): (inc, ("x", 1))}
+    assert str_graph(dsk) == {
+        str(("x", 1)): (inc, 1),
+        str(("x", 2)): (inc, str(("x", 1))),
+    }
 
-    dsks = [{'x': 1},
-            {('x', 1): (inc, 1), ('x', 2): (inc, ('x', 1))},
-            {('x', 1): (sum, [1, 2, 3]),
-             ('x', 2): (sum, [('x', 1), ('x', 1)])}]
+    dsks = [
+        {"x": 1},
+        {("x", 1): (inc, 1), ("x", 2): (inc, ("x", 1))},
+        {("x", 1): (sum, [1, 2, 3]), ("x", 2): (sum, [("x", 1), ("x", 1)])},
+    ]
     for dsk in dsks:
         sdsk = str_graph(dsk)
         keys = list(dsk)
@@ -209,59 +230,58 @@ def test_str_graph():
 
 def test_maybe_complex():
     assert not _maybe_complex(1)
-    assert not _maybe_complex('x')
+    assert not _maybe_complex("x")
     assert _maybe_complex((inc, 1))
     assert _maybe_complex([(inc, 1)])
     assert _maybe_complex([(inc, 1)])
-    assert _maybe_complex({'x': (inc, 1)})
+    assert _maybe_complex({"x": (inc, 1)})
 
 
 def test_read_block():
-    delimiter = b'\n'
-    data = delimiter.join([b'123', b'456', b'789'])
+    delimiter = b"\n"
+    data = delimiter.join([b"123", b"456", b"789"])
     f = io.BytesIO(data)
 
-    assert read_block(f, 1, 2) == b'23'
-    assert read_block(f, 0, 1, delimiter=b'\n') == b'123\n'
-    assert read_block(f, 0, 2, delimiter=b'\n') == b'123\n'
-    assert read_block(f, 0, 3, delimiter=b'\n') == b'123\n'
-    assert read_block(f, 0, 5, delimiter=b'\n') == b'123\n456\n'
-    assert read_block(f, 0, 8, delimiter=b'\n') == b'123\n456\n789'
-    assert read_block(f, 0, 100, delimiter=b'\n') == b'123\n456\n789'
-    assert read_block(f, 1, 1, delimiter=b'\n') == b''
-    assert read_block(f, 1, 5, delimiter=b'\n') == b'456\n'
-    assert read_block(f, 1, 8, delimiter=b'\n') == b'456\n789'
-
-    for ols in [[(0, 3), (3, 3), (6, 3), (9, 2)],
-                [(0, 4), (4, 4), (8, 4)]]:
-        out = [read_block(f, o, l, b'\n') for o, l in ols]
+    assert read_block(f, 1, 2) == b"23"
+    assert read_block(f, 0, 1, delimiter=b"\n") == b"123\n"
+    assert read_block(f, 0, 2, delimiter=b"\n") == b"123\n"
+    assert read_block(f, 0, 3, delimiter=b"\n") == b"123\n"
+    assert read_block(f, 0, 5, delimiter=b"\n") == b"123\n456\n"
+    assert read_block(f, 0, 8, delimiter=b"\n") == b"123\n456\n789"
+    assert read_block(f, 0, 100, delimiter=b"\n") == b"123\n456\n789"
+    assert read_block(f, 1, 1, delimiter=b"\n") == b""
+    assert read_block(f, 1, 5, delimiter=b"\n") == b"456\n"
+    assert read_block(f, 1, 8, delimiter=b"\n") == b"456\n789"
+
+    for ols in [[(0, 3), (3, 3), (6, 3), (9, 2)], [(0, 4), (4, 4), (8, 4)]]:
+        out = [read_block(f, o, l, b"\n") for o, l in ols]
         assert b"".join(filter(None, out)) == data
 
 
 def test_seek_delimiter_endline():
-    f = io.BytesIO(b'123\n456\n789')
+    f = io.BytesIO(b"123\n456\n789")
 
     # if at zero, stay at zero
-    seek_delimiter(f, b'\n', 5)
+    seek_delimiter(f, b"\n", 5)
     assert f.tell() == 0
 
     # choose the first block
     for bs in [1, 5, 100]:
         f.seek(1)
-        seek_delimiter(f, b'\n', blocksize=bs)
+        seek_delimiter(f, b"\n", blocksize=bs)
         assert f.tell() == 4
 
     # handle long delimiters well, even with short blocksizes
-    f = io.BytesIO(b'123abc456abc789')
+    f = io.BytesIO(b"123abc456abc789")
     for bs in [1, 2, 3, 4, 5, 6, 10]:
         f.seek(1)
-        seek_delimiter(f, b'abc', blocksize=bs)
+        seek_delimiter(f, b"abc", blocksize=bs)
         assert f.tell() == 6
 
     # End at the end
-    f = io.BytesIO(b'123\n456')
+    f = io.BytesIO(b"123\n456")
     f.seek(5)
-    seek_delimiter(f, b'\n', 5)
+    seek_delimiter(f, b"\n", 5)
     assert f.tell() == 7
 
 
@@ -269,19 +289,19 @@ def test_funcname():
     def f():
         pass
 
-    assert funcname(f) == 'f'
-    assert funcname(partial(f)) == 'f'
-    assert funcname(partial(partial(f))) == 'f'
+    assert funcname(f) == "f"
+    assert funcname(partial(f)) == "f"
+    assert funcname(partial(partial(f))) == "f"
 
 
 def test_ensure_bytes():
-    data = [b'1', '1', memoryview(b'1'), bytearray(b'1')]
+    data = [b"1", "1", memoryview(b"1"), bytearray(b"1")]
     if PY2:
-        data.append(buffer(b'1'))  # noqa: F821
+        data.append(buffer(b"1"))  # noqa: F821
     for d in data:
         result = ensure_bytes(d)
         assert isinstance(result, bytes)
-        assert result == b'1'
+        assert result == b"1"
 
 
 def test_nbytes():
@@ -289,8 +309,8 @@ def check(obj, expected):
         assert nbytes(obj) == expected
         assert nbytes(memoryview(obj)) == expected
 
-    check(b'123', 3)
-    check(bytearray(b'4567'), 4)
+    check(b"123", 3)
+    check(bytearray(b"4567"), 4)
 
     multi_dim = np.ones(shape=(10, 10))
     scalar = np.array(1)
@@ -302,7 +322,7 @@ def check(obj, expected):
 def test_open_port():
     port = open_port()
     s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    s.bind(('', port))
+    s.bind(("", port))
     s.close()
 
 
@@ -310,7 +330,7 @@ def test_set_thread_state():
     with set_thread_state(x=1):
         assert thread_state.x == 1
 
-    assert not hasattr(thread_state, 'x')
+    assert not hasattr(thread_state, "x")
 
 
 def assert_running(loop):
@@ -474,70 +494,73 @@ def test_loop_runner_gen():
 
 
 def test_parse_bytes():
-    assert parse_bytes('100') == 100
-    assert parse_bytes('100 MB') == 100000000
-    assert parse_bytes('100M') == 100000000
-    assert parse_bytes('5kB') == 5000
-    assert parse_bytes('5.4 kB') == 5400
-    assert parse_bytes('1kiB') == 1024
-    assert parse_bytes('1Mi') == 2**20
-    assert parse_bytes('1e6') == 1000000
-    assert parse_bytes('1e6 kB') == 1000000000
-    assert parse_bytes('MB') == 1000000
+    assert parse_bytes("100") == 100
+    assert parse_bytes("100 MB") == 100000000
+    assert parse_bytes("100M") == 100000000
+    assert parse_bytes("5kB") == 5000
+    assert parse_bytes("5.4 kB") == 5400
+    assert parse_bytes("1kiB") == 1024
+    assert parse_bytes("1Mi") == 2 ** 20
+    assert parse_bytes("1e6") == 1000000
+    assert parse_bytes("1e6 kB") == 1000000000
+    assert parse_bytes("MB") == 1000000
 
 
 def test_parse_timedelta():
-    for text, value in [('1s', 1),
-                        ('100ms', 0.1),
-                        ('5S', 5),
-                        ('5.5s', 5.5),
-                        ('5.5 s', 5.5),
-                        ('1 second', 1),
-                        ('3.3 seconds', 3.3),
-                        ('3.3 milliseconds', 0.0033),
-                        ('3500 us', 0.0035),
-                        ('1 ns', 1e-9),
-                        ('2m', 120),
-                        ('2 minutes', 120),
-                        (datetime.timedelta(seconds=2), 2),
-                        (datetime.timedelta(milliseconds=100), 0.1)]:
+    for text, value in [
+        ("1s", 1),
+        ("100ms", 0.1),
+        ("5S", 5),
+        ("5.5s", 5.5),
+        ("5.5 s", 5.5),
+        ("1 second", 1),
+        ("3.3 seconds", 3.3),
+        ("3.3 milliseconds", 0.0033),
+        ("3500 us", 0.0035),
+        ("1 ns", 1e-9),
+        ("2m", 120),
+        ("2 minutes", 120),
+        (datetime.timedelta(seconds=2), 2),
+        (datetime.timedelta(milliseconds=100), 0.1),
+    ]:
         result = parse_timedelta(text)
         assert abs(result - value) < 1e-14
 
-    assert parse_timedelta('1ms', default='seconds') == 0.001
-    assert parse_timedelta('1', default='seconds') == 1
-    assert parse_timedelta('1', default='ms') == 0.001
-    assert parse_timedelta(1, default='ms') == 0.001
+    assert parse_timedelta("1ms", default="seconds") == 0.001
+    assert parse_timedelta("1", default="seconds") == 1
+    assert parse_timedelta("1", default="ms") == 0.001
+    assert parse_timedelta(1, default="ms") == 0.001
 
 
 @gen_test()
 def test_all_exceptions_logging():
     @gen.coroutine
     def throws():
-        raise Exception('foo1234')
+        raise Exception("foo1234")
 
-    with captured_logger('') as sio:
+    with captured_logger("") as sio:
         try:
-            yield All([throws() for _ in range(5)],
-                      quiet_exceptions=Exception)
+            yield All([throws() for _ in range(5)], quiet_exceptions=Exception)
         except Exception:
             pass
 
-        import gc; gc.collect()
+        import gc
+
+        gc.collect()
         yield gen.sleep(0.1)
 
-    assert 'foo1234' not in sio.getvalue()
+    assert "foo1234" not in sio.getvalue()
 
 
 def test_warn_on_duration():
     with pytest.warns(None) as record:
-        with warn_on_duration('10s', 'foo'):
+        with warn_on_duration("10s", "foo"):
             pass
     assert not record
 
     with pytest.warns(None) as record:
-        with warn_on_duration('1ms', 'foo'):
+        with warn_on_duration("1ms", "foo"):
             sleep(0.100)
 
     assert record
-    assert any('foo' in str(rec.message) for rec in record)
+    assert any("foo" in str(rec.message) for rec in record)
diff --git a/distributed/tests/test_utils_comm.py b/distributed/tests/test_utils_comm.py
index 1e69eef6a03..c9750891dd7 100644
--- a/distributed/tests/test_utils_comm.py
+++ b/distributed/tests/test_utils_comm.py
@@ -4,23 +4,24 @@
 
 from distributed.core import rpc
 from distributed.utils_test import gen_cluster
-from distributed.utils_comm import (pack_data, gather_from_workers)
+from distributed.utils_comm import pack_data, gather_from_workers
 
 
 def test_pack_data():
-    data = {'x': 1}
-    assert pack_data(('x', 'y'), data) == (1, 'y')
-    assert pack_data({'a': 'x', 'b': 'y'}, data) == {'a': 1, 'b': 'y'}
-    assert pack_data({'a': ['x'], 'b': 'y'}, data) == {'a': [1], 'b': 'y'}
+    data = {"x": 1}
+    assert pack_data(("x", "y"), data) == (1, "y")
+    assert pack_data({"a": "x", "b": "y"}, data) == {"a": 1, "b": "y"}
+    assert pack_data({"a": ["x"], "b": "y"}, data) == {"a": [1], "b": "y"}
 
 
-@pytest.mark.xfail(reason='rpc now needs to be a connection pool')
+@pytest.mark.xfail(reason="rpc now needs to be a connection pool")
 @gen_cluster(client=True)
 def test_gather_from_workers_permissive(c, s, a, b):
-    x = yield c.scatter({'x': 1}, workers=a.address)
+    x = yield c.scatter({"x": 1}, workers=a.address)
 
     data, missing, bad_workers = yield gather_from_workers(
-        {'x': [a.address], 'y': [b.address]}, rpc=rpc)
+        {"x": [a.address], "y": [b.address]}, rpc=rpc
+    )
 
-    assert data == {'x': 1}
-    assert list(missing) == ['y']
+    assert data == {"x": 1}
+    assert list(missing) == ["y"]
diff --git a/distributed/tests/test_utils_perf.py b/distributed/tests/test_utils_perf.py
index e8e4cae9e37..55b250273c0 100644
--- a/distributed/tests/test_utils_perf.py
+++ b/distributed/tests/test_utils_perf.py
@@ -10,8 +10,7 @@
 
 from distributed.compatibility import PY2
 from distributed.metrics import thread_time
-from distributed.utils_perf import (FractionalTimer, GCDiagnosis,
-                                    disable_gc_diagnosis)
+from distributed.utils_perf import FractionalTimer, GCDiagnosis, disable_gc_diagnosis
 from distributed.utils_test import captured_logger, run_for
 
 
@@ -43,8 +42,9 @@ def check_fraction(timer, ft):
         # sum of last N "measurement" intervals over the sum of last
         # 2N intervals (not 2N - 1 or 2N + 1)
         actual = ft.running_fraction
-        expected = (sum(timer.durations[1][-N:]) /
-                    (sum(timer.durations[0][-N:] + timer.durations[1][-N:])))
+        expected = sum(timer.durations[1][-N:]) / (
+            sum(timer.durations[0][-N:] + timer.durations[1][-N:])
+        )
         assert actual == pytest.approx(expected)
 
     timer = RandomTimer()
@@ -68,13 +68,12 @@ def check_fraction(timer, ft):
 
 
 @contextlib.contextmanager
-def enable_gc_diagnosis_and_log(diag, level='INFO'):
+def enable_gc_diagnosis_and_log(diag, level="INFO"):
     disable_gc_diagnosis(force=True)  # just in case
     if gc.callbacks:
         print("Unexpected gc.callbacks", gc.callbacks)
 
-    with captured_logger('distributed.utils_perf', level=level,
-                         propagate=False) as sio:
+    with captured_logger("distributed.utils_perf", level=level, propagate=False) as sio:
         gc.disable()
         gc.collect()  # drain any leftover from previous tests
         diag.enable()
@@ -90,7 +89,7 @@ def test_gc_diagnosis_cpu_time():
     diag = GCDiagnosis(warn_over_frac=0.75)
     diag.N_SAMPLES = 3  # shorten tests
 
-    with enable_gc_diagnosis_and_log(diag, level='WARN') as sio:
+    with enable_gc_diagnosis_and_log(diag, level="WARN") as sio:
         # Spend some CPU time doing only full GCs
         for i in range(diag.N_SAMPLES):
             gc.collect()
@@ -99,10 +98,12 @@ def test_gc_diagnosis_cpu_time():
         lines = sio.getvalue().splitlines()
         assert len(lines) == 1
         # Between 80% and 100%
-        assert re.match(r"full garbage collections took (100|[89][0-9])% "
-                        r"CPU time recently", lines[0])
+        assert re.match(
+            r"full garbage collections took (100|[89][0-9])% " r"CPU time recently",
+            lines[0],
+        )
 
-    with enable_gc_diagnosis_and_log(diag, level='WARN') as sio:
+    with enable_gc_diagnosis_and_log(diag, level="WARN") as sio:
         # Spend half the CPU time doing full GCs
         for i in range(diag.N_SAMPLES + 1):
             t1 = thread_time()
@@ -113,7 +114,7 @@ def test_gc_diagnosis_cpu_time():
         assert not sio.getvalue()
 
 
-@pytest.mark.xfail(reason='unknown')
+@pytest.mark.xfail(reason="unknown")
 @pytest.mark.skipif(PY2, reason="requires Python 3")
 def test_gc_diagnosis_rss_win():
     diag = GCDiagnosis(info_over_rss_win=10e6)
@@ -137,5 +138,8 @@ def make_refcycle(nbytes):
         lines = sio.getvalue().splitlines()
         assert len(lines) == 1
         # Several MB released, and at least 1 reference cycles
-        assert re.match(r"full garbage collection released [\d\.]+ MB "
-                        r"from [1-9]\d* reference cycles", lines[0])
+        assert re.match(
+            r"full garbage collection released [\d\.]+ MB "
+            r"from [1-9]\d* reference cycles",
+            lines[0],
+        )
diff --git a/distributed/tests/test_utils_test.py b/distributed/tests/test_utils_test.py
index 84118750595..6f704c23f5b 100755
--- a/distributed/tests/test_utils_test.py
+++ b/distributed/tests/test_utils_test.py
@@ -12,11 +12,22 @@
 from distributed import Scheduler, Worker, Client, config, default_client
 from distributed.core import rpc
 from distributed.metrics import time
-from distributed.utils_test import (cluster, gen_cluster, inc,
-                                    gen_test, wait_for_port, new_config)
-
-from distributed.utils_test import (loop, tls_only_security, # noqa: F401
-                                    security, tls_client, tls_cluster)
+from distributed.utils_test import (  # noqa: F401
+    cluster,
+    gen_cluster,
+    inc,
+    gen_test,
+    wait_for_port,
+    new_config,
+)
+
+from distributed.utils_test import (  # noqa: F401
+    loop,
+    tls_only_security,
+    security,
+    tls_client,
+    tls_cluster,
+)
 from distributed.utils import get_ip
 
 
@@ -27,10 +38,10 @@ def test_bare_cluster(loop):
 
 def test_cluster(loop):
     with cluster() as (s, [a, b]):
-        with rpc(s['address']) as s:
+        with rpc(s["address"]) as s:
             ident = loop.run_sync(s.identity)
-            assert ident['type'] == 'Scheduler'
-            assert len(ident['workers']) == 2
+            assert ident["type"] == "Scheduler"
+            assert len(ident["workers"]) == 2
 
 
 @gen_cluster(client=True)
@@ -45,16 +56,17 @@ def test_gen_cluster(c, s, a, b):
 @pytest.mark.skip(reason="This hangs on travis")
 def test_gen_cluster_cleans_up_client(loop):
     import dask.context
-    assert not dask.config.get('get', None)
+
+    assert not dask.config.get("get", None)
 
     @gen_cluster(client=True)
     def f(c, s, a, b):
-        assert dask.config.get('get', None)
+        assert dask.config.get("get", None)
         yield c.submit(inc, 1)
 
     f()
 
-    assert not dask.config.get('get', None)
+    assert not dask.config.get("get", None)
 
 
 @gen_cluster(client=False)
@@ -65,16 +77,19 @@ def test_gen_cluster_without_client(s, a, b):
     assert s.ncores == {w.address: w.ncores for w in [a, b]}
 
 
-@gen_cluster(client=True, scheduler='tls://127.0.0.1',
-             ncores=[('tls://127.0.0.1', 1), ('tls://127.0.0.1', 2)],
-             security=tls_only_security())
+@gen_cluster(
+    client=True,
+    scheduler="tls://127.0.0.1",
+    ncores=[("tls://127.0.0.1", 1), ("tls://127.0.0.1", 2)],
+    security=tls_only_security(),
+)
 def test_gen_cluster_tls(e, s, a, b):
     assert isinstance(e, Client)
     assert isinstance(s, Scheduler)
-    assert s.address.startswith('tls://')
+    assert s.address.startswith("tls://")
     for w in [a, b]:
         assert isinstance(w, Worker)
-        assert w.address.startswith('tls://')
+        assert w.address.startswith("tls://")
     assert s.ncores == {w.address: w.ncores for w in [a, b]}
 
 
@@ -132,11 +147,11 @@ def test_wait_for_port():
 
 def test_new_config():
     c = config.copy()
-    with new_config({'xyzzy': 5}):
-        config['xyzzy'] == 5
+    with new_config({"xyzzy": 5}):
+        config["xyzzy"] == 5
 
     assert config == c
-    assert 'xyzzy' not in config
+    assert "xyzzy" not in config
 
 
 def test_lingering_client():
@@ -152,7 +167,7 @@ def f(s, a, b):
 
 def test_lingering_client(loop):
     with cluster() as (s, [a, b]):
-        client = Client(s['address'], loop=loop)
+        client = Client(s["address"], loop=loop)
 
 
 def test_tls_cluster(tls_client):
@@ -162,8 +177,8 @@ def test_tls_cluster(tls_client):
 
 def test_tls_scheduler(security, loop):
     s = Scheduler(security=security, loop=loop)
-    s.start('localhost')
-    assert s.address.startswith('tls')
+    s.start("localhost")
+    assert s.address.startswith("tls")
     s.close()
 
 
diff --git a/distributed/tests/test_variable.py b/distributed/tests/test_variable.py
index 0d0898923c4..5ae94d037c5 100644
--- a/distributed/tests/test_variable.py
+++ b/distributed/tests/test_variable.py
@@ -9,14 +9,14 @@
 
 from distributed import Client, Variable, worker_client, Nanny, wait
 from distributed.metrics import time
-from distributed.utils_test import (gen_cluster, inc, slow, div)
-from distributed.utils_test import client, cluster_fixture, loop # noqa: F401
+from distributed.utils_test import gen_cluster, inc, slow, div
+from distributed.utils_test import client, cluster_fixture, loop  # noqa: F401
 
 
 @gen_cluster(client=True)
 def test_variable(c, s, a, b):
-    x = Variable('x')
-    xx = Variable('x')
+    x = Variable("x")
+    xx = Variable("x")
     assert x.client is c
 
     future = c.submit(inc, 1)
@@ -40,20 +40,20 @@ def test_variable(c, s, a, b):
 
 @gen_cluster(client=True)
 def test_queue_with_data(c, s, a, b):
-    x = Variable('x')
-    xx = Variable('x')
+    x = Variable("x")
+    xx = Variable("x")
     assert x.client is c
 
-    yield x.set((1, 'hello'))
+    yield x.set((1, "hello"))
     data = yield xx.get()
 
-    assert data == (1, 'hello')
+    assert data == (1, "hello")
 
 
 def test_sync(client):
     future = client.submit(lambda x: x + 1, 10)
-    x = Variable('x')
-    xx = Variable('x')
+    x = Variable("x")
+    xx = Variable("x")
     x.set(future)
     future2 = xx.get()
 
@@ -64,7 +64,7 @@ def test_sync(client):
 def test_hold_futures(s, a, b):
     c1 = yield Client(s.address, asynchronous=True)
     future = c1.submit(lambda x: x + 1, 10)
-    x1 = Variable('x')
+    x1 = Variable("x")
     yield x1.set(future)
     del x1
     yield c1.close()
@@ -72,7 +72,7 @@ def test_hold_futures(s, a, b):
     yield gen.sleep(0.1)
 
     c2 = yield Client(s.address, asynchronous=True)
-    x2 = Variable('x')
+    x2 = Variable("x")
     future2 = yield x2.get()
     result = yield future2
 
@@ -82,7 +82,7 @@ def test_hold_futures(s, a, b):
 
 @gen_cluster(client=True)
 def test_timeout(c, s, a, b):
-    v = Variable('v')
+    v = Variable("v")
 
     start = time()
     with pytest.raises(gen.TimeoutError):
@@ -92,7 +92,7 @@ def test_timeout(c, s, a, b):
 
 
 def test_timeout_sync(client):
-    v = Variable('v')
+    v = Variable("v")
     start = time()
     with pytest.raises(gen.TimeoutError):
         v.get(timeout=0.1)
@@ -102,8 +102,8 @@ def test_timeout_sync(client):
 
 @gen_cluster(client=True)
 def test_cleanup(c, s, a, b):
-    v = Variable('v')
-    vv = Variable('v')
+    v = Variable("v")
+    vv = Variable("v")
 
     x = c.submit(lambda x: x + 1, 10)
     y = c.submit(lambda x: x + 1, 20)
@@ -124,7 +124,7 @@ def test_cleanup(c, s, a, b):
 
 
 def test_pickleable(client):
-    v = Variable('v')
+    v = Variable("v")
 
     def f(x):
         v.set(x + 1)
@@ -135,27 +135,26 @@ def f(x):
 
 @gen_cluster(client=True)
 def test_timeout_get(c, s, a, b):
-    v = Variable('v')
+    v = Variable("v")
 
     tornado_future = v.get()
 
-    vv = Variable('v')
+    vv = Variable("v")
     yield vv.set(1)
 
     result = yield tornado_future
     assert result == 1
 
 
-@pytest.mark.skipif(sys.version_info[0] == 2, reason='Multi-client issues')
+@pytest.mark.skipif(sys.version_info[0] == 2, reason="Multi-client issues")
 @slow
-@gen_cluster(client=True, ncores=[('127.0.0.1', 2)] * 5, Worker=Nanny,
-             timeout=None)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 2)] * 5, Worker=Nanny, timeout=None)
 def test_race(c, s, *workers):
     NITERS = 50
 
     def f(i):
         with worker_client() as c:
-            v = Variable('x', client=c)
+            v = Variable("x", client=c)
             for _ in range(NITERS):
                 future = v.get()
                 x = future.result()
@@ -166,7 +165,7 @@ def f(i):
             sleep(0.1)  # allow fire-and-forget messages to clear
             return result
 
-    v = Variable('x', client=c)
+    v = Variable("x", client=c)
     x = yield c.scatter(1)
     yield v.set(x)
 
@@ -175,7 +174,7 @@ def f(i):
     assert all(r > NITERS * 0.8 for r in results)
 
     start = time()
-    while len(s.wants_what['variable-x']) != 1:
+    while len(s.wants_what["variable-x"]) != 1:
         yield gen.sleep(0.01)
         assert time() - start < 2
 
@@ -183,20 +182,20 @@ def f(i):
 @gen_cluster(client=True)
 def test_Future_knows_status_immediately(c, s, a, b):
     x = yield c.scatter(123)
-    v = Variable('x')
+    v = Variable("x")
     yield v.set(x)
 
     c2 = yield Client(s.address, asynchronous=True)
-    v2 = Variable('x', client=c2)
+    v2 = Variable("x", client=c2)
     future = yield v2.get()
-    assert future.status == 'finished'
+    assert future.status == "finished"
 
     x = c.submit(div, 1, 0)
     yield wait(x)
     yield v.set(x)
 
     future2 = yield v2.get()
-    assert future2.status == 'error'
+    assert future2.status == "error"
     with pytest.raises(Exception):
         yield future2
 
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 3c9b51f2baa..05b61a997f4 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -19,8 +19,7 @@
 from tornado import gen
 from tornado.ioloop import TimeoutError
 
-from distributed import (Nanny, get_client, wait, default_client,
-        get_worker, Reschedule)
+from distributed import Nanny, get_client, wait, default_client, get_worker, Reschedule
 from distributed.compatibility import WINDOWS, cache_from_source
 from distributed.core import rpc
 from distributed.client import wait
@@ -28,14 +27,32 @@
 from distributed.metrics import time
 from distributed.worker import Worker, error_message, logger, TOTAL_MEMORY
 from distributed.utils import tmpfile, format_bytes
-from distributed.utils_test import (inc, mul, gen_cluster, div, dec,
-                                    slow, slowinc, gen_test, captured_logger)
-from distributed.utils_test import client, loop, nodebug, cluster_fixture, s, a, b  # noqa: F401
+from distributed.utils_test import (
+    inc,
+    mul,
+    gen_cluster,
+    div,
+    dec,
+    slow,
+    slowinc,
+    gen_test,
+    captured_logger,
+)
+from distributed.utils_test import (  # noqa: F401
+    client,
+    loop,
+    nodebug,
+    cluster_fixture,
+    s,
+    a,
+    b,
+)
 
 
 def test_worker_ncores():
     from distributed.worker import _ncores
-    w = Worker('127.0.0.1', 8019)
+
+    w = Worker("127.0.0.1", 8019)
     try:
         assert w.executor._max_workers == _ncores
     finally:
@@ -52,12 +69,12 @@ def test_str(s, a, b):
 
 
 def test_identity():
-    w = Worker('127.0.0.1', 8019)
+    w = Worker("127.0.0.1", 8019)
     ident = w.identity(None)
-    assert 'Worker' in ident['type']
-    assert ident['scheduler'] == 'tcp://127.0.0.1:8019'
-    assert isinstance(ident['ncores'], int)
-    assert isinstance(ident['memory_limit'], Number)
+    assert "Worker" in ident["type"]
+    assert ident["scheduler"] == "tcp://127.0.0.1:8019"
+    assert isinstance(ident["ncores"], int)
+    assert isinstance(ident["memory_limit"], Number)
 
 
 @gen_cluster(client=True)
@@ -91,11 +108,11 @@ def emit(self, record):
 
         def reset(self):
             self.messages = {
-                'debug': [],
-                'info': [],
-                'warning': [],
-                'error': [],
-                'critical': [],
+                "debug": [],
+                "info": [],
+                "warning": [],
+                "error": [],
+                "critical": [],
             }
 
     hdlr = MockLoggingHandler()
@@ -106,7 +123,7 @@ def reset(self):
     yield wait(y)
 
     assert not b.executing
-    assert y.status == 'error'
+    assert y.status == "error"
     # Make sure job died because of bad func and not because of bad
     # argument.
     with pytest.raises(ZeroDivisionError):
@@ -114,10 +131,10 @@ def reset(self):
 
     if sys.version_info[0] >= 3:
         tb = yield y._traceback()
-        assert any('1 / 0' in line
-                   for line in pluck(3, traceback.extract_tb(tb))
-                   if line)
-    assert "Compute Failed" in hdlr.messages['warning'][0]
+        assert any(
+            "1 / 0" in line for line in pluck(3, traceback.extract_tb(tb)) if line
+        )
+    assert "Compute Failed" in hdlr.messages["warning"][0]
     logger.setLevel(old_level)
 
     # Now we check that both workers are still alive.
@@ -133,18 +150,18 @@ def reset(self):
 @slow
 @gen_cluster()
 def dont_test_delete_data_with_missing_worker(c, a, b):
-    bad = '127.0.0.1:9001'  # this worker doesn't exist
-    c.who_has['z'].add(bad)
-    c.who_has['z'].add(a.address)
-    c.has_what[bad].add('z')
-    c.has_what[a.address].add('z')
-    a.data['z'] = 5
+    bad = "127.0.0.1:9001"  # this worker doesn't exist
+    c.who_has["z"].add(bad)
+    c.who_has["z"].add(a.address)
+    c.has_what[bad].add("z")
+    c.has_what[a.address].add("z")
+    a.data["z"] = 5
 
     cc = rpc(ip=c.ip, port=c.port)
 
-    yield cc.delete_data(keys=['z'])  # TODO: this hangs for a while
-    assert 'z' not in a.data
-    assert not c.who_has['z']
+    yield cc.delete_data(keys=["z"])  # TODO: this hangs for a while
+    assert "z" not in a.data
+    assert not c.who_has["z"]
     assert not c.has_what[bad]
     assert not c.has_what[a.address]
 
@@ -153,20 +170,23 @@ def dont_test_delete_data_with_missing_worker(c, a, b):
 
 @gen_cluster(client=True)
 def test_upload_file(c, s, a, b):
-    assert not os.path.exists(os.path.join(a.local_dir, 'foobar.py'))
-    assert not os.path.exists(os.path.join(b.local_dir, 'foobar.py'))
+    assert not os.path.exists(os.path.join(a.local_dir, "foobar.py"))
+    assert not os.path.exists(os.path.join(b.local_dir, "foobar.py"))
     assert a.local_dir != b.local_dir
 
     aa = rpc(a.address)
     bb = rpc(b.address)
-    yield [aa.upload_file(filename='foobar.py', data=b'x = 123'),
-           bb.upload_file(filename='foobar.py', data='x = 123')]
+    yield [
+        aa.upload_file(filename="foobar.py", data=b"x = 123"),
+        bb.upload_file(filename="foobar.py", data="x = 123"),
+    ]
 
-    assert os.path.exists(os.path.join(a.local_dir, 'foobar.py'))
-    assert os.path.exists(os.path.join(b.local_dir, 'foobar.py'))
+    assert os.path.exists(os.path.join(a.local_dir, "foobar.py"))
+    assert os.path.exists(os.path.join(b.local_dir, "foobar.py"))
 
     def g():
         import foobar
+
         return foobar.x
 
     future = c.submit(g, workers=a.address)
@@ -177,27 +197,29 @@ def g():
     yield b._close()
     aa.close_rpc()
     bb.close_rpc()
-    assert not os.path.exists(os.path.join(a.local_dir, 'foobar.py'))
+    assert not os.path.exists(os.path.join(a.local_dir, "foobar.py"))
 
 
 @pytest.mark.skip(reason="don't yet support uploading pyc files")
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)])
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)])
 def test_upload_file_pyc(c, s, w):
     with tmpfile() as dirname:
         os.mkdir(dirname)
-        with open(os.path.join(dirname, 'foo.py'), mode='w') as f:
-            f.write('def f():\n    return 123')
+        with open(os.path.join(dirname, "foo.py"), mode="w") as f:
+            f.write("def f():\n    return 123")
 
         sys.path.append(dirname)
         try:
             import foo
+
             assert foo.f() == 123
-            pyc = cache_from_source(os.path.join(dirname, 'foo.py'))
+            pyc = cache_from_source(os.path.join(dirname, "foo.py"))
             assert os.path.exists(pyc)
             yield c.upload_file(pyc)
 
             def g():
                 import foo
+
                 return foo.x
 
             future = c.submit(g)
@@ -209,8 +231,8 @@ def g():
 
 @gen_cluster(client=True)
 def test_upload_egg(c, s, a, b):
-    eggname = 'testegg-1.0.0-py3.4.egg'
-    local_file = __file__.replace('test_worker.py', eggname)
+    eggname = "testegg-1.0.0-py3.4.egg"
+    local_file = __file__.replace("test_worker.py", eggname)
     assert not os.path.exists(os.path.join(a.local_dir, eggname))
     assert not os.path.exists(os.path.join(b.local_dir, eggname))
     assert a.local_dir != b.local_dir
@@ -222,6 +244,7 @@ def test_upload_egg(c, s, a, b):
 
     def g(x):
         import testegg
+
         return testegg.inc(x)
 
     future = c.submit(g, 10, workers=a.address)
@@ -235,8 +258,8 @@ def g(x):
 
 @gen_cluster(client=True)
 def test_upload_pyz(c, s, a, b):
-    pyzname = 'mytest.pyz'
-    local_file = __file__.replace('test_worker.py', pyzname)
+    pyzname = "mytest.pyz"
+    local_file = __file__.replace("test_worker.py", pyzname)
     assert not os.path.exists(os.path.join(a.local_dir, pyzname))
     assert not os.path.exists(os.path.join(b.local_dir, pyzname))
     assert a.local_dir != b.local_dir
@@ -248,6 +271,7 @@ def test_upload_pyz(c, s, a, b):
 
     def g(x):
         from mytest import mytest
+
         return mytest.inc(x)
 
     future = c.submit(g, 10, workers=a.address)
@@ -259,22 +283,22 @@ def g(x):
     assert not os.path.exists(os.path.join(a.local_dir, pyzname))
 
 
-@pytest.mark.xfail(reason='Still lose time to network I/O')
+@pytest.mark.xfail(reason="Still lose time to network I/O")
 @gen_cluster(client=True)
 def test_upload_large_file(c, s, a, b):
-    pytest.importorskip('crick')
+    pytest.importorskip("crick")
     yield gen.sleep(0.05)
     with rpc(a.address) as aa:
-        yield aa.upload_file(filename='myfile.dat', data=b'0' * 100000000)
+        yield aa.upload_file(filename="myfile.dat", data=b"0" * 100000000)
         yield gen.sleep(0.05)
-        assert a.digests['tick-duration'].components[0].max() < 0.050
+        assert a.digests["tick-duration"].components[0].max() < 0.050
 
 
 @gen_cluster()
 def test_broadcast(s, a, b):
     with rpc(s.address) as cc:
-        results = yield cc.broadcast(msg={'op': 'ping'})
-        assert results == {a.address: b'pong', b.address: b'pong'}
+        results = yield cc.broadcast(msg={"op": "ping"})
+        assert results == {a.address: b"pong", b.address: b"pong"}
 
 
 @gen_test()
@@ -292,7 +316,7 @@ def test_worker_with_port_zero():
 def test_worker_waits_for_center_to_come_up(loop):
     @gen.coroutine
     def f():
-        w = yield Worker('127.0.0.1', 8007)
+        w = yield Worker("127.0.0.1", 8007)
 
     try:
         loop.run_sync(f, timeout=4)
@@ -300,7 +324,7 @@ def f():
         pass
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)])
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)])
 def test_worker_task_data(c, s, w):
     x = delayed(2)
     xx = c.persist(x)
@@ -316,20 +340,20 @@ def __init__(self, a, b):
         def __str__(self):
             return "MyException(%s)" % self.args
 
-    msg = error_message(MyException('Hello', 'World!'))
-    assert 'Hello' in str(msg['exception'])
+    msg = error_message(MyException("Hello", "World!"))
+    assert "Hello" in str(msg["exception"])
 
 
 @gen_cluster()
 def test_gather(s, a, b):
-    b.data['x'] = 1
-    b.data['y'] = 2
+    b.data["x"] = 1
+    b.data["y"] = 2
     with rpc(a.address) as aa:
-        resp = yield aa.gather(who_has={'x': [b.address], 'y': [b.address]})
-        assert resp['status'] == 'OK'
+        resp = yield aa.gather(who_has={"x": [b.address], "y": [b.address]})
+        assert resp["status"] == "OK"
 
-        assert a.data['x'] == b.data['x']
-        assert a.data['y'] == b.data['y']
+        assert a.data["x"] == b.data["x"]
+        assert a.data["y"] == b.data["y"]
 
 
 def test_io_loop(loop):
@@ -342,19 +366,24 @@ def test_io_loop(loop):
 
 @gen_cluster(client=True, ncores=[])
 def test_spill_to_disk(c, s):
-    np = pytest.importorskip('numpy')
-    w = yield Worker(s.address, loop=s.loop, memory_limit=1200 / 0.6,
-               memory_pause_fraction=None, memory_spill_fraction=None)
-
-    x = c.submit(np.random.randint, 0, 255, size=500, dtype='u1', key='x')
+    np = pytest.importorskip("numpy")
+    w = yield Worker(
+        s.address,
+        loop=s.loop,
+        memory_limit=1200 / 0.6,
+        memory_pause_fraction=None,
+        memory_spill_fraction=None,
+    )
+
+    x = c.submit(np.random.randint, 0, 255, size=500, dtype="u1", key="x")
     yield wait(x)
-    y = c.submit(np.random.randint, 0, 255, size=500, dtype='u1', key='y')
+    y = c.submit(np.random.randint, 0, 255, size=500, dtype="u1", key="y")
     yield wait(y)
 
     assert set(w.data) == {x.key, y.key}
     assert set(w.data.fast) == {x.key, y.key}
 
-    z = c.submit(np.random.randint, 0, 255, size=500, dtype='u1', key='z')
+    z = c.submit(np.random.randint, 0, 255, size=500, dtype="u1", key="z")
     yield wait(z)
     assert set(w.data) == {x.key, y.key, z.key}
     assert set(w.data.fast) == {y.key, z.key}
@@ -370,11 +399,12 @@ def test_spill_to_disk(c, s):
 def test_access_key(c, s, a, b):
     def f(i):
         from distributed.worker import thread_state
+
         return thread_state.key
 
-    futures = [c.submit(f, i, key='x-%d' % i) for i in range(20)]
+    futures = [c.submit(f, i, key="x-%d" % i) for i in range(20)]
     results = yield c._gather(futures)
-    assert list(results) == ['x-%d' % i for i in range(20)]
+    assert list(results) == ["x-%d" % i for i in range(20)]
 
 
 @gen_cluster(client=True)
@@ -417,32 +447,31 @@ def test_Executor(c, s):
 
 
 @pytest.mark.skip(reason="Leaks a large amount of memory")
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)], timeout=30)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)], timeout=30)
 def test_spill_by_default(c, s, w):
-    da = pytest.importorskip('dask.array')
-    x = da.ones(int(TOTAL_MEMORY * 0.7), chunks=10000000, dtype='u1')
+    da = pytest.importorskip("dask.array")
+    x = da.ones(int(TOTAL_MEMORY * 0.7), chunks=10000000, dtype="u1")
     y = c.persist(x)
     yield wait(y)
     assert len(w.data.slow)  # something is on disk
     del x, y
 
 
-@gen_cluster(ncores=[('127.0.0.1', 1)],
-             worker_kwargs={'reconnect': False})
+@gen_cluster(ncores=[("127.0.0.1", 1)], worker_kwargs={"reconnect": False})
 def test_close_on_disconnect(s, w):
     yield s.close()
 
     start = time()
-    while w.status != 'closed':
+    while w.status != "closed":
         yield gen.sleep(0.01)
         assert time() < start + 5
 
 
 def test_memory_limit_auto():
-    a = Worker('127.0.0.1', 8099, ncores=1)
-    b = Worker('127.0.0.1', 8099, ncores=2)
-    c = Worker('127.0.0.1', 8099, ncores=100)
-    d = Worker('127.0.0.1', 8099, ncores=200)
+    a = Worker("127.0.0.1", 8099, ncores=1)
+    b = Worker("127.0.0.1", 8099, ncores=2)
+    c = Worker("127.0.0.1", 8099, ncores=100)
+    d = Worker("127.0.0.1", 8099, ncores=200)
 
     assert isinstance(a.memory_limit, Number)
     assert isinstance(b.memory_limit, Number)
@@ -468,8 +497,17 @@ def test_clean(c, s, a, b):
 
     yield y
 
-    collections = [a.tasks, a.task_state, a.startstops, a.data, a.nbytes,
-                   a.durations, a.priorities, a.types, a.threads]
+    collections = [
+        a.tasks,
+        a.task_state,
+        a.startstops,
+        a.data,
+        a.nbytes,
+        a.durations,
+        a.priorities,
+        a.types,
+        a.threads,
+    ]
     for c in collections:
         assert c
 
@@ -489,15 +527,15 @@ def test_message_breakup(c, s, a, b):
     n = 100000
     a.target_message_size = 10 * n
     b.target_message_size = 10 * n
-    xs = [c.submit(mul, b'%d' % i, n, workers=a.address) for i in range(30)]
+    xs = [c.submit(mul, b"%d" % i, n, workers=a.address) for i in range(30)]
     y = c.submit(lambda *args: None, xs, workers=b.address)
     yield y
 
     assert 2 <= len(b.incoming_transfer_log) <= 20
     assert 2 <= len(a.outgoing_transfer_log) <= 20
 
-    assert all(msg['who'] == b.address for msg in a.outgoing_transfer_log)
-    assert all(msg['who'] == a.address for msg in a.incoming_transfer_log)
+    assert all(msg["who"] == b.address for msg in a.outgoing_transfer_log)
+    assert all(msg["who"] == a.address for msg in a.incoming_transfer_log)
 
 
 @gen_cluster(client=True)
@@ -528,13 +566,14 @@ def test_system_monitor(s, a, b):
     b.monitor.update()
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 2, {'resources': {'A': 1}}),
-                                  ('127.0.0.1', 1)])
+@gen_cluster(
+    client=True, ncores=[("127.0.0.1", 2, {"resources": {"A": 1}}), ("127.0.0.1", 1)]
+)
 def test_restrictions(c, s, a, b):
     # Resource restrictions
-    x = c.submit(inc, 1, resources={'A': 1})
+    x = c.submit(inc, 1, resources={"A": 1})
     yield x
-    assert a.resource_restrictions == {x.key: {'A': 1}}
+    assert a.resource_restrictions == {x.key: {"A": 1}}
     yield c._cancel(x)
 
     while x.key in a.task_state:
@@ -558,7 +597,7 @@ def test_clean_nbytes(c, s, a, b):
     assert len(a.nbytes) + len(b.nbytes) == 1
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 20)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 20)
 def test_gather_many_small(c, s, a, *workers):
     a.total_out_connections = 2
     futures = yield c._scatter(list(range(100)))
@@ -572,14 +611,14 @@ def f(*args):
     yield wait(future)
 
     types = list(pluck(0, a.log))
-    req = [i for i, t in enumerate(types) if t == 'request-dep']
-    recv = [i for i, t in enumerate(types) if t == 'receive-dep']
+    req = [i for i, t in enumerate(types) if t == "request-dep"]
+    recv = [i for i, t in enumerate(types) if t == "receive-dep"]
     assert min(recv) > max(req)
 
     assert a.comm_nbytes == 0
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 3)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 3)
 def test_multiple_transfers(c, s, w1, w2, w3):
     x = c.submit(inc, 1, workers=w1.address)
     y = c.submit(inc, 2, workers=w2.address)
@@ -588,14 +627,14 @@ def test_multiple_transfers(c, s, w1, w2, w3):
     yield wait(z)
 
     r = w3.startstops[z.key]
-    transfers = [t for t in r if t[0] == 'transfer']
+    transfers = [t for t in r if t[0] == "transfer"]
     assert len(transfers) == 2
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 3)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 3)
 def test_share_communication(c, s, w1, w2, w3):
-    x = c.submit(mul, b'1', int(w3.target_message_size + 1), workers=w1.address)
-    y = c.submit(mul, b'2', int(w3.target_message_size + 1), workers=w2.address)
+    x = c.submit(mul, b"1", int(w3.target_message_size + 1), workers=w1.address)
+    y = c.submit(mul, b"2", int(w3.target_message_size + 1), workers=w2.address)
     yield wait([x, y])
     yield c._replicate([x, y], workers=[w1.address, w2.address])
     z = c.submit(add, x, y, workers=w3.address)
@@ -607,15 +646,15 @@ def test_share_communication(c, s, w1, w2, w3):
 
 @gen_cluster(client=True)
 def test_dont_overlap_communications_to_same_worker(c, s, a, b):
-    x = c.submit(mul, b'1', int(b.target_message_size + 1), workers=a.address)
-    y = c.submit(mul, b'2', int(b.target_message_size + 1), workers=a.address)
+    x = c.submit(mul, b"1", int(b.target_message_size + 1), workers=a.address)
+    y = c.submit(mul, b"2", int(b.target_message_size + 1), workers=a.address)
     yield wait([x, y])
     z = c.submit(add, x, y, workers=b.address)
     yield wait(z)
     assert len(b.incoming_transfer_log) == 2
     l1, l2 = b.incoming_transfer_log
 
-    assert l1['stop'] < l2['start']
+    assert l1["stop"] < l2["start"]
 
 
 @pytest.mark.avoid_travis
@@ -625,6 +664,7 @@ def test_log_exception_on_failed_task(c, s, a, b):
         fh = logging.FileHandler(fn)
         try:
             from distributed.worker import logger
+
             logger.addHandler(fh)
 
             future = c.submit(div, 1, 0)
@@ -677,12 +717,12 @@ def test_hold_onto_dependents(c, s, a, b):
 @slow
 @gen_cluster(client=False, ncores=[])
 def test_worker_death_timeout(s):
-    with dask.config.set({'distributed.comm.timeouts.connect': '1s'}):
+    with dask.config.set({"distributed.comm.timeouts.connect": "1s"}):
         yield s.close()
         w = yield Worker(s.address, death_timeout=1)
 
     yield gen.sleep(2)
-    assert w.status == 'closed'
+    assert w.status == "closed"
 
 
 @gen_cluster(client=True)
@@ -698,14 +738,14 @@ def test_stop_doing_unnecessary_work(c, s, a, b):
         assert time() - start < 0.5
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)])
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)])
 def test_priorities(c, s, w):
     values = []
     for i in range(10):
-        a = delayed(slowinc)(i, dask_key_name='a-%d' % i, delay=0.01)
-        a1 = delayed(inc)(a, dask_key_name='a1-%d' % i)
-        a2 = delayed(inc)(a1, dask_key_name='a2-%d' % i)
-        b1 = delayed(dec)(a, dask_key_name='b1-%d' % i)  # <<-- least favored
+        a = delayed(slowinc)(i, dask_key_name="a-%d" % i, delay=0.01)
+        a1 = delayed(inc)(a, dask_key_name="a1-%d" % i)
+        a2 = delayed(inc)(a1, dask_key_name="a2-%d" % i)
+        b1 = delayed(dec)(a, dask_key_name="b1-%d" % i)  # <<-- least favored
 
         values.append(a2)
         values.append(b1)
@@ -713,29 +753,31 @@ def test_priorities(c, s, w):
     futures = c.compute(values)
     yield wait(futures)
 
-    log = [t[0] for t in w.log
-           if t[1] == 'executing'
-           and t[2] == 'memory'
-           and not t[0].startswith('finalize')]
+    log = [
+        t[0]
+        for t in w.log
+        if t[1] == "executing" and t[2] == "memory" and not t[0].startswith("finalize")
+    ]
 
-    assert any(key.startswith('b1') for key in log[:len(log) // 2])
+    assert any(key.startswith("b1") for key in log[: len(log) // 2])
 
 
 @gen_cluster(client=True)
 def test_heartbeats(c, s, a, b):
     x = s.workers[a.address].last_seen
     start = time()
-    yield gen.sleep(a.periodic_callbacks['heartbeat'].callback_time / 1000 + 0.1)
+    yield gen.sleep(a.periodic_callbacks["heartbeat"].callback_time / 1000 + 0.1)
     while s.workers[a.address].last_seen == x:
         yield gen.sleep(0.01)
         assert time() < start + 2
-    assert a.periodic_callbacks['heartbeat'].callback_time < 1000
+    assert a.periodic_callbacks["heartbeat"].callback_time < 1000
 
 
-@pytest.mark.parametrize('worker', [Worker, Nanny])
+@pytest.mark.parametrize("worker", [Worker, Nanny])
 def test_worker_dir(worker):
     with tmpfile() as fn:
-        @gen_cluster(client=True, worker_kwargs={'local_dir': fn})
+
+        @gen_cluster(client=True, worker_kwargs={"local_dir": fn})
         def test_worker_dir(c, s, a, b):
             directories = [w.local_directory for w in s.workers.values()]
             assert all(d.startswith(fn) for d in directories)
@@ -751,7 +793,7 @@ def __init__(self, data):
             self.data = data
 
         def __sizeof__(self):
-            raise TypeError('Hello')
+            raise TypeError("Hello")
 
     future = c.submit(BadSize, 123)
     result = yield future
@@ -770,7 +812,7 @@ def __sizeof__(self):
     future = c.submit(Bad)
     yield wait(future)
 
-    assert future.status == 'error'
+    assert future.status == "error"
 
     with pytest.raises(TypeError):
         yield future
@@ -781,8 +823,9 @@ def __sizeof__(self):
 
 
 @pytest.mark.skip(reason="Our logic here is faulty")
-@gen_cluster(ncores=[('127.0.0.1', 2)], client=True,
-             worker_kwargs={'memory_limit': 10e9})
+@gen_cluster(
+    ncores=[("127.0.0.1", 2)], client=True, worker_kwargs={"memory_limit": 10e9}
+)
 def test_fail_write_many_to_disk(c, s, a):
     a.validate = False
     yield gen.sleep(0.1)
@@ -863,8 +906,7 @@ def f():
         raise gen.Return(result)
 
     results = yield c.run(f)
-    assert results == {a.address: 11,
-                       b.address: 11}
+    assert results == {a.address: 11, b.address: 11}
 
 
 def test_get_client_coroutine_sync(client, s, a, b):
@@ -876,13 +918,13 @@ def f():
         raise gen.Return(result)
 
     results = client.run(f)
-    assert results == {a['address']: 11,
-                       b['address']: 11}
+    assert results == {a["address"]: 11, b["address"]: 11}
 
 
 @gen_cluster()
 def test_global_workers(s, a, b):
     from distributed.worker import _global_workers
+
     n = len(_global_workers)
     w = _global_workers[-1]()
     assert w is a or w is b
@@ -894,7 +936,7 @@ def test_global_workers(s, a, b):
 @pytest.mark.skipif(WINDOWS, reason="file descriptors")
 @gen_cluster(ncores=[])
 def test_worker_fds(s):
-    psutil = pytest.importorskip('psutil')
+    psutil = pytest.importorskip("psutil")
     yield gen.sleep(0.05)
     start = psutil.Process().num_fds()
 
@@ -916,39 +958,41 @@ def test_worker_fds(s):
 
 @gen_cluster(ncores=[])
 def test_service_hosts_match_worker(s):
-    pytest.importorskip('bokeh')
+    pytest.importorskip("bokeh")
     from distributed.bokeh.worker import BokehWorker
-    services = {('bokeh', ':0'): BokehWorker}
 
-    w = Worker(s.address, services={('bokeh', ':0'): BokehWorker})
-    yield w._start('tcp://0.0.0.0')
-    sock = first(w.services['bokeh'].server._http._sockets.values())
-    assert sock.getsockname()[0] in ('::', '0.0.0.0')
+    services = {("bokeh", ":0"): BokehWorker}
+
+    w = Worker(s.address, services={("bokeh", ":0"): BokehWorker})
+    yield w._start("tcp://0.0.0.0")
+    sock = first(w.services["bokeh"].server._http._sockets.values())
+    assert sock.getsockname()[0] in ("::", "0.0.0.0")
     yield w._close()
 
-    w = Worker(s.address, services={('bokeh', ':0'): BokehWorker})
-    yield w._start('tcp://127.0.0.1')
-    sock = first(w.services['bokeh'].server._http._sockets.values())
-    assert sock.getsockname()[0] in ('::', '0.0.0.0')
+    w = Worker(s.address, services={("bokeh", ":0"): BokehWorker})
+    yield w._start("tcp://127.0.0.1")
+    sock = first(w.services["bokeh"].server._http._sockets.values())
+    assert sock.getsockname()[0] in ("::", "0.0.0.0")
     yield w._close()
 
-    w = Worker(s.address, services={('bokeh', 0): BokehWorker})
-    yield w._start('tcp://127.0.0.1')
-    sock = first(w.services['bokeh'].server._http._sockets.values())
-    assert sock.getsockname()[0] == '127.0.0.1'
+    w = Worker(s.address, services={("bokeh", 0): BokehWorker})
+    yield w._start("tcp://127.0.0.1")
+    sock = first(w.services["bokeh"].server._http._sockets.values())
+    assert sock.getsockname()[0] == "127.0.0.1"
     yield w._close()
 
 
 @gen_cluster(ncores=[])
 def test_start_services(s):
-    pytest.importorskip('bokeh')
+    pytest.importorskip("bokeh")
     from distributed.bokeh.worker import BokehWorker
-    services = {('bokeh', ':1234'): BokehWorker}
+
+    services = {("bokeh", ":1234"): BokehWorker}
 
     w = Worker(s.address, services=services)
     yield w._start()
 
-    assert w.services['bokeh'].server.port == 1234
+    assert w.services["bokeh"].server.port == 1234
     yield w._close()
 
 
@@ -968,7 +1012,7 @@ def test_scheduler_delay(c, s, a, b):
     old = a.scheduler_delay
     assert abs(a.scheduler_delay) < 0.3
     assert abs(b.scheduler_delay) < 0.3
-    yield gen.sleep(a.periodic_callbacks['heartbeat'].callback_time / 1000 + .3)
+    yield gen.sleep(a.periodic_callbacks["heartbeat"].callback_time / 1000 + 0.3)
     assert a.scheduler_delay != old
 
 
@@ -977,26 +1021,30 @@ def test_statistical_profiling(c, s, a, b):
     futures = c.map(slowinc, range(10), delay=0.1)
     yield wait(futures)
 
-    profile = a.profile_keys['slowinc']
-    assert profile['count']
+    profile = a.profile_keys["slowinc"]
+    assert profile["count"]
 
 
 @nodebug
 @gen_cluster(client=True)
 def test_statistical_profiling_2(c, s, a, b):
-    da = pytest.importorskip('dask.array')
+    da = pytest.importorskip("dask.array")
     for i in range(5):
         x = da.random.random(1000000, chunks=(10000,))
         y = (x + x * 2) - x.sum().persist()
         yield wait(y)
     profile = a.get_profile()
-    assert profile['count']
-    assert 'sum' in str(profile) or 'random' in str(profile)
+    assert profile["count"]
+    assert "sum" in str(profile) or "random" in str(profile)
 
 
-@gen_cluster(ncores=[('127.0.0.1', 1)], client=True, worker_kwargs={'memory_monitor_interval': 10})
+@gen_cluster(
+    ncores=[("127.0.0.1", 1)],
+    client=True,
+    worker_kwargs={"memory_monitor_interval": 10},
+)
 def test_robust_to_bad_sizeof_estimates(c, s, a):
-    np = pytest.importorskip('numpy')
+    np = pytest.importorskip("numpy")
     memory = psutil.Process().memory_info().rss
     a.memory_limit = memory / 0.7 + 400e6
 
@@ -1008,7 +1056,7 @@ def __sizeof__(self):
             return 10
 
     def f(n):
-        x = np.ones(int(n), dtype='u1')
+        x = np.ones(int(n), dtype="u1")
         result = BadAccounting(x)
         return result
 
@@ -1021,42 +1069,48 @@ def f(n):
 
 
 @pytest.mark.slow
-@gen_cluster(ncores=[('127.0.0.1', 2)],
-             client=True,
-             worker_kwargs={'memory_monitor_interval': 10,
-                            'memory_spill_fraction': False,  # don't spill
-                            'memory_target_fraction': False,
-                            'memory_pause_fraction': 0.5},
-             timeout=20)
+@gen_cluster(
+    ncores=[("127.0.0.1", 2)],
+    client=True,
+    worker_kwargs={
+        "memory_monitor_interval": 10,
+        "memory_spill_fraction": False,  # don't spill
+        "memory_target_fraction": False,
+        "memory_pause_fraction": 0.5,
+    },
+    timeout=20,
+)
 def test_pause_executor(c, s, a):
     memory = psutil.Process().memory_info().rss
     a.memory_limit = memory / 0.5 + 200e6
-    np = pytest.importorskip('numpy')
+    np = pytest.importorskip("numpy")
 
     def f():
-        x = np.ones(int(400e6), dtype='u1')
+        x = np.ones(int(400e6), dtype="u1")
         sleep(1)
 
-    with captured_logger(logging.getLogger('distributed.worker')) as logger:
+    with captured_logger(logging.getLogger("distributed.worker")) as logger:
         future = c.submit(f)
         futures = c.map(slowinc, range(30), delay=0.1)
 
         start = time()
         while not a.paused:
             yield gen.sleep(0.01)
-            assert time() < start + 4,  (format_bytes(psutil.Process().memory_info().rss),
-                                         format_bytes(a.memory_limit),
-                                         len(a.data))
+            assert time() < start + 4, (
+                format_bytes(psutil.Process().memory_info().rss),
+                format_bytes(a.memory_limit),
+                len(a.data),
+            )
         out = logger.getvalue()
-        assert 'memory' in out.lower()
-        assert 'pausing' in out.lower()
+        assert "memory" in out.lower()
+        assert "pausing" in out.lower()
 
-    assert sum(f.status == 'finished' for f in futures) < 4
+    assert sum(f.status == "finished" for f in futures) < 4
 
     yield wait(futures)
 
 
-@gen_cluster(client=True, worker_kwargs={'profile_cycle_interval': '50 ms'})
+@gen_cluster(client=True, worker_kwargs={"profile_cycle_interval": "50 ms"})
 def test_statistical_profiling_cycle(c, s, a, b):
     futures = c.map(slowinc, range(20), delay=0.05)
     yield wait(futures)
@@ -1065,30 +1119,29 @@ def test_statistical_profiling_cycle(c, s, a, b):
     assert len(a.profile_history) > 3
 
     x = a.get_profile(start=time() + 10, stop=time() + 20)
-    assert not x['count']
+    assert not x["count"]
 
     x = a.get_profile(start=0, stop=time())
-    actual = sum(p['count'] for _, p in a.profile_history) + a.profile_recent['count']
+    actual = sum(p["count"] for _, p in a.profile_history) + a.profile_recent["count"]
     x2 = a.get_profile(start=0, stop=time())
-    assert x['count'] <= actual <= x2['count']
+    assert x["count"] <= actual <= x2["count"]
 
     y = a.get_profile(start=end - 0.300, stop=time())
-    assert 0 < y['count'] <= x['count']
+    assert 0 < y["count"] <= x["count"]
 
 
 @gen_cluster(client=True)
 def test_get_current_task(c, s, a, b):
-
     def some_name():
         return get_worker().get_current_task()
 
     result = yield c.submit(some_name)
-    assert result.startswith('some_name')
+    assert result.startswith("some_name")
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 2)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 2)
 def test_reschedule(c, s, a, b):
-    s.extensions['stealing']._pc.stop()
+    s.extensions["stealing"]._pc.stop()
     a_address = a.address
 
     def f(x):
@@ -1105,21 +1158,23 @@ def f(x):
 
 def test_deque_handler():
     from distributed.worker import logger
-    w = Worker('127.0.0.1', 8019)
+
+    w = Worker("127.0.0.1", 8019)
     deque_handler = w._deque_handler
-    logger.info('foo456')
+    logger.info("foo456")
     assert deque_handler.deque
     msg = deque_handler.deque[-1]
-    assert 'distributed.worker' in deque_handler.format(msg)
-    assert any(msg.msg == 'foo456' for msg in deque_handler.deque)
+    assert "distributed.worker" in deque_handler.format(msg)
+    assert any(msg.msg == "foo456" for msg in deque_handler.deque)
 
 
 @gen_cluster(ncores=[], client=True)
 def test_avoid_memory_monitor_if_zero_limit(c, s):
-    worker = yield Worker(s.address, loop=s.loop, memory_limit=0,
-                    memory_monitor_interval=10)
+    worker = yield Worker(
+        s.address, loop=s.loop, memory_limit=0, memory_monitor_interval=10
+    )
     assert type(worker.data) is dict
-    assert 'memory' not in worker.periodic_callbacks
+    assert "memory" not in worker.periodic_callbacks
 
     future = c.submit(inc, 1)
     assert (yield future) == 2
@@ -1130,9 +1185,13 @@ def test_avoid_memory_monitor_if_zero_limit(c, s):
     yield worker._close()
 
 
-@gen_cluster(ncores=[('127.0.0.1', 1)],
-             config={'distributed.worker.memory.spill': False,
-                     'distributed.worker.memory.target': False})
+@gen_cluster(
+    ncores=[("127.0.0.1", 1)],
+    config={
+        "distributed.worker.memory.spill": False,
+        "distributed.worker.memory.target": False,
+    },
+)
 def test_dict_data_if_no_spill_to_disk(s, w):
     assert type(w.data) is dict
 
@@ -1147,20 +1206,19 @@ def func(dask_scheduler):
         return list(dask_scheduler.clients)
 
     start = time()
-    while not any('worker' in n for n in client.run_on_scheduler(func)):
+    while not any("worker" in n for n in client.run_on_scheduler(func)):
         sleep(0.1)
         assert time() < start + 10
 
 
-@gen_cluster(ncores=[('127.0.0.1', 1)],
-        worker_kwargs={'memory_limit': '2e3 MB'})
+@gen_cluster(ncores=[("127.0.0.1", 1)], worker_kwargs={"memory_limit": "2e3 MB"})
 def test_parse_memory_limit(s, w):
     assert w.memory_limit == 2e9
 
 
 @gen_cluster(ncores=[], client=True)
 def test_scheduler_address_config(c, s):
-    with dask.config.set({'scheduler-address': s.address}):
+    with dask.config.set({"scheduler-address": s.address}):
         worker = yield Worker(loop=s.loop)
         assert worker.scheduler.address == s.address
     yield worker._close()
@@ -1169,7 +1227,7 @@ def test_scheduler_address_config(c, s):
 @slow
 @gen_cluster(client=True)
 def test_wait_for_outgoing(c, s, a, b):
-    np = pytest.importorskip('numpy')
+    np = pytest.importorskip("numpy")
     x = np.random.random(10000000)
     future = yield c.scatter(x, workers=a.address)
 
@@ -1177,36 +1235,39 @@ def test_wait_for_outgoing(c, s, a, b):
     yield wait(y)
 
     assert len(b.incoming_transfer_log) == len(a.outgoing_transfer_log) == 1
-    bb = b.incoming_transfer_log[0]['duration']
-    aa = a.outgoing_transfer_log[0]['duration']
+    bb = b.incoming_transfer_log[0]["duration"]
+    aa = a.outgoing_transfer_log[0]["duration"]
     ratio = aa / bb
 
     assert 1 / 3 < ratio < 3
 
 
-@pytest.mark.skipif(not sys.platform.startswith('linux'),
-                    reason="Need 127.0.0.2 to mean localhost")
-@gen_cluster(ncores=[('127.0.0.1', 1), ('127.0.0.1', 1), ('127.0.0.2', 1)],
-             client=True)
+@pytest.mark.skipif(
+    not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
+)
+@gen_cluster(ncores=[("127.0.0.1", 1), ("127.0.0.1", 1), ("127.0.0.2", 1)], client=True)
 def test_prefer_gather_from_local_address(c, s, w1, w2, w3):
     x = yield c.scatter(123, workers=[w1.address, w3.address], broadcast=True)
 
     y = c.submit(inc, x, workers=[w2.address])
     yield wait(y)
 
-    assert any(d['who'] == w2.address for d in w1.outgoing_transfer_log)
-    assert not any(d['who'] == w2.address for d in w3.outgoing_transfer_log)
+    assert any(d["who"] == w2.address for d in w1.outgoing_transfer_log)
+    assert not any(d["who"] == w2.address for d in w3.outgoing_transfer_log)
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 20, timeout=30,
-             config={'distributed.worker.connections.incoming': 1})
+@gen_cluster(
+    client=True,
+    ncores=[("127.0.0.1", 1)] * 20,
+    timeout=30,
+    config={"distributed.worker.connections.incoming": 1},
+)
 def test_avoid_oversubscription(c, s, *workers):
-    np = pytest.importorskip('numpy')
+    np = pytest.importorskip("numpy")
     x = c.submit(np.random.random, 1000000, workers=[workers[0].address])
     yield wait(x)
 
-    futures = [c.submit(len, x, pure=False, workers=[w.address])
-               for w in workers[1:]]
+    futures = [c.submit(len, x, pure=False, workers=[w.address]) for w in workers[1:]]
 
     yield wait(futures)
 
@@ -1217,31 +1278,33 @@ def test_avoid_oversubscription(c, s, *workers):
     assert len([w for w in workers if len(w.outgoing_transfer_log) > 0]) >= 3
 
 
-@gen_cluster(client=True, worker_kwargs={'metrics': {'my_port': lambda w: w.port}})
+@gen_cluster(client=True, worker_kwargs={"metrics": {"my_port": lambda w: w.port}})
 def test_custom_metrics(c, s, a, b):
-    assert s.workers[a.address].metrics['my_port'] == a.port
-    assert s.workers[b.address].metrics['my_port'] == b.port
+    assert s.workers[a.address].metrics["my_port"] == a.port
+    assert s.workers[b.address].metrics["my_port"] == b.port
 
 
 @gen_cluster(client=True)
 def test_register_worker_callbacks(c, s, a, b):
-    #preload function to run
+    # preload function to run
     def mystartup(dask_worker):
         dask_worker.init_variable = 1
 
     def mystartup2():
         import os
-        os.environ['MY_ENV_VALUE'] = 'WORKER_ENV_VALUE'
+
+        os.environ["MY_ENV_VALUE"] = "WORKER_ENV_VALUE"
         return "Env set."
 
-    #Check that preload function has been run
+    # Check that preload function has been run
     def test_import(dask_worker):
-        return hasattr(dask_worker, 'init_variable')
+        return hasattr(dask_worker, "init_variable")
         #       and dask_worker.init_variable == 1
 
     def test_startup2():
         import os
-        return os.getenv('MY_ENV_VALUE', None) == 'WORKER_ENV_VALUE'
+
+        return os.getenv("MY_ENV_VALUE", None) == "WORKER_ENV_VALUE"
 
     # Nothing has been run yet
     assert len(s.worker_setups) == 0
@@ -1309,7 +1372,7 @@ def __init__(self, x, y):
             self.x = x
             self.y = y
 
-    w = yield Worker(s.address, data=(Data, {'x': 123, 'y': 456}))
+    w = yield Worker(s.address, data=(Data, {"x": 123, "y": 456}))
     assert w.data.x == 123
     assert w.data.y == 456
     yield w._close()
diff --git a/distributed/tests/test_worker_client.py b/distributed/tests/test_worker_client.py
index b5d96edb238..b0dd338153f 100644
--- a/distributed/tests/test_worker_client.py
+++ b/distributed/tests/test_worker_client.py
@@ -10,8 +10,14 @@
 import pytest
 from tornado import gen
 
-from distributed import (worker_client, Client, as_completed, get_worker, wait,
-                         get_client)
+from distributed import (
+    worker_client,
+    Client,
+    as_completed,
+    get_worker,
+    wait,
+    get_client,
+)
 from distributed.metrics import time
 from distributed.utils_test import double, gen_cluster, inc
 from distributed.utils_test import client, cluster_fixture, loop  # noqa: F401
@@ -33,11 +39,10 @@ def func(x):
     assert yy == 20 + 1 + (20 + 1) * 2
 
     assert len(s.transition_log) > 10
-    assert len([id for id in s.wants_what
-                if id.lower().startswith('client')]) == 1
+    assert len([id for id in s.wants_what if id.lower().startswith("client")]) == 1
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 2)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 2)
 def test_scatter_from_worker(c, s, a, b):
     def func():
         with worker_client() as c:
@@ -64,8 +69,8 @@ def func():
                 correct &= type(futures) == type(data)
 
             o = object()
-            futures = c.scatter({'x': o})
-            correct &= get_worker().data['x'] is o
+            futures = c.scatter({"x": o})
+            correct &= get_worker().data["x"] is o
             return correct
 
     future = c.submit(func)
@@ -78,9 +83,9 @@ def func():
         assert time() < start + 5
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 2)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 2)
 def test_scatter_singleton(c, s, a, b):
-    np = pytest.importorskip('numpy')
+    np = pytest.importorskip("numpy")
 
     def func():
         with worker_client() as c:
@@ -91,7 +96,7 @@ def func():
     yield c.submit(func)
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 1)] * 2)
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 2)
 def test_gather_multi_machine(c, s, a, b):
     a_address = a.address
     b_address = b.address
@@ -157,7 +162,7 @@ def mysum():
         assert time() < start + 3
 
 
-@gen_cluster(client=True, ncores=[('127.0.0.1', 3)])
+@gen_cluster(client=True, ncores=[("127.0.0.1", 3)])
 def test_separate_thread_false(c, s, a):
     a.count = 0
 
@@ -230,13 +235,13 @@ def func(x):
 
     yield wait(c.map(func, range(10)))
     yield a._close()
-    assert c.status == 'running'
+    assert c.status == "running"
 
 
 def test_timeout(client):
     def func():
         with worker_client(timeout=0) as wc:
-            print('hello')
+            print("hello")
 
     future = client.submit(func)
     with pytest.raises(EnvironmentError):
@@ -254,12 +259,13 @@ def test_secede_without_stealing_issue_1262():
 
     # run the loop as an inner function so all workers are closed
     # and exceptions can be examined
-    @gen_cluster(client=True, scheduler_kwargs={'extensions': extensions})
+    @gen_cluster(client=True, scheduler_kwargs={"extensions": extensions})
     def secede_test(c, s, a, b):
         def func(x):
             with worker_client() as wc:
                 y = wc.submit(lambda: 1 + x)
                 return wc.gather(y)
+
         f = yield c.gather(c.submit(func, 1))
 
         raise gen.Return((c, s, a, b, f))
@@ -273,7 +279,6 @@ def func(x):
 
 @gen_cluster(client=True)
 def test_compute_within_worker_client(c, s, a, b):
-
     @dask.delayed
     def f():
         with worker_client():
@@ -298,9 +303,10 @@ def f():
 @gen_cluster()
 def test_submit_different_names(s, a, b):
     # https://github.com/dask/distributed/issues/2058
-    da = pytest.importorskip('dask.array')
-    c = yield Client('localhost:' + s.address.split(":")[-1], loop=s.loop,
-                     asynchronous=True)
+    da = pytest.importorskip("dask.array")
+    c = yield Client(
+        "localhost:" + s.address.split(":")[-1], loop=s.loop, asynchronous=True
+    )
     try:
         X = c.persist(da.random.uniform(size=(100, 10), chunks=50))
         yield wait(X)
diff --git a/distributed/threadpoolexecutor.py b/distributed/threadpoolexecutor.py
index 84c08f447da..d2d4e3b7921 100644
--- a/distributed/threadpoolexecutor.py
+++ b/distributed/threadpoolexecutor.py
@@ -61,7 +61,7 @@ def _worker(executor, work_queue):
                 return
         del executor
     except BaseException:
-        logger.critical('Exception in worker', exc_info=True)
+        logger.critical("Exception in worker", exc_info=True)
     finally:
         del thread_state.proceed
         del thread_state.executor
@@ -75,13 +75,18 @@ def __init__(self, *args, **kwargs):
         super(ThreadPoolExecutor, self).__init__(*args, **kwargs)
         self._rejoin_list = []
         self._rejoin_lock = threading.Lock()
-        self._thread_name_prefix = kwargs.get('thread_name_prefix', 'DaskThreadPoolExecutor')
+        self._thread_name_prefix = kwargs.get(
+            "thread_name_prefix", "DaskThreadPoolExecutor"
+        )
 
     def _adjust_thread_count(self):
         if len(self._threads) < self._max_workers:
-            t = threading.Thread(target=_worker,
-                                 name=self._thread_name_prefix + "-%d-%d" % (os.getpid(), next(self._counter)),
-                                 args=(self, self._work_queue))
+            t = threading.Thread(
+                target=_worker,
+                name=self._thread_name_prefix
+                + "-%d-%d" % (os.getpid(), next(self._counter)),
+                args=(self, self._work_queue),
+            )
             t.daemon = True
             self._threads.add(t)
             t.start()
diff --git a/distributed/utils.py b/distributed/utils.py
index dbc27251758..5259e567358 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -40,6 +40,7 @@
 import tornado
 from tornado import gen
 from tornado.ioloop import IOLoop
+
 try:
     from tornado.ioloop import PollIOLoop
 except ImportError:
@@ -57,17 +58,17 @@
 logger = _logger = logging.getLogger(__name__)
 
 
-no_default = '__no_default__'
+no_default = "__no_default__"
 
 
 def _initialize_mp_context():
-    if PY3 and not sys.platform.startswith('win') and 'PyPy' not in sys.version:
-        method = dask.config.get('distributed.worker.multiprocessing-method')
+    if PY3 and not sys.platform.startswith("win") and "PyPy" not in sys.version:
+        method = dask.config.get("distributed.worker.multiprocessing-method")
         ctx = multiprocessing.get_context(method)
         # Makes the test suite much faster
-        preload = ['distributed']
-        if 'pkg_resources' in sys.modules:
-            preload.append('pkg_resources')
+        preload = ["distributed"]
+        if "pkg_resources" in sys.modules:
+            preload.append("pkg_resources")
         ctx.set_forkserver_preload(preload)
     else:
         ctx = multiprocessing
@@ -80,7 +81,7 @@ def _initialize_mp_context():
 
 def funcname(func):
     """Get the name of a function."""
-    while hasattr(func, 'func'):
+    while hasattr(func, "func"):
         func = func.func
     try:
         return func.__name__
@@ -129,29 +130,31 @@ def _get_ip(host, port, family, default):
         return ip
     except EnvironmentError as e:
         # XXX Should first try getaddrinfo() on socket.gethostname() and getfqdn()
-        warnings.warn("Couldn't detect a suitable IP address for "
-                      "reaching %r, defaulting to %r: %s"
-                      % (host, default, e), RuntimeWarning)
+        warnings.warn(
+            "Couldn't detect a suitable IP address for "
+            "reaching %r, defaulting to %r: %s" % (host, default, e),
+            RuntimeWarning,
+        )
         return default
     finally:
         sock.close()
 
 
-def get_ip(host='8.8.8.8', port=80):
+def get_ip(host="8.8.8.8", port=80):
     """
     Get the local IP address through which the *host* is reachable.
 
     *host* defaults to a well-known Internet host (one of Google's public
     DNS servers).
     """
-    return _get_ip(host, port, family=socket.AF_INET, default='127.0.0.1')
+    return _get_ip(host, port, family=socket.AF_INET, default="127.0.0.1")
 
 
-def get_ipv6(host='2001:4860:4860::8888', port=80):
+def get_ipv6(host="2001:4860:4860::8888", port=80):
     """
     The same as get_ip(), but for IPv6.
     """
-    return _get_ip(host, port, family=socket.AF_INET6, default='::1')
+    return _get_ip(host, port, family=socket.AF_INET6, default="::1")
 
 
 def get_ip_interface(ifname):
@@ -163,6 +166,7 @@ def get_ip_interface(ifname):
     associated with it.
     """
     import psutil
+
     for info in psutil.net_if_addrs()[ifname]:
         if info.family == socket.AF_INET:
             return info.address
@@ -213,6 +217,7 @@ def All(args, quiet_exceptions=()):
         try:
             result = yield tasks.next()
         except Exception:
+
             @gen.coroutine
             def quiet():
                 """ Watch unfinished tasks
@@ -226,6 +231,7 @@ def quiet():
                         yield task
                     except quiet_exceptions:
                         pass
+
             quiet()
             raise
 
@@ -251,6 +257,7 @@ def Any(args, quiet_exceptions=()):
         try:
             result = yield tasks.next()
         except Exception:
+
             @gen.coroutine
             def quiet():
                 """ Watch unfinished tasks
@@ -264,6 +271,7 @@ def quiet():
                         yield task
                     except quiet_exceptions:
                         pass
+
             quiet()
             raise
 
@@ -277,8 +285,10 @@ def sync(loop, func, *args, **kwargs):
     Run coroutine in loop running in separate thread.
     """
     # Tornado's PollIOLoop doesn't raise when using closed, do it ourselves
-    if PollIOLoop and ((isinstance(loop, PollIOLoop) and getattr(loop, '_closing', False)) or
-            (hasattr(loop, 'asyncio_loop') and loop.asyncio_loop._closed)):
+    if PollIOLoop and (
+        (isinstance(loop, PollIOLoop) and getattr(loop, "_closing", False))
+        or (hasattr(loop, "asyncio_loop") and loop.asyncio_loop._closed)
+    ):
         raise RuntimeError("IOLoop is closed")
     try:
         if loop.asyncio_loop.is_closed():  # tornado 6
@@ -286,7 +296,7 @@ def sync(loop, func, *args, **kwargs):
     except AttributeError:
         pass
 
-    timeout = kwargs.pop('callback_timeout', None)
+    timeout = kwargs.pop("callback_timeout", None)
 
     e = threading.Event()
     main_tid = get_thread_identity()
@@ -339,6 +349,7 @@ class LoopRunner(object):
         If true, the loop is meant to run in the thread this
         object is instantiated from, and will not be started automatically.
     """
+
     # All loops currently associated to loop runners
     _all_loops = weakref.WeakKeyDictionary()
     _lock = threading.Lock()
@@ -376,7 +387,7 @@ def _start_unlocked(self):
         assert not self._started
 
         count, real_runner = self._all_loops[self._loop]
-        if (self._asynchronous or real_runner is not None or count > 0):
+        if self._asynchronous or real_runner is not None or count > 0:
             self._all_loops[self._loop] = count + 1, real_runner
             self._started = True
             return
@@ -414,7 +425,9 @@ def run_loop(loop=self._loop):
             # Loop already running in other thread (user-launched)
             done_evt.wait(5)
             if not isinstance(start_exc[0], RuntimeError):
-                if not isinstance(start_exc[0], Exception):  # track down infrequent error
+                if not isinstance(
+                    start_exc[0], Exception
+                ):  # track down infrequent error
                     raise TypeError("not an exception", start_exc[0])
                 raise start_exc[0]
             self._all_loops[self._loop] = count + 1, None
@@ -507,7 +520,7 @@ def set_thread_state(**kwargs):
 @contextmanager
 def tmp_text(filename, text):
     fn = os.path.join(tempfile.gettempdir(), filename)
-    with open(fn, 'w') as f:
+    with open(fn, "w") as f:
         f.write(text)
 
     try:
@@ -529,14 +542,15 @@ def is_kernel():
     False
     """
     # http://stackoverflow.com/questions/34091701/determine-if-were-in-an-ipython-notebook-session
-    if 'IPython' not in sys.modules:  # IPython hasn't been imported
+    if "IPython" not in sys.modules:  # IPython hasn't been imported
         return False
     from IPython import get_ipython
+
     # check for `kernel` attribute on the IPython instance
-    return getattr(get_ipython(), 'kernel', None) is not None
+    return getattr(get_ipython(), "kernel", None) is not None
 
 
-hex_pattern = re.compile('[a-f]+')
+hex_pattern = re.compile("[a-f]+")
 
 
 def key_split(s):
@@ -571,25 +585,26 @@ def key_split(s):
     if type(s) is tuple:
         s = s[0]
     try:
-        words = s.split('-')
+        words = s.split("-")
         if not words[0][0].isalpha():
             result = words[0].split(",")[0].strip("'(\"")
         else:
             result = words[0]
         for word in words[1:]:
-            if word.isalpha() and not (len(word) == 8 and
-                                       hex_pattern.match(word) is not None):
-                result += '-' + word
+            if word.isalpha() and not (
+                len(word) == 8 and hex_pattern.match(word) is not None
+            ):
+                result += "-" + word
             else:
                 break
-        if len(result) == 32 and re.match(r'[a-f0-9]{32}', result):
-            return 'data'
+        if len(result) == 32 and re.match(r"[a-f0-9]{32}", result):
+            return "data"
         else:
-            if result[0] == '<':
-                result = result.strip('<>').split()[0].split('.')[-1]
+            if result[0] == "<":
+                result = result.strip("<>").split()[0].split(".")[-1]
             return result
     except Exception:
-        return 'Other'
+        return "Other"
 
 
 try:
@@ -601,6 +616,7 @@ def key_split(s):
     key_split = lru_cache(100000)(key_split)
 
 if PY3:
+
     def key_split_group(x):
         """A more fine-grained version of key_split
 
@@ -631,19 +647,22 @@ def key_split_group(x):
         if typ is tuple:
             return x[0]
         elif typ is str:
-            if x[0] == '(':
-                return x.split(',', 1)[0].strip('()"\'')
-            elif len(x) == 32 and re.match(r'[a-f0-9]{32}', x):
-                return 'data'
-            elif x[0] == '<':
-                return x.strip('<>').split()[0].split('.')[-1]
+            if x[0] == "(":
+                return x.split(",", 1)[0].strip("()\"'")
+            elif len(x) == 32 and re.match(r"[a-f0-9]{32}", x):
+                return "data"
+            elif x[0] == "<":
+                return x.strip("<>").split()[0].split(".")[-1]
             else:
                 return x
         elif typ is bytes:
             return key_split_group(x.decode())
         else:
-            return 'Other'
+            return "Other"
+
+
 else:
+
     def key_split_group(x):
         """A more fine-grained version of key_split
 
@@ -674,21 +693,22 @@ def key_split_group(x):
         if typ is tuple:
             return x[0]
         elif typ is str or typ is unicode:
-            if x[0] == '(':
-                return x.split(',', 1)[0].strip('()"\'')
-            elif len(x) == 32 and re.match(r'[a-f0-9]{32}', x):
-                return 'data'
-            elif x[0] == '<':
-                return x.strip('<>').split()[0].split('.')[-1]
+            if x[0] == "(":
+                return x.split(",", 1)[0].strip("()\"'")
+            elif len(x) == 32 and re.match(r"[a-f0-9]{32}", x):
+                return "data"
+            elif x[0] == "<":
+                return x.strip("<>").split()[0].split(".")[-1]
             else:
                 return x
         else:
-            return 'Other'
+            return "Other"
 
 
 @contextmanager
 def log_errors(pdb=False):
     from .comm import CommClosedError
+
     try:
         yield
     except (CommClosedError, gen.Return):
@@ -700,11 +720,12 @@ def log_errors(pdb=False):
             pass
         if pdb:
             import pdb
+
             pdb.set_trace()
         raise
 
 
-def silence_logging(level, root='distributed'):
+def silence_logging(level, root="distributed"):
     """
     Force all existing loggers below *root* to the given level at least
     (or keep the existing level if less verbose).
@@ -737,9 +758,9 @@ def ensure_ip(hostname):
     families = [socket.AF_INET, socket.AF_INET6]
     for fam in families:
         try:
-            results = socket.getaddrinfo(hostname,
-                                         1234,  # dummy port number
-                                         fam, socket.SOCK_STREAM)
+            results = socket.getaddrinfo(
+                hostname, 1234, fam, socket.SOCK_STREAM  # dummy port number
+            )
         except socket.gaierror as e:
             exc = e
         else:
@@ -753,12 +774,15 @@ def ensure_ip(hostname):
 
 def get_traceback():
     exc_type, exc_value, exc_traceback = sys.exc_info()
-    bad = [os.path.join('distributed', 'worker'),
-           os.path.join('distributed', 'scheduler'),
-           os.path.join('tornado', 'gen.py'),
-           os.path.join('concurrent', 'futures')]
-    while exc_traceback and any(b in exc_traceback.tb_frame.f_code.co_filename
-                                for b in bad):
+    bad = [
+        os.path.join("distributed", "worker"),
+        os.path.join("distributed", "scheduler"),
+        os.path.join("tornado", "gen.py"),
+        os.path.join("concurrent", "futures"),
+    ]
+    while exc_traceback and any(
+        b in exc_traceback.tb_frame.f_code.co_filename for b in bad
+    ):
         exc_traceback = exc_traceback.tb_next
     return exc_traceback
 
@@ -767,25 +791,24 @@ def truncate_exception(e, n=10000):
     """ Truncate exception to be about a certain length """
     if len(str(e)) > n:
         try:
-            return type(e)("Long error message",
-                           str(e)[:n])
+            return type(e)("Long error message", str(e)[:n])
         except Exception:
-            return Exception("Long error message",
-                             type(e),
-                             str(e)[:n])
+            return Exception("Long error message", type(e), str(e)[:n])
     else:
         return e
 
 
 if sys.version_info >= (3,):
     # (re-)raising StopIteration is deprecated in 3.6+
-    exec("""def queue_to_iterator(q):
+    exec(
+        """def queue_to_iterator(q):
         while True:
             result = q.get()
             if isinstance(result, StopIteration):
                 return result.value
             yield result
-        """)
+        """
+    )
 else:
     # Returning non-None from generator is a syntax error in 2.x
     def queue_to_iterator(q):
@@ -836,15 +859,18 @@ def validate_key(k):
     """
     typ = type(k)
     if typ is not unicode and typ is not bytes:
-        raise TypeError("Unexpected key type %s (value: %r)"
-                        % (typ, k))
+        raise TypeError("Unexpected key type %s (value: %r)" % (typ, k))
 
 
 def _maybe_complex(task):
     """ Possibly contains a nested task """
-    return (istask(task) or
-            type(task) is list and any(map(_maybe_complex, task)) or
-            type(task) is dict and any(map(_maybe_complex, task.values())))
+    return (
+        istask(task)
+        or type(task) is list
+        and any(map(_maybe_complex, task))
+        or type(task) is dict
+        and any(map(_maybe_complex, task.values()))
+    )
 
 
 def convert(task, dsk, extra_values):
@@ -884,7 +910,7 @@ def seek_delimiter(file, delimiter, blocksize):
     if file.tell() == 0:
         return
 
-    last = b''
+    last = b""
     while True:
         current = file.read(blocksize)
         if not current:
@@ -896,7 +922,7 @@ def seek_delimiter(file, delimiter, blocksize):
             return
         except ValueError:
             pass
-        last = full[-len(delimiter):]
+        last = full[-len(delimiter) :]
 
 
 def read_block(f, offset, length, delimiter=None):
@@ -935,12 +961,12 @@ def read_block(f, offset, length, delimiter=None):
     """
     if delimiter:
         f.seek(offset)
-        seek_delimiter(f, delimiter, 2**16)
+        seek_delimiter(f, delimiter, 2 ** 16)
         start = f.tell()
         length -= start - offset
 
         f.seek(start + length)
-        seek_delimiter(f, delimiter, 2**16)
+        seek_delimiter(f, delimiter, 2 ** 16)
         end = f.tell()
 
         offset = start
@@ -952,8 +978,8 @@ def read_block(f, offset, length, delimiter=None):
 
 
 @contextmanager
-def tmpfile(extension=''):
-    extension = '.' + extension.lstrip('.')
+def tmpfile(extension=""):
+    extension = "." + extension.lstrip(".")
     handle, filename = tempfile.mkstemp(extension)
     os.close(handle)
     os.remove(filename)
@@ -984,10 +1010,9 @@ def ensure_bytes(s):
         return s.tobytes()
     if isinstance(s, bytearray) or PY2 and isinstance(s, buffer):  # noqa: F821
         return bytes(s)
-    if hasattr(s, 'encode'):
+    if hasattr(s, "encode"):
         return s.encode()
-    raise TypeError(
-        "Object %s is neither a bytes object nor has an encode method" % s)
+    raise TypeError("Object %s is neither a bytes object nor has an encode method" % s)
 
 
 def divide_n_among_bins(n, bins):
@@ -1020,9 +1045,11 @@ def mean(seq):
 
 
 if hasattr(sys, "is_finalizing"):
+
     def shutting_down(is_finalizing=sys.is_finalizing):
         return is_finalizing()
 
+
 else:
     _shutting_down = [False]
 
@@ -1043,7 +1070,7 @@ def shutting_down(l=_shutting_down):
     """
 
 
-def open_port(host=''):
+def open_port(host=""):
     """ Return a probably-open port
 
     There is a chance that this port will be taken by the operating system soon
@@ -1065,23 +1092,24 @@ def import_file(path):
     names_to_import = []
     tmp_python_path = None
 
-    if ext in ('.py',):  # , '.pyc'):
+    if ext in (".py",):  # , '.pyc'):
         if directory not in sys.path:
             tmp_python_path = directory
         names_to_import.append(name)
-    if ext == '.py':  # Ensure that no pyc file will be reused
+    if ext == ".py":  # Ensure that no pyc file will be reused
         cache_file = cache_from_source(path)
         with ignoring(OSError):
             os.remove(cache_file)
-    if ext in ('.egg', '.zip', '.pyz'):
+    if ext in (".egg", ".zip", ".pyz"):
         if path not in sys.path:
             sys.path.insert(0, path)
-        if ext == '.egg':
+        if ext == ".egg":
             import pkg_resources
+
             pkgs = pkg_resources.find_distributions(path)
             for pkg in pkgs:
                 names_to_import.append(pkg.project_name)
-        elif ext in ('.zip', '.pyz'):
+        elif ext in (".zip", ".pyz"):
             names_to_import.append(name)
 
     loaded = []
@@ -1111,7 +1139,8 @@ class itemgetter(object):
     >>> get_1(data)
     1
     """
-    __slots__ = ('index',)
+
+    __slots__ = ("index",)
 
     def __init__(self, index):
         self.index = index
@@ -1140,35 +1169,35 @@ def format_bytes(n):
     '1.23 PB'
     """
     if n > 1e15:
-        return '%0.2f PB' % (n / 1e15)
+        return "%0.2f PB" % (n / 1e15)
     if n > 1e12:
-        return '%0.2f TB' % (n / 1e12)
+        return "%0.2f TB" % (n / 1e12)
     if n > 1e9:
-        return '%0.2f GB' % (n / 1e9)
+        return "%0.2f GB" % (n / 1e9)
     if n > 1e6:
-        return '%0.2f MB' % (n / 1e6)
+        return "%0.2f MB" % (n / 1e6)
     if n > 1e3:
-        return '%0.2f kB' % (n / 1000)
-    return '%d B' % n
+        return "%0.2f kB" % (n / 1000)
+    return "%d B" % n
 
 
 byte_sizes = {
-        'kB': 10**3,
-        'MB': 10**6,
-        'GB': 10**9,
-        'TB': 10**12,
-        'PB': 10**15,
-        'KiB': 2**10,
-        'MiB': 2**20,
-        'GiB': 2**30,
-        'TiB': 2**40,
-        'PiB': 2**50,
-        'B': 1,
-        '': 1,
+    "kB": 10 ** 3,
+    "MB": 10 ** 6,
+    "GB": 10 ** 9,
+    "TB": 10 ** 12,
+    "PB": 10 ** 15,
+    "KiB": 2 ** 10,
+    "MiB": 2 ** 20,
+    "GiB": 2 ** 30,
+    "TiB": 2 ** 40,
+    "PiB": 2 ** 50,
+    "B": 1,
+    "": 1,
 }
 byte_sizes = {k.lower(): v for k, v in byte_sizes.items()}
-byte_sizes.update({k[0]: v for k, v in byte_sizes.items() if k and 'i' not in k})
-byte_sizes.update({k[:-1]: v for k, v in byte_sizes.items() if k and 'i' in k})
+byte_sizes.update({k[0]: v for k, v in byte_sizes.items() if k and "i" not in k})
+byte_sizes.update({k[:-1]: v for k, v in byte_sizes.items() if k and "i" in k})
 
 
 def parse_bytes(s):
@@ -1193,9 +1222,9 @@ def parse_bytes(s):
     >>> parse_bytes('MB')
     1000000
     """
-    s = s.replace(' ', '')
+    s = s.replace(" ", "")
     if not s[0].isdigit():
-        s = '1' + s
+        s = "1" + s
 
     for i in range(len(s) - 1, -1, -1):
         if not s[i].isalpha():
@@ -1214,30 +1243,30 @@ def parse_bytes(s):
 
 
 timedelta_sizes = {
-        's': 1,
-        'ms': 1e-3,
-        'us': 1e-6,
-        'ns': 1e-9,
-        'm': 60,
-        'h': 3600,
-        'd': 3600 * 24,
+    "s": 1,
+    "ms": 1e-3,
+    "us": 1e-6,
+    "ns": 1e-9,
+    "m": 60,
+    "h": 3600,
+    "d": 3600 * 24,
 }
 
 tds2 = {
-        'second': 1,
-        'minute': 60,
-        'hour': 60 * 60,
-        'day': 60 * 60 * 24,
-        'millisecond': 1e-3,
-        'microsecond': 1e-6,
-        'nanosecond': 1e-9,
+    "second": 1,
+    "minute": 60,
+    "hour": 60 * 60,
+    "day": 60 * 60 * 24,
+    "millisecond": 1e-3,
+    "microsecond": 1e-6,
+    "nanosecond": 1e-9,
 }
-tds2.update({k + 's': v for k, v in tds2.items()})
+tds2.update({k + "s": v for k, v in tds2.items()})
 timedelta_sizes.update(tds2)
 timedelta_sizes.update({k.upper(): v for k, v in timedelta_sizes.items()})
 
 
-def parse_timedelta(s, default='seconds'):
+def parse_timedelta(s, default="seconds"):
     """ Parse timedelta string to number of seconds
 
     Examples
@@ -1255,9 +1284,9 @@ def parse_timedelta(s, default='seconds'):
         return s.total_seconds()
     if isinstance(s, Number):
         s = str(s)
-    s = s.replace(' ', '')
+    s = s.replace(" ", "")
     if not s[0].isdigit():
-        s = '1' + s
+        s = "1" + s
 
     for i in range(len(s) - 1, -1, -1):
         if not s[i].isalpha():
@@ -1290,16 +1319,16 @@ def asciitable(columns, rows):
     """
     rows = [tuple(str(i) for i in r) for r in rows]
     columns = tuple(str(i) for i in columns)
-    widths = tuple(max(max(map(len, x)), len(c))
-                   for x, c in zip(zip(*rows), columns))
-    row_template = ('|' + (' %%-%ds |' * len(columns))) % widths
+    widths = tuple(max(max(map(len, x)), len(c)) for x, c in zip(zip(*rows), columns))
+    row_template = ("|" + (" %%-%ds |" * len(columns))) % widths
     header = row_template % tuple(columns)
-    bar = '+%s+' % '+'.join('-' * (w + 2) for w in widths)
-    data = '\n'.join(row_template % r for r in rows)
-    return '\n'.join([bar, header, bar, data, bar])
+    bar = "+%s+" % "+".join("-" * (w + 2) for w in widths)
+    data = "\n".join(row_template % r for r in rows)
+    return "\n".join([bar, header, bar, data, bar])
 
 
 if PY2:
+
     def nbytes(frame, _bytes_like=(bytes, bytearray, buffer)):  # noqa: F821
         """ Number of bytes of a frame or memoryview """
         if isinstance(frame, _bytes_like):
@@ -1308,11 +1337,13 @@ def nbytes(frame, _bytes_like=(bytes, bytearray, buffer)):  # noqa: F821
             if frame.shape is None:
                 return frame.itemsize
             else:
-                return functools.reduce(operator.mul, frame.shape,
-                                        frame.itemsize)
+                return functools.reduce(operator.mul, frame.shape, frame.itemsize)
         else:
             return frame.nbytes
+
+
 else:
+
     def nbytes(frame, _bytes_like=(bytes, bytearray)):
         """ Number of bytes of a frame or memoryview """
         if isinstance(frame, _bytes_like):
@@ -1341,7 +1372,7 @@ def time_warn(duration, text):
     yield
     end = time()
     if end - start > duration:
-        print('TIME WARNING', text, end - start)
+        print("TIME WARNING", text, end - start)
 
 
 def json_load_robust(fn, load=json.load):
@@ -1372,18 +1403,19 @@ def format_time(n):
     '123.46 s'
     """
     if n >= 1:
-        return '%.2f s' % n
+        return "%.2f s" % n
     if n >= 1e-3:
-        return '%.2f ms' % (n * 1e3)
-    return '%.2f us' % (n * 1e6)
+        return "%.2f ms" % (n * 1e3)
+    return "%.2f us" % (n * 1e6)
 
 
 class DequeHandler(logging.Handler):
     """ A logging.Handler that records records into a deque """
+
     _instances = weakref.WeakSet()
 
     def __init__(self, *args, **kwargs):
-        n = kwargs.pop('n', 10000)
+        n = kwargs.pop("n", 10000)
         self.deque = deque(maxlen=n)
         super(DequeHandler, self).__init__(*args, **kwargs)
         self._instances.add(self)
@@ -1417,22 +1449,25 @@ def reset_logger_locks():
 
 
 # Only bother if asyncio has been loaded by Tornado
-if 'asyncio' in sys.modules and tornado.version_info[0] >= 5:
+if "asyncio" in sys.modules and tornado.version_info[0] >= 5:
 
     jupyter_event_loop_initialized = False
 
-    if 'notebook' in sys.modules:
+    if "notebook" in sys.modules:
         import traitlets
         from notebook.notebookapp import NotebookApp
-        jupyter_event_loop_initialized = (
-            traitlets.config.Application.initialized() and
-            isinstance(traitlets.config.Application.instance(), NotebookApp)
+
+        jupyter_event_loop_initialized = traitlets.config.Application.initialized() and isinstance(
+            traitlets.config.Application.instance(), NotebookApp
         )
 
     if not jupyter_event_loop_initialized:
         import asyncio
         import tornado.platform.asyncio
-        asyncio.set_event_loop_policy(tornado.platform.asyncio.AnyThreadEventLoopPolicy())
+
+        asyncio.set_event_loop_policy(
+            tornado.platform.asyncio.AnyThreadEventLoopPolicy()
+        )
 
 
 def has_keyword(func, keyword):
@@ -1451,9 +1486,26 @@ def has_keyword(func, keyword):
 
 # from bokeh.palettes import viridis
 # palette = viridis(18)
-palette = ['#440154', '#471669', '#472A79', '#433C84', '#3C4D8A', '#355D8C',
-           '#2E6C8E', '#287A8E', '#23898D', '#1E978A', '#20A585', '#2EB27C',
-           '#45BF6F', '#64CB5D', '#88D547', '#AFDC2E', '#D7E219', '#FDE724']
+palette = [
+    "#440154",
+    "#471669",
+    "#472A79",
+    "#433C84",
+    "#3C4D8A",
+    "#355D8C",
+    "#2E6C8E",
+    "#287A8E",
+    "#23898D",
+    "#1E978A",
+    "#20A585",
+    "#2EB27C",
+    "#45BF6F",
+    "#64CB5D",
+    "#88D547",
+    "#AFDC2E",
+    "#D7E219",
+    "#FDE724",
+]
 
 
 @toolz.memoize
diff --git a/distributed/utils_comm.py b/distributed/utils_comm.py
index 63622044291..d2bd19908af 100644
--- a/distributed/utils_comm.py
+++ b/distributed/utils_comm.py
@@ -32,6 +32,7 @@ def gather_from_workers(who_has, rpc, close=True, serializers=None, who=None):
     _gather
     """
     from .worker import get_data_from_worker
+
     bad_addresses = set()
     missing_workers = set()
     original_who_has = who_has
@@ -57,11 +58,17 @@ def gather_from_workers(who_has, rpc, close=True, serializers=None, who=None):
 
         rpcs = {addr: rpc(addr) for addr in d}
         try:
-            coroutines = {address: get_data_from_worker(rpc, keys, address,
-                                                        who=who,
-                                                        serializers=serializers,
-                                                        max_connections=False)
-                          for address, keys in d.items()}
+            coroutines = {
+                address: get_data_from_worker(
+                    rpc,
+                    keys,
+                    address,
+                    who=who,
+                    serializers=serializers,
+                    max_connections=False,
+                )
+                for address, keys in d.items()
+            }
             response = {}
             for worker, c in coroutines.items():
                 try:
@@ -69,7 +76,7 @@ def gather_from_workers(who_has, rpc, close=True, serializers=None, who=None):
                 except EnvironmentError:
                     missing_workers.add(worker)
                 else:
-                    response.update(r['data'])
+                    response.update(r["data"])
         finally:
             for r in rpcs.values():
                 r.close_rpc()
@@ -91,6 +98,7 @@ class WrappedKey(object):
     only be accessed in a certain way.  Schedulers may have particular needs
     that can only be addressed by additional metadata.
     """
+
     def __init__(self, key):
         self.key = key
 
@@ -122,19 +130,23 @@ def scatter_to_workers(ncores, data, rpc=rpc, report=True, serializers=None):
 
     L = list(zip(worker_iter, names, data))
     d = groupby(0, L)
-    d = {worker: {key: value for _, key, value in v}
-         for worker, v in d.items()}
+    d = {worker: {key: value for _, key, value in v} for worker, v in d.items()}
 
     rpcs = {addr: rpc(addr) for addr in d}
     try:
-        out = yield All([rpcs[address].update_data(data=v, report=report,
-                                                   serializers=serializers)
-                         for address, v in d.items()])
+        out = yield All(
+            [
+                rpcs[address].update_data(
+                    data=v, report=report, serializers=serializers
+                )
+                for address, v in d.items()
+            ]
+        )
     finally:
         for r in rpcs.values():
             r.close_rpc()
 
-    nbytes = merge(o['nbytes'] for o in out)
+    nbytes = merge(o["nbytes"] for o in out)
 
     who_has = {k: [w for w, _, _ in v] for k, v in groupby(1, L).items()}
 
@@ -184,15 +196,23 @@ def unpack_remotedata(o, byte_keys=False, myset=None):
         if type(o[0]) is SubgraphCallable:
             sc = o[0]
             futures = set()
-            dsk = {k: unpack_remotedata(v, byte_keys, futures)
-                   for k, v in sc.dsk.items()}
+            dsk = {
+                k: unpack_remotedata(v, byte_keys, futures) for k, v in sc.dsk.items()
+            }
             args = tuple(unpack_remotedata(i, byte_keys, futures) for i in o[1:])
             if futures:
                 myset.update(futures)
-                futures = (tuple(tokey(f.key) for f in futures)
-                           if byte_keys else tuple(f.key for f in futures))
+                futures = (
+                    tuple(tokey(f.key) for f in futures)
+                    if byte_keys
+                    else tuple(f.key for f in futures)
+                )
                 inkeys = sc.inkeys + futures
-                return (SubgraphCallable(dsk, sc.outkey, inkeys, sc.name),) + args + futures
+                return (
+                    (SubgraphCallable(dsk, sc.outkey, inkeys, sc.name),)
+                    + args
+                    + futures
+                )
             else:
                 return o
         else:
diff --git a/distributed/utils_perf.py b/distributed/utils_perf.py
index c6c1d37b107..9f300c5f567 100644
--- a/distributed/utils_perf.py
+++ b/distributed/utils_perf.py
@@ -26,6 +26,7 @@ class ThrottledGC(object):
     to log a warning level message whenever an actual call to gc.collect()
     lasts too long.
     """
+
     def __init__(self, max_in_gc_frac=0.05, warn_if_longer=1, logger=None):
         self.max_in_gc_frac = max_in_gc_frac
         self.warn_if_longer = warn_if_longer
@@ -41,25 +42,30 @@ def collect(self):
         collect_start = thread_time()
         elapsed = max(collect_start - self.last_collect, MIN_RUNTIME)
         if self.last_gc_duration / elapsed < self.max_in_gc_frac:
-            self.logger.debug("Calling gc.collect(). %0.3fs elapsed since "
-                              "previous call.", elapsed)
+            self.logger.debug(
+                "Calling gc.collect(). %0.3fs elapsed since " "previous call.", elapsed
+            )
             gc.collect()
             self.last_collect = collect_start
             self.last_gc_duration = max(thread_time() - collect_start, MIN_RUNTIME)
             if self.last_gc_duration > self.warn_if_longer:
-                self.logger.warning("gc.collect() took %0.3fs. This is usually"
-                                    " a sign that the some tasks handle too"
-                                    " many Python objects at the same time."
-                                    " Rechunking the work into smaller tasks"
-                                    " might help.",
-                                    self.last_gc_duration)
+                self.logger.warning(
+                    "gc.collect() took %0.3fs. This is usually"
+                    " a sign that the some tasks handle too"
+                    " many Python objects at the same time."
+                    " Rechunking the work into smaller tasks"
+                    " might help.",
+                    self.last_gc_duration,
+                )
             else:
-                self.logger.debug("gc.collect() took %0.3fs",
-                                  self.last_gc_duration)
+                self.logger.debug("gc.collect() took %0.3fs", self.last_gc_duration)
         else:
-            self.logger.debug("gc.collect() lasts %0.3fs but only %0.3fs "
-                              "elapsed since last call: throttling.",
-                              self.last_gc_duration, elapsed)
+            self.logger.debug(
+                "gc.collect() lasts %0.3fs but only %0.3fs "
+                "elapsed since last call: throttling.",
+                self.last_gc_duration,
+                elapsed,
+            )
 
 
 class FractionalTimer(object):
@@ -178,33 +184,42 @@ def __exit__(self, *args):
     def _gc_callback(self, phase, info):
         # Young generations are small and collected very often,
         # don't waste time measuring them
-        if info['generation'] != 2:
+        if info["generation"] != 2:
             return
         if self._proc is not None:
             rss = self._proc.memory_info().rss
         else:
             rss = 0
-        if phase == 'start':
+        if phase == "start":
             self._fractional_timer.start_timing()
             self._gc_rss_before = rss
             return
-        assert phase == 'stop'
+        assert phase == "stop"
         self._fractional_timer.stop_timing()
         frac = self._fractional_timer.running_fraction
         if frac is not None and frac >= self._warn_over_frac:
-            logger.warning("full garbage collections took %d%% CPU time "
-                           "recently (threshold: %d%%)",
-                           100 * frac, 100 * self._warn_over_frac)
+            logger.warning(
+                "full garbage collections took %d%% CPU time "
+                "recently (threshold: %d%%)",
+                100 * frac,
+                100 * self._warn_over_frac,
+            )
         rss_saved = self._gc_rss_before - rss
         if rss_saved >= self._info_over_rss_win:
-            logger.info("full garbage collection released %s "
-                        "from %d reference cycles (threshold: %s)",
-                        format_bytes(rss_saved), info['collected'],
-                        format_bytes(self._info_over_rss_win))
-        if info['uncollectable'] > 0:
+            logger.info(
+                "full garbage collection released %s "
+                "from %d reference cycles (threshold: %s)",
+                format_bytes(rss_saved),
+                info["collected"],
+                format_bytes(self._info_over_rss_win),
+            )
+        if info["uncollectable"] > 0:
             # This should ideally never happen on Python 3, but who knows?
-            logger.warning("garbage collector couldn't collect %d objects, "
-                           "please look in gc.garbage", info['uncollectable'])
+            logger.warning(
+                "garbage collector couldn't collect %d objects, "
+                "please look in gc.garbage",
+                info["uncollectable"],
+            )
 
 
 _gc_diagnosis = GCDiagnosis()
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index 11f4df047e9..a3f76e4c477 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -48,9 +48,18 @@
 from .process import _cleanup_dangling
 from .proctitle import enable_proctitle_on_children
 from .security import Security
-from .utils import (ignoring, log_errors, mp_context, get_ip, get_ipv6,
-                    DequeHandler, reset_logger_locks, sync,
-                    iscoroutinefunction, thread_state)
+from .utils import (
+    ignoring,
+    log_errors,
+    mp_context,
+    get_ip,
+    get_ipv6,
+    DequeHandler,
+    reset_logger_locks,
+    sync,
+    iscoroutinefunction,
+    thread_state,
+)
 from .worker import Worker, TOTAL_MEMORY, _global_workers
 
 try:
@@ -62,33 +71,38 @@
 logger = logging.getLogger(__name__)
 
 
-logging_levels = {name: logger.level for name, logger in
-                  logging.root.manager.loggerDict.items()
-                  if isinstance(logger, logging.Logger)}
+logging_levels = {
+    name: logger.level
+    for name, logger in logging.root.manager.loggerDict.items()
+    if isinstance(logger, logging.Logger)
+}
 
 
 offload(lambda: None).result()  # create thread during import
 
 
-@pytest.fixture(scope='session')
+@pytest.fixture(scope="session")
 def valid_python_script(tmpdir_factory):
-    local_file = tmpdir_factory.mktemp('data').join('file.py')
+    local_file = tmpdir_factory.mktemp("data").join("file.py")
     local_file.write("print('hello world!')")
     return local_file
 
 
-@pytest.fixture(scope='session')
+@pytest.fixture(scope="session")
 def client_contract_script(tmpdir_factory):
-    local_file = tmpdir_factory.mktemp('data').join('distributed_script.py')
-    lines = ("from distributed import Client", "e = Client('127.0.0.1:8989')",
-             'print(e)')
-    local_file.write('\n'.join(lines))
+    local_file = tmpdir_factory.mktemp("data").join("distributed_script.py")
+    lines = (
+        "from distributed import Client",
+        "e = Client('127.0.0.1:8989')",
+        "print(e)",
+    )
+    local_file.write("\n".join(lines))
     return local_file
 
 
-@pytest.fixture(scope='session')
+@pytest.fixture(scope="session")
 def invalid_python_script(tmpdir_factory):
-    local_file = tmpdir_factory.mktemp('data').join('file.py')
+    local_file = tmpdir_factory.mktemp("data").join("file.py")
     local_file.write("a+1")
     return local_file
 
@@ -116,6 +130,7 @@ def start():
                 orig_start()
             finally:
                 is_stopped.set()
+
         loop.start = start
 
         yield loop
@@ -151,8 +166,7 @@ def start():
 @pytest.fixture
 def loop_in_thread():
     with pristine_loop() as loop:
-        thread = threading.Thread(target=loop.start,
-                                  name="test IOLoop")
+        thread = threading.Thread(target=loop.start, name="test IOLoop")
         thread.daemon = True
         thread.start()
         loop_started = threading.Event()
@@ -166,6 +180,7 @@ def loop_in_thread():
 @pytest.fixture
 def zmq_ctx():
     import zmq
+
     ctx = zmq.Context.instance()
     yield ctx
     ctx.destroy(linger=0)
@@ -193,6 +208,7 @@ def pristine_loop():
 def mock_ipython():
     import mock
     from distributed._ipython_utils import remote_magic
+
     ip = mock.Mock()
     ip.user_ns = {}
     ip.kernel = None
@@ -200,8 +216,9 @@ def mock_ipython():
     def get_ip():
         return ip
 
-    with mock.patch('IPython.get_ipython', get_ip), \
-            mock.patch('distributed._ipython_utils.get_ipython', get_ip):
+    with mock.patch("IPython.get_ipython", get_ip), mock.patch(
+        "distributed._ipython_utils.get_ipython", get_ip
+    ):
         yield ip
     # cleanup remote_magic client cache
     for kc in remote_magic._clients.values():
@@ -285,7 +302,7 @@ def deep(n):
 
 
 def throws(x):
-    raise RuntimeError('hello!')
+    raise RuntimeError("hello!")
 
 
 def double(x):
@@ -309,6 +326,7 @@ def slowdouble(x, delay=0.02):
 
 def randominc(x, scale=1):
     from random import random
+
     sleep(random() * scale)
     return x + 1
 
@@ -324,7 +342,7 @@ def slowsum(seq, delay=0.02):
 
 
 def slowidentity(*args, **kwargs):
-    delay = kwargs.get('delay', 0.02)
+    delay = kwargs.get("delay", 0.02)
     sleep(delay)
     if len(args) == 1:
         return args[0]
@@ -364,7 +382,7 @@ def varying(items):
     # used by *func* below, so we can't use `global <something>`.
     # Instead look up the module by name to get the original namespace
     # and not a copy.
-    slot = _ModuleSlot(__name__, '_varying_dict')
+    slot = _ModuleSlot(__name__, "_varying_dict")
     key = next(_varying_key_gen)
 
     def func():
@@ -388,6 +406,7 @@ def map_varying(itemslists):
     Like *varying*, but return the full specification for a map() call
     on multiple items lists.
     """
+
     def apply(func, *args, **kwargs):
         return func(*args, **kwargs)
 
@@ -403,17 +422,19 @@ def geninc(x, delay=0.02):
 def compile_snippet(code, dedent=True):
     if dedent:
         code = textwrap.dedent(code)
-    code = compile(code, '<dynamic>', 'exec')
+    code = compile(code, "<dynamic>", "exec")
     ns = globals()
     exec(code, ns, ns)
 
 
 if sys.version_info >= (3, 5):
-    compile_snippet("""
+    compile_snippet(
+        """
         async def asyncinc(x, delay=0.02):
             await gen.sleep(delay)
             return x + 1
-        """)
+        """
+    )
     assert asyncinc  # noqa: F821
 else:
     asyncinc = None
@@ -461,7 +482,7 @@ def run_scheduler(q, nputs, **kwargs):
     # so avoid inheriting the parent's IO loop.
     with pristine_loop() as loop:
         scheduler = Scheduler(validate=True, **kwargs)
-        done = scheduler.start('127.0.0.1')
+        done = scheduler.start("127.0.0.1")
 
         for i in range(nputs):
             q.put(scheduler.address)
@@ -482,6 +503,7 @@ def run_worker(q, scheduler_q, **kwargs):
             loop.run_sync(lambda: worker._start(0))
             q.put(worker.address)
             try:
+
                 @gen.coroutine
                 def wait_until_closed():
                     yield worker._closed.wait()
@@ -523,13 +545,17 @@ def check_active_rpc(loop, active_rpc_timeout=1):
     # (*) (example: gather_from_workers())
 
     def fail():
-        pytest.fail("some RPCs left active by test: %s"
-                    % (set(rpc.active) - active_before))
+        pytest.fail(
+            "some RPCs left active by test: %s" % (set(rpc.active) - active_before)
+        )
 
     @gen.coroutine
     def wait():
-        yield async_wait_for(lambda: len(set(rpc.active) - active_before) == 0,
-                             timeout=active_rpc_timeout, fail_func=fail)
+        yield async_wait_for(
+            lambda: len(set(rpc.active) - active_before) == 0,
+            timeout=active_rpc_timeout,
+            fail_func=fail,
+        )
 
     loop.run_sync(wait)
 
@@ -561,27 +587,28 @@ def b(cluster_fixture):
 @pytest.fixture
 def client(loop, cluster_fixture):
     scheduler, workers = cluster_fixture
-    with Client(scheduler['address'], loop=loop) as client:
+    with Client(scheduler["address"], loop=loop) as client:
         yield client
 
 
 @pytest.fixture
 def client_secondary(loop, cluster_fixture):
     scheduler, workers = cluster_fixture
-    with Client(scheduler['address'], loop=loop) as client:
+    with Client(scheduler["address"], loop=loop) as client:
         yield client
 
 
 @contextmanager
-def tls_cluster_context(worker_kwargs=None, scheduler_kwargs=None,
-                        security=None, **kwargs):
+def tls_cluster_context(
+    worker_kwargs=None, scheduler_kwargs=None, security=None, **kwargs
+):
     security = security or tls_only_security()
-    worker_kwargs = assoc(worker_kwargs or {}, 'security', security)
-    scheduler_kwargs = assoc(scheduler_kwargs or {}, 'security', security)
+    worker_kwargs = assoc(worker_kwargs or {}, "security", security)
+    scheduler_kwargs = assoc(scheduler_kwargs or {}, "security", security)
 
-    with cluster(worker_kwargs=worker_kwargs,
-                 scheduler_kwargs=scheduler_kwargs,
-                 **kwargs) as (s, workers):
+    with cluster(
+        worker_kwargs=worker_kwargs, scheduler_kwargs=scheduler_kwargs, **kwargs
+    ) as (s, workers):
         yield s, workers
 
 
@@ -594,7 +621,7 @@ def tls_cluster(loop, security):
 @pytest.fixture
 def tls_client(tls_cluster, loop, security):
     s, workers = tls_cluster
-    with Client(s['address'], security=security, loop=loop) as client:
+    with Client(s["address"], security=security, loop=loop) as client:
         yield client
 
 
@@ -604,8 +631,9 @@ def security():
 
 
 @contextmanager
-def cluster(nworkers=2, nanny=False, worker_kwargs={}, active_rpc_timeout=1,
-            scheduler_kwargs={}):
+def cluster(
+    nworkers=2, nanny=False, worker_kwargs={}, active_rpc_timeout=1, scheduler_kwargs={}
+):
     ws = weakref.WeakSet()
 
     reset_config()
@@ -626,9 +654,11 @@ def cluster(nworkers=2, nanny=False, worker_kwargs={}, active_rpc_timeout=1,
             scheduler_q = mp_context.Queue()
 
             # Launch scheduler
-            scheduler = mp_context.Process(target=run_scheduler,
-                                           args=(scheduler_q, nworkers + 1),
-                                           kwargs=scheduler_kwargs)
+            scheduler = mp_context.Process(
+                target=run_scheduler,
+                args=(scheduler_q, nworkers + 1),
+                kwargs=scheduler_kwargs,
+            )
             ws.add(scheduler)
             scheduler.daemon = True
             scheduler.start()
@@ -637,20 +667,22 @@ def cluster(nworkers=2, nanny=False, worker_kwargs={}, active_rpc_timeout=1,
             workers = []
             for i in range(nworkers):
                 q = mp_context.Queue()
-                fn = '_test_worker-%s' % uuid.uuid4()
-                kwargs = merge({'ncores': 1, 'local_dir': fn,
-                                'memory_limit': TOTAL_MEMORY}, worker_kwargs)
-                proc = mp_context.Process(target=_run_worker,
-                                          args=(q, scheduler_q),
-                                          kwargs=kwargs)
+                fn = "_test_worker-%s" % uuid.uuid4()
+                kwargs = merge(
+                    {"ncores": 1, "local_dir": fn, "memory_limit": TOTAL_MEMORY},
+                    worker_kwargs,
+                )
+                proc = mp_context.Process(
+                    target=_run_worker, args=(q, scheduler_q), kwargs=kwargs
+                )
                 ws.add(proc)
-                workers.append({'proc': proc, 'queue': q, 'dir': fn})
+                workers.append({"proc": proc, "queue": q, "dir": fn})
 
             for worker in workers:
-                worker['proc'].start()
+                worker["proc"].start()
             try:
                 for worker in workers:
-                    worker['address'] = worker['queue'].get(timeout=5)
+                    worker["address"] = worker["queue"].get(timeout=5)
             except Empty:
                 raise pytest.xfail.Exception("Worker failed to start in test")
 
@@ -659,8 +691,10 @@ def cluster(nworkers=2, nanny=False, worker_kwargs={}, active_rpc_timeout=1,
             start = time()
             try:
                 try:
-                    security = scheduler_kwargs['security']
-                    rpc_kwargs = {'connection_args': security.get_connection_args('client')}
+                    security = scheduler_kwargs["security"]
+                    rpc_kwargs = {
+                        "connection_args": security.get_connection_args("client")
+                    }
                 except KeyError:
                     rpc_kwargs = {}
 
@@ -673,16 +707,23 @@ def cluster(nworkers=2, nanny=False, worker_kwargs={}, active_rpc_timeout=1,
                             raise Exception("Timeout on cluster creation")
 
                 # avoid sending processes down to function
-                yield {'address': saddr}, [{'address': w['address'],
-                                            'proc': weakref.ref(w['proc'])}
-                                           for w in workers]
+                yield {"address": saddr}, [
+                    {"address": w["address"], "proc": weakref.ref(w["proc"])}
+                    for w in workers
+                ]
             finally:
                 logger.debug("Closing out test cluster")
 
-                loop.run_sync(lambda: disconnect_all([w['address'] for w in workers],
-                                                     timeout=0.5,
-                                                     rpc_kwargs=rpc_kwargs))
-                loop.run_sync(lambda: disconnect(saddr, timeout=0.5, rpc_kwargs=rpc_kwargs))
+                loop.run_sync(
+                    lambda: disconnect_all(
+                        [w["address"] for w in workers],
+                        timeout=0.5,
+                        rpc_kwargs=rpc_kwargs,
+                    )
+                )
+                loop.run_sync(
+                    lambda: disconnect(saddr, timeout=0.5, rpc_kwargs=rpc_kwargs)
+                )
 
                 scheduler.terminate()
                 scheduler_q.close()
@@ -690,21 +731,21 @@ def cluster(nworkers=2, nanny=False, worker_kwargs={}, active_rpc_timeout=1,
                 scheduler_q._writer.close()
 
                 for w in workers:
-                    w['proc'].terminate()
-                    w['queue'].close()
-                    w['queue']._reader.close()
-                    w['queue']._writer.close()
+                    w["proc"].terminate()
+                    w["queue"].close()
+                    w["queue"]._reader.close()
+                    w["queue"]._writer.close()
 
                 scheduler.join(2)
                 del scheduler
-                for proc in [w['proc'] for w in workers]:
+                for proc in [w["proc"] for w in workers]:
                     proc.join(timeout=2)
 
                 with ignoring(UnboundLocalError):
                     del worker, w, proc
                 del workers[:]
 
-                for fn in glob('_test_worker-*'):
+                for fn in glob("_test_worker-*"):
                     with ignoring(OSError):
                         shutil.rmtree(fn)
 
@@ -718,7 +759,7 @@ def cluster(nworkers=2, nanny=False, worker_kwargs={}, active_rpc_timeout=1,
     start = time()
     while list(ws):
         sleep(0.01)
-        assert time() < start + 1, 'Workers still around after one second'
+        assert time() < start + 1, "Workers still around after one second"
 
 
 @gen.coroutine
@@ -758,6 +799,7 @@ def gen_test(timeout=10):
     def test_foo():
         yield ...  # use tornado coroutines
     """
+
     def _(func):
         def test_func():
             with pristine_loop() as loop:
@@ -769,7 +811,9 @@ def test_func():
                     loop.run_sync(cor, timeout=timeout)
                 finally:
                     loop.stop()
+
         return test_func
+
     return _
 
 
@@ -778,25 +822,38 @@ def test_func():
 
 
 @gen.coroutine
-def start_cluster(ncores, scheduler_addr, loop, security=None,
-                  Worker=Worker, scheduler_kwargs={}, worker_kwargs={}):
-    s = Scheduler(loop=loop, validate=True, security=security,
-                  **scheduler_kwargs)
+def start_cluster(
+    ncores,
+    scheduler_addr,
+    loop,
+    security=None,
+    Worker=Worker,
+    scheduler_kwargs={},
+    worker_kwargs={},
+):
+    s = Scheduler(loop=loop, validate=True, security=security, **scheduler_kwargs)
     done = s.start(scheduler_addr)
-    workers = [Worker(s.address, ncores=ncore[1], name=i, security=security,
-                      loop=loop, validate=True,
-                      **(merge(worker_kwargs, ncore[2])
-                         if len(ncore) > 2
-                         else worker_kwargs))
-               for i, ncore in enumerate(ncores)]
+    workers = [
+        Worker(
+            s.address,
+            ncores=ncore[1],
+            name=i,
+            security=security,
+            loop=loop,
+            validate=True,
+            **(merge(worker_kwargs, ncore[2]) if len(ncore) > 2 else worker_kwargs)
+        )
+        for i, ncore in enumerate(ncores)
+    ]
     for w in workers:
         w.rpc = workers[0].rpc
 
     yield [w._start(ncore[0]) for ncore, w in zip(ncores, workers)]
 
     start = time()
-    while (len(s.workers) < len(ncores) or
-           any(comm.comm is None for comm in s.stream_comms.values())):
+    while len(s.workers) < len(ncores) or any(
+        comm.comm is None for comm in s.stream_comms.values()
+    ):
         yield gen.sleep(0.01)
         if time() - start > 5:
             yield [w._close(timeout=1) for w in workers]
@@ -819,12 +876,22 @@ def end_worker(w):
     s.stop()
 
 
-def gen_cluster(ncores=[('127.0.0.1', 1), ('127.0.0.1', 2)],
-                scheduler='127.0.0.1', timeout=10, security=None,
-                Worker=Worker, client=False, scheduler_kwargs={},
-                worker_kwargs={}, client_kwargs={}, active_rpc_timeout=1,
-                config={}, check_new_threads=True):
+def gen_cluster(
+    ncores=[("127.0.0.1", 1), ("127.0.0.1", 2)],
+    scheduler="127.0.0.1",
+    timeout=10,
+    security=None,
+    Worker=Worker,
+    client=False,
+    scheduler_kwargs={},
+    worker_kwargs={},
+    client_kwargs={},
+    active_rpc_timeout=1,
+    config={},
+    check_new_threads=True,
+):
     from distributed import Client
+
     """ Coroutine test with small cluster
 
     @gen_cluster()
@@ -835,8 +902,9 @@ def test_foo(scheduler, worker1, worker2):
         start
         end
     """
-    worker_kwargs = merge({'memory_limit': TOTAL_MEMORY, 'death_timeout': 5},
-                          worker_kwargs)
+    worker_kwargs = merge(
+        {"memory_limit": TOTAL_MEMORY, "death_timeout": 5}, worker_kwargs
+    )
 
     def _(func):
         if not iscoroutinefunction(func):
@@ -849,7 +917,7 @@ def test_func():
 
             reset_config()
 
-            dask.config.set({'distributed.comm.timeouts.connect': '5s'})
+            dask.config.set({"distributed.comm.timeouts.connect": "5s"})
             # Restore default logging levels
             # XXX use pytest hooks/fixtures instead?
             for name, level in logging_levels.items():
@@ -860,6 +928,7 @@ def test_func():
 
             with pristine_loop() as loop:
                 with check_active_rpc(loop, active_rpc_timeout):
+
                     @gen.coroutine
                     def coro():
                         with dask.config.set(config):
@@ -867,11 +936,19 @@ def coro():
                             for i in range(5):
                                 try:
                                     s, ws = yield start_cluster(
-                                        ncores, scheduler, loop, security=security,
-                                        Worker=Worker, scheduler_kwargs=scheduler_kwargs,
-                                        worker_kwargs=worker_kwargs)
+                                        ncores,
+                                        scheduler,
+                                        loop,
+                                        security=security,
+                                        Worker=Worker,
+                                        scheduler_kwargs=scheduler_kwargs,
+                                        worker_kwargs=worker_kwargs,
+                                    )
                                 except Exception as e:
-                                    logger.error("Failed to start gen_cluster, retrying", exc_info=True)
+                                    logger.error(
+                                        "Failed to start gen_cluster, retrying",
+                                        exc_info=True,
+                                    )
                                 else:
                                     workers[:] = ws
                                     args = [s] + workers
@@ -879,23 +956,30 @@ def coro():
                             if s is False:
                                 raise Exception("Could not start cluster")
                             if client:
-                                c = yield Client(s.address, loop=loop, security=security,
-                                                 asynchronous=True, **client_kwargs)
+                                c = yield Client(
+                                    s.address,
+                                    loop=loop,
+                                    security=security,
+                                    asynchronous=True,
+                                    **client_kwargs
+                                )
                                 args = [c] + args
                             try:
                                 future = func(*args)
                                 if timeout:
-                                    future = gen.with_timeout(timedelta(seconds=timeout),
-                                                              future)
+                                    future = gen.with_timeout(
+                                        timedelta(seconds=timeout), future
+                                    )
                                 result = yield future
                                 if s.validate:
                                     s.validate_state()
                             finally:
                                 if client:
-                                    yield c._close(fast=s.status == 'closed')
+                                    yield c._close(fast=s.status == "closed")
                                 yield end_cluster(s, workers)
-                                yield gen.with_timeout(timedelta(seconds=1),
-                                                       cleanup_global_workers())
+                                yield gen.with_timeout(
+                                    timedelta(seconds=1), cleanup_global_workers()
+                                )
 
                             try:
                                 c = yield default_client()
@@ -906,10 +990,12 @@ def coro():
 
                             raise gen.Return(result)
 
-                    result = loop.run_sync(coro, timeout=timeout * 2 if timeout else timeout)
+                    result = loop.run_sync(
+                        coro, timeout=timeout * 2 if timeout else timeout
+                    )
 
                 for w in workers:
-                    if getattr(w, 'data', None):
+                    if getattr(w, "data", None):
                         try:
                             w.data.clear()
                         except EnvironmentError:
@@ -921,24 +1007,28 @@ def coro():
                 for w in _global_workers:
                     w = w()
                     w._close(report=False, executor_wait=False)
-                    if w.status == 'running':
+                    if w.status == "running":
                         w.close()
                 del _global_workers[:]
 
             if PY3 and not WINDOWS and check_new_threads:
                 start = time()
                 while True:
-                    bad = [t for t, v in threading._active.items()
-                           if t not in active_threads_start and
-                          "Threaded" not in v.name and
-                          "watch message" not in v.name and
-                          "TCP-Executor" not in v.name]
+                    bad = [
+                        t
+                        for t, v in threading._active.items()
+                        if t not in active_threads_start
+                        and "Threaded" not in v.name
+                        and "watch message" not in v.name
+                        and "TCP-Executor" not in v.name
+                    ]
                     if not bad:
                         break
                     else:
                         sleep(0.01)
                     if time() > start + 5:
                         from distributed import profile
+
                         tid = bad[0]
                         thread = threading._active[tid]
                         call_stacks = profile.call_stack(sys._current_frames()[tid])
@@ -949,6 +1039,7 @@ def coro():
             return result
 
         return test_func
+
     return _
 
 
@@ -962,7 +1053,7 @@ def raises(func, exc=Exception):
 
 def terminate_process(proc):
     if proc.poll() is None:
-        if sys.platform.startswith('win'):
+        if sys.platform.startswith("win"):
             proc.send_signal(signal.CTRL_BREAK_EVENT)
         else:
             proc.send_signal(signal.SIGINT)
@@ -981,18 +1072,20 @@ def terminate_process(proc):
 
 @contextmanager
 def popen(args, **kwargs):
-    kwargs['stdout'] = subprocess.PIPE
-    kwargs['stderr'] = subprocess.PIPE
-    if sys.platform.startswith('win'):
+    kwargs["stdout"] = subprocess.PIPE
+    kwargs["stderr"] = subprocess.PIPE
+    if sys.platform.startswith("win"):
         # Allow using CTRL_C_EVENT / CTRL_BREAK_EVENT
-        kwargs['creationflags'] = subprocess.CREATE_NEW_PROCESS_GROUP
+        kwargs["creationflags"] = subprocess.CREATE_NEW_PROCESS_GROUP
     dump_stdout = False
 
     args = list(args)
-    if sys.platform.startswith('win'):
-        args[0] = os.path.join(sys.prefix, 'Scripts', args[0])
+    if sys.platform.startswith("win"):
+        args[0] = os.path.join(sys.prefix, "Scripts", args[0])
     else:
-        args[0] = os.path.join(os.environ.get('DESTDIR', '') + sys.prefix, 'bin', args[0])
+        args[0] = os.path.join(
+            os.environ.get("DESTDIR", "") + sys.prefix, "bin", args[0]
+        )
     proc = subprocess.Popen(args, **kwargs)
     try:
         yield proc
@@ -1007,10 +1100,10 @@ def popen(args, **kwargs):
             # XXX Also dump stdout if return code != 0 ?
             out, err = proc.communicate()
             if dump_stdout:
-                print('\n\nPrint from stderr\n  %s\n=================\n' % args[0][0])
+                print("\n\nPrint from stderr\n  %s\n=================\n" % args[0][0])
                 print(err.decode())
 
-                print('\n\nPrint from stdout\n=================\n')
+                print("\n\nPrint from stdout\n=================\n")
                 print(out.decode())
 
 
@@ -1061,7 +1154,7 @@ def has_ipv6():
     serv = cli = None
     try:
         serv = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
-        serv.bind(('::', 0))
+        serv.bind(("::", 0))
         serv.listen(5)
         cli = socket.create_connection(serv.getsockname()[:2])
     except EnvironmentError:
@@ -1076,9 +1169,11 @@ def has_ipv6():
 
 
 if has_ipv6():
+
     def requires_ipv6(test_func):
         return test_func
 
+
 else:
     requires_ipv6 = pytest.mark.skip("ipv6 required")
 
@@ -1091,13 +1186,14 @@ def assert_can_connect(addr, timeout=None, connection_args=None):
     """
     if timeout is None:
         timeout = 0.5
-    comm = yield connect(addr, timeout=timeout,
-                         connection_args=connection_args)
+    comm = yield connect(addr, timeout=timeout, connection_args=connection_args)
     comm.abort()
 
 
 @gen.coroutine
-def assert_cannot_connect(addr, timeout=None, connection_args=None, exception_class=EnvironmentError):
+def assert_cannot_connect(
+    addr, timeout=None, connection_args=None, exception_class=EnvironmentError
+):
     """
     Check that it is impossible to connect to the distributed *addr*
     within the given *timeout*.
@@ -1105,43 +1201,46 @@ def assert_cannot_connect(addr, timeout=None, connection_args=None, exception_cl
     if timeout is None:
         timeout = 0.5
     with pytest.raises(exception_class):
-        comm = yield connect(addr, timeout=timeout,
-                             connection_args=connection_args)
+        comm = yield connect(addr, timeout=timeout, connection_args=connection_args)
         comm.abort()
 
 
 @gen.coroutine
-def assert_can_connect_from_everywhere_4_6(port, timeout=None, connection_args=None, protocol='tcp'):
+def assert_can_connect_from_everywhere_4_6(
+    port, timeout=None, connection_args=None, protocol="tcp"
+):
     """
     Check that the local *port* is reachable from all IPv4 and IPv6 addresses.
     """
     args = (timeout, connection_args)
     futures = [
-        assert_can_connect('%s://127.0.0.1:%d' % (protocol, port), *args),
-        assert_can_connect('%s://%s:%d' % (protocol, get_ip(), port), *args),
+        assert_can_connect("%s://127.0.0.1:%d" % (protocol, port), *args),
+        assert_can_connect("%s://%s:%d" % (protocol, get_ip(), port), *args),
     ]
     if has_ipv6():
         futures += [
-            assert_can_connect('%s://[::1]:%d' % (protocol, port), *args),
-            assert_can_connect('%s://[%s]:%d' % (protocol, get_ipv6(), port), *args),
+            assert_can_connect("%s://[::1]:%d" % (protocol, port), *args),
+            assert_can_connect("%s://[%s]:%d" % (protocol, get_ipv6(), port), *args),
         ]
     yield futures
 
 
 @gen.coroutine
-def assert_can_connect_from_everywhere_4(port, timeout=None, connection_args=None, protocol='tcp'):
+def assert_can_connect_from_everywhere_4(
+    port, timeout=None, connection_args=None, protocol="tcp"
+):
     """
     Check that the local *port* is reachable from all IPv4 addresses.
     """
     args = (timeout, connection_args)
     futures = [
-        assert_can_connect('%s://127.0.0.1:%d' % (protocol, port), *args),
-        assert_can_connect('%s://%s:%d' % (protocol, get_ip(), port), *args),
+        assert_can_connect("%s://127.0.0.1:%d" % (protocol, port), *args),
+        assert_can_connect("%s://%s:%d" % (protocol, get_ip(), port), *args),
     ]
     if has_ipv6():
         futures += [
-            assert_cannot_connect('%s://[::1]:%d' % (protocol, port), *args),
-            assert_cannot_connect('%s://[%s]:%d' % (protocol, get_ipv6(), port), *args),
+            assert_cannot_connect("%s://[::1]:%d" % (protocol, port), *args),
+            assert_cannot_connect("%s://[%s]:%d" % (protocol, get_ipv6(), port), *args),
         ]
     yield futures
 
@@ -1152,17 +1251,13 @@ def assert_can_connect_locally_4(port, timeout=None, connection_args=None):
     Check that the local *port* is only reachable from local IPv4 addresses.
     """
     args = (timeout, connection_args)
-    futures = [
-        assert_can_connect('tcp://127.0.0.1:%d' % port, *args),
-    ]
-    if get_ip() != '127.0.0.1':  # No outside IPv4 connectivity?
-        futures += [
-            assert_cannot_connect('tcp://%s:%d' % (get_ip(), port), *args),
-        ]
+    futures = [assert_can_connect("tcp://127.0.0.1:%d" % port, *args)]
+    if get_ip() != "127.0.0.1":  # No outside IPv4 connectivity?
+        futures += [assert_cannot_connect("tcp://%s:%d" % (get_ip(), port), *args)]
     if has_ipv6():
         futures += [
-            assert_cannot_connect('tcp://[::1]:%d' % port, *args),
-            assert_cannot_connect('tcp://[%s]:%d' % (get_ipv6(), port), *args),
+            assert_cannot_connect("tcp://[::1]:%d" % port, *args),
+            assert_cannot_connect("tcp://[%s]:%d" % (get_ipv6(), port), *args),
         ]
     yield futures
 
@@ -1175,10 +1270,10 @@ def assert_can_connect_from_everywhere_6(port, timeout=None, connection_args=Non
     assert has_ipv6()
     args = (timeout, connection_args)
     futures = [
-        assert_cannot_connect('tcp://127.0.0.1:%d' % port, *args),
-        assert_cannot_connect('tcp://%s:%d' % (get_ip(), port), *args),
-        assert_can_connect('tcp://[::1]:%d' % port, *args),
-        assert_can_connect('tcp://[%s]:%d' % (get_ipv6(), port), *args),
+        assert_cannot_connect("tcp://127.0.0.1:%d" % port, *args),
+        assert_cannot_connect("tcp://%s:%d" % (get_ip(), port), *args),
+        assert_can_connect("tcp://[::1]:%d" % port, *args),
+        assert_can_connect("tcp://[%s]:%d" % (get_ipv6(), port), *args),
     ]
     yield futures
 
@@ -1191,14 +1286,12 @@ def assert_can_connect_locally_6(port, timeout=None, connection_args=None):
     assert has_ipv6()
     args = (timeout, connection_args)
     futures = [
-        assert_cannot_connect('tcp://127.0.0.1:%d' % port, *args),
-        assert_cannot_connect('tcp://%s:%d' % (get_ip(), port), *args),
-        assert_can_connect('tcp://[::1]:%d' % port, *args),
+        assert_cannot_connect("tcp://127.0.0.1:%d" % port, *args),
+        assert_cannot_connect("tcp://%s:%d" % (get_ip(), port), *args),
+        assert_can_connect("tcp://[::1]:%d" % port, *args),
     ]
-    if get_ipv6() != '::1':  # No outside IPv6 connectivity?
-        futures += [
-            assert_cannot_connect('tcp://[%s]:%d' % (get_ipv6(), port), *args),
-        ]
+    if get_ipv6() != "::1":  # No outside IPv6 connectivity?
+        futures += [assert_cannot_connect("tcp://[%s]:%d" % (get_ipv6(), port), *args)]
     yield futures
 
 
@@ -1244,6 +1337,7 @@ def new_config(new_config):
     Temporarily change configuration dictionary.
     """
     from .config import defaults
+
     config = dask.config.config
     orig_config = config.copy()
     try:
@@ -1275,25 +1369,25 @@ def new_config_file(c):
     Temporarily change configuration file to match dictionary *c*.
     """
     import yaml
-    old_file = os.environ.get('DASK_CONFIG')
-    fd, path = tempfile.mkstemp(prefix='dask-config')
+
+    old_file = os.environ.get("DASK_CONFIG")
+    fd, path = tempfile.mkstemp(prefix="dask-config")
     try:
-        with os.fdopen(fd, 'w') as f:
+        with os.fdopen(fd, "w") as f:
             f.write(yaml.dump(c))
-        os.environ['DASK_CONFIG'] = path
+        os.environ["DASK_CONFIG"] = path
         try:
             yield
         finally:
             if old_file:
-                os.environ['DASK_CONFIG'] = old_file
+                os.environ["DASK_CONFIG"] = old_file
             else:
-                del os.environ['DASK_CONFIG']
+                del os.environ["DASK_CONFIG"]
     finally:
         os.remove(path)
 
 
-certs_dir = os.path.abspath(os.path.join(os.path.dirname(__file__),
-                                         'tests'))
+certs_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "tests"))
 
 
 def get_cert(filename):
@@ -1309,22 +1403,16 @@ def tls_config():
     """
     A functional TLS configuration with our test certs.
     """
-    ca_file = get_cert('tls-ca-cert.pem')
-    keycert = get_cert('tls-key-cert.pem')
+    ca_file = get_cert("tls-ca-cert.pem")
+    keycert = get_cert("tls-key-cert.pem")
 
     c = {
-        'tls': {
-            'ca-file': ca_file,
-            'client': {
-                'cert': keycert,
-            },
-            'scheduler': {
-                'cert': keycert,
-            },
-            'worker': {
-                'cert': keycert,
-            },
-        },
+        "tls": {
+            "ca-file": ca_file,
+            "client": {"cert": keycert},
+            "scheduler": {"cert": keycert},
+            "worker": {"cert": keycert},
+        }
     }
     return c
 
@@ -1335,7 +1423,7 @@ def tls_only_config():
     plain TCP communications.
     """
     c = tls_config()
-    c['require-encryption'] = True
+    c["require-encryption"] = True
     return c
 
 
@@ -1359,20 +1447,20 @@ def tls_only_security():
     return sec
 
 
-def get_server_ssl_context(certfile='tls-cert.pem', keyfile='tls-key.pem',
-                           ca_file='tls-ca-cert.pem'):
-    ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH,
-                                     cafile=get_cert(ca_file))
+def get_server_ssl_context(
+    certfile="tls-cert.pem", keyfile="tls-key.pem", ca_file="tls-ca-cert.pem"
+):
+    ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH, cafile=get_cert(ca_file))
     ctx.check_hostname = False
     ctx.verify_mode = ssl.CERT_REQUIRED
     ctx.load_cert_chain(get_cert(certfile), get_cert(keyfile))
     return ctx
 
 
-def get_client_ssl_context(certfile='tls-cert.pem', keyfile='tls-key.pem',
-                           ca_file='tls-ca-cert.pem'):
-    ctx = ssl.create_default_context(ssl.Purpose.SERVER_AUTH,
-                                     cafile=get_cert(ca_file))
+def get_client_ssl_context(
+    certfile="tls-cert.pem", keyfile="tls-key.pem", ca_file="tls-ca-cert.pem"
+):
+    ctx = ssl.create_default_context(ssl.Purpose.SERVER_AUTH, cafile=get_cert(ca_file))
     ctx.check_hostname = False
     ctx.verify_mode = ssl.CERT_REQUIRED
     ctx.load_cert_chain(get_cert(certfile), get_cert(keyfile))
@@ -1380,18 +1468,17 @@ def get_client_ssl_context(certfile='tls-cert.pem', keyfile='tls-key.pem',
 
 
 def bump_rlimit(limit, desired):
-    resource = pytest.importorskip('resource')
+    resource = pytest.importorskip("resource")
     try:
         soft, hard = resource.getrlimit(limit)
         if soft < desired:
-            resource.setrlimit(limit,
-                               (desired, max(hard, desired)))
+            resource.setrlimit(limit, (desired, max(hard, desired)))
     except Exception as e:
-        pytest.skip("rlimit too low (%s) and can't be increased: %s"
-                    % (soft, e))
+        pytest.skip("rlimit too low (%s) and can't be increased: %s" % (soft, e))
 
 
 def gen_tls_cluster(**kwargs):
-    kwargs.setdefault('ncores', [('tls://127.0.0.1', 1), ('tls://127.0.0.1', 2)])
-    return gen_cluster(scheduler='tls://127.0.0.1',
-                       security=tls_only_security(), **kwargs)
+    kwargs.setdefault("ncores", [("tls://127.0.0.1", 1), ("tls://127.0.0.1", 2)])
+    return gen_cluster(
+        scheduler="tls://127.0.0.1", security=tls_only_security(), **kwargs
+    )
diff --git a/distributed/variable.py b/distributed/variable.py
index 5d905358a9e..7b775d3327a 100644
--- a/distributed/variable.py
+++ b/distributed/variable.py
@@ -37,27 +37,28 @@ def __init__(self, scheduler):
         self.waiting_conditions = defaultdict(tornado.locks.Condition)
         self.started = tornado.locks.Condition()
 
-        self.scheduler.handlers.update({'variable_set': self.set,
-                                        'variable_get': self.get})
+        self.scheduler.handlers.update(
+            {"variable_set": self.set, "variable_get": self.get}
+        )
 
-        self.scheduler.stream_handlers['variable-future-release'] = self.future_release
-        self.scheduler.stream_handlers['variable_delete'] = self.delete
+        self.scheduler.stream_handlers["variable-future-release"] = self.future_release
+        self.scheduler.stream_handlers["variable_delete"] = self.delete
 
-        self.scheduler.extensions['variables'] = self
+        self.scheduler.extensions["variables"] = self
 
     def set(self, stream=None, name=None, key=None, data=None, client=None):
         if key is not None:
-            record = {'type': 'Future', 'value': key}
-            self.scheduler.client_desires_keys(keys=[key], client='variable-%s' % name)
+            record = {"type": "Future", "value": key}
+            self.scheduler.client_desires_keys(keys=[key], client="variable-%s" % name)
         else:
-            record = {'type': 'msgpack', 'value': data}
+            record = {"type": "msgpack", "value": data}
         try:
             old = self.variables[name]
         except KeyError:
             pass
         else:
-            if old['type'] == 'Future' and old['value'] != key:
-                self.release(old['value'], name)
+            if old["type"] == "Future" and old["value"] != key:
+                self.release(old["value"], name)
         if name not in self.variables:
             self.started.notify_all()
         self.variables[name] = record
@@ -67,8 +68,7 @@ def release(self, key, name):
         while self.waiting[key, name]:
             yield self.waiting_conditions[name].wait()
 
-        self.scheduler.client_releases_keys(keys=[key],
-                                            client='variable-%s' % name)
+        self.scheduler.client_releases_keys(keys=[key], client="variable-%s" % name)
         del self.waiting[key, name]
 
     def future_release(self, name=None, key=None, token=None, client=None):
@@ -88,15 +88,15 @@ def get(self, stream=None, name=None, client=None, timeout=None):
                 raise gen.TimeoutError()
             yield self.started.wait(timeout=left)
         record = self.variables[name]
-        if record['type'] == 'Future':
-            key = record['value']
+        if record["type"] == "Future":
+            key = record["value"]
             token = uuid.uuid4().hex
             ts = self.scheduler.tasks.get(key)
-            state = ts.state if ts is not None else 'lost'
-            msg = {'token': token, 'state': state}
-            if state == 'erred':
-                msg['exception'] = ts.exception_blame.exception
-                msg['traceback'] = ts.exception_blame.traceback
+            state = ts.state if ts is not None else "lost"
+            msg = {"token": token, "state": state}
+            if state == "erred":
+                msg["exception"] = ts.exception_blame.exception
+                msg["traceback"] = ts.exception_blame.traceback
             record = merge(record, msg)
             self.waiting[key, name].add(token)
         raise gen.Return(record)
@@ -109,8 +109,8 @@ def delete(self, stream=None, name=None, client=None):
             except KeyError:
                 pass
             else:
-                if old['type'] == 'Future':
-                    yield self.release(old['value'], name)
+                if old["type"] == "Future":
+                    yield self.release(old["value"], name)
             del self.waiting_conditions[name]
             del self.variables[name]
 
@@ -149,16 +149,16 @@ class Variable(object):
 
     def __init__(self, name=None, client=None, maxsize=0):
         self.client = client or _get_global_client()
-        self.name = name or 'variable-' + uuid.uuid4().hex
+        self.name = name or "variable-" + uuid.uuid4().hex
 
     @gen.coroutine
     def _set(self, value):
         if isinstance(value, Future):
-            yield self.client.scheduler.variable_set(key=tokey(value.key),
-                                                     name=self.name)
+            yield self.client.scheduler.variable_set(
+                key=tokey(value.key), name=self.name
+            )
         else:
-            yield self.client.scheduler.variable_set(data=value,
-                                                     name=self.name)
+            yield self.client.scheduler.variable_set(data=value, name=self.name)
 
     def set(self, value, **kwargs):
         """ Set the value of this variable
@@ -172,19 +172,23 @@ def set(self, value, **kwargs):
 
     @gen.coroutine
     def _get(self, timeout=None):
-        d = yield self.client.scheduler.variable_get(timeout=timeout,
-                                                     name=self.name,
-                                                     client=self.client.id)
-        if d['type'] == 'Future':
-            value = Future(d['value'], self.client, inform=True, state=d['state'])
-            if d['state'] == 'erred':
-                value._state.set_error(d['exception'], d['traceback'])
-            self.client._send_to_scheduler({'op': 'variable-future-release',
-                                            'name': self.name,
-                                            'key': d['value'],
-                                            'token': d['token']})
+        d = yield self.client.scheduler.variable_get(
+            timeout=timeout, name=self.name, client=self.client.id
+        )
+        if d["type"] == "Future":
+            value = Future(d["value"], self.client, inform=True, state=d["state"])
+            if d["state"] == "erred":
+                value._state.set_error(d["exception"], d["traceback"])
+            self.client._send_to_scheduler(
+                {
+                    "op": "variable-future-release",
+                    "name": self.name,
+                    "key": d["value"],
+                    "token": d["token"],
+                }
+            )
         else:
-            value = d['value']
+            value = d["value"]
         raise gen.Return(value)
 
     def get(self, timeout=None, **kwargs):
@@ -196,9 +200,8 @@ def delete(self):
 
         Caution, this affects all clients currently pointing to this variable.
         """
-        if self.client.status == 'running':  # TODO: can leave zombie futures
-            self.client._send_to_scheduler({'op': 'variable_delete',
-                                            'name': self.name})
+        if self.client.status == "running":  # TODO: can leave zombie futures
+            self.client._send_to_scheduler({"op": "variable_delete", "name": self.name})
 
     def __getstate__(self):
         return (self.name, self.client.scheduler.address)
diff --git a/distributed/versions.py b/distributed/versions.py
index fa7bbc0835a..2baa47a1d8f 100644
--- a/distributed/versions.py
+++ b/distributed/versions.py
@@ -12,19 +12,23 @@
 from .utils import ignoring
 
 
-required_packages = [('dask', lambda p: p.__version__),
-                     ('distributed', lambda p: p.__version__),
-                     ('msgpack', lambda p: '.'.join([str(v) for v in p.version])),
-                     ('cloudpickle', lambda p: p.__version__),
-                     ('tornado', lambda p: p.version),
-                     ('toolz', lambda p: p.__version__)]
-
-optional_packages = [('numpy', lambda p: p.__version__),
-                     ('pandas', lambda p: p.__version__),
-                     ('bokeh', lambda p: p.__version__),
-                     ('lz4', lambda p: p.__version__),
-                     ('dask_ml', lambda p: p.__version__),
-                     ('blosc', lambda p: p.__version__)]
+required_packages = [
+    ("dask", lambda p: p.__version__),
+    ("distributed", lambda p: p.__version__),
+    ("msgpack", lambda p: ".".join([str(v) for v in p.version])),
+    ("cloudpickle", lambda p: p.__version__),
+    ("tornado", lambda p: p.version),
+    ("toolz", lambda p: p.__version__),
+]
+
+optional_packages = [
+    ("numpy", lambda p: p.__version__),
+    ("pandas", lambda p: p.__version__),
+    ("bokeh", lambda p: p.__version__),
+    ("lz4", lambda p: p.__version__),
+    ("dask_ml", lambda p: p.__version__),
+    ("blosc", lambda p: p.__version__),
+]
 
 
 def get_versions(packages=None):
@@ -34,27 +38,30 @@ def get_versions(packages=None):
     if packages is None:
         packages = []
 
-    d = {'host': get_system_info(),
-         'packages': {'required': get_package_info(required_packages),
-                      'optional': get_package_info(optional_packages + list(packages))}
-         }
+    d = {
+        "host": get_system_info(),
+        "packages": {
+            "required": get_package_info(required_packages),
+            "optional": get_package_info(optional_packages + list(packages)),
+        },
+    }
     return d
 
 
 def get_system_info():
-    (sysname, nodename, release,
-     version, machine, processor) = platform.uname()
-    host = [("python", "%d.%d.%d.%s.%s" % sys.version_info[:]),
-            ("python-bits", struct.calcsize("P") * 8),
-            ("OS", "%s" % (sysname)),
-            ("OS-release", "%s" % (release)),
-            ("machine", "%s" % (machine)),
-            ("processor", "%s" % (processor)),
-            ("byteorder", "%s" % sys.byteorder),
-            ("LC_ALL", "%s" % os.environ.get('LC_ALL', "None")),
-            ("LANG", "%s" % os.environ.get('LANG', "None")),
-            ("LOCALE", "%s.%s" % locale.getlocale()),
-            ]
+    (sysname, nodename, release, version, machine, processor) = platform.uname()
+    host = [
+        ("python", "%d.%d.%d.%s.%s" % sys.version_info[:]),
+        ("python-bits", struct.calcsize("P") * 8),
+        ("OS", "%s" % (sysname)),
+        ("OS-release", "%s" % (release)),
+        ("machine", "%s" % (machine)),
+        ("processor", "%s" % (processor)),
+        ("byteorder", "%s" % sys.byteorder),
+        ("LC_ALL", "%s" % os.environ.get("LC_ALL", "None")),
+        ("LANG", "%s" % os.environ.get("LANG", "None")),
+        ("LOCALE", "%s.%s" % locale.getlocale()),
+    ]
 
     return host
 
@@ -66,7 +73,7 @@ def version_of_package(pkg):
     with ignoring(AttributeError):
         return str(pkg.version)
     with ignoring(AttributeError):
-        return '.'.join(map(str, pkg.version_info))
+        return ".".join(map(str, pkg.version_info))
     return None
 
 
diff --git a/distributed/worker.py b/distributed/worker.py
index c7720888f4f..b9ed6c5a59d 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -16,6 +16,7 @@
 import dask
 from dask.core import istask
 from dask.compatibility import apply
+
 try:
     from cytoolz import pluck, partial, merge
 except ImportError:
@@ -29,27 +30,39 @@
 from .batched import BatchedSend
 from .comm import get_address_host, get_local_address_for, connect
 from .comm.utils import offload
-from .compatibility import (unicode, get_thread_identity, finalize,
-        MutableMapping)
-from .core import (error_message, CommClosedError, send_recv,
-                   pingpong, coerce_to_address)
+from .compatibility import unicode, get_thread_identity, finalize, MutableMapping
+from .core import error_message, CommClosedError, send_recv, pingpong, coerce_to_address
 from .diskutils import WorkSpace
 from .metrics import time
 from .node import ServerNode
 from .preloading import preload_modules
 from .proctitle import setproctitle
-from .protocol import (pickle, to_serialize, deserialize_bytes,
-                       serialize_bytelist)
+from .protocol import pickle, to_serialize, deserialize_bytes, serialize_bytelist
 from .pubsub import PubSubWorkerExtension
 from .security import Security
 from .sizeof import safe_sizeof as sizeof
 from .threadpoolexecutor import ThreadPoolExecutor, secede as tpe_secede
-from .utils import (funcname, get_ip, has_arg, _maybe_complex, log_errors,
-                    ignoring, mp_context, import_file,
-                    silence_logging, thread_state, json_load_robust, key_split,
-                    format_bytes, DequeHandler, PeriodicCallback,
-                    parse_bytes, parse_timedelta, iscoroutinefunction,
-                    warn_on_duration)
+from .utils import (
+    funcname,
+    get_ip,
+    has_arg,
+    _maybe_complex,
+    log_errors,
+    ignoring,
+    mp_context,
+    import_file,
+    silence_logging,
+    thread_state,
+    json_load_robust,
+    key_split,
+    format_bytes,
+    DequeHandler,
+    PeriodicCallback,
+    parse_bytes,
+    parse_timedelta,
+    iscoroutinefunction,
+    warn_on_duration,
+)
 from .utils_comm import pack_data, gather_from_workers
 from .utils_perf import ThrottledGC, enable_gc_diagnosis, disable_gc_diagnosis
 
@@ -57,12 +70,13 @@
 
 logger = logging.getLogger(__name__)
 
-LOG_PDB = dask.config.get('distributed.admin.pdb-on-err')
+LOG_PDB = dask.config.get("distributed.admin.pdb-on-err")
 
-no_value = '--no-value-sentinel--'
+no_value = "--no-value-sentinel--"
 
 try:
     import psutil
+
     TOTAL_MEMORY = psutil.virtual_memory().total
 except ImportError:
     logger.warning("Please install psutil to estimate worker memory use")
@@ -70,15 +84,13 @@
     psutil = None
 
 
-IN_PLAY = ('waiting', 'ready', 'executing', 'long-running')
-PENDING = ('waiting', 'ready', 'constrained')
-PROCESSING = ('waiting', 'ready', 'constrained', 'executing', 'long-running')
-READY = ('ready', 'constrained')
+IN_PLAY = ("waiting", "ready", "executing", "long-running")
+PENDING = ("waiting", "ready", "constrained")
+PROCESSING = ("waiting", "ready", "constrained", "executing", "long-running")
+READY = ("ready", "constrained")
 
 
-DEFAULT_EXTENSIONS = [
-    PubSubWorkerExtension,
-]
+DEFAULT_EXTENSIONS = [PubSubWorkerExtension]
 
 _global_workers = []
 
@@ -257,14 +269,33 @@ class Worker(ServerNode):
     distributed.nanny.Nanny
     """
 
-    def __init__(self, scheduler_ip=None, scheduler_port=None,
-                 scheduler_file=None, ncores=None, loop=None, local_dir='dask-worker-space',
-                 services=None, service_ports=None, name=None,
-                 reconnect=True, memory_limit='auto',
-                 executor=None, resources=None, silence_logs=None,
-                 death_timeout=None, preload=None, preload_argv=None, security=None,
-                 contact_address=None, memory_monitor_interval='200ms',
-                 extensions=None, metrics=None, data=None, **kwargs):
+    def __init__(
+        self,
+        scheduler_ip=None,
+        scheduler_port=None,
+        scheduler_file=None,
+        ncores=None,
+        loop=None,
+        local_dir="dask-worker-space",
+        services=None,
+        service_ports=None,
+        name=None,
+        reconnect=True,
+        memory_limit="auto",
+        executor=None,
+        resources=None,
+        silence_logs=None,
+        death_timeout=None,
+        preload=None,
+        preload_argv=None,
+        security=None,
+        contact_address=None,
+        memory_monitor_interval="200ms",
+        extensions=None,
+        metrics=None,
+        data=None,
+        **kwargs
+    ):
         self.tasks = dict()
         self.task_state = dict()
         self.dep_state = dict()
@@ -280,8 +311,12 @@ def __init__(self, scheduler_ip=None, scheduler_port=None,
 
         self.in_flight_tasks = dict()
         self.in_flight_workers = dict()
-        self.total_out_connections = dask.config.get('distributed.worker.connections.outgoing')
-        self.total_in_connections = dask.config.get('distributed.worker.connections.incoming')
+        self.total_out_connections = dask.config.get(
+            "distributed.worker.connections.outgoing"
+        )
+        self.total_in_connections = dask.config.get(
+            "distributed.worker.connections.incoming"
+        )
         self.total_comm_nbytes = 10e6
         self.comm_nbytes = 0
         self.suspicious_deps = defaultdict(lambda: 0)
@@ -313,33 +348,35 @@ def __init__(self, scheduler_ip=None, scheduler_port=None,
         self.long_running = set()
 
         self.batched_stream = None
-        self.recent_messages_log = deque(maxlen=dask.config.get('distributed.comm.recent-messages-log-length'))
+        self.recent_messages_log = deque(
+            maxlen=dask.config.get("distributed.comm.recent-messages-log-length")
+        )
         self.target_message_size = 50e6  # 50 MB
 
         self.log = deque(maxlen=100000)
-        self.validate = kwargs.pop('validate', False)
+        self.validate = kwargs.pop("validate", False)
 
         self._transitions = {
-            ('waiting', 'ready'): self.transition_waiting_ready,
-            ('waiting', 'memory'): self.transition_waiting_done,
-            ('waiting', 'error'): self.transition_waiting_done,
-            ('ready', 'executing'): self.transition_ready_executing,
-            ('ready', 'memory'): self.transition_ready_memory,
-            ('constrained', 'executing'): self.transition_constrained_executing,
-            ('executing', 'memory'): self.transition_executing_done,
-            ('executing', 'error'): self.transition_executing_done,
-            ('executing', 'rescheduled'): self.transition_executing_done,
-            ('executing', 'long-running'): self.transition_executing_long_running,
-            ('long-running', 'error'): self.transition_executing_done,
-            ('long-running', 'memory'): self.transition_executing_done,
-            ('long-running', 'rescheduled'): self.transition_executing_done,
+            ("waiting", "ready"): self.transition_waiting_ready,
+            ("waiting", "memory"): self.transition_waiting_done,
+            ("waiting", "error"): self.transition_waiting_done,
+            ("ready", "executing"): self.transition_ready_executing,
+            ("ready", "memory"): self.transition_ready_memory,
+            ("constrained", "executing"): self.transition_constrained_executing,
+            ("executing", "memory"): self.transition_executing_done,
+            ("executing", "error"): self.transition_executing_done,
+            ("executing", "rescheduled"): self.transition_executing_done,
+            ("executing", "long-running"): self.transition_executing_long_running,
+            ("long-running", "error"): self.transition_executing_done,
+            ("long-running", "memory"): self.transition_executing_done,
+            ("long-running", "rescheduled"): self.transition_executing_done,
         }
 
         self._dep_transitions = {
-            ('waiting', 'flight'): self.transition_dep_waiting_flight,
-            ('waiting', 'memory'): self.transition_dep_waiting_memory,
-            ('flight', 'waiting'): self.transition_dep_flight_waiting,
-            ('flight', 'memory'): self.transition_dep_flight_memory,
+            ("waiting", "flight"): self.transition_dep_waiting_flight,
+            ("waiting", "memory"): self.transition_dep_waiting_memory,
+            ("flight", "waiting"): self.transition_dep_flight_waiting,
+            ("flight", "memory"): self.transition_dep_flight_memory,
         }
 
         self.incoming_transfer_log = deque(maxlen=(100000))
@@ -350,17 +387,19 @@ def __init__(self, scheduler_ip=None, scheduler_port=None,
         self.repetitively_busy = 0
         self._client = None
 
-        profile_cycle_interval = kwargs.pop('profile_cycle_interval',
-                                            dask.config.get('distributed.worker.profile.cycle'))
-        profile_cycle_interval = parse_timedelta(profile_cycle_interval, default='ms')
+        profile_cycle_interval = kwargs.pop(
+            "profile_cycle_interval",
+            dask.config.get("distributed.worker.profile.cycle"),
+        )
+        profile_cycle_interval = parse_timedelta(profile_cycle_interval, default="ms")
 
         self._setup_logging()
 
         if scheduler_file:
             cfg = json_load_robust(scheduler_file)
-            scheduler_addr = cfg['address']
-        elif scheduler_ip is None and dask.config.get('scheduler-address', None):
-            scheduler_addr = dask.config.get('scheduler-address')
+            scheduler_addr = cfg["address"]
+        elif scheduler_ip is None and dask.config.get("scheduler-address", None):
+            scheduler_addr = dask.config.get("scheduler-address")
         elif scheduler_port is None:
             scheduler_addr = coerce_to_address(scheduler_ip)
         else:
@@ -372,48 +411,56 @@ def __init__(self, scheduler_ip=None, scheduler_port=None,
         self.death_timeout = death_timeout
         self.preload = preload
         if self.preload is None:
-            self.preload = dask.config.get('distributed.worker.preload')
+            self.preload = dask.config.get("distributed.worker.preload")
         self.preload_argv = preload_argv
         if self.preload_argv is None:
-            self.preload_argv = dask.config.get('distributed.worker.preload-argv')
+            self.preload_argv = dask.config.get("distributed.worker.preload-argv")
         self.contact_address = contact_address
-        self.memory_monitor_interval = parse_timedelta(memory_monitor_interval, default='ms')
+        self.memory_monitor_interval = parse_timedelta(
+            memory_monitor_interval, default="ms"
+        )
         self.extensions = dict()
         if silence_logs:
             silence_logging(level=silence_logs)
 
         with warn_on_duration(
-            '1s',
+            "1s",
             "Creating scratch directories is taking a surprisingly long time. "
             "This is often due to running workers on a network file system. "
             "Consider specifying a local-directory to point workers to write "
-            "scratch data to a local disk."
+            "scratch data to a local disk.",
         ):
             self._workspace = WorkSpace(os.path.abspath(local_dir))
-            self._workdir = self._workspace.new_work_dir(prefix='worker-')
+            self._workdir = self._workspace.new_work_dir(prefix="worker-")
             self.local_dir = self._workdir.dir_path
 
         self.security = security or Security()
         assert isinstance(self.security, Security)
-        self.connection_args = self.security.get_connection_args('worker')
-        self.listen_args = self.security.get_listen_args('worker')
+        self.connection_args = self.security.get_connection_args("worker")
+        self.listen_args = self.security.get_listen_args("worker")
 
         self.memory_limit = parse_memory_limit(memory_limit, self.ncores)
 
         self.paused = False
 
-        if 'memory_target_fraction' in kwargs:
-            self.memory_target_fraction = kwargs.pop('memory_target_fraction')
+        if "memory_target_fraction" in kwargs:
+            self.memory_target_fraction = kwargs.pop("memory_target_fraction")
         else:
-            self.memory_target_fraction = dask.config.get('distributed.worker.memory.target')
-        if 'memory_spill_fraction' in kwargs:
-            self.memory_spill_fraction = kwargs.pop('memory_spill_fraction')
+            self.memory_target_fraction = dask.config.get(
+                "distributed.worker.memory.target"
+            )
+        if "memory_spill_fraction" in kwargs:
+            self.memory_spill_fraction = kwargs.pop("memory_spill_fraction")
         else:
-            self.memory_spill_fraction = dask.config.get('distributed.worker.memory.spill')
-        if 'memory_pause_fraction' in kwargs:
-            self.memory_pause_fraction = kwargs.pop('memory_pause_fraction')
+            self.memory_spill_fraction = dask.config.get(
+                "distributed.worker.memory.spill"
+            )
+        if "memory_pause_fraction" in kwargs:
+            self.memory_pause_fraction = kwargs.pop("memory_pause_fraction")
         else:
-            self.memory_pause_fraction = dask.config.get('distributed.worker.memory.pause')
+            self.memory_pause_fraction = dask.config.get(
+                "distributed.worker.memory.pause"
+            )
 
         if isinstance(data, MutableMapping):
             self.data = data
@@ -421,17 +468,19 @@ def __init__(self, scheduler_ip=None, scheduler_port=None,
             self.data = data()
         elif isinstance(data, tuple):
             self.data = data[0](**data[1])
-        elif (self.memory_limit and
-                (self.memory_target_fraction or
-                 self.memory_spill_fraction)):
+        elif self.memory_limit and (
+            self.memory_target_fraction or self.memory_spill_fraction
+        ):
             try:
                 from zict import Buffer, File, Func
             except ImportError:
                 raise ImportError("Please `pip install zict` for spill-to-disk workers")
-            path = os.path.join(self.local_dir, 'storage')
-            storage = Func(partial(serialize_bytelist, on_error='raise'),
-                           deserialize_bytes,
-                           File(path))
+            path = os.path.join(self.local_dir, "storage")
+            storage = Func(
+                partial(serialize_bytelist, on_error="raise"),
+                deserialize_bytes,
+                File(path),
+            )
             target = int(float(self.memory_limit) * self.memory_target_fraction)
             self.data = Buffer({}, storage, target, weight)
         else:
@@ -442,8 +491,12 @@ def __init__(self, scheduler_ip=None, scheduler_port=None,
         self.status = None
         self._closed = Event()
         self.reconnect = reconnect
-        self.executor = executor or ThreadPoolExecutor(self.ncores, thread_name_prefix="Dask-Worker-Threads'")
-        self.actor_executor = ThreadPoolExecutor(1, thread_name_prefix="Dask-Actor-Threads")
+        self.executor = executor or ThreadPoolExecutor(
+            self.ncores, thread_name_prefix="Dask-Worker-Threads'"
+        )
+        self.actor_executor = ThreadPoolExecutor(
+            1, thread_name_prefix="Dask-Actor-Threads"
+        )
         self.name = name
         self.scheduler_delay = 0
         self.stream_comms = dict()
@@ -459,56 +512,61 @@ def __init__(self, scheduler_ip=None, scheduler_port=None,
         self.metrics = dict(metrics) if metrics else {}
 
         handlers = {
-            'gather': self.gather,
-            'run': self.run,
-            'run_coroutine': self.run_coroutine,
-            'get_data': self.get_data,
-            'update_data': self.update_data,
-            'delete_data': self.delete_data,
-            'terminate': self.terminate,
-            'ping': pingpong,
-            'upload_file': self.upload_file,
-            'start_ipython': self.start_ipython,
-            'call_stack': self.get_call_stack,
-            'profile': self.get_profile,
-            'profile_metadata': self.get_profile_metadata,
-            'get_logs': self.get_logs,
-            'keys': self.keys,
-            'versions': self.versions,
-            'actor_execute': self.actor_execute,
-            'actor_attribute': self.actor_attribute,
+            "gather": self.gather,
+            "run": self.run,
+            "run_coroutine": self.run_coroutine,
+            "get_data": self.get_data,
+            "update_data": self.update_data,
+            "delete_data": self.delete_data,
+            "terminate": self.terminate,
+            "ping": pingpong,
+            "upload_file": self.upload_file,
+            "start_ipython": self.start_ipython,
+            "call_stack": self.get_call_stack,
+            "profile": self.get_profile,
+            "profile_metadata": self.get_profile_metadata,
+            "get_logs": self.get_logs,
+            "keys": self.keys,
+            "versions": self.versions,
+            "actor_execute": self.actor_execute,
+            "actor_attribute": self.actor_attribute,
         }
 
         stream_handlers = {
-            'close': self._close,
-            'compute-task': self.add_task,
-            'release-task': partial(self.release_key, report=False),
-            'delete-data': self.delete_data,
-            'steal-request': self.steal_request,
+            "close": self._close,
+            "compute-task": self.add_task,
+            "release-task": partial(self.release_key, report=False),
+            "delete-data": self.delete_data,
+            "steal-request": self.steal_request,
         }
 
         super(Worker, self).__init__(
-                handlers=handlers,
-                stream_handlers=stream_handlers,
-                io_loop=self.loop,
-                connection_args=self.connection_args,
-                **kwargs)
+            handlers=handlers,
+            stream_handlers=stream_handlers,
+            io_loop=self.loop,
+            connection_args=self.connection_args,
+            **kwargs
+        )
 
         self.scheduler = self.rpc(scheduler_addr)
-        self.execution_state = {'scheduler': self.scheduler.address,
-                                'ioloop': self.loop,
-                                'worker': self}
+        self.execution_state = {
+            "scheduler": self.scheduler.address,
+            "ioloop": self.loop,
+            "worker": self,
+        }
 
         pc = PeriodicCallback(self.heartbeat, 1000, io_loop=self.io_loop)
-        self.periodic_callbacks['heartbeat'] = pc
+        self.periodic_callbacks["heartbeat"] = pc
         self._address = contact_address
 
         if self.memory_limit:
             self._memory_monitoring = False
-            pc = PeriodicCallback(self.memory_monitor,
-                                  self.memory_monitor_interval * 1000,
-                                  io_loop=self.io_loop)
-            self.periodic_callbacks['memory'] = pc
+            pc = PeriodicCallback(
+                self.memory_monitor,
+                self.memory_monitor_interval * 1000,
+                io_loop=self.io_loop,
+            )
+            self.periodic_callbacks["memory"] = pc
 
         if extensions is None:
             extensions = DEFAULT_EXTENSIONS
@@ -520,16 +578,19 @@ def __init__(self, scheduler_ip=None, scheduler_port=None,
         setproctitle("dask-worker [not started]")
 
         pc = PeriodicCallback(
-                self.trigger_profile,
-                parse_timedelta(dask.config.get('distributed.worker.profile.interval'), default='ms') * 1000,
-                io_loop=self.io_loop
+            self.trigger_profile,
+            parse_timedelta(
+                dask.config.get("distributed.worker.profile.interval"), default="ms"
+            )
+            * 1000,
+            io_loop=self.io_loop,
         )
-        self.periodic_callbacks['profile'] = pc
+        self.periodic_callbacks["profile"] = pc
 
-        pc = PeriodicCallback(self.cycle_profile,
-                              profile_cycle_interval * 1000,
-                              io_loop=self.io_loop)
-        self.periodic_callbacks['profile-cycle'] = pc
+        pc = PeriodicCallback(
+            self.cycle_profile, profile_cycle_interval * 1000, io_loop=self.io_loop
+        )
+        self.periodic_callbacks["profile-cycle"] = pc
 
         _global_workers.append(weakref.ref(self))
 
@@ -538,15 +599,28 @@ def __init__(self, scheduler_ip=None, scheduler_port=None,
     ##################
 
     def __repr__(self):
-        return "<%s: %s, %s, stored: %d, running: %d/%d, ready: %d, comm: %d, waiting: %d>" % (
-            self.__class__.__name__, self.address, self.status,
-            len(self.data), len(self.executing), self.ncores,
-            len(self.ready), len(self.in_flight_tasks),
-            len(self.waiting_for_data))
+        return (
+            "<%s: %s, %s, stored: %d, running: %d/%d, ready: %d, comm: %d, waiting: %d>"
+            % (
+                self.__class__.__name__,
+                self.address,
+                self.status,
+                len(self.data),
+                len(self.executing),
+                self.ncores,
+                len(self.ready),
+                len(self.in_flight_tasks),
+                len(self.waiting_for_data),
+            )
+        )
 
     def _setup_logging(self):
-        self._deque_handler = DequeHandler(n=dask.config.get('distributed.admin.log-length'))
-        self._deque_handler.setFormatter(logging.Formatter(dask.config.get('distributed.admin.log-format')))
+        self._deque_handler = DequeHandler(
+            n=dask.config.get("distributed.admin.log-length")
+        )
+        self._deque_handler.setFormatter(
+            logging.Formatter(dask.config.get("distributed.admin.log-format"))
+        )
         logger.addHandler(self._deque_handler)
         finalize(self, logger.removeHandler, self._deque_handler)
 
@@ -556,20 +630,24 @@ def worker_address(self):
         return self.address
 
     def get_metrics(self):
-        core = dict(executing=len(self.executing),
-                    in_memory=len(self.data),
-                    ready=len(self.ready),
-                    in_flight=len(self.in_flight_tasks))
+        core = dict(
+            executing=len(self.executing),
+            in_memory=len(self.data),
+            ready=len(self.ready),
+            in_flight=len(self.in_flight_tasks),
+        )
         custom = {k: metric(self) for k, metric in self.metrics.items()}
 
         return merge(custom, self.monitor.recent(), core)
 
     def identity(self, comm=None):
-        return {'type': type(self).__name__,
-                'id': self.id,
-                'scheduler': self.scheduler.address,
-                'ncores': self.ncores,
-                'memory_limit': self.memory_limit}
+        return {
+            "type": type(self).__name__,
+            "id": self.id,
+            "scheduler": self.scheduler.address,
+            "ncores": self.ncores,
+            "memory_limit": self.memory_limit,
+        }
 
     #####################
     # External Services #
@@ -577,37 +655,42 @@ def identity(self, comm=None):
 
     @gen.coroutine
     def _register_with_scheduler(self):
-        self.periodic_callbacks['heartbeat'].stop()
+        self.periodic_callbacks["heartbeat"].stop()
         start = time()
         if self.contact_address is None:
             self.contact_address = self.address
-        logger.info('-' * 49)
+        logger.info("-" * 49)
         while True:
             if self.death_timeout and time() > start + self.death_timeout:
                 yield self._close(timeout=1)
                 return
-            if self.status in ('closed', 'closing'):
+            if self.status in ("closed", "closing"):
                 raise gen.Return
             try:
                 _start = time()
-                comm = yield connect(self.scheduler.address,
-                                     connection_args=self.connection_args)
-                yield comm.write(dict(op='register-worker',
-                                      reply=False,
-                                      address=self.contact_address,
-                                      keys=list(self.data),
-                                      ncores=self.ncores,
-                                      name=self.name,
-                                      nbytes=self.nbytes,
-                                      now=time(),
-                                      resources=self.total_resources,
-                                      memory_limit=self.memory_limit,
-                                      local_directory=self.local_dir,
-                                      services=self.service_ports,
-                                      pid=os.getpid(),
-                                      metrics=self.get_metrics()),
-                                 serializers=['msgpack'])
-                future = comm.read(deserializers=['msgpack'])
+                comm = yield connect(
+                    self.scheduler.address, connection_args=self.connection_args
+                )
+                yield comm.write(
+                    dict(
+                        op="register-worker",
+                        reply=False,
+                        address=self.contact_address,
+                        keys=list(self.data),
+                        ncores=self.ncores,
+                        name=self.name,
+                        nbytes=self.nbytes,
+                        now=time(),
+                        resources=self.total_resources,
+                        memory_limit=self.memory_limit,
+                        local_directory=self.local_dir,
+                        services=self.service_ports,
+                        pid=os.getpid(),
+                        metrics=self.get_metrics(),
+                    ),
+                    serializers=["msgpack"],
+                )
+                future = comm.read(deserializers=["msgpack"])
                 if self.death_timeout:
                     diff = self.death_timeout - (time() - start)
                     if diff < 0:
@@ -616,33 +699,34 @@ def _register_with_scheduler(self):
                 response = yield future
                 _end = time()
                 middle = (_start + _end) / 2
-                self.scheduler_delay = response['time'] - middle
-                self.status = 'running'
+                self.scheduler_delay = response["time"] - middle
+                self.status = "running"
                 break
             except EnvironmentError:
-                logger.info('Waiting to connect to: %26s', self.scheduler.address)
+                logger.info("Waiting to connect to: %26s", self.scheduler.address)
                 yield gen.sleep(0.1)
             except gen.TimeoutError:
                 logger.info("Timed out when connecting to scheduler")
-        if response['status'] != 'OK':
-            raise ValueError("Unexpected response from register: %r" %
-                             (response,))
+        if response["status"] != "OK":
+            raise ValueError("Unexpected response from register: %r" % (response,))
         else:
             # Retrieve eventual init functions and run them
-            for function_bytes in response['worker-setups']:
+            for function_bytes in response["worker-setups"]:
                 setup_function = pickle.loads(function_bytes)
-                if has_arg(setup_function, 'dask_worker'):
+                if has_arg(setup_function, "dask_worker"):
                     result = setup_function(dask_worker=self)
                 else:
                     result = setup_function()
-                logger.info('Init function %s ran: output=%s' % (setup_function, result))
+                logger.info(
+                    "Init function %s ran: output=%s" % (setup_function, result)
+                )
 
-            logger.info('        Registered to: %26s', self.scheduler.address)
-            logger.info('-' * 49)
+            logger.info("        Registered to: %26s", self.scheduler.address)
+            logger.info("-" * 49)
 
-        self.batched_stream = BatchedSend(interval='2ms', loop=self.loop)
+        self.batched_stream = BatchedSend(interval="2ms", loop=self.loop)
         self.batched_stream.start(comm)
-        self.periodic_callbacks['heartbeat'].start()
+        self.periodic_callbacks["heartbeat"].start()
         self.loop.add_callback(self.handle_scheduler, comm)
 
     @gen.coroutine
@@ -653,18 +737,18 @@ def heartbeat(self):
             try:
                 start = time()
                 response = yield self.scheduler.heartbeat_worker(
-                    address=self.contact_address,
-                    now=time(),
-                    metrics=self.get_metrics()
+                    address=self.contact_address, now=time(), metrics=self.get_metrics()
                 )
                 end = time()
                 middle = (start + end) / 2
 
-                if response['status'] == 'missing':
+                if response["status"] == "missing":
                     yield self._register_with_scheduler()
                     return
-                self.scheduler_delay = response['time'] - middle
-                self.periodic_callbacks['heartbeat'].callback_time = response['heartbeat-interval'] * 1000
+                self.scheduler_delay = response["time"] - middle
+                self.periodic_callbacks["heartbeat"].callback_time = (
+                    response["heartbeat-interval"] * 1000
+                )
             except CommClosedError:
                 logger.warning("Heartbeat to scheduler failed")
             finally:
@@ -675,8 +759,9 @@ def heartbeat(self):
     @gen.coroutine
     def handle_scheduler(self, comm):
         try:
-            yield self.handle_stream(comm, every_cycle=[self.ensure_communicating,
-                                                        self.ensure_computing])
+            yield self.handle_stream(
+                comm, every_cycle=[self.ensure_communicating, self.ensure_computing]
+            )
         except Exception as e:
             logger.exception(e)
             raise
@@ -693,11 +778,10 @@ def start_ipython(self, comm):
         Returns Jupyter connection info dictionary.
         """
         from ._ipython_utils import start_ipython
+
         if self._ipython_kernel is None:
             self._ipython_kernel = start_ipython(
-                ip=self.ip,
-                ns={'worker': self},
-                log=logger,
+                ip=self.ip, ns={"worker": self}, log=logger
             )
         return self._ipython_kernel.get_connection_info()
 
@@ -708,7 +792,7 @@ def upload_file(self, comm, filename=None, data=None, load=True):
         def func(data):
             if isinstance(data, unicode):
                 data = data.encode()
-            with open(out_filename, 'wb') as f:
+            with open(out_filename, "wb") as f:
                 f.write(data)
                 f.flush()
             return data
@@ -723,29 +807,34 @@ def func(data):
                 import_file(out_filename)
             except Exception as e:
                 logger.exception(e)
-                raise gen.Return({'status': 'error',
-                                  'exception': to_serialize(e)})
+                raise gen.Return({"status": "error", "exception": to_serialize(e)})
 
-        raise gen.Return({'status': 'OK', 'nbytes': len(data)})
+        raise gen.Return({"status": "OK", "nbytes": len(data)})
 
     def keys(self, comm=None):
         return list(self.data)
 
     @gen.coroutine
     def gather(self, comm=None, who_has=None):
-        who_has = {k: [coerce_to_address(addr) for addr in v]
-                   for k, v in who_has.items()
-                   if k not in self.data}
+        who_has = {
+            k: [coerce_to_address(addr) for addr in v]
+            for k, v in who_has.items()
+            if k not in self.data
+        }
         result, missing_keys, missing_workers = yield gather_from_workers(
-            who_has, rpc=self.rpc, who=self.address)
+            who_has, rpc=self.rpc, who=self.address
+        )
         if missing_keys:
-            logger.warning("Could not find data: %s on workers: %s (who_has: %s)",
-                           missing_keys, missing_workers, who_has)
-            raise Return({'status': 'missing-data',
-                          'keys': missing_keys})
+            logger.warning(
+                "Could not find data: %s on workers: %s (who_has: %s)",
+                missing_keys,
+                missing_workers,
+                who_has,
+            )
+            raise Return({"status": "missing-data", "keys": missing_keys})
         else:
             self.update_data(data=result, report=False)
-            raise Return({'status': 'OK'})
+            raise Return({"status": "OK"})
 
     def get_logs(self, comm=None, n=None):
         deque_handler = self._deque_handler
@@ -761,8 +850,8 @@ def get_logs(self, comm=None, n=None):
     #############
 
     def start_services(self, default_listen_ip):
-        if default_listen_ip == '0.0.0.0':
-            default_listen_ip = ''  # for IPV6
+        if default_listen_ip == "0.0.0.0":
+            default_listen_ip = ""  # for IPV6
 
         for k, v in self.service_specs.items():
             listen_ip = None
@@ -772,7 +861,7 @@ def start_services(self, default_listen_ip):
                 port = 0
 
             if isinstance(port, (str, unicode)):
-                port = port.split(':')
+                port = port.split(":")
 
             if isinstance(port, (tuple, list)):
                 listen_ip, port = (port[0], int(port[1]))
@@ -783,7 +872,9 @@ def start_services(self, default_listen_ip):
                 kwargs = {}
 
             self.services[k] = v(self, io_loop=self.loop, **kwargs)
-            self.services[k].listen((listen_ip if listen_ip is not None else default_listen_ip, port))
+            self.services[k].listen(
+                (listen_ip if listen_ip is not None else default_listen_ip, port)
+            )
             self.service_ports[k] = self.services[k].port
 
     @gen.coroutine
@@ -797,16 +888,15 @@ def _start(self, addr_or_port=0):
         if not addr_or_port:
             # Default address is the required one to reach the scheduler
             listen_host = get_address_host(self.scheduler.address)
-            self.listen(get_local_address_for(self.scheduler.address),
-                        listen_args=self.listen_args)
+            self.listen(
+                get_local_address_for(self.scheduler.address),
+                listen_args=self.listen_args,
+            )
             self.ip = get_address_host(self.address)
         elif isinstance(addr_or_port, int):
             # addr_or_port is an integer => assume TCP
-            listen_host = self.ip = get_ip(
-                get_address_host(self.scheduler.address)
-            )
-            self.listen((listen_host, addr_or_port),
-                        listen_args=self.listen_args)
+            listen_host = self.ip = get_ip(get_address_host(self.scheduler.address))
+            self.listen((listen_host, addr_or_port), listen_args=self.listen_args)
         else:
             self.listen(addr_or_port, listen_args=self.listen_args)
             self.ip = get_address_host(self.address)
@@ -815,31 +905,40 @@ def _start(self, addr_or_port=0):
             except ValueError:
                 listen_host = addr_or_port
 
-        if '://' in listen_host:
-            protocol, listen_host = listen_host.split('://')
+        if "://" in listen_host:
+            protocol, listen_host = listen_host.split("://")
 
         self.name = self.name or self.address
-        preload_modules(self.preload, parameter=self, file_dir=self.local_dir, argv=self.preload_argv)
+        preload_modules(
+            self.preload,
+            parameter=self,
+            file_dir=self.local_dir,
+            argv=self.preload_argv,
+        )
         # Services listen on all addresses
         # Note Nanny is not a "real" service, just some metadata
         # passed in service_ports...
         self.start_services(listen_host)
 
         try:
-            listening_address = '%s%s:%d' % (self.listener.prefix, listen_host, self.port)
+            listening_address = "%s%s:%d" % (
+                self.listener.prefix,
+                listen_host,
+                self.port,
+            )
         except Exception:
-            listening_address = '%s%s' % (self.listener.prefix, listen_host)
+            listening_address = "%s%s" % (self.listener.prefix, listen_host)
 
-        logger.info('      Start worker at: %26s', self.address)
-        logger.info('         Listening to: %26s', listening_address)
+        logger.info("      Start worker at: %26s", self.address)
+        logger.info("         Listening to: %26s", listening_address)
         for k, v in self.service_ports.items():
-            logger.info('  %16s at: %26s' % (k, listen_host + ':' + str(v)))
-        logger.info('Waiting to connect to: %26s', self.scheduler.address)
-        logger.info('-' * 49)
-        logger.info('              Threads: %26d', self.ncores)
+            logger.info("  %16s at: %26s" % (k, listen_host + ":" + str(v)))
+        logger.info("Waiting to connect to: %26s", self.scheduler.address)
+        logger.info("-" * 49)
+        logger.info("              Threads: %26d", self.ncores)
         if self.memory_limit:
-            logger.info('               Memory: %26s', format_bytes(self.memory_limit))
-        logger.info('      Local Directory: %26s', self.local_dir)
+            logger.info("               Memory: %26s", format_bytes(self.memory_limit))
+        logger.info("      Local Directory: %26s", self.local_dir)
 
         setproctitle("dask-worker [%s]" % self.address)
 
@@ -857,7 +956,7 @@ def start(self, port=0):
     @gen.coroutine
     def _close(self, report=True, timeout=10, nanny=True, executor_wait=True):
         with log_errors():
-            if self.status in ('closed', 'closing'):
+            if self.status in ("closed", "closing"):
                 return
 
             disable_gc_diagnosis()
@@ -866,7 +965,7 @@ def _close(self, report=True, timeout=10, nanny=True, executor_wait=True):
                 logger.info("Stopping worker at %s", self.address)
             except ValueError:  # address not available if already closed
                 logger.info("Stopping worker")
-            self.status = 'closing'
+            self.status = "closing"
             setproctitle("dask-worker [closing]")
 
             self.stop()
@@ -874,8 +973,10 @@ def _close(self, report=True, timeout=10, nanny=True, executor_wait=True):
                 pc.stop()
             with ignoring(EnvironmentError, gen.TimeoutError):
                 if report:
-                    yield gen.with_timeout(timedelta(seconds=timeout),
-                                           self.scheduler.unregister(address=self.contact_address))
+                    yield gen.with_timeout(
+                        timedelta(seconds=timeout),
+                        self.scheduler.unregister(address=self.contact_address),
+                    )
             self.scheduler.close_rpc()
             self.actor_executor._work_queue.queue.clear()
             if isinstance(self.executor, ThreadPoolExecutor):
@@ -889,14 +990,14 @@ def _close(self, report=True, timeout=10, nanny=True, executor_wait=True):
             for k, v in self.services.items():
                 v.stop()
 
-            self.status = 'closed'
+            self.status = "closed"
 
-            if nanny and 'nanny' in self.service_ports:
-                with self.rpc((self.ip, self.service_ports['nanny'])) as r:
+            if nanny and "nanny" in self.service_ports:
+                with self.rpc((self.ip, self.service_ports["nanny"])) as r:
                     yield r.terminate()
 
             if self.batched_stream and not self.batched_stream.comm.closed():
-                self.batched_stream.send({'op': 'close-stream'})
+                self.batched_stream.send({"op": "close-stream"})
 
             if self.batched_stream:
                 self.batched_stream.close()
@@ -921,12 +1022,12 @@ def _remove_from_global_workers(self):
     @gen.coroutine
     def terminate(self, comm, report=True):
         yield self._close(report=report)
-        raise Return('OK')
+        raise Return("OK")
 
     @gen.coroutine
     def wait_until_closed(self):
         yield self._closed.wait()
-        assert self.status == 'closed'
+        assert self.status == "closed"
 
     ################
     # Worker Peers #
@@ -934,14 +1035,15 @@ def wait_until_closed(self):
 
     def send_to_worker(self, address, msg):
         if address not in self.stream_comms:
-            bcomm = BatchedSend(interval='1ms', loop=self.loop)
+            bcomm = BatchedSend(interval="1ms", loop=self.loop)
             self.stream_comms[address] = bcomm
 
             @gen.coroutine
             def batched_send_connect():
-                comm = yield connect(address,  # TODO, serialization
-                                     connection_args=self.connection_args)
-                yield comm.write({'op': 'connection_stream'})
+                comm = yield connect(
+                    address, connection_args=self.connection_args  # TODO, serialization
+                )
+                yield comm.write({"op": "connection_stream"})
 
                 bcomm.start(comm)
 
@@ -950,19 +1052,27 @@ def batched_send_connect():
         self.stream_comms[address].send(msg)
 
     @gen.coroutine
-    def get_data(self, comm, keys=None, who=None, serializers=None,
-                 max_connections=None):
+    def get_data(
+        self, comm, keys=None, who=None, serializers=None, max_connections=None
+    ):
         start = time()
 
         if max_connections is None:
             max_connections = self.total_in_connections
 
         # Allow same-host connections more liberally
-        if max_connections and comm and get_address_host(comm.peer_address) == get_address_host(self.address):
+        if (
+            max_connections
+            and comm
+            and get_address_host(comm.peer_address) == get_address_host(self.address)
+        ):
             max_connections = max_connections * 2
 
-        if max_connections is not False and self.outgoing_current_count > max_connections:
-            raise gen.Return({'status': 'busy'})
+        if (
+            max_connections is not False
+            and self.outgoing_current_count > max_connections
+        ):
+            raise gen.Return({"status": "busy"})
 
         self.outgoing_current_count += 1
         data = {k: self.data[k] for k in keys if k in self.data}
@@ -971,48 +1081,51 @@ def get_data(self, comm, keys=None, who=None, serializers=None,
             for k in set(keys) - set(data):
                 if k in self.actors:
                     from .actor import Actor
+
                     data[k] = Actor(type(self.actors[k]), self.address, k)
 
-        msg = {'status': 'OK',
-               'data': {k: to_serialize(v) for k, v in data.items()}}
+        msg = {"status": "OK", "data": {k: to_serialize(v) for k, v in data.items()}}
         nbytes = {k: self.nbytes.get(k) for k in data}
         stop = time()
         if self.digests is not None:
-            self.digests['get-data-load-duration'].add(stop - start)
+            self.digests["get-data-load-duration"].add(stop - start)
         start = time()
 
         try:
             compressed = yield comm.write(msg, serializers=serializers)
             response = yield comm.read(deserializers=serializers)
-            assert response == 'OK', response
+            assert response == "OK", response
         except EnvironmentError:
-            logger.exception('failed during get data with %s -> %s',
-                             self.address, who, exc_info=True)
+            logger.exception(
+                "failed during get data with %s -> %s", self.address, who, exc_info=True
+            )
             comm.abort()
             raise
         finally:
             self.outgoing_current_count -= 1
         stop = time()
         if self.digests is not None:
-            self.digests['get-data-send-duration'].add(stop - start)
+            self.digests["get-data-send-duration"].add(stop - start)
 
         total_bytes = sum(filter(None, nbytes.values()))
 
         self.outgoing_count += 1
         duration = (stop - start) or 0.5  # windows
-        self.outgoing_transfer_log.append({
-            'start': start + self.scheduler_delay,
-            'stop': stop + self.scheduler_delay,
-            'middle': (start + stop) / 2,
-            'duration': duration,
-            'who': who,
-            'keys': nbytes,
-            'total': total_bytes,
-            'compressed': compressed,
-            'bandwidth': total_bytes / duration
-        })
-
-        raise gen.Return('dont-reply')
+        self.outgoing_transfer_log.append(
+            {
+                "start": start + self.scheduler_delay,
+                "stop": stop + self.scheduler_delay,
+                "middle": (start + stop) / 2,
+                "duration": duration,
+                "who": who,
+                "keys": nbytes,
+                "total": total_bytes,
+                "compressed": compressed,
+                "bandwidth": total_bytes / duration,
+            }
+        )
+
+        raise gen.Return("dont-reply")
 
     ###################
     # Local Execution #
@@ -1021,32 +1134,30 @@ def get_data(self, comm, keys=None, who=None, serializers=None,
     def update_data(self, comm=None, data=None, report=True, serializers=None):
         for key, value in data.items():
             if key in self.task_state:
-                self.transition(key, 'memory', value=value)
+                self.transition(key, "memory", value=value)
             else:
                 self.put_key_in_memory(key, value)
-                self.task_state[key] = 'memory'
+                self.task_state[key] = "memory"
                 self.tasks[key] = None
                 self.priorities[key] = None
                 self.durations[key] = None
                 self.dependencies[key] = set()
 
             if key in self.dep_state:
-                self.transition_dep(key, 'memory', value=value)
+                self.transition_dep(key, "memory", value=value)
 
-            self.log.append((key, 'receive-from-scatter'))
+            self.log.append((key, "receive-from-scatter"))
 
         if report:
-            self.batched_stream.send({'op': 'add-keys',
-                                      'keys': list(data)})
-        info = {'nbytes': {k: sizeof(v) for k, v in data.items()},
-                'status': 'OK'}
+            self.batched_stream.send({"op": "add-keys", "keys": list(data)})
+        info = {"nbytes": {k: sizeof(v) for k, v in data.items()}, "status": "OK"}
         return info
 
     @gen.coroutine
     def delete_data(self, comm=None, keys=None, report=True):
         if keys:
             for key in list(keys):
-                self.log.append((key, 'delete'))
+                self.log.append((key, "delete"))
                 if key in self.task_state:
                     self.release_key(key)
 
@@ -1057,9 +1168,10 @@ def delete_data(self, comm=None, keys=None, report=True):
             if report:
                 logger.debug("Reporting loss of keys to scheduler")
                 # TODO: this route seems to not exist?
-                yield self.scheduler.remove_keys(address=self.contact_address,
-                                                 keys=list(keys))
-        raise Return('OK')
+                yield self.scheduler.remove_keys(
+                    address=self.contact_address, keys=list(keys)
+                )
+        raise Return("OK")
 
     @gen.coroutine
     def set_resources(self, **resources):
@@ -1070,28 +1182,42 @@ def set_resources(self, **resources):
                 self.available_resources[r] = quantity
             self.total_resources[r] = quantity
 
-        yield self.scheduler.set_resources(resources=self.total_resources,
-                                           worker=self.contact_address)
+        yield self.scheduler.set_resources(
+            resources=self.total_resources, worker=self.contact_address
+        )
 
     ###################
     # Task Management #
     ###################
 
-    def add_task(self, key, function=None, args=None, kwargs=None, task=None,
-                 who_has=None, nbytes=None, priority=None, duration=None,
-                 resource_restrictions=None, actor=False, **kwargs2):
+    def add_task(
+        self,
+        key,
+        function=None,
+        args=None,
+        kwargs=None,
+        task=None,
+        who_has=None,
+        nbytes=None,
+        priority=None,
+        duration=None,
+        resource_restrictions=None,
+        actor=False,
+        **kwargs2
+    ):
         try:
             if key in self.tasks:
                 state = self.task_state[key]
-                if state == 'memory':
+                if state == "memory":
                     assert key in self.data or key in self.actors
-                    logger.debug("Asked to compute pre-existing result: %s: %s",
-                                 key, state)
+                    logger.debug(
+                        "Asked to compute pre-existing result: %s: %s", key, state
+                    )
                     self.send_task_state_to_scheduler(key)
                     return
                 if state in IN_PLAY:
                     return
-                if state == 'erred':
+                if state == "erred":
                     del self.exceptions[key]
                     del self.tracebacks[key]
 
@@ -1099,16 +1225,16 @@ def add_task(self, key, function=None, args=None, kwargs=None, task=None,
                 priority = tuple(priority) + (self.generation,)
                 self.generation -= 1
 
-            if self.dep_state.get(key) == 'memory':
-                self.task_state[key] = 'memory'
+            if self.dep_state.get(key) == "memory":
+                self.task_state[key] = "memory"
                 self.send_task_state_to_scheduler(key)
                 self.tasks[key] = None
-                self.log.append((key, 'new-task-already-in-memory'))
+                self.log.append((key, "new-task-already-in-memory"))
                 self.priorities[key] = priority
                 self.durations[key] = duration
                 return
 
-            self.log.append((key, 'new'))
+            self.log.append((key, "new"))
             try:
                 start = time()
                 self.tasks[key] = _deserialize(function, args, kwargs, task)
@@ -1117,21 +1243,21 @@ def add_task(self, key, function=None, args=None, kwargs=None, task=None,
                 stop = time()
 
                 if stop - start > 0.010:
-                    self.startstops[key].append(('deserialize', start, stop))
+                    self.startstops[key].append(("deserialize", start, stop))
             except Exception as e:
                 logger.warning("Could not deserialize task", exc_info=True)
                 emsg = error_message(e)
-                emsg['key'] = key
-                emsg['op'] = 'task-erred'
+                emsg["key"] = key
+                emsg["op"] = "task-erred"
                 self.batched_stream.send(emsg)
-                self.log.append((key, 'deserialize-error'))
+                self.log.append((key, "deserialize-error"))
                 return
 
             self.priorities[key] = priority
             self.durations[key] = duration
             if resource_restrictions:
                 self.resource_restrictions[key] = resource_restrictions
-            self.task_state[key] = 'waiting'
+            self.task_state[key] = "waiting"
 
             if nbytes is not None:
                 self.nbytes.update(nbytes)
@@ -1146,14 +1272,14 @@ def add_task(self, key, function=None, args=None, kwargs=None, task=None,
                 self.dependents[dep].add(key)
 
                 if dep not in self.dep_state:
-                    if self.task_state.get(dep) == 'memory':
-                        state = 'memory'
+                    if self.task_state.get(dep) == "memory":
+                        state = "memory"
                     else:
-                        state = 'waiting'
+                        state = "waiting"
                     self.dep_state[dep] = state
-                    self.log.append((dep, 'new-dep', state))
+                    self.log.append((dep, "new-dep", state))
 
-                if self.dep_state[dep] != 'memory':
+                if self.dep_state[dep] != "memory":
                     self.waiting_for_data[key].add(dep)
 
             for dep, workers in who_has.items():
@@ -1164,13 +1290,13 @@ def add_task(self, key, function=None, args=None, kwargs=None, task=None,
 
                 for worker in workers:
                     self.has_what[worker].add(dep)
-                    if self.dep_state[dep] != 'memory':
+                    if self.dep_state[dep] != "memory":
                         self.pending_data_per_worker[worker].append(dep)
 
             if self.waiting_for_data[key]:
                 self.data_needed.append(key)
             else:
-                self.transition(key, 'ready')
+                self.transition(key, "ready")
             if self.validate:
                 if who_has:
                     assert all(dep in self.dep_state for dep in who_has)
@@ -1182,6 +1308,7 @@ def add_task(self, key, function=None, args=None, kwargs=None, task=None,
             logger.exception(e)
             if LOG_PDB:
                 import pdb
+
                 pdb.set_trace()
             raise
 
@@ -1194,7 +1321,7 @@ def transition_dep(self, dep, finish, **kwargs):
             return
         func = self._dep_transitions[start, finish]
         state = func(dep, **kwargs)
-        self.log.append(('dep', dep, start, state or finish))
+        self.log.append(("dep", dep, start, state or finish))
         if dep in self.dep_state:
             self.dep_state[dep] = state or finish
             if self.validate:
@@ -1211,6 +1338,7 @@ def transition_dep_waiting_flight(self, dep, worker=None):
             logger.exception(e)
             if LOG_PDB:
                 import pdb
+
                 pdb.set_trace()
             raise
 
@@ -1235,7 +1363,7 @@ def transition_dep_flight_waiting(self, dep, worker=None, remove=True):
                     self._missing_dep_flight.add(dep)
                     self.loop.add_callback(self.handle_missing_dep, dep)
             for key in self.dependents.get(dep, ()):
-                if self.task_state[key] == 'waiting':
+                if self.task_state[key] == "waiting":
                     if remove:  # try a new worker immediately
                         self.data_needed.appendleft(key)
                     else:  # worker was probably busy, wait a while
@@ -1247,6 +1375,7 @@ def transition_dep_flight_waiting(self, dep, worker=None, remove=True):
             logger.exception(e)
             if LOG_PDB:
                 import pdb
+
                 pdb.set_trace()
             raise
 
@@ -1257,10 +1386,9 @@ def transition_dep_flight_memory(self, dep, value=None):
 
             del self.in_flight_tasks[dep]
             if self.dependents[dep]:
-                self.dep_state[dep] = 'memory'
+                self.dep_state[dep] = "memory"
                 self.put_key_in_memory(dep, value)
-                self.batched_stream.send({'op': 'add-keys',
-                                          'keys': [dep]})
+                self.batched_stream.send({"op": "add-keys", "keys": [dep]})
             else:
                 self.release_dep(dep)
 
@@ -1268,6 +1396,7 @@ def transition_dep_flight_memory(self, dep, value=None):
             logger.exception(e)
             if LOG_PDB:
                 import pdb
+
                 pdb.set_trace()
             raise
 
@@ -1277,11 +1406,12 @@ def transition_dep_waiting_memory(self, dep, value=None):
                 assert dep in self.data
                 assert dep in self.nbytes
                 assert dep in self.types
-                assert self.task_state[dep] == 'memory'
+                assert self.task_state[dep] == "memory"
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
                 import pdb
+
                 pdb.set_trace()
             raise
         if value is not no_value and dep not in self.data:
@@ -1301,10 +1431,13 @@ def transition(self, key, finish, **kwargs):
     def transition_waiting_ready(self, key):
         try:
             if self.validate:
-                assert self.task_state[key] == 'waiting'
+                assert self.task_state[key] == "waiting"
                 assert key in self.waiting_for_data
                 assert not self.waiting_for_data[key]
-                assert all(dep in self.data or dep in self.actors for dep in self.dependencies[key])
+                assert all(
+                    dep in self.data or dep in self.actors
+                    for dep in self.dependencies[key]
+                )
                 assert key not in self.executing
                 assert key not in self.ready
 
@@ -1312,20 +1445,21 @@ def transition_waiting_ready(self, key):
 
             if key in self.resource_restrictions:
                 self.constrained.append(key)
-                return 'constrained'
+                return "constrained"
             else:
                 heapq.heappush(self.ready, (self.priorities[key], key))
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
                 import pdb
+
                 pdb.set_trace()
             raise
 
     def transition_waiting_done(self, key, value=None):
         try:
             if self.validate:
-                assert self.task_state[key] == 'waiting'
+                assert self.task_state[key] == "waiting"
                 assert key in self.waiting_for_data
                 assert key not in self.executing
                 assert key not in self.ready
@@ -1336,6 +1470,7 @@ def transition_waiting_done(self, key, value=None):
             logger.exception(e)
             if LOG_PDB:
                 import pdb
+
                 pdb.set_trace()
             raise
 
@@ -1346,7 +1481,10 @@ def transition_ready_executing(self, key):
                 # assert key not in self.data
                 assert self.task_state[key] in READY
                 assert key not in self.ready
-                assert all(dep in self.data or dep in self.actors for dep in self.dependencies[key])
+                assert all(
+                    dep in self.data or dep in self.actors
+                    for dep in self.dependencies[key]
+                )
 
             self.executing.add(key)
             self.loop.add_callback(self.execute, key)
@@ -1354,6 +1492,7 @@ def transition_ready_executing(self, key):
             logger.exception(e)
             if LOG_PDB:
                 import pdb
+
                 pdb.set_trace()
             raise
 
@@ -1380,26 +1519,26 @@ def transition_executing_done(self, key, value=no_value, report=True):
                 for resource, quantity in self.resource_restrictions[key].items():
                     self.available_resources[resource] += quantity
 
-            if self.task_state[key] == 'executing':
+            if self.task_state[key] == "executing":
                 self.executing.remove(key)
                 self.executed_count += 1
-            elif self.task_state[key] == 'long-running':
+            elif self.task_state[key] == "long-running":
                 self.long_running.remove(key)
 
             if value is not no_value:
                 try:
-                    self.task_state[key] = 'memory'
+                    self.task_state[key] = "memory"
                     self.put_key_in_memory(key, value, transition=False)
                 except Exception as e:
                     logger.info("Failed to put key in memory", exc_info=True)
                     msg = error_message(e)
-                    self.exceptions[key] = msg['exception']
-                    self.tracebacks[key] = msg['traceback']
-                    self.task_state[key] = 'error'
-                    out = 'error'
+                    self.exceptions[key] = msg["exception"]
+                    self.tracebacks[key] = msg["traceback"]
+                    self.task_state[key] = "error"
+                    out = "error"
 
                 if key in self.dep_state:
-                    self.transition_dep(key, 'memory')
+                    self.transition_dep(key, "memory")
 
             if report and self.batched_stream:
                 self.send_task_state_to_scheduler(key)
@@ -1414,6 +1553,7 @@ def transition_executing_done(self, key, value=no_value, report=True):
             logger.exception(e)
             if LOG_PDB:
                 import pdb
+
                 pdb.set_trace()
             raise
 
@@ -1424,45 +1564,59 @@ def transition_executing_long_running(self, key, compute_duration=None):
 
             self.executing.remove(key)
             self.long_running.add(key)
-            self.batched_stream.send({'op': 'long-running',
-                                      'key': key,
-                                      'compute_duration': compute_duration})
+            self.batched_stream.send(
+                {"op": "long-running", "key": key, "compute_duration": compute_duration}
+            )
 
             self.ensure_computing()
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
                 import pdb
+
                 pdb.set_trace()
             raise
 
     def maybe_transition_long_running(self, key, compute_duration=None):
-        if self.task_state.get(key) == 'executing':
-            self.transition(key, 'long-running', compute_duration=compute_duration)
+        if self.task_state.get(key) == "executing":
+            self.transition(key, "long-running", compute_duration=compute_duration)
 
     def stateof(self, key):
-        return {'executing': key in self.executing,
-                'waiting_for_data': key in self.waiting_for_data,
-                'heap': key in pluck(1, self.ready),
-                'data': key in self.data}
+        return {
+            "executing": key in self.executing,
+            "waiting_for_data": key in self.waiting_for_data,
+            "heap": key in pluck(1, self.ready),
+            "data": key in self.data,
+        }
 
     def story(self, *keys):
-        return [msg for msg in self.log
-                if any(key in msg for key in keys)
-                or any(key in c
-                       for key in keys
-                       for c in msg
-                       if isinstance(c, (tuple, list, set)))]
+        return [
+            msg
+            for msg in self.log
+            if any(key in msg for key in keys)
+            or any(
+                key in c
+                for key in keys
+                for c in msg
+                if isinstance(c, (tuple, list, set))
+            )
+        ]
 
     def ensure_communicating(self):
         changed = True
         try:
-            while changed and self.data_needed and len(self.in_flight_workers) < self.total_out_connections:
+            while (
+                changed
+                and self.data_needed
+                and len(self.in_flight_workers) < self.total_out_connections
+            ):
                 changed = False
-                logger.debug("Ensure communicating.  Pending: %d.  Connections: %d/%d",
-                             len(self.data_needed),
-                             len(self.in_flight_workers),
-                             self.total_out_connections)
+                logger.debug(
+                    "Ensure communicating.  Pending: %d.  Connections: %d/%d",
+                    len(self.data_needed),
+                    len(self.in_flight_workers),
+                    self.total_out_connections,
+                )
 
                 key = self.data_needed[0]
 
@@ -1471,8 +1625,8 @@ def ensure_communicating(self):
                     changed = True
                     continue
 
-                if self.task_state.get(key) != 'waiting':
-                    self.log.append((key, 'communication pass'))
+                if self.task_state.get(key) != "waiting":
+                    self.log.append((key, "communication pass"))
                     self.data_needed.popleft()
                     changed = True
                     continue
@@ -1481,33 +1635,38 @@ def ensure_communicating(self):
                 if self.validate:
                     assert all(dep in self.dep_state for dep in deps)
 
-                deps = [dep for dep in deps if self.dep_state[dep] == 'waiting']
+                deps = [dep for dep in deps if self.dep_state[dep] == "waiting"]
 
                 missing_deps = {dep for dep in deps if not self.who_has.get(dep)}
                 if missing_deps:
                     logger.info("Can't find dependencies for key %s", key)
-                    missing_deps2 = {dep for dep in missing_deps
-                                     if dep not in self._missing_dep_flight}
+                    missing_deps2 = {
+                        dep
+                        for dep in missing_deps
+                        if dep not in self._missing_dep_flight
+                    }
                     for dep in missing_deps2:
                         self._missing_dep_flight.add(dep)
-                    self.loop.add_callback(self.handle_missing_dep,
-                                           *missing_deps2)
+                    self.loop.add_callback(self.handle_missing_dep, *missing_deps2)
 
                     deps = [dep for dep in deps if dep not in missing_deps]
 
-                self.log.append(('gather-dependencies', key, deps))
+                self.log.append(("gather-dependencies", key, deps))
 
                 in_flight = False
 
-                while deps and (len(self.in_flight_workers) < self.total_out_connections
-                                or self.comm_nbytes < self.total_comm_nbytes):
+                while deps and (
+                    len(self.in_flight_workers) < self.total_out_connections
+                    or self.comm_nbytes < self.total_comm_nbytes
+                ):
                     dep = deps.pop()
-                    if self.dep_state[dep] != 'waiting':
+                    if self.dep_state[dep] != "waiting":
                         continue
                     if dep not in self.who_has:
                         continue
-                    workers = [w for w in self.who_has[dep]
-                               if w not in self.in_flight_workers]
+                    workers = [
+                        w for w in self.who_has[dep] if w not in self.in_flight_workers
+                    ]
                     if not workers:
                         in_flight = True
                         continue
@@ -1521,9 +1680,10 @@ def ensure_communicating(self):
                     self.comm_nbytes += total_nbytes
                     self.in_flight_workers[worker] = to_gather
                     for d in to_gather:
-                        self.transition_dep(d, 'flight', worker=worker)
-                    self.loop.add_callback(self.gather_dep, worker, dep,
-                                           to_gather, total_nbytes, cause=key)
+                        self.transition_dep(d, "flight", worker=worker)
+                    self.loop.add_callback(
+                        self.gather_dep, worker, dep, to_gather, total_nbytes, cause=key
+                    )
                     changed = True
 
                 if not deps and not in_flight:
@@ -1532,6 +1692,7 @@ def ensure_communicating(self):
             logger.exception(e)
             if LOG_PDB:
                 import pdb
+
                 pdb.set_trace()
             raise
 
@@ -1550,26 +1711,31 @@ def send_task_state_to_scheduler(self, key):
                 # Some types fail pickling (example: _thread.lock objects),
                 # send their name as a best effort.
                 typ = pickle.dumps(typ.__name__)
-            d = {'op': 'task-finished',
-                 'status': 'OK',
-                 'key': key,
-                 'nbytes': nbytes,
-                 'thread': self.threads.get(key),
-                 'type': typ}
+            d = {
+                "op": "task-finished",
+                "status": "OK",
+                "key": key,
+                "nbytes": nbytes,
+                "thread": self.threads.get(key),
+                "type": typ,
+            }
         elif key in self.exceptions:
-            d = {'op': 'task-erred',
-                 'status': 'error',
-                 'key': key,
-                 'thread': self.threads.get(key),
-                 'exception': self.exceptions[key],
-                 'traceback': self.tracebacks[key]}
+            d = {
+                "op": "task-erred",
+                "status": "error",
+                "key": key,
+                "thread": self.threads.get(key),
+                "exception": self.exceptions[key],
+                "traceback": self.tracebacks[key],
+            }
         else:
-            logger.error("Key not ready to send to worker, %s: %s",
-                         key, self.task_state[key])
+            logger.error(
+                "Key not ready to send to worker, %s: %s", key, self.task_state[key]
+            )
             return
 
         if key in self.startstops:
-            d['startstops'] = self.startstops[key]
+            d["startstops"] = self.startstops[key]
         self.batched_stream.send(d)
 
     def put_key_in_memory(self, key, value, transition=True):
@@ -1584,7 +1750,7 @@ def put_key_in_memory(self, key, value, transition=True):
             self.data[key] = value
             stop = time()
             if stop - start > 0.020:
-                self.startstops[key].append(('disk-write', start, stop))
+                self.startstops[key].append(("disk-write", start, stop))
 
         if key not in self.nbytes:
             self.nbytes[key] = sizeof(value)
@@ -1596,12 +1762,12 @@ def put_key_in_memory(self, key, value, transition=True):
                 if key in self.waiting_for_data[dep]:
                     self.waiting_for_data[dep].remove(key)
                 if not self.waiting_for_data[dep]:
-                    self.transition(dep, 'ready')
+                    self.transition(dep, "ready")
 
         if transition and key in self.task_state:
-            self.transition(key, 'memory')
+            self.transition(key, "memory")
 
-        self.log.append((key, 'put-in-memory'))
+        self.log.append((key, "put-in-memory"))
 
     def select_keys_for_gather(self, worker, dep):
         deps = {dep}
@@ -1611,7 +1777,7 @@ def select_keys_for_gather(self, worker, dep):
 
         while L:
             d = L.popleft()
-            if self.dep_state.get(d) != 'waiting':
+            if self.dep_state.get(d) != "waiting":
                 continue
             if total_bytes + self.nbytes[d] > self.target_message_size:
                 break
@@ -1622,7 +1788,7 @@ def select_keys_for_gather(self, worker, dep):
 
     @gen.coroutine
     def gather_dep(self, worker, dep, deps, total_nbytes, cause=None):
-        if self.status != 'running':
+        if self.status != "running":
             return
         with log_errors():
             response = {}
@@ -1632,58 +1798,64 @@ def gather_dep(self, worker, dep, deps, total_nbytes, cause=None):
 
                 # dep states may have changed before gather_dep runs
                 # if a dep is no longer in-flight then don't fetch it
-                deps = tuple(dep for dep in deps
-                                 if self.dep_state.get(dep) == 'flight')
+                deps = tuple(dep for dep in deps if self.dep_state.get(dep) == "flight")
 
-                self.log.append(('request-dep', dep, worker, deps))
+                self.log.append(("request-dep", dep, worker, deps))
                 logger.debug("Request %d keys", len(deps))
 
                 start = time()
-                response = yield get_data_from_worker(self.rpc, deps, worker,
-                                                      who=self.address)
+                response = yield get_data_from_worker(
+                    self.rpc, deps, worker, who=self.address
+                )
                 stop = time()
 
-                if response['status'] == 'busy':
-                    self.log.append(('busy-gather', worker, deps))
+                if response["status"] == "busy":
+                    self.log.append(("busy-gather", worker, deps))
                     for dep in deps:
-                        if self.dep_state.get(dep, None) == 'flight':
-                            self.transition_dep(dep, 'waiting')
+                        if self.dep_state.get(dep, None) == "flight":
+                            self.transition_dep(dep, "waiting")
                     return
 
                 if cause:
-                    self.startstops[cause].append((
-                        'transfer',
-                        start + self.scheduler_delay,
-                        stop + self.scheduler_delay
-                    ))
-
-                total_bytes = sum(self.nbytes.get(dep, 0) for dep in response['data'])
+                    self.startstops[cause].append(
+                        (
+                            "transfer",
+                            start + self.scheduler_delay,
+                            stop + self.scheduler_delay,
+                        )
+                    )
+
+                total_bytes = sum(self.nbytes.get(dep, 0) for dep in response["data"])
                 duration = (stop - start) or 0.5
-                self.incoming_transfer_log.append({
-                    'start': start + self.scheduler_delay,
-                    'stop': stop + self.scheduler_delay,
-                    'middle': (start + stop) / 2.0 + self.scheduler_delay,
-                    'duration': duration,
-                    'keys': {dep: self.nbytes.get(dep, None) for dep in response['data']},
-                    'total': total_bytes,
-                    'bandwidth': total_bytes / duration,
-                    'who': worker
-                })
+                self.incoming_transfer_log.append(
+                    {
+                        "start": start + self.scheduler_delay,
+                        "stop": stop + self.scheduler_delay,
+                        "middle": (start + stop) / 2.0 + self.scheduler_delay,
+                        "duration": duration,
+                        "keys": {
+                            dep: self.nbytes.get(dep, None) for dep in response["data"]
+                        },
+                        "total": total_bytes,
+                        "bandwidth": total_bytes / duration,
+                        "who": worker,
+                    }
+                )
                 if self.digests is not None:
-                    self.digests['transfer-bandwidth'].add(total_bytes / duration)
-                    self.digests['transfer-duration'].add(duration)
-                self.counters['transfer-count'].add(len(response['data']))
+                    self.digests["transfer-bandwidth"].add(total_bytes / duration)
+                    self.digests["transfer-duration"].add(duration)
+                self.counters["transfer-count"].add(len(response["data"]))
                 self.incoming_count += 1
 
-                self.log.append(('receive-dep', worker, list(response['data'])))
+                self.log.append(("receive-dep", worker, list(response["data"])))
 
-                if response['data']:
-                    self.batched_stream.send({'op': 'add-keys',
-                                              'keys': list(response['data'])})
+                if response["data"]:
+                    self.batched_stream.send(
+                        {"op": "add-keys", "keys": list(response["data"])}
+                    )
             except EnvironmentError as e:
-                logger.exception("Worker stream died during communication: %s",
-                                 worker)
-                self.log.append(('receive-dep-failed', worker))
+                logger.exception("Worker stream died during communication: %s", worker)
+                self.log.append(("receive-dep-failed", worker))
                 for d in self.has_what.pop(worker):
                     self.who_has[d].remove(worker)
                     if not self.who_has[d]:
@@ -1693,25 +1865,27 @@ def gather_dep(self, worker, dep, deps, total_nbytes, cause=None):
                 logger.exception(e)
                 if self.batched_stream and LOG_PDB:
                     import pdb
+
                     pdb.set_trace()
                 raise
             finally:
                 self.comm_nbytes -= total_nbytes
-                busy = response.get('status', '') == 'busy'
-                data = response.get('data', {})
+                busy = response.get("status", "") == "busy"
+                data = response.get("data", {})
 
                 for d in self.in_flight_workers.pop(worker):
                     if not busy and d in data:
-                        self.transition_dep(d, 'memory', value=data[d])
-                    elif self.dep_state.get(d) != 'memory':
-                        self.transition_dep(d, 'waiting', worker=worker,
-                                            remove=not busy)
+                        self.transition_dep(d, "memory", value=data[d])
+                    elif self.dep_state.get(d) != "memory":
+                        self.transition_dep(
+                            d, "waiting", worker=worker, remove=not busy
+                        )
 
                     if not busy and d not in data and d in self.dependents:
-                        self.log.append(('missing-dep', d))
-                        self.batched_stream.send({'op': 'missing-data',
-                                                  'errant_worker': worker,
-                                                  'key': d})
+                        self.log.append(("missing-dep", d))
+                        self.batched_stream.send(
+                            {"op": "missing-data", "errant_worker": worker, "key": d}
+                        )
 
                 if self.validate:
                     self.validate_state()
@@ -1734,15 +1908,15 @@ def bad_dep(self, dep):
         exc = ValueError("Could not find dependent %s.  Check worker logs" % str(dep))
         for key in self.dependents[dep]:
             msg = error_message(exc)
-            self.exceptions[key] = msg['exception']
-            self.tracebacks[key] = msg['traceback']
-            self.transition(key, 'error')
+            self.exceptions[key] = msg["exception"]
+            self.tracebacks[key] = msg["traceback"]
+            self.transition(key, "error")
         self.release_dep(dep)
 
     @gen.coroutine
     def handle_missing_dep(self, *deps, **kwargs):
         original_deps = list(deps)
-        self.log.append(('handle-missing', deps))
+        self.log.append(("handle-missing", deps))
         try:
             deps = {dep for dep in deps if dep in self.dependents}
             if not deps:
@@ -1757,8 +1931,11 @@ def handle_missing_dep(self, *deps, **kwargs):
                 return
 
             for dep in deps:
-                logger.info("Dependent not found: %s %s .  Asking scheduler",
-                            dep, self.suspicious_deps[dep])
+                logger.info(
+                    "Dependent not found: %s %s .  Asking scheduler",
+                    dep,
+                    self.suspicious_deps[dep],
+                )
 
             who_has = yield self.scheduler.who_has(keys=list(deps))
             who_has = {k: v for k, v in who_has.items() if v}
@@ -1767,19 +1944,18 @@ def handle_missing_dep(self, *deps, **kwargs):
                 self.suspicious_deps[dep] += 1
 
                 if not who_has.get(dep):
-                    self.log.append((dep, 'no workers found',
-                                     self.dependents.get(dep)))
+                    self.log.append((dep, "no workers found", self.dependents.get(dep)))
                     self.release_dep(dep)
                 else:
-                    self.log.append((dep, 'new workers found'))
+                    self.log.append((dep, "new workers found"))
                     for key in self.dependents.get(dep, ()):
                         if key in self.waiting_for_data:
                             self.data_needed.append(key)
 
         except Exception:
             logger.error("Handle missing dep failed, retrying", exc_info=True)
-            retries = kwargs.get('retries', 5)
-            self.log.append(('handle-missing-failed', retries, deps))
+            retries = kwargs.get("retries", 5)
+            self.log.append(("handle-missing-failed", retries, deps))
             if retries > 0:
                 yield self.handle_missing_dep(self, *deps, retries=retries - 1)
             else:
@@ -1816,18 +1992,17 @@ def update_who_has(self, who_has):
             logger.exception(e)
             if LOG_PDB:
                 import pdb
+
                 pdb.set_trace()
             raise
 
     def steal_request(self, key):
         state = self.task_state.get(key, None)
 
-        response = {'op': 'steal-response',
-                    'key': key,
-                    'state': state}
+        response = {"op": "steal-response", "key": key, "state": state}
         self.batched_stream.send(response)
 
-        if state in ('ready', 'waiting'):
+        if state in ("ready", "waiting"):
             self.release_key(key)
 
     def release_key(self, key, cause=None, reason=None, report=True):
@@ -1836,16 +2011,15 @@ def release_key(self, key, cause=None, reason=None, report=True):
                 return
             state = self.task_state.pop(key)
             if cause:
-                self.log.append((key, 'release-key', {'cause': cause}))
+                self.log.append((key, "release-key", {"cause": cause}))
             else:
-                self.log.append((key, 'release-key'))
+                self.log.append((key, "release-key"))
             del self.tasks[key]
             if key in self.data and key not in self.dep_state:
                 try:
                     del self.data[key]
                 except FileNotFoundError:
-                    logger.error("Tried to delete %s but no file found",
-                                 exc_info=True)
+                    logger.error("Tried to delete %s but no file found", exc_info=True)
                 del self.nbytes[key]
                 del self.types[key]
             if key in self.actors and key not in self.dep_state:
@@ -1859,7 +2033,10 @@ def release_key(self, key, cause=None, reason=None, report=True):
             for dep in self.dependencies.pop(key, ()):
                 if dep in self.dependents:
                     self.dependents[dep].discard(key)
-                    if not self.dependents[dep] and self.dep_state[dep] in ('waiting', 'flight'):
+                    if not self.dependents[dep] and self.dep_state[dep] in (
+                        "waiting",
+                        "flight",
+                    ):
                         self.release_dep(dep)
 
             if key in self.threads:
@@ -1879,21 +2056,20 @@ def release_key(self, key, cause=None, reason=None, report=True):
                 self.executing.remove(key)
 
             if key in self.resource_restrictions:
-                if state == 'executing':
+                if state == "executing":
                     for resource, quantity in self.resource_restrictions[key].items():
                         self.available_resources[resource] += quantity
                 del self.resource_restrictions[key]
 
             if report and state in PROCESSING:  # not finished
-                self.batched_stream.send({'op': 'release',
-                                          'key': key,
-                                          'cause': cause})
+                self.batched_stream.send({"op": "release", "key": key, "cause": cause})
         except CommClosedError:
             pass
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
                 import pdb
+
                 pdb.set_trace()
             raise
 
@@ -1901,7 +2077,7 @@ def release_dep(self, dep, report=False):
         try:
             if dep not in self.dep_state:
                 return
-            self.log.append((dep, 'release-dep'))
+            self.log.append((dep, "release-dep"))
             state = self.dep_state.pop(dep)
 
             if dep in self.suspicious_deps:
@@ -1925,16 +2101,16 @@ def release_dep(self, dep, report=False):
                 self.in_flight_workers[worker].remove(dep)
 
             for key in self.dependents.pop(dep, ()):
-                if self.task_state[key] != 'memory':
+                if self.task_state[key] != "memory":
                     self.release_key(key, cause=dep)
 
-            if report and state == 'memory':
-                self.batched_stream.send({'op': 'release-worker-data',
-                                          'keys': [dep]})
+            if report and state == "memory":
+                self.batched_stream.send({"op": "release-worker-data", "keys": [dep]})
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
                 import pdb
+
                 pdb.set_trace()
             raise
 
@@ -1963,6 +2139,7 @@ def rescind_key(self, key):
             logger.exception(e)
             if LOG_PDB:
                 import pdb
+
                 pdb.set_trace()
             raise
 
@@ -1971,8 +2148,7 @@ def rescind_key(self, key):
     ################
 
     @gen.coroutine
-    def executor_submit(self, key, function, args=(), kwargs=None,
-                        executor=None):
+    def executor_submit(self, key, function, args=(), kwargs=None, executor=None):
         """ Safely run function in thread pool executor
 
         We've run into issues running concurrent.future futures within
@@ -1985,8 +2161,9 @@ def executor_submit(self, key, function, args=(), kwargs=None,
         # logger.info("%s:%d Starts job %d, %s", self.ip, self.port, i, key)
         kwargs = kwargs or {}
         future = executor.submit(function, *args, **kwargs)
-        pc = PeriodicCallback(lambda: logger.debug("future state: %s - %s",
-                                                   key, future._state), 1000)
+        pc = PeriodicCallback(
+            lambda: logger.debug("future state: %s - %s", key, future._state), 1000
+        )
         pc.start()
         try:
             yield future
@@ -2000,39 +2177,43 @@ def executor_submit(self, key, function, args=(), kwargs=None,
 
     def run(self, comm, function, args=(), wait=True, kwargs=None):
         kwargs = kwargs or {}
-        return run(self, comm, function=function, args=args, kwargs=kwargs,
-                   wait=wait)
+        return run(self, comm, function=function, args=args, kwargs=kwargs, wait=wait)
 
     def run_coroutine(self, comm, function, args=(), kwargs=None, wait=True):
-        return run(self, comm, function=function, args=args, kwargs=kwargs,
-                   wait=wait)
+        return run(self, comm, function=function, args=args, kwargs=kwargs, wait=wait)
 
     @gen.coroutine
     def actor_execute(self, comm=None, actor=None, function=None, args=(), kwargs={}):
-        separate_thread = kwargs.pop('separate_thread', True)
+        separate_thread = kwargs.pop("separate_thread", True)
         key = actor
         actor = self.actors[key]
         func = getattr(actor, function)
-        name = key_split(key) + '.' + function
+        name = key_split(key) + "." + function
 
         if iscoroutinefunction(func):
             result = yield func(*args, **kwargs)
         elif separate_thread:
-            result = yield self.executor_submit(name,
-                                                apply_function_actor,
-                                                args=(func, args, kwargs,
-                                                      self.execution_state,
-                                                      name,
-                                                      self.active_threads,
-                                                      self.active_threads_lock),
-                                                executor=self.actor_executor)
+            result = yield self.executor_submit(
+                name,
+                apply_function_actor,
+                args=(
+                    func,
+                    args,
+                    kwargs,
+                    self.execution_state,
+                    name,
+                    self.active_threads,
+                    self.active_threads_lock,
+                ),
+                executor=self.actor_executor,
+            )
         else:
             result = func(*args, **kwargs)
-        raise gen.Return({'status': 'OK', 'result': to_serialize(result)})
+        raise gen.Return({"status": "OK", "result": to_serialize(result)})
 
     def actor_attribute(self, comm=None, actor=None, attribute=None):
         value = getattr(self.actors[actor], attribute)
-        return {'status': 'OK', 'result': to_serialize(value)}
+        return {"status": "OK", "result": to_serialize(value)}
 
     def meets_resource_constraints(self, key):
         if key not in self.resource_restrictions:
@@ -2049,36 +2230,37 @@ def ensure_computing(self):
         try:
             while self.constrained and len(self.executing) < self.ncores:
                 key = self.constrained[0]
-                if self.task_state.get(key) != 'constrained':
+                if self.task_state.get(key) != "constrained":
                     self.constrained.popleft()
                     continue
                 if self.meets_resource_constraints(key):
                     self.constrained.popleft()
-                    self.transition(key, 'executing')
+                    self.transition(key, "executing")
                 else:
                     break
             while self.ready and len(self.executing) < self.ncores:
                 _, key = heapq.heappop(self.ready)
                 if self.task_state.get(key) in READY:
-                    self.transition(key, 'executing')
+                    self.transition(key, "executing")
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
                 import pdb
+
                 pdb.set_trace()
             raise
 
     @gen.coroutine
     def execute(self, key, report=False):
         executor_error = None
-        if self.status in ('closing', 'closed'):
+        if self.status in ("closing", "closed"):
             return
         try:
             if key not in self.executing or key not in self.task_state:
                 return
             if self.validate:
                 assert key not in self.waiting_for_data
-                assert self.task_state[key] == 'executing'
+                assert self.task_state[key] == "executing"
 
             function, args, kwargs = self.tasks[key]
 
@@ -2089,64 +2271,74 @@ def execute(self, key, report=False):
                     data[k] = self.data[k]
                 except KeyError:
                     from .actor import Actor  # TODO: create local actor
+
                     data[k] = Actor(type(self.actors[k]), self.address, k, self)
             args2 = pack_data(args, data, key_types=(bytes, unicode))
             kwargs2 = pack_data(kwargs, data, key_types=(bytes, unicode))
             stop = time()
             if stop - start > 0.005:
-                self.startstops[key].append(('disk-read', start, stop))
+                self.startstops[key].append(("disk-read", start, stop))
                 if self.digests is not None:
-                    self.digests['disk-load-duration'].add(stop - start)
+                    self.digests["disk-load-duration"].add(stop - start)
 
-            logger.debug("Execute key: %s worker: %s", key, self.address)  # TODO: comment out?
+            logger.debug(
+                "Execute key: %s worker: %s", key, self.address
+            )  # TODO: comment out?
             try:
-                result = yield self.executor_submit(key, apply_function,
-                                                    args=(function, args2, kwargs2,
-                                                          self.execution_state, key,
-                                                          self.active_threads,
-                                                          self.active_threads_lock,
-                                                          self.scheduler_delay))
+                result = yield self.executor_submit(
+                    key,
+                    apply_function,
+                    args=(
+                        function,
+                        args2,
+                        kwargs2,
+                        self.execution_state,
+                        key,
+                        self.active_threads,
+                        self.active_threads_lock,
+                        self.scheduler_delay,
+                    ),
+                )
             except RuntimeError as e:
                 executor_error = e
                 raise
 
-            if self.task_state.get(key) not in ('executing', 'long-running'):
+            if self.task_state.get(key) not in ("executing", "long-running"):
                 return
 
-            result['key'] = key
-            value = result.pop('result', None)
-            self.startstops[key].append(('compute', result['start'],
-                                         result['stop']))
-            self.threads[key] = result['thread']
+            result["key"] = key
+            value = result.pop("result", None)
+            self.startstops[key].append(("compute", result["start"], result["stop"]))
+            self.threads[key] = result["thread"]
 
-            if result['op'] == 'task-finished':
-                self.nbytes[key] = result['nbytes']
-                self.types[key] = result['type']
-                self.transition(key, 'memory', value=value)
+            if result["op"] == "task-finished":
+                self.nbytes[key] = result["nbytes"]
+                self.types[key] = result["type"]
+                self.transition(key, "memory", value=value)
                 if self.digests is not None:
-                    self.digests['task-duration'].add(result['stop'] -
-                                                      result['start'])
+                    self.digests["task-duration"].add(result["stop"] - result["start"])
             else:
-                if isinstance(result.pop('actual-exception'), Reschedule):
-                    self.batched_stream.send({'op': 'reschedule', 'key': key})
-                    self.transition(key, 'rescheduled', report=False)
+                if isinstance(result.pop("actual-exception"), Reschedule):
+                    self.batched_stream.send({"op": "reschedule", "key": key})
+                    self.transition(key, "rescheduled", report=False)
                     self.release_key(key, report=False)
                 else:
-                    self.exceptions[key] = result['exception']
-                    self.tracebacks[key] = result['traceback']
-                    logger.warning(" Compute Failed\n"
-                                   "Function:  %s\n"
-                                   "args:      %s\n"
-                                   "kwargs:    %s\n"
-                                   "Exception: %s\n",
-                                   str(funcname(function))[:1000],
-                                   convert_args_to_str(args2, max_len=1000),
-                                   convert_kwargs_to_str(kwargs2, max_len=1000),
-                                   repr(result['exception'].data))
-                    self.transition(key, 'error')
-
-            logger.debug("Send compute response to scheduler: %s, %s", key,
-                         result)
+                    self.exceptions[key] = result["exception"]
+                    self.tracebacks[key] = result["traceback"]
+                    logger.warning(
+                        " Compute Failed\n"
+                        "Function:  %s\n"
+                        "args:      %s\n"
+                        "kwargs:    %s\n"
+                        "Exception: %s\n",
+                        str(funcname(function))[:1000],
+                        convert_args_to_str(args2, max_len=1000),
+                        convert_kwargs_to_str(kwargs2, max_len=1000),
+                        repr(result["exception"].data),
+                    )
+                    self.transition(key, "error")
+
+            logger.debug("Send compute response to scheduler: %s, %s", key, result)
 
             if self.validate:
                 assert key not in self.executing
@@ -2161,6 +2353,7 @@ def execute(self, key, report=False):
                 logger.exception(e)
                 if LOG_PDB:
                     import pdb
+
                     pdb.set_trace()
                 raise
         finally:
@@ -2193,18 +2386,22 @@ def memory_monitor(self):
             # Try to free some memory while in paused state
             self._throttled_gc.collect()
             if not self.paused:
-                logger.warning("Worker is at %d%% memory usage. Pausing worker.  "
-                               "Process memory: %s -- Worker memory limit: %s",
-                               int(frac * 100),
-                               format_bytes(proc.memory_info().rss),
-                               format_bytes(self.memory_limit))
+                logger.warning(
+                    "Worker is at %d%% memory usage. Pausing worker.  "
+                    "Process memory: %s -- Worker memory limit: %s",
+                    int(frac * 100),
+                    format_bytes(proc.memory_info().rss),
+                    format_bytes(self.memory_limit),
+                )
                 self.paused = True
         elif self.paused:
-            logger.warning("Worker is at %d%% memory usage. Resuming worker. "
-                           "Process memory: %s -- Worker memory limit: %s",
-                           int(frac * 100),
-                           format_bytes(proc.memory_info().rss),
-                           format_bytes(self.memory_limit))
+            logger.warning(
+                "Worker is at %d%% memory usage. Resuming worker. "
+                "Process memory: %s -- Worker memory limit: %s",
+                int(frac * 100),
+                format_bytes(proc.memory_info().rss),
+                format_bytes(self.memory_limit),
+            )
             self.paused = False
             self.ensure_computing()
 
@@ -2215,12 +2412,14 @@ def memory_monitor(self):
             need = memory - target
             while memory > target:
                 if not self.data.fast:
-                    logger.warning("Memory use is high but worker has no data "
-                                   "to store to disk.  Perhaps some other process "
-                                   "is leaking memory?  Process memory: %s -- "
-                                   "Worker memory limit: %s",
-                                   format_bytes(proc.memory_info().rss),
-                                   format_bytes(self.memory_limit))
+                    logger.warning(
+                        "Memory use is high but worker has no data "
+                        "to store to disk.  Perhaps some other process "
+                        "is leaking memory?  Process memory: %s -- "
+                        "Worker memory limit: %s",
+                        format_bytes(proc.memory_info().rss),
+                        format_bytes(self.memory_limit),
+                    )
                     break
                 k, v, weight = self.data.fast.evict()
                 del k, v
@@ -2235,8 +2434,11 @@ def memory_monitor(self):
                     self._throttled_gc.collect()
                     memory = proc.memory_info().rss
             if count:
-                logger.debug("Moved %d pieces of data data and %s to disk",
-                             count, format_bytes(total))
+                logger.debug(
+                    "Moved %d pieces of data data and %s to disk",
+                    count,
+                    format_bytes(total),
+                )
 
         self._memory_monitoring = False
         raise gen.Return(total)
@@ -2265,21 +2467,22 @@ def trigger_profile(self):
         for ident, frame in frames.items():
             if frame is not None:
                 key = key_split(active_threads[ident])
-                profile.process(frame, None, self.profile_recent,
-                                stop='distributed/worker.py')
-                profile.process(frame, None, self.profile_keys[key],
-                                stop='distributed/worker.py')
+                profile.process(
+                    frame, None, self.profile_recent, stop="distributed/worker.py"
+                )
+                profile.process(
+                    frame, None, self.profile_keys[key], stop="distributed/worker.py"
+                )
         stop = time()
         if self.digests is not None:
-            self.digests['profile-duration'].add(stop - start)
+            self.digests["profile-duration"].add(stop - start)
 
     def get_profile(self, comm=None, start=None, stop=None, key=None):
         now = time() + self.scheduler_delay
         if key is None:
             history = self.profile_history
         else:
-            history = [(t, d[key]) for t, d in self.profile_keys_history
-                       if key in d]
+            history = [(t, d[key]) for t, d in self.profile_keys_history if key in d]
         if start is None:
             istart = 0
         else:
@@ -2318,23 +2521,28 @@ def get_profile_metadata(self, comm=None, start=0, stop=None):
         now = time() + self.scheduler_delay
         stop = stop or now
         start = start or 0
-        result = {'counts': [(t, d['count']) for t, d in self.profile_history
-                             if start < t < stop],
-                  'keys': [(t, {k: d['count'] for k, d in v.items()})
-                           for t, v in self.profile_keys_history
-                           if start < t < stop]}
+        result = {
+            "counts": [
+                (t, d["count"]) for t, d in self.profile_history if start < t < stop
+            ],
+            "keys": [
+                (t, {k: d["count"] for k, d in v.items()})
+                for t, v in self.profile_keys_history
+                if start < t < stop
+            ],
+        }
         if add_recent:
-            result['counts'].append((now, self.profile_recent['count']))
-            result['keys'].append((now, {k: v['count']
-                                          for k, v in self.profile_keys.items()}))
+            result["counts"].append((now, self.profile_recent["count"]))
+            result["keys"].append(
+                (now, {k: v["count"] for k, v in self.profile_keys.items()})
+            )
         return result
 
     def get_call_stack(self, comm=None, keys=None):
         with self.active_threads_lock:
             frames = sys._current_frames()
             active_threads = self.active_threads.copy()
-            frames = {k: frames[ident]
-                      for ident, k in active_threads.items()}
+            frames = {k: frames[ident] for ident, k in active_threads.items()}
         if keys is not None:
             frames = {k: frame for k, frame in frames.items() if k in keys}
 
@@ -2352,20 +2560,24 @@ def validate_key_memory(self, key):
         assert key not in self.executing
         assert key not in self.ready
         if key in self.dep_state:
-            assert self.dep_state[key] == 'memory'
+            assert self.dep_state[key] == "memory"
 
     def validate_key_executing(self, key):
         assert key in self.executing
         assert key not in self.data
         assert key not in self.waiting_for_data
-        assert all(dep in self.data or dep in self.actors for dep in self.dependencies[key])
+        assert all(
+            dep in self.data or dep in self.actors for dep in self.dependencies[key]
+        )
 
     def validate_key_ready(self, key):
         assert key in pluck(1, self.ready)
         assert key not in self.data
         assert key not in self.executing
         assert key not in self.waiting_for_data
-        assert all(dep in self.data or dep in self.actors for dep in self.dependencies[key])
+        assert all(
+            dep in self.data or dep in self.actors for dep in self.dependencies[key]
+        )
 
     def validate_key_waiting(self, key):
         assert key not in self.data
@@ -2374,18 +2586,19 @@ def validate_key_waiting(self, key):
     def validate_key(self, key):
         try:
             state = self.task_state[key]
-            if state == 'memory':
+            if state == "memory":
                 self.validate_key_memory(key)
-            elif state == 'waiting':
+            elif state == "waiting":
                 self.validate_key_waiting(key)
-            elif state == 'ready':
+            elif state == "ready":
                 self.validate_key_ready(key)
-            elif state == 'executing':
+            elif state == "executing":
                 self.validate_key_executing(key)
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
                 import pdb
+
                 pdb.set_trace()
             raise
 
@@ -2407,16 +2620,16 @@ def validate_dep_memory(self, dep):
         assert dep in self.nbytes
         assert dep in self.types
         if dep in self.task_state:
-            assert self.task_state[dep] == 'memory'
+            assert self.task_state[dep] == "memory"
 
     def validate_dep(self, dep):
         try:
             state = self.dep_state[dep]
-            if state == 'waiting':
+            if state == "waiting":
                 self.validate_dep_waiting(dep)
-            elif state == 'flight':
+            elif state == "flight":
                 self.validate_dep_flight(dep)
-            elif state == 'memory':
+            elif state == "memory":
                 self.validate_dep_memory(dep)
             else:
                 raise ValueError("Unknown dependent state", state)
@@ -2424,11 +2637,12 @@ def validate_dep(self, dep):
             logger.exception(e)
             if LOG_PDB:
                 import pdb
+
                 pdb.set_trace()
             raise
 
     def validate_state(self):
-        if self.status != 'running':
+        if self.status != "running":
             return
         try:
             for key, workers in self.who_has.items():
@@ -2448,12 +2662,14 @@ def validate_state(self):
             for key, deps in self.waiting_for_data.items():
                 if key not in self.data_needed:
                     for dep in deps:
-                        assert (dep in self.in_flight_tasks or
-                                dep in self._missing_dep_flight or
-                                self.who_has[dep].issubset(self.in_flight_workers))
+                        assert (
+                            dep in self.in_flight_tasks
+                            or dep in self._missing_dep_flight
+                            or self.who_has[dep].issubset(self.in_flight_workers)
+                        )
 
             for key in self.tasks:
-                if self.task_state[key] == 'memory':
+                if self.task_state[key] == "memory":
                     assert isinstance(self.nbytes[key], int)
                     assert key not in self.waiting_for_data
                     assert key in self.data or key in self.actors
@@ -2462,6 +2678,7 @@ def validate_state(self):
             logger.exception(e)
             if LOG_PDB:
                 import pdb
+
                 pdb.set_trace()
             raise
 
@@ -2488,26 +2705,34 @@ def _get_client(self, timeout=3):
         """
         try:
             from .client import default_client
+
             client = default_client()
         except ValueError:  # no clients found, need to make a new one
             pass
         else:
-            if (client.scheduler and client.scheduler.address == self.scheduler.address
-                    or client._start_arg == self.scheduler.address):
+            if (
+                client.scheduler
+                and client.scheduler.address == self.scheduler.address
+                or client._start_arg == self.scheduler.address
+            ):
                 self._client = client
 
         if not self._client:
             from .client import Client
+
             asynchronous = self.loop is IOLoop.current()
-            self._client = Client(self.scheduler, loop=self.loop,
-                                  security=self.security,
-                                  set_as_default=True,
-                                  asynchronous=asynchronous,
-                                  direct_to_workers=True,
-                                  name='worker',
-                                  timeout=timeout)
+            self._client = Client(
+                self.scheduler,
+                loop=self.loop,
+                security=self.security,
+                set_as_default=True,
+                asynchronous=asynchronous,
+                direct_to_workers=True,
+                name="worker",
+                timeout=timeout,
+            )
             if not asynchronous:
-                assert self._client.status == 'running'
+                assert self._client.status == "running"
         return self._client
 
     def get_current_task(self):
@@ -2551,7 +2776,7 @@ def get_worker():
     worker_client
     """
     try:
-        return thread_state.execution_state['worker']
+        return thread_state.execution_state["worker"]
     except AttributeError:
         for ref in _global_workers[::-1]:
             worker = ref()
@@ -2608,11 +2833,13 @@ def get_client(address=None, timeout=3, resolve_address=True):
             return worker._get_client(timeout=timeout)
 
     from .client import _get_global_client
+
     client = _get_global_client()  # TODO: assumes the same scheduler
     if client and (not address or client.scheduler.address == address):
         return client
     elif address:
         from .client import Client
+
         return Client(address, timeout=timeout)
     else:
         raise ValueError("No global client found and no address provided")
@@ -2644,8 +2871,11 @@ def secede():
     worker = get_worker()
     tpe_secede()  # have this thread secede from the thread pool
     duration = time() - thread_state.start_time
-    worker.loop.add_callback(worker.maybe_transition_long_running,
-                             thread_state.key, compute_duration=duration)
+    worker.loop.add_callback(
+        worker.maybe_transition_long_running,
+        thread_state.key,
+        compute_duration=duration,
+    )
 
 
 class Reschedule(Exception):
@@ -2660,13 +2890,14 @@ class Reschedule(Exception):
     load across the cluster has significantly changed since first scheduling
     the task.
     """
+
     pass
 
 
 def parse_memory_limit(memory_limit, ncores, total_cores=_ncores):
     if memory_limit is None:
         return None
-    if memory_limit == 'auto':
+    if memory_limit == "auto":
         memory_limit = int(TOTAL_MEMORY * min(1, ncores / total_cores))
     with ignoring(ValueError, TypeError):
         x = float(memory_limit)
@@ -2680,8 +2911,15 @@ def parse_memory_limit(memory_limit, ncores, total_cores=_ncores):
 
 
 @gen.coroutine
-def get_data_from_worker(rpc, keys, worker, who=None, max_connections=None,
-                         serializers=None, deserializers=None):
+def get_data_from_worker(
+    rpc,
+    keys,
+    worker,
+    who=None,
+    max_connections=None,
+    serializers=None,
+    deserializers=None,
+):
     """ Get keys from worker
 
     The worker has a two step handshake to acknowledge when data has been fully
@@ -2700,18 +2938,22 @@ def get_data_from_worker(rpc, keys, worker, who=None, max_connections=None,
 
     comm = yield rpc.connect(worker)
     try:
-        response = yield send_recv(comm,
-                                   serializers=serializers,
-                                   deserializers=deserializers,
-                                   op='get_data', keys=keys, who=who,
-                                   max_connections=max_connections)
+        response = yield send_recv(
+            comm,
+            serializers=serializers,
+            deserializers=deserializers,
+            op="get_data",
+            keys=keys,
+            who=who,
+            max_connections=max_connections,
+        )
         try:
-            status = response['status']
+            status = response["status"]
         except KeyError:
             raise ValueError("Unexpected response", response)
         else:
-            if status == 'OK':
-                yield comm.write('OK')
+            if status == "OK":
+                yield comm.write("OK")
     finally:
         rpc.reuse(worker, comm)
 
@@ -2794,14 +3036,12 @@ def dumps_task(task):
     """
     if istask(task):
         if task[0] is apply and not any(map(_maybe_complex, task[2:])):
-            d = {'function': dumps_function(task[1]),
-                 'args': warn_dumps(task[2])}
+            d = {"function": dumps_function(task[1]), "args": warn_dumps(task[2])}
             if len(task) == 4:
-                d['kwargs'] = warn_dumps(task[3])
+                d["kwargs"] = warn_dumps(task[3])
             return d
         elif not any(map(_maybe_complex, task[1:])):
-            return {'function': dumps_function(task[0]),
-                    'args': warn_dumps(task[1:])}
+            return {"function": dumps_function(task[0]), "args": warn_dumps(task[1:])}
     return to_serialize(task)
 
 
@@ -2815,21 +3055,31 @@ def warn_dumps(obj, dumps=pickle.dumps, limit=1e6):
         _warn_dumps_warned[0] = True
         s = str(obj)
         if len(s) > 70:
-            s = s[:50] + ' ... ' + s[-15:]
-        warnings.warn("Large object of size %s detected in task graph: \n"
-                      "  %s\n"
-                      "Consider scattering large objects ahead of time\n"
-                      "with client.scatter to reduce scheduler burden and \n"
-                      "keep data on workers\n\n"
-                      "    future = client.submit(func, big_data)    # bad\n\n"
-                      "    big_future = client.scatter(big_data)     # good\n"
-                      "    future = client.submit(func, big_future)  # good"
-                      % (format_bytes(len(b)), s))
+            s = s[:50] + " ... " + s[-15:]
+        warnings.warn(
+            "Large object of size %s detected in task graph: \n"
+            "  %s\n"
+            "Consider scattering large objects ahead of time\n"
+            "with client.scatter to reduce scheduler burden and \n"
+            "keep data on workers\n\n"
+            "    future = client.submit(func, big_data)    # bad\n\n"
+            "    big_future = client.scatter(big_data)     # good\n"
+            "    future = client.submit(func, big_future)  # good"
+            % (format_bytes(len(b)), s)
+        )
     return b
 
 
-def apply_function(function, args, kwargs, execution_state, key,
-                   active_threads, active_threads_lock, time_delay):
+def apply_function(
+    function,
+    args,
+    kwargs,
+    execution_state,
+    key,
+    active_threads,
+    active_threads_lock,
+    time_delay,
+):
     """ Run a function, collect information
 
     Returns
@@ -2847,26 +3097,29 @@ def apply_function(function, args, kwargs, execution_state, key,
         result = function(*args, **kwargs)
     except Exception as e:
         msg = error_message(e)
-        msg['op'] = 'task-erred'
-        msg['actual-exception'] = e
+        msg["op"] = "task-erred"
+        msg["actual-exception"] = e
     else:
-        msg = {'op': 'task-finished',
-               'status': 'OK',
-               'result': result,
-               'nbytes': sizeof(result),
-               'type': type(result) if result is not None else None}
+        msg = {
+            "op": "task-finished",
+            "status": "OK",
+            "result": result,
+            "nbytes": sizeof(result),
+            "type": type(result) if result is not None else None,
+        }
     finally:
         end = time()
-    msg['start'] = start + time_delay
-    msg['stop'] = end + time_delay
-    msg['thread'] = ident
+    msg["start"] = start + time_delay
+    msg["stop"] = end + time_delay
+    msg["thread"] = ident
     with active_threads_lock:
         del active_threads[ident]
     return msg
 
 
-def apply_function_actor(function, args, kwargs, execution_state, key,
-                         active_threads, active_threads_lock):
+def apply_function_actor(
+    function, args, kwargs, execution_state, key, active_threads, active_threads_lock
+):
     """ Run a function, collect information
 
     Returns
@@ -2894,6 +3147,7 @@ def get_msg_safe_str(msg):
     allowing for some arguments to raise exceptions during conversion and
     ignoring them.
     """
+
     class Repr(object):
         def __init__(self, f, val):
             self._f = f
@@ -2924,7 +3178,7 @@ def convert_args_to_str(args, max_len=None):
         strs[i] = sarg
         length += len(sarg) + 2
         if max_len is not None and length > max_len:
-            return "({}".format(", ".join(strs[:i + 1]))[:max_len]
+            return "({}".format(", ".join(strs[: i + 1]))[:max_len]
     else:
         return "({})".format(", ".join(strs))
 
@@ -2944,7 +3198,7 @@ def convert_kwargs_to_str(kwargs, max_len=None):
         strs[i] = skwarg
         length += len(skwarg) + 2
         if max_len is not None and length > max_len:
-            return "{{{}".format(", ".join(strs[:i + 1]))[:max_len]
+            return "{{{}".format(", ".join(strs[: i + 1]))[:max_len]
     else:
         return "{{{}}}".format(", ".join(strs))
 
@@ -2959,17 +3213,19 @@ def run(server, comm, function, args=(), kwargs={}, is_coro=None, wait=True):
     if is_coro is None:
         is_coro = iscoroutinefunction(function)
     else:
-        warnings.warn("The is_coro= parameter is deprecated. "
-                      "We now automatically detect coroutines/async functions")
+        warnings.warn(
+            "The is_coro= parameter is deprecated. "
+            "We now automatically detect coroutines/async functions"
+        )
     assert wait or is_coro, "Combination not supported"
     if args:
         args = pickle.loads(args)
     if kwargs:
         kwargs = pickle.loads(kwargs)
-    if has_arg(function, 'dask_worker'):
-        kwargs['dask_worker'] = server
-    if has_arg(function, 'dask_scheduler'):
-        kwargs['dask_scheduler'] = server
+    if has_arg(function, "dask_worker"):
+        kwargs["dask_worker"] = server
+    if has_arg(function, "dask_scheduler"):
+        kwargs["dask_scheduler"] = server
     logger.info("Run out-of-band function %r", funcname(function))
     try:
         if not is_coro:
@@ -2982,18 +3238,15 @@ def run(server, comm, function, args=(), kwargs={}, is_coro=None, wait=True):
                 result = None
 
     except Exception as e:
-        logger.warning(" Run Failed\n"
-                       "Function: %s\n"
-                       "args:     %s\n"
-                       "kwargs:   %s\n",
-                       str(funcname(function))[:1000],
-                       convert_args_to_str(args, max_len=1000),
-                       convert_kwargs_to_str(kwargs, max_len=1000), exc_info=True)
+        logger.warning(
+            " Run Failed\n" "Function: %s\n" "args:     %s\n" "kwargs:   %s\n",
+            str(funcname(function))[:1000],
+            convert_args_to_str(args, max_len=1000),
+            convert_kwargs_to_str(kwargs, max_len=1000),
+            exc_info=True,
+        )
 
         response = error_message(e)
     else:
-        response = {
-            'status': 'OK',
-            'result': to_serialize(result),
-        }
+        response = {"status": "OK", "result": to_serialize(result)}
     raise Return(response)
diff --git a/distributed/worker_client.py b/distributed/worker_client.py
index 50cf6be25a5..ff6294430b5 100644
--- a/distributed/worker_client.py
+++ b/distributed/worker_client.py
@@ -44,7 +44,7 @@ def worker_client(timeout=3, separate_thread=True):
     client = get_client(timeout=timeout)
     if separate_thread:
         secede()  # have this thread secede from the thread pool
-        worker.loop.add_callback(worker.transition, thread_state.key, 'long-running')
+        worker.loop.add_callback(worker.transition, thread_state.key, "long-running")
 
     yield client
 

From cb6ed57573ef171988b372843489e12c8e5c5b6b Mon Sep 17 00:00:00 2001
From: Peter Andreas Entschev <peter@entschev.com>
Date: Tue, 16 Apr 2019 19:32:51 +0200
Subject: [PATCH 0237/1550] Integrate stacktrace for low-level profiling
 (#2575)

---
 continuous_integration/travis/install.sh   |   5 +
 distributed/bokeh/tests/test_components.py |   4 +-
 distributed/distributed.yaml               |   2 +
 distributed/profile.py                     | 113 ++++++++++++++++++++-
 distributed/tests/test_profile.py          |  46 ++++++++-
 distributed/worker.py                      |  16 ++-
 6 files changed, 178 insertions(+), 8 deletions(-)

diff --git a/continuous_integration/travis/install.sh b/continuous_integration/travis/install.sh
index 8e9d1ff7f12..bba69dd3ac8 100644
--- a/continuous_integration/travis/install.sh
+++ b/continuous_integration/travis/install.sh
@@ -54,6 +54,11 @@ conda install -q \
     tornado=$TORNADO \
     $PACKAGES
 
+# For low-level profiler, install libunwind and stacktrace from conda-forge
+# For stacktrace we use --no-deps to avoid upgrade of python
+conda install -c defaults -c conda-forge libunwind
+conda install --no-deps -c defaults -c numba -c conda-forge stacktrace
+
 pip install -q pytest-repeat pytest-faulthandler
 
 pip install -q git+https://github.com/dask/dask.git --upgrade --no-deps
diff --git a/distributed/bokeh/tests/test_components.py b/distributed/bokeh/tests/test_components.py
index 4f4df92f6cd..028f209b41a 100644
--- a/distributed/bokeh/tests/test_components.py
+++ b/distributed/bokeh/tests/test_components.py
@@ -30,10 +30,10 @@ def test_basic(Component):
 @gen_cluster(client=True, check_new_threads=False)
 def test_profile_plot(c, s, a, b):
     p = ProfilePlot()
-    assert len(p.source.data["left"]) <= 1
+    assert not p.source.data["left"]
     yield c.map(slowinc, range(10), delay=0.05)
     p.update(a.profile_recent)
-    assert len(p.source.data["left"]) > 1
+    assert len(p.source.data["left"]) >= 1
 
 
 @gen_cluster(client=True, check_new_threads=False)
diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index 156c90a127e..d625a103fe8 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -35,6 +35,8 @@ distributed:
     profile:
       interval: 10ms        # Time between statistical profiling queries
       cycle: 1000ms         # Time between starting new profile
+      low-level: False      # Whether or not to include low-level functions
+                            # Requires https://github.com/numba/stacktrace
 
     # Fractions of worker memory at which we take action to avoid memory blowup
     # Set any of the lower three values to False to turn off the behavior entirely
diff --git a/distributed/profile.py b/distributed/profile.py
index 385c7449e75..e240a872fb4 100644
--- a/distributed/profile.py
+++ b/distributed/profile.py
@@ -178,6 +178,7 @@ def plot_data(state, profile_interval=0.010):
 
     See Also
     --------
+    plot_figure
     distributed.bokeh.components.ProfilePlot
     """
     starts = []
@@ -213,9 +214,14 @@ def traverse(state, start, stop, height):
         ident = state["identifier"]
 
         try:
-            colors.append(color_of(desc["filename"]))
+            fn = desc["filename"]
         except IndexError:
             colors.append("gray")
+        else:
+            if fn == "<low-level>":
+                colors.append("lightgray")
+            else:
+                colors.append(color_of(fn))
 
         delta = (stop - start) / state["count"]
 
@@ -274,6 +280,29 @@ def watch(
     omit=None,
     stop=lambda: False,
 ):
+    """ Gather profile information on a particular thread
+
+    This starts a new thread to watch a particular thread and returns a deque
+    that holds periodic profile information.
+
+    Parameters
+    ----------
+    thread_id: int
+    interval: str
+        Time per sample
+    cycle: str
+        Time per refreshing to a new profile state
+    maxlen: int
+        Passed onto deque, maximum number of periods
+    omit: str
+        Don't include entries that start with this filename
+    stop: callable
+        Function to call to see if we should stop
+
+    Returns
+    -------
+    deque
+    """
     if thread_id is None:
         thread_id = get_thread_identity()
 
@@ -298,6 +327,17 @@ def watch(
 
 
 def get_profile(history, recent=None, start=None, stop=None, key=None):
+    """ Collect profile information from a sequence of profile states
+
+    Parameters
+    ----------
+    history: Sequence[Tuple[time, Dict]]
+        A list or deque of profile states
+    recent: dict
+        The most recent accumulating state
+    start: time
+    stop: time
+    """
     now = time()
     if start is None:
         istart = 0
@@ -329,6 +369,15 @@ def get_profile(history, recent=None, start=None, stop=None, key=None):
 
 
 def plot_figure(data, **kwargs):
+    """ Plot profile data using Bokeh
+
+    This takes the output from the function ``plot_data`` and produces a Bokeh
+    figure
+
+    See Also
+    --------
+    plot_data
+    """
     from bokeh.plotting import ColumnDataSource, figure
     from bokeh.models import HoverTool
 
@@ -388,3 +437,65 @@ def plot_figure(data, **kwargs):
     fig.grid.visible = False
 
     return fig, source
+
+
+def _remove_py_stack(frames):
+    for entry in frames:
+        if entry.is_python:
+            break
+        yield entry
+
+
+def llprocess(frames, child, state):
+    """ Add counts from low level profile information onto existing state
+
+    This uses the ``stacktrace`` module to collect low level stack trace
+    information and place it onto the given sttate.
+
+    It is configured with the ``distributed.worker.profile.low-level`` config
+    entry.
+
+    See Also
+    --------
+    process
+    ll_get_stack
+    """
+    if not frames:
+        return
+    frame = frames.pop()
+    if frames:
+        state = llprocess(frames, frame, state)
+
+    addr = hex(frame.addr - frame.offset)
+    ident = ";".join(map(str, (frame.name, "<low-level>", addr)))
+    try:
+        d = state["children"][ident]
+    except KeyError:
+        d = {
+            "count": 0,
+            "description": {
+                "filename": "<low-level>",
+                "name": frame.name,
+                "line_number": 0,
+                "line": str(frame),
+            },
+            "children": {},
+            "identifier": ident,
+        }
+        state["children"][ident] = d
+
+    state["count"] += 1
+
+    if child is not None:
+        return d
+    else:
+        d["count"] += 1
+
+
+def ll_get_stack(tid):
+    """ Collect low level stack information from thread id """
+    from stacktrace import get_thread_stack
+
+    frames = get_thread_stack(tid, show_python=False)
+    llframes = list(_remove_py_stack(frames))[::-1]
+    return llframes
diff --git a/distributed/tests/test_profile.py b/distributed/tests/test_profile.py
index 57a7ca657e4..ee49f130027 100644
--- a/distributed/tests/test_profile.py
+++ b/distributed/tests/test_profile.py
@@ -1,11 +1,22 @@
+import pytest
 import sys
 import time
 from toolz import first
 import threading
 
-from distributed.compatibility import get_thread_identity
+from distributed.compatibility import get_thread_identity, WINDOWS
 from distributed import metrics
-from distributed.profile import process, merge, create, call_stack, identifier, watch
+from distributed.profile import (
+    process,
+    merge,
+    create,
+    call_stack,
+    identifier,
+    watch,
+    llprocess,
+    ll_get_stack,
+    plot_data,
+)
 
 
 def test_basic():
@@ -44,6 +55,37 @@ def test_f():
     assert g["count"] < h["count"]
     assert 95 < g["count"] + h["count"] <= 100
 
+    pd = plot_data(state)
+    assert len(set(map(len, pd.values()))) == 1  # all same length
+    assert len(set(pd["color"])) > 1  # different colors
+
+
+@pytest.mark.skipif(
+    WINDOWS, reason="no low-level profiler support for Windows available"
+)
+def test_basic_low_level():
+    pytest.importorskip("stacktrace")
+
+    state = create()
+
+    for i in range(100):
+        time.sleep(0.02)
+        frame = sys._current_frames()[threading.get_ident()]
+        llframes = {threading.get_ident(): ll_get_stack(threading.get_ident())}
+        for f in llframes.values():
+            if f is not None:
+                llprocess(f, None, state)
+
+    assert state["count"] == 100
+    children = state.get("children")
+    assert children
+    expected = "<low-level>"
+    for k, v in zip(children.keys(), children.values()):
+        desc = v.get("description")
+        assert desc
+        filename = desc.get("filename")
+        assert expected in k and filename == expected
+
 
 def test_merge():
     a1 = {
diff --git a/distributed/worker.py b/distributed/worker.py
index b9ed6c5a59d..9f940f02f93 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -294,6 +294,7 @@ def __init__(
         extensions=None,
         metrics=None,
         data=None,
+        low_level_profiler=dask.config.get("distributed.worker.profile.low-level"),
         **kwargs
     ):
         self.tasks = dict()
@@ -511,6 +512,8 @@ def __init__(
         self.service_specs = services or {}
         self.metrics = dict(metrics) if metrics else {}
 
+        self.low_level_profiler = low_level_profiler
+
         handlers = {
             "gather": self.gather,
             "run": self.run,
@@ -2464,15 +2467,22 @@ def trigger_profile(self):
             active_threads = self.active_threads.copy()
         frames = sys._current_frames()
         frames = {ident: frames[ident] for ident in active_threads}
+        llframes = {}
+        if self.low_level_profiler:
+            llframes = {ident: profile.ll_get_stack(ident) for ident in active_threads}
         for ident, frame in frames.items():
             if frame is not None:
                 key = key_split(active_threads[ident])
-                profile.process(
-                    frame, None, self.profile_recent, stop="distributed/worker.py"
+                llframe = llframes.get(ident)
+
+                state = profile.process(
+                    frame, True, self.profile_recent, stop="distributed/worker.py"
                 )
+                profile.llprocess(llframe, None, state)
                 profile.process(
-                    frame, None, self.profile_keys[key], stop="distributed/worker.py"
+                    frame, True, self.profile_keys[key], stop="distributed/worker.py"
                 )
+
         stop = time()
         if self.digests is not None:
             self.digests["profile-duration"].add(stop - start)

From 33df62d5e6369e50f1c9e86381ea9f1f346ca931 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 18 Apr 2019 16:10:40 -0500
Subject: [PATCH 0238/1550] Allow Python 2 testing failures in Travis CI
 (#2615)

* Allow Python 2 build to fail in travis.ci

* Use the --check flag for black
---
 .travis.yml | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 96331468b55..23daef096ca 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -21,10 +21,9 @@ matrix:
   #- os: osx
     #env: PYTHON=3.6 RUNSLOW=false
   # Together with fast_finish, allow build to be marked successful before the OS X job finishes
-  #allow_failures:
-  #- os: osx
+  allow_failures:
     ## This needs to be the exact same line as above
-    #env: PYTHON=3.6 RUNSLOW=false
+    env: PYTHON=2.7 TESTS=true PACKAGES="python-blosc futures faulthandler lz4"
 
 install:
   - if [[ $TESTS == true ]]; then source continuous_integration/travis/install.sh ; fi
@@ -32,7 +31,7 @@ install:
 script:
   - if [[ $TESTS == true ]]; then source continuous_integration/travis/run_tests.sh ; fi
   - if [[ $LINT == true ]]; then pip install flake8 ; flake8 distributed ; fi
-  - if [[ $LINT == true ]]; then pip install black; black distributed ; fi
+  - if [[ $LINT == true ]]; then pip install black; black distributed --check; fi
 
 after_success:
   - if [[ $COVERAGE == true ]]; then coverage report; pip install -q coveralls ; coveralls ; fi

From c355744f988a23b7dbc388d93c0fd09de67e75cb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lo=C3=AFc=20Est=C3=A8ve?= <loic.esteve@ymail.com>
Date: Fri, 19 Apr 2019 15:39:01 +0200
Subject: [PATCH 0239/1550] Fix parameter name in LocalCluster docstring
 (#2626)

---
 distributed/deploy/local.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index 22a796bbb78..92de1b1c799 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -62,7 +62,7 @@ class LocalCluster(Cluster):
     asynchronous: bool (False by default)
         Set to True if using this cluster within async/await functions or within
         Tornado gen.coroutines.  This should remain False for normal use.
-    kwargs: dict
+    worker_kwargs: dict
         Extra worker arguments, will be passed to the Worker constructor.
     blocked_handlers: List[str]
         A list of strings specifying a blacklist of handlers to disallow on the Scheduler,

From 3b84c3e350f4b72d428c2bc44e5ce95652c63e5e Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sat, 20 Apr 2019 10:43:24 -0700
Subject: [PATCH 0240/1550] Add number of trials to diskutils test (#2630)

---
 distributed/tests/test_diskutils.py | 24 +++++++++++++++---------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/distributed/tests/test_diskutils.py b/distributed/tests/test_diskutils.py
index 8bf4000178e..d5abf5c1dee 100644
--- a/distributed/tests/test_diskutils.py
+++ b/distributed/tests/test_diskutils.py
@@ -19,14 +19,20 @@
 from distributed.utils_test import captured_logger, slow
 
 
-def assert_directory_contents(dir_path, expected):
+def assert_directory_contents(dir_path, expected, trials=2):
     expected = [os.path.join(dir_path, p) for p in expected]
-    actual = [
-        os.path.join(dir_path, p)
-        for p in os.listdir(dir_path)
-        if p not in ("global.lock", "purge.lock")
-    ]
-    assert sorted(actual) == sorted(expected)
+    for i in range(trials):
+        actual = [
+            os.path.join(dir_path, p)
+            for p in os.listdir(dir_path)
+            if p not in ("global.lock", "purge.lock")
+        ]
+        if sorted(actual) == sorted(expected):
+            break
+        else:
+            sleep(0.5)
+    else:
+        assert sorted(actual) == sorted(expected)
 
 
 def test_workdir_simple(tmpdir):
@@ -82,10 +88,10 @@ def test_two_workspaces_in_same_directory(tmpdir):
     del ws
     del b
     gc.collect()
-    assert_contents(["aa", "aa.dirlock"])
+    assert_contents(["aa", "aa.dirlock"], trials=5)
     del a
     gc.collect()
-    assert_contents([])
+    assert_contents([], trials=5)
 
 
 def test_workspace_process_crash(tmpdir):

From 291a140787168ac19625cd142f5189f21e179d00 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sat, 20 Apr 2019 10:44:08 -0700
Subject: [PATCH 0241/1550] Remove Python 2.7 from testing matrix (#2631)

---
 .travis.yml | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 23daef096ca..bcc09351eff 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -6,7 +6,6 @@ dist: trusty
 
 env:
   matrix:
-    - PYTHON=2.7 TESTS=true PACKAGES="python-blosc futures faulthandler lz4"
     - PYTHON=3.5.4 TESTS=true COVERAGE=true PACKAGES="python-blosc lz4" CRICK=true
     - PYTHON=3.6 TESTS=true PACKAGES="scikit-learn lz4" TORNADO=5
     - PYTHON=3.7 TESTS=true PACKAGES="scikit-learn python-snappy python-blosc" TORNADO=6
@@ -15,15 +14,8 @@ matrix:
   fast_finish: true
   include:
   - os: linux
-    # Using Travis-CI's python makes job faster by not downloading miniconda
     python: 3.6
     env: LINT=true
-  #- os: osx
-    #env: PYTHON=3.6 RUNSLOW=false
-  # Together with fast_finish, allow build to be marked successful before the OS X job finishes
-  allow_failures:
-    ## This needs to be the exact same line as above
-    env: PYTHON=2.7 TESTS=true PACKAGES="python-blosc futures faulthandler lz4"
 
 install:
   - if [[ $TESTS == true ]]; then source continuous_integration/travis/install.sh ; fi

From f459af1637c1f1cb3bf75e6d710b99bcf3886190 Mon Sep 17 00:00:00 2001
From: Peter Andreas Entschev <peter@entschev.com>
Date: Sat, 20 Apr 2019 19:49:34 +0200
Subject: [PATCH 0242/1550] Add worker_class argument to LocalCluster (#2625)

---
 distributed/deploy/local.py            | 12 +++++++----
 distributed/deploy/tests/test_local.py | 30 ++++++++++++++++++++++++++
 2 files changed, 38 insertions(+), 4 deletions(-)

diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index 92de1b1c799..4e63646ba27 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -74,6 +74,8 @@ class LocalCluster(Cluster):
         Protocol to use like ``tcp://``, ``tls://``, ``inproc://``
         This defaults to sensible choice given other keyword arguments like
         ``processes`` and ``security``
+    worker_class: Worker
+        Worker class used to instantiate workers from.
 
     Examples
     --------
@@ -115,6 +117,7 @@ def __init__(
         security=None,
         protocol=None,
         blocked_handlers=None,
+        worker_class=None,
         **worker_kwargs
     ):
         if start is not None:
@@ -203,6 +206,10 @@ def __init__(
         if security:
             self.worker_kwargs["security"] = security
 
+        if not worker_class:
+            worker_class = Worker if not processes else Nanny
+        self.worker_class = worker_class
+
         self.start(ip=ip, n_workers=n_workers)
 
         clusters_to_close.add(self)
@@ -279,12 +286,9 @@ def _start_worker(self, death_timeout=60, **kwargs):
             return
 
         if self.processes:
-            W = Nanny
             kwargs["quiet"] = True
-        else:
-            W = Worker
 
-        w = yield W(
+        w = yield self.worker_class(
             self.scheduler.address,
             loop=self.loop,
             death_timeout=death_timeout,
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index ee2d48c2df3..ab378ba0e8f 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -742,5 +742,35 @@ def test_protocol_ip(loop):
         assert cluster.scheduler.address.startswith("tcp://127.0.0.2")
 
 
+class MyWorker(Worker):
+    pass
+
+
+def test_worker_class_worker(loop):
+    with LocalCluster(
+        n_workers=2,
+        loop=loop,
+        worker_class=MyWorker,
+        processes=False,
+        scheduler_port=0,
+        dashboard_address=None,
+    ) as cluster:
+        assert all(isinstance(w, MyWorker) for w in cluster.workers)
+
+
+def test_worker_class_nanny(loop):
+    class MyNanny(Nanny):
+        pass
+
+    with LocalCluster(
+        n_workers=2,
+        loop=loop,
+        worker_class=MyNanny,
+        scheduler_port=0,
+        dashboard_address=None,
+    ) as cluster:
+        assert all(isinstance(w, MyNanny) for w in cluster.workers)
+
+
 if sys.version_info >= (3, 5):
     from distributed.deploy.tests.py3_test_deploy import *  # noqa F401

From 7de97bdbdc97599ae7c4bd8d9f3851463d7eedc1 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sun, 21 Apr 2019 08:25:40 -0700
Subject: [PATCH 0243/1550] Add interface= keyword to LocalCluster (#2629)

This is useful when you want to use a particular network interface, like
infiniband.

Fixes https://github.com/dask/distributed/issues/2618
---
 distributed/deploy/local.py      | 10 +++++++++-
 distributed/tests/test_client.py |  5 +++--
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index 4e63646ba27..73ddde8bdc1 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -15,6 +15,7 @@
 from ..compatibility import get_thread_identity
 from ..core import CommClosedError
 from ..utils import (
+    get_ip_interface,
     sync,
     ignoring,
     All,
@@ -74,6 +75,8 @@ class LocalCluster(Cluster):
         Protocol to use like ``tcp://``, ``tls://``, ``inproc://``
         This defaults to sensible choice given other keyword arguments like
         ``processes`` and ``security``
+    interface: str (optional)
+        Network interface to use.  Defaults to lo/localhost
     worker_class: Worker
         Worker class used to instantiate workers from.
 
@@ -117,6 +120,7 @@ def __init__(
         security=None,
         protocol=None,
         blocked_handlers=None,
+        interface=None,
         worker_class=None,
         **worker_kwargs
     ):
@@ -155,6 +159,7 @@ def __init__(
         self.silence_logs = silence_logs
         self._asynchronous = asynchronous
         self.security = security
+        self.interface = interface
         services = services or {}
         worker_services = worker_services or {}
         if silence_logs:
@@ -262,7 +267,10 @@ def _start(self, ip=None, n_workers=0):
             address = self.protocol
         else:
             if ip is None:
-                ip = "127.0.0.1"
+                if self.interface:
+                    ip = get_ip_interface(self.interface)
+                else:
+                    ip = "127.0.0.1"
 
             if "://" in ip:
                 address = ip
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 634834bf671..a9d4f21fd81 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -5061,8 +5061,9 @@ def test_call_stack_future(c, s, a, b):
 
 @gen_cluster([("127.0.0.1", 4)] * 2, client=True)
 def test_call_stack_all(c, s, a, b):
-    future = c.submit(slowinc, 1, delay=0.5)
-    yield gen.sleep(0.1)
+    future = c.submit(slowinc, 1, delay=0.8)
+    while not a.executing and not b.executing:
+        yield gen.sleep(0.01)
     result = yield c.call_stack()
     w = a if a.executing else b
     assert list(result) == [w.address]

From a3d2016a4fab9ef14ab5be0b0b722b59365b59ce Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sun, 21 Apr 2019 08:25:57 -0700
Subject: [PATCH 0244/1550] Increase GC thresholds (#2624)

Fixes https://github.com/dask/distributed/issues/1653
---
 distributed/cli/dask_scheduler.py | 3 +++
 distributed/cli/dask_worker.py    | 4 ++++
 2 files changed, 7 insertions(+)

diff --git a/distributed/cli/dask_scheduler.py b/distributed/cli/dask_scheduler.py
index 0e8415ac132..3b0aa5b4c70 100755
--- a/distributed/cli/dask_scheduler.py
+++ b/distributed/cli/dask_scheduler.py
@@ -3,6 +3,7 @@
 import atexit
 import dask
 import logging
+import gc
 import os
 import shutil
 import sys
@@ -137,6 +138,8 @@ def main(
     tls_key,
     dashboard_address,
 ):
+    g0, g1, g2 = gc.get_threshold()  # https://github.com/dask/distributed/issues/1653
+    gc.set_threshold(g0 * 3, g1 * 3, g2 * 3)
 
     enable_proctitle_on_current()
     enable_proctitle_on_children()
diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
index 0eb5a7973fb..73cb9970924 100755
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -2,6 +2,7 @@
 
 import atexit
 import logging
+import gc
 import os
 from sys import exit
 import warnings
@@ -207,6 +208,9 @@ def main(
     tls_key,
     dashboard_address,
 ):
+    g0, g1, g2 = gc.get_threshold()  # https://github.com/dask/distributed/issues/1653
+    gc.set_threshold(g0 * 3, g1 * 3, g2 * 3)
+
     enable_proctitle_on_current()
     enable_proctitle_on_children()
 

From 0c8918b5e6ff63857f89c76468a5ec2d2aa005ae Mon Sep 17 00:00:00 2001
From: Michael Delgado <delgadom@users.noreply.github.com>
Date: Wed, 24 Apr 2019 14:26:19 -0700
Subject: [PATCH 0245/1550] Adaptive: recommend close workers when any are idle
 (#2330)

* adaptive: recommend close workers if idle

* adaptive: check for idle workers before recommending scale up

* adaptive: check for waiting tasks in should_scale_up

* revert to changing needs_cpu only

* performance bump in adaptive.needs_cpu by looping through workers

* switch to checking number of cores, not workers

* apply black

* remove xfail

* flake
---
 distributed/deploy/adaptive.py            | 24 ++++++++++++++++++-----
 distributed/deploy/tests/test_adaptive.py |  2 --
 2 files changed, 19 insertions(+), 7 deletions(-)

diff --git a/distributed/deploy/adaptive.py b/distributed/deploy/adaptive.py
index 890e30c027f..8c260609638 100644
--- a/distributed/deploy/adaptive.py
+++ b/distributed/deploy/adaptive.py
@@ -135,10 +135,11 @@ def needs_cpu(self):
         Notes
         -----
         Returns ``True`` if the occupancy per core is some factor larger
-        than ``startup_cost``.
+        than ``startup_cost`` and the number of tasks exceeds the number of
+        cores
         """
         total_occupancy = self.scheduler.total_occupancy
-        total_cores = sum([ws.ncores for ws in self.scheduler.workers.values()])
+        total_cores = self.scheduler.total_ncores
 
         if total_occupancy / (total_cores + 1e-9) > self.startup_cost * 2:
             logger.info(
@@ -146,9 +147,22 @@ def needs_cpu(self):
                 total_occupancy,
                 total_cores,
             )
-            return True
-        else:
-            return False
+
+            tasks_processing = 0
+
+            for w in self.scheduler.workers.values():
+                tasks_processing += len(w.processing)
+
+                if tasks_processing > total_cores:
+                    logger.info(
+                        "pending tasks exceed number of cores " "[%d tasks / %d cores]",
+                        tasks_processing,
+                        total_cores,
+                    )
+
+                    return True
+
+        return False
 
     def needs_memory(self):
         """
diff --git a/distributed/deploy/tests/test_adaptive.py b/distributed/deploy/tests/test_adaptive.py
index 1d8a48bf7fc..50c4f0a45a3 100644
--- a/distributed/deploy/tests/test_adaptive.py
+++ b/distributed/deploy/tests/test_adaptive.py
@@ -2,7 +2,6 @@
 
 from time import sleep
 
-import pytest
 from toolz import frequencies, pluck
 from tornado import gen
 from tornado.ioloop import IOLoop
@@ -331,7 +330,6 @@ def test_adapt_down():
         yield cluster.close()
 
 
-@pytest.mark.xfail(reason="we currently only judge occupancy, not ntasks")
 @gen_test(timeout=30)
 def test_no_more_workers_than_tasks():
     loop = IOLoop.current()

From 7461488d6ecf9226870b159314565ea2ca477d28 Mon Sep 17 00:00:00 2001
From: Brett Randall <javabrett@gmail.com>
Date: Mon, 29 Apr 2019 23:20:15 +1000
Subject: [PATCH 0246/1550] Updated logging module doc links from
 docs.python.org/2 to docs.python.org/3. (#2635)

Signed-off-by: Brett Randall <javabrett@gmail.com>
---
 distributed/config.py         | 4 ++--
 docs/source/configuration.rst | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/distributed/config.py b/distributed/config.py
index 4b7b589d58f..5c71cf570c8 100644
--- a/distributed/config.py
+++ b/distributed/config.py
@@ -103,7 +103,7 @@ def _initialize_logging_old_style(config):
 def _initialize_logging_new_style(config):
     """
     Initialize logging using logging's "Configuration dictionary schema".
-    (ref.: https://docs.python.org/2/library/logging.config.html#logging-config-dictschema)
+    (ref.: https://docs.python.org/3/library/logging.config.html#configuration-dictionary-schema)
     """
     logging.config.dictConfig(config.get("logging"))
 
@@ -111,7 +111,7 @@ def _initialize_logging_new_style(config):
 def _initialize_logging_file_config(config):
     """
     Initialize logging using logging's "Configuration file format".
-    (ref.: https://docs.python.org/2/library/logging.config.html#configuration-file-format)
+    (ref.: https://docs.python.org/3/howto/logging.html#configuring-logging)
     """
     logging.config.fileConfig(
         config.get("logging-file-config"), disable_existing_loggers=False
diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst
index 86070de96f9..8967255f526 100644
--- a/docs/source/configuration.rst
+++ b/docs/source/configuration.rst
@@ -156,7 +156,7 @@ for each logger.  It also sets default values for several loggers such
 as ``distributed`` unless explicitly configured.
 
 A more extended format is possible following the :mod:`logging` module's
-`Configuration dictionary schema <https://docs.python.org/2/library/logging.config.html#logging-config-dictschema>`_.
+`Configuration dictionary schema <https://docs.python.org/3/library/logging.config.html#configuration-dictionary-schema>`_.
 To enable this extended format, there must be a ``version`` sub-key as
 mandated by the schema.  The extended format does not set any default values.
 
@@ -173,7 +173,7 @@ mandated by the schema.  The extended format does not set any default values.
 As an alternative to the two logging settings formats discussed above,
 you can specify a logging config file.
 Its format adheres to the :mod:`logging` module's
-`Configuration file format <https://docs.python.org/2/library/logging.config.html#configuration-file-format>`_.
+`Configuration file format <https://docs.python.org/3/howto/logging.html#configuring-logging>`_.
 
 .. note::
    The configuration options `logging-file-config` and `logging` are mutually exclusive.
\ No newline at end of file

From f62d6310827c387cb004fec8ee7202c62cf46a69 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 29 Apr 2019 09:39:55 -0500
Subject: [PATCH 0247/1550] bump version to 1.27.1

---
 docs/source/changelog.rst | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index affad66a759..34fe078fe18 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,10 +1,26 @@
 Changelog
 =========
 
+1.27.1 - 2019-04-29
+-------------------
+
+-  Adaptive: recommend close workers when any are idle (:pr:`2330`) `Michael Delgado`_
+-  Increase GC thresholds (:pr:`2624`) `Matthew Rocklin`_
+-  Add interface= keyword to LocalCluster (:pr:`2629`) `Matthew Rocklin`_
+-  Add worker_class argument to LocalCluster (:pr:`2625`) `Matthew Rocklin`_
+-  Remove Python 2.7 from testing matrix (:pr:`2631`) `Matthew Rocklin`_
+-  Add number of trials to diskutils test (:pr:`2630`) `Matthew Rocklin`_
+-  Fix parameter name in LocalCluster docstring (:pr:`2626`) `Loïc Estève`_
+-  Integrate stacktrace for low-level profiling (:pr:`2575`) `Peter Andreas Entschev`_
+-  Apply Black to standardize code styling (:pr:`2614`) `Matthew Rocklin`_
+-  added missing whitespace to start_worker cmd (:pr:`2613`) `condoratberlin`_
+-  Updated logging module doc links from docs.python.org/2 to docs.python.org/3. (:pr:`2635`) `Brett Randall`_
+
+
 1.27.0 - 2019-04-12
 -------------------
 
-   Add basic health endpoints to scheduler and worker bokeh. (#2607) `amerkel2`_
+   Add basic health endpoints to scheduler and worker bokeh. (:pr:`2607) `amerkel2`_
 -  Improved description accuracy of --memory-limit option. (:pr:`2601`) `Brett Randall`_
 -  Check self.dependencies when looking at dependent tasks in memory (:pr:`2606`) `deepthirajagopalan7`_
 -  Add RabbitMQ SchedulerPlugin example (:pr:`2604`) `Matt Nicolls`_
@@ -986,3 +1002,6 @@ significantly without many new features.
 .. _`Brian Chu`: https://github.com/bchu
 .. _`James Bourbeau`: https://github.com/jrbourbeau
 .. _`amerkel2`: https://github.com/amerkel2
+.. _`Michael Delgado`: https://github.com/delgadom
+.. _`Peter Andreas Entschev`: https://github.com/pentschev
+.. _`condoratberlin`: https://github.com/condoratberlin

From e0cf7e7300c9dcf10b7440abb1e3efc6cea3a91a Mon Sep 17 00:00:00 2001
From: Peter Andreas Entschev <peter@entschev.com>
Date: Mon, 29 Apr 2019 21:33:36 +0200
Subject: [PATCH 0248/1550] Fix deserialization of bytes chunks larger than
 64MB (#2637)

---
 distributed/protocol/serialize.py            | 2 +-
 distributed/protocol/tests/test_serialize.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index 3b0a45c8a6f..4ff0fb47a65 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -473,7 +473,7 @@ def _serialize_bytes(obj):
 
 @dask_deserialize.register((bytes, bytearray))
 def _deserialize_bytes(header, frames):
-    return frames[0]
+    return b"".join(frames)
 
 
 #########################
diff --git a/distributed/protocol/tests/test_serialize.py b/distributed/protocol/tests/test_serialize.py
index da43021d550..4f72ec9a538 100644
--- a/distributed/protocol/tests/test_serialize.py
+++ b/distributed/protocol/tests/test_serialize.py
@@ -192,7 +192,7 @@ def test_empty_loads_deep():
 
 
 def test_serialize_bytes():
-    for x in [1, "abc", np.arange(5)]:
+    for x in [1, "abc", np.arange(5), b"ab" * int(40e6)]:
         b = serialize_bytes(x)
         assert isinstance(b, bytes)
         y = deserialize_bytes(b)

From 8af282651d701569acda55e742f2db741700de9a Mon Sep 17 00:00:00 2001
From: Jim Crist <jcrist@users.noreply.github.com>
Date: Tue, 30 Apr 2019 09:21:41 -0500
Subject: [PATCH 0249/1550] Use proper address in worker -> nanny comms (#2640)

When a worker is shutdown explicitly it notifies the nanny that it
should also shutdown. Previously the address it used for this assumed
tcp in all cases, this changes that to use the same protocol as the
worker (which is currently always the same as the nanny's). This allows
`retire_workers` to properly work over TLS.
---
 distributed/tests/test_tls_functional.py | 14 ++++++++++++++
 distributed/worker.py                    |  7 ++++++-
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/distributed/tests/test_tls_functional.py b/distributed/tests/test_tls_functional.py
index 6b71941257c..74a9cf3cbd4 100644
--- a/distributed/tests/test_tls_functional.py
+++ b/distributed/tests/test_tls_functional.py
@@ -10,6 +10,8 @@
 
 from distributed import Nanny, worker_client, Queue
 from distributed.client import wait
+from distributed.metrics import time
+from distributed.nanny import Nanny
 from distributed.utils_test import gen_tls_cluster, inc, double, slowinc, slowadd
 
 
@@ -157,3 +159,15 @@ def mysum():
     future = c.submit(mysum)
     result = yield future
     assert result == 30 * 29
+
+
+@gen_tls_cluster(client=True, Worker=Nanny)
+def test_retire_workers(c, s, a, b):
+    assert set(s.workers) == {a.worker_address, b.worker_address}
+    yield c.retire_workers(workers=[a.worker_address], close_workers=True)
+    assert set(s.workers) == {b.worker_address}
+
+    start = time()
+    while a.status != "closed":
+        yield gen.sleep(0.01)
+        assert time() < start + 5
diff --git a/distributed/worker.py b/distributed/worker.py
index 9f940f02f93..33010836fdb 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -996,7 +996,12 @@ def _close(self, report=True, timeout=10, nanny=True, executor_wait=True):
             self.status = "closed"
 
             if nanny and "nanny" in self.service_ports:
-                with self.rpc((self.ip, self.service_ports["nanny"])) as r:
+                nanny_address = "%s%s:%d" % (
+                    self.listener.prefix,
+                    self.ip,
+                    self.service_ports["nanny"],
+                )
+                with self.rpc(nanny_address) as r:
                     yield r.terminate()
 
             if self.batched_stream and not self.batched_stream.comm.closed():

From 38afa51ca58aa2ea721caa95062fdb9024b0450a Mon Sep 17 00:00:00 2001
From: Peter Andreas Entschev <peter@entschev.com>
Date: Tue, 30 Apr 2019 16:46:15 +0200
Subject: [PATCH 0250/1550] Limit test_spill_by_default memory, reenable it
 (#2633)

---
 distributed/tests/test_worker.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 05b61a997f4..388414a4448 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -25,7 +25,7 @@
 from distributed.client import wait
 from distributed.scheduler import Scheduler
 from distributed.metrics import time
-from distributed.worker import Worker, error_message, logger, TOTAL_MEMORY
+from distributed.worker import Worker, error_message, logger
 from distributed.utils import tmpfile, format_bytes
 from distributed.utils_test import (
     inc,
@@ -446,11 +446,15 @@ def test_Executor(c, s):
         yield w._close()
 
 
-@pytest.mark.skip(reason="Leaks a large amount of memory")
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)], timeout=30)
+@gen_cluster(
+    client=True,
+    ncores=[("127.0.0.1", 1)],
+    timeout=30,
+    worker_kwargs={"memory_limit": 10e6},
+)
 def test_spill_by_default(c, s, w):
     da = pytest.importorskip("dask.array")
-    x = da.ones(int(TOTAL_MEMORY * 0.7), chunks=10000000, dtype="u1")
+    x = da.ones(int(10e6 * 0.7), chunks=1e6, dtype="u1")
     y = c.persist(x)
     yield wait(y)
     assert len(w.data.slow)  # something is on disk

From 0d115acada91fcfe4a685c7369a8b9736c9364ff Mon Sep 17 00:00:00 2001
From: Jim Crist <jcrist@users.noreply.github.com>
Date: Tue, 30 Apr 2019 13:11:38 -0500
Subject: [PATCH 0251/1550] Add timeout to Client._reconnect (#2639)

Previously if a client lost connection to the scheduler, it would try to
reconnect forever. We now use the same timeout as the initial connect.
On failure a nice message is logged and the client is shutdown.
---
 distributed/client.py            | 20 ++++++++++++++++----
 distributed/tests/test_client.py | 13 +++++++++++++
 distributed/utils_test.py        |  2 +-
 3 files changed, 30 insertions(+), 5 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 96d20a7ece2..b8bb42c3115 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -954,9 +954,10 @@ def _start(self, timeout=no_default, **kwargs):
         raise gen.Return(self)
 
     @gen.coroutine
-    def _reconnect(self, timeout=0.1):
+    def _reconnect(self):
         with log_errors():
             assert self.scheduler_comm.comm.closed()
+
             self.status = "connecting"
             self.scheduler_comm = None
 
@@ -964,12 +965,23 @@ def _reconnect(self, timeout=0.1):
                 st.cancel()
             self.futures.clear()
 
-            while self.status == "connecting":
+            timeout = self._timeout
+            deadline = self.loop.time() + timeout
+            while timeout > 0 and self.status == "connecting":
                 try:
-                    yield self._ensure_connected()
+                    yield self._ensure_connected(timeout=timeout)
                     break
                 except EnvironmentError:
-                    yield gen.sleep(timeout)
+                    # Wait a bit before retrying
+                    yield gen.sleep(0.1)
+                    timeout = deadline - self.loop.time()
+            else:
+                logger.error(
+                    "Failed to reconnect to scheduler after %.2f "
+                    "seconds, closing client",
+                    self._timeout,
+                )
+                yield self._close()
 
     @gen.coroutine
     def _ensure_connected(self, timeout=None):
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index a9d4f21fd81..545d3af67a9 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -3704,6 +3704,19 @@ def test_reconnect(loop):
     c.close()
 
 
+@gen_cluster(client=True, ncores=[], client_kwargs={"timeout": 0.5})
+def test_reconnect_timeout(c, s):
+    with captured_logger(logging.getLogger("distributed.client")) as logger:
+        yield s.close()
+        start = time()
+        while c.status != "closed":
+            yield c._update_scheduler_info()
+            yield gen.sleep(0.05)
+            assert time() < start + 5, "Timeout waiting for reconnect to fail"
+    text = logger.getvalue()
+    assert "Failed to reconnect" in text
+
+
 @slow
 @pytest.mark.skipif(
     sys.platform.startswith("win"), reason="num_fds not supported on windows"
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index a3f76e4c477..fcda695e2ec 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -974,7 +974,7 @@ def coro():
                                 if s.validate:
                                     s.validate_state()
                             finally:
-                                if client:
+                                if client and c.status not in ("closing", "closed"):
                                     yield c._close(fast=s.status == "closed")
                                 yield end_cluster(s, workers)
                                 yield gen.with_timeout(

From 2d431399ee32aef85f4ccd386623d599fa1ca50c Mon Sep 17 00:00:00 2001
From: Jim Crist <jcrist@users.noreply.github.com>
Date: Tue, 30 Apr 2019 13:36:20 -0500
Subject: [PATCH 0252/1550] Add as_completed methods to docs (#2642)

Adds ``as_completed`` methods to the docs, and cleans up the existing
docstrings slightly. Also adds a new method ``has_ready`` for checking
if there are any completed futures ready for processing.
---
 distributed/client.py            | 12 ++++++++----
 distributed/tests/test_client.py |  5 +++++
 docs/source/api.rst              |  4 +++-
 3 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index b8bb42c3115..332a6681508 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -4122,7 +4122,7 @@ def _notify(self):
             self.thread_condition.notify()
 
     @gen.coroutine
-    def track_future(self, future):
+    def _track_future(self, future):
         try:
             yield _wait(future)
         except CancelledError:
@@ -4148,7 +4148,7 @@ def update(self, futures):
                 if not isinstance(f, Future):
                     raise TypeError("Input must be a future, got %s" % f)
                 self.futures[f] += 1
-                self.loop.add_callback(self.track_future, f)
+                self.loop.add_callback(self._track_future, f)
 
     def add(self, future):
         """ Add a future to the collection
@@ -4158,9 +4158,13 @@ def add(self, future):
         self.update((future,))
 
     def is_empty(self):
-        """Return True if there no waiting futures, False otherwise"""
+        """Returns True if there no completed or computing futures"""
         return not self.count()
 
+    def has_ready(self):
+        """Returns True if there are completed futures available."""
+        return not self.queue.empty()
+
     def count(self):
         """ Return the number of futures yet to be returned
 
@@ -4207,7 +4211,7 @@ def __anext__(self):
     next = __next__
 
     def next_batch(self, block=True):
-        """ Get next batch of futures from as_completed iterator
+        """ Get the next batch of completed futures.
 
         Parameters
         ----------
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 545d3af67a9..d98d9039686 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -3590,8 +3590,13 @@ def test_as_completed_batches(c, with_results):
 def test_as_completed_next_batch(c):
     futures = c.map(slowinc, range(2), delay=0.1)
     ac = as_completed(futures)
+    assert not ac.is_empty()
     assert ac.next_batch(block=False) == []
     assert set(ac.next_batch(block=True)).issubset(futures)
+    while not ac.is_empty():
+        assert set(ac.next_batch(block=True)).issubset(futures)
+    assert ac.is_empty()
+    assert not ac.has_ready()
 
 
 @gen_test()
diff --git a/docs/source/api.rst b/docs/source/api.rst
index 33634bff6c7..47933be06d4 100644
--- a/docs/source/api.rst
+++ b/docs/source/api.rst
@@ -152,7 +152,9 @@ Future
 Other
 -----
 
-.. autofunction:: as_completed
+.. autoclass:: as_completed
+   :members:
+
 .. autofunction:: distributed.diagnostics.progress
 .. autofunction:: wait
 .. autofunction:: fire_and_forget

From 6fca31a13525d42ce7b2aa786e3e8a3a8f98cae5 Mon Sep 17 00:00:00 2001
From: Martin Durant <martindurant@users.noreply.github.com>
Date: Wed, 1 May 2019 09:59:16 -0400
Subject: [PATCH 0253/1550] Set working worker class for dask-ssh (#2646)

Fixes #2645

Suggestions welcome on how to test
---
 distributed/cli/dask_ssh.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/cli/dask_ssh.py b/distributed/cli/dask_ssh.py
index df2b1c6fe94..2d98992d969 100755
--- a/distributed/cli/dask_ssh.py
+++ b/distributed/cli/dask_ssh.py
@@ -96,7 +96,7 @@
 )
 @click.option(
     "--remote-dask-worker",
-    default=None,
+    default="distributed.cli.dask_worker",
     type=str,
     help="Worker to run. Defaults to distributed.cli.dask_worker",
 )

From 7b470c4cbedcd1b98d271983a2ac1c5a909e1230 Mon Sep 17 00:00:00 2001
From: plbertrand <bertrand.pierreluc@gmail.com>
Date: Wed, 1 May 2019 12:39:46 -0400
Subject: [PATCH 0254/1550] Add last worker into KilledWorker exception to help
 debug  (#2610)

Fixes #2549
---
 distributed/core.py                 |  2 +-
 distributed/scheduler.py            | 23 +++++++++++++++++++++--
 distributed/tests/test_client.py    |  4 +++-
 distributed/tests/test_scheduler.py |  9 +++++++++
 4 files changed, 34 insertions(+), 4 deletions(-)

diff --git a/distributed/core.py b/distributed/core.py
index e074fa68148..bb8a47c8525 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -959,7 +959,7 @@ def clean_exception(exception, traceback, **kwargs):
     --------
     error_message: create and serialize errors into message
     """
-    if isinstance(exception, bytes):
+    if isinstance(exception, bytes) or isinstance(exception, bytearray):
         try:
             exception = protocol.pickle.loads(exception)
         except Exception:
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 5e5e2843c2c..9aebcf11b4f 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -260,6 +260,20 @@ def __init__(
     def host(self):
         return get_address_host(self.address)
 
+    def clean(self):
+        """ Return a version of this object that is appropriate for serialization """
+        ws = WorkerState(
+            address=self.address,
+            pid=self.pid,
+            name=self.name,
+            ncores=self.ncores,
+            memory_limit=self.memory_limit,
+            local_directory=self.local_directory,
+            services=self.services,
+        )
+        ws.processing = {ts.key for ts in self.processing}
+        return ws
+
     def __repr__(self):
         return "<Worker %r, memory: %d, processing: %d>" % (
             self.address,
@@ -1872,7 +1886,9 @@ def remove_worker(self, comm=None, address=None, safe=False, close=True):
                     ts.suspicious += 1
                     if ts.suspicious > self.allowed_failures:
                         del recommendations[k]
-                        e = pickle.dumps(KilledWorker(k, address))
+                        e = pickle.dumps(
+                            KilledWorker(task=k, last_worker=ws.clean()), -1
+                        )
                         r = self.transition(k, "erred", exception=e, cause=k)
                         recommendations.update(r)
 
@@ -4827,4 +4843,7 @@ def heartbeat_interval(n):
 
 
 class KilledWorker(Exception):
-    pass
+    def __init__(self, task, last_worker):
+        super(KilledWorker, self).__init__(task, last_worker)
+        self.task = task
+        self.last_worker = last_worker
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index d98d9039686..ff94fba7787 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -3479,9 +3479,11 @@ def test_get_foo_lost_keys(c, s, u, v, w):
 @gen_cluster(client=True, Worker=Nanny, check_new_threads=False)
 def test_bad_tasks_fail(c, s, a, b):
     f = c.submit(sys.exit, 1)
-    with pytest.raises(KilledWorker):
+    with pytest.raises(KilledWorker) as info:
         yield f
 
+    assert info.value.last_worker.services["nanny"] in {a.port, b.port}
+
 
 def test_get_processing_sync(c, s, a, b):
     processing = c.processing()
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 02f15e1e1a2..5a68f287a1b 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -1,6 +1,7 @@
 from __future__ import print_function, division, absolute_import
 
 import cloudpickle
+import pickle
 from collections import defaultdict
 from datetime import timedelta
 import json
@@ -1496,3 +1497,11 @@ def qux(x):
     yield gen.sleep(0.1)
     f = c.submit(bar, x, key="y")
     yield f
+
+
+@gen_cluster()
+def test_workerstate_clean(s, a, b):
+    ws = s.workers[a.address].clean()
+    assert ws.address == a.address
+    b = pickle.dumps(ws)
+    assert len(b) < 1000

From 1082e3c91ad6576dc8956b86469268b52d4b7938 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 3 May 2019 11:00:12 -0700
Subject: [PATCH 0255/1550] Explain LocalCluster behavior in Client docstring
 (#2647)

---
 distributed/client.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/distributed/client.py b/distributed/client.py
index 332a6681508..24cf9ce5cf5 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -538,6 +538,9 @@ class resembles executors in ``concurrent.futures`` but also allows
         the scheduler to serve as intermediary.
     heartbeat_interval: int
         Time in milliseconds between heartbeats to scheduler
+    **kwargs:
+        If you do not pass a scheduler address, Client will create a
+        ``LocalCluster`` object, passing any extra keyword arguments.
 
     Examples
     --------
@@ -559,9 +562,19 @@ class resembles executors in ``concurrent.futures`` but also allows
     >>> client.gather(c)  # doctest: +SKIP
     33
 
+    You can also call Client with no arguments in order to create your own
+    local cluster.
+
+    >>> client = Client()  # makes your own local "cluster" # doctest: +SKIP
+
+    Extra keywords will be passed directly to LocalCluster
+
+    >>> client = Client(processes=False, threads_per_worker=1)  # doctest: +SKIP
+
     See Also
     --------
     distributed.scheduler.Scheduler: Internal scheduler
+    distributed.deploy.local.LocalCluster:
     """
 
     def __init__(

From fade817e361a6102b013f600e6947b8d2f7939df Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 3 May 2019 16:04:20 -0700
Subject: [PATCH 0256/1550] Add Comm closed bookkeeping (#2648)

We now track open Comms at every test in order to identify cases where
Comms may leak out and not be closed up.

This involves adding weak references and names to many objects, which
should hopefully help debugging in the future.
---
 distributed/client.py               | 11 ++++-------
 distributed/comm/core.py            | 10 +++++++++-
 distributed/comm/inproc.py          |  1 +
 distributed/comm/tcp.py             |  1 +
 distributed/core.py                 | 24 +++++++++++++++++++++++-
 distributed/node.py                 |  5 +++++
 distributed/scheduler.py            |  3 +++
 distributed/tests/test_scheduler.py |  2 +-
 distributed/utils_test.py           | 29 +++++++++++++++++++++++++++--
 distributed/worker.py               |  8 ++++++--
 10 files changed, 80 insertions(+), 14 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 24cf9ce5cf5..84728c62fb7 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -703,6 +703,7 @@ def __init__(
             io_loop=self.loop,
             serializers=serializers,
             deserializers=deserializers,
+            timeout=timeout,
         )
 
         for ext in extensions:
@@ -947,13 +948,7 @@ def _start(self, timeout=no_default, **kwargs):
             address = self.cluster.scheduler_address
 
         if self.scheduler is None:
-            self.scheduler = rpc(
-                address,
-                timeout=timeout,
-                connection_args=self.connection_args,
-                serializers=self._serializers,
-                deserializers=self._deserializers,
-            )
+            self.scheduler = self.rpc(address)
         self.scheduler_comm = None
 
         yield self._ensure_connected(timeout=timeout)
@@ -1014,6 +1009,7 @@ def _ensure_connected(self, timeout=None):
                 timeout=timeout,
                 connection_args=self.connection_args,
             )
+            comm.name = "Client->Scheduler"
             if timeout is not None:
                 yield gen.with_timeout(
                     timedelta(seconds=timeout), self._update_scheduler_info()
@@ -1238,6 +1234,7 @@ def _close(self, fast=False):
             if self._start_arg is None:
                 with ignoring(AttributeError):
                     yield self.cluster._close()
+            self.rpc.close()
             self.status = "closed"
             if _get_global_client() is self:
                 _set_global_client(None)
diff --git a/distributed/comm/core.py b/distributed/comm/core.py
index b66be0b6dc4..e0b236e7b96 100644
--- a/distributed/comm/core.py
+++ b/distributed/comm/core.py
@@ -3,6 +3,7 @@
 from abc import ABCMeta, abstractmethod, abstractproperty
 from datetime import timedelta
 import logging
+import weakref
 
 import dask
 from six import with_metaclass
@@ -37,6 +38,12 @@ class Comm(with_metaclass(ABCMeta)):
     depending on the underlying transport's characteristics.
     """
 
+    _instances = weakref.WeakSet()
+
+    def __init__(self):
+        self._instances.add(self)
+        self.name = None
+
     # XXX add set_close_callback()?
 
     @abstractmethod
@@ -116,8 +123,9 @@ def __repr__(self):
         if self.closed():
             return "<closed %s>" % (clsname,)
         else:
-            return "<%s local=%s remote=%s>" % (
+            return "<%s %s local=%s remote=%s>" % (
                 clsname,
+                self.name or "",
                 self.local_address,
                 self.peer_address,
             )
diff --git a/distributed/comm/inproc.py b/distributed/comm/inproc.py
index 8721a3df8ac..7f267978d51 100644
--- a/distributed/comm/inproc.py
+++ b/distributed/comm/inproc.py
@@ -152,6 +152,7 @@ class InProc(Comm):
     def __init__(
         self, local_addr, peer_addr, read_q, write_q, write_loop, deserialize=True
     ):
+        Comm.__init__(self)
         self._local_addr = local_addr
         self._peer_addr = peer_addr
         self.deserialize = deserialize
diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py
index 6d90a7bc9c7..85dbe2ce278 100644
--- a/distributed/comm/tcp.py
+++ b/distributed/comm/tcp.py
@@ -152,6 +152,7 @@ class TCP(Comm):
     _iostream_has_read_into = hasattr(IOStream, "read_into")
 
     def __init__(self, stream, local_addr, peer_addr, deserialize=True):
+        Comm.__init__(self)
         self._local_addr = local_addr
         self._peer_addr = peer_addr
         self.stream = stream
diff --git a/distributed/core.py b/distributed/core.py
index bb8a47c8525..3cf3f9b5bb2 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -593,6 +593,7 @@ def __init__(
         self.serializers = serializers
         self.deserializers = deserializers if deserializers is not None else serializers
         self.connection_args = connection_args
+        self._created = weakref.WeakSet()
         rpc.active.add(self)
 
     @gen.coroutine
@@ -632,6 +633,7 @@ def live_comm(self):
                 deserialize=self.deserialize,
                 connection_args=self.connection_args,
             )
+            comm.name = "rpc"
         self.comms[comm] = False  # mark as taken
         raise gen.Return(comm)
 
@@ -648,6 +650,9 @@ def _close_comm(comm):
         for comm in list(self.comms):
             if comm and not comm.closed():
                 _close_comm(comm)
+        for comm in list(self._created):
+            if comm and not comm.closed():
+                _close_comm(comm)
         self.comms.clear()
 
     def __getattr__(self, key):
@@ -659,6 +664,7 @@ def send_recv_from_rpc(**kwargs):
                 kwargs["deserializers"] = self.deserializers
             try:
                 comm = yield self.live_comm()
+                comm.name = "rpc." + key
                 result = yield send_recv(comm=comm, op=key, **kwargs)
             except (RPCClosed, CommClosedError) as e:
                 raise e.__class__(
@@ -723,10 +729,12 @@ def send_recv_from_rpc(**kwargs):
             if self.deserializers is not None and kwargs.get("deserializers") is None:
                 kwargs["deserializers"] = self.deserializers
             comm = yield self.pool.connect(self.addr)
+            name, comm.name = comm.name, "ConnectionPool." + key
             try:
                 result = yield send_recv(comm=comm, op=key, **kwargs)
             finally:
                 self.pool.reuse(self.addr, comm)
+                comm.name = name
 
             raise gen.Return(result)
 
@@ -780,6 +788,8 @@ class ConnectionPool(object):
         Whether or not to deserialize data by default or pass it through
     """
 
+    _instances = weakref.WeakSet()
+
     def __init__(
         self,
         limit=512,
@@ -787,6 +797,8 @@ def __init__(
         serializers=None,
         deserializers=None,
         connection_args=None,
+        timeout=None,
+        server=None,
     ):
         self.limit = limit  # Max number of open comms
         # Invariant: len(available) == open - active
@@ -797,7 +809,11 @@ def __init__(
         self.serializers = serializers
         self.deserializers = deserializers if deserializers is not None else serializers
         self.connection_args = connection_args
+        self.timeout = timeout
         self.event = Event()
+        self.server = weakref.ref(server) if server else None
+        self._created = weakref.WeakSet()
+        self._instances.add(self)
 
     @property
     def active(self):
@@ -838,10 +854,13 @@ def connect(self, addr, timeout=None):
         try:
             comm = yield connect(
                 addr,
-                timeout=timeout,
+                timeout=timeout or self.timeout,
                 deserialize=self.deserialize,
                 connection_args=self.connection_args,
             )
+            comm.name = "ConnectionPool"
+            comm._pool = weakref.ref(self)
+            self._created.add(comm)
         except Exception:
             raise
         occupied.add(comm)
@@ -907,6 +926,9 @@ def close(self):
             for comm in comms:
                 comm.abort()
 
+        for comm in self._created:
+            IOLoop.current().add_callback(comm.abort)
+
 
 def coerce_to_address(o):
     if isinstance(o, (list, tuple)):
diff --git a/distributed/node.py b/distributed/node.py
index 8a0b8c12195..8134546fa0b 100644
--- a/distributed/node.py
+++ b/distributed/node.py
@@ -19,6 +19,7 @@ def __init__(
         io_loop=None,
         serializers=None,
         deserializers=None,
+        timeout=None,
     ):
         self.io_loop = io_loop or IOLoop.current()
         self.rpc = ConnectionPool(
@@ -27,6 +28,8 @@ def __init__(
             serializers=serializers,
             deserializers=deserializers,
             connection_args=connection_args,
+            timeout=timeout,
+            server=self,
         )
 
 
@@ -51,6 +54,7 @@ def __init__(
         io_loop=None,
         serializers=None,
         deserializers=None,
+        timeout=None,
     ):
         Node.__init__(
             self,
@@ -60,6 +64,7 @@ def __init__(
             io_loop=io_loop,
             serializers=serializers,
             deserializers=deserializers,
+            timeout=timeout,
         )
         Server.__init__(
             self,
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 9aebcf11b4f..af87960f6b6 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -2176,6 +2176,7 @@ def add_client(self, comm, client=None):
         We listen to all future messages from this Comm.
         """
         assert client is not None
+        comm.name = "Scheduler->Client"
         logger.info("Receive client connection: %s", client)
         self.log_event(["all", client], {"action": "add-client", "client": client})
         self.clients[client] = ClientState(client)
@@ -2373,6 +2374,7 @@ def handle_worker(self, comm=None, worker=None):
         --------
         Scheduler.handle_client: Equivalent coroutine for clients
         """
+        comm.name = "Scheduler connection to worker"
         worker_comm = self.stream_comms[worker]
         worker_comm.start(comm)
         logger.info("Starting worker compute stream, %s", worker)
@@ -2633,6 +2635,7 @@ def send_message(addr):
             comm = yield connect(
                 addr, deserialize=self.deserialize, connection_args=self.connection_args
             )
+            comm.name = "Scheduler Broadcast"
             resp = yield send_recv(comm, close=True, serializers=serializers, **msg)
             raise gen.Return(resp)
 
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 5a68f287a1b..7bb114a4cbc 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -824,7 +824,7 @@ def test_file_descriptors(c, s):
     yield [n._close() for n in nannies]
 
     assert not s.rpc.open
-    assert not c.rpc.open
+    assert not c.rpc.active
     assert not s.stream_comms
 
     start = time()
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index fcda695e2ec..ba5567b1a80 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -41,6 +41,7 @@
 
 from .client import default_client, _global_clients, Client
 from .compatibility import PY3, Empty, WINDOWS, PY2
+from .comm import Comm
 from .comm.utils import offload
 from .config import initialize_logging
 from .core import connect, rpc, CommClosedError
@@ -637,6 +638,7 @@ def cluster(
     ws = weakref.WeakSet()
 
     reset_config()
+    Comm._instances.clear()
 
     for name, level in logging_levels.items():
         logging.getLogger(name).setLevel(level)
@@ -761,6 +763,17 @@ def cluster(
         sleep(0.01)
         assert time() < start + 1, "Workers still around after one second"
 
+    for i in range(5):
+        if all(c.closed() for c in Comm._instances):
+            break
+        else:
+            sleep(0.1)
+    else:
+        L = [c for c in Comm._instances if not c.closed()]
+        Comm._instances.clear()
+        print("Unclosed Comms", L)
+        # raise ValueError("Unclosed Comms", L)
+
 
 @gen.coroutine
 def disconnect(addr, timeout=3, rpc_kwargs=None):
@@ -845,8 +858,8 @@ def start_cluster(
         )
         for i, ncore in enumerate(ncores)
     ]
-    for w in workers:
-        w.rpc = workers[0].rpc
+    # for w in workers:
+    #     w.rpc = workers[0].rpc
 
     yield [w._start(ncore[0]) for ncore, w in zip(ncores, workers)]
 
@@ -913,6 +926,7 @@ def _(func):
         def test_func():
             del _global_workers[:]
             _global_clients.clear()
+            Comm._instances.clear()
             active_threads_start = set(threading._active)
 
             reset_config()
@@ -988,6 +1002,17 @@ def coro():
                             else:
                                 yield c._close(fast=True)
 
+                            for i in range(5):
+                                if all(c.closed() for c in Comm._instances):
+                                    break
+                                else:
+                                    yield gen.sleep(0.05)
+                            else:
+                                L = [c for c in Comm._instances if not c.closed()]
+                                Comm._instances.clear()
+                                # raise ValueError("Unclosed Comms", L)
+                                print("Unclosed Comms", L)
+
                             raise gen.Return(result)
 
                     result = loop.run_sync(
diff --git a/distributed/worker.py b/distributed/worker.py
index 33010836fdb..74ad395cc4d 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -674,6 +674,8 @@ def _register_with_scheduler(self):
                 comm = yield connect(
                     self.scheduler.address, connection_args=self.connection_args
                 )
+                comm.name = "Worker->Scheduler"
+                comm._server = weakref.ref(self)
                 yield comm.write(
                     dict(
                         op="register-worker",
@@ -993,8 +995,6 @@ def _close(self, report=True, timeout=10, nanny=True, executor_wait=True):
             for k, v in self.services.items():
                 v.stop()
 
-            self.status = "closed"
-
             if nanny and "nanny" in self.service_ports:
                 nanny_address = "%s%s:%d" % (
                     self.listener.prefix,
@@ -1013,6 +1013,8 @@ def _close(self, report=True, timeout=10, nanny=True, executor_wait=True):
             self.rpc.close()
             self._closed.set()
             self._remove_from_global_workers()
+
+            self.status = "closed"
             yield self.close()
 
             setproctitle("dask-worker [closed]")
@@ -1051,6 +1053,7 @@ def batched_send_connect():
                 comm = yield connect(
                     address, connection_args=self.connection_args  # TODO, serialization
                 )
+                comm.name = "Worker->Worker"
                 yield comm.write({"op": "connection_stream"})
 
                 bcomm.start(comm)
@@ -2952,6 +2955,7 @@ def get_data_from_worker(
         deserializers = rpc.deserializers
 
     comm = yield rpc.connect(worker)
+    comm.name = "Ephemeral Worker->Worker for gather"
     try:
         response = yield send_recv(
             comm,

From 2783024663c136318c3eca359951ac0cd2e2529e Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 3 May 2019 17:05:22 -0700
Subject: [PATCH 0257/1550] Rename Worker._close to Worker.close (#2650)

---
 distributed/deploy/local.py                   |  2 +-
 distributed/diagnostics/tests/test_plugin.py  |  6 ++--
 .../diagnostics/tests/test_progress.py        |  4 +--
 .../diagnostics/tests/test_progressbar.py     |  2 +-
 distributed/nanny.py                          | 17 ++++++----
 distributed/protocol/core.py                  | 20 ++---------
 distributed/protocol/serialize.py             | 17 +++++-----
 distributed/protocol/utils.py                 | 13 +++++++
 distributed/tests/test_client.py              | 34 +++++++++----------
 distributed/tests/test_nanny.py               | 24 ++++++-------
 distributed/tests/test_scheduler.py           | 24 ++++++-------
 distributed/tests/test_stress.py              |  4 +--
 distributed/tests/test_worker.py              | 24 ++++++-------
 distributed/utils_test.py                     | 10 +++---
 distributed/worker.py                         | 16 +++++----
 15 files changed, 111 insertions(+), 106 deletions(-)

diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index 73ddde8bdc1..68a47c85d48 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -338,7 +338,7 @@ def start_worker(self, **kwargs):
 
     @gen.coroutine
     def _stop_worker(self, w):
-        yield w._close()
+        yield w.close()
         if w in self.workers:
             self.workers.remove(w)
 
diff --git a/distributed/diagnostics/tests/test_plugin.py b/distributed/diagnostics/tests/test_plugin.py
index b1d5406e052..fa4449c74b7 100644
--- a/distributed/diagnostics/tests/test_plugin.py
+++ b/distributed/diagnostics/tests/test_plugin.py
@@ -55,8 +55,8 @@ def remove_worker(self, worker, scheduler):
     b = Worker(s.address)
     yield a
     yield b
-    yield a._close()
-    yield b._close()
+    yield a.close()
+    yield b.close()
 
     assert events == [
         ("add_worker", a.address),
@@ -68,5 +68,5 @@ def remove_worker(self, worker, scheduler):
     events[:] = []
     s.remove_plugin(plugin)
     a = yield Worker(s.address)
-    yield a._close()
+    yield a.close()
     assert events == []
diff --git a/distributed/diagnostics/tests/test_progress.py b/distributed/diagnostics/tests/test_progress.py
index d8435cc7ff0..097b2670247 100644
--- a/distributed/diagnostics/tests/test_progress.py
+++ b/distributed/diagnostics/tests/test_progress.py
@@ -185,8 +185,8 @@ def test_AllProgress_lost_key(c, s, a, b, timeout=None):
     yield wait(futures)
     assert len(p.state["memory"]["inc"]) == 5
 
-    yield a._close()
-    yield b._close()
+    yield a.close()
+    yield b.close()
 
     start = time()
     while len(p.state["memory"]["inc"]) > 0:
diff --git a/distributed/diagnostics/tests/test_progressbar.py b/distributed/diagnostics/tests/test_progressbar.py
index 8738cb60e22..d5a01410f5e 100644
--- a/distributed/diagnostics/tests/test_progressbar.py
+++ b/distributed/diagnostics/tests/test_progressbar.py
@@ -63,7 +63,7 @@ def f():
         assert progress.status == "finished"
         check_bar_completed(capsys)
 
-        yield [a._close(), b._close()]
+        yield [a.close(), b.close()]
         s.close()
         yield done
 
diff --git a/distributed/nanny.py b/distributed/nanny.py
index 356ebc3168d..4b81bec4646 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -8,6 +8,7 @@
 import shutil
 import threading
 import uuid
+import warnings
 
 import dask
 from tornado import gen
@@ -122,7 +123,7 @@ def __init__(
             "kill": self.kill,
             "restart": self.restart,
             # cannot call it 'close' on the rpc side for naming conflict
-            "terminate": self._close,
+            "terminate": self.close,
             "run": self.run,
         }
 
@@ -197,7 +198,7 @@ def _start(self, addr_or_port=0):
             assert self.worker_address
             self.status = "running"
         else:
-            yield self._close()
+            yield self.close()
 
         self.start_periodic_callbacks()
 
@@ -275,7 +276,7 @@ def instantiate(self, comm=None):
                     timedelta(seconds=self.death_timeout), self.process.start()
                 )
             except gen.TimeoutError:
-                yield self._close(timeout=self.death_timeout)
+                yield self.close(timeout=self.death_timeout)
                 raise gen.Return("timed out")
         else:
             result = yield self.process.start()
@@ -332,7 +333,7 @@ def _on_exit(self, exitcode):
                 yield self.scheduler.unregister(address=self.worker_address)
             except (EnvironmentError, CommClosedError):
                 if not self.reconnect:
-                    yield self._close()
+                    yield self.close()
                     return
 
             try:
@@ -349,8 +350,12 @@ def _on_exit(self, exitcode):
     def pid(self):
         return self.process and self.process.pid
 
+    def _close(self, *args, **kwargs):
+        warnings.warn("Worker._close has moved to Worker.close")
+        return self.close(*args, **kwargs)
+
     @gen.coroutine
-    def _close(self, comm=None, timeout=5, report=None):
+    def close(self, comm=None, timeout=5, report=None):
         """
         Close the worker process, stop all comms.
         """
@@ -584,7 +589,7 @@ def _run(
         @gen.coroutine
         def do_stop(timeout=5, executor_wait=True):
             try:
-                yield worker._close(
+                yield worker.close(
                     report=False,
                     nanny=False,
                     executor_wait=executor_wait,
diff --git a/distributed/protocol/core.py b/distributed/protocol/core.py
index 0b5f7eb0fea..c1b62b2491e 100644
--- a/distributed/protocol/core.py
+++ b/distributed/protocol/core.py
@@ -11,29 +11,13 @@
     from toolz import reduce
 
 from .compression import compressions, maybe_compress, decompress
-from .serialize import (
-    serialize,
-    deserialize,
-    Serialize,
-    Serialized,
-    extract_serialize,
-    msgpack_len_opts,
-)
-from .utils import frame_split_size, merge_frames
+from .serialize import serialize, deserialize, Serialize, Serialized, extract_serialize
+from .utils import frame_split_size, merge_frames, msgpack_opts
 from ..utils import nbytes
 
 _deserialize = deserialize
 
 
-try:
-    msgpack.loads(msgpack.dumps(""), raw=False, **msgpack_len_opts)
-    msgpack_opts = {"raw": False}
-    msgpack_opts.update(msgpack_len_opts)
-except TypeError:
-    # Backward compat with old msgpack (prior to 0.5.2)
-    msgpack_opts = {"encoding": "utf-8"}
-
-
 logger = logging.getLogger(__name__)
 
 
diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index 4ff0fb47a65..f47ea7388af 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -16,7 +16,13 @@
 from ..compatibility import PY2
 from ..utils import has_keyword
 from .compression import maybe_compress, decompress
-from .utils import unpack_frames, pack_frames_prelude, frame_split_size, ensure_bytes
+from .utils import (
+    unpack_frames,
+    pack_frames_prelude,
+    frame_split_size,
+    ensure_bytes,
+    msgpack_opts,
+)
 
 
 lazy_registrations = {}
@@ -58,11 +64,6 @@ def pickle_loads(header, frames):
     return pickle.loads(b"".join(frames))
 
 
-msgpack_len_opts = {
-    ("max_%s_len" % x): 2 ** 31 - 1 for x in ["str", "bin", "array", "map", "ext"]
-}
-
-
 def msgpack_dumps(x):
     try:
         frame = msgpack.dumps(x, use_bin_type=True)
@@ -73,9 +74,7 @@ def msgpack_dumps(x):
 
 
 def msgpack_loads(header, frames):
-    return msgpack.loads(
-        b"".join(frames), encoding="utf8", use_list=False, **msgpack_len_opts
-    )
+    return msgpack.loads(b"".join(frames), use_list=False, **msgpack_opts)
 
 
 def serialization_error_loads(header, frames):
diff --git a/distributed/protocol/utils.py b/distributed/protocol/utils.py
index 90d30342951..208caebb926 100644
--- a/distributed/protocol/utils.py
+++ b/distributed/protocol/utils.py
@@ -1,12 +1,25 @@
 from __future__ import print_function, division, absolute_import
 
 import struct
+import msgpack
 
 from ..utils import ensure_bytes, nbytes
 
 BIG_BYTES_SHARD_SIZE = 2 ** 26
 
 
+msgpack_opts = {
+    ("max_%s_len" % x): 2 ** 31 - 1 for x in ["str", "bin", "array", "map", "ext"]
+}
+
+try:
+    msgpack.loads(msgpack.dumps(""), raw=False, **msgpack_opts)
+    msgpack_opts["raw"] = False
+except TypeError:
+    # Backward compat with old msgpack (prior to 0.5.2)
+    msgpack_opts["encoding"] = "utf-8"
+
+
 def frame_split_size(frames, n=BIG_BYTES_SHARD_SIZE):
     """
     Split a list of frames into a list of frames of maximum size
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index ff94fba7787..6f1fb5c8a03 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -528,7 +528,7 @@ def test_gather_lost(c, s, a, b):
     [x] = yield c.scatter([1], workers=a.address)
     y = c.submit(inc, 1, workers=b.address)
 
-    yield a._close()
+    yield a.close()
 
     with pytest.raises(Exception):
         res = yield c.gather([x, y])
@@ -641,7 +641,7 @@ def g(a, b):
     with pytest.raises(AttributeError):
         yield c.gather(future_g)
 
-    yield a._close()
+    yield a.close()
 
 
 @gen_cluster(client=True)
@@ -946,7 +946,7 @@ def test_remove_worker(c, s, a, b):
     L = c.map(inc, range(20))
     yield wait(L)
 
-    yield b._close()
+    yield b.close()
 
     assert b.address not in s.workers
 
@@ -2845,7 +2845,7 @@ def test_worker_aliases():
         assert result == i + 1
 
     yield c.close()
-    yield [a._close(), b._close(), w._close()]
+    yield [a.close(), b.close(), w.close()]
     yield s.close()
 
 
@@ -3020,7 +3020,7 @@ def test_rebalance_unprepared(c, s, a, b):
 def test_receive_lost_key(c, s, a, b):
     x = c.submit(inc, 1, workers=[a.address])
     result = yield x
-    yield a._close()
+    yield a.close()
 
     start = time()
     while x.status == "finished":
@@ -3036,7 +3036,7 @@ def test_unrunnable_task_runs(c, s, a, b):
     x = c.submit(inc, 1, workers=[a.ip])
     result = yield x
 
-    yield a._close()
+    yield a.close()
     start = time()
     while x.status == "finished":
         assert time() < start + 5
@@ -3055,7 +3055,7 @@ def test_unrunnable_task_runs(c, s, a, b):
     assert s.tasks[x.key] not in s.unrunnable
     result = yield x
     assert result == 2
-    yield w._close()
+    yield w.close()
 
 
 @gen_cluster(client=True, ncores=[])
@@ -3067,7 +3067,7 @@ def test_add_worker_after_tasks(c, s):
 
     result = yield c.gather(futures)
 
-    yield n._close()
+    yield n.close()
 
 
 @pytest.mark.skipif(
@@ -3460,8 +3460,8 @@ def test_get_foo_lost_keys(c, s, u, v, w):
     d = yield c.scheduler.who_has(keys=[x.key, y.key])
     assert_dict_key_equal(d, {x.key: [ua], y.key: [va]})
 
-    yield u._close()
-    yield v._close()
+    yield u.close()
+    yield v.close()
 
     d = yield c.scheduler.has_what()
     assert_dict_key_equal(d, {wa: []})
@@ -3707,7 +3707,7 @@ def test_reconnect(loop):
         assert time() < start + 5
         sleep(0.1)
 
-    sync(loop, w._close)
+    sync(loop, w.close)
     c.close()
 
 
@@ -3753,7 +3753,7 @@ def start_worker(sleep, duration, repeat=1):
                 addr = w.worker_address
                 running[w] = addr
                 yield gen.sleep(duration)
-                yield w._close()
+                yield w.close()
                 del w
                 yield gen.moment
             done.release()
@@ -3882,7 +3882,7 @@ def f():
 def test_lose_scattered_data(c, s, a, b):
     [x] = yield c.scatter([1], workers=a.address)
 
-    yield a._close()
+    yield a.close()
     yield gen.sleep(0.1)
 
     assert x.status == "cancelled"
@@ -3894,7 +3894,7 @@ def test_partially_lose_scattered_data(e, s, a, b, c):
     [x] = yield e.scatter([1], workers=a.address)
     yield e.replicate(x, n=2)
 
-    yield a._close()
+    yield a.close()
     yield gen.sleep(0.1)
 
     assert x.status == "finished"
@@ -3909,7 +3909,7 @@ def test_scatter_compute_lose(c, s, a, b):
     z = c.submit(slowadd, x, y, delay=0.2)
     yield gen.sleep(0.1)
 
-    yield a._close()
+    yield a.close()
 
     with pytest.raises(CancelledError):
         yield wait(z)
@@ -3935,7 +3935,7 @@ def test_scatter_compute_store_lose(c, s, a, b):
     z = c.submit(slowadd, xx, y, delay=0.2, workers=b.address)
     yield wait(z)
 
-    yield a._close()
+    yield a.close()
 
     start = time()
     while x.status == "finished":
@@ -3980,7 +3980,7 @@ def test_scatter_compute_store_lose_processing(c, s, a, b):
     y = c.submit(slowinc, x, delay=0.2)
     z = c.submit(inc, y)
     yield gen.sleep(0.1)
-    yield a._close()
+    yield a.close()
 
     start = time()
     while x.status == "finished":
diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
index 932419015f3..08cd49fb3c9 100644
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -48,7 +48,7 @@ def test_nanny(s):
         yield nn.terminate()
         assert not n.is_alive()
 
-    yield n._close()
+    yield n.close()
 
 
 @gen_cluster(ncores=[])
@@ -57,7 +57,7 @@ def test_many_kills(s):
     assert n.is_alive()
     yield [n.kill() for i in range(5)]
     yield [n.kill() for i in range(5)]
-    yield n._close()
+    yield n.close()
 
 
 @gen_cluster(Worker=Nanny)
@@ -102,7 +102,7 @@ def test_nanny_process_failure(c, s):
 
     second_dir = n.worker_dir
 
-    yield n._close()
+    yield n.close()
     assert not os.path.exists(second_dir)
     assert not os.path.exists(first_dir)
     assert first_dir != n.worker_dir
@@ -124,7 +124,7 @@ def test_run(s):
         assert response["status"] == "OK"
         assert response["result"] == 1
 
-    yield n._close()
+    yield n.close()
 
 
 @slow
@@ -194,7 +194,7 @@ def test_num_fds(s):
 
     # Warm up
     w = yield Nanny(s.address)
-    yield w._close()
+    yield w.close()
     del w
     gc.collect()
 
@@ -203,7 +203,7 @@ def test_num_fds(s):
     for i in range(3):
         w = yield Nanny(s.address)
         yield gen.sleep(0.1)
-        yield w._close()
+        yield w.close()
 
     start = time()
     while proc.num_fds() > before:
@@ -226,7 +226,7 @@ def func(dask_worker):
 
         result = yield c.run(func)
         assert host in first(result.values())
-        yield n._close()
+        yield n.close()
 
 
 @gen_test()
@@ -236,7 +236,7 @@ def test_scheduler_file():
         s.start(8008)
         w = yield Nanny(scheduler_file=fn)
         assert set(s.workers) == {w.worker_address}
-        yield w._close()
+        yield w.close()
         s.stop()
 
 
@@ -301,7 +301,7 @@ def test_avoid_memory_monitor_if_zero_limit(c, s):
 
     yield c.submit(inc, 2)  # worker doesn't pause
 
-    yield nanny._close()
+    yield nanny.close()
 
 
 @gen_cluster(ncores=[], client=True)
@@ -315,7 +315,7 @@ def test_scheduler_address_config(c, s):
             yield gen.sleep(0.1)
             assert time() < start + 10
 
-    yield nanny._close()
+    yield nanny.close()
 
 
 @slow
@@ -338,7 +338,7 @@ def test_environment_variable(c, s):
     yield [a, b]
     results = yield c.run(lambda: os.environ["FOO"])
     assert results == {a.worker_address: "123", b.worker_address: "456"}
-    yield [a._close(), b._close()]
+    yield [a.close(), b.close()]
 
 
 @gen_cluster(ncores=[], client=True)
@@ -346,4 +346,4 @@ def test_data_types(c, s):
     w = yield Nanny(s.address, data=dict)
     r = yield c.run(lambda dask_worker: type(dask_worker.data))
     assert r[w.worker_address] == dict
-    yield w._close()
+    yield w.close()
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 7bb114a4cbc..8280c3cd120 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -275,7 +275,7 @@ def test_add_worker(s, a, b):
 
     assert w.ip in s.host_info
     assert s.host_info[w.ip]["addresses"] == {a.address, b.address, w.address}
-    yield w._close()
+    yield w.close()
 
 
 @gen_cluster(scheduler_kwargs={"blocked_handlers": ["feed"]})
@@ -541,10 +541,10 @@ def test_worker_name():
 
     with pytest.raises(ValueError):
         w2 = yield Worker(s.ip, s.port, name="alice")
-        yield w2._close()
+        yield w2.close()
 
     yield s.close()
-    yield w._close()
+    yield w.close()
 
 
 @gen_test()
@@ -585,7 +585,7 @@ def test_coerce_address():
         assert s.coerce_address("zzzt:8000", resolve=False) == "tcp://zzzt:8000"
 
         yield s.close()
-        yield [w._close() for w in [a, b, c]]
+        yield [w.close() for w in [a, b, c]]
 
 
 @pytest.mark.skipif(
@@ -598,7 +598,7 @@ def test_file_descriptors_dont_leak(s):
     before = proc.num_fds()
 
     w = yield Worker(s.ip, s.port)
-    yield w._close()
+    yield w.close()
 
     during = proc.num_fds()
 
@@ -668,7 +668,7 @@ def test_scatter_no_workers(c, s):
     yield [c.scatter(data={"y": 2}, timeout=5), w._start()]
 
     assert w.data["y"] == 2
-    yield w._close()
+    yield w.close()
 
 
 @gen_cluster(ncores=[])
@@ -676,7 +676,7 @@ def test_scheduler_sees_memory_limits(s):
     w = yield Worker(s.ip, s.port, ncores=3, memory_limit=12345)
 
     assert s.workers[w.address].memory_limit == 12345
-    yield w._close()
+    yield w.close()
 
 
 @gen_cluster(client=True, timeout=1000)
@@ -821,7 +821,7 @@ def test_file_descriptors(c, s):
     num_fds_6 = proc.num_fds()
     assert num_fds_6 < num_fds_5 + N
 
-    yield [n._close() for n in nannies]
+    yield [n.close() for n in nannies]
 
     assert not s.rpc.open
     assert not c.rpc.active
@@ -945,7 +945,7 @@ def test_worker_arrives_with_processing_data(c, s, a, b):
         z.key: "processing",
     }
 
-    yield w._close()
+    yield w.close()
 
 
 @gen_cluster(client=True, ncores=[("127.0.0.1", 1)])
@@ -996,7 +996,7 @@ def test_no_workers_to_memory(c, s):
         z.key: "processing",
     }
 
-    yield w._close()
+    yield w.close()
 
 
 @gen_cluster(client=True)
@@ -1025,7 +1025,7 @@ def test_no_worker_to_memory_restrictions(c, s, a, b):
         z.key: "processing",
     }
 
-    yield w._close()
+    yield w.close()
 
 
 def test_run_on_scheduler_sync(loop):
@@ -1333,7 +1333,7 @@ def test_mising_data_errant_worker(c, s, w1, w2, w3):
         y = c.submit(len, x, workers=w3.address)
         while not w3.tasks:
             yield gen.sleep(0.001)
-        w1._close()
+        w1.close()
         yield wait(y)
 
 
diff --git a/distributed/tests/test_stress.py b/distributed/tests/test_stress.py
index 8a36b8b3b94..f145a11b053 100644
--- a/distributed/tests/test_stress.py
+++ b/distributed/tests/test_stress.py
@@ -114,7 +114,7 @@ def create_and_destroy_worker(delay):
 
             yield gen.sleep(delay)
 
-            yield n._close()
+            yield n.close()
             print("Killed nanny")
 
     yield gen.with_timeout(
@@ -167,7 +167,7 @@ def test_stress_scatter_death(c, s, *workers):
             else:
                 raise
         w = random.choice(alive)
-        yield w._close()
+        yield w.close()
         alive.remove(w)
 
     try:
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 388414a4448..b51de0b7c0e 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -193,8 +193,8 @@ def g():
     result = yield future
     assert result == 123
 
-    yield a._close()
-    yield b._close()
+    yield a.close()
+    yield b.close()
     aa.close_rpc()
     bb.close_rpc()
     assert not os.path.exists(os.path.join(a.local_dir, "foobar.py"))
@@ -997,7 +997,7 @@ def test_start_services(s):
     yield w._start()
 
     assert w.services["bokeh"].server.port == 1234
-    yield w._close()
+    yield w.close()
 
 
 @gen_test()
@@ -1007,7 +1007,7 @@ def test_scheduler_file():
         s.start(8009)
         w = yield Worker(scheduler_file=fn)
         assert set(s.workers) == {w.address}
-        yield w._close()
+        yield w.close()
         s.stop()
 
 
@@ -1186,7 +1186,7 @@ def test_avoid_memory_monitor_if_zero_limit(c, s):
 
     yield c.submit(inc, 2)  # worker doesn't pause
 
-    yield worker._close()
+    yield worker.close()
 
 
 @gen_cluster(
@@ -1225,7 +1225,7 @@ def test_scheduler_address_config(c, s):
     with dask.config.set({"scheduler-address": s.address}):
         worker = yield Worker(loop=s.loop)
         assert worker.scheduler.address == s.address
-    yield worker._close()
+    yield worker.close()
 
 
 @slow
@@ -1321,7 +1321,7 @@ def test_startup2():
     worker = yield Worker(s.address, loop=s.loop)
     result = yield c.run(test_import, workers=[worker.address])
     assert list(result.values()) == [False]
-    yield worker._close()
+    yield worker.close()
 
     # Add a preload function
     response = yield c.register_worker_callbacks(setup=mystartup)
@@ -1336,7 +1336,7 @@ def test_startup2():
     worker = yield Worker(s.address, loop=s.loop)
     result = yield c.run(test_import, workers=[worker.address])
     assert list(result.values()) == [True]
-    yield worker._close()
+    yield worker.close()
 
     # Register another preload function
     response = yield c.register_worker_callbacks(setup=mystartup2)
@@ -1353,7 +1353,7 @@ def test_startup2():
     assert list(result.values()) == [True]
     result = yield c.run(test_startup2, workers=[worker.address])
     assert list(result.values()) == [True]
-    yield worker._close()
+    yield worker.close()
 
     # Final exception test
     with pytest.raises(ZeroDivisionError):
@@ -1364,12 +1364,12 @@ def test_startup2():
 def test_data_types(s):
     w = yield Worker(s.address, data=dict)
     assert isinstance(w.data, dict)
-    yield w._close()
+    yield w.close()
 
     data = dict()
     w = yield Worker(s.address, data=data)
     assert w.data is data
-    yield w._close()
+    yield w.close()
 
     class Data(dict):
         def __init__(self, x, y):
@@ -1379,4 +1379,4 @@ def __init__(self, x, y):
     w = yield Worker(s.address, data=(Data, {"x": 123, "y": 456}))
     assert w.data.x == 123
     assert w.data.y == 456
-    yield w._close()
+    yield w.close()
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index ba5567b1a80..b0c0d2d48cc 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -112,7 +112,7 @@ def invalid_python_script(tmpdir_factory):
 def cleanup_global_workers():
     for w in _global_workers:
         w = w()
-        w._close(report=False, executor_wait=False)
+        w.close(report=False, executor_wait=False)
 
 
 @pytest.fixture
@@ -526,7 +526,7 @@ def run_nanny(q, scheduler_q, **kwargs):
             try:
                 loop.start()
             finally:
-                loop.run_sync(worker._close)
+                loop.run_sync(worker.close)
                 loop.close(all_fds=True)
 
 
@@ -869,7 +869,7 @@ def start_cluster(
     ):
         yield gen.sleep(0.01)
         if time() - start > 5:
-            yield [w._close(timeout=1) for w in workers]
+            yield [w.close(timeout=1) for w in workers]
             yield s.close(fast=True)
             raise Exception("Cluster creation timeout")
     raise gen.Return((s, workers))
@@ -882,7 +882,7 @@ def end_cluster(s, workers):
     @gen.coroutine
     def end_worker(w):
         with ignoring(TimeoutError, CommClosedError, EnvironmentError):
-            yield w._close(report=False)
+            yield w.close(report=False)
 
     yield [end_worker(w) for w in workers]
     yield s.close()  # wait until scheduler stops completely
@@ -1031,7 +1031,7 @@ def coro():
                 DequeHandler.clear_all_instances()
                 for w in _global_workers:
                     w = w()
-                    w._close(report=False, executor_wait=False)
+                    w.close(report=False, executor_wait=False)
                     if w.status == "running":
                         w.close()
                 del _global_workers[:]
diff --git a/distributed/worker.py b/distributed/worker.py
index 74ad395cc4d..8d81e2781a0 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -536,7 +536,7 @@ def __init__(
         }
 
         stream_handlers = {
-            "close": self._close,
+            "close": self.close,
             "compute-task": self.add_task,
             "release-task": partial(self.release_key, report=False),
             "delete-data": self.delete_data,
@@ -665,7 +665,7 @@ def _register_with_scheduler(self):
         logger.info("-" * 49)
         while True:
             if self.death_timeout and time() > start + self.death_timeout:
-                yield self._close(timeout=1)
+                yield self.close(timeout=1)
                 return
             if self.status in ("closed", "closing"):
                 raise gen.Return
@@ -775,7 +775,7 @@ def handle_scheduler(self, comm):
                 logger.info("Connection to scheduler broken.  Reconnecting...")
                 self.loop.add_callback(self._register_with_scheduler)
             else:
-                yield self._close(report=False)
+                yield self.close(report=False)
 
     def start_ipython(self, comm):
         """Start an IPython kernel
@@ -958,8 +958,12 @@ def __await__(self):
     def start(self, port=0):
         self.loop.add_callback(self._start, port)
 
+    def _close(self, *args, **kwargs):
+        warnings.warn("Worker._close has moved to Worker.close")
+        return self.close(*args, **kwargs)
+
     @gen.coroutine
-    def _close(self, report=True, timeout=10, nanny=True, executor_wait=True):
+    def close(self, report=True, timeout=10, nanny=True, executor_wait=True):
         with log_errors():
             if self.status in ("closed", "closing"):
                 return
@@ -1015,7 +1019,7 @@ def _close(self, report=True, timeout=10, nanny=True, executor_wait=True):
             self._remove_from_global_workers()
 
             self.status = "closed"
-            yield self.close()
+            yield ServerNode.close(self)
 
             setproctitle("dask-worker [closed]")
 
@@ -1031,7 +1035,7 @@ def _remove_from_global_workers(self):
 
     @gen.coroutine
     def terminate(self, comm, report=True):
-        yield self._close(report=report)
+        yield self.close(report=report)
         raise Return("OK")
 
     @gen.coroutine

From 5c43091c2f61d25652d681898b7aade96e9ee811 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 3 May 2019 19:28:07 -0700
Subject: [PATCH 0258/1550] Use an LRU cache for deserialized functions (#2623)

Fixes https://github.com/dask/distributed/issues/2621
---
 distributed/worker.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/distributed/worker.py b/distributed/worker.py
index 8d81e2781a0..4cf0585b1de 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -3021,7 +3021,13 @@ def execute_task(task):
         return task
 
 
-cache = dict()
+try:
+    # a 10 MB cache of deserialized functions and their bytes
+    from zict import LRU
+
+    cache = LRU(10000000, dict(), weight=lambda k, v: len(v))
+except ImportError:
+    cache = dict()
 
 
 def dumps_function(func):

From 528c59b2b660e218c116823ad30671846cfa2530 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sat, 4 May 2019 08:33:50 -0700
Subject: [PATCH 0259/1550] Avoid deprecation warnings (#2653)

---
 distributed/client.py                 | 3 ++-
 distributed/deploy/local.py           | 4 +++-
 distributed/tests/py3_test_asyncio.py | 6 +++---
 distributed/tests/py3_test_pubsub.py  | 2 +-
 4 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 84728c62fb7..fef8b12f1fc 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -2391,7 +2391,8 @@ def run_coroutine(self, function, *args, **kwargs):
         warnings.warn(
             "This method has been deprecated. "
             "Instead use Client.run which detects async functions "
-            "automatically"
+            "automatically",
+            stacklevel=2,
         )
         return self.run(function, *args, **kwargs)
 
diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index 68a47c85d48..4cf67af150e 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -290,7 +290,9 @@ def _start(self, ip=None, n_workers=0):
     @gen.coroutine
     def _start_worker(self, death_timeout=60, **kwargs):
         if self.status and self.status.startswith("clos"):
-            warnings.warn("Tried to start a worker while status=='%s'" % self.status)
+            warnings.warn(
+                "Tried to start a worker while status=='%s'" % self.status, stacklevel=2
+            )
             return
 
         if self.processes:
diff --git a/distributed/tests/py3_test_asyncio.py b/distributed/tests/py3_test_asyncio.py
index 90e20268617..3754b282813 100644
--- a/distributed/tests/py3_test_asyncio.py
+++ b/distributed/tests/py3_test_asyncio.py
@@ -299,15 +299,15 @@ async def aiothrows(x, delay=0.02):
         raise RuntimeError("hello")
 
     async with AioClient(processes=False) as c:
-        results = await c.run_coroutine(aioinc, 1, delay=0.05)
+        results = await c.run(aioinc, 1, delay=0.05)
         assert len(results) > 0
         assert [value == 2 for value in results.values()]
 
-        results = await c.run_coroutine(aioinc, 1, workers=[])
+        results = await c.run(aioinc, 1, workers=[])
         assert results == {}
 
         with pytest.raises(RuntimeError) as exc_info:
-            await c.run_coroutine(aiothrows, 1)
+            await c.run(aiothrows, 1)
         assert "hello" in str(exc_info)
 
 
diff --git a/distributed/tests/py3_test_pubsub.py b/distributed/tests/py3_test_pubsub.py
index 172c8734819..0cedbb3bd31 100644
--- a/distributed/tests/py3_test_pubsub.py
+++ b/distributed/tests/py3_test_pubsub.py
@@ -22,7 +22,7 @@ def f(_):
         sub = Sub("a")
         return list(toolz.take(5, sub))
 
-    c.run_coroutine(publish, workers=[a.address])
+    c.run(publish, workers=[a.address])
 
     tasks = [c.submit(f, i) for i in range(4)]
     results = yield c.gather(tasks)

From ddaf73bea0cb18aedd2025b28db6247cc984aaba Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sat, 4 May 2019 10:00:54 -0700
Subject: [PATCH 0260/1550] Add idle timeout to scheduler (#2652)

Schedulers that haven't been touched in a while can choose to shut
themselves down.  This is useful as a stop-gap to clean up costly
forgotten resources.

* Avoid allocating works unnecessarily in LocalCluster tests
---
 distributed/deploy/tests/test_local.py | 27 ++++++++++++++++++-------
 distributed/distributed.yaml           |  1 +
 distributed/scheduler.py               | 28 ++++++++++++++++++++++++++
 distributed/tests/test_scheduler.py    | 16 +++++++++++++++
 4 files changed, 65 insertions(+), 7 deletions(-)

diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index ab378ba0e8f..6e1e71e83b2 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -52,7 +52,7 @@ def test_simple(loop):
 
 
 def test_local_cluster_supports_blocked_handlers(loop):
-    with LocalCluster(blocked_handlers=["run_function"], loop=loop) as c:
+    with LocalCluster(blocked_handlers=["run_function"], n_workers=0, loop=loop) as c:
         with Client(c) as client:
             with pytest.raises(ValueError) as exc:
                 client.run_on_scheduler(lambda x: x, 42)
@@ -309,11 +309,11 @@ def test_cleanup():
 
 def test_repeated():
     with LocalCluster(
-        scheduler_port=8448, silence_logs=False, dashboard_address=None
+        0, scheduler_port=8448, silence_logs=False, dashboard_address=None
     ) as c:
         pass
     with LocalCluster(
-        scheduler_port=8448, silence_logs=False, dashboard_address=None
+        0, scheduler_port=8448, silence_logs=False, dashboard_address=None
     ) as c:
         pass
 
@@ -323,6 +323,7 @@ def test_bokeh(loop, processes):
     pytest.importorskip("bokeh")
     requests = pytest.importorskip("requests")
     with LocalCluster(
+        n_workers=0,
         scheduler_port=0,
         silence_logs=False,
         loop=loop,
@@ -405,14 +406,19 @@ def test_silent_startup():
 
 def test_only_local_access(loop):
     with LocalCluster(
-        scheduler_port=0, silence_logs=False, dashboard_address=None, loop=loop
+        0, scheduler_port=0, silence_logs=False, dashboard_address=None, loop=loop
     ) as c:
         sync(loop, assert_can_connect_locally_4, c.scheduler.port)
 
 
 def test_remote_access(loop):
     with LocalCluster(
-        scheduler_port=0, silence_logs=False, dashboard_address=None, ip="", loop=loop
+        0,
+        scheduler_port=0,
+        silence_logs=False,
+        dashboard_address=None,
+        ip="",
+        loop=loop,
     ) as c:
         sync(loop, assert_can_connect_from_everywhere_4_6, c.scheduler.port)
 
@@ -463,6 +469,7 @@ def test_death_timeout_raises(loop):
 def test_bokeh_kwargs(loop):
     pytest.importorskip("bokeh")
     with LocalCluster(
+        n_workers=0,
         scheduler_port=0,
         silence_logs=False,
         loop=loop,
@@ -496,6 +503,7 @@ def test_logging():
 def test_ipywidgets(loop):
     ipywidgets = pytest.importorskip("ipywidgets")
     with LocalCluster(
+        n_workers=0,
         scheduler_port=0,
         silence_logs=False,
         loop=loop,
@@ -607,6 +615,7 @@ def test_local_tls(loop):
 
     security = tls_only_security()
     with LocalCluster(
+        n_workers=0,
         scheduler_port=8786,
         silence_logs=False,
         security=security,
@@ -730,7 +739,9 @@ def test_protocol_inproc(loop):
 
 
 def test_protocol_tcp(loop):
-    with LocalCluster(protocol="tcp", loop=loop, processes=False) as cluster:
+    with LocalCluster(
+        protocol="tcp", loop=loop, n_workers=0, processes=False
+    ) as cluster:
         assert cluster.scheduler.address.startswith("tcp://")
 
 
@@ -738,7 +749,9 @@ def test_protocol_tcp(loop):
     not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
 )
 def test_protocol_ip(loop):
-    with LocalCluster(ip="tcp://127.0.0.2", loop=loop, processes=False) as cluster:
+    with LocalCluster(
+        ip="tcp://127.0.0.2", loop=loop, n_workers=0, processes=False
+    ) as cluster:
         assert cluster.scheduler.address.startswith("tcp://127.0.0.2")
 
 
diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index d625a103fe8..3ae9b7ee690 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -16,6 +16,7 @@ distributed:
     # Number of seconds to wait until workers or clients are removed from the events log
     # after they have been removed from the scheduler
     events-cleanup-delay: 1h
+    idle-timeout: null      # Shut down after this duration, like "1h" or "30 minutes"
     transition-log-length: 100000
     work-stealing: True     # workers should steal tasks from each other
     worker-ttl: null        # like '60s'. Time to live for workers.  They must heartbeat faster than this
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index af87960f6b6..f9cbbb8c783 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -816,6 +816,7 @@ def __init__(
         scheduler_file=None,
         security=None,
         worker_ttl=None,
+        idle_timeout=None,
         **kwargs
     ):
 
@@ -836,6 +837,14 @@ def __init__(
         self.scheduler_file = scheduler_file
         worker_ttl = worker_ttl or dask.config.get("distributed.scheduler.worker-ttl")
         self.worker_ttl = parse_timedelta(worker_ttl) if worker_ttl else None
+        idle_timeout = idle_timeout or dask.config.get(
+            "distributed.scheduler.idle-timeout"
+        )
+        if idle_timeout:
+            self.idle_timeout = parse_timedelta(idle_timeout)
+        else:
+            self.idle_timeout = None
+        self.time_started = time()
 
         self.security = security or Security()
         assert isinstance(self.security, Security)
@@ -1054,6 +1063,10 @@ def __init__(
             pc = PeriodicCallback(self.check_worker_ttl, self.worker_ttl, io_loop=loop)
             self.periodic_callbacks["worker-ttl"] = pc
 
+        if self.idle_timeout:
+            pc = PeriodicCallback(self.check_idle, self.idle_timeout / 4, io_loop=loop)
+            self.periodic_callbacks["idle-timeout"] = pc
+
         if extensions is None:
             extensions = DEFAULT_EXTENSIONS
         for ext in extensions:
@@ -4651,6 +4664,21 @@ def check_worker_ttl(self):
                 )
                 self.remove_worker(address=ws.address)
 
+    def check_idle(self):
+        if any(ws.processing for ws in self.workers.values()):
+            return
+        if self.unrunnable:
+            return
+
+        if not self.transition_log:
+            close = time() > self.time_started + self.idle_timeout
+        else:
+            last_task = self.transition_log[-1][-1]
+            close = time() > last_task + self.idle_timeout
+
+        if close:
+            self.loop.add_callback(self.close)
+
 
 def decide_worker(ts, all_workers, valid_workers, objective):
     """
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 8280c3cd120..f4c13bfc852 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -1499,6 +1499,22 @@ def qux(x):
     yield f
 
 
+@gen_cluster(client=True, config={"distributed.scheduler.idle-timeout": "200ms"})
+def test_idle_timeout(c, s, a, b):
+    future = c.submit(slowinc, 1)
+    yield future
+
+    assert s.status != "closed"
+
+    start = time()
+    while s.status != "closed":
+        yield gen.sleep(0.01)
+    assert time() < start + 3
+
+    assert a.status == "closed"
+    assert b.status == "closed"
+
+
 @gen_cluster()
 def test_workerstate_clean(s, a, b):
     ws = s.workers[a.address].clean()

From d42173be416326aac4a48e73d4d169674a41c6e8 Mon Sep 17 00:00:00 2001
From: Brett Randall <javabrett@gmail.com>
Date: Sun, 5 May 2019 23:18:05 +1000
Subject: [PATCH 0261/1550] Fixed comment regarding keeping existing level if
 less verbose (#2655)

This behaviour changed in commit 6cc529979 .
---
 distributed/utils.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/distributed/utils.py b/distributed/utils.py
index 5259e567358..28ce2364190 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -727,8 +727,7 @@ def log_errors(pdb=False):
 
 def silence_logging(level, root="distributed"):
     """
-    Force all existing loggers below *root* to the given level at least
-    (or keep the existing level if less verbose).
+    Change all StreamHandlers for the given logger to the given level
     """
     if isinstance(level, str):
         level = getattr(logging, level.upper())

From 09b959a5667a51a2dc073510c784e01d44827457 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sun, 5 May 2019 11:42:04 -0700
Subject: [PATCH 0262/1550] Check direct_to_workers before using get_worker in
 Client (#2656)

Otherwise we would ignore direct_to_workers=True when there wasn't a
local worker
---
 distributed/client.py            | 10 +++++-----
 distributed/tests/test_client.py |  8 ++++++++
 2 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index fef8b12f1fc..97728929f33 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -591,7 +591,7 @@ def __init__(
         serializers=None,
         deserializers=None,
         extensions=DEFAULT_EXTENSIONS,
-        direct_to_workers=False,
+        direct_to_workers=None,
         **kwargs
     ):
         if timeout == no_default:
@@ -1607,6 +1607,8 @@ def _gather(self, futures, errors="raise", direct=None, local_worker=None):
         bad_data = dict()
         data = {}
 
+        if direct is None:
+            direct = self.direct_to_workers
         if direct is None:
             try:
                 w = get_worker()
@@ -1615,8 +1617,6 @@ def _gather(self, futures, errors="raise", direct=None, local_worker=None):
             else:
                 if w.scheduler.address == self.scheduler.address:
                     direct = True
-        if direct is None:
-            direct = self.direct_to_workers
 
         @gen.coroutine
         def wait(k):
@@ -1866,6 +1866,8 @@ def _scatter(
 
         types = valmap(type, data)
 
+        if direct is None:
+            direct = self.direct_to_workers
         if direct is None:
             try:
                 w = get_worker()
@@ -1874,8 +1876,6 @@ def _scatter(
             else:
                 if w.scheduler.address == self.scheduler.address:
                     direct = True
-        if direct is None:
-            direct = self.direct_to_workers
 
         if local_worker:  # running within task
             local_worker.update_data(data=data, report=False)
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 6f1fb5c8a03..429b92c5194 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -5707,5 +5707,13 @@ def test_get_mix_futures_and_SubgraphCallable_dask_dataframe(c, s, a, b):
     assert result.equals(df.astype("f8"))
 
 
+def test_direct_to_workers(s, loop):
+    with Client(s["address"], loop=loop, direct_to_workers=True) as client:
+        future = client.scatter(1)
+        future.result()
+        resp = client.run_on_scheduler(lambda dask_scheduler: dask_scheduler.events)
+        assert "gather" not in str(resp)
+
+
 if sys.version_info >= (3, 5):
     from distributed.tests.py3_test_client import *  # noqa F401

From 7a76a77428e64df486f34666bf2eb8869fecbce4 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 6 May 2019 09:13:19 -0500
Subject: [PATCH 0263/1550] Allow scheduler to politely close workers as part
 of shutdown (#2651)

---
 distributed/cli/dask_worker.py            |  2 +-
 distributed/core.py                       |  5 +++
 distributed/deploy/local.py               | 14 +++----
 distributed/nanny.py                      |  2 +-
 distributed/protocol/tests/test_pickle.py | 10 +++--
 distributed/scheduler.py                  | 11 ++++-
 distributed/tests/test_actor.py           |  2 +-
 distributed/tests/test_client.py          | 22 ++++------
 distributed/tests/test_failed_workers.py  |  8 ++--
 distributed/tests/test_resources.py       |  6 +--
 distributed/tests/test_scheduler.py       |  9 +++-
 distributed/tests/test_steal.py           |  6 +--
 distributed/tests/test_worker.py          | 50 ++++++++++++-----------
 distributed/tests/test_worker_client.py   |  2 +-
 distributed/worker.py                     | 14 +++----
 15 files changed, 92 insertions(+), 71 deletions(-)

diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
index 73cb9970924..a0bc801a960 100755
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -369,7 +369,7 @@ def del_pid_file():
     def close_all():
         # Unregister all workers from scheduler
         if nanny:
-            yield [n._close(timeout=2) for n in nannies]
+            yield [n.close(timeout=2) for n in nannies]
 
     def on_signal(signum):
         logger.info("Exiting on signal %d", signum)
diff --git a/distributed/core.py b/distributed/core.py
index 3cf3f9b5bb2..9b1d408a038 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -490,6 +490,11 @@ def handle_stream(self, comm, extra=None, every_cycle=[]):
     @gen.coroutine
     def close(self):
         self.listener.stop()
+        for i in range(20):  # let comms close naturally for a second
+            if not self._comms:
+                break
+            else:
+                yield gen.sleep(0.05)
         for comm in self._comms:
             comm.close()
         for cb in self._ongoing_coroutines:
diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index 4cf67af150e..ad8b36f6dd7 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -362,7 +362,11 @@ def _close(self, timeout="2s"):
             return
         self.status = "closing"
 
-        self.scheduler.clear_task_state()
+        with ignoring(gen.TimeoutError, CommClosedError, OSError):
+            yield gen.with_timeout(
+                timedelta(seconds=parse_timedelta(timeout)),
+                self.scheduler.close(close_workers=True),
+            )
 
         with ignoring(gen.TimeoutError):
             yield gen.with_timeout(
@@ -370,13 +374,7 @@ def _close(self, timeout="2s"):
                 All([self._stop_worker(w) for w in self.workers]),
             )
         del self.workers[:]
-
-        try:
-            with ignoring(gen.TimeoutError, CommClosedError, OSError):
-                yield self.scheduler.close(fast=True)
-            del self.workers[:]
-        finally:
-            self.status = "closed"
+        self.status = "closed"
 
     def close(self, timeout=20):
         """ Close the cluster """
diff --git a/distributed/nanny.py b/distributed/nanny.py
index 4b81bec4646..60e83e86da7 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -351,7 +351,7 @@ def pid(self):
         return self.process and self.process.pid
 
     def _close(self, *args, **kwargs):
-        warnings.warn("Worker._close has moved to Worker.close")
+        warnings.warn("Worker._close has moved to Worker.close", stacklevel=2)
         return self.close(*args, **kwargs)
 
     @gen.coroutine
diff --git a/distributed/protocol/tests/test_pickle.py b/distributed/protocol/tests/test_pickle.py
index 70d9cdaff22..0ba776e2758 100644
--- a/distributed/protocol/tests/test_pickle.py
+++ b/distributed/protocol/tests/test_pickle.py
@@ -1,10 +1,11 @@
-from distributed.protocol.pickle import dumps, loads
+from functools import partial
+import gc
+from operator import add
+import weakref
 
 import pytest
-import weakref
 
-from operator import add
-from functools import partial
+from distributed.protocol.pickle import dumps, loads
 
 
 def test_pickle_data():
@@ -42,5 +43,6 @@ def funcs():
         wr2 = weakref.ref(func2)
         assert func2(1) == func(1)
         del func, func2
+        gc.collect()
         assert wr() is None
         assert wr2() is None
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index f9cbbb8c783..cc6f39ff396 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1234,7 +1234,7 @@ def finished(self):
             yield All(self.coroutines)
 
     @gen.coroutine
-    def close(self, comm=None, fast=False):
+    def close(self, comm=None, fast=False, close_workers=False):
         """ Send cleanup signal to all coroutines then wait until finished
 
         See Also
@@ -1248,6 +1248,15 @@ def close(self, comm=None, fast=False):
         logger.info("Scheduler closing...")
         setproctitle("dask-scheduler [closing]")
 
+        if close_workers:
+            for worker in self.workers:
+                self.worker_send(worker, {"op": "close"})
+            for i in range(20):  # wait a second for send signals to clear
+                if self.workers:
+                    yield gen.sleep(0.05)
+                else:
+                    break
+
         for pc in self.periodic_callbacks.values():
             pc.stop()
         self.periodic_callbacks.clear()
diff --git a/distributed/tests/test_actor.py b/distributed/tests/test_actor.py
index fba0f50cbfe..ec2636ccd50 100644
--- a/distributed/tests/test_actor.py
+++ b/distributed/tests/test_actor.py
@@ -261,7 +261,7 @@ def test_failed_worker(c, s, a, b):
     yield wait(future)
     counter = yield future
 
-    yield a._close()
+    yield a.close()
 
     with pytest.raises(Exception) as info:
         yield counter.increment()
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 429b92c5194..8582c2abc83 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -3891,7 +3891,7 @@ def test_lose_scattered_data(c, s, a, b):
 
 @gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 3)
 def test_partially_lose_scattered_data(e, s, a, b, c):
-    [x] = yield e.scatter([1], workers=a.address)
+    x = yield e.scatter(1, workers=a.address)
     yield e.replicate(x, n=2)
 
     yield a.close()
@@ -4713,20 +4713,16 @@ def test_quiet_client_close(loop):
             ), line
 
 
+@slow
 def test_quiet_client_close_when_cluster_is_closed_before_client(loop):
-    n_attempts = 5
-    # Trying a few times to reduce the flakiness of the test. Without the bug
-    # fix in #2477 and with 5 attempts, this test passes by chance in about 10%
-    # of the cases.
-    for _ in range(n_attempts):
-        with captured_logger(logging.getLogger("tornado.application")) as logger:
-            cluster = LocalCluster(loop=loop)
-            client = Client(cluster, loop=loop)
-            cluster.close()
-            client.close()
+    with captured_logger(logging.getLogger("tornado.application")) as logger:
+        cluster = LocalCluster(loop=loop, n_workers=1)
+        client = Client(cluster, loop=loop)
+        cluster.close()
+        client.close()
 
-        out = logger.getvalue()
-        assert "CancelledError" not in out
+    out = logger.getvalue()
+    assert "CancelledError" not in out
 
 
 @gen_cluster()
diff --git a/distributed/tests/test_failed_workers.py b/distributed/tests/test_failed_workers.py
index 5bb1c61fb5b..0772ea52c32 100644
--- a/distributed/tests/test_failed_workers.py
+++ b/distributed/tests/test_failed_workers.py
@@ -53,14 +53,14 @@ def test_submit_after_failed_worker_async(c, s, a, b):
     result = yield total
     assert result == sum(map(inc, range(10)))
 
-    yield n._close()
+    yield n.close()
 
 
 @gen_cluster(client=True, timeout=60)
 def test_submit_after_failed_worker(c, s, a, b):
     L = c.map(inc, range(10))
     yield wait(L)
-    yield a._close()
+    yield a.close()
 
     total = c.submit(sum, L)
     result = yield total
@@ -353,7 +353,7 @@ def test_broken_worker_during_computation(c, s, a, b):
     assert isinstance(result, int)
     assert result == expected_result
 
-    yield n._close()
+    yield n.close()
 
 
 @gen_cluster(client=True, Worker=Nanny, timeout=60)
@@ -403,7 +403,7 @@ def test_worker_who_has_clears_after_failed_connection(c, s, a, b):
     assert not a.has_what.get(n_worker_address)
     assert not any(n_worker_address in s for s in a.who_has.values())
 
-    yield n._close()
+    yield n.close()
 
 
 @slow
diff --git a/distributed/tests/test_resources.py b/distributed/tests/test_resources.py
index 35f5e160969..429bbc2bb56 100644
--- a/distributed/tests/test_resources.py
+++ b/distributed/tests/test_resources.py
@@ -27,12 +27,12 @@ def test_resources(c, s):
     assert s.resources == {"GPU": {a.address: 2, b.address: 1}, "DB": {b.address: 1}}
     assert s.worker_resources == {a.address: {"GPU": 2}, b.address: {"GPU": 1, "DB": 1}}
 
-    yield b._close()
+    yield b.close()
 
     assert s.resources == {"GPU": {a.address: 2}, "DB": {}}
     assert s.worker_resources == {a.address: {"GPU": 2}}
 
-    yield a._close()
+    yield a.close()
 
 
 @gen_cluster(
@@ -60,7 +60,7 @@ def test_resource_submit(c, s, a, b):
     yield wait(z)
     assert z.key in d.data
 
-    yield d._close()
+    yield d.close()
 
 
 @gen_cluster(
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index f4c13bfc852..6a631a13498 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -543,8 +543,8 @@ def test_worker_name():
         w2 = yield Worker(s.ip, s.port, name="alice")
         yield w2.close()
 
-    yield s.close()
     yield w.close()
+    yield s.close()
 
 
 @gen_test()
@@ -1521,3 +1521,10 @@ def test_workerstate_clean(s, a, b):
     assert ws.address == a.address
     b = pickle.dumps(ws)
     assert len(b) < 1000
+
+
+@gen_cluster()
+def test_close_workers(s, a, b):
+    yield s.close(close_workers=True)
+    assert a.status == "closed"
+    assert b.status == "closed"
diff --git a/distributed/tests/test_steal.py b/distributed/tests/test_steal.py
index f93022e6d81..cb56fc0f263 100644
--- a/distributed/tests/test_steal.py
+++ b/distributed/tests/test_steal.py
@@ -182,7 +182,7 @@ def test_new_worker_steals(c, s, a):
 
     assert b.data
 
-    yield b._close()
+    yield b.close()
 
 
 @gen_cluster(client=True, timeout=20)
@@ -287,7 +287,7 @@ def test_steal_resource_restrictions(c, s, a):
     assert len(b.task_state) > 0
     assert len(a.task_state) < 101
 
-    yield b._close()
+    yield b.close()
 
 
 @gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 5, timeout=20)
@@ -550,7 +550,7 @@ def test_steal_twice(c, s, a, b):
     assert max(map(len, has_what.values())) < 30
 
     yield c._close()
-    yield [w._close() for w in workers]
+    yield [w.close() for w in workers]
 
 
 @gen_cluster(client=True)
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index b51de0b7c0e..07864ab4b64 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -174,12 +174,11 @@ def test_upload_file(c, s, a, b):
     assert not os.path.exists(os.path.join(b.local_dir, "foobar.py"))
     assert a.local_dir != b.local_dir
 
-    aa = rpc(a.address)
-    bb = rpc(b.address)
-    yield [
-        aa.upload_file(filename="foobar.py", data=b"x = 123"),
-        bb.upload_file(filename="foobar.py", data="x = 123"),
-    ]
+    with rpc(a.address) as aa, rpc(b.address) as bb:
+        yield [
+            aa.upload_file(filename="foobar.py", data=b"x = 123"),
+            bb.upload_file(filename="foobar.py", data="x = 123"),
+        ]
 
     assert os.path.exists(os.path.join(a.local_dir, "foobar.py"))
     assert os.path.exists(os.path.join(b.local_dir, "foobar.py"))
@@ -193,10 +192,8 @@ def g():
     result = yield future
     assert result == 123
 
-    yield a.close()
-    yield b.close()
-    aa.close_rpc()
-    bb.close_rpc()
+    yield c.close()
+    yield s.close(close_workers=True)
     assert not os.path.exists(os.path.join(a.local_dir, "foobar.py"))
 
 
@@ -251,8 +248,10 @@ def g(x):
     result = yield future
     assert result == 10 + 1
 
-    yield a._close()
-    yield b._close()
+    yield c.close()
+    yield s.close()
+    yield a.close()
+    yield b.close()
     assert not os.path.exists(os.path.join(a.local_dir, eggname))
 
 
@@ -278,8 +277,10 @@ def g(x):
     result = yield future
     assert result == 10 + 1
 
-    yield a._close()
-    yield b._close()
+    yield c.close()
+    yield s.close()
+    yield a.close()
+    yield b.close()
     assert not os.path.exists(os.path.join(a.local_dir, pyzname))
 
 
@@ -309,7 +310,7 @@ def test_worker_with_port_zero():
     assert isinstance(w.port, int)
     assert w.port > 1024
 
-    yield w._close()
+    yield w.close()
 
 
 @slow
@@ -392,7 +393,7 @@ def test_spill_to_disk(c, s):
     yield x
     assert set(w.data.fast) == {x.key, z.key}
     assert set(w.data.slow) == {y.key} or set(w.data.slow) == {x.key, y.key}
-    yield w._close()
+    yield w.close()
 
 
 @gen_cluster(client=True)
@@ -443,9 +444,12 @@ def test_Executor(c, s):
 
         assert e._threads  # had to do some work
 
-        yield w._close()
+        yield w.close()
 
 
+@pytest.mark.skip(
+    reason="Other tests leak memory, so process-level checks" "trigger immediately"
+)
 @gen_cluster(
     client=True,
     ncores=[("127.0.0.1", 1)],
@@ -932,8 +936,8 @@ def test_global_workers(s, a, b):
     n = len(_global_workers)
     w = _global_workers[-1]()
     assert w is a or w is b
-    yield a._close()
-    yield b._close()
+    yield a.close()
+    yield b.close()
     assert len(_global_workers) == n - 2
 
 
@@ -952,7 +956,7 @@ def test_worker_fds(s):
         yield gen.sleep(0.01)
         assert time() < start + 1
 
-    yield worker._close()
+    yield worker.close()
 
     start = time()
     while psutil.Process().num_fds() > start:
@@ -971,19 +975,19 @@ def test_service_hosts_match_worker(s):
     yield w._start("tcp://0.0.0.0")
     sock = first(w.services["bokeh"].server._http._sockets.values())
     assert sock.getsockname()[0] in ("::", "0.0.0.0")
-    yield w._close()
+    yield w.close()
 
     w = Worker(s.address, services={("bokeh", ":0"): BokehWorker})
     yield w._start("tcp://127.0.0.1")
     sock = first(w.services["bokeh"].server._http._sockets.values())
     assert sock.getsockname()[0] in ("::", "0.0.0.0")
-    yield w._close()
+    yield w.close()
 
     w = Worker(s.address, services={("bokeh", 0): BokehWorker})
     yield w._start("tcp://127.0.0.1")
     sock = first(w.services["bokeh"].server._http._sockets.values())
     assert sock.getsockname()[0] == "127.0.0.1"
-    yield w._close()
+    yield w.close()
 
 
 @gen_cluster(ncores=[])
diff --git a/distributed/tests/test_worker_client.py b/distributed/tests/test_worker_client.py
index b0dd338153f..2d4632b0b54 100644
--- a/distributed/tests/test_worker_client.py
+++ b/distributed/tests/test_worker_client.py
@@ -234,7 +234,7 @@ def func(x):
         return
 
     yield wait(c.map(func, range(10)))
-    yield a._close()
+    yield a.close()
     assert c.status == "running"
 
 
diff --git a/distributed/worker.py b/distributed/worker.py
index 4cf0585b1de..10258d5c28f 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -959,7 +959,7 @@ def start(self, port=0):
         self.loop.add_callback(self._start, port)
 
     def _close(self, *args, **kwargs):
-        warnings.warn("Worker._close has moved to Worker.close")
+        warnings.warn("Worker._close has moved to Worker.close", stacklevel=2)
         return self.close(*args, **kwargs)
 
     @gen.coroutine
@@ -999,6 +999,12 @@ def close(self, report=True, timeout=10, nanny=True, executor_wait=True):
             for k, v in self.services.items():
                 v.stop()
 
+            if self.batched_stream and not self.batched_stream.comm.closed():
+                self.batched_stream.send({"op": "close-stream"})
+
+            if self.batched_stream:
+                self.batched_stream.close()
+
             if nanny and "nanny" in self.service_ports:
                 nanny_address = "%s%s:%d" % (
                     self.listener.prefix,
@@ -1008,12 +1014,6 @@ def close(self, report=True, timeout=10, nanny=True, executor_wait=True):
                 with self.rpc(nanny_address) as r:
                     yield r.terminate()
 
-            if self.batched_stream and not self.batched_stream.comm.closed():
-                self.batched_stream.send({"op": "close-stream"})
-
-            if self.batched_stream:
-                self.batched_stream.close()
-
             self.rpc.close()
             self._closed.set()
             self._remove_from_global_workers()

From e5d2488310ef52ccfdd2f9b70d9ae20b89874eac Mon Sep 17 00:00:00 2001
From: "K.-Michael Aye" <michaelaye@users.noreply.github.com>
Date: Tue, 7 May 2019 07:24:22 -0600
Subject: [PATCH 0264/1550] DOC: Clean up reference to cluster object (#2664)

The current docs would overwrite the cluster objects, so it needs to get its own variable name.
As the Client is usually named `c`, I changed the cluster object to the name `cluster`
---
 distributed/deploy/local.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index ad8b36f6dd7..3431210a645 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -82,19 +82,19 @@ class LocalCluster(Cluster):
 
     Examples
     --------
-    >>> c = LocalCluster()  # Create a local cluster with as many workers as cores  # doctest: +SKIP
-    >>> c  # doctest: +SKIP
+    >>> cluster = LocalCluster()  # Create a local cluster with as many workers as cores  # doctest: +SKIP
+    >>> cluster  # doctest: +SKIP
     LocalCluster("127.0.0.1:8786", workers=8, ncores=8)
 
-    >>> c = Client(c)  # connect to local cluster  # doctest: +SKIP
+    >>> c = Client(cluster)  # connect to local cluster  # doctest: +SKIP
 
     Add a new worker to the cluster
 
-    >>> w = c.start_worker(ncores=2)  # doctest: +SKIP
+    >>> w = cluster.start_worker(ncores=2)  # doctest: +SKIP
 
     Shut down the extra worker
 
-    >>> c.stop_worker(w)  # doctest: +SKIP
+    >>> cluster.stop_worker(w)  # doctest: +SKIP
 
     Pass extra keyword arguments to Bokeh
 

From a61df1f54a7ae38c26d8d40de4e1e944067b8dea Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Tue, 7 May 2019 09:54:56 -0500
Subject: [PATCH 0265/1550] Add waiting task count to progress title bar
 (#2663)

---
 distributed/bokeh/scheduler.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/distributed/bokeh/scheduler.py b/distributed/bokeh/scheduler.py
index a6f07351730..2dd60f0690f 100644
--- a/distributed/bokeh/scheduler.py
+++ b/distributed/bokeh/scheduler.py
@@ -1049,7 +1049,7 @@ def __init__(self, scheduler, **kwargs):
     def update(self):
         with log_errors():
             state = {"all": valmap(len, self.plugin.all), "nbytes": self.plugin.nbytes}
-            for k in ["memory", "erred", "released", "processing"]:
+            for k in ["memory", "erred", "released", "processing", "waiting"]:
                 state[k] = valmap(len, self.plugin.state[k])
             if not state["all"] and not len(self.source.data["all"]):
                 return
@@ -1060,7 +1060,7 @@ def update(self):
 
             totals = {
                 k: sum(state[k].values())
-                for k in ["all", "memory", "erred", "released"]
+                for k in ["all", "memory", "erred", "released", "waiting"]
             }
             totals["processing"] = totals["all"] - sum(
                 v for k, v in totals.items() if k != "all"
@@ -1069,6 +1069,7 @@ def update(self):
             self.root.title.text = (
                 "Progress -- total: %(all)s, "
                 "in-memory: %(memory)s, processing: %(processing)s, "
+                "waiting: %(waiting)s, "
                 "erred: %(erred)s" % totals
             )
 

From f75ceb90ff294fd26428d3c8c2f54d6b524e0c0a Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 7 May 2019 18:29:08 -0500
Subject: [PATCH 0266/1550] Add Type Attribute to TaskState (#2657)

---
 distributed/bokeh/templates/task.html         |  6 ++++
 .../bokeh/tests/test_scheduler_bokeh_html.py  | 25 +++++++++++++-
 distributed/protocol/serialize.py             | 14 +-------
 distributed/scheduler.py                      | 33 ++++++++++++++++---
 distributed/tests/test_collections.py         |  5 +--
 distributed/tests/test_scheduler.py           |  8 +++++
 distributed/utils.py                          | 15 +++++++++
 distributed/worker.py                         | 10 ++++--
 8 files changed, 93 insertions(+), 23 deletions(-)

diff --git a/distributed/bokeh/templates/task.html b/distributed/bokeh/templates/task.html
index 9f3bb0f78f3..f396a4cba8f 100644
--- a/distributed/bokeh/templates/task.html
+++ b/distributed/bokeh/templates/task.html
@@ -19,6 +19,12 @@ <h1 class="title"> Task: {{ ts.key }} </h1>
               <td><a class="button is-primary" href="../call-stack/{{ url_escape(Task) }}.html">Call Stack</a></td>
           </tr>
           {% end %}
+          {% if ts.type %}
+          <tr>
+              <th> Type </th>
+              <td> {{ ts.type }} </td>
+          </tr>
+          {% end %}
           {% if ts.nbytes %}
           <tr>
               <th> Bytes </th>
diff --git a/distributed/bokeh/tests/test_scheduler_bokeh_html.py b/distributed/bokeh/tests/test_scheduler_bokeh_html.py
index d5ca1ee7f05..96fe3c2f5d2 100644
--- a/distributed/bokeh/tests/test_scheduler_bokeh_html.py
+++ b/distributed/bokeh/tests/test_scheduler_bokeh_html.py
@@ -11,7 +11,8 @@
 from tornado.escape import url_escape
 from tornado.httpclient import AsyncHTTPClient
 
-from distributed.utils_test import gen_cluster, slowinc
+from dask.sizeof import sizeof
+from distributed.utils_test import gen_cluster, slowinc, inc
 from distributed.bokeh.scheduler import BokehScheduler
 
 
@@ -105,3 +106,25 @@ def test_health(c, s, a, b):
 
     txt = response.body.decode("utf8")
     assert txt == "ok"
+
+
+@gen_cluster(client=True, scheduler_kwargs={"services": {("bokeh", 0): BokehScheduler}})
+def test_task_page(c, s, a, b):
+    future = c.submit(lambda x: x + 1, 1, workers=a.address)
+    x = c.submit(inc, 1)
+    yield future
+    http_client = AsyncHTTPClient()
+
+    "info/task/" + url_escape(future.key) + ".html",
+    response = yield http_client.fetch(
+        "http://localhost:%d/info/task/" % s.services["bokeh"].port
+        + url_escape(future.key)
+        + ".html"
+    )
+    assert response.code == 200
+    body = response.body.decode()
+
+    assert str(sizeof(1)) in body
+    assert "int" in body
+    assert a.address in body
+    assert "memory" in body
diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index f47ea7388af..26129f4e1c5 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -14,7 +14,7 @@
 
 from . import pickle
 from ..compatibility import PY2
-from ..utils import has_keyword
+from ..utils import has_keyword, typename
 from .compression import maybe_compress, decompress
 from .utils import (
     unpack_frames,
@@ -445,18 +445,6 @@ def register_serialization_lazy(toplevel, func):
     raise Exception("Serialization registration has changed. See documentation")
 
 
-def typename(typ):
-    """ Return name of type
-
-    Examples
-    --------
-    >>> from distributed import Scheduler
-    >>> typename(Scheduler)
-    'distributed.scheduler.Scheduler'
-    """
-    return typ.__module__ + "." + typ.__name__
-
-
 @partial(normalize_token.register, Serialized)
 def normalize_Serialized(o):
     return [o.header] + o.frames  # for dask.base.tokenize
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index cc6f39ff396..8442b6ddcea 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -453,6 +453,11 @@ class TaskState(object):
        of a finished task.  This number is used for diagnostics and to
        help prioritize work.
 
+    .. attribute:: type: str
+
+       The type of the object as a string.  Only present for tasks that have
+       been computed.
+
     .. attribute:: exception: object
 
        If this task failed executing, the exception object is stored here.
@@ -566,6 +571,7 @@ class TaskState(object):
         "suspicious",
         "retries",
         "nbytes",
+        "type",
     )
 
     def __init__(self, key, run_spec):
@@ -590,6 +596,7 @@ def __init__(self, key, run_spec):
         self.resource_restrictions = None
         self.loose_restrictions = False
         self.actor = None
+        self.type = None
 
     def get_nbytes(self):
         nbytes = self.nbytes
@@ -1382,6 +1389,7 @@ def add_worker(
         name=None,
         resolve_address=True,
         nbytes=None,
+        types=None,
         now=None,
         resources=None,
         host_info=None,
@@ -1460,7 +1468,11 @@ def add_worker(
                     ts = self.tasks.get(key)
                     if ts is not None and ts.state in ("processing", "waiting"):
                         recommendations = self.transition(
-                            key, "memory", worker=address, nbytes=nbytes[key]
+                            key,
+                            "memory",
+                            worker=address,
+                            nbytes=nbytes[key],
+                            typename=types[key],
                         )
                         self.transitions(recommendations)
 
@@ -3418,7 +3430,9 @@ def _remove_from_processing(self, ts, send_worker_msg=None):
             if send_worker_msg:
                 self.worker_send(w, send_worker_msg)
 
-    def _add_to_memory(self, ts, ws, recommendations, type=None, **kwargs):
+    def _add_to_memory(
+        self, ts, ws, recommendations, type=None, typename=None, **kwargs
+    ):
         """
         Add *ts* to the set of in-memory tasks.
         """
@@ -3454,6 +3468,7 @@ def _add_to_memory(self, ts, ws, recommendations, type=None, **kwargs):
             self.report(msg)
 
         ts.state = "memory"
+        ts.type = typename
 
         cs = self.clients["fire-and-forget"]
         if ts in cs.wants_what:
@@ -3676,7 +3691,14 @@ def transition_waiting_memory(self, key, nbytes=None, worker=None, **kwargs):
             raise
 
     def transition_processing_memory(
-        self, key, nbytes=None, type=None, worker=None, startstops=None, **kwargs
+        self,
+        key,
+        nbytes=None,
+        type=None,
+        typename=None,
+        worker=None,
+        startstops=None,
+        **kwargs
     ):
         try:
             ts = self.tasks[key]
@@ -3749,7 +3771,7 @@ def transition_processing_memory(
 
             self._remove_from_processing(ts)
 
-            self._add_to_memory(ts, ws, recommendations, type=type)
+            self._add_to_memory(ts, ws, recommendations, type=type, typename=typename)
 
             if self.validate:
                 assert not ts.processing_on
@@ -4801,6 +4823,9 @@ def validate_task_state(ts):
             str(ts),
             str(ts.who_has),
         )
+        if ts.run_spec:  # was computed
+            assert ts.type
+            assert isinstance(ts.type, str)
         assert not any(ts in dts.waiting_on for dts in ts.dependents)
         for ws in ts.who_has:
             assert ts in ws.has_what, (
diff --git a/distributed/tests/test_collections.py b/distributed/tests/test_collections.py
index f640d2d21e0..985b6f78fe9 100644
--- a/distributed/tests/test_collections.py
+++ b/distributed/tests/test_collections.py
@@ -73,10 +73,11 @@ def test_dataframes(c, s, a, b):
 def test__dask_array_collections(c, s, a, b):
     import dask.array as da
 
+    s.validate = False
     x_dsk = {("x", i, j): np.random.random((3, 3)) for i in range(3) for j in range(2)}
     y_dsk = {("y", i, j): np.random.random((3, 3)) for i in range(2) for j in range(3)}
-    x_futures = yield c._scatter(x_dsk)
-    y_futures = yield c._scatter(y_dsk)
+    x_futures = yield c.scatter(x_dsk)
+    y_futures = yield c.scatter(y_dsk)
 
     dt = np.random.random(0).dtype
     x_local = da.Array(x_dsk, "x", ((3, 3, 3), (3, 3)), dt)
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 6a631a13498..9224ed69030 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -1523,6 +1523,14 @@ def test_workerstate_clean(s, a, b):
     assert len(b) < 1000
 
 
+@gen_cluster(client=True)
+def test_result_type(c, s, a, b):
+    x = c.submit(lambda: 1)
+    yield x
+
+    assert "int" in s.tasks[x.key].type
+
+
 @gen_cluster()
 def test_close_workers(s, a, b):
     yield s.close(close_workers=True)
diff --git a/distributed/utils.py b/distributed/utils.py
index 28ce2364190..d6cc5ba62cf 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -1529,3 +1529,18 @@ def warn_on_duration(duration, msg):
     stop = time()
     if stop - start > parse_timedelta(duration):
         warnings.warn(msg, stacklevel=2)
+
+
+def typename(typ):
+    """ Return name of type
+
+    Examples
+    --------
+    >>> from distributed import Scheduler
+    >>> typename(Scheduler)
+    'distributed.scheduler.Scheduler'
+    """
+    try:
+        return typ.__module__ + "." + typ.__name__
+    except AttributeError:
+        return str(typ)
diff --git a/distributed/worker.py b/distributed/worker.py
index 10258d5c28f..915784edd88 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -44,6 +44,7 @@
 from .threadpoolexecutor import ThreadPoolExecutor, secede as tpe_secede
 from .utils import (
     funcname,
+    typename,
     get_ip,
     has_arg,
     _maybe_complex,
@@ -671,6 +672,7 @@ def _register_with_scheduler(self):
                 raise gen.Return
             try:
                 _start = time()
+                types = {k: typename(v) for k, v in self.data.items()}
                 comm = yield connect(
                     self.scheduler.address, connection_args=self.connection_args
                 )
@@ -685,6 +687,7 @@ def _register_with_scheduler(self):
                         ncores=self.ncores,
                         name=self.name,
                         nbytes=self.nbytes,
+                        types=types,
                         now=time(),
                         resources=self.total_resources,
                         memory_limit=self.memory_limit,
@@ -1721,18 +1724,19 @@ def send_task_state_to_scheduler(self, key):
             typ = self.types.get(key) or type(value)
             del value
             try:
-                typ = dumps_function(typ)
+                typ_serialized = dumps_function(typ)
             except PicklingError:
                 # Some types fail pickling (example: _thread.lock objects),
                 # send their name as a best effort.
-                typ = pickle.dumps(typ.__name__)
+                typ_serialized = pickle.dumps(typ.__name__)
             d = {
                 "op": "task-finished",
                 "status": "OK",
                 "key": key,
                 "nbytes": nbytes,
                 "thread": self.threads.get(key),
-                "type": typ,
+                "type": typ_serialized,
+                "typename": typename(typ),
             }
         elif key in self.exceptions:
             d = {

From a0d57710c9c836f8104b55c448eb2cec5f38e959 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Wed, 8 May 2019 14:12:01 -0500
Subject: [PATCH 0267/1550] bump version to 1.28.0

---
 docs/source/changelog.rst | 30 ++++++++++++++++++++++++++++--
 1 file changed, 28 insertions(+), 2 deletions(-)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index 34fe078fe18..1b584c02219 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,30 @@
 Changelog
 =========
 
+1.28.0 - 2019-05-08
+-------------------
+
+- Add Type Attribute to TaskState (:pr:`2657`) `Matthew Rocklin`_
+- Add waiting task count to progress title bar (:pr:`2663`) `James Bourbeau`_
+- DOC: Clean up reference to cluster object (:pr:`2664`) `K.-Michael Aye`_
+- Allow scheduler to politely close workers as part of shutdown (:pr:`2651`) `Matthew Rocklin`_
+- Check direct_to_workers before using get_worker in Client (:pr:`2656`) `Matthew Rocklin`_
+- Fixed comment regarding keeping existing level if less verbose (:pr:`2655`) `Brett Randall`_
+- Add idle timeout to scheduler (:pr:`2652`) `Matthew Rocklin`_
+- Avoid deprecation warnings (:pr:`2653`) `Matthew Rocklin`_
+- Use an LRU cache for deserialized functions (:pr:`2623`) `Matthew Rocklin`_
+- Rename Worker._close to Worker.close (:pr:`2650`) `Matthew Rocklin`_
+- Add Comm closed bookkeeping (:pr:`2648`) `Matthew Rocklin`_
+- Explain LocalCluster behavior in Client docstring (:pr:`2647`) `Matthew Rocklin`_
+- Add last worker into KilledWorker exception to help debug (:pr:`2610`) `@plbertrand`_
+- Set working worker class for dask-ssh (:pr:`2646`) `Martin Durant`_
+- Add as_completed methods to docs (:pr:`2642`) `Jim Crist`_
+- Add timeout to Client._reconnect (:pr:`2639`) `Jim Crist`_
+- Limit test_spill_by_default memory, reenable it (:pr:`2633`) `Peter Andreas Entschev`_
+- Use proper address in worker -> nanny comms (:pr:`2640`) `Jim Crist`_
+- Fix deserialization of bytes chunks larger than 64MB (:pr:`2637`) `Peter Andreas Entschev`_
+
+
 1.27.1 - 2019-04-29
 -------------------
 
@@ -20,7 +44,7 @@ Changelog
 1.27.0 - 2019-04-12
 -------------------
 
-   Add basic health endpoints to scheduler and worker bokeh. (:pr:`2607) `amerkel2`_
+-  Add basic health endpoints to scheduler and worker bokeh. (:pr:`2607`) `amerkel2`_
 -  Improved description accuracy of --memory-limit option. (:pr:`2601`) `Brett Randall`_
 -  Check self.dependencies when looking at dependent tasks in memory (:pr:`2606`) `deepthirajagopalan7`_
 -  Add RabbitMQ SchedulerPlugin example (:pr:`2604`) `Matt Nicolls`_
@@ -28,7 +52,7 @@ Changelog
 -  Use ensure_bytes in serialize_error (:pr:`2588`) `Matthew Rocklin`_
 -  Specify data storage explicitly from Worker constructor (:pr:`2600`) `Matthew Rocklin`_
 -  Change bokeh port keywords to dashboard_address (:pr:`2589`) `Matthew Rocklin`_
--  .detach_(`) pytorch tensor to serialize data as numpy array. (:pr:`2586`) `Muammar El Khatib`_
+-  .detach_() pytorch tensor to serialize data as numpy array. (:pr:`2586`) `Muammar El Khatib`_
 -  Add warning if creating scratch directories takes a long time (:pr:`2561`) `Matthew Rocklin`_
 -  Fix typo in pub-sub doc. (:pr:`2599`) `Loïc Estève`_
 -  Allow return_when='FIRST_COMPLETED' in wait (:pr:`2598`) `Nikos Tsaousis`_
@@ -1005,3 +1029,5 @@ significantly without many new features.
 .. _`Michael Delgado`: https://github.com/delgadom
 .. _`Peter Andreas Entschev`: https://github.com/pentschev
 .. _`condoratberlin`: https://github.com/condoratberlin
+.. _`K.-Michael Aye`: https://github.com/michaelaye
+.. _`@plbertrand`: https://github.com/plbertrand
\ No newline at end of file

From 7b526c0f436955d58144a8a76b64b5dea3e0b174 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Wed, 8 May 2019 14:32:51 -0500
Subject: [PATCH 0268/1550] Add release procedure doc (#2672)

---
 docs/release-procedure.md | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 docs/release-procedure.md

diff --git a/docs/release-procedure.md b/docs/release-procedure.md
new file mode 100644
index 00000000000..f9efd6a0ab1
--- /dev/null
+++ b/docs/release-procedure.md
@@ -0,0 +1,3 @@
+Distributed follows a similar procedure for releasing as the core Dask project.
+
+See https://github.com/dask/dask/blob/master/docs/release-procedure.md for instructions.
\ No newline at end of file

From ff6d3565b761b93f06af94cfb0d999af189a50e8 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 8 May 2019 17:06:47 -0500
Subject: [PATCH 0269/1550] Remove AioClient (#2668)

See also https://github.com/dask/dask-examples/pull/71

Fixes https://github.com/dask/distributed/issues/1164
Fixes https://github.com/dask/distributed/issues/1762
Fixes https://github.com/dask/distributed/issues/2537
Fixes https://github.com/dask/distributed/issues/2661
---
 distributed/asyncio.py                | 130 +--------
 distributed/tests/py3_test_asyncio.py | 365 --------------------------
 distributed/tests/test_asyncio.py     |   5 -
 docs/source/api.rst                   |   8 -
 docs/source/asynchronous.rst          |  46 +---
 5 files changed, 17 insertions(+), 537 deletions(-)
 delete mode 100644 distributed/tests/py3_test_asyncio.py
 delete mode 100644 distributed/tests/test_asyncio.py

diff --git a/distributed/asyncio.py b/distributed/asyncio.py
index b75bf2a1130..7a7225e60fd 100644
--- a/distributed/asyncio.py
+++ b/distributed/asyncio.py
@@ -1,126 +1,14 @@
-"""Experimental interface for asyncio, may disappear without warning"""
-
-# flake8: noqa
-
-import asyncio
-from functools import wraps
-
-from toolz import merge
-
-from tornado.platform.asyncio import BaseAsyncIOLoop
-from tornado.platform.asyncio import to_asyncio_future
-
-from . import client
-from .client import Client, Future
-from .variable import Variable
-from .utils import ignoring
-
-
-def to_asyncio(fn, **default_kwargs):
-    """Converts Tornado gen.coroutines and futures to asyncio ones"""
-
-    @wraps(fn)
-    def convert(*args, **kwargs):
-        if default_kwargs:
-            kwargs = merge(default_kwargs, kwargs)
-        return to_asyncio_future(fn(*args, **kwargs))
-
-    return convert
-
-
-class AioClient(Client):
-    """ Connect to and drive computation on a distributed Dask cluster
-
-    This class provides an asyncio compatible async/await interface for
-    dask.distributed.
-
-    The Client connects users to a dask.distributed compute cluster. It
-    provides an asynchronous user interface around functions and futures.
-    This class resembles executors in ``concurrent.futures`` but also
-    allows ``Future`` objects within ``submit/map`` calls.
-
-    AioClient is an **experimental** interface for distributed and may
-    disappear without warning!
-
-    Parameters
-    ----------
-    address: string, or Cluster
-        This can be the address of a ``Scheduler`` server like a string
-        ``'127.0.0.1:8786'`` or a cluster object like ``LocalCluster()``
-
-    Examples
-    --------
-    Provide cluster's scheduler address on initialization::
-
-        client = AioClient('127.0.0.1:8786')
-
-    Start the client::
-
-        async def start_the_client():
-            client = await AioClient()
-
-            # Use the client....
-
-            await client.close()
-
-    An ``async with`` statement is a more convenient way to start and shut down
-    the client::
-
-        async def start_the_client():
-            async with AioClient() as client:
-                # Use the client within this block.
-                pass
-
-    Use the ``submit`` method to send individual computations to the cluster,
-    and await the returned future to retrieve the result::
-
-        async def add_two_numbers():
-            async with AioClient() as client:
-                a = client.submit(add, 1, 2)
-                result = await a
-
-    Continue using submit or map on results to build up larger computations,
-    and gather results with the ``gather`` method::
-
-        async def gather_some_results():
-            async with AioClient() as client:
-                a = client.submit(add, 1, 2)
-                b = client.submit(add, 10, 20)
-                c = client.submit(add, a, b)
-                result = await client.gather([c])
-
-    See Also
-    --------
-    distributed.client.Client: Blocking Client
-    distributed.scheduler.Scheduler: Internal scheduler
+raise ImportError(
     """
 
-    def __init__(self, *args, **kwargs):
-        loop = asyncio.get_event_loop()
-        ioloop = BaseAsyncIOLoop(loop)
-        super().__init__(*args, loop=ioloop, asynchronous=True, **kwargs)
-
-    def __enter__(self):
-        raise RuntimeError("Use AioClient in an 'async with' block, not 'with'")
-
-    async def __aenter__(self):
-        await to_asyncio_future(self._started)
-        return self
-
-    async def __aexit__(self, type, value, traceback):
-        await to_asyncio_future(self._close())
-
-    def __await__(self):
-        return to_asyncio_future(self._started).__await__()
-
-    get = to_asyncio(Client.get, sync=False)
-    sync = to_asyncio(Client.sync)
-    close = to_asyncio(Client.close)
-    shutdown = to_asyncio(Client.shutdown)
-
+The dask.distributed.AioClient object has been removed.
+We recommend using the normal client with asynchonrous=True
 
-class as_completed(client.as_completed):
-    __anext__ = to_asyncio(client.as_completed.__anext__)
+        client = await Client(..., asynchronous=True)
 
+and a version of Tornado >= 5.
 
-wait = to_asyncio(client._wait)
+Documentation: https://distributed.dask.org/en/latest/asynchronous.html
+Example: https://examples.dask.org/applications/async-await.html
+"""
+)
diff --git a/distributed/tests/py3_test_asyncio.py b/distributed/tests/py3_test_asyncio.py
deleted file mode 100644
index 3754b282813..00000000000
--- a/distributed/tests/py3_test_asyncio.py
+++ /dev/null
@@ -1,365 +0,0 @@
-# flake8: noqa
-import pytest
-
-asyncio = pytest.importorskip("asyncio")
-
-import functools
-from time import time
-from operator import add
-from toolz import isdistinct
-from concurrent.futures import CancelledError
-from distributed.utils_test import slow
-from distributed.utils_test import slowinc
-
-from tornado.ioloop import IOLoop
-from tornado.platform.asyncio import BaseAsyncIOLoop
-
-from distributed.client import Future
-from distributed.variable import Variable
-from distributed.asyncio import AioClient
-from distributed.asyncio import as_completed, wait
-from distributed.utils_test import inc, div
-
-
-def coro_test(fn):
-    assert asyncio.iscoroutinefunction(fn)
-
-    @functools.wraps(fn)
-    def wrapper(*args, **kwargs):
-        loop = None
-        try:
-            IOLoop.clear_current()
-            loop = asyncio.new_event_loop()
-            asyncio.set_event_loop(loop)
-            loop.run_until_complete(fn(*args, **kwargs))
-        finally:
-            if loop is not None:
-                loop.close()
-
-            IOLoop.clear_current()
-            asyncio.set_event_loop(None)
-
-    return wrapper
-
-
-@coro_test
-async def test_coro_test():
-    assert asyncio.get_event_loop().is_running()
-
-
-@coro_test
-async def test_asyncio_start_close():
-    async with AioClient(processes=False, dashboard_address=False) as c:
-        assert c.status == "running"
-        # AioClient has installed its AioLoop shim.
-        assert isinstance(IOLoop.current(instance=False), BaseAsyncIOLoop)
-
-        result = await c.submit(inc, 10)
-        assert result == 11
-
-        await c.close()
-        assert c.status == "closed"
-        # assert IOLoop.current(instance=False) is None
-
-
-@coro_test
-async def test_asyncio_submit():
-    async with AioClient(processes=False) as c:
-        x = c.submit(inc, 10)
-        assert not x.done()
-
-        assert isinstance(x, Future)
-        assert x.client is c
-
-        result = await x.result()
-        assert result == 11
-        assert x.done()
-
-        y = c.submit(inc, 20)
-        z = c.submit(add, x, y)
-
-        result = await z.result()
-        assert result == 11 + 21
-
-
-@coro_test
-async def test_asyncio_future_await():
-    async with AioClient(processes=False) as c:
-        x = c.submit(inc, 10)
-        assert not x.done()
-
-        assert isinstance(x, Future)
-        assert x.client is c
-
-        result = await x
-        assert result == 11
-        assert x.done()
-
-        y = c.submit(inc, 20)
-        z = c.submit(add, x, y)
-
-        result = await z
-        assert result == 11 + 21
-
-
-@coro_test
-async def test_asyncio_map():
-    async with AioClient(processes=False) as c:
-        L1 = c.map(inc, range(5))
-        assert len(L1) == 5
-        assert isdistinct(x.key for x in L1)
-        assert all(isinstance(x, Future) for x in L1)
-
-        result = await L1[0]
-        assert result == inc(0)
-
-        L2 = c.map(inc, L1)
-
-        result = await L2[1]
-        assert result == inc(inc(1))
-
-        total = c.submit(sum, L2)
-        result = await total
-        assert result == sum(map(inc, map(inc, range(5))))
-
-        L3 = c.map(add, L1, L2)
-        result = await L3[1]
-        assert result == inc(1) + inc(inc(1))
-
-        L4 = c.map(add, range(3), range(4))
-        results = await c.gather(L4)
-        assert results == list(map(add, range(3), range(4)))
-
-        def f(x, y=10):
-            return x + y
-
-        L5 = c.map(f, range(5), y=5)
-        results = await c.gather(L5)
-        assert results == list(range(5, 10))
-
-        y = c.submit(f, 10)
-        L6 = c.map(f, range(5), y=y)
-        results = await c.gather(L6)
-        assert results == list(range(20, 25))
-
-
-@coro_test
-async def test_asyncio_gather():
-    async with AioClient(processes=False) as c:
-        x = c.submit(inc, 10)
-        y = c.submit(inc, x)
-
-        result = await c.gather(x)
-        assert result == 11
-        result = await c.gather([x])
-        assert result == [11]
-        result = await c.gather({"x": x, "y": [y]})
-        assert result == {"x": 11, "y": [12]}
-
-
-@coro_test
-async def test_asyncio_get():
-    async with AioClient(processes=False) as c:
-        result = await c.get({"x": (inc, 1)}, "x")
-        assert result == 2
-
-        result = await c.get({"x": (inc, 1)}, ["x"])
-        assert result == [2]
-
-        result = await c.get({}, [])
-        assert result == []
-
-        result = await c.get({("x", 1): (inc, 1), ("x", 2): (inc, ("x", 1))}, ("x", 2))
-        assert result == 3
-
-
-@coro_test
-async def test_asyncio_exceptions():
-    async with AioClient(processes=False) as c:
-        result = await c.submit(div, 1, 2)
-        assert result == 1 / 2
-
-        with pytest.raises(ZeroDivisionError):
-            result = await c.submit(div, 1, 0)
-
-        result = await c.submit(div, 10, 2)  # continues to operate
-        assert result == 10 / 2
-
-
-@coro_test
-async def test_asyncio_exception_on_exception():
-    async with AioClient(processes=False) as c:
-        x = c.submit(lambda: 1 / 0)
-        y = c.submit(inc, x)
-
-        with pytest.raises(ZeroDivisionError):
-            await y
-
-        z = c.submit(inc, y)
-        with pytest.raises(ZeroDivisionError):
-            await z
-
-
-@coro_test
-async def test_asyncio_as_completed():
-    async with AioClient(processes=False) as c:
-        futures = c.map(inc, range(10))
-
-        results = []
-        async for future in as_completed(futures):
-            results.append(await future)
-
-        assert set(results) == set(range(1, 11))
-
-
-@coro_test
-async def test_asyncio_cancel():
-    async with AioClient(processes=False) as c:
-        s = c.cluster.scheduler
-
-        x = c.submit(slowinc, 1)
-        y = c.submit(slowinc, x)
-
-        while y.key not in s.tasks:
-            await asyncio.sleep(0.01)
-
-        await c.cancel([x])
-
-        assert x.cancelled()
-        assert "cancel" in str(x)
-        s.validate_state()
-
-        start = time()
-        while not y.cancelled():
-            await asyncio.sleep(0.01)
-            assert time() < start + 5
-
-        assert not s.tasks
-        assert not s.who_has
-        s.validate_state()
-
-
-@coro_test
-async def test_asyncio_cancel_tuple_key():
-    async with AioClient(processes=False) as c:
-        x = c.submit(inc, 1, key=("x", 0, 1))
-        await x
-        await c.cancel(x)
-        with pytest.raises(CancelledError):
-            await x
-
-
-@coro_test
-async def test_asyncio_wait():
-    async with AioClient(processes=False) as c:
-        x = c.submit(inc, 1)
-        y = c.submit(inc, 2)
-        z = c.submit(inc, 3)
-
-        await wait(x)
-        assert x.done() is True
-
-        await wait([y, z])
-        assert y.done() is True
-        assert z.done() is True
-
-
-@coro_test
-async def test_asyncio_run():
-    async with AioClient(processes=False) as c:
-        results = await c.run(inc, 1)
-        assert len(results) > 0
-        assert [value == 2 for value in results.values()]
-
-        results = await c.run(inc, 1, workers=[])
-        assert results == {}
-
-
-@coro_test
-async def test_asyncio_run_on_scheduler():
-    def f(dask_scheduler=None):
-        return dask_scheduler.address
-
-    async with AioClient(processes=False) as c:
-        address = await c.run_on_scheduler(f)
-        assert address == c.cluster.scheduler.address
-
-        with pytest.raises(ZeroDivisionError):
-            await c.run_on_scheduler(div, 1, 0)
-
-
-@coro_test
-async def test_asyncio_run_coroutine():
-    async def aioinc(x, delay=0.02):
-        await asyncio.sleep(delay)
-        return x + 1
-
-    async def aiothrows(x, delay=0.02):
-        await asyncio.sleep(delay)
-        raise RuntimeError("hello")
-
-    async with AioClient(processes=False) as c:
-        results = await c.run(aioinc, 1, delay=0.05)
-        assert len(results) > 0
-        assert [value == 2 for value in results.values()]
-
-        results = await c.run(aioinc, 1, workers=[])
-        assert results == {}
-
-        with pytest.raises(RuntimeError) as exc_info:
-            await c.run(aiothrows, 1)
-        assert "hello" in str(exc_info)
-
-
-@slow
-@coro_test
-async def test_asyncio_restart():
-    async with AioClient(processes=False) as c:
-        assert c.status == "running"
-        x = c.submit(inc, 1)
-        assert x.key in c.refcount
-
-        await c.restart()
-        assert x.key not in c.refcount
-
-        key = x.key
-        del x
-        import gc
-
-        gc.collect()
-
-        assert key not in c.refcount
-
-
-@coro_test
-async def test_asyncio_nanny_workers():
-    async with AioClient(n_workers=2) as c:
-        assert await c.submit(inc, 1) == 2
-
-
-@coro_test
-async def test_asyncio_variable():
-    async with AioClient(processes=False) as c:
-        s = c.cluster.scheduler
-
-        x = Variable("x")
-        xx = Variable("x")
-        assert x.client is c
-
-        future = c.submit(inc, 1)
-
-        await x.set(future)
-        future2 = await xx.get()
-        assert future.key == future2.key
-
-        del future, future2
-
-        await asyncio.sleep(0.1)
-        assert s.tasks  # future still present
-
-        x.delete()
-
-        start = time()
-        while s.tasks:
-            await asyncio.sleep(0.01)
-            assert time() < start + 5
diff --git a/distributed/tests/test_asyncio.py b/distributed/tests/test_asyncio.py
deleted file mode 100644
index 4eab91a5a81..00000000000
--- a/distributed/tests/test_asyncio.py
+++ /dev/null
@@ -1,5 +0,0 @@
-import sys
-
-
-if sys.version_info >= (3, 5):
-    from distributed.tests.py3_test_asyncio import *  # noqa: F401, F403
diff --git a/docs/source/api.rst b/docs/source/api.rst
index 47933be06d4..e91c4ee6ac1 100644
--- a/docs/source/api.rst
+++ b/docs/source/api.rst
@@ -177,14 +177,6 @@ Other
    :members:
 
 
-Asyncio Client
---------------
-
-.. currentmodule:: distributed.asyncio
-.. autoclass:: AioClient
-   :members:
-
-
 Adaptive
 --------
 
diff --git a/docs/source/asynchronous.rst b/docs/source/asynchronous.rst
index 1c079035722..9d38a8b04fe 100644
--- a/docs/source/asynchronous.rst
+++ b/docs/source/asynchronous.rst
@@ -57,25 +57,6 @@ call.
        results = await client.gather(futures, asynchronous=True)
        return results
 
-AsyncIO
--------
-
-If you prefer to use the Asyncio event loop over the Tornado event loop you
-should use the ``AioClient``.
-
-.. code-block:: python
-
-   from distributed.asyncio import AioClient
-   client = await AioClient()
-
-All other operations remain the same:
-
-.. code-block:: python
-
-   future = client.submit(lambda x: x + 1, 10)
-   result = await future
-   # or
-   result = await client.gather(future)
 
 Python 2 Compatibility
 ----------------------
@@ -90,8 +71,8 @@ This self-contained example starts an asynchronous client, submits a trivial
 job, waits on the result, and then shuts down the client.  You can see
 implementations for Python 2 and 3 and for Asyncio and Tornado.
 
-Python 3 with Tornado
-+++++++++++++++++++++
+Python 3 with Tornado or Asyncio
+++++++++++++++++++++++++++++++++
 
 .. code-block:: python
 
@@ -104,9 +85,15 @@ Python 3 with Tornado
        await client.close()
        return result
 
+   # Either use Tornado
    from tornado.ioloop import IOLoop
    IOLoop().run_sync(f)
 
+   # Or use asyncio
+   import asyncio
+   asyncio.get_event_loop().run_until_complete(f())
+
+
 Python 2/3 with Tornado
 +++++++++++++++++++++++
 
@@ -126,23 +113,6 @@ Python 2/3 with Tornado
    from tornado.ioloop import IOLoop
    IOLoop().run_sync(f)
 
-Python 3 with Asyncio
-+++++++++++++++++++++
-
-.. code-block:: python
-
-   from distributed.asyncio import AioClient
-
-   async def f():
-       client = await AioClient()
-       future = client.submit(lambda x: x + 1, 10)
-       result = await future
-       await client.close()
-       return result
-
-   from asyncio import get_event_loop
-   get_event_loop().run_until_complete(f())
-
 Use Cases
 ---------
 

From 8dee90c3b093538188def4513d274886cae1c842 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 8 May 2019 19:16:01 -0500
Subject: [PATCH 0270/1550]  Move interface/host/port handling from CLI to
 classes (#2667)

This should allow other systems to benefit from this logic, but does make the
class constructors a bit more complicated.  Overall I think it's a win though.

* Improve debug information around test_file_descriptors
* Move port and host information into constructor in tests
* Pull out interface/host/port logic
* Test no leaked processes
---
 distributed/cli/dask_scheduler.py         | 31 +++++-----
 distributed/cli/dask_worker.py            | 25 ++------
 distributed/cli/tests/test_cli_utils.py   | 54 ------------------
 distributed/cli/tests/test_dask_worker.py |  3 +-
 distributed/cli/utils.py                  | 40 -------------
 distributed/comm/addressing.py            | 69 +++++++++++++++++++++++
 distributed/nanny.py                      | 15 +++++
 distributed/scheduler.py                  | 27 ++++++++-
 distributed/tests/test_client.py          |  9 +--
 distributed/tests/test_scheduler.py       | 21 ++++---
 distributed/tests/test_worker.py          | 21 +++++--
 distributed/utils_test.py                 | 38 +++++++++----
 distributed/worker.py                     | 18 +++++-
 13 files changed, 206 insertions(+), 165 deletions(-)
 delete mode 100644 distributed/cli/tests/test_cli_utils.py

diff --git a/distributed/cli/dask_scheduler.py b/distributed/cli/dask_scheduler.py
index 3b0aa5b4c70..57a7168a3a2 100755
--- a/distributed/cli/dask_scheduler.py
+++ b/distributed/cli/dask_scheduler.py
@@ -5,6 +5,7 @@
 import logging
 import gc
 import os
+import re
 import shutil
 import sys
 import tempfile
@@ -16,12 +17,7 @@
 
 from distributed import Scheduler
 from distributed.security import Security
-from distributed.utils import get_ip_interface
-from distributed.cli.utils import (
-    check_python_3,
-    install_signal_handlers,
-    uri_from_host_port,
-)
+from distributed.cli.utils import check_python_3, install_signal_handlers
 from distributed.preloading import preload_modules, validate_preload_argv
 from distributed.proctitle import (
     enable_proctitle_on_children,
@@ -151,6 +147,9 @@ def main(
         )
         dashboard_address = bokeh_port
 
+    if port is None and (not host or not re.search(r":\d", host)):
+        port = 8786
+
     sec = Security(
         tls_ca_file=tls_ca_file, tls_scheduler_cert=tls_cert, tls_scheduler_key=tls_key
     )
@@ -186,14 +185,6 @@ def del_pid_file():
         limit = max(soft, hard // 2)
         resource.setrlimit(resource.RLIMIT_NOFILE, (limit, hard))
 
-    if interface:
-        if host:
-            raise ValueError("Can not specify both interface and host")
-        else:
-            host = get_ip_interface(interface)
-
-    addr = uri_from_host_port(host, port, 8786)
-
     loop = IOLoop.current()
     logger.info("-" * 47)
 
@@ -213,9 +204,15 @@ def del_pid_file():
                 logger.info("Unable to import bokeh: %s" % str(error))
 
     scheduler = Scheduler(
-        loop=loop, services=services, scheduler_file=scheduler_file, security=sec
+        loop=loop,
+        services=services,
+        scheduler_file=scheduler_file,
+        security=sec,
+        host=host,
+        port=port,
+        interface=interface,
     )
-    scheduler.start(addr)
+    scheduler.start()
     if not preload:
         preload = dask.config.get("distributed.scheduler.preload")
     if not preload_argv:
@@ -237,7 +234,7 @@ def del_pid_file():
         if local_directory_created:
             shutil.rmtree(local_directory)
 
-        logger.info("End scheduler at %r", addr)
+        logger.info("End scheduler at %r", scheduler.address)
 
 
 def go():
diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
index a0bc801a960..6315939005d 100755
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -10,14 +10,10 @@
 import click
 from distributed import Nanny, Worker
 from distributed.config import config
-from distributed.utils import get_ip_interface, parse_timedelta
+from distributed.utils import parse_timedelta
 from distributed.worker import _ncores
 from distributed.security import Security
-from distributed.cli.utils import (
-    check_python_3,
-    uri_from_host_port,
-    install_signal_handlers,
-)
+from distributed.cli.utils import check_python_3, install_signal_handlers
 from distributed.comm import get_address_host_port
 from distributed.preloading import validate_preload_argv
 from distributed.proctitle import (
@@ -328,18 +324,6 @@ def del_pid_file():
             "dask-worker SCHEDULER_ADDRESS:8786"
         )
 
-    if interface:
-        if host:
-            raise ValueError("Can not specify both interface and host")
-        else:
-            host = get_ip_interface(interface)
-
-    if host or port:
-        addr = uri_from_host_port(host, port, 0)
-    else:
-        # Choose appropriate address for scheduler
-        addr = None
-
     if death_timeout is not None:
         death_timeout = parse_timedelta(death_timeout, "s")
 
@@ -359,6 +343,9 @@ def del_pid_file():
             preload_argv=preload_argv,
             security=sec,
             contact_address=contact_address,
+            interface=interface,
+            host=host,
+            port=port,
             name=name if nprocs == 1 or not name else name + "-" + str(i),
             **kwargs
         )
@@ -377,7 +364,7 @@ def on_signal(signum):
 
     @gen.coroutine
     def run():
-        yield [n._start(addr) for n in nannies]
+        yield nannies
         while all(n.status != "closed" for n in nannies):
             yield gen.sleep(0.2)
 
diff --git a/distributed/cli/tests/test_cli_utils.py b/distributed/cli/tests/test_cli_utils.py
deleted file mode 100644
index 4f07f699de5..00000000000
--- a/distributed/cli/tests/test_cli_utils.py
+++ /dev/null
@@ -1,54 +0,0 @@
-from __future__ import print_function, division, absolute_import
-
-import pytest
-
-pytest.importorskip("requests")
-
-from distributed.cli.utils import uri_from_host_port
-from distributed.utils import get_ip
-
-
-external_ip = get_ip()
-
-
-def test_uri_from_host_port():
-    f = uri_from_host_port
-
-    assert f("", 456, None) == "tcp://:456"
-    assert f("", 456, 123) == "tcp://:456"
-    assert f("", None, 123) == "tcp://:123"
-    assert f("", None, 0) == "tcp://"
-    assert f("", 0, 123) == "tcp://"
-
-    assert f("localhost", 456, None) == "tcp://localhost:456"
-    assert f("localhost", 456, 123) == "tcp://localhost:456"
-    assert f("localhost", None, 123) == "tcp://localhost:123"
-    assert f("localhost", None, 0) == "tcp://localhost"
-
-    assert f("192.168.1.2", 456, None) == "tcp://192.168.1.2:456"
-    assert f("192.168.1.2", 456, 123) == "tcp://192.168.1.2:456"
-    assert f("192.168.1.2", None, 123) == "tcp://192.168.1.2:123"
-    assert f("192.168.1.2", None, 0) == "tcp://192.168.1.2"
-
-    assert f("tcp://192.168.1.2", 456, None) == "tcp://192.168.1.2:456"
-    assert f("tcp://192.168.1.2", 456, 123) == "tcp://192.168.1.2:456"
-    assert f("tcp://192.168.1.2", None, 123) == "tcp://192.168.1.2:123"
-    assert f("tcp://192.168.1.2", None, 0) == "tcp://192.168.1.2"
-
-    assert f("tcp://192.168.1.2:456", None, None) == "tcp://192.168.1.2:456"
-    assert f("tcp://192.168.1.2:456", 0, 0) == "tcp://192.168.1.2:456"
-    assert f("tcp://192.168.1.2:456", 0, 123) == "tcp://192.168.1.2:456"
-    assert f("tcp://192.168.1.2:456", 456, 123) == "tcp://192.168.1.2:456"
-
-    with pytest.raises(ValueError):
-        # Two incompatible port values
-        f("tcp://192.168.1.2:456", 123, None)
-
-    assert f("tls://192.168.1.2:456", None, None) == "tls://192.168.1.2:456"
-    assert f("tls://192.168.1.2:456", 0, 0) == "tls://192.168.1.2:456"
-    assert f("tls://192.168.1.2:456", 0, 123) == "tls://192.168.1.2:456"
-    assert f("tls://192.168.1.2:456", 456, 123) == "tls://192.168.1.2:456"
-
-    assert f("tcp://[::1]:456", None, None) == "tcp://[::1]:456"
-
-    assert f("tls://[::1]:456", None, None) == "tls://[::1]:456"
diff --git a/distributed/cli/tests/test_dask_worker.py b/distributed/cli/tests/test_dask_worker.py
index 72084e53141..2fa3779d9b4 100644
--- a/distributed/cli/tests/test_dask_worker.py
+++ b/distributed/cli/tests/test_dask_worker.py
@@ -7,7 +7,6 @@
 import requests
 import sys
 from time import sleep
-from toolz import first
 
 from distributed import Client
 from distributed.metrics import time
@@ -52,7 +51,7 @@ def test_memory_limit(loop):
                 while not c.ncores():
                     sleep(0.1)
                 info = c.scheduler_info()
-                d = first(info["workers"].values())
+                [d] = info["workers"].values()
                 assert isinstance(d["memory_limit"], int)
                 assert d["memory_limit"] == 2e9
 
diff --git a/distributed/cli/utils.py b/distributed/cli/utils.py
index 4ce1d845821..2c2088a7556 100644
--- a/distributed/cli/utils.py
+++ b/distributed/cli/utils.py
@@ -3,13 +3,6 @@
 from tornado import gen
 from tornado.ioloop import IOLoop
 
-from distributed.comm import (
-    parse_address,
-    unparse_address,
-    parse_host_port,
-    unparse_host_port,
-)
-
 
 py3_err_msg = """
 Warning: Your terminal does not set locales.
@@ -75,36 +68,3 @@ def cleanup_and_stop():
 
     for sig in [signal.SIGINT, signal.SIGTERM]:
         old_handlers[sig] = signal.signal(sig, handle_signal)
-
-
-def uri_from_host_port(host_arg, port_arg, default_port):
-    """
-    Process the *host* and *port* CLI options.
-    Return a URI.
-    """
-    # Much of distributed depends on a well-known IP being assigned to
-    # each entity (Worker, Scheduler, etc.), so avoid "universal" addresses
-    # like '' which would listen on all registered IPs and interfaces.
-    scheme, loc = parse_address(host_arg or "")
-
-    host, port = parse_host_port(
-        loc, port_arg if port_arg is not None else default_port
-    )
-
-    if port is None and port_arg is None:
-        port_arg = default_port
-
-    if port and port_arg and port != port_arg:
-        raise ValueError(
-            "port number given twice in options: "
-            "host %r and port %r" % (host_arg, port_arg)
-        )
-    if port is None and port_arg is not None:
-        port = port_arg
-    # Note `port = 0` means "choose a random port"
-    if port is None:
-        port = default_port
-    loc = unparse_host_port(host, port)
-    addr = unparse_address(scheme, loc)
-
-    return addr
diff --git a/distributed/comm/addressing.py b/distributed/comm/addressing.py
index 20ddb2c863f..3d79befe0f1 100644
--- a/distributed/comm/addressing.py
+++ b/distributed/comm/addressing.py
@@ -5,6 +5,7 @@
 import dask
 
 from . import registry
+from ..utils import get_ip_interface
 
 
 DEFAULT_SCHEME = dask.config.get("distributed.comm.default-scheme")
@@ -172,3 +173,71 @@ def resolve_address(addr):
     scheme, loc = parse_address(addr)
     backend = registry.get_backend(scheme)
     return unparse_address(scheme, backend.resolve_address(loc))
+
+
+def uri_from_host_port(host_arg, port_arg, default_port):
+    """
+    Process the *host* and *port* CLI options.
+    Return a URI.
+    """
+    # Much of distributed depends on a well-known IP being assigned to
+    # each entity (Worker, Scheduler, etc.), so avoid "universal" addresses
+    # like '' which would listen on all registered IPs and interfaces.
+    scheme, loc = parse_address(host_arg or "")
+
+    host, port = parse_host_port(
+        loc, port_arg if port_arg is not None else default_port
+    )
+
+    if port is None and port_arg is None:
+        port_arg = default_port
+
+    if port and port_arg and port != port_arg:
+        raise ValueError(
+            "port number given twice in options: "
+            "host %r and port %r" % (host_arg, port_arg)
+        )
+    if port is None and port_arg is not None:
+        port = port_arg
+    # Note `port = 0` means "choose a random port"
+    if port is None:
+        port = default_port
+    loc = unparse_host_port(host, port)
+    addr = unparse_address(scheme, loc)
+
+    return addr
+
+
+def address_from_user_args(
+    host=None, port=None, interface=None, protocol=None, peer=None, security=None
+):
+    """ Get an address to listen on from common user provided arguments """
+    if security and security.require_encryption and not protocol:
+        protocol = "tls"
+
+    if protocol and protocol.rstrip("://") == "inplace":
+        if host or port or interface:
+            raise ValueError(
+                "Can not specify inproc protocol and host or port or interface"
+            )
+        else:
+            return "inproc://"
+
+    if interface:
+        if host:
+            raise ValueError("Can not specify both interface and host", interface, host)
+        else:
+            host = get_ip_interface(interface)
+
+    if protocol and host and "://" not in host:
+        host = protocol.rstrip("://") + "://" + host
+
+    if host or port:
+        addr = uri_from_host_port(host, port, 0)
+    else:
+        addr = ""
+
+    if protocol and "://" not in addr:
+        addr = protocol.rstrip("://") + "://" + addr
+
+    return addr
diff --git a/distributed/nanny.py b/distributed/nanny.py
index 60e83e86da7..ef0e0a38f0e 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -16,6 +16,7 @@
 from tornado.locks import Event
 
 from .comm import get_address_host, get_local_address_for, unparse_host_port
+from .comm.addressing import address_from_user_args
 from .core import rpc, RPCClosed, CommClosedError, coerce_to_address
 from .metrics import time
 from .node import ServerNode
@@ -69,6 +70,10 @@ def __init__(
         listen_address=None,
         worker_class=None,
         env=None,
+        interface=None,
+        host=None,
+        port=None,
+        protocol=None,
         **worker_kwargs
     ):
 
@@ -135,6 +140,14 @@ def __init__(
             pc = PeriodicCallback(self.memory_monitor, 100, io_loop=self.loop)
             self.periodic_callbacks["memory"] = pc
 
+        self._start_address = address_from_user_args(
+            host=host,
+            port=port,
+            interface=interface,
+            protocol=protocol,
+            security=security,
+        )
+
         self._listen_address = listen_address
         self.status = "init"
 
@@ -175,6 +188,7 @@ def worker_dir(self):
     @gen.coroutine
     def _start(self, addr_or_port=0):
         """ Start nanny, start local process, start watching """
+        addr_or_port = addr_or_port or self._start_address
 
         # XXX Factor this out
         if not addr_or_port:
@@ -419,6 +433,7 @@ def start(self):
 
         self.process = AsyncProcess(
             target=self._run,
+            name="Dask Worker process (from Nanny)",
             kwargs=dict(
                 worker_args=self.worker_args,
                 worker_kwargs=self.worker_kwargs,
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 8442b6ddcea..8ba4cedf468 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -35,6 +35,7 @@
     get_address_host,
     unparse_host_port,
 )
+from .comm.addressing import address_from_user_args
 from .compatibility import finalize, unicode, Mapping, Set
 from .core import rpc, connect, send_recv, clean_exception, CommClosedError
 from . import profile
@@ -824,9 +825,12 @@ def __init__(
         security=None,
         worker_ttl=None,
         idle_timeout=None,
+        interface=None,
+        host=None,
+        port=8786,
+        protocol=None,
         **kwargs
     ):
-
         self._setup_logging()
 
         # Attributes
@@ -1056,6 +1060,14 @@ def __init__(
 
         connection_limit = get_fileno_limit() / 2
 
+        self._start_address = address_from_user_args(
+            host=host,
+            port=port,
+            interface=interface,
+            protocol=protocol,
+            security=security,
+        )
+
         super(Scheduler, self).__init__(
             handlers=self.handlers,
             stream_handlers=merge(worker_handlers, client_handlers),
@@ -1172,10 +1184,12 @@ def stop_services(self):
         for service in self.services.values():
             service.stop()
 
-    def start(self, addr_or_port=8786, start_queues=True):
+    def start(self, addr_or_port=None, start_queues=True):
         """ Clear out old state and restart all running coroutines """
         enable_gc_diagnosis()
 
+        addr_or_port = addr_or_port or self._start_address
+
         self.clear_task_state()
 
         with ignoring(AttributeError):
@@ -1234,6 +1248,15 @@ def del_scheduler_file():
 
         return self.finished()
 
+    def __await__(self):
+        self.start()
+
+        @gen.coroutine
+        def _():
+            return self
+
+        return _().__await__()
+
     @gen.coroutine
     def finished(self):
         """ Wait until all coroutines have ceased """
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 8582c2abc83..3515be9ebcb 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -2824,8 +2824,7 @@ def test_diagnostic_nbytes(c, s, a, b):
 
 @gen_test()
 def test_worker_aliases():
-    s = Scheduler(validate=True)
-    s.start(0)
+    s = yield Scheduler(validate=True, port=0)
     a = Worker(s.ip, s.port, name="alice")
     b = Worker(s.ip, s.port, name="bob")
     w = Worker(s.ip, s.port, name=3)
@@ -3062,8 +3061,7 @@ def test_unrunnable_task_runs(c, s, a, b):
 def test_add_worker_after_tasks(c, s):
     futures = c.map(inc, range(10))
 
-    n = Nanny(s.ip, s.port, ncores=2, loop=s.loop)
-    n.start(0)
+    n = yield Nanny(s.ip, s.port, ncores=2, loop=s.loop, port=0)
 
     result = yield c.gather(futures)
 
@@ -3603,8 +3601,7 @@ def test_as_completed_next_batch(c):
 
 @gen_test()
 def test_status():
-    s = Scheduler()
-    s.start(0)
+    s = yield Scheduler(port=0)
 
     c = yield Client((s.ip, s.port), asynchronous=True)
     assert c.status == "running"
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 9224ed69030..805b0e06ed0 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -533,8 +533,7 @@ def test_broadcast_nanny(s, a, b):
 
 @gen_test()
 def test_worker_name():
-    s = Scheduler(validate=True)
-    s.start(0)
+    s = yield Scheduler(validate=True, port=0)
     w = yield Worker(s.ip, s.port, name="alice")
     assert s.workers[w.address].name == "alice"
     assert s.aliases["alice"] == w.address
@@ -550,8 +549,7 @@ def test_worker_name():
 @gen_test()
 def test_coerce_address():
     with dask.config.set({"distributed.comm.timeouts.connect": "100ms"}):
-        s = Scheduler(validate=True)
-        s.start(0)
+        s = yield Scheduler(validate=True, port=0)
         print("scheduler:", s.address, s.listen_address)
         a = Worker(s.ip, s.port, name="alice")
         b = Worker(s.ip, s.port, name=123)
@@ -824,7 +822,7 @@ def test_file_descriptors(c, s):
     yield [n.close() for n in nannies]
 
     assert not s.rpc.open
-    assert not c.rpc.active
+    assert not c.rpc.active, list(c.rpc._created)
     assert not s.stream_comms
 
     start = time()
@@ -1133,8 +1131,7 @@ def test_fifo_submission(c, s, w):
 @gen_test()
 def test_scheduler_file():
     with tmpfile() as fn:
-        s = Scheduler(scheduler_file=fn)
-        s.start(0)
+        s = yield Scheduler(scheduler_file=fn, port=0)
         with open(fn) as f:
             data = json.load(f)
         assert data["address"] == s.address
@@ -1536,3 +1533,13 @@ def test_close_workers(s, a, b):
     yield s.close(close_workers=True)
     assert a.status == "closed"
     assert b.status == "closed"
+
+
+@pytest.mark.skipif(
+    not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
+)
+@gen_test()
+def test_host_address():
+    s = yield Scheduler(host="127.0.0.2")
+    assert "127.0.0.2" in s.address
+    yield s.close()
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 07864ab4b64..8ca3c5d9682 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -304,8 +304,7 @@ def test_broadcast(s, a, b):
 
 @gen_test()
 def test_worker_with_port_zero():
-    s = Scheduler()
-    s.start(8007)
+    s = yield Scheduler(port=8007)
     w = yield Worker(s.address)
     assert isinstance(w.port, int)
     assert w.port > 1024
@@ -1007,8 +1006,7 @@ def test_start_services(s):
 @gen_test()
 def test_scheduler_file():
     with tmpfile() as fn:
-        s = Scheduler(scheduler_file=fn)
-        s.start(8009)
+        s = yield Scheduler(scheduler_file=fn, port=8009)
         w = yield Worker(scheduler_file=fn)
         assert set(s.workers) == {w.address}
         yield w.close()
@@ -1384,3 +1382,18 @@ def __init__(self, x, y):
     assert w.data.x == 123
     assert w.data.y == 456
     yield w.close()
+
+
+@pytest.mark.skipif(
+    not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
+)
+@gen_cluster(ncores=[], client=True)
+def test_host_address(c, s):
+    w = yield Worker(s.address, host="127.0.0.2")
+    assert "127.0.0.2" in w.address
+    yield w.close()
+
+    n = yield Nanny(s.address, host="127.0.0.3")
+    assert "127.0.0.3" in n.address
+    assert "127.0.0.3" in n.worker_address
+    yield n.close()
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index b0c0d2d48cc..7aaa5b1ed0d 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -11,7 +11,6 @@
 import logging
 import logging.config
 import os
-import psutil
 import re
 import shutil
 import signal
@@ -40,7 +39,7 @@
 from tornado.ioloop import IOLoop
 
 from .client import default_client, _global_clients, Client
-from .compatibility import PY3, Empty, WINDOWS, PY2
+from .compatibility import PY3, Empty, WINDOWS
 from .comm import Comm
 from .comm.utils import offload
 from .config import initialize_logging
@@ -156,10 +155,7 @@ def start():
 
     _cleanup_dangling()
 
-    if PY2:  # no forkserver, so no extra procs
-        for child in psutil.Process().children(recursive=True):
-            with ignoring(psutil.NoSuchProcess):
-                child.terminate()
+    assert_no_leaked_processes()
 
     _global_clients.clear()
 
@@ -482,8 +478,8 @@ def run_scheduler(q, nputs, **kwargs):
     # On Python 2.7 and Unix, fork() is used to spawn child processes,
     # so avoid inheriting the parent's IO loop.
     with pristine_loop() as loop:
-        scheduler = Scheduler(validate=True, **kwargs)
-        done = scheduler.start("127.0.0.1")
+        scheduler = Scheduler(validate=True, host="127.0.0.1", **kwargs)
+        done = scheduler.start()
 
         for i in range(nputs):
             q.put(scheduler.address)
@@ -501,7 +497,7 @@ def run_worker(q, scheduler_q, **kwargs):
         with pristine_loop() as loop:
             scheduler_addr = scheduler_q.get()
             worker = Worker(scheduler_addr, validate=True, **kwargs)
-            loop.run_sync(lambda: worker._start(0))
+            loop.run_sync(lambda: worker._start())
             q.put(worker.address)
             try:
 
@@ -521,7 +517,7 @@ def run_nanny(q, scheduler_q, **kwargs):
         with pristine_loop() as loop:
             scheduler_addr = scheduler_q.get()
             worker = Nanny(scheduler_addr, validate=True, **kwargs)
-            loop.run_sync(lambda: worker._start(0))
+            loop.run_sync(lambda: worker._start())
             q.put(worker.address)
             try:
                 loop.start()
@@ -657,6 +653,7 @@ def cluster(
 
             # Launch scheduler
             scheduler = mp_context.Process(
+                name="Dask cluster test: Scheduler",
                 target=run_scheduler,
                 args=(scheduler_q, nworkers + 1),
                 kwargs=scheduler_kwargs,
@@ -675,7 +672,10 @@ def cluster(
                     worker_kwargs,
                 )
                 proc = mp_context.Process(
-                    target=_run_worker, args=(q, scheduler_q), kwargs=kwargs
+                    name="Dask cluster test: Worker",
+                    target=_run_worker,
+                    args=(q, scheduler_q),
+                    kwargs=kwargs,
                 )
                 ws.add(proc)
                 workers.append({"proc": proc, "queue": q, "dir": fn})
@@ -774,6 +774,16 @@ def cluster(
         print("Unclosed Comms", L)
         # raise ValueError("Unclosed Comms", L)
 
+    assert_no_leaked_processes()
+
+
+def assert_no_leaked_processes():
+    for i in range(20):
+        if mp_context.active_children():
+            sleep(0.1)
+    else:
+        assert not mp_context.active_children()
+
 
 @gen.coroutine
 def disconnect(addr, timeout=3, rpc_kwargs=None):
@@ -854,6 +864,7 @@ def start_cluster(
             security=security,
             loop=loop,
             validate=True,
+            host=ncore[0],
             **(merge(worker_kwargs, ncore[2]) if len(ncore) > 2 else worker_kwargs)
         )
         for i, ncore in enumerate(ncores)
@@ -861,7 +872,7 @@ def start_cluster(
     # for w in workers:
     #     w.rpc = workers[0].rpc
 
-    yield [w._start(ncore[0]) for ncore, w in zip(ncores, workers)]
+    yield workers
 
     start = time()
     while len(s.workers) < len(ncores) or any(
@@ -1061,6 +1072,9 @@ def coro():
             _cleanup_dangling()
             with ignoring(AttributeError):
                 del thread_state.on_event_loop_thread
+
+            assert_no_leaked_processes()
+
             return result
 
         return test_func
diff --git a/distributed/worker.py b/distributed/worker.py
index 915784edd88..abbd2376c42 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -30,6 +30,7 @@
 from .batched import BatchedSend
 from .comm import get_address_host, get_local_address_for, connect
 from .comm.utils import offload
+from .comm.addressing import address_from_user_args
 from .compatibility import unicode, get_thread_identity, finalize, MutableMapping
 from .core import error_message, CommClosedError, send_recv, pingpong, coerce_to_address
 from .diskutils import WorkSpace
@@ -295,6 +296,10 @@ def __init__(
         extensions=None,
         metrics=None,
         data=None,
+        interface=None,
+        host=None,
+        port=None,
+        protocol=None,
         low_level_profiler=dask.config.get("distributed.worker.profile.low-level"),
         **kwargs
     ):
@@ -406,7 +411,16 @@ def __init__(
             scheduler_addr = coerce_to_address(scheduler_ip)
         else:
             scheduler_addr = coerce_to_address((scheduler_ip, scheduler_port))
-        self._port = 0
+        self.contact_address = contact_address
+
+        self._start_address = address_from_user_args(
+            host=host,
+            port=port,
+            interface=interface,
+            protocol=protocol,
+            security=security,
+        )
+
         self.ncores = ncores or _ncores
         self.total_resources = resources or {}
         self.available_resources = (resources or {}).copy()
@@ -417,7 +431,6 @@ def __init__(
         self.preload_argv = preload_argv
         if self.preload_argv is None:
             self.preload_argv = dask.config.get("distributed.worker.preload-argv")
-        self.contact_address = contact_address
         self.memory_monitor_interval = parse_timedelta(
             memory_monitor_interval, default="ms"
         )
@@ -888,6 +901,7 @@ def start_services(self, default_listen_ip):
     @gen.coroutine
     def _start(self, addr_or_port=0):
         assert self.status is None
+        addr_or_port = addr_or_port or self._start_address
 
         enable_gc_diagnosis()
         thread_state.on_event_loop_thread = True

From ffe08384d4eb98c7cd8e9891943b662c8301ea32 Mon Sep 17 00:00:00 2001
From: Peter Andreas Entschev <peter@entschev.com>
Date: Thu, 9 May 2019 02:55:52 +0200
Subject: [PATCH 0271/1550] Add memory and disk aliases to Worker.data (#2670)

---
 distributed/tests/test_worker.py | 19 ++++++++++++-------
 distributed/worker.py            | 13 ++++++++++++-
 2 files changed, 24 insertions(+), 8 deletions(-)

diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 8ca3c5d9682..0d8169fd6e5 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -381,17 +381,22 @@ def test_spill_to_disk(c, s):
     yield wait(y)
 
     assert set(w.data) == {x.key, y.key}
-    assert set(w.data.fast) == {x.key, y.key}
+    assert set(w.data.memory) == {x.key, y.key}
+    assert set(w.data.fast) == set(w.data.memory)
 
     z = c.submit(np.random.randint, 0, 255, size=500, dtype="u1", key="z")
     yield wait(z)
     assert set(w.data) == {x.key, y.key, z.key}
-    assert set(w.data.fast) == {y.key, z.key}
-    assert set(w.data.slow) == {x.key} or set(w.data.slow) == {x.key, y.key}
+    assert set(w.data.memory) == {y.key, z.key}
+    assert set(w.data.disk) == {x.key} or set(w.data.slow) == {x.key, y.key}
+    assert set(w.data.fast) == set(w.data.memory)
+    assert set(w.data.slow) == set(w.data.disk)
 
     yield x
-    assert set(w.data.fast) == {x.key, z.key}
-    assert set(w.data.slow) == {y.key} or set(w.data.slow) == {x.key, y.key}
+    assert set(w.data.memory) == {x.key, z.key}
+    assert set(w.data.disk) == {y.key} or set(w.data.slow) == {x.key, y.key}
+    assert set(w.data.fast) == set(w.data.memory)
+    assert set(w.data.slow) == set(w.data.disk)
     yield w.close()
 
 
@@ -460,7 +465,7 @@ def test_spill_by_default(c, s, w):
     x = da.ones(int(10e6 * 0.7), chunks=1e6, dtype="u1")
     y = c.persist(x)
     yield wait(y)
-    assert len(w.data.slow)  # something is on disk
+    assert len(w.data.disk)  # something is on disk
     del x, y
 
 
@@ -1069,7 +1074,7 @@ def f(n):
     futures = c.map(f, [100e6] * 8, pure=False)
 
     start = time()
-    while not a.data.slow:
+    while not a.data.disk:
         yield gen.sleep(0.1)
         assert time() < start + 5
 
diff --git a/distributed/worker.py b/distributed/worker.py
index abbd2376c42..a1846d85539 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -156,7 +156,16 @@ class Worker(ServerNode):
     that we want to collect from others.
 
     * **data:** ``{key: object}``:
-        Dictionary mapping keys to actual values
+        Prefer using the **host** attribute instead of this, unless
+        memory_limit and at least one of memory_target_fraction or
+        memory_spill_fraction values are defined, in that case, this attribute
+        is a zict.Buffer, from which information on LRU cache can be queried.
+    * **data.memory:** ``{key: object}``:
+        Dictionary mapping keys to actual values stored in memory. Only
+        available if condition for **data** being a zict.Buffer is met.
+    * **data.disk:** ``{key: object}``:
+        Dictionary mapping keys to actual values stored on disk. Only
+        available if condition for **data** being a zict.Buffer is met.
     * **task_state**: ``{key: string}``:
         The state of all tasks that the scheduler has asked us to compute.
         Valid states include waiting, constrained, executing, memory, erred
@@ -498,6 +507,8 @@ def __init__(
             )
             target = int(float(self.memory_limit) * self.memory_target_fraction)
             self.data = Buffer({}, storage, target, weight)
+            self.data.memory = self.data.fast
+            self.data.disk = self.data.slow
         else:
             self.data = dict()
 

From 0be60cccfb6b8739b119f7126eef94dd3deff609 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Thu, 9 May 2019 08:01:35 -0500
Subject: [PATCH 0272/1550] Use config accessor method for "scheduler-address"
 (#2676)

---
 distributed/cli/dask_worker.py            |  8 ++++++--
 distributed/cli/tests/test_dask_worker.py | 12 ++++++++++++
 2 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
index 6315939005d..1448395d109 100755
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -8,8 +8,8 @@
 import warnings
 
 import click
+import dask
 from distributed import Nanny, Worker
-from distributed.config import config
 from distributed.utils import parse_timedelta
 from distributed.worker import _ncores
 from distributed.security import Security
@@ -318,7 +318,11 @@ def del_pid_file():
             kwargs["service_ports"] = {"nanny": nanny_port}
         t = Worker
 
-    if not scheduler and not scheduler_file and "scheduler-address" not in config:
+    if (
+        not scheduler
+        and not scheduler_file
+        and dask.config.get("scheduler-address", None) is None
+    ):
         raise ValueError(
             "Need to provide scheduler address like\n"
             "dask-worker SCHEDULER_ADDRESS:8786"
diff --git a/distributed/cli/tests/test_dask_worker.py b/distributed/cli/tests/test_dask_worker.py
index 2fa3779d9b4..72f8327375a 100644
--- a/distributed/cli/tests/test_dask_worker.py
+++ b/distributed/cli/tests/test_dask_worker.py
@@ -6,6 +6,7 @@
 
 import requests
 import sys
+import os
 from time import sleep
 
 from distributed import Client
@@ -141,6 +142,17 @@ def test_scheduler_file(loop, nanny):
                         assert time() < start + 10
 
 
+def test_scheduler_address_env(loop, monkeypatch):
+    monkeypatch.setenv("DASK_SCHEDULER_ADDRESS", "tcp://127.0.0.1:8786")
+    with popen(["dask-scheduler", "--no-bokeh"]) as sched:
+        with popen(["dask-worker", "--no-bokeh"]):
+            with Client(os.environ["DASK_SCHEDULER_ADDRESS"], loop=loop) as c:
+                start = time()
+                while not c.scheduler_info()["workers"]:
+                    sleep(0.1)
+                    assert time() < start + 10
+
+
 def test_nprocs_requires_nanny(loop):
     with popen(["dask-scheduler", "--no-bokeh"]) as sched:
         with popen(

From 22c733ef852af72bb281af8f1534e6a942c96f8b Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Thu, 9 May 2019 14:44:43 -0500
Subject: [PATCH 0273/1550] Fix pytest.config deprecation warning (#2677)

---
 conftest.py                                 | 16 +++++++++---
 distributed/cli/tests/test_dask_worker.py   |  4 +--
 distributed/protocol/tests/test_numpy.py    |  4 +--
 distributed/protocol/tests/test_protocol.py |  3 +--
 distributed/tests/test_batched.py           |  6 ++---
 distributed/tests/test_client.py            | 27 ++++++++++-----------
 distributed/tests/test_core.py              |  3 +--
 distributed/tests/test_diskutils.py         |  4 +--
 distributed/tests/test_failed_workers.py    |  3 +--
 distributed/tests/test_nanny.py             |  8 +++---
 distributed/tests/test_queues.py            |  4 +--
 distributed/tests/test_scheduler.py         |  9 +++----
 distributed/tests/test_stress.py            |  5 ++--
 distributed/tests/test_variable.py          |  4 +--
 distributed/tests/test_worker.py            |  9 +++----
 distributed/utils_test.py                   | 11 ---------
 16 files changed, 56 insertions(+), 64 deletions(-)

diff --git a/conftest.py b/conftest.py
index cba68bddec1..b5db36f59d8 100644
--- a/conftest.py
+++ b/conftest.py
@@ -1,12 +1,11 @@
 # https://pytest.org/latest/example/simple.html#control-skipping-of-tests-according-to-command-line-option
-import os
 import pytest
 
 
 # Uncomment to enable more logging and checks
 # (https://docs.python.org/3/library/asyncio-dev.html)
 # Note this makes things slower and might consume much memory.
-#os.environ["PYTHONASYNCIODEBUG"] = "1"
+# os.environ["PYTHONASYNCIODEBUG"] = "1"
 
 try:
     import faulthandler
@@ -19,4 +18,15 @@
 def pytest_addoption(parser):
     parser.addoption("--runslow", action="store_true", help="run slow tests")
 
-pytest_plugins = ['distributed.pytest_resourceleaks']
+
+def pytest_collection_modifyitems(config, items):
+    if config.getoption("--runslow"):
+        # --runslow given in cli: do not skip slow tests
+        return
+    skip_slow = pytest.mark.skip(reason="need --runslow option to run")
+    for item in items:
+        if "slow" in item.keywords:
+            item.add_marker(skip_slow)
+
+
+pytest_plugins = ["distributed.pytest_resourceleaks"]
diff --git a/distributed/cli/tests/test_dask_worker.py b/distributed/cli/tests/test_dask_worker.py
index 72f8327375a..eec038ba9d6 100644
--- a/distributed/cli/tests/test_dask_worker.py
+++ b/distributed/cli/tests/test_dask_worker.py
@@ -12,7 +12,7 @@
 from distributed import Client
 from distributed.metrics import time
 from distributed.utils import sync, tmpfile
-from distributed.utils_test import popen, slow, terminate_process, wait_for_port
+from distributed.utils_test import popen, terminate_process, wait_for_port
 from distributed.utils_test import loop  # noqa: F401
 
 
@@ -65,7 +65,7 @@ def test_no_nanny(loop):
             assert any(b"Registered" in worker.stderr.readline() for i in range(15))
 
 
-@slow
+@pytest.mark.slow
 @pytest.mark.parametrize("nanny", ["--nanny", "--no-nanny"])
 def test_no_reconnect(nanny, loop):
     with popen(["dask-scheduler", "--no-bokeh"]) as sched:
diff --git a/distributed/protocol/tests/test_numpy.py b/distributed/protocol/tests/test_numpy.py
index 849c2964fd6..ede0eded3cf 100644
--- a/distributed/protocol/tests/test_numpy.py
+++ b/distributed/protocol/tests/test_numpy.py
@@ -18,7 +18,7 @@
 )
 from distributed.protocol.utils import BIG_BYTES_SHARD_SIZE
 from distributed.utils import tmpfile, nbytes
-from distributed.utils_test import slow, gen_cluster
+from distributed.utils_test import gen_cluster
 from distributed.protocol.numpy import itemsize
 from distributed.protocol.compression import maybe_compress
 
@@ -152,7 +152,7 @@ def test_memmap():
         np.testing.assert_equal(x, y)
 
 
-@slow
+@pytest.mark.slow
 def test_dumps_serialize_numpy_large():
     psutil = pytest.importorskip("psutil")
     if psutil.virtual_memory().total < 2e9:
diff --git a/distributed/protocol/tests/test_protocol.py b/distributed/protocol/tests/test_protocol.py
index f0dc1dc6c2f..2415e01b5f1 100644
--- a/distributed/protocol/tests/test_protocol.py
+++ b/distributed/protocol/tests/test_protocol.py
@@ -9,7 +9,6 @@
 from distributed.protocol.compression import compressions
 from distributed.protocol.serialize import Serialize, Serialized, serialize, deserialize
 from distributed.utils import nbytes
-from distributed.utils_test import slow
 
 
 def test_protocol():
@@ -110,7 +109,7 @@ def test_large_bytes():
         assert loads(frames, deserialize=False) == msg
 
 
-@slow
+@pytest.mark.slow
 def test_large_messages():
     np = pytest.importorskip("numpy")
     psutil = pytest.importorskip("psutil")
diff --git a/distributed/tests/test_batched.py b/distributed/tests/test_batched.py
index 2f22134f7ae..23d8e677774 100644
--- a/distributed/tests/test_batched.py
+++ b/distributed/tests/test_batched.py
@@ -10,7 +10,7 @@
 from distributed.core import listen, connect, CommClosedError
 from distributed.metrics import time
 from distributed.utils import All
-from distributed.utils_test import gen_test, slow, captured_logger
+from distributed.utils_test import gen_test, captured_logger
 from distributed.protocol import to_serialize
 
 
@@ -158,7 +158,7 @@ def test_close_twice():
         yield b.close()
 
 
-@slow
+@pytest.mark.slow
 @gen_test(timeout=50)
 def test_stress():
     with echo_server() as e:
@@ -231,7 +231,7 @@ def test_sending_traffic_jam():
     yield run_traffic_jam(50, 300000)
 
 
-@slow
+@pytest.mark.slow
 @gen_test()
 def test_large_traffic_jam():
     yield run_traffic_jam(500, 1500000)
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 3515be9ebcb..b77efc9f51d 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -60,7 +60,6 @@
 from distributed.utils import ignoring, mp_context, sync, tmp_text, tokey, tmpfile
 from distributed.utils_test import (
     cluster,
-    slow,
     slowinc,
     slowadd,
     slowdec,
@@ -771,7 +770,7 @@ def test_recompute_released_key(c, s, a, b):
     assert result1 == result2
 
 
-@slow
+@pytest.mark.slow
 @gen_cluster(client=True)
 def test_long_tasks_dont_trigger_timeout(c, s, a, b):
     from time import sleep
@@ -3473,7 +3472,7 @@ def test_get_foo_lost_keys(c, s, u, v, w):
     assert_dict_key_equal(d, {x.key: [], y.key: []})
 
 
-@slow
+@pytest.mark.slow
 @gen_cluster(client=True, Worker=Nanny, check_new_threads=False)
 def test_bad_tasks_fail(c, s, a, b):
     f = c.submit(sys.exit, 1)
@@ -3528,7 +3527,7 @@ def test_get_returns_early(c):
     assert x.key in c.futures
 
 
-@slow
+@pytest.mark.slow
 @gen_cluster(Worker=Nanny, client=True)
 def test_Client_clears_references_after_restart(c, s, a, b):
     x = c.submit(inc, 1)
@@ -3644,7 +3643,7 @@ def test_scatter_raises_if_no_workers(c, s):
         yield c.scatter(1, timeout=0.5)
 
 
-@slow
+@pytest.mark.slow
 def test_reconnect(loop):
     w = Worker("127.0.0.1", 9393, loop=loop)
     w.start()
@@ -3721,7 +3720,7 @@ def test_reconnect_timeout(c, s):
     assert "Failed to reconnect" in text
 
 
-@slow
+@pytest.mark.slow
 @pytest.mark.skipif(
     sys.platform.startswith("win"), reason="num_fds not supported on windows"
 )
@@ -4251,7 +4250,7 @@ def test_normalize_collection_dask_array(c, s, a, b):
     assert result1 == result2
 
 
-@slow
+@pytest.mark.slow
 def test_normalize_collection_with_released_futures(c):
     da = pytest.importorskip("dask.array")
 
@@ -4382,7 +4381,7 @@ def test_scatter_dict_workers(c, s, a, b):
     assert "a" in a.data or "a" in b.data
 
 
-@slow
+@pytest.mark.slow
 @gen_test()
 def test_client_timeout():
     loop = IOLoop.current()
@@ -4710,7 +4709,7 @@ def test_quiet_client_close(loop):
             ), line
 
 
-@slow
+@pytest.mark.slow
 def test_quiet_client_close_when_cluster_is_closed_before_client(loop):
     with captured_logger(logging.getLogger("tornado.application")) as logger:
         cluster = LocalCluster(loop=loop, n_workers=1)
@@ -4755,7 +4754,7 @@ def f(_):
         del results
 
 
-@slow
+@pytest.mark.slow
 def test_threadsafe_get(c):
     da = pytest.importorskip("dask.array")
     x = da.arange(100, chunks=(10,))
@@ -4774,7 +4773,7 @@ def f(_):
     assert results and all(results)
 
 
-@slow
+@pytest.mark.slow
 def test_threadsafe_compute(c):
     da = pytest.importorskip("dask.array")
     x = da.arange(100, chunks=(10,))
@@ -4864,7 +4863,7 @@ def f():
     assert result == 2
 
 
-@slow
+@pytest.mark.slow
 @gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 2, timeout=60)
 def test_secede_balances(c, s, a, b):
     count = threading.active_count()
@@ -4970,7 +4969,7 @@ def test_dynamic_workloads_sync(c):
     _test_dynamic_workloads_sync(c, delay=0.02)
 
 
-@slow
+@pytest.mark.slow
 def test_dynamic_workloads_sync_random(c):
     _test_dynamic_workloads_sync(c, delay="random")
 
@@ -5190,7 +5189,7 @@ def test_client_async_before_loop_starts():
         client.close()
 
 
-@slow
+@pytest.mark.slow
 @gen_cluster(
     client=True,
     Worker=Nanny if PY3 else Worker,
diff --git a/distributed/tests/test_core.py b/distributed/tests/test_core.py
index 2c8f63de6ff..4b3c0ac0ade 100644
--- a/distributed/tests/test_core.py
+++ b/distributed/tests/test_core.py
@@ -25,7 +25,6 @@
 from distributed.protocol import to_serialize
 from distributed.utils import get_ip, get_ipv6
 from distributed.utils_test import (
-    slow,
     gen_test,
     gen_cluster,
     has_ipv6,
@@ -409,7 +408,7 @@ def check_large_packets(listen_arg):
     server.stop()
 
 
-@slow
+@pytest.mark.slow
 @gen_test()
 def test_large_packets_tcp():
     yield check_large_packets("tcp://")
diff --git a/distributed/tests/test_diskutils.py b/distributed/tests/test_diskutils.py
index d5abf5c1dee..1bededf84ab 100644
--- a/distributed/tests/test_diskutils.py
+++ b/distributed/tests/test_diskutils.py
@@ -16,7 +16,7 @@
 from distributed.diskutils import WorkSpace
 from distributed.metrics import time
 from distributed.utils import mp_context
-from distributed.utils_test import captured_logger, slow
+from distributed.utils_test import captured_logger
 
 
 def assert_directory_contents(dir_path, expected, trials=2):
@@ -279,7 +279,7 @@ def test_workspace_concurrency(tmpdir):
     _test_workspace_concurrency(tmpdir, 2.0, 6)
 
 
-@slow
+@pytest.mark.slow
 def test_workspace_concurrency_intense(tmpdir):
     n_created, n_purged = _test_workspace_concurrency(tmpdir, 8.0, 16)
     assert n_created >= 100
diff --git a/distributed/tests/test_failed_workers.py b/distributed/tests/test_failed_workers.py
index 0772ea52c32..bae2e141ee2 100644
--- a/distributed/tests/test_failed_workers.py
+++ b/distributed/tests/test_failed_workers.py
@@ -19,7 +19,6 @@
     gen_cluster,
     cluster,
     inc,
-    slow,
     div,
     slowinc,
     slowadd,
@@ -406,7 +405,7 @@ def test_worker_who_has_clears_after_failed_connection(c, s, a, b):
     yield n.close()
 
 
-@slow
+@pytest.mark.slow
 @gen_cluster(client=True, timeout=60, Worker=Nanny, ncores=[("127.0.0.1", 1)])
 def test_restart_timeout_on_long_running_task(c, s, a):
     with captured_logger("distributed.scheduler") as sio:
diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
index 08cd49fb3c9..60de12dce4b 100644
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -18,7 +18,7 @@
 from distributed.metrics import time
 from distributed.protocol.pickle import dumps
 from distributed.utils import ignoring, tmpfile
-from distributed.utils_test import gen_cluster, gen_test, slow, inc, captured_logger
+from distributed.utils_test import gen_cluster, gen_test, inc, captured_logger
 
 
 @gen_cluster(ncores=[])
@@ -127,7 +127,7 @@ def test_run(s):
     yield n.close()
 
 
-@slow
+@pytest.mark.slow
 @gen_cluster(
     Worker=Nanny, ncores=[("127.0.0.1", 1)], worker_kwargs={"reconnect": False}
 )
@@ -159,7 +159,7 @@ def test_nanny_alt_worker_class(c, s, w1, w2):
     assert w1.Worker is Something
 
 
-@slow
+@pytest.mark.slow
 @gen_cluster(client=False, ncores=[])
 def test_nanny_death_timeout(s):
     yield s.close()
@@ -318,7 +318,7 @@ def test_scheduler_address_config(c, s):
     yield nanny.close()
 
 
-@slow
+@pytest.mark.slow
 @gen_test()
 def test_wait_for_scheduler():
     with captured_logger("distributed") as log:
diff --git a/distributed/tests/test_queues.py b/distributed/tests/test_queues.py
index e82b893989b..e40d3cd492c 100644
--- a/distributed/tests/test_queues.py
+++ b/distributed/tests/test_queues.py
@@ -9,7 +9,7 @@
 
 from distributed import Client, Queue, Nanny, worker_client, wait
 from distributed.metrics import time
-from distributed.utils_test import gen_cluster, inc, slow, div
+from distributed.utils_test import gen_cluster, inc, div
 from distributed.utils_test import client, cluster_fixture, loop  # noqa: F401
 
 
@@ -114,7 +114,7 @@ def f(x):
 
 
 @pytest.mark.skipif(sys.version_info[0] == 2, reason="Multi-client issues")
-@slow
+@pytest.mark.slow
 @gen_cluster(client=True, ncores=[("127.0.0.1", 2)] * 5, Worker=Nanny, timeout=None)
 def test_race(c, s, *workers):
     def f(i):
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 805b0e06ed0..07caff09869 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -35,7 +35,6 @@
     cluster,
     div,
     varying,
-    slow,
 )
 from distributed.utils_test import loop, nodebug  # noqa: F401
 from dask.compatibility import apply
@@ -775,7 +774,7 @@ def test_retire_workers_no_suspicious_tasks(c, s, a, b):
     assert all(ts.suspicious == 0 for ts in s.tasks.values())
 
 
-@slow
+@pytest.mark.slow
 @pytest.mark.skipif(
     sys.platform.startswith("win"), reason="file descriptors not really a thing"
 )
@@ -831,7 +830,7 @@ def test_file_descriptors(c, s):
         assert time() < start + 3
 
 
-@slow
+@pytest.mark.slow
 @nodebug
 @gen_cluster(client=True)
 def test_learn_occupancy(c, s, a, b):
@@ -844,7 +843,7 @@ def test_learn_occupancy(c, s, a, b):
         assert 50 < s.workers[w.address].occupancy < 700
 
 
-@slow
+@pytest.mark.slow
 @nodebug
 @gen_cluster(client=True)
 def test_learn_occupancy_2(c, s, a, b):
@@ -1062,7 +1061,7 @@ def test_close_worker(c, s, a, b):
     assert len(s.workers) == 1
 
 
-@slow
+@pytest.mark.slow
 @gen_cluster(client=True, Worker=Nanny, timeout=20)
 def test_close_nanny(c, s, a, b):
     assert len(s.workers) == 2
diff --git a/distributed/tests/test_stress.py b/distributed/tests/test_stress.py
index f145a11b053..8c37f5a82fb 100644
--- a/distributed/tests/test_stress.py
+++ b/distributed/tests/test_stress.py
@@ -21,7 +21,6 @@
     inc,
     slowinc,
     slowadd,
-    slow,
     slowsum,
     bump_rlimit,
 )
@@ -198,7 +197,7 @@ def vsum(*args):
 
 
 @pytest.mark.avoid_travis
-@slow
+@pytest.mark.slow
 @gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 80, timeout=1000)
 def test_stress_communication(c, s, *workers):
     s.validate = False  # very slow otherwise
@@ -244,7 +243,7 @@ def test_stress_steal(c, s, *workers):
             break
 
 
-@slow
+@pytest.mark.slow
 @gen_cluster(ncores=[("127.0.0.1", 1)] * 10, client=True, timeout=120)
 def test_close_connections(c, s, *workers):
     da = pytest.importorskip("dask.array")
diff --git a/distributed/tests/test_variable.py b/distributed/tests/test_variable.py
index 5ae94d037c5..4d8851668f9 100644
--- a/distributed/tests/test_variable.py
+++ b/distributed/tests/test_variable.py
@@ -9,7 +9,7 @@
 
 from distributed import Client, Variable, worker_client, Nanny, wait
 from distributed.metrics import time
-from distributed.utils_test import gen_cluster, inc, slow, div
+from distributed.utils_test import gen_cluster, inc, div
 from distributed.utils_test import client, cluster_fixture, loop  # noqa: F401
 
 
@@ -147,7 +147,7 @@ def test_timeout_get(c, s, a, b):
 
 
 @pytest.mark.skipif(sys.version_info[0] == 2, reason="Multi-client issues")
-@slow
+@pytest.mark.slow
 @gen_cluster(client=True, ncores=[("127.0.0.1", 2)] * 5, Worker=Nanny, timeout=None)
 def test_race(c, s, *workers):
     NITERS = 50
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 0d8169fd6e5..5fad86b2665 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -33,7 +33,6 @@
     gen_cluster,
     div,
     dec,
-    slow,
     slowinc,
     gen_test,
     captured_logger,
@@ -147,7 +146,7 @@ def reset(self):
     assert tuple(results) == (3, 7)
 
 
-@slow
+@pytest.mark.slow
 @gen_cluster()
 def dont_test_delete_data_with_missing_worker(c, a, b):
     bad = "127.0.0.1:9001"  # this worker doesn't exist
@@ -312,7 +311,7 @@ def test_worker_with_port_zero():
     yield w.close()
 
 
-@slow
+@pytest.mark.slow
 def test_worker_waits_for_center_to_come_up(loop):
     @gen.coroutine
     def f():
@@ -726,7 +725,7 @@ def test_hold_onto_dependents(c, s, a, b):
     assert x.key in b.data
 
 
-@slow
+@pytest.mark.slow
 @gen_cluster(client=False, ncores=[])
 def test_worker_death_timeout(s):
     with dask.config.set({"distributed.comm.timeouts.connect": "1s"}):
@@ -1235,7 +1234,7 @@ def test_scheduler_address_config(c, s):
     yield worker.close()
 
 
-@slow
+@pytest.mark.slow
 @gen_cluster(client=True)
 def test_wait_for_outgoing(c, s, a, b):
     np = pytest.importorskip("numpy")
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index 7aaa5b1ed0d..0a1cf447cfd 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -804,17 +804,6 @@ def disconnect_all(addresses, timeout=3, rpc_kwargs=None):
     yield [disconnect(addr, timeout, rpc_kwargs) for addr in addresses]
 
 
-def slow(func):
-    try:
-        if not pytest.config.getoption("--runslow"):
-            func = pytest.mark.skip("need --runslow option to run")(func)
-    except AttributeError:
-        # AttributeError: module 'pytest' has no attribute 'config'
-        pass
-
-    return nodebug(func)
-
-
 def gen_test(timeout=10):
     """ Coroutine test
 

From 9ea2dc3fdf0484339d13be3fb4485a782c6f4696 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 9 May 2019 14:53:57 -0500
Subject: [PATCH 0274/1550] Move dashboard_address logic into Scheduler/Worker
 (#2678)

This removes repetitive logic from the LocalCluster and
dask-scheduler/dask-worker CLI and moves it into the classes.
This also makes it easier to make other Cluster objects without
depending on LocalCluster

* fix test_file_descriptors
---
 distributed/cli/dask_scheduler.py   | 18 ++----------------
 distributed/cli/dask_worker.py      | 16 +++-------------
 distributed/deploy/local.py         | 25 ++++++++++---------------
 distributed/scheduler.py            | 13 +++++++++++++
 distributed/tests/test_scheduler.py |  2 +-
 distributed/worker.py               | 14 ++++++++++++++
 6 files changed, 43 insertions(+), 45 deletions(-)

diff --git a/distributed/cli/dask_scheduler.py b/distributed/cli/dask_scheduler.py
index 57a7168a3a2..3668be684d0 100755
--- a/distributed/cli/dask_scheduler.py
+++ b/distributed/cli/dask_scheduler.py
@@ -188,29 +188,15 @@ def del_pid_file():
     loop = IOLoop.current()
     logger.info("-" * 47)
 
-    services = {}
-    if _bokeh:
-        try:
-            from distributed.bokeh.scheduler import BokehScheduler
-
-            services[("bokeh", dashboard_address)] = (
-                BokehScheduler,
-                {"prefix": bokeh_prefix},
-            )
-        except ImportError as error:
-            if str(error).startswith("No module named"):
-                logger.info("Web dashboard not loaded.  Unable to import bokeh")
-            else:
-                logger.info("Unable to import bokeh: %s" % str(error))
-
     scheduler = Scheduler(
         loop=loop,
-        services=services,
         scheduler_file=scheduler_file,
         security=sec,
         host=host,
         port=port,
         interface=interface,
+        dashboard_address=dashboard_address if _bokeh else None,
+        service_kwargs={"bokeh": {"prefix": bokeh_prefix}},
     )
     scheduler.start()
     if not preload:
diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
index 1448395d109..e383095b382 100755
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -163,7 +163,7 @@
     default=None,
     help="Seconds to wait for a scheduler before closing",
 )
-@click.option("--bokeh-prefix", type=str, default=None, help="Prefix for the bokeh app")
+@click.option("--bokeh-prefix", type=str, default="", help="Prefix for the bokeh app")
 @click.option(
     "--preload",
     type=str,
@@ -288,18 +288,6 @@ def del_pid_file():
 
     services = {}
 
-    if bokeh:
-        try:
-            from distributed.bokeh.worker import BokehWorker
-        except ImportError:
-            pass
-        else:
-            if bokeh_prefix:
-                result = (BokehWorker, {"prefix": bokeh_prefix})
-            else:
-                result = BokehWorker
-            services[("bokeh", dashboard_address)] = result
-
     if resources:
         resources = resources.replace(",", " ").split()
         resources = dict(pair.split("=") for pair in resources)
@@ -350,6 +338,8 @@ def del_pid_file():
             interface=interface,
             host=host,
             port=port,
+            dashboard_address=dashboard_address if bokeh else None,
+            service_kwargs={"bokhe": {"prefix": bokeh_prefix}},
             name=name if nprocs == 1 or not name else name + "-" + str(i),
             **kwargs
         )
diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index 3431210a645..fb8793d0840 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -57,7 +57,7 @@ class LocalCluster(Cluster):
         Address on which to listen for the Bokeh diagnostics server like
         'localhost:8787' or '0.0.0.0:8787'.  Defaults to ':8787'.
         Set to ``None`` to disable the dashboard.
-        Use port 0 for a random port.
+        Use ':0' for a random port.
     diagnostics_port: int
         Deprecated.  See dashboard_address.
     asynchronous: bool (False by default)
@@ -112,6 +112,7 @@ def __init__(
         scheduler_port=0,
         silence_logs=logging.WARN,
         dashboard_address=":8787",
+        worker_dashboard_address=None,
         diagnostics_port=None,
         services=None,
         worker_services=None,
@@ -179,29 +180,23 @@ def __init__(
             worker_kwargs["memory_limit"] = parse_memory_limit("auto", 1, n_workers)
 
         worker_kwargs.update(
-            {"ncores": threads_per_worker, "services": worker_services}
+            {
+                "ncores": threads_per_worker,
+                "services": worker_services,
+                "dashboard_address": worker_dashboard_address,
+            }
         )
 
         self._loop_runner = LoopRunner(loop=loop, asynchronous=asynchronous)
         self.loop = self._loop_runner.loop
 
-        if dashboard_address is not False and dashboard_address is not None:
-            try:
-                from distributed.bokeh.scheduler import BokehScheduler
-                from distributed.bokeh.worker import BokehWorker
-            except ImportError:
-                logger.debug("To start diagnostics web server please install Bokeh")
-            else:
-                services[("bokeh", dashboard_address)] = (
-                    BokehScheduler,
-                    (service_kwargs or {}).get("bokeh", {}),
-                )
-                worker_services[("bokeh", 0)] = BokehWorker
-
         self.scheduler = Scheduler(
             loop=self.loop,
             services=services,
+            service_kwargs=service_kwargs,
             security=security,
+            interface=interface,
+            dashboard_address=dashboard_address,
             blocked_handlers=blocked_handlers,
         )
         self.scheduler_port = scheduler_port
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 8ba4cedf468..8500150204e 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -818,6 +818,7 @@ def __init__(
         delete_interval="500ms",
         synchronize_worker_interval="60s",
         services=None,
+        service_kwargs=None,
         allowed_failures=ALLOWED_FAILURES,
         extensions=None,
         validate=False,
@@ -829,6 +830,7 @@ def __init__(
         host=None,
         port=8786,
         protocol=None,
+        dashboard_address=None,
         **kwargs
     ):
         self._setup_logging()
@@ -862,6 +864,17 @@ def __init__(
         self.connection_args = self.security.get_connection_args("scheduler")
         self.listen_args = self.security.get_listen_args("scheduler")
 
+        if dashboard_address is not None:
+            try:
+                from distributed.bokeh.scheduler import BokehScheduler
+            except ImportError:
+                logger.debug("To start diagnostics web server please install Bokeh")
+            else:
+                self.service_specs[("bokeh", dashboard_address)] = (
+                    BokehScheduler,
+                    (service_kwargs or {}).get("bokeh", {}),
+                )
+
         # Communication state
         self.loop = loop or IOLoop.current()
         self.client_comms = dict()
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 07caff09869..3cc6579ed29 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -821,7 +821,7 @@ def test_file_descriptors(c, s):
     yield [n.close() for n in nannies]
 
     assert not s.rpc.open
-    assert not c.rpc.active, list(c.rpc._created)
+    assert not any(occ for addr, occ in c.rpc.occupied.items() if occ != s.address)
     assert not s.stream_comms
 
     start = time()
diff --git a/distributed/worker.py b/distributed/worker.py
index a1846d85539..1b103fe144a 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -290,6 +290,7 @@ def __init__(
         local_dir="dask-worker-space",
         services=None,
         service_ports=None,
+        service_kwargs=None,
         name=None,
         reconnect=True,
         memory_limit="auto",
@@ -309,6 +310,7 @@ def __init__(
         host=None,
         port=None,
         protocol=None,
+        dashboard_address=None,
         low_level_profiler=dask.config.get("distributed.worker.profile.low-level"),
         **kwargs
     ):
@@ -535,6 +537,18 @@ def __init__(
         self.services = {}
         self.service_ports = service_ports or {}
         self.service_specs = services or {}
+
+        if dashboard_address is not None:
+            try:
+                from distributed.bokeh.worker import BokehWorker
+            except ImportError:
+                logger.debug("To start diagnostics web server please install Bokeh")
+            else:
+                self.service_specs[("bokeh", dashboard_address)] = (
+                    BokehWorker,
+                    (service_kwargs or {}).get("bokeh", {}),
+                )
+
         self.metrics = dict(metrics) if metrics else {}
 
         self.low_level_profiler = low_level_profiler

From 94dd92ebc2345a326550a6b2e3f2de776727712f Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Fri, 10 May 2019 08:29:31 -0500
Subject: [PATCH 0275/1550] Fix uri_from_host_port import in dask-mpi (#2683)

---
 distributed/cli/dask_mpi.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/distributed/cli/dask_mpi.py b/distributed/cli/dask_mpi.py
index ef7dd0c59fa..398596508a3 100644
--- a/distributed/cli/dask_mpi.py
+++ b/distributed/cli/dask_mpi.py
@@ -8,7 +8,8 @@
 
 from distributed import Scheduler, Nanny, Worker
 from distributed.bokeh.worker import BokehWorker
-from distributed.cli.utils import check_python_3, uri_from_host_port
+from distributed.cli.utils import check_python_3
+from distributed.comm.addressing import uri_from_host_port
 from distributed.utils import get_ip_interface
 
 

From 14998926603310416eed91d0f6181bf73a6b1fb8 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 10 May 2019 08:43:21 -0500
Subject: [PATCH 0276/1550] Consolidate logic around services (#2679)

We move shared logic for services down to the Server class.
Additionally we remove special casing of the nanny in service_ports, and
instead tack on a nanny attribute to the Worker and WorkerState
directly.
---
 .../bokeh/tests/test_scheduler_bokeh.py       | 10 +--
 distributed/cli/tests/test_dask_worker.py     |  5 +-
 distributed/nanny.py                          |  2 +-
 distributed/node.py                           | 52 +++++++++++++++
 distributed/scheduler.py                      | 63 +++++--------------
 distributed/tests/test_client.py              |  2 +-
 distributed/tests/test_nanny.py               |  4 +-
 distributed/tests/test_scheduler.py           | 12 ++++
 distributed/worker.py                         | 43 +++----------
 9 files changed, 99 insertions(+), 94 deletions(-)

diff --git a/distributed/bokeh/tests/test_scheduler_bokeh.py b/distributed/bokeh/tests/test_scheduler_bokeh.py
index 380dff104e2..f3a57586c72 100644
--- a/distributed/bokeh/tests/test_scheduler_bokeh.py
+++ b/distributed/bokeh/tests/test_scheduler_bokeh.py
@@ -565,10 +565,12 @@ def test_GraphPlot_order(c, s, a, b):
 )
 def test_profile_server(c, s, a, b):
     ptp = ProfileServer(s)
-    ptp.trigger_update()
-    yield gen.sleep(0.200)
-    ptp.trigger_update()
-    assert 2 < len(ptp.ts_source.data["time"]) < 20
+    start = time()
+    yield gen.sleep(0.100)
+    while len(ptp.ts_source.data["time"]) < 2:
+        yield gen.sleep(0.100)
+        ptp.trigger_update()
+        assert time() < start + 2
 
 
 @gen_cluster(client=True, scheduler_kwargs={"services": {("bokeh", 0): BokehScheduler}})
diff --git a/distributed/cli/tests/test_dask_worker.py b/distributed/cli/tests/test_dask_worker.py
index eec038ba9d6..c26c99f2350 100644
--- a/distributed/cli/tests/test_dask_worker.py
+++ b/distributed/cli/tests/test_dask_worker.py
@@ -40,7 +40,10 @@ def test_nanny_worker_ports(loop):
                     else:
                         assert time() - start < 5
                         sleep(0.1)
-                assert d["workers"]["tcp://127.0.0.1:9684"]["services"]["nanny"] == 5273
+                assert (
+                    d["workers"]["tcp://127.0.0.1:9684"]["nanny"]
+                    == "tcp://127.0.0.1:5273"
+                )
 
 
 def test_memory_limit(loop):
diff --git a/distributed/nanny.py b/distributed/nanny.py
index ef0e0a38f0e..4bc0eeef6f9 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -259,7 +259,7 @@ def instantiate(self, comm=None):
                 ncores=self.ncores,
                 local_dir=self.local_dir,
                 services=self.services,
-                service_ports={"nanny": self.port},
+                nanny=self.address,
                 name=self.name,
                 memory_limit=self.memory_limit,
                 reconnect=self.reconnect,
diff --git a/distributed/node.py b/distributed/node.py
index 8134546fa0b..ff95a621877 100644
--- a/distributed/node.py
+++ b/distributed/node.py
@@ -1,7 +1,10 @@
 from __future__ import print_function, division, absolute_import
 
+import warnings
+
 from tornado.ioloop import IOLoop
 
+from .compatibility import unicode
 from .core import Server, ConnectionPool
 from .versions import get_versions
 
@@ -78,3 +81,52 @@ def __init__(
 
     def versions(self, comm=None, packages=None):
         return get_versions(packages=packages)
+
+    def start_services(self, default_listen_ip):
+        if default_listen_ip == "0.0.0.0":
+            default_listen_ip = ""  # for IPV6
+
+        for k, v in self.service_specs.items():
+            listen_ip = None
+            if isinstance(k, tuple):
+                k, port = k
+            else:
+                port = 0
+
+            if isinstance(port, (str, unicode)):
+                port = port.split(":")
+
+            if isinstance(port, (tuple, list)):
+                if len(port) == 2:
+                    listen_ip, port = (port[0], int(port[1]))
+                elif len(port) == 1:
+                    [listen_ip], port = port, 0
+                else:
+                    raise ValueError(port)
+
+            if isinstance(v, tuple):
+                v, kwargs = v
+            else:
+                kwargs = {}
+
+            try:
+                service = v(self, io_loop=self.loop, **kwargs)
+                service.listen(
+                    (listen_ip if listen_ip is not None else default_listen_ip, port)
+                )
+                self.services[k] = service
+            except Exception as e:
+                warnings.warn(
+                    "\nCould not launch service '%s' on port %s. " % (k, port)
+                    + "Got the following message:\n\n"
+                    + str(e),
+                    stacklevel=3,
+                )
+
+    def stop_services(self):
+        for service in self.services.values():
+            service.stop()
+
+    @property
+    def service_ports(self):
+        return {k: v.port for k, v in self.services.items()}
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 8500150204e..87cc4fda8a4 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -12,7 +12,6 @@
 import pickle
 import random
 import six
-import warnings
 
 import psutil
 import sortedcontainers
@@ -190,6 +189,10 @@ class WorkerState(object):
 
        The current status of the worker, either ``'running'`` or ``'closed'``
 
+    .. attribute:: nanny: str
+
+       Address of the associated Nanny, if present
+
     .. attribute:: last_seen: Number
 
        The last time we received a heartbeat from this worker, in local
@@ -214,6 +217,7 @@ class WorkerState(object):
         "memory_limit",
         "metrics",
         "name",
+        "nanny",
         "nbytes",
         "ncores",
         "occupancy",
@@ -235,6 +239,7 @@ def __init__(
         memory_limit=0,
         local_directory=None,
         services=None,
+        nanny=None,
     ):
         self.address = address
         self.pid = pid
@@ -243,6 +248,7 @@ def __init__(
         self.memory_limit = memory_limit
         self.local_directory = local_directory
         self.services = services or {}
+        self.nanny = nanny
 
         self.status = "running"
         self.nbytes = 0
@@ -271,6 +277,7 @@ def clean(self):
             memory_limit=self.memory_limit,
             local_directory=self.local_directory,
             services=self.services,
+            nanny=self.nanny,
         )
         ws.processing = {ts.key for ts in self.processing}
         return ws
@@ -298,6 +305,7 @@ def identity(self):
             "last_seen": self.last_seen,
             "services": self.services,
             "metrics": self.metrics,
+            "nanny": self.nanny,
         }
 
 
@@ -1157,46 +1165,6 @@ def get_worker_service_addr(self, worker, service_name, protocol=False):
         else:
             return ws.host, port
 
-    def start_services(self, default_listen_ip):
-        if default_listen_ip == "0.0.0.0":
-            default_listen_ip = ""  # for IPV6
-
-        for k, v in self.service_specs.items():
-            listen_ip = None
-            if isinstance(k, tuple):
-                k, port = k
-            else:
-                port = 0
-
-            if isinstance(port, (str, unicode)):
-                port = port.split(":")
-
-            if isinstance(port, (tuple, list)):
-                listen_ip, port = (port[0], int(port[1]))
-
-            if isinstance(v, tuple):
-                v, kwargs = v
-            else:
-                kwargs = {}
-
-            try:
-                service = v(self, io_loop=self.loop, **kwargs)
-                service.listen(
-                    (listen_ip if listen_ip is not None else default_listen_ip, port)
-                )
-                self.services[k] = service
-            except Exception as e:
-                warnings.warn(
-                    "\nCould not launch service '%s' on port %s. " % (k, port)
-                    + "Got the following message:\n\n"
-                    + str(e),
-                    stacklevel=3,
-                )
-
-    def stop_services(self):
-        for service in self.services.values():
-            service.stop()
-
     def start(self, addr_or_port=None, start_queues=True):
         """ Clear out old state and restart all running coroutines """
         enable_gc_diagnosis()
@@ -1347,7 +1315,7 @@ def close_worker(self, stream=None, worker=None, safe=None):
         logger.info("Closing worker %s", worker)
         with log_errors():
             self.log_event(worker, {"action": "close-worker"})
-            nanny_addr = self.get_worker_service_addr(worker, "nanny", protocol=True)
+            nanny_addr = self.workers[worker].nanny
             address = nanny_addr or worker
 
             self.worker_send(worker, {"op": "close", "report": False})
@@ -1434,6 +1402,7 @@ def add_worker(
         pid=0,
         services=None,
         local_directory=None,
+        nanny=None,
     ):
         """ Add a new worker to the cluster """
         with log_errors():
@@ -1453,6 +1422,7 @@ def add_worker(
                 name=name,
                 local_directory=local_directory,
                 services=services,
+                nanny=nanny,
             )
 
             if name in self.aliases:
@@ -2608,10 +2578,7 @@ def restart(self, client=None, timeout=3):
                     keys=[ts.key for ts in cs.wants_what], client=cs.client_key
                 )
 
-            nannies = {
-                addr: self.get_worker_service_addr(addr, "nanny", protocol=True)
-                for addr in self.workers
-            }
+            nannies = {addr: ws.nanny for addr, ws in self.workers.items()}
 
             for addr in list(self.workers):
                 try:
@@ -2694,9 +2661,7 @@ def broadcast(
         # TODO replace with worker_list
 
         if nanny:
-            addresses = [
-                self.get_worker_service_addr(w, "nanny", protocol=True) for w in workers
-            ]
+            addresses = [self.workers[w].nanny for w in workers]
         else:
             addresses = workers
 
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index b77efc9f51d..cf18e6b10e8 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -3479,7 +3479,7 @@ def test_bad_tasks_fail(c, s, a, b):
     with pytest.raises(KilledWorker) as info:
         yield f
 
-    assert info.value.last_worker.services["nanny"] in {a.port, b.port}
+    assert info.value.last_worker.nanny in {a.address, b.address}
 
 
 def test_get_processing_sync(c, s, a, b):
diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
index 60de12dce4b..bf9f91b6371 100644
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -28,7 +28,7 @@ def test_nanny(s):
     with rpc(n.address) as nn:
         assert n.is_alive()
         assert s.ncores[n.worker_address] == 2
-        assert s.workers[n.worker_address].services["nanny"] > 1024
+        assert s.workers[n.worker_address].nanny == n.address
 
         yield nn.kill()
         assert not n.is_alive()
@@ -43,7 +43,7 @@ def test_nanny(s):
         yield nn.instantiate()
         assert n.is_alive()
         assert s.ncores[n.worker_address] == 2
-        assert s.workers[n.worker_address].services["nanny"] > 1024
+        assert s.workers[n.worker_address].nanny == n.address
 
         yield nn.terminate()
         assert not n.is_alive()
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 3cc6579ed29..73d10ab8a55 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -1542,3 +1542,15 @@ def test_host_address():
     s = yield Scheduler(host="127.0.0.2")
     assert "127.0.0.2" in s.address
     yield s.close()
+
+
+@gen_test()
+def test_dashboard_address():
+    pytest.importorskip("bokeh")
+    s = yield Scheduler(dashboard_address="127.0.0.1:8901")
+    assert s.services["bokeh"].port == 8901
+    yield s.close()
+
+    s = yield Scheduler(dashboard_address="127.0.0.1")
+    assert s.services["bokeh"].port
+    yield s.close()
diff --git a/distributed/worker.py b/distributed/worker.py
index 1b103fe144a..64eca0cc770 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -261,6 +261,8 @@ class Worker(ServerNode):
     executor: concurrent.futures.Executor
     resources: dict
         Resources that this worker has like ``{'GPU': 2}``
+    nanny: str
+        Address on which to contact nanny, if it exists
 
     Examples
     --------
@@ -311,6 +313,7 @@ def __init__(
         port=None,
         protocol=None,
         dashboard_address=None,
+        nanny=None,
         low_level_profiler=dask.config.get("distributed.worker.profile.low-level"),
         **kwargs
     ):
@@ -323,6 +326,7 @@ def __init__(
         self.who_has = dict()
         self.has_what = defaultdict(set)
         self.pending_data_per_worker = defaultdict(deque)
+        self.nanny = nanny
         self._lock = threading.Lock()
 
         self.data_needed = deque()  # TODO: replace with heap?
@@ -535,7 +539,6 @@ def __init__(
             sys.path.insert(0, self.local_dir)
 
         self.services = {}
-        self.service_ports = service_ports or {}
         self.service_specs = services or {}
 
         if dashboard_address is not None:
@@ -731,6 +734,7 @@ def _register_with_scheduler(self):
                         memory_limit=self.memory_limit,
                         local_directory=self.local_dir,
                         services=self.service_ports,
+                        nanny=self.nanny,
                         pid=os.getpid(),
                         metrics=self.get_metrics(),
                     ),
@@ -895,34 +899,6 @@ def get_logs(self, comm=None, n=None):
     # Lifecycle #
     #############
 
-    def start_services(self, default_listen_ip):
-        if default_listen_ip == "0.0.0.0":
-            default_listen_ip = ""  # for IPV6
-
-        for k, v in self.service_specs.items():
-            listen_ip = None
-            if isinstance(k, tuple):
-                k, port = k
-            else:
-                port = 0
-
-            if isinstance(port, (str, unicode)):
-                port = port.split(":")
-
-            if isinstance(port, (tuple, list)):
-                listen_ip, port = (port[0], int(port[1]))
-
-            if isinstance(v, tuple):
-                v, kwargs = v
-            else:
-                kwargs = {}
-
-            self.services[k] = v(self, io_loop=self.loop, **kwargs)
-            self.services[k].listen(
-                (listen_ip if listen_ip is not None else default_listen_ip, port)
-            )
-            self.service_ports[k] = self.services[k].port
-
     @gen.coroutine
     def _start(self, addr_or_port=0):
         assert self.status is None
@@ -1047,13 +1023,8 @@ def close(self, report=True, timeout=10, nanny=True, executor_wait=True):
             if self.batched_stream:
                 self.batched_stream.close()
 
-            if nanny and "nanny" in self.service_ports:
-                nanny_address = "%s%s:%d" % (
-                    self.listener.prefix,
-                    self.ip,
-                    self.service_ports["nanny"],
-                )
-                with self.rpc(nanny_address) as r:
+            if nanny and self.nanny:
+                with self.rpc(self.nanny) as r:
                     yield r.terminate()
 
             self.rpc.close()

From 3e87f34f6fe110188e03c18c14bc27893ad0da59 Mon Sep 17 00:00:00 2001
From: Scott Sievert <stsievert@users.noreply.github.com>
Date: Fri, 10 May 2019 16:20:40 +0000
Subject: [PATCH 0277/1550] Add CONTRIBUTING.md (#2680)

---
 CONTRIBUTING.md | 29 +++++++++++++++++++++++++++++
 1 file changed, 29 insertions(+)
 create mode 100644 CONTRIBUTING.md

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
new file mode 100644
index 00000000000..cd35ad7c572
--- /dev/null
+++ b/CONTRIBUTING.md
@@ -0,0 +1,29 @@
+For more information, see https://docs.dask.org/en/latest/develop.html#contributing-to-code
+
+
+## Style
+Distributed conforms with the [flake8] and [black] styles. To make sure your
+code conforms with these styles, run
+
+``` shell
+$ pip install black flake8
+$ cd path/to/distributed
+$ black distributed
+$ flake8 distributed
+```
+
+[flake8]:http://flake8.pycqa.org/en/latest/
+[black]:https://github.com/python/black
+
+## Docstrings
+
+Dask Distributed roughly follows the [numpydoc] standard. More information is
+available at https://docs.dask.org/en/latest/develop.html#docstrings.
+
+[numpydoc]:https://github.com/numpy/numpy/blob/master/doc/HOWTO_DOCUMENT.rst.txt
+
+## Tests
+
+Dask employs extensive unit tests to ensure correctness of code both for today
+and for the future. Test coverage is expected for all code contributions. More
+detail is at https://docs.dask.org/en/latest/develop.html#test

From a8fa4c19f6da43d8605bb8fcd8d83b9ff8b214ee Mon Sep 17 00:00:00 2001
From: Muammar El Khatib <muammarelkhatib@gmail.com>
Date: Fri, 10 May 2019 09:28:45 -0700
Subject: [PATCH 0278/1550] Catch RuntimeError to avoid serialization fail when
 using pytorch (#2619)

* Fix `Failed to Serialize` error with pytorch tensors.

- When a tensor requires_grad then we have to t.detach().numpy()
  otherwise a .numpy() is used. This fixes the failed to serialized
  problem present in latest distributed version.
- Improved test_grad() test as suggested by @stsievert.
- The whole PR is included in a single commit.

* More improvements to test_torch

- Verify that t.requires_grad is not modified by serialization.
- Use `np.allclose()` instead of `==`.
---
 distributed/protocol/tests/test_torch.py | 18 +++++++++++++-----
 distributed/protocol/torch.py            |  7 ++++++-
 2 files changed, 19 insertions(+), 6 deletions(-)

diff --git a/distributed/protocol/tests/test_torch.py b/distributed/protocol/tests/test_torch.py
index 6cc8bb20986..efb5fa6610a 100644
--- a/distributed/protocol/tests/test_torch.py
+++ b/distributed/protocol/tests/test_torch.py
@@ -14,14 +14,22 @@ def test_tensor():
     assert (x == t2.numpy()).all()
 
 
-def test_grad():
+@pytest.mark.parametrize("requires_grad", [True, False])
+def test_grad(requires_grad):
     x = np.arange(10)
-    t = torch.Tensor(x)
-    t.grad = torch.zeros_like(t) + 1
+    t = torch.tensor(x, dtype=torch.float, requires_grad=requires_grad)
+
+    if requires_grad:
+        t.grad = torch.zeros_like(t) + 1
 
     t2 = deserialize(*serialize(t))
-    assert (t2.numpy() == x).all()
-    assert (t2.grad.numpy() == 1).all()
+
+    assert t2.requires_grad is requires_grad
+    assert t.requires_grad is requires_grad
+    assert np.allclose(t2.detach().numpy(), x)
+
+    if requires_grad:
+        assert np.allclose(t2.grad.numpy(), 1)
 
 
 def test_resnet():
diff --git a/distributed/protocol/torch.py b/distributed/protocol/torch.py
index e69be68b0c1..3b4c6d19c8d 100644
--- a/distributed/protocol/torch.py
+++ b/distributed/protocol/torch.py
@@ -7,7 +7,12 @@
 @dask_serialize.register(torch.Tensor)
 def serialize_torch_Tensor(t):
     requires_grad_ = t.requires_grad
-    header, frames = serialize(t.detach_().numpy())
+
+    if requires_grad_:
+        header, frames = serialize(t.detach().numpy())
+    else:
+        header, frames = serialize(t.numpy())
+
     if t.grad is not None:
         grad_header, grad_frames = serialize(t.grad.numpy())
         header["grad"] = {"header": grad_header, "start": len(frames)}

From edc001441841c748258a6f875f414f671e2dcf2a Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 10 May 2019 13:23:48 -0500
Subject: [PATCH 0279/1550] Switch from (ip, port) to address in tests (#2684)

This helps resolve issues on Windows CI
---
 distributed/batched.py                        |  2 +-
 .../diagnostics/tests/test_progressbar.py     | 33 ++++--------
 distributed/tests/test_client.py              | 52 +++++++++----------
 distributed/tests/test_failed_workers.py      | 10 ++--
 distributed/tests/test_nanny.py               |  6 +--
 distributed/tests/test_publish.py             | 16 +++---
 distributed/tests/test_resources.py           |  6 +--
 distributed/tests/test_scheduler.py           | 24 ++++-----
 distributed/tests/test_steal.py               |  6 +--
 distributed/tests/test_worker.py              |  2 +-
 10 files changed, 73 insertions(+), 84 deletions(-)

diff --git a/distributed/batched.py b/distributed/batched.py
index bc77cc7fda2..e17d7b1f1bd 100644
--- a/distributed/batched.py
+++ b/distributed/batched.py
@@ -26,7 +26,7 @@ class BatchedSend(object):
 
     Example
     -------
-    >>> stream = yield connect(ip, port)
+    >>> stream = yield connect(address)
     >>> bstream = BatchedSend(interval='10 ms')
     >>> bstream.start(stream)
     >>> bstream.send('Hello,')
diff --git a/distributed/diagnostics/tests/test_progressbar.py b/distributed/diagnostics/tests/test_progressbar.py
index d5a01410f5e..ac21f1637bc 100644
--- a/distributed/diagnostics/tests/test_progressbar.py
+++ b/distributed/diagnostics/tests/test_progressbar.py
@@ -2,12 +2,10 @@
 
 from time import sleep
 
-from tornado import gen
-
 from distributed import Scheduler, Worker
 from distributed.diagnostics.progressbar import TextProgressBar, progress
 from distributed.metrics import time
-from distributed.utils_test import inc, div, gen_cluster
+from distributed.utils_test import inc, div, gen_cluster, gen_test
 from distributed.utils_test import client, loop, cluster_fixture  # noqa: F401
 
 
@@ -30,34 +28,25 @@ def test_text_progressbar(capsys, client):
 def test_TextProgressBar_error(c, s, a, b):
     x = c.submit(div, 1, 0)
 
-    progress = TextProgressBar(
-        [x.key], scheduler=(s.ip, s.port), start=False, interval=0.01
-    )
+    progress = TextProgressBar([x.key], scheduler=s.address, start=False, interval=0.01)
     yield progress.listen()
 
     assert progress.status == "error"
     assert progress.comm.closed()
 
-    progress = TextProgressBar(
-        [x.key], scheduler=(s.ip, s.port), start=False, interval=0.01
-    )
+    progress = TextProgressBar([x.key], scheduler=s.address, start=False, interval=0.01)
     yield progress.listen()
     assert progress.status == "error"
     assert progress.comm.closed()
 
 
-def test_TextProgressBar_empty(loop, capsys):
-    @gen.coroutine
+def test_TextProgressBar_empty(capsys):
+    @gen_test()
     def f():
-        s = Scheduler(loop=loop)
-        done = s.start(0)
-        a = Worker(s.ip, s.port, loop=loop, ncores=1)
-        b = Worker(s.ip, s.port, loop=loop, ncores=1)
-        yield [a._start(0), b._start(0)]
-
-        progress = TextProgressBar(
-            [], scheduler=(s.ip, s.port), start=False, interval=0.01
-        )
+        s = yield Scheduler(port=0)
+        a, b = yield [Worker(s.address, ncores=1), Worker(s.address, ncores=1)]
+
+        progress = TextProgressBar([], scheduler=s.address, start=False, interval=0.01)
         yield progress.listen()
 
         assert progress.status == "finished"
@@ -65,9 +54,9 @@ def f():
 
         yield [a.close(), b.close()]
         s.close()
-        yield done
+        yield s.finished()
 
-    loop.run_sync(f)
+    f()
 
 
 def check_bar_completed(capsys, width=40):
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index cf18e6b10e8..bce1066c0d2 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -471,7 +471,7 @@ def test_exceptions(c, s, a, b):
 
 @gen_cluster()
 def test_gc(s, a, b):
-    c = yield Client((s.ip, s.port), asynchronous=True)
+    c = yield Client(s.address, asynchronous=True)
 
     x = c.submit(inc, 10)
     yield x
@@ -1006,12 +1006,12 @@ def assert_list(x, z=[]):
 
 @gen_cluster()
 def test_two_consecutive_clients_share_results(s, a, b):
-    c = yield Client((s.ip, s.port), asynchronous=True)
+    c = yield Client(s.address, asynchronous=True)
 
     x = c.submit(random.randint, 0, 1000, pure=True)
     xx = yield x
 
-    f = yield Client((s.ip, s.port), asynchronous=True)
+    f = yield Client(s.address, asynchronous=True)
 
     y = f.submit(random.randint, 0, 1000, pure=True)
     yy = yield y
@@ -1680,8 +1680,8 @@ def test_upload_file_exception_sync(c):
 @pytest.mark.skip
 @gen_cluster()
 def test_multiple_clients(s, a, b):
-    a = yield Client((s.ip, s.port), asynchronous=True)
-    b = yield Client((s.ip, s.port), asynchronous=True)
+    a = yield Client(s.address, asynchronous=True)
+    b = yield Client(s.address, asynchronous=True)
 
     x = a.submit(inc, 1)
     y = b.submit(inc, 2)
@@ -2102,8 +2102,8 @@ def test_waiting_data(c, s, a, b):
 
 @gen_cluster()
 def test_multi_client(s, a, b):
-    c = yield Client((s.ip, s.port), asynchronous=True)
-    f = yield Client((s.ip, s.port), asynchronous=True)
+    c = yield Client(s.address, asynchronous=True)
+    f = yield Client(s.address, asynchronous=True)
 
     assert set(s.client_comms) == {c.id, f.id}
 
@@ -2170,9 +2170,9 @@ def test_cleanup_after_broken_client_connection(s, a, b):
 
 @gen_cluster()
 def test_multi_garbage_collection(s, a, b):
-    c = yield Client((s.ip, s.port), asynchronous=True)
+    c = yield Client(s.address, asynchronous=True)
 
-    f = yield Client((s.ip, s.port), asynchronous=True)
+    f = yield Client(s.address, asynchronous=True)
 
     x = c.submit(inc, 1)
     y = f.submit(inc, 2)
@@ -2294,8 +2294,8 @@ def test__cancel_tuple_key(c, s, a, b):
 
 @gen_cluster()
 def test__cancel_multi_client(s, a, b):
-    c = yield Client((s.ip, s.port), asynchronous=True)
-    f = yield Client((s.ip, s.port), asynchronous=True)
+    c = yield Client(s.address, asynchronous=True)
+    f = yield Client(s.address, asynchronous=True)
 
     x = c.submit(slowinc, 1)
     y = f.submit(slowinc, 1)
@@ -2824,12 +2824,12 @@ def test_diagnostic_nbytes(c, s, a, b):
 @gen_test()
 def test_worker_aliases():
     s = yield Scheduler(validate=True, port=0)
-    a = Worker(s.ip, s.port, name="alice")
-    b = Worker(s.ip, s.port, name="bob")
-    w = Worker(s.ip, s.port, name=3)
+    a = Worker(s.address, name="alice")
+    b = Worker(s.address, name="bob")
+    w = Worker(s.address, name=3)
     yield [a, b, w]
 
-    c = yield Client((s.ip, s.port), asynchronous=True)
+    c = yield Client(s.address, asynchronous=True)
 
     L = c.map(inc, range(10), workers="alice")
     future = yield c.scatter(123, workers=3)
@@ -2905,10 +2905,10 @@ def test_client_num_fds(loop):
 
 @gen_cluster()
 def test_startup_close_startup(s, a, b):
-    c = yield Client((s.ip, s.port), asynchronous=True)
+    c = yield Client(s.address, asynchronous=True)
     yield c.close()
 
-    c = yield Client((s.ip, s.port), asynchronous=True)
+    c = yield Client(s.address, asynchronous=True)
     yield c.close()
 
 
@@ -3043,7 +3043,7 @@ def test_unrunnable_task_runs(c, s, a, b):
     assert s.tasks[x.key] in s.unrunnable
     assert s.get_task_status(keys=[x.key]) == {x.key: "no-worker"}
 
-    w = yield Worker(s.ip, s.port, loop=s.loop)
+    w = yield Worker(s.address, loop=s.loop)
 
     start = time()
     while x.status != "finished":
@@ -3060,7 +3060,7 @@ def test_unrunnable_task_runs(c, s, a, b):
 def test_add_worker_after_tasks(c, s):
     futures = c.map(inc, range(10))
 
-    n = yield Nanny(s.ip, s.port, ncores=2, loop=s.loop, port=0)
+    n = yield Nanny(s.address, ncores=2, loop=s.loop, port=0)
 
     result = yield c.gather(futures)
 
@@ -3602,7 +3602,7 @@ def test_as_completed_next_batch(c):
 def test_status():
     s = yield Scheduler(port=0)
 
-    c = yield Client((s.ip, s.port), asynchronous=True)
+    c = yield Client(s.address, asynchronous=True)
     assert c.status == "running"
     x = c.submit(inc, 1)
 
@@ -3782,8 +3782,8 @@ def start_worker(sleep, duration, repeat=1):
 
 @gen_cluster(client=False, timeout=None)
 def test_idempotence(s, a, b):
-    c = yield Client((s.ip, s.port), asynchronous=True)
-    f = yield Client((s.ip, s.port), asynchronous=True)
+    c = yield Client(s.address, asynchronous=True)
+    f = yield Client(s.address, asynchronous=True)
 
     # Submit
     x = c.submit(inc, 1)
@@ -3989,8 +3989,8 @@ def test_scatter_compute_store_lose_processing(c, s, a, b):
 
 @gen_cluster(client=False)
 def test_serialize_future(s, a, b):
-    c = yield Client((s.ip, s.port), asynchronous=True)
-    f = yield Client((s.ip, s.port), asynchronous=True)
+    c = yield Client(s.address, asynchronous=True)
+    f = yield Client(s.address, asynchronous=True)
 
     future = c.submit(lambda: 1)
     result = yield future
@@ -4008,8 +4008,8 @@ def test_serialize_future(s, a, b):
 
 @gen_cluster(client=False)
 def test_temp_client(s, a, b):
-    c = yield Client((s.ip, s.port), asynchronous=True)
-    f = yield Client((s.ip, s.port), asynchronous=True)
+    c = yield Client(s.address, asynchronous=True)
+    f = yield Client(s.address, asynchronous=True)
 
     with temp_default_client(c):
         assert default_client() is c
diff --git a/distributed/tests/test_failed_workers.py b/distributed/tests/test_failed_workers.py
index bae2e141ee2..dde92c6d24c 100644
--- a/distributed/tests/test_failed_workers.py
+++ b/distributed/tests/test_failed_workers.py
@@ -39,7 +39,7 @@ def test_submit_after_failed_worker_sync(loop):
 
 @gen_cluster(client=True, timeout=60, active_rpc_timeout=10)
 def test_submit_after_failed_worker_async(c, s, a, b):
-    n = Nanny(s.ip, s.port, ncores=2, loop=s.loop)
+    n = Nanny(s.address, ncores=2, loop=s.loop)
     n.start(0)
     while len(s.workers) < 3:
         yield gen.sleep(0.1)
@@ -267,8 +267,8 @@ def test_fast_kill(c, s, a, b):
 
 @gen_cluster(Worker=Nanny, timeout=60)
 def test_multiple_clients_restart(s, a, b):
-    e1 = yield Client((s.ip, s.port), asynchronous=True)
-    e2 = yield Client((s.ip, s.port), asynchronous=True)
+    e1 = yield Client(s.address, asynchronous=True)
+    e2 = yield Client(s.address, asynchronous=True)
 
     x = e1.submit(inc, 1)
     y = e2.submit(inc, 2)
@@ -315,7 +315,7 @@ def test_forgotten_futures_dont_clean_up_new_futures(c, s, a, b):
 @gen_cluster(client=True, timeout=60, active_rpc_timeout=10)
 def test_broken_worker_during_computation(c, s, a, b):
     s.allowed_failures = 100
-    n = Nanny(s.ip, s.port, ncores=2, loop=s.loop)
+    n = Nanny(s.address, ncores=2, loop=s.loop)
     n.start(0)
 
     start = time()
@@ -374,7 +374,7 @@ def test_restart_during_computation(c, s, a, b):
 
 @gen_cluster(client=True, timeout=60)
 def test_worker_who_has_clears_after_failed_connection(c, s, a, b):
-    n = Nanny(s.ip, s.port, ncores=2, loop=s.loop)
+    n = Nanny(s.address, ncores=2, loop=s.loop)
     n.start(0)
 
     start = time()
diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
index bf9f91b6371..4c18b5242a3 100644
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -23,7 +23,7 @@
 
 @gen_cluster(ncores=[])
 def test_nanny(s):
-    n = yield Nanny(s.ip, s.port, ncores=2, loop=s.loop)
+    n = yield Nanny(s.address, ncores=2, loop=s.loop)
 
     with rpc(n.address) as nn:
         assert n.is_alive()
@@ -70,7 +70,7 @@ def test_str(s, a, b):
 
 @gen_cluster(ncores=[], timeout=20, client=True)
 def test_nanny_process_failure(c, s):
-    n = yield Nanny(s.ip, s.port, ncores=2, loop=s.loop)
+    n = yield Nanny(s.address, ncores=2, loop=s.loop)
     first_dir = n.worker_dir
 
     assert os.path.exists(first_dir)
@@ -117,7 +117,7 @@ def test_nanny_no_port():
 @gen_cluster(ncores=[])
 def test_run(s):
     pytest.importorskip("psutil")
-    n = yield Nanny(s.ip, s.port, ncores=2, loop=s.loop)
+    n = yield Nanny(s.address, ncores=2, loop=s.loop)
 
     with rpc(n.address) as nn:
         response = yield nn.run(function=dumps(lambda: 1))
diff --git a/distributed/tests/test_publish.py b/distributed/tests/test_publish.py
index e4789589c48..7c0fd0db6d2 100644
--- a/distributed/tests/test_publish.py
+++ b/distributed/tests/test_publish.py
@@ -12,8 +12,8 @@
 
 @gen_cluster(client=False)
 def test_publish_simple(s, a, b):
-    c = yield Client((s.ip, s.port), asynchronous=True)
-    f = yield Client((s.ip, s.port), asynchronous=True)
+    c = yield Client(s.address, asynchronous=True)
+    f = yield Client(s.address, asynchronous=True)
 
     data = yield c.scatter(range(3))
     out = yield c.publish_dataset(data=data)
@@ -38,8 +38,8 @@ def test_publish_simple(s, a, b):
 
 @gen_cluster(client=False)
 def test_publish_non_string_key(s, a, b):
-    c = yield Client((s.ip, s.port), asynchronous=True)
-    f = yield Client((s.ip, s.port), asynchronous=True)
+    c = yield Client(s.address, asynchronous=True)
+    f = yield Client(s.address, asynchronous=True)
 
     try:
         for name in [("a", "b"), 9.0, 8]:
@@ -60,8 +60,8 @@ def test_publish_non_string_key(s, a, b):
 
 @gen_cluster(client=False)
 def test_publish_roundtrip(s, a, b):
-    c = yield Client((s.ip, s.port), asynchronous=True)
-    f = yield Client((s.ip, s.port), asynchronous=True)
+    c = yield Client(s.address, asynchronous=True)
+    f = yield Client(s.address, asynchronous=True)
 
     data = yield c.scatter([0, 1, 2])
     yield c.publish_dataset(data=data)
@@ -156,8 +156,8 @@ def test_unpublish_multiple_datasets_sync(client):
 @gen_cluster(client=False)
 def test_publish_bag(s, a, b):
     db = pytest.importorskip("dask.bag")
-    c = yield Client((s.ip, s.port), asynchronous=True)
-    f = yield Client((s.ip, s.port), asynchronous=True)
+    c = yield Client(s.address, asynchronous=True)
+    f = yield Client(s.address, asynchronous=True)
 
     bag = db.from_sequence([0, 1, 2])
     bagp = c.persist(bag)
diff --git a/distributed/tests/test_resources.py b/distributed/tests/test_resources.py
index 429bbc2bb56..d7102ef5301 100644
--- a/distributed/tests/test_resources.py
+++ b/distributed/tests/test_resources.py
@@ -19,8 +19,8 @@ def test_resources(c, s):
     assert not s.worker_resources
     assert not s.resources
 
-    a = Worker(s.ip, s.port, loop=s.loop, resources={"GPU": 2})
-    b = Worker(s.ip, s.port, loop=s.loop, resources={"GPU": 1, "DB": 1})
+    a = Worker(s.address, loop=s.loop, resources={"GPU": 2})
+    b = Worker(s.address, loop=s.loop, resources={"GPU": 1, "DB": 1})
 
     yield [a, b]
 
@@ -55,7 +55,7 @@ def test_resource_submit(c, s, a, b):
 
     assert s.get_task_status(keys=[z.key]) == {z.key: "no-worker"}
 
-    d = yield Worker(s.ip, s.port, loop=s.loop, resources={"C": 10})
+    d = yield Worker(s.address, loop=s.loop, resources={"C": 10})
 
     yield wait(z)
     assert z.key in d.data
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 73d10ab8a55..da750dd9196 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -253,7 +253,7 @@ def test_clear_events_client_removal(c, s, a, b):
 
 @gen_cluster()
 def test_add_worker(s, a, b):
-    w = Worker(s.ip, s.port, ncores=3)
+    w = Worker(s.address, ncores=3)
     w.data["x-5"] = 6
     w.data["y"] = 1
     yield w
@@ -533,12 +533,12 @@ def test_broadcast_nanny(s, a, b):
 @gen_test()
 def test_worker_name():
     s = yield Scheduler(validate=True, port=0)
-    w = yield Worker(s.ip, s.port, name="alice")
+    w = yield Worker(s.address, name="alice")
     assert s.workers[w.address].name == "alice"
     assert s.aliases["alice"] == w.address
 
     with pytest.raises(ValueError):
-        w2 = yield Worker(s.ip, s.port, name="alice")
+        w2 = yield Worker(s.address, name="alice")
         yield w2.close()
 
     yield w.close()
@@ -550,8 +550,8 @@ def test_coerce_address():
     with dask.config.set({"distributed.comm.timeouts.connect": "100ms"}):
         s = yield Scheduler(validate=True, port=0)
         print("scheduler:", s.address, s.listen_address)
-        a = Worker(s.ip, s.port, name="alice")
-        b = Worker(s.ip, s.port, name=123)
+        a = Worker(s.address, name="alice")
+        b = Worker(s.address, name=123)
         c = Worker("127.0.0.1", s.port, name="charlie")
         yield [a, b, c]
 
@@ -594,7 +594,7 @@ def test_file_descriptors_dont_leak(s):
     proc = psutil.Process()
     before = proc.num_fds()
 
-    w = yield Worker(s.ip, s.port)
+    w = yield Worker(s.address)
     yield w.close()
 
     during = proc.num_fds()
@@ -661,7 +661,7 @@ def test_scatter_no_workers(c, s):
         yield c.scatter(123, timeout=0.1)
     assert time() < start + 1.5
 
-    w = Worker(s.ip, s.port, ncores=3)
+    w = Worker(s.address, ncores=3)
     yield [c.scatter(data={"y": 2}, timeout=5), w._start()]
 
     assert w.data["y"] == 2
@@ -670,7 +670,7 @@ def test_scatter_no_workers(c, s):
 
 @gen_cluster(ncores=[])
 def test_scheduler_sees_memory_limits(s):
-    w = yield Worker(s.ip, s.port, ncores=3, memory_limit=12345)
+    w = yield Worker(s.address, ncores=3, memory_limit=12345)
 
     assert s.workers[w.address].memory_limit == 12345
     yield w.close()
@@ -788,7 +788,7 @@ def test_file_descriptors(c, s):
     num_fds_1 = proc.num_fds()
 
     N = 20
-    nannies = yield [Nanny(s.ip, s.port, loop=s.loop) for i in range(N)]
+    nannies = yield [Nanny(s.address, loop=s.loop) for i in range(N)]
 
     while len(s.ncores) < N:
         yield gen.sleep(0.1)
@@ -926,7 +926,7 @@ def test_worker_arrives_with_processing_data(c, s, a, b):
     while not any(w.processing for w in s.workers.values()):
         yield gen.sleep(0.01)
 
-    w = Worker(s.ip, s.port, ncores=1)
+    w = Worker(s.address, ncores=1)
     w.put_key_in_memory(y.key, 3)
 
     yield w
@@ -977,7 +977,7 @@ def test_no_workers_to_memory(c, s):
     while not s.tasks:
         yield gen.sleep(0.01)
 
-    w = Worker(s.ip, s.port, ncores=1)
+    w = Worker(s.address, ncores=1)
     w.put_key_in_memory(y.key, 3)
 
     yield w
@@ -1007,7 +1007,7 @@ def test_no_worker_to_memory_restrictions(c, s, a, b):
     while not s.tasks:
         yield gen.sleep(0.01)
 
-    w = Worker(s.ip, s.port, ncores=1, name="alice")
+    w = Worker(s.address, ncores=1, name="alice")
     w.put_key_in_memory(y.key, 3)
 
     yield w
diff --git a/distributed/tests/test_steal.py b/distributed/tests/test_steal.py
index cb56fc0f263..8edeb8e339c 100644
--- a/distributed/tests/test_steal.py
+++ b/distributed/tests/test_steal.py
@@ -172,7 +172,7 @@ def test_new_worker_steals(c, s, a):
     while len(a.task_state) < 10:
         yield gen.sleep(0.01)
 
-    b = yield Worker(s.ip, s.port, loop=s.loop, ncores=1, memory_limit=TOTAL_MEMORY)
+    b = yield Worker(s.address, loop=s.loop, ncores=1, memory_limit=TOTAL_MEMORY)
 
     result = yield total
     assert result == sum(map(inc, range(100)))
@@ -277,7 +277,7 @@ def test_steal_resource_restrictions(c, s, a):
         yield gen.sleep(0.01)
     assert len(a.task_state) == 101
 
-    b = yield Worker(s.ip, s.port, loop=s.loop, ncores=1, resources={"A": 4})
+    b = yield Worker(s.address, loop=s.loop, ncores=1, resources={"A": 4})
 
     start = time()
     while not b.task_state or len(a.task_state) == 101:
@@ -536,7 +536,7 @@ def test_steal_twice(c, s, a, b):
         yield gen.sleep(0.01)
 
     # Army of new workers arrives to help
-    workers = yield [Worker(s.ip, s.port, loop=s.loop) for _ in range(20)]
+    workers = yield [Worker(s.address, loop=s.loop) for _ in range(20)]
 
     yield wait(futures)
 
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 5fad86b2665..16a685fb09b 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -437,7 +437,7 @@ def f(dask_worker=None):
 @gen_cluster(client=True, ncores=[])
 def test_Executor(c, s):
     with ThreadPoolExecutor(2) as e:
-        w = Worker(s.ip, s.port, executor=e)
+        w = Worker(s.address, executor=e)
         assert w.executor is e
         w = yield w
 

From 2dc778d34ed68660885e8be8b87cf94c021458da Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Torsten=20W=C3=B6rtwein?=
 <twoertwein@users.noreply.github.com>
Date: Fri, 10 May 2019 20:21:05 -0400
Subject: [PATCH 0280/1550] Cap worker's memory limit by the hard limit of the
 maximum resident memory (#2665)

---
 distributed/tests/test_worker.py | 24 +++++++++++++++++++++-
 distributed/worker.py            | 34 +++++++++++++++++++-------------
 2 files changed, 43 insertions(+), 15 deletions(-)

diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 16a685fb09b..77df078fa89 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -25,7 +25,7 @@
 from distributed.client import wait
 from distributed.scheduler import Scheduler
 from distributed.metrics import time
-from distributed.worker import Worker, error_message, logger
+from distributed.worker import Worker, error_message, logger, parse_memory_limit
 from distributed.utils import tmpfile, format_bytes
 from distributed.utils_test import (
     inc,
@@ -1401,3 +1401,25 @@ def test_host_address(c, s):
     assert "127.0.0.3" in n.address
     assert "127.0.0.3" in n.worker_address
     yield n.close()
+
+
+def test_resource_limit():
+    assert parse_memory_limit("250MiB", 1, total_cores=1) == 1024 * 1024 * 250
+
+    # get current limit
+    resource = pytest.importorskip("resource")
+    try:
+        hard_limit = resource.getrlimit(resource.RLIMIT_RSS)[1]
+    except OSError:
+        pytest.skip("resource could not get the RSS limit")
+    memory_limit = psutil.virtual_memory().total
+    if hard_limit > memory_limit or hard_limit < 0:
+        hard_limit = memory_limit
+
+    # decrease memory limit by one byte
+    new_limit = hard_limit - 1
+    try:
+        resource.setrlimit(resource.RLIMIT_RSS, (new_limit, new_limit))
+        assert parse_memory_limit(hard_limit, 1, total_cores=1) == new_limit
+    except OSError:
+        pytest.skip("resource could not set the RSS limit")
diff --git a/distributed/worker.py b/distributed/worker.py
index 64eca0cc770..5302d94c6de 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -12,6 +12,7 @@
 import sys
 import warnings
 import weakref
+import psutil
 
 import dask
 from dask.core import istask
@@ -76,15 +77,7 @@
 
 no_value = "--no-value-sentinel--"
 
-try:
-    import psutil
-
-    TOTAL_MEMORY = psutil.virtual_memory().total
-except ImportError:
-    logger.warning("Please install psutil to estimate worker memory use")
-    TOTAL_MEMORY = 8e9
-    psutil = None
-
+TOTAL_MEMORY = psutil.virtual_memory().total
 
 IN_PLAY = ("waiting", "ready", "executing", "long-running")
 PENDING = ("waiting", "ready", "constrained")
@@ -2933,17 +2926,30 @@ class Reschedule(Exception):
 def parse_memory_limit(memory_limit, ncores, total_cores=_ncores):
     if memory_limit is None:
         return None
+
     if memory_limit == "auto":
         memory_limit = int(TOTAL_MEMORY * min(1, ncores / total_cores))
     with ignoring(ValueError, TypeError):
-        x = float(memory_limit)
-        if isinstance(x, float) and x <= 1:
-            return int(x * TOTAL_MEMORY)
+        memory_limit = float(memory_limit)
+        if isinstance(memory_limit, float) and memory_limit <= 1:
+            memory_limit = int(memory_limit * TOTAL_MEMORY)
 
     if isinstance(memory_limit, (unicode, str)):
-        return parse_bytes(memory_limit)
+        memory_limit = parse_bytes(memory_limit)
     else:
-        return int(memory_limit)
+        memory_limit = int(memory_limit)
+
+    # should be less than hard RSS limit
+    try:
+        import resource
+
+        hard_limit = resource.getrlimit(resource.RLIMIT_RSS)[1]
+        if hard_limit > 0:
+            memory_limit = min(memory_limit, hard_limit)
+    except (ImportError, OSError):
+        pass
+
+    return memory_limit
 
 
 @gen.coroutine

From 1cbd324248db1973f1cfeb393e2d0dc7a5490d41 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 10 May 2019 21:42:44 -0500
Subject: [PATCH 0281/1550] Add WeakSet _instances attributes to all classes
 (#2673)

These help us track leaking workers, schedulers, and clients
---
 distributed/client.py               |  3 +++
 distributed/nanny.py                |  3 +++
 distributed/scheduler.py            |  3 +++
 distributed/tests/test_client.py    | 11 +++++++---
 distributed/tests/test_scheduler.py |  4 +++-
 distributed/tests/test_worker.py    |  9 ++------
 distributed/utils_test.py           | 24 ++++++++++-----------
 distributed/worker.py               | 33 +++++++++++------------------
 8 files changed, 46 insertions(+), 44 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 97728929f33..e31b808abcf 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -577,6 +577,8 @@ class resembles executors in ``concurrent.futures`` but also allows
     distributed.deploy.local.LocalCluster:
     """
 
+    _instances = weakref.WeakSet()
+
     def __init__(
         self,
         address=None,
@@ -710,6 +712,7 @@ def __init__(
             ext(self)
 
         self.start(timeout=timeout)
+        Client._instances.add(self)
 
         from distributed.recreate_exceptions import ReplayExceptionClient
 
diff --git a/distributed/nanny.py b/distributed/nanny.py
index 4bc0eeef6f9..7be630cb4e9 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -9,6 +9,7 @@
 import threading
 import uuid
 import warnings
+import weakref
 
 import dask
 from tornado import gen
@@ -42,6 +43,7 @@ class Nanny(ServerNode):
     them as necessary.
     """
 
+    _instances = weakref.WeakSet()
     process = None
     status = None
 
@@ -149,6 +151,7 @@ def __init__(
         )
 
         self._listen_address = listen_address
+        Nanny._instances.add(self)
         self.status = "init"
 
     def __repr__(self):
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 87cc4fda8a4..68a80ac664b 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -12,6 +12,7 @@
 import pickle
 import random
 import six
+import weakref
 
 import psutil
 import sortedcontainers
@@ -819,6 +820,7 @@ class Scheduler(ServerNode):
     """
 
     default_port = 8786
+    _instances = weakref.WeakSet()
 
     def __init__(
         self,
@@ -1113,6 +1115,7 @@ def __init__(
             ext(self)
 
         setproctitle("dask-scheduler [not started]")
+        Scheduler._instances.add(self)
 
     ##################
     # Administration #
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index bce1066c0d2..5144c0f0868 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -4829,9 +4829,7 @@ def f(x):
 def test_get_client_no_cluster():
     # Clean up any global workers added by other tests. This test requires that
     # there are no global workers.
-    from distributed.worker import _global_workers
-
-    del _global_workers[:]
+    Worker._instances.clear()
 
     msg = "No global client found and no address provided"
     with pytest.raises(ValueError, match=r"^{}$".format(msg)):
@@ -5707,5 +5705,12 @@ def test_direct_to_workers(s, loop):
         assert "gather" not in str(resp)
 
 
+@gen_cluster(client=True)
+def test_instances(c, s, a, b):
+    assert list(Client._instances) == [c]
+    assert list(Scheduler._instances) == [s]
+    assert set(Worker._instances) == {a, b}
+
+
 if sys.version_info >= (3, 5):
     from distributed.tests.py3_test_client import *  # noqa F401
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index da750dd9196..ceb992d6e22 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -821,7 +821,9 @@ def test_file_descriptors(c, s):
     yield [n.close() for n in nannies]
 
     assert not s.rpc.open
-    assert not any(occ for addr, occ in c.rpc.occupied.items() if occ != s.address)
+    assert not any(
+        occ for addr, occ in c.rpc.occupied.items() if occ != s.address
+    ), list(c.rpc._created)
     assert not s.stream_comms
 
     start = time()
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 77df078fa89..d8ca4d31481 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -934,14 +934,9 @@ def f():
 
 @gen_cluster()
 def test_global_workers(s, a, b):
-    from distributed.worker import _global_workers
-
-    n = len(_global_workers)
-    w = _global_workers[-1]()
+    n = len(Worker._instances)
+    w = first(Worker._instances)
     assert w is a or w is b
-    yield a.close()
-    yield b.close()
-    assert len(_global_workers) == n - 2
 
 
 @pytest.mark.skipif(WINDOWS, reason="file descriptors")
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index 0a1cf447cfd..4cef981cd08 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -60,7 +60,8 @@
     iscoroutinefunction,
     thread_state,
 )
-from .worker import Worker, TOTAL_MEMORY, _global_workers
+from .worker import Worker, TOTAL_MEMORY
+from .nanny import Nanny
 
 try:
     import dask.array  # register config
@@ -109,14 +110,13 @@ def invalid_python_script(tmpdir_factory):
 
 @gen.coroutine
 def cleanup_global_workers():
-    for w in _global_workers:
-        w = w()
-        w.close(report=False, executor_wait=False)
+    for worker in Worker._instances:
+        worker.close(report=False, executor_wait=False)
 
 
 @pytest.fixture
 def loop():
-    del _global_workers[:]
+    Worker._instances.clear()
     _global_clients.clear()
     with pristine_loop() as loop:
         # Monkey-patch IOLoop.start to wait for loop stop
@@ -146,7 +146,7 @@ def start():
             pass
         else:
             is_stopped.wait()
-    del _global_workers[:]
+    Worker._instances.clear()
 
     start = time()
     while set(_global_clients):
@@ -511,8 +511,6 @@ def wait_until_closed():
 
 
 def run_nanny(q, scheduler_q, **kwargs):
-    from distributed import Nanny
-
     with log_errors():
         with pristine_loop() as loop:
             scheduler_addr = scheduler_q.get()
@@ -924,7 +922,10 @@ def _(func):
             func = gen.coroutine(func)
 
         def test_func():
-            del _global_workers[:]
+            Client._instances.clear()
+            Worker._instances.clear()
+            Scheduler._instances.clear()
+            Nanny._instances.clear()
             _global_clients.clear()
             Comm._instances.clear()
             active_threads_start = set(threading._active)
@@ -1029,12 +1030,11 @@ def coro():
                             pass
                         del w.data
                 DequeHandler.clear_all_instances()
-                for w in _global_workers:
-                    w = w()
+                for w in Worker._instances:
                     w.close(report=False, executor_wait=False)
                     if w.status == "running":
                         w.close()
-                del _global_workers[:]
+                Worker._instances.clear()
 
             if PY3 and not WINDOWS and check_new_threads:
                 start = time()
diff --git a/distributed/worker.py b/distributed/worker.py
index 5302d94c6de..f5f4f2bdd50 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -19,9 +19,9 @@
 from dask.compatibility import apply
 
 try:
-    from cytoolz import pluck, partial, merge
+    from cytoolz import pluck, partial, merge, first
 except ImportError:
-    from toolz import pluck, partial, merge
+    from toolz import pluck, partial, merge, first
 from tornado.gen import Return
 from tornado import gen
 from tornado.ioloop import IOLoop
@@ -87,8 +87,6 @@
 
 DEFAULT_EXTENSIONS = [PubSubWorkerExtension]
 
-_global_workers = []
-
 
 class Worker(ServerNode):
     """ Worker node in a Dask distributed cluster
@@ -275,6 +273,8 @@ class Worker(ServerNode):
     distributed.nanny.Nanny
     """
 
+    _instances = weakref.WeakSet()
+
     def __init__(
         self,
         scheduler_ip=None,
@@ -630,7 +630,7 @@ def __init__(
         )
         self.periodic_callbacks["profile-cycle"] = pc
 
-        _global_workers.append(weakref.ref(self))
+        Worker._instances.add(self)
 
     ##################
     # Administrative #
@@ -1022,23 +1022,12 @@ def close(self, report=True, timeout=10, nanny=True, executor_wait=True):
 
             self.rpc.close()
             self._closed.set()
-            self._remove_from_global_workers()
 
             self.status = "closed"
             yield ServerNode.close(self)
 
             setproctitle("dask-worker [closed]")
 
-    def __del__(self):
-        self._remove_from_global_workers()
-
-    def _remove_from_global_workers(self):
-        for ref in list(_global_workers):
-            if ref() is self:
-                _global_workers.remove(ref)
-            if ref() is None:
-                _global_workers.remove(ref)
-
     @gen.coroutine
     def terminate(self, comm, report=True):
         yield self.close(report=report)
@@ -2807,11 +2796,10 @@ def get_worker():
     try:
         return thread_state.execution_state["worker"]
     except AttributeError:
-        for ref in _global_workers[::-1]:
-            worker = ref()
-            if worker:
-                return worker
-        raise ValueError("No workers found")
+        try:
+            return first(Worker._instances)
+        except StopIteration:
+            raise ValueError("No workers found")
 
 
 def get_client(address=None, timeout=3, resolve_address=True):
@@ -3299,3 +3287,6 @@ def run(server, comm, function, args=(), kwargs={}, is_coro=None, wait=True):
     else:
         response = {"status": "OK", "result": to_serialize(result)}
     raise Return(response)
+
+
+_global_workers = Worker._instances

From ee1008416eb73418c6b656a1ab62af62c2087ffc Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sun, 12 May 2019 10:03:36 -0500
Subject: [PATCH 0282/1550] Organize thread/process/instance checking in
 utils_test.py (#2687)

This collects various state checking functionality in one place.
It also makes some modifications that were previously causing Dask tests
to fail.
---
 distributed/utils_test.py | 546 +++++++++++++++++++-------------------
 1 file changed, 280 insertions(+), 266 deletions(-)

diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index 4cef981cd08..e40912fcb33 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -148,15 +148,7 @@ def start():
             is_stopped.wait()
     Worker._instances.clear()
 
-    start = time()
-    while set(_global_clients):
-        sleep(0.1)
-        assert time() < start + 10
-
     _cleanup_dangling()
-
-    assert_no_leaked_processes()
-
     _global_clients.clear()
 
 
@@ -630,158 +622,123 @@ def cluster(
     nworkers=2, nanny=False, worker_kwargs={}, active_rpc_timeout=1, scheduler_kwargs={}
 ):
     ws = weakref.WeakSet()
-
-    reset_config()
-    Comm._instances.clear()
-
-    for name, level in logging_levels.items():
-        logging.getLogger(name).setLevel(level)
-
     enable_proctitle_on_children()
 
-    with pristine_loop() as loop:
-        with check_active_rpc(loop, active_rpc_timeout):
-            if nanny:
-                _run_worker = run_nanny
-            else:
-                _run_worker = run_worker
+    with clean(timeout=active_rpc_timeout, threads=False) as loop:
+        if nanny:
+            _run_worker = run_nanny
+        else:
+            _run_worker = run_worker
 
-            # The scheduler queue will receive the scheduler's address
-            scheduler_q = mp_context.Queue()
+        # The scheduler queue will receive the scheduler's address
+        scheduler_q = mp_context.Queue()
 
-            # Launch scheduler
-            scheduler = mp_context.Process(
-                name="Dask cluster test: Scheduler",
-                target=run_scheduler,
-                args=(scheduler_q, nworkers + 1),
-                kwargs=scheduler_kwargs,
+        # Launch scheduler
+        scheduler = mp_context.Process(
+            name="Dask cluster test: Scheduler",
+            target=run_scheduler,
+            args=(scheduler_q, nworkers + 1),
+            kwargs=scheduler_kwargs,
+        )
+        ws.add(scheduler)
+        scheduler.daemon = True
+        scheduler.start()
+
+        # Launch workers
+        workers = []
+        for i in range(nworkers):
+            q = mp_context.Queue()
+            fn = "_test_worker-%s" % uuid.uuid4()
+            kwargs = merge(
+                {"ncores": 1, "local_dir": fn, "memory_limit": TOTAL_MEMORY},
+                worker_kwargs,
             )
-            ws.add(scheduler)
-            scheduler.daemon = True
-            scheduler.start()
-
-            # Launch workers
-            workers = []
-            for i in range(nworkers):
-                q = mp_context.Queue()
-                fn = "_test_worker-%s" % uuid.uuid4()
-                kwargs = merge(
-                    {"ncores": 1, "local_dir": fn, "memory_limit": TOTAL_MEMORY},
-                    worker_kwargs,
-                )
-                proc = mp_context.Process(
-                    name="Dask cluster test: Worker",
-                    target=_run_worker,
-                    args=(q, scheduler_q),
-                    kwargs=kwargs,
-                )
-                ws.add(proc)
-                workers.append({"proc": proc, "queue": q, "dir": fn})
+            proc = mp_context.Process(
+                name="Dask cluster test: Worker",
+                target=_run_worker,
+                args=(q, scheduler_q),
+                kwargs=kwargs,
+            )
+            ws.add(proc)
+            workers.append({"proc": proc, "queue": q, "dir": fn})
 
+        for worker in workers:
+            worker["proc"].start()
+        try:
             for worker in workers:
-                worker["proc"].start()
-            try:
-                for worker in workers:
-                    worker["address"] = worker["queue"].get(timeout=5)
-            except Empty:
-                raise pytest.xfail.Exception("Worker failed to start in test")
+                worker["address"] = worker["queue"].get(timeout=5)
+        except Empty:
+            raise pytest.xfail.Exception("Worker failed to start in test")
 
-            saddr = scheduler_q.get()
+        saddr = scheduler_q.get()
 
-            start = time()
+        start = time()
+        try:
             try:
-                try:
-                    security = scheduler_kwargs["security"]
-                    rpc_kwargs = {
-                        "connection_args": security.get_connection_args("client")
-                    }
-                except KeyError:
-                    rpc_kwargs = {}
-
-                with rpc(saddr, **rpc_kwargs) as s:
-                    while True:
-                        ncores = loop.run_sync(s.ncores)
-                        if len(ncores) == nworkers:
-                            break
-                        if time() - start > 5:
-                            raise Exception("Timeout on cluster creation")
-
-                # avoid sending processes down to function
-                yield {"address": saddr}, [
-                    {"address": w["address"], "proc": weakref.ref(w["proc"])}
-                    for w in workers
-                ]
-            finally:
-                logger.debug("Closing out test cluster")
-
-                loop.run_sync(
-                    lambda: disconnect_all(
-                        [w["address"] for w in workers],
-                        timeout=0.5,
-                        rpc_kwargs=rpc_kwargs,
-                    )
-                )
-                loop.run_sync(
-                    lambda: disconnect(saddr, timeout=0.5, rpc_kwargs=rpc_kwargs)
+                security = scheduler_kwargs["security"]
+                rpc_kwargs = {"connection_args": security.get_connection_args("client")}
+            except KeyError:
+                rpc_kwargs = {}
+
+            with rpc(saddr, **rpc_kwargs) as s:
+                while True:
+                    ncores = loop.run_sync(s.ncores)
+                    if len(ncores) == nworkers:
+                        break
+                    if time() - start > 5:
+                        raise Exception("Timeout on cluster creation")
+
+            # avoid sending processes down to function
+            yield {"address": saddr}, [
+                {"address": w["address"], "proc": weakref.ref(w["proc"])}
+                for w in workers
+            ]
+        finally:
+            logger.debug("Closing out test cluster")
+
+            loop.run_sync(
+                lambda: disconnect_all(
+                    [w["address"] for w in workers], timeout=0.5, rpc_kwargs=rpc_kwargs
                 )
+            )
+            loop.run_sync(lambda: disconnect(saddr, timeout=0.5, rpc_kwargs=rpc_kwargs))
 
-                scheduler.terminate()
-                scheduler_q.close()
-                scheduler_q._reader.close()
-                scheduler_q._writer.close()
+            scheduler.terminate()
+            scheduler_q.close()
+            scheduler_q._reader.close()
+            scheduler_q._writer.close()
 
-                for w in workers:
-                    w["proc"].terminate()
-                    w["queue"].close()
-                    w["queue"]._reader.close()
-                    w["queue"]._writer.close()
+            for w in workers:
+                w["proc"].terminate()
+                w["queue"].close()
+                w["queue"]._reader.close()
+                w["queue"]._writer.close()
 
-                scheduler.join(2)
-                del scheduler
-                for proc in [w["proc"] for w in workers]:
-                    proc.join(timeout=2)
+            scheduler.join(2)
+            del scheduler
+            for proc in [w["proc"] for w in workers]:
+                proc.join(timeout=2)
 
-                with ignoring(UnboundLocalError):
-                    del worker, w, proc
-                del workers[:]
+            with ignoring(UnboundLocalError):
+                del worker, w, proc
+            del workers[:]
 
-                for fn in glob("_test_worker-*"):
-                    with ignoring(OSError):
-                        shutil.rmtree(fn)
+            for fn in glob("_test_worker-*"):
+                with ignoring(OSError):
+                    shutil.rmtree(fn)
 
-            try:
-                client = default_client()
-            except ValueError:
-                pass
-            else:
-                client.close()
+        try:
+            client = default_client()
+        except ValueError:
+            pass
+        else:
+            client.close()
 
     start = time()
     while list(ws):
         sleep(0.01)
         assert time() < start + 1, "Workers still around after one second"
 
-    for i in range(5):
-        if all(c.closed() for c in Comm._instances):
-            break
-        else:
-            sleep(0.1)
-    else:
-        L = [c for c in Comm._instances if not c.closed()]
-        Comm._instances.clear()
-        print("Unclosed Comms", L)
-        # raise ValueError("Unclosed Comms", L)
-
-    assert_no_leaked_processes()
-
-
-def assert_no_leaked_processes():
-    for i in range(20):
-        if mp_context.active_children():
-            sleep(0.1)
-    else:
-        assert not mp_context.active_children()
-
 
 @gen.coroutine
 def disconnect(addr, timeout=3, rpc_kwargs=None):
@@ -922,147 +879,95 @@ def _(func):
             func = gen.coroutine(func)
 
         def test_func():
-            Client._instances.clear()
-            Worker._instances.clear()
-            Scheduler._instances.clear()
-            Nanny._instances.clear()
-            _global_clients.clear()
-            Comm._instances.clear()
-            active_threads_start = set(threading._active)
-
-            reset_config()
-
-            dask.config.set({"distributed.comm.timeouts.connect": "5s"})
-            # Restore default logging levels
-            # XXX use pytest hooks/fixtures instead?
-            for name, level in logging_levels.items():
-                logging.getLogger(name).setLevel(level)
-
             result = None
             workers = []
+            with clean(threads=check_new_threads, timeout=active_rpc_timeout) as loop:
 
-            with pristine_loop() as loop:
-                with check_active_rpc(loop, active_rpc_timeout):
-
-                    @gen.coroutine
-                    def coro():
-                        with dask.config.set(config):
-                            s = False
-                            for i in range(5):
-                                try:
-                                    s, ws = yield start_cluster(
-                                        ncores,
-                                        scheduler,
-                                        loop,
-                                        security=security,
-                                        Worker=Worker,
-                                        scheduler_kwargs=scheduler_kwargs,
-                                        worker_kwargs=worker_kwargs,
-                                    )
-                                except Exception as e:
-                                    logger.error(
-                                        "Failed to start gen_cluster, retrying",
-                                        exc_info=True,
-                                    )
-                                else:
-                                    workers[:] = ws
-                                    args = [s] + workers
-                                    break
-                            if s is False:
-                                raise Exception("Could not start cluster")
-                            if client:
-                                c = yield Client(
-                                    s.address,
-                                    loop=loop,
+                @gen.coroutine
+                def coro():
+                    with dask.config.set(config):
+                        s = False
+                        for i in range(5):
+                            try:
+                                s, ws = yield start_cluster(
+                                    ncores,
+                                    scheduler,
+                                    loop,
                                     security=security,
-                                    asynchronous=True,
-                                    **client_kwargs
+                                    Worker=Worker,
+                                    scheduler_kwargs=scheduler_kwargs,
+                                    worker_kwargs=worker_kwargs,
                                 )
-                                args = [c] + args
-                            try:
-                                future = func(*args)
-                                if timeout:
-                                    future = gen.with_timeout(
-                                        timedelta(seconds=timeout), future
-                                    )
-                                result = yield future
-                                if s.validate:
-                                    s.validate_state()
-                            finally:
-                                if client and c.status not in ("closing", "closed"):
-                                    yield c._close(fast=s.status == "closed")
-                                yield end_cluster(s, workers)
-                                yield gen.with_timeout(
-                                    timedelta(seconds=1), cleanup_global_workers()
+                            except Exception as e:
+                                logger.error(
+                                    "Failed to start gen_cluster, retrying",
+                                    exc_info=True,
                                 )
-
-                            try:
-                                c = yield default_client()
-                            except ValueError:
-                                pass
-                            else:
-                                yield c._close(fast=True)
-
-                            for i in range(5):
-                                if all(c.closed() for c in Comm._instances):
-                                    break
-                                else:
-                                    yield gen.sleep(0.05)
                             else:
-                                L = [c for c in Comm._instances if not c.closed()]
-                                Comm._instances.clear()
-                                # raise ValueError("Unclosed Comms", L)
-                                print("Unclosed Comms", L)
-
-                            raise gen.Return(result)
-
-                    result = loop.run_sync(
-                        coro, timeout=timeout * 2 if timeout else timeout
-                    )
+                                workers[:] = ws
+                                args = [s] + workers
+                                break
+                        if s is False:
+                            raise Exception("Could not start cluster")
+                        if client:
+                            c = yield Client(
+                                s.address,
+                                loop=loop,
+                                security=security,
+                                asynchronous=True,
+                                **client_kwargs
+                            )
+                            args = [c] + args
+                        try:
+                            future = func(*args)
+                            if timeout:
+                                future = gen.with_timeout(
+                                    timedelta(seconds=timeout), future
+                                )
+                            result = yield future
+                            if s.validate:
+                                s.validate_state()
+                        finally:
+                            if client and c.status not in ("closing", "closed"):
+                                yield c._close(fast=s.status == "closed")
+                            yield end_cluster(s, workers)
+                            yield gen.with_timeout(
+                                timedelta(seconds=1), cleanup_global_workers()
+                            )
 
-                for w in workers:
-                    if getattr(w, "data", None):
                         try:
-                            w.data.clear()
-                        except EnvironmentError:
-                            # zict backends can fail if their storage directory
-                            # was already removed
+                            c = yield default_client()
+                        except ValueError:
                             pass
-                        del w.data
-                DequeHandler.clear_all_instances()
-                for w in Worker._instances:
-                    w.close(report=False, executor_wait=False)
-                    if w.status == "running":
-                        w.close()
-                Worker._instances.clear()
-
-            if PY3 and not WINDOWS and check_new_threads:
-                start = time()
-                while True:
-                    bad = [
-                        t
-                        for t, v in threading._active.items()
-                        if t not in active_threads_start
-                        and "Threaded" not in v.name
-                        and "watch message" not in v.name
-                        and "TCP-Executor" not in v.name
-                    ]
-                    if not bad:
-                        break
-                    else:
-                        sleep(0.01)
-                    if time() > start + 5:
-                        from distributed import profile
+                        else:
+                            yield c._close(fast=True)
 
-                        tid = bad[0]
-                        thread = threading._active[tid]
-                        call_stacks = profile.call_stack(sys._current_frames()[tid])
-                        assert False, (thread, call_stacks)
-            _cleanup_dangling()
-            with ignoring(AttributeError):
-                del thread_state.on_event_loop_thread
+                        for i in range(5):
+                            if all(c.closed() for c in Comm._instances):
+                                break
+                            else:
+                                yield gen.sleep(0.05)
+                        else:
+                            L = [c for c in Comm._instances if not c.closed()]
+                            Comm._instances.clear()
+                            # raise ValueError("Unclosed Comms", L)
+                            print("Unclosed Comms", L)
 
-            assert_no_leaked_processes()
+                        raise gen.Return(result)
+
+                result = loop.run_sync(
+                    coro, timeout=timeout * 2 if timeout else timeout
+                )
+
+            for w in workers:
+                if getattr(w, "data", None):
+                    try:
+                        w.data.clear()
+                    except EnvironmentError:
+                        # zict backends can fail if their storage directory
+                        # was already removed
+                        pass
+                    del w.data
 
             return result
 
@@ -1510,3 +1415,112 @@ def gen_tls_cluster(**kwargs):
     return gen_cluster(
         scheduler="tls://127.0.0.1", security=tls_only_security(), **kwargs
     )
+
+
+@contextmanager
+def check_thread_leak():
+    active_threads_start = set(threading._active)
+
+    yield
+
+    start = time()
+    while True:
+        bad = [
+            t
+            for t, v in threading._active.items()
+            if t not in active_threads_start
+            and "Threaded" not in v.name
+            and "watch message" not in v.name
+            and "TCP-Executor" not in v.name
+        ]
+        if not bad:
+            break
+        else:
+            sleep(0.01)
+        if time() > start + 5:
+            from distributed import profile
+
+            tid = bad[0]
+            thread = threading._active[tid]
+            call_stacks = profile.call_stack(sys._current_frames()[tid])
+            assert False, (thread, call_stacks)
+
+
+@contextmanager
+def check_process_leak():
+    start_children = set(mp_context.active_children())
+
+    yield
+
+    for i in range(50):
+        if not set(mp_context.active_children()) - start_children:
+            break
+        else:
+            sleep(0.2)
+    else:
+        assert not mp_context.active_children()
+
+    _cleanup_dangling()
+
+
+@contextmanager
+def check_instances():
+    Client._instances.clear()
+    Worker._instances.clear()
+    Scheduler._instances.clear()
+    Nanny._instances.clear()
+    _global_clients.clear()
+    Comm._instances.clear()
+
+    yield
+
+    start = time()
+    while set(_global_clients):
+        sleep(0.1)
+        assert time() < start + 10
+
+    _global_clients.clear()
+
+    for w in Worker._instances:
+        w.close(report=False, executor_wait=False)
+        if w.status == "running":
+            w.close()
+    Worker._instances.clear()
+
+    for i in range(5):
+        if all(c.closed() for c in Comm._instances):
+            break
+        else:
+            sleep(0.1)
+    else:
+        L = [c for c in Comm._instances if not c.closed()]
+        Comm._instances.clear()
+        print("Unclosed Comms", L)
+        # raise ValueError("Unclosed Comms", L)
+
+    DequeHandler.clear_all_instances()
+
+
+@contextmanager
+def clean(threads=not WINDOWS, processes=True, instances=True, timeout=1):
+    @contextmanager
+    def null():
+        yield
+
+    with check_thread_leak() if threads else null():
+        with pristine_loop() as loop:
+            with check_process_leak() if processes else null():
+                with check_instances() if instances else null():
+                    with check_active_rpc(loop, timeout):
+                        reset_config()
+
+                        dask.config.set({"distributed.comm.timeouts.connect": "5s"})
+                        # Restore default logging levels
+                        # XXX use pytest hooks/fixtures instead?
+                        for name, level in logging_levels.items():
+                            logging.getLogger(name).setLevel(level)
+
+                        yield loop
+
+                        with ignoring(AttributeError):
+                            del thread_state.on_event_loop_thread

From 3142dda225baaf1db0b8c73f240f7fad8941fc1e Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 13 May 2019 08:11:20 -0500
Subject: [PATCH 0283/1550] Learn bandwidth over time (#2658)

In order to schedule tasks intelligently we need to know how long
communications will take.  To do this, we need to estimate the bandwidth
of the network.  This can vary by orders of magnitude depending on
hardwware.

Previously we asked the user to specify this in configuration.

Now we learn it over time.  Each worker keeps an exponentially weighted
moving average for all of its data communications.  It sends this
information to the scheduler as part of the heartbeats (which include
lots of other diagnostic information).  The scheduler updates its own
measurement accordingly.
---
 distributed/scheduler.py            |  9 ++++++---
 distributed/stealing.py             |  3 +--
 distributed/tests/test_scheduler.py | 28 +++++++++++++++++++++-------
 distributed/tests/test_steal.py     |  8 ++++----
 distributed/utils.py                |  2 ++
 distributed/worker.py               | 11 +++++++++--
 6 files changed, 43 insertions(+), 18 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 68a80ac664b..446b479c769 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -54,6 +54,7 @@
     no_default,
     DequeHandler,
     parse_timedelta,
+    parse_bytes,
     PeriodicCallback,
     shutting_down,
 )
@@ -72,7 +73,6 @@
 logger = logging.getLogger(__name__)
 
 
-BANDWIDTH = dask.config.get("distributed.scheduler.bandwidth")
 ALLOWED_FAILURES = dask.config.get("distributed.scheduler.allowed-failures")
 
 LOG_PDB = dask.config.get("distributed.admin.pdb-on-err")
@@ -868,6 +868,7 @@ def __init__(
         else:
             self.idle_timeout = None
         self.time_started = time()
+        self.bandwidth = parse_bytes(dask.config.get("distributed.scheduler.bandwidth"))
 
         self.security = security or Security()
         assert isinstance(self.security, Security)
@@ -1359,6 +1360,8 @@ def heartbeat_worker(
         host_info = host_info or {}
 
         self.host_info[host]["last-seen"] = local_now
+        frac = 1 / 20 / len(self.workers)
+        self.bandwidth = self.bandwidth * (1 - frac) + metrics["bandwidth"] * frac
 
         ws = self.workers.get(address)
         if not ws:
@@ -3336,7 +3339,7 @@ def get_comm_cost(self, ts, ws):
         Get the estimated communication cost (in s.) to compute the task
         on the given worker.
         """
-        return sum(dts.nbytes for dts in ts.dependencies - ws.has_what) / BANDWIDTH
+        return sum(dts.nbytes for dts in ts.dependencies - ws.has_what) / self.bandwidth
 
     def get_task_duration(self, ts, default=0.5):
         """
@@ -4522,7 +4525,7 @@ def worker_objective(self, ts, ws):
             [dts.get_nbytes() for dts in ts.dependencies if ws not in dts.who_has]
         )
         stack_time = ws.occupancy / ws.ncores
-        start_time = comm_bytes / BANDWIDTH + stack_time
+        start_time = comm_bytes / self.bandwidth + stack_time
 
         if ts.actor:
             return (len(ws.actors), start_time, ws.nbytes)
diff --git a/distributed/stealing.py b/distributed/stealing.py
index d361305b105..dc8c989e39d 100644
--- a/distributed/stealing.py
+++ b/distributed/stealing.py
@@ -15,7 +15,6 @@
 except ImportError:
     from toolz import topk
 
-BANDWIDTH = 100e6
 LATENCY = 10e-3
 log_2 = log(2)
 
@@ -134,7 +133,7 @@ def steal_time_ratio(self, ts):
 
         nbytes = sum(dep.get_nbytes() for dep in ts.dependencies)
 
-        transfer_time = nbytes / BANDWIDTH + LATENCY
+        transfer_time = nbytes / self.scheduler.bandwidth + LATENCY
         split = ts.prefix
         if split in fast_tasks:
             return None, None
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index ceb992d6e22..9f61e5e710e 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -5,7 +5,7 @@
 from collections import defaultdict
 from datetime import timedelta
 import json
-from operator import add, mul
+import operator
 import sys
 from time import sleep
 
@@ -18,7 +18,7 @@
 
 from distributed import Nanny, Worker, Client, wait, fire_and_forget
 from distributed.core import connect, rpc
-from distributed.scheduler import Scheduler, BANDWIDTH
+from distributed.scheduler import Scheduler
 from distributed.client import wait
 from distributed.metrics import time
 from distributed.protocol.pickle import dumps
@@ -63,7 +63,7 @@ def test_respect_data_in_memory(c, s, a):
 
     assert s.tasks[y.key].who_has == {s.workers[a.address]}
 
-    z = delayed(add)(x, y)
+    z = delayed(operator.add)(x, y)
     f2 = c.persist(z)
     while f2.key not in s.tasks or not s.tasks[f2.key]:
         assert s.tasks[y.key].who_has
@@ -427,7 +427,10 @@ def test_filtered_communication(s, a, b):
     yield f.write(
         {
             "op": "update-graph",
-            "tasks": {"x": dumps_task((inc, 1)), "z": dumps_task((add, "x", 10))},
+            "tasks": {
+                "x": dumps_task((inc, 1)),
+                "z": dumps_task((operator.add, "x", 10)),
+            },
             "dependencies": {"x": [], "z": ["x"]},
             "client": "f",
             "keys": ["z"],
@@ -903,8 +906,8 @@ def test_learn_occupancy_multiple_workers(c, s, a, b):
 @gen_cluster(client=True)
 def test_include_communication_in_occupancy(c, s, a, b):
     s.task_duration["slowadd"] = 0.001
-    x = c.submit(mul, b"0", int(BANDWIDTH), workers=a.address)
-    y = c.submit(mul, b"1", int(BANDWIDTH * 1.5), workers=b.address)
+    x = c.submit(operator.mul, b"0", int(s.bandwidth), workers=a.address)
+    y = c.submit(operator.mul, b"1", int(s.bandwidth * 1.5), workers=b.address)
 
     z = c.submit(slowadd, x, y, delay=1)
     while z.key not in s.tasks or not s.tasks[z.key].processing_on:
@@ -1375,7 +1378,7 @@ def test_dont_recompute_if_persisted_3(c, s, a, b):
     x = delayed(inc)(1, dask_key_name="x")
     y = delayed(inc)(2, dask_key_name="y")
     z = delayed(inc)(y, dask_key_name="z")
-    w = delayed(add)(x, z, dask_key_name="w")
+    w = delayed(operator.add)(x, z, dask_key_name="w")
 
     ww = w.persist()
     yield wait(ww)
@@ -1513,6 +1516,17 @@ def test_idle_timeout(c, s, a, b):
     assert b.status == "closed"
 
 
+@gen_cluster(client=True, config={"distributed.scheduler.bandwidth": "100 GB"})
+def test_bandwidth(c, s, a, b):
+    start = s.bandwidth
+    x = c.submit(operator.mul, b"0", 20000, workers=a.address)
+    y = c.submit(lambda x: x, x, workers=b.address)
+    yield y
+    yield b.heartbeat()
+    assert s.bandwidth < start  # we've learned that we're slower
+    assert b.latency
+
+
 @gen_cluster()
 def test_workerstate_clean(s, a, b):
     ws = s.workers[a.address].clean()
diff --git a/distributed/tests/test_steal.py b/distributed/tests/test_steal.py
index 8edeb8e339c..7348d164c72 100644
--- a/distributed/tests/test_steal.py
+++ b/distributed/tests/test_steal.py
@@ -14,7 +14,7 @@
 from distributed import Nanny, Worker, wait, worker_client
 from distributed.config import config
 from distributed.metrics import time
-from distributed.scheduler import BANDWIDTH, key_split
+from distributed.scheduler import key_split
 from distributed.utils_test import (
     slowinc,
     slowadd,
@@ -394,7 +394,7 @@ def assert_balanced(inp, expected, c, s, *workers):
                 ts = s.tasks[dat.key]
                 # Ensure scheduler state stays consistent
                 old_nbytes = ts.nbytes
-                ts.nbytes = BANDWIDTH * t
+                ts.nbytes = s.bandwidth * t
                 for ws in ts.who_has:
                     ws.nbytes += ts.nbytes - old_nbytes
             else:
@@ -499,8 +499,8 @@ def test_restart(c, s, a, b):
 def test_steal_communication_heavy_tasks(c, s, a, b):
     steal = s.extensions["stealing"]
     s.task_duration["slowadd"] = 0.001
-    x = c.submit(mul, b"0", int(BANDWIDTH), workers=a.address)
-    y = c.submit(mul, b"1", int(BANDWIDTH), workers=b.address)
+    x = c.submit(mul, b"0", int(s.bandwidth), workers=a.address)
+    y = c.submit(mul, b"1", int(s.bandwidth), workers=b.address)
 
     futures = [
         c.submit(
diff --git a/distributed/utils.py b/distributed/utils.py
index d6cc5ba62cf..765035e5c13 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -1221,6 +1221,8 @@ def parse_bytes(s):
     >>> parse_bytes('MB')
     1000000
     """
+    if isinstance(s, (int, float)):
+        return int(s)
     s = s.replace(" ", "")
     if not s[0].isdigit():
         s = "1" + s
diff --git a/distributed/worker.py b/distributed/worker.py
index f5f4f2bdd50..8b8c139f356 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -400,6 +400,8 @@ def __init__(
         self.outgoing_count = 0
         self.outgoing_current_count = 0
         self.repetitively_busy = 0
+        self.bandwidth = parse_bytes(dask.config.get("distributed.scheduler.bandwidth"))
+        self.latency = 0.001
         self._client = None
 
         profile_cycle_interval = kwargs.pop(
@@ -673,6 +675,7 @@ def get_metrics(self):
             in_memory=len(self.data),
             ready=len(self.ready),
             in_flight=len(self.in_flight_tasks),
+            bandwidth=self.bandwidth,
         )
         custom = {k: metric(self) for k, metric in self.metrics.items()}
 
@@ -742,6 +745,7 @@ def _register_with_scheduler(self):
                 response = yield future
                 _end = time()
                 middle = (_start + _end) / 2
+                self.latency = (_end - start) * 0.05 + self.latency * 0.95
                 self.scheduler_delay = response["time"] - middle
                 self.status = "running"
                 break
@@ -1837,7 +1841,8 @@ def gather_dep(self, worker, dep, deps, total_nbytes, cause=None):
                     )
 
                 total_bytes = sum(self.nbytes.get(dep, 0) for dep in response["data"])
-                duration = (stop - start) or 0.5
+                duration = (stop - start) or 0.010
+                bandwidth = total_bytes / duration
                 self.incoming_transfer_log.append(
                     {
                         "start": start + self.scheduler_delay,
@@ -1848,10 +1853,12 @@ def gather_dep(self, worker, dep, deps, total_nbytes, cause=None):
                             dep: self.nbytes.get(dep, None) for dep in response["data"]
                         },
                         "total": total_bytes,
-                        "bandwidth": total_bytes / duration,
+                        "bandwidth": bandwidth,
                         "who": worker,
                     }
                 )
+                if total_bytes > 10000:
+                    self.bandwidth = self.bandwidth * 0.95 + bandwidth * 0.05
                 if self.digests is not None:
                     self.digests["transfer-bandwidth"].add(total_bytes / duration)
                     self.digests["transfer-duration"].add(duration)

From e3ffb9d12824b5a5434c5c15a2b6edfa7bf9ea12 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Thu, 9 May 2019 08:01:35 -0500
Subject: [PATCH 0284/1550] Use config accessor method for "scheduler-address"
 (#2676)

---
 distributed/cli/dask_worker.py            |  8 ++++++--
 distributed/cli/tests/test_dask_worker.py | 12 ++++++++++++
 2 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
index a0bc801a960..1e0ebe24176 100755
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -8,8 +8,8 @@
 import warnings
 
 import click
+import dask
 from distributed import Nanny, Worker
-from distributed.config import config
 from distributed.utils import get_ip_interface, parse_timedelta
 from distributed.worker import _ncores
 from distributed.security import Security
@@ -322,7 +322,11 @@ def del_pid_file():
             kwargs["service_ports"] = {"nanny": nanny_port}
         t = Worker
 
-    if not scheduler and not scheduler_file and "scheduler-address" not in config:
+    if (
+        not scheduler
+        and not scheduler_file
+        and dask.config.get("scheduler-address", None) is None
+    ):
         raise ValueError(
             "Need to provide scheduler address like\n"
             "dask-worker SCHEDULER_ADDRESS:8786"
diff --git a/distributed/cli/tests/test_dask_worker.py b/distributed/cli/tests/test_dask_worker.py
index 72084e53141..e1cfc8d5ad3 100644
--- a/distributed/cli/tests/test_dask_worker.py
+++ b/distributed/cli/tests/test_dask_worker.py
@@ -6,6 +6,7 @@
 
 import requests
 import sys
+import os
 from time import sleep
 from toolz import first
 
@@ -142,6 +143,17 @@ def test_scheduler_file(loop, nanny):
                         assert time() < start + 10
 
 
+def test_scheduler_address_env(loop, monkeypatch):
+    monkeypatch.setenv("DASK_SCHEDULER_ADDRESS", "tcp://127.0.0.1:8786")
+    with popen(["dask-scheduler", "--no-bokeh"]) as sched:
+        with popen(["dask-worker", "--no-bokeh"]):
+            with Client(os.environ["DASK_SCHEDULER_ADDRESS"], loop=loop) as c:
+                start = time()
+                while not c.scheduler_info()["workers"]:
+                    sleep(0.1)
+                    assert time() < start + 10
+
+
 def test_nprocs_requires_nanny(loop):
     with popen(["dask-scheduler", "--no-bokeh"]) as sched:
         with popen(

From 61be3a78375b730b78ce4e50aecea2fe432cdddd Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 13 May 2019 16:48:31 -0500
Subject: [PATCH 0285/1550] bump version to 1.28.1

---
 docs/source/changelog.rst | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index 1b584c02219..613773a0c1c 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,14 @@
 Changelog
 =========
 
+1.28.1 - 2019-05-13
+-------------------
+
+This is a small bugfix release due to a config change upstream.
+
+-  Use config accessor method for "scheduler-address" (#2676) `James Bourbeau`_
+
+
 1.28.0 - 2019-05-08
 -------------------
 
@@ -1030,4 +1038,4 @@ significantly without many new features.
 .. _`Peter Andreas Entschev`: https://github.com/pentschev
 .. _`condoratberlin`: https://github.com/condoratberlin
 .. _`K.-Michael Aye`: https://github.com/michaelaye
-.. _`@plbertrand`: https://github.com/plbertrand
\ No newline at end of file
+.. _`@plbertrand`: https://github.com/plbertrand

From a622ce4ac48861b07a2738baba361b9886ee1ca3 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 13 May 2019 18:13:48 -0500
Subject: [PATCH 0286/1550] Remove support for Iterators and Queues (#2671)

These add non-trivial code complexity, and don't seem to be commonly
used (based on bug reports and SO questions).  They're also a bit odd on
our tests (there are some lingering threads as a result.

This commit removes functionality for them and replaces them with
informative warnings pointing people towards normal for loops.
---
 distributed/client.py            | 156 +++++------------------
 distributed/tests/test_client.py | 206 +------------------------------
 distributed/tests/test_utils.py  |  23 +---
 distributed/utils.py             |  38 +-----
 distributed/utils_test.py        |   2 +-
 docs/source/queues.rst           | 196 +----------------------------
 6 files changed, 40 insertions(+), 581 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index e31b808abcf..7b6e14aa4ef 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -55,7 +55,6 @@
 from .cfexecutor import ClientExecutor
 from .compatibility import (
     Queue as pyQueue,
-    Empty,
     isqueue,
     html_escape,
     StopAsyncIteration,
@@ -77,7 +76,6 @@
     sync,
     funcname,
     ignoring,
-    queue_to_iterator,
     tokey,
     log_errors,
     str_graph,
@@ -1424,24 +1422,6 @@ def submit(self, func, *args, **kwargs):
 
         return futures[skey]
 
-    def _threaded_map(self, q_out, func, qs_in, **kwargs):
-        """ Internal function for mapping Queue """
-        if isqueue(qs_in[0]):
-            get = pyQueue.get
-        elif isinstance(qs_in[0], Iterator):
-            get = next
-        else:
-            raise NotImplementedError()
-
-        while True:
-            try:
-                args = [get(q) for q in qs_in]
-            except StopIteration as e:
-                q_out.put(e)
-                break
-            f = self.submit(func, *args, **kwargs)
-            q_out.put(f)
-
     def map(self, func, *iterables, **kwargs):
         """ Map a function on a sequence of arguments
 
@@ -1450,7 +1430,8 @@ def map(self, func, *iterables, **kwargs):
         Parameters
         ----------
         func: callable
-        iterables: Iterables, Iterators, or Queues
+        iterables: Iterables
+            List-like objects to map over.  They should have the same length.
         key: str, list
             Prefix for task names if string.  Explicit names if list.
         pure: bool (defaults to True)
@@ -1489,20 +1470,10 @@ def map(self, func, *iterables, **kwargs):
         if all(map(isqueue, iterables)) or all(
             isinstance(i, Iterator) for i in iterables
         ):
-            maxsize = kwargs.pop("maxsize", 0)
-            q_out = pyQueue(maxsize=maxsize)
-            t = threading.Thread(
-                target=self._threaded_map,
-                name="Threaded map()",
-                args=(q_out, func, iterables),
-                kwargs=kwargs,
+            raise TypeError(
+                "Dask no longer supports mapping over Iterators or Queues."
+                "Consider using a normal for loop and Client.submit"
             )
-            t.daemon = True
-            t.start()
-            if isqueue(iterables[0]):
-                return q_out
-            else:
-                return queue_to_iterator(q_out)
 
         key = kwargs.pop("key", None)
         key = key or funcname(func)
@@ -1738,22 +1709,7 @@ def _gather_remote(self, direct, local_worker):
 
         raise gen.Return(response)
 
-    def _threaded_gather(self, qin, qout, **kwargs):
-        """ Internal function for gathering Queue """
-        while True:
-            L = [qin.get()]
-            while qin.empty():
-                try:
-                    L.append(qin.get_nowait())
-                except Empty:
-                    break
-            results = self.gather(L, **kwargs)
-            for item in results:
-                qout.put(item)
-
-    def gather(
-        self, futures, errors="raise", maxsize=0, direct=None, asynchronous=None
-    ):
+    def gather(self, futures, errors="raise", direct=None, asynchronous=None):
         """ Gather futures from distributed memory
 
         Accepts a future, nested container of futures, iterator, or queue.
@@ -1763,7 +1719,7 @@ def gather(
         ----------
         futures: Collection of futures
             This can be a possibly nested collection of Future objects.
-            Collections can be lists, sets, iterators, queues or dictionaries
+            Collections can be lists, sets, or dictionaries
         errors: string
             Either 'raise' or 'skip' if we should raise if a future has erred
             or skip its inclusion in the output collection
@@ -1771,9 +1727,6 @@ def gather(
             Whether or not to connect directly to the workers, or to ask
             the scheduler to serve as intermediary.  This can also be set when
             creating the Client.
-        maxsize: int
-            If the input is a queue then this produces an output queue with a
-            maximum size.
 
         Returns
         -------
@@ -1790,25 +1743,16 @@ def gather(
         >>> c.gather([x, [x], x])  # support lists and dicts # doctest: +SKIP
         [3, [3], 3]
 
-        >>> seq = c.gather(iter([x, x]))  # support iterators # doctest: +SKIP
-        >>> next(seq)  # doctest: +SKIP
-        3
-
         See Also
         --------
         Client.scatter: Send data out to cluster
         """
         if isqueue(futures):
-            qout = pyQueue(maxsize=maxsize)
-            t = threading.Thread(
-                target=self._threaded_gather,
-                name="Threaded gather()",
-                args=(futures, qout),
-                kwargs={"errors": errors, "direct": direct},
+            raise TypeError(
+                "Dask no longer supports gathering over Iterators and Queues. "
+                "Consider using a normal for loop and Client.submit/gather"
             )
-            t.daemon = True
-            t.start()
-            return qout
+
         elif isinstance(futures, Iterator):
             return (self.gather(f, errors=errors, direct=direct) for f in futures)
         else:
@@ -1935,27 +1879,6 @@ def _scatter(
             out = list(out.values())[0]
         raise gen.Return(out)
 
-    def _threaded_scatter(self, q_or_i, qout, **kwargs):
-        """ Internal function for scattering Iterable/Queue data """
-        while True:
-            if isqueue(q_or_i):
-                L = [q_or_i.get()]
-                while not q_or_i.empty():
-                    try:
-                        L.append(q_or_i.get_nowait())
-                    except Empty:
-                        break
-            else:
-                try:
-                    L = [next(q_or_i)]
-                except StopIteration as e:
-                    qout.put(e)
-                    break
-
-            futures = self.scatter(L, **kwargs)
-            for future in futures:
-                qout.put(future)
-
     def scatter(
         self,
         data,
@@ -1963,7 +1886,6 @@ def scatter(
         broadcast=False,
         direct=None,
         hash=True,
-        maxsize=0,
         timeout=no_default,
         asynchronous=None,
     ):
@@ -1976,7 +1898,7 @@ def scatter(
 
         Parameters
         ----------
-        data: list, iterator, dict, Queue, or object
+        data: list, dict, or object
             Data to scatter out to workers.  Output type matches input type.
         workers: list of tuples (optional)
             Optionally constrain locations of data.
@@ -1988,8 +1910,6 @@ def scatter(
             Whether or not to connect directly to the workers, or to ask
             the scheduler to serve as intermediary.  This can also be set when
             creating the Client.
-        maxsize: int (optional)
-            Maximum size of queue if using queues, 0 implies infinite
         hash: bool (optional)
             Whether or not to hash data to determine key.
             If False then this uses a random key
@@ -2018,12 +1938,6 @@ def scatter(
 
         >>> c.scatter([1, 2, 3], workers=[('hostname', 8788)])   # doctest: +SKIP
 
-        Handle streaming sequences of data with iterators or queues
-
-        >>> seq = c.scatter(iter([1, 2, 3]))  # doctest: +SKIP
-        >>> next(seq)  # doctest: +SKIP
-        <Future: status: finished, key: c0a8a20f903a4915b94db8de3ea63195>,
-
         Broadcast data to all workers
 
         >>> [future] = c.scatter([element], broadcast=True)  # doctest: +SKIP
@@ -2041,38 +1955,26 @@ def scatter(
         if timeout == no_default:
             timeout = self._timeout
         if isqueue(data) or isinstance(data, Iterator):
-            logger.debug("Starting thread for streaming data")
-            qout = pyQueue(maxsize=maxsize)
-
-            t = threading.Thread(
-                target=self._threaded_scatter,
-                name="Threaded scatter()",
-                args=(data, qout),
-                kwargs={"workers": workers, "broadcast": broadcast},
+            raise TypeError(
+                "Dask no longer supports mapping over Iterators or Queues."
+                "Consider using a normal for loop and Client.submit"
             )
-            t.daemon = True
-            t.start()
 
-            if isqueue(data):
-                return qout
-            else:
-                return queue_to_iterator(qout)
+        if hasattr(thread_state, "execution_state"):  # within worker task
+            local_worker = thread_state.execution_state["worker"]
         else:
-            if hasattr(thread_state, "execution_state"):  # within worker task
-                local_worker = thread_state.execution_state["worker"]
-            else:
-                local_worker = None
-            return self.sync(
-                self._scatter,
-                data,
-                workers=workers,
-                broadcast=broadcast,
-                direct=direct,
-                local_worker=local_worker,
-                timeout=timeout,
-                asynchronous=asynchronous,
-                hash=hash,
-            )
+            local_worker = None
+        return self.sync(
+            self._scatter,
+            data,
+            workers=workers,
+            broadcast=broadcast,
+            direct=direct,
+            local_worker=local_worker,
+            timeout=timeout,
+            asynchronous=asynchronous,
+            hash=hash,
+        )
 
     @gen.coroutine
     def _cancel(self, futures, force=False):
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 5144c0f0868..449d207a91a 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -5,7 +5,6 @@
 from collections import deque
 from concurrent.futures import CancelledError
 import gc
-import itertools
 import logging
 import os
 import pickle
@@ -52,7 +51,7 @@
     futures_of,
     temp_default_client,
 )
-from distributed.compatibility import PY3, Iterator
+from distributed.compatibility import PY3
 
 from distributed.metrics import time
 from distributed.scheduler import Scheduler, KilledWorker
@@ -1321,104 +1320,6 @@ def test_directed_scatter_sync(c, s, a, b, loop):
     assert len(has_what[a["address"]]) == 0
 
 
-def test_iterator_scatter(c):
-    aa = c.scatter([1, 2, 3])
-    assert [1, 2, 3] == c.gather(aa)
-
-    g = (i for i in range(10))
-    futures = c.scatter(g)
-    assert isinstance(futures, Iterator)
-
-    a = next(futures)
-    assert c.gather(a) == 0
-
-    futures = list(futures)
-    assert len(futures) == 9
-    assert c.gather(futures) == [1, 2, 3, 4, 5, 6, 7, 8, 9]
-
-
-def test_queue_scatter(c):
-    from distributed.compatibility import Queue
-
-    q = Queue()
-    for d in range(10):
-        q.put(d)
-
-    futures = c.scatter(q)
-    assert isinstance(futures, Queue)
-    a = futures.get()
-    assert c.gather(a) == 0
-
-
-def test_queue_scatter_gather_maxsize(c):
-    from distributed.compatibility import Queue
-
-    q = Queue(maxsize=3)
-    out = c.scatter(q, maxsize=10)
-    assert out.maxsize == 10
-    local = c.gather(q)
-    assert not local.maxsize
-
-    q = Queue()
-    out = c.scatter(q)
-    assert not out.maxsize
-    local = c.gather(out, maxsize=10)
-    assert local.maxsize == 10
-
-    q = Queue(maxsize=3)
-    out = c.scatter(q)
-    assert not out.maxsize
-
-
-def test_queue_gather(c):
-    from distributed.compatibility import Queue
-
-    q = Queue()
-
-    qin = list(range(10))
-    for d in qin:
-        q.put(d)
-
-    futures = c.scatter(q)
-    assert isinstance(futures, Queue)
-
-    ff = c.gather(futures)
-    assert isinstance(ff, Queue)
-
-    qout = []
-    for f in range(10):
-        qout.append(ff.get())
-    assert qout == qin
-
-
-@pytest.mark.skip(reason="intermittent blocking failures")
-def test_iterator_gather(c, c2):
-    i_in = list(range(10))
-
-    g = (d for d in i_in)
-    futures = c.scatter(g)
-    assert isinstance(futures, Iterator)
-
-    ff = c.gather(futures)
-    assert isinstance(ff, Iterator)
-
-    i_out = list(ff)
-    assert i_out == i_in
-
-    i_in = ["a", "b", "c", StopIteration("f"), StopIteration, "d", "c"]
-
-    g = (d for d in i_in)
-    futures = c.scatter(g)
-
-    ff = c.gather(futures)
-    i_out = list(ff)
-    assert i_out[:3] == i_in[:3]
-    # This is because StopIteration('f') != StopIteration('f')
-    assert isinstance(i_out[3], StopIteration)
-    assert i_out[3].args == i_in[3].args
-    assert i_out[4:] == i_in[4:]
-
-
 @gen_cluster(client=True)
 def test_scatter_direct(c, s, a, b):
     future = yield c.scatter(123, direct=True)
@@ -2373,109 +2274,6 @@ def test_traceback_clean(c, s, a, b):
             tb = tb.tb_next
 
 
-@gen_cluster(client=True)
-def test_map_queue(c, s, a, b):
-    from distributed.compatibility import Queue, isqueue
-
-    q_1 = Queue(maxsize=2)
-    q_2 = c.map(inc, q_1)
-    assert isqueue(q_2)
-    assert not q_2.maxsize
-    q_3 = c.map(double, q_2, maxsize=3)
-    assert isqueue(q_3)
-    assert q_3.maxsize == 3
-    q_4 = yield c._gather(q_3)
-    assert isqueue(q_4)
-
-    q_1.put(1)
-
-    f = q_4.get()
-    assert isinstance(f, Future)
-    result = yield f
-    assert result == (1 + 1) * 2
-
-
-@pytest.mark.skipif(
-    sys.version_info >= (3, 7), reason="replace StopIteration with return"
-)
-@gen_cluster(client=True)
-def test_map_iterator_with_return(c, s, a, b):
-    def g():
-        yield 1
-        yield 2
-        raise StopIteration(3)  # py2.7 compat.
-
-    f1 = c.map(lambda x: x, g())
-    assert isinstance(f1, Iterator)
-
-    start = time()  # ensure that we compute eagerly
-    while not s.tasks:
-        yield gen.sleep(0.01)
-        assert time() < start + 5
-
-    g1 = g()
-    try:
-        while True:
-            f = next(f1)
-            n = yield f
-            assert n == next(g1)
-    except StopIteration as e:
-        with pytest.raises(StopIteration) as exc_info:
-            next(g1)
-        assert e.args == exc_info.value.args
-
-
-@gen_cluster(client=True)
-def test_map_iterator(c, s, a, b):
-    x = iter([1, 2, 3])
-    y = iter([10, 20, 30])
-    f1 = c.map(add, x, y)
-    assert isinstance(f1, Iterator)
-
-    start = time()  # ensure that we compute eagerly
-    while not s.tasks:
-        yield gen.sleep(0.01)
-        assert time() < start + 5
-
-    f2 = c.map(double, f1)
-    assert isinstance(f2, Iterator)
-
-    future = next(f2)
-    result = yield future
-    assert result == (1 + 10) * 2
-    futures = list(f2)
-    results = []
-    for f in futures:
-        r = yield f
-        results.append(r)
-    assert results == [(2 + 20) * 2, (3 + 30) * 2]
-
-    items = enumerate(range(10))
-    futures = c.map(lambda x: x, items)
-    assert isinstance(futures, Iterator)
-
-    result = yield next(futures)
-    assert result == (0, 0)
-    futures_l = list(futures)
-    results = []
-    for f in futures_l:
-        r = yield f
-        results.append(r)
-    assert results == [(i, i) for i in range(1, 10)]
-
-
-@gen_cluster(client=True)
-def test_map_infinite_iterators(c, s, a, b):
-    futures = c.map(add, [1, 2], itertools.repeat(10))
-    assert len(futures) == 2
-
-
-def test_map_iterator_sync(c):
-    items = enumerate(range(10))
-    futures = c.map(lambda x: x, items)
-    next(futures).result() == (0, 0)
-
-
 @gen_cluster(client=True)
 def test_map_differnet_lengths(c, s, a, b):
     assert len(c.map(add, [1, 2], [1, 2, 3])) == 2
@@ -3557,7 +3355,7 @@ def test_get_stops_work_after_error(c):
 
 
 def test_as_completed_list(c):
-    seq = c.map(inc, iter(range(5)))
+    seq = c.map(inc, range(5))
     seq2 = list(as_completed(seq))
     assert set(c.gather(seq2)) == {1, 2, 3, 4, 5}
 
diff --git a/distributed/tests/test_utils.py b/distributed/tests/test_utils.py
index f4423d26e4a..b82dce4e7d9 100644
--- a/distributed/tests/test_utils.py
+++ b/distributed/tests/test_utils.py
@@ -14,7 +14,7 @@
 from tornado.ioloop import IOLoop
 
 import dask
-from distributed.compatibility import Queue, Empty, isqueue, PY2, Iterator
+from distributed.compatibility import Queue, Empty, PY2
 from distributed.metrics import time
 from distributed.utils import (
     All,
@@ -24,8 +24,6 @@
     str_graph,
     truncate_exception,
     get_traceback,
-    queue_to_iterator,
-    iterator_to_queue,
     _maybe_complex,
     read_block,
     seek_delimiter,
@@ -183,25 +181,6 @@ def c(x):
         assert type(tb).__name__ == "traceback"
 
 
-def test_queue_to_iterator():
-    q = Queue()
-    q.put(1)
-    q.put(2)
-
-    seq = queue_to_iterator(q)
-    assert isinstance(seq, Iterator)
-    assert next(seq) == 1
-    assert next(seq) == 2
-
-
-def test_iterator_to_queue():
-    seq = iter([1, 2, 3])
-
-    q = iterator_to_queue(seq)
-    assert isqueue(q)
-    assert q.get() == 1
-
-
 def test_str_graph():
     dsk = {"x": 1}
     assert str_graph(dsk) == dsk
diff --git a/distributed/utils.py b/distributed/utils.py
index 765035e5c13..6debdedd24e 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -46,7 +46,7 @@
 except ImportError:
     PollIOLoop = None  # dropped in tornado 6.0
 
-from .compatibility import Queue, PY3, PY2, get_thread_identity, unicode
+from .compatibility import PY3, PY2, get_thread_identity, unicode
 from .metrics import time
 
 
@@ -797,42 +797,6 @@ def truncate_exception(e, n=10000):
         return e
 
 
-if sys.version_info >= (3,):
-    # (re-)raising StopIteration is deprecated in 3.6+
-    exec(
-        """def queue_to_iterator(q):
-        while True:
-            result = q.get()
-            if isinstance(result, StopIteration):
-                return result.value
-            yield result
-        """
-    )
-else:
-    # Returning non-None from generator is a syntax error in 2.x
-    def queue_to_iterator(q):
-        while True:
-            result = q.get()
-            if isinstance(result, StopIteration):
-                raise result
-            yield result
-
-
-def _dump_to_queue(seq, q):
-    for item in seq:
-        q.put(item)
-
-
-def iterator_to_queue(seq, maxsize=0):
-    q = Queue(maxsize=maxsize)
-
-    t = threading.Thread(target=_dump_to_queue, args=(seq, q))
-    t.daemon = True
-    t.start()
-
-    return q
-
-
 def tokey(o):
     """ Convert an object to a string.
 
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index e40912fcb33..7b90745ac6d 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -399,7 +399,7 @@ def map_varying(itemslists):
     def apply(func, *args, **kwargs):
         return func(*args, **kwargs)
 
-    return apply, map(varying, itemslists)
+    return apply, list(map(varying, itemslists))
 
 
 @gen.coroutine
diff --git a/docs/source/queues.rst b/docs/source/queues.rst
index ea1bc76a4f7..34a4cae538a 100644
--- a/docs/source/queues.rst
+++ b/docs/source/queues.rst
@@ -1,194 +1,10 @@
 Data Streams with Queues
 ========================
 
-The ``Client`` methods ``scatter``, ``map``, and ``gather`` can consume and
-produce standard Python ``Queue`` objects.  This is useful for processing
-continuous streams of data.  However, it does not constitute a full streaming
-data processing pipeline like Storm.
+This feature is no longer supported.
+Instead people may want to look at the following options:
 
-.. raw:: html
-
-   <iframe width="560"
-           height="315"
-           src="https://www.youtube.com/embed/hfmwXSeM8pk?rel=0"
-           frameborder="0"
-           allowfullscreen>
-   </iframe>
-
-Example
--------
-
-We connect to a local Client.
-
-.. code-block:: python
-
-   >>> from distributed import Client
-   >>> client = Client('127.0.0.1:8786')
-   >>> client
-   <Client: scheduler=127.0.0.1:8786 workers=1 threads=4>
-
-We build a couple of toy data processing functions:
-
-.. code-block:: python
-
-   from time import sleep
-   from random import random
-
-   def inc(x):
-       from random import random
-       sleep(random() * 2)
-       return x + 1
-
-   def double(x):
-       from random import random
-       sleep(random())
-       return 2 * x
-
-And we set up an input Queue and map our functions across it.
-
-.. code-block:: python
-
-   >>> from queue import Queue
-   >>> input_q = Queue()
-   >>> remote_q = client.scatter(input_q)
-   >>> inc_q = client.map(inc, remote_q)
-   >>> double_q = client.map(double, inc_q)
-
-We will fill the ``input_q`` with local data from some stream, and then
-``remote_q``, ``inc_q`` and ``double_q`` will fill with ``Future`` objects as
-data gets moved around.
-
-We gather the futures from the ``double_q`` back to a queue holding local
-data in the local process.
-
-.. code-block:: python
-
-   >>> result_q = client.gather(double_q)
-
-Insert Data Manually
-~~~~~~~~~~~~~~~~~~~~
-
-Because we haven't placed any data into any of the queues everything is empty,
-including the final output, ``result_q``.
-
-.. code-block:: python
-
-   >>> result_q.qsize()
-   0
-
-But when we insert an entry into the ``input_q``, it starts to make its way
-through the pipeline and ends up in the ``result_q``.
-
-.. code-block:: python
-
-   >>> input_q.put(10)
-   >>> result_q.get()
-   22
-
-Insert data in a separate thread
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-We simulate a slightly more realistic situation by dumping data into the
-``input_q`` in a separate thread.  This simulates what you might get if you
-were to read from an active data source.
-
-.. code-block:: python
-
-   def load_data(q):
-       i = 0
-       while True:
-           q.put(i)
-           sleep(random())
-           i += 1
-
-    >>> from threading import Thread
-    >>> load_thread = Thread(target=load_data, args=(input_q,))
-    >>> load_thread.start()
-
-    >>> result_q.qsize()
-    4
-    >>> result_q.qsize()
-    9
-
-We consume data from the ``result_q`` and print results to the screen.
-
-.. code-block:: python
-
-   >>> while True:
-   ...     item = result_q.get()
-   ...     print(item)
-   2
-   4
-   6
-   8
-   10
-   12
-   ...
-
-Limitations
------------
-
-*  This doesn't do any sort of auto-batching of computations, so ideally you
-   batch your data to take significantly longer than 1ms to run.
-*  This isn't a proper streaming system.  There is no support outside of what
-   you see here.  In particular there are no policies for dropping data, joining
-   over time windows, etc..
-
-Extensions
-----------
-
-We can extend this small example to more complex systems that have buffers,
-split queues, merge queues, etc. all by manipulating normal Python Queues.
-
-Here are a couple of useful function to multiplex and merge queues:
-
-.. code-block:: python
-
-    from queue import Queue
-    from threading import Thread
-
-    def multiplex(n, q, **kwargs):
-        """ Convert one queue into several equivalent Queues
-
-        >>> q1, q2, q3 = multiplex(3, in_q)
-        """
-        out_queues = [Queue(**kwargs) for i in range(n)]
-        def f():
-            while True:
-                x = q.get()
-                for out_q in out_queues:
-                    out_q.put(x)
-        t = Thread(target=f)
-        t.daemon = True
-        t.start()
-        return out_queues
-
-    def push(in_q, out_q):
-        while True:
-            x = in_q.get()
-            out_q.put(x)
-
-    def merge(*in_qs, **kwargs):
-        """ Merge multiple queues together
-
-        >>> out_q = merge(q1, q2, q3)
-        """
-        out_q = Queue(**kwargs)
-        threads = [Thread(target=push, args=(q, out_q)) for q in in_qs]
-        for t in threads:
-            t.daemon = True
-            t.start()
-        return out_q
-
-With useful functions like these we can build out more sophisticated data
-processing pipelines that split off and join back together.  By creating queues
-with ``maxsize=`` we can control buffering and apply back pressure.
-
-.. raw:: html
-
-   <iframe width="560"
-           height="315"
-           src="https://www.youtube.com/embed/4IzuoV6XW_4?rel=0"
-           frameborder="0"
-           allowfullscreen>
-   </iframe>
+1.  Use normal for loops with Client.submit/gather and as_completed
+2.  Use :doc:`asynchronous async/await <asynchronous>` code and a few coroutines
+3.  Try out the `Streamz <https://streamz.readthedocs.io>`_ project,
+    which has Dask support

From bb80c5e997d020c5b472819fd0751c8234d2b8cc Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 13 May 2019 18:19:59 -0500
Subject: [PATCH 0287/1550] Use 'temporary-directory' from dask.config for
 Worker's directory (#2654)

---
 distributed/tests/test_worker.py | 9 +++++++++
 distributed/worker.py            | 8 +++++++-
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index d8ca4d31481..4541e183e46 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -1383,6 +1383,15 @@ def __init__(self, x, y):
     yield w.close()
 
 
+@gen_cluster(ncores=[])
+def test_local_dir(s):
+    with tmpfile() as fn:
+        with dask.config.set(temporary_directory=fn):
+            w = yield Worker(s.address)
+            assert w.local_dir.startswith(fn)
+            assert "dask-worker-space" in w.local_dir
+
+
 @pytest.mark.skipif(
     not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
 )
diff --git a/distributed/worker.py b/distributed/worker.py
index 8b8c139f356..3b5e1cb233a 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -282,7 +282,7 @@ def __init__(
         scheduler_file=None,
         ncores=None,
         loop=None,
-        local_dir="dask-worker-space",
+        local_dir=None,
         services=None,
         service_ports=None,
         service_kwargs=None,
@@ -448,6 +448,12 @@ def __init__(
         if silence_logs:
             silence_logging(level=silence_logs)
 
+        if local_dir is None:
+            local_dir = dask.config.get("temporary-directory") or os.getcwd()
+            if not os.path.exists(local_dir):
+                os.mkdir(local_dir)
+            local_dir = os.path.join(local_dir, "dask-worker-space")
+
         with warn_on_duration(
             "1s",
             "Creating scratch directories is taking a surprisingly long time. "

From fd31ecca8017bae845a73d468de0376c02363fab Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 14 May 2019 12:49:49 -0500
Subject: [PATCH 0288/1550] Cleanup localcluster (#2693)

* Remove address handling (handled in scheduler)
* Move ip= keyword to host=
---
 distributed/deploy/local.py            | 57 +++++++++++---------------
 distributed/deploy/tests/test_local.py | 14 ++++---
 2 files changed, 33 insertions(+), 38 deletions(-)

diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index fb8793d0840..832e8f3e051 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -15,7 +15,6 @@
 from ..compatibility import get_thread_identity
 from ..core import CommClosedError
 from ..utils import (
-    get_ip_interface,
     sync,
     ignoring,
     All,
@@ -51,8 +50,10 @@ class LocalCluster(Cluster):
     silence_logs: logging level
         Level of logs to print out to stdout.  ``logging.WARN`` by default.
         Use a falsey value like False or None for no change.
+    host: string
+        Host address on which the scheduler will listen, defaults to only localhost
     ip: string
-        IP address on which the scheduler will listen, defaults to only localhost
+        Deprecated.  See ``host`` above.
     dashboard_address: str
         Address on which to listen for the Bokeh diagnostics server like
         'localhost:8787' or '0.0.0.0:8787'.  Defaults to ':8787'.
@@ -88,13 +89,9 @@ class LocalCluster(Cluster):
 
     >>> c = Client(cluster)  # connect to local cluster  # doctest: +SKIP
 
-    Add a new worker to the cluster
+    Scale the cluster to three workers
 
-    >>> w = cluster.start_worker(ncores=2)  # doctest: +SKIP
-
-    Shut down the extra worker
-
-    >>> cluster.stop_worker(w)  # doctest: +SKIP
+    >>> cluster.scale(3)  # doctest: +SKIP
 
     Pass extra keyword arguments to Bokeh
 
@@ -109,6 +106,7 @@ def __init__(
         loop=None,
         start=None,
         ip=None,
+        host=None,
         scheduler_port=0,
         silence_logs=logging.WARN,
         dashboard_address=":8787",
@@ -125,6 +123,10 @@ def __init__(
         worker_class=None,
         **worker_kwargs
     ):
+        if ip is not None:
+            warnings.warn("The ip keyword has been moved to host")
+            host = ip
+
         if start is not None:
             msg = (
                 "The start= parameter is deprecated. "
@@ -145,8 +147,8 @@ def __init__(
         self.processes = processes
 
         if protocol is None:
-            if ip and "://" in ip:
-                protocol = ip.split("://")[0]
+            if host and "://" in host:
+                protocol = host.split("://")[0]
             elif security:
                 protocol = "tls://"
             elif not self.processes and not scheduler_port:
@@ -155,12 +157,12 @@ def __init__(
                 protocol = "tcp://"
         if not protocol.endswith("://"):
             protocol = protocol + "://"
-        self.protocol = protocol
+
+        if host is None and not protocol.startswith("inproc") and not interface:
+            host = "127.0.0.1"
 
         self.silence_logs = silence_logs
         self._asynchronous = asynchronous
-        self.security = security
-        self.interface = interface
         services = services or {}
         worker_services = worker_services or {}
         if silence_logs:
@@ -184,6 +186,8 @@ def __init__(
                 "ncores": threads_per_worker,
                 "services": worker_services,
                 "dashboard_address": worker_dashboard_address,
+                "interface": interface,
+                "protocol": protocol,
             }
         )
 
@@ -192,14 +196,16 @@ def __init__(
 
         self.scheduler = Scheduler(
             loop=self.loop,
+            host=host,
             services=services,
             service_kwargs=service_kwargs,
             security=security,
+            port=scheduler_port,
             interface=interface,
+            protocol=protocol,
             dashboard_address=dashboard_address,
             blocked_handlers=blocked_handlers,
         )
-        self.scheduler_port = scheduler_port
 
         self.workers = []
         self.worker_kwargs = worker_kwargs
@@ -210,7 +216,7 @@ def __init__(
             worker_class = Worker if not processes else Nanny
         self.worker_class = worker_class
 
-        self.start(ip=ip, n_workers=n_workers)
+        self.start(n_workers=n_workers)
 
         clusters_to_close.add(self)
 
@@ -251,32 +257,17 @@ def start(self, **kwargs):
             self.sync(self._start, **kwargs)
 
     @gen.coroutine
-    def _start(self, ip=None, n_workers=0):
+    def _start(self, n_workers=0):
         """
         Start all cluster services.
         """
         if self.status == "running":
             return
 
-        if self.protocol == "inproc://":
-            address = self.protocol
-        else:
-            if ip is None:
-                if self.interface:
-                    ip = get_ip_interface(self.interface)
-                else:
-                    ip = "127.0.0.1"
-
-            if "://" in ip:
-                address = ip
-            else:
-                address = self.protocol + ip
-            if self.scheduler_port:
-                address += ":" + str(self.scheduler_port)
-
-        self.scheduler.start(address)
+        self.scheduler.start()
 
         yield [self._start_worker(**self.worker_kwargs) for i in range(n_workers)]
+        yield self.scheduler
 
         self.status = "running"
 
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index 6e1e71e83b2..ed9e3bb2dbe 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -128,7 +128,7 @@ def test_move_unserializable_data():
             assert y.result() is lock
 
 
-def test_transports():
+def test_transports_inproc():
     """
     Test the transport chosen by LocalCluster depending on arguments.
     """
@@ -140,6 +140,8 @@ def test_transports():
         with Client(c.scheduler.address) as e:
             assert e.submit(inc, 4).result() == 5
 
+
+def test_transports_tcp():
     # Have nannies => need TCP
     with LocalCluster(
         1, processes=True, silence_logs=False, dashboard_address=None
@@ -149,6 +151,8 @@ def test_transports():
         with Client(c.scheduler.address) as e:
             assert e.submit(inc, 4).result() == 5
 
+
+def test_transports_tcp_port():
     # Scheduler port specified => need TCP
     with LocalCluster(
         1,
@@ -417,7 +421,7 @@ def test_remote_access(loop):
         scheduler_port=0,
         silence_logs=False,
         dashboard_address=None,
-        ip="",
+        host="",
         loop=loop,
     ) as c:
         sync(loop, assert_can_connect_from_everywhere_4_6, c.scheduler.port)
@@ -620,7 +624,7 @@ def test_local_tls(loop):
         silence_logs=False,
         security=security,
         dashboard_address=False,
-        ip="tls://0.0.0.0",
+        host="tls://0.0.0.0",
         loop=loop,
     ) as c:
         sync(
@@ -690,7 +694,7 @@ def test_local_tls_restart(loop):
         silence_logs=False,
         security=security,
         dashboard_address=False,
-        ip="tls://0.0.0.0",
+        host="tls://0.0.0.0",
         loop=loop,
     ) as c:
         with Client(c.scheduler.address, loop=loop, security=security) as client:
@@ -750,7 +754,7 @@ def test_protocol_tcp(loop):
 )
 def test_protocol_ip(loop):
     with LocalCluster(
-        ip="tcp://127.0.0.2", loop=loop, n_workers=0, processes=False
+        host="tcp://127.0.0.2", loop=loop, n_workers=0, processes=False
     ) as cluster:
         assert cluster.scheduler.address.startswith("tcp://127.0.0.2")
 

From 8e449d392e91eff0a3454ee98ef362de8f78cc4f Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 15 May 2019 09:35:43 -0500
Subject: [PATCH 0289/1550] Support computation on delayed(None) (#2697)

Previously this conflicted with our sentinel value

Fixes #2696
---
 distributed/tests/test_collections.py | 10 ++++++++++
 distributed/worker.py                 |  6 +++---
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/distributed/tests/test_collections.py b/distributed/tests/test_collections.py
index 985b6f78fe9..dea4296769d 100644
--- a/distributed/tests/test_collections.py
+++ b/distributed/tests/test_collections.py
@@ -6,6 +6,7 @@
 pytest.importorskip("numpy")
 pytest.importorskip("pandas")
 
+import dask
 import dask.dataframe as dd
 import dask.bag as db
 from distributed.client import wait
@@ -185,3 +186,12 @@ def test_sparse_arrays(c, s, a, b):
     future = c.compute(s.sum(axis=0)[:10])
 
     yield future
+
+
+@gen_cluster(client=True, ncores=[("127.0.0.1", 1)])
+def test_delayed_none(c, s, w):
+    x = dask.delayed(None)
+    y = dask.delayed(123)
+    [xx, yy] = c.compute([x, y])
+    assert (yield xx) is None
+    assert (yield yy) == 123
diff --git a/distributed/worker.py b/distributed/worker.py
index 3b5e1cb233a..f4189393091 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -1216,7 +1216,7 @@ def add_task(
         function=None,
         args=None,
         kwargs=None,
-        task=None,
+        task=no_value,
         who_has=None,
         nbytes=None,
         priority=None,
@@ -3007,7 +3007,7 @@ def get_data_from_worker(
 job_counter = [0]
 
 
-def _deserialize(function=None, args=None, kwargs=None, task=None):
+def _deserialize(function=None, args=None, kwargs=None, task=no_value):
     """ Deserialize task inputs and regularize to func, args, kwargs """
     if function is not None:
         function = pickle.loads(function)
@@ -3016,7 +3016,7 @@ def _deserialize(function=None, args=None, kwargs=None, task=None):
     if kwargs:
         kwargs = pickle.loads(kwargs)
 
-    if task is not None:
+    if task is not no_value:
         assert not function and not args and not kwargs
         function = execute_task
         args = (task,)

From 73362eaf4bb1f941284658ae87f51dbaae753a4c Mon Sep 17 00:00:00 2001
From: Daniel Farrell <danpf@uw.edu>
Date: Wed, 15 May 2019 10:04:30 -0700
Subject: [PATCH 0290/1550] Add method to wait for n workers before continuing
 (#2688)

---
 distributed/client.py            | 11 +++++++++++
 distributed/tests/test_client.py | 13 +++++++++++++
 2 files changed, 24 insertions(+)

diff --git a/distributed/client.py b/distributed/client.py
index 7b6e14aa4ef..8dd85d7795e 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -1051,6 +1051,17 @@ def _update_scheduler_info(self):
         except EnvironmentError:
             logger.debug("Not able to query scheduler for identity")
 
+    @gen.coroutine
+    def _wait_for_workers(self, n_workers=0):
+        info = yield self.scheduler.identity()
+        while n_workers and len(info["workers"]) < n_workers:
+            yield gen.sleep(0.1)
+            info = yield self.scheduler.identity()
+
+    def wait_for_workers(self, n_workers=0):
+        """Blocking call to wait for n workers before continuing"""
+        return self.sync(self._wait_for_workers, n_workers)
+
     def _heartbeat(self):
         if self.scheduler_comm:
             self.scheduler_comm.send({"op": "heartbeat-client"})
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 449d207a91a..dfe677ddd1e 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -5510,5 +5510,18 @@ def test_instances(c, s, a, b):
     assert set(Worker._instances) == {a, b}
 
 
+@gen_cluster(client=True)
+def test_wait_for_workers(c, s, a, b):
+    future = c.wait_for_workers(n_workers=3)
+    yield gen.sleep(0.22)  # 2 chances
+    assert not future.done()
+
+    w = yield Worker(s.address)
+    start = time()
+    yield future
+    assert time() < start + 1
+    yield w.close()
+
+
 if sys.version_info >= (3, 5):
     from distributed.tests.py3_test_client import *  # noqa F401

From 0ce8f2bcb84d306e9a095d75497857dce30145b5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mathieu=20Dugr=C3=A9?= <math.dugre@gmail.com>
Date: Wed, 15 May 2019 15:56:44 -0400
Subject: [PATCH 0291/1550] Modify styling of histograms for many-worker
 dashboard plots (#2695)

Fixes #2691
---
 distributed/bokeh/scheduler.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/distributed/bokeh/scheduler.py b/distributed/bokeh/scheduler.py
index 2dd60f0690f..c078b612dd9 100644
--- a/distributed/bokeh/scheduler.py
+++ b/distributed/bokeh/scheduler.py
@@ -219,9 +219,10 @@ def __init__(self, scheduler, **kwargs):
             )
 
             self.root = figure(
-                title="Tasks Processing",
+                title="Tasks Processing (Histogram)",
                 id="bk-nprocessing-histogram-plot",
                 name="processing_hist",
+                y_axis_label="frequency",
                 **kwargs
             )
 
@@ -237,7 +238,8 @@ def __init__(self, scheduler, **kwargs):
                 right="right",
                 bottom=0,
                 top="top",
-                color="blue",
+                color="deepskyblue",
+                fill_alpha=0.5,
             )
 
     @without_property_validation
@@ -259,11 +261,13 @@ def __init__(self, scheduler, **kwargs):
             )
 
             self.root = figure(
-                title="Bytes Stored",
+                title="Bytes Stored (Histogram)",
                 name="nbytes_hist",
                 id="bk-nbytes-histogram-plot",
+                y_axis_label="frequency",
                 **kwargs
             )
+
             self.root.xaxis[0].formatter = NumeralTickFormatter(format="0.0 b")
             self.root.xaxis.major_label_orientation = -math.pi / 12
 
@@ -279,7 +283,8 @@ def __init__(self, scheduler, **kwargs):
                 right="right",
                 bottom=0,
                 top="top",
-                color="blue",
+                color="deepskyblue",
+                fill_alpha=0.5,
             )
 
     @without_property_validation
@@ -289,7 +294,7 @@ def update(self):
         d = {"left": x[:-1], "right": x[1:], "top": counts}
         self.source.data.update(d)
 
-        self.root.title.text = "Bytes stored: " + format_bytes(nbytes.sum())
+        self.root.title.text = "Bytes stored (Histogram): " + format_bytes(nbytes.sum())
 
 
 class CurrentLoad(DashboardComponent):

From d4f478672137481ad43abc6a44383a595e8485b5 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 16 May 2019 08:23:42 -0500
Subject: [PATCH 0292/1550] Handle heartbeat when worker has just left (#2702)

---
 distributed/scheduler.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 446b479c769..1d6a41a9acc 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1353,6 +1353,9 @@ def heartbeat_worker(
         address = self.coerce_address(address, resolve_address)
         address = normalize_address(address)
         host = get_address_host(address)
+        if address not in self.workers:
+            logger.info("Received heartbeat from removed worker: %s", address)
+            return
 
         local_now = time()
         now = now or time()

From 4feb90d7ddd4860648c161c1497bb55da0fc1b2a Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 16 May 2019 08:23:54 -0500
Subject: [PATCH 0293/1550] Except errors in Nanny's memory monitor if process
 no longer exists (#2701)

---
 distributed/nanny.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/distributed/nanny.py b/distributed/nanny.py
index 7be630cb4e9..842ec765d7f 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -326,9 +326,9 @@ def memory_monitor(self):
             return
         try:
             proc = psutil.Process(process.pid)
-        except psutil.NoSuchProcess:
+            memory = proc.memory_info().rss
+        except (ProcessLookupError, psutil.NoSuchProcess, psutil.AccessDenied):
             return
-        memory = proc.memory_info().rss
         frac = memory / self.memory_limit
         if self.memory_terminate_fraction and frac > self.memory_terminate_fraction:
             logger.warning(

From d9a0897cd3abc6f0c921f50b5e12dbb10fc5aac5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mathieu=20Dugr=C3=A9?= <math.dugre@gmail.com>
Date: Thu, 16 May 2019 16:25:15 -0400
Subject: [PATCH 0294/1550] Disable pan tool for the Progress, Byte Stored and
 Tasks Processing plot (#2703)

---
 distributed/bokeh/scheduler.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/distributed/bokeh/scheduler.py b/distributed/bokeh/scheduler.py
index c078b612dd9..471f93dd4b5 100644
--- a/distributed/bokeh/scheduler.py
+++ b/distributed/bokeh/scheduler.py
@@ -223,6 +223,7 @@ def __init__(self, scheduler, **kwargs):
                 id="bk-nprocessing-histogram-plot",
                 name="processing_hist",
                 y_axis_label="frequency",
+                tools="",
                 **kwargs
             )
 
@@ -265,6 +266,7 @@ def __init__(self, scheduler, **kwargs):
                 name="nbytes_hist",
                 id="bk-nbytes-histogram-plot",
                 y_axis_label="frequency",
+                tools="",
                 **kwargs
             )
 
@@ -943,6 +945,7 @@ def __init__(self, scheduler, **kwargs):
             x_range=x_range,
             y_range=y_range,
             toolbar_location=None,
+            tools="",
             **kwargs
         )
         self.root.line(  # just to define early ranges

From a42721656418ff4848c44e230b8f033d2db58a63 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 16 May 2019 16:36:07 -0500
Subject: [PATCH 0295/1550] Cleanly stop periodic callbacks in Client (#2705)

Previously we did this only in the asynchronous code, which left a gap
during which a heartbeat could sneak out.  Now we call it explicitly at
the beginning of the synchronous close command.
---
 distributed/client.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 8dd85d7795e..4aab8102d0c 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -1204,10 +1204,11 @@ def _close(self, fast=False):
         """ Send close signal and wait until scheduler completes """
         self.status = "closing"
 
+        for pc in self._periodic_callbacks.values():
+            pc.stop()
+
         with log_errors():
             _del_global_client(self)
-            for pc in self._periodic_callbacks.values():
-                pc.stop()
             self._scheduler_identity = {}
             with ignoring(AttributeError):
                 # clear the dask.config set keys
@@ -1289,6 +1290,9 @@ def close(self, timeout=no_default):
             return
         self.status = "closing"
 
+        for pc in self._periodic_callbacks.values():
+            pc.stop()
+
         if self.asynchronous:
             future = self._close()
             if timeout:

From 7ebe65980e7fef90fd25cc0d35e2fcfc0c266881 Mon Sep 17 00:00:00 2001
From: Benjamin Zaitlen <quasiben@users.noreply.github.com>
Date: Thu, 16 May 2019 17:36:36 -0400
Subject: [PATCH 0296/1550] Change the main workers bokeh page to /status
 (#2689)

This matches the behavior on the scheduler
---
 distributed/bokeh/scheduler.py            |  4 ++--
 distributed/bokeh/worker.py               | 16 ++++++++--------
 distributed/bokeh/worker_html.py          | 11 ++++++++++-
 distributed/cli/tests/test_dask_worker.py |  4 +++-
 docs/source/diagnosing-performance.rst    |  2 +-
 5 files changed, 24 insertions(+), 13 deletions(-)

diff --git a/distributed/bokeh/scheduler.py b/distributed/bokeh/scheduler.py
index 471f93dd4b5..cce94b356ae 100644
--- a/distributed/bokeh/scheduler.py
+++ b/distributed/bokeh/scheduler.py
@@ -152,7 +152,7 @@ def __init__(self, scheduler, **kwargs):
             # fig.xaxis[0].formatter = NumeralTickFormatter(format='0.0s')
             fig.x_range.start = 0
 
-            tap = TapTool(callback=OpenURL(url="http://@bokeh_address/main"))
+            tap = TapTool(callback=OpenURL(url="http://@bokeh_address/"))
 
             hover = HoverTool()
             hover.tooltips = "@worker : @occupancy s."
@@ -368,7 +368,7 @@ def __init__(self, scheduler, width=600, **kwargs):
                 fig.yaxis.visible = False
                 fig.ygrid.visible = False
 
-                tap = TapTool(callback=OpenURL(url="http://@bokeh_address/main"))
+                tap = TapTool(callback=OpenURL(url="http://@bokeh_address/"))
                 fig.add_tools(tap)
 
                 fig.toolbar.logo = None
diff --git a/distributed/bokeh/worker.py b/distributed/bokeh/worker.py
index c7ced4d90fc..ed7b68b76b4 100644
--- a/distributed/bokeh/worker.py
+++ b/distributed/bokeh/worker.py
@@ -51,7 +51,7 @@
 
 BOKEH_THEME = Theme(os.path.join(os.path.dirname(__file__), "theme.yaml"))
 
-template_variables = {"pages": ["main", "system", "profile", "crossfilter"]}
+template_variables = {"pages": ["status", "system", "profile", "crossfilter"]}
 
 
 class StateTable(DashboardComponent):
@@ -410,9 +410,9 @@ def process_msg(self, msg):
             def func(k):
                 return msg["keys"].get(k, 0)
 
-            main_key = max(msg["keys"], key=func)
-            typ = self.worker.types.get(main_key, object).__name__
-            keyname = key_split(main_key)
+            status_key = max(msg["keys"], key=func)
+            typ = self.worker.types.get(status_key, object).__name__
+            keyname = key_split(status_key)
             d = {
                 "nbytes": msg["total"],
                 "duration": msg["duration"],
@@ -659,7 +659,7 @@ def update(self):
 from bokeh.application import Application
 
 
-def main_doc(worker, extra, doc):
+def status_doc(worker, extra, doc):
     with log_errors():
         statetable = StateTable(worker)
         executing_ts = ExecutingTimeSeries(worker, sizing_mode="scale_width")
@@ -685,7 +685,7 @@ def main_doc(worker, extra, doc):
             )
         )
         doc.template = env.get_template("simple.html")
-        doc.template_variables["active_page"] = "main"
+        doc.template_variables["active_page"] = "status"
         doc.template_variables.update(extra)
         doc.theme = BOKEH_THEME
 
@@ -773,7 +773,7 @@ def __init__(self, worker, io_loop=None, prefix="", **kwargs):
 
         extra.update(template_variables)
 
-        main = Application(FunctionHandler(partial(main_doc, worker, extra)))
+        status = Application(FunctionHandler(partial(status_doc, worker, extra)))
         crossfilter = Application(
             FunctionHandler(partial(crossfilter_doc, worker, extra))
         )
@@ -787,7 +787,7 @@ def __init__(self, worker, io_loop=None, prefix="", **kwargs):
         )
 
         self.apps = {
-            "/main": main,
+            "/status": status,
             "/counters": counters,
             "/crossfilter": crossfilter,
             "/system": systemmonitor,
diff --git a/distributed/bokeh/worker_html.py b/distributed/bokeh/worker_html.py
index 3ddf9490c4d..c818c8fb1e6 100644
--- a/distributed/bokeh/worker_html.py
+++ b/distributed/bokeh/worker_html.py
@@ -67,7 +67,16 @@ def get(self):
         self.set_header("Content-Type", "text/plain")
 
 
-routes = [(r"metrics", PrometheusHandler), (r"health", HealthHandler)]
+class OldRoute(RequestHandler):
+    def get(self):
+        self.redirect("/status")
+
+
+routes = [
+    (r"metrics", PrometheusHandler),
+    (r"health", HealthHandler),
+    (r"main", OldRoute),
+]
 
 
 def get_handlers(server):
diff --git a/distributed/cli/tests/test_dask_worker.py b/distributed/cli/tests/test_dask_worker.py
index c26c99f2350..5ed668e758a 100644
--- a/distributed/cli/tests/test_dask_worker.py
+++ b/distributed/cli/tests/test_dask_worker.py
@@ -260,8 +260,10 @@ def test_bokeh_non_standard_ports(loop):
             start = time()
             while True:
                 try:
-                    response = requests.get("http://127.0.0.1:4833/main")
+                    response = requests.get("http://127.0.0.1:4833/status")
                     assert response.ok
+                    redirect_resp = requests.get("http://127.0.0.1:4833/main")
+                    redirect_resp.ok
                     break
                 except Exception:
                     sleep(0.5)
diff --git a/docs/source/diagnosing-performance.rst b/docs/source/diagnosing-performance.rst
index 28d7d9aba44..773a5d2316b 100644
--- a/docs/source/diagnosing-performance.rst
+++ b/docs/source/diagnosing-performance.rst
@@ -105,7 +105,7 @@ attributes including
 4.  Keys moved
 5.  Peer
 
-These are made available to users through the ``/main`` page of the Worker's
+These are made available to users through the ``/status`` page of the Worker's
 diagnostic dashboard.  You can capture their state explicitly by running a
 command on the workers:
 

From fc48c435f8f366c335c92dd6fc58af38065edcec Mon Sep 17 00:00:00 2001
From: Sam Grayson <sam@samgrayson.me>
Date: Fri, 17 May 2019 17:53:12 -0500
Subject: [PATCH 0297/1550] Support uploading files with multiple modules
 (#2587)

---
 distributed/tests/test_client.py | 96 ++++++++++++++++++++++++--------
 distributed/utils.py             | 15 ++---
 distributed/utils_test.py        | 15 +++++
 3 files changed, 94 insertions(+), 32 deletions(-)

diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index dfe677ddd1e..4cd196fa2f4 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -9,6 +9,7 @@
 import os
 import pickle
 import random
+import subprocess
 import sys
 import threading
 from threading import Semaphore
@@ -79,6 +80,7 @@
     wait_for,
     async_wait_for,
     pristine_loop,
+    save_sys_modules,
 )
 from distributed.utils_test import (  # noqa: F401
     client as c,
@@ -1484,7 +1486,7 @@ def g():
 
         return myfile.f()
 
-    try:
+    with save_sys_modules():
         for value in [123, 456]:
             with tmp_text("myfile.py", "def f():\n    return {}".format(value)) as fn:
                 yield c.upload_file(fn)
@@ -1492,10 +1494,6 @@ def g():
             x = c.submit(g, pure=False)
             result = yield x
             assert result == value
-    finally:
-        # Ensure that this test won't impact the others
-        if "myfile" in sys.modules:
-            del sys.modules["myfile"]
 
 
 @gen_cluster(client=True)
@@ -1511,28 +1509,80 @@ def g():
 
         return myfile.f()
 
-    try:
+    with save_sys_modules():
+        try:
+            for value in [123, 456]:
+                with tmp_text(
+                    "myfile.py", "def f():\n    return {}".format(value)
+                ) as fn_my_file:
+                    with zipfile.ZipFile("myfile.zip", "w") as z:
+                        z.write(fn_my_file, arcname=os.path.basename(fn_my_file))
+                    yield c.upload_file("myfile.zip")
+
+                    x = c.submit(g, pure=False)
+                    result = yield x
+                    assert result == value
+        finally:
+            if os.path.exists("myfile.zip"):
+                os.remove("myfile.zip")
+
+
+@gen_cluster(client=True)
+def test_upload_file_egg(c, s, a, b):
+    def g():
+        import package_1, package_2
+
+        return package_1.a, package_2.b
+
+    # c.upload_file tells each worker to
+    # - put this file in their local_dir
+    # - modify their sys.path to include it
+    # we don't care about the local_dir
+    # but we do care about restoring the path
+
+    with save_sys_modules():
         for value in [123, 456]:
-            with tmp_text(
-                "myfile.py", "def f():\n    return {}".format(value)
-            ) as fn_my_file:
-                with zipfile.ZipFile("myfile.zip", "w") as z:
-                    z.write(fn_my_file, arcname=os.path.basename(fn_my_file))
-                yield c.upload_file("myfile.zip")
+            with tmpfile() as dirname:
+                os.mkdir(dirname)
+
+                with open(os.path.join(dirname, "setup.py"), "w") as f:
+                    f.write("from setuptools import setup, find_packages\n")
+                    f.write(
+                        'setup(name="my_package", packages=find_packages(), version="{}")\n'.format(
+                            value
+                        )
+                    )
+
+                # test a package with an underscore in the name
+                package_1 = os.path.join(dirname, "package_1")
+                os.mkdir(package_1)
+                with open(os.path.join(package_1, "__init__.py"), "w") as f:
+                    f.write("a = {}\n".format(value))
+
+                # test multiple top-level packages
+                package_2 = os.path.join(dirname, "package_2")
+                os.mkdir(package_2)
+                with open(os.path.join(package_2, "__init__.py"), "w") as f:
+                    f.write("b = {}\n".format(value))
+
+                # compile these into an egg
+                subprocess.check_call(
+                    [sys.executable, "setup.py", "bdist_egg"], cwd=dirname
+                )
+
+                egg_root = os.path.join(dirname, "dist")
+                # first file ending with '.egg'
+                egg_name = [
+                    fname for fname in os.listdir(egg_root) if fname.endswith(".egg")
+                ][0]
+                egg_path = os.path.join(egg_root, egg_name)
+
+                yield c.upload_file(egg_path)
+                os.remove(egg_path)
 
                 x = c.submit(g, pure=False)
                 result = yield x
-                assert result == value
-    finally:
-        # Ensure that this test won't impact the others
-        if os.path.exists("myfile.zip"):
-            os.remove("myfile.zip")
-        if "myfile" in sys.modules:
-            del sys.modules["myfile"]
-        for path in sys.path:
-            if os.path.basename(path) == "myfile.zip":
-                sys.path.remove(path)
-                break
+                assert result == (value, value)
 
 
 @gen_cluster(client=True)
diff --git a/distributed/utils.py b/distributed/utils.py
index 6debdedd24e..466a96fdfd5 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -23,7 +23,7 @@
 import threading
 import warnings
 import weakref
-
+import pkgutil
 import six
 import tblib.pickling_support
 
@@ -1066,14 +1066,11 @@ def import_file(path):
     if ext in (".egg", ".zip", ".pyz"):
         if path not in sys.path:
             sys.path.insert(0, path)
-        if ext == ".egg":
-            import pkg_resources
-
-            pkgs = pkg_resources.find_distributions(path)
-            for pkg in pkgs:
-                names_to_import.append(pkg.project_name)
-        elif ext in (".zip", ".pyz"):
-            names_to_import.append(name)
+        if sys.version_info >= (3, 6):
+            names = (mod_info.name for mod_info in pkgutil.iter_modules([path]))
+        else:
+            names = (mod_info[1] for mod_info in pkgutil.iter_modules([path]))
+        names_to_import.extend(names)
 
     loaded = []
     if not names_to_import:
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index 7b90745ac6d..c44f4177472 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -1417,6 +1417,21 @@ def gen_tls_cluster(**kwargs):
     )
 
 
+@contextmanager
+def save_sys_modules():
+    old_modules = sys.modules
+    old_path = sys.path
+    try:
+        yield
+    finally:
+        for i, elem in enumerate(sys.path):
+            if elem not in old_path:
+                del sys.path[i]
+        for elem in sys.modules.keys():
+            if elem not in old_modules:
+                del sys.modules[elem]
+
+
 @contextmanager
 def check_thread_leak():
     active_threads_start = set(threading._active)

From 138842c9769c273b6edc6c086c80009102596198 Mon Sep 17 00:00:00 2001
From: Magnus Nord <magnunor@gmail.com>
Date: Mon, 20 May 2019 17:54:37 +0200
Subject: [PATCH 0298/1550] Fix two typos in Pub class docstring (#2714)

---
 distributed/pubsub.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/distributed/pubsub.py b/distributed/pubsub.py
index 5e086492923..f9cf1f6f7c3 100644
--- a/distributed/pubsub.py
+++ b/distributed/pubsub.py
@@ -234,8 +234,8 @@ class Pub(object):
     disappear without notice.
 
     When using a Pub or Sub from a Client all communications will be routed
-    through the scheduler.  This can cause some performance degredation.  Pubs
-    an Subs only operate at top-speed when they are both on workers.
+    through the scheduler.  This can cause some performance degradation.  Pubs
+    and Subs only operate at top-speed when they are both on workers.
 
     Parameters
     ----------

From 1a96f70d9c2d23d85a8550f5bfdf60c26bb4ed4f Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 20 May 2019 12:12:52 -0500
Subject: [PATCH 0299/1550] Remove special casing of Scikit-Learn BaseEstimator
 serialization (#2713)

Fixes https://github.com/dask/dask/issues/4769
---
 distributed/protocol/__init__.py           | 8 --------
 distributed/protocol/tests/test_sklearn.py | 4 ++++
 2 files changed, 4 insertions(+), 8 deletions(-)

diff --git a/distributed/protocol/__init__.py b/distributed/protocol/__init__.py
index cf1a3df8994..04691ce605d 100644
--- a/distributed/protocol/__init__.py
+++ b/distributed/protocol/__init__.py
@@ -60,14 +60,6 @@ def _register_arrow():
     from . import arrow
 
 
-@dask_serialize.register_lazy("sklearn")
-@dask_deserialize.register_lazy("sklearn")
-def _register_sklearn():
-    import sklearn.base
-
-    register_generic(sklearn.base.BaseEstimator)
-
-
 @dask_serialize.register_lazy("torch")
 @dask_deserialize.register_lazy("torch")
 @dask_serialize.register_lazy("torchvision")
diff --git a/distributed/protocol/tests/test_sklearn.py b/distributed/protocol/tests/test_sklearn.py
index 051a0440f3a..2a3835168ee 100644
--- a/distributed/protocol/tests/test_sklearn.py
+++ b/distributed/protocol/tests/test_sklearn.py
@@ -7,6 +7,10 @@
 from distributed.protocol import serialize, deserialize
 
 
+@pytest.mark.xfail(
+    reason="We no longer special-case the BaseEstimator "
+    "super class. It's hard to guarantee support for all subclasseses"
+)
 def test_basic():
     est = sklearn.linear_model.LinearRegression()
     est.fit([[0, 0], [1, 1], [2, 2]], [0, 1, 2])

From f47ed2e610590c644da052af64a34bffa1552a92 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 21 May 2019 17:45:13 -0500
Subject: [PATCH 0300/1550] Refer to LocalCluster in Client docstring (#2719)

---
 distributed/client.py | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 4aab8102d0c..6b5bc2811bb 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -504,12 +504,19 @@ class AllExit(Exception):
 
 
 class Client(Node):
-    """ Connect to and drive computation on a distributed Dask cluster
-
-    The Client connects users to a dask.distributed compute cluster.  It
-    provides an asynchronous user interface around functions and futures.  This
-    class resembles executors in ``concurrent.futures`` but also allows
-    ``Future`` objects within ``submit/map`` calls.
+    """ Connect to and submit computation to a Dask cluster
+
+    The Client connects users to a Dask cluster.  It provides an asynchronous
+    user interface around functions and futures.  This class resembles
+    executors in ``concurrent.futures`` but also allows ``Future`` objects
+    within ``submit/map`` calls.  When a Client is instantiated it takes over
+    all ``dask.compute`` and ``dask.persist`` calls by default.
+
+    It is also common to create a Client without specifying the scheduler
+    address , like ``Client()``.  In this case the Client creates a
+    ``LocalCluster`` in the background and connects to that.  Any extra
+    keywords are passed from Client to LocalCluster in this case.  See the
+    LocalCluster documentation for more information.
 
     Parameters
     ----------

From 62f604e7e567a1cc7806226adc5d7f288dc2fbad Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 22 May 2019 15:05:39 -0500
Subject: [PATCH 0301/1550] Add docstring to Scheduler.check_idle_saturated
 (#2721)

---
 distributed/scheduler.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 1d6a41a9acc..801c0c849d7 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -4357,6 +4357,19 @@ def reschedule(self, key=None, worker=None):
     ##############################
 
     def check_idle_saturated(self, ws, occ=None):
+        """ Update the status of the idle and saturated state
+
+        The scheduler keeps track of workers that are ..
+
+        -  Saturated: have enough work to stay busy
+        -  Idle: do not have enough work to stay busy
+
+        They are considered saturated if they both have enough tasks to occupy
+        all of their cores, and if the expected runtime of those tasks is large
+        enough.
+
+        This is useful for load balancing and adaptivity.
+        """
         if self.total_ncores == 0 or ws.status == "closed":
             return
         if occ is None:

From 28ce1eda0f6ab4940ce4daa1f309b29a496e6834 Mon Sep 17 00:00:00 2001
From: Benjamin Zaitlen <quasiben@users.noreply.github.com>
Date: Wed, 22 May 2019 16:06:50 -0400
Subject: [PATCH 0302/1550] Proxy worker dashboards from scheduler dashboard
 (#2715)

---
 dev-requirements.txt                          |   1 +
 distributed/bokeh/proxy.py                    | 130 ++++++++++++++++++
 distributed/bokeh/scheduler.py                |  35 +++--
 distributed/bokeh/scheduler_html.py           |   5 +-
 distributed/bokeh/templates/task.html         |   2 +-
 distributed/bokeh/templates/worker-table.html |   4 +-
 distributed/bokeh/templates/workers.html      |   1 -
 .../bokeh/tests/test_scheduler_bokeh.py       |  44 +++++-
 .../bokeh/tests/test_scheduler_bokeh_html.py  |   7 +-
 distributed/cli/tests/test_dask_worker.py     |  14 +-
 distributed/scheduler.py                      |   1 +
 11 files changed, 222 insertions(+), 22 deletions(-)
 create mode 100644 distributed/bokeh/proxy.py

diff --git a/dev-requirements.txt b/dev-requirements.txt
index 7d684343ca7..8cc8f7d256d 100644
--- a/dev-requirements.txt
+++ b/dev-requirements.txt
@@ -10,3 +10,4 @@ jupyter_client >= 4.4.0
 ipykernel >= 4.5.2
 pytest >= 3.0.5
 prometheus_client >= 0.6.0
+jupyter-server-proxy >= 1.1.0
diff --git a/distributed/bokeh/proxy.py b/distributed/bokeh/proxy.py
new file mode 100644
index 00000000000..9353e383112
--- /dev/null
+++ b/distributed/bokeh/proxy.py
@@ -0,0 +1,130 @@
+import logging
+
+from tornado import web
+
+logger = logging.getLogger(__name__)
+
+try:
+    from jupyter_server_proxy.handlers import ProxyHandler
+
+    class GlobalProxyHandler(ProxyHandler):
+        """
+        A tornado request handler that proxies HTTP and websockets
+        from a port to any valid endpoint'.
+        """
+
+        def initialize(self, server=None, extra=None):
+            self.scheduler = server
+            self.extra = extra or {}
+
+        async def http_get(self, port, host, proxied_path):
+            # route here first
+            # incoming URI /proxy/{port}/{host}/{proxied_path}
+
+            self.host = host
+
+            # rewrite uri for jupyter-server-proxy handling
+            uri = "/proxy/%s/%s" % (str(port), proxied_path)
+            self.request.uri = uri
+
+            # slash is removed during regex in handler
+            proxied_path = "/%s" % proxied_path
+
+            worker = "%s:%s" % (self.host, str(port))
+            if not check_worker_dashboard_exits(self.scheduler, worker):
+                msg = "Worker <%s> does not exist" % worker
+                self.set_status(400)
+                self.finish(msg)
+                return
+            return await self.proxy(port, proxied_path)
+
+        async def open(self, port, host, proxied_path):
+            # finally, proxy to other address/port
+            return await self.proxy_open(host, port, proxied_path)
+
+        def post(self, port, proxied_path):
+            return self.proxy(port, proxied_path)
+
+        def put(self, port, proxied_path):
+            return self.proxy(port, proxied_path)
+
+        def delete(self, port, proxied_path):
+            return self.proxy(port, proxied_path)
+
+        def head(self, port, proxied_path):
+            return self.proxy(port, proxied_path)
+
+        def patch(self, port, proxied_path):
+            return self.proxy(port, proxied_path)
+
+        def options(self, port, proxied_path):
+            return self.proxy(port, proxied_path)
+
+        def proxy(self, port, proxied_path):
+            # router here second
+            # returns ProxyHandler coroutine
+            return super().proxy(self.host, port, proxied_path)
+
+
+except ImportError:
+    logger.info(
+        "To route to workers diagnostics web server "
+        "please install jupyter-server-proxy: "
+        "pip install jupyter-server-proxy"
+    )
+
+    class GlobalProxyHandler(web.RequestHandler):
+        """Minimal Proxy handler when jupyter-server-proxy is not installed
+        """
+
+        def initialize(self, server=None, extra=None):
+            self.server = server
+            self.extra = extra or {}
+
+        def get(self, port, host, proxied_path):
+            worker_url = "%s:%s/%s" % (host, str(port), proxied_path)
+            msg = """
+                <p> Try navigating to <a href=http://%s>%s</a> for your worker dashboard </p>
+
+                <p>
+                Dask tried to proxy you to that page through your
+                Scheduler's dashboard connection, but you don't have
+                jupyter-server-proxy installed.  You may want to install it
+                with either conda or pip, and then restart your scheduler.
+                </p>
+
+                <p><pre> conda install jupyter-server-proxy -c conda-forge </pre></p>
+                <p><pre> pip install jupyter-server-proxy</pre></p>
+
+                <p>
+                The link above should work though if your workers are on a
+                sufficiently open network.  This is common on single machines,
+                but less common in production clusters.  Your IT administrators
+                will know more
+                </p>
+            """ % (
+                worker_url,
+                worker_url,
+            )
+            self.write(msg)
+
+
+def check_worker_dashboard_exits(scheduler, worker):
+    """Check addr:port exists as a worker in scheduler list
+
+    Parameters
+    ----------
+    worker : str
+        addr:port
+
+    Returns
+    -------
+    bool
+    """
+    addr, port = worker.split(":")
+    workers = list(scheduler.workers.values())
+    for w in workers:
+        bokeh_port = w.services.get("bokeh", "")
+        if addr == w.host and port == str(bokeh_port):
+            return True
+    return False
diff --git a/distributed/bokeh/scheduler.py b/distributed/bokeh/scheduler.py
index cce94b356ae..e0f5bfffab6 100644
--- a/distributed/bokeh/scheduler.py
+++ b/distributed/bokeh/scheduler.py
@@ -130,7 +130,8 @@ def __init__(self, scheduler, **kwargs):
                     "y": [1, 2],
                     "ms": [1, 2],
                     "color": ["red", "blue"],
-                    "bokeh_address": ["", ""],
+                    "dashboard_port": ["", ""],
+                    "dashboard_host": ["", ""],
                 }
             )
 
@@ -152,7 +153,9 @@ def __init__(self, scheduler, **kwargs):
             # fig.xaxis[0].formatter = NumeralTickFormatter(format='0.0s')
             fig.x_range.start = 0
 
-            tap = TapTool(callback=OpenURL(url="http://@bokeh_address/"))
+            tap = TapTool(
+                callback=OpenURL(url="./proxy/@dashboard_port/@dashboard_host/status")
+            )
 
             hover = HoverTool()
             hover.tooltips = "@worker : @occupancy s."
@@ -166,10 +169,8 @@ def update(self):
         with log_errors():
             workers = list(self.scheduler.workers.values())
 
-            bokeh_addresses = []
-            for ws in workers:
-                addr = self.scheduler.get_worker_service_addr(ws.address, "bokeh")
-                bokeh_addresses.append("%s:%d" % addr if addr is not None else "")
+            dashboard_host = [ws.host for ws in workers]
+            dashboard_port = [ws.services.get("bokeh", "") for ws in workers]
 
             y = list(range(len(workers)))
             occupancy = [ws.occupancy for ws in workers]
@@ -199,7 +200,8 @@ def update(self):
                     "worker": [ws.address for ws in workers],
                     "ms": ms,
                     "color": color,
-                    "bokeh_address": bokeh_addresses,
+                    "dashboard_host": dashboard_host,
+                    "dashboard_port": dashboard_port,
                     "x": x,
                     "y": y,
                 }
@@ -317,7 +319,8 @@ def __init__(self, scheduler, width=600, **kwargs):
                     "worker": ["a", "b"],
                     "y": [1, 2],
                     "nbytes-color": ["blue", "blue"],
-                    "bokeh_address": ["", ""],
+                    "dashboard_port": ["", ""],
+                    "dashboard_host": ["", ""],
                 }
             )
 
@@ -368,7 +371,11 @@ def __init__(self, scheduler, width=600, **kwargs):
                 fig.yaxis.visible = False
                 fig.ygrid.visible = False
 
-                tap = TapTool(callback=OpenURL(url="http://@bokeh_address/"))
+                tap = TapTool(
+                    callback=OpenURL(
+                        url="./proxy/@dashboard_port/@dashboard_host/status"
+                    )
+                )
                 fig.add_tools(tap)
 
                 fig.toolbar.logo = None
@@ -395,10 +402,8 @@ def update(self):
         with log_errors():
             workers = list(self.scheduler.workers.values())
 
-            bokeh_addresses = []
-            for ws in workers:
-                addr = self.scheduler.get_worker_service_addr(ws.address, "bokeh")
-                bokeh_addresses.append("%s:%d" % addr if addr is not None else "")
+            dashboard_host = [ws.host for ws in workers]
+            dashboard_port = [ws.services.get("bokeh", "") for ws in workers]
 
             y = list(range(len(workers)))
             nprocessing = [len(ws.processing) for ws in workers]
@@ -442,7 +447,8 @@ def update(self):
                     "nbytes-half": [nb / 2 for nb in nbytes],
                     "nbytes-color": nbytes_color,
                     "nbytes_text": nbytes_text,
-                    "bokeh_address": bokeh_addresses,
+                    "dashboard_host": dashboard_host,
+                    "dashboard_port": dashboard_port,
                     "worker": [ws.address for ws in workers],
                     "y": y,
                 }
@@ -1579,6 +1585,7 @@ def __init__(self, scheduler, io_loop=None, prefix="", **kwargs):
         self.prefix = prefix
 
         self.server_kwargs = kwargs
+
         self.server_kwargs["prefix"] = prefix or None
 
         self.apps = {
diff --git a/distributed/bokeh/scheduler_html.py b/distributed/bokeh/scheduler_html.py
index d1ba2646ed6..1d3635c37c5 100644
--- a/distributed/bokeh/scheduler_html.py
+++ b/distributed/bokeh/scheduler_html.py
@@ -7,6 +7,7 @@
 from tornado import web
 
 from ..utils import log_errors, format_bytes, format_time
+from .proxy import GlobalProxyHandler
 
 dirname = os.path.dirname(__file__)
 
@@ -42,6 +43,7 @@ def get(self, worker):
             self.render(
                 "worker.html",
                 title="Worker: " + worker,
+                scheduler=self.server,
                 Worker=worker,
                 **toolz.merge(self.server.__dict__, ns, self.extra)
             )
@@ -55,7 +57,7 @@ def get(self, task):
                 "task.html",
                 title="Task: " + task,
                 Task=task,
-                server=self.server,
+                scheduler=self.server,
                 **toolz.merge(self.server.__dict__, ns, self.extra)
             )
 
@@ -249,6 +251,7 @@ def get(self):
     (r"individual-plots.json", IndividualPlots),
     (r"metrics", PrometheusHandler),
     (r"health", HealthHandler),
+    (r"proxy/(\d+)/(.*?)/(.*)", GlobalProxyHandler),
 ]
 
 
diff --git a/distributed/bokeh/templates/task.html b/distributed/bokeh/templates/task.html
index f396a4cba8f..8c292da4e43 100644
--- a/distributed/bokeh/templates/task.html
+++ b/distributed/bokeh/templates/task.html
@@ -122,7 +122,7 @@ <h3 class="title is-5"> Transition Log </h3>
                   <th> Recommended Action </th>
               </thead>
 
-              {% for key, start, finish, recommendations, time in server.story(Task) %}
+              {% for key, start, finish, recommendations, time in scheduler.story(Task) %}
               <tr>
                   <td> {{ fromtimestamp(time) }} </td>
                   <td> <a href="{{ url_escape(key) }}.html">{{key}}</a> </td>
diff --git a/distributed/bokeh/templates/worker-table.html b/distributed/bokeh/templates/worker-table.html
index 90b59c08c54..8a86f8debd1 100644
--- a/distributed/bokeh/templates/worker-table.html
+++ b/distributed/bokeh/templates/worker-table.html
@@ -1,4 +1,4 @@
-  <table class="table is-striped is-hoverable">
+   <table class="table is-striped is-hoverable">
     <tr>
         <th> Worker </th>
         <th> Cores </th>
@@ -20,7 +20,7 @@
         <td> {{ len(ws.processing) }} </td>
         <td> {{ len(ws.has_what) }} </td>
         {% if 'bokeh' in ws.services %}
-        <td> <a href="http://{{ ws.host }}:{{ ws.services['bokeh'] }}">bokeh</a> </td>
+        <td> <a href="../../proxy/{{ ws.services['bokeh'] }}/{{ ws.host }}/status">bokeh</a> </td>
         {% else %}
         <td> </td>
         {% end %}
diff --git a/distributed/bokeh/templates/workers.html b/distributed/bokeh/templates/workers.html
index 6a2b7fc9345..f300855ac98 100644
--- a/distributed/bokeh/templates/workers.html
+++ b/distributed/bokeh/templates/workers.html
@@ -5,7 +5,6 @@ <h1 class="title"> Scheduler {{scheduler.address}} </h1>
 
   <a class="button is-primary" href="logs.html">Logs</a>
   <a class="button is-primary" href="../../status">Bokeh</a>
-
   {% set worker_list = list(workers.values()) %}
   {% include "worker-table.html" %}
 
diff --git a/distributed/bokeh/tests/test_scheduler_bokeh.py b/distributed/bokeh/tests/test_scheduler_bokeh.py
index f3a57586c72..057aa679655 100644
--- a/distributed/bokeh/tests/test_scheduler_bokeh.py
+++ b/distributed/bokeh/tests/test_scheduler_bokeh.py
@@ -89,7 +89,7 @@ def test_basic(c, s, a, b):
         data = ss.source.data
         assert len(first(data.values()))
         if component is Occupancy:
-            assert all(addr.startswith("127.0.0.1:") for addr in data["bokeh_address"])
+            assert all(addr == "127.0.0.1" for addr in data["dashboard_host"])
 
 
 @gen_cluster(client=True)
@@ -581,3 +581,45 @@ def test_root_redirect(c, s, a, b):
     )
     assert response.code == 200
     assert "/status" in response.effective_url
+
+
+@gen_cluster(
+    client=True,
+    scheduler_kwargs={"services": {("bokeh", 0): BokehScheduler}},
+    worker_kwargs={"services": {"bokeh": BokehWorker}},
+    timeout=180,
+)
+def test_proxy_to_workers(c, s, a, b):
+    try:
+        import jupyter_server_proxy  # noqa: F401
+
+        proxy_exists = True
+    except ImportError:
+        proxy_exists = False
+
+    dashboard_port = s.services["bokeh"].port
+    http_client = AsyncHTTPClient()
+    response = yield http_client.fetch("http://localhost:%d/" % dashboard_port)
+    assert response.code == 200
+    assert "/status" in response.effective_url
+
+    for w in [a, b]:
+        host = w.ip
+        port = w.service_ports["bokeh"]
+        proxy_url = "http://localhost:%d/proxy/%s/%s/status" % (
+            dashboard_port,
+            port,
+            host,
+        )
+        direct_url = "http://localhost:%s/status" % port
+        http_client = AsyncHTTPClient()
+        response_proxy = yield http_client.fetch(proxy_url)
+        response_direct = yield http_client.fetch(direct_url)
+
+        assert response_proxy.code == 200
+        if proxy_exists:
+            assert b"Crossfilter" in response_proxy.body
+        else:
+            assert b"pip install jupyter-server-proxy" in response_proxy.body
+        assert response_direct.code == 200
+        assert b"Crossfilter" in response_direct.body
diff --git a/distributed/bokeh/tests/test_scheduler_bokeh_html.py b/distributed/bokeh/tests/test_scheduler_bokeh_html.py
index 96fe3c2f5d2..691121f7514 100644
--- a/distributed/bokeh/tests/test_scheduler_bokeh_html.py
+++ b/distributed/bokeh/tests/test_scheduler_bokeh_html.py
@@ -14,9 +14,14 @@
 from dask.sizeof import sizeof
 from distributed.utils_test import gen_cluster, slowinc, inc
 from distributed.bokeh.scheduler import BokehScheduler
+from distributed.bokeh.worker import BokehWorker
 
 
-@gen_cluster(client=True, scheduler_kwargs={"services": {("bokeh", 0): BokehScheduler}})
+@gen_cluster(
+    client=True,
+    scheduler_kwargs={"services": {("bokeh", 0): BokehScheduler}},
+    worker_kwargs={"services": {"bokeh": BokehWorker}},
+)
 def test_connect(c, s, a, b):
     future = c.submit(lambda x: x + 1, 1)
     x = c.submit(slowinc, 1, delay=1, retries=5)
diff --git a/distributed/cli/tests/test_dask_worker.py b/distributed/cli/tests/test_dask_worker.py
index 5ed668e758a..aac27061b21 100644
--- a/distributed/cli/tests/test_dask_worker.py
+++ b/distributed/cli/tests/test_dask_worker.py
@@ -249,8 +249,14 @@ def func(dask_worker):
 
 def test_bokeh_non_standard_ports(loop):
     pytest.importorskip("bokeh")
+    try:
+        import jupyter_server_proxy  # noqa: F401
 
-    with popen(["dask-scheduler", "--port", "3449", "--no-bokeh"]):
+        proxy_exists = True
+    except ImportError:
+        proxy_exists = False
+
+    with popen(["dask-scheduler", "--port", "3449"]):
         with popen(
             ["dask-worker", "tcp://127.0.0.1:3449", "--dashboard-address", ":4833"]
         ) as proc:
@@ -264,9 +270,15 @@ def test_bokeh_non_standard_ports(loop):
                     assert response.ok
                     redirect_resp = requests.get("http://127.0.0.1:4833/main")
                     redirect_resp.ok
+                    # TEST PROXYING WORKS
+                    if proxy_exists:
+                        url = "http://127.0.0.1:8787/proxy/4833/127.0.0.1/status"
+                        response = requests.get(url)
+                        assert response.ok
                     break
                 except Exception:
                     sleep(0.5)
                     assert time() < start + 20
+
         with pytest.raises(Exception):
             requests.get("http://localhost:4833/status/")
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 801c0c849d7..cce66fb1767 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -856,6 +856,7 @@ def __init__(
         )
         self.digests = None
         self.service_specs = services or {}
+        self.service_kwargs = service_kwargs or {}
         self.services = {}
         self.scheduler_file = scheduler_file
         worker_ttl = worker_ttl or dask.config.get("distributed.scheduler.worker-ttl")

From 6134c754b08b35fd3e98d6128b9cdb2f28bb5300 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 22 May 2019 15:08:48 -0500
Subject: [PATCH 0303/1550] Replace register_worker_callbacks with worker
 plugins (#2453)

* Add worker plugins

* add docstring

* Replace legacy worker_callbacks with worker_plugins

* add and test name keyword

* fix missing import

* black

* respond to feedback

* Handle errors again

* Expand docstring
---
 distributed/client.py                    | 92 ++++++++++++++++++++----
 distributed/scheduler.py                 | 17 +++--
 distributed/tests/test_worker.py         |  7 +-
 distributed/tests/test_worker_plugins.py | 68 ++++++++++++++++++
 distributed/worker.py                    | 57 ++++++++++++---
 5 files changed, 206 insertions(+), 35 deletions(-)
 create mode 100644 distributed/tests/test_worker_plugins.py

diff --git a/distributed/client.py b/distributed/client.py
index 6b5bc2811bb..6b22f5b8ee5 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -89,6 +89,7 @@
     parse_timedelta,
     shutting_down,
     Any,
+    has_keyword,
 )
 from .versions import get_versions
 
@@ -3854,17 +3855,6 @@ def _get_task_stream(
         else:
             raise gen.Return(msgs)
 
-    @gen.coroutine
-    def _register_worker_callbacks(self, setup=None):
-        responses = yield self.scheduler.register_worker_callbacks(setup=dumps(setup))
-        results = {}
-        for key, resp in responses.items():
-            if resp["status"] == "OK":
-                results[key] = resp["result"]
-            elif resp["status"] == "error":
-                six.reraise(*clean_exception(**resp))
-        raise gen.Return(results)
-
     def register_worker_callbacks(self, setup=None):
         """
         Registers a setup callback function for all current and future workers.
@@ -3883,7 +3873,85 @@ def register_worker_callbacks(self, setup=None):
         setup : callable(dask_worker: Worker) -> None
             Function to register and run on all workers
         """
-        return self.sync(self._register_worker_callbacks, setup=setup)
+        return self.register_worker_plugin(_WorkerSetupPlugin(setup))
+
+    @gen.coroutine
+    def _register_worker_plugin(self, plugin=None, name=None):
+        responses = yield self.scheduler.register_worker_plugin(
+            plugin=dumps(plugin), name=name
+        )
+        for response in responses.values():
+            if response["status"] == "error":
+                exc = response["exception"]
+                typ = type(exc)
+                tb = response["traceback"]
+                six.reraise(typ, exc, tb)
+        raise gen.Return(responses)
+
+    def register_worker_plugin(self, plugin=None, name=None):
+        """
+        Registers a lifecycle worker plugin for all current and future workers.
+
+        This registers a new object to handle setup and teardown for workers in
+        this cluster. The plugin will instantiate itself on all currently
+        connected workers.  It will also be run on any worker that connects in
+        the future.
+
+        The plugin should be an object with ``setup`` and ``teardown`` methods.
+        It must be serializable with the pickle or cloudpickle modules.
+
+        If the plugin has a ``name`` attribute, or if the ``name=`` keyword is
+        used then that will control idempotency.  A a plugin with that name has
+        already registered then any future plugins will not run.
+
+        For alternatives to plugins, you may also wish to look into preload
+        scripts.
+
+        Parameters
+        ----------
+        plugin: object
+            The plugin object to pass to the workers
+        name: str, optional
+            A name for the plugin.
+            Registering a plugin with the same name will have no effect.
+
+        Examples
+        --------
+        >>> class MyPlugin:
+        ...     def __init__(self, *args, **kwargs):
+        ...         pass  # the constructor is up to you
+        ...     def setup(self, worker: dask.distributed.Worker):
+        ...         pass
+        ...     def teardown(self, worker: dask.distributed.Worker):
+        ...         pass
+
+        >>> plugin = MyPlugin(1, 2, 3)
+        >>> client.register_worker_plugin(plugin)
+
+        You can get access to the plugin with the ``get_worker`` function
+
+        >>> client.register_worker_plugin(other_plugin, name='my-plugin')
+        >>> def f():
+        ...    worker = get_worker()
+        ...    plugin = worker.plugins['my-plugin']
+        ...    return plugin.my_state
+
+        >>> future = client.run(f)
+        """
+        return self.sync(self._register_worker_plugin, plugin=plugin, name=name)
+
+
+class _WorkerSetupPlugin(object):
+    """ This is used to support older setup functions as callbacks """
+
+    def __init__(self, setup):
+        self._setup = setup
+
+    def setup(self, worker):
+        if has_keyword(self._setup, "dask_worker"):
+            return self._setup(dask_worker=worker)
+        else:
+            return self._setup()
 
 
 class Executor(Client):
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index cce66fb1767..57d768f95f6 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1003,7 +1003,7 @@ def __init__(
         self.log = deque(
             maxlen=dask.config.get("distributed.scheduler.transition-log-length")
         )
-        self.worker_setups = []
+        self.worker_plugins = []
 
         worker_handlers = {
             "task-finished": self.handle_task_finished,
@@ -1062,7 +1062,7 @@ def __init__(
             "heartbeat_worker": self.heartbeat_worker,
             "get_task_status": self.get_task_status,
             "get_task_stream": self.get_task_stream,
-            "register_worker_callbacks": self.register_worker_callbacks,
+            "register_worker_plugin": self.register_worker_plugin,
         }
 
         self._transitions = {
@@ -1510,7 +1510,7 @@ def add_worker(
                     "status": "OK",
                     "time": time(),
                     "heartbeat-interval": heartbeat_interval(len(self.workers)),
-                    "worker-setups": self.worker_setups,
+                    "worker-plugins": self.worker_plugins,
                 }
             )
             yield self.handle_worker(comm=comm, worker=address)
@@ -3407,14 +3407,13 @@ def get_task_stream(self, comm=None, start=None, stop=None, count=None):
         return ts.collect(start=start, stop=stop, count=count)
 
     @gen.coroutine
-    def register_worker_callbacks(self, comm, setup=None):
+    def register_worker_plugin(self, comm, plugin, name=None):
         """ Registers a setup function, and call it on every worker """
-        if setup is None:
-            raise gen.Return({})
-
-        self.worker_setups.append(setup)
+        self.worker_plugins.append(plugin)
 
-        responses = yield self.broadcast(msg=dict(op="run", function=setup))
+        responses = yield self.broadcast(
+            msg=dict(op="plugin-add", plugin=plugin, name=name)
+        )
         raise gen.Return(responses)
 
     #####################
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 4541e183e46..9fc967eef5a 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -1312,7 +1312,6 @@ def test_startup2():
         return os.getenv("MY_ENV_VALUE", None) == "WORKER_ENV_VALUE"
 
     # Nothing has been run yet
-    assert len(s.worker_setups) == 0
     result = yield c.run(test_import)
     assert list(result.values()) == [False] * 2
     result = yield c.run(test_startup2)
@@ -1327,7 +1326,6 @@ def test_startup2():
     # Add a preload function
     response = yield c.register_worker_callbacks(setup=mystartup)
     assert len(response) == 2
-    assert len(s.worker_setups) == 1
 
     # Check it has been ran on existing worker
     result = yield c.run(test_import)
@@ -1342,7 +1340,6 @@ def test_startup2():
     # Register another preload function
     response = yield c.register_worker_callbacks(setup=mystartup2)
     assert len(response) == 2
-    assert len(s.worker_setups) == 2
 
     # Check it has been run
     result = yield c.run(test_startup2)
@@ -1356,7 +1353,9 @@ def test_startup2():
     assert list(result.values()) == [True]
     yield worker.close()
 
-    # Final exception test
+
+@gen_cluster(client=True)
+def test_register_worker_callbacks_err(c, s, a, b):
     with pytest.raises(ZeroDivisionError):
         yield c.register_worker_callbacks(setup=lambda: 1 / 0)
 
diff --git a/distributed/tests/test_worker_plugins.py b/distributed/tests/test_worker_plugins.py
new file mode 100644
index 00000000000..25388459788
--- /dev/null
+++ b/distributed/tests/test_worker_plugins.py
@@ -0,0 +1,68 @@
+from distributed.utils_test import gen_cluster
+from distributed import Worker
+
+
+class MyPlugin:
+    name = "MyPlugin"
+
+    def __init__(self, data):
+        self.data = data
+
+    def setup(self, worker):
+        assert isinstance(worker, Worker)
+        self.worker = worker
+        self.worker._my_plugin_status = "setup"
+        self.worker._my_plugin_data = self.data
+
+    def teardown(self, worker):
+        assert isinstance(worker, Worker)
+        self.worker._my_plugin_status = "teardown"
+
+
+@gen_cluster(client=True, ncores=[])
+def test_create_with_client(c, s):
+    yield c.register_worker_plugin(MyPlugin(123))
+
+    worker = Worker(s.address, loop=s.loop)
+    yield worker._start()
+    assert worker._my_plugin_status == "setup"
+    assert worker._my_plugin_data == 123
+
+    yield worker._close()
+    assert worker._my_plugin_status == "teardown"
+
+
+@gen_cluster(client=True, worker_kwargs={"plugins": [MyPlugin(5)]})
+def test_create_on_construction(c, s, a, b):
+    assert len(a.plugins) == len(b.plugins) == 1
+    assert a._my_plugin_status == "setup"
+    assert a._my_plugin_data == 5
+
+
+@gen_cluster(client=True, worker_kwargs={"plugins": [MyPlugin(5)]})
+def test_idempotence_with_name(c, s, a, b):
+    a._my_plugin_data = 100
+
+    yield c.register_worker_plugin(MyPlugin(5))
+
+    assert a._my_plugin_data == 100  # call above has no effect
+
+
+@gen_cluster(client=True, worker_kwargs={"plugins": [MyPlugin(5)]})
+def test_duplicate_with_no_name(c, s, a, b):
+    assert len(a.plugins) == len(b.plugins) == 1
+
+    plugin = MyPlugin(10)
+    plugin.name = "other-name"
+
+    yield c.register_worker_plugin(plugin)
+
+    assert len(a.plugins) == len(b.plugins) == 2
+
+    assert a._my_plugin_data == 10
+
+    yield c.register_worker_plugin(plugin)
+    assert len(a.plugins) == len(b.plugins) == 2
+
+    yield c.register_worker_plugin(plugin, name="foo")
+    assert len(a.plugins) == len(b.plugins) == 3
diff --git a/distributed/worker.py b/distributed/worker.py
index f4189393091..3fcc477bf48 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -10,6 +10,7 @@
 import random
 import threading
 import sys
+import uuid
 import warnings
 import weakref
 import psutil
@@ -307,6 +308,7 @@ def __init__(
         protocol=None,
         dashboard_address=None,
         nanny=None,
+        plugins=(),
         low_level_profiler=dask.config.get("distributed.worker.profile.low-level"),
         **kwargs
     ):
@@ -576,6 +578,7 @@ def __init__(
             "versions": self.versions,
             "actor_execute": self.actor_execute,
             "actor_attribute": self.actor_attribute,
+            "plugin-add": self.plugin_add,
         }
 
         stream_handlers = {
@@ -638,6 +641,9 @@ def __init__(
         )
         self.periodic_callbacks["profile-cycle"] = pc
 
+        self.plugins = {}
+        self._pending_plugins = plugins
+
         Worker._instances.add(self)
 
     ##################
@@ -763,16 +769,9 @@ def _register_with_scheduler(self):
         if response["status"] != "OK":
             raise ValueError("Unexpected response from register: %r" % (response,))
         else:
-            # Retrieve eventual init functions and run them
-            for function_bytes in response["worker-setups"]:
-                setup_function = pickle.loads(function_bytes)
-                if has_arg(setup_function, "dask_worker"):
-                    result = setup_function(dask_worker=self)
-                else:
-                    result = setup_function()
-                logger.info(
-                    "Init function %s ran: output=%s" % (setup_function, result)
-                )
+            yield [
+                self.plugin_add(plugin=plugin) for plugin in response["worker-plugins"]
+            ]
 
             logger.info("        Registered to: %26s", self.scheduler.address)
             logger.info("-" * 49)
@@ -968,6 +967,9 @@ def _start(self, addr_or_port=0):
 
         setproctitle("dask-worker [%s]" % self.address)
 
+        yield [self.plugin_add(plugin=plugin) for plugin in self._pending_plugins]
+        self._pending_plugins = ()
+
         yield self._register_with_scheduler()
 
         self.start_periodic_callbacks()
@@ -998,6 +1000,12 @@ def close(self, report=True, timeout=10, nanny=True, executor_wait=True):
             self.status = "closing"
             setproctitle("dask-worker [closing]")
 
+            yield [
+                plugin.teardown(self)
+                for plugin in self.plugins.values()
+                if hasattr(plugin, "teardown")
+            ]
+
             self.stop()
             for pc in self.periodic_callbacks.values():
                 pc.stop()
@@ -2206,6 +2214,35 @@ def run(self, comm, function, args=(), wait=True, kwargs=None):
     def run_coroutine(self, comm, function, args=(), kwargs=None, wait=True):
         return run(self, comm, function=function, args=args, kwargs=kwargs, wait=wait)
 
+    @gen.coroutine
+    def plugin_add(self, comm=None, plugin=None, name=None):
+        with log_errors(pdb=False):
+            if isinstance(plugin, bytes):
+                plugin = pickle.loads(plugin)
+            if not name:
+                if hasattr(plugin, "name"):
+                    name = plugin.name
+                else:
+                    name = funcname(plugin) + "-" + str(uuid.uuid4())
+
+            assert name
+
+            if name in self.plugins:
+                return {"status": "repeat"}
+            else:
+                self.plugins[name] = plugin
+
+                logger.info("Starting Worker plugin %s" % name)
+                try:
+                    result = plugin.setup(worker=self)
+                    if isinstance(result, gen.Future):
+                        result = yield result
+                except Exception as e:
+                    msg = error_message(e)
+                    return msg
+                else:
+                    return {"status": "OK"}
+
     @gen.coroutine
     def actor_execute(self, comm=None, actor=None, function=None, args=(), kwargs={}):
         separate_thread = kwargs.pop("separate_thread", True)

From 6e0c0a6b90b1d3c3f686f0c968e9cf3d0c354413 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 22 May 2019 16:45:53 -0500
Subject: [PATCH 0304/1550] Add SpecificationCluster (#2675)

This is intended to be a base for LocalCluster (and others) that want to
specify more heterogeneous information about workers.

This forces the use of Python 3 and introduces more asyncio and async def handling.

This cleans up a number of intermittent testing failures and improves our testing harness hygeine.
---
 .../setup_conda_environment.cmd               |   2 +-
 continuous_integration/travis/install.sh      |   2 +-
 distributed/__init__.py                       |   2 +-
 distributed/cli/dask_worker.py                |   2 +-
 distributed/client.py                         |  24 +-
 distributed/comm/tcp.py                       |   1 +
 distributed/core.py                           |   8 +-
 distributed/deploy/__init__.py                |   1 +
 distributed/deploy/adaptive.py                |   4 +-
 distributed/deploy/cluster.py                 |  32 +-
 distributed/deploy/local.py                   | 326 ++----------------
 distributed/deploy/spec.py                    | 297 ++++++++++++++++
 distributed/deploy/tests/py3_test_deploy.py   |  15 +-
 distributed/deploy/tests/test_adaptive.py     |  56 ++-
 distributed/deploy/tests/test_local.py        | 120 ++++---
 distributed/deploy/tests/test_spec_cluster.py | 115 ++++++
 distributed/deploy/utils_test.py              |  17 +-
 distributed/nanny.py                          |  34 +-
 distributed/scheduler.py                      |  12 +-
 distributed/tests/test_as_completed.py        |   4 +-
 distributed/tests/test_asyncprocess.py        |   1 +
 distributed/tests/test_client.py              |  39 ++-
 distributed/tests/test_nanny.py               |   9 +-
 distributed/tests/test_scheduler.py           |  16 +-
 distributed/tests/test_worker.py              |  31 +-
 distributed/tests/test_worker_client.py       |   4 +-
 distributed/utils.py                          |   2 +
 distributed/utils_test.py                     |  84 ++---
 distributed/worker.py                         |  15 +-
 29 files changed, 774 insertions(+), 501 deletions(-)
 create mode 100644 distributed/deploy/spec.py
 create mode 100644 distributed/deploy/tests/test_spec_cluster.py

diff --git a/continuous_integration/setup_conda_environment.cmd b/continuous_integration/setup_conda_environment.cmd
index cd201ff46d5..5748a8cf20c 100644
--- a/continuous_integration/setup_conda_environment.cmd
+++ b/continuous_integration/setup_conda_environment.cmd
@@ -50,7 +50,7 @@ call activate %CONDA_ENV%
 %PIP_INSTALL% git+https://github.com/joblib/joblib.git --upgrade
 %PIP_INSTALL% git+https://github.com/dask/zict --upgrade
 
-%PIP_INSTALL% pytest-repeat pytest-timeout pytest-faulthandler sortedcollections
+%PIP_INSTALL% pytest-repeat pytest-timeout pytest-faulthandler sortedcollections pytest-asyncio
 
 @rem Display final environment (for reproducing)
 %CONDA% list
diff --git a/continuous_integration/travis/install.sh b/continuous_integration/travis/install.sh
index bba69dd3ac8..f1ff25a9bfa 100644
--- a/continuous_integration/travis/install.sh
+++ b/continuous_integration/travis/install.sh
@@ -59,7 +59,7 @@ conda install -q \
 conda install -c defaults -c conda-forge libunwind
 conda install --no-deps -c defaults -c numba -c conda-forge stacktrace
 
-pip install -q pytest-repeat pytest-faulthandler
+pip install -q pytest-repeat pytest-faulthandler pytest-asyncio
 
 pip install -q git+https://github.com/dask/dask.git --upgrade --no-deps
 pip install -q git+https://github.com/joblib/joblib.git --upgrade --no-deps
diff --git a/distributed/__init__.py b/distributed/__init__.py
index 7b2bc4ab082..2a632607cf9 100644
--- a/distributed/__init__.py
+++ b/distributed/__init__.py
@@ -4,7 +4,7 @@
 from dask.config import config
 from .actor import Actor, ActorFuture
 from .core import connect, rpc
-from .deploy import LocalCluster, Adaptive
+from .deploy import LocalCluster, Adaptive, SpecCluster
 from .diagnostics import progress
 from .client import (
     Client,
diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
index e383095b382..439bdaf4a62 100755
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -114,7 +114,7 @@
 @click.option(
     "--name",
     type=str,
-    default="",
+    default=None,
     help="A unique name for this worker like 'worker-1'. "
     "If used with --nprocs then the process number "
     "will be appended like name-0, name-1, name-2, ...",
diff --git a/distributed/client.py b/distributed/client.py
index 6b22f5b8ee5..afe6f6ef39f 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -2,8 +2,8 @@
 
 import atexit
 from collections import defaultdict
-from concurrent.futures import ThreadPoolExecutor
-from concurrent.futures._base import DoneAndNotDoneFutures, CancelledError
+from concurrent.futures import ThreadPoolExecutor, CancelledError
+from concurrent.futures._base import DoneAndNotDoneFutures
 from contextlib import contextmanager
 import copy
 from datetime import timedelta
@@ -44,6 +44,8 @@
 from tornado.ioloop import IOLoop
 from tornado.queues import Queue
 
+from asyncio import iscoroutine
+
 from .batched import BatchedSend
 from .utils_comm import (
     WrappedKey,
@@ -1309,7 +1311,13 @@ def close(self, timeout=no_default):
 
         if self._start_arg is None:
             with ignoring(AttributeError):
-                self.cluster.close()
+                f = self.cluster.close()
+                if iscoroutine(f):
+
+                    async def _():
+                        await f
+
+                    self.sync(_)
 
         sync(self.loop, self._close, fast=True)
 
@@ -1644,10 +1652,11 @@ def wait(k):
                             st = self.futures[key]
                             exception = st.exception
                             traceback = st.traceback
-                        except (AttributeError, KeyError):
-                            six.reraise(CancelledError, CancelledError(key), None)
+                        except (KeyError, AttributeError):
+                            exc = CancelledError(key)
                         else:
                             six.reraise(type(exception), exception, traceback)
+                        raise exc
                     if errors == "skip":
                         bad_keys.add(key)
                         bad_data[key] = None
@@ -4134,7 +4143,10 @@ def _track_future(self, future):
         except CancelledError:
             pass
         if self.with_results:
-            result = yield future._result(raiseit=False)
+            try:
+                result = yield future._result(raiseit=False)
+            except CancelledError as exc:
+                result = exc
         with self.lock:
             self.futures[future] -= 1
             if not self.futures[future]:
diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py
index 85dbe2ce278..d5351c7d565 100644
--- a/distributed/comm/tcp.py
+++ b/distributed/comm/tcp.py
@@ -431,6 +431,7 @@ def start(self):
                 break
         else:
             raise exc
+        self.get_host_port()  # trigger assignment to self.bound_address
 
     def stop(self):
         tcp_server, self.tcp_server = self.tcp_server, None
diff --git a/distributed/core.py b/distributed/core.py
index 9b1d408a038..17685c9d2d5 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -489,14 +489,16 @@ def handle_stream(self, comm, extra=None, every_cycle=[]):
 
     @gen.coroutine
     def close(self):
-        self.listener.stop()
+        for pc in self.periodic_callbacks.values():
+            pc.stop()
+        if self.listener:
+            self.listener.stop()
         for i in range(20):  # let comms close naturally for a second
             if not self._comms:
                 break
             else:
                 yield gen.sleep(0.05)
-        for comm in self._comms:
-            comm.close()
+        yield [comm.close() for comm in self._comms]
         for cb in self._ongoing_coroutines:
             cb.cancel()
         for i in range(10):
diff --git a/distributed/deploy/__init__.py b/distributed/deploy/__init__.py
index 35abf0a6439..9b5e478c303 100644
--- a/distributed/deploy/__init__.py
+++ b/distributed/deploy/__init__.py
@@ -4,6 +4,7 @@
 
 from .cluster import Cluster
 from .local import LocalCluster
+from .spec import SpecCluster
 from .adaptive import Adaptive
 
 with ignoring(ImportError):
diff --git a/distributed/deploy/adaptive.py b/distributed/deploy/adaptive.py
index 8c260609638..793e80d984c 100644
--- a/distributed/deploy/adaptive.py
+++ b/distributed/deploy/adaptive.py
@@ -272,7 +272,7 @@ def _retire_workers(self, workers=None):
 
             logger.info("Retiring workers %s", workers)
             f = self.cluster.scale_down(workers)
-            if gen.is_future(f):
+            if hasattr(f, "__await__"):
                 yield f
 
             raise gen.Return(workers)
@@ -354,7 +354,7 @@ def _adapt(self):
             if status == "up":
                 f = self.cluster.scale_up(**recommendations)
                 self.log.append((time(), "up", recommendations))
-                if gen.is_future(f):
+                if hasattr(f, "__await__"):
                     yield f
 
             elif status == "down":
diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
index f170d4ea5ad..8425b836a4d 100644
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -1,12 +1,23 @@
+from datetime import timedelta
 import logging
 import os
 from weakref import ref
 
 import dask
+from tornado import gen
 
 from .adaptive import Adaptive
 
-from ..utils import format_bytes, PeriodicCallback, log_errors, ignoring
+from ..compatibility import get_thread_identity
+from ..utils import (
+    format_bytes,
+    PeriodicCallback,
+    log_errors,
+    ignoring,
+    sync,
+    thread_state,
+)
+
 
 logger = logging.getLogger(__name__)
 
@@ -215,3 +226,22 @@ def update():
 
     def _ipython_display_(self, **kwargs):
         return self._widget()._ipython_display_(**kwargs)
+
+    @property
+    def asynchronous(self):
+        return (
+            self._asynchronous
+            or getattr(thread_state, "asynchronous", False)
+            or hasattr(self.loop, "_thread_identity")
+            and self.loop._thread_identity == get_thread_identity()
+        )
+
+    def sync(self, func, *args, **kwargs):
+        if kwargs.pop("asynchronous", None) or self.asynchronous:
+            callback_timeout = kwargs.pop("callback_timeout", None)
+            future = func(*args, **kwargs)
+            if callback_timeout is not None:
+                future = gen.with_timeout(timedelta(seconds=callback_timeout), future)
+            return future
+        else:
+            return sync(self.loop, func, *args, **kwargs)
diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index 832e8f3e051..17150fdf70f 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -1,29 +1,14 @@
 from __future__ import print_function, division, absolute_import
 
 import atexit
-from datetime import timedelta
 import logging
 import math
 import warnings
 import weakref
-import toolz
 
 from dask.utils import factors
-from tornado import gen
-
-from .cluster import Cluster
-from ..compatibility import get_thread_identity
-from ..core import CommClosedError
-from ..utils import (
-    sync,
-    ignoring,
-    All,
-    silence_logging,
-    LoopRunner,
-    log_errors,
-    thread_state,
-    parse_timedelta,
-)
+
+from .spec import SpecCluster
 from ..nanny import Nanny
 from ..scheduler import Scheduler
 from ..worker import Worker, parse_memory_limit, _ncores
@@ -31,7 +16,7 @@
 logger = logging.getLogger(__name__)
 
 
-class LocalCluster(Cluster):
+class LocalCluster(SpecCluster):
     """ Create local Scheduler and Workers
 
     This creates a "cluster" of a scheduler and workers running on the local
@@ -105,8 +90,8 @@ def __init__(
         processes=True,
         loop=None,
         start=None,
-        ip=None,
         host=None,
+        ip=None,
         scheduler_port=0,
         silence_logs=logging.WARN,
         dashboard_address=":8787",
@@ -127,15 +112,6 @@ def __init__(
             warnings.warn("The ip keyword has been moved to host")
             host = ip
 
-        if start is not None:
-            msg = (
-                "The start= parameter is deprecated. "
-                "LocalCluster always starts. "
-                "For asynchronous operation use the following: \n\n"
-                "  cluster = yield LocalCluster(asynchronous=True)"
-            )
-            raise ValueError(msg)
-
         if diagnostics_port is not None:
             warnings.warn(
                 "diagnostics_port has been deprecated. "
@@ -161,12 +137,8 @@ def __init__(
         if host is None and not protocol.startswith("inproc") and not interface:
             host = "127.0.0.1"
 
-        self.silence_logs = silence_logs
-        self._asynchronous = asynchronous
         services = services or {}
         worker_services = worker_services or {}
-        if silence_logs:
-            self._old_logging_level = silence_logging(level=silence_logs)
         if n_workers is None and threads_per_worker is None:
             if processes:
                 n_workers, threads_per_worker = nprocesses_nthreads(_ncores)
@@ -188,268 +160,42 @@ def __init__(
                 "dashboard_address": worker_dashboard_address,
                 "interface": interface,
                 "protocol": protocol,
+                "security": security,
+                "silence_logs": silence_logs,
             }
         )
 
-        self._loop_runner = LoopRunner(loop=loop, asynchronous=asynchronous)
-        self.loop = self._loop_runner.loop
-
-        self.scheduler = Scheduler(
-            loop=self.loop,
-            host=host,
-            services=services,
-            service_kwargs=service_kwargs,
-            security=security,
-            port=scheduler_port,
-            interface=interface,
-            protocol=protocol,
-            dashboard_address=dashboard_address,
-            blocked_handlers=blocked_handlers,
+        scheduler = {
+            "cls": Scheduler,
+            "options": dict(
+                host=host,
+                services=services,
+                service_kwargs=service_kwargs,
+                security=security,
+                port=scheduler_port,
+                interface=interface,
+                protocol=protocol,
+                dashboard_address=dashboard_address,
+                blocked_handlers=blocked_handlers,
+            ),
+        }
+
+        worker = {
+            "cls": worker_class or (Worker if not processes else Nanny),
+            "options": worker_kwargs,
+        }
+
+        workers = {i: worker for i in range(n_workers)}
+
+        super(LocalCluster, self).__init__(
+            scheduler=scheduler,
+            workers=workers,
+            worker=worker,
+            loop=loop,
+            asynchronous=asynchronous,
+            silence_logs=silence_logs,
         )
-
-        self.workers = []
-        self.worker_kwargs = worker_kwargs
-        if security:
-            self.worker_kwargs["security"] = security
-
-        if not worker_class:
-            worker_class = Worker if not processes else Nanny
-        self.worker_class = worker_class
-
-        self.start(n_workers=n_workers)
-
-        clusters_to_close.add(self)
-
-    def __repr__(self):
-        return "LocalCluster(%r, workers=%d, ncores=%d)" % (
-            self.scheduler_address,
-            len(self.workers),
-            sum(w.ncores for w in self.workers),
-        )
-
-    def __await__(self):
-        return self._started.__await__()
-
-    @property
-    def asynchronous(self):
-        return (
-            self._asynchronous
-            or getattr(thread_state, "asynchronous", False)
-            or hasattr(self.loop, "_thread_identity")
-            and self.loop._thread_identity == get_thread_identity()
-        )
-
-    def sync(self, func, *args, **kwargs):
-        if kwargs.pop("asynchronous", None) or self.asynchronous:
-            callback_timeout = kwargs.pop("callback_timeout", None)
-            future = func(*args, **kwargs)
-            if callback_timeout is not None:
-                future = gen.with_timeout(timedelta(seconds=callback_timeout), future)
-            return future
-        else:
-            return sync(self.loop, func, *args, **kwargs)
-
-    def start(self, **kwargs):
-        self._loop_runner.start()
-        if self._asynchronous:
-            self._started = self._start(**kwargs)
-        else:
-            self.sync(self._start, **kwargs)
-
-    @gen.coroutine
-    def _start(self, n_workers=0):
-        """
-        Start all cluster services.
-        """
-        if self.status == "running":
-            return
-
-        self.scheduler.start()
-
-        yield [self._start_worker(**self.worker_kwargs) for i in range(n_workers)]
-        yield self.scheduler
-
-        self.status = "running"
-
-        raise gen.Return(self)
-
-    @gen.coroutine
-    def _start_worker(self, death_timeout=60, **kwargs):
-        if self.status and self.status.startswith("clos"):
-            warnings.warn(
-                "Tried to start a worker while status=='%s'" % self.status, stacklevel=2
-            )
-            return
-
-        if self.processes:
-            kwargs["quiet"] = True
-
-        w = yield self.worker_class(
-            self.scheduler.address,
-            loop=self.loop,
-            death_timeout=death_timeout,
-            silence_logs=self.silence_logs,
-            **kwargs
-        )
-
-        self.workers.append(w)
-
-        while w.status != "closed" and w.worker_address not in self.scheduler.workers:
-            yield gen.sleep(0.01)
-
-        if w.status == "closed" and self.scheduler.status == "running":
-            self.workers.remove(w)
-            raise gen.TimeoutError("Worker failed to start")
-
-        raise gen.Return(w)
-
-    def start_worker(self, **kwargs):
-        """ Add a new worker to the running cluster
-
-        Parameters
-        ----------
-        port: int (optional)
-            Port on which to serve the worker, defaults to 0 or random
-        ncores: int (optional)
-            Number of threads to use.  Defaults to number of logical cores
-
-        Examples
-        --------
-        >>> c = LocalCluster()  # doctest: +SKIP
-        >>> c.start_worker(ncores=2)  # doctest: +SKIP
-
-        Returns
-        -------
-        The created Worker or Nanny object.  Can be discarded.
-        """
-        return self.sync(self._start_worker, **kwargs)
-
-    @gen.coroutine
-    def _stop_worker(self, w):
-        yield w.close()
-        if w in self.workers:
-            self.workers.remove(w)
-
-    def stop_worker(self, w):
-        """ Stop a running worker
-
-        Examples
-        --------
-        >>> c = LocalCluster()  # doctest: +SKIP
-        >>> w = c.start_worker(ncores=2)  # doctest: +SKIP
-        >>> c.stop_worker(w)  # doctest: +SKIP
-        """
-        self.sync(self._stop_worker, w)
-
-    @gen.coroutine
-    def _close(self, timeout="2s"):
-        # Can be 'closing' as we're called by close() below
-        if self.status == "closed":
-            return
-        self.status = "closing"
-
-        with ignoring(gen.TimeoutError, CommClosedError, OSError):
-            yield gen.with_timeout(
-                timedelta(seconds=parse_timedelta(timeout)),
-                self.scheduler.close(close_workers=True),
-            )
-
-        with ignoring(gen.TimeoutError):
-            yield gen.with_timeout(
-                timedelta(seconds=parse_timedelta(timeout)),
-                All([self._stop_worker(w) for w in self.workers]),
-            )
-        del self.workers[:]
-        self.status = "closed"
-
-    def close(self, timeout=20):
-        """ Close the cluster """
-        if self.status == "closed":
-            return
-
-        try:
-            result = self.sync(self._close, callback_timeout=timeout)
-        except RuntimeError:  # IOLoop is closed
-            result = None
-
-        if hasattr(self, "_old_logging_level"):
-            if self.asynchronous:
-                result.add_done_callback(
-                    lambda _: silence_logging(self._old_logging_level)
-                )
-            else:
-                silence_logging(self._old_logging_level)
-
-        if not self.asynchronous:
-            self._loop_runner.stop()
-
-        return result
-
-    @gen.coroutine
-    def scale_up(self, n, **kwargs):
-        """ Bring the total count of workers up to ``n``
-
-        This function/coroutine should bring the total number of workers up to
-        the number ``n``.
-
-        This can be implemented either as a function or as a Tornado coroutine.
-        """
-        with log_errors():
-            kwargs2 = toolz.merge(self.worker_kwargs, kwargs)
-            yield [
-                self._start_worker(**kwargs2)
-                for i in range(n - len(self.scheduler.workers))
-            ]
-
-            # clean up any closed worker
-            self.workers = [w for w in self.workers if w.status != "closed"]
-
-    @gen.coroutine
-    def scale_down(self, workers):
-        """ Remove ``workers`` from the cluster
-
-        Given a list of worker addresses this function should remove those
-        workers from the cluster.  This may require tracking which jobs are
-        associated to which worker address.
-
-        This can be implemented either as a function or as a Tornado coroutine.
-        """
-        with log_errors():
-            # clean up any closed worker
-            self.workers = [w for w in self.workers if w.status != "closed"]
-            workers = set(workers)
-
-            # we might be given addresses
-            if all(isinstance(w, str) for w in workers):
-                workers = {w for w in self.workers if w.worker_address in workers}
-
-            # stop the provided workers
-            yield [self._stop_worker(w) for w in workers]
-
-    def __del__(self):
-        self.close()
-
-    def __enter__(self):
-        return self
-
-    def __exit__(self, *args):
-        self.close()
-
-    @gen.coroutine
-    def __aenter__(self):
-        yield self._started
-        raise gen.Return(self)
-
-    @gen.coroutine
-    def __aexit__(self, typ, value, traceback):
-        yield self._close()
-
-    @property
-    def scheduler_address(self):
-        try:
-            return self.scheduler.address
-        except ValueError:
-            return "<unstarted>"
+        self.scale(n_workers)
 
 
 def nprocesses_nthreads(n):
diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
new file mode 100644
index 00000000000..9a4385e5054
--- /dev/null
+++ b/distributed/deploy/spec.py
@@ -0,0 +1,297 @@
+import asyncio
+import weakref
+
+from tornado import gen
+
+from .cluster import Cluster
+from ..utils import LoopRunner, silence_logging, ignoring
+from ..scheduler import Scheduler
+
+
+class SpecCluster(Cluster):
+    """ Cluster that requires a full specification of workers
+
+    The SpecCluster class expects a full specification of the Scheduler and
+    Workers to use.  It removes any handling of user inputs (like threads vs
+    processes, number of cores, and so on) and any handling of cluster resource
+    managers (like pods, jobs, and so on).  Instead, it expects this
+    information to be passed in scheduler and worker specifications.  This
+    class does handle all of the logic around asynchronously cleanly setting up
+    and tearing things down at the right times.  Hopefully it can form a base
+    for other more user-centric classes.
+
+    Parameters
+    ----------
+    workers: dict
+        A dictionary mapping names to worker classes and their specifications
+        See example below
+    scheduler: dict, optional
+        A similar mapping for a scheduler
+    worker: dict
+        A specification of a single worker.
+        This is used for any new workers that are created.
+    asynchronous: bool
+        If this is intended to be used directly within an event loop with
+        async/await
+    silence_logs: bool
+        Whether or not we should silence logging when setting up the cluster.
+
+    Examples
+    --------
+    To create a SpecCluster you specify how to set up a Scheduler and Workers
+
+    >>> from dask.distributed import Scheduler, Worker, Nanny
+    >>> scheduler = {'cls': Scheduler, 'options': {"dashboard_address": ':8787'}}
+    >>> workers = {
+    ...     'my-worker': {"cls": Worker, "options": {"ncores": 1}},
+    ...     'my-nanny': {"cls": Nanny, "options": {"ncores": 2}},
+    ... }
+    >>> cluster = SpecCluster(scheduler=scheduler, workers=workers)
+
+    The worker spec is stored as the ``.worker_spec`` attribute
+
+    >>> cluster.worker_spec
+    {
+       'my-worker': {"cls": Worker, "options": {"ncores": 1}},
+       'my-nanny': {"cls": Nanny, "options": {"ncores": 2}},
+    }
+
+    While the instantiation of this spec is stored in the ``.workers``
+    attribute
+
+    >>> cluster.workers
+    {
+        'my-worker': <Worker ...>
+        'my-nanny': <Nanny ...>
+    }
+
+    Should the spec change, we can await the cluster or call the
+    ``._correct_state`` method to align the actual state to the specified
+    state.
+
+    We can also ``.scale(...)`` the cluster, which adds new workers of a given
+    form.
+
+    >>> worker = {'cls': Worker, 'options': {}}
+    >>> cluster = SpecCluster(scheduler=scheduler, worker=worker)
+    >>> cluster.worker_spec
+    {}
+
+    >>> cluster.scale(3)
+    >>> cluster.worker_spec
+    {
+        0: {'cls': Worker, 'options': {}},
+        1: {'cls': Worker, 'options': {}},
+        2: {'cls': Worker, 'options': {}},
+    }
+
+    Note that above we are using the standard ``Worker`` and ``Nanny`` classes,
+    however in practice other classes could be used that handle resource
+    management like ``KubernetesPod`` or ``SLURMJob``.  The spec does not need
+    to conform to the expectations of the standard Dask Worker class.  It just
+    needs to be called with the provided options, support ``__await__`` and
+    ``close`` methods and the ``worker_address`` property..
+
+    Also note that uniformity of the specification is not required.  Other API
+    could be added externally (in subclasses) that adds workers of different
+    specifications into the same dictionary.
+    """
+
+    def __init__(
+        self,
+        workers=None,
+        scheduler=None,
+        worker=None,
+        asynchronous=False,
+        loop=None,
+        silence_logs=False,
+    ):
+        self._created = weakref.WeakSet()
+        if scheduler is None:
+            try:
+                from distributed.bokeh.scheduler import BokehScheduler
+            except ImportError:
+                services = {}
+            else:
+                services = {("bokeh", 8787): BokehScheduler}
+            scheduler = {"cls": Scheduler, "options": {"services": services}}
+
+        self.scheduler_spec = scheduler
+        self.worker_spec = workers or {}
+        self.new_spec = worker
+        self.workers = {}
+        self._i = 0
+        self._asynchronous = asynchronous
+
+        if silence_logs:
+            self._old_logging_level = silence_logging(level=silence_logs)
+
+        self._loop_runner = LoopRunner(loop=loop, asynchronous=asynchronous)
+        self.loop = self._loop_runner.loop
+
+        self.scheduler = self.scheduler_spec["cls"](
+            loop=self.loop, **self.scheduler_spec["options"]
+        )
+        self.status = "created"
+        self._correct_state_waiting = None
+
+        if not self.asynchronous:
+            self._loop_runner.start()
+            self.sync(self._start)
+            self.sync(self._correct_state)
+            self.sync(self._wait_for_workers)
+
+    async def _start(self):
+        while self.status == "starting":
+            await asyncio.sleep(0.01)
+        if self.status == "running":
+            return
+        if self.status == "closed":
+            raise ValueError("Cluster is closed")
+
+        self._lock = asyncio.Lock()
+        self.status = "starting"
+        self.scheduler = await self.scheduler
+        self.status = "running"
+
+    def _correct_state(self):
+        if self._correct_state_waiting:
+            # If people call this frequently, we only want to run it once
+            return self._correct_state_waiting
+        else:
+            task = asyncio.ensure_future(self._correct_state_internal())
+            self._correct_state_waiting = task
+            return task
+
+    async def _correct_state_internal(self):
+        async with self._lock:
+            self._correct_state_waiting = None
+
+            pre = list(set(self.workers))
+            to_close = set(self.workers) - set(self.worker_spec)
+            if to_close:
+                await self.scheduler.retire_workers(workers=list(to_close))
+                tasks = [self.workers[w].close() for w in to_close]
+                await asyncio.wait(tasks)
+                for task in tasks:  # for tornado gen.coroutine support
+                    await task
+            for name in to_close:
+                del self.workers[name]
+
+            to_open = set(self.worker_spec) - set(self.workers)
+            workers = []
+            for name in to_open:
+                d = self.worker_spec[name]
+                cls, opts = d["cls"], d.get("options", {})
+                if "name" not in opts:
+                    opts = opts.copy()
+                    opts["name"] = name
+                worker = cls(self.scheduler.address, **opts)
+                self._created.add(worker)
+                workers.append(worker)
+            if workers:
+                await asyncio.wait(workers)
+                for w in workers:
+                    w._cluster = weakref.ref(self)
+                    await w  # for tornado gen.coroutine support
+            self.workers.update(dict(zip(to_open, workers)))
+
+    def __await__(self):
+        async def _():
+            if self.status == "created":
+                await self._start()
+            await self.scheduler
+            await self._correct_state()
+            if self.workers:
+                await asyncio.wait(list(self.workers.values()))  # maybe there are more
+            await self._wait_for_workers()
+            return self
+
+        return _().__await__()
+
+    async def _wait_for_workers(self):
+        # TODO: this function needs to query scheduler and worker state
+        # remotely without assuming that they are local
+        while {d["name"] for d in self.scheduler.identity()["workers"].values()} != set(
+            self.workers
+        ):
+            if (
+                any(w.status == "closed" for w in self.workers.values())
+                and self.scheduler.status == "running"
+            ):
+                raise gen.TimeoutError("Worker unexpectedly closed")
+            await asyncio.sleep(0.1)
+
+    async def __aenter__(self):
+        await self
+        return self
+
+    async def __aexit__(self, typ, value, traceback):
+        await self.close()
+
+    async def _close(self):
+        while self.status == "closing":
+            await asyncio.sleep(0.1)
+        if self.status == "closed":
+            return
+        self.status = "closing"
+
+        async with self._lock:
+            await self.scheduler.close(close_workers=True)
+        self.scale(0)
+        await self._correct_state()
+        for w in self._created:
+            assert w.status == "closed"
+
+        if hasattr(self, "_old_logging_level"):
+            silence_logging(self._old_logging_level)
+
+        self.status = "closed"
+
+    def close(self):
+        with ignoring(RuntimeError):  # loop closed during process shutdown
+            return self.sync(self._close)
+
+    def __del__(self):
+        if self.status != "closed":
+            self.close()
+
+    def __enter__(self):
+        self.sync(self._correct_state)
+        self.sync(self._wait_for_workers)
+        assert self.status == "running"
+        return self
+
+    def __exit__(self, typ, value, traceback):
+        self.close()
+        self._loop_runner.stop()
+
+    def scale(self, n):
+        while len(self.worker_spec) > n:
+            self.worker_spec.popitem()
+
+        while len(self.worker_spec) < n:
+            while self._i in self.worker_spec:
+                self._i += 1
+            self.worker_spec[self._i] = self.new_spec
+
+        self.loop.add_callback(self._correct_state)
+
+    async def scale_down(self, workers):
+        workers = set(workers)
+
+        # TODO: this is linear cost.  We should be indexing by name or something
+        to_close = [w for w in self.workers.values() if w.address in workers]
+        for k, v in self.workers.items():
+            if v.worker_address in workers:
+                del self.worker_spec[k]
+
+        await self
+
+    scale_up = scale  # backwards compatibility
+
+    def __repr__(self):
+        return "SpecCluster(%r, workers=%d)" % (
+            self.scheduler_address,
+            len(self.workers),
+        )
diff --git a/distributed/deploy/tests/py3_test_deploy.py b/distributed/deploy/tests/py3_test_deploy.py
index 4c8fb2f86de..7a66ecf942c 100644
--- a/distributed/deploy/tests/py3_test_deploy.py
+++ b/distributed/deploy/tests/py3_test_deploy.py
@@ -1,14 +1,13 @@
 from distributed import LocalCluster
 from distributed.utils_test import loop  # noqa: F401
 
+import pytest
 
-def test_async_with(loop):
-    async def f():
 
-        async with LocalCluster(processes=False, asynchronous=True) as cluster:
-            w = cluster.workers
-            assert w
+@pytest.mark.asyncio
+async def test_async_with():
+    async with LocalCluster(processes=False, asynchronous=True) as cluster:
+        w = cluster.workers
+        assert w
 
-        assert not w
-
-    loop.run_sync(f)
+    assert not w
diff --git a/distributed/deploy/tests/test_adaptive.py b/distributed/deploy/tests/test_adaptive.py
index 50c4f0a45a3..8915c721353 100644
--- a/distributed/deploy/tests/test_adaptive.py
+++ b/distributed/deploy/tests/test_adaptive.py
@@ -2,12 +2,13 @@
 
 from time import sleep
 
+import pytest
 from toolz import frequencies, pluck
 from tornado import gen
 from tornado.ioloop import IOLoop
 
-from distributed import Client, wait, Adaptive, LocalCluster
-from distributed.utils_test import gen_cluster, gen_test, slowinc, inc
+from distributed import Client, wait, Adaptive, LocalCluster, SpecCluster, Worker
+from distributed.utils_test import gen_cluster, gen_test, slowinc, inc, clean
 from distributed.utils_test import loop, nodebug  # noqa: F401
 from distributed.metrics import time
 
@@ -162,19 +163,17 @@ def scale_down(self, workers):
     assert len(s.workers) == 2
 
 
+@pytest.mark.xfail(reason="need to rework adaptive")
 @gen_test(timeout=30)
 def test_min_max():
-    loop = IOLoop.current()
     cluster = yield LocalCluster(
         0,
         scheduler_port=0,
         silence_logs=False,
         processes=False,
         dashboard_address=None,
-        loop=loop,
         asynchronous=True,
     )
-    yield cluster._start()
     try:
         adapt = Adaptive(
             cluster.scheduler,
@@ -184,7 +183,7 @@ def test_min_max():
             interval="20 ms",
             wait_count=10,
         )
-        c = yield Client(cluster, asynchronous=True, loop=loop)
+        c = yield Client(cluster, asynchronous=True)
 
         start = time()
         while not cluster.scheduler.workers:
@@ -359,17 +358,18 @@ def test_no_more_workers_than_tasks():
 
 
 def test_basic_no_loop():
-    try:
-        with LocalCluster(
-            0, scheduler_port=0, silence_logs=False, dashboard_address=None
-        ) as cluster:
-            with Client(cluster) as client:
-                cluster.adapt()
-                future = client.submit(lambda x: x + 1, 1)
-                assert future.result() == 2
-            loop = cluster.loop
-    finally:
-        loop.add_callback(loop.stop)
+    with clean(threads=False):
+        try:
+            with LocalCluster(
+                0, scheduler_port=0, silence_logs=False, dashboard_address=None
+            ) as cluster:
+                with Client(cluster) as client:
+                    cluster.adapt()
+                    future = client.submit(lambda x: x + 1, 1)
+                    assert future.result() == 2
+                loop = cluster.loop
+        finally:
+            loop.add_callback(loop.stop)
 
 
 @gen_test(timeout=None)
@@ -408,25 +408,17 @@ def test_target_duration():
 @gen_test(timeout=None)
 def test_worker_keys():
     """ Ensure that redefining adapt with a lower maximum removes workers """
-    cluster = yield LocalCluster(
-        0,
+    cluster = yield SpecCluster(
+        workers={
+            "a-1": {"cls": Worker},
+            "a-2": {"cls": Worker},
+            "b-1": {"cls": Worker},
+            "b-2": {"cls": Worker},
+        },
         asynchronous=True,
-        processes=False,
-        scheduler_port=0,
-        silence_logs=False,
-        dashboard_address=None,
     )
 
     try:
-        yield [
-            cluster.start_worker(name="a-1"),
-            cluster.start_worker(name="a-2"),
-            cluster.start_worker(name="b-1"),
-            cluster.start_worker(name="b-2"),
-        ]
-
-        while len(cluster.scheduler.workers) != 4:
-            yield gen.sleep(0.01)
 
         def key(ws):
             return ws.name.split("-")[0]
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index ed9e3bb2dbe..4498611d7e8 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -17,6 +17,7 @@
 from distributed.deploy.local import LocalCluster, nprocesses_nthreads
 from distributed.metrics import time
 from distributed.utils_test import (
+    clean,
     inc,
     gen_test,
     slowinc,
@@ -46,7 +47,7 @@ def test_simple(loop):
             x = e.submit(inc, 1)
             x.result()
             assert x.key in c.scheduler.tasks
-            assert any(w.data == {x.key: 2} for w in c.workers)
+            assert any(w.data == {x.key: 2} for w in c.workers.values())
 
             assert e.loop is c.loop
 
@@ -87,10 +88,10 @@ def test_procs():
         silence_logs=False,
     ) as c:
         assert len(c.workers) == 2
-        assert all(isinstance(w, Worker) for w in c.workers)
+        assert all(isinstance(w, Worker) for w in c.workers.values())
         with Client(c.scheduler.address) as e:
-            assert all(w.ncores == 3 for w in c.workers)
-            assert all(isinstance(w, Worker) for w in c.workers)
+            assert all(w.ncores == 3 for w in c.workers.values())
+            assert all(isinstance(w, Worker) for w in c.workers.values())
         repr(c)
 
     with LocalCluster(
@@ -102,12 +103,12 @@ def test_procs():
         silence_logs=False,
     ) as c:
         assert len(c.workers) == 2
-        assert all(isinstance(w, Nanny) for w in c.workers)
+        assert all(isinstance(w, Nanny) for w in c.workers.values())
         with Client(c.scheduler.address) as e:
             assert all(v == 3 for v in e.ncores().values())
 
-            c.start_worker()
-            assert all(isinstance(w, Nanny) for w in c.workers)
+            c.scale(3)
+            assert all(isinstance(w, Nanny) for w in c.workers.values())
         repr(c)
 
 
@@ -171,7 +172,7 @@ def test_transports_tcp_port():
 @pytest.mark.skipif("sys.version_info[0] == 2", reason="")
 class LocalTest(ClusterTest, unittest.TestCase):
     Cluster = partial(LocalCluster, silence_logs=False, dashboard_address=None)
-    kwargs = {"dashboard_address": None}
+    kwargs = {"dashboard_address": None, "processes": False}
 
 
 @pytest.mark.skipif("sys.version_info[0] == 2", reason="")
@@ -208,12 +209,13 @@ def test_duplicate_clients():
         for msg in info.list
     )
     yield c1.close()
+    yield c2.close()
 
 
 def test_Client_kwargs(loop):
     with Client(loop=loop, processes=False, n_workers=2, silence_logs=False) as c:
         assert len(c.cluster.workers) == 2
-        assert all(isinstance(w, Worker) for w in c.cluster.workers)
+        assert all(isinstance(w, Worker) for w in c.cluster.workers.values())
     assert c.cluster.status == "closed"
 
 
@@ -230,14 +232,14 @@ def test_defaults():
     with LocalCluster(
         scheduler_port=0, silence_logs=False, dashboard_address=None
     ) as c:
-        assert sum(w.ncores for w in c.workers) == _ncores
-        assert all(isinstance(w, Nanny) for w in c.workers)
+        assert sum(w.ncores for w in c.workers.values()) == _ncores
+        assert all(isinstance(w, Nanny) for w in c.workers.values())
 
     with LocalCluster(
         processes=False, scheduler_port=0, silence_logs=False, dashboard_address=None
     ) as c:
-        assert sum(w.ncores for w in c.workers) == _ncores
-        assert all(isinstance(w, Worker) for w in c.workers)
+        assert sum(w.ncores for w in c.workers.values()) == _ncores
+        assert all(isinstance(w, Worker) for w in c.workers.values())
         assert len(c.workers) == 1
 
     with LocalCluster(
@@ -248,7 +250,7 @@ def test_defaults():
         else:
             # n_workers not a divisor of _ncores => threads are overcommitted
             expected_total_threads = max(2, _ncores + 1)
-        assert sum(w.ncores for w in c.workers) == expected_total_threads
+        assert sum(w.ncores for w in c.workers.values()) == expected_total_threads
 
     with LocalCluster(
         threads_per_worker=_ncores * 2,
@@ -264,7 +266,7 @@ def test_defaults():
         silence_logs=False,
         dashboard_address=None,
     ) as c:
-        assert all(w.ncores == 1 for w in c.workers)
+        assert all(w.ncores == 1 for w in c.workers.values())
     with LocalCluster(
         threads_per_worker=2,
         n_workers=3,
@@ -273,18 +275,19 @@ def test_defaults():
         dashboard_address=None,
     ) as c:
         assert len(c.workers) == 3
-        assert all(w.ncores == 2 for w in c.workers)
+        assert all(w.ncores == 2 for w in c.workers.values())
 
 
 def test_worker_params():
     with LocalCluster(
+        processes=False,
         n_workers=2,
         scheduler_port=0,
         silence_logs=False,
         dashboard_address=None,
         memory_limit=500,
     ) as c:
-        assert [w.memory_limit for w in c.workers] == [500] * 2
+        assert [w.memory_limit for w in c.workers.values()] == [500] * 2
 
 
 def test_memory_limit_none():
@@ -302,24 +305,28 @@ def test_memory_limit_none():
 
 
 def test_cleanup():
-    c = LocalCluster(2, scheduler_port=0, silence_logs=False, dashboard_address=None)
-    port = c.scheduler.port
-    c.close()
-    c2 = LocalCluster(
-        2, scheduler_port=port, silence_logs=False, dashboard_address=None
-    )
-    c.close()
+    with clean(threads=False):
+        c = LocalCluster(
+            2, scheduler_port=0, silence_logs=False, dashboard_address=None
+        )
+        port = c.scheduler.port
+        c.close()
+        c2 = LocalCluster(
+            2, scheduler_port=port, silence_logs=False, dashboard_address=None
+        )
+        c2.close()
 
 
 def test_repeated():
-    with LocalCluster(
-        0, scheduler_port=8448, silence_logs=False, dashboard_address=None
-    ) as c:
-        pass
-    with LocalCluster(
-        0, scheduler_port=8448, silence_logs=False, dashboard_address=None
-    ) as c:
-        pass
+    with clean(threads=False):
+        with LocalCluster(
+            0, scheduler_port=8448, silence_logs=False, dashboard_address=None
+        ) as c:
+            pass
+        with LocalCluster(
+            0, scheduler_port=8448, silence_logs=False, dashboard_address=None
+        ) as c:
+            pass
 
 
 @pytest.mark.parametrize("processes", [True, False])
@@ -373,15 +380,15 @@ def test_scale_up_and_down():
 
     assert not cluster.workers
 
-    yield cluster.scale_up(2)
+    cluster.scale(2)
+    yield cluster
     assert len(cluster.workers) == 2
     assert len(cluster.scheduler.ncores) == 2
 
-    addr = cluster.workers[0].address
-    yield cluster.scale_down([addr])
+    cluster.scale(1)
+    yield cluster
 
     assert len(cluster.workers) == 1
-    assert addr not in cluster.scheduler.ncores
 
     yield c.close()
     yield cluster.close()
@@ -437,7 +444,7 @@ def test_memory(loop, n_workers):
         dashboard_address=None,
         loop=loop,
     ) as cluster:
-        assert sum(w.memory_limit for w in cluster.workers) <= TOTAL_MEMORY
+        assert sum(w.memory_limit for w in cluster.workers.values()) <= TOTAL_MEMORY
 
 
 @pytest.mark.parametrize("n_workers", [None, 3])
@@ -486,11 +493,13 @@ def test_bokeh_kwargs(loop):
 
 
 def test_io_loop_periodic_callbacks(loop):
-    with LocalCluster(loop=loop, silence_logs=False) as cluster:
+    with LocalCluster(
+        loop=loop, port=0, dashboard_address=None, silence_logs=False
+    ) as cluster:
         assert cluster.scheduler.loop is loop
         for pc in cluster.scheduler.periodic_callbacks.values():
             assert pc.io_loop is loop
-        for worker in cluster.workers:
+        for worker in cluster.workers.values():
             for pc in worker.periodic_callbacks.values():
                 assert pc.io_loop is loop
 
@@ -772,7 +781,7 @@ def test_worker_class_worker(loop):
         scheduler_port=0,
         dashboard_address=None,
     ) as cluster:
-        assert all(isinstance(w, MyWorker) for w in cluster.workers)
+        assert all(isinstance(w, MyWorker) for w in cluster.workers.values())
 
 
 def test_worker_class_nanny(loop):
@@ -786,8 +795,37 @@ class MyNanny(Nanny):
         scheduler_port=0,
         dashboard_address=None,
     ) as cluster:
-        assert all(isinstance(w, MyNanny) for w in cluster.workers)
+        assert all(isinstance(w, MyNanny) for w in cluster.workers.values())
+
+
+@pytest.mark.asyncio
+async def test_worker_class_nanny_async():
+    class MyNanny(Nanny):
+        pass
+
+    async with LocalCluster(
+        n_workers=2,
+        worker_class=MyNanny,
+        scheduler_port=0,
+        dashboard_address=None,
+        asynchronous=True,
+    ) as cluster:
+        assert all(isinstance(w, MyNanny) for w in cluster.workers.values())
 
 
 if sys.version_info >= (3, 5):
     from distributed.deploy.tests.py3_test_deploy import *  # noqa F401
+
+
+def test_starts_up_sync(loop):
+    cluster = LocalCluster(
+        n_workers=2,
+        loop=loop,
+        processes=False,
+        scheduler_port=0,
+        dashboard_address=None,
+    )
+    try:
+        assert len(cluster.scheduler.workers) == 2
+    finally:
+        cluster.close()
diff --git a/distributed/deploy/tests/test_spec_cluster.py b/distributed/deploy/tests/test_spec_cluster.py
new file mode 100644
index 00000000000..cfc12427274
--- /dev/null
+++ b/distributed/deploy/tests/test_spec_cluster.py
@@ -0,0 +1,115 @@
+from dask.distributed import SpecCluster, Worker, Client, Scheduler
+from distributed.utils_test import loop  # noqa: F401
+import pytest
+
+
+class MyWorker(Worker):
+    pass
+
+
+class BrokenWorker(Worker):
+    def __await__(self):
+        async def _():
+            raise Exception("Worker Broken")
+
+        return _().__await__()
+
+
+worker_spec = {
+    0: {"cls": Worker, "options": {"ncores": 1}},
+    1: {"cls": Worker, "options": {"ncores": 2}},
+    "my-worker": {"cls": MyWorker, "options": {"ncores": 3}},
+}
+scheduler = {"cls": Scheduler, "options": {"port": 0}}
+
+
+@pytest.mark.asyncio
+async def test_specification():
+    async with SpecCluster(
+        workers=worker_spec, scheduler=scheduler, asynchronous=True
+    ) as cluster:
+        assert cluster.worker_spec is worker_spec
+
+        assert len(cluster.workers) == 3
+        assert set(cluster.workers) == set(worker_spec)
+        assert isinstance(cluster.workers[0], Worker)
+        assert isinstance(cluster.workers[1], Worker)
+        assert isinstance(cluster.workers["my-worker"], MyWorker)
+
+        assert cluster.workers[0].ncores == 1
+        assert cluster.workers[1].ncores == 2
+        assert cluster.workers["my-worker"].ncores == 3
+
+        async with Client(cluster, asynchronous=True) as client:
+            result = await client.submit(lambda x: x + 1, 10)
+            assert result == 11
+
+        for name in cluster.workers:
+            assert cluster.workers[name].name == name
+
+
+def test_spec_sync(loop):
+    worker_spec = {
+        0: {"cls": Worker, "options": {"ncores": 1}},
+        1: {"cls": Worker, "options": {"ncores": 2}},
+        "my-worker": {"cls": MyWorker, "options": {"ncores": 3}},
+    }
+    with SpecCluster(workers=worker_spec, scheduler=scheduler, loop=loop) as cluster:
+        assert cluster.worker_spec is worker_spec
+
+        assert len(cluster.workers) == 3
+        assert set(cluster.workers) == set(worker_spec)
+        assert isinstance(cluster.workers[0], Worker)
+        assert isinstance(cluster.workers[1], Worker)
+        assert isinstance(cluster.workers["my-worker"], MyWorker)
+
+        assert cluster.workers[0].ncores == 1
+        assert cluster.workers[1].ncores == 2
+        assert cluster.workers["my-worker"].ncores == 3
+
+        with Client(cluster, loop=loop) as client:
+            assert cluster.loop is cluster.scheduler.loop
+            assert cluster.loop is client.loop
+            result = client.submit(lambda x: x + 1, 10).result()
+            assert result == 11
+
+
+def test_loop_started():
+    cluster = SpecCluster(worker_spec)
+
+
+@pytest.mark.asyncio
+async def test_scale():
+    worker = {"cls": Worker, "options": {"ncores": 1}}
+    async with SpecCluster(
+        asynchronous=True, scheduler=scheduler, worker=worker
+    ) as cluster:
+        assert not cluster.workers
+        assert not cluster.worker_spec
+
+        # Scale up
+        cluster.scale(2)
+        assert not cluster.workers
+        assert cluster.worker_spec
+
+        await cluster
+        assert len(cluster.workers) == 2
+
+        # Scale down
+        cluster.scale(1)
+        assert len(cluster.workers) == 2
+
+        await cluster
+        assert len(cluster.workers) == 1
+
+
+@pytest.mark.asyncio
+async def test_broken_worker():
+    with pytest.raises(Exception) as info:
+        async with SpecCluster(
+            asynchronous=True,
+            workers={"good": {"cls": Worker}, "bad": {"cls": BrokenWorker}},
+        ) as cluster:
+            pass
+
+    assert "Broken" in str(info.value)
diff --git a/distributed/deploy/utils_test.py b/distributed/deploy/utils_test.py
index 9bc8cacccad..9da8d64cd50 100644
--- a/distributed/deploy/utils_test.py
+++ b/distributed/deploy/utils_test.py
@@ -1,5 +1,7 @@
 from ..client import Client
 
+import pytest
+
 
 class ClusterTest(object):
     Cluster = None
@@ -13,26 +15,15 @@ def tearDown(self):
         self.client.close()
         self.cluster.close()
 
+    @pytest.mark.xfail()
     def test_cores(self):
+        info = self.client.scheduler_info()
         assert len(self.client.ncores()) == 2
 
     def test_submit(self):
         future = self.client.submit(lambda x: x + 1, 1)
         assert future.result() == 2
 
-    def test_start_worker(self):
-        a = self.client.ncores()
-        w = self.cluster.start_worker(ncores=3)
-        b = self.client.ncores()
-
-        assert len(b) == 1 + len(a)
-        assert any(v == 3 for v in b.values())
-
-        self.cluster.stop_worker(w)
-
-        c = self.client.ncores()
-        assert c == a
-
     def test_context_manager(self):
         with self.Cluster(**self.kwargs) as c:
             with Client(c) as e:
diff --git a/distributed/nanny.py b/distributed/nanny.py
index 842ec765d7f..a27f713ea6b 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -18,7 +18,7 @@
 
 from .comm import get_address_host, get_local_address_for, unparse_host_port
 from .comm.addressing import address_from_user_args
-from .core import rpc, RPCClosed, CommClosedError, coerce_to_address
+from .core import RPCClosed, CommClosedError, coerce_to_address
 from .metrics import time
 from .node import ServerNode
 from .process import AsyncProcess
@@ -30,6 +30,7 @@
     silence_logging,
     json_load_robust,
     PeriodicCallback,
+    parse_timedelta,
 )
 from .worker import _ncores, run, parse_memory_limit, Worker
 
@@ -78,6 +79,11 @@ def __init__(
         protocol=None,
         **worker_kwargs
     ):
+        self.loop = loop or IOLoop.current()
+        self.security = security or Security()
+        assert isinstance(self.security, Security)
+        self.connection_args = self.security.get_connection_args("worker")
+        self.listen_args = self.security.get_listen_args("worker")
 
         if scheduler_file:
             cfg = json_load_robust(scheduler_file)
@@ -88,12 +94,13 @@ def __init__(
             self.scheduler_addr = coerce_to_address(scheduler_ip)
         else:
             self.scheduler_addr = coerce_to_address((scheduler_ip, scheduler_port))
+
         self._given_worker_port = worker_port
         self.ncores = ncores or _ncores
         self.reconnect = reconnect
         self.validate = validate
         self.resources = resources
-        self.death_timeout = death_timeout
+        self.death_timeout = parse_timedelta(death_timeout)
         self.preload = preload
         self.preload_argv = preload_argv
         self.Worker = Worker if worker_class is None else worker_class
@@ -105,15 +112,8 @@ def __init__(
             "distributed.worker.memory.terminate"
         )
 
-        self.security = security or Security()
-        assert isinstance(self.security, Security)
-        self.connection_args = self.security.get_connection_args("worker")
-        self.listen_args = self.security.get_listen_args("worker")
-
         self.local_dir = local_dir
 
-        self.loop = loop or IOLoop.current()
-        self.scheduler = rpc(self.scheduler_addr, connection_args=self.connection_args)
         self.services = services
         self.name = name
         self.quiet = quiet
@@ -135,9 +135,11 @@ def __init__(
         }
 
         super(Nanny, self).__init__(
-            handlers, io_loop=self.loop, connection_args=self.connection_args
+            handlers=handlers, io_loop=self.loop, connection_args=self.connection_args
         )
 
+        self.scheduler = self.rpc(self.scheduler_addr)
+
         if self.memory_limit:
             pc = PeriodicCallback(self.memory_monitor, 100, io_loop=self.loop)
             self.periodic_callbacks["memory"] = pc
@@ -240,7 +242,6 @@ def kill(self, comm=None, timeout=2):
 
         deadline = self.loop.time() + timeout
         yield self.process.kill(timeout=0.8 * (deadline - self.loop.time()))
-        yield self._unregister(deadline - self.loop.time())
 
     @gen.coroutine
     def instantiate(self, comm=None):
@@ -376,8 +377,12 @@ def close(self, comm=None, timeout=5, report=None):
         """
         Close the worker process, stop all comms.
         """
-        if self.status in ("closing", "closed"):
+        while self.status == "closing":
+            yield gen.sleep(0.01)
+
+        if self.status == "closed":
             raise gen.Return("OK")
+
         self.status = "closing"
         logger.info("Closing Nanny at %r", self.address)
         self.stop()
@@ -388,9 +393,10 @@ def close(self, comm=None, timeout=5, report=None):
             pass
         self.process = None
         self.rpc.close()
-        self.scheduler.close_rpc()
         self.status = "closed"
-        raise gen.Return("OK")
+        if comm:
+            yield comm.write("OK")
+        yield ServerNode.close(self)
 
 
 class WorkerProcess(object):
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 57d768f95f6..991ff1a2108 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1365,7 +1365,10 @@ def heartbeat_worker(
 
         self.host_info[host]["last-seen"] = local_now
         frac = 1 / 20 / len(self.workers)
-        self.bandwidth = self.bandwidth * (1 - frac) + metrics["bandwidth"] * frac
+        try:
+            self.bandwidth = self.bandwidth * (1 - frac) + metrics["bandwidth"] * frac
+        except KeyError:
+            pass
 
         ws = self.workers.get(address)
         if not ws:
@@ -1990,7 +1993,10 @@ def cancel_key(self, key, client, retries=5, force=False):
         """ Cancel a particular key and all dependents """
         # TODO: this should be converted to use the transition mechanism
         ts = self.tasks.get(key)
-        cs = self.clients[client]
+        try:
+            cs = self.clients[client]
+        except KeyError:
+            return
         if ts is None or not ts.who_wants:  # no key yet, lets try again in a moment
             if retries:
                 self.loop.add_future(
@@ -3085,7 +3091,7 @@ def retire_workers(
                     except KeyError:  # keys left during replicate
                         pass
 
-            workers = {self.workers[w] for w in workers}
+            workers = {self.workers[w] for w in workers if w in self.workers}
             if len(workers) > 0:
                 # Keys orphaned by retiring those workers
                 keys = set.union(*[w.has_what for w in workers])
diff --git a/distributed/tests/test_as_completed.py b/distributed/tests/test_as_completed.py
index 8e66b58dd4e..a584025ad03 100644
--- a/distributed/tests/test_as_completed.py
+++ b/distributed/tests/test_as_completed.py
@@ -1,4 +1,4 @@
-from concurrent.futures._base import CancelledError
+from concurrent.futures import CancelledError
 from operator import add
 import random
 from time import sleep
@@ -226,7 +226,7 @@ def test_as_completed_with_results_no_raise(client):
     assert y.status == "cancelled"
     assert z.status == "finished"
 
-    assert isinstance(dd[y][0], CancelledError)
+    assert isinstance(dd[y][0], CancelledError) or dd[y][0] == 6
     assert isinstance(dd[x][0][1], RuntimeError)
     assert dd[z][0] == 2
 
diff --git a/distributed/tests/test_asyncprocess.py b/distributed/tests/test_asyncprocess.py
index 1e7a5d2804f..3cb3eee14d4 100644
--- a/distributed/tests/test_asyncprocess.py
+++ b/distributed/tests/test_asyncprocess.py
@@ -49,6 +49,7 @@ def threads_info(q):
     q.put(threading.current_thread().name)
 
 
+@pytest.mark.xfail(reason="Intermittent failure")
 @nodebug
 @gen_test()
 def test_simple():
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 4cd196fa2f4..28c2f939eb7 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -22,7 +22,6 @@
 import pytest
 from toolz import identity, isdistinct, concat, pluck, valmap, partial, first, merge
 from tornado import gen
-from tornado.ioloop import IOLoop
 
 import dask
 from dask import delayed
@@ -3321,7 +3320,12 @@ def test_get_foo_lost_keys(c, s, u, v, w):
 
 
 @pytest.mark.slow
-@gen_cluster(client=True, Worker=Nanny, check_new_threads=False)
+@gen_cluster(
+    client=True,
+    Worker=Nanny,
+    check_new_threads=False,
+    worker_kwargs={"death_timeout": "500ms"},
+)
 def test_bad_tasks_fail(c, s, a, b):
     f = c.submit(sys.exit, 1)
     with pytest.raises(KilledWorker) as info:
@@ -3575,24 +3579,29 @@ def test_reconnect_timeout(c, s):
 @pytest.mark.skipif(
     sys.version_info[0] == 2, reason="Semaphore.acquire doesn't support timeout option"
 )
-@pytest.mark.xfail(reason="TODO: intermittent failures")
+# @pytest.mark.xfail(reason="TODO: intermittent failures")
 @pytest.mark.parametrize("worker,count,repeat", [(Worker, 100, 5), (Nanny, 10, 20)])
 def test_open_close_many_workers(loop, worker, count, repeat):
     psutil = pytest.importorskip("psutil")
     proc = psutil.Process()
 
-    with cluster(nworkers=0, active_rpc_timeout=20) as (s, _):
+    with cluster(nworkers=0, active_rpc_timeout=2) as (s, _):
         gc.collect()
         before = proc.num_fds()
         done = Semaphore(0)
         running = weakref.WeakKeyDictionary()
+        workers = set()
+        status = True
 
         @gen.coroutine
         def start_worker(sleep, duration, repeat=1):
             for i in range(repeat):
                 yield gen.sleep(sleep)
+                if not status:
+                    return
                 w = worker(s["address"], loop=loop)
                 running[w] = None
+                workers.add(w)
                 yield w
                 addr = w.worker_address
                 running[w] = addr
@@ -3621,6 +3630,12 @@ def start_worker(sleep, duration, repeat=1):
                 sleep(0.2)
                 assert time() < start + 10
 
+            status = False
+
+            [c.sync(w.close) for w in list(workers)]
+            for w in workers:
+                assert w.status == "closed"
+
     start = time()
     while proc.num_fds() > before:
         print("fds:", before, proc.num_fds())
@@ -4232,23 +4247,23 @@ def test_scatter_dict_workers(c, s, a, b):
 @pytest.mark.slow
 @gen_test()
 def test_client_timeout():
-    loop = IOLoop.current()
     c = Client("127.0.0.1:57484", asynchronous=True)
 
-    s = Scheduler(loop=loop)
+    s = Scheduler(loop=c.loop, port=57484)
     yield gen.sleep(4)
     try:
-        s.start(("127.0.0.1", 57484))
+        yield s
     except EnvironmentError:  # port in use
+        yield c.close()
         return
 
     start = time()
-    while not c.scheduler_comm:
-        yield gen.sleep(0.1)
+    yield c
+    try:
         assert time() < start + 2
-
-    yield c.close()
-    yield s.close()
+    finally:
+        yield c.close()
+        yield s.close()
 
 
 @gen_cluster(client=True)
diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
index 4c18b5242a3..be0a05afc20 100644
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -110,10 +110,6 @@ def test_nanny_process_failure(c, s):
     s.stop()
 
 
-def test_nanny_no_port():
-    _ = str(Nanny("127.0.0.1", 8786))
-
-
 @gen_cluster(ncores=[])
 def test_run(s):
     pytest.importorskip("psutil")
@@ -319,12 +315,13 @@ def test_scheduler_address_config(c, s):
 
 
 @pytest.mark.slow
-@gen_test()
+@gen_test(timeout=20)
 def test_wait_for_scheduler():
     with captured_logger("distributed") as log:
         w = Nanny("127.0.0.1:44737")
-        w._start()
+        w.start()
         yield gen.sleep(6)
+        yield w.close()
 
     log = log.getvalue()
     assert "error" not in log.lower(), log
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 9f61e5e710e..f5ce276b8d2 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -822,11 +822,12 @@ def test_file_descriptors(c, s):
     assert num_fds_6 < num_fds_5 + N
 
     yield [n.close() for n in nannies]
+    yield c.close()
 
     assert not s.rpc.open
-    assert not any(
-        occ for addr, occ in c.rpc.occupied.items() if occ != s.address
-    ), list(c.rpc._created)
+    for addr, occ in c.rpc.occupied.items():
+        for comm in occ:
+            assert comm.closed() or comm.peer_address != s.address, comm
     assert not s.stream_comms
 
     start = time()
@@ -1141,7 +1142,8 @@ def test_scheduler_file():
         assert data["address"] == s.address
 
         c = yield Client(scheduler_file=fn, loop=s.loop, asynchronous=True)
-    yield s.close()
+        yield c.close()
+        yield s.close()
 
 
 @pytest.mark.xfail(reason="")
@@ -1555,7 +1557,7 @@ def test_close_workers(s, a, b):
 )
 @gen_test()
 def test_host_address():
-    s = yield Scheduler(host="127.0.0.2")
+    s = yield Scheduler(host="127.0.0.2", port=0)
     assert "127.0.0.2" in s.address
     yield s.close()
 
@@ -1563,10 +1565,10 @@ def test_host_address():
 @gen_test()
 def test_dashboard_address():
     pytest.importorskip("bokeh")
-    s = yield Scheduler(dashboard_address="127.0.0.1:8901")
+    s = yield Scheduler(dashboard_address="127.0.0.1:8901", port=0)
     assert s.services["bokeh"].port == 8901
     yield s.close()
 
-    s = yield Scheduler(dashboard_address="127.0.0.1")
+    s = yield Scheduler(dashboard_address="127.0.0.1", port=0)
     assert s.services["bokeh"].port
     yield s.close()
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 9fc967eef5a..bf4e483f441 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -1,6 +1,7 @@
 from __future__ import print_function, division, absolute_import
 
 from concurrent.futures import ThreadPoolExecutor
+from datetime import timedelta
 import logging
 from numbers import Number
 from operator import add
@@ -312,15 +313,20 @@ def test_worker_with_port_zero():
 
 
 @pytest.mark.slow
-def test_worker_waits_for_center_to_come_up(loop):
+def test_worker_waits_for_scheduler(loop):
     @gen.coroutine
     def f():
-        w = yield Worker("127.0.0.1", 8007)
+        w = Worker("127.0.0.1", 8007)
+        try:
+            yield gen.with_timeout(timedelta(seconds=3), w)
+        except TimeoutError:
+            pass
+        else:
+            assert False
+        assert w.status not in ("closed", "running")
+        yield w.close(timeout=0.1)
 
-    try:
-        loop.run_sync(f, timeout=4)
-    except TimeoutError:
-        pass
+    loop.run_sync(f)
 
 
 @gen_cluster(client=True, ncores=[("127.0.0.1", 1)])
@@ -355,12 +361,13 @@ def test_gather(s, a, b):
         assert a.data["y"] == b.data["y"]
 
 
-def test_io_loop(loop):
-    s = Scheduler(loop=loop)
-    s.listen(0)
-    assert s.io_loop is loop
-    w = Worker(s.address, loop=loop)
-    assert w.io_loop is loop
+@pytest.mark.asyncio
+async def test_io_loop():
+    s = await Scheduler(port=0)
+    w = await Worker(s.address, loop=s.loop)
+    assert w.io_loop is s.loop
+    await s.close()
+    await w.close()
 
 
 @gen_cluster(client=True, ncores=[])
diff --git a/distributed/tests/test_worker_client.py b/distributed/tests/test_worker_client.py
index 2d4632b0b54..9c4616e9d26 100644
--- a/distributed/tests/test_worker_client.py
+++ b/distributed/tests/test_worker_client.py
@@ -202,7 +202,9 @@ def f(x):
     b = db.from_sequence([1, 2])
     b2 = b.map(f)
 
-    with Client(loop=loop, processes=False, set_as_default=True) as c:
+    with Client(
+        loop=loop, processes=False, set_as_default=True, dashboard_address=None
+    ) as c:
         assert dask.base.get_scheduler() == c.get
         for i in range(2):
             b2.compute()
diff --git a/distributed/utils.py b/distributed/utils.py
index 466a96fdfd5..55508a4c574 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -1242,6 +1242,8 @@ def parse_timedelta(s, default="seconds"):
     >>> parse_timedelta(timedelta(seconds=3))  # also supports timedeltas
     3
     """
+    if s is None:
+        return None
     if isinstance(s, timedelta):
         return s.total_seconds()
     if isinstance(s, Number):
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index c44f4177472..d61046f2a48 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -116,40 +116,35 @@ def cleanup_global_workers():
 
 @pytest.fixture
 def loop():
-    Worker._instances.clear()
-    _global_clients.clear()
-    with pristine_loop() as loop:
-        # Monkey-patch IOLoop.start to wait for loop stop
-        orig_start = loop.start
-        is_stopped = threading.Event()
-        is_stopped.set()
+    with check_instances():
+        with pristine_loop() as loop:
+            # Monkey-patch IOLoop.start to wait for loop stop
+            orig_start = loop.start
+            is_stopped = threading.Event()
+            is_stopped.set()
 
-        def start():
-            is_stopped.clear()
-            try:
-                orig_start()
-            finally:
-                is_stopped.set()
+            def start():
+                is_stopped.clear()
+                try:
+                    orig_start()
+                finally:
+                    is_stopped.set()
 
-        loop.start = start
+            loop.start = start
 
-        yield loop
+            yield loop
 
-        # Stop the loop in case it's still running
-        try:
-            sync(loop, cleanup_global_workers, callback_timeout=0.500)
-            loop.add_callback(loop.stop)
-        except RuntimeError as e:
-            if not re.match("IOLoop is clos(ed|ing)", str(e)):
-                raise
-        except gen.TimeoutError:
-            pass
-        else:
-            is_stopped.wait()
-    Worker._instances.clear()
-
-    _cleanup_dangling()
-    _global_clients.clear()
+            # Stop the loop in case it's still running
+            try:
+                sync(loop, cleanup_global_workers, callback_timeout=0.500)
+                loop.add_callback(loop.stop)
+            except RuntimeError as e:
+                if not re.match("IOLoop is clos(ed|ing)", str(e)):
+                    raise
+            except gen.TimeoutError:
+                pass
+            else:
+                is_stopped.wait()
 
 
 @pytest.fixture
@@ -464,13 +459,13 @@ def background_read():
         raise gen.Return(msg)
 
 
-def run_scheduler(q, nputs, **kwargs):
+def run_scheduler(q, nputs, port=0, **kwargs):
     from distributed import Scheduler
 
     # On Python 2.7 and Unix, fork() is used to spawn child processes,
     # so avoid inheriting the parent's IO loop.
     with pristine_loop() as loop:
-        scheduler = Scheduler(validate=True, host="127.0.0.1", **kwargs)
+        scheduler = Scheduler(validate=True, host="127.0.0.1", port=port, **kwargs)
         done = scheduler.start()
 
         for i in range(nputs):
@@ -735,9 +730,9 @@ def cluster(
             client.close()
 
     start = time()
-    while list(ws):
-        sleep(0.01)
-        assert time() < start + 1, "Workers still around after one second"
+    while len(ws):
+        sleep(0.1)
+        assert time() < start + 3, ("Workers still around after two seconds", list(ws))
 
 
 @gen.coroutine
@@ -769,15 +764,12 @@ def test_foo():
 
     def _(func):
         def test_func():
-            with pristine_loop() as loop:
+            with clean() as loop:
                 if iscoroutinefunction(func):
                     cor = func
                 else:
                     cor = gen.coroutine(func)
-                try:
-                    loop.run_sync(cor, timeout=timeout)
-                finally:
-                    loop.stop()
+                loop.run_sync(cor, timeout=timeout)
 
         return test_func
 
@@ -798,7 +790,9 @@ def start_cluster(
     scheduler_kwargs={},
     worker_kwargs={},
 ):
-    s = Scheduler(loop=loop, validate=True, security=security, **scheduler_kwargs)
+    s = Scheduler(
+        loop=loop, validate=True, security=security, port=0, **scheduler_kwargs
+    )
     done = s.start(scheduler_addr)
     workers = [
         Worker(
@@ -1483,6 +1477,9 @@ def check_instances():
     Client._instances.clear()
     Worker._instances.clear()
     Scheduler._instances.clear()
+    # assert all(n.status == "closed" for n in Nanny._instances), {
+    #     n: n.status for n in Nanny._instances
+    # }
     Nanny._instances.clear()
     _global_clients.clear()
     Comm._instances.clear()
@@ -1513,6 +1510,11 @@ def check_instances():
         print("Unclosed Comms", L)
         # raise ValueError("Unclosed Comms", L)
 
+    assert all(n.status == "closed" or n.status == "init" for n in Nanny._instances), {
+        n: n.status for n in Nanny._instances
+    }
+
+    Nanny._instances.clear()
     DequeHandler.clear_all_instances()
 
 
diff --git a/distributed/worker.py b/distributed/worker.py
index 3fcc477bf48..667bd83490f 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -436,7 +436,7 @@ def __init__(
         self.ncores = ncores or _ncores
         self.total_resources = resources or {}
         self.available_resources = (resources or {}).copy()
-        self.death_timeout = death_timeout
+        self.death_timeout = parse_timedelta(death_timeout)
         self.preload = preload
         if self.preload is None:
             self.preload = dask.config.get("distributed.worker.preload")
@@ -933,7 +933,8 @@ def _start(self, addr_or_port=0):
         if "://" in listen_host:
             protocol, listen_host = listen_host.split("://")
 
-        self.name = self.name or self.address
+        if self.name is None:
+            self.name = self.address
         preload_modules(
             self.preload,
             parameter=self,
@@ -976,7 +977,15 @@ def _start(self, addr_or_port=0):
         raise gen.Return(self)
 
     def __await__(self):
-        return self._start().__await__()
+        if self.status is not None:
+
+            @gen.coroutine  # idempotent
+            def _():
+                raise gen.Return(self)
+
+            return _().__await__()
+        else:
+            return self._start().__await__()
 
     def start(self, port=0):
         self.loop.add_callback(self._start, port)

From 6339d81e8de97b551c8cc908308c19ab89037df2 Mon Sep 17 00:00:00 2001
From: Matt Nicolls <2540582+nicolls1@users.noreply.github.com>
Date: Fri, 24 May 2019 14:36:05 -0500
Subject: [PATCH 0305/1550] Add SchedulerPlugin TaskState example (#2622)

---
 docs/source/plugins.rst | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/docs/source/plugins.rst b/docs/source/plugins.rst
index edf64362c56..b5f52f8843e 100644
--- a/docs/source/plugins.rst
+++ b/docs/source/plugins.rst
@@ -49,3 +49,27 @@ for more information on RabbitMQ and how to consume the messages.
        scheduler.add_plugin(plugin)
 
 Run with: ``dask-scheduler --preload <filename.py>``
+
+Accessing Full Task State
+-------------------------
+
+If you would like to access the full :class:`distributed.scheduler.TaskState`
+stored in the scheduler you can do this by passing and storing a reference to
+the scheduler as so:
+
+.. code-block:: python
+
+   from distributed.diagnostics.plugin import SchedulerPlugin
+
+   class MyPlugin(SchedulerPlugin):
+       def __init__(self, scheduler):
+            self.scheduler = scheduler
+
+       def transition(self, key, start, finish, *args, **kwargs):
+            # Get full TaskState
+            ts = self.scheduler.tasks[key]
+
+   @click.command()
+   def dask_setup(scheduler):
+       plugin = MyPlugin(scheduler)
+       scheduler.add_plugin(plugin)

From a818711f97fdf501823c35246b16f24fddd4035a Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 27 May 2019 14:47:10 -0500
Subject: [PATCH 0306/1550] Close clusters at exit (#2730)

---
 distributed/deploy/spec.py                    | 15 +++++++++++++--
 distributed/deploy/tests/test_spec_cluster.py | 11 +++++++++++
 2 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index 9a4385e5054..ad0aea25f6c 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -1,4 +1,5 @@
 import asyncio
+import atexit
 import weakref
 
 from tornado import gen
@@ -97,6 +98,8 @@ class does handle all of the logic around asynchronously cleanly setting up
     specifications into the same dictionary.
     """
 
+    _instances = weakref.WeakSet()
+
     def __init__(
         self,
         workers=None,
@@ -133,6 +136,7 @@ def __init__(
             loop=self.loop, **self.scheduler_spec["options"]
         )
         self.status = "created"
+        self._instances.add(self)
         self._correct_state_waiting = None
 
         if not self.asynchronous:
@@ -248,9 +252,9 @@ async def _close(self):
 
         self.status = "closed"
 
-    def close(self):
+    def close(self, timeout=None):
         with ignoring(RuntimeError):  # loop closed during process shutdown
-            return self.sync(self._close)
+            return self.sync(self._close, callback_timeout=timeout)
 
     def __del__(self):
         if self.status != "closed":
@@ -295,3 +299,10 @@ def __repr__(self):
             self.scheduler_address,
             len(self.workers),
         )
+
+
+@atexit.register
+def close_clusters():
+    for cluster in list(SpecCluster._instances):
+        with ignoring(gen.TimeoutError):
+            cluster.close(timeout=10)
diff --git a/distributed/deploy/tests/test_spec_cluster.py b/distributed/deploy/tests/test_spec_cluster.py
index cfc12427274..ac5706afe1c 100644
--- a/distributed/deploy/tests/test_spec_cluster.py
+++ b/distributed/deploy/tests/test_spec_cluster.py
@@ -1,4 +1,5 @@
 from dask.distributed import SpecCluster, Worker, Client, Scheduler
+from distributed.deploy.spec import close_clusters
 from distributed.utils_test import loop  # noqa: F401
 import pytest
 
@@ -113,3 +114,13 @@ async def test_broken_worker():
             pass
 
     assert "Broken" in str(info.value)
+
+
+@pytest.mark.slow
+def test_spec_close_clusters(loop):
+    workers = {0: {"cls": Worker}}
+    scheduler = {"cls": Scheduler, "options": {"port": 0}}
+    cluster = SpecCluster(workers=workers, scheduler=scheduler, loop=loop)
+    assert cluster in SpecCluster._instances
+    close_clusters()
+    assert cluster.status == "closed"

From d202e6253ed8ddc7919d0d4f128d88954e9859b8 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 28 May 2019 12:25:44 -0500
Subject: [PATCH 0307/1550] Move bokeh module to dashboard (#2724)

---
 distributed/bokeh.py                          |  1 +
 distributed/bokeh/__init__.py                 | 37 ----------
 distributed/cli/dask_mpi.py                   |  8 +--
 distributed/cli/dask_scheduler.py             | 23 +++---
 distributed/cli/dask_worker.py                | 18 ++---
 distributed/cli/tests/test_dask_scheduler.py  | 46 +++++-------
 distributed/cli/tests/test_dask_worker.py     | 56 +++++++++------
 distributed/cli/tests/test_tls_cli.py         | 17 ++---
 distributed/client.py                         |  6 +-
 distributed/dashboard/__init__.py             |  2 +
 .../{bokeh => dashboard}/components.py        |  0
 distributed/{bokeh => dashboard}/core.py      |  0
 .../{bokeh => dashboard}/export_tool.coffee   |  0
 .../{bokeh => dashboard}/export_tool.js       |  0
 .../{bokeh => dashboard}/export_tool.py       |  0
 distributed/{bokeh => dashboard}/proxy.py     |  2 +-
 distributed/{bokeh => dashboard}/scheduler.py |  4 +-
 .../{bokeh => dashboard}/scheduler_html.py    |  2 +-
 .../{bokeh => dashboard}/static/css/base.css  |  0
 .../static/css/status.css                     |  0
 .../static/css/system.css                     |  0
 .../static/images/dask-logo.svg               |  0
 .../static/images/fa-bars.svg                 |  0
 .../{bokeh => dashboard}/templates/base.html  |  0
 .../templates/call-stack.html                 |  0
 .../templates/json-index.html                 |  0
 .../{bokeh => dashboard}/templates/logs.html  |  0
 .../{bokeh => dashboard}/templates/main.html  |  0
 .../templates/simple.html                     |  0
 .../templates/status.html                     |  0
 .../templates/system.html                     |  0
 .../{bokeh => dashboard}/templates/task.html  |  0
 .../templates/worker-table.html               |  0
 .../templates/worker.html                     |  0
 .../templates/workers.html                    |  0
 distributed/dashboard/tests/test_bokeh.py     |  5 ++
 .../tests/test_components.py                  |  4 +-
 .../tests/test_scheduler_bokeh.py             | 30 ++++----
 .../tests/test_scheduler_bokeh_html.py        | 29 ++++----
 .../tests/test_worker_bokeh.py                | 24 ++++---
 .../tests/test_worker_bokeh_html.py           | 10 +--
 distributed/{bokeh => dashboard}/theme.yaml   |  0
 distributed/{bokeh => dashboard}/utils.py     |  0
 distributed/{bokeh => dashboard}/worker.py    |  0
 .../{bokeh => dashboard}/worker_html.py       |  0
 distributed/deploy/cluster.py                 |  4 +-
 distributed/deploy/spec.py                    |  4 +-
 distributed/deploy/tests/test_local.py        | 10 +--
 .../diagnostics/tests/test_eventstream.py     |  8 ++-
 distributed/scheduler.py                      |  6 +-
 distributed/tests/test_client.py              | 11 ++-
 distributed/tests/test_core.py                |  5 +-
 distributed/tests/test_scheduler.py           | 14 ++--
 distributed/tests/test_worker.py              | 22 +++---
 distributed/worker.py                         |  6 +-
 setup.py                                      | 70 ++++++++++---------
 56 files changed, 236 insertions(+), 248 deletions(-)
 create mode 100644 distributed/bokeh.py
 delete mode 100644 distributed/bokeh/__init__.py
 create mode 100644 distributed/dashboard/__init__.py
 rename distributed/{bokeh => dashboard}/components.py (100%)
 rename distributed/{bokeh => dashboard}/core.py (100%)
 rename distributed/{bokeh => dashboard}/export_tool.coffee (100%)
 rename distributed/{bokeh => dashboard}/export_tool.js (100%)
 rename distributed/{bokeh => dashboard}/export_tool.py (100%)
 rename distributed/{bokeh => dashboard}/proxy.py (98%)
 rename distributed/{bokeh => dashboard}/scheduler.py (99%)
 rename distributed/{bokeh => dashboard}/scheduler_html.py (99%)
 rename distributed/{bokeh => dashboard}/static/css/base.css (100%)
 rename distributed/{bokeh => dashboard}/static/css/status.css (100%)
 rename distributed/{bokeh => dashboard}/static/css/system.css (100%)
 rename distributed/{bokeh => dashboard}/static/images/dask-logo.svg (100%)
 rename distributed/{bokeh => dashboard}/static/images/fa-bars.svg (100%)
 rename distributed/{bokeh => dashboard}/templates/base.html (100%)
 rename distributed/{bokeh => dashboard}/templates/call-stack.html (100%)
 rename distributed/{bokeh => dashboard}/templates/json-index.html (100%)
 rename distributed/{bokeh => dashboard}/templates/logs.html (100%)
 rename distributed/{bokeh => dashboard}/templates/main.html (100%)
 rename distributed/{bokeh => dashboard}/templates/simple.html (100%)
 rename distributed/{bokeh => dashboard}/templates/status.html (100%)
 rename distributed/{bokeh => dashboard}/templates/system.html (100%)
 rename distributed/{bokeh => dashboard}/templates/task.html (100%)
 rename distributed/{bokeh => dashboard}/templates/worker-table.html (100%)
 rename distributed/{bokeh => dashboard}/templates/worker.html (100%)
 rename distributed/{bokeh => dashboard}/templates/workers.html (100%)
 create mode 100644 distributed/dashboard/tests/test_bokeh.py
 rename distributed/{bokeh => dashboard}/tests/test_components.py (92%)
 rename distributed/{bokeh => dashboard}/tests/test_scheduler_bokeh.py (95%)
 rename distributed/{bokeh => dashboard}/tests/test_scheduler_bokeh_html.py (79%)
 rename distributed/{bokeh => dashboard}/tests/test_worker_bokeh.py (79%)
 rename distributed/{bokeh => dashboard}/tests/test_worker_bokeh_html.py (75%)
 rename distributed/{bokeh => dashboard}/theme.yaml (100%)
 rename distributed/{bokeh => dashboard}/utils.py (100%)
 rename distributed/{bokeh => dashboard}/worker.py (100%)
 rename distributed/{bokeh => dashboard}/worker_html.py (100%)

diff --git a/distributed/bokeh.py b/distributed/bokeh.py
new file mode 100644
index 00000000000..e27bdffa33e
--- /dev/null
+++ b/distributed/bokeh.py
@@ -0,0 +1 @@
+raise ImportError("The distributed.bokeh module has moved to distributed.dashboard")
diff --git a/distributed/bokeh/__init__.py b/distributed/bokeh/__init__.py
deleted file mode 100644
index 24e082fa8e0..00000000000
--- a/distributed/bokeh/__init__.py
+++ /dev/null
@@ -1,37 +0,0 @@
-from collections import deque
-
-from ..metrics import time
-
-n = 60
-m = 100000
-
-messages = {
-    "workers": {
-        "interval": 1000,
-        "deque": deque(maxlen=n),
-        "times": deque(maxlen=n),
-        "index": deque(maxlen=n),
-        "plot-data": {
-            "time": deque(maxlen=n),
-            "cpu": deque(maxlen=n),
-            "memory_percent": deque(maxlen=n),
-            "network-send": deque(maxlen=n),
-            "network-recv": deque(maxlen=n),
-        },
-    },
-    "tasks": {"interval": 150, "deque": deque(maxlen=100), "times": deque(maxlen=100)},
-    "progress": {},
-    "processing": {"processing": {}, "memory": 0, "waiting": 0},
-    "task-events": {
-        "interval": 200,
-        "deque": deque(maxlen=m),
-        "times": deque(maxlen=m),
-        "index": deque(maxlen=m),
-        "rectangles": {
-            name: deque(maxlen=m)
-            for name in "start duration key name color worker worker_thread y alpha".split()
-        },
-        "workers": dict(),
-        "last_seen": [time()],
-    },
-}
diff --git a/distributed/cli/dask_mpi.py b/distributed/cli/dask_mpi.py
index 398596508a3..c7669073f79 100644
--- a/distributed/cli/dask_mpi.py
+++ b/distributed/cli/dask_mpi.py
@@ -7,7 +7,7 @@
 from warnings import warn
 
 from distributed import Scheduler, Nanny, Worker
-from distributed.bokeh.worker import BokehWorker
+from distributed.dashboard import BokehWorker
 from distributed.cli.utils import check_python_3
 from distributed.comm.addressing import uri_from_host_port
 from distributed.utils import get_ip_interface
@@ -82,12 +82,12 @@ def main(
 
     if rank == 0 and scheduler:
         try:
-            from distributed.bokeh.scheduler import BokehScheduler
+            from distributed.dashboard import BokehScheduler
         except ImportError:
             services = {}
         else:
             services = {
-                ("bokeh", bokeh_port): partial(BokehScheduler, prefix=bokeh_prefix)
+                ("dashboard", bokeh_port): partial(BokehScheduler, prefix=bokeh_prefix)
             }
         scheduler = Scheduler(
             scheduler_file=scheduler_file, loop=loop, services=services
@@ -107,7 +107,7 @@ def main(
             name=rank if scheduler else None,
             ncores=nthreads,
             local_dir=local_directory,
-            services={("bokeh", bokeh_worker_port): BokehWorker},
+            services={("dashboard", bokeh_worker_port): BokehWorker},
             memory_limit=memory_limit,
         )
         addr = uri_from_host_port(host, None, 0)
diff --git a/distributed/cli/dask_scheduler.py b/distributed/cli/dask_scheduler.py
index 3668be684d0..1f78426f635 100755
--- a/distributed/cli/dask_scheduler.py
+++ b/distributed/cli/dask_scheduler.py
@@ -68,27 +68,23 @@
     help="Address on which to listen for diagnostics dashboard",
 )
 @click.option(
-    "--bokeh/--no-bokeh",
-    "_bokeh",
+    "--dashboard/--no-dashboard",
+    "dashboard",
     default=True,
     show_default=True,
     required=False,
-    help="Launch Bokeh Web UI",
+    help="Launch the Dashboard",
 )
 @click.option("--show/--no-show", default=False, help="Show web UI")
 @click.option(
-    "--bokeh-whitelist",
-    default=None,
-    multiple=True,
-    help="IP addresses to whitelist for bokeh.",
+    "--dashboard-prefix", type=str, default=None, help="Prefix for the dashboard app"
 )
-@click.option("--bokeh-prefix", type=str, default=None, help="Prefix for the bokeh app")
 @click.option(
     "--use-xheaders",
     type=bool,
     default=False,
     show_default=True,
-    help="User xheaders in bokeh app for ssl termination in header",
+    help="User xheaders in dashboard app for ssl termination in header",
 )
 @click.option("--pid-file", type=str, default="", help="File to write the process PID")
 @click.option(
@@ -119,9 +115,8 @@ def main(
     port,
     bokeh_port,
     show,
-    _bokeh,
-    bokeh_whitelist,
-    bokeh_prefix,
+    dashboard,
+    dashboard_prefix,
     use_xheaders,
     pid_file,
     scheduler_file,
@@ -195,8 +190,8 @@ def del_pid_file():
         host=host,
         port=port,
         interface=interface,
-        dashboard_address=dashboard_address if _bokeh else None,
-        service_kwargs={"bokeh": {"prefix": bokeh_prefix}},
+        dashboard_address=dashboard_address if dashboard else None,
+        service_kwargs={"dashboard": {"prefix": dashboard_prefix}},
     )
     scheduler.start()
     if not preload:
diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
index 439bdaf4a62..c4f83f61405 100755
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -70,12 +70,12 @@
     help="Address on which to listen for diagnostics dashboard",
 )
 @click.option(
-    "--bokeh/--no-bokeh",
-    "bokeh",
+    "--dashboard/--no-dashboard",
+    "dashboard",
     default=True,
     show_default=True,
     required=False,
-    help="Launch Bokeh Web UI",
+    help="Launch the Dashboard",
 )
 @click.option(
     "--listen-address",
@@ -163,7 +163,9 @@
     default=None,
     help="Seconds to wait for a scheduler before closing",
 )
-@click.option("--bokeh-prefix", type=str, default="", help="Prefix for the bokeh app")
+@click.option(
+    "--dashboard-prefix", type=str, default="", help="Prefix for the dashboard"
+)
 @click.option(
     "--preload",
     type=str,
@@ -190,7 +192,7 @@ def main(
     pid_file,
     reconnect,
     resources,
-    bokeh,
+    dashboard,
     bokeh_port,
     local_directory,
     scheduler_file,
@@ -198,7 +200,7 @@ def main(
     death_timeout,
     preload,
     preload_argv,
-    bokeh_prefix,
+    dashboard_prefix,
     tls_ca_file,
     tls_cert,
     tls_key,
@@ -338,8 +340,8 @@ def del_pid_file():
             interface=interface,
             host=host,
             port=port,
-            dashboard_address=dashboard_address if bokeh else None,
-            service_kwargs={"bokhe": {"prefix": bokeh_prefix}},
+            dashboard_address=dashboard_address if dashboard else None,
+            service_kwargs={"bokhe": {"prefix": dashboard_prefix}},
             name=name if nprocs == 1 or not name else name + "-" + str(i),
             **kwargs
         )
diff --git a/distributed/cli/tests/test_dask_scheduler.py b/distributed/cli/tests/test_dask_scheduler.py
index 26fe607b901..754082f35eb 100644
--- a/distributed/cli/tests/test_dask_scheduler.py
+++ b/distributed/cli/tests/test_dask_scheduler.py
@@ -26,7 +26,7 @@
 
 
 def test_defaults(loop):
-    with popen(["dask-scheduler", "--no-bokeh"]) as proc:
+    with popen(["dask-scheduler", "--no-dashboard"]) as proc:
 
         @gen.coroutine
         def f():
@@ -43,7 +43,7 @@ def f():
 
 
 def test_hostport(loop):
-    with popen(["dask-scheduler", "--no-bokeh", "--host", "127.0.0.1:8978"]):
+    with popen(["dask-scheduler", "--no-dashboard", "--host", "127.0.0.1:8978"]):
 
         @gen.coroutine
         def f():
@@ -57,18 +57,18 @@ def f():
             c.sync(f)
 
 
-def test_no_bokeh(loop):
+def test_no_dashboard(loop):
     pytest.importorskip("bokeh")
-    with popen(["dask-scheduler", "--no-bokeh"]) as proc:
+    with popen(["dask-scheduler", "--no-dashboard"]) as proc:
         with Client("127.0.0.1:%d" % Scheduler.default_port, loop=loop) as c:
             for i in range(3):
                 line = proc.stderr.readline()
-                assert b"bokeh" not in line.lower()
+                assert b"dashboard" not in line.lower()
             with pytest.raises(Exception):
                 requests.get("http://127.0.0.1:8787/status/")
 
 
-def test_bokeh(loop):
+def test_dashboard(loop):
     pytest.importorskip("bokeh")
 
     with popen(["dask-scheduler"]) as proc:
@@ -97,7 +97,7 @@ def test_bokeh(loop):
         requests.get("http://127.0.0.1:8787/status/")
 
 
-def test_bokeh_non_standard_ports(loop):
+def test_dashboard_non_standard_ports(loop):
     pytest.importorskip("bokeh")
 
     with popen(
@@ -122,20 +122,12 @@ def test_bokeh_non_standard_ports(loop):
 @pytest.mark.skipif(
     not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
 )
-def test_bokeh_whitelist(loop):
+def test_dashboard_whitelist(loop):
     pytest.importorskip("bokeh")
     with pytest.raises(Exception):
         requests.get("http://localhost:8787/status/").ok
 
-    with popen(
-        [
-            "dask-scheduler",
-            "--bokeh-whitelist",
-            "127.0.0.2:8787",
-            "--bokeh-whitelist",
-            "127.0.0.3:8787",
-        ]
-    ) as proc:
+    with popen(["dask-scheduler"]) as proc:
         with Client("127.0.0.1:%d" % Scheduler.default_port, loop=loop) as c:
             pass
 
@@ -153,9 +145,9 @@ def test_bokeh_whitelist(loop):
 
 
 def test_multiple_workers(loop):
-    with popen(["dask-scheduler", "--no-bokeh"]) as s:
-        with popen(["dask-worker", "localhost:8786", "--no-bokeh"]) as a:
-            with popen(["dask-worker", "localhost:8786", "--no-bokeh"]) as b:
+    with popen(["dask-scheduler", "--no-dashboard"]) as s:
+        with popen(["dask-worker", "localhost:8786", "--no-dashboard"]) as a:
+            with popen(["dask-worker", "localhost:8786", "--no-dashboard"]) as b:
                 with Client("127.0.0.1:%d" % Scheduler.default_port, loop=loop) as c:
                     start = time()
                     while len(c.ncores()) < 2:
@@ -180,9 +172,9 @@ def test_interface(loop):
             "Available interfaces are: %s." % (if_names,)
         )
 
-    with popen(["dask-scheduler", "--no-bokeh", "--interface", if_name]) as s:
+    with popen(["dask-scheduler", "--no-dashboard", "--interface", if_name]) as s:
         with popen(
-            ["dask-worker", "127.0.0.1:8786", "--no-bokeh", "--interface", if_name]
+            ["dask-worker", "127.0.0.1:8786", "--no-dashboard", "--interface", if_name]
         ) as a:
             with Client("tcp://127.0.0.1:%d" % Scheduler.default_port, loop=loop) as c:
                 start = time()
@@ -217,12 +209,12 @@ def check_pidfile(proc, pidfile):
             assert proc.pid == pid
 
     with tmpfile() as s:
-        with popen(["dask-scheduler", "--pid-file", s, "--no-bokeh"]) as sched:
+        with popen(["dask-scheduler", "--pid-file", s, "--no-dashboard"]) as sched:
             check_pidfile(sched, s)
 
         with tmpfile() as w:
             with popen(
-                ["dask-worker", "127.0.0.1:8786", "--pid-file", w, "--no-bokeh"]
+                ["dask-worker", "127.0.0.1:8786", "--pid-file", w, "--no-dashboard"]
             ) as worker:
                 check_pidfile(worker, w)
 
@@ -230,21 +222,21 @@ def check_pidfile(proc, pidfile):
 def test_scheduler_port_zero(loop):
     with tmpfile() as fn:
         with popen(
-            ["dask-scheduler", "--no-bokeh", "--scheduler-file", fn, "--port", "0"]
+            ["dask-scheduler", "--no-dashboard", "--scheduler-file", fn, "--port", "0"]
         ) as sched:
             with Client(scheduler_file=fn, loop=loop) as c:
                 assert c.scheduler.port
                 assert c.scheduler.port != 8786
 
 
-def test_bokeh_port_zero(loop):
+def test_dashboard_port_zero(loop):
     pytest.importorskip("bokeh")
     with tmpfile() as fn:
         with popen(["dask-scheduler", "--dashboard-address", ":0"]) as proc:
             count = 0
             while count < 1:
                 line = proc.stderr.readline()
-                if b"bokeh" in line.lower() or b"web" in line.lower():
+                if b"dashboard" in line.lower():
                     sleep(0.01)
                     count += 1
                     assert b":0" not in line
diff --git a/distributed/cli/tests/test_dask_worker.py b/distributed/cli/tests/test_dask_worker.py
index aac27061b21..fa62594a753 100644
--- a/distributed/cli/tests/test_dask_worker.py
+++ b/distributed/cli/tests/test_dask_worker.py
@@ -17,7 +17,7 @@
 
 
 def test_nanny_worker_ports(loop):
-    with popen(["dask-scheduler", "--port", "9359", "--no-bokeh"]) as sched:
+    with popen(["dask-scheduler", "--port", "9359", "--no-dashboard"]) as sched:
         with popen(
             [
                 "dask-worker",
@@ -28,7 +28,7 @@ def test_nanny_worker_ports(loop):
                 "9684",
                 "--nanny-port",
                 "5273",
-                "--no-bokeh",
+                "--no-dashboard",
             ]
         ) as worker:
             with Client("127.0.0.1:9359", loop=loop) as c:
@@ -47,9 +47,15 @@ def test_nanny_worker_ports(loop):
 
 
 def test_memory_limit(loop):
-    with popen(["dask-scheduler", "--no-bokeh"]) as sched:
+    with popen(["dask-scheduler", "--no-dashboard"]) as sched:
         with popen(
-            ["dask-worker", "127.0.0.1:8786", "--memory-limit", "2e3MB", "--no-bokeh"]
+            [
+                "dask-worker",
+                "127.0.0.1:8786",
+                "--memory-limit",
+                "2e3MB",
+                "--no-dashboard",
+            ]
         ) as worker:
             with Client("127.0.0.1:8786", loop=loop) as c:
                 while not c.ncores():
@@ -61,9 +67,9 @@ def test_memory_limit(loop):
 
 
 def test_no_nanny(loop):
-    with popen(["dask-scheduler", "--no-bokeh"]) as sched:
+    with popen(["dask-scheduler", "--no-dashboard"]) as sched:
         with popen(
-            ["dask-worker", "127.0.0.1:8786", "--no-nanny", "--no-bokeh"]
+            ["dask-worker", "127.0.0.1:8786", "--no-nanny", "--no-dashboard"]
         ) as worker:
             assert any(b"Registered" in worker.stderr.readline() for i in range(15))
 
@@ -71,7 +77,7 @@ def test_no_nanny(loop):
 @pytest.mark.slow
 @pytest.mark.parametrize("nanny", ["--nanny", "--no-nanny"])
 def test_no_reconnect(nanny, loop):
-    with popen(["dask-scheduler", "--no-bokeh"]) as sched:
+    with popen(["dask-scheduler", "--no-dashboard"]) as sched:
         wait_for_port(("127.0.0.1", 8786))
         with popen(
             [
@@ -79,7 +85,7 @@ def test_no_reconnect(nanny, loop):
                 "tcp://127.0.0.1:8786",
                 "--no-reconnect",
                 nanny,
-                "--no-bokeh",
+                "--no-dashboard",
             ]
         ) as worker:
             sleep(2)
@@ -91,12 +97,12 @@ def test_no_reconnect(nanny, loop):
 
 
 def test_resources(loop):
-    with popen(["dask-scheduler", "--no-bokeh"]) as sched:
+    with popen(["dask-scheduler", "--no-dashboard"]) as sched:
         with popen(
             [
                 "dask-worker",
                 "tcp://127.0.0.1:8786",
-                "--no-bokeh",
+                "--no-dashboard",
                 "--resources",
                 "A=1 B=2,C=3",
             ]
@@ -112,13 +118,13 @@ def test_resources(loop):
 @pytest.mark.parametrize("nanny", ["--nanny", "--no-nanny"])
 def test_local_directory(loop, nanny):
     with tmpfile() as fn:
-        with popen(["dask-scheduler", "--no-bokeh"]) as sched:
+        with popen(["dask-scheduler", "--no-dashboard"]) as sched:
             with popen(
                 [
                     "dask-worker",
                     "127.0.0.1:8786",
                     nanny,
-                    "--no-bokeh",
+                    "--no-dashboard",
                     "--local-directory",
                     fn,
                 ]
@@ -136,8 +142,12 @@ def test_local_directory(loop, nanny):
 @pytest.mark.parametrize("nanny", ["--nanny", "--no-nanny"])
 def test_scheduler_file(loop, nanny):
     with tmpfile() as fn:
-        with popen(["dask-scheduler", "--no-bokeh", "--scheduler-file", fn]) as sched:
-            with popen(["dask-worker", "--scheduler-file", fn, nanny, "--no-bokeh"]):
+        with popen(
+            ["dask-scheduler", "--no-dashboard", "--scheduler-file", fn]
+        ) as sched:
+            with popen(
+                ["dask-worker", "--scheduler-file", fn, nanny, "--no-dashboard"]
+            ):
                 with Client(scheduler_file=fn, loop=loop) as c:
                     start = time()
                     while not c.scheduler_info()["workers"]:
@@ -147,8 +157,8 @@ def test_scheduler_file(loop, nanny):
 
 def test_scheduler_address_env(loop, monkeypatch):
     monkeypatch.setenv("DASK_SCHEDULER_ADDRESS", "tcp://127.0.0.1:8786")
-    with popen(["dask-scheduler", "--no-bokeh"]) as sched:
-        with popen(["dask-worker", "--no-bokeh"]):
+    with popen(["dask-scheduler", "--no-dashboard"]) as sched:
+        with popen(["dask-worker", "--no-dashboard"]):
             with Client(os.environ["DASK_SCHEDULER_ADDRESS"], loop=loop) as c:
                 start = time()
                 while not c.scheduler_info()["workers"]:
@@ -157,7 +167,7 @@ def test_scheduler_address_env(loop, monkeypatch):
 
 
 def test_nprocs_requires_nanny(loop):
-    with popen(["dask-scheduler", "--no-bokeh"]) as sched:
+    with popen(["dask-scheduler", "--no-dashboard"]) as sched:
         with popen(
             ["dask-worker", "127.0.0.1:8786", "--nprocs=2", "--no-nanny"]
         ) as worker:
@@ -168,7 +178,7 @@ def test_nprocs_requires_nanny(loop):
 
 
 def test_nprocs_expands_name(loop):
-    with popen(["dask-scheduler", "--no-bokeh"]) as sched:
+    with popen(["dask-scheduler", "--no-dashboard"]) as sched:
         with popen(
             ["dask-worker", "127.0.0.1:8786", "--nprocs", "2", "--name", "foo"]
         ) as worker:
@@ -194,13 +204,13 @@ def test_nprocs_expands_name(loop):
     "listen_address", ["tcp://0.0.0.0:39837", "tcp://127.0.0.2:39837"]
 )
 def test_contact_listen_address(loop, nanny, listen_address):
-    with popen(["dask-scheduler", "--no-bokeh"]) as sched:
+    with popen(["dask-scheduler", "--no-dashboard"]) as sched:
         with popen(
             [
                 "dask-worker",
                 "127.0.0.1:8786",
                 nanny,
-                "--no-bokeh",
+                "--no-dashboard",
                 "--contact-address",
                 "tcp://127.0.0.2:39837",
                 "--listen-address",
@@ -228,9 +238,9 @@ def func(dask_worker):
 @pytest.mark.parametrize("nanny", ["--nanny", "--no-nanny"])
 @pytest.mark.parametrize("host", ["127.0.0.2", "0.0.0.0"])
 def test_respect_host_listen_address(loop, nanny, host):
-    with popen(["dask-scheduler", "--no-bokeh"]) as sched:
+    with popen(["dask-scheduler", "--no-dashboard"]) as sched:
         with popen(
-            ["dask-worker", "127.0.0.1:8786", nanny, "--no-bokeh", "--host", host]
+            ["dask-worker", "127.0.0.1:8786", nanny, "--no-dashboard", "--host", host]
         ) as worker:
             with Client("127.0.0.1:8786") as client:
                 while not client.ncores():
@@ -247,7 +257,7 @@ def func(dask_worker):
                 assert all(host in v for v in listen_addresses.values())
 
 
-def test_bokeh_non_standard_ports(loop):
+def test_dashboard_non_standard_ports(loop):
     pytest.importorskip("bokeh")
     try:
         import jupyter_server_proxy  # noqa: F401
diff --git a/distributed/cli/tests/test_tls_cli.py b/distributed/cli/tests/test_tls_cli.py
index d983039c962..4663a9b38ff 100644
--- a/distributed/cli/tests/test_tls_cli.py
+++ b/distributed/cli/tests/test_tls_cli.py
@@ -33,9 +33,9 @@ def wait_for_cores(c, ncores=1):
 
 
 def test_basic(loop):
-    with popen(["dask-scheduler", "--no-bokeh"] + tls_args) as s:
+    with popen(["dask-scheduler", "--no-dashboard"] + tls_args) as s:
         with popen(
-            ["dask-worker", "--no-bokeh", "tls://127.0.0.1:8786"] + tls_args
+            ["dask-worker", "--no-dashboard", "tls://127.0.0.1:8786"] + tls_args
         ) as w:
             with Client(
                 "tls://127.0.0.1:8786", loop=loop, security=tls_security()
@@ -44,9 +44,10 @@ def test_basic(loop):
 
 
 def test_nanny(loop):
-    with popen(["dask-scheduler", "--no-bokeh"] + tls_args) as s:
+    with popen(["dask-scheduler", "--no-dashboard"] + tls_args) as s:
         with popen(
-            ["dask-worker", "--no-bokeh", "--nanny", "tls://127.0.0.1:8786"] + tls_args
+            ["dask-worker", "--no-dashboard", "--nanny", "tls://127.0.0.1:8786"]
+            + tls_args
         ) as w:
             with Client(
                 "tls://127.0.0.1:8786", loop=loop, security=tls_security()
@@ -55,9 +56,9 @@ def test_nanny(loop):
 
 
 def test_separate_key_cert(loop):
-    with popen(["dask-scheduler", "--no-bokeh"] + tls_args_2) as s:
+    with popen(["dask-scheduler", "--no-dashboard"] + tls_args_2) as s:
         with popen(
-            ["dask-worker", "--no-bokeh", "tls://127.0.0.1:8786"] + tls_args_2
+            ["dask-worker", "--no-dashboard", "tls://127.0.0.1:8786"] + tls_args_2
         ) as w:
             with Client(
                 "tls://127.0.0.1:8786", loop=loop, security=tls_security()
@@ -67,8 +68,8 @@ def test_separate_key_cert(loop):
 
 def test_use_config_file(loop):
     with new_config_file(tls_only_config()):
-        with popen(["dask-scheduler", "--no-bokeh", "--host", "tls://"]) as s:
-            with popen(["dask-worker", "--no-bokeh", "tls://127.0.0.1:8786"]) as w:
+        with popen(["dask-scheduler", "--no-dashboard", "--host", "tls://"]) as s:
+            with popen(["dask-worker", "--no-dashboard", "tls://127.0.0.1:8786"]) as w:
                 with Client(
                     "tls://127.0.0.1:8786", loop=loop, security=tls_security()
                 ) as c:
diff --git a/distributed/client.py b/distributed/client.py
index afe6f6ef39f..d924b608c61 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -812,9 +812,9 @@ def _repr_html_(self):
             text = (
                 "<h3>Client</h3>\n" "<ul>\n" "  <li><b>Scheduler: not connected</b>\n"
             )
-        if info and "bokeh" in info["services"]:
+        if info and "dashboard" in info["services"]:
             protocol, rest = scheduler.address.split("://")
-            port = info["services"]["bokeh"]
+            port = info["services"]["dashboard"]
             if protocol == "inproc":
                 host = "localhost"
             else:
@@ -3852,7 +3852,7 @@ def _get_task_stream(
             from .diagnostics.task_stream import rectangles
 
             rects = rectangles(msgs)
-            from .bokeh.components import task_stream_figure
+            from .dashboard.components import task_stream_figure
 
             source, figure = task_stream_figure(sizing_mode="stretch_both")
             source.data.update(rects)
diff --git a/distributed/dashboard/__init__.py b/distributed/dashboard/__init__.py
new file mode 100644
index 00000000000..675963b1463
--- /dev/null
+++ b/distributed/dashboard/__init__.py
@@ -0,0 +1,2 @@
+from .scheduler import BokehScheduler
+from .worker import BokehWorker
diff --git a/distributed/bokeh/components.py b/distributed/dashboard/components.py
similarity index 100%
rename from distributed/bokeh/components.py
rename to distributed/dashboard/components.py
diff --git a/distributed/bokeh/core.py b/distributed/dashboard/core.py
similarity index 100%
rename from distributed/bokeh/core.py
rename to distributed/dashboard/core.py
diff --git a/distributed/bokeh/export_tool.coffee b/distributed/dashboard/export_tool.coffee
similarity index 100%
rename from distributed/bokeh/export_tool.coffee
rename to distributed/dashboard/export_tool.coffee
diff --git a/distributed/bokeh/export_tool.js b/distributed/dashboard/export_tool.js
similarity index 100%
rename from distributed/bokeh/export_tool.js
rename to distributed/dashboard/export_tool.js
diff --git a/distributed/bokeh/export_tool.py b/distributed/dashboard/export_tool.py
similarity index 100%
rename from distributed/bokeh/export_tool.py
rename to distributed/dashboard/export_tool.py
diff --git a/distributed/bokeh/proxy.py b/distributed/dashboard/proxy.py
similarity index 98%
rename from distributed/bokeh/proxy.py
rename to distributed/dashboard/proxy.py
index 9353e383112..89f9f87aae6 100644
--- a/distributed/bokeh/proxy.py
+++ b/distributed/dashboard/proxy.py
@@ -124,7 +124,7 @@ def check_worker_dashboard_exits(scheduler, worker):
     addr, port = worker.split(":")
     workers = list(scheduler.workers.values())
     for w in workers:
-        bokeh_port = w.services.get("bokeh", "")
+        bokeh_port = w.services.get("dashboard", "")
         if addr == w.host and port == str(bokeh_port):
             return True
     return False
diff --git a/distributed/bokeh/scheduler.py b/distributed/dashboard/scheduler.py
similarity index 99%
rename from distributed/bokeh/scheduler.py
rename to distributed/dashboard/scheduler.py
index e0f5bfffab6..6476d3aa6e4 100644
--- a/distributed/bokeh/scheduler.py
+++ b/distributed/dashboard/scheduler.py
@@ -170,7 +170,7 @@ def update(self):
             workers = list(self.scheduler.workers.values())
 
             dashboard_host = [ws.host for ws in workers]
-            dashboard_port = [ws.services.get("bokeh", "") for ws in workers]
+            dashboard_port = [ws.services.get("dashboard", "") for ws in workers]
 
             y = list(range(len(workers)))
             occupancy = [ws.occupancy for ws in workers]
@@ -403,7 +403,7 @@ def update(self):
             workers = list(self.scheduler.workers.values())
 
             dashboard_host = [ws.host for ws in workers]
-            dashboard_port = [ws.services.get("bokeh", "") for ws in workers]
+            dashboard_port = [ws.services.get("dashboard", "") for ws in workers]
 
             y = list(range(len(workers)))
             nprocessing = [len(ws.processing) for ws in workers]
diff --git a/distributed/bokeh/scheduler_html.py b/distributed/dashboard/scheduler_html.py
similarity index 99%
rename from distributed/bokeh/scheduler_html.py
rename to distributed/dashboard/scheduler_html.py
index 1d3635c37c5..5f481f783be 100644
--- a/distributed/bokeh/scheduler_html.py
+++ b/distributed/dashboard/scheduler_html.py
@@ -177,7 +177,7 @@ def get(self):
 
 class IndividualPlots(RequestHandler):
     def get(self):
-        bokeh_server = self.server.services["bokeh"]
+        bokeh_server = self.server.services["dashboard"]
         result = {
             uri.strip("/").replace("-", " ").title(): uri
             for uri in bokeh_server.apps
diff --git a/distributed/bokeh/static/css/base.css b/distributed/dashboard/static/css/base.css
similarity index 100%
rename from distributed/bokeh/static/css/base.css
rename to distributed/dashboard/static/css/base.css
diff --git a/distributed/bokeh/static/css/status.css b/distributed/dashboard/static/css/status.css
similarity index 100%
rename from distributed/bokeh/static/css/status.css
rename to distributed/dashboard/static/css/status.css
diff --git a/distributed/bokeh/static/css/system.css b/distributed/dashboard/static/css/system.css
similarity index 100%
rename from distributed/bokeh/static/css/system.css
rename to distributed/dashboard/static/css/system.css
diff --git a/distributed/bokeh/static/images/dask-logo.svg b/distributed/dashboard/static/images/dask-logo.svg
similarity index 100%
rename from distributed/bokeh/static/images/dask-logo.svg
rename to distributed/dashboard/static/images/dask-logo.svg
diff --git a/distributed/bokeh/static/images/fa-bars.svg b/distributed/dashboard/static/images/fa-bars.svg
similarity index 100%
rename from distributed/bokeh/static/images/fa-bars.svg
rename to distributed/dashboard/static/images/fa-bars.svg
diff --git a/distributed/bokeh/templates/base.html b/distributed/dashboard/templates/base.html
similarity index 100%
rename from distributed/bokeh/templates/base.html
rename to distributed/dashboard/templates/base.html
diff --git a/distributed/bokeh/templates/call-stack.html b/distributed/dashboard/templates/call-stack.html
similarity index 100%
rename from distributed/bokeh/templates/call-stack.html
rename to distributed/dashboard/templates/call-stack.html
diff --git a/distributed/bokeh/templates/json-index.html b/distributed/dashboard/templates/json-index.html
similarity index 100%
rename from distributed/bokeh/templates/json-index.html
rename to distributed/dashboard/templates/json-index.html
diff --git a/distributed/bokeh/templates/logs.html b/distributed/dashboard/templates/logs.html
similarity index 100%
rename from distributed/bokeh/templates/logs.html
rename to distributed/dashboard/templates/logs.html
diff --git a/distributed/bokeh/templates/main.html b/distributed/dashboard/templates/main.html
similarity index 100%
rename from distributed/bokeh/templates/main.html
rename to distributed/dashboard/templates/main.html
diff --git a/distributed/bokeh/templates/simple.html b/distributed/dashboard/templates/simple.html
similarity index 100%
rename from distributed/bokeh/templates/simple.html
rename to distributed/dashboard/templates/simple.html
diff --git a/distributed/bokeh/templates/status.html b/distributed/dashboard/templates/status.html
similarity index 100%
rename from distributed/bokeh/templates/status.html
rename to distributed/dashboard/templates/status.html
diff --git a/distributed/bokeh/templates/system.html b/distributed/dashboard/templates/system.html
similarity index 100%
rename from distributed/bokeh/templates/system.html
rename to distributed/dashboard/templates/system.html
diff --git a/distributed/bokeh/templates/task.html b/distributed/dashboard/templates/task.html
similarity index 100%
rename from distributed/bokeh/templates/task.html
rename to distributed/dashboard/templates/task.html
diff --git a/distributed/bokeh/templates/worker-table.html b/distributed/dashboard/templates/worker-table.html
similarity index 100%
rename from distributed/bokeh/templates/worker-table.html
rename to distributed/dashboard/templates/worker-table.html
diff --git a/distributed/bokeh/templates/worker.html b/distributed/dashboard/templates/worker.html
similarity index 100%
rename from distributed/bokeh/templates/worker.html
rename to distributed/dashboard/templates/worker.html
diff --git a/distributed/bokeh/templates/workers.html b/distributed/dashboard/templates/workers.html
similarity index 100%
rename from distributed/bokeh/templates/workers.html
rename to distributed/dashboard/templates/workers.html
diff --git a/distributed/dashboard/tests/test_bokeh.py b/distributed/dashboard/tests/test_bokeh.py
new file mode 100644
index 00000000000..363272be5f6
--- /dev/null
+++ b/distributed/dashboard/tests/test_bokeh.py
@@ -0,0 +1,5 @@
+def test_old_import():
+    try:
+        from distributed.bokeh import BokehScheduler  # noqa: F401
+    except ImportError as e:
+        assert "distributed.dashboard" in str(e)
diff --git a/distributed/bokeh/tests/test_components.py b/distributed/dashboard/tests/test_components.py
similarity index 92%
rename from distributed/bokeh/tests/test_components.py
rename to distributed/dashboard/tests/test_components.py
index 028f209b41a..d441db57aec 100644
--- a/distributed/bokeh/tests/test_components.py
+++ b/distributed/dashboard/tests/test_components.py
@@ -7,10 +7,9 @@
 from bokeh.models import ColumnDataSource, Model
 from tornado import gen
 
-from distributed.bokeh import messages
 from distributed.utils_test import slowinc, gen_cluster
 
-from distributed.bokeh.components import (
+from distributed.dashboard.components import (
     TaskStream,
     MemoryUsage,
     Processing,
@@ -24,7 +23,6 @@ def test_basic(Component):
     c = Component()
     assert isinstance(c.source, ColumnDataSource)
     assert isinstance(c.root, Model)
-    c.update(messages)
 
 
 @gen_cluster(client=True, check_new_threads=False)
diff --git a/distributed/bokeh/tests/test_scheduler_bokeh.py b/distributed/dashboard/tests/test_scheduler_bokeh.py
similarity index 95%
rename from distributed/bokeh/tests/test_scheduler_bokeh.py
rename to distributed/dashboard/tests/test_scheduler_bokeh.py
index 057aa679655..f8a813514b4 100644
--- a/distributed/bokeh/tests/test_scheduler_bokeh.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh.py
@@ -17,8 +17,8 @@
 from distributed.client import wait
 from distributed.metrics import time
 from distributed.utils_test import gen_cluster, inc, dec, slowinc, div
-from distributed.bokeh.worker import Counters, BokehWorker
-from distributed.bokeh.scheduler import (
+from distributed.dashboard.worker import Counters, BokehWorker
+from distributed.dashboard.scheduler import (
     BokehScheduler,
     SystemMonitor,
     Occupancy,
@@ -36,7 +36,7 @@
     ProfileServer,
 )
 
-from distributed.bokeh import scheduler
+from distributed.dashboard import scheduler
 
 scheduler.PROFILING = False
 
@@ -44,10 +44,12 @@
 @pytest.mark.skipif(
     sys.version_info[0] == 2, reason="https://github.com/bokeh/bokeh/issues/5494"
 )
-@gen_cluster(client=True, scheduler_kwargs={"services": {("bokeh", 0): BokehScheduler}})
+@gen_cluster(
+    client=True, scheduler_kwargs={"services": {("dashboard", 0): BokehScheduler}}
+)
 def test_simple(c, s, a, b):
-    assert isinstance(s.services["bokeh"], BokehScheduler)
-    port = s.services["bokeh"].port
+    assert isinstance(s.services["dashboard"], BokehScheduler)
+    port = s.services["dashboard"].port
 
     future = c.submit(sleep, 1)
     yield gen.sleep(0.1)
@@ -80,7 +82,7 @@ def test_simple(c, s, a, b):
     assert response
 
 
-@gen_cluster(client=True, worker_kwargs=dict(services={"bokeh": BokehWorker}))
+@gen_cluster(client=True, worker_kwargs=dict(services={"dashboard": BokehWorker}))
 def test_basic(c, s, a, b):
     for component in [SystemMonitor, Occupancy, StealingTimeSeries]:
         ss = component(s)
@@ -573,11 +575,13 @@ def test_profile_server(c, s, a, b):
         assert time() < start + 2
 
 
-@gen_cluster(client=True, scheduler_kwargs={"services": {("bokeh", 0): BokehScheduler}})
+@gen_cluster(
+    client=True, scheduler_kwargs={"services": {("dashboard", 0): BokehScheduler}}
+)
 def test_root_redirect(c, s, a, b):
     http_client = AsyncHTTPClient()
     response = yield http_client.fetch(
-        "http://localhost:%d/" % s.services["bokeh"].port
+        "http://localhost:%d/" % s.services["dashboard"].port
     )
     assert response.code == 200
     assert "/status" in response.effective_url
@@ -585,8 +589,8 @@ def test_root_redirect(c, s, a, b):
 
 @gen_cluster(
     client=True,
-    scheduler_kwargs={"services": {("bokeh", 0): BokehScheduler}},
-    worker_kwargs={"services": {"bokeh": BokehWorker}},
+    scheduler_kwargs={"services": {("dashboard", 0): BokehScheduler}},
+    worker_kwargs={"services": {"dashboard": BokehWorker}},
     timeout=180,
 )
 def test_proxy_to_workers(c, s, a, b):
@@ -597,7 +601,7 @@ def test_proxy_to_workers(c, s, a, b):
     except ImportError:
         proxy_exists = False
 
-    dashboard_port = s.services["bokeh"].port
+    dashboard_port = s.services["dashboard"].port
     http_client = AsyncHTTPClient()
     response = yield http_client.fetch("http://localhost:%d/" % dashboard_port)
     assert response.code == 200
@@ -605,7 +609,7 @@ def test_proxy_to_workers(c, s, a, b):
 
     for w in [a, b]:
         host = w.ip
-        port = w.service_ports["bokeh"]
+        port = w.service_ports["dashboard"]
         proxy_url = "http://localhost:%d/proxy/%s/%s/status" % (
             dashboard_port,
             port,
diff --git a/distributed/bokeh/tests/test_scheduler_bokeh_html.py b/distributed/dashboard/tests/test_scheduler_bokeh_html.py
similarity index 79%
rename from distributed/bokeh/tests/test_scheduler_bokeh_html.py
rename to distributed/dashboard/tests/test_scheduler_bokeh_html.py
index 691121f7514..f872d02dc84 100644
--- a/distributed/bokeh/tests/test_scheduler_bokeh_html.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh_html.py
@@ -13,14 +13,13 @@
 
 from dask.sizeof import sizeof
 from distributed.utils_test import gen_cluster, slowinc, inc
-from distributed.bokeh.scheduler import BokehScheduler
-from distributed.bokeh.worker import BokehWorker
+from distributed.dashboard import BokehScheduler, BokehWorker
 
 
 @gen_cluster(
     client=True,
-    scheduler_kwargs={"services": {("bokeh", 0): BokehScheduler}},
-    worker_kwargs={"services": {"bokeh": BokehWorker}},
+    scheduler_kwargs={"services": {("dashboard", 0): BokehScheduler}},
+    worker_kwargs={"services": {"dashboard": BokehWorker}},
 )
 def test_connect(c, s, a, b):
     future = c.submit(lambda x: x + 1, 1)
@@ -41,7 +40,7 @@ def test_connect(c, s, a, b):
         "individual-plots.json",
     ]:
         response = yield http_client.fetch(
-            "http://localhost:%d/%s" % (s.services["bokeh"].port, suffix)
+            "http://localhost:%d/%s" % (s.services["dashboard"].port, suffix)
         )
         assert response.code == 200
         body = response.body.decode()
@@ -54,13 +53,15 @@ def test_connect(c, s, a, b):
 
 @gen_cluster(
     client=True,
-    scheduler_kwargs={"services": {("bokeh", 0): (BokehScheduler, {"prefix": "/foo"})}},
+    scheduler_kwargs={
+        "services": {("dashboard", 0): (BokehScheduler, {"prefix": "/foo"})}
+    },
 )
 def test_prefix(c, s, a, b):
     http_client = AsyncHTTPClient()
     for suffix in ["foo/info/main/workers.html", "foo/json/index.html", "foo/system"]:
         response = yield http_client.fetch(
-            "http://localhost:%d/%s" % (s.services["bokeh"].port, suffix)
+            "http://localhost:%d/%s" % (s.services["dashboard"].port, suffix)
         )
         assert response.code == 200
         body = response.body.decode()
@@ -73,7 +74,7 @@ def test_prefix(c, s, a, b):
 @gen_cluster(
     client=True,
     check_new_threads=False,
-    scheduler_kwargs={"services": {("bokeh", 0): BokehScheduler}},
+    scheduler_kwargs={"services": {("dashboard", 0): BokehScheduler}},
 )
 def test_prometheus(c, s, a, b):
     pytest.importorskip("prometheus_client")
@@ -85,7 +86,7 @@ def test_prometheus(c, s, a, b):
     # prometheus_client errors
     for _ in range(2):
         response = yield http_client.fetch(
-            "http://localhost:%d/metrics" % s.services["bokeh"].port
+            "http://localhost:%d/metrics" % s.services["dashboard"].port
         )
         assert response.code == 200
         assert response.headers["Content-Type"] == "text/plain; version=0.0.4"
@@ -98,13 +99,13 @@ def test_prometheus(c, s, a, b):
 @gen_cluster(
     client=True,
     check_new_threads=False,
-    scheduler_kwargs={"services": {("bokeh", 0): BokehScheduler}},
+    scheduler_kwargs={"services": {("dashboard", 0): BokehScheduler}},
 )
 def test_health(c, s, a, b):
     http_client = AsyncHTTPClient()
 
     response = yield http_client.fetch(
-        "http://localhost:%d/health" % s.services["bokeh"].port
+        "http://localhost:%d/health" % s.services["dashboard"].port
     )
     assert response.code == 200
     assert response.headers["Content-Type"] == "text/plain"
@@ -113,7 +114,9 @@ def test_health(c, s, a, b):
     assert txt == "ok"
 
 
-@gen_cluster(client=True, scheduler_kwargs={"services": {("bokeh", 0): BokehScheduler}})
+@gen_cluster(
+    client=True, scheduler_kwargs={"services": {("dashboard", 0): BokehScheduler}}
+)
 def test_task_page(c, s, a, b):
     future = c.submit(lambda x: x + 1, 1, workers=a.address)
     x = c.submit(inc, 1)
@@ -122,7 +125,7 @@ def test_task_page(c, s, a, b):
 
     "info/task/" + url_escape(future.key) + ".html",
     response = yield http_client.fetch(
-        "http://localhost:%d/info/task/" % s.services["bokeh"].port
+        "http://localhost:%d/info/task/" % s.services["dashboard"].port
         + url_escape(future.key)
         + ".html"
     )
diff --git a/distributed/bokeh/tests/test_worker_bokeh.py b/distributed/dashboard/tests/test_worker_bokeh.py
similarity index 79%
rename from distributed/bokeh/tests/test_worker_bokeh.py
rename to distributed/dashboard/tests/test_worker_bokeh.py
index 03a7ed3861b..11699d9ac83 100644
--- a/distributed/bokeh/tests/test_worker_bokeh.py
+++ b/distributed/dashboard/tests/test_worker_bokeh.py
@@ -14,7 +14,7 @@
 from distributed.client import wait
 from distributed.metrics import time
 from distributed.utils_test import gen_cluster, inc, dec
-from distributed.bokeh.worker import (
+from distributed.dashboard.worker import (
     BokehWorker,
     StateTable,
     CrossFilter,
@@ -29,10 +29,10 @@
 @pytest.mark.skipif(
     sys.version_info[0] == 2, reason="https://github.com/bokeh/bokeh/issues/5494"
 )
-@gen_cluster(client=True, worker_kwargs={"services": {("bokeh", 0): BokehWorker}})
+@gen_cluster(client=True, worker_kwargs={"services": {("dashboard", 0): BokehWorker}})
 def test_simple(c, s, a, b):
-    assert s.workers[a.address].services == {"bokeh": a.services["bokeh"].port}
-    assert s.workers[b.address].services == {"bokeh": b.services["bokeh"].port}
+    assert s.workers[a.address].services == {"dashboard": a.services["dashboard"].port}
+    assert s.workers[b.address].services == {"dashboard": b.services["dashboard"].port}
 
     future = c.submit(sleep, 1)
     yield gen.sleep(0.1)
@@ -40,15 +40,17 @@ def test_simple(c, s, a, b):
     http_client = AsyncHTTPClient()
     for suffix in ["main", "crossfilter", "system"]:
         response = yield http_client.fetch(
-            "http://localhost:%d/%s" % (a.services["bokeh"].port, suffix)
+            "http://localhost:%d/%s" % (a.services["dashboard"].port, suffix)
         )
         assert "bokeh" in response.body.decode().lower()
 
 
-@gen_cluster(client=True, worker_kwargs={"services": {("bokeh", 0): (BokehWorker, {})}})
+@gen_cluster(
+    client=True, worker_kwargs={"services": {("dashboard", 0): (BokehWorker, {})}}
+)
 def test_services_kwargs(c, s, a, b):
-    assert s.workers[a.address].services == {"bokeh": a.services["bokeh"].port}
-    assert isinstance(a.services["bokeh"], BokehWorker)
+    assert s.workers[a.address].services == {"dashboard": a.services["dashboard"].port}
+    assert isinstance(a.services["dashboard"], BokehWorker)
 
 
 @gen_cluster(client=True)
@@ -139,15 +141,15 @@ def test_CommunicatingStream(c, s, a, b):
 @gen_cluster(
     client=True,
     check_new_threads=False,
-    worker_kwargs={"services": {("bokeh", 0): BokehWorker}},
+    worker_kwargs={"services": {("dashboard", 0): BokehWorker}},
 )
 def test_prometheus(c, s, a, b):
     pytest.importorskip("prometheus_client")
-    assert s.workers[a.address].services == {"bokeh": a.services["bokeh"].port}
+    assert s.workers[a.address].services == {"dashboard": a.services["dashboard"].port}
 
     http_client = AsyncHTTPClient()
     for suffix in ["metrics"]:
         response = yield http_client.fetch(
-            "http://localhost:%d/%s" % (a.services["bokeh"].port, suffix)
+            "http://localhost:%d/%s" % (a.services["dashboard"].port, suffix)
         )
         assert response.code == 200
diff --git a/distributed/bokeh/tests/test_worker_bokeh_html.py b/distributed/dashboard/tests/test_worker_bokeh_html.py
similarity index 75%
rename from distributed/bokeh/tests/test_worker_bokeh_html.py
rename to distributed/dashboard/tests/test_worker_bokeh_html.py
index d59fec8d2d8..99916b3fdc7 100644
--- a/distributed/bokeh/tests/test_worker_bokeh_html.py
+++ b/distributed/dashboard/tests/test_worker_bokeh_html.py
@@ -4,10 +4,10 @@
 
 from tornado.httpclient import AsyncHTTPClient
 from distributed.utils_test import gen_cluster
-from distributed.bokeh.worker import BokehWorker
+from distributed.dashboard import BokehWorker
 
 
-@gen_cluster(client=True, worker_kwargs={"services": {("bokeh", 0): BokehWorker}})
+@gen_cluster(client=True, worker_kwargs={"services": {("dashboard", 0): BokehWorker}})
 def test_prometheus(c, s, a, b):
     pytest.importorskip("prometheus_client")
     from prometheus_client.parser import text_string_to_metric_families
@@ -18,7 +18,7 @@ def test_prometheus(c, s, a, b):
     # prometheus_client errors
     for _ in range(2):
         response = yield http_client.fetch(
-            "http://localhost:%d/metrics" % a.services["bokeh"].port
+            "http://localhost:%d/metrics" % a.services["dashboard"].port
         )
         assert response.code == 200
         assert response.headers["Content-Type"] == "text/plain; version=0.0.4"
@@ -28,12 +28,12 @@ def test_prometheus(c, s, a, b):
         assert len(families) > 0
 
 
-@gen_cluster(client=True, worker_kwargs={"services": {("bokeh", 0): BokehWorker}})
+@gen_cluster(client=True, worker_kwargs={"services": {("dashboard", 0): BokehWorker}})
 def test_health(c, s, a, b):
     http_client = AsyncHTTPClient()
 
     response = yield http_client.fetch(
-        "http://localhost:%d/health" % a.services["bokeh"].port
+        "http://localhost:%d/health" % a.services["dashboard"].port
     )
     assert response.code == 200
     assert response.headers["Content-Type"] == "text/plain"
diff --git a/distributed/bokeh/theme.yaml b/distributed/dashboard/theme.yaml
similarity index 100%
rename from distributed/bokeh/theme.yaml
rename to distributed/dashboard/theme.yaml
diff --git a/distributed/bokeh/utils.py b/distributed/dashboard/utils.py
similarity index 100%
rename from distributed/bokeh/utils.py
rename to distributed/dashboard/utils.py
diff --git a/distributed/bokeh/worker.py b/distributed/dashboard/worker.py
similarity index 100%
rename from distributed/bokeh/worker.py
rename to distributed/dashboard/worker.py
diff --git a/distributed/bokeh/worker_html.py b/distributed/dashboard/worker_html.py
similarity index 100%
rename from distributed/bokeh/worker_html.py
rename to distributed/dashboard/worker_html.py
diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
index 8425b836a4d..69cc5be9fac 100644
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -88,7 +88,7 @@ def scheduler_address(self):
     def dashboard_link(self):
         template = dask.config.get("distributed.dashboard.link")
         host = self.scheduler.address.split("://")[1].split(":")[0]
-        port = self.scheduler.services["bokeh"].port
+        port = self.scheduler.services["dashboard"].port
         return template.format(host=host, port=port, **os.environ)
 
     def scale(self, n):
@@ -165,7 +165,7 @@ def _widget(self):
 
         layout = Layout(width="150px")
 
-        if "bokeh" in self.scheduler.services:
+        if "dashboard" in self.scheduler.services:
             link = self.dashboard_link
             link = '<p><b>Dashboard: </b><a href="%s" target="_blank">%s</a></p>\n' % (
                 link,
diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index ad0aea25f6c..d5a954effc8 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -112,11 +112,11 @@ def __init__(
         self._created = weakref.WeakSet()
         if scheduler is None:
             try:
-                from distributed.bokeh.scheduler import BokehScheduler
+                from distributed.dashboard import BokehScheduler
             except ImportError:
                 services = {}
             else:
-                services = {("bokeh", 8787): BokehScheduler}
+                services = {("dashboard", 8787): BokehScheduler}
             scheduler = {"cls": Scheduler, "options": {"services": services}}
 
         self.scheduler_spec = scheduler
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index 4498611d7e8..6f9a4a03244 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -198,8 +198,8 @@ def test_duplicate_clients():
     with pytest.warns(Exception) as info:
         c2 = yield Client(processes=False, silence_logs=False, dashboard_address=9876)
 
-    assert "bokeh" in c1.cluster.scheduler.services
-    assert "bokeh" in c2.cluster.scheduler.services
+    assert "dashboard" in c1.cluster.scheduler.services
+    assert "dashboard" in c2.cluster.scheduler.services
 
     assert any(
         all(
@@ -341,7 +341,7 @@ def test_bokeh(loop, processes):
         processes=processes,
         dashboard_address=0,
     ) as c:
-        bokeh_port = c.scheduler.services["bokeh"].port
+        bokeh_port = c.scheduler.services["dashboard"].port
         url = "http://127.0.0.1:%d/status/" % bokeh_port
         start = time()
         while True:
@@ -485,10 +485,10 @@ def test_bokeh_kwargs(loop):
         silence_logs=False,
         loop=loop,
         dashboard_address=0,
-        service_kwargs={"bokeh": {"prefix": "/foo"}},
+        service_kwargs={"dashboard": {"prefix": "/foo"}},
     ) as c:
 
-        bs = c.scheduler.services["bokeh"]
+        bs = c.scheduler.services["dashboard"]
         assert bs.prefix == "/foo"
 
 
diff --git a/distributed/diagnostics/tests/test_eventstream.py b/distributed/diagnostics/tests/test_eventstream.py
index 0995d80db26..7ec646d7e91 100644
--- a/distributed/diagnostics/tests/test_eventstream.py
+++ b/distributed/diagnostics/tests/test_eventstream.py
@@ -1,6 +1,6 @@
 from __future__ import print_function, division, absolute_import
 
-from copy import deepcopy
+import collections
 
 import pytest
 from tornado import gen
@@ -26,10 +26,12 @@ def test_eventstream(c, s, *workers):
 
     assert len(es.buffer) == 11
 
-    from distributed.bokeh import messages
     from distributed.diagnostics.progress_stream import task_stream_append
 
-    lists = deepcopy(messages["task-events"]["rectangles"])
+    lists = {
+        name: collections.deque(maxlen=100)
+        for name in "start duration key name color worker worker_thread y alpha".split()
+    }
     workers = dict()
     for msg in es.buffer:
         task_stream_append(lists, msg, workers)
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 991ff1a2108..9db6477aeb7 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -878,13 +878,13 @@ def __init__(
 
         if dashboard_address is not None:
             try:
-                from distributed.bokeh.scheduler import BokehScheduler
+                from distributed.dashboard import BokehScheduler
             except ImportError:
                 logger.debug("To start diagnostics web server please install Bokeh")
             else:
-                self.service_specs[("bokeh", dashboard_address)] = (
+                self.service_specs[("dashboard", dashboard_address)] = (
                     BokehScheduler,
-                    (service_kwargs or {}).get("bokeh", {}),
+                    (service_kwargs or {}).get("dashboard", {}),
                 )
 
         # Communication state
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 28c2f939eb7..c731ae6e5ad 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -3506,7 +3506,7 @@ def test_reconnect(loop):
         "127.0.0.1",
         "--port",
         "9393",
-        "--no-bokeh",
+        "--no-dashboard",
     ]
     with popen(scheduler_cli) as s:
         c = Client("127.0.0.1:9393", loop=loop)
@@ -5221,12 +5221,11 @@ def test_quiet_scheduler_loss(c, s):
 @pytest.mark.skipif("USER" not in os.environ, reason="no USER env variable")
 def test_diagnostics_link_env_variable(loop):
     pytest.importorskip("bokeh")
-    from distributed.bokeh.scheduler import BokehScheduler
+    from distributed.dashboard import BokehScheduler
 
-    with cluster(scheduler_kwargs={"services": {("bokeh", 12355): BokehScheduler}}) as (
-        s,
-        [a, b],
-    ):
+    with cluster(
+        scheduler_kwargs={"services": {("dashboard", 12355): BokehScheduler}}
+    ) as (s, [a, b]):
         with Client(s["address"], loop=loop) as c:
             with dask.config.set(
                 {"distributed.dashboard.link": "http://foo-{USER}:{port}/status"}
diff --git a/distributed/tests/test_core.py b/distributed/tests/test_core.py
index 4b3c0ac0ade..f53340d1004 100644
--- a/distributed/tests/test_core.py
+++ b/distributed/tests/test_core.py
@@ -324,7 +324,10 @@ def check_rpc_message_lifetime(*listen_args):
     obj = CountedObject()
     assert CountedObject.n_instances == 1
     del obj
-    assert CountedObject.n_instances == 0
+    start = time()
+    while CountedObject.n_instances != 0:
+        yield gen.sleep(0.01)
+        assert time() < start + 1
 
     with rpc(server.address) as remote:
         obj = CountedObject()
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index f5ce276b8d2..6df271ae34e 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -1174,7 +1174,7 @@ def test_correct_bad_time_estimate(c, s, *workers):
 @gen_test()
 def test_service_hosts():
     pytest.importorskip("bokeh")
-    from distributed.bokeh.scheduler import BokehScheduler
+    from distributed.dashboard import BokehScheduler
 
     port = 0
     for url, expected in [
@@ -1182,12 +1182,12 @@ def test_service_hosts():
         ("tcp://127.0.0.1", "127.0.0.1"),
         ("tcp://127.0.0.1:38275", "127.0.0.1"),
     ]:
-        services = {("bokeh", port): BokehScheduler}
+        services = {("dashboard", port): BokehScheduler}
 
         s = Scheduler(services=services)
         yield s.start(url)
 
-        sock = first(s.services["bokeh"].server._http._sockets.values())
+        sock = first(s.services["dashboard"].server._http._sockets.values())
         if isinstance(expected, tuple):
             assert sock.getsockname()[0] in expected
         else:
@@ -1196,12 +1196,12 @@ def test_service_hosts():
 
     port = ("127.0.0.1", 0)
     for url in ["tcp://0.0.0.0", "tcp://127.0.0.1", "tcp://127.0.0.1:38275"]:
-        services = {("bokeh", port): BokehScheduler}
+        services = {("dashboard", port): BokehScheduler}
 
         s = Scheduler(services=services)
         yield s.start(url)
 
-        sock = first(s.services["bokeh"].server._http._sockets.values())
+        sock = first(s.services["dashboard"].server._http._sockets.values())
         assert sock.getsockname()[0] == "127.0.0.1"
         yield s.close()
 
@@ -1566,9 +1566,9 @@ def test_host_address():
 def test_dashboard_address():
     pytest.importorskip("bokeh")
     s = yield Scheduler(dashboard_address="127.0.0.1:8901", port=0)
-    assert s.services["bokeh"].port == 8901
+    assert s.services["dashboard"].port == 8901
     yield s.close()
 
     s = yield Scheduler(dashboard_address="127.0.0.1", port=0)
-    assert s.services["bokeh"].port
+    assert s.services["dashboard"].port
     yield s.close()
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index bf4e483f441..12a6b5ff68f 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -972,25 +972,25 @@ def test_worker_fds(s):
 @gen_cluster(ncores=[])
 def test_service_hosts_match_worker(s):
     pytest.importorskip("bokeh")
-    from distributed.bokeh.worker import BokehWorker
+    from distributed.dashboard import BokehWorker
 
-    services = {("bokeh", ":0"): BokehWorker}
+    services = {("dashboard", ":0"): BokehWorker}
 
-    w = Worker(s.address, services={("bokeh", ":0"): BokehWorker})
+    w = Worker(s.address, services={("dashboard", ":0"): BokehWorker})
     yield w._start("tcp://0.0.0.0")
-    sock = first(w.services["bokeh"].server._http._sockets.values())
+    sock = first(w.services["dashboard"].server._http._sockets.values())
     assert sock.getsockname()[0] in ("::", "0.0.0.0")
     yield w.close()
 
-    w = Worker(s.address, services={("bokeh", ":0"): BokehWorker})
+    w = Worker(s.address, services={("dashboard", ":0"): BokehWorker})
     yield w._start("tcp://127.0.0.1")
-    sock = first(w.services["bokeh"].server._http._sockets.values())
+    sock = first(w.services["dashboard"].server._http._sockets.values())
     assert sock.getsockname()[0] in ("::", "0.0.0.0")
     yield w.close()
 
-    w = Worker(s.address, services={("bokeh", 0): BokehWorker})
+    w = Worker(s.address, services={("dashboard", 0): BokehWorker})
     yield w._start("tcp://127.0.0.1")
-    sock = first(w.services["bokeh"].server._http._sockets.values())
+    sock = first(w.services["dashboard"].server._http._sockets.values())
     assert sock.getsockname()[0] == "127.0.0.1"
     yield w.close()
 
@@ -998,14 +998,14 @@ def test_service_hosts_match_worker(s):
 @gen_cluster(ncores=[])
 def test_start_services(s):
     pytest.importorskip("bokeh")
-    from distributed.bokeh.worker import BokehWorker
+    from distributed.dashboard import BokehWorker
 
-    services = {("bokeh", ":1234"): BokehWorker}
+    services = {("dashboard", ":1234"): BokehWorker}
 
     w = Worker(s.address, services=services)
     yield w._start()
 
-    assert w.services["bokeh"].server.port == 1234
+    assert w.services["dashboard"].server.port == 1234
     yield w.close()
 
 
diff --git a/distributed/worker.py b/distributed/worker.py
index 667bd83490f..711dad31651 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -546,13 +546,13 @@ def __init__(
 
         if dashboard_address is not None:
             try:
-                from distributed.bokeh.worker import BokehWorker
+                from distributed.dashboard import BokehWorker
             except ImportError:
                 logger.debug("To start diagnostics web server please install Bokeh")
             else:
-                self.service_specs[("bokeh", dashboard_address)] = (
+                self.service_specs[("dashboard", dashboard_address)] = (
                     BokehWorker,
-                    (service_kwargs or {}).get("bokeh", {}),
+                    (service_kwargs or {}).get("dashboard", {}),
                 )
 
         self.metrics = dict(metrics) if metrics else {}
diff --git a/setup.py b/setup.py
index 3ef26a047dc..0df22f3f911 100755
--- a/setup.py
+++ b/setup.py
@@ -2,46 +2,51 @@
 
 import os
 from setuptools import setup
-import sys
 import versioneer
 
-requires = open('requirements.txt').read().strip().split('\n')
+requires = open("requirements.txt").read().strip().split("\n")
 install_requires = []
 extras_require = {}
 for r in requires:
-    if ';' in r:
+    if ";" in r:
         # requirements.txt conditional dependencies need to be reformatted for wheels
         # to the form: `'[extra_name]:condition' : ['requirements']`
-        req, cond = r.split(';', 1)
-        cond = ':' + cond
+        req, cond = r.split(";", 1)
+        cond = ":" + cond
         cond_reqs = extras_require.setdefault(cond, [])
         cond_reqs.append(req)
     else:
         install_requires.append(r)
 
-setup(name='distributed',
-      version=versioneer.get_version(),
-      cmdclass=versioneer.get_cmdclass(),
-      description='Distributed scheduler for Dask',
-      url='https://distributed.readthedocs.io/en/latest/',
-      maintainer='Matthew Rocklin',
-      maintainer_email='mrocklin@gmail.com',
-      license='BSD',
-      package_data={'': ['templates/index.html', 'template.html'],
-                    'distributed': ['bokeh/templates/*.html']},
-      include_package_data=True,
-      install_requires=install_requires,
-      extras_require=extras_require,
-      packages=['distributed',
-                'distributed.bokeh',
-                'distributed.cli',
-                'distributed.comm',
-                'distributed.deploy',
-                'distributed.diagnostics',
-                'distributed.protocol'],
-      long_description=(open('README.rst').read() if os.path.exists('README.rst')
-                        else ''),
-      classifiers=[
+setup(
+    name="distributed",
+    version=versioneer.get_version(),
+    cmdclass=versioneer.get_cmdclass(),
+    description="Distributed scheduler for Dask",
+    url="https://distributed.readthedocs.io/en/latest/",
+    maintainer="Matthew Rocklin",
+    maintainer_email="mrocklin@gmail.com",
+    license="BSD",
+    package_data={
+        "": ["templates/index.html", "template.html"],
+        "distributed": ["dashboard/templates/*.html"],
+    },
+    include_package_data=True,
+    install_requires=install_requires,
+    extras_require=extras_require,
+    packages=[
+        "distributed",
+        "distributed.dashboard",
+        "distributed.cli",
+        "distributed.comm",
+        "distributed.deploy",
+        "distributed.diagnostics",
+        "distributed.protocol",
+    ],
+    long_description=(
+        open("README.rst").read() if os.path.exists("README.rst") else ""
+    ),
+    classifiers=[
         "Development Status :: 5 - Production/Stable",
         "Intended Audience :: Developers",
         "Intended Audience :: Science/Research",
@@ -54,8 +59,8 @@
         "Programming Language :: Python :: 3.7",
         "Topic :: Scientific/Engineering",
         "Topic :: System :: Distributed Computing",
-      ],
-      entry_points='''
+    ],
+    entry_points="""
         [console_scripts]
         dask-ssh=distributed.cli.dask_ssh:go
         dask-submit=distributed.cli.dask_submit:go
@@ -63,5 +68,6 @@
         dask-scheduler=distributed.cli.dask_scheduler:go
         dask-worker=distributed.cli.dask_worker:go
         dask-mpi=distributed.cli.dask_mpi:go
-      ''',
-      zip_safe=False)
+      """,
+    zip_safe=False,
+)

From 4e3ba76be99ae5d572364e3b8a05a5a7ec42cce5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lo=C3=AFc=20Est=C3=A8ve?= <loic.esteve@ymail.com>
Date: Wed, 29 May 2019 20:50:27 +0200
Subject: [PATCH 0308/1550] Add back LocalCluster.__repr__. (#2732)

LocalCluster.__repr__ was removed in #2675.
---
 distributed/deploy/local.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index 17150fdf70f..298c47d7a31 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -197,6 +197,13 @@ def __init__(
         )
         self.scale(n_workers)
 
+    def __repr__(self):
+        return "LocalCluster(%r, workers=%d, ncores=%d)" % (
+            self.scheduler_address,
+            len(self.workers),
+            sum(w.ncores for w in self.workers.values()),
+        )
+
 
 def nprocesses_nthreads(n):
     """

From 23b1d93ca7028e0b3dad0b55d6d133559b1d7f35 Mon Sep 17 00:00:00 2001
From: Manuel Garrido <manuel.garrido.pena@gmail.com>
Date: Wed, 29 May 2019 22:10:54 +0100
Subject: [PATCH 0309/1550] add kwargs to progressbars (#2638)

* add kwargs to progressbars

* remove assertion

* linting and add kwarg test for progress bar
---
 distributed/diagnostics/progressbar.py        | 25 ++++++++++++++-----
 .../diagnostics/tests/test_progressbar.py     |  8 ++++++
 2 files changed, 27 insertions(+), 6 deletions(-)

diff --git a/distributed/diagnostics/progressbar.py b/distributed/diagnostics/progressbar.py
index 08ba8f7da63..8a381562f27 100644
--- a/distributed/diagnostics/progressbar.py
+++ b/distributed/diagnostics/progressbar.py
@@ -118,6 +118,7 @@ def __init__(
         loop=None,
         complete=True,
         start=True,
+        **kwargs
     ):
         super(TextProgressBar, self).__init__(keys, scheduler, interval, complete)
         self.width = width
@@ -154,7 +155,13 @@ class ProgressWidget(ProgressBar):
     """
 
     def __init__(
-        self, keys, scheduler=None, interval="100ms", complete=False, loop=None
+        self,
+        keys,
+        scheduler=None,
+        interval="100ms",
+        complete=False,
+        loop=None,
+        **kwargs
     ):
         super(ProgressWidget, self).__init__(keys, scheduler, interval, complete)
 
@@ -207,7 +214,13 @@ def _draw_bar(self, remaining, all, **kwargs):
 
 class MultiProgressBar(object):
     def __init__(
-        self, keys, scheduler=None, func=key_split, interval="100ms", complete=False
+        self,
+        keys,
+        scheduler=None,
+        func=key_split,
+        interval="100ms",
+        complete=False,
+        **kwargs
     ):
         self.scheduler = get_scheduler(scheduler)
 
@@ -306,6 +319,7 @@ def __init__(
         interval=0.1,
         func=key_split,
         complete=False,
+        **kwargs
     ):
         super(MultiProgressWidget, self).__init__(
             keys, scheduler, func, interval, complete
@@ -425,7 +439,6 @@ def progress(*futures, **kwargs):
     notebook = kwargs.pop("notebook", None)
     multi = kwargs.pop("multi", True)
     complete = kwargs.pop("complete", True)
-    assert not kwargs
 
     futures = futures_of(futures)
     if not isinstance(futures, (set, list)):
@@ -434,9 +447,9 @@ def progress(*futures, **kwargs):
         notebook = is_kernel()  # often but not always correct assumption
     if notebook:
         if multi:
-            bar = MultiProgressWidget(futures, complete=complete)
+            bar = MultiProgressWidget(futures, complete=complete, **kwargs)
         else:
-            bar = ProgressWidget(futures, complete=complete)
+            bar = ProgressWidget(futures, complete=complete, **kwargs)
         return bar
     else:
-        TextProgressBar(futures, complete=complete)
+        TextProgressBar(futures, complete=complete, **kwargs)
diff --git a/distributed/diagnostics/tests/test_progressbar.py b/distributed/diagnostics/tests/test_progressbar.py
index ac21f1637bc..ba42f2ce6ea 100644
--- a/distributed/diagnostics/tests/test_progressbar.py
+++ b/distributed/diagnostics/tests/test_progressbar.py
@@ -76,3 +76,11 @@ def test_progress_function(client, capsys):
 
     progress(f)
     check_bar_completed(capsys)
+
+
+def test_progress_function_w_kwargs(client, capsys):
+    f = client.submit(lambda: 1)
+    g = client.submit(lambda: 2)
+
+    progress(f, interval="20ms")
+    check_bar_completed(capsys)

From d9626a59fa0ee5953293666591a083d2c249ddc1 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 30 May 2019 15:47:16 -0700
Subject: [PATCH 0310/1550] Close nannies gracefully (#2731)

Previously a worker process could be stopped before it told its nanny
that it was going away.  Now we intentionally tell the nanny ahead of
time from the scheduler (and the worker for good measure) before we
start the shutdown procedure.
---
 distributed/deploy/tests/test_local.py |  1 +
 distributed/nanny.py                   | 11 ++++++++++-
 distributed/scheduler.py               |  1 +
 distributed/utils_test.py              |  6 ++++++
 distributed/worker.py                  |  7 ++++++-
 5 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index 6f9a4a03244..8aad6675f8c 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -474,6 +474,7 @@ def test_death_timeout_raises(loop):
             loop=loop,
         ) as cluster:
             pass
+    LocalCluster._instances.clear()  # ignore test hygiene checks
 
 
 @pytest.mark.skipif(sys.version_info < (3, 6), reason="Unknown")
diff --git a/distributed/nanny.py b/distributed/nanny.py
index a27f713ea6b..8d2a38192d1 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -131,6 +131,7 @@ def __init__(
             "restart": self.restart,
             # cannot call it 'close' on the rpc side for naming conflict
             "terminate": self.close,
+            "close_gracefully": self.close_gracefully,
             "run": self.run,
         }
 
@@ -355,7 +356,7 @@ def _on_exit(self, exitcode):
                     return
 
             try:
-                if self.status not in ("closing", "closed"):
+                if self.status not in ("closing", "closed", "closing-gracefully"):
                     if self.auto_restart:
                         logger.warning("Restarting worker")
                         yield self.instantiate()
@@ -372,6 +373,14 @@ def _close(self, *args, **kwargs):
         warnings.warn("Worker._close has moved to Worker.close", stacklevel=2)
         return self.close(*args, **kwargs)
 
+    def close_gracefully(self, comm=None):
+        """
+        A signal that we shouldn't try to restart workers if they go away
+
+        This is used as part of the cluster shutdown process.
+        """
+        self.status = "closing-gracefully"
+
     @gen.coroutine
     def close(self, comm=None, timeout=5, report=None):
         """
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 9db6477aeb7..ca3c1241ea7 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1265,6 +1265,7 @@ def close(self, comm=None, fast=False, close_workers=False):
         setproctitle("dask-scheduler [closing]")
 
         if close_workers:
+            self.broadcast(msg={"op": "close_gracefully"}, nanny=True)
             for worker in self.workers:
                 self.worker_send(worker, {"op": "close"})
             for i in range(20):  # wait a second for send signals to clear
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index d61046f2a48..10784c6f759 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -44,6 +44,7 @@
 from .comm.utils import offload
 from .config import initialize_logging
 from .core import connect, rpc, CommClosedError
+from .deploy import SpecCluster
 from .metrics import time
 from .process import _cleanup_dangling
 from .proctitle import enable_proctitle_on_children
@@ -1477,6 +1478,7 @@ def check_instances():
     Client._instances.clear()
     Worker._instances.clear()
     Scheduler._instances.clear()
+    SpecCluster._instances.clear()
     # assert all(n.status == "closed" for n in Nanny._instances), {
     #     n: n.status for n in Nanny._instances
     # }
@@ -1514,6 +1516,10 @@ def check_instances():
         n: n.status for n in Nanny._instances
     }
 
+    # assert not list(SpecCluster._instances)  # TODO
+    assert all(c.status == "closed" for c in SpecCluster._instances)
+    SpecCluster._instances.clear()
+
     Nanny._instances.clear()
     DequeHandler.clear_all_instances()
 
diff --git a/distributed/worker.py b/distributed/worker.py
index 711dad31651..d0bc735ec67 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -1007,6 +1007,11 @@ def close(self, report=True, timeout=10, nanny=True, executor_wait=True):
             except ValueError:  # address not available if already closed
                 logger.info("Stopping worker")
             self.status = "closing"
+
+            if nanny and self.nanny:
+                with self.rpc(self.nanny) as r:
+                    yield r.close_gracefully()
+
             setproctitle("dask-worker [closing]")
 
             yield [
@@ -1015,7 +1020,6 @@ def close(self, report=True, timeout=10, nanny=True, executor_wait=True):
                 if hasattr(plugin, "teardown")
             ]
 
-            self.stop()
             for pc in self.periodic_callbacks.values():
                 pc.stop()
             with ignoring(EnvironmentError, gen.TimeoutError):
@@ -1047,6 +1051,7 @@ def close(self, report=True, timeout=10, nanny=True, executor_wait=True):
                 with self.rpc(self.nanny) as r:
                     yield r.terminate()
 
+            self.stop()
             self.rpc.close()
             self._closed.set()
 

From a8504d6d4a007ea5d427c2d17434b3dd22350e0a Mon Sep 17 00:00:00 2001
From: Benjamin Zaitlen <quasiben@users.noreply.github.com>
Date: Fri, 31 May 2019 09:51:47 -0400
Subject: [PATCH 0311/1550] Add Experimental UCX Comm (#2591)

---
 distributed/cli/dask_scheduler.py       |   5 +
 distributed/cli/dask_worker.py          |   5 +
 distributed/comm/__init__.py            |   5 +
 distributed/comm/addressing.py          |   4 +-
 distributed/comm/tests/__init__.py      |   0
 distributed/comm/tests/test_comms.py    |  14 +-
 distributed/comm/tests/test_ucx.py      | 296 +++++++++++++++++++++++
 distributed/comm/ucx.py                 | 308 ++++++++++++++++++++++++
 distributed/core.py                     |  10 +-
 distributed/deploy/local.py             |   8 +-
 distributed/preloading.py               |   2 +
 distributed/protocol/__init__.py        |  19 ++
 distributed/protocol/core.py            |   1 +
 distributed/protocol/cuda.py            |  33 +++
 distributed/protocol/cudf.py            |  74 ++++++
 distributed/protocol/cupy.py            |  42 ++++
 distributed/protocol/numba.py           |  61 +++++
 distributed/protocol/tests/test_cupy.py |  12 +
 distributed/protocol/utils.py           |   5 +-
 19 files changed, 893 insertions(+), 11 deletions(-)
 create mode 100644 distributed/comm/tests/__init__.py
 create mode 100644 distributed/comm/tests/test_ucx.py
 create mode 100644 distributed/comm/ucx.py
 create mode 100644 distributed/protocol/cuda.py
 create mode 100644 distributed/protocol/cudf.py
 create mode 100644 distributed/protocol/cupy.py
 create mode 100644 distributed/protocol/numba.py
 create mode 100644 distributed/protocol/tests/test_cupy.py

diff --git a/distributed/cli/dask_scheduler.py b/distributed/cli/dask_scheduler.py
index 1f78426f635..b27e68eaa9a 100755
--- a/distributed/cli/dask_scheduler.py
+++ b/distributed/cli/dask_scheduler.py
@@ -39,6 +39,9 @@
     default=None,
     help="Preferred network interface like 'eth0' or 'ib0'",
 )
+@click.option(
+    "--protocol", type=str, default=None, help="Protocol like tcp, tls, or ucx"
+)
 @click.option(
     "--tls-ca-file",
     type=pem_file_option_type,
@@ -121,6 +124,7 @@ def main(
     pid_file,
     scheduler_file,
     interface,
+    protocol,
     local_directory,
     preload,
     preload_argv,
@@ -190,6 +194,7 @@ def del_pid_file():
         host=host,
         port=port,
         interface=interface,
+        protocol=protocol,
         dashboard_address=dashboard_address if dashboard else None,
         service_kwargs={"dashboard": {"prefix": dashboard_prefix}},
     )
diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
index c4f83f61405..2cf570cfc1d 100755
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -104,6 +104,9 @@
 @click.option(
     "--interface", type=str, default=None, help="Network interface like 'eth0' or 'ib0'"
 )
+@click.option(
+    "--protocol", type=str, default=None, help="Protocol like tcp, tls, or ucx"
+)
 @click.option("--nthreads", type=int, default=0, help="Number of threads per process.")
 @click.option(
     "--nprocs",
@@ -197,6 +200,7 @@ def main(
     local_directory,
     scheduler_file,
     interface,
+    protocol,
     death_timeout,
     preload,
     preload_argv,
@@ -338,6 +342,7 @@ def del_pid_file():
             security=sec,
             contact_address=contact_address,
             interface=interface,
+            protocol=protocol,
             host=host,
             port=port,
             dashboard_address=dashboard_address if dashboard else None,
diff --git a/distributed/comm/__init__.py b/distributed/comm/__init__.py
index 0f7c701847d..e0615b38c7a 100644
--- a/distributed/comm/__init__.py
+++ b/distributed/comm/__init__.py
@@ -18,5 +18,10 @@ def _register_transports():
     from . import inproc
     from . import tcp
 
+    try:
+        from . import ucx
+    except ImportError:
+        pass
+
 
 _register_transports()
diff --git a/distributed/comm/addressing.py b/distributed/comm/addressing.py
index 3d79befe0f1..d707adb84ac 100644
--- a/distributed/comm/addressing.py
+++ b/distributed/comm/addressing.py
@@ -72,6 +72,8 @@ def _default():
             raise ValueError("missing port number in address %r" % (address,))
         return default_port
 
+    if "://" in address:
+        _, address = address.split("://")
     if address.startswith("["):
         # IPv6 notation: '[addr]:port' or '[addr]'.
         # The address may contain multiple colons.
@@ -101,7 +103,7 @@ def unparse_host_port(host, port=None):
     """
     if ":" in host and not host.startswith("["):
         host = "[%s]" % host
-    if port:
+    if port is not None:
         return "%s:%s" % (host, port)
     else:
         return host
diff --git a/distributed/comm/tests/__init__.py b/distributed/comm/tests/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/distributed/comm/tests/test_comms.py b/distributed/comm/tests/test_comms.py
index 0e8782718a0..e761deeab86 100644
--- a/distributed/comm/tests/test_comms.py
+++ b/distributed/comm/tests/test_comms.py
@@ -26,6 +26,7 @@
 
 from distributed.protocol import to_serialize, Serialized, serialize, deserialize
 
+from distributed.comm.registry import backends
 from distributed.comm import (
     tcp,
     inproc,
@@ -40,7 +41,6 @@
     get_local_address_for,
 )
 
-
 EXTERNAL_IP4 = get_ip()
 if has_ipv6():
     with warnings.catch_warnings(record=True):
@@ -154,7 +154,6 @@ def test_unparse_host_port():
     assert f("[::1]", 123) == "[::1]:123"
 
     assert f("127.0.0.1") == "127.0.0.1"
-    assert f("127.0.0.1", 0) == "127.0.0.1"
     assert f("127.0.0.1", None) == "127.0.0.1"
     assert f("127.0.0.1", "*") == "127.0.0.1:*"
 
@@ -488,7 +487,7 @@ def handle_comm(comm):
     # Check listener properties
     bound_addr = listener.listen_address
     bound_scheme, bound_loc = parse_address(bound_addr)
-    assert bound_scheme in ("inproc", "tcp", "tls")
+    assert bound_scheme in backends
     assert bound_scheme == parse_address(addr)[0]
 
     if check_listen_addr is not None:
@@ -530,6 +529,15 @@ def client_communicate(key, delay=0):
     listener.stop()
 
 
+@gen_test()
+def test_ucx_client_server():
+    pytest.importorskip("distributed.comm.ucx")
+    import ucp
+
+    addr = ucp.get_address()
+    yield check_client_server("ucx://" + addr)
+
+
 def tcp_eq(expected_host, expected_port=None):
     def checker(loc):
         host, port = parse_host_port(loc)
diff --git a/distributed/comm/tests/test_ucx.py b/distributed/comm/tests/test_ucx.py
new file mode 100644
index 00000000000..55a2f4ec82c
--- /dev/null
+++ b/distributed/comm/tests/test_ucx.py
@@ -0,0 +1,296 @@
+import asyncio
+
+import pytest
+
+ucp = pytest.importorskip("ucp")
+
+from distributed import Client
+from distributed.comm import ucx, listen, connect
+from distributed.comm.registry import backends, get_backend
+from distributed.comm import ucx, parse_address
+from distributed.protocol import to_serialize
+from distributed.deploy.local import LocalCluster
+from distributed.utils_test import gen_test, loop, inc  # noqa: 401
+
+from .test_comms import check_deserialize
+
+
+HOST = ucp.get_address()
+
+
+def test_registered():
+    assert "ucx" in backends
+    backend = get_backend("ucx")
+    assert isinstance(backend, ucx.UCXBackend)
+
+
+async def get_comm_pair(
+    listen_addr="ucx://" + HOST, listen_args=None, connect_args=None, **kwargs
+):
+    q = asyncio.queues.Queue()
+
+    async def handle_comm(comm):
+        await q.put(comm)
+
+    # Workaround for hanging test in
+    # pytest distributed/comm/tests/test_ucx.py::test_comm_objs -vs --count=2
+    # on the second time through.
+    ucp._libs.ucp_py.reader_added = 0
+
+    listener = listen(listen_addr, handle_comm, connection_args=listen_args, **kwargs)
+    with listener:
+        comm = await connect(
+            listener.contact_address, connection_args=connect_args, **kwargs
+        )
+        serv_com = await q.get()
+        return comm, serv_com
+
+
+@pytest.mark.asyncio
+async def test_ping_pong():
+    com, serv_com = await get_comm_pair()
+    msg = {"op": "ping"}
+    await com.write(msg)
+    result = await serv_com.read()
+    assert result == msg
+    result["op"] = "pong"
+
+    await serv_com.write(result)
+
+    result = await com.read()
+    assert result == {"op": "pong"}
+
+    await com.close()
+    await serv_com.close()
+
+
+@pytest.mark.asyncio
+async def test_comm_objs():
+    comm, serv_comm = await get_comm_pair()
+
+    scheme, loc = parse_address(comm.peer_address)
+    assert scheme == "ucx"
+
+    scheme, loc = parse_address(serv_comm.peer_address)
+    assert scheme == "ucx"
+
+    assert comm.peer_address == serv_comm.local_address
+
+
+def test_ucx_specific():
+    """
+    Test concrete UCX API.
+    """
+    # TODO:
+    # 1. ensure exceptions in handle_comm fail the test
+    # 2. Use dict in read / write, put seralization there.
+    # 3. Test peer_address
+    # 4. Test cleanup
+    async def f():
+        address = "ucx://{}:{}".format(HOST, 0)
+
+        async def handle_comm(comm):
+            msg = await comm.read()
+            msg["op"] = "pong"
+            await comm.write(msg)
+            assert comm.closed() is False
+            await comm.close()
+            assert comm.closed
+
+        listener = ucx.UCXListener(address, handle_comm)
+        listener.start()
+        host, port = listener.get_host_port()
+        assert host.count(".") == 3
+        assert port > 0
+
+        connector = ucx.UCXConnector()
+        l = []
+
+        async def client_communicate(key, delay=0):
+            addr = "%s:%d" % (host, port)
+            comm = await connector.connect(addr)
+            # TODO: peer_address
+            # assert comm.peer_address == 'ucx://' + addr
+            assert comm.extra_info == {}
+            msg = {"op": "ping", "data": key}
+            await comm.write(msg)
+            if delay:
+                await asyncio.sleep(delay)
+            msg = await comm.read()
+            assert msg == {"op": "pong", "data": key}
+            l.append(key)
+            return comm
+            assert comm.closed() is False
+            await comm.close()
+            assert comm.closed
+
+        comm = await client_communicate(key=1234, delay=0.5)
+
+        # Many clients at once
+        N = 2
+        futures = [client_communicate(key=i, delay=0.05) for i in range(N)]
+        await asyncio.gather(*futures)
+        assert set(l) == {1234} | set(range(N))
+
+    asyncio.run(f())
+
+
+@pytest.mark.asyncio
+async def test_ping_pong_data():
+    np = pytest.importorskip("numpy")
+
+    data = np.ones((10, 10))
+
+    com, serv_com = await get_comm_pair()
+    msg = {"op": "ping", "data": to_serialize(data)}
+    await com.write(msg)
+    result = await serv_com.read()
+    result["op"] = "pong"
+    data2 = result.pop("data")
+    np.testing.assert_array_equal(data2, data)
+
+    await serv_com.write(result)
+
+    result = await com.read()
+    assert result == {"op": "pong"}
+
+    await com.close()
+    await serv_com.close()
+
+
+@gen_test()
+def test_ucx_deserialize():
+    yield check_deserialize("tcp://")
+
+
+@pytest.mark.asyncio
+async def test_ping_pong_cudf():
+    # if this test appears after cupy an import error arises
+    # *** ImportError: /usr/lib/x86_64-linux-gnu/libstdc++.so.6: version `CXXABI_1.3.11'
+    # not found (required by python3.7/site-packages/pyarrow/../../../libarrow.so.12)
+    cudf = pytest.importorskip("cudf")
+
+    df = cudf.DataFrame({"A": [1, 2, None], "B": [1.0, 2.0, None]})
+
+    com, serv_com = await get_comm_pair()
+    msg = {"op": "ping", "data": to_serialize(df)}
+
+    await com.write(msg)
+    result = await serv_com.read()
+    data2 = result.pop("data")
+    assert result["op"] == "ping"
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("shape", [(100,), (10, 10), (4947,)])
+async def test_ping_pong_cupy(shape):
+    cupy = pytest.importorskip("cupy")
+    com, serv_com = await get_comm_pair()
+
+    arr = cupy.random.random(shape)
+    msg = {"op": "ping", "data": to_serialize(arr)}
+
+    _, result = await asyncio.gather(com.write(msg), serv_com.read())
+    data2 = result.pop("data")
+
+    assert result["op"] == "ping"
+    cupy.testing.assert_array_equal(arr, data2)
+    await com.close()
+    await serv_com.close()
+
+
+@pytest.mark.slow
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "n",
+    [
+        int(1e9),
+        pytest.param(
+            int(2.5e9), marks=[pytest.mark.xfail(reason="integer type in ucx-py")]
+        ),
+    ],
+)
+async def test_large_cupy(n):
+    cupy = pytest.importorskip("cupy")
+    com, serv_com = await get_comm_pair()
+
+    arr = cupy.ones(n, dtype="u1")
+    msg = {"op": "ping", "data": to_serialize(arr)}
+
+    _, result = await asyncio.gather(com.write(msg), serv_com.read())
+    data2 = result.pop("data")
+
+    assert result["op"] == "ping"
+    assert len(data2) == len(arr)
+    await com.close()
+    await serv_com.close()
+
+
+@pytest.mark.asyncio
+async def test_ping_pong_numba():
+    np = pytest.importorskip("numpy")
+    numba = pytest.importorskip("numba")
+    import numba.cuda
+
+    arr = np.arange(10)
+    arr = numba.cuda.to_device(arr)
+
+    com, serv_com = await get_comm_pair()
+    msg = {"op": "ping", "data": to_serialize(arr)}
+
+    await com.write(msg)
+    result = await serv_com.read()
+    data2 = result.pop("data")
+    assert result["op"] == "ping"
+
+
+@pytest.mark.skip(reason="hangs")
+@pytest.mark.parametrize("processes", [True, False])
+def test_ucx_localcluster(loop, processes):
+    if processes:
+        kwargs = {"env": {"UCX_MEMTYPE_CACHE": "n"}}
+    else:
+        kwargs = {}
+
+    ucx_addr = ucp.get_address()
+    with LocalCluster(
+        protocol="ucx",
+        interface="ib0",
+        dashboard_address=None,
+        n_workers=2,
+        threads_per_worker=1,
+        processes=processes,
+        loop=loop,
+        **kwargs,
+    ) as cluster:
+        with Client(cluster) as client:
+            x = client.submit(inc, 1)
+            x.result()
+            assert x.key in cluster.scheduler.tasks
+            if not processes:
+                assert any(w.data == {x.key: 2} for w in cluster.workers.values())
+            assert len(cluster.scheduler.workers) == 2
+
+
+def test_tcp_localcluster(loop):
+    ucx_addr = "127.0.0.1"
+    port = 13337
+    env = {"UCX_MEMTYPE_CACHE": "n"}
+    with LocalCluster(
+        2,
+        scheduler_port=port,
+        ip=ucx_addr,
+        processes=True,
+        threads_per_worker=1,
+        dashboard_address=None,
+        silence_logs=False,
+        env=env,
+    ) as cluster:
+        pass
+        # with Client(cluster) as e:
+        #     x = e.submit(inc, 1)
+        #     x.result()
+        #     assert x.key in c.scheduler.tasks
+        #     assert any(w.data == {x.key: 2} for w in c.workers)
+        #     assert e.loop is c.loop
+        #     print(c.scheduler.workers)
diff --git a/distributed/comm/ucx.py b/distributed/comm/ucx.py
new file mode 100644
index 00000000000..3f3f0bfe943
--- /dev/null
+++ b/distributed/comm/ucx.py
@@ -0,0 +1,308 @@
+"""
+:ref:`UCX`_ based communications for distributed.
+
+See :ref:`communications` for more.
+
+.. _UCX: https://github.com/openucx/ucx
+"""
+import asyncio
+import logging
+import struct
+
+from .addressing import parse_host_port, unparse_host_port
+from .core import Comm, Connector, Listener, CommClosedError
+from .registry import Backend, backends
+from .utils import ensure_concrete_host, to_frames, from_frames
+from ..utils import ensure_ip, get_ip, get_ipv6, nbytes
+
+import ucp
+
+import os
+
+os.environ.setdefault("UCX_RNDV_SCHEME", "put_zcopy")
+os.environ.setdefault("UCX_MEMTYPE_CACHE", "n")
+os.environ.setdefault("UCX_TLS", "rc,cuda_copy")
+
+logger = logging.getLogger(__name__)
+MAX_MSG_LOG = 23
+
+
+# ----------------------------------------------------------------------------
+# Comm Interface
+# ----------------------------------------------------------------------------
+
+
+class UCX(Comm):
+    """Comm object using UCP.
+
+    Parameters
+    ----------
+    ep : ucp.Endpoint
+        The UCP endpoint.
+    address : str
+        The address, prefixed with `ucx://` to use.
+    deserialize : bool, default True
+        Whether to deserialize data in :meth:`distributed.protocol.loads`
+
+    Notes
+    -----
+    The read-write cycle uses the following pattern:
+
+    Each msg is serialized into a number of "data" frames. We prepend these
+    real frames with two additional frames
+
+        1. is_gpus: Boolean indicator for whether the frame should be
+           received into GPU memory. Packed in '?' format. Unpack with
+           ``<n_frames>?`` format.
+        2. frame_size : Unsigned int describing the size of frame (in bytes)
+           to receive. Packed in 'Q' format, so a length-0 frame is equivalent
+           to an unsized frame. Unpacked with ``<n_frames>Q``.
+
+    The expected read cycle is
+
+    1. Read the frame describing number of frames
+    2. Read the frame describing whether each data frame is gpu-bound
+    3. Read the frame describing whether each data frame is sized
+    4. Read all the data frames.
+    """
+
+    def __init__(
+        self, ep: ucp.Endpoint, local_addr: str, peer_addr: str, deserialize=True
+    ):
+        Comm.__init__(self)
+        self._ep = ep
+        if local_addr:
+            assert local_addr.startswith("ucx")
+        assert peer_addr.startswith("ucx")
+        self._local_addr = local_addr
+        self._peer_addr = peer_addr
+        self.deserialize = deserialize
+        self.comm_flag = None
+        logger.debug("UCX.__init__ %s", self)
+
+    @property
+    def local_address(self) -> str:
+        return self._local_addr
+
+    @property
+    def peer_address(self) -> str:
+        return self._peer_addr
+
+    async def write(
+        self,
+        msg: dict,
+        serializers=("cuda", "dask", "pickle", "error"),
+        on_error: str = "message",
+    ):
+        if serializers is None:
+            serializers = ("cuda", "dask", "pickle", "error")
+        # msg can also be a list of dicts when sending batched messages
+        frames = await to_frames(msg, serializers=serializers, on_error=on_error)
+        is_gpus = b"".join(
+            [
+                struct.pack("?", hasattr(frame, "__cuda_array_interface__"))
+                for frame in frames
+            ]
+        )
+        sizes = b"".join([struct.pack("Q", nbytes(frame)) for frame in frames])
+
+        nframes = struct.pack("Q", len(frames))
+
+        meta = b"".join([nframes, is_gpus, sizes])
+
+        await self.ep.send_obj(meta)
+
+        for frame in frames:
+            await self.ep.send_obj(frame)
+        return sum(map(nbytes, frames))
+
+    async def read(self, deserializers=("cuda", "dask", "pickle", "error")):
+        if deserializers is None:
+            deserializers = ("cuda", "dask", "pickle", "error")
+        resp = await self.ep.recv_future()
+        obj = ucp.get_obj_from_msg(resp)
+        nframes, = struct.unpack("Q", obj[:8])  # first eight bytes for number of frames
+
+        gpu_frame_msg = obj[
+            8 : 8 + nframes
+        ]  # next nframes bytes for if they're GPU frames
+        is_gpus = struct.unpack("{}?".format(nframes), gpu_frame_msg)
+
+        sized_frame_msg = obj[8 + nframes :]  # then the rest for frame sizes
+        sizes = struct.unpack("{}Q".format(nframes), sized_frame_msg)
+
+        frames = []
+
+        for i, (is_gpu, size) in enumerate(zip(is_gpus, sizes)):
+            if size > 0:
+                resp = await self.ep.recv_obj(size, cuda=is_gpu)
+            else:
+                resp = await self.ep.recv_future()
+            frame = ucp.get_obj_from_msg(resp)
+            frames.append(frame)
+
+        msg = await from_frames(
+            frames, deserialize=self.deserialize, deserializers=deserializers
+        )
+
+        return msg
+
+    def abort(self):
+        if self._ep:
+            ucp.destroy_ep(self._ep)
+            logger.debug("Destroyed UCX endpoint")
+            self._ep = None
+
+    @property
+    def ep(self):
+        if self._ep:
+            return self._ep
+        else:
+            raise CommClosedError("UCX Endpoint is closed")
+
+    async def close(self):
+        # TODO: Handle in-flight messages?
+        # sleep is currently used to help flush buffer
+        self.abort()
+
+    def closed(self):
+        return self._ep is None
+
+
+class UCXConnector(Connector):
+    prefix = "ucx://"
+    comm_class = UCX
+    encrypted = False
+
+    async def connect(self, address: str, deserialize=True, **connection_args) -> UCX:
+        logger.debug("UCXConnector.connect: %s", address)
+        ucp.init()
+        ip, port = parse_host_port(address)
+        ep = await ucp.get_endpoint(ip.encode(), port)
+        return self.comm_class(
+            ep,
+            local_addr=None,
+            peer_addr=self.prefix + address,
+            deserialize=deserialize,
+        )
+
+
+class UCXListener(Listener):
+    # MAX_LISTENERS 256 in ucx-py
+    prefix = UCXConnector.prefix
+    comm_class = UCXConnector.comm_class
+    encrypted = UCXConnector.encrypted
+
+    def __init__(
+        self, address: str, comm_handler: None, deserialize=False, **connection_args
+    ):
+        if not address.startswith("ucx"):
+            address = "ucx://" + address
+        self.ip, self._input_port = parse_host_port(address, default_port=0)
+        self.comm_handler = comm_handler
+        self.deserialize = deserialize
+        self._ep = None  # type: ucp.Endpoint
+        self.listener_instance = None  # type: ucp.ListenerFuture
+        self.ucp_server = None
+        self._task = None
+
+        self.connection_args = connection_args
+        self._task = None
+
+    @property
+    def port(self):
+        return self.ucp_server.port
+
+    @property
+    def address(self):
+        return "ucx://" + self.ip + ":" + str(self.port)
+
+    def start(self):
+        async def serve_forever(client_ep, listener_instance):
+            ucx = UCX(
+                client_ep,
+                local_addr=self.address,
+                peer_addr=self.address,  # TODO: https://github.com/Akshay-Venkatesh/ucx-py/issues/111
+                deserialize=self.deserialize,
+            )
+            self.listener_instance = listener_instance
+            if self.comm_handler:
+                await self.comm_handler(ucx)
+
+        ucp.init()
+        self.ucp_server = ucp.start_listener(
+            serve_forever, listener_port=self._input_port, is_coroutine=True
+        )
+
+        try:
+            loop = asyncio.get_running_loop()
+        except (RuntimeError, AttributeError):
+            loop = asyncio.get_event_loop()
+
+        t = loop.create_task(self.ucp_server.coroutine)
+        self._task = t
+
+    def stop(self):
+        # What all should this do?
+        if self._task:
+            self._task.cancel()
+
+        if self._ep:
+            ucp.destroy_ep(self._ep)
+        # if self.listener_instance:
+        #   ucp.stop_listener(self.listener_instance)
+
+    def get_host_port(self):
+        # TODO: TCP raises if this hasn't started yet.
+        return self.ip, self.port
+
+    @property
+    def listen_address(self):
+        return self.prefix + unparse_host_port(*self.get_host_port())
+
+    @property
+    def contact_address(self):
+        host, port = self.get_host_port()
+        host = ensure_concrete_host(host)  # TODO: ensure_concrete_host
+        return self.prefix + unparse_host_port(host, port)
+
+    @property
+    def bound_address(self):
+        # TODO: Does this become part of the base API? Kinda hazy, since
+        # we exclude in for inproc.
+        return self.get_host_port()
+
+
+class UCXBackend(Backend):
+    # I / O
+
+    def get_connector(self):
+        return UCXConnector()
+
+    def get_listener(self, loc, handle_comm, deserialize, **connection_args):
+        return UCXListener(loc, handle_comm, deserialize, **connection_args)
+
+    # Address handling
+    # This duplicates BaseTCPBackend
+
+    def get_address_host(self, loc):
+        return parse_host_port(loc)[0]
+
+    def get_address_host_port(self, loc):
+        return parse_host_port(loc)
+
+    def resolve_address(self, loc):
+        host, port = parse_host_port(loc)
+        return unparse_host_port(ensure_ip(host), port)
+
+    def get_local_address_for(self, loc):
+        host, port = parse_host_port(loc)
+        host = ensure_ip(host)
+        if ":" in host:
+            local_host = get_ipv6(host)
+        else:
+            local_host = get_ip(host)
+        return unparse_host_port(local_host, None)
+
+
+backends["ucx"] = UCXBackend()
diff --git a/distributed/core.py b/distributed/core.py
index 17685c9d2d5..79c726eed6d 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -484,7 +484,7 @@ def handle_stream(self, comm, extra=None, every_cycle=[]):
                 pdb.set_trace()
             raise
         finally:
-            comm.close()  # TODO: why do we need this now?
+            yield comm.close()
             assert comm.closed()
 
     @gen.coroutine
@@ -498,7 +498,7 @@ def close(self):
                 break
             else:
                 yield gen.sleep(0.05)
-        yield [comm.close() for comm in self._comms]
+        yield [comm.close() for comm in self._comms]  # then forcefully close
         for cb in self._ongoing_coroutines:
             cb.cancel()
         for i in range(10):
@@ -901,7 +901,7 @@ def collect(self):
         )
         for addr, comms in self.available.items():
             for comm in comms:
-                comm.close()
+                IOLoop.current().add_callback(comm.close)
             comms.clear()
         if self.open < self.limit:
             self.event.set()
@@ -914,11 +914,11 @@ def remove(self, addr):
         if addr in self.available:
             comms = self.available.pop(addr)
             for comm in comms:
-                comm.close()
+                IOLoop.current().add_callback(comm.close)
         if addr in self.occupied:
             comms = self.occupied.pop(addr)
             for comm in comms:
-                comm.close()
+                IOLoop.current().add_callback(comm.close)
         if self.open < self.limit:
             self.event.set()
 
diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index 298c47d7a31..95f178c7c2e 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -195,7 +195,13 @@ def __init__(
             asynchronous=asynchronous,
             silence_logs=silence_logs,
         )
-        self.scale(n_workers)
+
+    def __repr__(self):
+        return "LocalCluster(%r, workers=%d, ncores=%d)" % (
+            self.scheduler_address,
+            len(self.workers),
+            sum(w.ncores for w in self.workers.values()),
+        )
 
     def __repr__(self):
         return "LocalCluster(%r, workers=%d, ncores=%d)" % (
diff --git a/distributed/preloading.py b/distributed/preloading.py
index 0f08f60f71c..a5e67c1611a 100644
--- a/distributed/preloading.py
+++ b/distributed/preloading.py
@@ -100,6 +100,7 @@ def _import_modules(names, file_dir=None):
                 import_module(name)
             module = sys.modules[name]
 
+        logger.info("Import preload module: %s", name)
         result_modules[name] = {
             attrname: getattr(module, attrname, None)
             for attrname in ("dask_setup", "dask_teardown")
@@ -137,6 +138,7 @@ def preload_modules(names, parameter=None, file_dir=None, argv=None):
                 dask_setup.callback(parameter, *context.args, **context.params)
             else:
                 dask_setup(parameter)
+                logger.info("Run preload setup function: %s", name)
 
         if interface["dask_teardown"]:
             atexit.register(interface["dask_teardown"], parameter)
diff --git a/distributed/protocol/__init__.py b/distributed/protocol/__init__.py
index 04691ce605d..3f98436f4b9 100644
--- a/distributed/protocol/__init__.py
+++ b/distributed/protocol/__init__.py
@@ -4,6 +4,7 @@
 
 from .compression import compressions, default_compression
 from .core import dumps, loads, maybe_compress, decompress, msgpack
+from .cuda import cuda_serialize, cuda_deserialize
 from .serialize import (
     serialize,
     deserialize,
@@ -66,3 +67,21 @@ def _register_arrow():
 @dask_deserialize.register_lazy("torchvision")
 def _register_torch():
     from . import torch
+
+
+@cuda_serialize.register_lazy("cupy")
+@cuda_deserialize.register_lazy("cupy")
+def _register_cupy():
+    from . import cupy
+
+
+@cuda_serialize.register_lazy("numba")
+@cuda_deserialize.register_lazy("numba")
+def _register_numba():
+    from . import numba
+
+
+@cuda_serialize.register_lazy("cudf")
+@cuda_deserialize.register_lazy("cudf")
+def _register_cudf():
+    from . import cudf
diff --git a/distributed/protocol/core.py b/distributed/protocol/core.py
index c1b62b2491e..d54dd2e533e 100644
--- a/distributed/protocol/core.py
+++ b/distributed/protocol/core.py
@@ -176,6 +176,7 @@ def loads_msgpack(header, payload):
     See Also:
         dumps_msgpack
     """
+    header = bytes(header)
     if header:
         header = msgpack.loads(header, use_list=False, **msgpack_opts)
     else:
diff --git a/distributed/protocol/cuda.py b/distributed/protocol/cuda.py
new file mode 100644
index 00000000000..13be1d75bb8
--- /dev/null
+++ b/distributed/protocol/cuda.py
@@ -0,0 +1,33 @@
+import dask
+
+from . import pickle
+from .serialize import register_serialization_family
+from dask.utils import typename
+
+cuda_serialize = dask.utils.Dispatch("cuda_serialize")
+cuda_deserialize = dask.utils.Dispatch("cuda_deserialize")
+
+
+def cuda_dumps(x):
+    type_name = typename(type(x))
+    try:
+        dumps = cuda_serialize.dispatch(type(x))
+    except TypeError:
+        raise NotImplementedError(type_name)
+
+    header, frames = dumps(x)
+
+    header["type"] = type_name
+    header["type-serialized"] = pickle.dumps(type(x))
+    header["serializer"] = "cuda"
+    header["compression"] = (None,) * len(frames)  # no compression for gpu data
+    return header, frames
+
+
+def cuda_loads(header, frames):
+    typ = pickle.loads(header["type-serialized"])
+    loads = cuda_deserialize.dispatch(typ)
+    return loads(header, frames)
+
+
+register_serialization_family("cuda", cuda_dumps, cuda_loads)
diff --git a/distributed/protocol/cudf.py b/distributed/protocol/cudf.py
new file mode 100644
index 00000000000..018596b1560
--- /dev/null
+++ b/distributed/protocol/cudf.py
@@ -0,0 +1,74 @@
+import cudf
+from .cuda import cuda_serialize, cuda_deserialize
+from .numba import serialize_numba_ndarray, deserialize_numba_ndarray
+
+
+# TODO:
+# 1. Just use positions
+#    a. Fixes duplicate columns
+#    b. Fixes non-msgpack-serializable names
+# 2. cudf.Series
+# 3. Serialize the index
+
+
+@cuda_serialize.register(cudf.DataFrame)
+def serialize_cudf_dataframe(x):
+    sub_headers = []
+    arrays = []
+    null_masks = []
+    null_headers = []
+    null_counts = {}
+
+    for label, col in x.iteritems():
+        header, [frame] = serialize_numba_ndarray(col.data.mem)
+        header["name"] = label
+        sub_headers.append(header)
+        arrays.append(frame)
+        if col.null_count:
+            header, [frame] = serialize_numba_ndarray(col.nullmask.mem)
+            header["name"] = label
+            null_headers.append(header)
+            null_masks.append(frame)
+            null_counts[label] = col.null_count
+
+    arrays.extend(null_masks)
+
+    header = {
+        "is_cuda": len(arrays),
+        "subheaders": sub_headers,
+        # TODO: the header must be msgpack (de)serializable.
+        # See if we can avoid names, and just use integer positions.
+        "columns": x.columns.tolist(),
+        "null_counts": null_counts,
+        "null_subheaders": null_headers,
+    }
+
+    return header, arrays
+
+
+@cuda_deserialize.register(cudf.DataFrame)
+def serialize_cudf_dataframe(header, frames):
+    columns = header["columns"]
+    n_columns = len(header["columns"])
+    n_masks = len(header["null_subheaders"])
+
+    masks = {}
+    pairs = []
+
+    for i in range(n_masks):
+        subheader = header["null_subheaders"][i]
+        frame = frames[n_columns + i]
+        mask = deserialize_numba_ndarray(subheader, [frame])
+        masks[subheader["name"]] = mask
+
+    for subheader, frame in zip(header["subheaders"], frames[:n_columns]):
+        name = subheader["name"]
+        array = deserialize_numba_ndarray(subheader, [frame])
+
+        if name in masks:
+            series = cudf.Series.from_masked_array(array, masks[name])
+        else:
+            series = cudf.Series(array)
+        pairs.append((name, series))
+
+    return cudf.DataFrame(pairs)
diff --git a/distributed/protocol/cupy.py b/distributed/protocol/cupy.py
new file mode 100644
index 00000000000..13c0348a821
--- /dev/null
+++ b/distributed/protocol/cupy.py
@@ -0,0 +1,42 @@
+"""
+Efficient serialization GPU arrays.
+"""
+import cupy
+from .cuda import cuda_serialize, cuda_deserialize
+
+
+@cuda_serialize.register(cupy.ndarray)
+def serialize_cupy_ndarray(x):
+    # TODO: handle non-contiguous
+    # TODO: Handle order='K' ravel
+    # TODO: 0d
+
+    if x.flags.c_contiguous or x.flags.f_contiguous:
+        strides = x.strides
+        data = x.ravel()  # order='K'
+    else:
+        x = cupy.ascontiguousarray(x)
+        strides = x.strides
+        data = x.ravel()
+
+    dtype = (0, x.dtype.str)
+
+    # used in the ucx comms for gpu/cpu message passing
+    # 'lengths' set by dask
+    header = x.__cuda_array_interface__.copy()
+    header["is_cuda"] = 1
+    header["dtype"] = dtype
+    return header, [data]
+
+
+@cuda_deserialize.register(cupy.ndarray)
+def deserialize_cupy_array(header, frames):
+    frame, = frames
+    # TODO: put this in ucx... as a kind of "fixup"
+    try:
+        frame.typestr = header["typestr"]
+        frame.shape = header["shape"]
+    except AttributeError:
+        pass
+    arr = cupy.asarray(frame)
+    return arr
diff --git a/distributed/protocol/numba.py b/distributed/protocol/numba.py
new file mode 100644
index 00000000000..18405ffebe0
--- /dev/null
+++ b/distributed/protocol/numba.py
@@ -0,0 +1,61 @@
+import numba.cuda
+from .cuda import cuda_serialize, cuda_deserialize
+
+
+@cuda_serialize.register(numba.cuda.devicearray.DeviceNDArray)
+def serialize_numba_ndarray(x):
+    # TODO: handle non-contiguous
+    # TODO: handle 2d
+    # TODO: 0d
+
+    if x.flags["C_CONTIGUOUS"] or x.flags["F_CONTIGUOUS"]:
+        strides = x.strides
+        if x.ndim > 1:
+            data = x.ravel()  # order='K'
+        else:
+            data = x
+    else:
+        raise ValueError("Array must be contiguous")
+        x = numba.ascontiguousarray(x)
+        strides = x.strides
+        if x.ndim > 1:
+            data = x.ravel()
+        else:
+            data = x
+
+    dtype = (0, x.dtype.str)
+    nbytes = data.dtype.itemsize * data.size
+
+    # used in the ucx comms for gpu/cpu message passing
+    # 'lengths' set by dask
+    header = x.__cuda_array_interface__.copy()
+    header["is_cuda"] = 1
+    header["dtype"] = dtype
+    return header, [data]
+
+
+@cuda_deserialize.register(numba.cuda.devicearray.DeviceNDArray)
+def deserialize_numba_ndarray(header, frames):
+    frame, = frames
+    # TODO: put this in ucx... as a kind of "fixup"
+    if isinstance(frame, bytes):
+        import numpy as np
+
+        arr2 = np.frombuffer(frame, header["typestr"])
+        return numba.cuda.to_device(arr2)
+
+    frame.typestr = header["typestr"]
+    frame.shape = header["shape"]
+
+    # numba & cupy don't properly roundtrip length-zero arrays.
+    if frame.shape[0] == 0:
+        arr = numba.cuda.device_array(
+            header["shape"],
+            header["typestr"]
+            # strides?
+            # order?
+        )
+        return arr
+
+    arr = numba.cuda.as_cuda_array(frame)
+    return arr
diff --git a/distributed/protocol/tests/test_cupy.py b/distributed/protocol/tests/test_cupy.py
new file mode 100644
index 00000000000..26940597f81
--- /dev/null
+++ b/distributed/protocol/tests/test_cupy.py
@@ -0,0 +1,12 @@
+from distributed.protocol import serialize, deserialize
+import pytest
+
+cupy = pytest.importorskip("cupy")
+
+
+def test_serialize_cupy():
+    x = cupy.arange(100)
+    header, frames = serialize(x, serializers=("cuda", "dask", "pickle"))
+    y = deserialize(header, frames, deserializers=("cuda", "dask", "pickle", "error"))
+
+    assert (x == y).all()
diff --git a/distributed/protocol/utils.py b/distributed/protocol/utils.py
index 208caebb926..caf4bb8833b 100644
--- a/distributed/protocol/utils.py
+++ b/distributed/protocol/utils.py
@@ -90,7 +90,10 @@ def merge_frames(header, frames):
                 L.append(mv[:l])
                 frames.append(mv[l:])
                 l = 0
-        out.append(b"".join(map(ensure_bytes, L)))
+        if len(L) == 1:  # no work necessary
+            out.extend(L)
+        else:
+            out.append(b"".join(map(ensure_bytes, L)))
     return out
 
 

From 7c3b4d1c59b74b39ccfc0f579ce8295713bfb15a Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 31 May 2019 11:04:13 -0700
Subject: [PATCH 0312/1550] Pin pytest >=4 with pip in appveyor and python 3.5
 (#2737)

---
 README.rst                                         | 1 +
 continuous_integration/setup_conda_environment.cmd | 2 +-
 continuous_integration/travis/install.sh           | 2 +-
 3 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/README.rst b/README.rst
index 2d2d285be2b..b6f0edd604f 100644
--- a/README.rst
+++ b/README.rst
@@ -3,4 +3,5 @@ Distributed
 
 A library for distributed computation.  See documentation_ for more details.
 
+
 .. _documentation: https://distributed.readthedocs.io/en/latest
diff --git a/continuous_integration/setup_conda_environment.cmd b/continuous_integration/setup_conda_environment.cmd
index 5748a8cf20c..3df89fa85fe 100644
--- a/continuous_integration/setup_conda_environment.cmd
+++ b/continuous_integration/setup_conda_environment.cmd
@@ -50,7 +50,7 @@ call activate %CONDA_ENV%
 %PIP_INSTALL% git+https://github.com/joblib/joblib.git --upgrade
 %PIP_INSTALL% git+https://github.com/dask/zict --upgrade
 
-%PIP_INSTALL% pytest-repeat pytest-timeout pytest-faulthandler sortedcollections pytest-asyncio
+%PIP_INSTALL% "pytest>=4" pytest-repeat pytest-timeout pytest-faulthandler sortedcollections pytest-asyncio
 
 @rem Display final environment (for reproducing)
 %CONDA% list
diff --git a/continuous_integration/travis/install.sh b/continuous_integration/travis/install.sh
index f1ff25a9bfa..cb2dbdf5c83 100644
--- a/continuous_integration/travis/install.sh
+++ b/continuous_integration/travis/install.sh
@@ -59,7 +59,7 @@ conda install -q \
 conda install -c defaults -c conda-forge libunwind
 conda install --no-deps -c defaults -c numba -c conda-forge stacktrace
 
-pip install -q pytest-repeat pytest-faulthandler pytest-asyncio
+pip install -q "pytest>=4" pytest-repeat pytest-faulthandler pytest-asyncio
 
 pip install -q git+https://github.com/dask/dask.git --upgrade --no-deps
 pip install -q git+https://github.com/joblib/joblib.git --upgrade --no-deps

From a16b8ff071938c08f64e78bb04636c3b4d619325 Mon Sep 17 00:00:00 2001
From: Caleb <caleb.yh.ho@gmail.com>
Date: Fri, 31 May 2019 16:03:39 -0700
Subject: [PATCH 0313/1550] Allow user to configure whether workers are daemon.
 (#2739)

Closes #2718
---
 .gitignore                      |  3 +++
 distributed/distributed.yaml    |  1 +
 distributed/nanny.py            |  2 +-
 distributed/process.py          |  2 +-
 distributed/tests/test_nanny.py | 35 +++++++++++++++++++++++++++++++++
 5 files changed, 41 insertions(+), 2 deletions(-)

diff --git a/.gitignore b/.gitignore
index a3a40e19289..2d70b7ebd7f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -13,3 +13,6 @@ continuous_integration/hdfs-initialized
 .pytest_cache/
 dask-worker-space/
 .vscode/
+*.swp
+.ycm_extra_conf.py
+tags
diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index 3ae9b7ee690..4d78a698e69 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -32,6 +32,7 @@ distributed:
       incoming: 10
     preload: []
     preload-argv: []
+    daemon: True
 
     profile:
       interval: 10ms        # Time between statistical profiling queries
diff --git a/distributed/nanny.py b/distributed/nanny.py
index 8d2a38192d1..59a8083e832 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -464,7 +464,7 @@ def start(self):
                 env=self.env,
             ),
         )
-        self.process.daemon = True
+        self.process.daemon = dask.config.get("distributed.worker.daemon", default=True)
         self.process.set_exit_callback(self._on_exit)
         self.running = Event()
         self.stopped = Event()
diff --git a/distributed/process.py b/distributed/process.py
index 5dd9368fdc1..556edae290e 100644
--- a/distributed/process.py
+++ b/distributed/process.py
@@ -330,7 +330,7 @@ def daemon(self, value):
 @atexit.register
 def _cleanup_dangling():
     for proc in list(_dangling):
-        if proc.daemon and proc.is_alive():
+        if proc.is_alive():
             try:
                 logger.warning("reaping stray process %s" % (proc,))
                 proc.terminate()
diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
index be0a05afc20..1357a3679e2 100644
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -5,6 +5,7 @@
 import os
 import random
 import sys
+import multiprocessing as mp
 
 import numpy as np
 
@@ -344,3 +345,37 @@ def test_data_types(c, s):
     r = yield c.run(lambda dask_worker: type(dask_worker.data))
     assert r[w.worker_address] == dict
     yield w.close()
+
+
+def _noop(x):
+    """Define here because closures aren't pickleable."""
+    pass
+
+
+@gen_cluster(
+    ncores=[("127.0.0.1", 1)],
+    client=True,
+    Worker=Nanny,
+    config={"distributed.worker.daemon": False},
+)
+def test_mp_process_worker_no_daemon(c, s, a):
+    def multiprocessing_worker():
+        p = mp.Process(target=_noop, args=(None,))
+        p.start()
+        p.join()
+
+    yield c.submit(multiprocessing_worker)
+
+
+@gen_cluster(
+    ncores=[("127.0.0.1", 1)],
+    client=True,
+    Worker=Nanny,
+    config={"distributed.worker.daemon": False},
+)
+def test_mp_pool_worker_no_daemon(c, s, a):
+    def pool_worker(world_size):
+        with mp.Pool(processes=world_size) as p:
+            p.map(_noop, range(world_size))
+
+    yield c.submit(pool_worker, 4)

From 861536ca2cbb6039ae9325c672f1b85f8124bc25 Mon Sep 17 00:00:00 2001
From: Michael Spiegel <Spiegel0@users.noreply.github.com>
Date: Mon, 3 Jun 2019 17:37:41 +0200
Subject: [PATCH 0314/1550] Fix the resource key representation before sending
 graphs (#2716) (#2733)

Convert resource key toples to a string representation before they are
submitted to the scheduler. The commit is intended to fix #2716.

The test case persists the result of a tiny DataFrame operation
and checks the resource restrictions.
---
 distributed/client.py               |  1 +
 distributed/tests/test_resources.py | 18 ++++++++++++++++++
 2 files changed, 19 insertions(+)

diff --git a/distributed/client.py b/distributed/client.py
index d924b608c61..22d89cda4e4 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -2351,6 +2351,7 @@ def _graph_to_futures(
                 resources = self._expand_resources(
                     resources, all_keys=itertools.chain(dsk, keys)
                 )
+                resources = {tokey(k): v for k, v in resources.items()}
 
             if retries:
                 retries = self._expand_retries(
diff --git a/distributed/tests/test_resources.py b/distributed/tests/test_resources.py
index d7102ef5301..480532d912e 100644
--- a/distributed/tests/test_resources.py
+++ b/distributed/tests/test_resources.py
@@ -202,6 +202,24 @@ def test_persist_tuple(c, s, a, b):
     assert not b.data
 
 
+@gen_cluster(client=True)
+def test_resources_str(c, s, a, b):
+    pd = pytest.importorskip("pandas")
+    dd = pytest.importorskip("dask.dataframe")
+
+    yield a.set_resources(MyRes=1)
+
+    x = dd.from_pandas(pd.DataFrame({"A": [1, 2], "B": [3, 4]}), npartitions=1)
+    y = x.apply(lambda row: row.sum(), axis=1, meta=(None, "int64"))
+    yy = y.persist(resources={"MyRes": 1})
+    yield wait(yy)
+
+    ts_first = s.tasks[tokey(y.__dask_keys__()[0])]
+    assert ts_first.resource_restrictions == {"MyRes": 1}
+    ts_last = s.tasks[tokey(y.__dask_keys__()[-1])]
+    assert ts_last.resource_restrictions == {"MyRes": 1}
+
+
 @gen_cluster(
     client=True,
     ncores=[

From bcb765de543a0f15ddce5d1a2e86b7f4bdefde4f Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 3 Jun 2019 17:26:06 -0700
Subject: [PATCH 0315/1550] Add async context managers to scheduler/worker
 classes (#2745)

---
 distributed/node.py                 |  7 +++++++
 distributed/tests/test_scheduler.py | 10 ++++++++++
 2 files changed, 17 insertions(+)

diff --git a/distributed/node.py b/distributed/node.py
index ff95a621877..4f0b9813a8e 100644
--- a/distributed/node.py
+++ b/distributed/node.py
@@ -130,3 +130,10 @@ def stop_services(self):
     @property
     def service_ports(self):
         return {k: v.port for k, v in self.services.items()}
+
+    async def __aenter__(self):
+        await self
+        return self
+
+    async def __aexit__(self, typ, value, traceback):
+        await self.close()
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 6df271ae34e..a0cbfabfa29 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -1572,3 +1572,13 @@ def test_dashboard_address():
     s = yield Scheduler(dashboard_address="127.0.0.1", port=0)
     assert s.services["dashboard"].port
     yield s.close()
+
+
+@pytest.mark.asyncio
+async def test_async_context_manager():
+    async with Scheduler(port=0) as s:
+        assert s.status == "running"
+        async with Worker(s.address) as w:
+            assert w.status == "running"
+            assert s.workers
+        assert not s.workers

From 2a1e089a9dc541a0a19c0fca575c42c719275bd9 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 5 Jun 2019 10:09:19 -0700
Subject: [PATCH 0316/1550] Worker dashboard fixes (#2747)

* bokeh -> dashboard in template

* Add doc to ProfileTimePlot

* Add test for bokeh worker routes

* Remove info route from worker

To do this we ...

1.  Remove the baked in "info" link in the base template
2.  Add that to the scheduler's list of links
3.  Add a redirect from "info" to the actual page
4.  Create a generic redirect route
5.  Move that and the RequestHandler to utils to avoid code duplication
    between scheduler and worker

Fixes https://github.com/dask/distributed/issues/2722

* Add worker name

Fixes https://github.com/dask/dask/issues/4878
---
 distributed/dashboard/scheduler.py            | 19 +++++++-----
 distributed/dashboard/scheduler_html.py       | 15 ++--------
 distributed/dashboard/templates/base.html     |  3 --
 .../dashboard/templates/worker-table.html     |  6 ++--
 distributed/dashboard/templates/worker.html   |  6 ++--
 .../dashboard/tests/test_scheduler_bokeh.py   |  6 ++--
 .../dashboard/tests/test_worker_bokeh.py      | 29 +++++++++++++++++++
 distributed/dashboard/utils.py                | 20 +++++++++++++
 distributed/dashboard/worker.py               |  2 +-
 distributed/dashboard/worker_html.py          | 22 ++------------
 10 files changed, 76 insertions(+), 52 deletions(-)

diff --git a/distributed/dashboard/scheduler.py b/distributed/dashboard/scheduler.py
index 6476d3aa6e4..86f56e9eda0 100644
--- a/distributed/dashboard/scheduler.py
+++ b/distributed/dashboard/scheduler.py
@@ -77,7 +77,7 @@
 )
 
 template_variables = {
-    "pages": ["status", "workers", "tasks", "system", "profile", "graph"]
+    "pages": ["status", "workers", "tasks", "system", "profile", "graph", "info"]
 }
 
 BOKEH_THEME = Theme(os.path.join(os.path.dirname(__file__), "theme.yaml"))
@@ -449,7 +449,7 @@ def update(self):
                     "nbytes_text": nbytes_text,
                     "dashboard_host": dashboard_host,
                     "dashboard_port": dashboard_port,
-                    "worker": [ws.address for ws in workers],
+                    "address": [ws.address for ws in workers],
                     "y": y,
                 }
 
@@ -1177,7 +1177,8 @@ class WorkerTable(DashboardComponent):
     def __init__(self, scheduler, width=800, **kwargs):
         self.scheduler = scheduler
         self.names = [
-            "worker",
+            "name",
+            "address",
             "ncores",
             "cpu",
             "memory",
@@ -1195,7 +1196,8 @@ def __init__(self, scheduler, width=800, **kwargs):
         )
 
         table_names = [
-            "worker",
+            "name",
+            "address",
             "ncores",
             "cpu",
             "memory",
@@ -1242,7 +1244,7 @@ def __init__(self, scheduler, width=800, **kwargs):
             if name in formatters:
                 table.columns[table_names.index(name)].formatter = formatters[name]
 
-        extra_names = ["worker"] + self.extra_names
+        extra_names = ["name", "address"] + self.extra_names
         extra_columns = {
             name: TableColumn(field=name, title=name.replace("_percent", "%"))
             for name in extra_names
@@ -1330,10 +1332,13 @@ def __init__(self, scheduler, width=800, **kwargs):
     @without_property_validation
     def update(self):
         data = {name: [] for name in self.names + self.extra_names}
-        for addr, ws in sorted(self.scheduler.workers.items()):
+        for i, (addr, ws) in enumerate(
+            sorted(self.scheduler.workers.items(), key=lambda kv: kv[1].name)
+        ):
             for name in self.names + self.extra_names:
                 data[name].append(ws.metrics.get(name, None))
-            data["worker"][-1] = ws.address
+            data["name"][-1] = ws.name if ws.name is not None else i
+            data["address"][-1] = ws.address
             if ws.memory_limit:
                 data["memory_percent"][-1] = ws.metrics["memory"] / ws.memory_limit
             else:
diff --git a/distributed/dashboard/scheduler_html.py b/distributed/dashboard/scheduler_html.py
index 5f481f783be..9f2bcd3cbb2 100644
--- a/distributed/dashboard/scheduler_html.py
+++ b/distributed/dashboard/scheduler_html.py
@@ -1,30 +1,18 @@
 from datetime import datetime
-import os
 
 import toolz
 from tornado import escape
 from tornado import gen
-from tornado import web
 
 from ..utils import log_errors, format_bytes, format_time
 from .proxy import GlobalProxyHandler
-
-dirname = os.path.dirname(__file__)
+from .utils import RequestHandler, redirect
 
 ns = {
     func.__name__: func for func in [format_bytes, format_time, datetime.fromtimestamp]
 }
 
 
-class RequestHandler(web.RequestHandler):
-    def initialize(self, server=None, extra=None):
-        self.server = server
-        self.extra = extra or {}
-
-    def get_template_path(self):
-        return os.path.join(dirname, "templates")
-
-
 class Workers(RequestHandler):
     def get(self):
         with log_errors():
@@ -238,6 +226,7 @@ def get(self):
 
 
 routes = [
+    (r"info", redirect("info/main/workers.html")),
     (r"info/main/workers.html", Workers),
     (r"info/worker/(.*).html", Worker),
     (r"info/task/(.*).html", Task),
diff --git a/distributed/dashboard/templates/base.html b/distributed/dashboard/templates/base.html
index da15df28b69..83f5e8527c6 100644
--- a/distributed/dashboard/templates/base.html
+++ b/distributed/dashboard/templates/base.html
@@ -29,9 +29,6 @@
           <a href="{{ page }}">{{ page|title }}</a>
         </li>
         {% endfor %}
-        <li>
-          <a href="info/main/workers.html">Info</a>
-        </li>
         <li id="navbar-toggle-icon">
           <a href="javascript:void(0);" onclick="myFunction()">
             <img src="statics/images/fa-bars.svg"></img>
diff --git a/distributed/dashboard/templates/worker-table.html b/distributed/dashboard/templates/worker-table.html
index 8a86f8debd1..4835849daad 100644
--- a/distributed/dashboard/templates/worker-table.html
+++ b/distributed/dashboard/templates/worker-table.html
@@ -1,6 +1,7 @@
    <table class="table is-striped is-hoverable">
     <tr>
         <th> Worker </th>
+        <th> Name </th>
         <th> Cores </th>
         <th> Memory </th>
         <th> Memory use </th>
@@ -13,14 +14,15 @@
     {% for ws in worker_list %}
     <tr>
         <td><a href="../worker/{{ url_escape(ws.address) }}.html">{{ws.address}}</a></td>
+        <td> {{ ws.name if ws.name is not None else "" }} </td>
         <td> {{ ws.ncores }} </td>
         <td> {{ format_bytes(ws.memory_limit) }} </td>
         <td> <progress class="progress" value="{{ ws.metrics['memory'] }}" max="{{ ws.memory_limit }}"></progress> </td>
         <td> {{ format_time(ws.occupancy) }} </td>
         <td> {{ len(ws.processing) }} </td>
         <td> {{ len(ws.has_what) }} </td>
-        {% if 'bokeh' in ws.services %}
-        <td> <a href="../../proxy/{{ ws.services['bokeh'] }}/{{ ws.host }}/status">bokeh</a> </td>
+        {% if 'dashboard' in ws.services %}
+        <td> <a href="../../proxy/{{ ws.services['dashboard'] }}/{{ ws.host }}/status">dashboard</a> </td>
         {% else %}
         <td> </td>
         {% end %}
diff --git a/distributed/dashboard/templates/worker.html b/distributed/dashboard/templates/worker.html
index 8b26d86e956..9c7608cb8c2 100644
--- a/distributed/dashboard/templates/worker.html
+++ b/distributed/dashboard/templates/worker.html
@@ -1,8 +1,8 @@
 {% extends main.html %}
 {% block content %}
-    <h1 class="title"> Worker: {{Worker}} </h1>
-      {% set ws = workers[Worker] %}
-      {% set worker_list = [ws] %}
+{% set ws = workers[Worker] %}
+{% set worker_list = [ws] %}
+    <h1 class="title"> Worker: {{ ws.address }} </h1>
       {% include "worker-table.html" %}
 
     <div class="columns">
diff --git a/distributed/dashboard/tests/test_scheduler_bokeh.py b/distributed/dashboard/tests/test_scheduler_bokeh.py
index f8a813514b4..692a29439c0 100644
--- a/distributed/dashboard/tests/test_scheduler_bokeh.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh.py
@@ -354,7 +354,7 @@ def metric_address(worker):
 
     assert all(data.values())
     assert all(len(v) == 2 for v in data.values())
-    my_index = data["worker"].index(a.address), data["worker"].index(b.address)
+    my_index = data["address"].index(a.address), data["address"].index(b.address)
     assert [data["metric_port"][i] for i in my_index] == [a.port, b.port]
     assert [data["metric_address"][i] for i in my_index] == [a.address, b.address]
 
@@ -379,7 +379,7 @@ def metric_port(worker):
     assert "metric_b" in data
     assert all(data.values())
     assert all(len(v) == 2 for v in data.values())
-    my_index = data["worker"].index(a.address), data["worker"].index(b.address)
+    my_index = data["address"].index(a.address), data["address"].index(b.address)
     assert [data["metric_a"][i] for i in my_index] == [a.port, None]
     assert [data["metric_b"][i] for i in my_index] == [None, b.port]
 
@@ -399,7 +399,7 @@ def metric_port(worker):
     assert "metric_a" in data
     assert all(data.values())
     assert all(len(v) == 2 for v in data.values())
-    my_index = data["worker"].index(a.address), data["worker"].index(b.address)
+    my_index = data["address"].index(a.address), data["address"].index(b.address)
     assert [data["metric_a"][i] for i in my_index] == [a.port, None]
 
 
diff --git a/distributed/dashboard/tests/test_worker_bokeh.py b/distributed/dashboard/tests/test_worker_bokeh.py
index 11699d9ac83..ef977127d23 100644
--- a/distributed/dashboard/tests/test_worker_bokeh.py
+++ b/distributed/dashboard/tests/test_worker_bokeh.py
@@ -1,6 +1,7 @@
 from __future__ import print_function, division, absolute_import
 
 from operator import add, sub
+import re
 from time import sleep
 
 import pytest
@@ -14,6 +15,7 @@
 from distributed.client import wait
 from distributed.metrics import time
 from distributed.utils_test import gen_cluster, inc, dec
+from distributed.dashboard.scheduler import BokehScheduler
 from distributed.dashboard.worker import (
     BokehWorker,
     StateTable,
@@ -26,6 +28,33 @@
 )
 
 
+@gen_cluster(
+    client=True,
+    worker_kwargs={"services": {("dashboard", 0): BokehWorker}},
+    scheduler_kwargs={"services": {("dashboard", 0): BokehScheduler}},
+)
+def test_routes(c, s, a, b):
+    assert isinstance(a.services["dashboard"], BokehWorker)
+    assert isinstance(b.services["dashboard"], BokehWorker)
+    port = a.services["dashboard"].port
+
+    future = c.submit(sleep, 1)
+    yield gen.sleep(0.1)
+
+    http_client = AsyncHTTPClient()
+    for suffix in ["status", "counters", "system", "profile", "profile-server"]:
+        response = yield http_client.fetch("http://localhost:%d/%s" % (port, suffix))
+        body = response.body.decode()
+        assert "bokeh" in body.lower()
+        assert not re.search("href=./", body)  # no absolute links
+
+    response = yield http_client.fetch(
+        "http://localhost:%d/info/main/workers.html" % s.services["dashboard"].port
+    )
+
+    assert str(port) in response.body.decode()
+
+
 @pytest.mark.skipif(
     sys.version_info[0] == 2, reason="https://github.com/bokeh/bokeh/issues/5494"
 )
diff --git a/distributed/dashboard/utils.py b/distributed/dashboard/utils.py
index 516ca5bfb88..a9b31345ca9 100644
--- a/distributed/dashboard/utils.py
+++ b/distributed/dashboard/utils.py
@@ -1,13 +1,16 @@
 from __future__ import print_function, division, absolute_import
 
 from distutils.version import LooseVersion
+import os
 
 import bokeh
+from tornado import web
 from toolz import partition
 
 from ..compatibility import PY2
 
 BOKEH_VERSION = LooseVersion(bokeh.__version__)
+dirname = os.path.dirname(__file__)
 
 
 if BOKEH_VERSION >= "1.0.0" and not PY2:
@@ -32,3 +35,20 @@ def parse_args(args):
 def transpose(lod):
     keys = list(lod[0].keys())
     return {k: [d[k] for d in lod] for k in keys}
+
+
+class RequestHandler(web.RequestHandler):
+    def initialize(self, server=None, extra=None):
+        self.server = server
+        self.extra = extra or {}
+
+    def get_template_path(self):
+        return os.path.join(dirname, "templates")
+
+
+def redirect(path):
+    class Redirect(RequestHandler):
+        def get(self):
+            self.redirect(path)
+
+    return Redirect
diff --git a/distributed/dashboard/worker.py b/distributed/dashboard/worker.py
index ed7b68b76b4..aa85afc4197 100644
--- a/distributed/dashboard/worker.py
+++ b/distributed/dashboard/worker.py
@@ -735,7 +735,7 @@ def counters_doc(server, extra, doc):
 def profile_doc(server, extra, doc):
     with log_errors():
         doc.title = "Dask Worker Profile"
-        profile = ProfileTimePlot(server, sizing_mode="scale_width")
+        profile = ProfileTimePlot(server, sizing_mode="scale_width", doc=doc)
         profile.trigger_update()
 
         doc.add_root(profile.root)
diff --git a/distributed/dashboard/worker_html.py b/distributed/dashboard/worker_html.py
index c818c8fb1e6..450cce56c8e 100644
--- a/distributed/dashboard/worker_html.py
+++ b/distributed/dashboard/worker_html.py
@@ -1,17 +1,4 @@
-import os
-
-from tornado import web
-
-dirname = os.path.dirname(__file__)
-
-
-class RequestHandler(web.RequestHandler):
-    def initialize(self, server=None, extra=None):
-        self.server = server
-        self.extra = extra or {}
-
-    def get_template_path(self):
-        return os.path.join(dirname, "templates")
+from .utils import RequestHandler, redirect
 
 
 class _PrometheusCollector(object):
@@ -67,15 +54,10 @@ def get(self):
         self.set_header("Content-Type", "text/plain")
 
 
-class OldRoute(RequestHandler):
-    def get(self):
-        self.redirect("/status")
-
-
 routes = [
     (r"metrics", PrometheusHandler),
     (r"health", HealthHandler),
-    (r"main", OldRoute),
+    (r"main", redirect("/status")),
 ]
 
 

From e846991d93054a29e28528224c63db69972ffc9c Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 5 Jun 2019 14:26:50 -0700
Subject: [PATCH 0317/1550] Add SpecCluster.new_worker_spec method (#2751)

* Add type name to LocalCluster.__repr__
* Add SpecCluster.new_worker_spec method

This is helpful for subclassing
---
 distributed/deploy/local.py                   | 10 ++-------
 distributed/deploy/spec.py                    | 22 ++++++++++++++++---
 distributed/deploy/tests/test_spec_cluster.py | 18 ++++++++++++++-
 3 files changed, 38 insertions(+), 12 deletions(-)

diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index 95f178c7c2e..a56cce8c2b2 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -197,14 +197,8 @@ def __init__(
         )
 
     def __repr__(self):
-        return "LocalCluster(%r, workers=%d, ncores=%d)" % (
-            self.scheduler_address,
-            len(self.workers),
-            sum(w.ncores for w in self.workers.values()),
-        )
-
-    def __repr__(self):
-        return "LocalCluster(%r, workers=%d, ncores=%d)" % (
+        return "%s(%r, workers=%d, ncores=%d)" % (
+            type(self).__name__,
             self.scheduler_address,
             len(self.workers),
             sum(w.ncores for w in self.workers.values()),
diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index d5a954effc8..2558a5df26a 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -275,12 +275,28 @@ def scale(self, n):
             self.worker_spec.popitem()
 
         while len(self.worker_spec) < n:
-            while self._i in self.worker_spec:
-                self._i += 1
-            self.worker_spec[self._i] = self.new_spec
+            k, spec = self.new_worker_spec()
+            self.worker_spec[k] = spec
 
         self.loop.add_callback(self._correct_state)
 
+    def new_worker_spec(self):
+        """ Return name and spec for the next worker
+
+        Returns
+        -------
+        name: identifier for worker
+        spec: dict
+
+        See Also
+        --------
+        scale
+        """
+        while self._i in self.worker_spec:
+            self._i += 1
+
+        return self._i, self.new_spec
+
     async def scale_down(self, workers):
         workers = set(workers)
 
diff --git a/distributed/deploy/tests/test_spec_cluster.py b/distributed/deploy/tests/test_spec_cluster.py
index ac5706afe1c..eb733f2e68f 100644
--- a/distributed/deploy/tests/test_spec_cluster.py
+++ b/distributed/deploy/tests/test_spec_cluster.py
@@ -76,7 +76,9 @@ def test_spec_sync(loop):
 
 
 def test_loop_started():
-    cluster = SpecCluster(worker_spec)
+    cluster = SpecCluster(
+        worker_spec, scheduler={"cls": Scheduler, "options": {"port": 0}}
+    )
 
 
 @pytest.mark.asyncio
@@ -110,6 +112,7 @@ async def test_broken_worker():
         async with SpecCluster(
             asynchronous=True,
             workers={"good": {"cls": Worker}, "bad": {"cls": BrokenWorker}},
+            scheduler={"cls": Scheduler, "options": {"port": 0}},
         ) as cluster:
             pass
 
@@ -124,3 +127,16 @@ def test_spec_close_clusters(loop):
     assert cluster in SpecCluster._instances
     close_clusters()
     assert cluster.status == "closed"
+
+
+@pytest.mark.asyncio
+async def test_new_worker_spec():
+    class MyCluster(SpecCluster):
+        def new_worker_spec(self):
+            i = len(self.worker_spec)
+            return i, {"cls": Worker, "options": {"ncores": i + 1}}
+
+    async with MyCluster(asynchronous=True, scheduler=scheduler) as cluster:
+        cluster.scale(3)
+        for i in range(3):
+            assert cluster.worker_spec[i]["options"]["ncores"] == i + 1

From 0696a1f6456b8010b19ac47b14cd2dca0d859246 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 6 Jun 2019 11:15:33 -0700
Subject: [PATCH 0318/1550] Move some of the adaptive logic into the scheduler
 (#2735)

* Move some of the adaptive logic into the scheduler
* don't close closed clusters
* require pytest >= 4 in CI
* use worker_spec if it exists
* Don't scale a closed cluster
* handle intermittent failures
---
 continuous_integration/travis/install.sh  |   2 +-
 distributed/deploy/adaptive.py            | 117 +++-------------------
 distributed/deploy/spec.py                |   7 +-
 distributed/deploy/tests/test_adaptive.py |  35 +++----
 distributed/scheduler.py                  |  55 ++++++++++
 distributed/tests/test_diskutils.py       |   2 +
 distributed/tests/test_scheduler.py       |  25 +++++
 distributed/utils.py                      |  12 +--
 8 files changed, 121 insertions(+), 134 deletions(-)

diff --git a/continuous_integration/travis/install.sh b/continuous_integration/travis/install.sh
index cb2dbdf5c83..2ab9724db25 100644
--- a/continuous_integration/travis/install.sh
+++ b/continuous_integration/travis/install.sh
@@ -44,7 +44,7 @@ conda install -q \
     paramiko \
     prometheus_client \
     psutil \
-    pytest \
+    pytest>=4 \
     pytest-timeout \
     python=$PYTHON \
     requests \
diff --git a/distributed/deploy/adaptive.py b/distributed/deploy/adaptive.py
index 793e80d984c..401acc3dc1d 100644
--- a/distributed/deploy/adaptive.py
+++ b/distributed/deploy/adaptive.py
@@ -4,7 +4,6 @@
 import logging
 import math
 
-import toolz
 from tornado import gen
 
 from ..metrics import time
@@ -128,104 +127,6 @@ def stop(self):
             self._adapt_callback = None
             del self._adapt_callback
 
-    def needs_cpu(self):
-        """
-        Check if the cluster is CPU constrained (too many tasks per core)
-
-        Notes
-        -----
-        Returns ``True`` if the occupancy per core is some factor larger
-        than ``startup_cost`` and the number of tasks exceeds the number of
-        cores
-        """
-        total_occupancy = self.scheduler.total_occupancy
-        total_cores = self.scheduler.total_ncores
-
-        if total_occupancy / (total_cores + 1e-9) > self.startup_cost * 2:
-            logger.info(
-                "CPU limit exceeded [%d occupancy / %d cores]",
-                total_occupancy,
-                total_cores,
-            )
-
-            tasks_processing = 0
-
-            for w in self.scheduler.workers.values():
-                tasks_processing += len(w.processing)
-
-                if tasks_processing > total_cores:
-                    logger.info(
-                        "pending tasks exceed number of cores " "[%d tasks / %d cores]",
-                        tasks_processing,
-                        total_cores,
-                    )
-
-                    return True
-
-        return False
-
-    def needs_memory(self):
-        """
-        Check if the cluster is RAM constrained
-
-        Notes
-        -----
-        Returns ``True`` if  the required bytes in distributed memory is some
-        factor larger than the actual distributed memory available.
-        """
-        limit_bytes = {
-            addr: ws.memory_limit for addr, ws in self.scheduler.workers.items()
-        }
-        worker_bytes = [ws.nbytes for ws in self.scheduler.workers.values()]
-
-        limit = sum(limit_bytes.values())
-        total = sum(worker_bytes)
-        if total > 0.6 * limit:
-            logger.info("Ram limit exceeded [%d/%d]", limit, total)
-            return True
-        else:
-            return False
-
-    def should_scale_up(self):
-        """
-        Determine whether additional workers should be added to the cluster
-
-        Returns
-        -------
-        scale_up : bool
-
-        Notes
-        ----
-        Additional workers are added whenever
-
-        1. There are unrunnable tasks and no workers
-        2. The cluster is CPU constrained
-        3. The cluster is RAM constrained
-        4. There are fewer workers than our minimum
-
-        See Also
-        --------
-        needs_cpu
-        needs_memory
-        """
-        with log_errors():
-            if len(self.scheduler.workers) < self.minimum:
-                return True
-
-            if self.maximum is not None and len(self.scheduler.workers) >= self.maximum:
-                return False
-
-            if self.scheduler.unrunnable and not self.scheduler.workers:
-                return True
-
-            needs_cpu = self.needs_cpu()
-            needs_memory = self.needs_memory()
-
-            if needs_cpu or needs_memory:
-                return True
-
-            return False
-
     def workers_to_close(self, **kwargs):
         """
         Determine which, if any, workers should potentially be removed from
@@ -305,9 +206,17 @@ def get_scale_up_kwargs(self):
         return {"n": instances}
 
     def recommendations(self, comm=None):
-        should_scale_up = self.should_scale_up()
+        n = self.scheduler.adaptive_target(target_duration=self.target_duration)
+        if self.maximum is not None:
+            n = min(self.maximum, n)
+        if self.minimum is not None:
+            n = max(self.minimum, n)
         workers = set(self.workers_to_close(key=self.worker_key, minimum=self.minimum))
-        if should_scale_up and workers:
+        try:
+            current = len(self.cluster.worker_spec)
+        except AttributeError:
+            current = len(self.cluster.workers)
+        if n > current and workers:
             logger.info("Attempting to scale up and scale down simultaneously.")
             self.close_counts.clear()
             return {
@@ -315,9 +224,9 @@ def recommendations(self, comm=None):
                 "msg": "Trying to scale up and down simultaneously",
             }
 
-        elif should_scale_up:
+        elif n > current:
             self.close_counts.clear()
-            return toolz.merge({"status": "up"}, self.get_scale_up_kwargs())
+            return {"status": "up", "n": n}
 
         elif workers:
             d = {}
@@ -352,7 +261,7 @@ def _adapt(self):
                 return
             status = recommendations.pop("status")
             if status == "up":
-                f = self.cluster.scale_up(**recommendations)
+                f = self.cluster.scale(**recommendations)
                 self.log.append((time(), "up", recommendations))
                 if hasattr(f, "__await__"):
                     yield f
diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index 2558a5df26a..85728a057e4 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -274,6 +274,10 @@ def scale(self, n):
         while len(self.worker_spec) > n:
             self.worker_spec.popitem()
 
+        if self.status in ("closing", "closed"):
+            self.loop.add_callback(self._correct_state)
+            return
+
         while len(self.worker_spec) < n:
             k, spec = self.new_worker_spec()
             self.worker_spec[k] = spec
@@ -321,4 +325,5 @@ def __repr__(self):
 def close_clusters():
     for cluster in list(SpecCluster._instances):
         with ignoring(gen.TimeoutError):
-            cluster.close(timeout=10)
+            if cluster.status != "closed":
+                cluster.close(timeout=10)
diff --git a/distributed/deploy/tests/test_adaptive.py b/distributed/deploy/tests/test_adaptive.py
index 8915c721353..cc860636e55 100644
--- a/distributed/deploy/tests/test_adaptive.py
+++ b/distributed/deploy/tests/test_adaptive.py
@@ -2,13 +2,12 @@
 
 from time import sleep
 
-import pytest
 from toolz import frequencies, pluck
 from tornado import gen
 from tornado.ioloop import IOLoop
 
 from distributed import Client, wait, Adaptive, LocalCluster, SpecCluster, Worker
-from distributed.utils_test import gen_cluster, gen_test, slowinc, inc, clean
+from distributed.utils_test import gen_cluster, gen_test, slowinc, clean
 from distributed.utils_test import loop, nodebug  # noqa: F401
 from distributed.metrics import time
 
@@ -116,11 +115,10 @@ def test_adaptive_local_cluster_multi_workers():
             yield gen.sleep(0.01)
             assert time() < start + 15, alc.log
 
-        # assert not cluster.workers
-        assert not cluster.scheduler.workers
-        yield gen.sleep(0.2)
-        # assert not cluster.workers
-        assert not cluster.scheduler.workers
+        # no workers for a while
+        for i in range(10):
+            assert not cluster.scheduler.workers
+            yield gen.sleep(0.05)
 
         futures = c.map(slowinc, range(100), delay=0.01)
         yield c.gather(futures)
@@ -152,6 +150,10 @@ def scale_up(self, n, **kwargs):
         def scale_down(self, workers):
             assert False
 
+        @property
+        def workers(self):
+            return s.workers
+
     assert len(s.workers) == 10
 
     # Assert that adaptive cycle does not reduce cluster below minimum size
@@ -163,8 +165,7 @@ def scale_down(self, workers):
     assert len(s.workers) == 2
 
 
-@pytest.mark.xfail(reason="need to rework adaptive")
-@gen_test(timeout=30)
+@gen_test()
 def test_min_max():
     cluster = yield LocalCluster(
         0,
@@ -242,7 +243,9 @@ def test_avoid_churn():
             yield client.submit(slowinc, i, delay=0.040)
             yield gen.sleep(0.040)
 
-        assert frequencies(pluck(1, adapt.log)) == {"up": 1}
+        from toolz.curried import pipe, unique, pluck, frequencies
+
+        assert pipe(adapt.log, unique(key=str), pluck(1), frequencies) == {"up": 1}
     finally:
         yield client.close()
         yield cluster.close()
@@ -435,15 +438,3 @@ def key(ws):
         assert names == {"a-1", "a-2"} or names == {"b-1", "b-2"}
     finally:
         yield cluster.close()
-
-
-@gen_cluster(client=True, ncores=[])
-def test_without_cluster(c, s):
-    adapt = Adaptive(scheduler=s)
-
-    future = c.submit(inc, 1)
-    while not s.tasks:
-        yield gen.sleep(0.01)
-
-    response = yield c.scheduler.adaptive_recommendations()
-    assert response["status"] == "up"
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index ca3c1241ea7..3cf8de49306 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -6,6 +6,7 @@
 import itertools
 import json
 import logging
+import math
 from numbers import Number
 import operator
 import os
@@ -1063,6 +1064,7 @@ def __init__(
             "get_task_status": self.get_task_status,
             "get_task_stream": self.get_task_stream,
             "register_worker_plugin": self.register_worker_plugin,
+            "adaptive_target": self.adaptive_target,
         }
 
         self._transitions = {
@@ -4740,6 +4742,59 @@ def check_idle(self):
         if close:
             self.loop.add_callback(self.close)
 
+    def adaptive_target(self, target_duration="5s"):
+        """ Desired number of workers based on the current workload
+
+        This looks at the current running tasks and memory use, and returns a
+        number of desired workers.  This is often used by adaptive scheduling.
+
+        Parameters
+        ----------
+        target_duration: str
+            A desired duration of time for computations to take.  This affects
+            how rapidly the scheduler will ask to scale.
+
+        See Also
+        --------
+        distributed.deploy.Adaptive
+        """
+        target_duration = parse_timedelta(target_duration)
+
+        # CPU
+        cpu = math.ceil(
+            self.total_occupancy / target_duration
+        )  # TODO: threads per worker
+
+        # Avoid a few long tasks from asking for many cores
+        tasks_processing = 0
+        for ws in self.workers.values():
+            tasks_processing += len(ws.processing)
+
+            if tasks_processing > cpu:
+                break
+        else:
+            cpu = min(tasks_processing, cpu)
+
+        if self.unrunnable and not self.workers:
+            cpu = max(1, cpu)
+
+        # Memory
+        limit_bytes = {addr: ws.memory_limit for addr, ws in self.workers.items()}
+        worker_bytes = [ws.nbytes for ws in self.workers.values()]
+        limit = sum(limit_bytes.values())
+        total = sum(worker_bytes)
+        if total > 0.6 * limit:
+            memory = 2 * len(self.workers)
+        else:
+            memory = 0
+
+        target = max(memory, cpu)
+        if target >= len(self.workers):
+            return target
+        else:  # Scale down?
+            to_close = self.workers_to_close()
+            return len(self.workers) - len(to_close)
+
 
 def decide_worker(ts, all_workers, valid_workers, objective):
     """
diff --git a/distributed/tests/test_diskutils.py b/distributed/tests/test_diskutils.py
index 1bededf84ab..a6dcf3497a3 100644
--- a/distributed/tests/test_diskutils.py
+++ b/distributed/tests/test_diskutils.py
@@ -276,6 +276,8 @@ def _test_workspace_concurrency(tmpdir, timeout, max_procs):
 def test_workspace_concurrency(tmpdir):
     if WINDOWS:
         raise pytest.xfail.Exception("TODO: unknown failure on windows")
+    if sys.version_info < (3, 6):
+        raise pytest.xfail.Exception("TODO: unknown failure on Python 3.5")
     _test_workspace_concurrency(tmpdir, 2.0, 6)
 
 
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index a0cbfabfa29..1c321a02906 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -1574,6 +1574,31 @@ def test_dashboard_address():
     yield s.close()
 
 
+@gen_cluster(client=True)
+async def test_adaptive_target(c, s, a, b):
+    assert s.adaptive_target() == 0
+    x = c.submit(inc, 1)
+    await x
+    assert s.adaptive_target() == 1
+
+    # Long task
+    s.task_duration["slowinc"] = 10
+    x = c.submit(slowinc, 1, delay=0.5)
+    while x.key not in s.tasks:
+        await gen.sleep(0.01)
+    assert s.adaptive_target(target_duration=".1s") == 1  # still one
+
+    s.task_duration["slowinc"] = 10
+    L = c.map(slowinc, range(100), delay=0.5)
+    while len(s.tasks) < 100:
+        await gen.sleep(0.01)
+    assert 10 < s.adaptive_target(target_duration=".1s") <= 100
+    del x, L
+    while s.tasks:
+        await gen.sleep(0.01)
+    assert s.adaptive_target(target_duration=".1s") == 0
+
+
 @pytest.mark.asyncio
 async def test_async_context_manager():
     async with Scheduler(port=0) as s:
diff --git a/distributed/utils.py b/distributed/utils.py
index 55508a4c574..e8de0bc5108 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -950,13 +950,13 @@ def tmpfile(extension=""):
     yield filename
 
     if os.path.exists(filename):
-        if os.path.isdir(filename):
-            shutil.rmtree(filename)
-        else:
-            try:
+        try:
+            if os.path.isdir(filename):
+                shutil.rmtree(filename)
+            else:
                 os.remove(filename)
-            except OSError:  # sometimes we can't remove a generated temp file
-                pass
+        except OSError:  # sometimes we can't remove a generated temp file
+            pass
 
 
 def ensure_bytes(s):

From 587be8d48536f52453594eebd1a23becf864ccf9 Mon Sep 17 00:00:00 2001
From: Tom Augspurger <TomAugspurger@users.noreply.github.com>
Date: Thu, 6 Jun 2019 16:42:32 -0500
Subject: [PATCH 0319/1550] Add nanny logs (#2744)

---
 distributed/client.py            | 20 ++++++++++++++------
 distributed/nanny.py             |  2 ++
 distributed/node.py              | 24 +++++++++++++++++++++++-
 distributed/scheduler.py         | 28 +++++-----------------------
 distributed/tests/test_client.py | 16 ++++++++++++++--
 distributed/worker.py            | 24 ++----------------------
 docs/source/api.rst              |  2 +-
 7 files changed, 61 insertions(+), 55 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 22d89cda4e4..ac098d7987e 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -2270,6 +2270,10 @@ def run(self, function, *args, **kwargs):
         wait: boolean (optional)
             If the function is asynchronous whether or not to wait until that
             function finishes.
+        nanny : bool, defualt False
+            Whether to run ``function`` on the nanny. By default, the function
+            is run on the worker process.  If specified, the addresses in
+            ``workers`` should still be the worker addresses, not the nanny addresses.
 
         Examples
         --------
@@ -3354,7 +3358,7 @@ def get_scheduler_logs(self, n=None):
 
         Parameters
         ----------
-        n: int
+        n : int
             Number of logs to retrive.  Maxes out at 10000 by default,
             confiruable in config.yaml::log-length
 
@@ -3364,23 +3368,27 @@ def get_scheduler_logs(self, n=None):
         """
         return self.sync(self.scheduler.logs, n=n)
 
-    def get_worker_logs(self, n=None, workers=None):
+    def get_worker_logs(self, n=None, workers=None, nanny=False):
         """ Get logs from workers
 
         Parameters
         ----------
-        n: int
+        n : int
             Number of logs to retrive.  Maxes out at 10000 by default,
             confiruable in config.yaml::log-length
-        workers: iterable
-            List of worker addresses to retrive.  Gets all workers by default.
+        workers : iterable
+            List of worker addresses to retrieve.  Gets all workers by default.
+        nanny : bool, default False
+            Whether to get the logs from the workers (False) or the nannies (True). If
+            specified, the addresses in `workers` should still be the worker addresses,
+            not the nanny addresses.
 
         Returns
         -------
         Dictionary mapping worker address to logs.
         Logs are returned in reversed order (newest first)
         """
-        return self.sync(self.scheduler.worker_logs, n=n, workers=workers)
+        return self.sync(self.scheduler.worker_logs, n=n, workers=workers, nanny=nanny)
 
     def retire_workers(self, workers=None, close_workers=True, **kwargs):
         """ Retire certain workers on the scheduler
diff --git a/distributed/nanny.py b/distributed/nanny.py
index 59a8083e832..9cf444fc7c4 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -79,6 +79,7 @@ def __init__(
         protocol=None,
         **worker_kwargs
     ):
+        self._setup_logging(logger)
         self.loop = loop or IOLoop.current()
         self.security = security or Security()
         assert isinstance(self.security, Security)
@@ -130,6 +131,7 @@ def __init__(
             "kill": self.kill,
             "restart": self.restart,
             # cannot call it 'close' on the rpc side for naming conflict
+            "get_logs": self.get_logs,
             "terminate": self.close,
             "close_gracefully": self.close_gracefully,
             "run": self.run,
diff --git a/distributed/node.py b/distributed/node.py
index 4f0b9813a8e..8bd81ffe5ae 100644
--- a/distributed/node.py
+++ b/distributed/node.py
@@ -1,12 +1,15 @@
 from __future__ import print_function, division, absolute_import
 
 import warnings
+import logging
 
 from tornado.ioloop import IOLoop
+import dask
 
-from .compatibility import unicode
+from .compatibility import unicode, finalize
 from .core import Server, ConnectionPool
 from .versions import get_versions
+from .utils import DequeHandler
 
 
 class Node(object):
@@ -131,6 +134,25 @@ def stop_services(self):
     def service_ports(self):
         return {k: v.port for k, v in self.services.items()}
 
+    def _setup_logging(self, logger):
+        self._deque_handler = DequeHandler(
+            n=dask.config.get("distributed.admin.log-length")
+        )
+        self._deque_handler.setFormatter(
+            logging.Formatter(dask.config.get("distributed.admin.log-format"))
+        )
+        logger.addHandler(self._deque_handler)
+        finalize(self, logger.removeHandler, self._deque_handler)
+
+    def get_logs(self, comm=None, n=None):
+        deque_handler = self._deque_handler
+        if n is None:
+            L = list(deque_handler.deque)
+        else:
+            L = deque_handler.deque
+            L = [L[-i] for i in range(min(n, len(L)))]
+        return [(msg.levelname, deque_handler.format(msg)) for msg in L]
+
     async def __aenter__(self):
         await self
         return self
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 3cf8de49306..2705971e155 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -53,7 +53,6 @@
     key_split,
     validate_key,
     no_default,
-    DequeHandler,
     parse_timedelta,
     parse_bytes,
     PeriodicCallback,
@@ -844,7 +843,7 @@ def __init__(
         dashboard_address=None,
         **kwargs
     ):
-        self._setup_logging()
+        self._setup_logging(logger)
 
         # Attributes
         self.allowed_failures = allowed_failures
@@ -1329,16 +1328,6 @@ def close_worker(self, stream=None, worker=None, safe=None):
             self.worker_send(worker, {"op": "close", "report": False})
             self.remove_worker(address=worker, safe=safe)
 
-    def _setup_logging(self):
-        self._deque_handler = DequeHandler(
-            n=dask.config.get("distributed.admin.log-length")
-        )
-        self._deque_handler.setFormatter(
-            logging.Formatter(dask.config.get("distributed.admin.log-format"))
-        )
-        logger.addHandler(self._deque_handler)
-        finalize(self, logger.removeHandler, self._deque_handler)
-
     ###########
     # Stimuli #
     ###########
@@ -4627,18 +4616,11 @@ def get_profile_metadata(
 
         raise gen.Return({"counts": counts, "keys": keys})
 
-    def get_logs(self, comm=None, n=None):
-        deque_handler = self._deque_handler
-        if n is None:
-            L = list(deque_handler.deque)
-        else:
-            L = deque_handler.deque
-            L = [L[-i] for i in range(min(n, len(L)))]
-        return [(msg.levelname, deque_handler.format(msg)) for msg in L]
-
     @gen.coroutine
-    def get_worker_logs(self, comm=None, n=None, workers=None):
-        results = yield self.broadcast(msg={"op": "get_logs", "n": n}, workers=workers)
+    def get_worker_logs(self, comm=None, n=None, workers=None, nanny=False):
+        results = yield self.broadcast(
+            msg={"op": "get_logs", "n": n}, workers=workers, nanny=nanny
+        )
         raise gen.Return(results)
 
     ###########
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index c731ae6e5ad..dc45b3025e5 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -5111,7 +5111,7 @@ def test_task_metadata(c, s, a, b):
     assert result == {"a": {"c": {"d": 1}}, "b": 2}
 
 
-@gen_cluster(client=True)
+@gen_cluster(client=True, Worker=Nanny)
 def test_logs(c, s, a, b):
     yield wait(c.map(inc, range(5)))
     logs = yield c.get_scheduler_logs(n=5)
@@ -5121,11 +5121,23 @@ def test_logs(c, s, a, b):
         assert "distributed.scheduler" in msg
 
     w_logs = yield c.get_worker_logs(n=5)
-    assert set(w_logs.keys()) == {a.address, b.address}
+    assert set(w_logs.keys()) == {a.worker_address, b.worker_address}
     for log in w_logs.values():
         for _, msg in log:
             assert "distributed.worker" in msg
 
+    n_logs = yield c.get_worker_logs(nanny=True)
+    assert set(n_logs.keys()) == {a.worker_address, b.worker_address}
+    for log in n_logs.values():
+        for _, msg in log:
+            assert "distributed.nanny" in msg
+
+    n_logs = yield c.get_worker_logs(nanny=True, workers=[a.worker_address])
+    assert set(n_logs.keys()) == {a.worker_address}
+    for log in n_logs.values():
+        for _, msg in log:
+            assert "distributed.nanny" in msg
+
 
 @gen_cluster(client=True)
 def test_avoid_delayed_finalize(c, s, a, b):
diff --git a/distributed/worker.py b/distributed/worker.py
index d0bc735ec67..37dcbc2eca1 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -33,7 +33,7 @@
 from .comm import get_address_host, get_local_address_for, connect
 from .comm.utils import offload
 from .comm.addressing import address_from_user_args
-from .compatibility import unicode, get_thread_identity, finalize, MutableMapping
+from .compatibility import unicode, get_thread_identity, MutableMapping
 from .core import error_message, CommClosedError, send_recv, pingpong, coerce_to_address
 from .diskutils import WorkSpace
 from .metrics import time
@@ -60,7 +60,6 @@
     json_load_robust,
     key_split,
     format_bytes,
-    DequeHandler,
     PeriodicCallback,
     parse_bytes,
     parse_timedelta,
@@ -412,7 +411,7 @@ def __init__(
         )
         profile_cycle_interval = parse_timedelta(profile_cycle_interval, default="ms")
 
-        self._setup_logging()
+        self._setup_logging(logger)
 
         if scheduler_file:
             cfg = json_load_robust(scheduler_file)
@@ -666,16 +665,6 @@ def __repr__(self):
             )
         )
 
-    def _setup_logging(self):
-        self._deque_handler = DequeHandler(
-            n=dask.config.get("distributed.admin.log-length")
-        )
-        self._deque_handler.setFormatter(
-            logging.Formatter(dask.config.get("distributed.admin.log-format"))
-        )
-        logger.addHandler(self._deque_handler)
-        finalize(self, logger.removeHandler, self._deque_handler)
-
     @property
     def worker_address(self):
         """ For API compatibility with Nanny """
@@ -888,15 +877,6 @@ def gather(self, comm=None, who_has=None):
             self.update_data(data=result, report=False)
             raise Return({"status": "OK"})
 
-    def get_logs(self, comm=None, n=None):
-        deque_handler = self._deque_handler
-        if n is None:
-            L = list(deque_handler.deque)
-        else:
-            L = deque_handler.deque
-            L = [L[-i] for i in range(min(n, len(L)))]
-        return [(msg.levelname, deque_handler.format(msg)) for msg in L]
-
     #############
     # Lifecycle #
     #############
diff --git a/docs/source/api.rst b/docs/source/api.rst
index e91c4ee6ac1..574a70d34b6 100644
--- a/docs/source/api.rst
+++ b/docs/source/api.rst
@@ -19,8 +19,8 @@ API
    Client.get_executor
    Client.get_metadata
    Client.get_scheduler_logs
-   Client.get_task_stream
    Client.get_worker_logs
+   Client.get_task_stream
    Client.has_what
    Client.list_datasets
    Client.map

From 5042f579b9b77576da319995cf36d0798875b621 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 7 Jun 2019 12:11:49 -0700
Subject: [PATCH 0320/1550] Add stress test for UCX (#2759)

This test generated https://github.com/rapidsai/ucx-py/pull/120
---
 distributed/comm/tests/test_ucx.py | 31 ++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/distributed/comm/tests/test_ucx.py b/distributed/comm/tests/test_ucx.py
index 55a2f4ec82c..8a0e8927cf6 100644
--- a/distributed/comm/tests/test_ucx.py
+++ b/distributed/comm/tests/test_ucx.py
@@ -294,3 +294,34 @@ def test_tcp_localcluster(loop):
         #     assert any(w.data == {x.key: 2} for w in c.workers)
         #     assert e.loop is c.loop
         #     print(c.scheduler.workers)
+
+
+@pytest.mark.slow
+@pytest.mark.asyncio
+async def test_stress():
+    from distributed.utils import get_ip_interface
+
+    try:  # this check should be removed once UCX + TCP works
+        get_ip_interface("ib0")
+    except Exception:
+        pytest.skip("ib0 interface not found")
+
+    import dask.array as da
+    from distributed import wait
+
+    chunksize = "10 MB"
+
+    async with LocalCluster(
+        protocol="ucx", interface="ib0", asynchronous=True
+    ) as cluster:
+        async with Client(cluster, asynchronous=True) as client:
+            rs = da.random.RandomState()
+            x = rs.random((10000, 10000), chunks=(-1, chunksize))
+            x = x.persist()
+            await wait(x)
+
+            for i in range(10):
+                x = x.rechunk((chunksize, -1))
+                x = x.rechunk((-1, chunksize))
+                x = x.persist()
+                await wait(x)

From 756bdd8eb891ee09af6340b7fef4bd883d9fcefb Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 7 Jun 2019 13:04:21 -0700
Subject: [PATCH 0321/1550] Remove module state in Prometheus Handlers (#2760)

This also fixes an ImportError in prometheus-client=0.7
---
 distributed/dashboard/scheduler_html.py | 24 ++++++++++--------------
 distributed/dashboard/worker_html.py    | 18 +++++++-----------
 2 files changed, 17 insertions(+), 25 deletions(-)

diff --git a/distributed/dashboard/scheduler_html.py b/distributed/dashboard/scheduler_html.py
index 9f2bcd3cbb2..8b1da2035ea 100644
--- a/distributed/dashboard/scheduler_html.py
+++ b/distributed/dashboard/scheduler_html.py
@@ -175,17 +175,18 @@ def get(self):
 
 
 class _PrometheusCollector(object):
-    def __init__(self, server, prometheus_client):
+    def __init__(self, server):
         self.server = server
-        self.prometheus_client = prometheus_client
 
     def collect(self):
-        yield self.prometheus_client.core.GaugeMetricFamily(
+        from prometheus_client.core import GaugeMetricFamily
+
+        yield GaugeMetricFamily(
             "dask_scheduler_workers",
             "Number of workers.",
             value=len(self.server.workers),
         )
-        yield self.prometheus_client.core.GaugeMetricFamily(
+        yield GaugeMetricFamily(
             "dask_scheduler_clients",
             "Number of clients.",
             value=len(self.server.clients),
@@ -196,26 +197,21 @@ class PrometheusHandler(RequestHandler):
     _initialized = False
 
     def __init__(self, *args, **kwargs):
-        import prometheus_client  # keep out of global namespace
-
-        self.prometheus_client = prometheus_client
+        import prometheus_client
 
         super(PrometheusHandler, self).__init__(*args, **kwargs)
 
-        self._init()
-
-    def _init(self):
         if PrometheusHandler._initialized:
             return
 
-        self.prometheus_client.REGISTRY.register(
-            _PrometheusCollector(self.server, self.prometheus_client)
-        )
+        prometheus_client.REGISTRY.register(_PrometheusCollector(self.server))
 
         PrometheusHandler._initialized = True
 
     def get(self):
-        self.write(self.prometheus_client.generate_latest())
+        import prometheus_client
+
+        self.write(prometheus_client.generate_latest())
         self.set_header("Content-Type", "text/plain; version=0.0.4")
 
 
diff --git a/distributed/dashboard/worker_html.py b/distributed/dashboard/worker_html.py
index 450cce56c8e..e1ae50f3afc 100644
--- a/distributed/dashboard/worker_html.py
+++ b/distributed/dashboard/worker_html.py
@@ -4,7 +4,6 @@
 class _PrometheusCollector(object):
     def __init__(self, server, prometheus_client):
         self.server = server
-        self.prometheus_client = prometheus_client
 
     def collect(self):
         # add your metrics here:
@@ -14,7 +13,7 @@ def collect(self):
             yield None
         #
         # 2. yield your metrics
-        #     yield self.prometheus_client.core.GaugeMetricFamily(
+        #     yield prometheus_client.core.GaugeMetricFamily(
         #         'dask_worker_connections',
         #         'Number of connections currently open.',
         #         value=???,
@@ -25,26 +24,23 @@ class PrometheusHandler(RequestHandler):
     _initialized = False
 
     def __init__(self, *args, **kwargs):
-        import prometheus_client  # keep out of global namespace
-
-        self.prometheus_client = prometheus_client
+        import prometheus_client
 
         super(PrometheusHandler, self).__init__(*args, **kwargs)
 
-        self._init()
-
-    def _init(self):
         if PrometheusHandler._initialized:
             return
 
-        self.prometheus_client.REGISTRY.register(
-            _PrometheusCollector(self.server, self.prometheus_client)
+        prometheus_client.REGISTRY.register(
+            _PrometheusCollector(self.server, prometheus_client)
         )
 
         PrometheusHandler._initialized = True
 
     def get(self):
-        self.write(self.prometheus_client.generate_latest())
+        import prometheus_client
+
+        self.write(prometheus_client.generate_latest())
         self.set_header("Content-Type", "text/plain; version=0.0.4")
 
 

From 309e435cbb383e437bb8af3c571b52fb163a0ac9 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 7 Jun 2019 13:25:56 -0700
Subject: [PATCH 0322/1550] Change address -> worker in ColumnDataSource for
 nbytes plot (#2755)

Fixes #2754
---
 distributed/dashboard/scheduler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/dashboard/scheduler.py b/distributed/dashboard/scheduler.py
index 86f56e9eda0..2cb916d0b5d 100644
--- a/distributed/dashboard/scheduler.py
+++ b/distributed/dashboard/scheduler.py
@@ -449,7 +449,7 @@ def update(self):
                     "nbytes_text": nbytes_text,
                     "dashboard_host": dashboard_host,
                     "dashboard_port": dashboard_port,
-                    "address": [ws.address for ws in workers],
+                    "worker": [ws.address for ws in workers],
                     "y": y,
                 }
 

From d378b41a89e33e660257522dd4b86d44e6d15fc5 Mon Sep 17 00:00:00 2001
From: Tom Augspurger <TomAugspurger@users.noreply.github.com>
Date: Sat, 8 Jun 2019 13:29:29 -0500
Subject: [PATCH 0323/1550] Delay lookup of allowed failures. (#2761)

This allows for setting the config after importing distributed

xref https://github.com/dask/dask-examples/pull/75#discussion_r291141404
---
 distributed/scheduler.py            |  6 +++---
 distributed/tests/test_scheduler.py | 14 ++++++++++++++
 2 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 2705971e155..ae449bcfafe 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -73,8 +73,6 @@
 logger = logging.getLogger(__name__)
 
 
-ALLOWED_FAILURES = dask.config.get("distributed.scheduler.allowed-failures")
-
 LOG_PDB = dask.config.get("distributed.admin.pdb-on-err")
 DEFAULT_DATA_SIZE = dask.config.get("distributed.scheduler.default-data-size")
 
@@ -829,7 +827,7 @@ def __init__(
         synchronize_worker_interval="60s",
         services=None,
         service_kwargs=None,
-        allowed_failures=ALLOWED_FAILURES,
+        allowed_failures=None,
         extensions=None,
         validate=False,
         scheduler_file=None,
@@ -846,6 +844,8 @@ def __init__(
         self._setup_logging(logger)
 
         # Attributes
+        if allowed_failures is None:
+            allowed_failures = dask.config.get("distributed.scheduler.allowed-failures")
         self.allowed_failures = allowed_failures
         self.validate = validate
         self.status = None
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 1c321a02906..66a8088ace5 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -1607,3 +1607,17 @@ async def test_async_context_manager():
             assert w.status == "running"
             assert s.workers
         assert not s.workers
+
+
+@pytest.mark.asyncio
+async def test_allowed_failures_config():
+    async with Scheduler(port=0, allowed_failures=10) as s:
+        assert s.allowed_failures == 10
+
+    with dask.config.set({"distributed.scheduler.allowed_failures": 100}):
+        async with Scheduler(port=0) as s:
+            assert s.allowed_failures == 100
+
+    with dask.config.set({"distributed.scheduler.allowed_failures": 0}):
+        async with Scheduler(port=0) as s:
+            assert s.allowed_failures == 0

From 2ba70b310dbefc5764ee43079ac1bb783a8cec08 Mon Sep 17 00:00:00 2001
From: Tom Augspurger <TomAugspurger@users.noreply.github.com>
Date: Mon, 10 Jun 2019 16:31:33 -0500
Subject: [PATCH 0324/1550] Add unknown pytest markers (#2764)

---
 setup.cfg | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/setup.cfg b/setup.cfg
index 434b1fd258c..5533437121b 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -41,6 +41,11 @@ universal=1
 [tool:pytest]
 addopts = -rsx -v --durations=10
 minversion = 3.2
+markers =
+    slow: marks tests as slow (deselect with '-m "not slow"')
+    avoid_travis: marks tests as flaky on TravisCI.
+    ipython: mark a test as exercising IPython
+
 # filterwarnings =
 #     error
 #     ignore::UserWarning

From a511f0ea480d7305d86c6439213c3cf3a6d95dc4 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 18 Jun 2019 14:55:26 +0200
Subject: [PATCH 0325/1550] Replace ncores with nthreads throughout codebase
 (#2758)

---
 distributed/cli/dask_mpi.py                   |   2 +-
 distributed/cli/dask_worker.py                |   6 +-
 distributed/cli/tests/test_dask_scheduler.py  |   6 +-
 distributed/cli/tests/test_dask_worker.py     |   6 +-
 distributed/cli/tests/test_tls_cli.py         |   4 +-
 distributed/client.py                         |  30 ++---
 distributed/dashboard/components.py           |  12 +-
 distributed/dashboard/scheduler.py            |  12 +-
 distributed/dashboard/scheduler_html.py       |   6 +-
 .../dashboard/templates/worker-table.html     |   2 +-
 .../dashboard/tests/test_scheduler_bokeh.py   |   4 +-
 distributed/dashboard/worker.py               |   4 +-
 distributed/deploy/local.py                   |  23 ++--
 distributed/deploy/spec.py                    |   8 +-
 distributed/deploy/tests/test_adaptive.py     |  16 +--
 distributed/deploy/tests/test_local.py        |  35 +++---
 distributed/deploy/tests/test_spec_cluster.py |  30 ++---
 distributed/deploy/utils_test.py              |   4 +-
 .../diagnostics/tests/test_eventstream.py     |   2 +-
 distributed/diagnostics/tests/test_plugin.py  |   2 +-
 .../diagnostics/tests/test_progressbar.py     |   2 +-
 .../diagnostics/tests/test_task_stream.py     |   2 +-
 distributed/nanny.py                          |  16 ++-
 distributed/scheduler.py                      |  60 +++++-----
 distributed/stealing.py                       |   4 +-
 distributed/tests/test_actor.py               |  10 +-
 distributed/tests/test_client.py              | 108 +++++++++---------
 distributed/tests/test_collections.py         |   2 +-
 distributed/tests/test_failed_workers.py      |  34 +++---
 distributed/tests/test_ipython.py             |   8 +-
 distributed/tests/test_locks.py               |   2 +-
 distributed/tests/test_nanny.py               |  54 ++++-----
 distributed/tests/test_priorities.py          |   4 +-
 distributed/tests/test_pubsub.py              |   2 +-
 distributed/tests/test_queues.py              |   4 +-
 distributed/tests/test_resources.py           |  36 +++---
 distributed/tests/test_scheduler.py           |  96 ++++++++--------
 distributed/tests/test_steal.py               |  54 ++++-----
 distributed/tests/test_stress.py              |  16 +--
 distributed/tests/test_tls_functional.py      |   6 +-
 distributed/tests/test_utils_test.py          |   8 +-
 distributed/tests/test_variable.py            |   2 +-
 distributed/tests/test_worker.py              |  79 ++++++-------
 distributed/tests/test_worker_client.py       |  10 +-
 distributed/tests/test_worker_plugins.py      |   2 +-
 distributed/utils_comm.py                     |   8 +-
 distributed/utils_test.py                     |  20 ++--
 distributed/worker.py                         |  40 ++++---
 docs/source/api.rst                           |   2 +-
 docs/source/local-cluster.rst                 |   2 +-
 docs/source/protocol.rst                      |   2 +-
 docs/source/scheduling-state.rst              |   2 +-
 docs/source/worker.rst                        |   4 +-
 53 files changed, 466 insertions(+), 449 deletions(-)

diff --git a/distributed/cli/dask_mpi.py b/distributed/cli/dask_mpi.py
index c7669073f79..2a965824662 100644
--- a/distributed/cli/dask_mpi.py
+++ b/distributed/cli/dask_mpi.py
@@ -105,7 +105,7 @@ def main(
             scheduler_file=scheduler_file,
             loop=loop,
             name=rank if scheduler else None,
-            ncores=nthreads,
+            nthreads=nthreads,
             local_dir=local_directory,
             services={("dashboard", bokeh_worker_port): BokehWorker},
             memory_limit=memory_limit,
diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
index 2cf570cfc1d..a53ddf99f6e 100755
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -2,6 +2,7 @@
 
 import atexit
 import logging
+import multiprocessing
 import gc
 import os
 from sys import exit
@@ -11,7 +12,6 @@
 import dask
 from distributed import Nanny, Worker
 from distributed.utils import parse_timedelta
-from distributed.worker import _ncores
 from distributed.security import Security
 from distributed.cli.utils import check_python_3, install_signal_handlers
 from distributed.comm import get_address_host_port
@@ -280,7 +280,7 @@ def main(
         port = worker_port
 
     if not nthreads:
-        nthreads = _ncores // nprocs
+        nthreads = multiprocessing.cpu_count() // nprocs
 
     if pid_file:
         with open(pid_file, "w") as f:
@@ -329,7 +329,7 @@ def del_pid_file():
         t(
             scheduler,
             scheduler_file=scheduler_file,
-            ncores=nthreads,
+            nthreads=nthreads,
             services=services,
             loop=loop,
             resources=resources,
diff --git a/distributed/cli/tests/test_dask_scheduler.py b/distributed/cli/tests/test_dask_scheduler.py
index 754082f35eb..e04fa24bad1 100644
--- a/distributed/cli/tests/test_dask_scheduler.py
+++ b/distributed/cli/tests/test_dask_scheduler.py
@@ -53,7 +53,7 @@ def f():
             ]
 
         with Client("127.0.0.1:8978", loop=loop) as c:
-            assert len(c.ncores()) == 0
+            assert len(c.nthreads()) == 0
             c.sync(f)
 
 
@@ -150,7 +150,7 @@ def test_multiple_workers(loop):
             with popen(["dask-worker", "localhost:8786", "--no-dashboard"]) as b:
                 with Client("127.0.0.1:%d" % Scheduler.default_port, loop=loop) as c:
                     start = time()
-                    while len(c.ncores()) < 2:
+                    while len(c.nthreads()) < 2:
                         sleep(0.1)
                         assert time() < start + 10
 
@@ -178,7 +178,7 @@ def test_interface(loop):
         ) as a:
             with Client("tcp://127.0.0.1:%d" % Scheduler.default_port, loop=loop) as c:
                 start = time()
-                while not len(c.ncores()):
+                while not len(c.nthreads()):
                     sleep(0.1)
                     assert time() - start < 5
                 info = c.scheduler_info()
diff --git a/distributed/cli/tests/test_dask_worker.py b/distributed/cli/tests/test_dask_worker.py
index fa62594a753..dc7c761fdf1 100644
--- a/distributed/cli/tests/test_dask_worker.py
+++ b/distributed/cli/tests/test_dask_worker.py
@@ -58,7 +58,7 @@ def test_memory_limit(loop):
             ]
         ) as worker:
             with Client("127.0.0.1:8786", loop=loop) as c:
-                while not c.ncores():
+                while not c.nthreads():
                     sleep(0.1)
                 info = c.scheduler_info()
                 [d] = info["workers"].values()
@@ -218,7 +218,7 @@ def test_contact_listen_address(loop, nanny, listen_address):
             ]
         ) as worker:
             with Client("127.0.0.1:8786") as client:
-                while not client.ncores():
+                while not client.nthreads():
                     sleep(0.1)
                 info = client.scheduler_info()
                 assert "tcp://127.0.0.2:39837" in info["workers"]
@@ -243,7 +243,7 @@ def test_respect_host_listen_address(loop, nanny, host):
             ["dask-worker", "127.0.0.1:8786", nanny, "--no-dashboard", "--host", host]
         ) as worker:
             with Client("127.0.0.1:8786") as client:
-                while not client.ncores():
+                while not client.nthreads():
                     sleep(0.1)
                 info = client.scheduler_info()
 
diff --git a/distributed/cli/tests/test_tls_cli.py b/distributed/cli/tests/test_tls_cli.py
index 4663a9b38ff..37fdc9bb00f 100644
--- a/distributed/cli/tests/test_tls_cli.py
+++ b/distributed/cli/tests/test_tls_cli.py
@@ -25,9 +25,9 @@
 tls_args_2 = ["--tls-ca-file", ca_file, "--tls-cert", cert, "--tls-key", key]
 
 
-def wait_for_cores(c, ncores=1):
+def wait_for_cores(c, nthreads=1):
     start = time()
-    while len(c.ncores()) < 1:
+    while len(c.nthreads()) < 1:
         sleep(0.1)
         assert time() < start + 10
 
diff --git a/distributed/client.py b/distributed/client.py
index ac098d7987e..7ad897bf616 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -770,12 +770,12 @@ def __repr__(self):
         if addr:
             workers = info.get("workers", {})
             nworkers = len(workers)
-            ncores = sum(w["ncores"] for w in workers.values())
+            nthreads = sum(w["nthreads"] for w in workers.values())
             return "<%s: scheduler=%r processes=%d cores=%d>" % (
                 self.__class__.__name__,
                 addr,
                 nworkers,
-                ncores,
+                nthreads,
             )
         elif self.scheduler is not None:
             return "<%s: scheduler=%r>" % (
@@ -830,7 +830,7 @@ def _repr_html_(self):
 
         if info:
             workers = len(info["workers"])
-            cores = sum(w["ncores"] for w in info["workers"].values())
+            cores = sum(w["nthreads"] for w in info["workers"].values())
             memory = sum(w["memory_limit"] for w in info["workers"].values())
             memory = format_bytes(memory)
             text2 = (
@@ -1868,19 +1868,19 @@ def _scatter(
         else:
             data2 = valmap(to_serialize, data)
             if direct:
-                ncores = None
+                nthreads = None
                 start = time()
-                while not ncores:
-                    if ncores is not None:
+                while not nthreads:
+                    if nthreads is not None:
                         yield gen.sleep(0.1)
                     if time() > start + timeout:
                         raise gen.TimeoutError("No valid workers found")
-                    ncores = yield self.scheduler.ncores(workers=workers)
-                if not ncores:
+                    nthreads = yield self.scheduler.ncores(workers=workers)
+                if not nthreads:
                     raise ValueError("No valid workers")
 
                 _, who_has, nbytes = yield scatter_to_workers(
-                    ncores, data2, report=False, rpc=self.rpc
+                    nthreads, data2, report=False, rpc=self.rpc
                 )
 
                 yield self.scheduler.update_data(
@@ -3013,7 +3013,7 @@ def replicate(self, futures, n=None, workers=None, branching_factor=2, **kwargs)
             **kwargs
         )
 
-    def ncores(self, workers=None, **kwargs):
+    def nthreads(self, workers=None, **kwargs):
         """ The number of threads/cores available on each worker node
 
         Parameters
@@ -3024,7 +3024,7 @@ def ncores(self, workers=None, **kwargs):
 
         Examples
         --------
-        >>> c.ncores()  # doctest: +SKIP
+        >>> c.threads()  # doctest: +SKIP
         {'192.168.1.141:46784': 8,
          '192.167.1.142:47548': 8,
          '192.167.1.143:47329': 8,
@@ -3043,6 +3043,8 @@ def ncores(self, workers=None, **kwargs):
             workers = [workers]
         return self.sync(self.scheduler.ncores, workers=workers, **kwargs)
 
+    ncores = nthreads
+
     def who_has(self, futures=None, **kwargs):
         """ The workers storing each future's data
 
@@ -3067,7 +3069,7 @@ def who_has(self, futures=None, **kwargs):
         See Also
         --------
         Client.has_what
-        Client.ncores
+        Client.nthreads
         """
         if futures is not None:
             futures = self.futures_of(futures)
@@ -3099,7 +3101,7 @@ def has_what(self, workers=None, **kwargs):
         See Also
         --------
         Client.who_has
-        Client.ncores
+        Client.nthreads
         Client.processing
         """
         if isinstance(workers, tuple) and all(
@@ -3130,7 +3132,7 @@ def processing(self, workers=None):
         --------
         Client.who_has
         Client.has_what
-        Client.ncores
+        Client.nthreads
         """
         if isinstance(workers, tuple) and all(
             isinstance(i, (str, tuple)) for i in workers
diff --git a/distributed/dashboard/components.py b/distributed/dashboard/components.py
index 16efa1d2eb0..e7234e2e6f7 100644
--- a/distributed/dashboard/components.py
+++ b/distributed/dashboard/components.py
@@ -276,7 +276,7 @@ class Processing(DashboardComponent):
     """
 
     def __init__(self, **kwargs):
-        data = self.processing_update({"processing": {}, "ncores": {}})
+        data = self.processing_update({"processing": {}, "nthreads": {}})
         self.source = ColumnDataSource(data)
 
         x_range = Range1d(-1, 1)
@@ -321,12 +321,12 @@ def __init__(self, **kwargs):
     def update(self, messages):
         with log_errors():
             msg = messages["processing"]
-            if not msg.get("ncores"):
+            if not msg.get("nthreads"):
                 return
             data = self.processing_update(msg)
             x_range = self.root.x_range
             max_right = max(data["right"])
-            cores = max(data["ncores"])
+            cores = max(data["nthreads"])
             if x_range.end < max_right:
                 x_range.end = max_right + 2
             elif x_range.end > 2 * max_right + cores:  # way out there, walk back
@@ -341,8 +341,8 @@ def processing_update(msg):
             names = sorted(names)
             processing = msg["processing"]
             processing = [processing[name] for name in names]
-            ncores = msg["ncores"]
-            ncores = [ncores[name] for name in names]
+            nthreads = msg["nthreads"]
+            nthreads = [nthreads[name] for name in names]
             n = len(names)
             d = {
                 "name": list(names),
@@ -350,7 +350,7 @@ def processing_update(msg):
                 "right": list(processing),
                 "top": list(range(n, 0, -1)),
                 "bottom": list(range(n - 1, -1, -1)),
-                "ncores": ncores,
+                "nthreads": nthreads,
             }
 
             d["alpha"] = [0.7] * n
diff --git a/distributed/dashboard/scheduler.py b/distributed/dashboard/scheduler.py
index 2cb916d0b5d..f6f1fef7590 100644
--- a/distributed/dashboard/scheduler.py
+++ b/distributed/dashboard/scheduler.py
@@ -189,7 +189,7 @@ def update(self):
             if total:
                 self.root.title.text = "Occupancy -- total time: %s  wall time: %s" % (
                     format_time(total),
-                    format_time(total / self.scheduler.total_ncores),
+                    format_time(total / self.scheduler.total_nthreads),
                 )
             else:
                 self.root.title.text = "Occupancy"
@@ -1179,7 +1179,7 @@ def __init__(self, scheduler, width=800, **kwargs):
         self.names = [
             "name",
             "address",
-            "ncores",
+            "nthreads",
             "cpu",
             "memory",
             "memory_limit",
@@ -1198,7 +1198,7 @@ def __init__(self, scheduler, width=800, **kwargs):
         table_names = [
             "name",
             "address",
-            "ncores",
+            "nthreads",
             "cpu",
             "memory",
             "memory_limit",
@@ -1223,7 +1223,7 @@ def __init__(self, scheduler, width=800, **kwargs):
             "read_bytes": NumberFormatter(format="0 b"),
             "write_bytes": NumberFormatter(format="0 b"),
             "num_fds": NumberFormatter(format="0"),
-            "ncores": NumberFormatter(format="0"),
+            "nthreads": NumberFormatter(format="0"),
         }
 
         if BOKEH_VERSION < "0.12.15":
@@ -1345,8 +1345,8 @@ def update(self):
                 data["memory_percent"][-1] = ""
             data["memory_limit"][-1] = ws.memory_limit
             data["cpu"][-1] = ws.metrics["cpu"] / 100.0
-            data["cpu_fraction"][-1] = ws.metrics["cpu"] / 100.0 / ws.ncores
-            data["ncores"][-1] = ws.ncores
+            data["cpu_fraction"][-1] = ws.metrics["cpu"] / 100.0 / ws.nthreads
+            data["nthreads"][-1] = ws.nthreads
 
         self.source.data.update(data)
 
diff --git a/distributed/dashboard/scheduler_html.py b/distributed/dashboard/scheduler_html.py
index 8b1da2035ea..65a89b33fbb 100644
--- a/distributed/dashboard/scheduler_html.py
+++ b/distributed/dashboard/scheduler_html.py
@@ -107,7 +107,7 @@ def get(self):
         scheduler = self.server
         erred = 0
         nbytes = 0
-        ncores = 0
+        nthreads = 0
         memory = 0
         processing = 0
         released = 0
@@ -124,7 +124,7 @@ def get(self):
             if ts.waiters:
                 waiting_data += 1
         for ws in scheduler.workers.values():
-            ncores += ws.ncores
+            nthreads += ws.nthreads
             memory += len(ws.has_what)
             nbytes += ws.nbytes
             processing += len(ws.processing)
@@ -132,7 +132,7 @@ def get(self):
         response = {
             "bytes": nbytes,
             "clients": len(scheduler.clients),
-            "cores": ncores,
+            "cores": nthreads,
             "erred": erred,
             "hosts": len(scheduler.host_info),
             "idle": len(scheduler.idle),
diff --git a/distributed/dashboard/templates/worker-table.html b/distributed/dashboard/templates/worker-table.html
index 4835849daad..a3566f90c3f 100644
--- a/distributed/dashboard/templates/worker-table.html
+++ b/distributed/dashboard/templates/worker-table.html
@@ -15,7 +15,7 @@
     <tr>
         <td><a href="../worker/{{ url_escape(ws.address) }}.html">{{ws.address}}</a></td>
         <td> {{ ws.name if ws.name is not None else "" }} </td>
-        <td> {{ ws.ncores }} </td>
+        <td> {{ ws.nthreads }} </td>
         <td> {{ format_bytes(ws.memory_limit) }} </td>
         <td> <progress class="progress" value="{{ ws.metrics['memory'] }}" max="{{ ws.memory_limit }}"></progress> </td>
         <td> {{ format_time(ws.occupancy) }} </td>
diff --git a/distributed/dashboard/tests/test_scheduler_bokeh.py b/distributed/dashboard/tests/test_scheduler_bokeh.py
index 692a29439c0..d9a83caf00b 100644
--- a/distributed/dashboard/tests/test_scheduler_bokeh.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh.py
@@ -321,8 +321,8 @@ def test_WorkerTable(c, s, a, b):
     assert all(wt.source.data.values())
     assert all(len(v) == 2 for v in wt.source.data.values())
 
-    ncores = wt.source.data["ncores"]
-    assert all(ncores)
+    nthreads = wt.source.data["nthreads"]
+    assert all(nthreads)
 
 
 @gen_cluster(client=True)
diff --git a/distributed/dashboard/worker.py b/distributed/dashboard/worker.py
index aa85afc4197..c6633a170aa 100644
--- a/distributed/dashboard/worker.py
+++ b/distributed/dashboard/worker.py
@@ -76,7 +76,7 @@ def update(self):
             w = self.worker
             d = {
                 "Stored": [len(w.data)],
-                "Executing": ["%d / %d" % (len(w.executing), w.ncores)],
+                "Executing": ["%d / %d" % (len(w.executing), w.nthreads)],
                 "Ready": [len(w.ready)],
                 "Waiting": [len(w.waiting_for_data)],
                 "Connections": [len(w.in_flight_workers)],
@@ -251,7 +251,7 @@ def __init__(self, worker, **kwargs):
         fig = figure(
             title="Executing History",
             x_axis_type="datetime",
-            y_range=[-0.1, worker.ncores + 0.1],
+            y_range=[-0.1, worker.nthreads + 0.1],
             height=150,
             tools="",
             x_range=x_range,
diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index a56cce8c2b2..ffb06b0a4bf 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -3,6 +3,7 @@
 import atexit
 import logging
 import math
+import multiprocessing
 import warnings
 import weakref
 
@@ -11,7 +12,7 @@
 from .spec import SpecCluster
 from ..nanny import Nanny
 from ..scheduler import Scheduler
-from ..worker import Worker, parse_memory_limit, _ncores
+from ..worker import Worker, parse_memory_limit
 
 logger = logging.getLogger(__name__)
 
@@ -70,7 +71,7 @@ class LocalCluster(SpecCluster):
     --------
     >>> cluster = LocalCluster()  # Create a local cluster with as many workers as cores  # doctest: +SKIP
     >>> cluster  # doctest: +SKIP
-    LocalCluster("127.0.0.1:8786", workers=8, ncores=8)
+    LocalCluster("127.0.0.1:8786", workers=8, threads=8)
 
     >>> c = Client(cluster)  # connect to local cluster  # doctest: +SKIP
 
@@ -141,21 +142,23 @@ def __init__(
         worker_services = worker_services or {}
         if n_workers is None and threads_per_worker is None:
             if processes:
-                n_workers, threads_per_worker = nprocesses_nthreads(_ncores)
+                n_workers, threads_per_worker = nprocesses_nthreads()
             else:
                 n_workers = 1
-                threads_per_worker = _ncores
+                threads_per_worker = multiprocessing.cpu_count()
         if n_workers is None and threads_per_worker is not None:
-            n_workers = max(1, _ncores // threads_per_worker)
+            n_workers = max(1, multiprocessing.cpu_count() // threads_per_worker)
         if n_workers and threads_per_worker is None:
             # Overcommit threads per worker, rather than undercommit
-            threads_per_worker = max(1, int(math.ceil(_ncores / n_workers)))
+            threads_per_worker = max(
+                1, int(math.ceil(multiprocessing.cpu_count() / n_workers))
+            )
         if n_workers and "memory_limit" not in worker_kwargs:
             worker_kwargs["memory_limit"] = parse_memory_limit("auto", 1, n_workers)
 
         worker_kwargs.update(
             {
-                "ncores": threads_per_worker,
+                "nthreads": threads_per_worker,
                 "services": worker_services,
                 "dashboard_address": worker_dashboard_address,
                 "interface": interface,
@@ -197,15 +200,15 @@ def __init__(
         )
 
     def __repr__(self):
-        return "%s(%r, workers=%d, ncores=%d)" % (
+        return "%s(%r, workers=%d, nthreads=%d)" % (
             type(self).__name__,
             self.scheduler_address,
             len(self.workers),
-            sum(w.ncores for w in self.workers.values()),
+            sum(w.nthreads for w in self.workers.values()),
         )
 
 
-def nprocesses_nthreads(n):
+def nprocesses_nthreads(n=multiprocessing.cpu_count()):
     """
     The default breakdown of processes and threads for a given number of cores
 
diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index 85728a057e4..bb46f81db88 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -44,8 +44,8 @@ class does handle all of the logic around asynchronously cleanly setting up
     >>> from dask.distributed import Scheduler, Worker, Nanny
     >>> scheduler = {'cls': Scheduler, 'options': {"dashboard_address": ':8787'}}
     >>> workers = {
-    ...     'my-worker': {"cls": Worker, "options": {"ncores": 1}},
-    ...     'my-nanny': {"cls": Nanny, "options": {"ncores": 2}},
+    ...     'my-worker': {"cls": Worker, "options": {"nthreads": 1}},
+    ...     'my-nanny': {"cls": Nanny, "options": {"nthreads": 2}},
     ... }
     >>> cluster = SpecCluster(scheduler=scheduler, workers=workers)
 
@@ -53,8 +53,8 @@ class does handle all of the logic around asynchronously cleanly setting up
 
     >>> cluster.worker_spec
     {
-       'my-worker': {"cls": Worker, "options": {"ncores": 1}},
-       'my-nanny': {"cls": Nanny, "options": {"ncores": 2}},
+       'my-worker': {"cls": Worker, "options": {"nthreads": 1}},
+       'my-nanny': {"cls": Nanny, "options": {"nthreads": 2}},
     }
 
     While the instantiation of this spec is stored in the ``.workers``
diff --git a/distributed/deploy/tests/test_adaptive.py b/distributed/deploy/tests/test_adaptive.py
index cc860636e55..146d7b95dbb 100644
--- a/distributed/deploy/tests/test_adaptive.py
+++ b/distributed/deploy/tests/test_adaptive.py
@@ -23,11 +23,11 @@ def test_get_scale_up_kwargs(loop):
         with Client(cluster, loop=loop) as c:
             future = c.submit(lambda x: x + 1, 1)
             assert future.result() == 2
-            assert c.ncores()
+            assert c.nthreads()
             assert alc.get_scale_up_kwargs() == {"n": 3}
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 4)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 4)
 def test_simultaneous_scale_up_and_down(c, s, *workers):
     class TestAdaptive(Adaptive):
         def get_scale_up_kwargs(self):
@@ -65,22 +65,22 @@ def test_adaptive_local_cluster(loop):
     ) as cluster:
         alc = Adaptive(cluster.scheduler, cluster, interval=100)
         with Client(cluster, loop=loop) as c:
-            assert not c.ncores()
+            assert not c.nthreads()
             future = c.submit(lambda x: x + 1, 1)
             assert future.result() == 2
-            assert c.ncores()
+            assert c.nthreads()
 
             sleep(0.1)
-            assert c.ncores()  # still there after some time
+            assert c.nthreads()  # still there after some time
 
             del future
 
             start = time()
-            while cluster.scheduler.ncores:
+            while cluster.scheduler.nthreads:
                 sleep(0.01)
                 assert time() < start + 5
 
-            assert not c.ncores()
+            assert not c.nthreads()
 
 
 @nodebug
@@ -128,7 +128,7 @@ def test_adaptive_local_cluster_multi_workers():
         yield cluster.close()
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 10, active_rpc_timeout=10)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 10, active_rpc_timeout=10)
 def test_adaptive_scale_down_override(c, s, *workers):
     class TestAdaptive(Adaptive):
         def __init__(self, *args, **kwargs):
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index 8aad6675f8c..520996f64a0 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -2,6 +2,7 @@
 
 from functools import partial
 import gc
+import multiprocessing
 import subprocess
 import sys
 from time import sleep
@@ -90,7 +91,7 @@ def test_procs():
         assert len(c.workers) == 2
         assert all(isinstance(w, Worker) for w in c.workers.values())
         with Client(c.scheduler.address) as e:
-            assert all(w.ncores == 3 for w in c.workers.values())
+            assert all(w.nthreads == 3 for w in c.workers.values())
             assert all(isinstance(w, Worker) for w in c.workers.values())
         repr(c)
 
@@ -105,7 +106,7 @@ def test_procs():
         assert len(c.workers) == 2
         assert all(isinstance(w, Nanny) for w in c.workers.values())
         with Client(c.scheduler.address) as e:
-            assert all(v == 3 for v in e.ncores().values())
+            assert all(v == 3 for v in e.nthreads().values())
 
             c.scale(3)
             assert all(isinstance(w, Nanny) for w in c.workers.values())
@@ -181,7 +182,7 @@ def test_Client_with_local(loop):
         1, scheduler_port=0, silence_logs=False, dashboard_address=None, loop=loop
     ) as c:
         with Client(c) as e:
-            assert len(e.ncores()) == len(c.workers)
+            assert len(e.nthreads()) == len(c.workers)
             assert c.scheduler_address in repr(c)
 
 
@@ -227,33 +228,33 @@ def test_Client_twice(loop):
 
 @pytest.mark.skipif("sys.version_info[0] == 2", reason="fork issues")
 def test_defaults():
-    from distributed.worker import _ncores
+    _nthreads = multiprocessing.cpu_count()
 
     with LocalCluster(
         scheduler_port=0, silence_logs=False, dashboard_address=None
     ) as c:
-        assert sum(w.ncores for w in c.workers.values()) == _ncores
+        assert sum(w.nthreads for w in c.workers.values()) == _nthreads
         assert all(isinstance(w, Nanny) for w in c.workers.values())
 
     with LocalCluster(
         processes=False, scheduler_port=0, silence_logs=False, dashboard_address=None
     ) as c:
-        assert sum(w.ncores for w in c.workers.values()) == _ncores
+        assert sum(w.nthreads for w in c.workers.values()) == _nthreads
         assert all(isinstance(w, Worker) for w in c.workers.values())
         assert len(c.workers) == 1
 
     with LocalCluster(
         n_workers=2, scheduler_port=0, silence_logs=False, dashboard_address=None
     ) as c:
-        if _ncores % 2 == 0:
-            expected_total_threads = max(2, _ncores)
+        if _nthreads % 2 == 0:
+            expected_total_threads = max(2, _nthreads)
         else:
-            # n_workers not a divisor of _ncores => threads are overcommitted
-            expected_total_threads = max(2, _ncores + 1)
-        assert sum(w.ncores for w in c.workers.values()) == expected_total_threads
+            # n_workers not a divisor of _nthreads => threads are overcommitted
+            expected_total_threads = max(2, _nthreads + 1)
+        assert sum(w.nthreads for w in c.workers.values()) == expected_total_threads
 
     with LocalCluster(
-        threads_per_worker=_ncores * 2,
+        threads_per_worker=_nthreads * 2,
         scheduler_port=0,
         silence_logs=False,
         dashboard_address=None,
@@ -261,12 +262,12 @@ def test_defaults():
         assert len(c.workers) == 1
 
     with LocalCluster(
-        n_workers=_ncores * 2,
+        n_workers=_nthreads * 2,
         scheduler_port=0,
         silence_logs=False,
         dashboard_address=None,
     ) as c:
-        assert all(w.ncores == 1 for w in c.workers.values())
+        assert all(w.nthreads == 1 for w in c.workers.values())
     with LocalCluster(
         threads_per_worker=2,
         n_workers=3,
@@ -275,7 +276,7 @@ def test_defaults():
         dashboard_address=None,
     ) as c:
         assert len(c.workers) == 3
-        assert all(w.ncores == 2 for w in c.workers.values())
+        assert all(w.nthreads == 2 for w in c.workers.values())
 
 
 def test_worker_params():
@@ -361,7 +362,7 @@ def test_bokeh(loop, processes):
 @pytest.mark.skipif(sys.version_info < (3, 6), reason="Unknown")
 def test_blocks_until_full(loop):
     with Client(loop=loop) as c:
-        assert len(c.ncores()) > 0
+        assert len(c.nthreads()) > 0
 
 
 @gen_test()
@@ -383,7 +384,7 @@ def test_scale_up_and_down():
     cluster.scale(2)
     yield cluster
     assert len(cluster.workers) == 2
-    assert len(cluster.scheduler.ncores) == 2
+    assert len(cluster.scheduler.nthreads) == 2
 
     cluster.scale(1)
     yield cluster
diff --git a/distributed/deploy/tests/test_spec_cluster.py b/distributed/deploy/tests/test_spec_cluster.py
index eb733f2e68f..0c062d3d3e0 100644
--- a/distributed/deploy/tests/test_spec_cluster.py
+++ b/distributed/deploy/tests/test_spec_cluster.py
@@ -17,9 +17,9 @@ async def _():
 
 
 worker_spec = {
-    0: {"cls": Worker, "options": {"ncores": 1}},
-    1: {"cls": Worker, "options": {"ncores": 2}},
-    "my-worker": {"cls": MyWorker, "options": {"ncores": 3}},
+    0: {"cls": Worker, "options": {"nthreads": 1}},
+    1: {"cls": Worker, "options": {"nthreads": 2}},
+    "my-worker": {"cls": MyWorker, "options": {"nthreads": 3}},
 }
 scheduler = {"cls": Scheduler, "options": {"port": 0}}
 
@@ -37,9 +37,9 @@ async def test_specification():
         assert isinstance(cluster.workers[1], Worker)
         assert isinstance(cluster.workers["my-worker"], MyWorker)
 
-        assert cluster.workers[0].ncores == 1
-        assert cluster.workers[1].ncores == 2
-        assert cluster.workers["my-worker"].ncores == 3
+        assert cluster.workers[0].nthreads == 1
+        assert cluster.workers[1].nthreads == 2
+        assert cluster.workers["my-worker"].nthreads == 3
 
         async with Client(cluster, asynchronous=True) as client:
             result = await client.submit(lambda x: x + 1, 10)
@@ -51,9 +51,9 @@ async def test_specification():
 
 def test_spec_sync(loop):
     worker_spec = {
-        0: {"cls": Worker, "options": {"ncores": 1}},
-        1: {"cls": Worker, "options": {"ncores": 2}},
-        "my-worker": {"cls": MyWorker, "options": {"ncores": 3}},
+        0: {"cls": Worker, "options": {"nthreads": 1}},
+        1: {"cls": Worker, "options": {"nthreads": 2}},
+        "my-worker": {"cls": MyWorker, "options": {"nthreads": 3}},
     }
     with SpecCluster(workers=worker_spec, scheduler=scheduler, loop=loop) as cluster:
         assert cluster.worker_spec is worker_spec
@@ -64,9 +64,9 @@ def test_spec_sync(loop):
         assert isinstance(cluster.workers[1], Worker)
         assert isinstance(cluster.workers["my-worker"], MyWorker)
 
-        assert cluster.workers[0].ncores == 1
-        assert cluster.workers[1].ncores == 2
-        assert cluster.workers["my-worker"].ncores == 3
+        assert cluster.workers[0].nthreads == 1
+        assert cluster.workers[1].nthreads == 2
+        assert cluster.workers["my-worker"].nthreads == 3
 
         with Client(cluster, loop=loop) as client:
             assert cluster.loop is cluster.scheduler.loop
@@ -83,7 +83,7 @@ def test_loop_started():
 
 @pytest.mark.asyncio
 async def test_scale():
-    worker = {"cls": Worker, "options": {"ncores": 1}}
+    worker = {"cls": Worker, "options": {"nthreads": 1}}
     async with SpecCluster(
         asynchronous=True, scheduler=scheduler, worker=worker
     ) as cluster:
@@ -134,9 +134,9 @@ async def test_new_worker_spec():
     class MyCluster(SpecCluster):
         def new_worker_spec(self):
             i = len(self.worker_spec)
-            return i, {"cls": Worker, "options": {"ncores": i + 1}}
+            return i, {"cls": Worker, "options": {"nthreads": i + 1}}
 
     async with MyCluster(asynchronous=True, scheduler=scheduler) as cluster:
         cluster.scale(3)
         for i in range(3):
-            assert cluster.worker_spec[i]["options"]["ncores"] == i + 1
+            assert cluster.worker_spec[i]["options"]["nthreads"] == i + 1
diff --git a/distributed/deploy/utils_test.py b/distributed/deploy/utils_test.py
index 9da8d64cd50..2bb55c7da08 100644
--- a/distributed/deploy/utils_test.py
+++ b/distributed/deploy/utils_test.py
@@ -18,7 +18,7 @@ def tearDown(self):
     @pytest.mark.xfail()
     def test_cores(self):
         info = self.client.scheduler_info()
-        assert len(self.client.ncores()) == 2
+        assert len(self.client.nthreads()) == 2
 
     def test_submit(self):
         future = self.client.submit(lambda x: x + 1, 1)
@@ -27,7 +27,7 @@ def test_submit(self):
     def test_context_manager(self):
         with self.Cluster(**self.kwargs) as c:
             with Client(c) as e:
-                assert e.ncores()
+                assert e.nthreads()
 
     def test_no_workers(self):
         with self.Cluster(0, scheduler_port=0, **self.kwargs):
diff --git a/distributed/diagnostics/tests/test_eventstream.py b/distributed/diagnostics/tests/test_eventstream.py
index 7ec646d7e91..9139f75eab3 100644
--- a/distributed/diagnostics/tests/test_eventstream.py
+++ b/distributed/diagnostics/tests/test_eventstream.py
@@ -11,7 +11,7 @@
 from distributed.utils_test import div, gen_cluster
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 3)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 3)
 def test_eventstream(c, s, *workers):
     pytest.importorskip("bokeh")
 
diff --git a/distributed/diagnostics/tests/test_plugin.py b/distributed/diagnostics/tests/test_plugin.py
index fa4449c74b7..1c9ebd7a1a8 100644
--- a/distributed/diagnostics/tests/test_plugin.py
+++ b/distributed/diagnostics/tests/test_plugin.py
@@ -34,7 +34,7 @@ def transition(self, key, start, finish, *args, **kwargs):
     assert counter not in s.plugins
 
 
-@gen_cluster(ncores=[], client=False)
+@gen_cluster(nthreads=[], client=False)
 def test_add_remove_worker(s):
     events = []
 
diff --git a/distributed/diagnostics/tests/test_progressbar.py b/distributed/diagnostics/tests/test_progressbar.py
index ba42f2ce6ea..3e5f0633d49 100644
--- a/distributed/diagnostics/tests/test_progressbar.py
+++ b/distributed/diagnostics/tests/test_progressbar.py
@@ -44,7 +44,7 @@ def test_TextProgressBar_empty(capsys):
     @gen_test()
     def f():
         s = yield Scheduler(port=0)
-        a, b = yield [Worker(s.address, ncores=1), Worker(s.address, ncores=1)]
+        a, b = yield [Worker(s.address, nthreads=1), Worker(s.address, nthreads=1)]
 
         progress = TextProgressBar([], scheduler=s.address, start=False, interval=0.01)
         yield progress.listen()
diff --git a/distributed/diagnostics/tests/test_task_stream.py b/distributed/diagnostics/tests/test_task_stream.py
index 366de8d79d5..ad23ca5ae8c 100644
--- a/distributed/diagnostics/tests/test_task_stream.py
+++ b/distributed/diagnostics/tests/test_task_stream.py
@@ -14,7 +14,7 @@
 from distributed.metrics import time
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 3)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 3)
 def test_TaskStreamPlugin(c, s, *workers):
     es = TaskStreamPlugin(s)
     assert not es.buffer
diff --git a/distributed/nanny.py b/distributed/nanny.py
index 9cf444fc7c4..d907c7171a1 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -3,6 +3,7 @@
 from datetime import timedelta
 import logging
 from multiprocessing.queues import Empty
+import multiprocessing
 import os
 import psutil
 import shutil
@@ -32,7 +33,7 @@
     PeriodicCallback,
     parse_timedelta,
 )
-from .worker import _ncores, run, parse_memory_limit, Worker
+from .worker import run, parse_memory_limit, Worker
 
 logger = logging.getLogger(__name__)
 
@@ -54,6 +55,7 @@ def __init__(
         scheduler_port=None,
         scheduler_file=None,
         worker_port=0,
+        nthreads=None,
         ncores=None,
         loop=None,
         local_dir="dask-worker-space",
@@ -96,8 +98,12 @@ def __init__(
         else:
             self.scheduler_addr = coerce_to_address((scheduler_ip, scheduler_port))
 
+        if ncores is not None:
+            warnings.warn("the ncores= parameter has moved to nthreads=")
+            nthreads = ncores
+
         self._given_worker_port = worker_port
-        self.ncores = ncores or _ncores
+        self.nthreads = nthreads or multiprocessing.cpu_count()
         self.reconnect = reconnect
         self.validate = validate
         self.resources = resources
@@ -120,7 +126,7 @@ def __init__(
         self.quiet = quiet
         self.auto_restart = True
 
-        self.memory_limit = parse_memory_limit(memory_limit, self.ncores)
+        self.memory_limit = parse_memory_limit(memory_limit, self.nthreads)
 
         if silence_logs:
             silence_logging(level=silence_logs)
@@ -160,7 +166,7 @@ def __init__(
         self.status = "init"
 
     def __repr__(self):
-        return "<Nanny: %s, threads: %d>" % (self.worker_address, self.ncores)
+        return "<Nanny: %s, threads: %d>" % (self.worker_address, self.nthreads)
 
     @gen.coroutine
     def _unregister(self, timeout=10):
@@ -263,7 +269,7 @@ def instantiate(self, comm=None):
         if self.process is None:
             worker_kwargs = dict(
                 scheduler_ip=self.scheduler_addr,
-                ncores=self.ncores,
+                nthreads=self.nthreads,
                 local_dir=self.local_dir,
                 services=self.services,
                 nanny=self.address,
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index ae449bcfafe..434d118a422 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -162,9 +162,9 @@ class WorkerState(object):
        The total memory size, in bytes, used by the tasks this worker
        holds in memory (i.e. the tasks in this worker's :attr:`has_what`).
 
-    .. attribute:: ncores: int
+    .. attribute:: nthreads: int
 
-       The number of CPU cores made available on this worker.
+       The number of CPU threads made available on this worker.
 
     .. attribute:: resources: {str: Number}
 
@@ -218,7 +218,7 @@ class WorkerState(object):
         "name",
         "nanny",
         "nbytes",
-        "ncores",
+        "nthreads",
         "occupancy",
         "pid",
         "processing",
@@ -234,7 +234,7 @@ def __init__(
         address=None,
         pid=0,
         name=None,
-        ncores=0,
+        nthreads=0,
         memory_limit=0,
         local_directory=None,
         services=None,
@@ -243,7 +243,7 @@ def __init__(
         self.address = address
         self.pid = pid
         self.name = name
-        self.ncores = ncores
+        self.nthreads = nthreads
         self.memory_limit = memory_limit
         self.local_directory = local_directory
         self.services = services or {}
@@ -272,7 +272,7 @@ def clean(self):
             address=self.address,
             pid=self.pid,
             name=self.name,
-            ncores=self.ncores,
+            nthreads=self.nthreads,
             memory_limit=self.memory_limit,
             local_directory=self.local_directory,
             services=self.services,
@@ -299,7 +299,7 @@ def identity(self):
             "resources": self.resources,
             "local_directory": self.local_directory,
             "name": self.name,
-            "ncores": self.ncores,
+            "nthreads": self.nthreads,
             "memory_limit": self.memory_limit,
             "last_seen": self.last_seen,
             "services": self.services,
@@ -963,7 +963,7 @@ def __init__(
         # Worker state
         self.workers = sortedcontainers.SortedDict()
         for old_attr, new_attr, wrap in [
-            ("ncores", "ncores", None),
+            ("nthreads", "nthreads", None),
             ("worker_bytes", "nbytes", None),
             ("worker_resources", "resources", None),
             ("used_resources", "used_resources", None),
@@ -980,7 +980,7 @@ def __init__(
         self.idle = sortedcontainers.SortedSet(key=operator.attrgetter("address"))
         self.saturated = set()
 
-        self.total_ncores = 0
+        self.total_nthreads = 0
         self.total_occupancy = 0
         self.host_info = defaultdict(dict)
         self.resources = defaultdict(dict)
@@ -1128,7 +1128,7 @@ def __repr__(self):
         return '<Scheduler: "%s" processes: %d cores: %d>' % (
             self.address,
             len(self.workers),
-            self.total_ncores,
+            self.total_nthreads,
         )
 
     def identity(self, comm=None):
@@ -1394,7 +1394,7 @@ def add_worker(
         comm=None,
         address=None,
         keys=(),
-        ncores=None,
+        nthreads=None,
         name=None,
         resolve_address=True,
         nbytes=None,
@@ -1422,7 +1422,7 @@ def add_worker(
             self.workers[address] = ws = WorkerState(
                 address=address,
                 pid=pid,
-                ncores=ncores,
+                nthreads=nthreads,
                 memory_limit=memory_limit,
                 name=name,
                 local_directory=local_directory,
@@ -1440,12 +1440,12 @@ def add_worker(
                 return
 
             if "addresses" not in self.host_info[host]:
-                self.host_info[host].update({"addresses": set(), "cores": 0})
+                self.host_info[host].update({"addresses": set(), "nthreads": 0})
 
             self.host_info[host]["addresses"].add(address)
-            self.host_info[host]["cores"] += ncores
+            self.host_info[host]["nthreads"] += nthreads
 
-            self.total_ncores += ncores
+            self.total_nthreads += nthreads
             self.aliases[name] = address
 
             response = self.heartbeat_worker(
@@ -1465,7 +1465,7 @@ def add_worker(
 
             self.stream_comms[address] = BatchedSend(interval="5ms", loop=self.loop)
 
-            if ws.ncores > len(ws.processing):
+            if ws.nthreads > len(ws.processing):
                 self.idle.add(ws)
 
             for plugin in self.plugins[:]:
@@ -1906,9 +1906,9 @@ def remove_worker(self, comm=None, address=None, safe=False, close=True):
 
             self.remove_resources(address)
 
-            self.host_info[host]["cores"] -= ws.ncores
+            self.host_info[host]["nthreads"] -= ws.nthreads
             self.host_info[host]["addresses"].remove(address)
-            self.total_ncores -= ws.ncores
+            self.total_nthreads -= ws.nthreads
 
             if not self.host_info[host]["addresses"]:
                 del self.host_info[host]
@@ -2489,22 +2489,22 @@ def scatter(
                 raise gen.TimeoutError("No workers found")
 
         if workers is None:
-            ncores = {w: ws.ncores for w, ws in self.workers.items()}
+            nthreads = {w: ws.nthreads for w, ws in self.workers.items()}
         else:
             workers = [self.coerce_address(w) for w in workers]
-            ncores = {w: self.workers[w].ncores for w in workers}
+            nthreads = {w: self.workers[w].nthreads for w in workers}
 
         assert isinstance(data, dict)
 
         keys, who_has, nbytes = yield scatter_to_workers(
-            ncores, data, rpc=self.rpc, report=False
+            nthreads, data, rpc=self.rpc, report=False
         )
 
         self.update_data(who_has=who_has, nbytes=nbytes, client=client)
 
         if broadcast:
             if broadcast == True:  # noqa: E712
-                n = len(ncores)
+                n = len(nthreads)
             else:
                 n = broadcast
             yield self.replicate(keys=keys, workers=workers, n=n)
@@ -3283,9 +3283,9 @@ def get_has_what(self, comm=None, workers=None):
     def get_ncores(self, comm=None, workers=None):
         if workers is not None:
             workers = map(self.coerce_address, workers)
-            return {w: self.workers[w].ncores for w in workers if w in self.workers}
+            return {w: self.workers[w].nthreads for w in workers if w in self.workers}
         else:
-            return {w: ws.ncores for w, ws in self.workers.items()}
+            return {w: ws.nthreads for w, ws in self.workers.items()}
 
     @gen.coroutine
     def get_call_stack(self, comm=None, keys=None):
@@ -4363,19 +4363,19 @@ def check_idle_saturated(self, ws, occ=None):
         -  Idle: do not have enough work to stay busy
 
         They are considered saturated if they both have enough tasks to occupy
-        all of their cores, and if the expected runtime of those tasks is large
-        enough.
+        all of their threads, and if the expected runtime of those tasks is
+        large enough.
 
         This is useful for load balancing and adaptivity.
         """
-        if self.total_ncores == 0 or ws.status == "closed":
+        if self.total_nthreads == 0 or ws.status == "closed":
             return
         if occ is None:
             occ = ws.occupancy
-        nc = ws.ncores
+        nc = ws.nthreads
         p = len(ws.processing)
 
-        avg = self.total_occupancy / self.total_ncores
+        avg = self.total_occupancy / self.total_nthreads
 
         if p < nc or occ / nc < avg / 2:
             self.idle.add(ws)
@@ -4538,7 +4538,7 @@ def worker_objective(self, ts, ws):
         comm_bytes = sum(
             [dts.get_nbytes() for dts in ts.dependencies if ws not in dts.who_has]
         )
-        stack_time = ws.occupancy / ws.ncores
+        stack_time = ws.occupancy / ws.nthreads
         start_time = comm_bytes / self.bandwidth + stack_time
 
         if ts.actor:
diff --git a/distributed/stealing.py b/distributed/stealing.py
index dc8c989e39d..afcdf2a1cfa 100644
--- a/distributed/stealing.py
+++ b/distributed/stealing.py
@@ -333,7 +333,7 @@ def maybe_move_task(level, ts, sat, idl, duration, cost_multiplier):
                 saturated = [
                     ws
                     for ws in saturated
-                    if combined_occupancy(ws) > 0.2 and len(ws.processing) > ws.ncores
+                    if combined_occupancy(ws) > 0.2 and len(ws.processing) > ws.nthreads
                 ]
             elif len(s.saturated) < 20:
                 saturated = sorted(saturated, key=combined_occupancy, reverse=True)
@@ -379,7 +379,7 @@ def maybe_move_task(level, ts, sat, idl, duration, cost_multiplier):
                             continue
                         if combined_occupancy(sat) < 0.2:
                             continue
-                        if len(sat.processing) <= sat.ncores:
+                        if len(sat.processing) <= sat.nthreads:
                             continue
 
                         i += 1
diff --git a/distributed/tests/test_actor.py b/distributed/tests/test_actor.py
index ec2636ccd50..fd6bf0335e1 100644
--- a/distributed/tests/test_actor.py
+++ b/distributed/tests/test_actor.py
@@ -341,13 +341,13 @@ def add(n, counter):
     done = c.submit(lambda x: None, futures)
 
     while not done.done():
-        assert len(s.processing) <= a.ncores + b.ncores
+        assert len(s.processing) <= a.nthreads + b.nthreads
         yield gen.sleep(0.01)
 
     yield done
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 5)] * 2)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 5)] * 2)
 def test_thread_safety(c, s, a, b):
     class Unsafe(object):
         def __init__(self):
@@ -394,7 +394,7 @@ def __init__(self, x):
     assert s.tasks[x.key].who_has != s.tasks[y.key].who_has  # second load balanced
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 5)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 5)
 def test_load_balance_map(c, s, *workers):
     class Foo(object):
         def __init__(self, x, y=None):
@@ -409,7 +409,7 @@ def __init__(self, x, y=None):
     assert all(len(w.actors) == 2 for w in workers)
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 4, Worker=Nanny)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 4, Worker=Nanny)
 def bench_param_server(c, s, *workers):
     import dask.array as da
     import numpy as np
@@ -506,7 +506,7 @@ def check(dask_worker):
 
 @gen_cluster(
     client=True,
-    ncores=[("127.0.0.1", 1)],
+    nthreads=[("127.0.0.1", 1)],
     config={"distributed.worker.profile.interval": "1ms"},
 )
 def test_actors_in_profile(c, s, a):
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index dc45b3025e5..d18216ef0ef 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -558,7 +558,7 @@ def test_gather_strict(c, s, a, b):
     assert xx == 2
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)])
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)])
 def test_gather_skip(c, s, a):
     x = c.submit(div, 1, 0, priority=10)
     y = c.submit(slowinc, 1, delay=0.5)
@@ -953,7 +953,7 @@ def test_remove_worker(c, s, a, b):
     assert result == list(map(inc, range(20)))
 
 
-@gen_cluster(ncores=[("127.0.0.1", 1)], client=True)
+@gen_cluster(nthreads=[("127.0.0.1", 1)], client=True)
 def test_errors_dont_block(c, s, w):
     L = [c.submit(inc, 1), c.submit(throws, 1), c.submit(inc, 2), c.submit(throws, 2)]
 
@@ -1359,13 +1359,13 @@ def test_scatter_direct_broadcast(c, s, a, b):
     assert not s.counters["op"].components[0]["scatter"]
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 4)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 4)
 def test_scatter_direct_balanced(c, s, *workers):
     futures = yield c.scatter([1, 2, 3], direct=True)
     assert sorted([len(w.data) for w in workers]) == [0, 1, 1, 1]
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 4)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 4)
 def test_scatter_direct_broadcast_target(c, s, *workers):
     futures = yield c.scatter([123, 456], direct=True, workers=workers[0].address)
     assert futures[0].key in workers[0].data
@@ -1384,13 +1384,13 @@ def test_scatter_direct_broadcast_target(c, s, *workers):
     )
 
 
-@gen_cluster(client=True, ncores=[])
+@gen_cluster(client=True, nthreads=[])
 def test_scatter_direct_empty(c, s):
     with pytest.raises((ValueError, gen.TimeoutError)):
         yield c.scatter(123, direct=True, timeout=0.1)
 
 
-@gen_cluster(client=True, timeout=None, ncores=[("127.0.0.1", 1)] * 5)
+@gen_cluster(client=True, timeout=None, nthreads=[("127.0.0.1", 1)] * 5)
 def test_scatter_direct_spread_evenly(c, s, *workers):
     futures = []
     for i in range(10):
@@ -1724,7 +1724,7 @@ def test_start_is_idempotent(c):
 
 @gen_cluster(client=True)
 def test_client_with_scheduler(c, s, a, b):
-    assert s.ncores == {a.address: a.ncores, b.address: b.ncores}
+    assert s.nthreads == {a.address: a.nthreads, b.address: b.nthreads}
 
     x = c.submit(inc, 1)
     y = c.submit(inc, 2)
@@ -2171,7 +2171,7 @@ def test__broadcast(c, s, a, b):
     assert a.data == b.data == {x.key: 1, y.key: 2}
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 4)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 4)
 def test__broadcast_integer(c, s, *workers):
     x, y = yield c.scatter([1, 2], broadcast=2)
     assert len(s.tasks[x.key].who_has) == 2
@@ -2486,7 +2486,7 @@ def test_futures_of_cancelled_raises(c, s, a, b):
 
 
 @pytest.mark.skip
-@gen_cluster(ncores=[("127.0.0.1", 1)], client=True)
+@gen_cluster(nthreads=[("127.0.0.1", 1)], client=True)
 def test_dont_delete_recomputed_results(c, s, w):
     x = c.submit(inc, 1)  # compute first time
     yield wait([x])
@@ -2504,7 +2504,7 @@ def test_dont_delete_recomputed_results(c, s, w):
         yield gen.sleep(0.01)
 
 
-@gen_cluster(ncores=[], client=True)
+@gen_cluster(nthreads=[], client=True)
 def test_fatally_serialized_input(c, s):
     o = FatallySerializedObject()
 
@@ -2613,14 +2613,14 @@ def test_diagnostic_ui(loop):
         a_addr = a["address"]
         b_addr = b["address"]
         with Client(s["address"], loop=loop) as c:
-            d = c.ncores()
+            d = c.nthreads()
             assert d == {a_addr: 1, b_addr: 1}
 
-            d = c.ncores([a_addr])
+            d = c.nthreads([a_addr])
             assert d == {a_addr: 1}
-            d = c.ncores(a_addr)
+            d = c.nthreads(a_addr)
             assert d == {a_addr: 1}
-            d = c.ncores(a["address"])
+            d = c.nthreads(a["address"])
             assert d == {a_addr: 1}
 
             x = c.submit(inc, 1)
@@ -2813,7 +2813,7 @@ def test_rebalance(c, s, a, b):
     assert aws not in s.tasks[x.key].who_has or aws not in s.tasks[y.key].who_has
 
 
-@gen_cluster(ncores=[("127.0.0.1", 1)] * 4, client=True)
+@gen_cluster(nthreads=[("127.0.0.1", 1)] * 4, client=True)
 def test_rebalance_workers(e, s, a, b, c, d):
     w, x, y, z = yield e.scatter([1, 2, 3, 4], workers=[a.address])
     assert len(a.data) == 4
@@ -2903,11 +2903,11 @@ def test_unrunnable_task_runs(c, s, a, b):
     yield w.close()
 
 
-@gen_cluster(client=True, ncores=[])
+@gen_cluster(client=True, nthreads=[])
 def test_add_worker_after_tasks(c, s):
     futures = c.map(inc, range(10))
 
-    n = yield Nanny(s.address, ncores=2, loop=s.loop, port=0)
+    n = yield Nanny(s.address, nthreads=2, loop=s.loop, port=0)
 
     result = yield c.gather(futures)
 
@@ -2939,7 +2939,7 @@ def test_submit_on_cancelled_future(c, s, a, b):
         y = c.submit(inc, x)
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 10)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 10)
 def test_replicate(c, s, *workers):
     [a, b] = yield c.scatter([1, 2])
     yield s.replicate(keys=[a.key, b.key], n=5)
@@ -2964,7 +2964,7 @@ def test_replicate_tuple_keys(c, s, a, b):
     s.validate_state()
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 10)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 10)
 def test_replicate_workers(c, s, *workers):
 
     [a, b] = yield c.scatter([1, 2], workers=[workers[0].address])
@@ -3015,7 +3015,7 @@ def __getstate__(self):
         return self.n
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 10)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 10)
 def test_replicate_tree_branching(c, s, *workers):
     obj = CountSerialization()
     [future] = yield c.scatter([obj])
@@ -3025,7 +3025,7 @@ def test_replicate_tree_branching(c, s, *workers):
     assert max_count > 1
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 10)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 10)
 def test_client_replicate(c, s, *workers):
     x = c.submit(inc, 1)
     y = c.submit(inc, 2)
@@ -3051,7 +3051,7 @@ def test_client_replicate(c, s, *workers):
 )
 @gen_cluster(
     client=True,
-    ncores=[("127.0.0.1", 1), ("127.0.0.2", 1), ("127.0.0.2", 1)],
+    nthreads=[("127.0.0.1", 1), ("127.0.0.2", 1), ("127.0.0.2", 1)],
     timeout=None,
 )
 def test_client_replicate_host(client, s, a, b, c):
@@ -3087,7 +3087,7 @@ def test_client_replicate_sync(c):
 @pytest.mark.skipif(
     sys.platform.startswith("win"), reason="Windows timer too coarse-grained"
 )
-@gen_cluster(client=True, ncores=[("127.0.0.1", 4)] * 1)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 4)] * 1)
 def test_task_load_adapts_quickly(c, s, a):
     future = c.submit(slowinc, 1, delay=0.2)  # slow
     yield wait(future)
@@ -3099,7 +3099,7 @@ def test_task_load_adapts_quickly(c, s, a):
     assert 0 < s.task_duration["slowinc"] < 0.1
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 2)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 2)
 def test_even_load_after_fast_functions(c, s, a, b):
     x = c.submit(inc, 1, workers=a.address)  # very fast
     y = c.submit(inc, 2, workers=b.address)  # very fast
@@ -3113,7 +3113,7 @@ def test_even_load_after_fast_functions(c, s, a, b):
     # assert abs(len(a.data) - len(b.data)) <= 3
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 2)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 2)
 def test_even_load_on_startup(c, s, a, b):
     x, y = c.map(inc, [1, 2])
     yield wait([x, y])
@@ -3121,7 +3121,7 @@ def test_even_load_on_startup(c, s, a, b):
 
 
 @pytest.mark.skip
-@gen_cluster(client=True, ncores=[("127.0.0.1", 2)] * 2)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 2)] * 2)
 def test_contiguous_load(c, s, a, b):
     w, x, y, z = c.map(inc, [1, 2, 3, 4])
     yield wait([w, x, y, z])
@@ -3131,7 +3131,7 @@ def test_contiguous_load(c, s, a, b):
     assert {y.key, z.key} in groups
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 4)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 4)
 def test_balanced_with_submit(c, s, *workers):
     L = [c.submit(slowinc, i) for i in range(4)]
     yield wait(L)
@@ -3139,7 +3139,7 @@ def test_balanced_with_submit(c, s, *workers):
         assert len(w.data) == 1
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 4)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 4)
 def test_balanced_with_submit_and_resident_data(c, s, *workers):
     [x] = yield c.scatter([10], broadcast=True)
     L = [c.submit(slowinc, x, pure=False) for i in range(4)]
@@ -3148,7 +3148,7 @@ def test_balanced_with_submit_and_resident_data(c, s, *workers):
         assert len(w.data) == 2
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 20)] * 2)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 20)] * 2)
 def test_scheduler_saturates_cores(c, s, a, b):
     for delay in [0, 0.01, 0.1]:
         futures = c.map(slowinc, range(100), delay=delay)
@@ -3163,7 +3163,7 @@ def test_scheduler_saturates_cores(c, s, a, b):
             yield gen.sleep(0.01)
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 20)] * 2)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 20)] * 2)
 def test_scheduler_saturates_cores_random(c, s, a, b):
     for delay in [0, 0.01, 0.1]:
         futures = c.map(randominc, range(100), scale=0.1)
@@ -3177,7 +3177,7 @@ def test_scheduler_saturates_cores_random(c, s, a, b):
             yield gen.sleep(0.01)
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 4)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 4)
 def test_cancel_clears_processing(c, s, *workers):
     da = pytest.importorskip("dask.array")
     x = c.submit(slowinc, 1, delay=0.2)
@@ -3255,10 +3255,10 @@ def test_get_foo(c, s, a, b):
     yield wait(futures)
 
     x = yield c.scheduler.ncores()
-    assert x == s.ncores
+    assert x == s.nthreads
 
     x = yield c.scheduler.ncores(workers=[a.address])
-    assert x == {a.address: s.ncores[a.address]}
+    assert x == {a.address: s.nthreads[a.address]}
 
     x = yield c.scheduler.has_what()
     assert valmap(sorted, x) == valmap(sorted, s.has_what)
@@ -3287,7 +3287,7 @@ def assert_dict_key_equal(expected, actual):
         assert list(ev) == list(av)
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 3)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 3)
 def test_get_foo_lost_keys(c, s, u, v, w):
     x = c.submit(inc, 1, workers=[u.address])
     y = yield c.scatter(3, workers=[v.address])
@@ -3489,7 +3489,7 @@ def test_persist_optimize_graph(c, s, a, b):
         assert not any(tokey(k) in s.tasks for k in b2.__dask_keys__())
 
 
-@gen_cluster(client=True, ncores=[])
+@gen_cluster(client=True, nthreads=[])
 def test_scatter_raises_if_no_workers(c, s):
     with pytest.raises(gen.TimeoutError):
         yield c.scatter(1, timeout=0.5)
@@ -3511,7 +3511,7 @@ def test_reconnect(loop):
     with popen(scheduler_cli) as s:
         c = Client("127.0.0.1:9393", loop=loop)
         start = time()
-        while len(c.ncores()) != 1:
+        while len(c.nthreads()) != 1:
             sleep(0.1)
             assert time() < start + 3
 
@@ -3524,7 +3524,7 @@ def test_reconnect(loop):
         sleep(0.01)
 
     with pytest.raises(Exception):
-        c.ncores()
+        c.nthreads()
 
     assert x.status == "cancelled"
     with pytest.raises(CancelledError):
@@ -3536,7 +3536,7 @@ def test_reconnect(loop):
             sleep(0.1)
             assert time() < start + 5
         start = time()
-        while len(c.ncores()) != 1:
+        while len(c.nthreads()) != 1:
             sleep(0.05)
             assert time() < start + 15
 
@@ -3559,7 +3559,7 @@ def test_reconnect(loop):
     c.close()
 
 
-@gen_cluster(client=True, ncores=[], client_kwargs={"timeout": 0.5})
+@gen_cluster(client=True, nthreads=[], client_kwargs={"timeout": 0.5})
 def test_reconnect_timeout(c, s):
     with captured_logger(logging.getLogger("distributed.client")) as logger:
         yield s.close()
@@ -3626,7 +3626,7 @@ def start_worker(sleep, duration, repeat=1):
                     break
 
             start = time()
-            while c.ncores():
+            while c.nthreads():
                 sleep(0.2)
                 assert time() < start + 10
 
@@ -3748,7 +3748,7 @@ def test_lose_scattered_data(c, s, a, b):
     assert x.key not in s.tasks
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 3)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 3)
 def test_partially_lose_scattered_data(e, s, a, b, c):
     x = yield e.scatter(1, workers=a.address)
     yield e.replicate(x, n=2)
@@ -3887,7 +3887,7 @@ def test_temp_client(s, a, b):
 
 
 @nodebug  # test timing is fragile
-@gen_cluster(ncores=[("127.0.0.1", 1)] * 3, client=True)
+@gen_cluster(nthreads=[("127.0.0.1", 1)] * 3, client=True)
 def test_persist_workers(e, s, a, b, c):
     L1 = [delayed(inc)(i) for i in range(4)]
     total = delayed(sum)(L1)
@@ -3912,7 +3912,7 @@ def test_persist_workers(e, s, a, b, c):
     assert s.loose_restrictions == {total2.key} | {v.key for v in L2}
 
 
-@gen_cluster(ncores=[("127.0.0.1", 1)] * 3, client=True)
+@gen_cluster(nthreads=[("127.0.0.1", 1)] * 3, client=True)
 def test_compute_workers(e, s, a, b, c):
     L1 = [delayed(inc)(i) for i in range(4)]
     total = delayed(sum)(L1)
@@ -3991,7 +3991,7 @@ def test_retire_workers_2(c, s, a, b):
     assert a.address not in s.workers
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 10)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 10)
 def test_retire_many_workers(c, s, *workers):
     futures = yield c.scatter(list(range(100)))
 
@@ -4000,14 +4000,14 @@ def test_retire_many_workers(c, s, *workers):
     results = yield c.gather(futures)
     assert results == list(range(100))
 
-    assert len(s.has_what) == len(s.ncores) == 3
+    assert len(s.has_what) == len(s.nthreads) == 3
     assert all(future.done() for future in futures)
     assert all(s.tasks[future.key].state == "memory" for future in futures)
     for w, keys in s.has_what.items():
         assert 15 < len(keys) < 50
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 3)] * 2)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 3)] * 2)
 def test_weight_occupancy_against_data_movement(c, s, a, b):
     s.extensions["stealing"]._pc.callback_time = 1000000
     s.task_duration["f"] = 0.01
@@ -4027,8 +4027,8 @@ def f(x, y=0, z=0):
     assert sum(f.key in b.data for f in futures) >= 1
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1), ("127.0.0.1", 10)])
-def test_distribute_tasks_by_ncores(c, s, a, b):
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1), ("127.0.0.1", 10)])
+def test_distribute_tasks_by_nthreads(c, s, a, b):
     s.task_duration["f"] = 0.01
     s.extensions["stealing"]._pc.callback_time = 1000000
 
@@ -4664,7 +4664,7 @@ def test_identity(c, s, a, b):
     assert s.id.lower().startswith("scheduler")
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 4)] * 2)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 4)] * 2)
 def test_get_client(c, s, a, b):
     assert get_client() is c
     assert c.asynchronous
@@ -4713,7 +4713,7 @@ def f(x):
     assert result == sum(range(10))
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 1, timeout=100)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 1, timeout=100)
 def test_secede_simple(c, s, a):
     def f():
         client = get_client()
@@ -4725,7 +4725,7 @@ def f():
 
 
 @pytest.mark.slow
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 2, timeout=60)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 2, timeout=60)
 def test_secede_balances(c, s, a, b):
     count = threading.active_count()
 
@@ -5055,7 +5055,7 @@ def test_client_async_before_loop_starts():
     client=True,
     Worker=Nanny if PY3 else Worker,
     timeout=60,
-    ncores=[("127.0.0.1", 3)] * 2,
+    nthreads=[("127.0.0.1", 3)] * 2,
 )
 def test_nested_compute(c, s, a, b):
     def fib(x):
@@ -5220,7 +5220,7 @@ def test_client_doesnt_close_given_loop(loop, s, a, b):
         assert c.submit(inc, 2).result() == 3
 
 
-@gen_cluster(client=True, ncores=[])
+@gen_cluster(client=True, nthreads=[])
 def test_quiet_scheduler_loss(c, s):
     c._periodic_callbacks["scheduler-info"].interval = 10
     with captured_logger(logging.getLogger("distributed.client")) as logger:
@@ -5368,7 +5368,7 @@ def test_client_repr_closed_sync(loop):
         c._repr_html_()
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)])
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)])
 def test_nested_prioritization(c, s, w):
     x = delayed(inc)(1, dask_key_name=("a", 2))
     y = delayed(inc)(2, dask_key_name=("a", 10))
diff --git a/distributed/tests/test_collections.py b/distributed/tests/test_collections.py
index dea4296769d..7cb509f6ac7 100644
--- a/distributed/tests/test_collections.py
+++ b/distributed/tests/test_collections.py
@@ -188,7 +188,7 @@ def test_sparse_arrays(c, s, a, b):
     yield future
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)])
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)])
 def test_delayed_none(c, s, w):
     x = dask.delayed(None)
     y = dask.delayed(123)
diff --git a/distributed/tests/test_failed_workers.py b/distributed/tests/test_failed_workers.py
index dde92c6d24c..b39dd3f3ae7 100644
--- a/distributed/tests/test_failed_workers.py
+++ b/distributed/tests/test_failed_workers.py
@@ -39,7 +39,7 @@ def test_submit_after_failed_worker_sync(loop):
 
 @gen_cluster(client=True, timeout=60, active_rpc_timeout=10)
 def test_submit_after_failed_worker_async(c, s, a, b):
-    n = Nanny(s.address, ncores=2, loop=s.loop)
+    n = Nanny(s.address, nthreads=2, loop=s.loop)
     n.start(0)
     while len(s.workers) < 3:
         yield gen.sleep(0.1)
@@ -79,7 +79,7 @@ def test_gather_after_failed_worker(loop):
 @gen_cluster(
     client=True,
     Worker=Nanny,
-    ncores=[("127.0.0.1", 1)] * 4,
+    nthreads=[("127.0.0.1", 1)] * 4,
     config={"distributed.comm.timeouts.connect": "1s"},
 )
 def test_gather_then_submit_after_failed_workers(c, s, w, x, y, z):
@@ -117,7 +117,7 @@ def test_failed_worker_without_warning(c, s, a, b):
     yield gen.sleep(0.5)
 
     start = time()
-    while len(s.ncores) < 2:
+    while len(s.nthreads) < 2:
         yield gen.sleep(0.01)
         assert time() - start < 10
 
@@ -126,7 +126,7 @@ def test_failed_worker_without_warning(c, s, a, b):
     L2 = c.map(inc, range(10, 20))
     yield wait(L2)
     assert all(len(keys) > 0 for keys in s.has_what.values())
-    ncores2 = dict(s.ncores)
+    nthreads2 = dict(s.nthreads)
 
     yield c._restart()
 
@@ -134,12 +134,12 @@ def test_failed_worker_without_warning(c, s, a, b):
     yield wait(L)
     assert all(len(keys) > 0 for keys in s.has_what.values())
 
-    assert not (set(ncores2) & set(s.ncores))  # no overlap
+    assert not (set(nthreads2) & set(s.nthreads))  # no overlap
 
 
 @gen_cluster(Worker=Nanny, client=True, timeout=60)
 def test_restart(c, s, a, b):
-    assert s.ncores == {a.worker_address: 1, b.worker_address: 2}
+    assert s.nthreads == {a.worker_address: 1, b.worker_address: 2}
 
     x = c.submit(inc, 1)
     y = c.submit(inc, x)
@@ -185,7 +185,7 @@ def test_restart_sync_no_center(loop):
             assert x.cancelled()
             y = c.submit(inc, 2)
             assert y.result() == 3
-            assert len(c.ncores()) == 2
+            assert len(c.nthreads()) == 2
 
 
 def test_restart_sync(loop):
@@ -198,7 +198,7 @@ def test_restart_sync(loop):
             c.restart()
             assert not sync(loop, c.scheduler.who_has)
             assert x.cancelled()
-            assert len(c.ncores()) == 2
+            assert len(c.nthreads()) == 2
 
             with pytest.raises(CancelledError):
                 x.result()
@@ -214,7 +214,7 @@ def test_restart_fast(c, s, a, b):
     start = time()
     yield c._restart()
     assert time() - start < 10
-    assert len(s.ncores) == 2
+    assert len(s.nthreads) == 2
 
     assert all(x.status == "cancelled" for x in L)
 
@@ -242,7 +242,7 @@ def test_restart_fast_sync(loop):
             start = time()
             c.restart()
             assert time() - start < 10
-            assert len(c.ncores()) == 2
+            assert len(c.nthreads()) == 2
 
             assert all(x.status == "cancelled" for x in L)
 
@@ -293,7 +293,7 @@ def test_restart_scheduler(s, a, b):
     gc.collect()
     addrs = (a.worker_address, b.worker_address)
     yield s.restart()
-    assert len(s.ncores) == 2
+    assert len(s.nthreads) == 2
     addrs2 = (a.worker_address, b.worker_address)
 
     assert addrs != addrs2
@@ -315,11 +315,11 @@ def test_forgotten_futures_dont_clean_up_new_futures(c, s, a, b):
 @gen_cluster(client=True, timeout=60, active_rpc_timeout=10)
 def test_broken_worker_during_computation(c, s, a, b):
     s.allowed_failures = 100
-    n = Nanny(s.address, ncores=2, loop=s.loop)
+    n = Nanny(s.address, nthreads=2, loop=s.loop)
     n.start(0)
 
     start = time()
-    while len(s.ncores) < 3:
+    while len(s.nthreads) < 3:
         yield gen.sleep(0.01)
         assert time() < start + 5
 
@@ -368,17 +368,17 @@ def test_restart_during_computation(c, s, a, b):
     yield c._restart()
     assert not s.rprocessing
 
-    assert len(s.ncores) == 2
+    assert len(s.nthreads) == 2
     assert not s.tasks
 
 
 @gen_cluster(client=True, timeout=60)
 def test_worker_who_has_clears_after_failed_connection(c, s, a, b):
-    n = Nanny(s.address, ncores=2, loop=s.loop)
+    n = Nanny(s.address, nthreads=2, loop=s.loop)
     n.start(0)
 
     start = time()
-    while len(s.ncores) < 3:
+    while len(s.nthreads) < 3:
         yield gen.sleep(0.01)
         assert time() < start + 5
 
@@ -406,7 +406,7 @@ def test_worker_who_has_clears_after_failed_connection(c, s, a, b):
 
 
 @pytest.mark.slow
-@gen_cluster(client=True, timeout=60, Worker=Nanny, ncores=[("127.0.0.1", 1)])
+@gen_cluster(client=True, timeout=60, Worker=Nanny, nthreads=[("127.0.0.1", 1)])
 def test_restart_timeout_on_long_running_task(c, s, a):
     with captured_logger("distributed.scheduler") as sio:
         future = c.submit(sleep, 3600)
diff --git a/distributed/tests/test_ipython.py b/distributed/tests/test_ipython.py
index 8bb64bb4e0b..a6f88ec5241 100644
--- a/distributed/tests/test_ipython.py
+++ b/distributed/tests/test_ipython.py
@@ -88,7 +88,7 @@ def test_start_ipython_workers_magic(loop, zmq_ctx):
     with cluster(2) as (s, [a, b]):
 
         with Client(s["address"], loop=loop) as e, mock_ipython() as ip:
-            workers = list(e.ncores())[:2]
+            workers = list(e.nthreads())[:2]
             names = ["magic%i" % i for i in range(len(workers))]
             info_dict = e.start_ipython_workers(workers, magic_names=names)
 
@@ -116,7 +116,7 @@ def test_start_ipython_workers_magic_asterix(loop, zmq_ctx):
     with cluster(2) as (s, [a, b]):
 
         with Client(s["address"], loop=loop) as e, mock_ipython() as ip:
-            workers = list(e.ncores())[:2]
+            workers = list(e.nthreads())[:2]
             info_dict = e.start_ipython_workers(workers, magic_names="magic_*")
 
         expected = [
@@ -144,7 +144,7 @@ def test_start_ipython_remote(loop, zmq_ctx):
 
     with cluster(1) as (s, [a]):
         with Client(s["address"], loop=loop) as e, mock_ipython() as ip:
-            worker = first(e.ncores())
+            worker = first(e.nthreads())
             ip.user_ns["info"] = e.start_ipython_workers(worker)[worker]
             remote_magic("info 1")  # line magic
             remote_magic("info", "worker")  # cell magic
@@ -165,7 +165,7 @@ def test_start_ipython_qtconsole(loop):
         with mock.patch("distributed._ipython_utils.Popen", Popen), Client(
             s["address"], loop=loop
         ) as e:
-            worker = first(e.ncores())
+            worker = first(e.nthreads())
             e.start_ipython_workers(worker, qtconsole=True)
             e.start_ipython_workers(worker, qtconsole=True, qtconsole_args=["--debug"])
     assert Popen.call_count == 2
diff --git a/distributed/tests/test_locks.py b/distributed/tests/test_locks.py
index 952d43ceb9b..226feec4faf 100644
--- a/distributed/tests/test_locks.py
+++ b/distributed/tests/test_locks.py
@@ -11,7 +11,7 @@
 from distributed.utils_test import client, cluster_fixture, loop  # noqa F401
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 8)] * 2)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 8)] * 2)
 def test_lock(c, s, a, b):
     c.set_metadata("locked", False)
 
diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
index 1357a3679e2..6b6d5bf939d 100644
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -22,28 +22,28 @@
 from distributed.utils_test import gen_cluster, gen_test, inc, captured_logger
 
 
-@gen_cluster(ncores=[])
+@gen_cluster(nthreads=[])
 def test_nanny(s):
-    n = yield Nanny(s.address, ncores=2, loop=s.loop)
+    n = yield Nanny(s.address, nthreads=2, loop=s.loop)
 
     with rpc(n.address) as nn:
         assert n.is_alive()
-        assert s.ncores[n.worker_address] == 2
+        assert s.nthreads[n.worker_address] == 2
         assert s.workers[n.worker_address].nanny == n.address
 
         yield nn.kill()
         assert not n.is_alive()
-        assert n.worker_address not in s.ncores
+        assert n.worker_address not in s.nthreads
         assert n.worker_address not in s.workers
 
         yield nn.kill()
         assert not n.is_alive()
-        assert n.worker_address not in s.ncores
+        assert n.worker_address not in s.nthreads
         assert n.worker_address not in s.workers
 
         yield nn.instantiate()
         assert n.is_alive()
-        assert s.ncores[n.worker_address] == 2
+        assert s.nthreads[n.worker_address] == 2
         assert s.workers[n.worker_address].nanny == n.address
 
         yield nn.terminate()
@@ -52,9 +52,9 @@ def test_nanny(s):
     yield n.close()
 
 
-@gen_cluster(ncores=[])
+@gen_cluster(nthreads=[])
 def test_many_kills(s):
-    n = yield Nanny(s.address, ncores=2, loop=s.loop)
+    n = yield Nanny(s.address, nthreads=2, loop=s.loop)
     assert n.is_alive()
     yield [n.kill() for i in range(5)]
     yield [n.kill() for i in range(5)]
@@ -65,13 +65,13 @@ def test_many_kills(s):
 def test_str(s, a, b):
     assert a.worker_address in str(a)
     assert a.worker_address in repr(a)
-    assert str(a.ncores) in str(a)
-    assert str(a.ncores) in repr(a)
+    assert str(a.nthreads) in str(a)
+    assert str(a.nthreads) in repr(a)
 
 
-@gen_cluster(ncores=[], timeout=20, client=True)
+@gen_cluster(nthreads=[], timeout=20, client=True)
 def test_nanny_process_failure(c, s):
-    n = yield Nanny(s.address, ncores=2, loop=s.loop)
+    n = yield Nanny(s.address, nthreads=2, loop=s.loop)
     first_dir = n.worker_dir
 
     assert os.path.exists(first_dir)
@@ -97,7 +97,7 @@ def test_nanny_process_failure(c, s):
     # assert n.worker_address != original_address  # most likely
 
     start = time()
-    while n.worker_address not in s.ncores or n.worker_dir is None:
+    while n.worker_address not in s.nthreads or n.worker_dir is None:
         yield gen.sleep(0.01)
         assert time() - start < 5
 
@@ -111,10 +111,10 @@ def test_nanny_process_failure(c, s):
     s.stop()
 
 
-@gen_cluster(ncores=[])
+@gen_cluster(nthreads=[])
 def test_run(s):
     pytest.importorskip("psutil")
-    n = yield Nanny(s.address, ncores=2, loop=s.loop)
+    n = yield Nanny(s.address, nthreads=2, loop=s.loop)
 
     with rpc(n.address) as nn:
         response = yield nn.run(function=dumps(lambda: 1))
@@ -126,7 +126,7 @@ def test_run(s):
 
 @pytest.mark.slow
 @gen_cluster(
-    Worker=Nanny, ncores=[("127.0.0.1", 1)], worker_kwargs={"reconnect": False}
+    Worker=Nanny, nthreads=[("127.0.0.1", 1)], worker_kwargs={"reconnect": False}
 )
 def test_close_on_disconnect(s, w):
     yield s.close()
@@ -157,7 +157,7 @@ def test_nanny_alt_worker_class(c, s, w1, w2):
 
 
 @pytest.mark.slow
-@gen_cluster(client=False, ncores=[])
+@gen_cluster(client=False, nthreads=[])
 def test_nanny_death_timeout(s):
     yield s.close()
     w = yield Nanny(s.address, death_timeout=1)
@@ -184,7 +184,7 @@ def check_func(func):
 @pytest.mark.skipif(
     sys.platform.startswith("win"), reason="num_fds not supported on windows"
 )
-@gen_cluster(client=False, ncores=[])
+@gen_cluster(client=False, nthreads=[])
 def test_num_fds(s):
     psutil = pytest.importorskip("psutil")
     proc = psutil.Process()
@@ -212,7 +212,7 @@ def test_num_fds(s):
 @pytest.mark.skipif(
     not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
 )
-@gen_cluster(client=True, ncores=[])
+@gen_cluster(client=True, nthreads=[])
 def test_worker_uses_same_host_as_nanny(c, s):
     for host in ["tcp://0.0.0.0", "tcp://127.0.0.2"]:
         n = Nanny(s.address)
@@ -237,7 +237,7 @@ def test_scheduler_file():
         s.stop()
 
 
-@gen_cluster(client=True, Worker=Nanny, ncores=[("127.0.0.1", 2)])
+@gen_cluster(client=True, Worker=Nanny, nthreads=[("127.0.0.1", 2)])
 def test_nanny_timeout(c, s, a):
     x = yield c.scatter(123)
     with captured_logger(
@@ -255,7 +255,7 @@ def test_nanny_timeout(c, s, a):
 
 
 @gen_cluster(
-    ncores=[("127.0.0.1", 1)],
+    nthreads=[("127.0.0.1", 1)],
     client=True,
     Worker=Nanny,
     worker_kwargs={"memory_limit": 1e8},
@@ -283,7 +283,7 @@ def leak():
         assert "memory" in out.lower()
 
 
-@gen_cluster(ncores=[], client=True)
+@gen_cluster(nthreads=[], client=True)
 def test_avoid_memory_monitor_if_zero_limit(c, s):
     nanny = yield Nanny(s.address, loop=s.loop, memory_limit=0)
     typ = yield c.run(lambda dask_worker: type(dask_worker.data))
@@ -301,7 +301,7 @@ def test_avoid_memory_monitor_if_zero_limit(c, s):
     yield nanny.close()
 
 
-@gen_cluster(ncores=[], client=True)
+@gen_cluster(nthreads=[], client=True)
 def test_scheduler_address_config(c, s):
     with dask.config.set({"scheduler-address": s.address}):
         nanny = yield Nanny(loop=s.loop)
@@ -329,7 +329,7 @@ def test_wait_for_scheduler():
     assert "restart" not in log.lower(), log
 
 
-@gen_cluster(ncores=[], client=True)
+@gen_cluster(nthreads=[], client=True)
 def test_environment_variable(c, s):
     a = Nanny(s.address, loop=s.loop, memory_limit=0, env={"FOO": "123"})
     b = Nanny(s.address, loop=s.loop, memory_limit=0, env={"FOO": "456"})
@@ -339,7 +339,7 @@ def test_environment_variable(c, s):
     yield [a.close(), b.close()]
 
 
-@gen_cluster(ncores=[], client=True)
+@gen_cluster(nthreads=[], client=True)
 def test_data_types(c, s):
     w = yield Nanny(s.address, data=dict)
     r = yield c.run(lambda dask_worker: type(dask_worker.data))
@@ -353,7 +353,7 @@ def _noop(x):
 
 
 @gen_cluster(
-    ncores=[("127.0.0.1", 1)],
+    nthreads=[("127.0.0.1", 1)],
     client=True,
     Worker=Nanny,
     config={"distributed.worker.daemon": False},
@@ -368,7 +368,7 @@ def multiprocessing_worker():
 
 
 @gen_cluster(
-    ncores=[("127.0.0.1", 1)],
+    nthreads=[("127.0.0.1", 1)],
     client=True,
     Worker=Nanny,
     config={"distributed.worker.daemon": False},
diff --git a/distributed/tests/test_priorities.py b/distributed/tests/test_priorities.py
index 421bf7e3028..6258c4e16a7 100644
--- a/distributed/tests/test_priorities.py
+++ b/distributed/tests/test_priorities.py
@@ -83,7 +83,7 @@ def test_expand_persist(c, s, a, b):
     assert s.tasks[low.key].state == "processing"
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)])
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)])
 def test_repeated_persists_same_priority(c, s, w):
     xs = [delayed(slowinc)(i, delay=0.05, dask_key_name="x-%d" % i) for i in range(10)]
     ys = [
@@ -107,7 +107,7 @@ def test_repeated_persists_same_priority(c, s, w):
     assert any(s.tasks[z.key].state == "memory" for z in zs)
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)])
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)])
 def test_last_in_first_out(c, s, w):
     xs = [c.submit(slowinc, i, delay=0.05) for i in range(5)]
     ys = [c.submit(slowinc, x, delay=0.05) for x in xs]
diff --git a/distributed/tests/test_pubsub.py b/distributed/tests/test_pubsub.py
index c44637cf9fd..9d2b30dab6f 100644
--- a/distributed/tests/test_pubsub.py
+++ b/distributed/tests/test_pubsub.py
@@ -49,7 +49,7 @@ def pingpong(a, b, start=False, n=1000, msg=1):
     # print('duration', stop - start)  # I get around 3ms/roundtrip on my laptop
 
 
-@gen_cluster(client=True, ncores=[])
+@gen_cluster(client=True, nthreads=[])
 def test_client(c, s):
     with pytest.raises(Exception):
         get_worker()
diff --git a/distributed/tests/test_queues.py b/distributed/tests/test_queues.py
index e40d3cd492c..2e7702171ad 100644
--- a/distributed/tests/test_queues.py
+++ b/distributed/tests/test_queues.py
@@ -2,7 +2,6 @@
 
 from datetime import timedelta
 from time import sleep
-import sys
 
 import pytest
 from tornado import gen
@@ -113,9 +112,8 @@ def f(x):
     assert q.get() == 11
 
 
-@pytest.mark.skipif(sys.version_info[0] == 2, reason="Multi-client issues")
 @pytest.mark.slow
-@gen_cluster(client=True, ncores=[("127.0.0.1", 2)] * 5, Worker=Nanny, timeout=None)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 2)] * 5, Worker=Nanny, timeout=None)
 def test_race(c, s, *workers):
     def f(i):
         with worker_client() as c:
diff --git a/distributed/tests/test_resources.py b/distributed/tests/test_resources.py
index 480532d912e..1985d44e2a3 100644
--- a/distributed/tests/test_resources.py
+++ b/distributed/tests/test_resources.py
@@ -14,7 +14,7 @@
 from distributed.utils_test import client, cluster_fixture, loop, s, a, b  # noqa: F401
 
 
-@gen_cluster(client=True, ncores=[])
+@gen_cluster(client=True, nthreads=[])
 def test_resources(c, s):
     assert not s.worker_resources
     assert not s.resources
@@ -37,7 +37,7 @@ def test_resources(c, s):
 
 @gen_cluster(
     client=True,
-    ncores=[
+    nthreads=[
         ("127.0.0.1", 1, {"resources": {"A": 5}}),
         ("127.0.0.1", 1, {"resources": {"A": 1, "B": 1}}),
     ],
@@ -65,7 +65,7 @@ def test_resource_submit(c, s, a, b):
 
 @gen_cluster(
     client=True,
-    ncores=[
+    nthreads=[
         ("127.0.0.1", 1, {"resources": {"A": 1}}),
         ("127.0.0.1", 1, {"resources": {"B": 1}}),
     ],
@@ -80,7 +80,7 @@ def test_submit_many_non_overlapping(c, s, a, b):
 
 @gen_cluster(
     client=True,
-    ncores=[
+    nthreads=[
         ("127.0.0.1", 1, {"resources": {"A": 1}}),
         ("127.0.0.1", 1, {"resources": {"B": 1}}),
     ],
@@ -96,7 +96,7 @@ def test_move(c, s, a, b):
 
 @gen_cluster(
     client=True,
-    ncores=[
+    nthreads=[
         ("127.0.0.1", 1, {"resources": {"A": 1}}),
         ("127.0.0.1", 1, {"resources": {"B": 1}}),
     ],
@@ -114,7 +114,7 @@ def test_dont_work_steal(c, s, a, b):
 
 @gen_cluster(
     client=True,
-    ncores=[
+    nthreads=[
         ("127.0.0.1", 1, {"resources": {"A": 1}}),
         ("127.0.0.1", 1, {"resources": {"B": 1}}),
     ],
@@ -128,7 +128,7 @@ def test_map(c, s, a, b):
 
 @gen_cluster(
     client=True,
-    ncores=[
+    nthreads=[
         ("127.0.0.1", 1, {"resources": {"A": 1}}),
         ("127.0.0.1", 1, {"resources": {"B": 1}}),
     ],
@@ -147,7 +147,7 @@ def test_persist(c, s, a, b):
 
 @gen_cluster(
     client=True,
-    ncores=[
+    nthreads=[
         ("127.0.0.1", 1, {"resources": {"A": 1}}),
         ("127.0.0.1", 1, {"resources": {"B": 11}}),
     ],
@@ -170,7 +170,7 @@ def test_compute(c, s, a, b):
 
 @gen_cluster(
     client=True,
-    ncores=[
+    nthreads=[
         ("127.0.0.1", 1, {"resources": {"A": 1}}),
         ("127.0.0.1", 1, {"resources": {"B": 1}}),
     ],
@@ -184,7 +184,7 @@ def test_get(c, s, a, b):
 
 @gen_cluster(
     client=True,
-    ncores=[
+    nthreads=[
         ("127.0.0.1", 1, {"resources": {"A": 1}}),
         ("127.0.0.1", 1, {"resources": {"B": 1}}),
     ],
@@ -222,7 +222,7 @@ def test_resources_str(c, s, a, b):
 
 @gen_cluster(
     client=True,
-    ncores=[
+    nthreads=[
         ("127.0.0.1", 4, {"resources": {"A": 2}}),
         ("127.0.0.1", 4, {"resources": {"A": 1}}),
     ],
@@ -240,7 +240,7 @@ def test_submit_many_non_overlapping(c, s, a, b):
     assert b.total_resources == b.available_resources
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 4, {"resources": {"A": 2, "B": 1}})])
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 4, {"resources": {"A": 2, "B": 1}})])
 def test_minimum_resource(c, s, a):
     futures = c.map(slowinc, range(30), resources={"A": 1, "B": 1}, delay=0.02)
 
@@ -252,7 +252,7 @@ def test_minimum_resource(c, s, a):
     assert a.total_resources == a.available_resources
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 2, {"resources": {"A": 1}})])
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 2, {"resources": {"A": 1}})])
 def test_prefer_constrained(c, s, a):
     futures = c.map(slowinc, range(1000), delay=0.1)
     constrained = c.map(inc, range(10), resources={"A": 1})
@@ -270,7 +270,7 @@ def test_prefer_constrained(c, s, a):
 @pytest.mark.skip(reason="")
 @gen_cluster(
     client=True,
-    ncores=[
+    nthreads=[
         ("127.0.0.1", 2, {"resources": {"A": 1}}),
         ("127.0.0.1", 2, {"resources": {"A": 1}}),
     ],
@@ -284,7 +284,7 @@ def test_balance_resources(c, s, a, b):
     assert any(f.key in b.data for f in constrained)
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 2)])
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 2)])
 def test_set_resources(c, s, a):
     yield a.set_resources(A=2)
     assert a.total_resources["A"] == 2
@@ -303,7 +303,7 @@ def test_set_resources(c, s, a):
 
 @gen_cluster(
     client=True,
-    ncores=[
+    nthreads=[
         ("127.0.0.1", 1, {"resources": {"A": 1}}),
         ("127.0.0.1", 1, {"resources": {"B": 1}}),
     ],
@@ -325,7 +325,7 @@ def test_persist_collections(c, s, a, b):
 @pytest.mark.skip(reason="Should protect resource keys from optimization")
 @gen_cluster(
     client=True,
-    ncores=[
+    nthreads=[
         ("127.0.0.1", 1, {"resources": {"A": 1}}),
         ("127.0.0.1", 1, {"resources": {"B": 1}}),
     ],
@@ -346,7 +346,7 @@ def test_dont_optimize_out(c, s, a, b):
 @pytest.mark.xfail(reason="atop fusion seemed to break this")
 @gen_cluster(
     client=True,
-    ncores=[
+    nthreads=[
         ("127.0.0.1", 1, {"resources": {"A": 1}}),
         ("127.0.0.1", 1, {"resources": {"B": 1}}),
     ],
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 66a8088ace5..e8d2a96ee60 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -50,11 +50,11 @@
 def test_administration(s, a, b):
     assert isinstance(s.address, str)
     assert s.address in str(s)
-    assert str(sum(s.ncores.values())) in repr(s)
-    assert str(len(s.ncores)) in repr(s)
+    assert str(sum(s.nthreads.values())) in repr(s)
+    assert str(len(s.nthreads)) in repr(s)
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)])
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)])
 def test_respect_data_in_memory(c, s, a):
     x = delayed(inc)(1)
     y = delayed(inc)(x)
@@ -106,14 +106,14 @@ def test_decide_worker_with_many_independent_leaves(c, s, a, b):
     assert nhits > 80
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 3)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 3)
 def test_decide_worker_with_restrictions(client, s, a, b, c):
     x = client.submit(inc, 1, workers=[a.address, b.address])
     yield wait(x)
     assert x.key in a.data or x.key in b.data
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 3)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 3)
 def test_move_data_over_break_restrictions(client, s, a, b, c):
     [x] = yield client.scatter([1], workers=b.address)
     y = client.submit(inc, x, workers=[a.address, b.address])
@@ -121,7 +121,7 @@ def test_move_data_over_break_restrictions(client, s, a, b, c):
     assert y.key in a.data or y.key in b.data
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 3)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 3)
 def test_balance_with_restrictions(client, s, a, b, c):
     [x], [y] = yield [
         client.scatter([[1, 2, 3]], workers=a.address),
@@ -133,7 +133,7 @@ def test_balance_with_restrictions(client, s, a, b, c):
     assert s.tasks[z.key].who_has == {s.workers[c.address]}
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 3)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 3)
 def test_no_valid_workers(client, s, a, b, c):
     x = client.submit(inc, 1, workers="127.0.0.5:9999")
     while not s.tasks:
@@ -145,7 +145,7 @@ def test_no_valid_workers(client, s, a, b, c):
         yield gen.with_timeout(timedelta(milliseconds=50), x)
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 3)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 3)
 def test_no_valid_workers_loose_restrictions(client, s, a, b, c):
     x = client.submit(inc, 1, workers="127.0.0.5:9999", allow_other_workers=True)
 
@@ -153,7 +153,7 @@ def test_no_valid_workers_loose_restrictions(client, s, a, b, c):
     assert result == 2
 
 
-@gen_cluster(client=True, ncores=[])
+@gen_cluster(client=True, nthreads=[])
 def test_no_workers(client, s):
     x = client.submit(inc, 1)
     while not s.tasks:
@@ -165,7 +165,7 @@ def test_no_workers(client, s):
         yield gen.with_timeout(timedelta(milliseconds=50), x)
 
 
-@gen_cluster(ncores=[])
+@gen_cluster(nthreads=[])
 def test_retire_workers_empty(s):
     yield s.retire_workers(workers=[])
 
@@ -207,7 +207,7 @@ def test_remove_worker_from_scheduler(s, a, b):
 
     assert a.address in s.stream_comms
     s.remove_worker(address=a.address)
-    assert a.address not in s.ncores
+    assert a.address not in s.nthreads
     assert len(s.workers[b.address].processing) == len(dsk)  # b owns everything
     s.validate_state()
 
@@ -215,14 +215,14 @@ def test_remove_worker_from_scheduler(s, a, b):
 @gen_cluster(config={"distributed.scheduler.events-cleanup-delay": "10 ms"})
 def test_clear_events_worker_removal(s, a, b):
     assert a.address in s.events
-    assert a.address in s.ncores
+    assert a.address in s.nthreads
     assert b.address in s.events
-    assert b.address in s.ncores
+    assert b.address in s.nthreads
 
     s.remove_worker(address=a.address)
     # Shortly after removal, the events should still be there
     assert a.address in s.events
-    assert a.address not in s.ncores
+    assert a.address not in s.nthreads
     s.validate_state()
 
     start = time()
@@ -253,7 +253,7 @@ def test_clear_events_client_removal(c, s, a, b):
 
 @gen_cluster()
 def test_add_worker(s, a, b):
-    w = Worker(s.address, ncores=3)
+    w = Worker(s.address, nthreads=3)
     w.data["x-5"] = 6
     w.data["y"] = 1
     yield w
@@ -267,7 +267,7 @@ def test_add_worker(s, a, b):
     )
 
     s.add_worker(
-        address=w.address, keys=list(w.data), ncores=w.ncores, services=s.services
+        address=w.address, keys=list(w.data), nthreads=w.nthreads, services=s.services
     )
 
     s.validate_state()
@@ -389,7 +389,7 @@ def test_delete_data(c, s, a, b):
         assert time() < start + 5
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)])
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)])
 def test_delete(c, s, a):
     x = c.submit(inc, 1)
     yield x
@@ -481,12 +481,12 @@ def test_ready_remove_worker(s, a, b):
         dependencies={"x-%d" % i: [] for i in range(20)},
     )
 
-    assert all(len(w.processing) > w.ncores for w in s.workers.values())
+    assert all(len(w.processing) > w.nthreads for w in s.workers.values())
 
     s.remove_worker(address=a.address)
 
     assert set(s.workers) == {b.address}
-    assert all(len(w.processing) > w.ncores for w in s.workers.values())
+    assert all(len(w.processing) > w.nthreads for w in s.workers.values())
 
 
 @gen_cluster(client=True, Worker=Nanny)
@@ -591,7 +591,7 @@ def test_coerce_address():
 @pytest.mark.skipif(
     sys.platform.startswith("win"), reason="file descriptors not really a thing"
 )
-@gen_cluster(ncores=[])
+@gen_cluster(nthreads=[])
 def test_file_descriptors_dont_leak(s):
     psutil = pytest.importorskip("psutil")
     proc = psutil.Process()
@@ -624,12 +624,12 @@ def test_update_graph_culls(s, a, b):
     assert "z" not in s.dependencies
 
 
-@gen_cluster(ncores=[])
+@gen_cluster(nthreads=[])
 def test_add_worker_is_idempotent(s):
-    s.add_worker(address=alice, ncores=1, resolve_address=False)
-    ncores = dict(s.ncores)
+    s.add_worker(address=alice, nthreads=1, resolve_address=False)
+    nthreads = dict(s.nthreads)
     s.add_worker(address=alice, resolve_address=False)
-    assert s.ncores == s.ncores
+    assert s.nthreads == s.nthreads
 
 
 def test_io_loop(loop):
@@ -654,7 +654,7 @@ def test_story(c, s, a, b):
     assert len(s.story(x.key, y.key)) > len(story)
 
 
-@gen_cluster(ncores=[], client=True)
+@gen_cluster(nthreads=[], client=True)
 def test_scatter_no_workers(c, s):
     with pytest.raises(gen.TimeoutError):
         yield s.scatter(data={"x": 1}, client="alice", timeout=0.1)
@@ -664,16 +664,16 @@ def test_scatter_no_workers(c, s):
         yield c.scatter(123, timeout=0.1)
     assert time() < start + 1.5
 
-    w = Worker(s.address, ncores=3)
+    w = Worker(s.address, nthreads=3)
     yield [c.scatter(data={"y": 2}, timeout=5), w._start()]
 
     assert w.data["y"] == 2
     yield w.close()
 
 
-@gen_cluster(ncores=[])
+@gen_cluster(nthreads=[])
 def test_scheduler_sees_memory_limits(s):
-    w = yield Worker(s.address, ncores=3, memory_limit=12345)
+    w = yield Worker(s.address, nthreads=3, memory_limit=12345)
 
     assert s.workers[w.address].memory_limit == 12345
     yield w.close()
@@ -688,8 +688,8 @@ def test_retire_workers(c, s, a, b):
 
     workers = yield s.retire_workers()
     assert list(workers) == [a.address]
-    assert workers[a.address]["ncores"] == a.ncores
-    assert list(s.ncores) == [b.address]
+    assert workers[a.address]["nthreads"] == a.nthreads
+    assert list(s.nthreads) == [b.address]
 
     assert s.workers_to_close() == []
 
@@ -717,7 +717,7 @@ def test_retire_workers_n(c, s, a, b):
         yield gen.sleep(0.01)
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 4)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 4)
 def test_workers_to_close(cl, s, *workers):
     s.task_duration["a"] = 4
     s.task_duration["b"] = 4
@@ -732,7 +732,7 @@ def test_workers_to_close(cl, s, *workers):
     assert len(wtc) == 1
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 4)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 4)
 def test_workers_to_close_grouped(c, s, *workers):
     groups = {
         workers[0].address: "a",
@@ -782,7 +782,7 @@ def test_retire_workers_no_suspicious_tasks(c, s, a, b):
     sys.platform.startswith("win"), reason="file descriptors not really a thing"
 )
 @pytest.mark.skipif(sys.version_info < (3, 6), reason="intermittent failure")
-@gen_cluster(client=True, ncores=[], timeout=240)
+@gen_cluster(client=True, nthreads=[], timeout=240)
 def test_file_descriptors(c, s):
     yield gen.sleep(0.1)
     psutil = pytest.importorskip("psutil")
@@ -793,7 +793,7 @@ def test_file_descriptors(c, s):
     N = 20
     nannies = yield [Nanny(s.address, loop=s.loop) for i in range(N)]
 
-    while len(s.ncores) < N:
+    while len(s.nthreads) < N:
         yield gen.sleep(0.1)
 
     num_fds_2 = proc.num_fds()
@@ -876,7 +876,7 @@ def test_occupancy_cleardown(c, s, a, b):
 
 
 @nodebug
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 30)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 30)
 def test_balance_many_workers(c, s, *workers):
     futures = c.map(slowinc, range(20), delay=0.2)
     yield wait(futures)
@@ -884,7 +884,7 @@ def test_balance_many_workers(c, s, *workers):
 
 
 @nodebug
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 30)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 30)
 def test_balance_many_workers_2(c, s, *workers):
     s.extensions["stealing"]._pc.callback_time = 100000000
     futures = c.map(slowinc, range(90), delay=0.2)
@@ -932,7 +932,7 @@ def test_worker_arrives_with_processing_data(c, s, a, b):
     while not any(w.processing for w in s.workers.values()):
         yield gen.sleep(0.01)
 
-    w = Worker(s.address, ncores=1)
+    w = Worker(s.address, nthreads=1)
     w.put_key_in_memory(y.key, 3)
 
     yield w
@@ -951,7 +951,7 @@ def test_worker_arrives_with_processing_data(c, s, a, b):
     yield w.close()
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)])
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)])
 def test_worker_breaks_and_returns(c, s, a):
     future = c.submit(slowinc, 1, delay=0.1)
     for i in range(10):
@@ -972,7 +972,7 @@ def test_worker_breaks_and_returns(c, s, a):
     assert states == {"memory": 1, "released": 10}
 
 
-@gen_cluster(client=True, ncores=[])
+@gen_cluster(client=True, nthreads=[])
 def test_no_workers_to_memory(c, s):
     x = delayed(slowinc)(1, delay=0.4)
     y = delayed(slowinc)(x, delay=0.4)
@@ -983,7 +983,7 @@ def test_no_workers_to_memory(c, s):
     while not s.tasks:
         yield gen.sleep(0.01)
 
-    w = Worker(s.address, ncores=1)
+    w = Worker(s.address, nthreads=1)
     w.put_key_in_memory(y.key, 3)
 
     yield w
@@ -1013,7 +1013,7 @@ def test_no_worker_to_memory_restrictions(c, s, a, b):
     while not s.tasks:
         yield gen.sleep(0.01)
 
-    w = Worker(s.address, ncores=1, name="alice")
+    w = Worker(s.address, nthreads=1, name="alice")
     w.put_key_in_memory(y.key, 3)
 
     yield w
@@ -1122,7 +1122,7 @@ def test_retire_nannies_close(c, s, a, b):
     assert not s.workers
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 2)])
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 2)])
 def test_fifo_submission(c, s, w):
     futures = []
     for i in range(20):
@@ -1147,17 +1147,17 @@ def test_scheduler_file():
 
 
 @pytest.mark.xfail(reason="")
-@gen_cluster(client=True, ncores=[])
+@gen_cluster(client=True, nthreads=[])
 def test_non_existent_worker(c, s):
     with dask.config.set({"distributed.comm.timeouts.connect": "100ms"}):
-        s.add_worker(address="127.0.0.1:5738", ncores=2, nbytes={}, host_info={})
+        s.add_worker(address="127.0.0.1:5738", nthreads=2, nbytes={}, host_info={})
         futures = c.map(inc, range(10))
         yield gen.sleep(0.300)
         assert not s.workers
         assert all(ts.state == "no-worker" for ts in s.tasks.values())
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 3)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 3)
 def test_correct_bad_time_estimate(c, s, *workers):
     future = c.submit(slowinc, 1, delay=0)
     yield wait(future)
@@ -1255,7 +1255,7 @@ def test_log_tasks_during_restart(c, s, a, b):
     assert "exit" in str(s.events)
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 2)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 2)
 def test_reschedule(c, s, a, b):
     yield c.submit(slowinc, -1, delay=0.1)  # learn cost
     x = c.map(slowinc, range(4), delay=0.1)
@@ -1324,7 +1324,7 @@ def test_retries(c, s, a, b):
 
 
 @pytest.mark.xfail(reason="second worker also errant for some reason")
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 3, timeout=5)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 3, timeout=5)
 def test_mising_data_errant_worker(c, s, w1, w2, w3):
     with dask.config.set({"distributed.comm.timeouts.connect": "1s"}):
         np = pytest.importorskip("numpy")
@@ -1457,7 +1457,7 @@ def test_closing_scheduler_closes_workers(s, a, b):
 
 
 @gen_cluster(
-    client=True, ncores=[("127.0.0.1", 1)], worker_kwargs={"resources": {"A": 1}}
+    client=True, nthreads=[("127.0.0.1", 1)], worker_kwargs={"resources": {"A": 1}}
 )
 def test_resources_reset_after_cancelled_task(c, s, w):
     future = c.submit(sleep, 0.2, resources={"A": 1})
diff --git a/distributed/tests/test_steal.py b/distributed/tests/test_steal.py
index 7348d164c72..d233fc28388 100644
--- a/distributed/tests/test_steal.py
+++ b/distributed/tests/test_steal.py
@@ -37,7 +37,7 @@
 @pytest.mark.skipif(
     not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
 )
-@gen_cluster(client=True, ncores=[("127.0.0.1", 2), ("127.0.0.2", 2)], timeout=20)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 2), ("127.0.0.2", 2)], timeout=20)
 def test_work_stealing(c, s, a, b):
     [x] = yield c._scatter([1], workers=a.address)
     futures = c.map(slowadd, range(50), [x] * 50)
@@ -46,7 +46,7 @@ def test_work_stealing(c, s, a, b):
     assert len(b.data) > 10
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 2)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 2)
 def test_dont_steal_expensive_data_fast_computation(c, s, a, b):
     np = pytest.importorskip("numpy")
     x = c.submit(np.arange, 1000000, workers=a.address)
@@ -64,7 +64,7 @@ def test_dont_steal_expensive_data_fast_computation(c, s, a, b):
     assert len(a.data) == 12
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 2)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 2)
 def test_steal_cheap_data_slow_computation(c, s, a, b):
     x = c.submit(slowinc, 100, delay=0.1)  # learn that slowinc is slow
     yield wait(x)
@@ -77,7 +77,7 @@ def test_steal_cheap_data_slow_computation(c, s, a, b):
 
 
 @pytest.mark.avoid_travis
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 2)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 2)
 def test_steal_expensive_data_slow_computation(c, s, a, b):
     np = pytest.importorskip("numpy")
 
@@ -94,7 +94,7 @@ def test_steal_expensive_data_slow_computation(c, s, a, b):
     assert b.data  # not empty
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 10)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 10)
 def test_worksteal_many_thieves(c, s, *workers):
     x = c.submit(slowinc, -1, delay=0.1)
     yield x
@@ -110,7 +110,7 @@ def test_worksteal_many_thieves(c, s, *workers):
     assert sum(map(len, s.has_what.values())) < 150
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 2)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 2)
 def test_dont_steal_unknown_functions(c, s, a, b):
     futures = c.map(inc, [1, 2], workers=a.address, allow_other_workers=True)
     yield wait(futures)
@@ -118,7 +118,7 @@ def test_dont_steal_unknown_functions(c, s, a, b):
     assert len(b.data) == 0
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 2)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 2)
 def test_eventually_steal_unknown_functions(c, s, a, b):
     futures = c.map(
         slowinc, range(10), delay=0.1, workers=a.address, allow_other_workers=True
@@ -129,7 +129,7 @@ def test_eventually_steal_unknown_functions(c, s, a, b):
 
 
 @pytest.mark.skip(reason="")
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 3)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 3)
 def test_steal_related_tasks(e, s, a, b, c):
     futures = e.map(
         slowinc, range(20), delay=0.05, workers=a.address, allow_other_workers=True
@@ -145,7 +145,7 @@ def test_steal_related_tasks(e, s, a, b, c):
     assert nearby > 10
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 10, timeout=1000)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 10, timeout=1000)
 def test_dont_steal_fast_tasks(c, s, *workers):
     np = pytest.importorskip("numpy")
     x = c.submit(np.random.random, 10000000, workers=workers[0].address)
@@ -163,7 +163,7 @@ def do_nothing(x, y=None):
     assert len(s.has_what[workers[0].address]) == 1001
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)], timeout=20)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)], timeout=20)
 def test_new_worker_steals(c, s, a):
     yield wait(c.submit(slowinc, 1, delay=0.01))
 
@@ -172,7 +172,7 @@ def test_new_worker_steals(c, s, a):
     while len(a.task_state) < 10:
         yield gen.sleep(0.01)
 
-    b = yield Worker(s.address, loop=s.loop, ncores=1, memory_limit=TOTAL_MEMORY)
+    b = yield Worker(s.address, loop=s.loop, nthreads=1, memory_limit=TOTAL_MEMORY)
 
     result = yield total
     assert result == sum(map(inc, range(100)))
@@ -204,7 +204,7 @@ def test_work_steal_no_kwargs(c, s, a, b):
     assert result == sum(map(inc, range(100)))
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1), ("127.0.0.1", 2)])
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1), ("127.0.0.1", 2)])
 def test_dont_steal_worker_restrictions(c, s, a, b):
     future = c.submit(slowinc, 1, delay=0.10, workers=a.address)
     yield future
@@ -228,7 +228,7 @@ def test_dont_steal_worker_restrictions(c, s, a, b):
 @pytest.mark.skipif(
     not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
 )
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1), ("127.0.0.2", 1)])
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1), ("127.0.0.2", 1)])
 def test_dont_steal_host_restrictions(c, s, a, b):
     future = c.submit(slowinc, 1, delay=0.10, workers=a.address)
     yield future
@@ -247,7 +247,7 @@ def test_dont_steal_host_restrictions(c, s, a, b):
 
 
 @gen_cluster(
-    client=True, ncores=[("127.0.0.1", 1, {"resources": {"A": 2}}), ("127.0.0.1", 1)]
+    client=True, nthreads=[("127.0.0.1", 1, {"resources": {"A": 2}}), ("127.0.0.1", 1)]
 )
 def test_dont_steal_resource_restrictions(c, s, a, b):
     future = c.submit(slowinc, 1, delay=0.10, workers=a.address)
@@ -267,7 +267,9 @@ def test_dont_steal_resource_restrictions(c, s, a, b):
 
 
 @pytest.mark.skip(reason="no stealing of resources")
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1, {"resources": {"A": 2}})], timeout=3)
+@gen_cluster(
+    client=True, nthreads=[("127.0.0.1", 1, {"resources": {"A": 2}})], timeout=3
+)
 def test_steal_resource_restrictions(c, s, a):
     future = c.submit(slowinc, 1, delay=0.10, workers=a.address)
     yield future
@@ -277,7 +279,7 @@ def test_steal_resource_restrictions(c, s, a):
         yield gen.sleep(0.01)
     assert len(a.task_state) == 101
 
-    b = yield Worker(s.address, loop=s.loop, ncores=1, resources={"A": 4})
+    b = yield Worker(s.address, loop=s.loop, nthreads=1, resources={"A": 4})
 
     start = time()
     while not b.task_state or len(a.task_state) == 101:
@@ -290,7 +292,7 @@ def test_steal_resource_restrictions(c, s, a):
     yield b.close()
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 5, timeout=20)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 5, timeout=20)
 def test_balance_without_dependencies(c, s, *workers):
     s.extensions["stealing"]._pc.callback_time = 20
 
@@ -306,7 +308,7 @@ def slow(x):
     assert max(durations) / min(durations) < 3
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 4)] * 2)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 4)] * 2)
 def test_dont_steal_executing_tasks(c, s, a, b):
     futures = c.map(
         slowinc, range(4), delay=0.1, workers=a.address, allow_other_workers=True
@@ -317,7 +319,7 @@ def test_dont_steal_executing_tasks(c, s, a, b):
     assert len(b.data) == 0
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 10)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 10)
 def test_dont_steal_few_saturated_tasks_many_workers(c, s, a, *rest):
     s.extensions["stealing"]._pc.callback_time = 20
     x = c.submit(mul, b"0", 100000000, workers=a.address)  # 100 MB
@@ -334,7 +336,7 @@ def test_dont_steal_few_saturated_tasks_many_workers(c, s, a, *rest):
 
 @gen_cluster(
     client=True,
-    ncores=[("127.0.0.1", 1)] * 10,
+    nthreads=[("127.0.0.1", 1)] * 10,
     worker_kwargs={"memory_limit": TOTAL_MEMORY},
 )
 def test_steal_when_more_tasks(c, s, a, *rest):
@@ -351,7 +353,7 @@ def test_steal_when_more_tasks(c, s, a, *rest):
         assert time() < start + 1
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 10)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 10)
 def test_steal_more_attractive_tasks(c, s, a, *rest):
     def slow2(x):
         sleep(1)
@@ -473,11 +475,11 @@ def assert_balanced(inp, expected, c, s, *workers):
 )
 def test_balance(inp, expected):
     test = lambda *args, **kwargs: assert_balanced(inp, expected, *args, **kwargs)
-    test = gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * len(inp))(test)
+    test = gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * len(inp))(test)
     test()
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 2, Worker=Nanny, timeout=20)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 2, Worker=Nanny, timeout=20)
 def test_restart(c, s, a, b):
     futures = c.map(
         slowinc, range(100), delay=0.1, workers=a.address, allow_other_workers=True
@@ -569,7 +571,7 @@ def test_dont_steal_executing_tasks(c, s, a, b):
     assert not b.executing
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 2)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 2)
 def test_dont_steal_long_running_tasks(c, s, a, b):
     def long(delay):
         with worker_client() as c:
@@ -603,7 +605,7 @@ def long(delay):
         ) <= 1
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 5)] * 2)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 5)] * 2)
 def test_cleanup_repeated_tasks(c, s, a, b):
     class Foo(object):
         pass
@@ -635,7 +637,7 @@ class Foo(object):
     assert not list(ws)
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 2)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 2)
 def test_lose_task(c, s, a, b):
     with captured_logger("distributed.stealing") as log:
         s.periodic_callbacks["stealing"].interval = 1
diff --git a/distributed/tests/test_stress.py b/distributed/tests/test_stress.py
index 8c37f5a82fb..81d7c4360f7 100644
--- a/distributed/tests/test_stress.py
+++ b/distributed/tests/test_stress.py
@@ -64,7 +64,7 @@ def test_stress_gc(loop, func, n):
 @pytest.mark.skipif(
     sys.platform.startswith("win"), reason="test can leave dangling RPC objects"
 )
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 8, timeout=None)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 8, timeout=None)
 def test_cancel_stress(c, s, *workers):
     da = pytest.importorskip("dask.array")
     x = da.random.random((50, 50), chunks=(2, 2))
@@ -93,7 +93,7 @@ def test_cancel_stress_sync(loop):
                 c.cancel(f)
 
 
-@gen_cluster(ncores=[], client=True, timeout=None)
+@gen_cluster(nthreads=[], client=True, timeout=None)
 def test_stress_creation_and_deletion(c, s):
     # Assertions are handled by the validate mechanism in the scheduler
     s.allowed_failures = 100000
@@ -108,7 +108,7 @@ def test_stress_creation_and_deletion(c, s):
     def create_and_destroy_worker(delay):
         start = time()
         while time() < start + 5:
-            n = Nanny(s.address, ncores=2, loop=s.loop)
+            n = Nanny(s.address, nthreads=2, loop=s.loop)
             n.start(0)
 
             yield gen.sleep(delay)
@@ -122,7 +122,7 @@ def create_and_destroy_worker(delay):
     )
 
 
-@gen_cluster(ncores=[("127.0.0.1", 1)] * 10, client=True, timeout=60)
+@gen_cluster(nthreads=[("127.0.0.1", 1)] * 10, client=True, timeout=60)
 def test_stress_scatter_death(c, s, *workers):
     import random
 
@@ -198,7 +198,7 @@ def vsum(*args):
 
 @pytest.mark.avoid_travis
 @pytest.mark.slow
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 80, timeout=1000)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 80, timeout=1000)
 def test_stress_communication(c, s, *workers):
     s.validate = False  # very slow otherwise
     da = pytest.importorskip("dask.array")
@@ -218,7 +218,7 @@ def test_stress_communication(c, s, *workers):
 
 
 @pytest.mark.skip
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 10, timeout=60)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 10, timeout=60)
 def test_stress_steal(c, s, *workers):
     s.validate = False
     for w in workers:
@@ -244,7 +244,7 @@ def test_stress_steal(c, s, *workers):
 
 
 @pytest.mark.slow
-@gen_cluster(ncores=[("127.0.0.1", 1)] * 10, client=True, timeout=120)
+@gen_cluster(nthreads=[("127.0.0.1", 1)] * 10, client=True, timeout=120)
 def test_close_connections(c, s, *workers):
     da = pytest.importorskip("dask.array")
     x = da.random.random(size=(1000, 1000), chunks=(1000, 1))
@@ -269,7 +269,7 @@ def test_close_connections(c, s, *workers):
     reason="IOStream._handle_write blocks on large write_buffer"
     " https://github.com/tornadoweb/tornado/issues/2110"
 )
-@gen_cluster(client=True, timeout=20, ncores=[("127.0.0.1", 1)])
+@gen_cluster(client=True, timeout=20, nthreads=[("127.0.0.1", 1)])
 def test_no_delay_during_large_transfer(c, s, w):
     pytest.importorskip("crick")
     np = pytest.importorskip("numpy")
diff --git a/distributed/tests/test_tls_functional.py b/distributed/tests/test_tls_functional.py
index 74a9cf3cbd4..7d097e28112 100644
--- a/distributed/tests/test_tls_functional.py
+++ b/distributed/tests/test_tls_functional.py
@@ -82,7 +82,7 @@ def test_nanny(c, s, a, b):
         assert isinstance(n, Nanny)
         assert n.address.startswith("tls://")
         assert n.worker_address.startswith("tls://")
-    assert s.ncores == {n.worker_address: n.ncores for n in [a, b]}
+    assert s.nthreads == {n.worker_address: n.nthreads for n in [a, b]}
 
     x = c.submit(inc, 10)
     result = yield x
@@ -101,7 +101,7 @@ def test_rebalance(c, s, a, b):
     assert len(b.data) == 1
 
 
-@gen_tls_cluster(client=True, ncores=[("tls://127.0.0.1", 2)] * 2)
+@gen_tls_cluster(client=True, nthreads=[("tls://127.0.0.1", 2)] * 2)
 def test_work_stealing(c, s, a, b):
     [x] = yield c._scatter([1], workers=a.address)
     futures = c.map(slowadd, range(50), [x] * 50, delay=0.1)
@@ -127,7 +127,7 @@ def func(x):
     assert yy == 20 + 1 + (20 + 1) * 2
 
 
-@gen_tls_cluster(client=True, ncores=[("tls://127.0.0.1", 1)] * 2)
+@gen_tls_cluster(client=True, nthreads=[("tls://127.0.0.1", 1)] * 2)
 def test_worker_client_gather(c, s, a, b):
     a_address = a.address
     b_address = b.address
diff --git a/distributed/tests/test_utils_test.py b/distributed/tests/test_utils_test.py
index 6f704c23f5b..c0afb9e2c7f 100755
--- a/distributed/tests/test_utils_test.py
+++ b/distributed/tests/test_utils_test.py
@@ -50,7 +50,7 @@ def test_gen_cluster(c, s, a, b):
     assert isinstance(s, Scheduler)
     for w in [a, b]:
         assert isinstance(w, Worker)
-    assert s.ncores == {w.address: w.ncores for w in [a, b]}
+    assert s.nthreads == {w.address: w.nthreads for w in [a, b]}
 
 
 @pytest.mark.skip(reason="This hangs on travis")
@@ -74,13 +74,13 @@ def test_gen_cluster_without_client(s, a, b):
     assert isinstance(s, Scheduler)
     for w in [a, b]:
         assert isinstance(w, Worker)
-    assert s.ncores == {w.address: w.ncores for w in [a, b]}
+    assert s.nthreads == {w.address: w.nthreads for w in [a, b]}
 
 
 @gen_cluster(
     client=True,
     scheduler="tls://127.0.0.1",
-    ncores=[("tls://127.0.0.1", 1), ("tls://127.0.0.1", 2)],
+    nthreads=[("tls://127.0.0.1", 1), ("tls://127.0.0.1", 2)],
     security=tls_only_security(),
 )
 def test_gen_cluster_tls(e, s, a, b):
@@ -90,7 +90,7 @@ def test_gen_cluster_tls(e, s, a, b):
     for w in [a, b]:
         assert isinstance(w, Worker)
         assert w.address.startswith("tls://")
-    assert s.ncores == {w.address: w.ncores for w in [a, b]}
+    assert s.nthreads == {w.address: w.nthreads for w in [a, b]}
 
 
 @gen_test()
diff --git a/distributed/tests/test_variable.py b/distributed/tests/test_variable.py
index 4d8851668f9..e734cc3094f 100644
--- a/distributed/tests/test_variable.py
+++ b/distributed/tests/test_variable.py
@@ -148,7 +148,7 @@ def test_timeout_get(c, s, a, b):
 
 @pytest.mark.skipif(sys.version_info[0] == 2, reason="Multi-client issues")
 @pytest.mark.slow
-@gen_cluster(client=True, ncores=[("127.0.0.1", 2)] * 5, Worker=Nanny, timeout=None)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 2)] * 5, Worker=Nanny, timeout=None)
 def test_race(c, s, *workers):
     NITERS = 50
 
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 12a6b5ff68f..fa0cf857a32 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -3,6 +3,7 @@
 from concurrent.futures import ThreadPoolExecutor
 from datetime import timedelta
 import logging
+import multiprocessing
 from numbers import Number
 from operator import add
 import os
@@ -49,12 +50,10 @@
 )
 
 
-def test_worker_ncores():
-    from distributed.worker import _ncores
-
+def test_worker_nthreads():
     w = Worker("127.0.0.1", 8019)
     try:
-        assert w.executor._max_workers == _ncores
+        assert w.executor._max_workers == multiprocessing.cpu_count()
     finally:
         shutil.rmtree(w.local_dir)
 
@@ -63,8 +62,8 @@ def test_worker_ncores():
 def test_str(s, a, b):
     assert a.address in str(a)
     assert a.address in repr(a)
-    assert str(a.ncores) in str(a)
-    assert str(a.ncores) in repr(a)
+    assert str(a.nthreads) in str(a)
+    assert str(a.nthreads) in repr(a)
     assert str(len(a.executing)) in repr(a)
 
 
@@ -73,7 +72,7 @@ def test_identity():
     ident = w.identity(None)
     assert "Worker" in ident["type"]
     assert ident["scheduler"] == "tcp://127.0.0.1:8019"
-    assert isinstance(ident["ncores"], int)
+    assert isinstance(ident["nthreads"], int)
     assert isinstance(ident["memory_limit"], Number)
 
 
@@ -198,7 +197,7 @@ def g():
 
 
 @pytest.mark.skip(reason="don't yet support uploading pyc files")
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)])
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)])
 def test_upload_file_pyc(c, s, w):
     with tmpfile() as dirname:
         os.mkdir(dirname)
@@ -329,7 +328,7 @@ def f():
     loop.run_sync(f)
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)])
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)])
 def test_worker_task_data(c, s, w):
     x = delayed(2)
     xx = c.persist(x)
@@ -370,7 +369,7 @@ async def test_io_loop():
     await w.close()
 
 
-@gen_cluster(client=True, ncores=[])
+@gen_cluster(client=True, nthreads=[])
 def test_spill_to_disk(c, s):
     np = pytest.importorskip("numpy")
     w = yield Worker(
@@ -441,7 +440,7 @@ def f(dask_worker=None):
     assert response == {a.address: a.id, b.address: b.id}
 
 
-@gen_cluster(client=True, ncores=[])
+@gen_cluster(client=True, nthreads=[])
 def test_Executor(c, s):
     with ThreadPoolExecutor(2) as e:
         w = Worker(s.address, executor=e)
@@ -462,7 +461,7 @@ def test_Executor(c, s):
 )
 @gen_cluster(
     client=True,
-    ncores=[("127.0.0.1", 1)],
+    nthreads=[("127.0.0.1", 1)],
     timeout=30,
     worker_kwargs={"memory_limit": 10e6},
 )
@@ -475,7 +474,7 @@ def test_spill_by_default(c, s, w):
     del x, y
 
 
-@gen_cluster(ncores=[("127.0.0.1", 1)], worker_kwargs={"reconnect": False})
+@gen_cluster(nthreads=[("127.0.0.1", 1)], worker_kwargs={"reconnect": False})
 def test_close_on_disconnect(s, w):
     yield s.close()
 
@@ -486,10 +485,10 @@ def test_close_on_disconnect(s, w):
 
 
 def test_memory_limit_auto():
-    a = Worker("127.0.0.1", 8099, ncores=1)
-    b = Worker("127.0.0.1", 8099, ncores=2)
-    c = Worker("127.0.0.1", 8099, ncores=100)
-    d = Worker("127.0.0.1", 8099, ncores=200)
+    a = Worker("127.0.0.1", 8099, nthreads=1)
+    b = Worker("127.0.0.1", 8099, nthreads=2)
+    c = Worker("127.0.0.1", 8099, nthreads=100)
+    d = Worker("127.0.0.1", 8099, nthreads=200)
 
     assert isinstance(a.memory_limit, Number)
     assert isinstance(b.memory_limit, Number)
@@ -585,7 +584,7 @@ def test_system_monitor(s, a, b):
 
 
 @gen_cluster(
-    client=True, ncores=[("127.0.0.1", 2, {"resources": {"A": 1}}), ("127.0.0.1", 1)]
+    client=True, nthreads=[("127.0.0.1", 2, {"resources": {"A": 1}}), ("127.0.0.1", 1)]
 )
 def test_restrictions(c, s, a, b):
     # Resource restrictions
@@ -615,7 +614,7 @@ def test_clean_nbytes(c, s, a, b):
     assert len(a.nbytes) + len(b.nbytes) == 1
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 20)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 20)
 def test_gather_many_small(c, s, a, *workers):
     a.total_out_connections = 2
     futures = yield c._scatter(list(range(100)))
@@ -636,7 +635,7 @@ def f(*args):
     assert a.comm_nbytes == 0
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 3)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 3)
 def test_multiple_transfers(c, s, w1, w2, w3):
     x = c.submit(inc, 1, workers=w1.address)
     y = c.submit(inc, 2, workers=w2.address)
@@ -649,7 +648,7 @@ def test_multiple_transfers(c, s, w1, w2, w3):
     assert len(transfers) == 2
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 3)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 3)
 def test_share_communication(c, s, w1, w2, w3):
     x = c.submit(mul, b"1", int(w3.target_message_size + 1), workers=w1.address)
     y = c.submit(mul, b"2", int(w3.target_message_size + 1), workers=w2.address)
@@ -733,7 +732,7 @@ def test_hold_onto_dependents(c, s, a, b):
 
 
 @pytest.mark.slow
-@gen_cluster(client=False, ncores=[])
+@gen_cluster(client=False, nthreads=[])
 def test_worker_death_timeout(s):
     with dask.config.set({"distributed.comm.timeouts.connect": "1s"}):
         yield s.close()
@@ -756,7 +755,7 @@ def test_stop_doing_unnecessary_work(c, s, a, b):
         assert time() - start < 0.5
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)])
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)])
 def test_priorities(c, s, w):
     values = []
     for i in range(10):
@@ -842,7 +841,7 @@ def __sizeof__(self):
 
 @pytest.mark.skip(reason="Our logic here is faulty")
 @gen_cluster(
-    ncores=[("127.0.0.1", 2)], client=True, worker_kwargs={"memory_limit": 10e9}
+    nthreads=[("127.0.0.1", 2)], client=True, worker_kwargs={"memory_limit": 10e9}
 )
 def test_fail_write_many_to_disk(c, s, a):
     a.validate = False
@@ -947,7 +946,7 @@ def test_global_workers(s, a, b):
 
 
 @pytest.mark.skipif(WINDOWS, reason="file descriptors")
-@gen_cluster(ncores=[])
+@gen_cluster(nthreads=[])
 def test_worker_fds(s):
     psutil = pytest.importorskip("psutil")
     yield gen.sleep(0.05)
@@ -969,7 +968,7 @@ def test_worker_fds(s):
         assert time() < start + 0.5
 
 
-@gen_cluster(ncores=[])
+@gen_cluster(nthreads=[])
 def test_service_hosts_match_worker(s):
     pytest.importorskip("bokeh")
     from distributed.dashboard import BokehWorker
@@ -995,7 +994,7 @@ def test_service_hosts_match_worker(s):
     yield w.close()
 
 
-@gen_cluster(ncores=[])
+@gen_cluster(nthreads=[])
 def test_start_services(s):
     pytest.importorskip("bokeh")
     from distributed.dashboard import BokehWorker
@@ -1051,7 +1050,7 @@ def test_statistical_profiling_2(c, s, a, b):
 
 
 @gen_cluster(
-    ncores=[("127.0.0.1", 1)],
+    nthreads=[("127.0.0.1", 1)],
     client=True,
     worker_kwargs={"memory_monitor_interval": 10},
 )
@@ -1082,7 +1081,7 @@ def f(n):
 
 @pytest.mark.slow
 @gen_cluster(
-    ncores=[("127.0.0.1", 2)],
+    nthreads=[("127.0.0.1", 2)],
     client=True,
     worker_kwargs={
         "memory_monitor_interval": 10,
@@ -1151,7 +1150,7 @@ def some_name():
     assert result.startswith("some_name")
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 2)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 2)
 def test_reschedule(c, s, a, b):
     s.extensions["stealing"]._pc.stop()
     a_address = a.address
@@ -1180,7 +1179,7 @@ def test_deque_handler():
     assert any(msg.msg == "foo456" for msg in deque_handler.deque)
 
 
-@gen_cluster(ncores=[], client=True)
+@gen_cluster(nthreads=[], client=True)
 def test_avoid_memory_monitor_if_zero_limit(c, s):
     worker = yield Worker(
         s.address, loop=s.loop, memory_limit=0, memory_monitor_interval=10
@@ -1198,7 +1197,7 @@ def test_avoid_memory_monitor_if_zero_limit(c, s):
 
 
 @gen_cluster(
-    ncores=[("127.0.0.1", 1)],
+    nthreads=[("127.0.0.1", 1)],
     config={
         "distributed.worker.memory.spill": False,
         "distributed.worker.memory.target": False,
@@ -1223,12 +1222,12 @@ def func(dask_scheduler):
         assert time() < start + 10
 
 
-@gen_cluster(ncores=[("127.0.0.1", 1)], worker_kwargs={"memory_limit": "2e3 MB"})
+@gen_cluster(nthreads=[("127.0.0.1", 1)], worker_kwargs={"memory_limit": "2e3 MB"})
 def test_parse_memory_limit(s, w):
     assert w.memory_limit == 2e9
 
 
-@gen_cluster(ncores=[], client=True)
+@gen_cluster(nthreads=[], client=True)
 def test_scheduler_address_config(c, s):
     with dask.config.set({"scheduler-address": s.address}):
         worker = yield Worker(loop=s.loop)
@@ -1257,7 +1256,9 @@ def test_wait_for_outgoing(c, s, a, b):
 @pytest.mark.skipif(
     not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
 )
-@gen_cluster(ncores=[("127.0.0.1", 1), ("127.0.0.1", 1), ("127.0.0.2", 1)], client=True)
+@gen_cluster(
+    nthreads=[("127.0.0.1", 1), ("127.0.0.1", 1), ("127.0.0.2", 1)], client=True
+)
 def test_prefer_gather_from_local_address(c, s, w1, w2, w3):
     x = yield c.scatter(123, workers=[w1.address, w3.address], broadcast=True)
 
@@ -1270,7 +1271,7 @@ def test_prefer_gather_from_local_address(c, s, w1, w2, w3):
 
 @gen_cluster(
     client=True,
-    ncores=[("127.0.0.1", 1)] * 20,
+    nthreads=[("127.0.0.1", 1)] * 20,
     timeout=30,
     config={"distributed.worker.connections.incoming": 1},
 )
@@ -1367,7 +1368,7 @@ def test_register_worker_callbacks_err(c, s, a, b):
         yield c.register_worker_callbacks(setup=lambda: 1 / 0)
 
 
-@gen_cluster(ncores=[])
+@gen_cluster(nthreads=[])
 def test_data_types(s):
     w = yield Worker(s.address, data=dict)
     assert isinstance(w.data, dict)
@@ -1389,7 +1390,7 @@ def __init__(self, x, y):
     yield w.close()
 
 
-@gen_cluster(ncores=[])
+@gen_cluster(nthreads=[])
 def test_local_dir(s):
     with tmpfile() as fn:
         with dask.config.set(temporary_directory=fn):
@@ -1401,7 +1402,7 @@ def test_local_dir(s):
 @pytest.mark.skipif(
     not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
 )
-@gen_cluster(ncores=[], client=True)
+@gen_cluster(nthreads=[], client=True)
 def test_host_address(c, s):
     w = yield Worker(s.address, host="127.0.0.2")
     assert "127.0.0.2" in w.address
diff --git a/distributed/tests/test_worker_client.py b/distributed/tests/test_worker_client.py
index 9c4616e9d26..fe1d49def6d 100644
--- a/distributed/tests/test_worker_client.py
+++ b/distributed/tests/test_worker_client.py
@@ -42,7 +42,7 @@ def func(x):
     assert len([id for id in s.wants_what if id.lower().startswith("client")]) == 1
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 2)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 2)
 def test_scatter_from_worker(c, s, a, b):
     def func():
         with worker_client() as c:
@@ -78,12 +78,12 @@ def func():
     assert result is True
 
     start = time()
-    while not all(v == 1 for v in s.ncores.values()):
+    while not all(v == 1 for v in s.nthreads.values()):
         yield gen.sleep(0.1)
         assert time() < start + 5
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 2)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 2)
 def test_scatter_singleton(c, s, a, b):
     np = pytest.importorskip("numpy")
 
@@ -96,7 +96,7 @@ def func():
     yield c.submit(func)
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 1)] * 2)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 2)
 def test_gather_multi_machine(c, s, a, b):
     a_address = a.address
     b_address = b.address
@@ -162,7 +162,7 @@ def mysum():
         assert time() < start + 3
 
 
-@gen_cluster(client=True, ncores=[("127.0.0.1", 3)])
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 3)])
 def test_separate_thread_false(c, s, a):
     a.count = 0
 
diff --git a/distributed/tests/test_worker_plugins.py b/distributed/tests/test_worker_plugins.py
index 25388459788..425a267923a 100644
--- a/distributed/tests/test_worker_plugins.py
+++ b/distributed/tests/test_worker_plugins.py
@@ -19,7 +19,7 @@ def teardown(self, worker):
         self.worker._my_plugin_status = "teardown"
 
 
-@gen_cluster(client=True, ncores=[])
+@gen_cluster(client=True, nthreads=[])
 def test_create_with_client(c, s):
     yield c.register_worker_plugin(MyPlugin(123))
 
diff --git a/distributed/utils_comm.py b/distributed/utils_comm.py
index d2bd19908af..e6f5235afe0 100644
--- a/distributed/utils_comm.py
+++ b/distributed/utils_comm.py
@@ -110,19 +110,19 @@ def __repr__(self):
 
 
 @gen.coroutine
-def scatter_to_workers(ncores, data, rpc=rpc, report=True, serializers=None):
+def scatter_to_workers(nthreads, data, rpc=rpc, report=True, serializers=None):
     """ Scatter data directly to workers
 
     This distributes data in a round-robin fashion to a set of workers based on
-    how many cores they have.  ncores should be a dictionary mapping worker
+    how many cores they have.  nthreads should be a dictionary mapping worker
     identities to numbers of cores.
 
     See scatter for parameter docstring
     """
-    assert isinstance(ncores, dict)
+    assert isinstance(nthreads, dict)
     assert isinstance(data, dict)
 
-    workers = list(concat([w] * nc for w, nc in ncores.items()))
+    workers = list(concat([w] * nc for w, nc in nthreads.items()))
     names, data = list(zip(*data.items()))
 
     worker_iter = drop(_round_robin_counter[0] % len(workers), cycle(workers))
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index 10784c6f759..89e0f3283e1 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -646,7 +646,7 @@ def cluster(
             q = mp_context.Queue()
             fn = "_test_worker-%s" % uuid.uuid4()
             kwargs = merge(
-                {"ncores": 1, "local_dir": fn, "memory_limit": TOTAL_MEMORY},
+                {"nthreads": 1, "local_dir": fn, "memory_limit": TOTAL_MEMORY},
                 worker_kwargs,
             )
             proc = mp_context.Process(
@@ -678,8 +678,8 @@ def cluster(
 
             with rpc(saddr, **rpc_kwargs) as s:
                 while True:
-                    ncores = loop.run_sync(s.ncores)
-                    if len(ncores) == nworkers:
+                    nthreads = loop.run_sync(s.ncores)
+                    if len(nthreads) == nworkers:
                         break
                     if time() - start > 5:
                         raise Exception("Timeout on cluster creation")
@@ -783,7 +783,7 @@ def test_func():
 
 @gen.coroutine
 def start_cluster(
-    ncores,
+    nthreads,
     scheduler_addr,
     loop,
     security=None,
@@ -798,7 +798,7 @@ def start_cluster(
     workers = [
         Worker(
             s.address,
-            ncores=ncore[1],
+            nthreads=ncore[1],
             name=i,
             security=security,
             loop=loop,
@@ -806,7 +806,7 @@ def start_cluster(
             host=ncore[0],
             **(merge(worker_kwargs, ncore[2]) if len(ncore) > 2 else worker_kwargs)
         )
-        for i, ncore in enumerate(ncores)
+        for i, ncore in enumerate(nthreads)
     ]
     # for w in workers:
     #     w.rpc = workers[0].rpc
@@ -814,7 +814,7 @@ def start_cluster(
     yield workers
 
     start = time()
-    while len(s.workers) < len(ncores) or any(
+    while len(s.workers) < len(nthreads) or any(
         comm.comm is None for comm in s.stream_comms.values()
     ):
         yield gen.sleep(0.01)
@@ -840,7 +840,7 @@ def end_worker(w):
 
 
 def gen_cluster(
-    ncores=[("127.0.0.1", 1), ("127.0.0.1", 2)],
+    nthreads=[("127.0.0.1", 1), ("127.0.0.1", 2)],
     scheduler="127.0.0.1",
     timeout=10,
     security=None,
@@ -885,7 +885,7 @@ def coro():
                         for i in range(5):
                             try:
                                 s, ws = yield start_cluster(
-                                    ncores,
+                                    nthreads,
                                     scheduler,
                                     loop,
                                     security=security,
@@ -1406,7 +1406,7 @@ def bump_rlimit(limit, desired):
 
 
 def gen_tls_cluster(**kwargs):
-    kwargs.setdefault("ncores", [("tls://127.0.0.1", 1), ("tls://127.0.0.1", 2)])
+    kwargs.setdefault("nthreads", [("tls://127.0.0.1", 1), ("tls://127.0.0.1", 2)])
     return gen_cluster(
         scheduler="tls://127.0.0.1", security=tls_only_security(), **kwargs
     )
diff --git a/distributed/worker.py b/distributed/worker.py
index 37dcbc2eca1..86d078254f0 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -5,6 +5,7 @@
 from datetime import timedelta
 import heapq
 import logging
+import multiprocessing
 import os
 from pickle import PicklingError
 import random
@@ -53,7 +54,6 @@
     _maybe_complex,
     log_errors,
     ignoring,
-    mp_context,
     import_file,
     silence_logging,
     thread_state,
@@ -69,8 +69,6 @@
 from .utils_comm import pack_data, gather_from_workers
 from .utils_perf import ThrottledGC, enable_gc_diagnosis, disable_gc_diagnosis
 
-_ncores = mp_context.cpu_count()
-
 logger = logging.getLogger(__name__)
 
 LOG_PDB = dask.config.get("distributed.admin.pdb-on-err")
@@ -116,8 +114,8 @@ class Worker(ServerNode):
 
     These attributes don't change significantly during execution.
 
-    * **ncores:** ``int``:
-        Number of cores used by this worker process
+    * **nthreads:** ``int``:
+        Number of nthreads used by this worker process
     * **executor:** ``concurrent.futures.ThreadPoolExecutor``:
         Executor used to perform computation
     * **local_dir:** ``path``:
@@ -233,7 +231,7 @@ class Worker(ServerNode):
     ip: str, optional
     data: MutableMapping, type, None
         The object to use for storage, builds a disk-backed LRU dict by default
-    ncores: int, optional
+    nthreads: int, optional
     loop: tornado.ioloop.IOLoop
     local_dir: str, optional
         Directory where we place local resources
@@ -241,7 +239,7 @@ class Worker(ServerNode):
     memory_limit: int, float, string
         Number of bytes of memory that this worker should use.
         Set to zero for no limit.  Set to 'auto' to calculate
-        as TOTAL_MEMORY * min(1, ncores / total_cores)
+        as TOTAL_MEMORY * min(1, nthreads / total_cores)
         Use strings or numbers like 5GB or 5e9
     memory_target_fraction: float
         Fraction of memory to try to stay beneath
@@ -281,6 +279,7 @@ def __init__(
         scheduler_port=None,
         scheduler_file=None,
         ncores=None,
+        nthreads=None,
         loop=None,
         local_dir=None,
         services=None,
@@ -432,7 +431,11 @@ def __init__(
             security=security,
         )
 
-        self.ncores = ncores or _ncores
+        if ncores is not None:
+            warnings.warn("the ncores= parameter has moved to nthreads=")
+            nthreads = ncores
+
+        self.nthreads = nthreads or multiprocessing.cpu_count()
         self.total_resources = resources or {}
         self.available_resources = (resources or {}).copy()
         self.death_timeout = parse_timedelta(death_timeout)
@@ -471,7 +474,7 @@ def __init__(
         self.connection_args = self.security.get_connection_args("worker")
         self.listen_args = self.security.get_listen_args("worker")
 
-        self.memory_limit = parse_memory_limit(memory_limit, self.ncores)
+        self.memory_limit = parse_memory_limit(memory_limit, self.nthreads)
 
         self.paused = False
 
@@ -526,7 +529,7 @@ def __init__(
         self._closed = Event()
         self.reconnect = reconnect
         self.executor = executor or ThreadPoolExecutor(
-            self.ncores, thread_name_prefix="Dask-Worker-Threads'"
+            self.nthreads, thread_name_prefix="Dask-Worker-Threads'"
         )
         self.actor_executor = ThreadPoolExecutor(
             1, thread_name_prefix="Dask-Actor-Threads"
@@ -658,7 +661,7 @@ def __repr__(self):
                 self.status,
                 len(self.data),
                 len(self.executing),
-                self.ncores,
+                self.nthreads,
                 len(self.ready),
                 len(self.in_flight_tasks),
                 len(self.waiting_for_data),
@@ -687,7 +690,8 @@ def identity(self, comm=None):
             "type": type(self).__name__,
             "id": self.id,
             "scheduler": self.scheduler.address,
-            "ncores": self.ncores,
+            "nthreads": self.nthreads,
+            "ncores": self.nthreads,  # backwards compatibility
             "memory_limit": self.memory_limit,
         }
 
@@ -722,7 +726,7 @@ def _register_with_scheduler(self):
                         reply=False,
                         address=self.contact_address,
                         keys=list(self.data),
-                        ncores=self.ncores,
+                        nthreads=self.nthreads,
                         name=self.name,
                         nbytes=self.nbytes,
                         types=types,
@@ -941,7 +945,7 @@ def _start(self, addr_or_port=0):
             logger.info("  %16s at: %26s" % (k, listen_host + ":" + str(v)))
         logger.info("Waiting to connect to: %26s", self.scheduler.address)
         logger.info("-" * 49)
-        logger.info("              Threads: %26d", self.ncores)
+        logger.info("              Threads: %26d", self.nthreads)
         if self.memory_limit:
             logger.info("               Memory: %26s", format_bytes(self.memory_limit))
         logger.info("      Local Directory: %26s", self.local_dir)
@@ -2283,7 +2287,7 @@ def ensure_computing(self):
         if self.paused:
             return
         try:
-            while self.constrained and len(self.executing) < self.ncores:
+            while self.constrained and len(self.executing) < self.nthreads:
                 key = self.constrained[0]
                 if self.task_state.get(key) != "constrained":
                     self.constrained.popleft()
@@ -2293,7 +2297,7 @@ def ensure_computing(self):
                     self.transition(key, "executing")
                 else:
                     break
-            while self.ready and len(self.executing) < self.ncores:
+            while self.ready and len(self.executing) < self.nthreads:
                 _, key = heapq.heappop(self.ready)
                 if self.task_state.get(key) in READY:
                     self.transition(key, "executing")
@@ -2955,12 +2959,12 @@ class Reschedule(Exception):
     pass
 
 
-def parse_memory_limit(memory_limit, ncores, total_cores=_ncores):
+def parse_memory_limit(memory_limit, nthreads, total_cores=multiprocessing.cpu_count()):
     if memory_limit is None:
         return None
 
     if memory_limit == "auto":
-        memory_limit = int(TOTAL_MEMORY * min(1, ncores / total_cores))
+        memory_limit = int(TOTAL_MEMORY * min(1, nthreads / total_cores))
     with ignoring(ValueError, TypeError):
         memory_limit = float(memory_limit)
         if isinstance(memory_limit, float) and memory_limit <= 1:
diff --git a/docs/source/api.rst b/docs/source/api.rst
index 574a70d34b6..adefe5b86c4 100644
--- a/docs/source/api.rst
+++ b/docs/source/api.rst
@@ -24,7 +24,7 @@ API
    Client.has_what
    Client.list_datasets
    Client.map
-   Client.ncores
+   Client.nthreads
    Client.persist
    Client.publish_dataset
    Client.profile
diff --git a/docs/source/local-cluster.rst b/docs/source/local-cluster.rst
index c415bbfbeba..d596ccaed24 100644
--- a/docs/source/local-cluster.rst
+++ b/docs/source/local-cluster.rst
@@ -7,7 +7,7 @@ For convenience you can start a local cluster from your Python session.
 
    >>> from distributed import Client, LocalCluster
    >>> cluster = LocalCluster()
-   LocalCluster("127.0.0.1:8786", workers=8, ncores=8)
+   LocalCluster("127.0.0.1:8786", workers=8, nthreads=8)
    >>> client = Client(cluster)
    <Client: scheduler=127.0.0.1:8786 processes=8 cores=8>
 
diff --git a/docs/source/protocol.rst b/docs/source/protocol.rst
index 645ba4aa905..334e2c0e4bd 100644
--- a/docs/source/protocol.rst
+++ b/docs/source/protocol.rst
@@ -25,7 +25,7 @@ In practice we represent these messages with dictionaries/mappings::
    {'op': 'register-worker',
     'address': '192.168.1.42',
     'name': 'alice',
-    'ncores': 4}
+    'nthreads': 4}
 
    {'x': b'...',
     'y': b'...'}
diff --git a/docs/source/scheduling-state.rst b/docs/source/scheduling-state.rst
index 90db367767f..515bb26cdb0 100644
--- a/docs/source/scheduling-state.rst
+++ b/docs/source/scheduling-state.rst
@@ -112,7 +112,7 @@ containers to help with scheduling tasks:
 .. attribute:: Scheduler.saturated: {WorkerState}
 
    A set of workers whose computing power (as
-   measured by :attr:`WorkerState.ncores`) is fully exploited by processing
+   measured by :attr:`WorkerState.nthreads`) is fully exploited by processing
    tasks, and whose current :attr:`~WorkerState.occupancy` is a lot greater
    than the average.
 
diff --git a/docs/source/worker.rst b/docs/source/worker.rst
index 4b835d7ba67..530a27b9505 100644
--- a/docs/source/worker.rst
+++ b/docs/source/worker.rst
@@ -100,7 +100,7 @@ are the available options::
      --name TEXT            Alias
      --memory-limit TEXT    Maximum bytes of memory that this worker should use.
                             Use 0 for unlimited, or 'auto' for
-                            TOTAL_MEMORY * min(1, ncores / total_cores)
+                            TOTAL_MEMORY * min(1, nthreads / total_nthreads)
      --no-nanny
      --help                 Show this message and exit.
 
@@ -151,7 +151,7 @@ command line ``--memory-limit`` keyword or the ``memory_limit=`` Python
 keyword argument, which sets the memory limit per worker processes launched
 by dask-worker ::
 
-    $ dask-worker tcp://scheduler:port --memory-limit=auto  # TOTAL_MEMORY * min(1, ncores / total_cores)
+    $ dask-worker tcp://scheduler:port --memory-limit=auto  # TOTAL_MEMORY * min(1, nthreads / total_nthreads)
     $ dask-worker tcp://scheduler:port --memory-limit=4e9  # four gigabytes per worker process.
 
 Workers use a few different heuristics to keep memory use beneath this limit:

From 1fb26c7ab0e84b7b678b52cb8ffd0e2948fc63ed Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 19 Jun 2019 09:30:38 +0200
Subject: [PATCH 0326/1550] Clean up lingering ncores->nthreads change in
 widget code (#2785)

---
 distributed/deploy/cluster.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
index 69cc5be9fac..74d61a995e1 100644
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -123,7 +123,7 @@ def scale(self, n):
 
     def _widget_status(self):
         workers = len(self.scheduler.workers)
-        cores = sum(ws.ncores for ws in self.scheduler.workers.values())
+        cores = sum(ws.nthreads for ws in self.scheduler.workers.values())
         memory = sum(ws.memory_limit for ws in self.scheduler.workers.values())
         memory = format_bytes(memory)
         text = """

From eba954b0ce589b2186e7ea78d697a79ac1faad62 Mon Sep 17 00:00:00 2001
From: Tom Augspurger <TomAugspurger@users.noreply.github.com>
Date: Wed, 19 Jun 2019 10:37:56 -0500
Subject: [PATCH 0327/1550] Raise when workers initialization times out (#2784)

This changes Worker / Nanny startup to raise when they timeout.
This bubbles up to the `dask-worker` CLI.

Closes #2781
---
 distributed/cli/dask_worker.py            |  5 ++++-
 distributed/cli/tests/test_dask_worker.py | 14 ++++++++++++++
 distributed/nanny.py                      |  8 +++++++-
 distributed/tests/test_nanny.py           |  5 +++--
 distributed/tests/test_worker.py          |  6 ++++--
 distributed/worker.py                     |  8 +++++++-
 6 files changed, 39 insertions(+), 7 deletions(-)

diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
index a53ddf99f6e..13b699ab0dd 100755
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -373,7 +373,10 @@ def run():
 
     try:
         loop.run_sync(run)
-    except (KeyboardInterrupt, TimeoutError):
+    except TimeoutError:
+        # We already log the exception in nanny / worker. Don't do it again.
+        raise TimeoutError("Timed out starting worker.") from None
+    except KeyboardInterrupt:
         pass
     finally:
         logger.info("End worker")
diff --git a/distributed/cli/tests/test_dask_worker.py b/distributed/cli/tests/test_dask_worker.py
index dc7c761fdf1..9191d7aba4d 100644
--- a/distributed/cli/tests/test_dask_worker.py
+++ b/distributed/cli/tests/test_dask_worker.py
@@ -1,6 +1,7 @@
 from __future__ import print_function, division, absolute_import
 
 import pytest
+from click.testing import CliRunner
 
 pytest.importorskip("requests")
 
@@ -9,6 +10,7 @@
 import os
 from time import sleep
 
+import distributed.cli.dask_worker
 from distributed import Client
 from distributed.metrics import time
 from distributed.utils import sync, tmpfile
@@ -292,3 +294,15 @@ def test_dashboard_non_standard_ports(loop):
 
         with pytest.raises(Exception):
             requests.get("http://localhost:4833/status/")
+
+
+@pytest.mark.slow
+@pytest.mark.parametrize("no_nanny", [True, False])
+def test_worker_timeout(no_nanny):
+    runner = CliRunner()
+    args = ["192.168.1.100:7777", "--death-timeout=1"]
+    if no_nanny:
+        args.append("--no-nanny")
+    result = runner.invoke(distributed.cli.dask_worker.main, args)
+    assert result.exit_code != 0
+    assert str(result.exception).startswith("Timed out")
diff --git a/distributed/nanny.py b/distributed/nanny.py
index d907c7171a1..f518b330d7c 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -304,7 +304,13 @@ def instantiate(self, comm=None):
                 )
             except gen.TimeoutError:
                 yield self.close(timeout=self.death_timeout)
-                raise gen.Return("timed out")
+                logger.exception(
+                    "Timed out connecting Nanny '%s' to scheduler '%s'",
+                    self,
+                    self.scheduler_addr,
+                )
+                raise
+
         else:
             result = yield self.process.start()
         raise gen.Return(result)
diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
index 6b6d5bf939d..40c8d49012d 100644
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -160,9 +160,10 @@ def test_nanny_alt_worker_class(c, s, w1, w2):
 @gen_cluster(client=False, nthreads=[])
 def test_nanny_death_timeout(s):
     yield s.close()
-    w = yield Nanny(s.address, death_timeout=1)
+    w = Nanny(s.address, death_timeout=1)
+    with pytest.raises(gen.TimeoutError):
+        yield w
 
-    yield gen.sleep(3)
     assert w.status == "closed"
 
 
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index fa0cf857a32..a0e8244e8bd 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -736,9 +736,11 @@ def test_hold_onto_dependents(c, s, a, b):
 def test_worker_death_timeout(s):
     with dask.config.set({"distributed.comm.timeouts.connect": "1s"}):
         yield s.close()
-        w = yield Worker(s.address, death_timeout=1)
+        w = Worker(s.address, death_timeout=1)
+
+    with pytest.raises(gen.TimeoutError):
+        yield w
 
-    yield gen.sleep(2)
     assert w.status == "closed"
 
 
diff --git a/distributed/worker.py b/distributed/worker.py
index 86d078254f0..63dfaed3114 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -708,8 +708,14 @@ def _register_with_scheduler(self):
         logger.info("-" * 49)
         while True:
             if self.death_timeout and time() > start + self.death_timeout:
+                logger.exception(
+                    "Timed out when connecting to scheduler '%s'",
+                    self.scheduler.address,
+                )
                 yield self.close(timeout=1)
-                return
+                raise gen.TimeoutError(
+                    "Timed out connecting to scheduler '%s'" % self.scheduler.address
+                )
             if self.status in ("closed", "closing"):
                 raise gen.Return
             try:

From c5f479ff28e91aed47c9e307ca3e0e65ea9c9150 Mon Sep 17 00:00:00 2001
From: Tom Augspurger <TomAugspurger@users.noreply.github.com>
Date: Thu, 20 Jun 2019 15:08:46 -0500
Subject: [PATCH 0328/1550] Add version option to scheduler and worker CLI
 (#2782)

* Add version option to scheduler and worker CLI

* Add version to other cli commands

* Add tests
---
 distributed/cli/dask_mpi.py                  | 1 +
 distributed/cli/dask_remote.py               | 1 +
 distributed/cli/dask_scheduler.py            | 1 +
 distributed/cli/dask_ssh.py                  | 1 +
 distributed/cli/dask_submit.py               | 1 +
 distributed/cli/dask_worker.py               | 1 +
 distributed/cli/tests/test_dask_mpi.py       | 8 ++++++++
 distributed/cli/tests/test_dask_remote.py    | 6 ++++++
 distributed/cli/tests/test_dask_scheduler.py | 9 +++++++++
 distributed/cli/tests/test_dask_ssh.py       | 8 ++++++++
 distributed/cli/tests/test_dask_submit.py    | 6 ++++++
 distributed/cli/tests/test_dask_worker.py    | 6 ++++++
 12 files changed, 49 insertions(+)
 create mode 100644 distributed/cli/tests/test_dask_ssh.py

diff --git a/distributed/cli/dask_mpi.py b/distributed/cli/dask_mpi.py
index 2a965824662..7b9aeaca213 100644
--- a/distributed/cli/dask_mpi.py
+++ b/distributed/cli/dask_mpi.py
@@ -63,6 +63,7 @@
     help="Worker's Bokeh port for visual diagnostics",
 )
 @click.option("--bokeh-prefix", type=str, default=None, help="Prefix for the bokeh app")
+@click.version_option()
 def main(
     scheduler_file,
     interface,
diff --git a/distributed/cli/dask_remote.py b/distributed/cli/dask_remote.py
index 933d8d318b0..29cc5c3c784 100644
--- a/distributed/cli/dask_remote.py
+++ b/distributed/cli/dask_remote.py
@@ -8,6 +8,7 @@
 @click.command()
 @click.option("--host", type=str, default=None, help="IP or hostname of this server")
 @click.option("--port", type=int, default=8788, help="Remote Client Port")
+@click.version_option()
 def main(host, port):
     _remote(host, port)
 
diff --git a/distributed/cli/dask_scheduler.py b/distributed/cli/dask_scheduler.py
index b27e68eaa9a..c38f405f04e 100755
--- a/distributed/cli/dask_scheduler.py
+++ b/distributed/cli/dask_scheduler.py
@@ -113,6 +113,7 @@
 @click.argument(
     "preload_argv", nargs=-1, type=click.UNPROCESSED, callback=validate_preload_argv
 )
+@click.version_option()
 def main(
     host,
     port,
diff --git a/distributed/cli/dask_ssh.py b/distributed/cli/dask_ssh.py
index 2d98992d969..1d264dc80e5 100755
--- a/distributed/cli/dask_ssh.py
+++ b/distributed/cli/dask_ssh.py
@@ -101,6 +101,7 @@
     help="Worker to run. Defaults to distributed.cli.dask_worker",
 )
 @click.pass_context
+@click.version_option()
 def main(
     ctx,
     scheduler,
diff --git a/distributed/cli/dask_submit.py b/distributed/cli/dask_submit.py
index 1ef759407c6..071dd5bbe32 100644
--- a/distributed/cli/dask_submit.py
+++ b/distributed/cli/dask_submit.py
@@ -9,6 +9,7 @@
 @click.command()
 @click.argument("remote_client_address", type=str, required=True)
 @click.argument("filepath", type=str, required=True)
+@click.version_option()
 def main(remote_client_address, filepath):
     @gen.coroutine
     def f():
diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
index 13b699ab0dd..1463c29afd1 100755
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -180,6 +180,7 @@
 @click.argument(
     "preload_argv", nargs=-1, type=click.UNPROCESSED, callback=validate_preload_argv
 )
+@click.version_option()
 def main(
     scheduler,
     host,
diff --git a/distributed/cli/tests/test_dask_mpi.py b/distributed/cli/tests/test_dask_mpi.py
index 8bc8dddca2e..89f1140bfab 100644
--- a/distributed/cli/tests/test_dask_mpi.py
+++ b/distributed/cli/tests/test_dask_mpi.py
@@ -8,12 +8,14 @@
 pytest.importorskip("mpi4py")
 
 import requests
+from click.testing import CliRunner
 
 from distributed import Client
 from distributed.utils import tmpfile
 from distributed.metrics import time
 from distributed.utils_test import popen
 from distributed.utils_test import loop  # noqa: F401
+from distributed.cli.dask_remote import main
 
 
 @pytest.mark.parametrize("nanny", ["--nanny", "--no-nanny"])
@@ -96,3 +98,9 @@ def test_bokeh(loop):
 
     with pytest.raises(Exception):
         requests.get("http://localhost:59583/status/")
+
+
+def test_version_option():
+    runner = CliRunner()
+    result = runner.invoke(main, ["--version"])
+    assert result.exit_code == 0
diff --git a/distributed/cli/tests/test_dask_remote.py b/distributed/cli/tests/test_dask_remote.py
index 04d04d62ecf..14da80f949c 100644
--- a/distributed/cli/tests/test_dask_remote.py
+++ b/distributed/cli/tests/test_dask_remote.py
@@ -7,3 +7,9 @@ def test_dask_remote():
     result = runner.invoke(main, ["--help"])
     assert "--host TEXT     IP or hostname of this server" in result.output
     assert result.exit_code == 0
+
+
+def test_version_option():
+    runner = CliRunner()
+    result = runner.invoke(main, ["--version"])
+    assert result.exit_code == 0
diff --git a/distributed/cli/tests/test_dask_scheduler.py b/distributed/cli/tests/test_dask_scheduler.py
index e04fa24bad1..7de7e881270 100644
--- a/distributed/cli/tests/test_dask_scheduler.py
+++ b/distributed/cli/tests/test_dask_scheduler.py
@@ -13,7 +13,9 @@
 from time import sleep
 
 from tornado import gen
+from click.testing import CliRunner
 
+import distributed
 from distributed import Scheduler, Client
 from distributed.utils import get_ip, get_ip_interface, tmpfile
 from distributed.utils_test import (
@@ -23,6 +25,7 @@
 )
 from distributed.utils_test import loop  # noqa: F401
 from distributed.metrics import time
+import distributed.cli.dask_scheduler
 
 
 def test_defaults(loop):
@@ -374,3 +377,9 @@ def check_passthrough():
 
     finally:
         shutil.rmtree(tmpdir)
+
+
+def test_version_option():
+    runner = CliRunner()
+    result = runner.invoke(distributed.cli.dask_scheduler.main, ["--version"])
+    assert result.exit_code == 0
diff --git a/distributed/cli/tests/test_dask_ssh.py b/distributed/cli/tests/test_dask_ssh.py
new file mode 100644
index 00000000000..9be8cb06f62
--- /dev/null
+++ b/distributed/cli/tests/test_dask_ssh.py
@@ -0,0 +1,8 @@
+from click.testing import CliRunner
+from distributed.cli.dask_ssh import main
+
+
+def test_version_option():
+    runner = CliRunner()
+    result = runner.invoke(main, ["--version"])
+    assert result.exit_code == 0
diff --git a/distributed/cli/tests/test_dask_submit.py b/distributed/cli/tests/test_dask_submit.py
index 83c7c1067fa..8f5f961ea96 100644
--- a/distributed/cli/tests/test_dask_submit.py
+++ b/distributed/cli/tests/test_dask_submit.py
@@ -7,3 +7,9 @@ def test_submit_runs_as_a_cli():
     result = runner.invoke(main, ["--help"])
     assert result.exit_code == 0
     assert "Usage: main [OPTIONS] REMOTE_CLIENT_ADDRESS FILEPATH" in result.output
+
+
+def test_version_option():
+    runner = CliRunner()
+    result = runner.invoke(main, ["--version"])
+    assert result.exit_code == 0
diff --git a/distributed/cli/tests/test_dask_worker.py b/distributed/cli/tests/test_dask_worker.py
index 9191d7aba4d..edba84d2ef4 100644
--- a/distributed/cli/tests/test_dask_worker.py
+++ b/distributed/cli/tests/test_dask_worker.py
@@ -296,6 +296,12 @@ def test_dashboard_non_standard_ports(loop):
             requests.get("http://localhost:4833/status/")
 
 
+def test_version_option():
+    runner = CliRunner()
+    result = runner.invoke(distributed.cli.dask_worker.main, ["--version"])
+    assert result.exit_code == 0
+
+
 @pytest.mark.slow
 @pytest.mark.parametrize("no_nanny", [True, False])
 def test_worker_timeout(no_nanny):

From 4ba820a1218d468692c36d4551eeba6491440366 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 21 Jun 2019 10:13:08 +0200
Subject: [PATCH 0329/1550] Add warnings around ncores= keywords (#2791)

---
 distributed/scheduler.py  | 6 ++++++
 distributed/utils_test.py | 6 ++++++
 2 files changed, 12 insertions(+)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 434d118a422..d370705e9af 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -13,6 +13,7 @@
 import pickle
 import random
 import six
+import warnings
 import weakref
 
 import psutil
@@ -307,6 +308,11 @@ def identity(self):
             "nanny": self.nanny,
         }
 
+    @property
+    def ncores(self):
+        warnings.warn("WorkerState.ncores has moved to WorkerState.nthreads")
+        return self.nthreads
+
 
 class TaskState(object):
     """
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index 89e0f3283e1..293cf5c0737 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -22,6 +22,7 @@
 import threading
 from time import sleep
 import uuid
+import warnings
 import weakref
 
 try:
@@ -841,6 +842,7 @@ def end_worker(w):
 
 def gen_cluster(
     nthreads=[("127.0.0.1", 1), ("127.0.0.1", 2)],
+    ncores=None,
     scheduler="127.0.0.1",
     timeout=10,
     security=None,
@@ -865,6 +867,10 @@ def test_foo(scheduler, worker1, worker2):
         start
         end
     """
+    if ncores is not None:
+        warnings.warn("ncores= has moved to nthreads=")
+        nthreads = ncores
+
     worker_kwargs = merge(
         {"memory_limit": TOTAL_MEMORY, "death_timeout": 5}, worker_kwargs
     )

From c5e830d4f386ef30664403d1d415f94c751cc8ce Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Fri, 21 Jun 2019 12:43:39 -0500
Subject: [PATCH 0330/1550] Remove "experimental" from TLS docs [skip ci]
 (#2793)

---
 docs/source/tls.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/source/tls.rst b/docs/source/tls.rst
index 8aaad99c45a..d367dabbf7b 100644
--- a/docs/source/tls.rst
+++ b/docs/source/tls.rst
@@ -4,9 +4,9 @@
 TLS/SSL
 =======
 
-Currently dask distributed has experimental support for TLS/SSL communication,
+Dask distributed has support for TLS/SSL communication,
 providing mutual authentication and encryption of communications between cluster
-endpoints (Clients, Schedulers and Workers).
+endpoints (Clients, Schedulers, and Workers).
 
 TLS is enabled by using a ``tls`` address such as ``tls://`` (the default
 being ``tcp``, which sends data unauthenticated and unencrypted).  In

From 9b4c8fc177a26a428b451677ed12fdf6bb577f09 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Sat, 22 Jun 2019 08:35:55 -0500
Subject: [PATCH 0331/1550] Update command line cli options docs (#2794)

---
 docs/requirements.txt                   |   1 +
 docs/source/conf.py                     | 230 ++++++++++++------------
 docs/source/submitting-applications.rst |  27 ++-
 3 files changed, 142 insertions(+), 116 deletions(-)

diff --git a/docs/requirements.txt b/docs/requirements.txt
index 329b7d7d23b..61dd185a5b9 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -5,3 +5,4 @@ dask
 numpydoc
 sphinx
 dask_sphinx_theme
+sphinx-click
diff --git a/docs/source/conf.py b/docs/source/conf.py
index 6c79073e3b8..c8ffc0ae50d 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -12,54 +12,51 @@
 # All configuration values have a default; values that are commented out
 # serve to show the default.
 
-import sys
-import os
-import shlex
-
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
-#sys.path.insert(0, os.path.abspath('.'))
+# sys.path.insert(0, os.path.abspath('.'))
 
 # -- General configuration ------------------------------------------------
 
 # If your documentation needs a minimal Sphinx version, state it here.
-#needs_sphinx = '1.0'
+# needs_sphinx = '1.0'
 
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
-    'sphinx.ext.autodoc',
-    'sphinx.ext.todo',
-    'sphinx.ext.ifconfig',
-    'sphinx.ext.viewcode',
-    'sphinx.ext.autosummary',
-    'sphinx.ext.extlinks',
-    'sphinx.ext.intersphinx',
-    'numpydoc',
+    "sphinx.ext.autodoc",
+    "sphinx.ext.todo",
+    "sphinx.ext.ifconfig",
+    "sphinx.ext.viewcode",
+    "sphinx.ext.autosummary",
+    "sphinx.ext.extlinks",
+    "sphinx.ext.intersphinx",
+    "numpydoc",
+    "sphinx_click.ext",
 ]
 
 numpydoc_show_class_members = False
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
 
 # The suffix(es) of source filenames.
 # You can specify multiple suffix as a list of string:
 # source_suffix = ['.rst', '.md']
-source_suffix = '.rst'
+source_suffix = ".rst"
 
 # The encoding of source files.
-#source_encoding = 'utf-8-sig'
+# source_encoding = 'utf-8-sig'
 
 # The master toctree document.
-master_doc = 'index'
+master_doc = "index"
 
 # General information about the project.
-project = u'Dask.distributed'
-copyright = u'2016, Anaconda, Inc.'
-author = u'Anaconda, Inc.'
+project = u"Dask.distributed"
+copyright = u"2016, Anaconda, Inc."
+author = u"Anaconda, Inc."
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
@@ -67,6 +64,7 @@
 #
 # The short X.Y version.
 import distributed
+
 version = distributed.__version__
 # The full version, including alpha/beta/rc tags.
 release = distributed.__version__
@@ -80,9 +78,9 @@
 
 # There are two options for replacing |today|: either, you set today to some
 # non-false value, then it is used:
-#today = ''
+# today = ''
 # Else, today_fmt is used as the format for a strftime call.
-#today_fmt = '%B %d, %Y'
+# today_fmt = '%B %d, %Y'
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
@@ -90,27 +88,27 @@
 
 # The reST default role (used for this markup: `text`) to use for all
 # documents.
-#default_role = None
+# default_role = None
 
 # If true, '()' will be appended to :func: etc. cross-reference text.
-#add_function_parentheses = True
+# add_function_parentheses = True
 
 # If true, the current module name will be prepended to all description
 # unit titles (such as .. function::).
-#add_module_names = True
+# add_module_names = True
 
 # If true, sectionauthor and moduleauthor directives will be shown in the
 # output. They are ignored by default.
-#show_authors = False
+# show_authors = False
 
 # The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'default'
+pygments_style = "default"
 
 # A list of ignored prefixes for module index sorting.
-#modindex_common_prefix = []
+# modindex_common_prefix = []
 
 # If true, keep warnings as "system message" paragraphs in the built documents.
-#keep_warnings = False
+# keep_warnings = False
 
 # If true, `todo` and `todoList` produce output, else they produce nothing.
 todo_include_todos = True
@@ -119,144 +117,147 @@
 # -- Options for HTML output ----------------------------------------------
 
 import dask_sphinx_theme
-html_theme = 'dask_sphinx_theme'
+
+html_theme = "dask_sphinx_theme"
 html_theme_path = [dask_sphinx_theme.get_html_theme_path()]
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
 # documentation.
-#html_theme_options = {}
+# html_theme_options = {}
 
 # Add any paths that contain custom themes here, relative to this directory.
-#html_theme_path = []
+# html_theme_path = []
 
 # The name for this set of Sphinx documents.  If None, it defaults to
 # "<project> v<release> documentation".
-#html_title = None
+# html_title = None
 
 # A shorter title for the navigation bar.  Default is the same as html_title.
-#html_short_title = None
+# html_short_title = None
 
 # The name of an image file (relative to this directory) to place at the top
 # of the sidebar.
-#html_logo = None
+# html_logo = None
 
 # The name of an image file (within the static path) to use as favicon of the
 # docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
 # pixels large.
-#html_favicon = None
+# html_favicon = None
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+html_static_path = ["_static"]
 
 # Add any extra paths that contain custom files (such as robots.txt or
 # .htaccess) here, relative to this directory. These files are copied
 # directly to the root of the documentation.
-#html_extra_path = []
+# html_extra_path = []
 
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
 # using the given strftime format.
-#html_last_updated_fmt = '%b %d, %Y'
+# html_last_updated_fmt = '%b %d, %Y'
 
 # If true, SmartyPants will be used to convert quotes and dashes to
 # typographically correct entities.
-#html_use_smartypants = True
+# html_use_smartypants = True
 
 # Custom sidebar templates, maps document names to template names.
-#html_sidebars = {}
+# html_sidebars = {}
 
 # Additional templates that should be rendered to pages, maps page names to
 # template names.
-#html_additional_pages = {}
+# html_additional_pages = {}
 
 # If false, no module index is generated.
-#html_domain_indices = True
+# html_domain_indices = True
 
 # If false, no index is generated.
-#html_use_index = True
+# html_use_index = True
 
 # If true, the index is split into individual pages for each letter.
-#html_split_index = False
+# html_split_index = False
 
 # If true, links to the reST sources are added to the pages.
-#html_show_sourcelink = True
+# html_show_sourcelink = True
 
 # If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
-#html_show_sphinx = True
+# html_show_sphinx = True
 
 # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
-#html_show_copyright = True
+# html_show_copyright = True
 
 # If true, an OpenSearch description file will be output, and all pages will
 # contain a <link> tag referring to it.  The value of this option must be the
 # base URL from which the finished HTML is served.
-#html_use_opensearch = ''
+# html_use_opensearch = ''
 
 # This is the file name suffix for HTML files (e.g. ".xhtml").
-#html_file_suffix = None
+# html_file_suffix = None
 
 # Language to be used for generating the HTML full-text search index.
 # Sphinx supports the following languages:
 #   'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'
 #   'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr'
-#html_search_language = 'en'
+# html_search_language = 'en'
 
 # A dictionary with options for the search language support, empty by default.
 # Now only 'ja' uses this config value
-#html_search_options = {'type': 'default'}
+# html_search_options = {'type': 'default'}
 
 # The name of a javascript file (relative to the configuration directory) that
 # implements a search results scorer. If empty, the default will be used.
-#html_search_scorer = 'scorer.js'
+# html_search_scorer = 'scorer.js'
 
 # Output file base name for HTML help builder.
-htmlhelp_basename = 'distributeddoc'
+htmlhelp_basename = "distributeddoc"
 
 # -- Options for LaTeX output ---------------------------------------------
 
 latex_elements = {
-# The paper size ('letterpaper' or 'a4paper').
-#'papersize': 'letterpaper',
-
-# The font size ('10pt', '11pt' or '12pt').
-#'pointsize': '10pt',
-
-# Additional stuff for the LaTeX preamble.
-#'preamble': '',
-
-# Latex figure (float) alignment
-#'figure_align': 'htbp',
+    # The paper size ('letterpaper' or 'a4paper').
+    #'papersize': 'letterpaper',
+    # The font size ('10pt', '11pt' or '12pt').
+    #'pointsize': '10pt',
+    # Additional stuff for the LaTeX preamble.
+    #'preamble': '',
+    # Latex figure (float) alignment
+    #'figure_align': 'htbp',
 }
 
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title,
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
-  (master_doc, 'distributed.tex', u'Dask.distributed Documentation',
-   u'Matthew Rocklin', 'manual'),
+    (
+        master_doc,
+        "distributed.tex",
+        u"Dask.distributed Documentation",
+        u"Matthew Rocklin",
+        "manual",
+    )
 ]
 
 # The name of an image file (relative to this directory) to place at the top of
 # the title page.
-#latex_logo = None
+# latex_logo = None
 
 # For "manual" documents, if this is true, then toplevel headings are parts,
 # not chapters.
-#latex_use_parts = False
+# latex_use_parts = False
 
 # If true, show page references after internal links.
-#latex_show_pagerefs = False
+# latex_show_pagerefs = False
 
 # If true, show URL addresses after external links.
-#latex_show_urls = False
+# latex_show_urls = False
 
 # Documents to append as an appendix to all manuals.
-#latex_appendices = []
+# latex_appendices = []
 
 # If false, no module index is generated.
-#latex_domain_indices = True
+# latex_domain_indices = True
 
 
 # -- Options for manual page output ---------------------------------------
@@ -264,12 +265,11 @@
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
 man_pages = [
-    (master_doc, 'Dask.distributed', u'Dask.distributed Documentation',
-     [author], 1)
+    (master_doc, "Dask.distributed", u"Dask.distributed Documentation", [author], 1)
 ]
 
 # If true, show URL addresses after external links.
-#man_show_urls = False
+# man_show_urls = False
 
 
 # -- Options for Texinfo output -------------------------------------------
@@ -278,22 +278,28 @@
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-  (master_doc, 'Dask.distributed', u'Dask.distributed Documentation',
-   author, 'Dask.distributed', 'One line description of project.',
-   'Miscellaneous'),
+    (
+        master_doc,
+        "Dask.distributed",
+        u"Dask.distributed Documentation",
+        author,
+        "Dask.distributed",
+        "One line description of project.",
+        "Miscellaneous",
+    )
 ]
 
 # Documents to append as an appendix to all manuals.
-#texinfo_appendices = []
+# texinfo_appendices = []
 
 # If false, no module index is generated.
-#texinfo_domain_indices = True
+# texinfo_domain_indices = True
 
 # How to display URL addresses: 'footnote', 'no', or 'inline'.
-#texinfo_show_urls = 'footnote'
+# texinfo_show_urls = 'footnote'
 
 # If true, do not generate a @detailmenu in the "Top" node's menu.
-#texinfo_no_detailmenu = False
+# texinfo_no_detailmenu = False
 
 
 # -- Options for Epub output ----------------------------------------------
@@ -305,85 +311,85 @@
 epub_copyright = copyright
 
 # The basename for the epub file. It defaults to the project name.
-#epub_basename = project
+# epub_basename = project
 
 # The HTML theme for the epub output. Since the default themes are not optimized
 # for small screen space, using the same theme for HTML and epub output is
 # usually not wise. This defaults to 'epub', a theme designed to save visual
 # space.
-#epub_theme = 'epub'
+# epub_theme = 'epub'
 
 # The language of the text. It defaults to the language option
 # or 'en' if the language is not set.
-#epub_language = ''
+# epub_language = ''
 
 # The scheme of the identifier. Typical schemes are ISBN or URL.
-#epub_scheme = ''
+# epub_scheme = ''
 
 # The unique identifier of the text. This can be a ISBN number
 # or the project homepage.
-#epub_identifier = ''
+# epub_identifier = ''
 
 # A unique identification for the text.
-#epub_uid = ''
+# epub_uid = ''
 
 # A tuple containing the cover image and cover page html template filenames.
-#epub_cover = ()
+# epub_cover = ()
 
 # A sequence of (type, uri, title) tuples for the guide element of content.opf.
-#epub_guide = ()
+# epub_guide = ()
 
 # HTML files that should be inserted before the pages created by sphinx.
 # The format is a list of tuples containing the path and title.
-#epub_pre_files = []
+# epub_pre_files = []
 
 # HTML files shat should be inserted after the pages created by sphinx.
 # The format is a list of tuples containing the path and title.
-#epub_post_files = []
+# epub_post_files = []
 
 # A list of files that should not be packed into the epub file.
-epub_exclude_files = ['search.html']
+epub_exclude_files = ["search.html"]
 
 # The depth of the table of contents in toc.ncx.
-#epub_tocdepth = 3
+# epub_tocdepth = 3
 
 # Allow duplicate toc entries.
-#epub_tocdup = True
+# epub_tocdup = True
 
 # Choose between 'default' and 'includehidden'.
-#epub_tocscope = 'default'
+# epub_tocscope = 'default'
 
 # Fix unsupported image types using the Pillow.
-#epub_fix_images = False
+# epub_fix_images = False
 
 # Scale large images.
-#epub_max_image_width = 0
+# epub_max_image_width = 0
 
 # How to display URL addresses: 'footnote', 'no', or 'inline'.
-#epub_show_urls = 'inline'
+# epub_show_urls = 'inline'
 
 # If false, no index is generated.
-#epub_use_index = True
+# epub_use_index = True
 
 # Link to GitHub issues and pull requests using :pr:`1234` and :issue:`1234`
 # syntax
 extlinks = {
-    'issue': ('https://github.com/dask/distributed/issues/%s', 'GH#'),
-    'pr': ('https://github.com/dask/distributed/pull/%s', 'GH#')
+    "issue": ("https://github.com/dask/distributed/issues/%s", "GH#"),
+    "pr": ("https://github.com/dask/distributed/pull/%s", "GH#"),
 }
 
 # Configuration for intersphinx: refer to the Python standard library
 # and the Numpy documentation.
 intersphinx_mapping = {
-    'python': ('https://docs.python.org/3', None),
-    'numpy': ('http://docs.scipy.org/doc/numpy', None),
-    }
+    "python": ("https://docs.python.org/3", None),
+    "numpy": ("http://docs.scipy.org/doc/numpy", None),
+}
 
 # Redirects
 # https://tech.signavio.com/2017/managing-sphinx-redirects
 redirect_files = [
     # old html, new html
-    ('joblib.html', 'https://ml.dask.org/joblib.html'),
+    ("joblib.html", "https://ml.dask.org/joblib.html")
 ]
 
 
@@ -400,13 +406,13 @@
 
 
 def copy_legacy_redirects(app, docname):
-    if app.builder.name == 'html':
+    if app.builder.name == "html":
         for html_src_path, new in redirect_files:
             page = redirect_template.format(new=new)
-            target_path = app.outdir + '/' + html_src_path
-            with open(target_path, 'w') as f:
+            target_path = app.outdir + "/" + html_src_path
+            with open(target_path, "w") as f:
                 f.write(page)
 
 
 def setup(app):
-    app.connect('build-finished', copy_legacy_redirects)
+    app.connect("build-finished", copy_legacy_redirects)
diff --git a/docs/source/submitting-applications.rst b/docs/source/submitting-applications.rst
index 5f81f4fd658..8b5ab1d61c8 100644
--- a/docs/source/submitting-applications.rst
+++ b/docs/source/submitting-applications.rst
@@ -10,8 +10,8 @@ For example, S3 buckets could not be visible from your local machine and hence a
 attempt to create a dask graph from local machine may not work.
 
 
-Submitting dask Applications with `dask-submit`
------------------------------------------------
+Submitting dask Applications with ``dask-submit``
+-------------------------------------------------
 
 In order to remotely submit scripts to the cluster from a local machine or a CI/CD
 environment, we need to run a remote client on the same machine as the scheduler::
@@ -20,7 +20,7 @@ environment, we need to run a remote client on the same machine as the scheduler
    dask-remote --port 8788
 
 
-After making sure the `dask-remote` is running, you can submit a script by::
+After making sure the ``dask-remote`` is running, you can submit a script by::
 
    #local machine
    dask-submit <dask-remote-address>:<port> <script.py>
@@ -28,7 +28,7 @@ After making sure the `dask-remote` is running, you can submit a script by::
 
 Some of the commonly used arguments are:
 
--  ``REMOTE_CLIENT_ADDRESS``: host name where dask-remote client is running
+-  ``REMOTE_CLIENT_ADDRESS``: host name where ``dask-remote`` client is running
 -  ``FILEPATH``: Local path to file containing dask application
 
 For example, given the following dask application saved in a file called
@@ -36,6 +36,7 @@ For example, given the following dask application saved in a file called
 
 .. code-block:: python
 
+   # script.py
    from distributed import Client
 
    def inc(x):
@@ -50,3 +51,21 @@ For example, given the following dask application saved in a file called
 We can submit this application from a local machine by running::
 
    dask-submit <remote-client-address>:<port> script.py
+
+
+CLI Options
+-----------
+
+.. note::
+
+   The command line documentation here may differ depending on your installed
+   version. We recommend referring to the output of ``dask-remote --help``
+   and ``dask-submit --help``.
+
+.. click:: distributed.cli.dask_remote:main
+   :prog: dask-remote
+   :show-nested:
+
+.. click:: distributed.cli.dask_submit:main
+   :prog: dask-submit
+   :show-nested:
\ No newline at end of file

From 912c8a38919079b502e934b1a76ba9b201ec21ab Mon Sep 17 00:00:00 2001
From: Tom Augspurger <TomAugspurger@users.noreply.github.com>
Date: Mon, 24 Jun 2019 08:11:20 -0500
Subject: [PATCH 0332/1550] Typo in bokeh service_kwargs for dask-worker
 (#2783)

---
 distributed/cli/dask_worker.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
index 1463c29afd1..f341a1abf78 100755
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -347,7 +347,7 @@ def del_pid_file():
             host=host,
             port=port,
             dashboard_address=dashboard_address if dashboard else None,
-            service_kwargs={"bokhe": {"prefix": dashboard_prefix}},
+            service_kwargs={"dashboard": {"prefix": dashboard_prefix}},
             name=name if nprocs == 1 or not name else name + "-" + str(i),
             **kwargs
         )

From 99444c24ec8d1c8248e273f5de5bd15b86f2ade2 Mon Sep 17 00:00:00 2001
From: Tom Augspurger <TomAugspurger@users.noreply.github.com>
Date: Tue, 25 Jun 2019 01:38:28 -0500
Subject: [PATCH 0333/1550] Deprecate --bokeh/--no-bokeh CLI (#2800)

Closes https://github.com/dask/distributed/issues/2799
---
 distributed/cli/dask_scheduler.py         | 13 +++++++++++++
 distributed/cli/dask_worker.py            | 13 +++++++++++++
 distributed/cli/tests/test_dask_worker.py | 19 +++++++++++++++++++
 3 files changed, 45 insertions(+)

diff --git a/distributed/cli/dask_scheduler.py b/distributed/cli/dask_scheduler.py
index c38f405f04e..f2799164a36 100755
--- a/distributed/cli/dask_scheduler.py
+++ b/distributed/cli/dask_scheduler.py
@@ -78,6 +78,13 @@
     required=False,
     help="Launch the Dashboard",
 )
+@click.option(
+    "--bokeh/--no-bokeh",
+    "bokeh",
+    default=None,
+    required=False,
+    help="Deprecated.  See --dashboard/--no-dashboard.",
+)
 @click.option("--show/--no-show", default=False, help="Show web UI")
 @click.option(
     "--dashboard-prefix", type=str, default=None, help="Prefix for the dashboard app"
@@ -120,6 +127,7 @@ def main(
     bokeh_port,
     show,
     dashboard,
+    bokeh,
     dashboard_prefix,
     use_xheaders,
     pid_file,
@@ -146,6 +154,11 @@ def main(
             "Consider adding ``--dashboard-address :%d`` " % bokeh_port
         )
         dashboard_address = bokeh_port
+    if bokeh is not None:
+        warnings.warn(
+            "The --bokeh/--no-bokeh flag has been renamed to --dashboard/--no-dashboard. "
+        )
+        dashboard = bokeh
 
     if port is None and (not host or not re.search(r":\d", host)):
         port = 8786
diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
index f341a1abf78..e86cfa41618 100755
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -77,6 +77,13 @@
     required=False,
     help="Launch the Dashboard",
 )
+@click.option(
+    "--bokeh/--no-bokeh",
+    "bokeh",
+    default=None,
+    help="Deprecated.  See --dashboard/--no-dashboard.",
+    required=False,
+)
 @click.option(
     "--listen-address",
     type=str,
@@ -197,6 +204,7 @@ def main(
     reconnect,
     resources,
     dashboard,
+    bokeh,
     bokeh_port,
     local_directory,
     scheduler_file,
@@ -223,6 +231,11 @@ def main(
             "Consider adding ``--dashboard-address :%d`` " % bokeh_port
         )
         dashboard_address = bokeh_port
+    if bokeh is not None:
+        warnings.warn(
+            "The --bokeh/--no-bokeh flag has been renamed to --dashboard/--no-dashboard. "
+        )
+        dashboard = bokeh
 
     sec = Security(
         tls_ca_file=tls_ca_file, tls_worker_cert=tls_cert, tls_worker_key=tls_key
diff --git a/distributed/cli/tests/test_dask_worker.py b/distributed/cli/tests/test_dask_worker.py
index edba84d2ef4..b6c7d393e3b 100644
--- a/distributed/cli/tests/test_dask_worker.py
+++ b/distributed/cli/tests/test_dask_worker.py
@@ -312,3 +312,22 @@ def test_worker_timeout(no_nanny):
     result = runner.invoke(distributed.cli.dask_worker.main, args)
     assert result.exit_code != 0
     assert str(result.exception).startswith("Timed out")
+
+
+def test_bokeh_deprecation():
+    pytest.importorskip("bokeh")
+
+    runner = CliRunner()
+    with pytest.warns(UserWarning, match="dashboard"):
+        try:
+            runner.invoke(distributed.cli.dask_worker.main, ["--bokeh"])
+        except ValueError:
+            # didn't pass scheduler
+            pass
+
+    with pytest.warns(UserWarning, match="dashboard"):
+        try:
+            runner.invoke(distributed.cli.dask_worker.main, ["--no-bokeh"])
+        except ValueError:
+            # didn't pass scheduler
+            pass

From 991391cb71492c2ecf10366bddd0dc8d526f212e Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 25 Jun 2019 09:01:02 +0200
Subject: [PATCH 0334/1550] Relax warnings before release (#2796)

Let's not be too strict about a couple of our warnings and missing
functions.
---
 distributed/deploy/local.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index ffb06b0a4bf..554459e43ac 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -110,7 +110,8 @@ def __init__(
         **worker_kwargs
     ):
         if ip is not None:
-            warnings.warn("The ip keyword has been moved to host")
+            # In the future we should warn users about this move
+            # warnings.warn("The ip keyword has been moved to host")
             host = ip
 
         if diagnostics_port is not None:
@@ -207,6 +208,12 @@ def __repr__(self):
             sum(w.nthreads for w in self.workers.values()),
         )
 
+    def start_worker(self, *args, **kwargs):
+        raise NotImplementedError(
+            "The `cluster.start_worker` function has been removed. "
+            "Please see the `cluster.scale` method instead."
+        )
+
 
 def nprocesses_nthreads(n=multiprocessing.cpu_count()):
     """

From e13f2984be5c5c388818576f78fcf30412374298 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 24 Jun 2019 17:20:54 +0200
Subject: [PATCH 0335/1550] bump version to 2.0

---
 distributed/tests/test_client.py | 20 +++++-----
 distributed/tests/test_worker.py | 13 ++++---
 docs/source/changelog.rst        | 63 +++++++++++++++++++++++++++++++-
 requirements.txt                 |  5 +--
 setup.cfg                        |  3 --
 setup.py                         |  1 -
 6 files changed, 80 insertions(+), 25 deletions(-)

diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index d18216ef0ef..dca108b57ee 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -51,7 +51,7 @@
     futures_of,
     temp_default_client,
 )
-from distributed.compatibility import PY3
+from distributed.compatibility import PY3, WINDOWS
 
 from distributed.metrics import time
 from distributed.scheduler import Scheduler, KilledWorker
@@ -2732,9 +2732,7 @@ def test_persist_get(c, s, a, b):
     assert result == ((1 + 1) + (2 + 2)) + 10
 
 
-@pytest.mark.skipif(
-    sys.platform.startswith("win"), reason="num_fds not supported on windows"
-)
+@pytest.mark.skipif(WINDOWS, reason="num_fds not supported on windows")
 def test_client_num_fds(loop):
     psutil = pytest.importorskip("psutil")
     with cluster() as (s, [a, b]):
@@ -3084,9 +3082,7 @@ def test_client_replicate_sync(c):
     assert y.result() == 3
 
 
-@pytest.mark.skipif(
-    sys.platform.startswith("win"), reason="Windows timer too coarse-grained"
-)
+@pytest.mark.skipif(WINDOWS, reason="Windows timer too coarse-grained")
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 4)] * 1)
 def test_task_load_adapts_quickly(c, s, a):
     future = c.submit(slowinc, 1, delay=0.2)  # slow
@@ -3573,9 +3569,7 @@ def test_reconnect_timeout(c, s):
 
 
 @pytest.mark.slow
-@pytest.mark.skipif(
-    sys.platform.startswith("win"), reason="num_fds not supported on windows"
-)
+@pytest.mark.skipif(WINDOWS, reason="num_fds not supported on windows")
 @pytest.mark.skipif(
     sys.version_info[0] == 2, reason="Semaphore.acquire doesn't support timeout option"
 )
@@ -5522,7 +5516,11 @@ def test_profile_bokeh(c, s, a, b):
     assert isinstance(figure, Model)
 
     with tmpfile("html") as fn:
-        yield c.profile(filename=fn)
+        try:
+            yield c.profile(filename=fn)
+        except PermissionError:
+            if WINDOWS:
+                pytest.xfail()
         assert os.path.exists(fn)
 
 
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index a0e8244e8bd..0f462d7de05 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -1038,17 +1038,20 @@ def test_statistical_profiling(c, s, a, b):
     assert profile["count"]
 
 
+@pytest.mark.slow
 @nodebug
-@gen_cluster(client=True)
+@gen_cluster(client=True, timeout=20)
 def test_statistical_profiling_2(c, s, a, b):
     da = pytest.importorskip("dask.array")
-    for i in range(5):
+    while True:
         x = da.random.random(1000000, chunks=(10000,))
         y = (x + x * 2) - x.sum().persist()
         yield wait(y)
-    profile = a.get_profile()
-    assert profile["count"]
-    assert "sum" in str(profile) or "random" in str(profile)
+
+        profile = a.get_profile()
+        text = str(profile)
+        if profile["count"] and "sum" in text and "random" in text:
+            break
 
 
 @gen_cluster(
diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index 613773a0c1c..37a44591ee5 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,12 +1,66 @@
 Changelog
 =========
 
+2.0.0 - 2019-06-25
+------------------
+
+-  **Drop support for Python 2**
+-  Relax warnings before release (:pr:`2796`) `Matthew Rocklin`_
+-  Deprecate --bokeh/--no-bokeh CLI (:pr:`2800`) `Tom Augspurger`_
+-  Typo in bokeh service_kwargs for dask-worker (:pr:`2783`) `Tom Augspurger`_
+-  Update command line cli options docs (:pr:`2794`) `James Bourbeau`_
+-  Remove "experimental" from TLS docs (:pr:`2793`) `James Bourbeau`_
+-  Add warnings around ncores= keywords (:pr:`2791`) `Matthew Rocklin`_
+-  Add --version option to scheduler and worker CLI (:pr:`2782`) `Tom Augspurger`_
+-  Raise when workers initialization times out (:pr:`2784`) `Tom Augspurger`_
+-  Replace ncores with nthreads throughout codebase (:pr:`2758`) `Matthew Rocklin`_
+-  Add unknown pytest markers (:pr:`2764`) `Tom Augspurger`_
+-  Delay lookup of allowed failures. (:pr:`2761`) `Tom Augspurger`_
+-  Change address -> worker in ColumnDataSource for nbytes plot (:pr:`2755`) `Matthew Rocklin`_
+-  Remove module state in Prometheus Handlers (:pr:`2760`) `Matthew Rocklin`_
+-  Add stress test for UCX (:pr:`2759`) `Matthew Rocklin`_
+-  Add nanny logs (:pr:`2744`) `Tom Augspurger`_
+-  Move some of the adaptive logic into the scheduler (:pr:`2735`) `Matthew Rocklin`_
+-  Add SpecCluster.new_worker_spec method (:pr:`2751`) `Matthew Rocklin`_
+-  Worker dashboard fixes (:pr:`2747`) `Matthew Rocklin`_
+-  Add async context managers to scheduler/worker classes (:pr:`2745`) `Matthew Rocklin`_
+-  Fix the resource key representation before sending graphs (:pr:`2733`) `Michael Spiegel`_
+-  Allow user to configure whether workers are daemon. (:pr:`2739`) `Caleb`_
+-  Pin pytest >=4 with pip in appveyor and python 3.5 (:pr:`2737`) `Matthew Rocklin`_
+-  Add Experimental UCX Comm (:pr:`2591`) `Ben Zaitlen`_ `Tom Augspurger`_ `Matthew Rocklin`_
+-  Close nannies gracefully (:pr:`2731`) `Matthew Rocklin`_
+-  add kwargs to progressbars (:pr:`2638`) `Manuel Garrido`_
+-  Add back LocalCluster.__repr__. (:pr:`2732`) `Loïc Estève`_
+-  Move bokeh module to dashboard (:pr:`2724`) `Matthew Rocklin`_
+-  Close clusters at exit (:pr:`2730`) `Matthew Rocklin`_
+-  Add SchedulerPlugin TaskState example (:pr:`2622`) `Matt Nicolls`_
+-  Add SpecificationCluster (:pr:`2675`) `Matthew Rocklin`_
+-  Replace register_worker_callbacks with worker plugins (:pr:`2453`) `Matthew Rocklin`_
+-  Proxy worker dashboards from scheduler dashboard (:pr:`2715`) `Ben Zaitlen`_
+-  Add docstring to Scheduler.check_idle_saturated (:pr:`2721`) `Matthew Rocklin`_
+-  Refer to LocalCluster in Client docstring (:pr:`2719`) `Matthew Rocklin`_
+-  Remove special casing of Scikit-Learn BaseEstimator serialization (:pr:`2713`) `Matthew Rocklin`_
+-  Fix two typos in Pub class docstring (:pr:`2714`) `Magnus Nord`_
+-  Support uploading files with multiple modules (:pr:`2587`) `Sam Grayson`_
+-  Change the main workers bokeh page to /status (:pr:`2689`) `Ben Zaitlen`_
+-  Cleanly stop periodic callbacks in Client (:pr:`2705`) `Matthew Rocklin`_
+-  Disable pan tool for the Progress, Byte Stored and Tasks Processing plot (:pr:`2703`) `Mathieu Dugré`_
+-  Except errors in Nanny's memory monitor if process no longer exists (:pr:`2701`) `Matthew Rocklin`_
+-  Handle heartbeat when worker has just left (:pr:`2702`) `Matthew Rocklin`_
+-  Modify styling of histograms for many-worker dashboard plots (:pr:`2695`) `Mathieu Dugré`_
+-  Add method to wait for n workers before continuing (:pr:`2688`) `Daniel Farrell`_
+-  Support computation on delayed(None) (:pr:`2697`)  `Matthew Rocklin`_
+-  Cleanup localcluster (:pr:`2693`)  `Matthew Rocklin`_
+-  Use 'temporary-directory' from dask.config for Worker's directory (:pr:`2654`) `Matthew Rocklin`_
+-  Remove support for Iterators and Queues (:pr:`2671`) `Matthew Rocklin`_
+
+
 1.28.1 - 2019-05-13
 -------------------
 
 This is a small bugfix release due to a config change upstream.
 
--  Use config accessor method for "scheduler-address" (#2676) `James Bourbeau`_
+-  Use config accessor method for "scheduler-address" (:pr:`2676`) `James Bourbeau`_
 
 
 1.28.0 - 2019-05-08
@@ -1039,3 +1093,10 @@ significantly without many new features.
 .. _`condoratberlin`: https://github.com/condoratberlin
 .. _`K.-Michael Aye`: https://github.com/michaelaye
 .. _`@plbertrand`: https://github.com/plbertrand
+.. _`Michael Spiegel`: https://github.com/Spiegel0
+.. _`Caleb`: https://github.com/calebho
+.. _`Ben Zaitlen`: https://github.com/quasiben
+.. _`Manuel Garrido`: https://github.com/manugarri
+.. _`Magnus Nord`: https://github.com/magnunor
+.. _`Sam Grayson`: https://github.com/charmoniumQ
+.. _`Mathieu Dugré`: https://github.com/mathdugre
diff --git a/requirements.txt b/requirements.txt
index a6c6b0f62f6..e376b2a50cc 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,6 @@
 click >= 6.6
 cloudpickle >= 0.2.2
-dask >= 0.18.0
+dask >= 2
 msgpack
 psutil >= 5.0
 six
@@ -9,7 +9,4 @@ tblib
 toolz >= 0.7.4
 tornado >= 5
 zict >= 0.1.3
-# Compatibility packages
-futures; python_version < '3.0'
-singledispatch; python_version < '3.4'
 pyyaml
diff --git a/setup.cfg b/setup.cfg
index 5533437121b..042a8b86f35 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -35,9 +35,6 @@ versionfile_build = distributed/_version.py
 tag_prefix =
 parentdir_prefix = distributed-
 
-[bdist_wheel]
-universal=1
-
 [tool:pytest]
 addopts = -rsx -v --durations=10
 minversion = 3.2
diff --git a/setup.py b/setup.py
index 0df22f3f911..6c4bce91d83 100755
--- a/setup.py
+++ b/setup.py
@@ -53,7 +53,6 @@
         "License :: OSI Approved :: BSD License",
         "Operating System :: OS Independent",
         "Programming Language :: Python",
-        "Programming Language :: Python :: 2.7",
         "Programming Language :: Python :: 3.5",
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",

From ded3f30ac2d161cede56ce58253e50e02b75b188 Mon Sep 17 00:00:00 2001
From: Brett Naul <brettnaul@gmail.com>
Date: Tue, 25 Jun 2019 14:55:20 -0700
Subject: [PATCH 0336/1550] Fix diagnostics page for memory_limit=None (#2770)

* Fix diagnostics page for memory_limit=None

* Apply black
---
 distributed/dashboard/templates/worker-table.html |  2 +-
 distributed/worker.py                             | 12 +++++++++---
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/distributed/dashboard/templates/worker-table.html b/distributed/dashboard/templates/worker-table.html
index a3566f90c3f..c12061fab46 100644
--- a/distributed/dashboard/templates/worker-table.html
+++ b/distributed/dashboard/templates/worker-table.html
@@ -16,7 +16,7 @@
         <td><a href="../worker/{{ url_escape(ws.address) }}.html">{{ws.address}}</a></td>
         <td> {{ ws.name if ws.name is not None else "" }} </td>
         <td> {{ ws.nthreads }} </td>
-        <td> {{ format_bytes(ws.memory_limit) }} </td>
+        <td> {{ format_bytes(ws.memory_limit) if ws.memory_limit is not None else "" }} </td>
         <td> <progress class="progress" value="{{ ws.metrics['memory'] }}" max="{{ ws.memory_limit }}"></progress> </td>
         <td> {{ format_time(ws.occupancy) }} </td>
         <td> {{ len(ws.processing) }} </td>
diff --git a/distributed/worker.py b/distributed/worker.py
index 63dfaed3114..bd4907372d5 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -2456,7 +2456,9 @@ def memory_monitor(self):
                     "Process memory: %s -- Worker memory limit: %s",
                     int(frac * 100),
                     format_bytes(proc.memory_info().rss),
-                    format_bytes(self.memory_limit),
+                    format_bytes(self.memory_limit)
+                    if self.memory_limit is not None
+                    else "None",
                 )
                 self.paused = True
         elif self.paused:
@@ -2465,7 +2467,9 @@ def memory_monitor(self):
                 "Process memory: %s -- Worker memory limit: %s",
                 int(frac * 100),
                 format_bytes(proc.memory_info().rss),
-                format_bytes(self.memory_limit),
+                format_bytes(self.memory_limit)
+                if self.memory_limit is not None
+                else "None",
             )
             self.paused = False
             self.ensure_computing()
@@ -2483,7 +2487,9 @@ def memory_monitor(self):
                         "is leaking memory?  Process memory: %s -- "
                         "Worker memory limit: %s",
                         format_bytes(proc.memory_info().rss),
-                        format_bytes(self.memory_limit),
+                        format_bytes(self.memory_limit)
+                        if self.memory_limit is not None
+                        else "None",
                     )
                     break
                 k, v, weight = self.data.fast.evict()

From 437c573627a89a11e0cc2e0fdd99209f586161f2 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 26 Jun 2019 10:21:53 +0200
Subject: [PATCH 0337/1550] Correctly manage tasks beyond deque limit in
 TaskStream plot (#2797)

Fixes #2501
---
 distributed/dashboard/scheduler.py            |  9 ++++--
 .../dashboard/tests/test_scheduler_bokeh.py   | 28 +++++++++++++++++++
 distributed/diagnostics/task_stream.py        | 16 +++++++++--
 distributed/distributed.yaml                  |  5 ++++
 4 files changed, 53 insertions(+), 5 deletions(-)

diff --git a/distributed/dashboard/scheduler.py b/distributed/dashboard/scheduler.py
index f6f1fef7590..bed13950fc0 100644
--- a/distributed/dashboard/scheduler.py
+++ b/distributed/dashboard/scheduler.py
@@ -35,6 +35,7 @@
 from bokeh.themes import Theme
 from bokeh.transform import factor_cmap
 from bokeh.io import curdoc
+import dask
 from toolz import pipe, merge
 from tornado import escape
 
@@ -1417,7 +1418,9 @@ def tasks_doc(scheduler, extra, doc):
     with log_errors():
         ts = TaskStream(
             scheduler,
-            n_rectangles=100000,
+            n_rectangles=dask.config.get(
+                "distributed.scheduler.dashboard.tasks.task-stream-length"
+            ),
             clear_interval="60s",
             sizing_mode="stretch_both",
         )
@@ -1447,7 +1450,9 @@ def status_doc(scheduler, extra, doc):
     with log_errors():
         task_stream = TaskStream(
             scheduler,
-            n_rectangles=1000,
+            n_rectangles=dask.config.get(
+                "distributed.scheduler.dashboard.status.task-stream-length"
+            ),
             clear_interval="10s",
             sizing_mode="stretch_both",
         )
diff --git a/distributed/dashboard/tests/test_scheduler_bokeh.py b/distributed/dashboard/tests/test_scheduler_bokeh.py
index d9a83caf00b..4d60f304876 100644
--- a/distributed/dashboard/tests/test_scheduler_bokeh.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh.py
@@ -627,3 +627,31 @@ def test_proxy_to_workers(c, s, a, b):
             assert b"pip install jupyter-server-proxy" in response_proxy.body
         assert response_direct.code == 200
         assert b"Crossfilter" in response_direct.body
+
+
+@gen_cluster(
+    client=True,
+    scheduler_kwargs={"services": {("dashboard", 0): BokehScheduler}},
+    config={
+        "distributed.scheduler.dashboard.tasks.task-stream-length": 10,
+        "distributed.scheduler.dashboard.status.task-stream-length": 10,
+    },
+)
+async def test_lots_of_tasks(c, s, a, b):
+    import toolz
+
+    ts = TaskStream(s)
+    ts.update()
+    futures = c.map(toolz.identity, range(100))
+    await wait(futures)
+
+    tsp = [p for p in s.plugins if "taskstream" in type(p).__name__.lower()][0]
+    assert len(tsp.buffer) == 10
+    ts.update()
+    assert len(ts.source.data["start"]) == 10
+    assert "identity" in str(ts.source.data)
+
+    futures = c.map(lambda x: x, range(100), pure=False)
+    await wait(futures)
+    ts.update()
+    assert "lambda" in str(ts.source.data)
diff --git a/distributed/diagnostics/task_stream.py b/distributed/diagnostics/task_stream.py
index 89cacb67c97..17e62c3045e 100644
--- a/distributed/diagnostics/task_stream.py
+++ b/distributed/diagnostics/task_stream.py
@@ -3,6 +3,7 @@
 from collections import deque
 import logging
 
+import dask
 from .progress_stream import color_of
 from .plugin import SchedulerPlugin
 from ..utils import key_split, format_time, parse_timedelta
@@ -13,7 +14,16 @@
 
 
 class TaskStreamPlugin(SchedulerPlugin):
-    def __init__(self, scheduler, maxlen=100000):
+    def __init__(self, scheduler, maxlen=None):
+        if maxlen is None:
+            maxlen = max(
+                dask.config.get(
+                    "distributed.scheduler.dashboard.status.task-stream-length"
+                ),
+                dask.config.get(
+                    "distributed.scheduler.dashboard.tasks.task-stream-length"
+                ),
+            )
         self.buffer = deque(maxlen=maxlen)
         self.scheduler = scheduler
         scheduler.add_plugin(self)
@@ -74,8 +84,8 @@ def rectangles(self, istart, istop=None, workers=None, start_boundary=0):
         msgs = []
         diff = self.index - len(self.buffer)
         if istop is None:
-            istop = len(self.buffer)
-        for i in range((istart or 0) - diff, istop - diff if istop else istop):
+            istop = self.index
+        for i in range(max(0, (istart or 0) - diff), istop - diff if istop else istop):
             msg = self.buffer[i]
             msgs.append(msg)
 
diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index 4d78a698e69..235c735946c 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -22,6 +22,11 @@ distributed:
     worker-ttl: null        # like '60s'. Time to live for workers.  They must heartbeat faster than this
     preload: []
     preload-argv: []
+    dashboard:
+      status:
+        task-stream-length: 1000
+      tasks:
+        task-stream-length: 100000
 
   worker:
     blocked-handlers: []

From 8990c98e593d90107bccd212a983aa4d3e5707cb Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 26 Jun 2019 15:23:14 +0200
Subject: [PATCH 0338/1550] Add python_requires entry to setup.py (#2807)

Alternative to #2806

Fixes #2804
---
 setup.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 6c4bce91d83..125ddc9c328 100755
--- a/setup.py
+++ b/setup.py
@@ -23,9 +23,10 @@
     version=versioneer.get_version(),
     cmdclass=versioneer.get_cmdclass(),
     description="Distributed scheduler for Dask",
-    url="https://distributed.readthedocs.io/en/latest/",
+    url="https://distributed.dask.org",
     maintainer="Matthew Rocklin",
     maintainer_email="mrocklin@gmail.com",
+    python_requires=">=3.5",
     license="BSD",
     package_data={
         "": ["templates/index.html", "template.html"],

From da6a01bdee1c6d90934c61ae056b14610cd56a6c Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 26 Jun 2019 15:25:43 +0200
Subject: [PATCH 0339/1550] bump version to 2.0.1

---
 docs/source/changelog.rst | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index 37a44591ee5..6162935b70d 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,18 @@
 Changelog
 =========
 
+2.0.1 - 2019-06-26
+------------------
+
+We neglected to include ``python_requires=`` in our setup.py file, resulting in
+confusion for Python 2 users who erroneously get packages for 2.0.0.
+This is fixed in 2.0.1 and we have removed the 2.0.0 files from PyPI.
+
+-  Add python_requires entry to setup.py (:pr:`2807`) `Matthew Rocklin`_
+-  Correctly manage tasks beyond deque limit in TaskStream plot (:pr:`2797`) `Matthew Rocklin`_
+-  Fix diagnostics page for memory_limit=None (:pr:`2770`) `Brett Naul`_
+
+
 2.0.0 - 2019-06-25
 ------------------
 

From f8af742fba80451b0db281eebe515951c53de9d4 Mon Sep 17 00:00:00 2001
From: Tom Augspurger <TomAugspurger@users.noreply.github.com>
Date: Thu, 27 Jun 2019 10:50:11 -0500
Subject: [PATCH 0340/1550] CLN: Use dask.utils.format_bytes (#2810)

---
 distributed/client.py                   |  3 +--
 distributed/dashboard/scheduler.py      |  3 ++-
 distributed/dashboard/scheduler_html.py |  3 ++-
 distributed/dashboard/worker.py         |  3 ++-
 distributed/deploy/cluster.py           | 10 ++------
 distributed/tests/test_utils.py         |  5 ++++
 distributed/tests/test_worker.py        |  3 ++-
 distributed/utils.py                    | 32 +++----------------------
 distributed/utils_perf.py               |  3 ++-
 distributed/worker.py                   |  2 +-
 10 files changed, 22 insertions(+), 45 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 7ad897bf616..a5e2b7f6103 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -28,7 +28,7 @@
 from dask.core import flatten, get_dependencies
 from dask.optimization import SubgraphCallable
 from dask.compatibility import apply, unicode
-from dask.utils import ensure_dict
+from dask.utils import ensure_dict, format_bytes
 
 try:
     from cytoolz import first, groupby, merge, valmap, keymap
@@ -82,7 +82,6 @@
     log_errors,
     str_graph,
     key_split,
-    format_bytes,
     asciitable,
     thread_state,
     no_default,
diff --git a/distributed/dashboard/scheduler.py b/distributed/dashboard/scheduler.py
index bed13950fc0..7cce430346a 100644
--- a/distributed/dashboard/scheduler.py
+++ b/distributed/dashboard/scheduler.py
@@ -36,6 +36,7 @@
 from bokeh.transform import factor_cmap
 from bokeh.io import curdoc
 import dask
+from dask.utils import format_bytes
 from toolz import pipe, merge
 from tornado import escape
 
@@ -55,7 +56,7 @@
 from .worker import SystemMonitor, counters_doc
 from .utils import transpose, BOKEH_VERSION, without_property_validation
 from ..metrics import time
-from ..utils import log_errors, format_bytes, format_time
+from ..utils import log_errors, format_time
 from ..diagnostics.progress_stream import color_of, progress_quads, nbytes_bar
 from ..diagnostics.progress import AllProgress
 from ..diagnostics.graph_layout import GraphLayout
diff --git a/distributed/dashboard/scheduler_html.py b/distributed/dashboard/scheduler_html.py
index 65a89b33fbb..08829241d47 100644
--- a/distributed/dashboard/scheduler_html.py
+++ b/distributed/dashboard/scheduler_html.py
@@ -1,10 +1,11 @@
 from datetime import datetime
 
+from dask.utils import format_bytes
 import toolz
 from tornado import escape
 from tornado import gen
 
-from ..utils import log_errors, format_bytes, format_time
+from ..utils import log_errors, format_time
 from .proxy import GlobalProxyHandler
 from .utils import RequestHandler, redirect
 
diff --git a/distributed/dashboard/worker.py b/distributed/dashboard/worker.py
index c6633a170aa..d8f8adc1c7d 100644
--- a/distributed/dashboard/worker.py
+++ b/distributed/dashboard/worker.py
@@ -22,6 +22,7 @@
 from bokeh.plotting import figure
 from bokeh.palettes import RdBu
 from bokeh.themes import Theme
+from dask.utils import format_bytes
 from toolz import merge, partition_all
 
 from .components import (
@@ -35,7 +36,7 @@
 from ..compatibility import WINDOWS
 from ..diagnostics.progress_stream import color_of
 from ..metrics import time
-from ..utils import log_errors, key_split, format_bytes, format_time
+from ..utils import log_errors, key_split, format_time
 
 
 logger = logging.getLogger(__name__)
diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
index 74d61a995e1..866910784e4 100644
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -4,19 +4,13 @@
 from weakref import ref
 
 import dask
+from dask.utils import format_bytes
 from tornado import gen
 
 from .adaptive import Adaptive
 
 from ..compatibility import get_thread_identity
-from ..utils import (
-    format_bytes,
-    PeriodicCallback,
-    log_errors,
-    ignoring,
-    sync,
-    thread_state,
-)
+from ..utils import PeriodicCallback, log_errors, ignoring, sync, thread_state
 
 
 logger = logging.getLogger(__name__)
diff --git a/distributed/tests/test_utils.py b/distributed/tests/test_utils.py
index b82dce4e7d9..df98bbe59e1 100644
--- a/distributed/tests/test_utils.py
+++ b/distributed/tests/test_utils.py
@@ -543,3 +543,8 @@ def test_warn_on_duration():
 
     assert record
     assert any("foo" in str(rec.message) for rec in record)
+
+
+def test_format_bytes_compat():
+    # moved to dask, but exported here for compatibility
+    from distributed.utils import format_bytes  # noqa
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 0f462d7de05..562a0e037b7 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -15,6 +15,7 @@
 
 import dask
 from dask import delayed
+from dask.utils import format_bytes
 import pytest
 from toolz import pluck, sliding_window, first
 import tornado
@@ -28,7 +29,7 @@
 from distributed.scheduler import Scheduler
 from distributed.metrics import time
 from distributed.worker import Worker, error_message, logger, parse_memory_limit
-from distributed.utils import tmpfile, format_bytes
+from distributed.utils import tmpfile
 from distributed.utils_test import (
     inc,
     mul,
diff --git a/distributed/utils.py b/distributed/utils.py
index e8de0bc5108..46982fd6324 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -36,6 +36,9 @@
 
 import dask
 from dask import istask
+
+# provide format_bytes here for backwards compatibility
+from dask.utils import format_bytes  # noqa
 import toolz
 import tornado
 from tornado import gen
@@ -1112,35 +1115,6 @@ def __reduce__(self):
         return (itemgetter, (self.index,))
 
 
-def format_bytes(n):
-    """ Format bytes as text
-
-    >>> format_bytes(1)
-    '1 B'
-    >>> format_bytes(1234)
-    '1.23 kB'
-    >>> format_bytes(12345678)
-    '12.35 MB'
-    >>> format_bytes(1234567890)
-    '1.23 GB'
-    >>> format_bytes(1234567890000)
-    '1.23 TB'
-    >>> format_bytes(1234567890000000)
-    '1.23 PB'
-    """
-    if n > 1e15:
-        return "%0.2f PB" % (n / 1e15)
-    if n > 1e12:
-        return "%0.2f TB" % (n / 1e12)
-    if n > 1e9:
-        return "%0.2f GB" % (n / 1e9)
-    if n > 1e6:
-        return "%0.2f MB" % (n / 1e6)
-    if n > 1e3:
-        return "%0.2f kB" % (n / 1000)
-    return "%d B" % n
-
-
 byte_sizes = {
     "kB": 10 ** 3,
     "MB": 10 ** 6,
diff --git a/distributed/utils_perf.py b/distributed/utils_perf.py
index 9f300c5f567..b1f65256c1e 100644
--- a/distributed/utils_perf.py
+++ b/distributed/utils_perf.py
@@ -5,9 +5,10 @@
 import logging
 import threading
 
+from dask.utils import format_bytes
+
 from .compatibility import PY2, PYPY
 from .metrics import thread_time
-from .utils import format_bytes
 
 
 logger = _logger = logging.getLogger(__name__)
diff --git a/distributed/worker.py b/distributed/worker.py
index bd4907372d5..582564e3f6d 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -19,6 +19,7 @@
 import dask
 from dask.core import istask
 from dask.compatibility import apply
+from dask.utils import format_bytes
 
 try:
     from cytoolz import pluck, partial, merge, first
@@ -59,7 +60,6 @@
     thread_state,
     json_load_robust,
     key_split,
-    format_bytes,
     PeriodicCallback,
     parse_bytes,
     parse_timedelta,

From bf65f7afccc952fa0d16d2974323e34a438934e1 Mon Sep 17 00:00:00 2001
From: Jim Crist <jcrist@users.noreply.github.com>
Date: Sat, 29 Jun 2019 10:06:27 -0500
Subject: [PATCH 0341/1550] Add HTTPS support for the dashboard (#2812)

Adds optional HTTPS support for the scheduler dashboard. This is only
available via the configuration file, by setting the following fields:

- `distributed.scheduler.tls.cert`: the certificate file
- `distributed.scheduler.tls.key`: the key file, optional if the key
  file is concatenated with the cert above
- `distributed.scheduler.tls.ca-file`: the CA file, optional

These certs *may* be the same as those used for the
scheduler/worker/client communication, but aren't required to be. The
user is responsible for making this decision and providing the proper
configuration.

Likewise, the user is responsible for providing trusted certificates, or
understanding the security implications of telling their browser "I
understand the risks, trust this certificate" (this is more likely,
given the transient nature of dask clusters).

The generated dashboard links now format on an optional `scheme`
parameter, which is either `http` or `https`, depending on if the TLS
configuration fields above are configured.
---
 distributed/client.py                         |  4 +-
 distributed/dashboard/scheduler.py            | 19 +++++++
 .../dashboard/tests/test_scheduler_bokeh.py   | 52 +++++++++++++++++--
 distributed/deploy/cluster.py                 | 14 +++--
 distributed/distributed.yaml                  |  7 ++-
 distributed/tests/test_client.py              |  9 ++--
 distributed/utils.py                          |  9 ++++
 7 files changed, 99 insertions(+), 15 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index a5e2b7f6103..ec564694c49 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -91,6 +91,7 @@
     shutting_down,
     Any,
     has_keyword,
+    format_dashboard_link,
 )
 from .versions import get_versions
 
@@ -818,8 +819,7 @@ def _repr_html_(self):
                 host = "localhost"
             else:
                 host = rest.split(":")[0]
-            template = dask.config.get("distributed.dashboard.link")
-            address = template.format(host=host, port=port, **os.environ)
+            address = format_dashboard_link(host, port)
             text += (
                 "  <li><b>Dashboard: </b><a href='%(web)s' target='_blank'>%(web)s</a>\n"
                 % {"web": address}
diff --git a/distributed/dashboard/scheduler.py b/distributed/dashboard/scheduler.py
index 7cce430346a..013edb39ace 100644
--- a/distributed/dashboard/scheduler.py
+++ b/distributed/dashboard/scheduler.py
@@ -1597,6 +1597,25 @@ def __init__(self, scheduler, io_loop=None, prefix="", **kwargs):
 
         self.server_kwargs = kwargs
 
+        # TLS configuration
+        http_server_kwargs = kwargs.setdefault("http_server_kwargs", {})
+        tls_key = dask.config.get("distributed.scheduler.dashboard.tls.key")
+        tls_cert = dask.config.get("distributed.scheduler.dashboard.tls.cert")
+        tls_ca_file = dask.config.get("distributed.scheduler.dashboard.tls.ca-file")
+        if tls_cert and "ssl_options" not in http_server_kwargs:
+            import ssl
+
+            ctx = ssl.create_default_context(
+                cafile=tls_ca_file, purpose=ssl.Purpose.SERVER_AUTH
+            )
+            ctx.load_cert_chain(tls_cert, keyfile=tls_key)
+            # Unlike the client/scheduler/worker TLS handling, we don't care
+            # about authenticating the user's webclient, TLS here is just for
+            # encryption. Disable these checks.
+            ctx.check_hostname = False
+            ctx.verify_mode = ssl.CERT_NONE
+            http_server_kwargs["ssl_options"] = ctx
+
         self.server_kwargs["prefix"] = prefix or None
 
         self.apps = {
diff --git a/distributed/dashboard/tests/test_scheduler_bokeh.py b/distributed/dashboard/tests/test_scheduler_bokeh.py
index 4d60f304876..3c7f85dc89a 100644
--- a/distributed/dashboard/tests/test_scheduler_bokeh.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh.py
@@ -2,6 +2,7 @@
 
 import json
 import re
+import ssl
 import sys
 from time import sleep
 
@@ -10,13 +11,13 @@
 pytest.importorskip("bokeh")
 from toolz import first
 from tornado import gen
-from tornado.httpclient import AsyncHTTPClient
+from tornado.httpclient import AsyncHTTPClient, HTTPRequest
 
 from dask.core import flatten
-from distributed.utils import tokey
+from distributed.utils import tokey, format_dashboard_link
 from distributed.client import wait
 from distributed.metrics import time
-from distributed.utils_test import gen_cluster, inc, dec, slowinc, div
+from distributed.utils_test import gen_cluster, inc, dec, slowinc, div, get_cert
 from distributed.dashboard.worker import Counters, BokehWorker
 from distributed.dashboard.scheduler import (
     BokehScheduler,
@@ -655,3 +656,48 @@ async def test_lots_of_tasks(c, s, a, b):
     await wait(futures)
     ts.update()
     assert "lambda" in str(ts.source.data)
+
+
+@gen_cluster(
+    client=True,
+    scheduler_kwargs={"services": {("dashboard", 0): BokehScheduler}},
+    config={
+        "distributed.scheduler.dashboard.tls.key": get_cert("tls-key.pem"),
+        "distributed.scheduler.dashboard.tls.cert": get_cert("tls-cert.pem"),
+        "distributed.scheduler.dashboard.tls.ca-file": get_cert("tls-ca-cert.pem"),
+    },
+)
+def test_https_support(c, s, a, b):
+    assert isinstance(s.services["dashboard"], BokehScheduler)
+    port = s.services["dashboard"].port
+
+    assert (
+        format_dashboard_link("localhost", port) == "https://localhost:%d/status" % port
+    )
+
+    ctx = ssl.create_default_context()
+    ctx.load_verify_locations(get_cert("tls-ca-cert.pem"))
+
+    http_client = AsyncHTTPClient()
+    for suffix in [
+        "system",
+        "counters",
+        "workers",
+        "status",
+        "tasks",
+        "stealing",
+        "graph",
+        "individual-task-stream",
+        "individual-progress",
+        "individual-graph",
+        "individual-nbytes",
+        "individual-nprocessing",
+        "individual-profile",
+    ]:
+        req = HTTPRequest(
+            url="https://localhost:%d/%s" % (port, suffix), ssl_options=ctx
+        )
+        response = yield http_client.fetch(req)
+        body = response.body.decode()
+        assert "bokeh" in body.lower()
+        assert not re.search("href=./", body)  # no absolute links
diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
index 866910784e4..9819c1ad017 100644
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -1,16 +1,21 @@
 from datetime import timedelta
 import logging
-import os
 from weakref import ref
 
-import dask
 from dask.utils import format_bytes
 from tornado import gen
 
 from .adaptive import Adaptive
 
 from ..compatibility import get_thread_identity
-from ..utils import PeriodicCallback, log_errors, ignoring, sync, thread_state
+from ..utils import (
+    PeriodicCallback,
+    log_errors,
+    ignoring,
+    sync,
+    thread_state,
+    format_dashboard_link,
+)
 
 
 logger = logging.getLogger(__name__)
@@ -80,10 +85,9 @@ def scheduler_address(self):
 
     @property
     def dashboard_link(self):
-        template = dask.config.get("distributed.dashboard.link")
         host = self.scheduler.address.split("://")[1].split(":")[0]
         port = self.scheduler.services["dashboard"].port
-        return template.format(host=host, port=port, **os.environ)
+        return format_dashboard_link(host, port)
 
     def scale(self, n):
         """ Scale cluster to n workers
diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index 235c735946c..e5bd3dd3140 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -27,6 +27,11 @@ distributed:
         task-stream-length: 1000
       tasks:
         task-stream-length: 100000
+      tls:
+        ca-file: null
+        key: null
+        cert: null
+
 
   worker:
     blocked-handlers: []
@@ -88,7 +93,7 @@ distributed:
   ###################
 
   dashboard:
-    link: "http://{host}:{port}/status"
+    link: "{scheme}://{host}:{port}/status"
     export-tool: False
 
   ##################
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index dca108b57ee..37d5550941f 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -5224,20 +5224,21 @@ def test_quiet_scheduler_loss(c, s):
     assert "BrokenPipeError" not in text
 
 
-@pytest.mark.skipif("USER" not in os.environ, reason="no USER env variable")
-def test_diagnostics_link_env_variable(loop):
+def test_dashboard_link(loop, monkeypatch):
     pytest.importorskip("bokeh")
     from distributed.dashboard import BokehScheduler
 
+    monkeypatch.setenv("USER", "myusername")
+
     with cluster(
         scheduler_kwargs={"services": {("dashboard", 12355): BokehScheduler}}
     ) as (s, [a, b]):
         with Client(s["address"], loop=loop) as c:
             with dask.config.set(
-                {"distributed.dashboard.link": "http://foo-{USER}:{port}/status"}
+                {"distributed.dashboard.link": "{scheme}://foo-{USER}:{port}/status"}
             ):
                 text = c._repr_html_()
-                link = "http://foo-" + os.environ["USER"] + ":12355/status"
+                link = "http://foo-myusername:12355/status"
                 assert link in text
 
 
diff --git a/distributed/utils.py b/distributed/utils.py
index 46982fd6324..afe43040bd9 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -1483,3 +1483,12 @@ def typename(typ):
         return typ.__module__ + "." + typ.__name__
     except AttributeError:
         return str(typ)
+
+
+def format_dashboard_link(host, port):
+    template = dask.config.get("distributed.dashboard.link")
+    if dask.config.get("distributed.scheduler.dashboard.tls.cert"):
+        scheme = "https"
+    else:
+        scheme = "http"
+    return template.format(scheme=scheme, host=host, port=port, **os.environ)

From fdc94d113c7fd2e29b530d9c277e76f2bf06a8d9 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sun, 30 Jun 2019 10:56:22 +0100
Subject: [PATCH 0342/1550] Relax check for worker references in cluster
 context manager (#2813)

---
 distributed/client.py     | 5 +++--
 distributed/utils_test.py | 7 ++++---
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index ec564694c49..3c15e68fe99 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -3278,9 +3278,10 @@ def _profile(
             if plot == "save" and not filename:
                 filename = "dask-profile.html"
 
-            from bokeh.plotting import save
+            if filename:
+                from bokeh.plotting import save
 
-            save(figure, title="Dask Profile", filename=filename)
+                save(figure, title="Dask Profile", filename=filename)
             raise gen.Return((state, figure))
 
         else:
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index 293cf5c0737..77568bb7595 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -732,9 +732,10 @@ def cluster(
             client.close()
 
     start = time()
-    while len(ws):
-        sleep(0.1)
-        assert time() < start + 3, ("Workers still around after two seconds", list(ws))
+    while any(proc.is_alive() for proc in ws):
+        text = str(list(ws))
+        sleep(0.2)
+        assert time() < start + 5, ("Workers still around after five seconds", text)
 
 
 @gen.coroutine

From 9aa0ea60c6ba0fa8dd65b2a8005d3dbf3d65db75 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sun, 30 Jun 2019 21:00:24 +0100
Subject: [PATCH 0343/1550] Use Keyword-only arguments (#2814)

Previously for functions with both `*args` and `**kwargs` inputs we
often explicitly popped off values explicitly.  Now that we no longer
support Python 2 we can use these as keyword arguments directly.

* Fix pytest 5.0 issues
---
 distributed/client.py                  | 90 +++++++++++++++-----------
 distributed/deploy/cluster.py          |  6 +-
 distributed/diagnostics/progressbar.py |  6 +-
 distributed/recreate_exceptions.py     |  4 +-
 distributed/tests/test_client.py       | 46 +++++--------
 distributed/utils.py                   | 17 ++---
 distributed/worker.py                  | 10 +--
 7 files changed, 85 insertions(+), 94 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 3c15e68fe99..8509d9e31a9 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -748,20 +748,20 @@ def asynchronous(self):
         """
         return self._asynchronous and self.loop is IOLoop.current()
 
-    def sync(self, func, *args, **kwargs):
-        asynchronous = kwargs.pop("asynchronous", None)
+    def sync(self, func, *args, asynchronous=None, callback_timeout=None, **kwargs):
         if (
             asynchronous
             or self.asynchronous
             or getattr(thread_state, "asynchronous", False)
         ):
-            callback_timeout = kwargs.pop("callback_timeout", None)
             future = func(*args, **kwargs)
             if callback_timeout is not None:
                 future = gen.with_timeout(timedelta(seconds=callback_timeout), future)
             return future
         else:
-            return sync(self.loop, func, *args, **kwargs)
+            return sync(
+                self.loop, func, *args, callback_timeout=callback_timeout, **kwargs
+            )
 
     def __repr__(self):
         # Note: avoid doing I/O here...
@@ -1351,7 +1351,23 @@ def get_executor(self, **kwargs):
         """
         return ClientExecutor(self, **kwargs)
 
-    def submit(self, func, *args, **kwargs):
+    def submit(
+        self,
+        func,
+        *args,
+        key=None,
+        workers=None,
+        resources=None,
+        retries=None,
+        priority=0,
+        fifo_timeout="100 ms",
+        allow_other_workers=False,
+        actor=False,
+        actors=False,
+        pure=None,
+        **kwargs
+    ):
+
         """ Submit a function application to the scheduler
 
         Parameters
@@ -1393,15 +1409,9 @@ def submit(self, func, *args, **kwargs):
         if not callable(func):
             raise TypeError("First input to submit must be a callable function")
 
-        key = kwargs.pop("key", None)
-        workers = kwargs.pop("workers", None)
-        resources = kwargs.pop("resources", None)
-        retries = kwargs.pop("retries", None)
-        priority = kwargs.pop("priority", 0)
-        fifo_timeout = kwargs.pop("fifo_timeout", "100ms")
-        allow_other_workers = kwargs.pop("allow_other_workers", False)
-        actor = kwargs.pop("actor", kwargs.pop("actors", False))
-        pure = kwargs.pop("pure", not actor)
+        actor = actor or actors
+        if pure is None:
+            pure = not actor
 
         if allow_other_workers not in (True, False, None):
             raise TypeError("allow_other_workers= must be True or False")
@@ -1452,7 +1462,22 @@ def submit(self, func, *args, **kwargs):
 
         return futures[skey]
 
-    def map(self, func, *iterables, **kwargs):
+    def map(
+        self,
+        func,
+        *iterables,
+        key=None,
+        workers=None,
+        retries=None,
+        resources=None,
+        priority=0,
+        allow_other_workers=False,
+        fifo_timeout="100 ms",
+        actor=False,
+        actors=False,
+        pure=None,
+        **kwargs
+    ):
         """ Map a function on a sequence of arguments
 
         Arguments can be normal objects or Futures
@@ -1494,6 +1519,11 @@ def map(self, func, *iterables, **kwargs):
         --------
         Client.submit: Submit a single function
         """
+        key = key or funcname(func)
+        actor = actor or actors
+        if pure is None:
+            pure = not actor
+
         if not callable(func):
             raise TypeError("First input to map must be a callable function")
 
@@ -1505,17 +1535,6 @@ def map(self, func, *iterables, **kwargs):
                 "Consider using a normal for loop and Client.submit"
             )
 
-        key = kwargs.pop("key", None)
-        key = key or funcname(func)
-        workers = kwargs.pop("workers", None)
-        retries = kwargs.pop("retries", None)
-        resources = kwargs.pop("resources", None)
-        user_priority = kwargs.pop("priority", 0)
-        allow_other_workers = kwargs.pop("allow_other_workers", False)
-        fifo_timeout = kwargs.pop("fifo_timeout", "100ms")
-        actor = kwargs.pop("actor", kwargs.pop("actors", False))
-        pure = kwargs.pop("pure", not actor)
-
         if allow_other_workers and workers is None:
             raise ValueError("Only use allow_other_workers= if using workers=")
 
@@ -1581,7 +1600,7 @@ def map(self, func, *iterables, **kwargs):
         else:
             loose_restrictions = set()
 
-        priority = dict(zip(keys, range(len(keys))))
+        internal_priority = dict(zip(keys, range(len(keys))))
 
         if resources:
             resources = {k: resources for k in keys}
@@ -1593,10 +1612,10 @@ def map(self, func, *iterables, **kwargs):
             keys,
             restrictions,
             loose_restrictions,
-            priority=priority,
+            priority=internal_priority,
             resources=resources,
             retries=retries,
-            user_priority=user_priority,
+            user_priority=priority,
             fifo_timeout=fifo_timeout,
             actors=actor,
         )
@@ -2051,7 +2070,7 @@ def retry(self, futures, asynchronous=None):
         return self.sync(self._retry, futures, asynchronous=asynchronous)
 
     @gen.coroutine
-    def _publish_dataset(self, *args, **kwargs):
+    def _publish_dataset(self, *args, name=None, **kwargs):
         with log_errors():
             coroutines = []
 
@@ -2063,7 +2082,6 @@ def add_coro(name, data):
                     )
                 )
 
-            name = kwargs.pop("name", None)
             if name:
                 if len(args) == 0:
                     raise ValueError(
@@ -2179,8 +2197,7 @@ def get_dataset(self, name, **kwargs):
         return self.sync(self._get_dataset, name, **kwargs)
 
     @gen.coroutine
-    def _run_on_scheduler(self, function, *args, **kwargs):
-        wait = kwargs.pop("wait", True)
+    def _run_on_scheduler(self, function, *args, wait=True, **kwargs):
         response = yield self.scheduler.run_function(
             function=dumps(function), args=dumps(args), kwargs=dumps(kwargs), wait=wait
         )
@@ -2222,10 +2239,7 @@ def run_on_scheduler(self, function, *args, **kwargs):
         return self.sync(self._run_on_scheduler, function, *args, **kwargs)
 
     @gen.coroutine
-    def _run(self, function, *args, **kwargs):
-        nanny = kwargs.pop("nanny", False)
-        workers = kwargs.pop("workers", None)
-        wait = kwargs.pop("wait", True)
+    def _run(self, function, *args, nanny=False, workers=None, wait=True, **kwargs):
         responses = yield self.scheduler.broadcast(
             msg=dict(
                 op="run",
@@ -2582,6 +2596,7 @@ def compute(
         priority=0,
         fifo_timeout="60s",
         actors=None,
+        traverse=True,
         **kwargs
     ):
         """ Compute dask collections on cluster
@@ -2645,7 +2660,6 @@ def compute(
             collections = [collections]
             singleton = True
 
-        traverse = kwargs.pop("traverse", True)
         if traverse:
             collections = tuple(
                 dask.delayed(a)
diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
index 9819c1ad017..d48f27603ff 100644
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -234,9 +234,9 @@ def asynchronous(self):
             and self.loop._thread_identity == get_thread_identity()
         )
 
-    def sync(self, func, *args, **kwargs):
-        if kwargs.pop("asynchronous", None) or self.asynchronous:
-            callback_timeout = kwargs.pop("callback_timeout", None)
+    def sync(self, func, *args, asynchronous=None, callback_timeout=None, **kwargs):
+        asynchronous = asynchronous or self.asynchronous
+        if asynchronous:
             future = func(*args, **kwargs)
             if callback_timeout is not None:
                 future = gen.with_timeout(timedelta(seconds=callback_timeout), future)
diff --git a/distributed/diagnostics/progressbar.py b/distributed/diagnostics/progressbar.py
index 8a381562f27..f25bf32a871 100644
--- a/distributed/diagnostics/progressbar.py
+++ b/distributed/diagnostics/progressbar.py
@@ -405,7 +405,7 @@ def _draw_bar(self, remaining, all, status, **kwargs):
             )
 
 
-def progress(*futures, **kwargs):
+def progress(*futures, notebook=None, multi=True, complete=True, **kwargs):
     """ Track progress of futures
 
     This operates differently in the notebook and the console
@@ -436,10 +436,6 @@ def progress(*futures, **kwargs):
     >>> progress(futures)  # doctest: +SKIP
     [########################################] | 100% Completed |  1.7s
     """
-    notebook = kwargs.pop("notebook", None)
-    multi = kwargs.pop("multi", True)
-    complete = kwargs.pop("complete", True)
-
     futures = futures_of(futures)
     if not isinstance(futures, (set, list)):
         futures = [futures]
diff --git a/distributed/recreate_exceptions.py b/distributed/recreate_exceptions.py
index d5351bb4d59..78b0f4de9ba 100644
--- a/distributed/recreate_exceptions.py
+++ b/distributed/recreate_exceptions.py
@@ -23,7 +23,7 @@ def __init__(self, scheduler):
         self.scheduler.handlers["cause_of_failure"] = self.cause_of_failure
         self.scheduler.extensions["exceptions"] = self
 
-    def cause_of_failure(self, *args, **kwargs):
+    def cause_of_failure(self, *args, keys=(), **kwargs):
         """
         Return details of first failed task required by set of keys
 
@@ -38,8 +38,6 @@ def cause_of_failure(self, *args, **kwargs):
         task: the definition of that key
         deps: keys that the task depends on
         """
-
-        keys = kwargs.pop("keys", [])
         for key in keys:
             if isinstance(key, list):
                 key = tuple(key)  # ensure not a list from msgpack
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 37d5550941f..8fa14269a1f 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -197,18 +197,15 @@ def test_map_retries(c, s, a, b):
     x, y, z = c.map(*map_varying(args), retries=1, pure=False)
     assert (yield x) == 2
     assert (yield y) == 4
-    with pytest.raises(ZeroDivisionError) as exc_info:
+    with pytest.raises(ZeroDivisionError, match="eight"):
         yield z
-    exc_info.match("eight")
 
     x, y, z = c.map(*map_varying(args), retries=0, pure=False)
-    with pytest.raises(ZeroDivisionError) as exc_info:
+    with pytest.raises(ZeroDivisionError, match="one"):
         yield x
-    exc_info.match("one")
     assert (yield y) == 4
-    with pytest.raises(ZeroDivisionError) as exc_info:
+    with pytest.raises(ZeroDivisionError, match="seven"):
         yield z
-    exc_info.match("seven")
 
 
 @gen_cluster(client=True)
@@ -217,15 +214,13 @@ def test_compute_retries(c, s, a, b):
 
     # Sanity check for varying() use
     x = c.compute(delayed(varying(args))())
-    with pytest.raises(ZeroDivisionError) as exc_info:
+    with pytest.raises(ZeroDivisionError, match="one"):
         yield x
-    exc_info.match("one")
 
     # Same retries for all
     x = c.compute(delayed(varying(args))(), retries=1)
-    with pytest.raises(ZeroDivisionError) as exc_info:
+    with pytest.raises(ZeroDivisionError, match="two"):
         yield x
-    exc_info.match("two")
 
     x = c.compute(delayed(varying(args))(), retries=2)
     assert (yield x) == 3
@@ -244,16 +239,14 @@ def test_compute_retries(c, s, a, b):
     gc.collect()
 
     assert (yield x) == 30
-    with pytest.raises(ZeroDivisionError) as exc_info:
+    with pytest.raises(ZeroDivisionError, match="five"):
         yield y
-    exc_info.match("five")
 
     x, y, z = [delayed(varying(args))() for args in (xargs, yargs, zargs)]
     x, y, z = c.compute([x, y, z], retries={(y, z): 2})
 
-    with pytest.raises(ZeroDivisionError) as exc_info:
+    with pytest.raises(ZeroDivisionError, match="one"):
         yield x
-    exc_info.match("one")
     assert (yield y) == 70
     assert (yield z) == 80
 
@@ -276,15 +269,13 @@ def test_compute_persisted_retries(c, s, a, b):
     # Sanity check
     x = c.persist(delayed(varying(args))())
     fut = c.compute(x)
-    with pytest.raises(ZeroDivisionError) as exc_info:
+    with pytest.raises(ZeroDivisionError, match="one"):
         yield fut
-    exc_info.match("one")
 
     x = c.persist(delayed(varying(args))())
     fut = c.compute(x, retries=1)
-    with pytest.raises(ZeroDivisionError) as exc_info:
+    with pytest.raises(ZeroDivisionError, match="two"):
         yield fut
-    exc_info.match("two")
 
     x = c.persist(delayed(varying(args))())
     fut = c.compute(x, retries=2)
@@ -303,9 +294,8 @@ def test_persist_retries(c, s, a, b):
 
     x = c.persist(delayed(varying(args))(), retries=1)
     x = c.compute(x)
-    with pytest.raises(ZeroDivisionError) as exc_info:
+    with pytest.raises(ZeroDivisionError, match="two"):
         yield x
-    exc_info.match("two")
 
     x = c.persist(delayed(varying(args))(), retries=2)
     x = c.compute(x)
@@ -320,9 +310,8 @@ def test_persist_retries(c, s, a, b):
     x, y, z = c.persist([x, y, z], retries={(y, z): 2})
     x, y, z = c.compute([x, y, z])
 
-    with pytest.raises(ZeroDivisionError) as exc_info:
+    with pytest.raises(ZeroDivisionError, match="one"):
         yield x
-    exc_info.match("one")
     assert (yield y) == 70
     assert (yield z) == 80
 
@@ -2575,9 +2564,8 @@ def test_run_coroutine(c, s, a, b):
     results = yield c.run(geninc, 1, workers=[])
     assert results == {}
 
-    with pytest.raises(RuntimeError) as exc_info:
+    with pytest.raises(RuntimeError, match="hello"):
         yield c.run(throws, 1)
-    assert "hello" in str(exc_info)
 
     if sys.version_info >= (3, 5):
         results = yield c.run(asyncinc, 2, delay=0.01)
@@ -2603,9 +2591,8 @@ def raise_exception(exc_type, exc_msg):
         raise exc_type(exc_msg)
 
     for exc_type in [ValueError, RuntimeError]:
-        with pytest.raises(exc_type) as excinfo:
+        with pytest.raises(exc_type, match="informative message"):
             c.run(raise_exception, exc_type, "informative message")
-        assert "informative message" in str(excinfo.value)
 
 
 def test_diagnostic_ui(loop):
@@ -4420,16 +4407,15 @@ def test_recreate_error_sync(c):
     tot = c.submit(sum, x, y)
     f = c.compute(tot)
 
-    with pytest.raises(ZeroDivisionError) as e:
+    with pytest.raises(ZeroDivisionError):
         c.recreate_error_locally(f)
     assert f.status == "error"
 
 
 def test_recreate_error_not_error(c):
     f = c.submit(dec, 2)
-    with pytest.raises(ValueError) as e:
+    with pytest.raises(ValueError, match="No errored futures passed"):
         c.recreate_error_locally(f)
-    assert "No errored futures passed" in str(e)
 
 
 @gen_cluster(client=True)
@@ -4497,7 +4483,7 @@ def __call__(self, *args):
             return 1
 
     future = c.submit(Foo(), 1)
-    with pytest.raises(MyException) as e:
+    with pytest.raises(MyException):
         yield future
 
     futures = c.map(inc, range(10))
diff --git a/distributed/utils.py b/distributed/utils.py
index afe43040bd9..2f4657439cf 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -283,7 +283,7 @@ def quiet():
     raise gen.Return(results)
 
 
-def sync(loop, func, *args, **kwargs):
+def sync(loop, func, *args, callback_timeout=None, **kwargs):
     """
     Run coroutine in loop running in separate thread.
     """
@@ -299,8 +299,6 @@ def sync(loop, func, *args, **kwargs):
     except AttributeError:
         pass
 
-    timeout = kwargs.pop("callback_timeout", None)
-
     e = threading.Event()
     main_tid = get_thread_identity()
     result = [None]
@@ -314,8 +312,8 @@ def f():
             yield gen.moment
             thread_state.asynchronous = True
             future = func(*args, **kwargs)
-            if timeout is not None:
-                future = gen.with_timeout(timedelta(seconds=timeout), future)
+            if callback_timeout is not None:
+                future = gen.with_timeout(timedelta(seconds=callback_timeout), future)
             result[0] = yield future
         except Exception as exc:
             error[0] = sys.exc_info()
@@ -324,9 +322,9 @@ def f():
             e.set()
 
     loop.add_callback(f)
-    if timeout is not None:
-        if not e.wait(timeout):
-            raise gen.TimeoutError("timed out after %s s." % (timeout,))
+    if callback_timeout is not None:
+        if not e.wait(callback_timeout):
+            raise gen.TimeoutError("timed out after %s s." % (callback_timeout,))
     else:
         while not e.is_set():
             e.wait(10)
@@ -1352,8 +1350,7 @@ class DequeHandler(logging.Handler):
 
     _instances = weakref.WeakSet()
 
-    def __init__(self, *args, **kwargs):
-        n = kwargs.pop("n", 10000)
+    def __init__(self, *args, n=10000, **kwargs):
         self.deque = deque(maxlen=n)
         super(DequeHandler, self).__init__(*args, **kwargs)
         self._instances.add(self)
diff --git a/distributed/worker.py b/distributed/worker.py
index 582564e3f6d..4ce564f1b7b 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -308,6 +308,8 @@ def __init__(
         nanny=None,
         plugins=(),
         low_level_profiler=dask.config.get("distributed.worker.profile.low-level"),
+        validate=False,
+        profile_cycle_interval=None,
         **kwargs
     ):
         self.tasks = dict()
@@ -369,7 +371,7 @@ def __init__(
         self.target_message_size = 50e6  # 50 MB
 
         self.log = deque(maxlen=100000)
-        self.validate = kwargs.pop("validate", False)
+        self.validate = validate
 
         self._transitions = {
             ("waiting", "ready"): self.transition_waiting_ready,
@@ -404,10 +406,8 @@ def __init__(
         self.latency = 0.001
         self._client = None
 
-        profile_cycle_interval = kwargs.pop(
-            "profile_cycle_interval",
-            dask.config.get("distributed.worker.profile.cycle"),
-        )
+        if profile_cycle_interval is None:
+            profile_cycle_interval = dask.config.get("distributed.worker.profile.cycle")
         profile_cycle_interval = parse_timedelta(profile_cycle_interval, default="ms")
 
         self._setup_logging(logger)

From b538d246d9f766f1e96a2635dce48f3f38fa6011 Mon Sep 17 00:00:00 2001
From: tjb900 <timb@dugeo.com>
Date: Tue, 2 Jul 2019 14:57:18 +0800
Subject: [PATCH 0344/1550] Fix case where key, rather than TaskState, could
 end up in ts.waiting_on (#2819)

---
 distributed/scheduler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index d370705e9af..66d8fdaac90 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -3559,7 +3559,7 @@ def transition_no_worker_waiting(self, key):
             for dts in ts.dependencies:
                 dep = dts.key
                 if not dts.who_has:
-                    ts.waiting_on.add(dep)
+                    ts.waiting_on.add(dts)
                 if dts.state == "released":
                     recommendations[dep] = "waiting"
                 else:

From e42c124787bcaa82ff18b892450e58a34e3af92d Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 2 Jul 2019 10:33:48 +0100
Subject: [PATCH 0345/1550] Fix Client repr with memory_info=None (#2816)

---
 distributed/client.py            | 16 ++++++++++------
 distributed/tests/test_client.py |  5 +++++
 2 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 8509d9e31a9..74e33716cb6 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -801,7 +801,7 @@ def _repr_html_(self):
             info = sync(self.loop, self.scheduler.identity)
             scheduler = self.scheduler
         else:
-            info = False
+            info = self._scheduler_identity
             scheduler = self.scheduler
 
         if scheduler is not None:
@@ -828,10 +828,14 @@ def _repr_html_(self):
         text += "</ul>\n"
 
         if info:
-            workers = len(info["workers"])
-            cores = sum(w["nthreads"] for w in info["workers"].values())
-            memory = sum(w["memory_limit"] for w in info["workers"].values())
-            memory = format_bytes(memory)
+            workers = list(info["workers"].values())
+            cores = sum(w["nthreads"] for w in workers)
+            if all(isinstance(w["memory_limit"], Number) for w in workers):
+                memory = sum(w["memory_limit"] for w in workers)
+                memory = format_bytes(memory)
+            else:
+                memory = ""
+
             text2 = (
                 "<h3>Cluster</h3>\n"
                 "<ul>\n"
@@ -839,7 +843,7 @@ def _repr_html_(self):
                 "  <li><b>Cores: </b>%d</li>\n"
                 "  <li><b>Memory: </b>%s</li>\n"
                 "</ul>\n"
-            ) % (workers, cores, memory)
+            ) % (len(workers), cores, memory)
 
             return (
                 '<table style="border: 2px solid white;">\n'
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 8fa14269a1f..1f9678583b0 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -1892,6 +1892,11 @@ def test_repr_async(c, s, a, b):
     c._repr_html_()
 
 
+@gen_cluster(client=True, worker_kwargs={"memory_limit": None})
+def test_repr_no_memory_limit(c, s, a, b):
+    c._repr_html_()
+
+
 @gen_test()
 def test_repr_localcluster():
     cluster = yield LocalCluster(

From bda0ab6f35be915d5d74660c90195912dd7e5355 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Tue, 2 Jul 2019 12:00:11 -0500
Subject: [PATCH 0346/1550] Updates to use update_graph in task journey docs
 (#2821)

---
 docs/source/journey.rst | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/source/journey.rst b/docs/source/journey.rst
index 9dd1e15d62b..dd7e60e8d42 100644
--- a/docs/source/journey.rst
+++ b/docs/source/journey.rst
@@ -41,15 +41,15 @@ Step 2: Arrive in the Scheduler
 A few milliseconds later, the scheduler receives this message on an open socket.
 
 The scheduler updates its state with this little graph that shows how to compute
-``z``.::
+``z``::
 
-    scheduler.tasks.update(msg['tasks'])
+    scheduler.update_graph(tasks=msg['tasks'], keys=msg['keys'])
 
 The scheduler also updates *a lot* of other state.  Notably, it has to identify
 that ``x`` and ``y`` are themselves variables, and connect all of those
 dependencies.  This is a long and detail oriented process that involves
 updating roughly 10 sets and dictionaries.  Interested readers should
-investigate ``distributed/scheduler.py::update_state()``.  While this is fairly
+investigate ``distributed/scheduler.py::update_graph()``.  While this is fairly
 complex and tedious to describe rest assured that it all happens in constant
 time and in about a millisecond.
 

From eca16ed89e40563bab4a01c7cb7e41db30fc10a8 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 5 Jul 2019 10:23:51 +0100
Subject: [PATCH 0347/1550] Remove dask-mpi (#2824)

This has moved to the dask-mpi project documented at
https://mpi.dask.org

Fixes #2823
---
 distributed/cli/dask_mpi.py            | 147 -------------------------
 distributed/cli/tests/test_dask_mpi.py | 106 ------------------
 setup.py                               |   1 -
 3 files changed, 254 deletions(-)
 delete mode 100644 distributed/cli/dask_mpi.py
 delete mode 100644 distributed/cli/tests/test_dask_mpi.py

diff --git a/distributed/cli/dask_mpi.py b/distributed/cli/dask_mpi.py
deleted file mode 100644
index 7b9aeaca213..00000000000
--- a/distributed/cli/dask_mpi.py
+++ /dev/null
@@ -1,147 +0,0 @@
-from functools import partial
-
-import click
-from mpi4py import MPI
-from tornado.ioloop import IOLoop
-from tornado import gen
-from warnings import warn
-
-from distributed import Scheduler, Nanny, Worker
-from distributed.dashboard import BokehWorker
-from distributed.cli.utils import check_python_3
-from distributed.comm.addressing import uri_from_host_port
-from distributed.utils import get_ip_interface
-
-
-comm = MPI.COMM_WORLD
-rank = comm.Get_rank()
-loop = IOLoop()
-
-
-@click.command()
-@click.option(
-    "--scheduler-file",
-    type=str,
-    default="scheduler.json",
-    help="Filename to JSON encoded scheduler information. ",
-)
-@click.option(
-    "--interface", type=str, default=None, help="Network interface like 'eth0' or 'ib0'"
-)
-@click.option("--nthreads", type=int, default=0, help="Number of threads per worker.")
-@click.option(
-    "--memory-limit",
-    default="auto",
-    help="Number of bytes before spilling data to disk. "
-    "This can be an integer (nbytes) "
-    "float (fraction of total memory) "
-    "or 'auto'",
-)
-@click.option(
-    "--local-directory", default="", type=str, help="Directory to place worker files"
-)
-@click.option(
-    "--scheduler/--no-scheduler",
-    default=True,
-    help=(
-        "Whether or not to include a scheduler. "
-        "Use --no-scheduler to increase an existing dask cluster"
-    ),
-)
-@click.option(
-    "--nanny/--no-nanny",
-    default=True,
-    help="Start workers in nanny process for management",
-)
-@click.option(
-    "--bokeh-port", type=int, default=8787, help="Bokeh port for visual diagnostics"
-)
-@click.option(
-    "--bokeh-worker-port",
-    type=int,
-    default=8789,
-    help="Worker's Bokeh port for visual diagnostics",
-)
-@click.option("--bokeh-prefix", type=str, default=None, help="Prefix for the bokeh app")
-@click.version_option()
-def main(
-    scheduler_file,
-    interface,
-    nthreads,
-    local_directory,
-    memory_limit,
-    scheduler,
-    bokeh_port,
-    bokeh_prefix,
-    nanny,
-    bokeh_worker_port,
-):
-    if interface:
-        host = get_ip_interface(interface)
-    else:
-        host = None
-
-    if rank == 0 and scheduler:
-        try:
-            from distributed.dashboard import BokehScheduler
-        except ImportError:
-            services = {}
-        else:
-            services = {
-                ("dashboard", bokeh_port): partial(BokehScheduler, prefix=bokeh_prefix)
-            }
-        scheduler = Scheduler(
-            scheduler_file=scheduler_file, loop=loop, services=services
-        )
-        addr = uri_from_host_port(host, None, 8786)
-        scheduler.start(addr)
-        try:
-            loop.start()
-            loop.close()
-        finally:
-            scheduler.stop()
-    else:
-        W = Nanny if nanny else Worker
-        worker = W(
-            scheduler_file=scheduler_file,
-            loop=loop,
-            name=rank if scheduler else None,
-            nthreads=nthreads,
-            local_dir=local_directory,
-            services={("dashboard", bokeh_worker_port): BokehWorker},
-            memory_limit=memory_limit,
-        )
-        addr = uri_from_host_port(host, None, 0)
-
-        @gen.coroutine
-        def run():
-            yield worker._start(addr)
-            while worker.status != "closed":
-                yield gen.sleep(0.2)
-
-        try:
-            loop.run_sync(run)
-            loop.close()
-        finally:
-            pass
-
-        @gen.coroutine
-        def close():
-            yield worker._close(timeout=2)
-
-        loop.run_sync(close)
-
-
-def go():
-    check_python_3()
-    warn(
-        "The dask-mpi command line utility in the `distributed` "
-        "package is deprecated.  "
-        "Please install the `dask-mpi` package instead. "
-        "More information is available at https://mpi.dask.org"
-    )
-    main()
-
-
-if __name__ == "__main__":
-    go()
diff --git a/distributed/cli/tests/test_dask_mpi.py b/distributed/cli/tests/test_dask_mpi.py
deleted file mode 100644
index 89f1140bfab..00000000000
--- a/distributed/cli/tests/test_dask_mpi.py
+++ /dev/null
@@ -1,106 +0,0 @@
-from __future__ import print_function, division, absolute_import
-
-import subprocess
-from time import sleep
-
-import pytest
-
-pytest.importorskip("mpi4py")
-
-import requests
-from click.testing import CliRunner
-
-from distributed import Client
-from distributed.utils import tmpfile
-from distributed.metrics import time
-from distributed.utils_test import popen
-from distributed.utils_test import loop  # noqa: F401
-from distributed.cli.dask_remote import main
-
-
-@pytest.mark.parametrize("nanny", ["--nanny", "--no-nanny"])
-def test_basic(loop, nanny):
-    with tmpfile() as fn:
-        with popen(
-            ["mpirun", "--np", "4", "dask-mpi", "--scheduler-file", fn, nanny],
-            stdin=subprocess.DEVNULL,
-        ):
-            with Client(scheduler_file=fn) as c:
-
-                start = time()
-                while len(c.scheduler_info()["workers"]) != 3:
-                    assert time() < start + 10
-                    sleep(0.2)
-
-                assert c.submit(lambda x: x + 1, 10, workers=1).result() == 11
-
-
-def test_no_scheduler(loop):
-    with tmpfile() as fn:
-        with popen(
-            ["mpirun", "--np", "2", "dask-mpi", "--scheduler-file", fn],
-            stdin=subprocess.DEVNULL,
-        ):
-            with Client(scheduler_file=fn) as c:
-
-                start = time()
-                while len(c.scheduler_info()["workers"]) != 1:
-                    assert time() < start + 10
-                    sleep(0.2)
-
-                assert c.submit(lambda x: x + 1, 10).result() == 11
-                with popen(
-                    [
-                        "mpirun",
-                        "--np",
-                        "1",
-                        "dask-mpi",
-                        "--scheduler-file",
-                        fn,
-                        "--no-scheduler",
-                    ]
-                ):
-
-                    start = time()
-                    while len(c.scheduler_info()["workers"]) != 2:
-                        assert time() < start + 10
-                        sleep(0.2)
-
-
-def test_bokeh(loop):
-    with tmpfile() as fn:
-        with popen(
-            [
-                "mpirun",
-                "--np",
-                "2",
-                "dask-mpi",
-                "--scheduler-file",
-                fn,
-                "--bokeh-port",
-                "59583",
-                "--bokeh-worker-port",
-                "59584",
-            ],
-            stdin=subprocess.DEVNULL,
-        ):
-
-            for port in [59853, 59584]:
-                start = time()
-                while True:
-                    try:
-                        response = requests.get("http://localhost:%d/status/" % port)
-                        assert response.ok
-                        break
-                    except Exception:
-                        sleep(0.1)
-                        assert time() < start + 20
-
-    with pytest.raises(Exception):
-        requests.get("http://localhost:59583/status/")
-
-
-def test_version_option():
-    runner = CliRunner()
-    result = runner.invoke(main, ["--version"])
-    assert result.exit_code == 0
diff --git a/setup.py b/setup.py
index 125ddc9c328..84054d199e0 100755
--- a/setup.py
+++ b/setup.py
@@ -67,7 +67,6 @@
         dask-remote=distributed.cli.dask_remote:go
         dask-scheduler=distributed.cli.dask_scheduler:go
         dask-worker=distributed.cli.dask_worker:go
-        dask-mpi=distributed.cli.dask_mpi:go
       """,
     zip_safe=False,
 )

From 776bb6b745448ca64b8e5ad99d2f7fab7bbd8a8e Mon Sep 17 00:00:00 2001
From: Russ Bubley <russ.bubley@googlemail.com>
Date: Fri, 5 Jul 2019 19:36:18 +0100
Subject: [PATCH 0348/1550] Fix typo that prevented error message (#2825)

---
 distributed/core.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/core.py b/distributed/core.py
index 79c726eed6d..6f08c17ac77 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -369,7 +369,7 @@ def handle_comm(self, comm, shutting_down=shutting_down):
                     op = msg.pop("op")
                 except KeyError:
                     raise ValueError(
-                        "Received unexpected message without 'op' key: " % str(msg)
+                        "Received unexpected message without 'op' key: " + str(msg)
                     )
                 if self.counters is not None:
                     self.counters["op"].add(op)

From 99ac4550d0231ae81076307dda8b821b51bc5792 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Mon, 8 Jul 2019 10:47:33 -0500
Subject: [PATCH 0349/1550] bump version to 2.1.0

---
 docs/source/changelog.rst | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index 6162935b70d..cb98ab79fc7 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,20 @@
 Changelog
 =========
 
+2.1.0 - 2019-07-08
+------------------
+
+- Fix typo that prevented error message (:pr:`2825`) `Russ Bubley`_
+- Remove ``dask-mpi`` (:pr:`2824`) `Matthew Rocklin`_
+- Updates to use ``update_graph`` in task journey docs (:pr:`2821`) `James Bourbeau`_
+- Fix Client repr with ``memory_info=None`` (:pr:`2816`) `Matthew Rocklin`_
+- Fix case where key, rather than ``TaskState``, could end up in ``ts.waiting_on`` (:pr:`2819`) `tjb900`_
+- Use Keyword-only arguments (:pr:`2814`) `Matthew Rocklin`_
+- Relax check for worker references in cluster context manager (:pr:`2813`) `Matthew Rocklin`_
+- Add HTTPS support for the dashboard (:pr:`2812`) `Jim Crist`_
+- Use ``dask.utils.format_bytes`` (:pr:`2810`) `Tom Augspurger`_
+
+
 2.0.1 - 2019-06-26
 ------------------
 
@@ -1084,7 +1098,6 @@ significantly without many new features.
 .. _`Diane Trout`: https://github.com/detrout
 .. _`tjb900`: https://github.com/tjb900
 .. _`Stephan Hoyer`: https://github.com/shoyer
-.. _`tjb900`: https://github.com/tjb900
 .. _`Dirk Petersen`: https://github.com/dirkpetersen
 .. _`Daniel Farrell`: https://github.com/danpf
 .. _`George Sakkis`: https://github.com/gsakkis

From 21370fa8f3904f548731bb93111e0d730a8e80da Mon Sep 17 00:00:00 2001
From: Russ Bubley <russ.bubley@googlemail.com>
Date: Wed, 10 Jul 2019 13:23:22 +0100
Subject: [PATCH 0350/1550] Respect security configuration in LocalCluster
 (#2822)

Fixes #2815
---
 distributed/deploy/local.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index 554459e43ac..cb1a1511e20 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -12,6 +12,7 @@
 from .spec import SpecCluster
 from ..nanny import Nanny
 from ..scheduler import Scheduler
+from ..security import Security
 from ..worker import Worker, parse_memory_limit
 
 logger = logging.getLogger(__name__)
@@ -123,11 +124,12 @@ def __init__(
 
         self.status = None
         self.processes = processes
+        security = security or Security()
 
         if protocol is None:
             if host and "://" in host:
                 protocol = host.split("://")[0]
-            elif security:
+            elif security and security.require_encryption:
                 protocol = "tls://"
             elif not self.processes and not scheduler_port:
                 protocol = "inproc://"

From 5b31a87b823792c4d8646dd3bf249fee116fa567 Mon Sep 17 00:00:00 2001
From: Christian Hudon <chrish@pianocktail.org>
Date: Sun, 14 Jul 2019 09:48:34 -0500
Subject: [PATCH 0351/1550] Add Nanny to worker docs (#2826)

Fixes #2771
---
 distributed/nanny.py             | 10 +++++++++-
 docs/source/resilience.rst       | 14 ++++++++------
 docs/source/scheduling-state.rst |  4 ++--
 docs/source/worker.rst           |  9 +++++++++
 4 files changed, 28 insertions(+), 9 deletions(-)

diff --git a/distributed/nanny.py b/distributed/nanny.py
index f518b330d7c..b6b43116a73 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -42,7 +42,15 @@ class Nanny(ServerNode):
     """ A process to manage worker processes
 
     The nanny spins up Worker processes, watches then, and kills or restarts
-    them as necessary.
+    them as necessary. It is necessary if you want to use the
+    ``Client.restart`` method, or to restart the worker automatically if
+    it gets to the terminate fractiom of its memory limit.
+
+    The parameters for the Nanny are mostly the same as those for the Worker.
+
+    See Also
+    --------
+    Worker
     """
 
     _instances = weakref.WeakSet()
diff --git a/docs/source/resilience.rst b/docs/source/resilience.rst
index f5300d4fbcd..1936d7ee995 100644
--- a/docs/source/resilience.rst
+++ b/docs/source/resilience.rst
@@ -48,11 +48,12 @@ This has some fail cases.
     causes a segmentation fault, then that bad function will repeatedly be
     called on other workers.  This function will be marked as "bad" after it
     kills a fixed number of workers (defaults to three).
-3.  Data scattered out to the workers is not kept in the scheduler (it is
-    often quite large) and so the loss of this data is irreparable.  You may
-    wish to call ``Client.replicate`` on the data with a suitable replication
-    factor to ensure that it remains long-lived or else back the data off of
-    some resilient store, like a file system.
+3.  Data sent out directly to the workers via a call to ``scatter()`` (instead
+    of being created from a Dask task graph via other Dask functions) is not
+    kept in the scheduler, as it is often quite large, and so the loss of this
+    data is irreparable.  You may wish to call ``Client.replicate`` on the data
+    with a suitable replication factor to ensure that it remains long-lived or
+    else back the data off of some resilient store, like a file system.
 
 
 Hardware Failures
@@ -81,4 +82,5 @@ The client provides a mechanism to restart all of the workers in the cluster.
 This is convenient if, during the course of experimentation, you find your
 workers in an inconvenient state that makes them unresponsive.  The
 ``Client.restart`` method kills all workers, flushes all scheduler state, and
-then brings all workers back online, resulting in a clean cluster.
+then brings all workers back online, resulting in a clean cluster. This
+requires the nanny process (which is started by default).
diff --git a/docs/source/scheduling-state.rst b/docs/source/scheduling-state.rst
index 515bb26cdb0..4bffd182439 100644
--- a/docs/source/scheduling-state.rst
+++ b/docs/source/scheduling-state.rst
@@ -131,8 +131,8 @@ Conversely, "saturated" workers may see their workload lightened through
 Client State
 ------------
 
-Information about each individual client is kept in a :class:`ClientState`
-object:
+Information about each individual client of the scheduler is kept
+in a :class:`ClientState` object:
 
 .. autoclass:: ClientState
 
diff --git a/docs/source/worker.rst b/docs/source/worker.rst
index 530a27b9505..be288ccf68c 100644
--- a/docs/source/worker.rst
+++ b/docs/source/worker.rst
@@ -233,6 +233,15 @@ YARN, Mesos, SGE, etc..).  After termination the nanny will restart the worker
 in a fresh state.
 
 
+Nanny
+~~~~~
+
+Dask workers are by default launched, monitored, and managed by a small Nanny
+process.
+
+.. autoclass:: distributed.worker.Nanny
+
+
 API Documentation
 -----------------
 

From d4934986301b5c944e6bd049912086ac2dfb928f Mon Sep 17 00:00:00 2001
From: tjb900 <ozburgess@gmail.com>
Date: Tue, 16 Jul 2019 00:12:58 +0800
Subject: [PATCH 0352/1550] Don't make False add-keys report to scheduler
 (#2421)

Fixes #2420
---
 distributed/worker.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/distributed/worker.py b/distributed/worker.py
index 4ce564f1b7b..e124ba6ab1f 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -1884,11 +1884,6 @@ def gather_dep(self, worker, dep, deps, total_nbytes, cause=None):
                 self.incoming_count += 1
 
                 self.log.append(("receive-dep", worker, list(response["data"])))
-
-                if response["data"]:
-                    self.batched_stream.send(
-                        {"op": "add-keys", "keys": list(response["data"])}
-                    )
             except EnvironmentError as e:
                 logger.exception("Worker stream died during communication: %s", worker)
                 self.log.append(("receive-dep-failed", worker))

From e5ec8daab0d2b30702cfce4acb6259c10aef8e05 Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Mon, 15 Jul 2019 20:33:37 +0100
Subject: [PATCH 0353/1550] Include type name in SpecCluster repr (#2834)

---
 distributed/deploy/spec.py                    |  3 ++-
 distributed/deploy/tests/test_spec_cluster.py | 13 +++++++++++++
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index bb46f81db88..445949dc200 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -315,7 +315,8 @@ async def scale_down(self, workers):
     scale_up = scale  # backwards compatibility
 
     def __repr__(self):
-        return "SpecCluster(%r, workers=%d)" % (
+        return "%s(%r, workers=%d)" % (
+            type(self).__name__,
             self.scheduler_address,
             len(self.workers),
         )
diff --git a/distributed/deploy/tests/test_spec_cluster.py b/distributed/deploy/tests/test_spec_cluster.py
index 0c062d3d3e0..e51e8f14260 100644
--- a/distributed/deploy/tests/test_spec_cluster.py
+++ b/distributed/deploy/tests/test_spec_cluster.py
@@ -81,6 +81,19 @@ def test_loop_started():
     )
 
 
+@pytest.mark.asyncio
+async def test_repr():
+    worker = {"cls": Worker, "options": {"nthreads": 1}}
+
+    class MyCluster(SpecCluster):
+        pass
+
+    async with MyCluster(
+        asynchronous=True, scheduler=scheduler, worker=worker
+    ) as cluster:
+        assert "MyCluster" in str(cluster)
+
+
 @pytest.mark.asyncio
 async def test_scale():
     worker = {"cls": Worker, "options": {"nthreads": 1}}

From af64e07a01e8ce0d76744099a93ca2155d835ba8 Mon Sep 17 00:00:00 2001
From: Gabriel Sailer <sublinus@riseup.net>
Date: Tue, 16 Jul 2019 17:24:26 +0200
Subject: [PATCH 0354/1550] Extend prometheus metrics endpoint (#2792) (#2833)

* Expose tasks prometheus metric at scheduler

* Add basic task metrics to worker

Number of tasks in states and number of threads are exposed on the
workers /metrics endpoints.

* Add worker metrics and reformat tasks

* Change log mesage in case of missing crick
---
 distributed/dashboard/scheduler_html.py       | 14 +++-
 .../dashboard/tests/test_worker_bokeh_html.py |  2 +-
 distributed/dashboard/worker_html.py          | 81 +++++++++++++++----
 3 files changed, 77 insertions(+), 20 deletions(-)

diff --git a/distributed/dashboard/scheduler_html.py b/distributed/dashboard/scheduler_html.py
index 08829241d47..3f119a929b9 100644
--- a/distributed/dashboard/scheduler_html.py
+++ b/distributed/dashboard/scheduler_html.py
@@ -184,14 +184,24 @@ def collect(self):
 
         yield GaugeMetricFamily(
             "dask_scheduler_workers",
-            "Number of workers.",
+            "Number of workers connected.",
             value=len(self.server.workers),
         )
         yield GaugeMetricFamily(
             "dask_scheduler_clients",
-            "Number of clients.",
+            "Number of clients connected.",
             value=len(self.server.clients),
         )
+        yield GaugeMetricFamily(
+            "dask_scheduler_received_tasks",
+            "Number of tasks received at scheduler",
+            value=len(self.server.tasks),
+        )
+        yield GaugeMetricFamily(
+            "dask_scheduler_unrunnable_tasks",
+            "Number of unrunnable tasks at scheduler",
+            value=len(self.server.unrunnable),
+        )
 
 
 class PrometheusHandler(RequestHandler):
diff --git a/distributed/dashboard/tests/test_worker_bokeh_html.py b/distributed/dashboard/tests/test_worker_bokeh_html.py
index 99916b3fdc7..7a4d70a037c 100644
--- a/distributed/dashboard/tests/test_worker_bokeh_html.py
+++ b/distributed/dashboard/tests/test_worker_bokeh_html.py
@@ -25,7 +25,7 @@ def test_prometheus(c, s, a, b):
 
         txt = response.body.decode("utf8")
         families = {familiy.name for familiy in text_string_to_metric_families(txt)}
-        assert len(families) > 0
+        assert "dask_worker_latency_seconds" in families
 
 
 @gen_cluster(client=True, worker_kwargs={"services": {("dashboard", 0): BokehWorker}})
diff --git a/distributed/dashboard/worker_html.py b/distributed/dashboard/worker_html.py
index e1ae50f3afc..27e1f9fe9d2 100644
--- a/distributed/dashboard/worker_html.py
+++ b/distributed/dashboard/worker_html.py
@@ -1,23 +1,72 @@
+import logging
 from .utils import RequestHandler, redirect
 
 
 class _PrometheusCollector(object):
-    def __init__(self, server, prometheus_client):
-        self.server = server
+    def __init__(self, server):
+        self.worker = server
+        self.logger = logging.getLogger("distributed.dask_worker")
+        self.crick_available = True
+        try:
+            import crick  # noqa: F401
+        except ImportError:
+            self.crick_available = False
+            self.logger.info(
+                "Not all prometheus metrics available are exported. Digest-based metrics require crick to be installed"
+            )
 
     def collect(self):
-        # add your metrics here:
-        #
-        # 1. remove the following lines
-        while False:
-            yield None
-        #
-        # 2. yield your metrics
-        #     yield prometheus_client.core.GaugeMetricFamily(
-        #         'dask_worker_connections',
-        #         'Number of connections currently open.',
-        #         value=???,
-        #     )
+        from prometheus_client.core import GaugeMetricFamily
+
+        tasks = GaugeMetricFamily(
+            "dask_worker_tasks", "Number of tasks at worker.", labels=["state"]
+        )
+        tasks.add_metric(["stored"], len(self.worker.data))
+        tasks.add_metric(["ready"], len(self.worker.ready))
+        tasks.add_metric(["waiting"], len(self.worker.waiting_for_data))
+        tasks.add_metric(["serving"], len(self.worker._comms))
+        yield tasks
+
+        yield GaugeMetricFamily(
+            "dask_worker_connections",
+            "Number of task connections to other workers.",
+            value=len(self.worker.in_flight_workers),
+        )
+
+        yield GaugeMetricFamily(
+            "dask_worker_threads",
+            "Number of worker threads.",
+            value=self.worker.nthreads,
+        )
+
+        yield GaugeMetricFamily(
+            "dask_worker_latency_seconds",
+            "Latency of worker connection.",
+            value=self.worker.latency,
+        )
+
+        # all metrics using digests require crick to be installed
+        # the following metrics will export NaN, if the corresponding digests are None
+        if self.crick_available:
+            yield GaugeMetricFamily(
+                "dask_worker_tick_duration_median_seconds",
+                "Median tick duration at worker.",
+                value=self.worker.digests["tick-duration"].components[1].quantile(50),
+            )
+
+            yield GaugeMetricFamily(
+                "dask_worker_task_duration_median_seconds",
+                "Median task runtime at worker.",
+                value=self.worker.digests["task-duration"].components[1].quantile(50),
+            )
+
+            yield GaugeMetricFamily(
+                "dask_worker_transfer_bandwidth_median_bytes",
+                "Bandwidth for transfer at worker in Bytes.",
+                value=self.worker.digests["transfer-bandwidth"]
+                .components[1]
+                .quantile(50),
+            )
 
 
 class PrometheusHandler(RequestHandler):
@@ -31,9 +80,7 @@ def __init__(self, *args, **kwargs):
         if PrometheusHandler._initialized:
             return
 
-        prometheus_client.REGISTRY.register(
-            _PrometheusCollector(self.server, prometheus_client)
-        )
+        prometheus_client.REGISTRY.register(_PrometheusCollector(self.server))
 
         PrometheusHandler._initialized = True
 

From df2addc62be91fc017b429c947afa8acd1a64127 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 17 Jul 2019 21:59:23 -0500
Subject: [PATCH 0355/1550] Add alternative SSHCluster implementation (#2827)

This is a proof of concept here for two reasons:

1.  It opens up a possible alternative for SSH deployment (which was
    surprisingly popular in the user survey)
2.  It is the first non-local application of `SpecCluster` and so serves
    as a proof of concept for other future deployments that are mostly
    defined by creating a remote Worker/Scheduler object

This forced some changes in `SpecCluster`, notably we now have an `rpc`
object that does remote calls rather than accessing the scheduler
directly.  Also, we're going to have to figure out how to handle all of
the keyword arguments.  In this case we need to pass them from Python
down to the CLI, and presumably we'll also want a `dask-ssh` CLI command
which has to translate the other way.
---
 .travis.yml                                |   1 +
 continuous_integration/travis/install.sh   |   1 +
 continuous_integration/travis/setup-ssh.sh |   2 +
 distributed/deploy/local.py                |   1 +
 distributed/deploy/spec.py                 |  30 ++--
 distributed/deploy/ssh2.py                 | 171 +++++++++++++++++++++
 distributed/deploy/tests/test_ssh2.py      |  17 ++
 7 files changed, 214 insertions(+), 9 deletions(-)
 create mode 100644 continuous_integration/travis/setup-ssh.sh
 create mode 100644 distributed/deploy/ssh2.py
 create mode 100644 distributed/deploy/tests/test_ssh2.py

diff --git a/.travis.yml b/.travis.yml
index bcc09351eff..35f4383748e 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -19,6 +19,7 @@ matrix:
 
 install:
   - if [[ $TESTS == true ]]; then source continuous_integration/travis/install.sh ; fi
+  - if [[ $TESTS == true ]]; then source continuous_integration/travis/setup-ssh.sh ; fi
 
 script:
   - if [[ $TESTS == true ]]; then source continuous_integration/travis/run_tests.sh ; fi
diff --git a/continuous_integration/travis/install.sh b/continuous_integration/travis/install.sh
index 2ab9724db25..b2fab6afb52 100644
--- a/continuous_integration/travis/install.sh
+++ b/continuous_integration/travis/install.sh
@@ -67,6 +67,7 @@ pip install -q git+https://github.com/dask/s3fs.git --upgrade --no-deps
 pip install -q git+https://github.com/dask/zict.git --upgrade --no-deps
 pip install -q sortedcollections msgpack --no-deps
 pip install -q keras --upgrade --no-deps
+pip install -q asyncssh 
 
 if [[ $CRICK == true ]]; then
     conda install -q cython
diff --git a/continuous_integration/travis/setup-ssh.sh b/continuous_integration/travis/setup-ssh.sh
new file mode 100644
index 00000000000..f102612bc96
--- /dev/null
+++ b/continuous_integration/travis/setup-ssh.sh
@@ -0,0 +1,2 @@
+ssh-keygen -t rsa -f ~/.ssh/id_rsa -N "" -q
+cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
\ No newline at end of file
diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index cb1a1511e20..5b0aec4e80c 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -200,6 +200,7 @@ def __init__(
             loop=loop,
             asynchronous=asynchronous,
             silence_logs=silence_logs,
+            security=security,
         )
 
     def __repr__(self):
diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index 445949dc200..6b95e2107fe 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -5,8 +5,10 @@
 from tornado import gen
 
 from .cluster import Cluster
+from ..core import rpc
 from ..utils import LoopRunner, silence_logging, ignoring
 from ..scheduler import Scheduler
+from ..security import Security
 
 
 class SpecCluster(Cluster):
@@ -107,6 +109,7 @@ def __init__(
         worker=None,
         asynchronous=False,
         loop=None,
+        security=None,
         silence_logs=False,
     ):
         self._created = weakref.WeakSet()
@@ -125,6 +128,8 @@ def __init__(
         self.workers = {}
         self._i = 0
         self._asynchronous = asynchronous
+        self.security = security or Security()
+        self.scheduler_comm = None
 
         if silence_logs:
             self._old_logging_level = silence_logging(level=silence_logs)
@@ -156,6 +161,10 @@ async def _start(self):
         self._lock = asyncio.Lock()
         self.status = "starting"
         self.scheduler = await self.scheduler
+        self.scheduler_comm = rpc(
+            self.scheduler.address,
+            connection_args=self.security.get_connection_args("client"),
+        )
         self.status = "running"
 
     def _correct_state(self):
@@ -174,11 +183,13 @@ async def _correct_state_internal(self):
             pre = list(set(self.workers))
             to_close = set(self.workers) - set(self.worker_spec)
             if to_close:
-                await self.scheduler.retire_workers(workers=list(to_close))
+                if self.scheduler.status == "running":
+                    await self.scheduler_comm.retire_workers(workers=list(to_close))
                 tasks = [self.workers[w].close() for w in to_close]
                 await asyncio.wait(tasks)
                 for task in tasks:  # for tornado gen.coroutine support
-                    await task
+                    with ignoring(RuntimeError):
+                        await task
             for name in to_close:
                 del self.workers[name]
 
@@ -214,11 +225,10 @@ async def _():
         return _().__await__()
 
     async def _wait_for_workers(self):
-        # TODO: this function needs to query scheduler and worker state
-        # remotely without assuming that they are local
-        while {d["name"] for d in self.scheduler.identity()["workers"].values()} != set(
-            self.workers
-        ):
+        while {
+            str(d["name"])
+            for d in (await self.scheduler_comm.identity())["workers"].values()
+        } != set(map(str, self.workers)):
             if (
                 any(w.status == "closed" for w in self.workers.values())
                 and self.scheduler.status == "running"
@@ -240,12 +250,14 @@ async def _close(self):
             return
         self.status = "closing"
 
-        async with self._lock:
-            await self.scheduler.close(close_workers=True)
         self.scale(0)
         await self._correct_state()
+        async with self._lock:
+            await self.scheduler_comm.close(close_workers=True)
+        await self.scheduler.close()
         for w in self._created:
             assert w.status == "closed"
+        self.scheduler_comm.close_rpc()
 
         if hasattr(self, "_old_logging_level"):
             silence_logging(self._old_logging_level)
diff --git a/distributed/deploy/ssh2.py b/distributed/deploy/ssh2.py
new file mode 100644
index 00000000000..0f8823cdab8
--- /dev/null
+++ b/distributed/deploy/ssh2.py
@@ -0,0 +1,171 @@
+import asyncio
+import logging
+import sys
+import warnings
+import weakref
+
+import asyncssh
+
+from .spec import SpecCluster
+
+logger = logging.getLogger(__name__)
+
+warnings.warn(
+    "the distributed.deploy.ssh2 module is experimental "
+    "and will move/change in the future without notice"
+)
+
+
+class Process:
+    """ A superclass for SSH Workers and Nannies
+
+    See Also
+    --------
+    Worker
+    Scheduler
+    """
+
+    def __init__(self):
+        self.lock = asyncio.Lock()
+        self.connection = None
+        self.proc = None
+        self.status = "created"
+
+    def __await__(self):
+        async def _():
+            async with self.lock:
+                if not self.connection:
+                    await self.start()
+                    assert self.connection
+                    weakref.finalize(self, self.proc.terminate)
+            return self
+
+        return _().__await__()
+
+    async def close(self):
+        self.proc.terminate()
+        self.connection.close()
+        self.status = "closed"
+
+    def __repr__(self):
+        return "<SSH %s: status=%s>" % (type(self).__name__, self.status)
+
+
+class Worker(Process):
+    """ A Remote Dask Worker controled by SSH
+
+    Parameters
+    ----------
+    scheduler: str
+        The address of the scheduler
+    address: str
+        The hostname where we should run this worker
+    connect_kwargs: dict
+        kwargs to be passed to asyncssh connections
+    kwargs:
+        TODO
+    """
+
+    def __init__(self, scheduler: str, address: str, connect_kwargs: dict, **kwargs):
+        self.address = address
+        self.scheduler = scheduler
+        self.connect_kwargs = connect_kwargs
+        self.kwargs = kwargs
+
+        super().__init__()
+
+    async def start(self):
+        self.connection = await asyncssh.connect(self.address, **self.connect_kwargs)
+        self.proc = await self.connection.create_process(
+            " ".join(
+                [
+                    sys.executable,
+                    "-m",
+                    "distributed.cli.dask_worker",
+                    self.scheduler,
+                    "--name",  # we need to have name for SpecCluster
+                    str(self.kwargs["name"]),
+                ]
+            )
+        )
+
+        # We watch stderr in order to get the address, then we return
+        while True:
+            line = await self.proc.stderr.readline()
+            if "worker at" in line:
+                self.address = line.split("worker at:")[1].strip()
+                self.status = "running"
+                break
+        logger.debug("%s", line)
+
+
+class Scheduler(Process):
+    """ A Remote Dask Scheduler controled by SSH
+
+    Parameters
+    ----------
+    address: str
+        The hostname where we should run this worker
+    connect_kwargs: dict
+        kwargs to be passed to asyncssh connections
+    kwargs:
+        TODO
+    """
+
+    def __init__(self, address: str, connect_kwargs: dict, **kwargs):
+        self.address = address
+        self.kwargs = kwargs
+        self.connect_kwargs = connect_kwargs
+
+        super().__init__()
+
+    async def start(self):
+        logger.debug("Created Scheduler Connection")
+
+        self.connection = await asyncssh.connect(self.address, **self.connect_kwargs)
+
+        self.proc = await self.connection.create_process(
+            " ".join([sys.executable, "-m", "distributed.cli.dask_scheduler"])
+        )
+
+        # We watch stderr in order to get the address, then we return
+        while True:
+            line = await self.proc.stderr.readline()
+            if "Scheduler at" in line:
+                self.address = line.split("Scheduler at:")[1].strip()
+                break
+        logger.debug("%s", line)
+
+
+def SSHCluster(hosts, connect_kwargs, **kwargs):
+    """ Deploy a Dask cluster using SSH
+
+    Parameters
+    ----------
+    hosts: List[str]
+        List of hostnames or addresses on which to launch our cluster
+        The first will be used for the scheduler and the rest for workers
+    connect_kwargs:
+        known_hosts: List[str] or None
+            The list of keys which will be used to validate the server host
+            key presented during the SSH handshake.  If this is not specified,
+            the keys will be looked up in the file .ssh/known_hosts.  If this
+            is explicitly set to None, server host key validation will be disabled.
+        TODO
+    kwargs:
+        TODO
+    ----
+    This doesn't handle any keyword arguments yet.  It is a proof of concept
+    """
+    scheduler = {
+        "cls": Scheduler,
+        "options": {"address": hosts[0], "connect_kwargs": connect_kwargs},
+    }
+    workers = {
+        i: {
+            "cls": Worker,
+            "options": {"address": host, "connect_kwargs": connect_kwargs},
+        }
+        for i, host in enumerate(hosts[1:])
+    }
+    return SpecCluster(workers, scheduler, **kwargs)
diff --git a/distributed/deploy/tests/test_ssh2.py b/distributed/deploy/tests/test_ssh2.py
new file mode 100644
index 00000000000..beb1c6ef91e
--- /dev/null
+++ b/distributed/deploy/tests/test_ssh2.py
@@ -0,0 +1,17 @@
+import pytest
+
+pytest.importorskip("asyncssh")
+
+from dask.distributed import Client
+from distributed.deploy.ssh2 import SSHCluster
+
+
+@pytest.mark.asyncio
+async def test_basic():
+    async with SSHCluster(
+        ["127.0.0.1"] * 3, connect_kwargs=dict(known_hosts=None), asynchronous=True
+    ) as cluster:
+        assert len(cluster.workers) == 2
+        async with Client(cluster, asynchronous=True) as client:
+            result = await client.submit(lambda x: x + 1, 10)
+            assert result == 11

From 27e8e6548e7a8401c9199a55c7dea2fa7331cb04 Mon Sep 17 00:00:00 2001
From: Pierre Glaser <pierreglaser@msn.com>
Date: Thu, 18 Jul 2019 08:27:09 +0200
Subject: [PATCH 0356/1550] Dont reuse closed worker in get_worker (#2841)

---
 distributed/deploy/tests/test_local.py | 25 ++++++++++++++++++++++++-
 distributed/worker.py                  |  2 +-
 2 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index 520996f64a0..ebe2d7ec99e 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -14,7 +14,7 @@
 from tornado import gen
 import pytest
 
-from distributed import Client, Worker, Nanny
+from distributed import Client, Worker, Nanny, get_client
 from distributed.deploy.local import LocalCluster, nprocesses_nthreads
 from distributed.metrics import time
 from distributed.utils_test import (
@@ -831,3 +831,26 @@ def test_starts_up_sync(loop):
         assert len(cluster.scheduler.workers) == 2
     finally:
         cluster.close()
+
+
+def test_dont_select_closed_worker():
+    # Make sure distributed does not try to reuse a client from a
+    # closed cluster (https://github.com/dask/distributed/issues/2840).
+    with clean(threads=False):
+        cluster = LocalCluster(n_workers=0)
+        c = Client(cluster)
+        cluster.scale(2)
+        assert c == get_client()
+
+        c.close()
+        cluster.close()
+
+        cluster2 = LocalCluster(n_workers=0)
+        c2 = Client(cluster2)
+        cluster2.scale(2)
+
+        current_client = get_client()
+        assert c2 == current_client
+
+        cluster2.close()
+        c2.close()
diff --git a/distributed/worker.py b/distributed/worker.py
index e124ba6ab1f..60884ebaba3 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -2852,7 +2852,7 @@ def get_worker():
         return thread_state.execution_state["worker"]
     except AttributeError:
         try:
-            return first(Worker._instances)
+            return first(w for w in Worker._instances if w.status == "running")
         except StopIteration:
             raise ValueError("No workers found")
 

From 0ada76c91bbf35f1e783159b63d3effe01c803dd Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Thu, 18 Jul 2019 15:21:25 +0100
Subject: [PATCH 0357/1550] SpecCluster: move init logic into start (#2850)

Move the scheduler creation from `__init__` to `_start`.
This allows clusters to make async calls within subclassed `_start` methods
before the scheduler object is created.

This also ignores exceptions from closing the scheduler if the scheduler has already timed out.
---
 distributed/deploy/spec.py | 28 +++++++++++++++-------------
 1 file changed, 15 insertions(+), 13 deletions(-)

diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index 6b95e2107fe..0d2d2d37021 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -5,7 +5,7 @@
 from tornado import gen
 
 from .cluster import Cluster
-from ..core import rpc
+from ..core import rpc, CommClosedError
 from ..utils import LoopRunner, silence_logging, ignoring
 from ..scheduler import Scheduler
 from ..security import Security
@@ -113,14 +113,6 @@ def __init__(
         silence_logs=False,
     ):
         self._created = weakref.WeakSet()
-        if scheduler is None:
-            try:
-                from distributed.dashboard import BokehScheduler
-            except ImportError:
-                services = {}
-            else:
-                services = {("dashboard", 8787): BokehScheduler}
-            scheduler = {"cls": Scheduler, "options": {"services": services}}
 
         self.scheduler_spec = scheduler
         self.worker_spec = workers or {}
@@ -137,9 +129,6 @@ def __init__(
         self._loop_runner = LoopRunner(loop=loop, asynchronous=asynchronous)
         self.loop = self._loop_runner.loop
 
-        self.scheduler = self.scheduler_spec["cls"](
-            loop=self.loop, **self.scheduler_spec["options"]
-        )
         self.status = "created"
         self._instances.add(self)
         self._correct_state_waiting = None
@@ -158,6 +147,18 @@ async def _start(self):
         if self.status == "closed":
             raise ValueError("Cluster is closed")
 
+        if self.scheduler_spec is None:
+            try:
+                from distributed.dashboard import BokehScheduler
+            except ImportError:
+                services = {}
+            else:
+                services = {("dashboard", 8787): BokehScheduler}
+            self.scheduler_spec = {"cls": Scheduler, "options": {"services": services}}
+        self.scheduler = self.scheduler_spec["cls"](
+            loop=self.loop, **self.scheduler_spec["options"]
+        )
+
         self._lock = asyncio.Lock()
         self.status = "starting"
         self.scheduler = await self.scheduler
@@ -253,7 +254,8 @@ async def _close(self):
         self.scale(0)
         await self._correct_state()
         async with self._lock:
-            await self.scheduler_comm.close(close_workers=True)
+            with ignoring(CommClosedError):
+                await self.scheduler_comm.close(close_workers=True)
         await self.scheduler.close()
         for w in self._created:
             assert w.status == "closed"

From 36b7585c7c84470c7ada34f1d0272f1696a6ecbf Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Fri, 19 Jul 2019 16:20:54 -0400
Subject: [PATCH 0358/1550] Document distributed.Reschedule in API docs (#2860)

---
 docs/source/api.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/source/api.rst b/docs/source/api.rst
index adefe5b86c4..8d739334b07 100644
--- a/docs/source/api.rst
+++ b/docs/source/api.rst
@@ -167,6 +167,7 @@ Other
 .. autofunction:: distributed.get_client
 .. autofunction:: distributed.secede
 .. autofunction:: distributed.rejoin
+.. autoclass:: distributed.Reschedule
 .. autoclass:: get_task_stream
 
 .. autoclass:: Lock

From 4c105056af643898d06ecd98ab2802cde495ac64 Mon Sep 17 00:00:00 2001
From: Martin Durant <martindurant@users.noreply.github.com>
Date: Fri, 19 Jul 2019 16:51:13 -0400
Subject: [PATCH 0359/1550] Add fsspec to installation of test builds (#2859)

---
 continuous_integration/setup_conda_environment.cmd | 1 +
 continuous_integration/travis/install.sh           | 1 +
 2 files changed, 2 insertions(+)

diff --git a/continuous_integration/setup_conda_environment.cmd b/continuous_integration/setup_conda_environment.cmd
index 3df89fa85fe..6fff1a5ca6a 100644
--- a/continuous_integration/setup_conda_environment.cmd
+++ b/continuous_integration/setup_conda_environment.cmd
@@ -40,6 +40,7 @@ call deactivate
     tblib ^
     tornado=5 ^
     zict ^
+    fsspec ^
     -c conda-forge
 
 call activate %CONDA_ENV%
diff --git a/continuous_integration/travis/install.sh b/continuous_integration/travis/install.sh
index b2fab6afb52..82993032e0b 100644
--- a/continuous_integration/travis/install.sh
+++ b/continuous_integration/travis/install.sh
@@ -63,6 +63,7 @@ pip install -q "pytest>=4" pytest-repeat pytest-faulthandler pytest-asyncio
 
 pip install -q git+https://github.com/dask/dask.git --upgrade --no-deps
 pip install -q git+https://github.com/joblib/joblib.git --upgrade --no-deps
+pip install -q git+https://github.com/intake/filesystem_spec.git --upgrade --no-deps
 pip install -q git+https://github.com/dask/s3fs.git --upgrade --no-deps
 pip install -q git+https://github.com/dask/zict.git --upgrade --no-deps
 pip install -q sortedcollections msgpack --no-deps

From 96ff5d3409c0146759cdc68a43d86c96e159daa4 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sat, 20 Jul 2019 13:19:10 -0500
Subject: [PATCH 0360/1550] Make await/start more consistent across
 Scheduler/Worker/Nanny (#2831)

Now every ServerNode has a start async method that returns itself.
And the __await__ method is handled in the superclass
---
 distributed/cli/tests/test_dask_worker.py     | 35 ++++-----
 distributed/comm/addressing.py                | 10 ++-
 distributed/deploy/spec.py                    |  2 +-
 distributed/deploy/tests/test_spec_cluster.py | 13 +++-
 distributed/nanny.py                          | 63 ++++++----------
 distributed/node.py                           | 10 +++
 distributed/scheduler.py                      | 36 +++-------
 distributed/tests/test_core.py                | 10 +--
 distributed/tests/test_failed_workers.py      | 23 +++---
 distributed/tests/test_nanny.py               |  6 +-
 distributed/tests/test_scheduler.py           | 36 ++++------
 distributed/tests/test_stress.py              |  5 +-
 distributed/tests/test_utils_test.py          |  5 +-
 distributed/tests/test_worker.py              | 71 ++++++++++++++++---
 distributed/tests/test_worker_plugins.py      |  3 +-
 distributed/utils_test.py                     | 13 ++--
 distributed/worker.py                         | 64 ++++-------------
 17 files changed, 201 insertions(+), 204 deletions(-)

diff --git a/distributed/cli/tests/test_dask_worker.py b/distributed/cli/tests/test_dask_worker.py
index b6c7d393e3b..e268229767d 100644
--- a/distributed/cli/tests/test_dask_worker.py
+++ b/distributed/cli/tests/test_dask_worker.py
@@ -268,29 +268,30 @@ def test_dashboard_non_standard_ports(loop):
     except ImportError:
         proxy_exists = False
 
-    with popen(["dask-scheduler", "--port", "3449"]):
+    with popen(["dask-scheduler", "--port", "3449"]) as s:
         with popen(
-            ["dask-worker", "tcp://127.0.0.1:3449", "--dashboard-address", ":4833"]
+            [
+                "dask-worker",
+                "tcp://127.0.0.1:3449",
+                "--dashboard-address",
+                ":4833",
+                "--host",
+                "127.0.0.1",
+            ]
         ) as proc:
             with Client("127.0.0.1:3449", loop=loop) as c:
+                c.wait_for_workers(1)
                 pass
 
-            start = time()
-            while True:
-                try:
-                    response = requests.get("http://127.0.0.1:4833/status")
+                response = requests.get("http://127.0.0.1:4833/status")
+                assert response.ok
+                redirect_resp = requests.get("http://127.0.0.1:4833/main")
+                redirect_resp.ok
+                # TEST PROXYING WORKS
+                if proxy_exists:
+                    url = "http://127.0.0.1:8787/proxy/4833/127.0.0.1/status"
+                    response = requests.get(url)
                     assert response.ok
-                    redirect_resp = requests.get("http://127.0.0.1:4833/main")
-                    redirect_resp.ok
-                    # TEST PROXYING WORKS
-                    if proxy_exists:
-                        url = "http://127.0.0.1:8787/proxy/4833/127.0.0.1/status"
-                        response = requests.get(url)
-                        assert response.ok
-                    break
-                except Exception:
-                    sleep(0.5)
-                    assert time() < start + 20
 
         with pytest.raises(Exception):
             requests.get("http://localhost:4833/status/")
diff --git a/distributed/comm/addressing.py b/distributed/comm/addressing.py
index d707adb84ac..54e37b77f6b 100644
--- a/distributed/comm/addressing.py
+++ b/distributed/comm/addressing.py
@@ -211,7 +211,13 @@ def uri_from_host_port(host_arg, port_arg, default_port):
 
 
 def address_from_user_args(
-    host=None, port=None, interface=None, protocol=None, peer=None, security=None
+    host=None,
+    port=None,
+    interface=None,
+    protocol=None,
+    peer=None,
+    security=None,
+    default_port=0,
 ):
     """ Get an address to listen on from common user provided arguments """
     if security and security.require_encryption and not protocol:
@@ -235,7 +241,7 @@ def address_from_user_args(
         host = protocol.rstrip("://") + "://" + host
 
     if host or port:
-        addr = uri_from_host_port(host, port, 0)
+        addr = uri_from_host_port(host, port, default_port)
     else:
         addr = ""
 
diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index 0d2d2d37021..441ef10a595 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -156,7 +156,7 @@ async def _start(self):
                 services = {("dashboard", 8787): BokehScheduler}
             self.scheduler_spec = {"cls": Scheduler, "options": {"services": services}}
         self.scheduler = self.scheduler_spec["cls"](
-            loop=self.loop, **self.scheduler_spec["options"]
+            loop=self.loop, **self.scheduler_spec.get("options", {})
         )
 
         self._lock = asyncio.Lock()
diff --git a/distributed/deploy/tests/test_spec_cluster.py b/distributed/deploy/tests/test_spec_cluster.py
index e51e8f14260..723c62a80c1 100644
--- a/distributed/deploy/tests/test_spec_cluster.py
+++ b/distributed/deploy/tests/test_spec_cluster.py
@@ -1,4 +1,4 @@
-from dask.distributed import SpecCluster, Worker, Client, Scheduler
+from dask.distributed import SpecCluster, Worker, Client, Scheduler, Nanny
 from distributed.deploy.spec import close_clusters
 from distributed.utils_test import loop  # noqa: F401
 import pytest
@@ -153,3 +153,14 @@ def new_worker_spec(self):
         cluster.scale(3)
         for i in range(3):
             assert cluster.worker_spec[i]["options"]["nthreads"] == i + 1
+
+
+@pytest.mark.asyncio
+async def test_nanny_port():
+    scheduler = {"cls": Scheduler}
+    workers = {0: {"cls": Nanny, "options": {"port": 9200}}}
+
+    async with SpecCluster(
+        scheduler=scheduler, workers=workers, asynchronous=True
+    ) as cluster:
+        pass
diff --git a/distributed/nanny.py b/distributed/nanny.py
index b6b43116a73..00370b81adb 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -17,7 +17,7 @@
 from tornado.ioloop import IOLoop, TimeoutError
 from tornado.locks import Event
 
-from .comm import get_address_host, get_local_address_for, unparse_host_port
+from .comm import get_address_host, unparse_host_port
 from .comm.addressing import address_from_user_args
 from .core import RPCClosed, CommClosedError, coerce_to_address
 from .metrics import time
@@ -120,6 +120,14 @@ def __init__(
         self.preload_argv = preload_argv
         self.Worker = Worker if worker_class is None else worker_class
         self.env = env or {}
+        worker_kwargs.update(
+            {
+                "port": worker_port,
+                "interface": interface,
+                "protocol": protocol,
+                "host": host,
+            }
+        )
         self.worker_kwargs = worker_kwargs
 
         self.contact_address = contact_address
@@ -161,6 +169,13 @@ def __init__(
             pc = PeriodicCallback(self.memory_monitor, 100, io_loop=self.loop)
             self.periodic_callbacks["memory"] = pc
 
+        if (
+            not host
+            and not interface
+            and not self.scheduler_addr.startswith("inproc://")
+        ):
+            host = get_ip(get_address_host(self.scheduler.address))
+
         self._start_address = address_from_user_args(
             host=host,
             port=port,
@@ -208,25 +223,10 @@ def worker_dir(self):
         return None if self.process is None else self.process.worker_dir
 
     @gen.coroutine
-    def _start(self, addr_or_port=0):
+    def start(self):
         """ Start nanny, start local process, start watching """
-        addr_or_port = addr_or_port or self._start_address
-
-        # XXX Factor this out
-        if not addr_or_port:
-            # Default address is the required one to reach the scheduler
-            self.listen(
-                get_local_address_for(self.scheduler.address),
-                listen_args=self.listen_args,
-            )
-            self.ip = get_address_host(self.address)
-        elif isinstance(addr_or_port, int):
-            # addr_or_port is an integer => assume TCP
-            self.ip = get_ip(get_address_host(self.scheduler.address))
-            self.listen((self.ip, addr_or_port), listen_args=self.listen_args)
-        else:
-            self.listen(addr_or_port, listen_args=self.listen_args)
-            self.ip = get_address_host(self.address)
+        self.listen(self._start_address, listen_args=self.listen_args)
+        self.ip = get_address_host(self.address)
 
         logger.info("        Start Nanny at: %r", self.address)
         response = yield self.instantiate()
@@ -238,13 +238,7 @@ def _start(self, addr_or_port=0):
 
         self.start_periodic_callbacks()
 
-        raise gen.Return(self)
-
-    def __await__(self):
-        return self._start().__await__()
-
-    def start(self, addr_or_port=0):
-        self.loop.add_callback(self._start, addr_or_port)
+        return self
 
     @gen.coroutine
     def kill(self, comm=None, timeout=2):
@@ -295,7 +289,6 @@ def instantiate(self, comm=None):
             )
             worker_kwargs.update(self.worker_kwargs)
             self.process = WorkerProcess(
-                worker_args=tuple(),
                 worker_kwargs=worker_kwargs,
                 worker_start_args=(start_arg,),
                 silence_logs=self.silence_logs,
@@ -432,18 +425,10 @@ def close(self, comm=None, timeout=5, report=None):
 
 class WorkerProcess(object):
     def __init__(
-        self,
-        worker_args,
-        worker_kwargs,
-        worker_start_args,
-        silence_logs,
-        on_exit,
-        worker,
-        env,
+        self, worker_kwargs, worker_start_args, silence_logs, on_exit, worker, env
     ):
         self.status = "init"
         self.silence_logs = silence_logs
-        self.worker_args = worker_args
         self.worker_kwargs = worker_kwargs
         self.worker_start_args = worker_start_args
         self.on_exit = on_exit
@@ -475,7 +460,6 @@ def start(self):
             target=self._run,
             name="Dask Worker process (from Nanny)",
             kwargs=dict(
-                worker_args=self.worker_args,
                 worker_kwargs=self.worker_kwargs,
                 worker_start_args=self.worker_start_args,
                 silence_logs=self.silence_logs,
@@ -615,7 +599,6 @@ def _wait_until_connected(self, uid):
     @classmethod
     def _run(
         cls,
-        worker_args,
         worker_kwargs,
         worker_start_args,
         silence_logs,
@@ -639,7 +622,7 @@ def _run(
         IOLoop.clear_instance()
         loop = IOLoop()
         loop.make_current()
-        worker = Worker(*worker_args, **worker_kwargs)
+        worker = Worker(**worker_kwargs)
 
         @gen.coroutine
         def do_stop(timeout=5, executor_wait=True):
@@ -679,7 +662,7 @@ def run():
             Try to start worker and inform parent of outcome.
             """
             try:
-                yield worker._start(*worker_start_args)
+                yield worker
             except Exception as e:
                 logger.exception("Failed to start worker")
                 init_result_q.put({"uid": uid, "exception": e})
diff --git a/distributed/node.py b/distributed/node.py
index 8bd81ffe5ae..323e2c3e49d 100644
--- a/distributed/node.py
+++ b/distributed/node.py
@@ -4,6 +4,7 @@
 import logging
 
 from tornado.ioloop import IOLoop
+from tornado import gen
 import dask
 
 from .compatibility import unicode, finalize
@@ -159,3 +160,12 @@ async def __aenter__(self):
 
     async def __aexit__(self, typ, value, traceback):
         await self.close()
+
+    def __await__(self):
+        if self.status == "running":
+            return gen.sleep(0).__await__()
+        else:
+            return self.start().__await__()
+
+    async def start(self):  # subclasses should implement this
+        return self
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 66d8fdaac90..63ae1ef33b2 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -48,7 +48,6 @@
 from .utils import (
     All,
     ignoring,
-    get_ip,
     get_fileno_limit,
     log_errors,
     key_split,
@@ -842,7 +841,7 @@ def __init__(
         idle_timeout=None,
         interface=None,
         host=None,
-        port=8786,
+        port=0,
         protocol=None,
         dashboard_address=None,
         **kwargs
@@ -1098,6 +1097,7 @@ def __init__(
             interface=interface,
             protocol=protocol,
             security=security,
+            default_port=self.default_port,
         )
 
         super(Scheduler, self).__init__(
@@ -1177,12 +1177,11 @@ def get_worker_service_addr(self, worker, service_name, protocol=False):
         else:
             return ws.host, port
 
-    def start(self, addr_or_port=None, start_queues=True):
+    @gen.coroutine
+    def start(self):
         """ Clear out old state and restart all running coroutines """
         enable_gc_diagnosis()
 
-        addr_or_port = addr_or_port or self._start_address
-
         self.clear_task_state()
 
         with ignoring(AttributeError):
@@ -1196,21 +1195,14 @@ def start(self, addr_or_port=None, start_queues=True):
                     raise exc
 
         if self.status != "running":
-            if isinstance(addr_or_port, int):
-                # Listen on all interfaces.  `get_ip()` is not suitable
-                # as it would prevent connecting via 127.0.0.1.
-                self.listen(("", addr_or_port), listen_args=self.listen_args)
-                self.ip = get_ip()
-                listen_ip = ""
-            else:
-                self.listen(addr_or_port, listen_args=self.listen_args)
-                self.ip = get_address_host(self.listen_address)
-                listen_ip = self.ip
+            self.listen(self._start_address, listen_args=self.listen_args)
+            self.ip = get_address_host(self.listen_address)
+            listen_ip = self.ip
 
             if listen_ip == "0.0.0.0":
                 listen_ip = ""
 
-            if isinstance(addr_or_port, str) and addr_or_port.startswith("inproc://"):
+            if self._start_address.startswith("inproc://"):
                 listen_ip = "localhost"
 
             # Services listen on all addresses
@@ -1239,16 +1231,8 @@ def del_scheduler_file():
 
         setproctitle("dask-scheduler [%s]" % (self.address,))
 
-        return self.finished()
-
-    def __await__(self):
-        self.start()
-
-        @gen.coroutine
-        def _():
-            return self
-
-        return _().__await__()
+        yield self.finished()
+        return self
 
     @gen.coroutine
     def finished(self):
diff --git a/distributed/tests/test_core.py b/distributed/tests/test_core.py
index f53340d1004..38a43a1a5c8 100644
--- a/distributed/tests/test_core.py
+++ b/distributed/tests/test_core.py
@@ -373,13 +373,13 @@ def g():
     server = Server({"ping": pingpong})
     server.listen(listen_arg)
 
-    remote = rpc(server.address)
-    yield [g() for i in range(10)]
+    with rpc(server.address) as remote:
+        yield [g() for i in range(10)]
 
-    server.stop()
+        server.stop()
 
-    remote.close_comms()
-    assert all(comm.closed() for comm in remote.comms)
+        remote.close_comms()
+        assert all(comm.closed() for comm in remote.comms)
 
 
 @gen_test()
diff --git a/distributed/tests/test_failed_workers.py b/distributed/tests/test_failed_workers.py
index b39dd3f3ae7..5465a7dd5f0 100644
--- a/distributed/tests/test_failed_workers.py
+++ b/distributed/tests/test_failed_workers.py
@@ -39,8 +39,7 @@ def test_submit_after_failed_worker_sync(loop):
 
 @gen_cluster(client=True, timeout=60, active_rpc_timeout=10)
 def test_submit_after_failed_worker_async(c, s, a, b):
-    n = Nanny(s.address, nthreads=2, loop=s.loop)
-    n.start(0)
+    n = yield Nanny(s.address, nthreads=2, loop=s.loop)
     while len(s.workers) < 3:
         yield gen.sleep(0.1)
 
@@ -128,7 +127,7 @@ def test_failed_worker_without_warning(c, s, a, b):
     assert all(len(keys) > 0 for keys in s.has_what.values())
     nthreads2 = dict(s.nthreads)
 
-    yield c._restart()
+    yield c.restart()
 
     L = c.map(inc, range(10))
     yield wait(L)
@@ -148,7 +147,7 @@ def test_restart(c, s, a, b):
 
     assert set(s.who_has) == {x.key, y.key}
 
-    f = yield c._restart()
+    f = yield c.restart()
     assert f is c
 
     assert len(s.workers) == 2
@@ -171,7 +170,7 @@ def test_restart_cleared(c, s, a, b):
     f = c.compute(x)
     yield wait([f])
 
-    yield c._restart()
+    yield c.restart()
 
     for coll in [s.tasks, s.unrunnable]:
         assert not coll
@@ -212,7 +211,7 @@ def test_restart_fast(c, s, a, b):
     L = c.map(sleep, range(10))
 
     start = time()
-    yield c._restart()
+    yield c.restart()
     assert time() - start < 10
     assert len(s.nthreads) == 2
 
@@ -255,7 +254,7 @@ def test_fast_kill(c, s, a, b):
     L = c.map(sleep, range(10))
 
     start = time()
-    yield c._restart()
+    yield c.restart()
     assert time() - start < 10
 
     assert all(x.status == "cancelled" for x in L)
@@ -302,7 +301,7 @@ def test_restart_scheduler(s, a, b):
 @gen_cluster(Worker=Nanny, client=True, timeout=60)
 def test_forgotten_futures_dont_clean_up_new_futures(c, s, a, b):
     x = c.submit(inc, 1)
-    yield c._restart()
+    yield c.restart()
     y = c.submit(inc, 1)
     del x
     import gc
@@ -315,8 +314,7 @@ def test_forgotten_futures_dont_clean_up_new_futures(c, s, a, b):
 @gen_cluster(client=True, timeout=60, active_rpc_timeout=10)
 def test_broken_worker_during_computation(c, s, a, b):
     s.allowed_failures = 100
-    n = Nanny(s.address, nthreads=2, loop=s.loop)
-    n.start(0)
+    n = yield Nanny(s.address, nthreads=2, loop=s.loop)
 
     start = time()
     while len(s.nthreads) < 3:
@@ -365,7 +363,7 @@ def test_restart_during_computation(c, s, a, b):
 
     yield gen.sleep(0.5)
     assert s.rprocessing
-    yield c._restart()
+    yield c.restart()
     assert not s.rprocessing
 
     assert len(s.nthreads) == 2
@@ -374,8 +372,7 @@ def test_restart_during_computation(c, s, a, b):
 
 @gen_cluster(client=True, timeout=60)
 def test_worker_who_has_clears_after_failed_connection(c, s, a, b):
-    n = Nanny(s.address, nthreads=2, loop=s.loop)
-    n.start(0)
+    n = yield Nanny(s.address, nthreads=2, loop=s.loop)
 
     start = time()
     while len(s.nthreads) < 3:
diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
index 40c8d49012d..7722476a2c5 100644
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -216,8 +216,7 @@ def test_num_fds(s):
 @gen_cluster(client=True, nthreads=[])
 def test_worker_uses_same_host_as_nanny(c, s):
     for host in ["tcp://0.0.0.0", "tcp://127.0.0.2"]:
-        n = Nanny(s.address)
-        yield n._start(host)
+        n = yield Nanny(s.address, host=host)
 
         def func(dask_worker):
             return dask_worker.listener.listen_address
@@ -230,8 +229,7 @@ def func(dask_worker):
 @gen_test()
 def test_scheduler_file():
     with tmpfile() as fn:
-        s = Scheduler(scheduler_file=fn)
-        s.start(8008)
+        s = yield Scheduler(scheduler_file=fn, port=8008)
         w = yield Nanny(scheduler_file=fn)
         assert set(s.workers) == {w.worker_address}
         yield w.close()
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index e8d2a96ee60..3d29bc79a1e 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -256,7 +256,6 @@ def test_add_worker(s, a, b):
     w = Worker(s.address, nthreads=3)
     w.data["x-5"] = 6
     w.data["y"] = 1
-    yield w
 
     dsk = {("x-%d" % i): (inc, i) for i in range(10)}
     s.update_graph(
@@ -265,11 +264,8 @@ def test_add_worker(s, a, b):
         client="client",
         dependencies={k: set() for k in dsk},
     )
-
-    s.add_worker(
-        address=w.address, keys=list(w.data), nthreads=w.nthreads, services=s.services
-    )
-
+    s.validate_state()
+    yield w
     s.validate_state()
 
     assert w.ip in s.host_info
@@ -665,7 +661,7 @@ def test_scatter_no_workers(c, s):
     assert time() < start + 1.5
 
     w = Worker(s.address, nthreads=3)
-    yield [c.scatter(data={"y": 2}, timeout=5), w._start()]
+    yield [c.scatter(data={"y": 2}, timeout=5), w]
 
     assert w.data["y"] == 2
     yield w.close()
@@ -1172,7 +1168,7 @@ def test_correct_bad_time_estimate(c, s, *workers):
 
 
 @gen_test()
-def test_service_hosts():
+async def test_service_hosts():
     pytest.importorskip("bokeh")
     from distributed.dashboard import BokehScheduler
 
@@ -1184,26 +1180,20 @@ def test_service_hosts():
     ]:
         services = {("dashboard", port): BokehScheduler}
 
-        s = Scheduler(services=services)
-        yield s.start(url)
-
-        sock = first(s.services["dashboard"].server._http._sockets.values())
-        if isinstance(expected, tuple):
-            assert sock.getsockname()[0] in expected
-        else:
-            assert sock.getsockname()[0] == expected
-        yield s.close()
+        async with Scheduler(host=url, services=services) as s:
+            sock = first(s.services["dashboard"].server._http._sockets.values())
+            if isinstance(expected, tuple):
+                assert sock.getsockname()[0] in expected
+            else:
+                assert sock.getsockname()[0] == expected
 
     port = ("127.0.0.1", 0)
     for url in ["tcp://0.0.0.0", "tcp://127.0.0.1", "tcp://127.0.0.1:38275"]:
         services = {("dashboard", port): BokehScheduler}
 
-        s = Scheduler(services=services)
-        yield s.start(url)
-
-        sock = first(s.services["dashboard"].server._http._sockets.values())
-        assert sock.getsockname()[0] == "127.0.0.1"
-        yield s.close()
+        async with Scheduler(services=services, host=url) as s:
+            sock = first(s.services["dashboard"].server._http._sockets.values())
+            assert sock.getsockname()[0] == "127.0.0.1"
 
 
 @gen_cluster(client=True, worker_kwargs={"profile_cycle_interval": 100})
diff --git a/distributed/tests/test_stress.py b/distributed/tests/test_stress.py
index 81d7c4360f7..6a5dbe72736 100644
--- a/distributed/tests/test_stress.py
+++ b/distributed/tests/test_stress.py
@@ -108,11 +108,8 @@ def test_stress_creation_and_deletion(c, s):
     def create_and_destroy_worker(delay):
         start = time()
         while time() < start + 5:
-            n = Nanny(s.address, nthreads=2, loop=s.loop)
-            n.start(0)
-
+            n = yield Nanny(s.address, nthreads=2, loop=s.loop)
             yield gen.sleep(delay)
-
             yield n.close()
             print("Killed nanny")
 
diff --git a/distributed/tests/test_utils_test.py b/distributed/tests/test_utils_test.py
index c0afb9e2c7f..eac2ec71529 100755
--- a/distributed/tests/test_utils_test.py
+++ b/distributed/tests/test_utils_test.py
@@ -176,10 +176,9 @@ def test_tls_cluster(tls_client):
 
 
 def test_tls_scheduler(security, loop):
-    s = Scheduler(security=security, loop=loop)
-    s.start("localhost")
+    s = yield Scheduler(security=security, loop=loop, host="localhost")
     assert s.address.startswith("tls")
-    s.close()
+    yield s.close()
 
 
 if sys.version_info >= (3, 5):
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 562a0e037b7..88186885db1 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -22,10 +22,18 @@
 from tornado import gen
 from tornado.ioloop import TimeoutError
 
-from distributed import Nanny, get_client, wait, default_client, get_worker, Reschedule
+from distributed import (
+    Client,
+    Nanny,
+    get_client,
+    wait,
+    default_client,
+    get_worker,
+    Reschedule,
+    wait,
+)
 from distributed.compatibility import WINDOWS, cache_from_source
 from distributed.core import rpc
-from distributed.client import wait
 from distributed.scheduler import Scheduler
 from distributed.metrics import time
 from distributed.worker import Worker, error_message, logger, parse_memory_limit
@@ -978,20 +986,23 @@ def test_service_hosts_match_worker(s):
 
     services = {("dashboard", ":0"): BokehWorker}
 
-    w = Worker(s.address, services={("dashboard", ":0"): BokehWorker})
-    yield w._start("tcp://0.0.0.0")
+    w = yield Worker(
+        s.address, services={("dashboard", ":0"): BokehWorker}, host="tcp://0.0.0.0"
+    )
     sock = first(w.services["dashboard"].server._http._sockets.values())
     assert sock.getsockname()[0] in ("::", "0.0.0.0")
     yield w.close()
 
-    w = Worker(s.address, services={("dashboard", ":0"): BokehWorker})
-    yield w._start("tcp://127.0.0.1")
+    w = yield Worker(
+        s.address, services={("dashboard", ":0"): BokehWorker}, host="tcp://127.0.0.1"
+    )
     sock = first(w.services["dashboard"].server._http._sockets.values())
     assert sock.getsockname()[0] in ("::", "0.0.0.0")
     yield w.close()
 
-    w = Worker(s.address, services={("dashboard", 0): BokehWorker})
-    yield w._start("tcp://127.0.0.1")
+    w = yield Worker(
+        s.address, services={("dashboard", 0): BokehWorker}, host="tcp://127.0.0.1"
+    )
     sock = first(w.services["dashboard"].server._http._sockets.values())
     assert sock.getsockname()[0] == "127.0.0.1"
     yield w.close()
@@ -1004,8 +1015,7 @@ def test_start_services(s):
 
     services = {("dashboard", ":1234"): BokehWorker}
 
-    w = Worker(s.address, services=services)
-    yield w._start()
+    w = yield Worker(s.address, services=services)
 
     assert w.services["dashboard"].server.port == 1234
     yield w.close()
@@ -1440,3 +1450,44 @@ def test_resource_limit():
         assert parse_memory_limit(hard_limit, 1, total_cores=1) == new_limit
     except OSError:
         pytest.skip("resource could not set the RSS limit")
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("Worker", [Worker, Nanny])
+async def test_interface_async(loop, Worker):
+    from distributed.utils import get_ip_interface
+
+    psutil = pytest.importorskip("psutil")
+    if_names = sorted(psutil.net_if_addrs())
+    for if_name in if_names:
+        try:
+            ipv4_addr = get_ip_interface(if_name)
+        except ValueError:
+            pass
+        else:
+            if ipv4_addr == "127.0.0.1":
+                break
+    else:
+        pytest.skip(
+            "Could not find loopback interface. "
+            "Available interfaces are: %s." % (if_names,)
+        )
+
+    async with Scheduler(interface=if_name) as s:
+        assert s.address.startswith("tcp://127.0.0.1")
+        async with Worker(s.address, interface=if_name) as w:
+            assert w.address.startswith("tcp://127.0.0.1")
+            assert w.ip == "127.0.0.1"
+            async with Client(s.address, asynchronous=True) as c:
+                info = c.scheduler_info()
+                assert "tcp://127.0.0.1" in info["address"]
+                assert all("127.0.0.1" == d["host"] for d in info["workers"].values())
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("Worker", [Worker, Nanny])
+async def test_worker_listens_on_same_interface_by_default(Worker):
+    async with Scheduler(host="localhost") as s:
+        assert s.ip in {"127.0.0.1", "localhost"}
+        async with Worker(s.address) as w:
+            assert s.ip == w.ip
diff --git a/distributed/tests/test_worker_plugins.py b/distributed/tests/test_worker_plugins.py
index 425a267923a..bbba39943fb 100644
--- a/distributed/tests/test_worker_plugins.py
+++ b/distributed/tests/test_worker_plugins.py
@@ -23,8 +23,7 @@ def teardown(self, worker):
 def test_create_with_client(c, s):
     yield c.register_worker_plugin(MyPlugin(123))
 
-    worker = Worker(s.address, loop=s.loop)
-    yield worker._start()
+    worker = yield Worker(s.address, loop=s.loop)
     assert worker._my_plugin_status == "setup"
     assert worker._my_plugin_data == 123
 
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index 77568bb7595..af28fe66168 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -486,7 +486,7 @@ def run_worker(q, scheduler_q, **kwargs):
         with pristine_loop() as loop:
             scheduler_addr = scheduler_q.get()
             worker = Worker(scheduler_addr, validate=True, **kwargs)
-            loop.run_sync(lambda: worker._start())
+            loop.run_sync(worker.start)
             q.put(worker.address)
             try:
 
@@ -504,7 +504,7 @@ def run_nanny(q, scheduler_q, **kwargs):
         with pristine_loop() as loop:
             scheduler_addr = scheduler_q.get()
             worker = Nanny(scheduler_addr, validate=True, **kwargs)
-            loop.run_sync(lambda: worker._start())
+            loop.run_sync(worker.start)
             q.put(worker.address)
             try:
                 loop.start()
@@ -794,9 +794,14 @@ def start_cluster(
     worker_kwargs={},
 ):
     s = Scheduler(
-        loop=loop, validate=True, security=security, port=0, **scheduler_kwargs
+        loop=loop,
+        validate=True,
+        security=security,
+        port=0,
+        host=scheduler_addr,
+        **scheduler_kwargs
     )
-    done = s.start(scheduler_addr)
+    done = s.start()
     workers = [
         Worker(
             s.address,
diff --git a/distributed/worker.py b/distributed/worker.py
index 60884ebaba3..791fc0ba101 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -32,7 +32,7 @@
 
 from . import profile, comm
 from .batched import BatchedSend
-from .comm import get_address_host, get_local_address_for, connect
+from .comm import get_address_host, connect
 from .comm.utils import offload
 from .comm.addressing import address_from_user_args
 from .compatibility import unicode, get_thread_identity, MutableMapping
@@ -423,6 +423,11 @@ def __init__(
             scheduler_addr = coerce_to_address((scheduler_ip, scheduler_port))
         self.contact_address = contact_address
 
+        # Target interface on which we contact the scheduler by default
+        # TODO: it is unfortunate that we special-case inproc here
+        if not host and not interface and not scheduler_addr.startswith("inproc://"):
+            host = get_ip(get_address_host(scheduler_addr))
+
         self._start_address = address_from_user_args(
             host=host,
             port=port,
@@ -892,39 +897,18 @@ def gather(self, comm=None, who_has=None):
     #############
 
     @gen.coroutine
-    def _start(self, addr_or_port=0):
+    def start(self):
         assert self.status is None
-        addr_or_port = addr_or_port or self._start_address
 
         enable_gc_diagnosis()
         thread_state.on_event_loop_thread = True
 
-        # XXX Factor this out
-        if not addr_or_port:
-            # Default address is the required one to reach the scheduler
-            listen_host = get_address_host(self.scheduler.address)
-            self.listen(
-                get_local_address_for(self.scheduler.address),
-                listen_args=self.listen_args,
-            )
-            self.ip = get_address_host(self.address)
-        elif isinstance(addr_or_port, int):
-            # addr_or_port is an integer => assume TCP
-            listen_host = self.ip = get_ip(get_address_host(self.scheduler.address))
-            self.listen((listen_host, addr_or_port), listen_args=self.listen_args)
-        else:
-            self.listen(addr_or_port, listen_args=self.listen_args)
-            self.ip = get_address_host(self.address)
-            try:
-                listen_host = get_address_host(addr_or_port)
-            except ValueError:
-                listen_host = addr_or_port
-
-        if "://" in listen_host:
-            protocol, listen_host = listen_host.split("://")
+        self.listen(self._start_address, listen_args=self.listen_args)
+        self.ip = get_address_host(self.address)
 
         if self.name is None:
             self.name = self.address
+
         preload_modules(
             self.preload,
             parameter=self,
@@ -934,21 +918,17 @@ def _start(self, addr_or_port=0):
         # Services listen on all addresses
         # Note Nanny is not a "real" service, just some metadata
         # passed in service_ports...
-        self.start_services(listen_host)
+        self.start_services(self.ip)
 
         try:
-            listening_address = "%s%s:%d" % (
-                self.listener.prefix,
-                listen_host,
-                self.port,
-            )
+            listening_address = "%s%s:%d" % (self.listener.prefix, self.ip, self.port)
         except Exception:
-            listening_address = "%s%s" % (self.listener.prefix, listen_host)
+            listening_address = "%s%s" % (self.listener.prefix, self.ip)
 
         logger.info("      Start worker at: %26s", self.address)
         logger.info("         Listening to: %26s", listening_address)
         for k, v in self.service_ports.items():
-            logger.info("  %16s at: %26s" % (k, listen_host + ":" + str(v)))
+            logger.info("  %16s at: %26s" % (k, self.ip + ":" + str(v)))
         logger.info("Waiting to connect to: %26s", self.scheduler.address)
         logger.info("-" * 49)
         logger.info("              Threads: %26d", self.nthreads)
@@ -964,21 +944,7 @@ def _start(self, addr_or_port=0):
         yield self._register_with_scheduler()
 
         self.start_periodic_callbacks()
-        raise gen.Return(self)
-
-    def __await__(self):
-        if self.status is not None:
-
-            @gen.coroutine  # idempotent
-            def _():
-                raise gen.Return(self)
-
-            return _().__await__()
-        else:
-            return self._start().__await__()
-
-    def start(self, port=0):
-        self.loop.add_callback(self._start, port)
+        return self
 
     def _close(self, *args, **kwargs):
         warnings.warn("Worker._close has moved to Worker.close", stacklevel=2)

From 2a145ac5ccbe34af44ec4d2814897a87b3fef3e7 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sat, 20 Jul 2019 13:19:40 -0500
Subject: [PATCH 0361/1550] Add cleanup fixture for asyncio tests (#2866)

---
 distributed/core.py                           |  4 ++--
 distributed/deploy/tests/test_local.py        |  5 +++--
 distributed/deploy/tests/test_spec_cluster.py | 10 +++++-----
 distributed/tests/test_scheduler.py           |  7 ++++---
 distributed/tests/test_worker.py              | 13 ++++++-------
 distributed/utils_test.py                     | 13 +++++++++++++
 6 files changed, 33 insertions(+), 19 deletions(-)

diff --git a/distributed/core.py b/distributed/core.py
index 6f08c17ac77..4d18547151a 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -154,11 +154,11 @@ def __init__(
         if not hasattr(self.io_loop, "profile"):
             ref = weakref.ref(self.io_loop)
 
-            if hasattr(self.io_loop, "closing"):
+            if hasattr(self.io_loop, "asyncio_loop"):
 
                 def stop():
                     loop = ref()
-                    return loop is None or loop.closing
+                    return loop is None or loop.asyncio_loop.is_closed()
 
             else:
 
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index ebe2d7ec99e..d489b84df0f 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -17,8 +17,9 @@
 from distributed import Client, Worker, Nanny, get_client
 from distributed.deploy.local import LocalCluster, nprocesses_nthreads
 from distributed.metrics import time
-from distributed.utils_test import (
+from distributed.utils_test import (  # noqa: F401
     clean,
+    cleanup,
     inc,
     gen_test,
     slowinc,
@@ -801,7 +802,7 @@ class MyNanny(Nanny):
 
 
 @pytest.mark.asyncio
-async def test_worker_class_nanny_async():
+async def test_worker_class_nanny_async(cleanup):
     class MyNanny(Nanny):
         pass
 
diff --git a/distributed/deploy/tests/test_spec_cluster.py b/distributed/deploy/tests/test_spec_cluster.py
index 723c62a80c1..bb992f8b7c7 100644
--- a/distributed/deploy/tests/test_spec_cluster.py
+++ b/distributed/deploy/tests/test_spec_cluster.py
@@ -1,6 +1,6 @@
 from dask.distributed import SpecCluster, Worker, Client, Scheduler, Nanny
 from distributed.deploy.spec import close_clusters
-from distributed.utils_test import loop  # noqa: F401
+from distributed.utils_test import loop, cleanup  # noqa: F401
 import pytest
 
 
@@ -25,7 +25,7 @@ async def _():
 
 
 @pytest.mark.asyncio
-async def test_specification():
+async def test_specification(cleanup):
     async with SpecCluster(
         workers=worker_spec, scheduler=scheduler, asynchronous=True
     ) as cluster:
@@ -82,7 +82,7 @@ def test_loop_started():
 
 
 @pytest.mark.asyncio
-async def test_repr():
+async def test_repr(cleanup):
     worker = {"cls": Worker, "options": {"nthreads": 1}}
 
     class MyCluster(SpecCluster):
@@ -95,7 +95,7 @@ class MyCluster(SpecCluster):
 
 
 @pytest.mark.asyncio
-async def test_scale():
+async def test_scale(cleanup):
     worker = {"cls": Worker, "options": {"nthreads": 1}}
     async with SpecCluster(
         asynchronous=True, scheduler=scheduler, worker=worker
@@ -143,7 +143,7 @@ def test_spec_close_clusters(loop):
 
 
 @pytest.mark.asyncio
-async def test_new_worker_spec():
+async def test_new_worker_spec(cleanup):
     class MyCluster(SpecCluster):
         def new_worker_spec(self):
             i = len(self.worker_spec)
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 3d29bc79a1e..9b512ddad64 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -24,7 +24,8 @@
 from distributed.protocol.pickle import dumps
 from distributed.worker import dumps_function, dumps_task
 from distributed.utils import tmpfile
-from distributed.utils_test import (
+from distributed.utils_test import (  # noqa: F401
+    cleanup,
     inc,
     dec,
     gen_cluster,
@@ -1590,7 +1591,7 @@ async def test_adaptive_target(c, s, a, b):
 
 
 @pytest.mark.asyncio
-async def test_async_context_manager():
+async def test_async_context_manager(cleanup):
     async with Scheduler(port=0) as s:
         assert s.status == "running"
         async with Worker(s.address) as w:
@@ -1600,7 +1601,7 @@ async def test_async_context_manager():
 
 
 @pytest.mark.asyncio
-async def test_allowed_failures_config():
+async def test_allowed_failures_config(cleanup):
     async with Scheduler(port=0, allowed_failures=10) as s:
         assert s.allowed_failures == 10
 
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 88186885db1..2e6af6e0bdc 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -38,7 +38,8 @@
 from distributed.metrics import time
 from distributed.worker import Worker, error_message, logger, parse_memory_limit
 from distributed.utils import tmpfile
-from distributed.utils_test import (
+from distributed.utils_test import (  # noqa: F401
+    cleanup,
     inc,
     mul,
     gen_cluster,
@@ -370,12 +371,10 @@ def test_gather(s, a, b):
 
 
 @pytest.mark.asyncio
-async def test_io_loop():
-    s = await Scheduler(port=0)
-    w = await Worker(s.address, loop=s.loop)
-    assert w.io_loop is s.loop
-    await s.close()
-    await w.close()
+async def test_io_loop(cleanup):
+    async with Scheduler(port=0) as s:
+        async with Worker(s.address, loop=s.loop) as w:
+            assert w.io_loop is s.loop
 
 
 @gen_cluster(client=True, nthreads=[])
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index af28fe66168..f0e75f8ed2c 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -1559,3 +1559,16 @@ def null():
 
                         with ignoring(AttributeError):
                             del thread_state.on_event_loop_thread
+
+
+@pytest.fixture
+def cleanup():
+    with check_thread_leak():
+        with check_process_leak():
+            with check_instances():
+                reset_config()
+                dask.config.set({"distributed.comm.timeouts.connect": "5s"})
+                for name, level in logging_levels.items():
+                    logging.getLogger(name).setLevel(level)
+
+                yield

From 967d97128ed6fa169231a83e6a7b1d22a92c9111 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sat, 20 Jul 2019 13:20:22 -0500
Subject: [PATCH 0362/1550]  Use only remote connection to scheduler in
 Adaptive (#2865)

This modifies the Adaptive class to only touch the scheduler through
communication, rather than direct access.  This should enable adaptive
scheduling when the scheduler is deployed in a remote process.

Fixes #2858

* Pickle worker_key function in adaptive
---
 distributed/deploy/adaptive.py            | 98 +++++++++-------------
 distributed/deploy/cluster.py             |  4 +-
 distributed/deploy/tests/test_adaptive.py | 99 ++++++++---------------
 distributed/distributed.yaml              |  1 +
 distributed/scheduler.py                  | 19 ++++-
 5 files changed, 94 insertions(+), 127 deletions(-)

diff --git a/distributed/deploy/adaptive.py b/distributed/deploy/adaptive.py
index 401acc3dc1d..761a7d300ee 100644
--- a/distributed/deploy/adaptive.py
+++ b/distributed/deploy/adaptive.py
@@ -2,12 +2,12 @@
 
 from collections import deque
 import logging
-import math
 
 from tornado import gen
 
 from ..metrics import time
 from ..utils import log_errors, PeriodicCallback, parse_timedelta
+from ..protocol import pickle
 
 logger = logging.getLogger(__name__)
 
@@ -80,14 +80,10 @@ class Adaptive(object):
     resized. The default implementation checks if there are too many tasks
     per worker or too little memory available (see :meth:`Adaptive.needs_cpu`
     and :meth:`Adaptive.needs_memory`).
-
-    :meth:`Adaptive.get_scale_up_kwargs` method controls the arguments passed to
-    the cluster's ``scale_up`` method.
     '''
 
     def __init__(
         self,
-        scheduler,
         cluster=None,
         interval="1s",
         startup_cost="1s",
@@ -96,20 +92,19 @@ def __init__(
         maximum=None,
         wait_count=3,
         target_duration="5s",
-        worker_key=lambda x: x,
+        worker_key=None,
         **kwargs
     ):
         interval = parse_timedelta(interval, default="ms")
         self.worker_key = worker_key
-        self.scheduler = scheduler
         self.cluster = cluster
         self.startup_cost = parse_timedelta(startup_cost, default="s")
         self.scale_factor = scale_factor
         if self.cluster:
             self._adapt_callback = PeriodicCallback(
-                self._adapt, interval * 1000, io_loop=scheduler.loop
+                self._adapt, interval * 1000, io_loop=self.loop
             )
-            self.scheduler.loop.add_callback(self._adapt_callback.start)
+            self.loop.add_callback(self._adapt_callback.start)
         self._adapting = False
         self._workers_to_close_kwargs = kwargs
         self.minimum = minimum
@@ -119,7 +114,9 @@ def __init__(
         self.wait_count = wait_count
         self.target_duration = parse_timedelta(target_duration)
 
-        self.scheduler.handlers["adaptive_recommendations"] = self.recommendations
+    @property
+    def scheduler(self):
+        return self.cluster.scheduler_comm
 
     def stop(self):
         if self.cluster:
@@ -127,7 +124,7 @@ def stop(self):
             self._adapt_callback = None
             del self._adapt_callback
 
-    def workers_to_close(self, **kwargs):
+    async def workers_to_close(self, **kwargs):
         """
         Determine which, if any, workers should potentially be removed from
         the cluster.
@@ -145,73 +142,53 @@ def workers_to_close(self, **kwargs):
         --------
         Scheduler.workers_to_close
         """
-        if len(self.scheduler.workers) <= self.minimum:
+        if len(self.cluster.workers) <= self.minimum:
             return []
 
         kw = dict(self._workers_to_close_kwargs)
         kw.update(kwargs)
 
-        if self.maximum is not None and len(self.scheduler.workers) > self.maximum:
-            kw["n"] = len(self.scheduler.workers) - self.maximum
+        if self.maximum is not None and len(self.cluster.workers) > self.maximum:
+            kw["n"] = len(self.cluster.workers) - self.maximum
 
-        L = self.scheduler.workers_to_close(**kw)
-        if len(self.scheduler.workers) - len(L) < self.minimum:
-            L = L[: len(self.scheduler.workers) - self.minimum]
+        L = await self.scheduler.workers_to_close(**kw)
+        if len(self.cluster.workers) - len(L) < self.minimum:
+            L = L[: len(self.cluster.workers) - self.minimum]
 
         return L
 
-    @gen.coroutine
-    def _retire_workers(self, workers=None):
+    async def _retire_workers(self, workers=None):
         if workers is None:
-            workers = self.workers_to_close(key=self.worker_key, minimum=self.minimum)
+            workers = await self.workers_to_close(
+                key=pickle.dumps(self.worker_key) if self.worker_key else None,
+                minimum=self.minimum,
+            )
         if not workers:
             raise gen.Return(workers)
         with log_errors():
-            yield self.scheduler.retire_workers(
+            await self.scheduler.retire_workers(
                 workers=workers, remove=True, close_workers=True
             )
 
             logger.info("Retiring workers %s", workers)
             f = self.cluster.scale_down(workers)
             if hasattr(f, "__await__"):
-                yield f
-
-            raise gen.Return(workers)
-
-    def get_scale_up_kwargs(self):
-        """
-        Get the arguments to be passed to ``self.cluster.scale_up``.
-
-        Notes
-        -----
-        By default the desired number of total workers is returned (``n``).
-        Subclasses should ensure that the return dictionary includes a key-
-        value pair for ``n``, either by implementing it or by calling the
-        parent's ``get_scale_up_kwargs``.
+                await f
 
-        See Also
-        --------
-        LocalCluster.scale_up
-        """
-        target = math.ceil(self.scheduler.total_occupancy / self.target_duration)
-        instances = max(
-            1, len(self.scheduler.workers) * self.scale_factor, target, self.minimum
-        )
-
-        if self.maximum:
-            instances = min(self.maximum, instances)
+            return workers
 
-        instances = int(instances)
-        logger.info("Scaling up to %d workers", instances)
-        return {"n": instances}
-
-    def recommendations(self, comm=None):
-        n = self.scheduler.adaptive_target(target_duration=self.target_duration)
+    async def recommendations(self, comm=None):
+        n = await self.scheduler.adaptive_target(target_duration=self.target_duration)
         if self.maximum is not None:
             n = min(self.maximum, n)
         if self.minimum is not None:
             n = max(self.minimum, n)
-        workers = set(self.workers_to_close(key=self.worker_key, minimum=self.minimum))
+        workers = set(
+            await self.workers_to_close(
+                key=pickle.dumps(self.worker_key) if self.worker_key else None,
+                minimum=self.minimum,
+            )
+        )
         try:
             current = len(self.cluster.worker_spec)
         except AttributeError:
@@ -249,14 +226,13 @@ def recommendations(self, comm=None):
             self.close_counts.clear()
             return None
 
-    @gen.coroutine
-    def _adapt(self):
+    async def _adapt(self):
         if self._adapting:  # Semaphore to avoid overlapping adapt calls
             return
 
         self._adapting = True
         try:
-            recommendations = self.recommendations()
+            recommendations = await self.recommendations()
             if not recommendations:
                 return
             status = recommendations.pop("status")
@@ -264,13 +240,17 @@ def _adapt(self):
                 f = self.cluster.scale(**recommendations)
                 self.log.append((time(), "up", recommendations))
                 if hasattr(f, "__await__"):
-                    yield f
+                    await f
 
             elif status == "down":
                 self.log.append((time(), "down", recommendations["workers"]))
-                workers = yield self._retire_workers(workers=recommendations["workers"])
+                workers = await self._retire_workers(workers=recommendations["workers"])
         finally:
             self._adapting = False
 
     def adapt(self):
-        self.scheduler.loop.add_callback(self._adapt)
+        self.loop.add_callback(self._adapt)
+
+    @property
+    def loop(self):
+        return self.cluster.loop
diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
index d48f27603ff..f5d991cd737 100644
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -62,7 +62,7 @@ def scale_down(self, workers: List[str]):
     LocalCluster: a simple implementation with local workers
     """
 
-    def adapt(self, **kwargs):
+    def adapt(self, Adaptive=Adaptive, **kwargs):
         """ Turn on adaptivity
 
         For keyword arguments see dask.distributed.Adaptive
@@ -76,7 +76,7 @@ def adapt(self, **kwargs):
         if not hasattr(self, "_adaptive_options"):
             self._adaptive_options = {}
         self._adaptive_options.update(kwargs)
-        self._adaptive = Adaptive(self.scheduler, self, **self._adaptive_options)
+        self._adaptive = Adaptive(self, **self._adaptive_options)
         return self._adaptive
 
     @property
diff --git a/distributed/deploy/tests/test_adaptive.py b/distributed/deploy/tests/test_adaptive.py
index 146d7b95dbb..2258a83cfe5 100644
--- a/distributed/deploy/tests/test_adaptive.py
+++ b/distributed/deploy/tests/test_adaptive.py
@@ -2,33 +2,19 @@
 
 from time import sleep
 
+import pytest
 from toolz import frequencies, pluck
 from tornado import gen
 from tornado.ioloop import IOLoop
 
 from distributed import Client, wait, Adaptive, LocalCluster, SpecCluster, Worker
-from distributed.utils_test import gen_cluster, gen_test, slowinc, clean
+from distributed.utils_test import gen_test, slowinc, clean
 from distributed.utils_test import loop, nodebug  # noqa: F401
 from distributed.metrics import time
 
 
-def test_get_scale_up_kwargs(loop):
-    with LocalCluster(
-        0, scheduler_port=0, silence_logs=False, dashboard_address=None, loop=loop
-    ) as cluster:
-
-        alc = Adaptive(cluster.scheduler, cluster, interval=100, scale_factor=3)
-        assert alc.get_scale_up_kwargs() == {"n": 1}
-
-        with Client(cluster, loop=loop) as c:
-            future = c.submit(lambda x: x + 1, 1)
-            assert future.result() == 2
-            assert c.nthreads()
-            assert alc.get_scale_up_kwargs() == {"n": 3}
-
-
-@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 4)
-def test_simultaneous_scale_up_and_down(c, s, *workers):
+@pytest.mark.asyncio
+async def test_simultaneous_scale_up_and_down():
     class TestAdaptive(Adaptive):
         def get_scale_up_kwargs(self):
             assert False
@@ -36,34 +22,35 @@ def get_scale_up_kwargs(self):
         def _retire_workers(self):
             assert False
 
-    class TestCluster(object):
+    class TestCluster(LocalCluster):
         def scale_up(self, n, **kwargs):
             assert False
 
         def scale_down(self, workers):
             assert False
 
-    cluster = TestCluster()
+    async with TestCluster(n_workers=4, processes=False, asynchronous=True) as cluster:
+        async with Client(cluster, asynchronous=True) as c:
+            s = cluster.scheduler
+            s.task_duration["a"] = 4
+            s.task_duration["b"] = 4
+            s.task_duration["c"] = 1
 
-    s.task_duration["a"] = 4
-    s.task_duration["b"] = 4
-    s.task_duration["c"] = 1
+            future = c.map(slowinc, [1, 1, 1], key=["a-4", "b-4", "c-1"])
 
-    future = c.map(slowinc, [1, 1, 1], key=["a-4", "b-4", "c-1"])
+            while len(s.rprocessing) < 3:
+                await gen.sleep(0.001)
 
-    while len(s.rprocessing) < 3:
-        yield gen.sleep(0.001)
+            ta = cluster.adapt(interval="100 ms", scale_factor=2, Adaptive=TestAdaptive)
 
-    ta = TestAdaptive(s, cluster, interval=100, scale_factor=2)
-
-    yield gen.sleep(0.3)
+            await gen.sleep(0.3)
 
 
 def test_adaptive_local_cluster(loop):
     with LocalCluster(
         0, scheduler_port=0, silence_logs=False, dashboard_address=None, loop=loop
     ) as cluster:
-        alc = Adaptive(cluster.scheduler, cluster, interval=100)
+        alc = cluster.adapt(interval="100 ms")
         with Client(cluster, loop=loop) as c:
             assert not c.nthreads()
             future = c.submit(lambda x: x + 1, 1)
@@ -128,41 +115,34 @@ def test_adaptive_local_cluster_multi_workers():
         yield cluster.close()
 
 
-@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 10, active_rpc_timeout=10)
-def test_adaptive_scale_down_override(c, s, *workers):
+@pytest.mark.asyncio
+async def test_adaptive_scale_down_override():
     class TestAdaptive(Adaptive):
         def __init__(self, *args, **kwargs):
             self.min_size = kwargs.pop("min_size", 0)
             Adaptive.__init__(self, *args, **kwargs)
 
-        def workers_to_close(self, **kwargs):
-            num_workers = len(self.scheduler.workers)
-            to_close = self.scheduler.workers_to_close(**kwargs)
+        async def workers_to_close(self, **kwargs):
+            num_workers = len(self.cluster.workers)
+            to_close = await self.scheduler.workers_to_close(**kwargs)
             if num_workers - len(to_close) < self.min_size:
                 to_close = to_close[: num_workers - self.min_size]
 
             return to_close
 
-    class TestCluster(object):
+    class TestCluster(LocalCluster):
         def scale_up(self, n, **kwargs):
             assert False
 
-        def scale_down(self, workers):
-            assert False
-
-        @property
-        def workers(self):
-            return s.workers
-
-    assert len(s.workers) == 10
-
-    # Assert that adaptive cycle does not reduce cluster below minimum size
-    # as determined via override.
-    cluster = TestCluster()
-    ta = TestAdaptive(s, cluster, min_size=2, interval=0.1, scale_factor=2)
-    yield gen.sleep(0.3)
+    async with TestCluster(n_workers=10, processes=False, asynchronous=True) as cluster:
+        ta = cluster.adapt(
+            min_size=2, interval=0.1, scale_factor=2, Adaptive=TestAdaptive
+        )
+        await gen.sleep(0.3)
 
-    assert len(s.workers) == 2
+        # Assert that adaptive cycle does not reduce cluster below minimum size
+        # as determined via override.
+        assert len(cluster.scheduler.workers) == 2
 
 
 @gen_test()
@@ -176,14 +156,7 @@ def test_min_max():
         asynchronous=True,
     )
     try:
-        adapt = Adaptive(
-            cluster.scheduler,
-            cluster,
-            minimum=1,
-            maximum=2,
-            interval="20 ms",
-            wait_count=10,
-        )
+        adapt = cluster.adapt(minimum=1, maximum=2, interval="20 ms", wait_count=10)
         c = yield Client(cluster, asynchronous=True)
 
         start = time()
@@ -237,7 +210,7 @@ def test_avoid_churn():
     )
     client = yield Client(cluster, asynchronous=True)
     try:
-        adapt = Adaptive(cluster.scheduler, cluster, interval="20 ms", wait_count=5)
+        adapt = cluster.adapt(interval="20 ms", wait_count=5)
 
         for i in range(10):
             yield client.submit(slowinc, i, delay=0.040)
@@ -267,7 +240,7 @@ def test_adapt_quickly():
         dashboard_address=None,
     )
     client = yield Client(cluster, asynchronous=True)
-    adapt = Adaptive(cluster.scheduler, cluster, interval=20, wait_count=5, maximum=10)
+    adapt = cluster.adapt(interval=20, wait_count=5, maximum=10)
     try:
         future = client.submit(slowinc, 1, delay=0.100)
         yield wait(future)
@@ -346,9 +319,7 @@ def test_no_more_workers_than_tasks():
     )
     yield cluster._start()
     try:
-        adapt = Adaptive(
-            cluster.scheduler, cluster, minimum=0, maximum=4, interval="10 ms"
-        )
+        adapt = cluster.adapt(minimum=0, maximum=4, interval="10 ms")
         client = yield Client(cluster, asynchronous=True, loop=loop)
         cluster.scheduler.task_duration["slowinc"] = 1000
 
diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index e5bd3dd3140..c6c3e3d1ba2 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -20,6 +20,7 @@ distributed:
     transition-log-length: 100000
     work-stealing: True     # workers should steal tasks from each other
     worker-ttl: null        # like '60s'. Time to live for workers.  They must heartbeat faster than this
+    pickle: True            # Is the scheduler allowed to deserialize arbitrary bytestrings
     preload: []
     preload-argv: []
     dashboard:
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 63ae1ef33b2..6e58fb36ffc 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1069,6 +1069,7 @@ def __init__(
             "get_task_stream": self.get_task_stream,
             "register_worker_plugin": self.register_worker_plugin,
             "adaptive_target": self.adaptive_target,
+            "workers_to_close": self.workers_to_close,
         }
 
         self._transitions = {
@@ -2919,7 +2920,9 @@ def replicate(
             },
         )
 
-    def workers_to_close(self, memory_ratio=None, n=None, key=None, minimum=None):
+    def workers_to_close(
+        self, comm=None, memory_ratio=None, n=None, key=None, minimum=None
+    ):
         """
         Find workers that we can close with low cost
 
@@ -2981,6 +2984,10 @@ def workers_to_close(self, memory_ratio=None, n=None, key=None, minimum=None):
 
             if key is None:
                 key = lambda ws: ws.address
+            if isinstance(key, bytes) and dask.config.get(
+                "distributed.scheduler.pickle"
+            ):
+                key = pickle.loads(key)
 
             groups = groupby(key, self.workers.values())
 
@@ -3209,6 +3216,14 @@ def feed(
         Caution: this runs arbitrary Python code on the scheduler.  This should
         eventually be phased out.  It is mostly used by diagnostics.
         """
+        if not dask.config.get("distributed.scheduler.pickle"):
+            logger.warn(
+                "Tried to call 'feed' route with custom fucntions, but "
+                "pickle is disallowed.  Set the 'distributed.scheduler.pickle'"
+                "config value to True to use the 'feed' route (this is mostly "
+                "commonly used with progress bars)"
+            )
+            return
         import pickle
 
         interval = parse_timedelta(interval)
@@ -4714,7 +4729,7 @@ def check_idle(self):
         if close:
             self.loop.add_callback(self.close)
 
-    def adaptive_target(self, target_duration="5s"):
+    def adaptive_target(self, comm=None, target_duration="5s"):
         """ Desired number of workers based on the current workload
 
         This looks at the current running tasks and memory use, and returns a

From d28d885437fdea1182277d974f61f4843e6612b7 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sat, 20 Jul 2019 17:49:14 -0500
Subject: [PATCH 0363/1550] Add Server.finished async function  (#2864)

---
 distributed/cli/dask_worker.py            |  3 +--
 distributed/core.py                       |  9 ++++++++-
 distributed/deploy/tests/test_adaptive.py |  6 +++---
 distributed/nanny.py                      |  9 +++++----
 distributed/scheduler.py                  | 20 +-------------------
 distributed/tests/test_scheduler.py       | 11 +++++++++++
 distributed/utils_test.py                 |  7 ++++---
 distributed/worker.py                     |  1 +
 8 files changed, 34 insertions(+), 32 deletions(-)

diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
index e86cfa41618..8752cd52448 100755
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -380,8 +380,7 @@ def on_signal(signum):
     @gen.coroutine
     def run():
         yield nannies
-        while all(n.status != "closed" for n in nannies):
-            yield gen.sleep(0.2)
+        yield [n.finished() for n in nannies]
 
     install_signal_handlers(loop, cleanup=on_signal)
 
diff --git a/distributed/core.py b/distributed/core.py
index 4d18547151a..8aac2edfc33 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -73,7 +73,7 @@ def _raise(*args, **kwargs):
 
 
 class Server(object):
-    """ Distributed TCP Server
+    """ Dask Distributed Server
 
     Superclass for endpoints in a distributed cluster, such as Worker
     and Scheduler objects.
@@ -146,6 +146,7 @@ def __init__(
         self.events = None
         self.event_counts = None
         self._ongoing_coroutines = weakref.WeakSet()
+        self._event_finished = Event()
 
         self.listener = None
         self.io_loop = io_loop or IOLoop.current()
@@ -211,6 +212,10 @@ def set_thread_ident():
 
         self.__stopped = False
 
+    async def finished(self):
+        """ Wait until the server has finished """
+        await self._event_finished.wait()
+
     def start_periodic_callbacks(self):
         """ Start Periodic Callbacks consistently
 
@@ -507,6 +512,8 @@ def close(self):
             else:
                 yield gen.sleep(0.01)
 
+        self._event_finished.set()
+
 
 def pingpong(comm):
     return b"pong"
diff --git a/distributed/deploy/tests/test_adaptive.py b/distributed/deploy/tests/test_adaptive.py
index 2258a83cfe5..861c5107348 100644
--- a/distributed/deploy/tests/test_adaptive.py
+++ b/distributed/deploy/tests/test_adaptive.py
@@ -9,12 +9,12 @@
 
 from distributed import Client, wait, Adaptive, LocalCluster, SpecCluster, Worker
 from distributed.utils_test import gen_test, slowinc, clean
-from distributed.utils_test import loop, nodebug  # noqa: F401
+from distributed.utils_test import loop, nodebug, cleanup  # noqa: F401
 from distributed.metrics import time
 
 
 @pytest.mark.asyncio
-async def test_simultaneous_scale_up_and_down():
+async def test_simultaneous_scale_up_and_down(cleanup):
     class TestAdaptive(Adaptive):
         def get_scale_up_kwargs(self):
             assert False
@@ -116,7 +116,7 @@ def test_adaptive_local_cluster_multi_workers():
 
 
 @pytest.mark.asyncio
-async def test_adaptive_scale_down_override():
+async def test_adaptive_scale_down_override(cleanup):
     class TestAdaptive(Adaptive):
         def __init__(self, *args, **kwargs):
             self.min_size = kwargs.pop("min_size", 0)
diff --git a/distributed/nanny.py b/distributed/nanny.py
index 00370b81adb..f3bebb1dcac 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -355,7 +355,7 @@ def memory_monitor(self):
             process.terminate()
 
     def is_alive(self):
-        return self.process is not None and self.process.status == "running"
+        return self.process is not None and self.process.is_alive()
 
     def run(self, *args, **kwargs):
         return run(self, *args, **kwargs)
@@ -401,11 +401,12 @@ def close(self, comm=None, timeout=5, report=None):
         """
         Close the worker process, stop all comms.
         """
-        while self.status == "closing":
-            yield gen.sleep(0.01)
+        if self.status == "closing":
+            yield self.finished()
+            assert self.status == "closed"
 
         if self.status == "closed":
-            raise gen.Return("OK")
+            return "OK"
 
         self.status = "closing"
         logger.info("Closing Nanny at %r", self.address)
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 6e58fb36ffc..4441b815642 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -818,8 +818,6 @@ class Scheduler(ServerNode):
         report results
     * **task_duration:** ``{key-prefix: time}``
         Time we expect certain functions to take, e.g. ``{'sum': 0.25}``
-    * **coroutines:** ``[Futures]``:
-        A list of active futures that control operation
     """
 
     default_port = 8786
@@ -896,7 +894,6 @@ def __init__(
         self.loop = loop or IOLoop.current()
         self.client_comms = dict()
         self.stream_comms = dict()
-        self.coroutines = []
         self._worker_coroutines = []
         self._ipython_kernel = None
 
@@ -1189,12 +1186,6 @@ def start(self):
             for c in self._worker_coroutines:
                 c.cancel()
 
-        for cor in self.coroutines:
-            if cor.done():
-                exc = cor.exception()
-                if exc:
-                    raise exc
-
         if self.status != "running":
             self.listen(self._start_address, listen_args=self.listen_args)
             self.ip = get_address_host(self.listen_address)
@@ -1232,15 +1223,8 @@ def del_scheduler_file():
 
         setproctitle("dask-scheduler [%s]" % (self.address,))
 
-        yield self.finished()
         return self
 
-    @gen.coroutine
-    def finished(self):
-        """ Wait until all coroutines have ceased """
-        while any(not c.done() for c in self.coroutines):
-            yield All(self.coroutines)
-
     @gen.coroutine
     def close(self, comm=None, fast=False, close_workers=False):
         """ Send cleanup signal to all coroutines then wait until finished
@@ -1250,6 +1234,7 @@ def close(self, comm=None, fast=False, close_workers=False):
         Scheduler.cleanup
         """
         if self.status.startswith("clos"):
+            yield self.finished()
             return
         self.status = "closing"
 
@@ -1287,9 +1272,6 @@ def close(self, comm=None, fast=False, close_workers=False):
         for future in futures:
             yield future
 
-        if not fast:
-            yield self.finished()
-
         for comm in self.client_comms.values():
             comm.abort()
 
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 9b512ddad64..20de5e7b7fd 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -1082,6 +1082,7 @@ def test_close_nanny(c, s, a, b):
         yield gen.sleep(0.1)
         assert time() < start + 5
 
+    assert not a.is_alive()
     assert a.pid is None
 
     for i in range(10):
@@ -1612,3 +1613,13 @@ async def test_allowed_failures_config(cleanup):
     with dask.config.set({"distributed.scheduler.allowed_failures": 0}):
         async with Scheduler(port=0) as s:
             assert s.allowed_failures == 0
+
+
+@pytest.mark.asyncio
+async def test_finished():
+    async with Scheduler(port=0) as s:
+        async with Worker(s.address) as w:
+            pass
+
+    await s.finished()
+    await w.finished()
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index f0e75f8ed2c..5f3dff548cf 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -1508,9 +1508,10 @@ def check_instances():
     _global_clients.clear()
 
     for w in Worker._instances:
-        w.close(report=False, executor_wait=False)
-        if w.status == "running":
-            w.close()
+        with ignoring(RuntimeError):  # closed IOLoop
+            w.close(report=False, executor_wait=False)
+            if w.status == "running":
+                w.close()
     Worker._instances.clear()
 
     for i in range(5):
diff --git a/distributed/worker.py b/distributed/worker.py
index 791fc0ba101..b052dd05799 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -954,6 +954,7 @@ def _close(self, *args, **kwargs):
     def close(self, report=True, timeout=10, nanny=True, executor_wait=True):
         with log_errors():
             if self.status in ("closed", "closing"):
+                yield self.finished()
                 return
 
             disable_gc_diagnosis()

From 5f8a4f9a0f549e01354f96e373fbe3abda23e25f Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sun, 21 Jul 2019 09:46:03 -0500
Subject: [PATCH 0364/1550] Align text and remove bullets in Client HTML repr
 (#2867)

---
 distributed/client.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 74e33716cb6..1dc5af16b81 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -804,14 +804,15 @@ def _repr_html_(self):
             info = self._scheduler_identity
             scheduler = self.scheduler
 
+        text = (
+            '<h3 style="text-align: left;">Client</h3>\n'
+            '<ul style="text-align: left; list-style: none; margin: 0; padding: 0;">\n'
+        )
         if scheduler is not None:
-            text = (
-                "<h3>Client</h3>\n" "<ul>\n" "  <li><b>Scheduler: </b>%s\n"
-            ) % scheduler.address
+            text += "  <li><b>Scheduler: </b>%s\n" % scheduler.address
         else:
-            text = (
-                "<h3>Client</h3>\n" "<ul>\n" "  <li><b>Scheduler: not connected</b>\n"
-            )
+            text += "  <li><b>Scheduler: not connected</b>\n"
+
         if info and "dashboard" in info["services"]:
             protocol, rest = scheduler.address.split("://")
             port = info["services"]["dashboard"]
@@ -837,8 +838,8 @@ def _repr_html_(self):
                 memory = ""
 
             text2 = (
-                "<h3>Cluster</h3>\n"
-                "<ul>\n"
+                '<h3 style="text-align: left;">Cluster</h3>\n'
+                '<ul style="text-align: left; list-style:none; margin: 0; padding: 0;">\n'
                 "  <li><b>Workers: </b>%d</li>\n"
                 "  <li><b>Cores: </b>%d</li>\n"
                 "  <li><b>Memory: </b>%s</li>\n"

From 586ead997dc72c1b7170dbf30af253257f53b1fc Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sun, 21 Jul 2019 13:06:59 -0500
Subject: [PATCH 0365/1550] Test dask-scheduler --idle-timeout flag (#2862)

Fixes #2846
---
 distributed/cli/dask_scheduler.py            | 15 +++++++++++++--
 distributed/cli/tests/test_dask_scheduler.py | 11 +++++++++++
 2 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/distributed/cli/dask_scheduler.py b/distributed/cli/dask_scheduler.py
index f2799164a36..7142331e861 100755
--- a/distributed/cli/dask_scheduler.py
+++ b/distributed/cli/dask_scheduler.py
@@ -120,6 +120,12 @@
 @click.argument(
     "preload_argv", nargs=-1, type=click.UNPROCESSED, callback=validate_preload_argv
 )
+@click.option(
+    "--idle-timeout",
+    default=None,
+    type=str,
+    help="Time of inactivity after which to kill the scheduler",
+)
 @click.version_option()
 def main(
     host,
@@ -141,6 +147,7 @@ def main(
     tls_cert,
     tls_key,
     dashboard_address,
+    idle_timeout,
 ):
     g0, g1, g2 = gc.get_threshold()  # https://github.com/dask/distributed/issues/1653
     gc.set_threshold(g0 * 3, g1 * 3, g2 * 3)
@@ -211,6 +218,7 @@ def del_pid_file():
         protocol=protocol,
         dashboard_address=dashboard_address if dashboard else None,
         service_kwargs={"dashboard": {"prefix": dashboard_prefix}},
+        idle_timeout=idle_timeout,
     )
     scheduler.start()
     if not preload:
@@ -226,9 +234,12 @@ def del_pid_file():
 
     install_signal_handlers(loop)
 
+    async def run():
+        await scheduler
+        await scheduler.finished()
+
     try:
-        loop.start()
-        loop.close()
+        loop.run_sync(run)
     finally:
         scheduler.stop()
         if local_directory_created:
diff --git a/distributed/cli/tests/test_dask_scheduler.py b/distributed/cli/tests/test_dask_scheduler.py
index 7de7e881270..24737474165 100644
--- a/distributed/cli/tests/test_dask_scheduler.py
+++ b/distributed/cli/tests/test_dask_scheduler.py
@@ -383,3 +383,14 @@ def test_version_option():
     runner = CliRunner()
     result = runner.invoke(distributed.cli.dask_scheduler.main, ["--version"])
     assert result.exit_code == 0
+
+
+@pytest.mark.slow
+def test_idle_timeout(loop):
+    start = time()
+    runner = CliRunner()
+    result = runner.invoke(
+        distributed.cli.dask_scheduler.main, ["--idle-timeout", "1s"]
+    )
+    stop = time()
+    assert 1 < stop - start < 10

From b13403727fc2dc1b1173aa0c347af2881418e36b Mon Sep 17 00:00:00 2001
From: Jim Crist <jcrist@users.noreply.github.com>
Date: Tue, 23 Jul 2019 18:06:42 -0500
Subject: [PATCH 0366/1550] Remove `Client.upload_environment` (#2877)

This method was undocumented and broken. Modifying the user environment
is better handled by the deployment solution, the current method is not
resilient to worker additions/removals, and uploading and replacing the
current environment is an inefficient method to add additional packages.
If users really want to upload packages to each worker at runtime, they
can use `upload_file` with a zipfile/egg.
---
 distributed/client.py | 33 ---------------------------------
 1 file changed, 33 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 1dc5af16b81..d8fb5dfa13d 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -9,7 +9,6 @@
 from datetime import timedelta
 import errno
 from functools import partial
-from glob import glob
 import itertools
 import json
 import logging
@@ -2823,38 +2822,6 @@ def persist(
         else:
             return result
 
-    @gen.coroutine
-    def _upload_environment(self, zipfile):
-        name = os.path.split(zipfile)[1]
-        yield self._upload_large_file(zipfile, name)
-
-        def unzip(dask_worker=None):
-            from distributed.utils import log_errors
-            import zipfile
-            import shutil
-
-            with log_errors():
-                a = os.path.join(dask_worker.worker_dir, name)
-                b = os.path.join(dask_worker.local_dir, name)
-                c = os.path.dirname(b)
-                shutil.move(a, b)
-
-                with zipfile.ZipFile(b) as f:
-                    f.extractall(path=c)
-
-                for fn in glob(os.path.join(c, name[:-4], "bin", "*")):
-                    st = os.stat(fn)
-                    os.chmod(fn, st.st_mode | 64)  # chmod u+x fn
-
-                assert os.path.exists(os.path.join(c, name[:-4]))
-                return c
-
-        yield self._run(unzip, nanny=True)
-        raise gen.Return(name[:-4])
-
-    def upload_environment(self, name, zipfile):
-        return self.sync(self._upload_environment, name, zipfile)
-
     @gen.coroutine
     def _restart(self, timeout=no_default):
         if timeout == no_default:

From f16ee17bc9481f2d9e19479f390d4248fc7c83a3 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 24 Jul 2019 09:02:42 -0700
Subject: [PATCH 0367/1550] Replace gen.coroutine with async/await in core
 (#2871)

---
 distributed/actor.py                          |  30 +-
 distributed/cli/dask_scheduler.py             |  14 +-
 distributed/client.py                         | 350 ++++++++----------
 distributed/comm/core.py                      |   9 +-
 distributed/comm/inproc.py                    |  31 +-
 distributed/comm/tcp.py                       |  53 ++-
 distributed/comm/tests/test_comms.py          |  12 +-
 distributed/core.py                           |  93 ++---
 distributed/dashboard/components.py           |   6 +-
 distributed/dashboard/scheduler_html.py       |  16 +-
 .../dashboard/tests/test_components.py        |   4 +-
 .../tests/test_scheduler_bokeh_html.py        |   4 +-
 .../dashboard/tests/test_worker_bokeh.py      |   2 +-
 distributed/deploy/spec.py                    |   2 +-
 distributed/deploy/tests/test_adaptive.py     |  12 +-
 distributed/deploy/tests/test_local.py        |   5 +-
 distributed/diagnostics/eventstream.py        |  15 +-
 distributed/diagnostics/progress.py           |  10 +-
 distributed/diagnostics/progress_stream.py    |  14 +-
 distributed/diagnostics/progressbar.py        |  37 +-
 .../diagnostics/tests/test_progressbar.py     |  31 +-
 distributed/lock.py                           |  19 +-
 distributed/nanny.py                          | 126 +++----
 distributed/process.py                        |   2 +-
 distributed/pubsub.py                         |  12 +-
 distributed/queues.py                         |  70 ++--
 distributed/recreate_exceptions.py            |  21 +-
 distributed/scheduler.py                      | 167 ++++-----
 distributed/tests/py3_test_pubsub.py          |   3 +-
 distributed/tests/test_as_completed.py        |   2 +-
 distributed/tests/test_batched.py             |   6 +-
 distributed/tests/test_client.py              |  26 +-
 distributed/tests/test_locks.py               |   2 +-
 distributed/tests/test_nanny.py               |  28 +-
 distributed/tests/test_priorities.py          |  52 +--
 distributed/tests/test_queues.py              |   9 +-
 distributed/tests/test_resources.py           |   4 +-
 distributed/tests/test_scheduler.py           |  32 +-
 distributed/tests/test_steal.py               |   4 +-
 distributed/tests/test_stress.py              |  29 +-
 distributed/tests/test_tls_functional.py      |   4 +-
 distributed/tests/test_utils_test.py          |   9 +-
 distributed/tests/test_worker.py              |  12 +-
 distributed/tests/test_worker_plugins.py      |   2 +-
 distributed/utils.py                          |  25 +-
 distributed/utils_comm.py                     |  34 +-
 distributed/utils_test.py                     | 188 +++++-----
 distributed/variable.py                       |  34 +-
 distributed/worker.py                         | 203 +++++-----
 docs/source/adaptive.rst                      |   6 +-
 docs/source/client.rst                        |  16 +-
 docs/source/foundations.rst                   |  59 ++-
 52 files changed, 921 insertions(+), 1035 deletions(-)

diff --git a/distributed/actor.py b/distributed/actor.py
index 4bbe6faf78d..e45f089effd 100644
--- a/distributed/actor.py
+++ b/distributed/actor.py
@@ -1,3 +1,4 @@
+import asyncio
 from tornado import gen
 import functools
 
@@ -128,10 +129,9 @@ def __getattr__(self, key):
 
             @functools.wraps(attr)
             def func(*args, **kwargs):
-                @gen.coroutine
-                def run_actor_function_on_worker():
+                async def run_actor_function_on_worker():
                     try:
-                        result = yield self._worker_rpc.actor_execute(
+                        result = await self._worker_rpc.actor_execute(
                             function=key,
                             actor=self.key,
                             args=[to_serialize(arg) for arg in args],
@@ -139,21 +139,20 @@ def run_actor_function_on_worker():
                         )
                     except OSError:
                         if self._future:
-                            yield self._future
+                            await self._future
                         else:
                             raise OSError("Unable to contact Actor's worker")
-                    raise gen.Return(result["result"])
+                    return result["result"]
 
                 if self._asynchronous:
-                    return run_actor_function_on_worker()
+                    return asyncio.ensure_future(run_actor_function_on_worker())
                 else:
                     # TODO: this mechanism is error prone
                     # we should endeavor to make dask's standard code work here
                     q = Queue()
 
-                    @gen.coroutine
-                    def wait_then_add_to_queue():
-                        x = yield run_actor_function_on_worker()
+                    async def wait_then_add_to_queue():
+                        x = await run_actor_function_on_worker()
                         q.put(x)
 
                     self._io_loop.add_callback(wait_then_add_to_queue)
@@ -164,11 +163,11 @@ def wait_then_add_to_queue():
 
         else:
 
-            @gen.coroutine
-            def get_actor_attribute_from_worker():
-                x = yield self._worker_rpc.actor_attribute(
+            async def get_actor_attribute_from_worker():
+                x = await self._worker_rpc.actor_attribute(
                     attribute=key, actor=self.key
                 )
+                return x["result"]
                 raise gen.Return(x["result"])
 
             return self._sync(get_actor_attribute_from_worker)
@@ -188,11 +187,10 @@ def __init__(self, rpc, address):
         self._address = address
 
     def __getattr__(self, key):
-        @gen.coroutine
-        def func(**msg):
+        async def func(**msg):
             msg["op"] = key
-            result = yield self.rpc.proxy(worker=self._address, msg=msg)
-            raise gen.Return(result)
+            result = await self.rpc.proxy(worker=self._address, msg=msg)
+            return result
 
         return func
 
diff --git a/distributed/cli/dask_scheduler.py b/distributed/cli/dask_scheduler.py
index 7142331e861..d41b98eb310 100755
--- a/distributed/cli/dask_scheduler.py
+++ b/distributed/cli/dask_scheduler.py
@@ -1,7 +1,6 @@
 from __future__ import print_function, division, absolute_import
 
 import atexit
-import dask
 import logging
 import gc
 import os
@@ -16,9 +15,9 @@
 from tornado.ioloop import IOLoop
 
 from distributed import Scheduler
+from distributed.preloading import validate_preload_argv
 from distributed.security import Security
 from distributed.cli.utils import check_python_3, install_signal_handlers
-from distributed.preloading import preload_modules, validate_preload_argv
 from distributed.proctitle import (
     enable_proctitle_on_children,
     enable_proctitle_on_current,
@@ -219,16 +218,9 @@ def del_pid_file():
         dashboard_address=dashboard_address if dashboard else None,
         service_kwargs={"dashboard": {"prefix": dashboard_prefix}},
         idle_timeout=idle_timeout,
+        preload=preload,
+        preload_argv=preload_argv,
     )
-    scheduler.start()
-    if not preload:
-        preload = dask.config.get("distributed.scheduler.preload")
-    if not preload_argv:
-        preload_argv = dask.config.get("distributed.scheduler.preload-argv")
-    preload_modules(
-        preload, parameter=scheduler, file_dir=local_directory, argv=preload_argv
-    )
-
     logger.info("Local Directory: %26s", local_directory)
     logger.info("-" * 47)
 
diff --git a/distributed/client.py b/distributed/client.py
index d8fb5dfa13d..c4b2f51426a 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -43,6 +43,7 @@
 from tornado.ioloop import IOLoop
 from tornado.queues import Queue
 
+import asyncio
 from asyncio import iscoroutine
 
 from .batched import BatchedSend
@@ -230,32 +231,30 @@ def result(self, timeout=None):
         else:
             return result
 
-    @gen.coroutine
-    def _result(self, raiseit=True):
-        yield self._state.wait()
+    async def _result(self, raiseit=True):
+        await self._state.wait()
         if self.status == "error":
             exc = clean_exception(self._state.exception, self._state.traceback)
             if raiseit:
                 six.reraise(*exc)
             else:
-                raise gen.Return(exc)
+                return exc
         elif self.status == "cancelled":
             exception = CancelledError(self.key)
             if raiseit:
                 raise exception
             else:
-                raise gen.Return(exception)
+                return exception
         else:
-            result = yield self.client._gather([self])
-            raise gen.Return(result[0])
+            result = await self.client._gather([self])
+            return result[0]
 
-    @gen.coroutine
-    def _exception(self):
-        yield self._state.wait()
+    async def _exception(self):
+        await self._state.wait()
         if self.status == "error":
-            raise gen.Return(self._state.exception)
+            return self._state.exception
         else:
-            raise gen.Return(None)
+            return None
 
     def exception(self, timeout=None, **kwargs):
         """ Return the exception of a failed task
@@ -320,13 +319,12 @@ def cancelled(self):
         """ Returns True if the future has been cancelled """
         return self._state.status == "cancelled"
 
-    @gen.coroutine
-    def _traceback(self):
-        yield self._state.wait()
+    async def _traceback(self):
+        await self._state.wait()
         if self.status == "error":
-            raise gen.Return(self._state.traceback)
+            return self._state.traceback
         else:
-            raise gen.Return(None)
+            return None
 
     def traceback(self, timeout=None, **kwargs):
         """ Return the traceback of a failed task
@@ -447,6 +445,7 @@ def _get_event(self):
 
     def cancel(self):
         self.status = "cancelled"
+        self.exception = CancelledError()
         self._get_event().set()
 
     def finish(self, type=None):
@@ -479,19 +478,17 @@ def reset(self):
         if self._event is not None:
             self._event.clear()
 
-    @gen.coroutine
-    def wait(self, timeout=None):
-        yield self._get_event().wait(timeout)
+    async def wait(self, timeout=None):
+        await self._get_event().wait(timeout)
 
     def __repr__(self):
         return "<%s: %s>" % (self.__class__.__name__, self.status)
 
 
-@gen.coroutine
-def done_callback(future, callback):
+async def done_callback(future, callback):
     """ Coroutine that waits on future, then calls callback """
     while future.status == "pending":
-        yield future._state.wait()
+        await future._state.wait()
     callback(future)
 
 
@@ -867,7 +864,7 @@ def start(self, **kwargs):
         self.status = "connecting"
 
         if self.asynchronous:
-            self._started = self._start(**kwargs)
+            self._started = asyncio.ensure_future(self._start(**kwargs))
         else:
             sync(self.loop, self._start, **kwargs)
 
@@ -876,9 +873,8 @@ def __await__(self):
             return self._started.__await__()
         else:
 
-            @gen.coroutine
-            def _():
-                raise gen.Return(self)
+            async def _():
+                return self
 
             return _().__await__()
 
@@ -901,8 +897,7 @@ def _send_to_scheduler(self, msg):
                 "Message: %s" % (self.status, msg)
             )
 
-    @gen.coroutine
-    def _start(self, timeout=no_default, **kwargs):
+    async def _start(self, timeout=no_default, **kwargs):
         if timeout == no_default:
             timeout = self._timeout
         if timeout is not None:
@@ -912,7 +907,7 @@ def _start(self, timeout=no_default, **kwargs):
         if self.cluster is not None:
             # Ensure the cluster is started (no-op if already running)
             try:
-                yield self.cluster._start()
+                await self.cluster._start()
             except AttributeError:  # Some clusters don't have this method
                 pass
             except Exception:
@@ -923,7 +918,7 @@ def _start(self, timeout=no_default, **kwargs):
             address = self.cluster.scheduler_address
         elif self.scheduler_file is not None:
             while not os.path.exists(self.scheduler_file):
-                yield gen.sleep(0.01)
+                await gen.sleep(0.01)
             for i in range(10):
                 try:
                     with open(self.scheduler_file) as f:
@@ -931,33 +926,31 @@ def _start(self, timeout=no_default, **kwargs):
                     address = cfg["address"]
                     break
                 except (ValueError, KeyError):  # JSON file not yet flushed
-                    yield gen.sleep(0.01)
+                    await gen.sleep(0.01)
         elif self._start_arg is None:
             from .deploy import LocalCluster
 
             try:
-                self.cluster = LocalCluster(
+                self.cluster = await LocalCluster(
                     loop=self.loop, asynchronous=True, **self._startup_kwargs
                 )
-                yield self.cluster
             except (OSError, socket.error) as e:
                 if e.errno != errno.EADDRINUSE:
                     raise
                 # The default port was taken, use a random one
-                self.cluster = LocalCluster(
+                self.cluster = await LocalCluster(
                     scheduler_port=0,
                     loop=self.loop,
                     asynchronous=True,
                     **self._startup_kwargs
                 )
-                yield self.cluster
 
             # Wait for all workers to be ready
             # XXX should be a LocalCluster method instead
             while not self.cluster.workers or len(self.cluster.scheduler.workers) < len(
                 self.cluster.workers
             ):
-                yield gen.sleep(0.01)
+                await gen.sleep(0.01)
 
             address = self.cluster.scheduler_address
 
@@ -965,18 +958,17 @@ def _start(self, timeout=no_default, **kwargs):
             self.scheduler = self.rpc(address)
         self.scheduler_comm = None
 
-        yield self._ensure_connected(timeout=timeout)
+        await self._ensure_connected(timeout=timeout)
 
         for pc in self._periodic_callbacks.values():
             pc.start()
 
-        self._handle_scheduler_coroutine = self._handle_report()
+        self._handle_scheduler_coroutine = asyncio.ensure_future(self._handle_report())
         self.coroutines.append(self._handle_scheduler_coroutine)
 
-        raise gen.Return(self)
+        return self
 
-    @gen.coroutine
-    def _reconnect(self):
+    async def _reconnect(self):
         with log_errors():
             assert self.scheduler_comm.comm.closed()
 
@@ -991,11 +983,11 @@ def _reconnect(self):
             deadline = self.loop.time() + timeout
             while timeout > 0 and self.status == "connecting":
                 try:
-                    yield self._ensure_connected(timeout=timeout)
+                    await self._ensure_connected(timeout=timeout)
                     break
                 except EnvironmentError:
                     # Wait a bit before retrying
-                    yield gen.sleep(0.1)
+                    await gen.sleep(0.1)
                     timeout = deadline - self.loop.time()
             else:
                 logger.error(
@@ -1003,10 +995,9 @@ def _reconnect(self):
                     "seconds, closing client",
                     self._timeout,
                 )
-                yield self._close()
+                await self._close()
 
-    @gen.coroutine
-    def _ensure_connected(self, timeout=None):
+    async def _ensure_connected(self, timeout=None):
         if (
             self.scheduler_comm
             and not self.scheduler_comm.closed()
@@ -1018,27 +1009,27 @@ def _ensure_connected(self, timeout=None):
         self._connecting_to_scheduler = True
 
         try:
-            comm = yield connect(
+            comm = await connect(
                 self.scheduler.address,
                 timeout=timeout,
                 connection_args=self.connection_args,
             )
             comm.name = "Client->Scheduler"
             if timeout is not None:
-                yield gen.with_timeout(
+                await gen.with_timeout(
                     timedelta(seconds=timeout), self._update_scheduler_info()
                 )
             else:
-                yield self._update_scheduler_info()
-            yield comm.write(
+                await self._update_scheduler_info()
+            await comm.write(
                 {"op": "register-client", "client": self.id, "reply": False}
             )
         finally:
             self._connecting_to_scheduler = False
         if timeout is not None:
-            msg = yield gen.with_timeout(timedelta(seconds=timeout), comm.read())
+            msg = await gen.with_timeout(timedelta(seconds=timeout), comm.read())
         else:
-            msg = yield comm.read()
+            msg = await comm.read()
         assert len(msg) == 1
         assert msg[0]["op"] == "stream-start"
 
@@ -1055,21 +1046,19 @@ def _ensure_connected(self, timeout=None):
 
         logger.debug("Started scheduling coroutines. Synchronized")
 
-    @gen.coroutine
-    def _update_scheduler_info(self):
+    async def _update_scheduler_info(self):
         if self.status not in ("running", "connecting"):
             return
         try:
-            self._scheduler_identity = yield self.scheduler.identity()
+            self._scheduler_identity = await self.scheduler.identity()
         except EnvironmentError:
             logger.debug("Not able to query scheduler for identity")
 
-    @gen.coroutine
-    def _wait_for_workers(self, n_workers=0):
-        info = yield self.scheduler.identity()
+    async def _wait_for_workers(self, n_workers=0):
+        info = await self.scheduler.identity()
         while n_workers and len(info["workers"]) < n_workers:
-            yield gen.sleep(0.1)
-            info = yield self.scheduler.identity()
+            await gen.sleep(0.1)
+            info = await self.scheduler.identity()
 
     def wait_for_workers(self, n_workers=0):
         """Blocking call to wait for n workers before continuing"""
@@ -1084,14 +1073,12 @@ def __enter__(self):
             self.start()
         return self
 
-    @gen.coroutine
-    def __aenter__(self):
-        yield self._started
-        raise gen.Return(self)
+    async def __aenter__(self):
+        await self._started
+        return self
 
-    @gen.coroutine
-    def __aexit__(self, typ, value, traceback):
-        yield self._close()
+    async def __aexit__(self, typ, value, traceback):
+        await self._close()
 
     def __exit__(self, type, value, traceback):
         self.close()
@@ -1121,8 +1108,7 @@ def _release_key(self, key):
                 {"op": "client-releases-keys", "keys": [key], "client": self.id}
             )
 
-    @gen.coroutine
-    def _handle_report(self):
+    async def _handle_report(self):
         """ Listen to scheduler """
         with log_errors():
             try:
@@ -1130,13 +1116,13 @@ def _handle_report(self):
                     if self.scheduler_comm is None:
                         break
                     try:
-                        msgs = yield self.scheduler_comm.comm.read()
+                        msgs = await self.scheduler_comm.comm.read()
                     except CommClosedError:
                         if self.status == "running":
                             logger.info("Client report stream closed to scheduler")
                             logger.info("Reconnecting...")
                             self.status = "connecting"
-                            yield self._reconnect()
+                            await self._reconnect()
                             continue
                         else:
                             break
@@ -1212,8 +1198,7 @@ def _handle_error(self, exception=None):
         logger.warning("Scheduler exception:")
         logger.exception(exception)
 
-    @gen.coroutine
-    def _close(self, fast=False):
+    async def _close(self, fast=False):
         """ Send close signal and wait until scheduler completes """
         self.status = "closing"
 
@@ -1243,7 +1228,7 @@ def _close(self, fast=False):
             # Give the scheduler 'stream-closed' message 100ms to come through
             # This makes the shutdown slightly smoother and quieter
             with ignoring(AttributeError, gen.TimeoutError):
-                yield gen.with_timeout(
+                await gen.with_timeout(
                     timedelta(milliseconds=100),
                     self._handle_scheduler_coroutine,
                     quiet_exceptions=(CancelledError,),
@@ -1254,12 +1239,12 @@ def _close(self, fast=False):
                 and self.scheduler_comm.comm
                 and not self.scheduler_comm.comm.closed()
             ):
-                yield self.scheduler_comm.close()
+                await self.scheduler_comm.close()
             for key in list(self.futures):
                 self._release_key(key=key)
             if self._start_arg is None:
                 with ignoring(AttributeError):
-                    yield self.cluster._close()
+                    await self.cluster._close()
             self.rpc.close()
             self.status = "closed"
             if _get_global_client() is self:
@@ -1275,7 +1260,7 @@ def _close(self, fast=False):
             del self.coroutines[:]
             if not fast:
                 with ignoring(TimeoutError):
-                    yield gen.with_timeout(timedelta(seconds=2), list(coroutines))
+                    await gen.with_timeout(timedelta(seconds=2), list(coroutines))
             with ignoring(AttributeError):
                 self.scheduler.close_rpc()
             self.scheduler = None
@@ -1627,8 +1612,7 @@ def map(
 
         return [futures[tokey(k)] for k in keys]
 
-    @gen.coroutine
-    def _gather(self, futures, errors="raise", direct=None, local_worker=None):
+    async def _gather(self, futures, errors="raise", direct=None, local_worker=None):
         unpacked, future_set = unpack_remotedata(futures, byte_keys=True)
         keys = [tokey(future.key) for future in future_set]
         bad_data = dict()
@@ -1645,11 +1629,10 @@ def _gather(self, futures, errors="raise", direct=None, local_worker=None):
                 if w.scheduler.address == self.scheduler.address:
                     direct = True
 
-        @gen.coroutine
-        def wait(k):
+        async def wait(k):
             """ Want to stop the All(...) early if we find an error """
             st = self.futures[k]
-            yield st.wait()
+            await st.wait()
             if st.status != "finished" and errors == "raise":
                 raise AllExit()
 
@@ -1657,7 +1640,7 @@ def wait(k):
             logger.debug("Waiting on futures to clear before gather")
 
             with ignoring(AllExit):
-                yield All(
+                await All(
                     [wait(key) for key in keys if key in self.futures],
                     quiet_exceptions=AllExit,
                 )
@@ -1696,15 +1679,17 @@ def wait(k):
             # We now do an actual remote communication with workers or scheduler
             if self._gather_future:  # attach onto another pending gather request
                 self._gather_keys |= set(keys)
-                response = yield self._gather_future
+                response = await self._gather_future
             else:  # no one waiting, go ahead
                 self._gather_keys = set(keys)
-                future = self._gather_remote(direct, local_worker)
+                future = asyncio.ensure_future(
+                    self._gather_remote(direct, local_worker)
+                )
                 if self._gather_keys is None:
                     self._gather_future = None
                 else:
                     self._gather_future = future
-                response = yield future
+                response = await future
 
             if response["status"] == "error":
                 log = logger.warning if errors == "raise" else logger.debug
@@ -1728,40 +1713,39 @@ def wait(k):
 
         data.update(response["data"])
         result = pack_data(unpacked, merge(data, bad_data))
-        raise gen.Return(result)
+        return result
 
-    @gen.coroutine
-    def _gather_remote(self, direct, local_worker):
+    async def _gather_remote(self, direct, local_worker):
         """ Perform gather with workers or scheduler
 
         This method exists to limit and batch many concurrent gathers into a
         few.  In controls access using a Tornado semaphore, and picks up keys
         from other requests made recently.
         """
-        yield self._gather_semaphore.acquire()
+        await self._gather_semaphore.acquire()
         keys = list(self._gather_keys)
         self._gather_keys = None  # clear state, these keys are being sent off
         self._gather_future = None
 
         try:
             if direct or local_worker:  # gather directly from workers
-                who_has = yield self.scheduler.who_has(keys=keys)
-                data2, missing_keys, missing_workers = yield gather_from_workers(
+                who_has = await self.scheduler.who_has(keys=keys)
+                data2, missing_keys, missing_workers = await gather_from_workers(
                     who_has, rpc=self.rpc, close=False
                 )
                 response = {"status": "OK", "data": data2}
                 if missing_keys:
                     keys2 = [key for key in keys if key not in data2]
-                    response = yield self.scheduler.gather(keys=keys2)
+                    response = await self.scheduler.gather(keys=keys2)
                     if response["status"] == "OK":
                         response["data"].update(data2)
 
             else:  # ask scheduler to gather data for us
-                response = yield self.scheduler.gather(keys=keys)
+                response = await self.scheduler.gather(keys=keys)
         finally:
             self._gather_semaphore.release()
 
-        raise gen.Return(response)
+        return response
 
     def gather(self, futures, errors="raise", direct=None, asynchronous=None):
         """ Gather futures from distributed memory
@@ -1823,8 +1807,7 @@ def gather(self, futures, errors="raise", direct=None, asynchronous=None):
                 asynchronous=asynchronous,
             )
 
-    @gen.coroutine
-    def _scatter(
+    async def _scatter(
         self,
         data,
         workers=None,
@@ -1841,7 +1824,7 @@ def _scatter(
         if isinstance(data, dict) and not all(
             isinstance(k, (bytes, unicode)) for k in data
         ):
-            d = yield self._scatter(keymap(tokey, data), workers, broadcast)
+            d = await self._scatter(keymap(tokey, data), workers, broadcast)
             raise gen.Return({k: d[tokey(k)] for k in data})
 
         if isinstance(data, type(range(0))):
@@ -1881,7 +1864,7 @@ def _scatter(
         if local_worker:  # running within task
             local_worker.update_data(data=data, report=False)
 
-            yield self.scheduler.update_data(
+            await self.scheduler.update_data(
                 who_has={key: [local_worker.address] for key in data},
                 nbytes=valmap(sizeof, data),
                 client=self.id,
@@ -1894,22 +1877,22 @@ def _scatter(
                 start = time()
                 while not nthreads:
                     if nthreads is not None:
-                        yield gen.sleep(0.1)
+                        await gen.sleep(0.1)
                     if time() > start + timeout:
                         raise gen.TimeoutError("No valid workers found")
-                    nthreads = yield self.scheduler.ncores(workers=workers)
+                    nthreads = await self.scheduler.ncores(workers=workers)
                 if not nthreads:
                     raise ValueError("No valid workers")
 
-                _, who_has, nbytes = yield scatter_to_workers(
+                _, who_has, nbytes = await scatter_to_workers(
                     nthreads, data2, report=False, rpc=self.rpc
                 )
 
-                yield self.scheduler.update_data(
+                await self.scheduler.update_data(
                     who_has=who_has, nbytes=nbytes, client=self.id
                 )
             else:
-                yield self.scheduler.scatter(
+                await self.scheduler.scatter(
                     data=data2,
                     workers=workers,
                     client=self.id,
@@ -1923,7 +1906,7 @@ def _scatter(
 
         if direct and broadcast:
             n = None if broadcast is True else broadcast
-            yield self._replicate(list(out.values()), workers=workers, n=n)
+            await self._replicate(list(out.values()), workers=workers, n=n)
 
         if issubclass(input_type, (list, tuple, set, frozenset)):
             out = input_type(out[k] for k in names)
@@ -1931,7 +1914,7 @@ def _scatter(
         if unpack:
             assert len(out) == 1
             out = list(out.values())[0]
-        raise gen.Return(out)
+        return out
 
     def scatter(
         self,
@@ -2030,10 +2013,9 @@ def scatter(
             hash=hash,
         )
 
-    @gen.coroutine
-    def _cancel(self, futures, force=False):
+    async def _cancel(self, futures, force=False):
         keys = list({tokey(f.key) for f in futures_of(futures)})
-        yield self.scheduler.cancel(keys=keys, client=self.id, force=force)
+        await self.scheduler.cancel(keys=keys, client=self.id, force=force)
         for k in keys:
             st = self.futures.pop(k, None)
             if st is not None:
@@ -2055,10 +2037,9 @@ def cancel(self, futures, asynchronous=None, force=False):
         """
         return self.sync(self._cancel, futures, asynchronous=asynchronous, force=force)
 
-    @gen.coroutine
-    def _retry(self, futures):
+    async def _retry(self, futures):
         keys = list({tokey(f.key) for f in futures_of(futures)})
-        response = yield self.scheduler.retry(keys=keys, client=self.id)
+        response = await self.scheduler.retry(keys=keys, client=self.id)
         for key in response:
             st = self.futures[key]
             st.retry()
@@ -2179,15 +2160,14 @@ def list_datasets(self, **kwargs):
         """
         return self.sync(self.scheduler.publish_list, **kwargs)
 
-    @gen.coroutine
-    def _get_dataset(self, name):
-        out = yield self.scheduler.publish_get(name=name, client=self.id)
+    async def _get_dataset(self, name):
+        out = await self.scheduler.publish_get(name=name, client=self.id)
         if out is None:
             raise KeyError("Dataset '%s' not found" % name)
 
         with temp_default_client(self):
             data = out["data"]
-        raise gen.Return(data)
+        return data
 
     def get_dataset(self, name, **kwargs):
         """
@@ -2200,15 +2180,14 @@ def get_dataset(self, name, **kwargs):
         """
         return self.sync(self._get_dataset, name, **kwargs)
 
-    @gen.coroutine
-    def _run_on_scheduler(self, function, *args, wait=True, **kwargs):
-        response = yield self.scheduler.run_function(
+    async def _run_on_scheduler(self, function, *args, wait=True, **kwargs):
+        response = await self.scheduler.run_function(
             function=dumps(function), args=dumps(args), kwargs=dumps(kwargs), wait=wait
         )
         if response["status"] == "error":
             six.reraise(*clean_exception(**response))
         else:
-            raise gen.Return(response["result"])
+            return response["result"]
 
     def run_on_scheduler(self, function, *args, **kwargs):
         """ Run a function on the scheduler process
@@ -2242,9 +2221,10 @@ def run_on_scheduler(self, function, *args, **kwargs):
         """
         return self.sync(self._run_on_scheduler, function, *args, **kwargs)
 
-    @gen.coroutine
-    def _run(self, function, *args, nanny=False, workers=None, wait=True, **kwargs):
-        responses = yield self.scheduler.broadcast(
+    async def _run(
+        self, function, *args, nanny=False, workers=None, wait=True, **kwargs
+    ):
+        responses = await self.scheduler.broadcast(
             msg=dict(
                 op="run",
                 function=dumps(function),
@@ -2262,7 +2242,7 @@ def _run(self, function, *args, nanny=False, workers=None, wait=True, **kwargs):
             elif resp["status"] == "error":
                 six.reraise(*clean_exception(**resp))
         if wait:
-            raise gen.Return(results)
+            return results
 
     def run(self, function, *args, **kwargs):
         """
@@ -2822,14 +2802,13 @@ def persist(
         else:
             return result
 
-    @gen.coroutine
-    def _restart(self, timeout=no_default):
+    async def _restart(self, timeout=no_default):
         if timeout == no_default:
             timeout = self._timeout * 2
         self._send_to_scheduler({"op": "restart", "timeout": timeout})
         self._restart_event = Event()
         try:
-            yield self._restart_event.wait(self.loop.time() + timeout)
+            await self._restart_event.wait(self.loop.time() + timeout)
         except gen.TimeoutError:
             logger.error("Restart timed out after %f seconds", timeout)
             pass
@@ -2837,7 +2816,7 @@ def _restart(self, timeout=no_default):
         with self._refcount_lock:
             self.refcount.clear()
 
-        raise gen.Return(self)
+        return self
 
     def restart(self, **kwargs):
         """ Restart the distributed network
@@ -2847,12 +2826,11 @@ def restart(self, **kwargs):
         """
         return self.sync(self._restart, **kwargs)
 
-    @gen.coroutine
-    def _upload_file(self, filename, raise_on_error=True):
+    async def _upload_file(self, filename, raise_on_error=True):
         with open(filename, "rb") as f:
             data = f.read()
         _, fn = os.path.split(filename)
-        d = yield self.scheduler.broadcast(
+        d = await self.scheduler.broadcast(
             msg={"op": "upload_file", "filename": fn, "data": to_serialize(data)}
         )
 
@@ -2861,21 +2839,20 @@ def _upload_file(self, filename, raise_on_error=True):
             if raise_on_error:
                 raise exceptions[0]
             else:
-                raise gen.Return(exceptions[0])
+                return exceptions[0]
 
         assert all(len(data) == v["nbytes"] for v in d.values())
 
-    @gen.coroutine
-    def _upload_large_file(self, local_filename, remote_filename=None):
+    async def _upload_large_file(self, local_filename, remote_filename=None):
         if remote_filename is None:
             remote_filename = os.path.split(local_filename)[1]
 
         with open(local_filename, "rb") as f:
             data = f.read()
 
-        [future] = yield self._scatter([data])
+        [future] = await self._scatter([data])
         key = future.key
-        yield self._replicate(future)
+        await self._replicate(future)
 
         def dump_to_file(dask_worker=None):
             if not os.path.isabs(remote_filename):
@@ -2887,7 +2864,7 @@ def dump_to_file(dask_worker=None):
 
             return len(dask_worker.data[key])
 
-        response = yield self._run(dump_to_file)
+        response = await self._run(dump_to_file)
 
         assert all(len(data) == v for v in response.values())
 
@@ -2917,11 +2894,10 @@ def upload_file(self, filename, **kwargs):
         else:
             return result
 
-    @gen.coroutine
-    def _rebalance(self, futures=None, workers=None):
-        yield _wait(futures)
+    async def _rebalance(self, futures=None, workers=None):
+        await _wait(futures)
         keys = list({tokey(f.key) for f in self.futures_of(futures)})
-        result = yield self.scheduler.rebalance(keys=keys, workers=workers)
+        result = await self.scheduler.rebalance(keys=keys, workers=workers)
         assert result["status"] == "OK"
 
     def rebalance(self, futures=None, workers=None, **kwargs):
@@ -2944,12 +2920,11 @@ def rebalance(self, futures=None, workers=None, **kwargs):
         """
         return self.sync(self._rebalance, futures, workers, **kwargs)
 
-    @gen.coroutine
-    def _replicate(self, futures, n=None, workers=None, branching_factor=2):
+    async def _replicate(self, futures, n=None, workers=None, branching_factor=2):
         futures = self.futures_of(futures)
-        yield _wait(futures)
+        await _wait(futures)
         keys = {tokey(f.key) for f in futures}
-        yield self.scheduler.replicate(
+        await self.scheduler.replicate(
             keys=list(keys), n=n, workers=workers, branching_factor=branching_factor
         )
 
@@ -3230,8 +3205,7 @@ def profile(
             filename=filename,
         )
 
-    @gen.coroutine
-    def _profile(
+    async def _profile(
         self,
         key=None,
         start=None,
@@ -3244,7 +3218,7 @@ def _profile(
         if isinstance(workers, six.string_types + (Number,)):
             workers = [workers]
 
-        state = yield self.scheduler.profile(
+        state = await self.scheduler.profile(
             key=key,
             workers=workers,
             merge_workers=merge_workers,
@@ -3268,10 +3242,10 @@ def _profile(
                 from bokeh.plotting import save
 
                 save(figure, title="Dask Profile", filename=filename)
-            raise gen.Return((state, figure))
+            return (state, figure)
 
         else:
-            raise gen.Return(state)
+            return state
 
     def scheduler_info(self, **kwargs):
         """ Basic information about the workers in the cluster
@@ -3517,15 +3491,14 @@ def futures_of(self, futures):
     def start_ipython(self, *args, **kwargs):
         raise Exception("Method moved to start_ipython_workers")
 
-    @gen.coroutine
-    def _start_ipython_workers(self, workers):
+    async def _start_ipython_workers(self, workers):
         if workers is None:
-            workers = yield self.scheduler.ncores()
+            workers = await self.scheduler.ncores()
 
-        responses = yield self.scheduler.broadcast(
+        responses = await self.scheduler.broadcast(
             msg=dict(op="start_ipython"), workers=workers
         )
-        raise gen.Return((workers, responses))
+        return workers, responses
 
     def start_ipython_workers(
         self, workers=None, magic_names=False, qtconsole=False, qtconsole_args=None
@@ -3840,11 +3813,10 @@ def get_task_stream(
             filename=filename,
         )
 
-    @gen.coroutine
-    def _get_task_stream(
+    async def _get_task_stream(
         self, start=None, stop=None, count=None, plot=False, filename="task-stream.html"
     ):
-        msgs = yield self.scheduler.get_task_stream(start=start, stop=stop, count=count)
+        msgs = await self.scheduler.get_task_stream(start=start, stop=stop, count=count)
         if plot:
             from .diagnostics.task_stream import rectangles
 
@@ -3857,9 +3829,9 @@ def _get_task_stream(
                 from bokeh.plotting import save
 
                 save(figure, title="Dask Task Stream", filename=filename)
-            raise gen.Return((msgs, figure))
+            return (msgs, figure)
         else:
-            raise gen.Return(msgs)
+            return msgs
 
     def register_worker_callbacks(self, setup=None):
         """
@@ -3881,9 +3853,8 @@ def register_worker_callbacks(self, setup=None):
         """
         return self.register_worker_plugin(_WorkerSetupPlugin(setup))
 
-    @gen.coroutine
-    def _register_worker_plugin(self, plugin=None, name=None):
-        responses = yield self.scheduler.register_worker_plugin(
+    async def _register_worker_plugin(self, plugin=None, name=None):
+        responses = await self.scheduler.register_worker_plugin(
             plugin=dumps(plugin), name=name
         )
         for response in responses.values():
@@ -3892,7 +3863,7 @@ def _register_worker_plugin(self, plugin=None, name=None):
                 typ = type(exc)
                 tb = response["traceback"]
                 six.reraise(typ, exc, tb)
-        raise gen.Return(responses)
+        return responses
 
     def register_worker_plugin(self, plugin=None, name=None):
         """
@@ -3976,8 +3947,7 @@ def CompatibleExecutor(*args, **kwargs):
 FIRST_COMPLETED = "FIRST_COMPLETED"
 
 
-@gen.coroutine
-def _wait(fs, timeout=None, return_when=ALL_COMPLETED):
+async def _wait(fs, timeout=None, return_when=ALL_COMPLETED):
     if timeout is not None and not isinstance(timeout, Number):
         raise TypeError(
             "timeout= keyword received a non-numeric value.\n"
@@ -3998,7 +3968,7 @@ def _wait(fs, timeout=None, return_when=ALL_COMPLETED):
     future = wait_for({f._state.wait() for f in fs})
     if timeout is not None:
         future = gen.with_timeout(timedelta(seconds=timeout), future)
-    yield future
+    await future
 
     done, not_done = (
         {fu for fu in fs if fu.status != "pending"},
@@ -4008,7 +3978,7 @@ def _wait(fs, timeout=None, return_when=ALL_COMPLETED):
     if cancelled:
         raise CancelledError(cancelled)
 
-    raise gen.Return(DoneAndNotDoneFutures(done, not_done))
+    return DoneAndNotDoneFutures(done, not_done)
 
 
 def wait(fs, timeout=None, return_when=ALL_COMPLETED):
@@ -4027,32 +3997,32 @@ def wait(fs, timeout=None, return_when=ALL_COMPLETED):
     return result
 
 
-@gen.coroutine
-def _as_completed(fs, queue):
+async def _as_completed(fs, queue):
     fs = futures_of(fs)
     groups = groupby(lambda f: f.key, fs)
     firsts = [v[0] for v in groups.values()]
-    wait_iterator = gen.WaitIterator(*[f._state.wait() for f in firsts])
+    wait_iterator = gen.WaitIterator(
+        *map(asyncio.ensure_future, [f._state.wait() for f in firsts])
+    )
 
     while not wait_iterator.done():
-        yield wait_iterator.next()
+        await wait_iterator.next()
         # TODO: handle case of restarted futures
         future = firsts[wait_iterator.current_index]
         for f in groups[future.key]:
             queue.put_nowait(f)
 
 
-@gen.coroutine
-def _first_completed(futures):
+async def _first_completed(futures):
     """ Return a single completed future
 
     See Also:
         _as_completed
     """
     q = Queue()
-    yield _as_completed(futures, q)
-    result = yield q.get()
-    raise gen.Return(result)
+    await _as_completed(futures, q)
+    result = await q.get()
+    return result
 
 
 class as_completed(object):
@@ -4133,15 +4103,14 @@ def _notify(self):
         with self.thread_condition:
             self.thread_condition.notify()
 
-    @gen.coroutine
-    def _track_future(self, future):
+    async def _track_future(self, future):
         try:
-            yield _wait(future)
+            await _wait(future)
         except CancelledError:
             pass
         if self.with_results:
             try:
-                result = yield future._result(raiseit=False)
+                result = await future._result(raiseit=False)
             except CancelledError as exc:
                 result = exc
         with self.lock:
@@ -4212,16 +4181,15 @@ def __next__(self):
                 self.thread_condition.wait(timeout=0.100)
         return self._get_and_raise()
 
-    @gen.coroutine
-    def __anext__(self):
+    async def __anext__(self):
         if not self.futures and self.queue.empty():
             raise StopAsyncIteration
         while self.queue.empty():
             if not self.futures:
                 raise StopAsyncIteration
-            yield self.condition.wait()
+            await self.condition.wait()
 
-        raise gen.Return(self._get_and_raise())
+        return self._get_and_raise()
 
     next = __next__
 
@@ -4443,13 +4411,11 @@ def __exit__(self, typ, value, traceback):
             L, self.figure = L
         self.data.extend(L)
 
-    @gen.coroutine
-    def __aenter__(self):
-        raise gen.Return(self)
+    async def __aenter__(self):
+        return self
 
-    @gen.coroutine
-    def __aexit__(self, typ, value, traceback):
-        L = yield self.client.get_task_stream(
+    async def __aexit__(self, typ, value, traceback):
+        L = await self.client.get_task_stream(
             start=self.start, plot=self._plot, filename=self._filename
         )
         if self._plot:
diff --git a/distributed/comm/core.py b/distributed/comm/core.py
index e0b236e7b96..869cb9b377f 100644
--- a/distributed/comm/core.py
+++ b/distributed/comm/core.py
@@ -178,8 +178,7 @@ def connect(self, address, deserialize=True):
         """
 
 
-@gen.coroutine
-def connect(addr, timeout=None, deserialize=True, connection_args=None):
+async def connect(addr, timeout=None, deserialize=True, connection_args=None):
     """
     Connect to the given address (a URI such as ``tcp://127.0.0.1:1234``)
     and yield a ``Comm`` object.  If the connection attempt fails, it is
@@ -212,7 +211,7 @@ def _raise(error):
             future = connector.connect(
                 loc, deserialize=deserialize, **(connection_args or {})
             )
-            comm = yield gen.with_timeout(
+            comm = await gen.with_timeout(
                 timedelta(seconds=deadline - time()),
                 future,
                 quiet_exceptions=EnvironmentError,
@@ -222,7 +221,7 @@ def _raise(error):
         except EnvironmentError as e:
             error = str(e)
             if time() < deadline:
-                yield gen.sleep(0.01)
+                await gen.sleep(0.01)
                 logger.debug("sleeping on connect")
             else:
                 _raise(error)
@@ -231,7 +230,7 @@ def _raise(error):
         else:
             break
 
-    raise gen.Return(comm)
+    return comm
 
 
 def listen(addr, handle_comm, deserialize=True, connection_args=None):
diff --git a/distributed/comm/inproc.py b/distributed/comm/inproc.py
index 7f267978d51..c9a6dc90281 100644
--- a/distributed/comm/inproc.py
+++ b/distributed/comm/inproc.py
@@ -7,7 +7,7 @@
 import threading
 import weakref
 
-from tornado import gen, locks
+from tornado import locks
 from tornado.concurrent import Future
 from tornado.ioloop import IOLoop
 
@@ -180,12 +180,11 @@ def local_address(self):
     def peer_address(self):
         return self._peer_addr
 
-    @gen.coroutine
-    def read(self, deserializers="ignored"):
+    async def read(self, deserializers="ignored"):
         if self._closed:
             raise CommClosedError
 
-        msg = yield self._read_q.get()
+        msg = await self._read_q.get()
         if msg is _EOF:
             self._closed = True
             self._finalizer.detach()
@@ -193,20 +192,18 @@ def read(self, deserializers="ignored"):
 
         if self.deserialize:
             msg = nested_deserialize(msg)
-        raise gen.Return(msg)
+        return msg
 
-    @gen.coroutine
-    def write(self, msg, serializers=None, on_error=None):
+    async def write(self, msg, serializers=None, on_error=None):
         if self.closed():
             raise CommClosedError
 
         # Ensure we feed the queue in the same thread it is read from.
         self._write_loop.add_callback(self._write_q.put_nowait, msg)
 
-        raise gen.Return(1)
+        return 1
 
-    @gen.coroutine
-    def close(self):
+    async def close(self):
         self.abort()
 
     def abort(self):
@@ -246,10 +243,9 @@ def __init__(self, address, comm_handler, deserialize=True):
         self.deserialize = deserialize
         self.listen_q = Queue()
 
-    @gen.coroutine
-    def _listen(self):
+    async def _listen(self):
         while True:
-            conn_req = yield self.listen_q.get()
+            conn_req = await self.listen_q.get()
             if conn_req is None:
                 break
             comm = InProc(
@@ -262,7 +258,7 @@ def _listen(self):
             )
             # Notify connector
             conn_req.c_loop.add_callback(conn_req.conn_event.set)
-            self.comm_handler(comm)
+            IOLoop.current().add_callback(self.comm_handler, comm)
 
     def connect_threadsafe(self, conn_req):
         self.loop.add_callback(self.listen_q.put_nowait, conn_req)
@@ -289,8 +285,7 @@ class InProcConnector(Connector):
     def __init__(self, manager):
         self.manager = manager
 
-    @gen.coroutine
-    def connect(self, address, deserialize=True, **connection_args):
+    async def connect(self, address, deserialize=True, **connection_args):
         listener = self.manager.get_listener_for(address)
         if listener is None:
             raise IOError("no endpoint for inproc address %r" % (address,))
@@ -306,7 +301,7 @@ def connect(self, address, deserialize=True, **connection_args):
         # Wait for connection acknowledgement
         # (do not pretend we're connected if the other comm never gets
         #  created, for example if the listener was stopped in the meantime)
-        yield conn_req.conn_event.wait()
+        await conn_req.conn_event.wait()
 
         comm = InProc(
             local_addr="inproc://" + conn_req.c_addr,
@@ -316,7 +311,7 @@ def connect(self, address, deserialize=True, **connection_args):
             write_loop=listener.loop,
             deserialize=deserialize,
         )
-        raise gen.Return(comm)
+        return comm
 
 
 class InProcBackend(Backend):
diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py
index d5351c7d565..602c9a36253 100644
--- a/distributed/comm/tcp.py
+++ b/distributed/comm/tcp.py
@@ -5,6 +5,7 @@
 import socket
 import struct
 import sys
+from tornado import gen
 
 try:
     import ssl
@@ -13,7 +14,7 @@
 
 import dask
 import tornado
-from tornado import gen, netutil
+from tornado import netutil
 from tornado.iostream import StreamClosedError, IOStream
 from tornado.tcpclient import TCPClient
 from tornado.tcpserver import TCPServer
@@ -184,16 +185,15 @@ def local_address(self):
     def peer_address(self):
         return self._peer_addr
 
-    @gen.coroutine
-    def read(self, deserializers=None):
+    async def read(self, deserializers=None):
         stream = self.stream
         if stream is None:
             raise CommClosedError
 
         try:
-            n_frames = yield stream.read_bytes(8)
+            n_frames = await stream.read_bytes(8)
             n_frames = struct.unpack("Q", n_frames)[0]
-            lengths = yield stream.read_bytes(8 * n_frames)
+            lengths = await stream.read_bytes(8 * n_frames)
             lengths = struct.unpack("Q" * n_frames, lengths)
 
             frames = []
@@ -201,10 +201,10 @@ def read(self, deserializers=None):
                 if length:
                     if PY3 and self._iostream_has_read_into:
                         frame = bytearray(length)
-                        n = yield stream.read_into(frame)
+                        n = await stream.read_into(frame)
                         assert n == length, (n, length)
                     else:
-                        frame = yield stream.read_bytes(length)
+                        frame = await stream.read_bytes(length)
                 else:
                     frame = b""
                 frames.append(frame)
@@ -214,14 +214,14 @@ def read(self, deserializers=None):
                 convert_stream_closed_error(self, e)
         else:
             try:
-                msg = yield from_frames(
+                msg = await from_frames(
                     frames, deserialize=self.deserialize, deserializers=deserializers
                 )
             except EOFError:
                 # Frames possibly garbled or truncated by communication error
                 self.abort()
                 raise CommClosedError("aborted stream on truncated data")
-            raise gen.Return(msg)
+            return msg
 
     @gen.coroutine
     def write(self, msg, serializers=None, on_error="message"):
@@ -268,16 +268,15 @@ def write(self, msg, serializers=None, on_error="message"):
             else:
                 raise
 
-        raise gen.Return(sum(map(nbytes, frames)))
+        return sum(map(nbytes, frames))
 
-    @gen.coroutine
-    def close(self):
+    async def close(self):
         stream, self.stream = self.stream, None
         if stream is not None and not stream.closed():
             try:
                 # Flush the stream's write buffer by waiting for a last write.
                 if stream.writing():
-                    yield stream.write(b"")
+                    await stream.write(b"")
                 stream.socket.shutdown(socket.SHUT_RDWR)
             except EnvironmentError:
                 pass
@@ -348,14 +347,13 @@ class BaseTCPConnector(Connector, RequireEncryptionMixin):
         _resolver = None
     client = TCPClient(resolver=_resolver)
 
-    @gen.coroutine
-    def connect(self, address, deserialize=True, **connection_args):
+    async def connect(self, address, deserialize=True, **connection_args):
         self._check_encryption(address, connection_args)
         ip, port = parse_host_port(address)
         kwargs = self._get_connect_args(**connection_args)
 
         try:
-            stream = yield BaseTCPConnector.client.connect(
+            stream = await BaseTCPConnector.client.connect(
                 ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs
             )
 
@@ -371,8 +369,8 @@ def connect(self, address, deserialize=True, **connection_args):
             convert_stream_closed_error(self, e)
 
         local_address = self.prefix + get_stream_address(stream)
-        raise gen.Return(
-            self.comm_class(stream, local_address, self.prefix + address, deserialize)
+        return self.comm_class(
+            stream, local_address, self.prefix + address, deserialize
         )
 
 
@@ -442,17 +440,16 @@ def _check_started(self):
         if self.tcp_server is None:
             raise ValueError("invalid operation on non-started TCPListener")
 
-    @gen.coroutine
-    def _handle_stream(self, stream, address):
+    async def _handle_stream(self, stream, address):
         address = self.prefix + unparse_host_port(*address[:2])
-        stream = yield self._prepare_stream(stream, address)
+        stream = await self._prepare_stream(stream, address)
         if stream is None:
             # Preparation failed
             return
         logger.debug("Incoming connection from %r to %r", address, self.contact_address)
         local_address = self.prefix + get_stream_address(stream)
         comm = self.comm_class(stream, local_address, address, self.deserialize)
-        yield self.comm_handler(comm)
+        await self.comm_handler(comm)
 
     def get_host_port(self):
         """
@@ -490,9 +487,8 @@ class TCPListener(BaseTCPListener):
     def _get_server_args(self, **connection_args):
         return {}
 
-    @gen.coroutine
-    def _prepare_stream(self, stream, address):
-        raise gen.Return(stream)
+    async def _prepare_stream(self, stream, address):
+        return stream
 
 
 class TLSListener(BaseTCPListener):
@@ -504,10 +500,9 @@ def _get_server_args(self, **connection_args):
         ctx = _expect_tls_context(connection_args)
         return {"ssl_options": ctx}
 
-    @gen.coroutine
-    def _prepare_stream(self, stream, address):
+    async def _prepare_stream(self, stream, address):
         try:
-            yield stream.wait_for_handshake()
+            await stream.wait_for_handshake()
         except EnvironmentError as e:
             # The handshake went wrong, log and ignore
             logger.warning(
@@ -517,7 +512,7 @@ def _prepare_stream(self, stream, address):
                 getattr(e, "real_error", None) or e,
             )
         else:
-            raise gen.Return(stream)
+            return stream
 
 
 class BaseTCPBackend(Backend):
diff --git a/distributed/comm/tests/test_comms.py b/distributed/comm/tests/test_comms.py
index e761deeab86..5d52b04a137 100644
--- a/distributed/comm/tests/test_comms.py
+++ b/distributed/comm/tests/test_comms.py
@@ -800,14 +800,14 @@ def handle_comm(comm):
             assert comm.closed()
             listener_errors.append(True)
         else:
-            comm.close()
+            yield comm.close()
 
     listener = listen("inproc://", handle_comm)
     listener.start()
     contact_addr = listener.contact_address
 
     comm = yield connect(contact_addr)
-    comm.close()
+    yield comm.close()
     assert comm.closed()
     start = time()
     while len(listener_errors) < 1:
@@ -821,7 +821,7 @@ def handle_comm(comm):
         yield comm.write("foo")
 
     comm = yield connect(contact_addr)
-    comm.write("foo")
+    yield comm.write("foo")
     with pytest.raises(CommClosedError):
         yield comm.read()
     with pytest.raises(CommClosedError):
@@ -829,15 +829,15 @@ def handle_comm(comm):
     assert comm.closed()
 
     comm = yield connect(contact_addr)
-    comm.write("foo")
+    yield comm.write("foo")
 
     start = time()
     while not comm.closed():
         yield gen.sleep(0.01)
         assert time() < start + 2
 
-    comm.close()
-    comm.close()
+    yield comm.close()
+    yield comm.close()
 
 
 #
diff --git a/distributed/core.py b/distributed/core.py
index 8aac2edfc33..d8a34859359 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -1,5 +1,6 @@
 from __future__ import print_function, division, absolute_import
 
+import asyncio
 from collections import defaultdict, deque
 from concurrent.futures import CancelledError
 from functools import partial
@@ -29,6 +30,7 @@
 from . import profile
 from .system_monitor import SystemMonitor
 from .utils import (
+    is_coroutine_function,
     get_traceback,
     truncate_exception,
     ignoring,
@@ -204,7 +206,6 @@ def stop():
 
         self.thread_id = 0
 
-        @gen.coroutine
         def set_thread_ident():
             self.thread_id = get_thread_identity()
 
@@ -326,8 +327,7 @@ def listen(self, port_or_addr=None, listen_args=None):
         )
         self.listener.start()
 
-    @gen.coroutine
-    def handle_comm(self, comm, shutting_down=shutting_down):
+    async def handle_comm(self, comm, shutting_down=shutting_down):
         """ Dispatch new communications to coroutine-handlers
 
         Handlers is a dictionary mapping operation names to functions or
@@ -349,7 +349,7 @@ def handle_comm(self, comm, shutting_down=shutting_down):
         try:
             while True:
                 try:
-                    msg = yield comm.read()
+                    msg = await comm.read()
                     logger.debug("Message from %r: %s", address, msg)
                 except EnvironmentError as e:
                     if not shutting_down():
@@ -363,7 +363,7 @@ def handle_comm(self, comm, shutting_down=shutting_down):
                     break
                 except Exception as e:
                     logger.exception(e)
-                    yield comm.write(error_message(e, status="uncaught-error"))
+                    await comm.write(error_message(e, status="uncaught-error"))
                     continue
                 if not isinstance(msg, dict):
                     raise TypeError(
@@ -384,7 +384,7 @@ def handle_comm(self, comm, shutting_down=shutting_down):
                 reply = msg.pop("reply", True)
                 if op == "close":
                     if reply:
-                        yield comm.write("OK")
+                        await comm.write("OK")
                     break
 
                 result = None
@@ -412,9 +412,10 @@ def handle_comm(self, comm, shutting_down=shutting_down):
                     logger.debug("Calling into handler %s", handler.__name__)
                     try:
                         result = handler(comm, **msg)
-                        if type(result) is gen.Future:
+                        if hasattr(result, "__await__"):
+                            result = asyncio.ensure_future(result)
                             self._ongoing_coroutines.add(result)
-                            result = yield result
+                            result = await result
                     except (CommClosedError, CancelledError) as e:
                         if self.status == "running":
                             logger.info("Lost connection to %r: %s", address, e)
@@ -425,7 +426,7 @@ def handle_comm(self, comm, shutting_down=shutting_down):
 
                 if reply and result != "dont-reply":
                     try:
-                        yield comm.write(result, serializers=serializers)
+                        await comm.write(result, serializers=serializers)
                     except (EnvironmentError, TypeError) as e:
                         logger.debug(
                             "Lost connection to %r while sending result for op %r: %s",
@@ -436,7 +437,7 @@ def handle_comm(self, comm, shutting_down=shutting_down):
                         break
                 msg = result = None
                 if close_desired:
-                    yield comm.close()
+                    await comm.close()
                 if comm.closed():
                     break
 
@@ -450,8 +451,7 @@ def handle_comm(self, comm, shutting_down=shutting_down):
                         "Failed while closing connection to %r: %s", address, e
                     )
 
-    @gen.coroutine
-    def handle_stream(self, comm, extra=None, every_cycle=[]):
+    async def handle_stream(self, comm, extra=None, every_cycle=[]):
         extra = extra or {}
         logger.info("Starting established connection")
 
@@ -459,7 +459,7 @@ def handle_stream(self, comm, extra=None, every_cycle=[]):
         closed = False
         try:
             while not closed:
-                msgs = yield comm.read()
+                msgs = await comm.read()
                 if not isinstance(msgs, (tuple, list)):
                     msgs = (msgs,)
 
@@ -473,9 +473,14 @@ def handle_stream(self, comm, extra=None, every_cycle=[]):
                                 closed = True
                                 break
                             handler = self.stream_handlers[op]
-                            handler(**merge(extra, msg))
+                            if is_coroutine_function(handler):
+                                self.loop.add_callback(handler, **merge(extra, msg))
+                            else:
+                                handler(**merge(extra, msg))
                         else:
                             logger.error("odd message %s", msg)
+                    await gen.sleep(0)
+
                 for func in every_cycle:
                     func()
 
@@ -489,7 +494,7 @@ def handle_stream(self, comm, extra=None, every_cycle=[]):
                 pdb.set_trace()
             raise
         finally:
-            yield comm.close()
+            await comm.close()
             assert comm.closed()
 
     @gen.coroutine
@@ -519,8 +524,7 @@ def pingpong(comm):
     return b"pong"
 
 
-@gen.coroutine
-def send_recv(comm, reply=True, serializers=None, deserializers=None, **kwargs):
+async def send_recv(comm, reply=True, serializers=None, deserializers=None, **kwargs):
     """ Send and recv with a Comm.
 
     Keyword arguments turn into the message
@@ -537,9 +541,9 @@ def send_recv(comm, reply=True, serializers=None, deserializers=None, **kwargs):
         msg["serializers"] = deserializers
 
     try:
-        yield comm.write(msg, serializers=serializers, on_error="raise")
+        await comm.write(msg, serializers=serializers, on_error="raise")
         if reply:
-            response = yield comm.read(deserializers=deserializers)
+            response = await comm.read(deserializers=deserializers)
         else:
             response = None
     except EnvironmentError:
@@ -548,7 +552,7 @@ def send_recv(comm, reply=True, serializers=None, deserializers=None, **kwargs):
         raise
     finally:
         if please_close:
-            yield comm.close()
+            await comm.close()
         elif force_close:
             comm.abort()
 
@@ -557,7 +561,7 @@ def send_recv(comm, reply=True, serializers=None, deserializers=None, **kwargs):
             six.reraise(*clean_exception(**response))
         else:
             raise Exception(response["text"])
-    raise gen.Return(response)
+    return response
 
 
 def addr_from_args(addr=None, ip=None, port=None):
@@ -610,8 +614,7 @@ def __init__(
         self._created = weakref.WeakSet()
         rpc.active.add(self)
 
-    @gen.coroutine
-    def live_comm(self):
+    async def live_comm(self):
         """ Get an open communication
 
         Some comms to the ip/port target may be in current use by other
@@ -641,7 +644,7 @@ def live_comm(self):
         for s in to_clear:
             del self.comms[s]
         if not open or comm.closed():
-            comm = yield connect(
+            comm = await connect(
                 self.address,
                 self.timeout,
                 deserialize=self.deserialize,
@@ -649,44 +652,46 @@ def live_comm(self):
             )
             comm.name = "rpc"
         self.comms[comm] = False  # mark as taken
-        raise gen.Return(comm)
+        return comm
 
     def close_comms(self):
         @gen.coroutine
         def _close_comm(comm):
             # Make sure we tell the peer to close
             try:
-                yield comm.write({"op": "close", "reply": False})
-                yield comm.close()
+                if not comm.closed():
+                    yield comm.write({"op": "close", "reply": False})
+                    yield comm.close()
             except EnvironmentError:
                 comm.abort()
 
         for comm in list(self.comms):
             if comm and not comm.closed():
-                _close_comm(comm)
+                # IOLoop.current().add_callback(_close_comm, comm)
+                task = asyncio.ensure_future(_close_comm(comm))
         for comm in list(self._created):
             if comm and not comm.closed():
-                _close_comm(comm)
+                # IOLoop.current().add_callback(_close_comm, comm)
+                task = asyncio.ensure_future(_close_comm(comm))
         self.comms.clear()
 
     def __getattr__(self, key):
-        @gen.coroutine
-        def send_recv_from_rpc(**kwargs):
+        async def send_recv_from_rpc(**kwargs):
             if self.serializers is not None and kwargs.get("serializers") is None:
                 kwargs["serializers"] = self.serializers
             if self.deserializers is not None and kwargs.get("deserializers") is None:
                 kwargs["deserializers"] = self.deserializers
             try:
-                comm = yield self.live_comm()
+                comm = await self.live_comm()
                 comm.name = "rpc." + key
-                result = yield send_recv(comm=comm, op=key, **kwargs)
+                result = await send_recv(comm=comm, op=key, **kwargs)
             except (RPCClosed, CommClosedError) as e:
                 raise e.__class__(
                     "%s: while trying to call remote method %r" % (e, key)
                 )
 
             self.comms[comm] = True  # mark as open
-            raise gen.Return(result)
+            return result
 
         return send_recv_from_rpc
 
@@ -736,21 +741,20 @@ def address(self):
         return self.addr
 
     def __getattr__(self, key):
-        @gen.coroutine
-        def send_recv_from_rpc(**kwargs):
+        async def send_recv_from_rpc(**kwargs):
             if self.serializers is not None and kwargs.get("serializers") is None:
                 kwargs["serializers"] = self.serializers
             if self.deserializers is not None and kwargs.get("deserializers") is None:
                 kwargs["deserializers"] = self.deserializers
-            comm = yield self.pool.connect(self.addr)
+            comm = await self.pool.connect(self.addr)
             name, comm.name = comm.name, "ConnectionPool." + key
             try:
-                result = yield send_recv(comm=comm, op=key, **kwargs)
+                result = await send_recv(comm=comm, op=key, **kwargs)
             finally:
                 self.pool.reuse(self.addr, comm)
                 comm.name = name
 
-            raise gen.Return(result)
+            return result
 
         return send_recv_from_rpc
 
@@ -847,8 +851,7 @@ def __call__(self, addr=None, ip=None, port=None):
             addr, self, serializers=self.serializers, deserializers=self.deserializers
         )
 
-    @gen.coroutine
-    def connect(self, addr, timeout=None):
+    async def connect(self, addr, timeout=None):
         """
         Get a Comm to the given address.  For internal use.
         """
@@ -858,15 +861,15 @@ def connect(self, addr, timeout=None):
             comm = available.pop()
             if not comm.closed():
                 occupied.add(comm)
-                raise gen.Return(comm)
+                return comm
 
         while self.open >= self.limit:
             self.event.clear()
             self.collect()
-            yield self.event.wait()
+            await self.event.wait()
 
         try:
-            comm = yield connect(
+            comm = await connect(
                 addr,
                 timeout=timeout or self.timeout,
                 deserialize=self.deserialize,
@@ -882,7 +885,7 @@ def connect(self, addr, timeout=None):
         if self.open >= self.limit:
             self.event.clear()
 
-        raise gen.Return(comm)
+        return comm
 
     def reuse(self, addr, comm):
         """
diff --git a/distributed/dashboard/components.py b/distributed/dashboard/components.py
index e7234e2e6f7..242a617706e 100644
--- a/distributed/dashboard/components.py
+++ b/distributed/dashboard/components.py
@@ -1,5 +1,6 @@
 from __future__ import print_function, division, absolute_import
 
+import asyncio
 from bisect import bisect
 from operator import add
 from time import time
@@ -549,8 +550,7 @@ def update(self, state, metadata=None):
 
     @without_property_validation
     def trigger_update(self, update_metadata=True):
-        @gen.coroutine
-        def cb():
+        async def cb():
             with log_errors():
                 prof = self.server.get_profile(
                     key=self.key, start=self.start, stop=self.stop
@@ -560,7 +560,7 @@ def cb():
                 else:
                     metadata = None
                 if isinstance(prof, gen.Future):
-                    prof, metadata = yield [prof, metadata]
+                    prof, metadata = await asyncio.gather(prof, metadata)
                 self.doc().add_next_tick_callback(lambda: self.update(prof, metadata))
 
         self.server.loop.add_callback(cb)
diff --git a/distributed/dashboard/scheduler_html.py b/distributed/dashboard/scheduler_html.py
index 3f119a929b9..3087f323b5f 100644
--- a/distributed/dashboard/scheduler_html.py
+++ b/distributed/dashboard/scheduler_html.py
@@ -3,7 +3,6 @@
 from dask.utils import format_bytes
 import toolz
 from tornado import escape
-from tornado import gen
 
 from ..utils import log_errors, format_time
 from .proxy import GlobalProxyHandler
@@ -59,22 +58,20 @@ def get(self):
 
 
 class WorkerLogs(RequestHandler):
-    @gen.coroutine
-    def get(self, worker):
+    async def get(self, worker):
         with log_errors():
             worker = escape.url_unescape(worker)
-            logs = yield self.server.get_worker_logs(workers=[worker])
+            logs = await self.server.get_worker_logs(workers=[worker])
             logs = logs[worker]
             self.render("logs.html", title="Logs: " + worker, logs=logs, **self.extra)
 
 
 class WorkerCallStacks(RequestHandler):
-    @gen.coroutine
-    def get(self, worker):
+    async def get(self, worker):
         with log_errors():
             worker = escape.url_unescape(worker)
             keys = self.server.processing[worker]
-            call_stack = yield self.server.get_call_stack(keys=keys)
+            call_stack = await self.server.get_call_stack(keys=keys)
             self.render(
                 "call-stack.html",
                 title="Call Stacks: " + worker,
@@ -84,11 +81,10 @@ def get(self, worker):
 
 
 class TaskCallStack(RequestHandler):
-    @gen.coroutine
-    def get(self, key):
+    async def get(self, key):
         with log_errors():
             key = escape.url_unescape(key)
-            call_stack = yield self.server.get_call_stack(keys=[key])
+            call_stack = await self.server.get_call_stack(keys=[key])
             if not call_stack:
                 self.write(
                     "<p>Task not actively running. "
diff --git a/distributed/dashboard/tests/test_components.py b/distributed/dashboard/tests/test_components.py
index d441db57aec..b12780f199b 100644
--- a/distributed/dashboard/tests/test_components.py
+++ b/distributed/dashboard/tests/test_components.py
@@ -25,7 +25,7 @@ def test_basic(Component):
     assert isinstance(c.root, Model)
 
 
-@gen_cluster(client=True, check_new_threads=False)
+@gen_cluster(client=True, clean_kwargs={"threads": False})
 def test_profile_plot(c, s, a, b):
     p = ProfilePlot()
     assert not p.source.data["left"]
@@ -34,7 +34,7 @@ def test_profile_plot(c, s, a, b):
     assert len(p.source.data["left"]) >= 1
 
 
-@gen_cluster(client=True, check_new_threads=False)
+@gen_cluster(client=True, clean_kwargs={"threads": False})
 def test_profile_time_plot(c, s, a, b):
     from bokeh.io import curdoc
 
diff --git a/distributed/dashboard/tests/test_scheduler_bokeh_html.py b/distributed/dashboard/tests/test_scheduler_bokeh_html.py
index f872d02dc84..b66aff02ddc 100644
--- a/distributed/dashboard/tests/test_scheduler_bokeh_html.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh_html.py
@@ -73,7 +73,7 @@ def test_prefix(c, s, a, b):
 
 @gen_cluster(
     client=True,
-    check_new_threads=False,
+    clean_kwargs={"threads": False},
     scheduler_kwargs={"services": {("dashboard", 0): BokehScheduler}},
 )
 def test_prometheus(c, s, a, b):
@@ -98,7 +98,7 @@ def test_prometheus(c, s, a, b):
 
 @gen_cluster(
     client=True,
-    check_new_threads=False,
+    clean_kwargs={"threads": False},
     scheduler_kwargs={"services": {("dashboard", 0): BokehScheduler}},
 )
 def test_health(c, s, a, b):
diff --git a/distributed/dashboard/tests/test_worker_bokeh.py b/distributed/dashboard/tests/test_worker_bokeh.py
index ef977127d23..d320ea24ee8 100644
--- a/distributed/dashboard/tests/test_worker_bokeh.py
+++ b/distributed/dashboard/tests/test_worker_bokeh.py
@@ -169,7 +169,7 @@ def test_CommunicatingStream(c, s, a, b):
 
 @gen_cluster(
     client=True,
-    check_new_threads=False,
+    clean_kwargs={"threads": False},
     worker_kwargs={"services": {("dashboard", 0): BokehWorker}},
 )
 def test_prometheus(c, s, a, b):
diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index 441ef10a595..1ba8e7fb213 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -272,7 +272,7 @@ def close(self, timeout=None):
 
     def __del__(self):
         if self.status != "closed":
-            self.close()
+            self.loop.add_callback(self.close)
 
     def __enter__(self):
         self.sync(self._correct_state)
diff --git a/distributed/deploy/tests/test_adaptive.py b/distributed/deploy/tests/test_adaptive.py
index 861c5107348..e0478a9cbdb 100644
--- a/distributed/deploy/tests/test_adaptive.py
+++ b/distributed/deploy/tests/test_adaptive.py
@@ -255,21 +255,27 @@ def test_adapt_quickly():
         d = [x for x in adapt.log[-1] if isinstance(x, dict)][0]
         assert 2 < d["n"] <= adapt.maximum
 
-        while len(cluster.scheduler.workers) < adapt.maximum:
+        while len(cluster.workers) < adapt.maximum:
             yield gen.sleep(0.01)
 
         del futures
 
-        while len(cluster.scheduler.workers) > 1:
+        while len(cluster.scheduler.tasks) > 1:
+            yield gen.sleep(0.01)
+
+        yield cluster
+
+        while len(cluster.scheduler.workers) > 1 or len(cluster.worker_spec) > 1:
             yield gen.sleep(0.01)
 
         # Don't scale up for large sequential computations
         x = yield client.scatter(1)
+        log = list(cluster._adaptive.log)
         for i in range(100):
             x = client.submit(slowinc, x)
 
         yield gen.sleep(0.1)
-        assert len(cluster.scheduler.workers) == 1
+        assert len(cluster.workers) == 1
     finally:
         yield client.close()
         yield cluster.close()
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index d489b84df0f..1c098c2b4c5 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -401,8 +401,9 @@ def test_silent_startup():
         from time import sleep
         from distributed import LocalCluster
 
-        with LocalCluster(1, dashboard_address=None, scheduler_port=0):
-            sleep(1.5)
+        if __name__ == "__main__":
+            with LocalCluster(1, dashboard_address=None, scheduler_port=0):
+                sleep(1.5)
         """
 
     out = subprocess.check_output(
diff --git a/distributed/diagnostics/eventstream.py b/distributed/diagnostics/eventstream.py
index a4eb0830534..b9213144d4e 100644
--- a/distributed/diagnostics/eventstream.py
+++ b/distributed/diagnostics/eventstream.py
@@ -2,8 +2,6 @@
 
 import logging
 
-from tornado import gen
-
 from .plugin import SchedulerPlugin
 
 from ..core import connect, coerce_to_address
@@ -37,8 +35,7 @@ def teardown(scheduler, es):
     scheduler.remove_plugin(es)
 
 
-@gen.coroutine
-def eventstream(address, interval):
+async def eventstream(address, interval):
     """ Open a TCP connection to scheduler, receive batched task messages
 
     The messages coming back are lists of dicts.  Each dict is of the following
@@ -59,14 +56,14 @@ def eventstream(address, interval):
 
     Examples
     --------
-    >>> stream = yield eventstream('127.0.0.1:8786', 0.100)  # doctest: +SKIP
-    >>> print(yield read(stream))  # doctest: +SKIP
+    >>> stream = await eventstream('127.0.0.1:8786', 0.100)  # doctest: +SKIP
+    >>> print(await read(stream))  # doctest: +SKIP
     [{'key': 'x', 'status': 'OK', 'worker': '192.168.0.1:54684', ...},
      {'key': 'y', 'status': 'error', 'worker': '192.168.0.1:54684', ...}]
     """
     address = coerce_to_address(address)
-    comm = yield connect(address)
-    yield comm.write(
+    comm = await connect(address)
+    await comm.write(
         {
             "op": "feed",
             "setup": dumps_function(EventStream),
@@ -75,4 +72,4 @@ def eventstream(address, interval):
             "teardown": dumps_function(teardown),
         }
     )
-    raise gen.Return(comm)
+    return comm
diff --git a/distributed/diagnostics/progress.py b/distributed/diagnostics/progress.py
index 38638a248dd..50c4cd9fad1 100644
--- a/distributed/diagnostics/progress.py
+++ b/distributed/diagnostics/progress.py
@@ -74,12 +74,11 @@ def __init__(self, keys, scheduler, minimum=0, dt=0.1, complete=False):
         self.status = None
         self.extra = {}
 
-    @gen.coroutine
-    def setup(self):
+    async def setup(self):
         keys = self.keys
 
         while not keys.issubset(self.scheduler.tasks):
-            yield gen.sleep(0.05)
+            await gen.sleep(0.05)
 
         tasks = [self.scheduler.tasks[k] for k in keys]
 
@@ -163,12 +162,11 @@ def __init__(
             self, keys, scheduler, minimum=minimum, dt=dt, complete=complete
         )
 
-    @gen.coroutine
-    def setup(self):
+    async def setup(self):
         keys = self.keys
 
         while not keys.issubset(self.scheduler.tasks):
-            yield gen.sleep(0.05)
+            await gen.sleep(0.05)
 
         tasks = [self.scheduler.tasks[k] for k in keys]
 
diff --git a/distributed/diagnostics/progress_stream.py b/distributed/diagnostics/progress_stream.py
index 1630251658a..b1e3787bd5a 100644
--- a/distributed/diagnostics/progress_stream.py
+++ b/distributed/diagnostics/progress_stream.py
@@ -3,7 +3,6 @@
 import logging
 
 from toolz import valmap, merge
-from tornado import gen
 
 from .progress import AllProgress
 
@@ -26,8 +25,7 @@ def counts(scheduler, allprogress):
     )
 
 
-@gen.coroutine
-def progress_stream(address, interval):
+async def progress_stream(address, interval):
     """ Open a TCP connection to scheduler, receive progress messages
 
     The messages coming back are dicts containing counts of key groups::
@@ -42,12 +40,12 @@ def progress_stream(address, interval):
 
     Examples
     --------
-    >>> stream = yield eventstream('127.0.0.1:8786', 0.100)  # doctest: +SKIP
-    >>> print(yield read(stream))  # doctest: +SKIP
+    >>> stream = await eventstream('127.0.0.1:8786', 0.100)  # doctest: +SKIP
+    >>> print(await read(stream))  # doctest: +SKIP
     """
     address = coerce_to_address(address)
-    comm = yield connect(address)
-    yield comm.write(
+    comm = await connect(address)
+    await comm.write(
         {
             "op": "feed",
             "setup": dumps_function(AllProgress),
@@ -56,7 +54,7 @@ def progress_stream(address, interval):
             "teardown": dumps_function(Scheduler.remove_plugin),
         }
     )
-    raise gen.Return(comm)
+    return comm
 
 
 def nbytes_bar(nbytes):
diff --git a/distributed/diagnostics/progressbar.py b/distributed/diagnostics/progressbar.py
index f25bf32a871..8d57da779c6 100644
--- a/distributed/diagnostics/progressbar.py
+++ b/distributed/diagnostics/progressbar.py
@@ -6,7 +6,6 @@
 import weakref
 
 from toolz import valmap
-from tornado import gen
 from tornado.ioloop import IOLoop
 
 from .progress import format_time, Progress, MultiProgress
@@ -46,16 +45,14 @@ def __init__(self, keys, scheduler=None, interval="100ms", complete=True):
     def elapsed(self):
         return default_timer() - self._start_time
 
-    @gen.coroutine
-    def listen(self):
+    async def listen(self):
         complete = self.complete
         keys = self.keys
 
-        @gen.coroutine
-        def setup(scheduler):
+        async def setup(scheduler):
             p = Progress(keys, scheduler, complete=complete)
-            yield p.setup()
-            raise gen.Return(p)
+            await p.setup()
+            return p
 
         def function(scheduler, p):
             result = {
@@ -67,13 +64,13 @@ def function(scheduler, p):
                 result.update(p.extra)
             return result
 
-        self.comm = yield connect(
+        self.comm = await connect(
             self.scheduler,
             connection_args=self.client().connection_args if self.client else None,
         )
         logger.debug("Progressbar Connected to scheduler")
 
-        yield self.comm.write(
+        await self.comm.write(
             {
                 "op": "feed",
                 "setup": dumps(setup),
@@ -85,7 +82,7 @@ def function(scheduler, p):
 
         while True:
             try:
-                response = yield self.comm.read(
+                response = await self.comm.read(
                     deserializers=self.client()._deserializers if self.client else None
                 )
             except CommClosedError:
@@ -94,7 +91,7 @@ def function(scheduler, p):
             self.status = response["status"]
             self._draw_bar(**response)
             if response["status"] in ("error", "finished"):
-                yield self.comm.close()
+                await self.comm.close()
                 self._draw_stop(**response)
                 break
 
@@ -240,17 +237,15 @@ def __init__(
     def elapsed(self):
         return default_timer() - self._start_time
 
-    @gen.coroutine
-    def listen(self):
+    async def listen(self):
         complete = self.complete
         keys = self.keys
         func = self.func
 
-        @gen.coroutine
-        def setup(scheduler):
+        async def setup(scheduler):
             p = MultiProgress(keys, scheduler, complete=complete, func=func)
-            yield p.setup()
-            raise gen.Return(p)
+            await p.setup()
+            return p
 
         def function(scheduler, p):
             result = {
@@ -262,13 +257,13 @@ def function(scheduler, p):
                 result.update(p.extra)
             return result
 
-        self.comm = yield connect(
+        self.comm = await connect(
             self.scheduler,
             connection_args=self.client().connection_args if self.client else None,
         )
         logger.debug("Progressbar Connected to scheduler")
 
-        yield self.comm.write(
+        await self.comm.write(
             {
                 "op": "feed",
                 "setup": dumps(setup),
@@ -278,14 +273,14 @@ def function(scheduler, p):
         )
 
         while True:
-            response = yield self.comm.read(
+            response = await self.comm.read(
                 deserializers=self.client()._deserializers if self.client else None
             )
             self._last_response = response
             self.status = response["status"]
             self._draw_bar(**response)
             if response["status"] in ("error", "finished"):
-                yield self.comm.close()
+                await self.comm.close()
                 self._draw_stop(**response)
                 break
         logger.debug("Progressbar disconnected from scheduler")
diff --git a/distributed/diagnostics/tests/test_progressbar.py b/distributed/diagnostics/tests/test_progressbar.py
index 3e5f0633d49..4e6ffe8c7e9 100644
--- a/distributed/diagnostics/tests/test_progressbar.py
+++ b/distributed/diagnostics/tests/test_progressbar.py
@@ -2,10 +2,12 @@
 
 from time import sleep
 
+import pytest
+
 from distributed import Scheduler, Worker
 from distributed.diagnostics.progressbar import TextProgressBar, progress
 from distributed.metrics import time
-from distributed.utils_test import inc, div, gen_cluster, gen_test
+from distributed.utils_test import inc, div, gen_cluster
 from distributed.utils_test import client, loop, cluster_fixture  # noqa: F401
 
 
@@ -40,23 +42,18 @@ def test_TextProgressBar_error(c, s, a, b):
     assert progress.comm.closed()
 
 
-def test_TextProgressBar_empty(capsys):
-    @gen_test()
-    def f():
-        s = yield Scheduler(port=0)
-        a, b = yield [Worker(s.address, nthreads=1), Worker(s.address, nthreads=1)]
-
-        progress = TextProgressBar([], scheduler=s.address, start=False, interval=0.01)
-        yield progress.listen()
-
-        assert progress.status == "finished"
-        check_bar_completed(capsys)
-
-        yield [a.close(), b.close()]
-        s.close()
-        yield s.finished()
+@pytest.mark.asyncio
+async def test_TextProgressBar_empty(capsys):
+    async with Scheduler(port=0) as s:
+        async with Worker(s.address, nthreads=1) as a:
+            async with Worker(s.address, nthreads=1) as b:
+                progress = TextProgressBar(
+                    [], scheduler=s.address, start=False, interval=0.01
+                )
+                await progress.listen()
 
-    f()
+                assert progress.status == "finished"
+                check_bar_completed(capsys)
 
 
 def check_bar_completed(capsys, width=40):
diff --git a/distributed/lock.py b/distributed/lock.py
index d12b1c41e15..6ad6ab607d3 100644
--- a/distributed/lock.py
+++ b/distributed/lock.py
@@ -35,8 +35,7 @@ def __init__(self, scheduler):
 
         self.scheduler.extensions["locks"] = self
 
-    @gen.coroutine
-    def acquire(self, stream=None, name=None, id=None, timeout=None):
+    async def acquire(self, stream=None, name=None, id=None, timeout=None):
         with log_errors():
             if isinstance(name, list):
                 name = tuple(name)
@@ -50,7 +49,7 @@ def acquire(self, stream=None, name=None, id=None, timeout=None):
                     if timeout is not None:
                         future = gen.with_timeout(timedelta(seconds=timeout), future)
                     try:
-                        yield future
+                        await future
                     except gen.TimeoutError:
                         result = False
                         break
@@ -62,7 +61,7 @@ def acquire(self, stream=None, name=None, id=None, timeout=None):
             if result:
                 assert name not in self.ids
                 self.ids[name] = id
-            raise gen.Return(result)
+            return result
 
     def release(self, stream=None, name=None, id=None):
         with log_errors():
@@ -155,14 +154,12 @@ def __enter__(self):
     def __exit__(self, *args, **kwargs):
         self.release()
 
-    @gen.coroutine
-    def __aenter__(self):
-        yield self.acquire()
-        raise gen.Return(self)
+    async def __aenter__(self):
+        await self.acquire()
+        return self
 
-    @gen.coroutine
-    def __aexit__(self, *args, **kwargs):
-        yield self.release()
+    async def __aexit__(self, *args, **kwargs):
+        await self.release()
 
     def __reduce__(self):
         return (Lock, (self.name,))
diff --git a/distributed/nanny.py b/distributed/nanny.py
index f3bebb1dcac..6c859115242 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -191,8 +191,7 @@ def __init__(
     def __repr__(self):
         return "<Nanny: %s, threads: %d>" % (self.worker_address, self.nthreads)
 
-    @gen.coroutine
-    def _unregister(self, timeout=10):
+    async def _unregister(self, timeout=10):
         if self.process is None:
             return
         worker_address = self.process.worker_address
@@ -206,7 +205,7 @@ def _unregister(self, timeout=10):
             RPCClosed,
         )
         try:
-            yield gen.with_timeout(
+            await gen.with_timeout(
                 timedelta(seconds=timeout),
                 self.scheduler.unregister(address=self.worker_address),
                 quiet_exceptions=allowed_errors,
@@ -222,26 +221,24 @@ def worker_address(self):
     def worker_dir(self):
         return None if self.process is None else self.process.worker_dir
 
-    @gen.coroutine
-    def start(self):
+    async def start(self):
         """ Start nanny, start local process, start watching """
         self.listen(self._start_address, listen_args=self.listen_args)
         self.ip = get_address_host(self.address)
 
         logger.info("        Start Nanny at: %r", self.address)
-        response = yield self.instantiate()
+        response = await self.instantiate()
         if response == "running":
             assert self.worker_address
             self.status = "running"
         else:
-            yield self.close()
+            await self.close()
 
         self.start_periodic_callbacks()
 
         return self
 
-    @gen.coroutine
-    def kill(self, comm=None, timeout=2):
+    async def kill(self, comm=None, timeout=2):
         """ Kill the local worker process
 
         Blocks until both the process is down and the scheduler is properly
@@ -249,13 +246,12 @@ def kill(self, comm=None, timeout=2):
         """
         self.auto_restart = False
         if self.process is None:
-            raise gen.Return("OK")
+            return "OK"
 
         deadline = self.loop.time() + timeout
-        yield self.process.kill(timeout=0.8 * (deadline - self.loop.time()))
+        await self.process.kill(timeout=0.8 * (deadline - self.loop.time()))
 
-    @gen.coroutine
-    def instantiate(self, comm=None):
+    async def instantiate(self, comm=None):
         """ Start a local worker process
 
         Blocks until the process is up and the scheduler is properly informed
@@ -292,7 +288,7 @@ def instantiate(self, comm=None):
                 worker_kwargs=worker_kwargs,
                 worker_start_args=(start_arg,),
                 silence_logs=self.silence_logs,
-                on_exit=self._on_exit,
+                on_exit=self._on_exit_sync,
                 worker=self.Worker,
                 env=self.env,
             )
@@ -300,11 +296,11 @@ def instantiate(self, comm=None):
         self.auto_restart = True
         if self.death_timeout:
             try:
-                result = yield gen.with_timeout(
+                result = await gen.with_timeout(
                     timedelta(seconds=self.death_timeout), self.process.start()
                 )
             except gen.TimeoutError:
-                yield self.close(timeout=self.death_timeout)
+                await self.close(timeout=self.death_timeout)
                 logger.exception(
                     "Timed out connecting Nanny '%s' to scheduler '%s'",
                     self,
@@ -313,26 +309,24 @@ def instantiate(self, comm=None):
                 raise
 
         else:
-            result = yield self.process.start()
-        raise gen.Return(result)
+            result = await self.process.start()
+        return result
 
-    @gen.coroutine
-    def restart(self, comm=None, timeout=2, executor_wait=True):
+    async def restart(self, comm=None, timeout=2, executor_wait=True):
         start = time()
 
-        @gen.coroutine
-        def _():
+        async def _():
             if self.process is not None:
-                yield self.kill()
-                yield self.instantiate()
+                await self.kill()
+                await self.instantiate()
 
         try:
-            yield gen.with_timeout(timedelta(seconds=timeout), _())
+            await gen.with_timeout(timedelta(seconds=timeout), _())
         except gen.TimeoutError:
             logger.error("Restart timed out, returning before finished")
-            raise gen.Return("timed out")
+            return "timed out"
         else:
-            raise gen.Return("OK")
+            return "OK"
 
     def memory_monitor(self):
         """ Track worker's memory.  Restart if it goes above terminate fraction """
@@ -360,21 +354,23 @@ def is_alive(self):
     def run(self, *args, **kwargs):
         return run(self, *args, **kwargs)
 
-    @gen.coroutine
-    def _on_exit(self, exitcode):
+    def _on_exit_sync(self, exitcode):
+        self.loop.add_callback(self._on_exit, exitcode)
+
+    async def _on_exit(self, exitcode):
         if self.status not in ("closing", "closed"):
             try:
-                yield self.scheduler.unregister(address=self.worker_address)
+                await self.scheduler.unregister(address=self.worker_address)
             except (EnvironmentError, CommClosedError):
                 if not self.reconnect:
-                    yield self.close()
+                    await self.close()
                     return
 
             try:
                 if self.status not in ("closing", "closed", "closing-gracefully"):
                     if self.auto_restart:
                         logger.warning("Restarting worker")
-                        yield self.instantiate()
+                        await self.instantiate()
             except Exception:
                 logger.error(
                     "Failed to restart worker after its process exited", exc_info=True
@@ -396,13 +392,12 @@ def close_gracefully(self, comm=None):
         """
         self.status = "closing-gracefully"
 
-    @gen.coroutine
-    def close(self, comm=None, timeout=5, report=None):
+    async def close(self, comm=None, timeout=5, report=None):
         """
         Close the worker process, stop all comms.
         """
         if self.status == "closing":
-            yield self.finished()
+            await self.finished()
             assert self.status == "closed"
 
         if self.status == "closed":
@@ -413,15 +408,15 @@ def close(self, comm=None, timeout=5, report=None):
         self.stop()
         try:
             if self.process is not None:
-                yield self.kill(timeout=timeout)
+                await self.kill(timeout=timeout)
         except Exception:
             pass
         self.process = None
         self.rpc.close()
         self.status = "closed"
         if comm:
-            yield comm.write("OK")
-        yield ServerNode.close(self)
+            await comm.write("OK")
+        await ServerNode.close(self)
 
 
 class WorkerProcess(object):
@@ -441,17 +436,16 @@ def __init__(
         self.worker_dir = None
         self.worker_address = None
 
-    @gen.coroutine
-    def start(self):
+    async def start(self):
         """
         Ensure the worker process is started.
         """
         enable_proctitle_on_children()
         if self.status == "running":
-            raise gen.Return(self.status)
+            return self.status
         if self.status == "starting":
-            yield self.running.wait()
-            raise gen.Return(self.status)
+            await self.running.wait()
+            return self.status
 
         self.init_result_q = init_q = mp_context.Queue()
         self.child_stop_q = mp_context.Queue()
@@ -476,10 +470,16 @@ def start(self):
         self.running = Event()
         self.stopped = Event()
         self.status = "starting"
-        yield self.process.start()
-        msg = yield self._wait_until_connected(uid)
+        try:
+            await self.process.start()
+        except OSError:
+            logger.exception("Nanny failed to start process", exc_info=True)
+            self.process.terminate()
+            return
+
+        msg = await self._wait_until_connected(uid)
         if not msg:
-            raise gen.Return(self.status)
+            return self.status
         self.worker_address = msg["address"]
         self.worker_dir = msg["dir"]
         assert self.worker_address
@@ -488,7 +488,7 @@ def start(self):
 
         init_q.close()
 
-        raise gen.Return(self.status)
+        return self.status
 
     def _on_exit(self, proc):
         if proc is not self.process:
@@ -518,7 +518,7 @@ def mark_stopped(self):
             assert r is not None
             if r != 0:
                 msg = self._death_message(self.process.pid, r)
-                logger.warning(msg)
+                logger.info(msg)
             self.status = "stopped"
             self.stopped.set()
             # Release resources
@@ -534,8 +534,7 @@ def mark_stopped(self):
             if self.on_exit is not None:
                 self.on_exit(r)
 
-    @gen.coroutine
-    def kill(self, timeout=2, executor_wait=True):
+    async def kill(self, timeout=2, executor_wait=True):
         """
         Ensure the worker process is stopped, waiting at most
         *timeout* seconds before terminating it abruptly.
@@ -546,7 +545,7 @@ def kill(self, timeout=2, executor_wait=True):
         if self.status == "stopped":
             return
         if self.status == "stopping":
-            yield self.stopped.wait()
+            await self.stopped.wait()
             return
         assert self.status in ("starting", "running")
         self.status = "stopping"
@@ -562,19 +561,18 @@ def kill(self, timeout=2, executor_wait=True):
         self.child_stop_q.close()
 
         while process.is_alive() and loop.time() < deadline:
-            yield gen.sleep(0.05)
+            await gen.sleep(0.05)
 
         if process.is_alive():
             logger.warning(
                 "Worker process still alive after %d seconds, killing", timeout
             )
             try:
-                yield process.terminate()
+                await process.terminate()
             except Exception as e:
                 logger.error("Failed to kill worker process: %s", e)
 
-    @gen.coroutine
-    def _wait_until_connected(self, uid):
+    async def _wait_until_connected(self, uid):
         delay = 0.05
         while True:
             if self.status != "starting":
@@ -582,7 +580,7 @@ def _wait_until_connected(self, uid):
             try:
                 msg = self.init_result_q.get_nowait()
             except Empty:
-                yield gen.sleep(delay)
+                await gen.sleep(delay)
                 continue
 
             if msg["uid"] != uid:  # ensure that we didn't cross queues
@@ -592,10 +590,10 @@ def _wait_until_connected(self, uid):
                 logger.error(
                     "Failed while trying to start worker process: %s", msg["exception"]
                 )
-                yield self.process.join()
+                await self.process.join()
                 raise msg
             else:
-                raise gen.Return(msg)
+                return msg
 
     @classmethod
     def _run(
@@ -625,10 +623,9 @@ def _run(
         loop.make_current()
         worker = Worker(**worker_kwargs)
 
-        @gen.coroutine
-        def do_stop(timeout=5, executor_wait=True):
+        async def do_stop(timeout=5, executor_wait=True):
             try:
-                yield worker.close(
+                await worker.close(
                     report=False,
                     nanny=False,
                     executor_wait=executor_wait,
@@ -657,13 +654,12 @@ def watch_stop_q():
         t.daemon = True
         t.start()
 
-        @gen.coroutine
-        def run():
+        async def run():
             """
             Try to start worker and inform parent of outcome.
             """
             try:
-                yield worker
+                await worker
             except Exception as e:
                 logger.exception("Failed to start worker")
                 init_result_q.put({"uid": uid, "exception": e})
@@ -674,7 +670,7 @@ def run():
                     {"address": worker.address, "dir": worker.local_dir, "uid": uid}
                 )
                 init_result_q.close()
-                yield worker.wait_until_closed()
+                await worker.wait_until_closed()
                 logger.info("Worker closed")
 
         try:
diff --git a/distributed/process.py b/distributed/process.py
index 556edae290e..e716d754db1 100644
--- a/distributed/process.py
+++ b/distributed/process.py
@@ -332,7 +332,7 @@ def _cleanup_dangling():
     for proc in list(_dangling):
         if proc.is_alive():
             try:
-                logger.warning("reaping stray process %s" % (proc,))
+                logger.info("reaping stray process %s" % (proc,))
                 proc.terminate()
             except OSError:
                 pass
diff --git a/distributed/pubsub.py b/distributed/pubsub.py
index f9cf1f6f7c3..f40c0b15b31 100644
--- a/distributed/pubsub.py
+++ b/distributed/pubsub.py
@@ -308,10 +308,9 @@ def __init__(self, name, worker=None, client=None):
             self.loop.add_callback(pubsub.publishers[name].add, self)
             finalize(self, pubsub.trigger_cleanup)
 
-    @gen.coroutine
-    def _start(self):
+    async def _start(self):
         if self.worker:
-            result = yield self.scheduler.pubsub_add_publisher(
+            result = await self.scheduler.pubsub_add_publisher(
                 name=self.name, worker=self.worker.address
             )
             pubsub = self.worker.extensions["pubsub"]
@@ -388,8 +387,7 @@ def __init__(self, name, worker=None, client=None):
 
         finalize(self, pubsub.trigger_cleanup)
 
-    @gen.coroutine
-    def _get(self, timeout=None):
+    async def _get(self, timeout=None):
         if timeout is not None:
             timeout = datetime.timedelta(seconds=timeout)
         start = datetime.datetime.now()
@@ -400,9 +398,9 @@ def _get(self, timeout=None):
                     raise gen.TimeoutError()
             else:
                 timeout2 = None
-            yield self.condition.wait(timeout=timeout2)
+            await self.condition.wait(timeout=timeout2)
 
-        raise gen.Return(self.buffer.popleft())
+        return self.buffer.popleft()
 
     __anext__ = _get
 
diff --git a/distributed/queues.py b/distributed/queues.py
index 72f0f9fe52c..12bd15b6318 100644
--- a/distributed/queues.py
+++ b/distributed/queues.py
@@ -5,8 +5,8 @@
 import logging
 import uuid
 
-from tornado import gen
 import tornado.queues
+from tornado.locks import Event
 
 from .client import Future, _get_global_client, Client
 from .utils import tokey, sync, thread_state
@@ -49,6 +49,7 @@ def __init__(self, scheduler):
         self.scheduler.extensions["queues"] = self
 
     def create(self, stream=None, name=None, client=None, maxsize=0):
+        print("name", name)
         if name not in self.queues:
             self.queues[name] = tornado.queues.Queue(maxsize=maxsize)
             self.client_refcount[name] = 1
@@ -64,13 +65,11 @@ def release(self, stream=None, name=None, client=None):
             del self.client_refcount[name]
             futures = self.queues[name]._queue
             del self.queues[name]
-            self.scheduler.client_releases_keys(
-                keys=[d["value"] for d in futures if d["type"] == "Future"],
-                client="queue-%s" % name,
-            )
+            keys = [d["value"] for d in futures if d["type"] == "Future"]
+            if keys:
+                self.scheduler.client_releases_keys(keys=keys, client="queue-%s" % name)
 
-    @gen.coroutine
-    def put(
+    async def put(
         self, stream=None, name=None, key=None, data=None, client=None, timeout=None
     ):
         if key is not None:
@@ -81,7 +80,7 @@ def put(
             record = {"type": "msgpack", "value": data}
         if timeout is not None:
             timeout = datetime.timedelta(seconds=(timeout))
-        yield self.queues[name].put(record, timeout=timeout)
+        await self.queues[name].put(record, timeout=timeout)
 
     def future_release(self, name=None, key=None, client=None):
         self.future_refcount[name, key] -= 1
@@ -89,8 +88,7 @@ def future_release(self, name=None, key=None, client=None):
             self.scheduler.client_releases_keys(keys=[key], client="queue-%s" % name)
             del self.future_refcount[name, key]
 
-    @gen.coroutine
-    def get(self, stream=None, name=None, client=None, timeout=None, batch=False):
+    async def get(self, stream=None, name=None, client=None, timeout=None, batch=False):
         def process(record):
             """ Add task status if known """
             if record["type"] == "Future":
@@ -111,7 +109,7 @@ def process(record):
             out = []
             if batch is True:
                 while not q.empty():
-                    record = yield q.get()
+                    record = await q.get()
                     out.append(record)
             else:
                 if timeout is not None:
@@ -121,16 +119,16 @@ def process(record):
                     )
                     raise NotImplementedError(msg)
                 for i in range(batch):
-                    record = yield q.get()
+                    record = await q.get()
                     out.append(record)
             out = [process(o) for o in out]
-            raise gen.Return(out)
+            return out
         else:
             if timeout is not None:
                 timeout = datetime.timedelta(seconds=timeout)
-            record = yield self.queues[name].get(timeout=timeout)
+            record = await self.queues[name].get(timeout=timeout)
             record = process(record)
-            raise gen.Return(record)
+            return record
 
     def qsize(self, stream=None, name=None, client=None):
         return self.queues[name].qsize()
@@ -168,12 +166,18 @@ class Queue(object):
     def __init__(self, name=None, client=None, maxsize=0):
         self.client = client or _get_global_client()
         self.name = name or "queue-" + uuid.uuid4().hex
+        self._event_started = Event()
         if self.client.asynchronous or getattr(
             thread_state, "on_event_loop_thread", False
         ):
-            self._started = self.client.scheduler.queue_create(
-                name=self.name, maxsize=maxsize
-            )
+
+            async def _create_queue():
+                await self.client.scheduler.queue_create(
+                    name=self.name, maxsize=maxsize
+                )
+                self._event_started.set()
+
+            self.client.loop.add_callback(_create_queue)
         else:
             sync(
                 self.client.loop,
@@ -181,24 +185,22 @@ def __init__(self, name=None, client=None, maxsize=0):
                 name=self.name,
                 maxsize=maxsize,
             )
-            self._started = gen.moment
+            self._event_started.set()
 
     def __await__(self):
-        @gen.coroutine
-        def _():
-            yield self._started
-            raise gen.Return(self)
+        async def _():
+            await self._event_started.wait()
+            return self
 
         return _().__await__()
 
-    @gen.coroutine
-    def _put(self, value, timeout=None):
+    async def _put(self, value, timeout=None):
         if isinstance(value, Future):
-            yield self.client.scheduler.queue_put(
+            await self.client.scheduler.queue_put(
                 key=tokey(value.key), timeout=timeout, name=self.name
             )
         else:
-            yield self.client.scheduler.queue_put(
+            await self.client.scheduler.queue_put(
                 data=value, timeout=timeout, name=self.name
             )
 
@@ -224,9 +226,8 @@ def qsize(self, **kwargs):
         """ Current number of elements in the queue """
         return self.client.sync(self._qsize, **kwargs)
 
-    @gen.coroutine
-    def _get(self, timeout=None, batch=False):
-        resp = yield self.client.scheduler.queue_get(
+    async def _get(self, timeout=None, batch=False):
+        resp = await self.client.scheduler.queue_get(
             timeout=timeout, name=self.name, batch=batch
         )
 
@@ -248,12 +249,11 @@ def process(d):
         else:
             result = list(map(process, resp))
 
-        raise gen.Return(result)
+        return result
 
-    @gen.coroutine
-    def _qsize(self):
-        result = yield self.client.scheduler.queue_qsize(name=self.name)
-        raise gen.Return(result)
+    async def _qsize(self):
+        result = await self.client.scheduler.queue_qsize(name=self.name)
+        return result
 
     def close(self):
         if self.client.status == "running":  # TODO: can leave zombie futures
diff --git a/distributed/recreate_exceptions.py b/distributed/recreate_exceptions.py
index 78b0f4de9ba..d02dc4d94f4 100644
--- a/distributed/recreate_exceptions.py
+++ b/distributed/recreate_exceptions.py
@@ -1,7 +1,6 @@
 from __future__ import print_function, division, absolute_import
 
 import logging
-from tornado import gen
 from .client import futures_of, wait
 from .utils import sync, tokey
 from .utils_comm import pack_data
@@ -77,20 +76,19 @@ def __init__(self, client):
     def scheduler(self):
         return self.client.scheduler
 
-    @gen.coroutine
-    def _get_futures_error(self, future):
+    async def _get_futures_error(self, future):
         # only get errors for futures that errored.
         futures = [f for f in futures_of(future) if f.status == "error"]
         if not futures:
             raise ValueError("No errored futures passed")
-        out = yield self.scheduler.cause_of_failure(keys=[f.key for f in futures])
+        out = await self.scheduler.cause_of_failure(keys=[f.key for f in futures])
         deps, task = out["deps"], out["task"]
         if isinstance(task, dict):
             function, args, kwargs = _deserialize(**task)
-            raise gen.Return((function, args, kwargs, deps))
+            return (function, args, kwargs, deps)
         else:
             function, args, kwargs = _deserialize(task=task)
-            raise gen.Return((function, args, kwargs, deps))
+            return (function, args, kwargs, deps)
 
     def get_futures_error(self, future):
         """
@@ -122,16 +120,15 @@ def get_futures_error(self, future):
         """
         return self.client.sync(self._get_futures_error, future)
 
-    @gen.coroutine
-    def _recreate_error_locally(self, future):
-        yield wait(future)
-        out = yield self._get_futures_error(future)
+    async def _recreate_error_locally(self, future):
+        await wait(future)
+        out = await self._get_futures_error(future)
         function, args, kwargs, deps = out
         futures = self.client._graph_to_futures({}, deps)
-        data = yield self.client._gather(futures)
+        data = await self.client._gather(futures)
         args = pack_data(args, data)
         kwargs = pack_data(kwargs, data)
-        raise gen.Return((function, args, kwargs))
+        return (function, args, kwargs)
 
     def recreate_error_locally(self, future):
         """
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 4441b815642..ace4d2483d5 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1,5 +1,6 @@
 from __future__ import print_function, division, absolute_import
 
+import asyncio
 from collections import defaultdict, deque, OrderedDict
 from datetime import timedelta
 from functools import partial
@@ -43,6 +44,7 @@
 from . import profile
 from .metrics import time
 from .node import ServerNode
+from .preloading import preload_modules
 from .proctitle import setproctitle
 from .security import Security
 from .utils import (
@@ -842,6 +844,8 @@ def __init__(
         port=0,
         protocol=None,
         dashboard_address=None,
+        preload=None,
+        preload_argv=(),
         **kwargs
     ):
         self._setup_logging(logger)
@@ -874,6 +878,13 @@ def __init__(
         self.time_started = time()
         self.bandwidth = parse_bytes(dask.config.get("distributed.scheduler.bandwidth"))
 
+        if not preload:
+            preload = dask.config.get("distributed.scheduler.preload")
+        if not preload_argv:
+            preload_argv = dask.config.get("distributed.scheduler.preload-argv")
+        self.preload = preload
+        self.preload_argv = preload_argv
+
         self.security = security or Security()
         assert isinstance(self.security, Security)
         self.connection_args = self.security.get_connection_args("scheduler")
@@ -1175,8 +1186,7 @@ def get_worker_service_addr(self, worker, service_name, protocol=False):
         else:
             return ws.host, port
 
-    @gen.coroutine
-    def start(self):
+    async def start(self):
         """ Clear out old state and restart all running coroutines """
         enable_gc_diagnosis()
 
@@ -1219,14 +1229,14 @@ def del_scheduler_file():
 
             finalize(self, del_scheduler_file)
 
+        preload_modules(self.preload, parameter=self, argv=self.preload_argv)
+
         self.start_periodic_callbacks()
 
         setproctitle("dask-scheduler [%s]" % (self.address,))
-
         return self
 
-    @gen.coroutine
-    def close(self, comm=None, fast=False, close_workers=False):
+    async def close(self, comm=None, fast=False, close_workers=False):
         """ Send cleanup signal to all coroutines then wait until finished
 
         See Also
@@ -1234,7 +1244,7 @@ def close(self, comm=None, fast=False, close_workers=False):
         Scheduler.cleanup
         """
         if self.status.startswith("clos"):
-            yield self.finished()
+            await self.finished()
             return
         self.status = "closing"
 
@@ -1242,12 +1252,12 @@ def close(self, comm=None, fast=False, close_workers=False):
         setproctitle("dask-scheduler [closing]")
 
         if close_workers:
-            self.broadcast(msg={"op": "close_gracefully"}, nanny=True)
+            await self.broadcast(msg={"op": "close_gracefully"}, nanny=True)
             for worker in self.workers:
                 self.worker_send(worker, {"op": "close"})
             for i in range(20):  # wait a second for send signals to clear
                 if self.workers:
-                    yield gen.sleep(0.05)
+                    await gen.sleep(0.05)
                 else:
                     break
 
@@ -1269,8 +1279,8 @@ def close(self, comm=None, fast=False, close_workers=False):
             with ignoring(AttributeError):
                 futures.append(comm.close())
 
-        for future in futures:
-            yield future
+        for future in futures:  # TODO: do all at once
+            await future
 
         for comm in self.client_comms.values():
             comm.abort()
@@ -1279,13 +1289,12 @@ def close(self, comm=None, fast=False, close_workers=False):
 
         self.status = "closed"
         self.stop()
-        yield super(Scheduler, self).close()
+        await super(Scheduler, self).close()
 
         setproctitle("dask-scheduler [closed]")
         disable_gc_diagnosis()
 
-    @gen.coroutine
-    def close_worker(self, stream=None, worker=None, safe=None):
+    async def close_worker(self, stream=None, worker=None, safe=None):
         """ Remove a worker from the cluster
 
         This both removes the worker from our local state and also sends a
@@ -1305,7 +1314,6 @@ def close_worker(self, stream=None, worker=None, safe=None):
     # Stimuli #
     ###########
 
-    @gen.coroutine
     def heartbeat_worker(
         self,
         comm=None,
@@ -1361,8 +1369,7 @@ def heartbeat_worker(
             "heartbeat-interval": heartbeat_interval(len(self.workers)),
         }
 
-    @gen.coroutine
-    def add_worker(
+    async def add_worker(
         self,
         comm=None,
         address=None,
@@ -1409,7 +1416,8 @@ def add_worker(
                     "message": "name taken, %s" % name,
                     "time": time(),
                 }
-                yield comm.write(msg)
+                if comm:
+                    await comm.write(msg)
                 return
 
             if "addresses" not in self.host_info[host]:
@@ -1473,15 +1481,16 @@ def add_worker(
             self.log_event("all", {"action": "add-worker", "worker": address})
             logger.info("Register %s", str(address))
 
-            yield comm.write(
-                {
-                    "status": "OK",
-                    "time": time(),
-                    "heartbeat-interval": heartbeat_interval(len(self.workers)),
-                    "worker-plugins": self.worker_plugins,
-                }
-            )
-            yield self.handle_worker(comm=comm, worker=address)
+            if comm:
+                await comm.write(
+                    {
+                        "status": "OK",
+                        "time": time(),
+                        "heartbeat-interval": heartbeat_interval(len(self.workers)),
+                        "worker-plugins": self.worker_plugins,
+                    }
+                )
+            await self.handle_worker(comm=comm, worker=address)
 
     def update_graph(
         self,
@@ -1930,7 +1939,6 @@ def remove_worker(self, comm=None, address=None, safe=False, close=True):
             if not self.workers:
                 logger.info("Lost all workers")
 
-            @gen.coroutine
             def remove_worker_from_events():
                 # If the worker isn't registered anymore after the delay, remove from events
                 if address not in self.workers and address in self.events:
@@ -2190,8 +2198,7 @@ def report(self, msg, ts=None, client=None):
                 if self.status == "running":
                     logger.critical("Tried writing to closed comm: %s", msg)
 
-    @gen.coroutine
-    def add_client(self, comm, client=None):
+    async def add_client(self, comm, client=None):
         """ Add client to network
 
         We listen to all future messages from this Comm.
@@ -2208,7 +2215,7 @@ def add_client(self, comm, client=None):
             bcomm.send({"op": "stream-start"})
 
             try:
-                yield self.handle_stream(comm=comm, extra={"client": client})
+                await self.handle_stream(comm=comm, extra={"client": client})
             finally:
                 self.remove_client(client=client)
                 logger.debug("Finished handling client %s", client)
@@ -2217,7 +2224,7 @@ def add_client(self, comm, client=None):
                 self.client_comms[client].send({"op": "stream-closed"})
             try:
                 if not shutting_down():
-                    yield self.client_comms[client].close()
+                    await self.client_comms[client].close()
                     del self.client_comms[client]
                     if self.status == "running":
                         logger.info("Close client connection: %s", client)
@@ -2240,7 +2247,6 @@ def remove_client(self, client=None):
             )
             del self.clients[client]
 
-        @gen.coroutine
         def remove_client_from_events():
             # If the client isn't registered anymore after the delay, remove from events
             if client not in self.clients and client in self.events:
@@ -2384,8 +2390,7 @@ def handle_long_running(self, key=None, worker=None, compute_duration=None):
         ws.processing[ts] = 0
         self.check_idle_saturated(ws)
 
-    @gen.coroutine
-    def handle_worker(self, comm=None, worker=None):
+    async def handle_worker(self, comm=None, worker=None):
         """
         Listen to responses from a single worker
 
@@ -2400,7 +2405,7 @@ def handle_worker(self, comm=None, worker=None):
         worker_comm.start(comm)
         logger.info("Starting worker compute stream, %s", worker)
         try:
-            yield self.handle_stream(comm=comm, extra={"worker": worker})
+            await self.handle_stream(comm=comm, extra={"worker": worker})
         finally:
             if worker in self.stream_comms:
                 worker_comm.abort()
@@ -2439,8 +2444,7 @@ def worker_send(self, worker, msg):
     # Less common interactions #
     ############################
 
-    @gen.coroutine
-    def scatter(
+    async def scatter(
         self,
         comm=None,
         data=None,
@@ -2457,7 +2461,7 @@ def scatter(
         """
         start = time()
         while not self.workers:
-            yield gen.sleep(0.2)
+            await gen.sleep(0.2)
             if time() > start + timeout:
                 raise gen.TimeoutError("No workers found")
 
@@ -2469,7 +2473,7 @@ def scatter(
 
         assert isinstance(data, dict)
 
-        keys, who_has, nbytes = yield scatter_to_workers(
+        keys, who_has, nbytes = await scatter_to_workers(
             nthreads, data, rpc=self.rpc, report=False
         )
 
@@ -2480,15 +2484,14 @@ def scatter(
                 n = len(nthreads)
             else:
                 n = broadcast
-            yield self.replicate(keys=keys, workers=workers, n=n)
+            await self.replicate(keys=keys, workers=workers, n=n)
 
         self.log_event(
             [client, "all"], {"action": "scatter", "client": client, "count": len(data)}
         )
-        raise gen.Return(keys)
+        return keys
 
-    @gen.coroutine
-    def gather(self, comm=None, keys=None, serializers=None):
+    async def gather(self, comm=None, keys=None, serializers=None):
         """ Collect data in from workers """
         keys = list(keys)
         who_has = {}
@@ -2499,7 +2502,7 @@ def gather(self, comm=None, keys=None, serializers=None):
             else:
                 who_has[key] = []
 
-        data, missing_keys, missing_workers = yield gather_from_workers(
+        data, missing_keys, missing_workers = await gather_from_workers(
             who_has, rpc=self.rpc, close=False, serializers=serializers
         )
         if not missing_keys:
@@ -2537,7 +2540,7 @@ def gather(self, comm=None, keys=None, serializers=None):
                             self.transitions({key: "released"})
 
         self.log_event("all", {"action": "gather", "count": len(keys)})
-        raise gen.Return(result)
+        return result
 
     def clear_task_state(self):
         # XXX what about nested state such as ClientState.wants_what
@@ -2546,8 +2549,7 @@ def clear_task_state(self):
         for collection in self._task_state_collections:
             collection.clear()
 
-    @gen.coroutine
-    def restart(self, client=None, timeout=3):
+    async def restart(self, client=None, timeout=3):
         """ Restart all workers.  Reset local state. """
         with log_errors():
 
@@ -2596,7 +2598,7 @@ def restart(self, client=None, timeout=3):
                         for nanny in nannies
                     ]
                 )
-                resps = yield gen.with_timeout(timedelta(seconds=timeout), resps)
+                resps = await gen.with_timeout(timedelta(seconds=timeout), resps)
                 if not all(resp == "OK" for resp in resps):
                     logger.error(
                         "Not all workers responded positively: %s", resps, exc_info=True
@@ -2610,17 +2612,16 @@ def restart(self, client=None, timeout=3):
                 for nanny in nannies:
                     nanny.close_rpc()
 
-            self.start()
+            await self.start()
 
             self.log_event([client, "all"], {"action": "restart", "client": client})
             start = time()
             while time() < start + 10 and len(self.workers) < n_workers:
-                yield gen.sleep(0.01)
+                await gen.sleep(0.01)
 
             self.report({"op": "restart"})
 
-    @gen.coroutine
-    def broadcast(
+    async def broadcast(
         self,
         comm=None,
         msg=None,
@@ -2646,28 +2647,26 @@ def broadcast(
         else:
             addresses = workers
 
-        @gen.coroutine
-        def send_message(addr):
-            comm = yield connect(
+        async def send_message(addr):
+            comm = await connect(
                 addr, deserialize=self.deserialize, connection_args=self.connection_args
             )
             comm.name = "Scheduler Broadcast"
-            resp = yield send_recv(comm, close=True, serializers=serializers, **msg)
-            raise gen.Return(resp)
+            resp = await send_recv(comm, close=True, serializers=serializers, **msg)
+            return resp
 
-        results = yield All(
+        results = await All(
             [send_message(address) for address in addresses if address is not None]
         )
 
-        raise Return(dict(zip(workers, results)))
+        return dict(zip(workers, results))
 
-    @gen.coroutine
-    def proxy(self, comm=None, msg=None, worker=None, serializers=None):
+    async def proxy(self, comm=None, msg=None, worker=None, serializers=None):
         """ Proxy a communication through the scheduler to some other worker """
-        d = yield self.broadcast(
+        d = await self.broadcast(
             comm=comm, msg=msg, workers=[worker], serializers=serializers
         )
-        raise gen.Return(d[worker])
+        return d[worker]
 
     @gen.coroutine
     def rebalance(self, comm=None, keys=None, workers=None):
@@ -2686,7 +2685,7 @@ def rebalance(self, comm=None, keys=None, workers=None):
                 tasks = {self.tasks[k] for k in keys}
                 missing_data = [ts.key for ts in tasks if not ts.who_has]
                 if missing_data:
-                    raise Return({"status": "missing-data", "keys": missing_data})
+                    return {"status": "missing-data", "keys": missing_data}
             else:
                 tasks = set(self.tasks.values())
 
@@ -3016,8 +3015,7 @@ def key(group):
 
             return result
 
-    @gen.coroutine
-    def retire_workers(
+    async def retire_workers(
         self, comm=None, workers=None, remove=True, close_workers=False, **kwargs
     ):
         """ Gracefully retire workers from cluster
@@ -3053,7 +3051,7 @@ def retire_workers(
                     try:
                         workers = self.workers_to_close(**kwargs)
                         if workers:
-                            workers = yield self.retire_workers(
+                            workers = await self.retire_workers(
                                 workers=workers,
                                 remove=remove,
                                 close_workers=close_workers,
@@ -3073,18 +3071,20 @@ def retire_workers(
             other_workers = set(self.workers.values()) - workers
             if keys:
                 if other_workers:
-                    yield self.replicate(
+                    await self.replicate(
                         keys=keys,
                         workers=[ws.address for ws in other_workers],
                         n=1,
                         delete=False,
                     )
                 else:
-                    raise gen.Return([])
+                    return []
 
             worker_keys = {ws.address: ws.identity() for ws in workers}
             if close_workers and worker_keys:
-                yield [self.close_worker(worker=w, safe=True) for w in worker_keys]
+                await asyncio.gather(
+                    *[self.close_worker(worker=w, safe=True) for w in worker_keys]
+                )
             if remove:
                 for w in worker_keys:
                     self.remove_worker(address=w, safe=True)
@@ -3099,7 +3099,7 @@ def retire_workers(
             )
             self.log_event(list(worker_keys), {"action": "retired"})
 
-            raise gen.Return(worker_keys)
+            return worker_keys
 
     def add_keys(self, comm=None, worker=None, keys=()):
         """
@@ -3188,8 +3188,7 @@ def report_on_key(self, key=None, ts=None, client=None):
                 client=client,
             )
 
-    @gen.coroutine
-    def feed(
+    async def feed(
         self, comm, function=None, setup=None, teardown=None, interval="1s", **kwargs
     ):
         """
@@ -3217,16 +3216,16 @@ def feed(
             if teardown:
                 teardown = pickle.loads(teardown)
             state = setup(self) if setup else None
-            if isinstance(state, gen.Future):
-                state = yield state
+            if hasattr(state, "__await__"):
+                state = await state
             try:
                 while self.status == "running":
                     if state is None:
                         response = function(self)
                     else:
                         response = function(self, state)
-                    yield comm.write(response)
-                    yield gen.sleep(interval)
+                    await comm.write(response)
+                    await gen.sleep(interval)
             except (EnvironmentError, CommClosedError):
                 pass
             finally:
@@ -3391,15 +3390,14 @@ def get_task_stream(self, comm=None, start=None, stop=None, count=None):
         ts = [p for p in self.plugins if isinstance(p, TaskStreamPlugin)][0]
         return ts.collect(start=start, stop=stop, count=count)
 
-    @gen.coroutine
-    def register_worker_plugin(self, comm, plugin, name=None):
+    async def register_worker_plugin(self, comm, plugin, name=None):
         """ Registers a setup function, and call it on every worker """
         self.worker_plugins.append(plugin)
 
-        responses = yield self.broadcast(
+        responses = await self.broadcast(
             msg=dict(op="plugin-add", plugin=plugin, name=name)
         )
-        raise gen.Return(responses)
+        return responses
 
     #####################
     # State Transitions #
@@ -4603,12 +4601,11 @@ def get_profile_metadata(
 
         raise gen.Return({"counts": counts, "keys": keys})
 
-    @gen.coroutine
-    def get_worker_logs(self, comm=None, n=None, workers=None, nanny=False):
-        results = yield self.broadcast(
+    async def get_worker_logs(self, comm=None, n=None, workers=None, nanny=False):
+        results = await self.broadcast(
             msg={"op": "get_logs", "n": n}, workers=workers, nanny=nanny
         )
-        raise gen.Return(results)
+        return results
 
     ###########
     # Cleanup #
diff --git a/distributed/tests/py3_test_pubsub.py b/distributed/tests/py3_test_pubsub.py
index 0cedbb3bd31..294ecfb90c8 100644
--- a/distributed/tests/py3_test_pubsub.py
+++ b/distributed/tests/py3_test_pubsub.py
@@ -1,6 +1,7 @@
 from distributed import Pub, Sub
 from distributed.utils_test import gen_cluster
 
+import asyncio
 import toolz
 from tornado import gen
 import pytest
@@ -22,7 +23,7 @@ def f(_):
         sub = Sub("a")
         return list(toolz.take(5, sub))
 
-    c.run(publish, workers=[a.address])
+    asyncio.ensure_future(c.run(publish, workers=[a.address]))
 
     tasks = [c.submit(f, i) for i in range(4)]
     results = yield c.gather(tasks)
diff --git a/distributed/tests/test_as_completed.py b/distributed/tests/test_as_completed.py
index a584025ad03..aa53b9b993a 100644
--- a/distributed/tests/test_as_completed.py
+++ b/distributed/tests/test_as_completed.py
@@ -238,7 +238,7 @@ def test_as_completed_with_results_no_raise_async(c, s, a, b):
     z = c.submit(inc, 1)
 
     ac = as_completed([x, y, z], with_results=True, raise_errors=False)
-    y.cancel()
+    c.loop.add_callback(y.cancel)
     first = yield ac.__anext__()
     second = yield ac.__anext__()
     third = yield ac.__anext__()
diff --git a/distributed/tests/test_batched.py b/distributed/tests/test_batched.py
index 23d8e677774..af281aff8c3 100644
--- a/distributed/tests/test_batched.py
+++ b/distributed/tests/test_batched.py
@@ -134,7 +134,7 @@ def test_close_closed():
         b.start(comm)
 
         b.send(123)
-        comm.close()  # external closing
+        yield comm.close()  # external closing
 
         yield b.close()
         assert "closed" in repr(b)
@@ -185,7 +185,7 @@ def recv():
         yield All([send(), recv()])
 
         assert L == list(range(0, 10000, 1))
-        comm.close()
+        yield comm.close()
 
 
 @gen.coroutine
@@ -222,7 +222,7 @@ def run_traffic_jam(nsends, nbytes):
 
         assert results == list(range(nsends))
 
-        comm.close()  # external closing
+        yield comm.close()  # external closing
         yield b.close()
 
 
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 1f9678583b0..50415971e20 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -1,11 +1,11 @@
 from __future__ import print_function, division, absolute_import
 
-from operator import add
-
+import asyncio
 from collections import deque
 from concurrent.futures import CancelledError
 import gc
 import logging
+from operator import add
 import os
 import pickle
 import random
@@ -3311,11 +3311,11 @@ def test_get_foo_lost_keys(c, s, u, v, w):
 @gen_cluster(
     client=True,
     Worker=Nanny,
-    check_new_threads=False,
     worker_kwargs={"death_timeout": "500ms"},
+    clean_kwargs={"processes": False, "threads": False},
 )
 def test_bad_tasks_fail(c, s, a, b):
-    f = c.submit(sys.exit, 1)
+    f = c.submit(sys.exit, 0)
     with pytest.raises(KilledWorker) as info:
         yield f
 
@@ -3486,7 +3486,7 @@ def test_scatter_raises_if_no_workers(c, s):
 @pytest.mark.slow
 def test_reconnect(loop):
     w = Worker("127.0.0.1", 9393, loop=loop)
-    w.start()
+    loop.add_callback(w.start)
 
     scheduler_cli = [
         "dask-scheduler",
@@ -4031,7 +4031,7 @@ def f(x, y=0):
     assert len(b.data) > 2 * len(a.data)
 
 
-@gen_cluster(client=True, check_new_threads=False)
+@gen_cluster(client=True, clean_kwargs={"threads": False})
 def test_add_done_callback(c, s, a, b):
     S = set()
 
@@ -4616,9 +4616,9 @@ def f(_):
 
     from concurrent.futures import ThreadPoolExecutor
 
-    e = ThreadPoolExecutor(30)
-    results = list(e.map(f, range(30)))
-    assert results and all(results)
+    with ThreadPoolExecutor(30) as e:
+        results = list(e.map(f, range(30)))
+        assert results and all(results)
 
 
 @pytest.mark.slow
@@ -5343,13 +5343,13 @@ def test_de_serialization_none(s, a, b):
 
 @gen_cluster()
 def test_client_repr_closed(s, a, b):
-    c = yield Client(s.address, asynchronous=True)
+    c = yield Client(s.address, asynchronous=True, dashboard_address=None)
     yield c.close()
     c._repr_html_()
 
 
 def test_client_repr_closed_sync(loop):
-    with Client(loop=loop, processes=False) as c:
+    with Client(loop=loop, processes=False, dashboard_address=None) as c:
         c.close()
         c._repr_html_()
 
@@ -5498,7 +5498,7 @@ def f(x):
         assert result == 101
 
 
-@gen_cluster(client=True, check_new_threads=False)
+@gen_cluster(client=True, clean_kwargs={"threads": False})
 def test_profile_bokeh(c, s, a, b):
     pytest.importorskip("bokeh.plotting")
     from bokeh.model import Model
@@ -5578,7 +5578,7 @@ def test_instances(c, s, a, b):
 
 @gen_cluster(client=True)
 def test_wait_for_workers(c, s, a, b):
-    future = c.wait_for_workers(n_workers=3)
+    future = asyncio.ensure_future(c.wait_for_workers(n_workers=3))
     yield gen.sleep(0.22)  # 2 chances
     assert not future.done()
 
diff --git a/distributed/tests/test_locks.py b/distributed/tests/test_locks.py
index 226feec4faf..9fa9a73787a 100644
--- a/distributed/tests/test_locks.py
+++ b/distributed/tests/test_locks.py
@@ -13,7 +13,7 @@
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 8)] * 2)
 def test_lock(c, s, a, b):
-    c.set_metadata("locked", False)
+    yield c.set_metadata("locked", False)
 
     def f(x):
         client = get_client()
diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
index 7722476a2c5..579af8dbc2c 100644
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -12,6 +12,7 @@
 import pytest
 from toolz import valmap, first
 from tornado import gen
+from tornado.ioloop import IOLoop
 
 import dask
 from distributed import Nanny, rpc, Scheduler, Worker
@@ -19,7 +20,13 @@
 from distributed.metrics import time
 from distributed.protocol.pickle import dumps
 from distributed.utils import ignoring, tmpfile
-from distributed.utils_test import gen_cluster, gen_test, inc, captured_logger
+from distributed.utils_test import (  # noqa: F401
+    gen_cluster,
+    gen_test,
+    inc,
+    captured_logger,
+    cleanup,
+)
 
 
 @gen_cluster(nthreads=[])
@@ -82,7 +89,7 @@ def test_nanny_process_failure(c, s):
     pid = n.pid
     assert pid is not None
     with ignoring(CommClosedError):
-        yield c._run(os._exit, 0, workers=[n.worker_address])
+        yield c.run(os._exit, 0, workers=[n.worker_address])
 
     start = time()
     while n.pid == pid:  # wait while process dies and comes back
@@ -90,6 +97,7 @@ def test_nanny_process_failure(c, s):
         assert time() - start < 5
 
     start = time()
+    yield gen.sleep(1)
     while not n.is_alive():  # wait while process comes back
         yield gen.sleep(0.01)
         assert time() - start < 5
@@ -259,7 +267,7 @@ def test_nanny_timeout(c, s, a):
     Worker=Nanny,
     worker_kwargs={"memory_limit": 1e8},
     timeout=20,
-    check_new_threads=False,
+    clean_kwargs={"threads": False},
 )
 def test_nanny_terminate(c, s, a):
     from time import sleep
@@ -319,7 +327,7 @@ def test_scheduler_address_config(c, s):
 def test_wait_for_scheduler():
     with captured_logger("distributed") as log:
         w = Nanny("127.0.0.1:44737")
-        w.start()
+        IOLoop.current().add_callback(w.start)
         yield gen.sleep(6)
         yield w.close()
 
@@ -378,3 +386,15 @@ def pool_worker(world_size):
             p.map(_noop, range(world_size))
 
     yield c.submit(pool_worker, 4)
+
+
+@pytest.mark.asyncio
+async def test_nanny_closes_cleanly(cleanup):
+    async with Scheduler() as s:
+        n = await Nanny(s.address)
+        assert n.process.pid
+        proc = n.process.process
+        await n.close()
+        assert not n.process
+        assert not proc.is_alive()
+        assert proc.exitcode == 0
diff --git a/distributed/tests/test_priorities.py b/distributed/tests/test_priorities.py
index 6258c4e16a7..ae96517f1ac 100644
--- a/distributed/tests/test_priorities.py
+++ b/distributed/tests/test_priorities.py
@@ -6,32 +6,34 @@
 from dask import delayed, persist
 
 from distributed.utils_test import gen_cluster, inc, slowinc, slowdec
-from distributed import wait
+from distributed import wait, Worker
 from distributed.utils import tokey
 
 
-@gen_cluster(client=True)
-def test_submit(c, s, a, b):
+@gen_cluster(client=True, nthreads=[])
+async def test_submit(c, s):
     low = c.submit(inc, 1, priority=-1)
     futures = c.map(slowinc, range(10), delay=0.1)
     high = c.submit(inc, 2, priority=1)
-    yield wait(high)
-    assert all(s.processing.values())
-    assert s.tasks[low.key].state == "processing"
+    async with Worker(s.address, nthreads=1):
+        await wait(high)
+        assert all(s.processing.values())
+        assert s.tasks[low.key].state == "processing"
 
 
-@gen_cluster(client=True)
-def test_map(c, s, a, b):
+@gen_cluster(client=True, nthreads=[])
+async def test_map(c, s):
     low = c.map(inc, [1, 2, 3], priority=-1)
     futures = c.map(slowinc, range(10), delay=0.1)
     high = c.map(inc, [4, 5, 6], priority=1)
-    yield wait(high)
-    assert all(s.processing.values())
-    assert s.tasks[low[0].key].state == "processing"
+    async with Worker(s.address, nthreads=1):
+        await wait(high)
+        assert all(s.processing.values())
+        assert s.tasks[low[0].key].state == "processing"
 
 
-@gen_cluster(client=True)
-def test_compute(c, s, a, b):
+@gen_cluster(client=True, nthreads=[])
+async def test_compute(c, s):
     da = pytest.importorskip("dask.array")
     x = da.random.random((10, 10), chunks=(5, 5))
     y = da.random.random((10, 10), chunks=(5, 5))
@@ -39,13 +41,14 @@ def test_compute(c, s, a, b):
     low = c.compute(x, priority=-1)
     futures = c.map(slowinc, range(10), delay=0.1)
     high = c.compute(y, priority=1)
-    yield wait(high)
-    assert all(s.processing.values())
-    assert s.tasks[tokey(low.key)].state in ("processing", "waiting")
+    async with Worker(s.address, nthreads=1):
+        await wait(high)
+        assert all(s.processing.values())
+        assert s.tasks[tokey(low.key)].state in ("processing", "waiting")
 
 
-@gen_cluster(client=True)
-def test_persist(c, s, a, b):
+@gen_cluster(client=True, nthreads=[])
+async def test_persist(c, s):
     da = pytest.importorskip("dask.array")
     x = da.random.random((10, 10), chunks=(5, 5))
     y = da.random.random((10, 10), chunks=(5, 5))
@@ -53,12 +56,13 @@ def test_persist(c, s, a, b):
     low = x.persist(priority=-1)
     futures = c.map(slowinc, range(10), delay=0.1)
     high = y.persist(priority=1)
-    yield wait(high)
-    assert all(s.processing.values())
-    assert all(
-        s.tasks[tokey(k)].state in ("processing", "waiting")
-        for k in flatten(low.__dask_keys__())
-    )
+    async with Worker(s.address, nthreads=1):
+        await wait(high)
+        assert all(s.processing.values())
+        assert all(
+            s.tasks[tokey(k)].state in ("processing", "waiting")
+            for k in flatten(low.__dask_keys__())
+        )
 
 
 @gen_cluster(client=True)
diff --git a/distributed/tests/test_queues.py b/distributed/tests/test_queues.py
index 2e7702171ad..a28d1e29082 100644
--- a/distributed/tests/test_queues.py
+++ b/distributed/tests/test_queues.py
@@ -223,7 +223,7 @@ def test_Future_knows_status_immediately(c, s, a, b):
 @gen_cluster(client=True)
 def test_erred_future(c, s, a, b):
     future = c.submit(div, 1, 0)
-    q = Queue()
+    q = yield Queue()
     yield q.put(future)
     yield gen.sleep(0.1)
     future2 = yield q.get()
@@ -236,10 +236,7 @@ def test_erred_future(c, s, a, b):
 
 @gen_cluster(client=True)
 def test_close(c, s, a, b):
-    q = Queue()
-
-    while q.name not in s.extensions["queues"].queues:
-        yield gen.sleep(0.01)
+    q = yield Queue()
 
     q.close()
     q.close()
@@ -250,7 +247,7 @@ def test_close(c, s, a, b):
 
 @gen_cluster(client=True)
 def test_timeout(c, s, a, b):
-    q = Queue("v", maxsize=1)
+    q = yield Queue("v", maxsize=1)
 
     start = time()
     with pytest.raises(gen.TimeoutError):
diff --git a/distributed/tests/test_resources.py b/distributed/tests/test_resources.py
index 1985d44e2a3..b3f5db36a76 100644
--- a/distributed/tests/test_resources.py
+++ b/distributed/tests/test_resources.py
@@ -380,8 +380,8 @@ def test_full_collections(c, s, a, b):
 def test_collections_get(client, optimize_graph, s, a, b):
     da = pytest.importorskip("dask.array")
 
-    def f(dask_worker):
-        dask_worker.set_resources(**{"A": 1})
+    async def f(dask_worker):
+        await dask_worker.set_resources(**{"A": 1})
 
     client.run(f, workers=[a["address"]])
 
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 20de5e7b7fd..4f1b2808102 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -110,7 +110,7 @@ def test_decide_worker_with_many_independent_leaves(c, s, a, b):
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 3)
 def test_decide_worker_with_restrictions(client, s, a, b, c):
     x = client.submit(inc, 1, workers=[a.address, b.address])
-    yield wait(x)
+    yield x
     assert x.key in a.data or x.key in b.data
 
 
@@ -621,14 +621,6 @@ def test_update_graph_culls(s, a, b):
     assert "z" not in s.dependencies
 
 
-@gen_cluster(nthreads=[])
-def test_add_worker_is_idempotent(s):
-    s.add_worker(address=alice, nthreads=1, resolve_address=False)
-    nthreads = dict(s.nthreads)
-    s.add_worker(address=alice, resolve_address=False)
-    assert s.nthreads == s.nthreads
-
-
 def test_io_loop(loop):
     s = Scheduler(loop=loop, validate=True)
     assert s.io_loop is loop
@@ -956,7 +948,7 @@ def test_worker_breaks_and_returns(c, s, a):
 
     yield wait(future)
 
-    a.batched_stream.comm.close()
+    yield a.batched_stream.comm.close()
 
     yield gen.sleep(0.1)
     start = time()
@@ -1146,11 +1138,13 @@ def test_scheduler_file():
 
 @pytest.mark.xfail(reason="")
 @gen_cluster(client=True, nthreads=[])
-def test_non_existent_worker(c, s):
+async def test_non_existent_worker(c, s):
     with dask.config.set({"distributed.comm.timeouts.connect": "100ms"}):
-        s.add_worker(address="127.0.0.1:5738", nthreads=2, nbytes={}, host_info={})
+        await s.add_worker(
+            address="127.0.0.1:5738", nthreads=2, nbytes={}, host_info={}
+        )
         futures = c.map(inc, range(10))
-        yield gen.sleep(0.300)
+        await gen.sleep(0.300)
         assert not s.workers
         assert all(ts.state == "no-worker" for ts in s.tasks.values())
 
@@ -1317,19 +1311,19 @@ def test_retries(c, s, a, b):
 
 @pytest.mark.xfail(reason="second worker also errant for some reason")
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 3, timeout=5)
-def test_mising_data_errant_worker(c, s, w1, w2, w3):
+async def test_mising_data_errant_worker(c, s, w1, w2, w3):
     with dask.config.set({"distributed.comm.timeouts.connect": "1s"}):
         np = pytest.importorskip("numpy")
 
         x = c.submit(np.random.random, 10000000, workers=w1.address)
-        yield wait(x)
-        yield c.replicate(x, workers=[w1.address, w2.address])
+        await wait(x)
+        await c.replicate(x, workers=[w1.address, w2.address])
 
         y = c.submit(len, x, workers=w3.address)
         while not w3.tasks:
-            yield gen.sleep(0.001)
-        w1.close()
-        yield wait(y)
+            await gen.sleep(0.001)
+        await w1.close()
+        await wait(y)
 
 
 @gen_cluster(client=True)
diff --git a/distributed/tests/test_steal.py b/distributed/tests/test_steal.py
index d233fc28388..45a110bbecf 100644
--- a/distributed/tests/test_steal.py
+++ b/distributed/tests/test_steal.py
@@ -234,7 +234,7 @@ def test_dont_steal_host_restrictions(c, s, a, b):
     yield future
 
     futures = c.map(slowinc, range(100), delay=0.1, workers="127.0.0.1")
-    while len(a.task_state) < 10:
+    while len(a.task_state) + len(b.task_state) < 100:
         yield gen.sleep(0.01)
     assert len(a.task_state) == 100
     assert len(b.task_state) == 0
@@ -254,7 +254,7 @@ def test_dont_steal_resource_restrictions(c, s, a, b):
     yield future
 
     futures = c.map(slowinc, range(100), delay=0.1, resources={"A": 1})
-    while len(a.task_state) < 10:
+    while len(a.task_state) + len(b.task_state) < 100:
         yield gen.sleep(0.01)
     assert len(a.task_state) == 100
     assert len(b.task_state) == 0
diff --git a/distributed/tests/test_stress.py b/distributed/tests/test_stress.py
index 6a5dbe72736..b5f51359239 100644
--- a/distributed/tests/test_stress.py
+++ b/distributed/tests/test_stress.py
@@ -14,7 +14,7 @@
 from distributed import Client, wait, Nanny
 from distributed.config import config
 from distributed.metrics import time
-from distributed.utils import All
+from distributed.utils import All, ignoring
 from distributed.utils_test import (
     gen_cluster,
     cluster,
@@ -126,7 +126,7 @@ def test_stress_scatter_death(c, s, *workers):
     s.allowed_failures = 1000
     np = pytest.importorskip("numpy")
     L = yield c.scatter([np.random.random(10000) for i in range(len(workers))])
-    yield c._replicate(L, n=2)
+    yield c.replicate(L, n=2)
 
     adds = [
         delayed(slowadd, pure=True)(
@@ -166,27 +166,10 @@ def test_stress_scatter_death(c, s, *workers):
         yield w.close()
         alive.remove(w)
 
-    try:
-        yield gen.with_timeout(timedelta(seconds=25), c._gather(futures))
-    except gen.TimeoutError:
-        ws = {w.address: w for w in workers if w.status != "closed"}
-        print(s.processing)
-        print(ws)
-        print(futures)
-        try:
-            worker = [w for w in ws.values() if w.waiting_for_data][0]
-        except Exception:
-            pass
-        if config.get("log-on-err"):
-            import pdb
-
-            pdb.set_trace()
-        else:
-            raise
-    except CancelledError:
-        pass
-    finally:
-        futures = None
+    with ignoring(CancelledError):
+        yield c.gather(futures)
+
+    futures = None
 
 
 def vsum(*args):
diff --git a/distributed/tests/test_tls_functional.py b/distributed/tests/test_tls_functional.py
index 7d097e28112..43c8c667bf4 100644
--- a/distributed/tests/test_tls_functional.py
+++ b/distributed/tests/test_tls_functional.py
@@ -19,8 +19,8 @@
 def test_Queue(c, s, a, b):
     assert s.address.startswith("tls://")
 
-    x = Queue("x")
-    y = Queue("y")
+    x = yield Queue("x")
+    y = yield Queue("y")
 
     size = yield x.qsize()
     assert size == 0
diff --git a/distributed/tests/test_utils_test.py b/distributed/tests/test_utils_test.py
index eac2ec71529..05b8066c707 100755
--- a/distributed/tests/test_utils_test.py
+++ b/distributed/tests/test_utils_test.py
@@ -13,6 +13,7 @@
 from distributed.core import rpc
 from distributed.metrics import time
 from distributed.utils_test import (  # noqa: F401
+    cleanup,
     cluster,
     gen_cluster,
     inc,
@@ -175,10 +176,10 @@ def test_tls_cluster(tls_client):
     assert tls_client.security
 
 
-def test_tls_scheduler(security, loop):
-    s = yield Scheduler(security=security, loop=loop, host="localhost")
-    assert s.address.startswith("tls")
-    yield s.close()
+@pytest.mark.asyncio
+async def test_tls_scheduler(security, cleanup):
+    async with Scheduler(security=security, host="localhost") as s:
+        assert s.address.startswith("tls")
 
 
 if sys.version_info >= (3, 5):
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 2e6af6e0bdc..1c1e70fcba7 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -501,7 +501,8 @@ def test_memory_limit_auto():
     assert isinstance(a.memory_limit, Number)
     assert isinstance(b.memory_limit, Number)
 
-    assert a.memory_limit < b.memory_limit
+    if multiprocessing.cpu_count() > 1:
+        assert a.memory_limit < b.memory_limit
 
     assert c.memory_limit == d.memory_limit
 
@@ -1050,7 +1051,14 @@ def test_statistical_profiling(c, s, a, b):
 
 @pytest.mark.slow
 @nodebug
-@gen_cluster(client=True, timeout=20)
+@gen_cluster(
+    client=True,
+    timeout=30,
+    config={
+        "distributed.worker.profile.interval": "1ms",
+        "distributed.worker.profile.cycle": "100ms",
+    },
+)
 def test_statistical_profiling_2(c, s, a, b):
     da = pytest.importorskip("dask.array")
     while True:
diff --git a/distributed/tests/test_worker_plugins.py b/distributed/tests/test_worker_plugins.py
index bbba39943fb..02db9419d4e 100644
--- a/distributed/tests/test_worker_plugins.py
+++ b/distributed/tests/test_worker_plugins.py
@@ -27,7 +27,7 @@ def test_create_with_client(c, s):
     assert worker._my_plugin_status == "setup"
     assert worker._my_plugin_data == 123
 
-    yield worker._close()
+    yield worker.close()
     assert worker._my_plugin_status == "teardown"
 
 
diff --git a/distributed/utils.py b/distributed/utils.py
index 2f4657439cf..6e8769979fb 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -1,5 +1,6 @@
 from __future__ import print_function, division, absolute_import
 
+import asyncio
 import atexit
 from collections import deque
 from contextlib import contextmanager
@@ -200,8 +201,7 @@ def ignore_exceptions(coroutines, *exceptions):
     raise gen.Return(results)
 
 
-@gen.coroutine
-def All(args, quiet_exceptions=()):
+async def All(args, quiet_exceptions=()):
     """ Wait on many tasks at the same time
 
     Err once any of the tasks err.
@@ -214,11 +214,11 @@ def All(args, quiet_exceptions=()):
     quiet_exceptions: tuple, Exception
         Exception types to avoid logging if they fail
     """
-    tasks = gen.WaitIterator(*args)
+    tasks = gen.WaitIterator(*map(asyncio.ensure_future, args))
     results = [None for _ in args]
     while not tasks.done():
         try:
-            result = yield tasks.next()
+            result = await tasks.next()
         except Exception:
 
             @gen.coroutine
@@ -237,13 +237,11 @@ def quiet():
 
             quiet()
             raise
-
         results[tasks.current_index] = result
-    raise gen.Return(results)
+    return results
 
 
-@gen.coroutine
-def Any(args, quiet_exceptions=()):
+async def Any(args, quiet_exceptions=()):
     """ Wait on many tasks at the same time and return when any is finished
 
     Err once any of the tasks err.
@@ -254,11 +252,11 @@ def Any(args, quiet_exceptions=()):
     quiet_exceptions: tuple, Exception
         Exception types to avoid logging if they fail
     """
-    tasks = gen.WaitIterator(*args)
+    tasks = gen.WaitIterator(*map(asyncio.ensure_future, args))
     results = [None for _ in args]
     while not tasks.done():
         try:
-            result = yield tasks.next()
+            result = await tasks.next()
         except Exception:
 
             @gen.coroutine
@@ -280,7 +278,7 @@ def quiet():
 
         results[tasks.current_index] = result
         break
-    raise gen.Return(results)
+    return results
 
 
 def sync(loop, func, *args, callback_timeout=None, **kwargs):
@@ -1397,7 +1395,6 @@ def reset_logger_locks():
         )
 
     if not jupyter_event_loop_initialized:
-        import asyncio
         import tornado.platform.asyncio
 
         asyncio.set_event_loop_policy(
@@ -1489,3 +1486,7 @@ def format_dashboard_link(host, port):
     else:
         scheme = "http"
     return template.format(scheme=scheme, host=host, port=port, **os.environ)
+
+
+def is_coroutine_function(f):
+    return asyncio.iscoroutinefunction(f) or gen.is_coroutine_function(f)
diff --git a/distributed/utils_comm.py b/distributed/utils_comm.py
index e6f5235afe0..af393cbd79e 100644
--- a/distributed/utils_comm.py
+++ b/distributed/utils_comm.py
@@ -1,12 +1,10 @@
 from __future__ import print_function, division, absolute_import
 
+import asyncio
 from collections import defaultdict
 from itertools import cycle
 import random
 
-from tornado import gen
-from tornado.gen import Return
-
 from dask.optimization import SubgraphCallable
 from toolz import merge, concat, groupby, drop
 
@@ -14,8 +12,7 @@
 from .utils import All, tokey
 
 
-@gen.coroutine
-def gather_from_workers(who_has, rpc, close=True, serializers=None, who=None):
+async def gather_from_workers(who_has, rpc, close=True, serializers=None, who=None):
     """ Gather data directly from peers
 
     Parameters
@@ -59,20 +56,22 @@ def gather_from_workers(who_has, rpc, close=True, serializers=None, who=None):
         rpcs = {addr: rpc(addr) for addr in d}
         try:
             coroutines = {
-                address: get_data_from_worker(
-                    rpc,
-                    keys,
-                    address,
-                    who=who,
-                    serializers=serializers,
-                    max_connections=False,
+                address: asyncio.ensure_future(
+                    get_data_from_worker(
+                        rpc,
+                        keys,
+                        address,
+                        who=who,
+                        serializers=serializers,
+                        max_connections=False,
+                    )
                 )
                 for address, keys in d.items()
             }
             response = {}
             for worker, c in coroutines.items():
                 try:
-                    r = yield c
+                    r = await c
                 except EnvironmentError:
                     missing_workers.add(worker)
                 else:
@@ -85,7 +84,7 @@ def gather_from_workers(who_has, rpc, close=True, serializers=None, who=None):
         results.update(response)
 
     bad_keys = {k: list(original_who_has[k]) for k in all_bad_keys}
-    raise Return((results, bad_keys, list(missing_workers)))
+    return (results, bad_keys, list(missing_workers))
 
 
 class WrappedKey(object):
@@ -109,8 +108,7 @@ def __repr__(self):
 _round_robin_counter = [0]
 
 
-@gen.coroutine
-def scatter_to_workers(nthreads, data, rpc=rpc, report=True, serializers=None):
+async def scatter_to_workers(nthreads, data, rpc=rpc, report=True, serializers=None):
     """ Scatter data directly to workers
 
     This distributes data in a round-robin fashion to a set of workers based on
@@ -134,7 +132,7 @@ def scatter_to_workers(nthreads, data, rpc=rpc, report=True, serializers=None):
 
     rpcs = {addr: rpc(addr) for addr in d}
     try:
-        out = yield All(
+        out = await All(
             [
                 rpcs[address].update_data(
                     data=v, report=report, serializers=serializers
@@ -150,7 +148,7 @@ def scatter_to_workers(nthreads, data, rpc=rpc, report=True, serializers=None):
 
     who_has = {k: [w for w, _, _ in v] for k, v in groupby(1, L).items()}
 
-    raise Return((names, who_has, nbytes))
+    return (names, who_has, nbytes)
 
 
 collection_types = (tuple, list, set, frozenset)
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index 5f3dff548cf..d2932857c40 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -1,5 +1,6 @@
 from __future__ import print_function, division, absolute_import
 
+import asyncio
 import collections
 from contextlib import contextmanager
 import copy
@@ -110,10 +111,9 @@ def invalid_python_script(tmpdir_factory):
     return local_file
 
 
-@gen.coroutine
-def cleanup_global_workers():
+async def cleanup_global_workers():
     for worker in Worker._instances:
-        worker.close(report=False, executor_wait=False)
+        await worker.close(report=False, executor_wait=False)
 
 
 @pytest.fixture
@@ -399,10 +399,9 @@ def apply(func, *args, **kwargs):
     return apply, list(map(varying, itemslists))
 
 
-@gen.coroutine
-def geninc(x, delay=0.02):
-    yield gen.sleep(delay)
-    raise gen.Return(x + 1)
+async def geninc(x, delay=0.02):
+    await gen.sleep(delay)
+    return x + 1
 
 
 def compile_snippet(code, dedent=True):
@@ -429,8 +428,7 @@ async def asyncinc(x, delay=0.02):
 _readone_queues = {}
 
 
-@gen.coroutine
-def readone(comm):
+async def readone(comm):
     """
     Read one message at a time from a comm that reads lists of
     messages.
@@ -440,11 +438,10 @@ def readone(comm):
     except KeyError:
         q = _readone_queues[comm] = queues.Queue()
 
-        @gen.coroutine
-        def background_read():
+        async def background_read():
             while True:
                 try:
-                    messages = yield comm.read()
+                    messages = await comm.read()
                 except CommClosedError:
                     break
                 for msg in messages:
@@ -454,11 +451,11 @@ def background_read():
 
         background_read()
 
-    msg = yield q.get()
+    msg = await q.get()
     if msg is None:
         raise CommClosedError
     else:
-        raise gen.Return(msg)
+        return msg
 
 
 def run_scheduler(q, nputs, port=0, **kwargs):
@@ -467,13 +464,17 @@ def run_scheduler(q, nputs, port=0, **kwargs):
     # On Python 2.7 and Unix, fork() is used to spawn child processes,
     # so avoid inheriting the parent's IO loop.
     with pristine_loop() as loop:
-        scheduler = Scheduler(validate=True, host="127.0.0.1", port=port, **kwargs)
-        done = scheduler.start()
 
-        for i in range(nputs):
-            q.put(scheduler.address)
+        async def _():
+            scheduler = await Scheduler(
+                validate=True, host="127.0.0.1", port=port, **kwargs
+            )
+            for i in range(nputs):
+                q.put(scheduler.address)
+            await scheduler.finished()
+
         try:
-            loop.start()
+            loop.run_sync(_)
         finally:
             loop.close(all_fds=True)
 
@@ -485,16 +486,14 @@ def run_worker(q, scheduler_q, **kwargs):
     with log_errors():
         with pristine_loop() as loop:
             scheduler_addr = scheduler_q.get()
-            worker = Worker(scheduler_addr, validate=True, **kwargs)
-            loop.run_sync(worker.start)
-            q.put(worker.address)
-            try:
 
-                @gen.coroutine
-                def wait_until_closed():
-                    yield worker._closed.wait()
+            async def _():
+                worker = await Worker(scheduler_addr, validate=True, **kwargs)
+                q.put(worker.address)
+                await worker.finished()
 
-                loop.run_sync(wait_until_closed)
+            try:
+                loop.run_sync(_)
             finally:
                 loop.close(all_fds=True)
 
@@ -503,13 +502,15 @@ def run_nanny(q, scheduler_q, **kwargs):
     with log_errors():
         with pristine_loop() as loop:
             scheduler_addr = scheduler_q.get()
-            worker = Nanny(scheduler_addr, validate=True, **kwargs)
-            loop.run_sync(worker.start)
-            q.put(worker.address)
+
+            async def _():
+                worker = await Nanny(scheduler_addr, validate=True, **kwargs)
+                q.put(worker.address)
+                await worker.finished()
+
             try:
-                loop.start()
+                loop.run_sync(_)
             finally:
-                loop.run_sync(worker.close)
                 loop.close(all_fds=True)
 
 
@@ -533,9 +534,8 @@ def fail():
             "some RPCs left active by test: %s" % (set(rpc.active) - active_before)
         )
 
-    @gen.coroutine
-    def wait():
-        yield async_wait_for(
+    async def wait():
+        await async_wait_for(
             lambda: len(set(rpc.active) - active_before) == 0,
             timeout=active_rpc_timeout,
             fail_func=fail,
@@ -738,23 +738,20 @@ def cluster(
         assert time() < start + 5, ("Workers still around after five seconds", text)
 
 
-@gen.coroutine
-def disconnect(addr, timeout=3, rpc_kwargs=None):
+async def disconnect(addr, timeout=3, rpc_kwargs=None):
     rpc_kwargs = rpc_kwargs or {}
 
-    @gen.coroutine
-    def do_disconnect():
+    async def do_disconnect():
         with ignoring(EnvironmentError, CommClosedError):
             with rpc(addr, **rpc_kwargs) as w:
-                yield w.terminate(close=True)
+                await w.terminate(close=True)
 
     with ignoring(TimeoutError):
-        yield gen.with_timeout(timedelta(seconds=timeout), do_disconnect())
+        await gen.with_timeout(timedelta(seconds=timeout), do_disconnect())
 
 
-@gen.coroutine
-def disconnect_all(addresses, timeout=3, rpc_kwargs=None):
-    yield [disconnect(addr, timeout, rpc_kwargs) for addr in addresses]
+async def disconnect_all(addresses, timeout=3, rpc_kwargs=None):
+    await asyncio.gather(*[disconnect(addr, timeout, rpc_kwargs) for addr in addresses])
 
 
 def gen_test(timeout=10):
@@ -783,8 +780,7 @@ def test_func():
 from .worker import Worker
 
 
-@gen.coroutine
-def start_cluster(
+async def start_cluster(
     nthreads,
     scheduler_addr,
     loop,
@@ -793,7 +789,7 @@ def start_cluster(
     scheduler_kwargs={},
     worker_kwargs={},
 ):
-    s = Scheduler(
+    s = await Scheduler(
         loop=loop,
         validate=True,
         security=security,
@@ -801,7 +797,6 @@ def start_cluster(
         host=scheduler_addr,
         **scheduler_kwargs
     )
-    done = s.start()
     workers = [
         Worker(
             s.address,
@@ -818,31 +813,29 @@ def start_cluster(
     # for w in workers:
     #     w.rpc = workers[0].rpc
 
-    yield workers
+    await asyncio.gather(*workers)
 
     start = time()
     while len(s.workers) < len(nthreads) or any(
         comm.comm is None for comm in s.stream_comms.values()
     ):
-        yield gen.sleep(0.01)
+        await gen.sleep(0.01)
         if time() - start > 5:
-            yield [w.close(timeout=1) for w in workers]
-            yield s.close(fast=True)
+            await asyncio.gather(*[w.close(timeout=1) for w in workers])
+            await s.close(fast=True)
             raise Exception("Cluster creation timeout")
-    raise gen.Return((s, workers))
+    return s, workers
 
 
-@gen.coroutine
-def end_cluster(s, workers):
+async def end_cluster(s, workers):
     logger.debug("Closing out test cluster")
 
-    @gen.coroutine
-    def end_worker(w):
+    async def end_worker(w):
         with ignoring(TimeoutError, CommClosedError, EnvironmentError):
-            yield w.close(report=False)
+            await w.close(report=False)
 
-    yield [end_worker(w) for w in workers]
-    yield s.close()  # wait until scheduler stops completely
+    await asyncio.gather(*[end_worker(w) for w in workers])
+    await s.close()  # wait until scheduler stops completely
     s.stop()
 
 
@@ -859,7 +852,7 @@ def gen_cluster(
     client_kwargs={},
     active_rpc_timeout=1,
     config={},
-    check_new_threads=True,
+    clean_kwargs={},
 ):
     from distributed import Client
 
@@ -874,7 +867,7 @@ def test_foo(scheduler, worker1, worker2):
         end
     """
     if ncores is not None:
-        warnings.warn("ncores= has moved to nthreads=")
+        warnings.warn("ncores= has moved to nthreads=", stacklevel=2)
         nthreads = ncores
 
     worker_kwargs = merge(
@@ -888,15 +881,14 @@ def _(func):
         def test_func():
             result = None
             workers = []
-            with clean(threads=check_new_threads, timeout=active_rpc_timeout) as loop:
+            with clean(timeout=active_rpc_timeout, **clean_kwargs) as loop:
 
-                @gen.coroutine
-                def coro():
+                async def coro():
                     with dask.config.set(config):
                         s = False
                         for i in range(5):
                             try:
-                                s, ws = yield start_cluster(
+                                s, ws = await start_cluster(
                                     nthreads,
                                     scheduler,
                                     loop,
@@ -917,7 +909,7 @@ def coro():
                         if s is False:
                             raise Exception("Could not start cluster")
                         if client:
-                            c = yield Client(
+                            c = await Client(
                                 s.address,
                                 loop=loop,
                                 security=security,
@@ -931,36 +923,36 @@ def coro():
                                 future = gen.with_timeout(
                                     timedelta(seconds=timeout), future
                                 )
-                            result = yield future
+                            result = await future
                             if s.validate:
                                 s.validate_state()
                         finally:
                             if client and c.status not in ("closing", "closed"):
-                                yield c._close(fast=s.status == "closed")
-                            yield end_cluster(s, workers)
-                            yield gen.with_timeout(
+                                await c._close(fast=s.status == "closed")
+                            await end_cluster(s, workers)
+                            await gen.with_timeout(
                                 timedelta(seconds=1), cleanup_global_workers()
                             )
 
                         try:
-                            c = yield default_client()
+                            c = await default_client()
                         except ValueError:
                             pass
                         else:
-                            yield c._close(fast=True)
+                            await c._close(fast=True)
 
                         for i in range(5):
                             if all(c.closed() for c in Comm._instances):
                                 break
                             else:
-                                yield gen.sleep(0.05)
+                                await gen.sleep(0.05)
                         else:
                             L = [c for c in Comm._instances if not c.closed()]
                             Comm._instances.clear()
                             # raise ValueError("Unclosed Comms", L)
                             print("Unclosed Comms", L)
 
-                        raise gen.Return(result)
+                        return result
 
                 result = loop.run_sync(
                     coro, timeout=timeout * 2 if timeout else timeout
@@ -1074,11 +1066,10 @@ def wait_for(predicate, timeout, fail_func=None, period=0.001):
             pytest.fail("condition not reached until %s seconds" % (timeout,))
 
 
-@gen.coroutine
-def async_wait_for(predicate, timeout, fail_func=None, period=0.001):
+async def async_wait_for(predicate, timeout, fail_func=None, period=0.001):
     deadline = time() + timeout
     while not predicate():
-        yield gen.sleep(period)
+        await gen.sleep(period)
         if time() > deadline:
             if fail_func is not None:
                 fail_func()
@@ -1118,20 +1109,18 @@ def requires_ipv6(test_func):
     requires_ipv6 = pytest.mark.skip("ipv6 required")
 
 
-@gen.coroutine
-def assert_can_connect(addr, timeout=None, connection_args=None):
+async def assert_can_connect(addr, timeout=None, connection_args=None):
     """
     Check that it is possible to connect to the distributed *addr*
     within the given *timeout*.
     """
     if timeout is None:
         timeout = 0.5
-    comm = yield connect(addr, timeout=timeout, connection_args=connection_args)
+    comm = await connect(addr, timeout=timeout, connection_args=connection_args)
     comm.abort()
 
 
-@gen.coroutine
-def assert_cannot_connect(
+async def assert_cannot_connect(
     addr, timeout=None, connection_args=None, exception_class=EnvironmentError
 ):
     """
@@ -1141,12 +1130,11 @@ def assert_cannot_connect(
     if timeout is None:
         timeout = 0.5
     with pytest.raises(exception_class):
-        comm = yield connect(addr, timeout=timeout, connection_args=connection_args)
+        comm = await connect(addr, timeout=timeout, connection_args=connection_args)
         comm.abort()
 
 
-@gen.coroutine
-def assert_can_connect_from_everywhere_4_6(
+async def assert_can_connect_from_everywhere_4_6(
     port, timeout=None, connection_args=None, protocol="tcp"
 ):
     """
@@ -1162,11 +1150,10 @@ def assert_can_connect_from_everywhere_4_6(
             assert_can_connect("%s://[::1]:%d" % (protocol, port), *args),
             assert_can_connect("%s://[%s]:%d" % (protocol, get_ipv6(), port), *args),
         ]
-    yield futures
+    await asyncio.gather(*futures)
 
 
-@gen.coroutine
-def assert_can_connect_from_everywhere_4(
+async def assert_can_connect_from_everywhere_4(
     port, timeout=None, connection_args=None, protocol="tcp"
 ):
     """
@@ -1182,11 +1169,10 @@ def assert_can_connect_from_everywhere_4(
             assert_cannot_connect("%s://[::1]:%d" % (protocol, port), *args),
             assert_cannot_connect("%s://[%s]:%d" % (protocol, get_ipv6(), port), *args),
         ]
-    yield futures
+    await asyncio.gather(*futures)
 
 
-@gen.coroutine
-def assert_can_connect_locally_4(port, timeout=None, connection_args=None):
+async def assert_can_connect_locally_4(port, timeout=None, connection_args=None):
     """
     Check that the local *port* is only reachable from local IPv4 addresses.
     """
@@ -1199,11 +1185,12 @@ def assert_can_connect_locally_4(port, timeout=None, connection_args=None):
             assert_cannot_connect("tcp://[::1]:%d" % port, *args),
             assert_cannot_connect("tcp://[%s]:%d" % (get_ipv6(), port), *args),
         ]
-    yield futures
+    await asyncio.gather(*futures)
 
 
-@gen.coroutine
-def assert_can_connect_from_everywhere_6(port, timeout=None, connection_args=None):
+async def assert_can_connect_from_everywhere_6(
+    port, timeout=None, connection_args=None
+):
     """
     Check that the local *port* is reachable from all IPv6 addresses.
     """
@@ -1215,11 +1202,10 @@ def assert_can_connect_from_everywhere_6(port, timeout=None, connection_args=Non
         assert_can_connect("tcp://[::1]:%d" % port, *args),
         assert_can_connect("tcp://[%s]:%d" % (get_ipv6(), port), *args),
     ]
-    yield futures
+    await asyncio.gather(*futures)
 
 
-@gen.coroutine
-def assert_can_connect_locally_6(port, timeout=None, connection_args=None):
+async def assert_can_connect_locally_6(port, timeout=None, connection_args=None):
     """
     Check that the local *port* is only reachable from local IPv6 addresses.
     """
@@ -1232,7 +1218,7 @@ def assert_can_connect_locally_6(port, timeout=None, connection_args=None):
     ]
     if get_ipv6() != "::1":  # No outside IPv6 connectivity?
         futures += [assert_cannot_connect("tcp://[%s]:%d" % (get_ipv6(), port), *args)]
-    yield futures
+    await asyncio.gather(*futures)
 
 
 @contextmanager
@@ -1509,9 +1495,9 @@ def check_instances():
 
     for w in Worker._instances:
         with ignoring(RuntimeError):  # closed IOLoop
-            w.close(report=False, executor_wait=False)
+            w.loop.add_callback(w.close, report=False, executor_wait=False)
             if w.status == "running":
-                w.close()
+                w.loop.add_callback(w.close)
     Worker._instances.clear()
 
     for i in range(5):
diff --git a/distributed/variable.py b/distributed/variable.py
index 7b775d3327a..30ffc5bf72d 100644
--- a/distributed/variable.py
+++ b/distributed/variable.py
@@ -1,5 +1,6 @@
 from __future__ import print_function, division, absolute_import
 
+import asyncio
 from collections import defaultdict
 import logging
 import uuid
@@ -58,15 +59,14 @@ def set(self, stream=None, name=None, key=None, data=None, client=None):
             pass
         else:
             if old["type"] == "Future" and old["value"] != key:
-                self.release(old["value"], name)
+                asyncio.ensure_future(self.release(old["value"], name))
         if name not in self.variables:
             self.started.notify_all()
         self.variables[name] = record
 
-    @gen.coroutine
-    def release(self, key, name):
+    async def release(self, key, name):
         while self.waiting[key, name]:
-            yield self.waiting_conditions[name].wait()
+            await self.waiting_conditions[name].wait()
 
         self.scheduler.client_releases_keys(keys=[key], client="variable-%s" % name)
         del self.waiting[key, name]
@@ -76,8 +76,7 @@ def future_release(self, name=None, key=None, token=None, client=None):
         if not self.waiting[key, name]:
             self.waiting_conditions[name].notify_all()
 
-    @gen.coroutine
-    def get(self, stream=None, name=None, client=None, timeout=None):
+    async def get(self, stream=None, name=None, client=None, timeout=None):
         start = time()
         while name not in self.variables:
             if timeout is not None:
@@ -86,7 +85,7 @@ def get(self, stream=None, name=None, client=None, timeout=None):
                 left = None
             if left and left < 0:
                 raise gen.TimeoutError()
-            yield self.started.wait(timeout=left)
+            await self.started.wait(timeout=left)
         record = self.variables[name]
         if record["type"] == "Future":
             key = record["value"]
@@ -99,10 +98,9 @@ def get(self, stream=None, name=None, client=None, timeout=None):
                 msg["traceback"] = ts.exception_blame.traceback
             record = merge(record, msg)
             self.waiting[key, name].add(token)
-        raise gen.Return(record)
+        return record
 
-    @gen.coroutine
-    def delete(self, stream=None, name=None, client=None):
+    async def delete(self, stream=None, name=None, client=None):
         with log_errors():
             try:
                 old = self.variables[name]
@@ -110,7 +108,7 @@ def delete(self, stream=None, name=None, client=None):
                 pass
             else:
                 if old["type"] == "Future":
-                    yield self.release(old["value"], name)
+                    await self.release(old["value"], name)
             del self.waiting_conditions[name]
             del self.variables[name]
 
@@ -151,14 +149,13 @@ def __init__(self, name=None, client=None, maxsize=0):
         self.client = client or _get_global_client()
         self.name = name or "variable-" + uuid.uuid4().hex
 
-    @gen.coroutine
-    def _set(self, value):
+    async def _set(self, value):
         if isinstance(value, Future):
-            yield self.client.scheduler.variable_set(
+            await self.client.scheduler.variable_set(
                 key=tokey(value.key), name=self.name
             )
         else:
-            yield self.client.scheduler.variable_set(data=value, name=self.name)
+            await self.client.scheduler.variable_set(data=value, name=self.name)
 
     def set(self, value, **kwargs):
         """ Set the value of this variable
@@ -170,9 +167,8 @@ def set(self, value, **kwargs):
         """
         return self.client.sync(self._set, value, **kwargs)
 
-    @gen.coroutine
-    def _get(self, timeout=None):
-        d = yield self.client.scheduler.variable_get(
+    async def _get(self, timeout=None):
+        d = await self.client.scheduler.variable_get(
             timeout=timeout, name=self.name, client=self.client.id
         )
         if d["type"] == "Future":
@@ -189,7 +185,7 @@ def _get(self, timeout=None):
             )
         else:
             value = d["value"]
-        raise gen.Return(value)
+        return value
 
     def get(self, timeout=None, **kwargs):
         """ Get the value of this variable """
diff --git a/distributed/worker.py b/distributed/worker.py
index b052dd05799..e65c1cbccdc 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -1,5 +1,6 @@
 from __future__ import print_function, division, absolute_import
 
+import asyncio
 import bisect
 from collections import defaultdict, deque
 from datetime import timedelta
@@ -25,7 +26,6 @@
     from cytoolz import pluck, partial, merge, first
 except ImportError:
     from toolz import pluck, partial, merge, first
-from tornado.gen import Return
 from tornado import gen
 from tornado.ioloop import IOLoop
 from tornado.locks import Event
@@ -48,9 +48,9 @@
 from .sizeof import safe_sizeof as sizeof
 from .threadpoolexecutor import ThreadPoolExecutor, secede as tpe_secede
 from .utils import (
+    get_ip,
     funcname,
     typename,
-    get_ip,
     has_arg,
     _maybe_complex,
     log_errors,
@@ -704,8 +704,7 @@ def identity(self, comm=None):
     # External Services #
     #####################
 
-    @gen.coroutine
-    def _register_with_scheduler(self):
+    async def _register_with_scheduler(self):
         self.periodic_callbacks["heartbeat"].stop()
         start = time()
         if self.contact_address is None:
@@ -717,21 +716,21 @@ def _register_with_scheduler(self):
                     "Timed out when connecting to scheduler '%s'",
                     self.scheduler.address,
                 )
-                yield self.close(timeout=1)
+                await self.close(timeout=1)
                 raise gen.TimeoutError(
                     "Timed out connecting to scheduler '%s'" % self.scheduler.address
                 )
             if self.status in ("closed", "closing"):
-                raise gen.Return
+                return
             try:
                 _start = time()
                 types = {k: typename(v) for k, v in self.data.items()}
-                comm = yield connect(
+                comm = await connect(
                     self.scheduler.address, connection_args=self.connection_args
                 )
                 comm.name = "Worker->Scheduler"
                 comm._server = weakref.ref(self)
-                yield comm.write(
+                await comm.write(
                     dict(
                         op="register-worker",
                         reply=False,
@@ -758,7 +757,7 @@ def _register_with_scheduler(self):
                     if diff < 0:
                         continue
                     future = gen.with_timeout(timedelta(seconds=diff), future)
-                response = yield future
+                response = await future
                 _end = time()
                 middle = (_start + _end) / 2
                 self.latency = (_end - start) * 0.05 + self.latency * 0.95
@@ -767,15 +766,18 @@ def _register_with_scheduler(self):
                 break
             except EnvironmentError:
                 logger.info("Waiting to connect to: %26s", self.scheduler.address)
-                yield gen.sleep(0.1)
+                await gen.sleep(0.1)
             except gen.TimeoutError:
                 logger.info("Timed out when connecting to scheduler")
         if response["status"] != "OK":
             raise ValueError("Unexpected response from register: %r" % (response,))
         else:
-            yield [
-                self.plugin_add(plugin=plugin) for plugin in response["worker-plugins"]
-            ]
+            await asyncio.gather(
+                *[
+                    self.plugin_add(plugin=plugin)
+                    for plugin in response["worker-plugins"]
+                ]
+            )
 
             logger.info("        Registered to: %26s", self.scheduler.address)
             logger.info("-" * 49)
@@ -785,21 +787,20 @@ def _register_with_scheduler(self):
         self.periodic_callbacks["heartbeat"].start()
         self.loop.add_callback(self.handle_scheduler, comm)
 
-    @gen.coroutine
-    def heartbeat(self):
+    async def heartbeat(self):
         if not self.heartbeat_active:
             self.heartbeat_active = True
             logger.debug("Heartbeat: %s" % self.address)
             try:
                 start = time()
-                response = yield self.scheduler.heartbeat_worker(
+                response = await self.scheduler.heartbeat_worker(
                     address=self.contact_address, now=time(), metrics=self.get_metrics()
                 )
                 end = time()
                 middle = (start + end) / 2
 
                 if response["status"] == "missing":
-                    yield self._register_with_scheduler()
+                    await self._register_with_scheduler()
                     return
                 self.scheduler_delay = response["time"] - middle
                 self.periodic_callbacks["heartbeat"].callback_time = (
@@ -812,10 +813,9 @@ def heartbeat(self):
         else:
             logger.debug("Heartbeat skipped: channel busy")
 
-    @gen.coroutine
-    def handle_scheduler(self, comm):
+    async def handle_scheduler(self, comm):
         try:
-            yield self.handle_stream(
+            await self.handle_stream(
                 comm, every_cycle=[self.ensure_communicating, self.ensure_computing]
             )
         except Exception as e:
@@ -826,7 +826,7 @@ def handle_scheduler(self, comm):
                 logger.info("Connection to scheduler broken.  Reconnecting...")
                 self.loop.add_callback(self._register_with_scheduler)
             else:
-                yield self.close(report=False)
+                await self.close(report=False)
 
     def start_ipython(self, comm):
         """Start an IPython kernel
@@ -841,8 +841,7 @@ def start_ipython(self, comm):
             )
         return self._ipython_kernel.get_connection_info()
 
-    @gen.coroutine
-    def upload_file(self, comm, filename=None, data=None, load=True):
+    async def upload_file(self, comm, filename=None, data=None, load=True):
         out_filename = os.path.join(self.local_dir, filename)
 
         def func(data):
@@ -856,28 +855,27 @@ def func(data):
         if len(data) < 10000:
             data = func(data)
         else:
-            data = yield offload(func, data)
+            data = await offload(func, data)
 
         if load:
             try:
                 import_file(out_filename)
             except Exception as e:
                 logger.exception(e)
-                raise gen.Return({"status": "error", "exception": to_serialize(e)})
+                return {"status": "error", "exception": to_serialize(e)}
 
-        raise gen.Return({"status": "OK", "nbytes": len(data)})
+        return {"status": "OK", "nbytes": len(data)}
 
     def keys(self, comm=None):
         return list(self.data)
 
-    @gen.coroutine
-    def gather(self, comm=None, who_has=None):
+    async def gather(self, comm=None, who_has=None):
         who_has = {
             k: [coerce_to_address(addr) for addr in v]
             for k, v in who_has.items()
             if k not in self.data
         }
-        result, missing_keys, missing_workers = yield gather_from_workers(
+        result, missing_keys, missing_workers = await gather_from_workers(
             who_has, rpc=self.rpc, who=self.address
         )
         if missing_keys:
@@ -887,18 +885,17 @@ def gather(self, comm=None, who_has=None):
                 missing_workers,
                 who_has,
             )
-            raise Return({"status": "missing-data", "keys": missing_keys})
+            return {"status": "missing-data", "keys": missing_keys}
         else:
             self.update_data(data=result, report=False)
-            raise Return({"status": "OK"})
+            return {"status": "OK"}
 
     #############
     # Lifecycle #
     #############
 
-    @gen.coroutine
-    def start(self):
-        assert self.status is None
+    async def start(self):
+        assert self.status is None, self.status
 
         enable_gc_diagnosis()
         thread_state.on_event_loop_thread = True
@@ -938,10 +935,12 @@ def start(self):
 
         setproctitle("dask-worker [%s]" % self.address)
 
-        yield [self.plugin_add(plugin=plugin) for plugin in self._pending_plugins]
+        await asyncio.gather(
+            *[self.plugin_add(plugin=plugin) for plugin in self._pending_plugins]
+        )
         self._pending_plugins = ()
 
-        yield self._register_with_scheduler()
+        await self._register_with_scheduler()
 
         self.start_periodic_callbacks()
         return self
@@ -950,11 +949,10 @@ def _close(self, *args, **kwargs):
         warnings.warn("Worker._close has moved to Worker.close", stacklevel=2)
         return self.close(*args, **kwargs)
 
-    @gen.coroutine
-    def close(self, report=True, timeout=10, nanny=True, executor_wait=True):
+    async def close(self, report=True, timeout=10, nanny=True, executor_wait=True):
         with log_errors():
             if self.status in ("closed", "closing"):
-                yield self.finished()
+                await self.finished()
                 return
 
             disable_gc_diagnosis()
@@ -963,25 +961,29 @@ def close(self, report=True, timeout=10, nanny=True, executor_wait=True):
                 logger.info("Stopping worker at %s", self.address)
             except ValueError:  # address not available if already closed
                 logger.info("Stopping worker")
+            if self.status != "running":
+                logger.info("Closed worker has not yet started: %s", self.status)
             self.status = "closing"
 
             if nanny and self.nanny:
                 with self.rpc(self.nanny) as r:
-                    yield r.close_gracefully()
+                    await r.close_gracefully()
 
             setproctitle("dask-worker [closing]")
 
-            yield [
+            teardowns = [
                 plugin.teardown(self)
                 for plugin in self.plugins.values()
                 if hasattr(plugin, "teardown")
             ]
 
+            await asyncio.gather(*[td for td in teardowns if hasattr(td, "__await__")])
+
             for pc in self.periodic_callbacks.values():
                 pc.stop()
             with ignoring(EnvironmentError, gen.TimeoutError):
                 if report:
-                    yield gen.with_timeout(
+                    await gen.with_timeout(
                         timedelta(seconds=timeout),
                         self.scheduler.unregister(address=self.contact_address),
                     )
@@ -1006,25 +1008,23 @@ def close(self, report=True, timeout=10, nanny=True, executor_wait=True):
 
             if nanny and self.nanny:
                 with self.rpc(self.nanny) as r:
-                    yield r.terminate()
+                    await r.terminate()
 
             self.stop()
             self.rpc.close()
             self._closed.set()
 
             self.status = "closed"
-            yield ServerNode.close(self)
+            await ServerNode.close(self)
 
             setproctitle("dask-worker [closed]")
 
-    @gen.coroutine
-    def terminate(self, comm, report=True):
-        yield self.close(report=report)
-        raise Return("OK")
+    async def terminate(self, comm, report=True):
+        await self.close(report=report)
+        return "OK"
 
-    @gen.coroutine
-    def wait_until_closed(self):
-        yield self._closed.wait()
+    async def wait_until_closed(self):
+        await self._closed.wait()
         assert self.status == "closed"
 
     ################
@@ -1036,13 +1036,12 @@ def send_to_worker(self, address, msg):
             bcomm = BatchedSend(interval="1ms", loop=self.loop)
             self.stream_comms[address] = bcomm
 
-            @gen.coroutine
-            def batched_send_connect():
-                comm = yield connect(
+            async def batched_send_connect():
+                comm = await connect(
                     address, connection_args=self.connection_args  # TODO, serialization
                 )
                 comm.name = "Worker->Worker"
-                yield comm.write({"op": "connection_stream"})
+                await comm.write({"op": "connection_stream"})
 
                 bcomm.start(comm)
 
@@ -1050,8 +1049,7 @@ def batched_send_connect():
 
         self.stream_comms[address].send(msg)
 
-    @gen.coroutine
-    def get_data(
+    async def get_data(
         self, comm, keys=None, who=None, serializers=None, max_connections=None
     ):
         start = time()
@@ -1071,7 +1069,7 @@ def get_data(
             max_connections is not False
             and self.outgoing_current_count > max_connections
         ):
-            raise gen.Return({"status": "busy"})
+            return {"status": "busy"}
 
         self.outgoing_current_count += 1
         data = {k: self.data[k] for k in keys if k in self.data}
@@ -1091,8 +1089,8 @@ def get_data(
         start = time()
 
         try:
-            compressed = yield comm.write(msg, serializers=serializers)
-            response = yield comm.read(deserializers=serializers)
+            compressed = await comm.write(msg, serializers=serializers)
+            response = await comm.read(deserializers=serializers)
             assert response == "OK", response
         except EnvironmentError:
             logger.exception(
@@ -1124,7 +1122,7 @@ def get_data(
             }
         )
 
-        raise gen.Return("dont-reply")
+        return "dont-reply"
 
     ###################
     # Local Execution #
@@ -1152,8 +1150,7 @@ def update_data(self, comm=None, data=None, report=True, serializers=None):
         info = {"nbytes": {k: sizeof(v) for k, v in data.items()}, "status": "OK"}
         return info
 
-    @gen.coroutine
-    def delete_data(self, comm=None, keys=None, report=True):
+    async def delete_data(self, comm=None, keys=None, report=True):
         if keys:
             for key in list(keys):
                 self.log.append((key, "delete"))
@@ -1167,13 +1164,12 @@ def delete_data(self, comm=None, keys=None, report=True):
             if report:
                 logger.debug("Reporting loss of keys to scheduler")
                 # TODO: this route seems to not exist?
-                yield self.scheduler.remove_keys(
+                await self.scheduler.remove_keys(
                     address=self.contact_address, keys=list(keys)
                 )
-        raise Return("OK")
+        return "OK"
 
-    @gen.coroutine
-    def set_resources(self, **resources):
+    async def set_resources(self, **resources):
         for r, quantity in resources.items():
             if r in self.total_resources:
                 self.available_resources[r] += quantity - self.total_resources[r]
@@ -1181,7 +1177,7 @@ def set_resources(self, **resources):
                 self.available_resources[r] = quantity
             self.total_resources[r] = quantity
 
-        yield self.scheduler.set_resources(
+        await self.scheduler.set_resources(
             resources=self.total_resources, worker=self.contact_address
         )
 
@@ -1786,8 +1782,7 @@ def select_keys_for_gather(self, worker, dep):
 
         return deps, total_bytes
 
-    @gen.coroutine
-    def gather_dep(self, worker, dep, deps, total_nbytes, cause=None):
+    async def gather_dep(self, worker, dep, deps, total_nbytes, cause=None):
         if self.status != "running":
             return
         with log_errors():
@@ -1804,7 +1799,7 @@ def gather_dep(self, worker, dep, deps, total_nbytes, cause=None):
                 logger.debug("Request %d keys", len(deps))
 
                 start = time()
-                response = yield get_data_from_worker(
+                response = await get_data_from_worker(
                     self.rpc, deps, worker, who=self.address
                 )
                 stop = time()
@@ -1896,10 +1891,10 @@ def gather_dep(self, worker, dep, deps, total_nbytes, cause=None):
                 else:
                     # Exponential backoff to avoid hammering scheduler/worker
                     self.repetitively_busy += 1
-                    yield gen.sleep(0.100 * 1.5 ** self.repetitively_busy)
+                    await gen.sleep(0.100 * 1.5 ** self.repetitively_busy)
 
                     # See if anyone new has the data
-                    yield self.query_who_has(dep)
+                    await self.query_who_has(dep)
                     self.ensure_communicating()
 
     def bad_dep(self, dep):
@@ -1911,8 +1906,7 @@ def bad_dep(self, dep):
             self.transition(key, "error")
         self.release_dep(dep)
 
-    @gen.coroutine
-    def handle_missing_dep(self, *deps, **kwargs):
+    async def handle_missing_dep(self, *deps, **kwargs):
         original_deps = list(deps)
         self.log.append(("handle-missing", deps))
         try:
@@ -1935,7 +1929,7 @@ def handle_missing_dep(self, *deps, **kwargs):
                     self.suspicious_deps[dep],
                 )
 
-            who_has = yield self.scheduler.who_has(keys=list(deps))
+            who_has = await self.scheduler.who_has(keys=list(deps))
             who_has = {k: v for k, v in who_has.items() if v}
             self.update_who_has(who_has)
             for dep in deps:
@@ -1955,7 +1949,7 @@ def handle_missing_dep(self, *deps, **kwargs):
             retries = kwargs.get("retries", 5)
             self.log.append(("handle-missing-failed", retries, deps))
             if retries > 0:
-                yield self.handle_missing_dep(self, *deps, retries=retries - 1)
+                await self.handle_missing_dep(self, *deps, retries=retries - 1)
             else:
                 raise
         finally:
@@ -1967,12 +1961,11 @@ def handle_missing_dep(self, *deps, **kwargs):
 
             self.ensure_communicating()
 
-    @gen.coroutine
-    def query_who_has(self, *deps):
+    async def query_who_has(self, *deps):
         with log_errors():
-            response = yield self.scheduler.who_has(keys=deps)
+            response = await self.scheduler.who_has(keys=deps)
             self.update_who_has(response)
-            raise gen.Return(response)
+            return response
 
     def update_who_has(self, who_has):
         try:
@@ -2180,8 +2173,7 @@ def run(self, comm, function, args=(), wait=True, kwargs=None):
     def run_coroutine(self, comm, function, args=(), kwargs=None, wait=True):
         return run(self, comm, function=function, args=args, kwargs=kwargs, wait=wait)
 
-    @gen.coroutine
-    def plugin_add(self, comm=None, plugin=None, name=None):
+    async def plugin_add(self, comm=None, plugin=None, name=None):
         with log_errors(pdb=False):
             if isinstance(plugin, bytes):
                 plugin = pickle.loads(plugin)
@@ -2201,16 +2193,17 @@ def plugin_add(self, comm=None, plugin=None, name=None):
                 logger.info("Starting Worker plugin %s" % name)
                 try:
                     result = plugin.setup(worker=self)
-                    if isinstance(result, gen.Future):
-                        result = yield result
+                    if hasattr(result, "__await__"):
+                        result = await result
                 except Exception as e:
                     msg = error_message(e)
                     return msg
                 else:
                     return {"status": "OK"}
 
-    @gen.coroutine
-    def actor_execute(self, comm=None, actor=None, function=None, args=(), kwargs={}):
+    async def actor_execute(
+        self, comm=None, actor=None, function=None, args=(), kwargs={}
+    ):
         separate_thread = kwargs.pop("separate_thread", True)
         key = actor
         actor = self.actors[key]
@@ -2218,9 +2211,9 @@ def actor_execute(self, comm=None, actor=None, function=None, args=(), kwargs={}
         name = key_split(key) + "." + function
 
         if iscoroutinefunction(func):
-            result = yield func(*args, **kwargs)
+            result = await func(*args, **kwargs)
         elif separate_thread:
-            result = yield self.executor_submit(
+            result = await self.executor_submit(
                 name,
                 apply_function_actor,
                 args=(
@@ -2236,7 +2229,7 @@ def actor_execute(self, comm=None, actor=None, function=None, args=(), kwargs={}
             )
         else:
             result = func(*args, **kwargs)
-        raise gen.Return({"status": "OK", "result": to_serialize(result)})
+        return {"status": "OK", "result": to_serialize(result)}
 
     def actor_attribute(self, comm=None, actor=None, attribute=None):
         value = getattr(self.actors[actor], attribute)
@@ -2277,8 +2270,7 @@ def ensure_computing(self):
                 pdb.set_trace()
             raise
 
-    @gen.coroutine
-    def execute(self, key, report=False):
+    async def execute(self, key, report=False):
         executor_error = None
         if self.status in ("closing", "closed"):
             return
@@ -2312,7 +2304,7 @@ def execute(self, key, report=False):
                 "Execute key: %s worker: %s", key, self.address
             )  # TODO: comment out?
             try:
-                result = yield self.executor_submit(
+                result = await self.executor_submit(
                     key,
                     apply_function,
                     args=(
@@ -2391,8 +2383,7 @@ def execute(self, key, report=False):
     # Administrative #
     ##################
 
-    @gen.coroutine
-    def memory_monitor(self):
+    async def memory_monitor(self):
         """ Track this process's memory usage and act accordingly
 
         If we rise above 70% memory use, start dumping data to disk.
@@ -2458,7 +2449,7 @@ def memory_monitor(self):
                 del k, v
                 total += weight
                 count += 1
-                yield gen.moment
+                await gen.sleep(0)
                 memory = proc.memory_info().rss
                 if total > need and memory > target:
                     # Issue a GC to ensure that the evicted data is actually
@@ -2474,7 +2465,7 @@ def memory_monitor(self):
                 )
 
         self._memory_monitoring = False
-        raise gen.Return(total)
+        return total
 
     def cycle_profile(self):
         now = time() + self.scheduler_delay
@@ -2962,8 +2953,7 @@ def parse_memory_limit(memory_limit, nthreads, total_cores=multiprocessing.cpu_c
     return memory_limit
 
 
-@gen.coroutine
-def get_data_from_worker(
+async def get_data_from_worker(
     rpc,
     keys,
     worker,
@@ -2988,10 +2978,10 @@ def get_data_from_worker(
     if deserializers is None:
         deserializers = rpc.deserializers
 
-    comm = yield rpc.connect(worker)
+    comm = await rpc.connect(worker)
     comm.name = "Ephemeral Worker->Worker for gather"
     try:
-        response = yield send_recv(
+        response = await send_recv(
             comm,
             serializers=serializers,
             deserializers=deserializers,
@@ -3006,11 +2996,11 @@ def get_data_from_worker(
             raise ValueError("Unexpected response", response)
         else:
             if status == "OK":
-                yield comm.write("OK")
+                await comm.write("OK")
     finally:
         rpc.reuse(worker, comm)
 
-    raise gen.Return(response)
+    return response
 
 
 job_counter = [0]
@@ -3266,8 +3256,7 @@ def weight(k, v):
     return sizeof(v)
 
 
-@gen.coroutine
-def run(server, comm, function, args=(), kwargs={}, is_coro=None, wait=True):
+async def run(server, comm, function, args=(), kwargs={}, is_coro=None, wait=True):
     function = pickle.loads(function)
     if is_coro is None:
         is_coro = iscoroutinefunction(function)
@@ -3291,7 +3280,7 @@ def run(server, comm, function, args=(), kwargs={}, is_coro=None, wait=True):
             result = function(*args, **kwargs)
         else:
             if wait:
-                result = yield function(*args, **kwargs)
+                result = await function(*args, **kwargs)
             else:
                 server.loop.add_callback(function, *args, **kwargs)
                 result = None
@@ -3308,7 +3297,7 @@ def run(server, comm, function, args=(), kwargs={}, is_coro=None, wait=True):
         response = error_message(e)
     else:
         response = {"status": "OK", "result": to_serialize(result)}
-    raise Return(response)
+    return response
 
 
 _global_workers = Worker._instances
diff --git a/docs/source/adaptive.rst b/docs/source/adaptive.rst
index 0e3a4bdefbc..774ae21e4a6 100644
--- a/docs/source/adaptive.rst
+++ b/docs/source/adaptive.rst
@@ -60,8 +60,7 @@ the correct times.
 .. code-block:: python
 
    class MyCluster(object):
-       @gen.coroutine
-       def scale_up(self, n, **kwargs):
+       async def scale_up(self, n, **kwargs):
            """
            Bring the total count of workers up to ``n``
 
@@ -72,8 +71,7 @@ the correct times.
            """
            raise NotImplementedError()
 
-       @gen.coroutine
-       def scale_down(self, workers):
+       async def scale_down(self, workers):
            """
            Remove ``workers`` from the cluster
 
diff --git a/docs/source/client.rst b/docs/source/client.rst
index b2b520d9614..b955d5ab504 100644
--- a/docs/source/client.rst
+++ b/docs/source/client.rst
@@ -149,8 +149,8 @@ keyword argument.  In this case keys are randomly generated (by ``uuid4``.)
 .. _pure: https://toolz.readthedocs.io/en/latest/purity.html
 
 
-Tornado Coroutines
-------------------
+Async/await Operation
+---------------------
 
 If we are operating in an asynchronous environment then the blocking functions
 listed above become asynchronous equivalents.  You must start your client
@@ -159,11 +159,10 @@ functions.
 
 .. code-block:: python
 
-   @gen.coroutine
-   def f():
-       client = yield Client(asynchronous=True)
+   async def f():
+       client = await Client(asynchronous=True)
        future = client.submit(func, *args)
-       result = yield future
+       result = await future
        return result
 
 If you want to reuse the same client in asynchronous and synchronous
@@ -174,10 +173,9 @@ call.
 
    client = Client()  # normal blocking client
 
-   @gen.coroutine
-   def f():
+   async def f():
        futures = client.map(func, L)
-       results = yield client.gather(futures, asynchronous=True)
+       results = await client.gather(futures, asynchronous=True)
        return results
 
 See the :doc:`Asynchronous <asynchronous>` documentation for more information.
diff --git a/docs/source/foundations.rst b/docs/source/foundations.rst
index 7b351f8f972..62253433763 100644
--- a/docs/source/foundations.rst
+++ b/docs/source/foundations.rst
@@ -86,25 +86,23 @@ Server Side
 
 .. code-block:: python
 
-   from tornado import gen
-   from tornado.ioloop import IOLoop
+   import asyncio
    from distributed.core import Server
 
    def add(comm, x=None, y=None):  # simple handler, just a function
        return x + y
 
-   @gen.coroutine
-   def stream_data(comm, interval=1):  # complex handler, multiple responses
+   async def stream_data(comm, interval=1):  # complex handler, multiple responses
        data = 0
        while True:
-           yield gen.sleep(interval)
+           await asyncio.sleep(interval)
            data += 1
-           yield comm.write(data)
+           await comm.write(data)
 
    s = Server({'add': add, 'stream_data': stream_data})
    s.listen('tcp://:8888')   # listen on TCP port 8888
 
-   IOLoop.current().start()
+   asyncio.get_event_loop().run_forever()
 
 
 Client Side
@@ -112,30 +110,27 @@ Client Side
 
 .. code-block:: python
 
-   from tornado import gen
-   from tornado.ioloop import IOLoop
+   import asyncio
    from distributed.core import connect
 
-   @gen.coroutine
-   def f():
-       comm = yield connect('tcp://127.0.0.1:8888')
-       yield comm.write({'op': 'add', 'x': 1, 'y': 2})
-       result = yield comm.read()
-       yield comm.close()
+   async def f():
+       comm = await connect('tcp://127.0.0.1:8888')
+       await comm.write({'op': 'add', 'x': 1, 'y': 2})
+       result = await comm.read()
+       await comm.close()
        print(result)
 
-   >>> IOLoop().run_sync(f)
+   >>> asyncio.get_event_loop().run_until_complete(g())
    3
 
-   @gen.coroutine
-   def g():
-       comm = yield connect('tcp://127.0.0.1:8888')
-       yield comm.write({'op': 'stream_data', 'interval': 1})
+   async def g():
+       comm = await connect('tcp://127.0.0.1:8888')
+       await comm.write({'op': 'stream_data', 'interval': 1})
        while True:
-           result = yield comm.read()
+           result = await comm.read()
            print(result)
 
-   >>> IOLoop().run_sync(g)
+   >>> asyncio.get_event_loop().run_until_complete(g())
    1
    2
    3
@@ -152,21 +147,17 @@ with the stream data case above.
 
 .. code-block:: python
 
-   from tornado import gen
-   from tornado.ioloop import IOLoop
+   import asyncio
    from distributed.core import rpc
 
-   @gen.coroutine
-   def f():
-       # comm = yield connect('tcp://127.0.0.1', 8888)
-       # yield comm.write({'op': 'add', 'x': 1, 'y': 2})
-       # result = yield comm.read()
-       r = rpc('tcp://127.0.0.1:8888')
-       result = yield r.add(x=1, y=2)
-       r.close_comms()
+   async def f():
+       # comm = await connect('tcp://127.0.0.1', 8888)
+       # await comm.write({'op': 'add', 'x': 1, 'y': 2})
+       # result = await comm.read()
+       with rpc('tcp://127.0.0.1:8888') as r:
+           result = await r.add(x=1, y=2)
 
        print(result)
 
-   >>> IOLoop().run_sync(f)
+   >>> asyncio.get_event_loop().run_until_complete(f())
    3
-

From 0c861360099fae9d352baa92538905b785d70a04 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 24 Jul 2019 15:11:52 -0700
Subject: [PATCH 0368/1550] Forcefully kill all processes before each test
 (#2882)

This should hopefully help with some intermittent testing failures
---
 distributed/tests/test_client.py |  2 +-
 distributed/utils_test.py        | 11 ++++++-----
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 50415971e20..bf05875fdee 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -3312,7 +3312,7 @@ def test_get_foo_lost_keys(c, s, u, v, w):
     client=True,
     Worker=Nanny,
     worker_kwargs={"death_timeout": "500ms"},
-    clean_kwargs={"processes": False, "threads": False},
+    clean_kwargs={"threads": False},
 )
 def test_bad_tasks_fail(c, s, a, b):
     f = c.submit(sys.exit, 0)
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index d2932857c40..51f1e907236 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -1456,12 +1456,13 @@ def check_thread_leak():
 
 @contextmanager
 def check_process_leak():
-    start_children = set(mp_context.active_children())
+    for proc in mp_context.active_children():
+        proc.terminate()
 
     yield
 
-    for i in range(50):
-        if not set(mp_context.active_children()) - start_children:
+    for i in range(100):
+        if not set(mp_context.active_children()):
             break
         else:
             sleep(0.2)
@@ -1524,14 +1525,14 @@ def check_instances():
 
 
 @contextmanager
-def clean(threads=not WINDOWS, processes=True, instances=True, timeout=1):
+def clean(threads=not WINDOWS, instances=True, timeout=1):
     @contextmanager
     def null():
         yield
 
     with check_thread_leak() if threads else null():
         with pristine_loop() as loop:
-            with check_process_leak() if processes else null():
+            with check_process_leak():
                 with check_instances() if instances else null():
                     with check_active_rpc(loop, timeout):
                         reset_config()

From 909a943b67b6b472a2d77afa13a8caa61f25f972 Mon Sep 17 00:00:00 2001
From: Jim Crist <jcrist@users.noreply.github.com>
Date: Thu, 25 Jul 2019 11:11:35 -0500
Subject: [PATCH 0369/1550] Cleanup Security class and configuration (#2873)

The previous implementation's constructor was a bit confusing, and
wasn't loading from the configuration file properly. This PR:

- Simplifies the class implementation
- Ensures all fields are set to the appropriate defaults in the config
  files (previously the defaults were in the code, not in the config).
- Documents the security class
- Updates tests to ensure parameters are loaded from the appropriate
  configuration fields.

* Add Security to API docs
---
 distributed/cli/dask_scheduler.py  |  10 +-
 distributed/cli/dask_worker.py     |  10 +-
 distributed/distributed.yaml       |  31 ++---
 distributed/security.py            | 177 ++++++++++++++---------------
 distributed/tests/test_security.py | 131 ++++++++++-----------
 distributed/utils_test.py          |  23 ++--
 docs/source/tls.rst                |   9 ++
 7 files changed, 206 insertions(+), 185 deletions(-)

diff --git a/distributed/cli/dask_scheduler.py b/distributed/cli/dask_scheduler.py
index d41b98eb310..2e6220b7d81 100755
--- a/distributed/cli/dask_scheduler.py
+++ b/distributed/cli/dask_scheduler.py
@@ -170,7 +170,15 @@ def main(
         port = 8786
 
     sec = Security(
-        tls_ca_file=tls_ca_file, tls_scheduler_cert=tls_cert, tls_scheduler_key=tls_key
+        **{
+            k: v
+            for k, v in [
+                ("tls_ca_file", tls_ca_file),
+                ("tls_scheduler_cert", tls_cert),
+                ("tls_scheduler_key", tls_key),
+            ]
+            if v is not None
+        }
     )
 
     if not host and (tls_ca_file or tls_cert or tls_key):
diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
index 8752cd52448..073c7c9c922 100755
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -238,7 +238,15 @@ def main(
         dashboard = bokeh
 
     sec = Security(
-        tls_ca_file=tls_ca_file, tls_worker_cert=tls_cert, tls_worker_key=tls_key
+        **{
+            k: v
+            for k, v in [
+                ("tls_ca_file", tls_ca_file),
+                ("tls_worker_cert", tls_cert),
+                ("tls_worker_key", tls_key),
+            ]
+            if v is not None
+        }
     )
 
     if nprocs > 1 and worker_port != 0:
diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index c6c3e3d1ba2..c3f14f114f1 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -72,21 +72,22 @@ distributed:
       connect: 10s          # time before connecting fails
       tcp: 30s              # time before calling an unresponsive connection dead
 
-  # require-encryption: False   # whether to require encryption on non-local comms
-  #
-  # tls:
-  #   ca-file: xxx.pem
-  #   scheduler:
-  #     key: xxx.pem
-  #     cert: xxx.pem
-  #   worker:
-  #     key: xxx.pem
-  #     cert: xxx.pem
-  #   client:
-  #     key: xxx.pem
-  #     cert: xxx.pem
-  #   ciphers:
-  #     ECDHE-ECDSA-AES128-GCM-SHA256
+    require-encryption: False   # Whether to require encryption on non-local comms
+
+    tls:
+      ciphers: null   # Allowed ciphers, specified as an OpenSSL cipher string.
+      ca-file: null   # Path to a CA file, in pem format, optional
+      scheduler:
+        cert: null    # Path to certificate file for scheduler.
+        key: null     # Path to key file for scheduler. Alternatively, the key
+                      # can be appended to the cert file above, and this field
+                      # left blank.
+      worker:
+        key: null
+        cert: null
+      client:
+        key: null
+        cert: null
 
 
   ###################
diff --git a/distributed/security.py b/distributed/security.py
index e86c0602860..a42cbeef646 100644
--- a/distributed/security.py
+++ b/distributed/security.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 try:
     import ssl
 except ImportError:
@@ -8,80 +6,85 @@
 import dask
 
 
-_roles = ["client", "scheduler", "worker"]
-
-_tls_per_role_fields = ["key", "cert"]
-
-_tls_fields = ["ca_file", "ciphers"]
-
-_misc_fields = ["require_encryption"]
-
-_fields = set(
-    _misc_fields
-    + ["tls_%s" % field for field in _tls_fields]
-    + ["tls_%s_%s" % (role, field) for role in _roles for field in _tls_per_role_fields]
-)
-
-
-def _field_to_config_key(field):
-    return field.replace("_", "-")
+__all__ = ("Security",)
 
 
 class Security(object):
-    """
-    An object to gather and pass around security configuration.
-    Default values are gathered from the global ``config`` object and
-    can be overriden by constructor args.
-
-    Supported fields:
-        - require_encryption
-        - tls_ca_file
-        - tls_ciphers
-        - tls_client_key
-        - tls_client_cert
-        - tls_scheduler_key
-        - tls_scheduler_cert
-        - tls_worker_key
-        - tls_worker_cert
+    """Security configuration for a Dask cluster.
+
+    Default values are loaded from Dask's configuration files, and can be
+    overridden in the constructor.
+
+    Parameters
+    ----------
+    require_encryption : bool, optional
+        Whether TLS encryption is required for all connections.
+    tls_ca_file : str, optional
+        Path to a CA certificate file encoded in PEM format.
+    tls_ciphers : str, optional
+        An OpenSSL cipher string of allowed ciphers. If not provided, the
+        system defaults will be used.
+    tls_client_cert : str, optional
+        Path to a certificate file for the client, encoded in PEM format.
+    tls_client_key : str, optional
+        Path to a key file for the client, encoded in PEM format.
+        Alternatively, the key may be appended to the cert file, and this
+        parameter be omitted.
+    tls_scheduler_cert : str, optional
+        Path to a certificate file for the scheduler, encoded in PEM format.
+    tls_scheduler_key : str, optional
+        Path to a key file for the scheduler, encoded in PEM format.
+        Alternatively, the key may be appended to the cert file, and this
+        parameter be omitted.
+    tls_worker_cert : str, optional
+        Path to a certificate file for a worker, encoded in PEM format.
+    tls_worker_key : str, optional
+        Path to a key file for a worker, encoded in PEM format.
+        Alternatively, the key may be appended to the cert file, and this
+        parameter be omitted.
     """
 
-    __slots__ = tuple(_fields)
+    __slots__ = (
+        "require_encryption",
+        "tls_ca_file",
+        "tls_ciphers",
+        "tls_client_key",
+        "tls_client_cert",
+        "tls_scheduler_key",
+        "tls_scheduler_cert",
+        "tls_worker_key",
+        "tls_worker_cert",
+    )
 
     def __init__(self, **kwargs):
-        self._init_from_dict(dask.config.config)
-        for k, v in kwargs.items():
-            if v is not None:
-                setattr(self, k, v)
-        for k in _fields:
-            if not hasattr(self, k):
-                setattr(self, k, None)
-
-    def _init_from_dict(self, d):
-        """
-        Initialize Security from nested dict.
-        """
-        self._init_fields_from_dict(d, "", _misc_fields, {})
-        self._init_fields_from_dict(d, "tls", _tls_fields, _tls_per_role_fields)
+        extra = set(kwargs).difference(self.__slots__)
+        if extra:
+            raise TypeError("Unknown parameters: %r" % sorted(extra))
+        self._set_field(
+            kwargs, "require_encryption", "distributed.comm.require-encryption"
+        )
+        self._set_field(kwargs, "tls_ciphers", "distributed.comm.tls.ciphers")
+        self._set_field(kwargs, "tls_ca_file", "distributed.comm.tls.ca-file")
+        self._set_field(kwargs, "tls_client_key", "distributed.comm.tls.client.key")
+        self._set_field(kwargs, "tls_client_cert", "distributed.comm.tls.client.cert")
+        self._set_field(
+            kwargs, "tls_scheduler_key", "distributed.comm.tls.scheduler.key"
+        )
+        self._set_field(
+            kwargs, "tls_scheduler_cert", "distributed.comm.tls.scheduler.cert"
+        )
+        self._set_field(kwargs, "tls_worker_key", "distributed.comm.tls.worker.key")
+        self._set_field(kwargs, "tls_worker_cert", "distributed.comm.tls.worker.cert")
 
-    def _init_fields_from_dict(self, d, category, fields, per_role_fields):
-        if category:
-            d = d.get(category, {})
-            category_prefix = category + "_"
+    def _set_field(self, kwargs, field, config_name):
+        if field in kwargs:
+            out = kwargs[field]
         else:
-            category_prefix = ""
-        for field in fields:
-            k = _field_to_config_key(field)
-            if k in d:
-                setattr(self, "%s%s" % (category_prefix, field), d[k])
-        for role in _roles:
-            dd = d.get(role, {})
-            for field in per_role_fields:
-                k = _field_to_config_key(field)
-                if k in dd:
-                    setattr(self, "%s%s_%s" % (category_prefix, role, field), dd[k])
+            out = dask.config.get(config_name)
+        setattr(self, field, out)
 
     def __repr__(self):
-        items = sorted((k, getattr(self, k)) for k in _fields)
+        items = sorted((k, getattr(self, k)) for k in self.__slots__)
         return (
             "Security("
             + ", ".join("%s=%r" % (k, v) for k, v in items if v is not None)
@@ -92,26 +95,18 @@ def get_tls_config_for_role(self, role):
         """
         Return the TLS configuration for the given role, as a flat dict.
         """
-        return self._get_config_for_role("tls", role, _tls_fields, _tls_per_role_fields)
-
-    def _get_config_for_role(self, category, role, fields, per_role_fields):
-        if role not in _roles:
+        if role not in {"client", "scheduler", "worker"}:
             raise ValueError("unknown role %r" % (role,))
-        d = {}
-        for field in fields:
-            k = "%s_%s" % (category, field)
-            d[field] = getattr(self, k)
-        for field in per_role_fields:
-            k = "%s_%s_%s" % (category, role, field)
-            d[field] = getattr(self, k)
-        return d
+        return {
+            "ca_file": self.tls_ca_file,
+            "ciphers": self.tls_ciphers,
+            "cert": getattr(self, "tls_%s_cert" % role),
+            "key": getattr(self, "tls_%s_key" % role),
+        }
 
     def _get_tls_context(self, tls, purpose):
         if tls.get("ca_file") and tls.get("cert"):
-            try:
-                ctx = ssl.create_default_context(purpose=purpose, cafile=tls["ca_file"])
-            except AttributeError:
-                raise RuntimeError("TLS functionality requires Python 2.7.9+")
+            ctx = ssl.create_default_context(purpose=purpose, cafile=tls["ca_file"])
             ctx.verify_mode = ssl.CERT_REQUIRED
             # We expect a dedicated CA for the cluster and people using
             # IP addresses rather than hostnames
@@ -126,23 +121,19 @@ def get_connection_args(self, role):
         Get the *connection_args* argument for a connect() call with
         the given *role*.
         """
-        d = {}
         tls = self.get_tls_config_for_role(role)
-        # Ensure backwards compatibility (ssl.Purpose is Python 2.7.9+ only)
-        purpose = ssl.Purpose.SERVER_AUTH if hasattr(ssl, "Purpose") else None
-        d["ssl_context"] = self._get_tls_context(tls, purpose)
-        d["require_encryption"] = self.require_encryption
-        return d
+        return {
+            "ssl_context": self._get_tls_context(tls, ssl.Purpose.SERVER_AUTH),
+            "require_encryption": self.require_encryption,
+        }
 
     def get_listen_args(self, role):
         """
         Get the *connection_args* argument for a listen() call with
         the given *role*.
         """
-        d = {}
         tls = self.get_tls_config_for_role(role)
-        # Ensure backwards compatibility (ssl.Purpose is Python 2.7.9+ only)
-        purpose = ssl.Purpose.CLIENT_AUTH if hasattr(ssl, "Purpose") else None
-        d["ssl_context"] = self._get_tls_context(tls, purpose)
-        d["require_encryption"] = self.require_encryption
-        return d
+        return {
+            "ssl_context": self._get_tls_context(tls, ssl.Purpose.CLIENT_AUTH),
+            "require_encryption": self.require_encryption,
+        }
diff --git a/distributed/tests/test_security.py b/distributed/tests/test_security.py
index 8e82db1308e..7f144625b04 100644
--- a/distributed/tests/test_security.py
+++ b/distributed/tests/test_security.py
@@ -13,8 +13,9 @@
 
 from distributed.comm import connect, listen
 from distributed.security import Security
-from distributed.utils_test import new_config, get_cert, gen_test
+from distributed.utils_test import get_cert, gen_test
 
+import dask
 
 ca_file = get_cert("tls-ca-cert.pem")
 
@@ -35,8 +36,7 @@
 
 
 def test_defaults():
-    with new_config({}):
-        sec = Security()
+    sec = Security()
     assert sec.require_encryption in (None, False)
     assert sec.tls_ca_file is None
     assert sec.tls_ciphers is None
@@ -48,6 +48,13 @@ def test_defaults():
     assert sec.tls_worker_cert is None
 
 
+def test_constructor_errors():
+    with pytest.raises(TypeError) as exc:
+        Security(unknown_keyword="bar")
+
+    assert "unknown_keyword" in str(exc.value)
+
+
 def test_attribute_error():
     sec = Security()
     assert hasattr(sec, "tls_ca_file")
@@ -59,16 +66,17 @@ def test_attribute_error():
 
 def test_from_config():
     c = {
-        "tls": {
-            "ca-file": "ca.pem",
-            "scheduler": {"key": "skey.pem", "cert": "scert.pem"},
-            "worker": {"cert": "wcert.pem"},
-            "ciphers": FORCED_CIPHER,
-        },
-        "require-encryption": True,
+        "distributed.comm.tls.ca-file": "ca.pem",
+        "distributed.comm.tls.scheduler.key": "skey.pem",
+        "distributed.comm.tls.scheduler.cert": "scert.pem",
+        "distributed.comm.tls.worker.cert": "wcert.pem",
+        "distributed.comm.tls.ciphers": FORCED_CIPHER,
+        "distributed.comm.require-encryption": True,
     }
-    with new_config(c):
+
+    with dask.config.set(c):
         sec = Security()
+
     assert sec.require_encryption is True
     assert sec.tls_ca_file == "ca.pem"
     assert sec.tls_ciphers == FORCED_CIPHER
@@ -82,18 +90,16 @@ def test_from_config():
 
 def test_kwargs():
     c = {
-        "tls": {
-            "ca-file": "ca.pem",
-            "scheduler": {"key": "skey.pem", "cert": "scert.pem"},
-        }
+        "distributed.comm.tls.ca-file": "ca.pem",
+        "distributed.comm.tls.scheduler.key": "skey.pem",
+        "distributed.comm.tls.scheduler.cert": "scert.pem",
     }
-    with new_config(c):
+    with dask.config.set(c):
         sec = Security(
             tls_scheduler_cert="newcert.pem", require_encryption=True, tls_ca_file=None
         )
     assert sec.require_encryption is True
-    # None value didn't override default
-    assert sec.tls_ca_file == "ca.pem"
+    assert sec.tls_ca_file is None
     assert sec.tls_ciphers is None
     assert sec.tls_client_key is None
     assert sec.tls_client_cert is None
@@ -104,24 +110,22 @@ def test_kwargs():
 
 
 def test_repr():
-    with new_config({}):
-        sec = Security(tls_ca_file="ca.pem", tls_scheduler_cert="scert.pem")
-        assert (
-            repr(sec)
-            == "Security(tls_ca_file='ca.pem', tls_scheduler_cert='scert.pem')"
-        )
+    sec = Security(tls_ca_file="ca.pem", tls_scheduler_cert="scert.pem")
+    assert (
+        repr(sec)
+        == "Security(require_encryption=False, tls_ca_file='ca.pem', tls_scheduler_cert='scert.pem')"
+    )
 
 
 def test_tls_config_for_role():
     c = {
-        "tls": {
-            "ca-file": "ca.pem",
-            "scheduler": {"key": "skey.pem", "cert": "scert.pem"},
-            "worker": {"cert": "wcert.pem"},
-            "ciphers": FORCED_CIPHER,
-        }
+        "distributed.comm.tls.ca-file": "ca.pem",
+        "distributed.comm.tls.scheduler.key": "skey.pem",
+        "distributed.comm.tls.scheduler.cert": "scert.pem",
+        "distributed.comm.tls.worker.cert": "wcert.pem",
+        "distributed.comm.tls.ciphers": FORCED_CIPHER,
     }
-    with new_config(c):
+    with dask.config.set(c):
         sec = Security()
     t = sec.get_tls_config_for_role("scheduler")
     assert t == {
@@ -158,13 +162,12 @@ def many_ciphers(ctx):
             assert len(ctx.get_ciphers()) > 2  # Most likely
 
     c = {
-        "tls": {
-            "ca-file": ca_file,
-            "scheduler": {"key": key1, "cert": cert1},
-            "worker": {"cert": keycert1},
-        }
+        "distributed.comm.tls.ca-file": ca_file,
+        "distributed.comm.tls.scheduler.key": key1,
+        "distributed.comm.tls.scheduler.cert": cert1,
+        "distributed.comm.tls.worker.cert": keycert1,
     }
-    with new_config(c):
+    with dask.config.set(c):
         sec = Security()
 
     d = sec.get_connection_args("scheduler")
@@ -183,10 +186,10 @@ def many_ciphers(ctx):
     assert d.get("ssl_context") is None
 
     # With more settings
-    c["tls"]["ciphers"] = FORCED_CIPHER
-    c["require-encryption"] = True
+    c["distributed.comm.tls.ciphers"] = FORCED_CIPHER
+    c["distributed.comm.require-encryption"] = True
 
-    with new_config(c):
+    with dask.config.set(c):
         sec = Security()
 
     d = sec.get_listen_args("scheduler")
@@ -212,13 +215,12 @@ def many_ciphers(ctx):
             assert len(ctx.get_ciphers()) > 2  # Most likely
 
     c = {
-        "tls": {
-            "ca-file": ca_file,
-            "scheduler": {"key": key1, "cert": cert1},
-            "worker": {"cert": keycert1},
-        }
+        "distributed.comm.tls.ca-file": ca_file,
+        "distributed.comm.tls.scheduler.key": key1,
+        "distributed.comm.tls.scheduler.cert": cert1,
+        "distributed.comm.tls.worker.cert": keycert1,
     }
-    with new_config(c):
+    with dask.config.set(c):
         sec = Security()
 
     d = sec.get_listen_args("scheduler")
@@ -237,10 +239,10 @@ def many_ciphers(ctx):
     assert d.get("ssl_context") is None
 
     # With more settings
-    c["tls"]["ciphers"] = FORCED_CIPHER
-    c["require-encryption"] = True
+    c["distributed.comm.tls.ciphers"] = FORCED_CIPHER
+    c["distributed.comm.require-encryption"] = True
 
-    with new_config(c):
+    with dask.config.set(c):
         sec = Security()
 
     d = sec.get_listen_args("scheduler")
@@ -270,17 +272,16 @@ def handle_comm(comm):
         yield comm.close()
 
     c = {
-        "tls": {
-            "ca-file": ca_file,
-            "scheduler": {"key": key1, "cert": cert1},
-            "worker": {"cert": keycert1},
-        }
+        "distributed.comm.tls.ca-file": ca_file,
+        "distributed.comm.tls.scheduler.key": key1,
+        "distributed.comm.tls.scheduler.cert": cert1,
+        "distributed.comm.tls.worker.cert": keycert1,
     }
-    with new_config(c):
+    with dask.config.set(c):
         sec = Security()
 
-    c["tls"]["ciphers"] = FORCED_CIPHER
-    with new_config(c):
+    c["distributed.comm.tls.ciphers"] = FORCED_CIPHER
+    with dask.config.set(c):
         forced_cipher_sec = Security()
 
     with listen(
@@ -321,16 +322,16 @@ def handle_comm(comm):
         comm.abort()
 
     c = {
-        "tls": {
-            "ca-file": ca_file,
-            "scheduler": {"key": key1, "cert": cert1},
-            "worker": {"cert": keycert1},
-        }
+        "distributed.comm.tls.ca-file": ca_file,
+        "distributed.comm.tls.scheduler.key": key1,
+        "distributed.comm.tls.scheduler.cert": cert1,
+        "distributed.comm.tls.worker.cert": keycert1,
     }
-    with new_config(c):
+    with dask.config.set(c):
         sec = Security()
-    c["require-encryption"] = True
-    with new_config(c):
+
+    c["distributed.comm.require-encryption"] = True
+    with dask.config.set(c):
         sec2 = Security()
 
     for listen_addr in ["inproc://", "tls://"]:
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index 51f1e907236..8055aec3f34 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -1265,10 +1265,10 @@ def new_config(new_config):
     from .config import defaults
 
     config = dask.config.config
-    orig_config = config.copy()
+    orig_config = copy.deepcopy(config)
     try:
         config.clear()
-        config.update(defaults.copy())
+        config.update(copy.deepcopy(defaults))
         dask.config.update(config, new_config)
         initialize_logging(config)
         yield
@@ -1332,15 +1332,18 @@ def tls_config():
     ca_file = get_cert("tls-ca-cert.pem")
     keycert = get_cert("tls-key-cert.pem")
 
-    c = {
-        "tls": {
-            "ca-file": ca_file,
-            "client": {"cert": keycert},
-            "scheduler": {"cert": keycert},
-            "worker": {"cert": keycert},
+    return {
+        "distributed": {
+            "comm": {
+                "tls": {
+                    "ca-file": ca_file,
+                    "client": {"cert": keycert},
+                    "scheduler": {"cert": keycert},
+                    "worker": {"cert": keycert},
+                }
+            }
         }
     }
-    return c
 
 
 def tls_only_config():
@@ -1349,7 +1352,7 @@ def tls_only_config():
     plain TCP communications.
     """
     c = tls_config()
-    c["require-encryption"] = True
+    c["distributed"]["comm"]["require-encryption"] = True
     return c
 
 
diff --git a/docs/source/tls.rst b/docs/source/tls.rst
index d367dabbf7b..0c635b85761 100644
--- a/docs/source/tls.rst
+++ b/docs/source/tls.rst
@@ -96,3 +96,12 @@ very large data over very high speed network links.
    `A study of AES-NI acceleration <https://calomel.org/aesni_ssl_performance.html>`_
    shows recent x86 CPUs can AES-encrypt more than 1 GB per second
    on each CPU core.
+
+
+API
+---
+
+.. currentmodule:: distributed
+
+.. autoclass:: distributed.security.Security
+   :members:

From 2cfaca1eeff3682239b9869daaa829ff0e002f0e Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Fri, 26 Jul 2019 16:56:30 +0100
Subject: [PATCH 0370/1550] Remove unused variable in SpecCluster scale down
 (#2870)

---
 distributed/deploy/spec.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index 1ba8e7fb213..912ccc79302 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -318,8 +318,6 @@ def new_worker_spec(self):
     async def scale_down(self, workers):
         workers = set(workers)
 
-        # TODO: this is linear cost.  We should be indexing by name or something
-        to_close = [w for w in self.workers.values() if w.address in workers]
         for k, v in self.workers.items():
             if v.worker_address in workers:
                 del self.worker_spec[k]

From 208e0bc2d313809b41e0998b75b0996cb39390b2 Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Fri, 26 Jul 2019 16:57:09 +0100
Subject: [PATCH 0371/1550] Add ProcessInterface (#2874)

Added ProcessInterface class which is an interface for custom schedulers and workers to inherit from for use in SpecCluster.
---
 distributed/deploy/__init__.py                |  2 +-
 distributed/deploy/spec.py                    | 40 ++++++++++++++++++-
 distributed/deploy/ssh2.py                    | 28 +++++--------
 distributed/deploy/tests/test_spec_cluster.py | 12 +++++-
 4 files changed, 62 insertions(+), 20 deletions(-)

diff --git a/distributed/deploy/__init__.py b/distributed/deploy/__init__.py
index 9b5e478c303..24a86e6d6d2 100644
--- a/distributed/deploy/__init__.py
+++ b/distributed/deploy/__init__.py
@@ -4,7 +4,7 @@
 
 from .cluster import Cluster
 from .local import LocalCluster
-from .spec import SpecCluster
+from .spec import SpecCluster, ProcessInterface
 from .adaptive import Adaptive
 
 with ignoring(ImportError):
diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index 912ccc79302..a6701807fa5 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -11,6 +11,44 @@
 from ..security import Security
 
 
+class ProcessInterface:
+    """ An interface for Scheduler and Worker processes for use in SpecCluster
+
+    Parameters
+    ----------
+    loop:
+        A pointer to the running loop.
+
+    """
+
+    def __init__(self, loop=None):
+        self.address = None
+        self.loop = loop
+        self.lock = asyncio.Lock()
+        self.status = "created"
+
+    def __await__(self):
+        async def _():
+            async with self.lock:
+                if self.status == "created":
+                    await self.start()
+                    assert self.status == "running"
+            return self
+
+        return _().__await__()
+
+    async def start(self):
+        """ Start the process. """
+        self.status = "running"
+
+    async def close(self):
+        """ Close the process. """
+        self.status = "closed"
+
+    def __repr__(self):
+        return "<%s: status=%s>" % (type(self).__name__, self.status)
+
+
 class SpecCluster(Cluster):
     """ Cluster that requires a full specification of workers
 
@@ -319,7 +357,7 @@ async def scale_down(self, workers):
         workers = set(workers)
 
         for k, v in self.workers.items():
-            if v.worker_address in workers:
+            if getattr(v, "worker_address", v.address) in workers:
                 del self.worker_spec[k]
 
         await self
diff --git a/distributed/deploy/ssh2.py b/distributed/deploy/ssh2.py
index 0f8823cdab8..bf471f1dee6 100644
--- a/distributed/deploy/ssh2.py
+++ b/distributed/deploy/ssh2.py
@@ -1,4 +1,3 @@
-import asyncio
 import logging
 import sys
 import warnings
@@ -6,7 +5,7 @@
 
 import asyncssh
 
-from .spec import SpecCluster
+from .spec import SpecCluster, ProcessInterface
 
 logger = logging.getLogger(__name__)
 
@@ -16,7 +15,7 @@
 )
 
 
-class Process:
+class Process(ProcessInterface):
     """ A superclass for SSH Workers and Nannies
 
     See Also
@@ -25,27 +24,20 @@ class Process:
     Scheduler
     """
 
-    def __init__(self):
-        self.lock = asyncio.Lock()
+    def __init__(self, **kwargs):
         self.connection = None
         self.proc = None
-        self.status = "created"
+        super().__init__(**kwargs)
 
-    def __await__(self):
-        async def _():
-            async with self.lock:
-                if not self.connection:
-                    await self.start()
-                    assert self.connection
-                    weakref.finalize(self, self.proc.terminate)
-            return self
-
-        return _().__await__()
+    async def start(self):
+        assert self.connection
+        weakref.finalize(self, self.proc.terminate)
+        await super().start()
 
     async def close(self):
         self.proc.terminate()
         self.connection.close()
-        self.status = "closed"
+        await super().close()
 
     def __repr__(self):
         return "<SSH %s: status=%s>" % (type(self).__name__, self.status)
@@ -97,6 +89,7 @@ async def start(self):
                 self.status = "running"
                 break
         logger.debug("%s", line)
+        await super().start()
 
 
 class Scheduler(Process):
@@ -135,6 +128,7 @@ async def start(self):
                 self.address = line.split("Scheduler at:")[1].strip()
                 break
         logger.debug("%s", line)
+        await super().start()
 
 
 def SSHCluster(hosts, connect_kwargs, **kwargs):
diff --git a/distributed/deploy/tests/test_spec_cluster.py b/distributed/deploy/tests/test_spec_cluster.py
index bb992f8b7c7..bef5a2e554c 100644
--- a/distributed/deploy/tests/test_spec_cluster.py
+++ b/distributed/deploy/tests/test_spec_cluster.py
@@ -1,5 +1,5 @@
 from dask.distributed import SpecCluster, Worker, Client, Scheduler, Nanny
-from distributed.deploy.spec import close_clusters
+from distributed.deploy.spec import close_clusters, ProcessInterface
 from distributed.utils_test import loop, cleanup  # noqa: F401
 import pytest
 
@@ -164,3 +164,13 @@ async def test_nanny_port():
         scheduler=scheduler, workers=workers, asynchronous=True
     ) as cluster:
         pass
+
+
+@pytest.mark.asyncio
+async def test_spec_process():
+    proc = ProcessInterface()
+    assert proc.status == "created"
+    await proc
+    assert proc.status == "running"
+    await proc.close()
+    assert proc.status == "closed"

From 594589e7dd7112c557661a4cb890440cceaacb9c Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Fri, 26 Jul 2019 19:28:27 +0100
Subject: [PATCH 0372/1550] Add Log(str) and Logs(dict) classes for nice HTML
 reprs (#2875)

---
 distributed/tests/test_utils.py | 12 ++++++++++++
 distributed/utils.py            | 20 ++++++++++++++++++++
 2 files changed, 32 insertions(+)

diff --git a/distributed/tests/test_utils.py b/distributed/tests/test_utils.py
index df98bbe59e1..9e3a1d90c4b 100644
--- a/distributed/tests/test_utils.py
+++ b/distributed/tests/test_utils.py
@@ -7,6 +7,7 @@
 import sys
 from time import sleep
 import traceback
+import xml.etree.ElementTree
 
 import numpy as np
 import pytest
@@ -18,6 +19,8 @@
 from distributed.metrics import time
 from distributed.utils import (
     All,
+    Log,
+    Logs,
     sync,
     is_kernel,
     ensure_ip,
@@ -548,3 +551,12 @@ def test_warn_on_duration():
 def test_format_bytes_compat():
     # moved to dask, but exported here for compatibility
     from distributed.utils import format_bytes  # noqa
+
+
+def test_logs():
+    d = Logs({"123": Log("Hello"), "456": Log("World!")})
+    text = d._repr_html_()
+    for line in text.split("\n"):
+        assert xml.etree.ElementTree.fromstring(line) is not None
+    assert "Hello" in text
+    assert "456" in text
diff --git a/distributed/utils.py b/distributed/utils.py
index 6e8769979fb..227406da76b 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -1490,3 +1490,23 @@ def format_dashboard_link(host, port):
 
 def is_coroutine_function(f):
     return asyncio.iscoroutinefunction(f) or gen.is_coroutine_function(f)
+
+
+class Log(str):
+    """ A container for logs """
+
+    def _repr_html_(self):
+        return "<pre><code>{log}</code></pre>".format(log=self)
+
+
+class Logs(dict):
+    """ A container for multiple logs """
+
+    def _repr_html_(self):
+        summaries = [
+            "<details><summary>{title}</summary>{log}</details>".format(
+                title=title, log=log._repr_html_()
+            )
+            for title, log in self.items()
+        ]
+        return "\n".join(summaries)

From 65001f2d1c796f652bf77bbf222cf900bf1062b8 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 26 Jul 2019 12:41:25 -0700
Subject: [PATCH 0373/1550] Pass Client._asynchronous to Cluster._asynchronous
 (#2890)

Previously when starting a client/cluster with `Client()` the
underlying cluster would always be started with `asynchronous=True`.
This could be troublesome in some cases.

Now we pass through the `asynchronous=` value that was originally passed
to the Client object.
---
 distributed/client.py                  | 4 +++-
 distributed/deploy/tests/test_local.py | 7 +++++++
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/distributed/client.py b/distributed/client.py
index c4b2f51426a..a830b147fb8 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -932,7 +932,9 @@ async def _start(self, timeout=no_default, **kwargs):
 
             try:
                 self.cluster = await LocalCluster(
-                    loop=self.loop, asynchronous=True, **self._startup_kwargs
+                    loop=self.loop,
+                    asynchronous=self._asynchronous,
+                    **self._startup_kwargs
                 )
             except (OSError, socket.error) as e:
                 if e.errno != errno.EADDRINUSE:
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index 1c098c2b4c5..91a792272e8 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -856,3 +856,10 @@ def test_dont_select_closed_worker():
 
         cluster2.close()
         c2.close()
+
+
+def test_client_cluster_synchronous(loop):
+    with clean(threads=False):
+        with Client(loop=loop, processes=False) as c:
+            assert not c.asynchronous
+            assert not c.cluster.asynchronous

From 3e50887d3d24fda7ec08364deb14090e72b4d484 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 26 Jul 2019 14:23:57 -0700
Subject: [PATCH 0374/1550] Add default logs method to Spec Cluster (#2889)

This gathers logs through the scheduler using existing methods,
and then returns them as nicely rendered summary/details outputs.
---
 distributed/deploy/spec.py                    | 35 ++++++++++++++++++-
 distributed/deploy/tests/test_spec_cluster.py | 31 ++++++++++++++++
 distributed/scheduler.py                      |  2 +-
 3 files changed, 66 insertions(+), 2 deletions(-)

diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index a6701807fa5..747ecc19a41 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -6,7 +6,7 @@
 
 from .cluster import Cluster
 from ..core import rpc, CommClosedError
-from ..utils import LoopRunner, silence_logging, ignoring
+from ..utils import LoopRunner, silence_logging, ignoring, Log, Logs
 from ..scheduler import Scheduler
 from ..security import Security
 
@@ -371,6 +371,39 @@ def __repr__(self):
             len(self.workers),
         )
 
+    async def _logs(self, scheduler=True, workers=True):
+        logs = Logs()
+
+        if scheduler:
+            L = await self.scheduler_comm.logs()
+            logs["Scheduler"] = Log("\n".join(line for level, line in L))
+
+        if workers:
+            d = await self.scheduler_comm.worker_logs(workers=workers)
+            for k, v in d.items():
+                logs[k] = Log("\n".join(line for level, line in v))
+
+        return logs
+
+    def logs(self, scheduler=True, workers=True):
+        """ Return logs for the scheduler and workers
+
+        Parameters
+        ----------
+        scheduler : boolean
+            Whether or not to collect logs for the scheduler
+        workers : boolean or Iterable[str], optional
+            A list of worker addresses to select.
+            Defaults to all workers if `True` or no workers if `False`
+
+        Returns
+        -------
+        logs: Dict[str]
+            A dictionary of logs, with one item for the scheduler and one for
+            each worker
+        """
+        return self.sync(self._logs, scheduler=scheduler, workers=workers)
+
 
 @atexit.register
 def close_clusters():
diff --git a/distributed/deploy/tests/test_spec_cluster.py b/distributed/deploy/tests/test_spec_cluster.py
index bef5a2e554c..51094dca2d5 100644
--- a/distributed/deploy/tests/test_spec_cluster.py
+++ b/distributed/deploy/tests/test_spec_cluster.py
@@ -1,6 +1,7 @@
 from dask.distributed import SpecCluster, Worker, Client, Scheduler, Nanny
 from distributed.deploy.spec import close_clusters, ProcessInterface
 from distributed.utils_test import loop, cleanup  # noqa: F401
+import toolz
 import pytest
 
 
@@ -174,3 +175,33 @@ async def test_spec_process():
     assert proc.status == "running"
     await proc.close()
     assert proc.status == "closed"
+
+
+@pytest.mark.asyncio
+async def test_logs(cleanup):
+    worker = {"cls": Worker, "options": {"nthreads": 1}}
+    async with SpecCluster(
+        asynchronous=True, scheduler=scheduler, worker=worker
+    ) as cluster:
+        cluster.scale(2)
+        await cluster
+
+        logs = await cluster.logs()
+        assert "Scheduler" in logs
+        for worker in cluster.scheduler.workers:
+            assert worker in logs
+
+        assert "Registered" in str(logs)
+
+        logs = await cluster.logs(scheduler=True, workers=False)
+        assert list(logs) == ["Scheduler"]
+
+        logs = await cluster.logs(scheduler=False, workers=False)
+        assert list(logs) == []
+
+        logs = await cluster.logs(scheduler=False, workers=True)
+        assert set(logs) == set(cluster.scheduler.workers)
+
+        w = toolz.first(cluster.scheduler.workers)
+        logs = await cluster.logs(scheduler=False, workers=[w])
+        assert set(logs) == {w}
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index ace4d2483d5..98a66bcc55f 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -2631,7 +2631,7 @@ async def broadcast(
         serializers=None,
     ):
         """ Broadcast message to workers, return all results """
-        if workers is None:
+        if workers is None or workers is True:
             if hosts is None:
                 workers = list(self.workers)
             else:

From c291175a975dfb9376724ececba10fcc9e2e43c0 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 26 Jul 2019 17:15:07 -0700
Subject: [PATCH 0375/1550] Add processes keyword back into clean (#2891)

This resolves some intermittent testing failures
---
 distributed/tests/test_failed_workers.py | 19 +++++++++++--------
 distributed/tests/test_scheduler.py      |  2 +-
 distributed/utils_test.py                |  4 ++--
 3 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/distributed/tests/test_failed_workers.py b/distributed/tests/test_failed_workers.py
index 5465a7dd5f0..1f27e067058 100644
--- a/distributed/tests/test_failed_workers.py
+++ b/distributed/tests/test_failed_workers.py
@@ -266,23 +266,26 @@ def test_fast_kill(c, s, a, b):
 
 @gen_cluster(Worker=Nanny, timeout=60)
 def test_multiple_clients_restart(s, a, b):
-    e1 = yield Client(s.address, asynchronous=True)
-    e2 = yield Client(s.address, asynchronous=True)
+    c1 = yield Client(s.address, asynchronous=True)
+    c2 = yield Client(s.address, asynchronous=True)
 
-    x = e1.submit(inc, 1)
-    y = e2.submit(inc, 2)
+    x = c1.submit(inc, 1)
+    y = c2.submit(inc, 2)
     xx = yield x
     yy = yield y
     assert xx == 2
     assert yy == 3
 
-    yield e1._restart()
+    yield c1.restart()
 
     assert x.cancelled()
-    assert y.cancelled()
+    start = time()
+    while not y.cancelled():
+        yield gen.sleep(0.01)
+        assert time() < start + 5
 
-    yield e1._close(fast=True)
-    yield e2._close(fast=True)
+    yield c1.close()
+    yield c2.close()
 
 
 @gen_cluster(Worker=Nanny, timeout=60)
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 4f1b2808102..300a1a5b2b0 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -1234,7 +1234,7 @@ def test_cancel_fire_and_forget(c, s, a, b):
     assert not s.tasks
 
 
-@gen_cluster(client=True, Worker=Nanny)
+@gen_cluster(client=True, Worker=Nanny, clean_kwargs={"processes": False})
 def test_log_tasks_during_restart(c, s, a, b):
     future = c.submit(sys.exit, 0)
     yield wait(future)
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index 8055aec3f34..52cc54b639d 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -1528,14 +1528,14 @@ def check_instances():
 
 
 @contextmanager
-def clean(threads=not WINDOWS, instances=True, timeout=1):
+def clean(threads=not WINDOWS, instances=True, timeout=1, processes=True):
     @contextmanager
     def null():
         yield
 
     with check_thread_leak() if threads else null():
         with pristine_loop() as loop:
-            with check_process_leak():
+            with check_process_leak() if processes else null():
                 with check_instances() if instances else null():
                     with check_active_rpc(loop, timeout):
                         reset_config()

From 741ffb60b94b15d2f243fc4ad4a849df76c46092 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sun, 28 Jul 2019 09:06:45 -0700
Subject: [PATCH 0376/1550] Update black (#2901)

This helps to resolve `**kwargs,` issues in Python 3.5
---
 .pre-commit-config.yaml                  | 2 +-
 distributed/comm/tests/test_comms.py     | 4 ++--
 distributed/protocol/tests/test_numpy.py | 8 ++++----
 distributed/queues.py                    | 2 +-
 distributed/scheduler.py                 | 2 +-
 distributed/tests/test_scheduler.py      | 5 ++---
 distributed/tests/test_security.py       | 2 +-
 distributed/versions.py                  | 8 ++++----
 distributed/worker.py                    | 4 ++--
 9 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 5939ad63655..6be2fcaa3bc 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,6 +1,6 @@
 repos:
 -   repo: https://github.com/ambv/black
-    rev: stable
+    rev: cad4138050b86d1c8570b926883e32f7465c2880
     hooks:
     - id: black
       language_version: python3.7
diff --git a/distributed/comm/tests/test_comms.py b/distributed/comm/tests/test_comms.py
index 5d52b04a137..f2bf7778221 100644
--- a/distributed/comm/tests/test_comms.py
+++ b/distributed/comm/tests/test_comms.py
@@ -969,7 +969,7 @@ def check_out_false(out_value):
         assert deserialize(ser.header, ser.frames) == 456
 
         assert isinstance(to_ser, list)
-        to_ser, = to_ser
+        (to_ser,) = to_ser
         # The to_serialize() value could have been actually serialized
         # or not (it's a transport-specific optimization)
         if isinstance(to_ser, Serialized):
@@ -1021,7 +1021,7 @@ def check_out(deserialize_flag, out_value):
             assert isinstance(ser, Serialized)
             assert deserialize(ser.header, ser.frames) == _uncompressible
             assert isinstance(to_ser, list)
-            to_ser, = to_ser
+            (to_ser,) = to_ser
             # The to_serialize() value could have been actually serialized
             # or not (it's a transport-specific optimization)
             if isinstance(to_ser, Serialized):
diff --git a/distributed/protocol/tests/test_numpy.py b/distributed/protocol/tests/test_numpy.py
index ede0eded3cf..eb39b57c351 100644
--- a/distributed/protocol/tests/test_numpy.py
+++ b/distributed/protocol/tests/test_numpy.py
@@ -107,7 +107,7 @@ def test_dumps_serialize_numpy(x):
     ],
 )
 def test_serialize_numpy_ma_masked_array(x):
-    y, = loads(dumps([to_serialize(x)]))
+    (y,) = loads(dumps([to_serialize(x)]))
     assert x.data.dtype == y.data.dtype
     np.testing.assert_equal(x.data, y.data)
     np.testing.assert_equal(x.mask, y.mask)
@@ -115,7 +115,7 @@ def test_serialize_numpy_ma_masked_array(x):
 
 
 def test_serialize_numpy_ma_masked():
-    y, = loads(dumps([to_serialize(np.ma.masked)]))
+    (y,) = loads(dumps([to_serialize(np.ma.masked)]))
     assert y is np.ma.masked
 
 
@@ -126,8 +126,8 @@ def test_dumps_serialize_numpy_custom_dtype():
     rational = test_rational.rational
     try:
         builtins.rational = (
-            rational
-        )  # Work around https://github.com/numpy/numpy/issues/9160
+            rational  # Work around https://github.com/numpy/numpy/issues/9160
+        )
         x = np.array([1], dtype=rational)
         header, frames = serialize(x)
         y = deserialize(header, frames)
diff --git a/distributed/queues.py b/distributed/queues.py
index 12bd15b6318..b97c317ac58 100644
--- a/distributed/queues.py
+++ b/distributed/queues.py
@@ -79,7 +79,7 @@ async def put(
         else:
             record = {"type": "msgpack", "value": data}
         if timeout is not None:
-            timeout = datetime.timedelta(seconds=(timeout))
+            timeout = datetime.timedelta(seconds=timeout)
         await self.queues[name].put(record, timeout=timeout)
 
     def future_release(self, name=None, key=None, client=None):
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 98a66bcc55f..999d4802730 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1643,7 +1643,7 @@ def update_graph(
         for key in set(priority) & touched_keys:
             ts = self.tasks[key]
             if ts.priority is None:
-                ts.priority = (-user_priority.get(key, 0), generation, priority[key])
+                ts.priority = (-(user_priority.get(key, 0)), generation, priority[key])
 
         # Ensure all runnables have a priority
         runnables = [ts for ts in touched_tasks if ts.run_spec]
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 300a1a5b2b0..6401cdd4b94 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -433,11 +433,10 @@ def test_filtered_communication(s, a, b):
             "keys": ["z"],
         }
     )
-
-    msg, = yield c.read()
+    (msg,) = yield c.read()
     assert msg["op"] == "key-in-memory"
     assert msg["key"] == "y"
-    msg, = yield f.read()
+    (msg,) = yield f.read()
     assert msg["op"] == "key-in-memory"
     assert msg["key"] == "z"
 
diff --git a/distributed/tests/test_security.py b/distributed/tests/test_security.py
index 7f144625b04..28438c6f359 100644
--- a/distributed/tests/test_security.py
+++ b/distributed/tests/test_security.py
@@ -306,7 +306,7 @@ def handle_comm(comm):
             listener.contact_address,
             connection_args=forced_cipher_sec.get_connection_args("worker"),
         )
-        cipher, _, _, = comm.extra_info["cipher"]
+        cipher, _, _ = comm.extra_info["cipher"]
         assert cipher in [FORCED_CIPHER] + TLS_13_CIPHERS
         comm.abort()
 
diff --git a/distributed/versions.py b/distributed/versions.py
index 2baa47a1d8f..d6a44096796 100644
--- a/distributed/versions.py
+++ b/distributed/versions.py
@@ -53,10 +53,10 @@ def get_system_info():
     host = [
         ("python", "%d.%d.%d.%s.%s" % sys.version_info[:]),
         ("python-bits", struct.calcsize("P") * 8),
-        ("OS", "%s" % (sysname)),
-        ("OS-release", "%s" % (release)),
-        ("machine", "%s" % (machine)),
-        ("processor", "%s" % (processor)),
+        ("OS", "%s" % sysname),
+        ("OS-release", "%s" % release),
+        ("machine", "%s" % machine),
+        ("processor", "%s" % processor),
         ("byteorder", "%s" % sys.byteorder),
         ("LC_ALL", "%s" % os.environ.get("LC_ALL", "None")),
         ("LANG", "%s" % os.environ.get("LANG", "None")),
diff --git a/distributed/worker.py b/distributed/worker.py
index e65c1cbccdc..2b762a4e751 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -396,9 +396,9 @@ def __init__(
             ("flight", "memory"): self.transition_dep_flight_memory,
         }
 
-        self.incoming_transfer_log = deque(maxlen=(100000))
+        self.incoming_transfer_log = deque(maxlen=100000)
         self.incoming_count = 0
-        self.outgoing_transfer_log = deque(maxlen=(100000))
+        self.outgoing_transfer_log = deque(maxlen=100000)
         self.outgoing_count = 0
         self.outgoing_current_count = 0
         self.repetitively_busy = 0

From 5437975b9c8f5bccb8ee0d39de80526412f83903 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sun, 28 Jul 2019 11:10:32 -0700
Subject: [PATCH 0377/1550] Move Worker.local_dir attribute to
 Worker.local_directory (#2900)

This matches the term elsewhere, including the Scheduler and the
dask-worker CLI
---
 distributed/cli/dask_worker.py   |  2 +-
 distributed/nanny.py             | 23 +++++++++++++---
 distributed/tests/test_worker.py | 46 ++++++++++++++++----------------
 distributed/worker.py            | 43 ++++++++++++++++++-----------
 4 files changed, 70 insertions(+), 44 deletions(-)

diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
index 073c7c9c922..08a1f47d1eb 100755
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -357,7 +357,7 @@ def del_pid_file():
             resources=resources,
             memory_limit=memory_limit,
             reconnect=reconnect,
-            local_dir=local_directory,
+            local_directory=local_directory,
             death_timeout=death_timeout,
             preload=preload,
             preload_argv=preload_argv,
diff --git a/distributed/nanny.py b/distributed/nanny.py
index 6c859115242..228e37c2839 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -66,7 +66,8 @@ def __init__(
         nthreads=None,
         ncores=None,
         loop=None,
-        local_dir="dask-worker-space",
+        local_dir=None,
+        local_directory="dask-worker-space",
         services=None,
         name=None,
         memory_limit="auto",
@@ -135,7 +136,11 @@ def __init__(
             "distributed.worker.memory.terminate"
         )
 
-        self.local_dir = local_dir
+        if local_dir is not None:
+            warnings.warn("The local_dir keyword has moved to local_directory")
+            local_directory = local_dir
+
+        self.local_directory = local_directory
 
         self.services = services
         self.name = name
@@ -221,6 +226,12 @@ def worker_address(self):
     def worker_dir(self):
         return None if self.process is None else self.process.worker_dir
 
+    @property
+    def local_dir(self):
+        """ For API compatibility with Nanny """
+        warnings.warn("The local_dir attribute has moved to local_directory")
+        return self.local_directory
+
     async def start(self):
         """ Start nanny, start local process, start watching """
         self.listen(self._start_address, listen_args=self.listen_args)
@@ -268,7 +279,7 @@ async def instantiate(self, comm=None):
             worker_kwargs = dict(
                 scheduler_ip=self.scheduler_addr,
                 nthreads=self.nthreads,
-                local_dir=self.local_dir,
+                local_directory=self.local_directory,
                 services=self.services,
                 nanny=self.address,
                 name=self.name,
@@ -667,7 +678,11 @@ async def run():
             else:
                 assert worker.address
                 init_result_q.put(
-                    {"address": worker.address, "dir": worker.local_dir, "uid": uid}
+                    {
+                        "address": worker.address,
+                        "dir": worker.local_directory,
+                        "uid": uid,
+                    }
                 )
                 init_result_q.close()
                 await worker.wait_until_closed()
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 1c1e70fcba7..96c673bf69a 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -65,7 +65,7 @@ def test_worker_nthreads():
     try:
         assert w.executor._max_workers == multiprocessing.cpu_count()
     finally:
-        shutil.rmtree(w.local_dir)
+        shutil.rmtree(w.local_directory)
 
 
 @gen_cluster()
@@ -179,9 +179,9 @@ def dont_test_delete_data_with_missing_worker(c, a, b):
 
 @gen_cluster(client=True)
 def test_upload_file(c, s, a, b):
-    assert not os.path.exists(os.path.join(a.local_dir, "foobar.py"))
-    assert not os.path.exists(os.path.join(b.local_dir, "foobar.py"))
-    assert a.local_dir != b.local_dir
+    assert not os.path.exists(os.path.join(a.local_directory, "foobar.py"))
+    assert not os.path.exists(os.path.join(b.local_directory, "foobar.py"))
+    assert a.local_directory != b.local_directory
 
     with rpc(a.address) as aa, rpc(b.address) as bb:
         yield [
@@ -189,8 +189,8 @@ def test_upload_file(c, s, a, b):
             bb.upload_file(filename="foobar.py", data="x = 123"),
         ]
 
-    assert os.path.exists(os.path.join(a.local_dir, "foobar.py"))
-    assert os.path.exists(os.path.join(b.local_dir, "foobar.py"))
+    assert os.path.exists(os.path.join(a.local_directory, "foobar.py"))
+    assert os.path.exists(os.path.join(b.local_directory, "foobar.py"))
 
     def g():
         import foobar
@@ -203,7 +203,7 @@ def g():
 
     yield c.close()
     yield s.close(close_workers=True)
-    assert not os.path.exists(os.path.join(a.local_dir, "foobar.py"))
+    assert not os.path.exists(os.path.join(a.local_directory, "foobar.py"))
 
 
 @pytest.mark.skip(reason="don't yet support uploading pyc files")
@@ -239,14 +239,14 @@ def g():
 def test_upload_egg(c, s, a, b):
     eggname = "testegg-1.0.0-py3.4.egg"
     local_file = __file__.replace("test_worker.py", eggname)
-    assert not os.path.exists(os.path.join(a.local_dir, eggname))
-    assert not os.path.exists(os.path.join(b.local_dir, eggname))
-    assert a.local_dir != b.local_dir
+    assert not os.path.exists(os.path.join(a.local_directory, eggname))
+    assert not os.path.exists(os.path.join(b.local_directory, eggname))
+    assert a.local_directory != b.local_directory
 
     yield c.upload_file(filename=local_file)
 
-    assert os.path.exists(os.path.join(a.local_dir, eggname))
-    assert os.path.exists(os.path.join(b.local_dir, eggname))
+    assert os.path.exists(os.path.join(a.local_directory, eggname))
+    assert os.path.exists(os.path.join(b.local_directory, eggname))
 
     def g(x):
         import testegg
@@ -261,21 +261,21 @@ def g(x):
     yield s.close()
     yield a.close()
     yield b.close()
-    assert not os.path.exists(os.path.join(a.local_dir, eggname))
+    assert not os.path.exists(os.path.join(a.local_directory, eggname))
 
 
 @gen_cluster(client=True)
 def test_upload_pyz(c, s, a, b):
     pyzname = "mytest.pyz"
     local_file = __file__.replace("test_worker.py", pyzname)
-    assert not os.path.exists(os.path.join(a.local_dir, pyzname))
-    assert not os.path.exists(os.path.join(b.local_dir, pyzname))
-    assert a.local_dir != b.local_dir
+    assert not os.path.exists(os.path.join(a.local_directory, pyzname))
+    assert not os.path.exists(os.path.join(b.local_directory, pyzname))
+    assert a.local_directory != b.local_directory
 
     yield c.upload_file(filename=local_file)
 
-    assert os.path.exists(os.path.join(a.local_dir, pyzname))
-    assert os.path.exists(os.path.join(b.local_dir, pyzname))
+    assert os.path.exists(os.path.join(a.local_directory, pyzname))
+    assert os.path.exists(os.path.join(b.local_directory, pyzname))
 
     def g(x):
         from mytest import mytest
@@ -290,7 +290,7 @@ def g(x):
     yield s.close()
     yield a.close()
     yield b.close()
-    assert not os.path.exists(os.path.join(a.local_dir, pyzname))
+    assert not os.path.exists(os.path.join(a.local_directory, pyzname))
 
 
 @pytest.mark.xfail(reason="Still lose time to network I/O")
@@ -805,7 +805,7 @@ def test_heartbeats(c, s, a, b):
 def test_worker_dir(worker):
     with tmpfile() as fn:
 
-        @gen_cluster(client=True, worker_kwargs={"local_dir": fn})
+        @gen_cluster(client=True, worker_kwargs={"local_directory": fn})
         def test_worker_dir(c, s, a, b):
             directories = [w.local_directory for w in s.workers.values()]
             assert all(d.startswith(fn) for d in directories)
@@ -1414,12 +1414,12 @@ def __init__(self, x, y):
 
 
 @gen_cluster(nthreads=[])
-def test_local_dir(s):
+def test_local_directory(s):
     with tmpfile() as fn:
         with dask.config.set(temporary_directory=fn):
             w = yield Worker(s.address)
-            assert w.local_dir.startswith(fn)
-            assert "dask-worker-space" in w.local_dir
+            assert w.local_directory.startswith(fn)
+            assert "dask-worker-space" in w.local_directory
 
 
 @pytest.mark.skipif(
diff --git a/distributed/worker.py b/distributed/worker.py
index 2b762a4e751..08927eb741d 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -118,7 +118,7 @@ class Worker(ServerNode):
         Number of nthreads used by this worker process
     * **executor:** ``concurrent.futures.ThreadPoolExecutor``:
         Executor used to perform computation
-    * **local_dir:** ``path``:
+    * **local_directory:** ``path``:
         Path on local machine to store temporary files
     * **scheduler:** ``rpc``:
         Location of scheduler.  See ``.ip/.port`` attributes.
@@ -233,7 +233,7 @@ class Worker(ServerNode):
         The object to use for storage, builds a disk-backed LRU dict by default
     nthreads: int, optional
     loop: tornado.ioloop.IOLoop
-    local_dir: str, optional
+    local_directory: str, optional
         Directory where we place local resources
     name: str, optional
     memory_limit: int, float, string
@@ -282,6 +282,7 @@ def __init__(
         nthreads=None,
         loop=None,
         local_dir=None,
+        local_directory=None,
         services=None,
         service_ports=None,
         service_kwargs=None,
@@ -457,11 +458,15 @@ def __init__(
         if silence_logs:
             silence_logging(level=silence_logs)
 
-        if local_dir is None:
-            local_dir = dask.config.get("temporary-directory") or os.getcwd()
-            if not os.path.exists(local_dir):
-                os.mkdir(local_dir)
-            local_dir = os.path.join(local_dir, "dask-worker-space")
+        if local_dir is not None:
+            warnings.warn("The local_dir keyword has moved to local_directory")
+            local_directory = local_dir
+
+        if local_directory is None:
+            local_directory = dask.config.get("temporary-directory") or os.getcwd()
+            if not os.path.exists(local_directory):
+                os.mkdir(local_directory)
+            local_directory = os.path.join(local_directory, "dask-worker-space")
 
         with warn_on_duration(
             "1s",
@@ -470,9 +475,9 @@ def __init__(
             "Consider specifying a local-directory to point workers to write "
             "scratch data to a local disk.",
         ):
-            self._workspace = WorkSpace(os.path.abspath(local_dir))
+            self._workspace = WorkSpace(os.path.abspath(local_directory))
             self._workdir = self._workspace.new_work_dir(prefix="worker-")
-            self.local_dir = self._workdir.dir_path
+            self.local_directory = self._workdir.dir_path
 
         self.security = security or Security()
         assert isinstance(self.security, Security)
@@ -515,7 +520,7 @@ def __init__(
                 from zict import Buffer, File, Func
             except ImportError:
                 raise ImportError("Please `pip install zict` for spill-to-disk workers")
-            path = os.path.join(self.local_dir, "storage")
+            path = os.path.join(self.local_directory, "storage")
             storage = Func(
                 partial(serialize_bytelist, on_error="raise"),
                 deserialize_bytes,
@@ -545,8 +550,8 @@ def __init__(
         self.heartbeat_active = False
         self._ipython_kernel = None
 
-        if self.local_dir not in sys.path:
-            sys.path.insert(0, self.local_dir)
+        if self.local_directory not in sys.path:
+            sys.path.insert(0, self.local_directory)
 
         self.services = {}
         self.service_specs = services or {}
@@ -678,6 +683,12 @@ def worker_address(self):
         """ For API compatibility with Nanny """
         return self.address
 
+    @property
+    def local_dir(self):
+        """ For API compatibility with Nanny """
+        warnings.warn("The local_dir attribute has moved to local_directory")
+        return self.local_directory
+
     def get_metrics(self):
         core = dict(
             executing=len(self.executing),
@@ -743,7 +754,7 @@ async def _register_with_scheduler(self):
                         now=time(),
                         resources=self.total_resources,
                         memory_limit=self.memory_limit,
-                        local_directory=self.local_dir,
+                        local_directory=self.local_directory,
                         services=self.service_ports,
                         nanny=self.nanny,
                         pid=os.getpid(),
@@ -842,7 +853,7 @@ def start_ipython(self, comm):
         return self._ipython_kernel.get_connection_info()
 
     async def upload_file(self, comm, filename=None, data=None, load=True):
-        out_filename = os.path.join(self.local_dir, filename)
+        out_filename = os.path.join(self.local_directory, filename)
 
         def func(data):
             if isinstance(data, unicode):
@@ -909,7 +920,7 @@ async def start(self):
         preload_modules(
             self.preload,
             parameter=self,
-            file_dir=self.local_dir,
+            file_dir=self.local_directory,
             argv=self.preload_argv,
         )
         # Services listen on all addresses
@@ -931,7 +942,7 @@ async def start(self):
         logger.info("              Threads: %26d", self.nthreads)
         if self.memory_limit:
             logger.info("               Memory: %26s", format_bytes(self.memory_limit))
-        logger.info("      Local Directory: %26s", self.local_dir)
+        logger.info("      Local Directory: %26s", self.local_directory)
 
         setproctitle("dask-worker [%s]" % self.address)
 

From 7810b731664ec179dc082657b6be725505d50225 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sun, 28 Jul 2019 11:11:42 -0700
Subject: [PATCH 0378/1550] Link from TapTools to worker info pages in
 dashboard (#2894)

---
 distributed/dashboard/scheduler.py            | 26 +++++--------------
 .../dashboard/tests/test_scheduler_bokeh.py   |  2 +-
 2 files changed, 7 insertions(+), 21 deletions(-)

diff --git a/distributed/dashboard/scheduler.py b/distributed/dashboard/scheduler.py
index 013edb39ace..e41862335cd 100644
--- a/distributed/dashboard/scheduler.py
+++ b/distributed/dashboard/scheduler.py
@@ -132,8 +132,7 @@ def __init__(self, scheduler, **kwargs):
                     "y": [1, 2],
                     "ms": [1, 2],
                     "color": ["red", "blue"],
-                    "dashboard_port": ["", ""],
-                    "dashboard_host": ["", ""],
+                    "escaped_worker": ["a", "b"],
                 }
             )
 
@@ -155,9 +154,7 @@ def __init__(self, scheduler, **kwargs):
             # fig.xaxis[0].formatter = NumeralTickFormatter(format='0.0s')
             fig.x_range.start = 0
 
-            tap = TapTool(
-                callback=OpenURL(url="./proxy/@dashboard_port/@dashboard_host/status")
-            )
+            tap = TapTool(callback=OpenURL(url="./info/worker/@escaped_worker.html"))
 
             hover = HoverTool()
             hover.tooltips = "@worker : @occupancy s."
@@ -171,9 +168,6 @@ def update(self):
         with log_errors():
             workers = list(self.scheduler.workers.values())
 
-            dashboard_host = [ws.host for ws in workers]
-            dashboard_port = [ws.services.get("dashboard", "") for ws in workers]
-
             y = list(range(len(workers)))
             occupancy = [ws.occupancy for ws in workers]
             ms = [occ * 1000 for occ in occupancy]
@@ -202,8 +196,7 @@ def update(self):
                     "worker": [ws.address for ws in workers],
                     "ms": ms,
                     "color": color,
-                    "dashboard_host": dashboard_host,
-                    "dashboard_port": dashboard_port,
+                    "escaped_worker": [escape.url_escape(ws.address) for ws in workers],
                     "x": x,
                     "y": y,
                 }
@@ -321,8 +314,7 @@ def __init__(self, scheduler, width=600, **kwargs):
                     "worker": ["a", "b"],
                     "y": [1, 2],
                     "nbytes-color": ["blue", "blue"],
-                    "dashboard_port": ["", ""],
-                    "dashboard_host": ["", ""],
+                    "escaped_worker": ["a", "b"],
                 }
             )
 
@@ -374,9 +366,7 @@ def __init__(self, scheduler, width=600, **kwargs):
                 fig.ygrid.visible = False
 
                 tap = TapTool(
-                    callback=OpenURL(
-                        url="./proxy/@dashboard_port/@dashboard_host/status"
-                    )
+                    callback=OpenURL(url="./info/worker/@escaped_worker.html")
                 )
                 fig.add_tools(tap)
 
@@ -404,9 +394,6 @@ def update(self):
         with log_errors():
             workers = list(self.scheduler.workers.values())
 
-            dashboard_host = [ws.host for ws in workers]
-            dashboard_port = [ws.services.get("dashboard", "") for ws in workers]
-
             y = list(range(len(workers)))
             nprocessing = [len(ws.processing) for ws in workers]
             processing_color = []
@@ -449,9 +436,8 @@ def update(self):
                     "nbytes-half": [nb / 2 for nb in nbytes],
                     "nbytes-color": nbytes_color,
                     "nbytes_text": nbytes_text,
-                    "dashboard_host": dashboard_host,
-                    "dashboard_port": dashboard_port,
                     "worker": [ws.address for ws in workers],
+                    "escaped_worker": [escape.url_escape(ws.address) for ws in workers],
                     "y": y,
                 }
 
diff --git a/distributed/dashboard/tests/test_scheduler_bokeh.py b/distributed/dashboard/tests/test_scheduler_bokeh.py
index 3c7f85dc89a..8544e72d9f4 100644
--- a/distributed/dashboard/tests/test_scheduler_bokeh.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh.py
@@ -92,7 +92,7 @@ def test_basic(c, s, a, b):
         data = ss.source.data
         assert len(first(data.values()))
         if component is Occupancy:
-            assert all(addr == "127.0.0.1" for addr in data["dashboard_host"])
+            assert all("127.0.0.1" in addr for addr in data["escaped_worker"])
 
 
 @gen_cluster(client=True)

From d1263246298a1978a3dd19f009b806fe24baaf77 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sun, 28 Jul 2019 11:11:56 -0700
Subject: [PATCH 0379/1550] Avoid exception in Client._ensure_connected if
 closed (#2893)

---
 distributed/client.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/distributed/client.py b/distributed/client.py
index a830b147fb8..c84a8160ad6 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -1026,6 +1026,11 @@ async def _ensure_connected(self, timeout=None):
             await comm.write(
                 {"op": "register-client", "client": self.id, "reply": False}
             )
+        except Exception as e:
+            if self.status == "closed":
+                return
+            else:
+                raise
         finally:
             self._connecting_to_scheduler = False
         if timeout is not None:

From e5f6d48db6c70080833293050e850afb7079fb73 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sun, 28 Jul 2019 11:13:01 -0700
Subject: [PATCH 0380/1550] Convert Pythonic kwargs to CLI Keywords for
 SSHCluster (#2898)

---
 distributed/deploy/ssh2.py            | 79 +++++++++++++++++++++------
 distributed/deploy/tests/test_ssh2.py | 24 +++++++-
 distributed/utils.py                  | 43 ++++++++++++++-
 3 files changed, 126 insertions(+), 20 deletions(-)

diff --git a/distributed/deploy/ssh2.py b/distributed/deploy/ssh2.py
index bf471f1dee6..5f9bb4f9a64 100644
--- a/distributed/deploy/ssh2.py
+++ b/distributed/deploy/ssh2.py
@@ -6,6 +6,9 @@
 import asyncssh
 
 from .spec import SpecCluster, ProcessInterface
+from ..utils import cli_keywords
+from ..scheduler import Scheduler as _Scheduler
+from ..worker import Worker as _Worker
 
 logger = logging.getLogger(__name__)
 
@@ -54,15 +57,25 @@ class Worker(Process):
         The hostname where we should run this worker
     connect_kwargs: dict
         kwargs to be passed to asyncssh connections
-    kwargs:
-        TODO
+    kwargs: dict
+        These will be passed through the dask-worker CLI to the
+        dask.distributed.Worker class
     """
 
-    def __init__(self, scheduler: str, address: str, connect_kwargs: dict, **kwargs):
+    def __init__(
+        self,
+        scheduler: str,
+        address: str,
+        connect_kwargs: dict,
+        kwargs: dict,
+        loop=None,
+        name=None,
+    ):
         self.address = address
         self.scheduler = scheduler
         self.connect_kwargs = connect_kwargs
         self.kwargs = kwargs
+        self.name = name
 
         super().__init__()
 
@@ -75,15 +88,19 @@ async def start(self):
                     "-m",
                     "distributed.cli.dask_worker",
                     self.scheduler,
-                    "--name",  # we need to have name for SpecCluster
-                    str(self.kwargs["name"]),
+                    "--name",
+                    str(self.name),
                 ]
+                + cli_keywords(self.kwargs, cls=_Worker)
             )
         )
 
         # We watch stderr in order to get the address, then we return
         while True:
             line = await self.proc.stderr.readline()
+            if not line.strip():
+                raise Exception("Worker failed to start")
+            logger.info(line.strip())
             if "worker at" in line:
                 self.address = line.split("worker at:")[1].strip()
                 self.status = "running"
@@ -101,11 +118,12 @@ class Scheduler(Process):
         The hostname where we should run this worker
     connect_kwargs: dict
         kwargs to be passed to asyncssh connections
-    kwargs:
-        TODO
+    kwargs: dict
+        These will be passed through the dask-scheduler CLI to the
+        dask.distributed.Scheduler class
     """
 
-    def __init__(self, address: str, connect_kwargs: dict, **kwargs):
+    def __init__(self, address: str, connect_kwargs: dict, kwargs: dict, loop=None):
         self.address = address
         self.kwargs = kwargs
         self.connect_kwargs = connect_kwargs
@@ -118,12 +136,18 @@ async def start(self):
         self.connection = await asyncssh.connect(self.address, **self.connect_kwargs)
 
         self.proc = await self.connection.create_process(
-            " ".join([sys.executable, "-m", "distributed.cli.dask_scheduler"])
+            " ".join(
+                [sys.executable, "-m", "distributed.cli.dask_scheduler"]
+                + cli_keywords(self.kwargs, cls=_Scheduler)
+            )
         )
 
         # We watch stderr in order to get the address, then we return
         while True:
             line = await self.proc.stderr.readline()
+            if not line.strip():
+                raise Exception("Worker failed to start")
+            logger.info(line.strip())
             if "Scheduler at" in line:
                 self.address = line.split("Scheduler at:")[1].strip()
                 break
@@ -131,7 +155,9 @@ async def start(self):
         await super().start()
 
 
-def SSHCluster(hosts, connect_kwargs, **kwargs):
+def SSHCluster(
+    hosts, connect_kwargs={}, worker_kwargs={}, scheduler_kwargs={}, **kwargs
+):
     """ Deploy a Dask cluster using SSH
 
     Parameters
@@ -140,25 +166,44 @@ def SSHCluster(hosts, connect_kwargs, **kwargs):
         List of hostnames or addresses on which to launch our cluster
         The first will be used for the scheduler and the rest for workers
     connect_kwargs:
+        Keywords to pass through to asyncssh.connect
         known_hosts: List[str] or None
             The list of keys which will be used to validate the server host
             key presented during the SSH handshake.  If this is not specified,
             the keys will be looked up in the file .ssh/known_hosts.  If this
             is explicitly set to None, server host key validation will be disabled.
-        TODO
-    kwargs:
-        TODO
-    ----
-    This doesn't handle any keyword arguments yet.  It is a proof of concept
+    scheduler_kwargs:
+        Keywords to pass on to dask-scheduler
+    worker_kwargs:
+        Keywords to pass on to dask-worker
+
+    Examples
+    --------
+    >>> from dask.distributed import Client
+    >>> from distributed.deploy.ssh2 import SSHCluster  # experimental for now
+    >>> cluster = SSHCluster(
+    ...     ["localhost"] * 4,
+    ...     connect_kwargs={"known_hosts": None},
+    ...     worker_kwargs={"nthreads": 2},
+    ...     scheduler_kwargs={"port": 0, "dashboard_address": ":8797"})
+    >>> client = Client(cluster)
     """
     scheduler = {
         "cls": Scheduler,
-        "options": {"address": hosts[0], "connect_kwargs": connect_kwargs},
+        "options": {
+            "address": hosts[0],
+            "connect_kwargs": connect_kwargs,
+            "kwargs": scheduler_kwargs,
+        },
     }
     workers = {
         i: {
             "cls": Worker,
-            "options": {"address": host, "connect_kwargs": connect_kwargs},
+            "options": {
+                "address": host,
+                "connect_kwargs": connect_kwargs,
+                "kwargs": worker_kwargs,
+            },
         }
         for i, host in enumerate(hosts[1:])
     }
diff --git a/distributed/deploy/tests/test_ssh2.py b/distributed/deploy/tests/test_ssh2.py
index beb1c6ef91e..07415ed47e6 100644
--- a/distributed/deploy/tests/test_ssh2.py
+++ b/distributed/deploy/tests/test_ssh2.py
@@ -9,9 +9,31 @@
 @pytest.mark.asyncio
 async def test_basic():
     async with SSHCluster(
-        ["127.0.0.1"] * 3, connect_kwargs=dict(known_hosts=None), asynchronous=True
+        ["127.0.0.1"] * 3,
+        connect_kwargs=dict(known_hosts=None),
+        asynchronous=True,
+        scheduler_kwargs={"port": 0},
     ) as cluster:
         assert len(cluster.workers) == 2
         async with Client(cluster, asynchronous=True) as client:
             result = await client.submit(lambda x: x + 1, 10)
             assert result == 11
+
+
+@pytest.mark.asyncio
+async def test_keywords():
+    async with SSHCluster(
+        ["127.0.0.1"] * 3,
+        connect_kwargs=dict(known_hosts=None),
+        asynchronous=True,
+        worker_kwargs={"nthreads": 2, "memory_limit": "2 GiB"},
+        scheduler_kwargs={"idle_timeout": "5s", "port": 0},
+    ) as cluster:
+        async with Client(cluster, asynchronous=True) as client:
+            assert (
+                await client.run_on_scheduler(
+                    lambda dask_scheduler: dask_scheduler.idle_timeout
+                )
+            ) == 5
+            d = client.scheduler_info()["workers"]
+            assert all(v["nthreads"] == 2 for v in d.values())
diff --git a/distributed/utils.py b/distributed/utils.py
index 227406da76b..71e4a09d2a1 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -803,7 +803,7 @@ def tokey(o):
     --------
 
     >>> tokey(b'x')
-    'x'
+    b'x'
     >>> tokey('x')
     'x'
     >>> tokey(1)
@@ -1210,7 +1210,7 @@ def parse_timedelta(s, default="seconds"):
     >>> parse_timedelta('300ms')
     0.3
     >>> parse_timedelta(timedelta(seconds=3))  # also supports timedeltas
-    3
+    3.0
     """
     if s is None:
         return None
@@ -1510,3 +1510,42 @@ def _repr_html_(self):
             for title, log in self.items()
         ]
         return "\n".join(summaries)
+
+
+def cli_keywords(d: dict, cls=None):
+    """ Convert a kwargs dictionary into a list of CLI keywords
+
+    Parameters
+    ----------
+    d: dict
+        The keywords to convert
+    cls: callable
+        The callable that consumes these terms to check them for validity
+
+    Examples
+    --------
+    >>> cli_keywords({"x": 123, "save_file": "foo.txt"})
+    ['--x', '123', '--save-file', 'foo.txt']
+
+    >>> from dask.distributed import Worker
+    >>> cli_keywords({"x": 123}, Worker)
+    Traceback (most recent call last):
+    ...
+    ValueError: Class distributed.worker.Worker does not support keyword x
+    """
+    if cls:
+        for k in d:
+            if not has_keyword(cls, k):
+                raise ValueError(
+                    "Class %s does not support keyword %s" % (typename(cls), k)
+                )
+
+    def convert_value(v):
+        out = str(v)
+        if " " in out and "'" not in out and '"' not in out:
+            out = '"' + out + '"'
+        return out
+
+    return sum(
+        [["--" + k.replace("_", "-"), convert_value(v)] for k, v in d.items()], []
+    )

From 3757cd497ad48cca737af291493eca5e660adcfe Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sun, 28 Jul 2019 15:19:55 -0700
Subject: [PATCH 0381/1550] Use kwargs in CLI (#2899)

---
 .travis.yml                       |  2 +-
 distributed/cli/dask_scheduler.py | 14 ++------------
 distributed/cli/dask_worker.py    | 29 +++--------------------------
 distributed/comm/ucx.py           |  4 +++-
 distributed/protocol/cupy.py      |  2 +-
 distributed/protocol/numba.py     |  2 +-
 6 files changed, 11 insertions(+), 42 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 35f4383748e..1726cffd4f1 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -24,7 +24,7 @@ install:
 script:
   - if [[ $TESTS == true ]]; then source continuous_integration/travis/run_tests.sh ; fi
   - if [[ $LINT == true ]]; then pip install flake8 ; flake8 distributed ; fi
-  - if [[ $LINT == true ]]; then pip install black; black distributed --check; fi
+  - if [[ $LINT == true ]]; then pip install git+https://github.com/psf/black@cad4138050b86d1c8570b926883e32f7465c2880; black distributed --check; fi
 
 after_success:
   - if [[ $COVERAGE == true ]]; then coverage report; pip install -q coveralls ; coveralls ; fi
diff --git a/distributed/cli/dask_scheduler.py b/distributed/cli/dask_scheduler.py
index 2e6220b7d81..54ecd69e595 100755
--- a/distributed/cli/dask_scheduler.py
+++ b/distributed/cli/dask_scheduler.py
@@ -136,17 +136,12 @@ def main(
     dashboard_prefix,
     use_xheaders,
     pid_file,
-    scheduler_file,
-    interface,
-    protocol,
     local_directory,
-    preload,
-    preload_argv,
     tls_ca_file,
     tls_cert,
     tls_key,
     dashboard_address,
-    idle_timeout,
+    **kwargs
 ):
     g0, g1, g2 = gc.get_threshold()  # https://github.com/dask/distributed/issues/1653
     gc.set_threshold(g0 * 3, g1 * 3, g2 * 3)
@@ -217,17 +212,12 @@ def del_pid_file():
 
     scheduler = Scheduler(
         loop=loop,
-        scheduler_file=scheduler_file,
         security=sec,
         host=host,
         port=port,
-        interface=interface,
-        protocol=protocol,
         dashboard_address=dashboard_address if dashboard else None,
         service_kwargs={"dashboard": {"prefix": dashboard_prefix}},
-        idle_timeout=idle_timeout,
-        preload=preload,
-        preload_argv=preload_argv,
+        **kwargs,
     )
     logger.info("Local Directory: %26s", local_directory)
     logger.info("-" * 47)
diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
index 08a1f47d1eb..084d7b59ccc 100755
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -11,7 +11,6 @@
 import click
 import dask
 from distributed import Nanny, Worker
-from distributed.utils import parse_timedelta
 from distributed.security import Security
 from distributed.cli.utils import check_python_3, install_signal_handlers
 from distributed.comm import get_address_host_port
@@ -199,25 +198,18 @@ def main(
     nprocs,
     nanny,
     name,
-    memory_limit,
     pid_file,
-    reconnect,
     resources,
     dashboard,
     bokeh,
     bokeh_port,
-    local_directory,
     scheduler_file,
-    interface,
-    protocol,
-    death_timeout,
-    preload,
-    preload_argv,
     dashboard_prefix,
     tls_ca_file,
     tls_cert,
     tls_key,
     dashboard_address,
+    **kwargs
 ):
     g0, g1, g2 = gc.get_threshold()  # https://github.com/dask/distributed/issues/1653
     gc.set_threshold(g0 * 3, g1 * 3, g2 * 3)
@@ -314,8 +306,6 @@ def del_pid_file():
 
         atexit.register(del_pid_file)
 
-    services = {}
-
     if resources:
         resources = resources.replace(",", " ").split()
         resources = dict(pair.split("=") for pair in resources)
@@ -326,10 +316,9 @@ def del_pid_file():
     loop = IOLoop.current()
 
     if nanny:
-        kwargs = {"worker_port": worker_port, "listen_address": listen_address}
+        kwargs.update({"worker_port": worker_port, "listen_address": listen_address})
         t = Nanny
     else:
-        kwargs = {}
         if nanny_port:
             kwargs["service_ports"] = {"nanny": nanny_port}
         t = Worker
@@ -344,33 +333,21 @@ def del_pid_file():
             "dask-worker SCHEDULER_ADDRESS:8786"
         )
 
-    if death_timeout is not None:
-        death_timeout = parse_timedelta(death_timeout, "s")
-
     nannies = [
         t(
             scheduler,
             scheduler_file=scheduler_file,
             nthreads=nthreads,
-            services=services,
             loop=loop,
             resources=resources,
-            memory_limit=memory_limit,
-            reconnect=reconnect,
-            local_directory=local_directory,
-            death_timeout=death_timeout,
-            preload=preload,
-            preload_argv=preload_argv,
             security=sec,
             contact_address=contact_address,
-            interface=interface,
-            protocol=protocol,
             host=host,
             port=port,
             dashboard_address=dashboard_address if dashboard else None,
             service_kwargs={"dashboard": {"prefix": dashboard_prefix}},
             name=name if nprocs == 1 or not name else name + "-" + str(i),
-            **kwargs
+            **kwargs,
         )
         for i in range(nprocs)
     ]
diff --git a/distributed/comm/ucx.py b/distributed/comm/ucx.py
index 3f3f0bfe943..eb1c7514133 100644
--- a/distributed/comm/ucx.py
+++ b/distributed/comm/ucx.py
@@ -121,7 +121,9 @@ async def read(self, deserializers=("cuda", "dask", "pickle", "error")):
             deserializers = ("cuda", "dask", "pickle", "error")
         resp = await self.ep.recv_future()
         obj = ucp.get_obj_from_msg(resp)
-        nframes, = struct.unpack("Q", obj[:8])  # first eight bytes for number of frames
+        (nframes,) = struct.unpack(
+            "Q", obj[:8]
+        )  # first eight bytes for number of frames
 
         gpu_frame_msg = obj[
             8 : 8 + nframes
diff --git a/distributed/protocol/cupy.py b/distributed/protocol/cupy.py
index 13c0348a821..f8d08ee3a1e 100644
--- a/distributed/protocol/cupy.py
+++ b/distributed/protocol/cupy.py
@@ -31,7 +31,7 @@ def serialize_cupy_ndarray(x):
 
 @cuda_deserialize.register(cupy.ndarray)
 def deserialize_cupy_array(header, frames):
-    frame, = frames
+    (frame,) = frames
     # TODO: put this in ucx... as a kind of "fixup"
     try:
         frame.typestr = header["typestr"]
diff --git a/distributed/protocol/numba.py b/distributed/protocol/numba.py
index 18405ffebe0..aa56a682b95 100644
--- a/distributed/protocol/numba.py
+++ b/distributed/protocol/numba.py
@@ -36,7 +36,7 @@ def serialize_numba_ndarray(x):
 
 @cuda_deserialize.register(numba.cuda.devicearray.DeviceNDArray)
 def deserialize_numba_ndarray(header, frames):
-    frame, = frames
+    (frame,) = frames
     # TODO: put this in ucx... as a kind of "fixup"
     if isinstance(frame, bytes):
         import numpy as np

From ec51220d3ff7b3607bd0899e492e253705a6e94f Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sun, 28 Jul 2019 15:20:25 -0700
Subject: [PATCH 0382/1550] Name SSHClusters by providing name= keyword to
 SpecCluster (#2903)

---
 distributed/deploy/spec.py            | 6 +++++-
 distributed/deploy/ssh2.py            | 2 +-
 distributed/deploy/tests/test_ssh2.py | 2 ++
 3 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index 747ecc19a41..cfafa5e94c2 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -76,6 +76,8 @@ class does handle all of the logic around asynchronously cleanly setting up
         async/await
     silence_logs: bool
         Whether or not we should silence logging when setting up the cluster.
+    name: str, optional
+        A name to use when printing out the cluster, defaults to type name
 
     Examples
     --------
@@ -149,6 +151,7 @@ def __init__(
         loop=None,
         security=None,
         silence_logs=False,
+        name=None,
     ):
         self._created = weakref.WeakSet()
 
@@ -170,6 +173,7 @@ def __init__(
         self.status = "created"
         self._instances.add(self)
         self._correct_state_waiting = None
+        self._name = name or type(self).__name__
 
         if not self.asynchronous:
             self._loop_runner.start()
@@ -366,7 +370,7 @@ async def scale_down(self, workers):
 
     def __repr__(self):
         return "%s(%r, workers=%d)" % (
-            type(self).__name__,
+            self._name,
             self.scheduler_address,
             len(self.workers),
         )
diff --git a/distributed/deploy/ssh2.py b/distributed/deploy/ssh2.py
index 5f9bb4f9a64..3e87b6d301b 100644
--- a/distributed/deploy/ssh2.py
+++ b/distributed/deploy/ssh2.py
@@ -207,4 +207,4 @@ def SSHCluster(
         }
         for i, host in enumerate(hosts[1:])
     }
-    return SpecCluster(workers, scheduler, **kwargs)
+    return SpecCluster(workers, scheduler, name="SSHCluster", **kwargs)
diff --git a/distributed/deploy/tests/test_ssh2.py b/distributed/deploy/tests/test_ssh2.py
index 07415ed47e6..df90d35cd6e 100644
--- a/distributed/deploy/tests/test_ssh2.py
+++ b/distributed/deploy/tests/test_ssh2.py
@@ -19,6 +19,8 @@ async def test_basic():
             result = await client.submit(lambda x: x + 1, 10)
             assert result == 11
 
+        assert "SSH" in repr(cluster)
+
 
 @pytest.mark.asyncio
 async def test_keywords():

From 50e486b67d01084896b92fb90b9dcf8d5be8eb21 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sun, 28 Jul 2019 18:23:56 -0700
Subject: [PATCH 0383/1550] Request feed of worker information from Scheduler
 to SpecCluster (#2902)

* Don't explicitly provide loop= in SpecCluster
    This is called from within the event loop, so IOLoop.current should be fine
* Ask scheduler for updates on adding and removing workers
* implement dashboard_link
* Add widgets to SpecCluster
* Don't include scaling buttons in SSHCluster
---
 distributed/deploy/spec.py                    | 185 +++++++++++++++++-
 distributed/deploy/ssh2.py                    |   2 +-
 distributed/deploy/tests/test_spec_cluster.py |  59 +++++-
 distributed/deploy/tests/test_ssh2.py         |   1 +
 distributed/scheduler.py                      |  46 ++++-
 5 files changed, 281 insertions(+), 12 deletions(-)

diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index cfafa5e94c2..87228d5693e 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -1,12 +1,24 @@
 import asyncio
 import atexit
+import copy
 import weakref
 
 from tornado import gen
+from dask.utils import format_bytes
 
 from .cluster import Cluster
+from ..comm import connect
 from ..core import rpc, CommClosedError
-from ..utils import LoopRunner, silence_logging, ignoring, Log, Logs
+from ..utils import (
+    log_errors,
+    LoopRunner,
+    silence_logging,
+    ignoring,
+    Log,
+    Logs,
+    PeriodicCallback,
+    format_dashboard_link,
+)
 from ..scheduler import Scheduler
 from ..security import Security
 
@@ -21,9 +33,8 @@ class ProcessInterface:
 
     """
 
-    def __init__(self, loop=None):
+    def __init__(self):
         self.address = None
-        self.loop = loop
         self.lock = asyncio.Lock()
         self.status = "created"
 
@@ -155,14 +166,16 @@ def __init__(
     ):
         self._created = weakref.WeakSet()
 
-        self.scheduler_spec = scheduler
-        self.worker_spec = workers or {}
-        self.new_spec = worker
+        self.scheduler_spec = copy.copy(scheduler)
+        self.worker_spec = copy.copy(workers) or {}
+        self.new_spec = copy.copy(worker)
         self.workers = {}
         self._i = 0
         self._asynchronous = asynchronous
         self.security = security or Security()
         self.scheduler_comm = None
+        self.scheduler_info = {}
+        self.periodic_callbacks = {}
 
         if silence_logs:
             self._old_logging_level = silence_logging(level=silence_logs)
@@ -189,6 +202,8 @@ async def _start(self):
         if self.status == "closed":
             raise ValueError("Cluster is closed")
 
+        self._lock = asyncio.Lock()
+
         if self.scheduler_spec is None:
             try:
                 from distributed.dashboard import BokehScheduler
@@ -198,18 +213,47 @@ async def _start(self):
                 services = {("dashboard", 8787): BokehScheduler}
             self.scheduler_spec = {"cls": Scheduler, "options": {"services": services}}
         self.scheduler = self.scheduler_spec["cls"](
-            loop=self.loop, **self.scheduler_spec.get("options", {})
+            **self.scheduler_spec.get("options", {})
         )
 
-        self._lock = asyncio.Lock()
         self.status = "starting"
         self.scheduler = await self.scheduler
         self.scheduler_comm = rpc(
             self.scheduler.address,
             connection_args=self.security.get_connection_args("client"),
         )
+        comm = await connect(
+            self.scheduler_address,
+            connection_args=self.security.get_connection_args("client"),
+        )
+        await comm.write({"op": "subscribe_worker_status"})
+        self.scheduler_info = await comm.read()
+        self._watch_worker_status_comm = comm
+        self._watch_worker_status_task = asyncio.ensure_future(
+            self._watch_worker_status(comm)
+        )
         self.status = "running"
 
+    async def _watch_worker_status(self, comm):
+        """ Listen to scheduler for updates on adding and removing workers """
+        while True:
+            try:
+                msgs = await comm.read()
+            except OSError:
+                break
+
+            for op, msg in msgs:
+                if op == "add":
+                    workers = msg.pop("workers")
+                    self.scheduler_info["workers"].update(workers)
+                    self.scheduler_info.update(msg)
+                elif op == "remove":
+                    del self.scheduler_info["workers"][msg]
+                else:
+                    raise ValueError("Invalid op", op, msg)
+
+        await comm.close()
+
     def _correct_state(self):
         if self._correct_state_waiting:
             # If people call this frequently, we only want to run it once
@@ -293,12 +337,17 @@ async def _close(self):
             return
         self.status = "closing"
 
+        for pc in self.periodic_callbacks.values():
+            pc.stop()
+
         self.scale(0)
         await self._correct_state()
         async with self._lock:
             with ignoring(CommClosedError):
                 await self.scheduler_comm.close(close_workers=True)
         await self.scheduler.close()
+        await self._watch_worker_status_comm.close()
+        await self._watch_worker_status_task
         for w in self._created:
             assert w.status == "closed"
         self.scheduler_comm.close_rpc()
@@ -357,6 +406,10 @@ def new_worker_spec(self):
 
         return self._i, self.new_spec
 
+    @property
+    def _supports_scaling(self):
+        return not not self.new_spec
+
     async def scale_down(self, workers):
         workers = set(workers)
 
@@ -408,6 +461,122 @@ def logs(self, scheduler=True, workers=True):
         """
         return self.sync(self._logs, scheduler=scheduler, workers=workers)
 
+    @property
+    def dashboard_link(self):
+        try:
+            port = self.scheduler_info["services"]["dashboard"]
+        except KeyError:
+            return ""
+        else:
+            host = self.scheduler_address.split("://")[1].split(":")[0]
+            return format_dashboard_link(host, port)
+
+    def _widget_status(self):
+        workers = len(self.scheduler_info["workers"])
+        cores = sum(v["nthreads"] for v in self.scheduler_info["workers"].values())
+        memory = sum(v["memory_limit"] for v in self.scheduler_info["workers"].values())
+        memory = format_bytes(memory)
+        text = """
+<div>
+  <style scoped>
+    .dataframe tbody tr th:only-of-type {
+        vertical-align: middle;
+    }
+
+    .dataframe tbody tr th {
+        vertical-align: top;
+    }
+
+    .dataframe thead th {
+        text-align: right;
+    }
+  </style>
+  <table style="text-align: right;">
+    <tr><th>Workers</th> <td>%d</td></tr>
+    <tr><th>Cores</th> <td>%d</td></tr>
+    <tr><th>Memory</th> <td>%s</td></tr>
+  </table>
+</div>
+""" % (
+            workers,
+            cores,
+            memory,
+        )
+        return text
+
+    def _widget(self):
+        """ Create IPython widget for display within a notebook """
+        try:
+            return self._cached_widget
+        except AttributeError:
+            pass
+
+        from ipywidgets import Layout, VBox, HBox, IntText, Button, HTML, Accordion
+
+        layout = Layout(width="150px")
+
+        if self.dashboard_link:
+            link = '<p><b>Dashboard: </b><a href="%s" target="_blank">%s</a></p>\n' % (
+                self.dashboard_link,
+                self.dashboard_link,
+            )
+        else:
+            link = ""
+
+        title = "<h2>%s</h2>" % type(self).__name__
+        title = HTML(title)
+        dashboard = HTML(link)
+
+        status = HTML(self._widget_status(), layout=Layout(min_width="150px"))
+
+        if self._supports_scaling:
+            request = IntText(0, description="Workers", layout=layout)
+            scale = Button(description="Scale", layout=layout)
+
+            minimum = IntText(0, description="Minimum", layout=layout)
+            maximum = IntText(0, description="Maximum", layout=layout)
+            adapt = Button(description="Adapt", layout=layout)
+
+            accordion = Accordion(
+                [HBox([request, scale]), HBox([minimum, maximum, adapt])],
+                layout=Layout(min_width="500px"),
+            )
+            accordion.selected_index = None
+            accordion.set_title(0, "Manual Scaling")
+            accordion.set_title(1, "Adaptive Scaling")
+
+            def adapt_cb(b):
+                self.adapt(minimum=minimum.value, maximum=maximum.value)
+
+            adapt.on_click(adapt_cb)
+
+            def scale_cb(b):
+                with log_errors():
+                    n = request.value
+                    with ignoring(AttributeError):
+                        self._adaptive.stop()
+                    self.scale(n)
+
+            scale.on_click(scale_cb)
+        else:
+            accordion = HTML("")
+
+        box = VBox([title, HBox([status, accordion]), dashboard])
+
+        self._cached_widget = box
+
+        def update():
+            status.value = self._widget_status()
+
+        pc = PeriodicCallback(update, 500, io_loop=self.loop)
+        self.periodic_callbacks["cluster-repr"] = pc
+        pc.start()
+
+        return box
+
+    def _ipython_display_(self, **kwargs):
+        return self._widget()._ipython_display_(**kwargs)
+
 
 @atexit.register
 def close_clusters():
diff --git a/distributed/deploy/ssh2.py b/distributed/deploy/ssh2.py
index 3e87b6d301b..189a61df5f1 100644
--- a/distributed/deploy/ssh2.py
+++ b/distributed/deploy/ssh2.py
@@ -123,7 +123,7 @@ class Scheduler(Process):
         dask.distributed.Scheduler class
     """
 
-    def __init__(self, address: str, connect_kwargs: dict, kwargs: dict, loop=None):
+    def __init__(self, address: str, connect_kwargs: dict, kwargs: dict):
         self.address = address
         self.kwargs = kwargs
         self.connect_kwargs = connect_kwargs
diff --git a/distributed/deploy/tests/test_spec_cluster.py b/distributed/deploy/tests/test_spec_cluster.py
index 51094dca2d5..84c868b2585 100644
--- a/distributed/deploy/tests/test_spec_cluster.py
+++ b/distributed/deploy/tests/test_spec_cluster.py
@@ -1,3 +1,6 @@
+import asyncio
+from time import time
+
 from dask.distributed import SpecCluster, Worker, Client, Scheduler, Nanny
 from distributed.deploy.spec import close_clusters, ProcessInterface
 from distributed.utils_test import loop, cleanup  # noqa: F401
@@ -30,7 +33,7 @@ async def test_specification(cleanup):
     async with SpecCluster(
         workers=worker_spec, scheduler=scheduler, asynchronous=True
     ) as cluster:
-        assert cluster.worker_spec is worker_spec
+        assert cluster.worker_spec == worker_spec
 
         assert len(cluster.workers) == 3
         assert set(cluster.workers) == set(worker_spec)
@@ -57,7 +60,7 @@ def test_spec_sync(loop):
         "my-worker": {"cls": MyWorker, "options": {"nthreads": 3}},
     }
     with SpecCluster(workers=worker_spec, scheduler=scheduler, loop=loop) as cluster:
-        assert cluster.worker_spec is worker_spec
+        assert cluster.worker_spec == worker_spec
 
         assert len(cluster.workers) == 3
         assert set(cluster.workers) == set(worker_spec)
@@ -205,3 +208,55 @@ async def test_logs(cleanup):
         w = toolz.first(cluster.scheduler.workers)
         logs = await cluster.logs(scheduler=False, workers=[w])
         assert set(logs) == {w}
+
+
+@pytest.mark.asyncio
+async def test_scheduler_info(cleanup):
+    async with SpecCluster(
+        workers=worker_spec, scheduler=scheduler, asynchronous=True
+    ) as cluster:
+        assert (
+            cluster.scheduler_info["id"] == cluster.scheduler.id
+        )  # present at startup
+
+        start = time()  # wait for all workers
+        while len(cluster.scheduler_info["workers"]) < len(cluster.workers):
+            await asyncio.sleep(0.01)
+            assert time() < start + 1
+
+        assert set(cluster.scheduler.identity()["workers"]) == set(
+            cluster.scheduler_info["workers"]
+        )
+        assert (
+            cluster.scheduler.identity()["services"]
+            == cluster.scheduler_info["services"]
+        )
+        assert len(cluster.scheduler_info["workers"]) == len(cluster.workers)
+
+
+@pytest.mark.asyncio
+async def test_dashboard_link(cleanup):
+    async with SpecCluster(
+        workers=worker_spec,
+        scheduler={
+            "cls": Scheduler,
+            "options": {"port": 0, "dashboard_address": ":12345"},
+        },
+        asynchronous=True,
+    ) as cluster:
+        assert "12345" in cluster.dashboard_link
+
+
+@pytest.mark.asyncio
+async def test_widget(cleanup):
+    async with SpecCluster(
+        workers=worker_spec, scheduler=scheduler, asynchronous=True
+    ) as cluster:
+
+        start = time()  # wait for all workers
+        while len(cluster.scheduler_info["workers"]) < len(cluster.worker_spec):
+            await asyncio.sleep(0.01)
+            assert time() < start + 1
+
+        assert "3" in cluster._widget_status()
+        assert "GB" in cluster._widget_status()
diff --git a/distributed/deploy/tests/test_ssh2.py b/distributed/deploy/tests/test_ssh2.py
index df90d35cd6e..b744d352b8b 100644
--- a/distributed/deploy/tests/test_ssh2.py
+++ b/distributed/deploy/tests/test_ssh2.py
@@ -18,6 +18,7 @@ async def test_basic():
         async with Client(cluster, asynchronous=True) as client:
             result = await client.submit(lambda x: x + 1, 10)
             assert result == 11
+        assert not cluster._supports_scaling
 
         assert "SSH" in repr(cluster)
 
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 999d4802730..b0db6653d2b 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -41,6 +41,7 @@
 from .comm.addressing import address_from_user_args
 from .compatibility import finalize, unicode, Mapping, Set
 from .core import rpc, connect, send_recv, clean_exception, CommClosedError
+from .diagnostics.plugin import SchedulerPlugin
 from . import profile
 from .metrics import time
 from .node import ServerNode
@@ -1078,6 +1079,7 @@ def __init__(
             "register_worker_plugin": self.register_worker_plugin,
             "adaptive_target": self.adaptive_target,
             "workers_to_close": self.workers_to_close,
+            "subscribe_worker_status": self.subscribe_worker_status,
         }
 
         self._transitions = {
@@ -1266,7 +1268,7 @@ async def close(self, comm=None, fast=False, close_workers=False):
         self.periodic_callbacks.clear()
 
         self.stop_services()
-        for ext in self.extensions:
+        for ext in self.extensions.values():
             with ignoring(AttributeError):
                 ext.teardown()
         logger.info("Scheduler closing all comms")
@@ -3232,6 +3234,14 @@ async def feed(
                 if teardown:
                     teardown(self, state)
 
+    def subscribe_worker_status(self, comm=None):
+        WorkerStatusPlugin(self, comm)
+        ident = self.identity()
+        for v in ident["workers"].values():
+            del v["metrics"]
+            del v["last_seen"]
+        return ident
+
     def get_processing(self, comm=None, workers=None):
         if workers is not None:
             workers = set(map(self.coerce_address, workers))
@@ -4963,3 +4973,37 @@ def __init__(self, task, last_worker):
         super(KilledWorker, self).__init__(task, last_worker)
         self.task = task
         self.last_worker = last_worker
+
+
+class WorkerStatusPlugin(SchedulerPlugin):
+    """
+    An plugin to share worker status with a remote observer
+
+    This is used in cluster managers to keep updated about the status of the
+    scheduler.
+    """
+
+    def __init__(self, scheduler, comm):
+        self.bcomm = BatchedSend(interval="5ms")
+        self.bcomm.start(comm)
+
+        self.scheduler = scheduler
+        self.scheduler.add_plugin(self)
+
+    def add_worker(self, worker=None, **kwargs):
+        ident = self.scheduler.workers[worker].identity()
+        del ident["metrics"]
+        del ident["last_seen"]
+        try:
+            self.bcomm.send(["add", {"workers": {worker: ident}}])
+        except CommClosedError:
+            self.scheduler.remove_plugin(self)
+
+    def remove_worker(self, worker=None, **kwargs):
+        try:
+            self.bcomm.send(["remove", worker])
+        except CommClosedError:
+            self.scheduler.remove_plugin(self)
+
+    def teardown(self):
+        self.bcomm.close()

From 157eada32c7c49c3f1b0fe9e020be74592ab27fd Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 29 Jul 2019 07:35:54 -0700
Subject: [PATCH 0384/1550] Clear out compatibillity file (#2896)

It has been a suitable time since we've dropped Python 2.
This clears out the file of unnecessary functions.
---
 distributed/actor.py                     |   5 +-
 distributed/client.py                    |  25 +--
 distributed/comm/inproc.py               |   3 +-
 distributed/comm/tcp.py                  |  15 +-
 distributed/comm/tests/test_comms.py     |   7 +-
 distributed/comm/utils.py                |   9 +-
 distributed/compatibility.py             | 251 +----------------------
 distributed/core.py                      |   4 +-
 distributed/dashboard/utils.py           |   4 +-
 distributed/deploy/cluster.py            |   4 +-
 distributed/diagnostics/progressbar.py   |   4 +-
 distributed/diskutils.py                 |   6 +-
 distributed/node.py                      |   8 +-
 distributed/process.py                   |   4 +-
 distributed/profile.py                   |   3 +-
 distributed/protocol/numpy.py            |   6 +-
 distributed/protocol/serialize.py        |   3 -
 distributed/protocol/tests/test_h5py.py  |  23 +--
 distributed/protocol/tests/test_numpy.py |   3 +-
 distributed/publish.py                   |   5 +-
 distributed/pubsub.py                    |   8 +-
 distributed/scheduler.py                 |   9 +-
 distributed/submit.py                    |  10 +-
 distributed/tests/test_as_completed.py   |  13 +-
 distributed/tests/test_client.py         |  11 +-
 distributed/tests/test_compatibility.py  |  38 ----
 distributed/tests/test_core.py           |   6 +-
 distributed/tests/test_diskutils.py      |   7 +-
 distributed/tests/test_metrics.py        |  35 ++--
 distributed/tests/test_profile.py        |   6 +-
 distributed/tests/test_utils.py          |  10 +-
 distributed/tests/test_utils_perf.py     |   3 -
 distributed/tests/test_worker.py         |   5 +-
 distributed/threadpoolexecutor.py        |   4 +-
 distributed/utils.py                     | 213 ++++++-------------
 distributed/utils_perf.py                |  10 +-
 distributed/utils_test.py                |  15 +-
 distributed/worker.py                    |  17 +-
 38 files changed, 200 insertions(+), 612 deletions(-)
 delete mode 100644 distributed/tests/test_compatibility.py

diff --git a/distributed/actor.py b/distributed/actor.py
index e45f089effd..e7e4afaacf0 100644
--- a/distributed/actor.py
+++ b/distributed/actor.py
@@ -1,9 +1,10 @@
 import asyncio
 from tornado import gen
 import functools
+import threading
+from queue import Queue
 
 from .client import Future, default_client
-from .compatibility import get_thread_identity, Queue
 from .protocol import to_serialize
 from .utils import sync
 from .utils_comm import WrappedKey
@@ -103,7 +104,7 @@ def _asynchronous(self):
         if self._client:
             return self._client.asynchronous
         else:
-            return get_thread_identity() == self._worker.thread_id
+            return threading.get_ident() == self._worker.thread_id
 
     def _sync(self, func, *args, **kwargs):
         if self._client:
diff --git a/distributed/client.py b/distributed/client.py
index c84a8160ad6..e1efda4e0d7 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -1,7 +1,7 @@
 from __future__ import print_function, division, absolute_import
 
 import atexit
-from collections import defaultdict
+from collections import defaultdict, Iterator
 from concurrent.futures import ThreadPoolExecutor, CancelledError
 from concurrent.futures._base import DoneAndNotDoneFutures
 from contextlib import contextmanager
@@ -9,6 +9,7 @@
 from datetime import timedelta
 import errno
 from functools import partial
+import html
 import itertools
 import json
 import logging
@@ -19,6 +20,7 @@
 import threading
 import six
 import socket
+from queue import Queue as pyQueue
 import warnings
 import weakref
 
@@ -26,7 +28,7 @@
 from dask.base import tokenize, normalize_token, collections_to_dsk
 from dask.core import flatten, get_dependencies
 from dask.optimization import SubgraphCallable
-from dask.compatibility import apply, unicode
+from dask.compatibility import apply
 from dask.utils import ensure_dict, format_bytes
 
 try:
@@ -55,13 +57,6 @@
     gather_from_workers,
 )
 from .cfexecutor import ClientExecutor
-from .compatibility import (
-    Queue as pyQueue,
-    isqueue,
-    html_escape,
-    StopAsyncIteration,
-    Iterator,
-)
 from .core import connect, rpc, clean_exception, CommClosedError, PooledRPCCall
 from .metrics import time
 from .node import Node
@@ -400,7 +395,7 @@ def __repr__(self):
             return "<Future: status: %s, key: %s>" % (self.status, self.key)
 
     def _repr_html_(self):
-        text = "<b>Future: %s</b> " % html_escape(key_split(self.key))
+        text = "<b>Future: %s</b> " % html.escape(key_split(self.key))
         text += (
             '<font color="gray">status: </font>'
             '<font color="%(color)s">%(status)s</font>, '
@@ -414,7 +409,7 @@ def _repr_html_(self):
             except AttributeError:
                 typ = str(self.type)
             text += '<font color="gray">type: </font>%s, ' % typ
-        text += '<font color="gray">key: </font>%s' % html_escape(str(self.key))
+        text += '<font color="gray">key: </font>%s' % html.escape(str(self.key))
         return text
 
     def __await__(self):
@@ -1523,7 +1518,7 @@ def map(
         if not callable(func):
             raise TypeError("First input to map must be a callable function")
 
-        if all(map(isqueue, iterables)) or all(
+        if all(isinstance(it, pyQueue) for it in iterables) or all(
             isinstance(i, Iterator) for i in iterables
         ):
             raise TypeError(
@@ -1792,7 +1787,7 @@ def gather(self, futures, errors="raise", direct=None, asynchronous=None):
         --------
         Client.scatter: Send data out to cluster
         """
-        if isqueue(futures):
+        if isinstance(futures, pyQueue):
             raise TypeError(
                 "Dask no longer supports gathering over Iterators and Queues. "
                 "Consider using a normal for loop and Client.submit/gather"
@@ -1829,7 +1824,7 @@ async def _scatter(
         if isinstance(workers, six.string_types + (Number,)):
             workers = [workers]
         if isinstance(data, dict) and not all(
-            isinstance(k, (bytes, unicode)) for k in data
+            isinstance(k, (bytes, str)) for k in data
         ):
             d = await self._scatter(keymap(tokey, data), workers, broadcast)
             raise gen.Return({k: d[tokey(k)] for k in data})
@@ -1998,7 +1993,7 @@ def scatter(
         """
         if timeout == no_default:
             timeout = self._timeout
-        if isqueue(data) or isinstance(data, Iterator):
+        if isinstance(data, pyQueue) or isinstance(data, Iterator):
             raise TypeError(
                 "Dask no longer supports mapping over Iterators or Queues."
                 "Consider using a normal for loop and Client.submit"
diff --git a/distributed/comm/inproc.py b/distributed/comm/inproc.py
index c9a6dc90281..3a781479bbc 100644
--- a/distributed/comm/inproc.py
+++ b/distributed/comm/inproc.py
@@ -11,7 +11,6 @@
 from tornado.concurrent import Future
 from tornado.ioloop import IOLoop
 
-from ..compatibility import finalize
 from ..protocol import nested_deserialize
 from ..utils import get_ip
 
@@ -161,7 +160,7 @@ def __init__(
         self._write_loop = write_loop
         self._closed = False
 
-        self._finalizer = finalize(self, self._get_finalizer())
+        self._finalizer = weakref.finalize(self, self._get_finalizer())
         self._finalizer.atexit = False
         self._initialized = True
 
diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py
index 602c9a36253..d23f381857d 100644
--- a/distributed/comm/tcp.py
+++ b/distributed/comm/tcp.py
@@ -6,6 +6,7 @@
 import struct
 import sys
 from tornado import gen
+import weakref
 
 try:
     import ssl
@@ -19,7 +20,6 @@
 from tornado.tcpclient import TCPClient
 from tornado.tcpserver import TCPServer
 
-from ..compatibility import finalize, PY3
 from ..threadpoolexecutor import ThreadPoolExecutor
 from ..utils import (
     ensure_bytes,
@@ -158,7 +158,7 @@ def __init__(self, stream, local_addr, peer_addr, deserialize=True):
         self._peer_addr = peer_addr
         self.stream = stream
         self.deserialize = deserialize
-        self._finalizer = finalize(self, self._get_finalizer())
+        self._finalizer = weakref.finalize(self, self._get_finalizer())
         self._finalizer.atexit = False
         self._extra = {}
 
@@ -199,7 +199,7 @@ async def read(self, deserializers=None):
             frames = []
             for length in lengths:
                 if length:
-                    if PY3 and self._iostream_has_read_into:
+                    if self._iostream_has_read_into:
                         frame = bytearray(length)
                         n = await stream.read_into(frame)
                         assert n == length, (n, length)
@@ -242,7 +242,7 @@ def write(self, msg, serializers=None, on_error="message"):
             length_bytes = [struct.pack("Q", len(frames))] + [
                 struct.pack("Q", x) for x in lengths
             ]
-            if PY3 and sum(lengths) < 2 ** 17:  # 128kiB
+            if sum(lengths) < 2 ** 17:  # 128kiB
                 b = b"".join(length_bytes + frames)  # small enough, send in one go
                 stream.write(b)
             else:
@@ -340,11 +340,8 @@ def _check_encryption(self, address, connection_args):
 
 
 class BaseTCPConnector(Connector, RequireEncryptionMixin):
-    if PY3:  # see github PR #2403 discussion for more info
-        _executor = ThreadPoolExecutor(2, thread_name_prefix="TCP-Executor")
-        _resolver = netutil.ExecutorResolver(close_executor=False, executor=_executor)
-    else:
-        _resolver = None
+    _executor = ThreadPoolExecutor(2, thread_name_prefix="TCP-Executor")
+    _resolver = netutil.ExecutorResolver(close_executor=False, executor=_executor)
     client = TCPClient(resolver=_resolver)
 
     async def connect(self, address, deserialize=True, **connection_args):
diff --git a/distributed/comm/tests/test_comms.py b/distributed/comm/tests/test_comms.py
index f2bf7778221..7fac117027b 100644
--- a/distributed/comm/tests/test_comms.py
+++ b/distributed/comm/tests/test_comms.py
@@ -11,7 +11,6 @@
 from tornado import gen, ioloop, locks, queues
 from tornado.concurrent import Future
 
-from distributed.compatibility import PY3
 from distributed.metrics import time
 from distributed.utils import get_ip, get_ipv6
 from distributed.utils_test import (
@@ -333,8 +332,7 @@ def sleep_for_60ms():
         yield connect("tcp://localhost:28400", 0.052)
     max_thread_count = yield sleep_future
     # 2 is the number set by BaseTCPConnector.executor (ThreadPoolExecutor)
-    if PY3:
-        assert max_thread_count <= 2 + original_thread_count
+    assert max_thread_count <= 2 + original_thread_count
 
     # tcp.TLSConnector()
     sleep_future = sleep_for_60ms()
@@ -345,8 +343,7 @@ def sleep_for_60ms():
             connection_args={"ssl_context": get_client_ssl_context()},
         )
     max_thread_count = yield sleep_future
-    if PY3:
-        assert max_thread_count <= 2 + original_thread_count
+    assert max_thread_count <= 2 + original_thread_count
 
 
 @gen.coroutine
diff --git a/distributed/comm/utils.py b/distributed/comm/utils.py
index bb6621e2021..1e23b25c46b 100644
--- a/distributed/comm/utils.py
+++ b/distributed/comm/utils.py
@@ -3,11 +3,11 @@
 from concurrent.futures import ThreadPoolExecutor
 import logging
 import socket
+import weakref
 
 from tornado import gen
 
 from .. import protocol
-from ..compatibility import finalize, PY3
 from ..utils import get_ip, get_ipv6, nbytes
 
 
@@ -25,7 +25,7 @@
     )
 except TypeError:
     _offload_executor = ThreadPoolExecutor(max_workers=1)
-finalize(_offload_executor, _offload_executor.shutdown)
+weakref.finalize(_offload_executor, _offload_executor.shutdown)
 
 
 def offload(fn, *args, **kwargs):
@@ -50,10 +50,7 @@ def _to_frames():
             logger.exception(e)
             raise
 
-    if PY3:
-        res = yield offload(_to_frames)
-    else:  # distributed/deploy/tests/test_adaptive.py::test_get_scale_up_kwargs fails on Py27.  Don't know why
-        res = _to_frames()
+    res = yield offload(_to_frames)
 
     raise gen.Return(res)
 
diff --git a/distributed/compatibility.py b/distributed/compatibility.py
index f3a85973802..fb79353d24b 100644
--- a/distributed/compatibility.py
+++ b/distributed/compatibility.py
@@ -1,256 +1,11 @@
 from __future__ import print_function, division, absolute_import
 
 import logging
+import platform
 import sys
 
-# flake8: noqa
-
-if sys.version_info[0] == 2:
-    from Queue import Queue, Empty
-    from io import BytesIO
-    from thread import get_ident as get_thread_identity
-    from inspect import getargspec
-    from cgi import escape as html_escape
-    from collections import Iterator, Mapping, Set, MutableMapping
-    from fractions import gcd
-
-    reload = reload
-    unicode = unicode
-    PY2 = True
-    PY3 = False
-    ConnectionRefusedError = OSError
-    FileExistsError = OSError
-
-    class StopAsyncIteration(Exception):
-        pass
-
-    import gzip
-
-    def gzip_decompress(b):
-        f = gzip.GzipFile(fileobj=BytesIO(b))
-        result = f.read()
-        f.close()
-        return result
-
-    def gzip_compress(b):
-        bio = BytesIO()
-        f = gzip.GzipFile(fileobj=bio, mode="w")
-        f.write(b)
-        f.close()
-        bio.seek(0)
-        result = bio.read()
-        return result
-
-    def isqueue(o):
-        return (
-            hasattr(o, "queue") and hasattr(o, "__module__") and o.__module__ == "Queue"
-        )
-
-    def invalidate_caches():
-        pass
-
-    def cache_from_source(path):
-        import os
-
-        name, ext = os.path.splitext(path)
-        return name + ".pyc"
-
-    logging_names = logging._levelNames
-
-    def iscoroutinefunction(func):
-        return False
-
-
-if sys.version_info[0] == 3:
-    from asyncio import iscoroutinefunction
-    from collections.abc import Iterator, Mapping, Set, MutableMapping
-    from queue import Queue, Empty
-    from importlib import reload
-    from threading import get_ident as get_thread_identity
-    from importlib import invalidate_caches
-    from importlib.util import cache_from_source
-    from inspect import getfullargspec as getargspec
-    from html import escape as html_escape
-    from math import gcd
-
-    PY2 = False
-    PY3 = True
-    unicode = str
-    from gzip import decompress as gzip_decompress
-    from gzip import compress as gzip_compress
-
-    ConnectionRefusedError = ConnectionRefusedError
-    FileExistsError = FileExistsError
-    StopAsyncIteration = StopAsyncIteration
-
-    def isqueue(o):
-        return isinstance(o, Queue)
-
-    logging_names = logging._levelToName.copy()
-    logging_names.update(logging._nameToLevel)
-
-
-import platform
+logging_names = logging._levelToName.copy()
+logging_names.update(logging._nameToLevel)
 
 PYPY = platform.python_implementation().lower() == "pypy"
 WINDOWS = sys.platform.startswith("win")
-
-
-try:
-    from json.decoder import JSONDecodeError
-except (ImportError, AttributeError):
-    JSONDecodeError = ValueError
-
-try:
-    from functools import singledispatch
-except ImportError:
-    from singledispatch import singledispatch
-
-try:
-    from weakref import finalize
-except ImportError:
-    # Backported from Python 3.6
-    import itertools
-    from weakref import ref
-
-    class finalize(object):
-        """Class for finalization of weakrefable objects
-
-        finalize(obj, func, *args, **kwargs) returns a callable finalizer
-        object which will be called when obj is garbage collected. The
-        first time the finalizer is called it evaluates func(*arg, **kwargs)
-        and returns the result. After this the finalizer is dead, and
-        calling it just returns None.
-
-        When the program exits any remaining finalizers for which the
-        atexit attribute is true will be run in reverse order of creation.
-        By default atexit is true.
-        """
-
-        # Finalizer objects don't have any state of their own.  They are
-        # just used as keys to lookup _Info objects in the registry.  This
-        # ensures that they cannot be part of a ref-cycle.
-
-        __slots__ = ()
-        _registry = {}
-        _shutdown = False
-        _index_iter = itertools.count()
-        _dirty = False
-        _registered_with_atexit = False
-
-        class _Info:
-            __slots__ = ("weakref", "func", "args", "kwargs", "atexit", "index")
-
-        def __init__(self, obj, func, *args, **kwargs):
-            if not self._registered_with_atexit:
-                # We may register the exit function more than once because
-                # of a thread race, but that is harmless
-                import atexit
-
-                atexit.register(self._exitfunc)
-                finalize._registered_with_atexit = True
-            info = self._Info()
-            info.weakref = ref(obj, self)
-            info.func = func
-            info.args = args
-            info.kwargs = kwargs or None
-            info.atexit = True
-            info.index = next(self._index_iter)
-            self._registry[self] = info
-            finalize._dirty = True
-
-        def __call__(self, _=None):
-            """If alive then mark as dead and return func(*args, **kwargs);
-            otherwise return None"""
-            info = self._registry.pop(self, None)
-            if info and not self._shutdown:
-                return info.func(*info.args, **(info.kwargs or {}))
-
-        def detach(self):
-            """If alive then mark as dead and return (obj, func, args, kwargs);
-            otherwise return None"""
-            info = self._registry.get(self)
-            obj = info and info.weakref()
-            if obj is not None and self._registry.pop(self, None):
-                return (obj, info.func, info.args, info.kwargs or {})
-
-        def peek(self):
-            """If alive then return (obj, func, args, kwargs);
-            otherwise return None"""
-            info = self._registry.get(self)
-            obj = info and info.weakref()
-            if obj is not None:
-                return (obj, info.func, info.args, info.kwargs or {})
-
-        @property
-        def alive(self):
-            """Whether finalizer is alive"""
-            return self in self._registry
-
-        @property
-        def atexit(self):
-            """Whether finalizer should be called at exit"""
-            info = self._registry.get(self)
-            return bool(info) and info.atexit
-
-        @atexit.setter
-        def atexit(self, value):
-            info = self._registry.get(self)
-            if info:
-                info.atexit = bool(value)
-
-        def __repr__(self):
-            info = self._registry.get(self)
-            obj = info and info.weakref()
-            if obj is None:
-                return "<%s object at %#x; dead>" % (type(self).__name__, id(self))
-            else:
-                return "<%s object at %#x; for %r at %#x>" % (
-                    type(self).__name__,
-                    id(self),
-                    type(obj).__name__,
-                    id(obj),
-                )
-
-        @classmethod
-        def _select_for_exit(cls):
-            # Return live finalizers marked for exit, oldest first
-            L = [(f, i) for (f, i) in cls._registry.items() if i.atexit]
-            L.sort(key=lambda item: item[1].index)
-            return [f for (f, i) in L]
-
-        @classmethod
-        def _exitfunc(cls):
-            # At shutdown invoke finalizers for which atexit is true.
-            # This is called once all other non-daemonic threads have been
-            # joined.
-            reenable_gc = False
-            try:
-                if cls._registry:
-                    import gc
-
-                    if gc.isenabled():
-                        reenable_gc = True
-                        gc.disable()
-                    pending = None
-                    while True:
-                        if pending is None or finalize._dirty:
-                            pending = cls._select_for_exit()
-                            finalize._dirty = False
-                        if not pending:
-                            break
-                        f = pending.pop()
-                        try:
-                            # gc is disabled, so (assuming no daemonic
-                            # threads) the following is the only line in
-                            # this function which might trigger creation
-                            # of a new finalizer
-                            f()
-                        except Exception:
-                            sys.excepthook(*sys.exc_info())
-                        assert f not in cls._registry
-            finally:
-                # prevent any more finalizers from executing during shutdown
-                finalize._shutdown = True
-                if reenable_gc:
-                    gc.enable()
diff --git a/distributed/core.py b/distributed/core.py
index d8a34859359..7db7b3e29e5 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -6,6 +6,7 @@
 from functools import partial
 import logging
 import six
+import threading
 import traceback
 import uuid
 import weakref
@@ -17,7 +18,6 @@
 from tornado.ioloop import IOLoop
 from tornado.locks import Event
 
-from .compatibility import get_thread_identity
 from .comm import (
     connect,
     listen,
@@ -207,7 +207,7 @@ def stop():
         self.thread_id = 0
 
         def set_thread_ident():
-            self.thread_id = get_thread_identity()
+            self.thread_id = threading.get_ident()
 
         self.io_loop.add_callback(set_thread_ident)
 
diff --git a/distributed/dashboard/utils.py b/distributed/dashboard/utils.py
index a9b31345ca9..8e6b5ff0b9c 100644
--- a/distributed/dashboard/utils.py
+++ b/distributed/dashboard/utils.py
@@ -7,13 +7,11 @@
 from tornado import web
 from toolz import partition
 
-from ..compatibility import PY2
-
 BOKEH_VERSION = LooseVersion(bokeh.__version__)
 dirname = os.path.dirname(__file__)
 
 
-if BOKEH_VERSION >= "1.0.0" and not PY2:
+if BOKEH_VERSION >= "1.0.0":
     # This decorator is only available in bokeh >= 1.0.0, and doesn't work for
     # callbacks in Python 2, since the signature introspection won't line up.
     from bokeh.core.properties import without_property_validation
diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
index f5d991cd737..58c6ce73644 100644
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -1,5 +1,6 @@
 from datetime import timedelta
 import logging
+import threading
 from weakref import ref
 
 from dask.utils import format_bytes
@@ -7,7 +8,6 @@
 
 from .adaptive import Adaptive
 
-from ..compatibility import get_thread_identity
 from ..utils import (
     PeriodicCallback,
     log_errors,
@@ -231,7 +231,7 @@ def asynchronous(self):
             self._asynchronous
             or getattr(thread_state, "asynchronous", False)
             or hasattr(self.loop, "_thread_identity")
-            and self.loop._thread_identity == get_thread_identity()
+            and self.loop._thread_identity == threading.get_ident()
         )
 
     def sync(self, func, *args, asynchronous=None, callback_timeout=None, **kwargs):
diff --git a/distributed/diagnostics/progressbar.py b/distributed/diagnostics/progressbar.py
index 8d57da779c6..4c9b781f61c 100644
--- a/distributed/diagnostics/progressbar.py
+++ b/distributed/diagnostics/progressbar.py
@@ -1,6 +1,7 @@
 from __future__ import print_function, division, absolute_import
 
 import logging
+import html
 from timeit import default_timer
 import sys
 import weakref
@@ -10,7 +11,6 @@
 
 from .progress import format_time, Progress, MultiProgress
 
-from ..compatibility import html_escape
 from ..core import connect, coerce_to_address, CommClosedError
 from ..client import default_client, futures_of
 from ..protocol.pickle import dumps
@@ -334,7 +334,7 @@ def make_widget(self, all):
                 '<div style="padding: 0px 10px 0px 10px;'
                 " text-align:left; word-wrap: "
                 'break-word;">'
-                + html_escape(key.decode() if isinstance(key, bytes) else key)
+                + html.escape(key.decode() if isinstance(key, bytes) else key)
                 + "</div>"
             )
             for key in all
diff --git a/distributed/diskutils.py b/distributed/diskutils.py
index 395f7828505..32e6be35adb 100644
--- a/distributed/diskutils.py
+++ b/distributed/diskutils.py
@@ -7,11 +7,11 @@
 import shutil
 import stat
 import tempfile
+import weakref
 
 import dask
 
 from . import locket
-from .compatibility import finalize
 
 
 logger = logging.getLogger(__name__)
@@ -73,7 +73,7 @@ def __init__(self, workspace, name=None, prefix=None):
                 raise
             workspace._known_locks.add(self._lock_path)
 
-            self._finalizer = finalize(
+            self._finalizer = weakref.finalize(
                 self,
                 self._finalize,
                 workspace,
@@ -82,7 +82,7 @@ def __init__(self, workspace, name=None, prefix=None):
                 self.dir_path,
             )
         else:
-            self._finalizer = finalize(
+            self._finalizer = weakref.finalize(
                 self, self._finalize, workspace, None, None, self.dir_path
             )
 
diff --git a/distributed/node.py b/distributed/node.py
index 323e2c3e49d..cbf2c00d8f7 100644
--- a/distributed/node.py
+++ b/distributed/node.py
@@ -1,13 +1,13 @@
 from __future__ import print_function, division, absolute_import
 
-import warnings
 import logging
+import warnings
+import weakref
 
 from tornado.ioloop import IOLoop
 from tornado import gen
 import dask
 
-from .compatibility import unicode, finalize
 from .core import Server, ConnectionPool
 from .versions import get_versions
 from .utils import DequeHandler
@@ -97,7 +97,7 @@ def start_services(self, default_listen_ip):
             else:
                 port = 0
 
-            if isinstance(port, (str, unicode)):
+            if isinstance(port, str):
                 port = port.split(":")
 
             if isinstance(port, (tuple, list)):
@@ -143,7 +143,7 @@ def _setup_logging(self, logger):
             logging.Formatter(dask.config.get("distributed.admin.log-format"))
         )
         logger.addHandler(self._deque_handler)
-        finalize(self, logger.removeHandler, self._deque_handler)
+        weakref.finalize(self, logger.removeHandler, self._deque_handler)
 
     def get_logs(self, comm=None, n=None):
         deque_handler = self._deque_handler
diff --git a/distributed/process.py b/distributed/process.py
index e716d754db1..b6e50122c36 100644
--- a/distributed/process.py
+++ b/distributed/process.py
@@ -4,11 +4,11 @@
 from datetime import timedelta
 import logging
 import os
+from queue import Queue as PyQueue
 import re
 import threading
 import weakref
 
-from .compatibility import finalize, Queue as PyQueue
 from .utils import mp_context
 
 from tornado import gen
@@ -112,7 +112,7 @@ def stop_thread(q):
             # We don't join the thread here as a finalizer can be called
             # asynchronously from anywhere
 
-        self._finalizer = finalize(self, stop_thread, q=self._watch_q)
+        self._finalizer = weakref.finalize(self, stop_thread, q=self._watch_q)
         self._finalizer.atexit = False
 
     def _on_exit(self, exitcode):
diff --git a/distributed/profile.py b/distributed/profile.py
index e240a872fb4..7f85f46312b 100644
--- a/distributed/profile.py
+++ b/distributed/profile.py
@@ -37,7 +37,6 @@
 
 from .metrics import time
 from .utils import format_time, color_of, parse_timedelta
-from .compatibility import get_thread_identity
 
 
 def identifier(frame):
@@ -304,7 +303,7 @@ def watch(
     deque
     """
     if thread_id is None:
-        thread_id = get_thread_identity()
+        thread_id = threading.get_ident()
 
     log = deque(maxlen=maxlen)
 
diff --git a/distributed/protocol/numpy.py b/distributed/protocol/numpy.py
index d8da4f204e4..b2375569ef6 100644
--- a/distributed/protocol/numpy.py
+++ b/distributed/protocol/numpy.py
@@ -1,12 +1,12 @@
 from __future__ import print_function, division, absolute_import
 
+import math
 import numpy as np
 
 from .utils import frame_split_size, merge_frames
 from .serialize import dask_serialize, dask_deserialize
 from . import pickle
 
-from ..compatibility import gcd
 from ..utils import log_errors
 
 
@@ -60,13 +60,13 @@ def serialize_numpy_ndarray(x):
         data = x.ravel()
 
     if data.dtype.fields or data.dtype.itemsize > 8:
-        data = data.view("u%d" % gcd(x.dtype.itemsize, 8))
+        data = data.view("u%d" % math.gcd(x.dtype.itemsize, 8))
 
     try:
         data = data.data
     except ValueError:
         # "ValueError: cannot include dtype 'M' in a buffer"
-        data = data.view("u%d" % gcd(x.dtype.itemsize, 8)).data
+        data = data.view("u%d" % math.gcd(x.dtype.itemsize, 8)).data
 
     header = {"dtype": dt, "shape": x.shape, "strides": strides}
 
diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index 26129f4e1c5..9e314703072 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -13,7 +13,6 @@
 import msgpack
 
 from . import pickle
-from ..compatibility import PY2
 from ..utils import has_keyword, typename
 from .compression import maybe_compress, decompress
 from .utils import (
@@ -376,8 +375,6 @@ def serialize_bytelist(x, **kwargs):
 
 def serialize_bytes(x, **kwargs):
     L = serialize_bytelist(x, **kwargs)
-    if PY2:
-        L = [bytes(y) for y in L]
     return b"".join(L)
 
 
diff --git a/distributed/protocol/tests/test_h5py.py b/distributed/protocol/tests/test_h5py.py
index f2f9a6625cb..6bae5b3b8d5 100644
--- a/distributed/protocol/tests/test_h5py.py
+++ b/distributed/protocol/tests/test_h5py.py
@@ -7,25 +7,22 @@
 
 from distributed.protocol import deserialize, serialize
 
-from distributed.utils import PY3, tmpfile
+from distributed.utils import tmpfile
 
 
 def silence_h5py_issue775(func):
     @functools.wraps(func)
     def wrapper():
-        if PY3:
-            try:
-                func()
-            except RuntimeError as e:
-                # https://github.com/h5py/h5py/issues/775
-                if str(e) != "dictionary changed size during iteration":
-                    raise
-                tb = traceback.extract_tb(e.__traceback__)
-                filename, lineno, _, _ = tb[-1]
-                if not filename.endswith("h5py/_objects.pyx"):
-                    raise
-        else:
+        try:
             func()
+        except RuntimeError as e:
+            # https://github.com/h5py/h5py/issues/775
+            if str(e) != "dictionary changed size during iteration":
+                raise
+            tb = traceback.extract_tb(e.__traceback__)
+            filename, lineno, _, _ = tb[-1]
+            if not filename.endswith("h5py/_objects.pyx"):
+                raise
 
     return wrapper
 
diff --git a/distributed/protocol/tests/test_numpy.py b/distributed/protocol/tests/test_numpy.py
index eb39b57c351..ed4e32c1137 100644
--- a/distributed/protocol/tests/test_numpy.py
+++ b/distributed/protocol/tests/test_numpy.py
@@ -6,7 +6,6 @@
 import numpy as np
 import pytest
 
-from distributed.compatibility import PY2
 from distributed.protocol import (
     serialize,
     deserialize,
@@ -79,7 +78,7 @@ def test_dumps_serialize_numpy(x):
     header, frames = serialize(x)
     if "compression" in header:
         frames = decompress(header, frames)
-    buffer_interface = buffer if PY2 else memoryview  # noqa: F821
+    buffer_interface = memoryview
     for frame in frames:
         assert isinstance(frame, (bytes, buffer_interface))
     y = deserialize(header, frames)
diff --git a/distributed/publish.py b/distributed/publish.py
index a21f5ef37ed..ea65efb4e74 100644
--- a/distributed/publish.py
+++ b/distributed/publish.py
@@ -1,5 +1,6 @@
-from distributed.compatibility import MutableMapping
-from distributed.utils import log_errors, tokey
+from collections import MutableMapping
+
+from .utils import log_errors, tokey
 
 
 class PublishExtension(object):
diff --git a/distributed/pubsub.py b/distributed/pubsub.py
index f40c0b15b31..0a4053191eb 100644
--- a/distributed/pubsub.py
+++ b/distributed/pubsub.py
@@ -1,12 +1,12 @@
 from collections import defaultdict, deque
 import datetime
 import logging
+import threading
 import weakref
 
 import tornado.locks
 from tornado import gen
 
-from .compatibility import finalize, get_thread_identity
 from .core import CommClosedError
 from .utils import sync
 from .protocol.serialize import to_serialize
@@ -306,7 +306,7 @@ def __init__(self, name, worker=None, client=None):
         if self.worker:
             pubsub = self.worker.extensions["pubsub"]
             self.loop.add_callback(pubsub.publishers[name].add, self)
-            finalize(self, pubsub.trigger_cleanup)
+            weakref.finalize(self, pubsub.trigger_cleanup)
 
     async def _start(self):
         if self.worker:
@@ -385,7 +385,7 @@ def __init__(self, name, worker=None, client=None):
         else:
             raise Exception()
 
-        finalize(self, pubsub.trigger_cleanup)
+        weakref.finalize(self, pubsub.trigger_cleanup)
 
     async def _get(self, timeout=None):
         if timeout is not None:
@@ -408,7 +408,7 @@ def get(self, timeout=None):
         """ Get a single message """
         if self.client:
             return self.client.sync(self._get, timeout=timeout)
-        elif self.worker.thread_id == get_thread_identity():
+        elif self.worker.thread_id == threading.get_ident():
             return self._get()
         else:
             if self.buffer:  # fastpath
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index b0db6653d2b..8eec8744849 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1,7 +1,7 @@
 from __future__ import print_function, division, absolute_import
 
 import asyncio
-from collections import defaultdict, deque, OrderedDict
+from collections import defaultdict, deque, OrderedDict, Mapping, Set
 from datetime import timedelta
 from functools import partial
 import itertools
@@ -39,7 +39,6 @@
     unparse_host_port,
 )
 from .comm.addressing import address_from_user_args
-from .compatibility import finalize, unicode, Mapping, Set
 from .core import rpc, connect, send_recv, clean_exception, CommClosedError
 from .diagnostics.plugin import SchedulerPlugin
 from . import profile
@@ -1229,7 +1228,7 @@ def del_scheduler_file():
                 if os.path.exists(fn):
                     os.remove(fn)
 
-            finalize(self, del_scheduler_file)
+            weakref.finalize(self, del_scheduler_file)
 
         preload_modules(self.preload, parameter=self, argv=self.preload_argv)
 
@@ -2124,7 +2123,7 @@ def validate_state(self, allow_overlap=False):
             raise ValueError("Workers not the same in all collections")
 
         for w, ws in self.workers.items():
-            assert isinstance(w, (str, unicode)), (type(w), w)
+            assert isinstance(w, str), (type(w), w)
             assert isinstance(ws, WorkerState), (type(ws), ws)
             assert ws.address == w
             if not ws.processing:
@@ -3706,7 +3705,7 @@ def transition_processing_memory(
         try:
             ts = self.tasks[key]
             assert worker
-            assert isinstance(worker, (str, unicode))
+            assert isinstance(worker, str)
 
             if self.validate:
                 assert ts.processing_on
diff --git a/distributed/submit.py b/distributed/submit.py
index bdbe3251a9d..f7e0a2f70aa 100644
--- a/distributed/submit.py
+++ b/distributed/submit.py
@@ -11,11 +11,9 @@
 
 from tornado.ioloop import IOLoop
 
-from distributed import rpc
-from distributed.compatibility import unicode
-from distributed.core import Server
-from distributed.security import Security
-from distributed.utils import get_ip
+from .core import rpc, Server
+from .security import Security
+from .utils import get_ip
 
 
 logger = logging.getLogger("distributed.remote")
@@ -62,7 +60,7 @@ def execute(self, stream=None, filename=None):
 
     def upload_file(self, stream, filename=None, file_payload=None):
         out_filename = os.path.join(self.local_dir, filename)
-        if isinstance(file_payload, unicode):
+        if isinstance(file_payload, str):
             file_payload = file_payload.encode()
         with open(out_filename, "wb") as f:
             f.write(file_payload)
diff --git a/distributed/tests/test_as_completed.py b/distributed/tests/test_as_completed.py
index aa53b9b993a..911ff388e06 100644
--- a/distributed/tests/test_as_completed.py
+++ b/distributed/tests/test_as_completed.py
@@ -1,5 +1,7 @@
 from concurrent.futures import CancelledError
+from collections import Iterator
 from operator import add
+import queue
 import random
 from time import sleep
 
@@ -7,7 +9,6 @@
 from tornado import gen
 
 from distributed.client import _as_completed, as_completed, _first_completed
-from distributed.compatibility import Empty, StopAsyncIteration, Queue, Iterator
 from distributed.utils_test import gen_cluster, inc, throws
 from distributed.utils_test import client, cluster_fixture, loop  # noqa: F401
 
@@ -18,11 +19,11 @@ def test__as_completed(c, s, a, b):
     y = c.submit(inc, 1)
     z = c.submit(inc, 2)
 
-    queue = Queue()
-    yield _as_completed([x, y, z], queue)
+    q = queue.Queue()
+    yield _as_completed([x, y, z], q)
 
-    assert queue.qsize() == 3
-    assert {queue.get(), queue.get(), queue.get()} == {x, y, z}
+    assert q.qsize() == 3
+    assert {q.get(), q.get(), q.get()} == {x, y, z}
 
     result = yield _first_completed([x, y, z])
     assert result in [x, y, z]
@@ -112,7 +113,7 @@ def test_as_completed_cancel(client):
     assert next(ac) is x or y
     assert next(ac) is y or x
 
-    with pytest.raises(Empty):
+    with pytest.raises(queue.Empty):
         ac.queue.get(timeout=0.1)
 
     res = list(as_completed([x, y, x]))
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index bf05875fdee..5e2ade3f247 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -51,7 +51,7 @@
     futures_of,
     temp_default_client,
 )
-from distributed.compatibility import PY3, WINDOWS
+from distributed.compatibility import WINDOWS
 
 from distributed.metrics import time
 from distributed.scheduler import Scheduler, KilledWorker
@@ -4833,7 +4833,6 @@ def test_bytes_keys(c, s, a, b):
 
 @gen_cluster(client=True)
 def test_unicode_ascii_keys(c, s, a, b):
-    # cross-version unicode type (py2: unicode, py3: str)
     uni_type = type(u"")
     key = u"inc-123"
     future = c.submit(inc, 1, key=key)
@@ -4846,7 +4845,6 @@ def test_unicode_ascii_keys(c, s, a, b):
 
 @gen_cluster(client=True)
 def test_unicode_keys(c, s, a, b):
-    # cross-version unicode type (py2: unicode, py3: str)
     uni_type = type(u"")
     key = u"inc-123\u03bc"
     future = c.submit(inc, 1, key=key)
@@ -5036,12 +5034,7 @@ def test_client_async_before_loop_starts():
 
 
 @pytest.mark.slow
-@gen_cluster(
-    client=True,
-    Worker=Nanny if PY3 else Worker,
-    timeout=60,
-    nthreads=[("127.0.0.1", 3)] * 2,
-)
+@gen_cluster(client=True, Worker=Nanny, timeout=60, nthreads=[("127.0.0.1", 3)] * 2)
 def test_nested_compute(c, s, a, b):
     def fib(x):
         assert get_worker().get_current_task()
diff --git a/distributed/tests/test_compatibility.py b/distributed/tests/test_compatibility.py
deleted file mode 100644
index 42eae448aa1..00000000000
--- a/distributed/tests/test_compatibility.py
+++ /dev/null
@@ -1,38 +0,0 @@
-from __future__ import print_function, division, absolute_import
-
-from distributed.compatibility import gzip_compress, gzip_decompress, finalize
-
-
-def test_gzip():
-    b = b"Hello, world!"
-    c = gzip_compress(b)
-    d = gzip_decompress(c)
-    assert b == d
-
-
-def test_finalize():
-    class C(object):
-        pass
-
-    l = []
-
-    def cb(value):
-        l.append(value)
-
-    o = C()
-    f = finalize(o, cb, 1)
-    assert f in f._select_for_exit()
-    f.atexit = False
-    assert f not in f._select_for_exit()
-    assert not l
-    del o
-    assert l.pop() == 1
-
-    o = C()
-    fin = finalize(o, cb, 2)
-    assert fin.alive
-    fin()
-    assert not fin.alive
-    assert l.pop() == 2
-    del o
-    assert not l
diff --git a/distributed/tests/test_core.py b/distributed/tests/test_core.py
index 38a43a1a5c8..82e4c709be5 100644
--- a/distributed/tests/test_core.py
+++ b/distributed/tests/test_core.py
@@ -3,13 +3,13 @@
 from contextlib import contextmanager
 import os
 import socket
+import threading
 import weakref
 
 from tornado import gen
 import pytest
 
 import dask
-from distributed.compatibility import finalize, get_thread_identity
 from distributed.core import (
     pingpong,
     Server,
@@ -63,7 +63,7 @@ class CountedObject(object):
     def __new__(cls):
         cls.n_instances += 1
         obj = object.__new__(cls)
-        finalize(obj, cls._finalize)
+        weakref.finalize(obj, cls._finalize)
         return obj
 
     @classmethod
@@ -702,7 +702,7 @@ def f():
 
 @gen_cluster()
 def test_thread_id(s, a, b):
-    assert s.thread_id == a.thread_id == b.thread_id == get_thread_identity()
+    assert s.thread_id == a.thread_id == b.thread_id == threading.get_ident()
 
 
 @gen_test()
diff --git a/distributed/tests/test_diskutils.py b/distributed/tests/test_diskutils.py
index a6dcf3497a3..561a4cd408b 100644
--- a/distributed/tests/test_diskutils.py
+++ b/distributed/tests/test_diskutils.py
@@ -3,6 +3,7 @@
 import functools
 import gc
 import os
+import queue
 import shutil
 import subprocess
 import sys
@@ -12,7 +13,7 @@
 import pytest
 
 import dask
-from distributed.compatibility import Empty, WINDOWS
+from distributed.compatibility import WINDOWS
 from distributed.diskutils import WorkSpace
 from distributed.metrics import time
 from distributed.utils import mp_context
@@ -258,7 +259,7 @@ def _test_workspace_concurrency(tmpdir, timeout, max_procs):
     # Any errors?
     try:
         err = err_q.get_nowait()
-    except Empty:
+    except queue.Empty:
         pass
     else:
         raise err
@@ -266,7 +267,7 @@ def _test_workspace_concurrency(tmpdir, timeout, max_procs):
     try:
         while True:
             n_purged += purged_q.get_nowait()
-    except Empty:
+    except queue.Empty:
         pass
     # We attempted to purge most directories at some point
     assert n_purged >= 0.5 * n_created > 0
diff --git a/distributed/tests/test_metrics.py b/distributed/tests/test_metrics.py
index d1eb4a1dad0..cdb4b8ee478 100644
--- a/distributed/tests/test_metrics.py
+++ b/distributed/tests/test_metrics.py
@@ -5,7 +5,6 @@
 import time
 
 from distributed import metrics
-from distributed.compatibility import PY3
 from distributed.utils_test import run_for
 
 
@@ -37,12 +36,11 @@ def test_process_time():
     dt = metrics.process_time() - start
     assert dt >= 0.05
 
-    if PY3:
-        # Sleep time not counted
-        start = metrics.process_time()
-        time.sleep(0.1)
-        dt = metrics.process_time() - start
-        assert dt <= 0.05
+    # Sleep time not counted
+    start = metrics.process_time()
+    time.sleep(0.1)
+    dt = metrics.process_time() - start
+    assert dt <= 0.05
 
 
 def test_thread_time():
@@ -51,18 +49,17 @@ def test_thread_time():
     dt = metrics.thread_time() - start
     assert 0.03 <= dt <= 0.2
 
-    if PY3:
-        # Sleep time not counted
+    # Sleep time not counted
+    start = metrics.thread_time()
+    time.sleep(0.1)
+    dt = metrics.thread_time() - start
+    assert dt <= 0.05
+
+    if sys.platform == "linux":
+        # Always per-thread on Linux
+        t = threading.Thread(target=run_for, args=(0.1,))
         start = metrics.thread_time()
-        time.sleep(0.1)
+        t.start()
+        t.join()
         dt = metrics.thread_time() - start
         assert dt <= 0.05
-
-        if sys.platform == "linux":
-            # Always per-thread on Linux
-            t = threading.Thread(target=run_for, args=(0.1,))
-            start = metrics.thread_time()
-            t.start()
-            t.join()
-            dt = metrics.thread_time() - start
-            assert dt <= 0.05
diff --git a/distributed/tests/test_profile.py b/distributed/tests/test_profile.py
index ee49f130027..a022600d819 100644
--- a/distributed/tests/test_profile.py
+++ b/distributed/tests/test_profile.py
@@ -4,7 +4,7 @@
 from toolz import first
 import threading
 
-from distributed.compatibility import get_thread_identity, WINDOWS
+from distributed.compatibility import WINDOWS
 from distributed import metrics
 from distributed.profile import (
     process,
@@ -164,7 +164,7 @@ def test_merge_empty():
 
 
 def test_call_stack():
-    frame = sys._current_frames()[get_thread_identity()]
+    frame = sys._current_frames()[threading.get_ident()]
     L = call_stack(frame)
     assert isinstance(L, list)
     assert all(isinstance(s, str) for s in L)
@@ -172,7 +172,7 @@ def test_call_stack():
 
 
 def test_identifier():
-    frame = sys._current_frames()[get_thread_identity()]
+    frame = sys._current_frames()[threading.get_ident()]
     assert identifier(frame) == identifier(frame)
     assert identifier(None) == identifier(None)
 
diff --git a/distributed/tests/test_utils.py b/distributed/tests/test_utils.py
index 9e3a1d90c4b..c547834626d 100644
--- a/distributed/tests/test_utils.py
+++ b/distributed/tests/test_utils.py
@@ -3,6 +3,7 @@
 import datetime
 from functools import partial
 import io
+import queue
 import socket
 import sys
 from time import sleep
@@ -15,7 +16,6 @@
 from tornado.ioloop import IOLoop
 
 import dask
-from distributed.compatibility import Queue, Empty, PY2
 from distributed.metrics import time
 from distributed.utils import (
     All,
@@ -278,8 +278,6 @@ def f():
 
 def test_ensure_bytes():
     data = [b"1", "1", memoryview(b"1"), bytearray(b"1")]
-    if PY2:
-        data.append(buffer(b"1"))  # noqa: F821
     for d in data:
         result = ensure_bytes(d)
         assert isinstance(result, bytes)
@@ -319,7 +317,7 @@ def assert_running(loop):
     """
     Raise if the given IOLoop is not running.
     """
-    q = Queue()
+    q = queue.Queue()
     loop.add_callback(q.put, 42)
     assert q.get(timeout=1) == 42
 
@@ -328,14 +326,14 @@ def assert_not_running(loop):
     """
     Raise if the given IOLoop is running.
     """
-    q = Queue()
+    q = queue.Queue()
     try:
         loop.add_callback(q.put, 42)
     except RuntimeError:
         # On AsyncIOLoop, can't add_callback() after the loop is closed
         pass
     else:
-        with pytest.raises(Empty):
+        with pytest.raises(queue.Empty):
             q.get(timeout=0.02)
 
 
diff --git a/distributed/tests/test_utils_perf.py b/distributed/tests/test_utils_perf.py
index 55b250273c0..95fa816a75b 100644
--- a/distributed/tests/test_utils_perf.py
+++ b/distributed/tests/test_utils_perf.py
@@ -8,7 +8,6 @@
 
 import pytest
 
-from distributed.compatibility import PY2
 from distributed.metrics import thread_time
 from distributed.utils_perf import FractionalTimer, GCDiagnosis, disable_gc_diagnosis
 from distributed.utils_test import captured_logger, run_for
@@ -84,7 +83,6 @@ def enable_gc_diagnosis_and_log(diag, level="INFO"):
             gc.enable()
 
 
-@pytest.mark.skipif(PY2, reason="requires Python 3")
 def test_gc_diagnosis_cpu_time():
     diag = GCDiagnosis(warn_over_frac=0.75)
     diag.N_SAMPLES = 3  # shorten tests
@@ -115,7 +113,6 @@ def test_gc_diagnosis_cpu_time():
 
 
 @pytest.mark.xfail(reason="unknown")
-@pytest.mark.skipif(PY2, reason="requires Python 3")
 def test_gc_diagnosis_rss_win():
     diag = GCDiagnosis(info_over_rss_win=10e6)
 
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 96c673bf69a..c7337d36424 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -2,6 +2,7 @@
 
 from concurrent.futures import ThreadPoolExecutor
 from datetime import timedelta
+import importlib
 import logging
 import multiprocessing
 from numbers import Number
@@ -32,7 +33,7 @@
     Reschedule,
     wait,
 )
-from distributed.compatibility import WINDOWS, cache_from_source
+from distributed.compatibility import WINDOWS
 from distributed.core import rpc
 from distributed.scheduler import Scheduler
 from distributed.metrics import time
@@ -219,7 +220,7 @@ def test_upload_file_pyc(c, s, w):
             import foo
 
             assert foo.f() == 123
-            pyc = cache_from_source(os.path.join(dirname, "foo.py"))
+            pyc = importlib.util.cache_from_source(os.path.join(dirname, "foo.py"))
             assert os.path.exists(pyc)
             yield c.upload_file(pyc)
 
diff --git a/distributed/threadpoolexecutor.py b/distributed/threadpoolexecutor.py
index d2d4e3b7921..f4cae3fd88e 100644
--- a/distributed/threadpoolexecutor.py
+++ b/distributed/threadpoolexecutor.py
@@ -23,9 +23,9 @@
 from __future__ import print_function, division, absolute_import
 
 from . import _concurrent_futures_thread as thread
-from .compatibility import Empty
 import os
 import logging
+import queue
 import threading
 import itertools
 
@@ -51,7 +51,7 @@ def _worker(executor, work_queue):
                     break
             try:
                 task = work_queue.get(timeout=1)
-            except Empty:
+            except queue.Empty:
                 continue
             if task is not None:  # sentinel
                 task.run()
diff --git a/distributed/utils.py b/distributed/utils.py
index 71e4a09d2a1..ab45350bf61 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -12,13 +12,14 @@
 import logging
 import multiprocessing
 from numbers import Number
-import operator
 import os
 import re
 import shutil
 import socket
 from time import sleep
-from importlib import import_module
+import importlib
+from importlib.util import cache_from_source
+import inspect
 import sys
 import tempfile
 import threading
@@ -28,8 +29,6 @@
 import six
 import tblib.pickling_support
 
-from .compatibility import cache_from_source, getargspec, invalidate_caches, reload
-
 try:
     import resource
 except ImportError:
@@ -50,7 +49,7 @@
 except ImportError:
     PollIOLoop = None  # dropped in tornado 6.0
 
-from .compatibility import PY3, PY2, get_thread_identity, unicode
+from .compatibility import PYPY, WINDOWS
 from .metrics import time
 
 
@@ -66,7 +65,9 @@
 
 
 def _initialize_mp_context():
-    if PY3 and not sys.platform.startswith("win") and "PyPy" not in sys.version:
+    if WINDOWS or PYPY:
+        return multiprocessing
+    else:
         method = dask.config.get("distributed.worker.multiprocessing-method")
         ctx = multiprocessing.get_context(method)
         # Makes the test suite much faster
@@ -74,10 +75,7 @@ def _initialize_mp_context():
         if "pkg_resources" in sys.modules:
             preload.append("pkg_resources")
         ctx.set_forkserver_preload(preload)
-    else:
-        ctx = multiprocessing
-
-    return ctx
+        return ctx
 
 
 mp_context = _initialize_mp_context()
@@ -99,7 +97,7 @@ def has_arg(func, argname):
     """
     while True:
         try:
-            if argname in getargspec(func).args:
+            if argname in inspect.getfullargspec(func).args:
                 return True
         except TypeError:
             break
@@ -298,14 +296,14 @@ def sync(loop, func, *args, callback_timeout=None, **kwargs):
         pass
 
     e = threading.Event()
-    main_tid = get_thread_identity()
+    main_tid = threading.get_ident()
     result = [None]
     error = [False]
 
     @gen.coroutine
     def f():
         try:
-            if main_tid == get_thread_identity():
+            if main_tid == threading.get_ident():
                 raise RuntimeError("sync() called from thread of running loop")
             yield gen.moment
             thread_state.asynchronous = True
@@ -552,6 +550,7 @@ def is_kernel():
 hex_pattern = re.compile("[a-f]+")
 
 
+@functools.lru_cache(100000)
 def key_split(s):
     """
     >>> key_split('x')
@@ -606,102 +605,48 @@ def key_split(s):
         return "Other"
 
 
-try:
-    from functools import lru_cache
-except ImportError:
-    lru_cache = False
-    pass
-else:
-    key_split = lru_cache(100000)(key_split)
-
-if PY3:
-
-    def key_split_group(x):
-        """A more fine-grained version of key_split
-
-        >>> key_split_group('x')
-        'x'
-        >>> key_split_group('x-1')
-        'x-1'
-        >>> key_split_group('x-1-2-3')
-        'x-1-2-3'
-        >>> key_split_group(('x-2', 1))
-        'x-2'
-        >>> key_split_group("('x-2', 1)")
-        'x-2'
-        >>> key_split_group('hello-world-1')
-        'hello-world-1'
-        >>> key_split_group(b'hello-world-1')
-        'hello-world-1'
-        >>> key_split_group('ae05086432ca935f6eba409a8ecd4896')
-        'data'
-        >>> key_split_group('<module.submodule.myclass object at 0xdaf372')
-        'myclass'
-        >>> key_split_group(None)
-        'Other'
-        >>> key_split_group('x-abcdefab')  # ignores hex
-        'x-abcdefab'
-        """
-        typ = type(x)
-        if typ is tuple:
-            return x[0]
-        elif typ is str:
-            if x[0] == "(":
-                return x.split(",", 1)[0].strip("()\"'")
-            elif len(x) == 32 and re.match(r"[a-f0-9]{32}", x):
-                return "data"
-            elif x[0] == "<":
-                return x.strip("<>").split()[0].split(".")[-1]
-            else:
-                return x
-        elif typ is bytes:
-            return key_split_group(x.decode())
-        else:
-            return "Other"
+def key_split_group(x):
+    """A more fine-grained version of key_split
 
-
-else:
-
-    def key_split_group(x):
-        """A more fine-grained version of key_split
-
-        >>> key_split_group('x')
-        'x'
-        >>> key_split_group('x-1')
-        'x-1'
-        >>> key_split_group('x-1-2-3')
-        'x-1-2-3'
-        >>> key_split_group(('x-2', 1))
-        'x-2'
-        >>> key_split_group("('x-2', 1)")
-        'x-2'
-        >>> key_split_group('hello-world-1')
-        'hello-world-1'
-        >>> key_split_group(b'hello-world-1')
-        'hello-world-1'
-        >>> key_split_group('ae05086432ca935f6eba409a8ecd4896')
-        'data'
-        >>> key_split_group('<module.submodule.myclass object at 0xdaf372')
-        'myclass'
-        >>> key_split_group(None)
-        'Other'
-        >>> key_split_group('x-abcdefab')  # ignores hex
-        'x-abcdefab'
-        """
-        typ = type(x)
-        if typ is tuple:
-            return x[0]
-        elif typ is str or typ is unicode:
-            if x[0] == "(":
-                return x.split(",", 1)[0].strip("()\"'")
-            elif len(x) == 32 and re.match(r"[a-f0-9]{32}", x):
-                return "data"
-            elif x[0] == "<":
-                return x.strip("<>").split()[0].split(".")[-1]
-            else:
-                return x
+    >>> key_split_group('x')
+    'x'
+    >>> key_split_group('x-1')
+    'x-1'
+    >>> key_split_group('x-1-2-3')
+    'x-1-2-3'
+    >>> key_split_group(('x-2', 1))
+    'x-2'
+    >>> key_split_group("('x-2', 1)")
+    'x-2'
+    >>> key_split_group('hello-world-1')
+    'hello-world-1'
+    >>> key_split_group(b'hello-world-1')
+    'hello-world-1'
+    >>> key_split_group('ae05086432ca935f6eba409a8ecd4896')
+    'data'
+    >>> key_split_group('<module.submodule.myclass object at 0xdaf372')
+    'myclass'
+    >>> key_split_group(None)
+    'Other'
+    >>> key_split_group('x-abcdefab')  # ignores hex
+    'x-abcdefab'
+    """
+    typ = type(x)
+    if typ is tuple:
+        return x[0]
+    elif typ is str:
+        if x[0] == "(":
+            return x.split(",", 1)[0].strip("()\"'")
+        elif len(x) == 32 and re.match(r"[a-f0-9]{32}", x):
+            return "data"
+        elif x[0] == "<":
+            return x.strip("<>").split()[0].split(".")[-1]
         else:
-            return "Other"
+            return x
+    elif typ is bytes:
+        return key_split_group(x.decode())
+    else:
+        return "Other"
 
 
 @contextmanager
@@ -810,7 +755,7 @@ def tokey(o):
     '1'
     """
     typ = type(o)
-    if typ is unicode or typ is bytes:
+    if typ is str or typ is bytes:
         return o
     else:
         return str(o)
@@ -820,7 +765,7 @@ def validate_key(k):
     """Validate a key as received on a stream.
     """
     typ = type(k)
-    if typ is not unicode and typ is not bytes:
+    if typ is not str and typ is not bytes:
         raise TypeError("Unexpected key type %s (value: %r)" % (typ, k))
 
 
@@ -970,7 +915,7 @@ def ensure_bytes(s):
         return s
     if isinstance(s, memoryview):
         return s.tobytes()
-    if isinstance(s, bytearray) or PY2 and isinstance(s, buffer):  # noqa: F821
+    if isinstance(s, bytearray):  # noqa: F821
         return bytes(s)
     if hasattr(s, "encode"):
         return s.encode()
@@ -1075,13 +1020,13 @@ def import_file(path):
     if not names_to_import:
         logger.warning("Found nothing to import from %s", filename)
     else:
-        invalidate_caches()
+        importlib.invalidate_caches()
         if tmp_python_path is not None:
             sys.path.insert(0, tmp_python_path)
         try:
             for name in names_to_import:
                 logger.info("Reload module %s from %s file", name, ext)
-                loaded.append(reload(import_module(name)))
+                loaded.append(importlib.reload(importlib.import_module(name)))
         finally:
             if tmp_python_path is not None:
                 sys.path.remove(tmp_python_path)
@@ -1261,32 +1206,15 @@ def asciitable(columns, rows):
     return "\n".join([bar, header, bar, data, bar])
 
 
-if PY2:
-
-    def nbytes(frame, _bytes_like=(bytes, bytearray, buffer)):  # noqa: F821
-        """ Number of bytes of a frame or memoryview """
-        if isinstance(frame, _bytes_like):
-            return len(frame)
-        elif isinstance(frame, memoryview):
-            if frame.shape is None:
-                return frame.itemsize
-            else:
-                return functools.reduce(operator.mul, frame.shape, frame.itemsize)
-        else:
+def nbytes(frame, _bytes_like=(bytes, bytearray)):
+    """ Number of bytes of a frame or memoryview """
+    if isinstance(frame, _bytes_like):
+        return len(frame)
+    else:
+        try:
             return frame.nbytes
-
-
-else:
-
-    def nbytes(frame, _bytes_like=(bytes, bytearray)):
-        """ Number of bytes of a frame or memoryview """
-        if isinstance(frame, _bytes_like):
+        except AttributeError:
             return len(frame)
-        else:
-            try:
-                return frame.nbytes
-            except AttributeError:
-                return len(frame)
 
 
 def PeriodicCallback(callback, callback_time, io_loop=None):
@@ -1402,18 +1330,9 @@ def reset_logger_locks():
         )
 
 
+@functools.lru_cache(1000)
 def has_keyword(func, keyword):
-    if PY3:
-        return keyword in inspect.signature(func).parameters
-    else:
-        # https://stackoverflow.com/questions/50100498/determine-keywords-of-a-tornado-coroutine
-        if gen.is_coroutine_function(func):
-            func = func.__wrapped__
-        return keyword in inspect.getargspec(func).args
-
-
-if lru_cache:
-    has_keyword = lru_cache(1000)(has_keyword)
+    return keyword in inspect.signature(func).parameters
 
 
 # from bokeh.palettes import viridis
diff --git a/distributed/utils_perf.py b/distributed/utils_perf.py
index b1f65256c1e..eb54ea0b381 100644
--- a/distributed/utils_perf.py
+++ b/distributed/utils_perf.py
@@ -7,7 +7,7 @@
 
 from dask.utils import format_bytes
 
-from .compatibility import PY2, PYPY
+from .compatibility import PYPY
 from .metrics import thread_time
 
 
@@ -147,7 +147,7 @@ def __init__(self, warn_over_frac=0.1, info_over_rss_win=10 * 1e6):
         self._enabled = False
 
     def enable(self):
-        if PY2 or PYPY:
+        if PYPY:
             return
         assert not self._enabled
         self._fractional_timer = FractionalTimer(n_samples=self.N_SAMPLES)
@@ -165,7 +165,7 @@ def enable(self):
         self._enabled = True
 
     def disable(self):
-        if PY2 or PYPY:
+        if PYPY:
             return
         assert self._enabled
         gc.callbacks.remove(self._gc_callback)
@@ -232,7 +232,7 @@ def enable_gc_diagnosis():
     """
     Ask to enable global GC diagnosis.
     """
-    if PY2 or PYPY:
+    if PYPY:
         return
     global _gc_diagnosis_users
     with _gc_diagnosis_lock:
@@ -247,7 +247,7 @@ def disable_gc_diagnosis(force=False):
     """
     Ask to disable global GC diagnosis.
     """
-    if PY2 or PYPY:
+    if PYPY:
         return
     global _gc_diagnosis_users
     with _gc_diagnosis_lock:
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index 52cc54b639d..0c7c8958a91 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -6,12 +6,12 @@
 import copy
 from datetime import timedelta
 import functools
-import gc
 from glob import glob
 import itertools
 import logging
 import logging.config
 import os
+import queue
 import re
 import shutil
 import signal
@@ -41,7 +41,7 @@
 from tornado.ioloop import IOLoop
 
 from .client import default_client, _global_clients, Client
-from .compatibility import PY3, Empty, WINDOWS
+from .compatibility import WINDOWS
 from .comm import Comm
 from .comm.utils import offload
 from .config import initialize_logging
@@ -225,11 +225,6 @@ def nodebug(func):
     A decorator to disable debug facilities during timing-sensitive tests.
     Warning: this doesn't affect already created IOLoops.
     """
-    if not PY3:
-        # py.test's runner magic breaks horridly on Python 2
-        # when a test function is wrapped, so avoid it
-        # (incidently, asyncio is irrelevant anyway)
-        return func
 
     @functools.wraps(func)
     def wrapped(*args, **kwargs):
@@ -517,10 +512,6 @@ async def _():
 @contextmanager
 def check_active_rpc(loop, active_rpc_timeout=1):
     active_before = set(rpc.active)
-    if active_before and not PY3:
-        # On Python 2, try to avoid dangling comms before forking workers
-        gc.collect()
-        active_before = set(rpc.active)
     yield
     # Some streams can take a bit of time to notice their peer
     # has closed, and keep a coroutine (*) waiting for a CommClosedError
@@ -664,7 +655,7 @@ def cluster(
         try:
             for worker in workers:
                 worker["address"] = worker["queue"].get(timeout=5)
-        except Empty:
+        except queue.Empty:
             raise pytest.xfail.Exception("Worker failed to start in test")
 
         saddr = scheduler_q.get()
diff --git a/distributed/worker.py b/distributed/worker.py
index 08927eb741d..ae7ee648ff9 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -2,7 +2,7 @@
 
 import asyncio
 import bisect
-from collections import defaultdict, deque
+from collections import defaultdict, deque, MutableMapping
 from datetime import timedelta
 import heapq
 import logging
@@ -35,7 +35,6 @@
 from .comm import get_address_host, connect
 from .comm.utils import offload
 from .comm.addressing import address_from_user_args
-from .compatibility import unicode, get_thread_identity, MutableMapping
 from .core import error_message, CommClosedError, send_recv, pingpong, coerce_to_address
 from .diskutils import WorkSpace
 from .metrics import time
@@ -856,7 +855,7 @@ async def upload_file(self, comm, filename=None, data=None, load=True):
         out_filename = os.path.join(self.local_directory, filename)
 
         def func(data):
-            if isinstance(data, unicode):
+            if isinstance(data, str):
                 data = data.encode()
             with open(out_filename, "wb") as f:
                 f.write(data)
@@ -2303,8 +2302,8 @@ async def execute(self, key, report=False):
                     from .actor import Actor  # TODO: create local actor
 
                     data[k] = Actor(type(self.actors[k]), self.address, k, self)
-            args2 = pack_data(args, data, key_types=(bytes, unicode))
-            kwargs2 = pack_data(kwargs, data, key_types=(bytes, unicode))
+            args2 = pack_data(args, data, key_types=(bytes, str))
+            kwargs2 = pack_data(kwargs, data, key_types=(bytes, str))
             stop = time()
             if stop - start > 0.005:
                 self.startstops[key].append(("disk-read", start, stop))
@@ -2796,7 +2795,7 @@ def get_current_task(self):
         --------
         get_worker
         """
-        return self.active_threads[get_thread_identity()]
+        return self.active_threads[threading.get_ident()]
 
 
 def get_worker():
@@ -2946,7 +2945,7 @@ def parse_memory_limit(memory_limit, nthreads, total_cores=multiprocessing.cpu_c
         if isinstance(memory_limit, float) and memory_limit <= 1:
             memory_limit = int(memory_limit * TOTAL_MEMORY)
 
-    if isinstance(memory_limit, (unicode, str)):
+    if isinstance(memory_limit, str):
         memory_limit = parse_bytes(memory_limit)
     else:
         memory_limit = int(memory_limit)
@@ -3146,7 +3145,7 @@ def apply_function(
     -------
     msg: dictionary with status, result/error, timings, etc..
     """
-    ident = get_thread_identity()
+    ident = threading.get_ident()
     with active_threads_lock:
         active_threads[ident] = key
     thread_state.start_time = time()
@@ -3186,7 +3185,7 @@ def apply_function_actor(
     -------
     msg: dictionary with status, result/error, timings, etc..
     """
-    ident = get_thread_identity()
+    ident = threading.get_ident()
 
     with active_threads_lock:
         active_threads[ident] = key

From cc4fc7d8fe42c41b328ade1db54dd88da904244a Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 29 Jul 2019 13:12:20 -0700
Subject: [PATCH 0385/1550] Remove future imports (#2897)

---
 distributed/__init__.py                                  | 2 --
 distributed/_ipython_utils.py                            | 2 --
 distributed/batched.py                                   | 2 --
 distributed/cfexecutor.py                                | 2 --
 distributed/cli/dask_remote.py                           | 2 --
 distributed/cli/dask_scheduler.py                        | 2 --
 distributed/cli/dask_ssh.py                              | 2 --
 distributed/cli/dask_worker.py                           | 2 --
 distributed/cli/tests/test_dask_scheduler.py             | 2 --
 distributed/cli/tests/test_dask_worker.py                | 2 --
 distributed/cli/tests/test_tls_cli.py                    | 3 ---
 distributed/cli/utils.py                                 | 2 --
 distributed/client.py                                    | 2 --
 distributed/comm/__init__.py                             | 2 --
 distributed/comm/addressing.py                           | 2 --
 distributed/comm/core.py                                 | 2 --
 distributed/comm/inproc.py                               | 2 --
 distributed/comm/registry.py                             | 2 --
 distributed/comm/tcp.py                                  | 2 --
 distributed/comm/tests/test_comms.py                     | 2 --
 distributed/comm/utils.py                                | 2 --
 distributed/compatibility.py                             | 2 --
 distributed/config.py                                    | 2 --
 distributed/core.py                                      | 2 --
 distributed/counter.py                                   | 2 --
 distributed/dashboard/components.py                      | 2 --
 distributed/dashboard/core.py                            | 2 --
 distributed/dashboard/export_tool.py                     | 2 --
 distributed/dashboard/scheduler.py                       | 2 --
 distributed/dashboard/tests/test_components.py           | 2 --
 distributed/dashboard/tests/test_scheduler_bokeh.py      | 2 --
 distributed/dashboard/tests/test_scheduler_bokeh_html.py | 2 --
 distributed/dashboard/tests/test_worker_bokeh.py         | 2 --
 distributed/dashboard/utils.py                           | 2 --
 distributed/dashboard/worker.py                          | 2 --
 distributed/deploy/__init__.py                           | 2 --
 distributed/deploy/adaptive.py                           | 2 --
 distributed/deploy/local.py                              | 2 --
 distributed/deploy/ssh.py                                | 2 --
 distributed/deploy/tests/test_adaptive.py                | 2 --
 distributed/deploy/tests/test_local.py                   | 2 --
 distributed/deploy/tests/test_ssh.py                     | 2 --
 distributed/diagnostics/__init__.py                      | 2 --
 distributed/diagnostics/eventstream.py                   | 2 --
 distributed/diagnostics/graph_layout.py                  | 2 --
 distributed/diagnostics/plugin.py                        | 2 --
 distributed/diagnostics/progress.py                      | 2 --
 distributed/diagnostics/progress_stream.py               | 2 --
 distributed/diagnostics/progressbar.py                   | 2 --
 distributed/diagnostics/task_stream.py                   | 2 --
 distributed/diagnostics/tests/test_eventstream.py        | 2 --
 distributed/diagnostics/tests/test_plugin.py             | 2 --
 distributed/diagnostics/tests/test_progress.py           | 2 --
 distributed/diagnostics/tests/test_progress_stream.py    | 3 ---
 distributed/diagnostics/tests/test_progressbar.py        | 2 --
 distributed/diagnostics/tests/test_task_stream.py        | 2 --
 distributed/diagnostics/tests/test_widgets.py            | 2 --
 distributed/diskutils.py                                 | 2 --
 distributed/lock.py                                      | 2 --
 distributed/metrics.py                                   | 2 --
 distributed/nanny.py                                     | 2 --
 distributed/node.py                                      | 2 --
 distributed/process.py                                   | 2 --
 distributed/proctitle.py                                 | 2 --
 distributed/profile.py                                   | 2 --
 distributed/protocol/__init__.py                         | 2 --
 distributed/protocol/arrow.py                            | 2 --
 distributed/protocol/compression.py                      | 2 --
 distributed/protocol/core.py                             | 2 --
 distributed/protocol/h5py.py                             | 2 --
 distributed/protocol/keras.py                            | 2 --
 distributed/protocol/netcdf4.py                          | 2 --
 distributed/protocol/numpy.py                            | 2 --
 distributed/protocol/pickle.py                           | 2 --
 distributed/protocol/serialize.py                        | 1 -
 distributed/protocol/sparse.py                           | 2 --
 distributed/protocol/tests/test_numpy.py                 | 2 --
 distributed/protocol/tests/test_pandas.py                | 3 ---
 distributed/protocol/tests/test_protocol.py              | 2 --
 distributed/protocol/tests/test_protocol_utils.py        | 2 --
 distributed/protocol/tests/test_serialize.py             | 2 --
 distributed/protocol/utils.py                            | 2 --
 distributed/pytest_resourceleaks.py                      | 2 --
 distributed/queues.py                                    | 2 --
 distributed/recreate_exceptions.py                       | 2 --
 distributed/scheduler.py                                 | 2 --
 distributed/sizeof.py                                    | 2 --
 distributed/stealing.py                                  | 2 --
 distributed/submit.py                                    | 2 --
 distributed/system_monitor.py                            | 2 --
 distributed/tests/test_asyncprocess.py                   | 2 --
 distributed/tests/test_client.py                         | 2 --
 distributed/tests/test_client_executor.py                | 2 --
 distributed/tests/test_collections.py                    | 3 ---
 distributed/tests/test_config.py                         | 2 --
 distributed/tests/test_core.py                           | 2 --
 distributed/tests/test_counter.py                        | 2 --
 distributed/tests/test_diskutils.py                      | 2 --
 distributed/tests/test_failed_workers.py                 | 2 --
 distributed/tests/test_ipython.py                        | 2 --
 distributed/tests/test_locks.py                          | 2 --
 distributed/tests/test_metrics.py                        | 2 --
 distributed/tests/test_nanny.py                          | 2 --
 distributed/tests/test_queues.py                         | 2 --
 distributed/tests/test_resources.py                      | 2 --
 distributed/tests/test_scheduler.py                      | 2 --
 distributed/tests/test_security.py                       | 2 --
 distributed/tests/test_steal.py                          | 6 ++----
 distributed/tests/test_stress.py                         | 2 --
 distributed/tests/test_submit_cli.py                     | 1 -
 distributed/tests/test_system_monitor.py                 | 2 --
 distributed/tests/test_tls_functional.py                 | 4 ----
 distributed/tests/test_utils.py                          | 2 --
 distributed/tests/test_utils_comm.py                     | 2 --
 distributed/tests/test_utils_perf.py                     | 2 --
 distributed/tests/test_utils_test.py                     | 2 --
 distributed/tests/test_variable.py                       | 2 --
 distributed/tests/test_worker.py                         | 2 --
 distributed/tests/test_worker_client.py                  | 2 --
 distributed/threadpoolexecutor.py                        | 2 --
 distributed/utils.py                                     | 2 --
 distributed/utils_comm.py                                | 2 --
 distributed/utils_perf.py                                | 2 --
 distributed/utils_test.py                                | 2 --
 distributed/variable.py                                  | 2 --
 distributed/versions.py                                  | 2 --
 distributed/worker.py                                    | 2 --
 distributed/worker_client.py                             | 2 --
 128 files changed, 2 insertions(+), 262 deletions(-)

diff --git a/distributed/__init__.py b/distributed/__init__.py
index 2a632607cf9..ca36613c815 100644
--- a/distributed/__init__.py
+++ b/distributed/__init__.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 from . import config
 from dask.config import config
 from .actor import Actor, ActorFuture
diff --git a/distributed/_ipython_utils.py b/distributed/_ipython_utils.py
index 512f8911588..1a999833786 100644
--- a/distributed/_ipython_utils.py
+++ b/distributed/_ipython_utils.py
@@ -4,8 +4,6 @@
 after which we can import them instead of having our own definitions.
 """
 
-from __future__ import print_function
-
 import atexit
 import os
 
diff --git a/distributed/batched.py b/distributed/batched.py
index e17d7b1f1bd..a3207b333ef 100644
--- a/distributed/batched.py
+++ b/distributed/batched.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 from collections import deque
 import logging
 
diff --git a/distributed/cfexecutor.py b/distributed/cfexecutor.py
index eb7bbf05646..34350462f8b 100644
--- a/distributed/cfexecutor.py
+++ b/distributed/cfexecutor.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import concurrent.futures as cf
 import weakref
 
diff --git a/distributed/cli/dask_remote.py b/distributed/cli/dask_remote.py
index 29cc5c3c784..3118da84ae7 100644
--- a/distributed/cli/dask_remote.py
+++ b/distributed/cli/dask_remote.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import click
 from distributed.cli.utils import check_python_3, install_signal_handlers
 from distributed.submit import _remote
diff --git a/distributed/cli/dask_scheduler.py b/distributed/cli/dask_scheduler.py
index 54ecd69e595..07e4c98e267 100755
--- a/distributed/cli/dask_scheduler.py
+++ b/distributed/cli/dask_scheduler.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import atexit
 import logging
 import gc
diff --git a/distributed/cli/dask_ssh.py b/distributed/cli/dask_ssh.py
index 1d264dc80e5..389e0327688 100755
--- a/distributed/cli/dask_ssh.py
+++ b/distributed/cli/dask_ssh.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 from distributed.deploy.ssh import SSHCluster
 import click
 
diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
index 084d7b59ccc..44931393522 100755
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import atexit
 import logging
 import multiprocessing
diff --git a/distributed/cli/tests/test_dask_scheduler.py b/distributed/cli/tests/test_dask_scheduler.py
index 24737474165..cb6cc306b6c 100644
--- a/distributed/cli/tests/test_dask_scheduler.py
+++ b/distributed/cli/tests/test_dask_scheduler.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import pytest
 
 pytest.importorskip("requests")
diff --git a/distributed/cli/tests/test_dask_worker.py b/distributed/cli/tests/test_dask_worker.py
index e268229767d..2dd74737b16 100644
--- a/distributed/cli/tests/test_dask_worker.py
+++ b/distributed/cli/tests/test_dask_worker.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import pytest
 from click.testing import CliRunner
 
diff --git a/distributed/cli/tests/test_tls_cli.py b/distributed/cli/tests/test_tls_cli.py
index 37fdc9bb00f..def31bc244d 100644
--- a/distributed/cli/tests/test_tls_cli.py
+++ b/distributed/cli/tests/test_tls_cli.py
@@ -1,8 +1,5 @@
-from __future__ import print_function, division, absolute_import
-
 from time import sleep
 
-
 from distributed import Client
 from distributed.utils_test import (
     popen,
diff --git a/distributed/cli/utils.py b/distributed/cli/utils.py
index 2c2088a7556..4cfb41abe0f 100644
--- a/distributed/cli/utils.py
+++ b/distributed/cli/utils.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 from tornado import gen
 from tornado.ioloop import IOLoop
 
diff --git a/distributed/client.py b/distributed/client.py
index e1efda4e0d7..d9f02ba80f8 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import atexit
 from collections import defaultdict, Iterator
 from concurrent.futures import ThreadPoolExecutor, CancelledError
diff --git a/distributed/comm/__init__.py b/distributed/comm/__init__.py
index e0615b38c7a..3537b301573 100644
--- a/distributed/comm/__init__.py
+++ b/distributed/comm/__init__.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 from .addressing import (
     parse_address,
     unparse_address,
diff --git a/distributed/comm/addressing.py b/distributed/comm/addressing.py
index 54e37b77f6b..8480134997c 100644
--- a/distributed/comm/addressing.py
+++ b/distributed/comm/addressing.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import six
 
 import dask
diff --git a/distributed/comm/core.py b/distributed/comm/core.py
index 869cb9b377f..602b3161657 100644
--- a/distributed/comm/core.py
+++ b/distributed/comm/core.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 from abc import ABCMeta, abstractmethod, abstractproperty
 from datetime import timedelta
 import logging
diff --git a/distributed/comm/inproc.py b/distributed/comm/inproc.py
index 3a781479bbc..5235b7535fd 100644
--- a/distributed/comm/inproc.py
+++ b/distributed/comm/inproc.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 from collections import deque, namedtuple
 import itertools
 import logging
diff --git a/distributed/comm/registry.py b/distributed/comm/registry.py
index a646b4d71b9..b7fcca912cd 100644
--- a/distributed/comm/registry.py
+++ b/distributed/comm/registry.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 from abc import ABCMeta, abstractmethod
 
 from six import with_metaclass
diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py
index d23f381857d..36783102b69 100644
--- a/distributed/comm/tcp.py
+++ b/distributed/comm/tcp.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import errno
 import logging
 import socket
diff --git a/distributed/comm/tests/test_comms.py b/distributed/comm/tests/test_comms.py
index 7fac117027b..620d4b89c94 100644
--- a/distributed/comm/tests/test_comms.py
+++ b/distributed/comm/tests/test_comms.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 from functools import partial
 import os
 import sys
diff --git a/distributed/comm/utils.py b/distributed/comm/utils.py
index 1e23b25c46b..dcc9e9a8b1a 100644
--- a/distributed/comm/utils.py
+++ b/distributed/comm/utils.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 from concurrent.futures import ThreadPoolExecutor
 import logging
 import socket
diff --git a/distributed/compatibility.py b/distributed/compatibility.py
index fb79353d24b..186e66e485c 100644
--- a/distributed/compatibility.py
+++ b/distributed/compatibility.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import logging
 import platform
 import sys
diff --git a/distributed/config.py b/distributed/config.py
index 5c71cf570c8..7e6075125fd 100644
--- a/distributed/config.py
+++ b/distributed/config.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import logging
 import logging.config
 import os
diff --git a/distributed/core.py b/distributed/core.py
index 7db7b3e29e5..f97d2df382a 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import asyncio
 from collections import defaultdict, deque
 from concurrent.futures import CancelledError
diff --git a/distributed/counter.py b/distributed/counter.py
index d5a3181b112..f41961e87ac 100644
--- a/distributed/counter.py
+++ b/distributed/counter.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 from collections import defaultdict
 
 from tornado.ioloop import IOLoop
diff --git a/distributed/dashboard/components.py b/distributed/dashboard/components.py
index 242a617706e..7fb8a6cb022 100644
--- a/distributed/dashboard/components.py
+++ b/distributed/dashboard/components.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import asyncio
 from bisect import bisect
 from operator import add
diff --git a/distributed/dashboard/core.py b/distributed/dashboard/core.py
index a85efb3233c..fd6ebef2834 100644
--- a/distributed/dashboard/core.py
+++ b/distributed/dashboard/core.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 from distutils.version import LooseVersion
 import os
 import warnings
diff --git a/distributed/dashboard/export_tool.py b/distributed/dashboard/export_tool.py
index 5d8f1c067ae..d93d21b881b 100644
--- a/distributed/dashboard/export_tool.py
+++ b/distributed/dashboard/export_tool.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import os
 
 from bokeh.core.properties import Int, String
diff --git a/distributed/dashboard/scheduler.py b/distributed/dashboard/scheduler.py
index e41862335cd..8396bbcb6ae 100644
--- a/distributed/dashboard/scheduler.py
+++ b/distributed/dashboard/scheduler.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 from functools import partial
 import logging
 import math
diff --git a/distributed/dashboard/tests/test_components.py b/distributed/dashboard/tests/test_components.py
index b12780f199b..5e96d788e45 100644
--- a/distributed/dashboard/tests/test_components.py
+++ b/distributed/dashboard/tests/test_components.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import pytest
 
 pytest.importorskip("bokeh")
diff --git a/distributed/dashboard/tests/test_scheduler_bokeh.py b/distributed/dashboard/tests/test_scheduler_bokeh.py
index 8544e72d9f4..e9ac62aad41 100644
--- a/distributed/dashboard/tests/test_scheduler_bokeh.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import json
 import re
 import ssl
diff --git a/distributed/dashboard/tests/test_scheduler_bokeh_html.py b/distributed/dashboard/tests/test_scheduler_bokeh_html.py
index b66aff02ddc..fc19efb0812 100644
--- a/distributed/dashboard/tests/test_scheduler_bokeh_html.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh_html.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import json
 import re
 import xml.etree.ElementTree
diff --git a/distributed/dashboard/tests/test_worker_bokeh.py b/distributed/dashboard/tests/test_worker_bokeh.py
index d320ea24ee8..c490c825ab4 100644
--- a/distributed/dashboard/tests/test_worker_bokeh.py
+++ b/distributed/dashboard/tests/test_worker_bokeh.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 from operator import add, sub
 import re
 from time import sleep
diff --git a/distributed/dashboard/utils.py b/distributed/dashboard/utils.py
index 8e6b5ff0b9c..285f6a5772a 100644
--- a/distributed/dashboard/utils.py
+++ b/distributed/dashboard/utils.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 from distutils.version import LooseVersion
 import os
 
diff --git a/distributed/dashboard/worker.py b/distributed/dashboard/worker.py
index d8f8adc1c7d..402d3fd0a70 100644
--- a/distributed/dashboard/worker.py
+++ b/distributed/dashboard/worker.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 from functools import partial
 import logging
 import math
diff --git a/distributed/deploy/__init__.py b/distributed/deploy/__init__.py
index 24a86e6d6d2..5a5a9106005 100644
--- a/distributed/deploy/__init__.py
+++ b/distributed/deploy/__init__.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 from ..utils import ignoring
 
 from .cluster import Cluster
diff --git a/distributed/deploy/adaptive.py b/distributed/deploy/adaptive.py
index 761a7d300ee..9b1d8511045 100644
--- a/distributed/deploy/adaptive.py
+++ b/distributed/deploy/adaptive.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 from collections import deque
 import logging
 
diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index 5b0aec4e80c..877a74587e9 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import atexit
 import logging
 import math
diff --git a/distributed/deploy/ssh.py b/distributed/deploy/ssh.py
index ba8ed01d1c7..9390d00a2ab 100644
--- a/distributed/deploy/ssh.py
+++ b/distributed/deploy/ssh.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import logging
 import socket
 import os
diff --git a/distributed/deploy/tests/test_adaptive.py b/distributed/deploy/tests/test_adaptive.py
index e0478a9cbdb..2d3d2235e21 100644
--- a/distributed/deploy/tests/test_adaptive.py
+++ b/distributed/deploy/tests/test_adaptive.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 from time import sleep
 
 import pytest
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index 91a792272e8..f434945c3af 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 from functools import partial
 import gc
 import multiprocessing
diff --git a/distributed/deploy/tests/test_ssh.py b/distributed/deploy/tests/test_ssh.py
index a86a8ddd280..492ee2c792d 100644
--- a/distributed/deploy/tests/test_ssh.py
+++ b/distributed/deploy/tests/test_ssh.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 from time import sleep
 
 import pytest
diff --git a/distributed/diagnostics/__init__.py b/distributed/diagnostics/__init__.py
index 9469c3855d1..2ab9fac731f 100644
--- a/distributed/diagnostics/__init__.py
+++ b/distributed/diagnostics/__init__.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 from ..utils import ignoring
 from .graph_layout import GraphLayout
 
diff --git a/distributed/diagnostics/eventstream.py b/distributed/diagnostics/eventstream.py
index b9213144d4e..c0fde24470b 100644
--- a/distributed/diagnostics/eventstream.py
+++ b/distributed/diagnostics/eventstream.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import logging
 
 from .plugin import SchedulerPlugin
diff --git a/distributed/diagnostics/graph_layout.py b/distributed/diagnostics/graph_layout.py
index 62e115a9ad4..c81c6edcafe 100644
--- a/distributed/diagnostics/graph_layout.py
+++ b/distributed/diagnostics/graph_layout.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 from .plugin import SchedulerPlugin
 
 
diff --git a/distributed/diagnostics/plugin.py b/distributed/diagnostics/plugin.py
index e1da4378fd4..cfe5fa42b49 100644
--- a/distributed/diagnostics/plugin.py
+++ b/distributed/diagnostics/plugin.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import logging
 
 logger = logging.getLogger(__name__)
diff --git a/distributed/diagnostics/progress.py b/distributed/diagnostics/progress.py
index 50c4cd9fad1..4136fd17a5c 100644
--- a/distributed/diagnostics/progress.py
+++ b/distributed/diagnostics/progress.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 from collections import defaultdict
 import logging
 from timeit import default_timer
diff --git a/distributed/diagnostics/progress_stream.py b/distributed/diagnostics/progress_stream.py
index b1e3787bd5a..038237b89e2 100644
--- a/distributed/diagnostics/progress_stream.py
+++ b/distributed/diagnostics/progress_stream.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import logging
 
 from toolz import valmap, merge
diff --git a/distributed/diagnostics/progressbar.py b/distributed/diagnostics/progressbar.py
index 4c9b781f61c..01dc9bbea39 100644
--- a/distributed/diagnostics/progressbar.py
+++ b/distributed/diagnostics/progressbar.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import logging
 import html
 from timeit import default_timer
diff --git a/distributed/diagnostics/task_stream.py b/distributed/diagnostics/task_stream.py
index 17e62c3045e..2491c8a89c0 100644
--- a/distributed/diagnostics/task_stream.py
+++ b/distributed/diagnostics/task_stream.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 from collections import deque
 import logging
 
diff --git a/distributed/diagnostics/tests/test_eventstream.py b/distributed/diagnostics/tests/test_eventstream.py
index 9139f75eab3..a111220b39e 100644
--- a/distributed/diagnostics/tests/test_eventstream.py
+++ b/distributed/diagnostics/tests/test_eventstream.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import collections
 
 import pytest
diff --git a/distributed/diagnostics/tests/test_plugin.py b/distributed/diagnostics/tests/test_plugin.py
index 1c9ebd7a1a8..af29e81674d 100644
--- a/distributed/diagnostics/tests/test_plugin.py
+++ b/distributed/diagnostics/tests/test_plugin.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 from distributed import Worker
 from distributed.utils_test import inc, gen_cluster
 from distributed.diagnostics.plugin import SchedulerPlugin
diff --git a/distributed/diagnostics/tests/test_progress.py b/distributed/diagnostics/tests/test_progress.py
index 097b2670247..8e3ba1688cc 100644
--- a/distributed/diagnostics/tests/test_progress.py
+++ b/distributed/diagnostics/tests/test_progress.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import pytest
 
 from tornado import gen
diff --git a/distributed/diagnostics/tests/test_progress_stream.py b/distributed/diagnostics/tests/test_progress_stream.py
index 9cf89817f34..56da9e974c1 100644
--- a/distributed/diagnostics/tests/test_progress_stream.py
+++ b/distributed/diagnostics/tests/test_progress_stream.py
@@ -1,6 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
-
 import pytest
 
 pytest.importorskip("bokeh")
diff --git a/distributed/diagnostics/tests/test_progressbar.py b/distributed/diagnostics/tests/test_progressbar.py
index 4e6ffe8c7e9..535efd0e9e2 100644
--- a/distributed/diagnostics/tests/test_progressbar.py
+++ b/distributed/diagnostics/tests/test_progressbar.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 from time import sleep
 
 import pytest
diff --git a/distributed/diagnostics/tests/test_task_stream.py b/distributed/diagnostics/tests/test_task_stream.py
index ad23ca5ae8c..58f1c4319f6 100644
--- a/distributed/diagnostics/tests/test_task_stream.py
+++ b/distributed/diagnostics/tests/test_task_stream.py
@@ -1,5 +1,3 @@
-from __future__ import absolute_import, division, print_function
-
 import os
 from time import sleep
 
diff --git a/distributed/diagnostics/tests/test_widgets.py b/distributed/diagnostics/tests/test_widgets.py
index 033d49251cb..03689c88c1d 100644
--- a/distributed/diagnostics/tests/test_widgets.py
+++ b/distributed/diagnostics/tests/test_widgets.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import pytest
 
 pytest.importorskip("ipywidgets")
diff --git a/distributed/diskutils.py b/distributed/diskutils.py
index 32e6be35adb..64dcf1dfc12 100644
--- a/distributed/diskutils.py
+++ b/distributed/diskutils.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import errno
 import glob
 import logging
diff --git a/distributed/lock.py b/distributed/lock.py
index 6ad6ab607d3..ed3eb4313f2 100644
--- a/distributed/lock.py
+++ b/distributed/lock.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 from collections import defaultdict, deque
 from datetime import timedelta
 import logging
diff --git a/distributed/metrics.py b/distributed/metrics.py
index 6c0bdb4dc7e..fefdfeb2e4c 100755
--- a/distributed/metrics.py
+++ b/distributed/metrics.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import collections
 from functools import wraps
 import sys
diff --git a/distributed/nanny.py b/distributed/nanny.py
index 228e37c2839..771b2d11d2d 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 from datetime import timedelta
 import logging
 from multiprocessing.queues import Empty
diff --git a/distributed/node.py b/distributed/node.py
index cbf2c00d8f7..8ef610a8481 100644
--- a/distributed/node.py
+++ b/distributed/node.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import logging
 import warnings
 import weakref
diff --git a/distributed/process.py b/distributed/process.py
index b6e50122c36..889787fe0bf 100644
--- a/distributed/process.py
+++ b/distributed/process.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import atexit
 from datetime import timedelta
 import logging
diff --git a/distributed/proctitle.py b/distributed/proctitle.py
index 50c9859e17e..961c74b91ab 100644
--- a/distributed/proctitle.py
+++ b/distributed/proctitle.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import os
 
 try:
diff --git a/distributed/profile.py b/distributed/profile.py
index 7f85f46312b..274dfcd1d20 100644
--- a/distributed/profile.py
+++ b/distributed/profile.py
@@ -24,8 +24,6 @@
                    'children': {...}}}
     }
 """
-from __future__ import print_function, division, absolute_import
-
 import bisect
 from collections import defaultdict, deque
 import linecache
diff --git a/distributed/protocol/__init__.py b/distributed/protocol/__init__.py
index 3f98436f4b9..e30786ab4a5 100644
--- a/distributed/protocol/__init__.py
+++ b/distributed/protocol/__init__.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 from functools import partial
 
 from .compression import compressions, default_compression
diff --git a/distributed/protocol/arrow.py b/distributed/protocol/arrow.py
index cac146a575c..1f2b4e83e9a 100644
--- a/distributed/protocol/arrow.py
+++ b/distributed/protocol/arrow.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 from .serialize import dask_serialize, dask_deserialize
 
 import pyarrow
diff --git a/distributed/protocol/compression.py b/distributed/protocol/compression.py
index f729748acc8..5035b465cee 100644
--- a/distributed/protocol/compression.py
+++ b/distributed/protocol/compression.py
@@ -3,8 +3,6 @@
 
 Includes utilities for determining whether or not to compress
 """
-from __future__ import print_function, division, absolute_import
-
 import logging
 import random
 
diff --git a/distributed/protocol/core.py b/distributed/protocol/core.py
index d54dd2e533e..3937c9c2fc8 100644
--- a/distributed/protocol/core.py
+++ b/distributed/protocol/core.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import logging
 import operator
 
diff --git a/distributed/protocol/h5py.py b/distributed/protocol/h5py.py
index cf08719e259..e129c166683 100644
--- a/distributed/protocol/h5py.py
+++ b/distributed/protocol/h5py.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 from .serialize import dask_serialize, dask_deserialize
 
 import h5py
diff --git a/distributed/protocol/keras.py b/distributed/protocol/keras.py
index 4c6fc4b4d0a..7471a3dbc93 100644
--- a/distributed/protocol/keras.py
+++ b/distributed/protocol/keras.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 from .serialize import dask_serialize, dask_deserialize, serialize, deserialize
 
 import keras
diff --git a/distributed/protocol/netcdf4.py b/distributed/protocol/netcdf4.py
index e04864d2b73..eb83461eddc 100644
--- a/distributed/protocol/netcdf4.py
+++ b/distributed/protocol/netcdf4.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 from .serialize import dask_serialize, dask_deserialize, serialize, deserialize
 
 import netCDF4
diff --git a/distributed/protocol/numpy.py b/distributed/protocol/numpy.py
index b2375569ef6..c7e48e63b1a 100644
--- a/distributed/protocol/numpy.py
+++ b/distributed/protocol/numpy.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import math
 import numpy as np
 
diff --git a/distributed/protocol/pickle.py b/distributed/protocol/pickle.py
index 080bb9037db..629fb962fbf 100644
--- a/distributed/protocol/pickle.py
+++ b/distributed/protocol/pickle.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import logging
 import sys
 
diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index 9e314703072..0069c6a264d 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -1,4 +1,3 @@
-from __future__ import print_function, division, absolute_import
 from functools import partial
 import traceback
 
diff --git a/distributed/protocol/sparse.py b/distributed/protocol/sparse.py
index b5a437a32a4..a22d661f849 100644
--- a/distributed/protocol/sparse.py
+++ b/distributed/protocol/sparse.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 from .serialize import dask_serialize, dask_deserialize, serialize, deserialize
 
 import sparse
diff --git a/distributed/protocol/tests/test_numpy.py b/distributed/protocol/tests/test_numpy.py
index ed4e32c1137..b334683b661 100644
--- a/distributed/protocol/tests/test_numpy.py
+++ b/distributed/protocol/tests/test_numpy.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import sys
 from zlib import crc32
 
diff --git a/distributed/protocol/tests/test_pandas.py b/distributed/protocol/tests/test_pandas.py
index 8f5827f7896..104151fb55a 100644
--- a/distributed/protocol/tests/test_pandas.py
+++ b/distributed/protocol/tests/test_pandas.py
@@ -1,6 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
-
 import pandas as pd
 import pandas.util.testing as tm
 import pytest
diff --git a/distributed/protocol/tests/test_protocol.py b/distributed/protocol/tests/test_protocol.py
index 2415e01b5f1..395c1ca7b97 100644
--- a/distributed/protocol/tests/test_protocol.py
+++ b/distributed/protocol/tests/test_protocol.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import sys
 
 import dask
diff --git a/distributed/protocol/tests/test_protocol_utils.py b/distributed/protocol/tests/test_protocol_utils.py
index f4b98ab0e1d..d4250fb3c05 100644
--- a/distributed/protocol/tests/test_protocol_utils.py
+++ b/distributed/protocol/tests/test_protocol_utils.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 from distributed.protocol.utils import merge_frames, pack_frames, unpack_frames
 from distributed.utils import ensure_bytes
 
diff --git a/distributed/protocol/tests/test_serialize.py b/distributed/protocol/tests/test_serialize.py
index 4f72ec9a538..09297793fc3 100644
--- a/distributed/protocol/tests/test_serialize.py
+++ b/distributed/protocol/tests/test_serialize.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import copy
 import pickle
 
diff --git a/distributed/protocol/utils.py b/distributed/protocol/utils.py
index caf4bb8833b..68de0bebd32 100644
--- a/distributed/protocol/utils.py
+++ b/distributed/protocol/utils.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import struct
 import msgpack
 
diff --git a/distributed/pytest_resourceleaks.py b/distributed/pytest_resourceleaks.py
index bb62d3916d0..0119a425722 100644
--- a/distributed/pytest_resourceleaks.py
+++ b/distributed/pytest_resourceleaks.py
@@ -2,8 +2,6 @@
 """
 A pytest plugin to trace resource leaks.
 """
-from __future__ import print_function, division
-
 import collections
 import gc
 import time
diff --git a/distributed/queues.py b/distributed/queues.py
index b97c317ac58..7174c48a63c 100644
--- a/distributed/queues.py
+++ b/distributed/queues.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 from collections import defaultdict
 import datetime
 import logging
diff --git a/distributed/recreate_exceptions.py b/distributed/recreate_exceptions.py
index d02dc4d94f4..9138c1fca5a 100644
--- a/distributed/recreate_exceptions.py
+++ b/distributed/recreate_exceptions.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import logging
 from .client import futures_of, wait
 from .utils import sync, tokey
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 8eec8744849..4e769ad420c 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import asyncio
 from collections import defaultdict, deque, OrderedDict, Mapping, Set
 from datetime import timedelta
diff --git a/distributed/sizeof.py b/distributed/sizeof.py
index 0bc094e35a7..bc51b3603ae 100644
--- a/distributed/sizeof.py
+++ b/distributed/sizeof.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import logging
 
 from dask.sizeof import sizeof
diff --git a/distributed/stealing.py b/distributed/stealing.py
index afcdf2a1cfa..e3537f647bf 100644
--- a/distributed/stealing.py
+++ b/distributed/stealing.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 from collections import defaultdict, deque
 import logging
 from math import log
diff --git a/distributed/submit.py b/distributed/submit.py
index f7e0a2f70aa..4cd7fb197a9 100644
--- a/distributed/submit.py
+++ b/distributed/submit.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import logging
 import os
 import socket
diff --git a/distributed/system_monitor.py b/distributed/system_monitor.py
index 30efc3ceb87..5b3bed3f98d 100644
--- a/distributed/system_monitor.py
+++ b/distributed/system_monitor.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 from collections import deque
 import psutil
 
diff --git a/distributed/tests/test_asyncprocess.py b/distributed/tests/test_asyncprocess.py
index 3cb3eee14d4..e496b35cb90 100644
--- a/distributed/tests/test_asyncprocess.py
+++ b/distributed/tests/test_asyncprocess.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 from datetime import timedelta
 import gc
 import os
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 5e2ade3f247..99e626de7fd 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import asyncio
 from collections import deque
 from concurrent.futures import CancelledError
diff --git a/distributed/tests/test_client_executor.py b/distributed/tests/test_client_executor.py
index a7f10491efb..7d08a63c5b2 100644
--- a/distributed/tests/test_client_executor.py
+++ b/distributed/tests/test_client_executor.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import random
 import time
 
diff --git a/distributed/tests/test_collections.py b/distributed/tests/test_collections.py
index 7cb509f6ac7..7fe8467b14b 100644
--- a/distributed/tests/test_collections.py
+++ b/distributed/tests/test_collections.py
@@ -1,6 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
-
 import pytest
 
 pytest.importorskip("numpy")
diff --git a/distributed/tests/test_config.py b/distributed/tests/test_config.py
index cdd4070f7bb..2017bb239f7 100644
--- a/distributed/tests/test_config.py
+++ b/distributed/tests/test_config.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import logging
 import subprocess
 import sys
diff --git a/distributed/tests/test_core.py b/distributed/tests/test_core.py
index 82e4c709be5..f91b8b64367 100644
--- a/distributed/tests/test_core.py
+++ b/distributed/tests/test_core.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 from contextlib import contextmanager
 import os
 import socket
diff --git a/distributed/tests/test_counter.py b/distributed/tests/test_counter.py
index 956a682920c..bb38a2812e5 100644
--- a/distributed/tests/test_counter.py
+++ b/distributed/tests/test_counter.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import pytest
 
 from distributed.counter import Counter
diff --git a/distributed/tests/test_diskutils.py b/distributed/tests/test_diskutils.py
index 561a4cd408b..c5cca9d5824 100644
--- a/distributed/tests/test_diskutils.py
+++ b/distributed/tests/test_diskutils.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import functools
 import gc
 import os
diff --git a/distributed/tests/test_failed_workers.py b/distributed/tests/test_failed_workers.py
index 1f27e067058..27bce439da4 100644
--- a/distributed/tests/test_failed_workers.py
+++ b/distributed/tests/test_failed_workers.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 from concurrent.futures import CancelledError
 import os
 import random
diff --git a/distributed/tests/test_ipython.py b/distributed/tests/test_ipython.py
index a6f88ec5241..8f2a40e45eb 100644
--- a/distributed/tests/test_ipython.py
+++ b/distributed/tests/test_ipython.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import mock
 
 import pytest
diff --git a/distributed/tests/test_locks.py b/distributed/tests/test_locks.py
index 9fa9a73787a..521a9b46114 100644
--- a/distributed/tests/test_locks.py
+++ b/distributed/tests/test_locks.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import pickle
 from time import sleep
 
diff --git a/distributed/tests/test_metrics.py b/distributed/tests/test_metrics.py
index cdb4b8ee478..3a27e638ef3 100644
--- a/distributed/tests/test_metrics.py
+++ b/distributed/tests/test_metrics.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import sys
 import threading
 import time
diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
index 579af8dbc2c..bd8a284df54 100644
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import gc
 import logging
 import os
diff --git a/distributed/tests/test_queues.py b/distributed/tests/test_queues.py
index a28d1e29082..817bfcbcea5 100644
--- a/distributed/tests/test_queues.py
+++ b/distributed/tests/test_queues.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 from datetime import timedelta
 from time import sleep
 
diff --git a/distributed/tests/test_resources.py b/distributed/tests/test_resources.py
index b3f5db36a76..648a191224e 100644
--- a/distributed/tests/test_resources.py
+++ b/distributed/tests/test_resources.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 from time import time
 
 from dask import delayed
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 6401cdd4b94..80cc04c81b3 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import cloudpickle
 import pickle
 from collections import defaultdict
diff --git a/distributed/tests/test_security.py b/distributed/tests/test_security.py
index 28438c6f359..bfc8358acf1 100644
--- a/distributed/tests/test_security.py
+++ b/distributed/tests/test_security.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 from contextlib import contextmanager
 import sys
 
diff --git a/distributed/tests/test_steal.py b/distributed/tests/test_steal.py
index 45a110bbecf..d7c396bb63f 100644
--- a/distributed/tests/test_steal.py
+++ b/distributed/tests/test_steal.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import itertools
 from operator import mul
 import random
@@ -114,8 +112,8 @@ def test_worksteal_many_thieves(c, s, *workers):
 def test_dont_steal_unknown_functions(c, s, a, b):
     futures = c.map(inc, [1, 2], workers=a.address, allow_other_workers=True)
     yield wait(futures)
-    assert len(a.data) == 2
-    assert len(b.data) == 0
+    assert len(a.data) == 2, [len(a.data), len(b.data)]
+    assert len(b.data) == 0, [len(a.data), len(b.data)]
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 2)
diff --git a/distributed/tests/test_stress.py b/distributed/tests/test_stress.py
index b5f51359239..db91ec0c004 100644
--- a/distributed/tests/test_stress.py
+++ b/distributed/tests/test_stress.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 from concurrent.futures import CancelledError
 from datetime import timedelta
 from operator import add
diff --git a/distributed/tests/test_submit_cli.py b/distributed/tests/test_submit_cli.py
index 04267a28e2b..9273261dc94 100644
--- a/distributed/tests/test_submit_cli.py
+++ b/distributed/tests/test_submit_cli.py
@@ -1,4 +1,3 @@
-from __future__ import print_function, division, absolute_import
 from mock import Mock
 
 from tornado import gen
diff --git a/distributed/tests/test_system_monitor.py b/distributed/tests/test_system_monitor.py
index f42fb8e3e08..f615549a686 100644
--- a/distributed/tests/test_system_monitor.py
+++ b/distributed/tests/test_system_monitor.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 from time import sleep
 
 from distributed.system_monitor import SystemMonitor
diff --git a/distributed/tests/test_tls_functional.py b/distributed/tests/test_tls_functional.py
index 43c8c667bf4..6d0e64b54e5 100644
--- a/distributed/tests/test_tls_functional.py
+++ b/distributed/tests/test_tls_functional.py
@@ -2,10 +2,6 @@
 Various functional tests for TLS networking.
 Most are taken from other test files and adapted.
 """
-
-from __future__ import print_function, division, absolute_import
-
-
 from tornado import gen
 
 from distributed import Nanny, worker_client, Queue
diff --git a/distributed/tests/test_utils.py b/distributed/tests/test_utils.py
index c547834626d..590f8c877b7 100644
--- a/distributed/tests/test_utils.py
+++ b/distributed/tests/test_utils.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import datetime
 from functools import partial
 import io
diff --git a/distributed/tests/test_utils_comm.py b/distributed/tests/test_utils_comm.py
index c9750891dd7..224b4b7f181 100644
--- a/distributed/tests/test_utils_comm.py
+++ b/distributed/tests/test_utils_comm.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import pytest
 
 from distributed.core import rpc
diff --git a/distributed/tests/test_utils_perf.py b/distributed/tests/test_utils_perf.py
index 95fa816a75b..4256548900c 100644
--- a/distributed/tests/test_utils_perf.py
+++ b/distributed/tests/test_utils_perf.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import contextlib
 import gc
 import itertools
diff --git a/distributed/tests/test_utils_test.py b/distributed/tests/test_utils_test.py
index 05b8066c707..1c6802b5637 100755
--- a/distributed/tests/test_utils_test.py
+++ b/distributed/tests/test_utils_test.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 from contextlib import contextmanager
 import socket
 import sys
diff --git a/distributed/tests/test_variable.py b/distributed/tests/test_variable.py
index e734cc3094f..88f96a241b0 100644
--- a/distributed/tests/test_variable.py
+++ b/distributed/tests/test_variable.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import random
 from time import sleep
 import sys
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index c7337d36424..bacd169d35c 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 from concurrent.futures import ThreadPoolExecutor
 from datetime import timedelta
 import importlib
diff --git a/distributed/tests/test_worker_client.py b/distributed/tests/test_worker_client.py
index fe1d49def6d..14a2d30f7d5 100644
--- a/distributed/tests/test_worker_client.py
+++ b/distributed/tests/test_worker_client.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import random
 import threading
 from time import sleep
diff --git a/distributed/threadpoolexecutor.py b/distributed/threadpoolexecutor.py
index f4cae3fd88e..44770900028 100644
--- a/distributed/threadpoolexecutor.py
+++ b/distributed/threadpoolexecutor.py
@@ -20,8 +20,6 @@
 
    Copyright 2001-2016 Python Software Foundation; All Rights Reserved
 """
-from __future__ import print_function, division, absolute_import
-
 from . import _concurrent_futures_thread as thread
 import os
 import logging
diff --git a/distributed/utils.py b/distributed/utils.py
index ab45350bf61..5cc5d414d3a 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import asyncio
 import atexit
 from collections import deque
diff --git a/distributed/utils_comm.py b/distributed/utils_comm.py
index af393cbd79e..f6b4ea36e4f 100644
--- a/distributed/utils_comm.py
+++ b/distributed/utils_comm.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import asyncio
 from collections import defaultdict
 from itertools import cycle
diff --git a/distributed/utils_perf.py b/distributed/utils_perf.py
index eb54ea0b381..048d9092d49 100644
--- a/distributed/utils_perf.py
+++ b/distributed/utils_perf.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 from collections import deque
 import gc
 import logging
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index 0c7c8958a91..8293bb474e2 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import asyncio
 import collections
 from contextlib import contextmanager
diff --git a/distributed/variable.py b/distributed/variable.py
index 30ffc5bf72d..2169c287f61 100644
--- a/distributed/variable.py
+++ b/distributed/variable.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import asyncio
 from collections import defaultdict
 import logging
diff --git a/distributed/versions.py b/distributed/versions.py
index d6a44096796..a769c9ab032 100644
--- a/distributed/versions.py
+++ b/distributed/versions.py
@@ -1,7 +1,5 @@
 """ utilities for package version introspection """
 
-from __future__ import print_function, division, absolute_import
-
 import platform
 import struct
 import os
diff --git a/distributed/worker.py b/distributed/worker.py
index ae7ee648ff9..b32ab2e52a4 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 import asyncio
 import bisect
 from collections import defaultdict, deque, MutableMapping
diff --git a/distributed/worker_client.py b/distributed/worker_client.py
index ff6294430b5..a45eb891f7d 100644
--- a/distributed/worker_client.py
+++ b/distributed/worker_client.py
@@ -1,5 +1,3 @@
-from __future__ import print_function, division, absolute_import
-
 from contextlib import contextmanager
 import warnings
 

From 4260790fc8c685cc306390f533fea8709b159830 Mon Sep 17 00:00:00 2001
From: Christian Hudon <chrish@pianocktail.org>
Date: Mon, 29 Jul 2019 18:52:30 -0400
Subject: [PATCH 0386/1550] Use click's show_default=True in relevant places
 (#2838)

* Use click's show_default=True in relevant places

* Make black happy... by running black

* Tweak Click default help text for bool options
---
 distributed/cli/dask_remote.py    |  4 +++-
 distributed/cli/dask_scheduler.py |  6 +++---
 distributed/cli/dask_ssh.py       | 16 ++++++++++++----
 distributed/cli/dask_worker.py    | 11 ++++++-----
 4 files changed, 24 insertions(+), 13 deletions(-)

diff --git a/distributed/cli/dask_remote.py b/distributed/cli/dask_remote.py
index 3118da84ae7..9fcfe7f3763 100644
--- a/distributed/cli/dask_remote.py
+++ b/distributed/cli/dask_remote.py
@@ -5,7 +5,9 @@
 
 @click.command()
 @click.option("--host", type=str, default=None, help="IP or hostname of this server")
-@click.option("--port", type=int, default=8788, help="Remote Client Port")
+@click.option(
+    "--port", type=int, default=8788, show_default=True, help="Remote Client Port"
+)
 @click.version_option()
 def main(host, port):
     _remote(host, port)
diff --git a/distributed/cli/dask_scheduler.py b/distributed/cli/dask_scheduler.py
index 07e4c98e267..a74f76102b9 100755
--- a/distributed/cli/dask_scheduler.py
+++ b/distributed/cli/dask_scheduler.py
@@ -65,15 +65,15 @@
     "--dashboard-address",
     type=str,
     default=":8787",
+    show_default=True,
     help="Address on which to listen for diagnostics dashboard",
 )
 @click.option(
     "--dashboard/--no-dashboard",
     "dashboard",
     default=True,
-    show_default=True,
     required=False,
-    help="Launch the Dashboard",
+    help="Launch the Dashboard [default: --dashboard]",
 )
 @click.option(
     "--bokeh/--no-bokeh",
@@ -82,7 +82,7 @@
     required=False,
     help="Deprecated.  See --dashboard/--no-dashboard.",
 )
-@click.option("--show/--no-show", default=False, help="Show web UI")
+@click.option("--show/--no-show", default=False, help="Show web UI [default: --show]")
 @click.option(
     "--dashboard-prefix", type=str, default=None, help="Prefix for the dashboard app"
 )
diff --git a/distributed/cli/dask_ssh.py b/distributed/cli/dask_ssh.py
index 389e0327688..97cf91f3519 100755
--- a/distributed/cli/dask_ssh.py
+++ b/distributed/cli/dask_ssh.py
@@ -20,8 +20,9 @@
 @click.option(
     "--scheduler-port",
     default=8786,
+    show_default=True,
     type=int,
-    help="Specify scheduler port number.  Defaults to port 8786.",
+    help="Specify scheduler port number.",
 )
 @click.option(
     "--nthreads",
@@ -36,8 +37,9 @@
 @click.option(
     "--nprocs",
     default=1,
+    show_default=True,
     type=int,
-    help="Number of worker processes per host.  Defaults to one.",
+    help="Number of worker processes per host.",
 )
 @click.argument("hostnames", nargs=-1, type=str)
 @click.option(
@@ -53,7 +55,11 @@
     help="Username to use when establishing SSH connections.",
 )
 @click.option(
-    "--ssh-port", default=22, type=int, help="Port to use for SSH connections."
+    "--ssh-port",
+    default=22,
+    type=int,
+    show_default=True,
+    help="Port to use for SSH connections.",
 )
 @click.option(
     "--ssh-private-key",
@@ -77,6 +83,7 @@
 @click.option(
     "--memory-limit",
     default="auto",
+    show_default=True,
     help="Bytes of memory that the worker can use. "
     "This can be an integer (bytes), "
     "float (fraction of total system memory), "
@@ -95,8 +102,9 @@
 @click.option(
     "--remote-dask-worker",
     default="distributed.cli.dask_worker",
+    show_default=True,
     type=str,
-    help="Worker to run. Defaults to distributed.cli.dask_worker",
+    help="Worker to run.",
 )
 @click.pass_context
 @click.version_option()
diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
index 44931393522..953d2e26fab 100755
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -70,9 +70,8 @@
     "--dashboard/--no-dashboard",
     "dashboard",
     default=True,
-    show_default=True,
     required=False,
-    help="Launch the Dashboard",
+    help="Launch the Dashboard [default: --dashboard]",
 )
 @click.option(
     "--bokeh/--no-bokeh",
@@ -116,7 +115,8 @@
     "--nprocs",
     type=int,
     default=1,
-    help="Number of worker processes to launch.  Defaults to one.",
+    show_default=True,
+    help="Number of worker processes to launch.",
 )
 @click.option(
     "--name",
@@ -129,6 +129,7 @@
 @click.option(
     "--memory-limit",
     default="auto",
+    show_default=True,
     help="Bytes of memory per process that the worker can use. "
     "This can be an integer (bytes), "
     "float (fraction of total system memory), "
@@ -138,12 +139,12 @@
 @click.option(
     "--reconnect/--no-reconnect",
     default=True,
-    help="Reconnect to scheduler if disconnected",
+    help="Reconnect to scheduler if disconnected [default: --reconnect]",
 )
 @click.option(
     "--nanny/--no-nanny",
     default=True,
-    help="Start workers in nanny process for management",
+    help="Start workers in nanny process for management [default: --nanny]",
 )
 @click.option("--pid-file", type=str, default="", help="File to write the process PID")
 @click.option(

From 051a79e05c501dcea57a19715f59d0cd203f6be6 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 30 Jul 2019 11:27:23 -0700
Subject: [PATCH 0387/1550] Close workers more gracefully (#2905)

This commit does two things:

1.  We wait to shutdown the executor a little longer in case it is still
    in use
2.  The worker no longer asks the Nanny to terminate it.  Instead it
    asks the nanny to shutdown gracefully after it is gone, and then
    continues closing itself as normal.
---
 distributed/nanny.py            |  3 +++
 distributed/tests/test_nanny.py | 17 ++++++++++++++++-
 distributed/worker.py           | 26 ++++++++++++--------------
 3 files changed, 31 insertions(+), 15 deletions(-)

diff --git a/distributed/nanny.py b/distributed/nanny.py
index 771b2d11d2d..e90b4ff33f8 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -380,6 +380,9 @@ async def _on_exit(self, exitcode):
                     if self.auto_restart:
                         logger.warning("Restarting worker")
                         await self.instantiate()
+                elif self.status == "closing-gracefully":
+                    await self.close()
+
             except Exception:
                 logger.error(
                     "Failed to restart worker after its process exited", exc_info=True
diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
index bd8a284df54..6187954e6be 100644
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -13,7 +13,7 @@
 from tornado.ioloop import IOLoop
 
 import dask
-from distributed import Nanny, rpc, Scheduler, Worker
+from distributed import Nanny, rpc, Scheduler, Worker, Client
 from distributed.core import CommClosedError
 from distributed.metrics import time
 from distributed.protocol.pickle import dumps
@@ -396,3 +396,18 @@ async def test_nanny_closes_cleanly(cleanup):
         assert not n.process
         assert not proc.is_alive()
         assert proc.exitcode == 0
+
+
+@pytest.mark.asyncio
+async def test_nanny_closes_cleanly(cleanup):
+    async with Scheduler() as s:
+        async with Nanny(s.address) as n:
+            async with Client(s.address, asynchronous=True) as client:
+                with client.rpc(n.worker_address) as w:
+                    IOLoop.current().add_callback(w.terminate)
+                    start = time()
+                    while n.status != "closed":
+                        await gen.sleep(0.01)
+                        assert time() < start + 5
+
+                    assert n.status == "closed"
diff --git a/distributed/worker.py b/distributed/worker.py
index b32ab2e52a4..67f753ad978 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -575,7 +575,7 @@ def __init__(
             "get_data": self.get_data,
             "update_data": self.update_data,
             "delete_data": self.delete_data,
-            "terminate": self.terminate,
+            "terminate": self.close,
             "ping": pingpong,
             "upload_file": self.upload_file,
             "start_ipython": self.start_ipython,
@@ -830,7 +830,7 @@ async def handle_scheduler(self, comm):
             logger.exception(e)
             raise
         finally:
-            if self.reconnect:
+            if self.reconnect and self.status == "running":
                 logger.info("Connection to scheduler broken.  Reconnecting...")
                 self.loop.add_callback(self._register_with_scheduler)
             else:
@@ -996,13 +996,6 @@ async def close(self, report=True, timeout=10, nanny=True, executor_wait=True):
                         self.scheduler.unregister(address=self.contact_address),
                     )
             self.scheduler.close_rpc()
-            self.actor_executor._work_queue.queue.clear()
-            if isinstance(self.executor, ThreadPoolExecutor):
-                self.executor._work_queue.queue.clear()
-                self.executor.shutdown(wait=executor_wait, timeout=timeout)
-            else:
-                self.executor.shutdown(wait=False)
-            self.actor_executor.shutdown(wait=executor_wait, timeout=timeout)
             self._workdir.release()
 
             for k, v in self.services.items():
@@ -1014,9 +1007,13 @@ async def close(self, report=True, timeout=10, nanny=True, executor_wait=True):
             if self.batched_stream:
                 self.batched_stream.close()
 
-            if nanny and self.nanny:
-                with self.rpc(self.nanny) as r:
-                    await r.terminate()
+            self.actor_executor._work_queue.queue.clear()
+            if isinstance(self.executor, ThreadPoolExecutor):
+                self.executor._work_queue.queue.clear()
+                self.executor.shutdown(wait=executor_wait, timeout=timeout)
+            else:
+                self.executor.shutdown(wait=False)
+            self.actor_executor.shutdown(wait=executor_wait, timeout=timeout)
 
             self.stop()
             self.rpc.close()
@@ -1026,9 +1023,10 @@ async def close(self, report=True, timeout=10, nanny=True, executor_wait=True):
             await ServerNode.close(self)
 
             setproctitle("dask-worker [closed]")
+        return "OK"
 
-    async def terminate(self, comm, report=True):
-        await self.close(report=report)
+    async def terminate(self, comm, report=True, **kwargs):
+        await self.close(report=report, **kwargs)
         return "OK"
 
     async def wait_until_closed(self):

From 5f120437a9f6101597b00bd2d5dfba8c851692d3 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 31 Jul 2019 07:37:48 -0700
Subject: [PATCH 0388/1550] Close workers gracefully with --lifetime keywords
 (#2892)

This allows workers to optionally terminate themselves gracefully after a predetermined time.  This can be helpful in a few contexts:

1.  We receive a SIGINT, and know that we need to clean up quickly (though note that the signal handlers are not implemented as part of this commit
2.  We know that we'll be kicked off at a certain time, such as in one hour from now, as is often specified by HPC job schedulers
3.  We just want to refresh our workers every once in a while, because we know that our code leaks some memory .  Fixes https://github.com/dask/distributed/issues/2861

This is configurable as keywords to the `Worker` or `Nanny` classes, in config values, or with CLI.  Here is an example with CLI.

### Restart to clear state

```
dask-worker scheduler:8786 --lifetime 1hr --lifetime-restart --lifetime-stagger 5m
```

This will kill the worker roughly 1 hour from now +- a range of 5 minutes (to avoid killing all of our workers at the same time).  It will also allow that worker to be restarted afterwards

### Restart to avoid walltime death

```
dask-worker scheduler:8786 --lifetime 58m
```

Here we don't try to restart the worker (no point) and we choose a time a bit before our 60m walltime.
---
 distributed/cli/dask_worker.py      | 24 ++++++++++-
 distributed/client.py               |  6 ++-
 distributed/diagnostics/__init__.py |  1 +
 distributed/distributed.yaml        |  5 ++-
 distributed/nanny.py                | 26 +++++++-----
 distributed/tests/test_client.py    |  2 +
 distributed/tests/test_nanny.py     | 25 +++++++++++
 distributed/tests/test_worker.py    | 39 ++++++++++++++++++
 distributed/worker.py               | 64 ++++++++++++++++++++++++++---
 9 files changed, 173 insertions(+), 19 deletions(-)

diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
index 953d2e26fab..eef1d648d40 100755
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -174,6 +174,28 @@
 @click.option(
     "--dashboard-prefix", type=str, default="", help="Prefix for the dashboard"
 )
+@click.option(
+    "--lifetime",
+    type=str,
+    default="",
+    help="If provided, shut down the worker after this duration.",
+)
+@click.option(
+    "--lifetime-stagger",
+    type=str,
+    default="0 seconds",
+    show_default=True,
+    help="Random amount by which to stagger lifetime values",
+)
+@click.option(
+    "--lifetime-restart/--no-lifetime-restart",
+    "lifetime_restart",
+    default=False,
+    show_default=True,
+    required=False,
+    help="Whether or not to restart the worker after the lifetime lapses. "
+    "This assumes that you are using the --lifetime and --nanny keywords",
+)
 @click.option(
     "--preload",
     type=str,
@@ -346,7 +368,7 @@ def del_pid_file():
             dashboard_address=dashboard_address if dashboard else None,
             service_kwargs={"dashboard": {"prefix": dashboard_prefix}},
             name=name if nprocs == 1 or not name else name + "-" + str(i),
-            **kwargs,
+            **kwargs
         )
         for i in range(nprocs)
     ]
diff --git a/distributed/client.py b/distributed/client.py
index d9f02ba80f8..0f7fe3d046b 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -953,7 +953,11 @@ async def _start(self, timeout=no_default, **kwargs):
             self.scheduler = self.rpc(address)
         self.scheduler_comm = None
 
-        await self._ensure_connected(timeout=timeout)
+        try:
+            await self._ensure_connected(timeout=timeout)
+        except OSError:
+            await self._close()
+            raise
 
         for pc in self._periodic_callbacks.values():
             pc.start()
diff --git a/distributed/diagnostics/__init__.py b/distributed/diagnostics/__init__.py
index 2ab9fac731f..337f41b7598 100644
--- a/distributed/diagnostics/__init__.py
+++ b/distributed/diagnostics/__init__.py
@@ -1,5 +1,6 @@
 from ..utils import ignoring
 from .graph_layout import GraphLayout
+from .plugin import SchedulerPlugin
 
 with ignoring(ImportError):
     from .progressbar import progress
diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index c3f14f114f1..9ad3e365e78 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -33,7 +33,6 @@ distributed:
         key: null
         cert: null
 
-
   worker:
     blocked-handlers: []
     multiprocessing-method: forkserver
@@ -44,6 +43,10 @@ distributed:
     preload: []
     preload-argv: []
     daemon: True
+    lifetime:
+      duration: null        # Time after which to gracefully shutdown the worker
+      stagger: 0 seconds    # Random amount by which to stagger lifetimes
+      restart: False        # Do we ressurrect the worker after the lifetime deadline?
 
     profile:
       interval: 10ms        # Time between statistical profiling queries
diff --git a/distributed/nanny.py b/distributed/nanny.py
index e90b4ff33f8..155fde98158 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -677,17 +677,21 @@ async def run():
                 init_result_q.put({"uid": uid, "exception": e})
                 init_result_q.close()
             else:
-                assert worker.address
-                init_result_q.put(
-                    {
-                        "address": worker.address,
-                        "dir": worker.local_directory,
-                        "uid": uid,
-                    }
-                )
-                init_result_q.close()
-                await worker.wait_until_closed()
-                logger.info("Worker closed")
+                try:
+                    assert worker.address
+                except ValueError:
+                    pass
+                else:
+                    init_result_q.put(
+                        {
+                            "address": worker.address,
+                            "dir": worker.local_directory,
+                            "uid": uid,
+                        }
+                    )
+                    init_result_q.close()
+                    await worker.wait_until_closed()
+                    logger.info("Worker closed")
 
         try:
             loop.run_sync(run)
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 99e626de7fd..5c86727c043 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -3318,6 +3318,7 @@ def test_bad_tasks_fail(c, s, a, b):
         yield f
 
     assert info.value.last_worker.nanny in {a.address, b.address}
+    yield [a.close(), b.close()]
 
 
 def test_get_processing_sync(c, s, a, b):
@@ -5233,6 +5234,7 @@ def test_client_timeout_2():
             yield c
         stop = time()
 
+        assert c.status == "closed"
         yield c.close()
 
         assert stop - start < 1
diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
index 6187954e6be..dec6bd91b20 100644
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -11,8 +11,10 @@
 from toolz import valmap, first
 from tornado import gen
 from tornado.ioloop import IOLoop
+from tornado.locks import Event
 
 import dask
+from distributed.diagnostics import SchedulerPlugin
 from distributed import Nanny, rpc, Scheduler, Worker, Client
 from distributed.core import CommClosedError
 from distributed.metrics import time
@@ -398,6 +400,29 @@ async def test_nanny_closes_cleanly(cleanup):
         assert proc.exitcode == 0
 
 
+@pytest.mark.slow
+@pytest.mark.asyncio
+async def test_lifetime(cleanup):
+    counter = 0
+    event = Event()
+
+    class Plugin(SchedulerPlugin):
+        def add_worker(self, **kwargs):
+            pass
+
+        def remove_worker(self, **kwargs):
+            nonlocal counter
+            counter += 1
+            if counter == 2:  # wait twice, then trigger closing event
+                event.set()
+
+    async with Scheduler() as s:
+        s.add_plugin(Plugin())
+        async with Nanny(s.address) as a:
+            async with Nanny(s.address, lifetime="500 ms", lifetime_restart=True) as b:
+                await event.wait()
+
+
 @pytest.mark.asyncio
 async def test_nanny_closes_cleanly(cleanup):
     async with Scheduler() as s:
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index bacd169d35c..52e92d474ce 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -1497,3 +1497,42 @@ async def test_worker_listens_on_same_interface_by_default(Worker):
         assert s.ip in {"127.0.0.1", "localhost"}
         async with Worker(s.address) as w:
             assert s.ip == w.ip
+
+
+@gen_cluster(client=True)
+async def test_close_gracefully(c, s, a, b):
+    futures = c.map(slowinc, range(200), delay=0.1)
+    while not b.data:
+        await gen.sleep(0.1)
+
+    mem = set(b.data)
+    proc = set(b.executing)
+
+    await b.close_gracefully()
+
+    assert b.status == "closed"
+    assert b.address not in s.workers
+    assert mem.issubset(set(a.data))
+    for key in proc:
+        assert s.tasks[key].state in ("processing", "memory")
+
+
+@pytest.mark.slow
+@pytest.mark.asyncio
+async def test_lifetime(cleanup):
+    async with Scheduler() as s:
+        async with Worker(s.address) as a, Worker(s.address, lifetime="1 seconds") as b:
+            async with Client(s.address, asynchronous=True) as c:
+                futures = c.map(slowinc, range(200), delay=0.1)
+                await gen.sleep(1.5)
+                assert b.status != "running"
+                await b.finished()
+
+                assert set(b.data).issubset(a.data)  # successfully moved data over
+
+
+@gen_cluster(client=True, worker_kwargs={"lifetime": "10s", "lifetime_stagger": "2s"})
+async def test_lifetime_stagger(c, s, a, b):
+    assert a.lifetime != b.lifetime
+    assert 8 <= a.lifetime <= 12
+    assert 8 <= b.lifetime <= 12
diff --git a/distributed/worker.py b/distributed/worker.py
index 67f753ad978..5648e67e353 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -249,6 +249,16 @@ class Worker(ServerNode):
         Resources that this worker has like ``{'GPU': 2}``
     nanny: str
         Address on which to contact nanny, if it exists
+    lifetime: str
+        Amount of time like "1 hour" after which we gracefully shut down the worker.
+        This defaults to None, meaning no explicit shutdown time.
+    lifetime_stagger: str
+        Amount of time like "5 minutes" to stagger the lifetime value
+        The actual lifetime will be selected uniformly at random between
+        lifetime +/- lifetime_stagger
+    lifetime_restart: bool
+        Whether or not to restart a worker after it has reached its lifetime
+        Default False
 
     Examples
     --------
@@ -308,6 +318,9 @@ def __init__(
         low_level_profiler=dask.config.get("distributed.worker.profile.low-level"),
         validate=False,
         profile_cycle_interval=None,
+        lifetime=None,
+        lifetime_stagger=None,
+        lifetime_restart=None,
         **kwargs
     ):
         self.tasks = dict()
@@ -653,6 +666,23 @@ def __init__(
         self.plugins = {}
         self._pending_plugins = plugins
 
+        self.lifetime = lifetime or dask.config.get(
+            "distributed.worker.lifetime.duration"
+        )
+        lifetime_stagger = lifetime_stagger or dask.config.get(
+            "distributed.worker.lifetime.stagger"
+        )
+        self.lifetime_restart = lifetime_restart or dask.config.get(
+            "distributed.worker.lifetime.restart"
+        )
+        if isinstance(self.lifetime, str):
+            self.lifetime = parse_timedelta(self.lifetime)
+        if isinstance(lifetime_stagger, str):
+            lifetime_stagger = parse_timedelta(lifetime_stagger)
+        if self.lifetime:
+            self.lifetime += (random.random() * 2 - 1) * lifetime_stagger
+            self.io_loop.call_later(self.lifetime, self.close_gracefully)
+
         Worker._instances.add(self)
 
     ##################
@@ -903,6 +933,8 @@ async def gather(self, comm=None, who_has=None):
     #############
 
     async def start(self):
+        if self.status and self.status.startswith("clos"):
+            return
         assert self.status is None, self.status
 
         enable_gc_diagnosis()
@@ -957,19 +989,22 @@ def _close(self, *args, **kwargs):
         warnings.warn("Worker._close has moved to Worker.close", stacklevel=2)
         return self.close(*args, **kwargs)
 
-    async def close(self, report=True, timeout=10, nanny=True, executor_wait=True):
+    async def close(
+        self, report=True, timeout=10, nanny=True, executor_wait=True, safe=False
+    ):
         with log_errors():
             if self.status in ("closed", "closing"):
                 await self.finished()
                 return
 
+            self.reconnect = False
             disable_gc_diagnosis()
 
             try:
                 logger.info("Stopping worker at %s", self.address)
             except ValueError:  # address not available if already closed
                 logger.info("Stopping worker")
-            if self.status != "running":
+            if self.status not in ("running", "closing-gracefully"):
                 logger.info("Closed worker has not yet started: %s", self.status)
             self.status = "closing"
 
@@ -993,7 +1028,9 @@ async def close(self, report=True, timeout=10, nanny=True, executor_wait=True):
                 if report:
                     await gen.with_timeout(
                         timedelta(seconds=timeout),
-                        self.scheduler.unregister(address=self.contact_address),
+                        self.scheduler.unregister(
+                            address=self.contact_address, safe=safe
+                        ),
                     )
             self.scheduler.close_rpc()
             self._workdir.release()
@@ -1025,6 +1062,23 @@ async def close(self, report=True, timeout=10, nanny=True, executor_wait=True):
             setproctitle("dask-worker [closed]")
         return "OK"
 
+    async def close_gracefully(self):
+        """ Gracefully shut down a worker
+
+        This first informs the scheduler that we're shutting down, and asks it
+        to move our data elsewhere.  Afterwards, we close as normal
+        """
+        if self.status.startswith("closing"):
+            await self.finished()
+
+        if self.status == "closed":
+            return
+
+        logger.info("Closing worker gracefully: %s", self.address)
+        self.status = "closing-gracefully"
+        await self.scheduler.retire_workers(workers=[self.address], remove=False)
+        await self.close(safe=True, nanny=not self.lifetime_restart)
+
     async def terminate(self, comm, report=True, **kwargs):
         await self.close(report=report, **kwargs)
         return "OK"
@@ -1541,7 +1595,7 @@ def transition_executing_done(self, key, value=no_value, report=True):
                 if key in self.dep_state:
                     self.transition_dep(key, "memory")
 
-            if report and self.batched_stream:
+            if report and self.batched_stream and self.status == "running":
                 self.send_task_state_to_scheduler(key)
             else:
                 raise CommClosedError
@@ -2278,7 +2332,7 @@ def ensure_computing(self):
 
     async def execute(self, key, report=False):
         executor_error = None
-        if self.status in ("closing", "closed"):
+        if self.status in ("closing", "closed", "closing-gracefully"):
             return
         try:
             if key not in self.executing or key not in self.task_state:

From f17ae0123735038eb0de355c4ce87cfb24b5221a Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 31 Jul 2019 12:25:32 -0700
Subject: [PATCH 0389/1550] Add closing <li> tags to Client._repr_html_ (#2911)

* Add is_valid_xml function

* Add closing <li> tags to Client._repr_html_
---
 distributed/client.py                                 |  4 ++--
 .../dashboard/tests/test_scheduler_bokeh_html.py      |  6 +++---
 distributed/deploy/spec.py                            |  6 +++---
 distributed/tests/test_client.py                      | 11 ++++++++++-
 distributed/tests/test_utils.py                       | 10 ++++++++--
 distributed/utils.py                                  |  5 +++++
 6 files changed, 31 insertions(+), 11 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 0f7fe3d046b..fab5ff0bf6e 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -798,9 +798,9 @@ def _repr_html_(self):
             '<ul style="text-align: left; list-style: none; margin: 0; padding: 0;">\n'
         )
         if scheduler is not None:
-            text += "  <li><b>Scheduler: </b>%s\n" % scheduler.address
+            text += "  <li><b>Scheduler: </b>%s</li>\n" % scheduler.address
         else:
-            text += "  <li><b>Scheduler: not connected</b>\n"
+            text += "  <li><b>Scheduler: not connected</b></li>\n"
 
         if info and "dashboard" in info["services"]:
             protocol, rest = scheduler.address.split("://")
diff --git a/distributed/dashboard/tests/test_scheduler_bokeh_html.py b/distributed/dashboard/tests/test_scheduler_bokeh_html.py
index fc19efb0812..660602df09a 100644
--- a/distributed/dashboard/tests/test_scheduler_bokeh_html.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh_html.py
@@ -1,6 +1,5 @@
 import json
 import re
-import xml.etree.ElementTree
 
 import pytest
 
@@ -10,6 +9,7 @@
 from tornado.httpclient import AsyncHTTPClient
 
 from dask.sizeof import sizeof
+from distributed.utils import is_valid_xml
 from distributed.utils_test import gen_cluster, slowinc, inc
 from distributed.dashboard import BokehScheduler, BokehWorker
 
@@ -45,7 +45,7 @@ def test_connect(c, s, a, b):
         if suffix.endswith(".json"):
             json.loads(body)
         else:
-            assert xml.etree.ElementTree.fromstring(body) is not None
+            assert is_valid_xml(body)
             assert not re.search("href=./", body)  # no absolute links
 
 
@@ -66,7 +66,7 @@ def test_prefix(c, s, a, b):
         if suffix.endswith(".json"):
             json.loads(body)
         else:
-            assert xml.etree.ElementTree.fromstring(body) is not None
+            assert is_valid_xml(body)
 
 
 @gen_cluster(
diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index 87228d5693e..c9e1aca1a87 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -492,9 +492,9 @@ def _widget_status(self):
     }
   </style>
   <table style="text-align: right;">
-    <tr><th>Workers</th> <td>%d</td></tr>
-    <tr><th>Cores</th> <td>%d</td></tr>
-    <tr><th>Memory</th> <td>%s</td></tr>
+    <tr> <th>Workers</th> <td>%d</td></tr>
+    <tr> <th>Cores</th> <td>%d</td></tr>
+    <tr> <th>Memory</th> <td>%s</td></tr>
   </table>
 </div>
 """ % (
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 5c86727c043..3eb9f39a2a4 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -54,7 +54,15 @@
 from distributed.metrics import time
 from distributed.scheduler import Scheduler, KilledWorker
 from distributed.sizeof import sizeof
-from distributed.utils import ignoring, mp_context, sync, tmp_text, tokey, tmpfile
+from distributed.utils import (
+    ignoring,
+    mp_context,
+    sync,
+    tmp_text,
+    tokey,
+    tmpfile,
+    is_valid_xml,
+)
 from distributed.utils_test import (
     cluster,
     slowinc,
@@ -1904,6 +1912,7 @@ def test_repr_localcluster():
     try:
         text = client._repr_html_()
         assert cluster.scheduler.address in text
+        assert is_valid_xml(client._repr_html_())
     finally:
         yield client.close()
         yield cluster.close()
diff --git a/distributed/tests/test_utils.py b/distributed/tests/test_utils.py
index 590f8c877b7..737b015646b 100644
--- a/distributed/tests/test_utils.py
+++ b/distributed/tests/test_utils.py
@@ -6,7 +6,6 @@
 import sys
 from time import sleep
 import traceback
-import xml.etree.ElementTree
 
 import numpy as np
 import pytest
@@ -21,6 +20,7 @@
     Logs,
     sync,
     is_kernel,
+    is_valid_xml,
     ensure_ip,
     str_graph,
     truncate_exception,
@@ -553,6 +553,12 @@ def test_logs():
     d = Logs({"123": Log("Hello"), "456": Log("World!")})
     text = d._repr_html_()
     for line in text.split("\n"):
-        assert xml.etree.ElementTree.fromstring(line) is not None
+        assert is_valid_xml(line)
     assert "Hello" in text
     assert "456" in text
+
+
+def test_is_valid_xml():
+    assert is_valid_xml("<a>foo</a>")
+    with pytest.raises(Exception):
+        assert is_valid_xml("<a>foo")
diff --git a/distributed/utils.py b/distributed/utils.py
index 5cc5d414d3a..a8cddb81498 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -26,6 +26,7 @@
 import pkgutil
 import six
 import tblib.pickling_support
+import xml.etree.ElementTree
 
 try:
     import resource
@@ -1466,3 +1467,7 @@ def convert_value(v):
     return sum(
         [["--" + k.replace("_", "-"), convert_value(v)] for k, v in d.items()], []
     )
+
+
+def is_valid_xml(text):
+    return xml.etree.ElementTree.fromstring(text) is not None

From a1e7b5212f8e96eaac6ebd4fbf45e24c4f50acc0 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 31 Jul 2019 15:51:14 -0700
Subject: [PATCH 0390/1550] Add endline spacing in Logs._repr_html_ (#2912)

---
 distributed/deploy/tests/test_spec_cluster.py | 2 ++
 distributed/tests/test_utils.py               | 3 +--
 distributed/utils.py                          | 6 +++---
 3 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/distributed/deploy/tests/test_spec_cluster.py b/distributed/deploy/tests/test_spec_cluster.py
index 84c868b2585..58bbbaef44d 100644
--- a/distributed/deploy/tests/test_spec_cluster.py
+++ b/distributed/deploy/tests/test_spec_cluster.py
@@ -4,6 +4,7 @@
 from dask.distributed import SpecCluster, Worker, Client, Scheduler, Nanny
 from distributed.deploy.spec import close_clusters, ProcessInterface
 from distributed.utils_test import loop, cleanup  # noqa: F401
+from distributed.utils import is_valid_xml
 import toolz
 import pytest
 
@@ -190,6 +191,7 @@ async def test_logs(cleanup):
         await cluster
 
         logs = await cluster.logs()
+        assert is_valid_xml("<div>" + logs._repr_html_() + "</div>")
         assert "Scheduler" in logs
         for worker in cluster.scheduler.workers:
             assert worker in logs
diff --git a/distributed/tests/test_utils.py b/distributed/tests/test_utils.py
index 737b015646b..e5e18eb393c 100644
--- a/distributed/tests/test_utils.py
+++ b/distributed/tests/test_utils.py
@@ -552,8 +552,7 @@ def test_format_bytes_compat():
 def test_logs():
     d = Logs({"123": Log("Hello"), "456": Log("World!")})
     text = d._repr_html_()
-    for line in text.split("\n"):
-        assert is_valid_xml(line)
+    assert is_valid_xml("<div>" + text + "</div>")
     assert "Hello" in text
     assert "456" in text
 
diff --git a/distributed/utils.py b/distributed/utils.py
index a8cddb81498..cdc5c4d1ae9 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -1414,7 +1414,7 @@ class Log(str):
     """ A container for logs """
 
     def _repr_html_(self):
-        return "<pre><code>{log}</code></pre>".format(log=self)
+        return "<pre><code>\n{log}\n</code></pre>".format(log=self.rstrip())
 
 
 class Logs(dict):
@@ -1422,12 +1422,12 @@ class Logs(dict):
 
     def _repr_html_(self):
         summaries = [
-            "<details><summary>{title}</summary>{log}</details>".format(
+            "<details>\n<summary>{title}</summary>\n{log}\n</details>".format(
                 title=title, log=log._repr_html_()
             )
             for title, log in self.items()
         ]
-        return "\n".join(summaries)
+        return "\n\n".join(summaries)
 
 
 def cli_keywords(d: dict, cls=None):

From 58844d01259ca2455a15267ceb2573a827761745 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 31 Jul 2019 16:36:06 -0700
Subject: [PATCH 0391/1550] bump version to 2.2.0

---
 docs/source/changelog.rst | 48 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 48 insertions(+)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index cb98ab79fc7..b7037d9d6c2 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,51 @@
 Changelog
 =========
 
+2.2.0 - 2019-07-31
+------------------
+
+-  Respect security configuration in LocalCluster (:pr:`2822`) `Russ Bubley`_
+-  Add Nanny to worker docs (:pr:`2826`) `Christian Hudon`_
+-  Don't make False add-keys report to scheduler (:pr:`2421`) `tjb900`_
+-  Include type name in SpecCluster repr (:pr:`2834`) `Jacob Tomlinson`_
+-  Extend prometheus metrics endpoint (:pr:`2833`) `Gabriel Sailer`_
+-  Add alternative SSHCluster implementation (:pr:`2827`) `Matthew Rocklin`_
+-  Dont reuse closed worker in get_worker (:pr:`2841`) `Pierre Glaser`_
+-  SpecCluster: move init logic into start (:pr:`2850`) `Jacob Tomlinson`_
+-  Document distributed.Reschedule in API docs (:pr:`2860`) `James Bourbeau`_
+-  Add fsspec to installation of test builds (:pr:`2859`) `Martin Durant`_
+-  Make await/start more consistent across Scheduler/Worker/Nanny (:pr:`2831`) `Matthew Rocklin`_
+-  Add cleanup fixture for asyncio tests (:pr:`2866`) `Matthew Rocklin`_
+-  Use only remote connection to scheduler in Adaptive (:pr:`2865`) `Matthew Rocklin`_
+-  Add Server.finished async function  (:pr:`2864`) `Matthew Rocklin`_
+-  Align text and remove bullets in Client HTML repr (:pr:`2867`) `Matthew Rocklin`_
+-  Test dask-scheduler --idle-timeout flag (:pr:`2862`) `Matthew Rocklin`_
+-  Remove ``Client.upload_environment`` (:pr:`2877`) `Jim Crist`_
+-  Replace gen.coroutine with async/await in core (:pr:`2871`) `Matthew Rocklin`_
+-  Forcefully kill all processes before each test (:pr:`2882`) `Matthew Rocklin`_
+-  Cleanup Security class and configuration (:pr:`2873`) `Jim Crist`_
+-  Remove unused variable in SpecCluster scale down (:pr:`2870`) `Jacob Tomlinson`_
+-  Add SpecCluster ProcessInterface (:pr:`2874`) `Jacob Tomlinson`_
+-  Add Log(str) and Logs(dict) classes for nice HTML reprs (:pr:`2875`) `Jacob Tomlinson`_
+-  Pass Client._asynchronous to Cluster._asynchronous (:pr:`2890`) `Matthew Rocklin`_
+-  Add default logs method to Spec Cluster (:pr:`2889`) `Matthew Rocklin`_
+-  Add processes keyword back into clean (:pr:`2891`) `Matthew Rocklin`_
+-  Update black (:pr:`2901`) `Matthew Rocklin`_
+-  Move Worker.local_dir attribute to Worker.local_directory (:pr:`2900`) `Matthew Rocklin`_
+-  Link from TapTools to worker info pages in dashboard (:pr:`2894`) `Matthew Rocklin`_
+-  Avoid exception in Client._ensure_connected if closed (:pr:`2893`) `Matthew Rocklin`_
+-  Convert Pythonic kwargs to CLI Keywords for SSHCluster (:pr:`2898`) `Matthew Rocklin`_
+-  Use kwargs in CLI (:pr:`2899`) `Matthew Rocklin`_
+-  Name SSHClusters by providing name= keyword to SpecCluster (:pr:`2903`) `Matthew Rocklin`_
+-  Request feed of worker information from Scheduler to SpecCluster (:pr:`2902`) `Matthew Rocklin`_
+-  Clear out compatibillity file (:pr:`2896`) `Matthew Rocklin`_
+-  Remove future imports (:pr:`2897`) `Matthew Rocklin`_
+-  Use click's show_default=True in relevant places (:pr:`2838`) `Christian Hudon`_
+-  Close workers more gracefully (:pr:`2905`) `Matthew Rocklin`_
+-  Close workers gracefully with --lifetime keywords (:pr:`2892`) `Matthew Rocklin`_
+-  Add closing <li> tags to Client._repr_html_ (:pr:`2911`) `Matthew Rocklin`_
+-  Add endline spacing in Logs._repr_html_ (:pr:`2912`) `Matthew Rocklin`_
+
 2.1.0 - 2019-07-08
 ------------------
 
@@ -1125,3 +1170,6 @@ significantly without many new features.
 .. _`Magnus Nord`: https://github.com/magnunor
 .. _`Sam Grayson`: https://github.com/charmoniumQ
 .. _`Mathieu Dugré`: https://github.com/mathdugre
+.. _`Christian Hudon`: https://github.com/chrish42
+.. _`Gabriel Sailer`: https://github.com/sublinus
+.. _`Pierre Glaser`: https://github.com/pierreglase

From e1e36e437a53a937d14519f06fced6b645cff361 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 1 Aug 2019 12:27:26 -0700
Subject: [PATCH 0392/1550] Call heartbeat rather than reconnect on
 disconnection (#2906)

Fixes https://github.com/dask/distributed/issues/2525

This avoids a subtle race condition.
---
 distributed/scheduler.py | 9 +++------
 distributed/worker.py    | 2 +-
 2 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 4e769ad420c..acabce22c63 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1325,11 +1325,10 @@ def heartbeat_worker(
     ):
         address = self.coerce_address(address, resolve_address)
         address = normalize_address(address)
-        host = get_address_host(address)
         if address not in self.workers:
-            logger.info("Received heartbeat from removed worker: %s", address)
-            return
+            return {"status": "missing"}
 
+        host = get_address_host(address)
         local_now = time()
         now = now or time()
         metrics = metrics or {}
@@ -1342,9 +1341,7 @@ def heartbeat_worker(
         except KeyError:
             pass
 
-        ws = self.workers.get(address)
-        if not ws:
-            return {"status": "missing"}
+        ws = self.workers[address]
 
         ws.last_seen = time()
 
diff --git a/distributed/worker.py b/distributed/worker.py
index 5648e67e353..672358d4460 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -862,7 +862,7 @@ async def handle_scheduler(self, comm):
         finally:
             if self.reconnect and self.status == "running":
                 logger.info("Connection to scheduler broken.  Reconnecting...")
-                self.loop.add_callback(self._register_with_scheduler)
+                self.loop.add_callback(self.heartbeat)
             else:
                 await self.close(report=False)
 

From ff3437c0a71f151ce1050a4e6a7177ab4a8a3b22 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 1 Aug 2019 13:47:09 -0700
Subject: [PATCH 0393/1550] Rewrite Adaptive/SpecCluster to support slowly
 arriving workers (#2904)

Previously SpecCluster waited until all workers had checked in with the
scheduler.  This made sense for LocalCluster or SSHCluster because
there isn't really a significant delay in starting things that we can't
control.  However, for other systems like dask-jobqueue or
dask-kubernetes workers might not ever start, so we need a different
system.

Now, SpecCluster still awaits the Worker object that it is passed, but
doesn't require that the worker has started in the scheduler.  We now
expect awaiting to mean

*"We have successfully handed control of starting the worker to some other robust system"*

Our job at this point is done.  We hope that the worker arrives, but
from our perspective this local Worker object is awaited and "running".

This commit also includes a minimal example of a Worker class,
`SlowWorker`, that serves as a nice minimal example for what SpecCluster
expects.

* Add AdaptiveCore class

* Back Adaptive with AdaptiveCore

* Include requested workers in widget

* Use worker names throughout adaptive
---
 distributed/deploy/adaptive.py                | 178 ++++------------
 distributed/deploy/adaptive_core.py           | 197 ++++++++++++++++++
 distributed/deploy/spec.py                    |  26 ++-
 distributed/deploy/tests/test_adaptive.py     | 120 +++++------
 .../deploy/tests/test_adaptive_core.py        |  90 ++++++++
 .../deploy/tests/test_slow_adaptive.py        |  98 +++++++++
 distributed/deploy/tests/test_spec_cluster.py |   8 +-
 distributed/nanny.py                          |   2 +-
 distributed/scheduler.py                      |  61 ++++--
 distributed/tests/test_scheduler.py           |   4 +-
 distributed/worker.py                         |   6 +-
 11 files changed, 555 insertions(+), 235 deletions(-)
 create mode 100644 distributed/deploy/adaptive_core.py
 create mode 100644 distributed/deploy/tests/test_adaptive_core.py
 create mode 100644 distributed/deploy/tests/test_slow_adaptive.py

diff --git a/distributed/deploy/adaptive.py b/distributed/deploy/adaptive.py
index 9b1d8511045..2efc18dfe0c 100644
--- a/distributed/deploy/adaptive.py
+++ b/distributed/deploy/adaptive.py
@@ -1,16 +1,14 @@
-from collections import deque
 import logging
+import math
 
-from tornado import gen
-
-from ..metrics import time
-from ..utils import log_errors, PeriodicCallback, parse_timedelta
+from .adaptive_core import AdaptiveCore
+from ..utils import log_errors, parse_timedelta
 from ..protocol import pickle
 
 logger = logging.getLogger(__name__)
 
 
-class Adaptive(object):
+class Adaptive(AdaptiveCore):
     '''
     Adaptively allocate workers based on scheduler load.  A superclass.
 
@@ -23,19 +21,13 @@ class Adaptive(object):
 
     Parameters
     ----------
-    scheduler: distributed.Scheduler
     cluster: object
-        Must have scale_up and scale_down methods/coroutines
-    startup_cost : timedelta or str, default "1s"
-        Estimate of the number of seconds for nnFactor representing how costly it is to start an additional worker.
-        Affects quickly to adapt to high tasks per worker loads
+        Must have scale and scale_down methods/coroutines
     interval : timedelta or str, default "1000 ms"
         Milliseconds between checks
     wait_count: int, default 3
         Number of consecutive times that a worker should be suggested for
         removal before we remove it.
-    scale_factor : int, default 2
-        Factor to scale by when it's determined additional workers are needed
     target_duration: timedelta or str, default "5s"
         Amount of time we want a computation to take.
         This affects how aggressively we scale up.
@@ -84,45 +76,47 @@ def __init__(
         self,
         cluster=None,
         interval="1s",
-        startup_cost="1s",
-        scale_factor=2,
         minimum=0,
-        maximum=None,
+        maximum=math.inf,
         wait_count=3,
         target_duration="5s",
         worker_key=None,
         **kwargs
     ):
-        interval = parse_timedelta(interval, default="ms")
-        self.worker_key = worker_key
         self.cluster = cluster
-        self.startup_cost = parse_timedelta(startup_cost, default="s")
-        self.scale_factor = scale_factor
-        if self.cluster:
-            self._adapt_callback = PeriodicCallback(
-                self._adapt, interval * 1000, io_loop=self.loop
-            )
-            self.loop.add_callback(self._adapt_callback.start)
-        self._adapting = False
+        self.worker_key = worker_key
         self._workers_to_close_kwargs = kwargs
-        self.minimum = minimum
-        self.maximum = maximum
-        self.log = deque(maxlen=1000)
-        self.close_counts = {}
-        self.wait_count = wait_count
         self.target_duration = parse_timedelta(target_duration)
 
+        super().__init__(
+            minimum=minimum, maximum=maximum, wait_count=wait_count, interval=interval
+        )
+
     @property
     def scheduler(self):
         return self.cluster.scheduler_comm
 
-    def stop(self):
-        if self.cluster:
-            self._adapt_callback.stop()
-            self._adapt_callback = None
-            del self._adapt_callback
+    @property
+    def plan(self):
+        try:
+            return set(self.cluster.worker_spec)
+        except AttributeError:
+            return set(self.cluster.workers)
+
+    @property
+    def requested(self):
+        return set(self.cluster.workers)
+
+    @property
+    def observed(self):
+        return {d["name"] for d in self.cluster.scheduler_info["workers"].values()}
+
+    async def target(self):
+        return await self.scheduler.adaptive_target(
+            target_duration=self.target_duration
+        )
 
-    async def workers_to_close(self, **kwargs):
+    async def workers_to_close(self, target: int):
         """
         Determine which, if any, workers should potentially be removed from
         the cluster.
@@ -140,114 +134,30 @@ async def workers_to_close(self, **kwargs):
         --------
         Scheduler.workers_to_close
         """
-        if len(self.cluster.workers) <= self.minimum:
-            return []
-
-        kw = dict(self._workers_to_close_kwargs)
-        kw.update(kwargs)
-
-        if self.maximum is not None and len(self.cluster.workers) > self.maximum:
-            kw["n"] = len(self.cluster.workers) - self.maximum
-
-        L = await self.scheduler.workers_to_close(**kw)
-        if len(self.cluster.workers) - len(L) < self.minimum:
-            L = L[: len(self.cluster.workers) - self.minimum]
-
-        return L
+        return await self.scheduler.workers_to_close(
+            target=target,
+            key=pickle.dumps(self.worker_key) if self.worker_key else None,
+            attribute="name",
+            **self._workers_to_close_kwargs
+        )
 
-    async def _retire_workers(self, workers=None):
-        if workers is None:
-            workers = await self.workers_to_close(
-                key=pickle.dumps(self.worker_key) if self.worker_key else None,
-                minimum=self.minimum,
-            )
+    async def scale_down(self, workers):
         if not workers:
-            raise gen.Return(workers)
+            return
         with log_errors():
+            # Ask scheduler to cleanly retire workers
             await self.scheduler.retire_workers(
-                workers=workers, remove=True, close_workers=True
+                names=workers, remove=True, close_workers=True
             )
 
+            # close workers more forcefully
             logger.info("Retiring workers %s", workers)
             f = self.cluster.scale_down(workers)
             if hasattr(f, "__await__"):
                 await f
 
-            return workers
-
-    async def recommendations(self, comm=None):
-        n = await self.scheduler.adaptive_target(target_duration=self.target_duration)
-        if self.maximum is not None:
-            n = min(self.maximum, n)
-        if self.minimum is not None:
-            n = max(self.minimum, n)
-        workers = set(
-            await self.workers_to_close(
-                key=pickle.dumps(self.worker_key) if self.worker_key else None,
-                minimum=self.minimum,
-            )
-        )
-        try:
-            current = len(self.cluster.worker_spec)
-        except AttributeError:
-            current = len(self.cluster.workers)
-        if n > current and workers:
-            logger.info("Attempting to scale up and scale down simultaneously.")
-            self.close_counts.clear()
-            return {
-                "status": "error",
-                "msg": "Trying to scale up and down simultaneously",
-            }
-
-        elif n > current:
-            self.close_counts.clear()
-            return {"status": "up", "n": n}
-
-        elif workers:
-            d = {}
-            to_close = []
-            for w, c in self.close_counts.items():
-                if w in workers:
-                    if c >= self.wait_count:
-                        to_close.append(w)
-                    else:
-                        d[w] = c
-
-            for w in workers:
-                d[w] = d.get(w, 0) + 1
-
-            self.close_counts = d
-
-            if to_close:
-                return {"status": "down", "workers": to_close}
-        else:
-            self.close_counts.clear()
-            return None
-
-    async def _adapt(self):
-        if self._adapting:  # Semaphore to avoid overlapping adapt calls
-            return
-
-        self._adapting = True
-        try:
-            recommendations = await self.recommendations()
-            if not recommendations:
-                return
-            status = recommendations.pop("status")
-            if status == "up":
-                f = self.cluster.scale(**recommendations)
-                self.log.append((time(), "up", recommendations))
-                if hasattr(f, "__await__"):
-                    await f
-
-            elif status == "down":
-                self.log.append((time(), "down", recommendations["workers"]))
-                workers = await self._retire_workers(workers=recommendations["workers"])
-        finally:
-            self._adapting = False
-
-    def adapt(self):
-        self.loop.add_callback(self._adapt)
+    async def scale_up(self, n):
+        self.cluster.scale(n)
 
     @property
     def loop(self):
diff --git a/distributed/deploy/adaptive_core.py b/distributed/deploy/adaptive_core.py
new file mode 100644
index 00000000000..6732bb20284
--- /dev/null
+++ b/distributed/deploy/adaptive_core.py
@@ -0,0 +1,197 @@
+import collections
+import math
+
+from tornado.ioloop import IOLoop
+import toolz
+
+from ..metrics import time
+from ..utils import parse_timedelta, PeriodicCallback
+
+
+class AdaptiveCore:
+    """
+    The core logic for adaptive deployments, with none of the cluster details
+
+    This class controls our adaptive scaling behavior.  It is intended to be
+    sued as a super-class or mixin.  It expects the following state and methods:
+
+    **State**
+
+    plan: set
+        A set of workers that we think should exist.
+        Here and below worker is just a token, often an address or name string
+
+    requested: set
+        A set of workers that the cluster class has successfully requested from
+        the resource manager.  We expect that resource manager to work to make
+        these exist.
+
+    observed: set
+        A set of workers that have successfully checked in with the scheduler
+
+    These sets are not necessarily equivalent.  Often plan and requested will
+    be very similar (requesting is usually fast) but there may be a large delay
+    between requested and observed (often resource managers don't give us what
+    we want).
+
+    **Functions**
+
+    target : -> int
+        Returns the target number of workers that should exist.
+        This is often obtained by querying the scheduler
+
+    workers_to_close : int -> Set[worker]
+        Given a target number of workers,
+        returns a set of workers that we should close when we're scaling down
+
+    scale_up : int -> None
+        Scales the cluster up to a target number of workers, presumably
+        changing at least ``plan`` and hopefully eventually also ``requested``
+
+   scale_down : Set[worker] -> None
+        Closes the provided set of workers
+
+    Parameters
+    ----------
+    minimum: int
+        The minimum number of allowed workers
+    maximum: int
+        The maximum number of allowed workers
+    wait_count: int
+        The number of scale-down requests we should receive before actually
+        scaling down
+    interval: str
+        The amount of time, like ``"1s"`` between checks
+    """
+
+    def __init__(
+        self,
+        minimum: int = 0,
+        maximum: int = math.inf,
+        wait_count: int = 3,
+        interval: str = "1s",
+    ):
+        self.minimum = minimum
+        self.maximum = maximum
+        self.wait_count = wait_count
+        self.interval = parse_timedelta(interval, "seconds") if interval else interval
+        self.periodic_callback = None
+
+        def f():
+            self.periodic_callback = PeriodicCallback(self.adapt, self.interval * 1000)
+            self.periodic_callback.start()
+
+        if self.interval:
+            try:
+                self.loop.add_callback(f)
+            except AttributeError:
+                IOLoop.current().add_callback(f)
+
+        try:
+            self.plan = set()
+            self.requested = set()
+            self.observed = set()
+        except Exception:
+            pass
+
+        # internal state
+        self.close_counts = collections.defaultdict(int)
+        self._adapting = False
+        self.log = collections.deque(maxlen=10000)
+
+    def stop(self):
+        if self.periodic_callback:
+            self.periodic_callback.stop()
+            self.periodic_callback = None
+
+    async def target(self) -> int:
+        """ The target number of workers that should exist """
+        raise NotImplementedError()
+
+    async def workers_to_close(self, target: int) -> list:
+        """
+        Give a list of workers to close that brings us down to target workers
+        """
+        # TODO, improve me with something that thinks about current load
+        return list(self.observed)[target:]
+
+    async def safe_target(self) -> int:
+        """ Used internally, like target, but respects minimum/maximum """
+        n = await self.target()
+        if n > self.maximum:
+            n = self.maximum
+
+        if n < self.minimum:
+            n = self.minimum
+
+        return n
+
+    async def recommendations(self, target: int) -> dict:
+        """
+        Make scale up/down recommendations based on current state and target
+        """
+        plan = self.plan
+        requested = self.requested
+        observed = self.observed
+
+        if target == len(plan):
+            self.close_counts.clear()
+            return {"status": "same"}
+
+        elif target > len(plan):
+            self.close_counts.clear()
+            return {"status": "up", "n": target}
+
+        elif target < len(plan):
+            not_yet_arrived = requested - observed
+            to_close = set()
+            if not_yet_arrived:
+                to_close.update((toolz.take(len(plan) - target, not_yet_arrived)))
+
+            if target < len(plan) - len(to_close):
+                L = await self.workers_to_close(target=target)
+                to_close.update(L)
+
+            firmly_close = set()
+            for w in to_close:
+                self.close_counts[w] += 1
+                if self.close_counts[w] >= self.wait_count:
+                    firmly_close.add(w)
+
+            for k in list(self.close_counts):  # clear out unseen keys
+                if k in firmly_close or k not in to_close:
+                    del self.close_counts[k]
+
+            if firmly_close:
+                return {"status": "down", "workers": list(firmly_close)}
+            else:
+                return {"status": "same"}
+
+    async def adapt(self) -> None:
+        """
+        Check the current state, make recommendations, call scale
+
+        This is the main event of the system
+        """
+        if self._adapting:  # Semaphore to avoid overlapping adapt calls
+            return
+        self._adapting = True
+
+        try:
+            target = await self.safe_target()
+            recommendations = await self.recommendations(target)
+
+            if recommendations["status"] != "same":
+                self.log.append((time(), dict(recommendations)))
+
+            status = recommendations.pop("status")
+            if status == "same":
+                return
+            if status == "up":
+                await self.scale_up(**recommendations)
+            if status == "down":
+                await self.scale_down(**recommendations)
+        except OSError:
+            self.stop()
+        finally:
+            self._adapting = False
diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index c9e1aca1a87..feb0dfe63b5 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -192,7 +192,6 @@ def __init__(
             self._loop_runner.start()
             self.sync(self._start)
             self.sync(self._correct_state)
-            self.sync(self._wait_for_workers)
 
     async def _start(self):
         while self.status == "starting":
@@ -306,7 +305,6 @@ async def _():
             await self._correct_state()
             if self.workers:
                 await asyncio.wait(list(self.workers.values()))  # maybe there are more
-            await self._wait_for_workers()
             return self
 
         return _().__await__()
@@ -367,7 +365,6 @@ def __del__(self):
 
     def __enter__(self):
         self.sync(self._correct_state)
-        self.sync(self._wait_for_workers)
         assert self.status == "running"
         return self
 
@@ -376,6 +373,13 @@ def __exit__(self, typ, value, traceback):
         self._loop_runner.stop()
 
     def scale(self, n):
+        if len(self.worker_spec) > n:
+            not_yet_launched = set(self.worker_spec) - {
+                v["name"] for v in self.scheduler_info["workers"].values()
+            }
+            while len(self.worker_spec) > n and not_yet_launched:
+                del self.worker_spec[not_yet_launched.pop()]
+
         while len(self.worker_spec) > n:
             self.worker_spec.popitem()
 
@@ -411,12 +415,9 @@ def _supports_scaling(self):
         return not not self.new_spec
 
     async def scale_down(self, workers):
-        workers = set(workers)
-
-        for k, v in self.workers.items():
-            if getattr(v, "worker_address", v.address) in workers:
-                del self.worker_spec[k]
-
+        for w in workers:
+            if w in self.worker_spec:
+                del self.worker_spec[w]
         await self
 
     scale_up = scale  # backwards compatibility
@@ -473,6 +474,7 @@ def dashboard_link(self):
 
     def _widget_status(self):
         workers = len(self.scheduler_info["workers"])
+        requested = len(self.worker_spec)
         cores = sum(v["nthreads"] for v in self.scheduler_info["workers"].values())
         memory = sum(v["memory_limit"] for v in self.scheduler_info["workers"].values())
         memory = format_bytes(memory)
@@ -492,13 +494,13 @@ def _widget_status(self):
     }
   </style>
   <table style="text-align: right;">
-    <tr> <th>Workers</th> <td>%d</td></tr>
+    <tr> <th>Workers</th> <td>%s</td></tr>
     <tr> <th>Cores</th> <td>%d</td></tr>
     <tr> <th>Memory</th> <td>%s</td></tr>
   </table>
 </div>
 """ % (
-            workers,
+            workers if workers == requested else "%d / %d" % (workers, requested),
             cores,
             memory,
         )
@@ -547,6 +549,7 @@ def _widget(self):
 
             def adapt_cb(b):
                 self.adapt(minimum=minimum.value, maximum=maximum.value)
+                update()
 
             adapt.on_click(adapt_cb)
 
@@ -556,6 +559,7 @@ def scale_cb(b):
                     with ignoring(AttributeError):
                         self._adaptive.stop()
                     self.scale(n)
+                    update()
 
             scale.on_click(scale_cb)
         else:
diff --git a/distributed/deploy/tests/test_adaptive.py b/distributed/deploy/tests/test_adaptive.py
index 2d3d2235e21..261b4355251 100644
--- a/distributed/deploy/tests/test_adaptive.py
+++ b/distributed/deploy/tests/test_adaptive.py
@@ -1,7 +1,6 @@
 from time import sleep
 
 import pytest
-from toolz import frequencies, pluck
 from tornado import gen
 from tornado.ioloop import IOLoop
 
@@ -68,51 +67,46 @@ def test_adaptive_local_cluster(loop):
             assert not c.nthreads()
 
 
-@nodebug
-@gen_test(timeout=30)
-def test_adaptive_local_cluster_multi_workers():
-    cluster = yield LocalCluster(
+@pytest.mark.asyncio
+async def test_adaptive_local_cluster_multi_workers(cleanup):
+    async with LocalCluster(
         0,
         scheduler_port=0,
         silence_logs=False,
         processes=False,
         dashboard_address=None,
         asynchronous=True,
-    )
-    try:
-        cluster.scheduler.allowed_failures = 1000
-        alc = cluster.adapt(interval=100)
-        c = yield Client(cluster, asynchronous=True)
-
-        futures = c.map(slowinc, range(100), delay=0.01)
+    ) as cluster:
 
-        start = time()
-        while not cluster.scheduler.workers:
-            yield gen.sleep(0.01)
-            assert time() < start + 15, alc.log
+        cluster.scheduler.allowed_failures = 1000
+        adapt = cluster.adapt(interval="100 ms")
+        async with Client(cluster, asynchronous=True) as c:
+            futures = c.map(slowinc, range(100), delay=0.01)
 
-        yield c.gather(futures)
-        del futures
+            start = time()
+            while not cluster.scheduler.workers:
+                await gen.sleep(0.01)
+                assert time() < start + 15, adapt.log
 
-        start = time()
-        # while cluster.workers:
-        while cluster.scheduler.workers:
-            yield gen.sleep(0.01)
-            assert time() < start + 15, alc.log
+            await c.gather(futures)
+            del futures
 
-        # no workers for a while
-        for i in range(10):
-            assert not cluster.scheduler.workers
-            yield gen.sleep(0.05)
+            start = time()
+            # while cluster.workers:
+            while cluster.scheduler.workers:
+                await gen.sleep(0.01)
+                assert time() < start + 15, adapt.log
 
-        futures = c.map(slowinc, range(100), delay=0.01)
-        yield c.gather(futures)
+            # no workers for a while
+            for i in range(10):
+                assert not cluster.scheduler.workers
+                await gen.sleep(0.05)
 
-    finally:
-        yield c.close()
-        yield cluster.close()
+            futures = c.map(slowinc, range(100), delay=0.01)
+            await c.gather(futures)
 
 
+@pytest.mark.xfail(reason="changed API")
 @pytest.mark.asyncio
 async def test_adaptive_scale_down_override(cleanup):
     class TestAdaptive(Adaptive):
@@ -164,7 +158,7 @@ def test_min_max():
 
         yield gen.sleep(0.2)
         assert len(cluster.scheduler.workers) == 1
-        assert frequencies(pluck(1, adapt.log)) == {"up": 1}
+        assert len(adapt.log) == 1 and adapt.log[-1][1] == {"status": "up", "n": 1}
 
         futures = c.map(slowinc, range(100), delay=0.1)
 
@@ -177,7 +171,7 @@ def test_min_max():
         yield gen.sleep(0.5)
         assert len(cluster.scheduler.workers) == 2
         assert len(cluster.workers) == 2
-        assert frequencies(pluck(1, adapt.log)) == {"up": 2}
+        assert len(adapt.log) == 2 and all(d["status"] == "up" for _, d in adapt.log)
 
         del futures
 
@@ -185,41 +179,35 @@ def test_min_max():
         while len(cluster.scheduler.workers) != 1:
             yield gen.sleep(0.01)
             assert time() < start + 2
-        assert frequencies(pluck(1, adapt.log)) == {"up": 2, "down": 1}
+        assert adapt.log[-1][1]["status"] == "down"
     finally:
         yield c.close()
         yield cluster.close()
 
 
-@gen_test()
-def test_avoid_churn():
+@pytest.mark.asyncio
+async def test_avoid_churn(cleanup):
     """ We want to avoid creating and deleting workers frequently
 
     Instead we want to wait a few beats before removing a worker in case the
     user is taking a brief pause between work
     """
-    cluster = yield LocalCluster(
+    async with LocalCluster(
         0,
         asynchronous=True,
         processes=False,
         scheduler_port=0,
         silence_logs=False,
         dashboard_address=None,
-    )
-    client = yield Client(cluster, asynchronous=True)
-    try:
-        adapt = cluster.adapt(interval="20 ms", wait_count=5)
+    ) as cluster:
+        async with Client(cluster, asynchronous=True) as client:
+            adapt = cluster.adapt(interval="20 ms", wait_count=5)
 
-        for i in range(10):
-            yield client.submit(slowinc, i, delay=0.040)
-            yield gen.sleep(0.040)
+            for i in range(10):
+                await client.submit(slowinc, i, delay=0.040)
+                await gen.sleep(0.040)
 
-        from toolz.curried import pipe, unique, pluck, frequencies
-
-        assert pipe(adapt.log, unique(key=str), pluck(1), frequencies) == {"up": 1}
-    finally:
-        yield client.close()
-        yield cluster.close()
+            assert len(adapt.log) == 1
 
 
 @gen_test(timeout=None)
@@ -238,7 +226,7 @@ def test_adapt_quickly():
         dashboard_address=None,
     )
     client = yield Client(cluster, asynchronous=True)
-    adapt = cluster.adapt(interval=20, wait_count=5, maximum=10)
+    adapt = cluster.adapt(interval="20 ms", wait_count=5, maximum=10)
     try:
         future = client.submit(slowinc, 1, delay=0.100)
         yield wait(future)
@@ -246,10 +234,10 @@ def test_adapt_quickly():
 
         # Scale up when there is plenty of available work
         futures = client.map(slowinc, range(1000), delay=0.100)
-        while frequencies(pluck(1, adapt.log)) == {"up": 1}:
+        while len(adapt.log) == 1:
             yield gen.sleep(0.01)
         assert len(adapt.log) == 2
-        assert "up" in adapt.log[-1]
+        assert adapt.log[-1][1]["status"] == "up"
         d = [x for x in adapt.log[-1] if isinstance(x, dict)][0]
         assert 2 < d["n"] <= adapt.maximum
 
@@ -362,7 +350,7 @@ def test_target_duration():
         dashboard_address=None,
     )
     client = yield Client(cluster, asynchronous=True)
-    adaptive = cluster.adapt(interval="20ms", minimum=2, target_duration="5s")
+    adapt = cluster.adapt(interval="20ms", minimum=2, target_duration="5s")
 
     cluster.scheduler.task_duration["slowinc"] = 1
 
@@ -372,21 +360,21 @@ def test_target_duration():
 
         futures = client.map(slowinc, range(100), delay=0.3)
 
-        while len(adaptive.log) < 2:
+        while len(adapt.log) < 2:
             yield gen.sleep(0.01)
 
-        assert adaptive.log[0][1:] == ("up", {"n": 2})
-        assert adaptive.log[1][1:] == ("up", {"n": 20})
+        assert adapt.log[0][1] == {"status": "up", "n": 2}
+        assert adapt.log[1][1] == {"status": "up", "n": 20}
 
     finally:
         yield client.close()
         yield cluster.close()
 
 
-@gen_test(timeout=None)
-def test_worker_keys():
+@pytest.mark.asyncio
+async def test_worker_keys(cleanup):
     """ Ensure that redefining adapt with a lower maximum removes workers """
-    cluster = yield SpecCluster(
+    async with SpecCluster(
         workers={
             "a-1": {"cls": Worker},
             "a-2": {"cls": Worker},
@@ -394,9 +382,7 @@ def test_worker_keys():
             "b-2": {"cls": Worker},
         },
         asynchronous=True,
-    )
-
-    try:
+    ) as cluster:
 
         def key(ws):
             return ws.name.split("-")[0]
@@ -404,12 +390,10 @@ def key(ws):
         cluster._adaptive_options = {"worker_key": key}
 
         adaptive = cluster.adapt(minimum=1)
-        yield adaptive._adapt()
+        await adaptive.adapt()
 
         while len(cluster.scheduler.workers) == 4:
-            yield gen.sleep(0.01)
+            await gen.sleep(0.01)
 
         names = {ws.name for ws in cluster.scheduler.workers.values()}
         assert names == {"a-1", "a-2"} or names == {"b-1", "b-2"}
-    finally:
-        yield cluster.close()
diff --git a/distributed/deploy/tests/test_adaptive_core.py b/distributed/deploy/tests/test_adaptive_core.py
new file mode 100644
index 00000000000..a073314223d
--- /dev/null
+++ b/distributed/deploy/tests/test_adaptive_core.py
@@ -0,0 +1,90 @@
+import asyncio
+import pytest
+
+from distributed.deploy.adaptive_core import AdaptiveCore
+from distributed.metrics import time
+
+
+class MyAdaptive(AdaptiveCore):
+    def __init__(self, *args, interval=None, **kwargs):
+        super().__init__(*args, interval=interval, **kwargs)
+        self._target = 0
+        self._log = []
+
+    async def target(self):
+        return self._target
+
+    async def scale_up(self, n=0):
+        self.plan = self.requested = set(range(n))
+
+    async def scale_down(self, workers=()):
+        for collection in [self.plan, self.requested, self.observed]:
+            for w in workers:
+                collection.discard(w)
+
+
+@pytest.mark.asyncio
+async def test_safe_target():
+    adapt = MyAdaptive(minimum=1, maximum=4)
+    assert await adapt.safe_target() == 1
+    adapt._target = 10
+    assert await adapt.safe_target() == 4
+
+
+@pytest.mark.asyncio
+async def test_scale_up():
+    adapt = MyAdaptive(minimum=1, maximum=4)
+    await adapt.adapt()
+    assert adapt.log[-1][1] == {"status": "up", "n": 1}
+    assert adapt.plan == {0}
+
+    adapt._target = 10
+    await adapt.adapt()
+    assert adapt.log[-1][1] == {"status": "up", "n": 4}
+    assert adapt.plan == {0, 1, 2, 3}
+
+
+@pytest.mark.asyncio
+async def test_scale_down():
+    adapt = MyAdaptive(minimum=1, maximum=4, wait_count=2)
+    adapt._target = 10
+    await adapt.adapt()
+    assert len(adapt.log) == 1
+
+    adapt.observed = {0, 1, 3}  # all but 2 have arrived
+
+    adapt._target = 2
+    await adapt.adapt()
+    assert len(adapt.log) == 1  # no change after only one call
+    await adapt.adapt()
+    assert len(adapt.log) == 2  # no change after only one call
+    assert adapt.log[-1][1]["status"] == "down"
+    assert 2 in adapt.log[-1][1]["workers"]
+    assert len(adapt.log[-1][1]["workers"]) == 2
+
+    old = list(adapt.log)
+    await adapt.adapt()
+    await adapt.adapt()
+    await adapt.adapt()
+    await adapt.adapt()
+    assert list(adapt.log) == old
+
+
+@pytest.mark.asyncio
+async def test_interval():
+    adapt = MyAdaptive(interval="5 ms")
+    assert not adapt.plan
+
+    for i in [0, 3, 1]:
+        start = time()
+        adapt._target = i
+        while len(adapt.plan) != i:
+            await asyncio.sleep(0.001)
+            assert time() < start + 2
+
+    adapt.stop()
+    await asyncio.sleep(0.050)
+
+    adapt._target = 10
+    await asyncio.sleep(0.020)
+    assert len(adapt.plan) == 1  # last value from before, unchanged
diff --git a/distributed/deploy/tests/test_slow_adaptive.py b/distributed/deploy/tests/test_slow_adaptive.py
new file mode 100644
index 00000000000..4f565a78289
--- /dev/null
+++ b/distributed/deploy/tests/test_slow_adaptive.py
@@ -0,0 +1,98 @@
+import asyncio
+import pytest
+
+from dask.distributed import Worker, Scheduler, SpecCluster, Client
+from distributed.utils_test import slowinc, cleanup  # noqa: F401
+from distributed.metrics import time
+
+
+class SlowWorker(object):
+    def __init__(self, *args, delay=0, **kwargs):
+        self.worker = Worker(*args, **kwargs)
+        self.delay = delay
+        self.status = None
+
+    @property
+    def address(self):
+        return self.worker.address
+
+    def __await__(self):
+        async def now():
+            if self.status != "running":
+                self.worker.loop.call_later(self.delay, self.worker.start)
+                self.status = "running"
+            return self
+
+        return now().__await__()
+
+    async def close(self):
+        await self.worker.close()
+        self.status = "closed"
+
+
+scheduler = {"cls": Scheduler, "options": {"port": 0}}
+
+
+@pytest.mark.asyncio
+async def test_startup(cleanup):
+    start = time()
+    async with SpecCluster(
+        scheduler=scheduler,
+        workers={
+            0: {"cls": Worker, "options": {}},
+            1: {"cls": SlowWorker, "options": {"delay": 5}},
+            2: {"cls": SlowWorker, "options": {"delay": 0}},
+        },
+        asynchronous=True,
+    ) as cluster:
+        assert len(cluster.workers) == len(cluster.worker_spec) == 3
+        assert time() < start + 5
+        assert 1 <= len(cluster.scheduler_info["workers"]) <= 2
+
+        async with Client(cluster, asynchronous=True) as client:
+            await client.wait_for_workers(n_workers=2)
+
+
+@pytest.mark.asyncio
+async def test_scale_up_down(cleanup):
+    start = time()
+    async with SpecCluster(
+        scheduler=scheduler,
+        workers={
+            "slow": {"cls": SlowWorker, "options": {"delay": 5}},
+            "fast": {"cls": Worker, "options": {}},
+        },
+        asynchronous=True,
+    ) as cluster:
+        cluster.scale(1)  # remove a worker, hopefully the one we don't have
+        await cluster
+
+        assert list(cluster.worker_spec) == ["fast"]
+
+        cluster.scale(0)
+        await cluster
+        assert not cluster.worker_spec
+
+
+@pytest.mark.asyncio
+async def test_adaptive(cleanup):
+    start = time()
+    async with SpecCluster(
+        scheduler=scheduler,
+        workers={"fast": {"cls": Worker, "options": {}}},
+        worker={"cls": SlowWorker, "options": {"delay": 5}},
+        asynchronous=True,
+    ) as cluster:
+        cluster.adapt(minimum=1, maximum=4, target_duration="1s", interval="20ms")
+        async with Client(cluster, asynchronous=True) as client:
+            futures = client.map(slowinc, range(200), delay=0.1)
+
+            while len(cluster.worker_spec) <= 1:
+                await asyncio.sleep(0.05)
+
+            del futures
+
+            while len(cluster.worker_spec) > 1:
+                await asyncio.sleep(0.05)
+
+            assert list(cluster.worker_spec) == ["fast"]
diff --git a/distributed/deploy/tests/test_spec_cluster.py b/distributed/deploy/tests/test_spec_cluster.py
index 58bbbaef44d..64633428a38 100644
--- a/distributed/deploy/tests/test_spec_cluster.py
+++ b/distributed/deploy/tests/test_spec_cluster.py
@@ -252,7 +252,10 @@ async def test_dashboard_link(cleanup):
 @pytest.mark.asyncio
 async def test_widget(cleanup):
     async with SpecCluster(
-        workers=worker_spec, scheduler=scheduler, asynchronous=True
+        workers=worker_spec,
+        scheduler=scheduler,
+        asynchronous=True,
+        worker={"cls": Worker, "options": {"nthreads": 1}},
     ) as cluster:
 
         start = time()  # wait for all workers
@@ -262,3 +265,6 @@ async def test_widget(cleanup):
 
         assert "3" in cluster._widget_status()
         assert "GB" in cluster._widget_status()
+
+        cluster.scale(5)
+        assert "3 / 5" in cluster._widget_status()
diff --git a/distributed/nanny.py b/distributed/nanny.py
index 155fde98158..b6d8dadbf9a 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -690,7 +690,7 @@ async def run():
                         }
                     )
                     init_result_q.close()
-                    await worker.wait_until_closed()
+                    await worker.finished()
                     logger.info("Worker closed")
 
         try:
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index acabce22c63..e9fdbe60b51 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -2898,7 +2898,14 @@ def replicate(
         )
 
     def workers_to_close(
-        self, comm=None, memory_ratio=None, n=None, key=None, minimum=None
+        self,
+        comm=None,
+        memory_ratio=None,
+        n=None,
+        key=None,
+        minimum=None,
+        target=None,
+        attribute="address",
     ):
         """
         Find workers that we can close with low cost
@@ -2925,6 +2932,11 @@ def workers_to_close(
             An optional callable mapping a WorkerState object to a group
             affiliation.  Groups will be closed together.  This is useful when
             closing workers must be done collectively, such as by hostname.
+        target: int
+            Target number of workers to have after we close
+        attribute : str
+            The attribute of the WorkerState object to return, like "address"
+            or "name".  Defaults to "address".
 
         Examples
         --------
@@ -2952,6 +2964,13 @@ def workers_to_close(
         --------
         Scheduler.retire_workers
         """
+        if target is not None and n is None:
+            n = len(self.workers) - target
+        if n is not None:
+            if n < 0:
+                n = 0
+            target = len(self.workers) - n
+
         if n is None and memory_ratio is None:
             memory_ratio = 2
 
@@ -2976,12 +2995,12 @@ def workers_to_close(
             limit = sum(limit_bytes.values())
             total = sum(group_bytes.values())
 
-            def key(group):
+            def _key(group):
                 is_idle = not any(ws.processing for ws in groups[group])
                 bytes = -group_bytes[group]
                 return (is_idle, bytes)
 
-            idle = sorted(groups, key=key)
+            idle = sorted(groups, key=_key)
 
             to_close = []
             n_remain = len(self.workers)
@@ -2996,7 +3015,7 @@ def key(group):
 
                 limit -= limit_bytes[group]
 
-                if (n is not None and len(to_close) < n) or (
+                if (n is not None and n_remain - len(groups[group]) >= target) or (
                     memory_ratio is not None and limit >= memory_ratio * total
                 ):
                     to_close.append(group)
@@ -3005,22 +3024,30 @@ def key(group):
                 else:
                     break
 
-            result = [ws.address for g in to_close for ws in groups[g]]
+            result = [getattr(ws, attribute) for g in to_close for ws in groups[g]]
             if result:
                 logger.info("Suggest closing workers: %s", result)
 
             return result
 
     async def retire_workers(
-        self, comm=None, workers=None, remove=True, close_workers=False, **kwargs
+        self,
+        comm=None,
+        workers=None,
+        remove=True,
+        close_workers=False,
+        names=None,
+        **kwargs
     ):
         """ Gracefully retire workers from cluster
 
         Parameters
         ----------
         workers: list (optional)
-            List of worker IDs to retire.
+            List of worker addresses to retire.
             If not provided we call ``workers_to_close`` which finds a good set
+        workers_names: list (optional)
+            List of worker names to retire.
         remove: bool (defaults to True)
             Whether or not to remove the worker metadata immediately or else
             wait for the worker to contact us
@@ -3042,6 +3069,11 @@ async def retire_workers(
         Scheduler.workers_to_close
         """
         with log_errors():
+            if names is not None:
+                names = set(names)
+                workers = [
+                    ws.address for ws in self.workers.values() if ws.name in names
+                ]
             if workers is None:
                 while True:
                     try:
@@ -3052,17 +3084,16 @@ async def retire_workers(
                                 remove=remove,
                                 close_workers=close_workers,
                             )
-                        raise gen.Return(workers)
+                        return workers
                     except KeyError:  # keys left during replicate
                         pass
-
             workers = {self.workers[w] for w in workers if w in self.workers}
-            if len(workers) > 0:
-                # Keys orphaned by retiring those workers
-                keys = set.union(*[w.has_what for w in workers])
-                keys = {ts.key for ts in keys if ts.who_has.issubset(workers)}
-            else:
-                keys = set()
+            if not workers:
+                return []
+
+            # Keys orphaned by retiring those workers
+            keys = set.union(*[w.has_what for w in workers])
+            keys = {ts.key for ts in keys if ts.who_has.issubset(workers)}
 
             other_workers = set(self.workers.values()) - workers
             if keys:
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 80cc04c81b3..0331ac0a972 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -1231,7 +1231,9 @@ def test_cancel_fire_and_forget(c, s, a, b):
     assert not s.tasks
 
 
-@gen_cluster(client=True, Worker=Nanny, clean_kwargs={"processes": False})
+@gen_cluster(
+    client=True, Worker=Nanny, clean_kwargs={"processes": False, "threads": False}
+)
 def test_log_tasks_during_restart(c, s, a, b):
     future = c.submit(sys.exit, 0)
     yield wait(future)
diff --git a/distributed/worker.py b/distributed/worker.py
index 672358d4460..290128c84e1 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -26,7 +26,6 @@
     from toolz import pluck, partial, merge, first
 from tornado import gen
 from tornado.ioloop import IOLoop
-from tornado.locks import Event
 
 from . import profile, comm
 from .batched import BatchedSend
@@ -546,7 +545,6 @@ def __init__(
         self.actors = {}
         self.loop = loop or IOLoop.current()
         self.status = None
-        self._closed = Event()
         self.reconnect = reconnect
         self.executor = executor or ThreadPoolExecutor(
             self.nthreads, thread_name_prefix="Dask-Worker-Threads'"
@@ -1054,7 +1052,6 @@ async def close(
 
             self.stop()
             self.rpc.close()
-            self._closed.set()
 
             self.status = "closed"
             await ServerNode.close(self)
@@ -1084,7 +1081,8 @@ async def terminate(self, comm, report=True, **kwargs):
         return "OK"
 
     async def wait_until_closed(self):
-        await self._closed.wait()
+        warnings.warn("wait_until_closed has moved to finished()")
+        await self.finished()
         assert self.status == "closed"
 
     ################

From 4dc3d196baafe5c3c704d894c9cbb7d80f61f6cc Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 2 Aug 2019 09:51:45 -0700
Subject: [PATCH 0394/1550] Add keep-alive message between worker and scheduler
 (#2907)

This is effectively a heartbeat, but much simpler and less frequent than
our current heartbeats

Fixes #2524
---
 distributed/scheduler.py         |  1 +
 distributed/tests/test_client.py |  2 +-
 distributed/tests/test_core.py   | 13 ++++++++++---
 distributed/worker.py            |  7 +++++++
 4 files changed, 19 insertions(+), 4 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index e9fdbe60b51..8a2fe03c8d1 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1025,6 +1025,7 @@ def __init__(
             "missing-data": self.handle_missing_data,
             "long-running": self.handle_long_running,
             "reschedule": self.reschedule,
+            "keep-alive": lambda *args, **kwargs: None,
         }
 
         client_handlers = {
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 3eb9f39a2a4..d89562dc1ff 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -3319,7 +3319,7 @@ def test_get_foo_lost_keys(c, s, u, v, w):
     client=True,
     Worker=Nanny,
     worker_kwargs={"death_timeout": "500ms"},
-    clean_kwargs={"threads": False},
+    clean_kwargs={"threads": False, "processes": False},
 )
 def test_bad_tasks_fail(c, s, a, b):
     f = c.submit(sys.exit, 0)
diff --git a/distributed/tests/test_core.py b/distributed/tests/test_core.py
index f91b8b64367..cad622980df 100644
--- a/distributed/tests/test_core.py
+++ b/distributed/tests/test_core.py
@@ -445,9 +445,16 @@ def test_identity_inproc():
 
 
 def test_ports(loop):
-    port = 9877
-    server = Server({}, io_loop=loop)
-    server.listen(port)
+    for port in range(9877, 9887):
+        server = Server({}, io_loop=loop)
+        try:
+            server.listen(port)
+        except OSError:  # port already taken?
+            pass
+        else:
+            break
+    else:
+        raise Exception()
     try:
         assert server.port == port
 
diff --git a/distributed/worker.py b/distributed/worker.py
index 290128c84e1..dfdd6df8e4a 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -820,6 +820,13 @@ async def _register_with_scheduler(self):
 
         self.batched_stream = BatchedSend(interval="2ms", loop=self.loop)
         self.batched_stream.start(comm)
+        pc = PeriodicCallback(
+            lambda: self.batched_stream.send({"op": "keep-alive"}),
+            60000,
+            io_loop=self.io_loop,
+        )
+        self.periodic_callbacks["keep-alive"] = pc
+        pc.start()
         self.periodic_callbacks["heartbeat"].start()
         self.loop.add_callback(self.handle_scheduler, comm)
 

From 6caa30896e66501483416812d44c861da75ceab6 Mon Sep 17 00:00:00 2001
From: Martin Durant <martindurant@users.noreply.github.com>
Date: Fri, 2 Aug 2019 15:11:17 -0400
Subject: [PATCH 0395/1550] Fix docstring [skip ci] (#2917)

Fixes #2914
---
 distributed/deploy/local.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index 877a74587e9..20476ad8065 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -80,7 +80,7 @@ class LocalCluster(SpecCluster):
 
     Pass extra keyword arguments to Bokeh
 
-    >>> LocalCluster(service_kwargs={'bokeh': {'prefix': '/foo'}})  # doctest: +SKIP
+    >>> LocalCluster(service_kwargs={'dashboard': {'prefix': '/foo'}})  # doctest: +SKIP
     """
 
     def __init__(

From 20ba1a7405a6a5eb59e14808abc5f6ff823ab48d Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Sat, 3 Aug 2019 12:25:05 -0400
Subject: [PATCH 0396/1550] Raise informative warning when rescheduling an
 unknown task (#2916)

---
 distributed/scheduler.py            |  9 ++++++++-
 distributed/tests/test_scheduler.py | 11 +++++++++++
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 8a2fe03c8d1..f84b3d1bce4 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -4364,7 +4364,14 @@ def reschedule(self, key=None, worker=None):
         Things may have shifted and this task may now be better suited to run
         elsewhere
         """
-        ts = self.tasks[key]
+        try:
+            ts = self.tasks[key]
+        except KeyError:
+            logger.warning(
+                "Attempting to reschedule task {}, which was not "
+                "found on the scheduler. Aborting reschedule.".format(key)
+            )
+            return
         if ts.state != "processing":
             return
         if worker and ts.processing_on.address != worker:
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 0331ac0a972..9035fbd8667 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -6,6 +6,7 @@
 import operator
 import sys
 from time import sleep
+import logging
 
 import dask
 from dask import delayed
@@ -23,6 +24,7 @@
 from distributed.worker import dumps_function, dumps_task
 from distributed.utils import tmpfile
 from distributed.utils_test import (  # noqa: F401
+    captured_logger,
     cleanup,
     inc,
     dec,
@@ -1260,6 +1262,15 @@ def test_reschedule(c, s, a, b):
     assert sum(future.key in a.data for future in x) <= 1
 
 
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 2)
+def test_reschedule_warns(c, s, a, b):
+    with captured_logger(logging.getLogger("distributed.scheduler")) as sched:
+        s.reschedule(key="__this-key-does-not-exist__")
+
+    assert "not found on the scheduler" in sched.getvalue()
+    assert "Aborting reschedule" in sched.getvalue()
+
+
 @gen_cluster(client=True)
 def test_get_task_status(c, s, a, b):
     future = c.submit(inc, 1)

From fb4e48fa5cb83bd0fe32d4c1fe23645d30798c21 Mon Sep 17 00:00:00 2001
From: Martin Durant <martindurant@users.noreply.github.com>
Date: Sat, 3 Aug 2019 17:33:00 -0400
Subject: [PATCH 0397/1550] Give 404 when requesting nonexistent tasks or
 workers (#2921)

---
 distributed/dashboard/scheduler_html.py       |  6 ++++++
 .../tests/test_scheduler_bokeh_html.py        | 21 ++++++++++++++++++-
 2 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/distributed/dashboard/scheduler_html.py b/distributed/dashboard/scheduler_html.py
index 3087f323b5f..1377b037173 100644
--- a/distributed/dashboard/scheduler_html.py
+++ b/distributed/dashboard/scheduler_html.py
@@ -27,6 +27,9 @@ def get(self):
 class Worker(RequestHandler):
     def get(self, worker):
         worker = escape.url_unescape(worker)
+        if worker not in self.server.workers:
+            self.send_error(404)
+            return
         with log_errors():
             self.render(
                 "worker.html",
@@ -40,6 +43,9 @@ def get(self, worker):
 class Task(RequestHandler):
     def get(self, task):
         task = escape.url_unescape(task)
+        if task not in self.server.tasks:
+            self.send_error(404)
+            return
         with log_errors():
             self.render(
                 "task.html",
diff --git a/distributed/dashboard/tests/test_scheduler_bokeh_html.py b/distributed/dashboard/tests/test_scheduler_bokeh_html.py
index 660602df09a..f2a2c880a94 100644
--- a/distributed/dashboard/tests/test_scheduler_bokeh_html.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh_html.py
@@ -6,7 +6,7 @@
 pytest.importorskip("bokeh")
 
 from tornado.escape import url_escape
-from tornado.httpclient import AsyncHTTPClient
+from tornado.httpclient import AsyncHTTPClient, HTTPClientError
 
 from dask.sizeof import sizeof
 from distributed.utils import is_valid_xml
@@ -49,6 +49,25 @@ def test_connect(c, s, a, b):
             assert not re.search("href=./", body)  # no absolute links
 
 
+@gen_cluster(
+    client=True,
+    nthreads=[],
+    scheduler_kwargs={"services": {("dashboard", 0): BokehScheduler}},
+)
+def test_worker_404(c, s):
+    http_client = AsyncHTTPClient()
+    with pytest.raises(HTTPClientError) as err:
+        yield http_client.fetch(
+            "http://localhost:%d/info/worker/unknown" % s.services["dashboard"].port
+        )
+    assert err.value.code == 404
+    with pytest.raises(HTTPClientError) as err:
+        yield http_client.fetch(
+            "http://localhost:%d/info/task/unknown" % s.services["dashboard"].port
+        )
+    assert err.value.code == 404
+
+
 @gen_cluster(
     client=True,
     scheduler_kwargs={

From b68660cb873e8efb9d9b47a92e39ab78f2fd7573 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sat, 3 Aug 2019 19:20:03 -0700
Subject: [PATCH 0398/1550] Cleanup async warnings in tests (#2920)

---
 distributed/client.py                  |  5 ++-
 distributed/tests/test_as_completed.py |  2 +-
 distributed/tests/test_client.py       | 51 +++++++++++++++++++-------
 distributed/tests/test_core.py         |  2 +-
 distributed/tests/test_variable.py     | 17 +++++----
 distributed/utils_test.py              |  2 +-
 distributed/worker.py                  |  4 +-
 7 files changed, 55 insertions(+), 28 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index fab5ff0bf6e..93501ae2077 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -2860,7 +2860,7 @@ async def _upload_large_file(self, local_filename, remote_filename=None):
 
         def dump_to_file(dask_worker=None):
             if not os.path.isabs(remote_filename):
-                fn = os.path.join(dask_worker.local_dir, remote_filename)
+                fn = os.path.join(dask_worker.local_directory, remote_filename)
             else:
                 fn = remote_filename
             with open(fn, "wb") as f:
@@ -3267,7 +3267,8 @@ def scheduler_info(self, **kwargs):
                                          'stored': 0,
                                          'time-delay': 0.0061032772064208984}}}
         """
-        self.sync(self._update_scheduler_info)
+        if not self.asynchronous:
+            self.sync(self._update_scheduler_info)
         return self._scheduler_identity
 
     def write_scheduler_file(self, scheduler_file):
diff --git a/distributed/tests/test_as_completed.py b/distributed/tests/test_as_completed.py
index 911ff388e06..d74d033c64a 100644
--- a/distributed/tests/test_as_completed.py
+++ b/distributed/tests/test_as_completed.py
@@ -204,7 +204,7 @@ def test_as_completed_with_results_async(c, s, a, b):
     z = c.submit(inc, 1)
 
     ac = as_completed([x, y, z], with_results=True)
-    y.cancel()
+    yield y.cancel()
     with pytest.raises(RuntimeError) as exc:
         first = yield ac.__anext__()
         second = yield ac.__anext__()
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index d89562dc1ff..7f0036e2a7a 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -6,6 +6,7 @@
 from operator import add
 import os
 import pickle
+import psutil
 import random
 import subprocess
 import sys
@@ -1529,9 +1530,9 @@ def g():
         return package_1.a, package_2.b
 
     # c.upload_file tells each worker to
-    # - put this file in their local_dir
+    # - put this file in their local_directory
     # - modify their sys.path to include it
-    # we don't care about the local_dir
+    # we don't care about the local_directory
     # but we do care about restoring the path
 
     with save_sys_modules():
@@ -1581,19 +1582,19 @@ def g():
 
 @gen_cluster(client=True)
 def test_upload_large_file(c, s, a, b):
-    assert a.local_dir
-    assert b.local_dir
+    assert a.local_directory
+    assert b.local_directory
     with tmp_text("myfile", "abc") as fn:
         with tmp_text("myfile2", "def") as fn2:
             yield c._upload_large_file(fn, remote_filename="x")
             yield c._upload_large_file(fn2)
 
             for w in [a, b]:
-                assert os.path.exists(os.path.join(w.local_dir, "x"))
-                assert os.path.exists(os.path.join(w.local_dir, "myfile2"))
-                with open(os.path.join(w.local_dir, "x")) as f:
+                assert os.path.exists(os.path.join(w.local_directory, "x"))
+                assert os.path.exists(os.path.join(w.local_directory, "myfile2"))
+                with open(os.path.join(w.local_directory, "x")) as f:
                     assert f.read() == "abc"
-                with open(os.path.join(w.local_dir, "myfile2")) as f:
+                with open(os.path.join(w.local_directory, "myfile2")) as f:
                     assert f.read() == "def"
 
 
@@ -4568,7 +4569,7 @@ def test_quiet_client_close(loop):
 @pytest.mark.slow
 def test_quiet_client_close_when_cluster_is_closed_before_client(loop):
     with captured_logger(logging.getLogger("tornado.application")) as logger:
-        cluster = LocalCluster(loop=loop, n_workers=1)
+        cluster = LocalCluster(loop=loop, n_workers=1, dashboard_address=":0")
         client = Client(cluster, loop=loop)
         cluster.close()
         client.close()
@@ -5179,7 +5180,7 @@ def test_scatter_direct(s, a, b):
         yield gen.sleep(0.10)
         assert time() < start + 5
 
-    yield c._close()
+    yield c.close()
 
 
 @pytest.mark.skipif(sys.version_info[0] < 3, reason="cloudpickle Py27 issue")
@@ -5196,7 +5197,7 @@ def test_client_name(s, a, b):
         c = yield Client(s.address, asynchronous=True)
         assert any("hello-world" in name for name in list(s.clients))
 
-    yield c._close()
+    yield c.close()
 
 
 def test_client_doesnt_close_given_loop(loop, s, a, b):
@@ -5301,7 +5302,7 @@ def test(s, a, b):
             with pytest.raises(TypeError):
                 yield c.run_on_scheduler(lambda: inc)
         finally:
-            yield c._close()
+            yield c.close()
 
     test()
 
@@ -5324,7 +5325,7 @@ def test_de_serialization(s, a, b):
         with pytest.raises(TypeError):
             result = yield future
     finally:
-        yield c._close()
+        yield c.close()
 
 
 @gen_cluster()
@@ -5340,7 +5341,7 @@ def test_de_serialization_none(s, a, b):
         with pytest.raises(TypeError):
             result = yield future
     finally:
-        yield c._close()
+        yield c.close()
 
 
 @gen_cluster()
@@ -5591,5 +5592,27 @@ def test_wait_for_workers(c, s, a, b):
     yield w.close()
 
 
+@pytest.mark.skipif(WINDOWS, reason="num_fds not supported on windows")
+@pytest.mark.asyncio
+@pytest.mark.parametrize("Worker", [Worker, Nanny])
+async def test_file_descriptors_dont_leak(Worker):
+    pytest.importorskip("pandas")
+    df = dask.datasets.timeseries(freq="10s", dtypes={"x": int, "y": float})
+
+    proc = psutil.Process()
+    start = proc.num_fds()
+    async with Scheduler(port=0, dashboard_address=":0") as s:
+        async with Worker(s.address, nthreads=2) as a, Worker(
+            s.address, nthreads=2
+        ) as b:
+            async with Client(s.address, asynchronous=True) as c:
+                await df.sum().persist()
+
+    begin = time()
+    while proc.num_fds() > begin:
+        await asyncio.sleep(0.01)
+        assert time() < begin + 5, (start, proc.num_fds())
+
+
 if sys.version_info >= (3, 5):
     from distributed.tests.py3_test_client import *  # noqa F401
diff --git a/distributed/tests/test_core.py b/distributed/tests/test_core.py
index cad622980df..e41866d6741 100644
--- a/distributed/tests/test_core.py
+++ b/distributed/tests/test_core.py
@@ -126,7 +126,7 @@ def f():
         assert isinstance(msg["exception"], ValueError)
         assert "'ping' handler has been explicitly disallowed" in repr(msg["exception"])
 
-        comm.close()
+        yield comm.close()
         server.stop()
 
     res = loop.run_sync(f)
diff --git a/distributed/tests/test_variable.py b/distributed/tests/test_variable.py
index 88f96a241b0..6dcca9c9cf4 100644
--- a/distributed/tests/test_variable.py
+++ b/distributed/tests/test_variable.py
@@ -1,3 +1,4 @@
+import asyncio
 import random
 from time import sleep
 import sys
@@ -99,7 +100,7 @@ def test_timeout_sync(client):
 
 
 @gen_cluster(client=True)
-def test_cleanup(c, s, a, b):
+async def test_cleanup(c, s, a, b):
     v = Variable("v")
     vv = Variable("v")
 
@@ -107,17 +108,17 @@ def test_cleanup(c, s, a, b):
     y = c.submit(lambda x: x + 1, 20)
     x_key = x.key
 
-    yield v.set(x)
+    await v.set(x)
     del x
-    yield gen.sleep(0.1)
+    await gen.sleep(0.1)
 
-    t_future = xx = vv._get()
-    yield gen.moment
-    v._set(y)
+    t_future = xx = asyncio.ensure_future(vv._get())
+    await gen.sleep(0)
+    asyncio.ensure_future(v.set(y))
 
-    future = yield t_future
+    future = await t_future
     assert future.key == x_key
-    result = yield future
+    result = await future
     assert result == 11
 
 
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index 8293bb474e2..13639f05fa5 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -636,7 +636,7 @@ def cluster(
             q = mp_context.Queue()
             fn = "_test_worker-%s" % uuid.uuid4()
             kwargs = merge(
-                {"nthreads": 1, "local_dir": fn, "memory_limit": TOTAL_MEMORY},
+                {"nthreads": 1, "local_directory": fn, "memory_limit": TOTAL_MEMORY},
                 worker_kwargs,
             )
             proc = mp_context.Process(
diff --git a/distributed/worker.py b/distributed/worker.py
index dfdd6df8e4a..795cb93e9f3 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -711,7 +711,9 @@ def worker_address(self):
     @property
     def local_dir(self):
         """ For API compatibility with Nanny """
-        warnings.warn("The local_dir attribute has moved to local_directory")
+        warnings.warn(
+            "The local_dir attribute has moved to local_directory", stacklevel=2
+        )
         return self.local_directory
 
     def get_metrics(self):

From 2428cc822a51bb12832c2bcc2bea2fa001e40d30 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sun, 4 Aug 2019 12:22:08 -0700
Subject: [PATCH 0399/1550] Add documentation around spec.ProcessInterface
 (#2923)

---
 distributed/deploy/spec.py | 29 +++++++++++++++++++++--------
 1 file changed, 21 insertions(+), 8 deletions(-)

diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index feb0dfe63b5..af4f4a3f23d 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -24,13 +24,12 @@
 
 
 class ProcessInterface:
-    """ An interface for Scheduler and Worker processes for use in SpecCluster
-
-    Parameters
-    ----------
-    loop:
-        A pointer to the running loop.
+    """
+    An interface for Scheduler and Worker processes for use in SpecCluster
 
+    This interface is responsible to submit a worker or scheduler process to a
+    resource manager like Kubernetes, Yarn, or SLURM/PBS/SGE/...
+    It should implement the methods below, like ``start`` and ``close``
     """
 
     def __init__(self):
@@ -49,11 +48,25 @@ async def _():
         return _().__await__()
 
     async def start(self):
-        """ Start the process. """
+        """ Submit the process to the resource manager
+
+        For workers this doesn't have to wait until the process actually starts,
+        but can return once the resource manager has the request, and will work
+        to make the job exist in the future
+
+        For the scheduler we will expect the scheduler's ``.address`` attribute
+        to be avaialble after this completes.
+        """
         self.status = "running"
 
     async def close(self):
-        """ Close the process. """
+        """ Close the process
+
+        This will be called by the Cluster object when we scale down a node,
+        but only after we ask the Scheduler to close the worker gracefully.
+        This method should kill the process a bit more forcefully and does not
+        need to worry about shutting down gracefully
+        """
         self.status = "closed"
 
     def __repr__(self):

From 29389702a88253f6504cca0b8304a86a94c2e677 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sun, 4 Aug 2019 14:31:00 -0700
Subject: [PATCH 0400/1550] Add timeouts to processes in SSH tests (#2925)

It turns out that OpenSSH doesn't pass through terminate/kill signals,
so we had some zombie processes hanging around sending signals around where
they shouldn't.

Now we place idle and death timeouts on the launched processes to keep them in
check.

See https://github.com/ronf/asyncssh/issues/112 for more information on the
underlying issue.
---
 distributed/deploy/ssh2.py            | 6 ++++--
 distributed/deploy/tests/test_ssh2.py | 5 +++--
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/distributed/deploy/ssh2.py b/distributed/deploy/ssh2.py
index 189a61df5f1..064e580b111 100644
--- a/distributed/deploy/ssh2.py
+++ b/distributed/deploy/ssh2.py
@@ -34,11 +34,13 @@ def __init__(self, **kwargs):
 
     async def start(self):
         assert self.connection
-        weakref.finalize(self, self.proc.terminate)
+        weakref.finalize(
+            self, self.proc.kill
+        )  # https://github.com/ronf/asyncssh/issues/112
         await super().start()
 
     async def close(self):
-        self.proc.terminate()
+        self.proc.kill()  # https://github.com/ronf/asyncssh/issues/112
         self.connection.close()
         await super().close()
 
diff --git a/distributed/deploy/tests/test_ssh2.py b/distributed/deploy/tests/test_ssh2.py
index b744d352b8b..076711bb841 100644
--- a/distributed/deploy/tests/test_ssh2.py
+++ b/distributed/deploy/tests/test_ssh2.py
@@ -12,7 +12,8 @@ async def test_basic():
         ["127.0.0.1"] * 3,
         connect_kwargs=dict(known_hosts=None),
         asynchronous=True,
-        scheduler_kwargs={"port": 0},
+        scheduler_kwargs={"port": 0, "idle_timeout": "5s"},
+        worker_kwargs={"death_timeout": "5s"},
     ) as cluster:
         assert len(cluster.workers) == 2
         async with Client(cluster, asynchronous=True) as client:
@@ -29,7 +30,7 @@ async def test_keywords():
         ["127.0.0.1"] * 3,
         connect_kwargs=dict(known_hosts=None),
         asynchronous=True,
-        worker_kwargs={"nthreads": 2, "memory_limit": "2 GiB"},
+        worker_kwargs={"nthreads": 2, "memory_limit": "2 GiB", "death_timeout": "5s"},
         scheduler_kwargs={"idle_timeout": "5s", "port": 0},
     ) as cluster:
         async with Client(cluster, asynchronous=True) as client:

From e02cc4409e352e40e4128fc542b8aaed51b5a01f Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sun, 4 Aug 2019 14:31:32 -0700
Subject: [PATCH 0401/1550] Always kill processes in clean tests, even if we
 don't check (#2924)

Also allow ValueErrors when collecting data from workers
---
 distributed/utils_comm.py |  8 ++++++++
 distributed/utils_test.py | 19 +++++++++++--------
 2 files changed, 19 insertions(+), 8 deletions(-)

diff --git a/distributed/utils_comm.py b/distributed/utils_comm.py
index f6b4ea36e4f..53504d11939 100644
--- a/distributed/utils_comm.py
+++ b/distributed/utils_comm.py
@@ -1,6 +1,7 @@
 import asyncio
 from collections import defaultdict
 from itertools import cycle
+import logging
 import random
 
 from dask.optimization import SubgraphCallable
@@ -9,6 +10,8 @@
 from .core import rpc
 from .utils import All, tokey
 
+logger = logging.getLogger(__name__)
+
 
 async def gather_from_workers(who_has, rpc, close=True, serializers=None, who=None):
     """ Gather data directly from peers
@@ -72,6 +75,11 @@ async def gather_from_workers(who_has, rpc, close=True, serializers=None, who=No
                     r = await c
                 except EnvironmentError:
                     missing_workers.add(worker)
+                except ValueError as e:
+                    logger.info(
+                        "Got an unexpected error while collecting from workers: %s", e
+                    )
+                    missing_workers.add(worker)
                 else:
                     response.update(r["data"])
         finally:
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index 13639f05fa5..505d269cae9 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -1447,21 +1447,24 @@ def check_thread_leak():
 
 
 @contextmanager
-def check_process_leak():
+def check_process_leak(check=True):
     for proc in mp_context.active_children():
         proc.terminate()
 
     yield
 
-    for i in range(100):
-        if not set(mp_context.active_children()):
-            break
+    if check:
+        for i in range(100):
+            if not set(mp_context.active_children()):
+                break
+            else:
+                sleep(0.2)
         else:
-            sleep(0.2)
-    else:
-        assert not mp_context.active_children()
+            assert not mp_context.active_children()
 
     _cleanup_dangling()
+    for proc in mp_context.active_children():
+        proc.terminate()
 
 
 @contextmanager
@@ -1524,7 +1527,7 @@ def null():
 
     with check_thread_leak() if threads else null():
         with pristine_loop() as loop:
-            with check_process_leak() if processes else null():
+            with check_process_leak(check=processes):
                 with check_instances() if instances else null():
                     with check_active_rpc(loop, timeout):
                         reset_config()

From be88537c4c6040e171c0644f507dae2a3b1e1ead Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 5 Aug 2019 14:54:37 -0700
Subject: [PATCH 0402/1550] Add real-time CPU utilization plot to dashboard
 (#2922)

This matches the styling of the nprocessing and memory use plots
---
 distributed/dashboard/scheduler.py | 50 ++++++++++++++++++++++++++++++
 1 file changed, 50 insertions(+)

diff --git a/distributed/dashboard/scheduler.py b/distributed/dashboard/scheduler.py
index 8396bbcb6ae..3332f4fc27b 100644
--- a/distributed/dashboard/scheduler.py
+++ b/distributed/dashboard/scheduler.py
@@ -18,6 +18,7 @@
     Range1d,
     Plot,
     Quad,
+    Span,
     value,
     LinearAxis,
     NumeralTickFormatter,
@@ -309,6 +310,8 @@ def __init__(self, scheduler, width=600, **kwargs):
                     "nbytes": [1, 2],
                     "nbytes-half": [0.5, 1],
                     "nbytes_text": ["1B", "2B"],
+                    "cpu": [1, 2],
+                    "cpu-half": [0.5, 1],
                     "worker": ["a", "b"],
                     "y": [1, 2],
                     "nbytes-color": ["blue", "blue"],
@@ -353,6 +356,32 @@ def __init__(self, scheduler, width=600, **kwargs):
             )
             rect.nonselection_glyph = None
 
+            cpu = figure(
+                title="CPU Utilization",
+                tools="",
+                id="bk-cpu-worker-plot",
+                width=int(width / 2),
+                name="cpu_hist",
+                **kwargs
+            )
+            rect = cpu.rect(
+                source=self.source,
+                x="cpu-half",
+                y="y",
+                width="cpu",
+                height=1,
+                color="blue",
+            )
+            rect.nonselection_glyph = None
+            hundred_span = Span(
+                location=100,
+                dimension="height",
+                line_color="gray",
+                line_dash="dashed",
+                line_width=3,
+            )
+            cpu.add_layout(hundred_span)
+
             nbytes.axis[0].ticker = BasicTicker(mantissas=[1, 256, 512], base=1024)
             nbytes.xaxis[0].formatter = NumeralTickFormatter(format="0.0 b")
             nbytes.xaxis.major_label_orientation = -math.pi / 12
@@ -382,10 +411,17 @@ def __init__(self, scheduler, width=600, **kwargs):
             hover.point_policy = "follow_mouse"
             nbytes.add_tools(hover)
 
+            hover = HoverTool()
+            hover.tooltips = "@worker : @cpu %"
+            hover.point_policy = "follow_mouse"
+            cpu.add_tools(hover)
+
             self.processing_figure = processing
             self.nbytes_figure = nbytes
+            self.cpu_figure = cpu
 
             processing.y_range = nbytes.y_range
+            cpu.y_range = nbytes.y_range
 
     @without_property_validation
     def update(self):
@@ -393,6 +429,9 @@ def update(self):
             workers = list(self.scheduler.workers.values())
 
             y = list(range(len(workers)))
+
+            cpu = [int(ws.metrics["cpu"]) for ws in workers]
+
             nprocessing = [len(ws.processing) for ws in workers]
             processing_color = []
             for ws in workers:
@@ -427,6 +466,8 @@ def update(self):
             if any(nprocessing) or self.last + 1 < now:
                 self.last = now
                 result = {
+                    "cpu": cpu,
+                    "cpu-half": [c / 2 for c in cpu],
                     "nprocessing": nprocessing,
                     "nprocessing-half": [np / 2 for np in nprocessing],
                     "nprocessing-color": processing_color,
@@ -1495,6 +1536,14 @@ def individual_nbytes_doc(scheduler, extra, doc):
     doc.theme = BOKEH_THEME
 
 
+def individual_cpu_doc(scheduler, extra, doc):
+    current_load = CurrentLoad(scheduler, sizing_mode="stretch_both")
+    current_load.update()
+    add_periodic_callback(doc, current_load, 100)
+    doc.add_root(current_load.cpu_figure)
+    doc.theme = BOKEH_THEME
+
+
 def individual_nprocessing_doc(scheduler, extra, doc):
     current_load = CurrentLoad(scheduler, sizing_mode="stretch_both")
     current_load.update()
@@ -1619,6 +1668,7 @@ def __init__(self, scheduler, io_loop=None, prefix="", **kwargs):
             "/individual-profile": individual_profile_doc,
             "/individual-profile-server": individual_profile_server_doc,
             "/individual-nbytes": individual_nbytes_doc,
+            "/individual-cpu": individual_cpu_doc,
             "/individual-nprocessing": individual_nprocessing_doc,
             "/individual-workers": individual_workers_doc,
         }

From 17889a976df6c9891de5ffcca4715b6b8adfb76b Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 5 Aug 2019 17:17:32 -0700
Subject: [PATCH 0403/1550] Add aenter/aexit protocols to ProcessInterface
 (#2927)

---
 distributed/deploy/spec.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index af4f4a3f23d..c2de8b9e2f8 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -72,6 +72,13 @@ async def close(self):
     def __repr__(self):
         return "<%s: status=%s>" % (type(self).__name__, self.status)
 
+    async def __aenter__(self):
+        await self
+        return self
+
+    async def __aexit__(self, *args, **kwargs):
+        await self.close()
+
 
 class SpecCluster(Cluster):
     """ Cluster that requires a full specification of workers

From f6c8818a39d8163de8152b8cd7bef8f034404d8d Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 6 Aug 2019 12:26:09 -0700
Subject: [PATCH 0404/1550] Move core functionality from SpecCluster to Cluster
 (#2913)

This moves standard functionality from SpecClsuter to the Cluster
superclass.  It also removes the assumption that the Scheduler will be
local to the Cluster class.
---
 distributed/deploy/cluster.py | 316 ++++++++++++++++++++++------------
 distributed/deploy/spec.py    | 227 +-----------------------
 2 files changed, 208 insertions(+), 335 deletions(-)

diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
index 58c6ce73644..e85ea2bc3dd 100644
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -1,7 +1,7 @@
+import asyncio
 from datetime import timedelta
 import logging
 import threading
-from weakref import ref
 
 from dask.utils import format_bytes
 from tornado import gen
@@ -13,6 +13,8 @@
     log_errors,
     ignoring,
     sync,
+    Log,
+    Logs,
     thread_state,
     format_dashboard_link,
 )
@@ -24,45 +26,88 @@
 class Cluster(object):
     """ Superclass for cluster objects
 
-    This expects a local Scheduler defined on the object.  It provides
-    common methods and an IPython widget display.
+    This class contains common functionality for Dask Cluster manager classes.
 
-    Clusters inheriting from this class should provide the following:
+    To implement this class, you must provide
 
-    1.  A local ``Scheduler`` object at ``.scheduler``
-    2.  scale_up and scale_down methods as defined below::
+    1.  A ``scheduler_comm`` attribute, which is a connection to the scheduler
+        following the ``distributed.core.rpc`` API.
+    2.  Implement ``scale``, which takes an integer and scales the cluster to
+        that many workers, or else set ``_supports_scaling`` to False
 
-        def scale_up(self, n: int):
-            ''' Brings total worker count up to ``n`` '''
+    For that, should should get the following:
 
-        def scale_down(self, workers: List[str]):
-            ''' Close the workers with the given addresses '''
-
-    This will provide a general ``scale`` method as well as an IPython widget
-    for display.
+    1.  A standard ``__repr__``
+    2.  A live IPython widget
+    3.  Adaptive scaling
+    4.  Integration with dask-labextension
+    5.  A ``scheduler_info`` attribute which contains an up-to-date copy of
+        ``Scheduler.identity()``, which is used for much of the above
+    6.  Methods to gather logs
+    """
 
-    Examples
-    --------
+    _supports_scaling = True
 
-    >>> from distributed.deploy import Cluster
-    >>> class MyCluster(cluster):
-    ...     def scale_up(self, n):
-    ...         ''' Bring the total worker count up to n '''
-    ...         pass
-    ...     def scale_down(self, workers):
-    ...         ''' Close the workers with the given addresses '''
-    ...         pass
+    def __init__(self, asynchronous):
+        self.scheduler_info = {}
+        self.periodic_callbacks = {}
+        self._asynchronous = asynchronous
 
-    >>> cluster = MyCluster()
-    >>> cluster.scale(5)                       # scale manually
-    >>> cluster.adapt(minimum=1, maximum=100)  # scale automatically
+        self.status = "created"
 
-    See Also
-    --------
-    LocalCluster: a simple implementation with local workers
-    """
-
-    def adapt(self, Adaptive=Adaptive, **kwargs):
+    async def _start(self):
+        comm = await self.scheduler_comm.live_comm()
+        await comm.write({"op": "subscribe_worker_status"})
+        self.scheduler_info = await comm.read()
+        self._watch_worker_status_comm = comm
+        self._watch_worker_status_task = asyncio.ensure_future(
+            self._watch_worker_status(comm)
+        )
+        self.status = "running"
+
+    async def _close(self):
+        if self.status == "closed":
+            return
+
+        await self._watch_worker_status_comm.close()
+        await self._watch_worker_status_task
+
+        for pc in self.periodic_callbacks.values():
+            pc.stop()
+        self.scheduler_comm.close_rpc()
+
+        self.status = "closed"
+
+    def close(self, timeout=None):
+        with ignoring(RuntimeError):  # loop closed during process shutdown
+            return self.sync(self._close, callback_timeout=timeout)
+
+    def __del__(self):
+        if self.status != "closed":
+            with ignoring(AttributeError, RuntimeError):  # during closing
+                self.loop.add_callback(self.close)
+
+    async def _watch_worker_status(self, comm):
+        """ Listen to scheduler for updates on adding and removing workers """
+        while True:
+            try:
+                msgs = await comm.read()
+            except OSError:
+                break
+
+            for op, msg in msgs:
+                if op == "add":
+                    workers = msg.pop("workers")
+                    self.scheduler_info["workers"].update(workers)
+                    self.scheduler_info.update(msg)
+                elif op == "remove":
+                    del self.scheduler_info["workers"][msg]
+                else:
+                    raise ValueError("Invalid op", op, msg)
+
+        await comm.close()
+
+    def adapt(self, Adaptive=Adaptive, **kwargs) -> Adaptive:
         """ Turn on adaptivity
 
         For keyword arguments see dask.distributed.Adaptive
@@ -79,17 +124,7 @@ def adapt(self, Adaptive=Adaptive, **kwargs):
         self._adaptive = Adaptive(self, **self._adaptive_options)
         return self._adaptive
 
-    @property
-    def scheduler_address(self):
-        return self.scheduler.address
-
-    @property
-    def dashboard_link(self):
-        host = self.scheduler.address.split("://")[1].split(":")[0]
-        port = self.scheduler.services["dashboard"].port
-        return format_dashboard_link(host, port)
-
-    def scale(self, n):
+    def scale(self, n: int) -> None:
         """ Scale cluster to n workers
 
         Parameters
@@ -100,29 +135,81 @@ def scale(self, n):
         Example
         -------
         >>> cluster.scale(10)  # scale cluster to ten workers
+        """
+        raise NotImplementedError()
 
-        See Also
-        --------
-        Cluster.scale_up
-        Cluster.scale_down
+    @property
+    def asynchronous(self):
+        return (
+            self._asynchronous
+            or getattr(thread_state, "asynchronous", False)
+            or hasattr(self.loop, "_thread_identity")
+            and self.loop._thread_identity == threading.get_ident()
+        )
+
+    def sync(self, func, *args, asynchronous=None, callback_timeout=None, **kwargs):
+        asynchronous = asynchronous or self.asynchronous
+        if asynchronous:
+            future = func(*args, **kwargs)
+            if callback_timeout is not None:
+                future = gen.with_timeout(timedelta(seconds=callback_timeout), future)
+            return future
+        else:
+            return sync(self.loop, func, *args, **kwargs)
+
+    async def _logs(self, scheduler=True, workers=True):
+        logs = Logs()
+
+        if scheduler:
+            L = await self.scheduler_comm.logs()
+            logs["Scheduler"] = Log("\n".join(line for level, line in L))
+
+        if workers:
+            d = await self.scheduler_comm.worker_logs(workers=workers)
+            for k, v in d.items():
+                logs[k] = Log("\n".join(line for level, line in v))
+
+        return logs
+
+    def logs(self, scheduler=True, workers=True):
+        """ Return logs for the scheduler and workers
+
+        Parameters
+        ----------
+        scheduler : boolean
+            Whether or not to collect logs for the scheduler
+        workers : boolean or Iterable[str], optional
+            A list of worker addresses to select.
+            Defaults to all workers if `True` or no workers if `False`
+
+        Returns
+        -------
+        logs: Dict[str]
+            A dictionary of logs, with one item for the scheduler and one for
+            each worker
         """
-        with log_errors():
-            if n >= len(self.scheduler.workers):
-                self.scheduler.loop.add_callback(self.scale_up, n)
-            else:
-                to_close = self.scheduler.workers_to_close(
-                    n=len(self.scheduler.workers) - n
-                )
-                logger.debug("Closing workers: %s", to_close)
-                self.scheduler.loop.add_callback(
-                    self.scheduler.retire_workers, workers=to_close
-                )
-                self.scheduler.loop.add_callback(self.scale_down, to_close)
+        return self.sync(self._logs, scheduler=scheduler, workers=workers)
+
+    @property
+    def dashboard_link(self):
+        try:
+            port = self.scheduler_info["services"]["dashboard"]
+        except KeyError:
+            return ""
+        else:
+            host = self.scheduler_address.split("://")[1].split(":")[0]
+            return format_dashboard_link(host, port)
 
     def _widget_status(self):
-        workers = len(self.scheduler.workers)
-        cores = sum(ws.nthreads for ws in self.scheduler.workers.values())
-        memory = sum(ws.memory_limit for ws in self.scheduler.workers.values())
+        workers = len(self.scheduler_info["workers"])
+        if hasattr(self, "worker_spec"):
+            requested = len(self.worker_spec)
+        elif hasattr(self, "workers"):
+            requested = len(self.workers)
+        else:
+            requested = workers
+        cores = sum(v["nthreads"] for v in self.scheduler_info["workers"].values())
+        memory = sum(v["memory_limit"] for v in self.scheduler_info["workers"].values())
         memory = format_bytes(memory)
         text = """
 <div>
@@ -140,13 +227,13 @@ def _widget_status(self):
     }
   </style>
   <table style="text-align: right;">
-    <tr><th>Workers</th> <td>%d</td></tr>
-    <tr><th>Cores</th> <td>%d</td></tr>
-    <tr><th>Memory</th> <td>%s</td></tr>
+    <tr> <th>Workers</th> <td>%s</td></tr>
+    <tr> <th>Cores</th> <td>%d</td></tr>
+    <tr> <th>Memory</th> <td>%s</td></tr>
   </table>
 </div>
 """ % (
-            workers,
+            workers if workers == requested else "%d / %d" % (workers, requested),
             cores,
             memory,
         )
@@ -163,11 +250,10 @@ def _widget(self):
 
         layout = Layout(width="150px")
 
-        if "dashboard" in self.scheduler.services:
-            link = self.dashboard_link
+        if self.dashboard_link:
             link = '<p><b>Dashboard: </b><a href="%s" target="_blank">%s</a></p>\n' % (
-                link,
-                link,
+                self.dashboard_link,
+                self.dashboard_link,
             )
         else:
             link = ""
@@ -178,46 +264,49 @@ def _widget(self):
 
         status = HTML(self._widget_status(), layout=Layout(min_width="150px"))
 
-        request = IntText(0, description="Workers", layout=layout)
-        scale = Button(description="Scale", layout=layout)
+        if self._supports_scaling:
+            request = IntText(0, description="Workers", layout=layout)
+            scale = Button(description="Scale", layout=layout)
 
-        minimum = IntText(0, description="Minimum", layout=layout)
-        maximum = IntText(0, description="Maximum", layout=layout)
-        adapt = Button(description="Adapt", layout=layout)
+            minimum = IntText(0, description="Minimum", layout=layout)
+            maximum = IntText(0, description="Maximum", layout=layout)
+            adapt = Button(description="Adapt", layout=layout)
 
-        accordion = Accordion(
-            [HBox([request, scale]), HBox([minimum, maximum, adapt])],
-            layout=Layout(min_width="500px"),
-        )
-        accordion.selected_index = None
-        accordion.set_title(0, "Manual Scaling")
-        accordion.set_title(1, "Adaptive Scaling")
+            accordion = Accordion(
+                [HBox([request, scale]), HBox([minimum, maximum, adapt])],
+                layout=Layout(min_width="500px"),
+            )
+            accordion.selected_index = None
+            accordion.set_title(0, "Manual Scaling")
+            accordion.set_title(1, "Adaptive Scaling")
 
-        box = VBox([title, HBox([status, accordion]), dashboard])
+            def adapt_cb(b):
+                self.adapt(minimum=minimum.value, maximum=maximum.value)
+                update()
 
-        self._cached_widget = box
+            adapt.on_click(adapt_cb)
 
-        def adapt_cb(b):
-            self.adapt(minimum=minimum.value, maximum=maximum.value)
+            def scale_cb(b):
+                with log_errors():
+                    n = request.value
+                    with ignoring(AttributeError):
+                        self._adaptive.stop()
+                    self.scale(n)
+                    update()
 
-        adapt.on_click(adapt_cb)
-
-        def scale_cb(b):
-            with log_errors():
-                n = request.value
-                with ignoring(AttributeError):
-                    self._adaptive.stop()
-                self.scale(n)
+            scale.on_click(scale_cb)
+        else:
+            accordion = HTML("")
 
-        scale.on_click(scale_cb)
+        box = VBox([title, HBox([status, accordion]), dashboard])
 
-        scheduler_ref = ref(self.scheduler)
+        self._cached_widget = box
 
         def update():
             status.value = self._widget_status()
 
-        pc = PeriodicCallback(update, 500, io_loop=self.scheduler.loop)
-        self.scheduler.periodic_callbacks["cluster-repr"] = pc
+        pc = PeriodicCallback(update, 500, io_loop=self.loop)
+        self.periodic_callbacks["cluster-repr"] = pc
         pc.start()
 
         return box
@@ -225,21 +314,20 @@ def update():
     def _ipython_display_(self, **kwargs):
         return self._widget()._ipython_display_(**kwargs)
 
-    @property
-    def asynchronous(self):
-        return (
-            self._asynchronous
-            or getattr(thread_state, "asynchronous", False)
-            or hasattr(self.loop, "_thread_identity")
-            and self.loop._thread_identity == threading.get_ident()
+    def __repr__(self):
+        return "%s(%r, workers=%d)" % (
+            type(self).__name__,
+            self.scheduler_address,
+            len(self.scheduler_info["workers"]),
         )
 
-    def sync(self, func, *args, asynchronous=None, callback_timeout=None, **kwargs):
-        asynchronous = asynchronous or self.asynchronous
-        if asynchronous:
-            future = func(*args, **kwargs)
-            if callback_timeout is not None:
-                future = gen.with_timeout(timedelta(seconds=callback_timeout), future)
-            return future
-        else:
-            return sync(self.loop, func, *args, **kwargs)
+    async def __aenter__(self):
+        await self
+        return self
+
+    async def __aexit__(self, typ, value, traceback):
+        await self.close()
+
+    @property
+    def scheduler_address(self):
+        return self.scheduler_comm.address
diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index c2de8b9e2f8..70a413fe1c0 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -4,21 +4,10 @@
 import weakref
 
 from tornado import gen
-from dask.utils import format_bytes
 
 from .cluster import Cluster
-from ..comm import connect
 from ..core import rpc, CommClosedError
-from ..utils import (
-    log_errors,
-    LoopRunner,
-    silence_logging,
-    ignoring,
-    Log,
-    Logs,
-    PeriodicCallback,
-    format_dashboard_link,
-)
+from ..utils import LoopRunner, silence_logging, ignoring
 from ..scheduler import Scheduler
 from ..security import Security
 
@@ -191,11 +180,8 @@ def __init__(
         self.new_spec = copy.copy(worker)
         self.workers = {}
         self._i = 0
-        self._asynchronous = asynchronous
         self.security = security or Security()
         self.scheduler_comm = None
-        self.scheduler_info = {}
-        self.periodic_callbacks = {}
 
         if silence_logs:
             self._old_logging_level = silence_logging(level=silence_logs)
@@ -203,11 +189,12 @@ def __init__(
         self._loop_runner = LoopRunner(loop=loop, asynchronous=asynchronous)
         self.loop = self._loop_runner.loop
 
-        self.status = "created"
         self._instances.add(self)
         self._correct_state_waiting = None
         self._name = name or type(self).__name__
 
+        super().__init__(asynchronous=asynchronous)
+
         if not self.asynchronous:
             self._loop_runner.start()
             self.sync(self._start)
@@ -241,37 +228,7 @@ async def _start(self):
             self.scheduler.address,
             connection_args=self.security.get_connection_args("client"),
         )
-        comm = await connect(
-            self.scheduler_address,
-            connection_args=self.security.get_connection_args("client"),
-        )
-        await comm.write({"op": "subscribe_worker_status"})
-        self.scheduler_info = await comm.read()
-        self._watch_worker_status_comm = comm
-        self._watch_worker_status_task = asyncio.ensure_future(
-            self._watch_worker_status(comm)
-        )
-        self.status = "running"
-
-    async def _watch_worker_status(self, comm):
-        """ Listen to scheduler for updates on adding and removing workers """
-        while True:
-            try:
-                msgs = await comm.read()
-            except OSError:
-                break
-
-            for op, msg in msgs:
-                if op == "add":
-                    workers = msg.pop("workers")
-                    self.scheduler_info["workers"].update(workers)
-                    self.scheduler_info.update(msg)
-                elif op == "remove":
-                    del self.scheduler_info["workers"][msg]
-                else:
-                    raise ValueError("Invalid op", op, msg)
-
-        await comm.close()
+        await super()._start()
 
     def _correct_state(self):
         if self._correct_state_waiting:
@@ -341,13 +298,6 @@ async def _wait_for_workers(self):
                 raise gen.TimeoutError("Worker unexpectedly closed")
             await asyncio.sleep(0.1)
 
-    async def __aenter__(self):
-        await self
-        return self
-
-    async def __aexit__(self, typ, value, traceback):
-        await self.close()
-
     async def _close(self):
         while self.status == "closing":
             await asyncio.sleep(0.1)
@@ -355,33 +305,20 @@ async def _close(self):
             return
         self.status = "closing"
 
-        for pc in self.periodic_callbacks.values():
-            pc.stop()
-
         self.scale(0)
         await self._correct_state()
         async with self._lock:
             with ignoring(CommClosedError):
                 await self.scheduler_comm.close(close_workers=True)
+
         await self.scheduler.close()
-        await self._watch_worker_status_comm.close()
-        await self._watch_worker_status_task
         for w in self._created:
             assert w.status == "closed"
-        self.scheduler_comm.close_rpc()
 
         if hasattr(self, "_old_logging_level"):
             silence_logging(self._old_logging_level)
 
-        self.status = "closed"
-
-    def close(self, timeout=None):
-        with ignoring(RuntimeError):  # loop closed during process shutdown
-            return self.sync(self._close, callback_timeout=timeout)
-
-    def __del__(self):
-        if self.status != "closed":
-            self.loop.add_callback(self.close)
+        await super()._close()
 
     def __enter__(self):
         self.sync(self._correct_state)
@@ -449,158 +386,6 @@ def __repr__(self):
             len(self.workers),
         )
 
-    async def _logs(self, scheduler=True, workers=True):
-        logs = Logs()
-
-        if scheduler:
-            L = await self.scheduler_comm.logs()
-            logs["Scheduler"] = Log("\n".join(line for level, line in L))
-
-        if workers:
-            d = await self.scheduler_comm.worker_logs(workers=workers)
-            for k, v in d.items():
-                logs[k] = Log("\n".join(line for level, line in v))
-
-        return logs
-
-    def logs(self, scheduler=True, workers=True):
-        """ Return logs for the scheduler and workers
-
-        Parameters
-        ----------
-        scheduler : boolean
-            Whether or not to collect logs for the scheduler
-        workers : boolean or Iterable[str], optional
-            A list of worker addresses to select.
-            Defaults to all workers if `True` or no workers if `False`
-
-        Returns
-        -------
-        logs: Dict[str]
-            A dictionary of logs, with one item for the scheduler and one for
-            each worker
-        """
-        return self.sync(self._logs, scheduler=scheduler, workers=workers)
-
-    @property
-    def dashboard_link(self):
-        try:
-            port = self.scheduler_info["services"]["dashboard"]
-        except KeyError:
-            return ""
-        else:
-            host = self.scheduler_address.split("://")[1].split(":")[0]
-            return format_dashboard_link(host, port)
-
-    def _widget_status(self):
-        workers = len(self.scheduler_info["workers"])
-        requested = len(self.worker_spec)
-        cores = sum(v["nthreads"] for v in self.scheduler_info["workers"].values())
-        memory = sum(v["memory_limit"] for v in self.scheduler_info["workers"].values())
-        memory = format_bytes(memory)
-        text = """
-<div>
-  <style scoped>
-    .dataframe tbody tr th:only-of-type {
-        vertical-align: middle;
-    }
-
-    .dataframe tbody tr th {
-        vertical-align: top;
-    }
-
-    .dataframe thead th {
-        text-align: right;
-    }
-  </style>
-  <table style="text-align: right;">
-    <tr> <th>Workers</th> <td>%s</td></tr>
-    <tr> <th>Cores</th> <td>%d</td></tr>
-    <tr> <th>Memory</th> <td>%s</td></tr>
-  </table>
-</div>
-""" % (
-            workers if workers == requested else "%d / %d" % (workers, requested),
-            cores,
-            memory,
-        )
-        return text
-
-    def _widget(self):
-        """ Create IPython widget for display within a notebook """
-        try:
-            return self._cached_widget
-        except AttributeError:
-            pass
-
-        from ipywidgets import Layout, VBox, HBox, IntText, Button, HTML, Accordion
-
-        layout = Layout(width="150px")
-
-        if self.dashboard_link:
-            link = '<p><b>Dashboard: </b><a href="%s" target="_blank">%s</a></p>\n' % (
-                self.dashboard_link,
-                self.dashboard_link,
-            )
-        else:
-            link = ""
-
-        title = "<h2>%s</h2>" % type(self).__name__
-        title = HTML(title)
-        dashboard = HTML(link)
-
-        status = HTML(self._widget_status(), layout=Layout(min_width="150px"))
-
-        if self._supports_scaling:
-            request = IntText(0, description="Workers", layout=layout)
-            scale = Button(description="Scale", layout=layout)
-
-            minimum = IntText(0, description="Minimum", layout=layout)
-            maximum = IntText(0, description="Maximum", layout=layout)
-            adapt = Button(description="Adapt", layout=layout)
-
-            accordion = Accordion(
-                [HBox([request, scale]), HBox([minimum, maximum, adapt])],
-                layout=Layout(min_width="500px"),
-            )
-            accordion.selected_index = None
-            accordion.set_title(0, "Manual Scaling")
-            accordion.set_title(1, "Adaptive Scaling")
-
-            def adapt_cb(b):
-                self.adapt(minimum=minimum.value, maximum=maximum.value)
-                update()
-
-            adapt.on_click(adapt_cb)
-
-            def scale_cb(b):
-                with log_errors():
-                    n = request.value
-                    with ignoring(AttributeError):
-                        self._adaptive.stop()
-                    self.scale(n)
-                    update()
-
-            scale.on_click(scale_cb)
-        else:
-            accordion = HTML("")
-
-        box = VBox([title, HBox([status, accordion]), dashboard])
-
-        self._cached_widget = box
-
-        def update():
-            status.value = self._widget_status()
-
-        pc = PeriodicCallback(update, 500, io_loop=self.loop)
-        self.periodic_callbacks["cluster-repr"] = pc
-        pc.start()
-
-        return box
-
-    def _ipython_display_(self, **kwargs):
-        return self._widget()._ipython_display_(**kwargs)
-
 
 @atexit.register
 def close_clusters():

From cf10db7b6a4fd091c2e1385162e3d36ab59c8f6e Mon Sep 17 00:00:00 2001
From: Jim Crist <jcrist@users.noreply.github.com>
Date: Tue, 6 Aug 2019 15:02:16 -0500
Subject: [PATCH 0405/1550] Fixup black string normalization (#2929)

After running black, several places in our codebase were rewritten from something like

```
raise ValueError("part one of the message "
                 "part two")
```

to

```
raise ValueError("part one of the message " "part two")
```

This fixes those cases, removing the unnecessary two-part string.
---
 distributed/cli/dask_scheduler.py  |  2 +-
 distributed/cli/dask_worker.py     |  2 +-
 distributed/client.py              | 10 ++++------
 distributed/comm/addressing.py     |  2 +-
 distributed/comm/tests/test_ucx.py |  2 +-
 distributed/diskutils.py           |  2 +-
 distributed/protocol/keras.py      |  2 +-
 distributed/scheduler.py           |  6 ++----
 distributed/tests/test_worker.py   |  2 +-
 distributed/utils_perf.py          |  2 +-
 distributed/worker.py              |  2 +-
 11 files changed, 15 insertions(+), 19 deletions(-)

diff --git a/distributed/cli/dask_scheduler.py b/distributed/cli/dask_scheduler.py
index a74f76102b9..29de26d7b4d 100755
--- a/distributed/cli/dask_scheduler.py
+++ b/distributed/cli/dask_scheduler.py
@@ -215,7 +215,7 @@ def del_pid_file():
         port=port,
         dashboard_address=dashboard_address if dashboard else None,
         service_kwargs={"dashboard": {"prefix": dashboard_prefix}},
-        **kwargs,
+        **kwargs
     )
     logger.info("Local Directory: %26s", local_directory)
     logger.info("-" * 47)
diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
index eef1d648d40..790b8b3a9ab 100755
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -84,7 +84,7 @@
     "--listen-address",
     type=str,
     default=None,
-    help="The address to which the worker binds. " "Example: tcp://0.0.0.0:9000",
+    help="The address to which the worker binds. Example: tcp://0.0.0.0:9000",
 )
 @click.option(
     "--contact-address",
diff --git a/distributed/client.py b/distributed/client.py
index 93501ae2077..9f59582b019 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -905,7 +905,7 @@ async def _start(self, timeout=no_default, **kwargs):
                 pass
             except Exception:
                 logger.info(
-                    "Tried to start cluster and received an error. " "Proceeding.",
+                    "Tried to start cluster and received an error. Proceeding.",
                     exc_info=True,
                 )
             address = self.cluster.scheduler_address
@@ -2383,9 +2383,7 @@ def _graph_to_futures(
             dsk3 = {k: v for k, v in dsk2.items() if k is not v}
             for future in extra_futures:
                 if future.client is not self:
-                    msg = (
-                        "Inputs contain futures that were created by " "another client."
-                    )
+                    msg = "Inputs contain futures that were created by another client."
                     raise ValueError(msg)
 
             if restrictions:
@@ -3485,7 +3483,7 @@ def to_packages(d):
                     errs.append("%s\n%s" % (pkg, asciitable(["", "version"], rows)))
 
                 raise ValueError(
-                    "Mismatched versions found\n" "\n" "%s" % ("\n\n".join(errs))
+                    "Mismatched versions found\n\n%s" % ("\n\n".join(errs))
                 )
 
         return result
@@ -3967,7 +3965,7 @@ async def _wait(fs, timeout=None, return_when=ALL_COMPLETED):
         wait_for = Any
     else:
         raise NotImplementedError(
-            "Only return_when='ALL_COMPLETED' and 'FIRST_COMPLETED' are " "supported"
+            "Only return_when='ALL_COMPLETED' and 'FIRST_COMPLETED' are supported"
         )
 
     future = wait_for({f._state.wait() for f in fs})
diff --git a/distributed/comm/addressing.py b/distributed/comm/addressing.py
index 8480134997c..21a23e1ef6e 100644
--- a/distributed/comm/addressing.py
+++ b/distributed/comm/addressing.py
@@ -123,7 +123,7 @@ def get_address_host_port(addr, strict=False):
         return backend.get_address_host_port(loc)
     except NotImplementedError:
         raise ValueError(
-            "don't know how to extract host and port " "for address %r" % (addr,)
+            "don't know how to extract host and port for address %r" % (addr,)
         )
 
 
diff --git a/distributed/comm/tests/test_ucx.py b/distributed/comm/tests/test_ucx.py
index 8a0e8927cf6..4bb4a341552 100644
--- a/distributed/comm/tests/test_ucx.py
+++ b/distributed/comm/tests/test_ucx.py
@@ -261,7 +261,7 @@ def test_ucx_localcluster(loop, processes):
         threads_per_worker=1,
         processes=processes,
         loop=loop,
-        **kwargs,
+        **kwargs
     ) as cluster:
         with Client(cluster) as client:
             x = client.submit(inc, 1)
diff --git a/distributed/diskutils.py b/distributed/diskutils.py
index 64dcf1dfc12..075ec7750c8 100644
--- a/distributed/diskutils.py
+++ b/distributed/diskutils.py
@@ -237,7 +237,7 @@ def new_work_dir(self, **kwargs):
             self._purge_leftovers()
         except OSError:
             logger.error(
-                "Failed to clean up lingering worker directories " "in path: %s ",
+                "Failed to clean up lingering worker directories in path: %s ",
                 exc_info=True,
             )
         return WorkDir(self, **kwargs)
diff --git a/distributed/protocol/keras.py b/distributed/protocol/keras.py
index 7471a3dbc93..020ce1cae3b 100644
--- a/distributed/protocol/keras.py
+++ b/distributed/protocol/keras.py
@@ -9,7 +9,7 @@ def serialize_keras_model(model):
 
     if keras.__version__ < "1.2.0":
         raise ImportError(
-            "Need Keras >= 1.2.0. " "Try pip install keras --upgrade --no-deps"
+            "Need Keras >= 1.2.0. Try pip install keras --upgrade --no-deps"
         )
 
     header = model._updated_config()
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index f84b3d1bce4..65e93d3e59c 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -2366,9 +2366,7 @@ def handle_long_running(self, key=None, worker=None, compute_duration=None):
 
         ws = ts.processing_on
         if ws is None:
-            logger.debug(
-                "Received long-running signal from duplicate task. " "Ignoring."
-            )
+            logger.debug("Received long-running signal from duplicate task. Ignoring.")
             return
 
         if compute_duration:
@@ -4730,7 +4728,7 @@ def check_worker_ttl(self):
         for ws in self.workers.values():
             if ws.last_seen < now - self.worker_ttl:
                 logger.warning(
-                    "Worker failed to heartbeat within %s seconds. " "Closing: %s",
+                    "Worker failed to heartbeat within %s seconds. Closing: %s",
                     self.worker_ttl,
                     ws,
                 )
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 52e92d474ce..64e774f582a 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -464,7 +464,7 @@ def test_Executor(c, s):
 
 
 @pytest.mark.skip(
-    reason="Other tests leak memory, so process-level checks" "trigger immediately"
+    reason="Other tests leak memory, so process-level checks trigger immediately"
 )
 @gen_cluster(
     client=True,
diff --git a/distributed/utils_perf.py b/distributed/utils_perf.py
index 048d9092d49..c2257f38fb0 100644
--- a/distributed/utils_perf.py
+++ b/distributed/utils_perf.py
@@ -42,7 +42,7 @@ def collect(self):
         elapsed = max(collect_start - self.last_collect, MIN_RUNTIME)
         if self.last_gc_duration / elapsed < self.max_in_gc_frac:
             self.logger.debug(
-                "Calling gc.collect(). %0.3fs elapsed since " "previous call.", elapsed
+                "Calling gc.collect(). %0.3fs elapsed since previous call.", elapsed
             )
             gc.collect()
             self.last_collect = collect_start
diff --git a/distributed/worker.py b/distributed/worker.py
index 795cb93e9f3..63d3ed92a21 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -3354,7 +3354,7 @@ async def run(server, comm, function, args=(), kwargs={}, is_coro=None, wait=Tru
 
     except Exception as e:
         logger.warning(
-            " Run Failed\n" "Function: %s\n" "args:     %s\n" "kwargs:   %s\n",
+            "Run Failed\nFunction: %s\nargs:     %s\nkwargs:   %s\n",
             str(funcname(function))[:1000],
             convert_args_to_str(args, max_len=1000),
             convert_kwargs_to_str(kwargs, max_len=1000),

From b1ba71a28aae83e4b9668d471b87a4aee762e2bf Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 6 Aug 2019 14:52:27 -0700
Subject: [PATCH 0406/1550] Change TCP.close to a coroutine to avoid task
 pending warning (#2930)

Previously this triggered an intermittent error
---
 distributed/comm/tcp.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py
index 36783102b69..bd76d0e6946 100644
--- a/distributed/comm/tcp.py
+++ b/distributed/comm/tcp.py
@@ -268,13 +268,17 @@ def write(self, msg, serializers=None, on_error="message"):
 
         return sum(map(nbytes, frames))
 
-    async def close(self):
+    @gen.coroutine
+    def close(self):
+        # We use gen.coroutine here rather than async def to avoid errors like
+        # Task was destroyed but it is pending!
+        # Triggered by distributed.deploy.tests.test_local::test_silent_startup
         stream, self.stream = self.stream, None
         if stream is not None and not stream.closed():
             try:
                 # Flush the stream's write buffer by waiting for a last write.
                 if stream.writing():
-                    await stream.write(b"")
+                    yield stream.write(b"")
                 stream.socket.shutdown(socket.SHUT_RDWR)
             except EnvironmentError:
                 pass

From 8d7e1664127e3967f12b8fbbd9348d118a7b4b8b Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 6 Aug 2019 17:45:29 -0700
Subject: [PATCH 0407/1550] Wrap offload in gen.coroutine (#2934)

Previously we would return the bare concurrent.future.Future
which was not awaitable.  Now we rely on Tornado's gen.coroutine
logic to handle this.

Fixes https://github.com/dask/distributed/issues/2928
---
 distributed/comm/utils.py | 16 +---------------
 distributed/utils.py      | 16 ++++++++++++++++
 distributed/utils_test.py |  4 ++--
 distributed/worker.py     |  2 +-
 4 files changed, 20 insertions(+), 18 deletions(-)

diff --git a/distributed/comm/utils.py b/distributed/comm/utils.py
index dcc9e9a8b1a..70cd2b4cd27 100644
--- a/distributed/comm/utils.py
+++ b/distributed/comm/utils.py
@@ -1,12 +1,10 @@
-from concurrent.futures import ThreadPoolExecutor
 import logging
 import socket
-import weakref
 
 from tornado import gen
 
 from .. import protocol
-from ..utils import get_ip, get_ipv6, nbytes
+from ..utils import get_ip, get_ipv6, nbytes, offload
 
 
 logger = logging.getLogger(__name__)
@@ -17,18 +15,6 @@
 
 FRAME_OFFLOAD_THRESHOLD = 10 * 1024 ** 2  # 10 MB
 
-try:
-    _offload_executor = ThreadPoolExecutor(
-        max_workers=1, thread_name_prefix="Dask-Offload"
-    )
-except TypeError:
-    _offload_executor = ThreadPoolExecutor(max_workers=1)
-weakref.finalize(_offload_executor, _offload_executor.shutdown)
-
-
-def offload(fn, *args, **kwargs):
-    return _offload_executor.submit(fn, *args, **kwargs)
-
 
 @gen.coroutine
 def to_frames(msg, serializers=None, on_error="message", context=None):
diff --git a/distributed/utils.py b/distributed/utils.py
index cdc5c4d1ae9..c8ea8d648eb 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -1,6 +1,7 @@
 import asyncio
 import atexit
 from collections import deque
+from concurrent.futures import ThreadPoolExecutor
 from contextlib import contextmanager
 from datetime import timedelta
 import functools
@@ -1471,3 +1472,18 @@ def convert_value(v):
 
 def is_valid_xml(text):
     return xml.etree.ElementTree.fromstring(text) is not None
+
+
+try:
+    _offload_executor = ThreadPoolExecutor(
+        max_workers=1, thread_name_prefix="Dask-Offload"
+    )
+except TypeError:
+    _offload_executor = ThreadPoolExecutor(max_workers=1)
+
+weakref.finalize(_offload_executor, _offload_executor.shutdown)
+
+
+@gen.coroutine
+def offload(fn, *args, **kwargs):
+    return (yield _offload_executor.submit(fn, *args, **kwargs))
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index 505d269cae9..e6b11ce2898 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -41,7 +41,6 @@
 from .client import default_client, _global_clients, Client
 from .compatibility import WINDOWS
 from .comm import Comm
-from .comm.utils import offload
 from .config import initialize_logging
 from .core import connect, rpc, CommClosedError
 from .deploy import SpecCluster
@@ -60,6 +59,7 @@
     sync,
     iscoroutinefunction,
     thread_state,
+    _offload_executor,
 )
 from .worker import Worker, TOTAL_MEMORY
 from .nanny import Nanny
@@ -80,7 +80,7 @@
 }
 
 
-offload(lambda: None).result()  # create thread during import
+_offload_executor.submit(lambda: None).result()  # create thread during import
 
 
 @pytest.fixture(scope="session")
diff --git a/distributed/worker.py b/distributed/worker.py
index 63d3ed92a21..9f686cfa508 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -30,7 +30,6 @@
 from . import profile, comm
 from .batched import BatchedSend
 from .comm import get_address_host, connect
-from .comm.utils import offload
 from .comm.addressing import address_from_user_args
 from .core import error_message, CommClosedError, send_recv, pingpong, coerce_to_address
 from .diskutils import WorkSpace
@@ -56,6 +55,7 @@
     thread_state,
     json_load_robust,
     key_split,
+    offload,
     PeriodicCallback,
     parse_bytes,
     parse_timedelta,

From bafc34fdc82949b404f10dcd4dcdbbb32d74c47d Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 7 Aug 2019 10:54:07 -0700
Subject: [PATCH 0408/1550] Redirect setup docs to docs.dask.org [skip ci]
 (#2936)

---
 docs/source/conf.py          |   3 +-
 docs/source/index.rst        |   4 +-
 docs/source/quickstart.rst   |   2 +-
 docs/source/related-work.rst |   2 +-
 docs/source/setup.rst        | 332 -----------------------------------
 5 files changed, 6 insertions(+), 337 deletions(-)
 delete mode 100644 docs/source/setup.rst

diff --git a/docs/source/conf.py b/docs/source/conf.py
index c8ffc0ae50d..afa33400fdc 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -389,7 +389,8 @@
 # https://tech.signavio.com/2017/managing-sphinx-redirects
 redirect_files = [
     # old html, new html
-    ("joblib.html", "https://ml.dask.org/joblib.html")
+    ("joblib.html", "https://ml.dask.org/joblib.html"),
+    ("setup.html", "https://docs.dask.org/en/latest/setup.html"),
 ]
 
 
diff --git a/docs/source/index.rst b/docs/source/index.rst
index cd27e9b4123..09257be3f58 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -28,7 +28,7 @@ In particular it meets the following needs:
     Python standard library.  Compatible with `dask`_ API for parallel
     algorithms
 *   **Easy Setup:** As a Pure Python package distributed is ``pip`` installable
-    and easy to :doc:`set up <setup>` on your own cluster.
+    and easy to :doc:`set up <https://docs.dask.org/en/latest/setup.html>`_ on your own cluster.
 
 .. _`concurrent.futures`: https://www.python.org/dev/peps/pep-3148/
 .. _`dask`: https://dask.org
@@ -77,7 +77,7 @@ Contents
 
    install
    quickstart
-   setup
+   Setup <https://docs.dask.org/en/latest/setup.html>
    client
    api
    faq
diff --git a/docs/source/quickstart.rst b/docs/source/quickstart.rst
index 3d1e326f528..4437f77a1ea 100644
--- a/docs/source/quickstart.rst
+++ b/docs/source/quickstart.rst
@@ -48,7 +48,7 @@ Launch a Client and point it to the IP/port of the scheduler.
    >>> from dask.distributed import Client
    >>> client = Client('127.0.0.1:8786')
 
-See :doc:`setup <setup>` for advanced use.
+See `setup documentation <https://docs.dask.org/en/latest/setup.html>`_ for advanced use.
 
 
 Map and Submit Functions
diff --git a/docs/source/related-work.rst b/docs/source/related-work.rst
index 4de458bad66..07b08c29787 100644
--- a/docs/source/related-work.rst
+++ b/docs/source/related-work.rst
@@ -185,7 +185,7 @@ IPython Parallel has the following advantages over ``distributed``
     serve in system administration tasks.
 3.  Deployment help:  IPython Parallel has mechanisms built-in to aid
     deployment on SGE, MPI, etc..  Distributed does not have any such sugar,
-    though is fairly simple to :doc:`set up <setup>` by hand.
+    though is fairly simple to `set up <https://docs.dask.org/en/latest/setup.html>`_ by hand.
 4.  Various other advantages:  Over the years IPython parallel has accrued a
     variety of helpful features like IPython interaction magics, ``@parallel``
     decorators, etc..
diff --git a/docs/source/setup.rst b/docs/source/setup.rst
deleted file mode 100644
index f1158712901..00000000000
--- a/docs/source/setup.rst
+++ /dev/null
@@ -1,332 +0,0 @@
-Setup Network
-=============
-
-A ``dask.distributed`` network consists of one ``Scheduler`` node and several
-``Worker`` nodes.  One can set these up in a variety of ways
-
-
-Using the Command Line
-----------------------
-
-We launch the ``dask-scheduler`` executable in one process and the
-``dask-worker`` executable in several processes, possibly on different
-machines.
-
-Launch ``dask-scheduler`` on one node::
-
-   $ dask-scheduler
-   Start scheduler at 192.168.0.1:8786
-
-Then launch ``dask-worker`` on the rest of the nodes, providing the address to the
-node that hosts ``dask-scheduler``::
-
-   $ dask-worker 192.168.0.1:8786
-   Start worker at:               192.168.0.2:12345
-   Registered with Scheduler at:  192.168.0.1:8786
-
-   $ dask-worker 192.168.0.1:8786
-   Start worker at:               192.168.0.3:12346
-   Registered with Scheduler at:  192.168.0.1:8786
-
-   $ dask-worker 192.168.0.1:8786
-   Start worker at:               192.168.0.4:12347
-   Registered with Scheduler at:  192.168.0.1:8786
-
-There are various mechanisms to deploy these executables on a cluster, ranging
-from manualy SSH-ing into all of the nodes to more automated systems like
-SGE/SLURM/Torque or Yarn/Mesos. Additionally, cluster SSH tools exist to
-send the same commands to many machines. One example is `tmux-cssh`__.
-
-.. note::
-
-  - The scheduler and worker both need to accept TCP connections.  By default
-    the scheduler uses port 8786 and the worker binds to a random open port.
-    If you are behind a firewall then you may have to open particular ports or
-    tell Dask to use particular ports with the ``--port`` and ``-worker-port``
-    keywords.    Other ports like 8787, 8788, and 8789 are also useful to keep
-    open for the diagnostic web interfaces.
-  - More information about relevant ports is available by looking at the help
-    pages with ``dask-scheduler --help`` and ``dask-worker --help``
-
-__ https://github.com/dennishafemann/tmux-cssh
-
-
-Using SSH
----------
-
-The convenience script ``dask-ssh`` opens several SSH connections to your
-target computers and initializes the network accordingly. You can
-give it a list of hostnames or IP addresses::
-
-   $ dask-ssh 192.168.0.1 192.168.0.2 192.168.0.3 192.168.0.4
-
-Or you can use normal UNIX grouping::
-
-   $ dask-ssh 192.168.0.{1,2,3,4}
-
-Or you can specify a hostfile that includes a list of hosts::
-
-   $ cat hostfile.txt
-   192.168.0.1
-   192.168.0.2
-   192.168.0.3
-   192.168.0.4
-
-   $ dask-ssh --hostfile hostfile.txt
-
-The ``dask-ssh`` utility depends on the ``paramiko``::
-
-    pip install paramiko
-
-
-Using a Shared Network File System and a Job Scheduler
-------------------------------------------------------
-
-Some clusters benefit from a shared network file system (NFS) and can use this
-to communicate the scheduler location to the workers::
-
-   dask-scheduler --scheduler-file /path/to/scheduler.json
-
-   dask-worker --scheduler-file /path/to/scheduler.json
-   dask-worker --scheduler-file /path/to/scheduler.json
-
-.. code-block:: python
-
-   >>> client = Client(scheduler_file='/path/to/scheduler.json')
-
-This can be particularly useful when deploying ``dask-scheduler`` and
-``dask-worker`` processes using a job scheduler like
-``SGE/SLURM/Torque/etc..``  Here is an example using SGE's ``qsub`` command::
-
-    # Start a dask-scheduler somewhere and write connection information to file
-    qsub -b y /path/to/dask-scheduler --scheduler-file /path/to/scheduler.json
-
-    # Start 100 dask-worker processes in an array job pointing to the same file
-    qsub -b y -t 1-100 /path/to/dask-worker --scheduler-file /path/to/scheduler.json
-
-Note, the ``--scheduler-file`` option is *only* valuable if your scheduler and
-workers share a standard POSIX file system.
-
-
-Using MPI
----------
-
-You can launch a Dask network using ``mpirun`` or ``mpiexec`` and the
-``dask-mpi`` command line executable.
-
-.. code-block:: bash
-
-   mpirun --np 4 dask-mpi --scheduler-file /path/to/scheduler.json
-
-.. code-block:: python
-
-   from dask.distributed import Client
-   client = Client(scheduler_file='/path/to/scheduler.json')
-
-This depends on the `mpi4py <http://mpi4py.readthedocs.io/>`_ library.  It only
-uses MPI to start the Dask cluster, and not for inter-node communication.  You
-may want to specify a high-bandwidth network interface like infiniband using
-the ``--interface`` keyword
-
-.. code-block:: bash
-
-   mpirun --np 4 dask-mpi --nthreads 1 \
-                          --interface ib0 \
-                          --scheduler-file /path/to/scheduler.json
-
-Using the Python API
---------------------
-
-Alternatively you can start up the ``distributed.scheduler.Scheduler`` and
-``distributed.worker.Worker`` objects within a Python session manually.
-
-Start the Scheduler, provide the listening port (defaults to 8786) and Tornado
-IOLoop (defaults to ``IOLoop.current()``)
-
-.. code-block:: python
-
-   from distributed import Scheduler
-   from tornado.ioloop import IOLoop
-   from threading import Thread
-
-   loop = IOLoop.current()
-   t = Thread(target=loop.start, daemon=True)
-   t.start()
-
-   s = Scheduler(loop=loop)
-   s.start('tcp://:8786')   # Listen on TCP port 8786
-
-On other nodes start worker processes that point to the URL of the scheduler.
-
-.. code-block:: python
-
-   from distributed import Worker
-   from tornado.ioloop import IOLoop
-   from threading import Thread
-
-   loop = IOLoop.current()
-   t = Thread(target=loop.start, daemon=True)
-   t.start()
-
-   w = Worker('tcp://127.0.0.1:8786', loop=loop)
-   w.start()  # choose randomly assigned port
-
-Alternatively, replace ``Worker`` with ``Nanny`` if you want your workers to be
-managed in a separate process by a local nanny process.  This allows workers to
-restart themselves in case of failure, provides some additional monitoring, and
-is useful when coordinating many workers that should live in different
-processes to avoid the GIL_.
-
-.. _GIL: https://docs.python.org/3/glossary.html#term-gil
-
-
-Using LocalCluster
-------------------
-
-You can do the work above easily using :doc:`LocalCluster<local-cluster>`.
-
-.. code-block:: python
-
-   from distributed import LocalCluster
-   c = LocalCluster(processes=False)
-
-A scheduler will be available under ``c.scheduler`` and a list of workers under
-``c.workers``.  There is an IOLoop running in a background thread.
-
-
-Using AWS
----------
-
-See `Cloud Deployments`_ for the latest information on deploying to Amazon
-cloud.
-
-.. _`Cloud Deployments`: https://docs.dask.org/en/latest/setup/cloud.html
-
-
-Using Google Cloud
-------------------
-
-See the dask-kubernetes_ project to easily launch clusters on `Google Kubernetes
-Engine`_.
-
-.. _dask-kubernetes: https://github.com/dask/dask-kubernetes
-.. _`Google Kubernetes Engine`: https://cloud.google.com/kubernetes-engine/
-
-Cluster Resource Managers
--------------------------
-
-Dask.distributed has been deployed on dozens of different cluster resource
-managers.  This section contains links to some external projects, scripts, and
-instructions that may serve as useful starting points.
-
-Kubernetes
-~~~~~~~~~~
-
-*  https://github.com/martindurant/dask-kubernetes
-*  https://github.com/ogrisel/docker-distributed
-*  https://github.com/hammerlab/dask-distributed-on-kubernetes/
-
-Marathon
-~~~~~~~~
-
-*  https://github.com/mrocklin/dask-marathon
-
-DRMAA (SGE, SLURM, Torque, etc..)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-*  https://github.com/dask/dask-drmaa
-*  https://github.com/mfouesneau/dasksge
-
-YARN
-~~~~
-
-*   https://github.com/dask/dask-yarn
-*   https://knit.readthedocs.io/en/latest/
-
-
-Software Environment
---------------------
-
-The workers and clients should all share the same software environment.  That
-means that they should all have access to the same libraries and that those
-libraries should be the same version.  Dask generally assumes that it can call
-a function on any worker with the same outcome (unless explicitly told
-otherwise.)
-
-This is typically enforced through external means, such as by having a network
-file system (NFS) mount for libraries, by starting the ``dask-worker``
-processes in equivalent Docker_ containers, using Conda_ environments, or
-through any of the other means typically employed by cluster administrators.
-
-.. _Docker: https://www.docker.com/
-.. _Conda: http://conda.pydata.org/docs/
-
-
-Windows
-~~~~~~~
-
-.. note::
-
-  - Running a ``dask-scheduler`` on Windows architectures is supported for only a
-    limited number of workers (roughly 100). This is a detail of the underlying tcp server
-    implementation and is discussed `here`__.
-
-  - Running ``dask-worker`` processes on Windows is well supported, performant, and without limit.
-
-If you wish to run in a primarily Windows environment, it is recommended
-to run a ``dask-scheduler`` on a linux or MacOSX environment, with ``dask-worker`` workers
-on the Windows boxes. This works because the scheduler environment is de-coupled from that of
-the workers.
-
-__ https://github.com/jfisteus/ztreamy/issues/26
-
-
-Customizing initialization
---------------------------
-
-Both ``dask-scheduler`` and ``dask-worker`` support a ``--preload`` option that
-allows custom initialization of each scheduler/worker respectively. A module or
-python file passed as a ``--preload`` value is guaranteed to be imported before
-establishing any connection. A ``dask_setup(service)`` function is called if
-found, with a ``Scheduler`` or ``Worker`` instance as the argument. As the
-service stops, ``dask_teardown(service)`` is called if present.
-
-To support additional configuration a single ``--preload`` module may register
-additional command-line arguments by exposing ``dask_setup`` as a  Click_
-command.  This command will be used to parse additional arguments provided to
-``dask-worker`` or ``dask-scheduler`` and will be called before service
-initialization.
-
-.. _Click: http://click.pocoo.org/
-
-
-As an example, consider the following file that creates a
-:doc:`scheduler plugin <plugins>` and registers it with the scheduler
-
-.. code-block:: python
-
-   # scheduler-setup.py
-   import click
-
-   from distributed.diagnostics.plugin import SchedulerPlugin
-
-   class MyPlugin(SchedulerPlugin):
-       def __init__(self, print_count):
-         self.print_count = print_count
-         SchedulerPlugin.__init__(self)
-
-       def add_worker(self, scheduler=None, worker=None, **kwargs):
-           print("Added a new worker at:", worker)
-           if self.print_count and scheduler is not None:
-               print("Total workers:", len(scheduler.workers))
-
-   @click.command()
-   @click.option("--print-count/--no-print-count", default=False)
-   def dask_setup(scheduler, print_count):
-       plugin = MyPlugin(print_count)
-       scheduler.add_plugin(plugin)
-
-We can then run this preload script by referring to its filename (or module name
-if it is on the path) when we start the scheduler::
-
-   dask-scheduler --preload scheduler-setup.py --print-count

From 8ee867be3b0de454467f736963bf7e5501aa3815 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Wed, 7 Aug 2019 14:29:21 -0400
Subject: [PATCH 0409/1550] Fixes Worker docstring formatting [skip ci] (#2939)

---
 distributed/worker.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/worker.py b/distributed/worker.py
index 9f686cfa508..f9b6348ebac 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -97,7 +97,7 @@ class Worker(ServerNode):
 
         $ dask-worker scheduler-ip:port
 
-    Use the ``--help`` flag to see more options
+    Use the ``--help`` flag to see more options::
 
         $ dask-worker --help
 

From 4d98bb5d6e8b45270c81e45a5662f1b1d4edee24 Mon Sep 17 00:00:00 2001
From: Jim Crist <jcrist@users.noreply.github.com>
Date: Thu, 8 Aug 2019 10:29:21 -0500
Subject: [PATCH 0410/1550] Import from collections.abc (#2938)

Silences deprecation warnings about importing from collections instead
of collections.abc.
---
 distributed/client.py                  | 3 ++-
 distributed/publish.py                 | 2 +-
 distributed/scheduler.py               | 3 ++-
 distributed/tests/test_as_completed.py | 2 +-
 distributed/worker.py                  | 3 ++-
 5 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 9f59582b019..678c3b4dbbe 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -1,5 +1,6 @@
 import atexit
-from collections import defaultdict, Iterator
+from collections import defaultdict
+from collections.abc import Iterator
 from concurrent.futures import ThreadPoolExecutor, CancelledError
 from concurrent.futures._base import DoneAndNotDoneFutures
 from contextlib import contextmanager
diff --git a/distributed/publish.py b/distributed/publish.py
index ea65efb4e74..c899b9fbaaa 100644
--- a/distributed/publish.py
+++ b/distributed/publish.py
@@ -1,4 +1,4 @@
-from collections import MutableMapping
+from collections.abc import MutableMapping
 
 from .utils import log_errors, tokey
 
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 65e93d3e59c..809a5bd303e 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1,5 +1,6 @@
 import asyncio
-from collections import defaultdict, deque, OrderedDict, Mapping, Set
+from collections import defaultdict, deque, OrderedDict
+from collections.abc import Mapping, Set
 from datetime import timedelta
 from functools import partial
 import itertools
diff --git a/distributed/tests/test_as_completed.py b/distributed/tests/test_as_completed.py
index d74d033c64a..45833b302e1 100644
--- a/distributed/tests/test_as_completed.py
+++ b/distributed/tests/test_as_completed.py
@@ -1,5 +1,5 @@
 from concurrent.futures import CancelledError
-from collections import Iterator
+from collections.abc import Iterator
 from operator import add
 import queue
 import random
diff --git a/distributed/worker.py b/distributed/worker.py
index f9b6348ebac..dce00f706f7 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -1,6 +1,7 @@
 import asyncio
 import bisect
-from collections import defaultdict, deque, MutableMapping
+from collections import defaultdict, deque
+from collections.abc import MutableMapping
 from datetime import timedelta
 import heapq
 import logging

From a55515569d4c5da734e5b14ae414cd342c37ed7b Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 8 Aug 2019 21:24:01 -0400
Subject: [PATCH 0411/1550] Pass GPU diagnostics from worker to scheduler
 (#2932)

This does a few things:

1.  Use `pynvml` to collect information about any CUDA GPUs present
2.  Optionally add those metrics to the worker's initial handshake and
    heartbeats
3.  Collect that information in the scheduler in the WorkerState object

For now these just hang out in the scheduler information,
but in the future they might be used for dashboards,
or possibly scheduling decisions in the future.

I believe that everything gpu-specific here is fairly well separated
and generalized (others should be able to follow this pattern to add
more diagnostics relatively easily) but it would be good to hear from
others on if this is out of scope.
---
 distributed/diagnostics/nvml.py  | 20 ++++++++++++
 distributed/scheduler.py         |  8 +++++
 distributed/tests/test_worker.py | 12 ++++++++
 distributed/worker.py            | 53 +++++++++++++++++++++++++++++---
 4 files changed, 88 insertions(+), 5 deletions(-)
 create mode 100644 distributed/diagnostics/nvml.py

diff --git a/distributed/diagnostics/nvml.py b/distributed/diagnostics/nvml.py
new file mode 100644
index 00000000000..25a11cde6b0
--- /dev/null
+++ b/distributed/diagnostics/nvml.py
@@ -0,0 +1,20 @@
+import pynvml
+
+pynvml.nvmlInit()
+count = pynvml.nvmlDeviceGetCount()
+
+handles = [pynvml.nvmlDeviceGetHandleByIndex(i) for i in range(count)]
+
+
+def real_time():
+    return {
+        "utilization": [pynvml.nvmlDeviceGetUtilizationRates(h).gpu for h in handles],
+        "memory-used": [pynvml.nvmlDeviceGetMemoryInfo(h).used for h in handles],
+    }
+
+
+def one_time():
+    return {
+        "memory-total": [pynvml.nvmlDeviceGetMemoryInfo(h).total for h in handles],
+        "name": [pynvml.nvmlDeviceGetName(h).decode() for h in handles],
+    }
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 809a5bd303e..8fc4a828bdf 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -211,6 +211,7 @@ class WorkerState(object):
     __slots__ = (
         "actors",
         "address",
+        "extra",
         "has_what",
         "last_seen",
         "local_directory",
@@ -240,6 +241,7 @@ def __init__(
         local_directory=None,
         services=None,
         nanny=None,
+        extra=None,
     ):
         self.address = address
         self.pid = pid
@@ -263,6 +265,8 @@ def __init__(
         self.resources = {}
         self.used_resources = {}
 
+        self.extra = extra or {}
+
     @property
     def host(self):
         return get_address_host(self.address)
@@ -278,6 +282,7 @@ def clean(self):
             local_directory=self.local_directory,
             services=self.services,
             nanny=self.nanny,
+            extra=self.extra,
         )
         ws.processing = {ts.key for ts in self.processing}
         return ws
@@ -306,6 +311,7 @@ def identity(self):
             "services": self.services,
             "metrics": self.metrics,
             "nanny": self.nanny,
+            **self.extra,
         }
 
     @property
@@ -1386,6 +1392,7 @@ async def add_worker(
         services=None,
         local_directory=None,
         nanny=None,
+        extra=None,
     ):
         """ Add a new worker to the cluster """
         with log_errors():
@@ -1406,6 +1413,7 @@ async def add_worker(
                 local_directory=local_directory,
                 services=services,
                 nanny=nanny,
+                extra=extra,
             )
 
             if name in self.aliases:
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 64e774f582a..4dab232487f 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -1536,3 +1536,15 @@ async def test_lifetime_stagger(c, s, a, b):
     assert a.lifetime != b.lifetime
     assert 8 <= a.lifetime <= 12
     assert 8 <= b.lifetime <= 12
+
+
+@gen_cluster()
+async def test_gpu_metrics(s, a, b):
+    pytest.importorskip("pynvml")
+    from distributed.diagnostics.nvml import count
+
+    assert "gpu" in a.metrics
+    assert len(s.workers[a.address].metrics["gpu"]["memory-used"]) == count
+
+    assert "gpu" in a.startup_information
+    assert len(s.workers[a.address].extra["gpu"]["name"]) == count
diff --git a/distributed/worker.py b/distributed/worker.py
index dce00f706f7..17d56aec79b 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -82,6 +82,10 @@
 
 DEFAULT_EXTENSIONS = [PubSubWorkerExtension]
 
+DEFAULT_METRICS = {}
+
+DEFAULT_STARTUP_INFORMATION = {}
+
 
 class Worker(ServerNode):
     """ Worker node in a Dask distributed cluster
@@ -306,7 +310,8 @@ def __init__(
         contact_address=None,
         memory_monitor_interval="200ms",
         extensions=None,
-        metrics=None,
+        metrics=DEFAULT_METRICS,
+        startup_information=DEFAULT_STARTUP_INFORMATION,
         data=None,
         interface=None,
         host=None,
@@ -577,6 +582,9 @@ def __init__(
                 )
 
         self.metrics = dict(metrics) if metrics else {}
+        self.startup_information = (
+            dict(startup_information) if startup_information else {}
+        )
 
         self.low_level_profiler = low_level_profiler
 
@@ -717,7 +725,7 @@ def local_dir(self):
         )
         return self.local_directory
 
-    def get_metrics(self):
+    async def get_metrics(self):
         core = dict(
             executing=len(self.executing),
             in_memory=len(self.data),
@@ -725,10 +733,24 @@ def get_metrics(self):
             in_flight=len(self.in_flight_tasks),
             bandwidth=self.bandwidth,
         )
-        custom = {k: metric(self) for k, metric in self.metrics.items()}
+        custom = {}
+        for k, metric in self.metrics.items():
+            result = metric(self)
+            if hasattr(result, "__await__"):
+                result = await result
+            custom[k] = result
 
         return merge(custom, self.monitor.recent(), core)
 
+    async def get_startup_information(self):
+        result = {}
+        for k, f in self.startup_information.items():
+            v = f(self)
+            if hasattr(v, "__await__"):
+                v = await v
+            result[k] = v
+        return result
+
     def identity(self, comm=None):
         return {
             "type": type(self).__name__,
@@ -786,7 +808,8 @@ async def _register_with_scheduler(self):
                         services=self.service_ports,
                         nanny=self.nanny,
                         pid=os.getpid(),
-                        metrics=self.get_metrics(),
+                        metrics=await self.get_metrics(),
+                        extra=await self.get_startup_information(),
                     ),
                     serializers=["msgpack"],
                 )
@@ -840,7 +863,9 @@ async def heartbeat(self):
             try:
                 start = time()
                 response = await self.scheduler.heartbeat_worker(
-                    address=self.contact_address, now=time(), metrics=self.get_metrics()
+                    address=self.contact_address,
+                    now=time(),
+                    metrics=await self.get_metrics(),
                 )
                 end = time()
                 middle = (start + end) / 2
@@ -3369,3 +3394,21 @@ async def run(server, comm, function, args=(), kwargs={}, is_coro=None, wait=Tru
 
 
 _global_workers = Worker._instances
+
+try:
+    from .diagnostics import nvml
+except ImportError:
+    pass
+else:
+
+    @gen.coroutine
+    def gpu_metric(worker):
+        result = yield offload(nvml.real_time)
+        return result
+
+    DEFAULT_METRICS["gpu"] = gpu_metric
+
+    def gpu_startup(worker):
+        return nvml.one_time()
+
+    DEFAULT_STARTUP_INFORMATION["gpu"] = gpu_startup

From b27f7b7ff4215c92b2e8f9e1dad5c3b2165c61ef Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 12 Aug 2019 08:31:58 -0400
Subject: [PATCH 0412/1550] Add GPUCurrentLoad dashboard plots (#2944)

---
 distributed/dashboard/nvml.py                 | 187 ++++++++++++++++++
 distributed/dashboard/scheduler.py            |  53 ++---
 .../dashboard/tests/test_scheduler_bokeh.py   |  19 +-
 3 files changed, 221 insertions(+), 38 deletions(-)
 create mode 100644 distributed/dashboard/nvml.py

diff --git a/distributed/dashboard/nvml.py b/distributed/dashboard/nvml.py
new file mode 100644
index 00000000000..7fd628dd469
--- /dev/null
+++ b/distributed/dashboard/nvml.py
@@ -0,0 +1,187 @@
+import math
+
+from .components import DashboardComponent, add_periodic_callback
+
+from bokeh.plotting import figure
+from bokeh.models import (
+    ColumnDataSource,
+    BasicTicker,
+    NumeralTickFormatter,
+    TapTool,
+    OpenURL,
+    HoverTool,
+)
+from tornado import escape
+from dask.utils import format_bytes
+from ..utils import log_errors
+from .scheduler import update, applications, BOKEH_THEME
+from .utils import without_property_validation
+
+
+class GPUCurrentLoad(DashboardComponent):
+    """ How many tasks are on each worker """
+
+    def __init__(self, scheduler, width=600, **kwargs):
+        with log_errors():
+            self.last = 0
+            self.scheduler = scheduler
+            self.source = ColumnDataSource(
+                {
+                    "memory": [1, 2],
+                    "memory-half": [0.5, 1],
+                    "memory_text": ["1B", "2B"],
+                    "utilization": [1, 2],
+                    "utilization-half": [0.5, 1],
+                    "worker": ["a", "b"],
+                    "gpu-index": [0, 0],
+                    "y": [1, 2],
+                    "escaped_worker": ["a", "b"],
+                }
+            )
+
+            memory = figure(
+                title="GPU Memory",
+                tools="",
+                id="bk-gpu-memory-worker-plot",
+                width=int(width / 2),
+                name="gpu_memory_histogram",
+                **kwargs
+            )
+            rect = memory.rect(
+                source=self.source,
+                x="memory-half",
+                y="y",
+                width="memory",
+                height=1,
+                color="#76B900",
+            )
+            rect.nonselection_glyph = None
+
+            utilization = figure(
+                title="GPU Utilization",
+                tools="",
+                id="bk-gpu-utilization-worker-plot",
+                width=int(width / 2),
+                name="gpu_utilization_histogram",
+                **kwargs
+            )
+            rect = utilization.rect(
+                source=self.source,
+                x="utilization-half",
+                y="y",
+                width="utilization",
+                height=1,
+                color="#76B900",
+            )
+            rect.nonselection_glyph = None
+
+            memory.axis[0].ticker = BasicTicker(mantissas=[1, 256, 512], base=1024)
+            memory.xaxis[0].formatter = NumeralTickFormatter(format="0.0 b")
+            memory.xaxis.major_label_orientation = -math.pi / 12
+            memory.x_range.start = 0
+
+            for fig in [memory, utilization]:
+                fig.xaxis.minor_tick_line_alpha = 0
+                fig.yaxis.visible = False
+                fig.ygrid.visible = False
+
+                tap = TapTool(
+                    callback=OpenURL(url="./info/worker/@escaped_worker.html")
+                )
+                fig.add_tools(tap)
+
+                fig.toolbar.logo = None
+                fig.toolbar_location = None
+                fig.yaxis.visible = False
+
+            hover = HoverTool()
+            hover.tooltips = "@worker : @utilization %"
+            hover.point_policy = "follow_mouse"
+            utilization.add_tools(hover)
+
+            hover = HoverTool()
+            hover.tooltips = "@worker : @memory_text"
+            hover.point_policy = "follow_mouse"
+            memory.add_tools(hover)
+
+            self.memory_figure = memory
+            self.utilization_figure = utilization
+
+            self.utilization_figure.y_range = memory.y_range
+            self.utilization_figure.x_range.start = 0
+            self.utilization_figure.x_range.end = 100
+
+    @without_property_validation
+    def update(self):
+        with log_errors():
+            workers = list(self.scheduler.workers.values())
+
+            utilization = []
+            memory = []
+            gpu_index = []
+            y = []
+            memory_total = 0
+            memory_max = 0
+            worker = []
+            i = 0
+
+            for ws in workers:
+                info = ws.extra["gpu"]
+                metrics = ws.metrics["gpu"]
+                for j, (u, mem_used, mem_total) in enumerate(
+                    zip(
+                        metrics["utilization"],
+                        metrics["memory-used"],
+                        info["memory-total"],
+                    )
+                ):
+                    memory_max = max(memory_max, mem_total)
+                    memory_total += mem_total
+                    utilization.append(int(u))
+                    memory.append(mem_used)
+                    worker.append(ws.address)
+                    gpu_index.append(j)
+                    y.append(i)
+                    i += 1
+
+            memory_text = [format_bytes(m) for m in memory]
+
+            result = {
+                "memory": memory,
+                "memory-half": [m / 2 for m in memory],
+                "memory_text": memory_text,
+                "utilization": utilization,
+                "utilization-half": [u / 2 for u in utilization],
+                "worker": worker,
+                "gpu-index": gpu_index,
+                "y": y,
+                "escaped_worker": [escape.url_escape(w) for w in worker],
+            }
+
+            self.memory_figure.title.text = "GPU Memory: %s / %s" % (
+                format_bytes(sum(memory)),
+                format_bytes(memory_total),
+            )
+            self.memory_figure.x_range.end = memory_max
+
+            update(self.source, result)
+
+
+def gpu_memory_doc(scheduler, extra, doc):
+    gpu_load = GPUCurrentLoad(scheduler, sizing_mode="stretch_both")
+    gpu_load.update()
+    add_periodic_callback(doc, gpu_load, 100)
+    doc.add_root(gpu_load.memory_figure)
+    doc.theme = BOKEH_THEME
+
+
+def gpu_utilization_doc(scheduler, extra, doc):
+    gpu_load = GPUCurrentLoad(scheduler, sizing_mode="stretch_both")
+    gpu_load.update()
+    add_periodic_callback(doc, gpu_load, 100)
+    doc.add_root(gpu_load.utilization_figure)
+    doc.theme = BOKEH_THEME
+
+
+applications["/individual-gpu-memory"] = gpu_memory_doc
+applications["/individual-gpu-utilization"] = gpu_utilization_doc
diff --git a/distributed/dashboard/scheduler.py b/distributed/dashboard/scheduler.py
index 3332f4fc27b..3dd108f5775 100644
--- a/distributed/dashboard/scheduler.py
+++ b/distributed/dashboard/scheduler.py
@@ -1651,28 +1651,7 @@ def __init__(self, scheduler, io_loop=None, prefix="", **kwargs):
 
         self.server_kwargs["prefix"] = prefix or None
 
-        self.apps = {
-            "/system": systemmonitor_doc,
-            "/stealing": stealing_doc,
-            "/workers": workers_doc,
-            "/events": events_doc,
-            "/counters": counters_doc,
-            "/tasks": tasks_doc,
-            "/status": status_doc,
-            "/profile": profile_doc,
-            "/profile-server": profile_server_doc,
-            "/graph": graph_doc,
-            "/individual-task-stream": individual_task_stream_doc,
-            "/individual-progress": individual_progress_doc,
-            "/individual-graph": individual_graph_doc,
-            "/individual-profile": individual_profile_doc,
-            "/individual-profile-server": individual_profile_server_doc,
-            "/individual-nbytes": individual_nbytes_doc,
-            "/individual-cpu": individual_cpu_doc,
-            "/individual-nprocessing": individual_nprocessing_doc,
-            "/individual-workers": individual_workers_doc,
-        }
-
+        self.apps = applications
         self.apps = {k: partial(v, scheduler, self.extra) for k, v in self.apps.items()}
 
         self.loop = io_loop or scheduler.loop
@@ -1701,3 +1680,33 @@ def listen(self, *args, **kwargs):
         ]
 
         self.server._tornado.add_handlers(r".*", handlers)
+
+
+applications = {
+    "/system": systemmonitor_doc,
+    "/stealing": stealing_doc,
+    "/workers": workers_doc,
+    "/events": events_doc,
+    "/counters": counters_doc,
+    "/tasks": tasks_doc,
+    "/status": status_doc,
+    "/profile": profile_doc,
+    "/profile-server": profile_server_doc,
+    "/graph": graph_doc,
+    "/individual-task-stream": individual_task_stream_doc,
+    "/individual-progress": individual_progress_doc,
+    "/individual-graph": individual_graph_doc,
+    "/individual-profile": individual_profile_doc,
+    "/individual-profile-server": individual_profile_server_doc,
+    "/individual-nbytes": individual_nbytes_doc,
+    "/individual-cpu": individual_cpu_doc,
+    "/individual-nprocessing": individual_nprocessing_doc,
+    "/individual-workers": individual_workers_doc,
+}
+
+try:
+    import pynvml  # noqa: 1708
+except ImportError:
+    pass
+else:
+    from . import nvml  # noqa: 1708
diff --git a/distributed/dashboard/tests/test_scheduler_bokeh.py b/distributed/dashboard/tests/test_scheduler_bokeh.py
index e9ac62aad41..2dc29572ea8 100644
--- a/distributed/dashboard/tests/test_scheduler_bokeh.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh.py
@@ -18,6 +18,7 @@
 from distributed.utils_test import gen_cluster, inc, dec, slowinc, div, get_cert
 from distributed.dashboard.worker import Counters, BokehWorker
 from distributed.dashboard.scheduler import (
+    applications,
     BokehScheduler,
     SystemMonitor,
     Occupancy,
@@ -54,22 +55,8 @@ def test_simple(c, s, a, b):
     yield gen.sleep(0.1)
 
     http_client = AsyncHTTPClient()
-    for suffix in [
-        "system",
-        "counters",
-        "workers",
-        "status",
-        "tasks",
-        "stealing",
-        "graph",
-        "individual-task-stream",
-        "individual-progress",
-        "individual-graph",
-        "individual-nbytes",
-        "individual-nprocessing",
-        "individual-profile",
-    ]:
-        response = yield http_client.fetch("http://localhost:%d/%s" % (port, suffix))
+    for suffix in applications:
+        response = yield http_client.fetch("http://localhost:%d%s" % (port, suffix))
         body = response.body.decode()
         assert "bokeh" in body.lower()
         assert not re.search("href=./", body)  # no absolute links

From b83edbef74a1718d62e51a9cee0379b7617048e1 Mon Sep 17 00:00:00 2001
From: Shayan Amani <SHi-ON@users.noreply.github.com>
Date: Mon, 12 Aug 2019 15:37:53 -0400
Subject: [PATCH 0413/1550] Update client.py (#2951)

---
 distributed/client.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/client.py b/distributed/client.py
index 678c3b4dbbe..9d05cde049e 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -4265,7 +4265,7 @@ def default_client(c=None):
     else:
         raise ValueError(
             "No clients found\n"
-            "Start an client and point it to the scheduler address\n"
+            "Start a client and point it to the scheduler address\n"
             "  from distributed import Client\n"
             "  client = Client('ip-addr-of-scheduler:8786')\n"
         )

From 28f300af7136b362a92afc0a93a9de23b374b0c8 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 13 Aug 2019 09:15:47 -0400
Subject: [PATCH 0414/1550] Normalize names with str in retire_workers (#2949)

This supports cases where names are passed through a CLI and become strings
---
 distributed/scheduler.py            |  8 ++++++--
 distributed/tests/test_scheduler.py | 14 ++++++++++++++
 2 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 8fc4a828bdf..ff9560767c0 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -3078,9 +3078,11 @@ async def retire_workers(
         """
         with log_errors():
             if names is not None:
-                names = set(names)
+                if names:
+                    logger.info("Retire worker names %s", names)
+                names = set(map(str, names))
                 workers = [
-                    ws.address for ws in self.workers.values() if ws.name in names
+                    ws.address for ws in self.workers.values() if str(ws.name) in names
                 ]
             if workers is None:
                 while True:
@@ -3098,6 +3100,7 @@ async def retire_workers(
             workers = {self.workers[w] for w in workers if w in self.workers}
             if not workers:
                 return []
+            logger.info("Retire workers %s", workers)
 
             # Keys orphaned by retiring those workers
             keys = set.union(*[w.has_what for w in workers])
@@ -3106,6 +3109,7 @@ async def retire_workers(
             other_workers = set(self.workers.values()) - workers
             if keys:
                 if other_workers:
+                    logger.info("Moving %d keys to other workers", len(keys))
                     await self.replicate(
                         keys=keys,
                         workers=[ws.address for ws in other_workers],
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 9035fbd8667..943daffde58 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -1627,3 +1627,17 @@ async def test_finished():
 
     await s.finished()
     await w.finished()
+
+
+@pytest.mark.asyncio
+async def test_retire_names_str(cleanup):
+    async with Scheduler(port=0) as s:
+        async with Worker(s.address, name="0") as a:
+            async with Worker(s.address, name="1") as b:
+                async with Client(s.address, asynchronous=True) as c:
+                    futures = c.map(inc, range(10))
+                    await wait(futures)
+                    assert a.data and b.data
+                    await s.retire_workers(names=[0])
+                    assert all(f.done() for f in futures)
+                    assert len(b.data) == 10

From 3bab3aaabefa5f958ef44894599d71fffaa59b03 Mon Sep 17 00:00:00 2001
From: Shayan Amani <SHi-ON@users.noreply.github.com>
Date: Tue, 13 Aug 2019 10:40:58 -0400
Subject: [PATCH 0415/1550] Update utils_perf.py (#2954)

---
 distributed/utils_perf.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/utils_perf.py b/distributed/utils_perf.py
index c2257f38fb0..f21e96d7353 100644
--- a/distributed/utils_perf.py
+++ b/distributed/utils_perf.py
@@ -50,7 +50,7 @@ def collect(self):
             if self.last_gc_duration > self.warn_if_longer:
                 self.logger.warning(
                     "gc.collect() took %0.3fs. This is usually"
-                    " a sign that the some tasks handle too"
+                    " a sign that some tasks handle too"
                     " many Python objects at the same time."
                     " Rechunking the work into smaller tasks"
                     " might help.",

From 6302175056742d101b8af52336bc6bbf4227da35 Mon Sep 17 00:00:00 2001
From: Bruce Merry <bmerry@ska.ac.za>
Date: Thu, 15 Aug 2019 15:17:35 +0200
Subject: [PATCH 0416/1550] Allow server_kwargs to override defaults in
 dashboard (#2955)

Fixes #2915

Add a unit test for overriding allow_websocket_origin
---
 distributed/dashboard/core.py                 |  6 ++---
 .../tests/test_scheduler_bokeh_html.py        | 26 ++++++++++++++++++-
 2 files changed, 28 insertions(+), 4 deletions(-)

diff --git a/distributed/dashboard/core.py b/distributed/dashboard/core.py
index fd6ebef2834..41e7c289c17 100644
--- a/distributed/dashboard/core.py
+++ b/distributed/dashboard/core.py
@@ -28,16 +28,16 @@ def listen(self, addr):
             ip = None
         for i in range(5):
             try:
-                self.server = Server(
-                    self.apps,
+                server_kwargs = dict(
                     port=port,
                     address=ip,
                     check_unused_sessions_milliseconds=500,
                     allow_websocket_origin=["*"],
                     use_index=False,
                     extra_patterns=[(r"/", web.RedirectHandler, {"url": "/status"})],
-                    **self.server_kwargs
                 )
+                server_kwargs.update(self.server_kwargs)
+                self.server = Server(self.apps, **server_kwargs)
                 self.server.start()
 
                 handlers = [
diff --git a/distributed/dashboard/tests/test_scheduler_bokeh_html.py b/distributed/dashboard/tests/test_scheduler_bokeh_html.py
index f2a2c880a94..55e4b797b4e 100644
--- a/distributed/dashboard/tests/test_scheduler_bokeh_html.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh_html.py
@@ -6,7 +6,8 @@
 pytest.importorskip("bokeh")
 
 from tornado.escape import url_escape
-from tornado.httpclient import AsyncHTTPClient, HTTPClientError
+from tornado.httpclient import AsyncHTTPClient, HTTPClientError, HTTPRequest
+from tornado.websocket import websocket_connect
 
 from dask.sizeof import sizeof
 from distributed.utils import is_valid_xml
@@ -153,3 +154,26 @@ def test_task_page(c, s, a, b):
     assert "int" in body
     assert a.address in body
     assert "memory" in body
+
+
+@gen_cluster(
+    client=True,
+    scheduler_kwargs={
+        "services": {
+            ("dashboard", 0): (
+                BokehScheduler,
+                {"allow_websocket_origin": ["good.invalid"]},
+            )
+        }
+    },
+)
+def test_allow_websocket_origin(c, s, a, b):
+    url = (
+        "ws://localhost:%d/status/ws?bokeh-protocol-version=1.0&bokeh-session-id=1"
+        % s.services["dashboard"].port
+    )
+    with pytest.raises(HTTPClientError) as err:
+        yield websocket_connect(
+            HTTPRequest(url, headers={"Origin": "http://evil.invalid"})
+        )
+    assert err.value.code == 403

From 2a4bc72a7384b5ed357cef890e7be9eb1a80acfc Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 16 Aug 2019 14:47:23 -0400
Subject: [PATCH 0417/1550] Use pytest.warning(Warning) rather than Exception
 (#2958)

---
 distributed/deploy/tests/test_local.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index f434945c3af..31493967d4c 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -195,7 +195,7 @@ def test_Client_solo(loop):
 def test_duplicate_clients():
     pytest.importorskip("bokeh")
     c1 = yield Client(processes=False, silence_logs=False, dashboard_address=9876)
-    with pytest.warns(Exception) as info:
+    with pytest.warns(Warning) as info:
         c2 = yield Client(processes=False, silence_logs=False, dashboard_address=9876)
 
     assert "dashboard" in c1.cluster.scheduler.services

From 41a4d41d174c7762d63e17067327518cb8d313e9 Mon Sep 17 00:00:00 2001
From: Peter Andreas Entschev <peter@entschev.com>
Date: Fri, 16 Aug 2019 21:51:17 +0200
Subject: [PATCH 0418/1550] Pass serialization down through small base
 collections (#2948)

This PR adds support for serialization of collections using objects' native types, rather than pickling the entire collection
---
 distributed/protocol/serialize.py             | 80 +++++++++++++++++++
 distributed/protocol/tests/test_collection.py | 50 ++++++++++++
 .../protocol/tests/test_collection_cuda.py    | 66 +++++++++++++++
 distributed/protocol/tests/test_serialize.py  |  4 +-
 distributed/tests/test_publish.py             |  2 +-
 5 files changed, 199 insertions(+), 3 deletions(-)
 create mode 100644 distributed/protocol/tests/test_collection.py
 create mode 100644 distributed/protocol/tests/test_collection_cuda.py

diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index 0069c6a264d..8d1d37a283e 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -135,6 +135,54 @@ def serialize(x, serializers=None, on_error="message", context=None):
     if isinstance(x, Serialized):
         return x.header, x.frames
 
+    # Determine whether keys are safe to be serialized with msgpack
+    if type(x) is dict and len(x) <= 5:
+        try:
+            msgpack.dumps(list(x.keys()))
+        except Exception:
+            dict_safe = False
+        else:
+            dict_safe = True
+
+    if (
+        type(x) in (list, set, tuple)
+        and len(x) <= 5
+        or type(x) is dict
+        and len(x) <= 5
+        and dict_safe
+    ):
+        if isinstance(x, dict):
+            headers_frames = []
+            for k, v in x.items():
+                _header, _frames = serialize(
+                    v, serializers=serializers, on_error=on_error, context=context
+                )
+                _header["key"] = k
+                headers_frames.append((_header, _frames))
+        else:
+            headers_frames = [
+                serialize(
+                    obj, serializers=serializers, on_error=on_error, context=context
+                )
+                for obj in x
+            ]
+
+        frames = []
+        lengths = []
+        for _header, _frames in headers_frames:
+            frames.extend(_frames)
+            length = len(_frames)
+            lengths.append(length)
+
+        headers = [obj[0] for obj in headers_frames]
+        headers = {
+            "sub-headers": headers,
+            "is-collection": True,
+            "frame-lengths": lengths,
+            "type-serialized": type(x).__name__,
+        }
+        return headers, frames
+
     tb = ""
 
     for name in serializers:
@@ -178,6 +226,38 @@ def deserialize(header, frames, deserializers=None):
     --------
     serialize
     """
+    if "is-collection" in header:
+        headers = header["sub-headers"]
+        lengths = header["frame-lengths"]
+        cls = {"tuple": tuple, "list": list, "set": set, "dict": dict}[
+            header["type-serialized"]
+        ]
+
+        start = 0
+        if cls is dict:
+            d = {}
+            for _header, _length in zip(headers, lengths):
+                k = _header.pop("key")
+                d[k] = deserialize(
+                    _header,
+                    frames[start : start + _length],
+                    deserializers=deserializers,
+                )
+                start += _length
+            return d
+        else:
+            lst = []
+            for _header, _length in zip(headers, lengths):
+                lst.append(
+                    deserialize(
+                        _header,
+                        frames[start : start + _length],
+                        deserializers=deserializers,
+                    )
+                )
+                start += _length
+            return cls(lst)
+
     name = header.get("serializer")
     if deserializers is not None and name not in deserializers:
         raise TypeError(
diff --git a/distributed/protocol/tests/test_collection.py b/distributed/protocol/tests/test_collection.py
new file mode 100644
index 00000000000..ddb8a44bd44
--- /dev/null
+++ b/distributed/protocol/tests/test_collection.py
@@ -0,0 +1,50 @@
+import pytest
+from distributed.protocol import serialize, deserialize
+import pandas as pd
+import numpy as np
+
+
+@pytest.mark.parametrize("collection", [tuple, dict, list])
+@pytest.mark.parametrize(
+    "y,y_serializer",
+    [
+        (np.arange(50), "dask"),
+        (pd.DataFrame({"C": ["a", "b", None], "D": [2.5, 3.5, 4.5]}), "pickle"),
+        (None, "pickle"),
+    ],
+)
+def test_serialize_collection(collection, y, y_serializer):
+    x = np.arange(100)
+    if issubclass(collection, dict):
+        header, frames = serialize({"x": x, "y": y}, serializers=("dask", "pickle"))
+    else:
+        header, frames = serialize(collection((x, y)), serializers=("dask", "pickle"))
+    t = deserialize(header, frames, deserializers=("dask", "pickle", "error"))
+    assert isinstance(t, collection)
+
+    assert header["is-collection"] is True
+    sub_headers = header["sub-headers"]
+
+    if collection is not dict:
+        assert sub_headers[0]["serializer"] == "dask"
+        assert sub_headers[1]["serializer"] == y_serializer
+
+    if collection is dict:
+        assert (t["x"] == x).all()
+        assert str(t["y"]) == str(y)
+    else:
+        assert (t[0] == x).all()
+        assert str(t[1]) == str(y)
+
+
+def test_large_collections_serialize_simply():
+    header, frames = serialize(tuple(range(1000)))
+    assert len(frames) == 1
+
+
+def test_nested_types():
+    x = np.ones(5)
+    header, frames = serialize([[[x]]])
+    assert "dask" in str(header)
+    assert len(frames) == 1
+    assert x.data in frames
diff --git a/distributed/protocol/tests/test_collection_cuda.py b/distributed/protocol/tests/test_collection_cuda.py
new file mode 100644
index 00000000000..e2602795782
--- /dev/null
+++ b/distributed/protocol/tests/test_collection_cuda.py
@@ -0,0 +1,66 @@
+import pytest
+
+from distributed.protocol import serialize, deserialize
+from dask.dataframe.utils import assert_eq
+import pandas as pd
+
+
+@pytest.mark.parametrize("collection", [tuple, dict])
+@pytest.mark.parametrize("y,y_serializer", [(50, "cuda"), (None, "pickle")])
+def test_serialize_cupy(collection, y, y_serializer):
+    cupy = pytest.importorskip("cupy")
+
+    x = cupy.arange(100)
+    if y is not None:
+        y = cupy.arange(y)
+    if issubclass(collection, dict):
+        header, frames = serialize(
+            {"x": x, "y": y}, serializers=("cuda", "dask", "pickle")
+        )
+    else:
+        header, frames = serialize((x, y), serializers=("cuda", "dask", "pickle"))
+    t = deserialize(header, frames, deserializers=("cuda", "dask", "pickle", "error"))
+
+    assert header["is-collection"] is True
+    sub_headers = header["sub-headers"]
+    assert sub_headers[0]["serializer"] == "cuda"
+    assert sub_headers[1]["serializer"] == y_serializer
+    assert isinstance(t, collection)
+
+    assert ((t["x"] if isinstance(t, dict) else t[0]) == x).all()
+    if y is None:
+        assert (t["y"] if isinstance(t, dict) else t[1]) is None
+    else:
+        assert ((t["y"] if isinstance(t, dict) else t[1]) == y).all()
+
+
+@pytest.mark.parametrize("collection", [tuple, dict])
+@pytest.mark.parametrize(
+    "df2,df2_serializer",
+    [(pd.DataFrame({"C": [3, 4, 5], "D": [2.5, 3.5, 4.5]}), "cuda"), (None, "pickle")],
+)
+def test_serialize_pandas_pandas(collection, df2, df2_serializer):
+    cudf = pytest.importorskip("cudf")
+
+    df1 = cudf.DataFrame({"A": [1, 2, None], "B": [1.0, 2.0, None]})
+    if df2 is not None:
+        df2 = cudf.from_pandas(df2)
+    if issubclass(collection, dict):
+        header, frames = serialize(
+            {"df1": df1, "df2": df2}, serializers=("cuda", "dask", "pickle")
+        )
+    else:
+        header, frames = serialize((df1, df2), serializers=("cuda", "dask", "pickle"))
+    t = deserialize(header, frames, deserializers=("cuda", "dask", "pickle"))
+
+    assert header["is-collection"] is True
+    sub_headers = header["sub-headers"]
+    assert sub_headers[0]["serializer"] == "cuda"
+    assert sub_headers[1]["serializer"] == df2_serializer
+    assert isinstance(t, collection)
+
+    assert_eq(t["df1"] if isinstance(t, dict) else t[0], df1)
+    if df2 is None:
+        assert (t["df2"] if isinstance(t, dict) else t[1]) is None
+    else:
+        assert_eq(t["df2"] if isinstance(t, dict) else t[1], df2)
diff --git a/distributed/protocol/tests/test_serialize.py b/distributed/protocol/tests/test_serialize.py
index 09297793fc3..6ba70f676a1 100644
--- a/distributed/protocol/tests/test_serialize.py
+++ b/distributed/protocol/tests/test_serialize.py
@@ -46,7 +46,7 @@ def deserialize_myobj(header, frames):
 
 
 def test_dumps_serialize():
-    for x in [123, [1, 2, 3]]:
+    for x in [123, [1, 2, 3, 4, 5, 6]]:
         header, frames = serialize(x)
         assert header["serializer"] == "pickle"
         assert len(frames) == 1
@@ -235,7 +235,7 @@ def __getstate__(self):
 
 
 def test_errors():
-    msg = {"data": {"foo": to_serialize(inc)}}
+    msg = {"data": {"foo": to_serialize(inc)}, "a": 1, "b": 2, "c": 3, "d": 4, "e": 5}
 
     header, frames = serialize(msg, serializers=["msgpack", "pickle"])
     assert header["serializer"] == "pickle"
diff --git a/distributed/tests/test_publish.py b/distributed/tests/test_publish.py
index 7c0fd0db6d2..32b2974a738 100644
--- a/distributed/tests/test_publish.py
+++ b/distributed/tests/test_publish.py
@@ -228,7 +228,7 @@ def test_pickle_safe(c, s, a, b):
     try:
         yield c2.publish_dataset(x=[1, 2, 3])
         result = yield c2.get_dataset("x")
-        assert result == (1, 2, 3)
+        assert result == [1, 2, 3]
 
         with pytest.raises(TypeError):
             yield c2.publish_dataset(y=lambda x: x)

From 3a2c83534cbacab0ebc6215e9bf5c85a3574255d Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 16 Aug 2019 16:53:29 -0400
Subject: [PATCH 0419/1550] Except all exceptions when checking pynvml (#2961)

pynvml uses a home-grown exception for this
---
 distributed/worker.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/worker.py b/distributed/worker.py
index 17d56aec79b..3dafe1e14df 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -3397,7 +3397,7 @@ async def run(server, comm, function, args=(), kwargs={}, is_coro=None, wait=Tru
 
 try:
     from .diagnostics import nvml
-except ImportError:
+except Exception:
     pass
 else:
 

From e16837c63c35e05764946563e6a397e3ab9597e8 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Fri, 16 Aug 2019 16:59:57 -0500
Subject: [PATCH 0420/1550] bump version to 2.3.0

---
 docs/source/changelog.rst | 34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index b7037d9d6c2..e6c75875765 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,39 @@
 Changelog
 =========
 
+2.3.0 - 2019-08-16
+------------------
+
+- Except all exceptions when checking ``pynvml`` (:pr:`2961`) `Matthew Rocklin`_
+- Pass serialization down through small base collections (:pr:`2948`) `Peter Andreas Entschev`_
+- Use ``pytest.warning(Warning)`` rather than ``Exception`` (:pr:`2958`) `Matthew Rocklin`_
+- Allow ``server_kwargs`` to override defaults in dashboard (:pr:`2955`) `Bruce Merry`_
+- Update ``utils_perf.py`` (:pr:`2954`) `Shayan Amani`_
+- Normalize names with ``str`` in ``retire_workers`` (:pr:`2949`) `Matthew Rocklin`_
+- Update ``client.py`` (:pr:`2951`) `Shayan Amani`_
+- Add ``GPUCurrentLoad`` dashboard plots (:pr:`2944`) `Matthew Rocklin`_
+- Pass GPU diagnostics from worker to scheduler (:pr:`2932`) `Matthew Rocklin`_
+- Import from ``collections.abc`` (:pr:`2938`) `Jim Crist`_
+- Fixes Worker docstring formatting (:pr:`2939`) `James Bourbeau`_
+- Redirect setup docs to docs.dask.org (:pr:`2936`) `Matthew Rocklin`_
+- Wrap offload in ``gen.coroutine`` (:pr:`2934`) `Matthew Rocklin`_
+- Change ``TCP.close`` to a coroutine to avoid task pending warning (:pr:`2930`) `Matthew Rocklin`_
+- Fixup black string normalization (:pr:`2929`) `Jim Crist`_
+- Move core functionality from ``SpecCluster`` to ``Cluster`` (:pr:`2913`) `Matthew Rocklin`_
+- Add aenter/aexit protocols to ``ProcessInterface`` (:pr:`2927`) `Matthew Rocklin`_
+- Add real-time CPU utilization plot to dashboard (:pr:`2922`) `Matthew Rocklin`_
+- Always kill processes in clean tests, even if we don't check (:pr:`2924`) `Matthew Rocklin`_
+- Add timeouts to processes in SSH tests (:pr:`2925`) `Matthew Rocklin`_
+- Add documentation around ``spec.ProcessInterface`` (:pr:`2923`) `Matthew Rocklin`_
+- Cleanup async warnings in tests (:pr:`2920`) `Matthew Rocklin`_
+- Give 404 when requesting nonexistent tasks or workers (:pr:`2921`) `Martin Durant`_
+- Raise informative warning when rescheduling an unknown task (:pr:`2916`) `James Bourbeau`_
+- Fix docstring (:pr:`2917`) `Martin Durant`_
+- Add keep-alive message between worker and scheduler (:pr:`2907`) `Matthew Rocklin`_
+- Rewrite ``Adaptive``/``SpecCluster`` to support slowly arriving workers (:pr:`2904`) `Matthew Rocklin`_
+- Call heartbeat rather than reconnect on disconnection (:pr:`2906`) `Matthew Rocklin`_
+
+
 2.2.0 - 2019-07-31
 ------------------
 
@@ -1173,3 +1206,4 @@ significantly without many new features.
 .. _`Christian Hudon`: https://github.com/chrish42
 .. _`Gabriel Sailer`: https://github.com/sublinus
 .. _`Pierre Glaser`: https://github.com/pierreglase
+.. _`Shayan Amani`: https://github.com/SHi-ON

From 31e775447a5642644cb5f5d9001b56000c4a0536 Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Sat, 17 Aug 2019 16:09:51 +0100
Subject: [PATCH 0421/1550] Add support for separate external address for
 SpecCluster scheduler (#2963)

---
 distributed/deploy/spec.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index 70a413fe1c0..feae250512b 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -23,6 +23,7 @@ class ProcessInterface:
 
     def __init__(self):
         self.address = None
+        self.external_address = None
         self.lock = asyncio.Lock()
         self.status = "created"
 
@@ -225,7 +226,7 @@ async def _start(self):
         self.status = "starting"
         self.scheduler = await self.scheduler
         self.scheduler_comm = rpc(
-            self.scheduler.address,
+            getattr(self.scheduler, "external_address", None) or self.scheduler.address,
             connection_args=self.security.get_connection_args("client"),
         )
         await super()._start()

From 2bff61d9bee59e0bf655937922d9d4c37e49820a Mon Sep 17 00:00:00 2001
From: Benjamin Zaitlen <quasiben@users.noreply.github.com>
Date: Sun, 18 Aug 2019 09:45:36 -0400
Subject: [PATCH 0422/1550] Defer cudf serialization/deserialization to that
 library (#2881)

Fixes #2830

Also log errors in UCX comm
---
 distributed/comm/tests/test_ucx.py | 32 ++++++++--
 distributed/comm/ucx.py            | 98 +++++++++++++++---------------
 distributed/protocol/cuda.py       |  2 -
 distributed/protocol/cudf.py       | 82 +++++--------------------
 4 files changed, 91 insertions(+), 123 deletions(-)

diff --git a/distributed/comm/tests/test_ucx.py b/distributed/comm/tests/test_ucx.py
index 4bb4a341552..afc0eee0676 100644
--- a/distributed/comm/tests/test_ucx.py
+++ b/distributed/comm/tests/test_ucx.py
@@ -1,5 +1,4 @@
 import asyncio
-
 import pytest
 
 ucp = pytest.importorskip("ucp")
@@ -10,6 +9,7 @@
 from distributed.comm import ucx, parse_address
 from distributed.protocol import to_serialize
 from distributed.deploy.local import LocalCluster
+from dask.dataframe.utils import assert_eq
 from distributed.utils_test import gen_test, loop, inc  # noqa: 401
 
 from .test_comms import check_deserialize
@@ -35,7 +35,7 @@ async def handle_comm(comm):
     # Workaround for hanging test in
     # pytest distributed/comm/tests/test_ucx.py::test_comm_objs -vs --count=2
     # on the second time through.
-    ucp._libs.ucp_py.reader_added = 0
+    # ucp._libs.ucp_py.reader_added = 0
 
     listener = listen(listen_addr, handle_comm, connection_args=listen_args, **kwargs)
     with listener:
@@ -164,21 +164,41 @@ def test_ucx_deserialize():
 
 
 @pytest.mark.asyncio
-async def test_ping_pong_cudf():
+@pytest.mark.parametrize(
+    "g",
+    [
+        lambda cudf: cudf.Series([1, 2, 3]),
+        lambda cudf: cudf.Series([]),
+        lambda cudf: cudf.DataFrame([]),
+        lambda cudf: cudf.DataFrame([1]).head(0),
+        lambda cudf: cudf.DataFrame([1.0]).head(0),
+        lambda cudf: cudf.DataFrame({"a": []}),
+        lambda cudf: cudf.DataFrame({"a": ["a"]}).head(0),
+        lambda cudf: cudf.DataFrame({"a": [1.0]}).head(0),
+        lambda cudf: cudf.DataFrame({"a": [1]}).head(0),
+        lambda cudf: cudf.DataFrame({"a": [1, 2, None], "b": [1.0, 2.0, None]}),
+    ],
+)
+async def test_ping_pong_cudf(g):
     # if this test appears after cupy an import error arises
     # *** ImportError: /usr/lib/x86_64-linux-gnu/libstdc++.so.6: version `CXXABI_1.3.11'
     # not found (required by python3.7/site-packages/pyarrow/../../../libarrow.so.12)
     cudf = pytest.importorskip("cudf")
 
-    df = cudf.DataFrame({"A": [1, 2, None], "B": [1.0, 2.0, None]})
+    cudf_obj = g(cudf)
 
     com, serv_com = await get_comm_pair()
-    msg = {"op": "ping", "data": to_serialize(df)}
+    msg = {"op": "ping", "data": to_serialize(cudf_obj)}
 
     await com.write(msg)
     result = await serv_com.read()
-    data2 = result.pop("data")
+
+    cudf_obj_2 = result.pop("data")
     assert result["op"] == "ping"
+    assert_eq(cudf_obj, cudf_obj_2)
+
+    await com.close()
+    await serv_com.close()
 
 
 @pytest.mark.asyncio
diff --git a/distributed/comm/ucx.py b/distributed/comm/ucx.py
index eb1c7514133..434c16c35ee 100644
--- a/distributed/comm/ucx.py
+++ b/distributed/comm/ucx.py
@@ -13,7 +13,7 @@
 from .core import Comm, Connector, Listener, CommClosedError
 from .registry import Backend, backends
 from .utils import ensure_concrete_host, to_frames, from_frames
-from ..utils import ensure_ip, get_ip, get_ipv6, nbytes
+from ..utils import ensure_ip, get_ip, get_ipv6, nbytes, log_errors
 
 import ucp
 
@@ -94,60 +94,62 @@ async def write(
         serializers=("cuda", "dask", "pickle", "error"),
         on_error: str = "message",
     ):
-        if serializers is None:
-            serializers = ("cuda", "dask", "pickle", "error")
-        # msg can also be a list of dicts when sending batched messages
-        frames = await to_frames(msg, serializers=serializers, on_error=on_error)
-        is_gpus = b"".join(
-            [
-                struct.pack("?", hasattr(frame, "__cuda_array_interface__"))
-                for frame in frames
-            ]
-        )
-        sizes = b"".join([struct.pack("Q", nbytes(frame)) for frame in frames])
+        with log_errors():
+            if serializers is None:
+                serializers = ("cuda", "dask", "pickle", "error")
+            # msg can also be a list of dicts when sending batched messages
+            frames = await to_frames(msg, serializers=serializers, on_error=on_error)
+            is_gpus = b"".join(
+                [
+                    struct.pack("?", hasattr(frame, "__cuda_array_interface__"))
+                    for frame in frames
+                ]
+            )
+            sizes = b"".join([struct.pack("Q", nbytes(frame)) for frame in frames])
 
-        nframes = struct.pack("Q", len(frames))
+            nframes = struct.pack("Q", len(frames))
 
-        meta = b"".join([nframes, is_gpus, sizes])
+            meta = b"".join([nframes, is_gpus, sizes])
 
-        await self.ep.send_obj(meta)
+            await self.ep.send_obj(meta)
 
-        for frame in frames:
-            await self.ep.send_obj(frame)
-        return sum(map(nbytes, frames))
+            for frame in frames:
+                await self.ep.send_obj(frame)
+            return sum(map(nbytes, frames))
 
     async def read(self, deserializers=("cuda", "dask", "pickle", "error")):
-        if deserializers is None:
-            deserializers = ("cuda", "dask", "pickle", "error")
-        resp = await self.ep.recv_future()
-        obj = ucp.get_obj_from_msg(resp)
-        (nframes,) = struct.unpack(
-            "Q", obj[:8]
-        )  # first eight bytes for number of frames
-
-        gpu_frame_msg = obj[
-            8 : 8 + nframes
-        ]  # next nframes bytes for if they're GPU frames
-        is_gpus = struct.unpack("{}?".format(nframes), gpu_frame_msg)
-
-        sized_frame_msg = obj[8 + nframes :]  # then the rest for frame sizes
-        sizes = struct.unpack("{}Q".format(nframes), sized_frame_msg)
-
-        frames = []
-
-        for i, (is_gpu, size) in enumerate(zip(is_gpus, sizes)):
-            if size > 0:
-                resp = await self.ep.recv_obj(size, cuda=is_gpu)
-            else:
-                resp = await self.ep.recv_future()
-            frame = ucp.get_obj_from_msg(resp)
-            frames.append(frame)
-
-        msg = await from_frames(
-            frames, deserialize=self.deserialize, deserializers=deserializers
-        )
+        with log_errors():
+            if deserializers is None:
+                deserializers = ("cuda", "dask", "pickle", "error")
+            resp = await self.ep.recv_future()
+            obj = ucp.get_obj_from_msg(resp)
+            (nframes,) = struct.unpack(
+                "Q", obj[:8]
+            )  # first eight bytes for number of frames
+
+            gpu_frame_msg = obj[
+                8 : 8 + nframes
+            ]  # next nframes bytes for if they're GPU frames
+            is_gpus = struct.unpack("{}?".format(nframes), gpu_frame_msg)
+
+            sized_frame_msg = obj[8 + nframes :]  # then the rest for frame sizes
+            sizes = struct.unpack("{}Q".format(nframes), sized_frame_msg)
+
+            frames = []
+
+            for i, (is_gpu, size) in enumerate(zip(is_gpus, sizes)):
+                if size > 0:
+                    resp = await self.ep.recv_obj(size, cuda=is_gpu)
+                else:
+                    resp = await self.ep.recv_future()
+                frame = ucp.get_obj_from_msg(resp)
+                frames.append(frame)
+
+            msg = await from_frames(
+                frames, deserialize=self.deserialize, deserializers=deserializers
+            )
 
-        return msg
+            return msg
 
     def abort(self):
         if self._ep:
diff --git a/distributed/protocol/cuda.py b/distributed/protocol/cuda.py
index 13be1d75bb8..51cb3ea42fa 100644
--- a/distributed/protocol/cuda.py
+++ b/distributed/protocol/cuda.py
@@ -16,8 +16,6 @@ def cuda_dumps(x):
         raise NotImplementedError(type_name)
 
     header, frames = dumps(x)
-
-    header["type"] = type_name
     header["type-serialized"] = pickle.dumps(type(x))
     header["serializer"] = "cuda"
     header["compression"] = (None,) * len(frames)  # no compression for gpu data
diff --git a/distributed/protocol/cudf.py b/distributed/protocol/cudf.py
index 018596b1560..e072570fe58 100644
--- a/distributed/protocol/cudf.py
+++ b/distributed/protocol/cudf.py
@@ -1,74 +1,22 @@
+import pickle
 import cudf
+import cudf.groupby.groupby
 from .cuda import cuda_serialize, cuda_deserialize
-from .numba import serialize_numba_ndarray, deserialize_numba_ndarray
+from ..utils import log_errors
 
-
-# TODO:
-# 1. Just use positions
-#    a. Fixes duplicate columns
-#    b. Fixes non-msgpack-serializable names
-# 2. cudf.Series
-# 3. Serialize the index
-
-
-@cuda_serialize.register(cudf.DataFrame)
+# all (de-)serializtion code lives in the cudf codebase
+# here we ammend the returned headers with `is_gpu` for
+# UCX buffer consumption
+@cuda_serialize.register((cudf.DataFrame, cudf.Series, cudf.groupby.groupby._Groupby))
 def serialize_cudf_dataframe(x):
-    sub_headers = []
-    arrays = []
-    null_masks = []
-    null_headers = []
-    null_counts = {}
-
-    for label, col in x.iteritems():
-        header, [frame] = serialize_numba_ndarray(col.data.mem)
-        header["name"] = label
-        sub_headers.append(header)
-        arrays.append(frame)
-        if col.null_count:
-            header, [frame] = serialize_numba_ndarray(col.nullmask.mem)
-            header["name"] = label
-            null_headers.append(header)
-            null_masks.append(frame)
-            null_counts[label] = col.null_count
-
-    arrays.extend(null_masks)
-
-    header = {
-        "is_cuda": len(arrays),
-        "subheaders": sub_headers,
-        # TODO: the header must be msgpack (de)serializable.
-        # See if we can avoid names, and just use integer positions.
-        "columns": x.columns.tolist(),
-        "null_counts": null_counts,
-        "null_subheaders": null_headers,
-    }
+    with log_errors():
+        header, frames = x.serialize()
+        return header, frames
 
-    return header, arrays
 
-
-@cuda_deserialize.register(cudf.DataFrame)
+@cuda_deserialize.register((cudf.DataFrame, cudf.Series, cudf.groupby.groupby._Groupby))
 def serialize_cudf_dataframe(header, frames):
-    columns = header["columns"]
-    n_columns = len(header["columns"])
-    n_masks = len(header["null_subheaders"])
-
-    masks = {}
-    pairs = []
-
-    for i in range(n_masks):
-        subheader = header["null_subheaders"][i]
-        frame = frames[n_columns + i]
-        mask = deserialize_numba_ndarray(subheader, [frame])
-        masks[subheader["name"]] = mask
-
-    for subheader, frame in zip(header["subheaders"], frames[:n_columns]):
-        name = subheader["name"]
-        array = deserialize_numba_ndarray(subheader, [frame])
-
-        if name in masks:
-            series = cudf.Series.from_masked_array(array, masks[name])
-        else:
-            series = cudf.Series(array)
-        pairs.append((name, series))
-
-    return cudf.DataFrame(pairs)
+    with log_errors():
+        cudf_typ = pickle.loads(header["type"])
+        cudf_obj = cudf_typ.deserialize(header, frames)
+        return cudf_obj

From b083b10d64763b38e559096127d6e3e0c0638c31 Mon Sep 17 00:00:00 2001
From: "Mads R. B. Kristensen" <madsbk@gmail.com>
Date: Tue, 20 Aug 2019 19:39:33 +0200
Subject: [PATCH 0423/1550] Workaround for hanging test now calls ucp.fin()
 (#2967)

---
 distributed/comm/tests/test_ucx.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/comm/tests/test_ucx.py b/distributed/comm/tests/test_ucx.py
index afc0eee0676..1355daf95b8 100644
--- a/distributed/comm/tests/test_ucx.py
+++ b/distributed/comm/tests/test_ucx.py
@@ -35,7 +35,7 @@ async def handle_comm(comm):
     # Workaround for hanging test in
     # pytest distributed/comm/tests/test_ucx.py::test_comm_objs -vs --count=2
     # on the second time through.
-    # ucp._libs.ucp_py.reader_added = 0
+    ucp._libs.ucp_py.fin()
 
     listener = listen(listen_addr, handle_comm, connection_args=listen_args, **kwargs)
     with listener:

From de2d529fe113f61979b47f1bc913f9791123c257 Mon Sep 17 00:00:00 2001
From: Pav A <rootsumsquared@gmail.com>
Date: Wed, 21 Aug 2019 19:05:10 +0100
Subject: [PATCH 0424/1550] [DOC] Remove unnecessary bullet point (#2972)

---
 docs/source/limitations.rst | 1 -
 1 file changed, 1 deletion(-)

diff --git a/docs/source/limitations.rst b/docs/source/limitations.rst
index 3c64d4458f2..e272359e701 100644
--- a/docs/source/limitations.rst
+++ b/docs/source/limitations.rst
@@ -35,7 +35,6 @@ Dask assumes the following about your functions and your data:
 -  Dask may run your functions multiple times,
    such as if a worker holding an intermediate result dies.  Any side effects
    should be `idempotent <https://en.wikipedia.org/wiki/Idempotence>`_.
--
 
 Security
 --------

From 6e0fecfe1bccdebd4db8bde71f5058d5534f3bea Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 21 Aug 2019 14:14:08 -0700
Subject: [PATCH 0425/1550] Directly import progress from
 diagnostics.progressbar (#2975)

For some reason the implicit import behavior has changed.

Fixes #2973
---
 distributed/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/__init__.py b/distributed/__init__.py
index ca36613c815..d79993dfef7 100644
--- a/distributed/__init__.py
+++ b/distributed/__init__.py
@@ -3,7 +3,7 @@
 from .actor import Actor, ActorFuture
 from .core import connect, rpc
 from .deploy import LocalCluster, Adaptive, SpecCluster
-from .diagnostics import progress
+from .diagnostics.progressbar import progress
 from .client import (
     Client,
     Executor,

From 4ae027155af116048440f08327a290863b3f5e0e Mon Sep 17 00:00:00 2001
From: Tom Augspurger <TomAugspurger@users.noreply.github.com>
Date: Wed, 21 Aug 2019 18:42:09 -0500
Subject: [PATCH 0426/1550] Handle buffer protocol objects in ensure_bytes
 (#2969)

---
 distributed/protocol/tests/test_arrow.py | 13 +++++++--
 distributed/tests/test_utils.py          | 15 +++++++++-
 distributed/utils.py                     | 37 +++++++++++++++++++-----
 3 files changed, 54 insertions(+), 11 deletions(-)

diff --git a/distributed/protocol/tests/test_arrow.py b/distributed/protocol/tests/test_arrow.py
index a363ee9511e..37aff3a2644 100644
--- a/distributed/protocol/tests/test_arrow.py
+++ b/distributed/protocol/tests/test_arrow.py
@@ -3,9 +3,9 @@
 
 pa = pytest.importorskip("pyarrow")
 
+import distributed
 from distributed.utils_test import gen_cluster
-from distributed.protocol import deserialize, serialize
-
+from distributed.protocol import deserialize, serialize, to_serialize
 
 df = pd.DataFrame({"A": list("abc"), "B": [1, 2, 3]})
 tbl = pa.Table.from_pandas(df, preserve_index=False)
@@ -35,3 +35,12 @@ def run_test(client, scheduler, worker1, worker2):
         assert obj.equals(result)
 
     run_test()
+
+
+def test_dumps_compression():
+    # https://github.com/dask/distributed/issues/2966
+    # large enough to trigger compression
+    t = pa.Table.from_pandas(pd.DataFrame({"A": [1] * 10000}))
+    msg = {"op": "update", "data": to_serialize(t)}
+    result = distributed.protocol.loads(distributed.protocol.dumps(msg))
+    assert result["data"].equals(t)
diff --git a/distributed/tests/test_utils.py b/distributed/tests/test_utils.py
index e5e18eb393c..81541686baf 100644
--- a/distributed/tests/test_utils.py
+++ b/distributed/tests/test_utils.py
@@ -1,3 +1,4 @@
+import array
 import datetime
 from functools import partial
 import io
@@ -275,13 +276,25 @@ def f():
 
 
 def test_ensure_bytes():
-    data = [b"1", "1", memoryview(b"1"), bytearray(b"1")]
+    data = [b"1", "1", memoryview(b"1"), bytearray(b"1"), array.array("b", [49])]
     for d in data:
         result = ensure_bytes(d)
         assert isinstance(result, bytes)
         assert result == b"1"
 
 
+def test_ensure_bytes_ndarray():
+    result = ensure_bytes(np.arange(12))
+    assert isinstance(result, bytes)
+
+
+def test_ensure_bytes_pyarrow_buffer():
+    pa = pytest.importorskip("pyarrow")
+    buf = pa.py_buffer(b"123")
+    result = ensure_bytes(buf)
+    assert isinstance(result, bytes)
+
+
 def test_nbytes():
     def check(obj, expected):
         assert nbytes(obj) == expected
diff --git a/distributed/utils.py b/distributed/utils.py
index c8ea8d648eb..65f5c188257 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -904,22 +904,43 @@ def tmpfile(extension=""):
 
 
 def ensure_bytes(s):
-    """ Turn string or bytes to bytes
+    """Attempt to turn `s` into bytes.
+
+    Parameters
+    ----------
+    s : Any
+        The object to be converted. Will correctly handled
+
+        * str
+        * bytes
+        * objects implementing the buffer protocol (memoryview, ndarray, etc.)
+
+    Returns
+    -------
+    b : bytes
+
+    Raises
+    ------
+    TypeError
+        When `s` cannot be converted
+
+    Examples
+    --------
 
     >>> ensure_bytes('123')
     b'123'
     >>> ensure_bytes(b'123')
     b'123'
     """
-    if isinstance(s, bytes):
-        return s
-    if isinstance(s, memoryview):
-        return s.tobytes()
-    if isinstance(s, bytearray):  # noqa: F821
-        return bytes(s)
     if hasattr(s, "encode"):
         return s.encode()
-    raise TypeError("Object %s is neither a bytes object nor has an encode method" % s)
+    else:
+        try:
+            return bytes(s)
+        except Exception as e:
+            raise TypeError(
+                "Object %s is neither a bytes object nor has an encode method" % s
+            ) from e
 
 
 def divide_n_among_bins(n, bins):

From 88fd0d23b1f1f89348e98baa62f6ddc30b75ab09 Mon Sep 17 00:00:00 2001
From: Pav A <rootsumsquared@gmail.com>
Date: Thu, 22 Aug 2019 00:43:03 +0100
Subject: [PATCH 0427/1550] Fix documentatation syntax and tree (#2981)

---
 docs/source/client.rst | 2 +-
 docs/source/conf.py    | 1 +
 docs/source/ec2.rst    | 1 -
 docs/source/index.rst  | 3 ++-
 docs/source/worker.rst | 2 +-
 5 files changed, 5 insertions(+), 4 deletions(-)
 delete mode 100644 docs/source/ec2.rst

diff --git a/docs/source/client.rst b/docs/source/client.rst
index b955d5ab504..444b651681e 100644
--- a/docs/source/client.rst
+++ b/docs/source/client.rst
@@ -3,7 +3,7 @@ Client
 
 The Client is the primary entry point for users of ``dask.distributed``.
 
-After we :doc:`setup a cluster <setup>`, we initialize a ``Client`` by pointing
+After we `setup a cluster <https://docs.dask.org/en/latest/setup.html>`_, we initialize a ``Client`` by pointing
 it to the address of a ``Scheduler``:
 
 .. code-block:: python
diff --git a/docs/source/conf.py b/docs/source/conf.py
index afa33400fdc..bb3361851b8 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -391,6 +391,7 @@
     # old html, new html
     ("joblib.html", "https://ml.dask.org/joblib.html"),
     ("setup.html", "https://docs.dask.org/en/latest/setup.html"),
+    ("ec2.html", "https://dask.pydata.org/en/latest/setup/cloud.html"),
 ]
 
 
diff --git a/docs/source/ec2.rst b/docs/source/ec2.rst
deleted file mode 100644
index 71747ba28dc..00000000000
--- a/docs/source/ec2.rst
+++ /dev/null
@@ -1 +0,0 @@
-See `Dask's cloud deployment documentation <https://dask.pydata.org/en/latest/setup/cloud.html>`_ for up-to-date documentation for deployment on Amazon's Cloud.
diff --git a/docs/source/index.rst b/docs/source/index.rst
index 09257be3f58..732c234a53b 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -28,7 +28,7 @@ In particular it meets the following needs:
     Python standard library.  Compatible with `dask`_ API for parallel
     algorithms
 *   **Easy Setup:** As a Pure Python package distributed is ``pip`` installable
-    and easy to :doc:`set up <https://docs.dask.org/en/latest/setup.html>`_ on your own cluster.
+    and easy to `set up <https://docs.dask.org/en/latest/setup.html>`_ on your own cluster.
 
 .. _`concurrent.futures`: https://www.python.org/dev/peps/pep-3148/
 .. _`dask`: https://dask.org
@@ -80,6 +80,7 @@ Contents
    Setup <https://docs.dask.org/en/latest/setup.html>
    client
    api
+   examples-overview
    faq
 
 .. toctree::
diff --git a/docs/source/worker.rst b/docs/source/worker.rst
index be288ccf68c..5ff66b613a6 100644
--- a/docs/source/worker.rst
+++ b/docs/source/worker.rst
@@ -239,7 +239,7 @@ Nanny
 Dask workers are by default launched, monitored, and managed by a small Nanny
 process.
 
-.. autoclass:: distributed.worker.Nanny
+.. autoclass:: distributed.nanny.Nanny
 
 
 API Documentation

From 58b3abe6d2e5b9898344ba87f93c20648247507e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lo=C3=AFc=20Est=C3=A8ve?= <loic.esteve@ymail.com>
Date: Thu, 22 Aug 2019 01:45:29 +0200
Subject: [PATCH 0428/1550] Improve get_ip_interface error message when
 interface does not exist (#2964)

---
 distributed/tests/test_utils.py | 12 ++++++++++--
 distributed/utils.py            | 11 ++++++++++-
 2 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/distributed/tests/test_utils.py b/distributed/tests/test_utils.py
index 81541686baf..bf2d8456681 100644
--- a/distributed/tests/test_utils.py
+++ b/distributed/tests/test_utils.py
@@ -150,8 +150,16 @@ def test_get_ip_interface():
         assert get_ip_interface("lo") == "127.0.0.1"
     else:
         pytest.skip("test needs to be enhanced for platform %r" % (sys.platform,))
-    with pytest.raises(KeyError):
-        get_ip_interface("__non-existent-interface")
+
+    non_existent_interface = "__non-existent-interface"
+    expected_error_message = "{!r}.+network interface.+".format(non_existent_interface)
+
+    if sys.platform == "darwin":
+        expected_error_message += "'lo0'"
+    elif sys.platform.startswith("linux"):
+        expected_error_message += "'lo'"
+    with pytest.raises(ValueError, match=expected_error_message):
+        get_ip_interface(non_existent_interface)
 
 
 def test_truncate_exception():
diff --git a/distributed/utils.py b/distributed/utils.py
index 65f5c188257..b83a2c1f2cf 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -169,7 +169,16 @@ def get_ip_interface(ifname):
     """
     import psutil
 
-    for info in psutil.net_if_addrs()[ifname]:
+    net_if_addrs = psutil.net_if_addrs()
+
+    if ifname not in net_if_addrs:
+        allowed_ifnames = list(net_if_addrs.keys())
+        raise ValueError(
+            "{!r} is not a valid network interface. "
+            "Valid network interfaces are: {}".format(ifname, allowed_ifnames)
+        )
+
+    for info in net_if_addrs[ifname]:
         if info.family == socket.AF_INET:
             return info.address
     raise ValueError("interface %r doesn't have an IPv4 address" % (ifname,))

From 799b5a0a0c95dc757a69e440e1d70a5e3afdd56f Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 22 Aug 2019 05:45:37 -0700
Subject: [PATCH 0429/1550] Add cores= and memory= keywords to scale (#2974)

Dask-Jobqueue did this internally.
It seems like a decent idea to pull upstream.
---
 distributed/deploy/spec.py             | 25 +++++++++++++++++++++++--
 distributed/deploy/tests/test_local.py | 22 ++++++++++++++++++++++
 distributed/protocol/cudf.py           |  1 +
 3 files changed, 46 insertions(+), 2 deletions(-)

diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index feae250512b..22fa6692d65 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -1,13 +1,14 @@
 import asyncio
 import atexit
 import copy
+import math
 import weakref
 
 from tornado import gen
 
 from .cluster import Cluster
 from ..core import rpc, CommClosedError
-from ..utils import LoopRunner, silence_logging, ignoring
+from ..utils import LoopRunner, silence_logging, ignoring, parse_bytes
 from ..scheduler import Scheduler
 from ..security import Security
 
@@ -330,7 +331,27 @@ def __exit__(self, typ, value, traceback):
         self.close()
         self._loop_runner.stop()
 
-    def scale(self, n):
+    def scale(self, n=0, memory=None, cores=None):
+        if memory is not None:
+            try:
+                limit = self.new_spec["options"]["memory_limit"]
+            except KeyError:
+                raise ValueError(
+                    "to use scale(memory=...) your worker definition must include a memory_limit definition"
+                )
+            else:
+                n = max(n, int(math.ceil(parse_bytes(memory) / parse_bytes(limit))))
+
+        if cores is not None:
+            try:
+                threads_per_worker = self.new_spec["options"]["nthreads"]
+            except KeyError:
+                raise ValueError(
+                    "to use scale(cores=...) your worker definition must include an nthreads= definition"
+                )
+            else:
+                n = max(n, int(math.ceil(cores / threads_per_worker)))
+
         if len(self.worker_spec) > n:
             not_yet_launched = set(self.worker_spec) - {
                 v["name"] for v in self.scheduler_info["workers"].values()
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index 31493967d4c..ad00e908d61 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -861,3 +861,25 @@ def test_client_cluster_synchronous(loop):
         with Client(loop=loop, processes=False) as c:
             assert not c.asynchronous
             assert not c.cluster.asynchronous
+
+
+@pytest.mark.asyncio
+async def test_scale_memory_cores(cleanup):
+    async with LocalCluster(
+        n_workers=0,
+        processes=False,
+        threads_per_worker=2,
+        memory_limit="2GB",
+        asynchronous=True,
+    ) as cluster:
+        cluster.scale(cores=4)
+        assert len(cluster.worker_spec) == 2
+
+        cluster.scale(memory="6GB")
+        assert len(cluster.worker_spec) == 3
+
+        cluster.scale(cores=1)
+        assert len(cluster.worker_spec) == 1
+
+        cluster.scale(memory="7GB")
+        assert len(cluster.worker_spec) == 4
diff --git a/distributed/protocol/cudf.py b/distributed/protocol/cudf.py
index e072570fe58..985314f3f2e 100644
--- a/distributed/protocol/cudf.py
+++ b/distributed/protocol/cudf.py
@@ -4,6 +4,7 @@
 from .cuda import cuda_serialize, cuda_deserialize
 from ..utils import log_errors
 
+
 # all (de-)serializtion code lives in the cudf codebase
 # here we ammend the returned headers with `is_gpu` for
 # UCX buffer consumption

From edc094348055e27d8ae45fe1ee397317a92b912f Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 22 Aug 2019 09:00:43 -0700
Subject: [PATCH 0430/1550] Make workers robust to bad custom metrics (#2984)

---
 .pre-commit-config.yaml          |  2 +-
 distributed/tests/test_worker.py | 10 ++++++++++
 distributed/worker.py            | 11 +++++++----
 3 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 6be2fcaa3bc..2c72a38ce93 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -5,6 +5,6 @@ repos:
     - id: black
       language_version: python3.7
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v1.2.3
+    rev: v2.3.0
     hooks:
     - id: flake8
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 4dab232487f..3d0844e66e6 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -1548,3 +1548,13 @@ async def test_gpu_metrics(s, a, b):
 
     assert "gpu" in a.startup_information
     assert len(s.workers[a.address].extra["gpu"]["name"]) == count
+
+
+@pytest.mark.asyncio
+async def test_bad_metrics(cleanup):
+    def bad_metric(w):
+        raise Exception("Hello")
+
+    async with Scheduler() as s:
+        async with Worker(s.address, metrics={"bad": bad_metric}) as w:
+            assert "bad" not in s.workers[w.address].metrics
diff --git a/distributed/worker.py b/distributed/worker.py
index 3dafe1e14df..b20f52161fa 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -735,10 +735,13 @@ async def get_metrics(self):
         )
         custom = {}
         for k, metric in self.metrics.items():
-            result = metric(self)
-            if hasattr(result, "__await__"):
-                result = await result
-            custom[k] = result
+            try:
+                result = metric(self)
+                if hasattr(result, "__await__"):
+                    result = await result
+                custom[k] = result
+            except Exception:  # TODO: log error once
+                pass
 
         return merge(custom, self.monitor.recent(), core)
 

From 4e9d5ecb373f03c4e71928c79cbabdfb3f257d26 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 22 Aug 2019 09:39:30 -0700
Subject: [PATCH 0431/1550] bump version to 2.3.1

---
 docs/source/changelog.rst | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index e6c75875765..07e7b8387f5 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,21 @@
 Changelog
 =========
 
+2.3.1 - 2019-08-22
+------------------
+
+-  Add support for separate external address for SpecCluster scheduler (:pr:`2963`) `Jacob Tomlinson`_
+-  Defer cudf serialization/deserialization to that library (:pr:`2881`) `Benjamin Zaitlen`_
+-  Workaround for hanging test now calls ucp.fin() (:pr:`2967`) `Mads R. B. Kristensen`_
+-  Remove unnecessary bullet point (:pr:`2972`) `Pav A`_
+-  Directly import progress from diagnostics.progressbar (:pr:`2975`) `Matthew Rocklin`_
+-  Handle buffer protocol objects in ensure_bytes (:pr:`2969`) `Tom Augspurger`_
+-  Fix documentatation syntax and tree (:pr:`2981`) `Pav A`_
+-  Improve get_ip_interface error message when interface does not exist (:pr:`2964`) `Loïc Estève`_
+-  Add cores= and memory= keywords to scale (:pr:`2974`) `Matthew Rocklin`_
+-  Make workers robust to bad custom metrics (:pr:`2984`) `Matthew Rocklin`_
+
+
 2.3.0 - 2019-08-16
 ------------------
 
@@ -1199,6 +1214,7 @@ significantly without many new features.
 .. _`Michael Spiegel`: https://github.com/Spiegel0
 .. _`Caleb`: https://github.com/calebho
 .. _`Ben Zaitlen`: https://github.com/quasiben
+.. _`Benjamin Zaitlen`: https://github.com/quasiben
 .. _`Manuel Garrido`: https://github.com/manugarri
 .. _`Magnus Nord`: https://github.com/magnunor
 .. _`Sam Grayson`: https://github.com/charmoniumQ
@@ -1207,3 +1223,5 @@ significantly without many new features.
 .. _`Gabriel Sailer`: https://github.com/sublinus
 .. _`Pierre Glaser`: https://github.com/pierreglase
 .. _`Shayan Amani`: https://github.com/SHi-ON
+.. _`Pav A`: https://github.com/rs2
+.. _`Mads R. B. Kristensen`: https://github.com/madsbk

From d3b075e7117a2e9a744641e4a9ea9deaca3681ff Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Fri, 23 Aug 2019 16:43:47 +0100
Subject: [PATCH 0432/1550] Skip exceptions in startup information (#2991)

---
 distributed/tests/test_worker.py | 12 ++++++++++++
 distributed/worker.py            | 12 ++++++++----
 2 files changed, 20 insertions(+), 4 deletions(-)

diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 3d0844e66e6..13dd92c00e0 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -1558,3 +1558,15 @@ def bad_metric(w):
     async with Scheduler() as s:
         async with Worker(s.address, metrics={"bad": bad_metric}) as w:
             assert "bad" not in s.workers[w.address].metrics
+
+
+@pytest.mark.asyncio
+async def test_bad_startup(cleanup):
+    def bad_startup(w):
+        raise Exception("Hello")
+
+    async with Scheduler() as s:
+        try:
+            w = await Worker(s.address, startup_information={"bad": bad_startup})
+        except Exception:
+            pytest.fail("Startup exception was raised")
diff --git a/distributed/worker.py b/distributed/worker.py
index b20f52161fa..ca4f4121af3 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -748,10 +748,14 @@ async def get_metrics(self):
     async def get_startup_information(self):
         result = {}
         for k, f in self.startup_information.items():
-            v = f(self)
-            if hasattr(v, "__await__"):
-                v = await v
-            result[k] = v
+            try:
+                v = f(self)
+                if hasattr(v, "__await__"):
+                    v = await v
+                result[k] = v
+            except Exception:  # TODO: log error once
+                pass
+
         return result
 
     def identity(self, comm=None):

From a0d68d42b81fe0215098d7f87783e2d57b2aa703 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 23 Aug 2019 09:13:17 -0700
Subject: [PATCH 0433/1550] bump version to 2.3.2

---
 docs/source/changelog.rst | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index 07e7b8387f5..7385567467e 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,11 @@
 Changelog
 =========
 
+2.3.2 - 2019-08-23
+------------------
+
+-  Skip exceptions in startup information (:pr:`2991`) `Jacob Tomlinson`_
+
 2.3.1 - 2019-08-22
 ------------------
 

From 512729206255154f511acbb3d054e0439f7e07c6 Mon Sep 17 00:00:00 2001
From: "Richard (Rick) Zamora" <rzamora217@gmail.com>
Date: Fri, 23 Aug 2019 15:47:03 -0500
Subject: [PATCH 0434/1550] Fix PyNVML initialization (#2993)

---
 distributed/diagnostics/nvml.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/distributed/diagnostics/nvml.py b/distributed/diagnostics/nvml.py
index 25a11cde6b0..a96a5547598 100644
--- a/distributed/diagnostics/nvml.py
+++ b/distributed/diagnostics/nvml.py
@@ -1,12 +1,19 @@
 import pynvml
 
-pynvml.nvmlInit()
-count = pynvml.nvmlDeviceGetCount()
+handles = None
 
-handles = [pynvml.nvmlDeviceGetHandleByIndex(i) for i in range(count)]
+
+def _pynvml_handles():
+    global handles
+    if handles is None:
+        pynvml.nvmlInit()
+        count = pynvml.nvmlDeviceGetCount()
+        handles = [pynvml.nvmlDeviceGetHandleByIndex(i) for i in range(count)]
+    return handles
 
 
 def real_time():
+    handles = _pynvml_handles()
     return {
         "utilization": [pynvml.nvmlDeviceGetUtilizationRates(h).gpu for h in handles],
         "memory-used": [pynvml.nvmlDeviceGetMemoryInfo(h).used for h in handles],
@@ -14,6 +21,7 @@ def real_time():
 
 
 def one_time():
+    handles = _pynvml_handles()
     return {
         "memory-total": [pynvml.nvmlDeviceGetMemoryInfo(h).total for h in handles],
         "name": [pynvml.nvmlDeviceGetName(h).decode() for h in handles],

From ad0c7c23c384e981e0622c36e7fb132429becc34 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 26 Aug 2019 09:40:31 -0700
Subject: [PATCH 0435/1550] Add threads= and memory= to Cluster and Client
 reprs (#2995)

---
 distributed/client.py                  |  8 +++++++-
 distributed/deploy/cluster.py          | 22 +++++++++++++++-------
 distributed/deploy/local.py            |  8 --------
 distributed/deploy/spec.py             |  7 -------
 distributed/deploy/tests/test_local.py | 21 +++++++++++++++++++++
 distributed/tests/test_client.py       |  8 ++++++--
 6 files changed, 49 insertions(+), 25 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 9d05cde049e..c11257ee74a 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -761,12 +761,18 @@ def __repr__(self):
             workers = info.get("workers", {})
             nworkers = len(workers)
             nthreads = sum(w["nthreads"] for w in workers.values())
-            return "<%s: scheduler=%r processes=%d cores=%d>" % (
+            text = "<%s: %r processes=%d threads=%d" % (
                 self.__class__.__name__,
                 addr,
                 nworkers,
                 nthreads,
             )
+            memory = [w["memory_limit"] for w in workers.values()]
+            if all(memory):
+                text += ", memory=" + format_bytes(sum(memory))
+            text += ">"
+            return text
+
         elif self.scheduler is not None:
             return "<%s: scheduler=%r>" % (
                 self.__class__.__name__,
diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
index e85ea2bc3dd..0c1a0364405 100644
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -314,13 +314,6 @@ def update():
     def _ipython_display_(self, **kwargs):
         return self._widget()._ipython_display_(**kwargs)
 
-    def __repr__(self):
-        return "%s(%r, workers=%d)" % (
-            type(self).__name__,
-            self.scheduler_address,
-            len(self.scheduler_info["workers"]),
-        )
-
     async def __aenter__(self):
         await self
         return self
@@ -331,3 +324,18 @@ async def __aexit__(self, typ, value, traceback):
     @property
     def scheduler_address(self):
         return self.scheduler_comm.address
+
+    def __repr__(self):
+        text = "%s(%r, workers=%d, threads=%d" % (
+            getattr(self, "_name", type(self).__name__),
+            self.scheduler_address,
+            len(self.workers),
+            sum(w["nthreads"] for w in self.scheduler_info["workers"].values()),
+        )
+
+        memory = [w["memory_limit"] for w in self.scheduler_info["workers"].values()]
+        if all(memory):
+            text += ", memory=" + format_bytes(sum(memory))
+
+        text += ")"
+        return text
diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index 20476ad8065..efe5ed03098 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -201,14 +201,6 @@ def __init__(
             security=security,
         )
 
-    def __repr__(self):
-        return "%s(%r, workers=%d, nthreads=%d)" % (
-            type(self).__name__,
-            self.scheduler_address,
-            len(self.workers),
-            sum(w.nthreads for w in self.workers.values()),
-        )
-
     def start_worker(self, *args, **kwargs):
         raise NotImplementedError(
             "The `cluster.start_worker` function has been removed. "
diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index 22fa6692d65..8ea03a8371d 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -401,13 +401,6 @@ async def scale_down(self, workers):
 
     scale_up = scale  # backwards compatibility
 
-    def __repr__(self):
-        return "%s(%r, workers=%d)" % (
-            self._name,
-            self.scheduler_address,
-            len(self.workers),
-        )
-
 
 @atexit.register
 def close_clusters():
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index ad00e908d61..59c75d545d8 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -883,3 +883,24 @@ async def test_scale_memory_cores(cleanup):
 
         cluster.scale(memory="7GB")
         assert len(cluster.worker_spec) == 4
+
+
+@pytest.mark.asyncio
+async def test_repr(cleanup):
+    async with LocalCluster(
+        n_workers=2,
+        processes=False,
+        threads_per_worker=2,
+        memory_limit="2GB",
+        asynchronous=True,
+    ) as cluster:
+        text = repr(cluster)
+        assert "workers=2" in text
+        assert cluster.scheduler_address in text
+        assert "cores=4" in text or "threads=4" in text
+        assert "GB" in text and "4" in text
+
+    async with LocalCluster(
+        n_workers=2, processes=False, memory_limit=None, asynchronous=True
+    ) as cluster:
+        assert "memory" not in repr(cluster)
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 7f0036e2a7a..a4e472c882a 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -1882,12 +1882,16 @@ def test_badly_serialized_input_stderr(capsys, c):
 
 def test_repr(loop):
     funcs = [str, repr, lambda x: x._repr_html_()]
-    with cluster(nworkers=3) as (s, [a, b, c]):
+    with cluster(nworkers=3, worker_kwargs={"memory_limit": "2 GB"}) as (s, [a, b, c]):
         with Client(s["address"], loop=loop) as c:
             for func in funcs:
                 text = func(c)
                 assert c.scheduler.address in text
                 assert "3" in text
+                assert "6" in text
+                assert "GB" in text
+                if "<table" not in text:
+                    assert len(text) < 80
 
         for func in funcs:
             text = func(c)
@@ -2037,7 +2041,7 @@ def test_repr_sync(c):
     assert c.scheduler.address in s
     assert c.scheduler.address in r
     assert str(2) in s  # nworkers
-    assert "cores" in s
+    assert "cores" in s or "threads" in s
 
 
 @gen_cluster(client=True)

From ca175924b1eb793ca2a217315fa1f76223d1d3bc Mon Sep 17 00:00:00 2001
From: Benjamin Zaitlen <quasiben@users.noreply.github.com>
Date: Mon, 26 Aug 2019 12:41:36 -0400
Subject: [PATCH 0436/1550] Add cuda_ipc to UCX environment for NVLink (#2996)

---
 distributed/comm/ucx.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/comm/ucx.py b/distributed/comm/ucx.py
index 434c16c35ee..8631bb18229 100644
--- a/distributed/comm/ucx.py
+++ b/distributed/comm/ucx.py
@@ -21,7 +21,7 @@
 
 os.environ.setdefault("UCX_RNDV_SCHEME", "put_zcopy")
 os.environ.setdefault("UCX_MEMTYPE_CACHE", "n")
-os.environ.setdefault("UCX_TLS", "rc,cuda_copy")
+os.environ.setdefault("UCX_TLS", "rc,cuda_copy,cuda_ipc")
 
 logger = logging.getLogger(__name__)
 MAX_MSG_LOG = 23

From 71dda4c326cf5ce523a025e7c299e424343a20f2 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 26 Aug 2019 13:39:45 -0700
Subject: [PATCH 0437/1550] Permit more keyword options when scaling with cores
 and memory (#2997)

Some cluster managers use options like nthreads or cores.
We should be robust to a few common choices.
---
 distributed/deploy/spec.py                    | 28 ++++++++++++-------
 distributed/deploy/tests/test_spec_cluster.py | 15 ++++++++++
 2 files changed, 33 insertions(+), 10 deletions(-)

diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index 8ea03a8371d..08a993def97 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -333,24 +333,32 @@ def __exit__(self, typ, value, traceback):
 
     def scale(self, n=0, memory=None, cores=None):
         if memory is not None:
-            try:
-                limit = self.new_spec["options"]["memory_limit"]
-            except KeyError:
+            for name in ["memory_limit", "memory"]:
+                try:
+                    limit = self.new_spec["options"][name]
+                except KeyError:
+                    pass
+                else:
+                    n = max(n, int(math.ceil(parse_bytes(memory) / parse_bytes(limit))))
+                    break
+            else:
                 raise ValueError(
                     "to use scale(memory=...) your worker definition must include a memory_limit definition"
                 )
-            else:
-                n = max(n, int(math.ceil(parse_bytes(memory) / parse_bytes(limit))))
 
         if cores is not None:
-            try:
-                threads_per_worker = self.new_spec["options"]["nthreads"]
-            except KeyError:
+            for name in ["nthreads", "ncores", "threads", "cores"]:
+                try:
+                    threads_per_worker = self.new_spec["options"][name]
+                except KeyError:
+                    pass
+                else:
+                    n = max(n, int(math.ceil(cores / threads_per_worker)))
+                    break
+            else:
                 raise ValueError(
                     "to use scale(cores=...) your worker definition must include an nthreads= definition"
                 )
-            else:
-                n = max(n, int(math.ceil(cores / threads_per_worker)))
 
         if len(self.worker_spec) > n:
             not_yet_launched = set(self.worker_spec) - {
diff --git a/distributed/deploy/tests/test_spec_cluster.py b/distributed/deploy/tests/test_spec_cluster.py
index 64633428a38..27ed20c9f20 100644
--- a/distributed/deploy/tests/test_spec_cluster.py
+++ b/distributed/deploy/tests/test_spec_cluster.py
@@ -268,3 +268,18 @@ async def test_widget(cleanup):
 
         cluster.scale(5)
         assert "3 / 5" in cluster._widget_status()
+
+
+@pytest.mark.asyncio
+async def test_scale_cores_memory(cleanup):
+    async with SpecCluster(
+        scheduler=scheduler,
+        worker={"cls": Worker, "options": {"nthreads": 1}},
+        asynchronous=True,
+    ) as cluster:
+        cluster.scale(cores=2)
+        assert len(cluster.worker_spec) == 2
+        with pytest.raises(ValueError) as info:
+            cluster.scale(memory="5GB")
+
+        assert "memory" in str(info.value)

From 83a844e9144e5310b6c9d140c2c644dcd56b8db1 Mon Sep 17 00:00:00 2001
From: Mohammad Noor <github@mdsalih.com>
Date: Wed, 28 Aug 2019 09:14:26 -0500
Subject: [PATCH 0438/1550] Fix minor typo in documentation (#3002)

---
 docs/source/manage-computation.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/source/manage-computation.rst b/docs/source/manage-computation.rst
index 20c01fe2c24..e4d2d4298d3 100644
--- a/docs/source/manage-computation.rst
+++ b/docs/source/manage-computation.rst
@@ -128,7 +128,7 @@ tasks directly to the cluster with ``client.scatter``, ``client.submit`` or ``cl
 .. code-block:: python
 
    futures = client.scatter(args)                        # Send data
-   future = client.submit(function, *args, **kwrags)     # Send single task
+   future = client.submit(function, *args, **kwargs)     # Send single task
    futures = client.map(function, sequence, **kwargs)    # Send many tasks
 
 In this case ``*args`` or ``**kwargs`` can be normal Python objects, like ``1``

From 52323b910b203d496849f61463c48fca8be92ac3 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 28 Aug 2019 08:29:05 -0700
Subject: [PATCH 0439/1550] Return dictionaries from new_worker_spec rather
 than name/worker pairs (#3000)

This allows for larger collections of workers, such as when launching
many workers in one HPC job.

Fixes https://github.com/dask/distributed/issues/2999
---
 distributed/deploy/spec.py                    | 8 +++-----
 distributed/deploy/tests/test_spec_cluster.py | 2 +-
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index 08a993def97..8b450c6af79 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -375,8 +375,7 @@ def scale(self, n=0, memory=None, cores=None):
             return
 
         while len(self.worker_spec) < n:
-            k, spec = self.new_worker_spec()
-            self.worker_spec[k] = spec
+            self.worker_spec.update(self.new_worker_spec())
 
         self.loop.add_callback(self._correct_state)
 
@@ -385,8 +384,7 @@ def new_worker_spec(self):
 
         Returns
         -------
-        name: identifier for worker
-        spec: dict
+        d: dict mapping names to worker specs
 
         See Also
         --------
@@ -395,7 +393,7 @@ def new_worker_spec(self):
         while self._i in self.worker_spec:
             self._i += 1
 
-        return self._i, self.new_spec
+        return {self._i: self.new_spec}
 
     @property
     def _supports_scaling(self):
diff --git a/distributed/deploy/tests/test_spec_cluster.py b/distributed/deploy/tests/test_spec_cluster.py
index 27ed20c9f20..2debf9dea68 100644
--- a/distributed/deploy/tests/test_spec_cluster.py
+++ b/distributed/deploy/tests/test_spec_cluster.py
@@ -152,7 +152,7 @@ async def test_new_worker_spec(cleanup):
     class MyCluster(SpecCluster):
         def new_worker_spec(self):
             i = len(self.worker_spec)
-            return i, {"cls": Worker, "options": {"nthreads": i + 1}}
+            return {i: {"cls": Worker, "options": {"nthreads": i + 1}}}
 
     async with MyCluster(asynchronous=True, scheduler=scheduler) as cluster:
         cluster.scale(3)

From 7a48850938c006ad16a2f99dfbcf44fa5df3ce00 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 28 Aug 2019 09:31:23 -0700
Subject: [PATCH 0440/1550] Make spec.ProcessInterface a valid no-op worker
 (#3004)

This allows it to be placed into specs as-is as a placeholder for
testing.
---
 distributed/deploy/spec.py                    |  2 +-
 distributed/deploy/tests/test_spec_cluster.py | 14 ++++++++++++++
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index 8b450c6af79..acb7e72368a 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -22,7 +22,7 @@ class ProcessInterface:
     It should implement the methods below, like ``start`` and ``close``
     """
 
-    def __init__(self):
+    def __init__(self, scheduler=None, name=None):
         self.address = None
         self.external_address = None
         self.lock = asyncio.Lock()
diff --git a/distributed/deploy/tests/test_spec_cluster.py b/distributed/deploy/tests/test_spec_cluster.py
index 2debf9dea68..ea0afb0488d 100644
--- a/distributed/deploy/tests/test_spec_cluster.py
+++ b/distributed/deploy/tests/test_spec_cluster.py
@@ -283,3 +283,17 @@ async def test_scale_cores_memory(cleanup):
             cluster.scale(memory="5GB")
 
         assert "memory" in str(info.value)
+
+
+@pytest.mark.asyncio
+async def test_ProcessInterfaceValid(cleanup):
+    async with SpecCluster(
+        scheduler=scheduler, worker={"cls": ProcessInterface}, asynchronous=True
+    ) as cluster:
+        cluster.scale(2)
+        await cluster
+        assert len(cluster.worker_spec) == len(cluster.workers) == 2
+
+        cluster.scale(1)
+        await cluster
+        assert len(cluster.worker_spec) == len(cluster.workers) == 1

From bc6f4a6395cbb28b493ff7984b941aa1e0dc4fd5 Mon Sep 17 00:00:00 2001
From: Benjamin Zaitlen <quasiben@users.noreply.github.com>
Date: Wed, 28 Aug 2019 15:57:03 -0400
Subject: [PATCH 0441/1550] better name for cudf deserialization function name
 (#3008)

---
 distributed/protocol/cudf.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/protocol/cudf.py b/distributed/protocol/cudf.py
index 985314f3f2e..f236a6c1f0c 100644
--- a/distributed/protocol/cudf.py
+++ b/distributed/protocol/cudf.py
@@ -16,7 +16,7 @@ def serialize_cudf_dataframe(x):
 
 
 @cuda_deserialize.register((cudf.DataFrame, cudf.Series, cudf.groupby.groupby._Groupby))
-def serialize_cudf_dataframe(header, frames):
+def deserialize_cudf_dataframe(header, frames):
     with log_errors():
         cudf_typ = pickle.loads(header["type"])
         cudf_obj = cudf_typ.deserialize(header, frames)

From f5f2aa3c313fc5002be8cb2f7f2af64d9043a6e0 Mon Sep 17 00:00:00 2001
From: Jim Crist <jcrist@users.noreply.github.com>
Date: Thu, 29 Aug 2019 08:47:01 -0500
Subject: [PATCH 0442/1550] Tweak `Logs` styling (#3012)

- Not all browsers render unstyled `summary` elements with the "carrot"
indicating they can be expanded. We style these so they always look the
same across browsers.
- We sort log elements when rendering.
---
 distributed/utils.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/distributed/utils.py b/distributed/utils.py
index b83a2c1f2cf..9f14d58de0b 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -1453,12 +1453,13 @@ class Logs(dict):
 
     def _repr_html_(self):
         summaries = [
-            "<details>\n<summary>{title}</summary>\n{log}\n</details>".format(
-                title=title, log=log._repr_html_()
-            )
-            for title, log in self.items()
+            "<details>\n"
+            "<summary style='display:list-item'>{title}</summary>\n"
+            "{log}\n"
+            "</details>".format(title=title, log=log._repr_html_())
+            for title, log in sorted(self.items())
         ]
-        return "\n\n".join(summaries)
+        return "\n".join(summaries)
 
 
 def cli_keywords(d: dict, cls=None):

From cf26e1a559e2c89c4a4b14b6622111eaf0954f12 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 29 Aug 2019 10:04:18 -0700
Subject: [PATCH 0443/1550] Support Spec jobs that generate multiple workers
 (#3013)

Sometimes a single entry in the worker_spec will generate multiple Dask
workers.  We add an entry, "group", to the spec that shows how
worker_spec entries map to dask-workers that connect to the scheduler
---
 distributed/deploy/adaptive.py                |  9 +--
 distributed/deploy/adaptive_core.py           |  2 +-
 distributed/deploy/cluster.py                 | 12 ++++
 distributed/deploy/spec.py                    | 65 +++++++++++++----
 distributed/deploy/tests/test_spec_cluster.py | 69 +++++++++++++++++++
 5 files changed, 138 insertions(+), 19 deletions(-)

diff --git a/distributed/deploy/adaptive.py b/distributed/deploy/adaptive.py
index 2efc18dfe0c..b8c3429a505 100644
--- a/distributed/deploy/adaptive.py
+++ b/distributed/deploy/adaptive.py
@@ -98,18 +98,15 @@ def scheduler(self):
 
     @property
     def plan(self):
-        try:
-            return set(self.cluster.worker_spec)
-        except AttributeError:
-            return set(self.cluster.workers)
+        return self.cluster.plan
 
     @property
     def requested(self):
-        return set(self.cluster.workers)
+        return self.cluster.requested
 
     @property
     def observed(self):
-        return {d["name"] for d in self.cluster.scheduler_info["workers"].values()}
+        return self.cluster.observed
 
     async def target(self):
         return await self.scheduler.adaptive_target(
diff --git a/distributed/deploy/adaptive_core.py b/distributed/deploy/adaptive_core.py
index 6732bb20284..db50f109ce3 100644
--- a/distributed/deploy/adaptive_core.py
+++ b/distributed/deploy/adaptive_core.py
@@ -48,7 +48,7 @@ class AdaptiveCore:
         Scales the cluster up to a target number of workers, presumably
         changing at least ``plan`` and hopefully eventually also ``requested``
 
-   scale_down : Set[worker] -> None
+    scale_down : Set[worker] -> None
         Closes the provided set of workers
 
     Parameters
diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
index 0c1a0364405..5e86c39ce8e 100644
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -339,3 +339,15 @@ def __repr__(self):
 
         text += ")"
         return text
+
+    @property
+    def plan(self):
+        return set(self.workers)
+
+    @property
+    def requested(self):
+        return set(self.workers)
+
+    @property
+    def observed(self):
+        return {d["name"] for d in self.scheduler_info["workers"].values()}
diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index acb7e72368a..d897dc6b7df 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -160,6 +160,23 @@ class does handle all of the logic around asynchronously cleanly setting up
     Also note that uniformity of the specification is not required.  Other API
     could be added externally (in subclasses) that adds workers of different
     specifications into the same dictionary.
+
+    If a single entry in the spec will generate multiple dask workers then
+    please provide a `"group"` element to the spec, that includes the suffixes
+    that will be added to each name (this should be handled by your worker
+    class).
+
+    >>> cluster.worker_spec
+    {
+        0: {"cls": MultiWorker, "options": {"processes": 3}, "group": ["-0", "-1", -2"]}
+        1: {"cls": MultiWorker, "options": {"processes": 2}, "group": ["-0", "-1"]}
+    }
+
+    These suffixes should correspond to the names used by the workers when
+    they deploy.
+
+    >>> [ws.name for ws in cluster.scheduler.workers.values()]
+    ["0-0", "0-1", "0-2", "1-0", "1-1"]
     """
 
     _instances = weakref.WeakSet()
@@ -288,18 +305,6 @@ async def _():
 
         return _().__await__()
 
-    async def _wait_for_workers(self):
-        while {
-            str(d["name"])
-            for d in (await self.scheduler_comm.identity())["workers"].values()
-        } != set(map(str, self.workers)):
-            if (
-                any(w.status == "closed" for w in self.workers.values())
-                and self.scheduler.status == "running"
-            ):
-                raise gen.TimeoutError("Worker unexpectedly closed")
-            await asyncio.sleep(0.1)
-
     async def _close(self):
         while self.status == "closing":
             await asyncio.sleep(0.1)
@@ -400,6 +405,18 @@ def _supports_scaling(self):
         return not not self.new_spec
 
     async def scale_down(self, workers):
+        # We may have groups, if so, map worker addresses to job names
+        if not all(w in self.worker_spec for w in workers):
+            mapping = {}
+            for name, spec in self.worker_spec.items():
+                if "group" in spec:
+                    for suffix in spec["group"]:
+                        mapping[str(name) + suffix] = name
+                else:
+                    mapping[name] = name
+
+            workers = {mapping.get(w, w) for w in workers}
+
         for w in workers:
             if w in self.worker_spec:
                 del self.worker_spec[w]
@@ -407,6 +424,30 @@ async def scale_down(self, workers):
 
     scale_up = scale  # backwards compatibility
 
+    @property
+    def plan(self):
+        out = set()
+        for name, spec in self.worker_spec.items():
+            if "group" in spec:
+                out.update({str(name) + suffix for suffix in spec["group"]})
+            else:
+                out.add(name)
+        return out
+
+    @property
+    def requested(self):
+        out = set()
+        for name in self.workers:
+            try:
+                spec = self.worker_spec[name]
+            except KeyError:
+                continue
+            if "group" in spec:
+                out.update({str(name) + suffix for suffix in spec["group"]})
+            else:
+                out.add(name)
+        return out
+
 
 @atexit.register
 def close_clusters():
diff --git a/distributed/deploy/tests/test_spec_cluster.py b/distributed/deploy/tests/test_spec_cluster.py
index ea0afb0488d..efc231ca030 100644
--- a/distributed/deploy/tests/test_spec_cluster.py
+++ b/distributed/deploy/tests/test_spec_cluster.py
@@ -297,3 +297,72 @@ async def test_ProcessInterfaceValid(cleanup):
         cluster.scale(1)
         await cluster
         assert len(cluster.worker_spec) == len(cluster.workers) == 1
+
+
+class MultiWorker(Worker, ProcessInterface):
+    def __init__(self, *args, n=1, name=None, nthreads=None, **kwargs):
+        self.workers = [
+            Worker(
+                *args, name=str(name) + "-" + str(i), nthreads=nthreads // n, **kwargs
+            )
+            for i in range(n)
+        ]
+
+    @property
+    def status(self):
+        return self.workers[0].status
+
+    def __str__(self):
+        return "<MultiWorker n=%d>" % len(self.workers)
+
+    __repr__ = __str__
+
+    async def start(self):
+        await asyncio.gather(*self.workers)
+
+    async def close(self):
+        await asyncio.gather(*[w.close() for w in self.workers])
+
+
+@pytest.mark.asyncio
+async def test_MultiWorker(cleanup):
+    async with SpecCluster(
+        scheduler=scheduler,
+        worker={
+            "cls": MultiWorker,
+            "options": {"n": 2, "nthreads": 4, "memory_limit": "4 GB"},
+            "group": ["-0", "-1"],
+        },
+        asynchronous=True,
+    ) as cluster:
+        s = cluster.scheduler
+        async with Client(cluster, asynchronous=True) as client:
+            cluster.scale(2)
+            await cluster
+            assert len(cluster.worker_spec) == 2
+            await client.wait_for_workers(4)
+
+            cluster.scale(1)
+            await cluster
+            assert len(s.workers) == 2
+
+            cluster.scale(memory="6GB")
+            await cluster
+            assert len(cluster.worker_spec) == 2
+            assert len(s.workers) == 4
+            assert cluster.plan == {ws.name for ws in s.workers.values()}
+
+            cluster.scale(cores=10)
+            await cluster
+            assert len(cluster.workers) == 3
+
+            adapt = cluster.adapt(minimum=0, maximum=4)
+
+            for i in range(adapt.wait_count):  # relax down to 0 workers
+                await adapt.adapt()
+            await cluster
+            assert not s.workers
+
+            future = client.submit(lambda x: x + 1, 10)
+            await future
+            assert len(cluster.workers) == 1

From 1ef4f70dc7f8fe048d11104ee8daad04b82227e9 Mon Sep 17 00:00:00 2001
From: byjott <jochen.ott@blue-yonder.com>
Date: Fri, 30 Aug 2019 18:18:03 +0200
Subject: [PATCH 0444/1550] Fix ConnectionPool limit handling (#3005)

Fixes #3001

* convert test_core to 'async def' style
---
 distributed/core.py            |  52 ++++++++-----
 distributed/tests/test_core.py | 138 ++++++++++++++++++---------------
 2 files changed, 109 insertions(+), 81 deletions(-)

diff --git a/distributed/core.py b/distributed/core.py
index f97d2df382a..6bda9c9e0be 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -826,7 +826,9 @@ def __init__(
         self.deserializers = deserializers if deserializers is not None else serializers
         self.connection_args = connection_args
         self.timeout = timeout
-        self.event = Event()
+        self._n_connecting = 0
+        # Invariant: semaphore._value == limit - open - _n_connecting
+        self.semaphore = asyncio.Semaphore(self.limit)
         self.server = weakref.ref(server) if server else None
         self._created = weakref.WeakSet()
         self._instances.add(self)
@@ -840,7 +842,11 @@ def open(self):
         return self.active + sum(map(len, self.available.values()))
 
     def __repr__(self):
-        return "<ConnectionPool: open=%d, active=%d>" % (self.open, self.active)
+        return "<ConnectionPool: open=%d, active=%d, connecting=%d>" % (
+            self.open,
+            self.active,
+            self._n_connecting,
+        )
 
     def __call__(self, addr=None, ip=None, port=None):
         """ Cached rpc objects """
@@ -861,10 +867,11 @@ async def connect(self, addr, timeout=None):
                 occupied.add(comm)
                 return comm
 
-        while self.open >= self.limit:
-            self.event.clear()
+        if self.semaphore.locked():
             self.collect()
-            await self.event.wait()
+
+        self._n_connecting += 1
+        await self.semaphore.acquire()
 
         try:
             comm = await connect(
@@ -877,11 +884,12 @@ async def connect(self, addr, timeout=None):
             comm._pool = weakref.ref(self)
             self._created.add(comm)
         except Exception:
+            self.semaphore.release()
             raise
-        occupied.add(comm)
+        finally:
+            self._n_connecting -= 1
 
-        if self.open >= self.limit:
-            self.event.clear()
+        occupied.add(comm)
 
         return comm
 
@@ -889,30 +897,34 @@ def reuse(self, addr, comm):
         """
         Reuse an open communication to the given address.  For internal use.
         """
-        try:
-            self.occupied[addr].remove(comm)
-        except KeyError:
-            pass
+        # if the pool is asked to re-use a comm it does not know about, ignore
+        # this comm: just close it.
+        if comm not in self.occupied[addr]:
+            IOLoop.current().add_callback(comm.close)
         else:
+            self.occupied[addr].remove(comm)
             if comm.closed():
-                if self.open < self.limit:
-                    self.event.set()
+                self.semaphore.release()
             else:
                 self.available[addr].add(comm)
+                if self.semaphore.locked() and self._n_connecting > 0:
+                    self.collect()
 
     def collect(self):
         """
         Collect open but unused communications, to allow opening other ones.
         """
         logger.info(
-            "Collecting unused comms.  open: %d, active: %d", self.open, self.active
+            "Collecting unused comms.  open: %d, active: %d, connecting: %d",
+            self.open,
+            self.active,
+            self._n_connecting,
         )
         for addr, comms in self.available.items():
             for comm in comms:
                 IOLoop.current().add_callback(comm.close)
+                self.semaphore.release()
             comms.clear()
-        if self.open < self.limit:
-            self.event.set()
 
     def remove(self, addr):
         """
@@ -923,12 +935,12 @@ def remove(self, addr):
             comms = self.available.pop(addr)
             for comm in comms:
                 IOLoop.current().add_callback(comm.close)
+                self.semaphore.release()
         if addr in self.occupied:
             comms = self.occupied.pop(addr)
             for comm in comms:
                 IOLoop.current().add_callback(comm.close)
-        if self.open < self.limit:
-            self.event.set()
+                self.semaphore.release()
 
     def close(self):
         """
@@ -937,8 +949,10 @@ def close(self):
         for comms in self.available.values():
             for comm in comms:
                 comm.abort()
+                self.semaphore.release()
         for comms in self.occupied.values():
             for comm in comms:
+                self.semaphore.release()
                 comm.abort()
 
         for comm in self._created:
diff --git a/distributed/tests/test_core.py b/distributed/tests/test_core.py
index e41866d6741..cbde7ac240b 100644
--- a/distributed/tests/test_core.py
+++ b/distributed/tests/test_core.py
@@ -1,10 +1,10 @@
+import asyncio
 from contextlib import contextmanager
 import os
 import socket
 import threading
 import weakref
 
-from tornado import gen
 import pytest
 
 import dask
@@ -82,8 +82,7 @@ def test_server(loop):
     Simple Server test.
     """
 
-    @gen.coroutine
-    def f():
+    async def f():
         server = Server({"ping": pingpong})
         with pytest.raises(ValueError):
             server.port
@@ -92,20 +91,20 @@ def f():
         assert server.address == ("tcp://%s:8881" % get_ip())
 
         for addr in ("127.0.0.1:8881", "tcp://127.0.0.1:8881", server.address):
-            comm = yield connect(addr)
+            comm = await connect(addr)
 
-            n = yield comm.write({"op": "ping"})
+            n = await comm.write({"op": "ping"})
             assert isinstance(n, int)
             assert 4 <= n <= 1000
 
-            response = yield comm.read()
+            response = await comm.read()
             assert response == b"pong"
 
-            yield comm.write({"op": "ping", "close": True})
-            response = yield comm.read()
+            await comm.write({"op": "ping", "close": True})
+            response = await comm.read()
             assert response == b"pong"
 
-            yield comm.close()
+            await comm.close()
 
         server.stop()
 
@@ -113,20 +112,19 @@ def f():
 
 
 def test_server_raises_on_blocked_handlers(loop):
-    @gen.coroutine
-    def f():
+    async def f():
         server = Server({"ping": pingpong}, blocked_handlers=["ping"])
         server.listen(8881)
 
-        comm = yield connect(server.address)
-        yield comm.write({"op": "ping"})
-        msg = yield comm.read()
+        comm = await connect(server.address)
+        await comm.write({"op": "ping"})
+        msg = await comm.read()
 
         assert "exception" in msg
         assert isinstance(msg["exception"], ValueError)
         assert "'ping' handler has been explicitly disallowed" in repr(msg["exception"])
 
-        yield comm.close()
+        await comm.close()
         server.stop()
 
     res = loop.run_sync(f)
@@ -251,21 +249,20 @@ def listen_on(cls, *args, **kwargs):
         yield assert_cannot_connect(inproc_addr2)
 
 
-@gen.coroutine
-def check_rpc(listen_addr, rpc_addr=None, listen_args=None, connection_args=None):
+async def check_rpc(listen_addr, rpc_addr=None, listen_args=None, connection_args=None):
     server = Server({"ping": pingpong})
     server.listen(listen_addr, listen_args=listen_args)
     if rpc_addr is None:
         rpc_addr = server.address
 
     with rpc(rpc_addr, connection_args=connection_args) as remote:
-        response = yield remote.ping()
+        response = await remote.ping()
         assert response == b"pong"
         assert remote.comms
 
-        response = yield remote.ping(close=True)
+        response = await remote.ping(close=True)
         assert response == b"pong"
-        response = yield remote.ping()
+        response = await remote.ping()
         assert response == b"pong"
 
     assert not remote.comms
@@ -311,8 +308,7 @@ def test_rpc_inputs():
         r.close_rpc()
 
 
-@gen.coroutine
-def check_rpc_message_lifetime(*listen_args):
+async def check_rpc_message_lifetime(*listen_args):
     # Issue #956: rpc arguments and result shouldn't be kept alive longer
     # than necessary
     server = Server({"echo": echo_serialize})
@@ -324,15 +320,15 @@ def check_rpc_message_lifetime(*listen_args):
     del obj
     start = time()
     while CountedObject.n_instances != 0:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 1
 
     with rpc(server.address) as remote:
         obj = CountedObject()
-        res = yield remote.echo(x=to_serialize(obj))
+        res = await remote.echo(x=to_serialize(obj))
         assert isinstance(res["result"], CountedObject)
         # Make sure resource cleanup code in coroutines runs
-        yield gen.sleep(0.05)
+        await asyncio.sleep(0.05)
 
         w1 = weakref.ref(obj)
         w2 = weakref.ref(res["result"])
@@ -361,18 +357,17 @@ def test_rpc_message_lifetime_inproc():
     yield check_rpc_message_lifetime("inproc://")
 
 
-@gen.coroutine
-def check_rpc_with_many_connections(listen_arg):
-    @gen.coroutine
-    def g():
+async def check_rpc_with_many_connections(listen_arg):
+    async def g():
         for i in range(10):
-            yield remote.ping()
+            await remote.ping()
 
     server = Server({"ping": pingpong})
     server.listen(listen_arg)
 
     with rpc(server.address) as remote:
-        yield [g() for i in range(10)]
+        for i in range(10):
+            await g()
 
         server.stop()
 
@@ -390,19 +385,18 @@ def test_rpc_with_many_connections_inproc():
     yield check_rpc_with_many_connections("inproc://")
 
 
-@gen.coroutine
-def check_large_packets(listen_arg):
+async def check_large_packets(listen_arg):
     """ tornado has a 100MB cap by default """
     server = Server({"echo": echo})
     server.listen(listen_arg)
 
     data = b"0" * int(200e6)  # slightly more than 100MB
     conn = rpc(server.address)
-    result = yield conn.echo(x=data)
+    result = await conn.echo(x=data)
     assert result == data
 
     d = {"x": data}
-    result = yield conn.echo(x=d)
+    result = await conn.echo(x=d)
     assert result == d
 
     conn.close_comms()
@@ -420,14 +414,13 @@ def test_large_packets_inproc():
     yield check_large_packets("inproc://")
 
 
-@gen.coroutine
-def check_identity(listen_arg):
+async def check_identity(listen_arg):
     server = Server({})
     server.listen(listen_arg)
 
     with rpc(server.address) as remote:
-        a = yield remote.identity()
-        b = yield remote.identity()
+        a = await remote.identity()
+        b = await remote.identity()
         assert a["type"] == "Server"
         assert a["id"] == b["id"]
 
@@ -489,7 +482,7 @@ def test_errors():
 
 @gen_test()
 def test_connect_raises():
-    with pytest.raises((gen.TimeoutError, IOError)):
+    with pytest.raises(IOError):
         yield connect("127.0.0.1:58259", timeout=0.01)
 
 
@@ -519,10 +512,9 @@ def test_coerce_to_address():
 
 @gen_test()
 def test_connection_pool():
-    @gen.coroutine
-    def ping(comm, delay=0.1):
-        yield gen.sleep(delay)
-        raise gen.Return("pong")
+    async def ping(comm, delay=0.1):
+        await asyncio.sleep(delay)
+        return "pong"
 
     servers = [Server({"ping": ping}) for i in range(10)]
     for server in servers:
@@ -553,12 +545,35 @@ def ping(comm, delay=0.1):
     rpc.collect()
     start = time()
     while any(rpc.available.values()):
-        yield gen.sleep(0.01)
+        yield asyncio.sleep(0.01)
         assert time() < start + 2
 
     rpc.close()
 
 
+@gen_test()
+def test_connection_pool_respects_limit():
+
+    limit = 5
+
+    async def ping(comm, delay=0.01):
+        await asyncio.sleep(delay)
+        return "pong"
+
+    async def do_ping(pool, port):
+        assert pool.open <= limit
+        await pool(ip="127.0.0.1", port=port).ping()
+        assert pool.open <= limit
+
+    servers = [Server({"ping": ping}) for i in range(10)]
+    for server in servers:
+        server.listen(0)
+
+    pool = ConnectionPool(limit=limit)
+
+    yield [do_ping(pool, s.port) for s in servers]
+
+
 @gen_test()
 def test_connection_pool_tls():
     """
@@ -568,10 +583,9 @@ def test_connection_pool_tls():
     connection_args = sec.get_connection_args("client")
     listen_args = sec.get_listen_args("scheduler")
 
-    @gen.coroutine
-    def ping(comm, delay=0.01):
-        yield gen.sleep(delay)
-        raise gen.Return("pong")
+    async def ping(comm, delay=0.01):
+        await asyncio.sleep(delay)
+        return "pong"
 
     servers = [Server({"ping": ping}) for i in range(10)]
     for server in servers:
@@ -589,10 +603,9 @@ def ping(comm, delay=0.01):
 
 @gen_test()
 def test_connection_pool_remove():
-    @gen.coroutine
-    def ping(comm, delay=0.01):
-        yield gen.sleep(delay)
-        raise gen.Return("pong")
+    async def ping(comm, delay=0.01):
+        await asyncio.sleep(delay)
+        return "pong"
 
     servers = [Server({"ping": ping}) for i in range(5)]
     for server in servers:
@@ -617,6 +630,9 @@ def ping(comm, delay=0.01):
     assert rpc.open == 4
 
     rpc.collect()
+
+    # this pattern of calls (esp. `reuse` after `remove`)
+    # can happen in case of worker failures:
     comm = yield rpc.connect(serv.address)
     rpc.remove(serv.address)
     rpc.reuse(serv.address, comm)
@@ -642,7 +658,7 @@ def test_counters():
 @gen_cluster()
 def test_ticks(s, a, b):
     pytest.importorskip("crick")
-    yield gen.sleep(0.1)
+    yield asyncio.sleep(0.1)
     c = s.digests["tick-duration"]
     assert c.size()
     assert 0.01 < c.components[0].quantile(0.5) < 0.5
@@ -657,7 +673,7 @@ def test_tick_logging(s, a, b):
     core.tick_maximum_delay = 0.001
     try:
         with captured_logger("distributed.core") as sio:
-            yield gen.sleep(0.1)
+            yield asyncio.sleep(0.1)
 
         text = sio.getvalue()
         assert "unresponsive" in text
@@ -671,14 +687,13 @@ def test_tick_logging(s, a, b):
 def test_compression(compression, serialize, loop):
     with dask.config.set(compression=compression):
 
-        @gen.coroutine
-        def f():
+        async def f():
             server = Server({"echo": serialize})
             server.listen("tcp://")
 
             with rpc(server.address) as r:
                 data = b"1" * 1000000
-                result = yield r.echo(x=to_serialize(data))
+                result = await r.echo(x=to_serialize(data))
                 assert result == {"result": data}
 
             server.stop()
@@ -687,17 +702,16 @@ def f():
 
 
 def test_rpc_serialization(loop):
-    @gen.coroutine
-    def f():
+    async def f():
         server = Server({"echo": echo_serialize})
         server.listen("tcp://")
 
         with rpc(server.address, serializers=["msgpack"]) as r:
             with pytest.raises(TypeError):
-                yield r.echo(x=to_serialize(inc))
+                await r.echo(x=to_serialize(inc))
 
         with rpc(server.address, serializers=["msgpack", "pickle"]) as r:
-            result = yield r.echo(x=to_serialize(inc))
+            result = await r.echo(x=to_serialize(inc))
             assert result == {"result": inc}
 
         server.stop()

From 2c148521abef4d0d5d46bd8f6e50da6096eb359b Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 3 Sep 2019 08:12:41 -0700
Subject: [PATCH 0445/1550] Avoid collision when using os.environ in
 dashboard_link (#3021)

Fixes #3016
---
 distributed/tests/test_utils.py | 16 ++++++++++++++++
 distributed/utils.py            |  4 +++-
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/distributed/tests/test_utils.py b/distributed/tests/test_utils.py
index bf2d8456681..ff733a1ad8c 100644
--- a/distributed/tests/test_utils.py
+++ b/distributed/tests/test_utils.py
@@ -2,6 +2,7 @@
 import datetime
 from functools import partial
 import io
+import os
 import queue
 import socket
 import sys
@@ -40,6 +41,7 @@
     parse_bytes,
     parse_timedelta,
     warn_on_duration,
+    format_dashboard_link,
 )
 from distributed.utils_test import loop, loop_in_thread  # noqa: F401
 from distributed.utils_test import div, has_ipv6, inc, throws, gen_test, captured_logger
@@ -582,3 +584,17 @@ def test_is_valid_xml():
     assert is_valid_xml("<a>foo</a>")
     with pytest.raises(Exception):
         assert is_valid_xml("<a>foo")
+
+
+def test_format_dashboard_link():
+    with dask.config.set({"distributed.dashboard.link": "foo"}):
+        assert format_dashboard_link("host", 1234) == "foo"
+
+    assert "host" in format_dashboard_link("host", 1234)
+    assert "1234" in format_dashboard_link("host", 1234)
+
+    try:
+        os.environ["host"] = "hello"
+        assert "hello" not in format_dashboard_link("host", 1234)
+    finally:
+        del os.environ["host"]
diff --git a/distributed/utils.py b/distributed/utils.py
index 9f14d58de0b..b7f6631ce93 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -1434,7 +1434,9 @@ def format_dashboard_link(host, port):
         scheme = "https"
     else:
         scheme = "http"
-    return template.format(scheme=scheme, host=host, port=port, **os.environ)
+    return template.format(
+        **toolz.merge(os.environ, dict(scheme=scheme, host=host, port=port))
+    )
 
 
 def is_coroutine_function(f):

From 04a1bb0140d181d36eb27a284cfad2e289b8cda0 Mon Sep 17 00:00:00 2001
From: Abael He <abaelhe@users.noreply.github.com>
Date: Wed, 4 Sep 2019 00:05:45 +0800
Subject: [PATCH 0446/1550] Add support for zstandard compression to comms
 (#2970)

Adds support for zstandard compression to dask comms.
---
 .../setup_conda_environment.cmd               |  1 +
 continuous_integration/travis/install.sh      |  2 +-
 distributed/distributed.yaml                  |  4 +++
 distributed/protocol/compression.py           | 20 +++++++++++
 distributed/protocol/tests/test_protocol.py   | 36 ++++++++-----------
 5 files changed, 40 insertions(+), 23 deletions(-)

diff --git a/continuous_integration/setup_conda_environment.cmd b/continuous_integration/setup_conda_environment.cmd
index 6fff1a5ca6a..d09846faedd 100644
--- a/continuous_integration/setup_conda_environment.cmd
+++ b/continuous_integration/setup_conda_environment.cmd
@@ -18,6 +18,7 @@ call deactivate
 @rem Create test environment
 @rem (note: no cytoolz as it seems to prevent faulthandler tracebacks on crash)
 %CONDA% create -n %CONDA_ENV% -q -y ^
+    zstandard ^
     bokeh ^
     click ^
     cloudpickle ^
diff --git a/continuous_integration/travis/install.sh b/continuous_integration/travis/install.sh
index 82993032e0b..ead4dbc3002 100644
--- a/continuous_integration/travis/install.sh
+++ b/continuous_integration/travis/install.sh
@@ -56,7 +56,7 @@ conda install -q \
 
 # For low-level profiler, install libunwind and stacktrace from conda-forge
 # For stacktrace we use --no-deps to avoid upgrade of python
-conda install -c defaults -c conda-forge libunwind
+conda install -c defaults -c conda-forge libunwind zstandard
 conda install --no-deps -c defaults -c numba -c conda-forge stacktrace
 
 pip install -q "pytest>=4" pytest-repeat pytest-faulthandler pytest-asyncio
diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index 9ad3e365e78..a0e801f26ac 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -71,6 +71,10 @@ distributed:
     socket-backlog: 2048
     recent-messages-log-length: 0  # number of messages to keep for debugging
 
+    zstd:
+      level: 3      # Compression level, between 1 and 22.
+      threads: 0    # Threads to use. 0 for single-threaded, -1 to infer from cpu count.
+
     timeouts:
       connect: 10s          # time before connecting fails
       tcp: 30s              # time before calling an unresponsive connection dead
diff --git a/distributed/protocol/compression.py b/distributed/protocol/compression.py
index 5035b465cee..5e81cdbaf1f 100644
--- a/distributed/protocol/compression.py
+++ b/distributed/protocol/compression.py
@@ -93,6 +93,26 @@ def _fixed_lz4_decompress(data):
     }
     default_compression = "lz4"
 
+
+with ignoring(ImportError):
+    import zstandard
+
+    zstd_compressor = zstandard.ZstdCompressor(
+        level=dask.config.get("distributed.comm.zstd.level"),
+        threads=dask.config.get("distributed.comm.zstd.threads"),
+    )
+
+    zstd_decompressor = zstandard.ZstdDecompressor()
+
+    def zstd_compress(data):
+        return zstd_compressor.compress(data)
+
+    def zstd_decompress(data):
+        return zstd_decompressor.decompress(data)
+
+    compressions["zstd"] = {"compress": zstd_compress, "decompress": zstd_decompress}
+
+
 with ignoring(ImportError):
     import blosc
 
diff --git a/distributed/protocol/tests/test_protocol.py b/distributed/protocol/tests/test_protocol.py
index 395c1ca7b97..3dd11ecc4d1 100644
--- a/distributed/protocol/tests/test_protocol.py
+++ b/distributed/protocol/tests/test_protocol.py
@@ -57,34 +57,26 @@ def test_small_and_big():
     # assert loads([big_header, big]) == {'y': d['y']}
 
 
-def test_maybe_compress():
-    pass
+@pytest.mark.parametrize(
+    "lib,compression",
+    [(None, None), ("zlib", "zlib"), ("lz4", "lz4"), ("zstandard", "zstd")],
+)
+def test_maybe_compress(lib, compression):
+    if lib:
+        pytest.importorskip(lib)
 
     try_converters = [bytes, memoryview]
-    try_compressions = ["zlib", "lz4"]
 
-    payload = b"123"
-
-    with dask.config.set({"distributed.comm.compression": None}):
+    with dask.config.set({"distributed.comm.compression": compression}):
         for f in try_converters:
+            payload = b"123"
             assert maybe_compress(f(payload)) == (None, payload)
 
-    for compression in try_compressions:
-        try:
-            __import__(compression)
-        except ImportError:
-            continue
-
-        with dask.config.set({"distributed.comm.compression": compression}):
-            for f in try_converters:
-                payload = b"123"
-                assert maybe_compress(f(payload)) == (None, payload)
-
-                payload = b"0" * 10000
-                rc, rd = maybe_compress(f(payload))
-                # For some reason compressing memoryviews can force blosc...
-                assert rc in (compression, "blosc")
-                assert compressions[rc]["decompress"](rd) == payload
+            payload = b"0" * 10000
+            rc, rd = maybe_compress(f(payload))
+            # For some reason compressing memoryviews can force blosc...
+            assert rc in (compression, "blosc")
+            assert compressions[rc]["decompress"](rd) == payload
 
 
 def test_maybe_compress_sample():

From 7a1a369270557b912cd6fde3f96cce8196672f23 Mon Sep 17 00:00:00 2001
From: Jim Crist <jcrist@users.noreply.github.com>
Date: Tue, 3 Sep 2019 14:02:16 -0500
Subject: [PATCH 0447/1550] Add fallback html repr for Cluster (#3023)

This PR fixes two things:

-  Cluster objects don't error when repr'd in notebooks without
    ipywidgets installed.
-  If ipywidgets isn't installed, a fallback HTML repr is used instead of
    the default `repr` string.

A test is added to test both of these.
---
 distributed/deploy/cluster.py          | 40 +++++++++++++++++++++++---
 distributed/deploy/tests/test_local.py | 25 ++++++++++++++++
 2 files changed, 61 insertions(+), 4 deletions(-)

diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
index 5e86c39ce8e..393be849b88 100644
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -246,7 +246,11 @@ def _widget(self):
         except AttributeError:
             pass
 
-        from ipywidgets import Layout, VBox, HBox, IntText, Button, HTML, Accordion
+        try:
+            from ipywidgets import Layout, VBox, HBox, IntText, Button, HTML, Accordion
+        except ImportError:
+            self._cached_widget = None
+            return None
 
         layout = Layout(width="150px")
 
@@ -258,7 +262,7 @@ def _widget(self):
         else:
             link = ""
 
-        title = "<h2>%s</h2>" % type(self).__name__
+        title = "<h2>%s</h2>" % self._cluster_class_name
         title = HTML(title)
         dashboard = HTML(link)
 
@@ -311,8 +315,32 @@ def update():
 
         return box
 
+    def _repr_html_(self):
+        if self.dashboard_link:
+            dashboard = "<a href='{0}' target='_blank'>{0}</a>".format(
+                self.dashboard_link
+            )
+        else:
+            dashboard = "Not Available"
+        return (
+            "<div style='background-color: #f2f2f2; display: inline-block; "
+            "padding: 10px; border: 1px solid #999999;'>\n"
+            "  <h3>{cls}</h3>\n"
+            "  <ul>\n"
+            "    <li><b>Dashboard: </b>{dashboard}\n"
+            "  </ul>\n"
+            "</div>\n"
+        ).format(cls=self._cluster_class_name, dashboard=dashboard)
+
     def _ipython_display_(self, **kwargs):
-        return self._widget()._ipython_display_(**kwargs)
+        widget = self._widget()
+        if widget is not None:
+            return widget._ipython_display_(**kwargs)
+        else:
+            from IPython.display import display
+
+            data = {"text/plain": repr(self), "text/html": self._repr_html_()}
+            display(data, raw=True)
 
     async def __aenter__(self):
         await self
@@ -325,9 +353,13 @@ async def __aexit__(self, typ, value, traceback):
     def scheduler_address(self):
         return self.scheduler_comm.address
 
+    @property
+    def _cluster_class_name(self):
+        return getattr(self, "_name", type(self).__name__)
+
     def __repr__(self):
         text = "%s(%r, workers=%d, threads=%d" % (
-            getattr(self, "_name", type(self).__name__),
+            self._cluster_class_name,
             self.scheduler_address,
             len(self.workers),
             sum(w["nthreads"] for w in self.scheduler_info["workers"].values()),
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index 59c75d545d8..2ad46035425 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -530,6 +530,31 @@ def test_ipywidgets(loop):
         assert isinstance(box, ipywidgets.Widget)
 
 
+def test_no_ipywidgets(loop, monkeypatch):
+    from unittest.mock import MagicMock
+
+    mock_display = MagicMock()
+
+    monkeypatch.setitem(sys.modules, "ipywidgets", None)
+    monkeypatch.setitem(sys.modules, "IPython.display", mock_display)
+
+    with LocalCluster(
+        n_workers=0,
+        scheduler_port=0,
+        silence_logs=False,
+        loop=loop,
+        dashboard_address=False,
+        processes=False,
+    ) as cluster:
+        cluster._ipython_display_()
+        args, kwargs = mock_display.display.call_args
+        res = args[0]
+        assert kwargs == {"raw": True}
+        assert isinstance(res, dict)
+        assert "text/plain" in res
+        assert "text/html" in res
+
+
 def test_scale(loop):
     """ Directly calling scale both up and down works as expected """
     with LocalCluster(

From 8d07bf162b1f6434fc982f36360c5cedbac369a6 Mon Sep 17 00:00:00 2001
From: Benjamin Zaitlen <quasiben@users.noreply.github.com>
Date: Tue, 3 Sep 2019 20:47:44 -0400
Subject: [PATCH 0448/1550] Rely on cudf codebase for cudf serialization
 (#2998)

* reinstate cudf serialization dispatch in distributed to support older versions of cudf -- there may be a distributed release *before* the next release of cudf
---
 distributed/protocol/__init__.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/distributed/protocol/__init__.py b/distributed/protocol/__init__.py
index e30786ab4a5..ef8b5564bbb 100644
--- a/distributed/protocol/__init__.py
+++ b/distributed/protocol/__init__.py
@@ -1,4 +1,5 @@
 from functools import partial
+from distutils.version import LooseVersion
 
 from .compression import compressions, default_compression
 from .core import dumps, loads, maybe_compress, decompress, msgpack
@@ -82,4 +83,9 @@ def _register_numba():
 @cuda_serialize.register_lazy("cudf")
 @cuda_deserialize.register_lazy("cudf")
 def _register_cudf():
-    from . import cudf
+    import cudf
+
+    if LooseVersion(cudf.__version__) > "0.9":
+        from cudf.comm import serialize
+    else:
+        from . import cudf

From adf247c15c32831b10c76a163cf0f769c4339483 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 6 Sep 2019 13:00:51 -0700
Subject: [PATCH 0449/1550] Set the x_range limit of the Meory utilization plot
 to memory-limit (#3034)

This should give a better sense for how close we are to filling up
memory, which seems to be the biggest use of the memory plot.
---
 distributed/dashboard/scheduler.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/distributed/dashboard/scheduler.py b/distributed/dashboard/scheduler.py
index 3dd108f5775..7f172c879d1 100644
--- a/distributed/dashboard/scheduler.py
+++ b/distributed/dashboard/scheduler.py
@@ -483,6 +483,7 @@ def update(self):
                 self.nbytes_figure.title.text = "Bytes stored: " + format_bytes(
                     sum(nbytes)
                 )
+                self.nbytes_figure.x_range.end = max_limit
 
                 update(self.source, result)
 

From 98822d4ab13081a49a3643f216dcf26009ff9496 Mon Sep 17 00:00:00 2001
From: Mikhail Akimov <roveo@users.noreply.github.com>
Date: Sat, 7 Sep 2019 00:30:46 +0300
Subject: [PATCH 0450/1550] Replace print statement in Queue.__init__ with
 debug message (#3035)

---
 distributed/queues.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/queues.py b/distributed/queues.py
index 7174c48a63c..1d0c2c0bdd3 100644
--- a/distributed/queues.py
+++ b/distributed/queues.py
@@ -47,7 +47,7 @@ def __init__(self, scheduler):
         self.scheduler.extensions["queues"] = self
 
     def create(self, stream=None, name=None, client=None, maxsize=0):
-        print("name", name)
+        logger.debug("Queue name: {}".format(name))
         if name not in self.queues:
             self.queues[name] = tornado.queues.Queue(maxsize=maxsize)
             self.client_refcount[name] = 1

From 8e544142602e05dab57331cc59f86c43550459db Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sat, 7 Sep 2019 08:37:01 -0700
Subject: [PATCH 0451/1550] Clean up test_local.py::test_defaults (#3017)

---
 distributed/deploy/tests/test_local.py | 112 +++++++++++++++----------
 1 file changed, 70 insertions(+), 42 deletions(-)

diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index 2ad46035425..5459574cccf 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -225,79 +225,112 @@ def test_Client_twice(loop):
             assert c.cluster.scheduler.port != f.cluster.scheduler.port
 
 
-@pytest.mark.skipif("sys.version_info[0] == 2", reason="fork issues")
-def test_defaults():
-    _nthreads = multiprocessing.cpu_count()
-
-    with LocalCluster(
-        scheduler_port=0, silence_logs=False, dashboard_address=None
+@pytest.mark.asyncio
+async def test_defaults(cleanup):
+    async with LocalCluster(
+        scheduler_port=0, silence_logs=False, dashboard_address=None, asynchronous=True
     ) as c:
-        assert sum(w.nthreads for w in c.workers.values()) == _nthreads
+        assert (
+            sum(w.nthreads for w in c.workers.values()) == multiprocessing.cpu_count()
+        )
         assert all(isinstance(w, Nanny) for w in c.workers.values())
 
-    with LocalCluster(
-        processes=False, scheduler_port=0, silence_logs=False, dashboard_address=None
+
+@pytest.mark.asyncio
+async def test_defaults_2(cleanup):
+    async with LocalCluster(
+        processes=False,
+        scheduler_port=0,
+        silence_logs=False,
+        dashboard_address=None,
+        asynchronous=True,
     ) as c:
-        assert sum(w.nthreads for w in c.workers.values()) == _nthreads
+        assert (
+            sum(w.nthreads for w in c.workers.values()) == multiprocessing.cpu_count()
+        )
         assert all(isinstance(w, Worker) for w in c.workers.values())
         assert len(c.workers) == 1
 
-    with LocalCluster(
-        n_workers=2, scheduler_port=0, silence_logs=False, dashboard_address=None
+
+@pytest.mark.asyncio
+async def test_defaults_3(cleanup):
+    async with LocalCluster(
+        n_workers=2,
+        scheduler_port=0,
+        silence_logs=False,
+        dashboard_address=None,
+        asynchronous=True,
     ) as c:
-        if _nthreads % 2 == 0:
-            expected_total_threads = max(2, _nthreads)
+        if multiprocessing.cpu_count() % 2 == 0:
+            expected_total_threads = max(2, multiprocessing.cpu_count())
         else:
             # n_workers not a divisor of _nthreads => threads are overcommitted
-            expected_total_threads = max(2, _nthreads + 1)
+            expected_total_threads = max(2, multiprocessing.cpu_count() + 1)
         assert sum(w.nthreads for w in c.workers.values()) == expected_total_threads
 
-    with LocalCluster(
-        threads_per_worker=_nthreads * 2,
+
+@pytest.mark.asyncio
+async def test_defaults_4(cleanup):
+    async with LocalCluster(
+        threads_per_worker=multiprocessing.cpu_count() * 2,
         scheduler_port=0,
         silence_logs=False,
         dashboard_address=None,
+        asynchronous=True,
     ) as c:
         assert len(c.workers) == 1
 
-    with LocalCluster(
-        n_workers=_nthreads * 2,
+
+@pytest.mark.asyncio
+async def test_defaults_5(cleanup):
+    async with LocalCluster(
+        n_workers=multiprocessing.cpu_count() * 2,
         scheduler_port=0,
         silence_logs=False,
         dashboard_address=None,
+        asynchronous=True,
     ) as c:
         assert all(w.nthreads == 1 for w in c.workers.values())
-    with LocalCluster(
+
+
+@pytest.mark.asyncio
+async def test_defaults_6(cleanup):
+    async with LocalCluster(
         threads_per_worker=2,
         n_workers=3,
         scheduler_port=0,
         silence_logs=False,
         dashboard_address=None,
+        asynchronous=True,
     ) as c:
         assert len(c.workers) == 3
         assert all(w.nthreads == 2 for w in c.workers.values())
 
 
-def test_worker_params():
-    with LocalCluster(
+@pytest.mark.asyncio
+async def test_worker_params(cleanup):
+    async with LocalCluster(
         processes=False,
         n_workers=2,
         scheduler_port=0,
         silence_logs=False,
         dashboard_address=None,
         memory_limit=500,
+        asynchronous=True,
     ) as c:
         assert [w.memory_limit for w in c.workers.values()] == [500] * 2
 
 
-def test_memory_limit_none():
-    with LocalCluster(
+@pytest.mark.asyncio
+async def test_memory_limit_none(cleanup):
+    async with LocalCluster(
         n_workers=2,
         scheduler_port=0,
         silence_logs=False,
         processes=False,
         dashboard_address=None,
         memory_limit=None,
+        asynchronous=True,
     ) as c:
         w = c.workers[0]
         assert type(w.data) is dict
@@ -364,34 +397,29 @@ def test_blocks_until_full(loop):
         assert len(c.nthreads()) > 0
 
 
-@gen_test()
-def test_scale_up_and_down():
-    loop = IOLoop.current()
-    cluster = yield LocalCluster(
+@pytest.mark.asyncio
+async def test_scale_up_and_down():
+    async with LocalCluster(
         0,
         scheduler_port=0,
         processes=False,
         silence_logs=False,
         dashboard_address=None,
-        loop=loop,
         asynchronous=True,
-    )
-    c = yield Client(cluster, asynchronous=True)
-
-    assert not cluster.workers
+    ) as cluster:
+        async with Client(cluster, asynchronous=True) as c:
 
-    cluster.scale(2)
-    yield cluster
-    assert len(cluster.workers) == 2
-    assert len(cluster.scheduler.nthreads) == 2
+            assert not cluster.workers
 
-    cluster.scale(1)
-    yield cluster
+            cluster.scale(2)
+            await cluster
+            assert len(cluster.workers) == 2
+            assert len(cluster.scheduler.nthreads) == 2
 
-    assert len(cluster.workers) == 1
+            cluster.scale(1)
+            await cluster
 
-    yield c.close()
-    yield cluster.close()
+            assert len(cluster.workers) == 1
 
 
 def test_silent_startup():

From 5735238e6fcd30a64cc16a3c60ba16b045966beb Mon Sep 17 00:00:00 2001
From: Guillaume Eynard-Bontemps <g.eynard.bontemps@gmail.com>
Date: Sat, 7 Sep 2019 17:37:41 +0200
Subject: [PATCH 0452/1550] Remove lost workers from SpecCluster.workers
 (#2990)

* Detect workers end in SpecCluster

* Make worker deletion checks robust to missing worker

* Convert integer names in dask-worker CLI

* lint

* remove logged warning (this happens under normal operation)

* Make sure that we close workers before we delete them

* Add a moderate delay before closing worker job
---
 distributed/cli/dask_worker.py                |  6 +++-
 distributed/cli/tests/test_dask_worker.py     | 15 ++++++--
 distributed/deploy/cluster.py                 | 22 +++++++-----
 distributed/deploy/spec.py                    | 33 +++++++++++++++---
 distributed/deploy/tests/test_spec_cluster.py | 34 ++++++++++++++++++-
 distributed/distributed.yaml                  |  3 ++
 6 files changed, 96 insertions(+), 17 deletions(-)

diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
index 790b8b3a9ab..952ba90984a 100755
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -8,6 +8,7 @@
 
 import click
 import dask
+from dask.utils import ignoring
 from distributed import Nanny, Worker
 from distributed.security import Security
 from distributed.cli.utils import check_python_3, install_signal_handlers
@@ -354,6 +355,9 @@ def del_pid_file():
             "dask-worker SCHEDULER_ADDRESS:8786"
         )
 
+    with ignoring(TypeError, ValueError):
+        name = int(name)
+
     nannies = [
         t(
             scheduler,
@@ -367,7 +371,7 @@ def del_pid_file():
             port=port,
             dashboard_address=dashboard_address if dashboard else None,
             service_kwargs={"dashboard": {"prefix": dashboard_prefix}},
-            name=name if nprocs == 1 or not name else name + "-" + str(i),
+            name=name if nprocs == 1 or not name else str(name) + "-" + str(i),
             **kwargs
         )
         for i in range(nprocs)
diff --git a/distributed/cli/tests/test_dask_worker.py b/distributed/cli/tests/test_dask_worker.py
index 2dd74737b16..0e871cf1b60 100644
--- a/distributed/cli/tests/test_dask_worker.py
+++ b/distributed/cli/tests/test_dask_worker.py
@@ -1,3 +1,4 @@
+import asyncio
 import pytest
 from click.testing import CliRunner
 
@@ -9,11 +10,11 @@
 from time import sleep
 
 import distributed.cli.dask_worker
-from distributed import Client
+from distributed import Client, Scheduler
 from distributed.metrics import time
 from distributed.utils import sync, tmpfile
 from distributed.utils_test import popen, terminate_process, wait_for_port
-from distributed.utils_test import loop  # noqa: F401
+from distributed.utils_test import loop, cleanup  # noqa: F401
 
 
 def test_nanny_worker_ports(loop):
@@ -330,3 +331,13 @@ def test_bokeh_deprecation():
         except ValueError:
             # didn't pass scheduler
             pass
+
+
+@pytest.mark.asyncio
+async def test_integer_names(cleanup):
+    async with Scheduler(port=0) as s:
+        with popen(["dask-worker", s.address, "--name", "123"]) as worker:
+            while not s.workers:
+                await asyncio.sleep(0.01)
+            [ws] = s.workers.values()
+            assert ws.name == 123
diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
index 393be849b88..32ceedc47bd 100644
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -95,18 +95,22 @@ async def _watch_worker_status(self, comm):
             except OSError:
                 break
 
-            for op, msg in msgs:
-                if op == "add":
-                    workers = msg.pop("workers")
-                    self.scheduler_info["workers"].update(workers)
-                    self.scheduler_info.update(msg)
-                elif op == "remove":
-                    del self.scheduler_info["workers"][msg]
-                else:
-                    raise ValueError("Invalid op", op, msg)
+            with log_errors():
+                for op, msg in msgs:
+                    self._update_worker_status(op, msg)
 
         await comm.close()
 
+    def _update_worker_status(self, op, msg):
+        if op == "add":
+            workers = msg.pop("workers")
+            self.scheduler_info["workers"].update(workers)
+            self.scheduler_info.update(msg)
+        elif op == "remove":
+            del self.scheduler_info["workers"][msg]
+        else:
+            raise ValueError("Invalid op", op, msg)
+
     def adapt(self, Adaptive=Adaptive, **kwargs) -> Adaptive:
         """ Turn on adaptivity
 
diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index d897dc6b7df..487e5192e17 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -1,18 +1,23 @@
 import asyncio
 import atexit
 import copy
+import logging
 import math
 import weakref
 
+import dask
 from tornado import gen
 
 from .cluster import Cluster
 from ..core import rpc, CommClosedError
-from ..utils import LoopRunner, silence_logging, ignoring, parse_bytes
+from ..utils import LoopRunner, silence_logging, ignoring, parse_bytes, parse_timedelta
 from ..scheduler import Scheduler
 from ..security import Security
 
 
+logger = logging.getLogger(__name__)
+
+
 class ProcessInterface:
     """
     An interface for Scheduler and Worker processes for use in SpecCluster
@@ -201,6 +206,7 @@ def __init__(
         self._i = 0
         self.security = security or Security()
         self.scheduler_comm = None
+        self._futures = set()
 
         if silence_logs:
             self._old_logging_level = silence_logging(level=silence_logs)
@@ -267,13 +273,14 @@ async def _correct_state_internal(self):
             if to_close:
                 if self.scheduler.status == "running":
                     await self.scheduler_comm.retire_workers(workers=list(to_close))
-                tasks = [self.workers[w].close() for w in to_close]
+                tasks = [self.workers[w].close() for w in to_close if w in self.workers]
                 await asyncio.wait(tasks)
                 for task in tasks:  # for tornado gen.coroutine support
                     with ignoring(RuntimeError):
                         await task
             for name in to_close:
-                del self.workers[name]
+                if name in self.workers:
+                    del self.workers[name]
 
             to_open = set(self.worker_spec) - set(self.workers)
             workers = []
@@ -293,6 +300,22 @@ async def _correct_state_internal(self):
                     await w  # for tornado gen.coroutine support
             self.workers.update(dict(zip(to_open, workers)))
 
+    def _update_worker_status(self, op, msg):
+        if op == "remove":
+            name = self.scheduler_info["workers"][msg]["name"]
+
+            def f():
+                if name in self.workers and msg not in self.scheduler_info:
+                    self._futures.add(asyncio.ensure_future(self.workers[name].close()))
+                    del self.workers[name]
+
+            delay = parse_timedelta(
+                dask.config.get("distributed.deploy.lost-worker-timeout")
+            )
+
+            asyncio.get_event_loop().call_later(delay, f)
+        super()._update_worker_status(op, msg)
+
     def __await__(self):
         async def _():
             if self.status == "created":
@@ -314,13 +337,15 @@ async def _close(self):
 
         self.scale(0)
         await self._correct_state()
+        for future in self._futures:
+            await future
         async with self._lock:
             with ignoring(CommClosedError):
                 await self.scheduler_comm.close(close_workers=True)
 
         await self.scheduler.close()
         for w in self._created:
-            assert w.status == "closed"
+            assert w.status == "closed", w.status
 
         if hasattr(self, "_old_logging_level"):
             silence_logging(self._old_logging_level)
diff --git a/distributed/deploy/tests/test_spec_cluster.py b/distributed/deploy/tests/test_spec_cluster.py
index efc231ca030..679adc0fd60 100644
--- a/distributed/deploy/tests/test_spec_cluster.py
+++ b/distributed/deploy/tests/test_spec_cluster.py
@@ -1,8 +1,9 @@
 import asyncio
-from time import time
 
+import dask
 from dask.distributed import SpecCluster, Worker, Client, Scheduler, Nanny
 from distributed.deploy.spec import close_clusters, ProcessInterface
+from distributed.metrics import time
 from distributed.utils_test import loop, cleanup  # noqa: F401
 from distributed.utils import is_valid_xml
 import toolz
@@ -124,6 +125,37 @@ async def test_scale(cleanup):
         assert len(cluster.workers) == 1
 
 
+@pytest.mark.asyncio
+async def test_unexpected_closed_worker(cleanup):
+    worker = {"cls": Worker, "options": {"nthreads": 1}}
+    with dask.config.set({"distributed.deploy.lost-worker-timeout": "10ms"}):
+        async with SpecCluster(
+            asynchronous=True, scheduler=scheduler, worker=worker
+        ) as cluster:
+            assert not cluster.workers
+            assert not cluster.worker_spec
+
+            # Scale up
+            cluster.scale(2)
+            assert not cluster.workers
+            assert cluster.worker_spec
+
+            await cluster
+            assert len(cluster.workers) == 2
+
+            # Close one
+            await list(cluster.workers.values())[0].close()
+            start = time()
+            while len(cluster.workers) > 1:  # wait for messages to flow around
+                await asyncio.sleep(0.01)
+                assert time() < start + 2
+            assert len(cluster.workers) == 1
+            assert len(cluster.worker_spec) == 2
+
+            await cluster
+            assert len(cluster.workers) == 2
+
+
 @pytest.mark.asyncio
 async def test_broken_worker():
     with pytest.raises(Exception) as info:
diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index a0e801f26ac..f277eb2f90b 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -65,6 +65,9 @@ distributed:
   client:
     heartbeat: 5s  # time between client heartbeats
 
+  deploy:
+    lost-worker-timeout: 15s  # Interval after which to hard-close a lost worker job
+
   comm:
     compression: auto
     default-scheme: tcp

From f0ccd366124f45da2bc86802d50684464b14fcb8 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sun, 8 Sep 2019 15:59:30 -0700
Subject: [PATCH 0453/1550] Support --name 0 and --nprocs keywords in
 dask-worker cli (#3037)

Previously a test for `not name` would incorrectly pass if the user
provided `--name 0`
---
 distributed/cli/dask_worker.py            | 4 +++-
 distributed/cli/tests/test_dask_worker.py | 4 ++--
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
index 952ba90984a..badecc4dad8 100755
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -371,7 +371,9 @@ def del_pid_file():
             port=port,
             dashboard_address=dashboard_address if dashboard else None,
             service_kwargs={"dashboard": {"prefix": dashboard_prefix}},
-            name=name if nprocs == 1 or not name else str(name) + "-" + str(i),
+            name=name
+            if nprocs == 1 or name is None or name == ""
+            else str(name) + "-" + str(i),
             **kwargs
         )
         for i in range(nprocs)
diff --git a/distributed/cli/tests/test_dask_worker.py b/distributed/cli/tests/test_dask_worker.py
index 0e871cf1b60..01327d64291 100644
--- a/distributed/cli/tests/test_dask_worker.py
+++ b/distributed/cli/tests/test_dask_worker.py
@@ -181,7 +181,7 @@ def test_nprocs_requires_nanny(loop):
 def test_nprocs_expands_name(loop):
     with popen(["dask-scheduler", "--no-dashboard"]) as sched:
         with popen(
-            ["dask-worker", "127.0.0.1:8786", "--nprocs", "2", "--name", "foo"]
+            ["dask-worker", "127.0.0.1:8786", "--nprocs", "2", "--name", "0"]
         ) as worker:
             with popen(["dask-worker", "127.0.0.1:8786", "--nprocs", "2"]) as worker:
                 with Client("tcp://127.0.0.1:8786", loop=loop) as c:
@@ -192,7 +192,7 @@ def test_nprocs_expands_name(loop):
 
                     info = c.scheduler_info()
                     names = [d["name"] for d in info["workers"].values()]
-                    foos = [n for n in names if n.startswith("foo")]
+                    foos = [n for n in names if n.startswith("0-")]
                     assert len(foos) == 2
                     assert len(set(names)) == 4
 

From fb733478bd30d71f0889119ec3859993dfa0315f Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sun, 8 Sep 2019 17:42:00 -0700
Subject: [PATCH 0454/1550] Redirect configuration doc page (#3038)

---
 docs/source/conf.py           |   3 +-
 docs/source/configuration.rst | 179 ----------------------------------
 2 files changed, 2 insertions(+), 180 deletions(-)
 delete mode 100644 docs/source/configuration.rst

diff --git a/docs/source/conf.py b/docs/source/conf.py
index bb3361851b8..9bd0ce6867e 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -391,7 +391,8 @@
     # old html, new html
     ("joblib.html", "https://ml.dask.org/joblib.html"),
     ("setup.html", "https://docs.dask.org/en/latest/setup.html"),
-    ("ec2.html", "https://dask.pydata.org/en/latest/setup/cloud.html"),
+    ("ec2.html", "https://docs.dask.org/en/latest/setup/cloud.html"),
+    ("configuration.html", "https://docs.dask.org/en/latest/configuration.html"),
 ]
 
 
diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst
deleted file mode 100644
index 8967255f526..00000000000
--- a/docs/source/configuration.rst
+++ /dev/null
@@ -1,179 +0,0 @@
-.. _configuration:
-
-=============
-Configuration
-=============
-
-As with any distributed computation system, taking full advantage of
-Dask distributed sometimes requires configuration.  Some options can be
-passed as :ref:`API <api>` parameters and/or command line options to the
-various Dask executables.  However, some options can also be entered in
-the Dask configuration file.
-
-
-User-wide configuration
-=======================
-
-Dask accepts some configuration options in a configuration file, which by
-default is a ``.dask/config.yaml`` file located in your home directory.
-The file path can be overriden using the ``DASK_CONFIG`` environment variable.
-In order to parse this configuration file, the ``pyyaml`` module needs to be
-installed. If the ``pyyaml`` module is not installed, the configuration file
-is ignored.
-
-The file is written in the YAML format, which allows for a human-readable
-hierarchical key-value configuration.  All keys in the configuration file
-are optional, though Dask will create a default configuration file for you
-on its first launch.
-
-Here is a synopsis of the configuration file:
-
-.. code-block:: yaml
-
-   logging:
-       distributed: info
-       distributed.client: warning
-       bokeh: critical
-
-   # Scheduler options
-   bandwidth: 100000000    # 100 MB/s estimated worker-worker bandwidth
-   allowed-failures: 3     # number of retries before a task is considered bad
-   pdb-on-err: False       # enter debug mode on scheduling error
-   transition-log-length: 100000
-
-   # Worker options
-   multiprocessing-method: forkserver
-
-   # Communication options
-   compression: auto
-   tcp-timeout: 30         # seconds delay before calling an unresponsive connection dead
-   default-scheme: tcp
-   require-encryption: False   # whether to require encryption on non-local comms
-   tls:
-       ca-file: myca.pem
-       scheduler:
-           cert: mycert.pem
-           key: mykey.pem
-       worker:
-           cert: mycert.pem
-           key: mykey.pem
-       client:
-           cert: mycert.pem
-           key: mykey.pem
-       #ciphers:
-           #ECDHE-ECDSA-AES128-GCM-SHA256
-
-   # Bokeh web dashboard
-   bokeh-export-tool: False
-
-
-We will review some of those options hereafter.
-
-
-Communication options
----------------------
-
-``compression``
-"""""""""""""""
-
-This key configures the desired compression scheme when transferring data
-over the network.  The default value, "auto", applies heuristics to try and
-select the best compression scheme for each piece of data.
-
-
-``default-scheme``
-""""""""""""""""""
-
-The :ref:`communication <communications>` scheme used by default.  You can
-override the default ("tcp") here, but it is recommended to use explicit URIs
-for the various endpoints instead (for example ``tls://`` if you want to
-enable :ref:`TLS <tls>` communications).
-
-
-``require-encryption``
-""""""""""""""""""""""
-
-Whether to require that all non-local communications be encrypted.  If true,
-then Dask will refuse establishing any clear-text communications (for example
-over TCP without TLS), forcing you to use a secure transport such as
-:ref:`TLS <tls>`.
-
-
-``tcp-timeout``
-"""""""""""""""
-
-The default "timeout" on TCP sockets.  If a remote endpoint is unresponsive
-(at the TCP layer, not at the distributed layer) for at least the specified
-number of seconds, the communication is considered closed.  This helps detect
-endpoints that have been killed or have disconnected abruptly.
-
-
-``tls``
-"""""""
-
-This key configures :ref:`TLS <tls>` communications.  Several sub-keys are
-recognized:
-
-* ``ca-file`` configures the CA certificate file used to authenticate
-  and authorize all endpoints.
-* ``ciphers`` restricts allowed ciphers on TLS communications.
-
-Each kind of endpoint has a dedicated endpoint sub-key: ``scheduler``,
-``worker`` and ``client``.  Each endpoint sub-key also supports several
-sub-keys:
-
-* ``cert`` configures the certificate file for the endpoint.
-* ``key`` configures the private key file for the endpoint.
-
-
-Scheduler options
------------------
-
-``allowed-failures``
-""""""""""""""""""""
-
-The number of retries before a "suspicious" task is considered bad.
-A task is considered "suspicious" if the worker died while executing it.
-
-
-``bandwidth``
-"""""""""""""
-
-The estimated network bandwidth, in bytes per second, from worker to worker.
-This value is used to estimate the time it takes to ship data from one node
-to another, and balance tasks and data accordingly.
-
-
-Misc options
-------------
-
-``logging``
-"""""""""""
-
-This key configures the logging settings.  There are two possible formats.
-The simple, recommended format configures the desired verbosity level
-for each logger.  It also sets default values for several loggers such
-as ``distributed`` unless explicitly configured.
-
-A more extended format is possible following the :mod:`logging` module's
-`Configuration dictionary schema <https://docs.python.org/3/library/logging.config.html#configuration-dictionary-schema>`_.
-To enable this extended format, there must be a ``version`` sub-key as
-mandated by the schema.  The extended format does not set any default values.
-
-.. note::
-   Python's :mod:`logging` module uses a hierarchical logger tree.
-   For example, configuring the logging level for the ``distributed``
-   logger will also affect its children such as ``distributed.scheduler``,
-   unless explicitly overriden.
-
-
-``logging-file-config``
-"""""""""""""""""""""""
-
-As an alternative to the two logging settings formats discussed above,
-you can specify a logging config file.
-Its format adheres to the :mod:`logging` module's
-`Configuration file format <https://docs.python.org/3/howto/logging.html#configuring-logging>`_.
-
-.. note::
-   The configuration options `logging-file-config` and `logging` are mutually exclusive.
\ No newline at end of file

From d419d41eebdc64d05336284ade8104a49d503e55 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Mon, 9 Sep 2019 22:51:21 +0200
Subject: [PATCH 0455/1550] Drop joblib shim module in distributed (#3040)

This already just raises an `ImportError` and has done so for a while.
Seems reasonable to just drop the module and let Python raise the
`ImportError` that it would anyways. Likely everyone has migrated to
this convention by now.
---
 distributed/joblib.py | 19 -------------------
 1 file changed, 19 deletions(-)
 delete mode 100644 distributed/joblib.py

diff --git a/distributed/joblib.py b/distributed/joblib.py
deleted file mode 100644
index fd81a8b078e..00000000000
--- a/distributed/joblib.py
+++ /dev/null
@@ -1,19 +0,0 @@
-msg = """ It is no longer necessary to `import dask_ml.joblib` or
-`import distributed.joblib`.
-
-This functionality has moved into the core Joblib codebase.
-
-To use Joblib's Dask backend with Scikit-Learn >= 0.20.0
-
-    from dask.distributed import Client
-    client = Client()
-
-    from sklearn.externals import joblib
-
-    with joblib.parallel_backend('dask'):
-        # your scikit-learn code
-
-See http://ml.dask.org/joblib.html for more information."""
-
-
-raise ImportError(msg)

From 810808e9ce4b3efeff1afc99e8fd8a9430f08a8c Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Mon, 9 Sep 2019 18:04:08 -0500
Subject: [PATCH 0456/1550] Move task deserialization to immediately before
 task execution (#3015)

This commit moves task deserialization on workers to just before the task is marked for execution on the worker executor.

This has a couple of benefits:

- In the case that a task is stolen, there won't be any unnecessary deserialization

- In the case of single-threaded workers, this will help reduce issues with task deserialization interfering with concurrently running tasks on the executor (ref #2965). This is because tasks are only transitioned to executing when the number of executing tasks is less than the number of executor theads https://github.com/dask/distributed/blob/cf26e1a559e2c89c4a4b14b6622111eaf0954f12/distributed/worker.py#L2361 IIUC this means, for single-threaded executors, tasks will only transition to executing when there are no currently executing tasks

On the other hand, a drawback to this approach is that deserialization errors are not immediately raised.
---
 distributed/worker.py | 54 ++++++++++++++++++++++++++++---------------
 1 file changed, 36 insertions(+), 18 deletions(-)

diff --git a/distributed/worker.py b/distributed/worker.py
index ca4f4121af3..06102a98405 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -1,6 +1,6 @@
 import asyncio
 import bisect
-from collections import defaultdict, deque
+from collections import defaultdict, deque, namedtuple
 from collections.abc import MutableMapping
 from datetime import timedelta
 import heapq
@@ -86,6 +86,8 @@
 
 DEFAULT_STARTUP_INFORMATION = {}
 
+SerializedTask = namedtuple("SerializedTask", ["function", "args", "kwargs", "task"])
+
 
 class Worker(ServerNode):
     """ Worker node in a Dask distributed cluster
@@ -1330,23 +1332,9 @@ def add_task(
                 return
 
             self.log.append((key, "new"))
-            try:
-                start = time()
-                self.tasks[key] = _deserialize(function, args, kwargs, task)
-                if actor:
-                    self.actors[key] = None
-                stop = time()
-
-                if stop - start > 0.010:
-                    self.startstops[key].append(("deserialize", start, stop))
-            except Exception as e:
-                logger.warning("Could not deserialize task", exc_info=True)
-                emsg = error_message(e)
-                emsg["key"] = key
-                emsg["op"] = "task-erred"
-                self.batched_stream.send(emsg)
-                self.log.append((key, "deserialize-error"))
-                return
+            self.tasks[key] = SerializedTask(function, args, kwargs, task)
+            if actor:
+                self.actors[key] = None
 
             self.priorities[key] = priority
             self.durations[key] = duration
@@ -2344,6 +2332,26 @@ def meets_resource_constraints(self, key):
 
         return True
 
+    def _maybe_deserialize_task(self, key):
+        if not isinstance(self.tasks[key], SerializedTask):
+            return self.tasks[key]
+        try:
+            start = time()
+            function, args, kwargs = _deserialize(*self.tasks[key])
+            stop = time()
+
+            if stop - start > 0.010:
+                self.startstops[key].append(("deserialize", start, stop))
+            return function, args, kwargs
+        except Exception as e:
+            logger.warning("Could not deserialize task", exc_info=True)
+            emsg = error_message(e)
+            emsg["key"] = key
+            emsg["op"] = "task-erred"
+            self.batched_stream.send(emsg)
+            self.log.append((key, "deserialize-error"))
+            raise
+
     def ensure_computing(self):
         if self.paused:
             return
@@ -2355,12 +2363,22 @@ def ensure_computing(self):
                     continue
                 if self.meets_resource_constraints(key):
                     self.constrained.popleft()
+                    try:
+                        # Ensure task is deserialized prior to execution
+                        self.tasks[key] = self._maybe_deserialize_task(key)
+                    except Exception:
+                        continue
                     self.transition(key, "executing")
                 else:
                     break
             while self.ready and len(self.executing) < self.nthreads:
                 _, key = heapq.heappop(self.ready)
                 if self.task_state.get(key) in READY:
+                    try:
+                        # Ensure task is deserialized prior to execution
+                        self.tasks[key] = self._maybe_deserialize_task(key)
+                    except Exception:
+                        continue
                     self.transition(key, "executing")
         except Exception as e:
             logger.exception(e)

From eecf25bd55b8bd6b58a3ee9c43e6e65f784ec4a4 Mon Sep 17 00:00:00 2001
From: Jim Crist <jcrist@users.noreply.github.com>
Date: Tue, 10 Sep 2019 13:26:18 -0500
Subject: [PATCH 0457/1550] Use cgroups resource limits to determine default
 threads and memory (#3039)

This adds support for detecting resource (CPU and memory) limits set
using `cgroups`. This makes dask resource detection play nicer with
container systems (e.g. `docker`), rather than detecting the host memory
and cpus avaialable.

This also centralizes all queries about the host platform to a single
module (`distributed.platform`), with top-level constants defined for
common usage.
---
 distributed/cli/dask_worker.py              |  4 +-
 distributed/comm/tcp.py                     | 12 +---
 distributed/core.py                         | 11 ---
 distributed/deploy/local.py                 | 12 ++--
 distributed/deploy/tests/test_local.py      | 25 +++----
 distributed/nanny.py                        |  4 +-
 distributed/protocol/tests/test_numpy.py    | 10 +--
 distributed/protocol/tests/test_protocol.py | 13 ++--
 distributed/system.py                       | 79 +++++++++++++++++++++
 distributed/tests/test_steal.py             |  6 +-
 distributed/tests/test_system.py            | 32 +++++++++
 distributed/tests/test_worker.py            | 30 +++-----
 distributed/utils_test.py                   | 11 ++-
 distributed/worker.py                       | 28 ++------
 14 files changed, 168 insertions(+), 109 deletions(-)
 create mode 100644 distributed/system.py
 create mode 100644 distributed/tests/test_system.py

diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
index badecc4dad8..35ca11da34c 100755
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -1,6 +1,5 @@
 import atexit
 import logging
-import multiprocessing
 import gc
 import os
 from sys import exit
@@ -11,6 +10,7 @@
 from dask.utils import ignoring
 from distributed import Nanny, Worker
 from distributed.security import Security
+from distributed.system import CPU_COUNT
 from distributed.cli.utils import check_python_3, install_signal_handlers
 from distributed.comm import get_address_host_port
 from distributed.preloading import validate_preload_argv
@@ -316,7 +316,7 @@ def main(
         port = worker_port
 
     if not nthreads:
-        nthreads = multiprocessing.cpu_count() // nprocs
+        nthreads = CPU_COUNT // nprocs
 
     if pid_file:
         with open(pid_file, "w") as f:
diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py
index bd76d0e6946..d0322e151d7 100644
--- a/distributed/comm/tcp.py
+++ b/distributed/comm/tcp.py
@@ -18,6 +18,7 @@
 from tornado.tcpclient import TCPClient
 from tornado.tcpserver import TCPServer
 
+from ..system import MEMORY_LIMIT
 from ..threadpoolexecutor import ThreadPoolExecutor
 from ..utils import (
     ensure_bytes,
@@ -38,16 +39,7 @@
 logger = logging.getLogger(__name__)
 
 
-def get_total_physical_memory():
-    try:
-        import psutil
-
-        return psutil.virtual_memory().total / 2
-    except ImportError:
-        return 2e9
-
-
-MAX_BUFFER_SIZE = get_total_physical_memory()
+MAX_BUFFER_SIZE = MEMORY_LIMIT / 2
 
 
 def set_tcp_timeout(stream):
diff --git a/distributed/core.py b/distributed/core.py
index 6bda9c9e0be..5da103802d2 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -47,15 +47,6 @@ class RPCClosed(IOError):
 logger = logging.getLogger(__name__)
 
 
-def get_total_physical_memory():
-    try:
-        import psutil
-
-        return psutil.virtual_memory().total / 2
-    except ImportError:
-        return 2e9
-
-
 def raise_later(exc):
     def _raise(*args, **kwargs):
         raise exc
@@ -63,8 +54,6 @@ def _raise(*args, **kwargs):
     return _raise
 
 
-MAX_BUFFER_SIZE = get_total_physical_memory()
-
 tick_maximum_delay = parse_timedelta(
     dask.config.get("distributed.admin.tick.limit"), default="ms"
 )
diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index efe5ed03098..29c344f6719 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -1,12 +1,12 @@
 import atexit
 import logging
 import math
-import multiprocessing
 import warnings
 import weakref
 
 from dask.utils import factors
 
+from .. import system
 from .spec import SpecCluster
 from ..nanny import Nanny
 from ..scheduler import Scheduler
@@ -146,14 +146,12 @@ def __init__(
                 n_workers, threads_per_worker = nprocesses_nthreads()
             else:
                 n_workers = 1
-                threads_per_worker = multiprocessing.cpu_count()
+                threads_per_worker = system.CPU_COUNT
         if n_workers is None and threads_per_worker is not None:
-            n_workers = max(1, multiprocessing.cpu_count() // threads_per_worker)
+            n_workers = max(1, system.CPU_COUNT // threads_per_worker)
         if n_workers and threads_per_worker is None:
             # Overcommit threads per worker, rather than undercommit
-            threads_per_worker = max(
-                1, int(math.ceil(multiprocessing.cpu_count() / n_workers))
-            )
+            threads_per_worker = max(1, int(math.ceil(system.CPU_COUNT / n_workers)))
         if n_workers and "memory_limit" not in worker_kwargs:
             worker_kwargs["memory_limit"] = parse_memory_limit("auto", 1, n_workers)
 
@@ -208,7 +206,7 @@ def start_worker(self, *args, **kwargs):
         )
 
 
-def nprocesses_nthreads(n=multiprocessing.cpu_count()):
+def nprocesses_nthreads(n=system.CPU_COUNT):
     """
     The default breakdown of processes and threads for a given number of cores
 
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index 5459574cccf..6611bfccc38 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -1,6 +1,5 @@
 from functools import partial
 import gc
-import multiprocessing
 import subprocess
 import sys
 from time import sleep
@@ -15,6 +14,7 @@
 from distributed import Client, Worker, Nanny, get_client
 from distributed.deploy.local import LocalCluster, nprocesses_nthreads
 from distributed.metrics import time
+from distributed.system import CPU_COUNT, MEMORY_LIMIT
 from distributed.utils_test import (  # noqa: F401
     clean,
     cleanup,
@@ -29,7 +29,6 @@
 )
 from distributed.utils_test import loop  # noqa: F401
 from distributed.utils import sync
-from distributed.worker import TOTAL_MEMORY
 
 from distributed.deploy.utils_test import ClusterTest
 
@@ -230,9 +229,7 @@ async def test_defaults(cleanup):
     async with LocalCluster(
         scheduler_port=0, silence_logs=False, dashboard_address=None, asynchronous=True
     ) as c:
-        assert (
-            sum(w.nthreads for w in c.workers.values()) == multiprocessing.cpu_count()
-        )
+        assert sum(w.nthreads for w in c.workers.values()) == CPU_COUNT
         assert all(isinstance(w, Nanny) for w in c.workers.values())
 
 
@@ -245,9 +242,7 @@ async def test_defaults_2(cleanup):
         dashboard_address=None,
         asynchronous=True,
     ) as c:
-        assert (
-            sum(w.nthreads for w in c.workers.values()) == multiprocessing.cpu_count()
-        )
+        assert sum(w.nthreads for w in c.workers.values()) == CPU_COUNT
         assert all(isinstance(w, Worker) for w in c.workers.values())
         assert len(c.workers) == 1
 
@@ -261,18 +256,18 @@ async def test_defaults_3(cleanup):
         dashboard_address=None,
         asynchronous=True,
     ) as c:
-        if multiprocessing.cpu_count() % 2 == 0:
-            expected_total_threads = max(2, multiprocessing.cpu_count())
+        if CPU_COUNT % 2 == 0:
+            expected_total_threads = max(2, CPU_COUNT)
         else:
             # n_workers not a divisor of _nthreads => threads are overcommitted
-            expected_total_threads = max(2, multiprocessing.cpu_count() + 1)
+            expected_total_threads = max(2, CPU_COUNT + 1)
         assert sum(w.nthreads for w in c.workers.values()) == expected_total_threads
 
 
 @pytest.mark.asyncio
 async def test_defaults_4(cleanup):
     async with LocalCluster(
-        threads_per_worker=multiprocessing.cpu_count() * 2,
+        threads_per_worker=CPU_COUNT * 2,
         scheduler_port=0,
         silence_logs=False,
         dashboard_address=None,
@@ -284,7 +279,7 @@ async def test_defaults_4(cleanup):
 @pytest.mark.asyncio
 async def test_defaults_5(cleanup):
     async with LocalCluster(
-        n_workers=multiprocessing.cpu_count() * 2,
+        n_workers=CPU_COUNT * 2,
         scheduler_port=0,
         silence_logs=False,
         dashboard_address=None,
@@ -473,7 +468,7 @@ def test_memory(loop, n_workers):
         dashboard_address=None,
         loop=loop,
     ) as cluster:
-        assert sum(w.memory_limit for w in cluster.workers.values()) <= TOTAL_MEMORY
+        assert sum(w.memory_limit for w in cluster.workers.values()) <= MEMORY_LIMIT
 
 
 @pytest.mark.parametrize("n_workers", [None, 3])
@@ -489,7 +484,7 @@ def test_memory_nanny(loop, n_workers):
         with Client(cluster.scheduler_address, loop=loop) as c:
             info = c.scheduler_info()
             assert (
-                sum(w["memory_limit"] for w in info["workers"].values()) <= TOTAL_MEMORY
+                sum(w["memory_limit"] for w in info["workers"].values()) <= MEMORY_LIMIT
             )
 
 
diff --git a/distributed/nanny.py b/distributed/nanny.py
index b6d8dadbf9a..c017eb54af7 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -1,7 +1,6 @@
 from datetime import timedelta
 import logging
 from multiprocessing.queues import Empty
-import multiprocessing
 import os
 import psutil
 import shutil
@@ -23,6 +22,7 @@
 from .process import AsyncProcess
 from .proctitle import enable_proctitle_on_children
 from .security import Security
+from .system import CPU_COUNT
 from .utils import (
     get_ip,
     mp_context,
@@ -110,7 +110,7 @@ def __init__(
             nthreads = ncores
 
         self._given_worker_port = worker_port
-        self.nthreads = nthreads or multiprocessing.cpu_count()
+        self.nthreads = nthreads or CPU_COUNT
         self.reconnect = reconnect
         self.validate = validate
         self.resources = resources
diff --git a/distributed/protocol/tests/test_numpy.py b/distributed/protocol/tests/test_numpy.py
index b334683b661..70d57a2e74f 100644
--- a/distributed/protocol/tests/test_numpy.py
+++ b/distributed/protocol/tests/test_numpy.py
@@ -14,10 +14,11 @@
     msgpack,
 )
 from distributed.protocol.utils import BIG_BYTES_SHARD_SIZE
-from distributed.utils import tmpfile, nbytes
-from distributed.utils_test import gen_cluster
 from distributed.protocol.numpy import itemsize
 from distributed.protocol.compression import maybe_compress
+from distributed.system import MEMORY_LIMIT
+from distributed.utils import tmpfile, nbytes
+from distributed.utils_test import gen_cluster
 
 
 def test_serialize():
@@ -151,9 +152,8 @@ def test_memmap():
 
 @pytest.mark.slow
 def test_dumps_serialize_numpy_large():
-    psutil = pytest.importorskip("psutil")
-    if psutil.virtual_memory().total < 2e9:
-        return
+    if MEMORY_LIMIT < 2e9:
+        pytest.skip("insufficient memory")
     x = np.random.random(size=int(BIG_BYTES_SHARD_SIZE * 2 // 8)).view("u1")
     assert x.nbytes == BIG_BYTES_SHARD_SIZE * 2
     frames = dumps([to_serialize(x)])
diff --git a/distributed/protocol/tests/test_protocol.py b/distributed/protocol/tests/test_protocol.py
index 3dd11ecc4d1..bf16aecf2f4 100644
--- a/distributed/protocol/tests/test_protocol.py
+++ b/distributed/protocol/tests/test_protocol.py
@@ -6,6 +6,7 @@
 from distributed.protocol import loads, dumps, msgpack, maybe_compress, to_serialize
 from distributed.protocol.compression import compressions
 from distributed.protocol.serialize import Serialize, Serialized, serialize, deserialize
+from distributed.system import MEMORY_LIMIT
 from distributed.utils import nbytes
 
 
@@ -102,13 +103,9 @@ def test_large_bytes():
 @pytest.mark.slow
 def test_large_messages():
     np = pytest.importorskip("numpy")
-    psutil = pytest.importorskip("psutil")
     pytest.importorskip("lz4")
-    if psutil.virtual_memory().total < 8e9:
-        return
-
-    if sys.version_info.major == 2:
-        return 2
+    if MEMORY_LIMIT < 8e9:
+        pytest.skip("insufficient memory")
 
     x = np.random.randint(0, 255, size=200000000, dtype="u1")
 
@@ -126,9 +123,7 @@ def test_large_messages():
 
 
 def test_large_messages_map():
-    import psutil
-
-    if psutil.virtual_memory().total < 8e9:
+    if MEMORY_LIMIT < 8e9:
         pytest.skip("insufficient memory")
 
     x = {i: "mystring_%d" % i for i in range(100000)}
diff --git a/distributed/system.py b/distributed/system.py
new file mode 100644
index 00000000000..e0735e1e34a
--- /dev/null
+++ b/distributed/system.py
@@ -0,0 +1,79 @@
+import os
+import sys
+
+import psutil
+
+__all__ = ("memory_limit", "cpu_count", "MEMORY_LIMIT", "CPU_COUNT")
+
+
+def memory_limit():
+    """Get the memory limit (in bytes) for this system.
+
+    Takes the minimum value from the following locations:
+
+    - Total system host memory
+    - Cgroups limit (if set)
+    - RSS rlimit (if set)
+    """
+    limit = psutil.virtual_memory().total
+
+    # Check cgroups if available
+    if sys.platform == "linux":
+        try:
+            with open("/sys/fs/cgroup/memory/memory.limit_in_bytes") as f:
+                cgroups_limit = int(f.read())
+            if cgroups_limit > 0:
+                limit = min(limit, cgroups_limit)
+        except Exception:
+            pass
+
+    # Check rlimit if available
+    try:
+        import resource
+
+        hard_limit = resource.getrlimit(resource.RLIMIT_RSS)[1]
+        if hard_limit > 0:
+            limit = min(limit, hard_limit)
+    except (ImportError, OSError):
+        pass
+
+    return limit
+
+
+def cpu_count():
+    """Get the available CPU count for this system.
+
+    Takes the minimum value from the following locations:
+
+    - Total system cpus available on the host.
+    - CPU Affinity (if set)
+    - Cgroups limit (if set)
+    """
+    count = os.cpu_count()
+
+    # Check CPU affinity if available
+    try:
+        affinity_count = len(psutil.Process().cpu_affinity())
+        if affinity_count > 0:
+            count = min(count, affinity_count)
+    except Exception:
+        pass
+
+    # Check cgroups if available
+    if sys.platform == "linux":
+        try:
+            with open("/sys/fs/cgroup/cpuacct,cpu/cpu.cfs_quota_us") as f:
+                quota = int(f.read())
+            with open("/sys/fs/cgroup/cpuacct,cpu/cpu.cfs_period_us") as f:
+                period = int(f.read())
+            cgroups_count = int(quota / period)
+            if cgroups_count > 0:
+                count = min(count, cgroups_count)
+        except Exception:
+            pass
+
+    return count
+
+
+MEMORY_LIMIT = memory_limit()
+CPU_COUNT = cpu_count()
diff --git a/distributed/tests/test_steal.py b/distributed/tests/test_steal.py
index d7c396bb63f..6d98e662034 100644
--- a/distributed/tests/test_steal.py
+++ b/distributed/tests/test_steal.py
@@ -13,6 +13,7 @@
 from distributed.config import config
 from distributed.metrics import time
 from distributed.scheduler import key_split
+from distributed.system import MEMORY_LIMIT
 from distributed.utils_test import (
     slowinc,
     slowadd,
@@ -22,7 +23,6 @@
     captured_logger,
 )
 from distributed.utils_test import nodebug_setup_module, nodebug_teardown_module
-from distributed.worker import TOTAL_MEMORY
 
 import pytest
 
@@ -170,7 +170,7 @@ def test_new_worker_steals(c, s, a):
     while len(a.task_state) < 10:
         yield gen.sleep(0.01)
 
-    b = yield Worker(s.address, loop=s.loop, nthreads=1, memory_limit=TOTAL_MEMORY)
+    b = yield Worker(s.address, loop=s.loop, nthreads=1, memory_limit=MEMORY_LIMIT)
 
     result = yield total
     assert result == sum(map(inc, range(100)))
@@ -335,7 +335,7 @@ def test_dont_steal_few_saturated_tasks_many_workers(c, s, a, *rest):
 @gen_cluster(
     client=True,
     nthreads=[("127.0.0.1", 1)] * 10,
-    worker_kwargs={"memory_limit": TOTAL_MEMORY},
+    worker_kwargs={"memory_limit": MEMORY_LIMIT},
 )
 def test_steal_when_more_tasks(c, s, a, *rest):
     s.extensions["stealing"]._pc.callback_time = 20
diff --git a/distributed/tests/test_system.py b/distributed/tests/test_system.py
new file mode 100644
index 00000000000..d0f00f495e3
--- /dev/null
+++ b/distributed/tests/test_system.py
@@ -0,0 +1,32 @@
+import os
+
+import psutil
+import pytest
+
+from distributed.system import cpu_count, memory_limit
+
+
+def test_cpu_count():
+    count = cpu_count()
+    assert isinstance(count, int)
+    assert count <= os.cpu_count()
+    assert count >= 1
+
+
+def test_memory_limit():
+    limit = memory_limit()
+    assert isinstance(limit, int)
+    assert limit <= psutil.virtual_memory().total
+    assert limit >= 1
+
+
+def test_rlimit():
+    resource = pytest.importorskip("resource")
+
+    # decrease memory limit by one byte
+    new_limit = memory_limit() - 1
+    try:
+        resource.setrlimit(resource.RLIMIT_RSS, (new_limit, new_limit))
+        assert memory_limit() == new_limit
+    except OSError:
+        pytest.skip("resource could not set the RSS limit")
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 13dd92c00e0..aa0de9fd90f 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -2,7 +2,6 @@
 from datetime import timedelta
 import importlib
 import logging
-import multiprocessing
 from numbers import Number
 from operator import add
 import os
@@ -30,6 +29,7 @@
     get_worker,
     Reschedule,
     wait,
+    system,
 )
 from distributed.compatibility import WINDOWS
 from distributed.core import rpc
@@ -62,7 +62,7 @@
 def test_worker_nthreads():
     w = Worker("127.0.0.1", 8019)
     try:
-        assert w.executor._max_workers == multiprocessing.cpu_count()
+        assert w.executor._max_workers == system.CPU_COUNT
     finally:
         shutil.rmtree(w.local_directory)
 
@@ -500,7 +500,7 @@ def test_memory_limit_auto():
     assert isinstance(a.memory_limit, Number)
     assert isinstance(b.memory_limit, Number)
 
-    if multiprocessing.cpu_count() > 1:
+    if system.CPU_COUNT > 1:
         assert a.memory_limit < b.memory_limit
 
     assert c.memory_limit == d.memory_limit
@@ -1436,26 +1436,14 @@ def test_host_address(c, s):
     yield n.close()
 
 
-def test_resource_limit():
+def test_resource_limit(monkeypatch):
     assert parse_memory_limit("250MiB", 1, total_cores=1) == 1024 * 1024 * 250
 
-    # get current limit
-    resource = pytest.importorskip("resource")
-    try:
-        hard_limit = resource.getrlimit(resource.RLIMIT_RSS)[1]
-    except OSError:
-        pytest.skip("resource could not get the RSS limit")
-    memory_limit = psutil.virtual_memory().total
-    if hard_limit > memory_limit or hard_limit < 0:
-        hard_limit = memory_limit
-
-    # decrease memory limit by one byte
-    new_limit = hard_limit - 1
-    try:
-        resource.setrlimit(resource.RLIMIT_RSS, (new_limit, new_limit))
-        assert parse_memory_limit(hard_limit, 1, total_cores=1) == new_limit
-    except OSError:
-        pytest.skip("resource could not set the RSS limit")
+    new_limit = 1024 * 1024 * 200
+    import distributed.worker
+
+    monkeypatch.setattr(distributed.system, "MEMORY_LIMIT", new_limit)
+    assert parse_memory_limit("250MiB", 1, total_cores=1) == new_limit
 
 
 @pytest.mark.asyncio
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index e6b11ce2898..8725e3eb7e0 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -38,6 +38,7 @@
 from tornado.gen import TimeoutError
 from tornado.ioloop import IOLoop
 
+from . import system
 from .client import default_client, _global_clients, Client
 from .compatibility import WINDOWS
 from .comm import Comm
@@ -61,7 +62,7 @@
     thread_state,
     _offload_executor,
 )
-from .worker import Worker, TOTAL_MEMORY
+from .worker import Worker
 from .nanny import Nanny
 
 try:
@@ -636,7 +637,11 @@ def cluster(
             q = mp_context.Queue()
             fn = "_test_worker-%s" % uuid.uuid4()
             kwargs = merge(
-                {"nthreads": 1, "local_directory": fn, "memory_limit": TOTAL_MEMORY},
+                {
+                    "nthreads": 1,
+                    "local_directory": fn,
+                    "memory_limit": system.MEMORY_LIMIT,
+                },
                 worker_kwargs,
             )
             proc = mp_context.Process(
@@ -860,7 +865,7 @@ def test_foo(scheduler, worker1, worker2):
         nthreads = ncores
 
     worker_kwargs = merge(
-        {"memory_limit": TOTAL_MEMORY, "death_timeout": 5}, worker_kwargs
+        {"memory_limit": system.MEMORY_LIMIT, "death_timeout": 5}, worker_kwargs
     )
 
     def _(func):
diff --git a/distributed/worker.py b/distributed/worker.py
index 06102a98405..bb00158ced8 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -5,7 +5,6 @@
 from datetime import timedelta
 import heapq
 import logging
-import multiprocessing
 import os
 from pickle import PicklingError
 import random
@@ -14,7 +13,6 @@
 import uuid
 import warnings
 import weakref
-import psutil
 
 import dask
 from dask.core import istask
@@ -28,7 +26,7 @@
 from tornado import gen
 from tornado.ioloop import IOLoop
 
-from . import profile, comm
+from . import profile, comm, system
 from .batched import BatchedSend
 from .comm import get_address_host, connect
 from .comm.addressing import address_from_user_args
@@ -72,8 +70,6 @@
 
 no_value = "--no-value-sentinel--"
 
-TOTAL_MEMORY = psutil.virtual_memory().total
-
 IN_PLAY = ("waiting", "ready", "executing", "long-running")
 PENDING = ("waiting", "ready", "constrained")
 PROCESSING = ("waiting", "ready", "constrained", "executing", "long-running")
@@ -242,7 +238,7 @@ class Worker(ServerNode):
     memory_limit: int, float, string
         Number of bytes of memory that this worker should use.
         Set to zero for no limit.  Set to 'auto' to calculate
-        as TOTAL_MEMORY * min(1, nthreads / total_cores)
+        as system.MEMORY_LIMIT * min(1, nthreads / total_cores)
         Use strings or numbers like 5GB or 5e9
     memory_target_fraction: float
         Fraction of memory to try to stay beneath
@@ -458,7 +454,7 @@ def __init__(
             warnings.warn("the ncores= parameter has moved to nthreads=")
             nthreads = ncores
 
-        self.nthreads = nthreads or multiprocessing.cpu_count()
+        self.nthreads = nthreads or system.CPU_COUNT
         self.total_resources = resources or {}
         self.available_resources = (resources or {}).copy()
         self.death_timeout = parse_timedelta(death_timeout)
@@ -3042,33 +3038,23 @@ class Reschedule(Exception):
     pass
 
 
-def parse_memory_limit(memory_limit, nthreads, total_cores=multiprocessing.cpu_count()):
+def parse_memory_limit(memory_limit, nthreads, total_cores=system.CPU_COUNT):
     if memory_limit is None:
         return None
 
     if memory_limit == "auto":
-        memory_limit = int(TOTAL_MEMORY * min(1, nthreads / total_cores))
+        memory_limit = int(system.MEMORY_LIMIT * min(1, nthreads / total_cores))
     with ignoring(ValueError, TypeError):
         memory_limit = float(memory_limit)
         if isinstance(memory_limit, float) and memory_limit <= 1:
-            memory_limit = int(memory_limit * TOTAL_MEMORY)
+            memory_limit = int(memory_limit * system.MEMORY_LIMIT)
 
     if isinstance(memory_limit, str):
         memory_limit = parse_bytes(memory_limit)
     else:
         memory_limit = int(memory_limit)
 
-    # should be less than hard RSS limit
-    try:
-        import resource
-
-        hard_limit = resource.getrlimit(resource.RLIMIT_RSS)[1]
-        if hard_limit > 0:
-            memory_limit = min(memory_limit, hard_limit)
-    except (ImportError, OSError):
-        pass
-
-    return memory_limit
+    return min(memory_limit, system.MEMORY_LIMIT)
 
 
 async def get_data_from_worker(

From 019f7a63464a73dcb50a246ce8bb6f2f69750bc2 Mon Sep 17 00:00:00 2001
From: Elliott Sales de Andrade <quantum.analyst@gmail.com>
Date: Thu, 12 Sep 2019 11:04:03 -0400
Subject: [PATCH 0458/1550] Use mock from unittest standard library. (#3049)

Since distributed depends on Python 3.5+, there's no need to use the
external mock package any more.
---
 continuous_integration/setup_conda_environment.cmd | 1 -
 continuous_integration/travis/install.sh           | 1 -
 dev-requirements.txt                               | 1 -
 distributed/tests/test_diskutils.py                | 2 +-
 distributed/tests/test_ipython.py                  | 2 +-
 distributed/tests/test_submit_cli.py               | 2 +-
 distributed/utils_test.py                          | 2 +-
 7 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/continuous_integration/setup_conda_environment.cmd b/continuous_integration/setup_conda_environment.cmd
index d09846faedd..87e37751548 100644
--- a/continuous_integration/setup_conda_environment.cmd
+++ b/continuous_integration/setup_conda_environment.cmd
@@ -30,7 +30,6 @@ call deactivate
     ipywidgets ^
     joblib ^
     jupyter_client ^
-    mock ^
     msgpack-python ^
     prometheus_client ^
     psutil ^
diff --git a/continuous_integration/travis/install.sh b/continuous_integration/travis/install.sh
index ead4dbc3002..f0c4a07be67 100644
--- a/continuous_integration/travis/install.sh
+++ b/continuous_integration/travis/install.sh
@@ -39,7 +39,6 @@ conda install -q \
     ipywidgets \
     joblib \
     jupyter_client \
-    mock \
     netcdf4 \
     paramiko \
     prometheus_client \
diff --git a/dev-requirements.txt b/dev-requirements.txt
index 8cc8f7d256d..3c4cf7954a3 100644
--- a/dev-requirements.txt
+++ b/dev-requirements.txt
@@ -1,5 +1,4 @@
 joblib >= 0.10.2
-mock >= 2.0.0
 pandas >= 0.19.2
 numpy >= 1.11.0
 bokeh >= 0.12.3
diff --git a/distributed/tests/test_diskutils.py b/distributed/tests/test_diskutils.py
index c5cca9d5824..e12fb324341 100644
--- a/distributed/tests/test_diskutils.py
+++ b/distributed/tests/test_diskutils.py
@@ -6,8 +6,8 @@
 import subprocess
 import sys
 from time import sleep
+from unittest import mock
 
-import mock
 import pytest
 
 import dask
diff --git a/distributed/tests/test_ipython.py b/distributed/tests/test_ipython.py
index 8f2a40e45eb..aa4a3e4092e 100644
--- a/distributed/tests/test_ipython.py
+++ b/distributed/tests/test_ipython.py
@@ -1,4 +1,4 @@
-import mock
+from unittest import mock
 
 import pytest
 from toolz import first
diff --git a/distributed/tests/test_submit_cli.py b/distributed/tests/test_submit_cli.py
index 9273261dc94..edc16e0a61e 100644
--- a/distributed/tests/test_submit_cli.py
+++ b/distributed/tests/test_submit_cli.py
@@ -1,4 +1,4 @@
-from mock import Mock
+from unittest.mock import Mock
 
 from tornado import gen
 from tornado.ioloop import IOLoop
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index 8725e3eb7e0..cd53fe6f86e 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -191,7 +191,7 @@ def pristine_loop():
 
 @contextmanager
 def mock_ipython():
-    import mock
+    from unittest import mock
     from distributed._ipython_utils import remote_magic
 
     ip = mock.Mock()

From bb127bba8e3a554ce7ca02675ec425d1336a30b2 Mon Sep 17 00:00:00 2001
From: Elliott Sales de Andrade <quantum.analyst@gmail.com>
Date: Thu, 12 Sep 2019 11:04:32 -0400
Subject: [PATCH 0459/1550] Add missing test data to sdist tarball. (#3050)

Fixes #2700.
---
 MANIFEST.in | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/MANIFEST.in b/MANIFEST.in
index a6c03274f24..b7a3764c87a 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -12,7 +12,8 @@ include README.rst
 include LICENSE.txt
 include MANIFEST.in
 include requirements.txt
-include distributed/tests/mytestegg-1.0.0-py3.4.egg
+include distributed/tests/testegg-1.0.0-py3.4.egg
+include distributed/tests/mytest.pyz
 include distributed/tests/*.pem
 
 prune docs/_build

From 0ed877267febb5c2eb2b759b7f77d5de56a146c9 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 12 Sep 2019 15:32:38 -0700
Subject: [PATCH 0460/1550] Remove six (#3045)

---
 distributed/cfexecutor.py                |  5 ++--
 distributed/client.py                    | 37 ++++++++++++++----------
 distributed/comm/addressing.py           |  4 +--
 distributed/comm/core.py                 |  9 +++---
 distributed/comm/registry.py             |  6 ++--
 distributed/core.py                      | 10 +++----
 distributed/protocol/tests/test_numpy.py |  2 +-
 distributed/scheduler.py                 |  3 +-
 distributed/utils.py                     |  4 +--
 distributed/utils_test.py                |  6 ++--
 requirements.txt                         |  1 -
 11 files changed, 41 insertions(+), 46 deletions(-)

diff --git a/distributed/cfexecutor.py b/distributed/cfexecutor.py
index 34350462f8b..373a3c4eb28 100644
--- a/distributed/cfexecutor.py
+++ b/distributed/cfexecutor.py
@@ -1,8 +1,6 @@
 import concurrent.futures as cf
 import weakref
 
-import six
-
 from toolz import merge
 
 from tornado import gen
@@ -27,7 +25,8 @@ def _cascade_future(future, cf_future):
         cf_future.set_running_or_notify_cancel()
     else:
         try:
-            six.reraise(*result)
+            typ, exc, tb = result
+            raise exc.with_traceback(tb)
         except BaseException as exc:
             cf_future.set_exception(exc)
 
diff --git a/distributed/client.py b/distributed/client.py
index c11257ee74a..63d8213c33f 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -17,7 +17,6 @@
 import sys
 import uuid
 import threading
-import six
 import socket
 from queue import Queue as pyQueue
 import warnings
@@ -219,7 +218,8 @@ def result(self, timeout=None):
         # shorten error traceback
         result = self.client.sync(self._result, callback_timeout=timeout, raiseit=False)
         if self.status == "error":
-            six.reraise(*result)
+            typ, exc, tb = result
+            raise exc.with_traceback(tb)
         elif self.status == "cancelled":
             raise result
         else:
@@ -230,7 +230,8 @@ async def _result(self, raiseit=True):
         if self.status == "error":
             exc = clean_exception(self._state.exception, self._state.traceback)
             if raiseit:
-                six.reraise(*exc)
+                typ, exc, tb = exc
+                raise exc.with_traceback(tb)
             else:
                 return exc
         elif self.status == "cancelled":
@@ -1145,7 +1146,8 @@ async def _handle_report(self):
                         logger.debug("Client receives message %s", msg)
 
                         if "status" in msg and "error" in msg["status"]:
-                            six.reraise(*clean_exception(**msg))
+                            typ, exc, tb = clean_exception(**msg)
+                            raise exc.with_traceback(tb)
 
                         op = msg.pop("op")
 
@@ -1431,7 +1433,7 @@ def submit(
         if allow_other_workers and workers is None:
             raise ValueError("Only use allow_other_workers= if using workers=")
 
-        if isinstance(workers, six.string_types + (Number,)):
+        if isinstance(workers, (str, Number)):
             workers = [workers]
         if workers is not None:
             restrictions = {skey: workers}
@@ -1577,7 +1579,7 @@ def map(
                 }
             )
 
-        if isinstance(workers, six.string_types + (Number,)):
+        if isinstance(workers, (str, Number)):
             workers = [workers]
         if isinstance(workers, (list, set)):
             if workers and isinstance(first(workers), (list, set)):
@@ -1671,7 +1673,7 @@ async def wait(k):
                         except (KeyError, AttributeError):
                             exc = CancelledError(key)
                         else:
-                            six.reraise(type(exception), exception, traceback)
+                            raise exception.with_traceback(traceback)
                         raise exc
                     if errors == "skip":
                         bad_keys.add(key)
@@ -1830,7 +1832,7 @@ async def _scatter(
     ):
         if timeout == no_default:
             timeout = self._timeout
-        if isinstance(workers, six.string_types + (Number,)):
+        if isinstance(workers, (str, Number)):
             workers = [workers]
         if isinstance(data, dict) and not all(
             isinstance(k, (bytes, str)) for k in data
@@ -2196,7 +2198,8 @@ async def _run_on_scheduler(self, function, *args, wait=True, **kwargs):
             function=dumps(function), args=dumps(args), kwargs=dumps(kwargs), wait=wait
         )
         if response["status"] == "error":
-            six.reraise(*clean_exception(**response))
+            typ, exc, tb = clean_exception(**response)
+            raise exc.with_traceback(tb)
         else:
             return response["result"]
 
@@ -2251,7 +2254,8 @@ async def _run(
             if resp["status"] == "OK":
                 results[key] = resp["result"]
             elif resp["status"] == "error":
-                six.reraise(*clean_exception(**resp))
+                typ, exc, tb = clean_exception(**resp)
+                raise exc.with_traceback(tb)
         if wait:
             return results
 
@@ -3200,7 +3204,7 @@ def profile(
         >>> client.profile()  # call on collections
         >>> client.profile(filename='dask-profile.html')  # save to html file
         """
-        if isinstance(workers, six.string_types + (Number,)):
+        if isinstance(workers, (str, Number)):
             workers = [workers]
 
         return self.sync(
@@ -3224,7 +3228,7 @@ async def _profile(
         plot=False,
         filename=None,
     ):
-        if isinstance(workers, six.string_types + (Number,)):
+        if isinstance(workers, (str, Number)):
             workers = [workers]
 
         state = await self.scheduler.profile(
@@ -3561,12 +3565,12 @@ def start_ipython_workers(
         --------
         Client.start_ipython_scheduler: start ipython on the scheduler
         """
-        if isinstance(workers, six.string_types + (Number,)):
+        if isinstance(workers, (str, Number)):
             workers = [workers]
 
         (workers, info_dict) = sync(self.loop, self._start_ipython_workers, workers)
 
-        if magic_names and isinstance(magic_names, six.string_types):
+        if magic_names and isinstance(magic_names, str):
             if "*" in magic_names:
                 magic_names = [
                     magic_names.replace("*", str(i)) for i in range(len(workers))
@@ -3872,7 +3876,7 @@ async def _register_worker_plugin(self, plugin=None, name=None):
                 exc = response["exception"]
                 typ = type(exc)
                 tb = response["traceback"]
-                six.reraise(typ, exc, tb)
+                raise exc.with_traceback(tb)
         return responses
 
     def register_worker_plugin(self, plugin=None, name=None):
@@ -4180,7 +4184,8 @@ def _get_and_raise(self):
         if self.with_results:
             future, result = res
             if self.raise_errors and future.status == "error":
-                six.reraise(*result)
+                typ, exc, tb = result
+                raise exc.with_traceback(tb)
         return res
 
     def __next__(self):
diff --git a/distributed/comm/addressing.py b/distributed/comm/addressing.py
index 21a23e1ef6e..35d5e1c3407 100644
--- a/distributed/comm/addressing.py
+++ b/distributed/comm/addressing.py
@@ -1,5 +1,3 @@
-import six
-
 import dask
 
 from . import registry
@@ -18,7 +16,7 @@ def parse_address(addr, strict=False):
 
     If strict is set to true the address must have a scheme.
     """
-    if not isinstance(addr, six.string_types):
+    if not isinstance(addr, str):
         raise TypeError("expected str, got %r" % addr.__class__.__name__)
     scheme, sep, loc = addr.rpartition("://")
     if strict and not sep:
diff --git a/distributed/comm/core.py b/distributed/comm/core.py
index 602b3161657..1bbc043f52d 100644
--- a/distributed/comm/core.py
+++ b/distributed/comm/core.py
@@ -1,10 +1,9 @@
-from abc import ABCMeta, abstractmethod, abstractproperty
+from abc import ABC, abstractmethod, abstractproperty
 from datetime import timedelta
 import logging
 import weakref
 
 import dask
-from six import with_metaclass
 from tornado import gen
 
 from ..metrics import time
@@ -24,7 +23,7 @@ class FatalCommClosedError(CommClosedError):
     pass
 
 
-class Comm(with_metaclass(ABCMeta)):
+class Comm(ABC):
     """
     A message-oriented communication object, representing an established
     communication channel.  There should be only one reader and one
@@ -129,7 +128,7 @@ def __repr__(self):
             )
 
 
-class Listener(with_metaclass(ABCMeta)):
+class Listener(ABC):
     @abstractmethod
     def start(self):
         """
@@ -165,7 +164,7 @@ def __exit__(self, *exc):
         self.stop()
 
 
-class Connector(with_metaclass(ABCMeta)):
+class Connector(ABC):
     @abstractmethod
     def connect(self, address, deserialize=True):
         """
diff --git a/distributed/comm/registry.py b/distributed/comm/registry.py
index b7fcca912cd..369f2415c35 100644
--- a/distributed/comm/registry.py
+++ b/distributed/comm/registry.py
@@ -1,9 +1,7 @@
-from abc import ABCMeta, abstractmethod
+from abc import ABC, abstractmethod
 
-from six import with_metaclass
 
-
-class Backend(with_metaclass(ABCMeta)):
+class Backend(ABC):
     """
     A communication backend, selected by a given URI scheme (e.g. 'tcp').
     """
diff --git a/distributed/core.py b/distributed/core.py
index 5da103802d2..d0f3c13aa97 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -3,14 +3,12 @@
 from concurrent.futures import CancelledError
 from functools import partial
 import logging
-import six
 import threading
 import traceback
 import uuid
 import weakref
 
 import dask
-from six import string_types
 from toolz import merge
 from tornado import gen
 from tornado.ioloop import IOLoop
@@ -305,7 +303,7 @@ def listen(self, port_or_addr=None, listen_args=None):
             addr = unparse_host_port(*port_or_addr)
         else:
             addr = port_or_addr
-            assert isinstance(addr, string_types)
+            assert isinstance(addr, str)
         self.listener = listen(
             addr,
             self.handle_comm,
@@ -545,7 +543,8 @@ async def send_recv(comm, reply=True, serializers=None, deserializers=None, **kw
 
     if isinstance(response, dict) and response.get("status") == "uncaught-error":
         if comm.deserialize:
-            six.reraise(*clean_exception(**response))
+            typ, exc, tb = clean_exception(**response)
+            raise exc.with_traceback(tb)
         else:
             raise Exception(response["text"])
     return response
@@ -969,7 +968,6 @@ def error_message(e, status="error"):
     See Also
     --------
     clean_exception: deserialize and unpack message into exception/traceback
-    six.reraise: raise exception/traceback
     """
     tb = get_traceback()
     e2 = truncate_exception(e, 1000)
@@ -1011,6 +1009,6 @@ def clean_exception(exception, traceback, **kwargs):
             traceback = protocol.pickle.loads(traceback)
         except (TypeError, AttributeError):
             traceback = None
-    elif isinstance(traceback, string_types):
+    elif isinstance(traceback, str):
         traceback = None  # happens if the traceback failed serializing
     return type(exception), exception, traceback
diff --git a/distributed/protocol/tests/test_numpy.py b/distributed/protocol/tests/test_numpy.py
index 70d57a2e74f..4fb20d58631 100644
--- a/distributed/protocol/tests/test_numpy.py
+++ b/distributed/protocol/tests/test_numpy.py
@@ -118,7 +118,7 @@ def test_serialize_numpy_ma_masked():
 
 
 def test_dumps_serialize_numpy_custom_dtype():
-    from six.moves import builtins
+    import builtins
 
     test_rational = pytest.importorskip("numpy.core.test_rational")
     rational = test_rational.rational
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index ff9560767c0..6bf1ba0adb4 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -12,7 +12,6 @@
 import os
 import pickle
 import random
-import six
 import warnings
 import weakref
 
@@ -4518,7 +4517,7 @@ def coerce_address(self, addr, resolve=True):
             addr = self.aliases[addr]
         if isinstance(addr, tuple):
             addr = unparse_host_port(*addr)
-        if not isinstance(addr, six.string_types):
+        if not isinstance(addr, str):
             raise TypeError("addresses should be strings or tuples, got %r" % (addr,))
 
         if resolve:
diff --git a/distributed/utils.py b/distributed/utils.py
index b7f6631ce93..015e4dbfb61 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -25,7 +25,6 @@
 import warnings
 import weakref
 import pkgutil
-import six
 import tblib.pickling_support
 import xml.etree.ElementTree
 
@@ -334,7 +333,8 @@ def f():
         while not e.is_set():
             e.wait(10)
     if error[0]:
-        six.reraise(*error[0])
+        typ, exc, tb = error[0]
+        raise exc.with_traceback(tb)
     else:
         return result[0]
 
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index cd53fe6f86e..97cbe783318 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -5,6 +5,7 @@
 from datetime import timedelta
 import functools
 from glob import glob
+import io
 import itertools
 import logging
 import logging.config
@@ -30,7 +31,6 @@
     ssl = None
 
 import pytest
-import six
 
 import dask
 from toolz import merge, memoize, assoc
@@ -1226,7 +1226,7 @@ def captured_logger(logger, level=logging.INFO, propagate=None):
     if propagate is not None:
         orig_propagate = logger.propagate
         logger.propagate = propagate
-    sio = six.StringIO()
+    sio = io.StringIO()
     logger.handlers[:] = [logging.StreamHandler(sio)]
     logger.setLevel(level)
     try:
@@ -1244,7 +1244,7 @@ def captured_handler(handler):
     """
     assert isinstance(handler, logging.StreamHandler)
     orig_stream = handler.stream
-    handler.stream = six.StringIO()
+    handler.stream = io.StringIO()
     try:
         yield handler.stream
     finally:
diff --git a/requirements.txt b/requirements.txt
index e376b2a50cc..804bdfe9637 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,7 +3,6 @@ cloudpickle >= 0.2.2
 dask >= 2
 msgpack
 psutil >= 5.0
-six
 sortedcontainers !=2.0.0, !=2.0.1
 tblib
 toolz >= 0.7.4

From 7d017c467590c758fa4b8cb2b1193205fe5aa7ad Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Fri, 13 Sep 2019 15:34:02 -0500
Subject: [PATCH 0461/1550] bump version to 2.4.0

---
 docs/source/changelog.rst | 38 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index 7385567467e..6bed266f32b 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,11 +1,44 @@
 Changelog
 =========
 
+2.4.0 - 2019-09-13
+------------------
+
+- Remove six (:pr:`3045`) `Matthew Rocklin`_
+- Add missing test data to sdist tarball (:pr:`3050`) `Elliott Sales de Andrade`_
+- Use mock from unittest standard library (:pr:`3049`) `Elliott Sales de Andrade`_
+- Use cgroups resource limits to determine default threads and memory (:pr:`3039`) `Jim Crist`_
+- Move task deserialization to immediately before task execution (:pr:`3015`) `James Bourbeau`_
+- Drop joblib shim module in distributed (:pr:`3040`) `John Kirkham`_
+- Redirect configuration doc page (:pr:`3038`) `Matthew Rocklin`_
+- Support ``--name 0`` and ``--nprocs`` keywords in dask-worker cli (:pr:`3037`) `Matthew Rocklin`_
+- Remove lost workers from ``SpecCluster.workers`` (:pr:`2990`) `Guillaume Eynard-Bontemps`_
+- Clean up ``test_local.py::test_defaults`` (:pr:`3017`) `Matthew Rocklin`_
+- Replace print statement in ``Queue.__init__`` with debug message (:pr:`3035`) `Mikhail Akimov`_
+- Set the ``x_range`` limit of the Meory utilization plot to memory-limit (:pr:`3034`) `Matthew Rocklin`_
+- Rely on cudf codebase for cudf serialization (:pr:`2998`) `Benjamin Zaitlen`_
+- Add fallback html repr for Cluster (:pr:`3023`) `Jim Crist`_
+- Add support for zstandard compression to comms (:pr:`2970`) `Abael He`_
+- Avoid collision when using ``os.environ`` in ``dashboard_link`` (:pr:`3021`) `Matthew Rocklin`_
+- Fix ``ConnectionPool`` limit handling (:pr:`3005`) `byjott`_
+- Support Spec jobs that generate multiple workers (:pr:`3013`) `Matthew Rocklin`_
+- Tweak ``Logs`` styling (:pr:`3012`) `Jim Crist`_
+- Better name for cudf deserialization function name (:pr:`3008`) `Benjamin Zaitlen`_
+- Make ``spec.ProcessInterface`` a valid no-op worker (:pr:`3004`) `Matthew Rocklin`_
+- Return dictionaries from ``new_worker_spec`` rather than name/worker pairs (:pr:`3000`) `Matthew Rocklin`_
+- Fix minor typo in documentation (:pr:`3002`) `Mohammad Noor`_
+- Permit more keyword options when scaling with cores and memory (:pr:`2997`) `Matthew Rocklin`_
+- Add ``cuda_ipc`` to UCX environment for NVLink (:pr:`2996`) `Benjamin Zaitlen`_
+- Add ``threads=`` and ``memory=`` to Cluster and Client reprs (:pr:`2995`) `Matthew Rocklin`_
+- Fix PyNVML initialization (:pr:`2993`) `Richard J Zamora`_
+
+
 2.3.2 - 2019-08-23
 ------------------
 
 -  Skip exceptions in startup information (:pr:`2991`) `Jacob Tomlinson`_
 
+
 2.3.1 - 2019-08-22
 ------------------
 
@@ -1230,3 +1263,8 @@ significantly without many new features.
 .. _`Shayan Amani`: https://github.com/SHi-ON
 .. _`Pav A`: https://github.com/rs2
 .. _`Mads R. B. Kristensen`: https://github.com/madsbk
+.. _`Mikhail Akimov`: https://github.com/roveo
+.. _`Abael He`: https://github.com/abaelhe
+.. _`byjott`: https://github.com/byjott
+.. _`Mohammad Noor`: https://github.com/MdSalih
+.. _`Richard J Zamora`: https://github.com/rjzamora

From 86d7c0385967e530208eeb18e8544564cda05c2b Mon Sep 17 00:00:00 2001
From: Chris White <chris@prefect.io>
Date: Sun, 15 Sep 2019 09:55:55 -0700
Subject: [PATCH 0462/1550] Add blurb about disabling work stealing (#3055)

---
 docs/source/work-stealing.rst | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/docs/source/work-stealing.rst b/docs/source/work-stealing.rst
index cf5a4bc48c1..83afd795b7f 100644
--- a/docs/source/work-stealing.rst
+++ b/docs/source/work-stealing.rst
@@ -125,3 +125,12 @@ the task and sends a response to the scheduler:
 This avoids redundant work, and also the duplication of side effects for more
 exotic tasks.  However, concurrent or repeated execution of the same task *is
 still possible* in the event of worker death or a disrupted network connection.
+
+
+Disabling Work Stealing
+---------------------------
+
+Work stealing is a toggleable setting on the Dask Scheduler; to disable 
+work stealing, you can toggle the scheduler ``work-stealing`` configuration 
+option to ``"False"`` either by setting ``DASK_DISTRIBUTED__SCHEDULER__WORK_STEALING="False"``
+or through your `Dask configuration file <https://docs.dask.org/en/latest/configuration.html>`_

From 8a41770e8ba219b4b114027d5ed61806ff8e8612 Mon Sep 17 00:00:00 2001
From: Jim Crist <jcrist@users.noreply.github.com>
Date: Mon, 16 Sep 2019 11:15:53 -0500
Subject: [PATCH 0463/1550] Check multiple cgroups dirs, ceil fractional cpus
 (#3056)

A few fixes for resource detection using cgroups:

- The directory for determining cpu availability isn't standardized
across linux distros, could be either `cpuacct,cpu`, or `cpu,cpuacct`.
We now check for both.
- When allotted fractional cpus (e.g. 1.5), we now round up.

Also adds tests for both CPU and memory limit detection under cgroups,
by monkeypatching in fake files.

Fixes #3053
---
 distributed/system.py            | 25 ++++++++------
 distributed/tests/test_system.py | 57 ++++++++++++++++++++++++++++++++
 2 files changed, 72 insertions(+), 10 deletions(-)

diff --git a/distributed/system.py b/distributed/system.py
index e0735e1e34a..291248ddded 100644
--- a/distributed/system.py
+++ b/distributed/system.py
@@ -1,3 +1,4 @@
+import math
 import os
 import sys
 
@@ -61,16 +62,20 @@ def cpu_count():
 
     # Check cgroups if available
     if sys.platform == "linux":
-        try:
-            with open("/sys/fs/cgroup/cpuacct,cpu/cpu.cfs_quota_us") as f:
-                quota = int(f.read())
-            with open("/sys/fs/cgroup/cpuacct,cpu/cpu.cfs_period_us") as f:
-                period = int(f.read())
-            cgroups_count = int(quota / period)
-            if cgroups_count > 0:
-                count = min(count, cgroups_count)
-        except Exception:
-            pass
+        # The directory name isn't standardized across linux distros, check both
+        for dirname in ["cpuacct,cpu", "cpu,cpuacct"]:
+            try:
+                with open("/sys/fs/cgroup/%s/cpu.cfs_quota_us" % dirname) as f:
+                    quota = int(f.read())
+                with open("/sys/fs/cgroup/%s/cpu.cfs_period_us" % dirname) as f:
+                    period = int(f.read())
+                # We round up on fractional CPUs
+                cgroups_count = math.ceil(quota / period)
+                if cgroups_count > 0:
+                    count = min(count, cgroups_count)
+                break
+            except Exception:
+                pass
 
     return count
 
diff --git a/distributed/tests/test_system.py b/distributed/tests/test_system.py
index d0f00f495e3..d276613b520 100644
--- a/distributed/tests/test_system.py
+++ b/distributed/tests/test_system.py
@@ -1,4 +1,7 @@
+import builtins
+import io
 import os
+import sys
 
 import psutil
 import pytest
@@ -13,6 +16,44 @@ def test_cpu_count():
     assert count >= 1
 
 
+@pytest.mark.parametrize("dirname", ["cpuacct,cpu", "cpu,cpuacct", None])
+def test_cpu_count_cgroups(dirname, monkeypatch):
+    def mycpu_count():
+        # Absurdly high, unlikely to match real value
+        return 250
+
+    monkeypatch.setattr(os, "cpu_count", mycpu_count)
+
+    class MyProcess(object):
+        def cpu_affinity(self):
+            # No affinity set
+            return []
+
+    monkeypatch.setattr(psutil, "Process", MyProcess)
+
+    if dirname:
+        paths = {
+            "/sys/fs/cgroup/%s/cpu.cfs_quota_us" % dirname: io.StringIO("2005"),
+            "/sys/fs/cgroup/%s/cpu.cfs_period_us" % dirname: io.StringIO("10"),
+        }
+        builtin_open = builtins.open
+
+        def myopen(path, *args, **kwargs):
+            if path in paths:
+                return paths.get(path)
+            return builtin_open(path, *args, **kwargs)
+
+        monkeypatch.setattr(builtins, "open", myopen)
+        monkeypatch.setattr(sys, "platform", "linux")
+
+    count = cpu_count()
+    if dirname:
+        # Rounds up
+        assert count == 201
+    else:
+        assert count == 250
+
+
 def test_memory_limit():
     limit = memory_limit()
     assert isinstance(limit, int)
@@ -20,6 +61,22 @@ def test_memory_limit():
     assert limit >= 1
 
 
+def test_memory_limit_cgroups(monkeypatch):
+    builtin_open = builtins.open
+
+    def myopen(path, *args, **kwargs):
+        if path == "/sys/fs/cgroup/memory/memory.limit_in_bytes":
+            # Absurdly low, unlikely to match real value
+            return io.StringIO("20")
+        return builtin_open(path, *args, **kwargs)
+
+    monkeypatch.setattr(builtins, "open", myopen)
+    monkeypatch.setattr(sys, "platform", "linux")
+
+    limit = memory_limit()
+    assert limit == 20
+
+
 def test_rlimit():
     resource = pytest.importorskip("resource")
 

From 1f5cc12654d743c15947b2a997eaba8770922a2a Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 16 Sep 2019 12:39:14 -0700
Subject: [PATCH 0464/1550] Allow full script in preload inputs (#3052)

Previously we allowed either a script or a module in Scheduler or
Worker preload arguments.

    worker = Worker(..., preload='myfile.py')

However for simple scripts we sometimes don't want to create a full
file, but just want to include the script as a  multi-line text value.

    script = """def dask_setup(worker):\n    worker.foo=123"""
    worker = Worker(..., preload=script)
---
 distributed/preloading.py         | 78 ++++++++++++++++---------------
 distributed/tests/test_preload.py | 17 ++++++-
 2 files changed, 56 insertions(+), 39 deletions(-)

diff --git a/distributed/preloading.py b/distributed/preloading.py
index a5e67c1611a..9b276b4337f 100644
--- a/distributed/preloading.py
+++ b/distributed/preloading.py
@@ -8,6 +8,8 @@
 
 import click
 
+from dask.utils import tmpfile
+
 from .utils import import_file
 
 logger = logging.getLogger(__name__)
@@ -29,7 +31,7 @@ def validate_preload_argv(ctx, param, value):
             % ("s" if len(value) > 1 else "", " ".join(value))
         )
 
-    preload_modules = _import_modules(ctx.params.get("preload"))
+    preload_modules = {name: _import_module(name) for name in ctx.params.get("preload")}
 
     preload_commands = [
         m["dask_setup"]
@@ -58,17 +60,16 @@ def validate_preload_argv(ctx, param, value):
     return value
 
 
-def _import_modules(names, file_dir=None):
-    """ Imports modules and extracts preload interface functions.
+def _import_module(name, file_dir=None):
+    """ Imports module and extract preload interface functions.
 
-    Imports modules specified by names and extracts 'dask_setup'
+    Import modules specified by name and extract 'dask_setup'
     and 'dask_teardown' if present.
 
-
     Parameters
     ----------
-    names: list of strings
-        Module names or file paths
+    name: str
+        Module name, file path, or text of module or script
     file_dir: string
         Path of a directory where files should be copied
 
@@ -77,36 +78,37 @@ def _import_modules(names, file_dir=None):
     Nest dict of names to extracted module interface components if present
     in imported module.
     """
-    result_modules = {}
-
-    for name in names:
-        # import
-        if name.endswith(".py"):
-            # name is a file path
-            if file_dir is not None:
-                basename = os.path.basename(name)
-                copy_dst = os.path.join(file_dir, basename)
-                if os.path.exists(copy_dst):
-                    if not filecmp.cmp(name, copy_dst):
-                        logger.error("File name collision: %s", basename)
-                shutil.copy(name, copy_dst)
-                module = import_file(copy_dst)[0]
-            else:
-                module = import_file(name)[0]
-
+    if name.endswith(".py"):
+        # name is a file path
+        if file_dir is not None:
+            basename = os.path.basename(name)
+            copy_dst = os.path.join(file_dir, basename)
+            if os.path.exists(copy_dst):
+                if not filecmp.cmp(name, copy_dst):
+                    logger.error("File name collision: %s", basename)
+            shutil.copy(name, copy_dst)
+            module = import_file(copy_dst)[0]
         else:
-            # name is a module name
-            if name not in sys.modules:
-                import_module(name)
-            module = sys.modules[name]
+            module = import_file(name)[0]
 
-        logger.info("Import preload module: %s", name)
-        result_modules[name] = {
-            attrname: getattr(module, attrname, None)
-            for attrname in ("dask_setup", "dask_teardown")
-        }
+    elif " " not in name:
+        # name is a module name
+        if name not in sys.modules:
+            import_module(name)
+        module = sys.modules[name]
 
-    return result_modules
+    else:
+        # not a name, actually the text of the script
+        with tmpfile(extension=".py") as fn:
+            with open(fn, mode="w") as f:
+                f.write(name)
+            return _import_module(fn, file_dir=file_dir)
+
+    logger.info("Import preload module: %s", name)
+    return {
+        attrname: getattr(module, attrname, None)
+        for attrname in ("dask_setup", "dask_teardown")
+    }
 
 
 def preload_modules(names, parameter=None, file_dir=None, argv=None):
@@ -123,10 +125,12 @@ def preload_modules(names, parameter=None, file_dir=None, argv=None):
     file_dir: string
         Path of a directory where files should be copied
     """
+    if isinstance(names, str):
+        names = [names]
 
-    imported_modules = _import_modules(names, file_dir=file_dir)
+    for name in names:
+        interface = _import_module(name, file_dir=file_dir)
 
-    for name, interface in imported_modules.items():
         dask_setup = interface.get("dask_setup", None)
         dask_teardown = interface.get("dask_teardown", None)
 
@@ -140,5 +144,5 @@ def preload_modules(names, parameter=None, file_dir=None, argv=None):
                 dask_setup(parameter)
                 logger.info("Run preload setup function: %s", name)
 
-        if interface["dask_teardown"]:
+        if dask_teardown:
             atexit.register(interface["dask_teardown"], parameter)
diff --git a/distributed/tests/test_preload.py b/distributed/tests/test_preload.py
index 07ee56d85a6..9ce804b752a 100644
--- a/distributed/tests/test_preload.py
+++ b/distributed/tests/test_preload.py
@@ -2,10 +2,11 @@
 import shutil
 import sys
 import tempfile
+import pytest
 
-from distributed import Client
+from distributed import Client, Scheduler, Worker
 from distributed.utils_test import cluster
-from distributed.utils_test import loop  # noqa F401
+from distributed.utils_test import cleanup, loop  # noqa F401
 
 
 PRELOAD_TEXT = """
@@ -42,6 +43,18 @@ def check_worker():
         shutil.rmtree(tmpdir)
 
 
+@pytest.mark.asyncio
+async def test_worker_preload_text(cleanup):
+    text = """
+def dask_setup(worker):
+    worker.foo = 'setup'
+"""
+    async with Scheduler(port=0, preload=text) as s:
+        assert s.foo == "setup"
+        async with Worker(s.address, preload=[text]) as w:
+            assert w.foo == "setup"
+
+
 def test_worker_preload_module(loop):
     def check_worker():
         import worker_info

From d16aabc012442cee0187e7af0c8ca30dff01e9a8 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 18 Sep 2019 11:01:03 -0500
Subject: [PATCH 0465/1550] Allow SpecCluster to scale by memory and cores
 (#3057)

```python
>>> cluster.adapt(minimum_cores=10, maximum_memory="100 GiB")
```
---
 distributed/deploy/spec.py                    | 115 ++++++++++++++----
 distributed/deploy/tests/test_adaptive.py     |  30 +++++
 .../deploy/tests/test_slow_adaptive.py        |   2 +-
 3 files changed, 122 insertions(+), 25 deletions(-)

diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index 487e5192e17..87336f96184 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -8,6 +8,7 @@
 import dask
 from tornado import gen
 
+from .adaptive import Adaptive
 from .cluster import Cluster
 from ..core import rpc, CommClosedError
 from ..utils import LoopRunner, silence_logging, ignoring, parse_bytes, parse_timedelta
@@ -361,34 +362,39 @@ def __exit__(self, typ, value, traceback):
         self.close()
         self._loop_runner.stop()
 
+    def _threads_per_worker(self) -> int:
+        """ Return the number of threads per worker for new workers """
+        if not self.new_spec:
+            raise ValueError("To scale by cores= you must specify cores per worker")
+
+        for name in ["nthreads", "ncores", "threads", "cores"]:
+            with ignoring(KeyError):
+                return self.new_spec["options"][name]
+
+        if not self.new_spec:
+            raise ValueError("To scale by cores= you must specify cores per worker")
+
+    def _memory_per_worker(self) -> int:
+        """ Return the memory limit per worker for new workers """
+        if not self.new_spec:
+            raise ValueError(
+                "to scale by memory= your worker definition must include a memory_limit definition"
+            )
+
+        for name in ["memory_limit", "memory"]:
+            with ignoring(KeyError):
+                return parse_bytes(self.new_spec["options"][name])
+
+        raise ValueError(
+            "to use scale(memory=...) your worker definition must include a memory_limit definition"
+        )
+
     def scale(self, n=0, memory=None, cores=None):
         if memory is not None:
-            for name in ["memory_limit", "memory"]:
-                try:
-                    limit = self.new_spec["options"][name]
-                except KeyError:
-                    pass
-                else:
-                    n = max(n, int(math.ceil(parse_bytes(memory) / parse_bytes(limit))))
-                    break
-            else:
-                raise ValueError(
-                    "to use scale(memory=...) your worker definition must include a memory_limit definition"
-                )
+            n = max(n, int(math.ceil(parse_bytes(memory) / self._memory_per_worker())))
 
         if cores is not None:
-            for name in ["nthreads", "ncores", "threads", "cores"]:
-                try:
-                    threads_per_worker = self.new_spec["options"][name]
-                except KeyError:
-                    pass
-                else:
-                    n = max(n, int(math.ceil(cores / threads_per_worker)))
-                    break
-            else:
-                raise ValueError(
-                    "to use scale(cores=...) your worker definition must include an nthreads= definition"
-                )
+            n = max(n, int(math.ceil(cores / self._threads_per_worker())))
 
         if len(self.worker_spec) > n:
             not_yet_launched = set(self.worker_spec) - {
@@ -473,6 +479,67 @@ def requested(self):
                 out.add(name)
         return out
 
+    def adapt(
+        self,
+        *args,
+        minimum=0,
+        maximum=math.inf,
+        minimum_cores: int = None,
+        maximum_cores: int = None,
+        minimum_memory: str = None,
+        maximum_memory: str = None,
+        **kwargs
+    ) -> Adaptive:
+        """ Turn on adaptivity
+
+        This scales Dask clusters automatically based on scheduler activity.
+
+        Parameters
+        ----------
+        minimum : int
+            Minimum number of workers
+        maximum : int
+            Maximum number of workers
+        minimum_cores : int
+            Minimum number of cores/threads to keep around in the cluster
+        maximum_cores : int
+            Maximum number of cores/threads to keep around in the cluster
+        minimum_memory : str
+            Minimum amount of memory to keep around in the cluster
+            Expressed as a string like "100 GiB"
+        maximum_cores : int
+            Maximum amount of memory to keep around in the cluster
+            Expressed as a string like "100 GiB"
+
+        Examples
+        --------
+        >>> cluster.adapt(minimum=0, maximum_memory="100 GiB", interval='500ms')
+
+        See Also
+        --------
+        dask.distributed.Adaptive : for more keyword arguments
+        """
+        if minimum_cores is not None:
+            minimum = max(
+                minimum or 0, math.ceil(minimum_cores / self._threads_per_worker())
+            )
+        if minimum_memory is not None:
+            minimum = max(
+                minimum or 0,
+                math.ceil(parse_bytes(minimum_memory) / self._memory_per_worker()),
+            )
+        if maximum_cores is not None:
+            maximum = min(
+                maximum, math.floor(maximum_cores / self._threads_per_worker())
+            )
+        if maximum_memory is not None:
+            maximum = min(
+                maximum,
+                math.floor(parse_bytes(maximum_memory) / self._memory_per_worker()),
+            )
+
+        return super().adapt(*args, minimum=minimum, maximum=maximum, **kwargs)
+
 
 @atexit.register
 def close_clusters():
diff --git a/distributed/deploy/tests/test_adaptive.py b/distributed/deploy/tests/test_adaptive.py
index 261b4355251..af198747822 100644
--- a/distributed/deploy/tests/test_adaptive.py
+++ b/distributed/deploy/tests/test_adaptive.py
@@ -397,3 +397,33 @@ def key(ws):
 
         names = {ws.name for ws in cluster.scheduler.workers.values()}
         assert names == {"a-1", "a-2"} or names == {"b-1", "b-2"}
+
+
+@pytest.mark.asyncio
+async def test_adapt_cores_memory(cleanup):
+    async with LocalCluster(
+        0,
+        threads_per_worker=2,
+        memory_limit="3 GB",
+        scheduler_port=0,
+        silence_logs=False,
+        processes=False,
+        dashboard_address=None,
+        asynchronous=True,
+    ) as cluster:
+        adapt = cluster.adapt(minimum_cores=3, maximum_cores=9)
+        assert adapt.minimum == 2
+        assert adapt.maximum == 4
+
+        adapt = cluster.adapt(minimum_memory="7GB", maximum_memory="20 GB")
+        assert adapt.minimum == 3
+        assert adapt.maximum == 6
+
+        adapt = cluster.adapt(
+            minimum_cores=1,
+            minimum_memory="7GB",
+            maximum_cores=10,
+            maximum_memory="1 TB",
+        )
+        assert adapt.minimum == 3
+        assert adapt.maximum == 5
diff --git a/distributed/deploy/tests/test_slow_adaptive.py b/distributed/deploy/tests/test_slow_adaptive.py
index 4f565a78289..09113fe3b23 100644
--- a/distributed/deploy/tests/test_slow_adaptive.py
+++ b/distributed/deploy/tests/test_slow_adaptive.py
@@ -47,7 +47,7 @@ async def test_startup(cleanup):
     ) as cluster:
         assert len(cluster.workers) == len(cluster.worker_spec) == 3
         assert time() < start + 5
-        assert 1 <= len(cluster.scheduler_info["workers"]) <= 2
+        assert 0 <= len(cluster.scheduler_info["workers"]) <= 2
 
         async with Client(cluster, asynchronous=True) as client:
             await client.wait_for_workers(n_workers=2)

From 4bfe42d8b00b7553609a1797bdb88b93c7efd062 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 18 Sep 2019 11:01:20 -0500
Subject: [PATCH 0466/1550] Use Cluster.scheduler_info for workers= value in
 repr (#3058)

Previously we would use `Cluster.workers`, which might differ from the
number of Dask workers if a single job generated several Dask workers.

Now, we use the reported number of workers from the scheduler.
---
 distributed/deploy/cluster.py                 | 2 +-
 distributed/deploy/tests/test_spec_cluster.py | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
index 32ceedc47bd..ad47881ba22 100644
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -365,7 +365,7 @@ def __repr__(self):
         text = "%s(%r, workers=%d, threads=%d" % (
             self._cluster_class_name,
             self.scheduler_address,
-            len(self.workers),
+            len(self.scheduler_info["workers"]),
             sum(w["nthreads"] for w in self.scheduler_info["workers"].values()),
         )
 
diff --git a/distributed/deploy/tests/test_spec_cluster.py b/distributed/deploy/tests/test_spec_cluster.py
index 679adc0fd60..d8c155c7c69 100644
--- a/distributed/deploy/tests/test_spec_cluster.py
+++ b/distributed/deploy/tests/test_spec_cluster.py
@@ -374,6 +374,8 @@ async def test_MultiWorker(cleanup):
             assert len(cluster.worker_spec) == 2
             await client.wait_for_workers(4)
 
+            assert "workers=4" in repr(cluster)
+
             cluster.scale(1)
             await cluster
             assert len(s.workers) == 2

From 71c7e4a6305942ae038db05cd55a29e184f998ce Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Wed, 18 Sep 2019 20:37:10 +0100
Subject: [PATCH 0467/1550] Allow specification of worker type in SSHCLuster
 (#3061)

---
 distributed/deploy/ssh2.py | 32 ++++++++++++++++++++++++++++----
 1 file changed, 28 insertions(+), 4 deletions(-)

diff --git a/distributed/deploy/ssh2.py b/distributed/deploy/ssh2.py
index 064e580b111..cb6b967d544 100644
--- a/distributed/deploy/ssh2.py
+++ b/distributed/deploy/ssh2.py
@@ -57,6 +57,8 @@ class Worker(Process):
         The address of the scheduler
     address: str
         The hostname where we should run this worker
+    worker_module: str
+        The python module to run to start the worker.
     connect_kwargs: dict
         kwargs to be passed to asyncssh connections
     kwargs: dict
@@ -70,11 +72,13 @@ def __init__(
         address: str,
         connect_kwargs: dict,
         kwargs: dict,
+        worker_module="distributed.cli.dask_worker",
         loop=None,
         name=None,
     ):
         self.address = address
         self.scheduler = scheduler
+        self.worker_module = worker_module
         self.connect_kwargs = connect_kwargs
         self.kwargs = kwargs
         self.name = name
@@ -88,7 +92,7 @@ async def start(self):
                 [
                     sys.executable,
                     "-m",
-                    "distributed.cli.dask_worker",
+                    self.worker_module,
                     self.scheduler,
                     "--name",
                     str(self.name),
@@ -158,7 +162,12 @@ async def start(self):
 
 
 def SSHCluster(
-    hosts, connect_kwargs={}, worker_kwargs={}, scheduler_kwargs={}, **kwargs
+    hosts,
+    connect_kwargs={},
+    worker_kwargs={},
+    scheduler_kwargs={},
+    worker_module="distributed.cli.dask_worker",
+    **kwargs
 ):
     """ Deploy a Dask cluster using SSH
 
@@ -174,10 +183,12 @@ def SSHCluster(
             key presented during the SSH handshake.  If this is not specified,
             the keys will be looked up in the file .ssh/known_hosts.  If this
             is explicitly set to None, server host key validation will be disabled.
-    scheduler_kwargs:
-        Keywords to pass on to dask-scheduler
     worker_kwargs:
         Keywords to pass on to dask-worker
+    scheduler_kwargs:
+        Keywords to pass on to dask-scheduler
+    worker_module:
+        Python module to call to start the worker
 
     Examples
     --------
@@ -189,6 +200,18 @@ def SSHCluster(
     ...     worker_kwargs={"nthreads": 2},
     ...     scheduler_kwargs={"port": 0, "dashboard_address": ":8797"})
     >>> client = Client(cluster)
+
+    Running GPU workers (requires ``dask_cuda`` to be installed on all hosts)
+
+    >>> from dask.distributed import Client
+    >>> from distributed.deploy.ssh2 import SSHCluster  # experimental for now
+    >>> cluster = SSHCluster(
+    ...     ["localhost", "hostwithgpus", "anothergpuhost"],
+    ...     connect_kwargs={"known_hosts": None},
+    ...     scheduler_kwargs={"port": 0, "dashboard_address": ":8797"},
+    ...     worker_module='dask_cuda.dask_cuda_worker')
+    >>> client = Client(cluster)
+
     """
     scheduler = {
         "cls": Scheduler,
@@ -205,6 +228,7 @@ def SSHCluster(
                 "address": host,
                 "connect_kwargs": connect_kwargs,
                 "kwargs": worker_kwargs,
+                "worker_module": worker_module,
             },
         }
         for i, host in enumerate(hosts[1:])

From face9e8273ef755aa658d89d8f3dd80b1c02cd7c Mon Sep 17 00:00:00 2001
From: "Mads R. B. Kristensen" <madsbk@gmail.com>
Date: Fri, 20 Sep 2019 16:16:41 +0200
Subject: [PATCH 0468/1550] Protocol of cupy and numba handles serialization
 exclusively  (#3047)

---
 distributed/protocol/cupy.py             | 32 +++---------
 distributed/protocol/numba.py            | 62 ++++++------------------
 distributed/protocol/tests/test_cupy.py  |  6 ++-
 distributed/protocol/tests/test_numba.py | 19 ++++++++
 4 files changed, 44 insertions(+), 75 deletions(-)
 create mode 100644 distributed/protocol/tests/test_numba.py

diff --git a/distributed/protocol/cupy.py b/distributed/protocol/cupy.py
index f8d08ee3a1e..d85f37d8a1e 100644
--- a/distributed/protocol/cupy.py
+++ b/distributed/protocol/cupy.py
@@ -7,36 +7,18 @@
 
 @cuda_serialize.register(cupy.ndarray)
 def serialize_cupy_ndarray(x):
-    # TODO: handle non-contiguous
-    # TODO: Handle order='K' ravel
-    # TODO: 0d
+    # Making sure `x` is behaving
+    if not x.flags.c_contiguous:
+        x = cupy.array(x, copy=True)
 
-    if x.flags.c_contiguous or x.flags.f_contiguous:
-        strides = x.strides
-        data = x.ravel()  # order='K'
-    else:
-        x = cupy.ascontiguousarray(x)
-        strides = x.strides
-        data = x.ravel()
-
-    dtype = (0, x.dtype.str)
-
-    # used in the ucx comms for gpu/cpu message passing
-    # 'lengths' set by dask
     header = x.__cuda_array_interface__.copy()
-    header["is_cuda"] = 1
-    header["dtype"] = dtype
-    return header, [data]
+    return header, [x]
 
 
 @cuda_deserialize.register(cupy.ndarray)
 def deserialize_cupy_array(header, frames):
     (frame,) = frames
-    # TODO: put this in ucx... as a kind of "fixup"
-    try:
-        frame.typestr = header["typestr"]
-        frame.shape = header["shape"]
-    except AttributeError:
-        pass
-    arr = cupy.asarray(frame)
+    arr = cupy.ndarray(
+        header["shape"], dtype=header["typestr"], memptr=cupy.asarray(frame).data
+    )
     return arr
diff --git a/distributed/protocol/numba.py b/distributed/protocol/numba.py
index aa56a682b95..ddf43adc182 100644
--- a/distributed/protocol/numba.py
+++ b/distributed/protocol/numba.py
@@ -1,61 +1,27 @@
+import numpy as np
 import numba.cuda
 from .cuda import cuda_serialize, cuda_deserialize
 
 
 @cuda_serialize.register(numba.cuda.devicearray.DeviceNDArray)
 def serialize_numba_ndarray(x):
-    # TODO: handle non-contiguous
-    # TODO: handle 2d
-    # TODO: 0d
-
-    if x.flags["C_CONTIGUOUS"] or x.flags["F_CONTIGUOUS"]:
-        strides = x.strides
-        if x.ndim > 1:
-            data = x.ravel()  # order='K'
-        else:
-            data = x
-    else:
-        raise ValueError("Array must be contiguous")
-        x = numba.ascontiguousarray(x)
-        strides = x.strides
-        if x.ndim > 1:
-            data = x.ravel()
-        else:
-            data = x
-
-    dtype = (0, x.dtype.str)
-    nbytes = data.dtype.itemsize * data.size
-
-    # used in the ucx comms for gpu/cpu message passing
-    # 'lengths' set by dask
+    # Making sure `x` is behaving
+    if not x.is_c_contiguous():
+        shape = x.shape
+        t = numba.cuda.device_array(shape, dtype=x.dtype)
+        t.copy_to_device(x)
+        x = t
     header = x.__cuda_array_interface__.copy()
-    header["is_cuda"] = 1
-    header["dtype"] = dtype
-    return header, [data]
+    return header, [x]
 
 
 @cuda_deserialize.register(numba.cuda.devicearray.DeviceNDArray)
 def deserialize_numba_ndarray(header, frames):
     (frame,) = frames
-    # TODO: put this in ucx... as a kind of "fixup"
-    if isinstance(frame, bytes):
-        import numpy as np
-
-        arr2 = np.frombuffer(frame, header["typestr"])
-        return numba.cuda.to_device(arr2)
-
-    frame.typestr = header["typestr"]
-    frame.shape = header["shape"]
-
-    # numba & cupy don't properly roundtrip length-zero arrays.
-    if frame.shape[0] == 0:
-        arr = numba.cuda.device_array(
-            header["shape"],
-            header["typestr"]
-            # strides?
-            # order?
-        )
-        return arr
-
-    arr = numba.cuda.as_cuda_array(frame)
+    arr = numba.cuda.devicearray.DeviceNDArray(
+        header["shape"],
+        header["strides"],
+        np.dtype(header["typestr"]),
+        gpu_data=numba.cuda.as_cuda_array(frame).gpu_data,
+    )
     return arr
diff --git a/distributed/protocol/tests/test_cupy.py b/distributed/protocol/tests/test_cupy.py
index 26940597f81..10335d14338 100644
--- a/distributed/protocol/tests/test_cupy.py
+++ b/distributed/protocol/tests/test_cupy.py
@@ -4,8 +4,10 @@
 cupy = pytest.importorskip("cupy")
 
 
-def test_serialize_cupy():
-    x = cupy.arange(100)
+@pytest.mark.parametrize("size", [0, 10])
+@pytest.mark.parametrize("dtype", ["u1", "u4", "u8", "f4"])
+def test_serialize_cupy(size, dtype):
+    x = cupy.arange(size, dtype=dtype)
     header, frames = serialize(x, serializers=("cuda", "dask", "pickle"))
     y = deserialize(header, frames, deserializers=("cuda", "dask", "pickle", "error"))
 
diff --git a/distributed/protocol/tests/test_numba.py b/distributed/protocol/tests/test_numba.py
new file mode 100644
index 00000000000..794db58b3c9
--- /dev/null
+++ b/distributed/protocol/tests/test_numba.py
@@ -0,0 +1,19 @@
+from distributed.protocol import serialize, deserialize
+import pytest
+
+cuda = pytest.importorskip("numba.cuda")
+np = pytest.importorskip("numpy")
+
+
+@pytest.mark.parametrize("dtype", ["u1", "u4", "u8", "f4"])
+def test_serialize_cupy(dtype):
+    ary = np.arange(100, dtype=dtype)
+    x = cuda.to_device(ary)
+    header, frames = serialize(x, serializers=("cuda", "dask", "pickle"))
+    y = deserialize(header, frames, deserializers=("cuda", "dask", "pickle", "error"))
+
+    hx = np.empty_like(ary)
+    hy = np.empty_like(ary)
+    x.copy_to_host(hx)
+    y.copy_to_host(hy)
+    assert (hx == hy).all()

From 2759f34b3de7e17ee97d07ab09e73e2d741ca6f8 Mon Sep 17 00:00:00 2001
From: Arpit Solanki <solankiarpit1997@gmail.com>
Date: Sat, 21 Sep 2019 00:50:17 +0530
Subject: [PATCH 0469/1550] Add monitoring with dask cluster docs (#3072)

---
 docs/source/index.rst      |  1 +
 docs/source/prometheus.rst | 36 ++++++++++++++++++++++++++++++++++++
 2 files changed, 37 insertions(+)
 create mode 100644 docs/source/prometheus.rst

diff --git a/docs/source/index.rst b/docs/source/index.rst
index 732c234a53b..ee32738f826 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -112,6 +112,7 @@ Contents
    local-cluster
    ipython
    Joblib Integration <https://ml.dask.org/joblib.html>
+   prometheus
    publish
    queues
    resources
diff --git a/docs/source/prometheus.rst b/docs/source/prometheus.rst
new file mode 100644
index 00000000000..5000858a045
--- /dev/null
+++ b/docs/source/prometheus.rst
@@ -0,0 +1,36 @@
+Prometheus Monitoring
+-----------------------
+
+Prometheus_ is a widely popular tool for monitoring and alerting a wide variety of systems. Dask.distributed exposes
+scheduler and worker metrics in a prometheus text based format. Metrics are available at ``http://scheduler-address:8787/metrics``.
+
+.. _Prometheus: https://prometheus.io
+
+Available metrics are as following
+
++---------------------------------------------+----------------------------------------------+
+| Metric name                                 | Description                                  |
++=========================+===================+==============================================+
+| dask_scheduler_workers                      | Number of workers connected.                 |
++---------------------------------------------+----------------------------------------------+
+| dask_scheduler_clients                      | Number of clients connected.                 |
++---------------------------------------------+----------------------------------------------+
+| dask_scheduler_received_tasks               | Number of tasks received at scheduler        |
++---------------------------------------------+----------------------------------------------+
+| dask_scheduler_unrunnable_tasks             | Number of unrunnable tasks at scheduler      |
++---------------------------------------------+----------------------------------------------+
+| dask_worker_tasks                           | Number of tasks at worker.                   |
++---------------------------------------------+----------------------------------------------+
+| dask_worker_connections                     | Number of task connections to other workers. |
++---------------------------------------------+----------------------------------------------+
+| dask_worker_threads                         | Number of worker threads.                    |
++---------------------------------------------+----------------------------------------------+
+| dask_worker_latency_seconds                 | Latency of worker connection.                |
++---------------------------------------------+----------------------------------------------+
+| dask_worker_tick_duration_median_seconds    | Median tick duration at worker.              |
++---------------------------------------------+----------------------------------------------+
+| dask_worker_task_duration_median_seconds    | Median task runtime at worker.               |
++---------------------------------------------+----------------------------------------------+
+| dask_worker_transfer_bandwidth_median_bytes | Bandwidth for transfer at worker in Bytes.   |
++---------------------------------------------+----------------------------------------------+
+

From 386ee6c181272d398881e0a749176671e5c3ed61 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 23 Sep 2019 08:21:23 -0500
Subject: [PATCH 0470/1550] Respect Cluster.dashboard_link in
 Client._repr_html_ if it exists (#3077)

---
 distributed/client.py            | 20 +++++++++++++-------
 distributed/tests/test_client.py | 13 +++++++++++++
 2 files changed, 26 insertions(+), 7 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 63d8213c33f..08fda7539cf 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -811,13 +811,19 @@ def _repr_html_(self):
             text += "  <li><b>Scheduler: not connected</b></li>\n"
 
         if info and "dashboard" in info["services"]:
-            protocol, rest = scheduler.address.split("://")
-            port = info["services"]["dashboard"]
-            if protocol == "inproc":
-                host = "localhost"
-            else:
-                host = rest.split(":")[0]
-            address = format_dashboard_link(host, port)
+            try:
+                address = self.cluster.dashboard_link
+            except AttributeError:
+                protocol, rest = scheduler.address.split("://")
+
+                port = info["services"]["dashboard"]
+                if protocol == "inproc":
+                    host = "localhost"
+                else:
+                    host = rest.split(":")[0]
+
+                address = format_dashboard_link(host, port)
+
             text += (
                 "  <li><b>Dashboard: </b><a href='%(web)s' target='_blank'>%(web)s</a>\n"
                 % {"web": address}
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index a4e472c882a..dfb0eb59386 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -91,6 +91,7 @@
 from distributed.utils_test import (  # noqa: F401
     client as c,
     client_secondary as c2,
+    cleanup,
     cluster_fixture,
     loop,
     loop_in_thread,
@@ -5618,5 +5619,17 @@ async def test_file_descriptors_dont_leak(Worker):
         assert time() < begin + 5, (start, proc.num_fds())
 
 
+@pytest.mark.asyncio
+async def test_dashboard_link_cluster(cleanup):
+    class MyCluster(LocalCluster):
+        @property
+        def dashboard_link(self):
+            return "http://foo.com"
+
+    async with MyCluster(processes=False, asynchronous=True) as cluster:
+        async with Client(cluster, asynchronous=True) as client:
+            assert "http://foo.com" in client._repr_html_()
+
+
 if sys.version_info >= (3, 5):
     from distributed.tests.py3_test_client import *  # noqa F401

From 549660e07c0c70fdb17e07c6a18ca438933bd8ba Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 23 Sep 2019 08:21:39 -0500
Subject: [PATCH 0471/1550] Have Client get Security from passed Cluster
 (#3079)

---
 distributed/client.py                  | 17 ++++++++++-------
 distributed/deploy/tests/test_local.py | 16 ++++++++++++++++
 2 files changed, 26 insertions(+), 7 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 08fda7539cf..5ca99b7f156 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -637,14 +637,7 @@ def __init__(
         self._gather_future = None
 
         # Communication
-        self.security = security or Security()
         self.scheduler_comm = None
-        assert isinstance(self.security, Security)
-
-        if name == "worker":
-            self.connection_args = self.security.get_connection_args("worker")
-        else:
-            self.connection_args = self.security.get_connection_args("client")
 
         if address is None:
             address = dask.config.get("scheduler-address", None)
@@ -658,6 +651,16 @@ def __init__(
             self.cluster = address
             with ignoring(AttributeError):
                 loop = address.loop
+            if security is None:
+                security = self.cluster.security
+
+        self.security = security or Security()
+        assert isinstance(self.security, Security)
+
+        if name == "worker":
+            self.connection_args = self.security.get_connection_args("worker")
+        else:
+            self.connection_args = self.security.get_connection_args("client")
 
         self._connecting_to_scheduler = False
         self._asynchronous = asynchronous
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index 6611bfccc38..520c99eb268 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -26,6 +26,7 @@
     assert_can_connect_from_everywhere_4,
     assert_can_connect_from_everywhere_4_6,
     captured_logger,
+    tls_only_security,
 )
 from distributed.utils_test import loop  # noqa: F401
 from distributed.utils import sync
@@ -952,3 +953,18 @@ async def test_repr(cleanup):
         n_workers=2, processes=False, memory_limit=None, asynchronous=True
     ) as cluster:
         assert "memory" not in repr(cluster)
+
+
+@pytest.mark.asyncio
+async def test_capture_security(cleanup):
+    security = tls_only_security()
+    async with LocalCluster(
+        n_workers=0,
+        silence_logs=False,
+        security=security,
+        asynchronous=True,
+        dashboard_address=False,
+        host="tls://0.0.0.0",
+    ) as cluster:
+        async with Client(cluster, asynchronous=True) as client:
+            assert client.security == cluster.security

From 4953f81d7b6856f6a22f00e27476d4a554a120c2 Mon Sep 17 00:00:00 2001
From: Daniel Farrell <danpf@uw.edu>
Date: Wed, 25 Sep 2019 09:23:52 -0700
Subject: [PATCH 0472/1550] Add configuation option for longer error tracebacks
 (#3086)

---
 distributed/core.py              |  5 +++--
 distributed/distributed.yaml     |  1 +
 distributed/tests/test_worker.py | 16 ++++++++++++++++
 3 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/distributed/core.py b/distributed/core.py
index d0f3c13aa97..32b509dc170 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -969,8 +969,9 @@ def error_message(e, status="error"):
     --------
     clean_exception: deserialize and unpack message into exception/traceback
     """
+    MAX_ERROR_LEN = dask.config.get("distributed.admin.max-error-length")
     tb = get_traceback()
-    e2 = truncate_exception(e, 1000)
+    e2 = truncate_exception(e, MAX_ERROR_LEN)
     try:
         e3 = protocol.pickle.dumps(e2)
         protocol.pickle.loads(e3)
@@ -982,7 +983,7 @@ def error_message(e, status="error"):
     except Exception:
         tb = tb2 = "".join(traceback.format_tb(tb))
 
-    if len(tb2) > 10000:
+    if len(tb2) > MAX_ERROR_LEN:
         tb_result = None
     else:
         tb_result = protocol.to_serialize(tb)
diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index f277eb2f90b..7d012a2f68b 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -117,6 +117,7 @@ distributed:
       interval: 20ms  # time between event loop health checks
       limit: 3s       # time allowed before triggering a warning
 
+    max-error-length: 10000 # Maximum size traceback after error to return
     log-length: 10000  # default length of logs to keep in memory
     log-format: '%(name)s - %(levelname)s - %(message)s'
     pdb-on-err: False       # enter debug mode on scheduling error
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index aa0de9fd90f..eb4c0f86c7b 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -356,6 +356,22 @@ def __str__(self):
     msg = error_message(MyException("Hello", "World!"))
     assert "Hello" in str(msg["exception"])
 
+    max_error_len = 100
+    with dask.config.set({"distributed.admin.max-error-length": max_error_len}):
+        msg = error_message(RuntimeError("-" * max_error_len))
+        assert len(msg["text"]) <= max_error_len
+        assert len(msg["text"]) < max_error_len * 2
+        msg = error_message(RuntimeError("-" * max_error_len * 20))
+        cut_text = msg["text"].replace("('Long error message', '", "")[:-2]
+        assert len(cut_text) == max_error_len
+
+    max_error_len = 1000000
+    with dask.config.set({"distributed.admin.max-error-length": max_error_len}):
+        msg = error_message(RuntimeError("-" * max_error_len * 2))
+        cut_text = msg["text"].replace("('Long error message', '", "")[:-2]
+        assert len(cut_text) == max_error_len
+        assert len(msg["text"]) > 10100  # default + 100
+
 
 @gen_cluster()
 def test_gather(s, a, b):

From dec8abead0c460fece9b9d15f216bee5ca2e0d11 Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Thu, 26 Sep 2019 14:42:24 +0100
Subject: [PATCH 0473/1550] Make Client.get_versions async friendly (#3064)

---
 distributed/client.py            | 13 +++++++------
 distributed/tests/test_client.py |  5 +++++
 2 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 5ca99b7f156..381516da5c6 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -3463,18 +3463,19 @@ def get_versions(self, check=False, packages=[]):
 
         >>> c.get_versions(packages=['sklearn', 'geopandas'])  # doctest: +SKIP
         """
+        return self.sync(self._get_versions, check=check, packages=packages)
+
+    async def _get_versions(self, check=False, packages=[]):
         client = get_versions(packages=packages)
         try:
-            scheduler = sync(self.loop, self.scheduler.versions, packages=packages)
+            scheduler = await self.scheduler.versions(packages=packages)
         except KeyError:
             scheduler = None
         except TypeError:  # packages keyword not supported
-            scheduler = sync(self.loop, self.scheduler.versions)  # this raises
+            scheduler = await self.scheduler.versions()  # this raises
 
-        workers = sync(
-            self.loop,
-            self.scheduler.broadcast,
-            msg={"op": "versions", "packages": packages},
+        workers = await self.scheduler.broadcast(
+            msg={"op": "versions", "packages": packages}
         )
         result = {"scheduler": scheduler, "workers": workers, "client": client}
 
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index dfb0eb59386..a95548689d1 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -3725,6 +3725,11 @@ def test_get_versions(c):
     assert dict(v["client"]["packages"]["optional"])["requests"] == requests.__version__
 
 
+@gen_cluster(client=True)
+async def test_async_get_versions(c, s, a, b):
+    await c.get_versions(check=True)
+
+
 def test_threaded_get_within_distributed(c):
     import dask.multiprocessing
 

From f7f6bd77ab8d28d8811b94aa84528d3a470f1ff1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lo=C3=AFc=20Est=C3=A8ve?= <loic.esteve@ymail.com>
Date: Thu, 26 Sep 2019 16:26:04 +0200
Subject: [PATCH 0474/1550] Fix widget with spec that generates multiple
 workers (#3067)

---
 distributed/deploy/cluster.py                 | 5 ++++-
 distributed/deploy/tests/test_spec_cluster.py | 3 +++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
index ad47881ba22..033f6877684 100644
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -207,7 +207,10 @@ def dashboard_link(self):
     def _widget_status(self):
         workers = len(self.scheduler_info["workers"])
         if hasattr(self, "worker_spec"):
-            requested = len(self.worker_spec)
+            requested = sum(
+                1 if "group" not in each else len(each["group"])
+                for each in self.worker_spec.values()
+            )
         elif hasattr(self, "workers"):
             requested = len(self.workers)
         else:
diff --git a/distributed/deploy/tests/test_spec_cluster.py b/distributed/deploy/tests/test_spec_cluster.py
index d8c155c7c69..485ae1989ea 100644
--- a/distributed/deploy/tests/test_spec_cluster.py
+++ b/distributed/deploy/tests/test_spec_cluster.py
@@ -1,4 +1,5 @@
 import asyncio
+import re
 
 import dask
 from dask.distributed import SpecCluster, Worker, Client, Scheduler, Nanny
@@ -375,6 +376,8 @@ async def test_MultiWorker(cleanup):
             await client.wait_for_workers(4)
 
             assert "workers=4" in repr(cluster)
+            workers_line = re.search("(Workers.+)", cluster._widget_status()).group(1)
+            assert re.match("Workers.*<td>4</td>", workers_line)
 
             cluster.scale(1)
             await cluster

From 031e3a29edd9b4979b910c4773586881b2b26401 Mon Sep 17 00:00:00 2001
From: byjott <jochen.ott@blue-yonder.com>
Date: Thu, 26 Sep 2019 20:18:42 +0200
Subject: [PATCH 0475/1550] Fix worker preload config (#3027)

* Fix preload option handling

Fixes #3026

* Get preload from config in nanny
---
 distributed/nanny.py              |  8 ++++++--
 distributed/tests/test_preload.py | 20 ++++++++++++++++++--
 2 files changed, 24 insertions(+), 4 deletions(-)

diff --git a/distributed/nanny.py b/distributed/nanny.py
index c017eb54af7..06b5a27dc79 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -75,8 +75,8 @@ def __init__(
         resources=None,
         silence_logs=None,
         death_timeout=None,
-        preload=(),
-        preload_argv=[],
+        preload=None,
+        preload_argv=None,
         security=None,
         contact_address=None,
         listen_address=None,
@@ -116,7 +116,11 @@ def __init__(
         self.resources = resources
         self.death_timeout = parse_timedelta(death_timeout)
         self.preload = preload
+        if self.preload is None:
+            self.preload = dask.config.get("distributed.worker.preload")
         self.preload_argv = preload_argv
+        if self.preload_argv is None:
+            self.preload_argv = dask.config.get("distributed.worker.preload-argv")
         self.Worker = Worker if worker_class is None else worker_class
         self.env = env or {}
         worker_kwargs.update(
diff --git a/distributed/tests/test_preload.py b/distributed/tests/test_preload.py
index 9ce804b752a..d3171ed6842 100644
--- a/distributed/tests/test_preload.py
+++ b/distributed/tests/test_preload.py
@@ -1,12 +1,14 @@
 import os
+import pytest
 import shutil
 import sys
 import tempfile
 import pytest
 
-from distributed import Client, Scheduler, Worker
+import dask
+from distributed import Client, Scheduler, Worker, Nanny
 from distributed.utils_test import cluster
-from distributed.utils_test import cleanup, loop  # noqa F401
+from distributed.utils_test import loop, cleanup  # noqa F401
 
 
 PRELOAD_TEXT = """
@@ -55,6 +57,20 @@ def dask_setup(worker):
             assert w.foo == "setup"
 
 
+@pytest.mark.asyncio
+async def test_worker_preload_config(cleanup):
+    text = """
+def dask_setup(worker):
+    worker.foo = 'setup'
+"""
+    with dask.config.set({"distributed.worker.preload": text}):
+        async with Scheduler(port=0) as s:
+            async with Nanny(s.address) as w:
+                async with Client(s.address, asynchronous=True) as c:
+                    d = await c.run(lambda dask_worker: dask_worker.foo)
+                    assert d == {w.worker_address: "setup"}
+
+
 def test_worker_preload_module(loop):
     def check_worker():
         import worker_info

From 316aedbae2ee9d8b14b6dbfb220e191bd15aa4c2 Mon Sep 17 00:00:00 2001
From: "Mads R. B. Kristensen" <madsbk@gmail.com>
Date: Fri, 27 Sep 2019 15:41:46 +0200
Subject: [PATCH 0476/1550] Use the new UCX Python bindings (#3059)

See also https://github.com/rapidsai/ucx-py/pull/180
---
 distributed/comm/tests/test_ucx.py |  52 +---------
 distributed/comm/ucx.py            | 157 ++++++++++++++---------------
 2 files changed, 82 insertions(+), 127 deletions(-)

diff --git a/distributed/comm/tests/test_ucx.py b/distributed/comm/tests/test_ucx.py
index 1355daf95b8..28348369899 100644
--- a/distributed/comm/tests/test_ucx.py
+++ b/distributed/comm/tests/test_ucx.py
@@ -32,11 +32,6 @@ async def get_comm_pair(
     async def handle_comm(comm):
         await q.put(comm)
 
-    # Workaround for hanging test in
-    # pytest distributed/comm/tests/test_ucx.py::test_comm_objs -vs --count=2
-    # on the second time through.
-    ucp._libs.ucp_py.fin()
-
     listener = listen(listen_addr, handle_comm, connection_args=listen_args, **kwargs)
     with listener:
         comm = await connect(
@@ -93,6 +88,7 @@ async def handle_comm(comm):
             msg = await comm.read()
             msg["op"] = "pong"
             await comm.write(msg)
+            await comm.read()
             assert comm.closed() is False
             await comm.close()
             assert comm.closed
@@ -118,11 +114,9 @@ async def client_communicate(key, delay=0):
                 await asyncio.sleep(delay)
             msg = await comm.read()
             assert msg == {"op": "pong", "data": key}
+            await comm.write({"op": "client closed"})
             l.append(key)
             return comm
-            assert comm.closed() is False
-            await comm.close()
-            assert comm.closed
 
         comm = await client_communicate(key=1234, delay=0.5)
 
@@ -177,6 +171,7 @@ def test_ucx_deserialize():
         lambda cudf: cudf.DataFrame({"a": [1.0]}).head(0),
         lambda cudf: cudf.DataFrame({"a": [1]}).head(0),
         lambda cudf: cudf.DataFrame({"a": [1, 2, None], "b": [1.0, 2.0, None]}),
+        lambda cudf: cudf.DataFrame({"a": ["Check", "str"], "b": ["Sup", "port"]}),
     ],
 )
 async def test_ping_pong_cudf(g):
@@ -264,24 +259,18 @@ async def test_ping_pong_numba():
     assert result["op"] == "ping"
 
 
-@pytest.mark.skip(reason="hangs")
 @pytest.mark.parametrize("processes", [True, False])
 def test_ucx_localcluster(loop, processes):
     if processes:
-        kwargs = {"env": {"UCX_MEMTYPE_CACHE": "n"}}
-    else:
-        kwargs = {}
+        pytest.skip("Known bug, processes=True doesn't work currently")
 
-    ucx_addr = ucp.get_address()
     with LocalCluster(
         protocol="ucx",
-        interface="ib0",
         dashboard_address=None,
         n_workers=2,
         threads_per_worker=1,
         processes=processes,
         loop=loop,
-        **kwargs
     ) as cluster:
         with Client(cluster) as client:
             x = client.submit(inc, 1)
@@ -292,47 +281,16 @@ def test_ucx_localcluster(loop, processes):
             assert len(cluster.scheduler.workers) == 2
 
 
-def test_tcp_localcluster(loop):
-    ucx_addr = "127.0.0.1"
-    port = 13337
-    env = {"UCX_MEMTYPE_CACHE": "n"}
-    with LocalCluster(
-        2,
-        scheduler_port=port,
-        ip=ucx_addr,
-        processes=True,
-        threads_per_worker=1,
-        dashboard_address=None,
-        silence_logs=False,
-        env=env,
-    ) as cluster:
-        pass
-        # with Client(cluster) as e:
-        #     x = e.submit(inc, 1)
-        #     x.result()
-        #     assert x.key in c.scheduler.tasks
-        #     assert any(w.data == {x.key: 2} for w in c.workers)
-        #     assert e.loop is c.loop
-        #     print(c.scheduler.workers)
-
-
 @pytest.mark.slow
 @pytest.mark.asyncio
 async def test_stress():
-    from distributed.utils import get_ip_interface
-
-    try:  # this check should be removed once UCX + TCP works
-        get_ip_interface("ib0")
-    except Exception:
-        pytest.skip("ib0 interface not found")
-
     import dask.array as da
     from distributed import wait
 
     chunksize = "10 MB"
 
     async with LocalCluster(
-        protocol="ucx", interface="ib0", asynchronous=True
+        protocol="ucx", dashboard_address=None, asynchronous=True, processes=False
     ) as cluster:
         async with Client(cluster, asynchronous=True) as client:
             rs = da.random.RandomState()
diff --git a/distributed/comm/ucx.py b/distributed/comm/ucx.py
index 8631bb18229..bbf2451e323 100644
--- a/distributed/comm/ucx.py
+++ b/distributed/comm/ucx.py
@@ -5,9 +5,7 @@
 
 .. _UCX: https://github.com/openucx/ucx
 """
-import asyncio
 import logging
-import struct
 
 from .addressing import parse_host_port, unparse_host_port
 from .core import Comm, Connector, Listener, CommClosedError
@@ -15,7 +13,10 @@
 from .utils import ensure_concrete_host, to_frames, from_frames
 from ..utils import ensure_ip, get_ip, get_ipv6, nbytes, log_errors
 
+from tornado.ioloop import IOLoop
 import ucp
+import numpy as np
+import numba.cuda
 
 import os
 
@@ -95,80 +96,100 @@ async def write(
         on_error: str = "message",
     ):
         with log_errors():
+            if self.closed():
+                raise CommClosedError("Endpoint is closed -- unable to send message")
+
             if serializers is None:
                 serializers = ("cuda", "dask", "pickle", "error")
             # msg can also be a list of dicts when sending batched messages
             frames = await to_frames(msg, serializers=serializers, on_error=on_error)
-            is_gpus = b"".join(
-                [
-                    struct.pack("?", hasattr(frame, "__cuda_array_interface__"))
-                    for frame in frames
-                ]
-            )
-            sizes = b"".join([struct.pack("Q", nbytes(frame)) for frame in frames])
-
-            nframes = struct.pack("Q", len(frames))
-
-            meta = b"".join([nframes, is_gpus, sizes])
-
-            await self.ep.send_obj(meta)
 
+            # Send meta data
+            await self.ep.send(np.array([len(frames)], dtype=np.uint64))
+            await self.ep.send(
+                np.array(
+                    [hasattr(f, "__cuda_array_interface__") for f in frames],
+                    dtype=np.bool,
+                )
+            )
+            await self.ep.send(np.array([nbytes(f) for f in frames], dtype=np.uint64))
+            # Send frames
             for frame in frames:
-                await self.ep.send_obj(frame)
+                if nbytes(frame) > 0:
+                    if hasattr(frame, "__array_interface__") or hasattr(
+                        frame, "__cuda_array_interface__"
+                    ):
+                        await self.ep.send(frame)
+                    else:
+                        await self.ep.send(frame)
             return sum(map(nbytes, frames))
 
     async def read(self, deserializers=("cuda", "dask", "pickle", "error")):
         with log_errors():
+            if self.closed():
+                raise CommClosedError("Endpoint is closed -- unable to read message")
+
             if deserializers is None:
                 deserializers = ("cuda", "dask", "pickle", "error")
-            resp = await self.ep.recv_future()
-            obj = ucp.get_obj_from_msg(resp)
-            (nframes,) = struct.unpack(
-                "Q", obj[:8]
-            )  # first eight bytes for number of frames
-
-            gpu_frame_msg = obj[
-                8 : 8 + nframes
-            ]  # next nframes bytes for if they're GPU frames
-            is_gpus = struct.unpack("{}?".format(nframes), gpu_frame_msg)
-
-            sized_frame_msg = obj[8 + nframes :]  # then the rest for frame sizes
-            sizes = struct.unpack("{}Q".format(nframes), sized_frame_msg)
-
-            frames = []
-
-            for i, (is_gpu, size) in enumerate(zip(is_gpus, sizes)):
-                if size > 0:
-                    resp = await self.ep.recv_obj(size, cuda=is_gpu)
-                else:
-                    resp = await self.ep.recv_future()
-                frame = ucp.get_obj_from_msg(resp)
-                frames.append(frame)
-
-            msg = await from_frames(
-                frames, deserialize=self.deserialize, deserializers=deserializers
-            )
 
-            return msg
+            try:
+                # Recv meta data
+                nframes = np.empty(1, dtype=np.uint64)
+                await self.ep.recv(nframes)
+                is_cudas = np.empty(nframes[0], dtype=np.bool)
+                await self.ep.recv(is_cudas)
+                sizes = np.empty(nframes[0], dtype=np.uint64)
+                await self.ep.recv(sizes)
+            except (ucp.exceptions.UCXCanceled, ucp.exceptions.UCXCloseError):
+                if self._ep is not None and not self._ep.closed():
+                    await self._ep.shutdown()
+                    self._ep.close()
+                self._ep = None
+                raise CommClosedError("While reading, the connection was canceled")
+            else:
+                # Recv frames
+                frames = []
+                for is_cuda, size in zip(is_cudas.tolist(), sizes.tolist()):
+                    if size > 0:
+                        if is_cuda:
+                            frame = numba.cuda.device_array((size,), dtype=np.uint8)
+                        else:
+                            frame = np.empty(size, dtype=np.uint8)
+                        await self.ep.recv(frame)
+                        if is_cuda:
+                            frames.append(frame)
+                        else:
+                            frames.append(frame.data)
+                    else:
+                        if is_cuda:
+                            frames.append(numba.cuda.device_array((0,), dtype=np.uint8))
+                        else:
+                            frames.append(b"")
+                msg = await from_frames(
+                    frames, deserialize=self.deserialize, deserializers=deserializers
+                )
+                return msg
+
+    async def close(self):
+        if self._ep is not None:
+            if not self._ep.closed():
+                await self._ep.signal_shutdown()
+                self._ep.close()
+            self._ep = None
 
     def abort(self):
-        if self._ep:
-            ucp.destroy_ep(self._ep)
+        if self._ep is not None:
             logger.debug("Destroyed UCX endpoint")
+            IOLoop.current().add_callback(self._ep.signal_shutdown)
             self._ep = None
 
     @property
     def ep(self):
-        if self._ep:
+        if self._ep is not None:
             return self._ep
         else:
             raise CommClosedError("UCX Endpoint is closed")
 
-    async def close(self):
-        # TODO: Handle in-flight messages?
-        # sleep is currently used to help flush buffer
-        self.abort()
-
     def closed(self):
         return self._ep is None
 
@@ -180,9 +201,8 @@ class UCXConnector(Connector):
 
     async def connect(self, address: str, deserialize=True, **connection_args) -> UCX:
         logger.debug("UCXConnector.connect: %s", address)
-        ucp.init()
         ip, port = parse_host_port(address)
-        ep = await ucp.get_endpoint(ip.encode(), port)
+        ep = await ucp.create_endpoint(ip, port)
         return self.comm_class(
             ep,
             local_addr=None,
@@ -206,12 +226,8 @@ def __init__(
         self.comm_handler = comm_handler
         self.deserialize = deserialize
         self._ep = None  # type: ucp.Endpoint
-        self.listener_instance = None  # type: ucp.ListenerFuture
         self.ucp_server = None
-        self._task = None
-
         self.connection_args = connection_args
-        self._task = None
 
     @property
     def port(self):
@@ -222,39 +238,20 @@ def address(self):
         return "ucx://" + self.ip + ":" + str(self.port)
 
     def start(self):
-        async def serve_forever(client_ep, listener_instance):
+        async def serve_forever(client_ep):
             ucx = UCX(
                 client_ep,
                 local_addr=self.address,
                 peer_addr=self.address,  # TODO: https://github.com/Akshay-Venkatesh/ucx-py/issues/111
                 deserialize=self.deserialize,
             )
-            self.listener_instance = listener_instance
             if self.comm_handler:
                 await self.comm_handler(ucx)
 
-        ucp.init()
-        self.ucp_server = ucp.start_listener(
-            serve_forever, listener_port=self._input_port, is_coroutine=True
-        )
-
-        try:
-            loop = asyncio.get_running_loop()
-        except (RuntimeError, AttributeError):
-            loop = asyncio.get_event_loop()
-
-        t = loop.create_task(self.ucp_server.coroutine)
-        self._task = t
+        self.ucp_server = ucp.create_listener(serve_forever, port=self._input_port)
 
     def stop(self):
-        # What all should this do?
-        if self._task:
-            self._task.cancel()
-
-        if self._ep:
-            ucp.destroy_ep(self._ep)
-        # if self.listener_instance:
-        #   ucp.stop_listener(self.listener_instance)
+        self.ucp_server = None
 
     def get_host_port(self):
         # TODO: TCP raises if this hasn't started yet.

From 50f11f3cbca6b3b8d42c95dd9431f191a9680c89 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 27 Sep 2019 10:47:37 -0500
Subject: [PATCH 0477/1550] bump version to 2.5.0

---
 docs/source/changelog.rst | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index 6bed266f32b..a55a6923a29 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,25 @@
 Changelog
 =========
 
+2.5.0 - 2019-09-27
+------------------
+
+-  Use the new UCX Python bindings (:pr:`3059`) `Mads R. B. Kristensen`_
+-  Fix worker preload config (:pr:`3027`) `byjott`_
+-  Fix widget with spec that generates multiple workers (:pr:`3067`) `Loïc Estève`_
+-  Make Client.get_versions async friendly (:pr:`3064`) `Jacob Tomlinson`_
+-  Add configuation option for longer error tracebacks (:pr:`3086`) `Daniel Farrell`_
+-  Have Client get Security from passed Cluster (:pr:`3079`) `Matthew Rocklin`_
+-  Respect Cluster.dashboard_link in Client._repr_html_ if it exists (:pr:`3077`) `Matthew Rocklin`_
+-  Add monitoring with dask cluster docs (:pr:`3072`) `Arpit Solanki`_
+-  Protocol of cupy and numba handles serialization exclusively  (:pr:`3047`) `Mads R. B. Kristensen`_
+-  Allow specification of worker type in SSHCLuster (:pr:`3061`) `Jacob Tomlinson`_
+-  Use Cluster.scheduler_info for workers= value in repr (:pr:`3058`) `Matthew Rocklin`_
+-  Allow SpecCluster to scale by memory and cores (:pr:`3057`) `Matthew Rocklin`_
+-  Allow full script in preload inputs (:pr:`3052`) `Matthew Rocklin`_
+-  Check multiple cgroups dirs, ceil fractional cpus (:pr:`3056`) `Jim Crist`_
+-  Add blurb about disabling work stealing (:pr:`3055`) `Chris White`_
+
 2.4.0 - 2019-09-13
 ------------------
 
@@ -1268,3 +1287,4 @@ significantly without many new features.
 .. _`byjott`: https://github.com/byjott
 .. _`Mohammad Noor`: https://github.com/MdSalih
 .. _`Richard J Zamora`: https://github.com/rjzamora
+.. _`Arpit Solanki`: https://github.com/arpit1997

From 1739811c48a73058e0db5eed38e8d46d0864d6d6 Mon Sep 17 00:00:00 2001
From: Daniel Farrell <danpf@uw.edu>
Date: Fri, 27 Sep 2019 10:11:00 -0700
Subject: [PATCH 0478/1550] Set known task durations with configuration (#3085)

---
 distributed/distributed.yaml        |  1 +
 distributed/scheduler.py            |  4 ++++
 distributed/tests/test_scheduler.py | 25 ++++++++++++++++++++++++-
 3 files changed, 29 insertions(+), 1 deletion(-)

diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index 7d012a2f68b..48484be12a6 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -23,6 +23,7 @@ distributed:
     pickle: True            # Is the scheduler allowed to deserialize arbitrary bytestrings
     preload: []
     preload-argv: []
+    default-task-durations: {}  # How long we expect function names to run ("1h", "1s") (helps for long tasks)
     dashboard:
       status:
         task-stream-length: 1000
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 6bf1ba0adb4..bdee7c7bdac 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -964,6 +964,10 @@ def __init__(
 
         # Prefix-keyed containers
         self.task_duration = {prefix: 0.00001 for prefix in fast_tasks}
+        for k, v in dask.config.get(
+            "distributed.scheduler.default-task-durations", {}
+        ).items():
+            self.task_duration[k] = parse_timedelta(v)
         self.unknown_durations = defaultdict(set)
 
         # Client state
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 943daffde58..71a19eba54f 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -17,7 +17,7 @@
 
 from distributed import Nanny, Worker, Client, wait, fire_and_forget
 from distributed.core import connect, rpc
-from distributed.scheduler import Scheduler
+from distributed.scheduler import Scheduler, TaskState
 from distributed.client import wait
 from distributed.metrics import time
 from distributed.protocol.pickle import dumps
@@ -1641,3 +1641,26 @@ async def test_retire_names_str(cleanup):
                     await s.retire_workers(names=[0])
                     assert all(f.done() for f in futures)
                     assert len(b.data) == 10
+
+
+def test_get_task_duration():
+    with dask.config.set(
+        {"distributed.scheduler.default-task-durations": {"prefix_1": 100}}
+    ):
+        s = Scheduler(port=0)
+        assert "prefix_1" in s.task_duration
+        assert s.task_duration["prefix_1"] == 100
+
+        ts_pref1 = TaskState("prefix_1-abcdefab", None)
+        assert s.get_task_duration(ts_pref1) == 100
+
+        # make sure get_task_duration adds TaskStates to unknown dict
+        assert len(s.unknown_durations) == 0
+        ts_pref2 = TaskState("prefix_2-abcdefab", None)
+        assert s.get_task_duration(ts_pref2) == 0.5  # default
+        assert len(s.unknown_durations) == 1
+        assert len(s.unknown_durations["prefix_2"]) == 1
+        ts_pref2_2 = TaskState("prefix_2-accdefab", None)
+        assert s.get_task_duration(ts_pref2_2) == 0.5  # default
+        assert len(s.unknown_durations) == 1
+        assert len(s.unknown_durations["prefix_2"]) == 2

From 95a2f4cbec9b26b37a85120841479232afc48432 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Fri, 27 Sep 2019 16:47:12 -0500
Subject: [PATCH 0479/1550] Fix tornado typo in asynchronous docs (#3101)

---
 docs/source/asynchronous.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/source/asynchronous.rst b/docs/source/asynchronous.rst
index 9d38a8b04fe..1ffc8d4f5c3 100644
--- a/docs/source/asynchronous.rst
+++ b/docs/source/asynchronous.rst
@@ -108,7 +108,7 @@ Python 2/3 with Tornado
        future = client.submit(lambda x: x + 1, 10)
        result = yield future
        yield client.close()
-       raise gen.Result(result)
+       raise gen.Return(result)
 
    from tornado.ioloop import IOLoop
    IOLoop().run_sync(f)

From 9bea7d78aaf4f830f9ef905fe193908d8da806cf Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 27 Sep 2019 17:21:06 -0500
Subject: [PATCH 0480/1550] Support clusters that don't have .security or
 ._close methods (#3100)

Previously clients checked for attributes that may not be universally
present.  Now we only check on scheduler, scheduler_comm, close,
workers, and other elements defined in the `Cluster` interface.

Fixes https://github.com/dask/dask-jobqueue/issues/341
---
 distributed/client.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 381516da5c6..75f1fd03d9f 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -652,7 +652,7 @@ def __init__(
             with ignoring(AttributeError):
                 loop = address.loop
             if security is None:
-                security = self.cluster.security
+                security = getattr(self.cluster, "security", None)
 
         self.security = security or Security()
         assert isinstance(self.security, Security)
@@ -917,9 +917,7 @@ async def _start(self, timeout=no_default, **kwargs):
         if self.cluster is not None:
             # Ensure the cluster is started (no-op if already running)
             try:
-                await self.cluster._start()
-            except AttributeError:  # Some clusters don't have this method
-                pass
+                await self.cluster
             except Exception:
                 logger.info(
                     "Tried to start cluster and received an error. Proceeding.",
@@ -1266,7 +1264,7 @@ async def _close(self, fast=False):
                 self._release_key(key=key)
             if self._start_arg is None:
                 with ignoring(AttributeError):
-                    await self.cluster._close()
+                    await self.cluster.close()
             self.rpc.close()
             self.status = "closed"
             if _get_global_client() is self:

From 8bc04e559d3c4d744b0453cf4aada4eece28c35a Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 27 Sep 2019 17:22:21 -0500
Subject: [PATCH 0481/1550] bump version to 2.5.1

---
 docs/source/changelog.rst | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index a55a6923a29..aff09d2c3c7 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,13 @@
 Changelog
 =========
 
+
+2.5.1 - 2019-09-27
+------------------
+
+-   Support clusters that don't have .security or ._close methods (:pr:`3100`) `Matthew Rocklin`_
+
+
 2.5.0 - 2019-09-27
 ------------------
 
@@ -20,6 +27,7 @@ Changelog
 -  Check multiple cgroups dirs, ceil fractional cpus (:pr:`3056`) `Jim Crist`_
 -  Add blurb about disabling work stealing (:pr:`3055`) `Chris White`_
 
+
 2.4.0 - 2019-09-13
 ------------------
 

From a76bd8a2db4358aeb02e5f97e053b63ebc280eca Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Mon, 30 Sep 2019 15:48:46 +0100
Subject: [PATCH 0482/1550] Check if self.cluster.scheduler is a local
 scheduler (#3099)

---
 distributed/client.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/distributed/client.py b/distributed/client.py
index 75f1fd03d9f..94165b4c380 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -786,10 +786,12 @@ def __repr__(self):
             return "<%s: not connected>" % (self.__class__.__name__,)
 
     def _repr_html_(self):
+        from .scheduler import Scheduler
+
         if (
             self.cluster
             and hasattr(self.cluster, "scheduler")
-            and self.cluster.scheduler
+            and isinstance(self.cluster.scheduler, Scheduler)
         ):
             info = self.cluster.scheduler.identity()
             scheduler = self.cluster.scheduler

From 430760b0e45ad8bbaeac40de34a497e554df1389 Mon Sep 17 00:00:00 2001
From: Philipp Rudiger <prudiger@anaconda.com>
Date: Mon, 30 Sep 2019 17:12:02 +0200
Subject: [PATCH 0483/1550] Lower default bokeh log level (#3087)

---
 distributed/config.py      | 2 +-
 distributed/deploy/spec.py | 5 +++++
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/distributed/config.py b/distributed/config.py
index 7e6075125fd..a313f18416b 100644
--- a/distributed/config.py
+++ b/distributed/config.py
@@ -76,7 +76,7 @@ def _initialize_logging_old_style(config):
     loggers = {  # default values
         "distributed": "info",
         "distributed.client": "warning",
-        "bokeh": "critical",
+        "bokeh": "error",
         "tornado": "critical",
         "tornado.application": "error",
     }
diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index 87336f96184..5cc0722ef72 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -211,6 +211,9 @@ def __init__(
 
         if silence_logs:
             self._old_logging_level = silence_logging(level=silence_logs)
+            self._old_bokeh_logging_level = silence_logging(
+                level=silence_logs, root="bokeh"
+            )
 
         self._loop_runner = LoopRunner(loop=loop, asynchronous=asynchronous)
         self.loop = self._loop_runner.loop
@@ -350,6 +353,8 @@ async def _close(self):
 
         if hasattr(self, "_old_logging_level"):
             silence_logging(self._old_logging_level)
+        if hasattr(self, "_old_bokeh_logging_level"):
+            silence_logging(self._old_bokeh_logging_level, root="bokeh")
 
         await super()._close()
 

From 8b4cbe2ba6b4015bf9f7fd46730e3c2bc2b3464e Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 30 Sep 2019 10:12:27 -0500
Subject: [PATCH 0484/1550] Remove utils.py functions for their dask/utils.py
 equivalents (#3042)

---
 distributed/client.py |   3 +-
 distributed/utils.py  | 168 ++----------------------------------------
 distributed/worker.py |   3 +-
 requirements.txt      |   2 +-
 4 files changed, 11 insertions(+), 165 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 94165b4c380..d00f545b161 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -27,7 +27,7 @@
 from dask.core import flatten, get_dependencies
 from dask.optimization import SubgraphCallable
 from dask.compatibility import apply
-from dask.utils import ensure_dict, format_bytes
+from dask.utils import ensure_dict, format_bytes, funcname
 
 try:
     from cytoolz import first, groupby, merge, valmap, keymap
@@ -69,7 +69,6 @@
 from .utils import (
     All,
     sync,
-    funcname,
     ignoring,
     tokey,
     log_errors,
diff --git a/distributed/utils.py b/distributed/utils.py
index 015e4dbfb61..fbac950df43 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -10,7 +10,6 @@
 import json
 import logging
 import multiprocessing
-from numbers import Number
 import os
 import re
 import shutil
@@ -37,7 +36,14 @@
 from dask import istask
 
 # provide format_bytes here for backwards compatibility
-from dask.utils import format_bytes  # noqa
+from dask.utils import (  # noqa
+    format_bytes,
+    funcname,
+    format_time,
+    parse_bytes,
+    parse_timedelta,
+)
+
 import toolz
 import tornado
 from tornado import gen
@@ -80,16 +86,6 @@ def _initialize_mp_context():
 mp_context = _initialize_mp_context()
 
 
-def funcname(func):
-    """Get the name of a function."""
-    while hasattr(func, "func"):
-        func = func.func
-    try:
-        return func.__name__
-    except AttributeError:
-        return str(func)
-
-
 def has_arg(func, argname):
     """
     Whether the function takes an argument with the given name.
@@ -1086,135 +1082,6 @@ def __reduce__(self):
         return (itemgetter, (self.index,))
 
 
-byte_sizes = {
-    "kB": 10 ** 3,
-    "MB": 10 ** 6,
-    "GB": 10 ** 9,
-    "TB": 10 ** 12,
-    "PB": 10 ** 15,
-    "KiB": 2 ** 10,
-    "MiB": 2 ** 20,
-    "GiB": 2 ** 30,
-    "TiB": 2 ** 40,
-    "PiB": 2 ** 50,
-    "B": 1,
-    "": 1,
-}
-byte_sizes = {k.lower(): v for k, v in byte_sizes.items()}
-byte_sizes.update({k[0]: v for k, v in byte_sizes.items() if k and "i" not in k})
-byte_sizes.update({k[:-1]: v for k, v in byte_sizes.items() if k and "i" in k})
-
-
-def parse_bytes(s):
-    """ Parse byte string to numbers
-
-    >>> parse_bytes('100')
-    100
-    >>> parse_bytes('100 MB')
-    100000000
-    >>> parse_bytes('100M')
-    100000000
-    >>> parse_bytes('5kB')
-    5000
-    >>> parse_bytes('5.4 kB')
-    5400
-    >>> parse_bytes('1kiB')
-    1024
-    >>> parse_bytes('1e6')
-    1000000
-    >>> parse_bytes('1e6 kB')
-    1000000000
-    >>> parse_bytes('MB')
-    1000000
-    """
-    if isinstance(s, (int, float)):
-        return int(s)
-    s = s.replace(" ", "")
-    if not s[0].isdigit():
-        s = "1" + s
-
-    for i in range(len(s) - 1, -1, -1):
-        if not s[i].isalpha():
-            break
-    index = i + 1
-
-    prefix = s[:index]
-    suffix = s[index:]
-
-    n = float(prefix)
-
-    multiplier = byte_sizes[suffix.lower()]
-
-    result = n * multiplier
-    return int(result)
-
-
-timedelta_sizes = {
-    "s": 1,
-    "ms": 1e-3,
-    "us": 1e-6,
-    "ns": 1e-9,
-    "m": 60,
-    "h": 3600,
-    "d": 3600 * 24,
-}
-
-tds2 = {
-    "second": 1,
-    "minute": 60,
-    "hour": 60 * 60,
-    "day": 60 * 60 * 24,
-    "millisecond": 1e-3,
-    "microsecond": 1e-6,
-    "nanosecond": 1e-9,
-}
-tds2.update({k + "s": v for k, v in tds2.items()})
-timedelta_sizes.update(tds2)
-timedelta_sizes.update({k.upper(): v for k, v in timedelta_sizes.items()})
-
-
-def parse_timedelta(s, default="seconds"):
-    """ Parse timedelta string to number of seconds
-
-    Examples
-    --------
-    >>> parse_timedelta('3s')
-    3
-    >>> parse_timedelta('3.5 seconds')
-    3.5
-    >>> parse_timedelta('300ms')
-    0.3
-    >>> parse_timedelta(timedelta(seconds=3))  # also supports timedeltas
-    3.0
-    """
-    if s is None:
-        return None
-    if isinstance(s, timedelta):
-        return s.total_seconds()
-    if isinstance(s, Number):
-        s = str(s)
-    s = s.replace(" ", "")
-    if not s[0].isdigit():
-        s = "1" + s
-
-    for i in range(len(s) - 1, -1, -1):
-        if not s[i].isalpha():
-            break
-    index = i + 1
-
-    prefix = s[:index]
-    suffix = s[index:] or default
-
-    n = float(prefix)
-
-    multiplier = timedelta_sizes[suffix.lower()]
-
-    result = n * multiplier
-    if int(result) == result:
-        result = int(result)
-    return result
-
-
 def asciitable(columns, rows):
     """Formats an ascii table for given columns and rows.
 
@@ -1282,25 +1149,6 @@ def json_load_robust(fn, load=json.load):
         sleep(0.1)
 
 
-def format_time(n):
-    """ format integers as time
-
-    >>> format_time(1)
-    '1.00 s'
-    >>> format_time(0.001234)
-    '1.23 ms'
-    >>> format_time(0.00012345)
-    '123.45 us'
-    >>> format_time(123.456)
-    '123.46 s'
-    """
-    if n >= 1:
-        return "%.2f s" % n
-    if n >= 1e-3:
-        return "%.2f ms" % (n * 1e3)
-    return "%.2f us" % (n * 1e6)
-
-
 class DequeHandler(logging.Handler):
     """ A logging.Handler that records records into a deque """
 
diff --git a/distributed/worker.py b/distributed/worker.py
index bb00158ced8..2bd345ea0c5 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -17,7 +17,7 @@
 import dask
 from dask.core import istask
 from dask.compatibility import apply
-from dask.utils import format_bytes
+from dask.utils import format_bytes, funcname
 
 try:
     from cytoolz import pluck, partial, merge, first
@@ -43,7 +43,6 @@
 from .threadpoolexecutor import ThreadPoolExecutor, secede as tpe_secede
 from .utils import (
     get_ip,
-    funcname,
     typename,
     has_arg,
     _maybe_complex,
diff --git a/requirements.txt b/requirements.txt
index 804bdfe9637..e599ab0de93 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,6 @@
 click >= 6.6
 cloudpickle >= 0.2.2
-dask >= 2
+dask >= 2.3
 msgpack
 psutil >= 5.0
 sortedcontainers !=2.0.0, !=2.0.1

From 790a4c032128f651c67b671b18a732034346f291 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Mon, 30 Sep 2019 11:26:05 -0500
Subject: [PATCH 0485/1550] Add favicon of logo to the dashboard (#3095)

* Add favicon to dashboard

* Include in package data

* Use relative links
---
 MANIFEST.in                                   |   1 +
 distributed/dashboard/scheduler_html.py       |  26 +++++++++++++-----
 .../dashboard/static/images/favicon.ico       | Bin 0 -> 15406 bytes
 distributed/dashboard/templates/base.html     |   1 +
 distributed/dashboard/templates/main.html     |   1 +
 5 files changed, 22 insertions(+), 7 deletions(-)
 create mode 100755 distributed/dashboard/static/images/favicon.ico

diff --git a/MANIFEST.in b/MANIFEST.in
index b7a3764c87a..2a8f054e213 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -4,6 +4,7 @@ recursive-include distributed *.coffee
 recursive-include distributed *.html
 recursive-include distributed *.css
 recursive-include distributed *.svg
+recursive-include distributed *.ico
 recursive-include distributed *.yaml
 recursive-include docs *.rst
 
diff --git a/distributed/dashboard/scheduler_html.py b/distributed/dashboard/scheduler_html.py
index 1377b037173..6ac13915523 100644
--- a/distributed/dashboard/scheduler_html.py
+++ b/distributed/dashboard/scheduler_html.py
@@ -12,6 +12,8 @@
     func.__name__: func for func in [format_bytes, format_time, datetime.fromtimestamp]
 }
 
+rel_path_statics = {"rel_path_statics": "../../"}
+
 
 class Workers(RequestHandler):
     def get(self):
@@ -20,7 +22,7 @@ def get(self):
                 "workers.html",
                 title="Workers",
                 scheduler=self.server,
-                **toolz.merge(self.server.__dict__, ns, self.extra)
+                **toolz.merge(self.server.__dict__, ns, self.extra, rel_path_statics),
             )
 
 
@@ -36,7 +38,7 @@ def get(self, worker):
                 title="Worker: " + worker,
                 scheduler=self.server,
                 Worker=worker,
-                **toolz.merge(self.server.__dict__, ns, self.extra)
+                **toolz.merge(self.server.__dict__, ns, self.extra, rel_path_statics),
             )
 
 
@@ -52,7 +54,7 @@ def get(self, task):
                 title="Task: " + task,
                 Task=task,
                 scheduler=self.server,
-                **toolz.merge(self.server.__dict__, ns, self.extra)
+                **toolz.merge(self.server.__dict__, ns, self.extra, rel_path_statics),
             )
 
 
@@ -60,7 +62,12 @@ class Logs(RequestHandler):
     def get(self):
         with log_errors():
             logs = self.server.get_logs()
-            self.render("logs.html", title="Logs", logs=logs, **self.extra)
+            self.render(
+                "logs.html",
+                title="Logs",
+                logs=logs,
+                **toolz.merge(self.extra, rel_path_statics),
+            )
 
 
 class WorkerLogs(RequestHandler):
@@ -69,7 +76,12 @@ async def get(self, worker):
             worker = escape.url_unescape(worker)
             logs = await self.server.get_worker_logs(workers=[worker])
             logs = logs[worker]
-            self.render("logs.html", title="Logs: " + worker, logs=logs, **self.extra)
+            self.render(
+                "logs.html",
+                title="Logs: " + worker,
+                logs=logs,
+                **toolz.merge(self.extra, rel_path_statics),
+            )
 
 
 class WorkerCallStacks(RequestHandler):
@@ -82,7 +94,7 @@ async def get(self, worker):
                 "call-stack.html",
                 title="Call Stacks: " + worker,
                 call_stack=call_stack,
-                **self.extra
+                **toolz.merge(self.extra, rel_path_statics),
             )
 
 
@@ -101,7 +113,7 @@ async def get(self, key):
                     "call-stack.html",
                     title="Call Stack: " + key,
                     call_stack=call_stack,
-                    **self.extra
+                    **toolz.merge(self.extra, rel_path_statics),
                 )
 
 
diff --git a/distributed/dashboard/static/images/favicon.ico b/distributed/dashboard/static/images/favicon.ico
new file mode 100755
index 0000000000000000000000000000000000000000..eac169d0c7ef3caad9b48ca6c142a5351e9f7dcc
GIT binary patch
literal 15406
zcmeHO30zZWwh!7)7h7v>-Dd1y9k)_JAc2IiMD~3NBm_u8AV4520xGy06&D1=x}mrr
zim1e;&WxRo)5VS(TQ))<5J({VhJe_1-g`4|^3KJgs7OU)-<$WF`}^JB&As>g&N<(A
zzwa#nbAm$IpeCcHO+&$T9xCHC6v`EaLd~1^;C{*bC{z_Zb9DUa{UQ|V&I}Z43ABMO
z(Br{#xJ>ByB(#2^6Q%wW6vr}YLLGl<jEQJ+!qWtI30dMxzH4MoKS#rkbZ7dqNZ~YF
zM$elll*PuDaQ#hO0$<?87Db0{?7T_b(Un6cb0?2Y$6|w}^LFDCfnU0aA6Ov09I>%G
z4jae+GnOKp7mzQRPTDQ6qwZI(MR7P|j%NY2VJeQ!ug50yumP*2=x|PFk}HMx2Mkr<
zf&#t~yQS|Y9#;N2^@KibY#C4}ER7%Mo-924;5~P;3#s{AmjuBo52mmT&EmaIt<WrH
zoz~T7a*gkdMPorw!&GR?BakKU%=i21J#<u41KN-Go>#iKB5YH~PUKsppV9Ay{`9f>
z!!sLqrs#r4su*qWecQ`9ys6EZ%tr(7x1+XoU!@&X1*O&)aM>437f=?9%~&+tlLX}+
zEHN2{=DqF|*653;3%((+#p}YhcR42=(R|A|IWR5zf|18KiF~K04;(|7x123*4K9>k
zjoK<-8L&>g#5<?;NbnlzUglA475lu|CA-deC+o82gQt5VPb`xNum*zG$?T$wWQEjy
z$_7TIdQ0fWj<fI$w`H9-#$=s0++@`beK>ai$Dc;-9hjDKLR*qrrQH^@tNRXppT>z@
zs~52<wQoP(a-2Ju_sp|~__#fajnw_US6HXCf8tDZU%-4*_V&!9?&~o#k84(r!)=^b
zoJsL}du!tkD1SXpqj6tF?&_I0Zi5NFMDz?oa5<caezM*Hd)@Zrm}|^>Jf2NyfosE5
z=jf)*m^j`#m$=4Hkv;V6*01D^4Ti`o!o~B3K&PlY(nZHSb6Nu)=*L&W?<u0rM4k`m
z(7WzQ!XH2f4Ep6sG=YWELq6*yc8E@Vs)LmuEC{+W154qxfnFE8r?oJA@+4mbuaypl
zZ|-a*Zto%o<;dQ6F_sn^JMzunI#L_#9HRvb?TIaO(eb<w;TylVAm>jHY0mP$HU%5Y
zy8t>+|0$*EBSL25?7%f`xe=RX^5`AiT1u&+0I>@bl>rU9)s@CCbW0TI5SxW(2u+@;
zBB^&)>zDp3TZ@C%c94VD$`?8Ki>6>myka-1V8E5yfJ68Y2-jZ=-qh|7{O?fqDEbrj
zt2X+V>ZeRp20L=&Y+R!7iW@`l50FC%{BL<@iB*9s+Xh0`b__&vWTGgJjOjvdUW23Z
zhp==($Ri#Wn=nq7Go`HOue1Zb=ETG5Tmt9%g!d{sj)!$i<jdfcW0IO&JlU-u`Q)|*
z2Nibgh}<l@8OiC~;zDU&34XwREQ25Oh(GenVnf{B<xD-G<Ru@~=-_;&U4FdZs5_WT
z7}qLXqF@Nm5EMUJ(-xZ`j%;qgTFC-*OmjZ$fgx8qkL(o3cUT;8>(TcHUm|{=GCb+1
zTA5a@yODC*xL_3Z=lbbD7JZDP3+nO7B8eMQwDhq~hz*Wx-U9pwG4cEuES+Bmb5(*(
z5ziWFhxirAm8#9ae<-6?w=-zpME8Wn#+Axj4?5<adx|jo(Hx*qb8&axLKB-$z#iyx
zis55$bY2zcf<f<0akbBiM>0^4_RUQ_slCHKYfxmI*FQdn5kbefVq%+?0$sIdn&=By
z=Tk?TBL0w5L{q&hnI~}sU4SO@e(RPbI^&bwIvBF9qtbWhPd)_LVrG?w$v&&UmsMvd
znW$~HBVT_9_Tg2pv=*&bQrp52{-1i^Tn0Xr9uwVk$^mo%@_ZRd_#5AxR&Lk^=^*UE
zjRejEKPdTxVSYxfu_@<*sVVbfzwHQ(=eoi=gf+O%BUN<YizSL4>0|GGC*LWw@kghq
z#=SELl7~K{SGr(f*!qqe<YI*`Zhvp+aC=ybZ14?%zd7fkIedi1^IqK;!YG8lCrh~T
zK_Bj9d!O4ookAM!x<of?T`7XYu2g{?3S>q0Uhw*kVA3wRihfA-b>d;wZ;(E$T0>Me
z*EpDW$+(@&mHg^?c_ZyylX#vUDMCHs>yGsM0DtS<r2)62U=3cvB?$UpjkJMp%3934
z^{yRR@+(rYyaf0Uq@2*O$z03iq(hpIvd<edxu2V^Ft3;&?b8v0<6JS+CKnH;sK+Z^
z^f{U(e-H6Nk#Qnj>;mpghd3e$eB)L^hPclsw{;f|Vv4XGJxdY~sXAGwbdB+~+7EMT
z`le>p85?pho7-4$KMuR6Ub)4IoL~=jd!@Gw_-40$9kg1qFQl+7C3w9Q<G%{g@7&4Q
zM81P(hWHP@d2Rg=;~WiKBb`S-pe#y1rN0Y)wKw9^f}eUm_mauXzG#^L6q9itI3@6H
z-IDkfUaXc|J~<M3&}wO4`1+1}#I4<3q@D6B<Pv!TX@_iv8?(*UZ-wL(`11qd8#)ig
zma4p1)!L?Pu6Zltq~%T6gGW|eH1+3Qxclii?8bOyv6)M{^A@uI39OC}16E4$u*dU>
zTV$V;b}5tz<-LRSgQ{<nDm7#n`{JNN>G|+Yo&6Bk6r`TeC1!C=HOVJ*GuX9;gE<$B
z{Tb))PFN3(ArvYor(tUNjt+N9dCxxJuVNlk8`Dl|jz#b8@($bFaU;5@OHC}5ld`$G
zboK><L&h0HRSwtG$E?vT9*f?KJ`37sc_aRy(j%!-eL3x<t}netUly^on@B9`(m-7F
zW#Uogr>tt@@6u~@-#~n&OgU?KXd_;f=UDyXE43f8PU<$nJWDca^>I-}o#nA5@`1Rr
z9wx0qKNDgj1?b>g>{=xt6R+kV?NPl2F<f#^o$-8Zx!jMuTOo<x+jArFKo2qPl+Mhq
z)g7HseD?xkB6c>1Yb3!L1bgGNgwJ{h7>Cv0rJdB?OFykmc~*S=0<50dCFoC<H!@+L
zA!{+dT+x?Uq5M1OLPbio9)XB{nFHe5sGaQ}#FZ-=6A!BX0e%=aY$CC>6{JMs0N}re
zc0f6V=m#s1oFEB{O>9{&HQ}?KuSol3fWH0m2f3ujF}6$*_@a2-%Sue(SYAW3EU&*T
zo&T@>5u5}dT2vC8*2PJH;4=>-MjspRd}f#bGcPQY76;bPbBL(N0+zgZNmL`?g|8lS
zu-@=Fuh|2B>JroZl}l{%Rfw_junBzh(zyCrsGvs}gZ21XpVHE>+e=(xcwb=Scmsfe
z7(9}Ne2+BoUVMrm1Ou_UogH^VKECxO9G~LSaPcgc82(l)rP++9@&6nA`@eyIZSu};
z75d~#4*BJ_272bT&lw%_j!*HIeLE+N_qKBsFAU;uIi4!`FL$P}&of=520VSxZ>8j3
z&>E>Ecx}ff|8?C<AH*xatb9Mu&0_OO^mhwgV|fjLAr0b_1lQeDTe6mCiO>4xOB6wc
z?caxP=+K36WLLvE?O~y75ub7Fyu{D*`k59;+)9S^ikzt77<SA&(34ag{BO^6B{!b|
z49*M~0Uu4}IlE^HKlRCL$qFoJy9T)_UE~&7fAse5+ki=?MizB_WcBp~vO7>37op?p
zJ<!zV<=A+hhhwb31)b1h?-(yw=*HyRx-cZOU2|kpQGf@H9s}gjreR{5irf<Tdf3A~
zE^&Me62m*tL{q%-TkIi@-yg9>CI{JnB<<?%pp<oQ8qN_|5#L2{Oc$yk)s4=-g<y?v
zs@xKVO29qk9w{O|?8&RJClAAZ%=2B{=I6VzeG%k;X4o>Wzd3_&{T;yJS%3ldxX}b!
z7dkHt!KEIIkvrLcL&wKqoQ{=IMKTe&L@^Xs-qV|~Pq{6+ymS7ua($!c=lOnHCOMJ$
z3jlvR2C@TQEBGgz|8QprhCqHZf(Za7WcJT%QwJBcw}r0l_)FM^&f;*6EX+HTZ|h8K
zB;w-vosbVT0M^Gqa*Yr9!aRqpmduXaCQBgimj4yxA4=HY+f6;FDxe&KwPY>(4Sc``
zc>>P#jT>F?H+-V-2N+8;<O|P&{9gn1+!m16rVJ`*Gl#D0xF4~(b1-VFj32(SgN7kD
zMd0Fi?QS%_32>q;`vmb*u^k~(@=i>#A~vp6UQgZE`vbi~DNU->X44@ycsSOV{9z0E
zVGNy5hcR>jZharH(0bQIo;Q{uoadR<;taS6#eY@%j*vAS*CID{c13ON5=Cw5WML?R
zV8|m#aIlB548bY{>mN&$0FGojxl|Dg^Dh8Qdnoy+wuM=#ilV&$zCIRx<XhTP8|H%i
zdtmPMaMgnRs~t&#8OWZ*1Fi-0I6G*S#4dD07ddK6*Se7P?V)ILvp?8?Vfmro>Lt|X
zIkRX89{Hqi2e>t0G!P5rFe+5-%ww9t)RVg3CspZO0G|ik&I$*tBQ&+e8DziV#^8UC
zPZrjr6PqF4d|Va;pCe=%-$B;8V|cSaV7Tp&%Q9hUycOtJp0jfTFU$_`T+sJV<&kKm
z-7{&2m1Tgd8vs}COFgCA$vpnzInO6YJ)iv5d#*Gd2iA`PpCmH5B@0#|wFghO8@{)_
z{-#4nQ#9oAWgx!+d^lK7d<yL2ZxAdVm%y_}F!;y%0v1a@qQn3${{_f@pH-t1rPpYQ
zglFW;AM5*EH{8iiv5j6(uh4)`65e-D5q<7L7eVF6)5k0a(UfHo^=Tk~59|T0Yn*`O
z63?T;I%)@drUuOSm={azw%`e$G;(jxG}r@)X;s?J^wWBC<~c)k`bqsfYuRVWrTiL8
z6_$eheeO&_Cq7LSGgfXBv0ski4clE~nsp$*!a0_QbEF9HxI{jZyXq&TwHSSK+PHpM
zQVg=@M$Lc?wZ=3D@Q|ZWbI}L5Xb=0GF`9OI;<>U>G#~y%d^0S)IRtE&%rix3^hy)$
z1)DnaVS}H3%y+vz6EOU<nCRv{Y;1G0Lsa9UIcVP7F7bRf$iJP3oP*IXS7Ht+Y_APm
z-{p+t_eN;gaU{Q@pV0Ur>xX>~avNM@4ePS;cO&m#?rO1F2DLF@ldpkaeAg>g^qt3p
zH8iR&gfr<AzwZ`0hc?{75Sz@dG5j;jLhlS?{qSAhbfJqeAb$<WZwgr>HAHY^`-3^{
zi$-G*jDyD6>DBsbnE%0SuBkiok}-%?X&s!vKJ@1Mu5>{$;N}L9qjx#0#dq{Np5<_D
zioNfxNaxT-1tzjd19_YRTf(S)4s-36){6ICA^v^n8tFZ-cM4K*cVSrBBkUf@@1&g4
zF*0inJ%H=>uej8=KBZ1I{pIHVL2ty5!O)uGAg=26N)sDB(pxe|+g4jb<7|7MJG-2M
z8~U6h8hadx4T&G%AI~?VVI6vBOMHS0+iwuJcHN6DQHUw!ipbG9;go8F1K_ncvg-`@
zAvbU_n>+BaHRK0fFv%sM(Gh$Ip+{QFfM<qq4=!Ceygm^6OFV8Y0{Q>w7~Jq9I-;>3
z)=({)$P0CfxCQkq4`X?oWenl8fI>+!%zrDTv}cgEU-f%>g~n-Q9i<*s&&@caI{>+r
zyDKi5MLAp@!CLmr8Eh#&S@2iLJ>K(Vi7w+(_#-)7n?;1%OYD7bA72(&-v#r34`TUy
zSPH)l@?e`0y^qufz%YmRi${)ZM%32sGMIl8{ea5IIHD;a?U4+x6Ds6|GR_$3AioOc
zUzvBsoW;EI&k5zHp7p&DU&bv-aM3eGG)PDn-|$F=9Noi%_xb|Y>vQe=Z;;U8O=mDM
z&7HV7{yp%$wP3fu#H9+UPE0<03$Wc_vqCshJYf0X(e^9vGb=UUFu{f)-vF6w<|&gC
z)Lpg1{2TKxo6B?RhOB~fBI_OUPRBw1ey~HW9w}mnhvGl@VS~f~_JK`{ouZm(kcYg8
z#4(_=1|V*b5Hedf;xk$nBmNT-GsNtc=hOE0s-f0Q1-X#TDaQtfzd=%!<|CwTEa#%>
zZr<nSOYG0@f^Yru2jVw?9X$v(0OW7cfISWKOlzOx!4|*&lRbN=Pi?`Mu#aw<53%GX
z&{-`Y|M$ytT6>|cWuFiDd`SDKo$^Jr&w8(c`~w*^h6d0*pMaD$8K?C#L5H4OalzaN
z^7B?)y!S9q{W7CK<j2wYp8>Xj$p51^TYSqeM{?XhU$QP>b!$w}N|6(xP&OA|0QuD6
znm3D$15q?{IA6>Xe+5`XpI=^^KA=Eay*#%KOIX?QVf<&k+03Jwo{Sp3DxGT}BKd%n
zW2))twfZv9p~k$+W(AvTMvv_8myv&Qg!EmIe;blJ^<=dSKpfML*wet(?HaI2UEv#~
zjZjBY6<s7}L%!Y}iO+HQs<&LBwgAr*dN0qGob$_XGXxg48-mwKt3%dyVu`zZos%oo
ze@Q>3Gi07K?Mwqdf?O$o4|LGRY_35E`CV1+MVKxt*fGRjAu%mdf8voQIs>(nU-{&;
z3IkTPcSDX<4>{HWhy#XVws)IJ#fmn-_z#fF<o=}Xs^2)!^i%LzVvKLD<TPL@NZh6i
z-zYmm+@@H@II78lTwH5*o#`6;T;C@V$Ib7ioi^ly4pM{s>by&4AIO=H_LpAxGaz=%
zKH=v3j>)nah&~}?@)tmk+ApY}JwIf9M|BjZQxH?st${q<y|_I+{SY5YAXjvldQjs_
zELP7&e7AstHg3pzsTpd!)KNv<8!7vho@v$EGLXMH>w<ZCLWS~u)@l6;$Zf*ho7J!n
ze61z_(*w8I5MVF*u5JGS$+3rR>T-iTb1wJ>+_*ADJJh81F%GJXNtLRG<P++()Cx5Y
z?5}_5dg&iQ{y}1qtUY>HH;Zvhy&QDT$(+lkSR{UdI)PPi53-xoNWPxTvCjNIE&r2G
zIFleoc^higeoZXxB>_Hi4(b*Z%%ke>Q%`90tZLmAsOMcCyiu9}b;d%-DgO|&Q~qsq
zNv9vXMje`c*+fpRh5DA$I!@L(gAV4tH;a4M&0042NomKsNBptqa^-x;^|6wUsJ=}-
zt}(Gr=}l1kbewuXy&U9U8&%Y$jM*jE#*`?^s1=IEuzoP9r}ZDFoz@(K`e8HZASwIY
z5TaNAcMi~_h;KkU)Qe9(q1g}ecR)V78+^1}$m6+1ZtXY+waVt$QbkKlsXCanPd9@K
zK2X{z?U!H!`XRr51KLigzT=e)6vmNoTxZLw(ye5lF$iF8zfL=&3l87b86C5;TSP(X
z0?HLt(Yw_PlMbsFLXDb;eMUEsS!XzxUf1`~Pkbe^Juguh15(qF3iaI}|5b=l4pJ+%
z&LDpU_;Pn4uig{4M@~zu=m}1()^=so=nUC)`t7XRjxjN$=Vfc%b`~2F_&Mx40}blG
zs-OnS52@DyJhUCb6Tu&<V;olP04zeAep=VfI-{qM>Qp~rsMeL_WvC%O0#Z+rf5n&#
z7@Rw)M1Fo){sX-xkYA8er5l7=qlT1Yh9`Z*mtkwgz7`wMqtmj%wjz3mTr5w4{|u->
zSN$F2|Bsa8n(tW<3sb-swIWmNl7e~;q;?L$f5_W=94TdTKG^HOGY%>L3AIlqsI$z3
z+LuT8wRO2$lbQ{&R6mnarl<jXJ;<m~|0mR@G$Xj2wJ<De3Og39hyv-ml#;HMF#js3
zJ?n=xRQxjk{TPeKt9*vwaj*ycse6?zP^Z`mIRU?sdf8XW*NU`l!i#!81|9TAsPEcK
zEbpCTMcTgzDH2CD9cmZX$Ch-v|0~w#AQ&jIRJ9c9@&6Tc&e-2{IBp*M)xV$qdj_mI
G1OEf;M`fJ=

literal 0
HcmV?d00001

diff --git a/distributed/dashboard/templates/base.html b/distributed/dashboard/templates/base.html
index 83f5e8527c6..a9f00ca17c8 100644
--- a/distributed/dashboard/templates/base.html
+++ b/distributed/dashboard/templates/base.html
@@ -5,6 +5,7 @@
     <title>Dask Diagnostic UI</title>
     <meta name='viewport' content='width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no' />
     <link rel="stylesheet" href="statics/css/base.css"/>
+    <link rel="shortcut icon" href="statics/images/favicon.ico"/>
     {% block resources %}
     {% block css_resources %}
     {{ bokeh_css | indent(8) if bokeh_css }}
diff --git a/distributed/dashboard/templates/main.html b/distributed/dashboard/templates/main.html
index 8d0d8264d52..88f7453a42a 100644
--- a/distributed/dashboard/templates/main.html
+++ b/distributed/dashboard/templates/main.html
@@ -3,6 +3,7 @@
     <meta charset="utf-8"></meta>
     <title> {{title}} </title>
     <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/bulma/0.6.0/css/bulma.min.css" media="all" />
+    <link rel="shortcut icon" href="{{ rel_path_statics }}statics/images/favicon.ico"/>
           <style>
          .commit-tease,
          .user-profile-mini-avatar,
          .avatar,
          .vcard-details,
          .signup-prompt-bg {
            display: none !IMPORTANT;
          }
        </style>
         <script>
          document.addEventListener('DOMContentLoaded', function() {
            this.querySelectorAll('a').forEach(anchor => {
              anchor.addEventListener('click', e => {
                e.preventDefault();

                const redact = new URLSearchParams(window.location.search).get('redact');
                const hasExistingParams = anchor.href.includes('?');
                window.location.href = anchor.href + (hasExistingParams ? `&redact=${redact}` : `?redact=${redact}`);
              });
            });
          });
        </script>
 </head>
 
   <body>

From 6fe62774aa7ad585cf2231ca6475f70fdc1cec24 Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Tue, 1 Oct 2019 00:16:57 +0100
Subject: [PATCH 0486/1550] Retry scheduler connect multiple times (#3104)

---
 distributed/comm/core.py | 26 +++++++++++++++-----------
 1 file changed, 15 insertions(+), 11 deletions(-)

diff --git a/distributed/comm/core.py b/distributed/comm/core.py
index 1bbc043f52d..256a17de3a5 100644
--- a/distributed/comm/core.py
+++ b/distributed/comm/core.py
@@ -7,7 +7,7 @@
 from tornado import gen
 
 from ..metrics import time
-from ..utils import parse_timedelta
+from ..utils import parse_timedelta, ignoring
 from . import registry
 from .addressing import parse_address
 
@@ -188,6 +188,7 @@ async def connect(addr, timeout=None, deserialize=True, connection_args=None):
     scheme, loc = parse_address(addr)
     backend = registry.get_backend(scheme)
     connector = backend.get_connector()
+    comm = None
 
     start = time()
     deadline = start + timeout
@@ -205,14 +206,19 @@ def _raise(error):
     # This starts a thread
     while True:
         try:
-            future = connector.connect(
-                loc, deserialize=deserialize, **(connection_args or {})
-            )
-            comm = await gen.with_timeout(
-                timedelta(seconds=deadline - time()),
-                future,
-                quiet_exceptions=EnvironmentError,
-            )
+            while deadline - time() > 0:
+                future = connector.connect(
+                    loc, deserialize=deserialize, **(connection_args or {})
+                )
+                with ignoring(gen.TimeoutError):
+                    comm = await gen.with_timeout(
+                        timedelta(seconds=min(deadline - time(), 1)),
+                        future,
+                        quiet_exceptions=EnvironmentError,
+                    )
+                    break
+            if not comm:
+                _raise(error)
         except FatalCommClosedError:
             raise
         except EnvironmentError as e:
@@ -222,8 +228,6 @@ def _raise(error):
                 logger.debug("sleeping on connect")
             else:
                 _raise(error)
-        except gen.TimeoutError:
-            _raise(error)
         else:
             break
 

From ad37905521c8c133707b44a4c7676ee4735e20f0 Mon Sep 17 00:00:00 2001
From: Gil Forsyth <gforsyth@users.noreply.github.com>
Date: Tue, 1 Oct 2019 14:35:16 -0400
Subject: [PATCH 0487/1550] Send noise over the wire to keep connection alive
 (#3105)

This change keeps the `dask-ssh` connection alive, preventing ssh
timeouts from closing a pipe and killing workers/schedulers without user
action.

Note that for the `asyncssh` version, this won't be required as
`asyncssh` has a specific `keepalive` parameter that can be passed
through, but before that is the default, this should ease some pain.
---
 distributed/deploy/ssh.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/distributed/deploy/ssh.py b/distributed/deploy/ssh.py
index 9390d00a2ab..30f6f819224 100644
--- a/distributed/deploy/ssh.py
+++ b/distributed/deploy/ssh.py
@@ -182,11 +182,16 @@ def communicate():
             )
             return True
 
+    # Get transport to current SSH client
+    transport = ssh.get_transport()
+
     # Wait for a message on the input_queue. Any message received signals this
     # thread to shut itself down.
     while cmd_dict["input_queue"].empty():
         # Kill some time so that this thread does not hog the CPU.
         time.sleep(1.0)
+        # Send noise down the pipe to keep connection active
+        transport.send_ignore()
         if communicate():
             break
 

From 52d3e057721802ee725d71448b6f7ef17a8e515a Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 1 Oct 2019 16:32:45 -0500
Subject: [PATCH 0488/1550] Collect worker-worker and type bandwidth
 information (#3094)

This collects the bandwidth that we observe both by type,
and by worker-worker pair.

We use this for visual diagnostics, and maybe scheduling
decisions in the future.
---
 distributed/dashboard/nvml.py                 |  11 +-
 distributed/dashboard/scheduler.py            | 183 +++++++++++++++++-
 .../dashboard/tests/test_scheduler_bokeh.py   |  13 +-
 distributed/scheduler.py                      |  30 ++-
 distributed/tests/test_scheduler.py           |  34 +++-
 distributed/worker.py                         |  27 ++-
 6 files changed, 277 insertions(+), 21 deletions(-)

diff --git a/distributed/dashboard/nvml.py b/distributed/dashboard/nvml.py
index 7fd628dd469..131a02a8397 100644
--- a/distributed/dashboard/nvml.py
+++ b/distributed/dashboard/nvml.py
@@ -183,5 +183,12 @@ def gpu_utilization_doc(scheduler, extra, doc):
     doc.theme = BOKEH_THEME
 
 
-applications["/individual-gpu-memory"] = gpu_memory_doc
-applications["/individual-gpu-utilization"] = gpu_utilization_doc
+try:
+    import pynvml
+
+    pynvml.nvmlInit()
+except Exception:
+    pass
+else:
+    applications["/individual-gpu-memory"] = gpu_memory_doc
+    applications["/individual-gpu-utilization"] = gpu_utilization_doc
diff --git a/distributed/dashboard/scheduler.py b/distributed/dashboard/scheduler.py
index 7f172c879d1..27a49b4fd68 100644
--- a/distributed/dashboard/scheduler.py
+++ b/distributed/dashboard/scheduler.py
@@ -8,6 +8,7 @@
 from bokeh.layouts import column, row
 from bokeh.models import (
     ColumnDataSource,
+    ColorBar,
     DataRange1d,
     HoverTool,
     ResetTool,
@@ -32,7 +33,7 @@
 from bokeh.plotting import figure
 from bokeh.palettes import Viridis11
 from bokeh.themes import Theme
-from bokeh.transform import factor_cmap
+from bokeh.transform import factor_cmap, linear_cmap
 from bokeh.io import curdoc
 import dask
 from dask.utils import format_bytes
@@ -295,6 +296,166 @@ def update(self):
         self.root.title.text = "Bytes stored (Histogram): " + format_bytes(nbytes.sum())
 
 
+class BandwidthTypes(DashboardComponent):
+    """ Bar chart showing bandwidth per type """
+
+    def __init__(self, scheduler, **kwargs):
+        with log_errors():
+            self.last = 0
+            self.scheduler = scheduler
+            self.source = ColumnDataSource(
+                {
+                    "bandwidth": [1, 2],
+                    "bandwidth-half": [0.5, 1],
+                    "type": ["a", "b"],
+                    "bandwidth_text": ["1", "2"],
+                }
+            )
+
+            fig = figure(
+                title="Bandwidth by Type",
+                tools="",
+                id="bk-bandwidth-type-plot",
+                name="bandwidth_type_histogram",
+                y_range=["a", "b"],
+                **kwargs
+            )
+            rect = fig.rect(
+                source=self.source,
+                x="bandwidth-half",
+                y="type",
+                width="bandwidth",
+                height=1,
+                color="blue",
+            )
+            fig.x_range.start = 0
+            fig.xaxis[0].formatter = NumeralTickFormatter(format="0.0 b")
+            rect.nonselection_glyph = None
+
+            fig.xaxis.minor_tick_line_alpha = 0
+            fig.ygrid.visible = False
+
+            fig.toolbar.logo = None
+            fig.toolbar_location = None
+
+            hover = HoverTool()
+            hover.tooltips = "@type: @bandwidth_text / s"
+            hover.point_policy = "follow_mouse"
+            fig.add_tools(hover)
+
+            self.fig = fig
+
+    @without_property_validation
+    def update(self):
+        with log_errors():
+            bw = self.scheduler.bandwidth_types
+            self.fig.y_range.factors = list(sorted(bw))
+            result = {
+                "bandwidth": list(bw.values()),
+                "bandwidth-half": [b / 2 for b in bw.values()],
+                "type": list(bw.keys()),
+                "bandwidth_text": list(map(format_bytes, bw.values())),
+            }
+            self.fig.title.text = "Bandwidth: " + format_bytes(self.scheduler.bandwidth)
+
+            update(self.source, result)
+
+
+class BandwidthWorkers(DashboardComponent):
+    """ How many tasks are on each worker """
+
+    def __init__(self, scheduler, **kwargs):
+        with log_errors():
+            self.last = 0
+            self.scheduler = scheduler
+            self.source = ColumnDataSource(
+                {
+                    "bandwidth": [1, 2],
+                    "source": ["a", "b"],
+                    "destination": ["a", "b"],
+                    "bandwidth_text": ["1", "2"],
+                }
+            )
+
+            values = [hex(x)[2:] for x in range(64, 256)][::-1]
+            mapper = linear_cmap(
+                field_name="bandwidth",
+                palette=["#" + x + x + "FF" for x in values],
+                low=0,
+                high=1,
+            )
+
+            fig = figure(
+                title="Bandwidth by Worker",
+                tools="",
+                id="bk-bandwidth-worker-plot",
+                name="bandwidth_worker_heatmap",
+                x_range=["a", "b"],
+                y_range=["a", "b"],
+                **kwargs
+            )
+            fig.xaxis.major_label_orientation = -math.pi / 12
+            rect = fig.rect(
+                source=self.source,
+                x="source",
+                y="destination",
+                color=mapper,
+                height=1,
+                width=1,
+            )
+
+            self.color_map = mapper["transform"]
+            color_bar = ColorBar(
+                color_mapper=self.color_map,
+                label_standoff=12,
+                border_line_color=None,
+                location=(0, 0),
+            )
+            color_bar.formatter = NumeralTickFormatter(format="0 b")
+            fig.add_layout(color_bar, "right")
+
+            fig.toolbar.logo = None
+            fig.toolbar_location = None
+
+            hover = HoverTool()
+            hover.tooltips = """
+            <div>
+                <p><b>Source:</b> @source </p>
+                <p><b>Destination:</b> @destination </p>
+                <p><b>Bandwidth:</b> @bandwidth_text / s</p>
+            </div>
+            """
+            hover.point_policy = "follow_mouse"
+            fig.add_tools(hover)
+
+            self.fig = fig
+
+    @without_property_validation
+    def update(self):
+        with log_errors():
+            bw = self.scheduler.bandwidth_workers
+            if not bw:
+                return
+            x, y, value = zip(*[(a, b, c) for (a, b), c in bw.items()])
+
+            if self.color_map.high < max(value):
+                self.color_map.high = max(value)
+
+            factors = list(sorted(set(x + y)))
+            self.fig.x_range.factors = factors
+            self.fig.y_range.factors = factors
+
+            result = {
+                "source": x,
+                "destination": y,
+                "bandwidth": value,
+                "bandwidth_text": list(map(format_bytes, value)),
+            }
+            self.fig.title.text = "Bandwidth: " + format_bytes(self.scheduler.bandwidth)
+
+            update(self.source, result)
+
+
 class CurrentLoad(DashboardComponent):
     """ How many tasks are on each worker """
 
@@ -1596,6 +1757,24 @@ def individual_workers_doc(scheduler, extra, doc):
         doc.theme = BOKEH_THEME
 
 
+def individual_bandwidth_types(scheduler, extra, doc):
+    with log_errors():
+        bw = BandwidthTypes(scheduler, sizing_mode="stretch_both")
+        bw.update()
+        add_periodic_callback(doc, bw, 500)
+        doc.add_root(bw.fig)
+        doc.theme = BOKEH_THEME
+
+
+def individual_bandwidth_workers(scheduler, extra, doc):
+    with log_errors():
+        bw = BandwidthWorkers(scheduler, sizing_mode="stretch_both")
+        bw.update()
+        add_periodic_callback(doc, bw, 500)
+        doc.add_root(bw.fig)
+        doc.theme = BOKEH_THEME
+
+
 def profile_doc(scheduler, extra, doc):
     with log_errors():
         doc.title = "Dask: Profile"
@@ -1703,6 +1882,8 @@ def listen(self, *args, **kwargs):
     "/individual-cpu": individual_cpu_doc,
     "/individual-nprocessing": individual_nprocessing_doc,
     "/individual-workers": individual_workers_doc,
+    "/individual-bandwidth-types": individual_bandwidth_types,
+    "/individual-bandwidth-workers": individual_bandwidth_workers,
 }
 
 try:
diff --git a/distributed/dashboard/tests/test_scheduler_bokeh.py b/distributed/dashboard/tests/test_scheduler_bokeh.py
index 2dc29572ea8..e68d7935583 100644
--- a/distributed/dashboard/tests/test_scheduler_bokeh.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh.py
@@ -664,6 +664,11 @@ def test_https_support(c, s, a, b):
     ctx.load_verify_locations(get_cert("tls-ca-cert.pem"))
 
     http_client = AsyncHTTPClient()
+    response = yield http_client.fetch(
+        "https://localhost:%d/individual-plots.json" % port, ssl_options=ctx
+    )
+    response = json.loads(response.body.decode())
+
     for suffix in [
         "system",
         "counters",
@@ -672,13 +677,7 @@ def test_https_support(c, s, a, b):
         "tasks",
         "stealing",
         "graph",
-        "individual-task-stream",
-        "individual-progress",
-        "individual-graph",
-        "individual-nbytes",
-        "individual-nprocessing",
-        "individual-profile",
-    ]:
+    ] + [url.strip("/") for url in response.values()]:
         req = HTTPRequest(
             url="https://localhost:%d/%s" % (port, suffix), ssl_options=ctx
         )
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index bdee7c7bdac..f3d3fc92ea8 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -210,6 +210,7 @@ class WorkerState(object):
     __slots__ = (
         "actors",
         "address",
+        "bandwidth",
         "extra",
         "has_what",
         "last_seen",
@@ -257,6 +258,7 @@ def __init__(
         self.metrics = {}
         self.last_seen = 0
         self.time_delay = 0
+        self.bandwidth = parse_bytes(dask.config.get("distributed.scheduler.bandwidth"))
 
         self.actors = set()
         self.has_what = set()
@@ -881,6 +883,8 @@ def __init__(
             self.idle_timeout = None
         self.time_started = time()
         self.bandwidth = parse_bytes(dask.config.get("distributed.scheduler.bandwidth"))
+        self.bandwidth_workers = defaultdict(float)
+        self.bandwidth_types = defaultdict(float)
 
         if not preload:
             preload = dask.config.get("distributed.scheduler.preload")
@@ -1346,9 +1350,27 @@ def heartbeat_worker(
         host_info = host_info or {}
 
         self.host_info[host]["last-seen"] = local_now
-        frac = 1 / 20 / len(self.workers)
+        frac = 1 / len(self.workers)
         try:
-            self.bandwidth = self.bandwidth * (1 - frac) + metrics["bandwidth"] * frac
+            self.bandwidth = (
+                self.bandwidth * (1 - frac) + metrics["bandwidth"]["total"] * frac
+            )
+            for other, (bw, count) in metrics["bandwidth"]["workers"].items():
+                if (address, other) not in self.bandwidth_workers:
+                    self.bandwidth_workers[address, other] = bw / count
+                else:
+                    alpha = (1 - frac) ** count
+                    self.bandwidth_workers[address, other] = self.bandwidth_workers[
+                        address, other
+                    ] * alpha + bw * (1 - alpha)
+            for typ, (bw, count) in metrics["bandwidth"]["types"].items():
+                if typ not in self.bandwidth_types:
+                    self.bandwidth_types[typ] = bw / count
+                else:
+                    alpha = (1 - frac) ** count
+                    self.bandwidth_types[typ] = self.bandwidth_types[
+                        typ
+                    ] * alpha + bw * (1 - alpha)
         except KeyError:
             pass
 
@@ -1948,6 +1970,10 @@ def remove_worker(self, comm=None, address=None, safe=False, close=True):
             if not self.workers:
                 logger.info("Lost all workers")
 
+            for w in self.workers:
+                self.bandwidth_workers.pop((address, w), None)
+                self.bandwidth_workers.pop((w, address), None)
+
             def remove_worker_from_events():
                 # If the worker isn't registered anymore after the delay, remove from events
                 if address not in self.workers and address in self.events:
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 71a19eba54f..4b56a4d084f 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -22,7 +22,7 @@
 from distributed.metrics import time
 from distributed.protocol.pickle import dumps
 from distributed.worker import dumps_function, dumps_task
-from distributed.utils import tmpfile
+from distributed.utils import tmpfile, typename
 from distributed.utils_test import (  # noqa: F401
     captured_logger,
     cleanup,
@@ -1515,14 +1515,38 @@ def test_idle_timeout(c, s, a, b):
 
 
 @gen_cluster(client=True, config={"distributed.scheduler.bandwidth": "100 GB"})
-def test_bandwidth(c, s, a, b):
+async def test_bandwidth(c, s, a, b):
     start = s.bandwidth
-    x = c.submit(operator.mul, b"0", 20000, workers=a.address)
+    x = c.submit(operator.mul, b"0", 1000000, workers=a.address)
     y = c.submit(lambda x: x, x, workers=b.address)
-    yield y
-    yield b.heartbeat()
+    await y
+    await b.heartbeat()
     assert s.bandwidth < start  # we've learned that we're slower
     assert b.latency
+    assert typename(bytes) in s.bandwidth_types
+    assert (b.address, a.address) in s.bandwidth_workers
+
+    await a.close()
+    assert not s.bandwidth_workers
+
+
+@gen_cluster(client=True, Worker=Nanny)
+async def test_bandwidth_clear(c, s, a, b):
+    np = pytest.importorskip("numpy")
+    x = c.submit(np.arange, 1000000, workers=[a.worker_address], pure=False)
+    y = c.submit(np.arange, 1000000, workers=[b.worker_address], pure=False)
+    z = c.submit(operator.add, x, y)  # force communication
+    await z
+
+    async def f(dask_worker):
+        await dask_worker.heartbeat()
+
+    await c.run(f)
+
+    assert s.bandwidth_workers
+
+    await s.restart()
+    assert not s.bandwidth_workers
 
 
 @gen_cluster()
diff --git a/distributed/worker.py b/distributed/worker.py
index 2bd345ea0c5..fba4eed57b3 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -20,9 +20,9 @@
 from dask.utils import format_bytes, funcname
 
 try:
-    from cytoolz import pluck, partial, merge, first
+    from cytoolz import pluck, partial, merge, first, keymap
 except ImportError:
-    from toolz import pluck, partial, merge, first
+    from toolz import pluck, partial, merge, first, keymap
 from tornado import gen
 from tornado.ioloop import IOLoop
 
@@ -416,6 +416,10 @@ def __init__(
         self.outgoing_current_count = 0
         self.repetitively_busy = 0
         self.bandwidth = parse_bytes(dask.config.get("distributed.scheduler.bandwidth"))
+        self.bandwidth_workers = defaultdict(
+            lambda: (0, 0)
+        )  # bw/count recent transfers
+        self.bandwidth_types = defaultdict(lambda: (0, 0))  # bw/count recent transfers
         self.latency = 0.001
         self._client = None
 
@@ -728,7 +732,11 @@ async def get_metrics(self):
             in_memory=len(self.data),
             ready=len(self.ready),
             in_flight=len(self.in_flight_tasks),
-            bandwidth=self.bandwidth,
+            bandwidth={
+                "total": self.bandwidth,
+                "workers": dict(self.bandwidth_workers),
+                "types": keymap(typename, self.bandwidth_types),
+            },
         )
         custom = {}
         for k, metric in self.metrics.items():
@@ -881,6 +889,8 @@ async def heartbeat(self):
                 self.periodic_callbacks["heartbeat"].callback_time = (
                     response["heartbeat-interval"] * 1000
                 )
+                self.bandwidth_workers.clear()
+                self.bandwidth_types.clear()
             except CommClosedError:
                 logger.warning("Heartbeat to scheduler failed")
             finally:
@@ -1920,8 +1930,17 @@ async def gather_dep(self, worker, dep, deps, total_nbytes, cause=None):
                         "who": worker,
                     }
                 )
-                if total_bytes > 10000:
+                if total_bytes > 1000000:
                     self.bandwidth = self.bandwidth * 0.95 + bandwidth * 0.05
+                    bw, cnt = self.bandwidth_workers[worker]
+                    self.bandwidth_workers[worker] = (bw + bandwidth, cnt + 1)
+
+                    types = set(map(type, response["data"].values()))
+                    if len(types) == 1:
+                        [typ] = types
+                        bw, cnt = self.bandwidth_types[typ]
+                        self.bandwidth_types[typ] = (bw + bandwidth, cnt + 1)
+
                 if self.digests is not None:
                     self.digests["transfer-bandwidth"].add(total_bytes / duration)
                     self.digests["transfer-duration"].add(duration)

From 00ac6f0dbae52aa085834b2e4f3a513e02463008 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 1 Oct 2019 17:08:47 -0500
Subject: [PATCH 0489/1550] Add Client.shutdown method (#3106)

This lets the client shut down the scheduler and workers

Asked for in https://stackoverflow.com/questions/50919227/is-it-possible-to-shutdown-a-dask-distributed-cluster-given-a-client-instance
---
 distributed/client.py            | 23 +++++++++++++++++------
 distributed/tests/test_client.py | 20 ++++++++++++++++++++
 2 files changed, 37 insertions(+), 6 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index d00f545b161..f918fffbf78 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -1335,14 +1335,25 @@ async def _():
         if self._should_close_loop and not shutting_down():
             self._loop_runner.stop()
 
-    def shutdown(self, *args, **kwargs):
-        """ Deprecated, see close instead
+    async def _shutdown(self):
+        logger.info("Shutting down scheduler from Client")
+        if self.cluster:
+            await self.cluster.close()
+        else:
+            with ignoring(CommClosedError):
+                await self.scheduler.terminate(close_workers=True)
+
+    def shutdown(self):
+        """ Shut down the connected scheduler and workers
+
+        Note, this may disrupt other clients that may be using the same
+        scheudler and workers.
 
-        This was deprecated because "shutdown" was sometimes confusingly
-        thought to refer to the cluster rather than the client
+        See also
+        --------
+        Client.close: close only this client
         """
-        warnings.warn("Shutdown is deprecated.  Please use close instead")
-        return self.close(*args, **kwargs)
+        return self.sync(self._shutdown)
 
     def get_executor(self, **kwargs):
         """
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index a95548689d1..d01088502f0 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -5636,5 +5636,25 @@ def dashboard_link(self):
             assert "http://foo.com" in client._repr_html_()
 
 
+@pytest.mark.asyncio
+async def test_shutdown(cleanup):
+    async with Scheduler(port=0) as s:
+        async with Worker(s.address) as w:
+            async with Client(s.address, asynchronous=True) as c:
+                await c.shutdown()
+
+            assert s.status == "closed"
+            assert w.status == "closed"
+
+
+@pytest.mark.asyncio
+async def test_shutdown_localcluster(cleanup):
+    async with LocalCluster(n_workers=1, asynchronous=True, processes=False) as lc:
+        async with Client(lc, asynchronous=True) as c:
+            await c.shutdown()
+
+        assert lc.scheduler.status == "closed"
+
+
 if sys.version_info >= (3, 5):
     from distributed.tests.py3_test_client import *  # noqa F401

From 78d8cad9e8934a3b863999832b7c46dfe8162683 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Thu, 3 Oct 2019 09:36:37 -0500
Subject: [PATCH 0490/1550] Identify lost workers in SpecCluster based on
 address not name (#3088)

Fixes #3062
---
 distributed/deploy/spec.py                    |  9 ++++++++-
 distributed/deploy/tests/test_spec_cluster.py | 20 +++++++++++++++++++
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index 5cc0722ef72..2dc910de2cc 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -309,7 +309,14 @@ def _update_worker_status(self, op, msg):
             name = self.scheduler_info["workers"][msg]["name"]
 
             def f():
-                if name in self.workers and msg not in self.scheduler_info:
+                if (
+                    name in self.workers
+                    and msg not in self.scheduler_info["workers"]
+                    and not any(
+                        d["name"] == name
+                        for d in self.scheduler_info["workers"].values()
+                    )
+                ):
                     self._futures.add(asyncio.ensure_future(self.workers[name].close()))
                     del self.workers[name]
 
diff --git a/distributed/deploy/tests/test_spec_cluster.py b/distributed/deploy/tests/test_spec_cluster.py
index 485ae1989ea..e0ea735f8ce 100644
--- a/distributed/deploy/tests/test_spec_cluster.py
+++ b/distributed/deploy/tests/test_spec_cluster.py
@@ -157,6 +157,26 @@ async def test_unexpected_closed_worker(cleanup):
             assert len(cluster.workers) == 2
 
 
+@pytest.mark.slow
+@pytest.mark.asyncio
+async def test_restart(cleanup):
+    # Regression test for https://github.com/dask/distributed/issues/3062
+    worker = {"cls": Nanny, "options": {"nthreads": 1}}
+    with dask.config.set({"distributed.deploy.lost-worker-timeout": "2s"}):
+        async with SpecCluster(
+            asynchronous=True, scheduler=scheduler, worker=worker
+        ) as cluster:
+            async with Client(cluster, asynchronous=True) as client:
+                cluster.scale(2)
+                await cluster
+                assert len(cluster.workers) == 2
+
+                await client.restart()
+                await asyncio.sleep(3)
+
+                assert len(cluster.workers) == 2
+
+
 @pytest.mark.asyncio
 async def test_broken_worker():
     with pytest.raises(Exception) as info:

From 4f11e7c6ecc3422b26d31124b8300ac4fbb5d1d2 Mon Sep 17 00:00:00 2001
From: Jim Crist <jcrist@users.noreply.github.com>
Date: Fri, 4 Oct 2019 05:16:48 -0500
Subject: [PATCH 0491/1550] Support calling `cluster.scale` as async method
 (#3110)

This adds optional support for calling `cluster.scale` as an async
method (i.e. `await cluster.scale(...)`). This is currently optional and
backwards compatible - perhaps in the future we may want to deprecate
calling in the non-async context.
---
 distributed/deploy/spec.py                    | 25 ++++++++++++++-----
 distributed/deploy/tests/test_spec_cluster.py |  5 ++++
 2 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index 2dc910de2cc..11ff3b44322 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -77,6 +77,19 @@ async def __aexit__(self, *args, **kwargs):
         await self.close()
 
 
+class NoOpAwaitable(object):
+    """An awaitable object that always returns None.
+
+    Useful to return from a method that can be called in both asynchronous and
+    synchronous contexts"""
+
+    def __await__(self):
+        async def f():
+            return None
+
+        return f().__await__()
+
+
 class SpecCluster(Cluster):
     """ Cluster that requires a full specification of workers
 
@@ -418,15 +431,15 @@ def scale(self, n=0, memory=None, cores=None):
         while len(self.worker_spec) > n:
             self.worker_spec.popitem()
 
-        if self.status in ("closing", "closed"):
-            self.loop.add_callback(self._correct_state)
-            return
-
-        while len(self.worker_spec) < n:
-            self.worker_spec.update(self.new_worker_spec())
+        if self.status not in ("closing", "closed"):
+            while len(self.worker_spec) < n:
+                self.worker_spec.update(self.new_worker_spec())
 
         self.loop.add_callback(self._correct_state)
 
+        if self.asynchronous:
+            return NoOpAwaitable()
+
     def new_worker_spec(self):
         """ Return name and spec for the next worker
 
diff --git a/distributed/deploy/tests/test_spec_cluster.py b/distributed/deploy/tests/test_spec_cluster.py
index e0ea735f8ce..1c8a01e98ce 100644
--- a/distributed/deploy/tests/test_spec_cluster.py
+++ b/distributed/deploy/tests/test_spec_cluster.py
@@ -125,6 +125,11 @@ async def test_scale(cleanup):
         await cluster
         assert len(cluster.workers) == 1
 
+        # Can use with await
+        await cluster.scale(2)
+        await cluster
+        assert len(cluster.workers) == 2
+
 
 @pytest.mark.asyncio
 async def test_unexpected_closed_worker(cleanup):

From f1c7bfdb7f6041d566b56ca1ffb439e8e09b39fb Mon Sep 17 00:00:00 2001
From: "Mads R. B. Kristensen" <madsbk@gmail.com>
Date: Fri, 4 Oct 2019 12:53:28 +0200
Subject: [PATCH 0492/1550] UCX: trying to allocate CUDA arrays using RMM and
 Numba (#3109)

* Removed duplicate code
* Added tcp to the default UCX_TLS
* Trying to allocate CUDA arrays using RMM and Numba
---
 distributed/comm/ucx.py | 36 ++++++++++++++++++++++--------------
 1 file changed, 22 insertions(+), 14 deletions(-)

diff --git a/distributed/comm/ucx.py b/distributed/comm/ucx.py
index bbf2451e323..77f65c661e8 100644
--- a/distributed/comm/ucx.py
+++ b/distributed/comm/ucx.py
@@ -16,13 +16,12 @@
 from tornado.ioloop import IOLoop
 import ucp
 import numpy as np
-import numba.cuda
 
 import os
 
 os.environ.setdefault("UCX_RNDV_SCHEME", "put_zcopy")
 os.environ.setdefault("UCX_MEMTYPE_CACHE", "n")
-os.environ.setdefault("UCX_TLS", "rc,cuda_copy,cuda_ipc")
+os.environ.setdefault("UCX_TLS", "tcp,rc,cuda_copy,cuda_ipc")
 
 logger = logging.getLogger(__name__)
 MAX_MSG_LOG = 23
@@ -32,6 +31,23 @@
 # Comm Interface
 # ----------------------------------------------------------------------------
 
+# Let's find the function, `cuda_array`, to use when allocating new CUDA arrays
+try:
+    import rmm
+
+    cuda_array = lambda n: rmm.device_array(n, dtype=np.uint8)
+except ImportError:
+    try:
+        import numba.cuda
+
+        cuda_array = lambda n: numba.cuda.device_array((n,), dtype=np.uint8)
+    except ImportError:
+
+        def cuda_array(n):
+            raise RuntimeError(
+                "In order to send/recv CUDA arrays, Numba or RMM is required"
+            )
+
 
 class UCX(Comm):
     """Comm object using UCP.
@@ -116,12 +132,7 @@ async def write(
             # Send frames
             for frame in frames:
                 if nbytes(frame) > 0:
-                    if hasattr(frame, "__array_interface__") or hasattr(
-                        frame, "__cuda_array_interface__"
-                    ):
-                        await self.ep.send(frame)
-                    else:
-                        await self.ep.send(frame)
+                    await self.ep.send(frame)
             return sum(map(nbytes, frames))
 
     async def read(self, deserializers=("cuda", "dask", "pickle", "error")):
@@ -152,17 +163,14 @@ async def read(self, deserializers=("cuda", "dask", "pickle", "error")):
                 for is_cuda, size in zip(is_cudas.tolist(), sizes.tolist()):
                     if size > 0:
                         if is_cuda:
-                            frame = numba.cuda.device_array((size,), dtype=np.uint8)
+                            frame = cuda_array(size)
                         else:
                             frame = np.empty(size, dtype=np.uint8)
                         await self.ep.recv(frame)
-                        if is_cuda:
-                            frames.append(frame)
-                        else:
-                            frames.append(frame.data)
+                        frames.append(frame)
                     else:
                         if is_cuda:
-                            frames.append(numba.cuda.device_array((0,), dtype=np.uint8))
+                            frames.append(cuda_array(size))
                         else:
                             frames.append(b"")
                 msg = await from_frames(

From 670f2e84a7607b685cbc2f8fbdbfb29aa8267c84 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 4 Oct 2019 09:33:03 -0500
Subject: [PATCH 0493/1550] Replace use of tornado.gen with asyncio in
 dask-worker (#3114)

---
 distributed/cli/dask_worker.py | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
index 35ca11da34c..31c17fed0d0 100755
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -1,3 +1,4 @@
+import asyncio
 import atexit
 import logging
 import gc
@@ -21,7 +22,6 @@
 
 from toolz import valmap
 from tornado.ioloop import IOLoop, TimeoutError
-from tornado import gen
 
 logger = logging.getLogger("distributed.dask_worker")
 
@@ -379,20 +379,18 @@ def del_pid_file():
         for i in range(nprocs)
     ]
 
-    @gen.coroutine
-    def close_all():
+    async def close_all():
         # Unregister all workers from scheduler
         if nanny:
-            yield [n.close(timeout=2) for n in nannies]
+            await asyncio.gather(*[n.close(timeout=2) for n in nannies])
 
     def on_signal(signum):
         logger.info("Exiting on signal %d", signum)
-        close_all()
+        asyncio.ensure_future(close_all())
 
-    @gen.coroutine
-    def run():
-        yield nannies
-        yield [n.finished() for n in nannies]
+    async def run():
+        await asyncio.gather(*nannies)
+        await asyncio.gather(*[n.finished() for n in nannies])
 
     install_signal_handlers(loop, cleanup=on_signal)
 

From 1630f4de2579ed0d99e56df5ae778cb35cceab77 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 4 Oct 2019 14:23:24 -0500
Subject: [PATCH 0494/1550] Make dask-worker close quietly when given sigint
 signal (#3116)

---
 distributed/cli/dask_worker.py            | 26 +++++++++++++++--------
 distributed/cli/tests/test_dask_worker.py |  1 -
 distributed/nanny.py                      |  2 +-
 3 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
index 31c17fed0d0..fb32fc2e882 100755
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -3,7 +3,8 @@
 import logging
 import gc
 import os
-from sys import exit
+import signal
+import sys
 import warnings
 
 import click
@@ -267,34 +268,34 @@ def main(
         logger.error(
             "Failed to launch worker.  You cannot use the --port argument when nprocs > 1."
         )
-        exit(1)
+        sys.exit(1)
 
     if nprocs > 1 and not nanny:
         logger.error(
             "Failed to launch worker.  You cannot use the --no-nanny argument when nprocs > 1."
         )
-        exit(1)
+        sys.exit(1)
 
     if contact_address and not listen_address:
         logger.error(
             "Failed to launch worker. "
             "Must specify --listen-address when --contact-address is given"
         )
-        exit(1)
+        sys.exit(1)
 
     if nprocs > 1 and listen_address:
         logger.error(
             "Failed to launch worker. "
             "You cannot specify --listen-address when nprocs > 1."
         )
-        exit(1)
+        sys.exit(1)
 
     if (worker_port or host) and listen_address:
         logger.error(
             "Failed to launch worker. "
             "You cannot specify --listen-address when --worker-port or --host is given."
         )
-        exit(1)
+        sys.exit(1)
 
     try:
         if listen_address:
@@ -308,7 +309,7 @@ def main(
             contact_address = listen_address
     except ValueError as e:
         logger.error("Failed to launch worker. " + str(e))
-        exit(1)
+        sys.exit(1)
 
     if nanny:
         port = nanny_port
@@ -384,8 +385,13 @@ async def close_all():
         if nanny:
             await asyncio.gather(*[n.close(timeout=2) for n in nannies])
 
+    signal_fired = False
+
     def on_signal(signum):
-        logger.info("Exiting on signal %d", signum)
+        nonlocal signal_fired
+        signal_fired = True
+        if signum != signal.SIGINT:
+            logger.info("Exiting on signal %d", signum)
         asyncio.ensure_future(close_all())
 
     async def run():
@@ -398,7 +404,9 @@ async def run():
         loop.run_sync(run)
     except TimeoutError:
         # We already log the exception in nanny / worker. Don't do it again.
-        raise TimeoutError("Timed out starting worker.") from None
+        if not signal_fired:
+            logger.info("Timed out starting worker")
+        sys.exit(1)
     except KeyboardInterrupt:
         pass
     finally:
diff --git a/distributed/cli/tests/test_dask_worker.py b/distributed/cli/tests/test_dask_worker.py
index 01327d64291..c509772d113 100644
--- a/distributed/cli/tests/test_dask_worker.py
+++ b/distributed/cli/tests/test_dask_worker.py
@@ -311,7 +311,6 @@ def test_worker_timeout(no_nanny):
         args.append("--no-nanny")
     result = runner.invoke(distributed.cli.dask_worker.main, args)
     assert result.exit_code != 0
-    assert str(result.exception).startswith("Timed out")
 
 
 def test_bokeh_deprecation():
diff --git a/distributed/nanny.py b/distributed/nanny.py
index 06b5a27dc79..8fbbf761368 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -314,7 +314,7 @@ async def instantiate(self, comm=None):
                 )
             except gen.TimeoutError:
                 await self.close(timeout=self.death_timeout)
-                logger.exception(
+                logger.error(
                     "Timed out connecting Nanny '%s' to scheduler '%s'",
                     self,
                     self.scheduler_addr,

From 8d300008e1c93525309198cc2e09405f6e2a8c04 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 4 Oct 2019 15:25:30 -0500
Subject: [PATCH 0495/1550] bump version to 2.5.2

---
 docs/source/changelog.rst | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index aff09d2c3c7..5b6288885fc 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,23 @@
 Changelog
 =========
 
+2.5.2 - 2019-10-04
+------------------
+
+-  Make dask-worker close quietly when given sigint signal (:pr:`3116`) `Matthew Rocklin`_
+-  Replace use of tornado.gen with asyncio in dask-worker (:pr:`3114`) `Matthew Rocklin`_
+-  UCX: allocate CUDA arrays using RMM and Numba (:pr:`3109`) `Mads R. B. Kristensen`_
+-  Support calling `cluster.scale` as async method (:pr:`3110`) `Jim Crist`_
+-  Identify lost workers in SpecCluster based on address not name (:pr:`3088`) `James Bourbeau`_
+-  Add Client.shutdown method (:pr:`3106`) `Matthew Rocklin`_
+-  Collect worker-worker and type bandwidth information (:pr:`3094`) `Matthew Rocklin`_
+-  Send noise over the wire to keep dask-ssh connection alive (:pr:`3105`) `Gil Forsyth`_
+-  Retry scheduler connect multiple times (:pr:`3104`) `Jacob Tomlinson`_
+-  Add favicon of logo to the dashboard (:pr:`3095`) `James Bourbeau`_
+-  Remove utils.py functions for their dask/utils.py equivalents (:pr:`3042`) `Matthew Rocklin`_
+-  Lower default bokeh log level (:pr:`3087`) `Philipp Rudiger`_
+-  Check if self.cluster.scheduler is a local scheduler (:pr:`3099`) `Jacob Tomlinson`_
+
 
 2.5.1 - 2019-09-27
 ------------------
@@ -1296,3 +1313,5 @@ significantly without many new features.
 .. _`Mohammad Noor`: https://github.com/MdSalih
 .. _`Richard J Zamora`: https://github.com/rjzamora
 .. _`Arpit Solanki`: https://github.com/arpit1997
+.. _`Gil Forsyth`: https://github.com/gforsyth
+.. _`Philipp Rudiger`: https://github.com/philippjfr

From 8dd912c1e71105f714292e2ecabb14804837fbe9 Mon Sep 17 00:00:00 2001
From: Jim Crist <jcrist@users.noreply.github.com>
Date: Tue, 8 Oct 2019 11:53:48 -0500
Subject: [PATCH 0496/1550] Bump dask dependency (#3124)

Due to moving some things around, distributed 2.5 now relies on dask
2.5. We bump the required version to account for this.
---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index e599ab0de93..d1335d0b3b1 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,6 @@
 click >= 6.6
 cloudpickle >= 0.2.2
-dask >= 2.3
+dask >= 2.5.2
 msgpack
 psutil >= 5.0
 sortedcontainers !=2.0.0, !=2.0.1

From dd0d60b9505952951736109f0b6e2aadea36744e Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 9 Oct 2019 08:24:16 -0500
Subject: [PATCH 0497/1550] Move new SSHCluster to top level (#3128)

We have two SSHCluster implementations, one that has been around for a
long while, and is still used in the dask-ssh CLI, and a newer
implementation that is based off of SpecCluster.

The old solution is battle hardened, and probably nicer for the CLI, but
not very modifyable.  The new solution is much simpler, and defers most
of the configuration to other systems, notably asyncssh and the
worker/scheduler classes.

This PR keeps both, but moves the new system to the main import, while
gracefully deferring to the old solution if the user seems to be using
those options (it is very obvious which the user intended to use).

We also move SSHCluster to a top-level import so users can do the following:

    from dask.distributed import SSHCluster, Client
---
 continuous_integration/travis/install.sh |   2 +-
 distributed/__init__.py                  |   2 +-
 distributed/cli/dask_ssh.py              |   2 +-
 distributed/deploy/__init__.py           |   1 +
 distributed/deploy/old_ssh.py            | 472 +++++++++++++++
 distributed/deploy/ssh.py                | 724 +++++++++--------------
 distributed/deploy/ssh2.py               | 236 --------
 distributed/deploy/tests/test_old_ssh.py |  31 +
 distributed/deploy/tests/test_ssh.py     |  72 ++-
 distributed/deploy/tests/test_ssh2.py    |  43 --
 10 files changed, 825 insertions(+), 760 deletions(-)
 create mode 100644 distributed/deploy/old_ssh.py
 delete mode 100644 distributed/deploy/ssh2.py
 create mode 100644 distributed/deploy/tests/test_old_ssh.py
 delete mode 100644 distributed/deploy/tests/test_ssh2.py

diff --git a/continuous_integration/travis/install.sh b/continuous_integration/travis/install.sh
index f0c4a07be67..8c34f38d276 100644
--- a/continuous_integration/travis/install.sh
+++ b/continuous_integration/travis/install.sh
@@ -55,7 +55,7 @@ conda install -q \
 
 # For low-level profiler, install libunwind and stacktrace from conda-forge
 # For stacktrace we use --no-deps to avoid upgrade of python
-conda install -c defaults -c conda-forge libunwind zstandard
+conda install -c defaults -c conda-forge libunwind zstandard asyncssh
 conda install --no-deps -c defaults -c numba -c conda-forge stacktrace
 
 pip install -q "pytest>=4" pytest-repeat pytest-faulthandler pytest-asyncio
diff --git a/distributed/__init__.py b/distributed/__init__.py
index d79993dfef7..07015ff44af 100644
--- a/distributed/__init__.py
+++ b/distributed/__init__.py
@@ -2,7 +2,7 @@
 from dask.config import config
 from .actor import Actor, ActorFuture
 from .core import connect, rpc
-from .deploy import LocalCluster, Adaptive, SpecCluster
+from .deploy import LocalCluster, Adaptive, SpecCluster, SSHCluster
 from .diagnostics.progressbar import progress
 from .client import (
     Client,
diff --git a/distributed/cli/dask_ssh.py b/distributed/cli/dask_ssh.py
index 97cf91f3519..07cbb57bf01 100755
--- a/distributed/cli/dask_ssh.py
+++ b/distributed/cli/dask_ssh.py
@@ -1,4 +1,4 @@
-from distributed.deploy.ssh import SSHCluster
+from distributed.deploy.old_ssh import SSHCluster
 import click
 
 from distributed.cli.utils import check_python_3
diff --git a/distributed/deploy/__init__.py b/distributed/deploy/__init__.py
index 5a5a9106005..35fc86fe393 100644
--- a/distributed/deploy/__init__.py
+++ b/distributed/deploy/__init__.py
@@ -2,6 +2,7 @@
 
 from .cluster import Cluster
 from .local import LocalCluster
+from .ssh import SSHCluster
 from .spec import SpecCluster, ProcessInterface
 from .adaptive import Adaptive
 
diff --git a/distributed/deploy/old_ssh.py b/distributed/deploy/old_ssh.py
new file mode 100644
index 00000000000..30f6f819224
--- /dev/null
+++ b/distributed/deploy/old_ssh.py
@@ -0,0 +1,472 @@
+import logging
+import socket
+import os
+import sys
+import time
+import traceback
+
+try:
+    from queue import Queue
+except ImportError:  # Python 2.7 fix
+    from Queue import Queue
+
+from threading import Thread
+
+from toolz import merge
+
+from tornado import gen
+
+
+logger = logging.getLogger(__name__)
+
+
+# These are handy for creating colorful terminal output to enhance readability
+# of the output generated by dask-ssh.
+class bcolors:
+    HEADER = "\033[95m"
+    OKBLUE = "\033[94m"
+    OKGREEN = "\033[92m"
+    WARNING = "\033[93m"
+    FAIL = "\033[91m"
+    ENDC = "\033[0m"
+    BOLD = "\033[1m"
+    UNDERLINE = "\033[4m"
+
+
+def async_ssh(cmd_dict):
+    import paramiko
+    from paramiko.buffered_pipe import PipeTimeout
+    from paramiko.ssh_exception import SSHException, PasswordRequiredException
+
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+    retries = 0
+    while True:  # Be robust to transient SSH failures.
+        try:
+            # Set paramiko logging to WARN or higher to squelch INFO messages.
+            logging.getLogger("paramiko").setLevel(logging.WARN)
+
+            ssh.connect(
+                hostname=cmd_dict["address"],
+                username=cmd_dict["ssh_username"],
+                port=cmd_dict["ssh_port"],
+                key_filename=cmd_dict["ssh_private_key"],
+                compress=True,
+                timeout=20,
+                banner_timeout=20,
+            )  # Helps prevent timeouts when many concurrent ssh connections are opened.
+            # Connection successful, break out of while loop
+            break
+
+        except (SSHException, PasswordRequiredException) as e:
+
+            print(
+                "[ dask-ssh ] : "
+                + bcolors.FAIL
+                + "SSH connection error when connecting to {addr}:{port}"
+                "to run '{cmd}'".format(
+                    addr=cmd_dict["address"],
+                    port=cmd_dict["ssh_port"],
+                    cmd=cmd_dict["cmd"],
+                )
+                + bcolors.ENDC
+            )
+
+            print(
+                bcolors.FAIL
+                + "               SSH reported this exception: "
+                + str(e)
+                + bcolors.ENDC
+            )
+
+            # Print an exception traceback
+            traceback.print_exc()
+
+            # Transient SSH errors can occur when many SSH connections are
+            # simultaneously opened to the same server. This makes a few
+            # attempts to retry.
+            retries += 1
+            if retries >= 3:
+                print(
+                    "[ dask-ssh ] : "
+                    + bcolors.FAIL
+                    + "SSH connection failed after 3 retries. Exiting."
+                    + bcolors.ENDC
+                )
+
+                # Connection failed after multiple attempts.  Terminate this thread.
+                os._exit(1)
+
+            # Wait a moment before retrying
+            print(
+                "               "
+                + bcolors.FAIL
+                + "Retrying... (attempt {n}/{total})".format(n=retries, total=3)
+                + bcolors.ENDC
+            )
+
+            time.sleep(1)
+
+    # Execute the command, and grab file handles for stdout and stderr. Note
+    # that we run the command using the user's default shell, but force it to
+    # run in an interactive login shell, which hopefully ensures that all of the
+    # user's normal environment variables (via the dot files) have been loaded
+    # before the command is run. This should help to ensure that important
+    # aspects of the environment like PATH and PYTHONPATH are configured.
+
+    print("[ {label} ] : {cmd}".format(label=cmd_dict["label"], cmd=cmd_dict["cmd"]))
+    stdin, stdout, stderr = ssh.exec_command(
+        "$SHELL -i -c '" + cmd_dict["cmd"] + "'", get_pty=True
+    )
+
+    # Set up channel timeout (which we rely on below to make readline() non-blocking)
+    channel = stdout.channel
+    channel.settimeout(0.1)
+
+    def read_from_stdout():
+        """
+        Read stdout stream, time out if necessary.
+        """
+        try:
+            line = stdout.readline()
+            while len(line) > 0:  # Loops until a timeout exception occurs
+                line = line.rstrip()
+                logger.debug("stdout from ssh channel: %s", line)
+                cmd_dict["output_queue"].put(
+                    "[ {label} ] : {output}".format(
+                        label=cmd_dict["label"], output=line
+                    )
+                )
+                line = stdout.readline()
+        except (PipeTimeout, socket.timeout):
+            pass
+
+    def read_from_stderr():
+        """
+        Read stderr stream, time out if necessary.
+        """
+        try:
+            line = stderr.readline()
+            while len(line) > 0:
+                line = line.rstrip()
+                logger.debug("stderr from ssh channel: %s", line)
+                cmd_dict["output_queue"].put(
+                    "[ {label} ] : ".format(label=cmd_dict["label"])
+                    + bcolors.FAIL
+                    + "{output}".format(output=line)
+                    + bcolors.ENDC
+                )
+                line = stderr.readline()
+        except (PipeTimeout, socket.timeout):
+            pass
+
+    def communicate():
+        """
+        Communicate a little bit, without blocking too long.
+        Return True if the command ended.
+        """
+        read_from_stdout()
+        read_from_stderr()
+
+        # Check to see if the process has exited. If it has, we let this thread
+        # terminate.
+        if channel.exit_status_ready():
+            exit_status = channel.recv_exit_status()
+            cmd_dict["output_queue"].put(
+                "[ {label} ] : ".format(label=cmd_dict["label"])
+                + bcolors.FAIL
+                + "remote process exited with exit status "
+                + str(exit_status)
+                + bcolors.ENDC
+            )
+            return True
+
+    # Get transport to current SSH client
+    transport = ssh.get_transport()
+
+    # Wait for a message on the input_queue. Any message received signals this
+    # thread to shut itself down.
+    while cmd_dict["input_queue"].empty():
+        # Kill some time so that this thread does not hog the CPU.
+        time.sleep(1.0)
+        # Send noise down the pipe to keep connection active
+        transport.send_ignore()
+        if communicate():
+            break
+
+    # Ctrl-C the executing command and wait a bit for command to end cleanly
+    start = time.time()
+    while time.time() < start + 5.0:
+        channel.send(b"\x03")  # Ctrl-C
+        if communicate():
+            break
+        time.sleep(1.0)
+
+    # Shutdown the channel, and close the SSH connection
+    channel.close()
+    ssh.close()
+
+
+def start_scheduler(
+    logdir, addr, port, ssh_username, ssh_port, ssh_private_key, remote_python=None
+):
+    cmd = "{python} -m distributed.cli.dask_scheduler --port {port}".format(
+        python=remote_python or sys.executable, port=port, logdir=logdir
+    )
+
+    # Optionally re-direct stdout and stderr to a logfile
+    if logdir is not None:
+        cmd = "mkdir -p {logdir} && ".format(logdir=logdir) + cmd
+        cmd += "&> {logdir}/dask_scheduler_{addr}:{port}.log".format(
+            addr=addr, port=port, logdir=logdir
+        )
+
+    # Format output labels we can prepend to each line of output, and create
+    # a 'status' key to keep track of jobs that terminate prematurely.
+    label = (
+        bcolors.BOLD
+        + "scheduler {addr}:{port}".format(addr=addr, port=port)
+        + bcolors.ENDC
+    )
+
+    # Create a command dictionary, which contains everything we need to run and
+    # interact with this command.
+    input_queue = Queue()
+    output_queue = Queue()
+    cmd_dict = {
+        "cmd": cmd,
+        "label": label,
+        "address": addr,
+        "port": port,
+        "input_queue": input_queue,
+        "output_queue": output_queue,
+        "ssh_username": ssh_username,
+        "ssh_port": ssh_port,
+        "ssh_private_key": ssh_private_key,
+    }
+
+    # Start the thread
+    thread = Thread(target=async_ssh, args=[cmd_dict])
+    thread.daemon = True
+    thread.start()
+
+    return merge(cmd_dict, {"thread": thread})
+
+
+def start_worker(
+    logdir,
+    scheduler_addr,
+    scheduler_port,
+    worker_addr,
+    nthreads,
+    nprocs,
+    ssh_username,
+    ssh_port,
+    ssh_private_key,
+    nohost,
+    memory_limit,
+    worker_port,
+    nanny_port,
+    remote_python=None,
+    remote_dask_worker="distributed.cli.dask_worker",
+):
+
+    cmd = (
+        "{python} -m {remote_dask_worker} "
+        "{scheduler_addr}:{scheduler_port} "
+        "--nthreads {nthreads}" + (" --nprocs {nprocs}" if nprocs != 1 else "")
+    )
+
+    if not nohost:
+        cmd += " --host {worker_addr}"
+
+    if memory_limit:
+        cmd += " --memory-limit {memory_limit}"
+
+    if worker_port:
+        cmd += " --worker-port {worker_port}"
+
+    if nanny_port:
+        cmd += " --nanny-port {nanny_port}"
+
+    cmd = cmd.format(
+        python=remote_python or sys.executable,
+        remote_dask_worker=remote_dask_worker,
+        scheduler_addr=scheduler_addr,
+        scheduler_port=scheduler_port,
+        worker_addr=worker_addr,
+        nthreads=nthreads,
+        nprocs=nprocs,
+        memory_limit=memory_limit,
+        worker_port=worker_port,
+        nanny_port=nanny_port,
+    )
+
+    # Optionally redirect stdout and stderr to a logfile
+    if logdir is not None:
+        cmd = "mkdir -p {logdir} && ".format(logdir=logdir) + cmd
+        cmd += "&> {logdir}/dask_scheduler_{addr}.log".format(
+            addr=worker_addr, logdir=logdir
+        )
+
+    label = "worker {addr}".format(addr=worker_addr)
+
+    # Create a command dictionary, which contains everything we need to run and
+    # interact with this command.
+    input_queue = Queue()
+    output_queue = Queue()
+    cmd_dict = {
+        "cmd": cmd,
+        "label": label,
+        "address": worker_addr,
+        "input_queue": input_queue,
+        "output_queue": output_queue,
+        "ssh_username": ssh_username,
+        "ssh_port": ssh_port,
+        "ssh_private_key": ssh_private_key,
+    }
+
+    # Start the thread
+    thread = Thread(target=async_ssh, args=[cmd_dict])
+    thread.daemon = True
+    thread.start()
+
+    return merge(cmd_dict, {"thread": thread})
+
+
+class SSHCluster(object):
+    def __init__(
+        self,
+        scheduler_addr,
+        scheduler_port,
+        worker_addrs,
+        nthreads=0,
+        nprocs=1,
+        ssh_username=None,
+        ssh_port=22,
+        ssh_private_key=None,
+        nohost=False,
+        logdir=None,
+        remote_python=None,
+        memory_limit=None,
+        worker_port=None,
+        nanny_port=None,
+        remote_dask_worker="distributed.cli.dask_worker",
+    ):
+
+        self.scheduler_addr = scheduler_addr
+        self.scheduler_port = scheduler_port
+        self.nthreads = nthreads
+        self.nprocs = nprocs
+
+        self.ssh_username = ssh_username
+        self.ssh_port = ssh_port
+        self.ssh_private_key = ssh_private_key
+
+        self.nohost = nohost
+
+        self.remote_python = remote_python
+
+        self.memory_limit = memory_limit
+        self.worker_port = worker_port
+        self.nanny_port = nanny_port
+        self.remote_dask_worker = remote_dask_worker
+
+        # Generate a universal timestamp to use for log files
+        import datetime
+
+        if logdir is not None:
+            logdir = os.path.join(
+                logdir,
+                "dask-ssh_" + datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S"),
+            )
+            print(
+                bcolors.WARNING + "Output will be redirected to logfiles "
+                'stored locally on individual worker nodes under "{logdir}".'.format(
+                    logdir=logdir
+                )
+                + bcolors.ENDC
+            )
+        self.logdir = logdir
+
+        # Keep track of all running threads
+        self.threads = []
+
+        # Start the scheduler node
+        self.scheduler = start_scheduler(
+            logdir,
+            scheduler_addr,
+            scheduler_port,
+            ssh_username,
+            ssh_port,
+            ssh_private_key,
+            remote_python,
+        )
+
+        # Start worker nodes
+        self.workers = []
+        for i, addr in enumerate(worker_addrs):
+            self.add_worker(addr)
+
+    @gen.coroutine
+    def _start(self):
+        pass
+
+    @property
+    def scheduler_address(self):
+        return "%s:%d" % (self.scheduler_addr, self.scheduler_port)
+
+    def monitor_remote_processes(self):
+
+        # Form a list containing all processes, since we treat them equally from here on out.
+        all_processes = [self.scheduler] + self.workers
+
+        try:
+            while True:
+                for process in all_processes:
+                    while not process["output_queue"].empty():
+                        print(process["output_queue"].get())
+
+                # Kill some time and free up CPU before starting the next sweep
+                # through the processes.
+                time.sleep(0.1)
+
+            # end while true
+
+        except KeyboardInterrupt:
+            pass  # Return execution to the calling process
+
+    def add_worker(self, address):
+        self.workers.append(
+            start_worker(
+                self.logdir,
+                self.scheduler_addr,
+                self.scheduler_port,
+                address,
+                self.nthreads,
+                self.nprocs,
+                self.ssh_username,
+                self.ssh_port,
+                self.ssh_private_key,
+                self.nohost,
+                self.memory_limit,
+                self.worker_port,
+                self.nanny_port,
+                self.remote_python,
+                self.remote_dask_worker,
+            )
+        )
+
+    def shutdown(self):
+        all_processes = [self.scheduler] + self.workers
+
+        for process in all_processes:
+            process["input_queue"].put("shutdown")
+            process["thread"].join()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        self.shutdown()
diff --git a/distributed/deploy/ssh.py b/distributed/deploy/ssh.py
index 30f6f819224..8aa3cc17d97 100644
--- a/distributed/deploy/ssh.py
+++ b/distributed/deploy/ssh.py
@@ -1,472 +1,286 @@
 import logging
-import socket
-import os
 import sys
-import time
-import traceback
-
-try:
-    from queue import Queue
-except ImportError:  # Python 2.7 fix
-    from Queue import Queue
-
-from threading import Thread
-
-from toolz import merge
-
-from tornado import gen
+from typing import List
+import warnings
+import weakref
 
+from .spec import SpecCluster, ProcessInterface
+from ..utils import cli_keywords
+from ..scheduler import Scheduler as _Scheduler
+from ..worker import Worker as _Worker
 
 logger = logging.getLogger(__name__)
 
 
-# These are handy for creating colorful terminal output to enhance readability
-# of the output generated by dask-ssh.
-class bcolors:
-    HEADER = "\033[95m"
-    OKBLUE = "\033[94m"
-    OKGREEN = "\033[92m"
-    WARNING = "\033[93m"
-    FAIL = "\033[91m"
-    ENDC = "\033[0m"
-    BOLD = "\033[1m"
-    UNDERLINE = "\033[4m"
-
-
-def async_ssh(cmd_dict):
-    import paramiko
-    from paramiko.buffered_pipe import PipeTimeout
-    from paramiko.ssh_exception import SSHException, PasswordRequiredException
-
-    ssh = paramiko.SSHClient()
-    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-
-    retries = 0
-    while True:  # Be robust to transient SSH failures.
-        try:
-            # Set paramiko logging to WARN or higher to squelch INFO messages.
-            logging.getLogger("paramiko").setLevel(logging.WARN)
-
-            ssh.connect(
-                hostname=cmd_dict["address"],
-                username=cmd_dict["ssh_username"],
-                port=cmd_dict["ssh_port"],
-                key_filename=cmd_dict["ssh_private_key"],
-                compress=True,
-                timeout=20,
-                banner_timeout=20,
-            )  # Helps prevent timeouts when many concurrent ssh connections are opened.
-            # Connection successful, break out of while loop
-            break
-
-        except (SSHException, PasswordRequiredException) as e:
-
-            print(
-                "[ dask-ssh ] : "
-                + bcolors.FAIL
-                + "SSH connection error when connecting to {addr}:{port}"
-                "to run '{cmd}'".format(
-                    addr=cmd_dict["address"],
-                    port=cmd_dict["ssh_port"],
-                    cmd=cmd_dict["cmd"],
-                )
-                + bcolors.ENDC
-            )
-
-            print(
-                bcolors.FAIL
-                + "               SSH reported this exception: "
-                + str(e)
-                + bcolors.ENDC
-            )
-
-            # Print an exception traceback
-            traceback.print_exc()
-
-            # Transient SSH errors can occur when many SSH connections are
-            # simultaneously opened to the same server. This makes a few
-            # attempts to retry.
-            retries += 1
-            if retries >= 3:
-                print(
-                    "[ dask-ssh ] : "
-                    + bcolors.FAIL
-                    + "SSH connection failed after 3 retries. Exiting."
-                    + bcolors.ENDC
-                )
-
-                # Connection failed after multiple attempts.  Terminate this thread.
-                os._exit(1)
-
-            # Wait a moment before retrying
-            print(
-                "               "
-                + bcolors.FAIL
-                + "Retrying... (attempt {n}/{total})".format(n=retries, total=3)
-                + bcolors.ENDC
-            )
-
-            time.sleep(1)
-
-    # Execute the command, and grab file handles for stdout and stderr. Note
-    # that we run the command using the user's default shell, but force it to
-    # run in an interactive login shell, which hopefully ensures that all of the
-    # user's normal environment variables (via the dot files) have been loaded
-    # before the command is run. This should help to ensure that important
-    # aspects of the environment like PATH and PYTHONPATH are configured.
-
-    print("[ {label} ] : {cmd}".format(label=cmd_dict["label"], cmd=cmd_dict["cmd"]))
-    stdin, stdout, stderr = ssh.exec_command(
-        "$SHELL -i -c '" + cmd_dict["cmd"] + "'", get_pty=True
-    )
-
-    # Set up channel timeout (which we rely on below to make readline() non-blocking)
-    channel = stdout.channel
-    channel.settimeout(0.1)
-
-    def read_from_stdout():
-        """
-        Read stdout stream, time out if necessary.
-        """
-        try:
-            line = stdout.readline()
-            while len(line) > 0:  # Loops until a timeout exception occurs
-                line = line.rstrip()
-                logger.debug("stdout from ssh channel: %s", line)
-                cmd_dict["output_queue"].put(
-                    "[ {label} ] : {output}".format(
-                        label=cmd_dict["label"], output=line
-                    )
-                )
-                line = stdout.readline()
-        except (PipeTimeout, socket.timeout):
-            pass
-
-    def read_from_stderr():
-        """
-        Read stderr stream, time out if necessary.
-        """
-        try:
-            line = stderr.readline()
-            while len(line) > 0:
-                line = line.rstrip()
-                logger.debug("stderr from ssh channel: %s", line)
-                cmd_dict["output_queue"].put(
-                    "[ {label} ] : ".format(label=cmd_dict["label"])
-                    + bcolors.FAIL
-                    + "{output}".format(output=line)
-                    + bcolors.ENDC
-                )
-                line = stderr.readline()
-        except (PipeTimeout, socket.timeout):
-            pass
-
-    def communicate():
-        """
-        Communicate a little bit, without blocking too long.
-        Return True if the command ended.
-        """
-        read_from_stdout()
-        read_from_stderr()
-
-        # Check to see if the process has exited. If it has, we let this thread
-        # terminate.
-        if channel.exit_status_ready():
-            exit_status = channel.recv_exit_status()
-            cmd_dict["output_queue"].put(
-                "[ {label} ] : ".format(label=cmd_dict["label"])
-                + bcolors.FAIL
-                + "remote process exited with exit status "
-                + str(exit_status)
-                + bcolors.ENDC
-            )
-            return True
-
-    # Get transport to current SSH client
-    transport = ssh.get_transport()
-
-    # Wait for a message on the input_queue. Any message received signals this
-    # thread to shut itself down.
-    while cmd_dict["input_queue"].empty():
-        # Kill some time so that this thread does not hog the CPU.
-        time.sleep(1.0)
-        # Send noise down the pipe to keep connection active
-        transport.send_ignore()
-        if communicate():
-            break
-
-    # Ctrl-C the executing command and wait a bit for command to end cleanly
-    start = time.time()
-    while time.time() < start + 5.0:
-        channel.send(b"\x03")  # Ctrl-C
-        if communicate():
-            break
-        time.sleep(1.0)
-
-    # Shutdown the channel, and close the SSH connection
-    channel.close()
-    ssh.close()
-
-
-def start_scheduler(
-    logdir, addr, port, ssh_username, ssh_port, ssh_private_key, remote_python=None
-):
-    cmd = "{python} -m distributed.cli.dask_scheduler --port {port}".format(
-        python=remote_python or sys.executable, port=port, logdir=logdir
-    )
-
-    # Optionally re-direct stdout and stderr to a logfile
-    if logdir is not None:
-        cmd = "mkdir -p {logdir} && ".format(logdir=logdir) + cmd
-        cmd += "&> {logdir}/dask_scheduler_{addr}:{port}.log".format(
-            addr=addr, port=port, logdir=logdir
-        )
-
-    # Format output labels we can prepend to each line of output, and create
-    # a 'status' key to keep track of jobs that terminate prematurely.
-    label = (
-        bcolors.BOLD
-        + "scheduler {addr}:{port}".format(addr=addr, port=port)
-        + bcolors.ENDC
-    )
-
-    # Create a command dictionary, which contains everything we need to run and
-    # interact with this command.
-    input_queue = Queue()
-    output_queue = Queue()
-    cmd_dict = {
-        "cmd": cmd,
-        "label": label,
-        "address": addr,
-        "port": port,
-        "input_queue": input_queue,
-        "output_queue": output_queue,
-        "ssh_username": ssh_username,
-        "ssh_port": ssh_port,
-        "ssh_private_key": ssh_private_key,
-    }
-
-    # Start the thread
-    thread = Thread(target=async_ssh, args=[cmd_dict])
-    thread.daemon = True
-    thread.start()
-
-    return merge(cmd_dict, {"thread": thread})
-
-
-def start_worker(
-    logdir,
-    scheduler_addr,
-    scheduler_port,
-    worker_addr,
-    nthreads,
-    nprocs,
-    ssh_username,
-    ssh_port,
-    ssh_private_key,
-    nohost,
-    memory_limit,
-    worker_port,
-    nanny_port,
-    remote_python=None,
-    remote_dask_worker="distributed.cli.dask_worker",
-):
+class Process(ProcessInterface):
+    """ A superclass for SSH Workers and Nannies
+
+    See Also
+    --------
+    Worker
+    Scheduler
+    """
+
+    def __init__(self, **kwargs):
+        self.connection = None
+        self.proc = None
+        super().__init__(**kwargs)
+
+    async def start(self):
+        assert self.connection
+        weakref.finalize(
+            self, self.proc.kill
+        )  # https://github.com/ronf/asyncssh/issues/112
+        await super().start()
+
+    async def close(self):
+        self.proc.kill()  # https://github.com/ronf/asyncssh/issues/112
+        self.connection.close()
+        await super().close()
+
+    def __repr__(self):
+        return "<SSH %s: status=%s>" % (type(self).__name__, self.status)
+
+
+class Worker(Process):
+    """ A Remote Dask Worker controled by SSH
+
+    Parameters
+    ----------
+    scheduler: str
+        The address of the scheduler
+    address: str
+        The hostname where we should run this worker
+    worker_module: str
+        The python module to run to start the worker.
+    connect_options: dict
+        kwargs to be passed to asyncssh connections
+    kwargs: dict
+        These will be passed through the dask-worker CLI to the
+        dask.distributed.Worker class
+    """
 
-    cmd = (
-        "{python} -m {remote_dask_worker} "
-        "{scheduler_addr}:{scheduler_port} "
-        "--nthreads {nthreads}" + (" --nprocs {nprocs}" if nprocs != 1 else "")
-    )
-
-    if not nohost:
-        cmd += " --host {worker_addr}"
-
-    if memory_limit:
-        cmd += " --memory-limit {memory_limit}"
-
-    if worker_port:
-        cmd += " --worker-port {worker_port}"
-
-    if nanny_port:
-        cmd += " --nanny-port {nanny_port}"
-
-    cmd = cmd.format(
-        python=remote_python or sys.executable,
-        remote_dask_worker=remote_dask_worker,
-        scheduler_addr=scheduler_addr,
-        scheduler_port=scheduler_port,
-        worker_addr=worker_addr,
-        nthreads=nthreads,
-        nprocs=nprocs,
-        memory_limit=memory_limit,
-        worker_port=worker_port,
-        nanny_port=nanny_port,
-    )
-
-    # Optionally redirect stdout and stderr to a logfile
-    if logdir is not None:
-        cmd = "mkdir -p {logdir} && ".format(logdir=logdir) + cmd
-        cmd += "&> {logdir}/dask_scheduler_{addr}.log".format(
-            addr=worker_addr, logdir=logdir
-        )
-
-    label = "worker {addr}".format(addr=worker_addr)
-
-    # Create a command dictionary, which contains everything we need to run and
-    # interact with this command.
-    input_queue = Queue()
-    output_queue = Queue()
-    cmd_dict = {
-        "cmd": cmd,
-        "label": label,
-        "address": worker_addr,
-        "input_queue": input_queue,
-        "output_queue": output_queue,
-        "ssh_username": ssh_username,
-        "ssh_port": ssh_port,
-        "ssh_private_key": ssh_private_key,
-    }
-
-    # Start the thread
-    thread = Thread(target=async_ssh, args=[cmd_dict])
-    thread.daemon = True
-    thread.start()
-
-    return merge(cmd_dict, {"thread": thread})
-
-
-class SSHCluster(object):
     def __init__(
         self,
-        scheduler_addr,
-        scheduler_port,
-        worker_addrs,
-        nthreads=0,
-        nprocs=1,
-        ssh_username=None,
-        ssh_port=22,
-        ssh_private_key=None,
-        nohost=False,
-        logdir=None,
-        remote_python=None,
-        memory_limit=None,
-        worker_port=None,
-        nanny_port=None,
-        remote_dask_worker="distributed.cli.dask_worker",
+        scheduler: str,
+        address: str,
+        connect_options: dict,
+        kwargs: dict,
+        worker_module="distributed.cli.dask_worker",
+        loop=None,
+        name=None,
     ):
-
-        self.scheduler_addr = scheduler_addr
-        self.scheduler_port = scheduler_port
-        self.nthreads = nthreads
-        self.nprocs = nprocs
-
-        self.ssh_username = ssh_username
-        self.ssh_port = ssh_port
-        self.ssh_private_key = ssh_private_key
-
-        self.nohost = nohost
-
-        self.remote_python = remote_python
-
-        self.memory_limit = memory_limit
-        self.worker_port = worker_port
-        self.nanny_port = nanny_port
-        self.remote_dask_worker = remote_dask_worker
-
-        # Generate a universal timestamp to use for log files
-        import datetime
-
-        if logdir is not None:
-            logdir = os.path.join(
-                logdir,
-                "dask-ssh_" + datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S"),
-            )
-            print(
-                bcolors.WARNING + "Output will be redirected to logfiles "
-                'stored locally on individual worker nodes under "{logdir}".'.format(
-                    logdir=logdir
-                )
-                + bcolors.ENDC
+        self.address = address
+        self.scheduler = scheduler
+        self.worker_module = worker_module
+        self.connect_options = connect_options
+        self.kwargs = kwargs
+        self.name = name
+
+        super().__init__()
+
+    async def start(self):
+        import asyncssh  # import now to avoid adding to module startup time
+
+        self.connection = await asyncssh.connect(self.address, **self.connect_options)
+        self.proc = await self.connection.create_process(
+            " ".join(
+                [
+                    sys.executable,
+                    "-m",
+                    self.worker_module,
+                    self.scheduler,
+                    "--name",
+                    str(self.name),
+                ]
+                + cli_keywords(self.kwargs, cls=_Worker)
             )
-        self.logdir = logdir
-
-        # Keep track of all running threads
-        self.threads = []
-
-        # Start the scheduler node
-        self.scheduler = start_scheduler(
-            logdir,
-            scheduler_addr,
-            scheduler_port,
-            ssh_username,
-            ssh_port,
-            ssh_private_key,
-            remote_python,
         )
 
-        # Start worker nodes
-        self.workers = []
-        for i, addr in enumerate(worker_addrs):
-            self.add_worker(addr)
-
-    @gen.coroutine
-    def _start(self):
-        pass
-
-    @property
-    def scheduler_address(self):
-        return "%s:%d" % (self.scheduler_addr, self.scheduler_port)
-
-    def monitor_remote_processes(self):
-
-        # Form a list containing all processes, since we treat them equally from here on out.
-        all_processes = [self.scheduler] + self.workers
-
-        try:
-            while True:
-                for process in all_processes:
-                    while not process["output_queue"].empty():
-                        print(process["output_queue"].get())
-
-                # Kill some time and free up CPU before starting the next sweep
-                # through the processes.
-                time.sleep(0.1)
-
-            # end while true
-
-        except KeyboardInterrupt:
-            pass  # Return execution to the calling process
-
-    def add_worker(self, address):
-        self.workers.append(
-            start_worker(
-                self.logdir,
-                self.scheduler_addr,
-                self.scheduler_port,
-                address,
-                self.nthreads,
-                self.nprocs,
-                self.ssh_username,
-                self.ssh_port,
-                self.ssh_private_key,
-                self.nohost,
-                self.memory_limit,
-                self.worker_port,
-                self.nanny_port,
-                self.remote_python,
-                self.remote_dask_worker,
+        # We watch stderr in order to get the address, then we return
+        while True:
+            line = await self.proc.stderr.readline()
+            if not line.strip():
+                raise Exception("Worker failed to start")
+            logger.info(line.strip())
+            if "worker at" in line:
+                self.address = line.split("worker at:")[1].strip()
+                self.status = "running"
+                break
+        logger.debug("%s", line)
+        await super().start()
+
+
+class Scheduler(Process):
+    """ A Remote Dask Scheduler controled by SSH
+
+    Parameters
+    ----------
+    address: str
+        The hostname where we should run this worker
+    connect_options: dict
+        kwargs to be passed to asyncssh connections
+    kwargs: dict
+        These will be passed through the dask-scheduler CLI to the
+        dask.distributed.Scheduler class
+    """
+
+    def __init__(self, address: str, connect_options: dict, kwargs: dict):
+        self.address = address
+        self.kwargs = kwargs
+        self.connect_options = connect_options
+
+        super().__init__()
+
+    async def start(self):
+        import asyncssh  # import now to avoid adding to module startup time
+
+        logger.debug("Created Scheduler Connection")
+
+        self.connection = await asyncssh.connect(self.address, **self.connect_options)
+
+        self.proc = await self.connection.create_process(
+            " ".join(
+                [sys.executable, "-m", "distributed.cli.dask_scheduler"]
+                + cli_keywords(self.kwargs, cls=_Scheduler)
             )
         )
 
-    def shutdown(self):
-        all_processes = [self.scheduler] + self.workers
-
-        for process in all_processes:
-            process["input_queue"].put("shutdown")
-            process["thread"].join()
-
-    def __enter__(self):
-        return self
-
-    def __exit__(self, *args):
-        self.shutdown()
+        # We watch stderr in order to get the address, then we return
+        while True:
+            line = await self.proc.stderr.readline()
+            if not line.strip():
+                raise Exception("Worker failed to start")
+            logger.info(line.strip())
+            if "Scheduler at" in line:
+                self.address = line.split("Scheduler at:")[1].strip()
+                break
+        logger.debug("%s", line)
+        await super().start()
+
+
+old_cluster_kwargs = {
+    "scheduler_addr",
+    "scheduler_port",
+    "worker_addrs",
+    "nthreads",
+    "nprocs",
+    "ssh_username",
+    "ssh_port",
+    "ssh_private_key",
+    "nohost",
+    "logdir",
+    "remote_python",
+    "memory_limit",
+    "worker_port",
+    "nanny_port",
+    "remote_dask_worker",
+}
+
+
+def SSHCluster(
+    hosts: List[str] = None,
+    connect_options: dict = {},
+    worker_options: dict = {},
+    scheduler_options: dict = {},
+    worker_module: str = "distributed.cli.dask_worker",
+    **kwargs
+):
+    """ Deploy a Dask cluster using SSH
+
+    The SSHCluster function deploys a Dask Scheduler and Workers for you on a
+    set of machine addresses that you provide.  The first address will be used
+    for the scheduler while the rest will be used for the workers (feel free to
+    repeat the first hostname if you want to have the scheudler and worker
+    co-habitate one machine.)
+
+    You may configure the scheduler and workers by passing
+    ``scheduler_options`` and ``worker_options`` dictionary keywords.  See the
+    ``dask.distributed.Scheduler`` and ``dask.distributed.Worker`` classes for
+    details on the available options, but the defaults should work in most
+    situations.
+
+    You may configure your use of SSH itself using the ``connect_options``
+    keyword, which passes values to the ``asyncssh.connect`` function.  For
+    more information on these see the documentation for the ``asyncssh``
+    library https://asyncssh.readthedocs.io .
+
+    Parameters
+    ----------
+    hosts: List[str]
+        List of hostnames or addresses on which to launch our cluster
+        The first will be used for the scheduler and the rest for workers
+    connect_options:
+        Keywords to pass through to asyncssh.connect
+        known_hosts: List[str] or None
+            The list of keys which will be used to validate the server host
+            key presented during the SSH handshake.  If this is not specified,
+            the keys will be looked up in the file .ssh/known_hosts.  If this
+            is explicitly set to None, server host key validation will be disabled.
+    worker_options:
+        Keywords to pass on to dask-worker
+    scheduler_options:
+        Keywords to pass on to dask-scheduler
+    worker_module:
+        Python module to call to start the worker
+
+    Examples
+    --------
+    >>> from dask.distributed import Client, SSHCluster
+    >>> cluster = SSHCluster(
+    ...     ["localhost", "localhost", "localhost", "localhost"],
+    ...     connect_options={"known_hosts": None},
+    ...     worker_options={"nthreads": 2},
+    ...     scheduler_options={"port": 0, "dashboard_address": ":8797"}
+    ... )
+    >>> client = Client(cluster)
+
+    An example using a different worker module, in particular the
+    ``dask-cuda-worker`` command from the ``dask-cuda`` project.
+
+    >>> from dask.distributed import Client, SSHCluster
+    >>> cluster = SSHCluster(
+    ...     ["localhost", "hostwithgpus", "anothergpuhost"],
+    ...     connect_options={"known_hosts": None},
+    ...     scheduler_options={"port": 0, "dashboard_address": ":8797"},
+    ...     worker_module='dask_cuda.dask_cuda_worker')
+    >>> client = Client(cluster)
+
+    See Also
+    --------
+    dask.distributed.Scheduler
+    dask.distributed.Worker
+    asyncssh.connect
+    """
+    if set(kwargs) & old_cluster_kwargs:
+        from .old_ssh import SSHCluster as OldSSHCluster
+
+        warnings.warn(
+            "Note that the SSHCluster API has been replaced.  "
+            "We're routing you to the older implementation.  "
+            "This will be removed in the future"
+        )
+        kwargs.setdefault("worker_addrs", hosts)
+        return OldSSHCluster(**kwargs)
+
+    scheduler = {
+        "cls": Scheduler,
+        "options": {
+            "address": hosts[0],
+            "connect_options": connect_options,
+            "kwargs": scheduler_options,
+        },
+    }
+    workers = {
+        i: {
+            "cls": Worker,
+            "options": {
+                "address": host,
+                "connect_options": connect_options,
+                "kwargs": worker_options,
+                "worker_module": worker_module,
+            },
+        }
+        for i, host in enumerate(hosts[1:])
+    }
+    return SpecCluster(workers, scheduler, name="SSHCluster", **kwargs)
diff --git a/distributed/deploy/ssh2.py b/distributed/deploy/ssh2.py
deleted file mode 100644
index cb6b967d544..00000000000
--- a/distributed/deploy/ssh2.py
+++ /dev/null
@@ -1,236 +0,0 @@
-import logging
-import sys
-import warnings
-import weakref
-
-import asyncssh
-
-from .spec import SpecCluster, ProcessInterface
-from ..utils import cli_keywords
-from ..scheduler import Scheduler as _Scheduler
-from ..worker import Worker as _Worker
-
-logger = logging.getLogger(__name__)
-
-warnings.warn(
-    "the distributed.deploy.ssh2 module is experimental "
-    "and will move/change in the future without notice"
-)
-
-
-class Process(ProcessInterface):
-    """ A superclass for SSH Workers and Nannies
-
-    See Also
-    --------
-    Worker
-    Scheduler
-    """
-
-    def __init__(self, **kwargs):
-        self.connection = None
-        self.proc = None
-        super().__init__(**kwargs)
-
-    async def start(self):
-        assert self.connection
-        weakref.finalize(
-            self, self.proc.kill
-        )  # https://github.com/ronf/asyncssh/issues/112
-        await super().start()
-
-    async def close(self):
-        self.proc.kill()  # https://github.com/ronf/asyncssh/issues/112
-        self.connection.close()
-        await super().close()
-
-    def __repr__(self):
-        return "<SSH %s: status=%s>" % (type(self).__name__, self.status)
-
-
-class Worker(Process):
-    """ A Remote Dask Worker controled by SSH
-
-    Parameters
-    ----------
-    scheduler: str
-        The address of the scheduler
-    address: str
-        The hostname where we should run this worker
-    worker_module: str
-        The python module to run to start the worker.
-    connect_kwargs: dict
-        kwargs to be passed to asyncssh connections
-    kwargs: dict
-        These will be passed through the dask-worker CLI to the
-        dask.distributed.Worker class
-    """
-
-    def __init__(
-        self,
-        scheduler: str,
-        address: str,
-        connect_kwargs: dict,
-        kwargs: dict,
-        worker_module="distributed.cli.dask_worker",
-        loop=None,
-        name=None,
-    ):
-        self.address = address
-        self.scheduler = scheduler
-        self.worker_module = worker_module
-        self.connect_kwargs = connect_kwargs
-        self.kwargs = kwargs
-        self.name = name
-
-        super().__init__()
-
-    async def start(self):
-        self.connection = await asyncssh.connect(self.address, **self.connect_kwargs)
-        self.proc = await self.connection.create_process(
-            " ".join(
-                [
-                    sys.executable,
-                    "-m",
-                    self.worker_module,
-                    self.scheduler,
-                    "--name",
-                    str(self.name),
-                ]
-                + cli_keywords(self.kwargs, cls=_Worker)
-            )
-        )
-
-        # We watch stderr in order to get the address, then we return
-        while True:
-            line = await self.proc.stderr.readline()
-            if not line.strip():
-                raise Exception("Worker failed to start")
-            logger.info(line.strip())
-            if "worker at" in line:
-                self.address = line.split("worker at:")[1].strip()
-                self.status = "running"
-                break
-        logger.debug("%s", line)
-        await super().start()
-
-
-class Scheduler(Process):
-    """ A Remote Dask Scheduler controled by SSH
-
-    Parameters
-    ----------
-    address: str
-        The hostname where we should run this worker
-    connect_kwargs: dict
-        kwargs to be passed to asyncssh connections
-    kwargs: dict
-        These will be passed through the dask-scheduler CLI to the
-        dask.distributed.Scheduler class
-    """
-
-    def __init__(self, address: str, connect_kwargs: dict, kwargs: dict):
-        self.address = address
-        self.kwargs = kwargs
-        self.connect_kwargs = connect_kwargs
-
-        super().__init__()
-
-    async def start(self):
-        logger.debug("Created Scheduler Connection")
-
-        self.connection = await asyncssh.connect(self.address, **self.connect_kwargs)
-
-        self.proc = await self.connection.create_process(
-            " ".join(
-                [sys.executable, "-m", "distributed.cli.dask_scheduler"]
-                + cli_keywords(self.kwargs, cls=_Scheduler)
-            )
-        )
-
-        # We watch stderr in order to get the address, then we return
-        while True:
-            line = await self.proc.stderr.readline()
-            if not line.strip():
-                raise Exception("Worker failed to start")
-            logger.info(line.strip())
-            if "Scheduler at" in line:
-                self.address = line.split("Scheduler at:")[1].strip()
-                break
-        logger.debug("%s", line)
-        await super().start()
-
-
-def SSHCluster(
-    hosts,
-    connect_kwargs={},
-    worker_kwargs={},
-    scheduler_kwargs={},
-    worker_module="distributed.cli.dask_worker",
-    **kwargs
-):
-    """ Deploy a Dask cluster using SSH
-
-    Parameters
-    ----------
-    hosts: List[str]
-        List of hostnames or addresses on which to launch our cluster
-        The first will be used for the scheduler and the rest for workers
-    connect_kwargs:
-        Keywords to pass through to asyncssh.connect
-        known_hosts: List[str] or None
-            The list of keys which will be used to validate the server host
-            key presented during the SSH handshake.  If this is not specified,
-            the keys will be looked up in the file .ssh/known_hosts.  If this
-            is explicitly set to None, server host key validation will be disabled.
-    worker_kwargs:
-        Keywords to pass on to dask-worker
-    scheduler_kwargs:
-        Keywords to pass on to dask-scheduler
-    worker_module:
-        Python module to call to start the worker
-
-    Examples
-    --------
-    >>> from dask.distributed import Client
-    >>> from distributed.deploy.ssh2 import SSHCluster  # experimental for now
-    >>> cluster = SSHCluster(
-    ...     ["localhost"] * 4,
-    ...     connect_kwargs={"known_hosts": None},
-    ...     worker_kwargs={"nthreads": 2},
-    ...     scheduler_kwargs={"port": 0, "dashboard_address": ":8797"})
-    >>> client = Client(cluster)
-
-    Running GPU workers (requires ``dask_cuda`` to be installed on all hosts)
-
-    >>> from dask.distributed import Client
-    >>> from distributed.deploy.ssh2 import SSHCluster  # experimental for now
-    >>> cluster = SSHCluster(
-    ...     ["localhost", "hostwithgpus", "anothergpuhost"],
-    ...     connect_kwargs={"known_hosts": None},
-    ...     scheduler_kwargs={"port": 0, "dashboard_address": ":8797"},
-    ...     worker_module='dask_cuda.dask_cuda_worker')
-    >>> client = Client(cluster)
-
-    """
-    scheduler = {
-        "cls": Scheduler,
-        "options": {
-            "address": hosts[0],
-            "connect_kwargs": connect_kwargs,
-            "kwargs": scheduler_kwargs,
-        },
-    }
-    workers = {
-        i: {
-            "cls": Worker,
-            "options": {
-                "address": host,
-                "connect_kwargs": connect_kwargs,
-                "kwargs": worker_kwargs,
-                "worker_module": worker_module,
-            },
-        }
-        for i, host in enumerate(hosts[1:])
-    }
-    return SpecCluster(workers, scheduler, name="SSHCluster", **kwargs)
diff --git a/distributed/deploy/tests/test_old_ssh.py b/distributed/deploy/tests/test_old_ssh.py
new file mode 100644
index 00000000000..e6960b3392d
--- /dev/null
+++ b/distributed/deploy/tests/test_old_ssh.py
@@ -0,0 +1,31 @@
+from time import sleep
+
+import pytest
+
+pytest.importorskip("paramiko")
+
+from distributed import Client
+from distributed.deploy.old_ssh import SSHCluster
+from distributed.metrics import time
+from distributed.utils_test import loop  # noqa: F401
+
+
+@pytest.mark.avoid_travis
+def test_cluster(loop):
+    with SSHCluster(
+        scheduler_addr="127.0.0.1",
+        scheduler_port=7437,
+        worker_addrs=["127.0.0.1", "127.0.0.1"],
+    ) as c:
+        with Client(c, loop=loop) as e:
+            start = time()
+            while len(e.ncores()) != 2:
+                sleep(0.01)
+                assert time() < start + 5
+
+            c.add_worker("127.0.0.1")
+
+            start = time()
+            while len(e.ncores()) != 3:
+                sleep(0.01)
+                assert time() < start + 5
diff --git a/distributed/deploy/tests/test_ssh.py b/distributed/deploy/tests/test_ssh.py
index 492ee2c792d..3124af4f177 100644
--- a/distributed/deploy/tests/test_ssh.py
+++ b/distributed/deploy/tests/test_ssh.py
@@ -1,31 +1,57 @@
-from time import sleep
-
 import pytest
 
-pytest.importorskip("paramiko")
+pytest.importorskip("asyncssh")
 
-from distributed import Client
+from dask.distributed import Client
 from distributed.deploy.ssh import SSHCluster
-from distributed.metrics import time
 from distributed.utils_test import loop  # noqa: F401
 
 
+@pytest.mark.asyncio
+async def test_basic():
+    async with SSHCluster(
+        ["127.0.0.1"] * 3,
+        connect_options=dict(known_hosts=None),
+        asynchronous=True,
+        scheduler_options={"port": 0, "idle_timeout": "5s"},
+        worker_options={"death_timeout": "5s"},
+    ) as cluster:
+        assert len(cluster.workers) == 2
+        async with Client(cluster, asynchronous=True) as client:
+            result = await client.submit(lambda x: x + 1, 10)
+            assert result == 11
+        assert not cluster._supports_scaling
+
+        assert "SSH" in repr(cluster)
+
+
+@pytest.mark.asyncio
+async def test_keywords():
+    async with SSHCluster(
+        ["127.0.0.1"] * 3,
+        connect_options=dict(known_hosts=None),
+        asynchronous=True,
+        worker_options={"nthreads": 2, "memory_limit": "2 GiB", "death_timeout": "5s"},
+        scheduler_options={"idle_timeout": "5s", "port": 0},
+    ) as cluster:
+        async with Client(cluster, asynchronous=True) as client:
+            assert (
+                await client.run_on_scheduler(
+                    lambda dask_scheduler: dask_scheduler.idle_timeout
+                )
+            ) == 5
+            d = client.scheduler_info()["workers"]
+            assert all(v["nthreads"] == 2 for v in d.values())
+
+
 @pytest.mark.avoid_travis
-def test_cluster(loop):
-    with SSHCluster(
-        scheduler_addr="127.0.0.1",
-        scheduler_port=7437,
-        worker_addrs=["127.0.0.1", "127.0.0.1"],
-    ) as c:
-        with Client(c, loop=loop) as e:
-            start = time()
-            while len(e.ncores()) != 2:
-                sleep(0.01)
-                assert time() < start + 5
-
-            c.add_worker("127.0.0.1")
-
-            start = time()
-            while len(e.ncores()) != 3:
-                sleep(0.01)
-                assert time() < start + 5
+def test_defer_to_old(loop):
+    with pytest.warns(Warning):
+        with SSHCluster(
+            scheduler_addr="127.0.0.1",
+            scheduler_port=7437,
+            worker_addrs=["127.0.0.1", "127.0.0.1"],
+        ) as c:
+            from distributed.deploy.old_ssh import SSHCluster as OldSSHCluster
+
+            assert isinstance(c, OldSSHCluster)
diff --git a/distributed/deploy/tests/test_ssh2.py b/distributed/deploy/tests/test_ssh2.py
deleted file mode 100644
index 076711bb841..00000000000
--- a/distributed/deploy/tests/test_ssh2.py
+++ /dev/null
@@ -1,43 +0,0 @@
-import pytest
-
-pytest.importorskip("asyncssh")
-
-from dask.distributed import Client
-from distributed.deploy.ssh2 import SSHCluster
-
-
-@pytest.mark.asyncio
-async def test_basic():
-    async with SSHCluster(
-        ["127.0.0.1"] * 3,
-        connect_kwargs=dict(known_hosts=None),
-        asynchronous=True,
-        scheduler_kwargs={"port": 0, "idle_timeout": "5s"},
-        worker_kwargs={"death_timeout": "5s"},
-    ) as cluster:
-        assert len(cluster.workers) == 2
-        async with Client(cluster, asynchronous=True) as client:
-            result = await client.submit(lambda x: x + 1, 10)
-            assert result == 11
-        assert not cluster._supports_scaling
-
-        assert "SSH" in repr(cluster)
-
-
-@pytest.mark.asyncio
-async def test_keywords():
-    async with SSHCluster(
-        ["127.0.0.1"] * 3,
-        connect_kwargs=dict(known_hosts=None),
-        asynchronous=True,
-        worker_kwargs={"nthreads": 2, "memory_limit": "2 GiB", "death_timeout": "5s"},
-        scheduler_kwargs={"idle_timeout": "5s", "port": 0},
-    ) as cluster:
-        async with Client(cluster, asynchronous=True) as client:
-            assert (
-                await client.run_on_scheduler(
-                    lambda dask_scheduler: dask_scheduler.idle_timeout
-                )
-            ) == 5
-            d = client.scheduler_info()["workers"]
-            assert all(v["nthreads"] == 2 for v in d.values())

From 856bf29c0c45cc4411e47575fb8f637dc0cf9b77 Mon Sep 17 00:00:00 2001
From: Jonathan De Troye <detroyejr@outlook.com>
Date: Wed, 9 Oct 2019 14:57:02 -0400
Subject: [PATCH 0498/1550] Raise exception if the user passes in unused
 keywords to Client (#3117)

Fixes #3014
---
 distributed/client.py                  |  5 +++++
 distributed/deploy/tests/test_local.py | 21 +++++++++++++++++++++
 distributed/tests/test_client.py       |  6 ++----
 3 files changed, 28 insertions(+), 4 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index f918fffbf78..a14929ba326 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -643,6 +643,11 @@ def __init__(
             if address:
                 logger.info("Config value `scheduler-address` found: %s", address)
 
+        if address is not None and kwargs:
+            raise ValueError(
+                "Unexpected keyword arguments: {}".format(str(sorted(kwargs)))
+            )
+
         if isinstance(address, (rpc, PooledRPCCall)):
             self.scheduler = address
         elif hasattr(address, "scheduler_address"):
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index 520c99eb268..7a340a9c6f8 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -219,6 +219,27 @@ def test_Client_kwargs(loop):
     assert c.cluster.status == "closed"
 
 
+def test_Client_unused_kwargs_with_cluster(loop):
+    with LocalCluster() as cluster:
+        with pytest.raises(Exception) as argexcept:
+            c = Client(cluster, n_workers=2, dashboard_port=8000, silence_logs=None)
+        assert (
+            str(argexcept.value)
+            == "Unexpected keyword arguments: ['dashboard_port', 'n_workers', 'silence_logs']"
+        )
+
+
+def test_Client_unused_kwargs_with_address(loop):
+    with pytest.raises(Exception) as argexcept:
+        c = Client(
+            "127.0.0.1:8786", n_workers=2, dashboard_port=8000, silence_logs=None
+        )
+    assert (
+        str(argexcept.value)
+        == "Unexpected keyword arguments: ['dashboard_port', 'n_workers', 'silence_logs']"
+    )
+
+
 def test_Client_twice(loop):
     with Client(loop=loop, silence_logs=False, dashboard_address=None) as c:
         with Client(loop=loop, silence_logs=False, dashboard_address=None) as f:
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index d01088502f0..c4ebe92b4f1 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -5010,9 +5010,7 @@ def test_profile_keys(c, s, a, b):
 @gen_cluster()
 def test_client_with_name(s, a, b):
     with captured_logger("distributed.scheduler") as sio:
-        client = yield Client(
-            s.address, asynchronous=True, name="foo", silence_logs=False
-        )
+        client = yield Client(s.address, asynchronous=True, name="foo")
         assert "foo" in client.id
         yield client.close()
 
@@ -5356,7 +5354,7 @@ def test_de_serialization_none(s, a, b):
 
 @gen_cluster()
 def test_client_repr_closed(s, a, b):
-    c = yield Client(s.address, asynchronous=True, dashboard_address=None)
+    c = yield Client(s.address, asynchronous=True)
     yield c.close()
     c._repr_html_()
 

From 935ec35eb7e5c84551ae50bd0bf267fb45a68c04 Mon Sep 17 00:00:00 2001
From: matthieubulte <matthieu.bulte.06@gmail.com>
Date: Wed, 9 Oct 2019 22:10:59 +0200
Subject: [PATCH 0499/1550] Extend Worker plugin API with transition method
 (#2994)

---
 distributed/__init__.py                       |  1 +
 distributed/client.py                         | 27 ++++--
 distributed/diagnostics/plugin.py             | 67 ++++++++++++-
 ...est_plugin.py => test_scheduler_plugin.py} |  3 +-
 .../diagnostics/tests/test_worker_plugin.py   | 93 +++++++++++++++++++
 distributed/tests/test_worker_plugins.py      | 67 -------------
 distributed/worker.py                         | 30 ++++--
 docs/source/plugins.rst                       |  6 ++
 8 files changed, 205 insertions(+), 89 deletions(-)
 rename distributed/diagnostics/tests/{test_plugin.py => test_scheduler_plugin.py} (94%)
 create mode 100644 distributed/diagnostics/tests/test_worker_plugin.py
 delete mode 100644 distributed/tests/test_worker_plugins.py

diff --git a/distributed/__init__.py b/distributed/__init__.py
index 07015ff44af..1eadee32307 100644
--- a/distributed/__init__.py
+++ b/distributed/__init__.py
@@ -4,6 +4,7 @@
 from .core import connect, rpc
 from .deploy import LocalCluster, Adaptive, SpecCluster, SSHCluster
 from .diagnostics.progressbar import progress
+from .diagnostics.plugin import WorkerPlugin, SchedulerPlugin
 from .client import (
     Client,
     Executor,
diff --git a/distributed/client.py b/distributed/client.py
index a14929ba326..08e808acd15 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -66,6 +66,7 @@
 from .sizeof import sizeof
 from .threadpoolexecutor import rejoin
 from .worker import dumps_task, get_client, get_worker, secede
+from .diagnostics.plugin import WorkerPlugin
 from .utils import (
     All,
     sync,
@@ -3908,13 +3909,15 @@ def register_worker_plugin(self, plugin=None, name=None):
         """
         Registers a lifecycle worker plugin for all current and future workers.
 
-        This registers a new object to handle setup and teardown for workers in
-        this cluster. The plugin will instantiate itself on all currently
-        connected workers.  It will also be run on any worker that connects in
-        the future.
+        This registers a new object to handle setup, task state transitions and
+        teardown for workers in this cluster. The plugin will instantiate itself
+        on all currently connected workers. It will also be run on any worker
+        that connects in the future.
 
-        The plugin should be an object with ``setup`` and ``teardown`` methods.
-        It must be serializable with the pickle or cloudpickle modules.
+        The plugin may include methods ``setup``, ``teardown``, and
+        ``transition``.  See the ``dask.distributed.WorkerPlugin`` class or the
+        examples below for the interface and docstrings.  It must be
+        serializable with the pickle or cloudpickle modules.
 
         If the plugin has a ``name`` attribute, or if the ``name=`` keyword is
         used then that will control idempotency.  A a plugin with that name has
@@ -3925,7 +3928,7 @@ def register_worker_plugin(self, plugin=None, name=None):
 
         Parameters
         ----------
-        plugin: object
+        plugin: WorkerPlugin
             The plugin object to pass to the workers
         name: str, optional
             A name for the plugin.
@@ -3933,13 +3936,15 @@ def register_worker_plugin(self, plugin=None, name=None):
 
         Examples
         --------
-        >>> class MyPlugin:
+        >>> class MyPlugin(WorkerPlugin):
         ...     def __init__(self, *args, **kwargs):
         ...         pass  # the constructor is up to you
         ...     def setup(self, worker: dask.distributed.Worker):
         ...         pass
         ...     def teardown(self, worker: dask.distributed.Worker):
         ...         pass
+        ...     def transition(self, key: str, start: str, finish: str, **kwargs):
+        ...         pass
 
         >>> plugin = MyPlugin(1, 2, 3)
         >>> client.register_worker_plugin(plugin)
@@ -3953,11 +3958,15 @@ def register_worker_plugin(self, plugin=None, name=None):
         ...    return plugin.my_state
 
         >>> future = client.run(f)
+
+        See Also
+        --------
+        distributed.WorkerPlugin
         """
         return self.sync(self._register_worker_plugin, plugin=plugin, name=name)
 
 
-class _WorkerSetupPlugin(object):
+class _WorkerSetupPlugin(WorkerPlugin):
     """ This is used to support older setup functions as callbacks """
 
     def __init__(self, setup):
diff --git a/distributed/diagnostics/plugin.py b/distributed/diagnostics/plugin.py
index cfe5fa42b49..8d56679e9a9 100644
--- a/distributed/diagnostics/plugin.py
+++ b/distributed/diagnostics/plugin.py
@@ -33,8 +33,8 @@ class SchedulerPlugin(object):
     ...     def restart(self, scheduler):
     ...         self.counter = 0
 
-    >>> c = Counter()
-    >>> scheduler.add_plugin(c)  # doctest: +SKIP
+    >>> plugin = Counter()
+    >>> scheduler.add_plugin(plugin)  # doctest: +SKIP
     """
 
     def update_graph(self, scheduler, dsk=None, keys=None, restrictions=None, **kwargs):
@@ -63,3 +63,66 @@ def add_worker(self, scheduler=None, worker=None, **kwargs):
 
     def remove_worker(self, scheduler=None, worker=None, **kwargs):
         """ Run when a worker leaves the cluster"""
+
+
+class WorkerPlugin(object):
+    """ Interface to extend the Worker
+
+    A worker plugin enables custom code to run at different stages of the Workers'
+    lifecycle: at setup, during task state transitions and at teardown.
+
+    A plugin enables custom code to run at each of step of a Workers's life. Whenever such
+    an event happens, the corresponding method on this class will be called. Note that the
+    user code always runs within the Worker's main thread.
+
+    To implement a plugin implement some of the methods of this class and register
+    the plugin to your client in order to have it attached to every existing and
+    future workers with ``Client.register_worker_plugin``.
+
+    Examples
+    --------
+    >>> class ErrorLogger(WorkerPlugin):
+    ...     def __init__(self, logger):
+    ...         self.logger = logger
+    ...
+    ...     def setup(self, worker):
+    ...         self.worker = worker
+    ...
+    ...     def transition(self, key, start, finish, *args, **kwargs):
+    ...         if finish == 'error':
+    ...             exc = self.worker.exceptions[key]
+    ...             self.logger.error("Task '%s' has failed with exception: %s" % (key, str(exc)))
+
+    >>> plugin = ErrorLogger()
+    >>> client.register_worker_plugin(plugin)  # doctest: +SKIP
+    """
+
+    def setup(self, worker):
+        """
+        Run when the plugin is attached to a worker. This happens when the plugin is registered
+        and attached to existing workers, or when a worker is created after the plugin has been
+        registered.
+        """
+
+    def teardown(self, worker):
+        """ Run when the worker to which the plugin is attached to is closed """
+
+    def transition(self, key, start, finish, **kwargs):
+        """
+        Throughout the lifecycle of a task (see :doc:`Worker <worker>`), Workers are
+        instructed by the scheduler to compute certain tasks, resulting in transitions
+        in the state of each task. The Worker owning the task is then notified of this
+        state transition.
+
+        Whenever a task changes its state, this method will be called.
+
+        Parameters
+        ----------
+        key: string
+        start: string
+            Start state of the transition.
+            One of waiting, ready, executing, long-running, memory, error.
+        finish: string
+            Final state of the transition.
+        kwargs: More options passed when transitioning
+        """
diff --git a/distributed/diagnostics/tests/test_plugin.py b/distributed/diagnostics/tests/test_scheduler_plugin.py
similarity index 94%
rename from distributed/diagnostics/tests/test_plugin.py
rename to distributed/diagnostics/tests/test_scheduler_plugin.py
index af29e81674d..2903214ba32 100644
--- a/distributed/diagnostics/tests/test_plugin.py
+++ b/distributed/diagnostics/tests/test_scheduler_plugin.py
@@ -1,6 +1,5 @@
-from distributed import Worker
+from distributed import Worker, SchedulerPlugin
 from distributed.utils_test import inc, gen_cluster
-from distributed.diagnostics.plugin import SchedulerPlugin
 
 
 @gen_cluster(client=True)
diff --git a/distributed/diagnostics/tests/test_worker_plugin.py b/distributed/diagnostics/tests/test_worker_plugin.py
new file mode 100644
index 00000000000..b3b919d7fe2
--- /dev/null
+++ b/distributed/diagnostics/tests/test_worker_plugin.py
@@ -0,0 +1,93 @@
+import pytest
+
+from distributed import Worker, WorkerPlugin
+from distributed.utils_test import gen_cluster
+
+
+class MyPlugin(WorkerPlugin):
+    name = "MyPlugin"
+
+    def __init__(self, data, expected_transitions=None):
+        self.data = data
+        self.expected_transitions = expected_transitions
+
+    def setup(self, worker):
+        assert isinstance(worker, Worker)
+        self.worker = worker
+        self.worker._my_plugin_status = "setup"
+        self.worker._my_plugin_data = self.data
+
+        self.observed_transitions = []
+
+    def teardown(self, worker):
+        self.worker._my_plugin_status = "teardown"
+
+        if self.expected_transitions is not None:
+            assert len(self.observed_transitions) == len(self.expected_transitions)
+            for expected, real in zip(
+                self.expected_transitions, self.observed_transitions
+            ):
+                assert expected == real
+
+    def transition(self, key, start, finish, **kwargs):
+        self.observed_transitions.append((key, start, finish))
+
+
+@gen_cluster(client=True, nthreads=[])
+def test_create_with_client(c, s):
+    yield c.register_worker_plugin(MyPlugin(123))
+
+    worker = yield Worker(s.address, loop=s.loop)
+    assert worker._my_plugin_status == "setup"
+    assert worker._my_plugin_data == 123
+
+    yield worker.close()
+    assert worker._my_plugin_status == "teardown"
+
+
+@gen_cluster(client=True, worker_kwargs={"plugins": [MyPlugin(5)]})
+def test_create_on_construction(c, s, a, b):
+    assert len(a.plugins) == len(b.plugins) == 1
+    assert a._my_plugin_status == "setup"
+    assert a._my_plugin_data == 5
+
+
+@gen_cluster(nthreads=[("127.0.0.1", 1)], client=True)
+def test_normal_task_transitions_called(c, s, w):
+    expected_transitions = [
+        ("task", "waiting", "ready"),
+        ("task", "ready", "executing"),
+        ("task", "executing", "memory"),
+    ]
+
+    plugin = MyPlugin(1, expected_transitions=expected_transitions)
+
+    yield c.register_worker_plugin(plugin)
+    yield c.submit(lambda x: x, 1, key="task")
+
+
+@gen_cluster(nthreads=[("127.0.0.1", 1)], client=True)
+def test_failing_task_transitions_called(c, s, w):
+    def failing(x):
+        raise Exception()
+
+    expected_transitions = [
+        ("task", "waiting", "ready"),
+        ("task", "ready", "executing"),
+        ("task", "executing", "error"),
+    ]
+
+    plugin = MyPlugin(1, expected_transitions=expected_transitions)
+
+    yield c.register_worker_plugin(plugin)
+
+    with pytest.raises(Exception):
+        yield c.submit(failing, 1, key="task")
+
+
+@gen_cluster(nthreads=[("127.0.0.1", 1)], client=True)
+async def test_empty_plugin(c, s, w):
+    class EmptyPlugin:
+        pass
+
+    await c.register_worker_plugin(EmptyPlugin())
diff --git a/distributed/tests/test_worker_plugins.py b/distributed/tests/test_worker_plugins.py
deleted file mode 100644
index 02db9419d4e..00000000000
--- a/distributed/tests/test_worker_plugins.py
+++ /dev/null
@@ -1,67 +0,0 @@
-from distributed.utils_test import gen_cluster
-from distributed import Worker
-
-
-class MyPlugin:
-    name = "MyPlugin"
-
-    def __init__(self, data):
-        self.data = data
-
-    def setup(self, worker):
-        assert isinstance(worker, Worker)
-        self.worker = worker
-        self.worker._my_plugin_status = "setup"
-        self.worker._my_plugin_data = self.data
-
-    def teardown(self, worker):
-        assert isinstance(worker, Worker)
-        self.worker._my_plugin_status = "teardown"
-
-
-@gen_cluster(client=True, nthreads=[])
-def test_create_with_client(c, s):
-    yield c.register_worker_plugin(MyPlugin(123))
-
-    worker = yield Worker(s.address, loop=s.loop)
-    assert worker._my_plugin_status == "setup"
-    assert worker._my_plugin_data == 123
-
-    yield worker.close()
-    assert worker._my_plugin_status == "teardown"
-
-
-@gen_cluster(client=True, worker_kwargs={"plugins": [MyPlugin(5)]})
-def test_create_on_construction(c, s, a, b):
-    assert len(a.plugins) == len(b.plugins) == 1
-    assert a._my_plugin_status == "setup"
-    assert a._my_plugin_data == 5
-
-
-@gen_cluster(client=True, worker_kwargs={"plugins": [MyPlugin(5)]})
-def test_idempotence_with_name(c, s, a, b):
-    a._my_plugin_data = 100
-
-    yield c.register_worker_plugin(MyPlugin(5))
-
-    assert a._my_plugin_data == 100  # call above has no effect
-
-
-@gen_cluster(client=True, worker_kwargs={"plugins": [MyPlugin(5)]})
-def test_duplicate_with_no_name(c, s, a, b):
-    assert len(a.plugins) == len(b.plugins) == 1
-
-    plugin = MyPlugin(10)
-    plugin.name = "other-name"
-
-    yield c.register_worker_plugin(plugin)
-
-    assert len(a.plugins) == len(b.plugins) == 2
-
-    assert a._my_plugin_data == 10
-
-    yield c.register_worker_plugin(plugin)
-    assert len(a.plugins) == len(b.plugins) == 2
-
-    yield c.register_worker_plugin(plugin, name="foo")
-    assert len(a.plugins) == len(b.plugins) == 3
diff --git a/distributed/worker.py b/distributed/worker.py
index fba4eed57b3..e06f224da68 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -1515,6 +1515,7 @@ def transition(self, key, finish, **kwargs):
         self.task_state[key] = state or finish
         if self.validate:
             self.validate_key(key)
+        self._notify_transition(key, start, finish, **kwargs)
 
     def transition_waiting_ready(self, key):
         try:
@@ -2293,15 +2294,16 @@ async def plugin_add(self, comm=None, plugin=None, name=None):
                 self.plugins[name] = plugin
 
                 logger.info("Starting Worker plugin %s" % name)
-                try:
-                    result = plugin.setup(worker=self)
-                    if hasattr(result, "__await__"):
-                        result = await result
-                except Exception as e:
-                    msg = error_message(e)
-                    return msg
-                else:
-                    return {"status": "OK"}
+                if hasattr(plugin, "setup"):
+                    try:
+                        result = plugin.setup(worker=self)
+                        if hasattr(result, "__await__"):
+                            result = await result
+                    except Exception as e:
+                        msg = error_message(e)
+                        return msg
+
+                return {"status": "OK"}
 
     async def actor_execute(
         self, comm=None, actor=None, function=None, args=(), kwargs={}
@@ -2712,6 +2714,16 @@ def get_call_stack(self, comm=None, keys=None):
         result = {k: profile.call_stack(frame) for k, frame in frames.items()}
         return result
 
+    def _notify_transition(self, key, start, finish, **kwargs):
+        for name, plugin in self.plugins.items():
+            if hasattr(plugin, "transition"):
+                try:
+                    plugin.transition(key, start, finish, **kwargs)
+                except Exception:
+                    logger.info(
+                        "Plugin '%s' failed with exception" % name, exc_info=True
+                    )
+
     ##############
     # Validation #
     ##############
diff --git a/docs/source/plugins.rst b/docs/source/plugins.rst
index b5f52f8843e..5c831fc167e 100644
--- a/docs/source/plugins.rst
+++ b/docs/source/plugins.rst
@@ -73,3 +73,9 @@ the scheduler as so:
    def dask_setup(scheduler):
        plugin = MyPlugin(scheduler)
        scheduler.add_plugin(plugin)
+
+Worker Plugins
+=================
+
+.. autoclass:: distributed.diagnostics.plugin.WorkerPlugin
+   :members:

From 7ca0c6b8bc5a6d54d93033e4f483cceafca75b9b Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 9 Oct 2019 15:38:43 -0500
Subject: [PATCH 0500/1550] Xfail test_worksapce_concurrency on Python 3.6
 (#3132)

---
 distributed/tests/test_diskutils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/distributed/tests/test_diskutils.py b/distributed/tests/test_diskutils.py
index e12fb324341..0057f96fb36 100644
--- a/distributed/tests/test_diskutils.py
+++ b/distributed/tests/test_diskutils.py
@@ -275,8 +275,8 @@ def _test_workspace_concurrency(tmpdir, timeout, max_procs):
 def test_workspace_concurrency(tmpdir):
     if WINDOWS:
         raise pytest.xfail.Exception("TODO: unknown failure on windows")
-    if sys.version_info < (3, 6):
-        raise pytest.xfail.Exception("TODO: unknown failure on Python 3.5")
+    if sys.version_info <= (3, 6):
+        raise pytest.xfail.Exception("TODO: unknown failure on Python 3.6")
     _test_workspace_concurrency(tmpdir, 2.0, 6)
 
 

From c2cc1a98cbaadc3e7952ea66e5096f0126492539 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 10 Oct 2019 09:51:12 -0700
Subject: [PATCH 0501/1550] Add Nanny(config={...}) keyword (#3134)

---
 distributed/nanny.py            | 16 +++++++++++++++-
 distributed/tests/test_nanny.py |  9 +++++++++
 2 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/distributed/nanny.py b/distributed/nanny.py
index 8fbbf761368..83ca2ebbf80 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -86,6 +86,7 @@ def __init__(
         host=None,
         port=None,
         protocol=None,
+        config=None,
         **worker_kwargs
     ):
         self._setup_logging(logger)
@@ -123,6 +124,7 @@ def __init__(
             self.preload_argv = dask.config.get("distributed.worker.preload-argv")
         self.Worker = Worker if worker_class is None else worker_class
         self.env = env or {}
+        self.config = config or {}
         worker_kwargs.update(
             {
                 "port": worker_port,
@@ -304,6 +306,7 @@ async def instantiate(self, comm=None):
                 on_exit=self._on_exit_sync,
                 worker=self.Worker,
                 env=self.env,
+                config=self.config,
             )
 
         self.auto_restart = True
@@ -437,7 +440,14 @@ async def close(self, comm=None, timeout=5, report=None):
 
 class WorkerProcess(object):
     def __init__(
-        self, worker_kwargs, worker_start_args, silence_logs, on_exit, worker, env
+        self,
+        worker_kwargs,
+        worker_start_args,
+        silence_logs,
+        on_exit,
+        worker,
+        env,
+        config,
     ):
         self.status = "init"
         self.silence_logs = silence_logs
@@ -447,6 +457,7 @@ def __init__(
         self.process = None
         self.Worker = worker
         self.env = env
+        self.config = config
 
         # Initialized when worker is ready
         self.worker_dir = None
@@ -479,6 +490,7 @@ async def start(self):
                 uid=uid,
                 Worker=self.Worker,
                 env=self.env,
+                config=self.config,
             ),
         )
         self.process.daemon = dask.config.get("distributed.worker.daemon", default=True)
@@ -621,9 +633,11 @@ def _run(
         child_stop_q,
         uid,
         env,
+        config,
         Worker,
     ):  # pragma: no cover
         os.environ.update(env)
+        dask.config.set(config)
         try:
             from dask.multiprocessing import initialize_worker_process
         except ImportError:  # old Dask version
diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
index dec6bd91b20..d54cf4e3b14 100644
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -436,3 +436,12 @@ async def test_nanny_closes_cleanly(cleanup):
                         assert time() < start + 5
 
                     assert n.status == "closed"
+
+
+@pytest.mark.asyncio
+async def test_config(cleanup):
+    async with Scheduler() as s:
+        async with Nanny(s.address, config={"foo": "bar"}) as n:
+            async with Client(s.address, asynchronous=True) as client:
+                config = await client.run(dask.config.get, "foo")
+                assert config[n.worker_address] == "bar"

From e39959e9b0e2e9746da0d1a53edeeea1931f9e47 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 11 Oct 2019 13:48:24 -0700
Subject: [PATCH 0502/1550] Only include metric in WorkerTable if it is a
 scalar (#3140)

---
 distributed/dashboard/scheduler.py                  | 7 ++++++-
 distributed/dashboard/tests/test_scheduler_bokeh.py | 5 +++++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/distributed/dashboard/scheduler.py b/distributed/dashboard/scheduler.py
index 27a49b4fd68..79484cd4196 100644
--- a/distributed/dashboard/scheduler.py
+++ b/distributed/dashboard/scheduler.py
@@ -1380,7 +1380,12 @@ def __init__(self, scheduler, width=800, **kwargs):
         ]
         workers = self.scheduler.workers.values()
         self.extra_names = sorted(
-            {m for ws in workers for m in ws.metrics if m not in self.names}
+            {
+                m
+                for ws in workers
+                for m, v in ws.metrics.items()
+                if m not in self.names and isinstance(v, (str, int, float))
+            }
             - self.excluded_names
         )
 
diff --git a/distributed/dashboard/tests/test_scheduler_bokeh.py b/distributed/dashboard/tests/test_scheduler_bokeh.py
index e68d7935583..1e48a3addec 100644
--- a/distributed/dashboard/tests/test_scheduler_bokeh.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh.py
@@ -305,6 +305,11 @@ def test_WorkerTable(c, s, a, b):
     wt = WorkerTable(s)
     wt.update()
     assert all(wt.source.data.values())
+    assert all(
+        not v or isinstance(v, (str, int, float))
+        for L in wt.source.data.values()
+        for v in L
+    ), {type(v).__name__ for L in wt.source.data.values() for v in L}
     assert all(len(v) == 2 for v in wt.source.data.values())
 
     nthreads = wt.source.data["nthreads"]

From 00d7f7dfff47cf2aa67dd220c15d3ccaebbabcc8 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 14 Oct 2019 07:55:24 -0700
Subject: [PATCH 0503/1550] Move death timeout logic up to Node.start (#3115)

Previously there were some cases where the death-timeout logic
wouldn't reliably be triggered.  Now we handle it higher up in the call
chain, where hopefully it will be more consistent.
---
 distributed/node.py              | 19 ++++++++++++++++++-
 distributed/tests/test_worker.py |  5 ++++-
 distributed/worker.py            | 16 ----------------
 3 files changed, 22 insertions(+), 18 deletions(-)

diff --git a/distributed/node.py b/distributed/node.py
index 8ef610a8481..2d7447b1a06 100644
--- a/distributed/node.py
+++ b/distributed/node.py
@@ -1,3 +1,4 @@
+import asyncio
 import logging
 import warnings
 import weakref
@@ -163,7 +164,23 @@ def __await__(self):
         if self.status == "running":
             return gen.sleep(0).__await__()
         else:
-            return self.start().__await__()
+            future = self.start()
+            timeout = getattr(self, "death_timeout", 0)
+            if timeout:
+
+                async def wait_for(future, timeout=None):
+                    try:
+                        await asyncio.wait_for(future, timeout=timeout)
+                    except Exception:
+                        await self.close(timeout=1)
+                        raise gen.TimeoutError(
+                            "{} failed to start in {} seconds".format(
+                                type(self).__name__, timeout
+                            )
+                        )
+
+                future = wait_for(future, timeout=timeout)
+            return future.__await__()
 
     async def start(self):  # subclasses should implement this
         return self
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index eb4c0f86c7b..53aac46216a 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -762,9 +762,12 @@ def test_worker_death_timeout(s):
         yield s.close()
         w = Worker(s.address, death_timeout=1)
 
-    with pytest.raises(gen.TimeoutError):
+    with pytest.raises(gen.TimeoutError) as info:
         yield w
 
+    assert "Worker" in str(info.value)
+    assert "timed out" in str(info.value) or "failed to start" in str(info.value)
+
     assert w.status == "closed"
 
 
diff --git a/distributed/worker.py b/distributed/worker.py
index e06f224da68..12dfe3fe178 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -784,17 +784,6 @@ async def _register_with_scheduler(self):
             self.contact_address = self.address
         logger.info("-" * 49)
         while True:
-            if self.death_timeout and time() > start + self.death_timeout:
-                logger.exception(
-                    "Timed out when connecting to scheduler '%s'",
-                    self.scheduler.address,
-                )
-                await self.close(timeout=1)
-                raise gen.TimeoutError(
-                    "Timed out connecting to scheduler '%s'" % self.scheduler.address
-                )
-            if self.status in ("closed", "closing"):
-                return
             try:
                 _start = time()
                 types = {k: typename(v) for k, v in self.data.items()}
@@ -826,11 +815,6 @@ async def _register_with_scheduler(self):
                     serializers=["msgpack"],
                 )
                 future = comm.read(deserializers=["msgpack"])
-                if self.death_timeout:
-                    diff = self.death_timeout - (time() - start)
-                    if diff < 0:
-                        continue
-                    future = gen.with_timeout(timedelta(seconds=diff), future)
                 response = await future
                 _end = time()
                 middle = (_start + _end) / 2

From eb7bccc7e3a202f11c17455740f3af4d913332c3 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 15 Oct 2019 12:06:59 -0700
Subject: [PATCH 0504/1550] Use setuptools.find_packages in setup.py (#3150)

---
 setup.py | 12 ++----------
 1 file changed, 2 insertions(+), 10 deletions(-)

diff --git a/setup.py b/setup.py
index 84054d199e0..5d900199256 100755
--- a/setup.py
+++ b/setup.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 import os
-from setuptools import setup
+from setuptools import setup, find_packages
 import versioneer
 
 requires = open("requirements.txt").read().strip().split("\n")
@@ -35,15 +35,7 @@
     include_package_data=True,
     install_requires=install_requires,
     extras_require=extras_require,
-    packages=[
-        "distributed",
-        "distributed.dashboard",
-        "distributed.cli",
-        "distributed.comm",
-        "distributed.deploy",
-        "distributed.diagnostics",
-        "distributed.protocol",
-    ],
+    packages=find_packages(exclude=["*tests*"]),
     long_description=(
         open("README.rst").read() if os.path.exists("README.rst") else ""
     ),

From c970ec0835648ea7ecced35b1909eddbddd925b4 Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Tue, 15 Oct 2019 20:08:20 +0100
Subject: [PATCH 0505/1550] Refactor dashboard module (#3138)

Closes #3048.

In an attempt to tidy up the dashboard code I've done some major shuffling of things within the dashboard submodule. I think overall this puts things in more obvious places and should make it more accessible to new contributors.

Notable changes:
- Components have been broken out in a new `distributed.dashboard.components` submodule and placed into logical groupings "scheduler specific", "worker specific", "shared" and "nvml/gpu". This could be broken down further but there is a balance to be struck between indirection and giant scary files.
- The additional server routes from `scheduler_html.py` and `worker_html.py` have been moved into the `scheduler.py` and `worker.py` files to keep all server things together.
- Shared functions and utilities have been moved around into more appropriate places.

Other things I'd like to do:
- [x] Make use of the [update source function](https://github.com/dask/distributed/pull/3138/files#diff-0db0a1f6d00335e7cc6e5e94eae3f8a1R89-R116) outside of the scheduler components
- [x] ~Re-order components to group `_doc` functions with their `DashboardComponent` classes to reduce indirection (or at least distance).~ _On second thoughts as some docs use multiple components having the docs together at the bottom actually makes more sense._
- [x] Rename functions for consistency (there are a few which should probably follow the `_doc` names)
- [x] Identify and remove unused components. (Could use some pointers on this)
---
 distributed/client.py                         |    6 +-
 distributed/dashboard/components/__init__.py  |   93 +
 .../dashboard/{ => components}/nvml.py        |   27 +-
 distributed/dashboard/components/scheduler.py | 1873 ++++++++++++++++
 .../{components.py => components/shared.py}   |  295 +--
 distributed/dashboard/components/worker.py    |  661 ++++++
 distributed/dashboard/scheduler.py            | 1954 ++---------------
 distributed/dashboard/scheduler_html.py       |  269 ---
 .../dashboard/tests/test_components.py        |    6 +-
 .../dashboard/tests/test_scheduler_bokeh.py   |   28 +-
 .../dashboard/tests/test_worker_bokeh.py      |    4 +-
 distributed/dashboard/utils.py                |   46 +
 distributed/dashboard/worker.py               |  818 +------
 distributed/dashboard/worker_html.py          |  108 -
 distributed/diagnostics/graph_layout.py       |    4 +-
 15 files changed, 3132 insertions(+), 3060 deletions(-)
 create mode 100644 distributed/dashboard/components/__init__.py
 rename distributed/dashboard/{ => components}/nvml.py (94%)
 create mode 100644 distributed/dashboard/components/scheduler.py
 rename distributed/dashboard/{components.py => components/shared.py} (75%)
 create mode 100644 distributed/dashboard/components/worker.py
 delete mode 100644 distributed/dashboard/scheduler_html.py
 delete mode 100644 distributed/dashboard/worker_html.py

diff --git a/distributed/client.py b/distributed/client.py
index 08e808acd15..ca7ca431c90 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -3861,7 +3861,7 @@ async def _get_task_stream(
             from .diagnostics.task_stream import rectangles
 
             rects = rectangles(msgs)
-            from .dashboard.components import task_stream_figure
+            from .dashboard.components.scheduler import task_stream_figure
 
             source, figure = task_stream_figure(sizing_mode="stretch_both")
             source.data.update(rects)
@@ -4424,7 +4424,7 @@ class get_task_stream(object):
 
     To share this file with others you may wish to upload and serve it online.
     A common way to do this is to upload the file as a gist, and then serve it
-    on https://rawgit.com ::
+    on https://raw.githack.com ::
 
        $ pip install gist
        $ gist task-stream.html
@@ -4432,7 +4432,7 @@ class get_task_stream(object):
 
     You can then navigate to that site, click the "Raw" button to the right of
     the ``task-stream.html`` file, and then provide that URL to
-    https://rawgit.com .  This process should provide a sharable link that
+    https://raw.githack.com .  This process should provide a sharable link that
     others can use to see your task stream plot.
 
     See Also
diff --git a/distributed/dashboard/components/__init__.py b/distributed/dashboard/components/__init__.py
new file mode 100644
index 00000000000..12f57b352b1
--- /dev/null
+++ b/distributed/dashboard/components/__init__.py
@@ -0,0 +1,93 @@
+import asyncio
+from bisect import bisect
+from operator import add
+from time import time
+import weakref
+
+from bokeh.layouts import row, column
+from bokeh.models import (
+    ColumnDataSource,
+    Plot,
+    DataRange1d,
+    LinearAxis,
+    HoverTool,
+    BoxZoomTool,
+    ResetTool,
+    PanTool,
+    WheelZoomTool,
+    Range1d,
+    Quad,
+    TapTool,
+    OpenURL,
+    Button,
+    Select,
+)
+from bokeh.palettes import Spectral9
+from bokeh.plotting import figure
+import dask
+from tornado import gen
+import toolz
+
+from distributed.dashboard.utils import without_property_validation, BOKEH_VERSION
+from distributed.diagnostics.progress_stream import nbytes_bar
+from distributed import profile
+from distributed.utils import log_errors, parse_timedelta
+
+if dask.config.get("distributed.dashboard.export-tool"):
+    from distributed.dashboard.export_tool import ExportTool
+else:
+    ExportTool = None
+
+
+profile_interval = dask.config.get("distributed.worker.profile.interval")
+profile_interval = parse_timedelta(profile_interval, default="ms")
+
+
+class DashboardComponent(object):
+    """ Base class for Dask.distributed UI dashboard components.
+
+    This class must have two attributes, ``root`` and ``source``, and one
+    method ``update``:
+
+    *  source: a Bokeh ColumnDataSource
+    *  root: a Bokeh Model
+    *  update: a method that consumes the messages dictionary found in
+               distributed.bokeh.messages
+    """
+
+    def __init__(self):
+        self.source = None
+        self.root = None
+
+    def update(self, messages):
+        """ Reads from bokeh.distributed.messages and updates self.source """
+
+
+def add_periodic_callback(doc, component, interval):
+    """ Add periodic callback to doc in a way that avoids reference cycles
+
+    If we instead use ``doc.add_periodic_callback(component.update, 100)`` then
+    the component stays in memory as a reference cycle because its method is
+    still around.  This way we avoid that and let things clean up a bit more
+    nicely.
+
+    TODO: we still have reference cycles.  Docs seem to be referred to by their
+    add_periodic_callback methods.
+    """
+    ref = weakref.ref(component)
+
+    doc.add_periodic_callback(lambda: update(ref), interval)
+    _attach(doc, component)
+
+
+def update(ref):
+    comp = ref()
+    if comp is not None:
+        comp.update()
+
+
+def _attach(doc, component):
+    if not hasattr(doc, "components"):
+        doc.components = set()
+
+    doc.components.add(component)
diff --git a/distributed/dashboard/nvml.py b/distributed/dashboard/components/nvml.py
similarity index 94%
rename from distributed/dashboard/nvml.py
rename to distributed/dashboard/components/nvml.py
index 131a02a8397..b0c56c4ef47 100644
--- a/distributed/dashboard/nvml.py
+++ b/distributed/dashboard/components/nvml.py
@@ -1,6 +1,6 @@
 import math
 
-from .components import DashboardComponent, add_periodic_callback
+from distributed.dashboard.components import DashboardComponent, add_periodic_callback
 
 from bokeh.plotting import figure
 from bokeh.models import (
@@ -13,9 +13,17 @@
 )
 from tornado import escape
 from dask.utils import format_bytes
-from ..utils import log_errors
-from .scheduler import update, applications, BOKEH_THEME
-from .utils import without_property_validation
+from distributed.utils import log_errors
+from distributed.dashboard.components.scheduler import BOKEH_THEME
+from distributed.dashboard.utils import without_property_validation, update
+
+
+try:
+    import pynvml
+
+    pynvml.nvmlInit()
+except Exception:
+    pass
 
 
 class GPUCurrentLoad(DashboardComponent):
@@ -181,14 +189,3 @@ def gpu_utilization_doc(scheduler, extra, doc):
     add_periodic_callback(doc, gpu_load, 100)
     doc.add_root(gpu_load.utilization_figure)
     doc.theme = BOKEH_THEME
-
-
-try:
-    import pynvml
-
-    pynvml.nvmlInit()
-except Exception:
-    pass
-else:
-    applications["/individual-gpu-memory"] = gpu_memory_doc
-    applications["/individual-gpu-utilization"] = gpu_utilization_doc
diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
new file mode 100644
index 00000000000..5e94a034cb0
--- /dev/null
+++ b/distributed/dashboard/components/scheduler.py
@@ -0,0 +1,1873 @@
+import logging
+import math
+from numbers import Number
+from operator import add
+import os
+
+from bokeh.layouts import column, row
+from bokeh.models import (
+    ColumnDataSource,
+    ColorBar,
+    DataRange1d,
+    HoverTool,
+    ResetTool,
+    PanTool,
+    WheelZoomTool,
+    TapTool,
+    OpenURL,
+    Range1d,
+    Plot,
+    Quad,
+    Span,
+    value,
+    LinearAxis,
+    NumeralTickFormatter,
+    BoxZoomTool,
+    BasicTicker,
+    NumberFormatter,
+    BoxSelectTool,
+    GroupFilter,
+    CDSView,
+)
+from bokeh.models.widgets import DataTable, TableColumn
+from bokeh.plotting import figure
+from bokeh.palettes import Viridis11
+from bokeh.themes import Theme
+from bokeh.transform import factor_cmap, linear_cmap
+from bokeh.io import curdoc
+import dask
+from dask.utils import format_bytes
+from toolz import pipe
+from tornado import escape
+
+try:
+    import numpy as np
+except ImportError:
+    np = False
+
+from distributed.dashboard.components import add_periodic_callback
+from distributed.dashboard.components.shared import (
+    DashboardComponent,
+    ProfileTimePlot,
+    ProfileServer,
+    SystemMonitor,
+)
+from distributed.dashboard.utils import (
+    transpose,
+    BOKEH_VERSION,
+    PROFILING,
+    without_property_validation,
+    update,
+)
+from distributed.metrics import time
+from distributed.utils import log_errors, format_time, parse_timedelta
+from distributed.diagnostics.progress_stream import color_of, progress_quads, nbytes_bar
+from distributed.diagnostics.progress import AllProgress
+from distributed.diagnostics.graph_layout import GraphLayout
+from distributed.diagnostics.task_stream import TaskStreamPlugin
+
+try:
+    from cytoolz.curried import map, concat, groupby, valmap
+except ImportError:
+    from toolz.curried import map, concat, groupby, valmap
+
+if dask.config.get("distributed.dashboard.export-tool"):
+    from distributed.dashboard.export_tool import ExportTool
+else:
+    ExportTool = None
+
+logger = logging.getLogger(__name__)
+
+from jinja2 import Environment, FileSystemLoader
+
+env = Environment(
+    loader=FileSystemLoader(os.path.join(os.path.dirname(__file__), "..", "templates"))
+)
+
+BOKEH_THEME = Theme(os.path.join(os.path.dirname(__file__), "..", "theme.yaml"))
+
+nan = float("nan")
+inf = float("inf")
+
+
+class Occupancy(DashboardComponent):
+    """ Occupancy (in time) per worker """
+
+    def __init__(self, scheduler, **kwargs):
+        with log_errors():
+            self.scheduler = scheduler
+            self.source = ColumnDataSource(
+                {
+                    "occupancy": [0, 0],
+                    "worker": ["a", "b"],
+                    "x": [0.0, 0.1],
+                    "y": [1, 2],
+                    "ms": [1, 2],
+                    "color": ["red", "blue"],
+                    "escaped_worker": ["a", "b"],
+                }
+            )
+
+            fig = figure(
+                title="Occupancy",
+                tools="",
+                id="bk-occupancy-plot",
+                x_axis_type="datetime",
+                **kwargs
+            )
+            rect = fig.rect(
+                source=self.source, x="x", width="ms", y="y", height=1, color="color"
+            )
+            rect.nonselection_glyph = None
+
+            fig.xaxis.minor_tick_line_alpha = 0
+            fig.yaxis.visible = False
+            fig.ygrid.visible = False
+            # fig.xaxis[0].formatter = NumeralTickFormatter(format='0.0s')
+            fig.x_range.start = 0
+
+            tap = TapTool(callback=OpenURL(url="./info/worker/@escaped_worker.html"))
+
+            hover = HoverTool()
+            hover.tooltips = "@worker : @occupancy s."
+            hover.point_policy = "follow_mouse"
+            fig.add_tools(hover, tap)
+
+            self.root = fig
+
+    @without_property_validation
+    def update(self):
+        with log_errors():
+            workers = list(self.scheduler.workers.values())
+
+            y = list(range(len(workers)))
+            occupancy = [ws.occupancy for ws in workers]
+            ms = [occ * 1000 for occ in occupancy]
+            x = [occ / 500 for occ in occupancy]
+            total = sum(occupancy)
+            color = []
+            for ws in workers:
+                if ws in self.scheduler.idle:
+                    color.append("red")
+                elif ws in self.scheduler.saturated:
+                    color.append("green")
+                else:
+                    color.append("blue")
+
+            if total:
+                self.root.title.text = "Occupancy -- total time: %s  wall time: %s" % (
+                    format_time(total),
+                    format_time(total / self.scheduler.total_nthreads),
+                )
+            else:
+                self.root.title.text = "Occupancy"
+
+            if occupancy:
+                result = {
+                    "occupancy": occupancy,
+                    "worker": [ws.address for ws in workers],
+                    "ms": ms,
+                    "color": color,
+                    "escaped_worker": [escape.url_escape(ws.address) for ws in workers],
+                    "x": x,
+                    "y": y,
+                }
+
+                update(self.source, result)
+
+
+class ProcessingHistogram(DashboardComponent):
+    """ How many tasks are on each worker """
+
+    def __init__(self, scheduler, **kwargs):
+        with log_errors():
+            self.last = 0
+            self.scheduler = scheduler
+            self.source = ColumnDataSource(
+                {"left": [1, 2], "right": [10, 10], "top": [0, 0]}
+            )
+
+            self.root = figure(
+                title="Tasks Processing (Histogram)",
+                id="bk-nprocessing-histogram-plot",
+                name="processing_hist",
+                y_axis_label="frequency",
+                tools="",
+                **kwargs
+            )
+
+            self.root.xaxis.minor_tick_line_alpha = 0
+            self.root.ygrid.visible = False
+
+            self.root.toolbar.logo = None
+            self.root.toolbar_location = None
+
+            self.root.quad(
+                source=self.source,
+                left="left",
+                right="right",
+                bottom=0,
+                top="top",
+                color="deepskyblue",
+                fill_alpha=0.5,
+            )
+
+    @without_property_validation
+    def update(self):
+        L = [len(ws.processing) for ws in self.scheduler.workers.values()]
+        counts, x = np.histogram(L, bins=40)
+        self.source.data.update({"left": x[:-1], "right": x[1:], "top": counts})
+
+
+class NBytesHistogram(DashboardComponent):
+    """ How many tasks are on each worker """
+
+    def __init__(self, scheduler, **kwargs):
+        with log_errors():
+            self.last = 0
+            self.scheduler = scheduler
+            self.source = ColumnDataSource(
+                {"left": [1, 2], "right": [10, 10], "top": [0, 0]}
+            )
+
+            self.root = figure(
+                title="Bytes Stored (Histogram)",
+                name="nbytes_hist",
+                id="bk-nbytes-histogram-plot",
+                y_axis_label="frequency",
+                tools="",
+                **kwargs
+            )
+
+            self.root.xaxis[0].formatter = NumeralTickFormatter(format="0.0 b")
+            self.root.xaxis.major_label_orientation = -math.pi / 12
+
+            self.root.xaxis.minor_tick_line_alpha = 0
+            self.root.ygrid.visible = False
+
+            self.root.toolbar.logo = None
+            self.root.toolbar_location = None
+
+            self.root.quad(
+                source=self.source,
+                left="left",
+                right="right",
+                bottom=0,
+                top="top",
+                color="deepskyblue",
+                fill_alpha=0.5,
+            )
+
+    @without_property_validation
+    def update(self):
+        nbytes = np.asarray([ws.nbytes for ws in self.scheduler.workers.values()])
+        counts, x = np.histogram(nbytes, bins=40)
+        d = {"left": x[:-1], "right": x[1:], "top": counts}
+        self.source.data.update(d)
+
+        self.root.title.text = "Bytes stored (Histogram): " + format_bytes(nbytes.sum())
+
+
+class BandwidthTypes(DashboardComponent):
+    """ Bar chart showing bandwidth per type """
+
+    def __init__(self, scheduler, **kwargs):
+        with log_errors():
+            self.last = 0
+            self.scheduler = scheduler
+            self.source = ColumnDataSource(
+                {
+                    "bandwidth": [1, 2],
+                    "bandwidth-half": [0.5, 1],
+                    "type": ["a", "b"],
+                    "bandwidth_text": ["1", "2"],
+                }
+            )
+
+            fig = figure(
+                title="Bandwidth by Type",
+                tools="",
+                id="bk-bandwidth-type-plot",
+                name="bandwidth_type_histogram",
+                y_range=["a", "b"],
+                **kwargs
+            )
+            rect = fig.rect(
+                source=self.source,
+                x="bandwidth-half",
+                y="type",
+                width="bandwidth",
+                height=1,
+                color="blue",
+            )
+            fig.x_range.start = 0
+            fig.xaxis[0].formatter = NumeralTickFormatter(format="0.0 b")
+            rect.nonselection_glyph = None
+
+            fig.xaxis.minor_tick_line_alpha = 0
+            fig.ygrid.visible = False
+
+            fig.toolbar.logo = None
+            fig.toolbar_location = None
+
+            hover = HoverTool()
+            hover.tooltips = "@type: @bandwidth_text / s"
+            hover.point_policy = "follow_mouse"
+            fig.add_tools(hover)
+
+            self.fig = fig
+
+    @without_property_validation
+    def update(self):
+        with log_errors():
+            bw = self.scheduler.bandwidth_types
+            self.fig.y_range.factors = list(sorted(bw))
+            result = {
+                "bandwidth": list(bw.values()),
+                "bandwidth-half": [b / 2 for b in bw.values()],
+                "type": list(bw.keys()),
+                "bandwidth_text": list(map(format_bytes, bw.values())),
+            }
+            self.fig.title.text = "Bandwidth: " + format_bytes(self.scheduler.bandwidth)
+
+            update(self.source, result)
+
+
+class BandwidthWorkers(DashboardComponent):
+    """ How many tasks are on each worker """
+
+    def __init__(self, scheduler, **kwargs):
+        with log_errors():
+            self.last = 0
+            self.scheduler = scheduler
+            self.source = ColumnDataSource(
+                {
+                    "bandwidth": [1, 2],
+                    "source": ["a", "b"],
+                    "destination": ["a", "b"],
+                    "bandwidth_text": ["1", "2"],
+                }
+            )
+
+            values = [hex(x)[2:] for x in range(64, 256)][::-1]
+            mapper = linear_cmap(
+                field_name="bandwidth",
+                palette=["#" + x + x + "FF" for x in values],
+                low=0,
+                high=1,
+            )
+
+            fig = figure(
+                title="Bandwidth by Worker",
+                tools="",
+                id="bk-bandwidth-worker-plot",
+                name="bandwidth_worker_heatmap",
+                x_range=["a", "b"],
+                y_range=["a", "b"],
+                **kwargs
+            )
+            fig.xaxis.major_label_orientation = -math.pi / 12
+            rect = fig.rect(
+                source=self.source,
+                x="source",
+                y="destination",
+                color=mapper,
+                height=1,
+                width=1,
+            )
+
+            self.color_map = mapper["transform"]
+            color_bar = ColorBar(
+                color_mapper=self.color_map,
+                label_standoff=12,
+                border_line_color=None,
+                location=(0, 0),
+            )
+            color_bar.formatter = NumeralTickFormatter(format="0 b")
+            fig.add_layout(color_bar, "right")
+
+            fig.toolbar.logo = None
+            fig.toolbar_location = None
+
+            hover = HoverTool()
+            hover.tooltips = """
+            <div>
+                <p><b>Source:</b> @source </p>
+                <p><b>Destination:</b> @destination </p>
+                <p><b>Bandwidth:</b> @bandwidth_text / s</p>
+            </div>
+            """
+            hover.point_policy = "follow_mouse"
+            fig.add_tools(hover)
+
+            self.fig = fig
+
+    @without_property_validation
+    def update(self):
+        with log_errors():
+            bw = self.scheduler.bandwidth_workers
+            if not bw:
+                return
+            x, y, value = zip(*[(a, b, c) for (a, b), c in bw.items()])
+
+            if self.color_map.high < max(value):
+                self.color_map.high = max(value)
+
+            factors = list(sorted(set(x + y)))
+            self.fig.x_range.factors = factors
+            self.fig.y_range.factors = factors
+
+            result = {
+                "source": x,
+                "destination": y,
+                "bandwidth": value,
+                "bandwidth_text": list(map(format_bytes, value)),
+            }
+            self.fig.title.text = "Bandwidth: " + format_bytes(self.scheduler.bandwidth)
+
+            update(self.source, result)
+
+
+class CurrentLoad(DashboardComponent):
+    """ How many tasks are on each worker """
+
+    def __init__(self, scheduler, width=600, **kwargs):
+        with log_errors():
+            self.last = 0
+            self.scheduler = scheduler
+            self.source = ColumnDataSource(
+                {
+                    "nprocessing": [1, 2],
+                    "nprocessing-half": [0.5, 1],
+                    "nprocessing-color": ["red", "blue"],
+                    "nbytes": [1, 2],
+                    "nbytes-half": [0.5, 1],
+                    "nbytes_text": ["1B", "2B"],
+                    "cpu": [1, 2],
+                    "cpu-half": [0.5, 1],
+                    "worker": ["a", "b"],
+                    "y": [1, 2],
+                    "nbytes-color": ["blue", "blue"],
+                    "escaped_worker": ["a", "b"],
+                }
+            )
+
+            processing = figure(
+                title="Tasks Processing",
+                tools="",
+                id="bk-nprocessing-plot",
+                name="processing_hist",
+                width=int(width / 2),
+                **kwargs
+            )
+            rect = processing.rect(
+                source=self.source,
+                x="nprocessing-half",
+                y="y",
+                width="nprocessing",
+                height=1,
+                color="nprocessing-color",
+            )
+            processing.x_range.start = 0
+            rect.nonselection_glyph = None
+
+            nbytes = figure(
+                title="Bytes stored",
+                tools="",
+                id="bk-nbytes-worker-plot",
+                width=int(width / 2),
+                name="nbytes_hist",
+                **kwargs
+            )
+            rect = nbytes.rect(
+                source=self.source,
+                x="nbytes-half",
+                y="y",
+                width="nbytes",
+                height=1,
+                color="nbytes-color",
+            )
+            rect.nonselection_glyph = None
+
+            cpu = figure(
+                title="CPU Utilization",
+                tools="",
+                id="bk-cpu-worker-plot",
+                width=int(width / 2),
+                name="cpu_hist",
+                **kwargs
+            )
+            rect = cpu.rect(
+                source=self.source,
+                x="cpu-half",
+                y="y",
+                width="cpu",
+                height=1,
+                color="blue",
+            )
+            rect.nonselection_glyph = None
+            hundred_span = Span(
+                location=100,
+                dimension="height",
+                line_color="gray",
+                line_dash="dashed",
+                line_width=3,
+            )
+            cpu.add_layout(hundred_span)
+
+            nbytes.axis[0].ticker = BasicTicker(mantissas=[1, 256, 512], base=1024)
+            nbytes.xaxis[0].formatter = NumeralTickFormatter(format="0.0 b")
+            nbytes.xaxis.major_label_orientation = -math.pi / 12
+            nbytes.x_range.start = 0
+
+            for fig in [processing, nbytes]:
+                fig.xaxis.minor_tick_line_alpha = 0
+                fig.yaxis.visible = False
+                fig.ygrid.visible = False
+
+                tap = TapTool(
+                    callback=OpenURL(url="./info/worker/@escaped_worker.html")
+                )
+                fig.add_tools(tap)
+
+                fig.toolbar.logo = None
+                fig.toolbar_location = None
+                fig.yaxis.visible = False
+
+            hover = HoverTool()
+            hover.tooltips = "@worker : @nprocessing tasks"
+            hover.point_policy = "follow_mouse"
+            processing.add_tools(hover)
+
+            hover = HoverTool()
+            hover.tooltips = "@worker : @nbytes_text"
+            hover.point_policy = "follow_mouse"
+            nbytes.add_tools(hover)
+
+            hover = HoverTool()
+            hover.tooltips = "@worker : @cpu %"
+            hover.point_policy = "follow_mouse"
+            cpu.add_tools(hover)
+
+            self.processing_figure = processing
+            self.nbytes_figure = nbytes
+            self.cpu_figure = cpu
+
+            processing.y_range = nbytes.y_range
+            cpu.y_range = nbytes.y_range
+
+    @without_property_validation
+    def update(self):
+        with log_errors():
+            workers = list(self.scheduler.workers.values())
+
+            y = list(range(len(workers)))
+
+            cpu = [int(ws.metrics["cpu"]) for ws in workers]
+
+            nprocessing = [len(ws.processing) for ws in workers]
+            processing_color = []
+            for ws in workers:
+                if ws in self.scheduler.idle:
+                    processing_color.append("red")
+                elif ws in self.scheduler.saturated:
+                    processing_color.append("green")
+                else:
+                    processing_color.append("blue")
+
+            nbytes = [ws.metrics["memory"] for ws in workers]
+            nbytes_text = [format_bytes(nb) for nb in nbytes]
+            nbytes_color = []
+            max_limit = 0
+            for ws, nb in zip(workers, nbytes):
+                limit = (
+                    getattr(self.scheduler.workers[ws.address], "memory_limit", inf)
+                    or inf
+                )
+
+                if limit > max_limit:
+                    max_limit = limit
+
+                if nb > limit:
+                    nbytes_color.append("red")
+                elif nb > limit / 2:
+                    nbytes_color.append("orange")
+                else:
+                    nbytes_color.append("blue")
+
+            now = time()
+            if any(nprocessing) or self.last + 1 < now:
+                self.last = now
+                result = {
+                    "cpu": cpu,
+                    "cpu-half": [c / 2 for c in cpu],
+                    "nprocessing": nprocessing,
+                    "nprocessing-half": [np / 2 for np in nprocessing],
+                    "nprocessing-color": processing_color,
+                    "nbytes": nbytes,
+                    "nbytes-half": [nb / 2 for nb in nbytes],
+                    "nbytes-color": nbytes_color,
+                    "nbytes_text": nbytes_text,
+                    "worker": [ws.address for ws in workers],
+                    "escaped_worker": [escape.url_escape(ws.address) for ws in workers],
+                    "y": y,
+                }
+
+                self.nbytes_figure.title.text = "Bytes stored: " + format_bytes(
+                    sum(nbytes)
+                )
+                self.nbytes_figure.x_range.end = max_limit
+
+                update(self.source, result)
+
+
+class StealingTimeSeries(DashboardComponent):
+    def __init__(self, scheduler, **kwargs):
+        self.scheduler = scheduler
+        self.source = ColumnDataSource(
+            {"time": [time(), time() + 1], "idle": [0, 0.1], "saturated": [0, 0.1]}
+        )
+
+        x_range = DataRange1d(follow="end", follow_interval=20000, range_padding=0)
+
+        fig = figure(
+            title="Idle and Saturated Workers Over Time",
+            x_axis_type="datetime",
+            y_range=[-0.1, len(scheduler.workers) + 0.1],
+            height=150,
+            tools="",
+            x_range=x_range,
+            **kwargs
+        )
+        fig.line(source=self.source, x="time", y="idle", color="red")
+        fig.line(source=self.source, x="time", y="saturated", color="green")
+        fig.yaxis.minor_tick_line_color = None
+
+        fig.add_tools(
+            ResetTool(), PanTool(dimensions="width"), WheelZoomTool(dimensions="width")
+        )
+
+        self.root = fig
+
+    @without_property_validation
+    def update(self):
+        with log_errors():
+            result = {
+                "time": [time() * 1000],
+                "idle": [len(self.scheduler.idle)],
+                "saturated": [len(self.scheduler.saturated)],
+            }
+            if PROFILING:
+                curdoc().add_next_tick_callback(
+                    lambda: self.source.stream(result, 10000)
+                )
+            else:
+                self.source.stream(result, 10000)
+
+
+class StealingEvents(DashboardComponent):
+    def __init__(self, scheduler, **kwargs):
+        self.scheduler = scheduler
+        self.steal = scheduler.extensions["stealing"]
+        self.last = 0
+        self.source = ColumnDataSource(
+            {
+                "time": [time() - 20, time()],
+                "level": [0, 15],
+                "color": ["white", "white"],
+                "duration": [0, 0],
+                "radius": [1, 1],
+                "cost_factor": [0, 10],
+                "count": [1, 1],
+            }
+        )
+
+        x_range = DataRange1d(follow="end", follow_interval=20000, range_padding=0)
+
+        fig = figure(
+            title="Stealing Events",
+            x_axis_type="datetime",
+            y_axis_type="log",
+            height=250,
+            tools="",
+            x_range=x_range,
+            **kwargs
+        )
+
+        fig.circle(
+            source=self.source,
+            x="time",
+            y="cost_factor",
+            color="color",
+            size="radius",
+            alpha=0.5,
+        )
+        fig.yaxis.axis_label = "Cost Multiplier"
+
+        hover = HoverTool()
+        hover.tooltips = "Level: @level, Duration: @duration, Count: @count, Cost factor: @cost_factor"
+        hover.point_policy = "follow_mouse"
+
+        fig.add_tools(
+            hover,
+            ResetTool(),
+            PanTool(dimensions="width"),
+            WheelZoomTool(dimensions="width"),
+        )
+
+        self.root = fig
+
+    def convert(self, msgs):
+        """ Convert a log message to a glyph """
+        total_duration = 0
+        for msg in msgs:
+            time, level, key, duration, sat, occ_sat, idl, occ_idl = msg
+            total_duration += duration
+
+        try:
+            color = Viridis11[level]
+        except (KeyError, IndexError):
+            color = "black"
+
+        radius = math.sqrt(min(total_duration, 10)) * 30 + 2
+
+        d = {
+            "time": time * 1000,
+            "level": level,
+            "count": len(msgs),
+            "color": color,
+            "duration": total_duration,
+            "radius": radius,
+            "cost_factor": min(10, self.steal.cost_multipliers[level]),
+        }
+
+        return d
+
+    @without_property_validation
+    def update(self):
+        with log_errors():
+            log = self.steal.log
+            n = self.steal.count - self.last
+            log = [log[-i] for i in range(1, n + 1) if isinstance(log[-i], list)]
+            self.last = self.steal.count
+
+            if log:
+                new = pipe(
+                    log,
+                    map(groupby(1)),
+                    map(dict.values),
+                    concat,
+                    map(self.convert),
+                    list,
+                    transpose,
+                )
+                if PROFILING:
+                    curdoc().add_next_tick_callback(
+                        lambda: self.source.stream(new, 10000)
+                    )
+                else:
+                    self.source.stream(new, 10000)
+
+
+class Events(DashboardComponent):
+    def __init__(self, scheduler, name, height=150, **kwargs):
+        self.scheduler = scheduler
+        self.action_ys = dict()
+        self.last = 0
+        self.name = name
+        self.source = ColumnDataSource(
+            {"time": [], "action": [], "hover": [], "y": [], "color": []}
+        )
+
+        x_range = DataRange1d(follow="end", follow_interval=200000)
+
+        fig = figure(
+            title=name,
+            x_axis_type="datetime",
+            height=height,
+            tools="",
+            x_range=x_range,
+            **kwargs
+        )
+
+        fig.circle(
+            source=self.source,
+            x="time",
+            y="y",
+            color="color",
+            size=50,
+            alpha=0.5,
+            legend="action",
+        )
+        fig.yaxis.axis_label = "Action"
+        fig.legend.location = "top_left"
+
+        hover = HoverTool()
+        hover.tooltips = "@action<br>@hover"
+        hover.point_policy = "follow_mouse"
+
+        fig.add_tools(
+            hover,
+            ResetTool(),
+            PanTool(dimensions="width"),
+            WheelZoomTool(dimensions="width"),
+        )
+
+        self.root = fig
+
+    @without_property_validation
+    def update(self):
+        with log_errors():
+            log = self.scheduler.events[self.name]
+            n = self.scheduler.event_counts[self.name] - self.last
+            if log:
+                log = [log[-i] for i in range(1, n + 1)]
+            self.last = self.scheduler.event_counts[self.name]
+
+            if log:
+                actions = []
+                times = []
+                hovers = []
+                ys = []
+                colors = []
+                for msg in log:
+                    times.append(msg["time"] * 1000)
+                    action = msg["action"]
+                    actions.append(action)
+                    try:
+                        ys.append(self.action_ys[action])
+                    except KeyError:
+                        self.action_ys[action] = len(self.action_ys)
+                        ys.append(self.action_ys[action])
+                    colors.append(color_of(action))
+                    hovers.append("TODO")
+
+                new = {
+                    "time": times,
+                    "action": actions,
+                    "hover": hovers,
+                    "y": ys,
+                    "color": colors,
+                }
+
+                if PROFILING:
+                    curdoc().add_next_tick_callback(
+                        lambda: self.source.stream(new, 10000)
+                    )
+                else:
+                    self.source.stream(new, 10000)
+
+
+class TaskStream(DashboardComponent):
+    def __init__(self, scheduler, n_rectangles=1000, clear_interval="20s", **kwargs):
+        self.scheduler = scheduler
+        self.offset = 0
+        es = [p for p in self.scheduler.plugins if isinstance(p, TaskStreamPlugin)]
+        if not es:
+            self.plugin = TaskStreamPlugin(self.scheduler)
+        else:
+            self.plugin = es[0]
+        self.index = max(0, self.plugin.index - n_rectangles)
+        self.workers = dict()
+        self.n_rectangles = n_rectangles
+        clear_interval = parse_timedelta(clear_interval, default="ms")
+        self.clear_interval = clear_interval
+        self.last = 0
+
+        self.source, self.root = task_stream_figure(clear_interval, **kwargs)
+
+        # Required for update callback
+        self.task_stream_index = [0]
+
+    @without_property_validation
+    def update(self):
+        if self.index == self.plugin.index:
+            return
+        with log_errors():
+            if self.index and len(self.source.data["start"]):
+                start = min(self.source.data["start"])
+                duration = max(self.source.data["duration"])
+                boundary = (self.offset + start - duration) / 1000
+            else:
+                boundary = self.offset
+            rectangles = self.plugin.rectangles(
+                istart=self.index, workers=self.workers, start_boundary=boundary
+            )
+            n = len(rectangles["name"])
+            self.index = self.plugin.index
+
+            if not rectangles["start"]:
+                return
+
+            # If there has been a significant delay then clear old rectangles
+            first_end = min(map(add, rectangles["start"], rectangles["duration"]))
+            if first_end > self.last:
+                last = self.last
+                self.last = first_end
+                if first_end > last + self.clear_interval * 1000:
+                    self.offset = min(rectangles["start"])
+                    self.source.data.update({k: [] for k in rectangles})
+
+            rectangles["start"] = [x - self.offset for x in rectangles["start"]]
+
+            # Convert to numpy for serialization speed
+            if n >= 10 and np:
+                for k, v in rectangles.items():
+                    if isinstance(v[0], Number):
+                        rectangles[k] = np.array(v)
+
+            if PROFILING:
+                curdoc().add_next_tick_callback(
+                    lambda: self.source.stream(rectangles, self.n_rectangles)
+                )
+            else:
+                self.source.stream(rectangles, self.n_rectangles)
+
+
+def task_stream_figure(clear_interval="20s", **kwargs):
+    """
+    kwargs are applied to the bokeh.models.plots.Plot constructor
+    """
+    clear_interval = parse_timedelta(clear_interval, default="ms")
+
+    source = ColumnDataSource(
+        data=dict(
+            start=[time() - clear_interval],
+            duration=[0.1],
+            key=["start"],
+            name=["start"],
+            color=["white"],
+            duration_text=["100 ms"],
+            worker=["foo"],
+            y=[0],
+            worker_thread=[1],
+            alpha=[0.0],
+        )
+    )
+
+    x_range = DataRange1d(range_padding=0)
+    y_range = DataRange1d(range_padding=0)
+
+    root = figure(
+        name="task_stream",
+        title="Task Stream",
+        id="bk-task-stream-plot",
+        x_range=x_range,
+        y_range=y_range,
+        toolbar_location="above",
+        x_axis_type="datetime",
+        min_border_right=35,
+        tools="",
+        **kwargs
+    )
+
+    rect = root.rect(
+        source=source,
+        x="start",
+        y="y",
+        width="duration",
+        height=0.4,
+        fill_color="color",
+        line_color="color",
+        line_alpha=0.6,
+        fill_alpha="alpha",
+        line_width=3,
+    )
+    rect.nonselection_glyph = None
+
+    root.yaxis.major_label_text_alpha = 0
+    root.yaxis.minor_tick_line_alpha = 0
+    root.yaxis.major_tick_line_alpha = 0
+    root.xgrid.visible = False
+
+    hover = HoverTool(
+        point_policy="follow_mouse",
+        tooltips="""
+            <div>
+                <span style="font-size: 12px; font-weight: bold;">@name:</span>&nbsp;
+                <span style="font-size: 10px; font-family: Monaco, monospace;">@duration_text</span>
+            </div>
+            """,
+    )
+
+    tap = TapTool(callback=OpenURL(url="/profile?key=@name"))
+
+    root.add_tools(
+        hover,
+        tap,
+        BoxZoomTool(),
+        ResetTool(),
+        PanTool(dimensions="width"),
+        WheelZoomTool(dimensions="width"),
+    )
+    if ExportTool:
+        export = ExportTool()
+        export.register_plot(root)
+        root.add_tools(export)
+
+    return source, root
+
+
+class TaskGraph(DashboardComponent):
+    """
+    A dynamic node-link diagram for the task graph on the scheduler
+
+    See also the GraphLayout diagnostic at
+    distributed/diagnostics/graph_layout.py
+    """
+
+    def __init__(self, scheduler, **kwargs):
+        self.scheduler = scheduler
+        self.layout = GraphLayout(scheduler)
+        self.invisible_count = 0  # number of invisible nodes
+
+        self.node_source = ColumnDataSource(
+            {"x": [], "y": [], "name": [], "state": [], "visible": [], "key": []}
+        )
+        self.edge_source = ColumnDataSource({"x": [], "y": [], "visible": []})
+
+        node_view = CDSView(
+            source=self.node_source,
+            filters=[GroupFilter(column_name="visible", group="True")],
+        )
+        edge_view = CDSView(
+            source=self.edge_source,
+            filters=[GroupFilter(column_name="visible", group="True")],
+        )
+
+        node_colors = factor_cmap(
+            "state",
+            factors=["waiting", "processing", "memory", "released", "erred"],
+            palette=["gray", "green", "red", "blue", "black"],
+        )
+
+        self.root = figure(title="Task Graph", **kwargs)
+        self.root.multi_line(
+            xs="x",
+            ys="y",
+            source=self.edge_source,
+            line_width=1,
+            view=edge_view,
+            color="black",
+            alpha=0.3,
+        )
+        rect = self.root.square(
+            x="x",
+            y="y",
+            size=10,
+            color=node_colors,
+            source=self.node_source,
+            view=node_view,
+            legend="state",
+        )
+        self.root.xgrid.grid_line_color = None
+        self.root.ygrid.grid_line_color = None
+
+        hover = HoverTool(
+            point_policy="follow_mouse",
+            tooltips="<b>@name</b>: @state",
+            renderers=[rect],
+        )
+        tap = TapTool(callback=OpenURL(url="info/task/@key.html"), renderers=[rect])
+        rect.nonselection_glyph = None
+        self.root.add_tools(hover, tap)
+
+    @without_property_validation
+    def update(self):
+        with log_errors():
+            # occasionally reset the column data source to remove old nodes
+            if self.invisible_count > len(self.node_source.data["x"]) / 2:
+                self.layout.reset_index()
+                self.invisible_count = 0
+                update = True
+            else:
+                update = False
+
+            new, self.layout.new = self.layout.new, []
+            new_edges = self.layout.new_edges
+            self.layout.new_edges = []
+
+            self.add_new_nodes_edges(new, new_edges, update=update)
+
+            self.patch_updates()
+
+    @without_property_validation
+    def add_new_nodes_edges(self, new, new_edges, update=False):
+        if new or update:
+            node_key = []
+            node_x = []
+            node_y = []
+            node_state = []
+            node_name = []
+            edge_x = []
+            edge_y = []
+
+            x = self.layout.x
+            y = self.layout.y
+
+            tasks = self.scheduler.tasks
+            for key in new:
+                try:
+                    task = tasks[key]
+                except KeyError:
+                    continue
+                xx = x[key]
+                yy = y[key]
+                node_key.append(escape.url_escape(key))
+                node_x.append(xx)
+                node_y.append(yy)
+                node_state.append(task.state)
+                node_name.append(task.prefix)
+
+            for a, b in new_edges:
+                try:
+                    edge_x.append([x[a], x[b]])
+                    edge_y.append([y[a], y[b]])
+                except KeyError:
+                    pass
+
+            node = {
+                "x": node_x,
+                "y": node_y,
+                "state": node_state,
+                "name": node_name,
+                "key": node_key,
+                "visible": ["True"] * len(node_x),
+            }
+            edge = {"x": edge_x, "y": edge_y, "visible": ["True"] * len(edge_x)}
+
+            if update or not len(self.node_source.data["x"]):
+                # see https://github.com/bokeh/bokeh/issues/7523
+                self.node_source.data.update(node)
+                self.edge_source.data.update(edge)
+            else:
+                self.node_source.stream(node)
+                self.edge_source.stream(edge)
+
+    @without_property_validation
+    def patch_updates(self):
+        """
+        Small updates like color changes or lost nodes from task transitions
+        """
+        n = len(self.node_source.data["x"])
+        m = len(self.edge_source.data["x"])
+
+        if self.layout.state_updates:
+            state_updates = self.layout.state_updates
+            self.layout.state_updates = []
+            updates = [(i, c) for i, c in state_updates if i < n]
+            self.node_source.patch({"state": updates})
+
+        if self.layout.visible_updates:
+            updates = self.layout.visible_updates
+            updates = [(i, c) for i, c in updates if i < n]
+            self.visible_updates = []
+            self.node_source.patch({"visible": updates})
+            self.invisible_count += len(updates)
+
+        if self.layout.visible_edge_updates:
+            updates = self.layout.visible_edge_updates
+            updates = [(i, c) for i, c in updates if i < m]
+            self.visible_updates = []
+            self.edge_source.patch({"visible": updates})
+
+    def __del__(self):
+        self.scheduler.remove_plugin(self.layout)
+
+
+class TaskProgress(DashboardComponent):
+    """ Progress bars per task type """
+
+    def __init__(self, scheduler, **kwargs):
+        self.scheduler = scheduler
+        ps = [p for p in scheduler.plugins if isinstance(p, AllProgress)]
+        if ps:
+            self.plugin = ps[0]
+        else:
+            self.plugin = AllProgress(scheduler)
+
+        data = progress_quads(
+            dict(all={}, memory={}, erred={}, released={}, processing={})
+        )
+        self.source = ColumnDataSource(data=data)
+
+        x_range = DataRange1d(range_padding=0)
+        y_range = Range1d(-8, 0)
+
+        self.root = figure(
+            id="bk-task-progress-plot",
+            title="Progress",
+            name="task_progress",
+            x_range=x_range,
+            y_range=y_range,
+            toolbar_location=None,
+            tools="",
+            **kwargs
+        )
+        self.root.line(  # just to define early ranges
+            x=[0, 0.9], y=[-1, 0], line_color="#FFFFFF", alpha=0.0
+        )
+        self.root.quad(
+            source=self.source,
+            top="top",
+            bottom="bottom",
+            left="left",
+            right="right",
+            fill_color="#aaaaaa",
+            line_color="#aaaaaa",
+            fill_alpha=0.1,
+            line_alpha=0.3,
+        )
+        self.root.quad(
+            source=self.source,
+            top="top",
+            bottom="bottom",
+            left="left",
+            right="released-loc",
+            fill_color="color",
+            line_color="color",
+            fill_alpha=0.6,
+        )
+        self.root.quad(
+            source=self.source,
+            top="top",
+            bottom="bottom",
+            left="released-loc",
+            right="memory-loc",
+            fill_color="color",
+            line_color="color",
+            fill_alpha=1.0,
+        )
+        self.root.quad(
+            source=self.source,
+            top="top",
+            bottom="bottom",
+            left="memory-loc",
+            right="erred-loc",
+            fill_color="black",
+            fill_alpha=0.5,
+            line_alpha=0,
+        )
+        self.root.quad(
+            source=self.source,
+            top="top",
+            bottom="bottom",
+            left="erred-loc",
+            right="processing-loc",
+            fill_color="gray",
+            fill_alpha=0.35,
+            line_alpha=0,
+        )
+        self.root.text(
+            source=self.source,
+            text="show-name",
+            y="bottom",
+            x="left",
+            x_offset=5,
+            text_font_size=value("10pt"),
+        )
+        self.root.text(
+            source=self.source,
+            text="done",
+            y="bottom",
+            x="right",
+            x_offset=-5,
+            text_align="right",
+            text_font_size=value("10pt"),
+        )
+        self.root.ygrid.visible = False
+        self.root.yaxis.minor_tick_line_alpha = 0
+        self.root.yaxis.visible = False
+        self.root.xgrid.visible = False
+        self.root.xaxis.minor_tick_line_alpha = 0
+        self.root.xaxis.visible = False
+
+        hover = HoverTool(
+            point_policy="follow_mouse",
+            tooltips="""
+                <div>
+                    <span style="font-size: 14px; font-weight: bold;">Name:</span>&nbsp;
+                    <span style="font-size: 10px; font-family: Monaco, monospace;">@name</span>
+                </div>
+                <div>
+                    <span style="font-size: 14px; font-weight: bold;">All:</span>&nbsp;
+                    <span style="font-size: 10px; font-family: Monaco, monospace;">@all</span>
+                </div>
+                <div>
+                    <span style="font-size: 14px; font-weight: bold;">Memory:</span>&nbsp;
+                    <span style="font-size: 10px; font-family: Monaco, monospace;">@memory</span>
+                </div>
+                <div>
+                    <span style="font-size: 14px; font-weight: bold;">Erred:</span>&nbsp;
+                    <span style="font-size: 10px; font-family: Monaco, monospace;">@erred</span>
+                </div>
+                <div>
+                    <span style="font-size: 14px; font-weight: bold;">Ready:</span>&nbsp;
+                    <span style="font-size: 10px; font-family: Monaco, monospace;">@processing</span>
+                </div>
+                """,
+        )
+        self.root.add_tools(hover)
+
+    @without_property_validation
+    def update(self):
+        with log_errors():
+            state = {"all": valmap(len, self.plugin.all), "nbytes": self.plugin.nbytes}
+            for k in ["memory", "erred", "released", "processing", "waiting"]:
+                state[k] = valmap(len, self.plugin.state[k])
+            if not state["all"] and not len(self.source.data["all"]):
+                return
+
+            d = progress_quads(state)
+
+            update(self.source, d)
+
+            totals = {
+                k: sum(state[k].values())
+                for k in ["all", "memory", "erred", "released", "waiting"]
+            }
+            totals["processing"] = totals["all"] - sum(
+                v for k, v in totals.items() if k != "all"
+            )
+
+            self.root.title.text = (
+                "Progress -- total: %(all)s, "
+                "in-memory: %(memory)s, processing: %(processing)s, "
+                "waiting: %(waiting)s, "
+                "erred: %(erred)s" % totals
+            )
+
+
+class MemoryUse(DashboardComponent):
+    """ The memory usage across the cluster, grouped by task type """
+
+    def __init__(self, scheduler, **kwargs):
+        self.scheduler = scheduler
+        ps = [p for p in scheduler.plugins if isinstance(p, AllProgress)]
+        if ps:
+            self.plugin = ps[0]
+        else:
+            self.plugin = AllProgress(scheduler)
+
+        self.source = ColumnDataSource(
+            data=dict(
+                name=[],
+                left=[],
+                right=[],
+                center=[],
+                color=[],
+                percent=[],
+                MB=[],
+                text=[],
+            )
+        )
+
+        self.root = Plot(
+            id="bk-nbytes-plot",
+            x_range=DataRange1d(),
+            y_range=DataRange1d(),
+            toolbar_location=None,
+            outline_line_color=None,
+            **kwargs
+        )
+
+        self.root.add_glyph(
+            self.source,
+            Quad(
+                top=1,
+                bottom=0,
+                left="left",
+                right="right",
+                fill_color="color",
+                fill_alpha=1,
+            ),
+        )
+
+        self.root.add_layout(LinearAxis(), "left")
+        self.root.add_layout(LinearAxis(), "below")
+
+        hover = HoverTool(
+            point_policy="follow_mouse",
+            tooltips="""
+                <div>
+                    <span style="font-size: 14px; font-weight: bold;">Name:</span>&nbsp;
+                    <span style="font-size: 10px; font-family: Monaco, monospace;">@name</span>
+                </div>
+                <div>
+                    <span style="font-size: 14px; font-weight: bold;">Percent:</span>&nbsp;
+                    <span style="font-size: 10px; font-family: Monaco, monospace;">@percent</span>
+                </div>
+                <div>
+                    <span style="font-size: 14px; font-weight: bold;">MB:</span>&nbsp;
+                    <span style="font-size: 10px; font-family: Monaco, monospace;">@MB</span>
+                </div>
+                """,
+        )
+        self.root.add_tools(hover)
+
+    @without_property_validation
+    def update(self):
+        with log_errors():
+            nb = nbytes_bar(self.plugin.nbytes)
+            update(self.source, nb)
+            self.root.title.text = "Memory Use: %0.2f MB" % (
+                sum(self.plugin.nbytes.values()) / 1e6
+            )
+
+
+class WorkerTable(DashboardComponent):
+    """ Status of the current workers
+
+    This is two plots, a text-based table for each host and a thin horizontal
+    plot laying out hosts by their current memory use.
+    """
+
+    excluded_names = {"executing", "in_flight", "in_memory", "ready", "time"}
+
+    def __init__(self, scheduler, width=800, **kwargs):
+        self.scheduler = scheduler
+        self.names = [
+            "name",
+            "address",
+            "nthreads",
+            "cpu",
+            "memory",
+            "memory_limit",
+            "memory_percent",
+            "num_fds",
+            "read_bytes",
+            "write_bytes",
+            "cpu_fraction",
+        ]
+        workers = self.scheduler.workers.values()
+        self.extra_names = sorted(
+            {
+                m
+                for ws in workers
+                for m, v in ws.metrics.items()
+                if m not in self.names and isinstance(v, (str, int, float))
+            }
+            - self.excluded_names
+        )
+
+        table_names = [
+            "name",
+            "address",
+            "nthreads",
+            "cpu",
+            "memory",
+            "memory_limit",
+            "memory_percent",
+            "num_fds",
+            "read_bytes",
+            "write_bytes",
+        ]
+
+        self.source = ColumnDataSource({k: [] for k in self.names})
+
+        columns = {
+            name: TableColumn(field=name, title=name.replace("_percent", " %"))
+            for name in table_names
+        }
+
+        formatters = {
+            "cpu": NumberFormatter(format="0.0 %"),
+            "memory_percent": NumberFormatter(format="0.0 %"),
+            "memory": NumberFormatter(format="0 b"),
+            "memory_limit": NumberFormatter(format="0 b"),
+            "read_bytes": NumberFormatter(format="0 b"),
+            "write_bytes": NumberFormatter(format="0 b"),
+            "num_fds": NumberFormatter(format="0"),
+            "nthreads": NumberFormatter(format="0"),
+        }
+
+        if BOKEH_VERSION < "0.12.15":
+            dt_kwargs = {"row_headers": False}
+        else:
+            dt_kwargs = {"index_position": None}
+
+        table = DataTable(
+            source=self.source,
+            columns=[columns[n] for n in table_names],
+            reorderable=True,
+            sortable=True,
+            width=width,
+            **dt_kwargs
+        )
+
+        for name in table_names:
+            if name in formatters:
+                table.columns[table_names.index(name)].formatter = formatters[name]
+
+        extra_names = ["name", "address"] + self.extra_names
+        extra_columns = {
+            name: TableColumn(field=name, title=name.replace("_percent", "%"))
+            for name in extra_names
+        }
+
+        extra_table = DataTable(
+            source=self.source,
+            columns=[extra_columns[n] for n in extra_names],
+            reorderable=True,
+            sortable=True,
+            width=width,
+            **dt_kwargs
+        )
+
+        hover = HoverTool(
+            point_policy="follow_mouse",
+            tooltips="""
+                <div>
+                  <span style="font-size: 10px; font-family: Monaco, monospace;">@worker: </span>
+                  <span style="font-size: 10px; font-family: Monaco, monospace;">@memory_percent</span>
+                </div>
+                """,
+        )
+
+        mem_plot = figure(
+            title="Memory Use (%)",
+            toolbar_location=None,
+            x_range=(0, 1),
+            y_range=(-0.1, 0.1),
+            height=60,
+            width=width,
+            tools="",
+            **kwargs
+        )
+        mem_plot.circle(
+            source=self.source, x="memory_percent", y=0, size=10, fill_alpha=0.5
+        )
+        mem_plot.ygrid.visible = False
+        mem_plot.yaxis.minor_tick_line_alpha = 0
+        mem_plot.xaxis.visible = False
+        mem_plot.yaxis.visible = False
+        mem_plot.add_tools(hover, BoxSelectTool())
+
+        hover = HoverTool(
+            point_policy="follow_mouse",
+            tooltips="""
+                <div>
+                  <span style="font-size: 10px; font-family: Monaco, monospace;">@worker: </span>
+                  <span style="font-size: 10px; font-family: Monaco, monospace;">@cpu</span>
+                </div>
+                """,
+        )
+
+        cpu_plot = figure(
+            title="CPU Use (%)",
+            toolbar_location=None,
+            x_range=(0, 1),
+            y_range=(-0.1, 0.1),
+            height=60,
+            width=width,
+            tools="",
+            **kwargs
+        )
+        cpu_plot.circle(
+            source=self.source, x="cpu_fraction", y=0, size=10, fill_alpha=0.5
+        )
+        cpu_plot.ygrid.visible = False
+        cpu_plot.yaxis.minor_tick_line_alpha = 0
+        cpu_plot.xaxis.visible = False
+        cpu_plot.yaxis.visible = False
+        cpu_plot.add_tools(hover, BoxSelectTool())
+        self.cpu_plot = cpu_plot
+
+        if "sizing_mode" in kwargs:
+            sizing_mode = {"sizing_mode": kwargs["sizing_mode"]}
+        else:
+            sizing_mode = {}
+
+        components = [cpu_plot, mem_plot, table]
+        if self.extra_names:
+            components.append(extra_table)
+
+        self.root = column(*components, id="bk-worker-table", **sizing_mode)
+
+    @without_property_validation
+    def update(self):
+        data = {name: [] for name in self.names + self.extra_names}
+        for i, (addr, ws) in enumerate(
+            sorted(self.scheduler.workers.items(), key=lambda kv: kv[1].name)
+        ):
+            for name in self.names + self.extra_names:
+                data[name].append(ws.metrics.get(name, None))
+            data["name"][-1] = ws.name if ws.name is not None else i
+            data["address"][-1] = ws.address
+            if ws.memory_limit:
+                data["memory_percent"][-1] = ws.metrics["memory"] / ws.memory_limit
+            else:
+                data["memory_percent"][-1] = ""
+            data["memory_limit"][-1] = ws.memory_limit
+            data["cpu"][-1] = ws.metrics["cpu"] / 100.0
+            data["cpu_fraction"][-1] = ws.metrics["cpu"] / 100.0 / ws.nthreads
+            data["nthreads"][-1] = ws.nthreads
+
+        self.source.data.update(data)
+
+
+def systemmonitor_doc(scheduler, extra, doc):
+    with log_errors():
+        sysmon = SystemMonitor(scheduler, sizing_mode="stretch_both")
+        doc.title = "Dask: Scheduler System Monitor"
+        add_periodic_callback(doc, sysmon, 500)
+
+        for subdoc in sysmon.root.children:
+            doc.add_root(subdoc)
+        doc.template = env.get_template("system.html")
+        doc.template_variables.update(extra)
+        doc.theme = BOKEH_THEME
+
+
+def stealing_doc(scheduler, extra, doc):
+    with log_errors():
+        occupancy = Occupancy(scheduler, height=200, sizing_mode="scale_width")
+        stealing_ts = StealingTimeSeries(scheduler, sizing_mode="scale_width")
+        stealing_events = StealingEvents(scheduler, sizing_mode="scale_width")
+        stealing_events.root.x_range = stealing_ts.root.x_range
+        doc.title = "Dask: Work Stealing"
+        add_periodic_callback(doc, occupancy, 500)
+        add_periodic_callback(doc, stealing_ts, 500)
+        add_periodic_callback(doc, stealing_events, 500)
+
+        doc.add_root(
+            column(
+                occupancy.root,
+                stealing_ts.root,
+                stealing_events.root,
+                sizing_mode="scale_width",
+            )
+        )
+
+        doc.template = env.get_template("simple.html")
+        doc.template_variables.update(extra)
+        doc.theme = BOKEH_THEME
+
+
+def events_doc(scheduler, extra, doc):
+    with log_errors():
+        events = Events(scheduler, "all", height=250)
+        events.update()
+        add_periodic_callback(doc, events, 500)
+        doc.title = "Dask: Scheduler Events"
+        doc.add_root(column(events.root, sizing_mode="scale_width"))
+        doc.template = env.get_template("simple.html")
+        doc.template_variables.update(extra)
+        doc.theme = BOKEH_THEME
+
+
+def workers_doc(scheduler, extra, doc):
+    with log_errors():
+        table = WorkerTable(scheduler)
+        table.update()
+        add_periodic_callback(doc, table, 500)
+        doc.title = "Dask: Workers"
+        doc.add_root(table.root)
+        doc.template = env.get_template("simple.html")
+        doc.template_variables.update(extra)
+        doc.theme = BOKEH_THEME
+
+
+def tasks_doc(scheduler, extra, doc):
+    with log_errors():
+        ts = TaskStream(
+            scheduler,
+            n_rectangles=dask.config.get(
+                "distributed.scheduler.dashboard.tasks.task-stream-length"
+            ),
+            clear_interval="60s",
+            sizing_mode="stretch_both",
+        )
+        ts.update()
+        add_periodic_callback(doc, ts, 5000)
+        doc.title = "Dask: Task Stream"
+        doc.add_root(ts.root)
+        doc.template = env.get_template("simple.html")
+        doc.template_variables.update(extra)
+        doc.theme = BOKEH_THEME
+
+
+def graph_doc(scheduler, extra, doc):
+    with log_errors():
+        graph = TaskGraph(scheduler, sizing_mode="stretch_both")
+        doc.title = "Dask: Task Graph"
+        graph.update()
+        add_periodic_callback(doc, graph, 200)
+        doc.add_root(graph.root)
+
+        doc.template = env.get_template("simple.html")
+        doc.template_variables.update(extra)
+        doc.theme = BOKEH_THEME
+
+
+def status_doc(scheduler, extra, doc):
+    with log_errors():
+        task_stream = TaskStream(
+            scheduler,
+            n_rectangles=dask.config.get(
+                "distributed.scheduler.dashboard.status.task-stream-length"
+            ),
+            clear_interval="10s",
+            sizing_mode="stretch_both",
+        )
+        task_stream.update()
+        add_periodic_callback(doc, task_stream, 100)
+
+        task_progress = TaskProgress(scheduler, sizing_mode="stretch_both")
+        task_progress.update()
+        add_periodic_callback(doc, task_progress, 100)
+
+        if len(scheduler.workers) < 50:
+            current_load = CurrentLoad(scheduler, sizing_mode="stretch_both")
+            current_load.update()
+            add_periodic_callback(doc, current_load, 100)
+            doc.add_root(current_load.nbytes_figure)
+            doc.add_root(current_load.processing_figure)
+        else:
+            nbytes_hist = NBytesHistogram(scheduler, sizing_mode="stretch_both")
+            nbytes_hist.update()
+            processing_hist = ProcessingHistogram(scheduler, sizing_mode="stretch_both")
+            processing_hist.update()
+            add_periodic_callback(doc, nbytes_hist, 100)
+            add_periodic_callback(doc, processing_hist, 100)
+            current_load_fig = row(
+                nbytes_hist.root, processing_hist.root, sizing_mode="stretch_both"
+            )
+
+            doc.add_root(nbytes_hist.root)
+            doc.add_root(processing_hist.root)
+
+        doc.title = "Dask: Status"
+        doc.add_root(task_progress.root)
+        doc.add_root(task_stream.root)
+        doc.theme = BOKEH_THEME
+        doc.template = env.get_template("status.html")
+        doc.template_variables.update(extra)
+        doc.theme = BOKEH_THEME
+
+
+def individual_task_stream_doc(scheduler, extra, doc):
+    task_stream = TaskStream(
+        scheduler, n_rectangles=1000, clear_interval="10s", sizing_mode="stretch_both"
+    )
+    task_stream.update()
+    add_periodic_callback(doc, task_stream, 100)
+    doc.add_root(task_stream.root)
+    doc.theme = BOKEH_THEME
+
+
+def individual_nbytes_doc(scheduler, extra, doc):
+    current_load = CurrentLoad(scheduler, sizing_mode="stretch_both")
+    current_load.update()
+    add_periodic_callback(doc, current_load, 100)
+    doc.add_root(current_load.nbytes_figure)
+    doc.theme = BOKEH_THEME
+
+
+def individual_memory_use_doc(scheduler, extra, doc):
+    memory_use = MemoryUse(scheduler, sizing_mode="stretch_both")
+    memory_use.update()
+    add_periodic_callback(doc, memory_use, 100)
+    doc.add_root(memory_use.root)
+    doc.theme = BOKEH_THEME
+
+
+def individual_cpu_doc(scheduler, extra, doc):
+    current_load = CurrentLoad(scheduler, sizing_mode="stretch_both")
+    current_load.update()
+    add_periodic_callback(doc, current_load, 100)
+    doc.add_root(current_load.cpu_figure)
+    doc.theme = BOKEH_THEME
+
+
+def individual_nprocessing_doc(scheduler, extra, doc):
+    current_load = CurrentLoad(scheduler, sizing_mode="stretch_both")
+    current_load.update()
+    add_periodic_callback(doc, current_load, 100)
+    doc.add_root(current_load.processing_figure)
+    doc.theme = BOKEH_THEME
+
+
+def individual_progress_doc(scheduler, extra, doc):
+    task_progress = TaskProgress(scheduler, height=160, sizing_mode="stretch_both")
+    task_progress.update()
+    add_periodic_callback(doc, task_progress, 100)
+    doc.add_root(task_progress.root)
+    doc.theme = BOKEH_THEME
+
+
+def individual_graph_doc(scheduler, extra, doc):
+    with log_errors():
+        graph = TaskGraph(scheduler, sizing_mode="stretch_both")
+        graph.update()
+
+        add_periodic_callback(doc, graph, 200)
+        doc.add_root(graph.root)
+        doc.theme = BOKEH_THEME
+
+
+def individual_profile_doc(scheduler, extra, doc):
+    with log_errors():
+        prof = ProfileTimePlot(scheduler, sizing_mode="scale_width", doc=doc)
+        doc.add_root(prof.root)
+        prof.trigger_update()
+        doc.theme = BOKEH_THEME
+
+
+def individual_profile_server_doc(scheduler, extra, doc):
+    with log_errors():
+        prof = ProfileServer(scheduler, sizing_mode="scale_width", doc=doc)
+        doc.add_root(prof.root)
+        prof.trigger_update()
+        doc.theme = BOKEH_THEME
+
+
+def individual_workers_doc(scheduler, extra, doc):
+    with log_errors():
+        table = WorkerTable(scheduler)
+        table.update()
+        add_periodic_callback(doc, table, 500)
+        doc.add_root(table.root)
+        doc.theme = BOKEH_THEME
+
+
+def individual_bandwidth_types_doc(scheduler, extra, doc):
+    with log_errors():
+        bw = BandwidthTypes(scheduler, sizing_mode="stretch_both")
+        bw.update()
+        add_periodic_callback(doc, bw, 500)
+        doc.add_root(bw.fig)
+        doc.theme = BOKEH_THEME
+
+
+def individual_bandwidth_workers_doc(scheduler, extra, doc):
+    with log_errors():
+        bw = BandwidthWorkers(scheduler, sizing_mode="stretch_both")
+        bw.update()
+        add_periodic_callback(doc, bw, 500)
+        doc.add_root(bw.fig)
+        doc.theme = BOKEH_THEME
+
+
+def profile_doc(scheduler, extra, doc):
+    with log_errors():
+        doc.title = "Dask: Profile"
+        prof = ProfileTimePlot(scheduler, sizing_mode="scale_width", doc=doc)
+        doc.add_root(prof.root)
+        doc.template = env.get_template("simple.html")
+        doc.template_variables.update(extra)
+        doc.theme = BOKEH_THEME
+
+        prof.trigger_update()
+
+
+def profile_server_doc(scheduler, extra, doc):
+    with log_errors():
+        doc.title = "Dask: Profile of Event Loop"
+        prof = ProfileServer(scheduler, sizing_mode="scale_width", doc=doc)
+        doc.add_root(prof.root)
+        doc.template = env.get_template("simple.html")
+        doc.template_variables.update(extra)
+        doc.theme = BOKEH_THEME
+
+        prof.trigger_update()
diff --git a/distributed/dashboard/components.py b/distributed/dashboard/components/shared.py
similarity index 75%
rename from distributed/dashboard/components.py
rename to distributed/dashboard/components/shared.py
index 7fb8a6cb022..882db411434 100644
--- a/distributed/dashboard/components.py
+++ b/distributed/dashboard/components/shared.py
@@ -1,7 +1,4 @@
 import asyncio
-from bisect import bisect
-from operator import add
-from time import time
 import weakref
 
 from bokeh.layouts import row, column
@@ -11,16 +8,11 @@
     DataRange1d,
     LinearAxis,
     HoverTool,
-    BoxZoomTool,
-    ResetTool,
-    PanTool,
-    WheelZoomTool,
     Range1d,
     Quad,
-    TapTool,
-    OpenURL,
     Button,
     Select,
+    NumeralTickFormatter,
 )
 from bokeh.palettes import Spectral9
 from bokeh.plotting import figure
@@ -28,13 +20,19 @@
 from tornado import gen
 import toolz
 
-from .utils import without_property_validation, BOKEH_VERSION
-from ..diagnostics.progress_stream import nbytes_bar
-from .. import profile
-from ..utils import log_errors, parse_timedelta
+from distributed.dashboard.components import DashboardComponent
+from distributed.dashboard.utils import (
+    without_property_validation,
+    BOKEH_VERSION,
+    update,
+)
+from distributed.diagnostics.progress_stream import nbytes_bar
+from distributed import profile
+from distributed.utils import log_errors, parse_timedelta
+from distributed.compatibility import WINDOWS
 
 if dask.config.get("distributed.dashboard.export-tool"):
-    from .export_tool import ExportTool
+    from distributed.dashboard.export_tool import ExportTool
 else:
     ExportTool = None
 
@@ -43,157 +41,6 @@
 profile_interval = parse_timedelta(profile_interval, default="ms")
 
 
-class DashboardComponent(object):
-    """ Base class for Dask.distributed UI dashboard components.
-
-    This class must have two attributes, ``root`` and ``source``, and one
-    method ``update``:
-
-    *  source: a Bokeh ColumnDataSource
-    *  root: a Bokeh Model
-    *  update: a method that consumes the messages dictionary found in
-               distributed.bokeh.messages
-    """
-
-    def __init__(self):
-        self.source = None
-        self.root = None
-
-    def update(self, messages):
-        """ Reads from bokeh.distributed.messages and updates self.source """
-
-
-class TaskStream(DashboardComponent):
-    """ Task Stream
-
-    The start and stop time of tasks as they occur on each core of the cluster.
-    """
-
-    def __init__(self, n_rectangles=1000, clear_interval="20s", **kwargs):
-        """
-        kwargs are applied to the bokeh.models.plots.Plot constructor
-        """
-        self.n_rectangles = n_rectangles
-        clear_interval = parse_timedelta(clear_interval, default="ms")
-        self.clear_interval = clear_interval
-        self.last = 0
-
-        self.source, self.root = task_stream_figure(clear_interval, **kwargs)
-
-        # Required for update callback
-        self.task_stream_index = [0]
-
-    @without_property_validation
-    def update(self, messages):
-        with log_errors():
-            index = messages["task-events"]["index"]
-            rectangles = messages["task-events"]["rectangles"]
-
-            if not index or index[-1] == self.task_stream_index[0]:
-                return
-
-            ind = bisect(index, self.task_stream_index[0])
-            rectangles = {
-                k: [v[i] for i in range(ind, len(index))] for k, v in rectangles.items()
-            }
-            self.task_stream_index[0] = index[-1]
-
-            # If there has been a significant delay then clear old rectangles
-            if rectangles["start"]:
-                m = min(map(add, rectangles["start"], rectangles["duration"]))
-                if m > self.last:
-                    self.last, last = m, self.last
-                    if m > last + self.clear_interval:
-                        self.source.data.update(rectangles)
-                        return
-
-            self.source.stream(rectangles, self.n_rectangles)
-
-
-def task_stream_figure(clear_interval="20s", **kwargs):
-    """
-    kwargs are applied to the bokeh.models.plots.Plot constructor
-    """
-    clear_interval = parse_timedelta(clear_interval, default="ms")
-
-    source = ColumnDataSource(
-        data=dict(
-            start=[time() - clear_interval],
-            duration=[0.1],
-            key=["start"],
-            name=["start"],
-            color=["white"],
-            duration_text=["100 ms"],
-            worker=["foo"],
-            y=[0],
-            worker_thread=[1],
-            alpha=[0.0],
-        )
-    )
-
-    x_range = DataRange1d(range_padding=0)
-    y_range = DataRange1d(range_padding=0)
-
-    root = figure(
-        name="task_stream",
-        title="Task Stream",
-        id="bk-task-stream-plot",
-        x_range=x_range,
-        y_range=y_range,
-        toolbar_location="above",
-        x_axis_type="datetime",
-        min_border_right=35,
-        tools="",
-        **kwargs
-    )
-
-    rect = root.rect(
-        source=source,
-        x="start",
-        y="y",
-        width="duration",
-        height=0.4,
-        fill_color="color",
-        line_color="color",
-        line_alpha=0.6,
-        fill_alpha="alpha",
-        line_width=3,
-    )
-    rect.nonselection_glyph = None
-
-    root.yaxis.major_label_text_alpha = 0
-    root.yaxis.minor_tick_line_alpha = 0
-    root.yaxis.major_tick_line_alpha = 0
-    root.xgrid.visible = False
-
-    hover = HoverTool(
-        point_policy="follow_mouse",
-        tooltips="""
-            <div>
-                <span style="font-size: 12px; font-weight: bold;">@name:</span>&nbsp;
-                <span style="font-size: 10px; font-family: Monaco, monospace;">@duration_text</span>
-            </div>
-            """,
-    )
-
-    tap = TapTool(callback=OpenURL(url="/profile?key=@name"))
-
-    root.add_tools(
-        hover,
-        tap,
-        BoxZoomTool(),
-        ResetTool(),
-        PanTool(dimensions="width"),
-        WheelZoomTool(dimensions="width"),
-    )
-    if ExportTool:
-        export = ExportTool()
-        export.register_plot(root)
-        root.add_tools(export)
-
-    return source, root
-
-
 class MemoryUsage(DashboardComponent):
     """ The memory usage across the cluster, grouped by task type """
 
@@ -261,7 +108,7 @@ def update(self, messages):
             if not msg:
                 return
             nb = nbytes_bar(msg["nbytes"])
-            self.source.data.update(nb)
+            update(self.source, nb)
             self.root.title.text = "Memory Use: %0.2f MB" % (
                 sum(msg["nbytes"].values()) / 1e6
             )
@@ -331,7 +178,7 @@ def update(self, messages):
             elif x_range.end > 2 * max_right + cores:  # way out there, walk back
                 x_range.end = x_range.end * 0.95 + max_right * 0.05
 
-            self.source.data.update(data)
+            update(self.source, data)
 
     @staticmethod
     def processing_update(msg):
@@ -383,7 +230,7 @@ def cb(attr, old, new):
                 data = profile.plot_data(self.states[ind], profile_interval)
                 del self.states[:]
                 self.states.extend(data.pop("states"))
-                self.source.data.update(data)
+                update(self.source, data)
                 self.source.selected = old
 
         if BOKEH_VERSION >= "1.0.0":
@@ -397,7 +244,7 @@ def update(self, state):
             self.state = state
             data = profile.plot_data(self.state, profile_interval)
             self.states = data.pop("states")
-            self.source.data.update(data)
+            update(self.source, data)
 
 
 class ProfileTimePlot(DashboardComponent):
@@ -450,7 +297,7 @@ def cb(attr, old, new):
                 del self.states[:]
                 self.states.extend(data.pop("states"))
                 changing[0] = True  # don't recursively trigger callback
-                self.source.data.update(data)
+                update(self.source, data)
                 if isinstance(new, list):  # bokeh >= 1.0
                     self.source.selected.indices = old
                 else:
@@ -532,7 +379,7 @@ def update(self, state, metadata=None):
             self.state = state
             data = profile.plot_data(self.state, profile_interval)
             self.states = data.pop("states")
-            self.source.data.update(data)
+            update(self.source, data)
 
             if metadata is not None and metadata["counts"]:
                 self.task_names = ["All"] + sorted(metadata["keys"])
@@ -602,7 +449,7 @@ def cb(attr, old, new):
                 del self.states[:]
                 self.states.extend(data.pop("states"))
                 changing[0] = True  # don't recursively trigger callback
-                self.source.data.update(data)
+                update(self.source, data)
                 if isinstance(new, list):  # bokeh >= 1.0
                     self.source.selected.indices = old
                 else:
@@ -669,44 +516,104 @@ def update(self, state):
             self.state = state
             data = profile.plot_data(self.state, profile_interval)
             self.states = data.pop("states")
-            self.source.data.update(data)
+            update(self.source, data)
 
     @without_property_validation
     def trigger_update(self):
         self.state = profile.get_profile(self.log, start=self.start, stop=self.stop)
         data = profile.plot_data(self.state, profile_interval)
         self.states = data.pop("states")
-        self.source.data.update(data)
+        update(self.source, data)
         times = [t * 1000 for t, _ in self.log]
         counts = list(toolz.pluck("count", toolz.pluck(1, self.log)))
         self.ts_source.data.update({"time": times, "count": counts})
 
 
-def add_periodic_callback(doc, component, interval):
-    """ Add periodic callback to doc in a way that avoids reference cycles
+class SystemMonitor(DashboardComponent):
+    def __init__(self, worker, height=150, **kwargs):
+        self.worker = worker
 
-    If we instead use ``doc.add_periodic_callback(component.update, 100)`` then
-    the component stays in memory as a reference cycle because its method is
-    still around.  This way we avoid that and let things clean up a bit more
-    nicely.
+        names = worker.monitor.quantities
+        self.last = 0
+        self.source = ColumnDataSource({name: [] for name in names})
+        update(self.source, self.get_data())
 
-    TODO: we still have reference cycles.  Docs seem to be referred to by their
-    add_periodic_callback methods.
-    """
-    ref = weakref.ref(component)
+        x_range = DataRange1d(follow="end", follow_interval=20000, range_padding=0)
 
-    doc.add_periodic_callback(lambda: update(ref), interval)
-    _attach(doc, component)
+        tools = "reset,xpan,xwheel_zoom"
 
+        self.cpu = figure(
+            title="CPU",
+            x_axis_type="datetime",
+            height=height,
+            tools=tools,
+            x_range=x_range,
+            **kwargs
+        )
+        self.cpu.line(source=self.source, x="time", y="cpu")
+        self.cpu.yaxis.axis_label = "Percentage"
+        self.mem = figure(
+            title="Memory",
+            x_axis_type="datetime",
+            height=height,
+            tools=tools,
+            x_range=x_range,
+            **kwargs
+        )
+        self.mem.line(source=self.source, x="time", y="memory")
+        self.mem.yaxis.axis_label = "Bytes"
+        self.bandwidth = figure(
+            title="Bandwidth",
+            x_axis_type="datetime",
+            height=height,
+            x_range=x_range,
+            tools=tools,
+            **kwargs
+        )
+        self.bandwidth.line(source=self.source, x="time", y="read_bytes", color="red")
+        self.bandwidth.line(source=self.source, x="time", y="write_bytes", color="blue")
+        self.bandwidth.yaxis.axis_label = "Bytes / second"
+
+        # self.cpu.yaxis[0].formatter = NumeralTickFormatter(format='0%')
+        self.bandwidth.yaxis[0].formatter = NumeralTickFormatter(format="0.0b")
+        self.mem.yaxis[0].formatter = NumeralTickFormatter(format="0.0b")
+
+        plots = [self.cpu, self.mem, self.bandwidth]
+
+        if not WINDOWS:
+            self.num_fds = figure(
+                title="Number of File Descriptors",
+                x_axis_type="datetime",
+                height=height,
+                x_range=x_range,
+                tools=tools,
+                **kwargs
+            )
+
+            self.num_fds.line(source=self.source, x="time", y="num_fds")
+            plots.append(self.num_fds)
+
+        if "sizing_mode" in kwargs:
+            kw = {"sizing_mode": kwargs["sizing_mode"]}
+        else:
+            kw = {}
 
-def update(ref):
-    comp = ref()
-    if comp is not None:
-        comp.update()
+        if not WINDOWS:
+            self.num_fds.y_range.start = 0
+        self.mem.y_range.start = 0
+        self.cpu.y_range.start = 0
+        self.bandwidth.y_range.start = 0
 
+        self.root = column(*plots, **kw)
+        self.worker.monitor.update()
 
-def _attach(doc, component):
-    if not hasattr(doc, "components"):
-        doc.components = set()
+    def get_data(self):
+        d = self.worker.monitor.range_query(start=self.last)
+        d["time"] = [x * 1000 for x in d["time"]]
+        self.last = self.worker.monitor.count
+        return d
 
-    doc.components.add(component)
+    @without_property_validation
+    def update(self):
+        with log_errors():
+            self.source.stream(self.get_data(), 1000)
diff --git a/distributed/dashboard/components/worker.py b/distributed/dashboard/components/worker.py
new file mode 100644
index 00000000000..9dc2b2ec82f
--- /dev/null
+++ b/distributed/dashboard/components/worker.py
@@ -0,0 +1,661 @@
+import logging
+import math
+import os
+
+from bokeh.layouts import row, column, widgetbox
+from bokeh.models import (
+    ColumnDataSource,
+    DataRange1d,
+    HoverTool,
+    BoxZoomTool,
+    ResetTool,
+    PanTool,
+    WheelZoomTool,
+    NumeralTickFormatter,
+    Select,
+)
+
+from bokeh.models.widgets import DataTable, TableColumn
+from bokeh.plotting import figure
+from bokeh.palettes import RdBu
+from bokeh.themes import Theme
+from dask.utils import format_bytes
+from toolz import merge, partition_all
+
+from distributed.dashboard.components import add_periodic_callback
+from distributed.dashboard.components.shared import (
+    DashboardComponent,
+    ProfileTimePlot,
+    ProfileServer,
+    SystemMonitor,
+)
+from distributed.dashboard.utils import transpose, without_property_validation, update
+from distributed.diagnostics.progress_stream import color_of
+from distributed.metrics import time
+from distributed.utils import log_errors, key_split, format_time
+
+
+logger = logging.getLogger(__name__)
+
+with open(os.path.join(os.path.dirname(__file__), "..", "templates", "base.html")) as f:
+    template_source = f.read()
+
+from jinja2 import Environment, FileSystemLoader
+
+env = Environment(
+    loader=FileSystemLoader(os.path.join(os.path.dirname(__file__), "..", "templates"))
+)
+
+BOKEH_THEME = Theme(os.path.join(os.path.dirname(__file__), "..", "theme.yaml"))
+
+template_variables = {"pages": ["status", "system", "profile", "crossfilter"]}
+
+
+class StateTable(DashboardComponent):
+    """ Currently running tasks """
+
+    def __init__(self, worker):
+        self.worker = worker
+
+        names = ["Stored", "Executing", "Ready", "Waiting", "Connections", "Serving"]
+        self.source = ColumnDataSource({name: [] for name in names})
+
+        columns = {name: TableColumn(field=name, title=name) for name in names}
+
+        table = DataTable(
+            source=self.source, columns=[columns[n] for n in names], height=70
+        )
+        self.root = table
+
+    @without_property_validation
+    def update(self):
+        with log_errors():
+            w = self.worker
+            d = {
+                "Stored": [len(w.data)],
+                "Executing": ["%d / %d" % (len(w.executing), w.nthreads)],
+                "Ready": [len(w.ready)],
+                "Waiting": [len(w.waiting_for_data)],
+                "Connections": [len(w.in_flight_workers)],
+                "Serving": [len(w._comms)],
+            }
+            update(self.source, d)
+
+
+class CommunicatingStream(DashboardComponent):
+    def __init__(self, worker, height=300, **kwargs):
+        with log_errors():
+            self.worker = worker
+            names = [
+                "start",
+                "stop",
+                "middle",
+                "duration",
+                "who",
+                "y",
+                "hover",
+                "alpha",
+                "bandwidth",
+                "total",
+            ]
+
+            self.incoming = ColumnDataSource({name: [] for name in names})
+            self.outgoing = ColumnDataSource({name: [] for name in names})
+
+            x_range = DataRange1d(range_padding=0)
+            y_range = DataRange1d(range_padding=0)
+
+            fig = figure(
+                title="Peer Communications",
+                x_axis_type="datetime",
+                x_range=x_range,
+                y_range=y_range,
+                height=height,
+                tools="",
+                **kwargs
+            )
+
+            fig.rect(
+                source=self.incoming,
+                x="middle",
+                y="y",
+                width="duration",
+                height=0.9,
+                color="red",
+                alpha="alpha",
+            )
+            fig.rect(
+                source=self.outgoing,
+                x="middle",
+                y="y",
+                width="duration",
+                height=0.9,
+                color="blue",
+                alpha="alpha",
+            )
+
+            hover = HoverTool(point_policy="follow_mouse", tooltips="""@hover""")
+            fig.add_tools(
+                hover,
+                ResetTool(),
+                PanTool(dimensions="width"),
+                WheelZoomTool(dimensions="width"),
+            )
+
+            self.root = fig
+
+            self.last_incoming = 0
+            self.last_outgoing = 0
+            self.who = dict()
+
+    @without_property_validation
+    def update(self):
+        with log_errors():
+            outgoing = self.worker.outgoing_transfer_log
+            n = self.worker.outgoing_count - self.last_outgoing
+            outgoing = [outgoing[-i].copy() for i in range(1, n + 1)]
+            self.last_outgoing = self.worker.outgoing_count
+
+            incoming = self.worker.incoming_transfer_log
+            n = self.worker.incoming_count - self.last_incoming
+            incoming = [incoming[-i].copy() for i in range(1, n + 1)]
+            self.last_incoming = self.worker.incoming_count
+
+            for [msgs, source] in [
+                [incoming, self.incoming],
+                [outgoing, self.outgoing],
+            ]:
+
+                for msg in msgs:
+                    if "compressed" in msg:
+                        del msg["compressed"]
+                    del msg["keys"]
+
+                    bandwidth = msg["total"] / (msg["duration"] or 0.5)
+                    bw = max(min(bandwidth / 500e6, 1), 0.3)
+                    msg["alpha"] = bw
+                    try:
+                        msg["y"] = self.who[msg["who"]]
+                    except KeyError:
+                        self.who[msg["who"]] = len(self.who)
+                        msg["y"] = self.who[msg["who"]]
+
+                    msg["hover"] = "%s / %s = %s/s" % (
+                        format_bytes(msg["total"]),
+                        format_time(msg["duration"]),
+                        format_bytes(msg["total"] / msg["duration"]),
+                    )
+
+                    for k in ["middle", "duration", "start", "stop"]:
+                        msg[k] = msg[k] * 1000
+
+                if msgs:
+                    msgs = transpose(msgs)
+                    if (
+                        len(source.data["stop"])
+                        and min(msgs["start"]) > source.data["stop"][-1] + 10000
+                    ):
+                        source.data.update(msgs)
+                    else:
+                        source.stream(msgs, rollover=10000)
+
+
+class CommunicatingTimeSeries(DashboardComponent):
+    def __init__(self, worker, **kwargs):
+        self.worker = worker
+        self.source = ColumnDataSource({"x": [], "in": [], "out": []})
+
+        x_range = DataRange1d(follow="end", follow_interval=20000, range_padding=0)
+
+        fig = figure(
+            title="Communication History",
+            x_axis_type="datetime",
+            y_range=[-0.1, worker.total_out_connections + 0.5],
+            height=150,
+            tools="",
+            x_range=x_range,
+            **kwargs
+        )
+        fig.line(source=self.source, x="x", y="in", color="red")
+        fig.line(source=self.source, x="x", y="out", color="blue")
+
+        fig.add_tools(
+            ResetTool(), PanTool(dimensions="width"), WheelZoomTool(dimensions="width")
+        )
+
+        self.root = fig
+
+    @without_property_validation
+    def update(self):
+        with log_errors():
+            self.source.stream(
+                {
+                    "x": [time() * 1000],
+                    "out": [len(self.worker._comms)],
+                    "in": [len(self.worker.in_flight_workers)],
+                },
+                10000,
+            )
+
+
+class ExecutingTimeSeries(DashboardComponent):
+    def __init__(self, worker, **kwargs):
+        self.worker = worker
+        self.source = ColumnDataSource({"x": [], "y": []})
+
+        x_range = DataRange1d(follow="end", follow_interval=20000, range_padding=0)
+
+        fig = figure(
+            title="Executing History",
+            x_axis_type="datetime",
+            y_range=[-0.1, worker.nthreads + 0.1],
+            height=150,
+            tools="",
+            x_range=x_range,
+            **kwargs
+        )
+        fig.line(source=self.source, x="x", y="y")
+
+        fig.add_tools(
+            ResetTool(), PanTool(dimensions="width"), WheelZoomTool(dimensions="width")
+        )
+
+        self.root = fig
+
+    @without_property_validation
+    def update(self):
+        with log_errors():
+            self.source.stream(
+                {"x": [time() * 1000], "y": [len(self.worker.executing)]}, 1000
+            )
+
+
+class CrossFilter(DashboardComponent):
+    def __init__(self, worker, **kwargs):
+        with log_errors():
+            self.worker = worker
+
+            quantities = ["nbytes", "duration", "bandwidth", "count", "start", "stop"]
+            colors = ["inout-color", "type-color", "key-color"]
+
+            # self.source = ColumnDataSource({name: [] for name in names})
+            self.source = ColumnDataSource(
+                {
+                    "nbytes": [1, 2],
+                    "duration": [0.01, 0.02],
+                    "bandwidth": [0.01, 0.02],
+                    "count": [1, 2],
+                    "type": ["int", "str"],
+                    "inout-color": ["blue", "red"],
+                    "type-color": ["blue", "red"],
+                    "key": ["add", "inc"],
+                    "start": [1, 2],
+                    "stop": [1, 2],
+                }
+            )
+
+            self.x = Select(title="X-Axis", value="nbytes", options=quantities)
+            self.x.on_change("value", self.update_figure)
+
+            self.y = Select(title="Y-Axis", value="bandwidth", options=quantities)
+            self.y.on_change("value", self.update_figure)
+
+            self.size = Select(
+                title="Size", value="None", options=["None"] + quantities
+            )
+            self.size.on_change("value", self.update_figure)
+
+            self.color = Select(
+                title="Color", value="inout-color", options=["black"] + colors
+            )
+            self.color.on_change("value", self.update_figure)
+
+            if "sizing_mode" in kwargs:
+                kw = {"sizing_mode": kwargs["sizing_mode"]}
+            else:
+                kw = {}
+
+            self.control = widgetbox(
+                [self.x, self.y, self.size, self.color], width=200, **kw
+            )
+
+            self.last_outgoing = 0
+            self.last_incoming = 0
+            self.kwargs = kwargs
+
+            self.layout = row(self.control, self.create_figure(**self.kwargs), **kw)
+
+            self.root = self.layout
+
+    @without_property_validation
+    def update(self):
+        with log_errors():
+            outgoing = self.worker.outgoing_transfer_log
+            n = self.worker.outgoing_count - self.last_outgoing
+            n = min(n, 1000)
+            outgoing = [outgoing[-i].copy() for i in range(1, n)]
+            self.last_outgoing = self.worker.outgoing_count
+
+            incoming = self.worker.incoming_transfer_log
+            n = self.worker.incoming_count - self.last_incoming
+            n = min(n, 1000)
+            incoming = [incoming[-i].copy() for i in range(1, n)]
+            self.last_incoming = self.worker.incoming_count
+
+            out = []
+
+            for msg in incoming:
+                if msg["keys"]:
+                    d = self.process_msg(msg)
+                    d["inout-color"] = "red"
+                    out.append(d)
+
+            for msg in outgoing:
+                if msg["keys"]:
+                    d = self.process_msg(msg)
+                    d["inout-color"] = "blue"
+                    out.append(d)
+
+            if out:
+                out = transpose(out)
+                if (
+                    len(self.source.data["stop"])
+                    and min(out["start"]) > self.source.data["stop"][-1] + 10
+                ):
+                    update(self.source, out)
+                else:
+                    self.source.stream(out, rollover=1000)
+
+    def create_figure(self, **kwargs):
+        with log_errors():
+            fig = figure(title="", tools="", **kwargs)
+
+            size = self.size.value
+            if size == "None":
+                size = 1
+
+            fig.circle(
+                source=self.source,
+                x=self.x.value,
+                y=self.y.value,
+                color=self.color.value,
+                size=10,
+                alpha=0.5,
+                hover_alpha=1,
+            )
+            fig.xaxis.axis_label = self.x.value
+            fig.yaxis.axis_label = self.y.value
+
+            fig.add_tools(
+                # self.hover,
+                ResetTool(),
+                PanTool(),
+                WheelZoomTool(),
+                BoxZoomTool(),
+            )
+            return fig
+
+    @without_property_validation
+    def update_figure(self, attr, old, new):
+        with log_errors():
+            fig = self.create_figure(**self.kwargs)
+            self.layout.children[1] = fig
+
+    def process_msg(self, msg):
+        try:
+
+            def func(k):
+                return msg["keys"].get(k, 0)
+
+            status_key = max(msg["keys"], key=func)
+            typ = self.worker.types.get(status_key, object).__name__
+            keyname = key_split(status_key)
+            d = {
+                "nbytes": msg["total"],
+                "duration": msg["duration"],
+                "bandwidth": msg["bandwidth"],
+                "count": len(msg["keys"]),
+                "type": typ,
+                "type-color": color_of(typ),
+                "key": keyname,
+                "key-color": color_of(keyname),
+                "start": msg["start"],
+                "stop": msg["stop"],
+            }
+            return d
+        except Exception as e:
+            logger.exception(e)
+            raise
+
+
+class Counters(DashboardComponent):
+    def __init__(self, server, sizing_mode="stretch_both", **kwargs):
+        self.server = server
+        self.counter_figures = {}
+        self.counter_sources = {}
+        self.digest_figures = {}
+        self.digest_sources = {}
+        self.sizing_mode = sizing_mode
+
+        if self.server.digests:
+            for name in self.server.digests:
+                self.add_digest_figure(name)
+        for name in self.server.counters:
+            self.add_counter_figure(name)
+
+        figures = merge(self.digest_figures, self.counter_figures)
+        figures = [figures[k] for k in sorted(figures)]
+
+        if len(figures) <= 5:
+            self.root = column(figures, sizing_mode=sizing_mode)
+        else:
+            self.root = column(
+                *[
+                    row(*pair, sizing_mode=sizing_mode)
+                    for pair in partition_all(2, figures)
+                ],
+                sizing_mode=sizing_mode
+            )
+
+    def add_digest_figure(self, name):
+        with log_errors():
+            n = len(self.server.digests[name].intervals)
+            sources = {i: ColumnDataSource({"x": [], "y": []}) for i in range(n)}
+
+            kwargs = {}
+            if name.endswith("duration"):
+                kwargs["x_axis_type"] = "datetime"
+
+            fig = figure(
+                title=name, tools="", height=150, sizing_mode=self.sizing_mode, **kwargs
+            )
+            fig.yaxis.visible = False
+            fig.ygrid.visible = False
+            if name.endswith("bandwidth") or name.endswith("bytes"):
+                fig.xaxis[0].formatter = NumeralTickFormatter(format="0.0b")
+
+            for i in range(n):
+                alpha = 0.3 + 0.3 * (n - i) / n
+                fig.line(
+                    source=sources[i],
+                    x="x",
+                    y="y",
+                    alpha=alpha,
+                    color=RdBu[max(n, 3)][-i],
+                )
+
+            fig.xaxis.major_label_orientation = math.pi / 12
+            fig.toolbar.logo = None
+            self.digest_sources[name] = sources
+            self.digest_figures[name] = fig
+            return fig
+
+    def add_counter_figure(self, name):
+        with log_errors():
+            n = len(self.server.counters[name].intervals)
+            sources = {
+                i: ColumnDataSource({"x": [], "y": [], "y-center": [], "counts": []})
+                for i in range(n)
+            }
+
+            fig = figure(
+                title=name,
+                tools="",
+                height=150,
+                sizing_mode=self.sizing_mode,
+                x_range=sorted(map(str, self.server.counters[name].components[0])),
+            )
+            fig.ygrid.visible = False
+
+            for i in range(n):
+                width = 0.5 + 0.4 * i / n
+                fig.rect(
+                    source=sources[i],
+                    x="x",
+                    y="y-center",
+                    width=width,
+                    height="y",
+                    alpha=0.3,
+                    color=RdBu[max(n, 3)][-i],
+                )
+                hover = HoverTool(
+                    point_policy="follow_mouse", tooltips="""@x : @counts"""
+                )
+                fig.add_tools(hover)
+                fig.xaxis.major_label_orientation = math.pi / 12
+
+            fig.toolbar.logo = None
+
+            self.counter_sources[name] = sources
+            self.counter_figures[name] = fig
+            return fig
+
+    @without_property_validation
+    def update(self):
+        with log_errors():
+            for name, fig in self.digest_figures.items():
+                digest = self.server.digests[name]
+                d = {}
+                for i, d in enumerate(digest.components):
+                    if d.size():
+                        ys, xs = d.histogram(100)
+                        xs = xs[1:]
+                        if name.endswith("duration"):
+                            xs *= 1000
+                        self.digest_sources[name][i].data.update({"x": xs, "y": ys})
+                fig.title.text = "%s: %d" % (name, digest.size())
+
+            for name, fig in self.counter_figures.items():
+                counter = self.server.counters[name]
+                d = {}
+                for i, d in enumerate(counter.components):
+                    if d:
+                        xs = sorted(d)
+                        factor = counter.intervals[0] / counter.intervals[i]
+                        counts = [d[x] for x in xs]
+                        ys = [factor * c for c in counts]
+                        y_centers = [y / 2 for y in ys]
+                        xs = list(map(str, xs))
+                        d = {"x": xs, "y": ys, "y-center": y_centers, "counts": counts}
+                        self.counter_sources[name][i].data.update(d)
+                    fig.title.text = "%s: %d" % (name, counter.size())
+                    fig.x_range.factors = list(map(str, xs))
+
+
+def status_doc(worker, extra, doc):
+    with log_errors():
+        statetable = StateTable(worker)
+        executing_ts = ExecutingTimeSeries(worker, sizing_mode="scale_width")
+        communicating_ts = CommunicatingTimeSeries(worker, sizing_mode="scale_width")
+        communicating_stream = CommunicatingStream(worker, sizing_mode="scale_width")
+
+        xr = executing_ts.root.x_range
+        communicating_ts.root.x_range = xr
+        communicating_stream.root.x_range = xr
+
+        doc.title = "Dask Worker Internal Monitor"
+        add_periodic_callback(doc, statetable, 200)
+        add_periodic_callback(doc, executing_ts, 200)
+        add_periodic_callback(doc, communicating_ts, 200)
+        add_periodic_callback(doc, communicating_stream, 200)
+        doc.add_root(
+            column(
+                statetable.root,
+                executing_ts.root,
+                communicating_ts.root,
+                communicating_stream.root,
+                sizing_mode="scale_width",
+            )
+        )
+        doc.template = env.get_template("simple.html")
+        doc.template_variables["active_page"] = "status"
+        doc.template_variables.update(extra)
+        doc.theme = BOKEH_THEME
+
+
+def crossfilter_doc(worker, extra, doc):
+    with log_errors():
+        statetable = StateTable(worker)
+        crossfilter = CrossFilter(worker)
+
+        doc.title = "Dask Worker Cross-filter"
+        add_periodic_callback(doc, statetable, 500)
+        add_periodic_callback(doc, crossfilter, 500)
+
+        doc.add_root(column(statetable.root, crossfilter.root))
+        doc.template = env.get_template("simple.html")
+        doc.template_variables["active_page"] = "crossfilter"
+        doc.template_variables.update(extra)
+        doc.theme = BOKEH_THEME
+
+
+def systemmonitor_doc(worker, extra, doc):
+    with log_errors():
+        sysmon = SystemMonitor(worker, sizing_mode="scale_width")
+        doc.title = "Dask Worker Monitor"
+        add_periodic_callback(doc, sysmon, 500)
+
+        doc.add_root(sysmon.root)
+        doc.template = env.get_template("simple.html")
+        doc.template_variables["active_page"] = "system"
+        doc.template_variables.update(extra)
+        doc.theme = BOKEH_THEME
+
+
+def counters_doc(server, extra, doc):
+    with log_errors():
+        doc.title = "Dask Worker Counters"
+        counter = Counters(server, sizing_mode="stretch_both")
+        add_periodic_callback(doc, counter, 500)
+
+        doc.add_root(counter.root)
+        doc.template = env.get_template("simple.html")
+        doc.template_variables["active_page"] = "counters"
+        doc.template_variables.update(extra)
+        doc.theme = BOKEH_THEME
+
+
+def profile_doc(server, extra, doc):
+    with log_errors():
+        doc.title = "Dask Worker Profile"
+        profile = ProfileTimePlot(server, sizing_mode="scale_width", doc=doc)
+        profile.trigger_update()
+
+        doc.add_root(profile.root)
+        doc.template = env.get_template("simple.html")
+        doc.template_variables["active_page"] = "profile"
+        doc.template_variables.update(extra)
+        doc.theme = BOKEH_THEME
+
+
+def profile_server_doc(server, extra, doc):
+    with log_errors():
+        doc.title = "Dask: Profile of Event Loop"
+        prof = ProfileServer(server, sizing_mode="scale_width", doc=doc)
+        doc.add_root(prof.root)
+        doc.template = env.get_template("simple.html")
+        # doc.template_variables['active_page'] = ''
+        doc.template_variables.update(extra)
+        doc.theme = BOKEH_THEME
+
+        prof.trigger_update()
diff --git a/distributed/dashboard/scheduler.py b/distributed/dashboard/scheduler.py
index 79484cd4196..1117fe7bd72 100644
--- a/distributed/dashboard/scheduler.py
+++ b/distributed/dashboard/scheduler.py
@@ -1,43 +1,11 @@
+from datetime import datetime
 from functools import partial
 import logging
-import math
-from numbers import Number
-from operator import add
-import os
-
-from bokeh.layouts import column, row
-from bokeh.models import (
-    ColumnDataSource,
-    ColorBar,
-    DataRange1d,
-    HoverTool,
-    ResetTool,
-    PanTool,
-    WheelZoomTool,
-    TapTool,
-    OpenURL,
-    Range1d,
-    Plot,
-    Quad,
-    Span,
-    value,
-    LinearAxis,
-    NumeralTickFormatter,
-    BasicTicker,
-    NumberFormatter,
-    BoxSelectTool,
-    GroupFilter,
-    CDSView,
-)
-from bokeh.models.widgets import DataTable, TableColumn
-from bokeh.plotting import figure
-from bokeh.palettes import Viridis11
-from bokeh.themes import Theme
-from bokeh.transform import factor_cmap, linear_cmap
-from bokeh.io import curdoc
+
 import dask
 from dask.utils import format_bytes
-from toolz import pipe, merge
+import toolz
+from toolz import merge
 from tornado import escape
 
 try:
@@ -45,1763 +13,303 @@
 except ImportError:
     np = False
 
-from . import components
-from .components import (
-    DashboardComponent,
-    ProfileTimePlot,
-    ProfileServer,
-    add_periodic_callback,
+from .components.worker import counters_doc
+from .components.scheduler import (
+    systemmonitor_doc,
+    stealing_doc,
+    workers_doc,
+    events_doc,
+    tasks_doc,
+    status_doc,
+    profile_doc,
+    profile_server_doc,
+    graph_doc,
+    individual_task_stream_doc,
+    individual_progress_doc,
+    individual_graph_doc,
+    individual_profile_doc,
+    individual_profile_server_doc,
+    individual_nbytes_doc,
+    individual_memory_use_doc,
+    individual_cpu_doc,
+    individual_nprocessing_doc,
+    individual_workers_doc,
+    individual_bandwidth_types_doc,
+    individual_bandwidth_workers_doc,
 )
 from .core import BokehServer
-from .worker import SystemMonitor, counters_doc
-from .utils import transpose, BOKEH_VERSION, without_property_validation
-from ..metrics import time
+from .worker import counters_doc
+from .proxy import GlobalProxyHandler
+from .utils import RequestHandler, redirect
 from ..utils import log_errors, format_time
-from ..diagnostics.progress_stream import color_of, progress_quads, nbytes_bar
-from ..diagnostics.progress import AllProgress
-from ..diagnostics.graph_layout import GraphLayout
-from ..diagnostics.task_stream import TaskStreamPlugin
-
-try:
-    from cytoolz.curried import map, concat, groupby, valmap, first
-except ImportError:
-    from toolz.curried import map, concat, groupby, valmap, first
 
-logger = logging.getLogger(__name__)
 
+ns = {
+    func.__name__: func for func in [format_bytes, format_time, datetime.fromtimestamp]
+}
 
-PROFILING = False
+rel_path_statics = {"rel_path_statics": "../../"}
 
-from jinja2 import Environment, FileSystemLoader
 
-env = Environment(
-    loader=FileSystemLoader(os.path.join(os.path.dirname(__file__), "templates"))
-)
+logger = logging.getLogger(__name__)
 
 template_variables = {
     "pages": ["status", "workers", "tasks", "system", "profile", "graph", "info"]
 }
 
-BOKEH_THEME = Theme(os.path.join(os.path.dirname(__file__), "theme.yaml"))
-
-nan = float("nan")
-inf = float("inf")
-
-
-@without_property_validation
-def update(source, data):
-    """ Update source with data
-
-    This checks a few things first
-
-    1.  If the data is the same, then don't update
-    2.  If numpy is available and the data is numeric, then convert to numpy
-        arrays
-    3.  If profiling then perform the update in another callback
-    """
-    if not np or not any(isinstance(v, np.ndarray) for v in source.data.values()):
-        if source.data == data:
-            return
-    if np and len(data[first(data)]) > 10:
-        d = {}
-        for k, v in data.items():
-            if type(v) is not np.ndarray and isinstance(v[0], Number):
-                d[k] = np.array(v)
-            else:
-                d[k] = v
-    else:
-        d = data
-
-    if PROFILING:
-        curdoc().add_next_tick_callback(lambda: source.data.update(d))
-    else:
-        source.data.update(d)
-
-
-class Occupancy(DashboardComponent):
-    """ Occupancy (in time) per worker """
 
-    def __init__(self, scheduler, **kwargs):
+class Workers(RequestHandler):
+    def get(self):
         with log_errors():
-            self.scheduler = scheduler
-            self.source = ColumnDataSource(
-                {
-                    "occupancy": [0, 0],
-                    "worker": ["a", "b"],
-                    "x": [0.0, 0.1],
-                    "y": [1, 2],
-                    "ms": [1, 2],
-                    "color": ["red", "blue"],
-                    "escaped_worker": ["a", "b"],
-                }
+            self.render(
+                "workers.html",
+                title="Workers",
+                scheduler=self.server,
+                **toolz.merge(self.server.__dict__, ns, self.extra, rel_path_statics),
             )
 
-            fig = figure(
-                title="Occupancy",
-                tools="",
-                id="bk-occupancy-plot",
-                x_axis_type="datetime",
-                **kwargs
-            )
-            rect = fig.rect(
-                source=self.source, x="x", width="ms", y="y", height=1, color="color"
-            )
-            rect.nonselection_glyph = None
-
-            fig.xaxis.minor_tick_line_alpha = 0
-            fig.yaxis.visible = False
-            fig.ygrid.visible = False
-            # fig.xaxis[0].formatter = NumeralTickFormatter(format='0.0s')
-            fig.x_range.start = 0
-
-            tap = TapTool(callback=OpenURL(url="./info/worker/@escaped_worker.html"))
-
-            hover = HoverTool()
-            hover.tooltips = "@worker : @occupancy s."
-            hover.point_policy = "follow_mouse"
-            fig.add_tools(hover, tap)
-
-            self.root = fig
-
-    @without_property_validation
-    def update(self):
-        with log_errors():
-            workers = list(self.scheduler.workers.values())
-
-            y = list(range(len(workers)))
-            occupancy = [ws.occupancy for ws in workers]
-            ms = [occ * 1000 for occ in occupancy]
-            x = [occ / 500 for occ in occupancy]
-            total = sum(occupancy)
-            color = []
-            for ws in workers:
-                if ws in self.scheduler.idle:
-                    color.append("red")
-                elif ws in self.scheduler.saturated:
-                    color.append("green")
-                else:
-                    color.append("blue")
-
-            if total:
-                self.root.title.text = "Occupancy -- total time: %s  wall time: %s" % (
-                    format_time(total),
-                    format_time(total / self.scheduler.total_nthreads),
-                )
-            else:
-                self.root.title.text = "Occupancy"
-
-            if occupancy:
-                result = {
-                    "occupancy": occupancy,
-                    "worker": [ws.address for ws in workers],
-                    "ms": ms,
-                    "color": color,
-                    "escaped_worker": [escape.url_escape(ws.address) for ws in workers],
-                    "x": x,
-                    "y": y,
-                }
 
-                update(self.source, result)
-
-
-class ProcessingHistogram(DashboardComponent):
-    """ How many tasks are on each worker """
-
-    def __init__(self, scheduler, **kwargs):
+class Worker(RequestHandler):
+    def get(self, worker):
+        worker = escape.url_unescape(worker)
+        if worker not in self.server.workers:
+            self.send_error(404)
+            return
         with log_errors():
-            self.last = 0
-            self.scheduler = scheduler
-            self.source = ColumnDataSource(
-                {"left": [1, 2], "right": [10, 10], "top": [0, 0]}
+            self.render(
+                "worker.html",
+                title="Worker: " + worker,
+                scheduler=self.server,
+                Worker=worker,
+                **toolz.merge(self.server.__dict__, ns, self.extra, rel_path_statics),
             )
 
-            self.root = figure(
-                title="Tasks Processing (Histogram)",
-                id="bk-nprocessing-histogram-plot",
-                name="processing_hist",
-                y_axis_label="frequency",
-                tools="",
-                **kwargs
-            )
-
-            self.root.xaxis.minor_tick_line_alpha = 0
-            self.root.ygrid.visible = False
 
-            self.root.toolbar.logo = None
-            self.root.toolbar_location = None
-
-            self.root.quad(
-                source=self.source,
-                left="left",
-                right="right",
-                bottom=0,
-                top="top",
-                color="deepskyblue",
-                fill_alpha=0.5,
-            )
-
-    @without_property_validation
-    def update(self):
-        L = [len(ws.processing) for ws in self.scheduler.workers.values()]
-        counts, x = np.histogram(L, bins=40)
-        self.source.data.update({"left": x[:-1], "right": x[1:], "top": counts})
-
-
-class NBytesHistogram(DashboardComponent):
-    """ How many tasks are on each worker """
-
-    def __init__(self, scheduler, **kwargs):
+class Task(RequestHandler):
+    def get(self, task):
+        task = escape.url_unescape(task)
+        if task not in self.server.tasks:
+            self.send_error(404)
+            return
         with log_errors():
-            self.last = 0
-            self.scheduler = scheduler
-            self.source = ColumnDataSource(
-                {"left": [1, 2], "right": [10, 10], "top": [0, 0]}
-            )
-
-            self.root = figure(
-                title="Bytes Stored (Histogram)",
-                name="nbytes_hist",
-                id="bk-nbytes-histogram-plot",
-                y_axis_label="frequency",
-                tools="",
-                **kwargs
+            self.render(
+                "task.html",
+                title="Task: " + task,
+                Task=task,
+                scheduler=self.server,
+                **toolz.merge(self.server.__dict__, ns, self.extra, rel_path_statics),
             )
 
-            self.root.xaxis[0].formatter = NumeralTickFormatter(format="0.0 b")
-            self.root.xaxis.major_label_orientation = -math.pi / 12
 
-            self.root.xaxis.minor_tick_line_alpha = 0
-            self.root.ygrid.visible = False
-
-            self.root.toolbar.logo = None
-            self.root.toolbar_location = None
-
-            self.root.quad(
-                source=self.source,
-                left="left",
-                right="right",
-                bottom=0,
-                top="top",
-                color="deepskyblue",
-                fill_alpha=0.5,
-            )
-
-    @without_property_validation
-    def update(self):
-        nbytes = np.asarray([ws.nbytes for ws in self.scheduler.workers.values()])
-        counts, x = np.histogram(nbytes, bins=40)
-        d = {"left": x[:-1], "right": x[1:], "top": counts}
-        self.source.data.update(d)
-
-        self.root.title.text = "Bytes stored (Histogram): " + format_bytes(nbytes.sum())
-
-
-class BandwidthTypes(DashboardComponent):
-    """ Bar chart showing bandwidth per type """
-
-    def __init__(self, scheduler, **kwargs):
+class Logs(RequestHandler):
+    def get(self):
         with log_errors():
-            self.last = 0
-            self.scheduler = scheduler
-            self.source = ColumnDataSource(
-                {
-                    "bandwidth": [1, 2],
-                    "bandwidth-half": [0.5, 1],
-                    "type": ["a", "b"],
-                    "bandwidth_text": ["1", "2"],
-                }
+            logs = self.server.get_logs()
+            self.render(
+                "logs.html",
+                title="Logs",
+                logs=logs,
+                **toolz.merge(self.extra, rel_path_statics),
             )
 
-            fig = figure(
-                title="Bandwidth by Type",
-                tools="",
-                id="bk-bandwidth-type-plot",
-                name="bandwidth_type_histogram",
-                y_range=["a", "b"],
-                **kwargs
-            )
-            rect = fig.rect(
-                source=self.source,
-                x="bandwidth-half",
-                y="type",
-                width="bandwidth",
-                height=1,
-                color="blue",
-            )
-            fig.x_range.start = 0
-            fig.xaxis[0].formatter = NumeralTickFormatter(format="0.0 b")
-            rect.nonselection_glyph = None
-
-            fig.xaxis.minor_tick_line_alpha = 0
-            fig.ygrid.visible = False
-
-            fig.toolbar.logo = None
-            fig.toolbar_location = None
 
-            hover = HoverTool()
-            hover.tooltips = "@type: @bandwidth_text / s"
-            hover.point_policy = "follow_mouse"
-            fig.add_tools(hover)
-
-            self.fig = fig
-
-    @without_property_validation
-    def update(self):
+class WorkerLogs(RequestHandler):
+    async def get(self, worker):
         with log_errors():
-            bw = self.scheduler.bandwidth_types
-            self.fig.y_range.factors = list(sorted(bw))
-            result = {
-                "bandwidth": list(bw.values()),
-                "bandwidth-half": [b / 2 for b in bw.values()],
-                "type": list(bw.keys()),
-                "bandwidth_text": list(map(format_bytes, bw.values())),
-            }
-            self.fig.title.text = "Bandwidth: " + format_bytes(self.scheduler.bandwidth)
-
-            update(self.source, result)
-
-
-class BandwidthWorkers(DashboardComponent):
-    """ How many tasks are on each worker """
-
-    def __init__(self, scheduler, **kwargs):
-        with log_errors():
-            self.last = 0
-            self.scheduler = scheduler
-            self.source = ColumnDataSource(
-                {
-                    "bandwidth": [1, 2],
-                    "source": ["a", "b"],
-                    "destination": ["a", "b"],
-                    "bandwidth_text": ["1", "2"],
-                }
-            )
-
-            values = [hex(x)[2:] for x in range(64, 256)][::-1]
-            mapper = linear_cmap(
-                field_name="bandwidth",
-                palette=["#" + x + x + "FF" for x in values],
-                low=0,
-                high=1,
-            )
-
-            fig = figure(
-                title="Bandwidth by Worker",
-                tools="",
-                id="bk-bandwidth-worker-plot",
-                name="bandwidth_worker_heatmap",
-                x_range=["a", "b"],
-                y_range=["a", "b"],
-                **kwargs
+            worker = escape.url_unescape(worker)
+            logs = await self.server.get_worker_logs(workers=[worker])
+            logs = logs[worker]
+            self.render(
+                "logs.html",
+                title="Logs: " + worker,
+                logs=logs,
+                **toolz.merge(self.extra, rel_path_statics),
             )
-            fig.xaxis.major_label_orientation = -math.pi / 12
-            rect = fig.rect(
-                source=self.source,
-                x="source",
-                y="destination",
-                color=mapper,
-                height=1,
-                width=1,
-            )
-
-            self.color_map = mapper["transform"]
-            color_bar = ColorBar(
-                color_mapper=self.color_map,
-                label_standoff=12,
-                border_line_color=None,
-                location=(0, 0),
-            )
-            color_bar.formatter = NumeralTickFormatter(format="0 b")
-            fig.add_layout(color_bar, "right")
-
-            fig.toolbar.logo = None
-            fig.toolbar_location = None
-
-            hover = HoverTool()
-            hover.tooltips = """
-            <div>
-                <p><b>Source:</b> @source </p>
-                <p><b>Destination:</b> @destination </p>
-                <p><b>Bandwidth:</b> @bandwidth_text / s</p>
-            </div>
-            """
-            hover.point_policy = "follow_mouse"
-            fig.add_tools(hover)
-
-            self.fig = fig
-
-    @without_property_validation
-    def update(self):
-        with log_errors():
-            bw = self.scheduler.bandwidth_workers
-            if not bw:
-                return
-            x, y, value = zip(*[(a, b, c) for (a, b), c in bw.items()])
-
-            if self.color_map.high < max(value):
-                self.color_map.high = max(value)
-
-            factors = list(sorted(set(x + y)))
-            self.fig.x_range.factors = factors
-            self.fig.y_range.factors = factors
-
-            result = {
-                "source": x,
-                "destination": y,
-                "bandwidth": value,
-                "bandwidth_text": list(map(format_bytes, value)),
-            }
-            self.fig.title.text = "Bandwidth: " + format_bytes(self.scheduler.bandwidth)
 
-            update(self.source, result)
 
-
-class CurrentLoad(DashboardComponent):
-    """ How many tasks are on each worker """
-
-    def __init__(self, scheduler, width=600, **kwargs):
+class WorkerCallStacks(RequestHandler):
+    async def get(self, worker):
         with log_errors():
-            self.last = 0
-            self.scheduler = scheduler
-            self.source = ColumnDataSource(
-                {
-                    "nprocessing": [1, 2],
-                    "nprocessing-half": [0.5, 1],
-                    "nprocessing-color": ["red", "blue"],
-                    "nbytes": [1, 2],
-                    "nbytes-half": [0.5, 1],
-                    "nbytes_text": ["1B", "2B"],
-                    "cpu": [1, 2],
-                    "cpu-half": [0.5, 1],
-                    "worker": ["a", "b"],
-                    "y": [1, 2],
-                    "nbytes-color": ["blue", "blue"],
-                    "escaped_worker": ["a", "b"],
-                }
-            )
-
-            processing = figure(
-                title="Tasks Processing",
-                tools="",
-                id="bk-nprocessing-plot",
-                name="processing_hist",
-                width=int(width / 2),
-                **kwargs
-            )
-            rect = processing.rect(
-                source=self.source,
-                x="nprocessing-half",
-                y="y",
-                width="nprocessing",
-                height=1,
-                color="nprocessing-color",
-            )
-            processing.x_range.start = 0
-            rect.nonselection_glyph = None
-
-            nbytes = figure(
-                title="Bytes stored",
-                tools="",
-                id="bk-nbytes-worker-plot",
-                width=int(width / 2),
-                name="nbytes_hist",
-                **kwargs
-            )
-            rect = nbytes.rect(
-                source=self.source,
-                x="nbytes-half",
-                y="y",
-                width="nbytes",
-                height=1,
-                color="nbytes-color",
-            )
-            rect.nonselection_glyph = None
-
-            cpu = figure(
-                title="CPU Utilization",
-                tools="",
-                id="bk-cpu-worker-plot",
-                width=int(width / 2),
-                name="cpu_hist",
-                **kwargs
+            worker = escape.url_unescape(worker)
+            keys = self.server.processing[worker]
+            call_stack = await self.server.get_call_stack(keys=keys)
+            self.render(
+                "call-stack.html",
+                title="Call Stacks: " + worker,
+                call_stack=call_stack,
+                **toolz.merge(self.extra, rel_path_statics),
             )
-            rect = cpu.rect(
-                source=self.source,
-                x="cpu-half",
-                y="y",
-                width="cpu",
-                height=1,
-                color="blue",
-            )
-            rect.nonselection_glyph = None
-            hundred_span = Span(
-                location=100,
-                dimension="height",
-                line_color="gray",
-                line_dash="dashed",
-                line_width=3,
-            )
-            cpu.add_layout(hundred_span)
-
-            nbytes.axis[0].ticker = BasicTicker(mantissas=[1, 256, 512], base=1024)
-            nbytes.xaxis[0].formatter = NumeralTickFormatter(format="0.0 b")
-            nbytes.xaxis.major_label_orientation = -math.pi / 12
-            nbytes.x_range.start = 0
-
-            for fig in [processing, nbytes]:
-                fig.xaxis.minor_tick_line_alpha = 0
-                fig.yaxis.visible = False
-                fig.ygrid.visible = False
-
-                tap = TapTool(
-                    callback=OpenURL(url="./info/worker/@escaped_worker.html")
-                )
-                fig.add_tools(tap)
-
-                fig.toolbar.logo = None
-                fig.toolbar_location = None
-                fig.yaxis.visible = False
 
-            hover = HoverTool()
-            hover.tooltips = "@worker : @nprocessing tasks"
-            hover.point_policy = "follow_mouse"
-            processing.add_tools(hover)
 
-            hover = HoverTool()
-            hover.tooltips = "@worker : @nbytes_text"
-            hover.point_policy = "follow_mouse"
-            nbytes.add_tools(hover)
-
-            hover = HoverTool()
-            hover.tooltips = "@worker : @cpu %"
-            hover.point_policy = "follow_mouse"
-            cpu.add_tools(hover)
-
-            self.processing_figure = processing
-            self.nbytes_figure = nbytes
-            self.cpu_figure = cpu
-
-            processing.y_range = nbytes.y_range
-            cpu.y_range = nbytes.y_range
-
-    @without_property_validation
-    def update(self):
+class TaskCallStack(RequestHandler):
+    async def get(self, key):
         with log_errors():
-            workers = list(self.scheduler.workers.values())
-
-            y = list(range(len(workers)))
-
-            cpu = [int(ws.metrics["cpu"]) for ws in workers]
-
-            nprocessing = [len(ws.processing) for ws in workers]
-            processing_color = []
-            for ws in workers:
-                if ws in self.scheduler.idle:
-                    processing_color.append("red")
-                elif ws in self.scheduler.saturated:
-                    processing_color.append("green")
-                else:
-                    processing_color.append("blue")
-
-            nbytes = [ws.metrics["memory"] for ws in workers]
-            nbytes_text = [format_bytes(nb) for nb in nbytes]
-            nbytes_color = []
-            max_limit = 0
-            for ws, nb in zip(workers, nbytes):
-                limit = (
-                    getattr(self.scheduler.workers[ws.address], "memory_limit", inf)
-                    or inf
-                )
-
-                if limit > max_limit:
-                    max_limit = limit
-
-                if nb > limit:
-                    nbytes_color.append("red")
-                elif nb > limit / 2:
-                    nbytes_color.append("orange")
-                else:
-                    nbytes_color.append("blue")
-
-            now = time()
-            if any(nprocessing) or self.last + 1 < now:
-                self.last = now
-                result = {
-                    "cpu": cpu,
-                    "cpu-half": [c / 2 for c in cpu],
-                    "nprocessing": nprocessing,
-                    "nprocessing-half": [np / 2 for np in nprocessing],
-                    "nprocessing-color": processing_color,
-                    "nbytes": nbytes,
-                    "nbytes-half": [nb / 2 for nb in nbytes],
-                    "nbytes-color": nbytes_color,
-                    "nbytes_text": nbytes_text,
-                    "worker": [ws.address for ws in workers],
-                    "escaped_worker": [escape.url_escape(ws.address) for ws in workers],
-                    "y": y,
-                }
-
-                self.nbytes_figure.title.text = "Bytes stored: " + format_bytes(
-                    sum(nbytes)
-                )
-                self.nbytes_figure.x_range.end = max_limit
-
-                update(self.source, result)
-
-
-class StealingTimeSeries(DashboardComponent):
-    def __init__(self, scheduler, **kwargs):
-        self.scheduler = scheduler
-        self.source = ColumnDataSource(
-            {"time": [time(), time() + 1], "idle": [0, 0.1], "saturated": [0, 0.1]}
-        )
-
-        x_range = DataRange1d(follow="end", follow_interval=20000, range_padding=0)
-
-        fig = figure(
-            title="Idle and Saturated Workers Over Time",
-            x_axis_type="datetime",
-            y_range=[-0.1, len(scheduler.workers) + 0.1],
-            height=150,
-            tools="",
-            x_range=x_range,
-            **kwargs
-        )
-        fig.line(source=self.source, x="time", y="idle", color="red")
-        fig.line(source=self.source, x="time", y="saturated", color="green")
-        fig.yaxis.minor_tick_line_color = None
-
-        fig.add_tools(
-            ResetTool(), PanTool(dimensions="width"), WheelZoomTool(dimensions="width")
-        )
-
-        self.root = fig
-
-    @without_property_validation
-    def update(self):
-        with log_errors():
-            result = {
-                "time": [time() * 1000],
-                "idle": [len(self.scheduler.idle)],
-                "saturated": [len(self.scheduler.saturated)],
-            }
-            if PROFILING:
-                curdoc().add_next_tick_callback(
-                    lambda: self.source.stream(result, 10000)
+            key = escape.url_unescape(key)
+            call_stack = await self.server.get_call_stack(keys=[key])
+            if not call_stack:
+                self.write(
+                    "<p>Task not actively running. "
+                    "It may be finished or not yet started</p>"
                 )
             else:
-                self.source.stream(result, 10000)
-
-
-class StealingEvents(DashboardComponent):
-    def __init__(self, scheduler, **kwargs):
-        self.scheduler = scheduler
-        self.steal = scheduler.extensions["stealing"]
-        self.last = 0
-        self.source = ColumnDataSource(
-            {
-                "time": [time() - 20, time()],
-                "level": [0, 15],
-                "color": ["white", "white"],
-                "duration": [0, 0],
-                "radius": [1, 1],
-                "cost_factor": [0, 10],
-                "count": [1, 1],
-            }
-        )
-
-        x_range = DataRange1d(follow="end", follow_interval=20000, range_padding=0)
-
-        fig = figure(
-            title="Stealing Events",
-            x_axis_type="datetime",
-            y_axis_type="log",
-            height=250,
-            tools="",
-            x_range=x_range,
-            **kwargs
-        )
-
-        fig.circle(
-            source=self.source,
-            x="time",
-            y="cost_factor",
-            color="color",
-            size="radius",
-            alpha=0.5,
-        )
-        fig.yaxis.axis_label = "Cost Multiplier"
-
-        hover = HoverTool()
-        hover.tooltips = "Level: @level, Duration: @duration, Count: @count, Cost factor: @cost_factor"
-        hover.point_policy = "follow_mouse"
-
-        fig.add_tools(
-            hover,
-            ResetTool(),
-            PanTool(dimensions="width"),
-            WheelZoomTool(dimensions="width"),
-        )
-
-        self.root = fig
-
-    def convert(self, msgs):
-        """ Convert a log message to a glyph """
-        total_duration = 0
-        for msg in msgs:
-            time, level, key, duration, sat, occ_sat, idl, occ_idl = msg
-            total_duration += duration
-
-        try:
-            color = Viridis11[level]
-        except (KeyError, IndexError):
-            color = "black"
-
-        radius = math.sqrt(min(total_duration, 10)) * 30 + 2
-
-        d = {
-            "time": time * 1000,
-            "level": level,
-            "count": len(msgs),
-            "color": color,
-            "duration": total_duration,
-            "radius": radius,
-            "cost_factor": min(10, self.steal.cost_multipliers[level]),
-        }
-
-        return d
-
-    @without_property_validation
-    def update(self):
-        with log_errors():
-            log = self.steal.log
-            n = self.steal.count - self.last
-            log = [log[-i] for i in range(1, n + 1) if isinstance(log[-i], list)]
-            self.last = self.steal.count
-
-            if log:
-                new = pipe(
-                    log,
-                    map(groupby(1)),
-                    map(dict.values),
-                    concat,
-                    map(self.convert),
-                    list,
-                    transpose,
-                )
-                if PROFILING:
-                    curdoc().add_next_tick_callback(
-                        lambda: self.source.stream(new, 10000)
-                    )
-                else:
-                    self.source.stream(new, 10000)
-
-
-class Events(DashboardComponent):
-    def __init__(self, scheduler, name, height=150, **kwargs):
-        self.scheduler = scheduler
-        self.action_ys = dict()
-        self.last = 0
-        self.name = name
-        self.source = ColumnDataSource(
-            {"time": [], "action": [], "hover": [], "y": [], "color": []}
-        )
-
-        x_range = DataRange1d(follow="end", follow_interval=200000)
-
-        fig = figure(
-            title=name,
-            x_axis_type="datetime",
-            height=height,
-            tools="",
-            x_range=x_range,
-            **kwargs
-        )
-
-        fig.circle(
-            source=self.source,
-            x="time",
-            y="y",
-            color="color",
-            size=50,
-            alpha=0.5,
-            legend="action",
-        )
-        fig.yaxis.axis_label = "Action"
-        fig.legend.location = "top_left"
-
-        hover = HoverTool()
-        hover.tooltips = "@action<br>@hover"
-        hover.point_policy = "follow_mouse"
-
-        fig.add_tools(
-            hover,
-            ResetTool(),
-            PanTool(dimensions="width"),
-            WheelZoomTool(dimensions="width"),
-        )
-
-        self.root = fig
-
-    @without_property_validation
-    def update(self):
-        with log_errors():
-            log = self.scheduler.events[self.name]
-            n = self.scheduler.event_counts[self.name] - self.last
-            if log:
-                log = [log[-i] for i in range(1, n + 1)]
-            self.last = self.scheduler.event_counts[self.name]
-
-            if log:
-                actions = []
-                times = []
-                hovers = []
-                ys = []
-                colors = []
-                for msg in log:
-                    times.append(msg["time"] * 1000)
-                    action = msg["action"]
-                    actions.append(action)
-                    try:
-                        ys.append(self.action_ys[action])
-                    except KeyError:
-                        self.action_ys[action] = len(self.action_ys)
-                        ys.append(self.action_ys[action])
-                    colors.append(color_of(action))
-                    hovers.append("TODO")
-
-                new = {
-                    "time": times,
-                    "action": actions,
-                    "hover": hovers,
-                    "y": ys,
-                    "color": colors,
-                }
-
-                if PROFILING:
-                    curdoc().add_next_tick_callback(
-                        lambda: self.source.stream(new, 10000)
-                    )
-                else:
-                    self.source.stream(new, 10000)
-
-
-class TaskStream(components.TaskStream):
-    def __init__(self, scheduler, n_rectangles=1000, clear_interval="20s", **kwargs):
-        self.scheduler = scheduler
-        self.offset = 0
-        es = [p for p in self.scheduler.plugins if isinstance(p, TaskStreamPlugin)]
-        if not es:
-            self.plugin = TaskStreamPlugin(self.scheduler)
-        else:
-            self.plugin = es[0]
-        self.index = max(0, self.plugin.index - n_rectangles)
-        self.workers = dict()
-
-        components.TaskStream.__init__(
-            self, n_rectangles=n_rectangles, clear_interval=clear_interval, **kwargs
-        )
-
-    @without_property_validation
-    def update(self):
-        if self.index == self.plugin.index:
-            return
-        with log_errors():
-            if self.index and len(self.source.data["start"]):
-                start = min(self.source.data["start"])
-                duration = max(self.source.data["duration"])
-                boundary = (self.offset + start - duration) / 1000
-            else:
-                boundary = self.offset
-            rectangles = self.plugin.rectangles(
-                istart=self.index, workers=self.workers, start_boundary=boundary
-            )
-            n = len(rectangles["name"])
-            self.index = self.plugin.index
-
-            if not rectangles["start"]:
-                return
-
-            # If there has been a significant delay then clear old rectangles
-            first_end = min(map(add, rectangles["start"], rectangles["duration"]))
-            if first_end > self.last:
-                last = self.last
-                self.last = first_end
-                if first_end > last + self.clear_interval * 1000:
-                    self.offset = min(rectangles["start"])
-                    self.source.data.update({k: [] for k in rectangles})
-
-            rectangles["start"] = [x - self.offset for x in rectangles["start"]]
-
-            # Convert to numpy for serialization speed
-            if n >= 10 and np:
-                for k, v in rectangles.items():
-                    if isinstance(v[0], Number):
-                        rectangles[k] = np.array(v)
-
-            if PROFILING:
-                curdoc().add_next_tick_callback(
-                    lambda: self.source.stream(rectangles, self.n_rectangles)
+                self.render(
+                    "call-stack.html",
+                    title="Call Stack: " + key,
+                    call_stack=call_stack,
+                    **toolz.merge(self.extra, rel_path_statics),
                 )
-            else:
-                self.source.stream(rectangles, self.n_rectangles)
-
-
-class GraphPlot(DashboardComponent):
-    """
-    A dynamic node-link diagram for the task graph on the scheduler
-
-    See also the GraphLayout diagnostic at
-    distributed/diagnostics/graph_layout.py
-    """
-
-    def __init__(self, scheduler, **kwargs):
-        self.scheduler = scheduler
-        self.layout = GraphLayout(scheduler)
-        self.invisible_count = 0  # number of invisible nodes
-
-        self.node_source = ColumnDataSource(
-            {"x": [], "y": [], "name": [], "state": [], "visible": [], "key": []}
-        )
-        self.edge_source = ColumnDataSource({"x": [], "y": [], "visible": []})
-
-        node_view = CDSView(
-            source=self.node_source,
-            filters=[GroupFilter(column_name="visible", group="True")],
-        )
-        edge_view = CDSView(
-            source=self.edge_source,
-            filters=[GroupFilter(column_name="visible", group="True")],
-        )
-
-        node_colors = factor_cmap(
-            "state",
-            factors=["waiting", "processing", "memory", "released", "erred"],
-            palette=["gray", "green", "red", "blue", "black"],
-        )
-
-        self.root = figure(title="Task Graph", **kwargs)
-        self.root.multi_line(
-            xs="x",
-            ys="y",
-            source=self.edge_source,
-            line_width=1,
-            view=edge_view,
-            color="black",
-            alpha=0.3,
-        )
-        rect = self.root.square(
-            x="x",
-            y="y",
-            size=10,
-            color=node_colors,
-            source=self.node_source,
-            view=node_view,
-            legend="state",
-        )
-        self.root.xgrid.grid_line_color = None
-        self.root.ygrid.grid_line_color = None
-
-        hover = HoverTool(
-            point_policy="follow_mouse",
-            tooltips="<b>@name</b>: @state",
-            renderers=[rect],
-        )
-        tap = TapTool(callback=OpenURL(url="info/task/@key.html"), renderers=[rect])
-        rect.nonselection_glyph = None
-        self.root.add_tools(hover, tap)
-
-    @without_property_validation
-    def update(self):
-        with log_errors():
-            # occasionally reset the column data source to remove old nodes
-            if self.invisible_count > len(self.node_source.data["x"]) / 2:
-                self.layout.reset_index()
-                self.invisible_count = 0
-                update = True
-            else:
-                update = False
-
-            new, self.layout.new = self.layout.new, []
-            new_edges = self.layout.new_edges
-            self.layout.new_edges = []
-
-            self.add_new_nodes_edges(new, new_edges, update=update)
-
-            self.patch_updates()
-
-    @without_property_validation
-    def add_new_nodes_edges(self, new, new_edges, update=False):
-        if new or update:
-            node_key = []
-            node_x = []
-            node_y = []
-            node_state = []
-            node_name = []
-            edge_x = []
-            edge_y = []
-
-            x = self.layout.x
-            y = self.layout.y
-
-            tasks = self.scheduler.tasks
-            for key in new:
-                try:
-                    task = tasks[key]
-                except KeyError:
-                    continue
-                xx = x[key]
-                yy = y[key]
-                node_key.append(escape.url_escape(key))
-                node_x.append(xx)
-                node_y.append(yy)
-                node_state.append(task.state)
-                node_name.append(task.prefix)
-
-            for a, b in new_edges:
-                try:
-                    edge_x.append([x[a], x[b]])
-                    edge_y.append([y[a], y[b]])
-                except KeyError:
-                    pass
-
-            node = {
-                "x": node_x,
-                "y": node_y,
-                "state": node_state,
-                "name": node_name,
-                "key": node_key,
-                "visible": ["True"] * len(node_x),
-            }
-            edge = {"x": edge_x, "y": edge_y, "visible": ["True"] * len(edge_x)}
-
-            if update or not len(self.node_source.data["x"]):
-                # see https://github.com/bokeh/bokeh/issues/7523
-                self.node_source.data.update(node)
-                self.edge_source.data.update(edge)
-            else:
-                self.node_source.stream(node)
-                self.edge_source.stream(edge)
-
-    @without_property_validation
-    def patch_updates(self):
-        """
-        Small updates like color changes or lost nodes from task transitions
-        """
-        n = len(self.node_source.data["x"])
-        m = len(self.edge_source.data["x"])
-
-        if self.layout.state_updates:
-            state_updates = self.layout.state_updates
-            self.layout.state_updates = []
-            updates = [(i, c) for i, c in state_updates if i < n]
-            self.node_source.patch({"state": updates})
-
-        if self.layout.visible_updates:
-            updates = self.layout.visible_updates
-            updates = [(i, c) for i, c in updates if i < n]
-            self.visible_updates = []
-            self.node_source.patch({"visible": updates})
-            self.invisible_count += len(updates)
-
-        if self.layout.visible_edge_updates:
-            updates = self.layout.visible_edge_updates
-            updates = [(i, c) for i, c in updates if i < m]
-            self.visible_updates = []
-            self.edge_source.patch({"visible": updates})
-
-    def __del__(self):
-        self.scheduler.remove_plugin(self.layout)
-
-
-class TaskProgress(DashboardComponent):
-    """ Progress bars per task type """
-
-    def __init__(self, scheduler, **kwargs):
-        self.scheduler = scheduler
-        ps = [p for p in scheduler.plugins if isinstance(p, AllProgress)]
-        if ps:
-            self.plugin = ps[0]
-        else:
-            self.plugin = AllProgress(scheduler)
-
-        data = progress_quads(
-            dict(all={}, memory={}, erred={}, released={}, processing={})
-        )
-        self.source = ColumnDataSource(data=data)
-
-        x_range = DataRange1d(range_padding=0)
-        y_range = Range1d(-8, 0)
-
-        self.root = figure(
-            id="bk-task-progress-plot",
-            title="Progress",
-            name="task_progress",
-            x_range=x_range,
-            y_range=y_range,
-            toolbar_location=None,
-            tools="",
-            **kwargs
-        )
-        self.root.line(  # just to define early ranges
-            x=[0, 0.9], y=[-1, 0], line_color="#FFFFFF", alpha=0.0
-        )
-        self.root.quad(
-            source=self.source,
-            top="top",
-            bottom="bottom",
-            left="left",
-            right="right",
-            fill_color="#aaaaaa",
-            line_color="#aaaaaa",
-            fill_alpha=0.1,
-            line_alpha=0.3,
-        )
-        self.root.quad(
-            source=self.source,
-            top="top",
-            bottom="bottom",
-            left="left",
-            right="released-loc",
-            fill_color="color",
-            line_color="color",
-            fill_alpha=0.6,
-        )
-        self.root.quad(
-            source=self.source,
-            top="top",
-            bottom="bottom",
-            left="released-loc",
-            right="memory-loc",
-            fill_color="color",
-            line_color="color",
-            fill_alpha=1.0,
-        )
-        self.root.quad(
-            source=self.source,
-            top="top",
-            bottom="bottom",
-            left="memory-loc",
-            right="erred-loc",
-            fill_color="black",
-            fill_alpha=0.5,
-            line_alpha=0,
-        )
-        self.root.quad(
-            source=self.source,
-            top="top",
-            bottom="bottom",
-            left="erred-loc",
-            right="processing-loc",
-            fill_color="gray",
-            fill_alpha=0.35,
-            line_alpha=0,
-        )
-        self.root.text(
-            source=self.source,
-            text="show-name",
-            y="bottom",
-            x="left",
-            x_offset=5,
-            text_font_size=value("10pt"),
-        )
-        self.root.text(
-            source=self.source,
-            text="done",
-            y="bottom",
-            x="right",
-            x_offset=-5,
-            text_align="right",
-            text_font_size=value("10pt"),
-        )
-        self.root.ygrid.visible = False
-        self.root.yaxis.minor_tick_line_alpha = 0
-        self.root.yaxis.visible = False
-        self.root.xgrid.visible = False
-        self.root.xaxis.minor_tick_line_alpha = 0
-        self.root.xaxis.visible = False
-
-        hover = HoverTool(
-            point_policy="follow_mouse",
-            tooltips="""
-                <div>
-                    <span style="font-size: 14px; font-weight: bold;">Name:</span>&nbsp;
-                    <span style="font-size: 10px; font-family: Monaco, monospace;">@name</span>
-                </div>
-                <div>
-                    <span style="font-size: 14px; font-weight: bold;">All:</span>&nbsp;
-                    <span style="font-size: 10px; font-family: Monaco, monospace;">@all</span>
-                </div>
-                <div>
-                    <span style="font-size: 14px; font-weight: bold;">Memory:</span>&nbsp;
-                    <span style="font-size: 10px; font-family: Monaco, monospace;">@memory</span>
-                </div>
-                <div>
-                    <span style="font-size: 14px; font-weight: bold;">Erred:</span>&nbsp;
-                    <span style="font-size: 10px; font-family: Monaco, monospace;">@erred</span>
-                </div>
-                <div>
-                    <span style="font-size: 14px; font-weight: bold;">Ready:</span>&nbsp;
-                    <span style="font-size: 10px; font-family: Monaco, monospace;">@processing</span>
-                </div>
-                """,
-        )
-        self.root.add_tools(hover)
 
-    @without_property_validation
-    def update(self):
-        with log_errors():
-            state = {"all": valmap(len, self.plugin.all), "nbytes": self.plugin.nbytes}
-            for k in ["memory", "erred", "released", "processing", "waiting"]:
-                state[k] = valmap(len, self.plugin.state[k])
-            if not state["all"] and not len(self.source.data["all"]):
-                return
-
-            d = progress_quads(state)
-
-            update(self.source, d)
-
-            totals = {
-                k: sum(state[k].values())
-                for k in ["all", "memory", "erred", "released", "waiting"]
-            }
-            totals["processing"] = totals["all"] - sum(
-                v for k, v in totals.items() if k != "all"
-            )
-
-            self.root.title.text = (
-                "Progress -- total: %(all)s, "
-                "in-memory: %(memory)s, processing: %(processing)s, "
-                "waiting: %(waiting)s, "
-                "erred: %(erred)s" % totals
-            )
 
+class CountsJSON(RequestHandler):
+    def get(self):
+        scheduler = self.server
+        erred = 0
+        nbytes = 0
+        nthreads = 0
+        memory = 0
+        processing = 0
+        released = 0
+        waiting = 0
+        waiting_data = 0
+
+        for ts in scheduler.tasks.values():
+            if ts.exception_blame is not None:
+                erred += 1
+            elif ts.state == "released":
+                released += 1
+            if ts.waiting_on:
+                waiting += 1
+            if ts.waiters:
+                waiting_data += 1
+        for ws in scheduler.workers.values():
+            nthreads += ws.nthreads
+            memory += len(ws.has_what)
+            nbytes += ws.nbytes
+            processing += len(ws.processing)
+
+        response = {
+            "bytes": nbytes,
+            "clients": len(scheduler.clients),
+            "cores": nthreads,
+            "erred": erred,
+            "hosts": len(scheduler.host_info),
+            "idle": len(scheduler.idle),
+            "memory": memory,
+            "processing": processing,
+            "released": released,
+            "saturated": len(scheduler.saturated),
+            "tasks": len(scheduler.tasks),
+            "unrunnable": len(scheduler.unrunnable),
+            "waiting": waiting,
+            "waiting_data": waiting_data,
+            "workers": len(scheduler.workers),
+        }
+        self.write(response)
 
-class MemoryUse(DashboardComponent):
-    """ The memory usage across the cluster, grouped by task type """
-
-    def __init__(self, scheduler, **kwargs):
-        self.scheduler = scheduler
-        ps = [p for p in scheduler.plugins if isinstance(p, AllProgress)]
-        if ps:
-            self.plugin = ps[0]
-        else:
-            self.plugin = AllProgress(scheduler)
-
-        self.source = ColumnDataSource(
-            data=dict(
-                name=[],
-                left=[],
-                right=[],
-                center=[],
-                color=[],
-                percent=[],
-                MB=[],
-                text=[],
-            )
-        )
-
-        self.root = Plot(
-            id="bk-nbytes-plot",
-            x_range=DataRange1d(),
-            y_range=DataRange1d(),
-            toolbar_location=None,
-            outline_line_color=None,
-            **kwargs
-        )
 
-        self.root.add_glyph(
-            self.source,
-            Quad(
-                top=1,
-                bottom=0,
-                left="left",
-                right="right",
-                fill_color="color",
-                fill_alpha=1,
-            ),
-        )
+class IdentityJSON(RequestHandler):
+    def get(self):
+        self.write(self.server.identity())
 
-        self.root.add_layout(LinearAxis(), "left")
-        self.root.add_layout(LinearAxis(), "below")
-
-        hover = HoverTool(
-            point_policy="follow_mouse",
-            tooltips="""
-                <div>
-                    <span style="font-size: 14px; font-weight: bold;">Name:</span>&nbsp;
-                    <span style="font-size: 10px; font-family: Monaco, monospace;">@name</span>
-                </div>
-                <div>
-                    <span style="font-size: 14px; font-weight: bold;">Percent:</span>&nbsp;
-                    <span style="font-size: 10px; font-family: Monaco, monospace;">@percent</span>
-                </div>
-                <div>
-                    <span style="font-size: 14px; font-weight: bold;">MB:</span>&nbsp;
-                    <span style="font-size: 10px; font-family: Monaco, monospace;">@MB</span>
-                </div>
-                """,
-        )
-        self.root.add_tools(hover)
 
-    @without_property_validation
-    def update(self):
+class IndexJSON(RequestHandler):
+    def get(self):
         with log_errors():
-            nb = nbytes_bar(self.plugin.nbytes)
-            update(self.source, nb)
-            self.root.title.text = "Memory Use: %0.2f MB" % (
-                sum(self.plugin.nbytes.values()) / 1e6
+            r = [url for url, _ in routes if url.endswith(".json")]
+            self.render(
+                "json-index.html", routes=r, title="Index of JSON routes", **self.extra
             )
 
 
-class WorkerTable(DashboardComponent):
-    """ Status of the current workers
-
-    This is two plots, a text-based table for each host and a thin horizontal
-    plot laying out hosts by their current memory use.
-    """
-
-    excluded_names = {"executing", "in_flight", "in_memory", "ready", "time"}
-
-    def __init__(self, scheduler, width=800, **kwargs):
-        self.scheduler = scheduler
-        self.names = [
-            "name",
-            "address",
-            "nthreads",
-            "cpu",
-            "memory",
-            "memory_limit",
-            "memory_percent",
-            "num_fds",
-            "read_bytes",
-            "write_bytes",
-            "cpu_fraction",
-        ]
-        workers = self.scheduler.workers.values()
-        self.extra_names = sorted(
-            {
-                m
-                for ws in workers
-                for m, v in ws.metrics.items()
-                if m not in self.names and isinstance(v, (str, int, float))
-            }
-            - self.excluded_names
-        )
-
-        table_names = [
-            "name",
-            "address",
-            "nthreads",
-            "cpu",
-            "memory",
-            "memory_limit",
-            "memory_percent",
-            "num_fds",
-            "read_bytes",
-            "write_bytes",
-        ]
-
-        self.source = ColumnDataSource({k: [] for k in self.names})
-
-        columns = {
-            name: TableColumn(field=name, title=name.replace("_percent", " %"))
-            for name in table_names
+class IndividualPlots(RequestHandler):
+    def get(self):
+        bokeh_server = self.server.services["dashboard"]
+        result = {
+            uri.strip("/").replace("-", " ").title(): uri
+            for uri in bokeh_server.apps
+            if uri.lstrip("/").startswith("individual-") and not uri.endswith(".json")
         }
+        self.write(result)
 
-        formatters = {
-            "cpu": NumberFormatter(format="0.0 %"),
-            "memory_percent": NumberFormatter(format="0.0 %"),
-            "memory": NumberFormatter(format="0 b"),
-            "memory_limit": NumberFormatter(format="0 b"),
-            "read_bytes": NumberFormatter(format="0 b"),
-            "write_bytes": NumberFormatter(format="0 b"),
-            "num_fds": NumberFormatter(format="0"),
-            "nthreads": NumberFormatter(format="0"),
-        }
 
-        if BOKEH_VERSION < "0.12.15":
-            dt_kwargs = {"row_headers": False}
-        else:
-            dt_kwargs = {"index_position": None}
-
-        table = DataTable(
-            source=self.source,
-            columns=[columns[n] for n in table_names],
-            reorderable=True,
-            sortable=True,
-            width=width,
-            **dt_kwargs
-        )
+class _PrometheusCollector(object):
+    def __init__(self, server):
+        self.server = server
 
-        for name in table_names:
-            if name in formatters:
-                table.columns[table_names.index(name)].formatter = formatters[name]
+    def collect(self):
+        from prometheus_client.core import GaugeMetricFamily
 
-        extra_names = ["name", "address"] + self.extra_names
-        extra_columns = {
-            name: TableColumn(field=name, title=name.replace("_percent", "%"))
-            for name in extra_names
-        }
-
-        extra_table = DataTable(
-            source=self.source,
-            columns=[extra_columns[n] for n in extra_names],
-            reorderable=True,
-            sortable=True,
-            width=width,
-            **dt_kwargs
-        )
-
-        hover = HoverTool(
-            point_policy="follow_mouse",
-            tooltips="""
-                <div>
-                  <span style="font-size: 10px; font-family: Monaco, monospace;">@worker: </span>
-                  <span style="font-size: 10px; font-family: Monaco, monospace;">@memory_percent</span>
-                </div>
-                """,
-        )
-
-        mem_plot = figure(
-            title="Memory Use (%)",
-            toolbar_location=None,
-            x_range=(0, 1),
-            y_range=(-0.1, 0.1),
-            height=60,
-            width=width,
-            tools="",
-            **kwargs
-        )
-        mem_plot.circle(
-            source=self.source, x="memory_percent", y=0, size=10, fill_alpha=0.5
+        yield GaugeMetricFamily(
+            "dask_scheduler_workers",
+            "Number of workers connected.",
+            value=len(self.server.workers),
         )
-        mem_plot.ygrid.visible = False
-        mem_plot.yaxis.minor_tick_line_alpha = 0
-        mem_plot.xaxis.visible = False
-        mem_plot.yaxis.visible = False
-        mem_plot.add_tools(hover, BoxSelectTool())
-
-        hover = HoverTool(
-            point_policy="follow_mouse",
-            tooltips="""
-                <div>
-                  <span style="font-size: 10px; font-family: Monaco, monospace;">@worker: </span>
-                  <span style="font-size: 10px; font-family: Monaco, monospace;">@cpu</span>
-                </div>
-                """,
+        yield GaugeMetricFamily(
+            "dask_scheduler_clients",
+            "Number of clients connected.",
+            value=len(self.server.clients),
         )
-
-        cpu_plot = figure(
-            title="CPU Use (%)",
-            toolbar_location=None,
-            x_range=(0, 1),
-            y_range=(-0.1, 0.1),
-            height=60,
-            width=width,
-            tools="",
-            **kwargs
+        yield GaugeMetricFamily(
+            "dask_scheduler_received_tasks",
+            "Number of tasks received at scheduler",
+            value=len(self.server.tasks),
         )
-        cpu_plot.circle(
-            source=self.source, x="cpu_fraction", y=0, size=10, fill_alpha=0.5
+        yield GaugeMetricFamily(
+            "dask_scheduler_unrunnable_tasks",
+            "Number of unrunnable tasks at scheduler",
+            value=len(self.server.unrunnable),
         )
-        cpu_plot.ygrid.visible = False
-        cpu_plot.yaxis.minor_tick_line_alpha = 0
-        cpu_plot.xaxis.visible = False
-        cpu_plot.yaxis.visible = False
-        cpu_plot.add_tools(hover, BoxSelectTool())
-        self.cpu_plot = cpu_plot
-
-        if "sizing_mode" in kwargs:
-            sizing_mode = {"sizing_mode": kwargs["sizing_mode"]}
-        else:
-            sizing_mode = {}
-
-        components = [cpu_plot, mem_plot, table]
-        if self.extra_names:
-            components.append(extra_table)
-
-        self.root = column(*components, id="bk-worker-table", **sizing_mode)
-
-    @without_property_validation
-    def update(self):
-        data = {name: [] for name in self.names + self.extra_names}
-        for i, (addr, ws) in enumerate(
-            sorted(self.scheduler.workers.items(), key=lambda kv: kv[1].name)
-        ):
-            for name in self.names + self.extra_names:
-                data[name].append(ws.metrics.get(name, None))
-            data["name"][-1] = ws.name if ws.name is not None else i
-            data["address"][-1] = ws.address
-            if ws.memory_limit:
-                data["memory_percent"][-1] = ws.metrics["memory"] / ws.memory_limit
-            else:
-                data["memory_percent"][-1] = ""
-            data["memory_limit"][-1] = ws.memory_limit
-            data["cpu"][-1] = ws.metrics["cpu"] / 100.0
-            data["cpu_fraction"][-1] = ws.metrics["cpu"] / 100.0 / ws.nthreads
-            data["nthreads"][-1] = ws.nthreads
-
-        self.source.data.update(data)
-
-
-def systemmonitor_doc(scheduler, extra, doc):
-    with log_errors():
-        sysmon = SystemMonitor(scheduler, sizing_mode="stretch_both")
-        doc.title = "Dask: Scheduler System Monitor"
-        add_periodic_callback(doc, sysmon, 500)
-
-        for subdoc in sysmon.root.children:
-            doc.add_root(subdoc)
-        doc.template = env.get_template("system.html")
-        doc.template_variables.update(extra)
-        doc.theme = BOKEH_THEME
-
-
-def stealing_doc(scheduler, extra, doc):
-    with log_errors():
-        occupancy = Occupancy(scheduler, height=200, sizing_mode="scale_width")
-        stealing_ts = StealingTimeSeries(scheduler, sizing_mode="scale_width")
-        stealing_events = StealingEvents(scheduler, sizing_mode="scale_width")
-        stealing_events.root.x_range = stealing_ts.root.x_range
-        doc.title = "Dask: Work Stealing"
-        add_periodic_callback(doc, occupancy, 500)
-        add_periodic_callback(doc, stealing_ts, 500)
-        add_periodic_callback(doc, stealing_events, 500)
-
-        doc.add_root(
-            column(
-                occupancy.root,
-                stealing_ts.root,
-                stealing_events.root,
-                sizing_mode="scale_width",
-            )
-        )
-
-        doc.template = env.get_template("simple.html")
-        doc.template_variables.update(extra)
-        doc.theme = BOKEH_THEME
-
-
-def events_doc(scheduler, extra, doc):
-    with log_errors():
-        events = Events(scheduler, "all", height=250)
-        events.update()
-        add_periodic_callback(doc, events, 500)
-        doc.title = "Dask: Scheduler Events"
-        doc.add_root(column(events.root, sizing_mode="scale_width"))
-        doc.template = env.get_template("simple.html")
-        doc.template_variables.update(extra)
-        doc.theme = BOKEH_THEME
-
-
-def workers_doc(scheduler, extra, doc):
-    with log_errors():
-        table = WorkerTable(scheduler)
-        table.update()
-        add_periodic_callback(doc, table, 500)
-        doc.title = "Dask: Workers"
-        doc.add_root(table.root)
-        doc.template = env.get_template("simple.html")
-        doc.template_variables.update(extra)
-        doc.theme = BOKEH_THEME
-
-
-def tasks_doc(scheduler, extra, doc):
-    with log_errors():
-        ts = TaskStream(
-            scheduler,
-            n_rectangles=dask.config.get(
-                "distributed.scheduler.dashboard.tasks.task-stream-length"
-            ),
-            clear_interval="60s",
-            sizing_mode="stretch_both",
-        )
-        ts.update()
-        add_periodic_callback(doc, ts, 5000)
-        doc.title = "Dask: Task Stream"
-        doc.add_root(ts.root)
-        doc.template = env.get_template("simple.html")
-        doc.template_variables.update(extra)
-        doc.theme = BOKEH_THEME
-
-
-def graph_doc(scheduler, extra, doc):
-    with log_errors():
-        graph = GraphPlot(scheduler, sizing_mode="stretch_both")
-        doc.title = "Dask: Task Graph"
-        graph.update()
-        add_periodic_callback(doc, graph, 200)
-        doc.add_root(graph.root)
-
-        doc.template = env.get_template("simple.html")
-        doc.template_variables.update(extra)
-        doc.theme = BOKEH_THEME
-
-
-def status_doc(scheduler, extra, doc):
-    with log_errors():
-        task_stream = TaskStream(
-            scheduler,
-            n_rectangles=dask.config.get(
-                "distributed.scheduler.dashboard.status.task-stream-length"
-            ),
-            clear_interval="10s",
-            sizing_mode="stretch_both",
-        )
-        task_stream.update()
-        add_periodic_callback(doc, task_stream, 100)
-
-        task_progress = TaskProgress(scheduler, sizing_mode="stretch_both")
-        task_progress.update()
-        add_periodic_callback(doc, task_progress, 100)
-
-        if len(scheduler.workers) < 50:
-            current_load = CurrentLoad(scheduler, sizing_mode="stretch_both")
-            current_load.update()
-            add_periodic_callback(doc, current_load, 100)
-            doc.add_root(current_load.nbytes_figure)
-            doc.add_root(current_load.processing_figure)
-        else:
-            nbytes_hist = NBytesHistogram(scheduler, sizing_mode="stretch_both")
-            nbytes_hist.update()
-            processing_hist = ProcessingHistogram(scheduler, sizing_mode="stretch_both")
-            processing_hist.update()
-            add_periodic_callback(doc, nbytes_hist, 100)
-            add_periodic_callback(doc, processing_hist, 100)
-            current_load_fig = row(
-                nbytes_hist.root, processing_hist.root, sizing_mode="stretch_both"
-            )
-
-            doc.add_root(nbytes_hist.root)
-            doc.add_root(processing_hist.root)
-
-        doc.title = "Dask: Status"
-        doc.add_root(task_progress.root)
-        doc.add_root(task_stream.root)
-        doc.theme = BOKEH_THEME
-        doc.template = env.get_template("status.html")
-        doc.template_variables.update(extra)
-        doc.theme = BOKEH_THEME
-
-
-def individual_task_stream_doc(scheduler, extra, doc):
-    task_stream = TaskStream(
-        scheduler, n_rectangles=1000, clear_interval="10s", sizing_mode="stretch_both"
-    )
-    task_stream.update()
-    add_periodic_callback(doc, task_stream, 100)
-    doc.add_root(task_stream.root)
-    doc.theme = BOKEH_THEME
 
 
-def individual_nbytes_doc(scheduler, extra, doc):
-    current_load = CurrentLoad(scheduler, sizing_mode="stretch_both")
-    current_load.update()
-    add_periodic_callback(doc, current_load, 100)
-    doc.add_root(current_load.nbytes_figure)
-    doc.theme = BOKEH_THEME
-
+class PrometheusHandler(RequestHandler):
+    _initialized = False
 
-def individual_cpu_doc(scheduler, extra, doc):
-    current_load = CurrentLoad(scheduler, sizing_mode="stretch_both")
-    current_load.update()
-    add_periodic_callback(doc, current_load, 100)
-    doc.add_root(current_load.cpu_figure)
-    doc.theme = BOKEH_THEME
+    def __init__(self, *args, **kwargs):
+        import prometheus_client
 
+        super(PrometheusHandler, self).__init__(*args, **kwargs)
 
-def individual_nprocessing_doc(scheduler, extra, doc):
-    current_load = CurrentLoad(scheduler, sizing_mode="stretch_both")
-    current_load.update()
-    add_periodic_callback(doc, current_load, 100)
-    doc.add_root(current_load.processing_figure)
-    doc.theme = BOKEH_THEME
-
-
-def individual_progress_doc(scheduler, extra, doc):
-    task_progress = TaskProgress(scheduler, height=160, sizing_mode="stretch_both")
-    task_progress.update()
-    add_periodic_callback(doc, task_progress, 100)
-    doc.add_root(task_progress.root)
-    doc.theme = BOKEH_THEME
-
-
-def individual_graph_doc(scheduler, extra, doc):
-    with log_errors():
-        graph = GraphPlot(scheduler, sizing_mode="stretch_both")
-        graph.update()
-
-        add_periodic_callback(doc, graph, 200)
-        doc.add_root(graph.root)
-        doc.theme = BOKEH_THEME
-
-
-def individual_profile_doc(scheduler, extra, doc):
-    with log_errors():
-        prof = ProfileTimePlot(scheduler, sizing_mode="scale_width", doc=doc)
-        doc.add_root(prof.root)
-        prof.trigger_update()
-        doc.theme = BOKEH_THEME
-
-
-def individual_profile_server_doc(scheduler, extra, doc):
-    with log_errors():
-        prof = ProfileServer(scheduler, sizing_mode="scale_width", doc=doc)
-        doc.add_root(prof.root)
-        prof.trigger_update()
-        doc.theme = BOKEH_THEME
-
-
-def individual_workers_doc(scheduler, extra, doc):
-    with log_errors():
-        table = WorkerTable(scheduler)
-        table.update()
-        add_periodic_callback(doc, table, 500)
-        doc.add_root(table.root)
-        doc.theme = BOKEH_THEME
+        if PrometheusHandler._initialized:
+            return
 
+        prometheus_client.REGISTRY.register(_PrometheusCollector(self.server))
 
-def individual_bandwidth_types(scheduler, extra, doc):
-    with log_errors():
-        bw = BandwidthTypes(scheduler, sizing_mode="stretch_both")
-        bw.update()
-        add_periodic_callback(doc, bw, 500)
-        doc.add_root(bw.fig)
-        doc.theme = BOKEH_THEME
+        PrometheusHandler._initialized = True
 
+    def get(self):
+        import prometheus_client
 
-def individual_bandwidth_workers(scheduler, extra, doc):
-    with log_errors():
-        bw = BandwidthWorkers(scheduler, sizing_mode="stretch_both")
-        bw.update()
-        add_periodic_callback(doc, bw, 500)
-        doc.add_root(bw.fig)
-        doc.theme = BOKEH_THEME
+        self.write(prometheus_client.generate_latest())
+        self.set_header("Content-Type", "text/plain; version=0.0.4")
 
 
-def profile_doc(scheduler, extra, doc):
-    with log_errors():
-        doc.title = "Dask: Profile"
-        prof = ProfileTimePlot(scheduler, sizing_mode="scale_width", doc=doc)
-        doc.add_root(prof.root)
-        doc.template = env.get_template("simple.html")
-        doc.template_variables.update(extra)
-        doc.theme = BOKEH_THEME
+class HealthHandler(RequestHandler):
+    def get(self):
+        self.write("ok")
+        self.set_header("Content-Type", "text/plain")
 
-        prof.trigger_update()
 
+routes = [
+    (r"info", redirect("info/main/workers.html")),
+    (r"info/main/workers.html", Workers),
+    (r"info/worker/(.*).html", Worker),
+    (r"info/task/(.*).html", Task),
+    (r"info/main/logs.html", Logs),
+    (r"info/call-stacks/(.*).html", WorkerCallStacks),
+    (r"info/call-stack/(.*).html", TaskCallStack),
+    (r"info/logs/(.*).html", WorkerLogs),
+    (r"json/counts.json", CountsJSON),
+    (r"json/identity.json", IdentityJSON),
+    (r"json/index.html", IndexJSON),
+    (r"individual-plots.json", IndividualPlots),
+    (r"metrics", PrometheusHandler),
+    (r"health", HealthHandler),
+    (r"proxy/(\d+)/(.*?)/(.*)", GlobalProxyHandler),
+]
 
-def profile_server_doc(scheduler, extra, doc):
-    with log_errors():
-        doc.title = "Dask: Profile of Event Loop"
-        prof = ProfileServer(scheduler, sizing_mode="scale_width", doc=doc)
-        doc.add_root(prof.root)
-        doc.template = env.get_template("simple.html")
-        doc.template_variables.update(extra)
-        doc.theme = BOKEH_THEME
 
-        prof.trigger_update()
+def get_handlers(server):
+    return [(url, cls, {"server": server}) for url, cls in routes]
 
 
 class BokehScheduler(BokehServer):
@@ -1853,8 +361,6 @@ def my_server(self):
     def listen(self, *args, **kwargs):
         super(BokehScheduler, self).listen(*args, **kwargs)
 
-        from .scheduler_html import routes
-
         handlers = [
             (
                 self.prefix + "/" + url,
@@ -1884,11 +390,12 @@ def listen(self, *args, **kwargs):
     "/individual-profile": individual_profile_doc,
     "/individual-profile-server": individual_profile_server_doc,
     "/individual-nbytes": individual_nbytes_doc,
+    "/individual-memory-use": individual_memory_use_doc,
     "/individual-cpu": individual_cpu_doc,
     "/individual-nprocessing": individual_nprocessing_doc,
     "/individual-workers": individual_workers_doc,
-    "/individual-bandwidth-types": individual_bandwidth_types,
-    "/individual-bandwidth-workers": individual_bandwidth_workers,
+    "/individual-bandwidth-types": individual_bandwidth_types_doc,
+    "/individual-bandwidth-workers": individual_bandwidth_workers_doc,
 }
 
 try:
@@ -1896,4 +403,7 @@ def listen(self, *args, **kwargs):
 except ImportError:
     pass
 else:
-    from . import nvml  # noqa: 1708
+    from .components.nvml import gpu_memory_doc, gpu_utilization_doc  # noqa: 1708
+
+    applications["/individual-gpu-memory"] = gpu_memory_doc
+    applications["/individual-gpu-utilization"] = gpu_utilization_doc
diff --git a/distributed/dashboard/scheduler_html.py b/distributed/dashboard/scheduler_html.py
deleted file mode 100644
index 6ac13915523..00000000000
--- a/distributed/dashboard/scheduler_html.py
+++ /dev/null
@@ -1,269 +0,0 @@
-from datetime import datetime
-
-from dask.utils import format_bytes
-import toolz
-from tornado import escape
-
-from ..utils import log_errors, format_time
-from .proxy import GlobalProxyHandler
-from .utils import RequestHandler, redirect
-
-ns = {
-    func.__name__: func for func in [format_bytes, format_time, datetime.fromtimestamp]
-}
-
-rel_path_statics = {"rel_path_statics": "../../"}
-
-
-class Workers(RequestHandler):
-    def get(self):
-        with log_errors():
-            self.render(
-                "workers.html",
-                title="Workers",
-                scheduler=self.server,
-                **toolz.merge(self.server.__dict__, ns, self.extra, rel_path_statics),
-            )
-
-
-class Worker(RequestHandler):
-    def get(self, worker):
-        worker = escape.url_unescape(worker)
-        if worker not in self.server.workers:
-            self.send_error(404)
-            return
-        with log_errors():
-            self.render(
-                "worker.html",
-                title="Worker: " + worker,
-                scheduler=self.server,
-                Worker=worker,
-                **toolz.merge(self.server.__dict__, ns, self.extra, rel_path_statics),
-            )
-
-
-class Task(RequestHandler):
-    def get(self, task):
-        task = escape.url_unescape(task)
-        if task not in self.server.tasks:
-            self.send_error(404)
-            return
-        with log_errors():
-            self.render(
-                "task.html",
-                title="Task: " + task,
-                Task=task,
-                scheduler=self.server,
-                **toolz.merge(self.server.__dict__, ns, self.extra, rel_path_statics),
-            )
-
-
-class Logs(RequestHandler):
-    def get(self):
-        with log_errors():
-            logs = self.server.get_logs()
-            self.render(
-                "logs.html",
-                title="Logs",
-                logs=logs,
-                **toolz.merge(self.extra, rel_path_statics),
-            )
-
-
-class WorkerLogs(RequestHandler):
-    async def get(self, worker):
-        with log_errors():
-            worker = escape.url_unescape(worker)
-            logs = await self.server.get_worker_logs(workers=[worker])
-            logs = logs[worker]
-            self.render(
-                "logs.html",
-                title="Logs: " + worker,
-                logs=logs,
-                **toolz.merge(self.extra, rel_path_statics),
-            )
-
-
-class WorkerCallStacks(RequestHandler):
-    async def get(self, worker):
-        with log_errors():
-            worker = escape.url_unescape(worker)
-            keys = self.server.processing[worker]
-            call_stack = await self.server.get_call_stack(keys=keys)
-            self.render(
-                "call-stack.html",
-                title="Call Stacks: " + worker,
-                call_stack=call_stack,
-                **toolz.merge(self.extra, rel_path_statics),
-            )
-
-
-class TaskCallStack(RequestHandler):
-    async def get(self, key):
-        with log_errors():
-            key = escape.url_unescape(key)
-            call_stack = await self.server.get_call_stack(keys=[key])
-            if not call_stack:
-                self.write(
-                    "<p>Task not actively running. "
-                    "It may be finished or not yet started</p>"
-                )
-            else:
-                self.render(
-                    "call-stack.html",
-                    title="Call Stack: " + key,
-                    call_stack=call_stack,
-                    **toolz.merge(self.extra, rel_path_statics),
-                )
-
-
-class CountsJSON(RequestHandler):
-    def get(self):
-        scheduler = self.server
-        erred = 0
-        nbytes = 0
-        nthreads = 0
-        memory = 0
-        processing = 0
-        released = 0
-        waiting = 0
-        waiting_data = 0
-
-        for ts in scheduler.tasks.values():
-            if ts.exception_blame is not None:
-                erred += 1
-            elif ts.state == "released":
-                released += 1
-            if ts.waiting_on:
-                waiting += 1
-            if ts.waiters:
-                waiting_data += 1
-        for ws in scheduler.workers.values():
-            nthreads += ws.nthreads
-            memory += len(ws.has_what)
-            nbytes += ws.nbytes
-            processing += len(ws.processing)
-
-        response = {
-            "bytes": nbytes,
-            "clients": len(scheduler.clients),
-            "cores": nthreads,
-            "erred": erred,
-            "hosts": len(scheduler.host_info),
-            "idle": len(scheduler.idle),
-            "memory": memory,
-            "processing": processing,
-            "released": released,
-            "saturated": len(scheduler.saturated),
-            "tasks": len(scheduler.tasks),
-            "unrunnable": len(scheduler.unrunnable),
-            "waiting": waiting,
-            "waiting_data": waiting_data,
-            "workers": len(scheduler.workers),
-        }
-        self.write(response)
-
-
-class IdentityJSON(RequestHandler):
-    def get(self):
-        self.write(self.server.identity())
-
-
-class IndexJSON(RequestHandler):
-    def get(self):
-        with log_errors():
-            r = [url for url, _ in routes if url.endswith(".json")]
-            self.render(
-                "json-index.html", routes=r, title="Index of JSON routes", **self.extra
-            )
-
-
-class IndividualPlots(RequestHandler):
-    def get(self):
-        bokeh_server = self.server.services["dashboard"]
-        result = {
-            uri.strip("/").replace("-", " ").title(): uri
-            for uri in bokeh_server.apps
-            if uri.lstrip("/").startswith("individual-") and not uri.endswith(".json")
-        }
-        self.write(result)
-
-
-class _PrometheusCollector(object):
-    def __init__(self, server):
-        self.server = server
-
-    def collect(self):
-        from prometheus_client.core import GaugeMetricFamily
-
-        yield GaugeMetricFamily(
-            "dask_scheduler_workers",
-            "Number of workers connected.",
-            value=len(self.server.workers),
-        )
-        yield GaugeMetricFamily(
-            "dask_scheduler_clients",
-            "Number of clients connected.",
-            value=len(self.server.clients),
-        )
-        yield GaugeMetricFamily(
-            "dask_scheduler_received_tasks",
-            "Number of tasks received at scheduler",
-            value=len(self.server.tasks),
-        )
-        yield GaugeMetricFamily(
-            "dask_scheduler_unrunnable_tasks",
-            "Number of unrunnable tasks at scheduler",
-            value=len(self.server.unrunnable),
-        )
-
-
-class PrometheusHandler(RequestHandler):
-    _initialized = False
-
-    def __init__(self, *args, **kwargs):
-        import prometheus_client
-
-        super(PrometheusHandler, self).__init__(*args, **kwargs)
-
-        if PrometheusHandler._initialized:
-            return
-
-        prometheus_client.REGISTRY.register(_PrometheusCollector(self.server))
-
-        PrometheusHandler._initialized = True
-
-    def get(self):
-        import prometheus_client
-
-        self.write(prometheus_client.generate_latest())
-        self.set_header("Content-Type", "text/plain; version=0.0.4")
-
-
-class HealthHandler(RequestHandler):
-    def get(self):
-        self.write("ok")
-        self.set_header("Content-Type", "text/plain")
-
-
-routes = [
-    (r"info", redirect("info/main/workers.html")),
-    (r"info/main/workers.html", Workers),
-    (r"info/worker/(.*).html", Worker),
-    (r"info/task/(.*).html", Task),
-    (r"info/main/logs.html", Logs),
-    (r"info/call-stacks/(.*).html", WorkerCallStacks),
-    (r"info/call-stack/(.*).html", TaskCallStack),
-    (r"info/logs/(.*).html", WorkerLogs),
-    (r"json/counts.json", CountsJSON),
-    (r"json/identity.json", IdentityJSON),
-    (r"json/index.html", IndexJSON),
-    (r"individual-plots.json", IndividualPlots),
-    (r"metrics", PrometheusHandler),
-    (r"health", HealthHandler),
-    (r"proxy/(\d+)/(.*?)/(.*)", GlobalProxyHandler),
-]
-
-
-def get_handlers(server):
-    return [(url, cls, {"server": server}) for url, cls in routes]
diff --git a/distributed/dashboard/tests/test_components.py b/distributed/dashboard/tests/test_components.py
index 5e96d788e45..195c947bdec 100644
--- a/distributed/dashboard/tests/test_components.py
+++ b/distributed/dashboard/tests/test_components.py
@@ -6,9 +6,7 @@
 from tornado import gen
 
 from distributed.utils_test import slowinc, gen_cluster
-
-from distributed.dashboard.components import (
-    TaskStream,
+from distributed.dashboard.components.shared import (
     MemoryUsage,
     Processing,
     ProfilePlot,
@@ -16,7 +14,7 @@
 )
 
 
-@pytest.mark.parametrize("Component", [TaskStream, MemoryUsage, Processing])
+@pytest.mark.parametrize("Component", [MemoryUsage, Processing])
 def test_basic(Component):
     c = Component()
     assert isinstance(c.source, ColumnDataSource)
diff --git a/distributed/dashboard/tests/test_scheduler_bokeh.py b/distributed/dashboard/tests/test_scheduler_bokeh.py
index 1e48a3addec..875f1064503 100644
--- a/distributed/dashboard/tests/test_scheduler_bokeh.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh.py
@@ -16,10 +16,10 @@
 from distributed.client import wait
 from distributed.metrics import time
 from distributed.utils_test import gen_cluster, inc, dec, slowinc, div, get_cert
-from distributed.dashboard.worker import Counters, BokehWorker
-from distributed.dashboard.scheduler import (
-    applications,
-    BokehScheduler,
+from distributed.dashboard.worker import BokehWorker
+from distributed.dashboard.components.worker import Counters
+from distributed.dashboard.scheduler import applications, BokehScheduler
+from distributed.dashboard.components.scheduler import (
     SystemMonitor,
     Occupancy,
     StealingTimeSeries,
@@ -32,7 +32,7 @@
     ProcessingHistogram,
     NBytesHistogram,
     WorkerTable,
-    GraphPlot,
+    TaskGraph,
     ProfileServer,
 )
 
@@ -70,7 +70,7 @@ def test_simple(c, s, a, b):
 
 @gen_cluster(client=True, worker_kwargs=dict(services={"dashboard": BokehWorker}))
 def test_basic(c, s, a, b):
-    for component in [SystemMonitor, Occupancy, StealingTimeSeries]:
+    for component in [TaskStream, SystemMonitor, Occupancy, StealingTimeSeries]:
         ss = component(s)
 
         ss.update()
@@ -443,8 +443,8 @@ def metric(worker):
 
 
 @gen_cluster(client=True)
-def test_GraphPlot(c, s, a, b):
-    gp = GraphPlot(s)
+def test_TaskGraph(c, s, a, b):
+    gp = TaskGraph(s)
     futures = c.map(inc, range(5))
     total = c.submit(sum, futures)
     yield total
@@ -483,8 +483,8 @@ def test_GraphPlot(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_GraphPlot_clear(c, s, a, b):
-    gp = GraphPlot(s)
+def test_TaskGraph_clear(c, s, a, b):
+    gp = TaskGraph(s)
     futures = c.map(inc, range(5))
     total = c.submit(sum, futures)
     yield total
@@ -507,9 +507,9 @@ def test_GraphPlot_clear(c, s, a, b):
 
 
 @gen_cluster(client=True, timeout=30)
-def test_GraphPlot_complex(c, s, a, b):
+def test_TaskGraph_complex(c, s, a, b):
     da = pytest.importorskip("dask.array")
-    gp = GraphPlot(s)
+    gp = TaskGraph(s)
     x = da.random.random((2000, 2000), chunks=(1000, 1000))
     y = ((x + x.T) - x.mean(axis=0)).persist()
     yield wait(y)
@@ -538,12 +538,12 @@ def test_GraphPlot_complex(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_GraphPlot_order(c, s, a, b):
+def test_TaskGraph_order(c, s, a, b):
     x = c.submit(inc, 1)
     y = c.submit(div, 1, 0)
     yield wait(y)
 
-    gp = GraphPlot(s)
+    gp = TaskGraph(s)
     gp.update()
 
     assert gp.node_source.data["state"][gp.layout.index[y.key]] == "erred"
diff --git a/distributed/dashboard/tests/test_worker_bokeh.py b/distributed/dashboard/tests/test_worker_bokeh.py
index c490c825ab4..b33fc3ba185 100644
--- a/distributed/dashboard/tests/test_worker_bokeh.py
+++ b/distributed/dashboard/tests/test_worker_bokeh.py
@@ -14,8 +14,8 @@
 from distributed.metrics import time
 from distributed.utils_test import gen_cluster, inc, dec
 from distributed.dashboard.scheduler import BokehScheduler
-from distributed.dashboard.worker import (
-    BokehWorker,
+from distributed.dashboard.worker import BokehWorker
+from distributed.dashboard.components.worker import (
     StateTable,
     CrossFilter,
     CommunicatingStream,
diff --git a/distributed/dashboard/utils.py b/distributed/dashboard/utils.py
index 285f6a5772a..b47cb75d6b0 100644
--- a/distributed/dashboard/utils.py
+++ b/distributed/dashboard/utils.py
@@ -1,14 +1,30 @@
 from distutils.version import LooseVersion
 import os
+from numbers import Number
 
 import bokeh
+from bokeh.io import curdoc
 from tornado import web
 from toolz import partition
 
+try:
+    import numpy as np
+except ImportError:
+    np = False
+
+
+try:
+    from cytoolz.curried import first
+except ImportError:
+    from toolz.curried import first
+
 BOKEH_VERSION = LooseVersion(bokeh.__version__)
 dirname = os.path.dirname(__file__)
 
 
+PROFILING = False
+
+
 if BOKEH_VERSION >= "1.0.0":
     # This decorator is only available in bokeh >= 1.0.0, and doesn't work for
     # callbacks in Python 2, since the signature introspection won't line up.
@@ -48,3 +64,33 @@ def get(self):
             self.redirect(path)
 
     return Redirect
+
+
+@without_property_validation
+def update(source, data):
+    """ Update source with data
+
+    This checks a few things first
+
+    1.  If the data is the same, then don't update
+    2.  If numpy is available and the data is numeric, then convert to numpy
+        arrays
+    3.  If profiling then perform the update in another callback
+    """
+    if not np or not any(isinstance(v, np.ndarray) for v in source.data.values()):
+        if source.data == data:
+            return
+    if np and len(data[first(data)]) > 10:
+        d = {}
+        for k, v in data.items():
+            if type(v) is not np.ndarray and isinstance(v[0], Number):
+                d[k] = np.array(v)
+            else:
+                d[k] = v
+    else:
+        d = data
+
+    if PROFILING:
+        curdoc().add_next_tick_callback(lambda: source.data.update(d))
+    else:
+        source.data.update(d)
diff --git a/distributed/dashboard/worker.py b/distributed/dashboard/worker.py
index 402d3fd0a70..4d635388512 100644
--- a/distributed/dashboard/worker.py
+++ b/distributed/dashboard/worker.py
@@ -1,40 +1,20 @@
 from functools import partial
 import logging
-import math
 import os
 
-from bokeh.layouts import row, column, widgetbox
-from bokeh.models import (
-    ColumnDataSource,
-    DataRange1d,
-    HoverTool,
-    BoxZoomTool,
-    ResetTool,
-    PanTool,
-    WheelZoomTool,
-    NumeralTickFormatter,
-    Select,
-)
-
-from bokeh.models.widgets import DataTable, TableColumn
-from bokeh.plotting import figure
-from bokeh.palettes import RdBu
 from bokeh.themes import Theme
-from dask.utils import format_bytes
-from toolz import merge, partition_all
-
-from .components import (
-    DashboardComponent,
-    ProfileTimePlot,
-    ProfileServer,
-    add_periodic_callback,
+from toolz import merge
+
+from .components.worker import (
+    status_doc,
+    crossfilter_doc,
+    systemmonitor_doc,
+    counters_doc,
+    profile_doc,
+    profile_server_doc,
 )
 from .core import BokehServer
-from .utils import transpose, without_property_validation
-from ..compatibility import WINDOWS
-from ..diagnostics.progress_stream import color_of
-from ..metrics import time
-from ..utils import log_errors, key_split, format_time
+from .utils import RequestHandler, redirect
 
 
 logger = logging.getLogger(__name__)
@@ -53,708 +33,110 @@
 template_variables = {"pages": ["status", "system", "profile", "crossfilter"]}
 
 
-class StateTable(DashboardComponent):
-    """ Currently running tasks """
-
-    def __init__(self, worker):
-        self.worker = worker
-
-        names = ["Stored", "Executing", "Ready", "Waiting", "Connections", "Serving"]
-        self.source = ColumnDataSource({name: [] for name in names})
-
-        columns = {name: TableColumn(field=name, title=name) for name in names}
-
-        table = DataTable(
-            source=self.source, columns=[columns[n] for n in names], height=70
-        )
-        self.root = table
-
-    @without_property_validation
-    def update(self):
-        with log_errors():
-            w = self.worker
-            d = {
-                "Stored": [len(w.data)],
-                "Executing": ["%d / %d" % (len(w.executing), w.nthreads)],
-                "Ready": [len(w.ready)],
-                "Waiting": [len(w.waiting_for_data)],
-                "Connections": [len(w.in_flight_workers)],
-                "Serving": [len(w._comms)],
-            }
-            self.source.data.update(d)
-
-
-class CommunicatingStream(DashboardComponent):
-    def __init__(self, worker, height=300, **kwargs):
-        with log_errors():
-            self.worker = worker
-            names = [
-                "start",
-                "stop",
-                "middle",
-                "duration",
-                "who",
-                "y",
-                "hover",
-                "alpha",
-                "bandwidth",
-                "total",
-            ]
-
-            self.incoming = ColumnDataSource({name: [] for name in names})
-            self.outgoing = ColumnDataSource({name: [] for name in names})
-
-            x_range = DataRange1d(range_padding=0)
-            y_range = DataRange1d(range_padding=0)
-
-            fig = figure(
-                title="Peer Communications",
-                x_axis_type="datetime",
-                x_range=x_range,
-                y_range=y_range,
-                height=height,
-                tools="",
-                **kwargs
-            )
-
-            fig.rect(
-                source=self.incoming,
-                x="middle",
-                y="y",
-                width="duration",
-                height=0.9,
-                color="red",
-                alpha="alpha",
-            )
-            fig.rect(
-                source=self.outgoing,
-                x="middle",
-                y="y",
-                width="duration",
-                height=0.9,
-                color="blue",
-                alpha="alpha",
+class _PrometheusCollector(object):
+    def __init__(self, server):
+        self.worker = server
+        self.logger = logging.getLogger("distributed.dask_worker")
+        self.crick_available = True
+        try:
+            import crick  # noqa: F401
+        except ImportError:
+            self.crick_available = False
+            self.logger.info(
+                "Not all prometheus metrics available are exported. Digest-based metrics require crick to be installed"
             )
 
-            hover = HoverTool(point_policy="follow_mouse", tooltips="""@hover""")
-            fig.add_tools(
-                hover,
-                ResetTool(),
-                PanTool(dimensions="width"),
-                WheelZoomTool(dimensions="width"),
-            )
+    def collect(self):
+        from prometheus_client.core import GaugeMetricFamily
 
-            self.root = fig
-
-            self.last_incoming = 0
-            self.last_outgoing = 0
-            self.who = dict()
-
-    @without_property_validation
-    def update(self):
-        with log_errors():
-            outgoing = self.worker.outgoing_transfer_log
-            n = self.worker.outgoing_count - self.last_outgoing
-            outgoing = [outgoing[-i].copy() for i in range(1, n + 1)]
-            self.last_outgoing = self.worker.outgoing_count
-
-            incoming = self.worker.incoming_transfer_log
-            n = self.worker.incoming_count - self.last_incoming
-            incoming = [incoming[-i].copy() for i in range(1, n + 1)]
-            self.last_incoming = self.worker.incoming_count
-
-            for [msgs, source] in [
-                [incoming, self.incoming],
-                [outgoing, self.outgoing],
-            ]:
-
-                for msg in msgs:
-                    if "compressed" in msg:
-                        del msg["compressed"]
-                    del msg["keys"]
-
-                    bandwidth = msg["total"] / (msg["duration"] or 0.5)
-                    bw = max(min(bandwidth / 500e6, 1), 0.3)
-                    msg["alpha"] = bw
-                    try:
-                        msg["y"] = self.who[msg["who"]]
-                    except KeyError:
-                        self.who[msg["who"]] = len(self.who)
-                        msg["y"] = self.who[msg["who"]]
-
-                    msg["hover"] = "%s / %s = %s/s" % (
-                        format_bytes(msg["total"]),
-                        format_time(msg["duration"]),
-                        format_bytes(msg["total"] / msg["duration"]),
-                    )
-
-                    for k in ["middle", "duration", "start", "stop"]:
-                        msg[k] = msg[k] * 1000
-
-                if msgs:
-                    msgs = transpose(msgs)
-                    if (
-                        len(source.data["stop"])
-                        and min(msgs["start"]) > source.data["stop"][-1] + 10000
-                    ):
-                        source.data.update(msgs)
-                    else:
-                        source.stream(msgs, rollover=10000)
-
-
-class CommunicatingTimeSeries(DashboardComponent):
-    def __init__(self, worker, **kwargs):
-        self.worker = worker
-        self.source = ColumnDataSource({"x": [], "in": [], "out": []})
-
-        x_range = DataRange1d(follow="end", follow_interval=20000, range_padding=0)
-
-        fig = figure(
-            title="Communication History",
-            x_axis_type="datetime",
-            y_range=[-0.1, worker.total_out_connections + 0.5],
-            height=150,
-            tools="",
-            x_range=x_range,
-            **kwargs
+        tasks = GaugeMetricFamily(
+            "dask_worker_tasks", "Number of tasks at worker.", labels=["state"]
         )
-        fig.line(source=self.source, x="x", y="in", color="red")
-        fig.line(source=self.source, x="x", y="out", color="blue")
-
-        fig.add_tools(
-            ResetTool(), PanTool(dimensions="width"), WheelZoomTool(dimensions="width")
+        tasks.add_metric(["stored"], len(self.worker.data))
+        tasks.add_metric(["ready"], len(self.worker.ready))
+        tasks.add_metric(["waiting"], len(self.worker.waiting_for_data))
+        tasks.add_metric(["serving"], len(self.worker._comms))
+        yield tasks
+
+        yield GaugeMetricFamily(
+            "dask_worker_connections",
+            "Number of task connections to other workers.",
+            value=len(self.worker.in_flight_workers),
         )
 
-        self.root = fig
-
-    @without_property_validation
-    def update(self):
-        with log_errors():
-            self.source.stream(
-                {
-                    "x": [time() * 1000],
-                    "out": [len(self.worker._comms)],
-                    "in": [len(self.worker.in_flight_workers)],
-                },
-                10000,
-            )
-
-
-class ExecutingTimeSeries(DashboardComponent):
-    def __init__(self, worker, **kwargs):
-        self.worker = worker
-        self.source = ColumnDataSource({"x": [], "y": []})
-
-        x_range = DataRange1d(follow="end", follow_interval=20000, range_padding=0)
-
-        fig = figure(
-            title="Executing History",
-            x_axis_type="datetime",
-            y_range=[-0.1, worker.nthreads + 0.1],
-            height=150,
-            tools="",
-            x_range=x_range,
-            **kwargs
+        yield GaugeMetricFamily(
+            "dask_worker_threads",
+            "Number of worker threads.",
+            value=self.worker.nthreads,
         )
-        fig.line(source=self.source, x="x", y="y")
 
-        fig.add_tools(
-            ResetTool(), PanTool(dimensions="width"), WheelZoomTool(dimensions="width")
+        yield GaugeMetricFamily(
+            "dask_worker_latency_seconds",
+            "Latency of worker connection.",
+            value=self.worker.latency,
         )
 
-        self.root = fig
-
-    @without_property_validation
-    def update(self):
-        with log_errors():
-            self.source.stream(
-                {"x": [time() * 1000], "y": [len(self.worker.executing)]}, 1000
+        # all metrics using digests require crick to be installed
+        # the following metrics will export NaN, if the corresponding digests are None
+        if self.crick_available:
+            yield GaugeMetricFamily(
+                "dask_worker_tick_duration_median_seconds",
+                "Median tick duration at worker.",
+                value=self.worker.digests["tick-duration"].components[1].quantile(50),
             )
 
-
-class CrossFilter(DashboardComponent):
-    def __init__(self, worker, **kwargs):
-        with log_errors():
-            self.worker = worker
-
-            quantities = ["nbytes", "duration", "bandwidth", "count", "start", "stop"]
-            colors = ["inout-color", "type-color", "key-color"]
-
-            # self.source = ColumnDataSource({name: [] for name in names})
-            self.source = ColumnDataSource(
-                {
-                    "nbytes": [1, 2],
-                    "duration": [0.01, 0.02],
-                    "bandwidth": [0.01, 0.02],
-                    "count": [1, 2],
-                    "type": ["int", "str"],
-                    "inout-color": ["blue", "red"],
-                    "type-color": ["blue", "red"],
-                    "key": ["add", "inc"],
-                    "start": [1, 2],
-                    "stop": [1, 2],
-                }
+            yield GaugeMetricFamily(
+                "dask_worker_task_duration_median_seconds",
+                "Median task runtime at worker.",
+                value=self.worker.digests["task-duration"].components[1].quantile(50),
             )
 
-            self.x = Select(title="X-Axis", value="nbytes", options=quantities)
-            self.x.on_change("value", self.update_figure)
-
-            self.y = Select(title="Y-Axis", value="bandwidth", options=quantities)
-            self.y.on_change("value", self.update_figure)
-
-            self.size = Select(
-                title="Size", value="None", options=["None"] + quantities
+            yield GaugeMetricFamily(
+                "dask_worker_transfer_bandwidth_median_bytes",
+                "Bandwidth for transfer at worker in Bytes.",
+                value=self.worker.digests["transfer-bandwidth"]
+                .components[1]
+                .quantile(50),
             )
-            self.size.on_change("value", self.update_figure)
 
-            self.color = Select(
-                title="Color", value="inout-color", options=["black"] + colors
-            )
-            self.color.on_change("value", self.update_figure)
 
-            if "sizing_mode" in kwargs:
-                kw = {"sizing_mode": kwargs["sizing_mode"]}
-            else:
-                kw = {}
+class PrometheusHandler(RequestHandler):
+    _initialized = False
 
-            self.control = widgetbox(
-                [self.x, self.y, self.size, self.color], width=200, **kw
-            )
+    def __init__(self, *args, **kwargs):
+        import prometheus_client
 
-            self.last_outgoing = 0
-            self.last_incoming = 0
-            self.kwargs = kwargs
-
-            self.layout = row(self.control, self.create_figure(**self.kwargs), **kw)
-
-            self.root = self.layout
-
-    @without_property_validation
-    def update(self):
-        with log_errors():
-            outgoing = self.worker.outgoing_transfer_log
-            n = self.worker.outgoing_count - self.last_outgoing
-            n = min(n, 1000)
-            outgoing = [outgoing[-i].copy() for i in range(1, n)]
-            self.last_outgoing = self.worker.outgoing_count
-
-            incoming = self.worker.incoming_transfer_log
-            n = self.worker.incoming_count - self.last_incoming
-            n = min(n, 1000)
-            incoming = [incoming[-i].copy() for i in range(1, n)]
-            self.last_incoming = self.worker.incoming_count
-
-            out = []
-
-            for msg in incoming:
-                if msg["keys"]:
-                    d = self.process_msg(msg)
-                    d["inout-color"] = "red"
-                    out.append(d)
-
-            for msg in outgoing:
-                if msg["keys"]:
-                    d = self.process_msg(msg)
-                    d["inout-color"] = "blue"
-                    out.append(d)
-
-            if out:
-                out = transpose(out)
-                if (
-                    len(self.source.data["stop"])
-                    and min(out["start"]) > self.source.data["stop"][-1] + 10
-                ):
-                    self.source.data.update(out)
-                else:
-                    self.source.stream(out, rollover=1000)
-
-    def create_figure(self, **kwargs):
-        with log_errors():
-            fig = figure(title="", tools="", **kwargs)
-
-            size = self.size.value
-            if size == "None":
-                size = 1
-
-            fig.circle(
-                source=self.source,
-                x=self.x.value,
-                y=self.y.value,
-                color=self.color.value,
-                size=10,
-                alpha=0.5,
-                hover_alpha=1,
-            )
-            fig.xaxis.axis_label = self.x.value
-            fig.yaxis.axis_label = self.y.value
-
-            fig.add_tools(
-                # self.hover,
-                ResetTool(),
-                PanTool(),
-                WheelZoomTool(),
-                BoxZoomTool(),
-            )
-            return fig
+        super(PrometheusHandler, self).__init__(*args, **kwargs)
 
-    @without_property_validation
-    def update_figure(self, attr, old, new):
-        with log_errors():
-            fig = self.create_figure(**self.kwargs)
-            self.layout.children[1] = fig
+        if PrometheusHandler._initialized:
+            return
 
-    def process_msg(self, msg):
-        try:
+        prometheus_client.REGISTRY.register(_PrometheusCollector(self.server))
 
-            def func(k):
-                return msg["keys"].get(k, 0)
-
-            status_key = max(msg["keys"], key=func)
-            typ = self.worker.types.get(status_key, object).__name__
-            keyname = key_split(status_key)
-            d = {
-                "nbytes": msg["total"],
-                "duration": msg["duration"],
-                "bandwidth": msg["bandwidth"],
-                "count": len(msg["keys"]),
-                "type": typ,
-                "type-color": color_of(typ),
-                "key": keyname,
-                "key-color": color_of(keyname),
-                "start": msg["start"],
-                "stop": msg["stop"],
-            }
-            return d
-        except Exception as e:
-            logger.exception(e)
-            raise
-
-
-class SystemMonitor(DashboardComponent):
-    def __init__(self, worker, height=150, **kwargs):
-        self.worker = worker
+        PrometheusHandler._initialized = True
 
-        names = worker.monitor.quantities
-        self.last = 0
-        self.source = ColumnDataSource({name: [] for name in names})
-        self.source.data.update(self.get_data())
+    def get(self):
+        import prometheus_client
 
-        x_range = DataRange1d(follow="end", follow_interval=20000, range_padding=0)
+        self.write(prometheus_client.generate_latest())
+        self.set_header("Content-Type", "text/plain; version=0.0.4")
 
-        tools = "reset,xpan,xwheel_zoom"
 
-        self.cpu = figure(
-            title="CPU",
-            x_axis_type="datetime",
-            height=height,
-            tools=tools,
-            x_range=x_range,
-            **kwargs
-        )
-        self.cpu.line(source=self.source, x="time", y="cpu")
-        self.cpu.yaxis.axis_label = "Percentage"
-        self.mem = figure(
-            title="Memory",
-            x_axis_type="datetime",
-            height=height,
-            tools=tools,
-            x_range=x_range,
-            **kwargs
-        )
-        self.mem.line(source=self.source, x="time", y="memory")
-        self.mem.yaxis.axis_label = "Bytes"
-        self.bandwidth = figure(
-            title="Bandwidth",
-            x_axis_type="datetime",
-            height=height,
-            x_range=x_range,
-            tools=tools,
-            **kwargs
-        )
-        self.bandwidth.line(source=self.source, x="time", y="read_bytes", color="red")
-        self.bandwidth.line(source=self.source, x="time", y="write_bytes", color="blue")
-        self.bandwidth.yaxis.axis_label = "Bytes / second"
-
-        # self.cpu.yaxis[0].formatter = NumeralTickFormatter(format='0%')
-        self.bandwidth.yaxis[0].formatter = NumeralTickFormatter(format="0.0b")
-        self.mem.yaxis[0].formatter = NumeralTickFormatter(format="0.0b")
-
-        plots = [self.cpu, self.mem, self.bandwidth]
-
-        if not WINDOWS:
-            self.num_fds = figure(
-                title="Number of File Descriptors",
-                x_axis_type="datetime",
-                height=height,
-                x_range=x_range,
-                tools=tools,
-                **kwargs
-            )
-
-            self.num_fds.line(source=self.source, x="time", y="num_fds")
-            plots.append(self.num_fds)
-
-        if "sizing_mode" in kwargs:
-            kw = {"sizing_mode": kwargs["sizing_mode"]}
-        else:
-            kw = {}
-
-        if not WINDOWS:
-            self.num_fds.y_range.start = 0
-        self.mem.y_range.start = 0
-        self.cpu.y_range.start = 0
-        self.bandwidth.y_range.start = 0
-
-        self.root = column(*plots, **kw)
-        self.worker.monitor.update()
-
-    def get_data(self):
-        d = self.worker.monitor.range_query(start=self.last)
-        d["time"] = [x * 1000 for x in d["time"]]
-        self.last = self.worker.monitor.count
-        return d
-
-    @without_property_validation
-    def update(self):
-        with log_errors():
-            self.source.stream(self.get_data(), 1000)
-
-
-class Counters(DashboardComponent):
-    def __init__(self, server, sizing_mode="stretch_both", **kwargs):
-        self.server = server
-        self.counter_figures = {}
-        self.counter_sources = {}
-        self.digest_figures = {}
-        self.digest_sources = {}
-        self.sizing_mode = sizing_mode
-
-        if self.server.digests:
-            for name in self.server.digests:
-                self.add_digest_figure(name)
-        for name in self.server.counters:
-            self.add_counter_figure(name)
-
-        figures = merge(self.digest_figures, self.counter_figures)
-        figures = [figures[k] for k in sorted(figures)]
-
-        if len(figures) <= 5:
-            self.root = column(figures, sizing_mode=sizing_mode)
-        else:
-            self.root = column(
-                *[
-                    row(*pair, sizing_mode=sizing_mode)
-                    for pair in partition_all(2, figures)
-                ],
-                sizing_mode=sizing_mode
-            )
-
-    def add_digest_figure(self, name):
-        with log_errors():
-            n = len(self.server.digests[name].intervals)
-            sources = {i: ColumnDataSource({"x": [], "y": []}) for i in range(n)}
-
-            kwargs = {}
-            if name.endswith("duration"):
-                kwargs["x_axis_type"] = "datetime"
-
-            fig = figure(
-                title=name, tools="", height=150, sizing_mode=self.sizing_mode, **kwargs
-            )
-            fig.yaxis.visible = False
-            fig.ygrid.visible = False
-            if name.endswith("bandwidth") or name.endswith("bytes"):
-                fig.xaxis[0].formatter = NumeralTickFormatter(format="0.0b")
-
-            for i in range(n):
-                alpha = 0.3 + 0.3 * (n - i) / n
-                fig.line(
-                    source=sources[i],
-                    x="x",
-                    y="y",
-                    alpha=alpha,
-                    color=RdBu[max(n, 3)][-i],
-                )
-
-            fig.xaxis.major_label_orientation = math.pi / 12
-            fig.toolbar.logo = None
-            self.digest_sources[name] = sources
-            self.digest_figures[name] = fig
-            return fig
-
-    def add_counter_figure(self, name):
-        with log_errors():
-            n = len(self.server.counters[name].intervals)
-            sources = {
-                i: ColumnDataSource({"x": [], "y": [], "y-center": [], "counts": []})
-                for i in range(n)
-            }
-
-            fig = figure(
-                title=name,
-                tools="",
-                height=150,
-                sizing_mode=self.sizing_mode,
-                x_range=sorted(map(str, self.server.counters[name].components[0])),
-            )
-            fig.ygrid.visible = False
-
-            for i in range(n):
-                width = 0.5 + 0.4 * i / n
-                fig.rect(
-                    source=sources[i],
-                    x="x",
-                    y="y-center",
-                    width=width,
-                    height="y",
-                    alpha=0.3,
-                    color=RdBu[max(n, 3)][-i],
-                )
-                hover = HoverTool(
-                    point_policy="follow_mouse", tooltips="""@x : @counts"""
-                )
-                fig.add_tools(hover)
-                fig.xaxis.major_label_orientation = math.pi / 12
-
-            fig.toolbar.logo = None
-
-            self.counter_sources[name] = sources
-            self.counter_figures[name] = fig
-            return fig
-
-    @without_property_validation
-    def update(self):
-        with log_errors():
-            for name, fig in self.digest_figures.items():
-                digest = self.server.digests[name]
-                d = {}
-                for i, d in enumerate(digest.components):
-                    if d.size():
-                        ys, xs = d.histogram(100)
-                        xs = xs[1:]
-                        if name.endswith("duration"):
-                            xs *= 1000
-                        self.digest_sources[name][i].data.update({"x": xs, "y": ys})
-                fig.title.text = "%s: %d" % (name, digest.size())
-
-            for name, fig in self.counter_figures.items():
-                counter = self.server.counters[name]
-                d = {}
-                for i, d in enumerate(counter.components):
-                    if d:
-                        xs = sorted(d)
-                        factor = counter.intervals[0] / counter.intervals[i]
-                        counts = [d[x] for x in xs]
-                        ys = [factor * c for c in counts]
-                        y_centers = [y / 2 for y in ys]
-                        xs = list(map(str, xs))
-                        d = {"x": xs, "y": ys, "y-center": y_centers, "counts": counts}
-                        self.counter_sources[name][i].data.update(d)
-                    fig.title.text = "%s: %d" % (name, counter.size())
-                    fig.x_range.factors = list(map(str, xs))
-
-
-from bokeh.application.handlers.function import FunctionHandler
-from bokeh.application import Application
-
-
-def status_doc(worker, extra, doc):
-    with log_errors():
-        statetable = StateTable(worker)
-        executing_ts = ExecutingTimeSeries(worker, sizing_mode="scale_width")
-        communicating_ts = CommunicatingTimeSeries(worker, sizing_mode="scale_width")
-        communicating_stream = CommunicatingStream(worker, sizing_mode="scale_width")
-
-        xr = executing_ts.root.x_range
-        communicating_ts.root.x_range = xr
-        communicating_stream.root.x_range = xr
-
-        doc.title = "Dask Worker Internal Monitor"
-        add_periodic_callback(doc, statetable, 200)
-        add_periodic_callback(doc, executing_ts, 200)
-        add_periodic_callback(doc, communicating_ts, 200)
-        add_periodic_callback(doc, communicating_stream, 200)
-        doc.add_root(
-            column(
-                statetable.root,
-                executing_ts.root,
-                communicating_ts.root,
-                communicating_stream.root,
-                sizing_mode="scale_width",
-            )
-        )
-        doc.template = env.get_template("simple.html")
-        doc.template_variables["active_page"] = "status"
-        doc.template_variables.update(extra)
-        doc.theme = BOKEH_THEME
-
-
-def crossfilter_doc(worker, extra, doc):
-    with log_errors():
-        statetable = StateTable(worker)
-        crossfilter = CrossFilter(worker)
-
-        doc.title = "Dask Worker Cross-filter"
-        add_periodic_callback(doc, statetable, 500)
-        add_periodic_callback(doc, crossfilter, 500)
-
-        doc.add_root(column(statetable.root, crossfilter.root))
-        doc.template = env.get_template("simple.html")
-        doc.template_variables["active_page"] = "crossfilter"
-        doc.template_variables.update(extra)
-        doc.theme = BOKEH_THEME
+class HealthHandler(RequestHandler):
+    def get(self):
+        self.write("ok")
+        self.set_header("Content-Type", "text/plain")
 
 
-def systemmonitor_doc(worker, extra, doc):
-    with log_errors():
-        sysmon = SystemMonitor(worker, sizing_mode="scale_width")
-        doc.title = "Dask Worker Monitor"
-        add_periodic_callback(doc, sysmon, 500)
+routes = [
+    (r"metrics", PrometheusHandler),
+    (r"health", HealthHandler),
+    (r"main", redirect("/status")),
+]
 
-        doc.add_root(sysmon.root)
-        doc.template = env.get_template("simple.html")
-        doc.template_variables["active_page"] = "system"
-        doc.template_variables.update(extra)
-        doc.theme = BOKEH_THEME
 
-
-def counters_doc(server, extra, doc):
-    with log_errors():
-        doc.title = "Dask Worker Counters"
-        counter = Counters(server, sizing_mode="stretch_both")
-        add_periodic_callback(doc, counter, 500)
-
-        doc.add_root(counter.root)
-        doc.template = env.get_template("simple.html")
-        doc.template_variables["active_page"] = "counters"
-        doc.template_variables.update(extra)
-        doc.theme = BOKEH_THEME
-
-
-def profile_doc(server, extra, doc):
-    with log_errors():
-        doc.title = "Dask Worker Profile"
-        profile = ProfileTimePlot(server, sizing_mode="scale_width", doc=doc)
-        profile.trigger_update()
-
-        doc.add_root(profile.root)
-        doc.template = env.get_template("simple.html")
-        doc.template_variables["active_page"] = "profile"
-        doc.template_variables.update(extra)
-        doc.theme = BOKEH_THEME
-
-
-def profile_server_doc(server, extra, doc):
-    with log_errors():
-        doc.title = "Dask: Profile of Event Loop"
-        prof = ProfileServer(server, sizing_mode="scale_width", doc=doc)
-        doc.add_root(prof.root)
-        doc.template = env.get_template("simple.html")
-        # doc.template_variables['active_page'] = ''
-        doc.template_variables.update(extra)
-        doc.theme = BOKEH_THEME
-
-        prof.trigger_update()
+def get_handlers(server):
+    return [(url, cls, {"server": server}) for url, cls in routes]
 
 
 class BokehWorker(BokehServer):
@@ -768,31 +150,15 @@ def __init__(self, worker, io_loop=None, prefix="", **kwargs):
             prefix = "/" + prefix
         self.prefix = prefix
 
-        extra = {"prefix": prefix}
-
-        extra.update(template_variables)
-
-        status = Application(FunctionHandler(partial(status_doc, worker, extra)))
-        crossfilter = Application(
-            FunctionHandler(partial(crossfilter_doc, worker, extra))
-        )
-        systemmonitor = Application(
-            FunctionHandler(partial(systemmonitor_doc, worker, extra))
-        )
-        counters = Application(FunctionHandler(partial(counters_doc, worker, extra)))
-        profile = Application(FunctionHandler(partial(profile_doc, worker, extra)))
-        profile_server = Application(
-            FunctionHandler(partial(profile_server_doc, worker, extra))
-        )
-
         self.apps = {
-            "/status": status,
-            "/counters": counters,
-            "/crossfilter": crossfilter,
-            "/system": systemmonitor,
-            "/profile": profile,
-            "/profile-server": profile_server,
+            "/status": status_doc,
+            "/counters": counters_doc,
+            "/crossfilter": crossfilter_doc,
+            "/system": systemmonitor_doc,
+            "/profile": profile_doc,
+            "/profile-server": profile_server_doc,
         }
+        self.apps = {k: partial(v, worker, self.extra) for k, v in self.apps.items()}
 
         self.loop = io_loop or worker.loop
         self.server = None
@@ -808,8 +174,6 @@ def my_server(self):
     def listen(self, *args, **kwargs):
         super(BokehWorker, self).listen(*args, **kwargs)
 
-        from .worker_html import routes
-
         handlers = [
             (
                 self.prefix + "/" + url,
diff --git a/distributed/dashboard/worker_html.py b/distributed/dashboard/worker_html.py
deleted file mode 100644
index 27e1f9fe9d2..00000000000
--- a/distributed/dashboard/worker_html.py
+++ /dev/null
@@ -1,108 +0,0 @@
-import logging
-from .utils import RequestHandler, redirect
-
-
-class _PrometheusCollector(object):
-    def __init__(self, server):
-        self.worker = server
-        self.logger = logging.getLogger("distributed.dask_worker")
-        self.crick_available = True
-        try:
-            import crick  # noqa: F401
-        except ImportError:
-            self.crick_available = False
-            self.logger.info(
-                "Not all prometheus metrics available are exported. Digest-based metrics require crick to be installed"
-            )
-
-    def collect(self):
-        from prometheus_client.core import GaugeMetricFamily
-
-        tasks = GaugeMetricFamily(
-            "dask_worker_tasks", "Number of tasks at worker.", labels=["state"]
-        )
-        tasks.add_metric(["stored"], len(self.worker.data))
-        tasks.add_metric(["ready"], len(self.worker.ready))
-        tasks.add_metric(["waiting"], len(self.worker.waiting_for_data))
-        tasks.add_metric(["serving"], len(self.worker._comms))
-        yield tasks
-
-        yield GaugeMetricFamily(
-            "dask_worker_connections",
-            "Number of task connections to other workers.",
-            value=len(self.worker.in_flight_workers),
-        )
-
-        yield GaugeMetricFamily(
-            "dask_worker_threads",
-            "Number of worker threads.",
-            value=self.worker.nthreads,
-        )
-
-        yield GaugeMetricFamily(
-            "dask_worker_latency_seconds",
-            "Latency of worker connection.",
-            value=self.worker.latency,
-        )
-
-        # all metrics using digests require crick to be installed
-        # the following metrics will export NaN, if the corresponding digests are None
-        if self.crick_available:
-            yield GaugeMetricFamily(
-                "dask_worker_tick_duration_median_seconds",
-                "Median tick duration at worker.",
-                value=self.worker.digests["tick-duration"].components[1].quantile(50),
-            )
-
-            yield GaugeMetricFamily(
-                "dask_worker_task_duration_median_seconds",
-                "Median task runtime at worker.",
-                value=self.worker.digests["task-duration"].components[1].quantile(50),
-            )
-
-            yield GaugeMetricFamily(
-                "dask_worker_transfer_bandwidth_median_bytes",
-                "Bandwidth for transfer at worker in Bytes.",
-                value=self.worker.digests["transfer-bandwidth"]
-                .components[1]
-                .quantile(50),
-            )
-
-
-class PrometheusHandler(RequestHandler):
-    _initialized = False
-
-    def __init__(self, *args, **kwargs):
-        import prometheus_client
-
-        super(PrometheusHandler, self).__init__(*args, **kwargs)
-
-        if PrometheusHandler._initialized:
-            return
-
-        prometheus_client.REGISTRY.register(_PrometheusCollector(self.server))
-
-        PrometheusHandler._initialized = True
-
-    def get(self):
-        import prometheus_client
-
-        self.write(prometheus_client.generate_latest())
-        self.set_header("Content-Type", "text/plain; version=0.0.4")
-
-
-class HealthHandler(RequestHandler):
-    def get(self):
-        self.write("ok")
-        self.set_header("Content-Type", "text/plain")
-
-
-routes = [
-    (r"metrics", PrometheusHandler),
-    (r"health", HealthHandler),
-    (r"main", redirect("/status")),
-]
-
-
-def get_handlers(server):
-    return [(url, cls, {"server": server}) for url, cls in routes]
diff --git a/distributed/diagnostics/graph_layout.py b/distributed/diagnostics/graph_layout.py
index c81c6edcafe..a348d2e04ee 100644
--- a/distributed/diagnostics/graph_layout.py
+++ b/distributed/diagnostics/graph_layout.py
@@ -7,7 +7,7 @@ class GraphLayout(SchedulerPlugin):
     This assigns (x, y) locations to all tasks quickly and dynamically as new
     tasks are added.  This scales to a few thousand nodes.
 
-    It is commonly used with distributed/bokeh/scheduler.py::GraphPlot, which
+    It is commonly used with distributed/bokeh/scheduler.py::TaskGraph, which
     is rendered at /graph on the diagnostic dashboard.
     """
 
@@ -113,7 +113,7 @@ def transition(self, key, start, finish, *args, **kwargs):
     def reset_index(self):
         """ Reset the index and refill new and new_edges
 
-        From time to time GraphPlot wants to remove invisible nodes and reset
+        From time to time TaskGraph wants to remove invisible nodes and reset
         all of its indices.  This helps.
         """
         self.new = []

From 159e6c2eba15b2a6bba9cabccbc994bcb81a8be7 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Tue, 15 Oct 2019 21:01:54 -0500
Subject: [PATCH 0506/1550] bump version to 2.6.0

---
 docs/source/changelog.rst | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index 5b6288885fc..41496953f66 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,21 @@
 Changelog
 =========
 
+2.6.0 - 2019-10-15
+------------------
+
+- Refactor dashboard module (:pr:`3138`) `Jacob Tomlinson`_
+- Use ``setuptools.find_packages`` in ``setup.py`` (:pr:`3150`) `Matthew Rocklin`_
+- Move death timeout logic up to ``Node.start`` (:pr:`3115`) `Matthew Rocklin`_
+- Only include metric in ``WorkerTable`` if it is a scalar (:pr:`3140`) `Matthew Rocklin`_
+- Add ``Nanny(config={...})`` keyword (:pr:`3134`) `Matthew Rocklin`_
+- Xfail ``test_worksapce_concurrency`` on Python 3.6 (:pr:`3132`) `Matthew Rocklin`_
+- Extend Worker plugin API with transition method (:pr:`2994`) `matthieubulte`_
+- Raise exception if the user passes in unused keywords to ``Client`` (:pr:`3117`) `Jonathan De Troye`_
+- Move new ``SSHCluster`` to top level (:pr:`3128`) `Matthew Rocklin`_
+- Bump dask dependency (:pr:`3124`) `Jim Crist`_
+
+
 2.5.2 - 2019-10-04
 ------------------
 
@@ -1315,3 +1330,5 @@ significantly without many new features.
 .. _`Arpit Solanki`: https://github.com/arpit1997
 .. _`Gil Forsyth`: https://github.com/gforsyth
 .. _`Philipp Rudiger`: https://github.com/philippjfr
+.. _`Jonathan De Troye`: https://github.com/detroyejr
+.. _`matthieubulte`: https://github.com/matthieubulte

From e7a2e6d41e0b719866769713d8f41cb5fcfbf6e8 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Wed, 16 Oct 2019 10:59:43 -0500
Subject: [PATCH 0507/1550] Adds badges to README.rst [skip ci] (#3152)

---
 README.rst | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/README.rst b/README.rst
index b6f0edd604f..3d9c02915fc 100644
--- a/README.rst
+++ b/README.rst
@@ -1,7 +1,20 @@
 Distributed
 ===========
 
-A library for distributed computation.  See documentation_ for more details.
+|Build Status| |Doc Status| |Gitter| |Version Status| |NumFOCUS|
 
+A library for distributed computation.  See documentation_ for more details.
 
-.. _documentation: https://distributed.readthedocs.io/en/latest
+.. _documentation: https://distributed.dask.org
+.. |Build Status| image:: https://travis-ci.org/dask/distributed.svg?branch=master
+   :target: https://travis-ci.org/dask/distributed
+.. |Doc Status| image:: https://readthedocs.org/projects/distributed/badge/?version=latest
+   :target: https://distributed.dask.org
+   :alt: Documentation Status
+.. |Gitter| image:: https://badges.gitter.im/Join%20Chat.svg
+   :alt: Join the chat at https://gitter.im/dask/dask
+   :target: https://gitter.im/dask/dask?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge
+.. |Version Status| image:: https://img.shields.io/pypi/v/distributed.svg
+   :target: https://pypi.python.org/pypi/distributed/
+.. |NumFOCUS| image:: https://img.shields.io/badge/powered%20by-NumFOCUS-orange.svg?style=flat&colorA=E1523D&colorB=007D8A
+   :target: https://www.numfocus.org/

From 8261e93dd98d233c8a5b262f4389b365c42697ee Mon Sep 17 00:00:00 2001
From: Gil Forsyth <gforsyth@users.noreply.github.com>
Date: Thu, 17 Oct 2019 08:44:07 -0400
Subject: [PATCH 0508/1550] Don't overwrite `self.address` if it is present
 (#3153)

Testing out the new `SSHCluster` I was unable to connect to an IP that turned
out to be `None` -- the call to `super().__init__()` was overwriting
`self.address` with `None` by default.  Quick one-line fix to check if the
attribute has already been declared in a child class.
---
 distributed/deploy/spec.py     |  2 +-
 distributed/tests/test_spec.py | 18 ++++++++++++++++++
 2 files changed, 19 insertions(+), 1 deletion(-)
 create mode 100644 distributed/tests/test_spec.py

diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index 11ff3b44322..72cae01e85c 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -29,7 +29,7 @@ class ProcessInterface:
     """
 
     def __init__(self, scheduler=None, name=None):
-        self.address = None
+        self.address = getattr(self, "address", None)
         self.external_address = None
         self.lock = asyncio.Lock()
         self.status = "created"
diff --git a/distributed/tests/test_spec.py b/distributed/tests/test_spec.py
new file mode 100644
index 00000000000..38719661fc9
--- /dev/null
+++ b/distributed/tests/test_spec.py
@@ -0,0 +1,18 @@
+from distributed.deploy.spec import ProcessInterface
+
+
+def test_address_default_none():
+    p = ProcessInterface()
+    assert p.address is None
+
+
+def test_child_address_persists():
+    class Child(ProcessInterface):
+        def __init__(self, address=None):
+            self.address = address
+            super().__init__()
+
+    c = Child()
+    assert c.address is None
+    c = Child("localhost")
+    assert c.address == "localhost"

From 4a7d16cf4ee6f528bbd1fd79dfbf550cdb3792f4 Mon Sep 17 00:00:00 2001
From: darindf <darin@omegasoft.org>
Date: Mon, 21 Oct 2019 13:22:54 -0700
Subject: [PATCH 0509/1550] Removed outdated references to debug scheduler and
 worker bokeh pages. (#3160)

---
 docs/source/web.rst | 21 +--------------------
 1 file changed, 1 insertion(+), 20 deletions(-)

diff --git a/docs/source/web.rst b/docs/source/web.rst
index c73838c13dd..cfef4902c35 100644
--- a/docs/source/web.rst
+++ b/docs/source/web.rst
@@ -19,31 +19,12 @@ a normal web page in real time.  This web interface is launched by default
 wherever the scheduler is launched if the scheduler machine has Bokeh_
 installed (``conda install bokeh -c bokeh``).
 
-List of Servers
----------------
-
-There are a few sets of diagnostic pages served at different ports:
+These diagnostic pages are:
 
 *   Main Scheduler pages at ``http://scheduler-address:8787``.  These pages,
     particularly the ``/status`` page are the main page that most people
     associate with Dask.  These pages are served from a separate standalone
     Bokeh server application running in a separate process.
-*   Debug Scheduler pages at ``http://scheduler-address:8788``.  These pages
-    have more detailed diagnostic information about the scheduler.  They are
-    more often used by developers than by users, but may still be of interest
-    to the performance-conscious.  These pages run from inside the scheduler
-    process, and so compete for resources with the main scheduler.
-*   Debug Worker pages for each worker at ``http://worker-address:8789``.
-    These pages have detailed diagnostic information about the worker.  Like the
-    diagnostic scheduler pages they are of more utility to developers or to
-    people looking to understand the performance of their underlying cluster.  If
-    port 8789 is unavailable (for example it is in use by another worker) then a
-    random port is chosen.  A list of all ports can be obtained from looking at
-    the service ports for each worker in the result of calling
-    ``client.scheduler_info()``
-
-The rest of this document will be about the main pages at
-``http://scheduler-address:8787``.
 
 The available pages are ``http://scheduler-address:8787/<page>/`` where ``<page>`` is one of
 

From 7cb76f57701db615ecea18976388c9110009e3a1 Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Tue, 22 Oct 2019 13:19:13 +0100
Subject: [PATCH 0510/1550] Update CONTRIBUTING.md (#3159)

Ref dask/community#17
---
 CONTRIBUTING.md | 30 ++----------------------------
 1 file changed, 2 insertions(+), 28 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index cd35ad7c572..ab4175a59fe 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -1,29 +1,3 @@
-For more information, see https://docs.dask.org/en/latest/develop.html#contributing-to-code
+Dask is a community maintained project. We welcome contributions in the form of bug reports, documentation, code, design proposals, and more. 
 
-
-## Style
-Distributed conforms with the [flake8] and [black] styles. To make sure your
-code conforms with these styles, run
-
-``` shell
-$ pip install black flake8
-$ cd path/to/distributed
-$ black distributed
-$ flake8 distributed
-```
-
-[flake8]:http://flake8.pycqa.org/en/latest/
-[black]:https://github.com/python/black
-
-## Docstrings
-
-Dask Distributed roughly follows the [numpydoc] standard. More information is
-available at https://docs.dask.org/en/latest/develop.html#docstrings.
-
-[numpydoc]:https://github.com/numpy/numpy/blob/master/doc/HOWTO_DOCUMENT.rst.txt
-
-## Tests
-
-Dask employs extensive unit tests to ensure correctness of code both for today
-and for the future. Test coverage is expected for all code contributions. More
-detail is at https://docs.dask.org/en/latest/develop.html#test
+For general information on how to contribute see https://docs.dask.org/en/latest/develop.html.

From 70bed6af661610abf3693cce66ee70b614aa50d2 Mon Sep 17 00:00:00 2001
From: darindf <darin@omegasoft.org>
Date: Tue, 22 Oct 2019 16:04:22 -0700
Subject: [PATCH 0511/1550] Add Prometheus metric for a worker's executing
 tasks count (#3163)

---
 distributed/dashboard/worker.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/distributed/dashboard/worker.py b/distributed/dashboard/worker.py
index 4d635388512..99b27557694 100644
--- a/distributed/dashboard/worker.py
+++ b/distributed/dashboard/worker.py
@@ -53,6 +53,7 @@ def collect(self):
             "dask_worker_tasks", "Number of tasks at worker.", labels=["state"]
         )
         tasks.add_metric(["stored"], len(self.worker.data))
+        tasks.add_metric(["executing"], len(self.worker.executing))
         tasks.add_metric(["ready"], len(self.worker.ready))
         tasks.add_metric(["waiting"], len(self.worker.waiting_for_data))
         tasks.add_metric(["serving"], len(self.worker._comms))

From 21e1dcdba13b1c2a4cb78ef3022d6acbe8bdc243 Mon Sep 17 00:00:00 2001
From: darindf <darin@omegasoft.org>
Date: Wed, 23 Oct 2019 12:08:06 -0700
Subject: [PATCH 0512/1550] Updated Prometheus documentation (#3165)

* Updated Prometheus documentation

* Changed formatting

* fixup
---
 docs/source/prometheus.rst | 58 ++++++++++++++++++++++----------------
 1 file changed, 33 insertions(+), 25 deletions(-)

diff --git a/docs/source/prometheus.rst b/docs/source/prometheus.rst
index 5000858a045..8d6759ad1fa 100644
--- a/docs/source/prometheus.rst
+++ b/docs/source/prometheus.rst
@@ -8,29 +8,37 @@ scheduler and worker metrics in a prometheus text based format. Metrics are avai
 
 Available metrics are as following
 
-+---------------------------------------------+----------------------------------------------+
-| Metric name                                 | Description                                  |
-+=========================+===================+==============================================+
-| dask_scheduler_workers                      | Number of workers connected.                 |
-+---------------------------------------------+----------------------------------------------+
-| dask_scheduler_clients                      | Number of clients connected.                 |
-+---------------------------------------------+----------------------------------------------+
-| dask_scheduler_received_tasks               | Number of tasks received at scheduler        |
-+---------------------------------------------+----------------------------------------------+
-| dask_scheduler_unrunnable_tasks             | Number of unrunnable tasks at scheduler      |
-+---------------------------------------------+----------------------------------------------+
-| dask_worker_tasks                           | Number of tasks at worker.                   |
-+---------------------------------------------+----------------------------------------------+
-| dask_worker_connections                     | Number of task connections to other workers. |
-+---------------------------------------------+----------------------------------------------+
-| dask_worker_threads                         | Number of worker threads.                    |
-+---------------------------------------------+----------------------------------------------+
-| dask_worker_latency_seconds                 | Latency of worker connection.                |
-+---------------------------------------------+----------------------------------------------+
-| dask_worker_tick_duration_median_seconds    | Median tick duration at worker.              |
-+---------------------------------------------+----------------------------------------------+
-| dask_worker_task_duration_median_seconds    | Median task runtime at worker.               |
-+---------------------------------------------+----------------------------------------------+
-| dask_worker_transfer_bandwidth_median_bytes | Bandwidth for transfer at worker in Bytes.   |
-+---------------------------------------------+----------------------------------------------+
++---------------------------------------------+------------------------------------------------+-----------+--------+
+| Metric name                                 | Description                                    | Scheduler | Worker |
++=========================+===================+================================================+===========+========+
+| python_gc_objects_collected_total           | Objects collected during gc.                   |    Yes    |  Yes   |
++---------------------------------------------+------------------------------------------------+-----------+--------+
+| python_gc_objects_uncollectable_total       | Uncollectable object found during GC.          |    Yes    |  Yes   |
++---------------------------------------------+------------------------------------------------+-----------+--------+
+| python_gc_collections_total                 | Number of times this generation was collected. |    Yes    |  Yes   |
++---------------------------------------------+------------------------------------------------+-----------+--------+
+| python_info                                 | Python platform information.                   |    Yes    |  Yes   |
++---------------------------------------------+------------------------------------------------+-----------+--------+
+| dask_scheduler_workers                      | Number of workers connected.                   |    Yes    |        |
++---------------------------------------------+------------------------------------------------+-----------+--------+
+| dask_scheduler_clients                      | Number of clients connected.                   |    Yes    |        |
++---------------------------------------------+------------------------------------------------+-----------+--------+
+| dask_scheduler_received_tasks               | Number of tasks received at scheduler.         |    Yes    |        |
++---------------------------------------------+------------------------------------------------+-----------+--------+
+| dask_scheduler_unrunnable_tasks             | Number of unrunnable tasks at scheduler.       |    Yes    |        |
++---------------------------------------------+------------------------------------------------+-----------+--------+
+| dask_worker_tasks                           | Number of tasks at worker.                     |           |  Yes   |
++---------------------------------------------+------------------------------------------------+-----------+--------+
+| dask_worker_connections                     | Number of task connections to other workers.   |           |  Yes   |
++---------------------------------------------+------------------------------------------------+-----------+--------+
+| dask_worker_threads                         | Number of worker threads.                      |           |  Yes   |
++---------------------------------------------+------------------------------------------------+-----------+--------+
+| dask_worker_latency_seconds                 | Latency of worker connection.                  |           |  Yes   |
++---------------------------------------------+------------------------------------------------+-----------+--------+
+| dask_worker_tick_duration_median_seconds    | Median tick duration at worker.                |           |  Yes   |
++---------------------------------------------+------------------------------------------------+-----------+--------+
+| dask_worker_task_duration_median_seconds    | Median task runtime at worker.                 |           |  Yes   |
++---------------------------------------------+------------------------------------------------+-----------+--------+
+| dask_worker_transfer_bandwidth_median_bytes | Bandwidth for transfer at worker in Bytes.     |           |  Yes   |
++---------------------------------------------+------------------------------------------------+-----------+--------+
 

From 876bca0384ba4e0eef8bbf600cf9b89ac99faa99 Mon Sep 17 00:00:00 2001
From: Peter Andreas Entschev <peter@entschev.com>
Date: Thu, 24 Oct 2019 13:56:10 +0200
Subject: [PATCH 0513/1550] Fix Numba serialization when strides is None
 (#3166)

---
 distributed/protocol/numba.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/distributed/protocol/numba.py b/distributed/protocol/numba.py
index ddf43adc182..9b33660e2bd 100644
--- a/distributed/protocol/numba.py
+++ b/distributed/protocol/numba.py
@@ -18,9 +18,18 @@ def serialize_numba_ndarray(x):
 @cuda_deserialize.register(numba.cuda.devicearray.DeviceNDArray)
 def deserialize_numba_ndarray(header, frames):
     (frame,) = frames
+    shape = header["shape"]
+    strides = header["strides"]
+
+    # Starting with __cuda_array_interface__ version 2, strides can be None,
+    # meaning the array is C-contiguous, so we have to calculate it.
+    if strides is None:
+        itemsize = np.dtype(header["typestr"]).itemsize
+        strides = tuple((np.cumprod((1,) + shape[:0:-1]) * itemsize).tolist())
+
     arr = numba.cuda.devicearray.DeviceNDArray(
-        header["shape"],
-        header["strides"],
+        shape,
+        strides,
         np.dtype(header["typestr"]),
         gpu_data=numba.cuda.as_cuda_array(frame).gpu_data,
     )

From 97fbaae0bbcfc91fccaaa43db4952bff312bcc12 Mon Sep 17 00:00:00 2001
From: SimonBoothroyd <simon.boothroyd@choderalab.org>
Date: Fri, 25 Oct 2019 10:35:41 -0400
Subject: [PATCH 0514/1550] Await cluster in Adaptive.recommendations (#3168)

Fixes #3154

This fixes adaptive scaling when we've lost all workers in some cases.
---
 distributed/deploy/adaptive.py                |  8 +++++
 distributed/deploy/adaptive_core.py           |  1 +
 distributed/deploy/tests/test_spec_cluster.py | 30 +++++++++++++++++++
 3 files changed, 39 insertions(+)

diff --git a/distributed/deploy/adaptive.py b/distributed/deploy/adaptive.py
index b8c3429a505..f173e36a396 100644
--- a/distributed/deploy/adaptive.py
+++ b/distributed/deploy/adaptive.py
@@ -113,6 +113,14 @@ async def target(self):
             target_duration=self.target_duration
         )
 
+    async def recommendations(self, target: int) -> dict:
+        if len(self.plan) != len(self.requested):
+            # Ensure that the number of planned and requested workers
+            # are in sync before making recommendations.
+            await self.cluster
+
+        return await super(Adaptive, self).recommendations(target)
+
     async def workers_to_close(self, target: int):
         """
         Determine which, if any, workers should potentially be removed from
diff --git a/distributed/deploy/adaptive_core.py b/distributed/deploy/adaptive_core.py
index db50f109ce3..44a708aca38 100644
--- a/distributed/deploy/adaptive_core.py
+++ b/distributed/deploy/adaptive_core.py
@@ -178,6 +178,7 @@ async def adapt(self) -> None:
         self._adapting = True
 
         try:
+
             target = await self.safe_target()
             recommendations = await self.recommendations(target)
 
diff --git a/distributed/deploy/tests/test_spec_cluster.py b/distributed/deploy/tests/test_spec_cluster.py
index 1c8a01e98ce..db78b66269e 100644
--- a/distributed/deploy/tests/test_spec_cluster.py
+++ b/distributed/deploy/tests/test_spec_cluster.py
@@ -1,5 +1,6 @@
 import asyncio
 import re
+from time import sleep
 
 import dask
 from dask.distributed import SpecCluster, Worker, Client, Scheduler, Nanny
@@ -131,6 +132,35 @@ async def test_scale(cleanup):
         assert len(cluster.workers) == 2
 
 
+@pytest.mark.slow
+@pytest.mark.asyncio
+async def test_adaptive_killed_worker(cleanup):
+    with dask.config.set({"distributed.deploy.lost-worker-timeout": 0.1}):
+
+        async with SpecCluster(
+            asynchronous=True,
+            worker={"cls": Nanny, "options": {"nthreads": 1}},
+            scheduler={"cls": Scheduler, "options": {"port": 0}},
+        ) as cluster:
+
+            async with Client(cluster, asynchronous=True) as client:
+
+                cluster.adapt(minimum=1, maximum=1)
+
+                # Scale up a cluster with 1 worker.
+                while len(cluster.workers) != 1:
+                    await asyncio.sleep(0.01)
+
+                future = client.submit(sleep, 0.1)
+
+                # Kill the only worker.
+                [worker_id] = cluster.workers
+                await cluster.workers[worker_id].kill()
+
+                # Wait for the worker to re-spawn and finish sleeping.
+                await future.result(timeout=5)
+
+
 @pytest.mark.asyncio
 async def test_unexpected_closed_worker(cleanup):
     worker = {"cls": Worker, "options": {"nthreads": 1}}

From 888675a2451f0908f2eb18a07ae48a42a9f4fe0d Mon Sep 17 00:00:00 2001
From: Jim Crist <jcrist@users.noreply.github.com>
Date: Fri, 25 Oct 2019 12:41:41 -0500
Subject: [PATCH 0515/1550] Support automatic TLS (#3164)

This adds support for automatically securing cluster communication with
TLS. This can be useful for situations where you don't have existing
credentials.

This required the following changes:

- ``Security`` objects now support either paths or contents for all
`key`/`cert`/`ca` fields. Due to limitations in Python's ``ssl`` module,
if contents are provided they must be written to a temporary directory
before being loaded back in. We make sure to use secure methods for
doing this. We also change the ``__repr__`` to not show the raw cert
values in the case they're stored in memory.

- ``Security`` objects now have a classmethod ``temporary``, which can
be used to create temporary credentials using self-signed certs. This
requires ``cryptography`` to be installed. Most environments will
already have ``cryptography``, so this isn't a huge new dependency.

```python
>>> sec = Security.temporary()
```

- Both ``Client`` and ``LocalCluster`` now support passing in
``security=True``, which will generate temporary credentials
automatically for use with that cluster. This api could be supported by
other cluster managers, but for now we restrict to ``LocalCluster``
only.

```python
>>> client = Client(security=True)
>>> client
<Client: 'tls://127.0.0.1:52616' processes=4 threads=8, memory=17.18 GB>
```
---
 distributed/client.py                  |  17 +++-
 distributed/deploy/local.py            |  15 +++-
 distributed/deploy/tests/test_local.py |  31 +++++--
 distributed/security.py                | 112 +++++++++++++++++++++++--
 distributed/tests/test_security.py     |  39 +++++++++
 5 files changed, 192 insertions(+), 22 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index ca7ca431c90..9fe32c58df0 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -523,8 +523,10 @@ class Client(Node):
         Claim this scheduler as the global dask scheduler
     scheduler_file: string (optional)
         Path to a file with scheduler information if available
-    security: (optional)
-        Optional security information
+    security: Security or bool, optional
+        Optional security information. If creating a local cluster can also
+        pass in ``True``, in which case temporary self-signed credentials will
+        be created automatically.
     asynchronous: bool (False by default)
         Set to True if using this client within async/await functions or within
         Tornado gen.coroutines.  Otherwise this should remain False for normal
@@ -659,8 +661,15 @@ def __init__(
             if security is None:
                 security = getattr(self.cluster, "security", None)
 
-        self.security = security or Security()
-        assert isinstance(self.security, Security)
+        if security is None:
+            security = Security()
+        elif security is True:
+            security = Security.temporary()
+            self._startup_kwargs["security"] = security
+        elif not isinstance(security, Security):
+            raise TypeError("security must be a Security object")
+
+        self.security = security
 
         if name == "worker":
             self.connection_args = self.security.get_connection_args("worker")
diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index 29c344f6719..fd1430baa21 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -56,7 +56,10 @@ class LocalCluster(SpecCluster):
         like ``['feed', 'run_function']``
     service_kwargs: Dict[str, Dict]
         Extra keywords to hand to the running services
-    security : Security
+    security : Security or bool, optional
+        Configures communication security in this cluster. Can be a security
+        object, or True. If True, temporary self-signed credentials will
+        be created automatically.
     protocol: str (optional)
         Protocol to use like ``tcp://``, ``tls://``, ``inproc://``
         This defaults to sensible choice given other keyword arguments like
@@ -122,7 +125,15 @@ def __init__(
 
         self.status = None
         self.processes = processes
-        security = security or Security()
+
+        if security is None:
+            # Falsey values load the default configuration
+            security = Security()
+        elif security is True:
+            # True indicates self-signed temporary credentials should be used
+            security = Security.temporary()
+        elif not isinstance(security, Security):
+            raise TypeError("security must be a Security object")
 
         if protocol is None:
             if host and "://" in host:
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index 7a340a9c6f8..452a5795ad7 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -246,6 +246,15 @@ def test_Client_twice(loop):
             assert c.cluster.scheduler.port != f.cluster.scheduler.port
 
 
+@pytest.mark.asyncio
+async def test_client_constructor_with_temporary_security(cleanup):
+    async with Client(
+        security=True, silence_logs=False, dashboard_address=None, asynchronous=True
+    ) as c:
+        assert c.cluster.scheduler_address.startswith("tls")
+        assert c.security == c.cluster.security
+
+
 @pytest.mark.asyncio
 async def test_defaults(cleanup):
     async with LocalCluster(
@@ -695,10 +704,12 @@ def test_adapt_then_manual(loop):
                 assert time() < start + 5
 
 
-def test_local_tls(loop):
-    from distributed.utils_test import tls_only_security
-
-    security = tls_only_security()
+@pytest.mark.parametrize("temporary", [True, False])
+def test_local_tls(loop, temporary):
+    if temporary:
+        security = True
+    else:
+        security = tls_only_security()
     with LocalCluster(
         n_workers=0,
         scheduler_port=8786,
@@ -712,7 +723,7 @@ def test_local_tls(loop):
             loop,
             assert_can_connect_from_everywhere_4,
             c.scheduler.port,
-            connection_args=security.get_connection_args("client"),
+            connection_args=c.security.get_connection_args("client"),
             protocol="tls",
             timeout=3,
         )
@@ -722,7 +733,7 @@ def test_local_tls(loop):
             loop,
             assert_cannot_connect,
             addr="tcp://127.0.0.1:%d" % c.scheduler.port,
-            connection_args=security.get_connection_args("client"),
+            connection_args=c.security.get_connection_args("client"),
             exception_class=RuntimeError,
         )
 
@@ -977,8 +988,12 @@ async def test_repr(cleanup):
 
 
 @pytest.mark.asyncio
-async def test_capture_security(cleanup):
-    security = tls_only_security()
+@pytest.mark.parametrize("temporary", [True, False])
+async def test_capture_security(cleanup, temporary):
+    if temporary:
+        security = True
+    else:
+        security = tls_only_security()
     async with LocalCluster(
         n_workers=0,
         silence_logs=False,
diff --git a/distributed/security.py b/distributed/security.py
index a42cbeef646..6b7d87b2715 100644
--- a/distributed/security.py
+++ b/distributed/security.py
@@ -1,3 +1,7 @@
+import datetime
+import tempfile
+import os
+
 try:
     import ssl
 except ImportError:
@@ -76,6 +80,67 @@ def __init__(self, **kwargs):
         self._set_field(kwargs, "tls_worker_key", "distributed.comm.tls.worker.key")
         self._set_field(kwargs, "tls_worker_cert", "distributed.comm.tls.worker.cert")
 
+    @classmethod
+    def temporary(cls):
+        """Create a new temporary Security object.
+
+        This creates a new self-signed key/cert pair suitable for securing
+        communication for all roles in a Dask cluster. These keys/certs exist
+        only in memory, and are stored in this object.
+
+        This method requires the library ``cryptography`` be installed.
+        """
+        try:
+            from cryptography import x509
+            from cryptography.hazmat.backends import default_backend
+            from cryptography.hazmat.primitives import hashes
+            from cryptography.hazmat.primitives import serialization
+            from cryptography.hazmat.primitives.asymmetric import rsa
+            from cryptography.x509.oid import NameOID
+        except ImportError:
+            raise ImportError(
+                "Using `Security.temporary` requires `cryptography`, please "
+                "install it using either pip or conda"
+            )
+        key = rsa.generate_private_key(
+            public_exponent=65537, key_size=2048, backend=default_backend()
+        )
+        key_contents = key.private_bytes(
+            encoding=serialization.Encoding.PEM,
+            format=serialization.PrivateFormat.PKCS8,
+            encryption_algorithm=serialization.NoEncryption(),
+        ).decode()
+
+        dask_internal = x509.Name(
+            [x509.NameAttribute(NameOID.COMMON_NAME, "dask-internal")]
+        )
+        altnames = x509.SubjectAlternativeName([x509.DNSName("dask-internal")])
+        now = datetime.datetime.utcnow()
+        cert = (
+            x509.CertificateBuilder()
+            .subject_name(dask_internal)
+            .issuer_name(dask_internal)
+            .add_extension(altnames, critical=False)
+            .public_key(key.public_key())
+            .serial_number(x509.random_serial_number())
+            .not_valid_before(now)
+            .not_valid_after(now + datetime.timedelta(days=365))
+            .sign(key, hashes.SHA256(), default_backend())
+        )
+
+        cert_contents = cert.public_bytes(serialization.Encoding.PEM).decode()
+
+        return cls(
+            require_encryption=True,
+            tls_ca_file=cert_contents,
+            tls_client_key=key_contents,
+            tls_client_cert=cert_contents,
+            tls_scheduler_key=key_contents,
+            tls_scheduler_cert=cert_contents,
+            tls_worker_key=key_contents,
+            tls_worker_cert=cert_contents,
+        )
+
     def _set_field(self, kwargs, field, config_name):
         if field in kwargs:
             out = kwargs[field]
@@ -84,12 +149,16 @@ def _set_field(self, kwargs, field, config_name):
         setattr(self, field, out)
 
     def __repr__(self):
-        items = sorted((k, getattr(self, k)) for k in self.__slots__)
-        return (
-            "Security("
-            + ", ".join("%s=%r" % (k, v) for k, v in items if v is not None)
-            + ")"
-        )
+        keys = sorted(self.__slots__)
+        items = []
+        for k in keys:
+            val = getattr(self, k)
+            if val is not None:
+                if isinstance(val, str) and "\n" in val:
+                    items.append((k, "..."))
+                else:
+                    items.append((k, repr(val)))
+        return "Security(" + ", ".join("%s=%s" % (k, v) for k, v in items) + ")"
 
     def get_tls_config_for_role(self, role):
         """
@@ -106,14 +175,41 @@ def get_tls_config_for_role(self, role):
 
     def _get_tls_context(self, tls, purpose):
         if tls.get("ca_file") and tls.get("cert"):
-            ctx = ssl.create_default_context(purpose=purpose, cafile=tls["ca_file"])
+            ca = tls["ca_file"]
+            cert_path = cert = tls["cert"]
+            key_path = key = tls.get("key")
+
+            if "\n" in ca:
+                ctx = ssl.create_default_context(purpose=purpose, cadata=ca)
+            else:
+                ctx = ssl.create_default_context(purpose=purpose, cafile=ca)
+
+            cert_in_memory = "\n" in cert
+            key_in_memory = key is not None and "\n" in key
+            if cert_in_memory or key_in_memory:
+                with tempfile.TemporaryDirectory() as tempdir:
+                    if cert_in_memory:
+                        cert_path = os.path.join(tempdir, "dask.crt")
+                        with open(cert_path, "w") as f:
+                            f.write(cert)
+                    if key_in_memory:
+                        key_path = os.path.join(tempdir, "dask.pem")
+                        with open(key_path, "w") as f:
+                            f.write(key)
+                    ctx.load_cert_chain(cert_path, key_path)
+            else:
+                ctx.load_cert_chain(cert_path, key_path)
+
+            # Bidirectional authentication
             ctx.verify_mode = ssl.CERT_REQUIRED
+
             # We expect a dedicated CA for the cluster and people using
             # IP addresses rather than hostnames
             ctx.check_hostname = False
-            ctx.load_cert_chain(tls["cert"], tls.get("key"))
+
             if tls.get("ciphers"):
                 ctx.set_ciphers(tls.get("ciphers"))
+
             return ctx
 
     def get_connection_args(self, role):
diff --git a/distributed/tests/test_security.py b/distributed/tests/test_security.py
index bfc8358acf1..7496c037ae7 100644
--- a/distributed/tests/test_security.py
+++ b/distributed/tests/test_security.py
@@ -379,3 +379,42 @@ def check_encryption_error():
                 handle_comm,
                 connection_args=sec2.get_listen_args("scheduler"),
             )
+
+
+def test_temporary_credentials():
+    sec = Security.temporary()
+    sec_repr = repr(sec)
+    fields = ["tls_ca_file"]
+    fields.extend(
+        "tls_%s_%s" % (role, kind)
+        for role in ["client", "scheduler", "worker"]
+        for kind in ["key", "cert"]
+    )
+    for f in fields:
+        val = getattr(sec, f)
+        assert "\n" in val
+        assert val not in sec_repr
+
+
+@gen_test()
+def test_tls_temporary_credentials_functional():
+    pytest.importorskip("cryptography")
+
+    @gen.coroutine
+    def handle_comm(comm):
+        peer_addr = comm.peer_address
+        assert peer_addr.startswith("tls://")
+        yield comm.write("hello")
+        yield comm.close()
+
+    sec = Security.temporary()
+
+    with listen(
+        "tls://", handle_comm, connection_args=sec.get_listen_args("scheduler")
+    ) as listener:
+        comm = yield connect(
+            listener.contact_address, connection_args=sec.get_connection_args("worker")
+        )
+        msg = yield comm.read()
+        assert msg == "hello"
+        comm.abort()

From c45023af13591e8502de1c68d4b07834eec8273d Mon Sep 17 00:00:00 2001
From: Tom Augspurger <TomAugspurger@users.noreply.github.com>
Date: Mon, 28 Oct 2019 14:21:05 -0500
Subject: [PATCH 0516/1550] Avoid swamping high-memory workers with data
 requests (#3071)

Reduces memory pressure on high-memory workers when many other workers
are requesting data from the high-memory worker. We throttle the number
of responses we'll fulfill simultaneously.
---
 distributed/nanny.py            |  1 +
 distributed/tests/test_nanny.py | 56 ++++++++++++++++++++++++++++++++-
 distributed/worker.py           | 16 +++++++++-
 3 files changed, 71 insertions(+), 2 deletions(-)

diff --git a/distributed/nanny.py b/distributed/nanny.py
index 83ca2ebbf80..b21974d0257 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -357,6 +357,7 @@ def memory_monitor(self):
         except (ProcessLookupError, psutil.NoSuchProcess, psutil.AccessDenied):
             return
         frac = memory / self.memory_limit
+
         if self.memory_terminate_fraction and frac > self.memory_terminate_fraction:
             logger.warning(
                 "Worker exceeded %d%% memory budget. Restarting",
diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
index d54cf4e3b14..952d9cb8c52 100644
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -15,7 +15,7 @@
 
 import dask
 from distributed.diagnostics import SchedulerPlugin
-from distributed import Nanny, rpc, Scheduler, Worker, Client
+from distributed import Nanny, rpc, Scheduler, Worker, Client, wait
 from distributed.core import CommClosedError
 from distributed.metrics import time
 from distributed.protocol.pickle import dumps
@@ -290,6 +290,60 @@ def leak():
         assert "memory" in out.lower()
 
 
+@gen_cluster(
+    nthreads=[("127.0.0.1", 1)] * 8,
+    client=True,
+    Worker=Nanny,
+    worker_kwargs={"memory_limit": 2e8},
+    timeout=20,
+    clean_kwargs={"threads": False},
+)
+async def test_nanny_throttle(c, s, *workers):
+    # Verify that get_data requests are throttled when the worker
+    # with the data is at high-memory by
+    # 1. Allocation some data on a worker
+    # 2. Pausing that worker
+    # 3. Requesting data from that worker from many other workers
+    a = workers[0]
+    proc = a.process.pid
+    size = 1000
+
+    def data(size):
+        return b"0" * size
+
+    def patch(dask_worker):
+        # Patch paused and memory_monitor on the one worker
+        # This is is very fragile, since a refactor of memory_monitor to
+        # remove _memory_monitoring will break this test.
+        dask_worker._memory_monitoring = True
+        dask_worker.paused = True
+
+    def check(dask_worker):
+        return dask_worker.paused
+
+    futures = [
+        c.submit(data, size, workers=[a.worker_address], pure=False) for i in range(4)
+    ]
+    await wait(futures)
+    await c.run(patch, workers=[a.worker_address])
+    paused = await c.run(check, workers=[a.worker_address])
+    assert paused[a.worker_address]
+
+    await c.run(lambda: logging.getLogger("distributed.worker").setLevel(logging.DEBUG))
+    # Cluster is in the correct state, now for the test.
+    n = len(workers)
+    result = c.map(
+        lambda x, i: x[i],
+        [futures[0]] * n,
+        range(n),
+        workers=[w.worker_address for w in workers[1:]],
+    )
+    await result[0]
+    wlogs = await c.get_worker_logs(workers=[a.worker_address])
+    wlogs = "\n".join(x[1] for x in wlogs[a.worker_address])
+    assert "throttling" in wlogs.lower()
+
+
 @gen_cluster(nthreads=[], client=True)
 def test_avoid_memory_monitor_if_zero_limit(c, s):
     nanny = yield Nanny(s.address, loop=s.loop, memory_limit=0)
diff --git a/distributed/worker.py b/distributed/worker.py
index 12dfe3fe178..822e9677cf1 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -1156,10 +1156,24 @@ async def get_data(
         ):
             max_connections = max_connections * 2
 
+        if self.paused:
+            max_connections = 1
+            throttle_msg = " Throttling outgoing connections because worker is paused."
+        else:
+            throttle_msg = ""
+
         if (
             max_connections is not False
-            and self.outgoing_current_count > max_connections
+            and self.outgoing_current_count >= max_connections
         ):
+            logger.debug(
+                "Worker %s has too many open connections to respond to data request from %s (%d/%d).%s",
+                self.address,
+                who,
+                self.outgoing_current_count,
+                max_connections,
+                throttle_msg,
+            )
             return {"status": "busy"}
 
         self.outgoing_current_count += 1

From ec1ffaa6086171ff21acd3ed8d879f293d2dd9b0 Mon Sep 17 00:00:00 2001
From: Peter Andreas Entschev <peter@entschev.com>
Date: Tue, 29 Oct 2019 16:20:40 +0100
Subject: [PATCH 0517/1550] Update UCX variables to use sockcm by default
 (#3177)

---
 distributed/comm/ucx.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/distributed/comm/ucx.py b/distributed/comm/ucx.py
index 77f65c661e8..cb3b93fbced 100644
--- a/distributed/comm/ucx.py
+++ b/distributed/comm/ucx.py
@@ -21,7 +21,8 @@
 
 os.environ.setdefault("UCX_RNDV_SCHEME", "put_zcopy")
 os.environ.setdefault("UCX_MEMTYPE_CACHE", "n")
-os.environ.setdefault("UCX_TLS", "tcp,rc,cuda_copy,cuda_ipc")
+os.environ.setdefault("UCX_TLS", "tcp,sockcm,rc,cuda_copy,cuda_ipc")
+os.environ.setdefault("UCX_SOCKADDR_TLS_PRIORITY", "sockcm")
 
 logger = logging.getLogger(__name__)
 MAX_MSG_LOG = 23

From 40d58b2a51f61a89a65db5af3f262d80bc80948f Mon Sep 17 00:00:00 2001
From: Peter Andreas Entschev <peter@entschev.com>
Date: Wed, 30 Oct 2019 00:12:17 +0100
Subject: [PATCH 0518/1550] Get protocol in Nanny/Worker from scheduler address
 (#3175)

---
 distributed/nanny.py             |  5 +++++
 distributed/tests/test_worker.py | 14 ++++++++++++++
 distributed/worker.py            |  5 +++++
 3 files changed, 24 insertions(+)

diff --git a/distributed/nanny.py b/distributed/nanny.py
index b21974d0257..6e58271c33a 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -106,6 +106,11 @@ def __init__(
         else:
             self.scheduler_addr = coerce_to_address((scheduler_ip, scheduler_port))
 
+        if protocol is None:
+            protocol_address = self.scheduler_addr.split("://")
+            if len(protocol_address) == 2:
+                protocol = protocol_address[0]
+
         if ncores is not None:
             warnings.warn("the ncores= parameter has moved to nthreads=")
             nthreads = ncores
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 53aac46216a..4b9c1ace01f 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -1497,6 +1497,20 @@ async def test_interface_async(loop, Worker):
                 assert all("127.0.0.1" == d["host"] for d in info["workers"].values())
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("Worker", [Worker, Nanny])
+async def test_protocol_from_scheduler_address(Worker):
+    ucp = pytest.importorskip("ucp")
+
+    async with Scheduler(protocol="ucx") as s:
+        assert s.address.startswith("ucx://")
+        async with Worker(s.address) as w:
+            assert w.address.startswith("ucx://")
+            async with Client(s.address, asynchronous=True) as c:
+                info = c.scheduler_info()
+                assert info["address"].startswith("ucx://")
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize("Worker", [Worker, Nanny])
 async def test_worker_listens_on_same_interface_by_default(Worker):
diff --git a/distributed/worker.py b/distributed/worker.py
index 822e9677cf1..fec0444ba4e 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -440,6 +440,11 @@ def __init__(
             scheduler_addr = coerce_to_address((scheduler_ip, scheduler_port))
         self.contact_address = contact_address
 
+        if protocol is None:
+            protocol_address = scheduler_addr.split("://")
+            if len(protocol_address) == 2:
+                protocol = protocol_address[0]
+
         # Target interface on which we contact the scheduler by default
         # TODO: it is unfortunate that we special-case inproc here
         if not host and not interface and not scheduler_addr.startswith("inproc://"):

From 9429ffe735e3767b427fd0b18aaa29d6a7da8513 Mon Sep 17 00:00:00 2001
From: darindf <darin@omegasoft.org>
Date: Tue, 29 Oct 2019 16:13:33 -0700
Subject: [PATCH 0519/1550] Add worker and tasks state for Prometheus data
 collection (#3174)

---
 distributed/dashboard/scheduler.py | 30 +++++++++++++++++-------------
 docs/source/prometheus.rst         |  4 +---
 2 files changed, 18 insertions(+), 16 deletions(-)

diff --git a/distributed/dashboard/scheduler.py b/distributed/dashboard/scheduler.py
index 1117fe7bd72..8928d468c5e 100644
--- a/distributed/dashboard/scheduler.py
+++ b/distributed/dashboard/scheduler.py
@@ -239,26 +239,30 @@ def __init__(self, server):
     def collect(self):
         from prometheus_client.core import GaugeMetricFamily
 
-        yield GaugeMetricFamily(
-            "dask_scheduler_workers",
-            "Number of workers connected.",
-            value=len(self.server.workers),
-        )
         yield GaugeMetricFamily(
             "dask_scheduler_clients",
             "Number of clients connected.",
             value=len(self.server.clients),
         )
-        yield GaugeMetricFamily(
-            "dask_scheduler_received_tasks",
-            "Number of tasks received at scheduler",
-            value=len(self.server.tasks),
+
+        tasks = GaugeMetricFamily(
+            "dask_scheduler_workers",
+            "Number of workers known by scheduler.",
+            labels=["state"],
         )
-        yield GaugeMetricFamily(
-            "dask_scheduler_unrunnable_tasks",
-            "Number of unrunnable tasks at scheduler",
-            value=len(self.server.unrunnable),
+        tasks.add_metric(["connected"], len(self.server.workers))
+        tasks.add_metric(["saturated"], len(self.server.saturated))
+        tasks.add_metric(["idle"], len(self.server.idle))
+        yield tasks
+
+        tasks = GaugeMetricFamily(
+            "dask_scheduler_tasks",
+            "Number of tasks known by scheduler.",
+            labels=["state"],
         )
+        tasks.add_metric(["received"], len(self.server.tasks))
+        tasks.add_metric(["unrunnable"], len(self.server.unrunnable))
+        yield tasks
 
 
 class PrometheusHandler(RequestHandler):
diff --git a/docs/source/prometheus.rst b/docs/source/prometheus.rst
index 8d6759ad1fa..097335ee0d7 100644
--- a/docs/source/prometheus.rst
+++ b/docs/source/prometheus.rst
@@ -23,9 +23,7 @@ Available metrics are as following
 +---------------------------------------------+------------------------------------------------+-----------+--------+
 | dask_scheduler_clients                      | Number of clients connected.                   |    Yes    |        |
 +---------------------------------------------+------------------------------------------------+-----------+--------+
-| dask_scheduler_received_tasks               | Number of tasks received at scheduler.         |    Yes    |        |
-+---------------------------------------------+------------------------------------------------+-----------+--------+
-| dask_scheduler_unrunnable_tasks             | Number of unrunnable tasks at scheduler.       |    Yes    |        |
+| dask_scheduler_tasks                        | Number of tasks at scheduler.                  |    Yes    |        |
 +---------------------------------------------+------------------------------------------------+-----------+--------+
 | dask_worker_tasks                           | Number of tasks at worker.                     |           |  Yes   |
 +---------------------------------------------+------------------------------------------------+-----------+--------+

From 8b7c47d7e93ee74d543f29ada1c7863bbad78310 Mon Sep 17 00:00:00 2001
From: "Mads R. B. Kristensen" <madsbk@gmail.com>
Date: Fri, 1 Nov 2019 05:30:28 +0100
Subject: [PATCH 0520/1550] Use async def functions for offload to/from_frames
 (#3171)

---
 distributed/client.py                  |  2 +-
 distributed/comm/tcp.py                |  7 +++---
 distributed/comm/ucx.py                |  3 +--
 distributed/comm/utils.py              | 16 +++++--------
 distributed/core.py                    | 18 ++++++++------
 distributed/deploy/cluster.py          |  2 +-
 distributed/deploy/tests/test_local.py | 16 ++++++++++++-
 distributed/scheduler.py               |  3 +--
 distributed/tests/test_core.py         |  5 ++--
 distributed/tests/test_nanny.py        |  2 +-
 distributed/tests/test_scheduler.py    | 16 +++++++++++++
 distributed/tests/test_worker.py       | 33 +++++++++++---------------
 distributed/utils.py                   |  6 ++---
 distributed/utils_comm.py              |  4 ++--
 distributed/worker.py                  |  2 +-
 15 files changed, 79 insertions(+), 56 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 9fe32c58df0..11aaacdf044 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -1298,7 +1298,7 @@ async def _close(self, fast=False):
                 with ignoring(TimeoutError):
                     await gen.with_timeout(timedelta(seconds=2), list(coroutines))
             with ignoring(AttributeError):
-                self.scheduler.close_rpc()
+                await self.scheduler.close_rpc()
             self.scheduler = None
 
         self.status = "closed"
diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py
index d0322e151d7..f0a24fe4fb7 100644
--- a/distributed/comm/tcp.py
+++ b/distributed/comm/tcp.py
@@ -213,14 +213,13 @@ async def read(self, deserializers=None):
                 raise CommClosedError("aborted stream on truncated data")
             return msg
 
-    @gen.coroutine
-    def write(self, msg, serializers=None, on_error="message"):
+    async def write(self, msg, serializers=None, on_error="message"):
         stream = self.stream
         bytes_since_last_yield = 0
         if stream is None:
             raise CommClosedError
 
-        frames = yield to_frames(
+        frames = await to_frames(
             msg,
             serializers=serializers,
             on_error=on_error,
@@ -247,7 +246,7 @@ def write(self, msg, serializers=None, on_error="message"):
                     future = stream.write(frame)
                     bytes_since_last_yield += nbytes(frame)
                     if bytes_since_last_yield > 32e6:
-                        yield future
+                        await future
                         bytes_since_last_yield = 0
         except StreamClosedError as e:
             stream = None
diff --git a/distributed/comm/ucx.py b/distributed/comm/ucx.py
index cb3b93fbced..fede1c91371 100644
--- a/distributed/comm/ucx.py
+++ b/distributed/comm/ucx.py
@@ -221,7 +221,6 @@ async def connect(self, address: str, deserialize=True, **connection_args) -> UC
 
 
 class UCXListener(Listener):
-    # MAX_LISTENERS 256 in ucx-py
     prefix = UCXConnector.prefix
     comm_class = UCXConnector.comm_class
     encrypted = UCXConnector.encrypted
@@ -251,7 +250,7 @@ async def serve_forever(client_ep):
             ucx = UCX(
                 client_ep,
                 local_addr=self.address,
-                peer_addr=self.address,  # TODO: https://github.com/Akshay-Venkatesh/ucx-py/issues/111
+                peer_addr=self.address,
                 deserialize=self.deserialize,
             )
             if self.comm_handler:
diff --git a/distributed/comm/utils.py b/distributed/comm/utils.py
index 70cd2b4cd27..80e1f163785 100644
--- a/distributed/comm/utils.py
+++ b/distributed/comm/utils.py
@@ -1,8 +1,6 @@
 import logging
 import socket
 
-from tornado import gen
-
 from .. import protocol
 from ..utils import get_ip, get_ipv6, nbytes, offload
 
@@ -16,8 +14,7 @@
 FRAME_OFFLOAD_THRESHOLD = 10 * 1024 ** 2  # 10 MB
 
 
-@gen.coroutine
-def to_frames(msg, serializers=None, on_error="message", context=None):
+async def to_frames(msg, serializers=None, on_error="message", context=None):
     """
     Serialize a message into a list of Distributed protocol frames.
     """
@@ -34,13 +31,12 @@ def _to_frames():
             logger.exception(e)
             raise
 
-    res = yield offload(_to_frames)
+    res = await offload(_to_frames)
 
-    raise gen.Return(res)
+    return res
 
 
-@gen.coroutine
-def from_frames(frames, deserialize=True, deserializers=None):
+async def from_frames(frames, deserialize=True, deserializers=None):
     """
     Unserialize a list of Distributed protocol frames.
     """
@@ -61,11 +57,11 @@ def _from_frames():
             raise
 
     if deserialize and size > FRAME_OFFLOAD_THRESHOLD:
-        res = yield offload(_from_frames)
+        res = await offload(_from_frames)
     else:
         res = _from_frames()
 
-    raise gen.Return(res)
+    return res
 
 
 def get_tcp_server_address(tcp_server):
diff --git a/distributed/core.py b/distributed/core.py
index 32b509dc170..716a7b035e2 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -641,25 +641,29 @@ async def live_comm(self):
         return comm
 
     def close_comms(self):
-        @gen.coroutine
-        def _close_comm(comm):
+        async def _close_comm(comm):
             # Make sure we tell the peer to close
             try:
                 if not comm.closed():
-                    yield comm.write({"op": "close", "reply": False})
-                    yield comm.close()
+                    await comm.write({"op": "close", "reply": False})
+                    await comm.close()
             except EnvironmentError:
                 comm.abort()
 
+        tasks = []
         for comm in list(self.comms):
             if comm and not comm.closed():
                 # IOLoop.current().add_callback(_close_comm, comm)
                 task = asyncio.ensure_future(_close_comm(comm))
+                tasks.append(task)
         for comm in list(self._created):
             if comm and not comm.closed():
                 # IOLoop.current().add_callback(_close_comm, comm)
                 task = asyncio.ensure_future(_close_comm(comm))
+                tasks.append(task)
+
         self.comms.clear()
+        return tasks
 
     def __getattr__(self, key):
         async def send_recv_from_rpc(**kwargs):
@@ -685,13 +689,13 @@ def close_rpc(self):
         if self.status != "closed":
             rpc.active.discard(self)
         self.status = "closed"
-        self.close_comms()
+        return asyncio.gather(*self.close_comms())
 
     def __enter__(self):
         return self
 
     def __exit__(self, *args):
-        self.close_rpc()
+        asyncio.ensure_future(self.close_rpc())
 
     def __del__(self):
         if self.status != "closed":
@@ -744,7 +748,7 @@ async def send_recv_from_rpc(**kwargs):
 
         return send_recv_from_rpc
 
-    def close_rpc(self):
+    async def close_rpc(self):
         pass
 
     # For compatibility with rpc()
diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
index 033f6877684..2631fb502df 100644
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -74,7 +74,7 @@ async def _close(self):
 
         for pc in self.periodic_callbacks.values():
             pc.stop()
-        self.scheduler_comm.close_rpc()
+        await self.scheduler_comm.close_rpc()
 
         self.status = "closed"
 
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index 452a5795ad7..e73ccd4721f 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -1,3 +1,4 @@
+import asyncio
 from functools import partial
 import gc
 import subprocess
@@ -455,7 +456,7 @@ def test_silent_startup():
 
         if __name__ == "__main__":
             with LocalCluster(1, dashboard_address=None, scheduler_port=0):
-                sleep(1.5)
+                sleep(.1)
         """
 
     out = subprocess.check_output(
@@ -1004,3 +1005,16 @@ async def test_capture_security(cleanup, temporary):
     ) as cluster:
         async with Client(cluster, asynchronous=True) as client:
             assert client.security == cluster.security
+
+
+@pytest.mark.asyncio
+@pytest.mark.skipif(
+    sys.version_info < (3, 7), reason="asyncio.all_tasks not implemented"
+)
+async def test_no_danglng_asyncio_tasks(cleanup):
+    start = asyncio.all_tasks()
+    async with LocalCluster(asynchronous=True, processes=False):
+        await asyncio.sleep(0.01)
+
+    tasks = asyncio.all_tasks()
+    assert tasks == start
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index f3d3fc92ea8..4319584735e 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -2642,8 +2642,7 @@ async def restart(self, client=None, timeout=3):
                     "timeout.  Continuuing with restart process"
                 )
             finally:
-                for nanny in nannies:
-                    nanny.close_rpc()
+                await asyncio.gather(*[nanny.close_rpc() for nanny in nannies])
 
             await self.start()
 
diff --git a/distributed/tests/test_core.py b/distributed/tests/test_core.py
index cbde7ac240b..99c07226c48 100644
--- a/distributed/tests/test_core.py
+++ b/distributed/tests/test_core.py
@@ -299,13 +299,14 @@ def test_rpc_inproc():
     yield check_rpc("inproc://", None)
 
 
-def test_rpc_inputs():
+@pytest.mark.asyncio
+async def test_rpc_inputs():
     L = [rpc("127.0.0.1:8884"), rpc(("127.0.0.1", 8884)), rpc("tcp://127.0.0.1:8884")]
 
     assert all(r.address == "tcp://127.0.0.1:8884" for r in L), L
 
     for r in L:
-        r.close_rpc()
+        await r.close_rpc()
 
 
 async def check_rpc_message_lifetime(*listen_args):
diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
index 952d9cb8c52..70497bf7909 100644
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -115,7 +115,7 @@ def test_nanny_process_failure(c, s):
     assert not os.path.exists(second_dir)
     assert not os.path.exists(first_dir)
     assert first_dir != n.worker_dir
-    ww.close_rpc()
+    yield ww.close_rpc()
     s.stop()
 
 
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 4b56a4d084f..f57dbfb9e07 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -1,3 +1,4 @@
+import asyncio
 import cloudpickle
 import pickle
 from collections import defaultdict
@@ -1688,3 +1689,18 @@ def test_get_task_duration():
         assert s.get_task_duration(ts_pref2_2) == 0.5  # default
         assert len(s.unknown_durations) == 1
         assert len(s.unknown_durations["prefix_2"]) == 2
+
+
+@pytest.mark.asyncio
+@pytest.mark.skipif(
+    sys.version_info < (3, 7), reason="asyncio.all_tasks not implemented"
+)
+async def test_no_danglng_asyncio_tasks(cleanup):
+    start = asyncio.all_tasks()
+    async with Scheduler(port=0) as s:
+        async with Worker(s.address, name="0") as a:
+            async with Client(s.address, asynchronous=True) as c:
+                await asyncio.sleep(0.01)
+
+    tasks = asyncio.all_tasks()
+    assert tasks == start
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 4b9c1ace01f..8b81e3afbe6 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -173,7 +173,7 @@ def dont_test_delete_data_with_missing_worker(c, a, b):
     assert not c.has_what[bad]
     assert not c.has_what[a.address]
 
-    cc.close_rpc()
+    yield cc.close_rpc()
 
 
 @gen_cluster(client=True)
@@ -998,32 +998,27 @@ def test_worker_fds(s):
 
 
 @gen_cluster(nthreads=[])
-def test_service_hosts_match_worker(s):
+async def test_service_hosts_match_worker(s):
     pytest.importorskip("bokeh")
     from distributed.dashboard import BokehWorker
 
-    services = {("dashboard", ":0"): BokehWorker}
-
-    w = yield Worker(
+    async with Worker(
         s.address, services={("dashboard", ":0"): BokehWorker}, host="tcp://0.0.0.0"
-    )
-    sock = first(w.services["dashboard"].server._http._sockets.values())
-    assert sock.getsockname()[0] in ("::", "0.0.0.0")
-    yield w.close()
+    ) as w:
+        sock = first(w.services["dashboard"].server._http._sockets.values())
+        assert sock.getsockname()[0] in ("::", "0.0.0.0")
 
-    w = yield Worker(
+    async with Worker(
         s.address, services={("dashboard", ":0"): BokehWorker}, host="tcp://127.0.0.1"
-    )
-    sock = first(w.services["dashboard"].server._http._sockets.values())
-    assert sock.getsockname()[0] in ("::", "0.0.0.0")
-    yield w.close()
+    ) as w:
+        sock = first(w.services["dashboard"].server._http._sockets.values())
+        assert sock.getsockname()[0] in ("::", "0.0.0.0")
 
-    w = yield Worker(
+    async with Worker(
         s.address, services={("dashboard", 0): BokehWorker}, host="tcp://127.0.0.1"
-    )
-    sock = first(w.services["dashboard"].server._http._sockets.values())
-    assert sock.getsockname()[0] == "127.0.0.1"
-    yield w.close()
+    ) as w:
+        sock = first(w.services["dashboard"].server._http._sockets.values())
+        assert sock.getsockname()[0] == "127.0.0.1"
 
 
 @gen_cluster(nthreads=[])
diff --git a/distributed/utils.py b/distributed/utils.py
index fbac950df43..c7ab77bcbd8 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -1365,6 +1365,6 @@ def is_valid_xml(text):
 weakref.finalize(_offload_executor, _offload_executor.shutdown)
 
 
-@gen.coroutine
-def offload(fn, *args, **kwargs):
-    return (yield _offload_executor.submit(fn, *args, **kwargs))
+async def offload(fn, *args, **kwargs):
+    loop = asyncio.get_event_loop()
+    return await loop.run_in_executor(_offload_executor, fn, *args, **kwargs)
diff --git a/distributed/utils_comm.py b/distributed/utils_comm.py
index 53504d11939..e2072189be0 100644
--- a/distributed/utils_comm.py
+++ b/distributed/utils_comm.py
@@ -84,7 +84,7 @@ async def gather_from_workers(who_has, rpc, close=True, serializers=None, who=No
                     response.update(r["data"])
         finally:
             for r in rpcs.values():
-                r.close_rpc()
+                await r.close_rpc()
 
         bad_addresses |= {v for k, v in rev.items() if k not in response}
         results.update(response)
@@ -148,7 +148,7 @@ async def scatter_to_workers(nthreads, data, rpc=rpc, report=True, serializers=N
         )
     finally:
         for r in rpcs.values():
-            r.close_rpc()
+            await r.close_rpc()
 
     nbytes = merge(o["nbytes"] for o in out)
 
diff --git a/distributed/worker.py b/distributed/worker.py
index fec0444ba4e..d1c35f68eb5 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -1068,7 +1068,7 @@ async def close(
                             address=self.contact_address, safe=safe
                         ),
                     )
-            self.scheduler.close_rpc()
+            await self.scheduler.close_rpc()
             self._workdir.release()
 
             for k, v in self.services.items():

From 5025b124bc7627837844e50d6f9c4b6df7ee36af Mon Sep 17 00:00:00 2001
From: "Mads R. B. Kristensen" <madsbk@gmail.com>
Date: Tue, 5 Nov 2019 16:48:19 +0100
Subject: [PATCH 0521/1550] Subprocesses inherit the global dask config (#3192)

---
 distributed/cli/dask_scheduler.py    |  7 +++++++
 distributed/cli/dask_worker.py       |  6 ++++++
 distributed/deploy/ssh.py            | 17 ++++++++++++---
 distributed/deploy/tests/test_ssh.py | 19 +++++++++++++++++
 distributed/process.py               | 16 ++++++++++++--
 distributed/tests/test_client.py     | 12 +++++++++++
 distributed/utils.py                 | 31 ++++++++++++++++++++++++++++
 7 files changed, 103 insertions(+), 5 deletions(-)

diff --git a/distributed/cli/dask_scheduler.py b/distributed/cli/dask_scheduler.py
index 29de26d7b4d..0951b8c3d27 100755
--- a/distributed/cli/dask_scheduler.py
+++ b/distributed/cli/dask_scheduler.py
@@ -9,6 +9,7 @@
 import warnings
 
 import click
+import dask
 
 from tornado.ioloop import IOLoop
 
@@ -16,6 +17,7 @@
 from distributed.preloading import validate_preload_argv
 from distributed.security import Security
 from distributed.cli.utils import check_python_3, install_signal_handlers
+from distributed.utils import deserialize_for_cli
 from distributed.proctitle import (
     enable_proctitle_on_children,
     enable_proctitle_on_current,
@@ -174,6 +176,11 @@ def main(
         }
     )
 
+    if "DASK_INTERNAL_INHERIT_CONFIG" in os.environ:
+        config = deserialize_for_cli(os.environ["DASK_INTERNAL_INHERIT_CONFIG"])
+        # Update the global config given priority to the existing global config
+        dask.config.update(dask.config.global_config, config, priority="old")
+
     if not host and (tls_ca_file or tls_cert or tls_key):
         host = "tls://"
 
diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
index fb32fc2e882..0f307398e04 100755
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -20,6 +20,7 @@
     enable_proctitle_on_children,
     enable_proctitle_on_current,
 )
+from distributed.utils import deserialize_for_cli
 
 from toolz import valmap
 from tornado.ioloop import IOLoop, TimeoutError
@@ -359,6 +360,11 @@ def del_pid_file():
     with ignoring(TypeError, ValueError):
         name = int(name)
 
+    if "DASK_INTERNAL_INHERIT_CONFIG" in os.environ:
+        config = deserialize_for_cli(os.environ["DASK_INTERNAL_INHERIT_CONFIG"])
+        # Update the global config given priority to the existing global config
+        dask.config.update(dask.config.global_config, config, priority="old")
+
     nannies = [
         t(
             scheduler,
diff --git a/distributed/deploy/ssh.py b/distributed/deploy/ssh.py
index 8aa3cc17d97..673cb7ba717 100644
--- a/distributed/deploy/ssh.py
+++ b/distributed/deploy/ssh.py
@@ -4,10 +4,13 @@
 import warnings
 import weakref
 
+import dask
+
 from .spec import SpecCluster, ProcessInterface
 from ..utils import cli_keywords
 from ..scheduler import Scheduler as _Scheduler
 from ..worker import Worker as _Worker
+from ..utils import serialize_for_cli
 
 logger = logging.getLogger(__name__)
 
@@ -86,6 +89,8 @@ async def start(self):
         self.proc = await self.connection.create_process(
             " ".join(
                 [
+                    'DASK_INTERNAL_INHERIT_CONFIG="%s"'
+                    % serialize_for_cli(dask.config.global_config),
                     sys.executable,
                     "-m",
                     self.worker_module,
@@ -112,7 +117,7 @@ async def start(self):
 
 
 class Scheduler(Process):
-    """ A Remote Dask Scheduler controled by SSH
+    """ A Remote Dask Scheduler controlled by SSH
 
     Parameters
     ----------
@@ -141,7 +146,13 @@ async def start(self):
 
         self.proc = await self.connection.create_process(
             " ".join(
-                [sys.executable, "-m", "distributed.cli.dask_scheduler"]
+                [
+                    'DASK_INTERNAL_INHERIT_CONFIG="%s"'
+                    % serialize_for_cli(dask.config.global_config),
+                    sys.executable,
+                    "-m",
+                    "distributed.cli.dask_scheduler",
+                ]
                 + cli_keywords(self.kwargs, cls=_Scheduler)
             )
         )
@@ -191,7 +202,7 @@ def SSHCluster(
     The SSHCluster function deploys a Dask Scheduler and Workers for you on a
     set of machine addresses that you provide.  The first address will be used
     for the scheduler while the rest will be used for the workers (feel free to
-    repeat the first hostname if you want to have the scheudler and worker
+    repeat the first hostname if you want to have the scheduler and worker
     co-habitate one machine.)
 
     You may configure the scheduler and workers by passing
diff --git a/distributed/deploy/tests/test_ssh.py b/distributed/deploy/tests/test_ssh.py
index 3124af4f177..376b0eae3a4 100644
--- a/distributed/deploy/tests/test_ssh.py
+++ b/distributed/deploy/tests/test_ssh.py
@@ -2,6 +2,7 @@
 
 pytest.importorskip("asyncssh")
 
+import dask
 from dask.distributed import Client
 from distributed.deploy.ssh import SSHCluster
 from distributed.utils_test import loop  # noqa: F401
@@ -55,3 +56,21 @@ def test_defer_to_old(loop):
             from distributed.deploy.old_ssh import SSHCluster as OldSSHCluster
 
             assert isinstance(c, OldSSHCluster)
+
+
+@pytest.mark.asyncio
+async def test_config_inherited_by_subprocess(loop):
+    def f(x):
+        return dask.config.get("foo") + 1
+
+    with dask.config.set(foo=100):
+        async with SSHCluster(
+            ["127.0.0.1"] * 2,
+            connect_options=dict(known_hosts=None),
+            asynchronous=True,
+            scheduler_options={"port": 0, "idle_timeout": "5s"},
+            worker_options={"death_timeout": "5s"},
+        ) as cluster:
+            async with Client(cluster, asynchronous=True) as client:
+                result = await client.submit(f, 1)
+                assert result == 101
diff --git a/distributed/process.py b/distributed/process.py
index 889787fe0bf..38527ecd9ab 100644
--- a/distributed/process.py
+++ b/distributed/process.py
@@ -6,6 +6,7 @@
 import re
 import threading
 import weakref
+import dask
 
 from .utils import mp_context
 
@@ -71,7 +72,14 @@ def __init__(self, loop=None, target=None, name=None, args=(), kwargs={}):
         self._process = mp_context.Process(
             target=self._run,
             name=name,
-            args=(target, args, kwargs, parent_alive_pipe, self._keep_child_alive),
+            args=(
+                target,
+                args,
+                kwargs,
+                parent_alive_pipe,
+                self._keep_child_alive,
+                dask.config.global_config,
+            ),
         )
         _dangling.add(self._process)
         self._name = self._process.name
@@ -163,7 +171,9 @@ def reset_logger_locks():
                 handler.createLock()
 
     @classmethod
-    def _run(cls, target, args, kwargs, parent_alive_pipe, _keep_child_alive):
+    def _run(
+        cls, target, args, kwargs, parent_alive_pipe, _keep_child_alive, inherit_config
+    ):
         # On Python 2 with the fork method, we inherit the _keep_child_alive fd,
         # whether it is passed or not. Therefore, pass it unconditionally and
         # close it here, so that there are no other references to the pipe lying
@@ -176,6 +186,8 @@ def _run(cls, target, args, kwargs, parent_alive_pipe, _keep_child_alive):
         cls._immediate_exit_when_closed(parent_alive_pipe)
 
         threading.current_thread().name = "MainThread"
+        # Update the global config given priority to the existing global config
+        dask.config.update(dask.config.global_config, inherit_config, priority="old")
         target(*args, **kwargs)
 
     @classmethod
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index c4ebe92b4f1..29aad876ab3 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -5654,5 +5654,17 @@ async def test_shutdown_localcluster(cleanup):
         assert lc.scheduler.status == "closed"
 
 
+@pytest.mark.asyncio
+async def test_config_inherited_by_subprocess(cleanup):
+    def f(x):
+        return dask.config.get("foo") + 1
+
+    with dask.config.set(foo=100):
+        async with LocalCluster(n_workers=1, asynchronous=True, processes=True) as lc:
+            async with Client(lc, asynchronous=True) as c:
+                result = await c.submit(f, 1)
+                assert result == 101
+
+
 if sys.version_info >= (3, 5):
     from distributed.tests.py3_test_client import *  # noqa F401
diff --git a/distributed/utils.py b/distributed/utils.py
index c7ab77bcbd8..251e1110be8 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -24,6 +24,7 @@
 import warnings
 import weakref
 import pkgutil
+import base64
 import tblib.pickling_support
 import xml.etree.ElementTree
 
@@ -1368,3 +1369,33 @@ def is_valid_xml(text):
 async def offload(fn, *args, **kwargs):
     loop = asyncio.get_event_loop()
     return await loop.run_in_executor(_offload_executor, fn, *args, **kwargs)
+
+
+def serialize_for_cli(data):
+    """ Serialize data into a string that can be passthrough cli
+
+    Parameters
+    ----------
+    data: json-serializable object
+        The data to serialize
+    Returns
+    -------
+    serialized_data: str
+        The serialized data as a string
+    """
+    return base64.urlsafe_b64encode(json.dumps(data).encode()).decode()
+
+
+def deserialize_for_cli(data):
+    """ De-serialize data into the original object
+
+    Parameters
+    ----------
+    data: str
+        String serialied by serialize_for_cli()
+    Returns
+    -------
+    deserialized_data: obj
+        The de-serialized data
+    """
+    return json.loads(base64.urlsafe_b64decode(data.encode()).decode())

From 4687879b0aebaa7435b26e80eb0669031ff2861e Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 5 Nov 2019 09:41:01 -0800
Subject: [PATCH 0522/1550] XFail test_open_close_many_workers (#3194)

This test is causing intermittent failures, and unfortunately no one is
available to resolve it.
---
 distributed/tests/test_client.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 29aad876ab3..00abbcd25ec 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -3576,10 +3576,7 @@ def test_reconnect_timeout(c, s):
 
 @pytest.mark.slow
 @pytest.mark.skipif(WINDOWS, reason="num_fds not supported on windows")
-@pytest.mark.skipif(
-    sys.version_info[0] == 2, reason="Semaphore.acquire doesn't support timeout option"
-)
-# @pytest.mark.xfail(reason="TODO: intermittent failures")
+@pytest.mark.xfail(reason="TODO: intermittent failures")
 @pytest.mark.parametrize("worker,count,repeat", [(Worker, 100, 5), (Nanny, 10, 20)])
 def test_open_close_many_workers(loop, worker, count, repeat):
     psutil = pytest.importorskip("psutil")

From e4a0404f0ed3f853a86adf79c48a60620f3e9fb6 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Tue, 5 Nov 2019 11:49:47 -0600
Subject: [PATCH 0523/1550] Drop Python 3.5 (#3179)

---
 .travis.yml                                   |  3 +-
 appveyor.yml                                  |  6 +--
 .../setup_conda_environment.cmd               |  1 -
 distributed/core.py                           |  6 +++
 distributed/tests/test_client.py              |  5 +-
 distributed/tests/test_nanny.py               | 51 +++++++++----------
 distributed/tests/test_security.py            |  8 +--
 distributed/utils_test.py                     |  2 +-
 docs/source/install.rst                       | 10 ++--
 setup.py                                      |  3 +-
 10 files changed, 44 insertions(+), 51 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 1726cffd4f1..5d3cbf0ec0b 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -6,8 +6,7 @@ dist: trusty
 
 env:
   matrix:
-    - PYTHON=3.5.4 TESTS=true COVERAGE=true PACKAGES="python-blosc lz4" CRICK=true
-    - PYTHON=3.6 TESTS=true PACKAGES="scikit-learn lz4" TORNADO=5
+    - PYTHON=3.6 TESTS=true COVERAGE=true PACKAGES="scikit-learn lz4" TORNADO=5 CRICK=true
     - PYTHON=3.7 TESTS=true PACKAGES="scikit-learn python-snappy python-blosc" TORNADO=6
 
 matrix:
diff --git a/appveyor.yml b/appveyor.yml
index 496640b3f30..e32c48f105a 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -11,14 +11,14 @@ environment:
 
   matrix:
     # Since appveyor is quite slow, we only use a single configuration
-    - PYTHON: "3.5"
+    - PYTHON: "3.6"
       ARCH: "64"
       CONDA_ENV: testenv
 
 init:
   # Use AppVeyor's provided Miniconda: https://www.appveyor.com/docs/installed-software#python
-  - if "%ARCH%" == "64" set MINICONDA=C:\Miniconda35-x64
-  - if "%ARCH%" == "32" set MINICONDA=C:\Miniconda35
+  - if "%ARCH%" == "64" set MINICONDA=C:\Miniconda36-x64
+  - if "%ARCH%" == "32" set MINICONDA=C:\Miniconda36
   - set PATH=%MINICONDA%;%MINICONDA%/Scripts;%MINICONDA%/Library/bin;%PATH%
 
 install:
diff --git a/continuous_integration/setup_conda_environment.cmd b/continuous_integration/setup_conda_environment.cmd
index 87e37751548..5efc7358dbe 100644
--- a/continuous_integration/setup_conda_environment.cmd
+++ b/continuous_integration/setup_conda_environment.cmd
@@ -24,7 +24,6 @@ call deactivate
     cloudpickle ^
     dask ^
     dill ^
-    futures ^
     lz4 ^
     ipykernel ^
     ipywidgets ^
diff --git a/distributed/core.py b/distributed/core.py
index 716a7b035e2..66205dbfd85 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -697,6 +697,12 @@ def __enter__(self):
     def __exit__(self, *args):
         asyncio.ensure_future(self.close_rpc())
 
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, *args):
+        await self.close_rpc()
+
     def __del__(self):
         if self.status != "closed":
             rpc.active.discard(self)
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 00abbcd25ec..3d582c4fe8f 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -3322,10 +3322,7 @@ def test_get_foo_lost_keys(c, s, u, v, w):
 
 @pytest.mark.slow
 @gen_cluster(
-    client=True,
-    Worker=Nanny,
-    worker_kwargs={"death_timeout": "500ms"},
-    clean_kwargs={"threads": False, "processes": False},
+    client=True, Worker=Nanny, clean_kwargs={"threads": False, "processes": False}
 )
 def test_bad_tasks_fail(c, s, a, b):
     f = c.submit(sys.exit, 0)
diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
index 70497bf7909..fccfd2efde6 100644
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -30,33 +30,30 @@
 
 
 @gen_cluster(nthreads=[])
-def test_nanny(s):
-    n = yield Nanny(s.address, nthreads=2, loop=s.loop)
-
-    with rpc(n.address) as nn:
-        assert n.is_alive()
-        assert s.nthreads[n.worker_address] == 2
-        assert s.workers[n.worker_address].nanny == n.address
-
-        yield nn.kill()
-        assert not n.is_alive()
-        assert n.worker_address not in s.nthreads
-        assert n.worker_address not in s.workers
-
-        yield nn.kill()
-        assert not n.is_alive()
-        assert n.worker_address not in s.nthreads
-        assert n.worker_address not in s.workers
-
-        yield nn.instantiate()
-        assert n.is_alive()
-        assert s.nthreads[n.worker_address] == 2
-        assert s.workers[n.worker_address].nanny == n.address
-
-        yield nn.terminate()
-        assert not n.is_alive()
-
-    yield n.close()
+async def test_nanny(s):
+    async with Nanny(s.address, nthreads=2, loop=s.loop) as n:
+        async with rpc(n.address) as nn:
+            assert n.is_alive()
+            assert s.nthreads[n.worker_address] == 2
+            assert s.workers[n.worker_address].nanny == n.address
+
+            await nn.kill()
+            assert not n.is_alive()
+            assert n.worker_address not in s.nthreads
+            assert n.worker_address not in s.workers
+
+            await nn.kill()
+            assert not n.is_alive()
+            assert n.worker_address not in s.nthreads
+            assert n.worker_address not in s.workers
+
+            await nn.instantiate()
+            assert n.is_alive()
+            assert s.nthreads[n.worker_address] == 2
+            assert s.workers[n.worker_address].nanny == n.address
+
+            await nn.terminate()
+            assert not n.is_alive()
 
 
 @gen_cluster(nthreads=[])
diff --git a/distributed/tests/test_security.py b/distributed/tests/test_security.py
index 7496c037ae7..167abc762ae 100644
--- a/distributed/tests/test_security.py
+++ b/distributed/tests/test_security.py
@@ -196,9 +196,9 @@ def many_ciphers(ctx):
     basic_checks(ctx)
     if sys.version_info >= (3, 6):
         supported_ciphers = ctx.get_ciphers()
-        tls_12_ciphers = [c for c in supported_ciphers if c["protocol"] == "TLSv1.2"]
+        tls_12_ciphers = [c for c in supported_ciphers if "TLSv1.2" in c["description"]]
         assert len(tls_12_ciphers) == 1
-        tls_13_ciphers = [c for c in supported_ciphers if c["protocol"] == "TLSv1.3"]
+        tls_13_ciphers = [c for c in supported_ciphers if "TLSv1.3" in c["description"]]
         if len(tls_13_ciphers):
             assert len(tls_13_ciphers) == 3
 
@@ -249,9 +249,9 @@ def many_ciphers(ctx):
     basic_checks(ctx)
     if sys.version_info >= (3, 6):
         supported_ciphers = ctx.get_ciphers()
-        tls_12_ciphers = [c for c in supported_ciphers if c["protocol"] == "TLSv1.2"]
+        tls_12_ciphers = [c for c in supported_ciphers if "TLSv1.2" in c["description"]]
         assert len(tls_12_ciphers) == 1
-        tls_13_ciphers = [c for c in supported_ciphers if c["protocol"] == "TLSv1.3"]
+        tls_13_ciphers = [c for c in supported_ciphers if "TLSv1.3" in c["description"]]
         if len(tls_13_ciphers):
             assert len(tls_13_ciphers) == 3
 
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index 97cbe783318..81b196639ac 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -865,7 +865,7 @@ def test_foo(scheduler, worker1, worker2):
         nthreads = ncores
 
     worker_kwargs = merge(
-        {"memory_limit": system.MEMORY_LIMIT, "death_timeout": 5}, worker_kwargs
+        {"memory_limit": system.MEMORY_LIMIT, "death_timeout": 10}, worker_kwargs
     )
 
     def _(func):
diff --git a/docs/source/install.rst b/docs/source/install.rst
index 2ca74bb3d5f..7cf4199eecd 100644
--- a/docs/source/install.rst
+++ b/docs/source/install.rst
@@ -35,13 +35,9 @@ Notes
 -----
 
 **Note for Macports users:** There `is a known issue
-<https://trac.macports.org/ticket/50058>`_.  with python from macports that
+<https://trac.macports.org/ticket/50058>`_.  with Python from macports that
 makes executables be placed in a location that is not available by default. A
 simple solution is to extend the ``PATH`` environment variable to the location
-where python from macports install the binaries::
+where Python from macports install the binaries. For example, for Python 3.6::
 
-    $ export PATH=/opt/local/Library/Frameworks/Python.framework/Versions/3.5/bin:$PATH
-
-    or
-
-    $ export PATH=/opt/local/Library/Frameworks/Python.framework/Versions/2.7/bin:$PATH
+    $ export PATH=/opt/local/Library/Frameworks/Python.framework/Versions/3.6/bin:$PATH
diff --git a/setup.py b/setup.py
index 5d900199256..310d5322e98 100755
--- a/setup.py
+++ b/setup.py
@@ -26,7 +26,7 @@
     url="https://distributed.dask.org",
     maintainer="Matthew Rocklin",
     maintainer_email="mrocklin@gmail.com",
-    python_requires=">=3.5",
+    python_requires=">=3.6",
     license="BSD",
     package_data={
         "": ["templates/index.html", "template.html"],
@@ -46,7 +46,6 @@
         "License :: OSI Approved :: BSD License",
         "Operating System :: OS Independent",
         "Programming Language :: Python",
-        "Programming Language :: Python :: 3.5",
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
         "Topic :: Scientific/Engineering",

From bc04d7642c0439bf3757b34bd1eb802b9679ba12 Mon Sep 17 00:00:00 2001
From: "Mads R. B. Kristensen" <madsbk@gmail.com>
Date: Tue, 5 Nov 2019 21:14:42 +0100
Subject: [PATCH 0524/1550] UCX: avoid double init after fork (#3178)

---
 distributed/comm/tests/test_comms.py |   2 +-
 distributed/comm/tests/test_ucx.py   |   3 -
 distributed/comm/ucx.py              | 122 +++++++++++++++------------
 3 files changed, 70 insertions(+), 57 deletions(-)

diff --git a/distributed/comm/tests/test_comms.py b/distributed/comm/tests/test_comms.py
index 620d4b89c94..301cb4f013f 100644
--- a/distributed/comm/tests/test_comms.py
+++ b/distributed/comm/tests/test_comms.py
@@ -527,7 +527,7 @@ def client_communicate(key, delay=0):
 @gen_test()
 def test_ucx_client_server():
     pytest.importorskip("distributed.comm.ucx")
-    import ucp
+    ucp = pytest.importorskip("ucp")
 
     addr = ucp.get_address()
     yield check_client_server("ucx://" + addr)
diff --git a/distributed/comm/tests/test_ucx.py b/distributed/comm/tests/test_ucx.py
index 28348369899..e5f4a4ab79b 100644
--- a/distributed/comm/tests/test_ucx.py
+++ b/distributed/comm/tests/test_ucx.py
@@ -261,9 +261,6 @@ async def test_ping_pong_numba():
 
 @pytest.mark.parametrize("processes", [True, False])
 def test_ucx_localcluster(loop, processes):
-    if processes:
-        pytest.skip("Known bug, processes=True doesn't work currently")
-
     with LocalCluster(
         protocol="ucx",
         dashboard_address=None,
diff --git a/distributed/comm/ucx.py b/distributed/comm/ucx.py
index fede1c91371..d6ab704ab0f 100644
--- a/distributed/comm/ucx.py
+++ b/distributed/comm/ucx.py
@@ -6,6 +6,11 @@
 .. _UCX: https://github.com/openucx/ucx
 """
 import logging
+import concurrent
+import os
+
+import dask
+import numpy as np
 
 from .addressing import parse_host_port, unparse_host_port
 from .core import Comm, Connector, Listener, CommClosedError
@@ -13,41 +18,48 @@
 from .utils import ensure_concrete_host, to_frames, from_frames
 from ..utils import ensure_ip, get_ip, get_ipv6, nbytes, log_errors
 
-from tornado.ioloop import IOLoop
-import ucp
-import numpy as np
-
-import os
 
 os.environ.setdefault("UCX_RNDV_SCHEME", "put_zcopy")
 os.environ.setdefault("UCX_MEMTYPE_CACHE", "n")
-os.environ.setdefault("UCX_TLS", "tcp,sockcm,rc,cuda_copy,cuda_ipc")
+os.environ.setdefault("UCX_TLS", "all")
 os.environ.setdefault("UCX_SOCKADDR_TLS_PRIORITY", "sockcm")
 
 logger = logging.getLogger(__name__)
-MAX_MSG_LOG = 23
 
 
-# ----------------------------------------------------------------------------
-# Comm Interface
-# ----------------------------------------------------------------------------
+# In order to avoid double init when forking/spawning new processes (multiprocess),
+# we make sure only to import and initialize UCX once at first use.
+ucp = None
+cuda_array = None
+
+
+def init_once():
+    global ucp, cuda_array
+    if ucp is not None:
+        return
+
+    import ucp as _ucp
 
-# Let's find the function, `cuda_array`, to use when allocating new CUDA arrays
-try:
-    import rmm
+    ucp = _ucp
+    options = dask.config.get("ucx", default={})
+    ucp.init(options=options)
 
-    cuda_array = lambda n: rmm.device_array(n, dtype=np.uint8)
-except ImportError:
+    # Find the function, `cuda_array()`, to use when allocating new CUDA arrays
     try:
-        import numba.cuda
+        import rmm
 
-        cuda_array = lambda n: numba.cuda.device_array((n,), dtype=np.uint8)
+        cuda_array = lambda n: rmm.device_array(n, dtype=np.uint8)
     except ImportError:
+        try:
+            import numba.cuda
 
-        def cuda_array(n):
-            raise RuntimeError(
-                "In order to send/recv CUDA arrays, Numba or RMM is required"
-            )
+            cuda_array = lambda n: numba.cuda.device_array((n,), dtype=np.uint8)
+        except ImportError:
+
+            def cuda_array(n):
+                raise RuntimeError(
+                    "In order to send/recv CUDA arrays, Numba or RMM is required"
+                )
 
 
 class UCX(Comm):
@@ -84,9 +96,7 @@ class UCX(Comm):
     4. Read all the data frames.
     """
 
-    def __init__(
-        self, ep: ucp.Endpoint, local_addr: str, peer_addr: str, deserialize=True
-    ):
+    def __init__(self, ep, local_addr: str, peer_addr: str, deserialize=True):
         Comm.__init__(self)
         self._ep = ep
         if local_addr:
@@ -115,26 +125,33 @@ async def write(
         with log_errors():
             if self.closed():
                 raise CommClosedError("Endpoint is closed -- unable to send message")
+            try:
+                if serializers is None:
+                    serializers = ("cuda", "dask", "pickle", "error")
+                # msg can also be a list of dicts when sending batched messages
+                frames = await to_frames(
+                    msg, serializers=serializers, on_error=on_error
+                )
 
-            if serializers is None:
-                serializers = ("cuda", "dask", "pickle", "error")
-            # msg can also be a list of dicts when sending batched messages
-            frames = await to_frames(msg, serializers=serializers, on_error=on_error)
-
-            # Send meta data
-            await self.ep.send(np.array([len(frames)], dtype=np.uint64))
-            await self.ep.send(
-                np.array(
-                    [hasattr(f, "__cuda_array_interface__") for f in frames],
-                    dtype=np.bool,
+                # Send meta data
+                await self.ep.send(np.array([len(frames)], dtype=np.uint64))
+                await self.ep.send(
+                    np.array(
+                        [hasattr(f, "__cuda_array_interface__") for f in frames],
+                        dtype=np.bool,
+                    )
                 )
-            )
-            await self.ep.send(np.array([nbytes(f) for f in frames], dtype=np.uint64))
-            # Send frames
-            for frame in frames:
-                if nbytes(frame) > 0:
-                    await self.ep.send(frame)
-            return sum(map(nbytes, frames))
+                await self.ep.send(
+                    np.array([nbytes(f) for f in frames], dtype=np.uint64)
+                )
+                # Send frames
+                for frame in frames:
+                    if nbytes(frame) > 0:
+                        await self.ep.send(frame)
+                return sum(map(nbytes, frames))
+            except (ucp.exceptions.UCXBaseException):
+                self.abort()
+                raise CommClosedError("While writing, the connection was closed")
 
     async def read(self, deserializers=("cuda", "dask", "pickle", "error")):
         with log_errors():
@@ -152,12 +169,12 @@ async def read(self, deserializers=("cuda", "dask", "pickle", "error")):
                 await self.ep.recv(is_cudas)
                 sizes = np.empty(nframes[0], dtype=np.uint64)
                 await self.ep.recv(sizes)
-            except (ucp.exceptions.UCXCanceled, ucp.exceptions.UCXCloseError):
-                if self._ep is not None and not self._ep.closed():
-                    await self._ep.shutdown()
-                    self._ep.close()
-                self._ep = None
-                raise CommClosedError("While reading, the connection was canceled")
+            except (
+                ucp.exceptions.UCXBaseException,
+                concurrent.futures._base.CancelledError,
+            ):
+                self.abort()
+                raise CommClosedError("While reading, the connection was closed")
             else:
                 # Recv frames
                 frames = []
@@ -181,15 +198,12 @@ async def read(self, deserializers=("cuda", "dask", "pickle", "error")):
 
     async def close(self):
         if self._ep is not None:
-            if not self._ep.closed():
-                await self._ep.signal_shutdown()
-                self._ep.close()
+            await self._ep.close()
             self._ep = None
 
     def abort(self):
         if self._ep is not None:
-            logger.debug("Destroyed UCX endpoint")
-            IOLoop.current().add_callback(self._ep.signal_shutdown)
+            self._ep.abort()
             self._ep = None
 
     @property
@@ -211,6 +225,7 @@ class UCXConnector(Connector):
     async def connect(self, address: str, deserialize=True, **connection_args) -> UCX:
         logger.debug("UCXConnector.connect: %s", address)
         ip, port = parse_host_port(address)
+        init_once()
         ep = await ucp.create_endpoint(ip, port)
         return self.comm_class(
             ep,
@@ -256,6 +271,7 @@ async def serve_forever(client_ep):
             if self.comm_handler:
                 await self.comm_handler(ucx)
 
+        init_once()
         self.ucp_server = ucp.create_listener(serve_forever, port=self._input_port)
 
     def stop(self):

From 2a3d0072cd906369a6bc5a933094f4d29712783c Mon Sep 17 00:00:00 2001
From: "James A. Bednar" <jbednar@users.noreply.github.com>
Date: Wed, 6 Nov 2019 17:56:50 -0600
Subject: [PATCH 0525/1550] Silenced warning when importing while offline
 (#3203)

---
 distributed/comm/inproc.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/distributed/comm/inproc.py b/distributed/comm/inproc.py
index 5235b7535fd..e9bed986ea0 100644
--- a/distributed/comm/inproc.py
+++ b/distributed/comm/inproc.py
@@ -4,6 +4,7 @@
 import os
 import threading
 import weakref
+import warnings
 
 from tornado import locks
 from tornado.concurrent import Future
@@ -31,7 +32,11 @@ class Manager(object):
     def __init__(self):
         self.listeners = weakref.WeakValueDictionary()
         self.addr_suffixes = itertools.count(1)
-        self.ip = get_ip()
+        with warnings.catch_warnings():
+            # Avoid immediate warning for unreachable network
+            # (will still warn for other get_ip() calls when actually used)
+            warnings.simplefilter("ignore")
+            self.ip = get_ip()
         self.lock = threading.Lock()
 
     def add_listener(self, addr, listener):

From 7b46b92e46358d13e8c9926d4d99f24bb2566fee Mon Sep 17 00:00:00 2001
From: IPetrik <petrikigor@gmail.com>
Date: Thu, 7 Nov 2019 11:25:19 -0800
Subject: [PATCH 0526/1550] Adds docs to Client methods for resources, actors,
 and traverse (#2851)

---
 distributed/client.py | 49 ++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 48 insertions(+), 1 deletion(-)

diff --git a/distributed/client.py b/distributed/client.py
index 11aaacdf044..c7f921ee348 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -1429,6 +1429,15 @@ def submit(
             Higher priorities take precedence
         fifo_timeout: str timedelta (default '100ms')
             Allowed amount of time between calls to consider the same priority
+        resources: dict (defaults to {})
+            Defines the `resources` this job requires on the worker; e.g.
+            ``{'GPU': 2}``. See :doc:`worker resources <resources>` for details
+            on defining resources.
+        actor: bool (default False)
+            Whether this task should exist on the worker as a stateful actor.
+            See :doc:`actors` for additional details.
+        actors: bool (default False)
+            Alias for `actor`
 
         Examples
         --------
@@ -1531,6 +1540,9 @@ def map(
         workers: set, iterable of sets
             A set of worker hostnames on which computations may be performed.
             Leave empty to default to all workers (common case)
+        allow_other_workers: bool (defaults to False)
+            Used with `workers`. Indicates whether or not the computations
+            may be performed on workers that are not in the `workers` set(s).
         retries: int (default to 0)
             Number of allowed automatic retries if a task fails
         priority: Number
@@ -1538,6 +1550,16 @@ def map(
             Higher priorities take precedence
         fifo_timeout: str timedelta (default '100ms')
             Allowed amount of time between calls to consider the same priority
+        resources: dict (defaults to {})
+            Defines the `resources` each instance of this mapped task requires
+            on the worker; e.g. ``{'GPU': 2}``. See
+            :doc:`worker resources <resources>` for details on defining
+            resources.
+        actor: bool (default False)
+            Whether these tasks should exist on the worker as stateful actors.
+            See :doc:`actors` for additional details.
+        actors: bool (default False)
+            Alias for `actor`
         **kwargs: dict
             Extra keywords to send to the function.
             Large values will be included explicitly in the task graph.
@@ -2656,6 +2678,21 @@ def compute(
             Higher priorities take precedence
         fifo_timeout: timedelta str (defaults to '60s')
             Allowed amount of time between calls to consider the same priority
+        traverse: bool (defaults to True)
+            By default dask traverses builtin python collections looking for
+            dask objects passed to ``compute``. For large collections this can
+            be expensive. If none of the arguments contain any dask objects,
+            set ``traverse=False`` to avoid doing this traversal.
+        resources: dict (defaults to {})
+            Defines the `resources` these tasks require on the worker. Can
+            specify global resources (``{'GPU': 2}``), or per-task resources
+            (``{'x': {'GPU': 1}, 'y': {'SSD': 4}}``), but not both.
+            See :doc:`worker resources <resources>` for details on defining
+            resources.
+        actors: bool or dict (default None)
+            Whether these tasks should exist on the worker as stateful actors.
+            Specified on a global (True/False) or per-task (``{'x': True,
+            'y': False}``) basis. See :doc:`actors` for additional details.
         **kwargs:
             Options to pass to the graph optimize calls
 
@@ -2791,7 +2828,17 @@ def persist(
             Higher priorities take precedence
         fifo_timeout: timedelta str (defaults to '60s')
             Allowed amount of time between calls to consider the same priority
-        kwargs:
+        resources: dict (defaults to {})
+            Defines the `resources` these tasks require on the worker. Can
+            specify global resources (``{'GPU': 2}``), or per-task resources
+            (``{'x': {'GPU': 1}, 'y': {'SSD': 4}}``), but not both.
+            See :doc:`worker resources <resources>` for details on defining
+            resources.
+        actors: bool or dict (default None)
+            Whether these tasks should exist on the worker as stateful actors.
+            Specified on a global (True/False) or per-task (``{'x': True,
+            'y': False}``) basis. See :doc:`actors` for additional details.
+        **kwargs:
             Options to pass to the graph optimize calls
 
         Returns

From 0766d78327a4d09e0f5e015570264a451a6607fa Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 7 Nov 2019 11:25:49 -0800
Subject: [PATCH 0527/1550] Add failing test for concurrent scatter operations
 (#2244)

---
 distributed/tests/test_client.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 3d582c4fe8f..4d747d8f3ec 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -5436,6 +5436,15 @@ def test_tuple_keys(c, s, a, b):
     assert (yield future) == 3
 
 
+@gen_cluster(client=True)
+def test_multiple_scatter(c, s, a, b):
+    for i in range(5):
+        x = c.scatter(1, direct=True)
+
+    x = yield x
+    x = yield x
+
+
 @gen_cluster(client=True)
 def test_map_large_kwargs_in_graph(c, s, a, b):
     np = pytest.importorskip("numpy")

From d575ea0bd0784da2368d486f07914ae669824a07 Mon Sep 17 00:00:00 2001
From: Dave Hirschfeld <david.hirschfeld@stanwell.com>
Date: Fri, 8 Nov 2019 05:57:56 +1000
Subject: [PATCH 0528/1550] Expand async docs (#2293)

---
 .gitignore                   |  1 +
 docs/source/asynchronous.rst | 18 ++++++++++++------
 2 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/.gitignore b/.gitignore
index 2d70b7ebd7f..86ee425adff 100644
--- a/.gitignore
+++ b/.gitignore
@@ -10,6 +10,7 @@ continuous_integration/hdfs-initialized
 *.lock
 .#*
 .idea/
+.vscode/
 .pytest_cache/
 dask-worker-space/
 .vscode/
diff --git a/docs/source/asynchronous.rst b/docs/source/asynchronous.rst
index 1ffc8d4f5c3..a49788e1fbe 100644
--- a/docs/source/asynchronous.rst
+++ b/docs/source/asynchronous.rst
@@ -44,18 +44,24 @@ received information from the scheduler should now be ``await``'ed.
 
    result = await client.gather(future)
 
-If you want to reuse the same client in asynchronous and synchronous
-environments you can apply the ``asynchronous=True`` keyword at each method
-call.
+
+If you want to use an asynchronous function with a synchronous ``Client``
+(one made without the ``asynchronous=True`` keyword) then you can apply the
+``asynchronous=True`` keyword at each method call and use the ``Client.sync``
+function to run the asynchronous function:
 
 .. code-block:: python
 
+   from dask.distributed import Client
+
    client = Client()  # normal blocking client
 
    async def f():
-       futures = client.map(func, L)
-       results = await client.gather(futures, asynchronous=True)
-       return results
+       future = client.submit(lambda x: x + 1, 10)
+       result = await client.gather(future, asynchronous=True)
+       return result
+
+   client.sync(f)
 
 
 Python 2 Compatibility

From 2f46ab18372e747dcf0179341ffcdaa5900aa522 Mon Sep 17 00:00:00 2001
From: "Richard (Rick) Zamora" <rzamora217@gmail.com>
Date: Fri, 8 Nov 2019 09:28:05 -0600
Subject: [PATCH 0529/1550] Adding PatchedDeviceArray to drop stride attribute
 for cupy<7.0 (#3198)

---
 distributed/protocol/cupy.py            | 13 +++++++++++++
 distributed/protocol/tests/test_cupy.py | 17 +++++++++++++++++
 2 files changed, 30 insertions(+)

diff --git a/distributed/protocol/cupy.py b/distributed/protocol/cupy.py
index d85f37d8a1e..d15c719359c 100644
--- a/distributed/protocol/cupy.py
+++ b/distributed/protocol/cupy.py
@@ -5,6 +5,17 @@
 from .cuda import cuda_serialize, cuda_deserialize
 
 
+class PatchedCudaArrayInterface(object):
+    # TODO: This class wont be necessary
+    #       once Cupy<7.0 is no longer supported
+    def __init__(self, ary):
+        cai = ary.__cuda_array_interface__
+        cai_cupy_vsn = cupy.ndarray(0).__cuda_array_interface__["version"]
+        if cai.get("strides") is None and cai_cupy_vsn < 2:
+            cai.pop("strides", None)
+        self.__cuda_array_interface__ = cai
+
+
 @cuda_serialize.register(cupy.ndarray)
 def serialize_cupy_ndarray(x):
     # Making sure `x` is behaving
@@ -18,6 +29,8 @@ def serialize_cupy_ndarray(x):
 @cuda_deserialize.register(cupy.ndarray)
 def deserialize_cupy_array(header, frames):
     (frame,) = frames
+    if not isinstance(frame, cupy.ndarray):
+        frame = PatchedCudaArrayInterface(frame)
     arr = cupy.ndarray(
         header["shape"], dtype=header["typestr"], memptr=cupy.asarray(frame).data
     )
diff --git a/distributed/protocol/tests/test_cupy.py b/distributed/protocol/tests/test_cupy.py
index 10335d14338..57d26ae679b 100644
--- a/distributed/protocol/tests/test_cupy.py
+++ b/distributed/protocol/tests/test_cupy.py
@@ -1,4 +1,5 @@
 from distributed.protocol import serialize, deserialize
+import pickle
 import pytest
 
 cupy = pytest.importorskip("cupy")
@@ -12,3 +13,19 @@ def test_serialize_cupy(size, dtype):
     y = deserialize(header, frames, deserializers=("cuda", "dask", "pickle", "error"))
 
     assert (x == y).all()
+
+
+@pytest.mark.parametrize("dtype", ["u1", "u4", "u8", "f4"])
+def test_serialize_cupy_from_numba(dtype):
+    numba = pytest.importorskip("numba")
+    np = pytest.importorskip("numpy")
+
+    size = 10
+    x_np = np.arange(size, dtype=dtype)
+    x = numba.cuda.to_device(x_np)
+    header, frames = serialize(x, serializers=("cuda", "dask", "pickle"))
+    header["type-serialized"] = pickle.dumps(cupy.ndarray)
+
+    y = deserialize(header, frames, deserializers=("cuda", "dask", "pickle", "error"))
+
+    assert (x_np == cupy.asnumpy(y)).all()

From f9ae2243e203195dd4069bda577bd1b8edb5ebda Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 8 Nov 2019 13:07:19 -0800
Subject: [PATCH 0530/1550] bump version to 2.7.0

---
 docs/source/changelog.rst | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index 41496953f66..3cd8e3d287d 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,35 @@
 Changelog
 =========
 
+2.7.0 - 2019-11-08
+------------------
+
+This release drops support for Python 3.5
+
+-  Adds badges to README.rst [skip ci] (:pr:`3152`) `James Bourbeau`_
+-  Don't overwrite `self.address` if it is present (:pr:`3153`) `Gil Forsyth`_
+-  Remove outdated references to debug scheduler and worker bokeh pages. (:pr:`3160`) `darindf`_
+-  Update CONTRIBUTING.md (:pr:`3159`) `Jacob Tomlinson`_
+-  Add Prometheus metric for a worker's executing tasks count (:pr:`3163`) `darindf`_
+-  Update Prometheus documentation (:pr:`3165`) `darindf`_
+-  Fix Numba serialization when strides is None (:pr:`3166`) `Peter Andreas Entschev`_
+-  Await cluster in Adaptive.recommendations (:pr:`3168`) `Simon Boothroyd`_
+-  Support automatic TLS (:pr:`3164`) `Jim Crist`_
+-  Avoid swamping high-memory workers with data requests (:pr:`3071`) `Tom Augspurger`_
+-  Update UCX variables to use sockcm by default (:pr:`3177`) `Peter Andreas Entschev`_
+-  Get protocol in Nanny/Worker from scheduler address (:pr:`3175`) `Peter Andreas Entschev`_
+-  Add worker and tasks state for Prometheus data collection (:pr:`3174`) `darindf`_
+-  Use async def functions for offload to/from_frames (:pr:`3171`) `Mads R. B. Kristensen`_
+-  Subprocesses inherit the global dask config (:pr:`3192`) `Mads R. B. Kristensen`_
+-  XFail test_open_close_many_workers (:pr:`3194`) `Matthew Rocklin`_
+-  Drop Python 3.5 (:pr:`3179`) `James Bourbeau`_
+-  UCX: avoid double init after fork (:pr:`3178`) `Mads R. B. Kristensen`_
+-  Silence warning when importing while offline (:pr:`3203`) `James A. Bednar`_
+-  Adds docs to Client methods for resources, actors, and traverse (:pr:`2851`) `IPetrik`_
+-  Add test for concurrent scatter operations (:pr:`2244`) `Matthew Rocklin`_
+-  Expand async docs (:pr:`2293`) `Dave Hirschfeld`_
+-  Add PatchedDeviceArray to drop stride attribute for cupy<7.0 (:pr:`3198`) `Richard J Zamora`_
+
 2.6.0 - 2019-10-15
 ------------------
 
@@ -1332,3 +1361,7 @@ significantly without many new features.
 .. _`Philipp Rudiger`: https://github.com/philippjfr
 .. _`Jonathan De Troye`: https://github.com/detroyejr
 .. _`matthieubulte`: https://github.com/matthieubulte
+.. _`darindf`: https://github.com/darindf
+.. _`James A. Bednar`: https://github.com/jbednar
+.. _`IPetrik`: https://github.com/IPetrik
+.. _`Simon Boothroyd`: https://github.com/SimonBoothroyd

From 763a649e3e272b657d394409e74584505510f064 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 8 Nov 2019 13:18:29 -0800
Subject: [PATCH 0531/1550] Add UCX config values (#3135)

---
 distributed/comm/tests/test_ucx.py | 52 ++++++++++++++++++++++--------
 distributed/comm/ucx.py            |  9 +++---
 distributed/distributed.yaml       |  2 ++
 3 files changed, 45 insertions(+), 18 deletions(-)

diff --git a/distributed/comm/tests/test_ucx.py b/distributed/comm/tests/test_ucx.py
index e5f4a4ab79b..4c2d2d0782e 100644
--- a/distributed/comm/tests/test_ucx.py
+++ b/distributed/comm/tests/test_ucx.py
@@ -3,19 +3,22 @@
 
 ucp = pytest.importorskip("ucp")
 
-from distributed import Client
+from distributed import Client, Worker, Scheduler, wait
 from distributed.comm import ucx, listen, connect
 from distributed.comm.registry import backends, get_backend
 from distributed.comm import ucx, parse_address
 from distributed.protocol import to_serialize
 from distributed.deploy.local import LocalCluster
 from dask.dataframe.utils import assert_eq
-from distributed.utils_test import gen_test, loop, inc  # noqa: 401
+from distributed.utils_test import gen_test, loop, inc, cleanup  # noqa: 401
 
 from .test_comms import check_deserialize
 
 
-HOST = ucp.get_address()
+try:
+    HOST = ucp.get_address()
+except Exception:
+    HOST = "127.0.0.1"
 
 
 def test_registered():
@@ -225,7 +228,7 @@ async def test_ping_pong_cupy(shape):
         ),
     ],
 )
-async def test_large_cupy(n):
+async def test_large_cupy(n, cleanup):
     cupy = pytest.importorskip("cupy")
     com, serv_com = await get_comm_pair()
 
@@ -242,7 +245,7 @@ async def test_large_cupy(n):
 
 
 @pytest.mark.asyncio
-async def test_ping_pong_numba():
+async def test_ping_pong_numba(cleanup):
     np = pytest.importorskip("numpy")
     numba = pytest.importorskip("numba")
     import numba.cuda
@@ -260,18 +263,19 @@ async def test_ping_pong_numba():
 
 
 @pytest.mark.parametrize("processes", [True, False])
-def test_ucx_localcluster(loop, processes):
-    with LocalCluster(
+@pytest.mark.asyncio
+async def test_ucx_localcluster(processes, cleanup):
+    async with LocalCluster(
         protocol="ucx",
         dashboard_address=None,
         n_workers=2,
         threads_per_worker=1,
         processes=processes,
-        loop=loop,
+        asynchronous=True,
     ) as cluster:
-        with Client(cluster) as client:
+        async with Client(cluster, asynchronous=True) as client:
             x = client.submit(inc, 1)
-            x.result()
+            await x.result()
             assert x.key in cluster.scheduler.tasks
             if not processes:
                 assert any(w.data == {x.key: 2} for w in cluster.workers.values())
@@ -280,9 +284,8 @@ def test_ucx_localcluster(loop, processes):
 
 @pytest.mark.slow
 @pytest.mark.asyncio
-async def test_stress():
-    import dask.array as da
-    from distributed import wait
+async def test_stress(cleanup):
+    da = pytest.importorskip("dask.array")
 
     chunksize = "10 MB"
 
@@ -300,3 +303,26 @@ async def test_stress():
                 x = x.rechunk((-1, chunksize))
                 x = x.persist()
                 await wait(x)
+
+
+@pytest.mark.asyncio
+async def test_simple(cleanup):
+    async with Scheduler(protocol="ucx") as s:
+        async with Worker(s.address) as a:
+            async with Client(s.address, asynchronous=True) as c:
+                result = await c.submit(lambda x: x + 1, 10)
+                assert result == 11
+
+
+@pytest.mark.asyncio
+async def test_transpose(cleanup):
+    da = pytest.importorskip("dask.array")
+
+    async with Scheduler(protocol="ucx") as s:
+        async with Worker(s.address) as a, Worker(s.address) as b:
+            async with Client(s.address, asynchronous=True) as c:
+                x = da.ones((10000, 10000), chunks=(1000, 1000)).persist()
+                await x
+
+                y = (x + x.T).sum()
+                await y
diff --git a/distributed/comm/ucx.py b/distributed/comm/ucx.py
index d6ab704ab0f..bcf9b19f412 100644
--- a/distributed/comm/ucx.py
+++ b/distributed/comm/ucx.py
@@ -7,7 +7,6 @@
 """
 import logging
 import concurrent
-import os
 
 import dask
 import numpy as np
@@ -18,11 +17,11 @@
 from .utils import ensure_concrete_host, to_frames, from_frames
 from ..utils import ensure_ip, get_ip, get_ipv6, nbytes, log_errors
 
+import dask
+import ucp
+import numpy as np
 
-os.environ.setdefault("UCX_RNDV_SCHEME", "put_zcopy")
-os.environ.setdefault("UCX_MEMTYPE_CACHE", "n")
-os.environ.setdefault("UCX_TLS", "all")
-os.environ.setdefault("UCX_SOCKADDR_TLS_PRIORITY", "sockcm")
+ucp.init(options=dask.config.get("ucx"), env_takes_precedence=True)
 
 logger = logging.getLogger(__name__)
 
diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index 48484be12a6..92b7c15e157 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -122,3 +122,5 @@ distributed:
     log-length: 10000  # default length of logs to keep in memory
     log-format: '%(name)s - %(levelname)s - %(message)s'
     pdb-on-err: False       # enter debug mode on scheduling error
+
+ucx: {}

From 690363b9515230e152d0507b8f785cd598249d68 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 8 Nov 2019 18:03:23 -0800
Subject: [PATCH 0532/1550] Relax test_MultiWorker (#3210)

The repr of a class only updates periodically,
so we need to relax a check a bit.
---
 distributed/deploy/tests/test_spec_cluster.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/distributed/deploy/tests/test_spec_cluster.py b/distributed/deploy/tests/test_spec_cluster.py
index db78b66269e..19e162ca67b 100644
--- a/distributed/deploy/tests/test_spec_cluster.py
+++ b/distributed/deploy/tests/test_spec_cluster.py
@@ -430,7 +430,9 @@ async def test_MultiWorker(cleanup):
             assert len(cluster.worker_spec) == 2
             await client.wait_for_workers(4)
 
-            assert "workers=4" in repr(cluster)
+            while "workers=4" not in repr(cluster):
+                await asyncio.sleep(0.1)
+
             workers_line = re.search("(Workers.+)", cluster._widget_status()).group(1)
             assert re.match("Workers.*<td>4</td>", workers_line)
 

From 559db6721e3f32d17606aaafce70eabdbf162198 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 8 Nov 2019 18:03:32 -0800
Subject: [PATCH 0533/1550] Avoid ucp.init at import time (#3211)

---
 distributed/comm/ucx.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/distributed/comm/ucx.py b/distributed/comm/ucx.py
index bcf9b19f412..0a4580eacaa 100644
--- a/distributed/comm/ucx.py
+++ b/distributed/comm/ucx.py
@@ -21,7 +21,6 @@
 import ucp
 import numpy as np
 
-ucp.init(options=dask.config.get("ucx"), env_takes_precedence=True)
 
 logger = logging.getLogger(__name__)
 
@@ -40,8 +39,7 @@ def init_once():
     import ucp as _ucp
 
     ucp = _ucp
-    options = dask.config.get("ucx", default={})
-    ucp.init(options=options)
+    ucp.init(options=dask.config.get("ucx"), env_takes_precedence=True)
 
     # Find the function, `cuda_array()`, to use when allocating new CUDA arrays
     try:

From dfc703eb3baa3dbfdc89f1f96fdba7db75964917 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sat, 9 Nov 2019 08:03:45 -0800
Subject: [PATCH 0534/1550] Clean up rpc to avoid intermittent test failure
 (#3215)

---
 distributed/tests/test_core.py | 25 ++++++++++++-------------
 1 file changed, 12 insertions(+), 13 deletions(-)

diff --git a/distributed/tests/test_core.py b/distributed/tests/test_core.py
index 99c07226c48..78fbd1211d7 100644
--- a/distributed/tests/test_core.py
+++ b/distributed/tests/test_core.py
@@ -255,7 +255,7 @@ async def check_rpc(listen_addr, rpc_addr=None, listen_args=None, connection_arg
     if rpc_addr is None:
         rpc_addr = server.address
 
-    with rpc(rpc_addr, connection_args=connection_args) as remote:
+    async with rpc(rpc_addr, connection_args=connection_args) as remote:
         response = await remote.ping()
         assert response == b"pong"
         assert remote.comms
@@ -324,7 +324,7 @@ async def check_rpc_message_lifetime(*listen_args):
         await asyncio.sleep(0.01)
         assert time() < start + 1
 
-    with rpc(server.address) as remote:
+    async with rpc(server.address) as remote:
         obj = CountedObject()
         res = await remote.echo(x=to_serialize(obj))
         assert isinstance(res["result"], CountedObject)
@@ -366,7 +366,7 @@ async def g():
     server = Server({"ping": pingpong})
     server.listen(listen_arg)
 
-    with rpc(server.address) as remote:
+    async with rpc(server.address) as remote:
         for i in range(10):
             await g()
 
@@ -392,15 +392,14 @@ async def check_large_packets(listen_arg):
     server.listen(listen_arg)
 
     data = b"0" * int(200e6)  # slightly more than 100MB
-    conn = rpc(server.address)
-    result = await conn.echo(x=data)
-    assert result == data
+    async with rpc(server.address) as conn:
+        result = await conn.echo(x=data)
+        assert result == data
 
-    d = {"x": data}
-    result = await conn.echo(x=d)
-    assert result == d
+        d = {"x": data}
+        result = await conn.echo(x=d)
+        assert result == d
 
-    conn.close_comms()
     server.stop()
 
 
@@ -419,7 +418,7 @@ async def check_identity(listen_arg):
     server = Server({})
     server.listen(listen_arg)
 
-    with rpc(server.address) as remote:
+    async with rpc(server.address) as remote:
         a = await remote.identity()
         b = await remote.identity()
         assert a["type"] == "Server"
@@ -707,11 +706,11 @@ async def f():
         server = Server({"echo": echo_serialize})
         server.listen("tcp://")
 
-        with rpc(server.address, serializers=["msgpack"]) as r:
+        async with rpc(server.address, serializers=["msgpack"]) as r:
             with pytest.raises(TypeError):
                 await r.echo(x=to_serialize(inc))
 
-        with rpc(server.address, serializers=["msgpack", "pickle"]) as r:
+        async with rpc(server.address, serializers=["msgpack", "pickle"]) as r:
             result = await r.echo(x=to_serialize(inc))
             assert result == {"result": inc}
 

From 2936803ef9bd5b434eb6f260b23edd62476715c9 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sat, 9 Nov 2019 13:08:51 -0800
Subject: [PATCH 0535/1550] Respect protocol if given to Scheduler (#3212)

Previously calling the following

    dask-scheduler --protocol ucx

Without specifying an interface would result in a tcp:// protocol

Now we are a bit more forceful about respecting a protocol if given.
---
 distributed/comm/addressing.py     | 4 ++--
 distributed/comm/tests/test_ucx.py | 7 +++++++
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/distributed/comm/addressing.py b/distributed/comm/addressing.py
index 35d5e1c3407..f0c18b9fbda 100644
--- a/distributed/comm/addressing.py
+++ b/distributed/comm/addressing.py
@@ -241,7 +241,7 @@ def address_from_user_args(
     else:
         addr = ""
 
-    if protocol and "://" not in addr:
-        addr = protocol.rstrip("://") + "://" + addr
+    if protocol:
+        addr = protocol.rstrip("://") + "://" + addr.split("://")[-1]
 
     return addr
diff --git a/distributed/comm/tests/test_ucx.py b/distributed/comm/tests/test_ucx.py
index 4c2d2d0782e..a9207e72e7a 100644
--- a/distributed/comm/tests/test_ucx.py
+++ b/distributed/comm/tests/test_ucx.py
@@ -326,3 +326,10 @@ async def test_transpose(cleanup):
 
                 y = (x + x.T).sum()
                 await y
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("port", [0, 1234])
+async def test_ucx_protocol(cleanup, port):
+    async with Scheduler(protocol="ucx", port=port) as s:
+        assert s.address.startswith("ucx://")

From 86bd07a0c58daf3a8249d8231251ed9c9fba600c Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 11 Nov 2019 08:04:20 -0800
Subject: [PATCH 0536/1550] Use legend_field= keyword in bokeh plots (#3218)

The old legend= keyword has been deprecated
---
 distributed/dashboard/components/scheduler.py | 40 +++++++++----------
 1 file changed, 20 insertions(+), 20 deletions(-)

diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index 5e94a034cb0..77f095bce0d 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -113,7 +113,7 @@ def __init__(self, scheduler, **kwargs):
                 tools="",
                 id="bk-occupancy-plot",
                 x_axis_type="datetime",
-                **kwargs
+                **kwargs,
             )
             rect = fig.rect(
                 source=self.source, x="x", width="ms", y="y", height=1, color="color"
@@ -193,7 +193,7 @@ def __init__(self, scheduler, **kwargs):
                 name="processing_hist",
                 y_axis_label="frequency",
                 tools="",
-                **kwargs
+                **kwargs,
             )
 
             self.root.xaxis.minor_tick_line_alpha = 0
@@ -236,7 +236,7 @@ def __init__(self, scheduler, **kwargs):
                 id="bk-nbytes-histogram-plot",
                 y_axis_label="frequency",
                 tools="",
-                **kwargs
+                **kwargs,
             )
 
             self.root.xaxis[0].formatter = NumeralTickFormatter(format="0.0 b")
@@ -290,7 +290,7 @@ def __init__(self, scheduler, **kwargs):
                 id="bk-bandwidth-type-plot",
                 name="bandwidth_type_histogram",
                 y_range=["a", "b"],
-                **kwargs
+                **kwargs,
             )
             rect = fig.rect(
                 source=self.source,
@@ -364,7 +364,7 @@ def __init__(self, scheduler, **kwargs):
                 name="bandwidth_worker_heatmap",
                 x_range=["a", "b"],
                 y_range=["a", "b"],
-                **kwargs
+                **kwargs,
             )
             fig.xaxis.major_label_orientation = -math.pi / 12
             rect = fig.rect(
@@ -458,7 +458,7 @@ def __init__(self, scheduler, width=600, **kwargs):
                 id="bk-nprocessing-plot",
                 name="processing_hist",
                 width=int(width / 2),
-                **kwargs
+                **kwargs,
             )
             rect = processing.rect(
                 source=self.source,
@@ -477,7 +477,7 @@ def __init__(self, scheduler, width=600, **kwargs):
                 id="bk-nbytes-worker-plot",
                 width=int(width / 2),
                 name="nbytes_hist",
-                **kwargs
+                **kwargs,
             )
             rect = nbytes.rect(
                 source=self.source,
@@ -495,7 +495,7 @@ def __init__(self, scheduler, width=600, **kwargs):
                 id="bk-cpu-worker-plot",
                 width=int(width / 2),
                 name="cpu_hist",
-                **kwargs
+                **kwargs,
             )
             rect = cpu.rect(
                 source=self.source,
@@ -637,7 +637,7 @@ def __init__(self, scheduler, **kwargs):
             height=150,
             tools="",
             x_range=x_range,
-            **kwargs
+            **kwargs,
         )
         fig.line(source=self.source, x="time", y="idle", color="red")
         fig.line(source=self.source, x="time", y="saturated", color="green")
@@ -691,7 +691,7 @@ def __init__(self, scheduler, **kwargs):
             height=250,
             tools="",
             x_range=x_range,
-            **kwargs
+            **kwargs,
         )
 
         fig.circle(
@@ -787,7 +787,7 @@ def __init__(self, scheduler, name, height=150, **kwargs):
             height=height,
             tools="",
             x_range=x_range,
-            **kwargs
+            **kwargs,
         )
 
         fig.circle(
@@ -797,7 +797,7 @@ def __init__(self, scheduler, name, height=150, **kwargs):
             color="color",
             size=50,
             alpha=0.5,
-            legend="action",
+            **{"legend_field" if BOKEH_VERSION >= "1.4" else "legend": "action"},
         )
         fig.yaxis.axis_label = "Action"
         fig.legend.location = "top_left"
@@ -958,7 +958,7 @@ def task_stream_figure(clear_interval="20s", **kwargs):
         x_axis_type="datetime",
         min_border_right=35,
         tools="",
-        **kwargs
+        **kwargs,
     )
 
     rect = root.rect(
@@ -1058,7 +1058,7 @@ def __init__(self, scheduler, **kwargs):
             color=node_colors,
             source=self.node_source,
             view=node_view,
-            legend="state",
+            **{"legend_field" if BOKEH_VERSION >= "1.4" else "legend": "state"},
         )
         self.root.xgrid.grid_line_color = None
         self.root.ygrid.grid_line_color = None
@@ -1202,7 +1202,7 @@ def __init__(self, scheduler, **kwargs):
             y_range=y_range,
             toolbar_location=None,
             tools="",
-            **kwargs
+            **kwargs,
         )
         self.root.line(  # just to define early ranges
             x=[0, 0.9], y=[-1, 0], line_color="#FFFFFF", alpha=0.0
@@ -1368,7 +1368,7 @@ def __init__(self, scheduler, **kwargs):
             y_range=DataRange1d(),
             toolbar_location=None,
             outline_line_color=None,
-            **kwargs
+            **kwargs,
         )
 
         self.root.add_glyph(
@@ -1492,7 +1492,7 @@ def __init__(self, scheduler, width=800, **kwargs):
             reorderable=True,
             sortable=True,
             width=width,
-            **dt_kwargs
+            **dt_kwargs,
         )
 
         for name in table_names:
@@ -1511,7 +1511,7 @@ def __init__(self, scheduler, width=800, **kwargs):
             reorderable=True,
             sortable=True,
             width=width,
-            **dt_kwargs
+            **dt_kwargs,
         )
 
         hover = HoverTool(
@@ -1532,7 +1532,7 @@ def __init__(self, scheduler, width=800, **kwargs):
             height=60,
             width=width,
             tools="",
-            **kwargs
+            **kwargs,
         )
         mem_plot.circle(
             source=self.source, x="memory_percent", y=0, size=10, fill_alpha=0.5
@@ -1561,7 +1561,7 @@ def __init__(self, scheduler, width=800, **kwargs):
             height=60,
             width=width,
             tools="",
-            **kwargs
+            **kwargs,
         )
         cpu_plot.circle(
             source=self.source, x="cpu_fraction", y=0, size=10, fill_alpha=0.5

From 2a25c5f3f16bf238787f8ca223442fdb79f89e0f Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 11 Nov 2019 08:04:39 -0800
Subject: [PATCH 0537/1550] Cache psutil.Process object in Nanny (#3207)

---
 distributed/nanny.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/distributed/nanny.py b/distributed/nanny.py
index 6e58271c33a..39bffbd80c0 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -349,6 +349,19 @@ async def _():
         else:
             return "OK"
 
+    @property
+    def _psutil_process(self):
+        pid = self.process.process.pid
+        try:
+            proc = self._psutil_process_obj
+        except AttributeError:
+            self._psutil_process_obj = psutil.Process(pid)
+
+        if self._psutil_process_obj.pid != pid:
+            self._psutil_process_obj = psutil.Process(pid)
+
+        return self._psutil_process_obj
+
     def memory_monitor(self):
         """ Track worker's memory.  Restart if it goes above terminate fraction """
         if self.status != "running":
@@ -357,7 +370,7 @@ def memory_monitor(self):
         if process is None:
             return
         try:
-            proc = psutil.Process(process.pid)
+            proc = self._psutil_process
             memory = proc.memory_info().rss
         except (ProcessLookupError, psutil.NoSuchProcess, psutil.AccessDenied):
             return

From 81df14f144d5358ed9d37a065c5f5220280a6f78 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 11 Nov 2019 09:23:12 -0800
Subject: [PATCH 0538/1550] Replace gen.sleep with asyncio.sleep (#3208)

---
 distributed/client.py                | 16 ++++++++--------
 distributed/comm/core.py             |  3 ++-
 distributed/comm/tests/test_comms.py |  4 ++--
 distributed/core.py                  |  6 +++---
 distributed/diagnostics/progress.py  |  6 +++---
 distributed/nanny.py                 |  5 +++--
 distributed/scheduler.py             |  8 ++++----
 distributed/utils_test.py            | 10 +++++-----
 distributed/worker.py                |  6 +++---
 9 files changed, 33 insertions(+), 31 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index c7f921ee348..6093bc9ec03 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -942,7 +942,7 @@ async def _start(self, timeout=no_default, **kwargs):
             address = self.cluster.scheduler_address
         elif self.scheduler_file is not None:
             while not os.path.exists(self.scheduler_file):
-                await gen.sleep(0.01)
+                await asyncio.sleep(0.01)
             for i in range(10):
                 try:
                     with open(self.scheduler_file) as f:
@@ -950,7 +950,7 @@ async def _start(self, timeout=no_default, **kwargs):
                     address = cfg["address"]
                     break
                 except (ValueError, KeyError):  # JSON file not yet flushed
-                    await gen.sleep(0.01)
+                    await asyncio.sleep(0.01)
         elif self._start_arg is None:
             from .deploy import LocalCluster
 
@@ -976,7 +976,7 @@ async def _start(self, timeout=no_default, **kwargs):
             while not self.cluster.workers or len(self.cluster.scheduler.workers) < len(
                 self.cluster.workers
             ):
-                await gen.sleep(0.01)
+                await asyncio.sleep(0.01)
 
             address = self.cluster.scheduler_address
 
@@ -1017,7 +1017,7 @@ async def _reconnect(self):
                     break
                 except EnvironmentError:
                     # Wait a bit before retrying
-                    await gen.sleep(0.1)
+                    await asyncio.sleep(0.1)
                     timeout = deadline - self.loop.time()
             else:
                 logger.error(
@@ -1092,7 +1092,7 @@ async def _update_scheduler_info(self):
     async def _wait_for_workers(self, n_workers=0):
         info = await self.scheduler.identity()
         while n_workers and len(info["workers"]) < n_workers:
-            await gen.sleep(0.1)
+            await asyncio.sleep(0.1)
             info = await self.scheduler.identity()
 
     def wait_for_workers(self, n_workers=0):
@@ -1946,7 +1946,7 @@ async def _scatter(
                 start = time()
                 while not nthreads:
                     if nthreads is not None:
-                        await gen.sleep(0.1)
+                        await asyncio.sleep(0.1)
                     if time() > start + timeout:
                         raise gen.TimeoutError("No valid workers found")
                     nthreads = await self.scheduler.ncores(workers=workers)
@@ -2280,7 +2280,7 @@ def run_on_scheduler(self, function, *args, **kwargs):
         >>> async def print_state(dask_scheduler):  # doctest: +SKIP
         ...    while True:
         ...        print(dask_scheduler.status)
-        ...        await gen.sleep(1)
+        ...        await asyncio.sleep(1)
 
         >>> c.run(print_state, wait=False)  # doctest: +SKIP
 
@@ -2370,7 +2370,7 @@ def run(self, function, *args, **kwargs):
         >>> async def print_state(dask_worker):  # doctest: +SKIP
         ...    while True:
         ...        print(dask_worker.status)
-        ...        await gen.sleep(1)
+        ...        await asyncio.sleep(1)
 
         >>> c.run(print_state, wait=False)  # doctest: +SKIP
         """
diff --git a/distributed/comm/core.py b/distributed/comm/core.py
index 256a17de3a5..39a8b123cd3 100644
--- a/distributed/comm/core.py
+++ b/distributed/comm/core.py
@@ -1,4 +1,5 @@
 from abc import ABC, abstractmethod, abstractproperty
+import asyncio
 from datetime import timedelta
 import logging
 import weakref
@@ -224,7 +225,7 @@ def _raise(error):
         except EnvironmentError as e:
             error = str(e)
             if time() < deadline:
-                await gen.sleep(0.01)
+                await asyncio.sleep(0.01)
                 logger.debug("sleeping on connect")
             else:
                 _raise(error)
diff --git a/distributed/comm/tests/test_comms.py b/distributed/comm/tests/test_comms.py
index 301cb4f013f..5839c3e8871 100644
--- a/distributed/comm/tests/test_comms.py
+++ b/distributed/comm/tests/test_comms.py
@@ -683,7 +683,7 @@ def handle_comm(comm):
 
     # Sanity check
     comm = yield connect(
-        listener.contact_address, timeout=0.5, connection_args={"ssl_context": cli_ctx}
+        listener.contact_address, timeout=2, connection_args={"ssl_context": cli_ctx}
     )
     yield comm.close()
 
@@ -696,7 +696,7 @@ def handle_comm(comm):
     with pytest.raises(EnvironmentError) as excinfo:
         yield connect(
             listener.contact_address,
-            timeout=0.5,
+            timeout=2,
             connection_args={"ssl_context": cli_ctx},
         )
     # The wrong error is reported on Python 2, see https://github.com/tornadoweb/tornado/pull/2028
diff --git a/distributed/core.py b/distributed/core.py
index 66205dbfd85..d096e0b2274 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -464,7 +464,7 @@ async def handle_stream(self, comm, extra=None, every_cycle=[]):
                                 handler(**merge(extra, msg))
                         else:
                             logger.error("odd message %s", msg)
-                    await gen.sleep(0)
+                    await asyncio.sleep(0)
 
                 for func in every_cycle:
                     func()
@@ -492,7 +492,7 @@ def close(self):
             if not self._comms:
                 break
             else:
-                yield gen.sleep(0.05)
+                yield asyncio.sleep(0.05)
         yield [comm.close() for comm in self._comms]  # then forcefully close
         for cb in self._ongoing_coroutines:
             cb.cancel()
@@ -500,7 +500,7 @@ def close(self):
             if all(cb.cancelled() for c in self._ongoing_coroutines):
                 break
             else:
-                yield gen.sleep(0.01)
+                yield asyncio.sleep(0.01)
 
         self._event_finished.set()
 
diff --git a/distributed/diagnostics/progress.py b/distributed/diagnostics/progress.py
index 4136fd17a5c..48f26570980 100644
--- a/distributed/diagnostics/progress.py
+++ b/distributed/diagnostics/progress.py
@@ -1,9 +1,9 @@
+import asyncio
 from collections import defaultdict
 import logging
 from timeit import default_timer
 
 from toolz import groupby, valmap
-from tornado import gen
 
 from .plugin import SchedulerPlugin
 from ..utils import key_split, key_split_group, log_errors, tokey
@@ -76,7 +76,7 @@ async def setup(self):
         keys = self.keys
 
         while not keys.issubset(self.scheduler.tasks):
-            await gen.sleep(0.05)
+            await asyncio.sleep(0.05)
 
         tasks = [self.scheduler.tasks[k] for k in keys]
 
@@ -164,7 +164,7 @@ async def setup(self):
         keys = self.keys
 
         while not keys.issubset(self.scheduler.tasks):
-            await gen.sleep(0.05)
+            await asyncio.sleep(0.05)
 
         tasks = [self.scheduler.tasks[k] for k in keys]
 
diff --git a/distributed/nanny.py b/distributed/nanny.py
index 39bffbd80c0..58491da154c 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -1,3 +1,4 @@
+import asyncio
 from datetime import timedelta
 import logging
 from multiprocessing.queues import Empty
@@ -608,7 +609,7 @@ async def kill(self, timeout=2, executor_wait=True):
         self.child_stop_q.close()
 
         while process.is_alive() and loop.time() < deadline:
-            await gen.sleep(0.05)
+            await asyncio.sleep(0.05)
 
         if process.is_alive():
             logger.warning(
@@ -627,7 +628,7 @@ async def _wait_until_connected(self, uid):
             try:
                 msg = self.init_result_q.get_nowait()
             except Empty:
-                await gen.sleep(delay)
+                await asyncio.sleep(delay)
                 continue
 
             if msg["uid"] != uid:  # ensure that we didn't cross queues
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 4319584735e..de04804370a 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1271,7 +1271,7 @@ async def close(self, comm=None, fast=False, close_workers=False):
                 self.worker_send(worker, {"op": "close"})
             for i in range(20):  # wait a second for send signals to clear
                 if self.workers:
-                    await gen.sleep(0.05)
+                    await asyncio.sleep(0.05)
                 else:
                     break
 
@@ -2494,7 +2494,7 @@ async def scatter(
         """
         start = time()
         while not self.workers:
-            await gen.sleep(0.2)
+            await asyncio.sleep(0.2)
             if time() > start + timeout:
                 raise gen.TimeoutError("No workers found")
 
@@ -2649,7 +2649,7 @@ async def restart(self, client=None, timeout=3):
             self.log_event([client, "all"], {"action": "restart", "client": client})
             start = time()
             while time() < start + 10 and len(self.workers) < n_workers:
-                await gen.sleep(0.01)
+                await asyncio.sleep(0.01)
 
             self.report({"op": "restart"})
 
@@ -3292,7 +3292,7 @@ async def feed(
                     else:
                         response = function(self, state)
                     await comm.write(response)
-                    await gen.sleep(interval)
+                    await asyncio.sleep(interval)
             except (EnvironmentError, CommClosedError):
                 pass
             finally:
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index 81b196639ac..aef7bde8eee 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -394,7 +394,7 @@ def apply(func, *args, **kwargs):
 
 
 async def geninc(x, delay=0.02):
-    await gen.sleep(delay)
+    await asyncio.sleep(delay)
     return x + 1
 
 
@@ -410,7 +410,7 @@ def compile_snippet(code, dedent=True):
     compile_snippet(
         """
         async def asyncinc(x, delay=0.02):
-            await gen.sleep(delay)
+            await asyncio.sleep(delay)
             return x + 1
         """
     )
@@ -813,7 +813,7 @@ async def start_cluster(
     while len(s.workers) < len(nthreads) or any(
         comm.comm is None for comm in s.stream_comms.values()
     ):
-        await gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         if time() - start > 5:
             await asyncio.gather(*[w.close(timeout=1) for w in workers])
             await s.close(fast=True)
@@ -939,7 +939,7 @@ async def coro():
                             if all(c.closed() for c in Comm._instances):
                                 break
                             else:
-                                await gen.sleep(0.05)
+                                await asyncio.sleep(0.05)
                         else:
                             L = [c for c in Comm._instances if not c.closed()]
                             Comm._instances.clear()
@@ -1063,7 +1063,7 @@ def wait_for(predicate, timeout, fail_func=None, period=0.001):
 async def async_wait_for(predicate, timeout, fail_func=None, period=0.001):
     deadline = time() + timeout
     while not predicate():
-        await gen.sleep(period)
+        await asyncio.sleep(period)
         if time() > deadline:
             if fail_func is not None:
                 fail_func()
diff --git a/distributed/worker.py b/distributed/worker.py
index d1c35f68eb5..c0836077a6c 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -829,7 +829,7 @@ async def _register_with_scheduler(self):
                 break
             except EnvironmentError:
                 logger.info("Waiting to connect to: %26s", self.scheduler.address)
-                await gen.sleep(0.1)
+                await asyncio.sleep(0.1)
             except gen.TimeoutError:
                 logger.info("Timed out when connecting to scheduler")
         if response["status"] != "OK":
@@ -1997,7 +1997,7 @@ async def gather_dep(self, worker, dep, deps, total_nbytes, cause=None):
                 else:
                     # Exponential backoff to avoid hammering scheduler/worker
                     self.repetitively_busy += 1
-                    await gen.sleep(0.100 * 1.5 ** self.repetitively_busy)
+                    await asyncio.sleep(0.100 * 1.5 ** self.repetitively_busy)
 
                     # See if anyone new has the data
                     await self.query_who_has(dep)
@@ -2586,7 +2586,7 @@ async def memory_monitor(self):
                 del k, v
                 total += weight
                 count += 1
-                await gen.sleep(0)
+                await asyncio.sleep(0)
                 memory = proc.memory_info().rss
                 if total > need and memory > target:
                     # Issue a GC to ensure that the evicted data is actually

From 8572887824cf438817e5e85ca7460f5f1d5870ac Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 11 Nov 2019 12:49:27 -0800
Subject: [PATCH 0539/1550] Avoid offloading serialization for small messages
 (#3224)

This was causing a fair amount of either cost or noise when profiling.

Fixes https://github.com/dask/distributed/issues/3223
---
 distributed/comm/utils.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/distributed/comm/utils.py b/distributed/comm/utils.py
index 80e1f163785..5b15d5c798c 100644
--- a/distributed/comm/utils.py
+++ b/distributed/comm/utils.py
@@ -1,6 +1,8 @@
 import logging
 import socket
 
+from dask.sizeof import sizeof
+
 from .. import protocol
 from ..utils import get_ip, get_ipv6, nbytes, offload
 
@@ -31,9 +33,10 @@ def _to_frames():
             logger.exception(e)
             raise
 
-    res = await offload(_to_frames)
-
-    return res
+    if sizeof(msg) > FRAME_OFFLOAD_THRESHOLD:
+        return await offload(_to_frames)
+    else:
+        return _to_frames()
 
 
 async def from_frames(frames, deserialize=True, deserializers=None):

From aff524ba2286e614ed9d5cc259bcab5cc921cc4a Mon Sep 17 00:00:00 2001
From: Gabriel Sailer <gabriel.sailer@blue-yonder.com>
Date: Mon, 11 Nov 2019 23:09:44 +0100
Subject: [PATCH 0540/1550] Add desired_workers metric (#3221)

---
 distributed/dashboard/scheduler.py | 8 ++++++++
 distributed/scheduler.py           | 2 +-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/distributed/dashboard/scheduler.py b/distributed/dashboard/scheduler.py
index 8928d468c5e..ecc413b5f0d 100644
--- a/distributed/dashboard/scheduler.py
+++ b/distributed/dashboard/scheduler.py
@@ -171,6 +171,7 @@ def get(self):
         released = 0
         waiting = 0
         waiting_data = 0
+        desired_workers = scheduler.adaptive_target()
 
         for ts in scheduler.tasks.values():
             if ts.exception_blame is not None:
@@ -203,6 +204,7 @@ def get(self):
             "waiting": waiting,
             "waiting_data": waiting_data,
             "workers": len(scheduler.workers),
+            "desired_workers": desired_workers,
         }
         self.write(response)
 
@@ -245,6 +247,12 @@ def collect(self):
             value=len(self.server.clients),
         )
 
+        yield GaugeMetricFamily(
+            "dask_scheduler_desired_workers",
+            "Number of workers scheduler needs for task graph.",
+            value=self.server.adaptive_target(),
+        )
+
         tasks = GaugeMetricFamily(
             "dask_scheduler_workers",
             "Number of workers known by scheduler.",
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index de04804370a..6724524344d 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -3062,7 +3062,7 @@ def _key(group):
 
             result = [getattr(ws, attribute) for g in to_close for ws in groups[g]]
             if result:
-                logger.info("Suggest closing workers: %s", result)
+                logger.debug("Suggest closing workers: %s", result)
 
             return result
 

From 43b2ed77218861c34dd2c98d88ec5153bd35f5c4 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 12 Nov 2019 11:57:43 -0800
Subject: [PATCH 0541/1550] Fail fast when importing distributed.comm.ucx
 (#3228)

Fixes https://github.com/dask/dask/issues/5572

If we don't have UCX installed then we shouldn't try anything further.
In this commit we move the `import ucp` line to the top of the file.
---
 distributed/comm/ucx.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/distributed/comm/ucx.py b/distributed/comm/ucx.py
index 0a4580eacaa..58b16eaaf7f 100644
--- a/distributed/comm/ucx.py
+++ b/distributed/comm/ucx.py
@@ -5,6 +5,8 @@
 
 .. _UCX: https://github.com/openucx/ucx
 """
+import ucp
+
 import logging
 import concurrent
 
@@ -18,7 +20,6 @@
 from ..utils import ensure_ip, get_ip, get_ipv6, nbytes, log_errors
 
 import dask
-import ucp
 import numpy as np
 
 

From 2060805bbe3074e050ef7ac81c485201caf0618e Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 13 Nov 2019 07:29:57 -0800
Subject: [PATCH 0542/1550] Add module name to Future repr (#3231)

---
 distributed/client.py            | 12 ++++--------
 distributed/tests/test_client.py | 11 ++++++++++-
 2 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 6093bc9ec03..6f5bf786367 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -383,16 +383,12 @@ def __del__(self):
     def __repr__(self):
         if self.type:
             try:
-                typ = self.type.__name__
+                typ = self.type.__module__.split(".")[0] + "." + self.type.__name__
             except AttributeError:
                 typ = str(self.type)
-            return "<Future: status: %s, type: %s, key: %s>" % (
-                self.status,
-                typ,
-                self.key,
-            )
+            return "<Future: %s, type: %s, key: %s>" % (self.status, typ, self.key)
         else:
-            return "<Future: status: %s, key: %s>" % (self.status, self.key)
+            return "<Future: %s, key: %s>" % (self.status, self.key)
 
     def _repr_html_(self):
         text = "<b>Future: %s</b> " % html.escape(key_split(self.key))
@@ -405,7 +401,7 @@ def _repr_html_(self):
         }
         if self.type:
             try:
-                typ = self.type.__name__
+                typ = self.type.__module__.split(".")[0] + "." + self.type.__name__
             except AttributeError:
                 typ = str(self.type)
             text += '<font color="gray">type: </font>%s, ' % typ
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 4d747d8f3ec..9f8e48298fa 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -334,13 +334,22 @@ def test_retries_dask_array(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_future_repr(c, s, a, b):
+async def test_future_repr(c, s, a, b):
+    pd = pytest.importorskip("pandas")
     x = c.submit(inc, 10)
+    y = c.submit(pd.DataFrame, {"x": [1, 2, 3]})
+    await x
+    await y
+
     for func in [repr, lambda x: x._repr_html_()]:
         assert str(x.key) in func(x)
         assert str(x.status) in func(x)
         assert str(x.status) in repr(c.futures[x.key])
 
+        assert "int" in func(x)
+        assert "pandas" in func(y)
+        assert "DataFrame" in func(y)
+
 
 @gen_cluster(client=True)
 def test_future_tuple_repr(c, s, a, b):

From e0fe1caa9b44919f1c59383d04e593bfc3f9ecda Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 14 Nov 2019 07:59:02 -0800
Subject: [PATCH 0543/1550] Add name to Pub/Sub repr (#3235)

---
 distributed/pubsub.py            | 10 ++++++++++
 distributed/tests/test_pubsub.py | 10 ++++++++++
 2 files changed, 20 insertions(+)

diff --git a/distributed/pubsub.py b/distributed/pubsub.py
index 0a4053191eb..5ed631c46ec 100644
--- a/distributed/pubsub.py
+++ b/distributed/pubsub.py
@@ -343,6 +343,11 @@ def put(self, msg):
         """ Publish a message to all subscribers of this topic """
         self.loop.add_callback(self._put, msg)
 
+    def __repr__(self):
+        return "<Pub: {}>".format(self.name)
+
+    __str__ = __repr__
+
 
 class Sub(object):
     """ Subscribe to a Publish/Subscribe topic
@@ -426,3 +431,8 @@ def __aiter__(self):
     def _put(self, msg):
         self.buffer.append(msg)
         self.condition.notify()
+
+    def __repr__(self):
+        return "<Sub: {}>".format(self.name)
+
+    __str__ = __repr__
diff --git a/distributed/tests/test_pubsub.py b/distributed/tests/test_pubsub.py
index 9d2b30dab6f..555afb71a73 100644
--- a/distributed/tests/test_pubsub.py
+++ b/distributed/tests/test_pubsub.py
@@ -125,5 +125,15 @@ def test_timeouts(c, s, a, b):
     assert stop - start < 1
 
 
+@gen_cluster(client=True)
+async def test_repr(c, s, a, b):
+    pub = Pub("my-topic")
+    sub = Sub("my-topic")
+    assert "my-topic" in str(pub)
+    assert "Pub" in str(pub)
+    assert "my-topic" in str(sub)
+    assert "Sub" in str(sub)
+
+
 if sys.version_info >= (3, 5):
     from distributed.tests.py3_test_pubsub import *  # noqa: F401, F403

From ddbec38ba1ec6de913ccbfcd090f1c85eea1b032 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Thu, 14 Nov 2019 10:14:10 -0600
Subject: [PATCH 0544/1550] Import CPU_COUNT from dask.system (#3199)

---
 distributed/cli/dask_worker.py         |  2 +-
 distributed/deploy/local.py            | 10 +++---
 distributed/deploy/tests/test_local.py |  3 +-
 distributed/nanny.py                   |  2 +-
 distributed/system.py                  | 44 +----------------------
 distributed/tests/test_system.py       | 48 +-------------------------
 distributed/tests/test_worker.py       |  6 ++--
 distributed/worker.py                  |  5 +--
 8 files changed, 17 insertions(+), 103 deletions(-)

diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
index 0f307398e04..9070024c430 100755
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -10,9 +10,9 @@
 import click
 import dask
 from dask.utils import ignoring
+from dask.system import CPU_COUNT
 from distributed import Nanny, Worker
 from distributed.security import Security
-from distributed.system import CPU_COUNT
 from distributed.cli.utils import check_python_3, install_signal_handlers
 from distributed.comm import get_address_host_port
 from distributed.preloading import validate_preload_argv
diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index fd1430baa21..8eb55c54997 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -5,8 +5,8 @@
 import weakref
 
 from dask.utils import factors
+from dask.system import CPU_COUNT
 
-from .. import system
 from .spec import SpecCluster
 from ..nanny import Nanny
 from ..scheduler import Scheduler
@@ -157,12 +157,12 @@ def __init__(
                 n_workers, threads_per_worker = nprocesses_nthreads()
             else:
                 n_workers = 1
-                threads_per_worker = system.CPU_COUNT
+                threads_per_worker = CPU_COUNT
         if n_workers is None and threads_per_worker is not None:
-            n_workers = max(1, system.CPU_COUNT // threads_per_worker)
+            n_workers = max(1, CPU_COUNT // threads_per_worker)
         if n_workers and threads_per_worker is None:
             # Overcommit threads per worker, rather than undercommit
-            threads_per_worker = max(1, int(math.ceil(system.CPU_COUNT / n_workers)))
+            threads_per_worker = max(1, int(math.ceil(CPU_COUNT / n_workers)))
         if n_workers and "memory_limit" not in worker_kwargs:
             worker_kwargs["memory_limit"] = parse_memory_limit("auto", 1, n_workers)
 
@@ -217,7 +217,7 @@ def start_worker(self, *args, **kwargs):
         )
 
 
-def nprocesses_nthreads(n=system.CPU_COUNT):
+def nprocesses_nthreads(n=CPU_COUNT):
     """
     The default breakdown of processes and threads for a given number of cores
 
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index e73ccd4721f..9b39d8f81f5 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -12,10 +12,11 @@
 from tornado import gen
 import pytest
 
+from dask.system import CPU_COUNT
 from distributed import Client, Worker, Nanny, get_client
 from distributed.deploy.local import LocalCluster, nprocesses_nthreads
 from distributed.metrics import time
-from distributed.system import CPU_COUNT, MEMORY_LIMIT
+from distributed.system import MEMORY_LIMIT
 from distributed.utils_test import (  # noqa: F401
     clean,
     cleanup,
diff --git a/distributed/nanny.py b/distributed/nanny.py
index 58491da154c..11cf0157c10 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -11,6 +11,7 @@
 import weakref
 
 import dask
+from dask.system import CPU_COUNT
 from tornado import gen
 from tornado.ioloop import IOLoop, TimeoutError
 from tornado.locks import Event
@@ -23,7 +24,6 @@
 from .process import AsyncProcess
 from .proctitle import enable_proctitle_on_children
 from .security import Security
-from .system import CPU_COUNT
 from .utils import (
     get_ip,
     mp_context,
diff --git a/distributed/system.py b/distributed/system.py
index 291248ddded..2b032a34024 100644
--- a/distributed/system.py
+++ b/distributed/system.py
@@ -1,10 +1,8 @@
-import math
-import os
 import sys
 
 import psutil
 
-__all__ = ("memory_limit", "cpu_count", "MEMORY_LIMIT", "CPU_COUNT")
+__all__ = ("memory_limit", "MEMORY_LIMIT")
 
 
 def memory_limit():
@@ -41,44 +39,4 @@ def memory_limit():
     return limit
 
 
-def cpu_count():
-    """Get the available CPU count for this system.
-
-    Takes the minimum value from the following locations:
-
-    - Total system cpus available on the host.
-    - CPU Affinity (if set)
-    - Cgroups limit (if set)
-    """
-    count = os.cpu_count()
-
-    # Check CPU affinity if available
-    try:
-        affinity_count = len(psutil.Process().cpu_affinity())
-        if affinity_count > 0:
-            count = min(count, affinity_count)
-    except Exception:
-        pass
-
-    # Check cgroups if available
-    if sys.platform == "linux":
-        # The directory name isn't standardized across linux distros, check both
-        for dirname in ["cpuacct,cpu", "cpu,cpuacct"]:
-            try:
-                with open("/sys/fs/cgroup/%s/cpu.cfs_quota_us" % dirname) as f:
-                    quota = int(f.read())
-                with open("/sys/fs/cgroup/%s/cpu.cfs_period_us" % dirname) as f:
-                    period = int(f.read())
-                # We round up on fractional CPUs
-                cgroups_count = math.ceil(quota / period)
-                if cgroups_count > 0:
-                    count = min(count, cgroups_count)
-                break
-            except Exception:
-                pass
-
-    return count
-
-
 MEMORY_LIMIT = memory_limit()
-CPU_COUNT = cpu_count()
diff --git a/distributed/tests/test_system.py b/distributed/tests/test_system.py
index d276613b520..3d44efe781d 100644
--- a/distributed/tests/test_system.py
+++ b/distributed/tests/test_system.py
@@ -1,57 +1,11 @@
 import builtins
 import io
-import os
 import sys
 
 import psutil
 import pytest
 
-from distributed.system import cpu_count, memory_limit
-
-
-def test_cpu_count():
-    count = cpu_count()
-    assert isinstance(count, int)
-    assert count <= os.cpu_count()
-    assert count >= 1
-
-
-@pytest.mark.parametrize("dirname", ["cpuacct,cpu", "cpu,cpuacct", None])
-def test_cpu_count_cgroups(dirname, monkeypatch):
-    def mycpu_count():
-        # Absurdly high, unlikely to match real value
-        return 250
-
-    monkeypatch.setattr(os, "cpu_count", mycpu_count)
-
-    class MyProcess(object):
-        def cpu_affinity(self):
-            # No affinity set
-            return []
-
-    monkeypatch.setattr(psutil, "Process", MyProcess)
-
-    if dirname:
-        paths = {
-            "/sys/fs/cgroup/%s/cpu.cfs_quota_us" % dirname: io.StringIO("2005"),
-            "/sys/fs/cgroup/%s/cpu.cfs_period_us" % dirname: io.StringIO("10"),
-        }
-        builtin_open = builtins.open
-
-        def myopen(path, *args, **kwargs):
-            if path in paths:
-                return paths.get(path)
-            return builtin_open(path, *args, **kwargs)
-
-        monkeypatch.setattr(builtins, "open", myopen)
-        monkeypatch.setattr(sys, "platform", "linux")
-
-    count = cpu_count()
-    if dirname:
-        # Rounds up
-        assert count == 201
-    else:
-        assert count == 250
+from distributed.system import memory_limit
 
 
 def test_memory_limit():
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 8b81e3afbe6..ff523342243 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -14,6 +14,7 @@
 import dask
 from dask import delayed
 from dask.utils import format_bytes
+from dask.system import CPU_COUNT
 import pytest
 from toolz import pluck, sliding_window, first
 import tornado
@@ -29,7 +30,6 @@
     get_worker,
     Reschedule,
     wait,
-    system,
 )
 from distributed.compatibility import WINDOWS
 from distributed.core import rpc
@@ -62,7 +62,7 @@
 def test_worker_nthreads():
     w = Worker("127.0.0.1", 8019)
     try:
-        assert w.executor._max_workers == system.CPU_COUNT
+        assert w.executor._max_workers == CPU_COUNT
     finally:
         shutil.rmtree(w.local_directory)
 
@@ -516,7 +516,7 @@ def test_memory_limit_auto():
     assert isinstance(a.memory_limit, Number)
     assert isinstance(b.memory_limit, Number)
 
-    if system.CPU_COUNT > 1:
+    if CPU_COUNT > 1:
         assert a.memory_limit < b.memory_limit
 
     assert c.memory_limit == d.memory_limit
diff --git a/distributed/worker.py b/distributed/worker.py
index c0836077a6c..3533d460299 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -18,6 +18,7 @@
 from dask.core import istask
 from dask.compatibility import apply
 from dask.utils import format_bytes, funcname
+from dask.system import CPU_COUNT
 
 try:
     from cytoolz import pluck, partial, merge, first, keymap
@@ -462,7 +463,7 @@ def __init__(
             warnings.warn("the ncores= parameter has moved to nthreads=")
             nthreads = ncores
 
-        self.nthreads = nthreads or system.CPU_COUNT
+        self.nthreads = nthreads or CPU_COUNT
         self.total_resources = resources or {}
         self.available_resources = (resources or {}).copy()
         self.death_timeout = parse_timedelta(death_timeout)
@@ -3071,7 +3072,7 @@ class Reschedule(Exception):
     pass
 
 
-def parse_memory_limit(memory_limit, nthreads, total_cores=system.CPU_COUNT):
+def parse_memory_limit(memory_limit, nthreads, total_cores=CPU_COUNT):
     if memory_limit is None:
         return None
 

From fd98e30ee129f07367ce4828523f9fd9eb4f7f58 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Thu, 14 Nov 2019 10:56:23 -0600
Subject: [PATCH 0545/1550] Efficiently serialize zero strided NumPy arrays
 (#3180)

---
 distributed/protocol/numpy.py            | 15 ++++++++++++
 distributed/protocol/tests/test_numpy.py | 29 ++++++++++++++++++++++++
 distributed/tests/test_sizeof.py         | 23 +++++++++++++++++++
 3 files changed, 67 insertions(+)
 create mode 100644 distributed/tests/test_sizeof.py

diff --git a/distributed/protocol/numpy.py b/distributed/protocol/numpy.py
index c7e48e63b1a..c5061a8f802 100644
--- a/distributed/protocol/numpy.py
+++ b/distributed/protocol/numpy.py
@@ -44,6 +44,13 @@ def serialize_numpy_ndarray(x):
     else:
         dt = (0, x.dtype.str)
 
+    # Only serialize non-broadcasted data for arrays with zero strided axes
+    if 0 in x.strides:
+        broadcast_to = (x.shape, x.flags.writeable)
+        x = x[tuple(slice(None) if s != 0 else slice(1) for s in x.strides)]
+    else:
+        broadcast_to = None
+
     if not x.shape:
         # 0d array
         strides = x.strides
@@ -68,6 +75,9 @@ def serialize_numpy_ndarray(x):
 
     header = {"dtype": dt, "shape": x.shape, "strides": strides}
 
+    if broadcast_to is not None:
+        header["broadcast_to"] = broadcast_to
+
     if x.nbytes > 1e5:
         frames = frame_split_size([data])
     else:
@@ -97,6 +107,11 @@ def deserialize_numpy_ndarray(header, frames):
             header["shape"], dtype=dt, buffer=frames[0], strides=header["strides"]
         )
 
+        if header.get("broadcast_to"):
+            shape, writeable = header["broadcast_to"]
+            x = np.broadcast_to(x, shape)
+            x.setflags(write=writeable)
+
         return x
 
 
diff --git a/distributed/protocol/tests/test_numpy.py b/distributed/protocol/tests/test_numpy.py
index 4fb20d58631..6e4712272d8 100644
--- a/distributed/protocol/tests/test_numpy.py
+++ b/distributed/protocol/tests/test_numpy.py
@@ -71,6 +71,7 @@ def test_serialize():
         np.zeros((1, 1000, 1000)),
         np.arange(12)[::2],  # non-contiguous array
         np.ones(shape=(5, 6)).astype(dtype=[("total", "<f8"), ("n", "<f8")]),
+        np.broadcast_to(np.arange(3), shape=(10, 3)),  # zero-strided array
     ],
 )
 def test_dumps_serialize_numpy(x):
@@ -259,3 +260,31 @@ def test_large_numpy_array():
     x = np.ones((100000000,), dtype="u4")
     header, frames = serialize(x)
     assert sum(header["lengths"]) == sum(map(nbytes, frames))
+
+
+@pytest.mark.parametrize(
+    "x",
+    [
+        np.broadcast_to(np.arange(10), (20, 10)),  # Some strides are 0
+        np.broadcast_to(1, (3, 4, 2)),  # All strides are 0
+        np.broadcast_to(np.arange(100)[:1], 5),  # x.base is larger than x
+    ],
+)
+@pytest.mark.parametrize("writeable", [True, False])
+def test_zero_strided_numpy_array(x, writeable):
+    assert 0 in x.strides
+    x.setflags(write=writeable)
+    header, frames = serialize(x)
+    y = deserialize(header, frames)
+    np.testing.assert_equal(x, y)
+    # Ensure we transmit fewer bytes than the full array
+    assert sum(map(nbytes, frames)) < x.nbytes
+    # Ensure both x and y are have same write flag
+    assert x.flags.writeable == y.flags.writeable
+
+
+def test_non_zero_strided_array():
+    x = np.arange(10)
+    header, frames = serialize(x)
+    assert "broadcast_to" not in header
+    assert sum(map(nbytes, frames)) == x.nbytes
diff --git a/distributed/tests/test_sizeof.py b/distributed/tests/test_sizeof.py
new file mode 100644
index 00000000000..a92d2900ae8
--- /dev/null
+++ b/distributed/tests/test_sizeof.py
@@ -0,0 +1,23 @@
+import pytest
+import logging
+from dask.sizeof import sizeof
+
+from distributed.sizeof import safe_sizeof
+from distributed.utils_test import captured_logger
+
+
+@pytest.mark.parametrize("obj", [list(range(10)), tuple(range(10)), set(range(10))])
+def test_safe_sizeof(obj):
+    assert safe_sizeof(obj) == sizeof(obj)
+
+
+def test_safe_sizeof_raises():
+    class BadlySized:
+        def __sizeof__(self):
+            raise ValueError("bar")
+
+    foo = BadlySized()
+    with captured_logger(logging.getLogger("distributed.sizeof")) as logs:
+        assert safe_sizeof(foo) == 1e6
+
+    assert "Sizeof calculation failed.  Defaulting to 1MB" in logs.getvalue()

From 332faa065203e49df5f7b4c4a3f3fe25fe70191a Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 14 Nov 2019 08:57:31 -0800
Subject: [PATCH 0546/1550] Cache function deserialization in workers (#3234)

This is particularly useful for numba.cuda.jit compiled functions where
there is some accumulated state.
---
 distributed/worker.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/distributed/worker.py b/distributed/worker.py
index 3533d460299..2a705320baa 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -3143,11 +3143,18 @@ async def get_data_from_worker(
 
 job_counter = [0]
 
+import functools
+
+
+@functools.lru_cache(100)
+def cached_function_deserialization(func):
+    return pickle.loads(func)
+
 
 def _deserialize(function=None, args=None, kwargs=None, task=no_value):
     """ Deserialize task inputs and regularize to func, args, kwargs """
     if function is not None:
-        function = pickle.loads(function)
+        function = cached_function_deserialization(function)
     if args:
         args = pickle.loads(args)
     if kwargs:

From 27b30fa2965d510d3afddb04807106ecb37f8d8d Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 14 Nov 2019 11:32:05 -0800
Subject: [PATCH 0547/1550] Respect ordering of futures in futures_of (#3236)

Previously we used to drop ordering and use only a set.
Now we maintain a list of when we first see each future.
---
 distributed/client.py            | 28 ++++++++++++++++++++++++----
 distributed/tests/test_client.py |  9 +++++++++
 2 files changed, 33 insertions(+), 4 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 6f5bf786367..4f315e89b6d 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -4385,9 +4385,27 @@ def redict_collection(c, dsk):
 
 
 def futures_of(o, client=None):
-    """ Future objects in a collection """
+    """ Future objects in a collection
+
+    Parameters
+    ----------
+    o: collection
+        A possibly nested collection of Dask objects
+
+    Examples
+    --------
+    >>> futures_of(my_dask_dataframe)
+    [<Future: finished key: ...>,
+     <Future: pending  key: ...>]
+
+    Returns
+    -------
+    futures : List[Future]
+        A list of futures held by those collections
+    """
     stack = [o]
-    futures = set()
+    seen = set()
+    futures = list()
     while stack:
         x = stack.pop()
         if type(x) in (tuple, set, list):
@@ -4397,7 +4415,9 @@ def futures_of(o, client=None):
         elif type(x) is SubgraphCallable:
             stack.extend(x.dsk.values())
         elif isinstance(x, Future):
-            futures.add(x)
+            if x not in seen:
+                seen.add(x)
+                futures.append(x)
         elif dask.is_dask_collection(x):
             stack.extend(x.__dask_graph__().values())
 
@@ -4406,7 +4426,7 @@ def futures_of(o, client=None):
         if bad:
             raise CancelledError(bad)
 
-    return list(futures)
+    return futures[::-1]
 
 
 def fire_and_forget(obj):
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 9f8e48298fa..73906517045 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -5678,5 +5678,14 @@ def f(x):
                 assert result == 101
 
 
+@gen_cluster(client=True)
+async def test_futures_of_sorted(c, s, a, b):
+    pytest.importorskip("dask.dataframe")
+    df = await dask.datasets.timeseries(dtypes={"x": int}).persist()
+    futures = futures_of(df)
+    for k, f in zip(df.__dask_keys__(), futures):
+        assert str(k) in str(f)
+
+
 if sys.version_info >= (3, 5):
     from distributed.tests.py3_test_client import *  # noqa F401

From 81821ad057968c813b2d0f112c6cee735cafb5a4 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Thu, 14 Nov 2019 15:10:12 -0600
Subject: [PATCH 0548/1550] Bump dask dependency to 2.7.0 (#3237)

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index d1335d0b3b1..b17e4620be6 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,6 @@
 click >= 6.6
 cloudpickle >= 0.2.2
-dask >= 2.5.2
+dask >= 2.7.0
 msgpack
 psutil >= 5.0
 sortedcontainers !=2.0.0, !=2.0.1

From 3670cc2ad9d1d885ddedeae3ac98cd1593d5a2d9 Mon Sep 17 00:00:00 2001
From: rockwellw <rockwellw@users.noreply.github.com>
Date: Thu, 14 Nov 2019 13:10:45 -0800
Subject: [PATCH 0549/1550] Avoid setting inf x_range (#3229)

---
 distributed/dashboard/components/scheduler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index 77f095bce0d..cb8f6e6f20e 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -585,7 +585,7 @@ def update(self):
                     or inf
                 )
 
-                if limit > max_limit:
+                if limit > max_limit and limit != inf:
                     max_limit = limit
 
                 if nb > limit:

From 886189aee41a18a08b15b4f48a65da91620e3cb7 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 14 Nov 2019 13:11:48 -0800
Subject: [PATCH 0550/1550] Clear task stream based on recent behavior (#3200)

---
 distributed/dashboard/components/scheduler.py | 36 +++++++++++++------
 1 file changed, 26 insertions(+), 10 deletions(-)

diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index cb8f6e6f20e..a53a0e744e8 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -1,7 +1,7 @@
 import logging
 import math
 from numbers import Number
-from operator import add
+import operator
 import os
 
 from bokeh.layouts import column, row
@@ -873,6 +873,7 @@ def __init__(self, scheduler, n_rectangles=1000, clear_interval="20s", **kwargs)
         clear_interval = parse_timedelta(clear_interval, default="ms")
         self.clear_interval = clear_interval
         self.last = 0
+        self.last_seen = 0
 
         self.source, self.root = task_stream_figure(clear_interval, **kwargs)
 
@@ -899,16 +900,31 @@ def update(self):
             if not rectangles["start"]:
                 return
 
-            # If there has been a significant delay then clear old rectangles
-            first_end = min(map(add, rectangles["start"], rectangles["duration"]))
-            if first_end > self.last:
-                last = self.last
-                self.last = first_end
-                if first_end > last + self.clear_interval * 1000:
-                    self.offset = min(rectangles["start"])
-                    self.source.data.update({k: [] for k in rectangles})
+            # If it has been a while since we've updated the plot
+            if time() > self.last_seen + self.clear_interval:
+                new_start = min(rectangles["start"]) - self.offset
+                old_start = min(self.source.data["start"])
+                old_end = max(
+                    map(
+                        operator.add,
+                        self.source.data["start"],
+                        self.source.data["duration"],
+                    )
+                )
+
+                density = (
+                    sum(self.source.data["duration"])
+                    / len(self.workers)
+                    / (old_end - old_start)
+                )
+
+                # If whitespace is more than 3x the old width
+                if (new_start - old_end) > (old_end - old_start) * 2 or density < 0.05:
+                    self.source.data.update({k: [] for k in rectangles})  # clear
+                    self.offset = min(rectangles["start"])  # redefine offset
 
             rectangles["start"] = [x - self.offset for x in rectangles["start"]]
+            self.last_seen = time()
 
             # Convert to numpy for serialization speed
             if n >= 10 and np:
@@ -1707,7 +1723,7 @@ def status_doc(scheduler, extra, doc):
             n_rectangles=dask.config.get(
                 "distributed.scheduler.dashboard.status.task-stream-length"
             ),
-            clear_interval="10s",
+            clear_interval="5s",
             sizing_mode="stretch_both",
         )
         task_stream.update()

From f8aca16c5dc5910618f0a7e1acfb1f284fd49759 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 14 Nov 2019 14:23:37 -0800
Subject: [PATCH 0551/1550] Use the percentage field for profile plots (#3238)

Previously we populated this data, but didn't use it correctly in the
tooltip
---
 distributed/profile.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/distributed/profile.py b/distributed/profile.py
index 274dfcd1d20..1bef6450974 100644
--- a/distributed/profile.py
+++ b/distributed/profile.py
@@ -230,7 +230,7 @@ def traverse(state, start, stop, height):
             x += width
 
     traverse(state, 0, 1, 0)
-    percentages = ["{:.2f}%".format(100 * w) for w in widths]
+    percentages = ["{:.1f}%".format(100 * w) for w in widths]
     return {
         "left": starts,
         "right": stops,
@@ -423,7 +423,7 @@ def plot_figure(data, **kwargs):
             </div>
             <div>
                 <span style="font-size: 14px; font-weight: bold;">Percentage:</span>&nbsp;
-                <span style="font-size: 10px; font-family: Monaco, monospace;">@width</span>
+                <span style="font-size: 10px; font-family: Monaco, monospace;">@percentage</span>
             </div>
             """,
     )

From 4d0d58aade4460fab6e7e85a3548353671036d2c Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 14 Nov 2019 14:58:28 -0800
Subject: [PATCH 0552/1550] bump version to 2.8.0

---
 docs/source/changelog.rst | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index 3cd8e3d287d..9187d0a1579 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,31 @@
 Changelog
 =========
 
+2.8.0 - 2019-11-14
+------------------
+
+-  Add UCX config values (:pr:`3135`) `Matthew Rocklin`_
+-  Relax test_MultiWorker (:pr:`3210`) `Matthew Rocklin`_
+-  Avoid ucp.init at import time (:pr:`3211`) `Matthew Rocklin`_
+-  Clean up rpc to avoid intermittent test failure (:pr:`3215`) `Matthew Rocklin`_
+-  Respect protocol if given to Scheduler (:pr:`3212`) `Matthew Rocklin`_
+-  Use legend_field= keyword in bokeh plots (:pr:`3218`) `Matthew Rocklin`_
+-  Cache psutil.Process object in Nanny (:pr:`3207`) `Matthew Rocklin`_
+-  Replace gen.sleep with asyncio.sleep (:pr:`3208`) `Matthew Rocklin`_
+-  Avoid offloading serialization for small messages (:pr:`3224`) `Matthew Rocklin`_
+-  Add desired_workers metric (:pr:`3221`) `Gabriel Sailer`_
+-  Fail fast when importing distributed.comm.ucx (:pr:`3228`) `Matthew Rocklin`_
+-  Add module name to Future repr (:pr:`3231`) `Matthew Rocklin`_
+-  Add name to Pub/Sub repr (:pr:`3235`) `Matthew Rocklin`_
+-  Import CPU_COUNT from dask.system (:pr:`3199`) `James Bourbeau`_
+-  Efficiently serialize zero strided NumPy arrays (:pr:`3180`) `James Bourbeau`_
+-  Cache function deserialization in workers (:pr:`3234`) `Matthew Rocklin`_
+-  Respect ordering of futures in futures_of (:pr:`3236`) `Matthew Rocklin`_
+-  Bump dask dependency to 2.7.0 (:pr:`3237`) `James Bourbeau`_
+-  Avoid setting inf x_range (:pr:`3229`) `rockwellw`_
+-  Clear task stream based on recent behavior (:pr:`3200`) `Matthew Rocklin`_
+-  Use the percentage field for profile plots (:pr:`3238`) `Matthew Rocklin`_
+
 2.7.0 - 2019-11-08
 ------------------
 
@@ -1365,3 +1390,4 @@ significantly without many new features.
 .. _`James A. Bednar`: https://github.com/jbednar
 .. _`IPetrik`: https://github.com/IPetrik
 .. _`Simon Boothroyd`: https://github.com/SimonBoothroyd
+.. _`rockwellw`: https://github.com/rockwellw

From 8eeb603e7518e692fa2aec940ca59572ed00e24a Mon Sep 17 00:00:00 2001
From: Jim Crist-Harif <jcrist@users.noreply.github.com>
Date: Fri, 15 Nov 2019 11:08:18 -0600
Subject: [PATCH 0553/1550] Use inspect.isawaitable where relevant (#3241)

Noticed a few places in distributed where `hasattr(x, '__await__')` was
checked. This should use `inspect.isawaitable` instead for
readability/correctness. Updated accordingly.
---
 distributed/core.py            | 3 ++-
 distributed/deploy/adaptive.py | 3 ++-
 distributed/scheduler.py       | 3 ++-
 distributed/worker.py          | 9 +++++----
 4 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/distributed/core.py b/distributed/core.py
index d096e0b2274..3e1a3b47cc3 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -2,6 +2,7 @@
 from collections import defaultdict, deque
 from concurrent.futures import CancelledError
 from functools import partial
+from inspect import isawaitable
 import logging
 import threading
 import traceback
@@ -397,7 +398,7 @@ async def handle_comm(self, comm, shutting_down=shutting_down):
                     logger.debug("Calling into handler %s", handler.__name__)
                     try:
                         result = handler(comm, **msg)
-                        if hasattr(result, "__await__"):
+                        if isawaitable(result):
                             result = asyncio.ensure_future(result)
                             self._ongoing_coroutines.add(result)
                             result = await result
diff --git a/distributed/deploy/adaptive.py b/distributed/deploy/adaptive.py
index f173e36a396..0d295200018 100644
--- a/distributed/deploy/adaptive.py
+++ b/distributed/deploy/adaptive.py
@@ -1,3 +1,4 @@
+from inspect import isawaitable
 import logging
 import math
 
@@ -158,7 +159,7 @@ async def scale_down(self, workers):
             # close workers more forcefully
             logger.info("Retiring workers %s", workers)
             f = self.cluster.scale_down(workers)
-            if hasattr(f, "__await__"):
+            if isawaitable(f):
                 await f
 
     async def scale_up(self, n):
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 6724524344d..5cb2267759c 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -3,6 +3,7 @@
 from collections.abc import Mapping, Set
 from datetime import timedelta
 from functools import partial
+from inspect import isawaitable
 import itertools
 import json
 import logging
@@ -3283,7 +3284,7 @@ async def feed(
             if teardown:
                 teardown = pickle.loads(teardown)
             state = setup(self) if setup else None
-            if hasattr(state, "__await__"):
+            if isawaitable(state):
                 state = await state
             try:
                 while self.status == "running":
diff --git a/distributed/worker.py b/distributed/worker.py
index 2a705320baa..e3ef6b260fa 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -4,6 +4,7 @@
 from collections.abc import MutableMapping
 from datetime import timedelta
 import heapq
+from inspect import isawaitable
 import logging
 import os
 from pickle import PicklingError
@@ -748,7 +749,7 @@ async def get_metrics(self):
         for k, metric in self.metrics.items():
             try:
                 result = metric(self)
-                if hasattr(result, "__await__"):
+                if isawaitable(result):
                     result = await result
                 custom[k] = result
             except Exception:  # TODO: log error once
@@ -761,7 +762,7 @@ async def get_startup_information(self):
         for k, f in self.startup_information.items():
             try:
                 v = f(self)
-                if hasattr(v, "__await__"):
+                if isawaitable(v):
                     v = await v
                 result[k] = v
             except Exception:  # TODO: log error once
@@ -1057,7 +1058,7 @@ async def close(
                 if hasattr(plugin, "teardown")
             ]
 
-            await asyncio.gather(*[td for td in teardowns if hasattr(td, "__await__")])
+            await asyncio.gather(*[td for td in teardowns if isawaitable(td)])
 
             for pc in self.periodic_callbacks.values():
                 pc.stop()
@@ -2301,7 +2302,7 @@ async def plugin_add(self, comm=None, plugin=None, name=None):
                 if hasattr(plugin, "setup"):
                     try:
                         result = plugin.setup(worker=self)
-                        if hasattr(result, "__await__"):
+                        if isawaitable(result):
                             result = await result
                     except Exception as e:
                         msg = error_message(e)

From 1a46657af40106fc1fdcc1f2ec6b7f48f5e98566 Mon Sep 17 00:00:00 2001
From: Jim Crist-Harif <jcrist@users.noreply.github.com>
Date: Fri, 15 Nov 2019 13:54:23 -0600
Subject: [PATCH 0554/1550] Remove `gen.coroutine` usage in scheduler (#3242)

Use `async`/`await` and `asyncio` idioms throughout.
---
 distributed/scheduler.py | 116 +++++++++++++++++++--------------------
 1 file changed, 58 insertions(+), 58 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 5cb2267759c..e013bbee9a9 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -25,7 +25,6 @@
     from toolz import frequencies, merge, pluck, merge_sorted, first
 from toolz import valmap, second, compose, groupby
 from tornado import gen
-from tornado.gen import Return
 from tornado.ioloop import IOLoop
 
 import dask
@@ -2008,8 +2007,8 @@ def cancel_key(self, key, client, retries=5, force=False):
             return
         if ts is None or not ts.who_wants:  # no key yet, lets try again in a moment
             if retries:
-                self.loop.add_future(
-                    gen.sleep(0.2), lambda _: self.cancel_key(key, client, retries - 1)
+                self.loop.call_later(
+                    0.2, lambda: self.cancel_key(key, client, retries - 1)
                 )
             return
         if force or ts.who_wants == {cs}:  # no one else wants this key
@@ -2701,8 +2700,7 @@ async def proxy(self, comm=None, msg=None, worker=None, serializers=None):
         )
         return d[worker]
 
-    @gen.coroutine
-    def rebalance(self, comm=None, keys=None, workers=None):
+    async def rebalance(self, comm=None, keys=None, workers=None):
         """ Rebalance keys so that each worker stores roughly equal bytes
 
         **Policy**
@@ -2778,9 +2776,9 @@ def rebalance(self, comm=None, keys=None, workers=None):
                 to_recipients[recipient.address][ts.key].append(sender.address)
                 to_senders[sender.address].append(ts.key)
 
-            result = yield {
-                r: self.rpc(addr=r).gather(who_has=v) for r, v in to_recipients.items()
-            }
+            result = await asyncio.gather(
+                *(self.rpc(addr=r).gather(who_has=v) for r, v in to_recipients.items())
+            )
             for r, v in to_recipients.items():
                 self.log_event(r, {"action": "rebalance", "who_has": v})
 
@@ -2795,13 +2793,11 @@ def rebalance(self, comm=None, keys=None, workers=None):
                 },
             )
 
-            if not all(r["status"] == "OK" for r in result.values()):
-                raise Return(
-                    {
-                        "status": "missing-data",
-                        "keys": sum([r["keys"] for r in result if "keys" in r], []),
-                    }
-                )
+            if not all(r["status"] == "OK" for r in result):
+                return {
+                    "status": "missing-data",
+                    "keys": sum([r["keys"] for r in result if "keys" in r], []),
+                }
 
             for sender, recipient, ts in msgs:
                 assert ts.state == "memory"
@@ -2812,20 +2808,21 @@ def rebalance(self, comm=None, keys=None, workers=None):
                     ("rebalance", ts.key, time(), sender.address, recipient.address)
                 )
 
-            result = yield {
-                r: self.rpc(addr=r).delete_data(keys=v, report=False)
-                for r, v in to_senders.items()
-            }
+            await asyncio.gather(
+                *(
+                    self.rpc(addr=r).delete_data(keys=v, report=False)
+                    for r, v in to_senders.items()
+                )
+            )
 
             for sender, recipient, ts in msgs:
                 ts.who_has.remove(sender)
                 sender.has_what.remove(ts)
                 sender.nbytes -= ts.get_nbytes()
 
-            raise Return({"status": "OK"})
+            return {"status": "OK"}
 
-    @gen.coroutine
-    def replicate(
+    async def replicate(
         self,
         comm=None,
         keys=None,
@@ -2868,7 +2865,7 @@ def replicate(
         tasks = {self.tasks[k] for k in keys}
         missing_data = [ts.key for ts in tasks if not ts.who_has]
         if missing_data:
-            raise Return({"status": "missing-data", "keys": missing_data})
+            return {"status": "missing-data", "keys": missing_data}
 
         # Delete extraneous data
         if delete:
@@ -2879,12 +2876,14 @@ def replicate(
                     for ws in random.sample(del_candidates, len(del_candidates) - n):
                         del_worker_tasks[ws].add(ts)
 
-            yield [
-                self.rpc(addr=ws.address).delete_data(
-                    keys=[ts.key for ts in tasks], report=False
+            await asyncio.gather(
+                *(
+                    self.rpc(addr=ws.address).delete_data(
+                        keys=[ts.key for ts in tasks], report=False
+                    )
+                    for ws, tasks in del_worker_tasks.items()
                 )
-                for ws, tasks in del_worker_tasks.items()
-            ]
+            )
 
             for ws, tasks in del_worker_tasks.items():
                 ws.has_what -= tasks
@@ -2912,11 +2911,13 @@ def replicate(
                 for ws in random.sample(workers - ts.who_has, count):
                     gathers[ws.address][ts.key] = [wws.address for wws in ts.who_has]
 
-            results = yield {
-                w: self.rpc(addr=w).gather(who_has=who_has)
-                for w, who_has in gathers.items()
-            }
-            for w, v in results.items():
+            results = await asyncio.gather(
+                *(
+                    self.rpc(addr=w).gather(who_has=who_has)
+                    for w, who_has in gathers.items()
+                )
+            )
+            for w, v in zip(gathers, results):
                 if v["status"] == "OK":
                     self.add_keys(worker=w, keys=list(gathers[w]))
                 else:
@@ -3349,8 +3350,7 @@ def get_ncores(self, comm=None, workers=None):
         else:
             return {w: ws.nthreads for w, ws in self.workers.items()}
 
-    @gen.coroutine
-    def get_call_stack(self, comm=None, keys=None):
+    async def get_call_stack(self, comm=None, keys=None):
         if keys is not None:
             stack = list(keys)
             processing = set()
@@ -3370,14 +3370,13 @@ def get_call_stack(self, comm=None, keys=None):
             workers = {w: None for w in self.workers}
 
         if not workers:
-            raise gen.Return({})
+            return {}
 
-        else:
-            response = yield {
-                w: self.rpc(w).call_stack(keys=v) for w, v in workers.items()
-            }
-            response = {k: v for k, v in response.items() if v}
-            raise gen.Return(response)
+        results = await asyncio.gather(
+            *(self.rpc(w).call_stack(keys=v) for w, v in workers.items())
+        )
+        response = {w: r for w, r in zip(workers, results) if r}
+        return response
 
     def get_nbytes(self, comm=None, keys=None, summary=True):
         with log_errors():
@@ -4614,8 +4613,7 @@ def worker_objective(self, ts, ws):
         else:
             return (start_time, ws.nbytes)
 
-    @gen.coroutine
-    def get_profile(
+    async def get_profile(
         self,
         comm=None,
         workers=None,
@@ -4628,15 +4626,17 @@ def get_profile(
             workers = self.workers
         else:
             workers = set(self.workers) & set(workers)
-        result = yield {
-            w: self.rpc(w).profile(start=start, stop=stop, key=key) for w in workers
-        }
+        results = await asyncio.gather(
+            *(self.rpc(w).profile(start=start, stop=stop, key=key) for w in workers)
+        )
+
         if merge_workers:
-            result = profile.merge(*result.values())
-        raise gen.Return(result)
+            response = profile.merge(*results)
+        else:
+            response = dict(zip(workers, results))
+        return response
 
-    @gen.coroutine
-    def get_profile_metadata(
+    async def get_profile_metadata(
         self,
         comm=None,
         workers=None,
@@ -4654,22 +4654,22 @@ def get_profile_metadata(
             workers = self.workers
         else:
             workers = set(self.workers) & set(workers)
-        result = yield {
-            w: self.rpc(w).profile_metadata(start=start, stop=stop) for w in workers
-        }
+        results = await asyncio.gather(
+            *(self.rpc(w).profile_metadata(start=start, stop=stop) for w in workers)
+        )
 
-        counts = [v["counts"] for v in result.values()]
+        counts = [v["counts"] for v in results]
         counts = itertools.groupby(merge_sorted(*counts), lambda t: t[0] // dt * dt)
         counts = [(time, sum(pluck(1, group))) for time, group in counts]
 
         keys = set()
-        for v in result.values():
+        for v in results:
             for t, d in v["keys"]:
                 for k in d:
                     keys.add(k)
         keys = {k: [] for k in keys}
 
-        groups1 = [v["keys"] for v in result.values()]
+        groups1 = [v["keys"] for v in results]
         groups2 = list(merge_sorted(*groups1, key=first))
 
         last = 0
@@ -4682,7 +4682,7 @@ def get_profile_metadata(
             for k, v in d.items():
                 keys[k][-1][1] += v
 
-        raise gen.Return({"counts": counts, "keys": keys})
+        return {"counts": counts, "keys": keys}
 
     async def get_worker_logs(self, comm=None, n=None, workers=None, nanny=False):
         results = await self.broadcast(

From 93d631ad84ee8517865a2c761e97ed84c4272ec8 Mon Sep 17 00:00:00 2001
From: "Mads R. B. Kristensen" <madsbk@gmail.com>
Date: Fri, 15 Nov 2019 22:57:46 +0100
Subject: [PATCH 0555/1550] Fixed cupy array going out of scope (#3240)

---
 distributed/protocol/cupy.py | 24 ++++++++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/distributed/protocol/cupy.py b/distributed/protocol/cupy.py
index d15c719359c..26a5accc6af 100644
--- a/distributed/protocol/cupy.py
+++ b/distributed/protocol/cupy.py
@@ -6,14 +6,34 @@
 
 
 class PatchedCudaArrayInterface(object):
-    # TODO: This class wont be necessary
-    #       once Cupy<7.0 is no longer supported
+    """This class do two things:
+        1) Makes sure that __cuda_array_interface__['strides']
+           behaves as specified in the protocol.
+        2) Makes sure that the cuda context is active
+           when deallocating the base cuda array.
+        Notice, this is only needed when the array to deserialize
+        isn't a native cupy array.
+    """
+
     def __init__(self, ary):
         cai = ary.__cuda_array_interface__
         cai_cupy_vsn = cupy.ndarray(0).__cuda_array_interface__["version"]
         if cai.get("strides") is None and cai_cupy_vsn < 2:
             cai.pop("strides", None)
         self.__cuda_array_interface__ = cai
+        # Save a ref to ary so it won't go out of scope
+        self.base = ary
+
+    def __del__(self):
+        # Making sure that the cuda context is active
+        # when deallocating the base cuda array
+        try:
+            import numba.cuda
+
+            numba.cuda.current_context()
+        except ImportError:
+            pass
+        del self.base
 
 
 @cuda_serialize.register(cupy.ndarray)

From 029ed174dce249ea1493fe60f1427827322aa2f5 Mon Sep 17 00:00:00 2001
From: He Jia <he.jia.phy@gmail.com>
Date: Sat, 16 Nov 2019 12:31:41 +0800
Subject: [PATCH 0556/1550] Fixed typos in pubsub.py (#3244)

---
 distributed/pubsub.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/distributed/pubsub.py b/distributed/pubsub.py
index 5ed631c46ec..0a5d82897fd 100644
--- a/distributed/pubsub.py
+++ b/distributed/pubsub.py
@@ -74,7 +74,7 @@ def remove_publisher(self, comm=None, name=None, worker=None):
 
     def remove_subscriber(self, comm=None, name=None, worker=None, client=None):
         if worker:
-            logger.debug("Add worker subscriber: %s %s", name, worker)
+            logger.debug("Remove worker subscriber: %s %s", name, worker)
             self.subscribers[name].remove(worker)
             for pub in self.publishers[name]:
                 self.scheduler.worker_send(
@@ -82,7 +82,7 @@ def remove_subscriber(self, comm=None, name=None, worker=None, client=None):
                     {"op": "pubsub-remove-subscriber", "address": worker, "name": name},
                 )
         elif client:
-            logger.debug("Add client subscriber: %s %s", name, client)
+            logger.debug("Remove client subscriber: %s %s", name, client)
             self.client_subscribers[name].remove(client)
             if not self.client_subscribers[name]:
                 del self.client_subscribers[name]

From 25976b4b8b592388f8b7503dcc4151157553ebaf Mon Sep 17 00:00:00 2001
From: Jed Brown <jed@jedbrown.org>
Date: Mon, 18 Nov 2019 21:44:38 -0700
Subject: [PATCH 0557/1550] docs: fix array.shape() -> array.shape (#3247)

---
 docs/source/efficiency.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/source/efficiency.rst b/docs/source/efficiency.rst
index 39ca16d93d5..94a603ea9a3 100644
--- a/docs/source/efficiency.rst
+++ b/docs/source/efficiency.rst
@@ -31,7 +31,7 @@ shape we might choose one of the following options:
 
 .. code-block:: python
 
-   >>> x.result().shape()  # Slow from lots of data transfer
+   >>> x.result().shape  # Slow from lots of data transfer
    (1000, 1000)
 
 **Fast**

From 35551998d7350cd5ae6a5c24970d8437fd8d521d Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 19 Nov 2019 09:24:32 -0700
Subject: [PATCH 0558/1550] Add new dashboard plot for memory use by key
 (#3243)

---
 distributed/dashboard/components/scheduler.py | 88 ++++++++++++++++++-
 distributed/dashboard/scheduler.py            |  2 +
 .../dashboard/tests/test_scheduler_bokeh.py   | 19 ++++
 3 files changed, 108 insertions(+), 1 deletion(-)

diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index a53a0e744e8..7f226bf7ddc 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -1,3 +1,4 @@
+from collections import defaultdict
 import logging
 import math
 from numbers import Number
@@ -36,7 +37,7 @@
 from bokeh.transform import factor_cmap, linear_cmap
 from bokeh.io import curdoc
 import dask
-from dask.utils import format_bytes
+from dask.utils import format_bytes, key_split
 from toolz import pipe
 from tornado import escape
 
@@ -428,6 +429,82 @@ def update(self):
             update(self.source, result)
 
 
+class MemoryByKey(DashboardComponent):
+    """ Bar chart showing memory use by key prefix"""
+
+    def __init__(self, scheduler, **kwargs):
+        with log_errors():
+            self.last = 0
+            self.scheduler = scheduler
+            self.source = ColumnDataSource(
+                {
+                    "name": ["a", "b"],
+                    "nbytes": [100, 1000],
+                    "count": [1, 2],
+                    "color": ["blue", "blue"],
+                }
+            )
+
+            fig = figure(
+                title="Memory Use",
+                tools="",
+                id="bk-memory-by-key-plot",
+                name="memory_by_key",
+                x_range=["a", "b"],
+                **kwargs,
+            )
+            rect = fig.vbar(
+                source=self.source, x="name", top="nbytes", width=0.9, color="color"
+            )
+            fig.yaxis[0].formatter = NumeralTickFormatter(format="0.0 b")
+            fig.xaxis.major_label_orientation = -math.pi / 12
+            rect.nonselection_glyph = None
+
+            fig.xaxis.minor_tick_line_alpha = 0
+            fig.ygrid.visible = False
+
+            fig.toolbar.logo = None
+            fig.toolbar_location = None
+
+            hover = HoverTool()
+            hover.tooltips = "@name: @nbytes_text"
+            hover.tooltips = """
+            <div>
+                <p><b>Name:</b> @name</p>
+                <p><b>Bytes:</b> @nbytes_text </p>
+                <p><b>Count:</b> @count objects </p>
+            </div>
+            """
+            hover.point_policy = "follow_mouse"
+            fig.add_tools(hover)
+
+            self.fig = fig
+
+    @without_property_validation
+    def update(self):
+        with log_errors():
+            counts = defaultdict(int)
+            nbytes = defaultdict(int)
+            for ws in self.scheduler.workers.values():
+                for ts in ws.has_what:
+                    ks = key_split(ts.key)
+                    counts[ks] += 1
+                    nbytes[ks] += ts.nbytes
+
+            names = list(sorted(counts))
+            self.fig.x_range.factors = names
+            result = {
+                "name": names,
+                "count": [counts[name] for name in names],
+                "nbytes": [nbytes[name] for name in names],
+                "nbytes_text": [format_bytes(nbytes[name]) for name in names],
+                "color": [color_of(name) for name in names],
+            }
+            self.fig.title.text = "Total Use: " + format_bytes(sum(nbytes.values()))
+
+            update(self.source, result)
+
+
 class CurrentLoad(DashboardComponent):
     """ How many tasks are on each worker """
 
@@ -1865,6 +1942,15 @@ def individual_bandwidth_workers_doc(scheduler, extra, doc):
         doc.theme = BOKEH_THEME
 
 
+def individual_memory_by_key_doc(scheduler, extra, doc):
+    with log_errors():
+        component = MemoryByKey(scheduler, sizing_mode="stretch_both")
+        component.update()
+        add_periodic_callback(doc, component, 500)
+        doc.add_root(component.fig)
+        doc.theme = BOKEH_THEME
+
+
 def profile_doc(scheduler, extra, doc):
     with log_errors():
         doc.title = "Dask: Profile"
diff --git a/distributed/dashboard/scheduler.py b/distributed/dashboard/scheduler.py
index ecc413b5f0d..67d08c50bd0 100644
--- a/distributed/dashboard/scheduler.py
+++ b/distributed/dashboard/scheduler.py
@@ -36,6 +36,7 @@
     individual_workers_doc,
     individual_bandwidth_types_doc,
     individual_bandwidth_workers_doc,
+    individual_memory_by_key_doc,
 )
 from .core import BokehServer
 from .worker import counters_doc
@@ -408,6 +409,7 @@ def listen(self, *args, **kwargs):
     "/individual-workers": individual_workers_doc,
     "/individual-bandwidth-types": individual_bandwidth_types_doc,
     "/individual-bandwidth-workers": individual_bandwidth_workers_doc,
+    "/individual-memory-by-key": individual_memory_by_key_doc,
 }
 
 try:
diff --git a/distributed/dashboard/tests/test_scheduler_bokeh.py b/distributed/dashboard/tests/test_scheduler_bokeh.py
index 875f1064503..14d055baa7f 100644
--- a/distributed/dashboard/tests/test_scheduler_bokeh.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh.py
@@ -11,6 +11,7 @@
 from tornado import gen
 from tornado.httpclient import AsyncHTTPClient, HTTPRequest
 
+import dask
 from dask.core import flatten
 from distributed.utils import tokey, format_dashboard_link
 from distributed.client import wait
@@ -34,6 +35,7 @@
     WorkerTable,
     TaskGraph,
     ProfileServer,
+    MemoryByKey,
 )
 
 from distributed.dashboard import scheduler
@@ -690,3 +692,20 @@ def test_https_support(c, s, a, b):
         body = response.body.decode()
         assert "bokeh" in body.lower()
         assert not re.search("href=./", body)  # no absolute links
+
+
+@gen_cluster(
+    client=True, scheduler_kwargs={"services": {("dashboard", 0): BokehScheduler}}
+)
+async def test_memory_by_key(c, s, a, b):
+    mbk = MemoryByKey(s)
+
+    da = pytest.importorskip("dask.array")
+    x = (da.random.random((20, 20), chunks=(10, 10)) + 1).persist(optimize_graph=False)
+    await x
+
+    y = await dask.delayed(inc)(1).persist()
+
+    mbk.update()
+    assert mbk.source.data["name"] == ["add", "inc"]
+    assert mbk.source.data["nbytes"] == [x.nbytes, sys.getsizeof(1)]

From be4e9661edc01dc3040871803da8b671c2188066 Mon Sep 17 00:00:00 2001
From: Peter Andreas Entschev <peter@entschev.com>
Date: Thu, 21 Nov 2019 15:04:15 +0100
Subject: [PATCH 0559/1550] Skip numba.cuda tests if CUDA is not available
 (#3255)

---
 distributed/protocol/tests/test_numba.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/distributed/protocol/tests/test_numba.py b/distributed/protocol/tests/test_numba.py
index 794db58b3c9..69ea73310d4 100644
--- a/distributed/protocol/tests/test_numba.py
+++ b/distributed/protocol/tests/test_numba.py
@@ -7,6 +7,9 @@
 
 @pytest.mark.parametrize("dtype", ["u1", "u4", "u8", "f4"])
 def test_serialize_cupy(dtype):
+    if not cuda.is_available():
+        pytest.skip("CUDA is not available")
+
     ary = np.arange(100, dtype=dtype)
     x = cuda.to_device(ary)
     header, frames = serialize(x, serializers=("cuda", "dask", "pickle"))

From 5b33d54cd9afd174f827bcd139a581aa215f95a7 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Fri, 22 Nov 2019 06:13:07 -0600
Subject: [PATCH 0560/1550] Fix NumPy writeable serialization bug (#3253)

* Add failing test

* Pass broadcasted shape to np.ndarray
---
 distributed/protocol/numpy.py            | 14 ++++++--------
 distributed/protocol/tests/test_numpy.py | 19 +++++++++++++++++++
 2 files changed, 25 insertions(+), 8 deletions(-)

diff --git a/distributed/protocol/numpy.py b/distributed/protocol/numpy.py
index c5061a8f802..9a1f493c333 100644
--- a/distributed/protocol/numpy.py
+++ b/distributed/protocol/numpy.py
@@ -46,7 +46,7 @@ def serialize_numpy_ndarray(x):
 
     # Only serialize non-broadcasted data for arrays with zero strided axes
     if 0 in x.strides:
-        broadcast_to = (x.shape, x.flags.writeable)
+        broadcast_to = x.shape
         x = x[tuple(slice(None) if s != 0 else slice(1) for s in x.strides)]
     else:
         broadcast_to = None
@@ -103,14 +103,12 @@ def deserialize_numpy_ndarray(header, frames):
         else:
             dt = np.dtype(dt)
 
-        x = np.ndarray(
-            header["shape"], dtype=dt, buffer=frames[0], strides=header["strides"]
-        )
-
         if header.get("broadcast_to"):
-            shape, writeable = header["broadcast_to"]
-            x = np.broadcast_to(x, shape)
-            x.setflags(write=writeable)
+            shape = header["broadcast_to"]
+        else:
+            shape = header["shape"]
+
+        x = np.ndarray(shape, dtype=dt, buffer=frames[0], strides=header["strides"])
 
         return x
 
diff --git a/distributed/protocol/tests/test_numpy.py b/distributed/protocol/tests/test_numpy.py
index 6e4712272d8..432b749e27e 100644
--- a/distributed/protocol/tests/test_numpy.py
+++ b/distributed/protocol/tests/test_numpy.py
@@ -288,3 +288,22 @@ def test_non_zero_strided_array():
     header, frames = serialize(x)
     assert "broadcast_to" not in header
     assert sum(map(nbytes, frames)) == x.nbytes
+
+
+def test_serialize_writeable_array_readonly_base_object():
+    # Regression test for https://github.com/dask/distributed/issues/3252
+
+    x = np.arange(3)
+    # Create array which doesn't own it's own memory
+    y = np.broadcast_to(x, (3, 3))
+
+    # Make y writeable and it's base object (x) read-only
+    y.setflags(write=True)
+    x.setflags(write=False)
+
+    # Serialize / deserialize y
+    z = deserialize(*serialize(y))
+    np.testing.assert_equal(z, y)
+
+    # Ensure z and y have the same flags (including WRITEABLE)
+    assert z.flags == y.flags

From 42a51ce086e9d8badbe68e139db471ce7b28befa Mon Sep 17 00:00:00 2001
From: Tom Augspurger <TomAugspurger@users.noreply.github.com>
Date: Fri, 22 Nov 2019 15:41:15 -0600
Subject: [PATCH 0561/1550] Fix hanging worker when the scheduler leaves
 (#3250)

Closes https://github.com/dask/distributed/issues/2880
---
 distributed/tests/test_nanny.py | 15 +++++++++++++++
 distributed/worker.py           |  6 ++++++
 2 files changed, 21 insertions(+)

diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
index fccfd2efde6..cacd98477e0 100644
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -1,3 +1,4 @@
+import asyncio
 import gc
 import logging
 import os
@@ -129,6 +130,20 @@ def test_run(s):
     yield n.close()
 
 
+@pytest.mark.slow
+@gen_cluster(config={"distributed.comm.timeouts.connect": "1s"})
+async def test_no_hang_when_scheduler_closes(s, a, b):
+    # https://github.com/dask/distributed/issues/2880
+    with captured_logger("tornado.application", logging.ERROR) as logger:
+        await s.close()
+        await asyncio.sleep(1.2)
+        assert a.status == "closed"
+        assert b.status == "closed"
+
+    out = logger.getvalue()
+    assert "Timed out trying to connect" not in out
+
+
 @pytest.mark.slow
 @gen_cluster(
     Worker=Nanny, nthreads=[("127.0.0.1", 1)], worker_kwargs={"reconnect": False}
diff --git a/distributed/worker.py b/distributed/worker.py
index e3ef6b260fa..e5666aca561 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -882,6 +882,12 @@ async def heartbeat(self):
                 )
                 self.bandwidth_workers.clear()
                 self.bandwidth_types.clear()
+            except IOError as e:
+                # Scheduler is gone. Respect distributed.comm.timeouts.connect
+                if "Timed out trying to connect" in str(e):
+                    await self.close(report=False)
+                else:
+                    raise e
             except CommClosedError:
                 logger.warning("Heartbeat to scheduler failed")
             finally:

From 507659d79434845e50d48c247ff42d5efd336686 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Fri, 22 Nov 2019 22:46:55 -0600
Subject: [PATCH 0562/1550] bump version to 2.8.1

---
 docs/source/changelog.rst | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index 9187d0a1579..21a57806533 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,20 @@
 Changelog
 =========
 
+2.8.1 - 2019-11-22
+------------------
+
+- Fix hanging worker when the scheduler leaves (:pr:`3250`) `Tom Augspurger`_
+- Fix NumPy writeable serialization bug (:pr:`3253`) `James Bourbeau`_
+- Skip ``numba.cuda`` tests if CUDA is not available (:pr:`3255`) `Peter Andreas Entschev`_
+- Add new dashboard plot for memory use by key (:pr:`3243`) `Matthew Rocklin`_
+- Fix ``array.shape()`` -> ``array.shape`` (:pr:`3247`) `Jed Brown`_
+- Fixed typos in ``pubsub.py`` (:pr:`3244`) `He Jia`_
+- Fixed cupy array going out of scope (:pr:`3240`) `Mads R. B. Kristensen`_
+- Remove ``gen.coroutine`` usage in scheduler (:pr:`3242`) `Jim Crist-Harif`_
+- Use ``inspect.isawaitable`` where relevant (:pr:`3241`) `Jim Crist-Harif`_
+
+
 2.8.0 - 2019-11-14
 ------------------
 
@@ -1391,3 +1405,6 @@ significantly without many new features.
 .. _`IPetrik`: https://github.com/IPetrik
 .. _`Simon Boothroyd`: https://github.com/SimonBoothroyd
 .. _`rockwellw`: https://github.com/rockwellw
+.. _`Jed Brown`: https://github.com/jedbrown
+.. _`He Jia`: https://github.com/HerculesJack
+.. _`Jim Crist-Harif`: https://github.com/jcrist

From bacfa51bf95dddcb75beaa62141e487b52e604cf Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sat, 23 Nov 2019 14:57:22 -0700
Subject: [PATCH 0563/1550] Add validate options to configuraation (#3258)

---
 distributed/distributed.yaml | 2 ++
 distributed/scheduler.py     | 4 +++-
 distributed/worker.py        | 4 +++-
 3 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index 92b7c15e157..59bf6f8dc1c 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -24,6 +24,7 @@ distributed:
     preload: []
     preload-argv: []
     default-task-durations: {}  # How long we expect function names to run ("1h", "1s") (helps for long tasks)
+    validate: False         # Check scheduler state at every step for debugging
     dashboard:
       status:
         task-stream-length: 1000
@@ -44,6 +45,7 @@ distributed:
     preload: []
     preload-argv: []
     daemon: True
+    validate: False         # Check worker state at every step for debugging
     lifetime:
       duration: null        # Time after which to gracefully shutdown the worker
       stagger: 0 seconds    # Random amount by which to stagger lifetimes
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index e013bbee9a9..94bb5ea756f 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -840,7 +840,7 @@ def __init__(
         service_kwargs=None,
         allowed_failures=None,
         extensions=None,
-        validate=False,
+        validate=None,
         scheduler_file=None,
         security=None,
         worker_ttl=None,
@@ -860,6 +860,8 @@ def __init__(
         if allowed_failures is None:
             allowed_failures = dask.config.get("distributed.scheduler.allowed-failures")
         self.allowed_failures = allowed_failures
+        if validate is None:
+            validate = dask.config.get("distributed.scheduler.validate")
         self.validate = validate
         self.status = None
         self.proc = psutil.Process()
diff --git a/distributed/worker.py b/distributed/worker.py
index e5666aca561..76ee7a723d5 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -320,7 +320,7 @@ def __init__(
         nanny=None,
         plugins=(),
         low_level_profiler=dask.config.get("distributed.worker.profile.low-level"),
-        validate=False,
+        validate=None,
         profile_cycle_interval=None,
         lifetime=None,
         lifetime_stagger=None,
@@ -386,6 +386,8 @@ def __init__(
         self.target_message_size = 50e6  # 50 MB
 
         self.log = deque(maxlen=100000)
+        if validate is None:
+            validate = dask.config.get("distributed.scheduler.validate")
         self.validate = validate
 
         self._transitions = {

From d2b43fbfed1fa0385c054987a170b09f32af4b9a Mon Sep 17 00:00:00 2001
From: Elliott Sales de Andrade <quantum.analyst@gmail.com>
Date: Mon, 25 Nov 2019 07:17:48 -0500
Subject: [PATCH 0564/1550] Fix dev requirements for pytest. (#3264)

The minimum version specified in `setup.cfg` is newer than
`dev-requirements.txt`.
---
 dev-requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dev-requirements.txt b/dev-requirements.txt
index 3c4cf7954a3..cd79b3e4317 100644
--- a/dev-requirements.txt
+++ b/dev-requirements.txt
@@ -7,6 +7,6 @@ pyzmq >= 16.0.2
 ipython >= 5.0.0
 jupyter_client >= 4.4.0
 ipykernel >= 4.5.2
-pytest >= 3.0.5
+pytest >= 3.2
 prometheus_client >= 0.6.0
 jupyter-server-proxy >= 1.1.0

From a285267d3c4042cc3aada675d6dabb793e43c500 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Mon, 25 Nov 2019 15:55:53 -0500
Subject: [PATCH 0565/1550] Use `DeviceBuffer` from newer RMM releases (#3261)

If a newer version of RMM is around, use `DeviceBuffer` instead.
Otherwise fallback to `device_array`. This is significantly faster to
allocate.
---
 distributed/comm/ucx.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/distributed/comm/ucx.py b/distributed/comm/ucx.py
index 58b16eaaf7f..2bdcff5e958 100644
--- a/distributed/comm/ucx.py
+++ b/distributed/comm/ucx.py
@@ -46,7 +46,10 @@ def init_once():
     try:
         import rmm
 
-        cuda_array = lambda n: rmm.device_array(n, dtype=np.uint8)
+        if hasattr(rmm, "DeviceBuffer"):
+            cuda_array = lambda n: rmm.DeviceBuffer(size=n)
+        else:  # pre-0.12.0
+            cuda_array = lambda n: rmm.device_array(n, dtype=np.uint8)
     except ImportError:
         try:
             import numba.cuda

From 1d9aaac6c67e6a6e7b49d3d1ec3216a4994482f1 Mon Sep 17 00:00:00 2001
From: fjetter <fjetter@users.noreply.github.com>
Date: Tue, 26 Nov 2019 16:27:34 +0100
Subject: [PATCH 0566/1550] Robust gather in case of connection failures
 (#3246)

---
 distributed/comm/core.py             |   2 +-
 distributed/scheduler.py             |  14 ++-
 distributed/tests/test_scheduler.py  | 145 ++++++++++++++++++++++++++-
 distributed/tests/test_utils_comm.py |  43 +++++++-
 distributed/worker.py                |  59 +++++++----
 5 files changed, 232 insertions(+), 31 deletions(-)

diff --git a/distributed/comm/core.py b/distributed/comm/core.py
index 39a8b123cd3..0befb36d712 100644
--- a/distributed/comm/core.py
+++ b/distributed/comm/core.py
@@ -59,7 +59,7 @@ def read(self, deserializers=None):
         """
 
     @abstractmethod
-    def write(self, msg, on_error=None):
+    def write(self, msg, serializers=None, on_error=None):
         """
         Write a message (a Python object).
 
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 94bb5ea756f..26b4cf1c67e 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -2547,7 +2547,7 @@ async def gather(self, comm=None, keys=None, serializers=None):
                 (self.tasks[key].state if key in self.tasks else None)
                 for key in missing_keys
             ]
-            logger.debug(
+            logger.exception(
                 "Couldn't gather keys %s state: %s workers: %s",
                 missing_keys,
                 missing_states,
@@ -2555,17 +2555,21 @@ async def gather(self, comm=None, keys=None, serializers=None):
             )
             result = {"status": "error", "keys": missing_keys}
             with log_errors():
+                # Remove suspicious workers from the scheduler but allow them to
+                # reconnect.
                 for worker in missing_workers:
-                    self.remove_worker(address=worker)  # this is extreme
+                    self.remove_worker(address=worker, close=False)
                 for key, workers in missing_keys.items():
-                    if not workers:
-                        continue
-                    ts = self.tasks[key]
+                    # Task may already be gone if it was held by a
+                    # `missing_worker`
+                    ts = self.tasks.get(key)
                     logger.exception(
                         "Workers don't have promised key: %s, %s",
                         str(workers),
                         str(key),
                     )
+                    if not workers or ts is None:
+                        continue
                     for worker in workers:
                         ws = self.workers.get(worker)
                         if ws is not None and ts in ws.has_what:
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index f57dbfb9e07..4e0e9a8710c 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -17,7 +17,8 @@
 import pytest
 
 from distributed import Nanny, Worker, Client, wait, fire_and_forget
-from distributed.core import connect, rpc
+from distributed.comm import Comm
+from distributed.core import connect, rpc, ConnectionPool
 from distributed.scheduler import Scheduler, TaskState
 from distributed.client import wait
 from distributed.metrics import time
@@ -1704,3 +1705,145 @@ async def test_no_danglng_asyncio_tasks(cleanup):
 
     tasks = asyncio.all_tasks()
     assert tasks == start
+
+
+class BrokenComm(Comm):
+    peer_address = None
+    local_address = None
+
+    def close(self):
+        pass
+
+    def closed(self):
+        pass
+
+    def abort(self):
+        pass
+
+    def read(self, deserializers=None):
+        raise EnvironmentError
+
+    def write(self, msg, serializers=None, on_error=None):
+        raise EnvironmentError
+
+
+class FlakyConnectionPool(ConnectionPool):
+    def __init__(self, *args, failing_connections=0, **kwargs):
+        self.cnn_count = 0
+        self.failing_connections = failing_connections
+        super(FlakyConnectionPool, self).__init__(*args, **kwargs)
+
+    async def connect(self, *args, **kwargs):
+        self.cnn_count += 1
+        if self.cnn_count > self.failing_connections:
+            return await super(FlakyConnectionPool, self).connect(*args, **kwargs)
+        else:
+            return BrokenComm()
+
+
+@gen_cluster(client=True)
+async def test_gather_failing_cnn_recover(c, s, a, b):
+    orig_rpc = s.rpc
+    x = await c.scatter({"x": 1}, workers=a.address)
+
+    s.rpc = FlakyConnectionPool(failing_connections=1)
+    res = await s.gather(keys=["x"])
+    assert res["status"] == "OK"
+
+
+@gen_cluster(client=True)
+async def test_gather_failing_cnn_error(c, s, a, b):
+    orig_rpc = s.rpc
+    x = await c.scatter({"x": 1}, workers=a.address)
+
+    s.rpc = FlakyConnectionPool(failing_connections=10)
+    res = await s.gather(keys=["x"])
+    assert res["status"] == "error"
+    assert list(res["keys"]) == ["x"]
+
+
+@gen_cluster(client=True)
+async def test_gather_no_workers(c, s, a, b):
+    await asyncio.sleep(1)
+    x = await c.scatter({"x": 1}, workers=a.address)
+
+    await a.close()
+    await b.close()
+
+    res = await s.gather(keys=["x"])
+    assert res["status"] == "error"
+    assert list(res["keys"]) == ["x"]
+
+
+@gen_cluster(client=True, client_kwargs={"direct_to_workers": False})
+async def test_gather_allow_worker_reconnect(c, s, a, b):
+    """
+    Test that client resubmissions allow failed workers to reconnect and re-use
+    their results. Failure scenario would be a connection issue during result
+    gathering.
+    Upon connection failure, the worker is flagged as suspicious and removed
+    from the scheduler. If the worker is healthy and reconnencts we want to use
+    its results instead of recomputing them.
+    """
+    # GH3246
+    ALREADY_CALCULATED = []
+
+    import time
+
+    def inc_slow(x):
+        # Once the graph below is rescheduled this computation runs again. We
+        # need to sleep for at least 0.5 seconds to give the worker a chance to
+        # reconnect (Heartbeat timing)
+        if x in ALREADY_CALCULATED:
+            time.sleep(0.5)
+        ALREADY_CALCULATED.append(x)
+        return x + 1
+
+    x = c.submit(inc_slow, 1)
+    y = c.submit(inc_slow, 2)
+
+    def reducer(x, y):
+        return x + y
+
+    z = c.submit(reducer, x, y)
+
+    s.rpc = FlakyConnectionPool(failing_connections=4)
+
+    with captured_logger(logging.getLogger("distributed.scheduler")) as sched_logger:
+        with captured_logger(logging.getLogger("distributed.client")) as client_logger:
+            with captured_logger(
+                logging.getLogger("distributed.worker")
+            ) as worker_logger:
+                # Gather using the client (as an ordinary user would)
+                # Upon a missing key, the client will reschedule the computations
+                res = await c.gather(z)
+
+    assert res == 5
+
+    sched_logger = sched_logger.getvalue()
+    client_logger = client_logger.getvalue()
+    worker_logger = worker_logger.getvalue()
+
+    # Ensure that the communication was done via the scheduler, i.e. we actually hit a bad connection
+    assert s.rpc.cnn_count > 0
+
+    assert "Encountered connection issue during data collection" in worker_logger
+
+    # The reducer task was actually not found upon first collection. The client will reschedule the graph
+    assert "Couldn't gather 1 keys, rescheduling" in client_logger
+    # There will also be a `Unexpected worker completed task` message but this
+    # is rather an artifact and not the intention
+    assert "Workers don't have promised key" in sched_logger
+
+    # Once the worker reconnects, it will also submit the keys it holds such
+    # that the scheduler again knows about the result.
+    # The final reduce step should then be used from the re-connected worker
+    # instead of recomputing it.
+
+    starts = []
+    finish_processing_transitions = 0
+    for transition in s.transition_log:
+        key, start, finish, recommendations, timestamp = transition
+        if "reducer" in key and finish == "processing":
+            finish_processing_transitions += 1
+    assert finish_processing_transitions == 1
diff --git a/distributed/tests/test_utils_comm.py b/distributed/tests/test_utils_comm.py
index 224b4b7f181..f66d3ba62d5 100644
--- a/distributed/tests/test_utils_comm.py
+++ b/distributed/tests/test_utils_comm.py
@@ -1,6 +1,5 @@
-import pytest
-
-from distributed.core import rpc
+from distributed.core import ConnectionPool
+from distributed.comm import Comm
 from distributed.utils_test import gen_cluster
 from distributed.utils_comm import pack_data, gather_from_workers
 
@@ -12,9 +11,9 @@ def test_pack_data():
     assert pack_data({"a": ["x"], "b": "y"}, data) == {"a": [1], "b": "y"}
 
 
-@pytest.mark.xfail(reason="rpc now needs to be a connection pool")
 @gen_cluster(client=True)
 def test_gather_from_workers_permissive(c, s, a, b):
+    rpc = ConnectionPool()
     x = yield c.scatter({"x": 1}, workers=a.address)
 
     data, missing, bad_workers = yield gather_from_workers(
@@ -23,3 +22,39 @@ def test_gather_from_workers_permissive(c, s, a, b):
 
     assert data == {"x": 1}
     assert list(missing) == ["y"]
+
+
+class BrokenComm(Comm):
+    peer_address = None
+    local_address = None
+
+    def close(self):
+        pass
+
+    def closed(self):
+        pass
+
+    def abort(self):
+        pass
+
+    def read(self, deserializers=None):
+        raise EnvironmentError
+
+    def write(self, msg, serializers=None, on_error=None):
+        raise EnvironmentError
+
+
+class BrokenConnectionPool(ConnectionPool):
+    async def connect(self, *args, **kwargs):
+        return BrokenComm()
+
+
+@gen_cluster(client=True)
+def test_gather_from_workers_permissive_flaky(c, s, a, b):
+    x = yield c.scatter({"x": 1}, workers=a.address)
+
+    rpc = BrokenConnectionPool()
+    data, missing, bad_workers = yield gather_from_workers({"x": [a.address]}, rpc=rpc)
+
+    assert missing == {"x": [a.address]}
+    assert bad_workers == [a.address]
diff --git a/distributed/worker.py b/distributed/worker.py
index 76ee7a723d5..fb46dc2391c 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -3125,27 +3125,46 @@ async def get_data_from_worker(
     if deserializers is None:
         deserializers = rpc.deserializers
 
-    comm = await rpc.connect(worker)
-    comm.name = "Ephemeral Worker->Worker for gather"
-    try:
-        response = await send_recv(
-            comm,
-            serializers=serializers,
-            deserializers=deserializers,
-            op="get_data",
-            keys=keys,
-            who=who,
-            max_connections=max_connections,
-        )
+    retry_count = 0
+    max_retries = 3
+
+    while True:
+        comm = await rpc.connect(worker)
+        comm.name = "Ephemeral Worker->Worker for gather"
         try:
-            status = response["status"]
-        except KeyError:
-            raise ValueError("Unexpected response", response)
-        else:
-            if status == "OK":
-                await comm.write("OK")
-    finally:
-        rpc.reuse(worker, comm)
+            response = await send_recv(
+                comm,
+                serializers=serializers,
+                deserializers=deserializers,
+                op="get_data",
+                keys=keys,
+                who=who,
+                max_connections=max_connections,
+            )
+            try:
+                status = response["status"]
+            except KeyError:
+                raise ValueError("Unexpected response", response)
+            else:
+                if status == "OK":
+                    await comm.write("OK")
+            break
+        except (EnvironmentError, CommClosedError):
+            if retry_count < max_retries:
+                await asyncio.sleep(0.1 * (2 ** retry_count))
+                retry_count += 1
+                logger.info(
+                    "Encountered connection issue during data collection of keys %s on worker %s. Retrying (%s / %s)",
+                    keys,
+                    worker,
+                    retry_count,
+                    max_retries,
+                )
+                continue
+            else:
+                raise
+        finally:
+            rpc.reuse(worker, comm)
 
     return response
 

From ec29c04a317cf5d0c418ee9803723839e04a16d2 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 26 Nov 2019 08:17:08 -0800
Subject: [PATCH 0567/1550] Use base-2 values for byte-valued axes in dashboard
 (#3267)

---
 distributed/dashboard/components/scheduler.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index 7f226bf7ddc..a02ca73b63d 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -24,6 +24,7 @@
     LinearAxis,
     NumeralTickFormatter,
     BoxZoomTool,
+    AdaptiveTicker,
     BasicTicker,
     NumberFormatter,
     BoxSelectTool,
@@ -241,6 +242,7 @@ def __init__(self, scheduler, **kwargs):
             )
 
             self.root.xaxis[0].formatter = NumeralTickFormatter(format="0.0 b")
+            self.root.xaxis.ticker = AdaptiveTicker(mantissas=[1, 256, 512], base=1024)
             self.root.xaxis.major_label_orientation = -math.pi / 12
 
             self.root.xaxis.minor_tick_line_alpha = 0
@@ -303,6 +305,7 @@ def __init__(self, scheduler, **kwargs):
             )
             fig.x_range.start = 0
             fig.xaxis[0].formatter = NumeralTickFormatter(format="0.0 b")
+            fig.xaxis.ticker = AdaptiveTicker(mantissas=[1, 256, 512], base=1024)
             rect.nonselection_glyph = None
 
             fig.xaxis.minor_tick_line_alpha = 0
@@ -457,6 +460,7 @@ def __init__(self, scheduler, **kwargs):
                 source=self.source, x="name", top="nbytes", width=0.9, color="color"
             )
             fig.yaxis[0].formatter = NumeralTickFormatter(format="0.0 b")
+            fig.yaxis.ticker = AdaptiveTicker(mantissas=[1, 256, 512], base=1024)
             fig.xaxis.major_label_orientation = -math.pi / 12
             rect.nonselection_glyph = None
 

From 6af261770072313392b0a4cac90d01bfb68f078c Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 26 Nov 2019 08:17:41 -0800
Subject: [PATCH 0568/1550] Set x_range in CPU plot based on the number of
 threads (#3266)

---
 distributed/dashboard/components/scheduler.py | 19 +++++++++----------
 .../dashboard/tests/test_scheduler_bokeh.py   |  2 ++
 2 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index a02ca73b63d..d5aa2c1d338 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -19,7 +19,6 @@
     Range1d,
     Plot,
     Quad,
-    Span,
     value,
     LinearAxis,
     NumeralTickFormatter,
@@ -576,6 +575,7 @@ def __init__(self, scheduler, width=600, **kwargs):
                 id="bk-cpu-worker-plot",
                 width=int(width / 2),
                 name="cpu_hist",
+                x_range=(0, None),
                 **kwargs,
             )
             rect = cpu.rect(
@@ -587,21 +587,13 @@ def __init__(self, scheduler, width=600, **kwargs):
                 color="blue",
             )
             rect.nonselection_glyph = None
-            hundred_span = Span(
-                location=100,
-                dimension="height",
-                line_color="gray",
-                line_dash="dashed",
-                line_width=3,
-            )
-            cpu.add_layout(hundred_span)
 
             nbytes.axis[0].ticker = BasicTicker(mantissas=[1, 256, 512], base=1024)
             nbytes.xaxis[0].formatter = NumeralTickFormatter(format="0.0 b")
             nbytes.xaxis.major_label_orientation = -math.pi / 12
             nbytes.x_range.start = 0
 
-            for fig in [processing, nbytes]:
+            for fig in [processing, nbytes, cpu]:
                 fig.xaxis.minor_tick_line_alpha = 0
                 fig.yaxis.visible = False
                 fig.ygrid.visible = False
@@ -698,6 +690,13 @@ def update(self):
                     sum(nbytes)
                 )
                 self.nbytes_figure.x_range.end = max_limit
+                if self.scheduler.workers:
+                    self.cpu_figure.x_range.end = (
+                        max(ws.nthreads or 1 for ws in self.scheduler.workers.values())
+                        * 100
+                    )
+                else:
+                    self.cpu_figure.x_range.end = 100
 
                 update(self.source, result)
 
diff --git a/distributed/dashboard/tests/test_scheduler_bokeh.py b/distributed/dashboard/tests/test_scheduler_bokeh.py
index 14d055baa7f..fe92c805efa 100644
--- a/distributed/dashboard/tests/test_scheduler_bokeh.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh.py
@@ -274,6 +274,8 @@ def test_CurrentLoad(c, s, a, b):
     assert all(len(L) == 2 for L in d.values())
     assert all(d["nbytes"])
 
+    assert cl.cpu_figure.x_range.end == 200
+
 
 @gen_cluster(client=True)
 def test_ProcessingHistogram(c, s, a, b):

From 856bba7ca913163493d406a87205629f1d021a23 Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Tue, 26 Nov 2019 17:07:11 +0000
Subject: [PATCH 0569/1550] Fix layout scaling on profile plots (#3268)

---
 distributed/dashboard/components/scheduler.py | 4 ++--
 distributed/dashboard/components/shared.py    | 9 +++++----
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index d5aa2c1d338..5b01a99f7a6 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -1957,7 +1957,7 @@ def individual_memory_by_key_doc(scheduler, extra, doc):
 def profile_doc(scheduler, extra, doc):
     with log_errors():
         doc.title = "Dask: Profile"
-        prof = ProfileTimePlot(scheduler, sizing_mode="scale_width", doc=doc)
+        prof = ProfileTimePlot(scheduler, sizing_mode="stretch_both", doc=doc)
         doc.add_root(prof.root)
         doc.template = env.get_template("simple.html")
         doc.template_variables.update(extra)
@@ -1969,7 +1969,7 @@ def profile_doc(scheduler, extra, doc):
 def profile_server_doc(scheduler, extra, doc):
     with log_errors():
         doc.title = "Dask: Profile of Event Loop"
-        prof = ProfileServer(scheduler, sizing_mode="scale_width", doc=doc)
+        prof = ProfileServer(scheduler, sizing_mode="stretch_both", doc=doc)
         doc.add_root(prof.root)
         doc.template = env.get_template("simple.html")
         doc.template_variables.update(extra)
diff --git a/distributed/dashboard/components/shared.py b/distributed/dashboard/components/shared.py
index 882db411434..ec3a207fc8b 100644
--- a/distributed/dashboard/components/shared.py
+++ b/distributed/dashboard/components/shared.py
@@ -312,12 +312,12 @@ def cb(attr, old, new):
         self.ts_source = ColumnDataSource({"time": [], "count": []})
         self.ts_plot = figure(
             title="Activity over time",
-            height=100,
+            height=150,
             x_axis_type="datetime",
             active_drag="xbox_select",
             y_range=[0, 1 / profile_interval],
             tools="xpan,xwheel_zoom,xbox_select,reset",
-            **kwargs
+            sizing_mode="stretch_width",
         )
         self.ts_plot.line("time", "count", source=self.ts_source)
         self.ts_plot.circle(
@@ -367,6 +367,7 @@ def select_cb(attr, old, new):
                 self.reset_button,
                 self.update_button,
                 sizing_mode="scale_width",
+                height=250,
             ),
             self.profile_plot,
             self.ts_plot,
@@ -464,12 +465,12 @@ def cb(attr, old, new):
         self.ts_source = ColumnDataSource({"time": [], "count": []})
         self.ts_plot = figure(
             title="Activity over time",
-            height=100,
+            height=150,
             x_axis_type="datetime",
             active_drag="xbox_select",
             y_range=[0, 1 / profile_interval],
             tools="xpan,xwheel_zoom,xbox_select,reset",
-            **kwargs
+            sizing_mode="stretch_width",
         )
         self.ts_plot.line("time", "count", source=self.ts_source)
         self.ts_plot.circle(

From b17a1ad04168dd12dac3ce5f31a13226d3d7e52a Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 26 Nov 2019 14:45:14 -0800
Subject: [PATCH 0570/1550] Add offload size to configuration (#3270)

---
 distributed/comm/utils.py    | 10 +++++++---
 distributed/distributed.yaml |  1 +
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/distributed/comm/utils.py b/distributed/comm/utils.py
index 5b15d5c798c..4862aace207 100644
--- a/distributed/comm/utils.py
+++ b/distributed/comm/utils.py
@@ -1,7 +1,9 @@
 import logging
 import socket
 
+import dask
 from dask.sizeof import sizeof
+from dask.utils import parse_bytes
 
 from .. import protocol
 from ..utils import get_ip, get_ipv6, nbytes, offload
@@ -13,7 +15,9 @@
 # Offload (de)serializing large frames to improve event loop responsiveness.
 # We use at most 4 threads to allow for parallel processing of large messages.
 
-FRAME_OFFLOAD_THRESHOLD = 10 * 1024 ** 2  # 10 MB
+FRAME_OFFLOAD_THRESHOLD = dask.config.get("distributed.comm.offload")
+if isinstance(FRAME_OFFLOAD_THRESHOLD, str):
+    FRAME_OFFLOAD_THRESHOLD = parse_bytes(FRAME_OFFLOAD_THRESHOLD)
 
 
 async def to_frames(msg, serializers=None, on_error="message", context=None):
@@ -33,7 +37,7 @@ def _to_frames():
             logger.exception(e)
             raise
 
-    if sizeof(msg) > FRAME_OFFLOAD_THRESHOLD:
+    if FRAME_OFFLOAD_THRESHOLD and sizeof(msg) > FRAME_OFFLOAD_THRESHOLD:
         return await offload(_to_frames)
     else:
         return _to_frames()
@@ -59,7 +63,7 @@ def _from_frames():
             logger.error("truncated data stream (%d bytes): %s", size, datastr)
             raise
 
-    if deserialize and size > FRAME_OFFLOAD_THRESHOLD:
+    if deserialize and FRAME_OFFLOAD_THRESHOLD and size > FRAME_OFFLOAD_THRESHOLD:
         res = await offload(_from_frames)
     else:
         res = _from_frames()
diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index 59bf6f8dc1c..ae42162bb2f 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -73,6 +73,7 @@ distributed:
 
   comm:
     compression: auto
+    offload: 10MiB # Size after which we choose to offload serialization to another thread
     default-scheme: tcp
     socket-backlog: 2048
     recent-messages-log-length: 0  # number of messages to keep for debugging

From 71c998d23b879c6fd8543c3937847318a23c5447 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 26 Nov 2019 14:45:48 -0800
Subject: [PATCH 0571/1550] Remove memory use plot (#3269)

This was replaced by another more traditional bar chart
---
 distributed/dashboard/components/__init__.py  |  1 -
 distributed/dashboard/components/scheduler.py | 90 +------------------
 distributed/dashboard/components/shared.py    | 77 ----------------
 distributed/dashboard/scheduler.py            |  2 -
 .../dashboard/tests/test_components.py        |  3 +-
 .../dashboard/tests/test_scheduler_bokeh.py   | 14 ---
 distributed/diagnostics/progress_stream.py    | 46 ----------
 .../diagnostics/tests/test_progress_stream.py | 24 +----
 8 files changed, 3 insertions(+), 254 deletions(-)

diff --git a/distributed/dashboard/components/__init__.py b/distributed/dashboard/components/__init__.py
index 12f57b352b1..a66be2eced6 100644
--- a/distributed/dashboard/components/__init__.py
+++ b/distributed/dashboard/components/__init__.py
@@ -29,7 +29,6 @@
 import toolz
 
 from distributed.dashboard.utils import without_property_validation, BOKEH_VERSION
-from distributed.diagnostics.progress_stream import nbytes_bar
 from distributed import profile
 from distributed.utils import log_errors, parse_timedelta
 
diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index 5b01a99f7a6..4049860bed5 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -17,10 +17,7 @@
     TapTool,
     OpenURL,
     Range1d,
-    Plot,
-    Quad,
     value,
-    LinearAxis,
     NumeralTickFormatter,
     BoxZoomTool,
     AdaptiveTicker,
@@ -62,7 +59,7 @@
 )
 from distributed.metrics import time
 from distributed.utils import log_errors, format_time, parse_timedelta
-from distributed.diagnostics.progress_stream import color_of, progress_quads, nbytes_bar
+from distributed.diagnostics.progress_stream import color_of, progress_quads
 from distributed.diagnostics.progress import AllProgress
 from distributed.diagnostics.graph_layout import GraphLayout
 from distributed.diagnostics.task_stream import TaskStreamPlugin
@@ -1434,83 +1431,6 @@ def update(self):
             )
 
 
-class MemoryUse(DashboardComponent):
-    """ The memory usage across the cluster, grouped by task type """
-
-    def __init__(self, scheduler, **kwargs):
-        self.scheduler = scheduler
-        ps = [p for p in scheduler.plugins if isinstance(p, AllProgress)]
-        if ps:
-            self.plugin = ps[0]
-        else:
-            self.plugin = AllProgress(scheduler)
-
-        self.source = ColumnDataSource(
-            data=dict(
-                name=[],
-                left=[],
-                right=[],
-                center=[],
-                color=[],
-                percent=[],
-                MB=[],
-                text=[],
-            )
-        )
-
-        self.root = Plot(
-            id="bk-nbytes-plot",
-            x_range=DataRange1d(),
-            y_range=DataRange1d(),
-            toolbar_location=None,
-            outline_line_color=None,
-            **kwargs,
-        )
-
-        self.root.add_glyph(
-            self.source,
-            Quad(
-                top=1,
-                bottom=0,
-                left="left",
-                right="right",
-                fill_color="color",
-                fill_alpha=1,
-            ),
-        )
-
-        self.root.add_layout(LinearAxis(), "left")
-        self.root.add_layout(LinearAxis(), "below")
-
-        hover = HoverTool(
-            point_policy="follow_mouse",
-            tooltips="""
-                <div>
-                    <span style="font-size: 14px; font-weight: bold;">Name:</span>&nbsp;
-                    <span style="font-size: 10px; font-family: Monaco, monospace;">@name</span>
-                </div>
-                <div>
-                    <span style="font-size: 14px; font-weight: bold;">Percent:</span>&nbsp;
-                    <span style="font-size: 10px; font-family: Monaco, monospace;">@percent</span>
-                </div>
-                <div>
-                    <span style="font-size: 14px; font-weight: bold;">MB:</span>&nbsp;
-                    <span style="font-size: 10px; font-family: Monaco, monospace;">@MB</span>
-                </div>
-                """,
-        )
-        self.root.add_tools(hover)
-
-    @without_property_validation
-    def update(self):
-        with log_errors():
-            nb = nbytes_bar(self.plugin.nbytes)
-            update(self.source, nb)
-            self.root.title.text = "Memory Use: %0.2f MB" % (
-                sum(self.plugin.nbytes.values()) / 1e6
-            )
-
-
 class WorkerTable(DashboardComponent):
     """ Status of the current workers
 
@@ -1860,14 +1780,6 @@ def individual_nbytes_doc(scheduler, extra, doc):
     doc.theme = BOKEH_THEME
 
 
-def individual_memory_use_doc(scheduler, extra, doc):
-    memory_use = MemoryUse(scheduler, sizing_mode="stretch_both")
-    memory_use.update()
-    add_periodic_callback(doc, memory_use, 100)
-    doc.add_root(memory_use.root)
-    doc.theme = BOKEH_THEME
-
-
 def individual_cpu_doc(scheduler, extra, doc):
     current_load = CurrentLoad(scheduler, sizing_mode="stretch_both")
     current_load.update()
diff --git a/distributed/dashboard/components/shared.py b/distributed/dashboard/components/shared.py
index ec3a207fc8b..d7554e6bb30 100644
--- a/distributed/dashboard/components/shared.py
+++ b/distributed/dashboard/components/shared.py
@@ -4,12 +4,9 @@
 from bokeh.layouts import row, column
 from bokeh.models import (
     ColumnDataSource,
-    Plot,
     DataRange1d,
-    LinearAxis,
     HoverTool,
     Range1d,
-    Quad,
     Button,
     Select,
     NumeralTickFormatter,
@@ -26,7 +23,6 @@
     BOKEH_VERSION,
     update,
 )
-from distributed.diagnostics.progress_stream import nbytes_bar
 from distributed import profile
 from distributed.utils import log_errors, parse_timedelta
 from distributed.compatibility import WINDOWS
@@ -41,79 +37,6 @@
 profile_interval = parse_timedelta(profile_interval, default="ms")
 
 
-class MemoryUsage(DashboardComponent):
-    """ The memory usage across the cluster, grouped by task type """
-
-    def __init__(self, **kwargs):
-        self.source = ColumnDataSource(
-            data=dict(
-                name=[],
-                left=[],
-                right=[],
-                center=[],
-                color=[],
-                percent=[],
-                MB=[],
-                text=[],
-            )
-        )
-
-        self.root = Plot(
-            id="bk-nbytes-plot",
-            x_range=DataRange1d(),
-            y_range=DataRange1d(),
-            toolbar_location=None,
-            outline_line_color=None,
-            **kwargs
-        )
-
-        self.root.add_glyph(
-            self.source,
-            Quad(
-                top=1,
-                bottom=0,
-                left="left",
-                right="right",
-                fill_color="color",
-                fill_alpha=1,
-            ),
-        )
-
-        self.root.add_layout(LinearAxis(), "left")
-        self.root.add_layout(LinearAxis(), "below")
-
-        hover = HoverTool(
-            point_policy="follow_mouse",
-            tooltips="""
-                <div>
-                    <span style="font-size: 14px; font-weight: bold;">Name:</span>&nbsp;
-                    <span style="font-size: 10px; font-family: Monaco, monospace;">@name</span>
-                </div>
-                <div>
-                    <span style="font-size: 14px; font-weight: bold;">Percent:</span>&nbsp;
-                    <span style="font-size: 10px; font-family: Monaco, monospace;">@percent</span>
-                </div>
-                <div>
-                    <span style="font-size: 14px; font-weight: bold;">MB:</span>&nbsp;
-                    <span style="font-size: 10px; font-family: Monaco, monospace;">@MB</span>
-                </div>
-                """,
-        )
-        self.root.add_tools(hover)
-
-    @without_property_validation
-    def update(self, messages):
-        with log_errors():
-            msg = messages["progress"]
-            if not msg:
-                return
-            nb = nbytes_bar(msg["nbytes"])
-            update(self.source, nb)
-            self.root.title.text = "Memory Use: %0.2f MB" % (
-                sum(msg["nbytes"].values()) / 1e6
-            )
-
-
 class Processing(DashboardComponent):
     """ Processing and distribution per core
 
diff --git a/distributed/dashboard/scheduler.py b/distributed/dashboard/scheduler.py
index 67d08c50bd0..2c1ec38c4f3 100644
--- a/distributed/dashboard/scheduler.py
+++ b/distributed/dashboard/scheduler.py
@@ -30,7 +30,6 @@
     individual_profile_doc,
     individual_profile_server_doc,
     individual_nbytes_doc,
-    individual_memory_use_doc,
     individual_cpu_doc,
     individual_nprocessing_doc,
     individual_workers_doc,
@@ -403,7 +402,6 @@ def listen(self, *args, **kwargs):
     "/individual-profile": individual_profile_doc,
     "/individual-profile-server": individual_profile_server_doc,
     "/individual-nbytes": individual_nbytes_doc,
-    "/individual-memory-use": individual_memory_use_doc,
     "/individual-cpu": individual_cpu_doc,
     "/individual-nprocessing": individual_nprocessing_doc,
     "/individual-workers": individual_workers_doc,
diff --git a/distributed/dashboard/tests/test_components.py b/distributed/dashboard/tests/test_components.py
index 195c947bdec..3e6a696cc6b 100644
--- a/distributed/dashboard/tests/test_components.py
+++ b/distributed/dashboard/tests/test_components.py
@@ -7,14 +7,13 @@
 
 from distributed.utils_test import slowinc, gen_cluster
 from distributed.dashboard.components.shared import (
-    MemoryUsage,
     Processing,
     ProfilePlot,
     ProfileTimePlot,
 )
 
 
-@pytest.mark.parametrize("Component", [MemoryUsage, Processing])
+@pytest.mark.parametrize("Component", [Processing])
 def test_basic(Component):
     c = Component()
     assert isinstance(c.source, ColumnDataSource)
diff --git a/distributed/dashboard/tests/test_scheduler_bokeh.py b/distributed/dashboard/tests/test_scheduler_bokeh.py
index fe92c805efa..0f262ec5809 100644
--- a/distributed/dashboard/tests/test_scheduler_bokeh.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh.py
@@ -28,7 +28,6 @@
     Events,
     TaskStream,
     TaskProgress,
-    MemoryUse,
     CurrentLoad,
     ProcessingHistogram,
     NBytesHistogram,
@@ -248,19 +247,6 @@ def test_TaskProgress_empty(c, s, a, b):
     assert not any(len(v) for v in tp.source.data.values())
 
 
-@gen_cluster(client=True)
-def test_MemoryUse(c, s, a, b):
-    mu = MemoryUse(s)
-
-    futures = c.map(slowinc, range(10), delay=0.001)
-    yield wait(futures)
-
-    mu.update()
-    d = dict(mu.source.data)
-    assert all(len(L) == 1 for L in d.values())
-    assert d["name"] == ["slowinc"]
-
-
 @gen_cluster(client=True)
 def test_CurrentLoad(c, s, a, b):
     cl = CurrentLoad(s)
diff --git a/distributed/diagnostics/progress_stream.py b/distributed/diagnostics/progress_stream.py
index 038237b89e2..d127ecfeb7e 100644
--- a/distributed/diagnostics/progress_stream.py
+++ b/distributed/diagnostics/progress_stream.py
@@ -55,52 +55,6 @@ async def progress_stream(address, interval):
     return comm
 
 
-def nbytes_bar(nbytes):
-    """ Convert nbytes message into rectangle placements
-
-    >>> nbytes_bar({'inc': 1000, 'dec': 3000}) # doctest: +NORMALIZE_WHITESPACE
-    {'names': ['dec', 'inc'],
-     'left': [0, 0.75],
-     'center': [0.375, 0.875],
-     'right': [0.75, 1.0]}
-    """
-    total = sum(nbytes.values())
-    names = sorted(nbytes)
-
-    d = {
-        "name": [],
-        "text": [],
-        "left": [],
-        "right": [],
-        "center": [],
-        "color": [],
-        "percent": [],
-        "MB": [],
-    }
-
-    if not total:
-        return d
-
-    right = 0
-    for name in names:
-        left = right
-        right = nbytes[name] / total + left
-        center = (right + left) / 2
-        d["MB"].append(nbytes[name] / 1000000)
-        d["percent"].append(round(nbytes[name] / total * 100, 2))
-        d["left"].append(left)
-        d["right"].append(right)
-        d["center"].append(center)
-        d["color"].append(color_of(name))
-        d["name"].append(name)
-        if right - left > 0.1:
-            d["text"].append(name)
-        else:
-            d["text"].append("")
-
-    return d
-
-
 def progress_quads(msg, nrows=8, ncols=3):
     """
 
diff --git a/distributed/diagnostics/tests/test_progress_stream.py b/distributed/diagnostics/tests/test_progress_stream.py
index 56da9e974c1..77b3922a42e 100644
--- a/distributed/diagnostics/tests/test_progress_stream.py
+++ b/distributed/diagnostics/tests/test_progress_stream.py
@@ -4,11 +4,7 @@
 
 from dask import delayed
 from distributed.client import wait
-from distributed.diagnostics.progress_stream import (
-    progress_quads,
-    nbytes_bar,
-    progress_stream,
-)
+from distributed.diagnostics.progress_stream import progress_quads, progress_stream
 from distributed.utils_test import div, gen_cluster, inc
 
 
@@ -88,24 +84,6 @@ def test_progress_stream(c, s, a, b):
     yield comm.close()
 
 
-def test_nbytes_bar():
-    nbytes = {"inc": 1000, "dec": 3000}
-    expected = {
-        "name": ["dec", "inc"],
-        "left": [0, 0.75],
-        "center": [0.375, 0.875],
-        "right": [0.75, 1.0],
-        "percent": [75, 25],
-        "MB": [0.003, 0.001],
-        "text": ["dec", "inc"],
-    }
-
-    result = nbytes_bar(nbytes)
-    color = result.pop("color")
-    assert len(set(color)) == 2
-    assert result == expected
-
-
 def test_progress_quads_many_functions():
     funcnames = ["fn%d" % i for i in range(1000)]
     msg = {

From 0b68318112b13d70a9cdd741e5db00da2ec6a8f5 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 26 Nov 2019 14:46:07 -0800
Subject: [PATCH 0572/1550] Enable saving profile information from server
 threads (#3271)

Previously we could only save information from the worker threads
and had to rely ont the dashboard to effectively operate on the
administrative threads of the scheduler and workers.

Now we can do both.
---
 distributed/client.py            | 16 ++++++++++++++++
 distributed/scheduler.py         | 11 ++++++++++-
 distributed/tests/test_client.py | 16 ++++++++++++++++
 distributed/worker.py            |  7 +++++--
 4 files changed, 47 insertions(+), 3 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 4f315e89b6d..3b2eb6e2863 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -3259,6 +3259,8 @@ def profile(
         merge_workers=True,
         plot=False,
         filename=None,
+        server=False,
+        scheduler=False,
     ):
         """ Collect statistical profiling information about recent work
 
@@ -3271,6 +3273,14 @@ def profile(
         stop: time
         workers: list
             List of workers to restrict profile information
+        server : bool
+            If true, return the profile of the worker's administrative thread
+            rather than the worker threads.
+            This is useful when profiling Dask itself, rather than user code.
+        scheduler: bool
+            If true, return the profile information from the scheduler's
+            administrative thread rather than the workers.
+            This is useful when profiling Dask's scheduling itself.
         plot: boolean or string
             Whether or not to return a plot object
         filename: str
@@ -3293,6 +3303,8 @@ def profile(
             stop=stop,
             plot=plot,
             filename=filename,
+            server=server,
+            scheduler=scheduler,
         )
 
     async def _profile(
@@ -3304,6 +3316,8 @@ async def _profile(
         merge_workers=True,
         plot=False,
         filename=None,
+        server=False,
+        scheduler=False,
     ):
         if isinstance(workers, (str, Number)):
             workers = [workers]
@@ -3314,6 +3328,8 @@ async def _profile(
             merge_workers=merge_workers,
             start=start,
             stop=stop,
+            server=server,
+            scheduler=scheduler,
         )
 
         if filename:
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 26b4cf1c67e..26b6bffa970 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -4623,6 +4623,8 @@ async def get_profile(
         self,
         comm=None,
         workers=None,
+        scheduler=False,
+        server=False,
         merge_workers=True,
         start=None,
         stop=None,
@@ -4632,8 +4634,15 @@ async def get_profile(
             workers = self.workers
         else:
             workers = set(self.workers) & set(workers)
+
+        if scheduler:
+            return profile.get_profile(self.io_loop.profile, start=start, stop=stop)
+
         results = await asyncio.gather(
-            *(self.rpc(w).profile(start=start, stop=stop, key=key) for w in workers)
+            *(
+                self.rpc(w).profile(start=start, stop=stop, key=key, server=server)
+                for w in workers
+            )
         )
 
         if merge_workers:
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 73906517045..e97cb023a48 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -5687,5 +5687,21 @@ async def test_futures_of_sorted(c, s, a, b):
         assert str(k) in str(f)
 
 
+@gen_cluster(client=True, worker_kwargs={"profile_cycle_interval": "10ms"})
+async def test_profile_server(c, s, a, b):
+    x = c.map(slowinc, range(10), delay=0.01, workers=a.address)
+    await wait(x)
+
+    await asyncio.gather(
+        c.run(slowinc, 1, delay=0.5), c.run_on_scheduler(slowdec, 1, delay=0.5)
+    )
+
+    p = await c.profile(server=True)  #  All worker servers
+    assert "slowinc" in str(p)
+
+    p = await c.profile(scheduler=True)  #  Scheduler
+    assert "slowdec" in str(p)
+
+
 if sys.version_info >= (3, 5):
     from distributed.tests.py3_test_client import *  # noqa F401
diff --git a/distributed/worker.py b/distributed/worker.py
index fb46dc2391c..8509f317a61 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -2655,12 +2655,15 @@ def trigger_profile(self):
         if self.digests is not None:
             self.digests["profile-duration"].add(stop - start)
 
-    def get_profile(self, comm=None, start=None, stop=None, key=None):
+    def get_profile(self, comm=None, start=None, stop=None, key=None, server=False):
         now = time() + self.scheduler_delay
-        if key is None:
+        if server:
+            history = self.io_loop.profile
+        elif key is None:
             history = self.profile_history
         else:
             history = [(t, d[key]) for t, d in self.profile_keys_history if key in d]
+
         if start is None:
             istart = 0
         else:

From 9a937535379fe786f08ce936962310307d5d99ae Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 27 Nov 2019 09:27:57 -0800
Subject: [PATCH 0573/1550] Make profile coroutines consistent between
 Scheduler and Worker (#3277)

Previously scheduler get_profile(_metadata) methods were asynchronous
while worker methods were synchronous.  This caused consistency issues
with dashboard plots.  Now we make the worker methods asynchronous
(needlessly) for consistency.
---
 distributed/dashboard/components/shared.py |  4 ++--
 distributed/tests/test_worker.py           | 10 +++++-----
 distributed/worker.py                      |  6 ++++--
 3 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/distributed/dashboard/components/shared.py b/distributed/dashboard/components/shared.py
index d7554e6bb30..611d281dd5e 100644
--- a/distributed/dashboard/components/shared.py
+++ b/distributed/dashboard/components/shared.py
@@ -321,11 +321,11 @@ def update(self, state, metadata=None):
     def trigger_update(self, update_metadata=True):
         async def cb():
             with log_errors():
-                prof = self.server.get_profile(
+                prof = await self.server.get_profile(
                     key=self.key, start=self.start, stop=self.stop
                 )
                 if update_metadata:
-                    metadata = self.server.get_profile_metadata()
+                    metadata = await self.server.get_profile_metadata()
                 else:
                     metadata = None
                 if isinstance(prof, gen.Future):
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index ff523342243..f67701f671a 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -1079,7 +1079,7 @@ def test_statistical_profiling_2(c, s, a, b):
         y = (x + x * 2) - x.sum().persist()
         yield wait(y)
 
-        profile = a.get_profile()
+        profile = yield a.get_profile()
         text = str(profile)
         if profile["count"] and "sum" in text and "random" in text:
             break
@@ -1165,15 +1165,15 @@ def test_statistical_profiling_cycle(c, s, a, b):
     end = time()
     assert len(a.profile_history) > 3
 
-    x = a.get_profile(start=time() + 10, stop=time() + 20)
+    x = yield a.get_profile(start=time() + 10, stop=time() + 20)
     assert not x["count"]
 
-    x = a.get_profile(start=0, stop=time())
+    x = yield a.get_profile(start=0, stop=time())
     actual = sum(p["count"] for _, p in a.profile_history) + a.profile_recent["count"]
-    x2 = a.get_profile(start=0, stop=time())
+    x2 = yield a.get_profile(start=0, stop=time())
     assert x["count"] <= actual <= x2["count"]
 
-    y = a.get_profile(start=end - 0.300, stop=time())
+    y = yield a.get_profile(start=end - 0.300, stop=time())
     assert 0 < y["count"] <= x["count"]
 
 
diff --git a/distributed/worker.py b/distributed/worker.py
index 8509f317a61..1eb8252534d 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -2655,7 +2655,9 @@ def trigger_profile(self):
         if self.digests is not None:
             self.digests["profile-duration"].add(stop - start)
 
-    def get_profile(self, comm=None, start=None, stop=None, key=None, server=False):
+    async def get_profile(
+        self, comm=None, start=None, stop=None, key=None, server=False
+    ):
         now = time() + self.scheduler_delay
         if server:
             history = self.io_loop.profile
@@ -2696,7 +2698,7 @@ def get_profile(self, comm=None, start=None, stop=None, key=None, server=False):
 
         return prof
 
-    def get_profile_metadata(self, comm=None, start=0, stop=None):
+    async def get_profile_metadata(self, comm=None, start=0, stop=None):
         if stop is None:
             add_recent = True
         now = time() + self.scheduler_delay

From 664d1339cbe159ed06065c211da1ca5f5bd8675f Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 27 Nov 2019 11:10:53 -0800
Subject: [PATCH 0574/1550] Improve bandwidth workers plot (#3273)

* Use new maximum for bandwidth workers at every step

* Use base 2 values for bandwidth workers plot

* Use worker names in bandwidth plot

* add an additional decimal place to workerbandwidth colorbar

We often have values like 1GB and 2GB, which are too coarse, and so can
be repeated

* switch around order of workers on y-axis

This matches intuition from matrices where the diagonal goes from upper
left to lower right
---
 distributed/dashboard/components/scheduler.py | 20 ++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index 4049860bed5..2b27c708111 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -383,7 +383,10 @@ def __init__(self, scheduler, **kwargs):
                 border_line_color=None,
                 location=(0, 0),
             )
-            color_bar.formatter = NumeralTickFormatter(format="0 b")
+            color_bar.formatter = NumeralTickFormatter(format="0.0 b")
+            color_bar.ticker = AdaptiveTicker(
+                mantissas=[1, 64, 128, 256, 512], base=1024
+            )
             fig.add_layout(color_bar, "right")
 
             fig.toolbar.logo = None
@@ -408,14 +411,21 @@ def update(self):
             bw = self.scheduler.bandwidth_workers
             if not bw:
                 return
-            x, y, value = zip(*[(a, b, c) for (a, b), c in bw.items()])
 
-            if self.color_map.high < max(value):
-                self.color_map.high = max(value)
+            def name(address):
+                ws = self.scheduler.workers[address]
+                if ws.name is not None:
+                    return str(ws.name)
+                else:
+                    return address
+
+            x, y, value = zip(*[(name(a), name(b), c) for (a, b), c in bw.items()])
+
+            self.color_map.high = max(value)
 
             factors = list(sorted(set(x + y)))
             self.fig.x_range.factors = factors
-            self.fig.y_range.factors = factors
+            self.fig.y_range.factors = factors[::-1]
 
             result = {
                 "source": x,

From 447b2c2ac2458090a0b8baf2f11774b89b6d78f9 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 27 Nov 2019 11:11:04 -0800
Subject: [PATCH 0575/1550] Worker profile server (#3274)

* Add profile-server button to Worker dashboard navbar

* Set worker's profile server plots to stretch_both layout
---
 distributed/dashboard/components/worker.py | 4 ++--
 distributed/dashboard/worker.py            | 4 +++-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/distributed/dashboard/components/worker.py b/distributed/dashboard/components/worker.py
index 9dc2b2ec82f..440e7279e3b 100644
--- a/distributed/dashboard/components/worker.py
+++ b/distributed/dashboard/components/worker.py
@@ -638,7 +638,7 @@ def counters_doc(server, extra, doc):
 def profile_doc(server, extra, doc):
     with log_errors():
         doc.title = "Dask Worker Profile"
-        profile = ProfileTimePlot(server, sizing_mode="scale_width", doc=doc)
+        profile = ProfileTimePlot(server, sizing_mode="stretch_both", doc=doc)
         profile.trigger_update()
 
         doc.add_root(profile.root)
@@ -651,7 +651,7 @@ def profile_doc(server, extra, doc):
 def profile_server_doc(server, extra, doc):
     with log_errors():
         doc.title = "Dask: Profile of Event Loop"
-        prof = ProfileServer(server, sizing_mode="scale_width", doc=doc)
+        prof = ProfileServer(server, sizing_mode="stretch_both", doc=doc)
         doc.add_root(prof.root)
         doc.template = env.get_template("simple.html")
         # doc.template_variables['active_page'] = ''
diff --git a/distributed/dashboard/worker.py b/distributed/dashboard/worker.py
index 99b27557694..5a34a261bf1 100644
--- a/distributed/dashboard/worker.py
+++ b/distributed/dashboard/worker.py
@@ -30,7 +30,9 @@
 
 BOKEH_THEME = Theme(os.path.join(os.path.dirname(__file__), "theme.yaml"))
 
-template_variables = {"pages": ["status", "system", "profile", "crossfilter"]}
+template_variables = {
+    "pages": ["status", "system", "profile", "crossfilter", "profile-server"]
+}
 
 
 class _PrometheusCollector(object):

From f7976d9397ab7ed4ce1224779914b18dc7e0f2a9 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 27 Nov 2019 13:51:37 -0800
Subject: [PATCH 0576/1550] Remove dask-submit and dask-remote (#3280)

These were seemingly unused for years, and have gotten far out of date.
---
 distributed/cli/dask_remote.py                | 23 -----
 distributed/cli/dask_submit.py                | 32 -------
 distributed/cli/tests/test_dask_remote.py     | 15 ---
 distributed/cli/tests/test_dask_submit.py     | 15 ---
 distributed/submit.py                         | 94 -------------------
 distributed/tests/test_submit_cli.py          | 56 -----------
 .../tests/test_submit_remote_client.py        | 63 -------------
 docs/source/index.rst                         |  1 -
 docs/source/submitting-applications.rst       | 71 --------------
 setup.py                                      |  2 -
 10 files changed, 372 deletions(-)
 delete mode 100644 distributed/cli/dask_remote.py
 delete mode 100644 distributed/cli/dask_submit.py
 delete mode 100644 distributed/cli/tests/test_dask_remote.py
 delete mode 100644 distributed/cli/tests/test_dask_submit.py
 delete mode 100644 distributed/submit.py
 delete mode 100644 distributed/tests/test_submit_cli.py
 delete mode 100644 distributed/tests/test_submit_remote_client.py
 delete mode 100644 docs/source/submitting-applications.rst

diff --git a/distributed/cli/dask_remote.py b/distributed/cli/dask_remote.py
deleted file mode 100644
index 9fcfe7f3763..00000000000
--- a/distributed/cli/dask_remote.py
+++ /dev/null
@@ -1,23 +0,0 @@
-import click
-from distributed.cli.utils import check_python_3, install_signal_handlers
-from distributed.submit import _remote
-
-
-@click.command()
-@click.option("--host", type=str, default=None, help="IP or hostname of this server")
-@click.option(
-    "--port", type=int, default=8788, show_default=True, help="Remote Client Port"
-)
-@click.version_option()
-def main(host, port):
-    _remote(host, port)
-
-
-def go():
-    install_signal_handlers()
-    check_python_3()
-    main()
-
-
-if __name__ == "__main__":
-    go()
diff --git a/distributed/cli/dask_submit.py b/distributed/cli/dask_submit.py
deleted file mode 100644
index 071dd5bbe32..00000000000
--- a/distributed/cli/dask_submit.py
+++ /dev/null
@@ -1,32 +0,0 @@
-import sys
-import click
-from tornado import gen
-from tornado.ioloop import IOLoop
-from distributed.cli.utils import check_python_3, install_signal_handlers
-from distributed.submit import _submit
-
-
-@click.command()
-@click.argument("remote_client_address", type=str, required=True)
-@click.argument("filepath", type=str, required=True)
-@click.version_option()
-def main(remote_client_address, filepath):
-    @gen.coroutine
-    def f():
-        stdout, stderr = yield _submit(remote_client_address, filepath)
-        if stdout:
-            sys.stdout.write(str(stdout))
-        if stderr:
-            sys.stderr.write(str(stderr))
-
-    IOLoop.instance().run_sync(f)
-
-
-def go():
-    install_signal_handlers()
-    check_python_3()
-    main()
-
-
-if __name__ == "__main__":
-    go()
diff --git a/distributed/cli/tests/test_dask_remote.py b/distributed/cli/tests/test_dask_remote.py
deleted file mode 100644
index 14da80f949c..00000000000
--- a/distributed/cli/tests/test_dask_remote.py
+++ /dev/null
@@ -1,15 +0,0 @@
-from click.testing import CliRunner
-from distributed.cli.dask_remote import main
-
-
-def test_dask_remote():
-    runner = CliRunner()
-    result = runner.invoke(main, ["--help"])
-    assert "--host TEXT     IP or hostname of this server" in result.output
-    assert result.exit_code == 0
-
-
-def test_version_option():
-    runner = CliRunner()
-    result = runner.invoke(main, ["--version"])
-    assert result.exit_code == 0
diff --git a/distributed/cli/tests/test_dask_submit.py b/distributed/cli/tests/test_dask_submit.py
deleted file mode 100644
index 8f5f961ea96..00000000000
--- a/distributed/cli/tests/test_dask_submit.py
+++ /dev/null
@@ -1,15 +0,0 @@
-from click.testing import CliRunner
-from distributed.cli.dask_submit import main
-
-
-def test_submit_runs_as_a_cli():
-    runner = CliRunner()
-    result = runner.invoke(main, ["--help"])
-    assert result.exit_code == 0
-    assert "Usage: main [OPTIONS] REMOTE_CLIENT_ADDRESS FILEPATH" in result.output
-
-
-def test_version_option():
-    runner = CliRunner()
-    result = runner.invoke(main, ["--version"])
-    assert result.exit_code == 0
diff --git a/distributed/submit.py b/distributed/submit.py
deleted file mode 100644
index 4cd7fb197a9..00000000000
--- a/distributed/submit.py
+++ /dev/null
@@ -1,94 +0,0 @@
-import logging
-import os
-import socket
-import subprocess
-import tempfile
-import sys
-
-from tornado import gen
-
-from tornado.ioloop import IOLoop
-
-from .core import rpc, Server
-from .security import Security
-from .utils import get_ip
-
-
-logger = logging.getLogger("distributed.remote")
-
-
-class RemoteClient(Server):
-    def __init__(
-        self,
-        ip=None,
-        local_dir=tempfile.mkdtemp(prefix="client-"),
-        loop=None,
-        security=None,
-        **kwargs
-    ):
-        self.ip = ip or get_ip()
-        self.loop = loop or IOLoop.current()
-        self.local_dir = local_dir
-        handlers = {"upload_file": self.upload_file, "execute": self.execute}
-
-        self.security = security or Security()
-        assert isinstance(self.security, Security)
-        self.listen_args = self.security.get_listen_args("scheduler")
-
-        super(RemoteClient, self).__init__(handlers, io_loop=self.loop, **kwargs)
-
-    @gen.coroutine
-    def _start(self, port=0):
-        self.listen(port, listen_args=self.listen_args)
-
-    def start(self, port=0):
-        self.loop.add_callback(self._start, port)
-        logger.info("Remote Client is running at {0}:{1}".format(self.ip, port))
-
-    @gen.coroutine
-    def execute(self, stream=None, filename=None):
-        script_path = os.path.join(self.local_dir, filename)
-        cmd = "{0} {1}".format(sys.executable, script_path)
-        process = subprocess.Popen(
-            cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE
-        )
-        out, err = process.communicate()
-        return_code = process.returncode
-        raise gen.Return({"stdout": out, "stderr": err, "returncode": return_code})
-
-    def upload_file(self, stream, filename=None, file_payload=None):
-        out_filename = os.path.join(self.local_dir, filename)
-        if isinstance(file_payload, str):
-            file_payload = file_payload.encode()
-        with open(out_filename, "wb") as f:
-            f.write(file_payload)
-        return {"status": "OK", "nbytes": len(file_payload)}
-
-    @gen.coroutine
-    def _close(self):
-        self.stop()
-
-
-def _remote(host, port, loop=IOLoop.current(), client=RemoteClient):
-    host = host or get_ip()
-    if ":" in host and port == 8788:
-        host, port = host.rsplit(":", 1)
-        port = int(port)
-    ip = socket.gethostbyname(host)
-    remote_client = client(ip=ip, loop=loop)
-    remote_client.start(port=port)
-    loop.start()
-    loop.close()
-    remote_client.stop()
-    logger.info("End remote client at %s:%d", host, port)
-
-
-@gen.coroutine
-def _submit(remote_client_address, filepath, connection_args=None):
-    rc = rpc(remote_client_address, connection_args=connection_args)
-    remote_file = os.path.basename(filepath)
-    with open(filepath, "rb") as f:
-        bytes_read = f.read()
-    yield rc.upload_file(filename=remote_file, file_payload=bytes_read)
-    result = yield rc.execute(filename=remote_file)
-    raise gen.Return((result["stdout"], result["stderr"]))
diff --git a/distributed/tests/test_submit_cli.py b/distributed/tests/test_submit_cli.py
deleted file mode 100644
index edc16e0a61e..00000000000
--- a/distributed/tests/test_submit_cli.py
+++ /dev/null
@@ -1,56 +0,0 @@
-from unittest.mock import Mock
-
-from tornado import gen
-from tornado.ioloop import IOLoop
-from distributed.submit import RemoteClient, _submit, _remote
-from distributed.utils_test import (  # noqa: F401
-    valid_python_script,
-    invalid_python_script,
-    loop,
-)
-
-
-def test_dask_submit_cli_writes_result_to_stdout(loop, tmpdir, valid_python_script):
-    @gen.coroutine
-    def test():
-        remote_client = RemoteClient(ip="127.0.0.1", local_dir=str(tmpdir))
-        yield remote_client._start()
-
-        out, err = yield _submit(
-            "127.0.0.1:{0}".format(remote_client.port), str(valid_python_script)
-        )
-        assert b"hello world!" in out
-        yield remote_client._close()
-
-    loop.run_sync(test, timeout=5)
-
-
-def test_dask_submit_cli_writes_traceback_to_stdout(
-    loop, tmpdir, invalid_python_script
-):
-    @gen.coroutine
-    def test():
-        remote_client = RemoteClient(ip="127.0.0.1", local_dir=str(tmpdir))
-        yield remote_client._start()
-
-        out, err = yield _submit(
-            "127.0.0.1:{0}".format(remote_client.port), str(invalid_python_script)
-        )
-        assert b"Traceback" in err
-        yield remote_client._close()
-
-    loop.run_sync(test, timeout=5)
-
-
-def test_cli_runs_remote_client():
-    mock_remote_client = Mock(spec=RemoteClient)
-    mock_ioloop = Mock(spec=IOLoop.current())
-
-    _remote("127.0.0.1:8799", 8788, loop=mock_ioloop, client=mock_remote_client)
-
-    mock_remote_client.assert_called_once_with(ip="127.0.0.1", loop=mock_ioloop)
-    mock_remote_client().start.assert_called_once_with(port=8799)
-
-    assert mock_ioloop.start.called
-    assert mock_ioloop.close.called
-    assert mock_remote_client().stop.called
diff --git a/distributed/tests/test_submit_remote_client.py b/distributed/tests/test_submit_remote_client.py
deleted file mode 100644
index e6527d8319b..00000000000
--- a/distributed/tests/test_submit_remote_client.py
+++ /dev/null
@@ -1,63 +0,0 @@
-import os
-
-from tornado import gen
-
-from distributed import rpc
-from distributed.submit import RemoteClient
-from distributed.utils_test import (  # noqa: F401
-    loop,
-    valid_python_script,
-    invalid_python_script,
-)
-
-
-def test_remote_client_uploads_a_file(loop, tmpdir):
-    @gen.coroutine
-    def test():
-        remote_client = RemoteClient(ip="127.0.0.1", local_dir=str(tmpdir))
-        yield remote_client._start(0)
-        remote_process = rpc(remote_client.address)
-        upload = yield remote_process.upload_file(
-            filename="script.py", file_payload="x=1"
-        )
-
-        assert upload == {"status": "OK", "nbytes": 3}
-        assert tmpdir.join("script.py").read() == "x=1"
-
-        yield remote_client._close()
-
-    loop.run_sync(test, timeout=5)
-
-
-def test_remote_client_execution_outputs_to_stdout(loop, tmpdir):
-    @gen.coroutine
-    def test():
-        remote_client = RemoteClient(ip="127.0.0.1", local_dir=str(tmpdir))
-        yield remote_client._start(0)
-        rr = rpc(remote_client.address)
-        yield rr.upload_file(filename="script.py", file_payload='print("hello world!")')
-
-        message = yield rr.execute(filename="script.py")
-        assert message["stdout"] == b"hello world!" + os.linesep.encode()
-        assert message["returncode"] == 0
-
-        yield remote_client._close()
-
-    loop.run_sync(test, timeout=5)
-
-
-def test_remote_client_execution_outputs_stderr(loop, tmpdir, invalid_python_script):
-    @gen.coroutine
-    def test():
-        remote_client = RemoteClient(ip="127.0.0.1", local_dir=str(tmpdir))
-        yield remote_client._start(0)
-        rr = rpc(remote_client.address)
-        yield rr.upload_file(filename="script.py", file_payload="a+1")
-
-        message = yield rr.execute(filename="script.py")
-        assert b"'a' is not defined" in message["stderr"]
-        assert message["returncode"] == 1
-
-        yield remote_client._close()
-
-    loop.run_sync(test, timeout=5)
diff --git a/docs/source/index.rst b/docs/source/index.rst
index ee32738f826..47419e014ec 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -116,7 +116,6 @@ Contents
    publish
    queues
    resources
-   submitting-applications
    task-launch
    tls
    web
diff --git a/docs/source/submitting-applications.rst b/docs/source/submitting-applications.rst
deleted file mode 100644
index 8b5ab1d61c8..00000000000
--- a/docs/source/submitting-applications.rst
+++ /dev/null
@@ -1,71 +0,0 @@
-Submitting Applications
-=======================
-
-The ``dask-submit`` cli can be used to submit an application to the dask cluster
-running remotely. If your code depends on resources that can only be access
-from cluster running dask, ``dask-submit`` provides a mechanism to send the script
-to the cluster for execution from a different machine.
-
-For example, S3 buckets could not be visible from your local machine and hence any
-attempt to create a dask graph from local machine may not work.
-
-
-Submitting dask Applications with ``dask-submit``
--------------------------------------------------
-
-In order to remotely submit scripts to the cluster from a local machine or a CI/CD
-environment, we need to run a remote client on the same machine as the scheduler::
-
-   #scheduler machine
-   dask-remote --port 8788
-
-
-After making sure the ``dask-remote`` is running, you can submit a script by::
-
-   #local machine
-   dask-submit <dask-remote-address>:<port> <script.py>
-
-
-Some of the commonly used arguments are:
-
--  ``REMOTE_CLIENT_ADDRESS``: host name where ``dask-remote`` client is running
--  ``FILEPATH``: Local path to file containing dask application
-
-For example, given the following dask application saved in a file called
-``script.py``:
-
-.. code-block:: python
-
-   # script.py
-   from distributed import Client
-
-   def inc(x):
-        return x + 1
-
-   if __name__=='__main__':
-        client = Client('127.0.0.1:8786')
-        x = client.submit(inc, 10)
-        print(x.result())
-
-
-We can submit this application from a local machine by running::
-
-   dask-submit <remote-client-address>:<port> script.py
-
-
-CLI Options
------------
-
-.. note::
-
-   The command line documentation here may differ depending on your installed
-   version. We recommend referring to the output of ``dask-remote --help``
-   and ``dask-submit --help``.
-
-.. click:: distributed.cli.dask_remote:main
-   :prog: dask-remote
-   :show-nested:
-
-.. click:: distributed.cli.dask_submit:main
-   :prog: dask-submit
-   :show-nested:
\ No newline at end of file
diff --git a/setup.py b/setup.py
index 310d5322e98..e8c419cb147 100755
--- a/setup.py
+++ b/setup.py
@@ -54,8 +54,6 @@
     entry_points="""
         [console_scripts]
         dask-ssh=distributed.cli.dask_ssh:go
-        dask-submit=distributed.cli.dask_submit:go
-        dask-remote=distributed.cli.dask_remote:go
         dask-scheduler=distributed.cli.dask_scheduler:go
         dask-worker=distributed.cli.dask_worker:go
       """,

From 1eb3a9dbdaae96aebb33d9be843906c684ca52f4 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 27 Nov 2019 17:54:10 -0800
Subject: [PATCH 0577/1550] Make Listener.start asynchronous (#3278)

Previously the Listener.start method was synchronous,
expecting comms to be able to start themselves and get an address
immediately.  This was a challenge when we tried adding asyncio comms,
which needed to await the server creation.

Now we make Listener.start asynchronous, and make sure that we await all
calls to it within the codebase.
---
 distributed/comm/core.py             |   8 +-
 distributed/comm/inproc.py           |   5 +-
 distributed/comm/tcp.py              |   2 +-
 distributed/comm/tests/test_comms.py | 627 +++++++++++++--------------
 distributed/comm/ucx.py              |   2 +-
 distributed/core.py                  |   4 +-
 distributed/nanny.py                 |   2 +-
 distributed/scheduler.py             |   2 +-
 distributed/tests/test_batched.py    | 174 ++++----
 distributed/tests/test_core.py       | 330 +++++++-------
 distributed/tests/test_security.py   |  62 ++-
 distributed/worker.py                |   2 +-
 12 files changed, 598 insertions(+), 622 deletions(-)

diff --git a/distributed/comm/core.py b/distributed/comm/core.py
index 0befb36d712..11f74a1aba8 100644
--- a/distributed/comm/core.py
+++ b/distributed/comm/core.py
@@ -131,7 +131,7 @@ def __repr__(self):
 
 class Listener(ABC):
     @abstractmethod
-    def start(self):
+    async def start(self):
         """
         Start listening for incoming connections.
         """
@@ -157,11 +157,11 @@ def contact_address(self):
         address such as 'tcp://0.0.0.0:123'.
         """
 
-    def __enter__(self):
-        self.start()
+    async def __aenter__(self):
+        await self.start()
         return self
 
-    def __exit__(self, *exc):
+    async def __aexit__(self, *exc):
         self.stop()
 
 
diff --git a/distributed/comm/inproc.py b/distributed/comm/inproc.py
index e9bed986ea0..e46c2804ed1 100644
--- a/distributed/comm/inproc.py
+++ b/distributed/comm/inproc.py
@@ -1,3 +1,4 @@
+import asyncio
 from collections import deque, namedtuple
 import itertools
 import logging
@@ -265,9 +266,9 @@ async def _listen(self):
     def connect_threadsafe(self, conn_req):
         self.loop.add_callback(self.listen_q.put_nowait, conn_req)
 
-    def start(self):
+    async def start(self):
         self.loop = IOLoop.current()
-        self.loop.add_callback(self._listen)
+        self._listen_future = asyncio.ensure_future(self._listen())
         self.manager.add_listener(self.address, self)
 
     def stop(self):
diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py
index f0a24fe4fb7..40b6e8104b3 100644
--- a/distributed/comm/tcp.py
+++ b/distributed/comm/tcp.py
@@ -395,7 +395,7 @@ def __init__(
         self.tcp_server = None
         self.bound_address = None
 
-    def start(self):
+    async def start(self):
         self.tcp_server = TCPServer(max_buffer_size=MAX_BUFFER_SIZE, **self.server_args)
         self.tcp_server.handle_stream = self._handle_stream
         backlog = int(dask.config.get("distributed.comm.socket-backlog"))
diff --git a/distributed/comm/tests/test_comms.py b/distributed/comm/tests/test_comms.py
index 5839c3e8871..465be11c7a4 100644
--- a/distributed/comm/tests/test_comms.py
+++ b/distributed/comm/tests/test_comms.py
@@ -1,3 +1,4 @@
+import asyncio
 from functools import partial
 import os
 import sys
@@ -6,13 +7,12 @@
 
 import pytest
 
-from tornado import gen, ioloop, locks, queues
+from tornado import ioloop, locks, queues
 from tornado.concurrent import Future
 
 from distributed.metrics import time
 from distributed.utils import get_ip, get_ipv6
 from distributed.utils_test import (
-    gen_test,
     requires_ipv6,
     has_ipv6,
     get_cert,
@@ -73,21 +73,21 @@ def check_tls_extra(info):
 )
 
 
-@gen.coroutine
-def get_comm_pair(listen_addr, listen_args=None, connect_args=None, **kwargs):
+@pytest.mark.asyncio
+async def get_comm_pair(listen_addr, listen_args=None, connect_args=None, **kwargs):
     q = queues.Queue()
 
     def handle_comm(comm):
         q.put(comm)
 
     listener = listen(listen_addr, handle_comm, connection_args=listen_args, **kwargs)
-    listener.start()
+    await listener.start()
 
-    comm = yield connect(
+    comm = await connect(
         listener.contact_address, connection_args=connect_args, **kwargs
     )
-    serv_comm = yield q.get()
-    raise gen.Return((comm, serv_comm))
+    serv_comm = await q.get()
+    return (comm, serv_comm)
 
 
 def get_tcp_comm_pair(**kwargs):
@@ -103,15 +103,14 @@ def get_inproc_comm_pair(**kwargs):
     return get_comm_pair("inproc://", **kwargs)
 
 
-@gen.coroutine
-def debug_loop():
+async def debug_loop():
     """
     Debug helper
     """
     while True:
         loop = ioloop.IOLoop.current()
         print(".", loop, loop._handlers)
-        yield gen.sleep(0.50)
+        await asyncio.sleep(0.50)
 
 
 #
@@ -205,23 +204,22 @@ def test_get_local_address_for():
 #
 
 
-@gen_test()
-def test_tcp_specific():
+@pytest.mark.asyncio
+async def test_tcp_specific():
     """
     Test concrete TCP API.
     """
 
-    @gen.coroutine
-    def handle_comm(comm):
+    async def handle_comm(comm):
         assert comm.peer_address.startswith("tcp://" + host)
         assert comm.extra_info == {}
-        msg = yield comm.read()
+        msg = await comm.read()
         msg["op"] = "pong"
-        yield comm.write(msg)
-        yield comm.close()
+        await comm.write(msg)
+        await comm.close()
 
     listener = tcp.TCPListener("localhost", handle_comm)
-    listener.start()
+    await listener.start()
     host, port = listener.get_host_port()
     assert host in ("localhost", "127.0.0.1", "::1")
     assert port > 0
@@ -229,49 +227,47 @@ def handle_comm(comm):
     connector = tcp.TCPConnector()
     l = []
 
-    @gen.coroutine
-    def client_communicate(key, delay=0):
+    async def client_communicate(key, delay=0):
         addr = "%s:%d" % (host, port)
-        comm = yield connector.connect(addr)
+        comm = await connector.connect(addr)
         assert comm.peer_address == "tcp://" + addr
         assert comm.extra_info == {}
-        yield comm.write({"op": "ping", "data": key})
+        await comm.write({"op": "ping", "data": key})
         if delay:
-            yield gen.sleep(delay)
-        msg = yield comm.read()
+            await asyncio.sleep(delay)
+        msg = await comm.read()
         assert msg == {"op": "pong", "data": key}
         l.append(key)
-        yield comm.close()
+        await comm.close()
 
-    yield client_communicate(key=1234)
+    await client_communicate(key=1234)
 
     # Many clients at once
     N = 100
     futures = [client_communicate(key=i, delay=0.05) for i in range(N)]
-    yield futures
+    await asyncio.gather(*futures)
     assert set(l) == {1234} | set(range(N))
 
 
-@gen_test()
-def test_tls_specific():
+@pytest.mark.asyncio
+async def test_tls_specific():
     """
     Test concrete TLS API.
     """
 
-    @gen.coroutine
-    def handle_comm(comm):
+    async def handle_comm(comm):
         assert comm.peer_address.startswith("tls://" + host)
         check_tls_extra(comm.extra_info)
-        msg = yield comm.read()
+        msg = await comm.read()
         msg["op"] = "pong"
-        yield comm.write(msg)
-        yield comm.close()
+        await comm.write(msg)
+        await comm.close()
 
     server_ctx = get_server_ssl_context()
     client_ctx = get_client_ssl_context()
 
     listener = tcp.TLSListener("localhost", handle_comm, ssl_context=server_ctx)
-    listener.start()
+    await listener.start()
     host, port = listener.get_host_port()
     assert host in ("localhost", "127.0.0.1", "::1")
     assert port > 0
@@ -279,31 +275,30 @@ def handle_comm(comm):
     connector = tcp.TLSConnector()
     l = []
 
-    @gen.coroutine
-    def client_communicate(key, delay=0):
+    async def client_communicate(key, delay=0):
         addr = "%s:%d" % (host, port)
-        comm = yield connector.connect(addr, ssl_context=client_ctx)
+        comm = await connector.connect(addr, ssl_context=client_ctx)
         assert comm.peer_address == "tls://" + addr
         check_tls_extra(comm.extra_info)
-        yield comm.write({"op": "ping", "data": key})
+        await comm.write({"op": "ping", "data": key})
         if delay:
-            yield gen.sleep(delay)
-        msg = yield comm.read()
+            await asyncio.sleep(delay)
+        msg = await comm.read()
         assert msg == {"op": "pong", "data": key}
         l.append(key)
-        yield comm.close()
+        await comm.close()
 
-    yield client_communicate(key=1234)
+    await client_communicate(key=1234)
 
     # Many clients at once
     N = 100
     futures = [client_communicate(key=i, delay=0.05) for i in range(N)]
-    yield futures
+    await asyncio.gather(*futures)
     assert set(l) == {1234} | set(range(N))
 
 
-@gen_test()
-def test_comm_failure_threading():
+@pytest.mark.asyncio
+async def test_comm_failure_threading():
     """
     When we fail to connect, make sure we don't make a lot
     of threads.
@@ -312,40 +307,38 @@ def test_comm_failure_threading():
     set for python 3.  See github PR #2403 discussion for info.
     """
 
-    @gen.coroutine
-    def sleep_for_60ms():
+    async def sleep_for_60ms():
         max_thread_count = 0
         for x in range(60):
-            yield gen.sleep(0.001)
+            await asyncio.sleep(0.001)
             thread_count = threading.active_count()
             if thread_count > max_thread_count:
                 max_thread_count = thread_count
-        raise gen.Return(max_thread_count)
+        return max_thread_count
 
     original_thread_count = threading.active_count()
 
     # tcp.TCPConnector()
     sleep_future = sleep_for_60ms()
     with pytest.raises(IOError):
-        yield connect("tcp://localhost:28400", 0.052)
-    max_thread_count = yield sleep_future
+        await connect("tcp://localhost:28400", 0.052)
+    max_thread_count = await sleep_future
     # 2 is the number set by BaseTCPConnector.executor (ThreadPoolExecutor)
     assert max_thread_count <= 2 + original_thread_count
 
     # tcp.TLSConnector()
     sleep_future = sleep_for_60ms()
     with pytest.raises(IOError):
-        yield connect(
+        await connect(
             "tls://localhost:28400",
             0.052,
             connection_args={"ssl_context": get_client_ssl_context()},
         )
-    max_thread_count = yield sleep_future
+    max_thread_count = await sleep_future
     assert max_thread_count <= 2 + original_thread_count
 
 
-@gen.coroutine
-def check_inproc_specific(run_client):
+async def check_inproc_specific(run_client):
     """
     Test concrete InProc API.
     """
@@ -356,18 +349,17 @@ def check_inproc_specific(run_client):
 
     N_MSGS = 3
 
-    @gen.coroutine
-    def handle_comm(comm):
+    async def handle_comm(comm):
         assert comm.peer_address.startswith("inproc://" + addr_head)
         client_addresses.add(comm.peer_address)
         for i in range(N_MSGS):
-            msg = yield comm.read()
+            msg = await comm.read()
             msg["op"] = "pong"
-            yield comm.write(msg)
-        yield comm.close()
+            await comm.write(msg)
+        await comm.close()
 
     listener = inproc.InProcListener(listener_addr, handle_comm)
-    listener.start()
+    await listener.start()
     assert (
         listener.listen_address
         == listener.contact_address
@@ -377,29 +369,28 @@ def handle_comm(comm):
     connector = inproc.InProcConnector(inproc.global_manager)
     l = []
 
-    @gen.coroutine
-    def client_communicate(key, delay=0):
-        comm = yield connector.connect(listener_addr)
+    async def client_communicate(key, delay=0):
+        comm = await connector.connect(listener_addr)
         assert comm.peer_address == "inproc://" + listener_addr
         for i in range(N_MSGS):
-            yield comm.write({"op": "ping", "data": key})
+            await comm.write({"op": "ping", "data": key})
             if delay:
-                yield gen.sleep(delay)
-            msg = yield comm.read()
+                await asyncio.sleep(delay)
+            msg = await comm.read()
         assert msg == {"op": "pong", "data": key}
         l.append(key)
         with pytest.raises(CommClosedError):
-            yield comm.read()
-        yield comm.close()
+            await comm.read()
+        await comm.close()
 
     client_communicate = partial(run_client, client_communicate)
 
-    yield client_communicate(key=1234)
+    await client_communicate(key=1234)
 
     # Many clients at once
     N = 20
     futures = [client_communicate(key=i, delay=0.001) for i in range(N)]
-    yield futures
+    await asyncio.gather(*futures)
     assert set(l) == {1234} | set(range(N))
 
     assert len(client_addresses) == N + 1
@@ -430,14 +421,14 @@ def run():
     return fut
 
 
-@gen_test()
-def test_inproc_specific_same_thread():
-    yield check_inproc_specific(run_coro)
+@pytest.mark.asyncio
+async def test_inproc_specific_same_thread():
+    await check_inproc_specific(run_coro)
 
 
-@gen_test()
-def test_inproc_specific_different_threads():
-    yield check_inproc_specific(run_coro_in_thread)
+@pytest.mark.asyncio
+async def test_inproc_specific_different_threads():
+    await check_inproc_specific(run_coro_in_thread)
 
 
 #
@@ -445,8 +436,7 @@ def test_inproc_specific_different_threads():
 #
 
 
-@gen.coroutine
-def check_client_server(
+async def check_client_server(
     addr,
     check_listen_addr=None,
     check_contact_addr=None,
@@ -457,27 +447,26 @@ def check_client_server(
     Abstract client / server test.
     """
 
-    @gen.coroutine
-    def handle_comm(comm):
+    async def handle_comm(comm):
         scheme, loc = parse_address(comm.peer_address)
         assert scheme == bound_scheme
 
-        msg = yield comm.read()
+        msg = await comm.read()
         assert msg["op"] == "ping"
         msg["op"] = "pong"
-        yield comm.write(msg)
+        await comm.write(msg)
 
-        msg = yield comm.read()
+        msg = await comm.read()
         assert msg["op"] == "foobar"
 
-        yield comm.close()
+        await comm.close()
 
     # Arbitrary connection args should be ignored
     listen_args = listen_args or {"xxx": "bar"}
     connect_args = connect_args or {"xxx": "foo"}
 
     listener = listen(addr, handle_comm, connection_args=listen_args)
-    listener.start()
+    await listener.start()
 
     # Check listener properties
     bound_addr = listener.listen_address
@@ -500,37 +489,36 @@ def handle_comm(comm):
     # Check client <-> server comms
     l = []
 
-    @gen.coroutine
-    def client_communicate(key, delay=0):
-        comm = yield connect(listener.contact_address, connection_args=connect_args)
+    async def client_communicate(key, delay=0):
+        comm = await connect(listener.contact_address, connection_args=connect_args)
         assert comm.peer_address == listener.contact_address
 
-        yield comm.write({"op": "ping", "data": key})
-        yield comm.write({"op": "foobar"})
+        await comm.write({"op": "ping", "data": key})
+        await comm.write({"op": "foobar"})
         if delay:
-            yield gen.sleep(delay)
-        msg = yield comm.read()
+            await asyncio.sleep(delay)
+        msg = await comm.read()
         assert msg == {"op": "pong", "data": key}
         l.append(key)
-        yield comm.close()
+        await comm.close()
 
-    yield client_communicate(key=1234)
+    await client_communicate(key=1234)
 
     # Many clients at once
     futures = [client_communicate(key=i, delay=0.05) for i in range(20)]
-    yield futures
+    await asyncio.gather(*futures)
     assert set(l) == {1234} | set(range(20))
 
     listener.stop()
 
 
-@gen_test()
-def test_ucx_client_server():
+@pytest.mark.asyncio
+async def test_ucx_client_server():
     pytest.importorskip("distributed.comm.ucx")
     ucp = pytest.importorskip("ucp")
 
     addr = ucp.get_address()
-    yield check_client_server("ucx://" + addr)
+    await check_client_server("ucx://" + addr)
 
 
 def tcp_eq(expected_host, expected_port=None):
@@ -560,79 +548,79 @@ def checker(loc):
     return checker
 
 
-@gen_test()
-def test_default_client_server_ipv4():
+@pytest.mark.asyncio
+async def test_default_client_server_ipv4():
     # Default scheme is (currently) TCP
-    yield check_client_server("127.0.0.1", tcp_eq("127.0.0.1"))
-    yield check_client_server("127.0.0.1:3201", tcp_eq("127.0.0.1", 3201))
-    yield check_client_server("0.0.0.0", tcp_eq("0.0.0.0"), tcp_eq(EXTERNAL_IP4))
-    yield check_client_server(
+    await check_client_server("127.0.0.1", tcp_eq("127.0.0.1"))
+    await check_client_server("127.0.0.1:3201", tcp_eq("127.0.0.1", 3201))
+    await check_client_server("0.0.0.0", tcp_eq("0.0.0.0"), tcp_eq(EXTERNAL_IP4))
+    await check_client_server(
         "0.0.0.0:3202", tcp_eq("0.0.0.0", 3202), tcp_eq(EXTERNAL_IP4, 3202)
     )
     # IPv4 is preferred for the bound address
-    yield check_client_server("", tcp_eq("0.0.0.0"), tcp_eq(EXTERNAL_IP4))
-    yield check_client_server(
+    await check_client_server("", tcp_eq("0.0.0.0"), tcp_eq(EXTERNAL_IP4))
+    await check_client_server(
         ":3203", tcp_eq("0.0.0.0", 3203), tcp_eq(EXTERNAL_IP4, 3203)
     )
 
 
 @requires_ipv6
-@gen_test()
-def test_default_client_server_ipv6():
-    yield check_client_server("[::1]", tcp_eq("::1"))
-    yield check_client_server("[::1]:3211", tcp_eq("::1", 3211))
-    yield check_client_server("[::]", tcp_eq("::"), tcp_eq(EXTERNAL_IP6))
-    yield check_client_server(
+@pytest.mark.asyncio
+async def test_default_client_server_ipv6():
+    await check_client_server("[::1]", tcp_eq("::1"))
+    await check_client_server("[::1]:3211", tcp_eq("::1", 3211))
+    await check_client_server("[::]", tcp_eq("::"), tcp_eq(EXTERNAL_IP6))
+    await check_client_server(
         "[::]:3212", tcp_eq("::", 3212), tcp_eq(EXTERNAL_IP6, 3212)
     )
 
 
-@gen_test()
-def test_tcp_client_server_ipv4():
-    yield check_client_server("tcp://127.0.0.1", tcp_eq("127.0.0.1"))
-    yield check_client_server("tcp://127.0.0.1:3221", tcp_eq("127.0.0.1", 3221))
-    yield check_client_server("tcp://0.0.0.0", tcp_eq("0.0.0.0"), tcp_eq(EXTERNAL_IP4))
-    yield check_client_server(
+@pytest.mark.asyncio
+async def test_tcp_client_server_ipv4():
+    await check_client_server("tcp://127.0.0.1", tcp_eq("127.0.0.1"))
+    await check_client_server("tcp://127.0.0.1:3221", tcp_eq("127.0.0.1", 3221))
+    await check_client_server("tcp://0.0.0.0", tcp_eq("0.0.0.0"), tcp_eq(EXTERNAL_IP4))
+    await check_client_server(
         "tcp://0.0.0.0:3222", tcp_eq("0.0.0.0", 3222), tcp_eq(EXTERNAL_IP4, 3222)
     )
-    yield check_client_server("tcp://", tcp_eq("0.0.0.0"), tcp_eq(EXTERNAL_IP4))
-    yield check_client_server(
+    await check_client_server("tcp://", tcp_eq("0.0.0.0"), tcp_eq(EXTERNAL_IP4))
+    await check_client_server(
         "tcp://:3223", tcp_eq("0.0.0.0", 3223), tcp_eq(EXTERNAL_IP4, 3223)
     )
 
 
 @requires_ipv6
-@gen_test()
-def test_tcp_client_server_ipv6():
-    yield check_client_server("tcp://[::1]", tcp_eq("::1"))
-    yield check_client_server("tcp://[::1]:3231", tcp_eq("::1", 3231))
-    yield check_client_server("tcp://[::]", tcp_eq("::"), tcp_eq(EXTERNAL_IP6))
-    yield check_client_server(
+@pytest.mark.asyncio
+async def test_tcp_client_server_ipv6():
+    await check_client_server("tcp://[::1]", tcp_eq("::1"))
+    await check_client_server("tcp://[::1]:3231", tcp_eq("::1", 3231))
+    await check_client_server("tcp://[::]", tcp_eq("::"), tcp_eq(EXTERNAL_IP6))
+    await check_client_server(
         "tcp://[::]:3232", tcp_eq("::", 3232), tcp_eq(EXTERNAL_IP6, 3232)
     )
 
 
-@gen_test()
-def test_tls_client_server_ipv4():
-    yield check_client_server("tls://127.0.0.1", tls_eq("127.0.0.1"), **tls_kwargs)
-    yield check_client_server(
+@pytest.mark.asyncio
+async def test_tls_client_server_ipv4():
+    await check_client_server("tls://127.0.0.1", tls_eq("127.0.0.1"), **tls_kwargs)
+    await check_client_server(
         "tls://127.0.0.1:3221", tls_eq("127.0.0.1", 3221), **tls_kwargs
     )
-    yield check_client_server(
+    await check_client_server(
         "tls://", tls_eq("0.0.0.0"), tls_eq(EXTERNAL_IP4), **tls_kwargs
     )
 
 
 @requires_ipv6
-@gen_test()
-def test_tls_client_server_ipv6():
-    yield check_client_server("tls://[::1]", tls_eq("::1"), **tls_kwargs)
+@pytest.mark.asyncio
+async def test_tls_client_server_ipv6():
+    await check_client_server("tls://[::1]", tls_eq("::1"), **tls_kwargs)
 
 
-@gen_test()
-def test_inproc_client_server():
-    yield check_client_server("inproc://", inproc_check())
-    yield check_client_server(inproc.new_address(), inproc_check())
+@pytest.mark.asyncio
+async def test_inproc_client_server():
+    await check_client_server("inproc://", inproc_check())
+    await check_client_server(inproc.new_address(), inproc_check())
 
 
 #
@@ -640,8 +628,8 @@ def test_inproc_client_server():
 #
 
 
-@gen_test()
-def test_tls_reject_certificate():
+@pytest.mark.asyncio
+async def test_tls_reject_certificate():
     cli_ctx = get_client_ssl_context()
     serv_ctx = get_server_ssl_context()
 
@@ -650,23 +638,22 @@ def test_tls_reject_certificate():
     bad_cli_ctx = get_client_ssl_context(*bad_cert_key)
     bad_serv_ctx = get_server_ssl_context(*bad_cert_key)
 
-    @gen.coroutine
-    def handle_comm(comm):
+    async def handle_comm(comm):
         scheme, loc = parse_address(comm.peer_address)
         assert scheme == "tls"
-        yield comm.close()
+        await comm.close()
 
     # Listener refuses a connector not signed by the CA
     listener = listen("tls://", handle_comm, connection_args={"ssl_context": serv_ctx})
-    listener.start()
+    await listener.start()
 
     with pytest.raises(EnvironmentError) as excinfo:
-        comm = yield connect(
+        comm = await connect(
             listener.contact_address,
             timeout=0.5,
             connection_args={"ssl_context": bad_cli_ctx},
         )
-        yield comm.write({"x": "foo"})  # TODO: why is this necessary in Tornado 6 ?
+        await comm.write({"x": "foo"})  # TODO: why is this necessary in Tornado 6 ?
 
     # The wrong error is reported on Python 2, see https://github.com/tornadoweb/tornado/pull/2028
     if sys.version_info >= (3,) and os.name != "nt":
@@ -682,19 +669,19 @@ def handle_comm(comm):
                 raise
 
     # Sanity check
-    comm = yield connect(
+    comm = await connect(
         listener.contact_address, timeout=2, connection_args={"ssl_context": cli_ctx}
     )
-    yield comm.close()
+    await comm.close()
 
     # Connector refuses a listener not signed by the CA
     listener = listen(
         "tls://", handle_comm, connection_args={"ssl_context": bad_serv_ctx}
     )
-    listener.start()
+    await listener.start()
 
     with pytest.raises(EnvironmentError) as excinfo:
-        yield connect(
+        await connect(
             listener.contact_address,
             timeout=2,
             connection_args={"ssl_context": cli_ctx},
@@ -709,130 +696,128 @@ def handle_comm(comm):
 #
 
 
-@gen.coroutine
-def check_comm_closed_implicit(addr, delay=None, listen_args=None, connect_args=None):
-    @gen.coroutine
-    def handle_comm(comm):
-        yield comm.close()
+async def check_comm_closed_implicit(
+    addr, delay=None, listen_args=None, connect_args=None
+):
+    async def handle_comm(comm):
+        await comm.close()
 
     listener = listen(addr, handle_comm, connection_args=listen_args)
-    listener.start()
+    await listener.start()
     contact_addr = listener.contact_address
 
-    comm = yield connect(contact_addr, connection_args=connect_args)
+    comm = await connect(contact_addr, connection_args=connect_args)
     with pytest.raises(CommClosedError):
-        yield comm.write({})
+        await comm.write({})
 
-    comm = yield connect(contact_addr, connection_args=connect_args)
+    comm = await connect(contact_addr, connection_args=connect_args)
     with pytest.raises(CommClosedError):
-        yield comm.read()
+        await comm.read()
 
 
-@gen_test()
-def test_tcp_comm_closed_implicit():
-    yield check_comm_closed_implicit("tcp://127.0.0.1")
+@pytest.mark.asyncio
+async def test_tcp_comm_closed_implicit():
+    await check_comm_closed_implicit("tcp://127.0.0.1")
 
 
-@gen_test()
-def test_tls_comm_closed_implicit():
-    yield check_comm_closed_implicit("tls://127.0.0.1", **tls_kwargs)
+@pytest.mark.asyncio
+async def test_tls_comm_closed_implicit():
+    await check_comm_closed_implicit("tls://127.0.0.1", **tls_kwargs)
 
 
-@gen_test()
-def test_inproc_comm_closed_implicit():
-    yield check_comm_closed_implicit(inproc.new_address())
+@pytest.mark.asyncio
+async def test_inproc_comm_closed_implicit():
+    await check_comm_closed_implicit(inproc.new_address())
 
 
-@gen.coroutine
-def check_comm_closed_explicit(addr, listen_args=None, connect_args=None):
-    a, b = yield get_comm_pair(addr, listen_args=listen_args, connect_args=connect_args)
+async def check_comm_closed_explicit(addr, listen_args=None, connect_args=None):
+    a, b = await get_comm_pair(addr, listen_args=listen_args, connect_args=connect_args)
     a_read = a.read()
     b_read = b.read()
-    yield a.close()
+    await a.close()
     # In-flight reads should abort with CommClosedError
     with pytest.raises(CommClosedError):
-        yield a_read
+        await a_read
     with pytest.raises(CommClosedError):
-        yield b_read
+        await b_read
     # New reads as well
     with pytest.raises(CommClosedError):
-        yield a.read()
+        await a.read()
     with pytest.raises(CommClosedError):
-        yield b.read()
+        await b.read()
     # And writes
     with pytest.raises(CommClosedError):
-        yield a.write({})
+        await a.write({})
     with pytest.raises(CommClosedError):
-        yield b.write({})
-    yield b.close()
+        await b.write({})
+    await b.close()
 
 
-@gen_test()
-def test_tcp_comm_closed_explicit():
-    yield check_comm_closed_explicit("tcp://127.0.0.1")
+@pytest.mark.asyncio
+async def test_tcp_comm_closed_explicit():
+    await check_comm_closed_explicit("tcp://127.0.0.1")
 
 
-@gen_test()
-def test_tls_comm_closed_explicit():
-    yield check_comm_closed_explicit("tls://127.0.0.1", **tls_kwargs)
+@pytest.mark.asyncio
+async def test_tls_comm_closed_explicit():
+    await check_comm_closed_explicit("tls://127.0.0.1", **tls_kwargs)
 
 
-@gen_test()
-def test_inproc_comm_closed_explicit():
-    yield check_comm_closed_explicit(inproc.new_address())
+@pytest.mark.asyncio
+async def test_inproc_comm_closed_explicit():
+    await check_comm_closed_explicit(inproc.new_address())
 
 
-@gen_test()
-def test_inproc_comm_closed_explicit_2():
+@pytest.mark.asyncio
+async def test_inproc_comm_closed_explicit_2():
     listener_errors = []
 
-    @gen.coroutine
-    def handle_comm(comm):
+    async def handle_comm(comm):
         # Wait
         try:
-            yield comm.read()
+            await comm.read()
         except CommClosedError:
             assert comm.closed()
             listener_errors.append(True)
         else:
-            yield comm.close()
+            await comm.close()
 
     listener = listen("inproc://", handle_comm)
-    listener.start()
+    await listener.start()
     contact_addr = listener.contact_address
 
-    comm = yield connect(contact_addr)
-    yield comm.close()
+    comm = await connect(contact_addr)
+    await comm.close()
     assert comm.closed()
     start = time()
     while len(listener_errors) < 1:
         assert time() < start + 1
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
     assert len(listener_errors) == 1
 
     with pytest.raises(CommClosedError):
-        yield comm.read()
+        await comm.read()
     with pytest.raises(CommClosedError):
-        yield comm.write("foo")
+        await comm.write("foo")
 
-    comm = yield connect(contact_addr)
-    yield comm.write("foo")
+    comm = await connect(contact_addr)
+    await comm.write("foo")
     with pytest.raises(CommClosedError):
-        yield comm.read()
+        await comm.read()
     with pytest.raises(CommClosedError):
-        yield comm.write("foo")
+        await comm.write("foo")
     assert comm.closed()
 
-    comm = yield connect(contact_addr)
-    yield comm.write("foo")
+    comm = await connect(contact_addr)
+    await comm.write("foo")
 
     start = time()
     while not comm.closed():
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 2
 
-    yield comm.close()
-    yield comm.close()
+    await comm.close()
+    await comm.close()
 
 
 #
@@ -840,28 +825,26 @@ def handle_comm(comm):
 #
 
 
-@gen.coroutine
-def check_connect_timeout(addr):
+async def check_connect_timeout(addr):
     t1 = time()
     with pytest.raises(IOError):
-        yield connect(addr, timeout=0.15)
+        await connect(addr, timeout=0.15)
     dt = time() - t1
     assert 1 >= dt >= 0.1
 
 
-@gen_test()
-def test_tcp_connect_timeout():
-    yield check_connect_timeout("tcp://127.0.0.1:44444")
+@pytest.mark.asyncio
+async def test_tcp_connect_timeout():
+    await check_connect_timeout("tcp://127.0.0.1:44444")
 
 
-@gen_test()
-def test_inproc_connect_timeout():
-    yield check_connect_timeout(inproc.new_address())
+@pytest.mark.asyncio
+async def test_inproc_connect_timeout():
+    await check_connect_timeout(inproc.new_address())
 
 
-def check_many_listeners(addr):
-    @gen.coroutine
-    def handle_comm(comm):
+async def check_many_listeners(addr):
+    async def handle_comm(comm):
         pass
 
     listeners = []
@@ -869,7 +852,7 @@ def handle_comm(comm):
 
     for i in range(N):
         listener = listen(addr, handle_comm)
-        listener.start()
+        await listener.start()
         listeners.append(listener)
 
     assert len(set(l.listen_address for l in listeners)) == N
@@ -879,16 +862,16 @@ def handle_comm(comm):
         listener.stop()
 
 
-@gen_test()
-def test_tcp_many_listeners():
-    check_many_listeners("tcp://127.0.0.1")
-    check_many_listeners("tcp://0.0.0.0")
-    check_many_listeners("tcp://")
+@pytest.mark.asyncio
+async def test_tcp_many_listeners():
+    await check_many_listeners("tcp://127.0.0.1")
+    await check_many_listeners("tcp://0.0.0.0")
+    await check_many_listeners("tcp://")
 
 
-@gen_test()
-def test_inproc_many_listeners():
-    check_many_listeners("inproc://")
+@pytest.mark.asyncio
+async def test_inproc_many_listeners():
+    await check_many_listeners("inproc://")
 
 
 #
@@ -896,47 +879,42 @@ def test_inproc_many_listeners():
 #
 
 
-@gen.coroutine
-def check_listener_deserialize(addr, deserialize, in_value, check_out):
+async def check_listener_deserialize(addr, deserialize, in_value, check_out):
     q = queues.Queue()
 
-    @gen.coroutine
-    def handle_comm(comm):
-        msg = yield comm.read()
+    async def handle_comm(comm):
+        msg = await comm.read()
         q.put_nowait(msg)
-        yield comm.close()
+        await comm.close()
 
-    with listen(addr, handle_comm, deserialize=deserialize) as listener:
-        comm = yield connect(listener.contact_address)
+    async with listen(addr, handle_comm, deserialize=deserialize) as listener:
+        comm = await connect(listener.contact_address)
 
-    yield comm.write(in_value)
+    await comm.write(in_value)
 
-    out_value = yield q.get()
+    out_value = await q.get()
     check_out(out_value)
-    yield comm.close()
+    await comm.close()
 
 
-@gen.coroutine
-def check_connector_deserialize(addr, deserialize, in_value, check_out):
+async def check_connector_deserialize(addr, deserialize, in_value, check_out):
     done = locks.Event()
 
-    @gen.coroutine
-    def handle_comm(comm):
-        yield comm.write(in_value)
-        yield done.wait()
-        yield comm.close()
+    async def handle_comm(comm):
+        await comm.write(in_value)
+        await done.wait()
+        await comm.close()
 
-    with listen(addr, handle_comm) as listener:
-        comm = yield connect(listener.contact_address, deserialize=deserialize)
+    async with listen(addr, handle_comm) as listener:
+        comm = await connect(listener.contact_address, deserialize=deserialize)
 
-    out_value = yield comm.read()
+    out_value = await comm.read()
     done.set()
-    yield comm.close()
+    await comm.close()
     check_out(out_value)
 
 
-@gen.coroutine
-def check_deserialize(addr):
+async def check_deserialize(addr):
     """
     Check the "deserialize" flag on connect() and listen().
     """
@@ -979,11 +957,11 @@ def check_out_true(out_value):
         expected_msg["to_ser"] = [123]
         assert out_value == expected_msg
 
-    yield check_listener_deserialize(addr, False, msg, check_out_false)
-    yield check_connector_deserialize(addr, False, msg, check_out_false)
+    await check_listener_deserialize(addr, False, msg, check_out_false)
+    await check_connector_deserialize(addr, False, msg, check_out_false)
 
-    yield check_listener_deserialize(addr, True, msg, check_out_true)
-    yield check_connector_deserialize(addr, True, msg, check_out_true)
+    await check_listener_deserialize(addr, True, msg, check_out_true)
+    await check_connector_deserialize(addr, True, msg, check_out_true)
 
     # Test with long bytestrings, large enough to be transferred
     # as a separate payload
@@ -1024,26 +1002,25 @@ def check_out(deserialize_flag, out_value):
             else:
                 assert to_ser == to_serialize(_uncompressible)
 
-    yield check_listener_deserialize(addr, False, msg, partial(check_out, False))
-    yield check_connector_deserialize(addr, False, msg, partial(check_out, False))
+    await check_listener_deserialize(addr, False, msg, partial(check_out, False))
+    await check_connector_deserialize(addr, False, msg, partial(check_out, False))
 
-    yield check_listener_deserialize(addr, True, msg, partial(check_out, True))
-    yield check_connector_deserialize(addr, True, msg, partial(check_out, True))
+    await check_listener_deserialize(addr, True, msg, partial(check_out, True))
+    await check_connector_deserialize(addr, True, msg, partial(check_out, True))
 
 
 @pytest.mark.xfail(reason="intermittent failure on windows")
-@gen_test()
-def test_tcp_deserialize():
-    yield check_deserialize("tcp://")
+@pytest.mark.asyncio
+async def test_tcp_deserialize():
+    await check_deserialize("tcp://")
 
 
-@gen_test()
-def test_inproc_deserialize():
-    yield check_deserialize("inproc://")
+@pytest.mark.asyncio
+async def test_inproc_deserialize():
+    await check_deserialize("inproc://")
 
 
-@gen.coroutine
-def check_deserialize_roundtrip(addr):
+async def check_deserialize_roundtrip(addr):
     """
     Sanity check round-tripping with "deserialize" on and off.
     """
@@ -1059,11 +1036,11 @@ def check_deserialize_roundtrip(addr):
     }
 
     for should_deserialize in (True, False):
-        a, b = yield get_comm_pair(addr, deserialize=should_deserialize)
-        yield a.write(msg)
-        got = yield b.read()
-        yield b.write(got)
-        got = yield a.read()
+        a, b = await get_comm_pair(addr, deserialize=should_deserialize)
+        await a.write(msg)
+        got = await b.read()
+        await b.write(got)
+        got = await a.read()
 
         assert sorted(got) == sorted(msg)
         for k in ("op", "x"):
@@ -1076,14 +1053,14 @@ def check_deserialize_roundtrip(addr):
             assert isinstance(got["ser"], Serialized)
 
 
-@gen_test()
-def test_inproc_deserialize_roundtrip():
-    yield check_deserialize_roundtrip("inproc://")
+@pytest.mark.asyncio
+async def test_inproc_deserialize_roundtrip():
+    await check_deserialize_roundtrip("inproc://")
 
 
-@gen_test()
-def test_tcp_deserialize_roundtrip():
-    yield check_deserialize_roundtrip("tcp://")
+@pytest.mark.asyncio
+async def test_tcp_deserialize_roundtrip():
+    await check_deserialize_roundtrip("tcp://")
 
 
 def _raise_eoferror():
@@ -1095,27 +1072,25 @@ def __reduce__(self):
         return _raise_eoferror, ()
 
 
-@gen.coroutine
-def check_deserialize_eoferror(addr):
+async def check_deserialize_eoferror(addr):
     """
     EOFError when deserializing should close the comm.
     """
 
-    @gen.coroutine
-    def handle_comm(comm):
-        yield comm.write({"data": to_serialize(_EOFRaising())})
+    async def handle_comm(comm):
+        await comm.write({"data": to_serialize(_EOFRaising())})
         with pytest.raises(CommClosedError):
-            yield comm.read()
+            await comm.read()
 
-    with listen(addr, handle_comm) as listener:
-        comm = yield connect(listener.contact_address, deserialize=deserialize)
+    async with listen(addr, handle_comm) as listener:
+        comm = await connect(listener.contact_address, deserialize=deserialize)
         with pytest.raises(CommClosedError):
-            yield comm.read()
+            await comm.read()
 
 
-@gen_test()
-def test_tcp_deserialize_eoferror():
-    yield check_deserialize_eoferror("tcp://")
+@pytest.mark.asyncio
+async def test_tcp_deserialize_eoferror():
+    await check_deserialize_eoferror("tcp://")
 
 
 #
@@ -1123,61 +1098,59 @@ def test_tcp_deserialize_eoferror():
 #
 
 
-@gen.coroutine
-def check_repr(a, b):
+async def check_repr(a, b):
     assert "closed" not in repr(a)
     assert "closed" not in repr(b)
-    yield a.close()
+    await a.close()
     assert "closed" in repr(a)
-    yield b.close()
+    await b.close()
     assert "closed" in repr(b)
 
 
-@gen_test()
-def test_tcp_repr():
-    a, b = yield get_tcp_comm_pair()
+@pytest.mark.asyncio
+async def test_tcp_repr():
+    a, b = await get_tcp_comm_pair()
     assert a.local_address in repr(b)
     assert b.local_address in repr(a)
-    yield check_repr(a, b)
+    await check_repr(a, b)
 
 
-@gen_test()
-def test_tls_repr():
-    a, b = yield get_tls_comm_pair()
+@pytest.mark.asyncio
+async def test_tls_repr():
+    a, b = await get_tls_comm_pair()
     assert a.local_address in repr(b)
     assert b.local_address in repr(a)
-    yield check_repr(a, b)
+    await check_repr(a, b)
 
 
-@gen_test()
-def test_inproc_repr():
-    a, b = yield get_inproc_comm_pair()
+@pytest.mark.asyncio
+async def test_inproc_repr():
+    a, b = await get_inproc_comm_pair()
     assert a.local_address in repr(b)
     assert b.local_address in repr(a)
-    yield check_repr(a, b)
+    await check_repr(a, b)
 
 
-@gen.coroutine
-def check_addresses(a, b):
+async def check_addresses(a, b):
     assert a.peer_address == b.local_address
     assert a.local_address == b.peer_address
     a.abort()
     b.abort()
 
 
-@gen_test()
-def test_tcp_adresses():
-    a, b = yield get_tcp_comm_pair()
-    yield check_addresses(a, b)
+@pytest.mark.asyncio
+async def test_tcp_adresses():
+    a, b = await get_tcp_comm_pair()
+    await check_addresses(a, b)
 
 
-@gen_test()
-def test_tls_adresses():
-    a, b = yield get_tls_comm_pair()
-    yield check_addresses(a, b)
+@pytest.mark.asyncio
+async def test_tls_adresses():
+    a, b = await get_tls_comm_pair()
+    await check_addresses(a, b)
 
 
-@gen_test()
-def test_inproc_adresses():
-    a, b = yield get_inproc_comm_pair()
-    yield check_addresses(a, b)
+@pytest.mark.asyncio
+async def test_inproc_adresses():
+    a, b = await get_inproc_comm_pair()
+    await check_addresses(a, b)
diff --git a/distributed/comm/ucx.py b/distributed/comm/ucx.py
index 2bdcff5e958..7c783b605a1 100644
--- a/distributed/comm/ucx.py
+++ b/distributed/comm/ucx.py
@@ -261,7 +261,7 @@ def port(self):
     def address(self):
         return "ucx://" + self.ip + ":" + str(self.port)
 
-    def start(self):
+    async def start(self):
         async def serve_forever(client_ep):
             ucx = UCX(
                 client_ep,
diff --git a/distributed/core.py b/distributed/core.py
index 3e1a3b47cc3..ec6e9969fea 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -295,7 +295,7 @@ def port(self):
     def identity(self, comm=None):
         return {"type": type(self).__name__, "id": self.id}
 
-    def listen(self, port_or_addr=None, listen_args=None):
+    async def listen(self, port_or_addr=None, listen_args=None):
         if port_or_addr is None:
             port_or_addr = self.default_port
         if isinstance(port_or_addr, int):
@@ -311,7 +311,7 @@ def listen(self, port_or_addr=None, listen_args=None):
             deserialize=self.deserialize,
             connection_args=listen_args,
         )
-        self.listener.start()
+        await self.listener.start()
 
     async def handle_comm(self, comm, shutting_down=shutting_down):
         """ Dispatch new communications to coroutine-handlers
diff --git a/distributed/nanny.py b/distributed/nanny.py
index 11cf0157c10..dc2e8a3ea48 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -244,7 +244,7 @@ def local_dir(self):
 
     async def start(self):
         """ Start nanny, start local process, start watching """
-        self.listen(self._start_address, listen_args=self.listen_args)
+        await self.listen(self._start_address, listen_args=self.listen_args)
         self.ip = get_address_host(self.address)
 
         logger.info("        Start Nanny at: %r", self.address)
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 26b6bffa970..b77c36477c1 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1213,7 +1213,7 @@ async def start(self):
                 c.cancel()
 
         if self.status != "running":
-            self.listen(self._start_address, listen_args=self.listen_args)
+            await self.listen(self._start_address, listen_args=self.listen_args)
             self.ip = get_address_host(self.listen_address)
             listen_ip = self.ip
 
diff --git a/distributed/tests/test_batched.py b/distributed/tests/test_batched.py
index af281aff8c3..a961157f948 100644
--- a/distributed/tests/test_batched.py
+++ b/distributed/tests/test_batched.py
@@ -1,4 +1,4 @@
-from contextlib import contextmanager
+import asyncio
 from datetime import timedelta
 import random
 
@@ -10,72 +10,67 @@
 from distributed.core import listen, connect, CommClosedError
 from distributed.metrics import time
 from distributed.utils import All
-from distributed.utils_test import gen_test, captured_logger
+from distributed.utils_test import captured_logger
 from distributed.protocol import to_serialize
 
 
 class EchoServer(object):
     count = 0
 
-    @gen.coroutine
-    def handle_comm(self, comm):
+    async def handle_comm(self, comm):
         while True:
             try:
-                msg = yield comm.read()
+                msg = await comm.read()
                 self.count += 1
-                yield comm.write(msg)
+                await comm.write(msg)
             except CommClosedError as e:
                 return
 
-    def listen(self):
+    async def listen(self):
         listener = listen("", self.handle_comm)
-        listener.start()
+        await listener.start()
         self.address = listener.contact_address
         self.stop = listener.stop
 
+    async def __aenter__(self):
+        await self.listen()
+        return self
 
-@contextmanager
-def echo_server():
-    server = EchoServer()
-    server.listen()
+    async def __aexit__(self, exc, typ, tb):
+        self.stop()
 
-    try:
-        yield server
-    finally:
-        server.stop()
 
-
-@gen_test()
-def test_BatchedSend():
-    with echo_server() as e:
-        comm = yield connect(e.address)
+@pytest.mark.asyncio
+async def test_BatchedSend():
+    async with EchoServer() as e:
+        comm = await connect(e.address)
 
         b = BatchedSend(interval=10)
         assert str(len(b.buffer)) in str(b)
         assert str(len(b.buffer)) in repr(b)
         b.start(comm)
 
-        yield gen.sleep(0.020)
+        await asyncio.sleep(0.020)
 
         b.send("hello")
         b.send("hello")
         b.send("world")
-        yield gen.sleep(0.020)
+        await asyncio.sleep(0.020)
         b.send("HELLO")
         b.send("HELLO")
 
-        result = yield comm.read()
+        result = await comm.read()
         assert result == ("hello", "hello", "world")
-        result = yield comm.read()
+        result = await comm.read()
         assert result == ("HELLO", "HELLO")
 
         assert b.byte_count > 1
 
 
-@gen_test()
-def test_send_before_start():
-    with echo_server() as e:
-        comm = yield connect(e.address)
+@pytest.mark.asyncio
+async def test_send_before_start():
+    async with EchoServer() as e:
+        comm = await connect(e.address)
 
         b = BatchedSend(interval=10)
 
@@ -83,120 +78,117 @@ def test_send_before_start():
         b.send("world")
 
         b.start(comm)
-        result = yield comm.read()
+        result = await comm.read()
         assert result == ("hello", "world")
 
 
-@gen_test()
-def test_send_after_stream_start():
-    with echo_server() as e:
-        comm = yield connect(e.address)
+@pytest.mark.asyncio
+async def test_send_after_stream_start():
+    async with EchoServer() as e:
+        comm = await connect(e.address)
 
         b = BatchedSend(interval=10)
 
         b.start(comm)
         b.send("hello")
         b.send("world")
-        result = yield comm.read()
+        result = await comm.read()
         if len(result) < 2:
-            result += yield comm.read()
+            result += await comm.read()
         assert result == ("hello", "world")
 
 
-@gen_test()
-def test_send_before_close():
-    with echo_server() as e:
-        comm = yield connect(e.address)
+@pytest.mark.asyncio
+async def test_send_before_close():
+    async with EchoServer() as e:
+        comm = await connect(e.address)
 
         b = BatchedSend(interval=10)
         b.start(comm)
 
         cnt = int(e.count)
         b.send("hello")
-        yield b.close()  # close immediately after sending
+        await b.close()  # close immediately after sending
         assert not b.buffer
 
         start = time()
         while e.count != cnt + 1:
-            yield gen.sleep(0.01)
+            await asyncio.sleep(0.01)
             assert time() < start + 5
 
         with pytest.raises(CommClosedError):
             b.send("123")
 
 
-@gen_test()
-def test_close_closed():
-    with echo_server() as e:
-        comm = yield connect(e.address)
+@pytest.mark.asyncio
+async def test_close_closed():
+    async with EchoServer() as e:
+        comm = await connect(e.address)
 
         b = BatchedSend(interval=10)
         b.start(comm)
 
         b.send(123)
-        yield comm.close()  # external closing
+        await comm.close()  # external closing
 
-        yield b.close()
+        await b.close()
         assert "closed" in repr(b)
         assert "closed" in str(b)
 
 
-@gen_test()
-def test_close_not_started():
+@pytest.mark.asyncio
+async def test_close_not_started():
     b = BatchedSend(interval=10)
-    yield b.close()
+    await b.close()
 
 
-@gen_test()
-def test_close_twice():
-    with echo_server() as e:
-        comm = yield connect(e.address)
+@pytest.mark.asyncio
+async def test_close_twice():
+    async with EchoServer() as e:
+        comm = await connect(e.address)
 
         b = BatchedSend(interval=10)
         b.start(comm)
-        yield b.close()
-        yield b.close()
+        await b.close()
+        await b.close()
 
 
 @pytest.mark.slow
-@gen_test(timeout=50)
-def test_stress():
-    with echo_server() as e:
-        comm = yield connect(e.address)
+@pytest.mark.asyncio
+async def test_stress():
+    async with EchoServer() as e:
+        comm = await connect(e.address)
         L = []
 
-        @gen.coroutine
-        def send():
+        async def send():
             b = BatchedSend(interval=3)
             b.start(comm)
             for i in range(0, 10000, 2):
                 b.send(i)
                 b.send(i + 1)
-                yield gen.sleep(0.00001 * random.randint(1, 10))
+                await asyncio.sleep(0.00001 * random.randint(1, 10))
 
-        @gen.coroutine
-        def recv():
+        async def recv():
             while True:
-                result = yield gen.with_timeout(timedelta(seconds=1), comm.read())
+                result = await gen.with_timeout(timedelta(seconds=1), comm.read())
                 L.extend(result)
                 if result[-1] == 9999:
                     break
 
-        yield All([send(), recv()])
+        await All([send(), recv()])
 
         assert L == list(range(0, 10000, 1))
-        yield comm.close()
+        await comm.close()
 
 
-@gen.coroutine
-def run_traffic_jam(nsends, nbytes):
+async def run_traffic_jam(nsends, nbytes):
     # This test eats `nsends * nbytes` bytes in RAM
     np = pytest.importorskip("numpy")
     from distributed.protocol import to_serialize
 
     data = bytes(np.random.randint(0, 255, size=(nbytes,)).astype("u1").data)
-    with echo_server() as e:
-        comm = yield connect(e.address)
+    async with EchoServer() as e:
+        comm = await connect(e.address)
 
         b = BatchedSend(interval=0.01)
         b.start(comm)
@@ -205,7 +197,7 @@ def run_traffic_jam(nsends, nbytes):
         for i in range(nsends):
             b.send(assoc(msg, "i", i))
             if np.random.random() > 0.5:
-                yield gen.sleep(0.001)
+                await asyncio.sleep(0.001)
 
         results = []
         count = 0
@@ -213,7 +205,7 @@ def run_traffic_jam(nsends, nbytes):
             # If this times out then I think it's a backpressure issue
             # Somehow we're able to flood the socket so that the receiving end
             # loses some of our messages
-            L = yield gen.with_timeout(timedelta(seconds=5), comm.read())
+            L = await gen.with_timeout(timedelta(seconds=5), comm.read())
             count += 1
             results.extend(r["i"] for r in L)
 
@@ -222,45 +214,45 @@ def run_traffic_jam(nsends, nbytes):
 
         assert results == list(range(nsends))
 
-        yield comm.close()  # external closing
-        yield b.close()
+        await comm.close()  # external closing
+        await b.close()
 
 
-@gen_test()
-def test_sending_traffic_jam():
-    yield run_traffic_jam(50, 300000)
+@pytest.mark.asyncio
+async def test_sending_traffic_jam():
+    await run_traffic_jam(50, 300000)
 
 
 @pytest.mark.slow
-@gen_test()
-def test_large_traffic_jam():
-    yield run_traffic_jam(500, 1500000)
+@pytest.mark.asyncio
+async def test_large_traffic_jam():
+    await run_traffic_jam(500, 1500000)
 
 
-@gen_test()
-def test_serializers():
-    with echo_server() as e:
-        comm = yield connect(e.address)
+@pytest.mark.asyncio
+async def test_serializers():
+    async with EchoServer() as e:
+        comm = await connect(e.address)
 
         b = BatchedSend(interval="10ms", serializers=["msgpack"])
         b.start(comm)
 
         b.send({"x": to_serialize(123)})
         b.send({"x": to_serialize("hello")})
-        yield gen.sleep(0.100)
+        await asyncio.sleep(0.100)
 
         b.send({"x": to_serialize(lambda x: x + 1)})
 
         with captured_logger("distributed.protocol") as sio:
-            yield gen.sleep(0.100)
+            await asyncio.sleep(0.100)
 
         value = sio.getvalue()
         assert "serialize" in value
         assert "type" in value
         assert "function" in value
 
-        msg = yield comm.read()
+        msg = await comm.read()
         assert list(msg) == [{"x": 123}, {"x": "hello"}]
 
         with pytest.raises(gen.TimeoutError):
-            msg = yield gen.with_timeout(timedelta(milliseconds=100), comm.read())
+            msg = await gen.with_timeout(timedelta(milliseconds=100), comm.read())
diff --git a/distributed/tests/test_core.py b/distributed/tests/test_core.py
index 78fbd1211d7..d3bcdaf8987 100644
--- a/distributed/tests/test_core.py
+++ b/distributed/tests/test_core.py
@@ -1,7 +1,7 @@
 import asyncio
-from contextlib import contextmanager
 import os
 import socket
+import sys
 import threading
 import weakref
 
@@ -23,7 +23,6 @@
 from distributed.protocol import to_serialize
 from distributed.utils import get_ip, get_ipv6
 from distributed.utils_test import (
-    gen_test,
     gen_cluster,
     has_ipv6,
     assert_can_connect,
@@ -86,7 +85,7 @@ async def f():
         server = Server({"ping": pingpong})
         with pytest.raises(ValueError):
             server.port
-        server.listen(8881)
+        await server.listen(8881)
         assert server.port == 8881
         assert server.address == ("tcp://%s:8881" % get_ip())
 
@@ -114,7 +113,7 @@ async def f():
 def test_server_raises_on_blocked_handlers(loop):
     async def f():
         server = Server({"ping": pingpong}, blocked_handlers=["ping"])
-        server.listen(8881)
+        await server.listen(8881)
 
         comm = await connect(server.address)
         await comm.write({"op": "ping"})
@@ -134,16 +133,21 @@ class MyServer(Server):
     default_port = 8756
 
 
-@gen_test()
-def test_server_listen():
+@pytest.mark.skipif(
+    sys.version_info < (3, 7),
+    reason="asynccontextmanager not avaiable before Python 3.7",
+)
+@pytest.mark.asyncio
+async def test_server_listen():
     """
     Test various Server.listen() arguments and their effect.
     """
+    from contextlib import asynccontextmanager
 
-    @contextmanager
-    def listen_on(cls, *args, **kwargs):
+    @asynccontextmanager
+    async def listen_on(cls, *args, **kwargs):
         server = cls({})
-        server.listen(*args, **kwargs)
+        await server.listen(*args, **kwargs)
         try:
             yield server
         finally:
@@ -151,107 +155,107 @@ def listen_on(cls, *args, **kwargs):
 
     # Note server.address is the concrete, contactable address
 
-    with listen_on(Server, 7800) as server:
+    async with listen_on(Server, 7800) as server:
         assert server.port == 7800
         assert server.address == "tcp://%s:%d" % (EXTERNAL_IP4, server.port)
-        yield assert_can_connect(server.address)
-        yield assert_can_connect_from_everywhere_4_6(server.port)
+        await assert_can_connect(server.address)
+        await assert_can_connect_from_everywhere_4_6(server.port)
 
-    with listen_on(Server) as server:
+    async with listen_on(Server) as server:
         assert server.port > 0
         assert server.address == "tcp://%s:%d" % (EXTERNAL_IP4, server.port)
-        yield assert_can_connect(server.address)
-        yield assert_can_connect_from_everywhere_4_6(server.port)
+        await assert_can_connect(server.address)
+        await assert_can_connect_from_everywhere_4_6(server.port)
 
-    with listen_on(MyServer) as server:
+    async with listen_on(MyServer) as server:
         assert server.port == MyServer.default_port
         assert server.address == "tcp://%s:%d" % (EXTERNAL_IP4, server.port)
-        yield assert_can_connect(server.address)
-        yield assert_can_connect_from_everywhere_4_6(server.port)
+        await assert_can_connect(server.address)
+        await assert_can_connect_from_everywhere_4_6(server.port)
 
-    with listen_on(Server, ("", 7801)) as server:
+    async with listen_on(Server, ("", 7801)) as server:
         assert server.port == 7801
         assert server.address == "tcp://%s:%d" % (EXTERNAL_IP4, server.port)
-        yield assert_can_connect(server.address)
-        yield assert_can_connect_from_everywhere_4_6(server.port)
+        await assert_can_connect(server.address)
+        await assert_can_connect_from_everywhere_4_6(server.port)
 
-    with listen_on(Server, "tcp://:7802") as server:
+    async with listen_on(Server, "tcp://:7802") as server:
         assert server.port == 7802
         assert server.address == "tcp://%s:%d" % (EXTERNAL_IP4, server.port)
-        yield assert_can_connect(server.address)
-        yield assert_can_connect_from_everywhere_4_6(server.port)
+        await assert_can_connect(server.address)
+        await assert_can_connect_from_everywhere_4_6(server.port)
 
     # Only IPv4
 
-    with listen_on(Server, ("0.0.0.0", 7810)) as server:
+    async with listen_on(Server, ("0.0.0.0", 7810)) as server:
         assert server.port == 7810
         assert server.address == "tcp://%s:%d" % (EXTERNAL_IP4, server.port)
-        yield assert_can_connect(server.address)
-        yield assert_can_connect_from_everywhere_4(server.port)
+        await assert_can_connect(server.address)
+        await assert_can_connect_from_everywhere_4(server.port)
 
-    with listen_on(Server, ("127.0.0.1", 7811)) as server:
+    async with listen_on(Server, ("127.0.0.1", 7811)) as server:
         assert server.port == 7811
         assert server.address == "tcp://127.0.0.1:%d" % server.port
-        yield assert_can_connect(server.address)
-        yield assert_can_connect_locally_4(server.port)
+        await assert_can_connect(server.address)
+        await assert_can_connect_locally_4(server.port)
 
-    with listen_on(Server, "tcp://127.0.0.1:7812") as server:
+    async with listen_on(Server, "tcp://127.0.0.1:7812") as server:
         assert server.port == 7812
         assert server.address == "tcp://127.0.0.1:%d" % server.port
-        yield assert_can_connect(server.address)
-        yield assert_can_connect_locally_4(server.port)
+        await assert_can_connect(server.address)
+        await assert_can_connect_locally_4(server.port)
 
     # Only IPv6
 
     if has_ipv6():
-        with listen_on(Server, ("::", 7813)) as server:
+        async with listen_on(Server, ("::", 7813)) as server:
             assert server.port == 7813
             assert server.address == "tcp://[%s]:%d" % (EXTERNAL_IP6, server.port)
-            yield assert_can_connect(server.address)
-            yield assert_can_connect_from_everywhere_6(server.port)
+            await assert_can_connect(server.address)
+            await assert_can_connect_from_everywhere_6(server.port)
 
-        with listen_on(Server, ("::1", 7814)) as server:
+        async with listen_on(Server, ("::1", 7814)) as server:
             assert server.port == 7814
             assert server.address == "tcp://[::1]:%d" % server.port
-            yield assert_can_connect(server.address)
-            yield assert_can_connect_locally_6(server.port)
+            await assert_can_connect(server.address)
+            await assert_can_connect_locally_6(server.port)
 
-        with listen_on(Server, "tcp://[::1]:7815") as server:
+        async with listen_on(Server, "tcp://[::1]:7815") as server:
             assert server.port == 7815
             assert server.address == "tcp://[::1]:%d" % server.port
-            yield assert_can_connect(server.address)
-            yield assert_can_connect_locally_6(server.port)
+            await assert_can_connect(server.address)
+            await assert_can_connect_locally_6(server.port)
 
     # TLS
 
     sec = tls_security()
-    with listen_on(
+    async with listen_on(
         Server, "tls://", listen_args=sec.get_listen_args("scheduler")
     ) as server:
         assert server.address.startswith("tls://")
-        yield assert_can_connect(
+        await assert_can_connect(
             server.address, connection_args=sec.get_connection_args("client")
         )
 
     # InProc
 
-    with listen_on(Server, "inproc://") as server:
+    async with listen_on(Server, "inproc://") as server:
         inproc_addr1 = server.address
         assert inproc_addr1.startswith("inproc://%s/%d/" % (get_ip(), os.getpid()))
-        yield assert_can_connect(inproc_addr1)
+        await assert_can_connect(inproc_addr1)
 
-        with listen_on(Server, "inproc://") as server2:
+        async with listen_on(Server, "inproc://") as server2:
             inproc_addr2 = server2.address
             assert inproc_addr2.startswith("inproc://%s/%d/" % (get_ip(), os.getpid()))
-            yield assert_can_connect(inproc_addr2)
+            await assert_can_connect(inproc_addr2)
 
-        yield assert_can_connect(inproc_addr1)
-        yield assert_cannot_connect(inproc_addr2)
+        await assert_can_connect(inproc_addr1)
+        await assert_cannot_connect(inproc_addr2)
 
 
 async def check_rpc(listen_addr, rpc_addr=None, listen_args=None, connection_args=None):
     server = Server({"ping": pingpong})
-    server.listen(listen_addr, listen_args=listen_args)
+    await server.listen(listen_addr, listen_args=listen_args)
     if rpc_addr is None:
         rpc_addr = server.address
 
@@ -269,24 +273,25 @@ async def check_rpc(listen_addr, rpc_addr=None, listen_args=None, connection_arg
     assert remote.status == "closed"
 
     server.stop()
+    await asyncio.sleep(0)
 
 
-@gen_test()
-def test_rpc_default():
-    yield check_rpc(8883, "127.0.0.1:8883")
-    yield check_rpc(8883)
+@pytest.mark.asyncio
+async def test_rpc_default():
+    await check_rpc(8883, "127.0.0.1:8883")
+    await check_rpc(8883)
 
 
-@gen_test()
-def test_rpc_tcp():
-    yield check_rpc("tcp://:8883", "tcp://127.0.0.1:8883")
-    yield check_rpc("tcp://")
+@pytest.mark.asyncio
+async def test_rpc_tcp():
+    await check_rpc("tcp://:8883", "tcp://127.0.0.1:8883")
+    await check_rpc("tcp://")
 
 
-@gen_test()
-def test_rpc_tls():
+@pytest.mark.asyncio
+async def test_rpc_tls():
     sec = tls_security()
-    yield check_rpc(
+    await check_rpc(
         "tcp://",
         None,
         sec.get_listen_args("scheduler"),
@@ -294,9 +299,9 @@ def test_rpc_tls():
     )
 
 
-@gen_test()
-def test_rpc_inproc():
-    yield check_rpc("inproc://", None)
+@pytest.mark.asyncio
+async def test_rpc_inproc():
+    await check_rpc("inproc://", None)
 
 
 @pytest.mark.asyncio
@@ -313,7 +318,7 @@ async def check_rpc_message_lifetime(*listen_args):
     # Issue #956: rpc arguments and result shouldn't be kept alive longer
     # than necessary
     server = Server({"echo": echo_serialize})
-    server.listen(*listen_args)
+    await server.listen(*listen_args)
 
     # Sanity check
     obj = CountedObject()
@@ -343,19 +348,19 @@ async def check_rpc_message_lifetime(*listen_args):
     server.stop()
 
 
-@gen_test()
-def test_rpc_message_lifetime_default():
-    yield check_rpc_message_lifetime()
+@pytest.mark.asyncio
+async def test_rpc_message_lifetime_default():
+    await check_rpc_message_lifetime()
 
 
-@gen_test()
-def test_rpc_message_lifetime_tcp():
-    yield check_rpc_message_lifetime("tcp://")
+@pytest.mark.asyncio
+async def test_rpc_message_lifetime_tcp():
+    await check_rpc_message_lifetime("tcp://")
 
 
-@gen_test()
-def test_rpc_message_lifetime_inproc():
-    yield check_rpc_message_lifetime("inproc://")
+@pytest.mark.asyncio
+async def test_rpc_message_lifetime_inproc():
+    await check_rpc_message_lifetime("inproc://")
 
 
 async def check_rpc_with_many_connections(listen_arg):
@@ -364,7 +369,7 @@ async def g():
             await remote.ping()
 
     server = Server({"ping": pingpong})
-    server.listen(listen_arg)
+    await server.listen(listen_arg)
 
     async with rpc(server.address) as remote:
         for i in range(10):
@@ -376,20 +381,20 @@ async def g():
         assert all(comm.closed() for comm in remote.comms)
 
 
-@gen_test()
-def test_rpc_with_many_connections_tcp():
-    yield check_rpc_with_many_connections("tcp://")
+@pytest.mark.asyncio
+async def test_rpc_with_many_connections_tcp():
+    await check_rpc_with_many_connections("tcp://")
 
 
-@gen_test()
-def test_rpc_with_many_connections_inproc():
-    yield check_rpc_with_many_connections("inproc://")
+@pytest.mark.asyncio
+async def test_rpc_with_many_connections_inproc():
+    await check_rpc_with_many_connections("inproc://")
 
 
 async def check_large_packets(listen_arg):
     """ tornado has a 100MB cap by default """
     server = Server({"echo": echo})
-    server.listen(listen_arg)
+    await server.listen(listen_arg)
 
     data = b"0" * int(200e6)  # slightly more than 100MB
     async with rpc(server.address) as conn:
@@ -404,19 +409,19 @@ async def check_large_packets(listen_arg):
 
 
 @pytest.mark.slow
-@gen_test()
-def test_large_packets_tcp():
-    yield check_large_packets("tcp://")
+@pytest.mark.asyncio
+async def test_large_packets_tcp():
+    await check_large_packets("tcp://")
 
 
-@gen_test()
-def test_large_packets_inproc():
-    yield check_large_packets("inproc://")
+@pytest.mark.asyncio
+async def test_large_packets_inproc():
+    await check_large_packets("inproc://")
 
 
 async def check_identity(listen_arg):
     server = Server({})
-    server.listen(listen_arg)
+    await server.listen(listen_arg)
 
     async with rpc(server.address) as remote:
         a = await remote.identity()
@@ -427,21 +432,22 @@ async def check_identity(listen_arg):
     server.stop()
 
 
-@gen_test()
-def test_identity_tcp():
-    yield check_identity("tcp://")
+@pytest.mark.asyncio
+async def test_identity_tcp():
+    await check_identity("tcp://")
 
 
-@gen_test()
-def test_identity_inproc():
-    yield check_identity("inproc://")
+@pytest.mark.asyncio
+async def test_identity_inproc():
+    await check_identity("inproc://")
 
 
-def test_ports(loop):
+@pytest.mark.asyncio
+async def test_ports(loop):
     for port in range(9877, 9887):
         server = Server({}, io_loop=loop)
         try:
-            server.listen(port)
+            await server.listen(port)
         except OSError:  # port already taken?
             pass
         else:
@@ -453,13 +459,13 @@ def test_ports(loop):
 
         with pytest.raises((OSError, socket.error)):
             server2 = Server({}, io_loop=loop)
-            server2.listen(port)
+            await server2.listen(port)
     finally:
         server.stop()
 
     try:
         server3 = Server({}, io_loop=loop)
-        server3.listen(0)
+        await server3.listen(0)
         assert isinstance(server3.port, int)
         assert server3.port > 1024
     finally:
@@ -470,35 +476,35 @@ def stream_div(stream=None, x=None, y=None):
     return x / y
 
 
-@gen_test()
-def test_errors():
+@pytest.mark.asyncio
+async def test_errors():
     server = Server({"div": stream_div})
-    server.listen(0)
+    await server.listen(0)
 
     with rpc(("127.0.0.1", server.port)) as r:
         with pytest.raises(ZeroDivisionError):
-            yield r.div(x=1, y=0)
+            await r.div(x=1, y=0)
 
 
-@gen_test()
-def test_connect_raises():
+@pytest.mark.asyncio
+async def test_connect_raises():
     with pytest.raises(IOError):
-        yield connect("127.0.0.1:58259", timeout=0.01)
+        await connect("127.0.0.1:58259", timeout=0.01)
 
 
-@gen_test()
-def test_send_recv_args():
+@pytest.mark.asyncio
+async def test_send_recv_args():
     server = Server({"echo": echo})
-    server.listen(0)
+    await server.listen(0)
 
-    comm = yield connect(server.address)
-    result = yield send_recv(comm, op="echo", x=b"1")
+    comm = await connect(server.address)
+    result = await send_recv(comm, op="echo", x=b"1")
     assert result == b"1"
     assert not comm.closed()
-    result = yield send_recv(comm, op="echo", x=b"2", reply=False)
+    result = await send_recv(comm, op="echo", x=b"2", reply=False)
     assert result is None
     assert not comm.closed()
-    result = yield send_recv(comm, op="echo", x=b"3", close=True)
+    result = await send_recv(comm, op="echo", x=b"3", close=True)
     assert result == b"3"
     assert comm.closed()
 
@@ -510,49 +516,57 @@ def test_coerce_to_address():
         assert coerce_to_address(arg) == "tcp://127.0.0.1:8786"
 
 
-@gen_test()
-def test_connection_pool():
+@pytest.mark.asyncio
+async def test_connection_pool():
     async def ping(comm, delay=0.1):
         await asyncio.sleep(delay)
         return "pong"
 
     servers = [Server({"ping": ping}) for i in range(10)]
     for server in servers:
-        server.listen(0)
+        await server.listen(0)
 
     rpc = ConnectionPool(limit=5)
 
     # Reuse connections
-    yield [rpc(ip="127.0.0.1", port=s.port).ping() for s in servers[:5]]
-    yield [rpc(s.address).ping() for s in servers[:5]]
-    yield [rpc("127.0.0.1:%d" % s.port).ping() for s in servers[:5]]
-    yield [rpc(ip="127.0.0.1", port=s.port).ping() for s in servers[:5]]
+    await asyncio.gather(
+        *[rpc(ip="127.0.0.1", port=s.port).ping() for s in servers[:5]]
+    )
+    await asyncio.gather(*[rpc(s.address).ping() for s in servers[:5]])
+    await asyncio.gather(*[rpc("127.0.0.1:%d" % s.port).ping() for s in servers[:5]])
+    await asyncio.gather(
+        *[rpc(ip="127.0.0.1", port=s.port).ping() for s in servers[:5]]
+    )
     assert sum(map(len, rpc.available.values())) == 5
     assert sum(map(len, rpc.occupied.values())) == 0
     assert rpc.active == 0
     assert rpc.open == 5
 
     # Clear out connections to make room for more
-    yield [rpc(ip="127.0.0.1", port=s.port).ping() for s in servers[5:]]
+    await asyncio.gather(
+        *[rpc(ip="127.0.0.1", port=s.port).ping() for s in servers[5:]]
+    )
     assert rpc.active == 0
     assert rpc.open == 5
 
     s = servers[0]
-    yield [rpc(ip="127.0.0.1", port=s.port).ping(delay=0.1) for i in range(3)]
+    await asyncio.gather(
+        *[rpc(ip="127.0.0.1", port=s.port).ping(delay=0.1) for i in range(3)]
+    )
     assert len(rpc.available["tcp://127.0.0.1:%d" % s.port]) == 3
 
     # Explicitly clear out connections
     rpc.collect()
     start = time()
     while any(rpc.available.values()):
-        yield asyncio.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 2
 
     rpc.close()
 
 
-@gen_test()
-def test_connection_pool_respects_limit():
+@pytest.mark.asyncio
+async def test_connection_pool_respects_limit():
 
     limit = 5
 
@@ -567,15 +581,15 @@ async def do_ping(pool, port):
 
     servers = [Server({"ping": ping}) for i in range(10)]
     for server in servers:
-        server.listen(0)
+        await server.listen(0)
 
     pool = ConnectionPool(limit=limit)
 
-    yield [do_ping(pool, s.port) for s in servers]
+    await asyncio.gather(*[do_ping(pool, s.port) for s in servers])
 
 
-@gen_test()
-def test_connection_pool_tls():
+@pytest.mark.asyncio
+async def test_connection_pool_tls():
     """
     Make sure connection args are supported.
     """
@@ -589,33 +603,33 @@ async def ping(comm, delay=0.01):
 
     servers = [Server({"ping": ping}) for i in range(10)]
     for server in servers:
-        server.listen("tls://", listen_args=listen_args)
+        await server.listen("tls://", listen_args=listen_args)
 
     rpc = ConnectionPool(limit=5, connection_args=connection_args)
 
-    yield [rpc(s.address).ping() for s in servers[:5]]
-    yield [rpc(s.address).ping() for s in servers[::2]]
-    yield [rpc(s.address).ping() for s in servers]
+    await asyncio.gather(*[rpc(s.address).ping() for s in servers[:5]])
+    await asyncio.gather(*[rpc(s.address).ping() for s in servers[::2]])
+    await asyncio.gather(*[rpc(s.address).ping() for s in servers])
     assert rpc.active == 0
 
     rpc.close()
 
 
-@gen_test()
-def test_connection_pool_remove():
+@pytest.mark.asyncio
+async def test_connection_pool_remove():
     async def ping(comm, delay=0.01):
         await asyncio.sleep(delay)
         return "pong"
 
     servers = [Server({"ping": ping}) for i in range(5)]
     for server in servers:
-        server.listen(0)
+        await server.listen(0)
 
     rpc = ConnectionPool(limit=10)
     serv = servers.pop()
-    yield [rpc(s.address).ping() for s in servers]
-    yield [rpc(serv.address).ping() for i in range(3)]
-    yield rpc.connect(serv.address)
+    await asyncio.gather(*[rpc(s.address).ping() for s in servers])
+    await asyncio.gather(*[rpc(serv.address).ping() for i in range(3)])
+    await rpc.connect(serv.address)
     assert sum(map(len, rpc.available.values())) == 6
     assert sum(map(len, rpc.occupied.values())) == 1
     assert rpc.active == 1
@@ -633,39 +647,39 @@ async def ping(comm, delay=0.01):
 
     # this pattern of calls (esp. `reuse` after `remove`)
     # can happen in case of worker failures:
-    comm = yield rpc.connect(serv.address)
+    comm = await rpc.connect(serv.address)
     rpc.remove(serv.address)
     rpc.reuse(serv.address, comm)
 
     rpc.close()
 
 
-@gen_test()
-def test_counters():
+@pytest.mark.asyncio
+async def test_counters():
     server = Server({"div": stream_div})
-    server.listen("tcp://")
+    await server.listen("tcp://")
 
-    with rpc(server.address) as r:
+    async with rpc(server.address) as r:
         for i in range(2):
-            yield r.identity()
+            await r.identity()
         with pytest.raises(ZeroDivisionError):
-            yield r.div(x=1, y=0)
+            await r.div(x=1, y=0)
 
         c = server.counters
         assert c["op"].components[0] == {"identity": 2, "div": 1}
 
 
 @gen_cluster()
-def test_ticks(s, a, b):
+async def test_ticks(s, a, b):
     pytest.importorskip("crick")
-    yield asyncio.sleep(0.1)
+    await asyncio.sleep(0.1)
     c = s.digests["tick-duration"]
     assert c.size()
     assert 0.01 < c.components[0].quantile(0.5) < 0.5
 
 
 @gen_cluster()
-def test_tick_logging(s, a, b):
+async def test_tick_logging(s, a, b):
     pytest.importorskip("crick")
     from distributed import core
 
@@ -673,7 +687,7 @@ def test_tick_logging(s, a, b):
     core.tick_maximum_delay = 0.001
     try:
         with captured_logger("distributed.core") as sio:
-            yield asyncio.sleep(0.1)
+            await asyncio.sleep(0.1)
 
         text = sio.getvalue()
         assert "unresponsive" in text
@@ -689,7 +703,7 @@ def test_compression(compression, serialize, loop):
 
         async def f():
             server = Server({"echo": serialize})
-            server.listen("tcp://")
+            await server.listen("tcp://")
 
             with rpc(server.address) as r:
                 data = b"1" * 1000000
@@ -704,7 +718,7 @@ async def f():
 def test_rpc_serialization(loop):
     async def f():
         server = Server({"echo": echo_serialize})
-        server.listen("tcp://")
+        await server.listen("tcp://")
 
         async with rpc(server.address, serializers=["msgpack"]) as r:
             with pytest.raises(TypeError):
@@ -724,14 +738,14 @@ def test_thread_id(s, a, b):
     assert s.thread_id == a.thread_id == b.thread_id == threading.get_ident()
 
 
-@gen_test()
-def test_deserialize_error():
+@pytest.mark.asyncio
+async def test_deserialize_error():
     server = Server({"throws": throws})
-    server.listen(0)
+    await server.listen(0)
 
-    comm = yield connect(server.address, deserialize=False)
+    comm = await connect(server.address, deserialize=False)
     with pytest.raises(Exception) as info:
-        yield send_recv(comm, op="throws")
+        await send_recv(comm, op="throws")
 
     assert type(info.value) == Exception
     for c in str(info.value):
diff --git a/distributed/tests/test_security.py b/distributed/tests/test_security.py
index 167abc762ae..8e9007f20f0 100644
--- a/distributed/tests/test_security.py
+++ b/distributed/tests/test_security.py
@@ -7,11 +7,10 @@
     ssl = None
 
 import pytest
-from tornado import gen
 
 from distributed.comm import connect, listen
 from distributed.security import Security
-from distributed.utils_test import get_cert, gen_test
+from distributed.utils_test import get_cert
 
 import dask
 
@@ -256,18 +255,17 @@ def many_ciphers(ctx):
             assert len(tls_13_ciphers) == 3
 
 
-@gen_test()
-def test_tls_listen_connect():
+@pytest.mark.asyncio
+async def test_tls_listen_connect():
     """
     Functional test for TLS connection args.
     """
 
-    @gen.coroutine
-    def handle_comm(comm):
+    async def handle_comm(comm):
         peer_addr = comm.peer_address
         assert peer_addr.startswith("tls://")
-        yield comm.write("hello")
-        yield comm.close()
+        await comm.write("hello")
+        await comm.close()
 
     c = {
         "distributed.comm.tls.ca-file": ca_file,
@@ -282,25 +280,25 @@ def handle_comm(comm):
     with dask.config.set(c):
         forced_cipher_sec = Security()
 
-    with listen(
+    async with listen(
         "tls://", handle_comm, connection_args=sec.get_listen_args("scheduler")
     ) as listener:
-        comm = yield connect(
+        comm = await connect(
             listener.contact_address, connection_args=sec.get_connection_args("worker")
         )
-        msg = yield comm.read()
+        msg = await comm.read()
         assert msg == "hello"
         comm.abort()
 
         # No SSL context for client
         with pytest.raises(TypeError):
-            yield connect(
+            await connect(
                 listener.contact_address,
                 connection_args=sec.get_connection_args("client"),
             )
 
         # Check forced cipher
-        comm = yield connect(
+        comm = await connect(
             listener.contact_address,
             connection_args=forced_cipher_sec.get_connection_args("worker"),
         )
@@ -309,14 +307,13 @@ def handle_comm(comm):
         comm.abort()
 
 
-@gen_test()
-def test_require_encryption():
+@pytest.mark.asyncio
+async def test_require_encryption():
     """
     Functional test for "require_encryption" setting.
     """
 
-    @gen.coroutine
-    def handle_comm(comm):
+    async def handle_comm(comm):
         comm.abort()
 
     c = {
@@ -333,19 +330,19 @@ def handle_comm(comm):
         sec2 = Security()
 
     for listen_addr in ["inproc://", "tls://"]:
-        with listen(
+        async with listen(
             listen_addr, handle_comm, connection_args=sec.get_listen_args("scheduler")
         ) as listener:
-            comm = yield connect(
+            comm = await connect(
                 listener.contact_address,
                 connection_args=sec2.get_connection_args("worker"),
             )
             comm.abort()
 
-        with listen(
+        async with listen(
             listen_addr, handle_comm, connection_args=sec2.get_listen_args("scheduler")
         ) as listener:
-            comm = yield connect(
+            comm = await connect(
                 listener.contact_address,
                 connection_args=sec2.get_connection_args("worker"),
             )
@@ -358,17 +355,17 @@ def check_encryption_error():
         assert "encryption required" in str(excinfo.value)
 
     for listen_addr in ["tcp://"]:
-        with listen(
+        async with listen(
             listen_addr, handle_comm, connection_args=sec.get_listen_args("scheduler")
         ) as listener:
-            comm = yield connect(
+            comm = await connect(
                 listener.contact_address,
                 connection_args=sec.get_connection_args("worker"),
             )
             comm.abort()
 
             with pytest.raises(RuntimeError):
-                yield connect(
+                await connect(
                     listener.contact_address,
                     connection_args=sec2.get_connection_args("worker"),
                 )
@@ -396,25 +393,24 @@ def test_temporary_credentials():
         assert val not in sec_repr
 
 
-@gen_test()
-def test_tls_temporary_credentials_functional():
+@pytest.mark.asyncio
+async def test_tls_temporary_credentials_functional():
     pytest.importorskip("cryptography")
 
-    @gen.coroutine
-    def handle_comm(comm):
+    async def handle_comm(comm):
         peer_addr = comm.peer_address
         assert peer_addr.startswith("tls://")
-        yield comm.write("hello")
-        yield comm.close()
+        await comm.write("hello")
+        await comm.close()
 
     sec = Security.temporary()
 
-    with listen(
+    async with listen(
         "tls://", handle_comm, connection_args=sec.get_listen_args("scheduler")
     ) as listener:
-        comm = yield connect(
+        comm = await connect(
             listener.contact_address, connection_args=sec.get_connection_args("worker")
         )
-        msg = yield comm.read()
+        msg = await comm.read()
         assert msg == "hello"
         comm.abort()
diff --git a/distributed/worker.py b/distributed/worker.py
index 1eb8252534d..b863e895f7d 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -986,7 +986,7 @@ async def start(self):
         enable_gc_diagnosis()
         thread_state.on_event_loop_thread = True
 
-        self.listen(self._start_address, listen_args=self.listen_args)
+        await self.listen(self._start_address, listen_args=self.listen_args)
         self.ip = get_address_host(self.address)
 
         if self.name is None:

From 59ad42ed24fd1a0a010eb6c4620cf9c48c209f44 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Wed, 27 Nov 2019 20:04:00 -0600
Subject: [PATCH 0578/1550] Update function serialization caches with custom
 LRU class (#3260)

---
 distributed/tests/test_utils.py | 19 +++++++++++++++++++
 distributed/utils.py            | 22 +++++++++++++++++++++-
 distributed/worker.py           | 33 ++++++++++++++++++---------------
 3 files changed, 58 insertions(+), 16 deletions(-)

diff --git a/distributed/tests/test_utils.py b/distributed/tests/test_utils.py
index ff733a1ad8c..ff2e42313ac 100644
--- a/distributed/tests/test_utils.py
+++ b/distributed/tests/test_utils.py
@@ -42,6 +42,7 @@
     parse_timedelta,
     warn_on_duration,
     format_dashboard_link,
+    LRU,
 )
 from distributed.utils_test import loop, loop_in_thread  # noqa: F401
 from distributed.utils_test import div, has_ipv6, inc, throws, gen_test, captured_logger
@@ -598,3 +599,21 @@ def test_format_dashboard_link():
         assert "hello" not in format_dashboard_link("host", 1234)
     finally:
         del os.environ["host"]
+
+
+def test_lru():
+
+    l = LRU(maxsize=3)
+    l["a"] = 1
+    l["b"] = 2
+    l["c"] = 3
+    assert list(l.keys()) == ["a", "b", "c"]
+
+    # Use "a" and ensure it becomes the most recently used item
+    l["a"]
+    assert list(l.keys()) == ["b", "c", "a"]
+
+    # Ensure maxsize is respected
+    l["d"] = 4
+    assert len(l) == 3
+    assert list(l.keys()) == ["c", "a", "d"]
diff --git a/distributed/utils.py b/distributed/utils.py
index 251e1110be8..978be4eae8a 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -1,6 +1,6 @@
 import asyncio
 import atexit
-from collections import deque
+from collections import deque, OrderedDict, UserDict
 from concurrent.futures import ThreadPoolExecutor
 from contextlib import contextmanager
 from datetime import timedelta
@@ -1399,3 +1399,23 @@ def deserialize_for_cli(data):
         The de-serialized data
     """
     return json.loads(base64.urlsafe_b64decode(data.encode()).decode())
+
+
+class LRU(UserDict):
+    """ Limited size mapping, evicting the least recently looked-up key when full
+    """
+
+    def __init__(self, maxsize):
+        super().__init__()
+        self.data = OrderedDict()
+        self.maxsize = maxsize
+
+    def __getitem__(self, key):
+        value = super().__getitem__(key)
+        self.data.move_to_end(key)
+        return value
+
+    def __setitem__(self, key, value):
+        if len(self) >= self.maxsize:
+            self.data.popitem(last=False)
+        super().__setitem__(key, value)
diff --git a/distributed/worker.py b/distributed/worker.py
index b863e895f7d..90059002e9a 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -61,6 +61,7 @@
     parse_timedelta,
     iscoroutinefunction,
     warn_on_duration,
+    LRU,
 )
 from .utils_comm import pack_data, gather_from_workers
 from .utils_perf import ThrottledGC, enable_gc_diagnosis, disable_gc_diagnosis
@@ -3176,18 +3177,26 @@ async def get_data_from_worker(
 
 job_counter = [0]
 
-import functools
 
+cache_loads = LRU(maxsize=100)
 
-@functools.lru_cache(100)
-def cached_function_deserialization(func):
-    return pickle.loads(func)
+
+def loads_function(bytes_object):
+    """ Load a function from bytes, cache bytes """
+    if len(bytes_object) < 100000:
+        try:
+            result = cache_loads[bytes_object]
+        except KeyError:
+            result = pickle.loads(bytes_object)
+            cache_loads[bytes_object] = result
+        return result
+    return pickle.loads(bytes_object)
 
 
 def _deserialize(function=None, args=None, kwargs=None, task=no_value):
     """ Deserialize task inputs and regularize to func, args, kwargs """
     if function is not None:
-        function = cached_function_deserialization(function)
+        function = loads_function(function)
     if args:
         args = pickle.loads(args)
     if kwargs:
@@ -3219,24 +3228,18 @@ def execute_task(task):
         return task
 
 
-try:
-    # a 10 MB cache of deserialized functions and their bytes
-    from zict import LRU
-
-    cache = LRU(10000000, dict(), weight=lambda k, v: len(v))
-except ImportError:
-    cache = dict()
+cache_dumps = LRU(maxsize=100)
 
 
 def dumps_function(func):
     """ Dump a function to bytes, cache functions """
     try:
-        result = cache[func]
+        result = cache_dumps[func]
     except KeyError:
         result = pickle.dumps(func)
         if len(result) < 100000:
-            cache[func] = result
-    except TypeError:
+            cache_dumps[func] = result
+    except TypeError:  # Unhashable function
         result = pickle.dumps(func)
     return result
 

From e3731a6adbf70e92852cd9b71075d5607185cd9b Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 28 Nov 2019 13:42:30 -0800
Subject: [PATCH 0579/1550] Add performance_report context manager for static
 report generation (#3282)

This generates a static HTML file with many of the same plots as the dashboard.

Example
-------

```python
from dask.distributed import Client
client = Client()
import dask.array as da
x = da.random.random((30000, 30000), chunks=(1000, 1000))
from dask.distributed import Client, performance_report
with performance_report():
    x = x.persist()
    (x + x.T).sum().compute()
```
---
 distributed/__init__.py          |  1 +
 distributed/client.py            | 41 ++++++++++++++++--
 distributed/scheduler.py         | 73 ++++++++++++++++++++++++++++++++
 distributed/tests/test_client.py | 17 ++++++++
 4 files changed, 129 insertions(+), 3 deletions(-)

diff --git a/distributed/__init__.py b/distributed/__init__.py
index 1eadee32307..06136dd72a2 100644
--- a/distributed/__init__.py
+++ b/distributed/__init__.py
@@ -16,6 +16,7 @@
     Future,
     futures_of,
     get_task_stream,
+    performance_report,
 )
 from .lock import Lock
 from .nanny import Nanny
diff --git a/distributed/client.py b/distributed/client.py
index 3b2eb6e2863..5ca517ae219 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -3291,9 +3291,6 @@ def profile(
         >>> client.profile()  # call on collections
         >>> client.profile(filename='dask-profile.html')  # save to html file
         """
-        if isinstance(workers, (str, Number)):
-            workers = [workers]
-
         return self.sync(
             self._profile,
             key=key,
@@ -4560,6 +4557,44 @@ async def __aexit__(self, typ, value, traceback):
         self.data.extend(L)
 
 
+class performance_report:
+    """ Gather performance report
+
+    This creates a static HTML file that includes many of the same plots of the
+    dashboard for later viewing.
+
+    The resulting file uses JavaScript, and so must be viewed with a web
+    browser.  Locally we recommend using ``python -m http.server`` or hosting
+    the file live online.
+
+    Examples
+    --------
+    >>> with performance_report(filename="myfile.html"):
+    ...     x.compute()
+
+    $ python -m http.server
+    $ open myfile.html
+    """
+
+    def __init__(self, filename="dask-report.html"):
+        self.filename = filename
+
+    async def __aenter__(self):
+        self.start = time()
+        await get_client().get_task_stream(start=0, stop=0)  # ensure plugin
+
+    async def __aexit__(self, typ, value, traceback):
+        data = await get_client().scheduler.performance_report(start=self.start)
+        with open(self.filename, "w") as f:
+            f.write(data)
+
+    def __enter__(self):
+        get_client().sync(self.__aenter__)
+
+    def __exit__(self, typ, value, traceback):
+        get_client().sync(self.__aexit__, type, value, traceback)
+
+
 @contextmanager
 def temp_default_client(c):
     """ Set the default client for the duration of the context
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index b77c36477c1..bd627f608aa 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -57,6 +57,7 @@
     parse_bytes,
     PeriodicCallback,
     shutting_down,
+    tmpfile,
 )
 from .utils_comm import scatter_to_workers, gather_from_workers
 from .utils_perf import enable_gc_diagnosis, disable_gc_diagnosis
@@ -1073,6 +1074,7 @@ def __init__(
             "processing": self.get_processing,
             "call_stack": self.get_call_stack,
             "profile": self.get_profile,
+            "performance_report": self.performance_report,
             "logs": self.get_logs,
             "worker_logs": self.get_worker_logs,
             "nbytes": self.get_nbytes,
@@ -4699,6 +4701,77 @@ async def get_profile_metadata(
 
         return {"counts": counts, "keys": keys}
 
+    async def performance_report(self, comm=None, start=None):
+        # Profiles
+        compute, scheduler, workers = await asyncio.gather(
+            *[
+                self.get_profile(start=start),
+                self.get_profile(scheduler=True, start=start),
+                self.get_profile(server=True, start=start),
+            ]
+        )
+        from . import profile
+
+        def profile_to_figure(state):
+            data = profile.plot_data(state)
+            figure, source = profile.plot_figure(data, sizing_mode="stretch_both")
+            return figure
+
+        compute, scheduler, workers = map(
+            profile_to_figure, (compute, scheduler, workers)
+        )
+
+        # Task stream
+        task_stream = self.get_task_stream(start=start)
+        from .diagnostics.task_stream import rectangles
+        from .dashboard.components.scheduler import task_stream_figure
+
+        rects = rectangles(task_stream)
+        source, task_stream = task_stream_figure(sizing_mode="stretch_both")
+        source.data.update(rects)
+
+        from distributed.dashboard.components.scheduler import (
+            BandwidthWorkers,
+            BandwidthTypes,
+        )
+
+        bandwidth_workers = BandwidthWorkers(self, sizing_mode="stretch_both")
+        bandwidth_workers.update()
+        bandwidth_types = BandwidthTypes(self, sizing_mode="stretch_both")
+        bandwidth_types.update()
+
+        from bokeh.models import Panel, Tabs
+
+        compute = Panel(child=compute, title="Worker Profile (compute)")
+        workers = Panel(child=workers, title="Worker Profile (administrative)")
+        scheduler = Panel(child=scheduler, title="Scheduler Profile (administrative)")
+        task_stream = Panel(child=task_stream, title="Task Stream")
+        bandwidth_workers = Panel(
+            child=bandwidth_workers.fig, title="Bandwidth (Workers)"
+        )
+        bandwidth_types = Panel(child=bandwidth_types.fig, title="Bandwidth (Types)")
+
+        tabs = Tabs(
+            tabs=[
+                task_stream,
+                compute,
+                workers,
+                scheduler,
+                bandwidth_workers,
+                bandwidth_types,
+            ]
+        )
+
+        from bokeh.plotting import save
+
+        with tmpfile(extension=".html") as fn:
+            save(tabs, filename=fn)
+
+            with open(fn) as f:
+                data = f.read()
+
+        return data
+
     async def get_worker_logs(self, comm=None, n=None, workers=None, nanny=False):
         results = await self.broadcast(
             msg={"op": "get_logs", "n": n}, workers=workers, nanny=nanny
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index e97cb023a48..b8d5be0b449 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -36,6 +36,7 @@
     get_worker,
     Executor,
     profile,
+    performance_report,
     TimeoutError,
 )
 from distributed.comm import CommClosedError
@@ -5703,5 +5704,21 @@ async def test_profile_server(c, s, a, b):
     assert "slowdec" in str(p)
 
 
+@gen_cluster(client=True)
+async def test_performance_report(c, s, a, b):
+    da = pytest.importorskip("dask.array")
+    x = da.random.random((1000, 1000), chunks=(100, 100))
+
+    with tmpfile(extension="html") as fn:
+        async with performance_report(filename=fn):
+            await c.compute((x + x.T).sum())
+
+        with open(fn) as f:
+            data = f.read()
+
+        assert "bokeh" in data
+        assert "random" in data
+
+
 if sys.version_info >= (3, 5):
     from distributed.tests.py3_test_client import *  # noqa F401

From 894910034230649b23547bddf10bc2d550552bae Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 28 Nov 2019 19:04:43 -0800
Subject: [PATCH 0580/1550] xfail test_workspace_concurrency for Python 3.6
 (#3283)

This only seems to fail for this version.
We actually intended to do this before, but the condition was imprecise.
---
 distributed/tests/test_diskutils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/tests/test_diskutils.py b/distributed/tests/test_diskutils.py
index 0057f96fb36..86b472e184a 100644
--- a/distributed/tests/test_diskutils.py
+++ b/distributed/tests/test_diskutils.py
@@ -275,7 +275,7 @@ def _test_workspace_concurrency(tmpdir, timeout, max_procs):
 def test_workspace_concurrency(tmpdir):
     if WINDOWS:
         raise pytest.xfail.Exception("TODO: unknown failure on windows")
-    if sys.version_info <= (3, 6):
+    if sys.version_info < (3, 7):
         raise pytest.xfail.Exception("TODO: unknown failure on Python 3.6")
     _test_workspace_concurrency(tmpdir, 2.0, 6)
 

From 892c371d7ab06b4bd5f39afb0770659d87918b6e Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 29 Nov 2019 16:53:44 -0800
Subject: [PATCH 0581/1550] Move Python 3 syntax tests into main tests (#3281)

Previously these were separated to support Python 2.
Now we can include them in normal tests
---
 distributed/deploy/tests/py3_test_deploy.py |  13 --
 distributed/deploy/tests/test_local.py      |  13 +-
 distributed/tests/py3_test_client.py        | 210 --------------------
 distributed/tests/py3_test_pubsub.py        |  38 ----
 distributed/tests/py3_test_utils_tst.py     |  17 --
 distributed/tests/test_client.py            | 190 +++++++++++++++++-
 distributed/tests/test_locks.py             |  12 +-
 distributed/tests/test_pubsub.py            |  40 +++-
 distributed/tests/test_utils_test.py        |  14 +-
 9 files changed, 251 insertions(+), 296 deletions(-)
 delete mode 100644 distributed/deploy/tests/py3_test_deploy.py
 delete mode 100644 distributed/tests/py3_test_client.py
 delete mode 100644 distributed/tests/py3_test_pubsub.py
 delete mode 100644 distributed/tests/py3_test_utils_tst.py

diff --git a/distributed/deploy/tests/py3_test_deploy.py b/distributed/deploy/tests/py3_test_deploy.py
deleted file mode 100644
index 7a66ecf942c..00000000000
--- a/distributed/deploy/tests/py3_test_deploy.py
+++ /dev/null
@@ -1,13 +0,0 @@
-from distributed import LocalCluster
-from distributed.utils_test import loop  # noqa: F401
-
-import pytest
-
-
-@pytest.mark.asyncio
-async def test_async_with():
-    async with LocalCluster(processes=False, asynchronous=True) as cluster:
-        w = cluster.workers
-        assert w
-
-    assert not w
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index 9b39d8f81f5..0f8eb6d8901 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -898,10 +898,6 @@ class MyNanny(Nanny):
         assert all(isinstance(w, MyNanny) for w in cluster.workers.values())
 
 
-if sys.version_info >= (3, 5):
-    from distributed.deploy.tests.py3_test_deploy import *  # noqa F401
-
-
 def test_starts_up_sync(loop):
     cluster = LocalCluster(
         n_workers=2,
@@ -1019,3 +1015,12 @@ async def test_no_danglng_asyncio_tasks(cleanup):
 
     tasks = asyncio.all_tasks()
     assert tasks == start
+
+
+@pytest.mark.asyncio
+async def test_async_with():
+    async with LocalCluster(processes=False, asynchronous=True) as cluster:
+        w = cluster.workers
+        assert w
+
+    assert not w
diff --git a/distributed/tests/py3_test_client.py b/distributed/tests/py3_test_client.py
deleted file mode 100644
index b5d10f8d553..00000000000
--- a/distributed/tests/py3_test_client.py
+++ /dev/null
@@ -1,210 +0,0 @@
-import gc
-import sys
-from time import sleep
-import weakref
-
-import pytest
-from tornado import gen
-
-from distributed.utils_test import div, gen_cluster, inc, loop, cluster  # noqa F401
-from distributed import as_completed, Client, Lock
-from distributed.metrics import time
-from distributed.utils import sync
-
-
-@gen_cluster(client=True)
-def test_await_future(c, s, a, b):
-    future = c.submit(inc, 1)
-
-    async def f():  # flake8: noqa
-        result = await future
-        assert result == 2
-
-    yield f()
-
-    future = c.submit(div, 1, 0)
-
-    async def f():
-        with pytest.raises(ZeroDivisionError):
-            await future
-
-    yield f()
-
-
-@gen_cluster(client=True)
-def test_as_completed_async_for(c, s, a, b):
-    futures = c.map(inc, range(10))
-    ac = as_completed(futures)
-    results = []
-
-    async def f():
-        async for future in ac:
-            result = await future
-            results.append(result)
-
-    yield f()
-
-    assert set(results) == set(range(1, 11))
-
-
-@gen_cluster(client=True)
-def test_as_completed_async_for_results(c, s, a, b):
-    futures = c.map(inc, range(10))
-    ac = as_completed(futures, with_results=True)
-    results = []
-
-    async def f():
-        async for future, result in ac:
-            results.append(result)
-
-    yield f()
-
-    assert set(results) == set(range(1, 11))
-    assert not s.counters["op"].components[0]["gather"]
-
-
-@gen_cluster(client=True)
-def test_as_completed_async_for_cancel(c, s, a, b):
-    x = c.submit(inc, 1)
-    y = c.submit(sleep, 0.3)
-    ac = as_completed([x, y])
-
-    async def _():
-        await gen.sleep(0.1)
-        await y.cancel(asynchronous=True)
-
-    c.loop.add_callback(_)
-
-    L = []
-
-    async def f():
-        async for future in ac:
-            L.append(future)
-
-    yield f()
-
-    assert L == [x, y]
-
-
-def test_async_with(loop):
-    result = None
-    client = None
-    cluster = None
-
-    async def f():
-        async with Client(processes=False, asynchronous=True) as c:
-            nonlocal result, client, cluster
-            result = await c.submit(lambda x: x + 1, 10)
-
-            client = c
-            cluster = c.cluster
-
-    loop.run_sync(f)
-
-    assert result == 11
-    assert client.status == "closed"
-    assert cluster.status == "closed"
-
-
-def test_locks(loop):
-    async def f():
-        async with Client(processes=False, asynchronous=True) as c:
-            assert c.asynchronous
-            async with Lock("x"):
-                lock2 = Lock("x")
-                result = await lock2.acquire(timeout=0.1)
-                assert result is False
-
-    loop.run_sync(f)
-
-
-def test_client_sync_with_async_def(loop):
-    async def ff():
-        await gen.sleep(0.01)
-        return 1
-
-    with cluster() as (s, [a, b]):
-        with Client(s["address"], loop=loop) as c:
-            assert sync(loop, ff) == 1
-            assert c.sync(ff) == 1
-
-
-@pytest.mark.xfail(reason="known intermittent failure")
-@gen_cluster(client=True)
-async def test_dont_hold_on_to_large_messages(c, s, a, b):
-    np = pytest.importorskip("numpy")
-    da = pytest.importorskip("dask.array")
-    x = np.random.random(1000000)
-    xr = weakref.ref(x)
-
-    d = da.from_array(x, chunks=(100000,))
-    d = d.persist()
-    del x
-
-    start = time()
-    while xr() is not None:
-        if time() > start + 5:
-            # Help diagnosing
-            from types import FrameType
-
-            x = xr()
-            if x is not None:
-                del x
-                rc = sys.getrefcount(xr())
-                refs = gc.get_referrers(xr())
-                print("refs to x:", rc, refs, gc.isenabled())
-                frames = [r for r in refs if isinstance(r, FrameType)]
-                for i, f in enumerate(frames):
-                    print(
-                        "frames #%d:" % i,
-                        f.f_code.co_name,
-                        f.f_code.co_filename,
-                        sorted(f.f_locals),
-                    )
-            pytest.fail("array should have been destroyed")
-
-        await gen.sleep(0.200)
-
-
-@gen_cluster(client=True)
-async def test_run_scheduler_async_def(c, s, a, b):
-    async def f(dask_scheduler):
-        await gen.sleep(0.01)
-        dask_scheduler.foo = "bar"
-
-    await c.run_on_scheduler(f)
-
-    assert s.foo == "bar"
-
-    async def f(dask_worker):
-        await gen.sleep(0.01)
-        dask_worker.foo = "bar"
-
-    await c.run(f)
-    assert a.foo == "bar"
-    assert b.foo == "bar"
-
-
-@gen_cluster(client=True)
-async def test_run_scheduler_async_def_wait(c, s, a, b):
-    async def f(dask_scheduler):
-        await gen.sleep(0.01)
-        dask_scheduler.foo = "bar"
-
-    await c.run_on_scheduler(f, wait=False)
-
-    while not hasattr(s, "foo"):
-        await gen.sleep(0.01)
-    assert s.foo == "bar"
-
-    async def f(dask_worker):
-        await gen.sleep(0.01)
-        dask_worker.foo = "bar"
-
-    await c.run(f, wait=False)
-
-    while not hasattr(a, "foo") or not hasattr(b, "foo"):
-        await gen.sleep(0.01)
-
-    assert a.foo == "bar"
-    assert b.foo == "bar"
diff --git a/distributed/tests/py3_test_pubsub.py b/distributed/tests/py3_test_pubsub.py
deleted file mode 100644
index 294ecfb90c8..00000000000
--- a/distributed/tests/py3_test_pubsub.py
+++ /dev/null
@@ -1,38 +0,0 @@
-from distributed import Pub, Sub
-from distributed.utils_test import gen_cluster
-
-import asyncio
-import toolz
-from tornado import gen
-import pytest
-
-
-@pytest.mark.xfail(reason="out of order execution")
-@gen_cluster(client=True)
-def test_basic(c, s, a, b):
-    async def publish():
-        pub = Pub("a")
-
-        i = 0
-        while True:
-            await gen.sleep(0.01)
-            pub._put(i)
-            i += 1
-
-    def f(_):
-        sub = Sub("a")
-        return list(toolz.take(5, sub))
-
-    asyncio.ensure_future(c.run(publish, workers=[a.address]))
-
-    tasks = [c.submit(f, i) for i in range(4)]
-    results = yield c.gather(tasks)
-
-    for r in results:
-        x = r[0]
-        # race conditions and unintended (but correct) messages
-        # can make this test not true
-        # assert r == [x, x + 1, x + 2, x + 3, x + 4]
-
-        assert len(r) == 5
-        assert all(r[i] < r[i + 1] for i in range(0, 4)), r
diff --git a/distributed/tests/py3_test_utils_tst.py b/distributed/tests/py3_test_utils_tst.py
deleted file mode 100644
index a4b1e242481..00000000000
--- a/distributed/tests/py3_test_utils_tst.py
+++ /dev/null
@@ -1,17 +0,0 @@
-from distributed.utils_test import gen_cluster, gen_test
-from distributed import Client
-
-from tornado import gen
-
-
-@gen_cluster()
-async def test_gen_cluster_async(s, a, b):  # flake8: noqa
-    async with Client(s.address, asynchronous=True) as c:
-        future = c.submit(lambda x: x + 1, 1)
-        result = await future
-        assert result == 2
-
-
-@gen_test()
-async def test_gen_test_async():  # flake8: noqa
-    await gen.sleep(0.001)
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index b8d5be0b449..02e2574a3ab 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -5704,6 +5704,192 @@ async def test_profile_server(c, s, a, b):
     assert "slowdec" in str(p)
 
 
+@gen_cluster(client=True)
+def test_await_future(c, s, a, b):
+    future = c.submit(inc, 1)
+
+    async def f():  # flake8: noqa
+        result = await future
+        assert result == 2
+
+    yield f()
+
+    future = c.submit(div, 1, 0)
+
+    async def f():
+        with pytest.raises(ZeroDivisionError):
+            await future
+
+    yield f()
+
+
+@gen_cluster(client=True)
+def test_as_completed_async_for(c, s, a, b):
+    futures = c.map(inc, range(10))
+    ac = as_completed(futures)
+    results = []
+
+    async def f():
+        async for future in ac:
+            result = await future
+            results.append(result)
+
+    yield f()
+
+    assert set(results) == set(range(1, 11))
+
+
+@gen_cluster(client=True)
+def test_as_completed_async_for_results(c, s, a, b):
+    futures = c.map(inc, range(10))
+    ac = as_completed(futures, with_results=True)
+    results = []
+
+    async def f():
+        async for future, result in ac:
+            results.append(result)
+
+    yield f()
+
+    assert set(results) == set(range(1, 11))
+    assert not s.counters["op"].components[0]["gather"]
+
+
+@gen_cluster(client=True)
+def test_as_completed_async_for_cancel(c, s, a, b):
+    x = c.submit(inc, 1)
+    y = c.submit(sleep, 0.3)
+    ac = as_completed([x, y])
+
+    async def _():
+        await gen.sleep(0.1)
+        await y.cancel(asynchronous=True)
+
+    c.loop.add_callback(_)
+
+    L = []
+
+    async def f():
+        async for future in ac:
+            L.append(future)
+
+    yield f()
+
+    assert L == [x, y]
+
+
+def test_async_with(loop):
+    result = None
+    client = None
+    cluster = None
+
+    async def f():
+        async with Client(processes=False, asynchronous=True) as c:
+            nonlocal result, client, cluster
+            result = await c.submit(lambda x: x + 1, 10)
+
+            client = c
+            cluster = c.cluster
+
+    loop.run_sync(f)
+
+    assert result == 11
+    assert client.status == "closed"
+    assert cluster.status == "closed"
+
+
+def test_client_sync_with_async_def(loop):
+    async def ff():
+        await gen.sleep(0.01)
+        return 1
+
+    with cluster() as (s, [a, b]):
+        with Client(s["address"], loop=loop) as c:
+            assert sync(loop, ff) == 1
+            assert c.sync(ff) == 1
+
+
+@pytest.mark.xfail(reason="known intermittent failure")
+@gen_cluster(client=True)
+async def test_dont_hold_on_to_large_messages(c, s, a, b):
+    np = pytest.importorskip("numpy")
+    da = pytest.importorskip("dask.array")
+    x = np.random.random(1000000)
+    xr = weakref.ref(x)
+
+    d = da.from_array(x, chunks=(100000,))
+    d = d.persist()
+    del x
+
+    start = time()
+    while xr() is not None:
+        if time() > start + 5:
+            # Help diagnosing
+            from types import FrameType
+
+            x = xr()
+            if x is not None:
+                del x
+                rc = sys.getrefcount(xr())
+                refs = gc.get_referrers(xr())
+                print("refs to x:", rc, refs, gc.isenabled())
+                frames = [r for r in refs if isinstance(r, FrameType)]
+                for i, f in enumerate(frames):
+                    print(
+                        "frames #%d:" % i,
+                        f.f_code.co_name,
+                        f.f_code.co_filename,
+                        sorted(f.f_locals),
+                    )
+            pytest.fail("array should have been destroyed")
+
+        await gen.sleep(0.200)
+
+
+@gen_cluster(client=True)
+async def test_run_scheduler_async_def(c, s, a, b):
+    async def f(dask_scheduler):
+        await gen.sleep(0.01)
+        dask_scheduler.foo = "bar"
+
+    await c.run_on_scheduler(f)
+
+    assert s.foo == "bar"
+
+    async def f(dask_worker):
+        await gen.sleep(0.01)
+        dask_worker.foo = "bar"
+
+    await c.run(f)
+    assert a.foo == "bar"
+    assert b.foo == "bar"
+
+
+@gen_cluster(client=True)
+async def test_run_scheduler_async_def_wait(c, s, a, b):
+    async def f(dask_scheduler):
+        await gen.sleep(0.01)
+        dask_scheduler.foo = "bar"
+
+    await c.run_on_scheduler(f, wait=False)
+
+    while not hasattr(s, "foo"):
+        await gen.sleep(0.01)
+    assert s.foo == "bar"
+
+    async def f(dask_worker):
+        await gen.sleep(0.01)
+        dask_worker.foo = "bar"
+
+    await c.run(f, wait=False)
+
+    while not hasattr(a, "foo") or not hasattr(b, "foo"):
+        await gen.sleep(0.01)
+
+    assert a.foo == "bar"
+    assert b.foo == "bar"
+
+
 @gen_cluster(client=True)
 async def test_performance_report(c, s, a, b):
     da = pytest.importorskip("dask.array")
@@ -5718,7 +5904,3 @@ async def test_performance_report(c, s, a, b):
 
         assert "bokeh" in data
         assert "random" in data
-
-
-if sys.version_info >= (3, 5):
-    from distributed.tests.py3_test_client import *  # noqa F401
diff --git a/distributed/tests/test_locks.py b/distributed/tests/test_locks.py
index 521a9b46114..4cf756ef178 100644
--- a/distributed/tests/test_locks.py
+++ b/distributed/tests/test_locks.py
@@ -3,7 +3,7 @@
 
 import pytest
 
-from distributed import Lock, get_client
+from distributed import Lock, get_client, Client
 from distributed.metrics import time
 from distributed.utils_test import gen_cluster
 from distributed.utils_test import client, cluster_fixture, loop  # noqa F401
@@ -128,3 +128,13 @@ def f(x, lock=None):
     lock2 = pickle.loads(pickle.dumps(lock))
     assert lock2.name == lock.name
     assert lock2.client is lock.client
+
+
+@pytest.mark.asyncio
+async def test_locks():
+    async with Client(processes=False, asynchronous=True) as c:
+        assert c.asynchronous
+        async with Lock("x"):
+            lock2 = Lock("x")
+            result = await lock2.acquire(timeout=0.1)
+            assert result is False
diff --git a/distributed/tests/test_pubsub.py b/distributed/tests/test_pubsub.py
index 555afb71a73..847b0b88bf0 100644
--- a/distributed/tests/test_pubsub.py
+++ b/distributed/tests/test_pubsub.py
@@ -1,13 +1,14 @@
-import sys
+import asyncio
 from time import sleep
 
+import pytest
+from tornado import gen
+import toolz
+
 from distributed import Pub, Sub, wait, get_worker, TimeoutError
 from distributed.utils_test import gen_cluster
 from distributed.metrics import time
 
-import pytest
-from tornado import gen
-
 
 @gen_cluster(client=True, timeout=None)
 def test_speed(c, s, a, b):
@@ -135,5 +136,32 @@ async def test_repr(c, s, a, b):
     assert "Sub" in str(sub)
 
 
-if sys.version_info >= (3, 5):
-    from distributed.tests.py3_test_pubsub import *  # noqa: F401, F403
+@pytest.mark.xfail(reason="out of order execution")
+@gen_cluster(client=True)
+def test_basic(c, s, a, b):
+    async def publish():
+        pub = Pub("a")
+
+        i = 0
+        while True:
+            await gen.sleep(0.01)
+            pub._put(i)
+            i += 1
+
+    def f(_):
+        sub = Sub("a")
+        return list(toolz.take(5, sub))
+
+    asyncio.ensure_future(c.run(publish, workers=[a.address]))
+
+    tasks = [c.submit(f, i) for i in range(4)]
+    results = yield c.gather(tasks)
+
+    for r in results:
+        x = r[0]
+        # race conditions and unintended (but correct) messages
+        # can make this test not true
+        # assert r == [x, x + 1, x + 2, x + 3, x + 4]
+
+        assert len(r) == 5
+        assert all(r[i] < r[i + 1] for i in range(0, 4)), r
diff --git a/distributed/tests/test_utils_test.py b/distributed/tests/test_utils_test.py
index 1c6802b5637..6a4a5ceaa5e 100755
--- a/distributed/tests/test_utils_test.py
+++ b/distributed/tests/test_utils_test.py
@@ -1,6 +1,5 @@
 from contextlib import contextmanager
 import socket
-import sys
 import threading
 from time import sleep
 
@@ -180,5 +179,14 @@ async def test_tls_scheduler(security, cleanup):
         assert s.address.startswith("tls")
 
 
-if sys.version_info >= (3, 5):
-    from distributed.tests.py3_test_utils_tst import *  # noqa: F401, F403
+@gen_cluster()
+async def test_gen_cluster_async(s, a, b):  # flake8: noqa
+    async with Client(s.address, asynchronous=True) as c:
+        future = c.submit(lambda x: x + 1, 1)
+        result = await future
+        assert result == 2
+
+
+@gen_test()
+async def test_gen_test_async():  # flake8: noqa
+    await gen.sleep(0.001)

From f15abc58718fa4ebb457cbbe76f29765bb7b7bd9 Mon Sep 17 00:00:00 2001
From: Tom Rochette <roctom@gmail.com>
Date: Sat, 30 Nov 2019 10:59:21 -0500
Subject: [PATCH 0582/1550] Fix distributed.wait documentation (#3289)

Properly display the return documentation.

Add documentation for the return_when parameter.
---
 distributed/client.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/distributed/client.py b/distributed/client.py
index 5ca517ae219..02337bb525f 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -4102,6 +4102,10 @@ def wait(fs, timeout=None, return_when=ALL_COMPLETED):
     fs: list of futures
     timeout: number, optional
         Time in seconds after which to raise a ``dask.distributed.TimeoutError``
+    return_when: str, optional
+        One of `ALL_COMPLETED` or `FIRST_COMPLETED`
+
+    Returns
     -------
     Named tuple of completed, not completed
     """

From 67007ba9f8f4a19e5bd520291a7756e7c1681a37 Mon Sep 17 00:00:00 2001
From: Benjamin Zaitlen <quasiben@users.noreply.github.com>
Date: Mon, 2 Dec 2019 11:32:50 -0500
Subject: [PATCH 0583/1550] xfail ucx empty object typed dataframe (#3279)

---
 distributed/comm/tests/test_ucx.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/distributed/comm/tests/test_ucx.py b/distributed/comm/tests/test_ucx.py
index a9207e72e7a..42b5275f5b0 100644
--- a/distributed/comm/tests/test_ucx.py
+++ b/distributed/comm/tests/test_ucx.py
@@ -170,7 +170,10 @@ def test_ucx_deserialize():
         lambda cudf: cudf.DataFrame([1]).head(0),
         lambda cudf: cudf.DataFrame([1.0]).head(0),
         lambda cudf: cudf.DataFrame({"a": []}),
-        lambda cudf: cudf.DataFrame({"a": ["a"]}).head(0),
+        pytest.param(
+            lambda cudf: cudf.DataFrame({"a": ["a"]}).head(0),
+            marks=pytest.mark.xfail(reason="0 length objects don't deseralize cleanly"),
+        ),
         lambda cudf: cudf.DataFrame({"a": [1.0]}).head(0),
         lambda cudf: cudf.DataFrame({"a": [1]}).head(0),
         lambda cudf: cudf.DataFrame({"a": [1, 2, None], "b": [1.0, 2.0, None]}),

From a6b8356918bfa9f832ebf858edee2d58b03392a2 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Mon, 2 Dec 2019 15:29:07 -0600
Subject: [PATCH 0584/1550] Avoid repeatedly adding deps to already in memory
 stack (#3293)

---
 distributed/scheduler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index bd627f608aa..7e799715ed4 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1608,7 +1608,7 @@ def update_graph(
                     else:
                         child_deps = self.dependencies[dep]
                     if all(d in done for d in child_deps):
-                        if dep in self.tasks:
+                        if dep in self.tasks and dep not in done:
                             done.add(dep)
                             stack.append(dep)
 

From e0f075eeedf0b9245ecd66702af4a35bce167f12 Mon Sep 17 00:00:00 2001
From: Benjamin Zaitlen <quasiben@users.noreply.github.com>
Date: Mon, 2 Dec 2019 18:40:52 -0500
Subject: [PATCH 0585/1550] Fix asynchronous listener in UCX (#3292)

---
 distributed/comm/tests/test_ucx.py | 26 ++++++++++++++++++--------
 1 file changed, 18 insertions(+), 8 deletions(-)

diff --git a/distributed/comm/tests/test_ucx.py b/distributed/comm/tests/test_ucx.py
index 42b5275f5b0..6fc892176a9 100644
--- a/distributed/comm/tests/test_ucx.py
+++ b/distributed/comm/tests/test_ucx.py
@@ -36,12 +36,12 @@ async def handle_comm(comm):
         await q.put(comm)
 
     listener = listen(listen_addr, handle_comm, connection_args=listen_args, **kwargs)
-    with listener:
+    async with listener:
         comm = await connect(
             listener.contact_address, connection_args=connect_args, **kwargs
         )
-        serv_com = await q.get()
-        return comm, serv_com
+        serv_comm = await q.get()
+        return (comm, serv_comm)
 
 
 @pytest.mark.asyncio
@@ -97,7 +97,7 @@ async def handle_comm(comm):
             assert comm.closed
 
         listener = ucx.UCXListener(address, handle_comm)
-        listener.start()
+        await listener.start()
         host, port = listener.get_host_port()
         assert host.count(".") == 3
         assert port > 0
@@ -174,10 +174,19 @@ def test_ucx_deserialize():
             lambda cudf: cudf.DataFrame({"a": ["a"]}).head(0),
             marks=pytest.mark.xfail(reason="0 length objects don't deseralize cleanly"),
         ),
-        lambda cudf: cudf.DataFrame({"a": [1.0]}).head(0),
-        lambda cudf: cudf.DataFrame({"a": [1]}).head(0),
+        pytest.param(
+            lambda cudf: cudf.DataFrame({"a": [1.0]}).head(0),
+            marks=pytest.mark.xfail(reason="0 length objects don't deseralize cleanly"),
+        ),
+        pytest.param(
+            lambda cudf: cudf.DataFrame({"a": [1]}).head(0),
+            marks=pytest.mark.xfail(reason="0 length objects don't deseralize cleanly"),
+        ),
         lambda cudf: cudf.DataFrame({"a": [1, 2, None], "b": [1.0, 2.0, None]}),
-        lambda cudf: cudf.DataFrame({"a": ["Check", "str"], "b": ["Sup", "port"]}),
+        pytest.param(
+            lambda cudf: cudf.DataFrame({"a": ["Check", "str"], "b": ["Sup", "port"]}),
+            marks=pytest.mark.xfail(reason="0 length objects don't deseralize cleanly"),
+        ),
     ],
 )
 async def test_ping_pong_cudf(g):
@@ -269,7 +278,8 @@ async def test_ping_pong_numba(cleanup):
 @pytest.mark.asyncio
 async def test_ucx_localcluster(processes, cleanup):
     async with LocalCluster(
-        protocol="ucx",
+        protocol="ucx:://",
+        host=HOST,
         dashboard_address=None,
         n_workers=2,
         threads_per_worker=1,

From 0259acc066c2f3210077b9e19f1d647f58173dd0 Mon Sep 17 00:00:00 2001
From: "Mads R. B. Kristensen" <madsbk@gmail.com>
Date: Tue, 3 Dec 2019 16:37:22 +0100
Subject: [PATCH 0586/1550] worker.close() awaits batched_stream.close()
 (#3291)

---
 distributed/batched.py              | 9 ++++++---
 distributed/tests/test_scheduler.py | 6 ++++--
 distributed/worker.py               | 3 ++-
 3 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/distributed/batched.py b/distributed/batched.py
index a3207b333ef..e066fcf7588 100644
--- a/distributed/batched.py
+++ b/distributed/batched.py
@@ -123,13 +123,16 @@ def send(self, msg):
             self.waker.set()
 
     @gen.coroutine
-    def close(self):
-        """ Flush existing messages and then close comm """
+    def close(self, timeout=None):
+        """ Flush existing messages and then close comm
+
+        If set, raises `tornado.util.TimeoutError` after a timeout.
+        """
         if self.comm is None:
             return
         self.please_stop = True
         self.waker.set()
-        yield self.stopped.wait()
+        yield self.stopped.wait(timeout=timeout)
         if not self.comm.closed():
             try:
                 if self.buffer:
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 4e0e9a8710c..859f56fef42 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -1512,8 +1512,10 @@ def test_idle_timeout(c, s, a, b):
         yield gen.sleep(0.01)
     assert time() < start + 3
 
-    assert a.status == "closed"
-    assert b.status == "closed"
+    start = time()
+    while not (a.status == "closed" and b.status == "closed"):
+        yield gen.sleep(0.01)
+        assert time() < start + 1
 
 
 @gen_cluster(client=True, config={"distributed.scheduler.bandwidth": "100 GB"})
diff --git a/distributed/worker.py b/distributed/worker.py
index 90059002e9a..ebddd042551 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -1089,7 +1089,8 @@ async def close(
                 self.batched_stream.send({"op": "close-stream"})
 
             if self.batched_stream:
-                self.batched_stream.close()
+                with ignoring(gen.TimeoutError):
+                    await self.batched_stream.close(timedelta(seconds=timeout))
 
             self.actor_executor._work_queue.queue.clear()
             if isinstance(self.executor, ThreadPoolExecutor):

From 54ce6785f82f9d1e180f303c4b2983f25c4db7ed Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Tue, 3 Dec 2019 12:58:29 -0600
Subject: [PATCH 0587/1550] Update SSHCluster docstring parameters (#3296)

---
 distributed/deploy/ssh.py | 25 ++++++++++---------------
 1 file changed, 10 insertions(+), 15 deletions(-)

diff --git a/distributed/deploy/ssh.py b/distributed/deploy/ssh.py
index 673cb7ba717..1f49f187a14 100644
--- a/distributed/deploy/ssh.py
+++ b/distributed/deploy/ssh.py
@@ -219,21 +219,16 @@ def SSHCluster(
     Parameters
     ----------
     hosts: List[str]
-        List of hostnames or addresses on which to launch our cluster
-        The first will be used for the scheduler and the rest for workers
-    connect_options:
-        Keywords to pass through to asyncssh.connect
-        known_hosts: List[str] or None
-            The list of keys which will be used to validate the server host
-            key presented during the SSH handshake.  If this is not specified,
-            the keys will be looked up in the file .ssh/known_hosts.  If this
-            is explicitly set to None, server host key validation will be disabled.
-    worker_options:
-        Keywords to pass on to dask-worker
-    scheduler_options:
-        Keywords to pass on to dask-scheduler
-    worker_module:
-        Python module to call to start the worker
+        List of hostnames or addresses on which to launch our cluster.
+        The first will be used for the scheduler and the rest for workers.
+    connect_options: dict, optional
+        Keywords to pass through to ``asyncssh.connect``.
+    worker_options: dict, optional
+        Keywords to pass on to workers.
+    scheduler_options: dict, optional
+        Keywords to pass on to scheduler.
+    worker_module: str, optional
+        Python module to call to start the worker.
 
     Examples
     --------

From b60c4bfc5f46a9aa2f1aacbda21eac38abcc807e Mon Sep 17 00:00:00 2001
From: Benjamin Zaitlen <quasiben@users.noreply.github.com>
Date: Tue, 3 Dec 2019 16:37:42 -0500
Subject: [PATCH 0588/1550] forgot to fix slow test (#3297)

---
 distributed/comm/tests/test_ucx.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/distributed/comm/tests/test_ucx.py b/distributed/comm/tests/test_ucx.py
index 6fc892176a9..7725bfa2432 100644
--- a/distributed/comm/tests/test_ucx.py
+++ b/distributed/comm/tests/test_ucx.py
@@ -303,7 +303,11 @@ async def test_stress(cleanup):
     chunksize = "10 MB"
 
     async with LocalCluster(
-        protocol="ucx", dashboard_address=None, asynchronous=True, processes=False
+        protocol="ucx",
+        dashboard_address=None,
+        asynchronous=True,
+        processes=False,
+        host=HOST,
     ) as cluster:
         async with Client(cluster, asynchronous=True) as client:
             rs = da.random.RandomState()

From 243fd0a8e665b5589982d647aed821578e413810 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 3 Dec 2019 15:51:29 -0800
Subject: [PATCH 0589/1550] Add title to performance_report (#3298)

---
 distributed/scheduler.py         | 3 ++-
 distributed/tests/test_client.py | 1 +
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 7e799715ed4..3ad82ece8c7 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -4762,9 +4762,10 @@ def profile_to_figure(state):
             ]
         )
 
-        from bokeh.plotting import save
+        from bokeh.plotting import save, output_file
 
         with tmpfile(extension=".html") as fn:
+            output_file(filename=fn, title="Dask Performance Report")
             save(tabs, filename=fn)
 
             with open(fn) as f:
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 02e2574a3ab..7e5442d01a5 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -5904,3 +5904,4 @@ async def test_performance_report(c, s, a, b):
 
         assert "bokeh" in data
         assert "random" in data
+        assert "Dask Performance Report" in data

From 4a8a4f3bce378406e83a099e8a12fc9bc12ef25c Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Tue, 3 Dec 2019 22:05:56 -0500
Subject: [PATCH 0590/1550] Updates RMM comment to the correct release (#3299)

This will land in RMM 0.11.0. So update this comment to be a bit more
precise.
---
 distributed/comm/ucx.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/comm/ucx.py b/distributed/comm/ucx.py
index 7c783b605a1..175d628a0f6 100644
--- a/distributed/comm/ucx.py
+++ b/distributed/comm/ucx.py
@@ -48,7 +48,7 @@ def init_once():
 
         if hasattr(rmm, "DeviceBuffer"):
             cuda_array = lambda n: rmm.DeviceBuffer(size=n)
-        else:  # pre-0.12.0
+        else:  # pre-0.11.0
             cuda_array = lambda n: rmm.device_array(n, dtype=np.uint8)
     except ImportError:
         try:

From b1e7ab4e599b8f3e93ffc3c4c95a471e923ac396 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 4 Dec 2019 17:16:42 -0800
Subject: [PATCH 0591/1550] Support multiple listeners in the scheduler (#3288)

This allows the scheduler to listen on multiple different listeners at
once. This can be useful when the client and workers are on different
interfaces, or have different security concerns.
---
 distributed/comm/addressing.py      | 52 ++++++++++++++++++++++++++++-
 distributed/core.py                 | 20 +++++++----
 distributed/scheduler.py            | 17 +++++-----
 distributed/tests/test_scheduler.py | 19 +++++++++++
 4 files changed, 93 insertions(+), 15 deletions(-)

diff --git a/distributed/comm/addressing.py b/distributed/comm/addressing.py
index f0c18b9fbda..2b1c4717407 100644
--- a/distributed/comm/addressing.py
+++ b/distributed/comm/addressing.py
@@ -1,3 +1,4 @@
+import itertools
 import dask
 
 from . import registry
@@ -206,6 +207,54 @@ def uri_from_host_port(host_arg, port_arg, default_port):
     return addr
 
 
+def addresses_from_user_args(
+    host=None,
+    port=None,
+    interface=None,
+    protocol=None,
+    peer=None,
+    security=None,
+    default_port=0,
+) -> list:
+    """ Get a list of addresses if the inputs are lists
+
+    This is like ``address_from_user_args`` except that it also accepts lists
+    for some of the arguments.  If these arguments are lists then it will map
+    over them accordingly.
+
+    Examples
+    --------
+    >>> addresses_from_user_args(host="127.0.0.1", protocol=["inproc", "tcp"])
+    ["inproc://127.0.0.1:", "tcp://127.0.0.1:"]
+    """
+
+    def listify(obj):
+        if isinstance(obj, (tuple, list)):
+            return obj
+        else:
+            return itertools.repeat(obj)
+
+    if any(isinstance(x, (tuple, list)) for x in (host, port, interface, protocol)):
+        return [
+            address_from_user_args(
+                host=h,
+                port=p,
+                interface=i,
+                protocol=pr,
+                peer=peer,
+                security=security,
+                default_port=default_port,
+            )
+            for h, p, i, pr in zip(*map(listify, (host, port, interface, protocol)))
+        ]
+    else:
+        return [
+            address_from_user_args(
+                host, port, interface, protocol, peer, security, default_port
+            )
+        ]
+
+
 def address_from_user_args(
     host=None,
     port=None,
@@ -214,8 +263,9 @@ def address_from_user_args(
     peer=None,
     security=None,
     default_port=0,
-):
+) -> str:
     """ Get an address to listen on from common user provided arguments """
+
     if security and security.require_encryption and not protocol:
         protocol = "tls"
 
diff --git a/distributed/core.py b/distributed/core.py
index ec6e9969fea..bf734070248 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -136,7 +136,7 @@ def __init__(
         self._ongoing_coroutines = weakref.WeakSet()
         self._event_finished = Event()
 
-        self.listener = None
+        self.listeners = []
         self.io_loop = io_loop or IOLoop.current()
         self.loop = self.io_loop
 
@@ -221,7 +221,7 @@ def start_pcs():
     def stop(self):
         if not self.__stopped:
             self.__stopped = True
-            if self.listener is not None:
+            for listener in self.listeners:
                 # Delay closing the server socket until the next IO loop tick.
                 # Otherwise race conditions can appear if an event handler
                 # for an accept() call is already scheduled by the IO loop,
@@ -229,7 +229,14 @@ def stop(self):
                 # The demonstrator for this is Worker.terminate(), which
                 # closes the server socket in response to an incoming message.
                 # See https://github.com/tornadoweb/tornado/issues/2069
-                self.io_loop.add_callback(self.listener.stop)
+                self.io_loop.add_callback(listener.stop)
+
+    @property
+    def listener(self):
+        if self.listeners:
+            return self.listeners[0]
+        else:
+            return None
 
     def _measure_tick(self):
         now = time()
@@ -305,13 +312,14 @@ async def listen(self, port_or_addr=None, listen_args=None):
         else:
             addr = port_or_addr
             assert isinstance(addr, str)
-        self.listener = listen(
+        listener = listen(
             addr,
             self.handle_comm,
             deserialize=self.deserialize,
             connection_args=listen_args,
         )
-        await self.listener.start()
+        await listener.start()
+        self.listeners.append(listener)
 
     async def handle_comm(self, comm, shutting_down=shutting_down):
         """ Dispatch new communications to coroutine-handlers
@@ -487,7 +495,7 @@ async def handle_stream(self, comm, extra=None, every_cycle=[]):
     def close(self):
         for pc in self.periodic_callbacks.values():
             pc.stop()
-        if self.listener:
+        for listener in self.listeners:
             self.listener.stop()
         for i in range(20):  # let comms close naturally for a second
             if not self._comms:
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 3ad82ece8c7..ab78ef05370 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -36,7 +36,7 @@
     get_address_host,
     unparse_host_port,
 )
-from .comm.addressing import address_from_user_args
+from .comm.addressing import addresses_from_user_args
 from .core import rpc, connect, send_recv, clean_exception, CommClosedError
 from .diagnostics.plugin import SchedulerPlugin
 from . import profile
@@ -1118,7 +1118,7 @@ def __init__(
 
         connection_limit = get_fileno_limit() / 2
 
-        self._start_address = address_from_user_args(
+        self._start_address = addresses_from_user_args(
             host=host,
             port=port,
             interface=interface,
@@ -1215,14 +1215,15 @@ async def start(self):
                 c.cancel()
 
         if self.status != "running":
-            await self.listen(self._start_address, listen_args=self.listen_args)
-            self.ip = get_address_host(self.listen_address)
-            listen_ip = self.ip
+            for addr in self._start_address:
+                await self.listen(addr, listen_args=self.listen_args)
+                self.ip = get_address_host(self.listen_address)
+                listen_ip = self.ip
 
-            if listen_ip == "0.0.0.0":
-                listen_ip = ""
+                if listen_ip == "0.0.0.0":
+                    listen_ip = ""
 
-            if self._start_address.startswith("inproc://"):
+            if self.address.startswith("inproc://"):
                 listen_ip = "localhost"
 
             # Services listen on all addresses
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 859f56fef42..ded056d8f96 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -1849,3 +1849,22 @@ def reducer(x, y):
         if "reducer" in key and finish == "processing":
             finish_processing_transitions += 1
     assert finish_processing_transitions == 1
+
+
+@pytest.mark.asyncio
+async def test_multiple_listeners(cleanup):
+    async with Scheduler(port=0, protocol=["inproc", "tcp"]) as s:
+        async with Worker(s.listeners[0].contact_address) as a:
+            async with Worker(s.listeners[1].contact_address) as b:
+                assert a.address.startswith("inproc")
+                assert a.scheduler.address.startswith("inproc")
+                assert b.address.startswith("tcp")
+                assert b.scheduler.address.startswith("tcp")
+
+                async with Client(s.address, asynchronous=True) as c:
+                    futures = c.map(inc, range(20))
+                    await wait(futures)
+
+                    # Force inter-worker communication both ways
+                    await c.submit(sum, futures, workers=[a.address])
+                    await c.submit(len, futures, workers=[b.address])

From 73b6bf989b3fbd5e164acb448bddf6334b7779e6 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Wed, 4 Dec 2019 20:46:47 -0600
Subject: [PATCH 0592/1550] Skip Security.temporary() tests if cryptography not
 installed (#3302)

---
 distributed/deploy/tests/test_local.py | 3 +++
 distributed/tests/test_security.py     | 2 ++
 2 files changed, 5 insertions(+)

diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index 0f8eb6d8901..370423771a5 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -250,6 +250,7 @@ def test_Client_twice(loop):
 
 @pytest.mark.asyncio
 async def test_client_constructor_with_temporary_security(cleanup):
+    pytest.importorskip("cryptography")
     async with Client(
         security=True, silence_logs=False, dashboard_address=None, asynchronous=True
     ) as c:
@@ -709,6 +710,7 @@ def test_adapt_then_manual(loop):
 @pytest.mark.parametrize("temporary", [True, False])
 def test_local_tls(loop, temporary):
     if temporary:
+        pytest.importorskip("cryptography")
         security = True
     else:
         security = tls_only_security()
@@ -989,6 +991,7 @@ async def test_repr(cleanup):
 @pytest.mark.parametrize("temporary", [True, False])
 async def test_capture_security(cleanup, temporary):
     if temporary:
+        pytest.importorskip("cryptography")
         security = True
     else:
         security = tls_only_security()
diff --git a/distributed/tests/test_security.py b/distributed/tests/test_security.py
index 8e9007f20f0..002e63d2855 100644
--- a/distributed/tests/test_security.py
+++ b/distributed/tests/test_security.py
@@ -379,6 +379,8 @@ def check_encryption_error():
 
 
 def test_temporary_credentials():
+    pytest.importorskip("cryptography")
+
     sec = Security.temporary()
     sec_repr = repr(sec)
     fields = ["tls_ca_file"]

From 4e9eb461df3d7ac4a693d094f488fbd4b44676ac Mon Sep 17 00:00:00 2001
From: byjott <jochen.ott@blue-yonder.com>
Date: Thu, 5 Dec 2019 16:33:29 +0100
Subject: [PATCH 0593/1550] Retry operations on network issues (#3294)

We operate distributed in the cloud and see tcp connection aborts. Unfortunately, distributed often does not recover cleanly from such situations, although a simple re-try would have helped in most cases.

This PR proposes to add a more generic retry to some operations.

Notes:

-  only some operations are re-tried, as for some operations, triggering it twice may have undesired effects. There are probably more operations that can / should be re-tried, so this is just a start for operations where it's "obviously" safe to retry.
-  parameters for the re-tries (maximum number of retry attempts, delay between re-tries) is configurable. The default is to not re-try at all to not change the current behavior (some might rely on/prefer seeing all connection failures, fast)
---
 dev-requirements.txt                 |  1 +
 distributed/client.py                |  7 ++-
 distributed/distributed.yaml         |  5 ++
 distributed/scheduler.py             | 17 ++++--
 distributed/tests/test_scheduler.py  | 44 ++++++++-------
 distributed/tests/test_utils_comm.py | 75 ++++++++++++++++++++++++-
 distributed/utils_comm.py            | 83 ++++++++++++++++++++++++++++
 distributed/worker.py                | 38 ++++---------
 8 files changed, 214 insertions(+), 56 deletions(-)

diff --git a/dev-requirements.txt b/dev-requirements.txt
index cd79b3e4317..a367f706e76 100644
--- a/dev-requirements.txt
+++ b/dev-requirements.txt
@@ -10,3 +10,4 @@ ipykernel >= 4.5.2
 pytest >= 3.2
 prometheus_client >= 0.6.0
 jupyter-server-proxy >= 1.1.0
+pytest-asyncio
diff --git a/distributed/client.py b/distributed/client.py
index 02337bb525f..027e24afeee 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -53,6 +53,7 @@
     pack_data,
     scatter_to_workers,
     gather_from_workers,
+    retry_operation,
 )
 from .cfexecutor import ClientExecutor
 from .core import connect, rpc, clean_exception, CommClosedError, PooledRPCCall
@@ -1794,19 +1795,19 @@ async def _gather_remote(self, direct, local_worker):
 
         try:
             if direct or local_worker:  # gather directly from workers
-                who_has = await self.scheduler.who_has(keys=keys)
+                who_has = await retry_operation(self.scheduler.who_has, keys=keys)
                 data2, missing_keys, missing_workers = await gather_from_workers(
                     who_has, rpc=self.rpc, close=False
                 )
                 response = {"status": "OK", "data": data2}
                 if missing_keys:
                     keys2 = [key for key in keys if key not in data2]
-                    response = await self.scheduler.gather(keys=keys2)
+                    response = await retry_operation(self.scheduler.gather, keys=keys2)
                     if response["status"] == "OK":
                         response["data"].update(data2)
 
             else:  # ask scheduler to gather data for us
-                response = await self.scheduler.gather(keys=keys)
+                response = await retry_operation(self.scheduler.gather, keys=keys)
         finally:
             self._gather_semaphore.release()
 
diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index ae42162bb2f..e6c6a49b484 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -72,6 +72,11 @@ distributed:
     lost-worker-timeout: 15s  # Interval after which to hard-close a lost worker job
 
   comm:
+    retry:  # some operations (such as gathering data) are subject to re-tries with the below parameters
+      count: 0  # the maximum retry attempts. 0 disables re-trying.
+      delay:
+         min: 1s  # the first non-zero delay between re-tries
+         max: 20s  # the maximum delay between re-tries
     compression: auto
     offload: 10MiB # Size after which we choose to offload serialization to another thread
     default-scheme: tcp
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index ab78ef05370..eeb7eb49732 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -59,7 +59,7 @@
     shutting_down,
     tmpfile,
 )
-from .utils_comm import scatter_to_workers, gather_from_workers
+from .utils_comm import scatter_to_workers, gather_from_workers, retry_operation
 from .utils_perf import enable_gc_diagnosis, disable_gc_diagnosis
 
 from .publish import PublishExtension
@@ -2786,7 +2786,10 @@ async def rebalance(self, comm=None, keys=None, workers=None):
                 to_senders[sender.address].append(ts.key)
 
             result = await asyncio.gather(
-                *(self.rpc(addr=r).gather(who_has=v) for r, v in to_recipients.items())
+                *(
+                    retry_operation(self.rpc(addr=r).gather, who_has=v)
+                    for r, v in to_recipients.items()
+                )
             )
             for r, v in to_recipients.items():
                 self.log_event(r, {"action": "rebalance", "who_has": v})
@@ -2819,7 +2822,7 @@ async def rebalance(self, comm=None, keys=None, workers=None):
 
             await asyncio.gather(
                 *(
-                    self.rpc(addr=r).delete_data(keys=v, report=False)
+                    retry_operation(self.rpc(addr=r).delete_data, keys=v, report=False)
                     for r, v in to_senders.items()
                 )
             )
@@ -2887,8 +2890,10 @@ async def replicate(
 
             await asyncio.gather(
                 *(
-                    self.rpc(addr=ws.address).delete_data(
-                        keys=[ts.key for ts in tasks], report=False
+                    retry_operation(
+                        self.rpc(addr=ws.address).delete_data,
+                        keys=[ts.key for ts in tasks],
+                        report=False,
                     )
                     for ws, tasks in del_worker_tasks.items()
                 )
@@ -2922,7 +2927,7 @@ async def replicate(
 
             results = await asyncio.gather(
                 *(
-                    self.rpc(addr=w).gather(who_has=who_has)
+                    retry_operation(self.rpc(addr=w).gather, who_has=who_has)
                     for w, who_has in gathers.items()
                 )
             )
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index ded056d8f96..7bf3f456085 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -7,6 +7,7 @@
 import operator
 import sys
 from time import sleep
+from unittest import mock
 import logging
 
 import dask
@@ -1749,7 +1750,8 @@ async def test_gather_failing_cnn_recover(c, s, a, b):
     x = await c.scatter({"x": 1}, workers=a.address)
 
     s.rpc = FlakyConnectionPool(failing_connections=1)
-    res = await s.gather(keys=["x"])
+    with mock.patch("distributed.utils_comm.retry_count", 1):
+        res = await s.gather(keys=["x"])
     assert res["status"] == "OK"
 
 
@@ -1811,25 +1813,31 @@ def reducer(x, y):
 
     s.rpc = FlakyConnectionPool(failing_connections=4)
 
-    with captured_logger(logging.getLogger("distributed.scheduler")) as sched_logger:
-        with captured_logger(logging.getLogger("distributed.client")) as client_logger:
-            with captured_logger(
-                logging.getLogger("distributed.worker")
-            ) as worker_logger:
-                # Gather using the client (as an ordinary user would)
-                # Upon a missing key, the client will reschedule the computations
-                res = await c.gather(z)
+    with captured_logger(
+        logging.getLogger("distributed.scheduler")
+    ) as sched_logger, captured_logger(
+        logging.getLogger("distributed.client")
+    ) as client_logger, captured_logger(
+        logging.getLogger("distributed.utils_comm")
+    ) as utils_comm_logger, mock.patch(
+        "distributed.utils_comm.retry_count", 3
+    ), mock.patch(
+        "distributed.utils_comm.retry_delay_min", 0.5
+    ):
+        # Gather using the client (as an ordinary user would)
+        # Upon a missing key, the client will reschedule the computations
+        res = await c.gather(z)
 
     assert res == 5
 
     sched_logger = sched_logger.getvalue()
     client_logger = client_logger.getvalue()
-    worker_logger = worker_logger.getvalue()
+    utils_comm_logger = utils_comm_logger.getvalue()
 
     # Ensure that the communication was done via the scheduler, i.e. we actually hit a bad connection
     assert s.rpc.cnn_count > 0
 
-    assert "Encountered connection issue during data collection" in worker_logger
+    assert "Retrying get_data_from_worker after exception" in utils_comm_logger
 
     # The reducer task was actually not found upon first collection. The client will reschedule the graph
     assert "Couldn't gather 1 keys, rescheduling" in client_logger
@@ -1841,14 +1849,12 @@ def reducer(x, y):
     # that the scheduler again knows about the result.
     # The final reduce step should then be used from the re-connected worker
     # instead of recomputing it.
-
-    starts = []
-    finish_processing_transitions = 0
-    for transition in s.transition_log:
-        key, start, finish, recommendations, timestamp = transition
-        if "reducer" in key and finish == "processing":
-            finish_processing_transitions += 1
-    assert finish_processing_transitions == 1
+    transitions_to_processing = [
+        (key, start, timestamp)
+        for key, start, finish, recommendations, timestamp in s.transition_log
+        if finish == "processing" and "reducer" in key
+    ]
+    assert len(transitions_to_processing) == 1
 
 
 @pytest.mark.asyncio
diff --git a/distributed/tests/test_utils_comm.py b/distributed/tests/test_utils_comm.py
index f66d3ba62d5..3f26fae623a 100644
--- a/distributed/tests/test_utils_comm.py
+++ b/distributed/tests/test_utils_comm.py
@@ -1,7 +1,11 @@
 from distributed.core import ConnectionPool
 from distributed.comm import Comm
-from distributed.utils_test import gen_cluster
-from distributed.utils_comm import pack_data, gather_from_workers
+from distributed.utils_test import gen_cluster, loop  # noqa: F401
+from distributed.utils_comm import pack_data, gather_from_workers, retry
+
+from unittest import mock
+
+import pytest
 
 
 def test_pack_data():
@@ -58,3 +62,70 @@ def test_gather_from_workers_permissive_flaky(c, s, a, b):
 
     assert missing == {"x": [a.address]}
     assert bad_workers == [a.address]
+
+
+def test_retry_no_exception(loop):
+    n_calls = 0
+    retval = object()
+
+    async def coro():
+        nonlocal n_calls
+        n_calls += 1
+        return retval
+
+    assert (
+        loop.run_sync(lambda: retry(coro, count=0, delay_min=-1, delay_max=-1))
+        is retval
+    )
+    assert n_calls == 1
+
+
+def test_retry0_raises_immediately(loop):
+    # test that using max_reties=0 raises after 1 call
+
+    n_calls = 0
+
+    async def coro():
+        nonlocal n_calls
+        n_calls += 1
+        raise RuntimeError(f"RT_ERROR {n_calls}")
+
+    with pytest.raises(RuntimeError, match="RT_ERROR 1"):
+        loop.run_sync(lambda: retry(coro, count=0, delay_min=-1, delay_max=-1))
+
+    assert n_calls == 1
+
+
+def test_retry_does_retry_and_sleep(loop):
+    # test the retry and sleep pattern of `retry`
+    n_calls = 0
+
+    class MyEx(Exception):
+        pass
+
+    async def coro():
+        nonlocal n_calls
+        n_calls += 1
+        raise MyEx(f"RT_ERROR {n_calls}")
+
+    sleep_calls = []
+
+    async def my_sleep(amount):
+        sleep_calls.append(amount)
+        return
+
+    with mock.patch("asyncio.sleep", my_sleep):
+        with pytest.raises(MyEx, match="RT_ERROR 6"):
+            loop.run_sync(
+                lambda: retry(
+                    coro,
+                    retry_on_exceptions=(MyEx,),
+                    count=5,
+                    delay_min=1.0,
+                    delay_max=6.0,
+                    jitter_fraction=0.0,
+                )
+            )
+
+    assert n_calls == 6
+    assert sleep_calls == [0.0, 1.0, 3.0, 6.0, 6.0]
diff --git a/distributed/utils_comm.py b/distributed/utils_comm.py
index e2072189be0..cb614602f7b 100644
--- a/distributed/utils_comm.py
+++ b/distributed/utils_comm.py
@@ -1,10 +1,13 @@
 import asyncio
 from collections import defaultdict
+from functools import partial
 from itertools import cycle
 import logging
 import random
 
 from dask.optimization import SubgraphCallable
+import dask.config
+from dask.utils import parse_timedelta
 from toolz import merge, concat, groupby, drop
 
 from .core import rpc
@@ -275,3 +278,83 @@ def pack_data(o, d, key_types=object):
         return {k: pack_data(v, d, key_types=key_types) for k, v in o.items()}
     else:
         return o
+
+
+retry_count = dask.config.get("distributed.comm.retry.count")
+retry_delay_min = parse_timedelta(
+    dask.config.get("distributed.comm.retry.delay.min"), default="s"
+)
+retry_delay_max = parse_timedelta(
+    dask.config.get("distributed.comm.retry.delay.max"), default="s"
+)
+
+
+async def retry(
+    coro,
+    count,
+    delay_min,
+    delay_max,
+    jitter_fraction=0.1,
+    retry_on_exceptions=(EnvironmentError, IOError),
+    operation=None,
+):
+    """
+    Return the result of ``await coro()``, re-trying in case of exceptions
+
+    The delay between attempts is ``delay_min * (2 ** i - 1)`` where ``i`` enumerates the attempt that just failed
+    (starting at 0), but never larger than ``delay_max``.
+    This yields no delay between the first and second attempt, then ``delay_min``, ``3 * delay_min``, etc.
+    (The reason to re-try with no delay is that in most cases this is sufficient and will thus recover faster
+    from a communication failure).
+
+    Parameters
+    ----------
+    coro
+        The coroutine function to call and await
+    count
+        The maximum number of re-tries before giving up. 0 means no re-try; must be >= 0.
+    delay_min
+        The base factor for the delay (in seconds); this is the first non-zero delay between re-tries.
+    delay_max
+        The maximum delay (in seconds) between consecutive re-tries (without jitter)
+    jitter_fraction
+        The maximum jitter to add to the delay, as fraction of the total delay. No jitter is added if this
+        value is <= 0.
+        Using a non-zero value here avoids "herd effects" of many operations re-tried at the same time
+    retry_on_exceptions
+        A tuple of exception classes to retry. Other exceptions are not caught and re-tried, but propagate immediately.
+    operation
+        A human-readable description of the operation attempted; used only for logging failures
+
+    Returns
+    -------
+    Any
+        Whatever `await `coro()` returned
+    """
+    # this loop is a no-op in case max_retries<=0
+    for i_try in range(count):
+        try:
+            return await coro()
+        except retry_on_exceptions as ex:
+            operation = operation or str(coro)
+            logger.info(
+                f"Retrying {operation} after exception in attempt {i_try}/{count}: {ex}"
+            )
+            delay = min(delay_min * (2 ** i_try - 1), delay_max)
+            if jitter_fraction > 0:
+                delay *= 1 + random.random() * jitter_fraction
+            await asyncio.sleep(delay)
+    return await coro()
+
+
+async def retry_operation(coro, *args, operation=None, **kwargs):
+    """
+    Retry an operation using the configuration values for the retry parameters
+    """
+    return await retry(
+        partial(coro, *args, **kwargs),
+        count=retry_count,
+        delay_min=retry_delay_min,
+        delay_max=retry_delay_max,
+        operation=operation,
+    )
diff --git a/distributed/worker.py b/distributed/worker.py
index ebddd042551..0326cbd157a 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -63,7 +63,7 @@
     warn_on_duration,
     LRU,
 )
-from .utils_comm import pack_data, gather_from_workers
+from .utils_comm import pack_data, gather_from_workers, retry_operation
 from .utils_perf import ThrottledGC, enable_gc_diagnosis, disable_gc_diagnosis
 
 logger = logging.getLogger(__name__)
@@ -868,7 +868,8 @@ async def heartbeat(self):
             logger.debug("Heartbeat: %s" % self.address)
             try:
                 start = time()
-                response = await self.scheduler.heartbeat_worker(
+                response = await retry_operation(
+                    self.scheduler.heartbeat_worker,
                     address=self.contact_address,
                     now=time(),
                     metrics=await self.get_metrics(),
@@ -1299,8 +1300,10 @@ async def set_resources(self, **resources):
                 self.available_resources[r] = quantity
             self.total_resources[r] = quantity
 
-        await self.scheduler.set_resources(
-            resources=self.total_resources, worker=self.contact_address
+        await retry_operation(
+            self.scheduler.set_resources,
+            resources=self.total_resources,
+            worker=self.contact_address,
         )
 
     ###################
@@ -2047,7 +2050,7 @@ async def handle_missing_dep(self, *deps, **kwargs):
                     self.suspicious_deps[dep],
                 )
 
-            who_has = await self.scheduler.who_has(keys=list(deps))
+            who_has = await retry_operation(self.scheduler.who_has, keys=list(deps))
             who_has = {k: v for k, v in who_has.items() if v}
             self.update_who_has(who_has)
             for dep in deps:
@@ -2081,7 +2084,7 @@ async def handle_missing_dep(self, *deps, **kwargs):
 
     async def query_who_has(self, *deps):
         with log_errors():
-            response = await self.scheduler.who_has(keys=deps)
+            response = await retry_operation(self.scheduler.who_has, keys=deps)
             self.update_who_has(response)
             return response
 
@@ -3132,10 +3135,7 @@ async def get_data_from_worker(
     if deserializers is None:
         deserializers = rpc.deserializers
 
-    retry_count = 0
-    max_retries = 3
-
-    while True:
+    async def _get_data():
         comm = await rpc.connect(worker)
         comm.name = "Ephemeral Worker->Worker for gather"
         try:
@@ -3155,25 +3155,11 @@ async def get_data_from_worker(
             else:
                 if status == "OK":
                     await comm.write("OK")
-            break
-        except (EnvironmentError, CommClosedError):
-            if retry_count < max_retries:
-                await asyncio.sleep(0.1 * (2 ** retry_count))
-                retry_count += 1
-                logger.info(
-                    "Encountered connection issue during data collection of keys %s on worker %s. Retrying (%s / %s)",
-                    keys,
-                    worker,
-                    retry_count,
-                    max_retries,
-                )
-                continue
-            else:
-                raise
+            return response
         finally:
             rpc.reuse(worker, comm)
 
-    return response
+    return await retry_operation(_get_data, operation="get_data_from_worker")
 
 
 job_counter = [0]

From a70a080fec56acafc3d3a1510862265525633511 Mon Sep 17 00:00:00 2001
From: byjott <jochen.ott@blue-yonder.com>
Date: Fri, 6 Dec 2019 00:28:03 +0100
Subject: [PATCH 0594/1550] Connectionpool: don't hand out closed connections
 (#3301)

Operating long-running dask clusters (sometimes, they run for many days without interruptions), we found that connection issues we observe are likely related to a behavior of the ConnectionPool of handing out connections that are already closed by the remote end (e.g. because a connection has been established some days ago, easily above connection timeouts).

It also fixes another bug in the connecitonpool's bookkeeping of connections.
---
 distributed/comm/tcp.py          | 14 ++++++++++++++
 distributed/core.py              | 14 ++++++++++++--
 distributed/tests/test_client.py |  3 ---
 distributed/tests/test_core.py   | 32 ++++++++++++++++++++++++++++++++
 4 files changed, 58 insertions(+), 5 deletions(-)

diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py
index 40b6e8104b3..c2f3feeb704 100644
--- a/distributed/comm/tcp.py
+++ b/distributed/comm/tcp.py
@@ -132,6 +132,10 @@ def convert_stream_closed_error(obj, exc):
         raise CommClosedError("in %s: %s" % (obj, exc))
 
 
+def _do_nothing():
+    pass
+
+
 class TCP(Comm):
     """
     An established communication based on an underlying Tornado IOStream.
@@ -154,6 +158,16 @@ def __init__(self, stream, local_addr, peer_addr, deserialize=True):
 
         stream.set_nodelay(True)
         set_tcp_timeout(stream)
+        # set a close callback, to make `self.stream.closed()` more reliable.
+        # Background: if `stream` is unused (e.g. because it's in `ConnectionPool.available`),
+        # the underlying fd is not watched for changes. In this case, even if the
+        # connection is actively closed by the remote end, `self.closed()` would still return `False`.
+        # Registering a closed callback will make tornado register the underlying fd
+        # for changes, and this would be reflected in `self.closed()` even without reading/writing.
+        #
+        # Use a global method (instead of a lambda) to avoid creating a reference
+        # to the local scope.
+        stream.set_close_callback(_do_nothing)
         self._read_extra()
 
     def _read_extra(self):
diff --git a/distributed/core.py b/distributed/core.py
index bf734070248..effc96831b2 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -840,6 +840,14 @@ def __init__(
         self._created = weakref.WeakSet()
         self._instances.add(self)
 
+    def _validate(self):
+        """
+        Validate important invariants of this class
+
+        Used only for testing / debugging
+        """
+        assert self.semaphore._value == self.limit - self.open - self._n_connecting
+
     @property
     def active(self):
         return sum(map(len, self.occupied.values()))
@@ -868,9 +876,11 @@ async def connect(self, addr, timeout=None):
         """
         available = self.available[addr]
         occupied = self.occupied[addr]
-        if available:
+        while available:
             comm = available.pop()
-            if not comm.closed():
+            if comm.closed():
+                self.semaphore.release()
+            else:
                 occupied.add(comm)
                 return comm
 
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 7e5442d01a5..2ec3a9f79d1 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -3532,9 +3532,6 @@ def test_reconnect(loop):
         assert time() < start + 5
         sleep(0.01)
 
-    with pytest.raises(Exception):
-        c.nthreads()
-
     assert x.status == "cancelled"
     with pytest.raises(CancelledError):
         x.result()
diff --git a/distributed/tests/test_core.py b/distributed/tests/test_core.py
index d3bcdaf8987..bda7bda2ad0 100644
--- a/distributed/tests/test_core.py
+++ b/distributed/tests/test_core.py
@@ -750,3 +750,35 @@ async def test_deserialize_error():
     assert type(info.value) == Exception
     for c in str(info.value):
         assert c.isalpha() or c in "(',!)"  # no crazy bytestrings
+
+
+@pytest.mark.asyncio
+async def test_connection_pool_detects_remote_close():
+    server = Server({"ping": pingpong})
+    await server.listen("tcp://")
+
+    # open a connection, use it and give it back to the pool
+    p = ConnectionPool(limit=10)
+    conn = await p.connect(server.address)
+    await send_recv(conn, op="ping")
+    p.reuse(server.address, conn)
+
+    # now close this connection on the *server*
+    assert len(server._comms) == 1
+    server_conn = list(server._comms.keys())[0]
+    await server_conn.close()
+
+    # give the ConnectionPool some time to realize that the connection is closed
+    await asyncio.sleep(0.1)
+
+    # the connection pool should not hand out `conn` again
+    conn2 = await p.connect(server.address)
+    assert conn2 is not conn
+    p.reuse(server.address, conn2)
+    # check that `conn` has ben removed from the internal data structures
+    assert p.open == 1 and p.active == 0
+
+    # check connection pool invariants hold even after it detects a closed connection
+    # while creating conn2:
+    p._validate()
+    p.close()

From f7a0d7a8e5a729e84f695b8efb593e590c0ec2f4 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 6 Dec 2019 09:53:17 -0800
Subject: [PATCH 0595/1550] Add dask-spec CLI tool (#3090)

This launches a fully configured spec, rather than command line arguments.

This is helpful for systems like SSH, that are trying to route Python arguments through the CLI.

## Example

```yaml
# foo.yaml
big:
  cls: dask.distributed.Worker
  opts:
    nanny: false,
    nthreads: 12
    resources: {"FOO": 1}
small:
  cls: dask.distributed.Worker
  opts:
    nanny: false,
    nthreads: 2
```

```bash
$ dask-scheduler
distributed.scheduler - INFO -   Scheduler at:   tcp://192.168.1.88:8786

$ python -m distributed.cli.dask_spec localhost:8786 --file foo.yaml
distributed.worker - INFO -       Start worker at:      tcp://127.0.0.1:63400
distributed.worker - INFO -          Listening to:      tcp://127.0.0.1:63400
distributed.worker - INFO - Waiting to connect to:       tcp://localhost:8786
distributed.worker - INFO - -------------------------------------------------
distributed.worker - INFO -               Threads:                         12
distributed.worker - INFO -                Memory:                   17.18 GB
distributed.worker - INFO -       Local Directory: /Users/mrocklin/workspace/distributed/dask-worker-space/worker-r762e6xl
distributed.worker - INFO - -------------------------------------------------
distributed.worker - INFO -       Start worker at:      tcp://127.0.0.1:63401
distributed.worker - INFO -          Listening to:      tcp://127.0.0.1:63401
distributed.worker - INFO - Waiting to connect to:       tcp://localhost:8786
distributed.worker - INFO - -------------------------------------------------
distributed.worker - INFO -               Threads:                          2
distributed.worker - INFO -                Memory:                    2.86 GB
distributed.worker - INFO -       Local Directory: /Users/mrocklin/workspace/distributed/dask-worker-space/worker-h8mjhnef
distributed.worker - INFO - -------------------------------------------------
distributed.worker - INFO -         Registered to:       tcp://localhost:8786
distributed.worker - INFO - -------------------------------------------------
distributed.core - INFO - Starting established connection
distributed.worker - INFO -         Registered to:       tcp://localhost:8786
distributed.worker - INFO - -------------------------------------------------
distributed.core - INFO - Starting established connection
```

As another example, here is starting a scheduler with two protocols

```
python -m distributed.cli.dask_spec --spec '{"cls": "dask.distributed.Scheduler", "opts": {"protocol": ["inproc", "tcp"]}}'
```
---
 distributed/cli/dask_spec.py                  | 41 +++++++++
 distributed/cli/tests/test_dask_spec.py       | 89 +++++++++++++++++++
 distributed/deploy/spec.py                    | 41 ++++++++-
 distributed/deploy/tests/test_spec_cluster.py | 20 ++++-
 distributed/utils.py                          | 17 ++++
 5 files changed, 202 insertions(+), 6 deletions(-)
 create mode 100644 distributed/cli/dask_spec.py
 create mode 100644 distributed/cli/tests/test_dask_spec.py

diff --git a/distributed/cli/dask_spec.py b/distributed/cli/dask_spec.py
new file mode 100644
index 00000000000..0a224e5b37c
--- /dev/null
+++ b/distributed/cli/dask_spec.py
@@ -0,0 +1,41 @@
+import asyncio
+import click
+import json
+import sys
+import yaml
+
+from distributed.deploy.spec import run_spec
+
+
+@click.command(context_settings=dict(ignore_unknown_options=True))
+@click.argument("args", nargs=-1)
+@click.option("--spec", type=str, default="", help="")
+@click.option("--spec-file", type=str, default=None, help="")
+@click.version_option()
+def main(args, spec: str, spec_file: str):
+    if spec and spec_file or not spec and not spec_file:
+        print("Must specify exactly one of --spec and --spec-file")
+        sys.exit(1)
+    _spec = {}
+    if spec_file:
+        with open(spec_file) as f:
+            _spec.update(yaml.safe_load(f))
+
+    if spec:
+        _spec.update(json.loads(spec))
+
+    if "cls" in _spec:  # single worker spec
+        _spec = {_spec["opts"].get("name", 0): _spec}
+
+    async def run():
+        servers = await run_spec(_spec, *args)
+        try:
+            await asyncio.gather(*[w.finished() for w in servers.values()])
+        except KeyboardInterrupt:
+            await asyncio.gather(*[w.close() for w in servers.values()])
+
+    asyncio.get_event_loop().run_until_complete(run())
+
+
+if __name__ == "__main__":
+    main()
diff --git a/distributed/cli/tests/test_dask_spec.py b/distributed/cli/tests/test_dask_spec.py
new file mode 100644
index 00000000000..a18b9fb383a
--- /dev/null
+++ b/distributed/cli/tests/test_dask_spec.py
@@ -0,0 +1,89 @@
+import pytest
+import sys
+import yaml
+
+from distributed import Client
+from distributed.utils_test import popen
+from distributed.utils_test import cleanup  # noqa: F401
+
+
+@pytest.mark.asyncio
+async def test_text(cleanup):
+    with popen(
+        [
+            sys.executable,
+            "-m",
+            "distributed.cli.dask_spec",
+            "--spec",
+            '{"cls": "dask.distributed.Scheduler", "opts": {"port": 9373}}',
+        ]
+    ) as sched:
+        with popen(
+            [
+                sys.executable,
+                "-m",
+                "distributed.cli.dask_spec",
+                "tcp://localhost:9373",
+                "--spec",
+                '{"cls": "dask.distributed.Worker", "opts": {"nanny": false, "nthreads": 3, "name": "foo"}}',
+            ]
+        ) as w:
+            async with Client("tcp://localhost:9373", asynchronous=True) as client:
+                await client.wait_for_workers(1)
+                info = await client.scheduler.identity()
+                [w] = info["workers"].values()
+                assert w["name"] == "foo"
+                assert w["nthreads"] == 3
+
+
+@pytest.mark.asyncio
+async def test_file(cleanup, tmp_path):
+    fn = str(tmp_path / "foo.yaml")
+    with open(fn, "w") as f:
+        yaml.dump(
+            {
+                "cls": "dask.distributed.Worker",
+                "opts": {"nanny": False, "nthreads": 3, "name": "foo"},
+            },
+            f,
+        )
+
+    with popen(["dask-scheduler", "--port", "9373", "--no-dashboard"]) as sched:
+        with popen(
+            [
+                sys.executable,
+                "-m",
+                "distributed.cli.dask_spec",
+                "tcp://localhost:9373",
+                "--spec-file",
+                fn,
+            ]
+        ) as w:
+            async with Client("tcp://localhost:9373", asynchronous=True) as client:
+                await client.wait_for_workers(1)
+                info = await client.scheduler.identity()
+                [w] = info["workers"].values()
+                assert w["name"] == "foo"
+                assert w["nthreads"] == 3
+
+
+def test_errors():
+    with popen(
+        [
+            sys.executable,
+            "-m",
+            "distributed.cli.dask_spec",
+            "--spec",
+            '{"foo": "bar"}',
+            "--spec-file",
+            "foo.yaml",
+        ]
+    ) as proc:
+        line = proc.stdout.readline().decode()
+        assert "exactly one" in line
+        assert "--spec" in line and "--spec-file" in line
+
+    with popen([sys.executable, "-m", "distributed.cli.dask_spec"]) as proc:
+        line = proc.stdout.readline().decode()
+        assert "exactly one" in line
+        assert "--spec" in line and "--spec-file" in line
diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index 72cae01e85c..fb06057bb64 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -7,11 +7,19 @@
 
 import dask
 from tornado import gen
+from tornado.locks import Event
 
 from .adaptive import Adaptive
 from .cluster import Cluster
 from ..core import rpc, CommClosedError
-from ..utils import LoopRunner, silence_logging, ignoring, parse_bytes, parse_timedelta
+from ..utils import (
+    LoopRunner,
+    silence_logging,
+    ignoring,
+    parse_bytes,
+    parse_timedelta,
+    import_term,
+)
 from ..scheduler import Scheduler
 from ..security import Security
 
@@ -33,6 +41,7 @@ def __init__(self, scheduler=None, name=None):
         self.external_address = None
         self.lock = asyncio.Lock()
         self.status = "created"
+        self._event_finished = Event()
 
     def __await__(self):
         async def _():
@@ -65,6 +74,11 @@ async def close(self):
         need to worry about shutting down gracefully
         """
         self.status = "closed"
+        self._event_finished.set()
+
+    async def finished(self):
+        """ Wait until the server has finished """
+        await self._event_finished.wait()
 
     def __repr__(self):
         return "<%s: status=%s>" % (type(self).__name__, self.status)
@@ -260,9 +274,11 @@ async def _start(self):
             else:
                 services = {("dashboard", 8787): BokehScheduler}
             self.scheduler_spec = {"cls": Scheduler, "options": {"services": services}}
-        self.scheduler = self.scheduler_spec["cls"](
-            **self.scheduler_spec.get("options", {})
-        )
+
+        cls = self.scheduler_spec["cls"]
+        if isinstance(cls, str):
+            cls = import_term(cls)
+        self.scheduler = cls(**self.scheduler_spec.get("options", {}))
 
         self.status = "starting"
         self.scheduler = await self.scheduler
@@ -307,6 +323,8 @@ async def _correct_state_internal(self):
                 if "name" not in opts:
                     opts = opts.copy()
                     opts["name"] = name
+                if isinstance(cls, str):
+                    cls = import_term(cls)
                 worker = cls(self.scheduler.address, **opts)
                 self._created.add(worker)
                 workers.append(worker)
@@ -566,6 +584,21 @@ def adapt(
         return super().adapt(*args, minimum=minimum, maximum=maximum, **kwargs)
 
 
+async def run_spec(spec: dict, *args):
+    workers = {}
+    for k, d in spec.items():
+        cls = d["cls"]
+        if isinstance(cls, str):
+            cls = import_term(cls)
+        workers[k] = cls(*args, **d.get("opts", {}))
+
+    if workers:
+        await asyncio.gather(*workers.values())
+        for w in workers.values():
+            await w  # for tornado gen.coroutine support
+    return workers
+
+
 @atexit.register
 def close_clusters():
     for cluster in list(SpecCluster._instances):
diff --git a/distributed/deploy/tests/test_spec_cluster.py b/distributed/deploy/tests/test_spec_cluster.py
index 19e162ca67b..68642cda9d2 100644
--- a/distributed/deploy/tests/test_spec_cluster.py
+++ b/distributed/deploy/tests/test_spec_cluster.py
@@ -4,7 +4,7 @@
 
 import dask
 from dask.distributed import SpecCluster, Worker, Client, Scheduler, Nanny
-from distributed.deploy.spec import close_clusters, ProcessInterface
+from distributed.deploy.spec import close_clusters, ProcessInterface, run_spec
 from distributed.metrics import time
 from distributed.utils_test import loop, cleanup  # noqa: F401
 from distributed.utils import is_valid_xml
@@ -25,7 +25,7 @@ async def _():
 
 
 worker_spec = {
-    0: {"cls": Worker, "options": {"nthreads": 1}},
+    0: {"cls": "dask.distributed.Worker", "options": {"nthreads": 1}},
     1: {"cls": Worker, "options": {"nthreads": 2}},
     "my-worker": {"cls": MyWorker, "options": {"nthreads": 3}},
 }
@@ -429,6 +429,8 @@ async def test_MultiWorker(cleanup):
             await cluster
             assert len(cluster.worker_spec) == 2
             await client.wait_for_workers(4)
+            while len(cluster.scheduler_info["workers"]) < 4:
+                await asyncio.sleep(0.01)
 
             while "workers=4" not in repr(cluster):
                 await asyncio.sleep(0.1)
@@ -460,3 +462,17 @@ async def test_MultiWorker(cleanup):
             future = client.submit(lambda x: x + 1, 10)
             await future
             assert len(cluster.workers) == 1
+
+
+@pytest.mark.asyncio
+async def test_run_spec(cleanup):
+    async with Scheduler(port=0) as s:
+        workers = await run_spec(worker_spec, s.address)
+        async with Client(s.address, asynchronous=True) as c:
+            await c.wait_for_workers(len(worker_spec))
+
+            await asyncio.gather(*[w.close() for w in workers.values()])
+
+            assert not s.workers
+
+            await asyncio.gather(*[w.finished() for w in workers.values()])
diff --git a/distributed/utils.py b/distributed/utils.py
index 978be4eae8a..22fae745d39 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -1366,6 +1366,23 @@ def is_valid_xml(text):
 weakref.finalize(_offload_executor, _offload_executor.shutdown)
 
 
+def import_term(name: str):
+    """ Return the fully qualified term
+
+    Examples
+    --------
+    >>> import_term("math.sin")
+    <function math.sin(x, /)>
+    """
+    try:
+        module_name, attr_name = name.rsplit(".", 1)
+    except ValueError:
+        return importlib.import_module(name)
+
+    module = importlib.import_module(module_name)
+    return getattr(module, attr_name)
+
+
 async def offload(fn, *args, **kwargs):
     loop = asyncio.get_event_loop()
     return await loop.run_in_executor(_offload_executor, fn, *args, **kwargs)

From 6c3bc6ef230354ffa7b7b65bb5b72e4bfc6f4f97 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Fri, 6 Dec 2019 16:07:23 -0600
Subject: [PATCH 0596/1550] bump version to 2.9.0

---
 docs/source/changelog.rst | 39 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index 21a57806533..3cff2b92ab5 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,44 @@
 Changelog
 =========
 
+2.9.0 - 2019-12-06
+------------------
+
+- Add ``dask-spec`` CLI tool (:pr:`3090`) `Matthew Rocklin`_
+- Connectionpool: don't hand out closed connections (:pr:`3301`) `byjott`_
+- Retry operations on network issues (:pr:`3294`) `byjott`_
+- Skip ``Security.temporary()`` tests if cryptography not installed (:pr:`3302`) `James Bourbeau`_
+- Support multiple listeners in the scheduler (:pr:`3288`) `Matthew Rocklin`_
+- Updates RMM comment to the correct release (:pr:`3299`) `John Kirkham`_
+- Add title to ``performance_report`` (:pr:`3298`) `Matthew Rocklin`_
+- Forgot to fix slow test (:pr:`3297`) `Benjamin Zaitlen`_
+- Update ``SSHCluster`` docstring parameters (:pr:`3296`) `James Bourbeau`_
+- ``worker.close()`` awaits ``batched_stream.close()`` (:pr:`3291`) `Mads R. B. Kristensen`_
+- Fix asynchronous listener in UCX (:pr:`3292`) `Benjamin Zaitlen`_
+- Avoid repeatedly adding deps to already in memory stack (:pr:`3293`) `James Bourbeau`_
+- xfail ucx empty object typed dataframe (:pr:`3279`) `Benjamin Zaitlen`_
+- Fix ``distributed.wait`` documentation (:pr:`3289`) `Tom Rochette`_
+- Move Python 3 syntax tests into main tests (:pr:`3281`) `Matthew Rocklin`_
+- xfail ``test_workspace_concurrency`` for Python 3.6 (:pr:`3283`) `Matthew Rocklin`_
+- Add ``performance_report`` context manager for static report generation (:pr:`3282`) `Matthew Rocklin`_
+- Update function serialization caches with custom LRU class (:pr:`3260`) `James Bourbeau`_
+- Make ``Listener.start`` asynchronous (:pr:`3278`) `Matthew Rocklin`_
+- Remove ``dask-submit`` and ``dask-remote`` (:pr:`3280`) `Matthew Rocklin`_
+- Worker profile server (:pr:`3274`) `Matthew Rocklin`_
+- Improve bandwidth workers plot (:pr:`3273`) `Matthew Rocklin`_
+- Make profile coroutines consistent between ``Scheduler`` and ``Worker`` (:pr:`3277`) `Matthew Rocklin`_
+- Enable saving profile information from server threads (:pr:`3271`) `Matthew Rocklin`_
+- Remove memory use plot (:pr:`3269`) `Matthew Rocklin`_
+- Add offload size to configuration (:pr:`3270`) `Matthew Rocklin`_
+- Fix layout scaling on profile plots (:pr:`3268`) `Jacob Tomlinson`_
+- Set ``x_range`` in CPU plot based on the number of threads (:pr:`3266`) `Matthew Rocklin`_
+- Use base-2 values for byte-valued axes in dashboard (:pr:`3267`) `Matthew Rocklin`_
+- Robust gather in case of connection failures (:pr:`3246`) `fjetter`_
+- Use ``DeviceBuffer`` from newer RMM releases (:pr:`3261`) `John Kirkham`_
+- Fix dev requirements for pytest (:pr:`3264`) `Elliott Sales de Andrade`_
+- Add validate options to configuration (:pr:`3258`) `Matthew Rocklin`_
+
+
 2.8.1 - 2019-11-22
 ------------------
 
@@ -1408,3 +1446,4 @@ significantly without many new features.
 .. _`Jed Brown`: https://github.com/jedbrown
 .. _`He Jia`: https://github.com/HerculesJack
 .. _`Jim Crist-Harif`: https://github.com/jcrist
+.. _`fjetter`: https://github.com/fjetter

From 07fba32f1f29136478edf676d608f9ef7e08bab2 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 6 Dec 2019 20:51:41 -0800
Subject: [PATCH 0597/1550] Make ConnectionPool.close asynchronous (#3304)

Previously we would call a hard abort rather than waiting for comms to
close more gracefully.
---
 distributed/client.py          |  2 +-
 distributed/core.py            | 17 ++++++++---------
 distributed/nanny.py           |  2 +-
 distributed/scheduler.py       |  2 +-
 distributed/tests/test_core.py |  6 +++---
 distributed/worker.py          |  2 +-
 6 files changed, 15 insertions(+), 16 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 027e24afeee..0cb9594f975 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -1278,7 +1278,7 @@ async def _close(self, fast=False):
             if self._start_arg is None:
                 with ignoring(AttributeError):
                     await self.cluster.close()
-            self.rpc.close()
+            await self.rpc.close()
             self.status = "closed"
             if _get_global_client() is self:
                 _set_global_client(None)
diff --git a/distributed/core.py b/distributed/core.py
index effc96831b2..81cd7adf8e4 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -959,18 +959,17 @@ def remove(self, addr):
                 IOLoop.current().add_callback(comm.close)
                 self.semaphore.release()
 
-    def close(self):
+    async def close(self):
         """
-        Close all communications abruptly.
+        Close all communications
         """
-        for comms in self.available.values():
-            for comm in comms:
-                comm.abort()
-                self.semaphore.release()
-        for comms in self.occupied.values():
-            for comm in comms:
+        for d in [self.available, self.occupied]:
+            comms = [comm for comms in d.values() for comm in comms]
+            await asyncio.gather(
+                *[comm.close() for comm in comms], return_exceptions=True
+            )
+            for _ in comms:
                 self.semaphore.release()
-                comm.abort()
 
         for comm in self._created:
             IOLoop.current().add_callback(comm.abort)
diff --git a/distributed/nanny.py b/distributed/nanny.py
index dc2e8a3ea48..7cf3c2cbbaf 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -451,7 +451,7 @@ async def close(self, comm=None, timeout=5, report=None):
         except Exception:
             pass
         self.process = None
-        self.rpc.close()
+        await self.rpc.close()
         self.status = "closed"
         if comm:
             await comm.write("OK")
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index eeb7eb49732..b81b2545298 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1304,7 +1304,7 @@ async def close(self, comm=None, fast=False, close_workers=False):
         for comm in self.client_comms.values():
             comm.abort()
 
-        self.rpc.close()
+        await self.rpc.close()
 
         self.status = "closed"
         self.stop()
diff --git a/distributed/tests/test_core.py b/distributed/tests/test_core.py
index bda7bda2ad0..d423c6ab6c3 100644
--- a/distributed/tests/test_core.py
+++ b/distributed/tests/test_core.py
@@ -562,7 +562,7 @@ async def ping(comm, delay=0.1):
         await asyncio.sleep(0.01)
         assert time() < start + 2
 
-    rpc.close()
+    await rpc.close()
 
 
 @pytest.mark.asyncio
@@ -612,7 +612,7 @@ async def ping(comm, delay=0.01):
     await asyncio.gather(*[rpc(s.address).ping() for s in servers])
     assert rpc.active == 0
 
-    rpc.close()
+    await rpc.close()
 
 
 @pytest.mark.asyncio
@@ -651,7 +651,7 @@ async def ping(comm, delay=0.01):
     rpc.remove(serv.address)
     rpc.reuse(serv.address, comm)
 
-    rpc.close()
+    await rpc.close()
 
 
 @pytest.mark.asyncio
diff --git a/distributed/worker.py b/distributed/worker.py
index 0326cbd157a..1e36728ca78 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -1102,7 +1102,7 @@ async def close(
             self.actor_executor.shutdown(wait=executor_wait, timeout=timeout)
 
             self.stop()
-            self.rpc.close()
+            await self.rpc.close()
 
             self.status = "closed"
             await ServerNode.close(self)

From cf0767536b2b2e17afd6c849c687695c83200979 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sun, 8 Dec 2019 14:45:49 -0800
Subject: [PATCH 0598/1550] Log address for each of the Scheduler listerners
 (#3306)

---
 distributed/scheduler.py            |  3 ++-
 distributed/tests/test_scheduler.py | 36 +++++++++++++++++------------
 2 files changed, 23 insertions(+), 16 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index b81b2545298..20d0c4ed239 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1230,7 +1230,8 @@ async def start(self):
             self.start_services(listen_ip)
 
             self.status = "running"
-            logger.info("  Scheduler at: %25s", self.address)
+            for listener in self.listeners:
+                logger.info("  Scheduler at: %25s", listener.contact_address)
             for k, v in self.services.items():
                 logger.info("%11s at: %25s", k, "%s:%d" % (listen_ip, v.port))
 
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 7bf3f456085..f9087a3029a 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -5,6 +5,7 @@
 from datetime import timedelta
 import json
 import operator
+import re
 import sys
 from time import sleep
 from unittest import mock
@@ -1859,18 +1860,23 @@ def reducer(x, y):
 
 @pytest.mark.asyncio
 async def test_multiple_listeners(cleanup):
-    async with Scheduler(port=0, protocol=["inproc", "tcp"]) as s:
-        async with Worker(s.listeners[0].contact_address) as a:
-            async with Worker(s.listeners[1].contact_address) as b:
-                assert a.address.startswith("inproc")
-                assert a.scheduler.address.startswith("inproc")
-                assert b.address.startswith("tcp")
-                assert b.scheduler.address.startswith("tcp")
-
-                async with Client(s.address, asynchronous=True) as c:
-                    futures = c.map(inc, range(20))
-                    await wait(futures)
-
-                    # Force inter-worker communication both ways
-                    await c.submit(sum, futures, workers=[a.address])
-                    await c.submit(len, futures, workers=[b.address])
+    with captured_logger(logging.getLogger("distributed.scheduler")) as log:
+        async with Scheduler(port=0, protocol=["inproc", "tcp"]) as s:
+            async with Worker(s.listeners[0].contact_address) as a:
+                async with Worker(s.listeners[1].contact_address) as b:
+                    assert a.address.startswith("inproc")
+                    assert a.scheduler.address.startswith("inproc")
+                    assert b.address.startswith("tcp")
+                    assert b.scheduler.address.startswith("tcp")
+
+                    async with Client(s.address, asynchronous=True) as c:
+                        futures = c.map(inc, range(20))
+                        await wait(futures)
+
+                        # Force inter-worker communication both ways
+                        await c.submit(sum, futures, workers=[a.address])
+                        await c.submit(len, futures, workers=[b.address])
+
+    log = log.getvalue()
+    assert re.search(r"Scheduler at:\s*tcp://", log)
+    assert re.search(r"Scheduler at:\s*inproc://", log)

From e591f322cfef90663e3b45b7154e96f9cf5adf1e Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 9 Dec 2019 07:50:31 -0800
Subject: [PATCH 0599/1550] Add lock to scheduler for sensitive operations
 (#3259)

Some operations like retiring workers or rebalancing data shouldn't
happen concurrently.  Here we add an asynchronous lock around these
operations in order to protect them from each other.
---
 distributed/scheduler.py | 470 ++++++++++++++++++++-------------------
 distributed/utils.py     |  17 ++
 2 files changed, 263 insertions(+), 224 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 20d0c4ed239..275058d5221 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -57,6 +57,7 @@
     parse_bytes,
     PeriodicCallback,
     shutting_down,
+    empty_context,
     tmpfile,
 )
 from .utils_comm import scatter_to_workers, gather_from_workers, retry_operation
@@ -885,6 +886,7 @@ def __init__(
         else:
             self.idle_timeout = None
         self.time_started = time()
+        self._lock = asyncio.Lock()
         self.bandwidth = parse_bytes(dask.config.get("distributed.scheduler.bandwidth"))
         self.bandwidth_workers = defaultdict(float)
         self.bandwidth_types = defaultdict(float)
@@ -2722,118 +2724,124 @@ async def rebalance(self, comm=None, keys=None, workers=None):
         average expected load.
         """
         with log_errors():
-            if keys:
-                tasks = {self.tasks[k] for k in keys}
-                missing_data = [ts.key for ts in tasks if not ts.who_has]
-                if missing_data:
-                    return {"status": "missing-data", "keys": missing_data}
-            else:
-                tasks = set(self.tasks.values())
-
-            if workers:
-                workers = {self.workers[w] for w in workers}
-                workers_by_task = {ts: ts.who_has & workers for ts in tasks}
-            else:
-                workers = set(self.workers.values())
-                workers_by_task = {ts: ts.who_has for ts in tasks}
+            async with self._lock:
+                if keys:
+                    tasks = {self.tasks[k] for k in keys}
+                    missing_data = [ts.key for ts in tasks if not ts.who_has]
+                    if missing_data:
+                        return {"status": "missing-data", "keys": missing_data}
+                else:
+                    tasks = set(self.tasks.values())
 
-            tasks_by_worker = {ws: set() for ws in workers}
+                if workers:
+                    workers = {self.workers[w] for w in workers}
+                    workers_by_task = {ts: ts.who_has & workers for ts in tasks}
+                else:
+                    workers = set(self.workers.values())
+                    workers_by_task = {ts: ts.who_has for ts in tasks}
 
-            for k, v in workers_by_task.items():
-                for vv in v:
-                    tasks_by_worker[vv].add(k)
+                tasks_by_worker = {ws: set() for ws in workers}
 
-            worker_bytes = {
-                ws: sum(ts.get_nbytes() for ts in v)
-                for ws, v in tasks_by_worker.items()
-            }
+                for k, v in workers_by_task.items():
+                    for vv in v:
+                        tasks_by_worker[vv].add(k)
 
-            avg = sum(worker_bytes.values()) / len(worker_bytes)
+                worker_bytes = {
+                    ws: sum(ts.get_nbytes() for ts in v)
+                    for ws, v in tasks_by_worker.items()
+                }
 
-            sorted_workers = list(
-                map(first, sorted(worker_bytes.items(), key=second, reverse=True))
-            )
+                avg = sum(worker_bytes.values()) / len(worker_bytes)
 
-            recipients = iter(reversed(sorted_workers))
-            recipient = next(recipients)
-            msgs = []  # (sender, recipient, key)
-            for sender in sorted_workers[: len(workers) // 2]:
-                sender_keys = {ts: ts.get_nbytes() for ts in tasks_by_worker[sender]}
-                sender_keys = iter(
-                    sorted(sender_keys.items(), key=second, reverse=True)
+                sorted_workers = list(
+                    map(first, sorted(worker_bytes.items(), key=second, reverse=True))
                 )
 
-                try:
-                    while worker_bytes[sender] > avg:
-                        while (
-                            worker_bytes[recipient] < avg and worker_bytes[sender] > avg
-                        ):
-                            ts, nb = next(sender_keys)
-                            if ts not in tasks_by_worker[recipient]:
-                                tasks_by_worker[recipient].add(ts)
-                                # tasks_by_worker[sender].remove(ts)
-                                msgs.append((sender, recipient, ts))
-                                worker_bytes[sender] -= nb
-                                worker_bytes[recipient] += nb
-                        if worker_bytes[sender] > avg:
-                            recipient = next(recipients)
-                except StopIteration:
-                    break
+                recipients = iter(reversed(sorted_workers))
+                recipient = next(recipients)
+                msgs = []  # (sender, recipient, key)
+                for sender in sorted_workers[: len(workers) // 2]:
+                    sender_keys = {
+                        ts: ts.get_nbytes() for ts in tasks_by_worker[sender]
+                    }
+                    sender_keys = iter(
+                        sorted(sender_keys.items(), key=second, reverse=True)
+                    )
+
+                    try:
+                        while worker_bytes[sender] > avg:
+                            while (
+                                worker_bytes[recipient] < avg
+                                and worker_bytes[sender] > avg
+                            ):
+                                ts, nb = next(sender_keys)
+                                if ts not in tasks_by_worker[recipient]:
+                                    tasks_by_worker[recipient].add(ts)
+                                    # tasks_by_worker[sender].remove(ts)
+                                    msgs.append((sender, recipient, ts))
+                                    worker_bytes[sender] -= nb
+                                    worker_bytes[recipient] += nb
+                            if worker_bytes[sender] > avg:
+                                recipient = next(recipients)
+                    except StopIteration:
+                        break
 
-            to_recipients = defaultdict(lambda: defaultdict(list))
-            to_senders = defaultdict(list)
-            for sender, recipient, ts in msgs:
-                to_recipients[recipient.address][ts.key].append(sender.address)
-                to_senders[sender.address].append(ts.key)
+                to_recipients = defaultdict(lambda: defaultdict(list))
+                to_senders = defaultdict(list)
+                for sender, recipient, ts in msgs:
+                    to_recipients[recipient.address][ts.key].append(sender.address)
+                    to_senders[sender.address].append(ts.key)
 
-            result = await asyncio.gather(
-                *(
-                    retry_operation(self.rpc(addr=r).gather, who_has=v)
-                    for r, v in to_recipients.items()
+                result = await asyncio.gather(
+                    *(
+                        retry_operation(self.rpc(addr=r).gather, who_has=v)
+                        for r, v in to_recipients.items()
+                    )
                 )
-            )
-            for r, v in to_recipients.items():
-                self.log_event(r, {"action": "rebalance", "who_has": v})
+                for r, v in to_recipients.items():
+                    self.log_event(r, {"action": "rebalance", "who_has": v})
 
-            self.log_event(
-                "all",
-                {
-                    "action": "rebalance",
-                    "total-keys": len(tasks),
-                    "senders": valmap(len, to_senders),
-                    "recipients": valmap(len, to_recipients),
-                    "moved_keys": len(msgs),
-                },
-            )
+                self.log_event(
+                    "all",
+                    {
+                        "action": "rebalance",
+                        "total-keys": len(tasks),
+                        "senders": valmap(len, to_senders),
+                        "recipients": valmap(len, to_recipients),
+                        "moved_keys": len(msgs),
+                    },
+                )
 
-            if not all(r["status"] == "OK" for r in result):
-                return {
-                    "status": "missing-data",
-                    "keys": sum([r["keys"] for r in result if "keys" in r], []),
-                }
+                if not all(r["status"] == "OK" for r in result):
+                    return {
+                        "status": "missing-data",
+                        "keys": sum([r["keys"] for r in result if "keys" in r], []),
+                    }
 
-            for sender, recipient, ts in msgs:
-                assert ts.state == "memory"
-                ts.who_has.add(recipient)
-                recipient.has_what.add(ts)
-                recipient.nbytes += ts.get_nbytes()
-                self.log.append(
-                    ("rebalance", ts.key, time(), sender.address, recipient.address)
-                )
+                for sender, recipient, ts in msgs:
+                    assert ts.state == "memory"
+                    ts.who_has.add(recipient)
+                    recipient.has_what.add(ts)
+                    recipient.nbytes += ts.get_nbytes()
+                    self.log.append(
+                        ("rebalance", ts.key, time(), sender.address, recipient.address)
+                    )
 
-            await asyncio.gather(
-                *(
-                    retry_operation(self.rpc(addr=r).delete_data, keys=v, report=False)
-                    for r, v in to_senders.items()
+                await asyncio.gather(
+                    *(
+                        retry_operation(
+                            self.rpc(addr=r).delete_data, keys=v, report=False
+                        )
+                        for r, v in to_senders.items()
+                    )
                 )
-            )
 
-            for sender, recipient, ts in msgs:
-                ts.who_has.remove(sender)
-                sender.has_what.remove(ts)
-                sender.nbytes -= ts.get_nbytes()
+                for sender, recipient, ts in msgs:
+                    ts.who_has.remove(sender)
+                    sender.has_what.remove(ts)
+                    sender.nbytes -= ts.get_nbytes()
 
-            return {"status": "OK"}
+                return {"status": "OK"}
 
     async def replicate(
         self,
@@ -2843,6 +2851,7 @@ async def replicate(
         workers=None,
         branching_factor=2,
         delete=True,
+        lock=True,
     ):
         """ Replicate data throughout cluster
 
@@ -2866,89 +2875,96 @@ async def replicate(
         Scheduler.rebalance
         """
         assert branching_factor > 0
+        async with self._lock if lock else empty_context:
+            workers = {self.workers[w] for w in self.workers_list(workers)}
+            if n is None:
+                n = len(workers)
+            else:
+                n = min(n, len(workers))
+            if n == 0:
+                raise ValueError("Can not use replicate to delete data")
+
+            tasks = {self.tasks[k] for k in keys}
+            missing_data = [ts.key for ts in tasks if not ts.who_has]
+            if missing_data:
+                return {"status": "missing-data", "keys": missing_data}
+
+            # Delete extraneous data
+            if delete:
+                del_worker_tasks = defaultdict(set)
+                for ts in tasks:
+                    del_candidates = ts.who_has & workers
+                    if len(del_candidates) > n:
+                        for ws in random.sample(
+                            del_candidates, len(del_candidates) - n
+                        ):
+                            del_worker_tasks[ws].add(ts)
 
-        workers = {self.workers[w] for w in self.workers_list(workers)}
-        if n is None:
-            n = len(workers)
-        else:
-            n = min(n, len(workers))
-        if n == 0:
-            raise ValueError("Can not use replicate to delete data")
-
-        tasks = {self.tasks[k] for k in keys}
-        missing_data = [ts.key for ts in tasks if not ts.who_has]
-        if missing_data:
-            return {"status": "missing-data", "keys": missing_data}
-
-        # Delete extraneous data
-        if delete:
-            del_worker_tasks = defaultdict(set)
-            for ts in tasks:
-                del_candidates = ts.who_has & workers
-                if len(del_candidates) > n:
-                    for ws in random.sample(del_candidates, len(del_candidates) - n):
-                        del_worker_tasks[ws].add(ts)
-
-            await asyncio.gather(
-                *(
-                    retry_operation(
-                        self.rpc(addr=ws.address).delete_data,
-                        keys=[ts.key for ts in tasks],
-                        report=False,
+                await asyncio.gather(
+                    *(
+                        retry_operation(
+                            self.rpc(addr=ws.address).delete_data,
+                            keys=[ts.key for ts in tasks],
+                            report=False,
+                        )
+                        for ws, tasks in del_worker_tasks.items()
                     )
-                    for ws, tasks in del_worker_tasks.items()
                 )
-            )
 
-            for ws, tasks in del_worker_tasks.items():
-                ws.has_what -= tasks
-                for ts in tasks:
-                    ts.who_has.remove(ws)
-                    ws.nbytes -= ts.get_nbytes()
-                self.log_event(
-                    ws.address,
-                    {"action": "replicate-remove", "keys": [ts.key for ts in tasks]},
-                )
+                for ws, tasks in del_worker_tasks.items():
+                    ws.has_what -= tasks
+                    for ts in tasks:
+                        ts.who_has.remove(ws)
+                        ws.nbytes -= ts.get_nbytes()
+                    self.log_event(
+                        ws.address,
+                        {
+                            "action": "replicate-remove",
+                            "keys": [ts.key for ts in tasks],
+                        },
+                    )
 
-        # Copy not-yet-filled data
-        while tasks:
-            gathers = defaultdict(dict)
-            for ts in list(tasks):
-                n_missing = n - len(ts.who_has & workers)
-                if n_missing <= 0:
-                    # Already replicated enough
-                    tasks.remove(ts)
-                    continue
+            # Copy not-yet-filled data
+            while tasks:
+                gathers = defaultdict(dict)
+                for ts in list(tasks):
+                    n_missing = n - len(ts.who_has & workers)
+                    if n_missing <= 0:
+                        # Already replicated enough
+                        tasks.remove(ts)
+                        continue
 
-                count = min(n_missing, branching_factor * len(ts.who_has))
-                assert count > 0
+                    count = min(n_missing, branching_factor * len(ts.who_has))
+                    assert count > 0
 
-                for ws in random.sample(workers - ts.who_has, count):
-                    gathers[ws.address][ts.key] = [wws.address for wws in ts.who_has]
+                    for ws in random.sample(workers - ts.who_has, count):
+                        gathers[ws.address][ts.key] = [
+                            wws.address for wws in ts.who_has
+                        ]
 
-            results = await asyncio.gather(
-                *(
-                    retry_operation(self.rpc(addr=w).gather, who_has=who_has)
-                    for w, who_has in gathers.items()
+                results = await asyncio.gather(
+                    *(
+                        retry_operation(self.rpc(addr=w).gather, who_has=who_has)
+                        for w, who_has in gathers.items()
+                    )
                 )
-            )
-            for w, v in zip(gathers, results):
-                if v["status"] == "OK":
-                    self.add_keys(worker=w, keys=list(gathers[w]))
-                else:
-                    logger.warning("Communication failed during replication: %s", v)
+                for w, v in zip(gathers, results):
+                    if v["status"] == "OK":
+                        self.add_keys(worker=w, keys=list(gathers[w]))
+                    else:
+                        logger.warning("Communication failed during replication: %s", v)
 
-                self.log_event(w, {"action": "replicate-add", "keys": gathers[w]})
+                    self.log_event(w, {"action": "replicate-add", "keys": gathers[w]})
 
-        self.log_event(
-            "all",
-            {
-                "action": "replicate",
-                "workers": list(workers),
-                "key-count": len(keys),
-                "branching-factor": branching_factor,
-            },
-        )
+            self.log_event(
+                "all",
+                {
+                    "action": "replicate",
+                    "workers": list(workers),
+                    "key-count": len(keys),
+                    "branching-factor": branching_factor,
+                },
+            )
 
     def workers_to_close(
         self,
@@ -3090,6 +3106,7 @@ async def retire_workers(
         remove=True,
         close_workers=False,
         names=None,
+        lock=True,
         **kwargs
     ):
         """ Gracefully retire workers from cluster
@@ -3122,68 +3139,73 @@ async def retire_workers(
         Scheduler.workers_to_close
         """
         with log_errors():
-            if names is not None:
-                if names:
-                    logger.info("Retire worker names %s", names)
-                names = set(map(str, names))
-                workers = [
-                    ws.address for ws in self.workers.values() if str(ws.name) in names
-                ]
-            if workers is None:
-                while True:
-                    try:
-                        workers = self.workers_to_close(**kwargs)
-                        if workers:
-                            workers = await self.retire_workers(
-                                workers=workers,
-                                remove=remove,
-                                close_workers=close_workers,
-                            )
-                        return workers
-                    except KeyError:  # keys left during replicate
-                        pass
-            workers = {self.workers[w] for w in workers if w in self.workers}
-            if not workers:
-                return []
-            logger.info("Retire workers %s", workers)
-
-            # Keys orphaned by retiring those workers
-            keys = set.union(*[w.has_what for w in workers])
-            keys = {ts.key for ts in keys if ts.who_has.issubset(workers)}
-
-            other_workers = set(self.workers.values()) - workers
-            if keys:
-                if other_workers:
-                    logger.info("Moving %d keys to other workers", len(keys))
-                    await self.replicate(
-                        keys=keys,
-                        workers=[ws.address for ws in other_workers],
-                        n=1,
-                        delete=False,
-                    )
-                else:
+            async with self._lock if lock else empty_context:
+                if names is not None:
+                    if names:
+                        logger.info("Retire worker names %s", names)
+                    names = set(map(str, names))
+                    workers = [
+                        ws.address
+                        for ws in self.workers.values()
+                        if str(ws.name) in names
+                    ]
+                if workers is None:
+                    while True:
+                        try:
+                            workers = self.workers_to_close(**kwargs)
+                            if workers:
+                                workers = await self.retire_workers(
+                                    workers=workers,
+                                    remove=remove,
+                                    close_workers=close_workers,
+                                    lock=False,
+                                )
+                            return workers
+                        except KeyError:  # keys left during replicate
+                            pass
+                workers = {self.workers[w] for w in workers if w in self.workers}
+                if not workers:
                     return []
+                logger.info("Retire workers %s", workers)
+
+                # Keys orphaned by retiring those workers
+                keys = set.union(*[w.has_what for w in workers])
+                keys = {ts.key for ts in keys if ts.who_has.issubset(workers)}
+
+                other_workers = set(self.workers.values()) - workers
+                if keys:
+                    if other_workers:
+                        logger.info("Moving %d keys to other workers", len(keys))
+                        await self.replicate(
+                            keys=keys,
+                            workers=[ws.address for ws in other_workers],
+                            n=1,
+                            delete=False,
+                            lock=False,
+                        )
+                    else:
+                        return []
 
-            worker_keys = {ws.address: ws.identity() for ws in workers}
-            if close_workers and worker_keys:
-                await asyncio.gather(
-                    *[self.close_worker(worker=w, safe=True) for w in worker_keys]
-                )
-            if remove:
-                for w in worker_keys:
-                    self.remove_worker(address=w, safe=True)
+                worker_keys = {ws.address: ws.identity() for ws in workers}
+                if close_workers and worker_keys:
+                    await asyncio.gather(
+                        *[self.close_worker(worker=w, safe=True) for w in worker_keys]
+                    )
+                if remove:
+                    for w in worker_keys:
+                        self.remove_worker(address=w, safe=True)
 
-            self.log_event(
-                "all",
-                {
-                    "action": "retire-workers",
-                    "workers": worker_keys,
-                    "moved-keys": len(keys),
-                },
-            )
-            self.log_event(list(worker_keys), {"action": "retired"})
+                self.log_event(
+                    "all",
+                    {
+                        "action": "retire-workers",
+                        "workers": worker_keys,
+                        "moved-keys": len(keys),
+                    },
+                )
+                self.log_event(list(worker_keys), {"action": "retired"})
 
-            return worker_keys
+                return worker_keys
 
     def add_keys(self, comm=None, worker=None, keys=()):
         """
diff --git a/distributed/utils.py b/distributed/utils.py
index 22fae745d39..cfb0ee921d6 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -1418,6 +1418,23 @@ def deserialize_for_cli(data):
     return json.loads(base64.urlsafe_b64decode(data.encode()).decode())
 
 
+class EmptyContext:
+    def __enter__(self):
+        pass
+
+    def __exit__(self, *args):
+        pass
+
+    async def __aenter__(self):
+        pass
+
+    async def __aexit__(self, *args):
+        pass
+
+
+empty_context = EmptyContext()
+
+
 class LRU(UserDict):
     """ Limited size mapping, evicting the least recently looked-up key when full
     """

From 3116655cd551fe84b9759220a6f34c299a8c36a6 Mon Sep 17 00:00:00 2001
From: Tom Augspurger <TomAugspurger@users.noreply.github.com>
Date: Mon, 9 Dec 2019 16:14:17 -0600
Subject: [PATCH 0600/1550] Clean up flaky test_nanny_throttle (#3295)

---
 distributed/tests/test_nanny.py | 56 +++++++++++----------------------
 1 file changed, 18 insertions(+), 38 deletions(-)

diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
index cacd98477e0..68f207a51ce 100644
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -305,54 +305,34 @@ def leak():
 @gen_cluster(
     nthreads=[("127.0.0.1", 1)] * 8,
     client=True,
-    Worker=Nanny,
-    worker_kwargs={"memory_limit": 2e8},
-    timeout=20,
+    Worker=Worker,
     clean_kwargs={"threads": False},
 )
-async def test_nanny_throttle(c, s, *workers):
-    # Verify that get_data requests are throttled when the worker
-    # with the data is at high-memory by
-    # 1. Allocation some data on a worker
-    # 2. Pausing that worker
-    # 3. Requesting data from that worker from many other workers
-    a = workers[0]
-    proc = a.process.pid
-    size = 1000
-
-    def data(size):
-        return b"0" * size
+async def test_throttle_outgoing_connections(c, s, a, *workers):
+    # But a bunch of small data on worker a
+    await c.run(lambda: logging.getLogger("distributed.worker").setLevel(logging.DEBUG))
+    remote_data = c.map(
+        lambda x: b"0" * 10000, range(10), pure=False, workers=[a.address]
+    )
+    await wait(remote_data)
 
-    def patch(dask_worker):
+    def pause(dask_worker):
         # Patch paused and memory_monitor on the one worker
         # This is is very fragile, since a refactor of memory_monitor to
         # remove _memory_monitoring will break this test.
         dask_worker._memory_monitoring = True
         dask_worker.paused = True
+        dask_worker.outgoing_current_count = 2
 
-    def check(dask_worker):
-        return dask_worker.paused
-
-    futures = [
-        c.submit(data, size, workers=[a.worker_address], pure=False) for i in range(4)
+    await c.run(pause, workers=[a.address])
+    requests = [
+        await a.get_data(await w.rpc.connect(w.address), keys=[f.key], who=w.address)
+        for w in workers
+        for f in remote_data
     ]
-    await wait(futures)
-    await c.run(patch, workers=[a.worker_address])
-    paused = await c.run(check, workers=[a.worker_address])
-    assert paused[a.worker_address]
-
-    await c.run(lambda: logging.getLogger("distributed.worker").setLevel(logging.DEBUG))
-    # Cluster is in the correct state, now for the test.
-    n = len(workers)
-    result = c.map(
-        lambda x, i: x[i],
-        [futures[0]] * n,
-        range(n),
-        workers=[w.worker_address for w in workers[1:]],
-    )
-    await result[0]
-    wlogs = await c.get_worker_logs(workers=[a.worker_address])
-    wlogs = "\n".join(x[1] for x in wlogs[a.worker_address])
+    await wait(requests)
+    wlogs = await c.get_worker_logs(workers=[a.address])
+    wlogs = "\n".join(x[1] for x in wlogs[a.address])
     assert "throttling" in wlogs.lower()
 
 

From 3151f09e883e9e93f8e06dd0ed9db266172ba479 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 10 Dec 2019 08:27:22 -0800
Subject: [PATCH 0601/1550] Use hostname as default IP address rather than
 localhost (#3308)

Previously if we couldn't connect to the scheduler we used localhost
this made sense for testing, but probably doesn't make sense for
operations.
---
 distributed/utils.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/distributed/utils.py b/distributed/utils.py
index cfb0ee921d6..26c503205aa 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -118,7 +118,7 @@ def get_fileno_limit():
 
 
 @toolz.memoize
-def _get_ip(host, port, family, default):
+def _get_ip(host, port, family):
     # By using a UDP socket, we don't actually try to connect but
     # simply select the local address through which *host* is reachable.
     sock = socket.socket(family, socket.SOCK_DGRAM)
@@ -130,10 +130,10 @@ def _get_ip(host, port, family, default):
         # XXX Should first try getaddrinfo() on socket.gethostname() and getfqdn()
         warnings.warn(
             "Couldn't detect a suitable IP address for "
-            "reaching %r, defaulting to %r: %s" % (host, default, e),
+            "reaching %r, defaulting to hostname: %s" % (host, e),
             RuntimeWarning,
         )
-        return default
+        return socket.gethostname()
     finally:
         sock.close()
 
@@ -145,14 +145,14 @@ def get_ip(host="8.8.8.8", port=80):
     *host* defaults to a well-known Internet host (one of Google's public
     DNS servers).
     """
-    return _get_ip(host, port, family=socket.AF_INET, default="127.0.0.1")
+    return _get_ip(host, port, family=socket.AF_INET)
 
 
 def get_ipv6(host="2001:4860:4860::8888", port=80):
     """
     The same as get_ip(), but for IPv6.
     """
-    return _get_ip(host, port, family=socket.AF_INET6, default="::1")
+    return _get_ip(host, port, family=socket.AF_INET6)
 
 
 def get_ip_interface(ifname):

From b92782d0eb8fecedeaf650f188e65ea4ef40de32 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Tue, 10 Dec 2019 10:28:44 -0600
Subject: [PATCH 0602/1550] Update inlining Futures in task graph in
 Client._graph_to_futures (#3303)

* Use pack_data to inline Futures

* Add subs_mutliple

* Check key mapping for keys to substitue

* Avoid unnecessary hash attempts
---
 distributed/client.py                |  7 ++++--
 distributed/tests/test_utils_comm.py | 16 +++++++++++++-
 distributed/utils_comm.py            | 32 ++++++++++++++++++++++++++++
 3 files changed, 52 insertions(+), 3 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 0cb9594f975..5ff715281ef 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -51,6 +51,7 @@
     WrappedKey,
     unpack_remotedata,
     pack_data,
+    subs_multiple,
     scatter_to_workers,
     gather_from_workers,
     retry_operation,
@@ -2435,10 +2436,12 @@ def _graph_to_futures(
             futures = {key: Future(key, self, inform=False) for key in keyset}
 
             values = {
-                k for k, v in dsk.items() if isinstance(v, Future) and k not in keyset
+                k: v
+                for k, v in dsk.items()
+                if isinstance(v, Future) and k not in keyset
             }
             if values:
-                dsk = dask.optimization.inline(dsk, keys=values)
+                dsk = subs_multiple(dsk, values)
 
             d = {k: unpack_remotedata(v, byte_keys=True) for k, v in dsk.items()}
             extra_futures = set.union(*[v[1] for v in d.values()]) if d else set()
diff --git a/distributed/tests/test_utils_comm.py b/distributed/tests/test_utils_comm.py
index 3f26fae623a..2d0159a2d3d 100644
--- a/distributed/tests/test_utils_comm.py
+++ b/distributed/tests/test_utils_comm.py
@@ -1,7 +1,7 @@
 from distributed.core import ConnectionPool
 from distributed.comm import Comm
 from distributed.utils_test import gen_cluster, loop  # noqa: F401
-from distributed.utils_comm import pack_data, gather_from_workers, retry
+from distributed.utils_comm import pack_data, subs_multiple, gather_from_workers, retry
 
 from unittest import mock
 
@@ -15,6 +15,20 @@ def test_pack_data():
     assert pack_data({"a": ["x"], "b": "y"}, data) == {"a": [1], "b": "y"}
 
 
+def test_subs_multiple():
+    data = {"x": 1, "y": 2}
+    assert subs_multiple((sum, [0, "x", "y", "z"]), data) == (sum, [0, 1, 2, "z"])
+    assert subs_multiple((sum, [0, ["x", "y", "z"]]), data) == (sum, [0, [1, 2, "z"]])
+
+    dsk = {"a": (sum, ["x", "y"])}
+    assert subs_multiple(dsk, data) == {"a": (sum, [1, 2])}
+
+    # Tuple key
+    data = {"x": 1, ("y", 0): 2}
+    dsk = {"a": (sum, ["x", ("y", 0)])}
+    assert subs_multiple(dsk, data) == {"a": (sum, [1, 2])}
+
+
 @gen_cluster(client=True)
 def test_gather_from_workers_permissive(c, s, a, b):
     rpc = ConnectionPool()
diff --git a/distributed/utils_comm.py b/distributed/utils_comm.py
index cb614602f7b..792e73227a9 100644
--- a/distributed/utils_comm.py
+++ b/distributed/utils_comm.py
@@ -280,6 +280,38 @@ def pack_data(o, d, key_types=object):
         return o
 
 
+def subs_multiple(o, d):
+    """ Perform substitutions on a tasks
+
+    Parameters
+    ----------
+    o:
+        Core data structures containing literals and keys
+    d: dict
+        Mapping of keys to values
+
+    Examples
+    --------
+    >>> dsk = {"a": (sum, ["x", 2])}
+    >>> data = {"x": 1}
+    >>> subs_multiple(dsk, data)  # doctest: +SKIP
+    {'a': (sum, [1, 2])}
+
+    """
+    typ = type(o)
+    if typ is tuple and o and callable(o[0]):  # istask(o)
+        return (o[0],) + tuple(subs_multiple(i, d) for i in o[1:])
+    elif typ is list:
+        return [subs_multiple(i, d) for i in o]
+    elif typ is dict:
+        return {k: subs_multiple(v, d) for (k, v) in o.items()}
+    else:
+        try:
+            return d.get(o, o)
+        except TypeError:
+            return o
+
+
 retry_count = dask.config.get("distributed.comm.retry.count")
 retry_delay_min = parse_timedelta(
     dask.config.get("distributed.comm.retry.delay.min"), default="s"

From 246eb9b4cf62430b3b9bc9cc1a64534bbd726730 Mon Sep 17 00:00:00 2001
From: fjetter <fjetter@users.noreply.github.com>
Date: Tue, 10 Dec 2019 18:55:41 +0100
Subject: [PATCH 0603/1550] Update latencies with heartbeats (#3310)

---
 distributed/tests/test_worker.py | 12 ++++++++++++
 distributed/worker.py            |  9 ++++++++-
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index f67701f671a..2cf316ccbca 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -1586,3 +1586,15 @@ def bad_startup(w):
             w = await Worker(s.address, startup_information={"bad": bad_startup})
         except Exception:
             pytest.fail("Startup exception was raised")
+
+
+@pytest.mark.asyncio
+async def test_update_latency(cleanup):
+    async with await Scheduler() as s:
+        async with await Worker(s.address) as w:
+            original = w.latency
+            await w.heartbeat()
+            assert original != w.latency
+
+            if w.digests is not None:
+                assert w.digests["latency"].size() > 0
diff --git a/distributed/worker.py b/distributed/worker.py
index 1e36728ca78..751fabce2b1 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -828,7 +828,7 @@ async def _register_with_scheduler(self):
                 response = await future
                 _end = time()
                 middle = (_start + _end) / 2
-                self.latency = (_end - start) * 0.05 + self.latency * 0.95
+                self._update_latency(_end - start)
                 self.scheduler_delay = response["time"] - middle
                 self.status = "running"
                 break
@@ -862,6 +862,11 @@ async def _register_with_scheduler(self):
         self.periodic_callbacks["heartbeat"].start()
         self.loop.add_callback(self.handle_scheduler, comm)
 
+    def _update_latency(self, latency):
+        self.latency = latency * 0.05 + self.latency * 0.95
+        if self.digests is not None:
+            self.digests["latency"].add(latency)
+
     async def heartbeat(self):
         if not self.heartbeat_active:
             self.heartbeat_active = True
@@ -877,6 +882,8 @@ async def heartbeat(self):
                 end = time()
                 middle = (start + end) / 2
 
+                self._update_latency(end - start)
+
                 if response["status"] == "missing":
                     await self._register_with_scheduler()
                     return

From 61238adb12d21985299fccd0f026bd42366d163f Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 11 Dec 2019 08:49:46 -0800
Subject: [PATCH 0604/1550] Add TaskGroup and TaskPrefix scheduler state
 (#3262)

This aggregates task information into hierarchies.
This should be helpful in improving both diagnostics and dashboards,
particularly when the number of tasks increases.
---
 distributed/dashboard/components/scheduler.py |  33 ++-
 distributed/deploy/tests/test_adaptive.py     | 126 ++++----
 distributed/diagnostics/progress.py           |   4 +-
 distributed/distributed.yaml                  |   4 +-
 distributed/scheduler.py                      | 276 +++++++++++++++---
 distributed/tests/test_client.py              |  24 +-
 distributed/tests/test_nanny.py               |  17 +-
 distributed/tests/test_scheduler.py           | 175 +++++++----
 distributed/tests/test_steal.py               |  40 ++-
 distributed/utils.py                          |  20 +-
 10 files changed, 506 insertions(+), 213 deletions(-)

diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index 2b27c708111..ad7cecea024 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -60,14 +60,13 @@
 from distributed.metrics import time
 from distributed.utils import log_errors, format_time, parse_timedelta
 from distributed.diagnostics.progress_stream import color_of, progress_quads
-from distributed.diagnostics.progress import AllProgress
 from distributed.diagnostics.graph_layout import GraphLayout
 from distributed.diagnostics.task_stream import TaskStreamPlugin
 
 try:
-    from cytoolz.curried import map, concat, groupby, valmap
+    from cytoolz.curried import map, concat, groupby
 except ImportError:
-    from toolz.curried import map, concat, groupby, valmap
+    from toolz.curried import map, concat, groupby
 
 if dask.config.get("distributed.dashboard.export-tool"):
     from distributed.dashboard.export_tool import ExportTool
@@ -1283,11 +1282,6 @@ class TaskProgress(DashboardComponent):
 
     def __init__(self, scheduler, **kwargs):
         self.scheduler = scheduler
-        ps = [p for p in scheduler.plugins if isinstance(p, AllProgress)]
-        if ps:
-            self.plugin = ps[0]
-        else:
-            self.plugin = AllProgress(scheduler)
 
         data = progress_quads(
             dict(all={}, memory={}, erred={}, released={}, processing={})
@@ -1415,9 +1409,26 @@ def __init__(self, scheduler, **kwargs):
     @without_property_validation
     def update(self):
         with log_errors():
-            state = {"all": valmap(len, self.plugin.all), "nbytes": self.plugin.nbytes}
-            for k in ["memory", "erred", "released", "processing", "waiting"]:
-                state[k] = valmap(len, self.plugin.state[k])
+            state = {
+                "memory": {},
+                "erred": {},
+                "released": {},
+                "processing": {},
+                "waiting": {},
+            }
+
+            for tp in self.scheduler.task_prefixes.values():
+                if any(tp.active_states.values()):
+                    state["memory"][tp.name] = tp.active_states["memory"]
+                    state["erred"][tp.name] = tp.active_states["erred"]
+                    state["released"][tp.name] = tp.active_states["released"]
+                    state["processing"][tp.name] = tp.active_states["processing"]
+                    state["waiting"][tp.name] = tp.active_states["waiting"]
+
+            state["all"] = {
+                k: sum(v[k] for v in state.values()) for k in state["memory"]
+            }
+
             if not state["all"] and not len(self.source.data["all"]):
                 return
 
diff --git a/distributed/deploy/tests/test_adaptive.py b/distributed/deploy/tests/test_adaptive.py
index af198747822..90f56c4bfde 100644
--- a/distributed/deploy/tests/test_adaptive.py
+++ b/distributed/deploy/tests/test_adaptive.py
@@ -1,8 +1,8 @@
 from time import sleep
 
+import dask
 import pytest
 from tornado import gen
-from tornado.ioloop import IOLoop
 
 from distributed import Client, wait, Adaptive, LocalCluster, SpecCluster, Worker
 from distributed.utils_test import gen_test, slowinc, clean
@@ -26,21 +26,25 @@ def scale_up(self, n, **kwargs):
         def scale_down(self, workers):
             assert False
 
-    async with TestCluster(n_workers=4, processes=False, asynchronous=True) as cluster:
-        async with Client(cluster, asynchronous=True) as c:
-            s = cluster.scheduler
-            s.task_duration["a"] = 4
-            s.task_duration["b"] = 4
-            s.task_duration["c"] = 1
+    with dask.config.set(
+        {"distributed.scheduler.default-task-durations": {"a": 4, "b": 4, "c": 1}}
+    ):
+        async with TestCluster(
+            n_workers=4, processes=False, asynchronous=True
+        ) as cluster:
+            async with Client(cluster, asynchronous=True) as c:
+                s = cluster.scheduler
 
-            future = c.map(slowinc, [1, 1, 1], key=["a-4", "b-4", "c-1"])
+                future = c.map(slowinc, [1, 1, 1], key=["a-4", "b-4", "c-1"])
 
-            while len(s.rprocessing) < 3:
-                await gen.sleep(0.001)
+                while len(s.rprocessing) < 3:
+                    await gen.sleep(0.001)
 
-            ta = cluster.adapt(interval="100 ms", scale_factor=2, Adaptive=TestAdaptive)
+                ta = cluster.adapt(
+                    interval="100 ms", scale_factor=2, Adaptive=TestAdaptive
+                )
 
-            await gen.sleep(0.3)
+                await gen.sleep(0.3)
 
 
 def test_adaptive_local_cluster(loop):
@@ -298,32 +302,25 @@ def test_adapt_down():
 
 
 @gen_test(timeout=30)
-def test_no_more_workers_than_tasks():
-    loop = IOLoop.current()
-    cluster = yield LocalCluster(
-        0,
-        scheduler_port=0,
-        silence_logs=False,
-        processes=False,
-        dashboard_address=None,
-        loop=loop,
-        asynchronous=True,
-    )
-    yield cluster._start()
-    try:
-        adapt = cluster.adapt(minimum=0, maximum=4, interval="10 ms")
-        client = yield Client(cluster, asynchronous=True, loop=loop)
-        cluster.scheduler.task_duration["slowinc"] = 1000
-
-        yield client.submit(slowinc, 1, delay=0.100)
-
-        assert len(cluster.scheduler.workers) <= 1
-    finally:
-        yield client.close()
-        yield cluster.close()
-
-
-def test_basic_no_loop():
+async def test_no_more_workers_than_tasks():
+    with dask.config.set(
+        {"distributed.scheduler.default-task-durations": {"slowinc": 1000}}
+    ):
+        async with LocalCluster(
+            0,
+            scheduler_port=0,
+            silence_logs=False,
+            processes=False,
+            dashboard_address=None,
+            asynchronous=True,
+        ) as cluster:
+            adapt = cluster.adapt(minimum=0, maximum=4, interval="10 ms")
+            async with Client(cluster, asynchronous=True) as client:
+                await client.submit(slowinc, 1, delay=0.100)
+                assert len(cluster.scheduler.workers) <= 1
+
+
+def test_basic_no_loop(loop):
     with clean(threads=False):
         try:
             with LocalCluster(
@@ -339,36 +336,31 @@ def test_basic_no_loop():
 
 
 @gen_test(timeout=None)
-def test_target_duration():
+async def test_target_duration():
     """ Ensure that redefining adapt with a lower maximum removes workers """
-    cluster = yield LocalCluster(
-        0,
-        asynchronous=True,
-        processes=False,
-        scheduler_port=0,
-        silence_logs=False,
-        dashboard_address=None,
-    )
-    client = yield Client(cluster, asynchronous=True)
-    adapt = cluster.adapt(interval="20ms", minimum=2, target_duration="5s")
-
-    cluster.scheduler.task_duration["slowinc"] = 1
-
-    try:
-        while len(cluster.scheduler.workers) < 2:
-            yield gen.sleep(0.01)
-
-        futures = client.map(slowinc, range(100), delay=0.3)
-
-        while len(adapt.log) < 2:
-            yield gen.sleep(0.01)
-
-        assert adapt.log[0][1] == {"status": "up", "n": 2}
-        assert adapt.log[1][1] == {"status": "up", "n": 20}
-
-    finally:
-        yield client.close()
-        yield cluster.close()
+    with dask.config.set(
+        {"distributed.scheduler.default-task-durations": {"slowinc": 1}}
+    ):
+        async with LocalCluster(
+            0,
+            asynchronous=True,
+            processes=False,
+            scheduler_port=0,
+            silence_logs=False,
+            dashboard_address=None,
+        ) as cluster:
+            adapt = cluster.adapt(interval="20ms", minimum=2, target_duration="5s")
+            async with Client(cluster, asynchronous=True) as client:
+                while len(cluster.scheduler.workers) < 2:
+                    await gen.sleep(0.01)
+
+                futures = client.map(slowinc, range(100), delay=0.3)
+
+                while len(adapt.log) < 2:
+                    await gen.sleep(0.01)
+
+                assert adapt.log[0][1] == {"status": "up", "n": 2}
+                assert adapt.log[1][1] == {"status": "up", "n": 20}
 
 
 @pytest.mark.asyncio
diff --git a/distributed/diagnostics/progress.py b/distributed/diagnostics/progress.py
index 48f26570980..1dcab0dc9e9 100644
--- a/distributed/diagnostics/progress.py
+++ b/distributed/diagnostics/progress.py
@@ -246,7 +246,7 @@ def __init__(self, scheduler):
 
         for ts in self.scheduler.tasks.values():
             key = ts.key
-            prefix = ts.prefix
+            prefix = ts.prefix.name
             self.all[prefix].add(key)
             self.state[ts.state][prefix].add(key)
             if ts.nbytes is not None:
@@ -256,7 +256,7 @@ def __init__(self, scheduler):
 
     def transition(self, key, start, finish, *args, **kwargs):
         ts = self.scheduler.tasks[key]
-        prefix = ts.prefix
+        prefix = ts.prefix.name
         self.all[prefix].add(key)
         try:
             self.state[start][prefix].remove(key)
diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index e6c6a49b484..ee38750f8ee 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -23,7 +23,9 @@ distributed:
     pickle: True            # Is the scheduler allowed to deserialize arbitrary bytestrings
     preload: []
     preload-argv: []
-    default-task-durations: {}  # How long we expect function names to run ("1h", "1s") (helps for long tasks)
+    default-task-durations:  # How long we expect function names to run ("1h", "1s") (helps for long tasks)
+      rechunk-split: 1us
+      shuffle-split: 1us
     validate: False         # Check scheduler state at every step for debugging
     dashboard:
       status:
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 275058d5221..98faba466e6 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -20,9 +20,9 @@
 import sortedcontainers
 
 try:
-    from cytoolz import frequencies, merge, pluck, merge_sorted, first
+    from cytoolz import frequencies, merge, pluck, merge_sorted, first, merge_with
 except ImportError:
-    from toolz import frequencies, merge, pluck, merge_sorted, first
+    from toolz import frequencies, merge, pluck, merge_sorted, first, merge_with
 from toolz import valmap, second, compose, groupby
 from tornado import gen
 from tornado.ioloop import IOLoop
@@ -57,6 +57,7 @@
     parse_bytes,
     PeriodicCallback,
     shutting_down,
+    key_split_group,
     empty_context,
     tmpfile,
 )
@@ -333,11 +334,10 @@ class TaskState(object):
        from the name of the function, followed by a hash of the function
        and arguments, like ``'inc-ab31c010444977004d656610d2d421ec'``.
 
-    .. attribute:: prefix: str
+    .. attribute:: prefix: TaskPrefix
 
-       The key prefix, used in certain calculations to get an estimate
-       of the task's duration based on the duration of other tasks in the
-       same "family" (for example ``'inc'``).
+       The broad class of tasks to which this task belongs like "inc" or
+       "read_csv"
 
     .. attribute:: run_spec: object
 
@@ -551,6 +551,10 @@ class TaskState(object):
     .. attribute: actor: bool
 
        Whether or not this task is an Actor.
+
+    .. attribute: group: TaskGroup
+
+:      The group of tasks to which this one belongs.
     """
 
     __slots__ = (
@@ -573,7 +577,7 @@ class TaskState(object):
         "resource_restrictions",
         "loose_restrictions",
         # === Task state ===
-        "state",
+        "_state",
         # Whether some dependencies were forgotten
         "has_lost_dependencies",
         # If in 'waiting' state, which tasks need to complete
@@ -595,13 +599,14 @@ class TaskState(object):
         "retries",
         "nbytes",
         "type",
+        "group_key",
+        "group",
     )
 
     def __init__(self, key, run_spec):
         self.key = key
-        self.prefix = key_split(key)
         self.run_spec = run_spec
-        self.state = None
+        self._state = None
         self.exception = self.traceback = self.exception_blame = None
         self.suspicious = self.retries = 0
         self.nbytes = None
@@ -620,14 +625,38 @@ def __init__(self, key, run_spec):
         self.loose_restrictions = False
         self.actor = None
         self.type = None
+        self.group_key = key_split_group(key)
+        self.group = None
+
+    @property
+    def state(self) -> str:
+        return self._state
 
-    def get_nbytes(self):
+    @property
+    def prefix_key(self):
+        return self.prefix.name
+
+    @state.setter
+    def state(self, value: str):
+        self.group.states[self._state] -= 1
+        self.group.states[value] += 1
+        self._state = value
+
+    def add_dependency(self, other: "TaskState"):
+        """ Add another task as a dependency of this task """
+        self.dependencies.add(other)
+        self.group.dependencies.add(other.group)
+        other.dependents.add(self)
+
+    def get_nbytes(self) -> int:
         nbytes = self.nbytes
         return nbytes if nbytes is not None else DEFAULT_DATA_SIZE
 
-    def set_nbytes(self, nbytes):
+    def set_nbytes(self, nbytes: int):
         old_nbytes = self.nbytes
         diff = nbytes - (old_nbytes or 0)
+        self.group.nbytes_total += diff
+        self.group.nbytes_in_memory += diff
         for ws in self.who_has:
             ws.nbytes += diff
         self.nbytes = nbytes
@@ -654,6 +683,161 @@ def validate(self):
                 pdb.set_trace()
 
 
+class TaskGroup(object):
+    """ Collection tracking all tasks within a group
+
+    Keys often have a structure like ``("x-123", 0)``
+    A group takes the first section, like ``"x-123"``
+
+    .. attribute:: name: str
+
+       The name of a group of tasks.
+       For a task like ``("x-123", 0)`` this is the text ``"x-123"``
+
+    .. attribute:: states: Dict[str, int]
+
+       The number of tasks in each state,
+       like ``{"memory": 10, "processing": 3, "released": 4, ...}``
+
+    .. attribute:: dependencies: Set[TaskGroup]
+
+       The other TaskGroups on which this one depends
+
+    .. attribute:: nbytes_total: int
+
+       The total number of bytes that this task group has produced
+
+    .. attribute:: nbytes_in_memory: int
+
+       The number of bytes currently stored by this TaskGroup
+
+    .. attribute:: duration: float
+
+       The total amount of time spent on all tasks in this TaskGroup
+
+    .. attribute:: types: Set[str]
+
+       The result types of this TaskGroup
+
+    See also
+    --------
+    TaskPrefix
+    """
+
+    def __init__(self, name):
+        self.name = name
+        self.states = {state: 0 for state in ALL_TASK_STATES}
+        self.states["forgotten"] = 0
+        self.dependencies = set()
+        self.nbytes_total = 0
+        self.nbytes_in_memory = 0
+        self.duration = 0
+        self.types = set()
+
+    def add(self, ts):
+        # self.tasks.add(ts)
+        self.states[ts.state] += 1
+        ts.group = self
+
+    def __repr__(self):
+        return (
+            "<"
+            + (self.name or "no-group")
+            + ": "
+            + ", ".join(
+                "%s: %d" % (k, v) for (k, v) in sorted(self.states.items()) if v
+            )
+            + ">"
+        )
+
+    def __len__(self):
+        return sum(self.states.values())
+
+
+class TaskPrefix(object):
+    """ Collection tracking all tasks within a group
+
+    Keys often have a structure like ``("x-123", 0)``
+    A group takes the first section, like ``"x"``
+
+    .. attribute:: name: str
+
+       The name of a group of tasks.
+       For a task like ``("x-123", 0)`` this is the text ``"x"``
+
+    .. attribute:: states: Dict[str, int]
+
+       The number of tasks in each state,
+       like ``{"memory": 10, "processing": 3, "released": 4, ...}``
+
+    .. attribute:: duration_average: float
+
+       An exponentially weighted moving average duration of all tasks with this prefix
+
+    See Also
+    --------
+    TaskGroup
+    """
+
+    def __init__(self, name):
+        self.name = name
+        self.groups = []
+        if self.name in dask.config.get("distributed.scheduler.default-task-durations"):
+            self.duration_average = parse_timedelta(
+                dask.config.get("distributed.scheduler.default-task-durations")[
+                    self.name
+                ]
+            )
+        else:
+            self.duration_average = None
+
+    @property
+    def states(self):
+        return merge_with(sum, [g.states for g in self.groups])
+
+    @property
+    def active(self):
+        return [
+            g
+            for g in self.groups
+            if any(v != 0 for k, v in g.states.items() if k != "forgotten")
+        ]
+
+    @property
+    def active_states(self):
+        return merge_with(sum, [g.states for g in self.active])
+
+    def __repr__(self):
+        return (
+            "<"
+            + self.name
+            + ": "
+            + ", ".join(
+                "%s: %d" % (k, v) for (k, v) in sorted(self.states.items()) if v
+            )
+            + ">"
+        )
+
+    @property
+    def nbytes_in_memory(self):
+        return sum(tg.nbytes_in_memory for tg in self.groups)
+
+    @property
+    def nbytes_total(self):
+        return sum(tg.nbytes_total for tg in self.groups)
+
+    def __len__(self):
+        return sum(map(len, self.groups))
+
+    @property
+    def duration(self):
+        return sum(tg.duration for tg in self.groups)
+
+    @property
+    def types(self):
+        return set.union(*[tg.types for tg in self.groups])
+
+
 class _StateLegacyMapping(Mapping):
     """
     A mapping interface mimicking the former Scheduler state dictionaries.
@@ -923,6 +1107,8 @@ def __init__(
 
         # Task state
         self.tasks = dict()
+        self.task_groups = dict()
+        self.task_prefixes = dict()
         for old_attr, new_attr, wrap in [
             ("priority", "priority", None),
             ("dependencies", "dependencies", _legacy_task_key_set),
@@ -972,11 +1158,6 @@ def __init__(
         self.datasets = dict()
 
         # Prefix-keyed containers
-        self.task_duration = {prefix: 0.00001 for prefix in fast_tasks}
-        for k, v in dask.config.get(
-            "distributed.scheduler.default-task-durations", {}
-        ).items():
-            self.task_duration[k] = parse_timedelta(v)
         self.unknown_durations = defaultdict(set)
 
         # Client state
@@ -1631,8 +1812,7 @@ def update_graph(
             # XXX Have a method get_task_state(self, k) ?
             ts = self.tasks.get(k)
             if ts is None:
-                ts = self.tasks[k] = TaskState(k, tasks.get(k))
-                ts.state = "released"
+                ts = self.new_task(k, tasks.get(k), "released")
             elif not ts.run_spec:
                 ts.run_spec = tasks.get(k)
 
@@ -1649,8 +1829,7 @@ def update_graph(
                 continue
             for dep in deps:
                 dts = self.tasks[dep]
-                ts.dependencies.add(dts)
-                dts.dependents.add(ts)
+                ts.add_dependency(dts)
 
         # Compute priorities
         if isinstance(user_priority, Number):
@@ -1775,6 +1954,27 @@ def update_graph(
 
         # TODO: balance workers
 
+    def new_task(self, key, spec, state):
+        """ Create a new task, and associated states """
+        ts = TaskState(key, spec)
+        ts._state = state
+        try:
+            tg = self.task_groups[ts.group_key]
+        except KeyError:
+            tg = self.task_groups[ts.group_key] = TaskGroup(ts.group_key)
+        tg.add(ts)
+        prefix_key = key_split(key)
+        try:
+            tp = self.task_prefixes[prefix_key]
+        except KeyError:
+            tp = TaskPrefix(prefix_key)
+            tp.groups.append(tg)
+            self.task_prefixes[prefix_key] = tp
+        ts.prefix = tp
+        tg.prefix = tp
+        self.tasks[key] = ts
+        return ts
+
     def stimulus_task_finished(self, key=None, worker=None, **kwargs):
         """ Mark that a task has finished execution on a particular worker """
         logger.debug("Stimulus task finished %s, %s", key, worker)
@@ -2037,8 +2237,7 @@ def client_desires_keys(self, keys=None, client=None):
             ts = self.tasks.get(k)
             if ts is None:
                 # For publish, queues etc.
-                ts = self.tasks[k] = TaskState(k, None)
-                ts.state = "released"
+                ts = self.new_task(k, None, "released")
             ts.who_wants.add(cs)
             cs.wants_what.add(ts)
 
@@ -2416,15 +2615,14 @@ def handle_long_running(self, key=None, worker=None, compute_duration=None):
             return
 
         if compute_duration:
-            prefix = ts.prefix
-            old_duration = self.task_duration.get(prefix, 0)
+            old_duration = ts.prefix.duration_average or 0
             new_duration = compute_duration
             if not old_duration:
                 avg_duration = new_duration
             else:
                 avg_duration = 0.5 * old_duration + 0.5 * new_duration
 
-            self.task_duration[prefix] = avg_duration
+            ts.prefix.duration_average = avg_duration
 
         ws.occupancy -= ws.processing[ts]
         self.total_occupancy -= ws.processing[ts]
@@ -3250,7 +3448,7 @@ def update_data(
             for key, workers in who_has.items():
                 ts = self.tasks.get(key)
                 if ts is None:
-                    ts = self.tasks[key] = TaskState(key, None)
+                    ts = self.new_task(key, None, "memory")
                 ts.state = "memory"
                 if key in nbytes:
                     ts.set_nbytes(nbytes[key])
@@ -3446,13 +3644,13 @@ def get_task_duration(self, ts, default=0.5):
         Get the estimated computation cost of the given task
         (not including any communication cost).
         """
-        prefix = ts.prefix
-        try:
-            return self.task_duration[prefix]
-        except KeyError:
-            self.unknown_durations[prefix].add(ts)
+        duration = ts.prefix.duration_average
+        if duration is None:
+            self.unknown_durations[ts.prefix.name].add(ts)
             return default
 
+        return duration
+
     def run_function(self, stream, function, args=(), kwargs={}, wait=True):
         """ Run a function within this process
 
@@ -3574,6 +3772,7 @@ def _add_to_memory(
 
         ts.state = "memory"
         ts.type = typename
+        ts.group.types.add(typename)
 
         cs = self.clients["fire-and-forget"]
         if ts in cs.wants_what:
@@ -3847,17 +4046,17 @@ def transition_processing_memory(
             #############################
             if compute_start and ws.processing.get(ts, True):
                 # Update average task duration for worker
-                prefix = ts.prefix
-                old_duration = self.task_duration.get(prefix, 0)
+                old_duration = ts.prefix.duration_average or 0
                 new_duration = compute_stop - compute_start
                 if not old_duration:
                     avg_duration = new_duration
                 else:
                     avg_duration = 0.5 * old_duration + 0.5 * new_duration
 
-                self.task_duration[prefix] = avg_duration
+                ts.prefix.duration_average = avg_duration
+                ts.group.duration += new_duration
 
-                for tts in self.unknown_durations.pop(prefix, ()):
+                for tts in self.unknown_durations.pop(ts.prefix.name, ()):
                     if tts.processing_on:
                         wws = tts.processing_on
                         old = wws.processing[tts]
@@ -3921,6 +4120,7 @@ def transition_memory_released(self, key, safe=False):
             for ws in ts.who_has:
                 ws.has_what.remove(ts)
                 ws.nbytes -= ts.get_nbytes()
+                ts.group.nbytes_in_memory -= ts.get_nbytes()
                 self.worker_send(
                     ws.address, {"op": "delete-data", "keys": [key], "report": False}
                 )
@@ -4236,6 +4436,9 @@ def _propagate_forgotten(self, ts, recommendations):
         ts.dependencies.clear()
         ts.waiting_on.clear()
 
+        if ts.who_has:
+            ts.group.nbytes_in_memory -= ts.get_nbytes()
+
         for ws in ts.who_has:
             ws.has_what.remove(ts)
             ws.nbytes -= ts.get_nbytes()
@@ -5142,9 +5345,6 @@ def validate_state(tasks, workers, clients):
 _round_robin = [0]
 
 
-fast_tasks = {"rechunk-split", "shuffle-split"}
-
-
 def heartbeat_interval(n):
     """
     Interval in seconds that we desire heartbeats based on number of workers
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 2ec3a9f79d1..7aba29040be 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -1262,7 +1262,9 @@ def test_if_intermediates_clear_on_error(c, s, a, b):
     assert not any(ts.who_has for ts in s.tasks.values())
 
 
-@gen_cluster(client=True)
+@gen_cluster(
+    client=True, config={"distributed.scheduler.default-task-durations": {"f": "1ms"}}
+)
 def test_pragmatic_move_small_data_to_large_data(c, s, a, b):
     np = pytest.importorskip("numpy")
     lists = c.map(np.ones, [10000] * 10, pure=False)
@@ -1272,7 +1274,6 @@ def test_pragmatic_move_small_data_to_large_data(c, s, a, b):
     def f(x, y):
         return None
 
-    s.task_duration["f"] = 0.001
     results = c.map(f, lists, [total] * 10)
 
     yield wait([total])
@@ -3102,12 +3103,12 @@ def test_client_replicate_sync(c):
 def test_task_load_adapts_quickly(c, s, a):
     future = c.submit(slowinc, 1, delay=0.2)  # slow
     yield wait(future)
-    assert 0.15 < s.task_duration["slowinc"] < 0.4
+    assert 0.15 < s.task_prefixes["slowinc"].duration_average < 0.4
 
     futures = c.map(slowinc, range(10), delay=0)  # very fast
     yield wait(futures)
 
-    assert 0 < s.task_duration["slowinc"] < 0.1
+    assert 0 < s.task_prefixes["slowinc"].duration_average < 0.1
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 2)
@@ -4013,10 +4014,13 @@ def test_retire_many_workers(c, s, *workers):
         assert 15 < len(keys) < 50
 
 
-@gen_cluster(client=True, nthreads=[("127.0.0.1", 3)] * 2)
+@gen_cluster(
+    client=True,
+    nthreads=[("127.0.0.1", 3)] * 2,
+    config={"distributed.scheduler.default-task-durations": {"f": "10ms"}},
+)
 def test_weight_occupancy_against_data_movement(c, s, a, b):
     s.extensions["stealing"]._pc.callback_time = 1000000
-    s.task_duration["f"] = 0.01
 
     def f(x, y=0, z=0):
         sleep(0.01)
@@ -4033,9 +4037,12 @@ def f(x, y=0, z=0):
     assert sum(f.key in b.data for f in futures) >= 1
 
 
-@gen_cluster(client=True, nthreads=[("127.0.0.1", 1), ("127.0.0.1", 10)])
+@gen_cluster(
+    client=True,
+    nthreads=[("127.0.0.1", 1), ("127.0.0.1", 10)],
+    config={"distributed.scheduler.default-task-durations": {"f": "10ms"}},
+)
 def test_distribute_tasks_by_nthreads(c, s, a, b):
-    s.task_duration["f"] = 0.01
     s.extensions["stealing"]._pc.callback_time = 1000000
 
     def f(x, y=0):
@@ -4748,7 +4755,6 @@ def f(x):
         yield gen.sleep(0.01)
         assert threading.active_count() < count + 50
 
-    # assert 0.005 < s.task_duration['f'] < 0.1
     assert len(a.log) < 2 * len(b.log)
     assert len(b.log) < 2 * len(a.log)
 
diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
index 68f207a51ce..88910c87069 100644
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -35,23 +35,26 @@ async def test_nanny(s):
     async with Nanny(s.address, nthreads=2, loop=s.loop) as n:
         async with rpc(n.address) as nn:
             assert n.is_alive()
-            assert s.nthreads[n.worker_address] == 2
-            assert s.workers[n.worker_address].nanny == n.address
+            [ws] = s.workers.values()
+            assert ws.nthreads == 2
+            assert ws.nanny == n.address
 
             await nn.kill()
             assert not n.is_alive()
-            assert n.worker_address not in s.nthreads
-            assert n.worker_address not in s.workers
+            start = time()
+            while n.worker_address in s.workers:
+                assert time() < start + 1
+                await asyncio.sleep(0.01)
 
             await nn.kill()
             assert not n.is_alive()
-            assert n.worker_address not in s.nthreads
             assert n.worker_address not in s.workers
 
             await nn.instantiate()
             assert n.is_alive()
-            assert s.nthreads[n.worker_address] == 2
-            assert s.workers[n.worker_address].nanny == n.address
+            [ws] = s.workers.values()
+            assert ws.nthreads == 2
+            assert ws.nanny == n.address
 
             await nn.terminate()
             assert not n.is_alive()
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index f9087a3029a..077c5530260 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -21,7 +21,7 @@
 from distributed import Nanny, Worker, Client, wait, fire_and_forget
 from distributed.comm import Comm
 from distributed.core import connect, rpc, ConnectionPool
-from distributed.scheduler import Scheduler, TaskState
+from distributed.scheduler import Scheduler
 from distributed.client import wait
 from distributed.metrics import time
 from distributed.protocol.pickle import dumps
@@ -710,18 +710,17 @@ def test_retire_workers_n(c, s, a, b):
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 4)
-def test_workers_to_close(cl, s, *workers):
-    s.task_duration["a"] = 4
-    s.task_duration["b"] = 4
-    s.task_duration["c"] = 1
-
-    futures = cl.map(slowinc, [1, 1, 1], key=["a-4", "b-4", "c-1"])
-    while sum(len(w.processing) for w in s.workers.values()) < 3:
-        yield gen.sleep(0.001)
+async def test_workers_to_close(cl, s, *workers):
+    with dask.config.set(
+        {"distributed.scheduler.default-task-durations": {"a": 4, "b": 4, "c": 1}}
+    ):
+        futures = cl.map(slowinc, [1, 1, 1], key=["a-4", "b-4", "c-1"])
+        while sum(len(w.processing) for w in s.workers.values()) < 3:
+            await gen.sleep(0.001)
 
-    wtc = s.workers_to_close()
-    assert all(not s.workers[w].processing for w in wtc)
-    assert len(wtc) == 1
+        wtc = s.workers_to_close()
+        assert all(not s.workers[w].processing for w in wtc)
+        assert len(wtc) == 1
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 4)
@@ -897,19 +896,19 @@ def test_learn_occupancy_multiple_workers(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_include_communication_in_occupancy(c, s, a, b):
-    s.task_duration["slowadd"] = 0.001
+async def test_include_communication_in_occupancy(c, s, a, b):
+    await c.submit(slowadd, 1, 2, delay=0)
     x = c.submit(operator.mul, b"0", int(s.bandwidth), workers=a.address)
     y = c.submit(operator.mul, b"1", int(s.bandwidth * 1.5), workers=b.address)
 
     z = c.submit(slowadd, x, y, delay=1)
     while z.key not in s.tasks or not s.tasks[z.key].processing_on:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
     ts = s.tasks[z.key]
     assert ts.processing_on == s.workers[b.address]
     assert s.workers[b.address].processing[ts] > 1
-    yield wait(z)
+    await wait(z)
     del z
 
 
@@ -1602,27 +1601,28 @@ def test_dashboard_address():
 
 @gen_cluster(client=True)
 async def test_adaptive_target(c, s, a, b):
-    assert s.adaptive_target() == 0
-    x = c.submit(inc, 1)
-    await x
-    assert s.adaptive_target() == 1
-
-    # Long task
-    s.task_duration["slowinc"] = 10
-    x = c.submit(slowinc, 1, delay=0.5)
-    while x.key not in s.tasks:
-        await gen.sleep(0.01)
-    assert s.adaptive_target(target_duration=".1s") == 1  # still one
-
-    s.task_duration["slowinc"] = 10
-    L = c.map(slowinc, range(100), delay=0.5)
-    while len(s.tasks) < 100:
-        await gen.sleep(0.01)
-    assert 10 < s.adaptive_target(target_duration=".1s") <= 100
-    del x, L
-    while s.tasks:
-        await gen.sleep(0.01)
-    assert s.adaptive_target(target_duration=".1s") == 0
+    with dask.config.set(
+        {"distributed.scheduler.default-task-durations": {"slowinc": 10}}
+    ):
+        assert s.adaptive_target() == 0
+        x = c.submit(inc, 1)
+        await x
+        assert s.adaptive_target() == 1
+
+        # Long task
+        x = c.submit(slowinc, 1, delay=0.5)
+        while x.key not in s.tasks:
+            await gen.sleep(0.01)
+        assert s.adaptive_target(target_duration=".1s") == 1  # still one
+
+        L = c.map(slowinc, range(100), delay=0.5)
+        while len(s.tasks) < 100:
+            await gen.sleep(0.01)
+        assert 10 < s.adaptive_target(target_duration=".1s") <= 100
+        del x, L
+        while s.tasks:
+            await gen.sleep(0.01)
+        assert s.adaptive_target(target_duration=".1s") == 0
 
 
 @pytest.mark.asyncio
@@ -1673,27 +1673,28 @@ async def test_retire_names_str(cleanup):
                     assert len(b.data) == 10
 
 
-def test_get_task_duration():
+@gen_cluster(client=True)
+async def test_get_task_duration(c, s, a, b):
     with dask.config.set(
-        {"distributed.scheduler.default-task-durations": {"prefix_1": 100}}
+        {"distributed.scheduler.default-task-durations": {"inc": 100}}
     ):
-        s = Scheduler(port=0)
-        assert "prefix_1" in s.task_duration
-        assert s.task_duration["prefix_1"] == 100
+        future = c.submit(inc, 1)
+        await future
+        assert 10 < s.task_prefixes["inc"].duration_average < 100
 
-        ts_pref1 = TaskState("prefix_1-abcdefab", None)
-        assert s.get_task_duration(ts_pref1) == 100
+        ts_pref1 = s.new_task("inc-abcdefab", None, "released")
+        assert 10 < s.get_task_duration(ts_pref1) < 100
 
         # make sure get_task_duration adds TaskStates to unknown dict
         assert len(s.unknown_durations) == 0
-        ts_pref2 = TaskState("prefix_2-abcdefab", None)
-        assert s.get_task_duration(ts_pref2) == 0.5  # default
-        assert len(s.unknown_durations) == 1
-        assert len(s.unknown_durations["prefix_2"]) == 1
-        ts_pref2_2 = TaskState("prefix_2-accdefab", None)
-        assert s.get_task_duration(ts_pref2_2) == 0.5  # default
+        x = c.submit(slowinc, 1, delay=0.5)
+        while len(s.tasks) < 3:
+            await asyncio.sleep(0.01)
+
+        ts = s.tasks[x.key]
+        assert s.get_task_duration(ts) == 0.5  # default
         assert len(s.unknown_durations) == 1
-        assert len(s.unknown_durations["prefix_2"]) == 2
+        assert len(s.unknown_durations["slowinc"]) == 1
 
 
 @pytest.mark.asyncio
@@ -1711,6 +1712,56 @@ async def test_no_danglng_asyncio_tasks(cleanup):
     assert tasks == start
 
 
+@gen_cluster(client=True)
+async def test_task_groups(c, s, a, b):
+    da = pytest.importorskip("dask.array")
+    x = da.arange(100, chunks=(20,))
+    y = (x + 1).persist(optimize_graph=False)
+    y = await y
+
+    tg = s.task_groups[x.name]
+    tp = s.task_prefixes["arange"]
+    repr(tg)
+    repr(tp)
+    assert tg.states["memory"] == 0
+    assert tg.states["released"] == 5
+    assert tp.states["memory"] == 0
+    assert tp.states["released"] == 5
+    assert tg.prefix is tp
+    assert tg in tp.groups
+    assert tg.duration == tp.duration
+    assert tg.nbytes_in_memory == tp.nbytes_in_memory
+    assert tg.nbytes_total == tp.nbytes_total
+
+    tg = s.task_groups[y.name]
+    assert tg.states["memory"] == 5
+
+    assert s.task_groups[y.name].dependencies == {s.task_groups[x.name]}
+
+    await c.replicate(y)
+    assert tg.nbytes_in_memory == y.nbytes
+
+    del y
+
+    while s.tasks:
+        await asyncio.sleep(0.01)
+
+    assert tg.nbytes_in_memory == 0
+    assert tg.states["forgotten"] == 5
+    assert "array" in str(tg.types)
+    assert "array" in str(tp.types)
+
+
+@gen_cluster(client=True)
+async def test_task_prefix(c, s, a, b):
+    da = pytest.importorskip("dask.array")
+    x = da.arange(100, chunks=(20,))
+    y = (x + 1).sum().persist()
+    y = await y
+
+    assert s.task_prefixes["sum-aggregate"].states["memory"] == 1
+
+
 class BrokenComm(Comm):
     peer_address = None
     local_address = None
@@ -1857,6 +1908,28 @@ def reducer(x, y):
     ]
     assert len(transitions_to_processing) == 1
 
+    starts = []
+    finish_processing_transitions = 0
+    for transition in s.transition_log:
+        key, start, finish, recommendations, timestamp = transition
+        if "reducer" in key and finish == "processing":
+            finish_processing_transitions += 1
+    assert finish_processing_transitions == 1
+
+
+@gen_cluster(client=True)
+async def test_too_many_groups(c, s, a, b):
+    x = dask.delayed(inc)(1)
+    y = dask.delayed(dec)(2)
+    z = dask.delayed(operator.add)(x, y)
+
+    await c.compute(z)
+
+    while s.tasks:
+        await asyncio.sleep(0.01)
+
+    assert len(s.task_groups) < 3
+
 
 @pytest.mark.asyncio
 async def test_multiple_listeners(cleanup):
diff --git a/distributed/tests/test_steal.py b/distributed/tests/test_steal.py
index 6d98e662034..de63e542807 100644
--- a/distributed/tests/test_steal.py
+++ b/distributed/tests/test_steal.py
@@ -317,12 +317,15 @@ def test_dont_steal_executing_tasks(c, s, a, b):
     assert len(b.data) == 0
 
 
-@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 10)
+@gen_cluster(
+    client=True,
+    nthreads=[("127.0.0.1", 1)] * 10,
+    config={"distributed.scheduler.default-task-durations": {"slowidentity": 0.2}},
+)
 def test_dont_steal_few_saturated_tasks_many_workers(c, s, a, *rest):
     s.extensions["stealing"]._pc.callback_time = 20
     x = c.submit(mul, b"0", 100000000, workers=a.address)  # 100 MB
     yield wait(x)
-    s.task_duration["slowidentity"] = 0.2
 
     futures = [c.submit(slowidentity, x, pure=False, delay=0.2) for i in range(2)]
 
@@ -336,12 +339,12 @@ def test_dont_steal_few_saturated_tasks_many_workers(c, s, a, *rest):
     client=True,
     nthreads=[("127.0.0.1", 1)] * 10,
     worker_kwargs={"memory_limit": MEMORY_LIMIT},
+    config={"distributed.scheduler.default-task-durations": {"slowidentity": 0.2}},
 )
 def test_steal_when_more_tasks(c, s, a, *rest):
     s.extensions["stealing"]._pc.callback_time = 20
     x = c.submit(mul, b"0", 50000000, workers=a.address)  # 50 MB
     yield wait(x)
-    s.task_duration["slowidentity"] = 0.2
 
     futures = [c.submit(slowidentity, x, pure=False, delay=0.2) for i in range(20)]
 
@@ -351,7 +354,16 @@ def test_steal_when_more_tasks(c, s, a, *rest):
         assert time() < start + 1
 
 
-@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 10)
+@gen_cluster(
+    client=True,
+    nthreads=[("127.0.0.1", 1)] * 10,
+    config={
+        "distributed.scheduler.default-task-durations": {
+            "slowidentity": 0.2,
+            "slow2": 1,
+        }
+    },
+)
 def test_steal_more_attractive_tasks(c, s, a, *rest):
     def slow2(x):
         sleep(1)
@@ -361,9 +373,6 @@ def slow2(x):
     x = c.submit(mul, b"0", 100000000, workers=a.address)  # 100 MB
     yield wait(x)
 
-    s.task_duration["slowidentity"] = 0.2
-    s.task_duration["slow2"] = 1
-
     futures = [c.submit(slowidentity, x, pure=False, delay=0.2) for i in range(10)]
     future = c.submit(slow2, x, priority=-1)
 
@@ -399,7 +408,6 @@ def assert_balanced(inp, expected, c, s, *workers):
                     ws.nbytes += ts.nbytes - old_nbytes
             else:
                 dat = 123
-            s.task_duration[str(int(t))] = 1
             i = next(counter)
             f = c.submit(
                 func,
@@ -473,7 +481,15 @@ def assert_balanced(inp, expected, c, s, *workers):
 )
 def test_balance(inp, expected):
     test = lambda *args, **kwargs: assert_balanced(inp, expected, *args, **kwargs)
-    test = gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * len(inp))(test)
+    test = gen_cluster(
+        client=True,
+        nthreads=[("127.0.0.1", 1)] * len(inp),
+        config={
+            "distributed.scheduler.default-task-durations": {
+                str(i): 1 for i in range(10)
+            }
+        },
+    )(test)
     test()
 
 
@@ -495,10 +511,12 @@ def test_restart(c, s, a, b):
     assert not any(x for L in steal.stealable.values() for x in L)
 
 
-@gen_cluster(client=True)
+@gen_cluster(
+    client=True,
+    config={"distributed.scheduler.default-task-durations": {"slowadd": 0.001}},
+)
 def test_steal_communication_heavy_tasks(c, s, a, b):
     steal = s.extensions["stealing"]
-    s.task_duration["slowadd"] = 0.001
     x = c.submit(mul, b"0", int(s.bandwidth), workers=a.address)
     y = c.submit(mul, b"1", int(s.bandwidth), workers=b.address)
 
diff --git a/distributed/utils.py b/distributed/utils.py
index 26c503205aa..f7009ece83b 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -614,28 +614,16 @@ def key_split(s):
 def key_split_group(x):
     """A more fine-grained version of key_split
 
-    >>> key_split_group('x')
-    'x'
-    >>> key_split_group('x-1')
-    'x-1'
-    >>> key_split_group('x-1-2-3')
-    'x-1-2-3'
     >>> key_split_group(('x-2', 1))
     'x-2'
     >>> key_split_group("('x-2', 1)")
     'x-2'
-    >>> key_split_group('hello-world-1')
-    'hello-world-1'
-    >>> key_split_group(b'hello-world-1')
-    'hello-world-1'
     >>> key_split_group('ae05086432ca935f6eba409a8ecd4896')
     'data'
     >>> key_split_group('<module.submodule.myclass object at 0xdaf372')
     'myclass'
-    >>> key_split_group(None)
-    'Other'
-    >>> key_split_group('x-abcdefab')  # ignores hex
-    'x-abcdefab'
+    >>> key_split_group('x')
+    >>> key_split_group('x-1')
     """
     typ = type(x)
     if typ is tuple:
@@ -648,11 +636,11 @@ def key_split_group(x):
         elif x[0] == "<":
             return x.strip("<>").split()[0].split(".")[-1]
         else:
-            return x
+            return ""
     elif typ is bytes:
         return key_split_group(x.decode())
     else:
-        return "Other"
+        return ""
 
 
 @contextmanager

From 74b6e1aa4980df76d72441ed72145e662b0211fd Mon Sep 17 00:00:00 2001
From: Stephan Erb <github@stephanerb.eu>
Date: Thu, 12 Dec 2019 02:11:56 +0100
Subject: [PATCH 0605/1550] Use worker name in logs (#3309)

---
 distributed/scheduler.py | 14 ++++++--------
 distributed/utils.py     |  5 ++++-
 distributed/worker.py    |  3 ++-
 3 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 98faba466e6..46851c2c059 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -292,15 +292,13 @@ def clean(self):
         return ws
 
     def __repr__(self):
-        return "<Worker %r, memory: %d, processing: %d>" % (
+        return "<Worker %r, name: %s, memory: %d, processing: %d>" % (
             self.address,
+            self.name,
             len(self.has_what),
             len(self.processing),
         )
 
-    def __str__(self):
-        return self.address
-
     def identity(self):
         return {
             "type": "Worker",
@@ -1616,7 +1614,7 @@ async def add_worker(
 
             ws = self.workers.get(address)
             if ws is not None:
-                raise ValueError("Worker already exists %s" % address)
+                raise ValueError("Worker already exists %s" % ws)
 
             self.workers[address] = ws = WorkerState(
                 address=address,
@@ -1699,7 +1697,7 @@ async def add_worker(
 
             self.log_event(address, {"action": "add-worker"})
             self.log_event("all", {"action": "add-worker", "worker": address})
-            logger.info("Register %s", str(address))
+            logger.info("Register worker %s", ws)
 
             if comm:
                 await comm.write(
@@ -2120,7 +2118,7 @@ def remove_worker(self, comm=None, address=None, safe=False, close=True):
                     "processing-tasks": dict(ws.processing),
                 },
             )
-            logger.info("Remove worker %s", address)
+            logger.info("Remove worker %s", ws)
             if close:
                 with ignoring(AttributeError, CommClosedError):
                     self.stream_comms[address].send({"op": "close", "report": False})
@@ -2191,7 +2189,7 @@ def remove_worker_from_events():
                 dask.config.get("distributed.scheduler.events-cleanup-delay")
             )
             self.loop.call_later(cleanup_delay, remove_worker_from_events)
-            logger.debug("Removed worker %s", address)
+            logger.debug("Removed worker %s", ws)
 
         return "OK"
 
diff --git a/distributed/utils.py b/distributed/utils.py
index f7009ece83b..d268e0b54b0 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -6,6 +6,7 @@
 from datetime import timedelta
 import functools
 from hashlib import md5
+import html
 import inspect
 import json
 import logging
@@ -1284,7 +1285,9 @@ class Log(str):
     """ A container for logs """
 
     def _repr_html_(self):
-        return "<pre><code>\n{log}\n</code></pre>".format(log=self.rstrip())
+        return "<pre><code>\n{log}\n</code></pre>".format(
+            log=html.escape(self.rstrip())
+        )
 
 
 class Logs(dict):
diff --git a/distributed/worker.py b/distributed/worker.py
index 751fabce2b1..639299fd477 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -709,10 +709,11 @@ def __init__(
 
     def __repr__(self):
         return (
-            "<%s: %s, %s, stored: %d, running: %d/%d, ready: %d, comm: %d, waiting: %d>"
+            "<%s: %r, %s, %s, stored: %d, running: %d/%d, ready: %d, comm: %d, waiting: %d>"
             % (
                 self.__class__.__name__,
                 self.address,
+                self.name,
                 self.status,
                 len(self.data),
                 len(self.executing),

From 5c1520a94b15c992044d5e70e2cfb91b22506f6f Mon Sep 17 00:00:00 2001
From: fjetter <fjetter@users.noreply.github.com>
Date: Fri, 13 Dec 2019 16:36:08 +0100
Subject: [PATCH 0606/1550] All scheduler task states in prometheus (#3307)

---
 distributed/dashboard/scheduler.py            | 62 +++++++-----
 .../tests/test_scheduler_bokeh_html.py        | 94 +++++++++++++++----
 2 files changed, 117 insertions(+), 39 deletions(-)

diff --git a/distributed/dashboard/scheduler.py b/distributed/dashboard/scheduler.py
index 2c1ec38c4f3..cb96389344e 100644
--- a/distributed/dashboard/scheduler.py
+++ b/distributed/dashboard/scheduler.py
@@ -4,8 +4,12 @@
 
 import dask
 from dask.utils import format_bytes
-import toolz
-from toolz import merge
+
+try:
+    from cytoolz import merge, merge_with
+except ImportError:
+    from toolz import merge, merge_with
+
 from tornado import escape
 
 try:
@@ -42,6 +46,7 @@
 from .proxy import GlobalProxyHandler
 from .utils import RequestHandler, redirect
 from ..utils import log_errors, format_time
+from ..scheduler import ALL_TASK_STATES
 
 
 ns = {
@@ -65,7 +70,7 @@ def get(self):
                 "workers.html",
                 title="Workers",
                 scheduler=self.server,
-                **toolz.merge(self.server.__dict__, ns, self.extra, rel_path_statics),
+                **merge(self.server.__dict__, ns, self.extra, rel_path_statics),
             )
 
 
@@ -81,7 +86,7 @@ def get(self, worker):
                 title="Worker: " + worker,
                 scheduler=self.server,
                 Worker=worker,
-                **toolz.merge(self.server.__dict__, ns, self.extra, rel_path_statics),
+                **merge(self.server.__dict__, ns, self.extra, rel_path_statics),
             )
 
 
@@ -97,7 +102,7 @@ def get(self, task):
                 title="Task: " + task,
                 Task=task,
                 scheduler=self.server,
-                **toolz.merge(self.server.__dict__, ns, self.extra, rel_path_statics),
+                **merge(self.server.__dict__, ns, self.extra, rel_path_statics),
             )
 
 
@@ -109,7 +114,7 @@ def get(self):
                 "logs.html",
                 title="Logs",
                 logs=logs,
-                **toolz.merge(self.extra, rel_path_statics),
+                **merge(self.extra, rel_path_statics),
             )
 
 
@@ -123,7 +128,7 @@ async def get(self, worker):
                 "logs.html",
                 title="Logs: " + worker,
                 logs=logs,
-                **toolz.merge(self.extra, rel_path_statics),
+                **merge(self.extra, rel_path_statics),
             )
 
 
@@ -137,7 +142,7 @@ async def get(self, worker):
                 "call-stack.html",
                 title="Call Stacks: " + worker,
                 call_stack=call_stack,
-                **toolz.merge(self.extra, rel_path_statics),
+                **merge(self.extra, rel_path_statics),
             )
 
 
@@ -156,7 +161,7 @@ async def get(self, key):
                     "call-stack.html",
                     title="Call Stack: " + key,
                     call_stack=call_stack,
-                    **toolz.merge(self.extra, rel_path_statics),
+                    **merge(self.extra, rel_path_statics),
                 )
 
 
@@ -239,7 +244,7 @@ def __init__(self, server):
         self.server = server
 
     def collect(self):
-        from prometheus_client.core import GaugeMetricFamily
+        from prometheus_client.core import GaugeMetricFamily, CounterMetricFamily
 
         yield GaugeMetricFamily(
             "dask_scheduler_clients",
@@ -253,40 +258,53 @@ def collect(self):
             value=self.server.adaptive_target(),
         )
 
-        tasks = GaugeMetricFamily(
+        worker_states = GaugeMetricFamily(
             "dask_scheduler_workers",
             "Number of workers known by scheduler.",
             labels=["state"],
         )
-        tasks.add_metric(["connected"], len(self.server.workers))
-        tasks.add_metric(["saturated"], len(self.server.saturated))
-        tasks.add_metric(["idle"], len(self.server.idle))
-        yield tasks
+        worker_states.add_metric(["connected"], len(self.server.workers))
+        worker_states.add_metric(["saturated"], len(self.server.saturated))
+        worker_states.add_metric(["idle"], len(self.server.idle))
+        yield worker_states
 
         tasks = GaugeMetricFamily(
             "dask_scheduler_tasks",
             "Number of tasks known by scheduler.",
             labels=["state"],
         )
-        tasks.add_metric(["received"], len(self.server.tasks))
-        tasks.add_metric(["unrunnable"], len(self.server.unrunnable))
+
+        task_counter = merge_with(
+            sum, (tp.states for tp in self.server.task_prefixes.values())
+        )
+
+        yield CounterMetricFamily(
+            "dask_scheduler_tasks_forgotten",
+            "Total number of processed tasks no longer in memory and already removed from the scheduler job queue.",
+            value=task_counter.get("forgotten", 0.0),
+        )
+
+        for state in ALL_TASK_STATES:
+            tasks.add_metric([state], task_counter.get(state, 0.0))
         yield tasks
 
 
 class PrometheusHandler(RequestHandler):
-    _initialized = False
+    _collector = None
 
     def __init__(self, *args, **kwargs):
         import prometheus_client
 
         super(PrometheusHandler, self).__init__(*args, **kwargs)
 
-        if PrometheusHandler._initialized:
+        if PrometheusHandler._collector:
+            # Especially during testing, multiple schedulers are started
+            # sequentially in the same python process
+            PrometheusHandler._collector.server = self.server
             return
 
-        prometheus_client.REGISTRY.register(_PrometheusCollector(self.server))
-
-        PrometheusHandler._initialized = True
+        PrometheusHandler._collector = _PrometheusCollector(self.server)
+        prometheus_client.REGISTRY.register(PrometheusHandler._collector)
 
     def get(self):
         import prometheus_client
diff --git a/distributed/dashboard/tests/test_scheduler_bokeh_html.py b/distributed/dashboard/tests/test_scheduler_bokeh_html.py
index 55e4b797b4e..39da4730a28 100644
--- a/distributed/dashboard/tests/test_scheduler_bokeh_html.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh_html.py
@@ -20,10 +20,10 @@
     scheduler_kwargs={"services": {("dashboard", 0): BokehScheduler}},
     worker_kwargs={"services": {"dashboard": BokehWorker}},
 )
-def test_connect(c, s, a, b):
+async def test_connect(c, s, a, b):
     future = c.submit(lambda x: x + 1, 1)
     x = c.submit(slowinc, 1, delay=1, retries=5)
-    yield future
+    await future
     http_client = AsyncHTTPClient()
     for suffix in [
         "info/main/workers.html",
@@ -38,7 +38,7 @@ def test_connect(c, s, a, b):
         "json/index.html",
         "individual-plots.json",
     ]:
-        response = yield http_client.fetch(
+        response = await http_client.fetch(
             "http://localhost:%d/%s" % (s.services["dashboard"].port, suffix)
         )
         assert response.code == 200
@@ -55,15 +55,15 @@ def test_connect(c, s, a, b):
     nthreads=[],
     scheduler_kwargs={"services": {("dashboard", 0): BokehScheduler}},
 )
-def test_worker_404(c, s):
+async def test_worker_404(c, s):
     http_client = AsyncHTTPClient()
     with pytest.raises(HTTPClientError) as err:
-        yield http_client.fetch(
+        await http_client.fetch(
             "http://localhost:%d/info/worker/unknown" % s.services["dashboard"].port
         )
     assert err.value.code == 404
     with pytest.raises(HTTPClientError) as err:
-        yield http_client.fetch(
+        await http_client.fetch(
             "http://localhost:%d/info/task/unknown" % s.services["dashboard"].port
         )
     assert err.value.code == 404
@@ -75,10 +75,10 @@ def test_worker_404(c, s):
         "services": {("dashboard", 0): (BokehScheduler, {"prefix": "/foo"})}
     },
 )
-def test_prefix(c, s, a, b):
+async def test_prefix(c, s, a, b):
     http_client = AsyncHTTPClient()
     for suffix in ["foo/info/main/workers.html", "foo/json/index.html", "foo/system"]:
-        response = yield http_client.fetch(
+        response = await http_client.fetch(
             "http://localhost:%d/%s" % (s.services["dashboard"].port, suffix)
         )
         assert response.code == 200
@@ -94,7 +94,7 @@ def test_prefix(c, s, a, b):
     clean_kwargs={"threads": False},
     scheduler_kwargs={"services": {("dashboard", 0): BokehScheduler}},
 )
-def test_prometheus(c, s, a, b):
+async def test_prometheus(c, s, a, b):
     pytest.importorskip("prometheus_client")
     from prometheus_client.parser import text_string_to_metric_families
 
@@ -103,7 +103,7 @@ def test_prometheus(c, s, a, b):
     # request data twice since there once was a case where metrics got registered multiple times resulting in
     # prometheus_client errors
     for _ in range(2):
-        response = yield http_client.fetch(
+        response = await http_client.fetch(
             "http://localhost:%d/metrics" % s.services["dashboard"].port
         )
         assert response.code == 200
@@ -119,10 +119,70 @@ def test_prometheus(c, s, a, b):
     clean_kwargs={"threads": False},
     scheduler_kwargs={"services": {("dashboard", 0): BokehScheduler}},
 )
-def test_health(c, s, a, b):
+async def test_prometheus_collect_task_states(c, s, a, b):
+    pytest.importorskip("prometheus_client")
+    from prometheus_client.parser import text_string_to_metric_families
+
+    http_client = AsyncHTTPClient()
+
+    async def fetch_metrics():
+        bokeh_scheduler = s.services["dashboard"]
+        assert s.services["dashboard"].scheduler is s
+        response = await http_client.fetch(
+            f"http://{bokeh_scheduler.server.address}:{bokeh_scheduler.port}/metrics"
+        )
+        txt = response.body.decode("utf8")
+        families = {
+            family.name: family for family in text_string_to_metric_families(txt)
+        }
+
+        active_metrics = {
+            sample.labels["state"]: sample.value
+            for sample in families["dask_scheduler_tasks"].samples
+        }
+        forgotten_tasks = [
+            sample.value
+            for sample in families["dask_scheduler_tasks_forgotten"].samples
+        ]
+        return active_metrics, forgotten_tasks
+
+    expected = {"memory", "released", "processing", "waiting", "no-worker", "erred"}
+
+    # Ensure that we get full zero metrics for all states even though the
+    # scheduler did nothing, yet
+    assert not s.tasks
+    active_metrics, forgotten_tasks = await fetch_metrics()
+    assert active_metrics.keys() == expected
+    assert sum(active_metrics.values()) == 0.0
+    assert sum(forgotten_tasks) == 0.0
+
+    # submit a task which should show up in the prometheus scraping
+    future = c.submit(slowinc, 1, delay=0.5)
+
+    active_metrics, forgotten_tasks = await fetch_metrics()
+    assert active_metrics.keys() == expected
+    assert sum(active_metrics.values()) == 1.0
+    assert sum(forgotten_tasks) == 0.0
+
+    res = await c.gather(future)
+    assert res == 2
+
+    del future
+    active_metrics, forgotten_tasks = await fetch_metrics()
+    assert active_metrics.keys() == expected
+    assert sum(active_metrics.values()) == 0.0
+    assert sum(forgotten_tasks) == 1.0
+
+
+@gen_cluster(
+    client=True,
+    clean_kwargs={"threads": False},
+    scheduler_kwargs={"services": {("dashboard", 0): BokehScheduler}},
+)
+async def test_health(c, s, a, b):
     http_client = AsyncHTTPClient()
 
-    response = yield http_client.fetch(
+    response = await http_client.fetch(
         "http://localhost:%d/health" % s.services["dashboard"].port
     )
     assert response.code == 200
@@ -135,14 +195,14 @@ def test_health(c, s, a, b):
 @gen_cluster(
     client=True, scheduler_kwargs={"services": {("dashboard", 0): BokehScheduler}}
 )
-def test_task_page(c, s, a, b):
+async def test_task_page(c, s, a, b):
     future = c.submit(lambda x: x + 1, 1, workers=a.address)
     x = c.submit(inc, 1)
-    yield future
+    await future
     http_client = AsyncHTTPClient()
 
     "info/task/" + url_escape(future.key) + ".html",
-    response = yield http_client.fetch(
+    response = await http_client.fetch(
         "http://localhost:%d/info/task/" % s.services["dashboard"].port
         + url_escape(future.key)
         + ".html"
@@ -167,13 +227,13 @@ def test_task_page(c, s, a, b):
         }
     },
 )
-def test_allow_websocket_origin(c, s, a, b):
+async def test_allow_websocket_origin(c, s, a, b):
     url = (
         "ws://localhost:%d/status/ws?bokeh-protocol-version=1.0&bokeh-session-id=1"
         % s.services["dashboard"].port
     )
     with pytest.raises(HTTPClientError) as err:
-        yield websocket_connect(
+        await websocket_connect(
             HTTPRequest(url, headers={"Origin": "http://evil.invalid"})
         )
     assert err.value.code == 403

From 06c0fc27f54b43f97ed89c5af48d8baeeb9175f3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Patrick=20Sodr=C3=A9?= <psodre@gmail.com>
Date: Fri, 13 Dec 2019 12:07:53 -0500
Subject: [PATCH 0607/1550] Add plugin entry point for out-of-tree comms
 library (#3305)

* Add distributed.comm.backends plugin entry-point

  - Allows end-users to create their own comms plugins and have them
    discovered at run-time without having to preload the library.
  - Existing behavior remains unchanged, i.e. entry-points has lower
    precedence than distributed.comm.registry

* Add tests

* Only import pkg_resources if backend is not in default registry.

* Update backends entry-point documentation.

* Cache the Backend classes found through package metadata.

* Indent the code-block.

* fix typo
---
 distributed/comm/registry.py         | 22 ++++++++++++++++++--
 distributed/comm/tests/test_comms.py | 30 +++++++++++++++++++++++++++-
 docs/source/communications.rst       | 16 +++++++++++++++
 3 files changed, 65 insertions(+), 3 deletions(-)

diff --git a/distributed/comm/registry.py b/distributed/comm/registry.py
index 369f2415c35..8fb7a6026f8 100644
--- a/distributed/comm/registry.py
+++ b/distributed/comm/registry.py
@@ -60,10 +60,28 @@ def get_local_address_for(self, loc):
 def get_backend(scheme):
     """
     Get the Backend instance for the given *scheme*.
+    It looks for matching scheme in dask's internal cache, and falls-back to
+    package metadata for the group name ``distributed.comm.backends``
     """
+
     backend = backends.get(scheme)
     if backend is None:
-        raise ValueError(
-            "unknown address scheme %r (known schemes: %s)" % (scheme, sorted(backends))
+        import pkg_resources
+
+        backend = next(
+            iter(
+                backend_class_ep.load()()
+                for backend_class_ep in pkg_resources.iter_entry_points(
+                    "distributed.comm.backends", scheme
+                )
+            ),
+            None,
         )
+        if backend is None:
+            raise ValueError(
+                "unknown address scheme %r (known schemes: %s)"
+                % (scheme, sorted(backends))
+            )
+        else:
+            backends[scheme] = backend
     return backend
diff --git a/distributed/comm/tests/test_comms.py b/distributed/comm/tests/test_comms.py
index 465be11c7a4..470c667b989 100644
--- a/distributed/comm/tests/test_comms.py
+++ b/distributed/comm/tests/test_comms.py
@@ -1,15 +1,18 @@
 import asyncio
+import types
 from functools import partial
 import os
 import sys
 import threading
 import warnings
 
+import pkg_resources
 import pytest
 
 from tornado import ioloop, locks, queues
 from tornado.concurrent import Future
 
+import distributed
 from distributed.metrics import time
 from distributed.utils import get_ip, get_ipv6
 from distributed.utils_test import (
@@ -23,7 +26,7 @@
 
 from distributed.protocol import to_serialize, Serialized, serialize, deserialize
 
-from distributed.comm.registry import backends
+from distributed.comm.registry import backends, get_backend
 from distributed.comm import (
     tcp,
     inproc,
@@ -1154,3 +1157,28 @@ async def test_tls_adresses():
 async def test_inproc_adresses():
     a, b = await get_inproc_comm_pair()
     await check_addresses(a, b)
+
+
+def test_register_backend_entrypoint():
+    # Code adapted from pandas backend entry point testing
+    # https://github.com/pandas-dev/pandas/blob/2470690b9f0826a8feb426927694fa3500c3e8d2/pandas/tests/plotting/test_backend.py#L50-L76
+
+    dist = pkg_resources.get_distribution("distributed")
+    if dist.module_path not in distributed.__file__:
+        # We are running from a non-installed distributed, and this test is invalid
+        pytest.skip("Testing a non-installed distributed")
+
+    mod = types.ModuleType("dask_udp")
+    mod.UDPBackend = lambda: 1
+    sys.modules[mod.__name__] = mod
+
+    entry_point_name = "distributed.comm.backends"
+    backends_entry_map = pkg_resources.get_entry_map("distributed")
+    if entry_point_name not in backends_entry_map:
+        backends_entry_map[entry_point_name] = dict()
+    backends_entry_map[entry_point_name]["udp"] = pkg_resources.EntryPoint(
+        "udp", mod.__name__, attrs=["UDPBackend"], dist=dist
+    )
+
+    result = get_backend("udp")
+    assert result == 1
diff --git a/docs/source/communications.rst b/docs/source/communications.rst
index 2869012ed48..b9406e6b3c5 100644
--- a/docs/source/communications.rst
+++ b/docs/source/communications.rst
@@ -97,6 +97,22 @@ Each transport is represented by a URI scheme (such as ``tcp``) and
 backed by a dedicated :class:`Backend` implementation, which provides
 entry points into all transport-specific routines.
 
+Out-of-tree backends can be registered under the group ``distributed.comm.backends``
+in setuptools `entry_points`_. For example, a hypothetical ``dask_udp`` package
+would register its UDP backend class by including the following in its ``setup.py`` file:
+
+.. code-block:: python
+
+    setup(name="dask_udp",
+          entry_points={
+            "distributed.comm.backends": [
+                "udp=dask_udp.backend:UDPBackend",
+            ]
+          },
+          ...
+    )
 
 .. autoclass:: distributed.comm.registry.Backend
    :members:
+
+.. _entry_points: https://packaging.python.org/guides/creating-and-discovering-plugins/#using-package-metadata

From d747f639d73c5f9b8961f2f80e407868de7da4cb Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Fri, 13 Dec 2019 17:46:55 +0000
Subject: [PATCH 0608/1550] Switch startstops to dicts and add worker name to
 transfer (#3319)

* Switch startstops to dicts and add worker name to transfer

* Fix task stream

* Rename `worker` to `source`
---
 distributed/diagnostics/progress_stream.py | 12 ++++-----
 distributed/diagnostics/task_stream.py     | 20 ++++++++-------
 distributed/scheduler.py                   |  6 ++++-
 distributed/tests/test_worker.py           |  2 +-
 distributed/worker.py                      | 29 ++++++++++++++--------
 5 files changed, 42 insertions(+), 27 deletions(-)

diff --git a/distributed/diagnostics/progress_stream.py b/distributed/diagnostics/progress_stream.py
index d127ecfeb7e..e417ee8e35b 100644
--- a/distributed/diagnostics/progress_stream.py
+++ b/distributed/diagnostics/progress_stream.py
@@ -156,17 +156,17 @@ def task_stream_append(lists, msg, workers):
     name = key_split(key)
     startstops = msg.get("startstops", [])
 
-    for action, start, stop in startstops:
-        color = colors[action]
+    for startstop in startstops:
+        color = colors[startstop["action"]]
         if type(color) is not str:
             color = color(msg)
 
-        lists["start"].append((start + stop) / 2 * 1000)
-        lists["duration"].append(1000 * (stop - start))
+        lists["start"].append((startstop["start"] + startstop["stop"]) / 2 * 1000)
+        lists["duration"].append(1000 * (startstop["stop"] - startstop["start"]))
         lists["key"].append(key)
-        lists["name"].append(prefix[action] + name)
+        lists["name"].append(prefix[startstop["action"]] + name)
         lists["color"].append(color)
-        lists["alpha"].append(alphas[action])
+        lists["alpha"].append(alphas[startstop["action"]])
         lists["worker"].append(msg["worker"])
 
         worker_thread = "%s-%d" % (msg["worker"], msg["thread"])
diff --git a/distributed/diagnostics/task_stream.py b/distributed/diagnostics/task_stream.py
index 2491c8a89c0..c319ca73d69 100644
--- a/distributed/diagnostics/task_stream.py
+++ b/distributed/diagnostics/task_stream.py
@@ -42,7 +42,9 @@ def bisect(target, left, right):
                 return left
 
             mid = (left + right) // 2
-            value = max(stop for _, start, stop in self.buffer[mid]["startstops"])
+            value = max(
+                startstop["stop"] for startstop in self.buffer[mid]["startstops"]
+            )
 
             if value < target:
                 return bisect(target, mid + 1, right)
@@ -119,20 +121,20 @@ def rectangles(msgs, workers=None, start_boundary=0):
         if worker_thread not in workers:
             workers[worker_thread] = len(workers) / 2
 
-        for action, start, stop in startstops:
-            if start < start_boundary:
+        for startstop in startstops:
+            if startstop["start"] < start_boundary:
                 continue
-            color = colors[action]
+            color = colors[startstop["action"]]
             if type(color) is not str:
                 color = color(msg)
 
-            L_start.append((start + stop) / 2 * 1000)
-            L_duration.append(1000 * (stop - start))
-            L_duration_text.append(format_time(stop - start))
+            L_start.append((startstop["start"] + startstop["stop"]) / 2 * 1000)
+            L_duration.append(1000 * (startstop["stop"] - startstop["start"]))
+            L_duration_text.append(format_time(startstop["stop"] - startstop["start"]))
             L_key.append(key)
-            L_name.append(prefix[action] + name)
+            L_name.append(prefix[startstop["action"]] + name)
             L_color.append(color)
-            L_alpha.append(alphas[action])
+            L_alpha.append(alphas[startstop["action"]])
             L_worker.append(msg["worker"])
             L_worker_thread.append(worker_thread)
             L_y.append(workers[worker_thread])
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 46851c2c059..c6b12e14e93 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -4031,7 +4031,11 @@ def transition_processing_memory(
                 return {}
 
             if startstops:
-                L = [(b, c) for a, b, c in startstops if a == "compute"]
+                L = [
+                    (startstop["start"], startstop["stop"])
+                    for startstop in startstops
+                    if startstop["action"] == "compute"
+                ]
                 if L:
                     compute_start, compute_stop = L[0]
                 else:  # This is very rare
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 2cf316ccbca..9cf32c1eec7 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -668,7 +668,7 @@ def test_multiple_transfers(c, s, w1, w2, w3):
     yield wait(z)
 
     r = w3.startstops[z.key]
-    transfers = [t for t in r if t[0] == "transfer"]
+    transfers = [t for t in r if t["action"] == "transfer"]
     assert len(transfers) == 2
 
 
diff --git a/distributed/worker.py b/distributed/worker.py
index 639299fd477..07695122f91 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -215,7 +215,7 @@ class Worker(ServerNode):
         The exception caused by running a task if it erred
     * **tracebacks**: ``{key: traceback}``
         The exception caused by running a task if it erred
-    * **startstops**: ``{key: [(str, float, float)]}``
+    * **startstops**: ``{key: [{startstop}]}``
         Log of transfer, load, and compute times for a task
 
     * **priorities**: ``{key: tuple}``
@@ -1866,7 +1866,9 @@ def put_key_in_memory(self, key, value, transition=True):
             self.data[key] = value
             stop = time()
             if stop - start > 0.020:
-                self.startstops[key].append(("disk-write", start, stop))
+                self.startstops[key].append(
+                    {"action": "disk-write", "start": start, "stop": stop}
+                )
 
         if key not in self.nbytes:
             self.nbytes[key] = sizeof(value)
@@ -1933,11 +1935,12 @@ async def gather_dep(self, worker, dep, deps, total_nbytes, cause=None):
 
                 if cause:
                     self.startstops[cause].append(
-                        (
-                            "transfer",
-                            start + self.scheduler_delay,
-                            stop + self.scheduler_delay,
-                        )
+                        {
+                            "action": "transfer",
+                            "start": start + self.scheduler_delay,
+                            "stop": stop + self.scheduler_delay,
+                            "source": worker,
+                        }
                     )
 
                 total_bytes = sum(self.nbytes.get(dep, 0) for dep in response["data"])
@@ -2383,7 +2386,9 @@ def _maybe_deserialize_task(self, key):
             stop = time()
 
             if stop - start > 0.010:
-                self.startstops[key].append(("deserialize", start, stop))
+                self.startstops[key].append(
+                    {"action": "deserialize", "start": start, "stop": stop}
+                )
             return function, args, kwargs
         except Exception as e:
             logger.warning("Could not deserialize task", exc_info=True)
@@ -2456,7 +2461,9 @@ async def execute(self, key, report=False):
             kwargs2 = pack_data(kwargs, data, key_types=(bytes, str))
             stop = time()
             if stop - start > 0.005:
-                self.startstops[key].append(("disk-read", start, stop))
+                self.startstops[key].append(
+                    {"action": "disk-read", "start": start, "stop": stop}
+                )
                 if self.digests is not None:
                     self.digests["disk-load-duration"].add(stop - start)
 
@@ -2487,7 +2494,9 @@ async def execute(self, key, report=False):
 
             result["key"] = key
             value = result.pop("result", None)
-            self.startstops[key].append(("compute", result["start"], result["stop"]))
+            self.startstops[key].append(
+                {"action": "compute", "start": result["start"], "stop": result["stop"]}
+            )
             self.threads[key] = result["thread"]
 
             if result["op"] == "task-finished":

From 5b1a1a99feeb934534d66621430e239f9564ee5a Mon Sep 17 00:00:00 2001
From: Benjamin Zaitlen <quasiben@users.noreply.github.com>
Date: Mon, 16 Dec 2019 11:57:24 -0500
Subject: [PATCH 0609/1550] Add performance report docs and color definitions
 to docs (#3325)

---
 docs/source/diagnosing-performance.rst | 29 ++++++++++++++++++++++++++
 docs/source/web.rst                    |  9 ++++++++
 2 files changed, 38 insertions(+)

diff --git a/docs/source/diagnosing-performance.rst b/docs/source/diagnosing-performance.rst
index 773a5d2316b..76ecfed944f 100644
--- a/docs/source/diagnosing-performance.rst
+++ b/docs/source/diagnosing-performance.rst
@@ -115,6 +115,35 @@ command on the workers:
    client.run(lambda dask_worker: dask_worker.incoming_transfer_log)
 
 
+Performance Reports
+-------------------
+
+Often when benchmarking and/or profiling, users may want to record a
+particular computation or even a full workflow.  Dask can save the bokeh
+dashboards as static HTML plots including the task stream, worker profiles,
+bandwidths, etc. This is done wrapping a computation with the ``performance_report`` context manager:
+
+.. code-block:: python
+
+    from dask.distributed import performance_report
+
+    with performance_report(filename="dask-report.html):
+        ## some dask computation
+
+The following video demonstrates the ``performance_report`` context manager in greater
+detail:
+
+.. raw:: html
+
+    <iframe width="560"
+            height="315"
+            src="https://www.youtube.com/embed/nTMGbkS761Q"
+            frameborder="0"
+            allow="autoplay; encrypted-media"
+            allowfullscreen>
+    </iframe>
+
+
 A note about times
 ------------------
 
diff --git a/docs/source/web.rst b/docs/source/web.rst
index cfef4902c35..6a5b58fac5e 100644
--- a/docs/source/web.rst
+++ b/docs/source/web.rst
@@ -126,6 +126,15 @@ accordingly.
 .. image:: https://raw.githubusercontent.com/dask/dask-org/master/images/bokeh-task-stream.gif
    :alt: Task stream plot of Dask web interface
 
+The colors signifying the following:
+
+1.  Serialization (gray)
+2.  Communication between workers (red)
+3.  Disk I/O (orange)
+4.  Error (black)
+5.  Execution times (colored by task: purple, green, yellow, etc)
+
+
 If data transfer occurs between workers a *red* bar appears preceding the
 task bar showing the duration of the transfer.  If an error occurs than a
 *black* bar replaces the normal color.  This plot show the last 1000 tasks.

From 94bf2ce3d28ee2e286086ef7c101d2c6c5d3cd89 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Tue, 17 Dec 2019 13:03:39 -0600
Subject: [PATCH 0610/1550] Add missing `"` in performance report example
 (#3329)

---
 docs/source/diagnosing-performance.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/source/diagnosing-performance.rst b/docs/source/diagnosing-performance.rst
index 76ecfed944f..330194076d3 100644
--- a/docs/source/diagnosing-performance.rst
+++ b/docs/source/diagnosing-performance.rst
@@ -127,7 +127,7 @@ bandwidths, etc. This is done wrapping a computation with the ``performance_repo
 
     from dask.distributed import performance_report
 
-    with performance_report(filename="dask-report.html):
+    with performance_report(filename="dask-report.html"):
         ## some dask computation
 
 The following video demonstrates the ``performance_report`` context manager in greater

From 0a8ccd8d720f001c03b51ef5b9be7c549b341aae Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 18 Dec 2019 10:54:30 -0800
Subject: [PATCH 0611/1550] Use TaskPrefix.name in Graph layout (#3328)

Fixes #3327
---
 distributed/dashboard/components/scheduler.py       | 2 +-
 distributed/dashboard/tests/test_scheduler_bokeh.py | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index ad7cecea024..9319937346c 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -1219,7 +1219,7 @@ def add_new_nodes_edges(self, new, new_edges, update=False):
                 node_x.append(xx)
                 node_y.append(yy)
                 node_state.append(task.state)
-                node_name.append(task.prefix)
+                node_name.append(task.prefix.name)
 
             for a, b in new_edges:
                 try:
diff --git a/distributed/dashboard/tests/test_scheduler_bokeh.py b/distributed/dashboard/tests/test_scheduler_bokeh.py
index 0f262ec5809..39d2ce84156 100644
--- a/distributed/dashboard/tests/test_scheduler_bokeh.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh.py
@@ -442,6 +442,8 @@ def test_TaskGraph(c, s, a, b):
     gp.update()
     assert set(map(len, gp.node_source.data.values())) == {6}
     assert set(map(len, gp.edge_source.data.values())) == {5}
+    json.dumps(gp.edge_source.data)
+    json.dumps(gp.node_source.data)
 
     da = pytest.importorskip("dask.array")
     x = da.random.random((20, 20), chunks=(10, 10)).persist()

From 54efd79fa535eaf2926f52eb056c4a5a6c4e0d3b Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Wed, 18 Dec 2019 14:28:56 -0600
Subject: [PATCH 0612/1550] Add setuptools to dependencies (#3320)

---
 requirements.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/requirements.txt b/requirements.txt
index b17e4620be6..67ab27f4edf 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -9,3 +9,4 @@ toolz >= 0.7.4
 tornado >= 5
 zict >= 0.1.3
 pyyaml
+setuptools

From 94f0219b26721bddb3af5539120add0a2a901298 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sat, 21 Dec 2019 11:25:15 -0800
Subject: [PATCH 0613/1550] Add lock around dumps_function cache (#3337)

Fixes #2727
---
 distributed/worker.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/distributed/worker.py b/distributed/worker.py
index 07695122f91..716806a67d6 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -3234,15 +3234,19 @@ def execute_task(task):
 
 cache_dumps = LRU(maxsize=100)
 
+_cache_lock = threading.Lock()
+
 
 def dumps_function(func):
     """ Dump a function to bytes, cache functions """
     try:
-        result = cache_dumps[func]
+        with _cache_lock:
+            result = cache_dumps[func]
     except KeyError:
         result = pickle.dumps(func)
         if len(result) < 100000:
-            cache_dumps[func] = result
+            with _cache_lock:
+                cache_dumps[func] = result
     except TypeError:  # Unhashable function
         result = pickle.dumps(func)
     return result

From 45eb9bf12c322be429a13979967b5854ceca31f1 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 27 Dec 2019 10:53:20 -0800
Subject: [PATCH 0614/1550] bump version to 2.9.1

---
 docs/source/changelog.rst | 24 ++++++++++++++++++++++++
 requirements.txt          |  2 +-
 2 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index 3cff2b92ab5..3e4a05fcc79 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,28 @@
 Changelog
 =========
 
+2.9.1 - 2019-12-27
+------------------
+
+-  Add lock around dumps_function cache (:pr:`3337`) `Matthew Rocklin`_
+-  Add setuptools to dependencies (:pr:`3320`) `James Bourbeau`_
+-  Use TaskPrefix.name in Graph layout (:pr:`3328`) `Matthew Rocklin`_
+-  Add missing `"` in performance report example (:pr:`3329`) `John Kirkham`_
+-  Add performance report docs and color definitions to docs (:pr:`3325`) `Benjamin Zaitlen`_
+-  Switch startstops to dicts and add worker name to transfer (:pr:`3319`) `Jacob Tomlinson`_
+-  Add plugin entry point for out-of-tree comms library (:pr:`3305`) `Patrick Sodré`_
+-  All scheduler task states in prometheus (:pr:`3307`) `fjetter`_
+-  Use worker name in logs (:pr:`3309`) `Stephan Erb`_
+-  Add TaskGroup and TaskPrefix scheduler state (:pr:`3262`)  `Matthew Rocklin`_
+-  Update latencies with heartbeats (:pr:`3310`) `fjetter`_
+-  Update inlining Futures in task graph in Client._graph_to_futures (:pr:`3303`) `James Bourbeau`_
+-  Use hostname as default IP address rather than localhost (:pr:`3308`) `Matthew Rocklin`_
+-  Clean up flaky test_nanny_throttle (:pr:`3295`) `Tom Augspurger`_
+-  Add lock to scheduler for sensitive operations (:pr:`3259`) `Matthew Rocklin`_
+-  Log address for each of the Scheduler listerners (:pr:`3306`) `Matthew Rocklin`_
+-  Make ConnectionPool.close asynchronous (:pr:`3304`) `Matthew Rocklin`_
+
+
 2.9.0 - 2019-12-06
 ------------------
 
@@ -1447,3 +1469,5 @@ significantly without many new features.
 .. _`He Jia`: https://github.com/HerculesJack
 .. _`Jim Crist-Harif`: https://github.com/jcrist
 .. _`fjetter`: https://github.com/fjetter
+.. _`Patrick Sodré`: https://github.com/sodre
+.. _`Stephan Erb`: https://github.com/StephanErb
diff --git a/requirements.txt b/requirements.txt
index 67ab27f4edf..545eba40c4d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,6 @@
 click >= 6.6
 cloudpickle >= 0.2.2
-dask >= 2.7.0
+dask >= 2.9.0
 msgpack
 psutil >= 5.0
 sortedcontainers !=2.0.0, !=2.0.1

From b6fb54154e8805ffe8fbec80bcb5ee7f2d24f328 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 27 Dec 2019 16:42:14 -0800
Subject: [PATCH 0615/1550] Relax intermittent failing test_profile_server
 (#3346)

---
 distributed/tests/test_client.py | 27 ++++++++++++++++++---------
 1 file changed, 18 insertions(+), 9 deletions(-)

diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 7aba29040be..079a673cbcf 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -5693,18 +5693,27 @@ async def test_futures_of_sorted(c, s, a, b):
 
 @gen_cluster(client=True, worker_kwargs={"profile_cycle_interval": "10ms"})
 async def test_profile_server(c, s, a, b):
-    x = c.map(slowinc, range(10), delay=0.01, workers=a.address)
-    await wait(x)
+    for i in range(5):
+        try:
+            x = c.map(slowinc, range(10), delay=0.01, workers=a.address, pure=False)
+            await wait(x)
 
-    await asyncio.gather(
-        c.run(slowinc, 1, delay=0.5), c.run_on_scheduler(slowdec, 1, delay=0.5)
-    )
+            await asyncio.gather(
+                c.run(slowinc, 1, delay=0.5), c.run_on_scheduler(slowdec, 1, delay=0.5)
+            )
 
-    p = await c.profile(server=True)  #  All worker servers
-    assert "slowinc" in str(p)
+            p = await c.profile(server=True)  #  All worker servers
+            assert "slowinc" in str(p)
 
-    p = await c.profile(scheduler=True)  #  Scheduler
-    assert "slowdec" in str(p)
+            p = await c.profile(scheduler=True)  #  Scheduler
+            assert "slowdec" in str(p)
+        except AssertionError:
+            if i == 4:
+                raise
+            else:
+                pass
+        else:
+            break
 
 
 @gen_cluster(client=True)

From 1e634e8da244db325007fb9e101b62bf2fb634cc Mon Sep 17 00:00:00 2001
From: Mana Borwornpadungkitti <potpath@users.noreply.github.com>
Date: Tue, 31 Dec 2019 23:53:02 +0700
Subject: [PATCH 0616/1550] Avoid setting event loop policy if within IPython
 kernel and no running event loop (#3336)

Setting asyncio event loop policy at these two places could cause problems.
1. When policy is set in Jupyter notebook server extension.
This causes the notebook server to hang. This is fixed in https://github.com/dask/distributed/pull/2343.
2. When policy is set in iPython startup config (`~/.ipython/profile_default/startup/whatever.py`) or by setting `get_config().InteractiveShellApp.exec_lines` in `~/.ipython/profile_default/ipython_config.py`.
This causes the kernel to hang. This can be reproduced by running either `jupyter console` or `jupyter notebook`. If running In Jupyter notebook, it will struck at "Kernel starting, please wait...".
Note that manually setting the policy in notebook cell, after the kernel has started, is fine.

In both cases, running `asyncio.get_running_loop()` just before setting the policy will raise `RuntimeError`, meaning there is no running event loop yet.

See https://github.com/dask/distributed/issues/3202.
---
 distributed/utils.py | 26 +++++++++++++++++---------
 1 file changed, 17 insertions(+), 9 deletions(-)

diff --git a/distributed/utils.py b/distributed/utils.py
index d268e0b54b0..e4884e4d16c 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -1177,25 +1177,33 @@ def reset_logger_locks():
             handler.createLock()
 
 
-# Only bother if asyncio has been loaded by Tornado
-if "asyncio" in sys.modules and tornado.version_info[0] >= 5:
+if tornado.version_info[0] >= 5:
 
-    jupyter_event_loop_initialized = False
+    is_server_extension = False
 
     if "notebook" in sys.modules:
         import traitlets
         from notebook.notebookapp import NotebookApp
 
-        jupyter_event_loop_initialized = traitlets.config.Application.initialized() and isinstance(
+        is_server_extension = traitlets.config.Application.initialized() and isinstance(
             traitlets.config.Application.instance(), NotebookApp
         )
 
-    if not jupyter_event_loop_initialized:
-        import tornado.platform.asyncio
+    if not is_server_extension:
+        is_kernel_and_no_running_loop = False
 
-        asyncio.set_event_loop_policy(
-            tornado.platform.asyncio.AnyThreadEventLoopPolicy()
-        )
+        if is_kernel():
+            try:
+                asyncio.get_running_loop()
+            except RuntimeError:
+                is_kernel_and_no_running_loop = True
+
+        if not is_kernel_and_no_running_loop:
+            import tornado.platform.asyncio
+
+            asyncio.set_event_loop_policy(
+                tornado.platform.asyncio.AnyThreadEventLoopPolicy()
+            )
 
 
 @functools.lru_cache(1000)

From fd451210617787a3bd77c370a2e113ad22fdab24 Mon Sep 17 00:00:00 2001
From: Markus Mohrhard <markus.mohrhard@googlemail.com>
Date: Thu, 2 Jan 2020 00:23:02 +0800
Subject: [PATCH 0617/1550] Avoid calling nbytes multiple times when sending
 data (#3349)

---
 distributed/comm/tcp.py | 20 ++++----------------
 1 file changed, 4 insertions(+), 16 deletions(-)

diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py
index c2f3feeb704..77876a04fbc 100644
--- a/distributed/comm/tcp.py
+++ b/distributed/comm/tcp.py
@@ -12,7 +12,6 @@
     ssl = None
 
 import dask
-import tornado
 from tornado import netutil
 from tornado.iostream import StreamClosedError, IOStream
 from tornado.tcpclient import TCPClient
@@ -20,15 +19,7 @@
 
 from ..system import MEMORY_LIMIT
 from ..threadpoolexecutor import ThreadPoolExecutor
-from ..utils import (
-    ensure_bytes,
-    ensure_ip,
-    get_ip,
-    get_ipv6,
-    nbytes,
-    parse_timedelta,
-    shutting_down,
-)
+from ..utils import ensure_ip, get_ip, get_ipv6, nbytes, parse_timedelta, shutting_down
 
 from .registry import Backend, backends
 from .addressing import parse_host_port, unparse_host_port
@@ -141,7 +132,6 @@ class TCP(Comm):
     An established communication based on an underlying Tornado IOStream.
     """
 
-    _iostream_allows_memoryview = tornado.version_info >= (4, 5)
     # IOStream.read_into() currently proposed in
     # https://github.com/tornadoweb/tornado/pull/2193
     _iostream_has_read_into = hasattr(IOStream, "read_into")
@@ -251,14 +241,12 @@ async def write(self, msg, serializers=None, on_error="message"):
             else:
                 stream.write(b"".join(length_bytes))  # avoid large memcpy, send in many
 
-                for frame in frames:
+                for frame, frame_bytes in zip(frames, lengths):
                     # Can't wait for the write() Future as it may be lost
                     # ("If write is called again before that Future has resolved,
                     #   the previous future will be orphaned and will never resolve")
-                    if not self._iostream_allows_memoryview:
-                        frame = ensure_bytes(frame)
                     future = stream.write(frame)
-                    bytes_since_last_yield += nbytes(frame)
+                    bytes_since_last_yield += frame_bytes
                     if bytes_since_last_yield > 32e6:
                         await future
                         bytes_since_last_yield = 0
@@ -271,7 +259,7 @@ async def write(self, msg, serializers=None, on_error="message"):
             else:
                 raise
 
-        return sum(map(nbytes, frames))
+        return sum(lengths)
 
     @gen.coroutine
     def close(self):

From 71a8f4551f362a901d1911628c30197f3afccab1 Mon Sep 17 00:00:00 2001
From: Benedikt Reinartz <filmor@gmail.com>
Date: Mon, 6 Jan 2020 18:30:14 +0100
Subject: [PATCH 0618/1550] Fix failures on mixed integer/string worker names
 (#3352)

---
 distributed/dashboard/components/scheduler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index 9319937346c..8eec6b8b772 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -1625,7 +1625,7 @@ def __init__(self, scheduler, width=800, **kwargs):
     def update(self):
         data = {name: [] for name in self.names + self.extra_names}
         for i, (addr, ws) in enumerate(
-            sorted(self.scheduler.workers.items(), key=lambda kv: kv[1].name)
+            sorted(self.scheduler.workers.items(), key=lambda kv: str(kv[1].name))
         ):
             for name in self.names + self.extra_names:
                 data[name].append(ws.metrics.get(name, None))

From 465c9975d922efd38c4c06e886bda51654ddfe04 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Mon, 6 Jan 2020 16:26:20 -0600
Subject: [PATCH 0619/1550] Return task in dask-worker on_signal function
 (#3354)

---
 distributed/cli/dask_worker.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
index 9070024c430..a252fe1a232 100755
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -398,7 +398,7 @@ def on_signal(signum):
         signal_fired = True
         if signum != signal.SIGINT:
             logger.info("Exiting on signal %d", signum)
-        asyncio.ensure_future(close_all())
+        return asyncio.ensure_future(close_all())
 
     async def run():
         await asyncio.gather(*nannies)

From f68119a49b6090b56c8fd5bebbc7def4090d463e Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Tue, 7 Jan 2020 16:54:26 +0000
Subject: [PATCH 0620/1550] Add websocket scheduler plugin (#3335)

---
 distributed/dashboard/scheduler.py   | 32 +++++++++++++++++
 distributed/diagnostics/websocket.py | 52 ++++++++++++++++++++++++++++
 2 files changed, 84 insertions(+)
 create mode 100644 distributed/diagnostics/websocket.py

diff --git a/distributed/dashboard/scheduler.py b/distributed/dashboard/scheduler.py
index cb96389344e..6a52063b9f1 100644
--- a/distributed/dashboard/scheduler.py
+++ b/distributed/dashboard/scheduler.py
@@ -1,5 +1,6 @@
 from datetime import datetime
 from functools import partial
+import json
 import logging
 
 import dask
@@ -11,6 +12,7 @@
     from toolz import merge, merge_with
 
 from tornado import escape
+from tornado.websocket import WebSocketHandler
 
 try:
     import numpy as np
@@ -45,6 +47,7 @@
 from .worker import counters_doc
 from .proxy import GlobalProxyHandler
 from .utils import RequestHandler, redirect
+from ..diagnostics.websocket import WebsocketPlugin
 from ..utils import log_errors, format_time
 from ..scheduler import ALL_TASK_STATES
 
@@ -319,6 +322,34 @@ def get(self):
         self.set_header("Content-Type", "text/plain")
 
 
+class EventstreamHandler(WebSocketHandler):
+    def initialize(self, server=None, extra=None):
+        self.server = server
+        self.extra = extra or {}
+        self.plugin = WebsocketPlugin(self, server)
+        self.server.add_plugin(self.plugin)
+
+    def send(self, name, data):
+        data["name"] = name
+        for k in list(data):
+            # Drop bytes objects for now
+            if isinstance(data[k], bytes):
+                del data[k]
+        self.write_message(data)
+
+    def open(self):
+        for worker in self.server.workers:
+            self.plugin.add_worker(self.server, worker)
+
+    def on_message(self, message):
+        message = json.loads(message)
+        if message["name"] == "ping":
+            self.send("pong", {"timestamp": str(datetime.now())})
+
+    def on_close(self):
+        self.server.remove_plugin(self.plugin)
+
+
 routes = [
     (r"info", redirect("info/main/workers.html")),
     (r"info/main/workers.html", Workers),
@@ -334,6 +365,7 @@ def get(self):
     (r"individual-plots.json", IndividualPlots),
     (r"metrics", PrometheusHandler),
     (r"health", HealthHandler),
+    (r"eventstream", EventstreamHandler),
     (r"proxy/(\d+)/(.*?)/(.*)", GlobalProxyHandler),
 ]
 
diff --git a/distributed/diagnostics/websocket.py b/distributed/diagnostics/websocket.py
new file mode 100644
index 00000000000..6682dd6a739
--- /dev/null
+++ b/distributed/diagnostics/websocket.py
@@ -0,0 +1,52 @@
+from .plugin import SchedulerPlugin
+from ..utils import key_split
+from .task_stream import colors
+
+
+class WebsocketPlugin(SchedulerPlugin):
+    def __init__(self, socket, scheduler):
+        self.socket = socket
+        self.scheduler = scheduler
+
+    def restart(self, scheduler, **kwargs):
+        """ Run when the scheduler restarts itself """
+        self.socket.send("restart", {})
+
+    def add_worker(self, scheduler=None, worker=None, **kwargs):
+        """ Run when a new worker enters the cluster """
+        self.socket.send("add_worker", {"worker": worker})
+
+    def remove_worker(self, scheduler=None, worker=None, **kwargs):
+        """ Run when a worker leaves the cluster"""
+        self.socket.send("remove_worker", {"worker": worker})
+
+    def transition(self, key, start, finish, *args, **kwargs):
+        """ Run whenever a task changes state
+
+        Parameters
+        ----------
+        key: string
+        start: string
+            Start state of the transition.
+            One of released, waiting, processing, memory, error.
+        finish: string
+            Final state of the transition.
+        *args, **kwargs: More options passed when transitioning
+            This may include worker ID, compute time, etc.
+        """
+        if key not in self.scheduler.tasks:
+            return
+        kwargs["key"] = key
+        startstops = kwargs.get("startstops", [])
+        for startstop in startstops:
+            color = colors[startstop["action"]]
+            if type(color) is not str:
+                color = color(kwargs)
+            data = {
+                "key": key,
+                "name": key_split(key),
+                "color": color,
+                **kwargs,
+                **startstop,
+            }
+            self.socket.send("transition", data)

From 32cb96effe9287fdc42dd73be36989be13abe99d Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 8 Jan 2020 18:38:21 -0800
Subject: [PATCH 0621/1550] Rework version checking (#2627)

This creates automatic version checks whenever a worker or client joins
the cluster and raises informative errors letting users know about the
mismatched versions.
---
 distributed/client.py                     | 55 ++++++-------
 distributed/deploy/tests/test_adaptive.py | 30 +++----
 distributed/scheduler.py                  | 97 ++++++++++++++---------
 distributed/tests/test_client.py          | 38 ++++-----
 distributed/tests/test_collections.py     | 12 +--
 distributed/tests/test_failed_workers.py  | 12 +--
 distributed/tests/test_scheduler.py       | 41 ++++++----
 distributed/tests/test_worker.py          |  5 +-
 distributed/utils.py                      | 10 +++
 distributed/versions.py                   | 58 +++++++++++---
 distributed/worker.py                     | 21 ++++-
 11 files changed, 233 insertions(+), 146 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 5ff715281ef..451a6628e73 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -77,7 +77,6 @@
     log_errors,
     str_graph,
     key_split,
-    asciitable,
     thread_state,
     no_default,
     PeriodicCallback,
@@ -88,7 +87,7 @@
     has_keyword,
     format_dashboard_link,
 )
-from .versions import get_versions
+from . import versions as version_module
 
 
 logger = logging.getLogger(__name__)
@@ -1050,7 +1049,12 @@ async def _ensure_connected(self, timeout=None):
             else:
                 await self._update_scheduler_info()
             await comm.write(
-                {"op": "register-client", "client": self.id, "reply": False}
+                {
+                    "op": "register-client",
+                    "client": self.id,
+                    "reply": False,
+                    "versions": version_module.get_versions(),
+                }
             )
         except Exception as e:
             if self.status == "closed":
@@ -1066,6 +1070,9 @@ async def _ensure_connected(self, timeout=None):
         assert len(msg) == 1
         assert msg[0]["op"] == "stream-start"
 
+        if msg[0].get("warning"):
+            warnings.warn(version_module.VersionMismatchWarning(msg[0]["warning"]))
+
         bcomm = BatchedSend(interval="10ms", loop=self.loop)
         bcomm.start(comm)
         self.scheduler_comm = bcomm
@@ -1249,7 +1256,7 @@ async def _close(self, fast=False):
             if self.get == dask.config.get("get", None):
                 del dask.config.config["get"]
             if self.status == "closed":
-                raise gen.Return()
+                return
 
             if (
                 self.scheduler_comm
@@ -1274,15 +1281,21 @@ async def _close(self, fast=False):
                 and not self.scheduler_comm.comm.closed()
             ):
                 await self.scheduler_comm.close()
+
             for key in list(self.futures):
                 self._release_key(key=key)
+
             if self._start_arg is None:
                 with ignoring(AttributeError):
                     await self.cluster.close()
+
             await self.rpc.close()
+
             self.status = "closed"
+
             if _get_global_client() is self:
                 _set_global_client(None)
+
             coroutines = set(self.coroutines)
             for f in self.coroutines:
                 # cancel() works on asyncio futures (Tornado 5)
@@ -1292,11 +1305,14 @@ async def _close(self, fast=False):
                 if f.cancelled():
                     coroutines.remove(f)
             del self.coroutines[:]
+
             if not fast:
                 with ignoring(TimeoutError):
                     await gen.with_timeout(timedelta(seconds=2), list(coroutines))
+
             with ignoring(AttributeError):
                 await self.scheduler.close_rpc()
+
             self.scheduler = None
 
         self.status = "closed"
@@ -3551,7 +3567,7 @@ def get_versions(self, check=False, packages=[]):
         return self.sync(self._get_versions, check=check, packages=packages)
 
     async def _get_versions(self, check=False, packages=[]):
-        client = get_versions(packages=packages)
+        client = version_module.get_versions(packages=packages)
         try:
             scheduler = await self.scheduler.versions(packages=packages)
         except KeyError:
@@ -3565,32 +3581,9 @@ async def _get_versions(self, check=False, packages=[]):
         result = {"scheduler": scheduler, "workers": workers, "client": client}
 
         if check:
-            # we care about the required & optional packages matching
-            def to_packages(d):
-                L = list(d["packages"].values())
-                return dict(sum(L, type(L[0])()))
-
-            client_versions = to_packages(result["client"])
-            versions = [("scheduler", to_packages(result["scheduler"]))]
-            versions.extend((w, to_packages(d)) for w, d in sorted(workers.items()))
-
-            mismatched = defaultdict(list)
-            for name, vers in versions:
-                for pkg, cv in client_versions.items():
-                    v = vers.get(pkg, "MISSING")
-                    if cv != v:
-                        mismatched[pkg].append((name, v))
-
-            if mismatched:
-                errs = []
-                for pkg, versions in sorted(mismatched.items()):
-                    rows = [("client", client_versions[pkg])]
-                    rows.extend(versions)
-                    errs.append("%s\n%s" % (pkg, asciitable(["", "version"], rows)))
-
-                raise ValueError(
-                    "Mismatched versions found\n\n%s" % ("\n\n".join(errs))
-                )
+            msg = version_module.error_message(scheduler, workers, client)
+            if msg:
+                raise ValueError(msg)
 
         return result
 
diff --git a/distributed/deploy/tests/test_adaptive.py b/distributed/deploy/tests/test_adaptive.py
index 90f56c4bfde..2eddeeceff8 100644
--- a/distributed/deploy/tests/test_adaptive.py
+++ b/distributed/deploy/tests/test_adaptive.py
@@ -272,33 +272,29 @@ def test_adapt_quickly():
 
 
 @gen_test(timeout=None)
-def test_adapt_down():
+async def test_adapt_down():
     """ Ensure that redefining adapt with a lower maximum removes workers """
-    cluster = yield LocalCluster(
+    async with LocalCluster(
         0,
         asynchronous=True,
         processes=False,
         scheduler_port=0,
         silence_logs=False,
         dashboard_address=None,
-    )
-    client = yield Client(cluster, asynchronous=True)
-    cluster.adapt(interval="20ms", maximum=5)
+    ) as cluster:
+        async with Client(cluster, asynchronous=True) as client:
+            cluster.adapt(interval="20ms", maximum=5)
 
-    try:
-        futures = client.map(slowinc, range(1000), delay=0.1)
-        while len(cluster.scheduler.workers) < 5:
-            yield gen.sleep(0.1)
+            futures = client.map(slowinc, range(1000), delay=0.1)
+            while len(cluster.scheduler.workers) < 5:
+                await gen.sleep(0.1)
 
-        cluster.adapt(maximum=2)
+            cluster.adapt(maximum=2)
 
-        start = time()
-        while len(cluster.scheduler.workers) != 2:
-            yield gen.sleep(0.1)
-            assert time() < start + 1
-    finally:
-        yield client.close()
-        yield cluster.close()
+            start = time()
+            while len(cluster.scheduler.workers) != 2:
+                await gen.sleep(0.1)
+                assert time() < start + 1
 
 
 @gen_test(timeout=30)
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index c6b12e14e93..66827935b19 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -63,6 +63,7 @@
 )
 from .utils_comm import scatter_to_workers, gather_from_workers, retry_operation
 from .utils_perf import enable_gc_diagnosis, disable_gc_diagnosis
+from . import versions as version_module
 
 from .publish import PublishExtension
 from .queues import QueueExtension
@@ -115,12 +116,13 @@ class ClientState(object):
 
     """
 
-    __slots__ = ("client_key", "wants_what", "last_seen")
+    __slots__ = ("client_key", "wants_what", "last_seen", "versions")
 
-    def __init__(self, client):
+    def __init__(self, client, versions=None):
         self.client_key = client
         self.wants_what = set()
         self.last_seen = time()
+        self.versions = versions or {}
 
     def __repr__(self):
         return "<Client %r>" % (self.client_key,)
@@ -232,6 +234,7 @@ class WorkerState(object):
         "status",
         "time_delay",
         "used_resources",
+        "versions",
     )
 
     def __init__(
@@ -243,6 +246,7 @@ def __init__(
         memory_limit=0,
         local_directory=None,
         services=None,
+        versions=None,
         nanny=None,
         extra=None,
     ):
@@ -253,6 +257,7 @@ def __init__(
         self.memory_limit = memory_limit
         self.local_directory = local_directory
         self.services = services or {}
+        self.versions = versions or {}
         self.nanny = nanny
 
         self.status = "running"
@@ -1533,33 +1538,30 @@ def heartbeat_worker(
         host = get_address_host(address)
         local_now = time()
         now = now or time()
-        metrics = metrics or {}
+        assert metrics
         host_info = host_info or {}
 
         self.host_info[host]["last-seen"] = local_now
         frac = 1 / len(self.workers)
-        try:
-            self.bandwidth = (
-                self.bandwidth * (1 - frac) + metrics["bandwidth"]["total"] * frac
-            )
-            for other, (bw, count) in metrics["bandwidth"]["workers"].items():
-                if (address, other) not in self.bandwidth_workers:
-                    self.bandwidth_workers[address, other] = bw / count
-                else:
-                    alpha = (1 - frac) ** count
-                    self.bandwidth_workers[address, other] = self.bandwidth_workers[
-                        address, other
-                    ] * alpha + bw * (1 - alpha)
-            for typ, (bw, count) in metrics["bandwidth"]["types"].items():
-                if typ not in self.bandwidth_types:
-                    self.bandwidth_types[typ] = bw / count
-                else:
-                    alpha = (1 - frac) ** count
-                    self.bandwidth_types[typ] = self.bandwidth_types[
-                        typ
-                    ] * alpha + bw * (1 - alpha)
-        except KeyError:
-            pass
+        self.bandwidth = (
+            self.bandwidth * (1 - frac) + metrics["bandwidth"]["total"] * frac
+        )
+        for other, (bw, count) in metrics["bandwidth"]["workers"].items():
+            if (address, other) not in self.bandwidth_workers:
+                self.bandwidth_workers[address, other] = bw / count
+            else:
+                alpha = (1 - frac) ** count
+                self.bandwidth_workers[address, other] = self.bandwidth_workers[
+                    address, other
+                ] * alpha + bw * (1 - alpha)
+        for typ, (bw, count) in metrics["bandwidth"]["types"].items():
+            if typ not in self.bandwidth_types:
+                self.bandwidth_types[typ] = bw / count
+            else:
+                alpha = (1 - frac) ** count
+                self.bandwidth_types[typ] = self.bandwidth_types[typ] * alpha + bw * (
+                    1 - alpha
+                )
 
         ws = self.workers[address]
 
@@ -1603,6 +1605,7 @@ async def add_worker(
         pid=0,
         services=None,
         local_directory=None,
+        versions=None,
         nanny=None,
         extra=None,
     ):
@@ -1624,6 +1627,7 @@ async def add_worker(
                 name=name,
                 local_directory=local_directory,
                 services=services,
+                versions=versions,
                 nanny=nanny,
                 extra=extra,
             )
@@ -1699,15 +1703,27 @@ async def add_worker(
             self.log_event("all", {"action": "add-worker", "worker": address})
             logger.info("Register worker %s", ws)
 
+            msg = {
+                "status": "OK",
+                "time": time(),
+                "heartbeat-interval": heartbeat_interval(len(self.workers)),
+                "worker-plugins": self.worker_plugins,
+            }
+
+            version_warning = version_module.error_message(
+                version_module.get_versions(),
+                merge(
+                    {w: ws.versions for w, ws in self.workers.items()},
+                    {c: cs.versions for c, cs in self.clients.items() if cs.versions},
+                ),
+                versions,
+                client_name="This Worker",
+            )
+            if version_warning:
+                msg["warning"] = version_warning
+
             if comm:
-                await comm.write(
-                    {
-                        "status": "OK",
-                        "time": time(),
-                        "heartbeat-interval": heartbeat_interval(len(self.workers)),
-                        "worker-plugins": self.worker_plugins,
-                    }
-                )
+                await comm.write(msg)
             await self.handle_worker(comm=comm, worker=address)
 
     def update_graph(
@@ -2438,7 +2454,7 @@ def report(self, msg, ts=None, client=None):
                 if self.status == "running":
                     logger.critical("Tried writing to closed comm: %s", msg)
 
-    async def add_client(self, comm, client=None):
+    async def add_client(self, comm, client=None, versions=None):
         """ Add client to network
 
         We listen to all future messages from this Comm.
@@ -2447,12 +2463,21 @@ async def add_client(self, comm, client=None):
         comm.name = "Scheduler->Client"
         logger.info("Receive client connection: %s", client)
         self.log_event(["all", client], {"action": "add-client", "client": client})
-        self.clients[client] = ClientState(client)
+        self.clients[client] = ClientState(client, versions=versions)
+
         try:
             bcomm = BatchedSend(interval="2ms", loop=self.loop)
             bcomm.start(comm)
             self.client_comms[client] = bcomm
-            bcomm.send({"op": "stream-start"})
+            msg = {"op": "stream-start"}
+            version_warning = version_module.error_message(
+                version_module.get_versions(),
+                {w: ws.versions for w, ws in self.workers.items()},
+                versions,
+            )
+            if version_warning:
+                msg["warning"] = version_warning
+            bcomm.send(msg)
 
             try:
                 await self.handle_stream(comm=comm, extra={"client": client})
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 079a673cbcf..9450f08fd75 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -3724,7 +3724,7 @@ def test_get_versions(c):
     # that this does not raise
 
     v = c.get_versions(packages=["requests"])
-    assert dict(v["client"]["packages"]["optional"])["requests"] == requests.__version__
+    assert v["client"]["packages"]["requests"] == requests.__version__
 
 
 @gen_cluster(client=True)
@@ -4007,7 +4007,11 @@ def test_retire_many_workers(c, s, *workers):
     results = yield c.gather(futures)
     assert results == list(range(100))
 
+    while len(s.workers) != 3:
+        yield gen.sleep(0.01)
+
     assert len(s.has_what) == len(s.nthreads) == 3
+
     assert all(future.done() for future in futures)
     assert all(s.tasks[future.key].state == "memory" for future in futures)
     for w, keys in s.has_what.items():
@@ -5284,40 +5288,39 @@ def test_client_active_bad_port():
 @pytest.mark.parametrize("direct", [True, False])
 def test_turn_off_pickle(direct):
     @gen_cluster()
-    def test(s, a, b):
+    async def test(s, a, b):
         import numpy as np
 
-        c = yield Client(s.address, asynchronous=True, serializers=["dask", "msgpack"])
-        try:
-            assert (yield c.submit(inc, 1)) == 2
-            yield c.submit(np.ones, 5)
-            yield c.scatter(1)
+        async with Client(
+            s.address, asynchronous=True, serializers=["dask", "msgpack"]
+        ) as c:
+            assert (await c.submit(inc, 1)) == 2
+            await c.submit(np.ones, 5)
+            await c.scatter(1)
 
             # Can't send complex data
             with pytest.raises(TypeError):
-                future = yield c.scatter(inc)
+                future = await c.scatter(inc)
 
             # can send complex tasks (this uses pickle regardless)
             future = c.submit(lambda x: x, inc)
-            yield wait(future)
+            await wait(future)
 
             # but can't receive complex results
             with pytest.raises(TypeError):
-                yield c.gather(future, direct=direct)
+                await c.gather(future, direct=direct)
 
             # Run works
-            result = yield c.run(lambda: 1)
+            result = await c.run(lambda: 1)
             assert list(result.values()) == [1, 1]
-            result = yield c.run_on_scheduler(lambda: 1)
+            result = await c.run_on_scheduler(lambda: 1)
             assert result == 1
 
             # But not with complex return values
             with pytest.raises(TypeError):
-                yield c.run(lambda: inc)
+                await c.run(lambda: inc)
             with pytest.raises(TypeError):
-                yield c.run_on_scheduler(lambda: inc)
-        finally:
-            yield c.close()
+                await c.run_on_scheduler(lambda: inc)
 
     test()
 
@@ -5697,7 +5700,6 @@ async def test_profile_server(c, s, a, b):
         try:
             x = c.map(slowinc, range(10), delay=0.01, workers=a.address, pure=False)
             await wait(x)
-
             await asyncio.gather(
                 c.run(slowinc, 1, delay=0.5), c.run_on_scheduler(slowdec, 1, delay=0.5)
             )
@@ -5821,7 +5823,7 @@ async def ff():
             assert c.sync(ff) == 1
 
 
-@pytest.mark.xfail(reason="known intermittent failure")
+@pytest.mark.skip(reason="known intermittent failure")
 @gen_cluster(client=True)
 async def test_dont_hold_on_to_large_messages(c, s, a, b):
     np = pytest.importorskip("numpy")
diff --git a/distributed/tests/test_collections.py b/distributed/tests/test_collections.py
index 7fe8467b14b..0843d711761 100644
--- a/distributed/tests/test_collections.py
+++ b/distributed/tests/test_collections.py
@@ -109,19 +109,19 @@ def test_bag_groupby_tasks_default(c, s, a, b):
 
 @pytest.mark.parametrize("wait", [wait, lambda x: None])
 def test_dataframe_set_index_sync(wait, client):
-    df = dd.demo.make_timeseries(
-        "2000",
-        "2001",
-        {"value": float, "name": str, "id": int},
+    df = dask.datasets.timeseries(
+        start="2000",
+        end="2001",
+        dtypes={"value": float, "name": str, "id": int},
         freq="2H",
         partition_freq="1M",
         seed=1,
     )
-    df = client.persist(df)
+    df = df.persist()
     wait(df)
 
     df2 = df.set_index("name", shuffle="tasks")
-    df2 = client.persist(df2)
+    df2 = df2.persist()
 
     assert len(df2)
 
diff --git a/distributed/tests/test_failed_workers.py b/distributed/tests/test_failed_workers.py
index 27bce439da4..8f790edf20e 100644
--- a/distributed/tests/test_failed_workers.py
+++ b/distributed/tests/test_failed_workers.py
@@ -415,8 +415,9 @@ def test_restart_timeout_on_long_running_task(c, s, a):
     assert "timeout" not in text.lower()
 
 
-@gen_cluster(client=True, scheduler_kwargs={"worker_ttl": "100ms"})
+@gen_cluster(client=True, scheduler_kwargs={"worker_ttl": "500ms"})
 def test_worker_time_to_live(c, s, a, b):
+    assert set(s.workers) == {a.address, b.address}
     a.periodic_callbacks["heartbeat"].stop()
     yield gen.sleep(0.010)
     assert set(s.workers) == {a.address, b.address}
@@ -424,13 +425,6 @@ def test_worker_time_to_live(c, s, a, b):
     start = time()
     while set(s.workers) == {a.address, b.address}:
         yield gen.sleep(0.050)
-        assert time() < start + 1
+        assert time() < start + 2
 
     set(s.workers) == {b.address}
-
-    start = time()
-    while b.status == "running":
-        yield gen.sleep(0.050)
-        assert time() < start + 1
-
-    assert b.status in ("closed", "closing")
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 077c5530260..a8b07ea499e 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -408,8 +408,8 @@ def test_delete(c, s, a):
 def test_filtered_communication(s, a, b):
     c = yield connect(s.address)
     f = yield connect(s.address)
-    yield c.write({"op": "register-client", "client": "c"})
-    yield f.write({"op": "register-client", "client": "f"})
+    yield c.write({"op": "register-client", "client": "c", "versions": {}})
+    yield f.write({"op": "register-client", "client": "f", "versions": {}})
     yield c.read()
     yield f.read()
 
@@ -942,10 +942,11 @@ def test_worker_arrives_with_processing_data(c, s, a, b):
     yield w.close()
 
 
+@pytest.mark.slow
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)])
 def test_worker_breaks_and_returns(c, s, a):
     future = c.submit(slowinc, 1, delay=0.1)
-    for i in range(10):
+    for i in range(20):
         future = c.submit(slowinc, future, delay=0.1)
 
     yield wait(future)
@@ -957,10 +958,10 @@ def test_worker_breaks_and_returns(c, s, a):
     yield wait(future, timeout=10)
     end = time()
 
-    assert end - start < 1
+    assert end - start < 2
 
     states = frequencies(ts.state for ts in s.tasks.values())
-    assert states == {"memory": 1, "released": 10}
+    assert states == {"memory": 1, "released": 20}
 
 
 @gen_cluster(client=True, nthreads=[])
@@ -1211,14 +1212,15 @@ def test_profile_metadata(c, s, a, b):
 
 @gen_cluster(client=True, worker_kwargs={"profile_cycle_interval": 100})
 def test_profile_metadata_keys(c, s, a, b):
-    start = time() - 1
     x = c.map(slowinc, range(10), delay=0.05)
     y = c.map(slowdec, range(10), delay=0.05)
     yield wait(x + y)
 
     meta = yield s.get_profile_metadata(profile_cycle_interval=0.100)
     assert set(meta["keys"]) == {"slowinc", "slowdec"}
-    assert len(meta["counts"]) == len(meta["keys"]["slowinc"])
+    assert (
+        len(meta["counts"]) - 3 <= len(meta["keys"]["slowinc"]) <= len(meta["counts"])
+    )
 
 
 @gen_cluster(client=True)
@@ -1501,21 +1503,32 @@ def qux(x):
     yield f
 
 
-@gen_cluster(client=True, config={"distributed.scheduler.idle-timeout": "200ms"})
-def test_idle_timeout(c, s, a, b):
+@gen_cluster(client=True)
+def test_collect_versions(c, s, a, b):
+    cs = s.clients[c.id]
+    (w1, w2) = s.workers.values()
+    assert cs.versions
+    assert w1.versions
+    assert w2.versions
+    assert "dask" in str(cs.versions)
+    assert cs.versions == w1.versions == w2.versions
+
+
+@gen_cluster(client=True, config={"distributed.scheduler.idle-timeout": "500ms"})
+async def test_idle_timeout(c, s, a, b):
     future = c.submit(slowinc, 1)
-    yield future
+    await future
 
     assert s.status != "closed"
 
     start = time()
     while s.status != "closed":
-        yield gen.sleep(0.01)
-    assert time() < start + 3
+        await gen.sleep(0.01)
+        assert time() < start + 3
 
     start = time()
     while not (a.status == "closed" and b.status == "closed"):
-        yield gen.sleep(0.01)
+        await gen.sleep(0.01)
         assert time() < start + 1
 
 
@@ -1851,7 +1864,7 @@ def inc_slow(x):
         # need to sleep for at least 0.5 seconds to give the worker a chance to
         # reconnect (Heartbeat timing)
         if x in ALREADY_CALCULATED:
-            time.sleep(0.5)
+            time.sleep(1)
         ALREADY_CALCULATED.append(x)
         return x + 1
 
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 9cf32c1eec7..df886b9431a 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -1168,9 +1168,10 @@ def test_statistical_profiling_cycle(c, s, a, b):
     x = yield a.get_profile(start=time() + 10, stop=time() + 20)
     assert not x["count"]
 
-    x = yield a.get_profile(start=0, stop=time())
+    x = yield a.get_profile(start=0, stop=time() + 10)
+    recent = a.profile_recent["count"]
     actual = sum(p["count"] for _, p in a.profile_history) + a.profile_recent["count"]
-    x2 = yield a.get_profile(start=0, stop=time())
+    x2 = yield a.get_profile(start=0, stop=time() + 10)
     assert x["count"] <= actual <= x2["count"]
 
     y = yield a.get_profile(start=end - 0.300, stop=time())
diff --git a/distributed/utils.py b/distributed/utils.py
index e4884e4d16c..df601291f06 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -81,6 +81,16 @@ def _initialize_mp_context():
         preload = ["distributed"]
         if "pkg_resources" in sys.modules:
             preload.append("pkg_resources")
+
+        from .versions import required_packages, optional_packages
+
+        for pkg, _ in required_packages + optional_packages:
+            try:
+                importlib.import_module(pkg)
+            except ImportError:
+                pass
+            else:
+                preload.append(pkg)
         ctx.set_forkserver_preload(preload)
         return ctx
 
diff --git a/distributed/versions.py b/distributed/versions.py
index a769c9ab032..e3a5e5d0f17 100644
--- a/distributed/versions.py
+++ b/distributed/versions.py
@@ -1,5 +1,8 @@
 """ utilities for package version introspection """
 
+from __future__ import print_function, division, absolute_import
+
+from collections import defaultdict
 import platform
 import struct
 import os
@@ -7,8 +10,6 @@
 import locale
 import importlib
 
-from .utils import ignoring
-
 
 required_packages = [
     ("dask", lambda p: p.__version__),
@@ -21,10 +22,7 @@
 
 optional_packages = [
     ("numpy", lambda p: p.__version__),
-    ("pandas", lambda p: p.__version__),
-    ("bokeh", lambda p: p.__version__),
     ("lz4", lambda p: p.__version__),
-    ("dask_ml", lambda p: p.__version__),
     ("blosc", lambda p: p.__version__),
 ]
 
@@ -38,11 +36,11 @@ def get_versions(packages=None):
 
     d = {
         "host": get_system_info(),
-        "packages": {
-            "required": get_package_info(required_packages),
-            "optional": get_package_info(optional_packages + list(packages)),
-        },
+        "packages": get_package_info(
+            required_packages + optional_packages + list(packages)
+        ),
     }
+
     return d
 
 
@@ -66,6 +64,8 @@ def get_system_info():
 
 def version_of_package(pkg):
     """ Try a variety of common ways to get the version of a package """
+    from .utils import ignoring
+
     with ignoring(AttributeError):
         return pkg.__version__
     with ignoring(AttributeError):
@@ -96,4 +96,42 @@ def get_package_info(pkgs):
         except Exception:
             pversions.append((modname, None))
 
-    return pversions
+    return dict(pversions)
+
+
+def error_message(scheduler, workers, client, client_name="client"):
+    # we care about the required & optional packages matching
+    try:
+        client_versions = client["packages"]
+        versions = [("scheduler", scheduler["packages"])]
+        versions.extend((w, d["packages"]) for w, d in sorted(workers.items()))
+    except KeyError:
+        return (
+            "Version mismatch for dask.distributed. "
+            "The scheduler has version >= 1.28.0 "
+            "but some other component is less than this"
+        )
+
+    mismatched = defaultdict(list)
+    for name, vers in versions:
+        for pkg, cv in client_versions.items():
+            v = vers.get(pkg, "MISSING")
+            if cv != v:
+                mismatched[pkg].append((name, v))
+
+    if mismatched:
+        from .utils import asciitable
+
+        errs = []
+        for pkg, versions in sorted(mismatched.items()):
+            rows = [(client_name, client_versions[pkg])]
+            rows.extend(versions)
+            errs.append("%s\n%s" % (pkg, asciitable(["", "version"], rows)))
+
+        return "Mismatched versions found\n" "\n" "%s" % ("\n\n".join(errs))
+    else:
+        return ""
+
+
+class VersionMismatchWarning(Warning):
+    """Indicates version mismatch between nodes"""
diff --git a/distributed/worker.py b/distributed/worker.py
index 716806a67d6..407c381c83f 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -65,6 +65,7 @@
 )
 from .utils_comm import pack_data, gather_from_workers, retry_operation
 from .utils_perf import ThrottledGC, enable_gc_diagnosis, disable_gc_diagnosis
+from .versions import get_versions
 
 logger = logging.getLogger(__name__)
 
@@ -724,6 +725,10 @@ def __repr__(self):
             )
         )
 
+    @property
+    def logs(self):
+        return self._deque_handler.deque
+
     @property
     def worker_address(self):
         """ For API compatibility with Nanny """
@@ -797,7 +802,6 @@ async def _register_with_scheduler(self):
         while True:
             try:
                 _start = time()
-                types = {k: typename(v) for k, v in self.data.items()}
                 comm = await connect(
                     self.scheduler.address, connection_args=self.connection_args
                 )
@@ -812,7 +816,7 @@ async def _register_with_scheduler(self):
                         nthreads=self.nthreads,
                         name=self.name,
                         nbytes=self.nbytes,
-                        types=types,
+                        types={k: typename(v) for k, v in self.data.items()},
                         now=time(),
                         resources=self.total_resources,
                         memory_limit=self.memory_limit,
@@ -820,13 +824,18 @@ async def _register_with_scheduler(self):
                         services=self.service_ports,
                         nanny=self.nanny,
                         pid=os.getpid(),
+                        versions=get_versions(),
                         metrics=await self.get_metrics(),
                         extra=await self.get_startup_information(),
                     ),
                     serializers=["msgpack"],
                 )
                 future = comm.read(deserializers=["msgpack"])
+
                 response = await future
+                if response.get("warning"):
+                    logger.warning(response["warning"])
+
                 _end = time()
                 middle = (_start + _end) / 2
                 self._update_latency(_end - start)
@@ -886,7 +895,13 @@ async def heartbeat(self):
                 self._update_latency(end - start)
 
                 if response["status"] == "missing":
-                    await self._register_with_scheduler()
+                    for i in range(10):
+                        if self.status != "running":
+                            break
+                        else:
+                            await asyncio.sleep(0.05)
+                    else:
+                        await self._register_with_scheduler()
                     return
                 self.scheduler_delay = response["time"] - middle
                 self.periodic_callbacks["heartbeat"].callback_time = (

From baf5903c6994a286e02c36f9df48c87776b062d7 Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Fri, 10 Jan 2020 17:04:43 +0000
Subject: [PATCH 0622/1550] Remove locale check that fails on OS X (#3360)

---
 distributed/versions.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/distributed/versions.py b/distributed/versions.py
index e3a5e5d0f17..0b97a6f7ac0 100644
--- a/distributed/versions.py
+++ b/distributed/versions.py
@@ -7,7 +7,6 @@
 import struct
 import os
 import sys
-import locale
 import importlib
 
 
@@ -56,7 +55,6 @@ def get_system_info():
         ("byteorder", "%s" % sys.byteorder),
         ("LC_ALL", "%s" % os.environ.get("LC_ALL", "None")),
         ("LANG", "%s" % os.environ.get("LANG", "None")),
-        ("LOCALE", "%s.%s" % locale.getlocale()),
     ]
 
     return host

From e7ac25a0e9655cf68f81ee5e30e423b6ab375f4b Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Sat, 11 Jan 2020 11:41:07 -0600
Subject: [PATCH 0623/1550] Add --worker-class option to dask-worker CLI
 (#3364)

---
 distributed/cli/dask_worker.py            | 18 ++++++++--
 distributed/cli/tests/test_dask_worker.py | 43 +++++++++++++++++++++++
 2 files changed, 58 insertions(+), 3 deletions(-)

diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
index a252fe1a232..e76bed2e9bc 100755
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -11,7 +11,7 @@
 import dask
 from dask.utils import ignoring
 from dask.system import CPU_COUNT
-from distributed import Nanny, Worker
+from distributed import Nanny
 from distributed.security import Security
 from distributed.cli.utils import check_python_3, install_signal_handlers
 from distributed.comm import get_address_host_port
@@ -20,7 +20,7 @@
     enable_proctitle_on_children,
     enable_proctitle_on_current,
 )
-from distributed.utils import deserialize_for_cli
+from distributed.utils import deserialize_for_cli, import_term
 
 from toolz import valmap
 from tornado.ioloop import IOLoop, TimeoutError
@@ -190,6 +190,13 @@
     show_default=True,
     help="Random amount by which to stagger lifetime values",
 )
+@click.option(
+    "--worker-class",
+    type=str,
+    default="dask.distributed.Worker",
+    show_default=True,
+    help="Worker class used to instantiate workers from.",
+)
 @click.option(
     "--lifetime-restart/--no-lifetime-restart",
     "lifetime_restart",
@@ -233,6 +240,7 @@ def main(
     tls_cert,
     tls_key,
     dashboard_address,
+    worker_class,
     **kwargs
 ):
     g0, g1, g2 = gc.get_threshold()  # https://github.com/dask/distributed/issues/1653
@@ -339,13 +347,17 @@ def del_pid_file():
 
     loop = IOLoop.current()
 
+    worker_class = import_term(worker_class)
+    if nanny:
+        kwargs["worker_class"] = worker_class
+
     if nanny:
         kwargs.update({"worker_port": worker_port, "listen_address": listen_address})
         t = Nanny
     else:
         if nanny_port:
             kwargs["service_ports"] = {"nanny": nanny_port}
-        t = Worker
+        t = worker_class
 
     if (
         not scheduler
diff --git a/distributed/cli/tests/test_dask_worker.py b/distributed/cli/tests/test_dask_worker.py
index c509772d113..767613f2a26 100644
--- a/distributed/cli/tests/test_dask_worker.py
+++ b/distributed/cli/tests/test_dask_worker.py
@@ -340,3 +340,46 @@ async def test_integer_names(cleanup):
                 await asyncio.sleep(0.01)
             [ws] = s.workers.values()
             assert ws.name == 123
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("nanny", ["--nanny", "--no-nanny"])
+async def test_worker_class(cleanup, tmp_path, nanny):
+    # Create module with custom worker class
+    WORKER_CLASS_TEXT = """
+from distributed.worker import Worker
+
+class MyWorker(Worker):
+    pass
+"""
+    tmpdir = str(tmp_path)
+    tmpfile = str(tmp_path / "myworker.py")
+    with open(tmpfile, "w") as f:
+        f.write(WORKER_CLASS_TEXT)
+
+    # Put module on PYTHONPATH
+    env = os.environ.copy()
+    if "PYTHONPATH" in env:
+        env["PYTHONPATH"] = tmpdir + ":" + env["PYTHONPATH"]
+    else:
+        env["PYTHONPATH"] = tmpdir
+
+    async with Scheduler(port=0) as s:
+        async with Client(s.address, asynchronous=True) as c:
+            with popen(
+                [
+                    "dask-worker",
+                    s.address,
+                    nanny,
+                    "--worker-class",
+                    "myworker.MyWorker",
+                ],
+                env=env,
+            ) as worker:
+                await c.wait_for_workers(1)
+
+                def worker_type(dask_worker):
+                    return type(dask_worker).__name__
+
+                worker_types = await c.run(worker_type)
+                assert all(name == "MyWorker" for name in worker_types.values())

From 44d24098b503d463ae5abfe1683934627973c14c Mon Sep 17 00:00:00 2001
From: byjott <jochen.ott@blue-yonder.com>
Date: Mon, 13 Jan 2020 17:38:21 +0100
Subject: [PATCH 0624/1550] Fix scheduler state in case of worker name
 collision (#3366)

---
 distributed/scheduler.py            | 24 +++++++++++++-----------
 distributed/tests/test_scheduler.py | 24 +++++++++++++++++++++++-
 distributed/worker.py               |  2 +-
 3 files changed, 37 insertions(+), 13 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 66827935b19..d16026b1c24 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1619,6 +1619,16 @@ async def add_worker(
             if ws is not None:
                 raise ValueError("Worker already exists %s" % ws)
 
+            if name in self.aliases:
+                msg = {
+                    "status": "error",
+                    "message": "name taken, %s" % name,
+                    "time": time(),
+                }
+                if comm:
+                    await comm.write(msg)
+                return
+
             self.workers[address] = ws = WorkerState(
                 address=address,
                 pid=pid,
@@ -1632,16 +1642,6 @@ async def add_worker(
                 extra=extra,
             )
 
-            if name in self.aliases:
-                msg = {
-                    "status": "error",
-                    "message": "name taken, %s" % name,
-                    "time": time(),
-                }
-                if comm:
-                    await comm.write(msg)
-                return
-
             if "addresses" not in self.host_info[host]:
                 self.host_info[host].update({"addresses": set(), "nthreads": 0})
 
@@ -2118,10 +2118,12 @@ def remove_worker(self, comm=None, address=None, safe=False, close=True):
         with log_errors():
             if self.status == "closed":
                 return
+
+            address = self.coerce_address(address)
+
             if address not in self.workers:
                 return "already-removed"
 
-            address = self.coerce_address(address)
             host = get_address_host(address)
 
             ws = self.workers[address]
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index a8b07ea499e..c493c41dd21 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -217,6 +217,15 @@ def test_remove_worker_from_scheduler(s, a, b):
     s.validate_state()
 
 
+@gen_cluster()
+def test_remove_worker_by_name_from_scheduler(s, a, b):
+    assert a.address in s.stream_comms
+    assert s.remove_worker(address=a.name) == "OK"
+    assert a.address not in s.nthreads
+    assert s.remove_worker(address=a.address) == "already-removed"
+    s.validate_state()
+
+
 @gen_cluster(config={"distributed.scheduler.events-cleanup-delay": "10 ms"})
 def test_clear_events_worker_removal(s, a, b):
     assert a.address in s.events
@@ -566,7 +575,7 @@ def test_coerce_address():
             "tcp://127.0.0.1:8000",
             "tcp://[::1]:8000",
         )
-        assert s.coerce_address(u"localhost:8000") in (
+        assert s.coerce_address("localhost:8000") in (
             "tcp://127.0.0.1:8000",
             "tcp://[::1]:8000",
         )
@@ -1966,3 +1975,16 @@ async def test_multiple_listeners(cleanup):
     log = log.getvalue()
     assert re.search(r"Scheduler at:\s*tcp://", log)
     assert re.search(r"Scheduler at:\s*inproc://", log)
+
+
+@gen_cluster(nthreads=[("127.0.0.1", 1)])
+async def test_worker_name_collision(s, a):
+    # test that a name collision for workers produces the expected respsone
+    # and leaves the data structures of Scheduler in a good state
+    # is not updated by the second worker
+    with pytest.raises(ValueError, match=f"name taken, {a.name!r}"):
+        await Worker(s.address, name=a.name, loop=s.loop, host="127.0.0.1")
+
+    s.validate_state()
+    assert set(s.workers) == {a.address}
+    assert s.aliases == {a.name: a.address}
diff --git a/distributed/worker.py b/distributed/worker.py
index 407c381c83f..f4a662ce44f 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -1096,7 +1096,7 @@ async def close(
             for pc in self.periodic_callbacks.values():
                 pc.stop()
             with ignoring(EnvironmentError, gen.TimeoutError):
-                if report:
+                if report and self.contact_address is not None:
                     await gen.with_timeout(
                         timedelta(seconds=timeout),
                         self.scheduler.unregister(

From d5864834af508016fa272db882829a4cbdd74dd0 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Mon, 13 Jan 2020 12:48:08 -0600
Subject: [PATCH 0625/1550] Close connection comm on retry (#3365)

---
 distributed/comm/core.py | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/distributed/comm/core.py b/distributed/comm/core.py
index 11f74a1aba8..42c95e3579e 100644
--- a/distributed/comm/core.py
+++ b/distributed/comm/core.py
@@ -1,11 +1,9 @@
 from abc import ABC, abstractmethod, abstractproperty
 import asyncio
-from datetime import timedelta
 import logging
 import weakref
 
 import dask
-from tornado import gen
 
 from ..metrics import time
 from ..utils import parse_timedelta, ignoring
@@ -211,11 +209,9 @@ def _raise(error):
                 future = connector.connect(
                     loc, deserialize=deserialize, **(connection_args or {})
                 )
-                with ignoring(gen.TimeoutError):
-                    comm = await gen.with_timeout(
-                        timedelta(seconds=min(deadline - time(), 1)),
-                        future,
-                        quiet_exceptions=EnvironmentError,
+                with ignoring(asyncio.TimeoutError):
+                    comm = await asyncio.wait_for(
+                        future, timeout=min(deadline - time(), 1)
                     )
                     break
             if not comm:

From 8472a0371ef9ddab5a49e089ef88d2fd16036448 Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Tue, 14 Jan 2020 11:23:31 +0000
Subject: [PATCH 0626/1550] Add cluster map dashboard (#3361)

* New diagram style dashboard

* Minor improvements

* Switch greensock for anime, refactor and apply prettier

* Move sending to the socket handler

* Add websocket scheduler plugin

* Rename eventstream endpoint and remove incorrect type hint

* Switch to event stream

* Revert conftest

* Simplify using anime.js timelines

* Tidy up code, fox reset bugs and reconnect websocket

* Add title and meta

* Check status code instead of looking for body content
---
 distributed/dashboard/scheduler.py            |  11 +-
 .../static/css/individual-cluster-map.css     |  54 +++
 .../static/individual-cluster-map.html        |  27 ++
 distributed/dashboard/static/js/anime.min.js  |   8 +
 .../static/js/individual-cluster-map.js       | 367 ++++++++++++++++++
 .../static/js/reconnecting-websocket.min.js   |   8 +
 .../dashboard/tests/test_scheduler_bokeh.py   |   2 +-
 7 files changed, 475 insertions(+), 2 deletions(-)
 create mode 100644 distributed/dashboard/static/css/individual-cluster-map.css
 create mode 100644 distributed/dashboard/static/individual-cluster-map.html
 create mode 100644 distributed/dashboard/static/js/anime.min.js
 create mode 100644 distributed/dashboard/static/js/individual-cluster-map.js
 create mode 100644 distributed/dashboard/static/js/reconnecting-websocket.min.js

diff --git a/distributed/dashboard/scheduler.py b/distributed/dashboard/scheduler.py
index 6a52063b9f1..2c0520161b3 100644
--- a/distributed/dashboard/scheduler.py
+++ b/distributed/dashboard/scheduler.py
@@ -1,5 +1,7 @@
 from datetime import datetime
 from functools import partial
+import os
+import os.path
 import json
 import logging
 
@@ -234,11 +236,18 @@ def get(self):
 class IndividualPlots(RequestHandler):
     def get(self):
         bokeh_server = self.server.services["dashboard"]
-        result = {
+        individual_bokeh = {
             uri.strip("/").replace("-", " ").title(): uri
             for uri in bokeh_server.apps
             if uri.lstrip("/").startswith("individual-") and not uri.endswith(".json")
         }
+        individual_static = {
+            uri.strip("/").replace(".html", "").replace("-", " ").title(): "/statics/"
+            + uri
+            for uri in os.listdir(os.path.join(os.path.dirname(__file__), "static"))
+            if uri.lstrip("/").startswith("individual-") and uri.endswith(".html")
+        }
+        result = {**individual_bokeh, **individual_static}
         self.write(result)
 
 
diff --git a/distributed/dashboard/static/css/individual-cluster-map.css b/distributed/dashboard/static/css/individual-cluster-map.css
new file mode 100644
index 00000000000..e45b735dd31
--- /dev/null
+++ b/distributed/dashboard/static/css/individual-cluster-map.css
@@ -0,0 +1,54 @@
+#vis {
+    height: 100%;
+}
+
+.node {
+    stroke: black;
+    stroke-width: 3px;
+    stroke-linejoin: round;
+    filter: drop-shadow(-1px 1px 2px rgba(0, 0, 0, .4));
+}
+
+.worker {
+    fill: #ECB172;
+}
+
+.scheduler {
+    fill: #c5c1ff
+}
+
+.projectile {
+    stroke-width: 4;
+    stroke-linecap: round;
+    fill: transparent;
+    stroke-dasharray: 20 1000;
+    stroke-dashoffset: 0;
+    animation: shoot 0.5s linear infinite;
+}
+
+.transfer {
+    stroke-width: 4;
+    stroke-linecap: round;
+    fill: transparent;
+    stroke-dasharray: 10;
+    stroke-dashoffset: 10;
+    animation: transfer 1s linear infinite;
+}
+
+@keyframes transfer {
+    from {
+        stroke-dashoffset: -100;
+    }
+    to {
+        stroke-dashoffset: 100;
+    }
+}
+
+@keyframes shoot {
+    from {
+        stroke-dashoffset: 0;
+    }
+    to {
+        stroke-dashoffset: 1000;
+    }
+}
\ No newline at end of file
diff --git a/distributed/dashboard/static/individual-cluster-map.html b/distributed/dashboard/static/individual-cluster-map.html
new file mode 100644
index 00000000000..20ab9d53f5b
--- /dev/null
+++ b/distributed/dashboard/static/individual-cluster-map.html
@@ -0,0 +1,27 @@
+<html>
+  <head>
+    <title>Dask: Cluster Map</title>
+    <meta charset="utf-8" />
+    <meta
+      name="viewport"
+      content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no"
+    />
+    <link rel="stylesheet" href="css/individual-cluster-map.css" />
+          <style>
          .commit-tease,
          .user-profile-mini-avatar,
          .avatar,
          .vcard-details,
          .signup-prompt-bg {
            display: none !IMPORTANT;
          }
        </style>
         <script>
          document.addEventListener('DOMContentLoaded', function() {
            this.querySelectorAll('a').forEach(anchor => {
              anchor.addEventListener('click', e => {
                e.preventDefault();

                const redact = new URLSearchParams(window.location.search).get('redact');
                const hasExistingParams = anchor.href.includes('?');
                window.location.href = anchor.href + (hasExistingParams ? `&redact=${redact}` : `?redact=${redact}`);
              });
            });
          });
        </script>
 </head>
+
+  <body>
+    <svg
+      id="vis"
+      version="1.1"
+      xmlns="http://www.w3.org/2000/svg"
+      xmlns:xlink="http://www.w3.org/1999/xlink"
+      width="100%"
+      height="100%"
+    ></svg>
+
+    <script src="js/anime.min.js"></script>
+    <script src="js/reconnecting-websocket.min.js"></script>
+
+    <script src="js/individual-cluster-map.js"></script>
+  </body>
+</html>
diff --git a/distributed/dashboard/static/js/anime.min.js b/distributed/dashboard/static/js/anime.min.js
new file mode 100644
index 00000000000..99b263aaebc
--- /dev/null
+++ b/distributed/dashboard/static/js/anime.min.js
@@ -0,0 +1,8 @@
+/*
+ * anime.js v3.1.0
+ * (c) 2019 Julian Garnier
+ * Released under the MIT license
+ * animejs.com
+ */
+
+!function(n,e){"object"==typeof exports&&"undefined"!=typeof module?module.exports=e():"function"==typeof define&&define.amd?define(e):n.anime=e()}(this,function(){"use strict";var n={update:null,begin:null,loopBegin:null,changeBegin:null,change:null,changeComplete:null,loopComplete:null,complete:null,loop:1,direction:"normal",autoplay:!0,timelineOffset:0},e={duration:1e3,delay:0,endDelay:0,easing:"easeOutElastic(1, .5)",round:0},r=["translateX","translateY","translateZ","rotate","rotateX","rotateY","rotateZ","scale","scaleX","scaleY","scaleZ","skew","skewX","skewY","perspective"],t={CSS:{},springs:{}};function a(n,e,r){return Math.min(Math.max(n,e),r)}function o(n,e){return n.indexOf(e)>-1}function u(n,e){return n.apply(null,e)}var i={arr:function(n){return Array.isArray(n)},obj:function(n){return o(Object.prototype.toString.call(n),"Object")},pth:function(n){return i.obj(n)&&n.hasOwnProperty("totalLength")},svg:function(n){return n instanceof SVGElement},inp:function(n){return n instanceof HTMLInputElement},dom:function(n){return n.nodeType||i.svg(n)},str:function(n){return"string"==typeof n},fnc:function(n){return"function"==typeof n},und:function(n){return void 0===n},hex:function(n){return/(^#[0-9A-F]{6}$)|(^#[0-9A-F]{3}$)/i.test(n)},rgb:function(n){return/^rgb/.test(n)},hsl:function(n){return/^hsl/.test(n)},col:function(n){return i.hex(n)||i.rgb(n)||i.hsl(n)},key:function(r){return!n.hasOwnProperty(r)&&!e.hasOwnProperty(r)&&"targets"!==r&&"keyframes"!==r}};function c(n){var e=/\(([^)]+)\)/.exec(n);return e?e[1].split(",").map(function(n){return parseFloat(n)}):[]}function s(n,e){var r=c(n),o=a(i.und(r[0])?1:r[0],.1,100),u=a(i.und(r[1])?100:r[1],.1,100),s=a(i.und(r[2])?10:r[2],.1,100),f=a(i.und(r[3])?0:r[3],.1,100),l=Math.sqrt(u/o),d=s/(2*Math.sqrt(u*o)),p=d<1?l*Math.sqrt(1-d*d):0,h=1,v=d<1?(d*l-f)/p:-f+l;function g(n){var r=e?e*n/1e3:n;return r=d<1?Math.exp(-r*d*l)*(h*Math.cos(p*r)+v*Math.sin(p*r)):(h+v*r)*Math.exp(-r*l),0===n||1===n?n:1-r}return e?g:function(){var e=t.springs[n];if(e)return e;for(var r=0,a=0;;)if(1===g(r+=1/6)){if(++a>=16)break}else a=0;var o=r*(1/6)*1e3;return t.springs[n]=o,o}}function f(n){return void 0===n&&(n=10),function(e){return Math.round(e*n)*(1/n)}}var l,d,p=function(){var n=11,e=1/(n-1);function r(n,e){return 1-3*e+3*n}function t(n,e){return 3*e-6*n}function a(n){return 3*n}function o(n,e,o){return((r(e,o)*n+t(e,o))*n+a(e))*n}function u(n,e,o){return 3*r(e,o)*n*n+2*t(e,o)*n+a(e)}return function(r,t,a,i){if(0<=r&&r<=1&&0<=a&&a<=1){var c=new Float32Array(n);if(r!==t||a!==i)for(var s=0;s<n;++s)c[s]=o(s*e,r,a);return function(n){return r===t&&a===i?n:0===n||1===n?n:o(f(n),t,i)}}function f(t){for(var i=0,s=1,f=n-1;s!==f&&c[s]<=t;++s)i+=e;var l=i+(t-c[--s])/(c[s+1]-c[s])*e,d=u(l,r,a);return d>=.001?function(n,e,r,t){for(var a=0;a<4;++a){var i=u(e,r,t);if(0===i)return e;e-=(o(e,r,t)-n)/i}return e}(t,l,r,a):0===d?l:function(n,e,r,t,a){for(var u,i,c=0;(u=o(i=e+(r-e)/2,t,a)-n)>0?r=i:e=i,Math.abs(u)>1e-7&&++c<10;);return i}(t,i,i+e,r,a)}}}(),h=(l={linear:function(){return function(n){return n}}},d={Sine:function(){return function(n){return 1-Math.cos(n*Math.PI/2)}},Circ:function(){return function(n){return 1-Math.sqrt(1-n*n)}},Back:function(){return function(n){return n*n*(3*n-2)}},Bounce:function(){return function(n){for(var e,r=4;n<((e=Math.pow(2,--r))-1)/11;);return 1/Math.pow(4,3-r)-7.5625*Math.pow((3*e-2)/22-n,2)}},Elastic:function(n,e){void 0===n&&(n=1),void 0===e&&(e=.5);var r=a(n,1,10),t=a(e,.1,2);return function(n){return 0===n||1===n?n:-r*Math.pow(2,10*(n-1))*Math.sin((n-1-t/(2*Math.PI)*Math.asin(1/r))*(2*Math.PI)/t)}}},["Quad","Cubic","Quart","Quint","Expo"].forEach(function(n,e){d[n]=function(){return function(n){return Math.pow(n,e+2)}}}),Object.keys(d).forEach(function(n){var e=d[n];l["easeIn"+n]=e,l["easeOut"+n]=function(n,r){return function(t){return 1-e(n,r)(1-t)}},l["easeInOut"+n]=function(n,r){return function(t){return t<.5?e(n,r)(2*t)/2:1-e(n,r)(-2*t+2)/2}}}),l);function v(n,e){if(i.fnc(n))return n;var r=n.split("(")[0],t=h[r],a=c(n);switch(r){case"spring":return s(n,e);case"cubicBezier":return u(p,a);case"steps":return u(f,a);default:return u(t,a)}}function g(n){try{return document.querySelectorAll(n)}catch(n){return}}function m(n,e){for(var r=n.length,t=arguments.length>=2?arguments[1]:void 0,a=[],o=0;o<r;o++)if(o in n){var u=n[o];e.call(t,u,o,n)&&a.push(u)}return a}function y(n){return n.reduce(function(n,e){return n.concat(i.arr(e)?y(e):e)},[])}function b(n){return i.arr(n)?n:(i.str(n)&&(n=g(n)||n),n instanceof NodeList||n instanceof HTMLCollection?[].slice.call(n):[n])}function M(n,e){return n.some(function(n){return n===e})}function x(n){var e={};for(var r in n)e[r]=n[r];return e}function w(n,e){var r=x(n);for(var t in n)r[t]=e.hasOwnProperty(t)?e[t]:n[t];return r}function k(n,e){var r=x(n);for(var t in e)r[t]=i.und(n[t])?e[t]:n[t];return r}function O(n){return i.rgb(n)?(r=/rgb\((\d+,\s*[\d]+,\s*[\d]+)\)/g.exec(e=n))?"rgba("+r[1]+",1)":e:i.hex(n)?(t=n.replace(/^#?([a-f\d])([a-f\d])([a-f\d])$/i,function(n,e,r,t){return e+e+r+r+t+t}),a=/^#?([a-f\d]{2})([a-f\d]{2})([a-f\d]{2})$/i.exec(t),"rgba("+parseInt(a[1],16)+","+parseInt(a[2],16)+","+parseInt(a[3],16)+",1)"):i.hsl(n)?function(n){var e,r,t,a=/hsl\((\d+),\s*([\d.]+)%,\s*([\d.]+)%\)/g.exec(n)||/hsla\((\d+),\s*([\d.]+)%,\s*([\d.]+)%,\s*([\d.]+)\)/g.exec(n),o=parseInt(a[1],10)/360,u=parseInt(a[2],10)/100,i=parseInt(a[3],10)/100,c=a[4]||1;function s(n,e,r){return r<0&&(r+=1),r>1&&(r-=1),r<1/6?n+6*(e-n)*r:r<.5?e:r<2/3?n+(e-n)*(2/3-r)*6:n}if(0==u)e=r=t=i;else{var f=i<.5?i*(1+u):i+u-i*u,l=2*i-f;e=s(l,f,o+1/3),r=s(l,f,o),t=s(l,f,o-1/3)}return"rgba("+255*e+","+255*r+","+255*t+","+c+")"}(n):void 0;var e,r,t,a}function C(n){var e=/[+-]?\d*\.?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?(%|px|pt|em|rem|in|cm|mm|ex|ch|pc|vw|vh|vmin|vmax|deg|rad|turn)?$/.exec(n);if(e)return e[1]}function B(n,e){return i.fnc(n)?n(e.target,e.id,e.total):n}function P(n,e){return n.getAttribute(e)}function I(n,e,r){if(M([r,"deg","rad","turn"],C(e)))return e;var a=t.CSS[e+r];if(!i.und(a))return a;var o=document.createElement(n.tagName),u=n.parentNode&&n.parentNode!==document?n.parentNode:document.body;u.appendChild(o),o.style.position="absolute",o.style.width=100+r;var c=100/o.offsetWidth;u.removeChild(o);var s=c*parseFloat(e);return t.CSS[e+r]=s,s}function T(n,e,r){if(e in n.style){var t=e.replace(/([a-z])([A-Z])/g,"$1-$2").toLowerCase(),a=n.style[e]||getComputedStyle(n).getPropertyValue(t)||"0";return r?I(n,a,r):a}}function D(n,e){return i.dom(n)&&!i.inp(n)&&(P(n,e)||i.svg(n)&&n[e])?"attribute":i.dom(n)&&M(r,e)?"transform":i.dom(n)&&"transform"!==e&&T(n,e)?"css":null!=n[e]?"object":void 0}function E(n){if(i.dom(n)){for(var e,r=n.style.transform||"",t=/(\w+)\(([^)]*)\)/g,a=new Map;e=t.exec(r);)a.set(e[1],e[2]);return a}}function F(n,e,r,t){var a,u=o(e,"scale")?1:0+(o(a=e,"translate")||"perspective"===a?"px":o(a,"rotate")||o(a,"skew")?"deg":void 0),i=E(n).get(e)||u;return r&&(r.transforms.list.set(e,i),r.transforms.last=e),t?I(n,i,t):i}function N(n,e,r,t){switch(D(n,e)){case"transform":return F(n,e,t,r);case"css":return T(n,e,r);case"attribute":return P(n,e);default:return n[e]||0}}function A(n,e){var r=/^(\*=|\+=|-=)/.exec(n);if(!r)return n;var t=C(n)||0,a=parseFloat(e),o=parseFloat(n.replace(r[0],""));switch(r[0][0]){case"+":return a+o+t;case"-":return a-o+t;case"*":return a*o+t}}function L(n,e){if(i.col(n))return O(n);if(/\s/g.test(n))return n;var r=C(n),t=r?n.substr(0,n.length-r.length):n;return e?t+e:t}function j(n,e){return Math.sqrt(Math.pow(e.x-n.x,2)+Math.pow(e.y-n.y,2))}function S(n){for(var e,r=n.points,t=0,a=0;a<r.numberOfItems;a++){var o=r.getItem(a);a>0&&(t+=j(e,o)),e=o}return t}function q(n){if(n.getTotalLength)return n.getTotalLength();switch(n.tagName.toLowerCase()){case"circle":return o=n,2*Math.PI*P(o,"r");case"rect":return 2*P(a=n,"width")+2*P(a,"height");case"line":return j({x:P(t=n,"x1"),y:P(t,"y1")},{x:P(t,"x2"),y:P(t,"y2")});case"polyline":return S(n);case"polygon":return r=(e=n).points,S(e)+j(r.getItem(r.numberOfItems-1),r.getItem(0))}var e,r,t,a,o}function $(n,e){var r=e||{},t=r.el||function(n){for(var e=n.parentNode;i.svg(e)&&i.svg(e.parentNode);)e=e.parentNode;return e}(n),a=t.getBoundingClientRect(),o=P(t,"viewBox"),u=a.width,c=a.height,s=r.viewBox||(o?o.split(" "):[0,0,u,c]);return{el:t,viewBox:s,x:s[0]/1,y:s[1]/1,w:u/s[2],h:c/s[3]}}function X(n,e){function r(r){void 0===r&&(r=0);var t=e+r>=1?e+r:0;return n.el.getPointAtLength(t)}var t=$(n.el,n.svg),a=r(),o=r(-1),u=r(1);switch(n.property){case"x":return(a.x-t.x)*t.w;case"y":return(a.y-t.y)*t.h;case"angle":return 180*Math.atan2(u.y-o.y,u.x-o.x)/Math.PI}}function Y(n,e){var r=/[+-]?\d*\.?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?/g,t=L(i.pth(n)?n.totalLength:n,e)+"";return{original:t,numbers:t.match(r)?t.match(r).map(Number):[0],strings:i.str(n)||e?t.split(r):[]}}function Z(n){return m(n?y(i.arr(n)?n.map(b):b(n)):[],function(n,e,r){return r.indexOf(n)===e})}function Q(n){var e=Z(n);return e.map(function(n,r){return{target:n,id:r,total:e.length,transforms:{list:E(n)}}})}function V(n,e){var r=x(e);if(/^spring/.test(r.easing)&&(r.duration=s(r.easing)),i.arr(n)){var t=n.length;2===t&&!i.obj(n[0])?n={value:n}:i.fnc(e.duration)||(r.duration=e.duration/t)}var a=i.arr(n)?n:[n];return a.map(function(n,r){var t=i.obj(n)&&!i.pth(n)?n:{value:n};return i.und(t.delay)&&(t.delay=r?0:e.delay),i.und(t.endDelay)&&(t.endDelay=r===a.length-1?e.endDelay:0),t}).map(function(n){return k(n,r)})}function z(n,e){var r=[],t=e.keyframes;for(var a in t&&(e=k(function(n){for(var e=m(y(n.map(function(n){return Object.keys(n)})),function(n){return i.key(n)}).reduce(function(n,e){return n.indexOf(e)<0&&n.push(e),n},[]),r={},t=function(t){var a=e[t];r[a]=n.map(function(n){var e={};for(var r in n)i.key(r)?r==a&&(e.value=n[r]):e[r]=n[r];return e})},a=0;a<e.length;a++)t(a);return r}(t),e)),e)i.key(a)&&r.push({name:a,tweens:V(e[a],n)});return r}function H(n,e){var r;return n.tweens.map(function(t){var a=function(n,e){var r={};for(var t in n){var a=B(n[t],e);i.arr(a)&&1===(a=a.map(function(n){return B(n,e)})).length&&(a=a[0]),r[t]=a}return r.duration=parseFloat(r.duration),r.delay=parseFloat(r.delay),r}(t,e),o=a.value,u=i.arr(o)?o[1]:o,c=C(u),s=N(e.target,n.name,c,e),f=r?r.to.original:s,l=i.arr(o)?o[0]:f,d=C(l)||C(s),p=c||d;return i.und(u)&&(u=f),a.from=Y(l,p),a.to=Y(A(u,l),p),a.start=r?r.end:0,a.end=a.start+a.delay+a.duration+a.endDelay,a.easing=v(a.easing,a.duration),a.isPath=i.pth(o),a.isColor=i.col(a.from.original),a.isColor&&(a.round=1),r=a,a})}var G={css:function(n,e,r){return n.style[e]=r},attribute:function(n,e,r){return n.setAttribute(e,r)},object:function(n,e,r){return n[e]=r},transform:function(n,e,r,t,a){if(t.list.set(e,r),e===t.last||a){var o="";t.list.forEach(function(n,e){o+=e+"("+n+") "}),n.style.transform=o}}};function R(n,e){Q(n).forEach(function(n){for(var r in e){var t=B(e[r],n),a=n.target,o=C(t),u=N(a,r,o,n),i=A(L(t,o||C(u)),u),c=D(a,r);G[c](a,r,i,n.transforms,!0)}})}function W(n,e){return m(y(n.map(function(n){return e.map(function(e){return function(n,e){var r=D(n.target,e.name);if(r){var t=H(e,n),a=t[t.length-1];return{type:r,property:e.name,animatable:n,tweens:t,duration:a.end,delay:t[0].delay,endDelay:a.endDelay}}}(n,e)})})),function(n){return!i.und(n)})}function J(n,e){var r=n.length,t=function(n){return n.timelineOffset?n.timelineOffset:0},a={};return a.duration=r?Math.max.apply(Math,n.map(function(n){return t(n)+n.duration})):e.duration,a.delay=r?Math.min.apply(Math,n.map(function(n){return t(n)+n.delay})):e.delay,a.endDelay=r?a.duration-Math.max.apply(Math,n.map(function(n){return t(n)+n.duration-n.endDelay})):e.endDelay,a}var K=0;var U,_=[],nn=[],en=function(){function n(){U=requestAnimationFrame(e)}function e(e){var r=_.length;if(r){for(var t=0;t<r;){var a=_[t];if(a.paused){var o=_.indexOf(a);o>-1&&(_.splice(o,1),r=_.length)}else a.tick(e);t++}n()}else U=cancelAnimationFrame(U)}return n}();function rn(r){void 0===r&&(r={});var t,o=0,u=0,i=0,c=0,s=null;function f(n){var e=window.Promise&&new Promise(function(n){return s=n});return n.finished=e,e}var l,d,p,h,v,g,y,b,M=(d=w(n,l=r),p=w(e,l),h=z(p,l),v=Q(l.targets),g=W(v,h),y=J(g,p),b=K,K++,k(d,{id:b,children:[],animatables:v,animations:g,duration:y.duration,delay:y.delay,endDelay:y.endDelay}));f(M);function x(){var n=M.direction;"alternate"!==n&&(M.direction="normal"!==n?"normal":"reverse"),M.reversed=!M.reversed,t.forEach(function(n){return n.reversed=M.reversed})}function O(n){return M.reversed?M.duration-n:n}function C(){o=0,u=O(M.currentTime)*(1/rn.speed)}function B(n,e){e&&e.seek(n-e.timelineOffset)}function P(n){for(var e=0,r=M.animations,t=r.length;e<t;){var o=r[e],u=o.animatable,i=o.tweens,c=i.length-1,s=i[c];c&&(s=m(i,function(e){return n<e.end})[0]||s);for(var f=a(n-s.start-s.delay,0,s.duration)/s.duration,l=isNaN(f)?1:s.easing(f),d=s.to.strings,p=s.round,h=[],v=s.to.numbers.length,g=void 0,y=0;y<v;y++){var b=void 0,x=s.to.numbers[y],w=s.from.numbers[y]||0;b=s.isPath?X(s.value,l*x):w+l*(x-w),p&&(s.isColor&&y>2||(b=Math.round(b*p)/p)),h.push(b)}var k=d.length;if(k){g=d[0];for(var O=0;O<k;O++){d[O];var C=d[O+1],B=h[O];isNaN(B)||(g+=C?B+C:B+" ")}}else g=h[0];G[o.type](u.target,o.property,g,u.transforms),o.currentValue=g,e++}}function I(n){M[n]&&!M.passThrough&&M[n](M)}function T(n){var e=M.duration,r=M.delay,l=e-M.endDelay,d=O(n);M.progress=a(d/e*100,0,100),M.reversePlayback=d<M.currentTime,t&&function(n){if(M.reversePlayback)for(var e=c;e--;)B(n,t[e]);else for(var r=0;r<c;r++)B(n,t[r])}(d),!M.began&&M.currentTime>0&&(M.began=!0,I("begin")),!M.loopBegan&&M.currentTime>0&&(M.loopBegan=!0,I("loopBegin")),d<=r&&0!==M.currentTime&&P(0),(d>=l&&M.currentTime!==e||!e)&&P(e),d>r&&d<l?(M.changeBegan||(M.changeBegan=!0,M.changeCompleted=!1,I("changeBegin")),I("change"),P(d)):M.changeBegan&&(M.changeCompleted=!0,M.changeBegan=!1,I("changeComplete")),M.currentTime=a(d,0,e),M.began&&I("update"),n>=e&&(u=0,M.remaining&&!0!==M.remaining&&M.remaining--,M.remaining?(o=i,I("loopComplete"),M.loopBegan=!1,"alternate"===M.direction&&x()):(M.paused=!0,M.completed||(M.completed=!0,I("loopComplete"),I("complete"),!M.passThrough&&"Promise"in window&&(s(),f(M)))))}return M.reset=function(){var n=M.direction;M.passThrough=!1,M.currentTime=0,M.progress=0,M.paused=!0,M.began=!1,M.loopBegan=!1,M.changeBegan=!1,M.completed=!1,M.changeCompleted=!1,M.reversePlayback=!1,M.reversed="reverse"===n,M.remaining=M.loop,t=M.children;for(var e=c=t.length;e--;)M.children[e].reset();(M.reversed&&!0!==M.loop||"alternate"===n&&1===M.loop)&&M.remaining++,P(M.reversed?M.duration:0)},M.set=function(n,e){return R(n,e),M},M.tick=function(n){i=n,o||(o=i),T((i+(u-o))*rn.speed)},M.seek=function(n){T(O(n))},M.pause=function(){M.paused=!0,C()},M.play=function(){M.paused&&(M.completed&&M.reset(),M.paused=!1,_.push(M),C(),U||en())},M.reverse=function(){x(),C()},M.restart=function(){M.reset(),M.play()},M.reset(),M.autoplay&&M.play(),M}function tn(n,e){for(var r=e.length;r--;)M(n,e[r].animatable.target)&&e.splice(r,1)}return"undefined"!=typeof document&&document.addEventListener("visibilitychange",function(){document.hidden?(_.forEach(function(n){return n.pause()}),nn=_.slice(0),rn.running=_=[]):nn.forEach(function(n){return n.play()})}),rn.version="3.1.0",rn.speed=1,rn.running=_,rn.remove=function(n){for(var e=Z(n),r=_.length;r--;){var t=_[r],a=t.animations,o=t.children;tn(e,a);for(var u=o.length;u--;){var i=o[u],c=i.animations;tn(e,c),c.length||i.children.length||o.splice(u,1)}a.length||o.length||t.pause()}},rn.get=N,rn.set=R,rn.convertPx=I,rn.path=function(n,e){var r=i.str(n)?g(n)[0]:n,t=e||100;return function(n){return{property:n,el:r,svg:$(r),totalLength:q(r)*(t/100)}}},rn.setDashoffset=function(n){var e=q(n);return n.setAttribute("stroke-dasharray",e),e},rn.stagger=function(n,e){void 0===e&&(e={});var r=e.direction||"normal",t=e.easing?v(e.easing):null,a=e.grid,o=e.axis,u=e.from||0,c="first"===u,s="center"===u,f="last"===u,l=i.arr(n),d=l?parseFloat(n[0]):parseFloat(n),p=l?parseFloat(n[1]):0,h=C(l?n[1]:n)||0,g=e.start||0+(l?d:0),m=[],y=0;return function(n,e,i){if(c&&(u=0),s&&(u=(i-1)/2),f&&(u=i-1),!m.length){for(var v=0;v<i;v++){if(a){var b=s?(a[0]-1)/2:u%a[0],M=s?(a[1]-1)/2:Math.floor(u/a[0]),x=b-v%a[0],w=M-Math.floor(v/a[0]),k=Math.sqrt(x*x+w*w);"x"===o&&(k=-x),"y"===o&&(k=-w),m.push(k)}else m.push(Math.abs(u-v));y=Math.max.apply(Math,m)}t&&(m=m.map(function(n){return t(n/y)*y})),"reverse"===r&&(m=m.map(function(n){return o?n<0?-1*n:-n:Math.abs(y-n)}))}return g+(l?(p-d)/y:d)*(Math.round(100*m[e])/100)+h}},rn.timeline=function(n){void 0===n&&(n={});var r=rn(n);return r.duration=0,r.add=function(t,a){var o=_.indexOf(r),u=r.children;function c(n){n.passThrough=!0}o>-1&&_.splice(o,1);for(var s=0;s<u.length;s++)c(u[s]);var f=k(t,w(e,n));f.targets=f.targets||n.targets;var l=r.duration;f.autoplay=!1,f.direction=r.direction,f.timelineOffset=i.und(a)?l:A(a,l),c(r),r.seek(f.timelineOffset);var d=rn(f);c(d),u.push(d);var p=J(u,n);return r.delay=p.delay,r.endDelay=p.endDelay,r.duration=p.duration,r.seek(0),r.reset(),r.autoplay&&r.play(),r},r},rn.easing=v,rn.penner=h,rn.random=function(n,e){return Math.floor(Math.random()*(e-n+1))+n},rn});
\ No newline at end of file
diff --git a/distributed/dashboard/static/js/individual-cluster-map.js b/distributed/dashboard/static/js/individual-cluster-map.js
new file mode 100644
index 00000000000..20aed45b7b1
--- /dev/null
+++ b/distributed/dashboard/static/js/individual-cluster-map.js
@@ -0,0 +1,367 @@
+// TODO Animate workers when performing tasks or swapping to show activity
+// TODO Add memory usage dial around outside of workers
+// TODO Add clients
+// TODO Show future retrieval
+// TODO Show graph submission
+
+const workerColor = "#ECB172";
+const workerIdleSize = 12;
+const workerBusySize = 15;
+
+// Expose in the global scope
+var dashboard;
+
+class Dashboard {
+  constructor() {
+    this.workers = [];
+    this.scheduler = "scheduler";
+    this.schedulerNode = null;
+    this.dashboard = document.getElementById("vis");
+    this.add_scheduler();
+  }
+
+  handle_event(event) {
+    switch (event["name"]) {
+      case "pong":
+        console.log(event);
+        break;
+      case "add_worker":
+        this.add_worker(this.hash_worker(event["worker"]));
+        break;
+      case "remove_worker":
+        this.remove_worker(this.hash_worker(event["worker"]));
+        break;
+      case "restart":
+        this.reset();
+        break;
+      case "transition":
+        if (event["action"] === "compute") {
+          this.run_task(
+            this.hash_worker(event["worker"]),
+            event["key"],
+            event["stop"] - event["start"],
+            event["color"]
+          );
+          break;
+        } else if (event["action"] === "transfer") {
+          this.run_transfer(
+            this.hash_worker(event["source"]),
+            this.hash_worker(event["worker"]),
+            event["stop"] - event["start"]
+          );
+          break;
+        } else if (
+          event["action"] === "disk-read" ||
+          event["action"] === "disk-write"
+        ) {
+          this.run_swap(
+            this.hash_worker(event["worker"]),
+            event["stop"] - event["start"]
+          );
+          break;
+        } else if (event["action"] === "deserialize") {
+          this.run_deserialize(
+            this.hash_worker(event["worker"]),
+            event["stop"] - event["start"]
+          );
+          break;
+        }
+      default:
+        console.log("Unknown event " + event["name"]);
+        console.log(event);
+    }
+  }
+
+  // Convert a worker address into a valid DOM ID
+  hash_worker(worker) {
+    let worker_id = "worker-" + worker.replace(/(:\/\/|\.|\:)/g, "-");
+    return worker_id;
+  }
+
+  add_scheduler() {
+    this.schedulerNode = document.createElementNS(
+      "http://www.w3.org/2000/svg",
+      "circle"
+    );
+    this.schedulerNode.setAttributeNS(null, "id", this.scheduler);
+    this.schedulerNode.setAttributeNS(null, "class", "node scheduler");
+    this.dashboard.appendChild(this.schedulerNode);
+    anime({
+      targets: "#" + this.scheduler,
+      r: [0, 30],
+      cx: ["50%", "50%"],
+      cy: ["50%", "50%"],
+      duration: 250
+    });
+  }
+
+  add_worker(id) {
+    // Create new circle element and add it to the SVG
+    let workerNode = document.createElementNS(
+      "http://www.w3.org/2000/svg",
+      "circle"
+    );
+    workerNode.setAttributeNS(null, "id", id);
+    workerNode.setAttributeNS(null, "r", "0");
+    workerNode.setAttributeNS(null, "cx", "50%");
+    workerNode.setAttributeNS(null, "cy", "50%");
+    workerNode.setAttributeNS(null, "class", "node worker");
+    this.dashboard.appendChild(workerNode);
+
+    // Add our new worker to the list of workers and then reposition them all
+    let position = Math.floor(Math.random() * this.workers.length) - 1;
+    this.workers.splice(position, 0, id);
+    this.update_worker_positions();
+  }
+
+  remove_worker(id) {
+    // Remove circle element from SVG
+    let worker = document.getElementById(id);
+    anime({
+      targets: worker,
+      opacity: 0,
+      duration: 1000,
+      complete: () => this.dashboard.removeChild(worker)
+    });
+
+    // Remove worker from list of workers
+    let index = this.workers.indexOf(id);
+    if (index > -1) {
+      this.workers.splice(index, 1);
+    }
+
+    // Reposition other workers to fill in the gap
+    this.update_worker_positions();
+  }
+
+  connected() {
+    anime({
+      targets: "#scheduler",
+      opacity: 1,
+      duration: 1000
+    });
+  }
+
+  disconnected() {
+    while (this.workers.length > 0) {
+      this.remove_worker(this.workers[0]);
+    }
+    anime({
+      targets: "#scheduler",
+      opacity: 0.3,
+      duration: 1000
+    });
+  }
+
+  update_worker_positions() {
+    // Calculate a circle around the scheduler and position our workers equally around it
+    for (var i = 0; i < this.workers.length; i++) {
+      let θ = (2 * Math.PI * i) / this.workers.length;
+      let r = 40;
+      let h = 50;
+      let k = 50;
+      let x = h + r * Math.cos(θ);
+      let y = k + r * Math.sin(θ);
+      anime({
+        targets: "#" + this.workers[i],
+        r: workerIdleSize,
+        cx: x + "%",
+        cy: y + "%",
+        easing: "easeInOutQuint",
+        duration: 500
+      });
+    }
+  }
+
+  run_task(worker_id, task_name, duration, color) {
+    let worker = document.getElementById(worker_id);
+    let scheduler = document.getElementById("scheduler");
+    let arc = this.draw_arc(scheduler, worker, color, "projectile");
+
+    anime
+      .timeline({
+        targets: "#" + worker_id
+      })
+      .add({
+        begin: () => this.dashboard.insertBefore(arc, this.schedulerNode)
+      })
+      .add(
+        {
+          fill: color,
+          r: workerBusySize,
+          begin: () => this.dashboard.removeChild(arc)
+        },
+        500
+      )
+      .add({ fill: workerColor, r: workerIdleSize }, "+=" + duration);
+  }
+
+  run_transfer(start_worker, end_worker, duration) {
+    start_worker = document.getElementById(start_worker);
+    end_worker = document.getElementById(end_worker);
+    duration = Math.max(250, duration / 1000);
+    let color = "rgba(255, 0, 0, .6)";
+    let arc = this.draw_arc(start_worker, end_worker, color, "transfer");
+
+    anime
+      .timeline({
+        targets: ["#" + start_worker, "#" + end_worker],
+        duration: 250
+      })
+      .add({
+        fill: color,
+        r: workerBusySize,
+        begin: () => this.dashboard.insertBefore(arc, this.schedulerNode)
+      })
+      .add(
+        {
+          fill: workerColor,
+          r: workerIdleSize,
+          begin: () => this.dashboard.removeChild(arc)
+        },
+        "+=" + duration
+      );
+  }
+
+  run_swap(worker, duration) {
+    anime
+      .timeline({
+        targets: "#" + worker,
+        duration: 250
+      })
+      .add({
+        fill: "#D67548",
+        r: workerBusySize
+      })
+      .add(
+        {
+          fill: workerColor,
+          r: workerIdleSize
+        },
+        "+=" + duration
+      );
+  }
+
+  run_deserialize(worker, duration) {
+    anime
+      .timeline({
+        targets: "#" + worker,
+        duration: 250
+      })
+      .add({
+        fill: "gray",
+        r: workerBusySize
+      })
+      .add(
+        {
+          fill: workerColor,
+          r: workerIdleSize
+        },
+        "+=" + duration
+      );
+  }
+
+  kill_worker(worker) {
+    anime({
+      targets: "#" + worker,
+      fill: "rgba(0, 0, 0, 1)",
+      duration: 250
+    });
+  }
+
+  reset() {
+    for (var i = 0; i < this.workers.length; i++) {
+      anime({
+        targets: "#" + this.workers[i],
+        fill: workerColor,
+        r: workerIdleSize,
+        duration: 250
+      });
+    }
+  }
+
+  calculate_arc(start_x, start_y, end_x, end_y) {
+    // mid-point of line:
+    let mpx = (start_x + end_x) * 0.5;
+    let mpy = (start_y + end_y) * 0.5;
+
+    // angle of perpendicular to line:
+    let theta = Math.atan2(start_y - end_y, start_x - end_x) - Math.PI / 2;
+
+    // distance of control point from mid-point of line:
+    let offset = Math.random() * 50;
+    if (Math.random() >= 0.5) {
+      offset = -offset;
+    }
+
+    // location of control point:
+    let c1x = mpx + offset * Math.cos(theta);
+    let c1y = mpy + offset * Math.sin(theta);
+
+    // construct the command to draw a quadratic curve
+    return (
+      "M" +
+      end_x +
+      " " +
+      end_y +
+      " Q " +
+      c1x +
+      " " +
+      c1y +
+      " " +
+      start_x +
+      " " +
+      start_y
+    );
+  }
+
+  draw_arc(start_element, end_element, color, class_name) {
+    let curve = this.calculate_arc(
+      this.getAbsoluteXY(start_element)[0],
+      this.getAbsoluteXY(start_element)[1],
+      this.getAbsoluteXY(end_element)[0],
+      this.getAbsoluteXY(end_element)[1]
+    );
+
+    let arc = document.createElementNS("http://www.w3.org/2000/svg", "path");
+    arc.setAttributeNS(null, "id", class_name);
+    arc.setAttributeNS(null, "class", class_name);
+    arc.setAttributeNS(null, "stroke", color);
+    arc.setAttribute("d", curve);
+    return arc;
+  }
+
+  getAbsoluteXY(element) {
+    var box = element.getBoundingClientRect();
+    var x = box.left + box.width / 4;
+    var y = box.top + box.height / 4;
+    return [x, y];
+  }
+}
+
+function get_websocket_url(endpoint) {
+  var l = window.location;
+  return (
+    (l.protocol === "https:" ? "wss://" : "ws://") +
+    l.hostname +
+    (l.port != 80 && l.port != 443 ? ":" + l.port : "") +
+    endpoint
+  );
+}
+
+function main() {
+  dashboard = new Dashboard();
+
+  var ws = new ReconnectingWebSocket(get_websocket_url("/eventstream"));
+  ws.onopen = function() {
+    dashboard.connected();
+  };
+  ws.onmessage = function(event) {
+    dashboard.handle_event(JSON.parse(event.data));
+  };
+  ws.onclose = function() {
+    dashboard.disconnected();
+  };
+}
+
+window.addEventListener("load", main);
diff --git a/distributed/dashboard/static/js/reconnecting-websocket.min.js b/distributed/dashboard/static/js/reconnecting-websocket.min.js
new file mode 100644
index 00000000000..b2c6e624ce0
--- /dev/null
+++ b/distributed/dashboard/static/js/reconnecting-websocket.min.js
@@ -0,0 +1,8 @@
+/*
+ * reconnecting-websocket.min.js v1.0.0
+ * Copyright (c) 2010-2012, Joe Walnes
+ * Released under the MIT license
+ * https://github.com/joewalnes/reconnecting-websocket
+ */
+
+!function(a,b){"function"==typeof define&&define.amd?define([],b):"undefined"!=typeof module&&module.exports?module.exports=b():a.ReconnectingWebSocket=b()}(this,function(){function a(b,c,d){function l(a,b){var c=document.createEvent("CustomEvent");return c.initCustomEvent(a,!1,!1,b),c}var e={debug:!1,automaticOpen:!0,reconnectInterval:1e3,maxReconnectInterval:3e4,reconnectDecay:1.5,timeoutInterval:2e3};d||(d={});for(var f in e)this[f]="undefined"!=typeof d[f]?d[f]:e[f];this.url=b,this.reconnectAttempts=0,this.readyState=WebSocket.CONNECTING,this.protocol=null;var h,g=this,i=!1,j=!1,k=document.createElement("div");k.addEventListener("open",function(a){g.onopen(a)}),k.addEventListener("close",function(a){g.onclose(a)}),k.addEventListener("connecting",function(a){g.onconnecting(a)}),k.addEventListener("message",function(a){g.onmessage(a)}),k.addEventListener("error",function(a){g.onerror(a)}),this.addEventListener=k.addEventListener.bind(k),this.removeEventListener=k.removeEventListener.bind(k),this.dispatchEvent=k.dispatchEvent.bind(k),this.open=function(b){h=new WebSocket(g.url,c||[]),b||k.dispatchEvent(l("connecting")),(g.debug||a.debugAll)&&console.debug("ReconnectingWebSocket","attempt-connect",g.url);var d=h,e=setTimeout(function(){(g.debug||a.debugAll)&&console.debug("ReconnectingWebSocket","connection-timeout",g.url),j=!0,d.close(),j=!1},g.timeoutInterval);h.onopen=function(){clearTimeout(e),(g.debug||a.debugAll)&&console.debug("ReconnectingWebSocket","onopen",g.url),g.protocol=h.protocol,g.readyState=WebSocket.OPEN,g.reconnectAttempts=0;var d=l("open");d.isReconnect=b,b=!1,k.dispatchEvent(d)},h.onclose=function(c){if(clearTimeout(e),h=null,i)g.readyState=WebSocket.CLOSED,k.dispatchEvent(l("close"));else{g.readyState=WebSocket.CONNECTING;var d=l("connecting");d.code=c.code,d.reason=c.reason,d.wasClean=c.wasClean,k.dispatchEvent(d),b||j||((g.debug||a.debugAll)&&console.debug("ReconnectingWebSocket","onclose",g.url),k.dispatchEvent(l("close")));var e=g.reconnectInterval*Math.pow(g.reconnectDecay,g.reconnectAttempts);setTimeout(function(){g.reconnectAttempts++,g.open(!0)},e>g.maxReconnectInterval?g.maxReconnectInterval:e)}},h.onmessage=function(b){(g.debug||a.debugAll)&&console.debug("ReconnectingWebSocket","onmessage",g.url,b.data);var c=l("message");c.data=b.data,k.dispatchEvent(c)},h.onerror=function(b){(g.debug||a.debugAll)&&console.debug("ReconnectingWebSocket","onerror",g.url,b),k.dispatchEvent(l("error"))}},1==this.automaticOpen&&this.open(!1),this.send=function(b){if(h)return(g.debug||a.debugAll)&&console.debug("ReconnectingWebSocket","send",g.url,b),h.send(b);throw"INVALID_STATE_ERR : Pausing to reconnect websocket"},this.close=function(a,b){"undefined"==typeof a&&(a=1e3),i=!0,h&&h.close(a,b)},this.refresh=function(){h&&h.close()}}return a.prototype.onopen=function(){},a.prototype.onclose=function(){},a.prototype.onconnecting=function(){},a.prototype.onmessage=function(){},a.prototype.onerror=function(){},a.debugAll=!1,a.CONNECTING=WebSocket.CONNECTING,a.OPEN=WebSocket.OPEN,a.CLOSING=WebSocket.CLOSING,a.CLOSED=WebSocket.CLOSED,a});
diff --git a/distributed/dashboard/tests/test_scheduler_bokeh.py b/distributed/dashboard/tests/test_scheduler_bokeh.py
index 39d2ce84156..6594ce2142f 100644
--- a/distributed/dashboard/tests/test_scheduler_bokeh.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh.py
@@ -679,8 +679,8 @@ def test_https_support(c, s, a, b):
             url="https://localhost:%d/%s" % (port, suffix), ssl_options=ctx
         )
         response = yield http_client.fetch(req)
+        assert response.code < 300
         body = response.body.decode()
-        assert "bokeh" in body.lower()
         assert not re.search("href=./", body)  # no absolute links
 
 

From 7d2ed43c794cf2a00a97f8d0f83b57f028f6be42 Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Tue, 14 Jan 2020 15:58:30 +0000
Subject: [PATCH 0627/1550] Add client join and leave hooks (#3371)

---
 distributed/diagnostics/plugin.py    |  8 +++++++-
 distributed/diagnostics/websocket.py | 12 ++++++++++++
 distributed/scheduler.py             | 12 ++++++++++++
 3 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/distributed/diagnostics/plugin.py b/distributed/diagnostics/plugin.py
index 8d56679e9a9..4d94f7c8859 100644
--- a/distributed/diagnostics/plugin.py
+++ b/distributed/diagnostics/plugin.py
@@ -62,7 +62,13 @@ def add_worker(self, scheduler=None, worker=None, **kwargs):
         """ Run when a new worker enters the cluster """
 
     def remove_worker(self, scheduler=None, worker=None, **kwargs):
-        """ Run when a worker leaves the cluster"""
+        """ Run when a worker leaves the cluster """
+
+    def add_client(self, scheduler=None, client=None, **kwargs):
+        """ Run when a new client connects """
+
+    def remove_client(self, scheduler=None, client=None, **kwargs):
+        """ Run when a client disconnects """
 
 
 class WorkerPlugin(object):
diff --git a/distributed/diagnostics/websocket.py b/distributed/diagnostics/websocket.py
index 6682dd6a739..641730faf54 100644
--- a/distributed/diagnostics/websocket.py
+++ b/distributed/diagnostics/websocket.py
@@ -20,6 +20,18 @@ def remove_worker(self, scheduler=None, worker=None, **kwargs):
         """ Run when a worker leaves the cluster"""
         self.socket.send("remove_worker", {"worker": worker})
 
+    def add_client(self, scheduler=None, client=None, **kwargs):
+        """ Run when a new client connects """
+        self.socket.send("add_client", {"client": client})
+
+    def remove_client(self, scheduler=None, client=None, **kwargs):
+        """ Run when a client disconnects """
+        self.socket.send("remove_client", {"client": client})
+
+    def update_graph(self, scheduler, client=None, **kwargs):
+        """ Run when a new graph / tasks enter the scheduler """
+        self.socket.send("update_graph", {"client": client})
+
     def transition(self, key, start, finish, *args, **kwargs):
         """ Run whenever a task changes state
 
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index d16026b1c24..478eb413422 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -2467,6 +2467,12 @@ async def add_client(self, comm, client=None, versions=None):
         self.log_event(["all", client], {"action": "add-client", "client": client})
         self.clients[client] = ClientState(client, versions=versions)
 
+        for plugin in self.plugins[:]:
+            try:
+                plugin.add_client(scheduler=self, client=client)
+            except Exception as e:
+                logger.exception(e)
+
         try:
             bcomm = BatchedSend(interval="2ms", loop=self.loop)
             bcomm.start(comm)
@@ -2514,6 +2520,12 @@ def remove_client(self, client=None):
             )
             del self.clients[client]
 
+            for plugin in self.plugins[:]:
+                try:
+                    plugin.remove_client(scheduler=self, client=client)
+                except Exception as e:
+                    logger.exception(e)
+
         def remove_client_from_events():
             # If the client isn't registered anymore after the delay, remove from events
             if client not in self.clients and client in self.events:

From 6a76ca7ac6db9f6243869057b6805f14c23e3811 Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Thu, 16 Jan 2020 08:39:36 +0000
Subject: [PATCH 0628/1550] Move Windows CI to GitHub Actions (#3373)

* Move Windows CI to GitHub Actions

* Switch to CMD and conda

* Remove indirection and add conda environment file

* Fix bad indentation

* Remove Python version

* Name steps, remove unnecessary conda activations, remove JUnit

* Align pytest options with Travis config

* Run single test for faster iterating

* Hmm pytest is fine. Trying two runners to improve performance

* Remove bad -n flag

* Add multicore support as GitHub Actions has dual-core runners

* Removing multicore again as that had some unexpected results

* Try bash instead of powershell
---
 .github/workflows/ci-windows.yaml             | 33 +++++++
 appveyor.yml                                  | 37 --------
 continuous_integration/build.cmd              |  6 --
 continuous_integration/environment.yml        | 36 ++++++++
 continuous_integration/run_tests.cmd          |  9 --
 continuous_integration/run_with_env.cmd       | 90 -------------------
 .../setup_conda_environment.cmd               | 61 -------------
 7 files changed, 69 insertions(+), 203 deletions(-)
 create mode 100644 .github/workflows/ci-windows.yaml
 delete mode 100644 appveyor.yml
 delete mode 100644 continuous_integration/build.cmd
 create mode 100644 continuous_integration/environment.yml
 delete mode 100644 continuous_integration/run_tests.cmd
 delete mode 100644 continuous_integration/run_with_env.cmd
 delete mode 100644 continuous_integration/setup_conda_environment.cmd

diff --git a/.github/workflows/ci-windows.yaml b/.github/workflows/ci-windows.yaml
new file mode 100644
index 00000000000..ecbf29f5a3d
--- /dev/null
+++ b/.github/workflows/ci-windows.yaml
@@ -0,0 +1,33 @@
+name: Windows CI
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    runs-on: windows-latest
+    strategy:
+      matrix:
+        python-version: ["3.6", "3.7"]
+
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v1
+
+      - name: Setup Conda Environment
+        uses: goanpeca/setup-miniconda@v1
+        with:
+          miniconda-version: "latest"
+          python-version: ${{ matrix.python-version }}
+          environment-file: continuous_integration/environment.yml
+          activate-environment: testenv
+          auto-activate-base: false
+
+      - name: Install distributed from source
+        shell: bash -l {0}
+        run: pip install -q --no-deps -e .
+
+      - name: Run tests
+        shell: bash -l {0}
+        env:
+          PYTHONFAULTHANDLER: 1
+        run: py.test -m "not avoid_travis" distributed --verbose -r s --timeout-method=thread --timeout=300 --durations=20
diff --git a/appveyor.yml b/appveyor.yml
deleted file mode 100644
index e32c48f105a..00000000000
--- a/appveyor.yml
+++ /dev/null
@@ -1,37 +0,0 @@
-# Environment loosely based on https://github.com/conda/conda/blob/master/appveyor.yml
-
-environment:
-
-  global:
-    # SDK v7.0 MSVC Express 2008's SetEnv.cmd script will fail if the
-    # /E:ON and /V:ON options are not enabled in the batch script intepreter
-    # See: http://stackoverflow.com/a/13751649/163740
-    CMD_IN_ENV: "cmd /E:ON /V:ON /C .\\continuous_integration\\run_with_env.cmd"
-    JUNIT_OUT: junit-results.xml
-
-  matrix:
-    # Since appveyor is quite slow, we only use a single configuration
-    - PYTHON: "3.6"
-      ARCH: "64"
-      CONDA_ENV: testenv
-
-init:
-  # Use AppVeyor's provided Miniconda: https://www.appveyor.com/docs/installed-software#python
-  - if "%ARCH%" == "64" set MINICONDA=C:\Miniconda36-x64
-  - if "%ARCH%" == "32" set MINICONDA=C:\Miniconda36
-  - set PATH=%MINICONDA%;%MINICONDA%/Scripts;%MINICONDA%/Library/bin;%PATH%
-
-install:
-  - continuous_integration\\setup_conda_environment.cmd
-
-build_script:
-  - continuous_integration\\build.cmd
-
-test_script:
-  # %CMD_IN_ENV% is needed for distutils/setuptools-based tests
-  # on certain build configurations.
-  - "%CMD_IN_ENV% continuous_integration\\run_tests.cmd"
-
-on_finish:
-  - ps: $wc = New-Object 'System.Net.WebClient'
-  - ps: $wc.UploadFile("https://ci.appveyor.com/api/testresults/junit/$($env:APPVEYOR_JOB_ID)", (Resolve-Path "$($env:JUNIT_OUT)"))
diff --git a/continuous_integration/build.cmd b/continuous_integration/build.cmd
deleted file mode 100644
index c29c3eafe82..00000000000
--- a/continuous_integration/build.cmd
+++ /dev/null
@@ -1,6 +0,0 @@
-call activate %CONDA_ENV%
-
-@echo on
-
-@rem Install Distributed
-%PIP_INSTALL% --no-deps -e .
diff --git a/continuous_integration/environment.yml b/continuous_integration/environment.yml
new file mode 100644
index 00000000000..f6651254af2
--- /dev/null
+++ b/continuous_integration/environment.yml
@@ -0,0 +1,36 @@
+name: testenv
+channels:
+  - defaults
+  - conda-forge
+dependencies:
+  - zstandard
+  - bokeh
+  - click
+  - cloudpickle
+  - dask
+  - dill
+  - lz4
+  - ipykernel
+  - ipywidgets
+  - joblib
+  - jupyter_client
+  - msgpack-python
+  - prometheus_client
+  - psutil
+  - pytest
+  - requests
+  - toolz
+  - tblib
+  - tornado=5
+  - zict
+  - fsspec
+  - pip
+  - pip:
+      - pytest-repeat
+      - pytest-timeout
+      - pytest-faulthandler
+      - sortedcollections
+      - pytest-asyncio
+      - git+https://github.com/dask/dask
+      - git+https://github.com/joblib/joblib.git
+      - git+https://github.com/dask/zict
diff --git a/continuous_integration/run_tests.cmd b/continuous_integration/run_tests.cmd
deleted file mode 100644
index f5ba5680dc2..00000000000
--- a/continuous_integration/run_tests.cmd
+++ /dev/null
@@ -1,9 +0,0 @@
-call activate %CONDA_ENV%
-
-@echo on
-
-set PYTHONFAULTHANDLER=1
-
-set PYTEST=py.test --tb=native --timeout=120 -r s
-
-%PYTEST% -v -m "not avoid_travis" --junit-xml="%JUNIT_OUT%" distributed
diff --git a/continuous_integration/run_with_env.cmd b/continuous_integration/run_with_env.cmd
deleted file mode 100644
index 3a56e3e840e..00000000000
--- a/continuous_integration/run_with_env.cmd
+++ /dev/null
@@ -1,90 +0,0 @@
-:: From https://github.com/ogrisel/python-appveyor-demo
-::
-:: To build extensions for 64 bit Python 3, we need to configure environment
-:: variables to use the MSVC 2010 C++ compilers from GRMSDKX_EN_DVD.iso of:
-:: MS Windows SDK for Windows 7 and .NET Framework 4 (SDK v7.1)
-::
-:: To build extensions for 64 bit Python 2, we need to configure environment
-:: variables to use the MSVC 2008 C++ compilers from GRMSDKX_EN_DVD.iso of:
-:: MS Windows SDK for Windows 7 and .NET Framework 3.5 (SDK v7.0)
-::
-:: 32 bit builds, and 64-bit builds for 3.5 and beyond, do not require specific
-:: environment configurations.
-::
-:: Note: this script needs to be run with the /E:ON and /V:ON flags for the
-:: cmd interpreter, at least for (SDK v7.0)
-::
-:: More details at:
-:: https://github.com/cython/cython/wiki/64BitCythonExtensionsOnWindows
-:: http://stackoverflow.com/a/13751649/163740
-::
-:: Author: Olivier Grisel
-:: License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/
-::
-:: Notes about batch files for Python people:
-::
-:: Quotes in values are literally part of the values:
-::      SET FOO="bar"
-:: FOO is now five characters long: " b a r "
-:: If you don't want quotes, don't include them on the right-hand side.
-::
-:: The CALL lines at the end of this file look redundant, but if you move them
-:: outside of the IF clauses, they do not run properly in the SET_SDK_64==Y
-:: case, I don't know why.
-@ECHO OFF
-
-SET COMMAND_TO_RUN=%*
-SET WIN_SDK_ROOT=C:\Program Files\Microsoft SDKs\Windows
-SET WIN_WDK=c:\Program Files (x86)\Windows Kits\10\Include\wdf
-
-:: Extract the major and minor versions, and allow for the minor version to be
-:: more than 9.  This requires the version number to have two dots in it.
-SET MAJOR_PYTHON_VERSION=%PYTHON:~0,1%
-IF "%PYTHON:~3,1%" == "." (
-    SET MINOR_PYTHON_VERSION=%PYTHON:~2,1%
-) ELSE (
-    SET MINOR_PYTHON_VERSION=%PYTHON:~2,2%
-)
-
-:: Based on the Python version, determine what SDK version to use, and whether
-:: to set the SDK for 64-bit.
-IF %MAJOR_PYTHON_VERSION% == 2 (
-    SET WINDOWS_SDK_VERSION="v7.0"
-    SET SET_SDK_64=Y
-) ELSE (
-    IF %MAJOR_PYTHON_VERSION% == 3 (
-        SET WINDOWS_SDK_VERSION="v7.1"
-        IF %MINOR_PYTHON_VERSION% LEQ 4 (
-            SET SET_SDK_64=Y
-        ) ELSE (
-            SET SET_SDK_64=N
-            IF EXIST "%WIN_WDK%" (
-                :: See: https://connect.microsoft.com/VisualStudio/feedback/details/1610302/
-                REN "%WIN_WDK%" 0wdf
-            )
-        )
-    ) ELSE (
-        ECHO Unsupported Python version: "%MAJOR_PYTHON_VERSION%"
-        EXIT 1
-    )
-)
-
-IF %ARCH% == 64 (
-    IF %SET_SDK_64% == Y (
-        ECHO Configuring Windows SDK %WINDOWS_SDK_VERSION% for Python %MAJOR_PYTHON_VERSION% on a 64 bit architecture
-        SET DISTUTILS_USE_SDK=1
-        SET MSSdk=1
-        "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Setup\WindowsSdkVer.exe" -q -version:%WINDOWS_SDK_VERSION%
-        "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Bin\SetEnv.cmd" /x64 /release
-        ECHO Executing: %COMMAND_TO_RUN%
-        call %COMMAND_TO_RUN% || EXIT 1
-    ) ELSE (
-        ECHO Using default MSVC build environment for 64 bit architecture
-        ECHO Executing: %COMMAND_TO_RUN%
-        call %COMMAND_TO_RUN% || EXIT 1
-    )
-) ELSE (
-    ECHO Using default MSVC build environment for 32 bit architecture
-    ECHO Executing: %COMMAND_TO_RUN%
-    call %COMMAND_TO_RUN% || EXIT 1
-)
diff --git a/continuous_integration/setup_conda_environment.cmd b/continuous_integration/setup_conda_environment.cmd
deleted file mode 100644
index 5efc7358dbe..00000000000
--- a/continuous_integration/setup_conda_environment.cmd
+++ /dev/null
@@ -1,61 +0,0 @@
-@rem The cmd /C hack circumvents a regression where conda installs a conda.bat
-@rem script in non-root environments.
-set CONDA=cmd /C conda
-set CONDA_INSTALL=%CONDA% install -q -y
-set PIP_INSTALL=pip install -q
-
-@echo on
-
-@rem Deactivate any environment
-call deactivate
-@rem Update conda
-%CONDA% update -q -y conda
-@rem Display root environment (for debugging)
-%CONDA% list
-@rem Clean up any left-over from a previous build
-%CONDA% remove --all -q -y -n %CONDA_ENV%
-
-@rem Create test environment
-@rem (note: no cytoolz as it seems to prevent faulthandler tracebacks on crash)
-%CONDA% create -n %CONDA_ENV% -q -y ^
-    zstandard ^
-    bokeh ^
-    click ^
-    cloudpickle ^
-    dask ^
-    dill ^
-    lz4 ^
-    ipykernel ^
-    ipywidgets ^
-    joblib ^
-    jupyter_client ^
-    msgpack-python ^
-    prometheus_client ^
-    psutil ^
-    pytest ^
-    python=%PYTHON% ^
-    requests ^
-    toolz ^
-    tblib ^
-    tornado=5 ^
-    zict ^
-    fsspec ^
-    -c conda-forge
-
-call activate %CONDA_ENV%
-
-%CONDA% uninstall -q -y --force dask joblib zict
-%PIP_INSTALL% pip --upgrade
-%PIP_INSTALL% git+https://github.com/dask/dask --upgrade
-%PIP_INSTALL% git+https://github.com/joblib/joblib.git --upgrade
-%PIP_INSTALL% git+https://github.com/dask/zict --upgrade
-
-%PIP_INSTALL% "pytest>=4" pytest-repeat pytest-timeout pytest-faulthandler sortedcollections pytest-asyncio
-
-@rem Display final environment (for reproducing)
-%CONDA% list
-%CONDA% list --explicit
-where python
-where pip
-pip list
-python -m site

From 2999ae08d3f2ce975f05ca66bf7b84604b9d281c Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Thu, 16 Jan 2020 18:08:31 -0600
Subject: [PATCH 0629/1550] bump version to 2.9.2

---
 docs/source/changelog.rst | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index 3e4a05fcc79..6410c2aa0b1 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,25 @@
 Changelog
 =========
 
+2.9.2 - 2020-01-16
+------------------
+
+- Move Windows CI to GitHub Actions (:pr:`3373`) `Jacob Tomlinson`_
+- Add client join and leave hooks (:pr:`3371`) `Jacob Tomlinson`_
+- Add cluster map dashboard (:pr:`3361`) `Jacob Tomlinson`_
+- Close connection comm on retry (:pr:`3365`) `James Bourbeau`_
+- Fix scheduler state in case of worker name collision (:pr:`3366`) `byjott`_
+- Add ``--worker-class`` option to ``dask-worker`` CLI (:pr:`3364`) `James Bourbeau`_
+- Remove ``locale`` check that fails on OS X (:pr:`3360`) `Jacob Tomlinson`_
+- Rework version checking (:pr:`2627`) `Matthew Rocklin`_
+- Add websocket scheduler plugin (:pr:`3335`) `Jacob Tomlinson`_
+- Return task in ``dask-worker`` ``on_signal`` function (:pr:`3354`) `James Bourbeau`_
+- Fix failures on mixed integer/string worker names (:pr:`3352`) `Benedikt Reinartz`_
+- Avoid calling ``nbytes`` multiple times when sending data (:pr:`3349`) `Markus Mohrhard`_
+- Avoid setting event loop policy if within IPython kernel and no running event loop (:pr:`3336`) `Mana Borwornpadungkitti`_
+- Relax intermittent failing ``test_profile_server`` (:pr:`3346`) `Matthew Rocklin`_
+
+
 2.9.1 - 2019-12-27
 ------------------
 
@@ -1471,3 +1490,6 @@ significantly without many new features.
 .. _`fjetter`: https://github.com/fjetter
 .. _`Patrick Sodré`: https://github.com/sodre
 .. _`Stephan Erb`: https://github.com/StephanErb
+.. _`Benedikt Reinartz`: https://github.com/filmor
+.. _`Markus Mohrhard`: https://github.com/mmohrhard
+.. _`Mana Borwornpadungkitti`: https://github.com/potpath

From d88118d25e9751c54c07dd90efa2fe1008a7d6b8 Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Fri, 17 Jan 2020 14:03:08 +0000
Subject: [PATCH 0630/1550] Get JavaScript document location instead of window
 and handle proxied url (#3382)

---
 distributed/dashboard/static/js/individual-cluster-map.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/distributed/dashboard/static/js/individual-cluster-map.js b/distributed/dashboard/static/js/individual-cluster-map.js
index 20aed45b7b1..57b5d210a65 100644
--- a/distributed/dashboard/static/js/individual-cluster-map.js
+++ b/distributed/dashboard/static/js/individual-cluster-map.js
@@ -340,12 +340,12 @@ class Dashboard {
 }
 
 function get_websocket_url(endpoint) {
-  var l = window.location;
+  var l = document.location;
   return (
     (l.protocol === "https:" ? "wss://" : "ws://") +
     l.hostname +
     (l.port != 80 && l.port != 443 ? ":" + l.port : "") +
-    endpoint
+    l.pathname.replace("/statics/individual-cluster-map.html", endpoint)
   );
 }
 

From db24547945ffd8ee126f54dd196e486edb3ea66f Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Fri, 17 Jan 2020 10:13:27 -0600
Subject: [PATCH 0631/1550] Fix get_running_loop import (#3383)

---
 distributed/compatibility.py | 5 +++++
 distributed/utils.py         | 4 ++--
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/distributed/compatibility.py b/distributed/compatibility.py
index 186e66e485c..33e50e429b8 100644
--- a/distributed/compatibility.py
+++ b/distributed/compatibility.py
@@ -7,3 +7,8 @@
 
 PYPY = platform.python_implementation().lower() == "pypy"
 WINDOWS = sys.platform.startswith("win")
+
+if sys.version_info[:2] >= (3, 7):
+    from asyncio import get_running_loop
+else:
+    from asyncio import _get_running_loop as get_running_loop  # noqa: F401
diff --git a/distributed/utils.py b/distributed/utils.py
index df601291f06..15824262ab7 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -56,7 +56,7 @@
 except ImportError:
     PollIOLoop = None  # dropped in tornado 6.0
 
-from .compatibility import PYPY, WINDOWS
+from .compatibility import PYPY, WINDOWS, get_running_loop
 from .metrics import time
 
 
@@ -1204,7 +1204,7 @@ def reset_logger_locks():
 
         if is_kernel():
             try:
-                asyncio.get_running_loop()
+                get_running_loop()
             except RuntimeError:
                 is_kernel_and_no_running_loop = True
 

From 7b3c6e9427082b8f2a87c1e19e055da6f2fb69ff Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Fri, 17 Jan 2020 11:11:15 -0600
Subject: [PATCH 0632/1550] Raise RuntimeError if no running loop (#3385)

---
 distributed/compatibility.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/distributed/compatibility.py b/distributed/compatibility.py
index 33e50e429b8..0dca141e0e9 100644
--- a/distributed/compatibility.py
+++ b/distributed/compatibility.py
@@ -11,4 +11,11 @@
 if sys.version_info[:2] >= (3, 7):
     from asyncio import get_running_loop
 else:
-    from asyncio import _get_running_loop as get_running_loop  # noqa: F401
+
+    def get_running_loop():
+        from asyncio import _get_running_loop
+
+        loop = _get_running_loop()
+        if loop is None:
+            raise RuntimeError("no running event loop")
+        return loop

From 26007f244911307f2dcc495357f3398e494d1237 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Fri, 17 Jan 2020 11:14:15 -0600
Subject: [PATCH 0633/1550] bump version to 2.9.3

---
 docs/source/changelog.rst | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index 6410c2aa0b1..1bb19e70330 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,14 @@
 Changelog
 =========
 
+2.9.3 - 2020-01-17
+------------------
+
+- Raise ``RuntimeError`` if no running loop (:pr:`3385`) `James Bourbeau`_
+- Fix ``get_running_loop`` import (:pr:`3383`) `James Bourbeau`_
+- Get JavaScript document location instead of window and handle proxied url (:pr:`3382`) `Jacob Tomlinson`_
+
+
 2.9.2 - 2020-01-16
 ------------------
 

From b464ae3e3abdea6b9577ed3dca5afe9f42efde60 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Fri, 17 Jan 2020 15:25:23 -0600
Subject: [PATCH 0634/1550] Replace gen.with_timeout with asyncio.wait_for
 (#3372)

---
 distributed/client.py                     | 23 +++-----
 distributed/deploy/cluster.py             |  4 +-
 distributed/deploy/tests/test_local.py    |  2 +-
 distributed/lock.py                       |  7 +--
 distributed/nanny.py                      | 20 +++----
 distributed/node.py                       |  2 +-
 distributed/process.py                    |  4 +-
 distributed/scheduler.py                  |  2 +-
 distributed/tests/test_batched.py         | 10 ++--
 distributed/tests/test_client.py          |  8 +--
 distributed/tests/test_client_executor.py | 67 +++++++++++++----------
 distributed/tests/test_failed_workers.py  |  4 +-
 distributed/tests/test_nanny.py           |  2 +-
 distributed/tests/test_publish.py         |  4 +-
 distributed/tests/test_queues.py          |  6 +-
 distributed/tests/test_scheduler.py       |  9 ++-
 distributed/tests/test_stress.py          |  7 +--
 distributed/tests/test_worker.py          |  9 ++-
 distributed/utils.py                      |  3 +-
 distributed/utils_test.py                 | 29 ++++++----
 distributed/worker.py                     |  6 +-
 21 files changed, 113 insertions(+), 115 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 451a6628e73..0a4080cd05d 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -5,7 +5,6 @@
 from concurrent.futures._base import DoneAndNotDoneFutures
 from contextlib import contextmanager
 import copy
-from datetime import timedelta
 import errno
 from functools import partial
 import html
@@ -761,7 +760,7 @@ def sync(self, func, *args, asynchronous=None, callback_timeout=None, **kwargs):
         ):
             future = func(*args, **kwargs)
             if callback_timeout is not None:
-                future = gen.with_timeout(timedelta(seconds=callback_timeout), future)
+                future = asyncio.wait_for(future, callback_timeout)
             return future
         else:
             return sync(
@@ -1043,9 +1042,7 @@ async def _ensure_connected(self, timeout=None):
             )
             comm.name = "Client->Scheduler"
             if timeout is not None:
-                await gen.with_timeout(
-                    timedelta(seconds=timeout), self._update_scheduler_info()
-                )
+                await asyncio.wait_for(self._update_scheduler_info(), timeout)
             else:
                 await self._update_scheduler_info()
             await comm.write(
@@ -1064,7 +1061,7 @@ async def _ensure_connected(self, timeout=None):
         finally:
             self._connecting_to_scheduler = False
         if timeout is not None:
-            msg = await gen.with_timeout(timedelta(seconds=timeout), comm.read())
+            msg = await asyncio.wait_for(comm.read(), timeout)
         else:
             msg = await comm.read()
         assert len(msg) == 1
@@ -1268,11 +1265,9 @@ async def _close(self, fast=False):
 
             # Give the scheduler 'stream-closed' message 100ms to come through
             # This makes the shutdown slightly smoother and quieter
-            with ignoring(AttributeError, gen.TimeoutError):
-                await gen.with_timeout(
-                    timedelta(milliseconds=100),
-                    self._handle_scheduler_coroutine,
-                    quiet_exceptions=(CancelledError,),
+            with ignoring(AttributeError, CancelledError, asyncio.TimeoutError):
+                await asyncio.wait_for(
+                    asyncio.shield(self._handle_scheduler_coroutine), 0.1
                 )
 
             if (
@@ -1308,7 +1303,7 @@ async def _close(self, fast=False):
 
             if not fast:
                 with ignoring(TimeoutError):
-                    await gen.with_timeout(timedelta(seconds=2), list(coroutines))
+                    await asyncio.wait_for(asyncio.gather(*coroutines), 2)
 
             with ignoring(AttributeError):
                 await self.scheduler.close_rpc()
@@ -1344,7 +1339,7 @@ def close(self, timeout=no_default):
         if self.asynchronous:
             future = self._close()
             if timeout:
-                future = gen.with_timeout(timedelta(seconds=timeout), future)
+                future = asyncio.wait_for(future, timeout)
             return future
 
         if self._start_arg is None:
@@ -4077,7 +4072,7 @@ async def _wait(fs, timeout=None, return_when=ALL_COMPLETED):
 
     future = wait_for({f._state.wait() for f in fs})
     if timeout is not None:
-        future = gen.with_timeout(timedelta(seconds=timeout), future)
+        future = asyncio.wait_for(future, timeout)
     await future
 
     done, not_done = (
diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
index 2631fb502df..1b304b0a53e 100644
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -1,10 +1,8 @@
 import asyncio
-from datetime import timedelta
 import logging
 import threading
 
 from dask.utils import format_bytes
-from tornado import gen
 
 from .adaptive import Adaptive
 
@@ -156,7 +154,7 @@ def sync(self, func, *args, asynchronous=None, callback_timeout=None, **kwargs):
         if asynchronous:
             future = func(*args, **kwargs)
             if callback_timeout is not None:
-                future = gen.with_timeout(timedelta(seconds=callback_timeout), future)
+                future = asyncio.wait_for(future, callback_timeout)
             return future
         else:
             return sync(self.loop, func, *args, **kwargs)
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index 370423771a5..4687d0c476f 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -523,7 +523,7 @@ def test_memory_nanny(loop, n_workers):
 
 
 def test_death_timeout_raises(loop):
-    with pytest.raises(gen.TimeoutError):
+    with pytest.raises(asyncio.TimeoutError):
         with LocalCluster(
             scheduler_port=0,
             silence_logs=False,
diff --git a/distributed/lock.py b/distributed/lock.py
index ed3eb4313f2..c581bb5d552 100644
--- a/distributed/lock.py
+++ b/distributed/lock.py
@@ -1,9 +1,8 @@
 from collections import defaultdict, deque
-from datetime import timedelta
 import logging
 import uuid
+import asyncio
 
-from tornado import gen
 import tornado.locks
 
 from .client import _get_global_client
@@ -45,10 +44,10 @@ async def acquire(self, stream=None, name=None, id=None, timeout=None):
                     self.events[name].append(event)
                     future = event.wait()
                     if timeout is not None:
-                        future = gen.with_timeout(timedelta(seconds=timeout), future)
+                        future = asyncio.wait_for(future, timeout)
                     try:
                         await future
-                    except gen.TimeoutError:
+                    except asyncio.TimeoutError:
                         result = False
                         break
                     else:
diff --git a/distributed/nanny.py b/distributed/nanny.py
index 7cf3c2cbbaf..19d48328f47 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -1,5 +1,4 @@
 import asyncio
-from datetime import timedelta
 import logging
 from multiprocessing.queues import Empty
 import os
@@ -31,6 +30,7 @@
     json_load_robust,
     PeriodicCallback,
     parse_timedelta,
+    ignoring,
 )
 from .worker import run, parse_memory_limit, Worker
 
@@ -219,14 +219,10 @@ async def _unregister(self, timeout=10):
             EnvironmentError,
             RPCClosed,
         )
-        try:
-            await gen.with_timeout(
-                timedelta(seconds=timeout),
-                self.scheduler.unregister(address=self.worker_address),
-                quiet_exceptions=allowed_errors,
+        with ignoring(allowed_errors):
+            await asyncio.wait_for(
+                self.scheduler.unregister(address=self.worker_address), timeout
             )
-        except allowed_errors:
-            pass
 
     @property
     def worker_address(self):
@@ -318,8 +314,8 @@ async def instantiate(self, comm=None):
         self.auto_restart = True
         if self.death_timeout:
             try:
-                result = await gen.with_timeout(
-                    timedelta(seconds=self.death_timeout), self.process.start()
+                result = await asyncio.wait_for(
+                    self.process.start(), self.death_timeout
                 )
             except gen.TimeoutError:
                 await self.close(timeout=self.death_timeout)
@@ -343,8 +339,8 @@ async def _():
                 await self.instantiate()
 
         try:
-            await gen.with_timeout(timedelta(seconds=timeout), _())
-        except gen.TimeoutError:
+            await asyncio.wait_for(_(), timeout)
+        except asyncio.TimeoutError:
             logger.error("Restart timed out, returning before finished")
             return "timed out"
         else:
diff --git a/distributed/node.py b/distributed/node.py
index 2d7447b1a06..6cf30f997fe 100644
--- a/distributed/node.py
+++ b/distributed/node.py
@@ -173,7 +173,7 @@ async def wait_for(future, timeout=None):
                         await asyncio.wait_for(future, timeout=timeout)
                     except Exception:
                         await self.close(timeout=1)
-                        raise gen.TimeoutError(
+                        raise asyncio.TimeoutError(
                             "{} failed to start in {} seconds".format(
                                 type(self).__name__, timeout
                             )
diff --git a/distributed/process.py b/distributed/process.py
index 38527ecd9ab..4ad86e2bb08 100644
--- a/distributed/process.py
+++ b/distributed/process.py
@@ -1,11 +1,11 @@
 import atexit
-from datetime import timedelta
 import logging
 import os
 from queue import Queue as PyQueue
 import re
 import threading
 import weakref
+import asyncio
 import dask
 
 from .utils import mp_context
@@ -282,7 +282,7 @@ def join(self, timeout=None):
             yield self._exit_future
         else:
             try:
-                yield gen.with_timeout(timedelta(seconds=timeout), self._exit_future)
+                yield asyncio.wait_for(self._exit_future, timeout)
             except gen.TimeoutError:
                 pass
 
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 478eb413422..152df2d705f 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -2878,7 +2878,7 @@ async def restart(self, client=None, timeout=3):
                         for nanny in nannies
                     ]
                 )
-                resps = await gen.with_timeout(timedelta(seconds=timeout), resps)
+                resps = await asyncio.wait_for(resps, timeout)
                 if not all(resp == "OK" for resp in resps):
                     logger.error(
                         "Not all workers responded positively: %s", resps, exc_info=True
diff --git a/distributed/tests/test_batched.py b/distributed/tests/test_batched.py
index a961157f948..3174f3f3022 100644
--- a/distributed/tests/test_batched.py
+++ b/distributed/tests/test_batched.py
@@ -1,10 +1,8 @@
 import asyncio
-from datetime import timedelta
 import random
 
 import pytest
 from toolz import assoc
-from tornado import gen
 
 from distributed.batched import BatchedSend
 from distributed.core import listen, connect, CommClosedError
@@ -170,7 +168,7 @@ async def send():
 
         async def recv():
             while True:
-                result = await gen.with_timeout(timedelta(seconds=1), comm.read())
+                result = await asyncio.wait_for(comm.read(), 1)
                 L.extend(result)
                 if result[-1] == 9999:
                     break
@@ -205,7 +203,7 @@ async def run_traffic_jam(nsends, nbytes):
             # If this times out then I think it's a backpressure issue
             # Somehow we're able to flood the socket so that the receiving end
             # loses some of our messages
-            L = await gen.with_timeout(timedelta(seconds=5), comm.read())
+            L = await asyncio.wait_for(comm.read(), 5)
             count += 1
             results.extend(r["i"] for r in L)
 
@@ -254,5 +252,5 @@ async def test_serializers():
         msg = await comm.read()
         assert list(msg) == [{"x": 123}, {"x": "hello"}]
 
-        with pytest.raises(gen.TimeoutError):
-            msg = await gen.with_timeout(timedelta(milliseconds=100), comm.read())
+        with pytest.raises(asyncio.TimeoutError):
+            msg = await asyncio.wait_for(comm.read(), 0.1)
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 9450f08fd75..da23b85df0e 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -496,7 +496,7 @@ def test_thread(c):
     assert x.result() == 2
 
     x = c.submit(slowinc, 1, delay=0.3)
-    with pytest.raises(gen.TimeoutError):
+    with pytest.raises((gen.TimeoutError, asyncio.TimeoutError)):
         x.result(timeout=0.01)
     assert x.result() == 2
 
@@ -681,7 +681,7 @@ def test_wait_first_completed(c, s, a, b):
 @gen_cluster(client=True, timeout=2)
 def test_wait_timeout(c, s, a, b):
     future = c.submit(sleep, 0.3)
-    with pytest.raises(gen.TimeoutError):
+    with pytest.raises(asyncio.TimeoutError):
         yield wait(future, timeout=0.01)
 
 
@@ -695,7 +695,7 @@ def test_wait_sync(c):
     assert x.status == y.status == "finished"
 
     future = c.submit(sleep, 0.3)
-    with pytest.raises(gen.TimeoutError):
+    with pytest.raises(asyncio.TimeoutError):
         wait(future, timeout=0.01)
 
 
@@ -5279,7 +5279,7 @@ def test_client_active_bad_port():
     http_server.listen(8080)
     with dask.config.set({"distributed.comm.timeouts.connect": "10ms"}):
         c = Client("127.0.0.1:8080", asynchronous=True)
-        with pytest.raises((TimeoutError, IOError)):
+        with pytest.raises((asyncio.TimeoutError, IOError)):
             yield c
         yield c._close(fast=True)
     http_server.stop()
diff --git a/distributed/tests/test_client_executor.py b/distributed/tests/test_client_executor.py
index 7d08a63c5b2..40639998852 100644
--- a/distributed/tests/test_client_executor.py
+++ b/distributed/tests/test_client_executor.py
@@ -14,7 +14,16 @@
 import pytest
 from toolz import take
 
-from distributed.utils_test import slowinc, slowadd, slowdec, inc, throws, varying
+from distributed import Client
+from distributed.utils_test import (
+    slowinc,
+    slowadd,
+    slowdec,
+    inc,
+    throws,
+    varying,
+    cluster,
+)
 from distributed.utils_test import client, cluster_fixture, loop, s, a, b  # noqa: F401
 
 
@@ -218,30 +227,32 @@ def test_retries(client):
         exc_info.match("one")
 
 
-def test_shutdown(client):
-    # shutdown(wait=True) waits for pending tasks to finish
-    e = client.get_executor()
-    fut = e.submit(time.sleep, 1.0)
-    t1 = time.time()
-    e.shutdown()
-    dt = time.time() - t1
-    assert 0.5 <= dt <= 2.0
-    time.sleep(0.1)  # wait for future outcome to propagate
-    assert fut.done()
-    fut.result()  # doesn't raise
-
-    with pytest.raises(RuntimeError):
-        e.submit(time.sleep, 1.0)
-
-    # shutdown(wait=False) cancels pending tasks
-    e = client.get_executor()
-    fut = e.submit(time.sleep, 2.0)
-    t1 = time.time()
-    e.shutdown(wait=False)
-    dt = time.time() - t1
-    assert dt < 0.5
-    time.sleep(0.1)  # wait for future outcome to propagate
-    assert fut.cancelled()
-
-    with pytest.raises(RuntimeError):
-        e.submit(time.sleep, 1.0)
+def test_shutdown(loop):
+    with cluster(disconnect_timeout=10) as (s, [a, b]):
+        with Client(s["address"], loop=loop) as client:
+            # shutdown(wait=True) waits for pending tasks to finish
+            e = client.get_executor()
+            fut = e.submit(time.sleep, 1.0)
+            t1 = time.time()
+            e.shutdown()
+            dt = time.time() - t1
+            assert 0.5 <= dt <= 2.0
+            time.sleep(0.1)  # wait for future outcome to propagate
+            assert fut.done()
+            fut.result()  # doesn't raise
+
+            with pytest.raises(RuntimeError):
+                e.submit(time.sleep, 1.0)
+
+            # shutdown(wait=False) cancels pending tasks
+            e = client.get_executor()
+            fut = e.submit(time.sleep, 2.0)
+            t1 = time.time()
+            e.shutdown(wait=False)
+            dt = time.time() - t1
+            assert dt < 0.5
+            time.sleep(0.1)  # wait for future outcome to propagate
+            assert fut.cancelled()
+
+            with pytest.raises(RuntimeError):
+                e.submit(time.sleep, 1.0)
diff --git a/distributed/tests/test_failed_workers.py b/distributed/tests/test_failed_workers.py
index 8f790edf20e..3cc055b5246 100644
--- a/distributed/tests/test_failed_workers.py
+++ b/distributed/tests/test_failed_workers.py
@@ -26,7 +26,7 @@
 
 
 def test_submit_after_failed_worker_sync(loop):
-    with cluster(active_rpc_timeout=10) as (s, [a, b]):
+    with cluster(active_rpc_timeout=10, disconnect_timeout=10) as (s, [a, b]):
         with Client(s["address"], loop=loop) as c:
             L = c.map(inc, range(10))
             wait(L)
@@ -64,7 +64,7 @@ def test_submit_after_failed_worker(c, s, a, b):
 
 
 def test_gather_after_failed_worker(loop):
-    with cluster(active_rpc_timeout=10) as (s, [a, b]):
+    with cluster(active_rpc_timeout=10, disconnect_timeout=10) as (s, [a, b]):
         with Client(s["address"], loop=loop) as c:
             L = c.map(inc, range(10))
             wait(L)
diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
index 88910c87069..b5631f0a47f 100644
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -184,7 +184,7 @@ def test_nanny_alt_worker_class(c, s, w1, w2):
 def test_nanny_death_timeout(s):
     yield s.close()
     w = Nanny(s.address, death_timeout=1)
-    with pytest.raises(gen.TimeoutError):
+    with pytest.raises(asyncio.TimeoutError):
         yield w
 
     assert w.status == "closed"
diff --git a/distributed/tests/test_publish.py b/distributed/tests/test_publish.py
index 32b2974a738..dde10b11cf1 100644
--- a/distributed/tests/test_publish.py
+++ b/distributed/tests/test_publish.py
@@ -54,8 +54,8 @@ def test_publish_non_string_key(s, a, b):
             assert name in datasets
 
     finally:
-        c.close()
-        f.close()
+        yield c.close()
+        yield f.close()
 
 
 @gen_cluster(client=False)
diff --git a/distributed/tests/test_queues.py b/distributed/tests/test_queues.py
index 817bfcbcea5..80ce977e9f1 100644
--- a/distributed/tests/test_queues.py
+++ b/distributed/tests/test_queues.py
@@ -1,5 +1,5 @@
-from datetime import timedelta
 from time import sleep
+import asyncio
 
 import pytest
 from tornado import gen
@@ -181,8 +181,8 @@ def test_get_many(c, s, a, b):
     data = yield xx.get(batch=2)
     assert data == [1, 2]
 
-    with pytest.raises(gen.TimeoutError):
-        data = yield gen.with_timeout(timedelta(seconds=0.100), xx.get(batch=2))
+    with pytest.raises(asyncio.TimeoutError):
+        data = yield asyncio.wait_for(xx.get(batch=2), 0.1)
 
 
 @gen_cluster(client=True)
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index c493c41dd21..df13f7a1fc1 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -2,7 +2,6 @@
 import cloudpickle
 import pickle
 from collections import defaultdict
-from datetime import timedelta
 import json
 import operator
 import re
@@ -146,8 +145,8 @@ def test_no_valid_workers(client, s, a, b, c):
 
     assert s.tasks[x.key] in s.unrunnable
 
-    with pytest.raises(gen.TimeoutError):
-        yield gen.with_timeout(timedelta(milliseconds=50), x)
+    with pytest.raises(asyncio.TimeoutError):
+        yield asyncio.wait_for(x, 0.05)
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 3)
@@ -166,8 +165,8 @@ def test_no_workers(client, s):
 
     assert s.tasks[x.key] in s.unrunnable
 
-    with pytest.raises(gen.TimeoutError):
-        yield gen.with_timeout(timedelta(milliseconds=50), x)
+    with pytest.raises(asyncio.TimeoutError):
+        yield asyncio.wait_for(x, 0.05)
 
 
 @gen_cluster(nthreads=[])
diff --git a/distributed/tests/test_stress.py b/distributed/tests/test_stress.py
index db91ec0c004..5275bc47fd8 100644
--- a/distributed/tests/test_stress.py
+++ b/distributed/tests/test_stress.py
@@ -1,9 +1,9 @@
 from concurrent.futures import CancelledError
-from datetime import timedelta
 from operator import add
 import random
 import sys
 from time import sleep
+import asyncio
 
 from dask import delayed
 import pytest
@@ -111,9 +111,8 @@ def create_and_destroy_worker(delay):
             yield n.close()
             print("Killed nanny")
 
-    yield gen.with_timeout(
-        timedelta(minutes=1),
-        All([create_and_destroy_worker(0.1 * i) for i in range(20)]),
+    yield asyncio.wait_for(
+        All([create_and_destroy_worker(0.1 * i) for i in range(20)]), 60
     )
 
 
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index df886b9431a..a57cbaf536c 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -1,5 +1,4 @@
 from concurrent.futures import ThreadPoolExecutor
-from datetime import timedelta
 import importlib
 import logging
 from numbers import Number
@@ -10,6 +9,7 @@
 import sys
 from time import sleep
 import traceback
+import asyncio
 
 import dask
 from dask import delayed
@@ -19,7 +19,6 @@
 from toolz import pluck, sliding_window, first
 import tornado
 from tornado import gen
-from tornado.ioloop import TimeoutError
 
 from distributed import (
     Client,
@@ -326,8 +325,8 @@ def test_worker_waits_for_scheduler(loop):
     def f():
         w = Worker("127.0.0.1", 8007)
         try:
-            yield gen.with_timeout(timedelta(seconds=3), w)
-        except TimeoutError:
+            yield asyncio.wait_for(w, 3)
+        except asyncio.TimeoutError:
             pass
         else:
             assert False
@@ -762,7 +761,7 @@ def test_worker_death_timeout(s):
         yield s.close()
         w = Worker(s.address, death_timeout=1)
 
-    with pytest.raises(gen.TimeoutError) as info:
+    with pytest.raises(asyncio.TimeoutError) as info:
         yield w
 
     assert "Worker" in str(info.value)
diff --git a/distributed/utils.py b/distributed/utils.py
index 15824262ab7..39bc973cdb7 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -3,7 +3,6 @@
 from collections import deque, OrderedDict, UserDict
 from concurrent.futures import ThreadPoolExecutor
 from contextlib import contextmanager
-from datetime import timedelta
 import functools
 from hashlib import md5
 import html
@@ -325,7 +324,7 @@ def f():
             thread_state.asynchronous = True
             future = func(*args, **kwargs)
             if callback_timeout is not None:
-                future = gen.with_timeout(timedelta(seconds=callback_timeout), future)
+                future = asyncio.wait_for(future, callback_timeout)
             result[0] = yield future
         except Exception as exc:
             error[0] = sys.exc_info()
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index aef7bde8eee..2fba3c74bfc 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -2,7 +2,6 @@
 import collections
 from contextlib import contextmanager
 import copy
-from datetime import timedelta
 import functools
 from glob import glob
 import io
@@ -606,7 +605,12 @@ def security():
 
 @contextmanager
 def cluster(
-    nworkers=2, nanny=False, worker_kwargs={}, active_rpc_timeout=1, scheduler_kwargs={}
+    nworkers=2,
+    nanny=False,
+    worker_kwargs={},
+    active_rpc_timeout=1,
+    disconnect_timeout=3,
+    scheduler_kwargs={},
 ):
     ws = weakref.WeakSet()
     enable_proctitle_on_children()
@@ -689,10 +693,16 @@ def cluster(
 
             loop.run_sync(
                 lambda: disconnect_all(
-                    [w["address"] for w in workers], timeout=0.5, rpc_kwargs=rpc_kwargs
+                    [w["address"] for w in workers],
+                    timeout=disconnect_timeout,
+                    rpc_kwargs=rpc_kwargs,
+                )
+            )
+            loop.run_sync(
+                lambda: disconnect(
+                    saddr, timeout=disconnect_timeout, rpc_kwargs=rpc_kwargs
                 )
             )
-            loop.run_sync(lambda: disconnect(saddr, timeout=0.5, rpc_kwargs=rpc_kwargs))
 
             scheduler.terminate()
             scheduler_q.close()
@@ -740,8 +750,7 @@ async def do_disconnect():
             with rpc(addr, **rpc_kwargs) as w:
                 await w.terminate(close=True)
 
-    with ignoring(TimeoutError):
-        await gen.with_timeout(timedelta(seconds=timeout), do_disconnect())
+    await asyncio.wait_for(do_disconnect(), timeout=timeout)
 
 
 async def disconnect_all(addresses, timeout=3, rpc_kwargs=None):
@@ -914,9 +923,7 @@ async def coro():
                         try:
                             future = func(*args)
                             if timeout:
-                                future = gen.with_timeout(
-                                    timedelta(seconds=timeout), future
-                                )
+                                future = asyncio.wait_for(future, timeout)
                             result = await future
                             if s.validate:
                                 s.validate_state()
@@ -924,9 +931,7 @@ async def coro():
                             if client and c.status not in ("closing", "closed"):
                                 await c._close(fast=s.status == "closed")
                             await end_cluster(s, workers)
-                            await gen.with_timeout(
-                                timedelta(seconds=1), cleanup_global_workers()
-                            )
+                            await asyncio.wait_for(cleanup_global_workers(), 1)
 
                         try:
                             c = await default_client()
diff --git a/distributed/worker.py b/distributed/worker.py
index f4a662ce44f..9bb30dfd39d 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -1095,13 +1095,13 @@ async def close(
 
             for pc in self.periodic_callbacks.values():
                 pc.stop()
-            with ignoring(EnvironmentError, gen.TimeoutError):
+            with ignoring(EnvironmentError, asyncio.TimeoutError):
                 if report and self.contact_address is not None:
-                    await gen.with_timeout(
-                        timedelta(seconds=timeout),
+                    await asyncio.wait_for(
                         self.scheduler.unregister(
                             address=self.contact_address, safe=safe
                         ),
+                        timeout,
                     )
             await self.scheduler.close_rpc()
             self._workdir.release()

From 726f65438815317bd6c430b983463cfdbe34712b Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Sun, 19 Jan 2020 18:27:19 -0600
Subject: [PATCH 0635/1550] Use latest release of black (#3388)

---
 .pre-commit-config.yaml | 2 +-
 .travis.yml             | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 2c72a38ce93..2b64eddd06a 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,6 +1,6 @@
 repos:
 -   repo: https://github.com/ambv/black
-    rev: cad4138050b86d1c8570b926883e32f7465c2880
+    rev: stable
     hooks:
     - id: black
       language_version: python3.7
diff --git a/.travis.yml b/.travis.yml
index 5d3cbf0ec0b..56c2588ff5f 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -23,7 +23,7 @@ install:
 script:
   - if [[ $TESTS == true ]]; then source continuous_integration/travis/run_tests.sh ; fi
   - if [[ $LINT == true ]]; then pip install flake8 ; flake8 distributed ; fi
-  - if [[ $LINT == true ]]; then pip install git+https://github.com/psf/black@cad4138050b86d1c8570b926883e32f7465c2880; black distributed --check; fi
+  - if [[ $LINT == true ]]; then pip install black ; black distributed --check; fi
 
 after_success:
   - if [[ $COVERAGE == true ]]; then coverage report; pip install -q coveralls ; coveralls ; fi

From 84f220a838cd4578671652bd1fae3a46574c5e92 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 20 Jan 2020 13:22:20 -0800
Subject: [PATCH 0636/1550] Add lifecycle hooks to SchedulerPlugin (#3391)

* Add lifecycle hooks to SchedulerPlugin

This adds start and close async functions to Scheduler Plugins
---
 distributed/diagnostics/plugin.py             | 15 +++++++++++
 .../tests/test_scheduler_plugin.py            | 26 +++++++++++++++++--
 distributed/scheduler.py                      |  7 ++++-
 distributed/tests/test_utils.py               |  3 ++-
 4 files changed, 47 insertions(+), 4 deletions(-)

diff --git a/distributed/diagnostics/plugin.py b/distributed/diagnostics/plugin.py
index 4d94f7c8859..1d218fe5ac8 100644
--- a/distributed/diagnostics/plugin.py
+++ b/distributed/diagnostics/plugin.py
@@ -37,6 +37,21 @@ class SchedulerPlugin(object):
     >>> scheduler.add_plugin(plugin)  # doctest: +SKIP
     """
 
+    async def start(self, scheduler):
+        """ Run when the scheduler starts up
+
+        This runs at the end of the Scheduler startup process
+        """
+        pass
+
+    async def close(self):
+        """ Run when the scheduler closes down
+
+        This runs at the beginning of the Scheduler shutdown process, but after
+        workers have been asked to shut down gracefully
+        """
+        pass
+
     def update_graph(self, scheduler, dsk=None, keys=None, restrictions=None, **kwargs):
         """ Run when a new graph / tasks enter the scheduler """
 
diff --git a/distributed/diagnostics/tests/test_scheduler_plugin.py b/distributed/diagnostics/tests/test_scheduler_plugin.py
index 2903214ba32..6fc9e22f3df 100644
--- a/distributed/diagnostics/tests/test_scheduler_plugin.py
+++ b/distributed/diagnostics/tests/test_scheduler_plugin.py
@@ -1,5 +1,6 @@
-from distributed import Worker, SchedulerPlugin
-from distributed.utils_test import inc, gen_cluster
+import pytest
+from distributed import Scheduler, Worker, SchedulerPlugin
+from distributed.utils_test import inc, gen_cluster, cleanup  # noqa: F401
 
 
 @gen_cluster(client=True)
@@ -67,3 +68,24 @@ def remove_worker(self, worker, scheduler):
     a = yield Worker(s.address)
     yield a.close()
     assert events == []
+
+
+@pytest.mark.asyncio
+async def test_lifecycle(cleanup):
+    class LifeCycle(SchedulerPlugin):
+        def __init__(self):
+            self.history = []
+
+        async def start(self, scheduler):
+            self.scheduler = scheduler
+            self.history.append("started")
+
+        async def close(self):
+            self.history.append("closed")
+
+    plugin = LifeCycle()
+    async with Scheduler(plugins=[plugin]) as s:
+        pass
+
+    assert plugin.history == ["started", "closed"]
+    assert plugin.scheduler is s
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 152df2d705f..31ce2596e7e 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1041,6 +1041,7 @@ def __init__(
         dashboard_address=None,
         preload=None,
         preload_argv=(),
+        plugins=(),
         **kwargs
     ):
         self._setup_logging(logger)
@@ -1210,7 +1211,7 @@ def __init__(
         ]
 
         self.extensions = {}
-        self.plugins = []
+        self.plugins = list(plugins)
         self.transition_log = deque(
             maxlen=dask.config.get("distributed.scheduler.transition-log-length")
         )
@@ -1437,6 +1438,8 @@ def del_scheduler_file():
 
         preload_modules(self.preload, parameter=self, argv=self.preload_argv)
 
+        await asyncio.gather(*[plugin.start(self) for plugin in self.plugins])
+
         self.start_periodic_callbacks()
 
         setproctitle("dask-scheduler [%s]" % (self.address,))
@@ -1467,6 +1470,8 @@ async def close(self, comm=None, fast=False, close_workers=False):
                 else:
                     break
 
+        await asyncio.gather(*[plugin.close() for plugin in self.plugins])
+
         for pc in self.periodic_callbacks.values():
             pc.stop()
         self.periodic_callbacks.clear()
diff --git a/distributed/tests/test_utils.py b/distributed/tests/test_utils.py
index ff2e42313ac..cf15985eb7a 100644
--- a/distributed/tests/test_utils.py
+++ b/distributed/tests/test_utils.py
@@ -1,3 +1,4 @@
+import asyncio
 import array
 import datetime
 from functools import partial
@@ -109,7 +110,7 @@ def function2(x):
 
 def test_sync_timeout(loop_in_thread):
     loop = loop_in_thread
-    with pytest.raises(gen.TimeoutError):
+    with pytest.raises((asyncio.TimeoutError, gen.TimeoutError)):
         sync(loop_in_thread, gen.sleep, 0.5, callback_timeout=0.05)
 
 

From 3a2f8a84121d27decf432cd4f379724907ea2b3d Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 21 Jan 2020 14:10:08 -0800
Subject: [PATCH 0637/1550] Replace gen.TimeoutError with utils.TimeoutError
 (#3394)

* Replace gen.TimeoutError with utils.TimeoutError

Previously we had a mix of gen.TimeoutError and asyncio.TimeoutError
within the code.  This import asyncio.TimeoutError within utils.py (and
our top level imports) and then uses that consistently throughout the
codebase

* Translate gen.TimeoutErrors into asyncio.TimeoutErrors

Sometimes when we use Tornado objects like events and queues we
experience gen.TimeoutErrors.  In these cases we raise instead
asyncio.TimeoutErrors
---
 distributed/__init__.py                |  4 +---
 distributed/cfexecutor.py              |  4 ++--
 distributed/client.py                  |  6 ++---
 distributed/comm/core.py               |  4 ++--
 distributed/deploy/spec.py             |  5 +++--
 distributed/deploy/tests/test_local.py |  4 ++--
 distributed/lock.py                    |  4 ++--
 distributed/nanny.py                   | 18 ++++++---------
 distributed/node.py                    |  4 ++--
 distributed/process.py                 |  4 ++--
 distributed/pubsub.py                  |  9 +++++---
 distributed/queues.py                  | 22 +++++++++++++-----
 distributed/scheduler.py               | 31 +++++++++++++-------------
 distributed/tests/test_batched.py      |  4 ++--
 distributed/tests/test_client.py       | 16 ++++++-------
 distributed/tests/test_nanny.py        |  4 ++--
 distributed/tests/test_queues.py       | 12 +++++-----
 distributed/tests/test_scheduler.py    | 10 ++++-----
 distributed/tests/test_utils.py        |  4 ++--
 distributed/tests/test_variable.py     |  6 ++---
 distributed/tests/test_worker.py       |  6 ++---
 distributed/utils.py                   |  3 ++-
 distributed/utils_test.py              |  4 ++--
 distributed/variable.py                | 11 +++++----
 distributed/worker.py                  |  7 +++---
 25 files changed, 110 insertions(+), 96 deletions(-)

diff --git a/distributed/__init__.py b/distributed/__init__.py
index 06136dd72a2..9238d57ccc9 100644
--- a/distributed/__init__.py
+++ b/distributed/__init__.py
@@ -24,13 +24,11 @@
 from .queues import Queue
 from .scheduler import Scheduler
 from .threadpoolexecutor import rejoin
-from .utils import sync
+from .utils import sync, TimeoutError
 from .variable import Variable
 from .worker import Worker, get_worker, get_client, secede, Reschedule
 from .worker_client import local_client, worker_client
 
-from tornado.gen import TimeoutError
-
 from ._version import get_versions
 
 versions = get_versions()
diff --git a/distributed/cfexecutor.py b/distributed/cfexecutor.py
index 373a3c4eb28..985a407bdb9 100644
--- a/distributed/cfexecutor.py
+++ b/distributed/cfexecutor.py
@@ -6,7 +6,7 @@
 from tornado import gen
 
 from .metrics import time
-from .utils import sync
+from .utils import sync, TimeoutError
 
 
 @gen.coroutine
@@ -135,7 +135,7 @@ def result_iterator():
                     if timeout is not None:
                         try:
                             yield future.result(end_time - time())
-                        except gen.TimeoutError:
+                        except TimeoutError:
                             raise cf.TimeoutError
                     else:
                         yield future.result()
diff --git a/distributed/client.py b/distributed/client.py
index 0a4080cd05d..7a39ec4b235 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -37,7 +37,6 @@
 except ImportError:
     single_key = first
 from tornado import gen
-from tornado.gen import TimeoutError
 from tornado.locks import Event, Condition, Semaphore
 from tornado.ioloop import IOLoop
 from tornado.queues import Queue
@@ -85,6 +84,7 @@
     Any,
     has_keyword,
     format_dashboard_link,
+    TimeoutError,
 )
 from . import versions as version_module
 
@@ -1265,7 +1265,7 @@ async def _close(self, fast=False):
 
             # Give the scheduler 'stream-closed' message 100ms to come through
             # This makes the shutdown slightly smoother and quieter
-            with ignoring(AttributeError, CancelledError, asyncio.TimeoutError):
+            with ignoring(AttributeError, CancelledError, TimeoutError):
                 await asyncio.wait_for(
                     asyncio.shield(self._handle_scheduler_coroutine), 0.1
                 )
@@ -1957,7 +1957,7 @@ async def _scatter(
                     if nthreads is not None:
                         await asyncio.sleep(0.1)
                     if time() > start + timeout:
-                        raise gen.TimeoutError("No valid workers found")
+                        raise TimeoutError("No valid workers found")
                     nthreads = await self.scheduler.ncores(workers=workers)
                 if not nthreads:
                     raise ValueError("No valid workers")
diff --git a/distributed/comm/core.py b/distributed/comm/core.py
index 42c95e3579e..e801242bb40 100644
--- a/distributed/comm/core.py
+++ b/distributed/comm/core.py
@@ -6,7 +6,7 @@
 import dask
 
 from ..metrics import time
-from ..utils import parse_timedelta, ignoring
+from ..utils import parse_timedelta, ignoring, TimeoutError
 from . import registry
 from .addressing import parse_address
 
@@ -209,7 +209,7 @@ def _raise(error):
                 future = connector.connect(
                     loc, deserialize=deserialize, **(connection_args or {})
                 )
-                with ignoring(asyncio.TimeoutError):
+                with ignoring(TimeoutError):
                     comm = await asyncio.wait_for(
                         future, timeout=min(deadline - time(), 1)
                     )
diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index fb06057bb64..96279d15323 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -6,8 +6,8 @@
 import weakref
 
 import dask
-from tornado import gen
 from tornado.locks import Event
+from tornado import gen
 
 from .adaptive import Adaptive
 from .cluster import Cluster
@@ -19,6 +19,7 @@
     parse_bytes,
     parse_timedelta,
     import_term,
+    TimeoutError,
 )
 from ..scheduler import Scheduler
 from ..security import Security
@@ -602,6 +603,6 @@ async def run_spec(spec: dict, *args):
 @atexit.register
 def close_clusters():
     for cluster in list(SpecCluster._instances):
-        with ignoring(gen.TimeoutError):
+        with ignoring((gen.TimeoutError, TimeoutError)):
             if cluster.status != "closed":
                 cluster.close(timeout=10)
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index 4687d0c476f..98d04c78d17 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -31,7 +31,7 @@
     tls_only_security,
 )
 from distributed.utils_test import loop  # noqa: F401
-from distributed.utils import sync
+from distributed.utils import sync, TimeoutError
 
 from distributed.deploy.utils_test import ClusterTest
 
@@ -523,7 +523,7 @@ def test_memory_nanny(loop, n_workers):
 
 
 def test_death_timeout_raises(loop):
-    with pytest.raises(asyncio.TimeoutError):
+    with pytest.raises(TimeoutError):
         with LocalCluster(
             scheduler_port=0,
             silence_logs=False,
diff --git a/distributed/lock.py b/distributed/lock.py
index c581bb5d552..c230a8e861c 100644
--- a/distributed/lock.py
+++ b/distributed/lock.py
@@ -6,7 +6,7 @@
 import tornado.locks
 
 from .client import _get_global_client
-from .utils import log_errors
+from .utils import log_errors, TimeoutError
 from .worker import get_worker
 
 logger = logging.getLogger(__name__)
@@ -47,7 +47,7 @@ async def acquire(self, stream=None, name=None, id=None, timeout=None):
                         future = asyncio.wait_for(future, timeout)
                     try:
                         await future
-                    except asyncio.TimeoutError:
+                    except TimeoutError:
                         result = False
                         break
                     else:
diff --git a/distributed/nanny.py b/distributed/nanny.py
index 19d48328f47..945f33041d3 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -11,9 +11,9 @@
 
 import dask
 from dask.system import CPU_COUNT
-from tornado import gen
-from tornado.ioloop import IOLoop, TimeoutError
+from tornado.ioloop import IOLoop
 from tornado.locks import Event
+from tornado import gen
 
 from .comm import get_address_host, unparse_host_port
 from .comm.addressing import address_from_user_args
@@ -31,6 +31,7 @@
     PeriodicCallback,
     parse_timedelta,
     ignoring,
+    TimeoutError,
 )
 from .worker import run, parse_memory_limit, Worker
 
@@ -213,12 +214,7 @@ async def _unregister(self, timeout=10):
         if worker_address is None:
             return
 
-        allowed_errors = (
-            gen.TimeoutError,
-            CommClosedError,
-            EnvironmentError,
-            RPCClosed,
-        )
+        allowed_errors = (TimeoutError, CommClosedError, EnvironmentError, RPCClosed)
         with ignoring(allowed_errors):
             await asyncio.wait_for(
                 self.scheduler.unregister(address=self.worker_address), timeout
@@ -317,7 +313,7 @@ async def instantiate(self, comm=None):
                 result = await asyncio.wait_for(
                     self.process.start(), self.death_timeout
                 )
-            except gen.TimeoutError:
+            except TimeoutError:
                 await self.close(timeout=self.death_timeout)
                 logger.error(
                     "Timed out connecting Nanny '%s' to scheduler '%s'",
@@ -340,7 +336,7 @@ async def _():
 
         try:
             await asyncio.wait_for(_(), timeout)
-        except asyncio.TimeoutError:
+        except TimeoutError:
             logger.error("Restart timed out, returning before finished")
             return "timed out"
         else:
@@ -729,7 +725,7 @@ async def run():
 
         try:
             loop.run_sync(run)
-        except TimeoutError:
+        except (TimeoutError, gen.TimeoutError):
             # Loop was stopped before wait_until_closed() returned, ignore
             pass
         except KeyboardInterrupt:
diff --git a/distributed/node.py b/distributed/node.py
index 6cf30f997fe..edee3e2dd7b 100644
--- a/distributed/node.py
+++ b/distributed/node.py
@@ -9,7 +9,7 @@
 
 from .core import Server, ConnectionPool
 from .versions import get_versions
-from .utils import DequeHandler
+from .utils import DequeHandler, TimeoutError
 
 
 class Node(object):
@@ -173,7 +173,7 @@ async def wait_for(future, timeout=None):
                         await asyncio.wait_for(future, timeout=timeout)
                     except Exception:
                         await self.close(timeout=1)
-                        raise asyncio.TimeoutError(
+                        raise TimeoutError(
                             "{} failed to start in {} seconds".format(
                                 type(self).__name__, timeout
                             )
diff --git a/distributed/process.py b/distributed/process.py
index 4ad86e2bb08..5899c853385 100644
--- a/distributed/process.py
+++ b/distributed/process.py
@@ -8,7 +8,7 @@
 import asyncio
 import dask
 
-from .utils import mp_context
+from .utils import mp_context, TimeoutError
 
 from tornado import gen
 from tornado.concurrent import Future
@@ -283,7 +283,7 @@ def join(self, timeout=None):
         else:
             try:
                 yield asyncio.wait_for(self._exit_future, timeout)
-            except gen.TimeoutError:
+            except TimeoutError:
                 pass
 
     def close(self):
diff --git a/distributed/pubsub.py b/distributed/pubsub.py
index 0a5d82897fd..9de133ddb47 100644
--- a/distributed/pubsub.py
+++ b/distributed/pubsub.py
@@ -8,7 +8,7 @@
 from tornado import gen
 
 from .core import CommClosedError
-from .utils import sync
+from .utils import sync, TimeoutError
 from .protocol.serialize import to_serialize
 
 logger = logging.getLogger(__name__)
@@ -400,10 +400,13 @@ async def _get(self, timeout=None):
             if timeout is not None:
                 timeout2 = timeout - (datetime.datetime.now() - start)
                 if timeout2.total_seconds() < 0:
-                    raise gen.TimeoutError()
+                    raise TimeoutError()
             else:
                 timeout2 = None
-            await self.condition.wait(timeout=timeout2)
+            try:
+                await self.condition.wait(timeout=timeout2)
+            except gen.TimeoutError:
+                raise TimeoutError("Timed out waiting on Sub")
 
         return self.buffer.popleft()
 
diff --git a/distributed/queues.py b/distributed/queues.py
index 1d0c2c0bdd3..6d1fc76571b 100644
--- a/distributed/queues.py
+++ b/distributed/queues.py
@@ -5,9 +5,10 @@
 
 import tornado.queues
 from tornado.locks import Event
+from tornado import gen
 
 from .client import Future, _get_global_client, Client
-from .utils import tokey, sync, thread_state
+from .utils import tokey, sync, thread_state, TimeoutError
 from .worker import get_client
 
 logger = logging.getLogger(__name__)
@@ -78,7 +79,10 @@ async def put(
             record = {"type": "msgpack", "value": data}
         if timeout is not None:
             timeout = datetime.timedelta(seconds=timeout)
-        await self.queues[name].put(record, timeout=timeout)
+        try:
+            await self.queues[name].put(record, timeout=timeout)
+        except gen.TimeoutError:
+            raise TimeoutError("Timed out waiting for Queue")
 
     def future_release(self, name=None, key=None, client=None):
         self.future_refcount[name, key] -= 1
@@ -124,7 +128,10 @@ def process(record):
         else:
             if timeout is not None:
                 timeout = datetime.timedelta(seconds=timeout)
-            record = await self.queues[name].get(timeout=timeout)
+            try:
+                record = await self.queues[name].get(timeout=timeout)
+            except gen.TimeoutError:
+                raise TimeoutError("Timed out waiting for Queue")
             record = process(record)
             return record
 
@@ -225,9 +232,12 @@ def qsize(self, **kwargs):
         return self.client.sync(self._qsize, **kwargs)
 
     async def _get(self, timeout=None, batch=False):
-        resp = await self.client.scheduler.queue_get(
-            timeout=timeout, name=self.name, batch=batch
-        )
+        try:
+            resp = await self.client.scheduler.queue_get(
+                timeout=timeout, name=self.name, batch=batch
+            )
+        except gen.TimeoutError:
+            raise TimeoutError("Timed out waiting for Queue")
 
         def process(d):
             if d["type"] == "Future":
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 31ce2596e7e..8080a1186da 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -24,7 +24,6 @@
 except ImportError:
     from toolz import frequencies, merge, pluck, merge_sorted, first, merge_with
 from toolz import valmap, second, compose, groupby
-from tornado import gen
 from tornado.ioloop import IOLoop
 
 import dask
@@ -60,6 +59,7 @@
     key_split_group,
     empty_context,
     tmpfile,
+    TimeoutError,
 )
 from .utils_comm import scatter_to_workers, gather_from_workers, retry_operation
 from .utils_perf import enable_gc_diagnosis, disable_gc_diagnosis
@@ -2744,7 +2744,7 @@ async def scatter(
         while not self.workers:
             await asyncio.sleep(0.2)
             if time() > start + timeout:
-                raise gen.TimeoutError("No workers found")
+                raise TimeoutError("No workers found")
 
         if workers is None:
             nthreads = {w: ws.nthreads for w, ws in self.workers.items()}
@@ -2874,25 +2874,26 @@ async def restart(self, client=None, timeout=3):
                 if nanny_address is not None
             ]
 
+            resps = All(
+                [
+                    nanny.restart(
+                        close=True, timeout=timeout * 0.8, executor_wait=False
+                    )
+                    for nanny in nannies
+                ]
+            )
             try:
-                resps = All(
-                    [
-                        nanny.restart(
-                            close=True, timeout=timeout * 0.8, executor_wait=False
-                        )
-                        for nanny in nannies
-                    ]
-                )
                 resps = await asyncio.wait_for(resps, timeout)
-                if not all(resp == "OK" for resp in resps):
-                    logger.error(
-                        "Not all workers responded positively: %s", resps, exc_info=True
-                    )
-            except gen.TimeoutError:
+            except TimeoutError:
                 logger.error(
                     "Nannies didn't report back restarted within "
                     "timeout.  Continuuing with restart process"
                 )
+            else:
+                if not all(resp == "OK" for resp in resps):
+                    logger.error(
+                        "Not all workers responded positively: %s", resps, exc_info=True
+                    )
             finally:
                 await asyncio.gather(*[nanny.close_rpc() for nanny in nannies])
 
diff --git a/distributed/tests/test_batched.py b/distributed/tests/test_batched.py
index 3174f3f3022..07dd32f4c68 100644
--- a/distributed/tests/test_batched.py
+++ b/distributed/tests/test_batched.py
@@ -7,7 +7,7 @@
 from distributed.batched import BatchedSend
 from distributed.core import listen, connect, CommClosedError
 from distributed.metrics import time
-from distributed.utils import All
+from distributed.utils import All, TimeoutError
 from distributed.utils_test import captured_logger
 from distributed.protocol import to_serialize
 
@@ -252,5 +252,5 @@ async def test_serializers():
         msg = await comm.read()
         assert list(msg) == [{"x": 123}, {"x": "hello"}]
 
-        with pytest.raises(asyncio.TimeoutError):
+        with pytest.raises(TimeoutError):
             msg = await asyncio.wait_for(comm.read(), 0.1)
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index da23b85df0e..77180eefa35 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -496,7 +496,7 @@ def test_thread(c):
     assert x.result() == 2
 
     x = c.submit(slowinc, 1, delay=0.3)
-    with pytest.raises((gen.TimeoutError, asyncio.TimeoutError)):
+    with pytest.raises(TimeoutError):
         x.result(timeout=0.01)
     assert x.result() == 2
 
@@ -681,7 +681,7 @@ def test_wait_first_completed(c, s, a, b):
 @gen_cluster(client=True, timeout=2)
 def test_wait_timeout(c, s, a, b):
     future = c.submit(sleep, 0.3)
-    with pytest.raises(asyncio.TimeoutError):
+    with pytest.raises(TimeoutError):
         yield wait(future, timeout=0.01)
 
 
@@ -695,7 +695,7 @@ def test_wait_sync(c):
     assert x.status == y.status == "finished"
 
     future = c.submit(sleep, 0.3)
-    with pytest.raises(asyncio.TimeoutError):
+    with pytest.raises(TimeoutError):
         wait(future, timeout=0.01)
 
 
@@ -1394,7 +1394,7 @@ def test_scatter_direct_broadcast_target(c, s, *workers):
 
 @gen_cluster(client=True, nthreads=[])
 def test_scatter_direct_empty(c, s):
-    with pytest.raises((ValueError, gen.TimeoutError)):
+    with pytest.raises((ValueError, TimeoutError)):
         yield c.scatter(123, direct=True, timeout=0.1)
 
 
@@ -1801,12 +1801,12 @@ def test_allow_restrictions(c, s, a, b):
 def test_bad_address():
     try:
         Client("123.123.123.123:1234", timeout=0.1)
-    except (IOError, gen.TimeoutError) as e:
+    except (IOError, TimeoutError) as e:
         assert "connect" in str(e).lower()
 
     try:
         Client("127.0.0.1:1234", timeout=0.1)
-    except (IOError, gen.TimeoutError) as e:
+    except (IOError, TimeoutError) as e:
         assert "connect" in str(e).lower()
 
 
@@ -3501,7 +3501,7 @@ def test_persist_optimize_graph(c, s, a, b):
 
 @gen_cluster(client=True, nthreads=[])
 def test_scatter_raises_if_no_workers(c, s):
-    with pytest.raises(gen.TimeoutError):
+    with pytest.raises(TimeoutError):
         yield c.scatter(1, timeout=0.5)
 
 
@@ -5279,7 +5279,7 @@ def test_client_active_bad_port():
     http_server.listen(8080)
     with dask.config.set({"distributed.comm.timeouts.connect": "10ms"}):
         c = Client("127.0.0.1:8080", asynchronous=True)
-        with pytest.raises((asyncio.TimeoutError, IOError)):
+        with pytest.raises((TimeoutError, IOError)):
             yield c
         yield c._close(fast=True)
     http_server.stop()
diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
index b5631f0a47f..2ddc3b7e5db 100644
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -20,7 +20,7 @@
 from distributed.core import CommClosedError
 from distributed.metrics import time
 from distributed.protocol.pickle import dumps
-from distributed.utils import ignoring, tmpfile
+from distributed.utils import ignoring, tmpfile, TimeoutError
 from distributed.utils_test import (  # noqa: F401
     gen_cluster,
     gen_test,
@@ -184,7 +184,7 @@ def test_nanny_alt_worker_class(c, s, w1, w2):
 def test_nanny_death_timeout(s):
     yield s.close()
     w = Nanny(s.address, death_timeout=1)
-    with pytest.raises(asyncio.TimeoutError):
+    with pytest.raises(TimeoutError):
         yield w
 
     assert w.status == "closed"
diff --git a/distributed/tests/test_queues.py b/distributed/tests/test_queues.py
index 80ce977e9f1..d797433d6b4 100644
--- a/distributed/tests/test_queues.py
+++ b/distributed/tests/test_queues.py
@@ -4,7 +4,7 @@
 import pytest
 from tornado import gen
 
-from distributed import Client, Queue, Nanny, worker_client, wait
+from distributed import Client, Queue, Nanny, worker_client, wait, TimeoutError
 from distributed.metrics import time
 from distributed.utils_test import gen_cluster, inc, div
 from distributed.utils_test import client, cluster_fixture, loop  # noqa: F401
@@ -24,7 +24,7 @@ def test_queue(c, s, a, b):
     future2 = yield xx.get()
     assert future.key == future2.key
 
-    with pytest.raises(gen.TimeoutError):
+    with pytest.raises(TimeoutError):
         yield x.get(timeout=0.1)
 
     del future, future2
@@ -50,7 +50,7 @@ def test_queue_with_data(c, s, a, b):
 
     assert data == (1, "hello")
 
-    with pytest.raises(gen.TimeoutError):
+    with pytest.raises(TimeoutError):
         yield x.get(timeout=0.1)
 
 
@@ -181,7 +181,7 @@ def test_get_many(c, s, a, b):
     data = yield xx.get(batch=2)
     assert data == [1, 2]
 
-    with pytest.raises(asyncio.TimeoutError):
+    with pytest.raises(TimeoutError):
         data = yield asyncio.wait_for(xx.get(batch=2), 0.1)
 
 
@@ -248,7 +248,7 @@ def test_timeout(c, s, a, b):
     q = yield Queue("v", maxsize=1)
 
     start = time()
-    with pytest.raises(gen.TimeoutError):
+    with pytest.raises(TimeoutError):
         yield q.get(timeout=0.3)
     stop = time()
     assert 0.2 < stop - start < 2.0
@@ -256,7 +256,7 @@ def test_timeout(c, s, a, b):
     yield q.put(1)
 
     start = time()
-    with pytest.raises(gen.TimeoutError):
+    with pytest.raises(TimeoutError):
         yield q.put(2, timeout=0.3)
     stop = time()
     assert 0.1 < stop - start < 2.0
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index df13f7a1fc1..fd0775ad003 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -25,7 +25,7 @@
 from distributed.metrics import time
 from distributed.protocol.pickle import dumps
 from distributed.worker import dumps_function, dumps_task
-from distributed.utils import tmpfile, typename
+from distributed.utils import tmpfile, typename, TimeoutError
 from distributed.utils_test import (  # noqa: F401
     captured_logger,
     cleanup,
@@ -145,7 +145,7 @@ def test_no_valid_workers(client, s, a, b, c):
 
     assert s.tasks[x.key] in s.unrunnable
 
-    with pytest.raises(asyncio.TimeoutError):
+    with pytest.raises(TimeoutError):
         yield asyncio.wait_for(x, 0.05)
 
 
@@ -165,7 +165,7 @@ def test_no_workers(client, s):
 
     assert s.tasks[x.key] in s.unrunnable
 
-    with pytest.raises(asyncio.TimeoutError):
+    with pytest.raises(TimeoutError):
         yield asyncio.wait_for(x, 0.05)
 
 
@@ -656,11 +656,11 @@ def test_story(c, s, a, b):
 
 @gen_cluster(nthreads=[], client=True)
 def test_scatter_no_workers(c, s):
-    with pytest.raises(gen.TimeoutError):
+    with pytest.raises(TimeoutError):
         yield s.scatter(data={"x": 1}, client="alice", timeout=0.1)
 
     start = time()
-    with pytest.raises(gen.TimeoutError):
+    with pytest.raises(TimeoutError):
         yield c.scatter(123, timeout=0.1)
     assert time() < start + 1.5
 
diff --git a/distributed/tests/test_utils.py b/distributed/tests/test_utils.py
index cf15985eb7a..93b843358a8 100644
--- a/distributed/tests/test_utils.py
+++ b/distributed/tests/test_utils.py
@@ -1,4 +1,3 @@
-import asyncio
 import array
 import datetime
 from functools import partial
@@ -44,6 +43,7 @@
     warn_on_duration,
     format_dashboard_link,
     LRU,
+    TimeoutError,
 )
 from distributed.utils_test import loop, loop_in_thread  # noqa: F401
 from distributed.utils_test import div, has_ipv6, inc, throws, gen_test, captured_logger
@@ -110,7 +110,7 @@ def function2(x):
 
 def test_sync_timeout(loop_in_thread):
     loop = loop_in_thread
-    with pytest.raises((asyncio.TimeoutError, gen.TimeoutError)):
+    with pytest.raises(TimeoutError):
         sync(loop_in_thread, gen.sleep, 0.5, callback_timeout=0.05)
 
 
diff --git a/distributed/tests/test_variable.py b/distributed/tests/test_variable.py
index 6dcca9c9cf4..962b7a40e42 100644
--- a/distributed/tests/test_variable.py
+++ b/distributed/tests/test_variable.py
@@ -6,7 +6,7 @@
 import pytest
 from tornado import gen
 
-from distributed import Client, Variable, worker_client, Nanny, wait
+from distributed import Client, Variable, worker_client, Nanny, wait, TimeoutError
 from distributed.metrics import time
 from distributed.utils_test import gen_cluster, inc, div
 from distributed.utils_test import client, cluster_fixture, loop  # noqa: F401
@@ -84,7 +84,7 @@ def test_timeout(c, s, a, b):
     v = Variable("v")
 
     start = time()
-    with pytest.raises(gen.TimeoutError):
+    with pytest.raises(TimeoutError):
         yield v.get(timeout=0.1)
     stop = time()
     assert 0.1 < stop - start < 2.0
@@ -93,7 +93,7 @@ def test_timeout(c, s, a, b):
 def test_timeout_sync(client):
     v = Variable("v")
     start = time()
-    with pytest.raises(gen.TimeoutError):
+    with pytest.raises(TimeoutError):
         v.get(timeout=0.1)
     stop = time()
     assert 0.1 < stop - start < 2.0
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index a57cbaf536c..11ab461ae25 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -35,7 +35,7 @@
 from distributed.scheduler import Scheduler
 from distributed.metrics import time
 from distributed.worker import Worker, error_message, logger, parse_memory_limit
-from distributed.utils import tmpfile
+from distributed.utils import tmpfile, TimeoutError
 from distributed.utils_test import (  # noqa: F401
     cleanup,
     inc,
@@ -326,7 +326,7 @@ def f():
         w = Worker("127.0.0.1", 8007)
         try:
             yield asyncio.wait_for(w, 3)
-        except asyncio.TimeoutError:
+        except TimeoutError:
             pass
         else:
             assert False
@@ -761,7 +761,7 @@ def test_worker_death_timeout(s):
         yield s.close()
         w = Worker(s.address, death_timeout=1)
 
-    with pytest.raises(asyncio.TimeoutError) as info:
+    with pytest.raises(TimeoutError) as info:
         yield w
 
     assert "Worker" in str(info.value)
diff --git a/distributed/utils.py b/distributed/utils.py
index 39bc973cdb7..ee9d7948a11 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -1,4 +1,5 @@
 import asyncio
+from asyncio import TimeoutError
 import atexit
 from collections import deque, OrderedDict, UserDict
 from concurrent.futures import ThreadPoolExecutor
@@ -335,7 +336,7 @@ def f():
     loop.add_callback(f)
     if callback_timeout is not None:
         if not e.wait(callback_timeout):
-            raise gen.TimeoutError("timed out after %s s." % (callback_timeout,))
+            raise TimeoutError("timed out after %s s." % (callback_timeout,))
     else:
         while not e.is_set():
             e.wait(10)
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index 2fba3c74bfc..1650e0426ef 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -34,7 +34,6 @@
 import dask
 from toolz import merge, memoize, assoc
 from tornado import gen, queues
-from tornado.gen import TimeoutError
 from tornado.ioloop import IOLoop
 
 from . import system
@@ -60,6 +59,7 @@
     iscoroutinefunction,
     thread_state,
     _offload_executor,
+    TimeoutError,
 )
 from .worker import Worker
 from .nanny import Nanny
@@ -141,7 +141,7 @@ def start():
             except RuntimeError as e:
                 if not re.match("IOLoop is clos(ed|ing)", str(e)):
                     raise
-            except gen.TimeoutError:
+            except TimeoutError:
                 pass
             else:
                 is_stopped.wait()
diff --git a/distributed/variable.py b/distributed/variable.py
index 2169c287f61..677e2997b32 100644
--- a/distributed/variable.py
+++ b/distributed/variable.py
@@ -3,8 +3,8 @@
 import logging
 import uuid
 
-from tornado import gen
 import tornado.locks
+from tornado import gen
 
 try:
     from cytoolz import merge
@@ -13,7 +13,7 @@
 
 from .client import Future, _get_global_client, Client
 from .metrics import time
-from .utils import tokey, log_errors
+from .utils import tokey, log_errors, TimeoutError
 from .worker import get_client
 
 logger = logging.getLogger(__name__)
@@ -82,8 +82,11 @@ async def get(self, stream=None, name=None, client=None, timeout=None):
             else:
                 left = None
             if left and left < 0:
-                raise gen.TimeoutError()
-            await self.started.wait(timeout=left)
+                raise TimeoutError()
+            try:
+                await self.started.wait(timeout=left)
+            except gen.TimeoutError:
+                raise TimeoutError("Timed out waiting for Variable.get")
         record = self.variables[name]
         if record["type"] == "Future":
             key = record["value"]
diff --git a/distributed/worker.py b/distributed/worker.py
index 9bb30dfd39d..7062aba5b87 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -62,6 +62,7 @@
     iscoroutinefunction,
     warn_on_duration,
     LRU,
+    TimeoutError,
 )
 from .utils_comm import pack_data, gather_from_workers, retry_operation
 from .utils_perf import ThrottledGC, enable_gc_diagnosis, disable_gc_diagnosis
@@ -845,7 +846,7 @@ async def _register_with_scheduler(self):
             except EnvironmentError:
                 logger.info("Waiting to connect to: %26s", self.scheduler.address)
                 await asyncio.sleep(0.1)
-            except gen.TimeoutError:
+            except TimeoutError:
                 logger.info("Timed out when connecting to scheduler")
         if response["status"] != "OK":
             raise ValueError("Unexpected response from register: %r" % (response,))
@@ -1095,7 +1096,7 @@ async def close(
 
             for pc in self.periodic_callbacks.values():
                 pc.stop()
-            with ignoring(EnvironmentError, asyncio.TimeoutError):
+            with ignoring(EnvironmentError, TimeoutError):
                 if report and self.contact_address is not None:
                     await asyncio.wait_for(
                         self.scheduler.unregister(
@@ -1113,7 +1114,7 @@ async def close(
                 self.batched_stream.send({"op": "close-stream"})
 
             if self.batched_stream:
-                with ignoring(gen.TimeoutError):
+                with ignoring(TimeoutError):
                     await self.batched_stream.close(timedelta(seconds=timeout))
 
             self.actor_executor._work_queue.queue.clear()

From a891a8316a2f28eeea77145a125e56570e51bc39 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 21 Jan 2020 18:40:22 -0800
Subject: [PATCH 0638/1550] Support args and kwargs in offload (#3392)

---
 distributed/tests/test_utils.py | 7 +++++++
 distributed/utils.py            | 2 +-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/distributed/tests/test_utils.py b/distributed/tests/test_utils.py
index 93b843358a8..e162b9fc2e1 100644
--- a/distributed/tests/test_utils.py
+++ b/distributed/tests/test_utils.py
@@ -43,6 +43,7 @@
     warn_on_duration,
     format_dashboard_link,
     LRU,
+    offload,
     TimeoutError,
 )
 from distributed.utils_test import loop, loop_in_thread  # noqa: F401
@@ -618,3 +619,9 @@ def test_lru():
     l["d"] = 4
     assert len(l) == 3
     assert list(l.keys()) == ["c", "a", "d"]
+
+
+@pytest.mark.asyncio
+async def test_offload():
+    assert (await offload(inc, 1)) == 2
+    assert (await offload(lambda x, y: x + y, 1, y=2)) == 3
diff --git a/distributed/utils.py b/distributed/utils.py
index ee9d7948a11..c2e32d849c4 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -1394,7 +1394,7 @@ def import_term(name: str):
 
 async def offload(fn, *args, **kwargs):
     loop = asyncio.get_event_loop()
-    return await loop.run_in_executor(_offload_executor, fn, *args, **kwargs)
+    return await loop.run_in_executor(_offload_executor, lambda: fn(*args, **kwargs))
 
 
 def serialize_for_cli(data):

From 84ee20548892c8d6dd3e9d0f44712392d9a66e04 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Thu, 23 Jan 2020 10:01:39 -0600
Subject: [PATCH 0639/1550] Add GitHub actions badge to README for windows
 build (#3403)

---
 README.rst | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/README.rst b/README.rst
index 3d9c02915fc..eb4c601bc8b 100644
--- a/README.rst
+++ b/README.rst
@@ -1,13 +1,15 @@
 Distributed
 ===========
 
-|Build Status| |Doc Status| |Gitter| |Version Status| |NumFOCUS|
+|Linux Build Status| |Windows Build Status| |Doc Status| |Gitter| |Version Status| |NumFOCUS|
 
 A library for distributed computation.  See documentation_ for more details.
 
 .. _documentation: https://distributed.dask.org
-.. |Build Status| image:: https://travis-ci.org/dask/distributed.svg?branch=master
+.. |Linux Build Status| image:: https://travis-ci.org/dask/distributed.svg?branch=master
    :target: https://travis-ci.org/dask/distributed
+.. |Windows Build Status| image:: https://github.com/dask/distributed/workflows/Windows%20CI/badge.svg?branch=master
+   :target: https://github.com/dask/distributed/actions?query=workflow%3A%22Windows+CI%22
 .. |Doc Status| image:: https://readthedocs.org/projects/distributed/badge/?version=latest
    :target: https://distributed.dask.org
    :alt: Documentation Status

From 036bcba0c1fa1e682f3b0d8a7b8c4dfccb7cf3e7 Mon Sep 17 00:00:00 2001
From: Darren Weber <dweber.consulting@gmail.com>
Date: Fri, 24 Jan 2020 11:29:24 -0800
Subject: [PATCH 0640/1550] Revise develop-docs: functional test example
 (#3398)

---
 docs/source/develop.rst | 35 ++++++++++++++++++++---------------
 1 file changed, 20 insertions(+), 15 deletions(-)

diff --git a/docs/source/develop.rst b/docs/source/develop.rst
index 9b3b70afb15..f7a10c64471 100644
--- a/docs/source/develop.rst
+++ b/docs/source/develop.rst
@@ -78,28 +78,33 @@ The test suite contains three kinds of tests
     These are rare and mostly for testing the command line interface.
 
 If you are comfortable with the Tornado interface then you will be happiest
-using the ``@gen_cluster`` style of test
+using the ``@gen_cluster`` style of test, e.g.
 
 .. code-block:: python
 
-   from distributed.utils_test import gen_cluster
+    # tests/test_submit.py
 
-   @gen_cluster(client=True)
-   def test_submit(c, s, a, b):
-       assert isinstance(c, Client)
-       assert isinstance(s, Scheduler)
-       assert isinstance(a, Worker)
-       assert isinstance(b, Worker)
+    from distributed.utils_test import gen_cluster, inc
+    from distributed import Client, Future, Scheduler, Worker
 
-       future = c.submit(inc, 1)
-       assert future.key in c.futures
+    @gen_cluster(client=True)
+    def test_submit(c, s, a, b):
+        assert isinstance(c, Client)
+        assert isinstance(s, Scheduler)
+        assert isinstance(a, Worker)
+        assert isinstance(b, Worker)
 
-       # result = future.result()  # This synchronous API call would block
-       result = yield future
-       assert result == 2
+        future = c.submit(inc, 1)
+        assert isinstance(future, Future)
+        assert future.key in c.futures
+
+        # result = future.result()  # This synchronous API call would block
+        result = yield future
+        assert result == 2
+
+        assert future.key in s.tasks
+        assert future.key in a.data or future.key in b.data
 
-       assert future.key in s.tasks
-       assert future.key in a.data or future.key in b.data
 
 The ``@gen_cluster`` decorator sets up a scheduler, client, and workers for
 you and cleans them up after the test.  It also allows you to directly inspect

From fe20eaaee60cfe5e4a8ad66dd733927a751985aa Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Fri, 24 Jan 2020 16:58:16 -0600
Subject: [PATCH 0641/1550] Use instance-level client instead of class-level
 (#3408)

---
 distributed/comm/tcp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py
index 77876a04fbc..ce6b7fa0b44 100644
--- a/distributed/comm/tcp.py
+++ b/distributed/comm/tcp.py
@@ -345,7 +345,7 @@ async def connect(self, address, deserialize=True, **connection_args):
         kwargs = self._get_connect_args(**connection_args)
 
         try:
-            stream = await BaseTCPConnector.client.connect(
+            stream = await self.client.connect(
                 ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs
             )
 

From 4081a0e09febd9099f04a1db9772f6cd95203b2e Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Fri, 24 Jan 2020 21:58:21 -0800
Subject: [PATCH 0642/1550] Enable WorkStealing case-by-case (#3410)

* Enable work-stealing dynamically

* Include test for work-stealing config

* Rewrite test to use `@pytest.mark.asyncio`
---
 distributed/scheduler.py            |  7 +++----
 distributed/tests/test_scheduler.py | 13 +++++++++++++
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 8080a1186da..ab2e615cef2 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -89,9 +89,6 @@
     PubSubSchedulerExtension,
 ]
 
-if dask.config.get("distributed.scheduler.work-stealing"):
-    DEFAULT_EXTENSIONS.append(WorkStealing)
-
 ALL_TASK_STATES = {"released", "waiting", "no-worker", "processing", "erred", "memory"}
 
 
@@ -1333,7 +1330,9 @@ def __init__(
             self.periodic_callbacks["idle-timeout"] = pc
 
         if extensions is None:
-            extensions = DEFAULT_EXTENSIONS
+            extensions = list(DEFAULT_EXTENSIONS)
+            if dask.config.get("distributed.scheduler.work-stealing"):
+                extensions.append(WorkStealing)
         for ext in extensions:
             ext(self)
 
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index fd0775ad003..3ce681ea546 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -596,6 +596,19 @@ def test_coerce_address():
         yield [w.close() for w in [a, b, c]]
 
 
+@pytest.mark.asyncio
+async def test_config_stealing(cleanup):
+    # Regression test for https://github.com/dask/distributed/issues/3409
+
+    with dask.config.set({"distributed.scheduler.work-stealing": True}):
+        async with Scheduler(port=0) as s:
+            assert "stealing" in s.extensions
+
+    with dask.config.set({"distributed.scheduler.work-stealing": False}):
+        async with Scheduler(port=0) as s:
+            assert "stealing" not in s.extensions
+
+
 @pytest.mark.skipif(
     sys.platform.startswith("win"), reason="file descriptors not really a thing"
 )

From 4f61b3d341c5098dfb9267a2a8fc4a6e74483dd4 Mon Sep 17 00:00:00 2001
From: Chrysostomos Nanakos <chris@include.gr>
Date: Mon, 27 Jan 2020 18:57:05 +0200
Subject: [PATCH 0643/1550] Respect dashboard prefix when redirecting root
 (#3387)

When --dashboard-prefix is used root location is always redirected to
/status without adding the prefix.

Fixes https://github.com/dask/distributed/issues/3405

Signed-off-by: Chrysostomos Nanakos <chris@include.gr>
---
 distributed/dashboard/core.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/distributed/dashboard/core.py b/distributed/dashboard/core.py
index 41e7c289c17..82cfe92da17 100644
--- a/distributed/dashboard/core.py
+++ b/distributed/dashboard/core.py
@@ -5,6 +5,7 @@
 import bokeh
 from bokeh.server.server import Server
 from tornado import web
+from urllib.parse import urljoin
 
 
 if LooseVersion(bokeh.__version__) < LooseVersion("0.13.0"):
@@ -34,7 +35,13 @@ def listen(self, addr):
                     check_unused_sessions_milliseconds=500,
                     allow_websocket_origin=["*"],
                     use_index=False,
-                    extra_patterns=[(r"/", web.RedirectHandler, {"url": "/status"})],
+                    extra_patterns=[
+                        (
+                            r"/",
+                            web.RedirectHandler,
+                            {"url": urljoin(self.prefix.rstrip("/") + "/", r"status")},
+                        )
+                    ],
                 )
                 server_kwargs.update(self.server_kwargs)
                 self.server = Server(self.apps, **server_kwargs)

From 77ffa7b78a1096ce67a0366abffeaed071ed02fa Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Mon, 27 Jan 2020 13:32:36 -0800
Subject: [PATCH 0644/1550] Drop custom cuDF serialization (#3404)

* Drop custom cuDF serialization

This is since handled in cuDF 0.9.0, which was released a while ago. So
go ahead and drop this from Distributed.

* Restore cuDF import of serializers
---
 distributed/protocol/__init__.py |  7 +------
 distributed/protocol/cudf.py     | 23 -----------------------
 2 files changed, 1 insertion(+), 29 deletions(-)
 delete mode 100644 distributed/protocol/cudf.py

diff --git a/distributed/protocol/__init__.py b/distributed/protocol/__init__.py
index ef8b5564bbb..30ae3935498 100644
--- a/distributed/protocol/__init__.py
+++ b/distributed/protocol/__init__.py
@@ -83,9 +83,4 @@ def _register_numba():
 @cuda_serialize.register_lazy("cudf")
 @cuda_deserialize.register_lazy("cudf")
 def _register_cudf():
-    import cudf
-
-    if LooseVersion(cudf.__version__) > "0.9":
-        from cudf.comm import serialize
-    else:
-        from . import cudf
+    from cudf.comm import serialize
diff --git a/distributed/protocol/cudf.py b/distributed/protocol/cudf.py
deleted file mode 100644
index f236a6c1f0c..00000000000
--- a/distributed/protocol/cudf.py
+++ /dev/null
@@ -1,23 +0,0 @@
-import pickle
-import cudf
-import cudf.groupby.groupby
-from .cuda import cuda_serialize, cuda_deserialize
-from ..utils import log_errors
-
-
-# all (de-)serializtion code lives in the cudf codebase
-# here we ammend the returned headers with `is_gpu` for
-# UCX buffer consumption
-@cuda_serialize.register((cudf.DataFrame, cudf.Series, cudf.groupby.groupby._Groupby))
-def serialize_cudf_dataframe(x):
-    with log_errors():
-        header, frames = x.serialize()
-        return header, frames
-
-
-@cuda_deserialize.register((cudf.DataFrame, cudf.Series, cudf.groupby.groupby._Groupby))
-def deserialize_cudf_dataframe(header, frames):
-    with log_errors():
-        cudf_typ = pickle.loads(header["type"])
-        cudf_obj = cudf_typ.deserialize(header, frames)
-        return cudf_obj

From 241d0d44d8a108bf2412696455dd7486bdfbff17 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Mon, 27 Jan 2020 16:25:03 -0600
Subject: [PATCH 0645/1550] Add CI documentation build (#3411)

---
 .github/workflows/ci-docs.yaml | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)
 create mode 100644 .github/workflows/ci-docs.yaml

diff --git a/.github/workflows/ci-docs.yaml b/.github/workflows/ci-docs.yaml
new file mode 100644
index 00000000000..e80e07b9c33
--- /dev/null
+++ b/.github/workflows/ci-docs.yaml
@@ -0,0 +1,28 @@
+name: Documentation CI
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v1
+
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.7
+
+      - name: Install Distributed
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e .
+
+      - name: Install doc dependencies
+        run: pip install -r docs/requirements.txt
+
+      - name: Build docs
+        run: |
+            cd docs
+            make html

From 6cf1afe012435216bb5a1f5e6fa3f9c8258dde00 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Mon, 27 Jan 2020 19:06:18 -0600
Subject: [PATCH 0646/1550] Ignore no-worker state in TaskProgress (#3407)

---
 distributed/dashboard/components/scheduler.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index 8eec6b8b772..2c9953e97e3 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -1418,12 +1418,13 @@ def update(self):
             }
 
             for tp in self.scheduler.task_prefixes.values():
-                if any(tp.active_states.values()):
-                    state["memory"][tp.name] = tp.active_states["memory"]
-                    state["erred"][tp.name] = tp.active_states["erred"]
-                    state["released"][tp.name] = tp.active_states["released"]
-                    state["processing"][tp.name] = tp.active_states["processing"]
-                    state["waiting"][tp.name] = tp.active_states["waiting"]
+                active_states = tp.active_states
+                if any(active_states.get(s) for s in state.keys()):
+                    state["memory"][tp.name] = active_states["memory"]
+                    state["erred"][tp.name] = active_states["erred"]
+                    state["released"][tp.name] = active_states["released"]
+                    state["processing"][tp.name] = active_states["processing"]
+                    state["waiting"][tp.name] = active_states["waiting"]
 
             state["all"] = {
                 k: sum(v[k] for v in state.values()) for k in state["memory"]

From 457281ba826df106737ed96c9327124cc85f6d29 Mon Sep 17 00:00:00 2001
From: Tom Augspurger <TomAugspurger@users.noreply.github.com>
Date: Tue, 28 Jan 2020 12:29:29 -0600
Subject: [PATCH 0647/1550] DOC: Update changelog for 2.10.0 (#3421)

[ci skip]
---
 docs/source/changelog.rst | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index 1bb19e70330..cd311309a30 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,16 @@
 Changelog
 =========
 
+2.10.0 - 2020-01-28
+-------------------
+
+- Fixed ``ZeroDivisionError`` in dashboard when no workers were present (:pr:`3407`) `James Bourbeau`_
+- Respect the ``dashboard-prefix`` when redirecting from the root (:pr:`3387`) `Chrysostomos Nanakos`_
+- Allow enabling / disabling work-stealing after the cluster has started (:pr:`3410`) `John Kirkham`_
+- Support ``*args`` and ``**kwargs`` in offload (:pr:`3392`) `Matthew Rocklin`_
+- Add lifecycle hooks to SchedulerPlugin (:pr:`3391`) `Matthew Rocklin`_
+
+
 2.9.3 - 2020-01-17
 ------------------
 
@@ -1501,3 +1511,4 @@ significantly without many new features.
 .. _`Benedikt Reinartz`: https://github.com/filmor
 .. _`Markus Mohrhard`: https://github.com/mmohrhard
 .. _`Mana Borwornpadungkitti`: https://github.com/potpath
+.. _`Chrysostomos Nanakos`: https://github.com/cnanakos

From 6083181cdcf4a8bc23697910d6d7f5712bfd8b47 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Wed, 29 Jan 2020 08:55:30 -0600
Subject: [PATCH 0648/1550] Add Mac OS build to CI (#3358)

---
 .travis.yml | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 56c2588ff5f..c0c30316c9a 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,4 +1,4 @@
-language: python
+language: generic
 # sudo shouldn't be required, but currently tests fail when run in a container
 # on travis instead of a vm. See https://github.com/dask/distributed/pull/1563.
 sudo: required
@@ -13,8 +13,15 @@ matrix:
   fast_finish: true
   include:
   - os: linux
+    language: python
     python: 3.6
     env: LINT=true
+  - os: osx
+    env: PYTHON=3.7 TESTS=true PACKAGES="scikit-learn python-snappy python-blosc" TORNADO=6
+    if: type != pull_request OR commit_message =~ test-osx  # Skip on PRs unless the commit message contains "test-osx"
+
+  allow_failures:
+    - os: osx
 
 install:
   - if [[ $TESTS == true ]]; then source continuous_integration/travis/install.sh ; fi

From fecf588c800f3230f08fe63d3f4f57360c3dd7cf Mon Sep 17 00:00:00 2001
From: Igor Gotlibovych <igor.gotlibovych@gmail.com>
Date: Wed, 29 Jan 2020 23:02:39 +0000
Subject: [PATCH 0649/1550] Support version checking with older versions of
 Dask (#3390)

The test fails of one of the components' version info is None (we treat it as UNKNOWN). In addition, the function fails to report if the `client` is missing a package.
---
 distributed/tests/test_versions.py | 119 +++++++++++++++++++++++++++++
 distributed/versions.py            |  61 +++++++++------
 2 files changed, 155 insertions(+), 25 deletions(-)
 create mode 100644 distributed/tests/test_versions.py

diff --git a/distributed/tests/test_versions.py b/distributed/tests/test_versions.py
new file mode 100644
index 00000000000..25087df795a
--- /dev/null
+++ b/distributed/tests/test_versions.py
@@ -0,0 +1,119 @@
+import re
+
+import pytest
+
+from distributed.versions import get_versions, error_message
+from distributed import Client, Worker
+from distributed.utils_test import gen_cluster
+
+
+# if one of the nodes reports this version, there's a mismatch
+mismatched_version = get_versions()
+mismatched_version["packages"]["distributed"] = "0.0.0.dev0"
+
+# for really old versions, the `package` key is missing - version is UNKNOWN
+key_err_version = {}
+
+# if no key is available for one package, we assume it's MISSING
+missing_version = get_versions()
+del missing_version["packages"]["distributed"]
+
+# if a node doesn't report any version info, we treat them as UNKNOWN
+# the happens if the node is pre-32cb96e, i.e. <=2.9.1
+unknown_version = None
+
+
+@pytest.fixture
+def kwargs_matching():
+    return dict(
+        scheduler=get_versions(),
+        workers={f"worker-{i}": get_versions() for i in range(3)},
+        client=get_versions(),
+    )
+
+
+def test_versions_match(kwargs_matching):
+    assert error_message(**kwargs_matching) == ""
+
+
+@pytest.fixture(params=["client", "scheduler", "worker-1"])
+def node(request):
+    """Node affected by version mismatch."""
+    return request.param
+
+
+@pytest.fixture(params=["MISMATCHED", "MISSING", "KEY_ERROR", "NONE"])
+def effect(request):
+    """Specify type of mismatch."""
+    return request.param
+
+
+@pytest.fixture
+def kwargs_not_matching(kwargs_matching, node, effect):
+    affected_version = {
+        "MISMATCHED": mismatched_version,
+        "MISSING": missing_version,
+        "KEY_ERROR": key_err_version,
+        "NONE": unknown_version,
+    }[effect]
+    kwargs = kwargs_matching
+    if node in kwargs["workers"]:
+        kwargs["workers"][node] = affected_version
+    else:
+        kwargs[node] = affected_version
+    return kwargs
+
+
+@pytest.fixture
+def pattern(effect):
+    """Pattern to match in the right-hand column."""
+    return {
+        "MISMATCHED": r"0\.0\.0\.dev0",
+        "MISSING": "MISSING",
+        "KEY_ERROR": "UNKNOWN",
+        "NONE": "UNKNOWN",
+    }[effect]
+
+
+def test_version_mismatch(node, effect, kwargs_not_matching, pattern):
+    msg = error_message(**kwargs_not_matching)
+
+    assert "Mismatched versions found" in msg
+    assert "distributed" in msg
+    assert re.search(node + r"\s+\|\s+" + pattern, msg)
+
+
+def test_scheduler_mismatched_irrelevant_package(kwargs_matching):
+    """An irrelevant package on the scheduler can have any version."""
+    kwargs_matching["scheduler"]["packages"]["numpy"] = "0.0.0"
+    assert "numpy" in kwargs_matching["client"]["packages"]
+
+    assert error_message(**kwargs_matching) == ""
+
+
+def test_scheduler_additional_irrelevant_package(kwargs_matching):
+    """An irrelevant package on the scheduler does not need to be present elsewhere."""
+    kwargs_matching["scheduler"]["packages"]["pyspark"] = "0.0.0"
+
+    assert error_message(**kwargs_matching) == ""
+
+
+@gen_cluster()
+async def test_version_warning_in_cluster(s, a, b):
+    s.workers[a.address].versions["packages"]["dask"] = "0.0.0"
+
+    with pytest.warns(None) as record:
+        async with Client(s.address, asynchronous=True) as client:
+            pass
+
+    assert record
+    assert any("dask" in str(r.message) for r in record)
+    assert any("0.0.0" in str(r.message) for r in record)
+    assert any(a.address in str(r.message) for r in record)
+
+    async with Worker(s.address) as w:
+        assert any("This Worker" in line.message for line in w.logs)
+        assert any("dask" in line.message for line in w.logs)
+        assert any(
+            "0.0.0" in line.message and a.address in line.message for line in w.logs
+        )
diff --git a/distributed/versions.py b/distributed/versions.py
index 0b97a6f7ac0..a7022c830f7 100644
--- a/distributed/versions.py
+++ b/distributed/versions.py
@@ -26,9 +26,15 @@
 ]
 
 
+# only these scheduler packages will be checked for version mismatch
+scheduler_relevant_packages = set(pkg for pkg, _ in required_packages) | set(
+    ["lz4", "blosc"]
+)
+
+
 def get_versions(packages=None):
     """
-    Return basic information on our software installation, and out installed versions of packages.
+    Return basic information on our software installation, and our installed versions of packages.
     """
     if packages is None:
         packages = []
@@ -98,34 +104,39 @@ def get_package_info(pkgs):
 
 
 def error_message(scheduler, workers, client, client_name="client"):
-    # we care about the required & optional packages matching
-    try:
-        client_versions = client["packages"]
-        versions = [("scheduler", scheduler["packages"])]
-        versions.extend((w, d["packages"]) for w, d in sorted(workers.items()))
-    except KeyError:
-        return (
-            "Version mismatch for dask.distributed. "
-            "The scheduler has version >= 1.28.0 "
-            "but some other component is less than this"
-        )
+    from .utils import asciitable
 
-    mismatched = defaultdict(list)
-    for name, vers in versions:
-        for pkg, cv in client_versions.items():
-            v = vers.get(pkg, "MISSING")
-            if cv != v:
-                mismatched[pkg].append((name, v))
+    nodes = {**{client_name: client}, **{"scheduler": scheduler}, **workers}
 
-    if mismatched:
-        from .utils import asciitable
+    # Hold all versions, e.g. versions["scheduler"]["distributed"] = 2.9.3
+    node_packages = defaultdict(dict)
 
-        errs = []
-        for pkg, versions in sorted(mismatched.items()):
-            rows = [(client_name, client_versions[pkg])]
-            rows.extend(versions)
-            errs.append("%s\n%s" % (pkg, asciitable(["", "version"], rows)))
+    # Collect all package versions
+    packages = set()
 
+    for node, info in nodes.items():
+        if info is None or not (isinstance(info, dict)) or "packages" not in info:
+            node_packages[node] = defaultdict(lambda: "UNKNOWN")
+        else:
+            node_packages[node] = defaultdict(lambda: "MISSING")
+            for pkg, version in info["packages"].items():
+                node_packages[node][pkg] = version
+                packages.add(pkg)
+
+    errs = []
+    for pkg in sorted(packages):
+        versions = set(
+            node_packages[node][pkg]
+            for node in nodes
+            if node != "scheduler" or pkg in scheduler_relevant_packages
+        )
+        if len(versions) <= 1:
+            continue
+        rows = [
+            (node_name, node_packages[node_name][pkg]) for node_name in nodes.keys()
+        ]
+        errs.append("%s\n%s" % (pkg, asciitable(["", "version"], rows)))
+    if errs:
         return "Mismatched versions found\n" "\n" "%s" % ("\n\n".join(errs))
     else:
         return ""

From 8eadf5e8fa56673aef9063fa91c52f4715638b6d Mon Sep 17 00:00:00 2001
From: Pierre Glaser <pierreglaser@msn.com>
Date: Thu, 30 Jan 2020 01:09:32 +0100
Subject: [PATCH 0650/1550] Make _get_ip return an IP address when defaulting
 (#3418)

* default to fully-qualified domain names in _get_ip
* make _get_ip return an IP address (not a hostname)
---
 distributed/utils.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/distributed/utils.py b/distributed/utils.py
index c2e32d849c4..efc948cff19 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -138,13 +138,15 @@ def _get_ip(host, port, family):
         ip = sock.getsockname()[0]
         return ip
     except EnvironmentError as e:
-        # XXX Should first try getaddrinfo() on socket.gethostname() and getfqdn()
         warnings.warn(
             "Couldn't detect a suitable IP address for "
             "reaching %r, defaulting to hostname: %s" % (host, e),
             RuntimeWarning,
         )
-        return socket.gethostname()
+        addr_info = socket.getaddrinfo(
+            socket.gethostname(), port, family, socket.SOCK_DGRAM, socket.IPPROTO_UDP
+        )[0]
+        return addr_info[4][0]
     finally:
         sock.close()
 

From 3e7bbbdb47c88b656e08ceab70e65a1877a7bee4 Mon Sep 17 00:00:00 2001
From: fjetter <fjetter@users.noreply.github.com>
Date: Thu, 30 Jan 2020 17:28:22 +0100
Subject: [PATCH 0651/1550] Allow memory monitor to evict data more
 aggressively (#3424)

---
 distributed/worker.py | 59 +++++++++++++++++++++++++++----------------
 1 file changed, 37 insertions(+), 22 deletions(-)

diff --git a/distributed/worker.py b/distributed/worker.py
index 7062aba5b87..e429cb75a4b 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -2584,36 +2584,44 @@ async def memory_monitor(self):
         memory = proc.memory_info().rss
         frac = memory / self.memory_limit
 
-        # Pause worker threads if above 80% memory use
-        if self.memory_pause_fraction and frac > self.memory_pause_fraction:
-            # Try to free some memory while in paused state
-            self._throttled_gc.collect()
-            if not self.paused:
+        def check_pause(memory):
+            frac = memory / self.memory_limit
+            # Pause worker threads if above 80% memory use
+            if self.memory_pause_fraction and frac > self.memory_pause_fraction:
+                # Try to free some memory while in paused state
+                self._throttled_gc.collect()
+                if not self.paused:
+                    logger.warning(
+                        "Worker is at %d%% memory usage. Pausing worker.  "
+                        "Process memory: %s -- Worker memory limit: %s",
+                        int(frac * 100),
+                        format_bytes(memory),
+                        format_bytes(self.memory_limit)
+                        if self.memory_limit is not None
+                        else "None",
+                    )
+                    self.paused = True
+            elif self.paused:
                 logger.warning(
-                    "Worker is at %d%% memory usage. Pausing worker.  "
+                    "Worker is at %d%% memory usage. Resuming worker. "
                     "Process memory: %s -- Worker memory limit: %s",
                     int(frac * 100),
-                    format_bytes(proc.memory_info().rss),
+                    format_bytes(memory),
                     format_bytes(self.memory_limit)
                     if self.memory_limit is not None
                     else "None",
                 )
-                self.paused = True
-        elif self.paused:
-            logger.warning(
-                "Worker is at %d%% memory usage. Resuming worker. "
-                "Process memory: %s -- Worker memory limit: %s",
-                int(frac * 100),
-                format_bytes(proc.memory_info().rss),
-                format_bytes(self.memory_limit)
-                if self.memory_limit is not None
-                else "None",
-            )
-            self.paused = False
-            self.ensure_computing()
+                self.paused = False
+                self.ensure_computing()
 
+        check_pause(memory)
         # Dump data to disk if above 70%
         if self.memory_spill_fraction and frac > self.memory_spill_fraction:
+            logger.debug(
+                "Worker is at %d%% memory usage. Start spilling data to disk.",
+                int(frac * 100),
+            )
+            start = time()
             target = self.memory_limit * self.memory_target_fraction
             count = 0
             need = memory - target
@@ -2624,7 +2632,7 @@ async def memory_monitor(self):
                         "to store to disk.  Perhaps some other process "
                         "is leaking memory?  Process memory: %s -- "
                         "Worker memory limit: %s",
-                        format_bytes(proc.memory_info().rss),
+                        format_bytes(memory),
                         format_bytes(self.memory_limit)
                         if self.memory_limit is not None
                         else "None",
@@ -2634,7 +2642,13 @@ async def memory_monitor(self):
                 del k, v
                 total += weight
                 count += 1
-                await asyncio.sleep(0)
+                # If the current buffer is filled with a lot of small values,
+                # evicting one at a time is very slow and the worker might
+                # generate new data faster than it is able to evict. Therefore,
+                # only pass on control if we spent at least 0.5s evicting
+                if time() - start > 0.5:
+                    await asyncio.sleep(0)
+                    start = time()
                 memory = proc.memory_info().rss
                 if total > need and memory > target:
                     # Issue a GC to ensure that the evicted data is actually
@@ -2642,6 +2656,7 @@ async def memory_monitor(self):
                     # before trying to evict even more data.
                     self._throttled_gc.collect()
                     memory = proc.memory_info().rss
+            check_pause(memory)
             if count:
                 logger.debug(
                     "Moved %d pieces of data data and %s to disk",

From a298fdaceca133b414b8182cb5e2e331fab06585 Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Thu, 30 Jan 2020 17:36:58 +0000
Subject: [PATCH 0652/1550] Add dashboard_link property to Client (#3429)

---
 distributed/client.py            | 74 ++++++++++++++++++--------------
 distributed/tests/test_client.py |  3 +-
 2 files changed, 43 insertions(+), 34 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 7a39ec4b235..5bde92a4bb1 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -752,6 +752,22 @@ def asynchronous(self):
         """
         return self._asynchronous and self.loop is IOLoop.current()
 
+    @property
+    def dashboard_link(self):
+        scheduler, info = self._get_scheduler_info()
+        try:
+            return self.cluster.dashboard_link
+        except AttributeError:
+            protocol, rest = scheduler.address.split("://")
+
+            port = info["services"]["dashboard"]
+            if protocol == "inproc":
+                host = "localhost"
+            else:
+                host = rest.split(":")[0]
+
+            return format_dashboard_link(host, port)
+
     def sync(self, func, *args, asynchronous=None, callback_timeout=None, **kwargs):
         if (
             asynchronous
@@ -767,6 +783,29 @@ def sync(self, func, *args, asynchronous=None, callback_timeout=None, **kwargs):
                 self.loop, func, *args, callback_timeout=callback_timeout, **kwargs
             )
 
+    def _get_scheduler_info(self):
+        from .scheduler import Scheduler
+
+        if (
+            self.cluster
+            and hasattr(self.cluster, "scheduler")
+            and isinstance(self.cluster.scheduler, Scheduler)
+        ):
+            info = self.cluster.scheduler.identity()
+            scheduler = self.cluster.scheduler
+        elif (
+            self._loop_runner.is_started()
+            and self.scheduler
+            and not (self.asynchronous and self.loop is IOLoop.current())
+        ):
+            info = sync(self.loop, self.scheduler.identity)
+            scheduler = self.scheduler
+        else:
+            info = self._scheduler_identity
+            scheduler = self.scheduler
+
+        return scheduler, info
+
     def __repr__(self):
         # Note: avoid doing I/O here...
         info = self._scheduler_identity
@@ -796,25 +835,7 @@ def __repr__(self):
             return "<%s: not connected>" % (self.__class__.__name__,)
 
     def _repr_html_(self):
-        from .scheduler import Scheduler
-
-        if (
-            self.cluster
-            and hasattr(self.cluster, "scheduler")
-            and isinstance(self.cluster.scheduler, Scheduler)
-        ):
-            info = self.cluster.scheduler.identity()
-            scheduler = self.cluster.scheduler
-        elif (
-            self._loop_runner.is_started()
-            and self.scheduler
-            and not (self.asynchronous and self.loop is IOLoop.current())
-        ):
-            info = sync(self.loop, self.scheduler.identity)
-            scheduler = self.scheduler
-        else:
-            info = self._scheduler_identity
-            scheduler = self.scheduler
+        scheduler, info = self._get_scheduler_info()
 
         text = (
             '<h3 style="text-align: left;">Client</h3>\n'
@@ -826,22 +847,9 @@ def _repr_html_(self):
             text += "  <li><b>Scheduler: not connected</b></li>\n"
 
         if info and "dashboard" in info["services"]:
-            try:
-                address = self.cluster.dashboard_link
-            except AttributeError:
-                protocol, rest = scheduler.address.split("://")
-
-                port = info["services"]["dashboard"]
-                if protocol == "inproc":
-                    host = "localhost"
-                else:
-                    host = rest.split(":")[0]
-
-                address = format_dashboard_link(host, port)
-
             text += (
                 "  <li><b>Dashboard: </b><a href='%(web)s' target='_blank'>%(web)s</a>\n"
-                % {"web": address}
+                % {"web": self.dashboard_link}
             )
 
         text += "</ul>\n"
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 77180eefa35..a205452b2f8 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -5249,8 +5249,9 @@ def test_dashboard_link(loop, monkeypatch):
             with dask.config.set(
                 {"distributed.dashboard.link": "{scheme}://foo-{USER}:{port}/status"}
             ):
-                text = c._repr_html_()
                 link = "http://foo-myusername:12355/status"
+                assert link == c.dashboard_link
+                text = c._repr_html_()
                 assert link in text
 
 

From 12a4f2d38f131ddb63849d1be0805f414fa6d816 Mon Sep 17 00:00:00 2001
From: Anderson Banihirwe <axbanihirwe@ualr.edu>
Date: Fri, 31 Jan 2020 12:48:36 -0700
Subject: [PATCH 0653/1550] Remove object from class hierarchy (#3432)

---
 distributed/_concurrent_futures_thread.py     |  2 +-
 distributed/actor.py                          |  4 ++--
 distributed/batched.py                        |  2 +-
 distributed/client.py                         |  6 +++---
 distributed/comm/inproc.py                    |  4 ++--
 distributed/comm/tcp.py                       |  2 +-
 distributed/comm/tests/test_comms.py          |  2 +-
 distributed/core.py                           |  8 ++++----
 distributed/counter.py                        |  4 ++--
 distributed/dashboard/components/__init__.py  |  2 +-
 distributed/dashboard/core.py                 |  2 +-
 distributed/dashboard/scheduler.py            |  2 +-
 distributed/dashboard/worker.py               |  2 +-
 distributed/deploy/cluster.py                 |  2 +-
 distributed/deploy/old_ssh.py                 |  2 +-
 distributed/deploy/spec.py                    |  2 +-
 .../deploy/tests/test_slow_adaptive.py        |  2 +-
 distributed/deploy/utils_test.py              |  2 +-
 distributed/diagnostics/plugin.py             |  4 ++--
 distributed/diagnostics/progressbar.py        |  4 ++--
 distributed/diskutils.py                      |  4 ++--
 distributed/lock.py                           |  4 ++--
 distributed/locket.py                         |  8 ++++----
 distributed/metrics.py                        |  2 +-
 distributed/nanny.py                          |  2 +-
 distributed/node.py                           |  2 +-
 distributed/process.py                        |  4 ++--
 distributed/protocol/cupy.py                  |  2 +-
 distributed/protocol/serialize.py             |  6 +++---
 distributed/protocol/tests/test_serialize.py  | 10 +++++-----
 distributed/publish.py                        |  2 +-
 distributed/pubsub.py                         | 10 +++++-----
 distributed/pytest_resourceleaks.py           |  4 ++--
 distributed/queues.py                         |  4 ++--
 distributed/recreate_exceptions.py            |  4 ++--
 distributed/scheduler.py                      | 10 +++++-----
 distributed/security.py                       |  2 +-
 distributed/system_monitor.py                 |  2 +-
 distributed/tests/test_actor.py               | 20 +++++++++----------
 distributed/tests/test_batched.py             |  2 +-
 distributed/tests/test_client.py              | 14 ++++++-------
 distributed/tests/test_core.py                |  2 +-
 distributed/tests/test_steal.py               |  2 +-
 distributed/tests/test_utils_perf.py          |  2 +-
 distributed/tests/test_worker.py              | 10 +++++-----
 distributed/utils.py                          |  4 ++--
 distributed/utils_comm.py                     |  2 +-
 distributed/utils_perf.py                     |  6 +++---
 distributed/utils_test.py                     |  2 +-
 distributed/variable.py                       |  4 ++--
 distributed/worker.py                         |  2 +-
 docs/source/actors.rst                        |  2 +-
 docs/source/adaptive.rst                      |  4 ++--
 docs/source/serialization.rst                 |  2 +-
 54 files changed, 111 insertions(+), 111 deletions(-)

diff --git a/distributed/_concurrent_futures_thread.py b/distributed/_concurrent_futures_thread.py
index 02ff7c649aa..b26da12cb7a 100644
--- a/distributed/_concurrent_futures_thread.py
+++ b/distributed/_concurrent_futures_thread.py
@@ -50,7 +50,7 @@ def _python_exit():
 atexit.register(_python_exit)
 
 
-class _WorkItem(object):
+class _WorkItem:
     def __init__(self, future, fn, args, kwargs):
         self.future = future
         self.fn = fn
diff --git a/distributed/actor.py b/distributed/actor.py
index e7e4afaacf0..37f43b69358 100644
--- a/distributed/actor.py
+++ b/distributed/actor.py
@@ -178,7 +178,7 @@ def client(self):
         return self._future.client
 
 
-class ProxyRPC(object):
+class ProxyRPC:
     """
     An rpc-like object that uses the scheduler's rpc to connect to a worker
     """
@@ -196,7 +196,7 @@ async def func(**msg):
         return func
 
 
-class ActorFuture(object):
+class ActorFuture:
     """ Future to an actor's method call
 
     Whenever you call a method on an Actor you get an ActorFuture immediately
diff --git a/distributed/batched.py b/distributed/batched.py
index e066fcf7588..13c241d1e1b 100644
--- a/distributed/batched.py
+++ b/distributed/batched.py
@@ -12,7 +12,7 @@
 logger = logging.getLogger(__name__)
 
 
-class BatchedSend(object):
+class BatchedSend:
     """ Batch messages in batches on a stream
 
     This takes an IOStream and an interval (in ms) and ensures that we send no
diff --git a/distributed/client.py b/distributed/client.py
index 5bde92a4bb1..706101d2fe4 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -412,7 +412,7 @@ def __await__(self):
         return self.result().__await__()
 
 
-class FutureState(object):
+class FutureState:
     """A Future's internal state.
 
     This is shared between all Futures with the same key and client.
@@ -4142,7 +4142,7 @@ async def _first_completed(futures):
     return result
 
 
-class as_completed(object):
+class as_completed:
     """
     Return futures in the order in which they complete
 
@@ -4480,7 +4480,7 @@ def fire_and_forget(obj):
         )
 
 
-class get_task_stream(object):
+class get_task_stream:
     """
     Collect task stream within a context block
 
diff --git a/distributed/comm/inproc.py b/distributed/comm/inproc.py
index e46c2804ed1..c0191f024f6 100644
--- a/distributed/comm/inproc.py
+++ b/distributed/comm/inproc.py
@@ -25,7 +25,7 @@
 )
 
 
-class Manager(object):
+class Manager:
     """
     An object coordinating listeners and their addresses.
     """
@@ -87,7 +87,7 @@ class QueueEmpty(Exception):
     pass
 
 
-class Queue(object):
+class Queue:
     """
     A single-reader, single-writer, non-threadsafe, peekable queue.
     """
diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py
index ce6b7fa0b44..7003053ce06 100644
--- a/distributed/comm/tcp.py
+++ b/distributed/comm/tcp.py
@@ -324,7 +324,7 @@ def _expect_tls_context(connection_args):
     return ctx
 
 
-class RequireEncryptionMixin(object):
+class RequireEncryptionMixin:
     def _check_encryption(self, address, connection_args):
         if not self.encrypted and connection_args.get("require_encryption"):
             # XXX Should we have a dedicated SecurityError class?
diff --git a/distributed/comm/tests/test_comms.py b/distributed/comm/tests/test_comms.py
index 470c667b989..b486912f281 100644
--- a/distributed/comm/tests/test_comms.py
+++ b/distributed/comm/tests/test_comms.py
@@ -1070,7 +1070,7 @@ def _raise_eoferror():
     raise EOFError
 
 
-class _EOFRaising(object):
+class _EOFRaising:
     def __reduce__(self):
         return _raise_eoferror, ()
 
diff --git a/distributed/core.py b/distributed/core.py
index 81cd7adf8e4..3dad1223030 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -60,7 +60,7 @@ def _raise(*args, **kwargs):
 LOG_PDB = dask.config.get("distributed.admin.pdb-on-err")
 
 
-class Server(object):
+class Server:
     """ Dask Distributed Server
 
     Superclass for endpoints in a distributed cluster, such as Worker
@@ -569,7 +569,7 @@ def addr_from_args(addr=None, ip=None, port=None):
     return normalize_address(addr)
 
 
-class rpc(object):
+class rpc:
     """ Conveniently interact with a remote server
 
     >>> remote = rpc(address)  # doctest: +SKIP
@@ -728,7 +728,7 @@ def __repr__(self):
         return "<rpc to %r, %d comms>" % (self.address, len(self.comms))
 
 
-class PooledRPCCall(object):
+class PooledRPCCall:
     """ The result of ConnectionPool()('host:port')
 
     See Also:
@@ -777,7 +777,7 @@ def __repr__(self):
         return "<pooled rpc to %r>" % (self.addr,)
 
 
-class ConnectionPool(object):
+class ConnectionPool:
     """ A maximum sized pool of Comm objects.
 
     This provides a connect method that mirrors the normal distributed.connect
diff --git a/distributed/counter.py b/distributed/counter.py
index f41961e87ac..ebc8cda6104 100644
--- a/distributed/counter.py
+++ b/distributed/counter.py
@@ -11,7 +11,7 @@
     pass
 else:
 
-    class Digest(object):
+    class Digest:
         def __init__(self, loop=None, intervals=(5, 60, 3600)):
             self.intervals = intervals
             self.components = [TDigest() for i in self.intervals]
@@ -39,7 +39,7 @@ def size(self):
             return sum(d.size() for d in self.components)
 
 
-class Counter(object):
+class Counter:
     def __init__(self, loop=None, intervals=(5, 60, 3600)):
         self.intervals = intervals
         self.components = [defaultdict(lambda: 0) for i in self.intervals]
diff --git a/distributed/dashboard/components/__init__.py b/distributed/dashboard/components/__init__.py
index a66be2eced6..bb8269083e9 100644
--- a/distributed/dashboard/components/__init__.py
+++ b/distributed/dashboard/components/__init__.py
@@ -42,7 +42,7 @@
 profile_interval = parse_timedelta(profile_interval, default="ms")
 
 
-class DashboardComponent(object):
+class DashboardComponent:
     """ Base class for Dask.distributed UI dashboard components.
 
     This class must have two attributes, ``root`` and ``source``, and one
diff --git a/distributed/dashboard/core.py b/distributed/dashboard/core.py
index 82cfe92da17..9b919917a67 100644
--- a/distributed/dashboard/core.py
+++ b/distributed/dashboard/core.py
@@ -16,7 +16,7 @@
     raise ImportError("Dask needs bokeh >= 0.13.0")
 
 
-class BokehServer(object):
+class BokehServer:
     server_kwargs = {}
 
     def listen(self, addr):
diff --git a/distributed/dashboard/scheduler.py b/distributed/dashboard/scheduler.py
index 2c0520161b3..a030ba434f7 100644
--- a/distributed/dashboard/scheduler.py
+++ b/distributed/dashboard/scheduler.py
@@ -251,7 +251,7 @@ def get(self):
         self.write(result)
 
 
-class _PrometheusCollector(object):
+class _PrometheusCollector:
     def __init__(self, server):
         self.server = server
 
diff --git a/distributed/dashboard/worker.py b/distributed/dashboard/worker.py
index 5a34a261bf1..db29480666b 100644
--- a/distributed/dashboard/worker.py
+++ b/distributed/dashboard/worker.py
@@ -35,7 +35,7 @@
 }
 
 
-class _PrometheusCollector(object):
+class _PrometheusCollector:
     def __init__(self, server):
         self.worker = server
         self.logger = logging.getLogger("distributed.dask_worker")
diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
index 1b304b0a53e..ad071a214be 100644
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -21,7 +21,7 @@
 logger = logging.getLogger(__name__)
 
 
-class Cluster(object):
+class Cluster:
     """ Superclass for cluster objects
 
     This class contains common functionality for Dask Cluster manager classes.
diff --git a/distributed/deploy/old_ssh.py b/distributed/deploy/old_ssh.py
index 30f6f819224..86d49c9cf15 100644
--- a/distributed/deploy/old_ssh.py
+++ b/distributed/deploy/old_ssh.py
@@ -335,7 +335,7 @@ def start_worker(
     return merge(cmd_dict, {"thread": thread})
 
 
-class SSHCluster(object):
+class SSHCluster:
     def __init__(
         self,
         scheduler_addr,
diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index 96279d15323..537fa3201f4 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -92,7 +92,7 @@ async def __aexit__(self, *args, **kwargs):
         await self.close()
 
 
-class NoOpAwaitable(object):
+class NoOpAwaitable:
     """An awaitable object that always returns None.
 
     Useful to return from a method that can be called in both asynchronous and
diff --git a/distributed/deploy/tests/test_slow_adaptive.py b/distributed/deploy/tests/test_slow_adaptive.py
index 09113fe3b23..e7021fc854a 100644
--- a/distributed/deploy/tests/test_slow_adaptive.py
+++ b/distributed/deploy/tests/test_slow_adaptive.py
@@ -6,7 +6,7 @@
 from distributed.metrics import time
 
 
-class SlowWorker(object):
+class SlowWorker:
     def __init__(self, *args, delay=0, **kwargs):
         self.worker = Worker(*args, **kwargs)
         self.delay = delay
diff --git a/distributed/deploy/utils_test.py b/distributed/deploy/utils_test.py
index 2bb55c7da08..fd6ba03aae9 100644
--- a/distributed/deploy/utils_test.py
+++ b/distributed/deploy/utils_test.py
@@ -3,7 +3,7 @@
 import pytest
 
 
-class ClusterTest(object):
+class ClusterTest:
     Cluster = None
     kwargs = {}
 
diff --git a/distributed/diagnostics/plugin.py b/distributed/diagnostics/plugin.py
index 1d218fe5ac8..12e7ad6ec3f 100644
--- a/distributed/diagnostics/plugin.py
+++ b/distributed/diagnostics/plugin.py
@@ -3,7 +3,7 @@
 logger = logging.getLogger(__name__)
 
 
-class SchedulerPlugin(object):
+class SchedulerPlugin:
     """ Interface to extend the Scheduler
 
     The scheduler operates by triggering and responding to events like
@@ -86,7 +86,7 @@ def remove_client(self, scheduler=None, client=None, **kwargs):
         """ Run when a client disconnects """
 
 
-class WorkerPlugin(object):
+class WorkerPlugin:
     """ Interface to extend the Worker
 
     A worker plugin enables custom code to run at different stages of the Workers'
diff --git a/distributed/diagnostics/progressbar.py b/distributed/diagnostics/progressbar.py
index 01dc9bbea39..ab7800c2125 100644
--- a/distributed/diagnostics/progressbar.py
+++ b/distributed/diagnostics/progressbar.py
@@ -24,7 +24,7 @@ def get_scheduler(scheduler):
     return coerce_to_address(scheduler)
 
 
-class ProgressBar(object):
+class ProgressBar:
     def __init__(self, keys, scheduler=None, interval="100ms", complete=True):
         self.scheduler = get_scheduler(scheduler)
 
@@ -207,7 +207,7 @@ def _draw_bar(self, remaining, all, **kwargs):
         )
 
 
-class MultiProgressBar(object):
+class MultiProgressBar:
     def __init__(
         self,
         keys,
diff --git a/distributed/diskutils.py b/distributed/diskutils.py
index 075ec7750c8..64124b753a8 100644
--- a/distributed/diskutils.py
+++ b/distributed/diskutils.py
@@ -30,7 +30,7 @@ def safe_unlink(path):
             logger.error("Failed to remove %r", str(e))
 
 
-class WorkDir(object):
+class WorkDir:
     """
     A temporary work directory inside a WorkSpace.
     """
@@ -102,7 +102,7 @@ def _finalize(cls, workspace, lock_path, lock_file, dir_path):
                 safe_unlink(lock_path)
 
 
-class WorkSpace(object):
+class WorkSpace:
     """
     An on-disk workspace that tracks disposable work directories inside it.
     If a process crashes or another event left stale directories behind,
diff --git a/distributed/lock.py b/distributed/lock.py
index c230a8e861c..48d538915f0 100644
--- a/distributed/lock.py
+++ b/distributed/lock.py
@@ -12,7 +12,7 @@
 logger = logging.getLogger(__name__)
 
 
-class LockExtension(object):
+class LockExtension:
     """ An extension for the scheduler to manage Locks
 
     This adds the following routes to the scheduler
@@ -73,7 +73,7 @@ def release(self, stream=None, name=None, id=None):
                 del self.events[name]
 
 
-class Lock(object):
+class Lock:
     """ Distributed Centralized Lock
 
     Parameters
diff --git a/distributed/locket.py b/distributed/locket.py
index 1ed7b023085..65a10f195f7 100644
--- a/distributed/locket.py
+++ b/distributed/locket.py
@@ -114,7 +114,7 @@ def _acquire_non_blocking(acquire, timeout, retry_period, path):
             time.sleep(retry_period)
 
 
-class _LockSet(object):
+class _LockSet:
     def __init__(self, locks):
         self._locks = locks
 
@@ -136,7 +136,7 @@ def release(self):
             lock.release()
 
 
-class _ThreadLock(object):
+class _ThreadLock:
     def __init__(self, path):
         self._path = path
         self._lock = threading.Lock()
@@ -156,7 +156,7 @@ def release(self):
         self._lock.release()
 
 
-class _LockFile(object):
+class _LockFile:
     def __init__(self, path):
         self._path = path
         self._file = None
@@ -181,7 +181,7 @@ def release(self):
         self._file = None
 
 
-class _Locker(object):
+class _Locker:
     """
     A lock wrapper to always apply the given *timeout* and *retry_period*
     to acquire() calls.
diff --git a/distributed/metrics.py b/distributed/metrics.py
index fefdfeb2e4c..f28e9f2ac7f 100755
--- a/distributed/metrics.py
+++ b/distributed/metrics.py
@@ -36,7 +36,7 @@ def wrapper():
 net_io_counters = _psutil_caller("net_io_counters")
 
 
-class _WindowsTime(object):
+class _WindowsTime:
     """
     Combine time.time() and time.perf_counter() to get an absolute clock
     with fine resolution.
diff --git a/distributed/nanny.py b/distributed/nanny.py
index 945f33041d3..9c95dd4a07a 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -450,7 +450,7 @@ async def close(self, comm=None, timeout=5, report=None):
         await ServerNode.close(self)
 
 
-class WorkerProcess(object):
+class WorkerProcess:
     def __init__(
         self,
         worker_kwargs,
diff --git a/distributed/node.py b/distributed/node.py
index edee3e2dd7b..4e26defeb08 100644
--- a/distributed/node.py
+++ b/distributed/node.py
@@ -12,7 +12,7 @@
 from .utils import DequeHandler, TimeoutError
 
 
-class Node(object):
+class Node:
     """
     Base class for nodes in a distributed cluster.
     """
diff --git a/distributed/process.py b/distributed/process.py
index 5899c853385..b070342b340 100644
--- a/distributed/process.py
+++ b/distributed/process.py
@@ -40,13 +40,13 @@ def _call_and_set_future(loop, future, func, *args, **kwargs):
         _loop_add_callback(loop, future.set_result, res)
 
 
-class _ProcessState(object):
+class _ProcessState:
     is_alive = False
     pid = None
     exitcode = None
 
 
-class AsyncProcess(object):
+class AsyncProcess:
     """
     A coroutine-compatible multiprocessing.Process-alike.
     All normally blocking methods are wrapped in Tornado coroutines.
diff --git a/distributed/protocol/cupy.py b/distributed/protocol/cupy.py
index 26a5accc6af..087de6f9663 100644
--- a/distributed/protocol/cupy.py
+++ b/distributed/protocol/cupy.py
@@ -5,7 +5,7 @@
 from .cuda import cuda_serialize, cuda_deserialize
 
 
-class PatchedCudaArrayInterface(object):
+class PatchedCudaArrayInterface:
     """This class do two things:
         1) Makes sure that __cuda_array_interface__['strides']
            behaves as specified in the protocol.
diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index 8d1d37a283e..ddab6130765 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -268,7 +268,7 @@ def deserialize(header, frames, deserializers=None):
     return loads(header, frames)
 
 
-class Serialize(object):
+class Serialize:
     """ Mark an object that should be serialized
 
     Example
@@ -301,7 +301,7 @@ def __hash__(self):
 to_serialize = Serialize
 
 
-class Serialized(object):
+class Serialized:
     """
     An object that is already serialized into header and frames
 
@@ -484,7 +484,7 @@ def register_serialization(cls, serialize, deserialize):
 
     Examples
     --------
-    >>> class Human(object):
+    >>> class Human:
     ...     def __init__(self, name):
     ...         self.name = name
 
diff --git a/distributed/protocol/tests/test_serialize.py b/distributed/protocol/tests/test_serialize.py
index 6ba70f676a1..b5a202f1520 100644
--- a/distributed/protocol/tests/test_serialize.py
+++ b/distributed/protocol/tests/test_serialize.py
@@ -26,7 +26,7 @@
 from distributed.comm.utils import to_frames, from_frames
 
 
-class MyObj(object):
+class MyObj:
     def __init__(self, data):
         self.data = data
 
@@ -151,7 +151,7 @@ def test_inter_worker_comms(c, s, a, b):
     assert o2.data == 123
 
 
-class Empty(object):
+class Empty:
     def __getstate__(self):
         raise Exception("Not picklable")
 
@@ -213,7 +213,7 @@ class BadException(Exception):
         def __setstate__(self):
             return Exception("Sneaky deserialization code")
 
-    class MyClass(object):
+    class MyClass:
         def __getstate__(self):
             raise BadException()
 
@@ -258,7 +258,7 @@ def test_err_on_bad_deserializer():
         yield from_frames(frames, deserializers=["msgpack"])
 
 
-class MyObject(object):
+class MyObject:
     def __init__(self, **kwargs):
         self.__dict__.update(kwargs)
 
@@ -348,7 +348,7 @@ def check(dask_worker):
 
 
 def test_serialize_raises():
-    class Foo(object):
+    class Foo:
         pass
 
     @dask_serialize.register(Foo)
diff --git a/distributed/publish.py b/distributed/publish.py
index c899b9fbaaa..758e5ccc34b 100644
--- a/distributed/publish.py
+++ b/distributed/publish.py
@@ -3,7 +3,7 @@
 from .utils import log_errors, tokey
 
 
-class PublishExtension(object):
+class PublishExtension:
     """ An extension for the scheduler to manage collections
 
     *  publish-list
diff --git a/distributed/pubsub.py b/distributed/pubsub.py
index 9de133ddb47..3c8b140b362 100644
--- a/distributed/pubsub.py
+++ b/distributed/pubsub.py
@@ -14,7 +14,7 @@
 logger = logging.getLogger(__name__)
 
 
-class PubSubSchedulerExtension(object):
+class PubSubSchedulerExtension:
     """ Extend Dask's scheduler with routes to handle PubSub machinery """
 
     def __init__(self, scheduler):
@@ -117,7 +117,7 @@ def handle_message(self, name=None, msg=None, worker=None, client=None):
                 )
 
 
-class PubSubWorkerExtension(object):
+class PubSubWorkerExtension:
     """ Extend Dask's Worker with routes to handle PubSub machinery """
 
     def __init__(self, worker):
@@ -170,7 +170,7 @@ def cleanup(self):
                 del self.publish_to_scheduler[name]
 
 
-class PubSubClientExtension(object):
+class PubSubClientExtension:
     """ Extend Dask's Client with handlers to handle PubSub machinery """
 
     def __init__(self, client):
@@ -199,7 +199,7 @@ def cleanup(self):
                 self.client.scheduler_comm.send(msg)
 
 
-class Pub(object):
+class Pub:
     """ Publish data with Publish-Subscribe pattern
 
     This allows clients and workers to directly communicate data between each
@@ -349,7 +349,7 @@ def __repr__(self):
     __str__ = __repr__
 
 
-class Sub(object):
+class Sub:
     """ Subscribe to a Publish/Subscribe topic
 
     See Also
diff --git a/distributed/pytest_resourceleaks.py b/distributed/pytest_resourceleaks.py
index 0119a425722..348472892d6 100644
--- a/distributed/pytest_resourceleaks.py
+++ b/distributed/pytest_resourceleaks.py
@@ -92,7 +92,7 @@ def decorate(cls):
     return decorate
 
 
-class ResourceChecker(object):
+class ResourceChecker:
     def on_start_test(self):
         pass
 
@@ -260,7 +260,7 @@ def format(self, before, after):
         return "\n".join(lines)
 
 
-class LeakChecker(object):
+class LeakChecker:
     def __init__(self, checkers, grace_delay, mark_failed, max_retries):
         self.checkers = checkers
         self.grace_delay = grace_delay
diff --git a/distributed/queues.py b/distributed/queues.py
index 6d1fc76571b..9f5db0af68e 100644
--- a/distributed/queues.py
+++ b/distributed/queues.py
@@ -14,7 +14,7 @@
 logger = logging.getLogger(__name__)
 
 
-class QueueExtension(object):
+class QueueExtension:
     """ An extension for the scheduler to manage queues
 
     This adds the following routes to the scheduler
@@ -139,7 +139,7 @@ def qsize(self, stream=None, name=None, client=None):
         return self.queues[name].qsize()
 
 
-class Queue(object):
+class Queue:
     """ Distributed Queue
 
     This allows multiple clients to share futures or small bits of data between
diff --git a/distributed/recreate_exceptions.py b/distributed/recreate_exceptions.py
index 9138c1fca5a..4aaa851ee23 100644
--- a/distributed/recreate_exceptions.py
+++ b/distributed/recreate_exceptions.py
@@ -7,7 +7,7 @@
 logger = logging.getLogger(__name__)
 
 
-class ReplayExceptionScheduler(object):
+class ReplayExceptionScheduler:
     """ A plugin for the scheduler to recreate exceptions locally
 
     This adds the following routes to the scheduler
@@ -50,7 +50,7 @@ def cause_of_failure(self, *args, keys=(), **kwargs):
                 }
 
 
-class ReplayExceptionClient(object):
+class ReplayExceptionClient:
     """
     A plugin for the client allowing replay of remote exceptions locally
 
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index ab2e615cef2..89c938f0dd5 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -92,7 +92,7 @@
 ALL_TASK_STATES = {"released", "waiting", "no-worker", "processing", "erred", "memory"}
 
 
-class ClientState(object):
+class ClientState:
     """
     A simple object holding information about a client.
 
@@ -128,7 +128,7 @@ def __str__(self):
         return self.client_key
 
 
-class WorkerState(object):
+class WorkerState:
     """
     A simple object holding information about a worker.
 
@@ -324,7 +324,7 @@ def ncores(self):
         return self.nthreads
 
 
-class TaskState(object):
+class TaskState:
     """
     A simple object holding information about a task.
 
@@ -683,7 +683,7 @@ def validate(self):
                 pdb.set_trace()
 
 
-class TaskGroup(object):
+class TaskGroup:
     """ Collection tracking all tasks within a group
 
     Keys often have a structure like ``("x-123", 0)``
@@ -754,7 +754,7 @@ def __len__(self):
         return sum(self.states.values())
 
 
-class TaskPrefix(object):
+class TaskPrefix:
     """ Collection tracking all tasks within a group
 
     Keys often have a structure like ``("x-123", 0)``
diff --git a/distributed/security.py b/distributed/security.py
index 6b7d87b2715..f3430ac7b3e 100644
--- a/distributed/security.py
+++ b/distributed/security.py
@@ -13,7 +13,7 @@
 __all__ = ("Security",)
 
 
-class Security(object):
+class Security:
     """Security configuration for a Dask cluster.
 
     Default values are loaded from Dask's configuration files, and can be
diff --git a/distributed/system_monitor.py b/distributed/system_monitor.py
index 5b3bed3f98d..cf305869a8c 100644
--- a/distributed/system_monitor.py
+++ b/distributed/system_monitor.py
@@ -5,7 +5,7 @@
 from .metrics import time
 
 
-class SystemMonitor(object):
+class SystemMonitor:
     def __init__(self, n=10000):
         self.proc = psutil.Process()
 
diff --git a/distributed/tests/test_actor.py b/distributed/tests/test_actor.py
index fd6bf0335e1..de69db5685a 100644
--- a/distributed/tests/test_actor.py
+++ b/distributed/tests/test_actor.py
@@ -11,7 +11,7 @@
 from distributed.metrics import time
 
 
-class Counter(object):
+class Counter:
     n = 0
 
     def __init__(self):
@@ -26,7 +26,7 @@ def add(self, x):
         return self.n
 
 
-class List(object):
+class List:
     L = []
 
     def __init__(self, dummy=None):
@@ -36,7 +36,7 @@ def append(self, x):
         self.L.append(x)
 
 
-class ParameterServer(object):
+class ParameterServer:
     def __init__(self):
         self.data = {}
 
@@ -156,7 +156,7 @@ def test_linear_access(c, s, a, b):
 
 @gen_cluster(client=True)
 def test_exceptions_create(c, s, a, b):
-    class Foo(object):
+    class Foo:
         x = 0
 
         def __init__(self):
@@ -170,7 +170,7 @@ def __init__(self):
 
 @gen_cluster(client=True)
 def test_exceptions_method(c, s, a, b):
-    class Foo(object):
+    class Foo:
         def throw(self):
             1 / 0
 
@@ -349,7 +349,7 @@ def add(n, counter):
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 5)] * 2)
 def test_thread_safety(c, s, a, b):
-    class Unsafe(object):
+    class Unsafe:
         def __init__(self):
             self.n = 0
 
@@ -378,7 +378,7 @@ def test_Actors_create_dependencies(c, s, a, b):
 
 @gen_cluster(client=True)
 def test_load_balance(c, s, a, b):
-    class Foo(object):
+    class Foo:
         def __init__(self, x):
             pass
 
@@ -396,7 +396,7 @@ def __init__(self, x):
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 5)
 def test_load_balance_map(c, s, *workers):
-    class Foo(object):
+    class Foo:
         def __init__(self, x, y=None):
             pass
 
@@ -510,7 +510,7 @@ def check(dask_worker):
     config={"distributed.worker.profile.interval": "1ms"},
 )
 def test_actors_in_profile(c, s, a):
-    class Sleeper(object):
+    class Sleeper:
         def sleep(self, time):
             sleep(time)
 
@@ -530,7 +530,7 @@ def sleep(self, time):
 def test_waiter(c, s, a, b):
     from tornado.locks import Event
 
-    class Waiter(object):
+    class Waiter:
         def __init__(self):
             self.event = Event()
 
diff --git a/distributed/tests/test_batched.py b/distributed/tests/test_batched.py
index 07dd32f4c68..74efba810d3 100644
--- a/distributed/tests/test_batched.py
+++ b/distributed/tests/test_batched.py
@@ -12,7 +12,7 @@
 from distributed.protocol import to_serialize
 
 
-class EchoServer(object):
+class EchoServer:
     count = 0
 
     async def handle_comm(self, comm):
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index a205452b2f8..16e660492f8 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -1130,7 +1130,7 @@ def test_scatter_hash(c, s, a, b):
 def test_scatter_tokenize_local(c, s, a, b):
     from dask.base import normalize_token
 
-    class MyObj(object):
+    class MyObj:
         pass
 
     L = []
@@ -1846,7 +1846,7 @@ def f(x, y=10):
     assert result == 100 + 1 + 200
 
 
-class BadlySerializedObject(object):
+class BadlySerializedObject:
     def __getstate__(self):
         return 1
 
@@ -1854,7 +1854,7 @@ def __setstate__(self, state):
         raise TypeError("hello!")
 
 
-class FatallySerializedObject(object):
+class FatallySerializedObject:
     def __getstate__(self):
         return 1
 
@@ -3018,7 +3018,7 @@ def test_replicate_workers(c, s, *workers):
     s.validate_state()
 
 
-class CountSerialization(object):
+class CountSerialization:
     def __init__(self):
         self.n = 0
 
@@ -4472,7 +4472,7 @@ class MyException(Exception):
 
 @gen_cluster(client=True)
 def test_robust_unserializable(c, s, a, b):
-    class Foo(object):
+    class Foo:
         def __getstate__(self):
             raise MyException()
 
@@ -4488,7 +4488,7 @@ def __getstate__(self):
 
 @gen_cluster(client=True)
 def test_robust_undeserializable(c, s, a, b):
-    class Foo(object):
+    class Foo:
         def __getstate__(self):
             return 1
 
@@ -4508,7 +4508,7 @@ def __setstate__(self, state):
 
 @gen_cluster(client=True)
 def test_robust_undeserializable_function(c, s, a, b):
-    class Foo(object):
+    class Foo:
         def __getstate__(self):
             return 1
 
diff --git a/distributed/tests/test_core.py b/distributed/tests/test_core.py
index d423c6ab6c3..0a9c48bc870 100644
--- a/distributed/tests/test_core.py
+++ b/distributed/tests/test_core.py
@@ -49,7 +49,7 @@ def echo(comm, x):
     return x
 
 
-class CountedObject(object):
+class CountedObject:
     """
     A class which counts the number of live instances.
     """
diff --git a/distributed/tests/test_steal.py b/distributed/tests/test_steal.py
index de63e542807..9c4fef57d2a 100644
--- a/distributed/tests/test_steal.py
+++ b/distributed/tests/test_steal.py
@@ -623,7 +623,7 @@ def long(delay):
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 5)] * 2)
 def test_cleanup_repeated_tasks(c, s, a, b):
-    class Foo(object):
+    class Foo:
         pass
 
     s.extensions["stealing"]._pc.callback_time = 20
diff --git a/distributed/tests/test_utils_perf.py b/distributed/tests/test_utils_perf.py
index 4256548900c..a1591df0280 100644
--- a/distributed/tests/test_utils_perf.py
+++ b/distributed/tests/test_utils_perf.py
@@ -11,7 +11,7 @@
 from distributed.utils_test import captured_logger, run_for
 
 
-class RandomTimer(object):
+class RandomTimer:
     """
     A mock timer producing random (but monotonic) values.
     """
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 11ab461ae25..6c1a0805817 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -86,7 +86,7 @@ def test_identity():
 
 @gen_cluster(client=True)
 def test_worker_bad_args(c, s, a, b):
-    class NoReprObj(object):
+    class NoReprObj:
         """ This object cannot be properly represented as a string. """
 
         def __str__(self):
@@ -833,7 +833,7 @@ def test_worker_dir(c, s, a, b):
 
 @gen_cluster(client=True)
 def test_dataframe_attribute_error(c, s, a, b):
-    class BadSize(object):
+    class BadSize:
         def __init__(self, data):
             self.data = data
 
@@ -847,7 +847,7 @@ def __sizeof__(self):
 
 @gen_cluster(client=True)
 def test_fail_write_to_disk(c, s, a, b):
-    class Bad(object):
+    class Bad:
         def __getstate__(self):
             raise TypeError()
 
@@ -876,7 +876,7 @@ def test_fail_write_many_to_disk(c, s, a):
     yield gen.sleep(0.1)
     assert not a.paused
 
-    class Bad(object):
+    class Bad:
         def __init__(self, x):
             pass
 
@@ -1094,7 +1094,7 @@ def test_robust_to_bad_sizeof_estimates(c, s, a):
     memory = psutil.Process().memory_info().rss
     a.memory_limit = memory / 0.7 + 400e6
 
-    class BadAccounting(object):
+    class BadAccounting:
         def __init__(self, data):
             self.data = data
 
diff --git a/distributed/utils.py b/distributed/utils.py
index efc948cff19..086555643ea 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -349,7 +349,7 @@ def f():
         return result[0]
 
 
-class LoopRunner(object):
+class LoopRunner:
     """
     A helper to start and stop an IO loop in a controlled way.
     Several loop runners can associate safely to the same IO loop.
@@ -1061,7 +1061,7 @@ def import_file(path):
     return loaded
 
 
-class itemgetter(object):
+class itemgetter:
     """A picklable itemgetter.
 
     Examples
diff --git a/distributed/utils_comm.py b/distributed/utils_comm.py
index 792e73227a9..3d10ba51038 100644
--- a/distributed/utils_comm.py
+++ b/distributed/utils_comm.py
@@ -96,7 +96,7 @@ async def gather_from_workers(who_has, rpc, close=True, serializers=None, who=No
     return (results, bad_keys, list(missing_workers))
 
 
-class WrappedKey(object):
+class WrappedKey:
     """ Interface for a key in a dask graph.
 
     Subclasses must have .key attribute that refers to a key in a dask graph.
diff --git a/distributed/utils_perf.py b/distributed/utils_perf.py
index f21e96d7353..3b97dd46327 100644
--- a/distributed/utils_perf.py
+++ b/distributed/utils_perf.py
@@ -12,7 +12,7 @@
 logger = _logger = logging.getLogger(__name__)
 
 
-class ThrottledGC(object):
+class ThrottledGC:
     """Wrap gc.collect to protect against excessively repeated calls.
 
     Allows to run throttled garbage collection in the workers as a
@@ -67,7 +67,7 @@ def collect(self):
             )
 
 
-class FractionalTimer(object):
+class FractionalTimer:
     """
     An object that measures runtimes, accumulates them and computes
     a running fraction of the recent runtimes over the corresponding
@@ -128,7 +128,7 @@ def running_fraction(self):
         return self._running_fraction
 
 
-class GCDiagnosis(object):
+class GCDiagnosis:
     """
     An object that hooks itself into the gc callbacks to collect
     timing and memory statistics, and log interesting info.
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index 1650e0426ef..e16983b1879 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -343,7 +343,7 @@ def run_for(duration, timer=time):
 _varying_key_gen = itertools.count()
 
 
-class _ModuleSlot(object):
+class _ModuleSlot:
     def __init__(self, modname, slotname):
         self.modname = modname
         self.slotname = slotname
diff --git a/distributed/variable.py b/distributed/variable.py
index 677e2997b32..bfd6ca250e2 100644
--- a/distributed/variable.py
+++ b/distributed/variable.py
@@ -19,7 +19,7 @@
 logger = logging.getLogger(__name__)
 
 
-class VariableExtension(object):
+class VariableExtension:
     """ An extension for the scheduler to manage queues
 
     This adds the following routes to the scheduler
@@ -114,7 +114,7 @@ async def delete(self, stream=None, name=None, client=None):
             del self.variables[name]
 
 
-class Variable(object):
+class Variable:
     """ Distributed Global Variable
 
     This allows multiple clients to share futures and data between each other
diff --git a/distributed/worker.py b/distributed/worker.py
index e429cb75a4b..77711058682 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -3417,7 +3417,7 @@ def get_msg_safe_str(msg):
     ignoring them.
     """
 
-    class Repr(object):
+    class Repr:
         def __init__(self, f, val):
             self._f = f
             self._val = val
diff --git a/docs/source/actors.rst b/docs/source/actors.rst
index b8bbebc743a..85d1300e502 100644
--- a/docs/source/actors.rst
+++ b/docs/source/actors.rst
@@ -197,7 +197,7 @@ will run on the Worker's event loop thread rather than a separate thread.
 
 .. code-block:: python
 
-   def Waiter(object):
+   def Waiter:
        def __init__(self):
            self.event = tornado.locks.Event()
 
diff --git a/docs/source/adaptive.rst b/docs/source/adaptive.rst
index 774ae21e4a6..f07246588cd 100644
--- a/docs/source/adaptive.rst
+++ b/docs/source/adaptive.rst
@@ -59,7 +59,7 @@ the correct times.
 
 .. code-block:: python
 
-   class MyCluster(object):
+   class MyCluster:
        async def scale_up(self, n, **kwargs):
            """
            Bring the total count of workers up to ``n``
@@ -110,7 +110,7 @@ We reproduce the full body of the implementation below as an example:
    from marathon import MarathonClient, MarathonApp
    from marathon.models.container import MarathonContainer
 
-   class MarathonCluster(object):
+   class MarathonCluster:
        def __init__(self, scheduler,
                     executable='dask-worker',
                     docker_image='mrocklin/dask-distributed',
diff --git a/docs/source/serialization.rst b/docs/source/serialization.rst
index 25b2ae49476..ec315cc321e 100644
--- a/docs/source/serialization.rst
+++ b/docs/source/serialization.rst
@@ -154,7 +154,7 @@ register them with Dask.
 
 .. code-block:: python
 
-    class Human(object):
+    class Human:
         def __init__(self, name):
             self.name = name
 

From 7c9da106b2edd543d3b420ad8270261c6a04ec5e Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 3 Feb 2020 09:09:37 -0800
Subject: [PATCH 0654/1550] Replace tornado.locks with asyncio for
 Events/Locks/Conditions/Semaphore (#3397)

---
 distributed/client.py                | 66 ++++++++++++++--------------
 distributed/comm/inproc.py           |  3 +-
 distributed/comm/tests/test_comms.py |  4 +-
 distributed/core.py                  |  3 +-
 distributed/deploy/spec.py           |  3 +-
 distributed/lock.py                  |  6 +--
 distributed/nanny.py                 |  5 +--
 distributed/pubsub.py                | 43 +++++++++---------
 distributed/queues.py                | 35 ++++-----------
 distributed/tests/test_nanny.py      |  3 +-
 distributed/tests/test_pubsub.py     |  2 +
 distributed/tests/test_variable.py   | 32 ++++++++++----
 distributed/variable.py              | 40 ++++++++++-------
 docs/source/actors.rst               |  2 +-
 14 files changed, 124 insertions(+), 123 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 706101d2fe4..93f395e2dc4 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -1,3 +1,4 @@
+import asyncio
 import atexit
 from collections import defaultdict
 from collections.abc import Iterator
@@ -8,6 +9,7 @@
 import errno
 from functools import partial
 import html
+from inspect import isawaitable
 import itertools
 import json
 import logging
@@ -37,12 +39,7 @@
 except ImportError:
     single_key = first
 from tornado import gen
-from tornado.locks import Event, Condition, Semaphore
 from tornado.ioloop import IOLoop
-from tornado.queues import Queue
-
-import asyncio
-from asyncio import iscoroutine
 
 from .batched import BatchedSend
 from .utils_comm import (
@@ -431,7 +428,7 @@ def _get_event(self):
         # (https://github.com/tornadoweb/tornado/issues/2189)
         event = self._event
         if event is None:
-            event = self._event = Event()
+            event = self._event = asyncio.Event()
         return event
 
     def cancel(self):
@@ -470,7 +467,7 @@ def reset(self):
             self._event.clear()
 
     async def wait(self, timeout=None):
-        await self._get_event().wait(timeout)
+        await asyncio.wait_for(self._get_event().wait(), timeout)
 
     def __repr__(self):
         return "<%s: %s>" % (self.__class__.__name__, self.status)
@@ -630,10 +627,6 @@ def __init__(
         self._deserializers = deserializers
         self.direct_to_workers = direct_to_workers
 
-        self._gather_semaphore = Semaphore(5)
-        self._gather_keys = None
-        self._gather_future = None
-
         # Communication
         self.scheduler_comm = None
 
@@ -678,6 +671,10 @@ def __init__(
         self._loop_runner = LoopRunner(loop=loop, asynchronous=asynchronous)
         self.loop = self._loop_runner.loop
 
+        self._gather_semaphore = asyncio.Semaphore(5, loop=self.loop.asyncio_loop)
+        self._gather_keys = None
+        self._gather_future = None
+
         if heartbeat_interval is None:
             heartbeat_interval = dask.config.get("distributed.client.heartbeat")
         heartbeat_interval = parse_timedelta(heartbeat_interval, default="ms")
@@ -1190,7 +1187,9 @@ async def _handle_report(self):
 
                         try:
                             handler = self._stream_handlers[op]
-                            handler(**msg)
+                            result = handler(**msg)
+                            if isawaitable(result):
+                                await result
                         except Exception as e:
                             logger.exception(e)
                     if breakout:
@@ -1246,6 +1245,8 @@ def _handle_error(self, exception=None):
 
     async def _close(self, fast=False):
         """ Send close signal and wait until scheduler completes """
+        if self.status == "closed":
+            return
         self.status = "closing"
 
         for pc in self._periodic_callbacks.values():
@@ -1260,8 +1261,6 @@ async def _close(self, fast=False):
                     pass
             if self.get == dask.config.get("get", None):
                 del dask.config.config["get"]
-            if self.status == "closed":
-                return
 
             if (
                 self.scheduler_comm
@@ -1353,7 +1352,7 @@ def close(self, timeout=no_default):
         if self._start_arg is None:
             with ignoring(AttributeError):
                 f = self.cluster.close()
-                if iscoroutine(f):
+                if asyncio.iscoroutine(f):
 
                     async def _():
                         await f
@@ -1373,6 +1372,7 @@ async def _shutdown(self):
             await self.cluster.close()
         else:
             with ignoring(CommClosedError):
+                self.status = "closing"
                 await self.scheduler.terminate(close_workers=True)
 
     def shutdown(self):
@@ -1808,12 +1808,11 @@ async def _gather_remote(self, direct, local_worker):
         few.  In controls access using a Tornado semaphore, and picks up keys
         from other requests made recently.
         """
-        await self._gather_semaphore.acquire()
-        keys = list(self._gather_keys)
-        self._gather_keys = None  # clear state, these keys are being sent off
-        self._gather_future = None
+        async with self._gather_semaphore:
+            keys = list(self._gather_keys)
+            self._gather_keys = None  # clear state, these keys are being sent off
+            self._gather_future = None
 
-        try:
             if direct or local_worker:  # gather directly from workers
                 who_has = await retry_operation(self.scheduler.who_has, keys=keys)
                 data2, missing_keys, missing_workers = await gather_from_workers(
@@ -1828,8 +1827,6 @@ async def _gather_remote(self, direct, local_worker):
 
             else:  # ask scheduler to gather data for us
                 response = await retry_operation(self.scheduler.gather, keys=keys)
-        finally:
-            self._gather_semaphore.release()
 
         return response
 
@@ -2919,10 +2916,12 @@ async def _restart(self, timeout=no_default):
         if timeout == no_default:
             timeout = self._timeout * 2
         self._send_to_scheduler({"op": "restart", "timeout": timeout})
-        self._restart_event = Event()
+        self._restart_event = asyncio.Event()
         try:
-            await self._restart_event.wait(self.loop.time() + timeout)
-        except gen.TimeoutError:
+            await asyncio.wait_for(
+                self._restart_event.wait(), self.loop.time() + timeout
+            )
+        except TimeoutError:
             logger.error("Restart timed out after %f seconds", timeout)
             pass
         self.generation += 1
@@ -4136,7 +4135,7 @@ async def _first_completed(futures):
     See Also:
         _as_completed
     """
-    q = Queue()
+    q = asyncio.Queue()
     await _as_completed(futures, q)
     result = await q.get()
     return result
@@ -4207,7 +4206,7 @@ def __init__(self, futures=None, loop=None, with_results=False, raise_errors=Tru
         self.queue = pyQueue()
         self.lock = threading.Lock()
         self.loop = loop or default_client().loop
-        self.condition = Condition()
+        self.condition = asyncio.Condition(loop=self.loop.asyncio_loop)
         self.thread_condition = threading.Condition()
         self.with_results = with_results
         self.raise_errors = raise_errors
@@ -4215,11 +4214,6 @@ def __init__(self, futures=None, loop=None, with_results=False, raise_errors=Tru
         if futures:
             self.update(futures)
 
-    def _notify(self):
-        self.condition.notify()
-        with self.thread_condition:
-            self.thread_condition.notify()
-
     async def _track_future(self, future):
         try:
             await _wait(future)
@@ -4238,7 +4232,10 @@ async def _track_future(self, future):
                 self.queue.put_nowait((future, result))
             else:
                 self.queue.put_nowait(future)
-            self._notify()
+            async with self.condition:
+                self.condition.notify()
+            with self.thread_condition:
+                self.thread_condition.notify()
 
     def update(self, futures):
         """ Add multiple futures to the collection.
@@ -4305,7 +4302,8 @@ async def __anext__(self):
         while self.queue.empty():
             if not self.futures:
                 raise StopAsyncIteration
-            await self.condition.wait()
+            async with self.condition:
+                await self.condition.wait()
 
         return self._get_and_raise()
 
diff --git a/distributed/comm/inproc.py b/distributed/comm/inproc.py
index c0191f024f6..0642cce7381 100644
--- a/distributed/comm/inproc.py
+++ b/distributed/comm/inproc.py
@@ -7,7 +7,6 @@
 import weakref
 import warnings
 
-from tornado import locks
 from tornado.concurrent import Future
 from tornado.ioloop import IOLoop
 
@@ -298,7 +297,7 @@ async def connect(self, address, deserialize=True, **connection_args):
             s2c_q=Queue(),
             c_loop=IOLoop.current(),
             c_addr=self.manager.new_address(),
-            conn_event=locks.Event(),
+            conn_event=asyncio.Event(),
         )
         listener.connect_threadsafe(conn_req)
         # Wait for connection acknowledgement
diff --git a/distributed/comm/tests/test_comms.py b/distributed/comm/tests/test_comms.py
index b486912f281..150251f3d59 100644
--- a/distributed/comm/tests/test_comms.py
+++ b/distributed/comm/tests/test_comms.py
@@ -9,7 +9,7 @@
 import pkg_resources
 import pytest
 
-from tornado import ioloop, locks, queues
+from tornado import ioloop, queues
 from tornado.concurrent import Future
 
 import distributed
@@ -901,7 +901,7 @@ async def handle_comm(comm):
 
 
 async def check_connector_deserialize(addr, deserialize, in_value, check_out):
-    done = locks.Event()
+    done = asyncio.Event()
 
     async def handle_comm(comm):
         await comm.write(in_value)
diff --git a/distributed/core.py b/distributed/core.py
index 3dad1223030..78dd618e8e2 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -13,7 +13,6 @@
 from toolz import merge
 from tornado import gen
 from tornado.ioloop import IOLoop
-from tornado.locks import Event
 
 from .comm import (
     connect,
@@ -134,7 +133,7 @@ def __init__(
         self.events = None
         self.event_counts = None
         self._ongoing_coroutines = weakref.WeakSet()
-        self._event_finished = Event()
+        self._event_finished = asyncio.Event()
 
         self.listeners = []
         self.io_loop = io_loop or IOLoop.current()
diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index 537fa3201f4..17b1af28148 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -6,7 +6,6 @@
 import weakref
 
 import dask
-from tornado.locks import Event
 from tornado import gen
 
 from .adaptive import Adaptive
@@ -42,7 +41,7 @@ def __init__(self, scheduler=None, name=None):
         self.external_address = None
         self.lock = asyncio.Lock()
         self.status = "created"
-        self._event_finished = Event()
+        self._event_finished = asyncio.Event()
 
     def __await__(self):
         async def _():
diff --git a/distributed/lock.py b/distributed/lock.py
index 48d538915f0..3eceba5ce7a 100644
--- a/distributed/lock.py
+++ b/distributed/lock.py
@@ -1,9 +1,7 @@
+import asyncio
 from collections import defaultdict, deque
 import logging
 import uuid
-import asyncio
-
-import tornado.locks
 
 from .client import _get_global_client
 from .utils import log_errors, TimeoutError
@@ -40,7 +38,7 @@ async def acquire(self, stream=None, name=None, id=None, timeout=None):
                 result = True
             else:
                 while name in self.ids:
-                    event = tornado.locks.Event()
+                    event = asyncio.Event()
                     self.events[name].append(event)
                     future = event.wait()
                     if timeout is not None:
diff --git a/distributed/nanny.py b/distributed/nanny.py
index 9c95dd4a07a..5e67b51b52b 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -12,7 +12,6 @@
 import dask
 from dask.system import CPU_COUNT
 from tornado.ioloop import IOLoop
-from tornado.locks import Event
 from tornado import gen
 
 from .comm import get_address_host, unparse_host_port
@@ -507,8 +506,8 @@ async def start(self):
         )
         self.process.daemon = dask.config.get("distributed.worker.daemon", default=True)
         self.process.set_exit_callback(self._on_exit)
-        self.running = Event()
-        self.stopped = Event()
+        self.running = asyncio.Event()
+        self.stopped = asyncio.Event()
         self.status = "starting"
         try:
             await self.process.start()
diff --git a/distributed/pubsub.py b/distributed/pubsub.py
index 3c8b140b362..355fee7ae59 100644
--- a/distributed/pubsub.py
+++ b/distributed/pubsub.py
@@ -1,14 +1,12 @@
+import asyncio
 from collections import defaultdict, deque
-import datetime
 import logging
 import threading
 import weakref
 
-import tornado.locks
-from tornado import gen
-
 from .core import CommClosedError
-from .utils import sync, TimeoutError
+from .metrics import time
+from .utils import sync, TimeoutError, ignoring
 from .protocol.serialize import to_serialize
 
 logger = logging.getLogger(__name__)
@@ -148,9 +146,9 @@ def remove_subscriber(self, name=None, address=None):
     def publish_scheduler(self, name=None, publish=None):
         self.publish_to_scheduler[name] = publish
 
-    def handle_message(self, name=None, msg=None):
+    async def handle_message(self, name=None, msg=None):
         for sub in self.subscribers.get(name, []):
-            sub._put(msg)
+            await sub._put(msg)
 
     def trigger_cleanup(self):
         self.worker.loop.add_callback(self.cleanup)
@@ -180,9 +178,9 @@ def __init__(self, client):
         self.subscribers = defaultdict(weakref.WeakSet)
         self.client.extensions["pubsub"] = self  # TODO: circular reference
 
-    def handle_message(self, name=None, msg=None):
+    async def handle_message(self, name=None, msg=None):
         for sub in self.subscribers[name]:
-            sub._put(msg)
+            await sub._put(msg)
 
         if not self.subscribers[name]:
             self.client.scheduler_comm.send(
@@ -374,7 +372,7 @@ def __init__(self, name, worker=None, client=None):
             self.loop = self.client.loop
         self.name = name
         self.buffer = deque()
-        self.condition = tornado.locks.Condition()
+        self.condition = asyncio.Condition(loop=self.loop.asyncio_loop)
 
         if self.worker:
             pubsub = self.worker.extensions["pubsub"]
@@ -393,20 +391,24 @@ def __init__(self, name, worker=None, client=None):
         weakref.finalize(self, pubsub.trigger_cleanup)
 
     async def _get(self, timeout=None):
-        if timeout is not None:
-            timeout = datetime.timedelta(seconds=timeout)
-        start = datetime.datetime.now()
+        start = time()
         while not self.buffer:
             if timeout is not None:
-                timeout2 = timeout - (datetime.datetime.now() - start)
-                if timeout2.total_seconds() < 0:
+                timeout2 = timeout - (time() - start)
+                if timeout2 < 0:
                     raise TimeoutError()
             else:
                 timeout2 = None
+
+            async def _():
+                await self.condition.acquire()
+                await self.condition.wait()
+
             try:
-                await self.condition.wait(timeout=timeout2)
-            except gen.TimeoutError:
-                raise TimeoutError("Timed out waiting on Sub")
+                await asyncio.wait_for(_(), timeout2)
+            finally:
+                with ignoring(RuntimeError):  # Python 3.6 fails here sometimes
+                    self.condition.release()
 
         return self.buffer.popleft()
 
@@ -431,9 +433,10 @@ def __iter__(self):
     def __aiter__(self):
         return self
 
-    def _put(self, msg):
+    async def _put(self, msg):
         self.buffer.append(msg)
-        self.condition.notify()
+        async with self.condition:
+            self.condition.notify()
 
     def __repr__(self):
         return "<Sub: {}>".format(self.name)
diff --git a/distributed/queues.py b/distributed/queues.py
index 9f5db0af68e..6cdab880aa6 100644
--- a/distributed/queues.py
+++ b/distributed/queues.py
@@ -1,14 +1,10 @@
+import asyncio
 from collections import defaultdict
-import datetime
 import logging
 import uuid
 
-import tornado.queues
-from tornado.locks import Event
-from tornado import gen
-
 from .client import Future, _get_global_client, Client
-from .utils import tokey, sync, thread_state, TimeoutError
+from .utils import tokey, sync, thread_state
 from .worker import get_client
 
 logger = logging.getLogger(__name__)
@@ -50,7 +46,7 @@ def __init__(self, scheduler):
     def create(self, stream=None, name=None, client=None, maxsize=0):
         logger.debug("Queue name: {}".format(name))
         if name not in self.queues:
-            self.queues[name] = tornado.queues.Queue(maxsize=maxsize)
+            self.queues[name] = asyncio.Queue(maxsize=maxsize)
             self.client_refcount[name] = 1
         else:
             self.client_refcount[name] += 1
@@ -77,12 +73,7 @@ async def put(
             self.scheduler.client_desires_keys(keys=[key], client="queue-%s" % name)
         else:
             record = {"type": "msgpack", "value": data}
-        if timeout is not None:
-            timeout = datetime.timedelta(seconds=timeout)
-        try:
-            await self.queues[name].put(record, timeout=timeout)
-        except gen.TimeoutError:
-            raise TimeoutError("Timed out waiting for Queue")
+        await asyncio.wait_for(self.queues[name].put(record), timeout=timeout)
 
     def future_release(self, name=None, key=None, client=None):
         self.future_refcount[name, key] -= 1
@@ -126,12 +117,7 @@ def process(record):
             out = [process(o) for o in out]
             return out
         else:
-            if timeout is not None:
-                timeout = datetime.timedelta(seconds=timeout)
-            try:
-                record = await self.queues[name].get(timeout=timeout)
-            except gen.TimeoutError:
-                raise TimeoutError("Timed out waiting for Queue")
+            record = await asyncio.wait_for(self.queues[name].get(), timeout=timeout)
             record = process(record)
             return record
 
@@ -171,7 +157,7 @@ class Queue:
     def __init__(self, name=None, client=None, maxsize=0):
         self.client = client or _get_global_client()
         self.name = name or "queue-" + uuid.uuid4().hex
-        self._event_started = Event()
+        self._event_started = asyncio.Event()
         if self.client.asynchronous or getattr(
             thread_state, "on_event_loop_thread", False
         ):
@@ -232,12 +218,9 @@ def qsize(self, **kwargs):
         return self.client.sync(self._qsize, **kwargs)
 
     async def _get(self, timeout=None, batch=False):
-        try:
-            resp = await self.client.scheduler.queue_get(
-                timeout=timeout, name=self.name, batch=batch
-            )
-        except gen.TimeoutError:
-            raise TimeoutError("Timed out waiting for Queue")
+        resp = await self.client.scheduler.queue_get(
+            timeout=timeout, name=self.name, batch=batch
+        )
 
         def process(d):
             if d["type"] == "Future":
diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
index 2ddc3b7e5db..0091a6126f1 100644
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -12,7 +12,6 @@
 from toolz import valmap, first
 from tornado import gen
 from tornado.ioloop import IOLoop
-from tornado.locks import Event
 
 import dask
 from distributed.diagnostics import SchedulerPlugin
@@ -453,7 +452,7 @@ async def test_nanny_closes_cleanly(cleanup):
 @pytest.mark.asyncio
 async def test_lifetime(cleanup):
     counter = 0
-    event = Event()
+    event = asyncio.Event()
 
     class Plugin(SchedulerPlugin):
         def add_worker(self, **kwargs):
diff --git a/distributed/tests/test_pubsub.py b/distributed/tests/test_pubsub.py
index 847b0b88bf0..2e372dea88b 100644
--- a/distributed/tests/test_pubsub.py
+++ b/distributed/tests/test_pubsub.py
@@ -124,6 +124,8 @@ def test_timeouts(c, s, a, b):
         yield sub.get(timeout=0.1)
     stop = time()
     assert stop - start < 1
+    with pytest.raises(TimeoutError):
+        yield sub.get(timeout=0.01)
 
 
 @gen_cluster(client=True)
diff --git a/distributed/tests/test_variable.py b/distributed/tests/test_variable.py
index 962b7a40e42..6e3b3bcdad6 100644
--- a/distributed/tests/test_variable.py
+++ b/distributed/tests/test_variable.py
@@ -5,9 +5,11 @@
 
 import pytest
 from tornado import gen
+from tornado.ioloop import IOLoop
 
 from distributed import Client, Variable, worker_client, Nanny, wait, TimeoutError
 from distributed.metrics import time
+from distributed.compatibility import WINDOWS
 from distributed.utils_test import gen_cluster, inc, div
 from distributed.utils_test import client, cluster_fixture, loop  # noqa: F401
 
@@ -83,20 +85,34 @@ def test_hold_futures(s, a, b):
 def test_timeout(c, s, a, b):
     v = Variable("v")
 
-    start = time()
+    start = IOLoop.current().time()
+    with pytest.raises(TimeoutError):
+        yield v.get(timeout=0.2)
+    stop = IOLoop.current().time()
+
+    if WINDOWS:  # timing is weird with asyncio and Windows
+        assert 0.1 < stop - start < 2.0
+    else:
+        assert 0.2 < stop - start < 2.0
+
     with pytest.raises(TimeoutError):
-        yield v.get(timeout=0.1)
-    stop = time()
-    assert 0.1 < stop - start < 2.0
+        yield v.get(timeout=0.01)
 
 
 def test_timeout_sync(client):
     v = Variable("v")
-    start = time()
+    start = IOLoop.current().time()
+    with pytest.raises(TimeoutError):
+        v.get(timeout=0.2)
+    stop = IOLoop.current().time()
+
+    if WINDOWS:
+        assert 0.1 < stop - start < 2.0
+    else:
+        assert 0.2 < stop - start < 2.0
+
     with pytest.raises(TimeoutError):
-        v.get(timeout=0.1)
-    stop = time()
-    assert 0.1 < stop - start < 2.0
+        yield v.get(timeout=0.01)
 
 
 @gen_cluster(client=True)
diff --git a/distributed/variable.py b/distributed/variable.py
index bfd6ca250e2..9024ab03d8b 100644
--- a/distributed/variable.py
+++ b/distributed/variable.py
@@ -3,17 +3,13 @@
 import logging
 import uuid
 
-import tornado.locks
-from tornado import gen
-
 try:
     from cytoolz import merge
 except ImportError:
     from toolz import merge
 
 from .client import Future, _get_global_client, Client
-from .metrics import time
-from .utils import tokey, log_errors, TimeoutError
+from .utils import tokey, log_errors, TimeoutError, ignoring
 from .worker import get_client
 
 logger = logging.getLogger(__name__)
@@ -33,8 +29,8 @@ def __init__(self, scheduler):
         self.scheduler = scheduler
         self.variables = dict()
         self.waiting = defaultdict(set)
-        self.waiting_conditions = defaultdict(tornado.locks.Condition)
-        self.started = tornado.locks.Condition()
+        self.waiting_conditions = defaultdict(asyncio.Condition)
+        self.started = asyncio.Condition()
 
         self.scheduler.handlers.update(
             {"variable_set": self.set, "variable_get": self.get}
@@ -45,7 +41,7 @@ def __init__(self, scheduler):
 
         self.scheduler.extensions["variables"] = self
 
-    def set(self, stream=None, name=None, key=None, data=None, client=None):
+    async def set(self, stream=None, name=None, key=None, data=None, client=None):
         if key is not None:
             record = {"type": "Future", "value": key}
             self.scheduler.client_desires_keys(keys=[key], client="variable-%s" % name)
@@ -59,34 +55,44 @@ def set(self, stream=None, name=None, key=None, data=None, client=None):
             if old["type"] == "Future" and old["value"] != key:
                 asyncio.ensure_future(self.release(old["value"], name))
         if name not in self.variables:
-            self.started.notify_all()
+            async with self.started:
+                self.started.notify_all()
         self.variables[name] = record
 
     async def release(self, key, name):
         while self.waiting[key, name]:
-            await self.waiting_conditions[name].wait()
+            async with self.waiting_conditions[name]:
+                await self.waiting_conditions[name].wait()
 
         self.scheduler.client_releases_keys(keys=[key], client="variable-%s" % name)
         del self.waiting[key, name]
 
-    def future_release(self, name=None, key=None, token=None, client=None):
+    async def future_release(self, name=None, key=None, token=None, client=None):
         self.waiting[key, name].remove(token)
         if not self.waiting[key, name]:
-            self.waiting_conditions[name].notify_all()
+            async with self.waiting_conditions[name]:
+                self.waiting_conditions[name].notify_all()
 
     async def get(self, stream=None, name=None, client=None, timeout=None):
-        start = time()
+        start = self.scheduler.loop.time()
         while name not in self.variables:
             if timeout is not None:
-                left = timeout - (time() - start)
+                left = timeout - (self.scheduler.loop.time() - start)
             else:
                 left = None
             if left and left < 0:
                 raise TimeoutError()
             try:
-                await self.started.wait(timeout=left)
-            except gen.TimeoutError:
-                raise TimeoutError("Timed out waiting for Variable.get")
+
+                async def _():  # Python 3.6 is odd and requires special help here
+                    await self.started.acquire()
+                    await self.started.wait()
+
+                await asyncio.wait_for(_(), timeout=left)
+            finally:
+                with ignoring(RuntimeError):  # Python 3.6 loses lock on finally clause
+                    self.started.release()
+
         record = self.variables[name]
         if record["type"] == "Future":
             key = record["value"]
diff --git a/docs/source/actors.rst b/docs/source/actors.rst
index 85d1300e502..370837629f3 100644
--- a/docs/source/actors.rst
+++ b/docs/source/actors.rst
@@ -199,7 +199,7 @@ will run on the Worker's event loop thread rather than a separate thread.
 
    def Waiter:
        def __init__(self):
-           self.event = tornado.locks.Event()
+           self.event = asyncio.Event()
 
        async def set(self):
            self.event.set()

From d9c481815e2dbfe4f4691a0f3c6071be6cf3a471 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=B8ren=20Fuglede=20J=C3=B8rgensen?= <s@fuglede.dk>
Date: Mon, 3 Feb 2020 19:06:18 +0100
Subject: [PATCH 0655/1550] Add documentation of parameters in coordination
 primitives (#3434)

---
 distributed/lock.py     |  8 ++++++--
 distributed/pubsub.py   |  8 +++++++-
 distributed/queues.py   | 12 ++++++++++++
 distributed/variable.py |  9 +++++++++
 4 files changed, 34 insertions(+), 3 deletions(-)

diff --git a/distributed/lock.py b/distributed/lock.py
index 3eceba5ce7a..3c893a419c2 100644
--- a/distributed/lock.py
+++ b/distributed/lock.py
@@ -76,9 +76,13 @@ class Lock:
 
     Parameters
     ----------
-    name: string
+    name: string (optional)
         Name of the lock to acquire.  Choosing the same name allows two
-        disconnected processes to coordinate a lock.
+        disconnected processes to coordinate a lock.  If not given, a random
+        name will be generated.
+    client: Client (optional)
+        Client to use for communication with the scheduler.  If not given, the
+        default global client will be used.
 
     Examples
     --------
diff --git a/distributed/pubsub.py b/distributed/pubsub.py
index 355fee7ae59..ca4e06c44d1 100644
--- a/distributed/pubsub.py
+++ b/distributed/pubsub.py
@@ -238,7 +238,13 @@ class Pub:
     Parameters
     ----------
     name: object (msgpack serializable)
-        The name of the group of Pubs and Subs on which to participate
+        The name of the group of Pubs and Subs on which to participate.
+    worker: Worker (optional)
+        The worker to be used for publishing data. Defaults to the value of
+        ```get_worker()```. If given, ``client`` must be ``None``.
+    client: Client (optional)
+        Client used for communication with the scheduler. Defaults to
+        the value of ``get_client()``. If given, ``worker`` must be ``None``.
 
     Examples
     --------
diff --git a/distributed/queues.py b/distributed/queues.py
index 6cdab880aa6..81262703ad4 100644
--- a/distributed/queues.py
+++ b/distributed/queues.py
@@ -141,6 +141,18 @@ class Queue:
 
        This object is experimental and has known issues in Python 2
 
+    Parameters
+    ----------
+    name: string (optional)
+        Name used by other clients and the scheduler to identify the queue. If
+        not given, a random name will be generated.
+    client: Client (optional)
+        Client used for communication with the scheduler. Defaults to the
+        value of ``_get_global_client()``.
+    maxsize: int (optional)
+        Number of items allowed in the queue. If 0 (the default), the queue
+        size is unbounded.
+
     Examples
     --------
     >>> from dask.distributed import Client, Queue  # doctest: +SKIP
diff --git a/distributed/variable.py b/distributed/variable.py
index 9024ab03d8b..fc4cc396dab 100644
--- a/distributed/variable.py
+++ b/distributed/variable.py
@@ -136,6 +136,15 @@ class Variable:
 
        This object is experimental and has known issues in Python 2
 
+    Parameters
+    ----------
+    name: string (optional)
+        Name used by other clients and the scheduler to identify the variable.
+        If not given, a random name will be generated.
+    client: Client (optional)
+        Client used for communication with the scheduler. Defaults to the
+        value of ``_get_global_client()``.
+
     Examples
     --------
     >>> from dask.distributed import Client, Variable # doctest: +SKIP

From eb8de64072a2ada03e195b5c2ea218e46abb1068 Mon Sep 17 00:00:00 2001
From: Cyril Shcherbin <cyril.shcherbin@gmail.com>
Date: Mon, 3 Feb 2020 19:11:21 +0100
Subject: [PATCH 0656/1550] Call pip as a module to avoid warnings (#3433)
 (#3436)

---
 .github/workflows/ci-docs.yaml                |  4 ++--
 .github/workflows/ci-windows.yaml             |  2 +-
 .travis.yml                                   |  6 ++---
 continuous_integration/travis/install.sh      | 24 +++++++++----------
 distributed/client.py                         |  2 +-
 distributed/dashboard/proxy.py                |  4 ++--
 .../dashboard/tests/test_scheduler_bokeh.py   |  2 +-
 distributed/protocol/keras.py                 |  2 +-
 distributed/worker.py                         |  4 +++-
 docs/source/install.rst                       |  2 +-
 docs/source/quickstart.rst                    |  2 +-
 11 files changed, 28 insertions(+), 26 deletions(-)

diff --git a/.github/workflows/ci-docs.yaml b/.github/workflows/ci-docs.yaml
index e80e07b9c33..780e2a251fd 100644
--- a/.github/workflows/ci-docs.yaml
+++ b/.github/workflows/ci-docs.yaml
@@ -17,10 +17,10 @@ jobs:
       - name: Install Distributed
         run: |
           python -m pip install --upgrade pip
-          pip install -e .
+          python -m pip install -e .
 
       - name: Install doc dependencies
-        run: pip install -r docs/requirements.txt
+        run: python -m pip install -r docs/requirements.txt
 
       - name: Build docs
         run: |
diff --git a/.github/workflows/ci-windows.yaml b/.github/workflows/ci-windows.yaml
index ecbf29f5a3d..992f65d6435 100644
--- a/.github/workflows/ci-windows.yaml
+++ b/.github/workflows/ci-windows.yaml
@@ -24,7 +24,7 @@ jobs:
 
       - name: Install distributed from source
         shell: bash -l {0}
-        run: pip install -q --no-deps -e .
+        run: python -m pip install -q --no-deps -e .
 
       - name: Run tests
         shell: bash -l {0}
diff --git a/.travis.yml b/.travis.yml
index c0c30316c9a..d00894dd3d6 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -29,11 +29,11 @@ install:
 
 script:
   - if [[ $TESTS == true ]]; then source continuous_integration/travis/run_tests.sh ; fi
-  - if [[ $LINT == true ]]; then pip install flake8 ; flake8 distributed ; fi
-  - if [[ $LINT == true ]]; then pip install black ; black distributed --check; fi
+  - if [[ $LINT == true ]]; then python -m pip install flake8 ; flake8 distributed ; fi
+  - if [[ $LINT == true ]]; then python -m pip install black ; black distributed --check; fi
 
 after_success:
-  - if [[ $COVERAGE == true ]]; then coverage report; pip install -q coveralls ; coveralls ; fi
+  - if [[ $COVERAGE == true ]]; then coverage report; python -m pip install -q coveralls ; coveralls ; fi
 
 notifications:
   email: false
diff --git a/continuous_integration/travis/install.sh b/continuous_integration/travis/install.sh
index 8c34f38d276..8eaed19df81 100644
--- a/continuous_integration/travis/install.sh
+++ b/continuous_integration/travis/install.sh
@@ -58,28 +58,28 @@ conda install -q \
 conda install -c defaults -c conda-forge libunwind zstandard asyncssh
 conda install --no-deps -c defaults -c numba -c conda-forge stacktrace
 
-pip install -q "pytest>=4" pytest-repeat pytest-faulthandler pytest-asyncio
+python -m pip install -q "pytest>=4" pytest-repeat pytest-faulthandler pytest-asyncio
 
-pip install -q git+https://github.com/dask/dask.git --upgrade --no-deps
-pip install -q git+https://github.com/joblib/joblib.git --upgrade --no-deps
-pip install -q git+https://github.com/intake/filesystem_spec.git --upgrade --no-deps
-pip install -q git+https://github.com/dask/s3fs.git --upgrade --no-deps
-pip install -q git+https://github.com/dask/zict.git --upgrade --no-deps
-pip install -q sortedcollections msgpack --no-deps
-pip install -q keras --upgrade --no-deps
-pip install -q asyncssh 
+python -m pip install -q git+https://github.com/dask/dask.git --upgrade --no-deps
+python -m pip install -q git+https://github.com/joblib/joblib.git --upgrade --no-deps
+python -m pip install -q git+https://github.com/intake/filesystem_spec.git --upgrade --no-deps
+python -m pip install -q git+https://github.com/dask/s3fs.git --upgrade --no-deps
+python -m pip install -q git+https://github.com/dask/zict.git --upgrade --no-deps
+python -m pip install -q sortedcollections msgpack --no-deps
+python -m pip install -q keras --upgrade --no-deps
+python -m pip install -q asyncssh
 
 if [[ $CRICK == true ]]; then
     conda install -q cython
-    pip install -q git+https://github.com/jcrist/crick.git
+    python -m pip install -q git+https://github.com/jcrist/crick.git
 fi;
 
 # Install distributed
-pip install --no-deps -e .
+python -m pip install --no-deps -e .
 
 # For debugging
 echo -e "--\n--Conda Environment\n--"
 conda list
 
 echo -e "--\n--Pip Environment\n--"
-pip list --format=columns
+python -m pip list --format=columns
diff --git a/distributed/client.py b/distributed/client.py
index 93f395e2dc4..fee50564963 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -4513,7 +4513,7 @@ class get_task_stream:
     A common way to do this is to upload the file as a gist, and then serve it
     on https://raw.githack.com ::
 
-       $ pip install gist
+       $ python -m pip install gist
        $ gist task-stream.html
        https://gist.github.com/8a5b3c74b10b413f612bb5e250856ceb
 
diff --git a/distributed/dashboard/proxy.py b/distributed/dashboard/proxy.py
index 89f9f87aae6..3e76ba11c0e 100644
--- a/distributed/dashboard/proxy.py
+++ b/distributed/dashboard/proxy.py
@@ -70,7 +70,7 @@ def proxy(self, port, proxied_path):
     logger.info(
         "To route to workers diagnostics web server "
         "please install jupyter-server-proxy: "
-        "pip install jupyter-server-proxy"
+        "python -m pip install jupyter-server-proxy"
     )
 
     class GlobalProxyHandler(web.RequestHandler):
@@ -94,7 +94,7 @@ def get(self, port, host, proxied_path):
                 </p>
 
                 <p><pre> conda install jupyter-server-proxy -c conda-forge </pre></p>
-                <p><pre> pip install jupyter-server-proxy</pre></p>
+                <p><pre> python -m pip install jupyter-server-proxy</pre></p>
 
                 <p>
                 The link above should work though if your workers are on a
diff --git a/distributed/dashboard/tests/test_scheduler_bokeh.py b/distributed/dashboard/tests/test_scheduler_bokeh.py
index 6594ce2142f..4ef90e48b8e 100644
--- a/distributed/dashboard/tests/test_scheduler_bokeh.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh.py
@@ -607,7 +607,7 @@ def test_proxy_to_workers(c, s, a, b):
         if proxy_exists:
             assert b"Crossfilter" in response_proxy.body
         else:
-            assert b"pip install jupyter-server-proxy" in response_proxy.body
+            assert b"python -m pip install jupyter-server-proxy" in response_proxy.body
         assert response_direct.code == 200
         assert b"Crossfilter" in response_direct.body
 
diff --git a/distributed/protocol/keras.py b/distributed/protocol/keras.py
index 020ce1cae3b..121aa0c4700 100644
--- a/distributed/protocol/keras.py
+++ b/distributed/protocol/keras.py
@@ -9,7 +9,7 @@ def serialize_keras_model(model):
 
     if keras.__version__ < "1.2.0":
         raise ImportError(
-            "Need Keras >= 1.2.0. Try pip install keras --upgrade --no-deps"
+            "Need Keras >= 1.2.0. Try python -m pip install keras --upgrade --no-deps"
         )
 
     header = model._updated_config()
diff --git a/distributed/worker.py b/distributed/worker.py
index 77711058682..e1ae8317148 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -547,7 +547,9 @@ def __init__(
             try:
                 from zict import Buffer, File, Func
             except ImportError:
-                raise ImportError("Please `pip install zict` for spill-to-disk workers")
+                raise ImportError(
+                    "Please `python -m pip install zict` for spill-to-disk workers"
+                )
             path = os.path.join(self.local_directory, "storage")
             storage = Func(
                 partial(serialize_bytelist, on_error="raise"),
diff --git a/docs/source/install.rst b/docs/source/install.rst
index 7cf4199eecd..db1bf316400 100644
--- a/docs/source/install.rst
+++ b/docs/source/install.rst
@@ -18,7 +18,7 @@ Pip
 
 Or install distributed with ``pip``::
 
-    pip install dask distributed --upgrade
+    python -m pip install dask distributed --upgrade
 
 Source
 ------
diff --git a/docs/source/quickstart.rst b/docs/source/quickstart.rst
index 4437f77a1ea..0172c376746 100644
--- a/docs/source/quickstart.rst
+++ b/docs/source/quickstart.rst
@@ -6,7 +6,7 @@ Install
 
 ::
 
-    $ pip install dask distributed --upgrade
+    $ python -m pip install dask distributed --upgrade
 
 See :doc:`installation <install>` document for more information.
 

From 812847960a40c2f31bfdb9b4b9af3a79a0c8c443 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 3 Feb 2020 16:57:03 -0800
Subject: [PATCH 0657/1550] Avoid loop= keyword in asyncio coordination
 primitives (#3437)

---
 distributed/client.py            | 12 ++++++++++--
 distributed/pubsub.py            |  9 ++++++++-
 distributed/tests/test_client.py | 14 ++++++++++++++
 3 files changed, 32 insertions(+), 3 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index fee50564963..516185cef23 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -671,7 +671,6 @@ def __init__(
         self._loop_runner = LoopRunner(loop=loop, asynchronous=asynchronous)
         self.loop = self._loop_runner.loop
 
-        self._gather_semaphore = asyncio.Semaphore(5, loop=self.loop.asyncio_loop)
         self._gather_keys = None
         self._gather_future = None
 
@@ -981,6 +980,8 @@ async def _start(self, timeout=no_default, **kwargs):
 
             address = self.cluster.scheduler_address
 
+        self._gather_semaphore = asyncio.Semaphore(5)
+
         if self.scheduler is None:
             self.scheduler = self.rpc(address)
         self.scheduler_comm = None
@@ -4206,7 +4207,6 @@ def __init__(self, futures=None, loop=None, with_results=False, raise_errors=Tru
         self.queue = pyQueue()
         self.lock = threading.Lock()
         self.loop = loop or default_client().loop
-        self.condition = asyncio.Condition(loop=self.loop.asyncio_loop)
         self.thread_condition = threading.Condition()
         self.with_results = with_results
         self.raise_errors = raise_errors
@@ -4214,6 +4214,14 @@ def __init__(self, futures=None, loop=None, with_results=False, raise_errors=Tru
         if futures:
             self.update(futures)
 
+    @property
+    def condition(self):
+        try:
+            return self._condition
+        except AttributeError:
+            self._condition = asyncio.Condition()
+            return self._condition
+
     async def _track_future(self, future):
         try:
             await _wait(future)
diff --git a/distributed/pubsub.py b/distributed/pubsub.py
index ca4e06c44d1..cdff73ffeca 100644
--- a/distributed/pubsub.py
+++ b/distributed/pubsub.py
@@ -378,7 +378,6 @@ def __init__(self, name, worker=None, client=None):
             self.loop = self.client.loop
         self.name = name
         self.buffer = deque()
-        self.condition = asyncio.Condition(loop=self.loop.asyncio_loop)
 
         if self.worker:
             pubsub = self.worker.extensions["pubsub"]
@@ -396,6 +395,14 @@ def __init__(self, name, worker=None, client=None):
 
         weakref.finalize(self, pubsub.trigger_cleanup)
 
+    @property
+    def condition(self):
+        try:
+            return self._condition
+        except AttributeError:
+            self._condition = asyncio.Condition()
+            return self._condition
+
     async def _get(self, timeout=None):
         start = time()
         while not self.buffer:
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 16e660492f8..ae03edd4faa 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -5920,3 +5920,17 @@ async def test_performance_report(c, s, a, b):
         assert "bokeh" in data
         assert "random" in data
         assert "Dask Performance Report" in data
+
+
+@pytest.mark.asyncio
+async def test_client_gather_semaphor_loop(cleanup):
+    async with Scheduler(port=0) as s:
+        async with Client(s.address, asynchronous=True) as c:
+            assert c._gather_semaphore._loop is c.loop.asyncio_loop
+
+
+@gen_cluster(client=True)
+def test_as_completed_condition_loop(c, s, a, b):
+    seq = c.map(inc, range(5))
+    ac = as_completed(seq)
+    assert ac.condition._loop == c.loop.asyncio_loop

From d134345c5c026c7472fbc0dcb0ce907ac2075e05 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Tue, 4 Feb 2020 00:19:34 -0600
Subject: [PATCH 0658/1550] Ensure scheduler updates task and worker states
 after successful worker data deletion (#3401)

---
 distributed/scheduler.py | 55 +++++++++++++++++++---------------------
 1 file changed, 26 insertions(+), 29 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 89c938f0dd5..8e5fdd17a34 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -2952,6 +2952,30 @@ async def proxy(self, comm=None, msg=None, worker=None, serializers=None):
         )
         return d[worker]
 
+    async def _delete_worker_data(self, worker_address, keys):
+        """ Delete data from a worker and update the corresponding worker/task states
+
+        Parameters
+        ----------
+        worker_address: str
+            Worker address to delete keys from
+        keys: List[str]
+            List of keys to delete on the specified worker
+        """
+        await retry_operation(
+            self.rpc(addr=worker_address).delete_data, keys=list(keys), report=False,
+        )
+
+        ws = self.workers[worker_address]
+        tasks = {self.tasks[key] for key in keys}
+        ws.has_what -= tasks
+        for ts in tasks:
+            ts.who_has.remove(ws)
+            ws.nbytes -= ts.get_nbytes()
+        self.log_event(
+            ws.address, {"action": "remove-worker-data", "keys": keys},
+        )
+
     async def rebalance(self, comm=None, keys=None, workers=None):
         """ Rebalance keys so that each worker stores roughly equal bytes
 
@@ -3068,19 +3092,9 @@ async def rebalance(self, comm=None, keys=None, workers=None):
                     )
 
                 await asyncio.gather(
-                    *(
-                        retry_operation(
-                            self.rpc(addr=r).delete_data, keys=v, report=False
-                        )
-                        for r, v in to_senders.items()
-                    )
+                    *(self._delete_worker_data(r, v) for r, v in to_senders.items())
                 )
 
-                for sender, recipient, ts in msgs:
-                    ts.who_has.remove(sender)
-                    sender.has_what.remove(ts)
-                    sender.nbytes -= ts.get_nbytes()
-
                 return {"status": "OK"}
 
     async def replicate(
@@ -3142,28 +3156,11 @@ async def replicate(
 
                 await asyncio.gather(
                     *(
-                        retry_operation(
-                            self.rpc(addr=ws.address).delete_data,
-                            keys=[ts.key for ts in tasks],
-                            report=False,
-                        )
+                        self._delete_worker_data(ws.address, [t.key for t in tasks])
                         for ws, tasks in del_worker_tasks.items()
                     )
                 )
 
-                for ws, tasks in del_worker_tasks.items():
-                    ws.has_what -= tasks
-                    for ts in tasks:
-                        ts.who_has.remove(ws)
-                        ws.nbytes -= ts.get_nbytes()
-                    self.log_event(
-                        ws.address,
-                        {
-                            "action": "replicate-remove",
-                            "keys": [ts.key for ts in tasks],
-                        },
-                    )
-
             # Copy not-yet-filled data
             while tasks:
                 gathers = defaultdict(dict)

From 3d454d719cda99e2fc0808d820a1a02e96a6260d Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Tue, 4 Feb 2020 12:01:12 -0600
Subject: [PATCH 0659/1550] Update worker_kwargs description in LocalCluster
 constructor [skip ci] (#3438)

---
 distributed/deploy/local.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index 8eb55c54997..d1744ed32c0 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -49,8 +49,6 @@ class LocalCluster(SpecCluster):
     asynchronous: bool (False by default)
         Set to True if using this cluster within async/await functions or within
         Tornado gen.coroutines.  This should remain False for normal use.
-    worker_kwargs: dict
-        Extra worker arguments, will be passed to the Worker constructor.
     blocked_handlers: List[str]
         A list of strings specifying a blacklist of handlers to disallow on the Scheduler,
         like ``['feed', 'run_function']``
@@ -68,6 +66,9 @@ class LocalCluster(SpecCluster):
         Network interface to use.  Defaults to lo/localhost
     worker_class: Worker
         Worker class used to instantiate workers from.
+    **worker_kwargs:
+        Extra worker arguments. Any additional keyword arguments will be passed
+        to the ``Worker`` class constructor.
 
     Examples
     --------

From f87e802102da07fce7c9958e5926d18a143b0fd8 Mon Sep 17 00:00:00 2001
From: Peter Andreas Entschev <peter@entschev.com>
Date: Wed, 5 Feb 2020 12:05:00 +0100
Subject: [PATCH 0660/1550] Checks for command parameters in ssh2 (#3078)

* Allow specification of worker type in SSHCLuster

* Default the worker_module to None and check further down

* Avoid ssh2 parameters from getting overwritten by superclass

* Check for a command's parameters in cli_keywords

* Pass ssh2's worker_module to cli_keywords

* Removed duplicate worker_module from ssh2

This was a result of a merge conflict that passed unnoticed

* Fix utils.py formatting

* Add tests for cli_keywords command options

* Add new command_has_keyword function, simplify cli_keywords

* Avoid ssh attributes from getting overwritten by superclass

* Pass ssh command parameters to cli_keywords

* Add test for cli_keywords command options

* Remove check for non-Worker class attribute "nprocs"

The "nprocs" is a command-line argument, and not an attribute
from the Worker class, thus it cannot be asserted for

* Improve documentation for cmd= in cli_keywords

Co-authored-by: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
---
 distributed/deploy/ssh.py            | 10 ++---
 distributed/deploy/tests/test_ssh.py | 25 ++++++++++++-
 distributed/utils.py                 | 55 +++++++++++++++++++++++++---
 3 files changed, 78 insertions(+), 12 deletions(-)

diff --git a/distributed/deploy/ssh.py b/distributed/deploy/ssh.py
index 1f49f187a14..a7b3526bcba 100644
--- a/distributed/deploy/ssh.py
+++ b/distributed/deploy/ssh.py
@@ -73,6 +73,8 @@ def __init__(
         loop=None,
         name=None,
     ):
+        super().__init__()
+
         self.address = address
         self.scheduler = scheduler
         self.worker_module = worker_module
@@ -80,8 +82,6 @@ def __init__(
         self.kwargs = kwargs
         self.name = name
 
-        super().__init__()
-
     async def start(self):
         import asyncssh  # import now to avoid adding to module startup time
 
@@ -98,7 +98,7 @@ async def start(self):
                     "--name",
                     str(self.name),
                 ]
-                + cli_keywords(self.kwargs, cls=_Worker)
+                + cli_keywords(self.kwargs, cls=_Worker, cmd=self.worker_module)
             )
         )
 
@@ -131,12 +131,12 @@ class Scheduler(Process):
     """
 
     def __init__(self, address: str, connect_options: dict, kwargs: dict):
+        super().__init__()
+
         self.address = address
         self.kwargs = kwargs
         self.connect_options = connect_options
 
-        super().__init__()
-
     async def start(self):
         import asyncssh  # import now to avoid adding to module startup time
 
diff --git a/distributed/deploy/tests/test_ssh.py b/distributed/deploy/tests/test_ssh.py
index 376b0eae3a4..eff7bf05a11 100644
--- a/distributed/deploy/tests/test_ssh.py
+++ b/distributed/deploy/tests/test_ssh.py
@@ -32,7 +32,12 @@ async def test_keywords():
         ["127.0.0.1"] * 3,
         connect_options=dict(known_hosts=None),
         asynchronous=True,
-        worker_options={"nthreads": 2, "memory_limit": "2 GiB", "death_timeout": "5s"},
+        worker_options={
+            "nprocs": 2,  # nprocs checks custom arguments with cli_keywords
+            "nthreads": 2,
+            "memory_limit": "2 GiB",
+            "death_timeout": "5s",
+        },
         scheduler_options={"idle_timeout": "5s", "port": 0},
     ) as cluster:
         async with Client(cluster, asynchronous=True) as client:
@@ -74,3 +79,21 @@ def f(x):
             async with Client(cluster, asynchronous=True) as client:
                 result = await client.submit(f, 1)
                 assert result == 101
+
+
+@pytest.mark.asyncio
+async def test_unimplemented_options():
+    with pytest.raises(Exception):
+        async with SSHCluster(
+            ["127.0.0.1"] * 3,
+            connect_kwargs=dict(known_hosts=None),
+            asynchronous=True,
+            worker_kwargs={
+                "nthreads": 2,
+                "memory_limit": "2 GiB",
+                "death_timeout": "5s",
+                "unimplemented_option": 2,
+            },
+            scheduler_kwargs={"idle_timeout": "5s", "port": 0},
+        ) as cluster:
+            assert cluster
diff --git a/distributed/utils.py b/distributed/utils.py
index 086555643ea..09c9c62cd1e 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -1,6 +1,7 @@
 import asyncio
 from asyncio import TimeoutError
 import atexit
+import click
 from collections import deque, OrderedDict, UserDict
 from concurrent.futures import ThreadPoolExecutor
 from contextlib import contextmanager
@@ -1223,6 +1224,32 @@ def has_keyword(func, keyword):
     return keyword in inspect.signature(func).parameters
 
 
+@functools.lru_cache(1000)
+def command_has_keyword(cmd, k):
+    if cmd is not None:
+        if isinstance(cmd, str):
+            try:
+                from importlib import import_module
+
+                cmd = import_module(cmd)
+            except ImportError:
+                raise ImportError("Module for command %s is not available" % cmd)
+
+        if isinstance(getattr(cmd, "main"), click.core.Command):
+            cmd = cmd.main
+        if isinstance(cmd, click.core.Command):
+            cmd_params = set(
+                [
+                    p.human_readable_name
+                    for p in cmd.params
+                    if isinstance(p, click.core.Option)
+                ]
+            )
+            return k in cmd_params
+
+    return False
+
+
 # from bokeh.palettes import viridis
 # palette = viridis(18)
 palette = [
@@ -1324,7 +1351,7 @@ def _repr_html_(self):
         return "\n".join(summaries)
 
 
-def cli_keywords(d: dict, cls=None):
+def cli_keywords(d: dict, cls=None, cmd=None):
     """ Convert a kwargs dictionary into a list of CLI keywords
 
     Parameters
@@ -1333,6 +1360,12 @@ def cli_keywords(d: dict, cls=None):
         The keywords to convert
     cls: callable
         The callable that consumes these terms to check them for validity
+    cmd: string or object
+        A string with the name of a module, or the module containing a
+        click-generated command with a "main" function, or the function itself.
+        It may be used to parse a module's custom arguments (i.e., arguments that
+        are not part of Worker class), such as nprocs from dask-worker CLI or
+        enable_nvlink from dask-cuda-worker CLI.
 
     Examples
     --------
@@ -1345,12 +1378,22 @@ def cli_keywords(d: dict, cls=None):
     ...
     ValueError: Class distributed.worker.Worker does not support keyword x
     """
-    if cls:
+    if cls or cmd:
         for k in d:
-            if not has_keyword(cls, k):
-                raise ValueError(
-                    "Class %s does not support keyword %s" % (typename(cls), k)
-                )
+            if not has_keyword(cls, k) and not command_has_keyword(cmd, k):
+                if cls and cmd:
+                    raise ValueError(
+                        "Neither class %s or module %s support keyword %s"
+                        % (typename(cls), typename(cmd), k)
+                    )
+                elif cls:
+                    raise ValueError(
+                        "Class %s does not support keyword %s" % (typename(cls), k)
+                    )
+                else:
+                    raise ValueError(
+                        "Module %s does not support keyword %s" % (typename(cmd), k)
+                    )
 
     def convert_value(v):
         out = str(v)

From ef5317feeae72cd23dd25f22ce278a3a91212e49 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Wed, 5 Feb 2020 07:45:12 -0800
Subject: [PATCH 0661/1550] Fix name of Numba serialization test (#3447)

---
 distributed/protocol/tests/test_numba.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/protocol/tests/test_numba.py b/distributed/protocol/tests/test_numba.py
index 69ea73310d4..78e2b1859b4 100644
--- a/distributed/protocol/tests/test_numba.py
+++ b/distributed/protocol/tests/test_numba.py
@@ -6,7 +6,7 @@
 
 
 @pytest.mark.parametrize("dtype", ["u1", "u4", "u8", "f4"])
-def test_serialize_cupy(dtype):
+def test_serialize_numba(dtype):
     if not cuda.is_available():
         pytest.skip("CUDA is not available")
 

From 88c354f68761b60c11ba5613806891f78810da0c Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Wed, 5 Feb 2020 10:58:31 -0800
Subject: [PATCH 0662/1550] Adjust `numba.cuda` import and add check (#3446)

* Import `numba.cuda` instead of just `numba`

Appears that an error will be raised when accessing `numba.cuda` unless
`numba.cuda` is imported as well. So go ahead and import `numba.cuda`
too.

* Check that CUDA is available in CuPy/Numba test
---
 distributed/protocol/tests/test_cupy.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/distributed/protocol/tests/test_cupy.py b/distributed/protocol/tests/test_cupy.py
index 57d26ae679b..f7feb4da5e2 100644
--- a/distributed/protocol/tests/test_cupy.py
+++ b/distributed/protocol/tests/test_cupy.py
@@ -17,12 +17,15 @@ def test_serialize_cupy(size, dtype):
 
 @pytest.mark.parametrize("dtype", ["u1", "u4", "u8", "f4"])
 def test_serialize_cupy_from_numba(dtype):
-    numba = pytest.importorskip("numba")
+    cuda = pytest.importorskip("numba.cuda")
     np = pytest.importorskip("numpy")
 
+    if not cuda.is_available():
+        pytest.skip("CUDA is not available")
+
     size = 10
     x_np = np.arange(size, dtype=dtype)
-    x = numba.cuda.to_device(x_np)
+    x = cuda.to_device(x_np)
     header, frames = serialize(x, serializers=("cuda", "dask", "pickle"))
     header["type-serialized"] = pickle.dumps(cupy.ndarray)
 

From 105d040fb9cabfa3581dcdce73165c77cb22de9a Mon Sep 17 00:00:00 2001
From: Alex Adamson <alex.b.adamson@gmail.com>
Date: Wed, 5 Feb 2020 17:07:15 -0500
Subject: [PATCH 0663/1550] Ensure __causes__s of exceptions raised on workers
 are serialized (#3430)

---
 distributed/core.py              | 10 ++++++++++
 distributed/tests/test_worker.py | 26 ++++++++++++++++++++++++++
 2 files changed, 36 insertions(+)

diff --git a/distributed/core.py b/distributed/core.py
index 78dd618e8e2..ac9be6728fc 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -10,6 +10,7 @@
 import weakref
 
 import dask
+import tblib
 from toolz import merge
 from tornado import gen
 from tornado.ioloop import IOLoop
@@ -981,6 +982,14 @@ def coerce_to_address(o):
     return normalize_address(o)
 
 
+def collect_causes(e):
+    causes = []
+    while e.__cause__ is not None:
+        causes.append(e.__cause__)
+        e = e.__cause__
+    return causes
+
+
 def error_message(e, status="error"):
     """ Produce message to send back given an exception has occurred
 
@@ -997,6 +1006,7 @@ def error_message(e, status="error"):
     clean_exception: deserialize and unpack message into exception/traceback
     """
     MAX_ERROR_LEN = dask.config.get("distributed.admin.max-error-length")
+    tblib.pickling_support.install(e, *collect_causes(e))
     tb = get_traceback()
     e2 = truncate_exception(e, MAX_ERROR_LEN)
     try:
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 6c1a0805817..55dc7faf417 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -372,6 +372,32 @@ def __str__(self):
         assert len(msg["text"]) > 10100  # default + 100
 
 
+@gen_cluster(client=True)
+def test_chained_error_message(c, s, a, b):
+    def chained_exception_fn():
+        class MyException(Exception):
+            def __init__(self, msg):
+                self.msg = msg
+
+            def __str__(self):
+                return "MyException(%s)" % self.msg
+
+        exception = MyException("Foo")
+        inner_exception = MyException("Bar")
+
+        try:
+            raise inner_exception
+        except Exception as e:
+            raise exception from e
+
+    f = c.submit(chained_exception_fn)
+    try:
+        yield f
+    except Exception as e:
+        assert e.__cause__ is not None
+        assert "Bar" in str(e.__cause__)
+
+
 @gen_cluster()
 def test_gather(s, a, b):
     b.data["x"] = 1

From 9af811d8f9858c63b9586bcfb78ce2dec8f5d6b3 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Thu, 6 Feb 2020 21:19:02 -0800
Subject: [PATCH 0664/1550] Rerun `black` on the code base (#3444)

---
 distributed/scheduler.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 8e5fdd17a34..5cfc2d7b9cb 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -2963,7 +2963,7 @@ async def _delete_worker_data(self, worker_address, keys):
             List of keys to delete on the specified worker
         """
         await retry_operation(
-            self.rpc(addr=worker_address).delete_data, keys=list(keys), report=False,
+            self.rpc(addr=worker_address).delete_data, keys=list(keys), report=False
         )
 
         ws = self.workers[worker_address]
@@ -2972,9 +2972,7 @@ async def _delete_worker_data(self, worker_address, keys):
         for ts in tasks:
             ts.who_has.remove(ws)
             ws.nbytes -= ts.get_nbytes()
-        self.log_event(
-            ws.address, {"action": "remove-worker-data", "keys": keys},
-        )
+        self.log_event(ws.address, {"action": "remove-worker-data", "keys": keys})
 
     async def rebalance(self, comm=None, keys=None, workers=None):
         """ Rebalance keys so that each worker stores roughly equal bytes

From 7bb23d5f3c586afdd5e03d9754ebe194b8491bb0 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Fri, 7 Feb 2020 12:56:24 -0600
Subject: [PATCH 0665/1550] Always add new TaskGroup to TaskPrefix (#3322)

---
 distributed/dashboard/scheduler.py            |  6 ++-
 .../tests/test_scheduler_bokeh_html.py        |  2 +-
 distributed/scheduler.py                      | 28 +++++++-----
 distributed/tests/test_scheduler.py           | 45 ++++++++++++++++++-
 distributed/utils.py                          |  4 +-
 5 files changed, 69 insertions(+), 16 deletions(-)

diff --git a/distributed/dashboard/scheduler.py b/distributed/dashboard/scheduler.py
index a030ba434f7..17e150e8df9 100644
--- a/distributed/dashboard/scheduler.py
+++ b/distributed/dashboard/scheduler.py
@@ -292,7 +292,11 @@ def collect(self):
 
         yield CounterMetricFamily(
             "dask_scheduler_tasks_forgotten",
-            "Total number of processed tasks no longer in memory and already removed from the scheduler job queue.",
+            (
+                "Total number of processed tasks no longer in memory and already "
+                "removed from the scheduler job queue. Note task groups on the "
+                "scheduler which have all tasks in the forgotten state are not included."
+            ),
             value=task_counter.get("forgotten", 0.0),
         )
 
diff --git a/distributed/dashboard/tests/test_scheduler_bokeh_html.py b/distributed/dashboard/tests/test_scheduler_bokeh_html.py
index 39da4730a28..de71b12a0d1 100644
--- a/distributed/dashboard/tests/test_scheduler_bokeh_html.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh_html.py
@@ -171,7 +171,7 @@ async def fetch_metrics():
     active_metrics, forgotten_tasks = await fetch_metrics()
     assert active_metrics.keys() == expected
     assert sum(active_metrics.values()) == 0.0
-    assert sum(forgotten_tasks) == 1.0
+    assert sum(forgotten_tasks) == 0.0
 
 
 @gen_cluster(
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 5cfc2d7b9cb..9dcaae16397 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -835,7 +835,7 @@ def duration(self):
 
     @property
     def types(self):
-        return set.union(*[tg.types for tg in self.groups])
+        return set().union(*[tg.types for tg in self.groups])
 
 
 class _StateLegacyMapping(Mapping):
@@ -1976,20 +1976,21 @@ def new_task(self, key, spec, state):
         """ Create a new task, and associated states """
         ts = TaskState(key, spec)
         ts._state = state
-        try:
-            tg = self.task_groups[ts.group_key]
-        except KeyError:
-            tg = self.task_groups[ts.group_key] = TaskGroup(ts.group_key)
-        tg.add(ts)
         prefix_key = key_split(key)
         try:
             tp = self.task_prefixes[prefix_key]
         except KeyError:
-            tp = TaskPrefix(prefix_key)
-            tp.groups.append(tg)
-            self.task_prefixes[prefix_key] = tp
+            tp = self.task_prefixes[prefix_key] = TaskPrefix(prefix_key)
         ts.prefix = tp
-        tg.prefix = tp
+
+        group_key = ts.group_key
+        try:
+            tg = self.task_groups[group_key]
+        except KeyError:
+            tg = self.task_groups[group_key] = TaskGroup(group_key)
+            tg.prefix = tp
+            tp.groups.append(tg)
+        tg.add(ts)
         self.tasks[key] = ts
         return ts
 
@@ -4642,6 +4643,13 @@ def transition(self, key, finish, *args, **kwargs):
                 if ts.state == "forgotten":
                     del self.tasks[ts.key]
 
+            if ts.state == "forgotten":
+                # Remove TaskGroup if all tasks are in the forgotten state
+                tg = ts.group
+                if not any(tg.states.get(s) for s in ALL_TASK_STATES):
+                    ts.prefix.groups.remove(tg)
+                    del self.task_groups[tg.name]
+
             return recommendations
         except Exception as e:
             logger.exception("Error transitioning %r from %r to %r", key, start, finish)
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 3ce681ea546..5c4d8cbc23e 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -1774,6 +1774,8 @@ async def test_task_groups(c, s, a, b):
 
     await c.replicate(y)
     assert tg.nbytes_in_memory == y.nbytes
+    assert "array" in str(tg.types)
+    assert "array" in str(tp.types)
 
     del y
 
@@ -1782,8 +1784,9 @@ async def test_task_groups(c, s, a, b):
 
     assert tg.nbytes_in_memory == 0
     assert tg.states["forgotten"] == 5
-    assert "array" in str(tg.types)
-    assert "array" in str(tp.types)
+    # Ensure TaskGroup is removed once all tasks are in forgotten state
+    assert tg.name not in s.task_groups
+    assert sys.getrefcount(tg) == 2
 
 
 @gen_cluster(client=True)
@@ -1795,6 +1798,44 @@ async def test_task_prefix(c, s, a, b):
 
     assert s.task_prefixes["sum-aggregate"].states["memory"] == 1
 
+    a = da.arange(101, chunks=(20,))
+    b = (a + 1).sum().persist()
+    b = await b
+
+    assert s.task_prefixes["sum-aggregate"].states["memory"] == 2
+
+
+@gen_cluster(client=True)
+async def test_task_group_non_tuple_key(c, s, a, b):
+    da = pytest.importorskip("dask.array")
+    np = pytest.importorskip("numpy")
+    x = da.arange(100, chunks=(20,))
+    y = (x + 1).sum().persist()
+    y = await y
+
+    assert s.task_prefixes["sum"].states["released"] == 4
+    assert "sum" not in s.task_groups
+
+    f = c.submit(np.sum, [1, 2, 3])
+    await f
+
+    assert s.task_prefixes["sum"].states["released"] == 4
+    assert s.task_prefixes["sum"].states["memory"] == 1
+    assert "sum" in s.task_groups
+
+
+@gen_cluster(client=True)
+async def test_task_unique_groups(c, s, a, b):
+    """ This test ensure that task groups remain unique when using submit
+    """
+    x = c.submit(sum, [1, 2])
+    y = c.submit(len, [1, 2])
+    z = c.submit(sum, [3, 4])
+    await asyncio.wait([x, y, z])
+
+    assert s.task_prefixes["len"].states["memory"] == 1
+    assert s.task_prefixes["sum"].states["memory"] == 2
+
 
 class BrokenComm(Comm):
     peer_address = None
diff --git a/distributed/utils.py b/distributed/utils.py
index 09c9c62cd1e..a771a3b280d 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -650,11 +650,11 @@ def key_split_group(x):
         elif x[0] == "<":
             return x.strip("<>").split()[0].split(".")[-1]
         else:
-            return ""
+            return key_split(x)
     elif typ is bytes:
         return key_split_group(x.decode())
     else:
-        return ""
+        return key_split(x)
 
 
 @contextmanager

From fdf1ece3081f716f880e6b1aa7765211a40821af Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Fri, 7 Feb 2020 11:54:22 -0800
Subject: [PATCH 0666/1550] Support serializing/deserializing
 `rmm.DeviceBuffer`s (#3442)

* Serialize and deserialize RMM `DeviceBuffer`'s

* Register RMM serializers and deserializers

* Test RMM serialization/deserialization

* Test serializing RMM `DeviceBuffer` from Numba

* Test deserializing a CuPy array from RMM data

* Test deserializing a Numba array from RMM data

* Fix some minor formatting issues

* Drop unneeded `cuda.as_cuda_array` call

This should already be deserialized as a Numba array. So go ahead and
treat it as such.

* Use namespace with `DeviceBuffer`

* Assert that `arr` is a `rmm.DeviceBuffer`

If we got here, we should already have an `rmm.DeviceBuffer` as that is
what we would have allocated. If that's not the case, something very
wrong has happened. So just `assert` that is true (erroring otherwise).

* Drop unneeded Numba-backed RMM serialization test

As RMM is used preferentially for allocations when available, there
shouldn't be a case where we need to serialize an RMM `DeviceBuffer`
from a Numba array. So drop this test.

* Drop unused import
---
 distributed/protocol/__init__.py         |  6 ++++++
 distributed/protocol/rmm.py              | 23 +++++++++++++++++++++++
 distributed/protocol/tests/test_cupy.py  | 20 ++++++++++++++++++++
 distributed/protocol/tests/test_numba.py | 24 ++++++++++++++++++++++++
 distributed/protocol/tests/test_rmm.py   | 22 ++++++++++++++++++++++
 5 files changed, 95 insertions(+)
 create mode 100644 distributed/protocol/rmm.py
 create mode 100644 distributed/protocol/tests/test_rmm.py

diff --git a/distributed/protocol/__init__.py b/distributed/protocol/__init__.py
index 30ae3935498..84ee9420c78 100644
--- a/distributed/protocol/__init__.py
+++ b/distributed/protocol/__init__.py
@@ -80,6 +80,12 @@ def _register_numba():
     from . import numba
 
 
+@cuda_serialize.register_lazy("rmm")
+@cuda_deserialize.register_lazy("rmm")
+def _register_rmm():
+    from . import rmm
+
+
 @cuda_serialize.register_lazy("cudf")
 @cuda_deserialize.register_lazy("cudf")
 def _register_cudf():
diff --git a/distributed/protocol/rmm.py b/distributed/protocol/rmm.py
new file mode 100644
index 00000000000..cdf22f8218f
--- /dev/null
+++ b/distributed/protocol/rmm.py
@@ -0,0 +1,23 @@
+import rmm
+from .cuda import cuda_serialize, cuda_deserialize
+
+
+# Used for RMM 0.11.0+ otherwise Numba serializers used
+if hasattr(rmm, "DeviceBuffer"):
+
+    @cuda_serialize.register(rmm.DeviceBuffer)
+    def serialize_rmm_device_buffer(x):
+        header = x.__cuda_array_interface__.copy()
+        frames = [x]
+        return header, frames
+
+    @cuda_deserialize.register(rmm.DeviceBuffer)
+    def deserialize_rmm_device_buffer(header, frames):
+        (arr,) = frames
+
+        # We should already have `DeviceBuffer`
+        # as RMM is used preferably for allocations
+        # when it is available (as it is here).
+        assert isinstance(arr, rmm.DeviceBuffer)
+
+        return arr
diff --git a/distributed/protocol/tests/test_cupy.py b/distributed/protocol/tests/test_cupy.py
index f7feb4da5e2..d2965d3af3f 100644
--- a/distributed/protocol/tests/test_cupy.py
+++ b/distributed/protocol/tests/test_cupy.py
@@ -32,3 +32,23 @@ def test_serialize_cupy_from_numba(dtype):
     y = deserialize(header, frames, deserializers=("cuda", "dask", "pickle", "error"))
 
     assert (x_np == cupy.asnumpy(y)).all()
+
+
+@pytest.mark.parametrize("size", [0, 3, 10])
+def test_serialize_cupy_from_rmm(size):
+    np = pytest.importorskip("numpy")
+    rmm = pytest.importorskip("rmm")
+
+    x_np = np.arange(size, dtype="u1")
+
+    x_np_desc = x_np.__array_interface__
+    (x_np_ptr, _) = x_np_desc["data"]
+    (x_np_size,) = x_np_desc["shape"]
+    x = rmm.DeviceBuffer(ptr=x_np_ptr, size=x_np_size)
+
+    header, frames = serialize(x, serializers=("cuda", "dask", "pickle"))
+    header["type-serialized"] = pickle.dumps(cupy.ndarray)
+
+    y = deserialize(header, frames, deserializers=("cuda", "dask", "pickle", "error"))
+
+    assert (x_np == cupy.asnumpy(y)).all()
diff --git a/distributed/protocol/tests/test_numba.py b/distributed/protocol/tests/test_numba.py
index 78e2b1859b4..21b722fb1b0 100644
--- a/distributed/protocol/tests/test_numba.py
+++ b/distributed/protocol/tests/test_numba.py
@@ -1,4 +1,5 @@
 from distributed.protocol import serialize, deserialize
+import pickle
 import pytest
 
 cuda = pytest.importorskip("numba.cuda")
@@ -20,3 +21,26 @@ def test_serialize_numba(dtype):
     x.copy_to_host(hx)
     y.copy_to_host(hy)
     assert (hx == hy).all()
+
+
+@pytest.mark.parametrize("size", [0, 3, 10])
+def test_serialize_numba_from_rmm(size):
+    np = pytest.importorskip("numpy")
+    rmm = pytest.importorskip("rmm")
+
+    if not cuda.is_available():
+        pytest.skip("CUDA is not available")
+
+    x_np = np.arange(size, dtype="u1")
+
+    x_np_desc = x_np.__array_interface__
+    (x_np_ptr, _) = x_np_desc["data"]
+    (x_np_size,) = x_np_desc["shape"]
+    x = rmm.DeviceBuffer(ptr=x_np_ptr, size=x_np_size)
+
+    header, frames = serialize(x, serializers=("cuda", "dask", "pickle"))
+    header["type-serialized"] = pickle.dumps(cuda.devicearray.DeviceNDArray)
+
+    y = deserialize(header, frames, deserializers=("cuda", "dask", "pickle", "error"))
+
+    assert (x_np == y.copy_to_host()).all()
diff --git a/distributed/protocol/tests/test_rmm.py b/distributed/protocol/tests/test_rmm.py
new file mode 100644
index 00000000000..eff3325289e
--- /dev/null
+++ b/distributed/protocol/tests/test_rmm.py
@@ -0,0 +1,22 @@
+from distributed.protocol import serialize, deserialize
+import pytest
+
+numpy = pytest.importorskip("numpy")
+cuda = pytest.importorskip("numba.cuda")
+rmm = pytest.importorskip("rmm")
+
+
+@pytest.mark.parametrize("size", [0, 3, 10])
+def test_serialize_rmm_device_buffer(size):
+    if not hasattr(rmm, "DeviceBuffer"):
+        pytest.skip("RMM pre-0.11.0 does not have DeviceBuffer")
+
+    x_np = numpy.arange(size, dtype="u1")
+    x = rmm.DeviceBuffer(size=size)
+    cuda.to_device(x_np, to=cuda.as_cuda_array(x))
+
+    header, frames = serialize(x, serializers=("cuda", "dask", "pickle"))
+    y = deserialize(header, frames, deserializers=("cuda", "dask", "pickle", "error"))
+    y_np = y.copy_to_host()
+
+    assert (x_np == y_np).all()

From 2a1ed3831c69c6acd256336c6f439f15d77fee95 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sun, 9 Feb 2020 16:27:46 -0800
Subject: [PATCH 0667/1550] Skip test_open_close_many_workers on Python 3.6
 (#3459)

This test has caused intermittent failures in the past.
Previously we had marked it as xfail, but it would still cause CI to
break because it would cause things to hang.

In #3419 it was observed that the failure seems to only occur on Python
3.6.  This commit changes the universal xfail to a skipif for Python 3.6
and below.

We still don't know what causes the failure, other than that GC seems to
take up all of the CPU time

Fixes #3419
---
 distributed/tests/test_client.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index ae03edd4faa..9879c9ff408 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -3581,7 +3581,7 @@ def test_reconnect_timeout(c, s):
 
 @pytest.mark.slow
 @pytest.mark.skipif(WINDOWS, reason="num_fds not supported on windows")
-@pytest.mark.xfail(reason="TODO: intermittent failures")
+@pytest.mark.skipif(sys.version_info < (3, 7), reason="TODO: intermittent failures")
 @pytest.mark.parametrize("worker,count,repeat", [(Worker, 100, 5), (Nanny, 10, 20)])
 def test_open_close_many_workers(loop, worker, count, repeat):
     psutil = pytest.importorskip("psutil")

From 0af1f4ff4f7c49cb365045011e0f565c0516b1e1 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 10 Feb 2020 07:21:27 -0800
Subject: [PATCH 0668/1550] Include code and summary in performance report
 (#3462)

---
 distributed/client.py            | 17 +++++++++-----
 distributed/scheduler.py         | 38 ++++++++++++++++++++++++++++++--
 distributed/tests/test_client.py |  1 +
 3 files changed, 49 insertions(+), 7 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 516185cef23..36c487dc334 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -9,7 +9,7 @@
 import errno
 from functools import partial
 import html
-from inspect import isawaitable
+import inspect
 import itertools
 import json
 import logging
@@ -1189,7 +1189,7 @@ async def _handle_report(self):
                         try:
                             handler = self._stream_handlers[op]
                             result = handler(**msg)
-                            if isawaitable(result):
+                            if inspect.isawaitable(result):
                                 await result
                         except Exception as e:
                             logger.exception(e)
@@ -4593,8 +4593,13 @@ async def __aenter__(self):
         self.start = time()
         await get_client().get_task_stream(start=0, stop=0)  # ensure plugin
 
-    async def __aexit__(self, typ, value, traceback):
-        data = await get_client().scheduler.performance_report(start=self.start)
+    async def __aexit__(self, typ, value, traceback, code=None):
+        if not code:
+            frame = inspect.currentframe().f_back
+            code = inspect.getsource(frame)
+        data = await get_client().scheduler.performance_report(
+            start=self.start, code=code
+        )
         with open(self.filename, "w") as f:
             f.write(data)
 
@@ -4602,7 +4607,9 @@ def __enter__(self):
         get_client().sync(self.__aenter__)
 
     def __exit__(self, typ, value, traceback):
-        get_client().sync(self.__aexit__, type, value, traceback)
+        frame = inspect.currentframe().f_back
+        code = inspect.getsource(frame)
+        get_client().sync(self.__aexit__, type, value, traceback, code=code)
 
 
 @contextmanager
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 9dcaae16397..b332b92349d 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -59,6 +59,8 @@
     key_split_group,
     empty_context,
     tmpfile,
+    format_bytes,
+    format_time,
     TimeoutError,
 )
 from .utils_comm import scatter_to_workers, gather_from_workers, retry_operation
@@ -4982,7 +4984,7 @@ async def get_profile_metadata(
 
         return {"counts": counts, "keys": keys}
 
-    async def performance_report(self, comm=None, start=None):
+    async def performance_report(self, comm=None, start=None, code=""):
         # Profiles
         compute, scheduler, workers = await asyncio.gather(
             *[
@@ -5021,8 +5023,39 @@ def profile_to_figure(state):
         bandwidth_types = BandwidthTypes(self, sizing_mode="stretch_both")
         bandwidth_types.update()
 
-        from bokeh.models import Panel, Tabs
+        from bokeh.models import Panel, Tabs, Div
 
+        # HTML
+        html = """
+        <h1> Dask Performance Report </h1>
+
+        <i> Select different tabs on the top for additional information </i>
+
+        <h2> Duration: {time} </h2>
+
+        <h2> Scheduler Information </h2>
+        <ul>
+          <li> Address: {address} </li>
+          <li> Workers: {nworkers} </li>
+          <li> Threads: {threads} </li>
+          <li> Memory: {memory} </li>
+        </ul>
+
+        <h2> Calling Code </h2>
+        <pre>
+{code}
+        </pre>
+        """.format(
+            time=format_time(time() - start),
+            address=self.address,
+            nworkers=len(self.workers),
+            threads=sum(w.nthreads for w in self.workers.values()),
+            memory=format_bytes(sum(w.memory_limit for w in self.workers.values())),
+            code=code,
+        )
+        html = Div(text=html)
+
+        html = Panel(child=html, title="Summary")
         compute = Panel(child=compute, title="Worker Profile (compute)")
         workers = Panel(child=workers, title="Worker Profile (administrative)")
         scheduler = Panel(child=scheduler, title="Scheduler Profile (administrative)")
@@ -5034,6 +5067,7 @@ def profile_to_figure(state):
 
         tabs = Tabs(
             tabs=[
+                html,
                 task_stream,
                 compute,
                 workers,
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 9879c9ff408..8abbd89386d 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -5920,6 +5920,7 @@ async def test_performance_report(c, s, a, b):
         assert "bokeh" in data
         assert "random" in data
         assert "Dask Performance Report" in data
+        assert "x = da.random" in data
 
 
 @pytest.mark.asyncio

From 9d79de1abf28e3784ce03f6622fd07ab1131e0be Mon Sep 17 00:00:00 2001
From: Tom Augspurger <TomAugspurger@users.noreply.github.com>
Date: Mon, 10 Feb 2020 09:36:11 -0600
Subject: [PATCH 0669/1550] Workaround RecursionError on profile data (#3455)

---
 distributed/comm/utils.py                    |  8 +++++++-
 distributed/protocol/tests/test_serialize.py | 15 +++++++++++++++
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/distributed/comm/utils.py b/distributed/comm/utils.py
index 4862aace207..b75663a14f2 100644
--- a/distributed/comm/utils.py
+++ b/distributed/comm/utils.py
@@ -1,4 +1,5 @@
 import logging
+import math
 import socket
 
 import dask
@@ -37,7 +38,12 @@ def _to_frames():
             logger.exception(e)
             raise
 
-    if FRAME_OFFLOAD_THRESHOLD and sizeof(msg) > FRAME_OFFLOAD_THRESHOLD:
+    try:
+        msg_size = sizeof(msg)
+    except RecursionError:
+        msg_size = math.inf
+
+    if FRAME_OFFLOAD_THRESHOLD and msg_size > FRAME_OFFLOAD_THRESHOLD:
         return await offload(_to_frames)
     else:
         return _to_frames()
diff --git a/distributed/protocol/tests/test_serialize.py b/distributed/protocol/tests/test_serialize.py
index b5a202f1520..caf1bbe0ad5 100644
--- a/distributed/protocol/tests/test_serialize.py
+++ b/distributed/protocol/tests/test_serialize.py
@@ -359,3 +359,18 @@ def dumps(f):
         deserialize(*serialize(Foo()))
 
     assert "Hello-123" in str(info.value)
+
+
+@pytest.mark.asyncio
+async def test_profile_nested_sizeof():
+    # https://github.com/dask/distributed/issues/1674
+    n = 500
+    original = outer = {}
+    inner = {}
+
+    for i in range(n):
+        outer["children"] = inner
+        outer, inner = inner, {}
+
+    msg = {"data": original}
+    frames = await to_frames(msg)

From e9cdc9e9c2ac4fb04bb2f9cf79d88a85043d4650 Mon Sep 17 00:00:00 2001
From: Julia Signell <jsignell@gmail.com>
Date: Mon, 10 Feb 2020 17:14:04 -0500
Subject: [PATCH 0670/1550] Add total row to workers plot in dashboard (#3464)

---
 distributed/dashboard/components/scheduler.py     | 15 +++++++++++++--
 .../dashboard/tests/test_scheduler_bokeh.py       | 11 +++++++----
 2 files changed, 20 insertions(+), 6 deletions(-)

diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index 2c9953e97e3..c70e41ca436 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -1556,7 +1556,7 @@ def __init__(self, scheduler, width=800, **kwargs):
             point_policy="follow_mouse",
             tooltips="""
                 <div>
-                  <span style="font-size: 10px; font-family: Monaco, monospace;">@worker: </span>
+                  <span style="font-size: 10px; font-family: Monaco, monospace;">Worker (@name): </span>
                   <span style="font-size: 10px; font-family: Monaco, monospace;">@memory_percent</span>
                 </div>
                 """,
@@ -1585,7 +1585,7 @@ def __init__(self, scheduler, width=800, **kwargs):
             point_policy="follow_mouse",
             tooltips="""
                 <div>
-                  <span style="font-size: 10px; font-family: Monaco, monospace;">@worker: </span>
+                  <span style="font-size: 10px; font-family: Monaco, monospace;">Worker (@name): </span>
                   <span style="font-size: 10px; font-family: Monaco, monospace;">@cpu</span>
                 </div>
                 """,
@@ -1641,6 +1641,17 @@ def update(self):
             data["cpu_fraction"][-1] = ws.metrics["cpu"] / 100.0 / ws.nthreads
             data["nthreads"][-1] = ws.nthreads
 
+        for name in self.names + self.extra_names:
+            if name == "name":
+                data[name].insert(
+                    0, "Total ({nworkers})".format(nworkers=len(data[name]))
+                )
+                continue
+            try:
+                data[name].insert(0, sum(data[name]))
+            except TypeError:
+                data[name].insert(0, None)
+
         self.source.data.update(data)
 
 
diff --git a/distributed/dashboard/tests/test_scheduler_bokeh.py b/distributed/dashboard/tests/test_scheduler_bokeh.py
index 4ef90e48b8e..4977ee8fa76 100644
--- a/distributed/dashboard/tests/test_scheduler_bokeh.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh.py
@@ -300,10 +300,13 @@ def test_WorkerTable(c, s, a, b):
         for L in wt.source.data.values()
         for v in L
     ), {type(v).__name__ for L in wt.source.data.values() for v in L}
-    assert all(len(v) == 2 for v in wt.source.data.values())
+
+    assert all(len(v) == 3 for v in wt.source.data.values())
+    assert wt.source.data["name"][0] == "Total (2)"
 
     nthreads = wt.source.data["nthreads"]
     assert all(nthreads)
+    assert nthreads[0] == nthreads[1] + nthreads[2]
 
 
 @gen_cluster(client=True)
@@ -334,7 +337,7 @@ def metric_address(worker):
         assert name in data
 
     assert all(data.values())
-    assert all(len(v) == 2 for v in data.values())
+    assert all(len(v) == 3 for v in data.values())
     my_index = data["address"].index(a.address), data["address"].index(b.address)
     assert [data["metric_port"][i] for i in my_index] == [a.port, b.port]
     assert [data["metric_address"][i] for i in my_index] == [a.address, b.address]
@@ -359,7 +362,7 @@ def metric_port(worker):
     assert "metric_a" in data
     assert "metric_b" in data
     assert all(data.values())
-    assert all(len(v) == 2 for v in data.values())
+    assert all(len(v) == 3 for v in data.values())
     my_index = data["address"].index(a.address), data["address"].index(b.address)
     assert [data["metric_a"][i] for i in my_index] == [a.port, None]
     assert [data["metric_b"][i] for i in my_index] == [None, b.port]
@@ -379,7 +382,7 @@ def metric_port(worker):
 
     assert "metric_a" in data
     assert all(data.values())
-    assert all(len(v) == 2 for v in data.values())
+    assert all(len(v) == 3 for v in data.values())
     my_index = data["address"].index(a.address), data["address"].index(b.address)
     assert [data["metric_a"][i] for i in my_index] == [a.port, None]
 

From e3c97aec94c6df505c59c8a69210339f4a008ac6 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Tue, 11 Feb 2020 07:25:14 -0600
Subject: [PATCH 0671/1550] Update minimum tblib version to 1.6.0 (#3451)

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 545eba40c4d..87e148bb244 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,7 +4,7 @@ dask >= 2.9.0
 msgpack
 psutil >= 5.0
 sortedcontainers !=2.0.0, !=2.0.1
-tblib
+tblib >= 1.6.0
 toolz >= 0.7.4
 tornado >= 5
 zict >= 0.1.3

From f561e9646a121f74d1aecc6d1ee31baeabffad49 Mon Sep 17 00:00:00 2001
From: rockwellw <rockwellw@users.noreply.github.com>
Date: Tue, 11 Feb 2020 08:09:48 -0800
Subject: [PATCH 0672/1550] Update comparison logic for worker state (#3321)

---
 distributed/scheduler.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index b332b92349d..b56c08eb38b 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -275,6 +275,12 @@ def __init__(
 
         self.extra = extra or {}
 
+    def __hash__(self):
+        return hash((self.name, self.host))
+
+    def __eq__(self, other):
+        return type(self) == type(other) and hash(self) == hash(other)
+
     @property
     def host(self):
         return get_address_host(self.address)
@@ -2603,7 +2609,7 @@ def handle_release_data(self, key=None, worker=None, client=None, **msg):
         if ts is None:
             return
         ws = self.workers[worker]
-        if ts.processing_on is not ws:
+        if ts.processing_on != ws:
             return
         r = self.stimulus_missing_data(key=key, ensure=False, **msg)
         self.transitions(r)
@@ -4062,7 +4068,7 @@ def transition_processing_memory(
             if ws is None:
                 return {key: "released"}
 
-            if ws is not ts.processing_on:  # someone else has this task
+            if ws != ts.processing_on:  # someone else has this task
                 logger.info(
                     "Unexpected worker completed task, likely due to"
                     " work stealing.  Expected: %s, Got: %s, Key: %s",

From 386a9d836b05b2e3f8daacdeb97f8081e987272b Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Wed, 12 Feb 2020 09:45:24 -0600
Subject: [PATCH 0673/1550] Minor gen.Return cleanup (#3469)

---
 distributed/actor.py  | 2 --
 distributed/client.py | 2 +-
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/distributed/actor.py b/distributed/actor.py
index 37f43b69358..69172bf23ec 100644
--- a/distributed/actor.py
+++ b/distributed/actor.py
@@ -1,5 +1,4 @@
 import asyncio
-from tornado import gen
 import functools
 import threading
 from queue import Queue
@@ -169,7 +168,6 @@ async def get_actor_attribute_from_worker():
                     attribute=key, actor=self.key
                 )
                 return x["result"]
-                raise gen.Return(x["result"])
 
             return self._sync(get_actor_attribute_from_worker)
 
diff --git a/distributed/client.py b/distributed/client.py
index 36c487dc334..dbc4cd5bc11 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -1909,7 +1909,7 @@ async def _scatter(
             isinstance(k, (bytes, str)) for k in data
         ):
             d = await self._scatter(keymap(tokey, data), workers, broadcast)
-            raise gen.Return({k: d[tokey(k)] for k in data})
+            return {k: d[tokey(k)] for k in data}
 
         if isinstance(data, type(range(0))):
             data = list(data)

From 51f1a22fb8742c90f9870d224a9336235957a3df Mon Sep 17 00:00:00 2001
From: Dustin Tindall <dustin.tindall@gmail.com>
Date: Wed, 12 Feb 2020 09:47:53 -0600
Subject: [PATCH 0674/1550] Update locality.rst (#3470)

---
 docs/source/locality.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/source/locality.rst b/docs/source/locality.rst
index 4ee6568395d..caab601f191 100644
--- a/docs/source/locality.rst
+++ b/docs/source/locality.rst
@@ -73,7 +73,7 @@ used.
                           allow_other_workers=True)
 
 Additionally the ``scatter`` function supports a ``broadcast=`` keyword
-argument to enforce that the all data is sent to all workers rather than
+argument to enforce that all the data is sent to all workers rather than
 round-robined.  If new workers arrive they will not automatically receive this
 data.
 

From 346b2dbc56d682aa96d0b3f072f7411ec7d695a0 Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Wed, 12 Feb 2020 17:42:22 +0000
Subject: [PATCH 0675/1550] Split dashboard host on additional slashes to
 handle inproc (#3466)

---
 distributed/deploy/cluster.py    | 2 +-
 distributed/tests/test_client.py | 7 +++++++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
index ad071a214be..c616f13c826 100644
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -199,7 +199,7 @@ def dashboard_link(self):
         except KeyError:
             return ""
         else:
-            host = self.scheduler_address.split("://")[1].split(":")[0]
+            host = self.scheduler_address.split("://")[1].split("/")[0].split(":")[0]
             return format_dashboard_link(host, port)
 
     def _widget_status(self):
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 8abbd89386d..4f075d582f3 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -5255,6 +5255,13 @@ def test_dashboard_link(loop, monkeypatch):
                 assert link in text
 
 
+@pytest.mark.asyncio
+async def test_dashboard_link_inproc(cleanup):
+    async with Client(processes=False, asynchronous=True) as c:
+        with dask.config.set({"distributed.dashboard.link": "{host}"}):
+            assert "/" not in c.dashboard_link
+
+
 @gen_test()
 def test_client_timeout_2():
     with dask.config.set({"distributed.comm.timeouts.connect": "10ms"}):

From 7c1b4dfdc1dc9d690ee1731a8fc94f5665620de7 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 12 Feb 2020 09:43:09 -0800
Subject: [PATCH 0676/1550] Change default multiprocessing behavior to spawn
 (#3461)

---
 distributed/deploy/tests/test_ssh.py | 4 ++--
 distributed/distributed.yaml         | 2 +-
 distributed/nanny.py                 | 1 +
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/distributed/deploy/tests/test_ssh.py b/distributed/deploy/tests/test_ssh.py
index eff7bf05a11..af6bf1566f2 100644
--- a/distributed/deploy/tests/test_ssh.py
+++ b/distributed/deploy/tests/test_ssh.py
@@ -38,14 +38,14 @@ async def test_keywords():
             "memory_limit": "2 GiB",
             "death_timeout": "5s",
         },
-        scheduler_options={"idle_timeout": "5s", "port": 0},
+        scheduler_options={"idle_timeout": "10s", "port": 0},
     ) as cluster:
         async with Client(cluster, asynchronous=True) as client:
             assert (
                 await client.run_on_scheduler(
                     lambda dask_scheduler: dask_scheduler.idle_timeout
                 )
-            ) == 5
+            ) == 10
             d = client.scheduler_info()["workers"]
             assert all(v["nthreads"] == 2 for v in d.values())
 
diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index ee38750f8ee..487e72e215e 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -39,7 +39,7 @@ distributed:
 
   worker:
     blocked-handlers: []
-    multiprocessing-method: forkserver
+    multiprocessing-method: spawn
     use-file-locking: True
     connections:            # Maximum concurrent connections for data
       outgoing: 50          # This helps to control network saturation
diff --git a/distributed/nanny.py b/distributed/nanny.py
index 5e67b51b52b..ff653ba096c 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -597,6 +597,7 @@ async def kill(self, timeout=2, executor_wait=True):
                 "executor_wait": executor_wait,
             }
         )
+        await asyncio.sleep(0)  # otherwise we get broken pipe errors
         self.child_stop_q.close()
 
         while process.is_alive() and loop.time() < deadline:

From cf051bf7050a00ff7c5f738283b2634fbb5b5178 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 12 Feb 2020 09:43:32 -0800
Subject: [PATCH 0677/1550] Clear old docs (#3458)

* Remove local-cluster doc
* remove adaptive and prometheus docs
* remove dashboard docs
---
 docs/source/adaptive.rst      | 175 --------------------------
 docs/source/conf.py           |   7 ++
 docs/source/index.rst         |   5 -
 docs/source/local-cluster.rst |  50 --------
 docs/source/prometheus.rst    |  42 -------
 docs/source/web.rst           | 229 ----------------------------------
 6 files changed, 7 insertions(+), 501 deletions(-)
 delete mode 100644 docs/source/adaptive.rst
 delete mode 100644 docs/source/local-cluster.rst
 delete mode 100644 docs/source/prometheus.rst
 delete mode 100644 docs/source/web.rst

diff --git a/docs/source/adaptive.rst b/docs/source/adaptive.rst
deleted file mode 100644
index f07246588cd..00000000000
--- a/docs/source/adaptive.rst
+++ /dev/null
@@ -1,175 +0,0 @@
-Adaptive Deployments
-====================
-
-It is possible to grow and shrink Dask clusters based on current use.  This
-allows you to run Dask permanently on your cluster and have it only take up
-resources when necessary.  Dask contains the logic about when to grow and
-shrink but relies on external cluster managers to launch and kill
-``dask-worker`` jobs.  This page describes the policies about adaptively
-resizing Dask clusters based on load, how to connect these policies to a
-particular job scheduler, and an example implementation.
-
-Dynamically scaling a Dask cluster up and down requires tight integration with
-an external cluster management system that can deploy ``dask-worker`` jobs
-throughout the cluster.  Several systems are in wide use today, including
-common examples like SGE, SLURM, Torque, Condor, LSF, Yarn, Mesos, Marathon,
-Kubernetes, etc... These systems can be quite different from each other, but
-all are used to manage distributed services throughout different kinds of
-clusters.
-
-The large number of relevant systems, the challenges of rigorously testing
-each, and finite development time precludes the systematic inclusion of all
-solutions within the dask/distributed repository.  Instead, we include a
-generic interface that can be extended by someone with basic understanding of
-their cluster management tool.  We encourage these as third party modules.
-
-
-Policies
---------
-
-We control the number of workers based on current load and memory use.  The
-scheduler checks itself periodically to determine if more or fewer workers are
-needed.
-
-If there are excess unclaimed tasks, or if the memory of the current workers is
-more nearing full then the scheduler tries to increase the number of workers by
-a fixed factor, defaulting to 2.  This causes exponential growth while growth
-is useful.
-
-If there are idle workers and if the memory of the current workers is nearing
-empty then we gracefully retire the idle workers with the least amount of data
-in memory.  We first move these results to the surviving workers and then
-remove the idle workers from the cluster.  This shrinks the cluster while
-gracefully preserving intermediate results, shrinking the cluster when excess
-size is not useful.
-
-
-Adaptive class interface
-------------------------
-
-The ``distributed.deploy.Adaptive`` class contains the logic about when to ask
-for new workers, and when to close idle ones.  This class requires both a
-scheduler and a cluster object.
-
-The cluster object must support two methods, ``scale_up(n, **kwargs)``, which
-takes in a target number of total workers for the cluster and
-``scale_down(workers)``, which takes in a list of addresses to remove from the
-cluster.  The Adaptive class will call these methods with the correct values at
-the correct times.
-
-.. code-block:: python
-
-   class MyCluster:
-       async def scale_up(self, n, **kwargs):
-           """
-           Bring the total count of workers up to ``n``
-
-           This function/coroutine should bring the total number of workers up to
-           the number ``n``.
-
-           This can be implemented either as a function or as a Tornado coroutine.
-           """
-           raise NotImplementedError()
-
-       async def scale_down(self, workers):
-           """
-           Remove ``workers`` from the cluster
-
-           Given a list of worker addresses this function should remove those
-           workers from the cluster.  This may require tracking which jobs are
-           associated to which worker address.
-
-           This can be implemented either as a function or as a Tornado coroutine.
-           """
-
-   from distributed.deploy import Adaptive
-
-   scheduler = Scheduler()
-   cluster = MyCluster()
-   adapative_cluster = Adaptive(scheduler, cluster)
-   scheduler.start()
-
-Implementing these ``scale_up`` and ``scale_down`` functions depends strongly
-on the cluster management system.  See :doc:`LocalCluster <local-cluster>` for
-an example.
-
-
-Marathon: an example
---------------------
-
-We now present an example project that implements this cluster interface backed
-by the Marathon cluster management tool on Mesos.  Full source code and testing
-apparatus is available here: http://github.com/mrocklin/dask-marathon
-
-The implementation is small.  It uses the Marathon HTTP API through the
-`marathon Python client library <https://github.com/thefactory/marathon-python>`_.
-We reproduce the full body of the implementation below as an example:
-
-.. code-block:: python
-
-   from marathon import MarathonClient, MarathonApp
-   from marathon.models.container import MarathonContainer
-
-   class MarathonCluster:
-       def __init__(self, scheduler,
-                    executable='dask-worker',
-                    docker_image='mrocklin/dask-distributed',
-                    marathon_address='http://localhost:8080',
-                    name=None, **kwargs):
-           self.scheduler = scheduler
-
-           # Create Marathon App to run dask-worker
-           args = [executable, scheduler.address,
-                   '--name', '$MESOS_TASK_ID']  # use Mesos task ID as worker name
-           if 'mem' in kwargs:
-               args.extend(['--memory-limit',
-                            str(int(kwargs['mem'] * 0.6 * 1e6))])
-           kwargs['cmd'] = ' '.join(args)
-           container = MarathonContainer({'image': docker_image})
-
-           app = MarathonApp(instances=0, container=container, **kwargs)
-
-           # Connect and register app
-           self.client = MarathonClient(marathon_address)
-           self.app = self.client.create_app(name or 'dask-%s' % uuid.uuid4(), app)
-
-       def scale_up(self, instances):
-           self.marathon_client.scale_app(self.app.id, instances=instances)
-
-       def scale_down(self, workers):
-           for w in workers:
-               self.marathon_client.kill_task(self.app.id,
-                                              self.scheduler.worker_info[w]['name'],
-                                              scale=True)
-
-Subclassing Adaptive
---------------------
-
-The default behaviors of ``Adaptive`` controlling when to scale up or down, and
-by how much, may not be appropriate for your cluster manager or workload. For
-example, you may have tasks that require a worker with more memory than usual.
-This means we need to pass through some additional keyword arguments to
-``cluster.scale_up`` call.
-
-.. code-block:: python
-
-   from distributed.deploy import Adaptive
-
-   class MyAdaptive(Adaptive):
-       def get_scale_up_kwargs(self):
-           kwargs = super(Adaptive, self).get_scale_up_kwargs()
-           # resource_restrictions maps task keys to a dict of restrictions
-           restrictions = self.scheduler.resource_restrictions.values()
-           memory_restrictions = [x.get('memory') for x in restrictions
-                                  if 'memory' in x]
-
-           if memory_restrictions:
-               kwargs['memory'] = max(memory_restrictions)
-
-           return kwargs
-
-
-So if there are any tasks that are waiting to be run on a worker with enough
-memory, the ``kwargs`` dictionary passed to ``cluster.scale_up`` will include
-a key and value for ``'memory'`` (your ``Cluster.scale_up`` method needs to be
-able to support this).
diff --git a/docs/source/conf.py b/docs/source/conf.py
index 9bd0ce6867e..f8ab5a31797 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -393,6 +393,13 @@
     ("setup.html", "https://docs.dask.org/en/latest/setup.html"),
     ("ec2.html", "https://docs.dask.org/en/latest/setup/cloud.html"),
     ("configuration.html", "https://docs.dask.org/en/latest/configuration.html"),
+    (
+        "local-cluster.html",
+        "https://docs.dask.org/en/latest/setup/single-distributed.html",
+    ),
+    ("adaptive.html", "https://docs.dask.org/en/latest/setup/adaptive.html"),
+    ("prometheus.html", "https://docs.dask.org/en/latest/setup/prometheus.html"),
+    ("web.html", "https://docs.dask.org/en/latest/diagnostics-distributed.html"),
 ]
 
 
diff --git a/docs/source/index.rst b/docs/source/index.rst
index 47419e014ec..3cbdd18792a 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -106,19 +106,14 @@ Contents
    :caption: Additional Features
 
    actors
-   adaptive
    asynchronous
    configuration
-   local-cluster
    ipython
-   Joblib Integration <https://ml.dask.org/joblib.html>
    prometheus
    publish
-   queues
    resources
    task-launch
    tls
-   web
 
 .. toctree::
    :maxdepth: 1
diff --git a/docs/source/local-cluster.rst b/docs/source/local-cluster.rst
deleted file mode 100644
index d596ccaed24..00000000000
--- a/docs/source/local-cluster.rst
+++ /dev/null
@@ -1,50 +0,0 @@
-Local Cluster
-=============
-
-For convenience you can start a local cluster from your Python session.
-
-.. code-block:: python
-
-   >>> from distributed import Client, LocalCluster
-   >>> cluster = LocalCluster()
-   LocalCluster("127.0.0.1:8786", workers=8, nthreads=8)
-   >>> client = Client(cluster)
-   <Client: scheduler=127.0.0.1:8786 processes=8 cores=8>
-
-You can dynamically scale this cluster up and down:
-
-.. code-block:: python
-
-   >>> worker = cluster.add_worker()
-   >>> cluster.remove_worker(worker)
-
-Alternatively, a ``LocalCluster`` is made for you automatically if you create
-an ``Client`` with no arguments:
-
-.. code-block:: python
-
-   >>> from distributed import Client
-   >>> client = Client()
-   >>> client
-   <Client: scheduler=127.0.0.1:8786 processes=8 cores=8>
-
-.. note::
-
-   Within a Python script you need to start a local cluster in the
-   ``if __name__ == '__main__'`` block:
-
-   .. code-block:: python
-
-      if __name__ == '__main__':
-          cluster = LocalCluster()
-          client = Client(cluster)
-          # Your code follows here
-
-API
----
-
-.. currentmodule:: distributed.deploy.local
-
-.. autoclass:: LocalCluster
-   :members:
-
diff --git a/docs/source/prometheus.rst b/docs/source/prometheus.rst
deleted file mode 100644
index 097335ee0d7..00000000000
--- a/docs/source/prometheus.rst
+++ /dev/null
@@ -1,42 +0,0 @@
-Prometheus Monitoring
------------------------
-
-Prometheus_ is a widely popular tool for monitoring and alerting a wide variety of systems. Dask.distributed exposes
-scheduler and worker metrics in a prometheus text based format. Metrics are available at ``http://scheduler-address:8787/metrics``.
-
-.. _Prometheus: https://prometheus.io
-
-Available metrics are as following
-
-+---------------------------------------------+------------------------------------------------+-----------+--------+
-| Metric name                                 | Description                                    | Scheduler | Worker |
-+=========================+===================+================================================+===========+========+
-| python_gc_objects_collected_total           | Objects collected during gc.                   |    Yes    |  Yes   |
-+---------------------------------------------+------------------------------------------------+-----------+--------+
-| python_gc_objects_uncollectable_total       | Uncollectable object found during GC.          |    Yes    |  Yes   |
-+---------------------------------------------+------------------------------------------------+-----------+--------+
-| python_gc_collections_total                 | Number of times this generation was collected. |    Yes    |  Yes   |
-+---------------------------------------------+------------------------------------------------+-----------+--------+
-| python_info                                 | Python platform information.                   |    Yes    |  Yes   |
-+---------------------------------------------+------------------------------------------------+-----------+--------+
-| dask_scheduler_workers                      | Number of workers connected.                   |    Yes    |        |
-+---------------------------------------------+------------------------------------------------+-----------+--------+
-| dask_scheduler_clients                      | Number of clients connected.                   |    Yes    |        |
-+---------------------------------------------+------------------------------------------------+-----------+--------+
-| dask_scheduler_tasks                        | Number of tasks at scheduler.                  |    Yes    |        |
-+---------------------------------------------+------------------------------------------------+-----------+--------+
-| dask_worker_tasks                           | Number of tasks at worker.                     |           |  Yes   |
-+---------------------------------------------+------------------------------------------------+-----------+--------+
-| dask_worker_connections                     | Number of task connections to other workers.   |           |  Yes   |
-+---------------------------------------------+------------------------------------------------+-----------+--------+
-| dask_worker_threads                         | Number of worker threads.                      |           |  Yes   |
-+---------------------------------------------+------------------------------------------------+-----------+--------+
-| dask_worker_latency_seconds                 | Latency of worker connection.                  |           |  Yes   |
-+---------------------------------------------+------------------------------------------------+-----------+--------+
-| dask_worker_tick_duration_median_seconds    | Median tick duration at worker.                |           |  Yes   |
-+---------------------------------------------+------------------------------------------------+-----------+--------+
-| dask_worker_task_duration_median_seconds    | Median task runtime at worker.                 |           |  Yes   |
-+---------------------------------------------+------------------------------------------------+-----------+--------+
-| dask_worker_transfer_bandwidth_median_bytes | Bandwidth for transfer at worker in Bytes.     |           |  Yes   |
-+---------------------------------------------+------------------------------------------------+-----------+--------+
-
diff --git a/docs/source/web.rst b/docs/source/web.rst
deleted file mode 100644
index 6a5b58fac5e..00000000000
--- a/docs/source/web.rst
+++ /dev/null
@@ -1,229 +0,0 @@
-Web Interface
-=============
-
-.. raw:: html
-
-    <iframe width="560"
-            height="315"
-            src="https://www.youtube.com/embed/N_GqzcuGLCY"
-            frameborder="0"
-            allow="autoplay; encrypted-media"
-            allowfullscreen>
-    </iframe>
-
-Information about the current state of the network helps to track progress,
-identify performance issues, and debug failures.
-
-Dask.distributed includes a web interface to help deliver this information over
-a normal web page in real time.  This web interface is launched by default
-wherever the scheduler is launched if the scheduler machine has Bokeh_
-installed (``conda install bokeh -c bokeh``).
-
-These diagnostic pages are:
-
-*   Main Scheduler pages at ``http://scheduler-address:8787``.  These pages,
-    particularly the ``/status`` page are the main page that most people
-    associate with Dask.  These pages are served from a separate standalone
-    Bokeh server application running in a separate process.
-
-The available pages are ``http://scheduler-address:8787/<page>/`` where ``<page>`` is one of
-
-- ``status``: a stream of recently run tasks, progress bars, resource use
-- ``tasks``: a larger stream of the last 100k tasks
-- ``workers``: basic information about workers and their current load
-- ``health``: basic health check, returns ``ok`` if service is running
-
-.. _Bokeh: http://bokeh.pydata.org/en/latest/
-
-Plots
------
-
-Example Computation
-~~~~~~~~~~~~~~~~~~~
-
-The following plots show a trace of the following computation:
-
-.. code-block:: python
-
-   from distributed import Client
-   from time import sleep
-   import random
-
-   def inc(x):
-       sleep(random.random() / 10)
-       return x + 1
-
-   def dec(x):
-       sleep(random.random() / 10)
-       return x - 1
-
-   def add(x, y):
-       sleep(random.random() / 10)
-       return x + y
-
-
-   client = Client('127.0.0.1:8786')
-
-   incs = client.map(inc, range(100))
-   decs = client.map(dec, range(100))
-   adds = client.map(add, incs, decs)
-   total = client.submit(sum, adds)
-
-   del incs, decs, adds
-   total.result()
-
-Progress
-~~~~~~~~
-
-The interface shows the progress of the various computations as well as the
-exact number completed.
-
-.. image:: https://raw.githubusercontent.com/dask/dask-org/master/images/bokeh-progress.gif
-   :alt: Resources view of Dask web interface
-
-Each bar is assigned a color according to the function being run.  Each bar
-has a few components.  On the left the lighter shade is the number of tasks
-that have both completed and have been released from memory.  The darker shade
-to the right corresponds to the tasks that are completed and whose data still
-reside  in memory.  If errors occur then they appear as a black colored block
-to the right.
-
-Typical computations may involve dozens of kinds of functions.  We handle this
-visually with the following approaches:
-
-1.  Functions are ordered by the number of total tasks
-2.  The colors are assigned in a round-robin fashion from a standard palette
-3.  The progress bars shrink horizontally to make space for more functions
-4.  Only the largest functions (in terms of number of tasks) are displayed
-
-.. image:: https://raw.githubusercontent.com/dask/dask-org/master/images/bokeh-progress-large.gif
-   :alt: Progress bar plot of Dask web interface
-
-Counts of tasks processing, waiting for dependencies, processing, etc.. are
-displayed in the title bar.
-
-Memory Use
-~~~~~~~~~~
-
-The interface shows the relative memory use of each function with a horizontal
-bar sorted by function name.
-
-.. image:: https://raw.githubusercontent.com/dask/dask-org/master/images/bokeh-memory-use.gif
-   :alt: Memory use plot of Dask web interface
-
-The title shows the number of total bytes in use.  Hovering over any bar
-tells you the specific function and how many bytes its results are actively
-taking up in memory.  This does not count data that has been released.
-
-Task Stream
-~~~~~~~~~~~
-
-The task stream plot shows when tasks complete on which workers.  Worker cores
-are on the y-axis and time is on the x-axis.  As a worker completes a task its
-start and end times are recorded and a rectangle is added to this plot
-accordingly.
-
-.. image:: https://raw.githubusercontent.com/dask/dask-org/master/images/bokeh-task-stream.gif
-   :alt: Task stream plot of Dask web interface
-
-The colors signifying the following:
-
-1.  Serialization (gray)
-2.  Communication between workers (red)
-3.  Disk I/O (orange)
-4.  Error (black)
-5.  Execution times (colored by task: purple, green, yellow, etc)
-
-
-If data transfer occurs between workers a *red* bar appears preceding the
-task bar showing the duration of the transfer.  If an error occurs than a
-*black* bar replaces the normal color.  This plot show the last 1000 tasks.
-It resets if there is a delay greater than 10 seconds.
-
-For a full history of the last 100,000 tasks see the ``tasks/`` page.
-
-Resources
-~~~~~~~~~
-
-The resources plot show the average CPU and Memory use over time as well as
-average network traffic.  More detailed information on a per-worker basis is
-available in the ``workers/`` page.
-
-.. image:: https://raw.githubusercontent.com/dask/dask-org/master/images/bokeh-resources.gif
-   :alt: Resources view of Dask web interface
-
-Per-worker resources
-~~~~~~~~~~~~~~~~~~~~
-
-The ``workers/`` page shows per-worker resources, the main ones being CPU and
-memory use. Custom metrics can be registered and displayed in this page. Here
-is an example showing how to display GPU utilization and GPU memory use:
-
-.. code-block:: python
-
-   import subprocess
-
-   def nvidia_data(name):
-       def dask_function(dask_worker):
-           cmd = 'nvidia-smi --query-gpu={} --format=csv,noheader'.format(name)
-           result = subprocess.check_output(cmd.split())
-           return result.strip().decode()
-       return dask_function
-
-   def register_metrics(dask_worker):
-       for name in ['utilization.gpu', 'utilization.memory']:
-           dask_worker.metrics[name] = nvidia_data(name)
-
-   client.run(register_metrics)
-
-Connecting to Web Interface
----------------------------
-
-Default
-~~~~~~~
-
-By default, ``dask-scheduler`` prints out the address of the web interface::
-
-   INFO -  Bokeh UI at:  http://10.129.39.91:8787/status
-   ...
-   INFO - Starting Bokeh server on port 8787 with applications at paths ['/status', '/tasks']
-
-The machine hosting the scheduler runs an HTTP server serving at that address.
-
-
-Troubleshooting
----------------
-
-Some clusters restrict the ports that are visible to the outside world.  These
-ports may include the default port for the web interface, ``8787``.  There are
-a few ways to handle this:
-
-1.  Open port ``8787`` to the outside world.  Often this involves asking your
-    cluster administrator.
-2.  Use a different port that is publicly accessible using the
-    ``--dashboard-address :8787`` option on the ``dask-scheduler`` command.
-3.  Use fancier techniques, like `Port Forwarding`_
-
-Running distributed on a remote machine can cause issues with viewing the web
-UI -- this depends on the remote machines network configuration.
-
-.. _`Port Forwarding`: https://en.wikipedia.org/wiki/Port_forwarding
-
-
-Port Forwarding
-~~~~~~~~~~~~~~~
-
-If you have SSH access then one way to gain access to a blocked port is through
-SSH port forwarding. A typical use case looks like the following:
-
-.. code:: bash
-
-   local$ ssh -L 8000:localhost:8787 user@remote
-   remote$ dask-scheduler  # now, the web UI is visible at localhost:8000
-   remote$ # continue to set up dask if needed -- add workers, etc
-
-It is then possible to go to ``localhost:8000`` and see Dask Web UI. This same approach is
-not specific to dask.distributed, but can be used by any service that operates over a
-network, such as Jupyter notebooks. For example, if we chose to do this we could
-forward port 8888 (the default Jupyter port) to port 8001 with
-``ssh -L 8001:localhost:8888 user@remote``.

From 49328dcd0e13556c7b3bae8c3d542786cbb07703 Mon Sep 17 00:00:00 2001
From: condoratberlin <49398997+condoratberlin@users.noreply.github.com>
Date: Wed, 12 Feb 2020 18:50:37 +0100
Subject: [PATCH 0678/1550] Change default value of local_directory from empty
 string to None (#3441)

Fixes #3440
---
 distributed/cli/dask_worker.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
index e76bed2e9bc..5188333b75c 100755
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -151,12 +151,12 @@
 )
 @click.option("--pid-file", type=str, default="", help="File to write the process PID")
 @click.option(
-    "--local-directory", default="", type=str, help="Directory to place worker files"
+    "--local-directory", default=None, type=str, help="Directory to place worker files"
 )
 @click.option(
     "--resources",
     type=str,
-    default="",
+    default=None,
     help='Resources for task constraints like "GPU=2 MEM=10e9". '
     "Resources are applied separately to each worker process "
     "(only relevant when starting multiple worker processes with '--nprocs').",
@@ -164,7 +164,7 @@
 @click.option(
     "--scheduler-file",
     type=str,
-    default="",
+    default=None,
     help="Filename to JSON encoded scheduler information. "
     "Use with dask-scheduler --scheduler-file",
 )
@@ -180,7 +180,7 @@
 @click.option(
     "--lifetime",
     type=str,
-    default="",
+    default=None,
     help="If provided, shut down the worker after this duration.",
 )
 @click.option(

From ca88aa7327820fc934b2254b884721cea94991e5 Mon Sep 17 00:00:00 2001
From: kaelgreco <kaelgreco@users.noreply.github.com>
Date: Thu, 13 Feb 2020 08:43:26 -0800
Subject: [PATCH 0679/1550] Add last seen column to worker table and highlight
 errant workers (#3468)

---
 distributed/dashboard/scheduler.py                | 4 +++-
 distributed/dashboard/templates/task.html         | 4 ++--
 distributed/dashboard/templates/worker-table.html | 6 ++++--
 3 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/distributed/dashboard/scheduler.py b/distributed/dashboard/scheduler.py
index 17e150e8df9..836cefbbd6c 100644
--- a/distributed/dashboard/scheduler.py
+++ b/distributed/dashboard/scheduler.py
@@ -50,12 +50,14 @@
 from .proxy import GlobalProxyHandler
 from .utils import RequestHandler, redirect
 from ..diagnostics.websocket import WebsocketPlugin
+from ..metrics import time
 from ..utils import log_errors, format_time
 from ..scheduler import ALL_TASK_STATES
 
 
 ns = {
-    func.__name__: func for func in [format_bytes, format_time, datetime.fromtimestamp]
+    func.__name__: func
+    for func in [format_bytes, format_time, datetime.fromtimestamp, time]
 }
 
 rel_path_statics = {"rel_path_statics": "../../"}
diff --git a/distributed/dashboard/templates/task.html b/distributed/dashboard/templates/task.html
index 8c292da4e43..bcc0d17c0a8 100644
--- a/distributed/dashboard/templates/task.html
+++ b/distributed/dashboard/templates/task.html
@@ -122,9 +122,9 @@ <h3 class="title is-5"> Transition Log </h3>
                   <th> Recommended Action </th>
               </thead>
 
-              {% for key, start, finish, recommendations, time in scheduler.story(Task) %}
+              {% for key, start, finish, recommendations, transition_time in scheduler.story(Task) %}
               <tr>
-                  <td> {{ fromtimestamp(time) }} </td>
+                  <td> {{ fromtimestamp(transition_time) }} </td>
                   <td> <a href="{{ url_escape(key) }}.html">{{key}}</a> </td>
                   <td> {{ start }} </td>
                   <td> {{ finish }} </td>
diff --git a/distributed/dashboard/templates/worker-table.html b/distributed/dashboard/templates/worker-table.html
index c12061fab46..87512ee3860 100644
--- a/distributed/dashboard/templates/worker-table.html
+++ b/distributed/dashboard/templates/worker-table.html
@@ -1,4 +1,4 @@
-   <table class="table is-striped is-hoverable">
+<table class="table is-striped is-hoverable">
     <tr>
         <th> Worker </th>
         <th> Name </th>
@@ -10,9 +10,10 @@
         <th> In-memory </th>
         <th> Services</th>
         <th> Logs </th>
+        <th> Last seen </th>
     </tr>
     {% for ws in worker_list %}
-    <tr>
+    <tr {{ "style=background-color:#ffcdd2" if time() - ws.last_seen > 60 else ""}}>
         <td><a href="../worker/{{ url_escape(ws.address) }}.html">{{ws.address}}</a></td>
         <td> {{ ws.name if ws.name is not None else "" }} </td>
         <td> {{ ws.nthreads }} </td>
@@ -27,6 +28,7 @@
         <td> </td>
         {% end %}
         <td> <a href="../logs/{{ url_escape(ws.address) }}.html">logs</a></td>
+        <td> {{ format_time(time() - ws.last_seen) }} </td>
     </tr>
     {% end %}
   </table>

From 04de4b2adc1f0c94ab86e9aa46a4c67382a7eaea Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Thu, 13 Feb 2020 16:10:34 -0600
Subject: [PATCH 0680/1550] Add support for Python 3.8 (#3249)

---
 .github/workflows/ci-windows.yaml         | 15 ++++++++++++-
 .travis.yml                               |  1 +
 continuous_integration/environment.yml    |  1 -
 continuous_integration/travis/install.sh  | 19 +++++++++++------
 distributed/__init__.py                   |  2 +-
 distributed/client.py                     |  8 ++++---
 distributed/comm/ucx.py                   |  8 ++-----
 distributed/core.py                       |  2 +-
 distributed/deploy/tests/test_local.py    |  7 ++++++
 distributed/protocol/tests/test_pickle.py |  6 ++++++
 distributed/tests/test_as_completed.py    |  2 +-
 distributed/tests/test_client.py          |  2 +-
 distributed/tests/test_client_executor.py |  2 +-
 distributed/tests/test_failed_workers.py  |  3 +--
 distributed/tests/test_steal.py           |  5 +++++
 distributed/tests/test_stress.py          |  3 +--
 distributed/tests/test_worker.py          |  5 +++++
 distributed/utils.py                      | 26 ++++++++++++++++++-----
 requirements.txt                          |  3 ++-
 setup.py                                  |  1 +
 20 files changed, 88 insertions(+), 33 deletions(-)

diff --git a/.github/workflows/ci-windows.yaml b/.github/workflows/ci-windows.yaml
index 992f65d6435..707c50ffc76 100644
--- a/.github/workflows/ci-windows.yaml
+++ b/.github/workflows/ci-windows.yaml
@@ -7,7 +7,7 @@ jobs:
     runs-on: windows-latest
     strategy:
       matrix:
-        python-version: ["3.6", "3.7"]
+        python-version: ["3.6", "3.7", "3.8"]
 
     steps:
       - name: Checkout source
@@ -22,6 +22,19 @@ jobs:
           activate-environment: testenv
           auto-activate-base: false
 
+      - name: Install tornado
+        shell: bash -l {0}
+        run: |
+          if [[ "${{ matrix.python-version }}" = "3.8" ]]; then
+            conda install -c conda-forge tornado=6
+          else
+            conda install -c conda-forge tornado=5
+          fi
+
+      - name: List packages in environment
+        shell: bash -l {0}
+        run: conda list
+
       - name: Install distributed from source
         shell: bash -l {0}
         run: python -m pip install -q --no-deps -e .
diff --git a/.travis.yml b/.travis.yml
index d00894dd3d6..e8f2afc5057 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -8,6 +8,7 @@ env:
   matrix:
     - PYTHON=3.6 TESTS=true COVERAGE=true PACKAGES="scikit-learn lz4" TORNADO=5 CRICK=true
     - PYTHON=3.7 TESTS=true PACKAGES="scikit-learn python-snappy python-blosc" TORNADO=6
+    - PYTHON=3.8 TESTS=true PACKAGES="scikit-learn python-snappy python-blosc" TORNADO=6
 
 matrix:
   fast_finish: true
diff --git a/continuous_integration/environment.yml b/continuous_integration/environment.yml
index f6651254af2..f69d919e879 100644
--- a/continuous_integration/environment.yml
+++ b/continuous_integration/environment.yml
@@ -21,7 +21,6 @@ dependencies:
   - requests
   - toolz
   - tblib
-  - tornado=5
   - zict
   - fsspec
   - pip
diff --git a/continuous_integration/travis/install.sh b/continuous_integration/travis/install.sh
index 8eaed19df81..b631ac3bc6c 100644
--- a/continuous_integration/travis/install.sh
+++ b/continuous_integration/travis/install.sh
@@ -27,7 +27,8 @@ conda create -q -n test-environment python=$PYTHON
 source activate test-environment
 
 # Install dependencies
-conda install -q \
+conda install -c conda-forge -q \
+    asyncssh \
     bokeh \
     click \
     coverage \
@@ -48,15 +49,20 @@ conda install -q \
     python=$PYTHON \
     requests \
     scipy \
-    tblib \
+    tblib>=1.5.0 \
     toolz \
     tornado=$TORNADO \
+    zstandard \
     $PACKAGES
 
-# For low-level profiler, install libunwind and stacktrace from conda-forge
-# For stacktrace we use --no-deps to avoid upgrade of python
-conda install -c defaults -c conda-forge libunwind zstandard asyncssh
-conda install --no-deps -c defaults -c numba -c conda-forge stacktrace
+# stacktrace is not currently avaiable for Python 3.8.
+# Remove the version check block below when it is avaiable.
+if [[ $PYTHON != 3.8 ]]; then
+    # For low-level profiler, install libunwind and stacktrace from conda-forge
+    # For stacktrace we use --no-deps to avoid upgrade of python
+    conda install -c defaults -c conda-forge libunwind
+    conda install --no-deps -c defaults -c numba -c conda-forge stacktrace
+fi;
 
 python -m pip install -q "pytest>=4" pytest-repeat pytest-faulthandler pytest-asyncio
 
@@ -67,7 +73,6 @@ python -m pip install -q git+https://github.com/dask/s3fs.git --upgrade --no-dep
 python -m pip install -q git+https://github.com/dask/zict.git --upgrade --no-deps
 python -m pip install -q sortedcollections msgpack --no-deps
 python -m pip install -q keras --upgrade --no-deps
-python -m pip install -q asyncssh
 
 if [[ $CRICK == true ]]; then
     conda install -q cython
diff --git a/distributed/__init__.py b/distributed/__init__.py
index 9238d57ccc9..be750f9daed 100644
--- a/distributed/__init__.py
+++ b/distributed/__init__.py
@@ -24,7 +24,7 @@
 from .queues import Queue
 from .scheduler import Scheduler
 from .threadpoolexecutor import rejoin
-from .utils import sync, TimeoutError
+from .utils import sync, TimeoutError, CancelledError
 from .variable import Variable
 from .worker import Worker, get_worker, get_client, secede, Reschedule
 from .worker_client import local_client, worker_client
diff --git a/distributed/client.py b/distributed/client.py
index dbc4cd5bc11..ce820d2c6e6 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -2,7 +2,7 @@
 import atexit
 from collections import defaultdict
 from collections.abc import Iterator
-from concurrent.futures import ThreadPoolExecutor, CancelledError
+from concurrent.futures import ThreadPoolExecutor
 from concurrent.futures._base import DoneAndNotDoneFutures
 from contextlib import contextmanager
 import copy
@@ -82,6 +82,7 @@
     has_keyword,
     format_dashboard_link,
     TimeoutError,
+    CancelledError,
 )
 from . import versions as version_module
 
@@ -1248,6 +1249,7 @@ async def _close(self, fast=False):
         """ Send close signal and wait until scheduler completes """
         if self.status == "closed":
             return
+
         self.status = "closing"
 
         for pc in self._periodic_callbacks.values():
@@ -1273,7 +1275,7 @@ async def _close(self, fast=False):
 
             # Give the scheduler 'stream-closed' message 100ms to come through
             # This makes the shutdown slightly smoother and quieter
-            with ignoring(AttributeError, CancelledError, TimeoutError):
+            with ignoring(AttributeError, asyncio.CancelledError, TimeoutError):
                 await asyncio.wait_for(
                     asyncio.shield(self._handle_scheduler_coroutine), 0.1
                 )
@@ -1310,7 +1312,7 @@ async def _close(self, fast=False):
             del self.coroutines[:]
 
             if not fast:
-                with ignoring(TimeoutError):
+                with ignoring(TimeoutError, asyncio.CancelledError):
                     await asyncio.wait_for(asyncio.gather(*coroutines), 2)
 
             with ignoring(AttributeError):
diff --git a/distributed/comm/ucx.py b/distributed/comm/ucx.py
index 175d628a0f6..629a179e43e 100644
--- a/distributed/comm/ucx.py
+++ b/distributed/comm/ucx.py
@@ -8,7 +8,6 @@
 import ucp
 
 import logging
-import concurrent
 
 import dask
 import numpy as np
@@ -17,7 +16,7 @@
 from .core import Comm, Connector, Listener, CommClosedError
 from .registry import Backend, backends
 from .utils import ensure_concrete_host, to_frames, from_frames
-from ..utils import ensure_ip, get_ip, get_ipv6, nbytes, log_errors
+from ..utils import ensure_ip, get_ip, get_ipv6, nbytes, log_errors, CancelledError
 
 import dask
 import numpy as np
@@ -170,10 +169,7 @@ async def read(self, deserializers=("cuda", "dask", "pickle", "error")):
                 await self.ep.recv(is_cudas)
                 sizes = np.empty(nframes[0], dtype=np.uint64)
                 await self.ep.recv(sizes)
-            except (
-                ucp.exceptions.UCXBaseException,
-                concurrent.futures._base.CancelledError,
-            ):
+            except (ucp.exceptions.UCXBaseException, CancelledError):
                 self.abort()
                 raise CommClosedError("While reading, the connection was closed")
             else:
diff --git a/distributed/core.py b/distributed/core.py
index ac9be6728fc..5768f0f4d8e 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -1,6 +1,5 @@
 import asyncio
 from collections import defaultdict, deque
-from concurrent.futures import CancelledError
 from functools import partial
 from inspect import isawaitable
 import logging
@@ -35,6 +34,7 @@
     PeriodicCallback,
     parse_timedelta,
     has_keyword,
+    CancelledError,
 )
 from . import protocol
 
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index 98d04c78d17..8ca780a4eb2 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -7,9 +7,11 @@
 from threading import Lock
 import unittest
 import weakref
+from distutils.version import LooseVersion
 
 from tornado.ioloop import IOLoop
 from tornado import gen
+import tornado
 import pytest
 
 from dask.system import CPU_COUNT
@@ -451,6 +453,11 @@ async def test_scale_up_and_down():
             assert len(cluster.workers) == 1
 
 
+@pytest.mark.xfail(
+    sys.version_info >= (3, 8) and LooseVersion(tornado.version) < "6.0.3",
+    reason="Known issue with Python 3.8 and Tornado < 6.0.3. See https://github.com/tornadoweb/tornado/pull/2683.",
+    strict=True,
+)
 def test_silent_startup():
     code = """if 1:
         from time import sleep
diff --git a/distributed/protocol/tests/test_pickle.py b/distributed/protocol/tests/test_pickle.py
index 0ba776e2758..681992ef844 100644
--- a/distributed/protocol/tests/test_pickle.py
+++ b/distributed/protocol/tests/test_pickle.py
@@ -2,6 +2,7 @@
 import gc
 from operator import add
 import weakref
+import sys
 
 import pytest
 
@@ -23,6 +24,11 @@ def test_pickle_numpy():
     assert (loads(dumps(x)) == x).all()
 
 
+@pytest.mark.xfail(
+    sys.version_info[:2] == (3, 8),
+    reason="Sporadic failure on Python 3.8",
+    strict=False,
+)
 def test_pickle_functions():
     def make_closure():
         value = 1
diff --git a/distributed/tests/test_as_completed.py b/distributed/tests/test_as_completed.py
index 45833b302e1..d0249b121d6 100644
--- a/distributed/tests/test_as_completed.py
+++ b/distributed/tests/test_as_completed.py
@@ -1,4 +1,3 @@
-from concurrent.futures import CancelledError
 from collections.abc import Iterator
 from operator import add
 import queue
@@ -9,6 +8,7 @@
 from tornado import gen
 
 from distributed.client import _as_completed, as_completed, _first_completed
+from distributed.utils import CancelledError
 from distributed.utils_test import gen_cluster, inc, throws
 from distributed.utils_test import client, cluster_fixture, loop  # noqa: F401
 
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 4f075d582f3..392aec73be8 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -1,6 +1,5 @@
 import asyncio
 from collections import deque
-from concurrent.futures import CancelledError
 import gc
 import logging
 from operator import add
@@ -38,6 +37,7 @@
     profile,
     performance_report,
     TimeoutError,
+    CancelledError,
 )
 from distributed.comm import CommClosedError
 from distributed.client import (
diff --git a/distributed/tests/test_client_executor.py b/distributed/tests/test_client_executor.py
index 40639998852..1024990216d 100644
--- a/distributed/tests/test_client_executor.py
+++ b/distributed/tests/test_client_executor.py
@@ -2,7 +2,6 @@
 import time
 
 from concurrent.futures import (
-    CancelledError,
     TimeoutError,
     Future,
     wait,
@@ -15,6 +14,7 @@
 from toolz import take
 
 from distributed import Client
+from distributed.utils import CancelledError
 from distributed.utils_test import (
     slowinc,
     slowadd,
diff --git a/distributed/tests/test_failed_workers.py b/distributed/tests/test_failed_workers.py
index 3cc055b5246..cf0387c1cd2 100644
--- a/distributed/tests/test_failed_workers.py
+++ b/distributed/tests/test_failed_workers.py
@@ -1,4 +1,3 @@
-from concurrent.futures import CancelledError
 import os
 import random
 from time import sleep
@@ -12,7 +11,7 @@
 from distributed.comm import CommClosedError
 from distributed.client import wait
 from distributed.metrics import time
-from distributed.utils import sync, ignoring
+from distributed.utils import sync, ignoring, CancelledError
 from distributed.utils_test import (
     gen_cluster,
     cluster,
diff --git a/distributed/tests/test_steal.py b/distributed/tests/test_steal.py
index 9c4fef57d2a..a6a19332f5f 100644
--- a/distributed/tests/test_steal.py
+++ b/distributed/tests/test_steal.py
@@ -621,6 +621,11 @@ def long(delay):
         ) <= 1
 
 
+@pytest.mark.xfail(
+    sys.version_info[:2] == (3, 8),
+    reason="Sporadic failure on Python 3.8",
+    strict=False,
+)
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 5)] * 2)
 def test_cleanup_repeated_tasks(c, s, a, b):
     class Foo:
diff --git a/distributed/tests/test_stress.py b/distributed/tests/test_stress.py
index 5275bc47fd8..ab996e2b30d 100644
--- a/distributed/tests/test_stress.py
+++ b/distributed/tests/test_stress.py
@@ -1,4 +1,3 @@
-from concurrent.futures import CancelledError
 from operator import add
 import random
 import sys
@@ -12,7 +11,7 @@
 from distributed import Client, wait, Nanny
 from distributed.config import config
 from distributed.metrics import time
-from distributed.utils import All, ignoring
+from distributed.utils import All, ignoring, CancelledError
 from distributed.utils_test import (
     gen_cluster,
     cluster,
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 55dc7faf417..0bc2cf10988 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -1141,6 +1141,11 @@ def f(n):
 
 
 @pytest.mark.slow
+@pytest.mark.xfail(
+    sys.version_info[:2] == (3, 8),
+    reason="Sporadic failure on Python 3.8",
+    strict=False,
+)
 @gen_cluster(
     nthreads=[("127.0.0.1", 2)],
     client=True,
diff --git a/distributed/utils.py b/distributed/utils.py
index a771a3b280d..429a53cddde 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -3,7 +3,7 @@
 import atexit
 import click
 from collections import deque, OrderedDict, UserDict
-from concurrent.futures import ThreadPoolExecutor
+from concurrent.futures import ThreadPoolExecutor, CancelledError  # noqa: F401
 from contextlib import contextmanager
 import functools
 from hashlib import md5
@@ -1212,11 +1212,27 @@ def reset_logger_locks():
                 is_kernel_and_no_running_loop = True
 
         if not is_kernel_and_no_running_loop:
-            import tornado.platform.asyncio
 
-            asyncio.set_event_loop_policy(
-                tornado.platform.asyncio.AnyThreadEventLoopPolicy()
-            )
+            # TODO: Use tornado's AnyThreadEventLoopPolicy, instead of class below,
+            # once tornado > 6.0.3 is available.
+            if WINDOWS and hasattr(asyncio, "WindowsSelectorEventLoopPolicy"):
+                # WindowsProactorEventLoopPolicy is not compatible with tornado 6
+                # fallback to the pre-3.8 default of Selector
+                # https://github.com/tornadoweb/tornado/issues/2608
+                BaseEventLoopPolicy = asyncio.WindowsSelectorEventLoopPolicy
+            else:
+                BaseEventLoopPolicy = asyncio.DefaultEventLoopPolicy
+
+            class AnyThreadEventLoopPolicy(BaseEventLoopPolicy):
+                def get_event_loop(self):
+                    try:
+                        return super().get_event_loop()
+                    except (RuntimeError, AssertionError):
+                        loop = self.new_event_loop()
+                        self.set_event_loop(loop)
+                        return loop
+
+            asyncio.set_event_loop_policy(AnyThreadEventLoopPolicy())
 
 
 @functools.lru_cache(1000)
diff --git a/requirements.txt b/requirements.txt
index 87e148bb244..49b4d21940b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,7 +6,8 @@ psutil >= 5.0
 sortedcontainers !=2.0.0, !=2.0.1
 tblib >= 1.6.0
 toolz >= 0.7.4
-tornado >= 5
+tornado >= 5;python_version<'3.8'
+tornado >= 6.0.3;python_version>='3.8'
 zict >= 0.1.3
 pyyaml
 setuptools
diff --git a/setup.py b/setup.py
index e8c419cb147..155ae0c0274 100755
--- a/setup.py
+++ b/setup.py
@@ -48,6 +48,7 @@
         "Programming Language :: Python",
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.8",
         "Topic :: Scientific/Engineering",
         "Topic :: System :: Distributed Computing",
     ],

From 3f23aa30a50c1a5d382e467a17b74d0c80709a17 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Fri, 14 Feb 2020 12:45:11 -0800
Subject: [PATCH 0681/1550] Register Dask cuDF serializers (#3478)

As cuDF is gaining support for serializing using the Dask protocol as
well, make sure to register it's serializers there as well. cuDF
versions lacking this support will continue to behave the same (falling
back to pickle). However cuDF versions with this support will bypass
pickle. So only pay the cost of a host-to-device transfer.
---
 distributed/protocol/__init__.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/distributed/protocol/__init__.py b/distributed/protocol/__init__.py
index 84ee9420c78..6830f375e35 100644
--- a/distributed/protocol/__init__.py
+++ b/distributed/protocol/__init__.py
@@ -88,5 +88,7 @@ def _register_rmm():
 
 @cuda_serialize.register_lazy("cudf")
 @cuda_deserialize.register_lazy("cudf")
+@dask_serialize.register_lazy("cudf")
+@dask_deserialize.register_lazy("cudf")
 def _register_cudf():
     from cudf.comm import serialize

From 288c9577eb7a51a6a662499b686b948059da86ed Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= <berykubik@gmail.com>
Date: Fri, 14 Feb 2020 21:50:30 +0100
Subject: [PATCH 0682/1550] Do not duplicate messages in scheduler report
 (#3477)

---
 distributed/scheduler.py | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index b56c08eb38b..e6e6adf7ced 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -2439,13 +2439,10 @@ def report(self, msg, ts=None, client=None):
         If the message contains a key then we only send the message to those
         comms that care about the key.
         """
+        comms = set()
         if client is not None:
             try:
-                comm = self.client_comms[client]
-                comm.send(msg)
-            except CommClosedError:
-                if self.status == "running":
-                    logger.critical("Tried writing to closed comm: %s", msg)
+                comms.add(self.client_comms[client])
             except KeyError:
                 pass
 
@@ -2453,14 +2450,14 @@ def report(self, msg, ts=None, client=None):
             ts = self.tasks.get(msg["key"])
         if ts is None:
             # Notify all clients
-            comms = self.client_comms.values()
+            comms |= set(self.client_comms.values())
         else:
             # Notify clients interested in key
-            comms = [
+            comms |= {
                 self.client_comms[c.client_key]
                 for c in ts.who_wants
                 if c.client_key in self.client_comms
-            ]
+            }
         for c in comms:
             try:
                 c.send(msg)

From c1f265125ab068a54a4c67501250550dcfff688e Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sat, 15 Feb 2020 17:49:44 -0800
Subject: [PATCH 0683/1550] Remove --verbose flag from CI runs (#3484)

Historically we included the verbose flag because if things hung it was
useful to see which test in particular was causing the hang.  This
hasn't been so important recently.

It's somewhat annoying to scroll through all of the tests one by one.
This seems more important today than the hung test case.

* Give test_workspace_concurrency more time

This is more important now that we're using spawn
---
 .github/workflows/ci-windows.yaml          | 2 +-
 continuous_integration/travis/run_tests.sh | 2 +-
 distributed/tests/test_diskutils.py        | 3 ++-
 setup.cfg                                  | 2 +-
 4 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/ci-windows.yaml b/.github/workflows/ci-windows.yaml
index 707c50ffc76..75c4b294e88 100644
--- a/.github/workflows/ci-windows.yaml
+++ b/.github/workflows/ci-windows.yaml
@@ -43,4 +43,4 @@ jobs:
         shell: bash -l {0}
         env:
           PYTHONFAULTHANDLER: 1
-        run: py.test -m "not avoid_travis" distributed --verbose -r s --timeout-method=thread --timeout=300 --durations=20
+        run: py.test -m "not avoid_travis" distributed -r s --timeout-method=thread --timeout=300 --durations=20
diff --git a/continuous_integration/travis/run_tests.sh b/continuous_integration/travis/run_tests.sh
index dbc0b21ff03..14c3db7750a 100644
--- a/continuous_integration/travis/run_tests.sh
+++ b/continuous_integration/travis/run_tests.sh
@@ -1,4 +1,4 @@
-export PYTEST_OPTIONS="--verbose -r s --timeout-method=thread --timeout=300 --durations=20"
+export PYTEST_OPTIONS="-r s --timeout-method=thread --timeout=300 --durations=20"
 if [[ $RUNSLOW != false ]]; then
     export PYTEST_OPTIONS="$PYTEST_OPTIONS --runslow"
 fi
diff --git a/distributed/tests/test_diskutils.py b/distributed/tests/test_diskutils.py
index 86b472e184a..f69485cfa46 100644
--- a/distributed/tests/test_diskutils.py
+++ b/distributed/tests/test_diskutils.py
@@ -272,12 +272,13 @@ def _test_workspace_concurrency(tmpdir, timeout, max_procs):
     return n_created, n_purged
 
 
+@pytest.mark.slow
 def test_workspace_concurrency(tmpdir):
     if WINDOWS:
         raise pytest.xfail.Exception("TODO: unknown failure on windows")
     if sys.version_info < (3, 7):
         raise pytest.xfail.Exception("TODO: unknown failure on Python 3.6")
-    _test_workspace_concurrency(tmpdir, 2.0, 6)
+    _test_workspace_concurrency(tmpdir, 5.0, 6)
 
 
 @pytest.mark.slow
diff --git a/setup.cfg b/setup.cfg
index 042a8b86f35..764ac7ad02c 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -36,7 +36,7 @@ tag_prefix =
 parentdir_prefix = distributed-
 
 [tool:pytest]
-addopts = -rsx -v --durations=10
+addopts = -rsx --durations=10
 minversion = 3.2
 markers =
     slow: marks tests as slow (deselect with '-m "not slow"')

From ba8dad3fc76c54e788e54b8b425d7debf08d12a6 Mon Sep 17 00:00:00 2001
From: Chris Roat <1053153+chrisroat@users.noreply.github.com>
Date: Sun, 16 Feb 2020 09:21:21 -0800
Subject: [PATCH 0684/1550] Propose fix for collection based resources docs
 (#3480)

Besides a missing end brace, the dask keys need to be flattened before they can be turned into a tuple (they can contain lists, which are not hashable as dict keys).
---
 docs/source/resources.rst | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/docs/source/resources.rst b/docs/source/resources.rst
index bd4d8b5e81c..f9449dbd8be 100644
--- a/docs/source/resources.rst
+++ b/docs/source/resources.rst
@@ -96,12 +96,13 @@ delayed objects.  You can pass a dictionary mapping keys of the collection to
 resource requirements during compute or persist calls.
 
 .. code-block:: python
-
+    from dask import core
+    
     x = dd.read_csv(...)
     y = x.map_partitions(func1)
     z = y.map_partitions(func2)
 
-    z.compute(resources={tuple(y.__dask_keys__()): {'GPU': 1})
+    z.compute(resources={tuple(core.flatten(y.__dask_keys__())): {'GPU': 1}})
 
 In some cases (such as the case above) the keys for ``y`` may be optimized away
 before execution.  You can avoid that either by requiring them as an explicit
@@ -110,4 +111,4 @@ output, or by passing the ``optimize_graph=False`` keyword.
 
 .. code-block:: python
 
-    z.compute(resources={tuple(y.__dask_keys__()): {'GPU': 1}, optimize_graph=False)
+    z.compute(resources={tuple(core.flatten(y.__dask_keys__())): {'GPU': 1}}, optimize_graph=False)

From c9b8fc35e2209855c45eb7b70f585193b65516c1 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Sun, 16 Feb 2020 11:24:06 -0600
Subject: [PATCH 0685/1550] Update NumPy array serialization to handle
 non-contiguous slices (#3474)

---
 distributed/protocol/numpy.py            | 17 +++++++++++++----
 distributed/protocol/tests/test_numpy.py |  1 +
 2 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/distributed/protocol/numpy.py b/distributed/protocol/numpy.py
index 9a1f493c333..a2c9c2933e6 100644
--- a/distributed/protocol/numpy.py
+++ b/distributed/protocol/numpy.py
@@ -44,12 +44,21 @@ def serialize_numpy_ndarray(x):
     else:
         dt = (0, x.dtype.str)
 
-    # Only serialize non-broadcasted data for arrays with zero strided axes
+    # Only serialize broadcastable data for arrays with zero strided axes
+    broadcast_to = None
     if 0 in x.strides:
         broadcast_to = x.shape
-        x = x[tuple(slice(None) if s != 0 else slice(1) for s in x.strides)]
-    else:
-        broadcast_to = None
+        strides = x.strides
+        writeable = x.flags.writeable
+        x = x[tuple(slice(None) if s != 0 else slice(1) for s in strides)]
+        if not x.flags.c_contiguous and not x.flags.f_contiguous:
+            # Broadcasting can only be done with contiguous arrays
+            x = np.ascontiguousarray(x)
+            x = np.lib.stride_tricks.as_strided(
+                x,
+                strides=[j if i != 0 else i for i, j in zip(strides, x.strides)],
+                writeable=writeable,
+            )
 
     if not x.shape:
         # 0d array
diff --git a/distributed/protocol/tests/test_numpy.py b/distributed/protocol/tests/test_numpy.py
index 432b749e27e..e6dfd9764e2 100644
--- a/distributed/protocol/tests/test_numpy.py
+++ b/distributed/protocol/tests/test_numpy.py
@@ -268,6 +268,7 @@ def test_large_numpy_array():
         np.broadcast_to(np.arange(10), (20, 10)),  # Some strides are 0
         np.broadcast_to(1, (3, 4, 2)),  # All strides are 0
         np.broadcast_to(np.arange(100)[:1], 5),  # x.base is larger than x
+        np.broadcast_to(np.arange(5), (4, 5))[:, ::-1],
     ],
 )
 @pytest.mark.parametrize("writeable", [True, False])

From 9408ebcb3f9ef1420640febf67b48c48fdcd2dd8 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Sun, 16 Feb 2020 11:38:54 -0800
Subject: [PATCH 0686/1550] Serialize 1-D, contiguous, `uint8` CUDA frames
 (#3475)

* Space things out to improve readability

* Assign `frames` for clarity

* Check CUDA contiguous arrays the same way

* Use keyword arguments with CUDA array constructors

Should make it a little clearer what the arguments relate to. Also makes
them less dependent on changes in the signature of the constructors.

* Always serialize the CUDA array's `strides`

* Make sure CuPy always uses the `strides` provided

* Drop workarounds for unknown `strides`

As we always include the actual `strides` in the `header`, we can rely
on this when deserializing the data. So drop the workarounds added for
C-contiguous data where `strides` could be `None` as we still have the
exact `strides` in that case.

* Convert CUDA arrays into 1-D contiguous arrays

* Avoid copying C/F contiguous CUDA arrays

* Cast CUDA array data to `uint8` before serializing

As we will ultimately read any data in as `uint8`, go ahead and cast to
`uint8` as part of serialization. This is a good first order check to
make sure that we are able to serialize the dat

To simplify handling of the data a bit, go ahead and cast it to `uint8`
before serializing. This makes contiguity checks trivial.

* Test serialization of F-contiguous CUDA arrays
---
 distributed/protocol/cupy.py             | 28 ++++++++++++++----------
 distributed/protocol/numba.py            | 24 ++++++++++----------
 distributed/protocol/tests/test_cupy.py  |  9 +++++---
 distributed/protocol/tests/test_numba.py |  7 ++++--
 4 files changed, 40 insertions(+), 28 deletions(-)

diff --git a/distributed/protocol/cupy.py b/distributed/protocol/cupy.py
index 087de6f9663..3ba5ca51597 100644
--- a/distributed/protocol/cupy.py
+++ b/distributed/protocol/cupy.py
@@ -6,21 +6,15 @@
 
 
 class PatchedCudaArrayInterface:
-    """This class do two things:
-        1) Makes sure that __cuda_array_interface__['strides']
-           behaves as specified in the protocol.
-        2) Makes sure that the cuda context is active
+    """This class does one thing:
+        1) Makes sure that the cuda context is active
            when deallocating the base cuda array.
         Notice, this is only needed when the array to deserialize
         isn't a native cupy array.
     """
 
     def __init__(self, ary):
-        cai = ary.__cuda_array_interface__
-        cai_cupy_vsn = cupy.ndarray(0).__cuda_array_interface__["version"]
-        if cai.get("strides") is None and cai_cupy_vsn < 2:
-            cai.pop("strides", None)
-        self.__cuda_array_interface__ = cai
+        self.__cuda_array_interface__ = ary.__cuda_array_interface__
         # Save a ref to ary so it won't go out of scope
         self.base = ary
 
@@ -39,11 +33,18 @@ def __del__(self):
 @cuda_serialize.register(cupy.ndarray)
 def serialize_cupy_ndarray(x):
     # Making sure `x` is behaving
-    if not x.flags.c_contiguous:
+    if not (x.flags["C_CONTIGUOUS"] or x.flags["F_CONTIGUOUS"]):
         x = cupy.array(x, copy=True)
 
     header = x.__cuda_array_interface__.copy()
-    return header, [x]
+    header["strides"] = tuple(x.strides)
+    frames = [
+        cupy.ndarray(
+            shape=(x.nbytes,), dtype=cupy.dtype("u1"), memptr=x.data, strides=(1,)
+        )
+    ]
+
+    return header, frames
 
 
 @cuda_deserialize.register(cupy.ndarray)
@@ -52,6 +53,9 @@ def deserialize_cupy_array(header, frames):
     if not isinstance(frame, cupy.ndarray):
         frame = PatchedCudaArrayInterface(frame)
     arr = cupy.ndarray(
-        header["shape"], dtype=header["typestr"], memptr=cupy.asarray(frame).data
+        shape=header["shape"],
+        dtype=header["typestr"],
+        memptr=cupy.asarray(frame).data,
+        strides=header["strides"],
     )
     return arr
diff --git a/distributed/protocol/numba.py b/distributed/protocol/numba.py
index 9b33660e2bd..3d2b4879c3b 100644
--- a/distributed/protocol/numba.py
+++ b/distributed/protocol/numba.py
@@ -6,13 +6,21 @@
 @cuda_serialize.register(numba.cuda.devicearray.DeviceNDArray)
 def serialize_numba_ndarray(x):
     # Making sure `x` is behaving
-    if not x.is_c_contiguous():
+    if not (x.flags["C_CONTIGUOUS"] or x.flags["F_CONTIGUOUS"]):
         shape = x.shape
         t = numba.cuda.device_array(shape, dtype=x.dtype)
         t.copy_to_device(x)
         x = t
+
     header = x.__cuda_array_interface__.copy()
-    return header, [x]
+    header["strides"] = tuple(x.strides)
+    frames = [
+        numba.cuda.cudadrv.devicearray.DeviceNDArray(
+            shape=(x.nbytes,), strides=(1,), dtype=np.dtype("u1"), gpu_data=x.gpu_data,
+        )
+    ]
+
+    return header, frames
 
 
 @cuda_deserialize.register(numba.cuda.devicearray.DeviceNDArray)
@@ -21,16 +29,10 @@ def deserialize_numba_ndarray(header, frames):
     shape = header["shape"]
     strides = header["strides"]
 
-    # Starting with __cuda_array_interface__ version 2, strides can be None,
-    # meaning the array is C-contiguous, so we have to calculate it.
-    if strides is None:
-        itemsize = np.dtype(header["typestr"]).itemsize
-        strides = tuple((np.cumprod((1,) + shape[:0:-1]) * itemsize).tolist())
-
     arr = numba.cuda.devicearray.DeviceNDArray(
-        shape,
-        strides,
-        np.dtype(header["typestr"]),
+        shape=shape,
+        strides=strides,
+        dtype=np.dtype(header["typestr"]),
         gpu_data=numba.cuda.as_cuda_array(frame).gpu_data,
     )
     return arr
diff --git a/distributed/protocol/tests/test_cupy.py b/distributed/protocol/tests/test_cupy.py
index d2965d3af3f..4b3ea27cc9c 100644
--- a/distributed/protocol/tests/test_cupy.py
+++ b/distributed/protocol/tests/test_cupy.py
@@ -3,12 +3,15 @@
 import pytest
 
 cupy = pytest.importorskip("cupy")
+numpy = pytest.importorskip("numpy")
 
 
-@pytest.mark.parametrize("size", [0, 10])
+@pytest.mark.parametrize("shape", [(0,), (5,), (4, 6), (10, 11), (2, 3, 5)])
 @pytest.mark.parametrize("dtype", ["u1", "u4", "u8", "f4"])
-def test_serialize_cupy(size, dtype):
-    x = cupy.arange(size, dtype=dtype)
+@pytest.mark.parametrize("order", ["C", "F"])
+def test_serialize_cupy(shape, dtype, order):
+    x = cupy.arange(numpy.product(shape), dtype=dtype)
+    x = cupy.ndarray(shape, dtype=x.dtype, memptr=x.data, order=order)
     header, frames = serialize(x, serializers=("cuda", "dask", "pickle"))
     y = deserialize(header, frames, deserializers=("cuda", "dask", "pickle", "error"))
 
diff --git a/distributed/protocol/tests/test_numba.py b/distributed/protocol/tests/test_numba.py
index 21b722fb1b0..4f3a9e7116e 100644
--- a/distributed/protocol/tests/test_numba.py
+++ b/distributed/protocol/tests/test_numba.py
@@ -6,12 +6,15 @@
 np = pytest.importorskip("numpy")
 
 
+@pytest.mark.parametrize("shape", [(0,), (5,), (4, 6), (10, 11), (2, 3, 5)])
 @pytest.mark.parametrize("dtype", ["u1", "u4", "u8", "f4"])
-def test_serialize_numba(dtype):
+@pytest.mark.parametrize("order", ["C", "F"])
+def test_serialize_numba(shape, dtype, order):
     if not cuda.is_available():
         pytest.skip("CUDA is not available")
 
-    ary = np.arange(100, dtype=dtype)
+    ary = np.arange(np.product(shape), dtype=dtype)
+    ary = np.ndarray(shape, dtype=ary.dtype, buffer=ary.data, order=order)
     x = cuda.to_device(ary)
     header, frames = serialize(x, serializers=("cuda", "dask", "pickle"))
     y = deserialize(header, frames, deserializers=("cuda", "dask", "pickle", "error"))

From cc7ecdf2abb76a7ed21d0cb4c9a9c92559f638c4 Mon Sep 17 00:00:00 2001
From: Brett Naul <brettnaul@gmail.com>
Date: Sun, 16 Feb 2020 17:32:13 -0800
Subject: [PATCH 0687/1550] Check exact equality for worker state (#3483)

---
 distributed/scheduler.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index e6e6adf7ced..cba399318cc 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -276,10 +276,10 @@ def __init__(
         self.extra = extra or {}
 
     def __hash__(self):
-        return hash((self.name, self.host))
+        return hash(self.address)
 
     def __eq__(self, other):
-        return type(self) == type(other) and hash(self) == hash(other)
+        return type(self) == type(other) and self.address == other.address
 
     @property
     def host(self):

From e11674a987f98571f92f11fe74cc4588adab4af8 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Tue, 18 Feb 2020 14:11:50 -0800
Subject: [PATCH 0688/1550] Register cuML serializers (#3485)

---
 distributed/protocol/__init__.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/distributed/protocol/__init__.py b/distributed/protocol/__init__.py
index 6830f375e35..b82461ef054 100644
--- a/distributed/protocol/__init__.py
+++ b/distributed/protocol/__init__.py
@@ -92,3 +92,11 @@ def _register_rmm():
 @dask_deserialize.register_lazy("cudf")
 def _register_cudf():
     from cudf.comm import serialize
+
+
+@cuda_serialize.register_lazy("cuml")
+@cuda_deserialize.register_lazy("cuml")
+@dask_serialize.register_lazy("cuml")
+@dask_deserialize.register_lazy("cuml")
+def _register_cuml():
+    from cuml.comm import serialize

From a04a6321a46870f8432da3ee3789bc7cd7c27bbb Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Tue, 18 Feb 2020 16:47:34 -0600
Subject: [PATCH 0689/1550] Msgpack 1.0 compatibility (#3494)

---
 continuous_integration/environment.yml      | 2 +-
 continuous_integration/travis/install.sh    | 3 ++-
 distributed/protocol/tests/test_numpy.py    | 2 +-
 distributed/protocol/tests/test_protocol.py | 4 +++-
 distributed/protocol/utils.py               | 1 +
 requirements.txt                            | 2 +-
 6 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/continuous_integration/environment.yml b/continuous_integration/environment.yml
index f69d919e879..8f8e425dcab 100644
--- a/continuous_integration/environment.yml
+++ b/continuous_integration/environment.yml
@@ -14,7 +14,7 @@ dependencies:
   - ipywidgets
   - joblib
   - jupyter_client
-  - msgpack-python
+  - msgpack-python>=0.6.0
   - prometheus_client
   - psutil
   - pytest
diff --git a/continuous_integration/travis/install.sh b/continuous_integration/travis/install.sh
index b631ac3bc6c..68b842aa033 100644
--- a/continuous_integration/travis/install.sh
+++ b/continuous_integration/travis/install.sh
@@ -40,6 +40,7 @@ conda install -c conda-forge -q \
     ipywidgets \
     joblib \
     jupyter_client \
+    msgpack-python>=0.6.0 \
     netcdf4 \
     paramiko \
     prometheus_client \
@@ -71,7 +72,7 @@ python -m pip install -q git+https://github.com/joblib/joblib.git --upgrade --no
 python -m pip install -q git+https://github.com/intake/filesystem_spec.git --upgrade --no-deps
 python -m pip install -q git+https://github.com/dask/s3fs.git --upgrade --no-deps
 python -m pip install -q git+https://github.com/dask/zict.git --upgrade --no-deps
-python -m pip install -q sortedcollections msgpack --no-deps
+python -m pip install -q sortedcollections --no-deps
 python -m pip install -q keras --upgrade --no-deps
 
 if [[ $CRICK == true ]]; then
diff --git a/distributed/protocol/tests/test_numpy.py b/distributed/protocol/tests/test_numpy.py
index e6dfd9764e2..99a298d9694 100644
--- a/distributed/protocol/tests/test_numpy.py
+++ b/distributed/protocol/tests/test_numpy.py
@@ -196,7 +196,7 @@ def test_compress_numpy():
     frames = dumps({"x": to_serialize(x)})
     assert sum(map(nbytes, frames)) < x.nbytes
 
-    header = msgpack.loads(frames[2], raw=False, use_list=False)
+    header = msgpack.loads(frames[2], raw=False, use_list=False, strict_map_key=False)
     try:
         import blosc  # noqa: F401
     except ImportError:
diff --git a/distributed/protocol/tests/test_protocol.py b/distributed/protocol/tests/test_protocol.py
index bf16aecf2f4..d3536933a96 100644
--- a/distributed/protocol/tests/test_protocol.py
+++ b/distributed/protocol/tests/test_protocol.py
@@ -166,7 +166,9 @@ def test_loads_without_deserialization_avoids_compression():
 
 def eq_frames(a, b):
     if b"headers" in a:
-        return msgpack.loads(a, use_list=False) == msgpack.loads(b, use_list=False)
+        return msgpack.loads(a, use_list=False, strict_map_key=False) == msgpack.loads(
+            b, use_list=False, strict_map_key=False
+        )
     else:
         return a == b
 
diff --git a/distributed/protocol/utils.py b/distributed/protocol/utils.py
index 68de0bebd32..e5b9247e77f 100644
--- a/distributed/protocol/utils.py
+++ b/distributed/protocol/utils.py
@@ -9,6 +9,7 @@
 msgpack_opts = {
     ("max_%s_len" % x): 2 ** 31 - 1 for x in ["str", "bin", "array", "map", "ext"]
 }
+msgpack_opts["strict_map_key"] = False
 
 try:
     msgpack.loads(msgpack.dumps(""), raw=False, **msgpack_opts)
diff --git a/requirements.txt b/requirements.txt
index 49b4d21940b..3f827e250e6 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,7 +1,7 @@
 click >= 6.6
 cloudpickle >= 0.2.2
 dask >= 2.9.0
-msgpack
+msgpack >= 0.6.0
 psutil >= 5.0
 sortedcontainers !=2.0.0, !=2.0.1
 tblib >= 1.6.0

From 54657393afb49ef3c6cb41ab739f0d2b0d22621f Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Tue, 18 Feb 2020 19:54:58 -0800
Subject: [PATCH 0690/1550] Suppress cuML `ImportError` (#3499)

* Suppress cuML `ImportError`

If cuML is present, but `cuml.comm` does not yet exist, make sure to
suppress the `ImportError`. After all there is nothing to do here in
this case and we don't want to raise unnecessary errors.

* Use `ignoring` instead of `suppress`
---
 distributed/protocol/__init__.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/distributed/protocol/__init__.py b/distributed/protocol/__init__.py
index b82461ef054..c83d33c4868 100644
--- a/distributed/protocol/__init__.py
+++ b/distributed/protocol/__init__.py
@@ -99,4 +99,5 @@ def _register_cudf():
 @dask_serialize.register_lazy("cuml")
 @dask_deserialize.register_lazy("cuml")
 def _register_cuml():
-    from cuml.comm import serialize
+    with ignoring(ImportError):
+        from cuml.comm import serialize

From b5e95ed77d5df7b07f43f02fc39881d483f061a4 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Tue, 18 Feb 2020 19:56:36 -0800
Subject: [PATCH 0691/1550] Add dask serialization of CUDA objects (#3482)

* Run `isort` on CUDA protocol `import`s

* Align CuPy serialize/deserialize function names

* Prefix CUDA serializers with `cuda_`

This should make room for Dask serializers to also be specified and
added.

* Add Dask serializers for RMM `DeviceBuffer`s

To make TCP a bit more performant with RMM, provide Dask serializers to
allow going to and from host memory.

* Add Dask serializers for Numba `DeviceNDArray`s

* Add Dask serializers for CuPy `ndarray`s

* Parametrize serializers in CUDA object tests

To make sure that different CUDA objects can use different serialization
protocols, test with each one individual and ensure it completes. In
particular test both "cuda" and "dask". Where supported also test
"pickle", but skip it when it is not (like with Numba).

* Check frames are the expected type

To make sure Dask can handle transmission of the frames serialized, test
they match the type expected by the protocol used. With "cuda" ensure we
get something that supports `__cuda_array_interface__`. With "dask" make
sure we get a `memoryview`.
---
 distributed/protocol/__init__.py         |  6 +++++
 distributed/protocol/cupy.py             | 27 ++++++++++++++++---
 distributed/protocol/numba.py            | 33 +++++++++++++++++++++---
 distributed/protocol/rmm.py              | 28 +++++++++++++++++---
 distributed/protocol/tests/test_cupy.py  | 12 ++++++---
 distributed/protocol/tests/test_numba.py | 12 ++++++---
 distributed/protocol/tests/test_rmm.py   | 12 ++++++---
 7 files changed, 111 insertions(+), 19 deletions(-)

diff --git a/distributed/protocol/__init__.py b/distributed/protocol/__init__.py
index c83d33c4868..212051427f5 100644
--- a/distributed/protocol/__init__.py
+++ b/distributed/protocol/__init__.py
@@ -70,18 +70,24 @@ def _register_torch():
 
 @cuda_serialize.register_lazy("cupy")
 @cuda_deserialize.register_lazy("cupy")
+@dask_serialize.register_lazy("cupy")
+@dask_deserialize.register_lazy("cupy")
 def _register_cupy():
     from . import cupy
 
 
 @cuda_serialize.register_lazy("numba")
 @cuda_deserialize.register_lazy("numba")
+@dask_serialize.register_lazy("numba")
+@dask_deserialize.register_lazy("numba")
 def _register_numba():
     from . import numba
 
 
 @cuda_serialize.register_lazy("rmm")
 @cuda_deserialize.register_lazy("rmm")
+@dask_serialize.register_lazy("rmm")
+@dask_deserialize.register_lazy("rmm")
 def _register_rmm():
     from . import rmm
 
diff --git a/distributed/protocol/cupy.py b/distributed/protocol/cupy.py
index 3ba5ca51597..40bf6efda4f 100644
--- a/distributed/protocol/cupy.py
+++ b/distributed/protocol/cupy.py
@@ -2,7 +2,14 @@
 Efficient serialization GPU arrays.
 """
 import cupy
-from .cuda import cuda_serialize, cuda_deserialize
+
+from .cuda import cuda_deserialize, cuda_serialize
+from .serialize import dask_deserialize, dask_serialize
+
+try:
+    from .rmm import dask_deserialize_rmm_device_buffer as dask_deserialize_cuda_buffer
+except ImportError:
+    from .numba import dask_deserialize_numba_array as dask_deserialize_cuda_buffer
 
 
 class PatchedCudaArrayInterface:
@@ -31,7 +38,7 @@ def __del__(self):
 
 
 @cuda_serialize.register(cupy.ndarray)
-def serialize_cupy_ndarray(x):
+def cuda_serialize_cupy_ndarray(x):
     # Making sure `x` is behaving
     if not (x.flags["C_CONTIGUOUS"] or x.flags["F_CONTIGUOUS"]):
         x = cupy.array(x, copy=True)
@@ -48,7 +55,7 @@ def serialize_cupy_ndarray(x):
 
 
 @cuda_deserialize.register(cupy.ndarray)
-def deserialize_cupy_array(header, frames):
+def cuda_deserialize_cupy_ndarray(header, frames):
     (frame,) = frames
     if not isinstance(frame, cupy.ndarray):
         frame = PatchedCudaArrayInterface(frame)
@@ -59,3 +66,17 @@ def deserialize_cupy_array(header, frames):
         strides=header["strides"],
     )
     return arr
+
+
+@dask_serialize.register(cupy.ndarray)
+def dask_serialize_cupy_ndarray(x):
+    header, frames = cuda_serialize_cupy_ndarray(x)
+    frames = [memoryview(cupy.asnumpy(f)) for f in frames]
+    return header, frames
+
+
+@dask_deserialize.register(cupy.ndarray)
+def dask_deserialize_cupy_ndarray(header, frames):
+    frames = [dask_deserialize_cuda_buffer(header, frames)]
+    arr = cuda_deserialize_cupy_ndarray(header, frames)
+    return arr
diff --git a/distributed/protocol/numba.py b/distributed/protocol/numba.py
index 3d2b4879c3b..1070c080e61 100644
--- a/distributed/protocol/numba.py
+++ b/distributed/protocol/numba.py
@@ -1,10 +1,17 @@
-import numpy as np
 import numba.cuda
-from .cuda import cuda_serialize, cuda_deserialize
+import numpy as np
+
+from .cuda import cuda_deserialize, cuda_serialize
+from .serialize import dask_deserialize, dask_serialize
+
+try:
+    from .rmm import dask_deserialize_rmm_device_buffer
+except ImportError:
+    dask_deserialize_rmm_device_buffer = None
 
 
 @cuda_serialize.register(numba.cuda.devicearray.DeviceNDArray)
-def serialize_numba_ndarray(x):
+def cuda_serialize_numba_ndarray(x):
     # Making sure `x` is behaving
     if not (x.flags["C_CONTIGUOUS"] or x.flags["F_CONTIGUOUS"]):
         shape = x.shape
@@ -24,7 +31,7 @@ def serialize_numba_ndarray(x):
 
 
 @cuda_deserialize.register(numba.cuda.devicearray.DeviceNDArray)
-def deserialize_numba_ndarray(header, frames):
+def cuda_deserialize_numba_ndarray(header, frames):
     (frame,) = frames
     shape = header["shape"]
     strides = header["strides"]
@@ -36,3 +43,21 @@ def deserialize_numba_ndarray(header, frames):
         gpu_data=numba.cuda.as_cuda_array(frame).gpu_data,
     )
     return arr
+
+
+@dask_serialize.register(numba.cuda.devicearray.DeviceNDArray)
+def dask_serialize_numba_ndarray(x):
+    header, frames = cuda_serialize_numba_ndarray(x)
+    frames = [memoryview(f.copy_to_host()) for f in frames]
+    return header, frames
+
+
+@dask_deserialize.register(numba.cuda.devicearray.DeviceNDArray)
+def dask_deserialize_numba_array(header, frames):
+    if dask_deserialize_rmm_device_buffer:
+        frames = [dask_deserialize_rmm_device_buffer(header, frames)]
+    else:
+        frames = [numba.cuda.to_device(np.asarray(memoryview(f))) for f in frames]
+
+    arr = cuda_deserialize_numba_ndarray(header, frames)
+    return arr
diff --git a/distributed/protocol/rmm.py b/distributed/protocol/rmm.py
index cdf22f8218f..ae2db0d528b 100644
--- a/distributed/protocol/rmm.py
+++ b/distributed/protocol/rmm.py
@@ -1,18 +1,22 @@
+import numba
+import numba.cuda
+import numpy
 import rmm
-from .cuda import cuda_serialize, cuda_deserialize
 
+from .cuda import cuda_deserialize, cuda_serialize
+from .serialize import dask_deserialize, dask_serialize
 
 # Used for RMM 0.11.0+ otherwise Numba serializers used
 if hasattr(rmm, "DeviceBuffer"):
 
     @cuda_serialize.register(rmm.DeviceBuffer)
-    def serialize_rmm_device_buffer(x):
+    def cuda_serialize_rmm_device_buffer(x):
         header = x.__cuda_array_interface__.copy()
         frames = [x]
         return header, frames
 
     @cuda_deserialize.register(rmm.DeviceBuffer)
-    def deserialize_rmm_device_buffer(header, frames):
+    def cuda_deserialize_rmm_device_buffer(header, frames):
         (arr,) = frames
 
         # We should already have `DeviceBuffer`
@@ -21,3 +25,21 @@ def deserialize_rmm_device_buffer(header, frames):
         assert isinstance(arr, rmm.DeviceBuffer)
 
         return arr
+
+    @dask_serialize.register(rmm.DeviceBuffer)
+    def dask_serialize_rmm_device_buffer(x):
+        header = x.__cuda_array_interface__.copy()
+        frames = [numba.cuda.as_cuda_array(x).copy_to_host().data]
+        return header, frames
+
+    @dask_deserialize.register(rmm.DeviceBuffer)
+    def dask_deserialize_rmm_device_buffer(header, frames):
+        (frame,) = frames
+
+        arr = numpy.asarray(memoryview(frame))
+        ptr = arr.__array_interface__["data"][0]
+        size = arr.nbytes
+
+        buf = rmm.DeviceBuffer(ptr=ptr, size=size)
+
+        return buf
diff --git a/distributed/protocol/tests/test_cupy.py b/distributed/protocol/tests/test_cupy.py
index 4b3ea27cc9c..5470266fce5 100644
--- a/distributed/protocol/tests/test_cupy.py
+++ b/distributed/protocol/tests/test_cupy.py
@@ -9,11 +9,17 @@
 @pytest.mark.parametrize("shape", [(0,), (5,), (4, 6), (10, 11), (2, 3, 5)])
 @pytest.mark.parametrize("dtype", ["u1", "u4", "u8", "f4"])
 @pytest.mark.parametrize("order", ["C", "F"])
-def test_serialize_cupy(shape, dtype, order):
+@pytest.mark.parametrize("serializers", [("cuda",), ("dask",), ("pickle",)])
+def test_serialize_cupy(shape, dtype, order, serializers):
     x = cupy.arange(numpy.product(shape), dtype=dtype)
     x = cupy.ndarray(shape, dtype=x.dtype, memptr=x.data, order=order)
-    header, frames = serialize(x, serializers=("cuda", "dask", "pickle"))
-    y = deserialize(header, frames, deserializers=("cuda", "dask", "pickle", "error"))
+    header, frames = serialize(x, serializers=serializers)
+    y = deserialize(header, frames, deserializers=serializers)
+
+    if serializers[0] == "cuda":
+        assert all(hasattr(f, "__cuda_array_interface__") for f in frames)
+    elif serializers[0] == "dask":
+        assert all(isinstance(f, memoryview) for f in frames)
 
     assert (x == y).all()
 
diff --git a/distributed/protocol/tests/test_numba.py b/distributed/protocol/tests/test_numba.py
index 4f3a9e7116e..61213640715 100644
--- a/distributed/protocol/tests/test_numba.py
+++ b/distributed/protocol/tests/test_numba.py
@@ -9,15 +9,21 @@
 @pytest.mark.parametrize("shape", [(0,), (5,), (4, 6), (10, 11), (2, 3, 5)])
 @pytest.mark.parametrize("dtype", ["u1", "u4", "u8", "f4"])
 @pytest.mark.parametrize("order", ["C", "F"])
-def test_serialize_numba(shape, dtype, order):
+@pytest.mark.parametrize("serializers", [("cuda",), ("dask",)])
+def test_serialize_numba(shape, dtype, order, serializers):
     if not cuda.is_available():
         pytest.skip("CUDA is not available")
 
     ary = np.arange(np.product(shape), dtype=dtype)
     ary = np.ndarray(shape, dtype=ary.dtype, buffer=ary.data, order=order)
     x = cuda.to_device(ary)
-    header, frames = serialize(x, serializers=("cuda", "dask", "pickle"))
-    y = deserialize(header, frames, deserializers=("cuda", "dask", "pickle", "error"))
+    header, frames = serialize(x, serializers=serializers)
+    y = deserialize(header, frames, deserializers=serializers)
+
+    if serializers[0] == "cuda":
+        assert all(hasattr(f, "__cuda_array_interface__") for f in frames)
+    elif serializers[0] == "dask":
+        assert all(isinstance(f, memoryview) for f in frames)
 
     hx = np.empty_like(ary)
     hy = np.empty_like(ary)
diff --git a/distributed/protocol/tests/test_rmm.py b/distributed/protocol/tests/test_rmm.py
index eff3325289e..8176f4d22f7 100644
--- a/distributed/protocol/tests/test_rmm.py
+++ b/distributed/protocol/tests/test_rmm.py
@@ -7,7 +7,8 @@
 
 
 @pytest.mark.parametrize("size", [0, 3, 10])
-def test_serialize_rmm_device_buffer(size):
+@pytest.mark.parametrize("serializers", [("cuda",), ("dask",), ("pickle",)])
+def test_serialize_rmm_device_buffer(size, serializers):
     if not hasattr(rmm, "DeviceBuffer"):
         pytest.skip("RMM pre-0.11.0 does not have DeviceBuffer")
 
@@ -15,8 +16,13 @@ def test_serialize_rmm_device_buffer(size):
     x = rmm.DeviceBuffer(size=size)
     cuda.to_device(x_np, to=cuda.as_cuda_array(x))
 
-    header, frames = serialize(x, serializers=("cuda", "dask", "pickle"))
-    y = deserialize(header, frames, deserializers=("cuda", "dask", "pickle", "error"))
+    header, frames = serialize(x, serializers=serializers)
+    y = deserialize(header, frames, deserializers=serializers)
     y_np = y.copy_to_host()
 
+    if serializers[0] == "cuda":
+        assert all(hasattr(f, "__cuda_array_interface__") for f in frames)
+    elif serializers[0] == "dask":
+        assert all(isinstance(f, memoryview) for f in frames)
+
     assert (x_np == y_np).all()

From 2a05299a934a2557b985dee93da1e0eff8689178 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Wed, 19 Feb 2020 11:45:15 -0600
Subject: [PATCH 0692/1550] bump version to 2.11.0

---
 docs/source/changelog.rst | 58 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 58 insertions(+)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index cd311309a30..498668d3c88 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,55 @@
 Changelog
 =========
 
+2.11.0 - 2020-02-19
+-------------------
+
+- Add dask serialization of CUDA objects (:pr:`3482`) `John Kirkham`_
+- Suppress cuML ``ImportError`` (:pr:`3499`) `John Kirkham`_
+- Msgpack 1.0 compatibility (:pr:`3494`) `James Bourbeau`_
+- Register cuML serializers (:pr:`3485`) `John Kirkham`_
+- Check exact equality for worker state (:pr:`3483`) `Brett Naul`_
+- Serialize 1-D, contiguous, ``uint8`` CUDA frames (:pr:`3475`) `John Kirkham`_
+- Update NumPy array serialization to handle non-contiguous slices (:pr:`3474`) `James Bourbeau`_
+- Propose fix for collection based resources docs (:pr:`3480`) `Chris Roat`_
+- Remove ``--verbose`` flag from CI runs (:pr:`3484`) `Matthew Rocklin`_
+- Do not duplicate messages in scheduler report (:pr:`3477`) `Jakub Beránek`_
+- Register Dask cuDF serializers (:pr:`3478`) `John Kirkham`_
+- Add support for Python 3.8 (:pr:`3249`) `James Bourbeau`_
+- Add last seen column to worker table and highlight errant workers (:pr:`3468`) `kaelgreco`_
+- Change default value of ``local_directory`` from empty string to ``None`` (:pr:`3441`) `condoratberlin`_
+- Clear old docs (:pr:`3458`) `Matthew Rocklin`_
+- Change default multiprocessing behavior to spawn (:pr:`3461`) `Matthew Rocklin`_
+- Split dashboard host on additional slashes to handle inproc (:pr:`3466`) `Jacob Tomlinson`_
+- Update ``locality.rst`` (:pr:`3470`) `Dustin Tindall`_
+- Minor ``gen.Return`` cleanup (:pr:`3469`) `James Bourbeau`_
+- Update comparison logic for worker state (:pr:`3321`) `rockwellw`_
+- Update minimum ``tblib`` version to 1.6.0 (:pr:`3451`) `James Bourbeau`_
+- Add total row to workers plot in dashboard (:pr:`3464`) `Julia Signell`_
+- Workaround ``RecursionError`` on profile data (:pr:`3455`) `Tom Augspurger`_
+- Include code and summary in performance report (:pr:`3462`) `Matthew Rocklin`_
+- Skip ``test_open_close_many_workers`` on Python 3.6 (:pr:`3459`) `Matthew Rocklin`_
+- Support serializing/deserializing ``rmm.DeviceBuffer`` s (:pr:`3442`) `John Kirkham`_
+- Always add new ``TaskGroup`` to ``TaskPrefix`` (:pr:`3322`) `James Bourbeau`_
+- Rerun ``black`` on the code base (:pr:`3444`) `John Kirkham`_
+- Ensure ``__causes__`` s of exceptions raised on workers are serialized (:pr:`3430`) `Alex Adamson`_
+- Adjust ``numba.cuda`` import and add check (:pr:`3446`) `John Kirkham`_
+- Fix name of Numba serialization test (:pr:`3447`) `John Kirkham`_
+- Checks for command parameters in ``ssh2`` (:pr:`3078`) `Peter Andreas Entschev`_
+- Update ``worker_kwargs`` description in ``LocalCluster`` constructor (:pr:`3438`) `James Bourbeau`_
+- Ensure scheduler updates task and worker states after successful worker data deletion (:pr:`3401`) `James Bourbeau`_
+- Avoid ``loop=`` keyword in asyncio coordination primitives (:pr:`3437`) `Matthew Rocklin`_
+- Call pip as a module to avoid warnings (:pr:`3436`) `Cyril Shcherbin`_
+- Add documentation of parameters in coordination primitives (:pr:`3434`) `Søren Fuglede Jørgensen`_
+- Replace ``tornado.locks`` with asyncio for Events/Locks/Conditions/Semaphore (:pr:`3397`) `Matthew Rocklin`_
+- Remove object from class hierarchy (:pr:`3432`) `Anderson Banihirwe`_
+- Add ``dashboard_link`` property to ``Client`` (:pr:`3429`) `Jacob Tomlinson`_
+- Allow memory monitor to evict data more aggressively (:pr:`3424`) `fjetter`_
+- Make ``_get_ip`` return an IP address when defaulting (:pr:`3418`) `Pierre Glaser`_
+- Support version checking with older versions of Dask (:pr:`3390`) `Igor Gotlibovych`_
+- Add Mac OS build to CI (:pr:`3358`) `James Bourbeau`_
+
+
 2.10.0 - 2020-01-28
 -------------------
 
@@ -1512,3 +1561,12 @@ significantly without many new features.
 .. _`Markus Mohrhard`: https://github.com/mmohrhard
 .. _`Mana Borwornpadungkitti`: https://github.com/potpath
 .. _`Chrysostomos Nanakos`: https://github.com/cnanakos
+.. _`Chris Roat`: https://github.com/chrisroat
+.. _`Jakub Beránek`: https://github.com/Kobzol
+.. _`kaelgreco`: https://github.com/kaelgreco
+.. _`Dustin Tindall`: https://github.com/dustindall
+.. _`Julia Signell`: https://github.com/jsignell
+.. _`Alex Adamson`: https://github.com/aadamson
+.. _`Cyril Shcherbin`: https://github.com/shcherbin
+.. _`Søren Fuglede Jørgensen`: https://github.com/fuglede
+.. _`Igor Gotlibovych`: https://github.com/ig248

From 83f8febd32d99b58b75ecd3da710dd8a25618867 Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Wed, 19 Feb 2020 20:06:48 +0000
Subject: [PATCH 0693/1550] Stop keep alives when worker reconnecting to the
 scheduler (#3493)

---
 distributed/worker.py | 25 +++++++++++++++----------
 1 file changed, 15 insertions(+), 10 deletions(-)

diff --git a/distributed/worker.py b/distributed/worker.py
index e1ae8317148..cb32be8111e 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -382,7 +382,6 @@ def __init__(
         self.executed_count = 0
         self.long_running = set()
 
-        self.batched_stream = None
         self.recent_messages_log = deque(
             maxlen=dask.config.get("distributed.comm.recent-messages-log-length")
         )
@@ -573,6 +572,7 @@ def __init__(
         self.actor_executor = ThreadPoolExecutor(
             1, thread_name_prefix="Dask-Actor-Threads"
         )
+        self.batched_stream = BatchedSend(interval="2ms", loop=self.loop)
         self.name = name
         self.scheduler_delay = 0
         self.stream_comms = dict()
@@ -650,6 +650,13 @@ def __init__(
 
         pc = PeriodicCallback(self.heartbeat, 1000, io_loop=self.io_loop)
         self.periodic_callbacks["heartbeat"] = pc
+        pc = PeriodicCallback(
+            lambda: self.batched_stream.send({"op": "keep-alive"}),
+            60000,
+            io_loop=self.io_loop,
+        )
+        self.periodic_callbacks["keep-alive"] = pc
+
         self._address = contact_address
 
         if self.memory_limit:
@@ -797,6 +804,7 @@ def identity(self, comm=None):
     #####################
 
     async def _register_with_scheduler(self):
+        self.periodic_callbacks["keep-alive"].stop()
         self.periodic_callbacks["heartbeat"].stop()
         start = time()
         if self.contact_address is None:
@@ -863,15 +871,8 @@ async def _register_with_scheduler(self):
             logger.info("        Registered to: %26s", self.scheduler.address)
             logger.info("-" * 49)
 
-        self.batched_stream = BatchedSend(interval="2ms", loop=self.loop)
         self.batched_stream.start(comm)
-        pc = PeriodicCallback(
-            lambda: self.batched_stream.send({"op": "keep-alive"}),
-            60000,
-            io_loop=self.io_loop,
-        )
-        self.periodic_callbacks["keep-alive"] = pc
-        pc.start()
+        self.periodic_callbacks["keep-alive"].start()
         self.periodic_callbacks["heartbeat"].start()
         self.loop.add_callback(self.handle_scheduler, comm)
 
@@ -1112,7 +1113,11 @@ async def close(
             for k, v in self.services.items():
                 v.stop()
 
-            if self.batched_stream and not self.batched_stream.comm.closed():
+            if (
+                self.batched_stream
+                and self.batched_stream.comm
+                and not self.batched_stream.comm.closed()
+            ):
                 self.batched_stream.send({"op": "close-stream"})
 
             if self.batched_stream:

From 6ea63bdf4bb04ef11fdb06019d84a161681f761b Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Thu, 20 Feb 2020 15:36:12 +0000
Subject: [PATCH 0694/1550] Rename logs to get_logs (#3473)

* Rename get_logs to logs

* Update distributed/node.py

Co-Authored-By: James Bourbeau <jrbourbeau@users.noreply.github.com>

* Revert changes

* Rename logs to get_logs

Co-authored-by: James Bourbeau <jrbourbeau@users.noreply.github.com>
---
 distributed/deploy/cluster.py | 13 +++++++++----
 distributed/scheduler.py      |  1 +
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
index c616f13c826..81f3d578fb2 100644
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -1,6 +1,7 @@
 import asyncio
 import logging
 import threading
+import warnings
 
 from dask.utils import format_bytes
 
@@ -159,11 +160,11 @@ def sync(self, func, *args, asynchronous=None, callback_timeout=None, **kwargs):
         else:
             return sync(self.loop, func, *args, **kwargs)
 
-    async def _logs(self, scheduler=True, workers=True):
+    async def _get_logs(self, scheduler=True, workers=True):
         logs = Logs()
 
         if scheduler:
-            L = await self.scheduler_comm.logs()
+            L = await self.scheduler_comm.get_logs()
             logs["Scheduler"] = Log("\n".join(line for level, line in L))
 
         if workers:
@@ -173,7 +174,7 @@ async def _logs(self, scheduler=True, workers=True):
 
         return logs
 
-    def logs(self, scheduler=True, workers=True):
+    def get_logs(self, scheduler=True, workers=True):
         """ Return logs for the scheduler and workers
 
         Parameters
@@ -190,7 +191,11 @@ def logs(self, scheduler=True, workers=True):
             A dictionary of logs, with one item for the scheduler and one for
             each worker
         """
-        return self.sync(self._logs, scheduler=scheduler, workers=workers)
+        return self.sync(self._get_logs, scheduler=scheduler, workers=workers)
+
+    def logs(self, *args, **kwargs):
+        warnings.warn("logs is deprecated, use get_logs instead", DeprecationWarning)
+        return self.get_logs(*args, **kwargs)
 
     @property
     def dashboard_link(self):
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index cba399318cc..c78c4b1b218 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1267,6 +1267,7 @@ def __init__(
             "call_stack": self.get_call_stack,
             "profile": self.get_profile,
             "performance_report": self.performance_report,
+            "get_logs": self.get_logs,
             "logs": self.get_logs,
             "worker_logs": self.get_worker_logs,
             "nbytes": self.get_nbytes,

From 0bed9fe57fa6c0f9416b337aa816024a6bb31acf Mon Sep 17 00:00:00 2001
From: Peter Andreas Entschev <peter@entschev.com>
Date: Sat, 22 Feb 2020 13:27:45 +0100
Subject: [PATCH 0695/1550] Remove `import ucp` from the top of `ucx.py`
 (#3510)

This is needed to ensure Dask configurations will be propagated
to UCX upon importing. Since `ucx.py` is imported upon
`import distributed`, Dask configurations passed to
`Nanny(..., config=ucx_config)` won't be read by UCX since it has
already been loaded.
---
 distributed/comm/ucx.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/distributed/comm/ucx.py b/distributed/comm/ucx.py
index 629a179e43e..330dde8a2d3 100644
--- a/distributed/comm/ucx.py
+++ b/distributed/comm/ucx.py
@@ -5,8 +5,6 @@
 
 .. _UCX: https://github.com/openucx/ucx
 """
-import ucp
-
 import logging
 
 import dask
@@ -26,7 +24,9 @@
 
 
 # In order to avoid double init when forking/spawning new processes (multiprocess),
-# we make sure only to import and initialize UCX once at first use.
+# we make sure only to import and initialize UCX once at first use. This is also
+# required to ensure Dask configuration gets propagated to UCX, which needs
+# variables to be set before being imported.
 ucp = None
 cuda_array = None
 

From 1868dfe18e50aea20d3242c6955523b57c4f4e50 Mon Sep 17 00:00:00 2001
From: Darren Weber <darren.weber@jupiterintel.com>
Date: Mon, 24 Feb 2020 08:53:40 -0800
Subject: [PATCH 0696/1550] Revise develop-docs: conda env example (#3406)

---
 docs/source/develop.rst | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/docs/source/develop.rst b/docs/source/develop.rst
index f7a10c64471..68e8385ca5e 100644
--- a/docs/source/develop.rst
+++ b/docs/source/develop.rst
@@ -22,6 +22,26 @@ and install it from source::
    cd distributed
    python setup.py install
 
+Using conda, for example::
+
+   git clone git@github.com:{your-fork}/distributed.git
+   cd distributed
+   conda create -y -n distributed python=3.6
+   conda activate distributed
+   python -m pip install -U -r requirements.txt
+   python -m pip install -U -r dev-requirements.txt
+   python -m pip install -e .
+
+To keep a fork in sync with the upstream source::
+
+   cd distributed
+   git remote add upstream git@github.com:dask/distributed.git
+   git remote -v
+   git fetch -a upstream
+   git checkout master
+   git pull upstream master
+   git push origin master
+
 Test
 ----
 

From 56fd9b8eaef22896acca01c97bc7061ef2f114fb Mon Sep 17 00:00:00 2001
From: Benjamin Zaitlen <quasiben@users.noreply.github.com>
Date: Tue, 25 Feb 2020 10:25:56 -0500
Subject: [PATCH 0697/1550] RMM/UCX Config Flags (#3515)

* initial setup and test of ucx config flags in dask

* add rmm, rework ucx config a bit, add keys in distributed.yaml

* move ucx conf to solitary file

* rework ucx config ingestion

* lint

* dgx check

* simplify dask.ucx flag scrubbing

* move scrub function back to ucx

* remove tcp-over-ucx flag and do not assume tls when only using net-devices
---
 distributed/comm/tests/test_ucx.py        |  2 +-
 distributed/comm/tests/test_ucx_config.py | 84 +++++++++++++++++++++++
 distributed/comm/ucx.py                   | 69 +++++++++++++++++--
 distributed/distributed.yaml              |  7 +-
 4 files changed, 154 insertions(+), 8 deletions(-)
 create mode 100644 distributed/comm/tests/test_ucx_config.py

diff --git a/distributed/comm/tests/test_ucx.py b/distributed/comm/tests/test_ucx.py
index 7725bfa2432..ead799f8158 100644
--- a/distributed/comm/tests/test_ucx.py
+++ b/distributed/comm/tests/test_ucx.py
@@ -10,7 +10,7 @@
 from distributed.protocol import to_serialize
 from distributed.deploy.local import LocalCluster
 from dask.dataframe.utils import assert_eq
-from distributed.utils_test import gen_test, loop, inc, cleanup  # noqa: 401
+from distributed.utils_test import gen_test, loop, inc, cleanup, popen  # noqa: 401
 
 from .test_comms import check_deserialize
 
diff --git a/distributed/comm/tests/test_ucx_config.py b/distributed/comm/tests/test_ucx_config.py
new file mode 100644
index 00000000000..5b45a844093
--- /dev/null
+++ b/distributed/comm/tests/test_ucx_config.py
@@ -0,0 +1,84 @@
+import pytest
+from time import sleep
+
+import dask
+from dask.utils import format_bytes
+from distributed import Client
+from distributed.utils_test import gen_test, loop, inc, cleanup, popen  # noqa: 401
+from distributed.utils import get_ip
+from distributed.comm.ucx import _scrub_ucx_config
+
+try:
+    HOST = get_ip()
+except Exception:
+    HOST = "127.0.0.1"
+
+ucp = pytest.importorskip("ucp")
+rmm = pytest.importorskip("rmm")
+
+
+@pytest.mark.asyncio
+async def test_ucx_config(cleanup):
+
+    ucx = {"nvlink": True, "infiniband": True, "net-devices": ""}
+
+    with dask.config.set(ucx=ucx):
+        ucx_config = _scrub_ucx_config()
+        assert ucx_config.get("TLS") == "rc,tcp,sockcm,cuda_copy,cuda_ipc"
+        assert ucx_config.get("NET_DEVICES") is None
+
+    ucx = {"nvlink": False, "infiniband": True, "net-devices": "mlx5_0:1"}
+
+    with dask.config.set(ucx=ucx):
+        ucx_config = _scrub_ucx_config()
+        assert ucx_config.get("TLS") == "rc,tcp,sockcm,cuda_copy"
+        assert ucx_config.get("NET_DEVICES") == "mlx5_0:1"
+
+    ucx = {
+        "nvlink": False,
+        "infiniband": True,
+        "net-devices": "all",
+        "MEMTYPE_CACHE": "y",
+    }
+
+    with dask.config.set(ucx=ucx):
+        ucx_config = _scrub_ucx_config()
+        assert ucx_config.get("TLS") == "rc,tcp,sockcm,cuda_copy"
+        assert ucx_config.get("MEMTYPE_CACHE") == "y"
+
+
+def test_ucx_config_w_env_var(cleanup, loop, monkeypatch):
+    size = "1000.00 MB"
+    monkeypatch.setenv("DASK_RMM__POOL_SIZE", size)
+
+    dask.config.refresh()
+
+    port = "13339"
+    sched_addr = "ucx://%s:%s" % (HOST, port)
+
+    with popen(
+        ["dask-scheduler", "--no-dashboard", "--protocol", "ucx", "--port", port]
+    ) as sched:
+        with popen(
+            [
+                "dask-worker",
+                sched_addr,
+                "--no-dashboard",
+                "--protocol",
+                "ucx",
+                "--no-nanny",
+            ]
+        ) as w:
+            with Client(sched_addr, loop=loop, timeout=10) as c:
+                while not c.scheduler_info()["workers"]:
+                    sleep(0.1)
+
+                # configured with 1G pool
+                rmm_usage = c.run_on_scheduler(rmm.get_info)
+                assert size == format_bytes(rmm_usage.free)
+
+                # configured with 1G pool
+                worker_addr = list(c.scheduler_info()["workers"])[0]
+                worker_rmm_usage = c.run(rmm.get_info)
+                rmm_usage = worker_rmm_usage[worker_addr]
+                assert size == format_bytes(rmm_usage.free)
diff --git a/distributed/comm/ucx.py b/distributed/comm/ucx.py
index 330dde8a2d3..5ea2d16ec45 100644
--- a/distributed/comm/ucx.py
+++ b/distributed/comm/ucx.py
@@ -14,11 +14,15 @@
 from .core import Comm, Connector, Listener, CommClosedError
 from .registry import Backend, backends
 from .utils import ensure_concrete_host, to_frames, from_frames
-from ..utils import ensure_ip, get_ip, get_ipv6, nbytes, log_errors, CancelledError
-
-import dask
-import numpy as np
-
+from ..utils import (
+    ensure_ip,
+    get_ip,
+    get_ipv6,
+    nbytes,
+    log_errors,
+    CancelledError,
+    parse_bytes,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -39,7 +43,11 @@ def init_once():
     import ucp as _ucp
 
     ucp = _ucp
-    ucp.init(options=dask.config.get("ucx"), env_takes_precedence=True)
+
+    # remove/process dask.ucx flags for valid ucx options
+    ucx_config = _scrub_ucx_config()
+
+    ucp.init(options=ucx_config, env_takes_precedence=True)
 
     # Find the function, `cuda_array()`, to use when allocating new CUDA arrays
     try:
@@ -61,6 +69,13 @@ def cuda_array(n):
                     "In order to send/recv CUDA arrays, Numba or RMM is required"
                 )
 
+    pool_size_str = dask.config.get("rmm.pool-size")
+    if pool_size_str is not None:
+        pool_size = parse_bytes(pool_size_str)
+        rmm.reinitialize(
+            pool_allocator=True, managed_memory=False, initial_pool_size=pool_size
+        )
+
 
 class UCX(Comm):
     """Comm object using UCP.
@@ -328,3 +343,45 @@ def get_local_address_for(self, loc):
 
 
 backends["ucx"] = UCXBackend()
+
+
+def _scrub_ucx_config():
+    """Function to scrub dask config options for valid UCX config options"""
+
+    # configuration of UCX can happen in two ways:
+    # 1) high level on/off flags which correspond to UCX configuration
+    # 2) explicity defined UCX configuration flags
+
+    # import does not initialize ucp -- this will occur outside this function
+    from ucp import get_config
+
+    options = {}
+
+    # if any of the high level flags are set, as long as they are not Null/None,
+    # we assume we should configure basic TLS settings for UCX
+    if any([dask.config.get("ucx.nvlink"), dask.config.get("ucx.infiniband")]):
+        tls = "tcp,sockcm,cuda_copy"
+        tls_priority = "sockcm"
+
+        if dask.config.get("ucx.infiniband"):
+            tls = "rc," + tls
+        if dask.config.get("ucx.nvlink"):
+            tls = tls + ",cuda_ipc"
+
+        options = {"TLS": tls, "SOCKADDR_TLS_PRIORITY": tls_priority}
+
+        net_devices = dask.config.get("ucx.net-devices")
+        if net_devices is not None and net_devices != "":
+            options["NET_DEVICES"] = net_devices
+
+    # ANY UCX options defined in config will overwrite high level dask.ucx flags
+    valid_ucx_keys = list(get_config().keys())
+    for k, v in dask.config.get("ucx").items():
+        if k in valid_ucx_keys:
+            options[k] = v
+        else:
+            logger.debug(
+                "Key: %s with value: %s not a valid UCX configuration option" % (k, v)
+            )
+
+    return options
diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index 487e72e215e..417b7bd5be8 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -132,5 +132,10 @@ distributed:
     log-length: 10000  # default length of logs to keep in memory
     log-format: '%(name)s - %(levelname)s - %(message)s'
     pdb-on-err: False       # enter debug mode on scheduling error
+rmm:
+  pool-size: null
+ucx:
+  nvlink: null
+  infiniband: null
+  net-devices: null
 
-ucx: {}

From fc3e8d68e9e867e4cdd0d90cef86ca46f758f48e Mon Sep 17 00:00:00 2001
From: Lucas Rademaker <44430780+lr4d@users.noreply.github.com>
Date: Thu, 27 Feb 2020 22:59:08 +0100
Subject: [PATCH 0698/1550] make work stealing callback time configurable
 (#3523)

---
 distributed/distributed.yaml    |  1 +
 distributed/stealing.py         | 11 +++++++++--
 distributed/tests/test_steal.py | 18 ++++++++++++++++++
 3 files changed, 28 insertions(+), 2 deletions(-)

diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index 417b7bd5be8..ed21507e041 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -19,6 +19,7 @@ distributed:
     idle-timeout: null      # Shut down after this duration, like "1h" or "30 minutes"
     transition-log-length: 100000
     work-stealing: True     # workers should steal tasks from each other
+    work-stealing-interval: 100ms  # Callback time for work stealing
     worker-ttl: null        # like '60s'. Time to live for workers.  They must heartbeat faster than this
     pickle: True            # Is the scheduler allowed to deserialize arbitrary bytestrings
     preload: []
diff --git a/distributed/stealing.py b/distributed/stealing.py
index e3537f647bf..b14a2a8de6d 100644
--- a/distributed/stealing.py
+++ b/distributed/stealing.py
@@ -6,7 +6,7 @@
 import dask
 from .core import CommClosedError
 from .diagnostics.plugin import SchedulerPlugin
-from .utils import log_errors, PeriodicCallback
+from .utils import log_errors, parse_timedelta, PeriodicCallback
 
 try:
     from cytoolz import topk
@@ -40,8 +40,15 @@ def __init__(self, scheduler):
         for worker in scheduler.workers:
             self.add_worker(worker=worker)
 
+        # `callback_time` is in milliseconds
+        callback_time = 1000 * parse_timedelta(
+            dask.config.get("distributed.scheduler.work-stealing-interval"),
+            default="ms",
+        )
         pc = PeriodicCallback(
-            callback=self.balance, callback_time=100, io_loop=self.scheduler.loop
+            callback=self.balance,
+            callback_time=callback_time,
+            io_loop=self.scheduler.loop,
         )
         self._pc = pc
         self.scheduler.periodic_callbacks["stealing"] = pc
diff --git a/distributed/tests/test_steal.py b/distributed/tests/test_steal.py
index a6a19332f5f..b017bff4371 100644
--- a/distributed/tests/test_steal.py
+++ b/distributed/tests/test_steal.py
@@ -9,6 +9,7 @@
 from toolz import sliding_window, concat
 from tornado import gen
 
+import dask
 from distributed import Nanny, Worker, wait, worker_client
 from distributed.config import config
 from distributed.metrics import time
@@ -676,3 +677,20 @@ def test_lose_task(c, s, a, b):
 
     out = log.getvalue()
     assert "Error" not in out
+
+
+@gen_cluster(client=True)
+def test_worker_stealing_interval(c, s, a, b):
+    from distributed.scheduler import WorkStealing
+
+    ws = WorkStealing(s)
+    assert ws._pc.callback_time == 100
+
+    with dask.config.set({"distributed.scheduler.work-stealing-interval": "500ms"}):
+        ws = WorkStealing(s)
+    assert ws._pc.callback_time == 500
+
+    # Default unit is `ms`
+    with dask.config.set({"distributed.scheduler.work-stealing-interval": 2}):
+        ws = WorkStealing(s)
+    assert ws._pc.callback_time == 2

From 0d7a31adaabd801a189fa529c6b7670fe98395b1 Mon Sep 17 00:00:00 2001
From: Davis Bennett <davis.v.bennett@gmail.com>
Date: Fri, 28 Feb 2020 11:15:44 -0500
Subject: [PATCH 0699/1550] fix typo in docstring (#3528)

---
 distributed/deploy/adaptive_core.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/deploy/adaptive_core.py b/distributed/deploy/adaptive_core.py
index 44a708aca38..dfd82ea33ba 100644
--- a/distributed/deploy/adaptive_core.py
+++ b/distributed/deploy/adaptive_core.py
@@ -13,7 +13,7 @@ class AdaptiveCore:
     The core logic for adaptive deployments, with none of the cluster details
 
     This class controls our adaptive scaling behavior.  It is intended to be
-    sued as a super-class or mixin.  It expects the following state and methods:
+    used as a super-class or mixin.  It expects the following state and methods:
 
     **State**
 

From 3b915a2adeddaf991590fd0192178836a7594fcf Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sat, 29 Feb 2020 18:09:47 -0500
Subject: [PATCH 0700/1550] Add try-except around getting source code in
 performance report (#3505)

See https://github.com/dask/distributed/issues/1674#issuecomment-589028369

Co-authored-by: James Bourbeau <jrbourbeau@users.noreply.github.com>
---
 distributed/client.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index ce820d2c6e6..4e84ea278d2 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -4597,8 +4597,11 @@ async def __aenter__(self):
 
     async def __aexit__(self, typ, value, traceback, code=None):
         if not code:
-            frame = inspect.currentframe().f_back
-            code = inspect.getsource(frame)
+            try:
+                frame = inspect.currentframe().f_back
+                code = inspect.getsource(frame)
+            except Exception:
+                code = ""
         data = await get_client().scheduler.performance_report(
             start=self.start, code=code
         )
@@ -4609,8 +4612,11 @@ def __enter__(self):
         get_client().sync(self.__aenter__)
 
     def __exit__(self, typ, value, traceback):
-        frame = inspect.currentframe().f_back
-        code = inspect.getsource(frame)
+        try:
+            frame = inspect.currentframe().f_back
+            code = inspect.getsource(frame)
+        except Exception:
+            code = ""
         get_client().sync(self.__aexit__, type, value, traceback, code=code)
 
 

From f49b4b0166699755bcca47fd4c86573a7d4ce72f Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Sun, 1 Mar 2020 07:29:01 -0800
Subject: [PATCH 0701/1550] Use 'temporary-directory' from dask.config for
 Nanny's directory (#3531)

Make sure to respect the `temporary-directory` config value in `Nanny`
when determining an appropriate temporary directory for things like
spilling.
---
 distributed/nanny.py            |  8 +++++++-
 distributed/tests/test_nanny.py | 10 ++++++++++
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/distributed/nanny.py b/distributed/nanny.py
index ff653ba096c..676291da3f8 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -66,7 +66,7 @@ def __init__(
         ncores=None,
         loop=None,
         local_dir=None,
-        local_directory="dask-worker-space",
+        local_directory=None,
         services=None,
         name=None,
         memory_limit="auto",
@@ -150,6 +150,12 @@ def __init__(
             warnings.warn("The local_dir keyword has moved to local_directory")
             local_directory = local_dir
 
+        if local_directory is None:
+            local_directory = dask.config.get("temporary-directory") or os.getcwd()
+            if not os.path.exists(local_directory):
+                os.mkdir(local_directory)
+            local_directory = os.path.join(local_directory, "dask-worker-space")
+
         self.local_directory = local_directory
 
         self.services = services
diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
index 0091a6126f1..c80974d9970 100644
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -402,6 +402,16 @@ def test_data_types(c, s):
     yield w.close()
 
 
+@gen_cluster(nthreads=[])
+def test_local_directory(s):
+    with tmpfile() as fn:
+        with dask.config.set(temporary_directory=fn):
+            w = yield Nanny(s.address)
+            assert w.local_directory.startswith(fn)
+            assert "dask-worker-space" in w.local_directory
+            yield w.close()
+
+
 def _noop(x):
     """Define here because closures aren't pickleable."""
     pass

From 953314f64d780f68848b42f6478e343129adef11 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Sun, 1 Mar 2020 13:21:12 -0800
Subject: [PATCH 0702/1550] Mark `bool` as MessagePack serializable (#3535)

As `bool` values can be serialized by MessagePack (see code below), mark
them as such in `_is_msgpack_serializable`.

```python
In [1]: import msgpack

In [2]: (msgpack.dumps(False), msgpack.dumps(True))
Out[2]: (b'\xc2', b'\xc3')

In [3]: (msgpack.loads(b'\xc2'), msgpack.loads(b'\xc3'))
Out[3]: (False, True)
```
---
 distributed/protocol/serialize.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index ddab6130765..b9c6c33d318 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -548,6 +548,7 @@ def _is_msgpack_serializable(v):
     typ = type(v)
     return (
         typ is str
+        or typ is bool
         or typ is int
         or typ is float
         or isinstance(v, dict)

From 5fd58327b0cb23c401178f8388548b4ebd74c93f Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Sun, 1 Mar 2020 18:19:21 -0800
Subject: [PATCH 0703/1550] Mark `None` as MessagePack serializable (#3537)

---
 distributed/protocol/serialize.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index b9c6c33d318..3f3207ab58f 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -547,7 +547,8 @@ def _deserialize_bytes(header, frames):
 def _is_msgpack_serializable(v):
     typ = type(v)
     return (
-        typ is str
+        v is None
+        or typ is str
         or typ is bool
         or typ is int
         or typ is float

From b9936bfe01d87f89cd60d2ea7abf21749f031781 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Mon, 2 Mar 2020 13:41:39 -0800
Subject: [PATCH 0704/1550] Use `makedirs` when constructing `local_directory`
 (#3538)

---
 distributed/nanny.py  | 2 +-
 distributed/worker.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/distributed/nanny.py b/distributed/nanny.py
index 676291da3f8..ec5397efb93 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -153,7 +153,7 @@ def __init__(
         if local_directory is None:
             local_directory = dask.config.get("temporary-directory") or os.getcwd()
             if not os.path.exists(local_directory):
-                os.mkdir(local_directory)
+                os.makedirs(local_directory)
             local_directory = os.path.join(local_directory, "dask-worker-space")
 
         self.local_directory = local_directory
diff --git a/distributed/worker.py b/distributed/worker.py
index cb32be8111e..185db2e193b 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -492,7 +492,7 @@ def __init__(
         if local_directory is None:
             local_directory = dask.config.get("temporary-directory") or os.getcwd()
             if not os.path.exists(local_directory):
-                os.mkdir(local_directory)
+                os.makedirs(local_directory)
             local_directory = os.path.join(local_directory, "dask-worker-space")
 
         with warn_on_duration(

From 72213c9b14fe7f828371f187a01b1d8c9b773ae1 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Mon, 2 Mar 2020 15:47:43 -0600
Subject: [PATCH 0705/1550] Update heartbeat CommClosedError error handling
 (#3529)

* Update worker heartbeat error catching logic

* Run black
---
 distributed/tests/test_worker.py | 25 ++++++++++++++++++++++++-
 distributed/worker.py            |  6 ++++--
 2 files changed, 28 insertions(+), 3 deletions(-)

diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 0bc2cf10988..b6da294c749 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -31,7 +31,7 @@
     wait,
 )
 from distributed.compatibility import WINDOWS
-from distributed.core import rpc
+from distributed.core import rpc, CommClosedError
 from distributed.scheduler import Scheduler
 from distributed.metrics import time
 from distributed.worker import Worker, error_message, logger, parse_memory_limit
@@ -1629,3 +1629,26 @@ async def test_update_latency(cleanup):
 
             if w.digests is not None:
                 assert w.digests["latency"].size() > 0
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("reconnect", [True, False])
+async def test_heartbeat_comm_closed(cleanup, monkeypatch, reconnect):
+    with captured_logger("distributed.worker", level=logging.WARNING) as logger:
+        async with await Scheduler() as s:
+
+            def bad_heartbeat_worker(*args, **kwargs):
+                raise CommClosedError()
+
+            async with await Worker(s.address, reconnect=reconnect) as w:
+                # Trigger CommClosedError during worker heartbeat
+                monkeypatch.setattr(
+                    w.scheduler, "heartbeat_worker", bad_heartbeat_worker
+                )
+
+                await w.heartbeat()
+                if reconnect:
+                    assert w.status == "running"
+                else:
+                    assert w.status == "closed"
+    assert "Heartbeat to scheduler failed" in logger.getvalue()
diff --git a/distributed/worker.py b/distributed/worker.py
index 185db2e193b..a5a39fe22b3 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -913,14 +913,16 @@ async def heartbeat(self):
                 )
                 self.bandwidth_workers.clear()
                 self.bandwidth_types.clear()
+            except CommClosedError:
+                logger.warning("Heartbeat to scheduler failed")
+                if not self.reconnect:
+                    await self.close(report=False)
             except IOError as e:
                 # Scheduler is gone. Respect distributed.comm.timeouts.connect
                 if "Timed out trying to connect" in str(e):
                     await self.close(report=False)
                 else:
                     raise e
-            except CommClosedError:
-                logger.warning("Heartbeat to scheduler failed")
             finally:
                 self.heartbeat_active = False
         else:

From 0140fc61835745ee41305d944ac037649e096059 Mon Sep 17 00:00:00 2001
From: Benjamin Zaitlen <quasiben@users.noreply.github.com>
Date: Mon, 2 Mar 2020 19:56:41 -0500
Subject: [PATCH 0706/1550] Fix/more ucx config options (#3539)

* add tcp and cuda_copy config flags

* update tests

* raise error if no transport methods are set
---
 distributed/comm/tests/test_ucx_config.py | 32 ++++++++++++++++++++---
 distributed/comm/ucx.py                   | 20 ++++++++++++--
 distributed/distributed.yaml              |  8 +++---
 3 files changed, 52 insertions(+), 8 deletions(-)

diff --git a/distributed/comm/tests/test_ucx_config.py b/distributed/comm/tests/test_ucx_config.py
index 5b45a844093..5746cc80454 100644
--- a/distributed/comm/tests/test_ucx_config.py
+++ b/distributed/comm/tests/test_ucx_config.py
@@ -20,18 +20,30 @@
 @pytest.mark.asyncio
 async def test_ucx_config(cleanup):
 
-    ucx = {"nvlink": True, "infiniband": True, "net-devices": ""}
+    ucx = {
+        "nvlink": True,
+        "infiniband": True,
+        "net-devices": "",
+        "tcp": True,
+        "cuda_copy": True,
+    }
 
     with dask.config.set(ucx=ucx):
         ucx_config = _scrub_ucx_config()
         assert ucx_config.get("TLS") == "rc,tcp,sockcm,cuda_copy,cuda_ipc"
         assert ucx_config.get("NET_DEVICES") is None
 
-    ucx = {"nvlink": False, "infiniband": True, "net-devices": "mlx5_0:1"}
+    ucx = {
+        "nvlink": False,
+        "infiniband": True,
+        "net-devices": "mlx5_0:1",
+        "tcp": True,
+        "cuda_copy": False,
+    }
 
     with dask.config.set(ucx=ucx):
         ucx_config = _scrub_ucx_config()
-        assert ucx_config.get("TLS") == "rc,tcp,sockcm,cuda_copy"
+        assert ucx_config.get("TLS") == "rc,tcp,sockcm"
         assert ucx_config.get("NET_DEVICES") == "mlx5_0:1"
 
     ucx = {
@@ -39,6 +51,8 @@ async def test_ucx_config(cleanup):
         "infiniband": True,
         "net-devices": "all",
         "MEMTYPE_CACHE": "y",
+        "tcp": True,
+        "cuda_copy": True,
     }
 
     with dask.config.set(ucx=ucx):
@@ -46,6 +60,18 @@ async def test_ucx_config(cleanup):
         assert ucx_config.get("TLS") == "rc,tcp,sockcm,cuda_copy"
         assert ucx_config.get("MEMTYPE_CACHE") == "y"
 
+    ucx = {
+        "nvlink": False,
+        "infiniband": False,
+        "net-devices": "all",
+        "MEMTYPE_CACHE": "y",
+        "tcp": False,
+        "cuda_copy": True,
+    }
+    with dask.config.set(ucx=ucx):
+        with raises(ValueError):
+            ucx_config = _scrub_ucx_config()
+
 
 def test_ucx_config_w_env_var(cleanup, loop, monkeypatch):
     size = "1000.00 MB"
diff --git a/distributed/comm/ucx.py b/distributed/comm/ucx.py
index 5ea2d16ec45..9484c9c08b9 100644
--- a/distributed/comm/ucx.py
+++ b/distributed/comm/ucx.py
@@ -359,10 +359,22 @@ def _scrub_ucx_config():
 
     # if any of the high level flags are set, as long as they are not Null/None,
     # we assume we should configure basic TLS settings for UCX
-    if any([dask.config.get("ucx.nvlink"), dask.config.get("ucx.infiniband")]):
-        tls = "tcp,sockcm,cuda_copy"
+    if any(
+        [
+            dask.config.get("ucx.tcp"),
+            dask.config.get("ucx.nvlink"),
+            dask.config.get("ucx.infiniband"),
+        ]
+    ):
+        tls = "tcp,sockcm"
         tls_priority = "sockcm"
 
+        # CUDA COPY can optionally be used with ucx -- we rely on the user
+        # to define when messages will include CUDA objects.  Note:
+        # defining only the Infiniband flag will not enable cuda_copy
+        if any([dask.config.get("ucx.nvlink"), dask.config.get("ucx.cuda_copy")]):
+            tls = tls + ",cuda_copy"
+
         if dask.config.get("ucx.infiniband"):
             tls = "rc," + tls
         if dask.config.get("ucx.nvlink"):
@@ -373,6 +385,10 @@ def _scrub_ucx_config():
         net_devices = dask.config.get("ucx.net-devices")
         if net_devices is not None and net_devices != "":
             options["NET_DEVICES"] = net_devices
+    else:
+        raise ValueError(
+            "UCX Dask config not set.  Please define at least one: ucx.tcp, ucx.nvlink, ucx.infiniband"
+        )
 
     # ANY UCX options defined in config will overwrite high level dask.ucx flags
     valid_ucx_keys = list(get_config().keys())
diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index ed21507e041..05f27604328 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -136,7 +136,9 @@ distributed:
 rmm:
   pool-size: null
 ucx:
-  nvlink: null
-  infiniband: null
-  net-devices: null
+  tcp: null  # enable tcp
+  nvlink: null  # enable cuda_ipc
+  infiniband: null # enable Infiniband
+  cuda_copy: null  # enable cuda-copy
+  net-devices: null  # define which Infiniband device to use
 

From 6a66df019cceec14cbf7397c24cb2766967cd704 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Tue, 3 Mar 2020 03:36:50 -0800
Subject: [PATCH 0707/1550] Use `pytest.raises` in `test_ucx_config.py` (#3541)

Should fix a linting error we are seeing in `master`.

ref: https://travis-ci.org/dask/distributed/jobs/657551806#L460
---
 distributed/comm/tests/test_ucx_config.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/comm/tests/test_ucx_config.py b/distributed/comm/tests/test_ucx_config.py
index 5746cc80454..695f2e7575d 100644
--- a/distributed/comm/tests/test_ucx_config.py
+++ b/distributed/comm/tests/test_ucx_config.py
@@ -69,7 +69,7 @@ async def test_ucx_config(cleanup):
         "cuda_copy": True,
     }
     with dask.config.set(ucx=ucx):
-        with raises(ValueError):
+        with pytest.raises(ValueError):
             ucx_config = _scrub_ucx_config()
 
 

From b049bd71f8ef28adb96aa0cdd91254242c38ea2c Mon Sep 17 00:00:00 2001
From: Tom Augspurger <TomAugspurger@users.noreply.github.com>
Date: Tue, 3 Mar 2020 08:26:51 -0600
Subject: [PATCH 0708/1550] DOC: update to async await (#3543)

---
 docs/source/develop.rst | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/docs/source/develop.rst b/docs/source/develop.rst
index 68e8385ca5e..8d0a02fd73d 100644
--- a/docs/source/develop.rst
+++ b/docs/source/develop.rst
@@ -108,20 +108,20 @@ using the ``@gen_cluster`` style of test, e.g.
     from distributed import Client, Future, Scheduler, Worker
 
     @gen_cluster(client=True)
-    def test_submit(c, s, a, b):
+    async def test_submit(c, s, a, b):
         assert isinstance(c, Client)
         assert isinstance(s, Scheduler)
         assert isinstance(a, Worker)
         assert isinstance(b, Worker)
-
+    
         future = c.submit(inc, 1)
         assert isinstance(future, Future)
         assert future.key in c.futures
-
+    
         # result = future.result()  # This synchronous API call would block
-        result = yield future
+        result = await future
         assert result == 2
-
+    
         assert future.key in s.tasks
         assert future.key in a.data or future.key in b.data
 
@@ -131,8 +131,8 @@ you and cleans them up after the test.  It also allows you to directly inspect
 the state of every element of the cluster directly.  However, you can not use
 the normal synchronous API (doing so will cause the test to wait forever) and
 instead you need to use the coroutine API, where all blocking functions are
-prepended with an underscore (``_``).  Beware, it is a common mistake to use
-the blocking interface within these tests.
+prepended with an underscore (``_``) and awaited with ``await``.
+Beware, it is a common mistake to use the blocking interface within these tests.
 
 If you want to test the normal synchronous API you can use the ``client``
 pytest fixture style test, which sets up a scheduler and workers for you in
@@ -166,7 +166,7 @@ also add the ``s, a, b`` fixtures as well.
 In this style of test you do not have access to the scheduler or workers.  The
 variables ``s, a, b`` are now dictionaries holding a
 ``multiprocessing.Process`` object and a port integer.  However, you can now
-use the normal synchronous API (never use yield in this style of test) and you
+use the normal synchronous API (never use ``await`` in this style of test) and you
 can close processes easily by terminating them.
 
 Typically for most user-facing functions you will find both kinds of tests.

From 52a56c50623b6e95c3ee84cdf094b185043dbe17 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Tue, 3 Mar 2020 09:58:14 -0800
Subject: [PATCH 0709/1550] Support using other serializers with
 `register_generic` (#3536)

* Build `ObjectDictSerializer`

Makes a class to handle serialization and deserialization of objects
with `__dict__`s. Should make it easier to specify other serializers one
can use with such objects.

Note if we are unable to use the serializer specified, we fallback to
pickling things.

* Use `ObjectDictSerializer` for "dask" serialization

* Make `register_generic` handle other serializers

Allow users to call `register_generic` with other serializers in mind.
By default still use `dask_serialize` and `dask_deserialize`. Though
allow these to be subbed out by other `*_serialize` and `*_deserialize`
functions that will also dispatch based on type to serialize and
deserialize with the expected serialization mode.

* Ensure `dict` can deserialized with "cuda"

As nested serialization can run into `dict` objects in the process, make
sure that they can be deserialized with "cuda" as well to produce the
original object.
---
 distributed/protocol/cuda.py      |   7 +-
 distributed/protocol/serialize.py | 103 +++++++++++++++++-------------
 2 files changed, 63 insertions(+), 47 deletions(-)

diff --git a/distributed/protocol/cuda.py b/distributed/protocol/cuda.py
index 51cb3ea42fa..aa638f70c0d 100644
--- a/distributed/protocol/cuda.py
+++ b/distributed/protocol/cuda.py
@@ -1,7 +1,7 @@
 import dask
 
 from . import pickle
-from .serialize import register_serialization_family
+from .serialize import ObjectDictSerializer, register_serialization_family
 from dask.utils import typename
 
 cuda_serialize = dask.utils.Dispatch("cuda_serialize")
@@ -29,3 +29,8 @@ def cuda_loads(header, frames):
 
 
 register_serialization_family("cuda", cuda_dumps, cuda_loads)
+
+
+cuda_object_with_dict_serializer = ObjectDictSerializer("cuda")
+
+cuda_deserialize.register(dict)(cuda_object_with_dict_serializer.deserialize)
diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index 3f3207ab58f..c462568cc40 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -560,59 +560,69 @@ def _is_msgpack_serializable(v):
     )
 
 
-def serialize_object_with_dict(est):
-    header = {
-        "serializer": "dask",
-        "type-serialized": pickle.dumps(type(est)),
-        "simple": {},
-        "complex": {},
-    }
-    frames = []
-
-    if isinstance(est, dict):
-        d = est
-    else:
-        d = est.__dict__
+class ObjectDictSerializer:
+    def __init__(self, serializer):
+        self.serializer = serializer
+
+    def serialize(self, est):
+        header = {
+            "serializer": self.serializer,
+            "type-serialized": pickle.dumps(type(est)),
+            "simple": {},
+            "complex": {},
+        }
+        frames = []
 
-    for k, v in d.items():
-        if _is_msgpack_serializable(v):
-            header["simple"][k] = v
+        if isinstance(est, dict):
+            d = est
         else:
-            if isinstance(v, dict):
-                h, f = serialize_object_with_dict(v)
-            else:
-                h, f = serialize(v)
-            header["complex"][k] = {
-                "header": h,
-                "start": len(frames),
-                "stop": len(frames) + len(f),
-            }
-            frames += f
-    return header, frames
+            d = est.__dict__
 
+        for k, v in d.items():
+            if _is_msgpack_serializable(v):
+                header["simple"][k] = v
+            else:
+                if isinstance(v, dict):
+                    h, f = self.serialize(v)
+                else:
+                    h, f = serialize(v, serializers=(self.serializer, "pickle"))
+                header["complex"][k] = {
+                    "header": h,
+                    "start": len(frames),
+                    "stop": len(frames) + len(f),
+                }
+                frames += f
+        return header, frames
+
+    def deserialize(self, header, frames):
+        cls = pickle.loads(header["type-serialized"])
+        if issubclass(cls, dict):
+            dd = obj = {}
+        else:
+            obj = object.__new__(cls)
+            dd = obj.__dict__
+        dd.update(header["simple"])
+        for k, d in header["complex"].items():
+            h = d["header"]
+            f = frames[d["start"] : d["stop"]]
+            v = deserialize(h, f)
+            dd[k] = v
 
-def deserialize_object_with_dict(header, frames):
-    cls = pickle.loads(header["type-serialized"])
-    if issubclass(cls, dict):
-        dd = obj = {}
-    else:
-        obj = object.__new__(cls)
-        dd = obj.__dict__
-    dd.update(header["simple"])
-    for k, d in header["complex"].items():
-        h = d["header"]
-        f = frames[d["start"] : d["stop"]]
-        v = deserialize(h, f)
-        dd[k] = v
+        return obj
 
-    return obj
 
+dask_object_with_dict_serializer = ObjectDictSerializer("dask")
 
-dask_deserialize.register(dict)(deserialize_object_with_dict)
+dask_deserialize.register(dict)(dask_object_with_dict_serializer.deserialize)
 
 
-def register_generic(cls):
-    """ Register dask_(de)serialize to traverse through __dict__
+def register_generic(
+    cls,
+    serializer_name="dask",
+    serialize_func=dask_serialize,
+    deserialize_func=dask_deserialize,
+):
+    """ Register (de)serialize to traverse through __dict__
 
     Normally when registering new classes for Dask's custom serialization you
     need to manage headers and frames, which can be tedious.  If all you want
@@ -643,5 +653,6 @@ def register_generic(cls):
     dask_serialize
     dask_deserialize
     """
-    dask_serialize.register(cls)(serialize_object_with_dict)
-    dask_deserialize.register(cls)(deserialize_object_with_dict)
+    object_with_dict_serializer = ObjectDictSerializer(serializer_name)
+    serialize_func.register(cls)(object_with_dict_serializer.serialize)
+    deserialize_func.register(cls)(object_with_dict_serializer.deserialize)

From 384080422f5bd54d16ba23161a1c2d18f74ff299 Mon Sep 17 00:00:00 2001
From: Peter Andreas Entschev <peter@entschev.com>
Date: Tue, 3 Mar 2020 18:58:40 +0100
Subject: [PATCH 0710/1550] Use UCX default configuration instead of raising
 (#3544)

* Use UCX default configuration instead of raising

* Remove UCX test raising ValueError
---
 distributed/comm/tests/test_ucx_config.py | 12 ------------
 distributed/comm/ucx.py                   |  7 ++-----
 2 files changed, 2 insertions(+), 17 deletions(-)

diff --git a/distributed/comm/tests/test_ucx_config.py b/distributed/comm/tests/test_ucx_config.py
index 695f2e7575d..c2e86ed0b49 100644
--- a/distributed/comm/tests/test_ucx_config.py
+++ b/distributed/comm/tests/test_ucx_config.py
@@ -60,18 +60,6 @@ async def test_ucx_config(cleanup):
         assert ucx_config.get("TLS") == "rc,tcp,sockcm,cuda_copy"
         assert ucx_config.get("MEMTYPE_CACHE") == "y"
 
-    ucx = {
-        "nvlink": False,
-        "infiniband": False,
-        "net-devices": "all",
-        "MEMTYPE_CACHE": "y",
-        "tcp": False,
-        "cuda_copy": True,
-    }
-    with dask.config.set(ucx=ucx):
-        with pytest.raises(ValueError):
-            ucx_config = _scrub_ucx_config()
-
 
 def test_ucx_config_w_env_var(cleanup, loop, monkeypatch):
     size = "1000.00 MB"
diff --git a/distributed/comm/ucx.py b/distributed/comm/ucx.py
index 9484c9c08b9..7295b11bb48 100644
--- a/distributed/comm/ucx.py
+++ b/distributed/comm/ucx.py
@@ -358,7 +358,8 @@ def _scrub_ucx_config():
     options = {}
 
     # if any of the high level flags are set, as long as they are not Null/None,
-    # we assume we should configure basic TLS settings for UCX
+    # we assume we should configure basic TLS settings for UCX, otherwise we
+    # leave UCX to its default configuration
     if any(
         [
             dask.config.get("ucx.tcp"),
@@ -385,10 +386,6 @@ def _scrub_ucx_config():
         net_devices = dask.config.get("ucx.net-devices")
         if net_devices is not None and net_devices != "":
             options["NET_DEVICES"] = net_devices
-    else:
-        raise ValueError(
-            "UCX Dask config not set.  Please define at least one: ucx.tcp, ucx.nvlink, ucx.infiniband"
-        )
 
     # ANY UCX options defined in config will overwrite high level dask.ucx flags
     valid_ucx_keys = list(get_config().keys())

From d8d0d4e71023ac6c1507b443b90d7805e2bf7ad2 Mon Sep 17 00:00:00 2001
From: Stan Seibert <stan@mtrr.org>
Date: Tue, 3 Mar 2020 14:56:45 -0600
Subject: [PATCH 0711/1550] Allow tasks with restrictions to be stolen (#3069)

Addresses stealing tasks with resource restrictions, as mentioned in #1851.
If a task has hard restrictions, do not just give up on stealing.
Instead, use the restrictions to determine which workers can steal it
before attempting to execute a steal operation.

A follow up PR will be needed to address the issue of long-running tasks
not being stolen because the scheduler has no information about their
runtime.

Supercedes #2740
---
 distributed/stealing.py         | 70 ++++++++++++++++++++++++++++-----
 distributed/tests/test_steal.py | 55 +++++++++++++++++++++++++-
 distributed/worker.py           |  2 +-
 3 files changed, 115 insertions(+), 12 deletions(-)

diff --git a/distributed/stealing.py b/distributed/stealing.py
index b14a2a8de6d..4fbb753e131 100644
--- a/distributed/stealing.py
+++ b/distributed/stealing.py
@@ -4,6 +4,7 @@
 from time import time
 
 import dask
+from .comm.addressing import get_address_host
 from .core import CommClosedError
 from .diagnostics.plugin import SchedulerPlugin
 from .utils import log_errors, parse_timedelta, PeriodicCallback
@@ -128,11 +129,6 @@ def steal_time_ratio(self, ts):
         For example a result of zero implies a task without dependencies.
         level: The location within a stealable list to place this value
         """
-        if not ts.loose_restrictions and (
-            ts.host_restrictions or ts.worker_restrictions or ts.resource_restrictions
-        ):
-            return None, None  # don't steal
-
         if not ts.dependencies:  # no dependencies fast path
             return 0, 0
 
@@ -258,7 +254,7 @@ def move_task_confirm(self, key=None, worker=None, state=None):
                 self.scheduler.check_idle_saturated(victim)
 
             # Victim was waiting, has given up task, enact steal
-            elif state in ("waiting", "ready"):
+            elif state in ("waiting", "ready", "constrained"):
                 self.remove_key_from_stealable(ts)
                 ts.processing_on = thief
                 duration = victim.processing.pop(ts)
@@ -360,14 +356,23 @@ def maybe_move_task(level, ts, sat, idl, duration, cost_multiplier):
                         i += 1
                         if not idle:
                             break
-                        idl = idle[i % len(idle)]
+
+                        if _has_restrictions(ts):
+                            thieves = [ws for ws in idle if _can_steal(ws, ts, sat)]
+                        else:
+                            thieves = idle
+                        if not thieves:
+                            break
+                        thief = thieves[i % len(thieves)]
 
                         duration = sat.processing.get(ts)
                         if duration is None:
                             stealable.discard(ts)
                             continue
 
-                        maybe_move_task(level, ts, sat, idl, duration, cost_multiplier)
+                        maybe_move_task(
+                            level, ts, sat, thief, duration, cost_multiplier
+                        )
 
                 if self.cost_multipliers[level] < 20:  # don't steal from public at cost
                     stealable = self.stealable_all[level]
@@ -388,10 +393,18 @@ def maybe_move_task(level, ts, sat, idl, duration, cost_multiplier):
                             continue
 
                         i += 1
-                        idl = idle[i % len(idle)]
+                        if _has_restrictions(ts):
+                            thieves = [ws for ws in idle if _can_steal(ws, ts, sat)]
+                        else:
+                            thieves = idle
+                        if not thieves:
+                            continue
+                        thief = thieves[i % len(thieves)]
                         duration = sat.processing[ts]
 
-                        maybe_move_task(level, ts, sat, idl, duration, cost_multiplier)
+                        maybe_move_task(
+                            level, ts, sat, thief, duration, cost_multiplier
+                        )
 
             if log:
                 self.log.append(log)
@@ -422,4 +435,41 @@ def story(self, *keys):
         return out
 
 
+def _has_restrictions(ts):
+    """Determine whether the given task has restrictions and whether these
+    restrictions are strict.
+    """
+    return not ts.loose_restrictions and (
+        ts.host_restrictions or ts.worker_restrictions or ts.resource_restrictions
+    )
+
+
+def _can_steal(thief, ts, victim):
+    """Determine whether worker ``thief`` can steal task ``ts`` from worker
+    ``victim``.
+
+    Assumes that `ts` has some restrictions.
+    """
+    if (
+        ts.host_restrictions
+        and get_address_host(thief.address) not in ts.host_restrictions
+    ):
+        return False
+    elif ts.worker_restrictions and thief.address not in ts.worker_restrictions:
+        return False
+
+    if victim.resources is None:
+        return True
+
+    for resource, value in victim.resources.items():
+        try:
+            supplied = thief.resources[resource]
+        except KeyError:
+            return False
+        else:
+            if supplied < value:
+                return False
+    return True
+
+
 fast_tasks = {"shuffle-split"}
diff --git a/distributed/tests/test_steal.py b/distributed/tests/test_steal.py
index b017bff4371..71f408749a1 100644
--- a/distributed/tests/test_steal.py
+++ b/distributed/tests/test_steal.py
@@ -224,6 +224,32 @@ def test_dont_steal_worker_restrictions(c, s, a, b):
     assert len(b.task_state) == 0
 
 
+@gen_cluster(
+    client=True, nthreads=[("127.0.0.1", 1), ("127.0.0.1", 2), ("127.0.0.1", 2)]
+)
+def test_steal_worker_restrictions(c, s, wa, wb, wc):
+    future = c.submit(slowinc, 1, delay=0.1, workers={wa.address, wb.address})
+    yield future
+
+    ntasks = 100
+    futures = c.map(slowinc, range(ntasks), delay=0.1, workers={wa.address, wb.address})
+
+    while sum(len(w.task_state) for w in [wa, wb, wc]) < ntasks:
+        yield gen.sleep(0.01)
+
+    assert 0 < len(wa.task_state) < ntasks
+    assert 0 < len(wb.task_state) < ntasks
+    assert len(wc.task_state) == 0
+
+    s.extensions["stealing"].balance()
+
+    yield gen.sleep(0.1)
+
+    assert 0 < len(wa.task_state) < ntasks
+    assert 0 < len(wb.task_state) < ntasks
+    assert len(wc.task_state) == 0
+
+
 @pytest.mark.skipif(
     not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
 )
@@ -245,6 +271,34 @@ def test_dont_steal_host_restrictions(c, s, a, b):
     assert len(b.task_state) == 0
 
 
+@pytest.mark.skipif(
+    not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
+)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1), ("127.0.0.2", 2)])
+def test_steal_host_restrictions(c, s, wa, wb):
+    future = c.submit(slowinc, 1, delay=0.10, workers=wa.address)
+    yield future
+
+    ntasks = 100
+    futures = c.map(slowinc, range(ntasks), delay=0.1, workers="127.0.0.1")
+    while len(wa.task_state) < ntasks:
+        yield gen.sleep(0.01)
+    assert len(wa.task_state) == ntasks
+    assert len(wb.task_state) == 0
+
+    wc = yield Worker(s.address, ncores=1)
+
+    start = time()
+    while not wc.task_state or len(wa.task_state) == ntasks:
+        yield gen.sleep(0.01)
+        assert time() < start + 3
+
+    yield gen.sleep(0.1)
+    assert 0 < len(wa.task_state) < ntasks
+    assert len(wb.task_state) == 0
+    assert 0 < len(wc.task_state) < ntasks
+
+
 @gen_cluster(
     client=True, nthreads=[("127.0.0.1", 1, {"resources": {"A": 2}}), ("127.0.0.1", 1)]
 )
@@ -265,7 +319,6 @@ def test_dont_steal_resource_restrictions(c, s, a, b):
     assert len(b.task_state) == 0
 
 
-@pytest.mark.skip(reason="no stealing of resources")
 @gen_cluster(
     client=True, nthreads=[("127.0.0.1", 1, {"resources": {"A": 2}})], timeout=3
 )
diff --git a/distributed/worker.py b/distributed/worker.py
index a5a39fe22b3..aa71a16640b 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -2150,7 +2150,7 @@ def steal_request(self, key):
         response = {"op": "steal-response", "key": key, "state": state}
         self.batched_stream.send(response)
 
-        if state in ("ready", "waiting"):
+        if state in ("ready", "waiting", "constrained"):
             self.release_key(key)
 
     def release_key(self, key, cause=None, reason=None, report=True):

From 15550952aabfecf9e6a6bbccb7dea82d72857a4c Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Tue, 3 Mar 2020 18:20:47 -0800
Subject: [PATCH 0712/1550] Serialize sparse arrays (#3545)

* Serialize SciPy sparse matrices

As SciPy sparse matrices just consist of a handful of NumPy arrays as
attributes (which Dask already knows how to serialize), we can just ask
Dask to comb through them for us and serialize their components.

* Special case scipy.sparse's `dok_matrix`

As `dok_matrix` is a subclass of `spmatrix` and `dict`, this confuses
Dask's `register_generic` machinery. Not to mention this doesn't
actually contain any NumPy `ndarray`s. Instead it just stores
coordinates as `tuple`s in keys and data in values.

Ideally we would just pack the dictionary into the header and move on.
However as the data included is not MessagePack serializable, this
strategy does not work in practice. So simply convert the `dok_matrix`
to a `coo_matrix`, which has very similar layout and is easier for us to
serialize. When deserializing, just extract the `coo_matrix` and convert
it back to a `dok_matrix`.

This let's us bypass the oddities of the `dok_matrix` while still having
reasonably efficient serialization.

* Register SciPy serialization

* Test serializing SciPy sparse matrices

* Serialize CuPy sparse matrices

As CuPy sparse matrices just consist of a handful of CuPy arrays as
attributes (which Dask already knows how to serialize), we can just ask
Dask to comb through them for us and serialize their components.

* Run `isort` on `test_cupy`

* Test serializing CuPy sparse matrices
---
 distributed/protocol/__init__.py         |  6 ++++
 distributed/protocol/cupy.py             | 19 +++++++++++-
 distributed/protocol/scipy.py            | 30 +++++++++++++++++++
 distributed/protocol/tests/test_cupy.py  | 35 +++++++++++++++++++++-
 distributed/protocol/tests/test_scipy.py | 37 ++++++++++++++++++++++++
 5 files changed, 125 insertions(+), 2 deletions(-)
 create mode 100644 distributed/protocol/scipy.py
 create mode 100644 distributed/protocol/tests/test_scipy.py

diff --git a/distributed/protocol/__init__.py b/distributed/protocol/__init__.py
index 212051427f5..c34f161a1fe 100644
--- a/distributed/protocol/__init__.py
+++ b/distributed/protocol/__init__.py
@@ -30,6 +30,12 @@ def _register_numpy():
     from . import numpy
 
 
+@dask_serialize.register_lazy("scipy")
+@dask_deserialize.register_lazy("scipy")
+def _register_scipy():
+    from . import scipy
+
+
 @dask_serialize.register_lazy("h5py")
 @dask_deserialize.register_lazy("h5py")
 def _register_h5py():
diff --git a/distributed/protocol/cupy.py b/distributed/protocol/cupy.py
index 40bf6efda4f..9245412de6e 100644
--- a/distributed/protocol/cupy.py
+++ b/distributed/protocol/cupy.py
@@ -4,7 +4,7 @@
 import cupy
 
 from .cuda import cuda_deserialize, cuda_serialize
-from .serialize import dask_deserialize, dask_serialize
+from .serialize import dask_deserialize, dask_serialize, register_generic
 
 try:
     from .rmm import dask_deserialize_rmm_device_buffer as dask_deserialize_cuda_buffer
@@ -80,3 +80,20 @@ def dask_deserialize_cupy_ndarray(header, frames):
     frames = [dask_deserialize_cuda_buffer(header, frames)]
     arr = cuda_deserialize_cupy_ndarray(header, frames)
     return arr
+
+
+try:
+    from cupy.cusparse import MatDescriptor
+    from cupyx.scipy.sparse import spmatrix
+
+    cupy_sparse_types = [MatDescriptor, spmatrix]
+except ImportError:
+    cupy_sparse_types = []
+
+
+for t in cupy_sparse_types:
+    for n, s, d in [
+        ("cuda", cuda_serialize, cuda_deserialize),
+        ("dask", dask_serialize, dask_deserialize),
+    ]:
+        register_generic(t, n, s, d)
diff --git a/distributed/protocol/scipy.py b/distributed/protocol/scipy.py
new file mode 100644
index 00000000000..9ed533bc850
--- /dev/null
+++ b/distributed/protocol/scipy.py
@@ -0,0 +1,30 @@
+"""
+Efficient serialization of SciPy sparse matrices.
+"""
+import scipy
+
+from .serialize import dask_deserialize, dask_serialize, register_generic
+
+register_generic(scipy.sparse.spmatrix, "dask", dask_serialize, dask_deserialize)
+
+
+@dask_serialize.register(scipy.sparse.dok.dok_matrix)
+def serialize_scipy_sparse_dok(x):
+    x_coo = x.tocoo()
+    coo_header, coo_frames = dask_serialize(x.tocoo())
+
+    header = {"coo_header": coo_header}
+    frames = coo_frames
+
+    return header, frames
+
+
+@dask_deserialize.register(scipy.sparse.dok.dok_matrix)
+def deserialize_scipy_sparse_dok(header, frames):
+    coo_header = header["coo_header"]
+    coo_frames = frames
+    x_coo = dask_deserialize(coo_header, coo_frames)
+
+    x = x_coo.todok()
+
+    return x
diff --git a/distributed/protocol/tests/test_cupy.py b/distributed/protocol/tests/test_cupy.py
index 5470266fce5..44d4b80d66d 100644
--- a/distributed/protocol/tests/test_cupy.py
+++ b/distributed/protocol/tests/test_cupy.py
@@ -1,8 +1,10 @@
-from distributed.protocol import serialize, deserialize
 import pickle
+
 import pytest
+from distributed.protocol import deserialize, serialize
 
 cupy = pytest.importorskip("cupy")
+cupy_sparse = pytest.importorskip("cupyx.scipy.sparse")
 numpy = pytest.importorskip("numpy")
 
 
@@ -61,3 +63,34 @@ def test_serialize_cupy_from_rmm(size):
     y = deserialize(header, frames, deserializers=("cuda", "dask", "pickle", "error"))
 
     assert (x_np == cupy.asnumpy(y)).all()
+
+
+@pytest.mark.parametrize(
+    "sparse_type",
+    [
+        cupy_sparse.coo_matrix,
+        cupy_sparse.csc_matrix,
+        cupy_sparse.csr_matrix,
+        cupy_sparse.dia_matrix,
+    ],
+)
+@pytest.mark.parametrize(
+    "dtype",
+    [numpy.dtype("<f4"), numpy.dtype(">f4"), numpy.dtype("<f8"), numpy.dtype(">f8"),],
+)
+@pytest.mark.parametrize("serializer", ["cuda", "dask",])
+def test_serialize_cupy_sparse(sparse_type, dtype, serializer):
+    a_host = numpy.array([[0, 1, 0], [2, 0, 3], [0, 4, 0]], dtype=dtype)
+    a = cupy.asarray(a_host)
+
+    anz = a.nonzero()
+    acoo = cupy_sparse.coo_matrix((a[anz], anz))
+    asp = sparse_type(acoo)
+
+    header, frames = serialize(asp, serializers=[serializer])
+    asp2 = deserialize(header, frames)
+
+    a2 = asp2.todense()
+    a2_host = cupy.asnumpy(a2)
+
+    assert (a_host == a2_host).all()
diff --git a/distributed/protocol/tests/test_scipy.py b/distributed/protocol/tests/test_scipy.py
new file mode 100644
index 00000000000..2cb5d7477e5
--- /dev/null
+++ b/distributed/protocol/tests/test_scipy.py
@@ -0,0 +1,37 @@
+import pytest
+from distributed.protocol import deserialize, serialize
+
+numpy = pytest.importorskip("numpy")
+scipy = pytest.importorskip("scipy")
+scipy_sparse = pytest.importorskip("scipy.sparse")
+
+
+@pytest.mark.parametrize(
+    "sparse_type",
+    [
+        scipy_sparse.bsr_matrix,
+        scipy_sparse.coo_matrix,
+        scipy_sparse.csc_matrix,
+        scipy_sparse.csr_matrix,
+        scipy_sparse.dia_matrix,
+        scipy_sparse.dok_matrix,
+        scipy_sparse.lil_matrix,
+    ],
+)
+@pytest.mark.parametrize(
+    "dtype",
+    [numpy.dtype("<f4"), numpy.dtype(">f4"), numpy.dtype("<f8"), numpy.dtype(">f8"),],
+)
+def test_serialize_scipy_sparse(sparse_type, dtype):
+    a = numpy.array([[0, 1, 0], [2, 0, 3], [0, 4, 0]], dtype=dtype)
+
+    anz = a.nonzero()
+    acoo = scipy_sparse.coo_matrix((a[anz], anz))
+    asp = sparse_type(acoo)
+
+    header, frames = serialize(asp, serializers=["dask"])
+    asp2 = deserialize(header, frames)
+
+    a2 = asp2.todense()
+
+    assert (a == a2).all()

From 9b5bf448af478a166069aefc9c0c1354a29ae482 Mon Sep 17 00:00:00 2001
From: Tom Augspurger <TomAugspurger@users.noreply.github.com>
Date: Wed, 4 Mar 2020 09:59:20 -0600
Subject: [PATCH 0713/1550] API docs for LocalCluster and SpecCluster (#3548)

---
 distributed/client.py |  4 ++--
 docs/source/api.rst   | 22 ++++++++++++++++++++++
 2 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 4e84ea278d2..287425a70f1 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -502,7 +502,7 @@ class Client(Node):
 
     It is also common to create a Client without specifying the scheduler
     address , like ``Client()``.  In this case the Client creates a
-    ``LocalCluster`` in the background and connects to that.  Any extra
+    :class:`LocalCluster` in the background and connects to that.  Any extra
     keywords are passed from Client to LocalCluster in this case.  See the
     LocalCluster documentation for more information.
 
@@ -569,7 +569,7 @@ class Client(Node):
     See Also
     --------
     distributed.scheduler.Scheduler: Internal scheduler
-    distributed.deploy.local.LocalCluster:
+    distributed.LocalCluster:
     """
 
     _instances = weakref.WeakSet()
diff --git a/docs/source/api.rst b/docs/source/api.rst
index 8d739334b07..9d2f6c7f870 100644
--- a/docs/source/api.rst
+++ b/docs/source/api.rst
@@ -148,6 +148,28 @@ Future
 .. autoclass:: Future
    :members:
 
+Cluster
+-------
+
+Classes relevant for cluster creation and management. Other libraries
+(like `dask-jobqueue`_, `dask-gateway`_, `dask-kubernetes`_, `dask-yarn`_ etc.)
+provide additional cluster objects.
+
+.. _dask-jobqueue: https://jobqueue.dask.org/
+.. _dask-gateway: https://gateway.dask.org/
+.. _dask-kubernetes: https://kubernetes.dask.org/
+.. _dask-yarn: https://yarn.dask.org/en/latest/
+
+.. autosummary::
+   LocalCluster
+   SpecCluster
+
+.. autoclass:: LocalCluster
+   :members:
+
+.. autoclass:: SpecCluster
+   :members:
+
 
 Other
 -----

From ae74b5ea18fac7e272c9f25b5d9f2775956aa943 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Thu, 5 Mar 2020 15:17:37 -0800
Subject: [PATCH 0714/1550] Fix-up CuPy sparse serialization (#3556)

Fix-up CuPy sparse serialization
---
 distributed/protocol/__init__.py        |  4 +++
 distributed/protocol/cupy.py            | 32 +++++++++++++++---
 distributed/protocol/tests/test_cupy.py | 45 ++++++++++++++-----------
 3 files changed, 56 insertions(+), 25 deletions(-)

diff --git a/distributed/protocol/__init__.py b/distributed/protocol/__init__.py
index c34f161a1fe..bb919019e04 100644
--- a/distributed/protocol/__init__.py
+++ b/distributed/protocol/__init__.py
@@ -78,6 +78,10 @@ def _register_torch():
 @cuda_deserialize.register_lazy("cupy")
 @dask_serialize.register_lazy("cupy")
 @dask_deserialize.register_lazy("cupy")
+@cuda_serialize.register_lazy("cupyx")
+@cuda_deserialize.register_lazy("cupyx")
+@dask_serialize.register_lazy("cupyx")
+@dask_deserialize.register_lazy("cupyx")
 def _register_cupy():
     from . import cupy
 
diff --git a/distributed/protocol/cupy.py b/distributed/protocol/cupy.py
index 9245412de6e..3d074266245 100644
--- a/distributed/protocol/cupy.py
+++ b/distributed/protocol/cupy.py
@@ -1,6 +1,8 @@
 """
 Efficient serialization GPU arrays.
 """
+import copyreg
+
 import cupy
 
 from .cuda import cuda_deserialize, cuda_serialize
@@ -85,15 +87,35 @@ def dask_deserialize_cupy_ndarray(header, frames):
 try:
     from cupy.cusparse import MatDescriptor
     from cupyx.scipy.sparse import spmatrix
-
-    cupy_sparse_types = [MatDescriptor, spmatrix]
 except ImportError:
-    cupy_sparse_types = []
+    MatDescriptor = None
+    spmatrix = None
+
+
+if MatDescriptor is not None:
+
+    def reduce_matdescriptor(other):
+        # Pickling MatDescriptor errors
+        # xref: https://github.com/cupy/cupy/issues/3061
+        return cupy.cusparse.MatDescriptor.create, ()
+
+    copyreg.pickle(MatDescriptor, reduce_matdescriptor)
+
+    @cuda_serialize.register(MatDescriptor)
+    @dask_serialize.register(MatDescriptor)
+    def serialize_cupy_matdescriptor(x):
+        header, frames = {}, []
+        return header, frames
+
+    @cuda_deserialize.register(MatDescriptor)
+    @dask_deserialize.register(MatDescriptor)
+    def deserialize_cupy_matdescriptor(header, frames):
+        return MatDescriptor.create()
 
 
-for t in cupy_sparse_types:
+if spmatrix is not None:
     for n, s, d in [
         ("cuda", cuda_serialize, cuda_deserialize),
         ("dask", dask_serialize, dask_deserialize),
     ]:
-        register_generic(t, n, s, d)
+        register_generic(spmatrix, n, s, d)
diff --git a/distributed/protocol/tests/test_cupy.py b/distributed/protocol/tests/test_cupy.py
index 44d4b80d66d..95cb530c4db 100644
--- a/distributed/protocol/tests/test_cupy.py
+++ b/distributed/protocol/tests/test_cupy.py
@@ -4,7 +4,6 @@
 from distributed.protocol import deserialize, serialize
 
 cupy = pytest.importorskip("cupy")
-cupy_sparse = pytest.importorskip("cupyx.scipy.sparse")
 numpy = pytest.importorskip("numpy")
 
 
@@ -66,31 +65,37 @@ def test_serialize_cupy_from_rmm(size):
 
 
 @pytest.mark.parametrize(
-    "sparse_type",
-    [
-        cupy_sparse.coo_matrix,
-        cupy_sparse.csc_matrix,
-        cupy_sparse.csr_matrix,
-        cupy_sparse.dia_matrix,
-    ],
+    "sparse_name", ["coo_matrix", "csc_matrix", "csr_matrix", "dia_matrix",],
 )
 @pytest.mark.parametrize(
     "dtype",
     [numpy.dtype("<f4"), numpy.dtype(">f4"), numpy.dtype("<f8"), numpy.dtype(">f8"),],
 )
-@pytest.mark.parametrize("serializer", ["cuda", "dask",])
-def test_serialize_cupy_sparse(sparse_type, dtype, serializer):
-    a_host = numpy.array([[0, 1, 0], [2, 0, 3], [0, 4, 0]], dtype=dtype)
-    a = cupy.asarray(a_host)
-
-    anz = a.nonzero()
-    acoo = cupy_sparse.coo_matrix((a[anz], anz))
-    asp = sparse_type(acoo)
+@pytest.mark.parametrize("serializer", ["cuda", "dask", "pickle"])
+def test_serialize_cupy_sparse(sparse_name, dtype, serializer):
+    scipy_sparse = pytest.importorskip("scipy.sparse")
+    cupy_sparse = pytest.importorskip("cupyx.scipy.sparse")
 
-    header, frames = serialize(asp, serializers=[serializer])
-    asp2 = deserialize(header, frames)
+    scipy_sparse_type = getattr(scipy_sparse, sparse_name)
+    cupy_sparse_type = getattr(cupy_sparse, sparse_name)
 
-    a2 = asp2.todense()
-    a2_host = cupy.asnumpy(a2)
+    a_host = numpy.array([[0, 1, 0], [2, 0, 3], [0, 4, 0]], dtype=dtype)
+    asp_host = scipy_sparse_type(a_host)
+    if sparse_name == "dia_matrix":
+        # CuPy `dia_matrix` cannot be created from SciPy one
+        # xref: https://github.com/cupy/cupy/issues/3158
+        asp_dev = cupy_sparse_type(
+            (asp_host.data, asp_host.offsets),
+            shape=asp_host.shape,
+            dtype=asp_host.dtype,
+        )
+    else:
+        asp_dev = cupy_sparse_type(asp_host)
+
+    header, frames = serialize(asp_dev, serializers=[serializer])
+    a2sp_dev = deserialize(header, frames)
+
+    a2sp_host = a2sp_dev.get()
+    a2_host = a2sp_host.todense()
 
     assert (a_host == a2_host).all()

From e619fc99ad5b635f52f69ce2edafed77cfa7c898 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Fri, 6 Mar 2020 10:23:06 -0600
Subject: [PATCH 0715/1550] Update TaskGroup remove logic (#3557)

---
 distributed/scheduler.py            |  2 +-
 distributed/tests/test_scheduler.py | 11 +++++++++++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index c78c4b1b218..ab026f61d06 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -4649,7 +4649,7 @@ def transition(self, key, finish, *args, **kwargs):
                 if ts.state == "forgotten":
                     del self.tasks[ts.key]
 
-            if ts.state == "forgotten":
+            if ts.state == "forgotten" and ts.group.name in self.task_groups:
                 # Remove TaskGroup if all tasks are in the forgotten state
                 tg = ts.group
                 if not any(tg.states.get(s) for s in ALL_TASK_STATES):
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 5c4d8cbc23e..a5649dbfc82 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -1837,6 +1837,17 @@ async def test_task_unique_groups(c, s, a, b):
     assert s.task_prefixes["sum"].states["memory"] == 2
 
 
+@gen_cluster(client=True)
+async def test_task_group_on_fire_and_forget(c, s, a, b):
+    # Regression test for https://github.com/dask/distributed/issues/3465
+    with captured_logger("distributed.scheduler") as logs:
+        x = await c.scatter(list(range(10)))
+        fire_and_forget([c.submit(slowadd, i, x[i]) for i in range(len(x))])
+        await asyncio.sleep(1)
+
+    assert "Error transitioning" not in logs.getvalue()
+
+
 class BrokenComm(Comm):
     peer_address = None
     local_address = None

From 15591929f3a6b7b390a7a5394e1f53fe6a6c16f4 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Fri, 6 Mar 2020 14:16:22 -0600
Subject: [PATCH 0716/1550] bump version to 2.12.0

---
 docs/source/changelog.rst | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index 498668d3c88..9c7ca9b01f4 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,34 @@
 Changelog
 =========
 
+2.12.0 - 2020-03-06
+-------------------
+
+- Update ``TaskGroup`` remove logic (:pr:`3557`) `James Bourbeau`_
+- Fix-up CuPy sparse serialization (:pr:`3556`) `John Kirkham`_
+- API docs for ``LocalCluster`` and ``SpecCluster`` (:pr:`3548`) `Tom Augspurger`_
+- Serialize sparse arrays (:pr:`3545`) `John Kirkham`_
+- Allow tasks with restrictions to be stolen (:pr:`3069`) `Stan Seibert`_
+- Use UCX default configuration instead of raising (:pr:`3544`) `Peter Andreas Entschev`_
+- Support using other serializers with ``register_generic`` (:pr:`3536`) `John Kirkham`_
+- DOC: update to async await (:pr:`3543`) `Tom Augspurger`_
+- Use ``pytest.raises`` in ``test_ucx_config.py`` (:pr:`3541`) `John Kirkham`_
+- Fix/more ucx config options (:pr:`3539`) `Benjamin Zaitlen`_
+- Update heartbeat ``CommClosedError`` error handling (:pr:`3529`) `James Bourbeau`_
+- Use ``makedirs`` when constructing ``local_directory`` (:pr:`3538`) `John Kirkham`_
+- Mark ``None`` as MessagePack serializable (:pr:`3537`) `John Kirkham`_
+- Mark ``bool`` as MessagePack serializable (:pr:`3535`) `John Kirkham`_
+- Use 'temporary-directory' from ``dask.config`` for Nanny's directory (:pr:`3531`) `John Kirkham`_
+- Add try-except around getting source code in performance report (:pr:`3505`) `Matthew Rocklin`_
+- Fix typo in docstring (:pr:`3528`) `Davis Bennett`_
+- Make work stealing callback time configurable (:pr:`3523`) `Lucas Rademaker`_
+- RMM/UCX Config Flags (:pr:`3515`) `Benjamin Zaitlen`_
+- Revise develop-docs: conda env example (:pr:`3406`) `Darren Weber`_
+- Remove ``import ucp`` from the top of ``ucx.py`` (:pr:`3510`) `Peter Andreas Entschev`_
+- Rename ``logs`` to ``get_logs`` (:pr:`3473`) `Jacob Tomlinson`_
+- Stop keep alives when worker reconnecting to the scheduler (:pr:`3493`) `Jacob Tomlinson`_
+
+
 2.11.0 - 2020-02-19
 -------------------
 
@@ -1570,3 +1598,7 @@ significantly without many new features.
 .. _`Cyril Shcherbin`: https://github.com/shcherbin
 .. _`Søren Fuglede Jørgensen`: https://github.com/fuglede
 .. _`Igor Gotlibovych`: https://github.com/ig248
+.. _`Stan Seibert`: https://github.com/seibert
+.. _`Davis Bennett`: https://github.com/d-v-b
+.. _`Lucas Rademaker`: https://github.com/lr4d
+.. _`Darren Weber`: https://github.com/dazza-codes

From 73f8ae229d558e2c25ef0baea6fb6d127e78191e Mon Sep 17 00:00:00 2001
From: Abdulelah Bin Mahfoodh <abduh.bm@gmail.com>
Date: Mon, 9 Mar 2020 18:24:13 +0300
Subject: [PATCH 0717/1550] Add 'local_directory' option to dask-ssh (#3554)

---
 distributed/cli/dask_ssh.py   | 10 ++++++++++
 distributed/deploy/old_ssh.py | 24 +++++++++++++++++++++++-
 2 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/distributed/cli/dask_ssh.py b/distributed/cli/dask_ssh.py
index 07cbb57bf01..eb09f49cfed 100755
--- a/distributed/cli/dask_ssh.py
+++ b/distributed/cli/dask_ssh.py
@@ -77,6 +77,14 @@
         "dask-scheduler and dask-worker commands."
     ),
 )
+@click.option(
+    "--local-directory",
+    default=None,
+    type=click.Path(exists=True),
+    help=(
+        "Directory to use on all cluster nodes to place workers " "and scheduler files."
+    ),
+)
 @click.option(
     "--remote-python", default=None, type=str, help="Path to Python on remote nodes."
 )
@@ -126,6 +134,7 @@ def main(
     worker_port,
     nanny_port,
     remote_dask_worker,
+    local_directory,
 ):
     try:
         hostnames = list(hostnames)
@@ -157,6 +166,7 @@ def main(
         worker_port,
         nanny_port,
         remote_dask_worker,
+        local_directory,
     )
 
     import distributed
diff --git a/distributed/deploy/old_ssh.py b/distributed/deploy/old_ssh.py
index 86d49c9cf15..b524e2d7c45 100644
--- a/distributed/deploy/old_ssh.py
+++ b/distributed/deploy/old_ssh.py
@@ -209,12 +209,24 @@ def communicate():
 
 
 def start_scheduler(
-    logdir, addr, port, ssh_username, ssh_port, ssh_private_key, remote_python=None
+    logdir,
+    addr,
+    port,
+    ssh_username,
+    ssh_port,
+    ssh_private_key,
+    remote_python=None,
+    local_directory=None,
 ):
     cmd = "{python} -m distributed.cli.dask_scheduler --port {port}".format(
         python=remote_python or sys.executable, port=port, logdir=logdir
     )
 
+    if local_directory is not None:
+        cmd += " --local-directory {local_directory}".format(
+            local_directory=local_directory
+        )
+
     # Optionally re-direct stdout and stderr to a logfile
     if logdir is not None:
         cmd = "mkdir -p {logdir} && ".format(logdir=logdir) + cmd
@@ -270,6 +282,7 @@ def start_worker(
     nanny_port,
     remote_python=None,
     remote_dask_worker="distributed.cli.dask_worker",
+    local_directory=None,
 ):
 
     cmd = (
@@ -303,6 +316,11 @@ def start_worker(
         nanny_port=nanny_port,
     )
 
+    if local_directory is not None:
+        cmd += " --local-directory {local_directory}".format(
+            local_directory=local_directory
+        )
+
     # Optionally redirect stdout and stderr to a logfile
     if logdir is not None:
         cmd = "mkdir -p {logdir} && ".format(logdir=logdir) + cmd
@@ -353,6 +371,7 @@ def __init__(
         worker_port=None,
         nanny_port=None,
         remote_dask_worker="distributed.cli.dask_worker",
+        local_directory=None,
     ):
 
         self.scheduler_addr = scheduler_addr
@@ -372,6 +391,7 @@ def __init__(
         self.worker_port = worker_port
         self.nanny_port = nanny_port
         self.remote_dask_worker = remote_dask_worker
+        self.local_directory = local_directory
 
         # Generate a universal timestamp to use for log files
         import datetime
@@ -402,6 +422,7 @@ def __init__(
             ssh_port,
             ssh_private_key,
             remote_python,
+            local_directory,
         )
 
         # Start worker nodes
@@ -455,6 +476,7 @@ def add_worker(self, address):
                 self.nanny_port,
                 self.remote_python,
                 self.remote_dask_worker,
+                self.local_directory,
             )
         )
 

From b809777d250152edb52b846e9c1c12a20f500878 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Mon, 9 Mar 2020 12:00:20 -0700
Subject: [PATCH 0718/1550] Fix typo in Client.shutdown docstring (#3562)

---
 distributed/client.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/client.py b/distributed/client.py
index 287425a70f1..679e625470f 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -1382,7 +1382,7 @@ def shutdown(self):
         """ Shut down the connected scheduler and workers
 
         Note, this may disrupt other clients that may be using the same
-        scheudler and workers.
+        scheduler and workers.
 
         See also
         --------

From 81e303afc2bae5d3696ad4c29cb189e9f55cfcb8 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Thu, 12 Mar 2020 14:10:12 -0500
Subject: [PATCH 0719/1550] Disable fast fail on GitHub Actions Windows CI
 (#3569)

---
 .github/workflows/ci-windows.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/ci-windows.yaml b/.github/workflows/ci-windows.yaml
index 75c4b294e88..78db494a6fb 100644
--- a/.github/workflows/ci-windows.yaml
+++ b/.github/workflows/ci-windows.yaml
@@ -6,6 +6,7 @@ jobs:
   build:
     runs-on: windows-latest
     strategy:
+      fail-fast: false
       matrix:
         python-version: ["3.6", "3.7", "3.8"]
 

From 8e8438324b4323131e941428cd48df8e52b2edd8 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Fri, 13 Mar 2020 16:40:42 -0500
Subject: [PATCH 0720/1550] Pin bokeh in CI builds (#3570)

---
 .github/workflows/ci-windows.yaml      | 8 ++++----
 continuous_integration/environment.yml | 3 +--
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/ci-windows.yaml b/.github/workflows/ci-windows.yaml
index 78db494a6fb..3b99a8c8ec0 100644
--- a/.github/workflows/ci-windows.yaml
+++ b/.github/workflows/ci-windows.yaml
@@ -32,14 +32,14 @@ jobs:
             conda install -c conda-forge tornado=5
           fi
 
-      - name: List packages in environment
-        shell: bash -l {0}
-        run: conda list
-
       - name: Install distributed from source
         shell: bash -l {0}
         run: python -m pip install -q --no-deps -e .
 
+      - name: List packages in environment
+        shell: bash -l {0}
+        run: conda list
+
       - name: Run tests
         shell: bash -l {0}
         env:
diff --git a/continuous_integration/environment.yml b/continuous_integration/environment.yml
index 8f8e425dcab..8218d721e85 100644
--- a/continuous_integration/environment.yml
+++ b/continuous_integration/environment.yml
@@ -1,10 +1,9 @@
 name: testenv
 channels:
-  - defaults
   - conda-forge
 dependencies:
   - zstandard
-  - bokeh
+  - bokeh=1.4.0
   - click
   - cloudpickle
   - dask

From 0ad75b044731338484cbde7a60986bc6a0258483 Mon Sep 17 00:00:00 2001
From: Krishan Bhasin <KrishanBhasin@users.noreply.github.com>
Date: Fri, 13 Mar 2020 22:15:25 +0000
Subject: [PATCH 0721/1550] Avoid performance_report crashing when a worker
 dies mid-compute (#3575)

---
 distributed/dashboard/components/scheduler.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index c70e41ca436..ee037a4aabb 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -412,7 +412,10 @@ def update(self):
                 return
 
             def name(address):
-                ws = self.scheduler.workers[address]
+                try:
+                    ws = self.scheduler.workers[address]
+                except KeyError:
+                    return address
                 if ws.name is not None:
                     return str(ws.name)
                 else:

From 806a7e97285c5534b3e37e912cbc060a8036c56f Mon Sep 17 00:00:00 2001
From: Gil Forsyth <gforsyth@users.noreply.github.com>
Date: Sat, 14 Mar 2020 11:49:30 -0400
Subject: [PATCH 0722/1550] Pin `numpydoc` to avoid double escaped * (#3530)

Similar to dask/dask#5961, recent changes to `numpydoc` lead to function
signatures displayed using `autosummary` to have doubly-escaped `*`s.

This pins `numpydoc` to version 0.8.0 to avoid the regression until a
patch is merged upstream.

Also small changes to avoid a sphinx timeout looking for an intersphinx
inventory, and a fixed a misformatted code-block
---
 docs/requirements.txt     | 3 ++-
 docs/source/conf.py       | 3 ++-
 docs/source/resources.rst | 1 +
 3 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/docs/requirements.txt b/docs/requirements.txt
index 61dd185a5b9..6bcd69b284d 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -1,8 +1,9 @@
+# We pin numpydoc to avoid doubly-escaped *args and **kwargs in rendered docs
+numpydoc==0.8.0
 tornado
 toolz
 cloudpickle
 dask
-numpydoc
 sphinx
 dask_sphinx_theme
 sphinx-click
diff --git a/docs/source/conf.py b/docs/source/conf.py
index f8ab5a31797..9bda8cb1a14 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -39,6 +39,7 @@
 
 numpydoc_show_class_members = False
 
+
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ["_templates"]
 
@@ -382,7 +383,7 @@
 # and the Numpy documentation.
 intersphinx_mapping = {
     "python": ("https://docs.python.org/3", None),
-    "numpy": ("http://docs.scipy.org/doc/numpy", None),
+    "numpy": ("https://docs.scipy.org/doc/numpy", None),
 }
 
 # Redirects
diff --git a/docs/source/resources.rst b/docs/source/resources.rst
index f9449dbd8be..7931b980d03 100644
--- a/docs/source/resources.rst
+++ b/docs/source/resources.rst
@@ -96,6 +96,7 @@ delayed objects.  You can pass a dictionary mapping keys of the collection to
 resource requirements during compute or persist calls.
 
 .. code-block:: python
+
     from dask import core
     
     x = dd.read_csv(...)

From f2f82c6c2e8d36731cb3fb82fb1f80ea0323358e Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Tue, 17 Mar 2020 08:51:52 -0700
Subject: [PATCH 0723/1550] Import tlz (#3579)

Import from `tlz` for optional `cytoolz` support
---
 distributed/cfexecutor.py                       |  2 +-
 distributed/cli/dask_worker.py                  |  2 +-
 distributed/client.py                           |  6 ++----
 distributed/core.py                             |  2 +-
 distributed/dashboard/components/__init__.py    |  1 -
 distributed/dashboard/components/scheduler.py   |  8 ++------
 distributed/dashboard/components/shared.py      |  2 +-
 distributed/dashboard/components/worker.py      |  2 +-
 distributed/dashboard/scheduler.py              |  5 +----
 .../dashboard/tests/test_scheduler_bokeh.py     |  4 ++--
 .../dashboard/tests/test_worker_bokeh.py        |  2 +-
 distributed/dashboard/utils.py                  |  8 ++------
 distributed/dashboard/worker.py                 |  2 +-
 distributed/deploy/adaptive_core.py             |  2 +-
 distributed/deploy/old_ssh.py                   |  2 +-
 distributed/deploy/tests/test_spec_cluster.py   |  2 +-
 distributed/diagnostics/progress.py             |  2 +-
 distributed/diagnostics/progress_stream.py      |  2 +-
 distributed/diagnostics/progressbar.py          |  2 +-
 .../diagnostics/tests/test_task_stream.py       |  2 +-
 distributed/diagnostics/tests/test_widgets.py   |  2 +-
 distributed/profile.py                          |  2 +-
 distributed/protocol/compression.py             |  3 ++-
 distributed/protocol/core.py                    |  6 +-----
 distributed/protocol/serialize.py               |  5 +----
 distributed/protocol/tests/test_serialize.py    |  2 +-
 distributed/scheduler.py                        | 17 ++++++++++++-----
 distributed/stealing.py                         |  5 +----
 distributed/tests/test_batched.py               |  2 +-
 distributed/tests/test_client.py                |  3 ++-
 distributed/tests/test_client_executor.py       |  2 +-
 distributed/tests/test_failed_workers.py        |  2 +-
 distributed/tests/test_ipython.py               |  2 +-
 distributed/tests/test_nanny.py                 |  2 +-
 distributed/tests/test_profile.py               |  2 +-
 distributed/tests/test_pubsub.py                |  2 +-
 distributed/tests/test_scheduler.py             |  2 +-
 distributed/tests/test_steal.py                 |  2 +-
 distributed/tests/test_stress.py                |  2 +-
 distributed/tests/test_worker.py                |  2 +-
 distributed/utils.py                            |  2 +-
 distributed/utils_comm.py                       |  2 +-
 distributed/utils_test.py                       |  2 +-
 distributed/variable.py                         |  5 +----
 distributed/worker.py                           |  6 ++----
 docs/source/efficiency.rst                      |  2 +-
 requirements.txt                                |  2 +-
 47 files changed, 64 insertions(+), 84 deletions(-)

diff --git a/distributed/cfexecutor.py b/distributed/cfexecutor.py
index 985a407bdb9..545dbbced09 100644
--- a/distributed/cfexecutor.py
+++ b/distributed/cfexecutor.py
@@ -1,7 +1,7 @@
 import concurrent.futures as cf
 import weakref
 
-from toolz import merge
+from tlz import merge
 
 from tornado import gen
 
diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
index 5188333b75c..29261b52451 100755
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -22,7 +22,7 @@
 )
 from distributed.utils import deserialize_for_cli, import_term
 
-from toolz import valmap
+from tlz import valmap
 from tornado.ioloop import IOLoop, TimeoutError
 
 logger = logging.getLogger("distributed.dask_worker")
diff --git a/distributed/client.py b/distributed/client.py
index 679e625470f..06c6d245c07 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -30,10 +30,8 @@
 from dask.compatibility import apply
 from dask.utils import ensure_dict, format_bytes, funcname
 
-try:
-    from cytoolz import first, groupby, merge, valmap, keymap
-except ImportError:
-    from toolz import first, groupby, merge, valmap, keymap
+from tlz import first, groupby, merge, valmap, keymap
+
 try:
     from dask.delayed import single_key
 except ImportError:
diff --git a/distributed/core.py b/distributed/core.py
index 5768f0f4d8e..ec1e6c5214c 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -10,7 +10,7 @@
 
 import dask
 import tblib
-from toolz import merge
+from tlz import merge
 from tornado import gen
 from tornado.ioloop import IOLoop
 
diff --git a/distributed/dashboard/components/__init__.py b/distributed/dashboard/components/__init__.py
index bb8269083e9..f6159e83bcf 100644
--- a/distributed/dashboard/components/__init__.py
+++ b/distributed/dashboard/components/__init__.py
@@ -26,7 +26,6 @@
 from bokeh.plotting import figure
 import dask
 from tornado import gen
-import toolz
 
 from distributed.dashboard.utils import without_property_validation, BOKEH_VERSION
 from distributed import profile
diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index ee037a4aabb..c371210c701 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -35,7 +35,8 @@
 from bokeh.io import curdoc
 import dask
 from dask.utils import format_bytes, key_split
-from toolz import pipe
+from tlz import pipe
+from tlz.curried import map, concat, groupby
 from tornado import escape
 
 try:
@@ -63,11 +64,6 @@
 from distributed.diagnostics.graph_layout import GraphLayout
 from distributed.diagnostics.task_stream import TaskStreamPlugin
 
-try:
-    from cytoolz.curried import map, concat, groupby
-except ImportError:
-    from toolz.curried import map, concat, groupby
-
 if dask.config.get("distributed.dashboard.export-tool"):
     from distributed.dashboard.export_tool import ExportTool
 else:
diff --git a/distributed/dashboard/components/shared.py b/distributed/dashboard/components/shared.py
index 611d281dd5e..24db46385e7 100644
--- a/distributed/dashboard/components/shared.py
+++ b/distributed/dashboard/components/shared.py
@@ -15,7 +15,7 @@
 from bokeh.plotting import figure
 import dask
 from tornado import gen
-import toolz
+import tlz as toolz
 
 from distributed.dashboard.components import DashboardComponent
 from distributed.dashboard.utils import (
diff --git a/distributed/dashboard/components/worker.py b/distributed/dashboard/components/worker.py
index 440e7279e3b..a11d3047838 100644
--- a/distributed/dashboard/components/worker.py
+++ b/distributed/dashboard/components/worker.py
@@ -20,7 +20,7 @@
 from bokeh.palettes import RdBu
 from bokeh.themes import Theme
 from dask.utils import format_bytes
-from toolz import merge, partition_all
+from tlz import merge, partition_all
 
 from distributed.dashboard.components import add_periodic_callback
 from distributed.dashboard.components.shared import (
diff --git a/distributed/dashboard/scheduler.py b/distributed/dashboard/scheduler.py
index 836cefbbd6c..acaab24cd17 100644
--- a/distributed/dashboard/scheduler.py
+++ b/distributed/dashboard/scheduler.py
@@ -8,10 +8,7 @@
 import dask
 from dask.utils import format_bytes
 
-try:
-    from cytoolz import merge, merge_with
-except ImportError:
-    from toolz import merge, merge_with
+from tlz import merge, merge_with
 
 from tornado import escape
 from tornado.websocket import WebSocketHandler
diff --git a/distributed/dashboard/tests/test_scheduler_bokeh.py b/distributed/dashboard/tests/test_scheduler_bokeh.py
index 4977ee8fa76..f36bfd897e1 100644
--- a/distributed/dashboard/tests/test_scheduler_bokeh.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh.py
@@ -7,7 +7,7 @@
 import pytest
 
 pytest.importorskip("bokeh")
-from toolz import first
+from tlz import first
 from tornado import gen
 from tornado.httpclient import AsyncHTTPClient, HTTPRequest
 
@@ -624,7 +624,7 @@ def test_proxy_to_workers(c, s, a, b):
     },
 )
 async def test_lots_of_tasks(c, s, a, b):
-    import toolz
+    import tlz as toolz
 
     ts = TaskStream(s)
     ts.update()
diff --git a/distributed/dashboard/tests/test_worker_bokeh.py b/distributed/dashboard/tests/test_worker_bokeh.py
index b33fc3ba185..97729fce14f 100644
--- a/distributed/dashboard/tests/test_worker_bokeh.py
+++ b/distributed/dashboard/tests/test_worker_bokeh.py
@@ -6,7 +6,7 @@
 
 pytest.importorskip("bokeh")
 import sys
-from toolz import first
+from tlz import first
 from tornado import gen
 from tornado.httpclient import AsyncHTTPClient
 
diff --git a/distributed/dashboard/utils.py b/distributed/dashboard/utils.py
index b47cb75d6b0..394e016a4da 100644
--- a/distributed/dashboard/utils.py
+++ b/distributed/dashboard/utils.py
@@ -5,7 +5,8 @@
 import bokeh
 from bokeh.io import curdoc
 from tornado import web
-from toolz import partition
+from tlz import partition
+from tlz.curried import first
 
 try:
     import numpy as np
@@ -13,11 +14,6 @@
     np = False
 
 
-try:
-    from cytoolz.curried import first
-except ImportError:
-    from toolz.curried import first
-
 BOKEH_VERSION = LooseVersion(bokeh.__version__)
 dirname = os.path.dirname(__file__)
 
diff --git a/distributed/dashboard/worker.py b/distributed/dashboard/worker.py
index db29480666b..54b3a0a4a51 100644
--- a/distributed/dashboard/worker.py
+++ b/distributed/dashboard/worker.py
@@ -3,7 +3,7 @@
 import os
 
 from bokeh.themes import Theme
-from toolz import merge
+from tlz import merge
 
 from .components.worker import (
     status_doc,
diff --git a/distributed/deploy/adaptive_core.py b/distributed/deploy/adaptive_core.py
index dfd82ea33ba..192e244bd08 100644
--- a/distributed/deploy/adaptive_core.py
+++ b/distributed/deploy/adaptive_core.py
@@ -2,7 +2,7 @@
 import math
 
 from tornado.ioloop import IOLoop
-import toolz
+import tlz as toolz
 
 from ..metrics import time
 from ..utils import parse_timedelta, PeriodicCallback
diff --git a/distributed/deploy/old_ssh.py b/distributed/deploy/old_ssh.py
index b524e2d7c45..33e69772f9b 100644
--- a/distributed/deploy/old_ssh.py
+++ b/distributed/deploy/old_ssh.py
@@ -12,7 +12,7 @@
 
 from threading import Thread
 
-from toolz import merge
+from tlz import merge
 
 from tornado import gen
 
diff --git a/distributed/deploy/tests/test_spec_cluster.py b/distributed/deploy/tests/test_spec_cluster.py
index 68642cda9d2..90ce9923c69 100644
--- a/distributed/deploy/tests/test_spec_cluster.py
+++ b/distributed/deploy/tests/test_spec_cluster.py
@@ -8,7 +8,7 @@
 from distributed.metrics import time
 from distributed.utils_test import loop, cleanup  # noqa: F401
 from distributed.utils import is_valid_xml
-import toolz
+import tlz as toolz
 import pytest
 
 
diff --git a/distributed/diagnostics/progress.py b/distributed/diagnostics/progress.py
index 1dcab0dc9e9..2aeba986839 100644
--- a/distributed/diagnostics/progress.py
+++ b/distributed/diagnostics/progress.py
@@ -3,7 +3,7 @@
 import logging
 from timeit import default_timer
 
-from toolz import groupby, valmap
+from tlz import groupby, valmap
 
 from .plugin import SchedulerPlugin
 from ..utils import key_split, key_split_group, log_errors, tokey
diff --git a/distributed/diagnostics/progress_stream.py b/distributed/diagnostics/progress_stream.py
index e417ee8e35b..c5e74a30f34 100644
--- a/distributed/diagnostics/progress_stream.py
+++ b/distributed/diagnostics/progress_stream.py
@@ -1,6 +1,6 @@
 import logging
 
-from toolz import valmap, merge
+from tlz import valmap, merge
 
 from .progress import AllProgress
 
diff --git a/distributed/diagnostics/progressbar.py b/distributed/diagnostics/progressbar.py
index ab7800c2125..11da7a30d3d 100644
--- a/distributed/diagnostics/progressbar.py
+++ b/distributed/diagnostics/progressbar.py
@@ -4,7 +4,7 @@
 import sys
 import weakref
 
-from toolz import valmap
+from tlz import valmap
 from tornado.ioloop import IOLoop
 
 from .progress import format_time, Progress, MultiProgress
diff --git a/distributed/diagnostics/tests/test_task_stream.py b/distributed/diagnostics/tests/test_task_stream.py
index 58f1c4319f6..4639c7a7a0b 100644
--- a/distributed/diagnostics/tests/test_task_stream.py
+++ b/distributed/diagnostics/tests/test_task_stream.py
@@ -2,7 +2,7 @@
 from time import sleep
 
 import pytest
-from toolz import frequencies
+from tlz import frequencies
 
 from distributed import get_task_stream
 from distributed.utils_test import gen_cluster, div, inc, slowinc
diff --git a/distributed/diagnostics/tests/test_widgets.py b/distributed/diagnostics/tests/test_widgets.py
index 03689c88c1d..c217d17e293 100644
--- a/distributed/diagnostics/tests/test_widgets.py
+++ b/distributed/diagnostics/tests/test_widgets.py
@@ -74,7 +74,7 @@ def record_display(*args):
 from operator import add
 import re
 
-from toolz import valmap
+from tlz import valmap
 
 from distributed.client import wait
 from distributed.worker import dumps_task
diff --git a/distributed/profile.py b/distributed/profile.py
index 1bef6450974..5bf071e20da 100644
--- a/distributed/profile.py
+++ b/distributed/profile.py
@@ -31,7 +31,7 @@
 import threading
 from time import sleep
 
-import toolz
+import tlz as toolz
 
 from .metrics import time
 from .utils import format_time, color_of, parse_timedelta
diff --git a/distributed/protocol/compression.py b/distributed/protocol/compression.py
index 5e81cdbaf1f..adb3c888be6 100644
--- a/distributed/protocol/compression.py
+++ b/distributed/protocol/compression.py
@@ -3,11 +3,12 @@
 
 Includes utilities for determining whether or not to compress
 """
+from functools import partial
 import logging
 import random
 
 import dask
-from toolz import identity, partial
+from tlz import identity
 
 try:
     import blosc
diff --git a/distributed/protocol/core.py b/distributed/protocol/core.py
index 3937c9c2fc8..3bb863f78c2 100644
--- a/distributed/protocol/core.py
+++ b/distributed/protocol/core.py
@@ -1,13 +1,9 @@
+from functools import reduce
 import logging
 import operator
 
 import msgpack
 
-try:
-    from cytoolz import reduce
-except ImportError:
-    from toolz import reduce
-
 from .compression import compressions, maybe_compress, decompress
 from .serialize import serialize, deserialize, Serialize, Serialized, extract_serialize
 from .utils import frame_split_size, merge_frames, msgpack_opts
diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index c462568cc40..c0fdb98449a 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -4,10 +4,7 @@
 import dask
 from dask.base import normalize_token
 
-try:
-    from cytoolz import valmap, get_in
-except ImportError:
-    from toolz import valmap, get_in
+from tlz import valmap, get_in
 
 import msgpack
 
diff --git a/distributed/protocol/tests/test_serialize.py b/distributed/protocol/tests/test_serialize.py
index caf1bbe0ad5..10e4c5e797d 100644
--- a/distributed/protocol/tests/test_serialize.py
+++ b/distributed/protocol/tests/test_serialize.py
@@ -4,7 +4,7 @@
 import msgpack
 import numpy as np
 import pytest
-from toolz import identity
+from tlz import identity
 
 from distributed import wait
 from distributed.protocol import (
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index ab026f61d06..d543808340d 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -19,11 +19,18 @@
 import psutil
 import sortedcontainers
 
-try:
-    from cytoolz import frequencies, merge, pluck, merge_sorted, first, merge_with
-except ImportError:
-    from toolz import frequencies, merge, pluck, merge_sorted, first, merge_with
-from toolz import valmap, second, compose, groupby
+from tlz import (
+    frequencies,
+    merge,
+    pluck,
+    merge_sorted,
+    first,
+    merge_with,
+    valmap,
+    second,
+    compose,
+    groupby,
+)
 from tornado.ioloop import IOLoop
 
 import dask
diff --git a/distributed/stealing.py b/distributed/stealing.py
index 4fbb753e131..fcceba4824a 100644
--- a/distributed/stealing.py
+++ b/distributed/stealing.py
@@ -9,10 +9,7 @@
 from .diagnostics.plugin import SchedulerPlugin
 from .utils import log_errors, parse_timedelta, PeriodicCallback
 
-try:
-    from cytoolz import topk
-except ImportError:
-    from toolz import topk
+from tlz import topk
 
 LATENCY = 10e-3
 log_2 = log(2)
diff --git a/distributed/tests/test_batched.py b/distributed/tests/test_batched.py
index 74efba810d3..f2b0be99ab0 100644
--- a/distributed/tests/test_batched.py
+++ b/distributed/tests/test_batched.py
@@ -2,7 +2,7 @@
 import random
 
 import pytest
-from toolz import assoc
+from tlz import assoc
 
 from distributed.batched import BatchedSend
 from distributed.core import listen, connect, CommClosedError
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 392aec73be8..5c853916558 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -1,5 +1,6 @@
 import asyncio
 from collections import deque
+from functools import partial
 import gc
 import logging
 from operator import add
@@ -18,7 +19,7 @@
 import zipfile
 
 import pytest
-from toolz import identity, isdistinct, concat, pluck, valmap, partial, first, merge
+from tlz import identity, isdistinct, concat, pluck, valmap, first, merge
 from tornado import gen
 
 import dask
diff --git a/distributed/tests/test_client_executor.py b/distributed/tests/test_client_executor.py
index 1024990216d..e7e3fc24c7d 100644
--- a/distributed/tests/test_client_executor.py
+++ b/distributed/tests/test_client_executor.py
@@ -11,7 +11,7 @@
 )
 
 import pytest
-from toolz import take
+from tlz import take
 
 from distributed import Client
 from distributed.utils import CancelledError
diff --git a/distributed/tests/test_failed_workers.py b/distributed/tests/test_failed_workers.py
index cf0387c1cd2..99b1b4a42a7 100644
--- a/distributed/tests/test_failed_workers.py
+++ b/distributed/tests/test_failed_workers.py
@@ -3,7 +3,7 @@
 from time import sleep
 
 import pytest
-from toolz import partition_all, first
+from tlz import partition_all, first
 from tornado import gen
 
 from dask import delayed
diff --git a/distributed/tests/test_ipython.py b/distributed/tests/test_ipython.py
index aa4a3e4092e..a6d387589e6 100644
--- a/distributed/tests/test_ipython.py
+++ b/distributed/tests/test_ipython.py
@@ -1,7 +1,7 @@
 from unittest import mock
 
 import pytest
-from toolz import first
+from tlz import first
 import tornado
 
 from distributed import Client
diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
index c80974d9970..2a19bdf8742 100644
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -9,7 +9,7 @@
 import numpy as np
 
 import pytest
-from toolz import valmap, first
+from tlz import valmap, first
 from tornado import gen
 from tornado.ioloop import IOLoop
 
diff --git a/distributed/tests/test_profile.py b/distributed/tests/test_profile.py
index a022600d819..9f673e8caaf 100644
--- a/distributed/tests/test_profile.py
+++ b/distributed/tests/test_profile.py
@@ -1,7 +1,7 @@
 import pytest
 import sys
 import time
-from toolz import first
+from tlz import first
 import threading
 
 from distributed.compatibility import WINDOWS
diff --git a/distributed/tests/test_pubsub.py b/distributed/tests/test_pubsub.py
index 2e372dea88b..639542df5ca 100644
--- a/distributed/tests/test_pubsub.py
+++ b/distributed/tests/test_pubsub.py
@@ -3,7 +3,7 @@
 
 import pytest
 from tornado import gen
-import toolz
+import tlz as toolz
 
 from distributed import Pub, Sub, wait, get_worker, TimeoutError
 from distributed.utils_test import gen_cluster
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index a5649dbfc82..5459716ca85 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -12,7 +12,7 @@
 
 import dask
 from dask import delayed
-from toolz import merge, concat, valmap, first, frequencies
+from tlz import merge, concat, valmap, first, frequencies
 from tornado import gen
 
 import pytest
diff --git a/distributed/tests/test_steal.py b/distributed/tests/test_steal.py
index 71f408749a1..5b13d9157e8 100644
--- a/distributed/tests/test_steal.py
+++ b/distributed/tests/test_steal.py
@@ -6,7 +6,7 @@
 import weakref
 
 import pytest
-from toolz import sliding_window, concat
+from tlz import sliding_window, concat
 from tornado import gen
 
 import dask
diff --git a/distributed/tests/test_stress.py b/distributed/tests/test_stress.py
index ab996e2b30d..d5e1e62c574 100644
--- a/distributed/tests/test_stress.py
+++ b/distributed/tests/test_stress.py
@@ -6,7 +6,7 @@
 
 from dask import delayed
 import pytest
-from toolz import concat, sliding_window
+from tlz import concat, sliding_window
 
 from distributed import Client, wait, Nanny
 from distributed.config import config
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index b6da294c749..0bda344fd96 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -16,7 +16,7 @@
 from dask.utils import format_bytes
 from dask.system import CPU_COUNT
 import pytest
-from toolz import pluck, sliding_window, first
+from tlz import pluck, sliding_window, first
 import tornado
 from tornado import gen
 
diff --git a/distributed/utils.py b/distributed/utils.py
index 429a53cddde..eb622f7b837 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -47,7 +47,7 @@
     parse_timedelta,
 )
 
-import toolz
+import tlz as toolz
 import tornado
 from tornado import gen
 from tornado.ioloop import IOLoop
diff --git a/distributed/utils_comm.py b/distributed/utils_comm.py
index 3d10ba51038..42404754527 100644
--- a/distributed/utils_comm.py
+++ b/distributed/utils_comm.py
@@ -8,7 +8,7 @@
 from dask.optimization import SubgraphCallable
 import dask.config
 from dask.utils import parse_timedelta
-from toolz import merge, concat, groupby, drop
+from tlz import merge, concat, groupby, drop
 
 from .core import rpc
 from .utils import All, tokey
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index e16983b1879..741fc76a8dd 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -32,7 +32,7 @@
 import pytest
 
 import dask
-from toolz import merge, memoize, assoc
+from tlz import merge, memoize, assoc
 from tornado import gen, queues
 from tornado.ioloop import IOLoop
 
diff --git a/distributed/variable.py b/distributed/variable.py
index fc4cc396dab..3c6cc931166 100644
--- a/distributed/variable.py
+++ b/distributed/variable.py
@@ -3,10 +3,7 @@
 import logging
 import uuid
 
-try:
-    from cytoolz import merge
-except ImportError:
-    from toolz import merge
+from tlz import merge
 
 from .client import Future, _get_global_client, Client
 from .utils import tokey, log_errors, TimeoutError, ignoring
diff --git a/distributed/worker.py b/distributed/worker.py
index aa71a16640b..247ffc99510 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -3,6 +3,7 @@
 from collections import defaultdict, deque, namedtuple
 from collections.abc import MutableMapping
 from datetime import timedelta
+from functools import partial
 import heapq
 from inspect import isawaitable
 import logging
@@ -21,10 +22,7 @@
 from dask.utils import format_bytes, funcname
 from dask.system import CPU_COUNT
 
-try:
-    from cytoolz import pluck, partial, merge, first, keymap
-except ImportError:
-    from toolz import pluck, partial, merge, first, keymap
+from tlz import pluck, merge, first, keymap
 from tornado import gen
 from tornado.ioloop import IOLoop
 
diff --git a/docs/source/efficiency.rst b/docs/source/efficiency.rst
index 94a603ea9a3..ed3ad2428d5 100644
--- a/docs/source/efficiency.rst
+++ b/docs/source/efficiency.rst
@@ -67,7 +67,7 @@ A common solution is to batch your input into larger chunks.
    >>> def f_many(chunk):
    ...     return [f(x) for x in chunk]
 
-   >>> from toolz import partition_all
+   >>> from tlz import partition_all
    >>> chunks = partition_all(1000000, seq)  # Collect into groups of size 1000
 
    >>> futures = client.map(f_many, chunks)
diff --git a/requirements.txt b/requirements.txt
index 3f827e250e6..4cb3ba60ae7 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -5,7 +5,7 @@ msgpack >= 0.6.0
 psutil >= 5.0
 sortedcontainers !=2.0.0, !=2.0.1
 tblib >= 1.6.0
-toolz >= 0.7.4
+toolz >= 0.8.2
 tornado >= 5;python_version<'3.8'
 tornado >= 6.0.3;python_version>='3.8'
 zict >= 0.1.3

From 511427b81f599105dc1eb4d2f35fd33aa249b4b8 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Tue, 17 Mar 2020 15:47:37 -0500
Subject: [PATCH 0724/1550] Add Python version to version check (#3567)

* Add Python to version checks

* Use dict for system info
---
 distributed/tests/test_versions.py | 17 +++++++++++++++++
 distributed/versions.py            | 26 ++++++++++++++------------
 2 files changed, 31 insertions(+), 12 deletions(-)

diff --git a/distributed/tests/test_versions.py b/distributed/tests/test_versions.py
index 25087df795a..ab3547820ca 100644
--- a/distributed/tests/test_versions.py
+++ b/distributed/tests/test_versions.py
@@ -1,4 +1,5 @@
 import re
+import uuid
 
 import pytest
 
@@ -117,3 +118,19 @@ async def test_version_warning_in_cluster(s, a, b):
         assert any(
             "0.0.0" in line.message and a.address in line.message for line in w.logs
         )
+
+
+@gen_cluster()
+async def test_python_version_mismatch_warning(s, a, b):
+    # Set random Python version for one worker
+    random_version = uuid.uuid4().hex
+    orig = s.workers[a.address].versions["host"]["python"] = random_version
+
+    with pytest.warns(None) as record:
+        async with Client(s.address, asynchronous=True) as client:
+            pass
+
+    assert record
+    assert any("python" in str(r.message) for r in record)
+    assert any(random_version in str(r.message) for r in record)
+    assert any(a.address in str(r.message) for r in record)
diff --git a/distributed/versions.py b/distributed/versions.py
index a7022c830f7..403d79f6aef 100644
--- a/distributed/versions.py
+++ b/distributed/versions.py
@@ -51,17 +51,17 @@ def get_versions(packages=None):
 
 def get_system_info():
     (sysname, nodename, release, version, machine, processor) = platform.uname()
-    host = [
-        ("python", "%d.%d.%d.%s.%s" % sys.version_info[:]),
-        ("python-bits", struct.calcsize("P") * 8),
-        ("OS", "%s" % sysname),
-        ("OS-release", "%s" % release),
-        ("machine", "%s" % machine),
-        ("processor", "%s" % processor),
-        ("byteorder", "%s" % sys.byteorder),
-        ("LC_ALL", "%s" % os.environ.get("LC_ALL", "None")),
-        ("LANG", "%s" % os.environ.get("LANG", "None")),
-    ]
+    host = {
+        "python": "%d.%d.%d.%s.%s" % sys.version_info[:],
+        "python-bits": struct.calcsize("P") * 8,
+        "OS": "%s" % sysname,
+        "OS-release": "%s" % release,
+        "machine": "%s" % machine,
+        "processor": "%s" % processor,
+        "byteorder": "%s" % sys.byteorder,
+        "LC_ALL": "%s" % os.environ.get("LC_ALL", "None"),
+        "LANG": "%s" % os.environ.get("LANG", "None"),
+    }
 
     return host
 
@@ -113,7 +113,6 @@ def error_message(scheduler, workers, client, client_name="client"):
 
     # Collect all package versions
     packages = set()
-
     for node, info in nodes.items():
         if info is None or not (isinstance(info, dict)) or "packages" not in info:
             node_packages[node] = defaultdict(lambda: "UNKNOWN")
@@ -122,6 +121,9 @@ def error_message(scheduler, workers, client, client_name="client"):
             for pkg, version in info["packages"].items():
                 node_packages[node][pkg] = version
                 packages.add(pkg)
+            # Collect Python version for each node
+            node_packages[node]["python"] = info["host"]["python"]
+            packages.add("python")
 
     errs = []
     for pkg in sorted(packages):

From 2acffc3172ec32e173547ee4c39a01b6c94e74a1 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 18 Mar 2020 18:04:05 -0700
Subject: [PATCH 0725/1550] Optionally compress on a frame-by-frame basis
 (#3586)

Previously this converted a list of bytes-like objects into a list.
Now we consume a single one and use map when dealing with lists.

* Handle compression on a frame-by-frame basis

* Set cuda serialization to False rather than None

We've changed the convention so that None now means "proceed as usual"
rather than "don't do anything please"
---
 distributed/protocol/core.py                 | 28 +++++++++-----
 distributed/protocol/cuda.py                 |  2 +-
 distributed/protocol/numpy.py                |  2 +-
 distributed/protocol/serialize.py            |  6 ++-
 distributed/protocol/tests/test_serialize.py | 14 +++++++
 distributed/protocol/utils.py                | 39 +++++++++-----------
 6 files changed, 57 insertions(+), 34 deletions(-)

diff --git a/distributed/protocol/core.py b/distributed/protocol/core.py
index 3bb863f78c2..0947b3a6292 100644
--- a/distributed/protocol/core.py
+++ b/distributed/protocol/core.py
@@ -48,17 +48,27 @@ def dumps(msg, serializers=None, on_error="message", context=None):
         for key, (head, frames) in data.items():
             if "lengths" not in head:
                 head["lengths"] = tuple(map(nbytes, frames))
-            if "compression" not in head:
-                frames = frame_split_size(frames)
-                if frames:
-                    compression, frames = zip(*map(maybe_compress, frames))
-                else:
-                    compression = []
-                head["compression"] = compression
-            head["count"] = len(frames)
+
+            # Compress frames that are not yet compressed
+            out_compression = []
+            _out_frames = []
+            for frame, compression in zip(
+                frames, head.get("compression") or [None] * len(frames)
+            ):
+                if compression is None:  # default behavior
+                    _frames = frame_split_size(frame)
+                    _compression, _frames = zip(*map(maybe_compress, _frames))
+                    out_compression.extend(_compression)
+                    _out_frames.extend(_frames)
+                else:  # already specified, so pass
+                    out_compression.append(compression)
+                    _out_frames.append(frame)
+
+            head["compression"] = out_compression
+            head["count"] = len(_out_frames)
             header["headers"][key] = head
             header["keys"].append(key)
-            out_frames.extend(frames)
+            out_frames.extend(_out_frames)
 
         for key, (head, frames) in pre.items():
             if "lengths" not in head:
diff --git a/distributed/protocol/cuda.py b/distributed/protocol/cuda.py
index aa638f70c0d..44ed6a033df 100644
--- a/distributed/protocol/cuda.py
+++ b/distributed/protocol/cuda.py
@@ -18,7 +18,7 @@ def cuda_dumps(x):
     header, frames = dumps(x)
     header["type-serialized"] = pickle.dumps(type(x))
     header["serializer"] = "cuda"
-    header["compression"] = (None,) * len(frames)  # no compression for gpu data
+    header["compression"] = (False,) * len(frames)  # no compression for gpu data
     return header, frames
 
 
diff --git a/distributed/protocol/numpy.py b/distributed/protocol/numpy.py
index a2c9c2933e6..a0df77c8b37 100644
--- a/distributed/protocol/numpy.py
+++ b/distributed/protocol/numpy.py
@@ -88,7 +88,7 @@ def serialize_numpy_ndarray(x):
         header["broadcast_to"] = broadcast_to
 
     if x.nbytes > 1e5:
-        frames = frame_split_size([data])
+        frames = frame_split_size(data)
     else:
         frames = [data]
 
diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index c0fdb98449a..6db7ca70c13 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -166,10 +166,12 @@ def serialize(x, serializers=None, on_error="message", context=None):
 
         frames = []
         lengths = []
+        compressions = []
         for _header, _frames in headers_frames:
             frames.extend(_frames)
             length = len(_frames)
             lengths.append(length)
+            compressions.extend(_header.get("compression") or [None] * len(_frames))
 
         headers = [obj[0] for obj in headers_frames]
         headers = {
@@ -178,6 +180,8 @@ def serialize(x, serializers=None, on_error="message", context=None):
             "frame-lengths": lengths,
             "type-serialized": type(x).__name__,
         }
+        if any(compression is not None for compression in compressions):
+            headers["compression"] = compressions
         return headers, frames
 
     tb = ""
@@ -436,7 +440,7 @@ def replace_inner(x):
 
 def serialize_bytelist(x, **kwargs):
     header, frames = serialize(x, **kwargs)
-    frames = frame_split_size(frames)
+    frames = sum(map(frame_split_size, frames), [])
     if frames:
         compression, frames = zip(*map(maybe_compress, frames))
     else:
diff --git a/distributed/protocol/tests/test_serialize.py b/distributed/protocol/tests/test_serialize.py
index 10e4c5e797d..41e2af51b70 100644
--- a/distributed/protocol/tests/test_serialize.py
+++ b/distributed/protocol/tests/test_serialize.py
@@ -374,3 +374,17 @@ async def test_profile_nested_sizeof():
 
     msg = {"data": original}
     frames = await to_frames(msg)
+
+
+def test_compression_numpy_list():
+    class MyObj:
+        pass
+
+    @dask_serialize.register(MyObj)
+    def _(x):
+        header = {"compression": [False]}
+        frames = [b""]
+        return header, frames
+
+    header, frames = serialize([MyObj(), MyObj()])
+    assert header["compression"] == [False, False]
diff --git a/distributed/protocol/utils.py b/distributed/protocol/utils.py
index e5b9247e77f..3af203881ff 100644
--- a/distributed/protocol/utils.py
+++ b/distributed/protocol/utils.py
@@ -19,9 +19,9 @@
     msgpack_opts["encoding"] = "utf-8"
 
 
-def frame_split_size(frames, n=BIG_BYTES_SHARD_SIZE):
+def frame_split_size(frame, n=BIG_BYTES_SHARD_SIZE) -> list:
     """
-    Split a list of frames into a list of frames of maximum size
+    Split a frame into a list of frames of maximum size
 
     This helps us to avoid passing around very large bytestrings.
 
@@ -30,26 +30,21 @@ def frame_split_size(frames, n=BIG_BYTES_SHARD_SIZE):
     >>> frame_split_size([b'12345', b'678'], n=3)  # doctest: +SKIP
     [b'123', b'45', b'678']
     """
-    if not frames:
-        return frames
-
-    if max(map(nbytes, frames)) <= n:
-        return frames
-
-    out = []
-    for frame in frames:
-        if nbytes(frame) > n:
-            if isinstance(frame, (bytes, bytearray)):
-                frame = memoryview(frame)
-            try:
-                itemsize = frame.itemsize
-            except AttributeError:
-                itemsize = 1
-            for i in range(0, nbytes(frame) // itemsize, n // itemsize):
-                out.append(frame[i : i + n // itemsize])
-        else:
-            out.append(frame)
-    return out
+    if nbytes(frame) <= n:
+        return [frame]
+
+    if nbytes(frame) > n:
+        if isinstance(frame, (bytes, bytearray)):
+            frame = memoryview(frame)
+        try:
+            itemsize = frame.itemsize
+        except AttributeError:
+            itemsize = 1
+
+        return [
+            frame[i : i + n // itemsize]
+            for i in range(0, nbytes(frame) // itemsize, n // itemsize)
+        ]
 
 
 def merge_frames(header, frames):

From 3c1bfa8838e385daf7e86910283969699d169d40 Mon Sep 17 00:00:00 2001
From: Julia Signell <jsignell@gmail.com>
Date: Thu, 19 Mar 2020 18:37:05 -0400
Subject: [PATCH 0726/1550] Change Adaptive docs to reference adaptive_target
 (#3597)

---
 distributed/deploy/adaptive.py | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/distributed/deploy/adaptive.py b/distributed/deploy/adaptive.py
index 0d295200018..d16f577168a 100644
--- a/distributed/deploy/adaptive.py
+++ b/distributed/deploy/adaptive.py
@@ -66,11 +66,11 @@ class Adaptive(AdaptiveCore):
 
     Notes
     -----
-    Subclasses can override :meth:`Adaptive.should_scale_up` and
+    Subclasses can override :meth:`Adaptive.target` and
     :meth:`Adaptive.workers_to_close` to control when the cluster should be
     resized. The default implementation checks if there are too many tasks
-    per worker or too little memory available (see :meth:`Adaptive.needs_cpu`
-    and :meth:`Adaptive.needs_memory`).
+    per worker or too little memory available (see
+    :meth:`Scheduler.adaptive_target`).
     '''
 
     def __init__(
@@ -110,6 +110,22 @@ def observed(self):
         return self.cluster.observed
 
     async def target(self):
+        """
+        Determine target number of workers that should exist.
+
+        Notes
+        -----
+        ``Adaptive.target`` dispatches to Scheduler.adaptive_target(),
+        but may be overridden in subclasses.
+
+        Returns
+        -------
+        Target number of workers
+
+        See Also
+        --------
+        Scheduler.adaptive_target
+        """
         return await self.scheduler.adaptive_target(
             target_duration=self.target_duration
         )

From 700fa17913e3f7ad8906a8468f6e9cb680746b9f Mon Sep 17 00:00:00 2001
From: Gabriel Sailer <sublinus@users.noreply.github.com>
Date: Fri, 20 Mar 2020 02:15:05 +0100
Subject: [PATCH 0727/1550] Add configuration for Adaptive arguments (#3509)

---
 distributed/deploy/adaptive.py            | 26 +++++++++++++++++------
 distributed/deploy/tests/test_adaptive.py | 12 +++++++++++
 distributed/distributed.yaml              |  8 ++++++-
 distributed/scheduler.py                  |  4 +++-
 4 files changed, 42 insertions(+), 8 deletions(-)

diff --git a/distributed/deploy/adaptive.py b/distributed/deploy/adaptive.py
index d16f577168a..1c53155de15 100644
--- a/distributed/deploy/adaptive.py
+++ b/distributed/deploy/adaptive.py
@@ -1,6 +1,6 @@
 from inspect import isawaitable
 import logging
-import math
+import dask.config
 
 from .adaptive_core import AdaptiveCore
 from ..utils import log_errors, parse_timedelta
@@ -71,22 +71,36 @@ class Adaptive(AdaptiveCore):
     resized. The default implementation checks if there are too many tasks
     per worker or too little memory available (see
     :meth:`Scheduler.adaptive_target`).
+    The values for interval, min, max, wait_count and target_duration can be
+    specified in the dask config under the distributed.adaptive key.
     '''
 
     def __init__(
         self,
         cluster=None,
-        interval="1s",
-        minimum=0,
-        maximum=math.inf,
-        wait_count=3,
-        target_duration="5s",
+        interval=None,
+        minimum=None,
+        maximum=None,
+        wait_count=None,
+        target_duration=None,
         worker_key=None,
         **kwargs
     ):
         self.cluster = cluster
         self.worker_key = worker_key
         self._workers_to_close_kwargs = kwargs
+
+        if interval is None:
+            interval = dask.config.get("distributed.adaptive.interval")
+        if minimum is None:
+            minimum = dask.config.get("distributed.adaptive.minimum")
+        if maximum is None:
+            maximum = dask.config.get("distributed.adaptive.maximum")
+        if wait_count is None:
+            wait_count = dask.config.get("distributed.adaptive.wait-count")
+        if target_duration is None:
+            target_duration = dask.config.get("distributed.adaptive.target-duration")
+
         self.target_duration = parse_timedelta(target_duration)
 
         super().__init__(
diff --git a/distributed/deploy/tests/test_adaptive.py b/distributed/deploy/tests/test_adaptive.py
index 2eddeeceff8..9c68e6ddf53 100644
--- a/distributed/deploy/tests/test_adaptive.py
+++ b/distributed/deploy/tests/test_adaptive.py
@@ -1,3 +1,4 @@
+import math
 from time import sleep
 
 import dask
@@ -415,3 +416,14 @@ async def test_adapt_cores_memory(cleanup):
         )
         assert adapt.minimum == 3
         assert adapt.maximum == 5
+
+
+def test_adaptive_config():
+    with dask.config.set(
+        {"distributed.adaptive.minimum": 10, "distributed.adaptive.wait-count": 8}
+    ):
+        adapt = Adaptive(interval="5s")
+        assert adapt.minimum == 10
+        assert adapt.maximum == math.inf
+        assert adapt.interval == 5
+        assert adapt.wait_count == 8
diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index 05f27604328..311eeaae829 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -74,6 +74,13 @@ distributed:
   deploy:
     lost-worker-timeout: 15s  # Interval after which to hard-close a lost worker job
 
+  adaptive:
+    interval: 1s         # Interval between scaling evaluations
+    target-duration: 5s  # Time an entire graph calculation is desired to take ("1m", "30m")
+    minimum: 0           # Minimum number of workers
+    maximum: .inf        # Maximum number of workers
+    wait-count: 3        # Number of times a worker should be suggested for removal before removing it
+
   comm:
     retry:  # some operations (such as gathering data) are subject to re-tries with the below parameters
       count: 0  # the maximum retry attempts. 0 disables re-trying.
@@ -141,4 +148,3 @@ ucx:
   infiniband: null # enable Infiniband
   cuda_copy: null  # enable cuda-copy
   net-devices: null  # define which Infiniband device to use
-
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index d543808340d..8a61ba31fca 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -5206,7 +5206,7 @@ def check_idle(self):
         if close:
             self.loop.add_callback(self.close)
 
-    def adaptive_target(self, comm=None, target_duration="5s"):
+    def adaptive_target(self, comm=None, target_duration=None):
         """ Desired number of workers based on the current workload
 
         This looks at the current running tasks and memory use, and returns a
@@ -5222,6 +5222,8 @@ def adaptive_target(self, comm=None, target_duration="5s"):
         --------
         distributed.deploy.Adaptive
         """
+        if target_duration is None:
+            target_duration = dask.config.get("distributed.adaptive.target-duration")
         target_duration = parse_timedelta(target_duration)
 
         # CPU

From 0d64f3a3c2f72543420b6f2967e8e789ad265a27 Mon Sep 17 00:00:00 2001
From: Peter Andreas Entschev <peter@entschev.com>
Date: Fri, 20 Mar 2020 03:10:06 +0100
Subject: [PATCH 0728/1550] Synchronize default CUDA stream before UCX
 send/recv (#3598)

* Synchronize default CUDA stream before UCX send/recv

* Add more clarity on UCX.write comment

Co-Authored-By: Mark Harris <mharris@nvidia.com>

* Add more clarity on UCX.read comment

Co-Authored-By: Mark Harris <mharris@nvidia.com>

Co-authored-by: Mark Harris <mharris@nvidia.com>
---
 distributed/comm/ucx.py | 26 +++++++++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)

diff --git a/distributed/comm/ucx.py b/distributed/comm/ucx.py
index 7295b11bb48..04eecdf4482 100644
--- a/distributed/comm/ucx.py
+++ b/distributed/comm/ucx.py
@@ -35,6 +35,15 @@
 cuda_array = None
 
 
+def synchronize_stream(stream=0):
+    import numba.cuda
+
+    ctx = numba.cuda.current_context()
+    cu_stream = numba.cuda.driver.drvapi.cu_stream(stream)
+    stream = numba.cuda.driver.Stream(ctx, cu_stream, None)
+    stream.synchronize()
+
+
 def init_once():
     global ucp, cuda_array
     if ucp is not None:
@@ -160,6 +169,14 @@ async def write(
                     np.array([nbytes(f) for f in frames], dtype=np.uint64)
                 )
                 # Send frames
+
+                # It is necessary to first synchronize the default stream before start sending
+                # We synchronize the default stream because UCX is not stream-ordered and
+                #  syncing the default stream will wait for other non-blocking CUDA streams.
+                # Note this is only sufficient if the memory being sent is not currently in use on
+                # non-blocking CUDA streams. 
+                synchronize_stream(0)
+
                 for frame in frames:
                     if nbytes(frame) > 0:
                         await self.ep.send(frame)
@@ -196,13 +213,20 @@ async def read(self, deserializers=("cuda", "dask", "pickle", "error")):
                             frame = cuda_array(size)
                         else:
                             frame = np.empty(size, dtype=np.uint8)
-                        await self.ep.recv(frame)
                         frames.append(frame)
                     else:
                         if is_cuda:
                             frames.append(cuda_array(size))
                         else:
                             frames.append(b"")
+
+                # It is necessary to first populate `frames` with CUDA arrays and synchronize
+                # the default stream before starting receiving to ensure buffers have been allocated
+                synchronize_stream(0)
+                for i, (is_cuda, size) in enumerate(zip(is_cudas.tolist(), sizes.tolist())):
+                    if size > 0:
+                        await self.ep.recv(frames[i])
+
                 msg = await from_frames(
                     frames, deserialize=self.deserialize, deserializers=deserializers
                 )

From e928cc0090015a7e07c18d2d64255d33a849224a Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Fri, 20 Mar 2020 10:36:24 -0500
Subject: [PATCH 0729/1550] Fix linting errors (#3604)

---
 distributed/comm/ucx.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/distributed/comm/ucx.py b/distributed/comm/ucx.py
index 04eecdf4482..22d7e361e97 100644
--- a/distributed/comm/ucx.py
+++ b/distributed/comm/ucx.py
@@ -174,7 +174,7 @@ async def write(
                 # We synchronize the default stream because UCX is not stream-ordered and
                 #  syncing the default stream will wait for other non-blocking CUDA streams.
                 # Note this is only sufficient if the memory being sent is not currently in use on
-                # non-blocking CUDA streams. 
+                # non-blocking CUDA streams.
                 synchronize_stream(0)
 
                 for frame in frames:
@@ -223,7 +223,9 @@ async def read(self, deserializers=("cuda", "dask", "pickle", "error")):
                 # It is necessary to first populate `frames` with CUDA arrays and synchronize
                 # the default stream before starting receiving to ensure buffers have been allocated
                 synchronize_stream(0)
-                for i, (is_cuda, size) in enumerate(zip(is_cudas.tolist(), sizes.tolist())):
+                for i, (is_cuda, size) in enumerate(
+                    zip(is_cudas.tolist(), sizes.tolist())
+                ):
                     if size > 0:
                         await self.ep.recv(frames[i])
 

From 6adf0696601fc575978e4f1c3ea6805cf323283a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lo=C3=AFc=20Est=C3=A8ve?= <loic.esteve@ymail.com>
Date: Fri, 20 Mar 2020 19:32:16 +0100
Subject: [PATCH 0730/1550] Remove dill from CI environments. (#3608)

---
 continuous_integration/environment.yml   | 1 -
 continuous_integration/travis/install.sh | 1 -
 2 files changed, 2 deletions(-)

diff --git a/continuous_integration/environment.yml b/continuous_integration/environment.yml
index 8218d721e85..7458c9f64ac 100644
--- a/continuous_integration/environment.yml
+++ b/continuous_integration/environment.yml
@@ -7,7 +7,6 @@ dependencies:
   - click
   - cloudpickle
   - dask
-  - dill
   - lz4
   - ipykernel
   - ipywidgets
diff --git a/continuous_integration/travis/install.sh b/continuous_integration/travis/install.sh
index 68b842aa033..4ee0790f6c5 100644
--- a/continuous_integration/travis/install.sh
+++ b/continuous_integration/travis/install.sh
@@ -33,7 +33,6 @@ conda install -c conda-forge -q \
     click \
     coverage \
     dask \
-    dill \
     flake8 \
     h5py \
     ipykernel \

From 5f1c6bcdea47c2d8bd67cc58cfe6c29ff4fab9ef Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Fri, 20 Mar 2020 14:20:38 -0500
Subject: [PATCH 0731/1550] Replace tornado.queues with asyncio.queues (#3607)

---
 distributed/comm/tests/test_comms.py | 10 +++++-----
 distributed/utils_test.py            |  4 ++--
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/distributed/comm/tests/test_comms.py b/distributed/comm/tests/test_comms.py
index 150251f3d59..16036184755 100644
--- a/distributed/comm/tests/test_comms.py
+++ b/distributed/comm/tests/test_comms.py
@@ -9,7 +9,7 @@
 import pkg_resources
 import pytest
 
-from tornado import ioloop, queues
+from tornado import ioloop
 from tornado.concurrent import Future
 
 import distributed
@@ -78,10 +78,10 @@ def check_tls_extra(info):
 
 @pytest.mark.asyncio
 async def get_comm_pair(listen_addr, listen_args=None, connect_args=None, **kwargs):
-    q = queues.Queue()
+    q = asyncio.Queue()
 
-    def handle_comm(comm):
-        q.put(comm)
+    async def handle_comm(comm):
+        await q.put(comm)
 
     listener = listen(listen_addr, handle_comm, connection_args=listen_args, **kwargs)
     await listener.start()
@@ -883,7 +883,7 @@ async def test_inproc_many_listeners():
 
 
 async def check_listener_deserialize(addr, deserialize, in_value, check_out):
-    q = queues.Queue()
+    q = asyncio.Queue()
 
     async def handle_comm(comm):
         msg = await comm.read()
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index 741fc76a8dd..b521826647a 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -33,7 +33,7 @@
 
 import dask
 from tlz import merge, memoize, assoc
-from tornado import gen, queues
+from tornado import gen
 from tornado.ioloop import IOLoop
 
 from . import system
@@ -429,7 +429,7 @@ async def readone(comm):
     try:
         q = _readone_queues[comm]
     except KeyError:
-        q = _readone_queues[comm] = queues.Queue()
+        q = _readone_queues[comm] = asyncio.Queue()
 
         async def background_read():
             while True:

From 51428dc5f6f4181157e819d628d5f8dd43de3217 Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Sat, 21 Mar 2020 01:02:51 +0000
Subject: [PATCH 0732/1550] Pin openssl to 1.1.1d for Travis (#3602)

---
 continuous_integration/travis/install.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/continuous_integration/travis/install.sh b/continuous_integration/travis/install.sh
index 4ee0790f6c5..09d13962bd4 100644
--- a/continuous_integration/travis/install.sh
+++ b/continuous_integration/travis/install.sh
@@ -79,6 +79,9 @@ if [[ $CRICK == true ]]; then
     python -m pip install -q git+https://github.com/jcrist/crick.git
 fi;
 
+# Pin openssl==1.1.1d (see https://github.com/dask/distributed/issues/3588)
+conda install -c conda-forge openssl==1.1.1d
+
 # Install distributed
 python -m pip install --no-deps -e .
 

From 3f03e1cf851ba1950cff6de8e17e7749cc3569d3 Mon Sep 17 00:00:00 2001
From: Scott Sievert <stsievert@users.noreply.github.com>
Date: Sat, 21 Mar 2020 02:55:06 +0000
Subject: [PATCH 0733/1550] Increase number of visible mantissas in dashboard
 plots (#3585)

---
 distributed/dashboard/components/nvml.py      |  4 ++--
 distributed/dashboard/components/scheduler.py | 13 ++++++-------
 2 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/distributed/dashboard/components/nvml.py b/distributed/dashboard/components/nvml.py
index b0c56c4ef47..00346e81e66 100644
--- a/distributed/dashboard/components/nvml.py
+++ b/distributed/dashboard/components/nvml.py
@@ -14,7 +14,7 @@
 from tornado import escape
 from dask.utils import format_bytes
 from distributed.utils import log_errors
-from distributed.dashboard.components.scheduler import BOKEH_THEME
+from distributed.dashboard.components.scheduler import BOKEH_THEME, TICKS_1024
 from distributed.dashboard.utils import without_property_validation, update
 
 
@@ -83,7 +83,7 @@ def __init__(self, scheduler, width=600, **kwargs):
             )
             rect.nonselection_glyph = None
 
-            memory.axis[0].ticker = BasicTicker(mantissas=[1, 256, 512], base=1024)
+            memory.axis[0].ticker = BasicTicker(**TICKS_1024)
             memory.xaxis[0].formatter = NumeralTickFormatter(format="0.0 b")
             memory.xaxis.major_label_orientation = -math.pi / 12
             memory.x_range.start = 0
diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index c371210c701..9519d3629ff 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -78,6 +78,7 @@
 )
 
 BOKEH_THEME = Theme(os.path.join(os.path.dirname(__file__), "..", "theme.yaml"))
+TICKS_1024 = {"base": 1024, "mantissas": [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]}
 
 nan = float("nan")
 inf = float("inf")
@@ -233,7 +234,7 @@ def __init__(self, scheduler, **kwargs):
             )
 
             self.root.xaxis[0].formatter = NumeralTickFormatter(format="0.0 b")
-            self.root.xaxis.ticker = AdaptiveTicker(mantissas=[1, 256, 512], base=1024)
+            self.root.xaxis.ticker = AdaptiveTicker(**TICKS_1024)
             self.root.xaxis.major_label_orientation = -math.pi / 12
 
             self.root.xaxis.minor_tick_line_alpha = 0
@@ -296,7 +297,7 @@ def __init__(self, scheduler, **kwargs):
             )
             fig.x_range.start = 0
             fig.xaxis[0].formatter = NumeralTickFormatter(format="0.0 b")
-            fig.xaxis.ticker = AdaptiveTicker(mantissas=[1, 256, 512], base=1024)
+            fig.xaxis.ticker = AdaptiveTicker(**TICKS_1024)
             rect.nonselection_glyph = None
 
             fig.xaxis.minor_tick_line_alpha = 0
@@ -379,9 +380,7 @@ def __init__(self, scheduler, **kwargs):
                 location=(0, 0),
             )
             color_bar.formatter = NumeralTickFormatter(format="0.0 b")
-            color_bar.ticker = AdaptiveTicker(
-                mantissas=[1, 64, 128, 256, 512], base=1024
-            )
+            color_bar.ticker = AdaptiveTicker(**TICKS_1024)
             fig.add_layout(color_bar, "right")
 
             fig.toolbar.logo = None
@@ -464,7 +463,7 @@ def __init__(self, scheduler, **kwargs):
                 source=self.source, x="name", top="nbytes", width=0.9, color="color"
             )
             fig.yaxis[0].formatter = NumeralTickFormatter(format="0.0 b")
-            fig.yaxis.ticker = AdaptiveTicker(mantissas=[1, 256, 512], base=1024)
+            fig.yaxis.ticker = AdaptiveTicker(**TICKS_1024)
             fig.xaxis.major_label_orientation = -math.pi / 12
             rect.nonselection_glyph = None
 
@@ -593,7 +592,7 @@ def __init__(self, scheduler, width=600, **kwargs):
             )
             rect.nonselection_glyph = None
 
-            nbytes.axis[0].ticker = BasicTicker(mantissas=[1, 256, 512], base=1024)
+            nbytes.axis[0].ticker = BasicTicker(**TICKS_1024)
             nbytes.xaxis[0].formatter = NumeralTickFormatter(format="0.0 b")
             nbytes.xaxis.major_label_orientation = -math.pi / 12
             nbytes.x_range.start = 0

From 2a795df67d3521dbfbf49a650238bab7266a86ee Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sat, 21 Mar 2020 13:09:07 -0700
Subject: [PATCH 0734/1550] Make Listeners awaitable (#3611)

---
 distributed/comm/core.py             |  7 ++++++
 distributed/comm/tests/test_comms.py | 34 ++++++++++++----------------
 distributed/comm/tests/test_ucx.py   |  3 +--
 distributed/core.py                  |  3 +--
 distributed/tests/test_batched.py    |  3 +--
 5 files changed, 24 insertions(+), 26 deletions(-)

diff --git a/distributed/comm/core.py b/distributed/comm/core.py
index e801242bb40..b4c93644a2c 100644
--- a/distributed/comm/core.py
+++ b/distributed/comm/core.py
@@ -162,6 +162,13 @@ async def __aenter__(self):
     async def __aexit__(self, *exc):
         self.stop()
 
+    def __await__(self):
+        async def _():
+            await self.start()
+            return self
+
+        return _().__await__()
+
 
 class Connector(ABC):
     @abstractmethod
diff --git a/distributed/comm/tests/test_comms.py b/distributed/comm/tests/test_comms.py
index 16036184755..2e5602a9ac5 100644
--- a/distributed/comm/tests/test_comms.py
+++ b/distributed/comm/tests/test_comms.py
@@ -83,8 +83,9 @@ async def get_comm_pair(listen_addr, listen_args=None, connect_args=None, **kwar
     async def handle_comm(comm):
         await q.put(comm)
 
-    listener = listen(listen_addr, handle_comm, connection_args=listen_args, **kwargs)
-    await listener.start()
+    listener = await listen(
+        listen_addr, handle_comm, connection_args=listen_args, **kwargs
+    )
 
     comm = await connect(
         listener.contact_address, connection_args=connect_args, **kwargs
@@ -221,8 +222,7 @@ async def handle_comm(comm):
         await comm.write(msg)
         await comm.close()
 
-    listener = tcp.TCPListener("localhost", handle_comm)
-    await listener.start()
+    listener = await tcp.TCPListener("localhost", handle_comm)
     host, port = listener.get_host_port()
     assert host in ("localhost", "127.0.0.1", "::1")
     assert port > 0
@@ -269,8 +269,7 @@ async def handle_comm(comm):
     server_ctx = get_server_ssl_context()
     client_ctx = get_client_ssl_context()
 
-    listener = tcp.TLSListener("localhost", handle_comm, ssl_context=server_ctx)
-    await listener.start()
+    listener = await tcp.TLSListener("localhost", handle_comm, ssl_context=server_ctx)
     host, port = listener.get_host_port()
     assert host in ("localhost", "127.0.0.1", "::1")
     assert port > 0
@@ -361,8 +360,7 @@ async def handle_comm(comm):
             await comm.write(msg)
         await comm.close()
 
-    listener = inproc.InProcListener(listener_addr, handle_comm)
-    await listener.start()
+    listener = await inproc.InProcListener(listener_addr, handle_comm)
     assert (
         listener.listen_address
         == listener.contact_address
@@ -468,8 +466,7 @@ async def handle_comm(comm):
     listen_args = listen_args or {"xxx": "bar"}
     connect_args = connect_args or {"xxx": "foo"}
 
-    listener = listen(addr, handle_comm, connection_args=listen_args)
-    await listener.start()
+    listener = await listen(addr, handle_comm, connection_args=listen_args)
 
     # Check listener properties
     bound_addr = listener.listen_address
@@ -647,8 +644,9 @@ async def handle_comm(comm):
         await comm.close()
 
     # Listener refuses a connector not signed by the CA
-    listener = listen("tls://", handle_comm, connection_args={"ssl_context": serv_ctx})
-    await listener.start()
+    listener = await listen(
+        "tls://", handle_comm, connection_args={"ssl_context": serv_ctx}
+    )
 
     with pytest.raises(EnvironmentError) as excinfo:
         comm = await connect(
@@ -678,10 +676,9 @@ async def handle_comm(comm):
     await comm.close()
 
     # Connector refuses a listener not signed by the CA
-    listener = listen(
+    listener = await listen(
         "tls://", handle_comm, connection_args={"ssl_context": bad_serv_ctx}
     )
-    await listener.start()
 
     with pytest.raises(EnvironmentError) as excinfo:
         await connect(
@@ -705,8 +702,7 @@ async def check_comm_closed_implicit(
     async def handle_comm(comm):
         await comm.close()
 
-    listener = listen(addr, handle_comm, connection_args=listen_args)
-    await listener.start()
+    listener = await listen(addr, handle_comm, connection_args=listen_args)
     contact_addr = listener.contact_address
 
     comm = await connect(contact_addr, connection_args=connect_args)
@@ -785,8 +781,7 @@ async def handle_comm(comm):
         else:
             await comm.close()
 
-    listener = listen("inproc://", handle_comm)
-    await listener.start()
+    listener = await listen("inproc://", handle_comm)
     contact_addr = listener.contact_address
 
     comm = await connect(contact_addr)
@@ -854,8 +849,7 @@ async def handle_comm(comm):
     N = 100
 
     for i in range(N):
-        listener = listen(addr, handle_comm)
-        await listener.start()
+        listener = await listen(addr, handle_comm)
         listeners.append(listener)
 
     assert len(set(l.listen_address for l in listeners)) == N
diff --git a/distributed/comm/tests/test_ucx.py b/distributed/comm/tests/test_ucx.py
index ead799f8158..84da6e4f1aa 100644
--- a/distributed/comm/tests/test_ucx.py
+++ b/distributed/comm/tests/test_ucx.py
@@ -96,8 +96,7 @@ async def handle_comm(comm):
             await comm.close()
             assert comm.closed
 
-        listener = ucx.UCXListener(address, handle_comm)
-        await listener.start()
+        listener = await ucx.UCXListener(address, handle_comm)
         host, port = listener.get_host_port()
         assert host.count(".") == 3
         assert port > 0
diff --git a/distributed/core.py b/distributed/core.py
index ec1e6c5214c..dd40fa7a4d0 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -312,13 +312,12 @@ async def listen(self, port_or_addr=None, listen_args=None):
         else:
             addr = port_or_addr
             assert isinstance(addr, str)
-        listener = listen(
+        listener = await listen(
             addr,
             self.handle_comm,
             deserialize=self.deserialize,
             connection_args=listen_args,
         )
-        await listener.start()
         self.listeners.append(listener)
 
     async def handle_comm(self, comm, shutting_down=shutting_down):
diff --git a/distributed/tests/test_batched.py b/distributed/tests/test_batched.py
index f2b0be99ab0..a288a25bbb9 100644
--- a/distributed/tests/test_batched.py
+++ b/distributed/tests/test_batched.py
@@ -25,8 +25,7 @@ async def handle_comm(self, comm):
                 return
 
     async def listen(self):
-        listener = listen("", self.handle_comm)
-        await listener.start()
+        listener = await listen("", self.handle_comm)
         self.address = listener.contact_address
         self.stop = listener.stop
 

From 7deb3b059f9ed335e2c405d0fd801c806215f9f9 Mon Sep 17 00:00:00 2001
From: Matthias Urlichs <matthias@urlichs.de>
Date: Sun, 22 Mar 2020 20:46:05 +0100
Subject: [PATCH 0735/1550] Add backoff to comm connect attempts. (#3496)

Closes #3487
---
 distributed/comm/core.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/distributed/comm/core.py b/distributed/comm/core.py
index b4c93644a2c..26ecdd1c54f 100644
--- a/distributed/comm/core.py
+++ b/distributed/comm/core.py
@@ -209,6 +209,10 @@ def _raise(error):
         )
         raise IOError(msg)
 
+    backoff = 0.01
+    if timeout and timeout / 20 < backoff:
+        backoff = timeout / 20
+
     # This starts a thread
     while True:
         try:
@@ -228,8 +232,10 @@ def _raise(error):
         except EnvironmentError as e:
             error = str(e)
             if time() < deadline:
-                await asyncio.sleep(0.01)
-                logger.debug("sleeping on connect")
+                logger.debug("Could not connect, waiting before retrying")
+                await asyncio.sleep(backoff)
+                backoff *= 1.5
+                backoff = min(backoff, 1)  # wait at most one second
             else:
                 _raise(error)
         else:

From e1f871a3a807ef4c86f6cb8909c41cf7ec9d429e Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 23 Mar 2020 12:44:43 -0700
Subject: [PATCH 0736/1550] Add str/repr methods to as_completed (#3618)

---
 distributed/client.py                  |  5 +++++
 distributed/tests/test_as_completed.py | 19 +++++++++++++++++++
 2 files changed, 24 insertions(+)

diff --git a/distributed/client.py b/distributed/client.py
index 06c6d245c07..24f25a79a09 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -4281,6 +4281,11 @@ def count(self):
         with self.lock:
             return len(self.futures) + len(self.queue.queue)
 
+    def __repr__(self):
+        return "<as_completed: waiting={} done={}>".format(
+            len(self.futures), len(self.queue.queue)
+        )
+
     def __iter__(self):
         return self
 
diff --git a/distributed/tests/test_as_completed.py b/distributed/tests/test_as_completed.py
index d0249b121d6..c9780d196cd 100644
--- a/distributed/tests/test_as_completed.py
+++ b/distributed/tests/test_as_completed.py
@@ -1,3 +1,4 @@
+import asyncio
 from collections.abc import Iterator
 from operator import add
 import queue
@@ -8,6 +9,7 @@
 from tornado import gen
 
 from distributed.client import _as_completed, as_completed, _first_completed
+from distributed.metrics import time
 from distributed.utils import CancelledError
 from distributed.utils_test import gen_cluster, inc, throws
 from distributed.utils_test import client, cluster_fixture, loop  # noqa: F401
@@ -232,6 +234,23 @@ def test_as_completed_with_results_no_raise(client):
     assert dd[z][0] == 2
 
 
+@gen_cluster(client=True)
+async def test_str(c, s, a, b):
+    futures = c.map(inc, range(3))
+    ac = as_completed(futures)
+    assert "waiting=3" in str(ac)
+    assert "waiting=3" in repr(ac)
+    assert "done=0" in str(ac)
+    assert "done=0" in repr(ac)
+
+    await ac.__anext__()
+
+    start = time()
+    while "done=2" not in str(ac):
+        await asyncio.sleep(0.01)
+        assert time() < start + 2
+
+
 @gen_cluster(client=True)
 def test_as_completed_with_results_no_raise_async(c, s, a, b):
     x = c.submit(throws, 1)

From e22f2fbf2d3fbe4a79c218d2b487bb0fc4ebc12f Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 23 Mar 2020 13:26:36 -0700
Subject: [PATCH 0737/1550] Support async Listener.stop functions (#3613)

---
 distributed/comm/core.py | 5 ++++-
 distributed/core.py      | 8 +++++---
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/distributed/comm/core.py b/distributed/comm/core.py
index 26ecdd1c54f..6ef26568853 100644
--- a/distributed/comm/core.py
+++ b/distributed/comm/core.py
@@ -1,5 +1,6 @@
 from abc import ABC, abstractmethod, abstractproperty
 import asyncio
+import inspect
 import logging
 import weakref
 
@@ -160,7 +161,9 @@ async def __aenter__(self):
         return self
 
     async def __aexit__(self, *exc):
-        self.stop()
+        future = self.stop()
+        if inspect.isawaitable(future):
+            await future
 
     def __await__(self):
         async def _():
diff --git a/distributed/core.py b/distributed/core.py
index dd40fa7a4d0..5bff3276e99 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -1,7 +1,7 @@
 import asyncio
 from collections import defaultdict, deque
 from functools import partial
-from inspect import isawaitable
+import inspect
 import logging
 import threading
 import traceback
@@ -405,7 +405,7 @@ async def handle_comm(self, comm, shutting_down=shutting_down):
                     logger.debug("Calling into handler %s", handler.__name__)
                     try:
                         result = handler(comm, **msg)
-                        if isawaitable(result):
+                        if inspect.isawaitable(result):
                             result = asyncio.ensure_future(result)
                             self._ongoing_coroutines.add(result)
                             result = await result
@@ -495,7 +495,9 @@ def close(self):
         for pc in self.periodic_callbacks.values():
             pc.stop()
         for listener in self.listeners:
-            self.listener.stop()
+            future = self.listener.stop()
+            if inspect.isawaitable(future):
+                yield future
         for i in range(20):  # let comms close naturally for a second
             if not self._comms:
                 break

From 4d4d935f46619ad9d96d64190274201ebe894eec Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Mon, 23 Mar 2020 21:46:05 +0100
Subject: [PATCH 0738/1550] Ensure that we don't steal blacklisted fast tasks
 (#3591)

---
 distributed/stealing.py         |  4 +-
 distributed/tests/test_steal.py | 69 +++++++++++++++++++++++++--------
 2 files changed, 54 insertions(+), 19 deletions(-)

diff --git a/distributed/stealing.py b/distributed/stealing.py
index fcceba4824a..38524f1722c 100644
--- a/distributed/stealing.py
+++ b/distributed/stealing.py
@@ -84,7 +84,7 @@ def transition(
         if start == "processing":
             self.remove_key_from_stealable(ts)
             if finish == "memory":
-                for tts in self.stealable_unknown_durations.pop(ts.prefix, ()):
+                for tts in self.stealable_unknown_durations.pop(ts.prefix.name, ()):
                     if tts not in self.in_flight and tts.state == "processing":
                         self.put_key_in_stealable(tts)
             else:
@@ -132,7 +132,7 @@ def steal_time_ratio(self, ts):
         nbytes = sum(dep.get_nbytes() for dep in ts.dependencies)
 
         transfer_time = nbytes / self.scheduler.bandwidth + LATENCY
-        split = ts.prefix
+        split = ts.prefix.name
         if split in fast_tasks:
             return None, None
         ws = ts.processing_on
diff --git a/distributed/tests/test_steal.py b/distributed/tests/test_steal.py
index 5b13d9157e8..0ed9051cc95 100644
--- a/distributed/tests/test_steal.py
+++ b/distributed/tests/test_steal.py
@@ -1,32 +1,29 @@
 import itertools
-from operator import mul
 import random
 import sys
-from time import sleep
 import weakref
-
-import pytest
-from tlz import sliding_window, concat
-from tornado import gen
+from operator import mul
+from time import sleep
 
 import dask
+import pytest
 from distributed import Nanny, Worker, wait, worker_client
 from distributed.config import config
 from distributed.metrics import time
 from distributed.scheduler import key_split
 from distributed.system import MEMORY_LIMIT
 from distributed.utils_test import (
-    slowinc,
-    slowadd,
-    inc,
+    captured_logger,
     gen_cluster,
+    inc,
+    nodebug_setup_module,
+    nodebug_teardown_module,
+    slowadd,
     slowidentity,
-    captured_logger,
+    slowinc,
 )
-from distributed.utils_test import nodebug_setup_module, nodebug_teardown_module
-
-import pytest
-
+from tlz import concat, sliding_window
+from tornado import gen
 
 # Most tests here are timing-dependent
 setup_module = nodebug_setup_module
@@ -145,23 +142,61 @@ def test_steal_related_tasks(e, s, a, b, c):
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 10, timeout=1000)
-def test_dont_steal_fast_tasks(c, s, *workers):
+async def test_dont_steal_fast_tasks_compute_time(c, s, *workers):
     np = pytest.importorskip("numpy")
     x = c.submit(np.random.random, 10000000, workers=workers[0].address)
 
     def do_nothing(x, y=None):
         pass
 
-    yield wait(c.submit(do_nothing, 1))
+    # execute and meassure runtime once
+    await wait(c.submit(do_nothing, 1))
 
     futures = c.map(do_nothing, range(1000), y=x)
 
-    yield wait(futures)
+    await wait(futures)
 
     assert len(s.who_has[x.key]) == 1
     assert len(s.has_what[workers[0].address]) == 1001
 
 
+@gen_cluster(client=True)
+async def test_dont_steal_fast_tasks_blacklist(c, s, a, b):
+    # create a dependency
+    x = c.submit(slowinc, 1, workers=[b.address])
+
+    # If the blacklist of fast tasks is tracked somewhere else, this needs to be
+    # changed. This test requies *any* key which is blacklisted.
+    from distributed.stealing import fast_tasks
+
+    blacklisted_key = next(iter(fast_tasks))
+
+    def fast_blacklisted(x, y=None):
+        # The task should observe a certain computation time such that we can
+        # ensure that it is not stolen due to the blacklisting. If it is too
+        # fast, the standard mechansim shouldn't allow stealing
+        import time
+
+        time.sleep(0.01)
+
+    futures = c.map(
+        fast_blacklisted,
+        range(100),
+        y=x,
+        # Submit the task to one worker but allow it to be distributed else,
+        # i.e. this is not a task restriction
+        workers=[a.address],
+        allow_other_workers=True,
+        key=blacklisted_key,
+    )
+
+    await wait(futures)
+
+    # The +1 is the dependency we initially submitted to worker B
+    assert len(s.has_what[a.address]) == 101
+    assert len(s.has_what[b.address]) == 1
+
+
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)], timeout=20)
 def test_new_worker_steals(c, s, a):
     yield wait(c.submit(slowinc, 1, delay=0.01))

From 706de86a255671eecbff5b03f0b4b8db456b1823 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Tue, 24 Mar 2020 09:59:33 -0700
Subject: [PATCH 0739/1550] Check `nbytes` and `types` before reading `data`
 (#3628)

* Check `nbytes` and `types` before reading `data`

To avoid reading `data` when it is not needed, try checking `nbytes` and
`types` beforehand. If the metadata is already there, continue on
without reading `data`. Otherwise fallback to the reading `data`, but do
make sure to cache the results of that read to avoid doing it again.

* Use `.get(...)` with `self.nbytes` as well

* Restart GitHub CI

Appears GitHub CI failed to checkout the code and clicking restart in
the UI does not work. So pushing a dummy commit to restart it.
---
 distributed/worker.py | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/distributed/worker.py b/distributed/worker.py
index 247ffc99510..191e4df085f 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -1836,13 +1836,16 @@ def ensure_communicating(self):
 
     def send_task_state_to_scheduler(self, key):
         if key in self.data or self.actors.get(key):
-            try:
-                value = self.data[key]
-            except KeyError:
-                value = self.actors[key]
-            nbytes = self.nbytes[key] or sizeof(value)
-            typ = self.types.get(key) or type(value)
-            del value
+            nbytes = self.nbytes.get(key)
+            typ = self.types.get(key)
+            if nbytes is None or typ is None:
+                try:
+                    value = self.data[key]
+                except KeyError:
+                    value = self.actors[key]
+                nbytes = self.nbytes[key] = sizeof(value)
+                typ = self.types[key] = type(value)
+                del value
             try:
                 typ_serialized = dumps_function(typ)
             except PicklingError:

From 0f834e7984ebcdf1333bae41dfd80fa56fd51023 Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Tue, 24 Mar 2020 18:54:05 +0100
Subject: [PATCH 0740/1550] Remove dead stealing code (#3619)

---
 distributed/stealing.py | 29 +++++++++--------------------
 1 file changed, 9 insertions(+), 20 deletions(-)

diff --git a/distributed/stealing.py b/distributed/stealing.py
index 38524f1722c..0d552d1689f 100644
--- a/distributed/stealing.py
+++ b/distributed/stealing.py
@@ -29,8 +29,6 @@ def __init__(self, scheduler):
         self.stealable = dict()
         # { task state: (worker, level) }
         self.key_stealable = dict()
-        # { prefix: { task states } }
-        self.stealable_unknown_durations = defaultdict(set)
 
         self.cost_multipliers = [1 + 2 ** (i - 6) for i in range(15)]
         self.cost_multipliers[0] = 1
@@ -83,11 +81,7 @@ def transition(
 
         if start == "processing":
             self.remove_key_from_stealable(ts)
-            if finish == "memory":
-                for tts in self.stealable_unknown_durations.pop(ts.prefix.name, ()):
-                    if tts not in self.in_flight and tts.state == "processing":
-                        self.put_key_in_stealable(tts)
-            else:
+            if finish != "memory":
                 self.in_flight.pop(ts, None)
 
     def put_key_in_stealable(self, ts):
@@ -136,20 +130,16 @@ def steal_time_ratio(self, ts):
         if split in fast_tasks:
             return None, None
         ws = ts.processing_on
-        if ws is None:
-            self.stealable_unknown_durations[split].add(ts)
+        compute_time = ws.processing[ts]
+        if compute_time < 0.005:  # 5ms, just give up
+            return None, None
+        cost_multiplier = transfer_time / compute_time
+        if cost_multiplier > 100:
             return None, None
-        else:
-            compute_time = ws.processing[ts]
-            if compute_time < 0.005:  # 5ms, just give up
-                return None, None
-            cost_multiplier = transfer_time / compute_time
-            if cost_multiplier > 100:
-                return None, None
 
-            level = int(round(log(cost_multiplier) / log_2 + 6, 0))
-            level = max(1, level)
-            return cost_multiplier, level
+        level = int(round(log(cost_multiplier) / log_2 + 6, 0))
+        level = max(1, level)
+        return cost_multiplier, level
 
     def move_task_request(self, ts, victim, thief):
         try:
@@ -418,7 +408,6 @@ def restart(self, scheduler):
         for s in self.stealable_all:
             s.clear()
         self.key_stealable.clear()
-        self.stealable_unknown_durations.clear()
 
     def story(self, *keys):
         keys = set(keys)

From dd28d08ca22f7ae874ba10e524ed322a15b4cacd Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Tue, 24 Mar 2020 17:55:45 -0500
Subject: [PATCH 0741/1550] Ensure Client connection pool semaphore attaches to
 the Client's event loop (#3546)

* Add Node and ConnectionPool start methods
* Make ConnectionPools awaitable
---
 distributed/client.py                |  3 +++
 distributed/core.py                  | 13 +++++++++++--
 distributed/nanny.py                 |  3 +++
 distributed/node.py                  |  7 ++++++-
 distributed/scheduler.py             |  3 +++
 distributed/tests/test_client.py     |  5 +++++
 distributed/tests/test_core.py       | 10 +++++-----
 distributed/tests/test_scheduler.py  |  6 +++---
 distributed/tests/test_utils_comm.py | 16 ++++++++--------
 distributed/worker.py                |  2 ++
 10 files changed, 49 insertions(+), 19 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 24f25a79a09..4065aad17e9 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -923,6 +923,9 @@ def _send_to_scheduler(self, msg):
             )
 
     async def _start(self, timeout=no_default, **kwargs):
+
+        await super().start()
+
         if timeout == no_default:
             timeout = self._timeout
         if timeout is not None:
diff --git a/distributed/core.py b/distributed/core.py
index 5bff3276e99..1bf3b172b68 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -835,8 +835,6 @@ def __init__(
         self.connection_args = connection_args
         self.timeout = timeout
         self._n_connecting = 0
-        # Invariant: semaphore._value == limit - open - _n_connecting
-        self.semaphore = asyncio.Semaphore(self.limit)
         self.server = weakref.ref(server) if server else None
         self._created = weakref.WeakSet()
         self._instances.add(self)
@@ -871,6 +869,17 @@ def __call__(self, addr=None, ip=None, port=None):
             addr, self, serializers=self.serializers, deserializers=self.deserializers
         )
 
+    def __await__(self):
+        async def _():
+            await self.start()
+            return self
+
+        return _().__await__()
+
+    async def start(self):
+        # Invariant: semaphore._value == limit - open - _n_connecting
+        self.semaphore = asyncio.Semaphore(self.limit)
+
     async def connect(self, addr, timeout=None):
         """
         Get a Comm to the given address.  For internal use.
diff --git a/distributed/nanny.py b/distributed/nanny.py
index ec5397efb93..baa77e3ce10 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -241,6 +241,9 @@ def local_dir(self):
 
     async def start(self):
         """ Start nanny, start local process, start watching """
+
+        await super().start()
+
         await self.listen(self._start_address, listen_args=self.listen_args)
         self.ip = get_address_host(self.address)
 
diff --git a/distributed/node.py b/distributed/node.py
index 4e26defeb08..af15b5a409f 100644
--- a/distributed/node.py
+++ b/distributed/node.py
@@ -38,6 +38,9 @@ def __init__(
             server=self,
         )
 
+    async def start(self):
+        await self.rpc.start()
+
 
 class ServerNode(Node, Server):
     """
@@ -182,5 +185,7 @@ async def wait_for(future, timeout=None):
                 future = wait_for(future, timeout=timeout)
             return future.__await__()
 
-    async def start(self):  # subclasses should implement this
+    async def start(self):
+        # subclasses should implement their own start method whichs calls super().start()
+        await Node.start(self)
         return self
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 8a61ba31fca..cea1f9fd136 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1408,6 +1408,9 @@ def get_worker_service_addr(self, worker, service_name, protocol=False):
 
     async def start(self):
         """ Clear out old state and restart all running coroutines """
+
+        await super().start()
+
         enable_gc_diagnosis()
 
         self.clear_task_state()
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 5c853916558..d9633876cb2 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -5943,3 +5943,8 @@ def test_as_completed_condition_loop(c, s, a, b):
     seq = c.map(inc, range(5))
     ac = as_completed(seq)
     assert ac.condition._loop == c.loop.asyncio_loop
+
+
+def test_client_connectionpool_semaphore_loop(s, a, b):
+    with Client(s["address"]) as c:
+        assert c.rpc.semaphore._loop is c.loop.asyncio_loop
diff --git a/distributed/tests/test_core.py b/distributed/tests/test_core.py
index 0a9c48bc870..76f2b285500 100644
--- a/distributed/tests/test_core.py
+++ b/distributed/tests/test_core.py
@@ -526,7 +526,7 @@ async def ping(comm, delay=0.1):
     for server in servers:
         await server.listen(0)
 
-    rpc = ConnectionPool(limit=5)
+    rpc = await ConnectionPool(limit=5)
 
     # Reuse connections
     await asyncio.gather(
@@ -583,7 +583,7 @@ async def do_ping(pool, port):
     for server in servers:
         await server.listen(0)
 
-    pool = ConnectionPool(limit=limit)
+    pool = await ConnectionPool(limit=limit)
 
     await asyncio.gather(*[do_ping(pool, s.port) for s in servers])
 
@@ -605,7 +605,7 @@ async def ping(comm, delay=0.01):
     for server in servers:
         await server.listen("tls://", listen_args=listen_args)
 
-    rpc = ConnectionPool(limit=5, connection_args=connection_args)
+    rpc = await ConnectionPool(limit=5, connection_args=connection_args)
 
     await asyncio.gather(*[rpc(s.address).ping() for s in servers[:5]])
     await asyncio.gather(*[rpc(s.address).ping() for s in servers[::2]])
@@ -625,7 +625,7 @@ async def ping(comm, delay=0.01):
     for server in servers:
         await server.listen(0)
 
-    rpc = ConnectionPool(limit=10)
+    rpc = await ConnectionPool(limit=10)
     serv = servers.pop()
     await asyncio.gather(*[rpc(s.address).ping() for s in servers])
     await asyncio.gather(*[rpc(serv.address).ping() for i in range(3)])
@@ -758,7 +758,7 @@ async def test_connection_pool_detects_remote_close():
     await server.listen("tcp://")
 
     # open a connection, use it and give it back to the pool
-    p = ConnectionPool(limit=10)
+    p = await ConnectionPool(limit=10)
     conn = await p.connect(server.address)
     await send_recv(conn, op="ping")
     p.reuse(server.address, conn)
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 5459716ca85..24a40dccdab 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -1887,7 +1887,7 @@ async def test_gather_failing_cnn_recover(c, s, a, b):
     orig_rpc = s.rpc
     x = await c.scatter({"x": 1}, workers=a.address)
 
-    s.rpc = FlakyConnectionPool(failing_connections=1)
+    s.rpc = await FlakyConnectionPool(failing_connections=1)
     with mock.patch("distributed.utils_comm.retry_count", 1):
         res = await s.gather(keys=["x"])
     assert res["status"] == "OK"
@@ -1898,7 +1898,7 @@ async def test_gather_failing_cnn_error(c, s, a, b):
     orig_rpc = s.rpc
     x = await c.scatter({"x": 1}, workers=a.address)
 
-    s.rpc = FlakyConnectionPool(failing_connections=10)
+    s.rpc = await FlakyConnectionPool(failing_connections=10)
     res = await s.gather(keys=["x"])
     assert res["status"] == "error"
     assert list(res["keys"]) == ["x"]
@@ -1949,7 +1949,7 @@ def reducer(x, y):
 
     z = c.submit(reducer, x, y)
 
-    s.rpc = FlakyConnectionPool(failing_connections=4)
+    s.rpc = await FlakyConnectionPool(failing_connections=4)
 
     with captured_logger(
         logging.getLogger("distributed.scheduler")
diff --git a/distributed/tests/test_utils_comm.py b/distributed/tests/test_utils_comm.py
index 2d0159a2d3d..7ab793e18e4 100644
--- a/distributed/tests/test_utils_comm.py
+++ b/distributed/tests/test_utils_comm.py
@@ -30,11 +30,11 @@ def test_subs_multiple():
 
 
 @gen_cluster(client=True)
-def test_gather_from_workers_permissive(c, s, a, b):
-    rpc = ConnectionPool()
-    x = yield c.scatter({"x": 1}, workers=a.address)
+async def test_gather_from_workers_permissive(c, s, a, b):
+    rpc = await ConnectionPool()
+    x = await c.scatter({"x": 1}, workers=a.address)
 
-    data, missing, bad_workers = yield gather_from_workers(
+    data, missing, bad_workers = await gather_from_workers(
         {"x": [a.address], "y": [b.address]}, rpc=rpc
     )
 
@@ -68,11 +68,11 @@ async def connect(self, *args, **kwargs):
 
 
 @gen_cluster(client=True)
-def test_gather_from_workers_permissive_flaky(c, s, a, b):
-    x = yield c.scatter({"x": 1}, workers=a.address)
+async def test_gather_from_workers_permissive_flaky(c, s, a, b):
+    x = await c.scatter({"x": 1}, workers=a.address)
 
-    rpc = BrokenConnectionPool()
-    data, missing, bad_workers = yield gather_from_workers({"x": [a.address]}, rpc=rpc)
+    rpc = await BrokenConnectionPool()
+    data, missing, bad_workers = await gather_from_workers({"x": [a.address]}, rpc=rpc)
 
     assert missing == {"x": [a.address]}
     assert bad_workers == [a.address]
diff --git a/distributed/worker.py b/distributed/worker.py
index 191e4df085f..ba25c91d979 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -1012,6 +1012,8 @@ async def start(self):
             return
         assert self.status is None, self.status
 
+        await super().start()
+
         enable_gc_diagnosis()
         thread_state.on_event_loop_thread = True
 

From 13419fb5631df548d9fc977a548b128a8751060f Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Wed, 25 Mar 2020 06:47:42 -0700
Subject: [PATCH 0742/1550] WIP: Include frame lengths of CUDA objects in
 `header["lengths"]` (#3631)

* Reuse "cuda" serialization in "dask" serialization

Make sure that RMM `DeviceBuffer`s use "cuda" serialization before they
are passed through "dask" serialization.

* Include `"lengths"` in CUDA `header`s

As going through `"dask"` serialization can result in data being split
for better compression, ensure the original number of bytes in the frame
is stored in `header["lengths"]`. That way on `"dask"` deserialization
the frames can be merged back into their original sizes before `"cuda"`
serialization is performed.
---
 distributed/protocol/cupy.py  | 1 +
 distributed/protocol/numba.py | 1 +
 distributed/protocol/rmm.py   | 5 +++--
 3 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/distributed/protocol/cupy.py b/distributed/protocol/cupy.py
index 3d074266245..b3465fee424 100644
--- a/distributed/protocol/cupy.py
+++ b/distributed/protocol/cupy.py
@@ -47,6 +47,7 @@ def cuda_serialize_cupy_ndarray(x):
 
     header = x.__cuda_array_interface__.copy()
     header["strides"] = tuple(x.strides)
+    header["lengths"] = [x.nbytes]
     frames = [
         cupy.ndarray(
             shape=(x.nbytes,), dtype=cupy.dtype("u1"), memptr=x.data, strides=(1,)
diff --git a/distributed/protocol/numba.py b/distributed/protocol/numba.py
index 1070c080e61..03bf4aa9f16 100644
--- a/distributed/protocol/numba.py
+++ b/distributed/protocol/numba.py
@@ -21,6 +21,7 @@ def cuda_serialize_numba_ndarray(x):
 
     header = x.__cuda_array_interface__.copy()
     header["strides"] = tuple(x.strides)
+    header["lengths"] = [x.nbytes]
     frames = [
         numba.cuda.cudadrv.devicearray.DeviceNDArray(
             shape=(x.nbytes,), strides=(1,), dtype=np.dtype("u1"), gpu_data=x.gpu_data,
diff --git a/distributed/protocol/rmm.py b/distributed/protocol/rmm.py
index ae2db0d528b..76706d49d89 100644
--- a/distributed/protocol/rmm.py
+++ b/distributed/protocol/rmm.py
@@ -12,6 +12,7 @@
     @cuda_serialize.register(rmm.DeviceBuffer)
     def cuda_serialize_rmm_device_buffer(x):
         header = x.__cuda_array_interface__.copy()
+        header["lengths"] = [x.nbytes]
         frames = [x]
         return header, frames
 
@@ -28,8 +29,8 @@ def cuda_deserialize_rmm_device_buffer(header, frames):
 
     @dask_serialize.register(rmm.DeviceBuffer)
     def dask_serialize_rmm_device_buffer(x):
-        header = x.__cuda_array_interface__.copy()
-        frames = [numba.cuda.as_cuda_array(x).copy_to_host().data]
+        header, frames = cuda_serialize_rmm_device_buffer(x)
+        frames = [numba.cuda.as_cuda_array(f).copy_to_host().data for f in frames]
         return header, frames
 
     @dask_deserialize.register(rmm.DeviceBuffer)

From b0c000883eb10d4801b967fe348c4d6281ca3f1d Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 25 Mar 2020 06:50:55 -0700
Subject: [PATCH 0743/1550] Add logging message when closing idle dask
 scheduler (#3632)

```
$ dask-scheduler --idle-timeout "5 seconds"
distributed.scheduler - INFO - -----------------------------------------------
distributed.scheduler - INFO - Local Directory:    /tmp/scheduler-niju4kje
distributed.scheduler - INFO - -----------------------------------------------
distributed.scheduler - INFO - Clear task state
distributed.scheduler - INFO -   Scheduler at:   tcp://192.168.0.11:8786
distributed.scheduler - INFO -   dashboard at:                     :8787
distributed.scheduler - INFO - Scheduler closing after being idle for 5.00 s
distributed.scheduler - INFO - Scheduler closing...
distributed.scheduler - INFO - Scheduler closing all comms
distributed.scheduler - INFO - End scheduler at 'tcp://192.168.0.11:8786'
```
---
 distributed/scheduler.py            |  4 ++++
 distributed/tests/test_scheduler.py | 21 +++++++++++++--------
 2 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index cea1f9fd136..f99c26d9aba 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -5207,6 +5207,10 @@ def check_idle(self):
             close = time() > last_task + self.idle_timeout
 
         if close:
+            logger.info(
+                "Scheduler closing after being idle for %s",
+                format_time(self.idle_timeout),
+            )
             self.loop.add_callback(self.close)
 
     def adaptive_target(self, comm=None, target_duration=None):
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 24a40dccdab..1068169b200 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -1542,15 +1542,20 @@ async def test_idle_timeout(c, s, a, b):
 
     assert s.status != "closed"
 
-    start = time()
-    while s.status != "closed":
-        await gen.sleep(0.01)
-        assert time() < start + 3
+    with captured_logger("distributed.scheduler") as logs:
+        start = time()
+        while s.status != "closed":
+            await gen.sleep(0.01)
+            assert time() < start + 3
 
-    start = time()
-    while not (a.status == "closed" and b.status == "closed"):
-        await gen.sleep(0.01)
-        assert time() < start + 1
+        start = time()
+        while not (a.status == "closed" and b.status == "closed"):
+            await gen.sleep(0.01)
+            assert time() < start + 1
+
+    assert "idle" in logs.getvalue()
+    assert "500" in logs.getvalue()
+    assert "ms" in logs.getvalue()
 
 
 @gen_cluster(client=True, config={"distributed.scheduler.bandwidth": "100 GB"})

From fbdb067ed514024aad14dbdc3790bae9344548df Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Wed, 25 Mar 2020 07:04:06 -0700
Subject: [PATCH 0744/1550] Drop unused line from `pack_frames_prelude` (#3634)

---
 distributed/protocol/utils.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/distributed/protocol/utils.py b/distributed/protocol/utils.py
index 3af203881ff..e58732b881c 100644
--- a/distributed/protocol/utils.py
+++ b/distributed/protocol/utils.py
@@ -92,7 +92,6 @@ def merge_frames(header, frames):
 
 
 def pack_frames_prelude(frames):
-    lengths = [len(f) for f in frames]
     lengths = [struct.pack("Q", len(frames))] + [
         struct.pack("Q", nbytes(frame)) for frame in frames
     ]

From 77d103f10971bea68fa733e53c00d9ea2a6a431a Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 25 Mar 2020 08:14:57 -0700
Subject: [PATCH 0745/1550] Add as_completed.clear method (#3617)

---
 distributed/client.py                  | 30 ++++++++++++++++----------
 distributed/tests/test_as_completed.py | 16 +++++++++++++-
 2 files changed, 34 insertions(+), 12 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 4065aad17e9..c45e0718b43 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -4236,17 +4236,18 @@ async def _track_future(self, future):
             except CancelledError as exc:
                 result = exc
         with self.lock:
-            self.futures[future] -= 1
-            if not self.futures[future]:
-                del self.futures[future]
-            if self.with_results:
-                self.queue.put_nowait((future, result))
-            else:
-                self.queue.put_nowait(future)
-            async with self.condition:
-                self.condition.notify()
-            with self.thread_condition:
-                self.thread_condition.notify()
+            if future in self.futures:
+                self.futures[future] -= 1
+                if not self.futures[future]:
+                    del self.futures[future]
+                if self.with_results:
+                    self.queue.put_nowait((future, result))
+                else:
+                    self.queue.put_nowait(future)
+                async with self.condition:
+                    self.condition.notify()
+                with self.thread_condition:
+                    self.thread_condition.notify()
 
     def update(self, futures):
         """ Add multiple futures to the collection.
@@ -4380,6 +4381,13 @@ def batches(self):
             except StopIteration:
                 return
 
+    def clear(self):
+        """ Clear out all submitted futures """
+        with self.lock:
+            self.futures.clear()
+            while not self.queue.empty():
+                self.queue.get()
+
 
 def AsCompleted(*args, **kwargs):
     raise Exception("This has moved to as_completed")
diff --git a/distributed/tests/test_as_completed.py b/distributed/tests/test_as_completed.py
index c9780d196cd..f71c6f7492e 100644
--- a/distributed/tests/test_as_completed.py
+++ b/distributed/tests/test_as_completed.py
@@ -8,7 +8,7 @@
 import pytest
 from tornado import gen
 
-from distributed.client import _as_completed, as_completed, _first_completed
+from distributed.client import _as_completed, as_completed, _first_completed, wait
 from distributed.metrics import time
 from distributed.utils import CancelledError
 from distributed.utils_test import gen_cluster, inc, throws
@@ -273,3 +273,17 @@ def test_as_completed_with_results_no_raise_async(c, s, a, b):
     assert isinstance(dd[y][0], CancelledError)
     assert isinstance(dd[x][0][1], RuntimeError)
     assert dd[z][0] == 2
+
+
+@gen_cluster(client=True, timeout=None)
+async def test_clear(c, s, a, b):
+    futures = c.map(inc, range(3))
+    ac = as_completed(futures)
+    await wait(futures)
+    ac.clear()
+    with pytest.raises(StopAsyncIteration):
+        await ac.__anext__()
+    del futures
+
+    while s.tasks:
+        await asyncio.sleep(0.3)

From 2277379f6249ecbb132d6b3872550d7ee7665ef0 Mon Sep 17 00:00:00 2001
From: Peter Andreas Entschev <peter@entschev.com>
Date: Wed, 25 Mar 2020 19:38:34 +0100
Subject: [PATCH 0746/1550] UCX synchronize default stream only on CUDA frames
 (#3638)

* UCX synchronize default stream only on CUDA frames

* Improve check for CUDA frames in UCX comms

* Further improvements to CUDA frame synchronization in UCX

* Fix black formatting
---
 distributed/comm/ucx.py | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/distributed/comm/ucx.py b/distributed/comm/ucx.py
index 22d7e361e97..fc187dcc614 100644
--- a/distributed/comm/ucx.py
+++ b/distributed/comm/ucx.py
@@ -158,13 +158,12 @@ async def write(
                 )
 
                 # Send meta data
-                await self.ep.send(np.array([len(frames)], dtype=np.uint64))
-                await self.ep.send(
-                    np.array(
-                        [hasattr(f, "__cuda_array_interface__") for f in frames],
-                        dtype=np.bool,
-                    )
+                cuda_frames = np.array(
+                    [hasattr(f, "__cuda_array_interface__") for f in frames],
+                    dtype=np.bool,
                 )
+                await self.ep.send(np.array([len(frames)], dtype=np.uint64))
+                await self.ep.send(cuda_frames)
                 await self.ep.send(
                     np.array([nbytes(f) for f in frames], dtype=np.uint64)
                 )
@@ -175,7 +174,8 @@ async def write(
                 #  syncing the default stream will wait for other non-blocking CUDA streams.
                 # Note this is only sufficient if the memory being sent is not currently in use on
                 # non-blocking CUDA streams.
-                synchronize_stream(0)
+                if cuda_frames.any():
+                    synchronize_stream(0)
 
                 for frame in frames:
                     if nbytes(frame) > 0:
@@ -222,7 +222,8 @@ async def read(self, deserializers=("cuda", "dask", "pickle", "error")):
 
                 # It is necessary to first populate `frames` with CUDA arrays and synchronize
                 # the default stream before starting receiving to ensure buffers have been allocated
-                synchronize_stream(0)
+                if is_cudas.any():
+                    synchronize_stream(0)
                 for i, (is_cuda, size) in enumerate(
                     zip(is_cudas.tolist(), sizes.tolist())
                 ):

From 2e64ae9c256069d8e5ca93b1a1b7356a8c29f3c5 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Wed, 25 Mar 2020 14:42:12 -0500
Subject: [PATCH 0747/1550] bump version to 2.13.0

---
 docs/source/changelog.rst | 39 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index 9c7ca9b01f4..12288fc4aba 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,41 @@
 Changelog
 =========
 
+2.13.0 - 2020-03-25
+-------------------
+
+- UCX synchronize default stream only on CUDA frames (:pr:`3638`) `Peter Andreas Entschev`_
+- Add ``as_completed.clear`` method (:pr:`3617`) `Matthew Rocklin`_
+- Drop unused line from ``pack_frames_prelude`` (:pr:`3634`) `John Kirkham`_
+- Add logging message when closing idle dask scheduler (:pr:`3632`) `Matthew Rocklin`_
+- Include frame lengths of CUDA objects in ``header["lengths"]`` (:pr:`3631`) `John Kirkham`_
+- Ensure ``Client`` connection pool semaphore attaches to the ``Client`` event loop (:pr:`3546`) `James Bourbeau`_
+- Remove dead stealing code (:pr:`3619`) `Florian Jetter`_
+- Check ``nbytes`` and ``types`` before reading ``data`` (:pr:`3628`) `John Kirkham`_
+- Ensure that we don't steal blacklisted fast tasks (:pr:`3591`) `Florian Jetter`_
+- Support async ``Listener.stop`` functions (:pr:`3613`) `Matthew Rocklin`_
+- Add str/repr methods to ``as_completed`` (:pr:`3618`) `Matthew Rocklin`_
+- Add backoff to comm connect attempts. (:pr:`3496`) `Matthias Urlichs`_
+- Make ``Listeners`` awaitable (:pr:`3611`) `Matthew Rocklin`_
+- Increase number of visible mantissas in dashboard plots (:pr:`3585`) `Scott Sievert`_
+- Pin openssl to 1.1.1d for Travis (:pr:`3602`) `Jacob Tomlinson`_
+- Replace ``tornado.queues`` with ``asyncio.queues`` (:pr:`3607`) `James Bourbeau`_
+- Remove ``dill`` from CI environments (:pr:`3608`) `Loïc Estève`_
+- Fix linting errors (:pr:`3604`) `James Bourbeau`_
+- Synchronize default CUDA stream before UCX send/recv (:pr:`3598`) `Peter Andreas Entschev`_
+- Add configuration for ``Adaptive`` arguments (:pr:`3509`) `Gabriel Sailer`_
+- Change ``Adaptive`` docs to reference ``adaptive_target`` (:pr:`3597`) `Julia Signell`_
+- Optionally compress on a frame-by-frame basis (:pr:`3586`) `Matthew Rocklin`_
+- Add Python version to version check (:pr:`3567`) `James Bourbeau`_
+- Import ``tlz`` (:pr:`3579`) `John Kirkham`_
+- Pin ``numpydoc`` to avoid double escaped ``*`` (:pr:`3530`) `Gil Forsyth`_
+- Avoid ``performance_report`` crashing when a worker dies mid-compute (:pr:`3575`) `Krishan Bhasin`_
+- Pin ``bokeh`` in CI builds (:pr:`3570`) `James Bourbeau`_
+- Disable fast fail on GitHub Actions Windows CI (:pr:`3569`) `James Bourbeau`_
+- Fix typo in ``Client.shutdown`` docstring (:pr:`3562`) `John Kirkham`_
+- Add ``local_directory`` option to ``dask-ssh`` (:pr:`3554`) `Abdulelah Bin Mahfoodh`_
+
+
 2.12.0 - 2020-03-06
 -------------------
 
@@ -1583,6 +1618,7 @@ significantly without many new features.
 .. _`He Jia`: https://github.com/HerculesJack
 .. _`Jim Crist-Harif`: https://github.com/jcrist
 .. _`fjetter`: https://github.com/fjetter
+.. _`Florian Jetter`: https://github.com/fjetter
 .. _`Patrick Sodré`: https://github.com/sodre
 .. _`Stephan Erb`: https://github.com/StephanErb
 .. _`Benedikt Reinartz`: https://github.com/filmor
@@ -1602,3 +1638,6 @@ significantly without many new features.
 .. _`Davis Bennett`: https://github.com/d-v-b
 .. _`Lucas Rademaker`: https://github.com/lr4d
 .. _`Darren Weber`: https://github.com/dazza-codes
+.. _`Matthias Urlichs`: https://github.com/smurfix
+.. _`Krishan Bhasin`: https://github.com/KrishanBhasin
+.. _`Abdulelah Bin Mahfoodh`: https://github.com/abduhbm

From 0bf20a4312b113659900be7e647d28b3bde4015a Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Thu, 26 Mar 2020 13:08:35 -0500
Subject: [PATCH 0748/1550] Update bokeh dependency in CI builds (#3637)

---
 continuous_integration/environment.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/continuous_integration/environment.yml b/continuous_integration/environment.yml
index 7458c9f64ac..5f09525caae 100644
--- a/continuous_integration/environment.yml
+++ b/continuous_integration/environment.yml
@@ -3,7 +3,7 @@ channels:
   - conda-forge
 dependencies:
   - zstandard
-  - bokeh=1.4.0
+  - bokeh!=2.0.0
   - click
   - cloudpickle
   - dask

From 9a02d7c7c6ecf8010d5286e625a5d842e99df8ad Mon Sep 17 00:00:00 2001
From: Prasun Anand <prasunanand.bitsp@gmail.com>
Date: Thu, 26 Mar 2020 23:40:23 +0530
Subject: [PATCH 0749/1550] Add link to contributing.md (#3621)

---
 CONTRIBUTING.md | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index ab4175a59fe..3859e21af8d 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -1,3 +1,5 @@
-Dask is a community maintained project. We welcome contributions in the form of bug reports, documentation, code, design proposals, and more. 
+Dask is a community maintained project. We welcome contributions in the form of bug reports, documentation, code, design proposals, and more.
 
-For general information on how to contribute see https://docs.dask.org/en/latest/develop.html.
+Please see https://distributed.dask.org/en/latest/develop.html for more information.
+
+Also for general information on how to contribute see https://docs.dask.org/en/latest/develop.html.

From f244ab64e634f46ad3d26de7a2f536e0055a5214 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Fri, 27 Mar 2020 03:50:41 -0500
Subject: [PATCH 0750/1550] Don't create output Futures in Client when there
 are mixed Client Futures (#3643)

* Don't create Futures if raising mixed Clients error

* Add test
---
 distributed/client.py            | 5 ++---
 distributed/tests/test_client.py | 3 +++
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index c45e0718b43..97c2fd60e17 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -2452,8 +2452,6 @@ def _graph_to_futures(
                 actors = list(self._expand_key(actors))
 
             keyset = set(keys)
-            flatkeys = list(map(tokey, keys))
-            futures = {key: Future(key, self, inform=False) for key in keyset}
 
             values = {
                 k: v
@@ -2506,12 +2504,13 @@ def _graph_to_futures(
             if isinstance(retries, Number) and retries > 0:
                 retries = {k: retries for k in dsk3}
 
+            futures = {key: Future(key, self, inform=False) for key in keyset}
             self._send_to_scheduler(
                 {
                     "op": "update-graph",
                     "tasks": valmap(dumps_task, dsk3),
                     "dependencies": dependencies,
-                    "keys": list(flatkeys),
+                    "keys": list(map(tokey, keys)),
                     "restrictions": restrictions or {},
                     "loose_restrictions": loose_restrictions,
                     "priority": priority,
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index d9633876cb2..98d0a9bf290 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -5449,6 +5449,9 @@ def test_mixing_clients(s, a, b):
     future = c1.submit(inc, 1)
     with pytest.raises(ValueError):
         c2.submit(inc, future)
+
+    assert not c2.futures  # Don't create Futures on second Client
+
     yield c1.close()
     yield c2.close()
 

From 926eb12a03ac750712e13ac60560c64f652661b1 Mon Sep 17 00:00:00 2001
From: Prasun Anand <prasunanand.bitsp@gmail.com>
Date: Fri, 27 Mar 2020 20:34:54 +0530
Subject: [PATCH 0751/1550] Remove local-directory keyword (#3620)

---
 distributed/cli/dask_scheduler.py | 20 --------------------
 1 file changed, 20 deletions(-)

diff --git a/distributed/cli/dask_scheduler.py b/distributed/cli/dask_scheduler.py
index 0951b8c3d27..2394dd65dea 100755
--- a/distributed/cli/dask_scheduler.py
+++ b/distributed/cli/dask_scheduler.py
@@ -3,9 +3,7 @@
 import gc
 import os
 import re
-import shutil
 import sys
-import tempfile
 import warnings
 
 import click
@@ -104,9 +102,6 @@
     "This may be a good way to share connection information if your "
     "cluster is on a shared network file system.",
 )
-@click.option(
-    "--local-directory", default="", type=str, help="Directory to place scheduler files"
-)
 @click.option(
     "--preload",
     type=str,
@@ -136,7 +131,6 @@ def main(
     dashboard_prefix,
     use_xheaders,
     pid_file,
-    local_directory,
     tls_ca_file,
     tls_cert,
     tls_key,
@@ -194,17 +188,6 @@ def del_pid_file():
 
         atexit.register(del_pid_file)
 
-    local_directory_created = False
-    if local_directory:
-        if not os.path.exists(local_directory):
-            os.mkdir(local_directory)
-            local_directory_created = True
-    else:
-        local_directory = tempfile.mkdtemp(prefix="scheduler-")
-        local_directory_created = True
-    if local_directory not in sys.path:
-        sys.path.insert(0, local_directory)
-
     if sys.platform.startswith("linux"):
         import resource  # module fails importing on Windows
 
@@ -224,7 +207,6 @@ def del_pid_file():
         service_kwargs={"dashboard": {"prefix": dashboard_prefix}},
         **kwargs
     )
-    logger.info("Local Directory: %26s", local_directory)
     logger.info("-" * 47)
 
     install_signal_handlers(loop)
@@ -237,8 +219,6 @@ async def run():
         loop.run_sync(run)
     finally:
         scheduler.stop()
-        if local_directory_created:
-            shutil.rmtree(local_directory)
 
         logger.info("End scheduler at %r", scheduler.address)
 

From aa979741787248e4d45abec9affb6240ab61dd1e Mon Sep 17 00:00:00 2001
From: Gabriel Sailer <sublinus@users.noreply.github.com>
Date: Fri, 27 Mar 2020 16:42:48 +0100
Subject: [PATCH 0752/1550] Add prometheus metric for suspicious tasks (#3550)

---
 distributed/dashboard/scheduler.py  | 10 ++++++++++
 distributed/scheduler.py            |  7 +++++++
 distributed/tests/test_scheduler.py | 14 ++++++++++++++
 3 files changed, 31 insertions(+)

diff --git a/distributed/dashboard/scheduler.py b/distributed/dashboard/scheduler.py
index acaab24cd17..982bc424826 100644
--- a/distributed/dashboard/scheduler.py
+++ b/distributed/dashboard/scheduler.py
@@ -289,6 +289,16 @@ def collect(self):
             sum, (tp.states for tp in self.server.task_prefixes.values())
         )
 
+        suspicious_tasks = CounterMetricFamily(
+            "dask_scheduler_tasks_suspicious",
+            "Total number of times a task has been marked suspicious",
+            labels=["task_prefix_name"],
+        )
+
+        for tp in self.server.task_prefixes.values():
+            suspicious_tasks.add_metric([tp.name], tp.suspicious)
+        yield suspicious_tasks
+
         yield CounterMetricFamily(
             "dask_scheduler_tasks_forgotten",
             (
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index f99c26d9aba..3de506cfade 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -789,6 +789,11 @@ class TaskPrefix:
 
        An exponentially weighted moving average duration of all tasks with this prefix
 
+    .. attribute:: suspicious: int
+
+       Numbers of times a task was marked as suspicious with this prefix
+
+
     See Also
     --------
     TaskGroup
@@ -805,6 +810,7 @@ def __init__(self, name):
             )
         else:
             self.duration_average = None
+        self.suspicious = 0
 
     @property
     def states(self):
@@ -2190,6 +2196,7 @@ def remove_worker(self, comm=None, address=None, safe=False, close=True):
                 recommendations[k] = "released"
                 if not safe:
                     ts.suspicious += 1
+                    ts.prefix.suspicious += 1
                     if ts.suspicious > self.allowed_failures:
                         del recommendations[k]
                         e = pickle.dumps(
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 1068169b200..46e4d0c885c 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -787,6 +787,7 @@ def test_retire_workers_no_suspicious_tasks(c, s, a, b):
     yield s.retire_workers(workers=[a.address])
 
     assert all(ts.suspicious == 0 for ts in s.tasks.values())
+    assert all(tp.suspicious == 0 for tp in s.task_prefixes.values())
 
 
 @pytest.mark.slow
@@ -1810,6 +1811,19 @@ async def test_task_prefix(c, s, a, b):
     assert s.task_prefixes["sum-aggregate"].states["memory"] == 2
 
 
+@gen_cluster(
+    client=True, Worker=Nanny, config={"distributed.scheduler.allowed-failures": 0}
+)
+async def test_failing_task_increments_suspicious(client, s, a, b):
+    future = client.submit(sys.exit, 0)
+    await wait(future)
+
+    assert s.task_prefixes["exit"].suspicious == 1
+    assert sum(tp.suspicious for tp in s.task_prefixes.values()) == sum(
+        ts.suspicious for ts in s.tasks.values()
+    )
+
+
 @gen_cluster(client=True)
 async def test_task_group_non_tuple_key(c, s, a, b):
     da = pytest.importorskip("dask.array")

From b8a0b8e110d37b7986d0693e9f35fdddb85ded17 Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Fri, 27 Mar 2020 15:43:23 +0000
Subject: [PATCH 0753/1550] Handle exception in faulthandler (#3646)

---
 conftest.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/conftest.py b/conftest.py
index b5db36f59d8..07adc4982f6 100644
--- a/conftest.py
+++ b/conftest.py
@@ -12,7 +12,10 @@
 except ImportError:
     pass
 else:
-    faulthandler.enable()
+    try:
+        faulthandler.enable()
+    except Exception:
+        pass
 
 
 def pytest_addoption(parser):

From 7802bf3bffe0c870bb27543e4c334820c3b8fae7 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Fri, 27 Mar 2020 12:16:30 -0500
Subject: [PATCH 0754/1550] Bump checkout GitHub action to v2 (#3649)

---
 .github/workflows/ci-docs.yaml    | 2 +-
 .github/workflows/ci-windows.yaml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci-docs.yaml b/.github/workflows/ci-docs.yaml
index 780e2a251fd..c519427f140 100644
--- a/.github/workflows/ci-docs.yaml
+++ b/.github/workflows/ci-docs.yaml
@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v1
+        uses: actions/checkout@v2
 
       - name: Set up Python 3.7
         uses: actions/setup-python@v1
diff --git a/.github/workflows/ci-windows.yaml b/.github/workflows/ci-windows.yaml
index 3b99a8c8ec0..e0c95d0f234 100644
--- a/.github/workflows/ci-windows.yaml
+++ b/.github/workflows/ci-windows.yaml
@@ -12,7 +12,7 @@ jobs:
 
     steps:
       - name: Checkout source
-        uses: actions/checkout@v1
+        uses: actions/checkout@v2
 
       - name: Setup Conda Environment
         uses: goanpeca/setup-miniconda@v1

From 3fceec696b81f02c8082f253bf340e3f494fc42c Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Fri, 27 Mar 2020 13:36:04 -0700
Subject: [PATCH 0755/1550] UCX simplify receiving frames in `comm`s (#3651)

* Prefix `for`-loop variables with `each_*`

Should make it easier to disambiguate things like `frame` and `frames`
as they are now `each_frame` and `frames`.

* Allocate frames the same way in 0-length case

* Always allocate frames, receive non-trivial ones

* Allocate all frames to fill before receiving

* Filter out non-trivial frames to transmit
---
 distributed/comm/ucx.py | 40 ++++++++++++++++++----------------------
 1 file changed, 18 insertions(+), 22 deletions(-)

diff --git a/distributed/comm/ucx.py b/distributed/comm/ucx.py
index fc187dcc614..a29441ec4d5 100644
--- a/distributed/comm/ucx.py
+++ b/distributed/comm/ucx.py
@@ -156,6 +156,9 @@ async def write(
                 frames = await to_frames(
                     msg, serializers=serializers, on_error=on_error
                 )
+                send_frames = [
+                    each_frame for each_frame in frames if len(each_frame) > 0
+                ]
 
                 # Send meta data
                 cuda_frames = np.array(
@@ -167,6 +170,7 @@ async def write(
                 await self.ep.send(
                     np.array([nbytes(f) for f in frames], dtype=np.uint64)
                 )
+
                 # Send frames
 
                 # It is necessary to first synchronize the default stream before start sending
@@ -177,10 +181,9 @@ async def write(
                 if cuda_frames.any():
                     synchronize_stream(0)
 
-                for frame in frames:
-                    if nbytes(frame) > 0:
-                        await self.ep.send(frame)
-                return sum(map(nbytes, frames))
+                for each_frame in send_frames:
+                    await self.ep.send(each_frame)
+                return sum(map(nbytes, send_frames))
             except (ucp.exceptions.UCXBaseException):
                 self.abort()
                 raise CommClosedError("While writing, the connection was closed")
@@ -206,30 +209,23 @@ async def read(self, deserializers=("cuda", "dask", "pickle", "error")):
                 raise CommClosedError("While reading, the connection was closed")
             else:
                 # Recv frames
-                frames = []
-                for is_cuda, size in zip(is_cudas.tolist(), sizes.tolist()):
-                    if size > 0:
-                        if is_cuda:
-                            frame = cuda_array(size)
-                        else:
-                            frame = np.empty(size, dtype=np.uint8)
-                        frames.append(frame)
-                    else:
-                        if is_cuda:
-                            frames.append(cuda_array(size))
-                        else:
-                            frames.append(b"")
+                frames = [
+                    cuda_array(each_size)
+                    if is_cuda
+                    else np.empty(each_size, dtype=np.uint8)
+                    for is_cuda, each_size in zip(is_cudas.tolist(), sizes.tolist())
+                ]
+                recv_frames = [
+                    each_frame for each_frame in frames if len(each_frame) > 0
+                ]
 
                 # It is necessary to first populate `frames` with CUDA arrays and synchronize
                 # the default stream before starting receiving to ensure buffers have been allocated
                 if is_cudas.any():
                     synchronize_stream(0)
-                for i, (is_cuda, size) in enumerate(
-                    zip(is_cudas.tolist(), sizes.tolist())
-                ):
-                    if size > 0:
-                        await self.ep.recv(frames[i])
 
+                for each_frame in recv_frames:
+                    await self.ep.recv(each_frame)
                 msg = await from_frames(
                     frames, deserialize=self.deserialize, deserializers=deserializers
                 )

From eda27bee472090f5370121d7d35c201d2facd79e Mon Sep 17 00:00:00 2001
From: Gabriel Sailer <sublinus@users.noreply.github.com>
Date: Fri, 27 Mar 2020 23:10:14 +0100
Subject: [PATCH 0756/1550] Introduce config for default task duration (#3642)

---
 distributed/distributed.yaml        |  1 +
 distributed/scheduler.py            |  6 +++++-
 distributed/tests/test_scheduler.py | 11 +++++++++++
 3 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index 311eeaae829..17326aebd54 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -24,6 +24,7 @@ distributed:
     pickle: True            # Is the scheduler allowed to deserialize arbitrary bytestrings
     preload: []
     preload-argv: []
+    unknown-task-duration: 500ms  # Default duration for all tasks with unknown durations ("15m", "2h")
     default-task-durations:  # How long we expect function names to run ("1h", "1s") (helps for long tasks)
       rechunk-split: 1us
       shuffle-split: 1us
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 3de506cfade..0e998e7296a 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -3700,7 +3700,7 @@ def get_comm_cost(self, ts, ws):
         """
         return sum(dts.nbytes for dts in ts.dependencies - ws.has_what) / self.bandwidth
 
-    def get_task_duration(self, ts, default=0.5):
+    def get_task_duration(self, ts, default=None):
         """
         Get the estimated computation cost of the given task
         (not including any communication cost).
@@ -3708,6 +3708,10 @@ def get_task_duration(self, ts, default=0.5):
         duration = ts.prefix.duration_average
         if duration is None:
             self.unknown_durations[ts.prefix.name].add(ts)
+            if default is None:
+                default = parse_timedelta(
+                    dask.config.get("distributed.scheduler.unknown-task-duration")
+                )
             return default
 
         return duration
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 46e4d0c885c..a0e1b11de2b 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -2071,3 +2071,14 @@ async def test_worker_name_collision(s, a):
     s.validate_state()
     assert set(s.workers) == {a.address}
     assert s.aliases == {a.name: a.address}
+
+
+@gen_cluster(client=True, config={"distributed.scheduler.unknown-task-duration": "1h"})
+async def test_unknown_task_duration_config(client, s, a, b):
+    future = client.submit(slowinc, 1)
+    while not s.tasks:
+        await asyncio.sleep(0.001)
+    assert sum(s.get_task_duration(ts) for ts in s.tasks.values()) == 3600
+    assert len(s.unknown_durations) == 1
+    await wait(future)
+    assert len(s.unknown_durations) == 0

From f765242811199801553822a99e80559465926357 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sat, 28 Mar 2020 10:15:05 -0700
Subject: [PATCH 0757/1550] Avoid diangostics time in performance report
 (#3654)

Previously we would include all of the time taken to generate the
performance report in the reported time.  Now we record the time before
we generate plots and use that instead.
---
 distributed/scheduler.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 0e998e7296a..882970838df 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -5010,6 +5010,7 @@ async def get_profile_metadata(
         return {"counts": counts, "keys": keys}
 
     async def performance_report(self, comm=None, start=None, code=""):
+        stop = time()
         # Profiles
         compute, scheduler, workers = await asyncio.gather(
             *[
@@ -5071,7 +5072,7 @@ def profile_to_figure(state):
 {code}
         </pre>
         """.format(
-            time=format_time(time() - start),
+            time=format_time(stop - start),
             address=self.address,
             nworkers=len(self.workers),
             threads=sum(w.nthreads for w in self.workers.values()),

From d7948ce499c6788b45758efc0d31103cf57c2d22 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sat, 28 Mar 2020 10:55:46 -0700
Subject: [PATCH 0758/1550] Clean up performance report test (#3655)

Previously all of the text in the assertions was being included in the
performance report itself (ever since we started including the
surrounding frame).  This made these tests pass trivially.

Now we wrap the performance report in a function of its own.
---
 distributed/tests/test_client.py | 38 +++++++++++++++++++++-----------
 1 file changed, 25 insertions(+), 13 deletions(-)

diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 98d0a9bf290..8f121c7f27c 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -5916,22 +5916,34 @@ async def f(dask_worker):
     assert b.foo == "bar"
 
 
-@gen_cluster(client=True)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 2)] * 2)
 async def test_performance_report(c, s, a, b):
     da = pytest.importorskip("dask.array")
-    x = da.random.random((1000, 1000), chunks=(100, 100))
-
-    with tmpfile(extension="html") as fn:
-        async with performance_report(filename=fn):
-            await c.compute((x + x.T).sum())
 
-        with open(fn) as f:
-            data = f.read()
-
-        assert "bokeh" in data
-        assert "random" in data
-        assert "Dask Performance Report" in data
-        assert "x = da.random" in data
+    async def f():
+        """
+        We wrap this in a function so that the assertions aren't in the
+        performanace report itself
+
+        Also, we want this comment to appear
+        """
+        x = da.random.random((1000, 1000), chunks=(100, 100))
+        with tmpfile(extension="html") as fn:
+            async with performance_report(filename=fn):
+                await c.compute((x + x.T).sum())
+
+            with open(fn) as f:
+                data = f.read()
+        return data
+
+    data = await f()
+
+    assert "Also, we want this comment to appear" in data
+    assert "bokeh" in data
+    assert "random" in data
+    assert "Dask Performance Report" in data
+    assert "x = da.random" in data
+    assert "Threads: 4" in data
 
 
 @pytest.mark.asyncio

From 362896a87927cbca48cccf3ba6ceb495169ecaee Mon Sep 17 00:00:00 2001
From: Rami Chowdhury <460769+necaris@users.noreply.github.com>
Date: Mon, 30 Mar 2020 12:24:25 -0400
Subject: [PATCH 0759/1550] Add newlines to ensure code formatting for
 `retire_workers`(#3661)

Without the newline, the code samples don't seem to be parsed as such,
and so aren't set apart or formatted properly.
---
 distributed/client.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/distributed/client.py b/distributed/client.py
index 97c2fd60e17..f2b25b74f60 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -3489,9 +3489,11 @@ def retire_workers(self, workers=None, close_workers=True, **kwargs):
         Examples
         --------
         You can get information about active workers using the following:
+
         >>> workers = client.scheduler_info()['workers']
 
         From that list you may want to select some workers to close
+
         >>> client.retire_workers(workers=['tcp://address:port', ...])
 
         See Also

From 067fd1cc35ec9d5eae62c50e1bc439111f613662 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Mon, 30 Mar 2020 13:51:51 -0500
Subject: [PATCH 0760/1550] Update Python version checking (#3660)

---
 distributed/tests/test_versions.py | 20 +++++---------------
 distributed/versions.py            |  5 +----
 2 files changed, 6 insertions(+), 19 deletions(-)

diff --git a/distributed/tests/test_versions.py b/distributed/tests/test_versions.py
index ab3547820ca..64b94ba37b1 100644
--- a/distributed/tests/test_versions.py
+++ b/distributed/tests/test_versions.py
@@ -1,5 +1,5 @@
 import re
-import uuid
+import sys
 
 import pytest
 
@@ -120,17 +120,7 @@ async def test_version_warning_in_cluster(s, a, b):
         )
 
 
-@gen_cluster()
-async def test_python_version_mismatch_warning(s, a, b):
-    # Set random Python version for one worker
-    random_version = uuid.uuid4().hex
-    orig = s.workers[a.address].versions["host"]["python"] = random_version
-
-    with pytest.warns(None) as record:
-        async with Client(s.address, asynchronous=True) as client:
-            pass
-
-    assert record
-    assert any("python" in str(r.message) for r in record)
-    assert any(random_version in str(r.message) for r in record)
-    assert any(a.address in str(r.message) for r in record)
+def test_python_version():
+    required = get_versions()["packages"]
+    assert "python" in required
+    assert required["python"] == ".".join(map(str, sys.version_info))
diff --git a/distributed/versions.py b/distributed/versions.py
index 403d79f6aef..d800f65ec63 100644
--- a/distributed/versions.py
+++ b/distributed/versions.py
@@ -82,7 +82,7 @@ def version_of_package(pkg):
 def get_package_info(pkgs):
     """ get package versions for the passed required & optional packages """
 
-    pversions = []
+    pversions = [("python", ".".join(map(str, sys.version_info)))]
     for pkg in pkgs:
         if isinstance(pkg, (tuple, list)):
             modname, ver_f = pkg
@@ -121,9 +121,6 @@ def error_message(scheduler, workers, client, client_name="client"):
             for pkg, version in info["packages"].items():
                 node_packages[node][pkg] = version
                 packages.add(pkg)
-            # Collect Python version for each node
-            node_packages[node]["python"] = info["host"]["python"]
-            packages.add("python")
 
     errs = []
     for pkg in sorted(packages):

From 26e28c348a1101a62631a07c152c6ce0365621d0 Mon Sep 17 00:00:00 2001
From: Martin Durant <martindurant@users.noreply.github.com>
Date: Tue, 31 Mar 2020 11:12:36 -0400
Subject: [PATCH 0761/1550] Write "why killed" docs [skip ci] (#3596)

* Write "why killed" docs [skip ci]

* try specific [skip ci]

* Responses
---
 docs/source/faq.rst     |   8 +++
 docs/source/index.rst   |   1 +
 docs/source/ipython.rst |   1 +
 docs/source/killed.rst  | 139 ++++++++++++++++++++++++++++++++++++++++
 docs/source/worker.rst  |   1 +
 5 files changed, 150 insertions(+)
 create mode 100644 docs/source/killed.rst

diff --git a/docs/source/faq.rst b/docs/source/faq.rst
index 5803f10096c..0c0e3e84b70 100644
--- a/docs/source/faq.rst
+++ b/docs/source/faq.rst
@@ -82,3 +82,11 @@ subprocess, head to `this section`_ of the supervisor documentation to see
 how to pass the ``$HOME`` and ``$USER`` variables through.
 
 .. _this section: http://supervisord.org/subprocess.html#subprocess-environment
+
+
+KilledWorker, CommsClosed, etc.
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+In the case that workers disappear unexpectedly from your cluster, you may see
+a range of error messages. After checking the logs of the workers affected, you
+should read the section :doc:`killed`.
diff --git a/docs/source/index.rst b/docs/source/index.rst
index 3cbdd18792a..249f9eb9faf 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -100,6 +100,7 @@ Contents
    scheduling-state
    worker
    work-stealing
+   killed
 
 .. toctree::
    :maxdepth: 1
diff --git a/docs/source/ipython.rst b/docs/source/ipython.rst
index a70229d508f..df5f44eb1d1 100644
--- a/docs/source/ipython.rst
+++ b/docs/source/ipython.rst
@@ -37,6 +37,7 @@ This is possible with the Client.become_dask_ method::
 
 .. _Client.become_dask: https://ipyparallel.readthedocs.io/en/latest/api/ipyparallel.html#ipyparallel.Client.become_dask
 
+.. _ipylaunch:
 
 Launch IPython within Dask Workers
 ----------------------------------
diff --git a/docs/source/killed.rst b/docs/source/killed.rst
new file mode 100644
index 00000000000..837ccd944b4
--- /dev/null
+++ b/docs/source/killed.rst
@@ -0,0 +1,139 @@
+.. _killed:
+
+Why did my worker die?
+----------------------
+
+A Dask worker can cease functioning for a number of reasons. These fall into the
+following categories:
+
+- the worker chooses to exit
+- an unrecoverable exception happens within the worker
+- the worker process is shut down by some external action
+
+Each of these cases will be described in more detail below. The *symptoms* you will
+experience when these things happen range from simply work not getting done anymore,
+to various exceptions appearing when you interact with your local client, such as
+``KilledWorker``, ``TimeoutError`` and ``CommClosedError``.
+
+Note the special case of ``KilledWorker``: this means that a particular task was
+tried on a worker, and it died, and then the same task was sent to another worker,
+which also died. After a configurable number of deaths (config key "
+``distributed.scheduler.allowed-failures``), Dask decides to blame the
+task itself, and returns this exception. Note, that it is possible for a task to be
+unfairly blamed - the worker happened to die while the task was active, perhaps
+due to another thread - complicating diagnosis.
+
+In every case, the first place to look for further information is the logs of
+the given worker, which may well give a complete description of what happened. These
+logs are printed by the worker to its "standard error", which may appear in the text
+console from which you launched the worker, or some logging system maintained by
+the cluster infrastructure. It is also helpful to watch the diagnostic dashboard to
+look for memory spikes, but of course this is only possible while the worker is still
+alive.
+
+In all cases, the scheduler will notice that the worker has gone, either because
+of an explicit de-registration, or because the worker no longer produces heartbeats,
+and it should be possible to reroute tasks to other workers and have the system
+keep running.
+
+Scenarios
+~~~~~~~~~
+
+Worker chose to exit
+''''''''''''''''''''
+
+Workers may exit in normal functioning because they have been asked to, e.g.,
+they received a keyboard interrupt (^C), or the scheduler scaled down the cluster.
+In such cases, the work that was being done by the worker will be redirected to
+other workers, if there are any left.
+
+You should expect to see the following message at the end of the worker's log:
+
+::
+
+   distributed.dask_worker - INFO - End worker
+
+In these cases, there is not normally anything which you need to do, since the
+behaviour is expected.
+
+Unrecoverable Exception
+'''''''''''''''''''''''
+
+The worker is a python process, and like any other code, an exception may occur
+which causes the process to exit. One typical example of this might be a
+version mismatch between the packages of the client and worker, so that
+a message sent to the worker errors while being unpacked. There are a number of
+packages that need to match, not only ``dask`` and ``distributed``.
+
+In this case, you should expect to see the full python traceback in the worker's
+log. In the event of a version mismatch, this might be complaining about a bad
+import or missing attribute. However, other fatal exceptions are also possible,
+such as trying to allocate more memory than the system has available, or writing
+temporary files without appropriate permissions.
+
+To assure that you have matching versions, you should run (more recent versions
+of distributed may do this automatically)
+
+.. code-block::
+
+   client.get_versions(check=True)
+
+For other errors, you might want to run the computation in your local client, if
+possible, or try grabbing just the task that errored and using
+:func:`recreate_error_locally <distributed.recreate_exceptions.ReplayExceptionClient.recreate_error_locally>`,
+as you would for ordinary exceptions happening during task execution.
+
+Specifically for connectivity problems (e.g., timeout exceptions in the worker
+logs), you will need to diagnose your networking infrastructure, which is more
+complicated than can be described here. Commonly, it may involve logging into
+the machine running the affected worker
+(although you can :ref:`ipylaunch`).
+
+Killed by Nanny
+'''''''''''''''
+
+The Dask "nanny" is a process which watches the worker, and restarts it if
+necessary. It also tracks the worker's memory usage, and if it should cross
+a given fraction of total memory, then also the worker will be restarted,
+interrupting any work in progress. The log will show a message like
+
+::
+
+    Worker exceeded X memory budget. Restarting
+
+Where X is the memory fraction. You can set this critical fraction using
+the configuration, see :ref:`memman`. If you have an external system for
+watching memory usage provided by your cluster infrastructure (HPC,
+kubernetes, etc.), then it may be reasonable to turn off this memory
+limit. Indeed, in these cases, restarts might be handled for you too, so
+you could do without the nanny at all (``--no-nanny`` CLI option or
+configuration equivalent).
+
+Sudden Exit
+'''''''''''
+
+The worker process may stop working without notice. This can happen due to
+something internal to the worker, e.g., a memory violation (common if interfacing
+with compiled code), or due to something external, e.g., the ``kill`` command, or
+stopping of the container or machine on which the worker is running.
+
+In the best case, you may have a line in the logs from the OS saying that the
+worker was shut down, such as the single word "killed"  or something more descriptive.
+In these cases, the fault may well be in your code, and you might be able to use the
+same debugging tools as in the previous section.
+
+However, if the action was initiated by some outside framework, then the worker will
+have no time to leave a logging message, and the death *may* have nothing to do with
+what the worker was doing at the time. For example, if kubernetes decides to evict a
+pod, or your ec2 instance goes down for maintenance, the worker is not at fault.
+Hopefully, the system provides a reasonable message of what happened in the process
+output.
+However, if the memory allocation (or other resource) exceeds toleration, then it
+*is* the code's fault - although you may be able to fix with better configuration
+of Dask's own limits, or simply with a bigger cluster. In any case, your deployment
+framework has its own logging system, and you should look there for the reason that
+the dask worker was taken down.
+
+Specifically for memory issues, refer to the memory section of `best practices`_.
+
+.. _best practices: https://docs.dask.org/en/latest/best-practices.html#avoid-very-large-partitions
diff --git a/docs/source/worker.rst b/docs/source/worker.rst
index 5ff66b613a6..dc4e56d9ac7 100644
--- a/docs/source/worker.rst
+++ b/docs/source/worker.rst
@@ -142,6 +142,7 @@ thread pool.
 A task either errs or its result is put into memory.  In either case a response
 is sent back to the scheduler.
 
+.. _memman::
 
 Memory Management
 -----------------

From dedcb1350e6106a21ddcdad1edaa94b346b3e9d8 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 31 Mar 2020 08:15:56 -0700
Subject: [PATCH 0762/1550] Clean up some test warnings (#3662)

* remove errant yield in variable tests

* logs -> get_logs

* add additional awaits
---
 distributed/deploy/tests/test_spec_cluster.py | 10 +++++-----
 distributed/tests/test_client.py              |  9 ++++-----
 distributed/tests/test_core.py                |  2 +-
 distributed/tests/test_variable.py            |  2 +-
 4 files changed, 11 insertions(+), 12 deletions(-)

diff --git a/distributed/deploy/tests/test_spec_cluster.py b/distributed/deploy/tests/test_spec_cluster.py
index 90ce9923c69..ae24e7400e2 100644
--- a/distributed/deploy/tests/test_spec_cluster.py
+++ b/distributed/deploy/tests/test_spec_cluster.py
@@ -278,7 +278,7 @@ async def test_logs(cleanup):
         cluster.scale(2)
         await cluster
 
-        logs = await cluster.logs()
+        logs = await cluster.get_logs()
         assert is_valid_xml("<div>" + logs._repr_html_() + "</div>")
         assert "Scheduler" in logs
         for worker in cluster.scheduler.workers:
@@ -286,17 +286,17 @@ async def test_logs(cleanup):
 
         assert "Registered" in str(logs)
 
-        logs = await cluster.logs(scheduler=True, workers=False)
+        logs = await cluster.get_logs(scheduler=True, workers=False)
         assert list(logs) == ["Scheduler"]
 
-        logs = await cluster.logs(scheduler=False, workers=False)
+        logs = await cluster.get_logs(scheduler=False, workers=False)
         assert list(logs) == []
 
-        logs = await cluster.logs(scheduler=False, workers=True)
+        logs = await cluster.get_logs(scheduler=False, workers=True)
         assert set(logs) == set(cluster.scheduler.workers)
 
         w = toolz.first(cluster.scheduler.workers)
-        logs = await cluster.logs(scheduler=False, workers=[w])
+        logs = await cluster.get_logs(scheduler=False, workers=[w])
         assert set(logs) == {w}
 
 
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 8f121c7f27c..1ce1c1dfa6f 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -5465,12 +5465,11 @@ def test_tuple_keys(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_multiple_scatter(c, s, a, b):
-    for i in range(5):
-        x = c.scatter(1, direct=True)
+async def test_multiple_scatter(c, s, a, b):
+    futures = await asyncio.gather(*[c.scatter(1, direct=True) for _ in range(5)])
 
-    x = yield x
-    x = yield x
+    x = await futures[0]
+    x = await futures[0]
 
 
 @gen_cluster(client=True)
diff --git a/distributed/tests/test_core.py b/distributed/tests/test_core.py
index 76f2b285500..b4993bb4ce8 100644
--- a/distributed/tests/test_core.py
+++ b/distributed/tests/test_core.py
@@ -781,4 +781,4 @@ async def test_connection_pool_detects_remote_close():
     # check connection pool invariants hold even after it detects a closed connection
     # while creating conn2:
     p._validate()
-    p.close()
+    await p.close()
diff --git a/distributed/tests/test_variable.py b/distributed/tests/test_variable.py
index 6e3b3bcdad6..64765d808c7 100644
--- a/distributed/tests/test_variable.py
+++ b/distributed/tests/test_variable.py
@@ -112,7 +112,7 @@ def test_timeout_sync(client):
         assert 0.2 < stop - start < 2.0
 
     with pytest.raises(TimeoutError):
-        yield v.get(timeout=0.01)
+        v.get(timeout=0.01)
 
 
 @gen_cluster(client=True)

From 10e6018c737d33cc46a14778ca42ce2e0d26d434 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Tue, 31 Mar 2020 21:24:34 -0500
Subject: [PATCH 0763/1550] Replace ncores with nthreads in work stealing tests
 (#3615)

---
 distributed/tests/test_steal.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/tests/test_steal.py b/distributed/tests/test_steal.py
index 0ed9051cc95..1c9fe22e2e8 100644
--- a/distributed/tests/test_steal.py
+++ b/distributed/tests/test_steal.py
@@ -321,7 +321,7 @@ def test_steal_host_restrictions(c, s, wa, wb):
     assert len(wa.task_state) == ntasks
     assert len(wb.task_state) == 0
 
-    wc = yield Worker(s.address, ncores=1)
+    wc = yield Worker(s.address, nthreads=1)
 
     start = time()
     while not wc.task_state or len(wa.task_state) == ntasks:

From 2129b740c1e3f524e5ba40a0b6a77b239d4c1f94 Mon Sep 17 00:00:00 2001
From: Lucas Rademaker <44430780+lr4d@users.noreply.github.com>
Date: Wed, 1 Apr 2020 08:48:31 +0200
Subject: [PATCH 0764/1550] Add Semaphore extension (#3573)

The complexity of the internal structure comes in since we do not support any notion of an ephemeral key, i.e. a value which expires together with the session. In this context this translates best to the Client. Therefore, the Semaphore tracks which lease stems from which Semaphore client instance and stores its associated Client ID. If the client is lost/closed, the semaphore will release all it's acquired values eventually.

This behavior is quite important for resilience: If a worker is shut down ungracefully, all acquired leases should be released eventually since otherwise we may cause a deadlock.

gh3573 and gh2690
---
 distributed/__init__.py             |   1 +
 distributed/distributed.yaml        |   2 +
 distributed/scheduler.py            |   2 +
 distributed/semaphore.py            | 341 ++++++++++++++++++++++++++++
 distributed/tests/test_semaphore.py | 246 ++++++++++++++++++++
 distributed/worker.py               |   2 +-
 6 files changed, 593 insertions(+), 1 deletion(-)
 create mode 100644 distributed/semaphore.py
 create mode 100644 distributed/tests/test_semaphore.py

diff --git a/distributed/__init__.py b/distributed/__init__.py
index be750f9daed..608e23a58e3 100644
--- a/distributed/__init__.py
+++ b/distributed/__init__.py
@@ -22,6 +22,7 @@
 from .nanny import Nanny
 from .pubsub import Pub, Sub
 from .queues import Queue
+from .semaphore import Semaphore
 from .scheduler import Scheduler
 from .threadpoolexecutor import rejoin
 from .utils import sync, TimeoutError, CancelledError
diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index 17326aebd54..f4bfc7d76ab 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -38,6 +38,8 @@ distributed:
         ca-file: null
         key: null
         cert: null
+    locks:
+      lease-validation-interval: 10s
 
   worker:
     blocked-handlers: []
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 882970838df..bc6c0ea0fd5 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -76,6 +76,7 @@
 
 from .publish import PublishExtension
 from .queues import QueueExtension
+from .semaphore import SemaphoreExtension
 from .recreate_exceptions import ReplayExceptionScheduler
 from .lock import LockExtension
 from .pubsub import PubSubSchedulerExtension
@@ -96,6 +97,7 @@
     QueueExtension,
     VariableExtension,
     PubSubSchedulerExtension,
+    SemaphoreExtension,
 ]
 
 ALL_TASK_STATES = {"released", "waiting", "no-worker", "processing", "erred", "memory"}
diff --git a/distributed/semaphore.py b/distributed/semaphore.py
new file mode 100644
index 00000000000..23f253b7ede
--- /dev/null
+++ b/distributed/semaphore.py
@@ -0,0 +1,341 @@
+import uuid
+from collections import defaultdict, deque
+from functools import partial
+import asyncio
+import dask
+from asyncio import TimeoutError
+from .utils import PeriodicCallback, log_errors, parse_timedelta
+from .worker import get_client
+from .metrics import time
+import warnings
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class _Watch:
+    def __init__(self, duration=None):
+        self.duration = duration
+        self.started_at = None
+
+    def start(self):
+        self.started_at = time()
+
+    def leftover(self):
+        if self.duration is None:
+            return None
+        else:
+            elapsed = time() - self.started_at
+            return max(0, self.duration - elapsed)
+
+
+class SemaphoreExtension:
+    """ An extension for the scheduler to manage Semaphores
+
+    This adds the following routes to the scheduler
+
+    * semaphore_acquire
+    * semaphore_release
+    * semaphore_create
+    """
+
+    def __init__(self, scheduler):
+        self.scheduler = scheduler
+        self.leases = defaultdict(deque)
+        self.events = defaultdict(asyncio.Event)
+        self.max_leases = dict()
+        self.leases_per_client = defaultdict(partial(defaultdict, deque))
+        self.scheduler.handlers.update(
+            {
+                "semaphore_create": self.create,
+                "semaphore_acquire": self.acquire,
+                "semaphore_release": self.release,
+                "semaphore_close": self.close,
+            }
+        )
+
+        self.scheduler.extensions["semaphores"] = self
+        self.pc_validate_leases = PeriodicCallback(
+            self._validate_leases,
+            1000
+            * parse_timedelta(
+                dask.config.get(
+                    "distributed.scheduler.locks.lease-validation-interval"
+                ),
+                default="s",
+            ),
+            io_loop=self.scheduler.loop,
+        )
+        self.pc_validate_leases.start()
+        self._validation_running = False
+
+    # `comm` here is required by the handler interface
+    def create(self, comm=None, name=None, max_leases=None):
+        # We use `self.max_leases.keys()` as the point of truth to find out if a semaphore with a specific
+        # `name` has been created.
+        if name not in self.max_leases:
+            assert isinstance(max_leases, int), max_leases
+            self.max_leases[name] = max_leases
+        else:
+            if max_leases != self.max_leases[name]:
+                raise ValueError(
+                    "Inconsistent max leases: %s, expected: %s"
+                    % (max_leases, self.max_leases[name])
+                )
+
+    async def _get_lease(self, client, name, identifier):
+        result = True
+        if len(self.leases[name]) < self.max_leases[name]:
+            # naive: self.leases[resource] += 1
+            # not naive:
+            self.leases[name].append(identifier)
+            self.leases_per_client[client][name].append(identifier)
+        else:
+            result = False
+        return result
+
+    def _semaphore_exists(self, name):
+        if name not in self.max_leases:
+            return False
+        return True
+
+    async def acquire(
+        self, comm=None, name=None, client=None, timeout=None, identifier=None
+    ):
+        with log_errors():
+            if not self._semaphore_exists(name):
+                raise RuntimeError(f"Semaphore `{name}` not known or already closed.")
+
+            if isinstance(name, list):
+                name = tuple(name)
+            w = _Watch(timeout)
+            w.start()
+
+            while True:
+                # Reset the event and try to get a release. The event will be set if the state
+                # is changed and helps to identify when it is worth to retry an acquire
+                self.events[name].clear()
+
+                # If we hit the timeout, this cancels the _get_lease
+                future = asyncio.wait_for(
+                    self._get_lease(client, name, identifier), timeout=w.leftover()
+                )
+
+                try:
+                    result = await future
+                except TimeoutError:
+                    result = False
+
+                # If acquiring fails, we wait for the event to be set, i.e. something has
+                # been released and we can try to acquire again (continue loop)
+                if not result:
+                    future = asyncio.wait_for(
+                        self.events[name].wait(), timeout=w.leftover()
+                    )
+                    try:
+                        await future
+                        continue
+                    except TimeoutError:
+                        result = False
+                return result
+
+    def release(self, comm=None, name=None, client=None, identifier=None):
+        with log_errors():
+            if not self._semaphore_exists(name):
+                logger.warning(
+                    f"Tried to release semaphore `{name}` but it is not known or already closed."
+                )
+                return
+            if isinstance(name, list):
+                name = tuple(name)
+            if name in self.leases and identifier in self.leases[name]:
+                self._release_value(name, client, identifier)
+            else:
+                raise ValueError(
+                    f"Tried to release semaphore but it was already released: "
+                    f"client={client}, name={name}, identifier={identifier}"
+                )
+
+    def _release_value(self, name, client, identifier):
+        # Everything needs to be atomic here.
+        self.leases_per_client[client][name].remove(identifier)
+        self.leases[name].remove(identifier)
+        self.events[name].set()
+
+    def _release_client(self, client):
+        semaphore_names = list(self.leases_per_client[client])
+        for name in semaphore_names:
+            ids = list(self.leases_per_client[client][name])
+            for _id in list(ids):
+                self._release_value(name=name, client=client, identifier=_id)
+        del self.leases_per_client[client]
+
+    def _validate_leases(self):
+        if not self._validation_running:
+            self._validation_running = True
+            known_clients_with_leases = set(self.leases_per_client.keys())
+            scheduler_clients = set(self.scheduler.clients.keys())
+            for dead_client in known_clients_with_leases - scheduler_clients:
+                self._release_client(dead_client)
+            else:
+                self._validation_running = False
+
+    def close(self, comm=None, name=None):
+        """Hard close the semaphore without warning clients which still hold a lease."""
+        with log_errors():
+            if not self._semaphore_exists(name):
+                return
+
+            del self.max_leases[name]
+            if name in self.events:
+                del self.events[name]
+            if name in self.leases:
+                del self.leases[name]
+
+            for client, client_leases in self.leases_per_client.items():
+                if name in client_leases:
+                    warnings.warn(
+                        f"Closing semaphore `{name}` but client `{client}` still has a lease open.",
+                        RuntimeWarning,
+                    )
+                    del client_leases[name]
+
+
+class Semaphore:
+    """ Semaphore
+
+    Parameters
+    ----------
+    max_leases: int (optional)
+        The maximum amount of leases that may be granted at the same time. This
+        effectively sets an upper limit to the amount of parallel access to a specific resource.
+        Defaults to 1.
+    name: string (optional)
+        Name of the semaphore to acquire.  Choosing the same name allows two
+        disconnected processes to coordinate.  If not given, a random
+        name will be generated.
+    client: Client (optional)
+        Client to use for communication with the scheduler.  If not given, the
+        default global client will be used.
+
+    Examples
+    --------
+    >>> from distributed import Semaphore
+    >>> sem = Semaphore(max_leases=2, name='my_database')
+    >>> def access_resource(s, sem):
+    >>>     # This automatically acquires a lease from the semaphore (if available) which will be
+    >>>     # released when leaving the context manager.
+    >>>     with sem:
+    >>>         pass
+    >>>
+    >>> futures = client.map(access_resource, range(10), sem=sem)
+    >>> client.gather(futures)
+    >>> # Once done, close the semaphore to clean up the state on scheduler side.
+    >>> sem.close()
+
+    Notes
+    -----
+    If a client attempts to release the semaphore but doesn't have a lease acquired, this will raise an exception.
+
+
+    When a semaphore is closed, if, for that closed semaphore, a client attempts to:
+
+    - Acquire a lease: an exception will be raised.
+    - Release: a warning will be logged.
+    - Close: nothing will happen.
+
+
+    dask executes functions by default assuming they are pure, when using semaphore acquire/releases inside
+    such a function, it must be noted that there *are* in fact side-effects, thus, the function can no longer be
+    considered pure. If this is not taken into account, this may lead to unexpected behavior.
+
+    """
+
+    def __init__(self, max_leases=1, name=None, client=None):
+        # NOTE: the `id` of the `Semaphore` instance will always be unique, even among different
+        # instances for the same resource. The actual attribute that identifies a specific resource is `name`,
+        # which will be the same for all instances of this class which limit the same resource.
+        self.client = client or get_client()
+        self.id = uuid.uuid4().hex
+        self.name = name or "semaphore-" + uuid.uuid4().hex
+        self.max_leases = max_leases
+
+        if self.client.asynchronous:
+            self._started = self.client.scheduler.semaphore_create(
+                name=self.name, max_leases=max_leases
+            )
+        else:
+            self.client.sync(
+                self.client.scheduler.semaphore_create,
+                name=self.name,
+                max_leases=max_leases,
+            )
+            self._started = asyncio.sleep(0)
+
+    def __await__(self):
+        async def create_semaphore():
+            await self._started
+            return self
+
+        return create_semaphore().__await__()
+
+    def acquire(self, timeout=None):
+        """
+        Acquire a semaphore.
+
+        If the internal counter is greater than zero, decrement it by one and return True immediately.
+        If it is zero, wait until a release() is called and return True.
+        """
+        # TODO: This (may?) keep the HTTP request open until timeout runs out (forever if None).
+        #  Can do this in batches of smaller timeouts.
+        # TODO: what if connection breaks up?
+        return self.client.sync(
+            self.client.scheduler.semaphore_acquire,
+            name=self.name,
+            timeout=timeout,
+            client=self.client.id,
+            identifier=self.id,
+        )
+
+    def release(self):
+        """
+        Release a semaphore.
+
+        Increment the internal counter by one.
+        """
+
+        """ Release the lock if already acquired """
+        # TODO: what if connection breaks up?
+        return self.client.sync(
+            self.client.scheduler.semaphore_release,
+            name=self.name,
+            client=self.client.id,
+            identifier=self.id,
+        )
+
+    def __enter__(self):
+        self.acquire()
+        return self
+
+    def __exit__(self, *args, **kwargs):
+        self.release()
+
+    async def __aenter__(self):
+        await self.acquire()
+        return self
+
+    async def __aexit__(self, *args, **kwargs):
+        await self.release()
+
+    def __getstate__(self):
+        # Do not serialize the address since workers may have different
+        # addresses for the scheduler (e.g. if a proxy is between them)
+        return (self.name, self.max_leases)
+
+    def __setstate__(self, state):
+        name, max_leases = state
+        client = get_client()
+        self.__init__(name=name, client=client, max_leases=max_leases)
+
+    def close(self):
+        return self.client.sync(self.client.scheduler.semaphore_close, name=self.name)
diff --git a/distributed/tests/test_semaphore.py b/distributed/tests/test_semaphore.py
new file mode 100644
index 00000000000..9d94b83515a
--- /dev/null
+++ b/distributed/tests/test_semaphore.py
@@ -0,0 +1,246 @@
+import pickle
+
+import dask
+from dask.distributed import Client
+
+from distributed import Semaphore
+from distributed.metrics import time
+from distributed.utils_test import cluster, gen_cluster
+from distributed.utils_test import client, loop, cluster_fixture  # noqa: F401
+import pytest
+
+
+@gen_cluster(client=True)
+async def test_semaphore(c, s, a, b):
+    semaphore = await Semaphore(max_leases=2, name="resource_we_want_to_limit")
+
+    result = await semaphore.acquire()  # allowed_leases: 2 - 1 -> 1
+    assert result is True
+
+    second = await semaphore.acquire()  # allowed_leases: 1 - 1 -> 0
+    assert second is True
+    start = time()
+    result = await semaphore.acquire(timeout=0.025)  # allowed_leases: 0 -> False
+    stop = time()
+    assert stop - start < 0.2
+    assert result is False
+
+
+@gen_cluster(client=True)
+async def test_serializable(c, s, a, b):
+    sem = await Semaphore(max_leases=2, name="x")
+    res = await sem.acquire()
+    assert len(s.extensions["semaphores"].leases["x"]) == 1
+    assert res
+    sem2 = pickle.loads(pickle.dumps(sem))
+    assert sem2.name == sem.name
+    assert sem2.client.scheduler.address == sem.client.scheduler.address
+
+    # actual leases didn't change
+    assert len(s.extensions["semaphores"].leases["x"]) == 1
+
+    res = await sem2.acquire()
+    assert res
+    assert len(s.extensions["semaphores"].leases["x"]) == 2
+
+    # Ensure that both objects access the same semaphore
+    res = await sem.acquire(timeout=0.025)
+
+    assert not res
+    res = await sem2.acquire(timeout=0.025)
+
+    assert not res
+
+
+@gen_cluster(client=True)
+async def test_release_simple(c, s, a, b):
+    def f(x, semaphore):
+        with semaphore:
+            assert semaphore.name == "x"
+            return x + 1
+
+    sem = await Semaphore(max_leases=2, name="x")
+    futures = c.map(f, list(range(10)), semaphore=sem)
+    await c.gather(futures)
+
+
+@gen_cluster(client=True)
+async def test_acquires_with_timeout(c, s, a, b):
+    sem = await Semaphore(1, "x")
+    assert await sem.acquire(timeout=0.025)
+    assert not await sem.acquire(timeout=0.025)
+    await sem.release()
+    assert await sem.acquire(timeout=0.025)
+    await sem.release()
+
+
+def test_timeout_sync(client):
+    s = Semaphore(name="x")
+    # Using the context manager already acquires a lease, so the line below won't be able to acquire another one
+    with s:
+        assert s.acquire(timeout=0.025) is False
+
+
+@pytest.mark.slow
+@gen_cluster(client=True, timeout=20)
+async def test_release_semaphore_after_timeout(c, s, a, b):
+    with dask.config.set(
+        {"distributed.scheduler.locks.lease-validation-interval": "50ms"}
+    ):
+        sem = await Semaphore(name="x", max_leases=2)
+        await sem.acquire()  # leases: 2 - 1 = 1
+        semY = await Semaphore(name="y")
+
+        async with Client(s.address, asynchronous=True, name="ClientB") as clientB:
+            semB = await Semaphore(name="x", max_leases=2, client=clientB)
+            semYB = await Semaphore(name="y", client=clientB)
+
+            assert await semB.acquire()  # leases: 1 - 1 = 0
+            assert await semYB.acquire()
+
+            assert not (await sem.acquire(timeout=0.01))
+            assert not (await semB.acquire(timeout=0.01))
+            assert not (await semYB.acquire(timeout=0.01))
+
+        # `ClientB` goes out of scope, leases should be released
+        # At this point, we should be able to acquire x and y once
+        assert await sem.acquire()
+        assert await semY.acquire()
+
+        assert not (await semY.acquire(timeout=0.01))
+        assert not (await sem.acquire(timeout=0.01))
+
+        assert clientB.id not in s.extensions["semaphores"].leases_per_client
+
+
+@gen_cluster()
+async def test_async_ctx(s, a, b):
+    sem = await Semaphore(name="x")
+    async with sem:
+        assert not await sem.acquire(timeout=0.025)
+    assert await sem.acquire()
+
+
+@pytest.mark.slow
+def test_worker_dies():
+    with cluster(disconnect_timeout=10) as (scheduler, workers):
+        with Client(scheduler["address"]) as client:
+            sem = Semaphore(name="x", max_leases=1)
+
+            def f(x, sem, kill_address):
+                with sem:
+                    from distributed.worker import get_worker
+
+                    worker = get_worker()
+                    if worker.address == kill_address:
+                        import os
+
+                        os.kill(os.getpid(), 15)
+                    return x
+
+            futures = client.map(
+                f, range(100), sem=sem, kill_address=workers[0]["address"]
+            )
+            results = client.gather(futures)
+
+            assert sorted(results) == list(range(100))
+
+
+@gen_cluster(client=True)
+async def test_access_semaphore_by_name(c, s, a, b):
+    def f(x, release=True):
+        sem = Semaphore(name="x")
+        if not sem.acquire(timeout=0.1):
+            return False
+        if release:
+            sem.release()
+
+        return True
+
+    sem = await Semaphore(name="x")
+    futures = c.map(f, list(range(10)))
+    assert all(await c.gather(futures))
+
+    # Clean-up the state, otherwise we would get the same result when calling `f` with the same arguments
+    del futures
+
+    assert len(s.extensions["semaphores"].leases["x"]) == 0
+    assert await sem.acquire()
+    assert len(s.extensions["semaphores"].leases["x"]) == 1
+    futures = c.map(f, list(range(10)))
+    assert not any(await c.gather(futures))
+    await sem.release()
+
+    del futures
+
+    futures = c.map(f, list(range(10)), release=False)
+    result = await c.gather(futures)
+    assert result.count(True) == 1
+    assert result.count(False) == 9
+
+
+@gen_cluster(client=True)
+async def test_close_async(c, s, a, b):
+    sem = await Semaphore(name="test")
+
+    assert await sem.acquire()
+    with pytest.warns(
+        RuntimeWarning, match="Closing semaphore .* but client .* still has a lease"
+    ):
+        await sem.close()
+
+    with pytest.raises(
+        RuntimeError, match="Semaphore `test` not known or already closed."
+    ):
+        await sem.acquire()
+
+    semaphore_object = s.extensions["semaphores"]
+    assert not semaphore_object.max_leases
+    assert not semaphore_object.leases
+    assert not semaphore_object.events
+    assert not any(semaphore_object.leases_per_client.values())
+
+
+def test_close_sync(client):
+    sem = Semaphore()
+    sem.close()
+
+    with pytest.raises(RuntimeError, match="Semaphore .* not known or already closed."):
+        sem.acquire()
+
+
+@gen_cluster(client=True)
+async def test_release_once_too_many(c, s, a, b):
+    sem = await Semaphore(name="x")
+    assert await sem.acquire()
+    await sem.release()
+
+    with pytest.raises(
+        ValueError, match="Tried to release semaphore but it was already released"
+    ):
+        await sem.release()
+
+    assert await sem.acquire()
+    await sem.release()
+
+
+@gen_cluster(client=True)
+async def test_release_once_too_many_resilience(c, s, a, b):
+    def f(x, sem):
+        sem.acquire()
+        sem.release()
+        with pytest.raises(
+            ValueError, match="Tried to release semaphore but it was already released"
+        ):
+            sem.release()
+        return x
+
+    sem = await Semaphore(max_leases=3, name="x")
+
+    inpt = list(range(20))
+    futures = c.map(f, inpt, sem=sem)
+    assert sorted(await c.gather(futures)) == inpt
+
+    assert not s.extensions["semaphores"].leases["x"]
+    await sem.acquire()
+    assert len(s.extensions["semaphores"].leases["x"]) == 1
diff --git a/distributed/worker.py b/distributed/worker.py
index ba25c91d979..c6ae63f0ef1 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -1160,7 +1160,7 @@ async def close_gracefully(self):
         await self.scheduler.retire_workers(workers=[self.address], remove=False)
         await self.close(safe=True, nanny=not self.lifetime_restart)
 
-    async def terminate(self, comm, report=True, **kwargs):
+    async def terminate(self, comm=None, report=True, **kwargs):
         await self.close(report=report, **kwargs)
         return "OK"
 

From 66fe0acf041add720d5cc9d9d1ed99d9e90e7e2e Mon Sep 17 00:00:00 2001
From: Prasun Anand <prasunanand.bitsp@gmail.com>
Date: Wed, 1 Apr 2020 20:56:23 +0530
Subject: [PATCH 0765/1550] Add Resouces option to get_task_stream and call
 output_file (#3653)

---
 distributed/client.py | 29 +++++++++++++++++++++++------
 1 file changed, 23 insertions(+), 6 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index f2b25b74f60..d42c29c2314 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -3367,9 +3367,10 @@ async def _profile(
                 filename = "dask-profile.html"
 
             if filename:
-                from bokeh.plotting import save
+                from bokeh.plotting import output_file, save
 
-                save(figure, title="Dask Profile", filename=filename)
+                output_file(filename=filename, title="Dask Profile")
+                save(figure, filename=filename)
             return (state, figure)
 
         else:
@@ -3852,7 +3853,13 @@ def collections_to_dsk(collections, *args, **kwargs):
         return collections_to_dsk(collections, *args, **kwargs)
 
     def get_task_stream(
-        self, start=None, stop=None, count=None, plot=False, filename="task-stream.html"
+        self,
+        start=None,
+        stop=None,
+        count=None,
+        plot=False,
+        filename="task-stream.html",
+        bokeh_resources=None,
     ):
         """ Get task stream data from scheduler
 
@@ -3881,6 +3888,8 @@ def get_task_stream(
             If plot == 'save' then save the figure to a file
         filename: str (optional)
             The filename to save to if you set ``plot='save'``
+        bokeh_resources: bokeh.resources.Resources (optional)
+            Specifies if the resource component is INLINE or CDN
 
         Examples
         --------
@@ -3920,10 +3929,17 @@ def get_task_stream(
             count=count,
             plot=plot,
             filename=filename,
+            bokeh_resources=bokeh_resources,
         )
 
     async def _get_task_stream(
-        self, start=None, stop=None, count=None, plot=False, filename="task-stream.html"
+        self,
+        start=None,
+        stop=None,
+        count=None,
+        plot=False,
+        filename="task-stream.html",
+        bokeh_resources=None,
     ):
         msgs = await self.scheduler.get_task_stream(start=start, stop=stop, count=count)
         if plot:
@@ -3935,9 +3951,10 @@ async def _get_task_stream(
             source, figure = task_stream_figure(sizing_mode="stretch_both")
             source.data.update(rects)
             if plot == "save":
-                from bokeh.plotting import save
+                from bokeh.plotting import save, output_file
 
-                save(figure, title="Dask Task Stream", filename=filename)
+                output_file(filename=filename, title="Dask Task Stream")
+                save(figure, filename=filename, resources=bokeh_resources)
             return (msgs, figure)
         else:
             return msgs

From 09f86837e9b908b43991f6b4159f0083ffeb799b Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Thu, 2 Apr 2020 07:36:35 -0700
Subject: [PATCH 0766/1550] Get CUDA context to finalize Numba `DeviceNDArray`
 (#3666)

* Ensure CUDA context on `DeviceNDArray` cleanup

As the CUDA context does not seem to always be established when cleaning
up Numba `DeviceNDArray`s, make sure to get the context right before
cleanup to ensure the context is available.

This is done when allocating new CUDA frames to when using UCX. Make
sure this is handled both for Numba and old versions of RMM that also
use Numba under-the-hood. Newer versions of RMM don't need this.

Also do this when performing a host-to-device transfer where a Numba
`DeviceNDArray` is the first object created on device.

* Drop `PatchedCudaArrayInterface` and usage thereof

This was used to ensure that Numba had acquired a CUDA context before
finalizing any Numba `DeviceNDArray` objects that might be used to back
a CuPy `ndarray`. As we now make sure of this when creating Numba
`DeviceNDArray` by using `weakref.finalize`, this should be correctly
handled for all CUDA objects backed by Numba `DeviceNDArray`s (not just
CuPy `ndarray`s). So there should be no need to do this for CuPy
`ndarray`s separately. Should simplify the CuPy serialization and
cleanup paths a bit.
---
 distributed/comm/ucx.py       | 17 +++++++++++++++--
 distributed/protocol/cupy.py  | 27 ---------------------------
 distributed/protocol/numba.py |  4 ++++
 3 files changed, 19 insertions(+), 29 deletions(-)

diff --git a/distributed/comm/ucx.py b/distributed/comm/ucx.py
index a29441ec4d5..4e6ca8116c8 100644
--- a/distributed/comm/ucx.py
+++ b/distributed/comm/ucx.py
@@ -6,6 +6,7 @@
 .. _UCX: https://github.com/openucx/ucx
 """
 import logging
+import weakref
 
 import dask
 import numpy as np
@@ -65,12 +66,24 @@ def init_once():
         if hasattr(rmm, "DeviceBuffer"):
             cuda_array = lambda n: rmm.DeviceBuffer(size=n)
         else:  # pre-0.11.0
-            cuda_array = lambda n: rmm.device_array(n, dtype=np.uint8)
+            import numba.cuda
+
+            def rmm_cuda_array(n):
+                a = rmm.device_array(n, dtype=np.uint8)
+                weakref.finalize(a, numba.cuda.current_context)
+                return a
+
+            cuda_array = rmm_cuda_array
     except ImportError:
         try:
             import numba.cuda
 
-            cuda_array = lambda n: numba.cuda.device_array((n,), dtype=np.uint8)
+            def numba_cuda_array(n):
+                a = numba.cuda.device_array((n,), dtype=np.uint8)
+                weakref.finalize(a, numba.cuda.current_context)
+                return a
+
+            cuda_array = numba_cuda_array
         except ImportError:
 
             def cuda_array(n):
diff --git a/distributed/protocol/cupy.py b/distributed/protocol/cupy.py
index b3465fee424..0a2c53be4a5 100644
--- a/distributed/protocol/cupy.py
+++ b/distributed/protocol/cupy.py
@@ -14,31 +14,6 @@
     from .numba import dask_deserialize_numba_array as dask_deserialize_cuda_buffer
 
 
-class PatchedCudaArrayInterface:
-    """This class does one thing:
-        1) Makes sure that the cuda context is active
-           when deallocating the base cuda array.
-        Notice, this is only needed when the array to deserialize
-        isn't a native cupy array.
-    """
-
-    def __init__(self, ary):
-        self.__cuda_array_interface__ = ary.__cuda_array_interface__
-        # Save a ref to ary so it won't go out of scope
-        self.base = ary
-
-    def __del__(self):
-        # Making sure that the cuda context is active
-        # when deallocating the base cuda array
-        try:
-            import numba.cuda
-
-            numba.cuda.current_context()
-        except ImportError:
-            pass
-        del self.base
-
-
 @cuda_serialize.register(cupy.ndarray)
 def cuda_serialize_cupy_ndarray(x):
     # Making sure `x` is behaving
@@ -60,8 +35,6 @@ def cuda_serialize_cupy_ndarray(x):
 @cuda_deserialize.register(cupy.ndarray)
 def cuda_deserialize_cupy_ndarray(header, frames):
     (frame,) = frames
-    if not isinstance(frame, cupy.ndarray):
-        frame = PatchedCudaArrayInterface(frame)
     arr = cupy.ndarray(
         shape=header["shape"],
         dtype=header["typestr"],
diff --git a/distributed/protocol/numba.py b/distributed/protocol/numba.py
index 03bf4aa9f16..20eec8e11b6 100644
--- a/distributed/protocol/numba.py
+++ b/distributed/protocol/numba.py
@@ -1,3 +1,5 @@
+import weakref
+
 import numba.cuda
 import numpy as np
 
@@ -59,6 +61,8 @@ def dask_deserialize_numba_array(header, frames):
         frames = [dask_deserialize_rmm_device_buffer(header, frames)]
     else:
         frames = [numba.cuda.to_device(np.asarray(memoryview(f))) for f in frames]
+        for f in frames:
+            weakref.finalize(f, numba.cuda.current_context)
 
     arr = cuda_deserialize_numba_ndarray(header, frames)
     return arr

From c05899e01189d4bbabeb53d46c37f5d7cdde5eb8 Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Thu, 2 Apr 2020 20:28:06 +0200
Subject: [PATCH 0767/1550] More documentation for Semaphore (#3664)

---
 distributed/distributed.yaml |  2 +-
 distributed/semaphore.py     | 21 +++++++++++++++++++++
 docs/source/api.rst          |  2 ++
 3 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index f4bfc7d76ab..ca31b17776e 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -39,7 +39,7 @@ distributed:
         key: null
         cert: null
     locks:
-      lease-validation-interval: 10s
+      lease-validation-interval: 10s  # The time to wait until an acquired semaphore is released if the Client goes out of scope
 
   worker:
     blocked-handlers: []
diff --git a/distributed/semaphore.py b/distributed/semaphore.py
index 23f253b7ede..6f5553af0e8 100644
--- a/distributed/semaphore.py
+++ b/distributed/semaphore.py
@@ -204,6 +204,26 @@ def close(self, comm=None, name=None):
 class Semaphore:
     """ Semaphore
 
+    This `semaphore <https://en.wikipedia.org/wiki/Semaphore_(programming)>`_
+    will track leases on the scheduler which can be acquired and
+    released by an instance of this class. If the maximum amount of leases are
+    already acquired, it is not possible to acquire more and the caller waits
+    until another lease has been released.
+
+    The lifetime of a lease is coupled to the ``Client`` it was acquired with.
+    Once the Client goes out of scope, the leases associated to it are freed.
+    This behavior can be controlled with the
+    ``distributed.scheduler.locks.lease-validation-interval`` configuration
+    option.
+
+    A noticeable difference to the Semaphore of the python standard library is
+    that this implementation does not allow to release more often than it was
+    acquired. If this happens, a warning is emitted but the internal state is
+    not modified.
+
+    This implementation is still in an experimental state and subtle changes in
+    behavior may occur without any change in the major version of this library.
+
     Parameters
     ----------
     max_leases: int (optional)
@@ -222,6 +242,7 @@ class Semaphore:
     --------
     >>> from distributed import Semaphore
     >>> sem = Semaphore(max_leases=2, name='my_database')
+    >>>
     >>> def access_resource(s, sem):
     >>>     # This automatically acquires a lease from the semaphore (if available) which will be
     >>>     # released when leaving the context manager.
diff --git a/docs/source/api.rst b/docs/source/api.rst
index 9d2f6c7f870..da9a76eed9b 100644
--- a/docs/source/api.rst
+++ b/docs/source/api.rst
@@ -194,6 +194,8 @@ Other
 
 .. autoclass:: Lock
    :members:
+.. autoclass:: Semaphore
+   :members:
 .. autoclass:: Queue
    :members:
 .. autoclass:: Variable

From 46314d88ebf7e1d2fe19bd97b6cda72c5d087419 Mon Sep 17 00:00:00 2001
From: "Jonathan J. Helmus" <jjhelmus@gmail.com>
Date: Thu, 2 Apr 2020 17:30:44 -0500
Subject: [PATCH 0768/1550] Remove openssl 1.1.1d pin for Travis (#3668)

OpenSSL 1.1.1f has been released with a fix for the bug introduced in
1.1.1e.  Pinning to 1.1.1d is no longer necessary.
---
 continuous_integration/travis/install.sh | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/continuous_integration/travis/install.sh b/continuous_integration/travis/install.sh
index 09d13962bd4..4ee0790f6c5 100644
--- a/continuous_integration/travis/install.sh
+++ b/continuous_integration/travis/install.sh
@@ -79,9 +79,6 @@ if [[ $CRICK == true ]]; then
     python -m pip install -q git+https://github.com/jcrist/crick.git
 fi;
 
-# Pin openssl==1.1.1d (see https://github.com/dask/distributed/issues/3588)
-conda install -c conda-forge openssl==1.1.1d
-
 # Install distributed
 python -m pip install --no-deps -e .
 

From 4f11509b844c3569f164704983bb0affd009dd4c Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Thu, 2 Apr 2020 16:36:15 -0700
Subject: [PATCH 0769/1550] Enable more UCX tests (#3667)

* Re-enable cuDF serialization tests

* Skip cuDF strings test that segfaults test

* Run CuPy test as this appears to work

* Contain `check_deserialize` import

Sometimes this doesn't work on some machines. We are still working out
how to get this to work more reliably. For now just skip the test if
this `import` doesn't work for any reason. Also contain the `import` to
this test as other tests don't need it and we don't want the tests to
fail to run entirely.

* Rerurn `black`

* Allow `test_ucx_deserialize` to error

Instead of trying to catch and skip certain errors, go ahead and just add a
comment to this test about how it can error on some systems. Since this no
longer blocks the full test suite from running, it is less of an issue. Plus
seeing it may give us motivation to fix either the system or the test in the
future ;)

* Rerun black
---
 distributed/comm/tests/test_ucx.py | 34 +++++++++++-------------------
 1 file changed, 12 insertions(+), 22 deletions(-)

diff --git a/distributed/comm/tests/test_ucx.py b/distributed/comm/tests/test_ucx.py
index 84da6e4f1aa..f61ea22128a 100644
--- a/distributed/comm/tests/test_ucx.py
+++ b/distributed/comm/tests/test_ucx.py
@@ -12,8 +12,6 @@
 from dask.dataframe.utils import assert_eq
 from distributed.utils_test import gen_test, loop, inc, cleanup, popen  # noqa: 401
 
-from .test_comms import check_deserialize
-
 
 try:
     HOST = ucp.get_address()
@@ -156,6 +154,11 @@ async def test_ping_pong_data():
 
 @gen_test()
 def test_ucx_deserialize():
+    # Note we see this error on some systems with this test:
+    # `socket.gaierror: [Errno -5] No address associated with hostname`
+    # This may be due to a system configuration issue.
+    from .test_comms import check_deserialize
+
     yield check_deserialize("tcp://")
 
 
@@ -169,22 +172,15 @@ def test_ucx_deserialize():
         lambda cudf: cudf.DataFrame([1]).head(0),
         lambda cudf: cudf.DataFrame([1.0]).head(0),
         lambda cudf: cudf.DataFrame({"a": []}),
-        pytest.param(
-            lambda cudf: cudf.DataFrame({"a": ["a"]}).head(0),
-            marks=pytest.mark.xfail(reason="0 length objects don't deseralize cleanly"),
-        ),
-        pytest.param(
-            lambda cudf: cudf.DataFrame({"a": [1.0]}).head(0),
-            marks=pytest.mark.xfail(reason="0 length objects don't deseralize cleanly"),
-        ),
-        pytest.param(
-            lambda cudf: cudf.DataFrame({"a": [1]}).head(0),
-            marks=pytest.mark.xfail(reason="0 length objects don't deseralize cleanly"),
-        ),
+        lambda cudf: cudf.DataFrame({"a": ["a"]}).head(0),
+        lambda cudf: cudf.DataFrame({"a": [1.0]}).head(0),
+        lambda cudf: cudf.DataFrame({"a": [1]}).head(0),
         lambda cudf: cudf.DataFrame({"a": [1, 2, None], "b": [1.0, 2.0, None]}),
         pytest.param(
             lambda cudf: cudf.DataFrame({"a": ["Check", "str"], "b": ["Sup", "port"]}),
-            marks=pytest.mark.xfail(reason="0 length objects don't deseralize cleanly"),
+            marks=pytest.mark.skip(
+                reason="This test segfaults for some reason. So skip running it entirely."
+            ),
         ),
     ],
 )
@@ -231,13 +227,7 @@ async def test_ping_pong_cupy(shape):
 @pytest.mark.slow
 @pytest.mark.asyncio
 @pytest.mark.parametrize(
-    "n",
-    [
-        int(1e9),
-        pytest.param(
-            int(2.5e9), marks=[pytest.mark.xfail(reason="integer type in ucx-py")]
-        ),
-    ],
+    "n", [int(1e9), int(2.5e9),],
 )
 async def test_large_cupy(n, cleanup):
     cupy = pytest.importorskip("cupy")

From a5d1961a579ba934370fe166d84885f948851305 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Fri, 3 Apr 2020 15:57:13 -0500
Subject: [PATCH 0770/1550] bump version to 2.14.0

---
 docs/source/changelog.rst | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index 12288fc4aba..c1bcab71eb5 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,33 @@
 Changelog
 =========
 
+2.14.0 - 2020-04-03
+-------------------
+
+- Enable more UCX tests (:pr:`3667`) `jakirkham`_
+- Remove openssl 1.1.1d pin for Travis (:pr:`3668`) `Jonathan J. Helmus`_
+- More documentation for ``Semaphore`` (:pr:`3664`) `Florian Jetter`_
+- Get CUDA context to finalize Numba ``DeviceNDArray`` (:pr:`3666`) `jakirkham`_
+- Add Resouces option to ``get_task_stream`` and call ``output_file`` (:pr:`3653`) `Prasun Anand`_
+- Add ``Semaphore`` extension (:pr:`3573`) `Lucas Rademaker`_
+- Replace ``ncores`` with ``nthreads`` in work stealing tests (:pr:`3615`) `James Bourbeau`_
+- Clean up some test warnings (:pr:`3662`) `Matthew Rocklin`_
+- Write "why killed" docs (:pr:`3596`) `Martin Durant`_
+- Update Python version checking (:pr:`3660`) `James Bourbeau`_
+- Add newlines to ensure code formatting for ``retire_workers`` (:pr:`3661`) `Rami Chowdhury`_
+- Clean up performance report test (:pr:`3655`) `Matthew Rocklin`_
+- Avoid diagnostics time in performance report (:pr:`3654`) `Matthew Rocklin`_
+- Introduce config for default task duration (:pr:`3642`) `Gabriel Sailer`_
+- UCX simplify receiving frames in ``comm`` (:pr:`3651`) `jakirkham`_
+- Bump checkout GitHub action to v2 (:pr:`3649`) `James Bourbeau`_
+- Handle exception in ``faulthandler`` (:pr:`3646`) `Jacob Tomlinson`_
+- Add prometheus metric for suspicious tasks (:pr:`3550`) `Gabriel Sailer`_
+- Remove ``local-directory`` keyword (:pr:`3620`) `Prasun Anand`_
+- Don't create output Futures in Client when there are mixed Client Futures (:pr:`3643`) `James Bourbeau`_
+- Add link to ``contributing.md`` (:pr:`3621`) `Prasun Anand`_
+- Update bokeh dependency in CI builds (:pr:`3637`) `James Bourbeau`_
+
+
 2.13.0 - 2020-03-25
 -------------------
 
@@ -1641,3 +1668,7 @@ significantly without many new features.
 .. _`Matthias Urlichs`: https://github.com/smurfix
 .. _`Krishan Bhasin`: https://github.com/KrishanBhasin
 .. _`Abdulelah Bin Mahfoodh`: https://github.com/abduhbm
+.. _`jakirkham`: https://github.com/jakirkham
+.. _`Prasun Anand`: https://github.com/prasunanand
+.. _`Jonathan J. Helmus`: https://github.com/jjhelmus
+.. _`Rami Chowdhury`: https://github.com/necaris

From 88d05134a97c8f5d26639c31bccd1af8e20a3abf Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Sat, 4 Apr 2020 12:35:51 -0500
Subject: [PATCH 0771/1550] Update Scheduler.rebalance return value when data
 is missing (#3670)

---
 distributed/client.py            | 26 +++++++++++++++-----------
 distributed/scheduler.py         |  3 ++-
 distributed/tests/test_client.py | 21 +++++++++++++++------
 3 files changed, 32 insertions(+), 18 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index d42c29c2314..fc6098d5ee1 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -587,7 +587,7 @@ def __init__(
         deserializers=None,
         extensions=DEFAULT_EXTENSIONS,
         direct_to_workers=None,
-        **kwargs
+        **kwargs,
     ):
         if timeout == no_default:
             timeout = dask.config.get("distributed.comm.timeouts.connect")
@@ -960,7 +960,7 @@ async def _start(self, timeout=no_default, **kwargs):
                 self.cluster = await LocalCluster(
                     loop=self.loop,
                     asynchronous=self._asynchronous,
-                    **self._startup_kwargs
+                    **self._startup_kwargs,
                 )
             except (OSError, socket.error) as e:
                 if e.errno != errno.EADDRINUSE:
@@ -970,7 +970,7 @@ async def _start(self, timeout=no_default, **kwargs):
                     scheduler_port=0,
                     loop=self.loop,
                     asynchronous=True,
-                    **self._startup_kwargs
+                    **self._startup_kwargs,
                 )
 
             # Wait for all workers to be ready
@@ -1422,7 +1422,7 @@ def submit(
         actor=False,
         actors=False,
         pure=None,
-        **kwargs
+        **kwargs,
     ):
 
         """ Submit a function application to the scheduler
@@ -1542,7 +1542,7 @@ def map(
         actor=False,
         actors=False,
         pure=None,
-        **kwargs
+        **kwargs,
     ):
         """ Map a function on a sequence of arguments
 
@@ -2538,7 +2538,7 @@ def get(
         priority=0,
         fifo_timeout="60s",
         actors=None,
-        **kwargs
+        **kwargs,
     ):
         """ Compute dask graph
 
@@ -2669,7 +2669,7 @@ def compute(
         fifo_timeout="60s",
         actors=None,
         traverse=True,
-        **kwargs
+        **kwargs,
     ):
         """ Compute dask collections on cluster
 
@@ -2817,7 +2817,7 @@ def persist(
         priority=0,
         fifo_timeout="60s",
         actors=None,
-        **kwargs
+        **kwargs,
     ):
         """ Persist dask collections on cluster
 
@@ -3013,6 +3013,10 @@ async def _rebalance(self, futures=None, workers=None):
         await _wait(futures)
         keys = list({tokey(f.key) for f in self.futures_of(futures)})
         result = await self.scheduler.rebalance(keys=keys, workers=workers)
+        if result["status"] == "missing-data":
+            raise ValueError(
+                f"During rebalance {len(result['keys'])} keys were found to be missing"
+            )
         assert result["status"] == "OK"
 
     def rebalance(self, futures=None, workers=None, **kwargs):
@@ -3023,7 +3027,7 @@ def rebalance(self, futures=None, workers=None, **kwargs):
         depending on keyword arguments.
 
         This operation is generally not well tested against normal operation of
-        the scheduler.  It it not recommended to use it while waiting on
+        the scheduler.  It is not recommended to use it while waiting on
         computations.
 
         Parameters
@@ -3085,7 +3089,7 @@ def replicate(self, futures, n=None, workers=None, branching_factor=2, **kwargs)
             n=n,
             workers=workers,
             branching_factor=branching_factor,
-            **kwargs
+            **kwargs,
         )
 
     def nthreads(self, workers=None, **kwargs):
@@ -3505,7 +3509,7 @@ def retire_workers(self, workers=None, close_workers=True, **kwargs):
             self.scheduler.retire_workers,
             workers=workers,
             close_workers=close_workers,
-            **kwargs
+            **kwargs,
         )
 
     def set_metadata(self, key, value):
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index bc6c0ea0fd5..235cb01931e 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -30,6 +30,7 @@
     second,
     compose,
     groupby,
+    concat,
 )
 from tornado.ioloop import IOLoop
 
@@ -3103,7 +3104,7 @@ async def rebalance(self, comm=None, keys=None, workers=None):
                 if not all(r["status"] == "OK" for r in result):
                     return {
                         "status": "missing-data",
-                        "keys": sum([r["keys"] for r in result if "keys" in r], []),
+                        "keys": tuple(concat(r["keys"].keys() for r in result)),
                     }
 
                 for sender, recipient, ts in msgs:
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 1ce1c1dfa6f..68507d889f0 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -2876,6 +2876,15 @@ def test_rebalance_unprepared(c, s, a, b):
     s.validate_state()
 
 
+@gen_cluster(client=True)
+async def test_rebalance_raises_missing_data(c, s, a, b):
+    with pytest.raises(ValueError, match=f"keys were found to be missing"):
+        futures = await c.scatter(range(100))
+        keys = [f.key for f in futures]
+        del futures
+        await c.rebalance(keys)
+
+
 @gen_cluster(client=True)
 def test_receive_lost_key(c, s, a, b):
     x = c.submit(inc, 1, workers=[a.address])
@@ -4864,8 +4873,8 @@ def test_bytes_keys(c, s, a, b):
 
 @gen_cluster(client=True)
 def test_unicode_ascii_keys(c, s, a, b):
-    uni_type = type(u"")
-    key = u"inc-123"
+    uni_type = type("")
+    key = "inc-123"
     future = c.submit(inc, 1, key=key)
     result = yield future
     assert type(future.key) is uni_type
@@ -4876,8 +4885,8 @@ def test_unicode_ascii_keys(c, s, a, b):
 
 @gen_cluster(client=True)
 def test_unicode_keys(c, s, a, b):
-    uni_type = type(u"")
-    key = u"inc-123\u03bc"
+    uni_type = type("")
+    key = "inc-123\u03bc"
     future = c.submit(inc, 1, key=key)
     result = yield future
     assert type(future.key) is uni_type
@@ -4889,8 +4898,8 @@ def test_unicode_keys(c, s, a, b):
     result2 = yield future2
     assert result2 == 3
 
-    future3 = yield c.scatter({u"data-123": 123})
-    result3 = yield future3[u"data-123"]
+    future3 = yield c.scatter({"data-123": 123})
+    result3 = yield future3["data-123"]
     assert result3 == 123
 
 

From beee00a3fa96c26075a024241e55eb70fc1eed29 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Sun, 5 Apr 2020 12:09:20 -0500
Subject: [PATCH 0772/1550] Add zoom tools to profile plots (#3672)

---
 distributed/profile.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/profile.py b/distributed/profile.py
index 5bf071e20da..1bf81ad6ff0 100644
--- a/distributed/profile.py
+++ b/distributed/profile.py
@@ -383,7 +383,7 @@ def plot_figure(data, **kwargs):
 
     source = ColumnDataSource(data=data)
 
-    fig = figure(tools="tap", **kwargs)
+    fig = figure(tools="tap,box_zoom,xwheel_zoom,reset", **kwargs)
     r = fig.quad(
         "left",
         "right",

From cc57f10e636f83d8bbbb2af83bf32a785dfb87d4 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 6 Apr 2020 10:17:12 -0700
Subject: [PATCH 0773/1550] Expose Security object as public API (#3675)

---
 distributed/__init__.py           | 1 +
 distributed/cli/dask_scheduler.py | 3 +--
 distributed/cli/dask_worker.py    | 3 +--
 3 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/distributed/__init__.py b/distributed/__init__.py
index 608e23a58e3..2ad25d05093 100644
--- a/distributed/__init__.py
+++ b/distributed/__init__.py
@@ -22,6 +22,7 @@
 from .nanny import Nanny
 from .pubsub import Pub, Sub
 from .queues import Queue
+from .security import Security
 from .semaphore import Semaphore
 from .scheduler import Scheduler
 from .threadpoolexecutor import rejoin
diff --git a/distributed/cli/dask_scheduler.py b/distributed/cli/dask_scheduler.py
index 2394dd65dea..78d6623608f 100755
--- a/distributed/cli/dask_scheduler.py
+++ b/distributed/cli/dask_scheduler.py
@@ -11,9 +11,8 @@
 
 from tornado.ioloop import IOLoop
 
-from distributed import Scheduler
+from distributed import Scheduler, Security
 from distributed.preloading import validate_preload_argv
-from distributed.security import Security
 from distributed.cli.utils import check_python_3, install_signal_handlers
 from distributed.utils import deserialize_for_cli
 from distributed.proctitle import (
diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
index 29261b52451..ff6d09b4c9a 100755
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -11,8 +11,7 @@
 import dask
 from dask.utils import ignoring
 from dask.system import CPU_COUNT
-from distributed import Nanny
-from distributed.security import Security
+from distributed import Nanny, Security
 from distributed.cli.utils import check_python_3, install_signal_handlers
 from distributed.comm import get_address_host_port
 from distributed.preloading import validate_preload_argv

From f2b13c935cd92af80d3d8f410c8f02053626b422 Mon Sep 17 00:00:00 2001
From: Nicholas Smith <nick.smith@cern.ch>
Date: Mon, 6 Apr 2020 18:08:57 -0500
Subject: [PATCH 0774/1550] Use relative URL in scheduler dashboard (#3676)

As done for other examples of `OpenURL` in the same file. This is important for cases where the dashboard is exposed at a non-root prefix (e.g. via the `--dashboard-prefix` option to `dask-scheduler`)
---
 distributed/dashboard/components/scheduler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index 9519d3629ff..c376e2098e4 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -1090,7 +1090,7 @@ def task_stream_figure(clear_interval="20s", **kwargs):
             """,
     )
 
-    tap = TapTool(callback=OpenURL(url="/profile?key=@name"))
+    tap = TapTool(callback=OpenURL(url="./profile?key=@name"))
 
     root.add_tools(
         hover,

From bd84d9d2ac5896e27743a1e0b07b1919cec8e0d3 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 6 Apr 2020 19:01:35 -0700
Subject: [PATCH 0775/1550] Run preload at import, start, and teardown (#3673)

Previously we only ran preload scripts after the server had started.
This made it difficult to modify the server before certain actions had
taken place.  For example, a test in this PR registers a new Comm
backend for the server to use.  This would not have been possible
before.

Now we run different preload functions when

1.  we first instantiate the server
2.  we first start the server
3.  we close the server (we used to run teardown at `atexit` time
---
 distributed/preloading.py         | 53 +++++++++++++++++++++++--------
 distributed/scheduler.py          | 11 ++++---
 distributed/tests/test_preload.py | 19 +++++++++++
 distributed/worker.py             | 29 ++++++++++-------
 4 files changed, 83 insertions(+), 29 deletions(-)

diff --git a/distributed/preloading.py b/distributed/preloading.py
index 9b276b4337f..2e0469419b9 100644
--- a/distributed/preloading.py
+++ b/distributed/preloading.py
@@ -1,4 +1,4 @@
-import atexit
+import inspect
 import logging
 import os
 import shutil
@@ -111,38 +111,65 @@ def _import_module(name, file_dir=None):
     }
 
 
-def preload_modules(names, parameter=None, file_dir=None, argv=None):
-    """ Imports modules, handles `dask_setup` and `dask_teardown`.
+def on_creation(names, file_dir: str = None) -> dict:
+    """ Imports each of the preload modules
 
     Parameters
     ----------
     names: list of strings
         Module names or file paths
-    parameter: object
-        Parameter passed to `dask_setup` and `dask_teardown`
-    argv: [string]
-        List of string arguments passed to click-configurable `dask_setup`.
     file_dir: string
         Path of a directory where files should be copied
     """
     if isinstance(names, str):
         names = [names]
 
-    for name in names:
-        interface = _import_module(name, file_dir=file_dir)
+    return {name: _import_module(name, file_dir=file_dir) for name in names}
+
 
+async def on_start(modules: dict, dask_server=None, argv=None):
+    """ Run when the server finishes its start method
+
+    Parameters
+    ----------
+    modules: Dict[str, module]
+        The imported modules, from on_creation
+    dask_server: dask.distributed.Server
+        The Worker or Scheduler
+    argv: [string]
+        List of string arguments passed to click-configurable `dask_setup`.
+    file_dir: string
+        Path of a directory where files should be copied
+    """
+    for name, interface in modules.items():
         dask_setup = interface.get("dask_setup", None)
-        dask_teardown = interface.get("dask_teardown", None)
 
         if dask_setup:
             if isinstance(dask_setup, click.Command):
                 context = dask_setup.make_context(
                     "dask_setup", list(argv), allow_extra_args=False
                 )
-                dask_setup.callback(parameter, *context.args, **context.params)
+                dask_setup.callback(dask_server, *context.args, **context.params)
             else:
-                dask_setup(parameter)
+                future = dask_setup(dask_server)
+                if inspect.isawaitable(future):
+                    await future
                 logger.info("Run preload setup function: %s", name)
 
+
+async def on_teardown(modules: dict, dask_server=None):
+    """ Run when the server starts its close method
+
+    Parameters
+    ----------
+    modules: Dict[str, module]
+        The imported modules, from on_creation
+    dask_server: dask.distributed.Server
+        The Worker or Scheduler
+    """
+    for name, interface in modules.items():
+        dask_teardown = interface.get("dask_teardown", None)
         if dask_teardown:
-            atexit.register(interface["dask_teardown"], parameter)
+            future = dask_teardown(dask_server)
+            if inspect.isawaitable(future):
+                await future
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 235cb01931e..525170c5b14 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -3,7 +3,7 @@
 from collections.abc import Mapping, Set
 from datetime import timedelta
 from functools import partial
-from inspect import isawaitable
+import inspect
 import itertools
 import json
 import logging
@@ -49,7 +49,7 @@
 from . import profile
 from .metrics import time
 from .node import ServerNode
-from .preloading import preload_modules
+from . import preloading
 from .proctitle import setproctitle
 from .security import Security
 from .utils import (
@@ -1106,6 +1106,7 @@ def __init__(
             preload_argv = dask.config.get("distributed.scheduler.preload-argv")
         self.preload = preload
         self.preload_argv = preload_argv
+        self._preload_modules = preloading.on_creation(self.preload)
 
         self.security = security or Security()
         assert isinstance(self.security, Security)
@@ -1463,7 +1464,7 @@ def del_scheduler_file():
 
             weakref.finalize(self, del_scheduler_file)
 
-        preload_modules(self.preload, parameter=self, argv=self.preload_argv)
+        await preloading.on_start(self._preload_modules, self, argv=self.preload_argv)
 
         await asyncio.gather(*[plugin.start(self) for plugin in self.plugins])
 
@@ -1487,6 +1488,8 @@ async def close(self, comm=None, fast=False, close_workers=False):
         logger.info("Scheduler closing...")
         setproctitle("dask-scheduler [closing]")
 
+        await preloading.on_teardown(self._preload_modules, self)
+
         if close_workers:
             await self.broadcast(msg={"op": "close_gracefully"}, nanny=True)
             for worker in self.workers:
@@ -3584,7 +3587,7 @@ async def feed(
             if teardown:
                 teardown = pickle.loads(teardown)
             state = setup(self) if setup else None
-            if isawaitable(state):
+            if inspect.isawaitable(state):
                 state = await state
             try:
                 while self.status == "running":
diff --git a/distributed/tests/test_preload.py b/distributed/tests/test_preload.py
index d3171ed6842..888e7c42ea2 100644
--- a/distributed/tests/test_preload.py
+++ b/distributed/tests/test_preload.py
@@ -95,3 +95,22 @@ def check_worker():
     finally:
         sys.path.remove(tmpdir)
         shutil.rmtree(tmpdir)
+
+
+@pytest.mark.asyncio
+async def test_preload_import_time(cleanup):
+    text = """
+from distributed.comm.registry import backends
+from distributed.comm.tcp import TCPBackend
+
+backends["foo"] = TCPBackend()
+""".strip()
+    try:
+        async with Scheduler(port=0, preload=text, protocol="foo") as s:
+            async with Nanny(s.address, preload=text, protocol="foo") as n:
+                async with Client(s.address, asynchronous=True) as c:
+                    await c.wait_for_workers(1)
+    finally:
+        from distributed.comm.registry import backends
+
+        del backends["foo"]
diff --git a/distributed/worker.py b/distributed/worker.py
index c6ae63f0ef1..ff781202393 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -34,7 +34,7 @@
 from .diskutils import WorkSpace
 from .metrics import time
 from .node import ServerNode
-from .preloading import preload_modules
+from . import preloading
 from .proctitle import setproctitle
 from .protocol import pickle, to_serialize, deserialize_bytes, serialize_bytelist
 from .pubsub import PubSubWorkerExtension
@@ -470,12 +470,7 @@ def __init__(
         self.total_resources = resources or {}
         self.available_resources = (resources or {}).copy()
         self.death_timeout = parse_timedelta(death_timeout)
-        self.preload = preload
-        if self.preload is None:
-            self.preload = dask.config.get("distributed.worker.preload")
-        self.preload_argv = preload_argv
-        if self.preload_argv is None:
-            self.preload_argv = dask.config.get("distributed.worker.preload-argv")
+
         self.memory_monitor_interval = parse_timedelta(
             memory_monitor_interval, default="ms"
         )
@@ -504,6 +499,16 @@ def __init__(
             self._workdir = self._workspace.new_work_dir(prefix="worker-")
             self.local_directory = self._workdir.dir_path
 
+        self.preload = preload
+        if self.preload is None:
+            self.preload = dask.config.get("distributed.worker.preload")
+        self.preload_argv = preload_argv
+        if self.preload_argv is None:
+            self.preload_argv = dask.config.get("distributed.worker.preload-argv")
+        self._preload_modules = preloading.on_creation(
+            self.preload, file_dir=self.local_directory
+        )
+
         self.security = security or Security()
         assert isinstance(self.security, Security)
         self.connection_args = self.security.get_connection_args("worker")
@@ -1023,12 +1028,10 @@ async def start(self):
         if self.name is None:
             self.name = self.address
 
-        preload_modules(
-            self.preload,
-            parameter=self,
-            file_dir=self.local_directory,
-            argv=self.preload_argv,
+        await preloading.on_start(
+            self._preload_modules, self, argv=self.preload_argv,
         )
+
         # Services listen on all addresses
         # Note Nanny is not a "real" service, just some metadata
         # passed in service_ports...
@@ -1085,6 +1088,8 @@ async def close(
                 logger.info("Closed worker has not yet started: %s", self.status)
             self.status = "closing"
 
+            await preloading.on_teardown(self._preload_modules, self)
+
             if nanny and self.nanny:
                 with self.rpc(self.nanny) as r:
                     await r.close_gracefully()

From e395c86b7bab29c5388f888d989b3ba6d069484b Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 6 Apr 2020 19:31:51 -0700
Subject: [PATCH 0776/1550] Pass through connection/listen_args as splatted
 keywords (#3674)

Previously we would accept a specific keyword connection_args or
listen_args for security information like `ssl_context`.

Now we pass in these keywords directly,
and let the listener handle them without intermediary.
---
 distributed/cli/tests/test_dask_scheduler.py |  6 +-
 distributed/client.py                        |  4 +-
 distributed/comm/core.py                     | 12 ++-
 distributed/comm/tests/test_comms.py         | 54 +++++--------
 distributed/comm/tests/test_ucx.py           |  8 +-
 distributed/core.py                          | 15 ++--
 distributed/deploy/tests/test_local.py       |  4 +-
 distributed/diagnostics/progressbar.py       |  3 +-
 distributed/nanny.py                         |  5 +-
 distributed/scheduler.py                     |  3 +-
 distributed/tests/test_core.py               | 12 ++-
 distributed/tests/test_security.py           | 36 ++++-----
 distributed/tests/test_tls_functional.py     |  5 ++
 distributed/utils_test.py                    | 81 +++++++++-----------
 distributed/worker.py                        | 11 ++-
 15 files changed, 109 insertions(+), 150 deletions(-)

diff --git a/distributed/cli/tests/test_dask_scheduler.py b/distributed/cli/tests/test_dask_scheduler.py
index cb6cc306b6c..2206c173925 100644
--- a/distributed/cli/tests/test_dask_scheduler.py
+++ b/distributed/cli/tests/test_dask_scheduler.py
@@ -32,7 +32,9 @@ def test_defaults(loop):
         @gen.coroutine
         def f():
             # Default behaviour is to listen on all addresses
-            yield [assert_can_connect_from_everywhere_4_6(8786, 5.0)]  # main port
+            yield [
+                assert_can_connect_from_everywhere_4_6(8786, timeout=5.0)
+            ]  # main port
 
         with Client("127.0.0.1:%d" % Scheduler.default_port, loop=loop) as c:
             c.sync(f)
@@ -50,7 +52,7 @@ def test_hostport(loop):
         def f():
             yield [
                 # The scheduler's main port can't be contacted from the outside
-                assert_can_connect_locally_4(8978, 5.0)
+                assert_can_connect_locally_4(8978, timeout=5.0)
             ]
 
         with Client("127.0.0.1:8978", loop=loop) as c:
diff --git a/distributed/client.py b/distributed/client.py
index fc6098d5ee1..73cccc8a18a 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -1044,9 +1044,7 @@ async def _ensure_connected(self, timeout=None):
 
         try:
             comm = await connect(
-                self.scheduler.address,
-                timeout=timeout,
-                connection_args=self.connection_args,
+                self.scheduler.address, timeout=timeout, **self.connection_args,
             )
             comm.name = "Client->Scheduler"
             if timeout is not None:
diff --git a/distributed/comm/core.py b/distributed/comm/core.py
index 6ef26568853..bfae9e8dcc0 100644
--- a/distributed/comm/core.py
+++ b/distributed/comm/core.py
@@ -184,7 +184,7 @@ def connect(self, address, deserialize=True):
         """
 
 
-async def connect(addr, timeout=None, deserialize=True, connection_args=None):
+async def connect(addr, timeout=None, deserialize=True, **connection_args):
     """
     Connect to the given address (a URI such as ``tcp://127.0.0.1:1234``)
     and yield a ``Comm`` object.  If the connection attempt fails, it is
@@ -221,7 +221,7 @@ def _raise(error):
         try:
             while deadline - time() > 0:
                 future = connector.connect(
-                    loc, deserialize=deserialize, **(connection_args or {})
+                    loc, deserialize=deserialize, **connection_args
                 )
                 with ignoring(TimeoutError):
                     comm = await asyncio.wait_for(
@@ -247,7 +247,7 @@ def _raise(error):
     return comm
 
 
-def listen(addr, handle_comm, deserialize=True, connection_args=None):
+def listen(addr, handle_comm, deserialize=True, **kwargs):
     """
     Create a listener object with the given parameters.  When its ``start()``
     method is called, the listener will listen on the given address
@@ -259,7 +259,7 @@ def listen(addr, handle_comm, deserialize=True, connection_args=None):
     try:
         scheme, loc = parse_address(addr, strict=True)
     except ValueError:
-        if connection_args and connection_args.get("ssl_context"):
+        if kwargs.get("ssl_context"):
             addr = "tls://" + addr
         else:
             addr = "tcp://" + addr
@@ -267,6 +267,4 @@ def listen(addr, handle_comm, deserialize=True, connection_args=None):
 
     backend = registry.get_backend(scheme)
 
-    return backend.get_listener(
-        loc, handle_comm, deserialize, **(connection_args or {})
-    )
+    return backend.get_listener(loc, handle_comm, deserialize, **kwargs)
diff --git a/distributed/comm/tests/test_comms.py b/distributed/comm/tests/test_comms.py
index 2e5602a9ac5..035a95513fb 100644
--- a/distributed/comm/tests/test_comms.py
+++ b/distributed/comm/tests/test_comms.py
@@ -77,19 +77,15 @@ def check_tls_extra(info):
 
 
 @pytest.mark.asyncio
-async def get_comm_pair(listen_addr, listen_args=None, connect_args=None, **kwargs):
+async def get_comm_pair(listen_addr, listen_args={}, connect_args={}, **kwargs):
     q = asyncio.Queue()
 
     async def handle_comm(comm):
         await q.put(comm)
 
-    listener = await listen(
-        listen_addr, handle_comm, connection_args=listen_args, **kwargs
-    )
+    listener = await listen(listen_addr, handle_comm, **listen_args, **kwargs)
 
-    comm = await connect(
-        listener.contact_address, connection_args=connect_args, **kwargs
-    )
+    comm = await connect(listener.contact_address, **connect_args, **kwargs)
     serv_comm = await q.get()
     return (comm, serv_comm)
 
@@ -332,9 +328,7 @@ async def sleep_for_60ms():
     sleep_future = sleep_for_60ms()
     with pytest.raises(IOError):
         await connect(
-            "tls://localhost:28400",
-            0.052,
-            connection_args={"ssl_context": get_client_ssl_context()},
+            "tls://localhost:28400", 0.052, ssl_context=get_client_ssl_context(),
         )
     max_thread_count = await sleep_future
     assert max_thread_count <= 2 + original_thread_count
@@ -441,8 +435,8 @@ async def check_client_server(
     addr,
     check_listen_addr=None,
     check_contact_addr=None,
-    listen_args=None,
-    connect_args=None,
+    listen_args={},
+    connect_args={},
 ):
     """
     Abstract client / server test.
@@ -466,7 +460,7 @@ async def handle_comm(comm):
     listen_args = listen_args or {"xxx": "bar"}
     connect_args = connect_args or {"xxx": "foo"}
 
-    listener = await listen(addr, handle_comm, connection_args=listen_args)
+    listener = await listen(addr, handle_comm, **listen_args)
 
     # Check listener properties
     bound_addr = listener.listen_address
@@ -490,7 +484,7 @@ async def handle_comm(comm):
     l = []
 
     async def client_communicate(key, delay=0):
-        comm = await connect(listener.contact_address, connection_args=connect_args)
+        comm = await connect(listener.contact_address, **connect_args)
         assert comm.peer_address == listener.contact_address
 
         await comm.write({"op": "ping", "data": key})
@@ -644,15 +638,11 @@ async def handle_comm(comm):
         await comm.close()
 
     # Listener refuses a connector not signed by the CA
-    listener = await listen(
-        "tls://", handle_comm, connection_args={"ssl_context": serv_ctx}
-    )
+    listener = await listen("tls://", handle_comm, ssl_context=serv_ctx)
 
     with pytest.raises(EnvironmentError) as excinfo:
         comm = await connect(
-            listener.contact_address,
-            timeout=0.5,
-            connection_args={"ssl_context": bad_cli_ctx},
+            listener.contact_address, timeout=0.5, ssl_context=bad_cli_ctx,
         )
         await comm.write({"x": "foo"})  # TODO: why is this necessary in Tornado 6 ?
 
@@ -670,21 +660,15 @@ async def handle_comm(comm):
                 raise
 
     # Sanity check
-    comm = await connect(
-        listener.contact_address, timeout=2, connection_args={"ssl_context": cli_ctx}
-    )
+    comm = await connect(listener.contact_address, timeout=2, ssl_context=cli_ctx,)
     await comm.close()
 
     # Connector refuses a listener not signed by the CA
-    listener = await listen(
-        "tls://", handle_comm, connection_args={"ssl_context": bad_serv_ctx}
-    )
+    listener = await listen("tls://", handle_comm, ssl_context=bad_serv_ctx)
 
     with pytest.raises(EnvironmentError) as excinfo:
         await connect(
-            listener.contact_address,
-            timeout=2,
-            connection_args={"ssl_context": cli_ctx},
+            listener.contact_address, timeout=2, ssl_context=cli_ctx,
         )
     # The wrong error is reported on Python 2, see https://github.com/tornadoweb/tornado/pull/2028
     if sys.version_info >= (3,):
@@ -696,20 +680,18 @@ async def handle_comm(comm):
 #
 
 
-async def check_comm_closed_implicit(
-    addr, delay=None, listen_args=None, connect_args=None
-):
+async def check_comm_closed_implicit(addr, delay=None, listen_args={}, connect_args={}):
     async def handle_comm(comm):
         await comm.close()
 
-    listener = await listen(addr, handle_comm, connection_args=listen_args)
+    listener = await listen(addr, handle_comm, **listen_args)
     contact_addr = listener.contact_address
 
-    comm = await connect(contact_addr, connection_args=connect_args)
+    comm = await connect(contact_addr, **connect_args)
     with pytest.raises(CommClosedError):
         await comm.write({})
 
-    comm = await connect(contact_addr, connection_args=connect_args)
+    comm = await connect(contact_addr, **connect_args)
     with pytest.raises(CommClosedError):
         await comm.read()
 
@@ -729,7 +711,7 @@ async def test_inproc_comm_closed_implicit():
     await check_comm_closed_implicit(inproc.new_address())
 
 
-async def check_comm_closed_explicit(addr, listen_args=None, connect_args=None):
+async def check_comm_closed_explicit(addr, listen_args={}, connect_args={}):
     a, b = await get_comm_pair(addr, listen_args=listen_args, connect_args=connect_args)
     a_read = a.read()
     b_read = b.read()
diff --git a/distributed/comm/tests/test_ucx.py b/distributed/comm/tests/test_ucx.py
index f61ea22128a..9ac97deeb7e 100644
--- a/distributed/comm/tests/test_ucx.py
+++ b/distributed/comm/tests/test_ucx.py
@@ -26,18 +26,16 @@ def test_registered():
 
 
 async def get_comm_pair(
-    listen_addr="ucx://" + HOST, listen_args=None, connect_args=None, **kwargs
+    listen_addr="ucx://" + HOST, listen_args={}, connect_args={}, **kwargs
 ):
     q = asyncio.queues.Queue()
 
     async def handle_comm(comm):
         await q.put(comm)
 
-    listener = listen(listen_addr, handle_comm, connection_args=listen_args, **kwargs)
+    listener = listen(listen_addr, handle_comm, **listen_args, **kwargs)
     async with listener:
-        comm = await connect(
-            listener.contact_address, connection_args=connect_args, **kwargs
-        )
+        comm = await connect(listener.contact_address, **connect_args, **kwargs)
         serv_comm = await q.get()
         return (comm, serv_comm)
 
diff --git a/distributed/core.py b/distributed/core.py
index 1bf3b172b68..dd5e18d0007 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -302,7 +302,7 @@ def port(self):
     def identity(self, comm=None):
         return {"type": type(self).__name__, "id": self.id}
 
-    async def listen(self, port_or_addr=None, listen_args=None):
+    async def listen(self, port_or_addr=None, **kwargs):
         if port_or_addr is None:
             port_or_addr = self.default_port
         if isinstance(port_or_addr, int):
@@ -313,10 +313,7 @@ async def listen(self, port_or_addr=None, listen_args=None):
             addr = port_or_addr
             assert isinstance(addr, str)
         listener = await listen(
-            addr,
-            self.handle_comm,
-            deserialize=self.deserialize,
-            connection_args=listen_args,
+            addr, self.handle_comm, deserialize=self.deserialize, **kwargs,
         )
         self.listeners.append(listener)
 
@@ -606,7 +603,7 @@ def __init__(
         self.deserialize = deserialize
         self.serializers = serializers
         self.deserializers = deserializers if deserializers is not None else serializers
-        self.connection_args = connection_args
+        self.connection_args = connection_args or {}
         self._created = weakref.WeakSet()
         rpc.active.add(self)
 
@@ -644,7 +641,7 @@ async def live_comm(self):
                 self.address,
                 self.timeout,
                 deserialize=self.deserialize,
-                connection_args=self.connection_args,
+                **self.connection_args,
             )
             comm.name = "rpc"
         self.comms[comm] = False  # mark as taken
@@ -832,7 +829,7 @@ def __init__(
         self.deserialize = deserialize
         self.serializers = serializers
         self.deserializers = deserializers if deserializers is not None else serializers
-        self.connection_args = connection_args
+        self.connection_args = connection_args or {}
         self.timeout = timeout
         self._n_connecting = 0
         self.server = weakref.ref(server) if server else None
@@ -905,7 +902,7 @@ async def connect(self, addr, timeout=None):
                 addr,
                 timeout=timeout or self.timeout,
                 deserialize=self.deserialize,
-                connection_args=self.connection_args,
+                **self.connection_args,
             )
             comm.name = "ConnectionPool"
             comm._pool = weakref.ref(self)
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index 8ca780a4eb2..403beb3aa41 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -734,9 +734,9 @@ def test_local_tls(loop, temporary):
             loop,
             assert_can_connect_from_everywhere_4,
             c.scheduler.port,
-            connection_args=c.security.get_connection_args("client"),
             protocol="tls",
             timeout=3,
+            **c.security.get_connection_args("client"),
         )
 
         # If we connect to a TLS localculster without ssl information we should fail
@@ -744,8 +744,8 @@ def test_local_tls(loop, temporary):
             loop,
             assert_cannot_connect,
             addr="tcp://127.0.0.1:%d" % c.scheduler.port,
-            connection_args=c.security.get_connection_args("client"),
             exception_class=RuntimeError,
+            **c.security.get_connection_args("client"),
         )
 
 
diff --git a/distributed/diagnostics/progressbar.py b/distributed/diagnostics/progressbar.py
index 11da7a30d3d..4ef7254f52e 100644
--- a/distributed/diagnostics/progressbar.py
+++ b/distributed/diagnostics/progressbar.py
@@ -63,8 +63,7 @@ def function(scheduler, p):
             return result
 
         self.comm = await connect(
-            self.scheduler,
-            connection_args=self.client().connection_args if self.client else None,
+            self.scheduler, **(self.client().connection_args if self.client else {}),
         )
         logger.debug("Progressbar Connected to scheduler")
 
diff --git a/distributed/nanny.py b/distributed/nanny.py
index baa77e3ce10..d3a4d2dc82b 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -95,7 +95,6 @@ def __init__(
         self.security = security or Security()
         assert isinstance(self.security, Security)
         self.connection_args = self.security.get_connection_args("worker")
-        self.listen_args = self.security.get_listen_args("worker")
 
         if scheduler_file:
             cfg = json_load_robust(scheduler_file)
@@ -244,7 +243,9 @@ async def start(self):
 
         await super().start()
 
-        await self.listen(self._start_address, listen_args=self.listen_args)
+        await self.listen(
+            self._start_address, **self.security.get_listen_args("worker")
+        )
         self.ip = get_address_host(self.address)
 
         logger.info("        Start Nanny at: %r", self.address)
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 525170c5b14..55692b875ab 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1111,7 +1111,6 @@ def __init__(
         self.security = security or Security()
         assert isinstance(self.security, Security)
         self.connection_args = self.security.get_connection_args("scheduler")
-        self.listen_args = self.security.get_listen_args("scheduler")
 
         if dashboard_address is not None:
             try:
@@ -1431,7 +1430,7 @@ async def start(self):
 
         if self.status != "running":
             for addr in self._start_address:
-                await self.listen(addr, listen_args=self.listen_args)
+                await self.listen(addr, **self.security.get_listen_args("scheduler"))
                 self.ip = get_address_host(self.listen_address)
                 listen_ip = self.ip
 
diff --git a/distributed/tests/test_core.py b/distributed/tests/test_core.py
index b4993bb4ce8..49033a6a11e 100644
--- a/distributed/tests/test_core.py
+++ b/distributed/tests/test_core.py
@@ -230,12 +230,10 @@ async def listen_on(cls, *args, **kwargs):
 
     sec = tls_security()
     async with listen_on(
-        Server, "tls://", listen_args=sec.get_listen_args("scheduler")
+        Server, "tls://", **sec.get_listen_args("scheduler")
     ) as server:
         assert server.address.startswith("tls://")
-        await assert_can_connect(
-            server.address, connection_args=sec.get_connection_args("client")
-        )
+        await assert_can_connect(server.address, **sec.get_connection_args("client"))
 
     # InProc
 
@@ -253,9 +251,9 @@ async def listen_on(cls, *args, **kwargs):
         await assert_cannot_connect(inproc_addr2)
 
 
-async def check_rpc(listen_addr, rpc_addr=None, listen_args=None, connection_args=None):
+async def check_rpc(listen_addr, rpc_addr=None, listen_args={}, connection_args={}):
     server = Server({"ping": pingpong})
-    await server.listen(listen_addr, listen_args=listen_args)
+    await server.listen(listen_addr, **listen_args)
     if rpc_addr is None:
         rpc_addr = server.address
 
@@ -603,7 +601,7 @@ async def ping(comm, delay=0.01):
 
     servers = [Server({"ping": ping}) for i in range(10)]
     for server in servers:
-        await server.listen("tls://", listen_args=listen_args)
+        await server.listen("tls://", **listen_args)
 
     rpc = await ConnectionPool(limit=5, connection_args=connection_args)
 
diff --git a/distributed/tests/test_security.py b/distributed/tests/test_security.py
index 002e63d2855..8665ebead33 100644
--- a/distributed/tests/test_security.py
+++ b/distributed/tests/test_security.py
@@ -281,10 +281,10 @@ async def handle_comm(comm):
         forced_cipher_sec = Security()
 
     async with listen(
-        "tls://", handle_comm, connection_args=sec.get_listen_args("scheduler")
+        "tls://", handle_comm, **sec.get_listen_args("scheduler")
     ) as listener:
         comm = await connect(
-            listener.contact_address, connection_args=sec.get_connection_args("worker")
+            listener.contact_address, **sec.get_connection_args("worker")
         )
         msg = await comm.read()
         assert msg == "hello"
@@ -293,14 +293,12 @@ async def handle_comm(comm):
         # No SSL context for client
         with pytest.raises(TypeError):
             await connect(
-                listener.contact_address,
-                connection_args=sec.get_connection_args("client"),
+                listener.contact_address, **sec.get_connection_args("client"),
             )
 
         # Check forced cipher
         comm = await connect(
-            listener.contact_address,
-            connection_args=forced_cipher_sec.get_connection_args("worker"),
+            listener.contact_address, **forced_cipher_sec.get_connection_args("worker"),
         )
         cipher, _, _ = comm.extra_info["cipher"]
         assert cipher in [FORCED_CIPHER] + TLS_13_CIPHERS
@@ -331,20 +329,18 @@ async def handle_comm(comm):
 
     for listen_addr in ["inproc://", "tls://"]:
         async with listen(
-            listen_addr, handle_comm, connection_args=sec.get_listen_args("scheduler")
+            listen_addr, handle_comm, **sec.get_listen_args("scheduler")
         ) as listener:
             comm = await connect(
-                listener.contact_address,
-                connection_args=sec2.get_connection_args("worker"),
+                listener.contact_address, **sec2.get_connection_args("worker"),
             )
             comm.abort()
 
         async with listen(
-            listen_addr, handle_comm, connection_args=sec2.get_listen_args("scheduler")
+            listen_addr, handle_comm, **sec2.get_listen_args("scheduler")
         ) as listener:
             comm = await connect(
-                listener.contact_address,
-                connection_args=sec2.get_connection_args("worker"),
+                listener.contact_address, **sec2.get_connection_args("worker"),
             )
             comm.abort()
 
@@ -356,25 +352,21 @@ def check_encryption_error():
 
     for listen_addr in ["tcp://"]:
         async with listen(
-            listen_addr, handle_comm, connection_args=sec.get_listen_args("scheduler")
+            listen_addr, handle_comm, **sec.get_listen_args("scheduler")
         ) as listener:
             comm = await connect(
-                listener.contact_address,
-                connection_args=sec.get_connection_args("worker"),
+                listener.contact_address, **sec.get_connection_args("worker"),
             )
             comm.abort()
 
             with pytest.raises(RuntimeError):
                 await connect(
-                    listener.contact_address,
-                    connection_args=sec2.get_connection_args("worker"),
+                    listener.contact_address, **sec2.get_connection_args("worker"),
                 )
 
         with pytest.raises(RuntimeError):
             listen(
-                listen_addr,
-                handle_comm,
-                connection_args=sec2.get_listen_args("scheduler"),
+                listen_addr, handle_comm, **sec2.get_listen_args("scheduler"),
             )
 
 
@@ -408,10 +400,10 @@ async def handle_comm(comm):
     sec = Security.temporary()
 
     async with listen(
-        "tls://", handle_comm, connection_args=sec.get_listen_args("scheduler")
+        "tls://", handle_comm, **sec.get_listen_args("scheduler")
     ) as listener:
         comm = await connect(
-            listener.contact_address, connection_args=sec.get_connection_args("worker")
+            listener.contact_address, **sec.get_connection_args("worker")
         )
         msg = await comm.read()
         assert msg == "hello"
diff --git a/distributed/tests/test_tls_functional.py b/distributed/tests/test_tls_functional.py
index 6d0e64b54e5..3a2bebf790d 100644
--- a/distributed/tests/test_tls_functional.py
+++ b/distributed/tests/test_tls_functional.py
@@ -11,6 +11,11 @@
 from distributed.utils_test import gen_tls_cluster, inc, double, slowinc, slowadd
 
 
+@gen_tls_cluster(client=True)
+def test_basic(c, s, a, b):
+    pass
+
+
 @gen_tls_cluster(client=True)
 def test_Queue(c, s, a, b):
     assert s.address.startswith("tls://")
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index b521826647a..983eaac48f5 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -798,7 +798,7 @@ async def start_cluster(
         security=security,
         port=0,
         host=scheduler_addr,
-        **scheduler_kwargs
+        **scheduler_kwargs,
     )
     workers = [
         Worker(
@@ -809,7 +809,7 @@ async def start_cluster(
             loop=loop,
             validate=True,
             host=ncore[0],
-            **(merge(worker_kwargs, ncore[2]) if len(ncore) > 2 else worker_kwargs)
+            **(merge(worker_kwargs, ncore[2]) if len(ncore) > 2 else worker_kwargs),
         )
         for i, ncore in enumerate(nthreads)
     ]
@@ -917,7 +917,7 @@ async def coro():
                                 loop=loop,
                                 security=security,
                                 asynchronous=True,
-                                **client_kwargs
+                                **client_kwargs,
                             )
                             args = [c] + args
                         try:
@@ -1108,115 +1108,106 @@ def requires_ipv6(test_func):
     requires_ipv6 = pytest.mark.skip("ipv6 required")
 
 
-async def assert_can_connect(addr, timeout=None, connection_args=None):
+async def assert_can_connect(addr, timeout=0.5, **kwargs):
     """
     Check that it is possible to connect to the distributed *addr*
     within the given *timeout*.
     """
-    if timeout is None:
-        timeout = 0.5
-    comm = await connect(addr, timeout=timeout, connection_args=connection_args)
+    comm = await connect(addr, timeout=timeout, **kwargs)
     comm.abort()
 
 
 async def assert_cannot_connect(
-    addr, timeout=None, connection_args=None, exception_class=EnvironmentError
+    addr, timeout=0.5, exception_class=EnvironmentError, **kwargs
 ):
     """
     Check that it is impossible to connect to the distributed *addr*
     within the given *timeout*.
     """
-    if timeout is None:
-        timeout = 0.5
     with pytest.raises(exception_class):
-        comm = await connect(addr, timeout=timeout, connection_args=connection_args)
+        comm = await connect(addr, timeout=timeout, **kwargs)
         comm.abort()
 
 
-async def assert_can_connect_from_everywhere_4_6(
-    port, timeout=None, connection_args=None, protocol="tcp"
-):
+async def assert_can_connect_from_everywhere_4_6(port, protocol="tcp", **kwargs):
     """
     Check that the local *port* is reachable from all IPv4 and IPv6 addresses.
     """
-    args = (timeout, connection_args)
     futures = [
-        assert_can_connect("%s://127.0.0.1:%d" % (protocol, port), *args),
-        assert_can_connect("%s://%s:%d" % (protocol, get_ip(), port), *args),
+        assert_can_connect("%s://127.0.0.1:%d" % (protocol, port), **kwargs),
+        assert_can_connect("%s://%s:%d" % (protocol, get_ip(), port), **kwargs),
     ]
     if has_ipv6():
         futures += [
-            assert_can_connect("%s://[::1]:%d" % (protocol, port), *args),
-            assert_can_connect("%s://[%s]:%d" % (protocol, get_ipv6(), port), *args),
+            assert_can_connect("%s://[::1]:%d" % (protocol, port), **kwargs),
+            assert_can_connect("%s://[%s]:%d" % (protocol, get_ipv6(), port), **kwargs),
         ]
     await asyncio.gather(*futures)
 
 
 async def assert_can_connect_from_everywhere_4(
-    port, timeout=None, connection_args=None, protocol="tcp"
+    port, protocol="tcp", **kwargs,
 ):
     """
     Check that the local *port* is reachable from all IPv4 addresses.
     """
-    args = (timeout, connection_args)
     futures = [
-        assert_can_connect("%s://127.0.0.1:%d" % (protocol, port), *args),
-        assert_can_connect("%s://%s:%d" % (protocol, get_ip(), port), *args),
+        assert_can_connect("%s://127.0.0.1:%d" % (protocol, port), **kwargs),
+        assert_can_connect("%s://%s:%d" % (protocol, get_ip(), port), **kwargs),
     ]
     if has_ipv6():
         futures += [
-            assert_cannot_connect("%s://[::1]:%d" % (protocol, port), *args),
-            assert_cannot_connect("%s://[%s]:%d" % (protocol, get_ipv6(), port), *args),
+            assert_cannot_connect("%s://[::1]:%d" % (protocol, port), **kwargs),
+            assert_cannot_connect(
+                "%s://[%s]:%d" % (protocol, get_ipv6(), port), **kwargs
+            ),
         ]
     await asyncio.gather(*futures)
 
 
-async def assert_can_connect_locally_4(port, timeout=None, connection_args=None):
+async def assert_can_connect_locally_4(port, **kwargs):
     """
     Check that the local *port* is only reachable from local IPv4 addresses.
     """
-    args = (timeout, connection_args)
-    futures = [assert_can_connect("tcp://127.0.0.1:%d" % port, *args)]
+    futures = [assert_can_connect("tcp://127.0.0.1:%d" % port, **kwargs)]
     if get_ip() != "127.0.0.1":  # No outside IPv4 connectivity?
-        futures += [assert_cannot_connect("tcp://%s:%d" % (get_ip(), port), *args)]
+        futures += [assert_cannot_connect("tcp://%s:%d" % (get_ip(), port), **kwargs)]
     if has_ipv6():
         futures += [
-            assert_cannot_connect("tcp://[::1]:%d" % port, *args),
-            assert_cannot_connect("tcp://[%s]:%d" % (get_ipv6(), port), *args),
+            assert_cannot_connect("tcp://[::1]:%d" % port, **kwargs),
+            assert_cannot_connect("tcp://[%s]:%d" % (get_ipv6(), port), **kwargs),
         ]
     await asyncio.gather(*futures)
 
 
-async def assert_can_connect_from_everywhere_6(
-    port, timeout=None, connection_args=None
-):
+async def assert_can_connect_from_everywhere_6(port, **kwargs):
     """
     Check that the local *port* is reachable from all IPv6 addresses.
     """
     assert has_ipv6()
-    args = (timeout, connection_args)
     futures = [
-        assert_cannot_connect("tcp://127.0.0.1:%d" % port, *args),
-        assert_cannot_connect("tcp://%s:%d" % (get_ip(), port), *args),
-        assert_can_connect("tcp://[::1]:%d" % port, *args),
-        assert_can_connect("tcp://[%s]:%d" % (get_ipv6(), port), *args),
+        assert_cannot_connect("tcp://127.0.0.1:%d" % port, **kwargs),
+        assert_cannot_connect("tcp://%s:%d" % (get_ip(), port), **kwargs),
+        assert_can_connect("tcp://[::1]:%d" % port, **kwargs),
+        assert_can_connect("tcp://[%s]:%d" % (get_ipv6(), port), **kwargs),
     ]
     await asyncio.gather(*futures)
 
 
-async def assert_can_connect_locally_6(port, timeout=None, connection_args=None):
+async def assert_can_connect_locally_6(port, **kwargs):
     """
     Check that the local *port* is only reachable from local IPv6 addresses.
     """
     assert has_ipv6()
-    args = (timeout, connection_args)
     futures = [
-        assert_cannot_connect("tcp://127.0.0.1:%d" % port, *args),
-        assert_cannot_connect("tcp://%s:%d" % (get_ip(), port), *args),
-        assert_can_connect("tcp://[::1]:%d" % port, *args),
+        assert_cannot_connect("tcp://127.0.0.1:%d" % port, **kwargs),
+        assert_cannot_connect("tcp://%s:%d" % (get_ip(), port), **kwargs),
+        assert_can_connect("tcp://[::1]:%d" % port, **kwargs),
     ]
     if get_ipv6() != "::1":  # No outside IPv6 connectivity?
-        futures += [assert_cannot_connect("tcp://[%s]:%d" % (get_ipv6(), port), *args)]
+        futures += [
+            assert_cannot_connect("tcp://[%s]:%d" % (get_ipv6(), port), **kwargs)
+        ]
     await asyncio.gather(*futures)
 
 
diff --git a/distributed/worker.py b/distributed/worker.py
index ff781202393..0c6c8b46ba5 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -512,7 +512,6 @@ def __init__(
         self.security = security or Security()
         assert isinstance(self.security, Security)
         self.connection_args = self.security.get_connection_args("worker")
-        self.listen_args = self.security.get_listen_args("worker")
 
         self.memory_limit = parse_memory_limit(memory_limit, self.nthreads)
 
@@ -816,9 +815,7 @@ async def _register_with_scheduler(self):
         while True:
             try:
                 _start = time()
-                comm = await connect(
-                    self.scheduler.address, connection_args=self.connection_args
-                )
+                comm = await connect(self.scheduler.address, **self.connection_args)
                 comm.name = "Worker->Scheduler"
                 comm._server = weakref.ref(self)
                 await comm.write(
@@ -1022,7 +1019,9 @@ async def start(self):
         enable_gc_diagnosis()
         thread_state.on_event_loop_thread = True
 
-        await self.listen(self._start_address, listen_args=self.listen_args)
+        await self.listen(
+            self._start_address, **self.security.get_listen_args("worker")
+        )
         self.ip = get_address_host(self.address)
 
         if self.name is None:
@@ -1185,7 +1184,7 @@ def send_to_worker(self, address, msg):
 
             async def batched_send_connect():
                 comm = await connect(
-                    address, connection_args=self.connection_args  # TODO, serialization
+                    address, **self.connection_args  # TODO, serialization
                 )
                 comm.name = "Worker->Worker"
                 await comm.write({"op": "connection_stream"})

From 4055fd7f94c49e15583753cb76befbabbbc37f5c Mon Sep 17 00:00:00 2001
From: Martin Durant <martindurant@users.noreply.github.com>
Date: Tue, 7 Apr 2020 11:15:17 -0400
Subject: [PATCH 0777/1550] don't make task graphs too big (#3671)

Set a maximum size for which task graphs are displayed in the dashboard
---
 distributed/dashboard/components/scheduler.py |  6 ++++
 .../dashboard/tests/test_scheduler_bokeh.py   | 30 +++++++++++++++++++
 distributed/distributed.yaml                  |  1 +
 3 files changed, 37 insertions(+)

diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index c376e2098e4..6b7a77d9479 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -34,6 +34,7 @@
 from bokeh.transform import factor_cmap, linear_cmap
 from bokeh.io import curdoc
 import dask
+from dask import config
 from dask.utils import format_bytes, key_split
 from tlz import pipe
 from tlz.curried import map, concat, groupby
@@ -1171,6 +1172,7 @@ def __init__(self, scheduler, **kwargs):
         tap = TapTool(callback=OpenURL(url="info/task/@key.html"), renderers=[rect])
         rect.nonselection_glyph = None
         self.root.add_tools(hover, tap)
+        self.max_items = config.get("distributed.dashboard.graph-max-items", 5000)
 
     @without_property_validation
     def update(self):
@@ -1206,6 +1208,10 @@ def add_new_nodes_edges(self, new, new_edges, update=False):
             y = self.layout.y
 
             tasks = self.scheduler.tasks
+            if len(tasks) > self.max_items:
+                # graph to big - no update, reset for next time
+                self.invisible_count = len(tasks)
+                return
             for key in new:
                 try:
                     task = tasks[key]
diff --git a/distributed/dashboard/tests/test_scheduler_bokeh.py b/distributed/dashboard/tests/test_scheduler_bokeh.py
index f36bfd897e1..65b5fa25d50 100644
--- a/distributed/dashboard/tests/test_scheduler_bokeh.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh.py
@@ -36,6 +36,7 @@
     ProfileServer,
     MemoryByKey,
 )
+from distributed.utils_test import async_wait_for
 
 from distributed.dashboard import scheduler
 
@@ -501,6 +502,35 @@ def test_TaskGraph_clear(c, s, a, b):
         assert time() < start + 5
 
 
+@gen_cluster(
+    client=True, config={"distributed.dashboard.graph-max-items": 2,},
+)
+def test_TaskGraph_limit(c, s, a, b):
+    gp = TaskGraph(s)
+
+    def func(x):
+        return x
+
+    f1 = c.submit(func, 1)
+    yield wait(f1)
+    gp.update()
+    assert len(gp.node_source.data["x"]) == 1
+    f2 = c.submit(func, 2)
+    yield wait(f2)
+    gp.update()
+    assert len(gp.node_source.data["x"]) == 2
+    f3 = c.submit(func, 3)
+    yield wait(f3)
+    gp.update()
+    assert len(gp.node_source.data["x"]) == 2
+    del f1
+    del f2
+    del f3
+    _ = c.submit(func, 1)
+
+    async_wait_for(lambda: len(gp.node_source.data["x"]) == 1, timeout=1)
+
+
 @gen_cluster(client=True, timeout=30)
 def test_TaskGraph_complex(c, s, a, b):
     da = pytest.importorskip("dask.array")
diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index ca31b17776e..97ca6be3945 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -129,6 +129,7 @@ distributed:
   dashboard:
     link: "{scheme}://{host}:{port}/status"
     export-tool: False
+    graph-max-items: 5000  # maximum number of tasks to try to plot in graph view
 
   ##################
   # Administrative #

From ffeaa97500d1ac0ea0bd0d38c95ca3a6f8c7e1dd Mon Sep 17 00:00:00 2001
From: Lucas Rademaker <44430780+lr4d@users.noreply.github.com>
Date: Wed, 8 Apr 2020 11:19:30 +0200
Subject: [PATCH 0778/1550] Refactor semaphore internals: make `_get_lease`
 synchronous (#3679)

Co-authored-by: lr4d <lr4d@users.noreply.github.com>
---
 distributed/semaphore.py | 14 ++------------
 1 file changed, 2 insertions(+), 12 deletions(-)

diff --git a/distributed/semaphore.py b/distributed/semaphore.py
index 6f5553af0e8..2d506c8ed0e 100644
--- a/distributed/semaphore.py
+++ b/distributed/semaphore.py
@@ -83,11 +83,9 @@ def create(self, comm=None, name=None, max_leases=None):
                     % (max_leases, self.max_leases[name])
                 )
 
-    async def _get_lease(self, client, name, identifier):
+    def _get_lease(self, client, name, identifier):
         result = True
         if len(self.leases[name]) < self.max_leases[name]:
-            # naive: self.leases[resource] += 1
-            # not naive:
             self.leases[name].append(identifier)
             self.leases_per_client[client][name].append(identifier)
         else:
@@ -116,15 +114,7 @@ async def acquire(
                 # is changed and helps to identify when it is worth to retry an acquire
                 self.events[name].clear()
 
-                # If we hit the timeout, this cancels the _get_lease
-                future = asyncio.wait_for(
-                    self._get_lease(client, name, identifier), timeout=w.leftover()
-                )
-
-                try:
-                    result = await future
-                except TimeoutError:
-                    result = False
+                result = self._get_lease(client, name, identifier)
 
                 # If acquiring fails, we wait for the event to be set, i.e. something has
                 # been released and we can try to acquire again (continue loop)

From 2aa3ee7be1b11e4326d679d9d3c7bf906ea0623d Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 8 Apr 2020 12:08:30 -0700
Subject: [PATCH 0779/1550] Support preload modules in Nanny (#3678)

Adds support for running preload scripts on the Nanny
---
 distributed/cli/dask_worker.py               | 10 ++++++++
 distributed/cli/tests/test_dask_scheduler.py | 25 ++++++++++++++++++++
 distributed/distributed.yaml                 | 12 ++++++----
 distributed/nanny.py                         | 24 +++++++++++++++++++
 distributed/tests/test_preload.py            |  9 ++++++-
 distributed/worker.py                        |  2 +-
 6 files changed, 76 insertions(+), 6 deletions(-)

diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
index ff6d09b4c9a..efc330a4a24 100755
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -216,6 +216,14 @@
 @click.argument(
     "preload_argv", nargs=-1, type=click.UNPROCESSED, callback=validate_preload_argv
 )
+@click.option(
+    "--preload-nanny",
+    type=str,
+    multiple=True,
+    is_eager=True,
+    help="Module that should be loaded by each nanny "
+    'like "foo.bar" or "/path/to/foo.py"',
+)
 @click.version_option()
 def main(
     scheduler,
@@ -240,6 +248,7 @@ def main(
     tls_key,
     dashboard_address,
     worker_class,
+    preload_nanny,
     **kwargs
 ):
     g0, g1, g2 = gc.get_threshold()  # https://github.com/dask/distributed/issues/1653
@@ -349,6 +358,7 @@ def del_pid_file():
     worker_class = import_term(worker_class)
     if nanny:
         kwargs["worker_class"] = worker_class
+        kwargs["preload_nanny"] = preload_nanny
 
     if nanny:
         kwargs.update({"worker_port": worker_port, "listen_address": listen_address})
diff --git a/distributed/cli/tests/test_dask_scheduler.py b/distributed/cli/tests/test_dask_scheduler.py
index 2206c173925..62c79f8c0b8 100644
--- a/distributed/cli/tests/test_dask_scheduler.py
+++ b/distributed/cli/tests/test_dask_scheduler.py
@@ -394,3 +394,28 @@ def test_idle_timeout(loop):
     )
     stop = time()
     assert 1 < stop - start < 10
+
+
+def test_multiple_workers(loop):
+    text = """
+def dask_setup(worker):
+    worker.foo = 'setup'
+"""
+    with popen(["dask-scheduler", "--no-dashboard"]) as s:
+        with popen(
+            [
+                "dask-worker",
+                "localhost:8786",
+                "--no-dashboard",
+                "--preload",
+                text,
+                "--preload-nanny",
+                text,
+            ]
+        ) as a:
+            with Client("127.0.0.1:8786", loop=loop) as c:
+                c.wait_for_workers(1)
+                [foo] = c.run(lambda dask_worker: dask_worker.foo).values()
+                assert foo == "setup"
+                [foo] = c.run(lambda dask_worker: dask_worker.foo, nanny=True).values()
+                assert foo == "setup"
diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index 97ca6be3945..bd1dd85c57e 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -22,8 +22,8 @@ distributed:
     work-stealing-interval: 100ms  # Callback time for work stealing
     worker-ttl: null        # like '60s'. Time to live for workers.  They must heartbeat faster than this
     pickle: True            # Is the scheduler allowed to deserialize arbitrary bytestrings
-    preload: []
-    preload-argv: []
+    preload: []             # Run custom modules with Scheduler
+    preload-argv: []        # See https://docs.dask.org/en/latest/setup/custom-startup.html
     unknown-task-duration: 500ms  # Default duration for all tasks with unknown durations ("15m", "2h")
     default-task-durations:  # How long we expect function names to run ("1h", "1s") (helps for long tasks)
       rechunk-split: 1us
@@ -48,8 +48,8 @@ distributed:
     connections:            # Maximum concurrent connections for data
       outgoing: 50          # This helps to control network saturation
       incoming: 10
-    preload: []
-    preload-argv: []
+    preload: []             # Run custom modules with Worker
+    preload-argv: []        # See https://docs.dask.org/en/latest/setup/custom-startup.html
     daemon: True
     validate: False         # Check worker state at every step for debugging
     lifetime:
@@ -71,6 +71,10 @@ distributed:
       pause: 0.80  # fraction at which we pause worker threads
       terminate: 0.95  # fraction at which we terminate the worker
 
+  nanny:
+    preload: []             # Run custom modules with Nanny
+    preload-argv: []        # See https://docs.dask.org/en/latest/setup/custom-startup.html
+
   client:
     heartbeat: 5s  # time between client heartbeats
 
diff --git a/distributed/nanny.py b/distributed/nanny.py
index d3a4d2dc82b..3f7c20f98f9 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -19,6 +19,7 @@
 from .core import RPCClosed, CommClosedError, coerce_to_address
 from .metrics import time
 from .node import ServerNode
+from . import preloading
 from .process import AsyncProcess
 from .proctitle import enable_proctitle_on_children
 from .security import Security
@@ -78,6 +79,8 @@ def __init__(
         death_timeout=None,
         preload=None,
         preload_argv=None,
+        preload_nanny=None,
+        preload_nanny_argv=None,
         security=None,
         contact_address=None,
         listen_address=None,
@@ -121,12 +124,21 @@ def __init__(
         self.validate = validate
         self.resources = resources
         self.death_timeout = parse_timedelta(death_timeout)
+
         self.preload = preload
         if self.preload is None:
             self.preload = dask.config.get("distributed.worker.preload")
         self.preload_argv = preload_argv
         if self.preload_argv is None:
             self.preload_argv = dask.config.get("distributed.worker.preload-argv")
+
+        self.preload_nanny = preload_nanny
+        if self.preload_nanny is None:
+            self.preload_nanny = dask.config.get("distributed.nanny.preload")
+        self.preload_nanny_argv = preload_nanny_argv
+        if self.preload_nanny_argv is None:
+            self.preload_nanny_argv = dask.config.get("distributed.nanny.preload-argv")
+
         self.Worker = Worker if worker_class is None else worker_class
         self.env = env or {}
         self.config = config or {}
@@ -157,6 +169,10 @@ def __init__(
 
         self.local_directory = local_directory
 
+        self._preload_modules = preloading.on_creation(
+            self.preload_nanny, file_dir=self.local_directory
+        )
+
         self.services = services
         self.name = name
         self.quiet = quiet
@@ -248,6 +264,10 @@ async def start(self):
         )
         self.ip = get_address_host(self.address)
 
+        await preloading.on_start(
+            self._preload_modules, self, argv=self.preload_nanny_argv,
+        )
+
         logger.info("        Start Nanny at: %r", self.address)
         response = await self.instantiate()
         if response == "running":
@@ -445,6 +465,9 @@ async def close(self, comm=None, timeout=5, report=None):
 
         self.status = "closing"
         logger.info("Closing Nanny at %r", self.address)
+
+        await preloading.on_teardown(self._preload_modules, self)
+
         self.stop()
         try:
             if self.process is not None:
@@ -519,6 +542,7 @@ async def start(self):
         self.running = asyncio.Event()
         self.stopped = asyncio.Event()
         self.status = "starting"
+
         try:
             await self.process.start()
         except OSError:
diff --git a/distributed/tests/test_preload.py b/distributed/tests/test_preload.py
index 888e7c42ea2..4f60ca586f9 100644
--- a/distributed/tests/test_preload.py
+++ b/distributed/tests/test_preload.py
@@ -62,13 +62,20 @@ async def test_worker_preload_config(cleanup):
     text = """
 def dask_setup(worker):
     worker.foo = 'setup'
+
+def dask_teardown(worker):
+    worker.foo = 'teardown'
 """
-    with dask.config.set({"distributed.worker.preload": text}):
+    with dask.config.set(
+        {"distributed.worker.preload": text, "distributed.nanny.preload": text,}
+    ):
         async with Scheduler(port=0) as s:
             async with Nanny(s.address) as w:
+                assert w.foo == "setup"
                 async with Client(s.address, asynchronous=True) as c:
                     d = await c.run(lambda dask_worker: dask_worker.foo)
                     assert d == {w.worker_address: "setup"}
+            assert w.foo == "teardown"
 
 
 def test_worker_preload_module(loop):
diff --git a/distributed/worker.py b/distributed/worker.py
index 0c6c8b46ba5..f2f832f6d83 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -452,7 +452,7 @@ def __init__(
         # Target interface on which we contact the scheduler by default
         # TODO: it is unfortunate that we special-case inproc here
         if not host and not interface and not scheduler_addr.startswith("inproc://"):
-            host = get_ip(get_address_host(scheduler_addr))
+            host = get_ip(get_address_host(scheduler_addr.split("://")[-1]))
 
         self._start_address = address_from_user_args(
             host=host,

From 8116a266109cab5bcdb8d950f72235e9e050a7d5 Mon Sep 17 00:00:00 2001
From: Abdulelah Bin Mahfoodh <abduh.bm@gmail.com>
Date: Thu, 9 Apr 2020 19:45:42 +0300
Subject: [PATCH 0780/1550] Fix dask-ssh after removing local-directory from
 dask_scheduler cli (#3684)

* Fix dask-ssh after removing local-directory keyword from dask_scheduler

* black changes

* Add a test to dask-ssh with local directory parameter
---
 distributed/cli/dask_ssh.py          |  4 +---
 distributed/deploy/old_ssh.py        | 15 +--------------
 distributed/deploy/tests/test_ssh.py | 18 ++++++++++++++++++
 3 files changed, 20 insertions(+), 17 deletions(-)

diff --git a/distributed/cli/dask_ssh.py b/distributed/cli/dask_ssh.py
index eb09f49cfed..7674632807c 100755
--- a/distributed/cli/dask_ssh.py
+++ b/distributed/cli/dask_ssh.py
@@ -81,9 +81,7 @@
     "--local-directory",
     default=None,
     type=click.Path(exists=True),
-    help=(
-        "Directory to use on all cluster nodes to place workers " "and scheduler files."
-    ),
+    help=("Directory to use on all cluster nodes to place workers files."),
 )
 @click.option(
     "--remote-python", default=None, type=str, help="Path to Python on remote nodes."
diff --git a/distributed/deploy/old_ssh.py b/distributed/deploy/old_ssh.py
index 33e69772f9b..648d7b80905 100644
--- a/distributed/deploy/old_ssh.py
+++ b/distributed/deploy/old_ssh.py
@@ -209,24 +209,12 @@ def communicate():
 
 
 def start_scheduler(
-    logdir,
-    addr,
-    port,
-    ssh_username,
-    ssh_port,
-    ssh_private_key,
-    remote_python=None,
-    local_directory=None,
+    logdir, addr, port, ssh_username, ssh_port, ssh_private_key, remote_python=None,
 ):
     cmd = "{python} -m distributed.cli.dask_scheduler --port {port}".format(
         python=remote_python or sys.executable, port=port, logdir=logdir
     )
 
-    if local_directory is not None:
-        cmd += " --local-directory {local_directory}".format(
-            local_directory=local_directory
-        )
-
     # Optionally re-direct stdout and stderr to a logfile
     if logdir is not None:
         cmd = "mkdir -p {logdir} && ".format(logdir=logdir) + cmd
@@ -422,7 +410,6 @@ def __init__(
             ssh_port,
             ssh_private_key,
             remote_python,
-            local_directory,
         )
 
         # Start worker nodes
diff --git a/distributed/deploy/tests/test_ssh.py b/distributed/deploy/tests/test_ssh.py
index af6bf1566f2..11885dd8612 100644
--- a/distributed/deploy/tests/test_ssh.py
+++ b/distributed/deploy/tests/test_ssh.py
@@ -63,6 +63,24 @@ def test_defer_to_old(loop):
             assert isinstance(c, OldSSHCluster)
 
 
+@pytest.mark.avoid_travis
+def test_old_ssh_wih_local_dir(loop):
+    with pytest.warns(Warning):
+        from distributed.deploy.old_ssh import SSHCluster as OldSSHCluster
+
+        with OldSSHCluster(
+            scheduler_addr="127.0.0.1",
+            scheduler_port=7437,
+            worker_addrs=["127.0.0.1", "127.0.0.1"],
+            local_directory="/tmp",
+        ) as c:
+            assert len(c.workers) == 2
+            with Client(c) as client:
+                result = client.submit(lambda x: x + 1, 10)
+                result = result.result()
+                assert result == 11
+
+
 @pytest.mark.asyncio
 async def test_config_inherited_by_subprocess(loop):
     def f(x):

From c77618c0e62b2f7be70cca5da747d2fcf1ce4f98 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 9 Apr 2020 13:35:06 -0700
Subject: [PATCH 0781/1550] Replace Bokeh Server with Tornado HTTPServer
 (#3658)

This creates a vanilla Tornado HTTPServer on the Worker and Schedulers, and then optionally attaches a Bokeh application to that server.

This lets us always have an HTTP Server running, even if we aren't running the bokeh dashboard. This is helpful for example for prometheus and health checks, and to allow for other uses of the HTTP Server in the future. I hope that it also makes it easier to configure HTTPS globally across the project.

The info, json, proxy, health, and prometheus routes have been moved out of the dashboard directory into a new distributed/http directory.
---
 distributed/cli/dask_scheduler.py             |   7 +-
 distributed/cli/dask_worker.py                |   4 +-
 distributed/cli/tests/test_dask_scheduler.py  |  12 +-
 distributed/client.py                         |   2 +-
 distributed/comm/__init__.py                  |   1 +
 distributed/dashboard/__init__.py             |   2 -
 distributed/dashboard/components/scheduler.py |   4 +-
 distributed/dashboard/components/worker.py    |   7 +-
 distributed/dashboard/core.py                 | 114 ++---
 distributed/dashboard/scheduler.py            | 433 +-----------------
 .../dashboard/tests/test_scheduler_bokeh.py   |  37 +-
 .../dashboard/tests/test_worker_bokeh.py      |  38 +-
 distributed/dashboard/utils.py                |  20 -
 distributed/dashboard/worker.py               | 188 +-------
 distributed/deploy/cluster.py                 |   2 +-
 distributed/deploy/local.py                   |  27 +-
 distributed/deploy/spec.py                    |   8 +-
 distributed/deploy/tests/test_local.py        |  37 +-
 distributed/deploy/tests/test_spec_cluster.py |   8 +-
 distributed/distributed.yaml                  |  19 +
 distributed/http/__init__.py                  |   1 +
 distributed/http/health.py                    |  12 +
 distributed/{dashboard => http}/proxy.py      |  13 +-
 distributed/http/routing.py                   |  22 +
 distributed/http/scheduler/__init__.py        |   0
 distributed/http/scheduler/info.py            | 203 ++++++++
 distributed/http/scheduler/json.py            |  72 +++
 distributed/http/scheduler/prometheus.py      |  99 ++++
 .../scheduler/tests/test_scheduler_http.py}   |  94 ++--
 .../{dashboard => http}/static/css/base.css   |   0
 .../static/css/individual-cluster-map.css     |   0
 .../{dashboard => http}/static/css/status.css |   0
 .../{dashboard => http}/static/css/system.css |   0
 .../static/images/dask-logo.svg               |   0
 .../static/images/fa-bars.svg                 |   0
 .../static/images/favicon.ico                 | Bin
 .../static/individual-cluster-map.html        |   0
 .../static/js/anime.min.js                    |   0
 .../static/js/individual-cluster-map.js       |   0
 .../static/js/reconnecting-websocket.min.js   |   0
 distributed/http/statics.py                   |  10 +
 .../{dashboard => http}/templates/base.html   |   0
 .../templates/call-stack.html                 |   0
 .../templates/json-index.html                 |   0
 .../{dashboard => http}/templates/logs.html   |   0
 .../{dashboard => http}/templates/main.html   |   0
 .../{dashboard => http}/templates/simple.html |   0
 .../{dashboard => http}/templates/status.html |   0
 .../{dashboard => http}/templates/system.html |   0
 .../{dashboard => http}/templates/task.html   |   0
 .../templates/worker-table.html               |   0
 .../{dashboard => http}/templates/worker.html |   0
 .../templates/workers.html                    |   0
 distributed/http/tests/__init__.py            |   0
 distributed/http/tests/test_core.py           |  11 +
 distributed/http/tests/test_routing.py        |  38 ++
 distributed/http/utils.py                     |  51 +++
 distributed/http/worker/__init__.py           |   0
 distributed/http/worker/prometheus.py         |  98 ++++
 .../worker/tests/test_worker_http.py}         |  11 +-
 distributed/node.py                           |  57 ++-
 distributed/scheduler.py                      |  41 +-
 distributed/tests/test_client.py              |   7 +-
 distributed/tests/test_scheduler.py           |  15 +-
 distributed/tests/test_worker.py              | 133 +++---
 distributed/utils.py                          |  42 ++
 distributed/utils_test.py                     |   4 +-
 distributed/worker.py                         |  22 +-
 setup.py                                      |   2 +-
 69 files changed, 1055 insertions(+), 973 deletions(-)
 create mode 100644 distributed/http/__init__.py
 create mode 100644 distributed/http/health.py
 rename distributed/{dashboard => http}/proxy.py (93%)
 create mode 100644 distributed/http/routing.py
 create mode 100644 distributed/http/scheduler/__init__.py
 create mode 100644 distributed/http/scheduler/info.py
 create mode 100644 distributed/http/scheduler/json.py
 create mode 100644 distributed/http/scheduler/prometheus.py
 rename distributed/{dashboard/tests/test_scheduler_bokeh_html.py => http/scheduler/tests/test_scheduler_http.py} (72%)
 rename distributed/{dashboard => http}/static/css/base.css (100%)
 rename distributed/{dashboard => http}/static/css/individual-cluster-map.css (100%)
 rename distributed/{dashboard => http}/static/css/status.css (100%)
 rename distributed/{dashboard => http}/static/css/system.css (100%)
 rename distributed/{dashboard => http}/static/images/dask-logo.svg (100%)
 rename distributed/{dashboard => http}/static/images/fa-bars.svg (100%)
 rename distributed/{dashboard => http}/static/images/favicon.ico (100%)
 rename distributed/{dashboard => http}/static/individual-cluster-map.html (100%)
 rename distributed/{dashboard => http}/static/js/anime.min.js (100%)
 rename distributed/{dashboard => http}/static/js/individual-cluster-map.js (100%)
 rename distributed/{dashboard => http}/static/js/reconnecting-websocket.min.js (100%)
 create mode 100644 distributed/http/statics.py
 rename distributed/{dashboard => http}/templates/base.html (100%)
 rename distributed/{dashboard => http}/templates/call-stack.html (100%)
 rename distributed/{dashboard => http}/templates/json-index.html (100%)
 rename distributed/{dashboard => http}/templates/logs.html (100%)
 rename distributed/{dashboard => http}/templates/main.html (100%)
 rename distributed/{dashboard => http}/templates/simple.html (100%)
 rename distributed/{dashboard => http}/templates/status.html (100%)
 rename distributed/{dashboard => http}/templates/system.html (100%)
 rename distributed/{dashboard => http}/templates/task.html (100%)
 rename distributed/{dashboard => http}/templates/worker-table.html (100%)
 rename distributed/{dashboard => http}/templates/worker.html (100%)
 rename distributed/{dashboard => http}/templates/workers.html (100%)
 create mode 100644 distributed/http/tests/__init__.py
 create mode 100644 distributed/http/tests/test_core.py
 create mode 100644 distributed/http/tests/test_routing.py
 create mode 100644 distributed/http/utils.py
 create mode 100644 distributed/http/worker/__init__.py
 create mode 100644 distributed/http/worker/prometheus.py
 rename distributed/{dashboard/tests/test_worker_bokeh_html.py => http/worker/tests/test_worker_http.py} (73%)

diff --git a/distributed/cli/dask_scheduler.py b/distributed/cli/dask_scheduler.py
index 78d6623608f..1eeb1e2715f 100755
--- a/distributed/cli/dask_scheduler.py
+++ b/distributed/cli/dask_scheduler.py
@@ -83,7 +83,7 @@
 )
 @click.option("--show/--no-show", default=False, help="Show web UI [default: --show]")
 @click.option(
-    "--dashboard-prefix", type=str, default=None, help="Prefix for the dashboard app"
+    "--dashboard-prefix", type=str, default="", help="Prefix for the dashboard app"
 )
 @click.option(
     "--use-xheaders",
@@ -202,8 +202,9 @@ def del_pid_file():
         security=sec,
         host=host,
         port=port,
-        dashboard_address=dashboard_address if dashboard else None,
-        service_kwargs={"dashboard": {"prefix": dashboard_prefix}},
+        dashboard=dashboard,
+        dashboard_address=dashboard_address,
+        http_prefix=dashboard_prefix,
         **kwargs
     )
     logger.info("-" * 47)
diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
index efc330a4a24..9d73f7af5b2 100755
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -397,8 +397,8 @@ def del_pid_file():
             contact_address=contact_address,
             host=host,
             port=port,
-            dashboard_address=dashboard_address if dashboard else None,
-            service_kwargs={"dashboard": {"prefix": dashboard_prefix}},
+            dashboard=dashboard,
+            dashboard_address=dashboard_address,
             name=name
             if nprocs == 1 or name is None or name == ""
             else str(name) + "-" + str(i),
diff --git a/distributed/cli/tests/test_dask_scheduler.py b/distributed/cli/tests/test_dask_scheduler.py
index 62c79f8c0b8..3e867b1f377 100644
--- a/distributed/cli/tests/test_dask_scheduler.py
+++ b/distributed/cli/tests/test_dask_scheduler.py
@@ -39,8 +39,9 @@ def f():
         with Client("127.0.0.1:%d" % Scheduler.default_port, loop=loop) as c:
             c.sync(f)
 
-    with pytest.raises(Exception):
-        requests.get("http://127.0.0.1:8787/status/")
+        response = requests.get("http://127.0.0.1:8787/status/")
+        assert response.status_code == 404
+
     with pytest.raises(Exception):
         response = requests.get("http://127.0.0.1:9786/info.json")
 
@@ -64,11 +65,8 @@ def test_no_dashboard(loop):
     pytest.importorskip("bokeh")
     with popen(["dask-scheduler", "--no-dashboard"]) as proc:
         with Client("127.0.0.1:%d" % Scheduler.default_port, loop=loop) as c:
-            for i in range(3):
-                line = proc.stderr.readline()
-                assert b"dashboard" not in line.lower()
-            with pytest.raises(Exception):
-                requests.get("http://127.0.0.1:8787/status/")
+            response = requests.get("http://127.0.0.1:8787/status/")
+            assert response.status_code == 404
 
 
 def test_dashboard(loop):
diff --git a/distributed/client.py b/distributed/client.py
index 73cccc8a18a..6545e938511 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -843,7 +843,7 @@ def _repr_html_(self):
 
         if info and "dashboard" in info["services"]:
             text += (
-                "  <li><b>Dashboard: </b><a href='%(web)s' target='_blank'>%(web)s</a>\n"
+                "  <li><b>Dashboard: </b><a href='%(web)s' target='_blank'>%(web)s</a></li>\n"
                 % {"web": self.dashboard_link}
             )
 
diff --git a/distributed/comm/__init__.py b/distributed/comm/__init__.py
index 3537b301573..2ff679ada3d 100644
--- a/distributed/comm/__init__.py
+++ b/distributed/comm/__init__.py
@@ -10,6 +10,7 @@
     get_local_address_for,
 )
 from .core import connect, listen, Comm, CommClosedError
+from .utils import get_tcp_server_address
 
 
 def _register_transports():
diff --git a/distributed/dashboard/__init__.py b/distributed/dashboard/__init__.py
index 675963b1463..e69de29bb2d 100644
--- a/distributed/dashboard/__init__.py
+++ b/distributed/dashboard/__init__.py
@@ -1,2 +0,0 @@
-from .scheduler import BokehScheduler
-from .worker import BokehWorker
diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index 6b7a77d9479..26e60c55bce 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -75,7 +75,9 @@
 from jinja2 import Environment, FileSystemLoader
 
 env = Environment(
-    loader=FileSystemLoader(os.path.join(os.path.dirname(__file__), "..", "templates"))
+    loader=FileSystemLoader(
+        os.path.join(os.path.dirname(__file__), "..", "..", "http", "templates")
+    )
 )
 
 BOKEH_THEME = Theme(os.path.join(os.path.dirname(__file__), "..", "theme.yaml"))
diff --git a/distributed/dashboard/components/worker.py b/distributed/dashboard/components/worker.py
index a11d3047838..a6feb3911e1 100644
--- a/distributed/dashboard/components/worker.py
+++ b/distributed/dashboard/components/worker.py
@@ -37,13 +37,12 @@
 
 logger = logging.getLogger(__name__)
 
-with open(os.path.join(os.path.dirname(__file__), "..", "templates", "base.html")) as f:
-    template_source = f.read()
-
 from jinja2 import Environment, FileSystemLoader
 
 env = Environment(
-    loader=FileSystemLoader(os.path.join(os.path.dirname(__file__), "..", "templates"))
+    loader=FileSystemLoader(
+        os.path.join(os.path.dirname(__file__), "..", "..", "http", "templates")
+    )
 )
 
 BOKEH_THEME = Theme(os.path.join(os.path.dirname(__file__), "..", "theme.yaml"))
diff --git a/distributed/dashboard/core.py b/distributed/dashboard/core.py
index 9b919917a67..6843b0659b3 100644
--- a/distributed/dashboard/core.py
+++ b/distributed/dashboard/core.py
@@ -1,11 +1,14 @@
 from distutils.version import LooseVersion
-import os
+import functools
 import warnings
 
 import bokeh
-from bokeh.server.server import Server
-from tornado import web
-from urllib.parse import urljoin
+from bokeh.server.server import BokehTornado
+from bokeh.server.util import create_hosts_whitelist
+from bokeh.application.handlers.function import FunctionHandler
+from bokeh.application import Application
+import dask
+import toolz
 
 
 if LooseVersion(bokeh.__version__) < LooseVersion("0.13.0"):
@@ -16,87 +19,28 @@
     raise ImportError("Dask needs bokeh >= 0.13.0")
 
 
-class BokehServer:
-    server_kwargs = {}
+def BokehApplication(applications, server, prefix="/", template_variables={}):
+    prefix = prefix or ""
+    prefix = "/" + prefix.strip("/")
+    if not prefix.endswith("/"):
+        prefix = prefix + "/"
 
-    def listen(self, addr):
-        if self.server:
-            return
-        if isinstance(addr, tuple):
-            ip, port = addr
-        else:
-            port = addr
-            ip = None
-        for i in range(5):
-            try:
-                server_kwargs = dict(
-                    port=port,
-                    address=ip,
-                    check_unused_sessions_milliseconds=500,
-                    allow_websocket_origin=["*"],
-                    use_index=False,
-                    extra_patterns=[
-                        (
-                            r"/",
-                            web.RedirectHandler,
-                            {"url": urljoin(self.prefix.rstrip("/") + "/", r"status")},
-                        )
-                    ],
-                )
-                server_kwargs.update(self.server_kwargs)
-                self.server = Server(self.apps, **server_kwargs)
-                self.server.start()
+    extra = toolz.merge({"prefix": prefix}, template_variables)
 
-                handlers = [
-                    (
-                        self.prefix + r"/statics/(.*)",
-                        web.StaticFileHandler,
-                        {"path": os.path.join(os.path.dirname(__file__), "static")},
-                    )
-                ]
-
-                self.server._tornado.add_handlers(r".*", handlers)
-
-                return
-            except (SystemExit, EnvironmentError) as exc:
-                if port != 0:
-                    if "already in use" in str(
-                        exc
-                    ) or "Only one usage of" in str(  # Unix/Mac
-                        exc
-                    ):  # Windows
-                        msg = (
-                            "Port %d is already in use. "
-                            "\nPerhaps you already have a cluster running?"
-                            "\nHosting the diagnostics dashboard on a random port instead."
-                            % port
-                        )
-                    else:
-                        msg = (
-                            "Failed to start diagnostics server on port %d. " % port
-                            + str(exc)
-                        )
-                    warnings.warn("\n" + msg)
-                    port = 0
-                if i == 4:
-                    raise
-
-    @property
-    def port(self):
-        return (
-            self.server.port
-            or list(self.server._http._sockets.values())[0].getsockname()[1]
-        )
-
-    def stop(self):
-        for context in self.server._tornado._applications.values():
-            context.run_unload_hook()
-
-        self.server._tornado._stats_job.stop()
-        self.server._tornado._cleanup_job.stop()
-        if self.server._tornado._ping_job is not None:
-            self.server._tornado._ping_job.stop()
+    apps = {
+        prefix + k.lstrip("/"): functools.partial(v, server, extra)
+        for k, v in applications.items()
+    }
+    apps = {k: Application(FunctionHandler(v)) for k, v in apps.items()}
+    kwargs = dask.config.get("distributed.scheduler.dashboard.bokeh-application").copy()
+    extra_websocket_origins = create_hosts_whitelist(
+        kwargs.pop("allow_websocket_origin"), server.http_server.port
+    )
 
-        # https://github.com/bokeh/bokeh/issues/5494
-        if LooseVersion(bokeh.__version__) >= "0.12.4":
-            self.server.stop()
+    application = BokehTornado(
+        apps,
+        use_index=False,
+        extra_websocket_origins=extra_websocket_origins,
+        **kwargs,
+    )
+    return application
diff --git a/distributed/dashboard/scheduler.py b/distributed/dashboard/scheduler.py
index 982bc424826..825195ecefa 100644
--- a/distributed/dashboard/scheduler.py
+++ b/distributed/dashboard/scheduler.py
@@ -1,23 +1,14 @@
-from datetime import datetime
-from functools import partial
-import os
-import os.path
-import json
-import logging
+from urllib.parse import urljoin
 
-import dask
-from dask.utils import format_bytes
-
-from tlz import merge, merge_with
-
-from tornado import escape
-from tornado.websocket import WebSocketHandler
+from tornado.ioloop import IOLoop
+from tornado import web
 
 try:
     import numpy as np
 except ImportError:
     np = False
 
+from .core import BokehApplication
 from .components.worker import counters_doc
 from .components.scheduler import (
     systemmonitor_doc,
@@ -42,419 +33,31 @@
     individual_bandwidth_workers_doc,
     individual_memory_by_key_doc,
 )
-from .core import BokehServer
 from .worker import counters_doc
-from .proxy import GlobalProxyHandler
-from .utils import RequestHandler, redirect
-from ..diagnostics.websocket import WebsocketPlugin
-from ..metrics import time
-from ..utils import log_errors, format_time
-from ..scheduler import ALL_TASK_STATES
-
-
-ns = {
-    func.__name__: func
-    for func in [format_bytes, format_time, datetime.fromtimestamp, time]
-}
-
-rel_path_statics = {"rel_path_statics": "../../"}
 
 
-logger = logging.getLogger(__name__)
-
 template_variables = {
     "pages": ["status", "workers", "tasks", "system", "profile", "graph", "info"]
 }
 
 
-class Workers(RequestHandler):
-    def get(self):
-        with log_errors():
-            self.render(
-                "workers.html",
-                title="Workers",
-                scheduler=self.server,
-                **merge(self.server.__dict__, ns, self.extra, rel_path_statics),
-            )
-
-
-class Worker(RequestHandler):
-    def get(self, worker):
-        worker = escape.url_unescape(worker)
-        if worker not in self.server.workers:
-            self.send_error(404)
-            return
-        with log_errors():
-            self.render(
-                "worker.html",
-                title="Worker: " + worker,
-                scheduler=self.server,
-                Worker=worker,
-                **merge(self.server.__dict__, ns, self.extra, rel_path_statics),
-            )
-
-
-class Task(RequestHandler):
-    def get(self, task):
-        task = escape.url_unescape(task)
-        if task not in self.server.tasks:
-            self.send_error(404)
-            return
-        with log_errors():
-            self.render(
-                "task.html",
-                title="Task: " + task,
-                Task=task,
-                scheduler=self.server,
-                **merge(self.server.__dict__, ns, self.extra, rel_path_statics),
-            )
-
-
-class Logs(RequestHandler):
-    def get(self):
-        with log_errors():
-            logs = self.server.get_logs()
-            self.render(
-                "logs.html",
-                title="Logs",
-                logs=logs,
-                **merge(self.extra, rel_path_statics),
-            )
-
-
-class WorkerLogs(RequestHandler):
-    async def get(self, worker):
-        with log_errors():
-            worker = escape.url_unescape(worker)
-            logs = await self.server.get_worker_logs(workers=[worker])
-            logs = logs[worker]
-            self.render(
-                "logs.html",
-                title="Logs: " + worker,
-                logs=logs,
-                **merge(self.extra, rel_path_statics),
-            )
-
-
-class WorkerCallStacks(RequestHandler):
-    async def get(self, worker):
-        with log_errors():
-            worker = escape.url_unescape(worker)
-            keys = self.server.processing[worker]
-            call_stack = await self.server.get_call_stack(keys=keys)
-            self.render(
-                "call-stack.html",
-                title="Call Stacks: " + worker,
-                call_stack=call_stack,
-                **merge(self.extra, rel_path_statics),
-            )
-
-
-class TaskCallStack(RequestHandler):
-    async def get(self, key):
-        with log_errors():
-            key = escape.url_unescape(key)
-            call_stack = await self.server.get_call_stack(keys=[key])
-            if not call_stack:
-                self.write(
-                    "<p>Task not actively running. "
-                    "It may be finished or not yet started</p>"
-                )
-            else:
-                self.render(
-                    "call-stack.html",
-                    title="Call Stack: " + key,
-                    call_stack=call_stack,
-                    **merge(self.extra, rel_path_statics),
-                )
-
-
-class CountsJSON(RequestHandler):
-    def get(self):
-        scheduler = self.server
-        erred = 0
-        nbytes = 0
-        nthreads = 0
-        memory = 0
-        processing = 0
-        released = 0
-        waiting = 0
-        waiting_data = 0
-        desired_workers = scheduler.adaptive_target()
-
-        for ts in scheduler.tasks.values():
-            if ts.exception_blame is not None:
-                erred += 1
-            elif ts.state == "released":
-                released += 1
-            if ts.waiting_on:
-                waiting += 1
-            if ts.waiters:
-                waiting_data += 1
-        for ws in scheduler.workers.values():
-            nthreads += ws.nthreads
-            memory += len(ws.has_what)
-            nbytes += ws.nbytes
-            processing += len(ws.processing)
-
-        response = {
-            "bytes": nbytes,
-            "clients": len(scheduler.clients),
-            "cores": nthreads,
-            "erred": erred,
-            "hosts": len(scheduler.host_info),
-            "idle": len(scheduler.idle),
-            "memory": memory,
-            "processing": processing,
-            "released": released,
-            "saturated": len(scheduler.saturated),
-            "tasks": len(scheduler.tasks),
-            "unrunnable": len(scheduler.unrunnable),
-            "waiting": waiting,
-            "waiting_data": waiting_data,
-            "workers": len(scheduler.workers),
-            "desired_workers": desired_workers,
-        }
-        self.write(response)
-
-
-class IdentityJSON(RequestHandler):
-    def get(self):
-        self.write(self.server.identity())
-
-
-class IndexJSON(RequestHandler):
-    def get(self):
-        with log_errors():
-            r = [url for url, _ in routes if url.endswith(".json")]
-            self.render(
-                "json-index.html", routes=r, title="Index of JSON routes", **self.extra
-            )
-
-
-class IndividualPlots(RequestHandler):
-    def get(self):
-        bokeh_server = self.server.services["dashboard"]
-        individual_bokeh = {
-            uri.strip("/").replace("-", " ").title(): uri
-            for uri in bokeh_server.apps
-            if uri.lstrip("/").startswith("individual-") and not uri.endswith(".json")
-        }
-        individual_static = {
-            uri.strip("/").replace(".html", "").replace("-", " ").title(): "/statics/"
-            + uri
-            for uri in os.listdir(os.path.join(os.path.dirname(__file__), "static"))
-            if uri.lstrip("/").startswith("individual-") and uri.endswith(".html")
-        }
-        result = {**individual_bokeh, **individual_static}
-        self.write(result)
-
-
-class _PrometheusCollector:
-    def __init__(self, server):
-        self.server = server
-
-    def collect(self):
-        from prometheus_client.core import GaugeMetricFamily, CounterMetricFamily
-
-        yield GaugeMetricFamily(
-            "dask_scheduler_clients",
-            "Number of clients connected.",
-            value=len(self.server.clients),
-        )
-
-        yield GaugeMetricFamily(
-            "dask_scheduler_desired_workers",
-            "Number of workers scheduler needs for task graph.",
-            value=self.server.adaptive_target(),
-        )
+def connect(application, http_server, scheduler, prefix=""):
+    bokeh_app = BokehApplication(
+        applications, scheduler, prefix=prefix, template_variables=template_variables
+    )
+    application.add_application(bokeh_app)
+    bokeh_app.initialize(IOLoop.current())
 
-        worker_states = GaugeMetricFamily(
-            "dask_scheduler_workers",
-            "Number of workers known by scheduler.",
-            labels=["state"],
-        )
-        worker_states.add_metric(["connected"], len(self.server.workers))
-        worker_states.add_metric(["saturated"], len(self.server.saturated))
-        worker_states.add_metric(["idle"], len(self.server.idle))
-        yield worker_states
-
-        tasks = GaugeMetricFamily(
-            "dask_scheduler_tasks",
-            "Number of tasks known by scheduler.",
-            labels=["state"],
-        )
-
-        task_counter = merge_with(
-            sum, (tp.states for tp in self.server.task_prefixes.values())
-        )
-
-        suspicious_tasks = CounterMetricFamily(
-            "dask_scheduler_tasks_suspicious",
-            "Total number of times a task has been marked suspicious",
-            labels=["task_prefix_name"],
-        )
-
-        for tp in self.server.task_prefixes.values():
-            suspicious_tasks.add_metric([tp.name], tp.suspicious)
-        yield suspicious_tasks
-
-        yield CounterMetricFamily(
-            "dask_scheduler_tasks_forgotten",
+    bokeh_app.add_handlers(
+        r".*",
+        [
             (
-                "Total number of processed tasks no longer in memory and already "
-                "removed from the scheduler job queue. Note task groups on the "
-                "scheduler which have all tasks in the forgotten state are not included."
-            ),
-            value=task_counter.get("forgotten", 0.0),
-        )
-
-        for state in ALL_TASK_STATES:
-            tasks.add_metric([state], task_counter.get(state, 0.0))
-        yield tasks
-
-
-class PrometheusHandler(RequestHandler):
-    _collector = None
-
-    def __init__(self, *args, **kwargs):
-        import prometheus_client
-
-        super(PrometheusHandler, self).__init__(*args, **kwargs)
-
-        if PrometheusHandler._collector:
-            # Especially during testing, multiple schedulers are started
-            # sequentially in the same python process
-            PrometheusHandler._collector.server = self.server
-            return
-
-        PrometheusHandler._collector = _PrometheusCollector(self.server)
-        prometheus_client.REGISTRY.register(PrometheusHandler._collector)
-
-    def get(self):
-        import prometheus_client
-
-        self.write(prometheus_client.generate_latest())
-        self.set_header("Content-Type", "text/plain; version=0.0.4")
-
-
-class HealthHandler(RequestHandler):
-    def get(self):
-        self.write("ok")
-        self.set_header("Content-Type", "text/plain")
-
-
-class EventstreamHandler(WebSocketHandler):
-    def initialize(self, server=None, extra=None):
-        self.server = server
-        self.extra = extra or {}
-        self.plugin = WebsocketPlugin(self, server)
-        self.server.add_plugin(self.plugin)
-
-    def send(self, name, data):
-        data["name"] = name
-        for k in list(data):
-            # Drop bytes objects for now
-            if isinstance(data[k], bytes):
-                del data[k]
-        self.write_message(data)
-
-    def open(self):
-        for worker in self.server.workers:
-            self.plugin.add_worker(self.server, worker)
-
-    def on_message(self, message):
-        message = json.loads(message)
-        if message["name"] == "ping":
-            self.send("pong", {"timestamp": str(datetime.now())})
-
-    def on_close(self):
-        self.server.remove_plugin(self.plugin)
-
-
-routes = [
-    (r"info", redirect("info/main/workers.html")),
-    (r"info/main/workers.html", Workers),
-    (r"info/worker/(.*).html", Worker),
-    (r"info/task/(.*).html", Task),
-    (r"info/main/logs.html", Logs),
-    (r"info/call-stacks/(.*).html", WorkerCallStacks),
-    (r"info/call-stack/(.*).html", TaskCallStack),
-    (r"info/logs/(.*).html", WorkerLogs),
-    (r"json/counts.json", CountsJSON),
-    (r"json/identity.json", IdentityJSON),
-    (r"json/index.html", IndexJSON),
-    (r"individual-plots.json", IndividualPlots),
-    (r"metrics", PrometheusHandler),
-    (r"health", HealthHandler),
-    (r"eventstream", EventstreamHandler),
-    (r"proxy/(\d+)/(.*?)/(.*)", GlobalProxyHandler),
-]
-
-
-def get_handlers(server):
-    return [(url, cls, {"server": server}) for url, cls in routes]
-
-
-class BokehScheduler(BokehServer):
-    def __init__(self, scheduler, io_loop=None, prefix="", **kwargs):
-        self.scheduler = scheduler
-        prefix = prefix or ""
-        prefix = prefix.rstrip("/")
-        if prefix and not prefix.startswith("/"):
-            prefix = "/" + prefix
-        self.prefix = prefix
-
-        self.server_kwargs = kwargs
-
-        # TLS configuration
-        http_server_kwargs = kwargs.setdefault("http_server_kwargs", {})
-        tls_key = dask.config.get("distributed.scheduler.dashboard.tls.key")
-        tls_cert = dask.config.get("distributed.scheduler.dashboard.tls.cert")
-        tls_ca_file = dask.config.get("distributed.scheduler.dashboard.tls.ca-file")
-        if tls_cert and "ssl_options" not in http_server_kwargs:
-            import ssl
-
-            ctx = ssl.create_default_context(
-                cafile=tls_ca_file, purpose=ssl.Purpose.SERVER_AUTH
+                r"/",
+                web.RedirectHandler,
+                {"url": urljoin((prefix or "").strip("/") + "/", r"status")},
             )
-            ctx.load_cert_chain(tls_cert, keyfile=tls_key)
-            # Unlike the client/scheduler/worker TLS handling, we don't care
-            # about authenticating the user's webclient, TLS here is just for
-            # encryption. Disable these checks.
-            ctx.check_hostname = False
-            ctx.verify_mode = ssl.CERT_NONE
-            http_server_kwargs["ssl_options"] = ctx
-
-        self.server_kwargs["prefix"] = prefix or None
-
-        self.apps = applications
-        self.apps = {k: partial(v, scheduler, self.extra) for k, v in self.apps.items()}
-
-        self.loop = io_loop or scheduler.loop
-        self.server = None
-
-    @property
-    def extra(self):
-        return merge({"prefix": self.prefix}, template_variables)
-
-    @property
-    def my_server(self):
-        return self.scheduler
-
-    def listen(self, *args, **kwargs):
-        super(BokehScheduler, self).listen(*args, **kwargs)
-
-        handlers = [
-            (
-                self.prefix + "/" + url,
-                cls,
-                {"server": self.my_server, "extra": self.extra},
-            )
-            for url, cls in routes
-        ]
-
-        self.server._tornado.add_handlers(r".*", handlers)
+        ],
+    )
 
 
 applications = {
diff --git a/distributed/dashboard/tests/test_scheduler_bokeh.py b/distributed/dashboard/tests/test_scheduler_bokeh.py
index 65b5fa25d50..49bdfe448bf 100644
--- a/distributed/dashboard/tests/test_scheduler_bokeh.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh.py
@@ -17,9 +17,8 @@
 from distributed.client import wait
 from distributed.metrics import time
 from distributed.utils_test import gen_cluster, inc, dec, slowinc, div, get_cert
-from distributed.dashboard.worker import BokehWorker
 from distributed.dashboard.components.worker import Counters
-from distributed.dashboard.scheduler import applications, BokehScheduler
+from distributed.dashboard.scheduler import applications
 from distributed.dashboard.components.scheduler import (
     SystemMonitor,
     Occupancy,
@@ -46,12 +45,9 @@
 @pytest.mark.skipif(
     sys.version_info[0] == 2, reason="https://github.com/bokeh/bokeh/issues/5494"
 )
-@gen_cluster(
-    client=True, scheduler_kwargs={"services": {("dashboard", 0): BokehScheduler}}
-)
+@gen_cluster(client=True, scheduler_kwargs={"dashboard": True})
 def test_simple(c, s, a, b):
-    assert isinstance(s.services["dashboard"], BokehScheduler)
-    port = s.services["dashboard"].port
+    port = s.http_server.port
 
     future = c.submit(sleep, 1)
     yield gen.sleep(0.1)
@@ -70,7 +66,7 @@ def test_simple(c, s, a, b):
     assert response
 
 
-@gen_cluster(client=True, worker_kwargs=dict(services={"dashboard": BokehWorker}))
+@gen_cluster(client=True, worker_kwargs={"dashboard": True})
 def test_basic(c, s, a, b):
     for component in [TaskStream, SystemMonitor, Occupancy, StealingTimeSeries]:
         ss = component(s)
@@ -592,21 +588,19 @@ def test_profile_server(c, s, a, b):
 
 
 @gen_cluster(
-    client=True, scheduler_kwargs={"services": {("dashboard", 0): BokehScheduler}}
+    client=True, scheduler_kwargs={"dashboard": True},
 )
 def test_root_redirect(c, s, a, b):
     http_client = AsyncHTTPClient()
-    response = yield http_client.fetch(
-        "http://localhost:%d/" % s.services["dashboard"].port
-    )
+    response = yield http_client.fetch("http://localhost:%d/" % s.http_server.port)
     assert response.code == 200
     assert "/status" in response.effective_url
 
 
 @gen_cluster(
     client=True,
-    scheduler_kwargs={"services": {("dashboard", 0): BokehScheduler}},
-    worker_kwargs={"services": {"dashboard": BokehWorker}},
+    scheduler_kwargs={"dashboard": True},
+    worker_kwargs={"dashboard": True},
     timeout=180,
 )
 def test_proxy_to_workers(c, s, a, b):
@@ -617,7 +611,7 @@ def test_proxy_to_workers(c, s, a, b):
     except ImportError:
         proxy_exists = False
 
-    dashboard_port = s.services["dashboard"].port
+    dashboard_port = s.http_server.port
     http_client = AsyncHTTPClient()
     response = yield http_client.fetch("http://localhost:%d/" % dashboard_port)
     assert response.code == 200
@@ -625,7 +619,7 @@ def test_proxy_to_workers(c, s, a, b):
 
     for w in [a, b]:
         host = w.ip
-        port = w.service_ports["dashboard"]
+        port = w.http_server.port
         proxy_url = "http://localhost:%d/proxy/%s/%s/status" % (
             dashboard_port,
             port,
@@ -647,7 +641,7 @@ def test_proxy_to_workers(c, s, a, b):
 
 @gen_cluster(
     client=True,
-    scheduler_kwargs={"services": {("dashboard", 0): BokehScheduler}},
+    scheduler_kwargs={"dashboard": True},
     config={
         "distributed.scheduler.dashboard.tasks.task-stream-length": 10,
         "distributed.scheduler.dashboard.status.task-stream-length": 10,
@@ -675,7 +669,7 @@ async def test_lots_of_tasks(c, s, a, b):
 
 @gen_cluster(
     client=True,
-    scheduler_kwargs={"services": {("dashboard", 0): BokehScheduler}},
+    scheduler_kwargs={"dashboard": True},
     config={
         "distributed.scheduler.dashboard.tls.key": get_cert("tls-key.pem"),
         "distributed.scheduler.dashboard.tls.cert": get_cert("tls-cert.pem"),
@@ -683,8 +677,7 @@ async def test_lots_of_tasks(c, s, a, b):
     },
 )
 def test_https_support(c, s, a, b):
-    assert isinstance(s.services["dashboard"], BokehScheduler)
-    port = s.services["dashboard"].port
+    port = s.http_server.port
 
     assert (
         format_dashboard_link("localhost", port) == "https://localhost:%d/status" % port
@@ -717,9 +710,7 @@ def test_https_support(c, s, a, b):
         assert not re.search("href=./", body)  # no absolute links
 
 
-@gen_cluster(
-    client=True, scheduler_kwargs={"services": {("dashboard", 0): BokehScheduler}}
-)
+@gen_cluster(client=True, scheduler_kwargs={"dashboard": True})
 async def test_memory_by_key(c, s, a, b):
     mbk = MemoryByKey(s)
 
diff --git a/distributed/dashboard/tests/test_worker_bokeh.py b/distributed/dashboard/tests/test_worker_bokeh.py
index 97729fce14f..873cc1c1f3e 100644
--- a/distributed/dashboard/tests/test_worker_bokeh.py
+++ b/distributed/dashboard/tests/test_worker_bokeh.py
@@ -5,7 +5,6 @@
 import pytest
 
 pytest.importorskip("bokeh")
-import sys
 from tlz import first
 from tornado import gen
 from tornado.httpclient import AsyncHTTPClient
@@ -13,8 +12,6 @@
 from distributed.client import wait
 from distributed.metrics import time
 from distributed.utils_test import gen_cluster, inc, dec
-from distributed.dashboard.scheduler import BokehScheduler
-from distributed.dashboard.worker import BokehWorker
 from distributed.dashboard.components.worker import (
     StateTable,
     CrossFilter,
@@ -28,13 +25,11 @@
 
 @gen_cluster(
     client=True,
-    worker_kwargs={"services": {("dashboard", 0): BokehWorker}},
-    scheduler_kwargs={"services": {("dashboard", 0): BokehScheduler}},
+    worker_kwargs={"dashboard": True},
+    scheduler_kwargs={"dashboard": True},
 )
 def test_routes(c, s, a, b):
-    assert isinstance(a.services["dashboard"], BokehWorker)
-    assert isinstance(b.services["dashboard"], BokehWorker)
-    port = a.services["dashboard"].port
+    port = a.http_server.port
 
     future = c.submit(sleep, 1)
     yield gen.sleep(0.1)
@@ -47,37 +42,33 @@ def test_routes(c, s, a, b):
         assert not re.search("href=./", body)  # no absolute links
 
     response = yield http_client.fetch(
-        "http://localhost:%d/info/main/workers.html" % s.services["dashboard"].port
+        "http://localhost:%d/info/main/workers.html" % s.http_server.port
     )
 
     assert str(port) in response.body.decode()
 
 
-@pytest.mark.skipif(
-    sys.version_info[0] == 2, reason="https://github.com/bokeh/bokeh/issues/5494"
-)
-@gen_cluster(client=True, worker_kwargs={"services": {("dashboard", 0): BokehWorker}})
+@gen_cluster(client=True, worker_kwargs={"dashboard": True})
 def test_simple(c, s, a, b):
-    assert s.workers[a.address].services == {"dashboard": a.services["dashboard"].port}
-    assert s.workers[b.address].services == {"dashboard": b.services["dashboard"].port}
+    assert s.workers[a.address].services == {"dashboard": a.http_server.port}
+    assert s.workers[b.address].services == {"dashboard": b.http_server.port}
 
     future = c.submit(sleep, 1)
     yield gen.sleep(0.1)
 
     http_client = AsyncHTTPClient()
-    for suffix in ["main", "crossfilter", "system"]:
+    for suffix in ["crossfilter", "system"]:
         response = yield http_client.fetch(
-            "http://localhost:%d/%s" % (a.services["dashboard"].port, suffix)
+            "http://localhost:%d/%s" % (a.http_server.port, suffix)
         )
         assert "bokeh" in response.body.decode().lower()
 
 
 @gen_cluster(
-    client=True, worker_kwargs={"services": {("dashboard", 0): (BokehWorker, {})}}
+    client=True, worker_kwargs={"dashboard": True},
 )
 def test_services_kwargs(c, s, a, b):
-    assert s.workers[a.address].services == {"dashboard": a.services["dashboard"].port}
-    assert isinstance(a.services["dashboard"], BokehWorker)
+    assert s.workers[a.address].services == {"dashboard": a.http_server.port}
 
 
 @gen_cluster(client=True)
@@ -166,17 +157,14 @@ def test_CommunicatingStream(c, s, a, b):
 
 
 @gen_cluster(
-    client=True,
-    clean_kwargs={"threads": False},
-    worker_kwargs={"services": {("dashboard", 0): BokehWorker}},
+    client=True, clean_kwargs={"threads": False}, worker_kwargs={"dashboard": True},
 )
 def test_prometheus(c, s, a, b):
     pytest.importorskip("prometheus_client")
-    assert s.workers[a.address].services == {"dashboard": a.services["dashboard"].port}
 
     http_client = AsyncHTTPClient()
     for suffix in ["metrics"]:
         response = yield http_client.fetch(
-            "http://localhost:%d/%s" % (a.services["dashboard"].port, suffix)
+            "http://localhost:%d/%s" % (a.http_server.port, suffix)
         )
         assert response.code == 200
diff --git a/distributed/dashboard/utils.py b/distributed/dashboard/utils.py
index 394e016a4da..0de536a6050 100644
--- a/distributed/dashboard/utils.py
+++ b/distributed/dashboard/utils.py
@@ -1,10 +1,8 @@
 from distutils.version import LooseVersion
-import os
 from numbers import Number
 
 import bokeh
 from bokeh.io import curdoc
-from tornado import web
 from tlz import partition
 from tlz.curried import first
 
@@ -15,7 +13,6 @@
 
 
 BOKEH_VERSION = LooseVersion(bokeh.__version__)
-dirname = os.path.dirname(__file__)
 
 
 PROFILING = False
@@ -45,23 +42,6 @@ def transpose(lod):
     return {k: [d[k] for d in lod] for k in keys}
 
 
-class RequestHandler(web.RequestHandler):
-    def initialize(self, server=None, extra=None):
-        self.server = server
-        self.extra = extra or {}
-
-    def get_template_path(self):
-        return os.path.join(dirname, "templates")
-
-
-def redirect(path):
-    class Redirect(RequestHandler):
-        def get(self):
-            self.redirect(path)
-
-    return Redirect
-
-
 @without_property_validation
 def update(source, data):
     """ Update source with data
diff --git a/distributed/dashboard/worker.py b/distributed/dashboard/worker.py
index 54b3a0a4a51..ff9ae3b2f7d 100644
--- a/distributed/dashboard/worker.py
+++ b/distributed/dashboard/worker.py
@@ -1,10 +1,3 @@
-from functools import partial
-import logging
-import os
-
-from bokeh.themes import Theme
-from tlz import merge
-
 from .components.worker import (
     status_doc,
     crossfilter_doc,
@@ -13,177 +6,28 @@
     profile_doc,
     profile_server_doc,
 )
-from .core import BokehServer
-from .utils import RequestHandler, redirect
-
-
-logger = logging.getLogger(__name__)
-
-with open(os.path.join(os.path.dirname(__file__), "templates", "base.html")) as f:
-    template_source = f.read()
-
-from jinja2 import Environment, FileSystemLoader
-
-env = Environment(
-    loader=FileSystemLoader(os.path.join(os.path.dirname(__file__), "templates"))
-)
+from .core import BokehApplication
+from tornado.ioloop import IOLoop
 
-BOKEH_THEME = Theme(os.path.join(os.path.dirname(__file__), "theme.yaml"))
 
 template_variables = {
     "pages": ["status", "system", "profile", "crossfilter", "profile-server"]
 }
 
 
-class _PrometheusCollector:
-    def __init__(self, server):
-        self.worker = server
-        self.logger = logging.getLogger("distributed.dask_worker")
-        self.crick_available = True
-        try:
-            import crick  # noqa: F401
-        except ImportError:
-            self.crick_available = False
-            self.logger.info(
-                "Not all prometheus metrics available are exported. Digest-based metrics require crick to be installed"
-            )
-
-    def collect(self):
-        from prometheus_client.core import GaugeMetricFamily
-
-        tasks = GaugeMetricFamily(
-            "dask_worker_tasks", "Number of tasks at worker.", labels=["state"]
-        )
-        tasks.add_metric(["stored"], len(self.worker.data))
-        tasks.add_metric(["executing"], len(self.worker.executing))
-        tasks.add_metric(["ready"], len(self.worker.ready))
-        tasks.add_metric(["waiting"], len(self.worker.waiting_for_data))
-        tasks.add_metric(["serving"], len(self.worker._comms))
-        yield tasks
-
-        yield GaugeMetricFamily(
-            "dask_worker_connections",
-            "Number of task connections to other workers.",
-            value=len(self.worker.in_flight_workers),
-        )
-
-        yield GaugeMetricFamily(
-            "dask_worker_threads",
-            "Number of worker threads.",
-            value=self.worker.nthreads,
-        )
-
-        yield GaugeMetricFamily(
-            "dask_worker_latency_seconds",
-            "Latency of worker connection.",
-            value=self.worker.latency,
-        )
-
-        # all metrics using digests require crick to be installed
-        # the following metrics will export NaN, if the corresponding digests are None
-        if self.crick_available:
-            yield GaugeMetricFamily(
-                "dask_worker_tick_duration_median_seconds",
-                "Median tick duration at worker.",
-                value=self.worker.digests["tick-duration"].components[1].quantile(50),
-            )
-
-            yield GaugeMetricFamily(
-                "dask_worker_task_duration_median_seconds",
-                "Median task runtime at worker.",
-                value=self.worker.digests["task-duration"].components[1].quantile(50),
-            )
-
-            yield GaugeMetricFamily(
-                "dask_worker_transfer_bandwidth_median_bytes",
-                "Bandwidth for transfer at worker in Bytes.",
-                value=self.worker.digests["transfer-bandwidth"]
-                .components[1]
-                .quantile(50),
-            )
-
-
-class PrometheusHandler(RequestHandler):
-    _initialized = False
-
-    def __init__(self, *args, **kwargs):
-        import prometheus_client
-
-        super(PrometheusHandler, self).__init__(*args, **kwargs)
+def connect(application, http_server, worker, prefix=""):
+    bokeh_app = BokehApplication(
+        applications, worker, prefix=prefix, template_variables=template_variables
+    )
+    application.add_application(bokeh_app)
+    bokeh_app.initialize(IOLoop.current())
 
-        if PrometheusHandler._initialized:
-            return
 
-        prometheus_client.REGISTRY.register(_PrometheusCollector(self.server))
-
-        PrometheusHandler._initialized = True
-
-    def get(self):
-        import prometheus_client
-
-        self.write(prometheus_client.generate_latest())
-        self.set_header("Content-Type", "text/plain; version=0.0.4")
-
-
-class HealthHandler(RequestHandler):
-    def get(self):
-        self.write("ok")
-        self.set_header("Content-Type", "text/plain")
-
-
-routes = [
-    (r"metrics", PrometheusHandler),
-    (r"health", HealthHandler),
-    (r"main", redirect("/status")),
-]
-
-
-def get_handlers(server):
-    return [(url, cls, {"server": server}) for url, cls in routes]
-
-
-class BokehWorker(BokehServer):
-    def __init__(self, worker, io_loop=None, prefix="", **kwargs):
-        self.worker = worker
-        self.server_kwargs = kwargs
-        self.server_kwargs["prefix"] = prefix or None
-        prefix = prefix or ""
-        prefix = prefix.rstrip("/")
-        if prefix and not prefix.startswith("/"):
-            prefix = "/" + prefix
-        self.prefix = prefix
-
-        self.apps = {
-            "/status": status_doc,
-            "/counters": counters_doc,
-            "/crossfilter": crossfilter_doc,
-            "/system": systemmonitor_doc,
-            "/profile": profile_doc,
-            "/profile-server": profile_server_doc,
-        }
-        self.apps = {k: partial(v, worker, self.extra) for k, v in self.apps.items()}
-
-        self.loop = io_loop or worker.loop
-        self.server = None
-
-    @property
-    def extra(self):
-        return merge({"prefix": self.prefix}, template_variables)
-
-    @property
-    def my_server(self):
-        return self.worker
-
-    def listen(self, *args, **kwargs):
-        super(BokehWorker, self).listen(*args, **kwargs)
-
-        handlers = [
-            (
-                self.prefix + "/" + url,
-                cls,
-                {"server": self.my_server, "extra": self.extra},
-            )
-            for url, cls in routes
-        ]
-
-        self.server._tornado.add_handlers(r".*", handlers)
+applications = {
+    "/status": status_doc,
+    "/counters": counters_doc,
+    "/crossfilter": crossfilter_doc,
+    "/system": systemmonitor_doc,
+    "/profile": profile_doc,
+    "/profile-server": profile_server_doc,
+}
diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
index 81f3d578fb2..8082d278483 100644
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -48,7 +48,7 @@ class Cluster:
     _supports_scaling = True
 
     def __init__(self, asynchronous):
-        self.scheduler_info = {}
+        self.scheduler_info = {"workers": {}}
         self.periodic_callbacks = {}
         self._asynchronous = asynchronous
 
diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index d1744ed32c0..cd33ad26a12 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -6,6 +6,7 @@
 
 from dask.utils import factors
 from dask.system import CPU_COUNT
+import toolz
 
 from .spec import SpecCluster
 from ..nanny import Nanny
@@ -110,6 +111,7 @@ def __init__(
         blocked_handlers=None,
         interface=None,
         worker_class=None,
+        scheduler_kwargs=None,
         **worker_kwargs
     ):
         if ip is not None:
@@ -172,6 +174,7 @@ def __init__(
                 "nthreads": threads_per_worker,
                 "services": worker_services,
                 "dashboard_address": worker_dashboard_address,
+                "dashboard": worker_dashboard_address is not None,
                 "interface": interface,
                 "protocol": protocol,
                 "security": security,
@@ -181,16 +184,20 @@ def __init__(
 
         scheduler = {
             "cls": Scheduler,
-            "options": dict(
-                host=host,
-                services=services,
-                service_kwargs=service_kwargs,
-                security=security,
-                port=scheduler_port,
-                interface=interface,
-                protocol=protocol,
-                dashboard_address=dashboard_address,
-                blocked_handlers=blocked_handlers,
+            "options": toolz.merge(
+                dict(
+                    host=host,
+                    services=services,
+                    service_kwargs=service_kwargs,
+                    security=security,
+                    port=scheduler_port,
+                    interface=interface,
+                    protocol=protocol,
+                    dashboard=dashboard_address is not None,
+                    dashboard_address=dashboard_address,
+                    blocked_handlers=blocked_handlers,
+                ),
+                scheduler_kwargs or {},
             ),
         }
 
diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index 17b1af28148..99ab70d2de1 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -268,12 +268,12 @@ async def _start(self):
 
         if self.scheduler_spec is None:
             try:
-                from distributed.dashboard import BokehScheduler
+                import distributed.dashboard  # noqa: F401
             except ImportError:
-                services = {}
+                pass
             else:
-                services = {("dashboard", 8787): BokehScheduler}
-            self.scheduler_spec = {"cls": Scheduler, "options": {"services": services}}
+                options = {"dashboard": True}
+            self.scheduler_spec = {"cls": Scheduler, "options": options}
 
         cls = self.scheduler_spec["cls"]
         if isinstance(cls, str):
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index 403beb3aa41..31fbcebd3b8 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -12,6 +12,7 @@
 from tornado.ioloop import IOLoop
 from tornado import gen
 import tornado
+from tornado.httpclient import AsyncHTTPClient
 import pytest
 
 from dask.system import CPU_COUNT
@@ -196,11 +197,18 @@ def test_Client_solo(loop):
 
 
 @gen_test()
-def test_duplicate_clients():
+async def test_duplicate_clients():
     pytest.importorskip("bokeh")
-    c1 = yield Client(processes=False, silence_logs=False, dashboard_address=9876)
+    c1 = await Client(
+        processes=False, silence_logs=False, dashboard_address=9876, asynchronous=True
+    )
     with pytest.warns(Warning) as info:
-        c2 = yield Client(processes=False, silence_logs=False, dashboard_address=9876)
+        c2 = await Client(
+            processes=False,
+            silence_logs=False,
+            dashboard_address=9876,
+            asynchronous=True,
+        )
 
     assert "dashboard" in c1.cluster.scheduler.services
     assert "dashboard" in c2.cluster.scheduler.services
@@ -212,8 +220,8 @@ def test_duplicate_clients():
         )
         for msg in info.list
     )
-    yield c1.close()
-    yield c2.close()
+    await c1.close()
+    await c2.close()
 
 
 def test_Client_kwargs(loop):
@@ -405,7 +413,7 @@ def test_bokeh(loop, processes):
         processes=processes,
         dashboard_address=0,
     ) as c:
-        bokeh_port = c.scheduler.services["dashboard"].port
+        bokeh_port = c.scheduler.http_server.port
         url = "http://127.0.0.1:%d/status/" % bokeh_port
         start = time()
         while True:
@@ -543,19 +551,22 @@ def test_death_timeout_raises(loop):
 
 
 @pytest.mark.skipif(sys.version_info < (3, 6), reason="Unknown")
-def test_bokeh_kwargs(loop):
+@pytest.mark.asyncio
+async def test_bokeh_kwargs(cleanup):
     pytest.importorskip("bokeh")
-    with LocalCluster(
+    async with LocalCluster(
         n_workers=0,
         scheduler_port=0,
         silence_logs=False,
-        loop=loop,
         dashboard_address=0,
-        service_kwargs={"dashboard": {"prefix": "/foo"}},
+        asynchronous=True,
+        scheduler_kwargs={"http_prefix": "/foo"},
     ) as c:
-
-        bs = c.scheduler.services["dashboard"]
-        assert bs.prefix == "/foo"
+        client = AsyncHTTPClient()
+        response = await client.fetch(
+            "http://localhost:{}/foo/status".format(c.scheduler.http_server.port)
+        )
+        assert "bokeh" in response.body.decode()
 
 
 def test_io_loop_periodic_callbacks(loop):
diff --git a/distributed/deploy/tests/test_spec_cluster.py b/distributed/deploy/tests/test_spec_cluster.py
index ae24e7400e2..c9482f5da56 100644
--- a/distributed/deploy/tests/test_spec_cluster.py
+++ b/distributed/deploy/tests/test_spec_cluster.py
@@ -4,6 +4,7 @@
 
 import dask
 from dask.distributed import SpecCluster, Worker, Client, Scheduler, Nanny
+from distributed.compatibility import WINDOWS
 from distributed.deploy.spec import close_clusters, ProcessInterface, run_spec
 from distributed.metrics import time
 from distributed.utils_test import loop, cleanup  # noqa: F401
@@ -84,9 +85,10 @@ def test_spec_sync(loop):
 
 
 def test_loop_started():
-    cluster = SpecCluster(
+    with SpecCluster(
         worker_spec, scheduler={"cls": Scheduler, "options": {"port": 0}}
-    )
+    ) as cluster:
+        pass
 
 
 @pytest.mark.asyncio
@@ -212,6 +214,7 @@ async def test_restart(cleanup):
                 assert len(cluster.workers) == 2
 
 
+@pytest.mark.skipif(WINDOWS, reason="HTTP Server doesn't close out")
 @pytest.mark.asyncio
 async def test_broken_worker():
     with pytest.raises(Exception) as info:
@@ -225,6 +228,7 @@ async def test_broken_worker():
     assert "Broken" in str(info.value)
 
 
+@pytest.mark.skipif(WINDOWS, reason="HTTP Server doesn't close out")
 @pytest.mark.slow
 def test_spec_close_clusters(loop):
     workers = {0: {"cls": Worker}}
diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index bd1dd85c57e..b11270f4704 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -38,9 +38,22 @@ distributed:
         ca-file: null
         key: null
         cert: null
+      bokeh-application:  # keywords to pass to BokehTornado application
+        allow_websocket_origin: ["*"]
+        keep_alive_milliseconds: 500
+        check_unused_sessions_milliseconds: 500
     locks:
       lease-validation-interval: 10s  # The time to wait until an acquired semaphore is released if the Client goes out of scope
 
+    http:
+      routes:
+        - distributed.http.scheduler.prometheus
+        - distributed.http.scheduler.info
+        - distributed.http.scheduler.json
+        - distributed.http.health
+        - distributed.http.proxy
+        - distributed.http.statics
+
   worker:
     blocked-handlers: []
     multiprocessing-method: spawn
@@ -71,6 +84,12 @@ distributed:
       pause: 0.80  # fraction at which we pause worker threads
       terminate: 0.95  # fraction at which we terminate the worker
 
+    http:
+      routes:
+        - distributed.http.worker.prometheus
+        - distributed.http.health
+        - distributed.http.statics
+
   nanny:
     preload: []             # Run custom modules with Nanny
     preload-argv: []        # See https://docs.dask.org/en/latest/setup/custom-startup.html
diff --git a/distributed/http/__init__.py b/distributed/http/__init__.py
new file mode 100644
index 00000000000..b41a454ed2f
--- /dev/null
+++ b/distributed/http/__init__.py
@@ -0,0 +1 @@
+from .utils import get_handlers
diff --git a/distributed/http/health.py b/distributed/http/health.py
new file mode 100644
index 00000000000..2a45c4abf77
--- /dev/null
+++ b/distributed/http/health.py
@@ -0,0 +1,12 @@
+from tornado import web
+
+
+class HealthHandler(web.RequestHandler):
+    def get(self):
+        self.write("ok")
+        self.set_header("Content-Type", "text/plain")
+
+
+routes = [
+    ("/health", HealthHandler, {}),
+]
diff --git a/distributed/dashboard/proxy.py b/distributed/http/proxy.py
similarity index 93%
rename from distributed/dashboard/proxy.py
rename to distributed/http/proxy.py
index 3e76ba11c0e..c1f437d9b5f 100644
--- a/distributed/dashboard/proxy.py
+++ b/distributed/http/proxy.py
@@ -13,8 +13,8 @@ class GlobalProxyHandler(ProxyHandler):
         from a port to any valid endpoint'.
         """
 
-        def initialize(self, server=None, extra=None):
-            self.scheduler = server
+        def initialize(self, dask_server=None, extra=None):
+            self.scheduler = dask_server
             self.extra = extra or {}
 
         async def http_get(self, port, host, proxied_path):
@@ -77,8 +77,8 @@ class GlobalProxyHandler(web.RequestHandler):
         """Minimal Proxy handler when jupyter-server-proxy is not installed
         """
 
-        def initialize(self, server=None, extra=None):
-            self.server = server
+        def initialize(self, dask_server=None, extra=None):
+            self.server = dask_server
             self.extra = extra or {}
 
         def get(self, port, host, proxied_path):
@@ -128,3 +128,8 @@ def check_worker_dashboard_exits(scheduler, worker):
         if addr == w.host and port == str(bokeh_port):
             return True
     return False
+
+
+routes = [
+    (r"proxy/(\d+)/(.*?)/(.*)", GlobalProxyHandler, {}),
+]
diff --git a/distributed/http/routing.py b/distributed/http/routing.py
new file mode 100644
index 00000000000..ac51086493d
--- /dev/null
+++ b/distributed/http/routing.py
@@ -0,0 +1,22 @@
+from tornado import web
+import tornado.httputil
+
+
+class RoutingApplication(web.Application):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.applications = []
+
+    def find_handler(self, request: tornado.httputil.HTTPServerRequest, **kwargs):
+        handler = super().find_handler(request, **kwargs)
+        if handler and not issubclass(handler.handler_class, web.ErrorHandler):
+            return handler
+        else:
+            for app in self.applications:
+                handler = app.find_handler(request, **kwargs) or handler
+                if handler and not issubclass(handler.handler_class, web.ErrorHandler):
+                    break
+            return handler
+
+    def add_application(self, application: web.Application):
+        self.applications.append(application)
diff --git a/distributed/http/scheduler/__init__.py b/distributed/http/scheduler/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/distributed/http/scheduler/info.py b/distributed/http/scheduler/info.py
new file mode 100644
index 00000000000..6e5a222dd23
--- /dev/null
+++ b/distributed/http/scheduler/info.py
@@ -0,0 +1,203 @@
+from datetime import datetime
+import json
+import logging
+import os
+import os.path
+
+from dask.utils import format_bytes
+
+from tornado import escape
+from tornado.websocket import WebSocketHandler
+from tlz import first, merge
+
+from ..utils import RequestHandler, redirect
+from ...diagnostics.websocket import WebsocketPlugin
+from ...metrics import time
+from ...utils import log_errors, format_time
+
+ns = {
+    func.__name__: func
+    for func in [format_bytes, format_time, datetime.fromtimestamp, time]
+}
+
+rel_path_statics = {"rel_path_statics": "../../.."}
+
+
+logger = logging.getLogger(__name__)
+
+
+class Workers(RequestHandler):
+    def get(self):
+        with log_errors():
+            self.render(
+                "workers.html",
+                title="Workers",
+                scheduler=self.server,
+                **merge(self.server.__dict__, ns, self.extra, rel_path_statics),
+            )
+
+
+class Worker(RequestHandler):
+    def get(self, worker):
+        worker = escape.url_unescape(worker)
+        if worker not in self.server.workers:
+            self.send_error(404)
+            return
+        with log_errors():
+            self.render(
+                "worker.html",
+                title="Worker: " + worker,
+                scheduler=self.server,
+                Worker=worker,
+                **merge(self.server.__dict__, ns, self.extra, rel_path_statics),
+            )
+
+
+class Task(RequestHandler):
+    def get(self, task):
+        task = escape.url_unescape(task)
+        if task not in self.server.tasks:
+            self.send_error(404)
+            return
+        with log_errors():
+            self.render(
+                "task.html",
+                title="Task: " + task,
+                Task=task,
+                scheduler=self.server,
+                **merge(self.server.__dict__, ns, self.extra, rel_path_statics),
+            )
+
+
+class Logs(RequestHandler):
+    def get(self):
+        with log_errors():
+            logs = self.server.get_logs()
+            self.render(
+                "logs.html",
+                title="Logs",
+                logs=logs,
+                **merge(self.extra, rel_path_statics),
+            )
+
+
+class WorkerLogs(RequestHandler):
+    async def get(self, worker):
+        with log_errors():
+            worker = escape.url_unescape(worker)
+            logs = await self.server.get_worker_logs(workers=[worker])
+            logs = logs[worker]
+            self.render(
+                "logs.html",
+                title="Logs: " + worker,
+                logs=logs,
+                **merge(self.extra, rel_path_statics),
+            )
+
+
+class WorkerCallStacks(RequestHandler):
+    async def get(self, worker):
+        with log_errors():
+            worker = escape.url_unescape(worker)
+            keys = self.server.processing[worker]
+            call_stack = await self.server.get_call_stack(keys=keys)
+            self.render(
+                "call-stack.html",
+                title="Call Stacks: " + worker,
+                call_stack=call_stack,
+                **merge(self.extra, rel_path_statics),
+            )
+
+
+class TaskCallStack(RequestHandler):
+    async def get(self, key):
+        with log_errors():
+            key = escape.url_unescape(key)
+            call_stack = await self.server.get_call_stack(keys=[key])
+            if not call_stack:
+                self.write(
+                    "<p>Task not actively running. "
+                    "It may be finished or not yet started</p>"
+                )
+            else:
+                self.render(
+                    "call-stack.html",
+                    title="Call Stack: " + key,
+                    call_stack=call_stack,
+                    **merge(self.extra, rel_path_statics),
+                )
+
+
+class IndividualPlots(RequestHandler):
+    def get(self):
+        try:
+            from bokeh.server.tornado import BokehTornado
+
+            bokeh_application = first(
+                app
+                for app in self.server.http_application.applications
+                if isinstance(app, BokehTornado)
+            )
+            individual_bokeh = {
+                uri.strip("/").replace("-", " ").title(): uri
+                for uri in bokeh_application.app_paths
+                if uri.lstrip("/").startswith("individual-")
+                and not uri.endswith(".json")
+            }
+            individual_static = {
+                uri.strip("/")
+                .replace(".html", "")
+                .replace("-", " ")
+                .title(): "/statics/"
+                + uri
+                for uri in os.listdir(
+                    os.path.join(os.path.dirname(__file__), "..", "static")
+                )
+                if uri.lstrip("/").startswith("individual-") and uri.endswith(".html")
+            }
+            result = {**individual_bokeh, **individual_static}
+            self.write(result)
+        except (ImportError, StopIteration):
+            self.write({})
+
+
+class EventstreamHandler(WebSocketHandler):
+    def initialize(self, dask_server=None, extra=None):
+        self.server = dask_server
+        self.extra = extra or {}
+        self.plugin = WebsocketPlugin(self, self.server)
+        self.server.add_plugin(self.plugin)
+
+    def send(self, name, data):
+        data["name"] = name
+        for k in list(data):
+            # Drop bytes objects for now
+            if isinstance(data[k], bytes):
+                del data[k]
+        self.write_message(data)
+
+    def open(self):
+        for worker in self.server.workers:
+            self.plugin.add_worker(self.server, worker)
+
+    def on_message(self, message):
+        message = json.loads(message)
+        if message["name"] == "ping":
+            self.send("pong", {"timestamp": str(datetime.now())})
+
+    def on_close(self):
+        self.server.remove_plugin(self.plugin)
+
+
+routes = [
+    (r"info", redirect("info/main/workers.html"), {}),
+    (r"info/main/workers.html", Workers, {}),
+    (r"info/worker/(.*).html", Worker, {}),
+    (r"info/task/(.*).html", Task, {}),
+    (r"info/main/logs.html", Logs, {}),
+    (r"info/call-stacks/(.*).html", WorkerCallStacks, {}),
+    (r"info/call-stack/(.*).html", TaskCallStack, {}),
+    (r"info/logs/(.*).html", WorkerLogs, {}),
+    (r"individual-plots.json", IndividualPlots, {}),
+    (r"eventstream", EventstreamHandler, {}),
+]
diff --git a/distributed/http/scheduler/json.py b/distributed/http/scheduler/json.py
new file mode 100644
index 00000000000..5dc09b4b6fe
--- /dev/null
+++ b/distributed/http/scheduler/json.py
@@ -0,0 +1,72 @@
+from ..utils import RequestHandler
+from ...utils import log_errors
+
+
+class CountsJSON(RequestHandler):
+    def get(self):
+        scheduler = self.server
+        erred = 0
+        nbytes = 0
+        nthreads = 0
+        memory = 0
+        processing = 0
+        released = 0
+        waiting = 0
+        waiting_data = 0
+        desired_workers = scheduler.adaptive_target()
+
+        for ts in scheduler.tasks.values():
+            if ts.exception_blame is not None:
+                erred += 1
+            elif ts.state == "released":
+                released += 1
+            if ts.waiting_on:
+                waiting += 1
+            if ts.waiters:
+                waiting_data += 1
+        for ws in scheduler.workers.values():
+            nthreads += ws.nthreads
+            memory += len(ws.has_what)
+            nbytes += ws.nbytes
+            processing += len(ws.processing)
+
+        response = {
+            "bytes": nbytes,
+            "clients": len(scheduler.clients),
+            "cores": nthreads,
+            "erred": erred,
+            "hosts": len(scheduler.host_info),
+            "idle": len(scheduler.idle),
+            "memory": memory,
+            "processing": processing,
+            "released": released,
+            "saturated": len(scheduler.saturated),
+            "tasks": len(scheduler.tasks),
+            "unrunnable": len(scheduler.unrunnable),
+            "waiting": waiting,
+            "waiting_data": waiting_data,
+            "workers": len(scheduler.workers),
+            "desired_workers": desired_workers,
+        }
+        self.write(response)
+
+
+class IdentityJSON(RequestHandler):
+    def get(self):
+        self.write(self.server.identity())
+
+
+class IndexJSON(RequestHandler):
+    def get(self):
+        with log_errors():
+            r = [url[5:] for url, _, _ in routes if url.endswith(".json")]
+            self.render(
+                "json-index.html", routes=r, title="Index of JSON routes", **self.extra
+            )
+
+
+routes = [
+    (r"json/counts.json", CountsJSON, {}),
+    (r"json/identity.json", IdentityJSON, {}),
+    (r"json/index.html", IndexJSON, {}),
+]
diff --git a/distributed/http/scheduler/prometheus.py b/distributed/http/scheduler/prometheus.py
new file mode 100644
index 00000000000..0f1f9c3c14f
--- /dev/null
+++ b/distributed/http/scheduler/prometheus.py
@@ -0,0 +1,99 @@
+import toolz
+
+from ..utils import RequestHandler
+from ...scheduler import ALL_TASK_STATES
+
+
+class _PrometheusCollector:
+    def __init__(self, dask_server):
+        self.server = dask_server
+
+    def collect(self):
+        from prometheus_client.core import GaugeMetricFamily, CounterMetricFamily
+
+        yield GaugeMetricFamily(
+            "dask_scheduler_clients",
+            "Number of clients connected.",
+            value=len(self.server.clients),
+        )
+
+        yield GaugeMetricFamily(
+            "dask_scheduler_desired_workers",
+            "Number of workers scheduler needs for task graph.",
+            value=self.server.adaptive_target(),
+        )
+
+        worker_states = GaugeMetricFamily(
+            "dask_scheduler_workers",
+            "Number of workers known by scheduler.",
+            labels=["state"],
+        )
+        worker_states.add_metric(["connected"], len(self.server.workers))
+        worker_states.add_metric(["saturated"], len(self.server.saturated))
+        worker_states.add_metric(["idle"], len(self.server.idle))
+        yield worker_states
+
+        tasks = GaugeMetricFamily(
+            "dask_scheduler_tasks",
+            "Number of tasks known by scheduler.",
+            labels=["state"],
+        )
+
+        task_counter = toolz.merge_with(
+            sum, (tp.states for tp in self.server.task_prefixes.values())
+        )
+
+        suspicious_tasks = CounterMetricFamily(
+            "dask_scheduler_tasks_suspicious",
+            "Total number of times a task has been marked suspicious",
+            labels=["task_prefix_name"],
+        )
+
+        for tp in self.server.task_prefixes.values():
+            suspicious_tasks.add_metric([tp.name], tp.suspicious)
+        yield suspicious_tasks
+
+        yield CounterMetricFamily(
+            "dask_scheduler_tasks_forgotten",
+            (
+                "Total number of processed tasks no longer in memory and already "
+                "removed from the scheduler job queue. Note task groups on the "
+                "scheduler which have all tasks in the forgotten state are not included."
+            ),
+            value=task_counter.get("forgotten", 0.0),
+        )
+
+        for state in ALL_TASK_STATES:
+            tasks.add_metric([state], task_counter.get(state, 0.0))
+        yield tasks
+
+
+class PrometheusHandler(RequestHandler):
+    _collector = None
+
+    def __init__(self, *args, dask_server=None, **kwargs):
+        import prometheus_client
+
+        super(PrometheusHandler, self).__init__(
+            *args, dask_server=dask_server, **kwargs
+        )
+
+        if PrometheusHandler._collector:
+            # Especially during testing, multiple schedulers are started
+            # sequentially in the same python process
+            PrometheusHandler._collector.server = self.server
+            return
+
+        PrometheusHandler._collector = _PrometheusCollector(self.server)
+        prometheus_client.REGISTRY.register(PrometheusHandler._collector)
+
+    def get(self):
+        import prometheus_client
+
+        self.write(prometheus_client.generate_latest())
+        self.set_header("Content-Type", "text/plain; version=0.0.4")
+
+
+routes = [
+    ("/metrics", PrometheusHandler, {}),
+]
diff --git a/distributed/dashboard/tests/test_scheduler_bokeh_html.py b/distributed/http/scheduler/tests/test_scheduler_http.py
similarity index 72%
rename from distributed/dashboard/tests/test_scheduler_bokeh_html.py
rename to distributed/http/scheduler/tests/test_scheduler_http.py
index de71b12a0d1..f1c3a8ed064 100644
--- a/distributed/dashboard/tests/test_scheduler_bokeh_html.py
+++ b/distributed/http/scheduler/tests/test_scheduler_http.py
@@ -6,20 +6,14 @@
 pytest.importorskip("bokeh")
 
 from tornado.escape import url_escape
-from tornado.httpclient import AsyncHTTPClient, HTTPClientError, HTTPRequest
-from tornado.websocket import websocket_connect
+from tornado.httpclient import AsyncHTTPClient, HTTPClientError
 
 from dask.sizeof import sizeof
 from distributed.utils import is_valid_xml
 from distributed.utils_test import gen_cluster, slowinc, inc
-from distributed.dashboard import BokehScheduler, BokehWorker
 
 
-@gen_cluster(
-    client=True,
-    scheduler_kwargs={"services": {("dashboard", 0): BokehScheduler}},
-    worker_kwargs={"services": {"dashboard": BokehWorker}},
-)
+@gen_cluster(client=True)
 async def test_connect(c, s, a, b):
     future = c.submit(lambda x: x + 1, 1)
     x = c.submit(slowinc, 1, delay=1, retries=5)
@@ -39,7 +33,7 @@ async def test_connect(c, s, a, b):
         "individual-plots.json",
     ]:
         response = await http_client.fetch(
-            "http://localhost:%d/%s" % (s.services["dashboard"].port, suffix)
+            "http://localhost:%d/%s" % (s.http_server.port, suffix)
         )
         assert response.code == 200
         body = response.body.decode()
@@ -50,36 +44,27 @@ async def test_connect(c, s, a, b):
             assert not re.search("href=./", body)  # no absolute links
 
 
-@gen_cluster(
-    client=True,
-    nthreads=[],
-    scheduler_kwargs={"services": {("dashboard", 0): BokehScheduler}},
-)
+@gen_cluster(client=True, nthreads=[])
 async def test_worker_404(c, s):
     http_client = AsyncHTTPClient()
     with pytest.raises(HTTPClientError) as err:
         await http_client.fetch(
-            "http://localhost:%d/info/worker/unknown" % s.services["dashboard"].port
+            "http://localhost:%d/info/worker/unknown" % s.http_server.port
         )
     assert err.value.code == 404
     with pytest.raises(HTTPClientError) as err:
         await http_client.fetch(
-            "http://localhost:%d/info/task/unknown" % s.services["dashboard"].port
+            "http://localhost:%d/info/task/unknown" % s.http_server.port
         )
     assert err.value.code == 404
 
 
-@gen_cluster(
-    client=True,
-    scheduler_kwargs={
-        "services": {("dashboard", 0): (BokehScheduler, {"prefix": "/foo"})}
-    },
-)
+@gen_cluster(client=True, scheduler_kwargs={"http_prefix": "/foo", "dashboard": True})
 async def test_prefix(c, s, a, b):
     http_client = AsyncHTTPClient()
     for suffix in ["foo/info/main/workers.html", "foo/json/index.html", "foo/system"]:
         response = await http_client.fetch(
-            "http://localhost:%d/%s" % (s.services["dashboard"].port, suffix)
+            "http://localhost:%d/%s" % (s.http_server.port, suffix)
         )
         assert response.code == 200
         body = response.body.decode()
@@ -89,11 +74,7 @@ async def test_prefix(c, s, a, b):
             assert is_valid_xml(body)
 
 
-@gen_cluster(
-    client=True,
-    clean_kwargs={"threads": False},
-    scheduler_kwargs={"services": {("dashboard", 0): BokehScheduler}},
-)
+@gen_cluster(client=True, clean_kwargs={"threads": False})
 async def test_prometheus(c, s, a, b):
     pytest.importorskip("prometheus_client")
     from prometheus_client.parser import text_string_to_metric_families
@@ -104,7 +85,7 @@ async def test_prometheus(c, s, a, b):
     # prometheus_client errors
     for _ in range(2):
         response = await http_client.fetch(
-            "http://localhost:%d/metrics" % s.services["dashboard"].port
+            "http://localhost:%d/metrics" % s.http_server.port
         )
         assert response.code == 200
         assert response.headers["Content-Type"] == "text/plain; version=0.0.4"
@@ -114,11 +95,7 @@ async def test_prometheus(c, s, a, b):
         assert "dask_scheduler_workers" in families
 
 
-@gen_cluster(
-    client=True,
-    clean_kwargs={"threads": False},
-    scheduler_kwargs={"services": {("dashboard", 0): BokehScheduler}},
-)
+@gen_cluster(client=True, clean_kwargs={"threads": False})
 async def test_prometheus_collect_task_states(c, s, a, b):
     pytest.importorskip("prometheus_client")
     from prometheus_client.parser import text_string_to_metric_families
@@ -126,11 +103,8 @@ async def test_prometheus_collect_task_states(c, s, a, b):
     http_client = AsyncHTTPClient()
 
     async def fetch_metrics():
-        bokeh_scheduler = s.services["dashboard"]
-        assert s.services["dashboard"].scheduler is s
-        response = await http_client.fetch(
-            f"http://{bokeh_scheduler.server.address}:{bokeh_scheduler.port}/metrics"
-        )
+        port = s.http_server.port
+        response = await http_client.fetch(f"http://localhost:{port}/metrics")
         txt = response.body.decode("utf8")
         families = {
             family.name: family for family in text_string_to_metric_families(txt)
@@ -174,16 +148,12 @@ async def fetch_metrics():
     assert sum(forgotten_tasks) == 0.0
 
 
-@gen_cluster(
-    client=True,
-    clean_kwargs={"threads": False},
-    scheduler_kwargs={"services": {("dashboard", 0): BokehScheduler}},
-)
+@gen_cluster(client=True, clean_kwargs={"threads": False})
 async def test_health(c, s, a, b):
     http_client = AsyncHTTPClient()
 
     response = await http_client.fetch(
-        "http://localhost:%d/health" % s.services["dashboard"].port
+        "http://localhost:%d/health" % s.http_server.port
     )
     assert response.code == 200
     assert response.headers["Content-Type"] == "text/plain"
@@ -192,9 +162,7 @@ async def test_health(c, s, a, b):
     assert txt == "ok"
 
 
-@gen_cluster(
-    client=True, scheduler_kwargs={"services": {("dashboard", 0): BokehScheduler}}
-)
+@gen_cluster(client=True)
 async def test_task_page(c, s, a, b):
     future = c.submit(lambda x: x + 1, 1, workers=a.address)
     x = c.submit(inc, 1)
@@ -203,7 +171,7 @@ async def test_task_page(c, s, a, b):
 
     "info/task/" + url_escape(future.key) + ".html",
     response = await http_client.fetch(
-        "http://localhost:%d/info/task/" % s.services["dashboard"].port
+        "http://localhost:%d/info/task/" % s.http_server.port
         + url_escape(future.key)
         + ".html"
     )
@@ -218,22 +186,34 @@ async def test_task_page(c, s, a, b):
 
 @gen_cluster(
     client=True,
-    scheduler_kwargs={
-        "services": {
-            ("dashboard", 0): (
-                BokehScheduler,
-                {"allow_websocket_origin": ["good.invalid"]},
-            )
-        }
+    scheduler_kwargs={"dashboard": True},
+    config={
+        "distributed.scheduler.dashboard.bokeh-application.allow_websocket_origin": [
+            "good.invalid"
+        ]
     },
 )
 async def test_allow_websocket_origin(c, s, a, b):
+    from tornado.httpclient import HTTPRequest
+    from tornado.websocket import websocket_connect
+
     url = (
         "ws://localhost:%d/status/ws?bokeh-protocol-version=1.0&bokeh-session-id=1"
-        % s.services["dashboard"].port
+        % s.http_server.port
     )
     with pytest.raises(HTTPClientError) as err:
         await websocket_connect(
             HTTPRequest(url, headers={"Origin": "http://evil.invalid"})
         )
     assert err.value.code == 403
+
+
+@gen_cluster(client=True)
+async def test_eventstream(c, s, a, b):
+    from tornado.websocket import websocket_connect
+
+    ws_client = await websocket_connect(
+        "ws://localhost:%d/%s" % (s.http_server.port, "eventstream")
+    )
+    assert "websocket" in str(s.plugins).lower()
+    ws_client.close()
diff --git a/distributed/dashboard/static/css/base.css b/distributed/http/static/css/base.css
similarity index 100%
rename from distributed/dashboard/static/css/base.css
rename to distributed/http/static/css/base.css
diff --git a/distributed/dashboard/static/css/individual-cluster-map.css b/distributed/http/static/css/individual-cluster-map.css
similarity index 100%
rename from distributed/dashboard/static/css/individual-cluster-map.css
rename to distributed/http/static/css/individual-cluster-map.css
diff --git a/distributed/dashboard/static/css/status.css b/distributed/http/static/css/status.css
similarity index 100%
rename from distributed/dashboard/static/css/status.css
rename to distributed/http/static/css/status.css
diff --git a/distributed/dashboard/static/css/system.css b/distributed/http/static/css/system.css
similarity index 100%
rename from distributed/dashboard/static/css/system.css
rename to distributed/http/static/css/system.css
diff --git a/distributed/dashboard/static/images/dask-logo.svg b/distributed/http/static/images/dask-logo.svg
similarity index 100%
rename from distributed/dashboard/static/images/dask-logo.svg
rename to distributed/http/static/images/dask-logo.svg
diff --git a/distributed/dashboard/static/images/fa-bars.svg b/distributed/http/static/images/fa-bars.svg
similarity index 100%
rename from distributed/dashboard/static/images/fa-bars.svg
rename to distributed/http/static/images/fa-bars.svg
diff --git a/distributed/dashboard/static/images/favicon.ico b/distributed/http/static/images/favicon.ico
similarity index 100%
rename from distributed/dashboard/static/images/favicon.ico
rename to distributed/http/static/images/favicon.ico
diff --git a/distributed/dashboard/static/individual-cluster-map.html b/distributed/http/static/individual-cluster-map.html
similarity index 100%
rename from distributed/dashboard/static/individual-cluster-map.html
rename to distributed/http/static/individual-cluster-map.html
diff --git a/distributed/dashboard/static/js/anime.min.js b/distributed/http/static/js/anime.min.js
similarity index 100%
rename from distributed/dashboard/static/js/anime.min.js
rename to distributed/http/static/js/anime.min.js
diff --git a/distributed/dashboard/static/js/individual-cluster-map.js b/distributed/http/static/js/individual-cluster-map.js
similarity index 100%
rename from distributed/dashboard/static/js/individual-cluster-map.js
rename to distributed/http/static/js/individual-cluster-map.js
diff --git a/distributed/dashboard/static/js/reconnecting-websocket.min.js b/distributed/http/static/js/reconnecting-websocket.min.js
similarity index 100%
rename from distributed/dashboard/static/js/reconnecting-websocket.min.js
rename to distributed/http/static/js/reconnecting-websocket.min.js
diff --git a/distributed/http/statics.py b/distributed/http/statics.py
new file mode 100644
index 00000000000..4a8a60298fe
--- /dev/null
+++ b/distributed/http/statics.py
@@ -0,0 +1,10 @@
+from tornado import web
+import os
+
+routes = [
+    (
+        r"/statics/(.*)",
+        web.StaticFileHandler,
+        {"path": os.path.join(os.path.dirname(__file__), "static")},
+    ),
+]
diff --git a/distributed/dashboard/templates/base.html b/distributed/http/templates/base.html
similarity index 100%
rename from distributed/dashboard/templates/base.html
rename to distributed/http/templates/base.html
diff --git a/distributed/dashboard/templates/call-stack.html b/distributed/http/templates/call-stack.html
similarity index 100%
rename from distributed/dashboard/templates/call-stack.html
rename to distributed/http/templates/call-stack.html
diff --git a/distributed/dashboard/templates/json-index.html b/distributed/http/templates/json-index.html
similarity index 100%
rename from distributed/dashboard/templates/json-index.html
rename to distributed/http/templates/json-index.html
diff --git a/distributed/dashboard/templates/logs.html b/distributed/http/templates/logs.html
similarity index 100%
rename from distributed/dashboard/templates/logs.html
rename to distributed/http/templates/logs.html
diff --git a/distributed/dashboard/templates/main.html b/distributed/http/templates/main.html
similarity index 100%
rename from distributed/dashboard/templates/main.html
rename to distributed/http/templates/main.html
diff --git a/distributed/dashboard/templates/simple.html b/distributed/http/templates/simple.html
similarity index 100%
rename from distributed/dashboard/templates/simple.html
rename to distributed/http/templates/simple.html
diff --git a/distributed/dashboard/templates/status.html b/distributed/http/templates/status.html
similarity index 100%
rename from distributed/dashboard/templates/status.html
rename to distributed/http/templates/status.html
diff --git a/distributed/dashboard/templates/system.html b/distributed/http/templates/system.html
similarity index 100%
rename from distributed/dashboard/templates/system.html
rename to distributed/http/templates/system.html
diff --git a/distributed/dashboard/templates/task.html b/distributed/http/templates/task.html
similarity index 100%
rename from distributed/dashboard/templates/task.html
rename to distributed/http/templates/task.html
diff --git a/distributed/dashboard/templates/worker-table.html b/distributed/http/templates/worker-table.html
similarity index 100%
rename from distributed/dashboard/templates/worker-table.html
rename to distributed/http/templates/worker-table.html
diff --git a/distributed/dashboard/templates/worker.html b/distributed/http/templates/worker.html
similarity index 100%
rename from distributed/dashboard/templates/worker.html
rename to distributed/http/templates/worker.html
diff --git a/distributed/dashboard/templates/workers.html b/distributed/http/templates/workers.html
similarity index 100%
rename from distributed/dashboard/templates/workers.html
rename to distributed/http/templates/workers.html
diff --git a/distributed/http/tests/__init__.py b/distributed/http/tests/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/distributed/http/tests/test_core.py b/distributed/http/tests/test_core.py
new file mode 100644
index 00000000000..c1bffedb72e
--- /dev/null
+++ b/distributed/http/tests/test_core.py
@@ -0,0 +1,11 @@
+from distributed.utils_test import gen_cluster
+from tornado.httpclient import AsyncHTTPClient
+
+
+@gen_cluster(client=True)
+async def test_scheduler(c, s, a, b):
+    client = AsyncHTTPClient()
+    response = await client.fetch(
+        "http://localhost:{}/health".format(s.http_server.port)
+    )
+    assert response.code == 200
diff --git a/distributed/http/tests/test_routing.py b/distributed/http/tests/test_routing.py
new file mode 100644
index 00000000000..481cfb3a209
--- /dev/null
+++ b/distributed/http/tests/test_routing.py
@@ -0,0 +1,38 @@
+from tornado import web
+from tornado.httpclient import AsyncHTTPClient, HTTPClientError
+import pytest
+
+from distributed.http.routing import RoutingApplication
+
+
+class OneHandler(web.RequestHandler):
+    def get(self):
+        self.write("one")
+
+
+class TwoHandler(web.RequestHandler):
+    def get(self):
+        self.write("two")
+
+
+@pytest.mark.asyncio
+async def test_basic():
+    application = RoutingApplication([(r"/one", OneHandler),])
+    two = web.Application([(r"/two", TwoHandler),])
+    server = application.listen(1234)
+
+    client = AsyncHTTPClient("http://localhost:1234")
+    response = await client.fetch("http://localhost:1234/one")
+    assert response.body.decode() == "one"
+
+    with pytest.raises(HTTPClientError):
+        response = await client.fetch("http://localhost:1234/two")
+
+    application.applications.append(two)
+
+    response = await client.fetch("http://localhost:1234/two")
+    assert response.body.decode() == "two"
+
+    application.add_handlers(".*", [(r"/three", OneHandler, {})])
+    response = await client.fetch("http://localhost:1234/three")
+    assert response.body.decode() == "one"
diff --git a/distributed/http/utils.py b/distributed/http/utils.py
new file mode 100644
index 00000000000..5977ccd5bad
--- /dev/null
+++ b/distributed/http/utils.py
@@ -0,0 +1,51 @@
+import importlib
+import os
+from typing import List
+
+from tornado import web
+import toolz
+
+from ..utils import has_keyword
+
+
+dirname = os.path.dirname(__file__)
+
+
+class RequestHandler(web.RequestHandler):
+    def initialize(self, dask_server=None, extra=None):
+        self.server = dask_server
+        self.extra = extra or {}
+
+    def get_template_path(self):
+        return os.path.join(dirname, "templates")
+
+
+def redirect(path):
+    class Redirect(RequestHandler):
+        def get(self):
+            self.redirect(path)
+
+    return Redirect
+
+
+def get_handlers(server, modules: List[str], prefix="/"):
+    prefix = prefix or ""
+    prefix = "/" + prefix.strip("/")
+
+    if not prefix.endswith("/"):
+        prefix = prefix + "/"
+
+    _routes = []
+    for module_name in modules:
+        module = importlib.import_module(module_name)
+        _routes.extend(module.routes)
+
+    routes = []
+
+    for url, cls, kwargs in _routes:
+        if has_keyword(cls.initialize, "dask_server"):
+            kwargs = toolz.assoc(kwargs, "dask_server", server)
+
+        routes.append((prefix + url.lstrip("/"), cls, kwargs))
+
+    return routes
diff --git a/distributed/http/worker/__init__.py b/distributed/http/worker/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/distributed/http/worker/prometheus.py b/distributed/http/worker/prometheus.py
new file mode 100644
index 00000000000..a60de3a6b64
--- /dev/null
+++ b/distributed/http/worker/prometheus.py
@@ -0,0 +1,98 @@
+from ..utils import RequestHandler
+
+import logging
+
+
+class _PrometheusCollector:
+    def __init__(self, server):
+        self.worker = server
+        self.logger = logging.getLogger("distributed.dask_worker")
+        self.crick_available = True
+        try:
+            import crick  # noqa: F401
+        except ImportError:
+            self.crick_available = False
+            self.logger.info(
+                "Not all prometheus metrics available are exported. Digest-based metrics require crick to be installed"
+            )
+
+    def collect(self):
+        from prometheus_client.core import GaugeMetricFamily
+
+        tasks = GaugeMetricFamily(
+            "dask_worker_tasks", "Number of tasks at worker.", labels=["state"]
+        )
+        tasks.add_metric(["stored"], len(self.worker.data))
+        tasks.add_metric(["executing"], len(self.worker.executing))
+        tasks.add_metric(["ready"], len(self.worker.ready))
+        tasks.add_metric(["waiting"], len(self.worker.waiting_for_data))
+        tasks.add_metric(["serving"], len(self.worker._comms))
+        yield tasks
+
+        yield GaugeMetricFamily(
+            "dask_worker_connections",
+            "Number of task connections to other workers.",
+            value=len(self.worker.in_flight_workers),
+        )
+
+        yield GaugeMetricFamily(
+            "dask_worker_threads",
+            "Number of worker threads.",
+            value=self.worker.nthreads,
+        )
+
+        yield GaugeMetricFamily(
+            "dask_worker_latency_seconds",
+            "Latency of worker connection.",
+            value=self.worker.latency,
+        )
+
+        # all metrics using digests require crick to be installed
+        # the following metrics will export NaN, if the corresponding digests are None
+        if self.crick_available:
+            yield GaugeMetricFamily(
+                "dask_worker_tick_duration_median_seconds",
+                "Median tick duration at worker.",
+                value=self.worker.digests["tick-duration"].components[1].quantile(50),
+            )
+
+            yield GaugeMetricFamily(
+                "dask_worker_task_duration_median_seconds",
+                "Median task runtime at worker.",
+                value=self.worker.digests["task-duration"].components[1].quantile(50),
+            )
+
+            yield GaugeMetricFamily(
+                "dask_worker_transfer_bandwidth_median_bytes",
+                "Bandwidth for transfer at worker in Bytes.",
+                value=self.worker.digests["transfer-bandwidth"]
+                .components[1]
+                .quantile(50),
+            )
+
+
+class PrometheusHandler(RequestHandler):
+    _initialized = False
+
+    def __init__(self, *args, **kwargs):
+        import prometheus_client
+
+        super(PrometheusHandler, self).__init__(*args, **kwargs)
+
+        if PrometheusHandler._initialized:
+            return
+
+        prometheus_client.REGISTRY.register(_PrometheusCollector(self.server))
+
+        PrometheusHandler._initialized = True
+
+    def get(self):
+        import prometheus_client
+
+        self.write(prometheus_client.generate_latest())
+        self.set_header("Content-Type", "text/plain; version=0.0.4")
+
+
+routes = [
+    (r"metrics", PrometheusHandler, {}),
+]
diff --git a/distributed/dashboard/tests/test_worker_bokeh_html.py b/distributed/http/worker/tests/test_worker_http.py
similarity index 73%
rename from distributed/dashboard/tests/test_worker_bokeh_html.py
rename to distributed/http/worker/tests/test_worker_http.py
index 7a4d70a037c..0a4135fba7f 100644
--- a/distributed/dashboard/tests/test_worker_bokeh_html.py
+++ b/distributed/http/worker/tests/test_worker_http.py
@@ -1,13 +1,10 @@
 import pytest
 
-pytest.importorskip("bokeh")
-
 from tornado.httpclient import AsyncHTTPClient
 from distributed.utils_test import gen_cluster
-from distributed.dashboard import BokehWorker
 
 
-@gen_cluster(client=True, worker_kwargs={"services": {("dashboard", 0): BokehWorker}})
+@gen_cluster(client=True)
 def test_prometheus(c, s, a, b):
     pytest.importorskip("prometheus_client")
     from prometheus_client.parser import text_string_to_metric_families
@@ -18,7 +15,7 @@ def test_prometheus(c, s, a, b):
     # prometheus_client errors
     for _ in range(2):
         response = yield http_client.fetch(
-            "http://localhost:%d/metrics" % a.services["dashboard"].port
+            "http://localhost:%d/metrics" % a.http_server.port
         )
         assert response.code == 200
         assert response.headers["Content-Type"] == "text/plain; version=0.0.4"
@@ -28,12 +25,12 @@ def test_prometheus(c, s, a, b):
         assert "dask_worker_latency_seconds" in families
 
 
-@gen_cluster(client=True, worker_kwargs={"services": {("dashboard", 0): BokehWorker}})
+@gen_cluster(client=True)
 def test_health(c, s, a, b):
     http_client = AsyncHTTPClient()
 
     response = yield http_client.fetch(
-        "http://localhost:%d/health" % a.services["dashboard"].port
+        "http://localhost:%d/health" % a.http_server.port
     )
     assert response.code == 200
     assert response.headers["Content-Type"] == "text/plain"
diff --git a/distributed/node.py b/distributed/node.py
index af15b5a409f..11645e86317 100644
--- a/distributed/node.py
+++ b/distributed/node.py
@@ -4,12 +4,17 @@
 import weakref
 
 from tornado.ioloop import IOLoop
+from tornado.httpserver import HTTPServer
 from tornado import gen
+import tlz
 import dask
 
+from .comm import get_tcp_server_address
+from .comm import get_address_host
 from .core import Server, ConnectionPool
+from .http.routing import RoutingApplication
 from .versions import get_versions
-from .utils import DequeHandler, TimeoutError
+from .utils import DequeHandler, TimeoutError, clean_dashboard_address, ignoring
 
 
 class Node:
@@ -189,3 +194,53 @@ async def start(self):
         # subclasses should implement their own start method whichs calls super().start()
         await Node.start(self)
         return self
+
+    def start_http_server(
+        self, routes, dashboard_address, default_port=0, ssl_options=None,
+    ):
+        """ This creates an HTTP Server running on this node """
+
+        self.http_application = RoutingApplication(routes,)
+
+        # TLS configuration
+        tls_key = dask.config.get("distributed.scheduler.dashboard.tls.key")
+        tls_cert = dask.config.get("distributed.scheduler.dashboard.tls.cert")
+        tls_ca_file = dask.config.get("distributed.scheduler.dashboard.tls.ca-file")
+        if tls_cert:
+            import ssl
+
+            ssl_options = ssl.create_default_context(
+                cafile=tls_ca_file, purpose=ssl.Purpose.SERVER_AUTH
+            )
+            ssl_options.load_cert_chain(tls_cert, keyfile=tls_key)
+            # We don't care about auth here, just encryption
+            ssl_options.check_hostname = False
+            ssl_options.verify_mode = ssl.CERT_NONE
+
+        self.http_server = HTTPServer(self.http_application, ssl_options=ssl_options)
+        http_address = clean_dashboard_address(dashboard_address or default_port)
+
+        if not http_address["address"]:
+            address = self._start_address
+            if isinstance(address, (list, tuple)):
+                address = address[0]
+            if address:
+                with ignoring(ValueError):
+                    http_address["address"] = get_address_host(address)
+        changed_port = False
+        try:
+            self.http_server.listen(**http_address)
+        except Exception:
+            changed_port = True
+            self.http_server.listen(**tlz.merge(http_address, {"port": 0}))
+        self.http_server.port = get_tcp_server_address(self.http_server)[1]
+        self.services["dashboard"] = self.http_server
+
+        if changed_port and dashboard_address:
+            warnings.warn(
+                "Port {} is already in use.\n"
+                "Perhaps you already have a cluster running?\n"
+                "Hosting the HTTP server on port {} instead".format(
+                    http_address["port"], self.http_server.port
+                )
+            )
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 55692b875ab..86dd6b9203e 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -36,6 +36,7 @@
 
 import dask
 
+from . import profile
 from .batched import BatchedSend
 from .comm import (
     normalize_address,
@@ -46,7 +47,8 @@
 from .comm.addressing import addresses_from_user_args
 from .core import rpc, connect, send_recv, clean_exception, CommClosedError
 from .diagnostics.plugin import SchedulerPlugin
-from . import profile
+
+from .http import get_handlers
 from .metrics import time
 from .node import ServerNode
 from . import preloading
@@ -1060,6 +1062,8 @@ def __init__(
         port=0,
         protocol=None,
         dashboard_address=None,
+        dashboard=None,
+        http_prefix="/",
         preload=None,
         preload_argv=(),
         plugins=(),
@@ -1112,15 +1116,30 @@ def __init__(
         assert isinstance(self.security, Security)
         self.connection_args = self.security.get_connection_args("scheduler")
 
-        if dashboard_address is not None:
+        self._start_address = addresses_from_user_args(
+            host=host,
+            port=port,
+            interface=interface,
+            protocol=protocol,
+            security=security,
+            default_port=self.default_port,
+        )
+
+        routes = get_handlers(
+            server=self,
+            modules=dask.config.get("distributed.scheduler.http.routes"),
+            prefix=http_prefix,
+        )
+        self.start_http_server(routes, dashboard_address, default_port=8787)
+
+        if dashboard:
             try:
-                from distributed.dashboard import BokehScheduler
+                import distributed.dashboard.scheduler
             except ImportError:
                 logger.debug("To start diagnostics web server please install Bokeh")
             else:
-                self.service_specs[("dashboard", dashboard_address)] = (
-                    BokehScheduler,
-                    (service_kwargs or {}).get("dashboard", {}),
+                distributed.dashboard.scheduler.connect(
+                    self.http_application, self.http_server, self, prefix=http_prefix,
                 )
 
         # Communication state
@@ -1327,15 +1346,6 @@ def __init__(
 
         connection_limit = get_fileno_limit() / 2
 
-        self._start_address = addresses_from_user_args(
-            host=host,
-            port=port,
-            interface=interface,
-            protocol=protocol,
-            security=security,
-            default_port=self.default_port,
-        )
-
         super(Scheduler, self).__init__(
             handlers=self.handlers,
             stream_handlers=merge(worker_handlers, client_handlers),
@@ -1506,6 +1516,7 @@ async def close(self, comm=None, fast=False, close_workers=False):
         self.periodic_callbacks.clear()
 
         self.stop_services()
+
         for ext in self.extensions.values():
             with ignoring(AttributeError):
                 ext.teardown()
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 68507d889f0..297394631ae 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -5247,14 +5247,9 @@ def test_quiet_scheduler_loss(c, s):
 
 
 def test_dashboard_link(loop, monkeypatch):
-    pytest.importorskip("bokeh")
-    from distributed.dashboard import BokehScheduler
-
     monkeypatch.setenv("USER", "myusername")
 
-    with cluster(
-        scheduler_kwargs={"services": {("dashboard", 12355): BokehScheduler}}
-    ) as (s, [a, b]):
+    with cluster(scheduler_kwargs={"dashboard_address": ":12355"}) as (s, [a, b]):
         with Client(s["address"], loop=loop) as c:
             with dask.config.set(
                 {"distributed.dashboard.link": "{scheme}://foo-{USER}:{port}/status"}
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index a0e1b11de2b..2b48fa030e4 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -1190,19 +1190,14 @@ def test_correct_bad_time_estimate(c, s, *workers):
 
 @gen_test()
 async def test_service_hosts():
-    pytest.importorskip("bokeh")
-    from distributed.dashboard import BokehScheduler
-
     port = 0
     for url, expected in [
         ("tcp://0.0.0.0", ("::", "0.0.0.0")),
         ("tcp://127.0.0.1", "127.0.0.1"),
         ("tcp://127.0.0.1:38275", "127.0.0.1"),
     ]:
-        services = {("dashboard", port): BokehScheduler}
-
-        async with Scheduler(host=url, services=services) as s:
-            sock = first(s.services["dashboard"].server._http._sockets.values())
+        async with Scheduler(host=url) as s:
+            sock = first(s.http_server._sockets.values())
             if isinstance(expected, tuple):
                 assert sock.getsockname()[0] in expected
             else:
@@ -1210,10 +1205,8 @@ async def test_service_hosts():
 
     port = ("127.0.0.1", 0)
     for url in ["tcp://0.0.0.0", "tcp://127.0.0.1", "tcp://127.0.0.1:38275"]:
-        services = {("dashboard", port): BokehScheduler}
-
-        async with Scheduler(services=services, host=url) as s:
-            sock = first(s.services["dashboard"].server._http._sockets.values())
+        async with Scheduler(dashboard_address="127.0.0.1:0", host=url) as s:
+            sock = first(s.http_server._sockets.values())
             assert sock.getsockname()[0] == "127.0.0.1"
 
 
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 0bda344fd96..a5e364ec0cd 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -5,7 +5,6 @@
 from operator import add
 import os
 import psutil
-import shutil
 import sys
 from time import sleep
 import traceback
@@ -58,12 +57,11 @@
 )
 
 
-def test_worker_nthreads():
-    w = Worker("127.0.0.1", 8019)
-    try:
-        assert w.executor._max_workers == CPU_COUNT
-    finally:
-        shutil.rmtree(w.local_directory)
+@pytest.mark.asyncio
+async def test_worker_nthreads(cleanup):
+    async with Scheduler() as s:
+        async with Worker(s.address) as w:
+            assert w.executor._max_workers == CPU_COUNT
 
 
 @gen_cluster()
@@ -75,13 +73,15 @@ def test_str(s, a, b):
     assert str(len(a.executing)) in repr(a)
 
 
-def test_identity():
-    w = Worker("127.0.0.1", 8019)
-    ident = w.identity(None)
-    assert "Worker" in ident["type"]
-    assert ident["scheduler"] == "tcp://127.0.0.1:8019"
-    assert isinstance(ident["nthreads"], int)
-    assert isinstance(ident["memory_limit"], Number)
+@pytest.mark.asyncio
+async def test_identity(cleanup):
+    async with Scheduler() as s:
+        async with Worker(s.address) as w:
+            ident = w.identity(None)
+            assert "Worker" in ident["type"]
+            assert ident["scheduler"] == s.address
+            assert isinstance(ident["nthreads"], int)
+            assert isinstance(ident["memory_limit"], Number)
 
 
 @gen_cluster(client=True)
@@ -320,20 +320,17 @@ def test_worker_with_port_zero():
 
 
 @pytest.mark.slow
-def test_worker_waits_for_scheduler(loop):
-    @gen.coroutine
-    def f():
-        w = Worker("127.0.0.1", 8007)
-        try:
-            yield asyncio.wait_for(w, 3)
-        except TimeoutError:
-            pass
-        else:
-            assert False
-        assert w.status not in ("closed", "running")
-        yield w.close(timeout=0.1)
-
-    loop.run_sync(f)
+@pytest.mark.asyncio
+async def test_worker_waits_for_scheduler(cleanup):
+    w = Worker("127.0.0.1:8724")
+    try:
+        await asyncio.wait_for(w, 3)
+    except TimeoutError:
+        pass
+    else:
+        assert False
+    assert w.status not in ("closed", "running")
+    await w.close(timeout=0.1)
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)])
@@ -532,19 +529,21 @@ def test_close_on_disconnect(s, w):
         assert time() < start + 5
 
 
-def test_memory_limit_auto():
-    a = Worker("127.0.0.1", 8099, nthreads=1)
-    b = Worker("127.0.0.1", 8099, nthreads=2)
-    c = Worker("127.0.0.1", 8099, nthreads=100)
-    d = Worker("127.0.0.1", 8099, nthreads=200)
-
-    assert isinstance(a.memory_limit, Number)
-    assert isinstance(b.memory_limit, Number)
+@pytest.mark.asyncio
+async def test_memory_limit_auto():
+    async with Scheduler() as s:
+        async with Worker(s.address, nthreads=1) as a, Worker(
+            s.address, nthreads=2
+        ) as b, Worker(s.address, nthreads=100) as c, Worker(
+            s.address, nthreads=200
+        ) as d:
+            assert isinstance(a.memory_limit, Number)
+            assert isinstance(b.memory_limit, Number)
 
-    if CPU_COUNT > 1:
-        assert a.memory_limit < b.memory_limit
+            if CPU_COUNT > 1:
+                assert a.memory_limit < b.memory_limit
 
-    assert c.memory_limit == d.memory_limit
+            assert c.memory_limit == d.memory_limit
 
 
 @gen_cluster(client=True)
@@ -782,13 +781,13 @@ def test_hold_onto_dependents(c, s, a, b):
 
 @pytest.mark.slow
 @gen_cluster(client=False, nthreads=[])
-def test_worker_death_timeout(s):
+async def test_worker_death_timeout(s):
     with dask.config.set({"distributed.comm.timeouts.connect": "1s"}):
-        yield s.close()
+        await s.close()
         w = Worker(s.address, death_timeout=1)
 
     with pytest.raises(TimeoutError) as info:
-        yield w
+        await w
 
     assert "Worker" in str(info.value)
     assert "timed out" in str(info.value) or "failed to start" in str(info.value)
@@ -1024,39 +1023,25 @@ def test_worker_fds(s):
 
 @gen_cluster(nthreads=[])
 async def test_service_hosts_match_worker(s):
-    pytest.importorskip("bokeh")
-    from distributed.dashboard import BokehWorker
-
-    async with Worker(
-        s.address, services={("dashboard", ":0"): BokehWorker}, host="tcp://0.0.0.0"
-    ) as w:
-        sock = first(w.services["dashboard"].server._http._sockets.values())
+    async with Worker(s.address, host="tcp://0.0.0.0") as w:
+        sock = first(w.http_server._sockets.values())
         assert sock.getsockname()[0] in ("::", "0.0.0.0")
 
     async with Worker(
-        s.address, services={("dashboard", ":0"): BokehWorker}, host="tcp://127.0.0.1"
+        s.address, host="tcp://127.0.0.1", dashboard_address="0.0.0.0:0"
     ) as w:
-        sock = first(w.services["dashboard"].server._http._sockets.values())
+        sock = first(w.http_server._sockets.values())
         assert sock.getsockname()[0] in ("::", "0.0.0.0")
 
-    async with Worker(
-        s.address, services={("dashboard", 0): BokehWorker}, host="tcp://127.0.0.1"
-    ) as w:
-        sock = first(w.services["dashboard"].server._http._sockets.values())
+    async with Worker(s.address, host="tcp://127.0.0.1") as w:
+        sock = first(w.http_server._sockets.values())
         assert sock.getsockname()[0] == "127.0.0.1"
 
 
 @gen_cluster(nthreads=[])
-def test_start_services(s):
-    pytest.importorskip("bokeh")
-    from distributed.dashboard import BokehWorker
-
-    services = {("dashboard", ":1234"): BokehWorker}
-
-    w = yield Worker(s.address, services=services)
-
-    assert w.services["dashboard"].server.port == 1234
-    yield w.close()
+async def test_start_services(s):
+    async with Worker(s.address, dashboard_address=1234) as w:
+        assert w.http_server.port == 1234
 
 
 @gen_test()
@@ -1234,16 +1219,18 @@ def f(x):
     assert all(f.key in b.data for f in futures)
 
 
-def test_deque_handler():
+@pytest.mark.asyncio
+async def test_deque_handler(cleanup):
     from distributed.worker import logger
 
-    w = Worker("127.0.0.1", 8019)
-    deque_handler = w._deque_handler
-    logger.info("foo456")
-    assert deque_handler.deque
-    msg = deque_handler.deque[-1]
-    assert "distributed.worker" in deque_handler.format(msg)
-    assert any(msg.msg == "foo456" for msg in deque_handler.deque)
+    async with Scheduler() as s:
+        async with Worker(s.address) as w:
+            deque_handler = w._deque_handler
+            logger.info("foo456")
+            assert deque_handler.deque
+            msg = deque_handler.deque[-1]
+            assert "distributed.worker" in deque_handler.format(msg)
+            assert any(msg.msg == "foo456" for msg in deque_handler.deque)
 
 
 @gen_cluster(nthreads=[], client=True)
diff --git a/distributed/utils.py b/distributed/utils.py
index eb622f7b837..adc20d4f368 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -1523,3 +1523,45 @@ def __setitem__(self, key, value):
         if len(self) >= self.maxsize:
             self.data.popitem(last=False)
         super().__setitem__(key, value)
+
+
+def clean_dashboard_address(addr, default_listen_ip=""):
+    """
+
+    Examples
+    --------
+    >>> clean_dashboard_address(8787)
+    {'address': '', 'port': 8787}
+    >>> clean_dashboard_address(":8787")
+    {'address': '', 'port': 8787}
+    >>> clean_dashboard_address("8787")
+    {'address': '', 'port': 8787}
+    >>> clean_dashboard_address("8787")
+    {'address': '', 'port': 8787}
+    >>> clean_dashboard_address("foo:8787")
+    {'address': 'foo', 'port': 8787}
+    """
+
+    if default_listen_ip == "0.0.0.0":
+        default_listen_ip = ""  # for IPV6
+
+    try:
+        addr = int(addr)
+    except (TypeError, ValueError):
+        pass
+
+    if isinstance(addr, str):
+        addr = addr.split(":")
+
+    if isinstance(addr, (tuple, list)):
+        if len(addr) == 2:
+            host, port = (addr[0], int(addr[1]))
+        elif len(addr) == 1:
+            [host], port = addr, 0
+        else:
+            raise ValueError(addr)
+    elif isinstance(addr, int):
+        host = default_listen_ip
+        port = addr
+
+    return {"address": host, "port": port}
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index 983eaac48f5..e1db066b732 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -1513,7 +1513,9 @@ def check_instances():
     }
 
     # assert not list(SpecCluster._instances)  # TODO
-    assert all(c.status == "closed" for c in SpecCluster._instances)
+    assert all(c.status == "closed" for c in SpecCluster._instances), list(
+        SpecCluster._instances
+    )
     SpecCluster._instances.clear()
 
     Nanny._instances.clear()
diff --git a/distributed/worker.py b/distributed/worker.py
index f2f832f6d83..a50103bacab 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -32,6 +32,7 @@
 from .comm.addressing import address_from_user_args
 from .core import error_message, CommClosedError, send_recv, pingpong, coerce_to_address
 from .diskutils import WorkSpace
+from .http import get_handlers
 from .metrics import time
 from .node import ServerNode
 from . import preloading
@@ -318,6 +319,8 @@ def __init__(
         port=None,
         protocol=None,
         dashboard_address=None,
+        dashboard=False,
+        http_prefix="/",
         nanny=None,
         plugins=(),
         low_level_profiler=dask.config.get("distributed.worker.profile.low-level"),
@@ -587,15 +590,21 @@ def __init__(
         self.services = {}
         self.service_specs = services or {}
 
-        if dashboard_address is not None:
+        routes = get_handlers(
+            server=self,
+            modules=dask.config.get("distributed.worker.http.routes"),
+            prefix=http_prefix,
+        )
+        self.start_http_server(routes, dashboard_address)
+
+        if dashboard:
             try:
-                from distributed.dashboard import BokehWorker
+                import distributed.dashboard.worker
             except ImportError:
                 logger.debug("To start diagnostics web server please install Bokeh")
             else:
-                self.service_specs[("dashboard", dashboard_address)] = (
-                    BokehWorker,
-                    (service_kwargs or {}).get("dashboard", {}),
+                distributed.dashboard.worker.connect(
+                    self.http_application, self.http_server, self, prefix=http_prefix,
                 )
 
         self.metrics = dict(metrics) if metrics else {}
@@ -1116,8 +1125,7 @@ async def close(
             await self.scheduler.close_rpc()
             self._workdir.release()
 
-            for k, v in self.services.items():
-                v.stop()
+            self.stop_services()
 
             if (
                 self.batched_stream
diff --git a/setup.py b/setup.py
index 155ae0c0274..6b70d0638ce 100755
--- a/setup.py
+++ b/setup.py
@@ -30,7 +30,7 @@
     license="BSD",
     package_data={
         "": ["templates/index.html", "template.html"],
-        "distributed": ["dashboard/templates/*.html"],
+        "distributed": ["http/templates/*.html"],
     },
     include_package_data=True,
     install_requires=install_requires,

From 74f10aaef2827194c58514451bea9dcd55167ae1 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Thu, 9 Apr 2020 19:04:16 -0500
Subject: [PATCH 0782/1550] Add Client.wait_to_workers to Client autosummary
 table (#3692)

---
 docs/source/api.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/source/api.rst b/docs/source/api.rst
index da9a76eed9b..c036fdf96db 100644
--- a/docs/source/api.rst
+++ b/docs/source/api.rst
@@ -43,6 +43,7 @@ API
    Client.submit
    Client.unpublish_dataset
    Client.upload_file
+   Client.wait_for_workers
    Client.who_has
 
 .. currentmodule:: distributed

From 22dbe7147fcb5d44dae4c37dc3f026b59369e5c1 Mon Sep 17 00:00:00 2001
From: Abdulelah Bin Mahfoodh <abduh.bm@gmail.com>
Date: Mon, 13 Apr 2020 16:37:00 +0300
Subject: [PATCH 0783/1550] Fix propagating inherit config in SSHCluster for
 non-bash shells (#3688)

* Fix propagating inherit config in SSHCluster for non-bash shells
---
 distributed/deploy/ssh.py | 71 ++++++++++++++++++++++++++-------------
 1 file changed, 48 insertions(+), 23 deletions(-)

diff --git a/distributed/deploy/ssh.py b/distributed/deploy/ssh.py
index a7b3526bcba..fc629f8445c 100644
--- a/distributed/deploy/ssh.py
+++ b/distributed/deploy/ssh.py
@@ -86,22 +86,38 @@ async def start(self):
         import asyncssh  # import now to avoid adding to module startup time
 
         self.connection = await asyncssh.connect(self.address, **self.connect_options)
-        self.proc = await self.connection.create_process(
-            " ".join(
-                [
-                    'DASK_INTERNAL_INHERIT_CONFIG="%s"'
-                    % serialize_for_cli(dask.config.global_config),
-                    sys.executable,
-                    "-m",
-                    self.worker_module,
-                    self.scheduler,
-                    "--name",
-                    str(self.name),
-                ]
-                + cli_keywords(self.kwargs, cls=_Worker, cmd=self.worker_module)
+
+        result = await self.connection.run("uname")
+        if result.exit_status == 0:
+            set_env = 'env DASK_INTERNAL_INHERIT_CONFIG="{}"'.format(
+                serialize_for_cli(dask.config.global_config)
             )
+        else:
+            result = await self.connection.run("cmd /c ver")
+            if result.exit_status == 0:
+                set_env = "set DASK_INTERNAL_INHERIT_CONFIG={} &&".format(
+                    serialize_for_cli(dask.config.global_config)
+                )
+            else:
+                raise Exception(
+                    "Worker failed to set DASK_INTERNAL_INHERIT_CONFIG variable "
+                )
+
+        cmd = " ".join(
+            [
+                set_env,
+                sys.executable,
+                "-m",
+                self.worker_module,
+                self.scheduler,
+                "--name",
+                str(self.name),
+            ]
+            + cli_keywords(self.kwargs, cls=_Worker, cmd=self.worker_module)
         )
 
+        self.proc = await self.connection.create_process(cmd)
+
         # We watch stderr in order to get the address, then we return
         while True:
             line = await self.proc.stderr.readline()
@@ -144,18 +160,27 @@ async def start(self):
 
         self.connection = await asyncssh.connect(self.address, **self.connect_options)
 
-        self.proc = await self.connection.create_process(
-            " ".join(
-                [
-                    'DASK_INTERNAL_INHERIT_CONFIG="%s"'
-                    % serialize_for_cli(dask.config.global_config),
-                    sys.executable,
-                    "-m",
-                    "distributed.cli.dask_scheduler",
-                ]
-                + cli_keywords(self.kwargs, cls=_Scheduler)
+        result = await self.connection.run("uname")
+        if result.exit_status == 0:
+            set_env = 'env DASK_INTERNAL_INHERIT_CONFIG="{}"'.format(
+                serialize_for_cli(dask.config.global_config)
             )
+        else:
+            result = await self.connection.run("cmd /c ver")
+            if result.exit_status == 0:
+                set_env = "set DASK_INTERNAL_INHERIT_CONFIG={} &&".format(
+                    serialize_for_cli(dask.config.global_config)
+                )
+            else:
+                raise Exception(
+                    "Scheduler failed to set DASK_INTERNAL_INHERIT_CONFIG variable "
+                )
+
+        cmd = " ".join(
+            [set_env, sys.executable, "-m", "distributed.cli.dask_scheduler",]
+            + cli_keywords(self.kwargs, cls=_Scheduler)
         )
+        self.proc = await self.connection.create_process(cmd)
 
         # We watch stderr in order to get the address, then we return
         while True:

From 8a0efe4ad8e05833e905f55f2e329a4aa02d2711 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 13 Apr 2020 07:18:36 -0700
Subject: [PATCH 0784/1550] Add Cluster __enter__ and __exit__ methods (#3699)

These just call sync on the async versions
---
 distributed/deploy/cluster.py | 6 ++++++
 distributed/deploy/spec.py    | 7 ++++---
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
index 8082d278483..65199c48bd5 100644
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -352,6 +352,12 @@ def _ipython_display_(self, **kwargs):
             data = {"text/plain": repr(self), "text/html": self._repr_html_()}
             display(data, raw=True)
 
+    def __enter__(self):
+        return self.sync(self.__aenter__)
+
+    def __exit__(self, typ, value, traceback):
+        return self.sync(self.__aexit__, typ, value, traceback)
+
     async def __aenter__(self):
         await self
         return self
diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index 99ab70d2de1..c6338d3b93f 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -396,13 +396,14 @@ async def _close(self):
 
         await super()._close()
 
-    def __enter__(self):
-        self.sync(self._correct_state)
+    async def __aenter__(self):
+        await self
+        await self._correct_state()
         assert self.status == "running"
         return self
 
     def __exit__(self, typ, value, traceback):
-        self.close()
+        super().__exit__(typ, value, traceback)
         self._loop_runner.stop()
 
     def _threads_per_worker(self) -> int:

From 20c3e29e49798d4d446e1991b1ea6042763fd3ab Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 13 Apr 2020 07:22:48 -0700
Subject: [PATCH 0785/1550] Replace Example with Examples in docstrings (#3697)

This was causing warnings in sphinx, and doesn't get rendered properly
---
 distributed/batched.py            | 4 ++--
 distributed/deploy/cluster.py     | 4 ++--
 distributed/profile.py            | 4 ++--
 distributed/protocol/serialize.py | 4 ++--
 4 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/distributed/batched.py b/distributed/batched.py
index 13c241d1e1b..07eb8e41014 100644
--- a/distributed/batched.py
+++ b/distributed/batched.py
@@ -22,8 +22,8 @@ class BatchedSend:
     Batching several messages at once helps performance when sending
     a myriad of tiny messages.
 
-    Example
-    -------
+    Examples
+    --------
     >>> stream = yield connect(address)
     >>> bstream = BatchedSend(interval='10 ms')
     >>> bstream.start(stream)
diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
index 65199c48bd5..7164b17b076 100644
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -135,8 +135,8 @@ def scale(self, n: int) -> None:
         n: int
             Target number of workers
 
-        Example
-        -------
+        Examples
+        --------
         >>> cluster.scale(10)  # scale cluster to ten workers
         """
         raise NotImplementedError()
diff --git a/distributed/profile.py b/distributed/profile.py
index 1bf81ad6ff0..33eba502ef9 100644
--- a/distributed/profile.py
+++ b/distributed/profile.py
@@ -79,8 +79,8 @@ def process(frame, child, state, stop=None, omit=None):
     This recursively adds counts to the existing state dictionary and creates
     new entries for new functions.
 
-    Example
-    -------
+    Examples
+    --------
     >>> import sys, threading
     >>> ident = threading.get_ident()  # replace with your thread of interest
     >>> frame = sys._current_frames()[ident]
diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index 6db7ca70c13..f20bc490752 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -272,8 +272,8 @@ def deserialize(header, frames, deserializers=None):
 class Serialize:
     """ Mark an object that should be serialized
 
-    Example
-    -------
+    Examples
+    --------
     >>> msg = {'op': 'update', 'data': to_serialize(123)}
     >>> msg  # doctest: +SKIP
     {'op': 'update', 'data': <Serialize: 123>}

From fee5c42ca5dab297d5e3829f1ccd94689e9d5b6c Mon Sep 17 00:00:00 2001
From: Abdulelah Bin Mahfoodh <abduh.bm@gmail.com>
Date: Mon, 13 Apr 2020 17:51:32 +0300
Subject: [PATCH 0786/1550] Add remote_python keyword to the new SSHCluster
 (#3701)

* Fix dask-ssh after removing local-directory keyword from dask_scheduler

* black changes

* Add a test to dask-ssh with local directory parameter

* Add python_remote keyword to SSHCluster

* Modify docstring for Scheduler and Worker

* Rename to remote_python
---
 distributed/deploy/ssh.py | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/distributed/deploy/ssh.py b/distributed/deploy/ssh.py
index fc629f8445c..4f0e713ffa9 100644
--- a/distributed/deploy/ssh.py
+++ b/distributed/deploy/ssh.py
@@ -58,6 +58,8 @@ class Worker(Process):
         The python module to run to start the worker.
     connect_options: dict
         kwargs to be passed to asyncssh connections
+    remote_python: str
+        Path to Python on remote node to run this worker.
     kwargs: dict
         These will be passed through the dask-worker CLI to the
         dask.distributed.Worker class
@@ -70,6 +72,7 @@ def __init__(
         connect_options: dict,
         kwargs: dict,
         worker_module="distributed.cli.dask_worker",
+        remote_python=None,
         loop=None,
         name=None,
     ):
@@ -81,6 +84,7 @@ def __init__(
         self.connect_options = connect_options
         self.kwargs = kwargs
         self.name = name
+        self.remote_python = remote_python
 
     async def start(self):
         import asyncssh  # import now to avoid adding to module startup time
@@ -141,17 +145,22 @@ class Scheduler(Process):
         The hostname where we should run this worker
     connect_options: dict
         kwargs to be passed to asyncssh connections
+    remote_python: str
+        Path to Python on remote node to run this scheduler.
     kwargs: dict
         These will be passed through the dask-scheduler CLI to the
         dask.distributed.Scheduler class
     """
 
-    def __init__(self, address: str, connect_options: dict, kwargs: dict):
+    def __init__(
+        self, address: str, connect_options: dict, kwargs: dict, remote_python=None
+    ):
         super().__init__()
 
         self.address = address
         self.kwargs = kwargs
         self.connect_options = connect_options
+        self.remote_python = remote_python
 
     async def start(self):
         import asyncssh  # import now to avoid adding to module startup time
@@ -220,6 +229,7 @@ def SSHCluster(
     worker_options: dict = {},
     scheduler_options: dict = {},
     worker_module: str = "distributed.cli.dask_worker",
+    remote_python: str = None,
     **kwargs
 ):
     """ Deploy a Dask cluster using SSH
@@ -254,6 +264,8 @@ def SSHCluster(
         Keywords to pass on to scheduler.
     worker_module: str, optional
         Python module to call to start the worker.
+    remote_python: str, optional
+        Path to Python on remote nodes.
 
     Examples
     --------
@@ -300,6 +312,7 @@ def SSHCluster(
             "address": hosts[0],
             "connect_options": connect_options,
             "kwargs": scheduler_options,
+            "remote_python": remote_python,
         },
     }
     workers = {
@@ -310,6 +323,7 @@ def SSHCluster(
                 "connect_options": connect_options,
                 "kwargs": worker_options,
                 "worker_module": worker_module,
+                "remote_python": remote_python,
             },
         }
         for i, host in enumerate(hosts[1:])

From 549528434dc4805d76cc2cfae74f7bff4ee6b645 Mon Sep 17 00:00:00 2001
From: "Jonathan J. Helmus" <jjhelmus@gmail.com>
Date: Tue, 14 Apr 2020 10:00:17 -0500
Subject: [PATCH 0787/1550] do not log an error on unset variable delete
 (#3652)

If a Variable is never set or accessed no entries are made in the
tracking attributes of VariableExtension.  Therefore there is no need to
raise or log an error when the variable is deleted.
---
 distributed/tests/test_variable.py | 13 +++++++++++++
 distributed/variable.py            |  6 ++++--
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/distributed/tests/test_variable.py b/distributed/tests/test_variable.py
index 64765d808c7..0e450aa7a02 100644
--- a/distributed/tests/test_variable.py
+++ b/distributed/tests/test_variable.py
@@ -2,6 +2,7 @@
 import random
 from time import sleep
 import sys
+import logging
 
 import pytest
 from tornado import gen
@@ -12,6 +13,7 @@
 from distributed.compatibility import WINDOWS
 from distributed.utils_test import gen_cluster, inc, div
 from distributed.utils_test import client, cluster_fixture, loop  # noqa: F401
+from distributed.utils_test import captured_logger
 
 
 @gen_cluster(client=True)
@@ -39,6 +41,17 @@ def test_variable(c, s, a, b):
         assert time() < start + 5
 
 
+@gen_cluster(client=True)
+async def test_delete_unset_variable(c, s, a, b):
+    x = Variable()
+    assert x.client is c
+    with captured_logger(logging.getLogger("distributed.utils")) as logger:
+        x.delete()
+        await c.close()
+    text = logger.getvalue()
+    assert "KeyError" not in text
+
+
 @gen_cluster(client=True)
 def test_queue_with_data(c, s, a, b):
     x = Variable("x")
diff --git a/distributed/variable.py b/distributed/variable.py
index 3c6cc931166..a47064b1397 100644
--- a/distributed/variable.py
+++ b/distributed/variable.py
@@ -113,8 +113,10 @@ async def delete(self, stream=None, name=None, client=None):
             else:
                 if old["type"] == "Future":
                     await self.release(old["value"], name)
-            del self.waiting_conditions[name]
-            del self.variables[name]
+            with ignoring(KeyError):
+                del self.waiting_conditions[name]
+            with ignoring(KeyError):
+                del self.variables[name]
 
 
 class Variable:

From 3a70aa6dc84cc29c2318b566d2d064ff84fed940 Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Tue, 14 Apr 2020 18:08:53 +0200
Subject: [PATCH 0788/1550] Allow modification of distributed.comm.retry at
 runtime (#3705)

---
 distributed/tests/test_scheduler.py | 28 +++++++++++++---------------
 distributed/utils_comm.py           | 17 ++++++++---------
 2 files changed, 21 insertions(+), 24 deletions(-)

diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 2b48fa030e4..d64c88fceea 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -7,7 +7,6 @@
 import re
 import sys
 from time import sleep
-from unittest import mock
 import logging
 
 import dask
@@ -1900,7 +1899,7 @@ async def test_gather_failing_cnn_recover(c, s, a, b):
     x = await c.scatter({"x": 1}, workers=a.address)
 
     s.rpc = await FlakyConnectionPool(failing_connections=1)
-    with mock.patch("distributed.utils_comm.retry_count", 1):
+    with dask.config.set({"distributed.comm.retry.count": 1}):
         res = await s.gather(keys=["x"])
     assert res["status"] == "OK"
 
@@ -1963,20 +1962,19 @@ def reducer(x, y):
 
     s.rpc = await FlakyConnectionPool(failing_connections=4)
 
-    with captured_logger(
-        logging.getLogger("distributed.scheduler")
-    ) as sched_logger, captured_logger(
-        logging.getLogger("distributed.client")
-    ) as client_logger, captured_logger(
-        logging.getLogger("distributed.utils_comm")
-    ) as utils_comm_logger, mock.patch(
-        "distributed.utils_comm.retry_count", 3
-    ), mock.patch(
-        "distributed.utils_comm.retry_delay_min", 0.5
+    with dask.config.set(
+        {"distributed.comm.retry.delay_min": 0.5, "distributed.comm.retry.count": 3,}
     ):
-        # Gather using the client (as an ordinary user would)
-        # Upon a missing key, the client will reschedule the computations
-        res = await c.gather(z)
+        with captured_logger(
+            logging.getLogger("distributed.scheduler")
+        ) as sched_logger, captured_logger(
+            logging.getLogger("distributed.client")
+        ) as client_logger, captured_logger(
+            logging.getLogger("distributed.utils_comm")
+        ) as utils_comm_logger:
+            # Gather using the client (as an ordinary user would)
+            # Upon a missing key, the client will reschedule the computations
+            res = await c.gather(z)
 
     assert res == 5
 
diff --git a/distributed/utils_comm.py b/distributed/utils_comm.py
index 42404754527..b7e33656ab8 100644
--- a/distributed/utils_comm.py
+++ b/distributed/utils_comm.py
@@ -312,15 +312,6 @@ def subs_multiple(o, d):
             return o
 
 
-retry_count = dask.config.get("distributed.comm.retry.count")
-retry_delay_min = parse_timedelta(
-    dask.config.get("distributed.comm.retry.delay.min"), default="s"
-)
-retry_delay_max = parse_timedelta(
-    dask.config.get("distributed.comm.retry.delay.max"), default="s"
-)
-
-
 async def retry(
     coro,
     count,
@@ -383,6 +374,14 @@ async def retry_operation(coro, *args, operation=None, **kwargs):
     """
     Retry an operation using the configuration values for the retry parameters
     """
+
+    retry_count = dask.config.get("distributed.comm.retry.count")
+    retry_delay_min = parse_timedelta(
+        dask.config.get("distributed.comm.retry.delay.min"), default="s"
+    )
+    retry_delay_max = parse_timedelta(
+        dask.config.get("distributed.comm.retry.delay.max"), default="s"
+    )
     return await retry(
         partial(coro, *args, **kwargs),
         count=retry_count,

From 65c4a4989350ae79c46e98a7f6ea79708b289e94 Mon Sep 17 00:00:00 2001
From: Tom Augspurger <TomAugspurger@users.noreply.github.com>
Date: Wed, 15 Apr 2020 16:00:28 -0500
Subject: [PATCH 0789/1550] Avoid DeprecationWarning from pandas (#3712)

---
 distributed/protocol/tests/test_pandas.py | 51 +++++++++++++++++------
 distributed/tests/test_collections.py     | 19 ++++++---
 2 files changed, 52 insertions(+), 18 deletions(-)

diff --git a/distributed/protocol/tests/test_pandas.py b/distributed/protocol/tests/test_pandas.py
index 104151fb55a..b1f96bfd486 100644
--- a/distributed/protocol/tests/test_pandas.py
+++ b/distributed/protocol/tests/test_pandas.py
@@ -1,5 +1,5 @@
+import numpy as np
 import pandas as pd
-import pandas.util.testing as tm
 import pytest
 
 from dask.dataframe.utils import assert_eq
@@ -22,18 +22,43 @@
     pd.DataFrame({"x": [b"a", b"b", b"c"]}),
     pd.DataFrame({"x": pd.Categorical(["a", "b", "a"], ordered=True)}),
     pd.DataFrame({"x": pd.Categorical(["a", "b", "a"], ordered=False)}),
-    tm.makeCategoricalIndex(),
-    tm.makeCustomDataframe(5, 3),
-    tm.makeDataFrame(),
-    tm.makeDateIndex(),
-    tm.makeMissingDataframe(),
-    tm.makeMixedDataFrame(),
-    tm.makeObjectSeries(),
-    tm.makePeriodFrame(),
-    tm.makeRangeIndex(),
-    tm.makeTimeDataFrame(),
-    tm.makeTimeSeries(),
-    tm.makeUnicodeIndex(),
+    pd.Index(pd.Categorical(["a"], categories=["a", "b"], ordered=True)),
+    pd.date_range("2000", periods=12, freq="B"),
+    pd.RangeIndex(10),
+    pd.DataFrame(
+        "a",
+        index=pd.Index(["a", "b", "c", "d"], name="a"),
+        columns=pd.Index(["A", "B", "C", "D"], name="columns"),
+    ),
+    pd.DataFrame(
+        np.random.randn(10, 5), columns=list("ABCDE"), index=list("abcdefghij")
+    ),
+    pd.DataFrame(
+        np.random.randn(10, 5), columns=list("ABCDE"), index=list("abcdefghij")
+    ).where(lambda x: x > 0),
+    pd.DataFrame(
+        {
+            "a": [0.0, 0.1],
+            "B": [0.0, 1.0],
+            "C": ["a", "b"],
+            "D": pd.to_datetime(["2000", "2001"]),
+        }
+    ),
+    pd.Series(["a", "b", "c"], index=["a", "b", "c"]),
+    pd.DataFrame(
+        np.random.randn(10, 5),
+        columns=list("ABCDE"),
+        index=pd.period_range("2000", periods=10, freq="B"),
+    ),
+    pd.DataFrame(
+        np.random.randn(10, 5),
+        columns=list("ABCDE"),
+        index=pd.date_range("2000", periods=10, freq="B"),
+    ),
+    pd.Series(
+        np.random.randn(10), name="a", index=pd.date_range("2000", periods=10, freq="B")
+    ),
+    pd.Index(["סשםקה7ךשץא", "8טלכז6לרפל"]),
 ]
 
 
diff --git a/distributed/tests/test_collections.py b/distributed/tests/test_collections.py
index 0843d711761..61424c68f38 100644
--- a/distributed/tests/test_collections.py
+++ b/distributed/tests/test_collections.py
@@ -11,7 +11,7 @@
 from distributed.utils_test import client, cluster_fixture, loop  # noqa F401
 import numpy as np
 import pandas as pd
-import pandas.util.testing as tm
+import pandas.testing as tm
 
 
 dfs = [
@@ -126,28 +126,37 @@ def test_dataframe_set_index_sync(wait, client):
     assert len(df2)
 
 
+def make_time_dataframe():
+    return pd.DataFrame(
+        np.random.randn(30, 4),
+        columns=list("ABCD"),
+        index=pd.date_range("2000", periods=30, freq="B"),
+    )
+
+
 def test_loc_sync(client):
-    df = pd.util.testing.makeTimeDataFrame()
+    df = make_time_dataframe()
     ddf = dd.from_pandas(df, npartitions=10)
     ddf.loc["2000-01-17":"2000-01-24"].compute()
 
 
 def test_rolling_sync(client):
-    df = pd.util.testing.makeTimeDataFrame()
+    df = make_time_dataframe()
     ddf = dd.from_pandas(df, npartitions=10)
     ddf.A.rolling(2).mean().compute()
 
 
 @gen_cluster(client=True)
 def test_loc(c, s, a, b):
-    df = pd.util.testing.makeTimeDataFrame()
+    df = make_time_dataframe()
     ddf = dd.from_pandas(df, npartitions=10)
     future = c.compute(ddf.loc["2000-01-17":"2000-01-24"])
     yield future
 
 
 def test_dataframe_groupby_tasks(client):
-    df = pd.util.testing.makeTimeDataFrame()
+    df = make_time_dataframe()
+
     df["A"] = df.A // 0.1
     df["B"] = df.B // 0.1
     ddf = dd.from_pandas(df, npartitions=10)

From 5243d23df9efb58d4f0d643c29a40062b365b65e Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Wed, 15 Apr 2020 14:05:48 -0700
Subject: [PATCH 0790/1550] Always use `readinto` in TCP (#3711)

As of Tornado 5.0.0+, `read_into` is always available. Given this is our
minimum requirement for Tornado, there is no need to handle earlier
Tornado versions that don't have this feature. So drop that code path to
simplify maintenance.
---
 distributed/comm/tcp.py | 17 ++++-------------
 1 file changed, 4 insertions(+), 13 deletions(-)

diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py
index 7003053ce06..769e9132abe 100644
--- a/distributed/comm/tcp.py
+++ b/distributed/comm/tcp.py
@@ -13,7 +13,7 @@
 
 import dask
 from tornado import netutil
-from tornado.iostream import StreamClosedError, IOStream
+from tornado.iostream import StreamClosedError
 from tornado.tcpclient import TCPClient
 from tornado.tcpserver import TCPServer
 
@@ -132,10 +132,6 @@ class TCP(Comm):
     An established communication based on an underlying Tornado IOStream.
     """
 
-    # IOStream.read_into() currently proposed in
-    # https://github.com/tornadoweb/tornado/pull/2193
-    _iostream_has_read_into = hasattr(IOStream, "read_into")
-
     def __init__(self, stream, local_addr, peer_addr, deserialize=True):
         Comm.__init__(self)
         self._local_addr = local_addr
@@ -192,15 +188,10 @@ async def read(self, deserializers=None):
 
             frames = []
             for length in lengths:
+                frame = bytearray(length)
                 if length:
-                    if self._iostream_has_read_into:
-                        frame = bytearray(length)
-                        n = await stream.read_into(frame)
-                        assert n == length, (n, length)
-                    else:
-                        frame = await stream.read_bytes(length)
-                else:
-                    frame = b""
+                    n = await stream.read_into(frame)
+                    assert n == length, (n, length)
                 frames.append(frame)
         except StreamClosedError as e:
             self.stream = None

From ee8cff496da9d26c2e140df549067d1683cab8ea Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Thu, 16 Apr 2020 18:05:17 +0200
Subject: [PATCH 0791/1550] Idempotent semaphore acquire with retries (#3690)

Semaphore.acquire now performs idempotent acquire requests and retries in case of
connection failures.
Each lease is now unique and is assigned a unique timeout which is controlled using the
configuration option ``distributed.scheduler.locks.lease-timeout``
---
 distributed/distributed.yaml        |   1 +
 distributed/semaphore.py            | 280 +++++++++++++++++++---------
 distributed/tests/test_semaphore.py | 235 +++++++++++++++++++----
 3 files changed, 388 insertions(+), 128 deletions(-)

diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index b11270f4704..71ecd840a10 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -44,6 +44,7 @@ distributed:
         check_unused_sessions_milliseconds: 500
     locks:
       lease-validation-interval: 10s  # The time to wait until an acquired semaphore is released if the Client goes out of scope
+      lease-timeout: 30s  # The timeout after which a lease will be released if not refreshed
 
     http:
       routes:
diff --git a/distributed/semaphore.py b/distributed/semaphore.py
index 2d506c8ed0e..0727c279a3a 100644
--- a/distributed/semaphore.py
+++ b/distributed/semaphore.py
@@ -1,6 +1,5 @@
 import uuid
 from collections import defaultdict, deque
-from functools import partial
 import asyncio
 import dask
 from asyncio import TimeoutError
@@ -9,6 +8,7 @@
 from .metrics import time
 import warnings
 import logging
+from distributed.utils_comm import retry_operation
 
 logger = logging.getLogger(__name__)
 
@@ -21,12 +21,14 @@ def __init__(self, duration=None):
     def start(self):
         self.started_at = time()
 
+    def elapsed(self):
+        return time() - self.started_at
+
     def leftover(self):
         if self.duration is None:
             return None
         else:
-            elapsed = time() - self.started_at
-            return max(0, self.duration - elapsed)
+            return max(0, self.duration - self.elapsed())
 
 
 class SemaphoreExtension:
@@ -37,41 +39,49 @@ class SemaphoreExtension:
     * semaphore_acquire
     * semaphore_release
     * semaphore_create
+    * semaphore_close
+    * semaphore_refresh_leases
     """
 
     def __init__(self, scheduler):
         self.scheduler = scheduler
-        self.leases = defaultdict(deque)
+
+        # {semaphore_name: asyncio.Event}
         self.events = defaultdict(asyncio.Event)
+        # {semaphore_name: max_leases}
         self.max_leases = dict()
-        self.leases_per_client = defaultdict(partial(defaultdict, deque))
+        # {semaphore_name: {lease_id: lease_last_seen_timestamp}}
+        self.leases = defaultdict(dict)
+
         self.scheduler.handlers.update(
             {
                 "semaphore_create": self.create,
                 "semaphore_acquire": self.acquire,
                 "semaphore_release": self.release,
                 "semaphore_close": self.close,
+                "semaphore_refresh_leases": self.refresh_leases,
             }
         )
 
         self.scheduler.extensions["semaphores"] = self
-        self.pc_validate_leases = PeriodicCallback(
-            self._validate_leases,
-            1000
-            * parse_timedelta(
-                dask.config.get(
-                    "distributed.scheduler.locks.lease-validation-interval"
-                ),
-                default="s",
-            ),
+
+        validation_callback_time = 1000 * parse_timedelta(
+            dask.config.get("distributed.scheduler.locks.lease-validation-interval"),
+            default="s",
+        )
+        self._pc_lease_timeout = PeriodicCallback(
+            self._check_lease_timeout,
+            validation_callback_time,
             io_loop=self.scheduler.loop,
         )
-        self.pc_validate_leases.start()
-        self._validation_running = False
+        self._pc_lease_timeout.start()
+        self.lease_timeout = parse_timedelta(
+            dask.config.get("distributed.scheduler.locks.lease-timeout"), default="s",
+        )
 
     # `comm` here is required by the handler interface
     def create(self, comm=None, name=None, max_leases=None):
-        # We use `self.max_leases.keys()` as the point of truth to find out if a semaphore with a specific
+        # We use `self.max_leases` as the point of truth to find out if a semaphore with a specific
         # `name` has been created.
         if name not in self.max_leases:
             assert isinstance(max_leases, int), max_leases
@@ -83,11 +93,32 @@ def create(self, comm=None, name=None, max_leases=None):
                     % (max_leases, self.max_leases[name])
                 )
 
-    def _get_lease(self, client, name, identifier):
+    def refresh_leases(self, comm=None, name=None, lease_ids=None):
+        with log_errors():
+            now = time()
+            logger.debug(
+                "Refresh leases for %s with ids %s at %s", name, lease_ids, now
+            )
+            for id_ in lease_ids:
+                if id_ not in self.leases[name]:
+                    logger.critical(
+                        f"Trying to refresh an unknown lease ID {id_} for {name}. This might be due to leases "
+                        f"timing out and may cause overbooking of the semaphore!"
+                        f"This is often caused by long-running GIL-holding in the task which acquired the lease."
+                    )
+                self.leases[name][id_] = now
+
+    def _get_lease(self, name, lease_id):
         result = True
-        if len(self.leases[name]) < self.max_leases[name]:
-            self.leases[name].append(identifier)
-            self.leases_per_client[client][name].append(identifier)
+
+        if (
+            # This allows request idempotency
+            lease_id in self.leases[name]
+            or len(self.leases[name]) < self.max_leases[name]
+        ):
+            now = time()
+            logger.info("Acquire lease %s for %s at %s", lease_id, name, now)
+            self.leases[name][lease_id] = now
         else:
             result = False
         return result
@@ -97,9 +128,7 @@ def _semaphore_exists(self, name):
             return False
         return True
 
-    async def acquire(
-        self, comm=None, name=None, client=None, timeout=None, identifier=None
-    ):
+    async def acquire(self, comm=None, name=None, timeout=None, lease_id=None):
         with log_errors():
             if not self._semaphore_exists(name):
                 raise RuntimeError(f"Semaphore `{name}` not known or already closed.")
@@ -110,11 +139,17 @@ async def acquire(
             w.start()
 
             while True:
+                logger.info(
+                    "Trying to acquire %s for %s with %ss left.",
+                    lease_id,
+                    name,
+                    w.leftover(),
+                )
                 # Reset the event and try to get a release. The event will be set if the state
                 # is changed and helps to identify when it is worth to retry an acquire
                 self.events[name].clear()
 
-                result = self._get_lease(client, name, identifier)
+                result = self._get_lease(name, lease_id)
 
                 # If acquiring fails, we wait for the event to be set, i.e. something has
                 # been released and we can try to acquire again (continue loop)
@@ -127,9 +162,16 @@ async def acquire(
                         continue
                     except TimeoutError:
                         result = False
+                logger.info(
+                    "Acquisition of lease %s for %s is %s after waiting for %ss.",
+                    lease_id,
+                    name,
+                    result,
+                    w.elapsed(),
+                )
                 return result
 
-    def release(self, comm=None, name=None, client=None, identifier=None):
+    def release(self, comm=None, name=None, lease_id=None):
         with log_errors():
             if not self._semaphore_exists(name):
                 logger.warning(
@@ -138,37 +180,41 @@ def release(self, comm=None, name=None, client=None, identifier=None):
                 return
             if isinstance(name, list):
                 name = tuple(name)
-            if name in self.leases and identifier in self.leases[name]:
-                self._release_value(name, client, identifier)
+            if name in self.leases and lease_id in self.leases[name]:
+                self._release_value(name, lease_id)
             else:
-                raise ValueError(
+                logger.warning(
                     f"Tried to release semaphore but it was already released: "
-                    f"client={client}, name={name}, identifier={identifier}"
+                    f"name={name}, lease_id={lease_id}. This can happen if the semaphore timed out before."
                 )
 
-    def _release_value(self, name, client, identifier):
+    def _release_value(self, name, lease_id):
+        logger.info("Releasing %s for %s", lease_id, name)
         # Everything needs to be atomic here.
-        self.leases_per_client[client][name].remove(identifier)
-        self.leases[name].remove(identifier)
+        del self.leases[name][lease_id]
         self.events[name].set()
 
-    def _release_client(self, client):
-        semaphore_names = list(self.leases_per_client[client])
+    def _check_lease_timeout(self):
+        now = time()
+        semaphore_names = list(self.leases.keys())
         for name in semaphore_names:
-            ids = list(self.leases_per_client[client][name])
-            for _id in list(ids):
-                self._release_value(name=name, client=client, identifier=_id)
-        del self.leases_per_client[client]
-
-    def _validate_leases(self):
-        if not self._validation_running:
-            self._validation_running = True
-            known_clients_with_leases = set(self.leases_per_client.keys())
-            scheduler_clients = set(self.scheduler.clients.keys())
-            for dead_client in known_clients_with_leases - scheduler_clients:
-                self._release_client(dead_client)
-            else:
-                self._validation_running = False
+            ids = list(self.leases[name])
+            logger.debug(
+                "Validating leases for %s at time %s. Currently known %s",
+                name,
+                now,
+                self.leases[name],
+            )
+            for _id in ids:
+                time_since_refresh = now - self.leases[name][_id]
+                if time_since_refresh > self.lease_timeout:
+                    logger.info(
+                        "Lease %s for %s timed out after %ss.",
+                        _id,
+                        name,
+                        time_since_refresh,
+                    )
+                    self._release_value(name=name, lease_id=_id)
 
     def close(self, comm=None, name=None):
         """Hard close the semaphore without warning clients which still hold a lease."""
@@ -180,15 +226,12 @@ def close(self, comm=None, name=None):
             if name in self.events:
                 del self.events[name]
             if name in self.leases:
-                del self.leases[name]
-
-            for client, client_leases in self.leases_per_client.items():
-                if name in client_leases:
+                if self.leases[name]:
                     warnings.warn(
-                        f"Closing semaphore `{name}` but client `{client}` still has a lease open.",
+                        f"Closing semaphore {name} but there remain unreleased leases {sorted(self.leases[name])}",
                         RuntimeWarning,
                     )
-                    del client_leases[name]
+                del self.leases[name]
 
 
 class Semaphore:
@@ -200,19 +243,31 @@ class Semaphore:
     already acquired, it is not possible to acquire more and the caller waits
     until another lease has been released.
 
-    The lifetime of a lease is coupled to the ``Client`` it was acquired with.
-    Once the Client goes out of scope, the leases associated to it are freed.
-    This behavior can be controlled with the
-    ``distributed.scheduler.locks.lease-validation-interval`` configuration
-    option.
+    The lifetime or leases are controlled using a timeout. This timeout is
+    refreshed in regular intervals by the ``Client`` of this instance and
+    provides protection from deadlocks or resource starvation in case of worker
+    failure.
+    The timeout can be controlled using the configuration option
+    ``distributed.scheduler.locks.lease-timeout`` and the interval in which the
+    scheduler verifies the timeout is set using the option
+    ``distributed.scheduler.locks.lease-validation-interval``.
 
     A noticeable difference to the Semaphore of the python standard library is
     that this implementation does not allow to release more often than it was
     acquired. If this happens, a warning is emitted but the internal state is
     not modified.
 
-    This implementation is still in an experimental state and subtle changes in
-    behavior may occur without any change in the major version of this library.
+    .. warning::
+
+        This implementation is still in an experimental state and subtle
+        changes in behavior may occur without any change in the major version
+        of this library.
+
+    .. warning::
+
+        This implementation is susceptible to lease overbooking in case of
+        lease timeouts. It is advised to monitor log information and adjust
+        above configuration options to suitable values for the user application.
 
     Parameters
     ----------
@@ -263,25 +318,37 @@ class Semaphore:
     """
 
     def __init__(self, max_leases=1, name=None, client=None):
-        # NOTE: the `id` of the `Semaphore` instance will always be unique, even among different
-        # instances for the same resource. The actual attribute that identifies a specific resource is `name`,
-        # which will be the same for all instances of this class which limit the same resource.
         self.client = client or get_client()
-        self.id = uuid.uuid4().hex
         self.name = name or "semaphore-" + uuid.uuid4().hex
         self.max_leases = max_leases
+        self.id = uuid.uuid4().hex
+        self._leases = deque()
 
-        if self.client.asynchronous:
-            self._started = self.client.scheduler.semaphore_create(
-                name=self.name, max_leases=max_leases
-            )
-        else:
-            self.client.sync(
-                self.client.scheduler.semaphore_create,
-                name=self.name,
-                max_leases=max_leases,
+        self._started = self.client.sync(
+            self.client.scheduler.semaphore_create,
+            name=self.name,
+            max_leases=max_leases,
+        )
+        # this should give ample time to refresh without introducing another
+        # config parameter since this *must* be smaller than the timeout anyhow
+        refresh_leases_interval = (
+            parse_timedelta(
+                dask.config.get("distributed.scheduler.locks.lease-timeout"),
+                default="s",
             )
-            self._started = asyncio.sleep(0)
+            / 5
+        )
+        self._refreshing_leases = False
+        pc = PeriodicCallback(
+            self._refresh_leases,
+            callback_time=1000 * refresh_leases_interval,
+            io_loop=self.client.io_loop,
+        )
+        self.refresh_callback = pc
+        # Registering the pc to the client here is important for proper cleanup
+        self._periodic_callback_name = f"refresh_semaphores_{self.id}"
+        self.client._periodic_callbacks[self._periodic_callback_name] = pc
+        pc.start()
 
     def __await__(self):
         async def create_semaphore():
@@ -290,6 +357,40 @@ async def create_semaphore():
 
         return create_semaphore().__await__()
 
+    async def _refresh_leases(self):
+        if self.client.scheduler is not None and not self._refreshing_leases:
+            self._refreshing_leases = True
+            if self._leases:
+                logger.debug(
+                    "%s refreshing leases for %s with IDs %s",
+                    self.client.id,
+                    self.name,
+                    self._leases,
+                )
+                await self.client.scheduler.semaphore_refresh_leases(
+                    lease_ids=list(self._leases), name=self.name
+                )
+            self._refreshing_leases = False
+
+    async def _acquire(self, timeout=None):
+        lease_id = uuid.uuid4().hex
+        logger.info(
+            "%s requests lease for %s with ID %s", self.client.id, self.name, lease_id,
+        )
+
+        # Using a unique lease id generated here allows us to retry since the
+        # server handle is idempotent
+
+        result = await retry_operation(
+            self.client.scheduler.semaphore_acquire,
+            name=self.name,
+            timeout=timeout,
+            lease_id=lease_id,
+        )
+        if result:
+            self._leases.append(lease_id)
+        return result
+
     def acquire(self, timeout=None):
         """
         Acquire a semaphore.
@@ -297,16 +398,7 @@ def acquire(self, timeout=None):
         If the internal counter is greater than zero, decrement it by one and return True immediately.
         If it is zero, wait until a release() is called and return True.
         """
-        # TODO: This (may?) keep the HTTP request open until timeout runs out (forever if None).
-        #  Can do this in batches of smaller timeouts.
-        # TODO: what if connection breaks up?
-        return self.client.sync(
-            self.client.scheduler.semaphore_acquire,
-            name=self.name,
-            timeout=timeout,
-            client=self.client.id,
-            identifier=self.id,
-        )
+        return self.client.sync(self._acquire, timeout=timeout)
 
     def release(self):
         """
@@ -316,12 +408,13 @@ def release(self):
         """
 
         """ Release the lock if already acquired """
-        # TODO: what if connection breaks up?
+        if not self._leases:
+            raise RuntimeError("Released too often")
+        # popleft to release the oldest lease first
+        lease_id = self._leases.popleft()
+        logger.info("%s releases %s for %s", self.client.id, lease_id, self.name)
         return self.client.sync(
-            self.client.scheduler.semaphore_release,
-            name=self.name,
-            client=self.client.id,
-            identifier=self.id,
+            self.client.scheduler.semaphore_release, name=self.name, lease_id=lease_id,
         )
 
     def __enter__(self):
@@ -350,3 +443,8 @@ def __setstate__(self, state):
 
     def close(self):
         return self.client.sync(self.client.scheduler.semaphore_close, name=self.name)
+
+    def __del__(self):
+        if self._periodic_callback_name in self.client._periodic_callbacks:
+            self.client._periodic_callbacks[self._periodic_callback_name].stop()
+            del self.client._periodic_callbacks[self._periodic_callback_name]
diff --git a/distributed/tests/test_semaphore.py b/distributed/tests/test_semaphore.py
index 9d94b83515a..3c68f685eff 100644
--- a/distributed/tests/test_semaphore.py
+++ b/distributed/tests/test_semaphore.py
@@ -1,17 +1,24 @@
 import pickle
-
 import dask
+import pytest
 from dask.distributed import Client
 
 from distributed import Semaphore
+from distributed.comm import Comm
+from distributed.core import ConnectionPool
 from distributed.metrics import time
-from distributed.utils_test import cluster, gen_cluster
-from distributed.utils_test import client, loop, cluster_fixture  # noqa: F401
-import pytest
+from distributed.utils_test import (  # noqa: F401
+    client,
+    cluster,
+    cluster_fixture,
+    gen_cluster,
+    slowidentity,
+    loop,
+)
 
 
 @gen_cluster(client=True)
-async def test_semaphore(c, s, a, b):
+async def test_semaphore_trivial(c, s, a, b):
     semaphore = await Semaphore(max_leases=2, name="resource_we_want_to_limit")
 
     result = await semaphore.acquire()  # allowed_leases: 2 - 1 -> 1
@@ -81,36 +88,37 @@ def test_timeout_sync(client):
         assert s.acquire(timeout=0.025) is False
 
 
-@pytest.mark.slow
-@gen_cluster(client=True, timeout=20)
+@gen_cluster(
+    client=True,
+    timeout=20,
+    config={
+        "distributed.scheduler.locks.lease-validation-interval": "500ms",
+        "distributed.scheduler.locks.lease-timeout": "500ms",
+    },
+)
 async def test_release_semaphore_after_timeout(c, s, a, b):
-    with dask.config.set(
-        {"distributed.scheduler.locks.lease-validation-interval": "50ms"}
-    ):
-        sem = await Semaphore(name="x", max_leases=2)
-        await sem.acquire()  # leases: 2 - 1 = 1
-        semY = await Semaphore(name="y")
-
-        async with Client(s.address, asynchronous=True, name="ClientB") as clientB:
-            semB = await Semaphore(name="x", max_leases=2, client=clientB)
-            semYB = await Semaphore(name="y", client=clientB)
-
-            assert await semB.acquire()  # leases: 1 - 1 = 0
-            assert await semYB.acquire()
+    sem = await Semaphore(name="x", max_leases=2)
+    await sem.acquire()  # leases: 2 - 1 = 1
+    semY = await Semaphore(name="y")
 
-            assert not (await sem.acquire(timeout=0.01))
-            assert not (await semB.acquire(timeout=0.01))
-            assert not (await semYB.acquire(timeout=0.01))
+    async with Client(s.address, asynchronous=True, name="ClientB") as clientB:
+        semB = await Semaphore(name="x", max_leases=2, client=clientB)
+        semYB = await Semaphore(name="y", client=clientB)
 
-        # `ClientB` goes out of scope, leases should be released
-        # At this point, we should be able to acquire x and y once
-        assert await sem.acquire()
-        assert await semY.acquire()
+        assert await semB.acquire()  # leases: 1 - 1 = 0
+        assert await semYB.acquire()
 
-        assert not (await semY.acquire(timeout=0.01))
         assert not (await sem.acquire(timeout=0.01))
+        assert not (await semB.acquire(timeout=0.01))
+        assert not (await semYB.acquire(timeout=0.01))
 
-        assert clientB.id not in s.extensions["semaphores"].leases_per_client
+    # `ClientB` goes out of scope, leases should be released
+    # At this point, we should be able to acquire x and y once
+    assert await sem.acquire()
+    assert await semY.acquire()
+
+    assert not (await semY.acquire(timeout=0.5))
+    assert not (await sem.acquire(timeout=0.5))
 
 
 @gen_cluster()
@@ -185,7 +193,8 @@ async def test_close_async(c, s, a, b):
 
     assert await sem.acquire()
     with pytest.warns(
-        RuntimeWarning, match="Closing semaphore .* but client .* still has a lease"
+        RuntimeWarning,
+        match="Closing semaphore .* but there remain unreleased leases .*",
     ):
         await sem.close()
 
@@ -198,7 +207,6 @@ async def test_close_async(c, s, a, b):
     assert not semaphore_object.max_leases
     assert not semaphore_object.leases
     assert not semaphore_object.events
-    assert not any(semaphore_object.leases_per_client.values())
 
 
 def test_close_sync(client):
@@ -215,9 +223,7 @@ async def test_release_once_too_many(c, s, a, b):
     assert await sem.acquire()
     await sem.release()
 
-    with pytest.raises(
-        ValueError, match="Tried to release semaphore but it was already released"
-    ):
+    with pytest.raises(RuntimeError, match="Released too often"):
         await sem.release()
 
     assert await sem.acquire()
@@ -229,9 +235,7 @@ async def test_release_once_too_many_resilience(c, s, a, b):
     def f(x, sem):
         sem.acquire()
         sem.release()
-        with pytest.raises(
-            ValueError, match="Tried to release semaphore but it was already released"
-        ):
+        with pytest.raises(RuntimeError, match="Released too often"):
             sem.release()
         return x
 
@@ -244,3 +248,160 @@ def f(x, sem):
     assert not s.extensions["semaphores"].leases["x"]
     await sem.acquire()
     assert len(s.extensions["semaphores"].leases["x"]) == 1
+
+
+class BrokenComm(Comm):
+    peer_address = None
+    local_address = None
+
+    def close(self):
+        pass
+
+    def closed(self):
+        return True
+
+    def abort(self):
+        pass
+
+    def read(self, deserializers=None):
+        raise EnvironmentError
+
+    def write(self, msg, serializers=None, on_error=None):
+        raise EnvironmentError
+
+
+class FlakyConnectionPool(ConnectionPool):
+    def __init__(self, *args, failing_connections=0, **kwargs):
+        self.cnn_count = 0
+        self.failing_connections = failing_connections
+        self._flaky_active = False
+        super().__init__(*args, **kwargs)
+
+    def activate(self):
+        self._flaky_active = True
+
+    async def connect(self, *args, **kwargs):
+        if self.cnn_count >= self.failing_connections or not self._flaky_active:
+            return await super().connect(*args, **kwargs)
+        else:
+            self.cnn_count += 1
+            return BrokenComm()
+
+
+@gen_cluster(client=True)
+async def test_retry_acquire(c, s, a, b):
+    with dask.config.set({"distributed.comm.retry.count": 1}):
+
+        pool = await FlakyConnectionPool(failing_connections=1)
+        rpc = pool(s.address)
+        c.scheduler = rpc
+        semaphore = await Semaphore(
+            max_leases=2, name="resource_we_want_to_limit", client=c
+        )
+        pool.activate()
+
+        result = await semaphore.acquire()
+        assert result is True
+
+        second = await semaphore.acquire()
+        assert second is True
+        start = time()
+        result = await semaphore.acquire(timeout=0.025)
+        stop = time()
+        assert stop - start < 0.2
+        assert result is False
+
+
+@gen_cluster(
+    client=True,
+    config={
+        "distributed.scheduler.locks.lease-timeout": "100ms",
+        "distributed.scheduler.locks.lease-validation-interval": "10ms",
+    },
+)
+async def test_oversubscribing_leases(c, s, a, b):
+    """
+    This test ensures that we detect oversubscription scenarios and will not
+    accept new leases as long as the semaphore is oversubscribed.
+
+    Oversubscription may occur if tasks hold the GIL for a longer time than the
+    lease-timeout is configured causing the lease refreshs to go stale and
+    timeout.
+
+    We cannot protect ourselves entirely from this but we can ensure that while
+    a task with a timed out lease is still running, we block further
+    acquisitions until we return to normal.
+
+    An example would be a task which continuously locks the GIL for a longer
+    time than the lease timeout but this continous lock only makes up a
+    fraction of the tasks runtime.
+
+    """
+    # GH3705
+
+    from distributed.worker import Worker, get_client
+
+    # Using the metadata as a crude "asyncio.Event" since the proper event
+    # implementation cannot be serialized. For the purpose of this test a
+    # metadata check with a sleep loop is not elegant but practical.
+    await c.set_metadata("release", False)
+    sem = await Semaphore()
+
+    def guaranteed_lease_timeout(x, sem):
+        """
+        This function simulates a payload computation with some GIL
+        locking in the beginning.
+
+        To simulate this we will manually disable the refresh callback, i.e.
+        all leases will eventually timeout. The function will only
+        release/return once the "Event" is set, i.e. our observer is done.
+        """
+        sem.refresh_callback.stop()
+        client = get_client()
+
+        with sem:
+            # This simulates a task which holds the GIL for longer than the
+            # lease-timeout.
+            slowidentity(delay=0.2)
+            old_value = client.set_metadata(x, "locked")
+
+            # Now the GIL is free again, i.e. we enable the callback again
+            sem.refresh_callback.start()
+
+            # This is the poormans Event.wait()
+            while not client.get_metadata("release"):
+                slowidentity(delay=0.02)
+            return x
+
+    def observe_state(sem):
+        """
+        This function is 100% artificial and acts as an observer to verify
+        our assumptions. The function will wait until both payload tasks are
+        executing, i.e. we're in an oversubscription scenario. It will then
+        try to acquire and hopefully fail showing that the semaphore is
+        protected if the oversubscription is recognized.
+        """
+        client = get_client()
+        x_locked = False
+        y_locked = False
+        # We wait until we're in an oversubscribed state, i.e. both tasks
+        # are executed although there should only be one allowed
+        while not x_locked and y_locked:
+            slowidentity(delay=0.005)
+            x_locked = client.get_metadata(0) == "locked"
+            y_locked = client.get_metadata(1) == "locked"
+
+        # Once we're in an oversubscribed state, we must not be able to
+        # acquire a lease.
+        assert not sem.acquire(timeout=0.05)
+        client.set_metadata("release", True)
+
+    observer = await Worker(s.address)
+
+    futures = c.map(
+        guaranteed_lease_timeout, range(2), sem=sem, workers=[a.address, b.address]
+    )
+    fut_observe = c.submit(observe_state, sem=sem, workers=[observer.address])
+
+    payload, observer = await c.gather([futures, fut_observe])
+    assert sorted(payload) == [0, 1]

From 6a3dc40891b9df0e2b94073a945641ebb39bfb5a Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Fri, 17 Apr 2020 00:36:27 +0100
Subject: [PATCH 0792/1550] Force threads_per_worker (#3715)

---
 distributed/deploy/tests/test_adaptive.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/distributed/deploy/tests/test_adaptive.py b/distributed/deploy/tests/test_adaptive.py
index 9c68e6ddf53..7fb91292540 100644
--- a/distributed/deploy/tests/test_adaptive.py
+++ b/distributed/deploy/tests/test_adaptive.py
@@ -1,3 +1,4 @@
+import gc
 import math
 from time import sleep
 
@@ -151,6 +152,7 @@ def test_min_max():
         processes=False,
         dashboard_address=None,
         asynchronous=True,
+        threads_per_worker=1,
     )
     try:
         adapt = cluster.adapt(minimum=1, maximum=2, interval="20 ms", wait_count=10)
@@ -179,6 +181,7 @@ def test_min_max():
         assert len(adapt.log) == 2 and all(d["status"] == "up" for _, d in adapt.log)
 
         del futures
+        gc.collect()
 
         start = time()
         while len(cluster.scheduler.workers) != 1:

From d5cb312496708d84b01a423ce5cd4867240cc4d3 Mon Sep 17 00:00:00 2001
From: "Richard (Rick) Zamora" <rzamora217@gmail.com>
Date: Fri, 17 Apr 2020 10:16:02 -0500
Subject: [PATCH 0793/1550] Dask-serialize dicts longer than five elements
 (#3689)

---
 distributed/protocol/serialize.py            | 34 +++++++++++++++--
 distributed/protocol/tests/test_serialize.py | 40 ++++++++++++++++++++
 2 files changed, 71 insertions(+), 3 deletions(-)

diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index f20bc490752..a1b35ec4463 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -90,6 +90,20 @@ def register_serialization_family(name, dumps, loads):
 register_serialization_family("error", None, serialization_error_loads)
 
 
+def check_dask_serializable(x):
+    if type(x) in (list, set, tuple) and len(x):
+        return check_dask_serializable(next(iter(x)))
+    elif type(x) is dict and len(x):
+        return check_dask_serializable(next(iter(x.items()))[1])
+    else:
+        try:
+            dask_serialize.dispatch(type(x))
+            return True
+        except TypeError:
+            pass
+    return False
+
+
 def serialize(x, serializers=None, on_error="message", context=None):
     r"""
     Convert object to a header and list of bytestrings
@@ -132,8 +146,22 @@ def serialize(x, serializers=None, on_error="message", context=None):
     if isinstance(x, Serialized):
         return x.header, x.frames
 
+    if type(x) in (list, set, tuple, dict):
+        iterate_collection = False
+        if type(x) is list and "msgpack" in serializers:
+            # Note: "msgpack" will always convert lists to tuples
+            #       (see GitHub #3716), so we should iterate
+            #       through the list if "msgpack" comes before "pickle"
+            #       in the list of serializers.
+            iterate_collection = ("pickle" not in serializers) or (
+                serializers.index("pickle") > serializers.index("msgpack")
+            )
+        if not iterate_collection:
+            # Check for "dask"-serializable data in dict/list/set
+            iterate_collection = check_dask_serializable(x)
+
     # Determine whether keys are safe to be serialized with msgpack
-    if type(x) is dict and len(x) <= 5:
+    if type(x) is dict and iterate_collection:
         try:
             msgpack.dumps(list(x.keys()))
         except Exception:
@@ -143,9 +171,9 @@ def serialize(x, serializers=None, on_error="message", context=None):
 
     if (
         type(x) in (list, set, tuple)
-        and len(x) <= 5
+        and iterate_collection
         or type(x) is dict
-        and len(x) <= 5
+        and iterate_collection
         and dict_safe
     ):
         if isinstance(x, dict):
diff --git a/distributed/protocol/tests/test_serialize.py b/distributed/protocol/tests/test_serialize.py
index 41e2af51b70..dd23e5e635d 100644
--- a/distributed/protocol/tests/test_serialize.py
+++ b/distributed/protocol/tests/test_serialize.py
@@ -21,6 +21,7 @@
     register_serialization_family,
     dask_serialize,
 )
+from distributed.protocol.serialize import check_dask_serializable
 from distributed.utils import nbytes
 from distributed.utils_test import inc, gen_test
 from distributed.comm.utils import to_frames, from_frames
@@ -388,3 +389,42 @@ def _(x):
 
     header, frames = serialize([MyObj(), MyObj()])
     assert header["compression"] == [False, False]
+
+
+@pytest.mark.parametrize(
+    "data,is_serializable",
+    [
+        ([], False),
+        ({}, False),
+        ({i: i for i in range(10)}, False),
+        (set(range(10)), False),
+        (tuple(range(100)), False),
+        ({"x": MyObj(5)}, True),
+        ({"x": {"y": MyObj(5)}}, True),
+        pytest.param(
+            [1, MyObj(5)],
+            True,
+            marks=pytest.mark.xfail(reason="Only checks 0th element for now."),
+        ),
+        ([MyObj([0, 1, 2]), 1], True),
+        (tuple([MyObj(None)]), True),
+        ({("x", i): MyObj(5) for i in range(100)}, True),
+    ],
+)
+def test_check_dask_serializable(data, is_serializable):
+    result = check_dask_serializable(data)
+    expected = is_serializable
+
+    assert result == expected
+
+
+@pytest.mark.parametrize(
+    "serializers",
+    [["msgpack"], ["pickle"], ["msgpack", "pickle"], ["pickle", "msgpack"]],
+)
+def test_serialize_lists(serializers):
+    data_in = ["a", 2, "c", None, "e", 6]
+    header, frames = serialize(data_in, serializers=serializers)
+    data_out = deserialize(header, frames)
+
+    assert data_in == data_out

From 07b0cfeef4d2515361c1ee89222c18d30ad26a67 Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Fri, 17 Apr 2020 18:49:28 +0200
Subject: [PATCH 0794/1550] Adjust semaphore test timeouts (#3720)

---
 distributed/tests/test_semaphore.py | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/distributed/tests/test_semaphore.py b/distributed/tests/test_semaphore.py
index 3c68f685eff..2f08fe45751 100644
--- a/distributed/tests/test_semaphore.py
+++ b/distributed/tests/test_semaphore.py
@@ -361,7 +361,8 @@ def guaranteed_lease_timeout(x, sem):
 
         with sem:
             # This simulates a task which holds the GIL for longer than the
-            # lease-timeout.
+            # lease-timeout. This is twice the lease timeout to ensurre that the
+            # leases are actually timed out
             slowidentity(delay=0.2)
             old_value = client.set_metadata(x, "locked")
 
@@ -391,9 +392,12 @@ def observe_state(sem):
             x_locked = client.get_metadata(0) == "locked"
             y_locked = client.get_metadata(1) == "locked"
 
+        # Once both are locked we should give the refresh time to notify the scheduler
+        # This parameter should be larger than ``lease-validation-interval``
+        slowidentity(delay=0.15)
         # Once we're in an oversubscribed state, we must not be able to
         # acquire a lease.
-        assert not sem.acquire(timeout=0.05)
+        assert not sem.acquire(timeout=0)
         client.set_metadata("release", True)
 
     observer = await Worker(s.address)
@@ -405,3 +409,16 @@ def observe_state(sem):
 
     payload, observer = await c.gather([futures, fut_observe])
     assert sorted(payload) == [0, 1]
+
+
+@gen_cluster(client=True,)
+async def test_timeout_zero(c, s, a, b):
+    # Depending on the internals a timeout zero cannot work, e.g. when the
+    # initial try already includes a wait. Since some test cases use this, it is
+    # worth testing against.
+
+    sem = await Semaphore()
+
+    assert await sem.acquire(timeout=0)
+    assert not await sem.acquire(timeout=0)
+    await sem.release()

From 41f1568975d51815e87df006f70589a8a4f6b84d Mon Sep 17 00:00:00 2001
From: Tom Augspurger <TomAugspurger@users.noreply.github.com>
Date: Fri, 17 Apr 2020 15:35:32 -0500
Subject: [PATCH 0795/1550] Add batch_size to Client.map (#3650)

An informal benchmark

```python
In [9]: %time _ = wait(c.map(inc, range(100_000), pure=False,
batch_size=1_000))
CPU times: user 31.8 s, sys: 1.07 s, total: 32.9 s
Wall time: 33.3 s

In [10]: %time _ = wait(c.map(inc, range(100_000), pure=False))

CPU times: user 45.3 s, sys: 2.13 s, total: 47.5 s
Wall time: 48.4 s
```

The difference likely increases in the size of the iterable.

Closes https://github.com/dask/distributed/issues/2181
---
 distributed/client.py            | 47 +++++++++++++++++++++++++++-----
 distributed/tests/test_client.py | 16 +++++++++++
 2 files changed, 56 insertions(+), 7 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 6545e938511..607ca9f5b48 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -30,7 +30,7 @@
 from dask.compatibility import apply
 from dask.utils import ensure_dict, format_bytes, funcname
 
-from tlz import first, groupby, merge, valmap, keymap
+from tlz import first, groupby, merge, valmap, keymap, partition_all
 
 try:
     from dask.delayed import single_key
@@ -1044,7 +1044,7 @@ async def _ensure_connected(self, timeout=None):
 
         try:
             comm = await connect(
-                self.scheduler.address, timeout=timeout, **self.connection_args,
+                self.scheduler.address, timeout=timeout, **self.connection_args
             )
             comm.name = "Client->Scheduler"
             if timeout is not None:
@@ -1540,6 +1540,7 @@ def map(
         actor=False,
         actors=False,
         pure=None,
+        batch_size=None,
         **kwargs,
     ):
         """ Map a function on a sequence of arguments
@@ -1579,6 +1580,11 @@ def map(
             See :doc:`actors` for additional details.
         actors: bool (default False)
             Alias for `actor`
+        batch_size : int, optional
+            Submit tasks to the scheduler in batches of (at most) ``batch_size``.
+            Larger batch sizes can be useful for very large ``iterables``,
+            as the cluster can start processing tasks while later ones are
+            submitted asynchronously.
         **kwargs: dict
             Extra keywords to send to the function.
             Large values will be included explicitly in the task graph.
@@ -1596,11 +1602,6 @@ def map(
         --------
         Client.submit: Submit a single function
         """
-        key = key or funcname(func)
-        actor = actor or actors
-        if pure is None:
-            pure = not actor
-
         if not callable(func):
             raise TypeError("First input to map must be a callable function")
 
@@ -1611,6 +1612,38 @@ def map(
                 "Dask no longer supports mapping over Iterators or Queues."
                 "Consider using a normal for loop and Client.submit"
             )
+        total_length = sum(len(x) for x in iterables)
+
+        if batch_size and batch_size > 1 and total_length > batch_size:
+            batches = list(
+                zip(*[partition_all(batch_size, iterable) for iterable in iterables])
+            )
+            return sum(
+                [
+                    self.map(
+                        func,
+                        *batch,
+                        key=key,
+                        workers=workers,
+                        retries=retries,
+                        priority=priority,
+                        allow_other_workers=allow_other_workers,
+                        fifo_timeout=fifo_timeout,
+                        resources=resources,
+                        actor=actor,
+                        actors=actors,
+                        pure=pure,
+                        **kwargs,
+                    )
+                    for batch in batches
+                ],
+                [],
+            )
+
+        key = key or funcname(func)
+        actor = actor or actors
+        if pure is None:
+            pure = not actor
 
         if allow_other_workers and workers is None:
             raise ValueError("Only use allow_other_workers= if using workers=")
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 297394631ae..17aea54f78a 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -218,6 +218,22 @@ def test_map_retries(c, s, a, b):
         yield z
 
 
+@gen_cluster(client=True)
+async def test_map_batch_size(c, s, a, b):
+    result = c.map(inc, range(100), batch_size=10)
+    result = await c.gather(result)
+    assert result == list(range(1, 101))
+
+    result = c.map(add, range(100), range(100), batch_size=10)
+    result = await c.gather(result)
+    assert result == list(range(0, 200, 2))
+
+    # mismatch shape
+    result = c.map(add, range(100, 200), range(10), batch_size=2)
+    result = await c.gather(result)
+    assert result == list(range(100, 120, 2))
+
+
 @gen_cluster(client=True)
 def test_compute_retries(c, s, a, b):
     args = [ZeroDivisionError("one"), ZeroDivisionError("two"), 3]

From 5c027a6fe08e387362ab4e59264b80b652d22377 Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Sat, 18 Apr 2020 17:09:10 +0200
Subject: [PATCH 0796/1550] Fix flaky test_oversubscribing_leases (#3726)

* Add get_value method to semaphore

* Introduce refresh_leases to control lease refreshing
---
 distributed/semaphore.py            | 36 ++++++++++-------
 distributed/tests/test_semaphore.py | 61 ++++++++++++++++++++---------
 2 files changed, 64 insertions(+), 33 deletions(-)

diff --git a/distributed/semaphore.py b/distributed/semaphore.py
index 0727c279a3a..976f54704c4 100644
--- a/distributed/semaphore.py
+++ b/distributed/semaphore.py
@@ -60,6 +60,7 @@ def __init__(self, scheduler):
                 "semaphore_release": self.release,
                 "semaphore_close": self.close,
                 "semaphore_refresh_leases": self.refresh_leases,
+                "semaphore_value": self.get_value,
             }
         )
 
@@ -79,6 +80,9 @@ def __init__(self, scheduler):
             dask.config.get("distributed.scheduler.locks.lease-timeout"), default="s",
         )
 
+    async def get_value(self, comm=None, name=None):
+        return len(self.leases[name])
+
     # `comm` here is required by the handler interface
     def create(self, comm=None, name=None, max_leases=None):
         # We use `self.max_leases` as the point of truth to find out if a semaphore with a specific
@@ -102,7 +106,7 @@ def refresh_leases(self, comm=None, name=None, lease_ids=None):
             for id_ in lease_ids:
                 if id_ not in self.leases[name]:
                     logger.critical(
-                        f"Trying to refresh an unknown lease ID {id_} for {name}. This might be due to leases "
+                        f"Refreshing an unknown lease ID {id_} for {name}. This might be due to leases "
                         f"timing out and may cause overbooking of the semaphore!"
                         f"This is often caused by long-running GIL-holding in the task which acquired the lease."
                     )
@@ -349,6 +353,7 @@ def __init__(self, max_leases=1, name=None, client=None):
         self._periodic_callback_name = f"refresh_semaphores_{self.id}"
         self.client._periodic_callbacks[self._periodic_callback_name] = pc
         pc.start()
+        self.refresh_leases = True
 
     def __await__(self):
         async def create_semaphore():
@@ -358,19 +363,16 @@ async def create_semaphore():
         return create_semaphore().__await__()
 
     async def _refresh_leases(self):
-        if self.client.scheduler is not None and not self._refreshing_leases:
-            self._refreshing_leases = True
-            if self._leases:
-                logger.debug(
-                    "%s refreshing leases for %s with IDs %s",
-                    self.client.id,
-                    self.name,
-                    self._leases,
-                )
-                await self.client.scheduler.semaphore_refresh_leases(
-                    lease_ids=list(self._leases), name=self.name
-                )
-            self._refreshing_leases = False
+        if self.refresh_leases and self._leases:
+            logger.debug(
+                "%s refreshing leases for %s with IDs %s",
+                self.client.id,
+                self.name,
+                self._leases,
+            )
+            await self.client.scheduler.semaphore_refresh_leases(
+                lease_ids=list(self._leases), name=self.name
+            )
 
     async def _acquire(self, timeout=None):
         lease_id = uuid.uuid4().hex
@@ -417,6 +419,12 @@ def release(self):
             self.client.scheduler.semaphore_release, name=self.name, lease_id=lease_id,
         )
 
+    def get_value(self):
+        """
+        Return the number of currently registered leases.
+        """
+        return self.client.sync(self.client.scheduler.semaphore_value, name=self.name)
+
     def __enter__(self):
         self.acquire()
         return self
diff --git a/distributed/tests/test_semaphore.py b/distributed/tests/test_semaphore.py
index 2f08fe45751..7a36431042f 100644
--- a/distributed/tests/test_semaphore.py
+++ b/distributed/tests/test_semaphore.py
@@ -2,7 +2,7 @@
 import dask
 import pytest
 from dask.distributed import Client
-
+from time import sleep
 from distributed import Semaphore
 from distributed.comm import Comm
 from distributed.core import ConnectionPool
@@ -10,6 +10,8 @@
 from distributed.utils_test import (  # noqa: F401
     client,
     cluster,
+    async_wait_for,
+    captured_logger,
     cluster_fixture,
     gen_cluster,
     slowidentity,
@@ -316,7 +318,7 @@ async def test_retry_acquire(c, s, a, b):
     client=True,
     config={
         "distributed.scheduler.locks.lease-timeout": "100ms",
-        "distributed.scheduler.locks.lease-validation-interval": "10ms",
+        "distributed.scheduler.locks.lease-validation-interval": "100ms",
     },
 )
 async def test_oversubscribing_leases(c, s, a, b):
@@ -346,6 +348,7 @@ async def test_oversubscribing_leases(c, s, a, b):
     # metadata check with a sleep loop is not elegant but practical.
     await c.set_metadata("release", False)
     sem = await Semaphore()
+    sem.refresh_callback.stop()
 
     def guaranteed_lease_timeout(x, sem):
         """
@@ -356,7 +359,7 @@ def guaranteed_lease_timeout(x, sem):
         all leases will eventually timeout. The function will only
         release/return once the "Event" is set, i.e. our observer is done.
         """
-        sem.refresh_callback.stop()
+        sem.refresh_leases = False
         client = get_client()
 
         with sem:
@@ -364,14 +367,17 @@ def guaranteed_lease_timeout(x, sem):
             # lease-timeout. This is twice the lease timeout to ensurre that the
             # leases are actually timed out
             slowidentity(delay=0.2)
-            old_value = client.set_metadata(x, "locked")
 
+            assert sem._leases
             # Now the GIL is free again, i.e. we enable the callback again
-            sem.refresh_callback.start()
+            sem.refresh_leases = True
+            sleep(0.1)
 
             # This is the poormans Event.wait()
-            while not client.get_metadata("release"):
-                slowidentity(delay=0.02)
+            while client.get_metadata("release") is not True:
+                sleep(0.05)
+
+            assert sem.get_value() >= 1
             return x
 
     def observe_state(sem):
@@ -382,22 +388,17 @@ def observe_state(sem):
         try to acquire and hopefully fail showing that the semaphore is
         protected if the oversubscription is recognized.
         """
-        client = get_client()
-        x_locked = False
-        y_locked = False
+        sem.refresh_callback.stop()
         # We wait until we're in an oversubscribed state, i.e. both tasks
         # are executed although there should only be one allowed
-        while not x_locked and y_locked:
-            slowidentity(delay=0.005)
-            x_locked = client.get_metadata(0) == "locked"
-            y_locked = client.get_metadata(1) == "locked"
-
-        # Once both are locked we should give the refresh time to notify the scheduler
-        # This parameter should be larger than ``lease-validation-interval``
-        slowidentity(delay=0.15)
+        while not sem.get_value() > 1:
+            sleep(0.2)
+
         # Once we're in an oversubscribed state, we must not be able to
         # acquire a lease.
         assert not sem.acquire(timeout=0)
+
+        client = get_client()
         client.set_metadata("release", True)
 
     observer = await Worker(s.address)
@@ -407,8 +408,18 @@ def observe_state(sem):
     )
     fut_observe = c.submit(observe_state, sem=sem, workers=[observer.address])
 
-    payload, observer = await c.gather([futures, fut_observe])
+    with captured_logger("distributed.semaphore") as caplog:
+        payload, observer = await c.gather([futures, fut_observe])
+
+    logs = caplog.getvalue().split("\n")
+    timeouts = [log for log in logs if "timed out" in log]
+    refresh_unknown = [log for log in logs if "Refreshing an unknown lease ID" in log]
+    assert len(timeouts) == 2
+    assert len(refresh_unknown) == 2
+
     assert sorted(payload) == [0, 1]
+    # Back to normal
+    assert await sem.get_value() == 0
 
 
 @gen_cluster(client=True,)
@@ -422,3 +433,15 @@ async def test_timeout_zero(c, s, a, b):
     assert await sem.acquire(timeout=0)
     assert not await sem.acquire(timeout=0)
     await sem.release()
+
+
+@gen_cluster(client=True)
+async def test_getvalue(c, s, a, b):
+
+    sem = await Semaphore()
+
+    assert await sem.get_value() == 0
+    await sem.acquire()
+    assert await sem.get_value() == 1
+    await sem.release()
+    assert await sem.get_value() == 0

From 9622b8f9bef1855412e9b23265378e2da1f47f2f Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Sat, 18 Apr 2020 18:06:06 +0100
Subject: [PATCH 0797/1550] Replace gen.coroutine with async-await in tests
 (#3706)

---
 .gitignore                                    |    1 +
 distributed/cli/tests/test_dask_scheduler.py  |   17 +-
 distributed/cli/utils.py                      |    6 +-
 distributed/client.py                         |    5 +-
 distributed/comm/tests/test_ucx.py            |    4 +-
 .../dashboard/tests/test_components.py        |   13 +-
 .../dashboard/tests/test_scheduler_bokeh.py   |  181 +-
 .../dashboard/tests/test_worker_bokeh.py      |   40 +-
 distributed/deploy/tests/test_adaptive.py     |   76 +-
 distributed/deploy/tests/test_local.py        |   22 +-
 .../diagnostics/tests/test_eventstream.py     |   20 +-
 .../diagnostics/tests/test_graph_layout.py    |   30 +-
 .../diagnostics/tests/test_progress.py        |   64 +-
 .../diagnostics/tests/test_progress_stream.py |   10 +-
 .../diagnostics/tests/test_progressbar.py     |    6 +-
 .../tests/test_scheduler_plugin.py            |   18 +-
 .../diagnostics/tests/test_task_stream.py     |   28 +-
 distributed/diagnostics/tests/test_widgets.py |   34 +-
 .../diagnostics/tests/test_worker_plugin.py   |   22 +-
 .../http/worker/tests/test_worker_http.py     |   12 +-
 distributed/node.py                           |    2 +-
 distributed/protocol/tests/test_arrow.py      |    6 +-
 distributed/protocol/tests/test_h5py.py       |    8 +-
 distributed/protocol/tests/test_netcdf4.py    |    4 +-
 distributed/protocol/tests/test_numpy.py      |    4 +-
 distributed/protocol/tests/test_serialize.py  |   44 +-
 distributed/tests/test_actor.py               |  196 +-
 distributed/tests/test_as_completed.py        |   49 +-
 distributed/tests/test_asyncprocess.py        |  106 +-
 distributed/tests/test_client.py              | 1875 ++++++++---------
 distributed/tests/test_collections.py         |   30 +-
 distributed/tests/test_core.py                |    2 +-
 distributed/tests/test_failed_workers.py      |  160 +-
 distributed/tests/test_locks.py               |   48 +-
 distributed/tests/test_nanny.py               |  181 +-
 distributed/tests/test_priorities.py          |   19 +-
 distributed/tests/test_publish.py             |  116 +-
 distributed/tests/test_pubsub.py              |   33 +-
 distributed/tests/test_queues.py              |  177 +-
 distributed/tests/test_resources.py           |  107 +-
 distributed/tests/test_scheduler.py           |  603 +++---
 distributed/tests/test_security.py            |   47 +-
 distributed/tests/test_semaphore.py           |    6 +-
 distributed/tests/test_steal.py               |  231 +-
 distributed/tests/test_stress.py              |   64 +-
 distributed/tests/test_tls_functional.py      |   72 +-
 distributed/tests/test_utils.py               |   42 +-
 distributed/tests/test_utils_test.py          |   38 +-
 distributed/tests/test_variable.py            |   99 +-
 distributed/tests/test_worker.py              |  499 +++--
 distributed/tests/test_worker_client.py       |   74 +-
 distributed/utils.py                          |    1 +
 distributed/utils_test.py                     |   54 +-
 distributed/worker.py                         |    1 +
 docs/source/asynchronous.rst                  |   29 +-
 55 files changed, 2764 insertions(+), 2872 deletions(-)

diff --git a/.gitignore b/.gitignore
index 86ee425adff..cf6732eaa70 100644
--- a/.gitignore
+++ b/.gitignore
@@ -17,3 +17,4 @@ dask-worker-space/
 *.swp
 .ycm_extra_conf.py
 tags
+.ipynb_checkpoints
diff --git a/distributed/cli/tests/test_dask_scheduler.py b/distributed/cli/tests/test_dask_scheduler.py
index 3e867b1f377..6f4129514b9 100644
--- a/distributed/cli/tests/test_dask_scheduler.py
+++ b/distributed/cli/tests/test_dask_scheduler.py
@@ -10,7 +10,6 @@
 import tempfile
 from time import sleep
 
-from tornado import gen
 from click.testing import CliRunner
 
 import distributed
@@ -29,12 +28,9 @@
 def test_defaults(loop):
     with popen(["dask-scheduler", "--no-dashboard"]) as proc:
 
-        @gen.coroutine
-        def f():
+        async def f():
             # Default behaviour is to listen on all addresses
-            yield [
-                assert_can_connect_from_everywhere_4_6(8786, timeout=5.0)
-            ]  # main port
+            await assert_can_connect_from_everywhere_4_6(8786, timeout=5.0)
 
         with Client("127.0.0.1:%d" % Scheduler.default_port, loop=loop) as c:
             c.sync(f)
@@ -49,12 +45,9 @@ def f():
 def test_hostport(loop):
     with popen(["dask-scheduler", "--no-dashboard", "--host", "127.0.0.1:8978"]):
 
-        @gen.coroutine
-        def f():
-            yield [
-                # The scheduler's main port can't be contacted from the outside
-                assert_can_connect_locally_4(8978, timeout=5.0)
-            ]
+        async def f():
+            # The scheduler's main port can't be contacted from the outside
+            await assert_can_connect_locally_4(8978, timeout=5.0)
 
         with Client("127.0.0.1:8978", loop=loop) as c:
             assert len(c.nthreads()) == 0
diff --git a/distributed/cli/utils.py b/distributed/cli/utils.py
index 4cfb41abe0f..c1bff051534 100644
--- a/distributed/cli/utils.py
+++ b/distributed/cli/utils.py
@@ -1,4 +1,3 @@
-from tornado import gen
 from tornado.ioloop import IOLoop
 
 
@@ -51,11 +50,10 @@ def install_signal_handlers(loop=None, cleanup=None):
     old_handlers = {}
 
     def handle_signal(sig, frame):
-        @gen.coroutine
-        def cleanup_and_stop():
+        async def cleanup_and_stop():
             try:
                 if cleanup is not None:
-                    yield cleanup(sig)
+                    await cleanup(sig)
             finally:
                 loop.stop()
 
diff --git a/distributed/client.py b/distributed/client.py
index 607ca9f5b48..5ba05a84a3b 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -102,7 +102,6 @@ def _get_global_client():
             return c
         else:
             del _global_clients[k]
-    del L
     return None
 
 
@@ -1339,6 +1338,10 @@ def close(self, timeout=no_default):
             timeout = self._timeout * 2
         # XXX handling of self.status here is not thread-safe
         if self.status == "closed":
+            if self.asynchronous:
+                future = asyncio.Future()
+                future.set_result(None)
+                return future
             return
         self.status = "closing"
 
diff --git a/distributed/comm/tests/test_ucx.py b/distributed/comm/tests/test_ucx.py
index 9ac97deeb7e..7e3cb61e375 100644
--- a/distributed/comm/tests/test_ucx.py
+++ b/distributed/comm/tests/test_ucx.py
@@ -151,13 +151,13 @@ async def test_ping_pong_data():
 
 
 @gen_test()
-def test_ucx_deserialize():
+async def test_ucx_deserialize():
     # Note we see this error on some systems with this test:
     # `socket.gaierror: [Errno -5] No address associated with hostname`
     # This may be due to a system configuration issue.
     from .test_comms import check_deserialize
 
-    yield check_deserialize("tcp://")
+    await check_deserialize("tcp://")
 
 
 @pytest.mark.asyncio
diff --git a/distributed/dashboard/tests/test_components.py b/distributed/dashboard/tests/test_components.py
index 3e6a696cc6b..a3e444e17e6 100644
--- a/distributed/dashboard/tests/test_components.py
+++ b/distributed/dashboard/tests/test_components.py
@@ -1,9 +1,10 @@
+import asyncio
+
 import pytest
 
 pytest.importorskip("bokeh")
 
 from bokeh.models import ColumnDataSource, Model
-from tornado import gen
 
 from distributed.utils_test import slowinc, gen_cluster
 from distributed.dashboard.components.shared import (
@@ -21,16 +22,16 @@ def test_basic(Component):
 
 
 @gen_cluster(client=True, clean_kwargs={"threads": False})
-def test_profile_plot(c, s, a, b):
+async def test_profile_plot(c, s, a, b):
     p = ProfilePlot()
     assert not p.source.data["left"]
-    yield c.map(slowinc, range(10), delay=0.05)
+    await c.gather(c.map(slowinc, range(10), delay=0.05))
     p.update(a.profile_recent)
     assert len(p.source.data["left"]) >= 1
 
 
 @gen_cluster(client=True, clean_kwargs={"threads": False})
-def test_profile_time_plot(c, s, a, b):
+async def test_profile_time_plot(c, s, a, b):
     from bokeh.io import curdoc
 
     sp = ProfileTimePlot(s, doc=curdoc())
@@ -42,7 +43,7 @@ def test_profile_time_plot(c, s, a, b):
     assert len(sp.source.data["left"]) <= 1
     assert len(ap.source.data["left"]) <= 1
 
-    yield c.map(slowinc, range(10), delay=0.05)
+    await c.gather(c.map(slowinc, range(10), delay=0.05))
     ap.trigger_update()
     sp.trigger_update()
-    yield gen.sleep(0.05)
+    await asyncio.sleep(0.05)
diff --git a/distributed/dashboard/tests/test_scheduler_bokeh.py b/distributed/dashboard/tests/test_scheduler_bokeh.py
index 49bdfe448bf..8ed1bb0f8a1 100644
--- a/distributed/dashboard/tests/test_scheduler_bokeh.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh.py
@@ -1,3 +1,4 @@
+import asyncio
 import json
 import re
 import ssl
@@ -8,7 +9,6 @@
 
 pytest.importorskip("bokeh")
 from tlz import first
-from tornado import gen
 from tornado.httpclient import AsyncHTTPClient, HTTPRequest
 
 import dask
@@ -35,31 +35,26 @@
     ProfileServer,
     MemoryByKey,
 )
-from distributed.utils_test import async_wait_for
-
 from distributed.dashboard import scheduler
 
 scheduler.PROFILING = False
 
 
-@pytest.mark.skipif(
-    sys.version_info[0] == 2, reason="https://github.com/bokeh/bokeh/issues/5494"
-)
 @gen_cluster(client=True, scheduler_kwargs={"dashboard": True})
-def test_simple(c, s, a, b):
+async def test_simple(c, s, a, b):
     port = s.http_server.port
 
     future = c.submit(sleep, 1)
-    yield gen.sleep(0.1)
+    await asyncio.sleep(0.1)
 
     http_client = AsyncHTTPClient()
     for suffix in applications:
-        response = yield http_client.fetch("http://localhost:%d%s" % (port, suffix))
+        response = await http_client.fetch("http://localhost:%d%s" % (port, suffix))
         body = response.body.decode()
         assert "bokeh" in body.lower()
         assert not re.search("href=./", body)  # no absolute links
 
-    response = yield http_client.fetch(
+    response = await http_client.fetch(
         "http://localhost:%d/individual-plots.json" % port
     )
     response = json.loads(response.body.decode())
@@ -67,7 +62,7 @@ def test_simple(c, s, a, b):
 
 
 @gen_cluster(client=True, worker_kwargs={"dashboard": True})
-def test_basic(c, s, a, b):
+async def test_basic(c, s, a, b):
     for component in [TaskStream, SystemMonitor, Occupancy, StealingTimeSeries]:
         ss = component(s)
 
@@ -79,24 +74,24 @@ def test_basic(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_counters(c, s, a, b):
+async def test_counters(c, s, a, b):
     pytest.importorskip("crick")
     while "tick-duration" not in s.digests:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
     ss = Counters(s)
 
     ss.update()
-    yield gen.sleep(0.1)
+    await asyncio.sleep(0.1)
     ss.update()
 
     start = time()
     while not len(ss.digest_sources["tick-duration"][0].data["x"]):
-        yield gen.sleep(1)
+        await asyncio.sleep(1)
         assert time() < start + 5
 
 
 @gen_cluster(client=True)
-def test_stealing_events(c, s, a, b):
+async def test_stealing_events(c, s, a, b):
     se = StealingEvents(s)
 
     futures = c.map(
@@ -104,7 +99,7 @@ def test_stealing_events(c, s, a, b):
     )
 
     while not b.task_state:  # will steal soon
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
     se.update()
 
@@ -112,7 +107,7 @@ def test_stealing_events(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_events(c, s, a, b):
+async def test_events(c, s, a, b):
     e = Events(s, "all")
 
     futures = c.map(
@@ -120,7 +115,7 @@ def test_events(c, s, a, b):
     )
 
     while not b.task_state:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
     e.update()
     d = dict(e.source.data)
@@ -128,12 +123,12 @@ def test_events(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_task_stream(c, s, a, b):
+async def test_task_stream(c, s, a, b):
     ts = TaskStream(s)
 
     futures = c.map(slowinc, range(10), delay=0.001)
 
-    yield wait(futures)
+    await wait(futures)
 
     ts.update()
     d = dict(ts.source.data)
@@ -146,7 +141,7 @@ def test_task_stream(c, s, a, b):
     assert all(len(L) == 10 for L in d.values())
 
     total = c.submit(sum, futures)
-    yield wait(total)
+    await wait(total)
 
     ts.update()
     d = dict(ts.source.data)
@@ -154,21 +149,21 @@ def test_task_stream(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_task_stream_n_rectangles(c, s, a, b):
+async def test_task_stream_n_rectangles(c, s, a, b):
     ts = TaskStream(s, n_rectangles=10)
     futures = c.map(slowinc, range(10), delay=0.001)
-    yield wait(futures)
+    await wait(futures)
     ts.update()
 
     assert len(ts.source.data["start"]) == 10
 
 
 @gen_cluster(client=True)
-def test_task_stream_second_plugin(c, s, a, b):
+async def test_task_stream_second_plugin(c, s, a, b):
     ts = TaskStream(s, n_rectangles=10, clear_interval=10)
     ts.update()
     futures = c.map(inc, range(10))
-    yield wait(futures)
+    await wait(futures)
     ts.update()
 
     ts2 = TaskStream(s, n_rectangles=5, clear_interval=10)
@@ -176,21 +171,21 @@ def test_task_stream_second_plugin(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_task_stream_clear_interval(c, s, a, b):
+async def test_task_stream_clear_interval(c, s, a, b):
     ts = TaskStream(s, clear_interval=200)
 
-    yield wait(c.map(inc, range(10)))
+    await wait(c.map(inc, range(10)))
     ts.update()
-    yield gen.sleep(0.010)
-    yield wait(c.map(dec, range(10)))
+    await asyncio.sleep(0.010)
+    await wait(c.map(dec, range(10)))
     ts.update()
 
     assert len(set(map(len, ts.source.data.values()))) == 1
     assert ts.source.data["name"].count("inc") == 10
     assert ts.source.data["name"].count("dec") == 10
 
-    yield gen.sleep(0.300)
-    yield wait(c.map(inc, range(10, 20)))
+    await asyncio.sleep(0.300)
+    await wait(c.map(inc, range(10, 20)))
     ts.update()
 
     assert len(set(map(len, ts.source.data.values()))) == 1
@@ -199,11 +194,11 @@ def test_task_stream_clear_interval(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_TaskProgress(c, s, a, b):
+async def test_TaskProgress(c, s, a, b):
     tp = TaskProgress(s)
 
     futures = c.map(slowinc, range(10), delay=0.001)
-    yield wait(futures)
+    await wait(futures)
 
     tp.update()
     d = dict(tp.source.data)
@@ -211,7 +206,7 @@ def test_TaskProgress(c, s, a, b):
     assert d["name"] == ["slowinc"]
 
     futures2 = c.map(dec, range(5))
-    yield wait(futures2)
+    await wait(futures2)
 
     tp.update()
     d = dict(tp.source.data)
@@ -221,35 +216,35 @@ def test_TaskProgress(c, s, a, b):
     del futures, futures2
 
     while s.tasks:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
     tp.update()
     assert not tp.source.data["all"]
 
 
 @gen_cluster(client=True)
-def test_TaskProgress_empty(c, s, a, b):
+async def test_TaskProgress_empty(c, s, a, b):
     tp = TaskProgress(s)
     tp.update()
 
     futures = [c.submit(inc, i, key="f-" + "a" * i) for i in range(20)]
-    yield wait(futures)
+    await wait(futures)
     tp.update()
 
     del futures
     while s.tasks:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
     tp.update()
 
     assert not any(len(v) for v in tp.source.data.values())
 
 
 @gen_cluster(client=True)
-def test_CurrentLoad(c, s, a, b):
+async def test_CurrentLoad(c, s, a, b):
     cl = CurrentLoad(s)
 
     futures = c.map(slowinc, range(10), delay=0.001)
-    yield wait(futures)
+    await wait(futures)
 
     cl.update()
     d = dict(cl.source.data)
@@ -261,34 +256,34 @@ def test_CurrentLoad(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_ProcessingHistogram(c, s, a, b):
+async def test_ProcessingHistogram(c, s, a, b):
     ph = ProcessingHistogram(s)
     ph.update()
     assert (ph.source.data["top"] != 0).sum() == 1
 
     futures = c.map(slowinc, range(10), delay=0.050)
     while not s.tasks:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
     ph.update()
     assert ph.source.data["right"][-1] > 2
 
 
 @gen_cluster(client=True)
-def test_NBytesHistogram(c, s, a, b):
+async def test_NBytesHistogram(c, s, a, b):
     nh = NBytesHistogram(s)
     nh.update()
     assert (nh.source.data["top"] != 0).sum() == 1
 
     futures = c.map(inc, range(10))
-    yield wait(futures)
+    await wait(futures)
 
     nh.update()
     assert nh.source.data["right"][-1] > 5 * 20
 
 
 @gen_cluster(client=True)
-def test_WorkerTable(c, s, a, b):
+async def test_WorkerTable(c, s, a, b):
     wt = WorkerTable(s)
     wt.update()
     assert all(wt.source.data.values())
@@ -307,7 +302,7 @@ def test_WorkerTable(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_WorkerTable_custom_metrics(c, s, a, b):
+async def test_WorkerTable_custom_metrics(c, s, a, b):
     def metric_port(worker):
         return worker.port
 
@@ -320,7 +315,7 @@ def metric_address(worker):
         for name, func in metrics.items():
             w.metrics[name] = func
 
-    yield [a.heartbeat(), b.heartbeat()]
+    await asyncio.gather(a.heartbeat(), b.heartbeat())
 
     for w in [a, b]:
         assert s.workers[w.address].metrics["metric_port"] == w.port
@@ -341,13 +336,13 @@ def metric_address(worker):
 
 
 @gen_cluster(client=True)
-def test_WorkerTable_different_metrics(c, s, a, b):
+async def test_WorkerTable_different_metrics(c, s, a, b):
     def metric_port(worker):
         return worker.port
 
     a.metrics["metric_a"] = metric_port
     b.metrics["metric_b"] = metric_port
-    yield [a.heartbeat(), b.heartbeat()]
+    await asyncio.gather(a.heartbeat(), b.heartbeat())
 
     assert s.workers[a.address].metrics["metric_a"] == a.port
     assert s.workers[b.address].metrics["metric_b"] == b.port
@@ -366,12 +361,12 @@ def metric_port(worker):
 
 
 @gen_cluster(client=True)
-def test_WorkerTable_metrics_with_different_metric_2(c, s, a, b):
+async def test_WorkerTable_metrics_with_different_metric_2(c, s, a, b):
     def metric_port(worker):
         return worker.port
 
     a.metrics["metric_a"] = metric_port
-    yield [a.heartbeat(), b.heartbeat()]
+    await asyncio.gather(a.heartbeat(), b.heartbeat())
 
     wt = WorkerTable(s)
     wt.update()
@@ -385,13 +380,13 @@ def metric_port(worker):
 
 
 @gen_cluster(client=True, worker_kwargs={"metrics": {"my_port": lambda w: w.port}})
-def test_WorkerTable_add_and_remove_metrics(c, s, a, b):
+async def test_WorkerTable_add_and_remove_metrics(c, s, a, b):
     def metric_port(worker):
         return worker.port
 
     a.metrics["metric_a"] = metric_port
     b.metrics["metric_b"] = metric_port
-    yield [a.heartbeat(), b.heartbeat()]
+    await asyncio.gather(a.heartbeat(), b.heartbeat())
 
     assert s.workers[a.address].metrics["metric_a"] == a.port
     assert s.workers[b.address].metrics["metric_b"] == b.port
@@ -403,14 +398,14 @@ def metric_port(worker):
 
     # Remove 'metric_b' from worker b
     del b.metrics["metric_b"]
-    yield [a.heartbeat(), b.heartbeat()]
+    await asyncio.gather(a.heartbeat(), b.heartbeat())
 
     wt = WorkerTable(s)
     wt.update()
     assert "metric_a" in wt.source.data
 
     del a.metrics["metric_a"]
-    yield [a.heartbeat(), b.heartbeat()]
+    await asyncio.gather(a.heartbeat(), b.heartbeat())
 
     wt = WorkerTable(s)
     wt.update()
@@ -418,14 +413,14 @@ def metric_port(worker):
 
 
 @gen_cluster(client=True)
-def test_WorkerTable_custom_metric_overlap_with_core_metric(c, s, a, b):
+async def test_WorkerTable_custom_metric_overlap_with_core_metric(c, s, a, b):
     def metric(worker):
         return -999
 
     a.metrics["executing"] = metric
     a.metrics["cpu"] = metric
     a.metrics["metric"] = metric
-    yield [a.heartbeat(), b.heartbeat()]
+    await asyncio.gather(a.heartbeat(), b.heartbeat())
 
     assert s.workers[a.address].metrics["executing"] != -999
     assert s.workers[a.address].metrics["cpu"] != -999
@@ -433,11 +428,11 @@ def metric(worker):
 
 
 @gen_cluster(client=True)
-def test_TaskGraph(c, s, a, b):
+async def test_TaskGraph(c, s, a, b):
     gp = TaskGraph(s)
     futures = c.map(inc, range(5))
     total = c.submit(sum, futures)
-    yield total
+    await total
 
     gp.update()
     assert set(map(len, gp.node_source.data.values())) == {6}
@@ -449,22 +444,22 @@ def test_TaskGraph(c, s, a, b):
     x = da.random.random((20, 20), chunks=(10, 10)).persist()
     y = (x + x.T) - x.mean(axis=0)
     y = y.persist()
-    yield wait(y)
+    await wait(y)
 
     gp.update()
     gp.update()
 
-    yield c.compute((x + y).sum())
+    await c.compute((x + y).sum())
 
     gp.update()
 
     future = c.submit(inc, 10)
     future2 = c.submit(inc, future)
-    yield wait(future2)
+    await wait(future2)
     key = future.key
     del future, future2
     while key in s.tasks:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
     assert "memory" in gp.node_source.data["state"]
 
@@ -475,25 +470,25 @@ def test_TaskGraph(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_TaskGraph_clear(c, s, a, b):
+async def test_TaskGraph_clear(c, s, a, b):
     gp = TaskGraph(s)
     futures = c.map(inc, range(5))
     total = c.submit(sum, futures)
-    yield total
+    await total
 
     gp.update()
 
     del total, futures
 
     while s.tasks:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
     gp.update()
     gp.update()
 
     start = time()
     while any(gp.node_source.data.values()) or any(gp.edge_source.data.values()):
-        yield gen.sleep(0.1)
+        await asyncio.sleep(0.1)
         gp.update()
         assert time() < start + 5
 
@@ -501,49 +496,43 @@ def test_TaskGraph_clear(c, s, a, b):
 @gen_cluster(
     client=True, config={"distributed.dashboard.graph-max-items": 2,},
 )
-def test_TaskGraph_limit(c, s, a, b):
+async def test_TaskGraph_limit(c, s, a, b):
     gp = TaskGraph(s)
 
     def func(x):
         return x
 
     f1 = c.submit(func, 1)
-    yield wait(f1)
+    await wait(f1)
     gp.update()
     assert len(gp.node_source.data["x"]) == 1
     f2 = c.submit(func, 2)
-    yield wait(f2)
+    await wait(f2)
     gp.update()
     assert len(gp.node_source.data["x"]) == 2
     f3 = c.submit(func, 3)
-    yield wait(f3)
+    await wait(f3)
     gp.update()
     assert len(gp.node_source.data["x"]) == 2
-    del f1
-    del f2
-    del f3
-    _ = c.submit(func, 1)
-
-    async_wait_for(lambda: len(gp.node_source.data["x"]) == 1, timeout=1)
 
 
 @gen_cluster(client=True, timeout=30)
-def test_TaskGraph_complex(c, s, a, b):
+async def test_TaskGraph_complex(c, s, a, b):
     da = pytest.importorskip("dask.array")
     gp = TaskGraph(s)
     x = da.random.random((2000, 2000), chunks=(1000, 1000))
     y = ((x + x.T) - x.mean(axis=0)).persist()
-    yield wait(y)
+    await wait(y)
     gp.update()
     assert len(gp.layout.index) == len(gp.node_source.data["x"])
     assert len(gp.layout.index) == len(s.tasks)
     z = (x - y).sum().persist()
-    yield wait(z)
+    await wait(z)
     gp.update()
     assert len(gp.layout.index) == len(gp.node_source.data["x"])
     assert len(gp.layout.index) == len(s.tasks)
     del z
-    yield gen.sleep(0.2)
+    await asyncio.sleep(0.2)
     gp.update()
     assert len(gp.layout.index) == sum(
         v == "True" for v in gp.node_source.data["visible"]
@@ -559,10 +548,10 @@ def test_TaskGraph_complex(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_TaskGraph_order(c, s, a, b):
+async def test_TaskGraph_order(c, s, a, b):
     x = c.submit(inc, 1)
     y = c.submit(div, 1, 0)
-    yield wait(y)
+    await wait(y)
 
     gp = TaskGraph(s)
     gp.update()
@@ -577,12 +566,12 @@ def test_TaskGraph_order(c, s, a, b):
         "distributed.worker.profile.cycle": "50ms",
     },
 )
-def test_profile_server(c, s, a, b):
+async def test_profile_server(c, s, a, b):
     ptp = ProfileServer(s)
     start = time()
-    yield gen.sleep(0.100)
+    await asyncio.sleep(0.100)
     while len(ptp.ts_source.data["time"]) < 2:
-        yield gen.sleep(0.100)
+        await asyncio.sleep(0.100)
         ptp.trigger_update()
         assert time() < start + 2
 
@@ -590,9 +579,9 @@ def test_profile_server(c, s, a, b):
 @gen_cluster(
     client=True, scheduler_kwargs={"dashboard": True},
 )
-def test_root_redirect(c, s, a, b):
+async def test_root_redirect(c, s, a, b):
     http_client = AsyncHTTPClient()
-    response = yield http_client.fetch("http://localhost:%d/" % s.http_server.port)
+    response = await http_client.fetch("http://localhost:%d/" % s.http_server.port)
     assert response.code == 200
     assert "/status" in response.effective_url
 
@@ -603,7 +592,7 @@ def test_root_redirect(c, s, a, b):
     worker_kwargs={"dashboard": True},
     timeout=180,
 )
-def test_proxy_to_workers(c, s, a, b):
+async def test_proxy_to_workers(c, s, a, b):
     try:
         import jupyter_server_proxy  # noqa: F401
 
@@ -613,7 +602,7 @@ def test_proxy_to_workers(c, s, a, b):
 
     dashboard_port = s.http_server.port
     http_client = AsyncHTTPClient()
-    response = yield http_client.fetch("http://localhost:%d/" % dashboard_port)
+    response = await http_client.fetch("http://localhost:%d/" % dashboard_port)
     assert response.code == 200
     assert "/status" in response.effective_url
 
@@ -627,8 +616,8 @@ def test_proxy_to_workers(c, s, a, b):
         )
         direct_url = "http://localhost:%s/status" % port
         http_client = AsyncHTTPClient()
-        response_proxy = yield http_client.fetch(proxy_url)
-        response_direct = yield http_client.fetch(direct_url)
+        response_proxy = await http_client.fetch(proxy_url)
+        response_direct = await http_client.fetch(direct_url)
 
         assert response_proxy.code == 200
         if proxy_exists:
@@ -676,7 +665,7 @@ async def test_lots_of_tasks(c, s, a, b):
         "distributed.scheduler.dashboard.tls.ca-file": get_cert("tls-ca-cert.pem"),
     },
 )
-def test_https_support(c, s, a, b):
+async def test_https_support(c, s, a, b):
     port = s.http_server.port
 
     assert (
@@ -687,7 +676,7 @@ def test_https_support(c, s, a, b):
     ctx.load_verify_locations(get_cert("tls-ca-cert.pem"))
 
     http_client = AsyncHTTPClient()
-    response = yield http_client.fetch(
+    response = await http_client.fetch(
         "https://localhost:%d/individual-plots.json" % port, ssl_options=ctx
     )
     response = json.loads(response.body.decode())
@@ -704,7 +693,7 @@ def test_https_support(c, s, a, b):
         req = HTTPRequest(
             url="https://localhost:%d/%s" % (port, suffix), ssl_options=ctx
         )
-        response = yield http_client.fetch(req)
+        response = await http_client.fetch(req)
         assert response.code < 300
         body = response.body.decode()
         assert not re.search("href=./", body)  # no absolute links
diff --git a/distributed/dashboard/tests/test_worker_bokeh.py b/distributed/dashboard/tests/test_worker_bokeh.py
index 873cc1c1f3e..47ac89c6b0a 100644
--- a/distributed/dashboard/tests/test_worker_bokeh.py
+++ b/distributed/dashboard/tests/test_worker_bokeh.py
@@ -1,12 +1,12 @@
-from operator import add, sub
+import asyncio
 import re
+from operator import add, sub
 from time import sleep
 
 import pytest
 
 pytest.importorskip("bokeh")
 from tlz import first
-from tornado import gen
 from tornado.httpclient import AsyncHTTPClient
 
 from distributed.client import wait
@@ -28,20 +28,20 @@
     worker_kwargs={"dashboard": True},
     scheduler_kwargs={"dashboard": True},
 )
-def test_routes(c, s, a, b):
+async def test_routes(c, s, a, b):
     port = a.http_server.port
 
     future = c.submit(sleep, 1)
-    yield gen.sleep(0.1)
+    await asyncio.sleep(0.1)
 
     http_client = AsyncHTTPClient()
     for suffix in ["status", "counters", "system", "profile", "profile-server"]:
-        response = yield http_client.fetch("http://localhost:%d/%s" % (port, suffix))
+        response = await http_client.fetch("http://localhost:%d/%s" % (port, suffix))
         body = response.body.decode()
         assert "bokeh" in body.lower()
         assert not re.search("href=./", body)  # no absolute links
 
-    response = yield http_client.fetch(
+    response = await http_client.fetch(
         "http://localhost:%d/info/main/workers.html" % s.http_server.port
     )
 
@@ -49,16 +49,16 @@ def test_routes(c, s, a, b):
 
 
 @gen_cluster(client=True, worker_kwargs={"dashboard": True})
-def test_simple(c, s, a, b):
+async def test_simple(c, s, a, b):
     assert s.workers[a.address].services == {"dashboard": a.http_server.port}
     assert s.workers[b.address].services == {"dashboard": b.http_server.port}
 
     future = c.submit(sleep, 1)
-    yield gen.sleep(0.1)
+    await asyncio.sleep(0.1)
 
     http_client = AsyncHTTPClient()
     for suffix in ["crossfilter", "system"]:
-        response = yield http_client.fetch(
+        response = await http_client.fetch(
             "http://localhost:%d/%s" % (a.http_server.port, suffix)
         )
         assert "bokeh" in response.body.decode().lower()
@@ -67,12 +67,12 @@ def test_simple(c, s, a, b):
 @gen_cluster(
     client=True, worker_kwargs={"dashboard": True},
 )
-def test_services_kwargs(c, s, a, b):
+async def test_services_kwargs(c, s, a, b):
     assert s.workers[a.address].services == {"dashboard": a.http_server.port}
 
 
 @gen_cluster(client=True)
-def test_basic(c, s, a, b):
+async def test_basic(c, s, a, b):
     for component in [
         StateTable,
         ExecutingTimeSeries,
@@ -92,7 +92,7 @@ def slowall(*args):
 
         x = c.submit(slowall, xs, ys, 1, workers=a.address)
         y = c.submit(slowall, xs, ys, 2, workers=b.address)
-        yield gen.sleep(0.1)
+        await asyncio.sleep(0.1)
 
         aa.update()
         bb.update()
@@ -103,19 +103,19 @@ def slowall(*args):
 
 
 @gen_cluster(client=True)
-def test_counters(c, s, a, b):
+async def test_counters(c, s, a, b):
     pytest.importorskip("crick")
     while "tick-duration" not in a.digests:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
     aa = Counters(a)
 
     aa.update()
-    yield gen.sleep(0.1)
+    await asyncio.sleep(0.1)
     aa.update()
 
     start = time()
     while not len(aa.digest_sources["tick-duration"][0].data["x"]):
-        yield gen.sleep(1)
+        await asyncio.sleep(1)
         assert time() < start + 5
 
     a.digests["foo"].add(1)
@@ -134,7 +134,7 @@ def test_counters(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_CommunicatingStream(c, s, a, b):
+async def test_CommunicatingStream(c, s, a, b):
     aa = CommunicatingStream(a)
     bb = CommunicatingStream(b)
 
@@ -143,7 +143,7 @@ def test_CommunicatingStream(c, s, a, b):
     adds = c.map(add, xs, ys, workers=a.address)
     subs = c.map(sub, xs, ys, workers=b.address)
 
-    yield wait([adds, subs])
+    await wait([adds, subs])
 
     aa.update()
     bb.update()
@@ -159,12 +159,12 @@ def test_CommunicatingStream(c, s, a, b):
 @gen_cluster(
     client=True, clean_kwargs={"threads": False}, worker_kwargs={"dashboard": True},
 )
-def test_prometheus(c, s, a, b):
+async def test_prometheus(c, s, a, b):
     pytest.importorskip("prometheus_client")
 
     http_client = AsyncHTTPClient()
     for suffix in ["metrics"]:
-        response = yield http_client.fetch(
+        response = await http_client.fetch(
             "http://localhost:%d/%s" % (a.http_server.port, suffix)
         )
         assert response.code == 200
diff --git a/distributed/deploy/tests/test_adaptive.py b/distributed/deploy/tests/test_adaptive.py
index 7fb91292540..651717aeee4 100644
--- a/distributed/deploy/tests/test_adaptive.py
+++ b/distributed/deploy/tests/test_adaptive.py
@@ -1,10 +1,10 @@
+import asyncio
 import gc
 import math
 from time import sleep
 
 import dask
 import pytest
-from tornado import gen
 
 from distributed import Client, wait, Adaptive, LocalCluster, SpecCluster, Worker
 from distributed.utils_test import gen_test, slowinc, clean
@@ -40,13 +40,13 @@ def scale_down(self, workers):
                 future = c.map(slowinc, [1, 1, 1], key=["a-4", "b-4", "c-1"])
 
                 while len(s.rprocessing) < 3:
-                    await gen.sleep(0.001)
+                    await asyncio.sleep(0.001)
 
                 ta = cluster.adapt(
                     interval="100 ms", scale_factor=2, Adaptive=TestAdaptive
                 )
 
-                await gen.sleep(0.3)
+                await asyncio.sleep(0.3)
 
 
 def test_adaptive_local_cluster(loop):
@@ -91,7 +91,7 @@ async def test_adaptive_local_cluster_multi_workers(cleanup):
 
             start = time()
             while not cluster.scheduler.workers:
-                await gen.sleep(0.01)
+                await asyncio.sleep(0.01)
                 assert time() < start + 15, adapt.log
 
             await c.gather(futures)
@@ -100,13 +100,13 @@ async def test_adaptive_local_cluster_multi_workers(cleanup):
             start = time()
             # while cluster.workers:
             while cluster.scheduler.workers:
-                await gen.sleep(0.01)
+                await asyncio.sleep(0.01)
                 assert time() < start + 15, adapt.log
 
             # no workers for a while
             for i in range(10):
                 assert not cluster.scheduler.workers
-                await gen.sleep(0.05)
+                await asyncio.sleep(0.05)
 
             futures = c.map(slowinc, range(100), delay=0.01)
             await c.gather(futures)
@@ -136,7 +136,7 @@ def scale_up(self, n, **kwargs):
         ta = cluster.adapt(
             min_size=2, interval=0.1, scale_factor=2, Adaptive=TestAdaptive
         )
-        await gen.sleep(0.3)
+        await asyncio.sleep(0.3)
 
         # Assert that adaptive cycle does not reduce cluster below minimum size
         # as determined via override.
@@ -144,8 +144,8 @@ def scale_up(self, n, **kwargs):
 
 
 @gen_test()
-def test_min_max():
-    cluster = yield LocalCluster(
+async def test_min_max():
+    cluster = await LocalCluster(
         0,
         scheduler_port=0,
         silence_logs=False,
@@ -156,14 +156,14 @@ def test_min_max():
     )
     try:
         adapt = cluster.adapt(minimum=1, maximum=2, interval="20 ms", wait_count=10)
-        c = yield Client(cluster, asynchronous=True)
+        c = await Client(cluster, asynchronous=True)
 
         start = time()
         while not cluster.scheduler.workers:
-            yield gen.sleep(0.01)
+            await asyncio.sleep(0.01)
             assert time() < start + 1
 
-        yield gen.sleep(0.2)
+        await asyncio.sleep(0.2)
         assert len(cluster.scheduler.workers) == 1
         assert len(adapt.log) == 1 and adapt.log[-1][1] == {"status": "up", "n": 1}
 
@@ -171,11 +171,11 @@ def test_min_max():
 
         start = time()
         while len(cluster.scheduler.workers) < 2:
-            yield gen.sleep(0.01)
+            await asyncio.sleep(0.01)
             assert time() < start + 1
 
         assert len(cluster.scheduler.workers) == 2
-        yield gen.sleep(0.5)
+        await asyncio.sleep(0.5)
         assert len(cluster.scheduler.workers) == 2
         assert len(cluster.workers) == 2
         assert len(adapt.log) == 2 and all(d["status"] == "up" for _, d in adapt.log)
@@ -185,12 +185,12 @@ def test_min_max():
 
         start = time()
         while len(cluster.scheduler.workers) != 1:
-            yield gen.sleep(0.01)
+            await asyncio.sleep(0.01)
             assert time() < start + 2
         assert adapt.log[-1][1]["status"] == "down"
     finally:
-        yield c.close()
-        yield cluster.close()
+        await c.close()
+        await cluster.close()
 
 
 @pytest.mark.asyncio
@@ -213,19 +213,19 @@ async def test_avoid_churn(cleanup):
 
             for i in range(10):
                 await client.submit(slowinc, i, delay=0.040)
-                await gen.sleep(0.040)
+                await asyncio.sleep(0.040)
 
             assert len(adapt.log) == 1
 
 
-@gen_test(timeout=None)
-def test_adapt_quickly():
+@pytest.mark.asyncio
+async def test_adapt_quickly():
     """ We want to avoid creating and deleting workers frequently
 
     Instead we want to wait a few beats before removing a worker in case the
     user is taking a brief pause between work
     """
-    cluster = yield LocalCluster(
+    cluster = await LocalCluster(
         0,
         asynchronous=True,
         processes=False,
@@ -233,46 +233,46 @@ def test_adapt_quickly():
         silence_logs=False,
         dashboard_address=None,
     )
-    client = yield Client(cluster, asynchronous=True)
+    client = await Client(cluster, asynchronous=True)
     adapt = cluster.adapt(interval="20 ms", wait_count=5, maximum=10)
     try:
         future = client.submit(slowinc, 1, delay=0.100)
-        yield wait(future)
+        await wait(future)
         assert len(adapt.log) == 1
 
         # Scale up when there is plenty of available work
         futures = client.map(slowinc, range(1000), delay=0.100)
         while len(adapt.log) == 1:
-            yield gen.sleep(0.01)
+            await asyncio.sleep(0.01)
         assert len(adapt.log) == 2
         assert adapt.log[-1][1]["status"] == "up"
         d = [x for x in adapt.log[-1] if isinstance(x, dict)][0]
         assert 2 < d["n"] <= adapt.maximum
 
         while len(cluster.workers) < adapt.maximum:
-            yield gen.sleep(0.01)
+            await asyncio.sleep(0.01)
 
         del futures
 
         while len(cluster.scheduler.tasks) > 1:
-            yield gen.sleep(0.01)
+            await asyncio.sleep(0.01)
 
-        yield cluster
+        await cluster
 
         while len(cluster.scheduler.workers) > 1 or len(cluster.worker_spec) > 1:
-            yield gen.sleep(0.01)
+            await asyncio.sleep(0.01)
 
         # Don't scale up for large sequential computations
-        x = yield client.scatter(1)
+        x = await client.scatter(1)
         log = list(cluster._adaptive.log)
         for i in range(100):
             x = client.submit(slowinc, x)
 
-        yield gen.sleep(0.1)
+        await asyncio.sleep(0.1)
         assert len(cluster.workers) == 1
     finally:
-        yield client.close()
-        yield cluster.close()
+        await client.close()
+        await cluster.close()
 
 
 @gen_test(timeout=None)
@@ -291,13 +291,13 @@ async def test_adapt_down():
 
             futures = client.map(slowinc, range(1000), delay=0.1)
             while len(cluster.scheduler.workers) < 5:
-                await gen.sleep(0.1)
+                await asyncio.sleep(0.1)
 
             cluster.adapt(maximum=2)
 
             start = time()
             while len(cluster.scheduler.workers) != 2:
-                await gen.sleep(0.1)
+                await asyncio.sleep(0.1)
                 assert time() < start + 1
 
 
@@ -335,7 +335,7 @@ def test_basic_no_loop(loop):
             loop.add_callback(loop.stop)
 
 
-@gen_test(timeout=None)
+@pytest.mark.asyncio
 async def test_target_duration():
     """ Ensure that redefining adapt with a lower maximum removes workers """
     with dask.config.set(
@@ -352,12 +352,12 @@ async def test_target_duration():
             adapt = cluster.adapt(interval="20ms", minimum=2, target_duration="5s")
             async with Client(cluster, asynchronous=True) as client:
                 while len(cluster.scheduler.workers) < 2:
-                    await gen.sleep(0.01)
+                    await asyncio.sleep(0.01)
 
                 futures = client.map(slowinc, range(100), delay=0.3)
 
                 while len(adapt.log) < 2:
-                    await gen.sleep(0.01)
+                    await asyncio.sleep(0.01)
 
                 assert adapt.log[0][1] == {"status": "up", "n": 2}
                 assert adapt.log[1][1] == {"status": "up", "n": 20}
@@ -385,7 +385,7 @@ def key(ws):
         await adaptive.adapt()
 
         while len(cluster.scheduler.workers) == 4:
-            await gen.sleep(0.01)
+            await asyncio.sleep(0.01)
 
         names = {ws.name for ws in cluster.scheduler.workers.values()}
         assert names == {"a-1", "a-2"} or names == {"b-1", "b-2"}
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index 31fbcebd3b8..94a6016dd2a 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -10,7 +10,6 @@
 from distutils.version import LooseVersion
 
 from tornado.ioloop import IOLoop
-from tornado import gen
 import tornado
 from tornado.httpclient import AsyncHTTPClient
 import pytest
@@ -761,13 +760,13 @@ def test_local_tls(loop, temporary):
 
 
 @gen_test()
-def test_scale_retires_workers():
+async def test_scale_retires_workers():
     class MyCluster(LocalCluster):
         def scale_down(self, *args, **kwargs):
             pass
 
     loop = IOLoop.current()
-    cluster = yield MyCluster(
+    cluster = await MyCluster(
         0,
         scheduler_port=0,
         processes=False,
@@ -776,26 +775,26 @@ def scale_down(self, *args, **kwargs):
         loop=loop,
         asynchronous=True,
     )
-    c = yield Client(cluster, asynchronous=True)
+    c = await Client(cluster, asynchronous=True)
 
     assert not cluster.workers
 
-    yield cluster.scale(2)
+    await cluster.scale(2)
 
     start = time()
     while len(cluster.scheduler.workers) != 2:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 3
 
-    yield cluster.scale(1)
+    await cluster.scale(1)
 
     start = time()
     while len(cluster.scheduler.workers) != 1:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 3
 
-    yield c.close()
-    yield cluster.close()
+    await c.close()
+    await cluster.close()
 
 
 def test_local_tls_restart(loop):
@@ -844,8 +843,7 @@ def test_asynchronous_property(loop):
         loop=loop,
     ) as cluster:
 
-        @gen.coroutine
-        def _():
+        async def _():
             assert cluster.asynchronous
 
         cluster.sync(_)
diff --git a/distributed/diagnostics/tests/test_eventstream.py b/distributed/diagnostics/tests/test_eventstream.py
index a111220b39e..4af97799893 100644
--- a/distributed/diagnostics/tests/test_eventstream.py
+++ b/distributed/diagnostics/tests/test_eventstream.py
@@ -1,7 +1,7 @@
+import asyncio
 import collections
 
 import pytest
-from tornado import gen
 
 from distributed.client import wait
 from distributed.diagnostics.eventstream import EventStream, eventstream
@@ -10,7 +10,7 @@
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 3)
-def test_eventstream(c, s, *workers):
+async def test_eventstream(c, s, *workers):
     pytest.importorskip("bokeh")
 
     es = EventStream()
@@ -19,8 +19,8 @@ def test_eventstream(c, s, *workers):
 
     futures = c.map(div, [1] * 10, range(10))
     total = c.submit(sum, futures[1:])
-    yield wait(total)
-    yield wait(futures)
+    await wait(total)
+    await wait(futures)
 
     assert len(es.buffer) == 11
 
@@ -43,13 +43,13 @@ def test_eventstream(c, s, *workers):
 
 
 @gen_cluster(client=True)
-def test_eventstream_remote(c, s, a, b):
+async def test_eventstream_remote(c, s, a, b):
     base_plugins = len(s.plugins)
-    comm = yield eventstream(s.address, interval=0.010)
+    comm = await eventstream(s.address, interval=0.010)
 
     start = time()
     while len(s.plugins) == base_plugins:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 5
 
     futures = c.map(div, [1] * 10, range(10))
@@ -57,13 +57,13 @@ def test_eventstream_remote(c, s, a, b):
     start = time()
     total = []
     while len(total) < 10:
-        msgs = yield comm.read()
+        msgs = await comm.read()
         assert isinstance(msgs, tuple)
         total.extend(msgs)
         assert time() < start + 5
 
-    yield comm.close()
+    await comm.close()
     start = time()
     while len(s.plugins) > base_plugins:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 5
diff --git a/distributed/diagnostics/tests/test_graph_layout.py b/distributed/diagnostics/tests/test_graph_layout.py
index fc8fba8d028..b63311f8432 100644
--- a/distributed/diagnostics/tests/test_graph_layout.py
+++ b/distributed/diagnostics/tests/test_graph_layout.py
@@ -1,18 +1,18 @@
+import asyncio
 import operator
 
 from distributed.utils_test import gen_cluster, inc
 from distributed.diagnostics import GraphLayout
 from distributed import wait
-from tornado import gen
 
 
 @gen_cluster(client=True)
-def test_basic(c, s, a, b):
+async def test_basic(c, s, a, b):
     gl = GraphLayout(s)
     futures = c.map(inc, range(5))
     total = c.submit(sum, futures)
 
-    yield total
+    await total
 
     assert len(gl.x) == len(gl.y) == 6
     assert all(gl.x[f.key] == 0 for f in futures)
@@ -21,11 +21,11 @@ def test_basic(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_construct_after_call(c, s, a, b):
+async def test_construct_after_call(c, s, a, b):
     futures = c.map(inc, range(5))
     total = c.submit(sum, futures)
 
-    yield total
+    await total
 
     gl = GraphLayout(s)
 
@@ -36,13 +36,13 @@ def test_construct_after_call(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_states(c, s, a, b):
+async def test_states(c, s, a, b):
     gl = GraphLayout(s)
     futures = c.map(inc, range(5))
     total = c.submit(sum, futures)
     del futures
 
-    yield total
+    await total
 
     updates = {state for idx, state in gl.state_updates}
     assert "memory" in updates
@@ -51,31 +51,31 @@ def test_states(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_release_tasks(c, s, a, b):
+async def test_release_tasks(c, s, a, b):
     gl = GraphLayout(s)
     futures = c.map(inc, range(5))
     total = c.submit(sum, futures)
 
-    yield total
+    await total
     key = total.key
     del total
     while key in s.tasks:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
     assert len(gl.visible_updates) == 1
     assert len(gl.visible_edge_updates) == 5
 
 
 @gen_cluster(client=True)
-def test_forget(c, s, a, b):
+async def test_forget(c, s, a, b):
     gl = GraphLayout(s)
 
     futures = c.map(inc, range(10))
     futures = c.map(inc, futures)
-    yield wait(futures)
+    await wait(futures)
     del futures
     while s.tasks:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
     assert not gl.x
     assert not gl.y
@@ -85,12 +85,12 @@ def test_forget(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_unique_positions(c, s, a, b):
+async def test_unique_positions(c, s, a, b):
     gl = GraphLayout(s)
 
     x = c.submit(inc, 1)
     ys = [c.submit(operator.add, x, i) for i in range(5)]
-    yield wait(ys)
+    await wait(ys)
 
     y_positions = [(gl.x[k], gl.y[k]) for k in gl.x]
     assert len(y_positions) == len(set(y_positions))
diff --git a/distributed/diagnostics/tests/test_progress.py b/distributed/diagnostics/tests/test_progress.py
index 8e3ba1688cc..871dcb0c5a5 100644
--- a/distributed/diagnostics/tests/test_progress.py
+++ b/distributed/diagnostics/tests/test_progress.py
@@ -1,6 +1,6 @@
-import pytest
+import asyncio
 
-from tornado import gen
+import pytest
 
 from distributed import Nanny
 from distributed.client import wait
@@ -29,24 +29,24 @@ def h(*args):
 
 @nodebug
 @gen_cluster(client=True)
-def test_many_Progress(c, s, a, b):
+async def test_many_Progress(c, s, a, b):
     x = c.submit(f, 1)
     y = c.submit(g, x)
     z = c.submit(h, y)
 
-    bars = [Progress(keys=[z], scheduler=s) for i in range(10)]
-    yield [bar.setup() for bar in bars]
+    bars = [Progress(keys=[z], scheduler=s) for _ in range(10)]
+    await asyncio.gather(*(bar.setup() for bar in bars))
 
-    yield z
+    await z
 
     start = time()
     while not all(b.status == "finished" for b in bars):
-        yield gen.sleep(0.1)
+        await asyncio.sleep(0.1)
         assert time() < start + 5
 
 
 @gen_cluster(client=True)
-def test_multiprogress(c, s, a, b):
+async def test_multiprogress(c, s, a, b):
     x1 = c.submit(f, 1)
     x2 = c.submit(f, x1)
     x3 = c.submit(f, x2)
@@ -54,18 +54,18 @@ def test_multiprogress(c, s, a, b):
     y2 = c.submit(g, y1)
 
     p = MultiProgress([y2], scheduler=s, complete=True)
-    yield p.setup()
+    await p.setup()
 
     assert p.all_keys == {
         "f": {f.key for f in [x1, x2, x3]},
         "g": {f.key for f in [y1, y2]},
     }
 
-    yield x3
+    await x3
 
     assert p.keys["f"] == set()
 
-    yield y2
+    await y2
 
     assert p.keys == {"f": set(), "g": set()}
 
@@ -73,7 +73,7 @@ def test_multiprogress(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_robust_to_bad_plugin(c, s, a, b):
+async def test_robust_to_bad_plugin(c, s, a, b):
     class Bad(SchedulerPlugin):
         def transition(self, key, start, finish, **kwargs):
             raise Exception()
@@ -83,7 +83,7 @@ def transition(self, key, start, finish, **kwargs):
 
     x = c.submit(inc, 1)
     y = c.submit(inc, x)
-    result = yield y
+    result = await y
     assert result == 3
 
 
@@ -95,11 +95,11 @@ def check_bar_completed(capsys, width=40):
 
 
 @gen_cluster(client=True, Worker=Nanny, timeout=None)
-def test_AllProgress(c, s, a, b):
+async def test_AllProgress(c, s, a, b):
     x, y, z = c.map(inc, [1, 2, 3])
     xx, yy, zz = c.map(dec, [x, y, z])
 
-    yield wait([x, y, z])
+    await wait([x, y, z])
     p = AllProgress(s)
     assert p.all["inc"] == {x.key, y.key, z.key}
     assert p.state["memory"]["inc"] == {x.key, y.key, z.key}
@@ -109,7 +109,7 @@ def test_AllProgress(c, s, a, b):
     assert isinstance(p.nbytes["inc"], int)
     assert p.nbytes["inc"] > 0
 
-    yield wait([xx, yy, zz])
+    await wait([xx, yy, zz])
     assert p.all["dec"] == {xx.key, yy.key, zz.key}
     assert p.state["memory"]["dec"] == {xx.key, yy.key, zz.key}
     assert p.state["released"] == {}
@@ -117,7 +117,7 @@ def test_AllProgress(c, s, a, b):
     assert p.nbytes["inc"] == p.nbytes["dec"]
 
     t = c.submit(sum, [x, y, z])
-    yield t
+    await t
 
     keys = {x.key, y.key, z.key}
     del x, y, z
@@ -126,7 +126,7 @@ def test_AllProgress(c, s, a, b):
     gc.collect()
 
     while any(k in s.who_has for k in keys):
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
     assert p.state["released"]["inc"] == keys
     assert p.all["inc"] == keys
@@ -135,7 +135,7 @@ def test_AllProgress(c, s, a, b):
         assert p.nbytes["inc"] == 0
 
     xxx = c.submit(div, 1, 0)
-    yield wait([xxx])
+    await wait([xxx])
     assert p.state["erred"] == {"div": {xxx.key}}
 
     tkey = t.key
@@ -145,7 +145,7 @@ def test_AllProgress(c, s, a, b):
     gc.collect()
 
     while tkey in s.tasks:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
     for coll in [p.all, p.nbytes] + list(p.state.values()):
         assert "inc" not in coll
@@ -160,47 +160,47 @@ def f(x):
 
     gc.collect()
 
-    yield gen.sleep(1)
+    await asyncio.sleep(1)
 
-    yield wait([future])
+    await wait([future])
     assert p.state["memory"] == {"f": {future.key}}
 
-    yield c._restart()
+    await c._restart()
 
     for coll in [p.all] + list(p.state.values()):
         assert not coll
 
     x = c.submit(div, 1, 2)
-    yield wait([x])
+    await wait([x])
     assert set(p.all) == {"div"}
     assert all(set(d) == {"div"} for d in p.state.values())
 
 
 @gen_cluster(client=True, Worker=Nanny)
-def test_AllProgress_lost_key(c, s, a, b, timeout=None):
+async def test_AllProgress_lost_key(c, s, a, b, timeout=None):
     p = AllProgress(s)
     futures = c.map(inc, range(5))
-    yield wait(futures)
+    await wait(futures)
     assert len(p.state["memory"]["inc"]) == 5
 
-    yield a.close()
-    yield b.close()
+    await a.close()
+    await b.close()
 
     start = time()
     while len(p.state["memory"]["inc"]) > 0:
-        yield gen.sleep(0.1)
+        await asyncio.sleep(0.1)
         assert time() < start + 5
 
 
 @gen_cluster(client=True)
-def test_GroupProgress(c, s, a, b):
+async def test_GroupProgress(c, s, a, b):
     da = pytest.importorskip("dask.array")
     fp = GroupProgress(s)
     x = da.ones(100, chunks=10)
     y = x + 1
     z = (x * y).sum().persist(optimize_graph=False)
 
-    yield wait(z)
+    await wait(z)
     assert 3 < len(fp.groups) < 10
     for k, g in fp.groups.items():
         assert fp.keys[k]
@@ -212,6 +212,6 @@ def test_GroupProgress(c, s, a, b):
 
     del x, y, z
     while s.tasks:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
     assert not fp.groups
diff --git a/distributed/diagnostics/tests/test_progress_stream.py b/distributed/diagnostics/tests/test_progress_stream.py
index 77b3922a42e..8f506b7a7bb 100644
--- a/distributed/diagnostics/tests/test_progress_stream.py
+++ b/distributed/diagnostics/tests/test_progress_stream.py
@@ -56,7 +56,7 @@ def test_progress_quads_too_many():
 
 
 @gen_cluster(client=True)
-def test_progress_stream(c, s, a, b):
+async def test_progress_stream(c, s, a, b):
     futures = c.map(div, [1] * 10, range(10))
 
     x = 1
@@ -64,10 +64,10 @@ def test_progress_stream(c, s, a, b):
         x = delayed(inc)(x)
     future = c.compute(x)
 
-    yield wait(futures + [future])
+    await wait(futures + [future])
 
-    comm = yield progress_stream(s.address, interval=0.010)
-    msg = yield comm.read()
+    comm = await progress_stream(s.address, interval=0.010)
+    msg = await comm.read()
     nbytes = msg.pop("nbytes")
     assert msg == {
         "all": {"div": 10, "inc": 5},
@@ -81,7 +81,7 @@ def test_progress_stream(c, s, a, b):
 
     assert progress_quads(msg)
 
-    yield comm.close()
+    await comm.close()
 
 
 def test_progress_quads_many_functions():
diff --git a/distributed/diagnostics/tests/test_progressbar.py b/distributed/diagnostics/tests/test_progressbar.py
index 535efd0e9e2..f19dbd2df26 100644
--- a/distributed/diagnostics/tests/test_progressbar.py
+++ b/distributed/diagnostics/tests/test_progressbar.py
@@ -25,17 +25,17 @@ def test_text_progressbar(capsys, client):
 
 
 @gen_cluster(client=True)
-def test_TextProgressBar_error(c, s, a, b):
+async def test_TextProgressBar_error(c, s, a, b):
     x = c.submit(div, 1, 0)
 
     progress = TextProgressBar([x.key], scheduler=s.address, start=False, interval=0.01)
-    yield progress.listen()
+    await progress.listen()
 
     assert progress.status == "error"
     assert progress.comm.closed()
 
     progress = TextProgressBar([x.key], scheduler=s.address, start=False, interval=0.01)
-    yield progress.listen()
+    await progress.listen()
     assert progress.status == "error"
     assert progress.comm.closed()
 
diff --git a/distributed/diagnostics/tests/test_scheduler_plugin.py b/distributed/diagnostics/tests/test_scheduler_plugin.py
index 6fc9e22f3df..31ada3f9e12 100644
--- a/distributed/diagnostics/tests/test_scheduler_plugin.py
+++ b/distributed/diagnostics/tests/test_scheduler_plugin.py
@@ -4,7 +4,7 @@
 
 
 @gen_cluster(client=True)
-def test_simple(c, s, a, b):
+async def test_simple(c, s, a, b):
     class Counter(SchedulerPlugin):
         def start(self, scheduler):
             self.scheduler = scheduler
@@ -25,7 +25,7 @@ def transition(self, key, start, finish, *args, **kwargs):
     y = c.submit(inc, x)
     z = c.submit(inc, y)
 
-    yield z
+    await z
 
     assert counter.count == 3
     s.remove_plugin(counter)
@@ -33,7 +33,7 @@ def transition(self, key, start, finish, *args, **kwargs):
 
 
 @gen_cluster(nthreads=[], client=False)
-def test_add_remove_worker(s):
+async def test_add_remove_worker(s):
     events = []
 
     class MyPlugin(SchedulerPlugin):
@@ -51,10 +51,10 @@ def remove_worker(self, worker, scheduler):
 
     a = Worker(s.address)
     b = Worker(s.address)
-    yield a
-    yield b
-    yield a.close()
-    yield b.close()
+    await a
+    await b
+    await a.close()
+    await b.close()
 
     assert events == [
         ("add_worker", a.address),
@@ -65,8 +65,8 @@ def remove_worker(self, worker, scheduler):
 
     events[:] = []
     s.remove_plugin(plugin)
-    a = yield Worker(s.address)
-    yield a.close()
+    a = await Worker(s.address)
+    await a.close()
     assert events == []
 
 
diff --git a/distributed/diagnostics/tests/test_task_stream.py b/distributed/diagnostics/tests/test_task_stream.py
index 4639c7a7a0b..4b57d18ee7a 100644
--- a/distributed/diagnostics/tests/test_task_stream.py
+++ b/distributed/diagnostics/tests/test_task_stream.py
@@ -13,13 +13,13 @@
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 3)
-def test_TaskStreamPlugin(c, s, *workers):
+async def test_TaskStreamPlugin(c, s, *workers):
     es = TaskStreamPlugin(s)
     assert not es.buffer
 
     futures = c.map(div, [1] * 10, range(10))
     total = c.submit(sum, futures[1:])
-    yield wait(total)
+    await wait(total)
 
     assert len(es.buffer) == 11
 
@@ -45,19 +45,19 @@ def test_TaskStreamPlugin(c, s, *workers):
 
 
 @gen_cluster(client=True)
-def test_maxlen(c, s, a, b):
+async def test_maxlen(c, s, a, b):
     tasks = TaskStreamPlugin(s, maxlen=5)
     futures = c.map(inc, range(10))
-    yield wait(futures)
+    await wait(futures)
     assert len(tasks.buffer) == 5
 
 
 @gen_cluster(client=True)
-def test_collect(c, s, a, b):
+async def test_collect(c, s, a, b):
     tasks = TaskStreamPlugin(s)
     start = time()
     futures = c.map(slowinc, range(10), delay=0.1)
-    yield wait(futures)
+    await wait(futures)
 
     L = tasks.collect()
     assert len(L) == len(futures)
@@ -82,15 +82,15 @@ def test_collect(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_client(c, s, a, b):
-    L = yield c.get_task_stream()
+async def test_client(c, s, a, b):
+    L = await c.get_task_stream()
     assert L == ()
 
     futures = c.map(slowinc, range(10), delay=0.1)
-    yield wait(futures)
+    await wait(futures)
 
     tasks = [p for p in s.plugins if isinstance(p, TaskStreamPlugin)][0]
-    L = yield c.get_task_stream()
+    L = await c.get_task_stream()
     assert L == tuple(tasks.buffer)
 
 
@@ -105,14 +105,14 @@ def test_client_sync(client):
 
 
 @gen_cluster(client=True)
-def test_get_task_stream_plot(c, s, a, b):
+async def test_get_task_stream_plot(c, s, a, b):
     bokeh = pytest.importorskip("bokeh")
-    yield c.get_task_stream()
+    await c.get_task_stream()
 
     futures = c.map(slowinc, range(10), delay=0.1)
-    yield wait(futures)
+    await wait(futures)
 
-    data, figure = yield c.get_task_stream(plot=True)
+    data, figure = await c.get_task_stream(plot=True)
     assert isinstance(figure, bokeh.plotting.Figure)
 
 
diff --git a/distributed/diagnostics/tests/test_widgets.py b/distributed/diagnostics/tests/test_widgets.py
index c217d17e293..6064462d893 100644
--- a/distributed/diagnostics/tests/test_widgets.py
+++ b/distributed/diagnostics/tests/test_widgets.py
@@ -88,24 +88,24 @@ def record_display(*args):
 
 
 @gen_cluster(client=True)
-def test_progressbar_widget(c, s, a, b):
+async def test_progressbar_widget(c, s, a, b):
     x = c.submit(inc, 1)
     y = c.submit(inc, x)
     z = c.submit(inc, y)
-    yield wait(z)
+    await wait(z)
 
     progress = ProgressWidget([z.key], scheduler=s.address, complete=True)
-    yield progress.listen()
+    await progress.listen()
 
     assert progress.bar.value == 1.0
     assert "3 / 3" in progress.bar_text.value
 
     progress = ProgressWidget([z.key], scheduler=s.address)
-    yield progress.listen()
+    await progress.listen()
 
 
 @gen_cluster(client=True)
-def test_multi_progressbar_widget(c, s, a, b):
+async def test_multi_progressbar_widget(c, s, a, b):
     x1 = c.submit(inc, 1)
     x2 = c.submit(inc, x1)
     x3 = c.submit(inc, x2)
@@ -113,10 +113,10 @@ def test_multi_progressbar_widget(c, s, a, b):
     y2 = c.submit(dec, y1)
     e = c.submit(throws, y2)
     other = c.submit(inc, 123)
-    yield wait([other, e])
+    await wait([other, e])
 
     p = MultiProgressWidget([e.key], scheduler=s.address, complete=True)
-    yield p.listen()
+    await p.listen()
 
     assert p.bars["inc"].value == 1.0
     assert p.bars["dec"].value == 1.0
@@ -145,7 +145,7 @@ def test_multi_progressbar_widget(c, s, a, b):
 
 
 @gen_cluster()
-def test_multi_progressbar_widget_after_close(s, a, b):
+async def test_multi_progressbar_widget_after_close(s, a, b):
     s.update_graph(
         tasks=valmap(
             dumps_task,
@@ -170,7 +170,7 @@ def test_multi_progressbar_widget_after_close(s, a, b):
     )
 
     p = MultiProgressWidget(["x-1", "x-2", "x-3"], scheduler=s.address)
-    yield p.listen()
+    await p.listen()
 
     assert "x" in p.bars
 
@@ -231,7 +231,7 @@ def test_progressbar_cancel(client):
 
 
 @gen_cluster()
-def test_multibar_complete(s, a, b):
+async def test_multibar_complete(s, a, b):
     s.update_graph(
         tasks=valmap(
             dumps_task,
@@ -256,7 +256,7 @@ def test_multibar_complete(s, a, b):
     )
 
     p = MultiProgressWidget(["e"], scheduler=s.address, complete=True)
-    yield p.listen()
+    await p.listen()
 
     assert p._last_response["all"] == {"x": 3, "y": 2, "e": 1}
     assert all(b.value == 1.0 for k, b in p.bars.items() if k != "e")
@@ -274,28 +274,28 @@ def test_fast(client):
 
 
 @gen_cluster(client=True, client_kwargs={"serializers": ["msgpack"]})
-def test_serializers(c, s, a, b):
+async def test_serializers(c, s, a, b):
     x = c.submit(inc, 1)
     y = c.submit(inc, x)
     z = c.submit(inc, y)
-    yield wait(z)
+    await wait(z)
 
     progress = ProgressWidget([z], scheduler=s.address, complete=True)
-    yield progress.listen()
+    await progress.listen()
 
     assert progress.bar.value == 1.0
     assert "3 / 3" in progress.bar_text.value
 
 
 @gen_tls_cluster(client=True)
-def test_tls(c, s, a, b):
+async def test_tls(c, s, a, b):
     x = c.submit(inc, 1)
     y = c.submit(inc, x)
     z = c.submit(inc, y)
-    yield wait(z)
+    await wait(z)
 
     progress = ProgressWidget([z], scheduler=s.address, complete=True)
-    yield progress.listen()
+    await progress.listen()
 
     assert progress.bar.value == 1.0
     assert "3 / 3" in progress.bar_text.value
diff --git a/distributed/diagnostics/tests/test_worker_plugin.py b/distributed/diagnostics/tests/test_worker_plugin.py
index b3b919d7fe2..2ee5a28c780 100644
--- a/distributed/diagnostics/tests/test_worker_plugin.py
+++ b/distributed/diagnostics/tests/test_worker_plugin.py
@@ -34,26 +34,26 @@ def transition(self, key, start, finish, **kwargs):
 
 
 @gen_cluster(client=True, nthreads=[])
-def test_create_with_client(c, s):
-    yield c.register_worker_plugin(MyPlugin(123))
+async def test_create_with_client(c, s):
+    await c.register_worker_plugin(MyPlugin(123))
 
-    worker = yield Worker(s.address, loop=s.loop)
+    worker = await Worker(s.address, loop=s.loop)
     assert worker._my_plugin_status == "setup"
     assert worker._my_plugin_data == 123
 
-    yield worker.close()
+    await worker.close()
     assert worker._my_plugin_status == "teardown"
 
 
 @gen_cluster(client=True, worker_kwargs={"plugins": [MyPlugin(5)]})
-def test_create_on_construction(c, s, a, b):
+async def test_create_on_construction(c, s, a, b):
     assert len(a.plugins) == len(b.plugins) == 1
     assert a._my_plugin_status == "setup"
     assert a._my_plugin_data == 5
 
 
 @gen_cluster(nthreads=[("127.0.0.1", 1)], client=True)
-def test_normal_task_transitions_called(c, s, w):
+async def test_normal_task_transitions_called(c, s, w):
     expected_transitions = [
         ("task", "waiting", "ready"),
         ("task", "ready", "executing"),
@@ -62,12 +62,12 @@ def test_normal_task_transitions_called(c, s, w):
 
     plugin = MyPlugin(1, expected_transitions=expected_transitions)
 
-    yield c.register_worker_plugin(plugin)
-    yield c.submit(lambda x: x, 1, key="task")
+    await c.register_worker_plugin(plugin)
+    await c.submit(lambda x: x, 1, key="task")
 
 
 @gen_cluster(nthreads=[("127.0.0.1", 1)], client=True)
-def test_failing_task_transitions_called(c, s, w):
+async def test_failing_task_transitions_called(c, s, w):
     def failing(x):
         raise Exception()
 
@@ -79,10 +79,10 @@ def failing(x):
 
     plugin = MyPlugin(1, expected_transitions=expected_transitions)
 
-    yield c.register_worker_plugin(plugin)
+    await c.register_worker_plugin(plugin)
 
     with pytest.raises(Exception):
-        yield c.submit(failing, 1, key="task")
+        await c.submit(failing, 1, key="task")
 
 
 @gen_cluster(nthreads=[("127.0.0.1", 1)], client=True)
diff --git a/distributed/http/worker/tests/test_worker_http.py b/distributed/http/worker/tests/test_worker_http.py
index 0a4135fba7f..2282a4daa66 100644
--- a/distributed/http/worker/tests/test_worker_http.py
+++ b/distributed/http/worker/tests/test_worker_http.py
@@ -5,16 +5,16 @@
 
 
 @gen_cluster(client=True)
-def test_prometheus(c, s, a, b):
+async def test_prometheus(c, s, a, b):
     pytest.importorskip("prometheus_client")
     from prometheus_client.parser import text_string_to_metric_families
 
     http_client = AsyncHTTPClient()
 
-    # request data twice since there once was a case where metrics got registered multiple times resulting in
-    # prometheus_client errors
+    # request data twice since there once was a case where metrics got registered
+    # multiple times resulting in prometheus_client errors
     for _ in range(2):
-        response = yield http_client.fetch(
+        response = await http_client.fetch(
             "http://localhost:%d/metrics" % a.http_server.port
         )
         assert response.code == 200
@@ -26,10 +26,10 @@ def test_prometheus(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_health(c, s, a, b):
+async def test_health(c, s, a, b):
     http_client = AsyncHTTPClient()
 
-    response = yield http_client.fetch(
+    response = await http_client.fetch(
         "http://localhost:%d/health" % a.http_server.port
     )
     assert response.code == 200
diff --git a/distributed/node.py b/distributed/node.py
index 11645e86317..740776bed68 100644
--- a/distributed/node.py
+++ b/distributed/node.py
@@ -3,9 +3,9 @@
 import warnings
 import weakref
 
+from tornado import gen
 from tornado.ioloop import IOLoop
 from tornado.httpserver import HTTPServer
-from tornado import gen
 import tlz
 import dask
 
diff --git a/distributed/protocol/tests/test_arrow.py b/distributed/protocol/tests/test_arrow.py
index 37aff3a2644..e86bfa6f827 100644
--- a/distributed/protocol/tests/test_arrow.py
+++ b/distributed/protocol/tests/test_arrow.py
@@ -28,10 +28,10 @@ def echo(arg):
 @pytest.mark.parametrize("obj", [batch, tbl], ids=["RecordBatch", "Table"])
 def test_scatter(obj):
     @gen_cluster(client=True)
-    def run_test(client, scheduler, worker1, worker2):
-        obj_fut = yield client.scatter(obj)
+    async def run_test(client, scheduler, worker1, worker2):
+        obj_fut = await client.scatter(obj)
         fut = client.submit(echo, obj_fut)
-        result = yield fut
+        result = await fut
         assert obj.equals(result)
 
     run_test()
diff --git a/distributed/protocol/tests/test_h5py.py b/distributed/protocol/tests/test_h5py.py
index 6bae5b3b8d5..80eeb2c05f5 100644
--- a/distributed/protocol/tests/test_h5py.py
+++ b/distributed/protocol/tests/test_h5py.py
@@ -90,7 +90,7 @@ def test_raise_error_on_serialize_write_permissions():
 
 @silence_h5py_issue775
 @gen_cluster(client=True)
-def test_h5py_serialize(c, s, a, b):
+async def test_h5py_serialize(c, s, a, b):
     from dask.utils import SerializableLock
 
     lock = SerializableLock("hdf5")
@@ -102,12 +102,12 @@ def test_h5py_serialize(c, s, a, b):
             dset = f["/group/x"]
             x = da.from_array(dset, chunks=dset.chunks, lock=lock)
             y = c.compute(x)
-            y = yield y
+            y = await y
             assert (y[:] == dset[:]).all()
 
 
 @gen_cluster(client=True)
-def test_h5py_serialize_2(c, s, a, b):
+async def test_h5py_serialize_2(c, s, a, b):
     with tmpfile() as fn:
         with h5py.File(fn, mode="a") as f:
             x = f.create_dataset("/group/x", shape=(12,), dtype="i4", chunks=(4,))
@@ -116,5 +116,5 @@ def test_h5py_serialize_2(c, s, a, b):
             dset = f["/group/x"]
             x = da.from_array(dset, chunks=(3,))
             y = c.compute(x.sum())
-            y = yield y
+            y = await y
             assert y == (1 + 2 + 3 + 4) * 3
diff --git a/distributed/protocol/tests/test_netcdf4.py b/distributed/protocol/tests/test_netcdf4.py
index f1ddcead3ef..1ed78508156 100644
--- a/distributed/protocol/tests/test_netcdf4.py
+++ b/distributed/protocol/tests/test_netcdf4.py
@@ -82,12 +82,12 @@ def test_serialize_deserialize_group():
 
 
 @gen_cluster(client=True)
-def test_netcdf4_serialize(c, s, a, b):
+async def test_netcdf4_serialize(c, s, a, b):
     with tmpfile() as fn:
         create_test_dataset(fn)
         with netCDF4.Dataset(fn, mode="r") as f:
             dset = f.variables["x"]
             x = da.from_array(dset, chunks=2)
             y = c.compute(x)
-            y = yield y
+            y = await y
             assert (y[:] == dset[:]).all()
diff --git a/distributed/protocol/tests/test_numpy.py b/distributed/protocol/tests/test_numpy.py
index 99a298d9694..08a7c2df244 100644
--- a/distributed/protocol/tests/test_numpy.py
+++ b/distributed/protocol/tests/test_numpy.py
@@ -233,9 +233,9 @@ def test_dont_compress_uncompressable_data():
 
 
 @gen_cluster(client=True, timeout=60)
-def test_dumps_large_blosc(c, s, a, b):
+async def test_dumps_large_blosc(c, s, a, b):
     x = c.submit(np.ones, BIG_BYTES_SHARD_SIZE * 2, dtype="u1")
-    result = yield x
+    await x
 
 
 @pytest.mark.skipif(sys.version_info[0] < 3, reason="numpy doesnt use memoryviews")
diff --git a/distributed/protocol/tests/test_serialize.py b/distributed/protocol/tests/test_serialize.py
index dd23e5e635d..6a5af842ddd 100644
--- a/distributed/protocol/tests/test_serialize.py
+++ b/distributed/protocol/tests/test_serialize.py
@@ -120,34 +120,34 @@ def test_nested_deserialize():
 
 
 @gen_cluster(client=True)
-def test_object_in_graph(c, s, a, b):
+async def test_object_in_graph(c, s, a, b):
     o = MyObj(123)
     v = delayed(o)
     v2 = delayed(identity)(v)
 
     future = c.compute(v2)
-    result = yield future
+    result = await future
 
     assert isinstance(result, MyObj)
     assert result.data == 123
 
 
 @gen_cluster(client=True)
-def test_scatter(c, s, a, b):
+async def test_scatter(c, s, a, b):
     o = MyObj(123)
-    [future] = yield c._scatter([o])
-    yield c._replicate(o)
-    o2 = yield c._gather(future)
+    [future] = await c._scatter([o])
+    await c._replicate(o)
+    o2 = await c._gather(future)
     assert isinstance(o2, MyObj)
     assert o2.data == 123
 
 
 @gen_cluster(client=True)
-def test_inter_worker_comms(c, s, a, b):
+async def test_inter_worker_comms(c, s, a, b):
     o = MyObj(123)
-    [future] = yield c._scatter([o], workers=a.address)
+    [future] = await c._scatter([o], workers=a.address)
     future2 = c.submit(identity, future, workers=b.address)
-    o2 = yield c._gather(future2)
+    o2 = await c._gather(future2)
     assert isinstance(o2, MyObj)
     assert o2.data == 123
 
@@ -249,14 +249,14 @@ def test_errors():
 
 
 @gen_test()
-def test_err_on_bad_deserializer():
-    frames = yield to_frames({"x": to_serialize(1234)}, serializers=["pickle"])
+async def test_err_on_bad_deserializer():
+    frames = await to_frames({"x": to_serialize(1234)}, serializers=["pickle"])
 
-    result = yield from_frames(frames, deserializers=["pickle", "foo"])
+    result = await from_frames(frames, deserializers=["pickle", "foo"])
     assert result == {"x": 1234}
 
-    with pytest.raises(TypeError) as info:
-        yield from_frames(frames, deserializers=["msgpack"])
+    with pytest.raises(TypeError):
+        await from_frames(frames, deserializers=["msgpack"])
 
 
 class MyObject:
@@ -290,7 +290,7 @@ def my_loads(header, frames):
     client_kwargs={"serializers": ["my-ser", "pickle"]},
     worker_kwargs={"serializers": ["my-ser", "pickle"]},
 )
-def test_context_specific_serialization(c, s, a, b):
+async def test_context_specific_serialization(c, s, a, b):
     register_serialization_family("my-ser", my_dumps, my_loads)
 
     try:
@@ -298,7 +298,7 @@ def test_context_specific_serialization(c, s, a, b):
         x = c.submit(MyObject, x=1, y=2, workers=a.address)
         y = c.submit(lambda x: x, x, workers=b.address)
 
-        yield wait(y)
+        await wait(y)
 
         key = y.key
 
@@ -307,11 +307,11 @@ def check(dask_worker):
             my_obj = dask_worker.data[key]
             return my_obj.context
 
-        result = yield c.run(check, workers=[b.address])
+        result = await c.run(check, workers=[b.address])
         expected = {"sender": a.address, "recipient": b.address}
         assert result[b.address]["sender"] == a.address  # see origin worker
 
-        z = yield y  # bring object to local process
+        z = await y  # bring object to local process
 
         assert z.x == 1 and z.y == 2
         assert z.context["sender"] == b.address
@@ -322,14 +322,14 @@ def check(dask_worker):
 
 
 @gen_cluster(client=True)
-def test_context_specific_serialization_class(c, s, a, b):
+async def test_context_specific_serialization_class(c, s, a, b):
     register_serialization(MyObject, my_dumps, my_loads)
 
     # Create the object on A, force communication to B
     x = c.submit(MyObject, x=1, y=2, workers=a.address)
     y = c.submit(lambda x: x, x, workers=b.address)
 
-    yield wait(y)
+    await wait(y)
 
     key = y.key
 
@@ -338,11 +338,11 @@ def check(dask_worker):
         my_obj = dask_worker.data[key]
         return my_obj.context
 
-    result = yield c.run(check, workers=[b.address])
+    result = await c.run(check, workers=[b.address])
     expected = {"sender": a.address, "recipient": b.address}
     assert result[b.address]["sender"] == a.address  # see origin worker
 
-    z = yield y  # bring object to local process
+    z = await y  # bring object to local process
 
     assert z.x == 1 and z.y == 2
     assert z.context["sender"] == b.address
diff --git a/distributed/tests/test_actor.py b/distributed/tests/test_actor.py
index de69db5685a..89233eaca24 100644
--- a/distributed/tests/test_actor.py
+++ b/distributed/tests/test_actor.py
@@ -1,6 +1,6 @@
+import asyncio
 import operator
 from time import sleep
-from tornado import gen
 
 import pytest
 
@@ -50,20 +50,20 @@ def get(self, key):
 @pytest.mark.parametrize("direct_to_workers", [True, False])
 def test_client_actions(direct_to_workers):
     @gen_cluster(client=True)
-    def test(c, s, a, b):
-        c = yield Client(
+    async def test(c, s, a, b):
+        c = await Client(
             s.address, asynchronous=True, direct_to_workers=direct_to_workers
         )
 
         counter = c.submit(Counter, workers=[a.address], actor=True)
         assert isinstance(counter, Future)
-        counter = yield counter
+        counter = await counter
         assert counter._address
         assert hasattr(counter, "increment")
         assert hasattr(counter, "add")
         assert hasattr(counter, "n")
 
-        n = yield counter.n
+        n = await counter.n
         assert n == 0
 
         assert counter._address == a.address
@@ -71,17 +71,17 @@ def test(c, s, a, b):
         assert isinstance(a.actors[counter.key], Counter)
         assert s.tasks[counter.key].actor
 
-        yield [counter.increment(), counter.increment()]
+        await asyncio.gather(counter.increment(), counter.increment())
 
-        n = yield counter.n
+        n = await counter.n
         assert n == 2
 
         counter.add(10)
-        while (yield counter.n) != 10 + 2:
-            n = yield counter.n
-            yield gen.sleep(0.01)
+        while (await counter.n) != 10 + 2:
+            n = await counter.n
+            await asyncio.sleep(0.01)
 
-        yield c.close()
+        await c.close()
 
     test()
 
@@ -89,7 +89,7 @@ def test(c, s, a, b):
 @pytest.mark.parametrize("separate_thread", [False, True])
 def test_worker_actions(separate_thread):
     @gen_cluster(client=True)
-    def test(c, s, a, b):
+    async def test(c, s, a, b):
         counter = c.submit(Counter, workers=[a.address], actor=True)
         a_address = a.address
 
@@ -106,17 +106,17 @@ def f(counter):
             assert end > start
 
         futures = [c.submit(f, counter, pure=False) for _ in range(10)]
-        yield futures
+        await c.gather(futures)
 
-        counter = yield counter
-        assert (yield counter.n) == 10
+        counter = await counter
+        assert await counter.n == 10
 
     test()
 
 
 @gen_cluster(client=True)
-def test_Actor(c, s, a, b):
-    counter = yield c.submit(Counter, actor=True)
+async def test_Actor(c, s, a, b):
+    counter = await c.submit(Counter, actor=True)
 
     assert counter._cls == Counter
 
@@ -132,22 +132,22 @@ def test_Actor(c, s, a, b):
     + "Should rely on sending small messages rather than rpc"
 )
 @gen_cluster(client=True)
-def test_linear_access(c, s, a, b):
+async def test_linear_access(c, s, a, b):
     start = time()
     future = c.submit(sleep, 0.2)
     actor = c.submit(List, actor=True, dummy=future)
-    actor = yield actor
+    actor = await actor
 
     for i in range(100):
         actor.append(i)
 
     while True:
-        yield gen.sleep(0.1)
-        L = yield actor.L
+        await asyncio.sleep(0.1)
+        L = await actor.L
         if len(L) == 100:
             break
 
-    L = yield actor.L
+    L = await actor.L
     stop = time()
     assert L == tuple(range(100))
 
@@ -155,7 +155,7 @@ def test_linear_access(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_exceptions_create(c, s, a, b):
+async def test_exceptions_create(c, s, a, b):
     class Foo:
         x = 0
 
@@ -163,62 +163,62 @@ def __init__(self):
             raise ValueError("bar")
 
     with pytest.raises(ValueError) as info:
-        future = yield c.submit(Foo, actor=True)
+        await c.submit(Foo, actor=True)
 
     assert "bar" in str(info.value)
 
 
 @gen_cluster(client=True)
-def test_exceptions_method(c, s, a, b):
+async def test_exceptions_method(c, s, a, b):
     class Foo:
         def throw(self):
             1 / 0
 
-    foo = yield c.submit(Foo, actor=True)
+    foo = await c.submit(Foo, actor=True)
     with pytest.raises(ZeroDivisionError):
-        yield foo.throw()
+        await foo.throw()
 
 
 @gen_cluster(client=True)
-def test_gc(c, s, a, b):
+async def test_gc(c, s, a, b):
     actor = c.submit(Counter, actor=True)
-    yield wait(actor)
+    await wait(actor)
     del actor
 
     while a.actors or b.actors:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
 
 @gen_cluster(client=True)
-def test_track_dependencies(c, s, a, b):
+async def test_track_dependencies(c, s, a, b):
     actor = c.submit(Counter, actor=True)
-    yield wait(actor)
+    await wait(actor)
     x = c.submit(sleep, 0.5)
     y = c.submit(lambda x, y: x, x, actor)
     del actor
 
-    yield gen.sleep(0.3)
+    await asyncio.sleep(0.3)
 
     assert a.actors or b.actors
 
 
 @gen_cluster(client=True)
-def test_future(c, s, a, b):
+async def test_future(c, s, a, b):
     counter = c.submit(Counter, actor=True, workers=[a.address])
     assert isinstance(counter, Future)
-    yield wait(counter)
+    await wait(counter)
     assert isinstance(a.actors[counter.key], Counter)
 
-    counter = yield counter
+    counter = await counter
     assert isinstance(counter, Actor)
     assert counter._address
 
-    yield gen.sleep(0.1)
+    await asyncio.sleep(0.1)
     assert counter.key in c.futures  # don't lose future
 
 
 @gen_cluster(client=True)
-def test_future_dependencies(c, s, a, b):
+async def test_future_dependencies(c, s, a, b):
     counter = c.submit(Counter, actor=True, workers=[a.address])
 
     def f(a):
@@ -226,13 +226,13 @@ def f(a):
         assert a._cls == Counter
 
     x = c.submit(f, counter, workers=[b.address])
-    yield x
+    await x
 
     assert {ts.key for ts in s.tasks[x.key].dependencies} == {counter.key}
     assert {ts.key for ts in s.tasks[counter.key].dependents} == {x.key}
 
     y = c.submit(f, counter, workers=[a.address], pure=False)
-    yield y
+    await y
 
     assert {ts.key for ts in s.tasks[y.key].dependencies} == {counter.key}
     assert {ts.key for ts in s.tasks[counter.key].dependents} == {x.key, y.key}
@@ -256,15 +256,15 @@ def test_sync(client):
 
 
 @gen_cluster(client=True, config={"distributed.comm.timeouts.connect": "1s"})
-def test_failed_worker(c, s, a, b):
+async def test_failed_worker(c, s, a, b):
     future = c.submit(Counter, actor=True, workers=[a.address])
-    yield wait(future)
-    counter = yield future
+    await wait(future)
+    counter = await future
 
-    yield a.close()
+    await a.close()
 
     with pytest.raises(Exception) as info:
-        yield counter.increment()
+        await counter.increment()
 
     assert "actor" in str(info.value).lower()
     assert "worker" in str(info.value).lower()
@@ -272,45 +272,45 @@ def test_failed_worker(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def bench(c, s, a, b):
-    counter = yield c.submit(Counter, actor=True)
+async def bench(c, s, a, b):
+    counter = await c.submit(Counter, actor=True)
 
     for i in range(1000):
-        yield counter.increment()
+        await counter.increment()
 
 
 @gen_cluster(client=True)
-def test_numpy_roundtrip(c, s, a, b):
+async def test_numpy_roundtrip(c, s, a, b):
     np = pytest.importorskip("numpy")
 
-    server = yield c.submit(ParameterServer, actor=True)
+    server = await c.submit(ParameterServer, actor=True)
 
     x = np.random.random(1000)
-    yield server.put("x", x)
+    await server.put("x", x)
 
-    y = yield server.get("x")
+    y = await server.get("x")
 
     assert (x == y).all()
 
 
 @gen_cluster(client=True)
-def test_numpy_roundtrip_getattr(c, s, a, b):
+async def test_numpy_roundtrip_getattr(c, s, a, b):
     np = pytest.importorskip("numpy")
 
-    counter = yield c.submit(Counter, actor=True)
+    counter = await c.submit(Counter, actor=True)
 
     x = np.random.random(1000)
 
-    yield counter.add(x)
+    await counter.add(x)
 
-    y = yield counter.n
+    y = await counter.n
 
     assert (x == y).all()
 
 
 @gen_cluster(client=True)
-def test_repr(c, s, a, b):
-    counter = yield c.submit(Counter, actor=True)
+async def test_repr(c, s, a, b):
+    counter = await c.submit(Counter, actor=True)
 
     assert "Counter" in repr(counter)
     assert "Actor" in repr(counter)
@@ -319,8 +319,8 @@ def test_repr(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_dir(c, s, a, b):
-    counter = yield c.submit(Counter, actor=True)
+async def test_dir(c, s, a, b):
+    counter = await c.submit(Counter, actor=True)
 
     d = set(dir(counter))
 
@@ -330,8 +330,8 @@ def test_dir(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_many_computations(c, s, a, b):
-    counter = yield c.submit(Counter, actor=True)
+async def test_many_computations(c, s, a, b):
+    counter = await c.submit(Counter, actor=True)
 
     def add(n, counter):
         for i in range(n):
@@ -342,13 +342,13 @@ def add(n, counter):
 
     while not done.done():
         assert len(s.processing) <= a.nthreads + b.nthreads
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
-    yield done
+    await done
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 5)] * 2)
-def test_thread_safety(c, s, a, b):
+async def test_thread_safety(c, s, a, b):
     class Unsafe:
         def __init__(self):
             self.n = 0
@@ -362,32 +362,32 @@ def f(self):
                 assert self.n == 1
             self.n = 0
 
-    unsafe = yield c.submit(Unsafe, actor=True)
+    unsafe = await c.submit(Unsafe, actor=True)
 
     futures = [unsafe.f() for i in range(10)]
-    yield futures
+    await c.gather(futures)
 
 
 @gen_cluster(client=True)
-def test_Actors_create_dependencies(c, s, a, b):
-    counter = yield c.submit(Counter, actor=True)
+async def test_Actors_create_dependencies(c, s, a, b):
+    counter = await c.submit(Counter, actor=True)
     future = c.submit(lambda x: None, counter)
-    yield wait(future)
+    await wait(future)
     assert s.tasks[future.key].dependencies == {s.tasks[counter.key]}
 
 
 @gen_cluster(client=True)
-def test_load_balance(c, s, a, b):
+async def test_load_balance(c, s, a, b):
     class Foo:
         def __init__(self, x):
             pass
 
     b = c.submit(operator.mul, "b", 1000000)
-    yield wait(b)
+    await wait(b)
     [ws] = s.tasks[b.key].who_has
 
-    x = yield c.submit(Foo, b, actor=True)
-    y = yield c.submit(Foo, b, actor=True)
+    x = await c.submit(Foo, b, actor=True)
+    y = await c.submit(Foo, b, actor=True)
     assert x.key != y.key  # actors assumed not pure
 
     assert s.tasks[x.key].who_has == {ws}  # first went to best match
@@ -395,28 +395,28 @@ def __init__(self, x):
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 5)
-def test_load_balance_map(c, s, *workers):
+async def test_load_balance_map(c, s, *workers):
     class Foo:
         def __init__(self, x, y=None):
             pass
 
     b = c.submit(operator.mul, "b", 1000000)
-    yield wait(b)
+    await wait(b)
 
     actors = c.map(Foo, range(10), y=b, actor=True)
-    yield wait(actors)
+    await wait(actors)
 
     assert all(len(w.actors) == 2 for w in workers)
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 4, Worker=Nanny)
-def bench_param_server(c, s, *workers):
+async def bench_param_server(c, s, *workers):
     import dask.array as da
     import numpy as np
 
     x = da.random.random((500000, 1000), chunks=(1000, 1000))
     x = x.persist()
-    yield wait(x)
+    await wait(x)
 
     class ParameterServer:
         data = None
@@ -443,17 +443,17 @@ def f(block, ps=None):
     from distributed.utils import format_time
 
     start = time()
-    ps = yield c.submit(ParameterServer, x.shape[1], actor=True)
+    ps = await c.submit(ParameterServer, x.shape[1], actor=True)
     y = x.map_blocks(f, ps=ps, dtype=x.dtype)
-    # result = yield c.compute(y.mean())
-    yield wait(y.persist())
+    # result = await c.compute(y.mean())
+    await wait(y.persist())
     end = time()
     print(format_time(end - start))
 
 
 @pytest.mark.xfail(reason="unknown")
 @gen_cluster(client=True)
-def test_compute(c, s, a, b):
+async def test_compute(c, s, a, b):
     @dask.delayed
     def f(n, counter):
         assert isinstance(counter, Actor)
@@ -468,12 +468,12 @@ def check(counter, blanks):
     values = [f(i, counter) for i in range(5)]
     final = check(counter, values)
 
-    result = yield c.compute(final, actors=counter)
+    result = await c.compute(final, actors=counter)
     assert result == 0 + 1 + 2 + 3 + 4
 
     start = time()
     while a.data or b.data:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 5
 
 
@@ -509,15 +509,15 @@ def check(dask_worker):
     nthreads=[("127.0.0.1", 1)],
     config={"distributed.worker.profile.interval": "1ms"},
 )
-def test_actors_in_profile(c, s, a):
+async def test_actors_in_profile(c, s, a):
     class Sleeper:
         def sleep(self, time):
             sleep(time)
 
-    sleeper = yield c.submit(Sleeper, actor=True)
+    sleeper = await c.submit(Sleeper, actor=True)
 
     for i in range(5):
-        yield sleeper.sleep(0.200)
+        await sleeper.sleep(0.200)
         if (
             list(a.profile_recent["children"])[0].startswith("sleep")
             or "Sleeper.sleep" in a.profile_keys
@@ -527,28 +527,26 @@ def sleep(self, time):
 
 
 @gen_cluster(client=True)
-def test_waiter(c, s, a, b):
+async def test_waiter(c, s, a, b):
     from tornado.locks import Event
 
     class Waiter:
         def __init__(self):
             self.event = Event()
 
-        @gen.coroutine
-        def set(self):
+        async def set(self):
             self.event.set()
 
-        @gen.coroutine
-        def wait(self):
-            yield self.event.wait()
+        async def wait(self):
+            await self.event.wait()
 
-    waiter = yield c.submit(Waiter, actor=True)
+    waiter = await c.submit(Waiter, actor=True)
 
-    futures = [waiter.wait() for i in range(5)]  # way more than we have actor threads
+    futures = [waiter.wait() for _ in range(5)]  # way more than we have actor threads
 
-    yield gen.sleep(0.1)
+    await asyncio.sleep(0.1)
     assert not any(future.done() for future in futures)
 
-    yield waiter.set()
+    await waiter.set()
 
-    yield futures
+    await c.gather(futures)
diff --git a/distributed/tests/test_as_completed.py b/distributed/tests/test_as_completed.py
index f71c6f7492e..ae257f9bb8e 100644
--- a/distributed/tests/test_as_completed.py
+++ b/distributed/tests/test_as_completed.py
@@ -6,7 +6,6 @@
 from time import sleep
 
 import pytest
-from tornado import gen
 
 from distributed.client import _as_completed, as_completed, _first_completed, wait
 from distributed.metrics import time
@@ -16,18 +15,18 @@
 
 
 @gen_cluster(client=True)
-def test__as_completed(c, s, a, b):
+async def test__as_completed(c, s, a, b):
     x = c.submit(inc, 1)
     y = c.submit(inc, 1)
     z = c.submit(inc, 2)
 
     q = queue.Queue()
-    yield _as_completed([x, y, z], q)
+    await _as_completed([x, y, z], q)
 
     assert q.qsize() == 3
     assert {q.get(), q.get(), q.get()} == {x, y, z}
 
-    result = yield _first_completed([x, y, z])
+    result = await _first_completed([x, y, z])
     assert result in [x, y, z]
 
 
@@ -129,11 +128,10 @@ def test_as_completed_cancel_last(client):
     x = client.submit(inc, 1)
     y = client.submit(inc, 0.3)
 
-    @gen.coroutine
-    def _():
-        yield gen.sleep(0.1)
-        yield w.cancel(asynchronous=True)
-        yield y.cancel(asynchronous=True)
+    async def _():
+        await asyncio.sleep(0.1)
+        await w.cancel(asynchronous=True)
+        await y.cancel(asynchronous=True)
 
     client.loop.add_callback(_)
 
@@ -144,32 +142,23 @@ def _():
 
 
 @gen_cluster(client=True)
-def test_async_for_py2_equivalent(c, s, a, b):
+async def test_async_for_py2_equivalent(c, s, a, b):
     futures = c.map(sleep, [0.01] * 3, pure=False)
     seq = as_completed(futures)
-    x = yield seq.__anext__()
-    y = yield seq.__anext__()
-    z = yield seq.__anext__()
-
+    x, y, z = [el async for el in seq]
     assert x.done()
     assert y.done()
     assert z.done()
     assert x.key != y.key
 
-    with pytest.raises(StopAsyncIteration):
-        yield seq.__anext__()
-
 
 @gen_cluster(client=True)
-def test_as_completed_error_async(c, s, a, b):
+async def test_as_completed_error_async(c, s, a, b):
     x = c.submit(throws, 1)
     y = c.submit(inc, 1)
 
     ac = as_completed([x, y])
-    first = yield ac.__anext__()
-    second = yield ac.__anext__()
-    result = {first, second}
-
+    result = {el async for el in ac}
     assert result == {x, y}
     assert x.status == "error"
     assert y.status == "finished"
@@ -200,17 +189,16 @@ def test_as_completed_with_results(client):
 
 
 @gen_cluster(client=True)
-def test_as_completed_with_results_async(c, s, a, b):
+async def test_as_completed_with_results_async(c, s, a, b):
     x = c.submit(throws, 1)
     y = c.submit(inc, 5)
     z = c.submit(inc, 1)
 
     ac = as_completed([x, y, z], with_results=True)
-    yield y.cancel()
+    await y.cancel()
     with pytest.raises(RuntimeError) as exc:
-        first = yield ac.__anext__()
-        second = yield ac.__anext__()
-        third = yield ac.__anext__()
+        async for _ in ac:
+            pass
     assert str(exc.value) == "hello!"
 
 
@@ -252,17 +240,14 @@ async def test_str(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_as_completed_with_results_no_raise_async(c, s, a, b):
+async def test_as_completed_with_results_no_raise_async(c, s, a, b):
     x = c.submit(throws, 1)
     y = c.submit(inc, 5)
     z = c.submit(inc, 1)
 
     ac = as_completed([x, y, z], with_results=True, raise_errors=False)
     c.loop.add_callback(y.cancel)
-    first = yield ac.__anext__()
-    second = yield ac.__anext__()
-    third = yield ac.__anext__()
-    res = [first, second, third]
+    res = [el async for el in ac]
 
     dd = {r[0]: r[1:] for r in res}
     assert set(dd.keys()) == {y, x, z}
diff --git a/distributed/tests/test_asyncprocess.py b/distributed/tests/test_asyncprocess.py
index e496b35cb90..3923d81cf2c 100644
--- a/distributed/tests/test_asyncprocess.py
+++ b/distributed/tests/test_asyncprocess.py
@@ -1,11 +1,12 @@
-from datetime import timedelta
+import asyncio
 import gc
 import os
 import signal
 import sys
 import threading
-from time import sleep
 import weakref
+from datetime import timedelta
+from time import sleep
 
 import pytest
 from tornado import gen
@@ -50,7 +51,7 @@ def threads_info(q):
 @pytest.mark.xfail(reason="Intermittent failure")
 @nodebug
 @gen_test()
-def test_simple():
+async def test_simple():
     to_child = mp_context.Queue()
     from_child = mp_context.Queue()
 
@@ -67,15 +68,15 @@ def test_simple():
 
     # join() before start()
     with pytest.raises(AssertionError):
-        yield proc.join()
+        await proc.join()
 
-    yield proc.start()
+    await proc.start()
     assert proc.is_alive()
     assert proc.pid is not None
     assert proc.exitcode is None
 
     t1 = time()
-    yield proc.join(timeout=0.02)
+    await proc.join(timeout=0.02)
     dt = time() - t1
     assert 0.2 >= dt >= 0.01
     assert proc.is_alive()
@@ -91,7 +92,7 @@ def test_simple():
 
     # child should be stopping now
     t1 = time()
-    yield proc.join(timeout=10)
+    await proc.join(timeout=10)
     dt = time() - t1
     assert dt <= 1.0
     assert not proc.is_alive()
@@ -100,7 +101,7 @@ def test_simple():
 
     # join() again
     t1 = time()
-    yield proc.join()
+    await proc.join()
     dt = time() - t1
     assert dt <= 0.6
 
@@ -133,14 +134,14 @@ def test_simple():
         pytest.fail("AsyncProcess should have been destroyed")
     t1 = time()
     while wr2() is not None:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         gc.collect()
         dt = time() - t1
         assert dt < 2.0
 
 
 @gen_test()
-def test_exitcode():
+async def test_exitcode():
     q = mp_context.Queue()
 
     proc = AsyncProcess(target=exit, kwargs={"q": q})
@@ -148,80 +149,81 @@ def test_exitcode():
     assert not proc.is_alive()
     assert proc.exitcode is None
 
-    yield proc.start()
+    await proc.start()
     assert proc.is_alive()
     assert proc.exitcode is None
 
     q.put(5)
-    yield proc.join(timeout=3.0)
+    await proc.join(timeout=3.0)
     assert not proc.is_alive()
     assert proc.exitcode == 5
 
 
 @pytest.mark.skipif(os.name == "nt", reason="POSIX only")
 @gen_test()
-def test_signal():
+async def test_signal():
     proc = AsyncProcess(target=exit_with_signal, args=(signal.SIGINT,))
     proc.daemon = True
     assert not proc.is_alive()
     assert proc.exitcode is None
 
-    yield proc.start()
-    yield proc.join(timeout=3.0)
+    await proc.start()
+    await proc.join(timeout=3.0)
 
     assert not proc.is_alive()
     # Can be 255 with forkserver, see https://bugs.python.org/issue30589
     assert proc.exitcode in (-signal.SIGINT, 255)
 
     proc = AsyncProcess(target=wait)
-    yield proc.start()
+    await proc.start()
     os.kill(proc.pid, signal.SIGTERM)
-    yield proc.join(timeout=3.0)
+    await proc.join(timeout=3.0)
 
     assert not proc.is_alive()
     assert proc.exitcode in (-signal.SIGTERM, 255)
 
 
 @gen_test()
-def test_terminate():
+async def test_terminate():
     proc = AsyncProcess(target=wait)
     proc.daemon = True
-    yield proc.start()
-    yield proc.terminate()
+    await proc.start()
+    await proc.terminate()
 
-    yield proc.join(timeout=3.0)
+    await proc.join(timeout=3.0)
     assert not proc.is_alive()
     assert proc.exitcode in (-signal.SIGTERM, 255)
 
 
 @gen_test()
-def test_close():
+async def test_close():
     proc = AsyncProcess(target=exit_now)
     proc.close()
     with pytest.raises(ValueError):
-        yield proc.start()
+        await proc.start()
 
     proc = AsyncProcess(target=exit_now)
-    yield proc.start()
+    await proc.start()
     proc.close()
     with pytest.raises(ValueError):
-        yield proc.terminate()
+        await proc.terminate()
 
     proc = AsyncProcess(target=exit_now)
-    yield proc.start()
-    yield proc.join()
+    await proc.start()
+    await proc.join()
     proc.close()
     with pytest.raises(ValueError):
-        yield proc.join()
+        await proc.join()
     proc.close()
 
 
 @gen_test()
-def test_exit_callback():
+async def test_exit_callback():
     to_child = mp_context.Queue()
     from_child = mp_context.Queue()
     evt = Event()
 
+    # FIXME: this breaks if changed to async def...
     @gen.coroutine
     def on_stop(_proc):
         assert _proc is proc
@@ -234,13 +236,13 @@ def on_stop(_proc):
     proc.set_exit_callback(on_stop)
     proc.daemon = True
 
-    yield proc.start()
-    yield gen.sleep(0.05)
+    await proc.start()
+    await asyncio.sleep(0.05)
     assert proc.is_alive()
     assert not evt.is_set()
 
     to_child.put(None)
-    yield evt.wait(timedelta(seconds=3))
+    await evt.wait(timedelta(seconds=3))
     assert evt.is_set()
     assert not proc.is_alive()
 
@@ -250,25 +252,25 @@ def on_stop(_proc):
     proc.set_exit_callback(on_stop)
     proc.daemon = True
 
-    yield proc.start()
-    yield gen.sleep(0.05)
+    await proc.start()
+    await asyncio.sleep(0.05)
     assert proc.is_alive()
     assert not evt.is_set()
 
-    yield proc.terminate()
-    yield evt.wait(timedelta(seconds=3))
+    await proc.terminate()
+    await evt.wait(timedelta(seconds=3))
     assert evt.is_set()
 
 
 @gen_test()
-def test_child_main_thread():
+async def test_child_main_thread():
     """
     The main thread in the child should be called "MainThread".
     """
     q = mp_context.Queue()
     proc = AsyncProcess(target=threads_info, args=(q,))
-    yield proc.start()
-    yield proc.join()
+    await proc.start()
+    await proc.join()
     n_threads = q.get()
     main_name = q.get()
     assert n_threads <= 3
@@ -282,38 +284,38 @@ def test_child_main_thread():
     sys.platform.startswith("win"), reason="num_fds not supported on windows"
 )
 @gen_test()
-def test_num_fds():
+async def test_num_fds():
     psutil = pytest.importorskip("psutil")
 
     # Warm up
     proc = AsyncProcess(target=exit_now)
     proc.daemon = True
-    yield proc.start()
-    yield proc.join()
+    await proc.start()
+    await proc.join()
 
     p = psutil.Process()
     before = p.num_fds()
 
     proc = AsyncProcess(target=exit_now)
     proc.daemon = True
-    yield proc.start()
-    yield proc.join()
+    await proc.start()
+    await proc.join()
     assert not proc.is_alive()
     assert proc.exitcode == 0
 
     start = time()
     while p.num_fds() > before:
-        yield gen.sleep(0.1)
+        await asyncio.sleep(0.1)
         print("fds:", before, p.num_fds())
         assert time() < start + 10
 
 
 @gen_test()
-def test_terminate_after_stop():
+async def test_terminate_after_stop():
     proc = AsyncProcess(target=sleep, args=(0,))
-    yield proc.start()
-    yield gen.sleep(0.1)
-    yield proc.terminate()
+    await proc.start()
+    await asyncio.sleep(0.1)
+    await proc.terminate()
 
 
 def _worker_process(worker_ready, child_pipe):
@@ -342,12 +344,12 @@ def _parent_process(child_pipe):
     The child_alive pipe is held open for as long as the child is alive, and can
     be used to determine if it exited correctly. """
 
-    def parent_process_coroutine():
+    async def parent_process_coroutine():
         worker_ready = mp_context.Event()
 
         worker = AsyncProcess(target=_worker_process, args=(worker_ready, child_pipe))
 
-        yield worker.start()
+        await worker.start()
 
         # Wait for the child process to have started.
         worker_ready.wait()
@@ -359,7 +361,7 @@ def parent_process_coroutine():
 
     with pristine_loop() as loop:
         try:
-            loop.run_sync(gen.coroutine(parent_process_coroutine), timeout=10)
+            loop.run_sync(parent_process_coroutine(), timeout=10)
         finally:
             loop.stop()
 
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 17aea54f78a..fd95895c84e 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -20,7 +20,6 @@
 
 import pytest
 from tlz import identity, isdistinct, concat, pluck, valmap, first, merge
-from tornado import gen
 
 import dask
 from dask import delayed
@@ -105,80 +104,79 @@
 
 
 @gen_cluster(client=True, timeout=None)
-def test_submit(c, s, a, b):
+async def test_submit(c, s, a, b):
     x = c.submit(inc, 10)
     assert not x.done()
 
     assert isinstance(x, Future)
     assert x.client is c
 
-    result = yield x
+    result = await x
     assert result == 11
     assert x.done()
 
     y = c.submit(inc, 20)
     z = c.submit(add, x, y)
 
-    result = yield z
+    result = await z
     assert result == 11 + 21
     s.validate_state()
 
 
 @gen_cluster(client=True)
-def test_map(c, s, a, b):
+async def test_map(c, s, a, b):
     L1 = c.map(inc, range(5))
     assert len(L1) == 5
     assert isdistinct(x.key for x in L1)
     assert all(isinstance(x, Future) for x in L1)
 
-    result = yield L1[0]
+    result = await L1[0]
     assert result == inc(0)
     assert len(s.tasks) == 5
 
     L2 = c.map(inc, L1)
 
-    result = yield L2[1]
+    result = await L2[1]
     assert result == inc(inc(1))
     assert len(s.tasks) == 10
     # assert L1[0].key in s.tasks[L2[0].key]
 
     total = c.submit(sum, L2)
-    result = yield total
+    result = await total
     assert result == sum(map(inc, map(inc, range(5))))
 
     L3 = c.map(add, L1, L2)
-    result = yield L3[1]
+    result = await L3[1]
     assert result == inc(1) + inc(inc(1))
 
     L4 = c.map(add, range(3), range(4))
-    results = yield c.gather(L4)
-    if sys.version_info[0] >= 3:
-        assert results == list(map(add, range(3), range(4)))
+    results = await c.gather(L4)
+    assert results == list(map(add, range(3), range(4)))
 
     def f(x, y=10):
         return x + y
 
     L5 = c.map(f, range(5), y=5)
-    results = yield c.gather(L5)
+    results = await c.gather(L5)
     assert results == list(range(5, 10))
 
     y = c.submit(f, 10)
     L6 = c.map(f, range(5), y=y)
-    results = yield c.gather(L6)
+    results = await c.gather(L6)
     assert results == list(range(20, 25))
     s.validate_state()
 
 
 @gen_cluster(client=True)
-def test_map_empty(c, s, a, b):
+async def test_map_empty(c, s, a, b):
     L1 = c.map(inc, [], pure=False)
     assert len(L1) == 0
-    results = yield c.gather(L1)
+    results = await c.gather(L1)
     assert results == []
 
 
 @gen_cluster(client=True)
-def test_map_keynames(c, s, a, b):
+async def test_map_keynames(c, s, a, b):
     futures = c.map(inc, range(4), key="INC")
     assert all(f.key.startswith("INC") for f in futures)
     assert isdistinct(f.key for f in futures)
@@ -192,7 +190,7 @@ def test_map_keynames(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_map_retries(c, s, a, b):
+async def test_map_retries(c, s, a, b):
     args = [
         [ZeroDivisionError("one"), 2, 3],
         [4, 5, 6],
@@ -200,22 +198,22 @@ def test_map_retries(c, s, a, b):
     ]
 
     x, y, z = c.map(*map_varying(args), retries=2)
-    assert (yield x) == 2
-    assert (yield y) == 4
-    assert (yield z) == 9
+    assert await x == 2
+    assert await y == 4
+    assert await z == 9
 
     x, y, z = c.map(*map_varying(args), retries=1, pure=False)
-    assert (yield x) == 2
-    assert (yield y) == 4
+    assert await x == 2
+    assert await y == 4
     with pytest.raises(ZeroDivisionError, match="eight"):
-        yield z
+        await z
 
     x, y, z = c.map(*map_varying(args), retries=0, pure=False)
     with pytest.raises(ZeroDivisionError, match="one"):
-        yield x
-    assert (yield y) == 4
+        await x
+    assert await y == 4
     with pytest.raises(ZeroDivisionError, match="seven"):
-        yield z
+        await z
 
 
 @gen_cluster(client=True)
@@ -235,25 +233,25 @@ async def test_map_batch_size(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_compute_retries(c, s, a, b):
+async def test_compute_retries(c, s, a, b):
     args = [ZeroDivisionError("one"), ZeroDivisionError("two"), 3]
 
     # Sanity check for varying() use
     x = c.compute(delayed(varying(args))())
     with pytest.raises(ZeroDivisionError, match="one"):
-        yield x
+        await x
 
     # Same retries for all
     x = c.compute(delayed(varying(args))(), retries=1)
     with pytest.raises(ZeroDivisionError, match="two"):
-        yield x
+        await x
 
     x = c.compute(delayed(varying(args))(), retries=2)
-    assert (yield x) == 3
+    assert await x == 3
 
     args.append(4)
     x = c.compute(delayed(varying(args))(), retries=2)
-    assert (yield x) == 3
+    assert await x == 3
 
     # Per-future retries
     xargs = [ZeroDivisionError("one"), ZeroDivisionError("two"), 30, 40]
@@ -264,17 +262,17 @@ def test_compute_retries(c, s, a, b):
     x, y = c.compute([x, y], retries={x: 2})
     gc.collect()
 
-    assert (yield x) == 30
+    assert await x == 30
     with pytest.raises(ZeroDivisionError, match="five"):
-        yield y
+        await y
 
     x, y, z = [delayed(varying(args))() for args in (xargs, yargs, zargs)]
     x, y, z = c.compute([x, y, z], retries={(y, z): 2})
 
     with pytest.raises(ZeroDivisionError, match="one"):
-        yield x
-    assert (yield y) == 70
-    assert (yield z) == 80
+        await x
+    assert await y == 70
+    assert await z == 80
 
 
 def test_retries_get(c):
@@ -289,43 +287,43 @@ def test_retries_get(c):
 
 
 @gen_cluster(client=True)
-def test_compute_persisted_retries(c, s, a, b):
+async def test_compute_persisted_retries(c, s, a, b):
     args = [ZeroDivisionError("one"), ZeroDivisionError("two"), 3]
 
     # Sanity check
     x = c.persist(delayed(varying(args))())
     fut = c.compute(x)
     with pytest.raises(ZeroDivisionError, match="one"):
-        yield fut
+        await fut
 
     x = c.persist(delayed(varying(args))())
     fut = c.compute(x, retries=1)
     with pytest.raises(ZeroDivisionError, match="two"):
-        yield fut
+        await fut
 
     x = c.persist(delayed(varying(args))())
     fut = c.compute(x, retries=2)
-    assert (yield fut) == 3
+    assert await fut == 3
 
     args.append(4)
     x = c.persist(delayed(varying(args))())
     fut = c.compute(x, retries=3)
-    assert (yield fut) == 3
+    assert await fut == 3
 
 
 @gen_cluster(client=True)
-def test_persist_retries(c, s, a, b):
+async def test_persist_retries(c, s, a, b):
     # Same retries for all
     args = [ZeroDivisionError("one"), ZeroDivisionError("two"), 3]
 
     x = c.persist(delayed(varying(args))(), retries=1)
     x = c.compute(x)
     with pytest.raises(ZeroDivisionError, match="two"):
-        yield x
+        await x
 
     x = c.persist(delayed(varying(args))(), retries=2)
     x = c.compute(x)
-    assert (yield x) == 3
+    assert await x == 3
 
     # Per-key retries
     xargs = [ZeroDivisionError("one"), ZeroDivisionError("two"), 30, 40]
@@ -337,17 +335,17 @@ def test_persist_retries(c, s, a, b):
     x, y, z = c.compute([x, y, z])
 
     with pytest.raises(ZeroDivisionError, match="one"):
-        yield x
-    assert (yield y) == 70
-    assert (yield z) == 80
+        await x
+    assert await y == 70
+    assert await z == 80
 
 
 @gen_cluster(client=True)
-def test_retries_dask_array(c, s, a, b):
+async def test_retries_dask_array(c, s, a, b):
     da = pytest.importorskip("dask.array")
     x = da.ones((10, 10), chunks=(3, 3))
     future = c.compute(x.sum(), retries=2)
-    y = yield future
+    y = await future
     assert y == 100
 
 
@@ -370,7 +368,7 @@ async def test_future_repr(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_future_tuple_repr(c, s, a, b):
+async def test_future_tuple_repr(c, s, a, b):
     da = pytest.importorskip("dask.array")
     y = da.arange(10, chunks=(5,)).persist()
     f = futures_of(y)[0]
@@ -380,13 +378,13 @@ def test_future_tuple_repr(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_Future_exception(c, s, a, b):
+async def test_Future_exception(c, s, a, b):
     x = c.submit(div, 1, 0)
-    result = yield x.exception()
+    result = await x.exception()
     assert isinstance(result, ZeroDivisionError)
 
     x = c.submit(div, 1, 1)
-    result = yield x.exception()
+    result = await x.exception()
     assert result is None
 
 
@@ -399,23 +397,23 @@ def test_Future_exception_sync(c):
 
 
 @gen_cluster(client=True)
-def test_Future_release(c, s, a, b):
+async def test_Future_release(c, s, a, b):
     # Released Futures should be removed timely from the Client
     x = c.submit(div, 1, 1)
-    yield x
+    await x
     x.release()
-    yield gen.moment
+    await asyncio.sleep(0)
     assert not c.futures
 
     x = c.submit(slowinc, 1, delay=0.5)
     x.release()
-    yield gen.moment
+    await asyncio.sleep(0)
     assert not c.futures
 
     x = c.submit(div, 1, 0)
-    yield x.exception()
+    await x.exception()
     x.release()
-    yield gen.moment
+    await asyncio.sleep(0)
     assert not c.futures
 
 
@@ -454,7 +452,7 @@ def test_short_tracebacks(loop, c):
 
 
 @gen_cluster(client=True)
-def test_map_naming(c, s, a, b):
+async def test_map_naming(c, s, a, b):
     L1 = c.map(inc, range(5))
     L2 = c.map(inc, range(5))
 
@@ -468,7 +466,7 @@ def test_map_naming(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_submit_naming(c, s, a, b):
+async def test_submit_naming(c, s, a, b):
     a = c.submit(inc, 1)
     b = c.submit(inc, 1)
 
@@ -479,33 +477,33 @@ def test_submit_naming(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_exceptions(c, s, a, b):
+async def test_exceptions(c, s, a, b):
     x = c.submit(div, 1, 2)
-    result = yield x
+    result = await x
     assert result == 1 / 2
 
     x = c.submit(div, 1, 0)
     with pytest.raises(ZeroDivisionError):
-        result = yield x
+        await x
 
     x = c.submit(div, 10, 2)  # continues to operate
-    result = yield x
+    result = await x
     assert result == 10 / 2
 
 
 @gen_cluster()
-def test_gc(s, a, b):
-    c = yield Client(s.address, asynchronous=True)
+async def test_gc(s, a, b):
+    c = await Client(s.address, asynchronous=True)
 
     x = c.submit(inc, 10)
-    yield x
+    await x
     assert s.tasks[x.key].who_has
     x.__del__()
-    yield async_wait_for(
+    await async_wait_for(
         lambda: x.key not in s.tasks or not s.tasks[x.key].who_has, timeout=0.3
     )
 
-    yield c.close()
+    await c.close()
 
 
 def test_thread(c):
@@ -534,27 +532,27 @@ def test_sync_exceptions(c):
 
 
 @gen_cluster(client=True)
-def test_gather(c, s, a, b):
+async def test_gather(c, s, a, b):
     x = c.submit(inc, 10)
     y = c.submit(inc, x)
 
-    result = yield c.gather(x)
+    result = await c.gather(x)
     assert result == 11
-    result = yield c.gather([x])
+    result = await c.gather([x])
     assert result == [11]
-    result = yield c.gather({"x": x, "y": [y]})
+    result = await c.gather({"x": x, "y": [y]})
     assert result == {"x": 11, "y": [12]}
 
 
 @gen_cluster(client=True)
-def test_gather_lost(c, s, a, b):
-    [x] = yield c.scatter([1], workers=a.address)
+async def test_gather_lost(c, s, a, b):
+    [x] = await c.scatter([1], workers=a.address)
     y = c.submit(inc, 1, workers=b.address)
 
-    yield a.close()
+    await a.close()
 
     with pytest.raises(Exception):
-        res = yield c.gather([x, y])
+        await c.gather([x, y])
 
 
 def test_gather_sync(c):
@@ -571,25 +569,25 @@ def test_gather_sync(c):
 
 
 @gen_cluster(client=True)
-def test_gather_strict(c, s, a, b):
+async def test_gather_strict(c, s, a, b):
     x = c.submit(div, 2, 1)
     y = c.submit(div, 1, 0)
 
     with pytest.raises(ZeroDivisionError):
-        yield c.gather([x, y])
+        await c.gather([x, y])
 
-    [xx] = yield c.gather([x, y], errors="skip")
+    [xx] = await c.gather([x, y], errors="skip")
     assert xx == 2
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)])
-def test_gather_skip(c, s, a):
+async def test_gather_skip(c, s, a):
     x = c.submit(div, 1, 0, priority=10)
     y = c.submit(slowinc, 1, delay=0.5)
 
     with captured_logger(logging.getLogger("distributed.scheduler")) as sched:
         with captured_logger(logging.getLogger("distributed.client")) as client:
-            L = yield c.gather([x, y], errors="skip")
+            L = await c.gather([x, y], errors="skip")
             assert L == [2]
 
     assert not client.getvalue()
@@ -597,28 +595,29 @@ def test_gather_skip(c, s, a):
 
 
 @gen_cluster(client=True)
-def test_limit_concurrent_gathering(c, s, a, b):
+async def test_limit_concurrent_gathering(c, s, a, b):
     futures = c.map(inc, range(100))
-    results = yield futures
+    await c.gather(futures)
     assert len(a.outgoing_transfer_log) + len(b.outgoing_transfer_log) < 100
 
 
 @gen_cluster(client=True, timeout=None)
-def test_get(c, s, a, b):
+async def test_get(c, s, a, b):
     future = c.get({"x": (inc, 1)}, "x", sync=False)
     assert isinstance(future, Future)
-    result = yield future
+    result = await future
     assert result == 2
 
     futures = c.get({"x": (inc, 1)}, ["x"], sync=False)
     assert isinstance(futures[0], Future)
-    result = yield futures
+    result = await c.gather(futures)
     assert result == [2]
 
-    result = yield c.get({}, [], sync=False)
+    futures = c.get({}, [], sync=False)
+    result = await c.gather(futures)
     assert result == []
 
-    result = yield c.get(
+    result = await c.get(
         {("x", 1): (inc, 1), ("x", 2): (inc, ("x", 1))}, ("x", 2), sync=False
     )
     assert result == 3
@@ -650,7 +649,7 @@ def test_get_sync_optimize_graph_passes_through(c):
 
 
 @gen_cluster(client=True)
-def test_gather_errors(c, s, a, b):
+async def test_gather_errors(c, s, a, b):
     def f(a, b):
         raise TypeError
 
@@ -660,20 +659,20 @@ def g(a, b):
     future_f = c.submit(f, 1, 2)
     future_g = c.submit(g, 1, 2)
     with pytest.raises(TypeError):
-        yield c.gather(future_f)
+        await c.gather(future_f)
     with pytest.raises(AttributeError):
-        yield c.gather(future_g)
+        await c.gather(future_g)
 
-    yield a.close()
+    await a.close()
 
 
 @gen_cluster(client=True)
-def test_wait(c, s, a, b):
+async def test_wait(c, s, a, b):
     x = c.submit(inc, 1)
     y = c.submit(inc, 1)
     z = c.submit(inc, 2)
 
-    done, not_done = yield wait([x, y, z])
+    done, not_done = await wait([x, y, z])
 
     assert done == {x, y, z}
     assert not_done == set()
@@ -681,12 +680,12 @@ def test_wait(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_wait_first_completed(c, s, a, b):
+async def test_wait_first_completed(c, s, a, b):
     x = c.submit(slowinc, 1)
     y = c.submit(slowinc, 1)
     z = c.submit(inc, 2)
 
-    done, not_done = yield wait([x, y, z], return_when="FIRST_COMPLETED")
+    done, not_done = await wait([x, y, z], return_when="FIRST_COMPLETED")
 
     assert done == {z}
     assert not_done == {x, y}
@@ -696,10 +695,10 @@ def test_wait_first_completed(c, s, a, b):
 
 
 @gen_cluster(client=True, timeout=2)
-def test_wait_timeout(c, s, a, b):
+async def test_wait_timeout(c, s, a, b):
     future = c.submit(sleep, 0.3)
     with pytest.raises(TimeoutError):
-        yield wait(future, timeout=0.01)
+        await wait(future, timeout=0.01)
 
 
 def test_wait_sync(c):
@@ -728,31 +727,31 @@ def test_wait_informative_error_for_timeouts(c):
 
 
 @gen_cluster(client=True)
-def test_garbage_collection(c, s, a, b):
+async def test_garbage_collection(c, s, a, b):
     x = c.submit(inc, 1)
     y = c.submit(inc, 1)
 
     assert c.refcount[x.key] == 2
     x.__del__()
-    yield gen.moment
+    await asyncio.sleep(0)
     assert c.refcount[x.key] == 1
 
     z = c.submit(inc, y)
     y.__del__()
-    yield gen.moment
+    await asyncio.sleep(0)
 
-    result = yield z
+    result = await z
     assert result == 3
 
     ykey = y.key
     y.__del__()
-    yield gen.moment
+    await asyncio.sleep(0)
     assert ykey not in c.futures
 
 
 @gen_cluster(client=True)
-def test_garbage_collection_with_scatter(c, s, a, b):
-    [future] = yield c.scatter([1])
+async def test_garbage_collection_with_scatter(c, s, a, b):
+    [future] = await c.scatter([1])
     assert future.key in c.futures
     assert future.status == "finished"
     assert s.who_wants[future.key] == {c.id}
@@ -760,7 +759,7 @@ def test_garbage_collection_with_scatter(c, s, a, b):
     key = future.key
     assert c.refcount[key] == 1
     future.__del__()
-    yield gen.moment
+    await asyncio.sleep(0)
     assert c.refcount[key] == 0
 
     start = time()
@@ -769,50 +768,50 @@ def test_garbage_collection_with_scatter(c, s, a, b):
             break
         else:
             assert time() < start + 3
-            yield gen.sleep(0.1)
+            await asyncio.sleep(0.1)
 
 
 @gen_cluster(timeout=1000, client=True)
-def test_recompute_released_key(c, s, a, b):
+async def test_recompute_released_key(c, s, a, b):
     x = c.submit(inc, 100)
-    result1 = yield x
+    result1 = await x
     xkey = x.key
     del x
     import gc
 
     gc.collect()
-    yield gen.moment
+    await asyncio.sleep(0)
     assert c.refcount[xkey] == 0
 
     # 1 second batching needs a second action to trigger
     while xkey in s.tasks and s.tasks[xkey].who_has or xkey in a.data or xkey in b.data:
-        yield gen.sleep(0.1)
+        await asyncio.sleep(0.1)
 
     x = c.submit(inc, 100)
     assert x.key in c.futures
-    result2 = yield x
+    result2 = await x
     assert result1 == result2
 
 
 @pytest.mark.slow
 @gen_cluster(client=True)
-def test_long_tasks_dont_trigger_timeout(c, s, a, b):
+async def test_long_tasks_dont_trigger_timeout(c, s, a, b):
     from time import sleep
 
     x = c.submit(sleep, 3)
-    yield x
+    await x
 
 
 @pytest.mark.skip
 @gen_cluster(client=True)
-def test_missing_data_heals(c, s, a, b):
+async def test_missing_data_heals(c, s, a, b):
     a.validate = False
     b.validate = False
     x = c.submit(inc, 1)
     y = c.submit(inc, x)
     z = c.submit(inc, y)
 
-    yield wait([x, y, z])
+    await wait([x, y, z])
 
     # Secretly delete y's key
     if y.key in a.data:
@@ -821,36 +820,36 @@ def test_missing_data_heals(c, s, a, b):
     if y.key in b.data:
         del b.data[y.key]
         b.release_key(y.key)
-    yield gen.moment
+    await asyncio.sleep(0)
 
     w = c.submit(add, y, z)
 
-    result = yield w
+    result = await w
     assert result == 3 + 4
 
 
 @pytest.mark.skip
 @gen_cluster(client=True)
-def test_gather_robust_to_missing_data(c, s, a, b):
+async def test_gather_robust_to_missing_data(c, s, a, b):
     a.validate = False
     b.validate = False
     x, y, z = c.map(inc, range(3))
-    yield wait([x, y, z])  # everything computed
+    await wait([x, y, z])  # everything computed
 
     for f in [x, y]:
         for w in [a, b]:
             if f.key in w.data:
                 del w.data[f.key]
-                yield gen.moment
+                await asyncio.sleep(0)
                 w.release_key(f.key)
 
-    xx, yy, zz = yield c.gather([x, y, z])
+    xx, yy, zz = await c.gather([x, y, z])
     assert (xx, yy, zz) == (1, 2, 3)
 
 
 @pytest.mark.skip
 @gen_cluster(client=True)
-def test_gather_robust_to_nested_missing_data(c, s, a, b):
+async def test_gather_robust_to_nested_missing_data(c, s, a, b):
     a.validate = False
     b.validate = False
     w = c.submit(inc, 1)
@@ -858,22 +857,22 @@ def test_gather_robust_to_nested_missing_data(c, s, a, b):
     y = c.submit(inc, x)
     z = c.submit(inc, y)
 
-    yield wait([z])
+    await wait([z])
 
     for worker in [a, b]:
         for datum in [y, z]:
             if datum.key in worker.data:
                 del worker.data[datum.key]
-                yield gen.moment
+                await asyncio.sleep(0)
                 worker.release_key(datum.key)
 
-    result = yield c.gather([z])
+    result = await c.gather([z])
 
     assert result == [inc(inc(inc(inc(1))))]
 
 
 @gen_cluster(client=True)
-def test_tokenize_on_futures(c, s, a, b):
+async def test_tokenize_on_futures(c, s, a, b):
     x = c.submit(inc, 1)
     y = c.submit(inc, 1)
     tok = tokenize(x)
@@ -889,10 +888,10 @@ def test_tokenize_on_futures(c, s, a, b):
     not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
 )
 @gen_cluster([("127.0.0.1", 1), ("127.0.0.2", 2)], client=True)
-def test_restrictions_submit(c, s, a, b):
+async def test_restrictions_submit(c, s, a, b):
     x = c.submit(inc, 1, workers={a.ip})
     y = c.submit(inc, x, workers={b.ip})
-    yield wait([x, y])
+    await wait([x, y])
 
     assert s.host_restrictions[x.key] == {a.ip}
     assert x.key in a.data
@@ -902,10 +901,10 @@ def test_restrictions_submit(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_restrictions_ip_port(c, s, a, b):
+async def test_restrictions_ip_port(c, s, a, b):
     x = c.submit(inc, 1, workers={a.address})
     y = c.submit(inc, x, workers={b.address})
-    yield wait([x, y])
+    await wait([x, y])
 
     assert s.worker_restrictions[x.key] == {a.address}
     assert x.key in a.data
@@ -918,9 +917,9 @@ def test_restrictions_ip_port(c, s, a, b):
     not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
 )
 @gen_cluster([("127.0.0.1", 1), ("127.0.0.2", 2)], client=True)
-def test_restrictions_map(c, s, a, b):
+async def test_restrictions_map(c, s, a, b):
     L = c.map(inc, range(5), workers={a.ip})
-    yield wait(L)
+    await wait(L)
 
     assert set(a.data) == {x.key for x in L}
     assert not b.data
@@ -928,7 +927,7 @@ def test_restrictions_map(c, s, a, b):
         assert s.host_restrictions[x.key] == {a.ip}
 
     L = c.map(inc, [10, 11, 12], workers=[{a.ip}, {a.ip, b.ip}, {b.ip}])
-    yield wait(L)
+    await wait(L)
 
     assert s.host_restrictions[L[0].key] == {a.ip}
     assert s.host_restrictions[L[1].key] == {a.ip, b.ip}
@@ -942,22 +941,22 @@ def test_restrictions_map(c, s, a, b):
     not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
 )
 @gen_cluster([("127.0.0.1", 1), ("127.0.0.2", 2)], client=True)
-def test_restrictions_get(c, s, a, b):
+async def test_restrictions_get(c, s, a, b):
     dsk = {"x": 1, "y": (inc, "x"), "z": (inc, "y")}
     restrictions = {"y": {a.ip}, "z": {b.ip}}
 
     futures = c.get(dsk, ["y", "z"], restrictions, sync=False)
-    result = yield futures
+    result = await c.gather(futures)
     assert result == [2, 3]
     assert "y" in a.data
     assert "z" in b.data
 
 
 @gen_cluster(client=True)
-def dont_test_bad_restrictions_raise_exception(c, s, a, b):
+async def dont_test_bad_restrictions_raise_exception(c, s, a, b):
     z = c.submit(inc, 2, workers={"bad-address"})
     try:
-        yield z
+        await z
         assert False
     except ValueError as e:
         assert "bad-address" in str(e)
@@ -965,133 +964,133 @@ def dont_test_bad_restrictions_raise_exception(c, s, a, b):
 
 
 @gen_cluster(client=True, timeout=None)
-def test_remove_worker(c, s, a, b):
+async def test_remove_worker(c, s, a, b):
     L = c.map(inc, range(20))
-    yield wait(L)
+    await wait(L)
 
-    yield b.close()
+    await b.close()
 
     assert b.address not in s.workers
 
-    result = yield c.gather(L)
+    result = await c.gather(L)
     assert result == list(map(inc, range(20)))
 
 
 @gen_cluster(nthreads=[("127.0.0.1", 1)], client=True)
-def test_errors_dont_block(c, s, w):
+async def test_errors_dont_block(c, s, w):
     L = [c.submit(inc, 1), c.submit(throws, 1), c.submit(inc, 2), c.submit(throws, 2)]
 
     start = time()
     while not (L[0].status == L[2].status == "finished"):
         assert time() < start + 5
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
-    result = yield c.gather([L[0], L[2]])
+    result = await c.gather([L[0], L[2]])
     assert result == [2, 3]
 
 
 @gen_cluster(client=True)
-def test_submit_quotes(c, s, a, b):
+async def test_submit_quotes(c, s, a, b):
     def assert_list(x, z=[]):
         return isinstance(x, list) and isinstance(z, list)
 
     x = c.submit(assert_list, [1, 2, 3])
-    result = yield x
+    result = await x
     assert result
 
     x = c.submit(assert_list, [1, 2, 3], z=[4, 5, 6])
-    result = yield x
+    result = await x
     assert result
 
     x = c.submit(inc, 1)
     y = c.submit(inc, 2)
     z = c.submit(assert_list, [x, y])
-    result = yield z
+    result = await z
     assert result
 
 
 @gen_cluster(client=True)
-def test_map_quotes(c, s, a, b):
+async def test_map_quotes(c, s, a, b):
     def assert_list(x, z=[]):
         return isinstance(x, list) and isinstance(z, list)
 
     L = c.map(assert_list, [[1, 2, 3], [4]])
-    result = yield c.gather(L)
+    result = await c.gather(L)
     assert all(result)
 
     L = c.map(assert_list, [[1, 2, 3], [4]], z=[10])
-    result = yield c.gather(L)
+    result = await c.gather(L)
     assert all(result)
 
     L = c.map(assert_list, [[1, 2, 3], [4]], [[]] * 3)
-    result = yield c.gather(L)
+    result = await c.gather(L)
     assert all(result)
 
 
 @gen_cluster()
-def test_two_consecutive_clients_share_results(s, a, b):
-    c = yield Client(s.address, asynchronous=True)
+async def test_two_consecutive_clients_share_results(s, a, b):
+    c = await Client(s.address, asynchronous=True)
 
     x = c.submit(random.randint, 0, 1000, pure=True)
-    xx = yield x
+    xx = await x
 
-    f = yield Client(s.address, asynchronous=True)
+    f = await Client(s.address, asynchronous=True)
 
     y = f.submit(random.randint, 0, 1000, pure=True)
-    yy = yield y
+    yy = await y
 
     assert xx == yy
 
-    yield c.close()
-    yield f.close()
+    await c.close()
+    await f.close()
 
 
 @gen_cluster(client=True)
-def test_submit_then_get_with_Future(c, s, a, b):
+async def test_submit_then_get_with_Future(c, s, a, b):
     x = c.submit(slowinc, 1)
     dsk = {"y": (inc, x)}
 
-    result = yield c.get(dsk, "y", sync=False)
+    result = await c.get(dsk, "y", sync=False)
     assert result == 3
 
 
 @gen_cluster(client=True)
-def test_aliases(c, s, a, b):
+async def test_aliases(c, s, a, b):
     x = c.submit(inc, 1)
 
     dsk = {"y": x}
-    result = yield c.get(dsk, "y", sync=False)
+    result = await c.get(dsk, "y", sync=False)
     assert result == 2
 
 
 @gen_cluster(client=True)
-def test_aliases_2(c, s, a, b):
+async def test_aliases_2(c, s, a, b):
     dsk_keys = [
         ({"x": (inc, 1), "y": "x", "z": "x", "w": (add, "y", "z")}, ["y", "w"]),
         ({"x": "y", "y": 1}, ["x"]),
         ({"x": 1, "y": "x", "z": "y", "w": (inc, "z")}, ["w"]),
     ]
     for dsk, keys in dsk_keys:
-        result = yield c.get(dsk, keys, sync=False)
+        result = await c.gather(c.get(dsk, keys, sync=False))
         assert list(result) == list(dask.get(dsk, keys))
-        yield gen.moment
+        await asyncio.sleep(0)
 
 
 @gen_cluster(client=True)
-def test__scatter(c, s, a, b):
-    d = yield c.scatter({"y": 20})
+async def test_scatter(c, s, a, b):
+    d = await c.scatter({"y": 20})
     assert isinstance(d["y"], Future)
     assert a.data.get("y") == 20 or b.data.get("y") == 20
     y_who_has = s.get_who_has(keys=["y"])["y"]
     assert a.address in y_who_has or b.address in y_who_has
     assert s.get_nbytes(summary=False) == {"y": sizeof(20)}
-    yy = yield c.gather([d["y"]])
+    yy = await c.gather([d["y"]])
     assert yy == [20]
 
-    [x] = yield c.scatter([10])
+    [x] = await c.scatter([10])
     assert isinstance(x, Future)
     assert a.data.get(x.key) == 10 or b.data.get(x.key) == 10
-    xx = yield c.gather([x])
+    xx = await c.gather([x])
     x_who_has = s.get_who_has(keys=[x.key])[x.key]
     assert s.tasks[x.key].who_has
     assert (
@@ -1102,49 +1101,49 @@ def test__scatter(c, s, a, b):
     assert xx == [10]
 
     z = c.submit(add, x, d["y"])  # submit works on Future
-    result = yield z
+    result = await z
     assert result == 10 + 20
-    result = yield c.gather([z, x])
+    result = await c.gather([z, x])
     assert result == [30, 10]
 
 
 @gen_cluster(client=True)
-def test__scatter_types(c, s, a, b):
-    d = yield c.scatter({"x": 1})
+async def test_scatter_types(c, s, a, b):
+    d = await c.scatter({"x": 1})
     assert isinstance(d, dict)
     assert list(d) == ["x"]
 
     for seq in [[1], (1,), {1}, frozenset([1])]:
-        L = yield c.scatter(seq)
+        L = await c.scatter(seq)
         assert isinstance(L, type(seq))
         assert len(L) == 1
         s.validate_state()
 
-    seq = yield c.scatter(range(5))
+    seq = await c.scatter(range(5))
     assert isinstance(seq, list)
     assert len(seq) == 5
     s.validate_state()
 
 
 @gen_cluster(client=True)
-def test__scatter_non_list(c, s, a, b):
-    x = yield c.scatter(1)
+async def test_scatter_non_list(c, s, a, b):
+    x = await c.scatter(1)
     assert isinstance(x, Future)
-    result = yield x
+    result = await x
     assert result == 1
 
 
 @gen_cluster(client=True)
-def test_scatter_hash(c, s, a, b):
-    [a] = yield c.scatter([1])
-    [b] = yield c.scatter([1])
+async def test_scatter_hash(c, s, a, b):
+    [a] = await c.scatter([1])
+    [b] = await c.scatter([1])
 
     assert a.key == b.key
     s.validate_state()
 
 
 @gen_cluster(client=True)
-def test_scatter_tokenize_local(c, s, a, b):
+async def test_scatter_tokenize_local(c, s, a, b):
     from dask.base import normalize_token
 
     class MyObj:
@@ -1159,46 +1158,46 @@ def f(x):
 
     obj = MyObj()
 
-    future = yield c.scatter(obj)
+    future = await c.scatter(obj)
     assert L and L[0] is obj
 
 
 @gen_cluster(client=True)
-def test_scatter_singletons(c, s, a, b):
+async def test_scatter_singletons(c, s, a, b):
     np = pytest.importorskip("numpy")
     pd = pytest.importorskip("pandas")
     for x in [1, np.ones(5), pd.DataFrame({"x": [1, 2, 3]})]:
-        future = yield c.scatter(x)
-        result = yield future
+        future = await c.scatter(x)
+        result = await future
         assert str(result) == str(x)
 
 
 @gen_cluster(client=True)
-def test_scatter_typename(c, s, a, b):
-    future = yield c.scatter(123)
+async def test_scatter_typename(c, s, a, b):
+    future = await c.scatter(123)
     assert future.key.startswith("int")
 
 
 @gen_cluster(client=True)
-def test_scatter_hash(c, s, a, b):
-    x = yield c.scatter(123)
-    y = yield c.scatter(123)
+async def test_scatter_hash(c, s, a, b):
+    x = await c.scatter(123)
+    y = await c.scatter(123)
     assert x.key == y.key
 
-    z = yield c.scatter(123, hash=False)
+    z = await c.scatter(123, hash=False)
     assert z.key != y.key
 
 
 @gen_cluster(client=True)
-def test_get_releases_data(c, s, a, b):
-    [x] = yield c.get({"x": (inc, 1)}, ["x"], sync=False)
+async def test_get_releases_data(c, s, a, b):
+    await c.gather(c.get({"x": (inc, 1)}, ["x"], sync=False))
     import gc
 
     gc.collect()
 
     start = time()
     while c.refcount["x"]:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 2
 
 
@@ -1229,26 +1228,26 @@ def test_global_clients(loop):
 
 
 @gen_cluster(client=True)
-def test_exception_on_exception(c, s, a, b):
+async def test_exception_on_exception(c, s, a, b):
     x = c.submit(lambda: 1 / 0)
     y = c.submit(inc, x)
 
     with pytest.raises(ZeroDivisionError):
-        yield y
+        await y
 
     z = c.submit(inc, y)
 
     with pytest.raises(ZeroDivisionError):
-        yield z
+        await z
 
 
 @gen_cluster(client=True)
-def test_get_nbytes(c, s, a, b):
-    [x] = yield c.scatter([1])
+async def test_get_nbytes(c, s, a, b):
+    [x] = await c.scatter([1])
     assert s.get_nbytes(summary=False) == {x.key: sizeof(1)}
 
     y = c.submit(inc, x)
-    yield y
+    await y
 
     assert s.get_nbytes(summary=False) == {x.key: sizeof(1), y.key: sizeof(2)}
 
@@ -1257,24 +1256,24 @@ def test_get_nbytes(c, s, a, b):
     not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
 )
 @gen_cluster([("127.0.0.1", 1), ("127.0.0.2", 2)], client=True)
-def test_nbytes_determines_worker(c, s, a, b):
+async def test_nbytes_determines_worker(c, s, a, b):
     x = c.submit(identity, 1, workers=[a.ip])
     y = c.submit(identity, tuple(range(100)), workers=[b.ip])
-    yield c.gather([x, y])
+    await c.gather([x, y])
 
     z = c.submit(lambda x, y: None, x, y)
-    yield z
+    await z
     assert s.tasks[z.key].who_has == {s.workers[b.address]}
 
 
 @gen_cluster(client=True)
-def test_if_intermediates_clear_on_error(c, s, a, b):
+async def test_if_intermediates_clear_on_error(c, s, a, b):
     x = delayed(div, pure=True)(1, 0)
     y = delayed(div, pure=True)(1, 2)
     z = delayed(add, pure=True)(x, y)
     f = c.compute(z)
     with pytest.raises(ZeroDivisionError):
-        yield f
+        await f
     s.validate_state()
     assert not any(ts.who_has for ts in s.tasks.values())
 
@@ -1282,7 +1281,7 @@ def test_if_intermediates_clear_on_error(c, s, a, b):
 @gen_cluster(
     client=True, config={"distributed.scheduler.default-task-durations": {"f": "1ms"}}
 )
-def test_pragmatic_move_small_data_to_large_data(c, s, a, b):
+async def test_pragmatic_move_small_data_to_large_data(c, s, a, b):
     np = pytest.importorskip("numpy")
     lists = c.map(np.ones, [10000] * 10, pure=False)
     sums = c.map(np.sum, lists)
@@ -1293,9 +1292,8 @@ def f(x, y):
 
     results = c.map(f, lists, [total] * 10)
 
-    yield wait([total])
-
-    yield wait(results)
+    await wait([total])
+    await wait(results)
 
     assert (
         sum(
@@ -1307,20 +1305,20 @@ def f(x, y):
 
 
 @gen_cluster(client=True)
-def test_get_with_non_list_key(c, s, a, b):
+async def test_get_with_non_list_key(c, s, a, b):
     dsk = {("x", 0): (inc, 1), 5: (inc, 2)}
 
-    x = yield c.get(dsk, ("x", 0), sync=False)
-    y = yield c.get(dsk, 5, sync=False)
+    x = await c.get(dsk, ("x", 0), sync=False)
+    y = await c.get(dsk, 5, sync=False)
     assert x == 2
     assert y == 3
 
 
 @gen_cluster(client=True)
-def test_get_with_error(c, s, a, b):
+async def test_get_with_error(c, s, a, b):
     dsk = {"x": (div, 1, 0), "y": (inc, "x")}
     with pytest.raises(ZeroDivisionError):
-        yield c.get(dsk, "y", sync=False)
+        await c.get(dsk, "y", sync=False)
 
 
 def test_get_with_error_sync(c):
@@ -1330,12 +1328,12 @@ def test_get_with_error_sync(c):
 
 
 @gen_cluster(client=True)
-def test_directed_scatter(c, s, a, b):
-    yield c.scatter([1, 2, 3], workers=[a.address])
+async def test_directed_scatter(c, s, a, b):
+    await c.scatter([1, 2, 3], workers=[a.address])
     assert len(a.data) == 3
     assert not b.data
 
-    yield c.scatter([4, 5], workers=[b.name])
+    await c.scatter([4, 5], workers=[b.name])
     assert len(b.data) == 2
 
 
@@ -1347,56 +1345,56 @@ def test_directed_scatter_sync(c, s, a, b, loop):
 
 
 @gen_cluster(client=True)
-def test_scatter_direct(c, s, a, b):
-    future = yield c.scatter(123, direct=True)
+async def test_scatter_direct(c, s, a, b):
+    future = await c.scatter(123, direct=True)
     assert future.key in a.data or future.key in b.data
     assert s.tasks[future.key].who_has
     assert future.status == "finished"
-    result = yield future
+    result = await future
     assert result == 123
     assert not s.counters["op"].components[0]["scatter"]
 
-    result = yield future
+    result = await future
     assert not s.counters["op"].components[0]["gather"]
 
-    result = yield c.gather(future)
+    result = await c.gather(future)
     assert not s.counters["op"].components[0]["gather"]
 
 
 @gen_cluster(client=True)
-def test_scatter_direct_numpy(c, s, a, b):
+async def test_scatter_direct_numpy(c, s, a, b):
     np = pytest.importorskip("numpy")
     x = np.ones(5)
-    future = yield c.scatter(x, direct=True)
-    result = yield future
+    future = await c.scatter(x, direct=True)
+    result = await future
     assert np.allclose(x, result)
     assert not s.counters["op"].components[0]["scatter"]
 
 
 @gen_cluster(client=True)
-def test_scatter_direct_broadcast(c, s, a, b):
-    future2 = yield c.scatter(456, direct=True, broadcast=True)
+async def test_scatter_direct_broadcast(c, s, a, b):
+    future2 = await c.scatter(456, direct=True, broadcast=True)
     assert future2.key in a.data
     assert future2.key in b.data
     assert s.tasks[future2.key].who_has == {s.workers[a.address], s.workers[b.address]}
-    result = yield future2
+    result = await future2
     assert result == 456
     assert not s.counters["op"].components[0]["scatter"]
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 4)
-def test_scatter_direct_balanced(c, s, *workers):
-    futures = yield c.scatter([1, 2, 3], direct=True)
+async def test_scatter_direct_balanced(c, s, *workers):
+    futures = await c.scatter([1, 2, 3], direct=True)
     assert sorted([len(w.data) for w in workers]) == [0, 1, 1, 1]
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 4)
-def test_scatter_direct_broadcast_target(c, s, *workers):
-    futures = yield c.scatter([123, 456], direct=True, workers=workers[0].address)
+async def test_scatter_direct_broadcast_target(c, s, *workers):
+    futures = await c.scatter([123, 456], direct=True, workers=workers[0].address)
     assert futures[0].key in workers[0].data
     assert futures[1].key in workers[0].data
 
-    futures = yield c.scatter(
+    futures = await c.scatter(
         [123, 456],
         direct=True,
         broadcast=True,
@@ -1410,16 +1408,16 @@ def test_scatter_direct_broadcast_target(c, s, *workers):
 
 
 @gen_cluster(client=True, nthreads=[])
-def test_scatter_direct_empty(c, s):
+async def test_scatter_direct_empty(c, s):
     with pytest.raises((ValueError, TimeoutError)):
-        yield c.scatter(123, direct=True, timeout=0.1)
+        await c.scatter(123, direct=True, timeout=0.1)
 
 
 @gen_cluster(client=True, timeout=None, nthreads=[("127.0.0.1", 1)] * 5)
-def test_scatter_direct_spread_evenly(c, s, *workers):
+async def test_scatter_direct_spread_evenly(c, s, *workers):
     futures = []
     for i in range(10):
-        future = yield c.scatter(i, direct=True)
+        future = await c.scatter(i, direct=True)
         futures.append(future)
 
     assert all(w.data for w in workers)
@@ -1436,34 +1434,32 @@ def test_scatter_gather_sync(c, direct, broadcast):
 
 
 @gen_cluster(client=True)
-def test_gather_direct(c, s, a, b):
-    futures = yield c.scatter([1, 2, 3])
+async def test_gather_direct(c, s, a, b):
+    futures = await c.scatter([1, 2, 3])
 
-    data = yield c.gather(futures, direct=True)
+    data = await c.gather(futures, direct=True)
     assert data == [1, 2, 3]
 
 
 @gen_cluster(client=True)
-def test_many_submits_spread_evenly(c, s, a, b):
+async def test_many_submits_spread_evenly(c, s, a, b):
     L = [c.submit(inc, i) for i in range(10)]
-    yield wait(L)
+    await wait(L)
 
     assert a.data and b.data
 
 
 @gen_cluster(client=True)
-def test_traceback(c, s, a, b):
+async def test_traceback(c, s, a, b):
     x = c.submit(div, 1, 0)
-    tb = yield x.traceback()
-
-    if sys.version_info[0] >= 3:
-        assert any("x / y" in line for line in pluck(3, traceback.extract_tb(tb)))
+    tb = await x.traceback()
+    assert any("x / y" in line for line in pluck(3, traceback.extract_tb(tb)))
 
 
 @gen_cluster(client=True)
-def test_get_traceback(c, s, a, b):
+async def test_get_traceback(c, s, a, b):
     try:
-        yield c.get({"x": (div, 1, 0)}, "x", sync=False)
+        await c.get({"x": (div, 1, 0)}, "x", sync=False)
     except ZeroDivisionError:
         exc_type, exc_value, exc_traceback = sys.exc_info()
         L = traceback.format_tb(exc_traceback)
@@ -1471,10 +1467,10 @@ def test_get_traceback(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_gather_traceback(c, s, a, b):
+async def test_gather_traceback(c, s, a, b):
     x = c.submit(div, 1, 0)
     try:
-        yield c.gather(x)
+        await c.gather(x)
     except ZeroDivisionError:
         exc_type, exc_value, exc_traceback = sys.exc_info()
         L = traceback.format_tb(exc_traceback)
@@ -1484,12 +1480,11 @@ def test_gather_traceback(c, s, a, b):
 def test_traceback_sync(c):
     x = c.submit(div, 1, 0)
     tb = x.traceback()
-    if sys.version_info[0] >= 3:
-        assert any(
-            "x / y" in line
-            for line in concat(traceback.extract_tb(tb))
-            if isinstance(line, str)
-        )
+    assert any(
+        "x / y" in line
+        for line in concat(traceback.extract_tb(tb))
+        if isinstance(line, str)
+    )
 
     y = c.submit(inc, x)
     tb2 = y.traceback()
@@ -1504,7 +1499,7 @@ def test_traceback_sync(c):
 
 
 @gen_cluster(client=True)
-def test_upload_file(c, s, a, b):
+async def test_upload_file(c, s, a, b):
     def g():
         import myfile
 
@@ -1513,21 +1508,21 @@ def g():
     with save_sys_modules():
         for value in [123, 456]:
             with tmp_text("myfile.py", "def f():\n    return {}".format(value)) as fn:
-                yield c.upload_file(fn)
+                await c.upload_file(fn)
 
             x = c.submit(g, pure=False)
-            result = yield x
+            result = await x
             assert result == value
 
 
 @gen_cluster(client=True)
-def test_upload_file_no_extension(c, s, a, b):
+async def test_upload_file_no_extension(c, s, a, b):
     with tmp_text("myfile", "") as fn:
-        yield c.upload_file(fn)
+        await c.upload_file(fn)
 
 
 @gen_cluster(client=True)
-def test_upload_file_zip(c, s, a, b):
+async def test_upload_file_zip(c, s, a, b):
     def g():
         import myfile
 
@@ -1541,10 +1536,10 @@ def g():
                 ) as fn_my_file:
                     with zipfile.ZipFile("myfile.zip", "w") as z:
                         z.write(fn_my_file, arcname=os.path.basename(fn_my_file))
-                    yield c.upload_file("myfile.zip")
+                    await c.upload_file("myfile.zip")
 
                     x = c.submit(g, pure=False)
-                    result = yield x
+                    result = await x
                     assert result == value
         finally:
             if os.path.exists("myfile.zip"):
@@ -1552,7 +1547,7 @@ def g():
 
 
 @gen_cluster(client=True)
-def test_upload_file_egg(c, s, a, b):
+async def test_upload_file_egg(c, s, a, b):
     def g():
         import package_1, package_2
 
@@ -1601,22 +1596,22 @@ def g():
                 ][0]
                 egg_path = os.path.join(egg_root, egg_name)
 
-                yield c.upload_file(egg_path)
+                await c.upload_file(egg_path)
                 os.remove(egg_path)
 
                 x = c.submit(g, pure=False)
-                result = yield x
+                result = await x
                 assert result == (value, value)
 
 
 @gen_cluster(client=True)
-def test_upload_large_file(c, s, a, b):
+async def test_upload_large_file(c, s, a, b):
     assert a.local_directory
     assert b.local_directory
     with tmp_text("myfile", "abc") as fn:
         with tmp_text("myfile2", "def") as fn2:
-            yield c._upload_large_file(fn, remote_filename="x")
-            yield c._upload_large_file(fn2)
+            await c._upload_large_file(fn, remote_filename="x")
+            await c._upload_large_file(fn2)
 
             for w in [a, b]:
                 assert os.path.exists(os.path.join(w.local_directory, "x"))
@@ -1640,10 +1635,10 @@ def g():
 
 
 @gen_cluster(client=True)
-def test_upload_file_exception(c, s, a, b):
+async def test_upload_file_exception(c, s, a, b):
     with tmp_text("myfile.py", "syntax-error!") as fn:
         with pytest.raises(SyntaxError):
-            yield c.upload_file(fn)
+            await c.upload_file(fn)
 
 
 def test_upload_file_exception_sync(c):
@@ -1654,29 +1649,29 @@ def test_upload_file_exception_sync(c):
 
 @pytest.mark.skip
 @gen_cluster()
-def test_multiple_clients(s, a, b):
-    a = yield Client(s.address, asynchronous=True)
-    b = yield Client(s.address, asynchronous=True)
+async def test_multiple_clients(s, a, b):
+    a = await Client(s.address, asynchronous=True)
+    b = await Client(s.address, asynchronous=True)
 
     x = a.submit(inc, 1)
     y = b.submit(inc, 2)
     assert x.client is a
     assert y.client is b
-    xx = yield x
-    yy = yield y
+    xx = await x
+    yy = await y
     assert xx == 2
     assert yy == 3
     z = a.submit(add, x, y)
     assert z.client is a
-    zz = yield z
+    zz = await z
     assert zz == 5
 
-    yield a.close()
-    yield b.close()
+    await a.close()
+    await b.close()
 
 
 @gen_cluster(client=True)
-def test_async_compute(c, s, a, b):
+async def test_async_compute(c, s, a, b):
     from dask.delayed import delayed
 
     x = delayed(1)
@@ -1688,7 +1683,7 @@ def test_async_compute(c, s, a, b):
     assert isinstance(zz, Future)
     assert aa == 3
 
-    result = yield c.gather([yy, zz])
+    result = await c.gather([yy, zz])
     assert result == [2, 0]
 
     assert isinstance(c.compute(y), Future)
@@ -1696,8 +1691,8 @@ def test_async_compute(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_async_compute_with_scatter(c, s, a, b):
-    d = yield c.scatter({("x", 1): 1, ("y", 1): 2})
+async def test_async_compute_with_scatter(c, s, a, b):
+    d = await c.scatter({("x", 1): 1, ("y", 1): 2})
     x, y = d[("x", 1)], d[("y", 1)]
 
     from dask.delayed import delayed
@@ -1705,7 +1700,7 @@ def test_async_compute_with_scatter(c, s, a, b):
     z = delayed(add)(delayed(inc)(x), delayed(inc)(y))
     zz = c.compute(z)
 
-    [result] = yield c.gather([zz])
+    [result] = await c.gather([zz])
     assert result == 2 + 3
 
 
@@ -1719,22 +1714,22 @@ def test_sync_compute(c):
 
 
 @gen_cluster(client=True)
-def test_remote_scatter_gather(c, s, a, b):
-    x, y, z = yield c.scatter([1, 2, 3])
+async def test_remote_scatter_gather(c, s, a, b):
+    x, y, z = await c.scatter([1, 2, 3])
 
     assert x.key in a.data or x.key in b.data
     assert y.key in a.data or y.key in b.data
     assert z.key in a.data or z.key in b.data
 
-    xx, yy, zz = yield c.gather([x, y, z])
+    xx, yy, zz = await c.gather([x, y, z])
     assert (xx, yy, zz) == (1, 2, 3)
 
 
 @gen_cluster(timeout=1000, client=True)
-def test_remote_submit_on_Future(c, s, a, b):
+async def test_remote_submit_on_Future(c, s, a, b):
     x = c.submit(lambda x: x + 1, 1)
     y = c.submit(lambda x: x + 1, x)
-    result = yield y
+    result = await y
     assert result == 3
 
 
@@ -1748,22 +1743,22 @@ def test_start_is_idempotent(c):
 
 
 @gen_cluster(client=True)
-def test_client_with_scheduler(c, s, a, b):
+async def test_client_with_scheduler(c, s, a, b):
     assert s.nthreads == {a.address: a.nthreads, b.address: b.nthreads}
 
     x = c.submit(inc, 1)
     y = c.submit(inc, 2)
     z = c.submit(add, x, y)
-    result = yield x
+    result = await x
     assert result == 1 + 1
-    result = yield z
+    result = await z
     assert result == 1 + 1 + 1 + 2
 
-    A, B, C = yield c.scatter([1, 2, 3])
-    AA, BB, xx = yield c.gather([A, B, x])
+    A, B, C = await c.scatter([1, 2, 3])
+    AA, BB, xx = await c.gather([A, B, x])
     assert (AA, BB, xx) == (1, 2, 2)
 
-    result = yield c.get({"x": (inc, 1), "y": (add, "x", 10)}, "y", sync=False)
+    result = await c.get({"x": (inc, 1), "y": (add, "x", 10)}, "y", sync=False)
     assert result == 12
 
 
@@ -1771,33 +1766,33 @@ def test_client_with_scheduler(c, s, a, b):
     not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
 )
 @gen_cluster([("127.0.0.1", 1), ("127.0.0.2", 2)], client=True)
-def test_allow_restrictions(c, s, a, b):
+async def test_allow_restrictions(c, s, a, b):
     aws = s.workers[a.address]
     bws = s.workers[a.address]
 
     x = c.submit(inc, 1, workers=a.ip)
-    yield x
+    await x
     assert s.tasks[x.key].who_has == {aws}
     assert not s.loose_restrictions
 
     x = c.submit(inc, 2, workers=a.ip, allow_other_workers=True)
-    yield x
+    await x
     assert s.tasks[x.key].who_has == {aws}
     assert x.key in s.loose_restrictions
 
     L = c.map(inc, range(3, 13), workers=a.ip, allow_other_workers=True)
-    yield wait(L)
+    await wait(L)
     assert all(s.tasks[f.key].who_has == {aws} for f in L)
     assert {f.key for f in L}.issubset(s.loose_restrictions)
 
     x = c.submit(inc, 15, workers="127.0.0.3", allow_other_workers=True)
 
-    yield x
+    await x
     assert s.tasks[x.key].who_has
     assert x.key in s.loose_restrictions
 
     L = c.map(inc, range(15, 25), workers="127.0.0.3", allow_other_workers=True)
-    yield wait(L)
+    await wait(L)
     assert all(s.tasks[f.key].who_has for f in L)
     assert {f.key for f in L}.issubset(s.loose_restrictions)
 
@@ -1828,18 +1823,18 @@ def test_bad_address():
 
 
 @gen_cluster(client=True)
-def test_long_error(c, s, a, b):
+async def test_long_error(c, s, a, b):
     def bad(x):
         raise ValueError("a" * 100000)
 
     x = c.submit(bad, 10)
 
     try:
-        yield x
+        await x
     except ValueError as e:
         assert len(str(e)) < 100000
 
-    tb = yield x.traceback()
+    tb = await x.traceback()
     assert all(
         len(line) < 100000
         for line in concat(traceback.extract_tb(tb))
@@ -1848,18 +1843,18 @@ def bad(x):
 
 
 @gen_cluster(client=True)
-def test_map_on_futures_with_kwargs(c, s, a, b):
+async def test_map_on_futures_with_kwargs(c, s, a, b):
     def f(x, y=10):
         return x + y
 
     futures = c.map(inc, range(10))
     futures2 = c.map(f, futures, y=20)
-    results = yield c.gather(futures2)
+    results = await c.gather(futures2)
     assert results == [i + 1 + 20 for i in range(10)]
 
     future = c.submit(inc, 100)
     future2 = c.submit(f, future, y=200)
-    result = yield future2
+    result = await future2
     assert result == 100 + 1 + 200
 
 
@@ -1883,19 +1878,19 @@ def __setstate__(self, state):
 
 
 @gen_cluster(client=True)
-def test_badly_serialized_input(c, s, a, b):
+async def test_badly_serialized_input(c, s, a, b):
     o = BadlySerializedObject()
 
     future = c.submit(inc, o)
     futures = c.map(inc, range(10))
 
-    L = yield c.gather(futures)
+    L = await c.gather(futures)
     assert list(L) == list(map(inc, range(10)))
     assert future.status == "error"
 
 
 @pytest.mark.skipif("True", reason="")
-def test_badly_serialized_input_stderr(capsys, c):
+async def test_badly_serialized_input_stderr(capsys, c):
     o = BadlySerializedObject()
     future = c.submit(inc, o)
 
@@ -1928,37 +1923,37 @@ def test_repr(loop):
 
 
 @gen_cluster(client=True)
-def test_repr_async(c, s, a, b):
+async def test_repr_async(c, s, a, b):
     c._repr_html_()
 
 
 @gen_cluster(client=True, worker_kwargs={"memory_limit": None})
-def test_repr_no_memory_limit(c, s, a, b):
+async def test_repr_no_memory_limit(c, s, a, b):
     c._repr_html_()
 
 
 @gen_test()
-def test_repr_localcluster():
-    cluster = yield LocalCluster(
+async def test_repr_localcluster():
+    cluster = await LocalCluster(
         processes=False, dashboard_address=None, asynchronous=True
     )
-    client = yield Client(cluster, asynchronous=True)
+    client = await Client(cluster, asynchronous=True)
     try:
         text = client._repr_html_()
         assert cluster.scheduler.address in text
         assert is_valid_xml(client._repr_html_())
     finally:
-        yield client.close()
-        yield cluster.close()
+        await client.close()
+        await cluster.close()
 
 
 @gen_cluster(client=True)
-def test_forget_simple(c, s, a, b):
+async def test_forget_simple(c, s, a, b):
     x = c.submit(inc, 1, retries=2)
     y = c.submit(inc, 2)
     z = c.submit(add, x, y, workers=[a.ip], allow_other_workers=True)
 
-    yield wait([x, y, z])
+    await wait([x, y, z])
     assert not s.waiting_data.get(x.key)
     assert not s.waiting_data.get(y.key)
 
@@ -1977,14 +1972,14 @@ def test_forget_simple(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_forget_complex(e, s, A, B):
-    a, b, c, d = yield e.scatter(list(range(4)))
+async def test_forget_complex(e, s, A, B):
+    a, b, c, d = await e.scatter(list(range(4)))
     ab = e.submit(add, a, b)
     cd = e.submit(add, c, d)
     ac = e.submit(add, a, c)
     acab = e.submit(add, ac, ab)
 
-    yield wait([a, b, c, d, ab, ac, cd, acab])
+    await wait([a, b, c, d, ab, ac, cd, acab])
 
     assert set(s.tasks) == {f.key for f in [ab, ac, cd, acab, a, b, c, d]}
 
@@ -2000,7 +1995,7 @@ def test_forget_complex(e, s, A, B):
 
     start = time()
     while b.key in A.data or b.key in B.data:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 10
 
     s.client_releases_keys(keys=[ac.key], client=e.id)
@@ -2008,7 +2003,7 @@ def test_forget_complex(e, s, A, B):
 
 
 @gen_cluster(client=True)
-def test_forget_in_flight(e, s, A, B):
+async def test_forget_in_flight(e, s, A, B):
     delayed2 = partial(delayed, pure=True)
     a, b, c, d = [delayed2(slowinc)(i) for i in range(4)]
     ab = delayed2(slowadd)(a, b, dask_key_name="ab")
@@ -2020,7 +2015,7 @@ def test_forget_in_flight(e, s, A, B):
     s.validate_state()
 
     for i in range(5):
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         s.validate_state()
 
     s.client_releases_keys(keys=[y.key], client=e.id)
@@ -2031,11 +2026,11 @@ def test_forget_in_flight(e, s, A, B):
 
 
 @gen_cluster(client=True)
-def test_forget_errors(c, s, a, b):
+async def test_forget_errors(c, s, a, b):
     x = c.submit(div, 1, 0)
     y = c.submit(inc, x)
     z = c.submit(inc, y)
-    yield wait([y])
+    await wait([y])
 
     assert x.key in s.exceptions
     assert x.key in s.exceptions_blame
@@ -2074,21 +2069,21 @@ def test_repr_sync(c):
 
 
 @gen_cluster(client=True)
-def test_waiting_data(c, s, a, b):
+async def test_waiting_data(c, s, a, b):
     x = c.submit(inc, 1)
     y = c.submit(inc, 2)
     z = c.submit(add, x, y, workers=[a.ip], allow_other_workers=True)
 
-    yield wait([x, y, z])
+    await wait([x, y, z])
 
     assert not s.waiting_data.get(x.key)
     assert not s.waiting_data.get(y.key)
 
 
 @gen_cluster()
-def test_multi_client(s, a, b):
-    c = yield Client(s.address, asynchronous=True)
-    f = yield Client(s.address, asynchronous=True)
+async def test_multi_client(s, a, b):
+    c = await Client(s.address, asynchronous=True)
+    f = await Client(s.address, asynchronous=True)
 
     assert set(s.client_comms) == {c.id, f.id}
 
@@ -2098,7 +2093,7 @@ def test_multi_client(s, a, b):
 
     assert y.key == y2.key
 
-    yield wait([x, y])
+    await wait([x, y])
 
     assert s.wants_what == {
         c.id: {x.key, y.key},
@@ -2107,22 +2102,22 @@ def test_multi_client(s, a, b):
     }
     assert s.who_wants == {x.key: {c.id}, y.key: {c.id, f.id}}
 
-    yield c.close()
+    await c.close()
 
     start = time()
     while c.id in s.wants_what:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 5
 
     assert c.id not in s.wants_what
     assert c.id not in s.who_wants[y.key]
     assert x.key not in s.who_wants
 
-    yield f.close()
+    await f.close()
 
     start = time()
     while s.tasks:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 2, s.tasks
 
 
@@ -2135,29 +2130,29 @@ def long_running_client_connection(address):
 
 
 @gen_cluster()
-def test_cleanup_after_broken_client_connection(s, a, b):
+async def test_cleanup_after_broken_client_connection(s, a, b):
     proc = mp_context.Process(target=long_running_client_connection, args=(s.address,))
     proc.daemon = True
     proc.start()
 
     start = time()
     while not s.tasks:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 5
 
     proc.terminate()
 
     start = time()
     while s.tasks:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 5
 
 
 @gen_cluster()
-def test_multi_garbage_collection(s, a, b):
-    c = yield Client(s.address, asynchronous=True)
+async def test_multi_garbage_collection(s, a, b):
+    c = await Client(s.address, asynchronous=True)
 
-    f = yield Client(s.address, asynchronous=True)
+    f = await Client(s.address, asynchronous=True)
 
     x = c.submit(inc, 1)
     y = f.submit(inc, 2)
@@ -2165,12 +2160,12 @@ def test_multi_garbage_collection(s, a, b):
 
     assert y.key == y2.key
 
-    yield wait([x, y])
+    await wait([x, y])
 
     x.__del__()
     start = time()
     while x.key in a.data or x.key in b.data:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 5
 
     assert s.wants_what == {c.id: {y.key}, f.id: {y.key}, "fire-and-forget": set()}
@@ -2179,10 +2174,10 @@ def test_multi_garbage_collection(s, a, b):
     y.__del__()
     start = time()
     while x.key in s.wants_what[f.id]:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 5
 
-    yield gen.sleep(0.1)
+    await asyncio.sleep(0.1)
     assert y.key in a.data or y.key in b.data
     assert s.wants_what == {c.id: {y.key}, f.id: set(), "fire-and-forget": set()}
     assert s.who_wants == {y.key: {c.id}}
@@ -2190,32 +2185,32 @@ def test_multi_garbage_collection(s, a, b):
     y2.__del__()
     start = time()
     while y.key in a.data or y.key in b.data:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 5
 
     assert not any(v for v in s.wants_what.values())
     assert not s.who_wants
 
-    yield c.close()
-    yield f.close()
+    await c.close()
+    await f.close()
 
 
 @gen_cluster(client=True)
-def test__broadcast(c, s, a, b):
-    x, y = yield c.scatter([1, 2], broadcast=True)
+async def test__broadcast(c, s, a, b):
+    x, y = await c.scatter([1, 2], broadcast=True)
     assert a.data == b.data == {x.key: 1, y.key: 2}
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 4)
-def test__broadcast_integer(c, s, *workers):
-    x, y = yield c.scatter([1, 2], broadcast=2)
+async def test__broadcast_integer(c, s, *workers):
+    x, y = await c.scatter([1, 2], broadcast=2)
     assert len(s.tasks[x.key].who_has) == 2
     assert len(s.tasks[y.key].who_has) == 2
 
 
 @gen_cluster(client=True)
-def test__broadcast_dict(c, s, a, b):
-    d = yield c.scatter({"x": 1}, broadcast=True)
+async def test__broadcast_dict(c, s, a, b):
+    d = await c.scatter({"x": 1}, broadcast=True)
     assert a.data == b.data == {"x": 1}
 
 
@@ -2239,20 +2234,20 @@ def test_broadcast(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_proxy(c, s, a, b):
-    msg = yield c.scheduler.proxy(msg={"op": "identity"}, worker=a.address)
+async def test_proxy(c, s, a, b):
+    msg = await c.scheduler.proxy(msg={"op": "identity"}, worker=a.address)
     assert msg["id"] == a.identity()["id"]
 
 
 @gen_cluster(client=True)
-def test__cancel(c, s, a, b):
+async def test__cancel(c, s, a, b):
     x = c.submit(slowinc, 1)
     y = c.submit(slowinc, x)
 
     while y.key not in s.tasks:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
-    yield c.cancel([x])
+    await c.cancel([x])
 
     assert x.cancelled()
     assert "cancel" in str(x)
@@ -2260,7 +2255,7 @@ def test__cancel(c, s, a, b):
 
     start = time()
     while not y.cancelled():
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 5
 
     assert not s.tasks
@@ -2268,54 +2263,56 @@ def test__cancel(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test__cancel_tuple_key(c, s, a, b):
+async def test_cancel_tuple_key(c, s, a, b):
     x = c.submit(inc, 1, key=("x", 0, 1))
-
-    result = yield x
-    yield c.cancel(x)
+    await x
+    await c.cancel(x)
     with pytest.raises(CancelledError):
-        yield x
+        await x
 
 
 @gen_cluster()
-def test__cancel_multi_client(s, a, b):
-    c = yield Client(s.address, asynchronous=True)
-    f = yield Client(s.address, asynchronous=True)
+async def test_cancel_multi_client(s, a, b):
+    c = await Client(s.address, asynchronous=True)
+    f = await Client(s.address, asynchronous=True)
 
     x = c.submit(slowinc, 1)
     y = f.submit(slowinc, 1)
 
     assert x.key == y.key
 
-    yield c.cancel([x])
+    await c.cancel([x])
 
     assert x.cancelled()
     assert not y.cancelled()
 
     start = time()
     while y.key not in s.tasks:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 5
 
-    out = yield y
+    out = await y
     assert out == 2
 
     with pytest.raises(CancelledError):
-        yield x
+        await x
 
-    yield c.close()
-    yield f.close()
+    await c.close()
+    await f.close()
 
 
 @gen_cluster(client=True)
-def test__cancel_collection(c, s, a, b):
+async def test_cancel_collection(c, s, a, b):
     L = c.map(double, [[1], [2], [3]])
     x = db.Bag({("b", i): f for i, f in enumerate(L)}, "b", 3)
 
-    yield c.cancel(x)
-    yield c.cancel([x])
+    await c.cancel(x)
+    await c.cancel([x])
     assert all(f.cancelled() for f in L)
-    assert not s.tasks
+    start = time()
+    while s.tasks:
+        assert time() < start + 1
+        await asyncio.sleep(0.01)
 
 
 def test_cancel(c):
@@ -2337,18 +2334,18 @@ def test_cancel(c):
 
 
 @gen_cluster(client=True)
-def test_future_type(c, s, a, b):
+async def test_future_type(c, s, a, b):
     x = c.submit(inc, 1)
-    yield wait([x])
+    await wait([x])
     assert x.type == int
     assert "int" in str(x)
 
 
 @gen_cluster(client=True)
-def test_traceback_clean(c, s, a, b):
+async def test_traceback_clean(c, s, a, b):
     x = c.submit(div, 1, 0)
     try:
-        yield x
+        await x
     except Exception as e:
         f = e
         exc_type, exc_value, tb = sys.exc_info()
@@ -2359,7 +2356,7 @@ def test_traceback_clean(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_map_differnet_lengths(c, s, a, b):
+async def test_map_differnet_lengths(c, s, a, b):
     assert len(c.map(add, [1, 2], [1, 2, 3])) == 2
 
 
@@ -2375,7 +2372,7 @@ def test_Future_exception_sync_2(loop, capsys):
 
 
 @gen_cluster(timeout=60, client=True)
-def test_async_persist(c, s, a, b):
+async def test_async_persist(c, s, a, b):
     from dask.delayed import delayed, Delayed
 
     x = delayed(1)
@@ -2393,13 +2390,13 @@ def test_async_persist(c, s, a, b):
     assert w.__dask_keys__() == ww.__dask_keys__()
 
     while y.key not in s.tasks and w.key not in s.tasks:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
     assert s.who_wants[y.key] == {c.id}
     assert s.who_wants[w.key] == {c.id}
 
     yyf, wwf = c.compute([yy, ww])
-    yyy, www = yield c.gather([yyf, wwf])
+    yyy, www = await c.gather([yyf, wwf])
     assert yyy == inc(1)
     assert www == add(inc(1), dec(1))
 
@@ -2408,7 +2405,7 @@ def test_async_persist(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test__persist(c, s, a, b):
+async def test__persist(c, s, a, b):
     pytest.importorskip("dask.array")
     import dask.array as da
 
@@ -2424,7 +2421,7 @@ def test__persist(c, s, a, b):
 
     g, h = c.compute([y, yy])
 
-    gg, hh = yield c.gather([g, h])
+    gg, hh = await c.gather([g, h])
     assert (gg == hh).all()
 
 
@@ -2447,7 +2444,7 @@ def test_persist(c):
 
 
 @gen_cluster(timeout=60, client=True)
-def test_long_traceback(c, s, a, b):
+async def test_long_traceback(c, s, a, b):
     from distributed.protocol.pickle import dumps
 
     def deep(n):
@@ -2457,22 +2454,22 @@ def deep(n):
             return deep(n - 1)
 
     x = c.submit(deep, 200)
-    yield wait([x])
+    await wait([x])
     assert len(dumps(c.futures[x.key].traceback)) < 10000
     assert isinstance(c.futures[x.key].exception, ZeroDivisionError)
 
 
 @gen_cluster(client=True)
-def test_wait_on_collections(c, s, a, b):
+async def test_wait_on_collections(c, s, a, b):
     L = c.map(double, [[1], [2], [3]])
     x = db.Bag({("b", i): f for i, f in enumerate(L)}, "b", 3)
 
-    yield wait(x)
+    await wait(x)
     assert all(f.key in a.data or f.key in b.data for f in L)
 
 
 @gen_cluster(client=True)
-def test_futures_of_get(c, s, a, b):
+async def test_futures_of_get(c, s, a, b):
     x, y, z = c.map(inc, [1, 2, 3])
 
     assert set(futures_of(0)) == set()
@@ -2498,15 +2495,15 @@ def test_futures_of_class():
 
 
 @gen_cluster(client=True)
-def test_futures_of_cancelled_raises(c, s, a, b):
+async def test_futures_of_cancelled_raises(c, s, a, b):
     x = c.submit(inc, 1)
-    yield c.cancel([x])
+    await c.cancel([x])
 
     with pytest.raises(CancelledError):
-        yield x
+        await x
 
     with pytest.raises(CancelledError):
-        yield c.get({"x": (inc, x), "y": (inc, 2)}, ["x", "y"], sync=False)
+        await c.get({"x": (inc, x), "y": (inc, 2)}, ["x", "y"], sync=False)
 
     with pytest.raises(CancelledError):
         c.submit(inc, x)
@@ -2522,69 +2519,69 @@ def test_futures_of_cancelled_raises(c, s, a, b):
 
 @pytest.mark.skip
 @gen_cluster(nthreads=[("127.0.0.1", 1)], client=True)
-def test_dont_delete_recomputed_results(c, s, w):
+async def test_dont_delete_recomputed_results(c, s, w):
     x = c.submit(inc, 1)  # compute first time
-    yield wait([x])
+    await wait([x])
     x.__del__()  # trigger garbage collection
-    yield gen.moment
+    await asyncio.sleep(0)
     xx = c.submit(inc, 1)  # compute second time
 
     start = time()
     while xx.key not in w.data:  # data shows up
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 1
 
     while time() < start + (s.delete_interval + 100) / 1000:  # and stays
         assert xx.key in w.data
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
 
 @gen_cluster(nthreads=[], client=True)
-def test_fatally_serialized_input(c, s):
+async def test_fatally_serialized_input(c, s):
     o = FatallySerializedObject()
 
     future = c.submit(inc, o)
 
     while not s.tasks:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
 
 @pytest.mark.skip(reason="Use fast random selection now")
 @gen_cluster(client=True)
-def test_balance_tasks_by_stacks(c, s, a, b):
+async def test_balance_tasks_by_stacks(c, s, a, b):
     x = c.submit(inc, 1)
-    yield wait(x)
+    await wait(x)
 
     y = c.submit(inc, 2)
-    yield wait(y)
+    await wait(y)
 
     assert len(a.data) == len(b.data) == 1
 
 
 @gen_cluster(client=True)
-def test_run(c, s, a, b):
-    results = yield c.run(inc, 1)
+async def test_run(c, s, a, b):
+    results = await c.run(inc, 1)
     assert results == {a.address: 2, b.address: 2}
 
-    results = yield c.run(inc, 1, workers=[a.address])
+    results = await c.run(inc, 1, workers=[a.address])
     assert results == {a.address: 2}
 
-    results = yield c.run(inc, 1, workers=[])
+    results = await c.run(inc, 1, workers=[])
     assert results == {}
 
 
 @gen_cluster(client=True)
-def test_run_handles_picklable_data(c, s, a, b):
+async def test_run_handles_picklable_data(c, s, a, b):
     futures = c.map(inc, range(10))
-    yield wait(futures)
+    await wait(futures)
 
     def func():
         return {}, set(), [], (), 1, "hello", b"100"
 
-    results = yield c.run_on_scheduler(func)
+    results = await c.run_on_scheduler(func)
     assert results == func()
 
-    results = yield c.run(func)
+    results = await c.run(func)
     assert results == {w.address: func() for w in [a, b]}
 
 
@@ -2600,22 +2597,21 @@ def func(x, y=10):
 
 
 @gen_cluster(client=True)
-def test_run_coroutine(c, s, a, b):
-    results = yield c.run(geninc, 1, delay=0.05)
+async def test_run_coroutine(c, s, a, b):
+    results = await c.run(geninc, 1, delay=0.05)
     assert results == {a.address: 2, b.address: 2}
 
-    results = yield c.run(geninc, 1, delay=0.05, workers=[a.address])
+    results = await c.run(geninc, 1, delay=0.05, workers=[a.address])
     assert results == {a.address: 2}
 
-    results = yield c.run(geninc, 1, workers=[])
+    results = await c.run(geninc, 1, workers=[])
     assert results == {}
 
     with pytest.raises(RuntimeError, match="hello"):
-        yield c.run(throws, 1)
+        await c.run(throws, 1)
 
-    if sys.version_info >= (3, 5):
-        results = yield c.run(asyncinc, 2, delay=0.01)
-        assert results == {a.address: 3, b.address: 3}
+    results = await c.run(asyncinc, 2, delay=0.01)
+    assert results == {a.address: 3, b.address: 3}
 
 
 def test_run_coroutine_sync(c, s, a, b):
@@ -2692,39 +2688,38 @@ def test_diagnostic_nbytes_sync(c):
 
 
 @gen_cluster(client=True)
-def test_diagnostic_nbytes(c, s, a, b):
+async def test_diagnostic_nbytes(c, s, a, b):
     incs = c.map(inc, [1, 2, 3])
     doubles = c.map(double, [1, 2, 3])
-    yield wait(incs + doubles)
+    await wait(incs + doubles)
 
     assert s.get_nbytes(summary=False) == {k.key: sizeof(1) for k in incs + doubles}
     assert s.get_nbytes(summary=True) == {"inc": sizeof(1) * 3, "double": sizeof(1) * 3}
 
 
 @gen_test()
-def test_worker_aliases():
-    s = yield Scheduler(validate=True, port=0)
+async def test_worker_aliases():
+    s = await Scheduler(validate=True, port=0)
     a = Worker(s.address, name="alice")
     b = Worker(s.address, name="bob")
     w = Worker(s.address, name=3)
-    yield [a, b, w]
-
-    c = yield Client(s.address, asynchronous=True)
+    await asyncio.gather(a, b, w)
+    c = await Client(s.address, asynchronous=True)
 
     L = c.map(inc, range(10), workers="alice")
-    future = yield c.scatter(123, workers=3)
-    yield wait(L)
+    future = await c.scatter(123, workers=3)
+    await wait(L)
     assert len(a.data) == 10
     assert len(b.data) == 0
     assert dict(w.data) == {future.key: 123}
 
     for i, alias in enumerate([3, [3], "alice"]):
-        result = yield c.submit(lambda x: x + 1, i, workers=alias)
+        result = await c.submit(lambda x: x + 1, i, workers=alias)
         assert result == i + 1
 
-    yield c.close()
-    yield [a.close(), b.close(), w.close()]
-    yield s.close()
+    await c.close()
+    await asyncio.gather(a.close(), b.close(), w.close())
+    await s.close()
 
 
 def test_persist_get_sync(c):
@@ -2741,7 +2736,7 @@ def test_persist_get_sync(c):
 
 
 @gen_cluster(client=True)
-def test_persist_get(c, s, a, b):
+async def test_persist_get(c, s, a, b):
     dadd = delayed(add)
     x, y = delayed(1), delayed(2)
     xx = delayed(add)(x, x)
@@ -2751,17 +2746,17 @@ def test_persist_get(c, s, a, b):
     xxyy2 = c.persist(xxyy)
     xxyy3 = delayed(add)(xxyy2, 10)
 
-    yield gen.sleep(0.5)
-    result = yield c.get(xxyy3.dask, xxyy3.__dask_keys__(), sync=False)
+    await asyncio.sleep(0.5)
+    result = await c.gather(c.get(xxyy3.dask, xxyy3.__dask_keys__(), sync=False))
     assert result[0] == ((1 + 1) + (2 + 2)) + 10
 
-    result = yield c.compute(xxyy3)
+    result = await c.compute(xxyy3)
     assert result == ((1 + 1) + (2 + 2)) + 10
 
-    result = yield c.compute(xxyy3)
+    result = await c.compute(xxyy3)
     assert result == ((1 + 1) + (2 + 2)) + 10
 
-    result = yield c.compute(xxyy3)
+    result = await c.compute(xxyy3)
     assert result == ((1 + 1) + (2 + 2)) + 10
 
 
@@ -2782,12 +2777,12 @@ def test_client_num_fds(loop):
 
 
 @gen_cluster()
-def test_startup_close_startup(s, a, b):
-    c = yield Client(s.address, asynchronous=True)
-    yield c.close()
+async def test_startup_close_startup(s, a, b):
+    c = await Client(s.address, asynchronous=True)
+    await c.close()
 
-    c = yield Client(s.address, asynchronous=True)
-    yield c.close()
+    c = await Client(s.address, asynchronous=True)
+    await c.close()
 
 
 def test_startup_close_startup_sync(loop):
@@ -2804,7 +2799,7 @@ def test_startup_close_startup_sync(loop):
 
 
 @gen_cluster(client=True)
-def test_badly_serialized_exceptions(c, s, a, b):
+async def test_badly_serialized_exceptions(c, s, a, b):
     def f():
         class BadlySerializedException(Exception):
             def __reduce__(self):
@@ -2815,7 +2810,7 @@ def __reduce__(self):
     x = c.submit(f)
 
     try:
-        result = yield x
+        result = await x
     except Exception as e:
         assert "hello world" in str(e)
     else:
@@ -2823,16 +2818,16 @@ def __reduce__(self):
 
 
 @gen_cluster(client=True)
-def test_rebalance(c, s, a, b):
+async def test_rebalance(c, s, a, b):
     aws = s.workers[a.address]
     bws = s.workers[b.address]
 
-    x, y = yield c.scatter([1, 2], workers=[a.address])
+    x, y = await c.scatter([1, 2], workers=[a.address])
     assert len(a.data) == 2
     assert len(b.data) == 0
 
     s.validate_state()
-    yield c.rebalance()
+    await c.rebalance()
     s.validate_state()
 
     assert len(b.data) == 1
@@ -2845,21 +2840,21 @@ def test_rebalance(c, s, a, b):
 
 
 @gen_cluster(nthreads=[("127.0.0.1", 1)] * 4, client=True)
-def test_rebalance_workers(e, s, a, b, c, d):
-    w, x, y, z = yield e.scatter([1, 2, 3, 4], workers=[a.address])
+async def test_rebalance_workers(e, s, a, b, c, d):
+    w, x, y, z = await e.scatter([1, 2, 3, 4], workers=[a.address])
     assert len(a.data) == 4
     assert len(b.data) == 0
     assert len(c.data) == 0
     assert len(d.data) == 0
 
-    yield e.rebalance([x, y], workers=[a.address, c.address])
+    await e.rebalance([x, y], workers=[a.address, c.address])
     assert len(a.data) == 3
     assert len(b.data) == 0
     assert len(c.data) == 1
     assert len(d.data) == 0
     assert c.data == {x.key: 2} or c.data == {y.key: 3}
 
-    yield e.rebalance()
+    await e.rebalance()
     assert len(a.data) == 1
     assert len(b.data) == 1
     assert len(c.data) == 1
@@ -2868,9 +2863,9 @@ def test_rebalance_workers(e, s, a, b, c, d):
 
 
 @gen_cluster(client=True)
-def test_rebalance_execution(c, s, a, b):
+async def test_rebalance_execution(c, s, a, b):
     futures = c.map(inc, range(10), workers=a.address)
-    yield c.rebalance(futures)
+    await c.rebalance(futures)
     assert len(a.data) == len(b.data) == 5
     s.validate_state()
 
@@ -2885,10 +2880,10 @@ def test_rebalance_sync(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_rebalance_unprepared(c, s, a, b):
+async def test_rebalance_unprepared(c, s, a, b):
     futures = c.map(slowinc, range(10), delay=0.05, workers=a.address)
-    yield gen.sleep(0.1)
-    yield c.rebalance(futures)
+    await asyncio.sleep(0.1)
+    await c.rebalance(futures)
     s.validate_state()
 
 
@@ -2902,66 +2897,63 @@ async def test_rebalance_raises_missing_data(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_receive_lost_key(c, s, a, b):
+async def test_receive_lost_key(c, s, a, b):
     x = c.submit(inc, 1, workers=[a.address])
-    result = yield x
-    yield a.close()
+    await x
+    await a.close()
 
     start = time()
     while x.status == "finished":
         assert time() < start + 5
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
 
 @pytest.mark.skipif(
     not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
 )
 @gen_cluster([("127.0.0.1", 1), ("127.0.0.2", 2)], client=True)
-def test_unrunnable_task_runs(c, s, a, b):
+async def test_unrunnable_task_runs(c, s, a, b):
     x = c.submit(inc, 1, workers=[a.ip])
-    result = yield x
+    await x
 
-    yield a.close()
+    await a.close()
     start = time()
     while x.status == "finished":
         assert time() < start + 5
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
     assert s.tasks[x.key] in s.unrunnable
     assert s.get_task_status(keys=[x.key]) == {x.key: "no-worker"}
 
-    w = yield Worker(s.address, loop=s.loop)
+    w = await Worker(s.address, loop=s.loop)
 
     start = time()
     while x.status != "finished":
         assert time() < start + 2
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
     assert s.tasks[x.key] not in s.unrunnable
-    result = yield x
+    result = await x
     assert result == 2
-    yield w.close()
+    await w.close()
 
 
 @gen_cluster(client=True, nthreads=[])
-def test_add_worker_after_tasks(c, s):
+async def test_add_worker_after_tasks(c, s):
     futures = c.map(inc, range(10))
-
-    n = yield Nanny(s.address, nthreads=2, loop=s.loop, port=0)
-
-    result = yield c.gather(futures)
-
-    yield n.close()
+    n = await Nanny(s.address, nthreads=2, loop=s.loop, port=0)
+    await c.gather(futures)
+    await n.close()
 
 
 @pytest.mark.skipif(
     not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
 )
 @gen_cluster([("127.0.0.1", 1), ("127.0.0.2", 2)], client=True)
-def test_workers_register_indirect_data(c, s, a, b):
-    [x] = yield c.scatter([1], workers=a.address)
+async def test_workers_register_indirect_data(c, s, a, b):
+    [x] = await c.scatter([1], workers=a.address)
     y = c.submit(inc, x, workers=b.ip)
-    yield y
+    await y
     assert b.data[x.key] == 1
     assert s.tasks[x.key].who_has == {s.workers[a.address], s.workers[b.address]}
     assert s.workers[b.address].has_what == {s.tasks[x.key], s.tasks[y.key]}
@@ -2969,20 +2961,20 @@ def test_workers_register_indirect_data(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_submit_on_cancelled_future(c, s, a, b):
+async def test_submit_on_cancelled_future(c, s, a, b):
     x = c.submit(inc, 1)
-    yield x
+    await x
 
-    yield c.cancel(x)
+    await c.cancel(x)
 
     with pytest.raises(CancelledError):
-        y = c.submit(inc, x)
+        c.submit(inc, x)
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 10)
-def test_replicate(c, s, *workers):
-    [a, b] = yield c.scatter([1, 2])
-    yield s.replicate(keys=[a.key, b.key], n=5)
+async def test_replicate(c, s, *workers):
+    [a, b] = await c.scatter([1, 2])
+    await s.replicate(keys=[a.key, b.key], n=5)
     s.validate_state()
 
     assert len(s.tasks[a.key].who_has) == 5
@@ -2993,22 +2985,22 @@ def test_replicate(c, s, *workers):
 
 
 @gen_cluster(client=True)
-def test_replicate_tuple_keys(c, s, a, b):
+async def test_replicate_tuple_keys(c, s, a, b):
     x = delayed(inc)(1, dask_key_name=("x", 1))
     f = c.persist(x)
-    yield c.replicate(f, n=5)
+    await c.replicate(f, n=5)
     s.validate_state()
     assert a.data and b.data
 
-    yield c.rebalance(f)
+    await c.rebalance(f)
     s.validate_state()
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 10)
-def test_replicate_workers(c, s, *workers):
+async def test_replicate_workers(c, s, *workers):
 
-    [a, b] = yield c.scatter([1, 2], workers=[workers[0].address])
-    yield s.replicate(
+    [a, b] = await c.scatter([1, 2], workers=[workers[0].address])
+    await s.replicate(
         keys=[a.key, b.key], n=5, workers=[w.address for w in workers[:5]]
     )
 
@@ -3020,7 +3012,7 @@ def test_replicate_workers(c, s, *workers):
     assert sum(a.key in w.data for w in workers[5:]) == 0
     assert sum(b.key in w.data for w in workers[5:]) == 0
 
-    yield s.replicate(keys=[a.key, b.key], n=1)
+    await s.replicate(keys=[a.key, b.key], n=1)
 
     assert len(s.tasks[a.key].who_has) == 1
     assert len(s.tasks[b.key].who_has) == 1
@@ -3029,12 +3021,12 @@ def test_replicate_workers(c, s, *workers):
 
     s.validate_state()
 
-    yield s.replicate(keys=[a.key, b.key], n=None)  # all
+    await s.replicate(keys=[a.key, b.key], n=None)  # all
     assert len(s.tasks[a.key].who_has) == 10
     assert len(s.tasks[b.key].who_has) == 10
     s.validate_state()
 
-    yield s.replicate(
+    await s.replicate(
         keys=[a.key, b.key], n=1, workers=[w.address for w in workers[:5]]
     )
     assert sum(a.key in w.data for w in workers[:5]) == 1
@@ -3056,30 +3048,30 @@ def __getstate__(self):
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 10)
-def test_replicate_tree_branching(c, s, *workers):
+async def test_replicate_tree_branching(c, s, *workers):
     obj = CountSerialization()
-    [future] = yield c.scatter([obj])
-    yield s.replicate(keys=[future.key], n=10)
+    [future] = await c.scatter([obj])
+    await s.replicate(keys=[future.key], n=10)
 
     max_count = max(w.data[future.key].n for w in workers)
     assert max_count > 1
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 10)
-def test_client_replicate(c, s, *workers):
+async def test_client_replicate(c, s, *workers):
     x = c.submit(inc, 1)
     y = c.submit(inc, 2)
-    yield c.replicate([x, y], n=5)
+    await c.replicate([x, y], n=5)
 
     assert len(s.tasks[x.key].who_has) == 5
     assert len(s.tasks[y.key].who_has) == 5
 
-    yield c.replicate([x, y], n=3)
+    await c.replicate([x, y], n=3)
 
     assert len(s.tasks[x.key].who_has) == 3
     assert len(s.tasks[y.key].who_has) == 3
 
-    yield c.replicate([x, y])
+    await c.replicate([x, y])
     s.validate_state()
 
     assert len(s.tasks[x.key].who_has) == 10
@@ -3094,19 +3086,19 @@ def test_client_replicate(c, s, *workers):
     nthreads=[("127.0.0.1", 1), ("127.0.0.2", 1), ("127.0.0.2", 1)],
     timeout=None,
 )
-def test_client_replicate_host(client, s, a, b, c):
+async def test_client_replicate_host(client, s, a, b, c):
     aws = s.workers[a.address]
     bws = s.workers[b.address]
     cws = s.workers[c.address]
 
     x = client.submit(inc, 1, workers="127.0.0.2")
-    yield wait([x])
+    await wait([x])
     assert s.tasks[x.key].who_has == {bws} or s.tasks[x.key].who_has == {cws}
 
-    yield client.replicate([x], workers=["127.0.0.2"])
+    await client.replicate([x], workers=["127.0.0.2"])
     assert s.tasks[x.key].who_has == {bws, cws}
 
-    yield client.replicate([x], workers=["127.0.0.1"])
+    await client.replicate([x], workers=["127.0.0.1"])
     assert s.tasks[x.key].who_has == {aws, bws, cws}
 
 
@@ -3126,25 +3118,25 @@ def test_client_replicate_sync(c):
 
 @pytest.mark.skipif(WINDOWS, reason="Windows timer too coarse-grained")
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 4)] * 1)
-def test_task_load_adapts_quickly(c, s, a):
+async def test_task_load_adapts_quickly(c, s, a):
     future = c.submit(slowinc, 1, delay=0.2)  # slow
-    yield wait(future)
+    await wait(future)
     assert 0.15 < s.task_prefixes["slowinc"].duration_average < 0.4
 
     futures = c.map(slowinc, range(10), delay=0)  # very fast
-    yield wait(futures)
+    await wait(futures)
 
     assert 0 < s.task_prefixes["slowinc"].duration_average < 0.1
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 2)
-def test_even_load_after_fast_functions(c, s, a, b):
+async def test_even_load_after_fast_functions(c, s, a, b):
     x = c.submit(inc, 1, workers=a.address)  # very fast
     y = c.submit(inc, 2, workers=b.address)  # very fast
-    yield wait([x, y])
+    await wait([x, y])
 
     futures = c.map(inc, range(2, 11))
-    yield wait(futures)
+    await wait(futures)
     assert any(f.key in a.data for f in futures)
     assert any(f.key in b.data for f in futures)
 
@@ -3152,17 +3144,17 @@ def test_even_load_after_fast_functions(c, s, a, b):
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 2)
-def test_even_load_on_startup(c, s, a, b):
+async def test_even_load_on_startup(c, s, a, b):
     x, y = c.map(inc, [1, 2])
-    yield wait([x, y])
+    await wait([x, y])
     assert len(a.data) == len(b.data) == 1
 
 
 @pytest.mark.skip
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 2)] * 2)
-def test_contiguous_load(c, s, a, b):
+async def test_contiguous_load(c, s, a, b):
     w, x, y, z = c.map(inc, [1, 2, 3, 4])
-    yield wait([w, x, y, z])
+    await wait([w, x, y, z])
 
     groups = [set(a.data), set(b.data)]
     assert {w.key, x.key} in groups
@@ -3170,24 +3162,24 @@ def test_contiguous_load(c, s, a, b):
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 4)
-def test_balanced_with_submit(c, s, *workers):
+async def test_balanced_with_submit(c, s, *workers):
     L = [c.submit(slowinc, i) for i in range(4)]
-    yield wait(L)
+    await wait(L)
     for w in workers:
         assert len(w.data) == 1
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 4)
-def test_balanced_with_submit_and_resident_data(c, s, *workers):
-    [x] = yield c.scatter([10], broadcast=True)
+async def test_balanced_with_submit_and_resident_data(c, s, *workers):
+    [x] = await c.scatter([10], broadcast=True)
     L = [c.submit(slowinc, x, pure=False) for i in range(4)]
-    yield wait(L)
+    await wait(L)
     for w in workers:
         assert len(w.data) == 2
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 20)] * 2)
-def test_scheduler_saturates_cores(c, s, a, b):
+async def test_scheduler_saturates_cores(c, s, a, b):
     for delay in [0, 0.01, 0.1]:
         futures = c.map(slowinc, range(100), delay=delay)
         futures = c.map(slowinc, futures, delay=delay / 10)
@@ -3198,11 +3190,11 @@ def test_scheduler_saturates_cores(c, s, a, b):
                     for w in s.workers.values()
                     for p in w.processing.values()
                 )
-            yield gen.sleep(0.01)
+            await asyncio.sleep(0.01)
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 20)] * 2)
-def test_scheduler_saturates_cores_random(c, s, a, b):
+async def test_scheduler_saturates_cores_random(c, s, a, b):
     for delay in [0, 0.01, 0.1]:
         futures = c.map(randominc, range(100), scale=0.1)
         while not s.tasks:
@@ -3212,22 +3204,22 @@ def test_scheduler_saturates_cores_random(c, s, a, b):
                     for w in s.workers.values()
                     for p in w.processing.values()
                 )
-            yield gen.sleep(0.01)
+            await asyncio.sleep(0.01)
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 4)
-def test_cancel_clears_processing(c, s, *workers):
+async def test_cancel_clears_processing(c, s, *workers):
     da = pytest.importorskip("dask.array")
     x = c.submit(slowinc, 1, delay=0.2)
     while not s.tasks:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
-    yield c.cancel(x)
+    await c.cancel(x)
 
     start = time()
     while any(v for w in s.workers.values() for v in w.processing):
         assert time() < start + 0.2
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
     s.validate_state()
 
 
@@ -3270,50 +3262,50 @@ def test_default_get():
 
 
 @gen_cluster(client=True)
-def test_get_processing(c, s, a, b):
-    processing = yield c.processing()
+async def test_get_processing(c, s, a, b):
+    processing = await c.processing()
     assert processing == valmap(tuple, s.processing)
 
     futures = c.map(
         slowinc, range(10), delay=0.1, workers=[a.address], allow_other_workers=True
     )
 
-    yield gen.sleep(0.2)
+    await asyncio.sleep(0.2)
 
-    x = yield c.processing()
+    x = await c.processing()
     assert set(x) == {a.address, b.address}
 
-    x = yield c.processing(workers=[a.address])
+    x = await c.processing(workers=[a.address])
     assert isinstance(x[a.address], (list, tuple))
 
 
 @gen_cluster(client=True)
-def test_get_foo(c, s, a, b):
+async def test_get_foo(c, s, a, b):
     futures = c.map(inc, range(10))
-    yield wait(futures)
+    await wait(futures)
 
-    x = yield c.scheduler.ncores()
+    x = await c.scheduler.ncores()
     assert x == s.nthreads
 
-    x = yield c.scheduler.ncores(workers=[a.address])
+    x = await c.scheduler.ncores(workers=[a.address])
     assert x == {a.address: s.nthreads[a.address]}
 
-    x = yield c.scheduler.has_what()
+    x = await c.scheduler.has_what()
     assert valmap(sorted, x) == valmap(sorted, s.has_what)
 
-    x = yield c.scheduler.has_what(workers=[a.address])
+    x = await c.scheduler.has_what(workers=[a.address])
     assert valmap(sorted, x) == {a.address: sorted(s.has_what[a.address])}
 
-    x = yield c.scheduler.nbytes(summary=False)
+    x = await c.scheduler.nbytes(summary=False)
     assert x == s.get_nbytes(summary=False)
 
-    x = yield c.scheduler.nbytes(keys=[futures[0].key], summary=False)
+    x = await c.scheduler.nbytes(keys=[futures[0].key], summary=False)
     assert x == {futures[0].key: s.tasks[futures[0].key].nbytes}
 
-    x = yield c.scheduler.who_has()
+    x = await c.scheduler.who_has()
     assert valmap(sorted, x) == valmap(sorted, s.who_has)
 
-    x = yield c.scheduler.who_has(keys=[futures[0].key])
+    x = await c.scheduler.who_has(keys=[futures[0].key])
     assert valmap(sorted, x) == {futures[0].key: sorted(s.who_has[futures[0].key])}
 
 
@@ -3326,34 +3318,34 @@ def assert_dict_key_equal(expected, actual):
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 3)
-def test_get_foo_lost_keys(c, s, u, v, w):
+async def test_get_foo_lost_keys(c, s, u, v, w):
     x = c.submit(inc, 1, workers=[u.address])
-    y = yield c.scatter(3, workers=[v.address])
-    yield wait([x, y])
+    y = await c.scatter(3, workers=[v.address])
+    await wait([x, y])
 
     ua, va, wa = u.address, v.address, w.address
 
-    d = yield c.scheduler.has_what()
+    d = await c.scheduler.has_what()
     assert_dict_key_equal(d, {ua: [x.key], va: [y.key], wa: []})
-    d = yield c.scheduler.has_what(workers=[ua, va])
+    d = await c.scheduler.has_what(workers=[ua, va])
     assert_dict_key_equal(d, {ua: [x.key], va: [y.key]})
-    d = yield c.scheduler.who_has()
+    d = await c.scheduler.who_has()
     assert_dict_key_equal(d, {x.key: [ua], y.key: [va]})
-    d = yield c.scheduler.who_has(keys=[x.key, y.key])
+    d = await c.scheduler.who_has(keys=[x.key, y.key])
     assert_dict_key_equal(d, {x.key: [ua], y.key: [va]})
 
-    yield u.close()
-    yield v.close()
+    await u.close()
+    await v.close()
 
-    d = yield c.scheduler.has_what()
+    d = await c.scheduler.has_what()
     assert_dict_key_equal(d, {wa: []})
-    d = yield c.scheduler.has_what(workers=[ua, va])
+    d = await c.scheduler.has_what(workers=[ua, va])
     assert_dict_key_equal(d, {ua: [], va: []})
     # The scattered key cannot be recomputed so it is forgotten
-    d = yield c.scheduler.who_has()
+    d = await c.scheduler.who_has()
     assert_dict_key_equal(d, {x.key: []})
     # ... but when passed explicitly, it is included in the result
-    d = yield c.scheduler.who_has(keys=[x.key, y.key])
+    d = await c.scheduler.who_has(keys=[x.key, y.key])
     assert_dict_key_equal(d, {x.key: [], y.key: []})
 
 
@@ -3361,13 +3353,13 @@ def test_get_foo_lost_keys(c, s, u, v, w):
 @gen_cluster(
     client=True, Worker=Nanny, clean_kwargs={"threads": False, "processes": False}
 )
-def test_bad_tasks_fail(c, s, a, b):
+async def test_bad_tasks_fail(c, s, a, b):
     f = c.submit(sys.exit, 0)
     with pytest.raises(KilledWorker) as info:
-        yield f
+        await f
 
     assert info.value.last_worker.nanny in {a.address, b.address}
-    yield [a.close(), b.close()]
+    await asyncio.gather(a.close(), b.close())
 
 
 def test_get_processing_sync(c, s, a, b):
@@ -3417,11 +3409,11 @@ def test_get_returns_early(c):
 
 @pytest.mark.slow
 @gen_cluster(Worker=Nanny, client=True)
-def test_Client_clears_references_after_restart(c, s, a, b):
+async def test_Client_clears_references_after_restart(c, s, a, b):
     x = c.submit(inc, 1)
     assert x.key in c.refcount
 
-    yield c.restart()
+    await c.restart()
     assert x.key not in c.refcount
 
     key = x.key
@@ -3429,7 +3421,7 @@ def test_Client_clears_references_after_restart(c, s, a, b):
     import gc
 
     gc.collect()
-    yield gen.moment
+    await asyncio.sleep(0)
 
     assert key not in c.refcount
 
@@ -3487,21 +3479,21 @@ def test_as_completed_next_batch(c):
 
 
 @gen_test()
-def test_status():
-    s = yield Scheduler(port=0)
+async def test_status():
+    s = await Scheduler(port=0)
 
-    c = yield Client(s.address, asynchronous=True)
+    c = await Client(s.address, asynchronous=True)
     assert c.status == "running"
     x = c.submit(inc, 1)
 
-    yield c.close()
+    await c.close()
     assert c.status == "closed"
 
-    yield s.close()
+    await s.close()
 
 
 @gen_cluster(client=True)
-def test_persist_optimize_graph(c, s, a, b):
+async def test_persist_optimize_graph(c, s, a, b):
     i = 10
     for method in [c.persist, c.compute]:
         b = db.range(i, npartitions=2)
@@ -3510,7 +3502,7 @@ def test_persist_optimize_graph(c, s, a, b):
         b3 = b2.map(inc)
 
         b4 = method(b3, optimize_graph=False)
-        yield wait(b4)
+        await wait(b4)
 
         assert set(map(tokey, b3.__dask_keys__())).issubset(s.tasks)
 
@@ -3520,15 +3512,15 @@ def test_persist_optimize_graph(c, s, a, b):
         b3 = b2.map(inc)
 
         b4 = method(b3, optimize_graph=True)
-        yield wait(b4)
+        await wait(b4)
 
         assert not any(tokey(k) in s.tasks for k in b2.__dask_keys__())
 
 
 @gen_cluster(client=True, nthreads=[])
-def test_scatter_raises_if_no_workers(c, s):
+async def test_scatter_raises_if_no_workers(c, s):
     with pytest.raises(TimeoutError):
-        yield c.scatter(1, timeout=0.5)
+        await c.scatter(1, timeout=0.5)
 
 
 @pytest.mark.slow
@@ -3593,13 +3585,13 @@ def test_reconnect(loop):
 
 
 @gen_cluster(client=True, nthreads=[], client_kwargs={"timeout": 0.5})
-def test_reconnect_timeout(c, s):
+async def test_reconnect_timeout(c, s):
     with captured_logger(logging.getLogger("distributed.client")) as logger:
-        yield s.close()
+        await s.close()
         start = time()
         while c.status != "closed":
-            yield c._update_scheduler_info()
-            yield gen.sleep(0.05)
+            await c._update_scheduler_info()
+            await asyncio.sleep(0.05)
             assert time() < start + 5, "Timeout waiting for reconnect to fail"
     text = logger.getvalue()
     assert "Failed to reconnect" in text
@@ -3621,22 +3613,21 @@ def test_open_close_many_workers(loop, worker, count, repeat):
         workers = set()
         status = True
 
-        @gen.coroutine
-        def start_worker(sleep, duration, repeat=1):
+        async def start_worker(sleep, duration, repeat=1):
             for i in range(repeat):
-                yield gen.sleep(sleep)
+                await asyncio.sleep(sleep)
                 if not status:
                     return
                 w = worker(s["address"], loop=loop)
                 running[w] = None
                 workers.add(w)
-                yield w
+                await w
                 addr = w.worker_address
                 running[w] = addr
-                yield gen.sleep(duration)
-                yield w.close()
+                await asyncio.sleep(duration)
+                await w.close()
                 del w
-                yield gen.moment
+                await asyncio.sleep(0)
             done.release()
 
         for i in range(count):
@@ -3672,34 +3663,34 @@ def start_worker(sleep, duration, repeat=1):
 
 
 @gen_cluster(client=False, timeout=None)
-def test_idempotence(s, a, b):
-    c = yield Client(s.address, asynchronous=True)
-    f = yield Client(s.address, asynchronous=True)
+async def test_idempotence(s, a, b):
+    c = await Client(s.address, asynchronous=True)
+    f = await Client(s.address, asynchronous=True)
 
     # Submit
     x = c.submit(inc, 1)
-    yield x
+    await x
     log = list(s.transition_log)
 
     len_single_submit = len(log)  # see last assert
 
     y = f.submit(inc, 1)
     assert x.key == y.key
-    yield y
-    yield gen.sleep(0.1)
+    await y
+    await asyncio.sleep(0.1)
     log2 = list(s.transition_log)
     assert log == log2
 
     # Error
     a = c.submit(div, 1, 0)
-    yield wait(a)
+    await wait(a)
     assert a.status == "error"
     log = list(s.transition_log)
 
     b = f.submit(div, 1, 0)
     assert a.key == b.key
-    yield wait(b)
-    yield gen.sleep(0.1)
+    await wait(b)
+    await asyncio.sleep(0.1)
     log2 = list(s.transition_log)
     assert log == log2
 
@@ -3707,12 +3698,12 @@ def test_idempotence(s, a, b):
     # Simultaneous Submit
     d = c.submit(inc, 2)
     e = c.submit(inc, 2)
-    yield wait([d, e])
+    await wait([d, e])
 
     assert len(s.transition_log) == len_single_submit
 
-    yield c.close()
-    yield f.close()
+    await c.close()
+    await f.close()
 
 
 def test_scheduler_info(c):
@@ -3771,40 +3762,40 @@ def f():
 
 
 @gen_cluster(client=True)
-def test_lose_scattered_data(c, s, a, b):
-    [x] = yield c.scatter([1], workers=a.address)
+async def test_lose_scattered_data(c, s, a, b):
+    [x] = await c.scatter([1], workers=a.address)
 
-    yield a.close()
-    yield gen.sleep(0.1)
+    await a.close()
+    await asyncio.sleep(0.1)
 
     assert x.status == "cancelled"
     assert x.key not in s.tasks
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 3)
-def test_partially_lose_scattered_data(e, s, a, b, c):
-    x = yield e.scatter(1, workers=a.address)
-    yield e.replicate(x, n=2)
+async def test_partially_lose_scattered_data(e, s, a, b, c):
+    x = await e.scatter(1, workers=a.address)
+    await e.replicate(x, n=2)
 
-    yield a.close()
-    yield gen.sleep(0.1)
+    await a.close()
+    await asyncio.sleep(0.1)
 
     assert x.status == "finished"
     assert s.get_task_status(keys=[x.key]) == {x.key: "memory"}
 
 
 @gen_cluster(client=True)
-def test_scatter_compute_lose(c, s, a, b):
-    [x] = yield c.scatter([[1, 2, 3, 4]], workers=a.address)
+async def test_scatter_compute_lose(c, s, a, b):
+    [x] = await c.scatter([[1, 2, 3, 4]], workers=a.address)
     y = c.submit(inc, 1, workers=b.address)
 
     z = c.submit(slowadd, x, y, delay=0.2)
-    yield gen.sleep(0.1)
+    await asyncio.sleep(0.1)
 
-    yield a.close()
+    await a.close()
 
     with pytest.raises(CancelledError):
-        yield wait(z)
+        await wait(z)
 
     assert x.status == "cancelled"
     assert y.status == "finished"
@@ -3812,7 +3803,7 @@ def test_scatter_compute_lose(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_scatter_compute_store_lose(c, s, a, b):
+async def test_scatter_compute_store_lose(c, s, a, b):
     """
     Create irreplaceable data on one machine,
     cause a dependent computation to occur on another and complete
@@ -3820,18 +3811,18 @@ def test_scatter_compute_store_lose(c, s, a, b):
     Kill the machine with the irreplaceable data.  What happens to the complete
     result?  How about after it GCs and tries to come back?
     """
-    x = yield c.scatter(1, workers=a.address)
+    x = await c.scatter(1, workers=a.address)
     xx = c.submit(inc, x, workers=a.address)
     y = c.submit(inc, 1)
 
     z = c.submit(slowadd, xx, y, delay=0.2, workers=b.address)
-    yield wait(z)
+    await wait(z)
 
-    yield a.close()
+    await a.close()
 
     start = time()
     while x.status == "finished":
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 2
 
     # assert xx.status == 'finished'
@@ -3839,14 +3830,14 @@ def test_scatter_compute_store_lose(c, s, a, b):
     assert z.status == "finished"
 
     zz = c.submit(inc, z)
-    yield wait(zz)
+    await wait(zz)
 
     zkey = z.key
     del z
 
     start = time()
     while s.get_task_status(keys=[zkey]) != {zkey: "released"}:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 2
 
     xxkey = xx.key
@@ -3854,12 +3845,12 @@ def test_scatter_compute_store_lose(c, s, a, b):
 
     start = time()
     while x.key in s.tasks and zkey not in s.tasks and xxkey not in s.tasks:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 2
 
 
 @gen_cluster(client=True)
-def test_scatter_compute_store_lose_processing(c, s, a, b):
+async def test_scatter_compute_store_lose_processing(c, s, a, b):
     """
     Create irreplaceable data on one machine,
     cause a dependent computation to occur on another and complete
@@ -3867,16 +3858,16 @@ def test_scatter_compute_store_lose_processing(c, s, a, b):
     Kill the machine with the irreplaceable data.  What happens to the complete
     result?  How about after it GCs and tries to come back?
     """
-    [x] = yield c.scatter([1], workers=a.address)
+    [x] = await c.scatter([1], workers=a.address)
 
     y = c.submit(slowinc, x, delay=0.2)
     z = c.submit(inc, y)
-    yield gen.sleep(0.1)
-    yield a.close()
+    await asyncio.sleep(0.1)
+    await a.close()
 
     start = time()
     while x.status == "finished":
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 2
 
     assert y.status == "cancelled"
@@ -3884,28 +3875,28 @@ def test_scatter_compute_store_lose_processing(c, s, a, b):
 
 
 @gen_cluster(client=False)
-def test_serialize_future(s, a, b):
-    c = yield Client(s.address, asynchronous=True)
-    f = yield Client(s.address, asynchronous=True)
+async def test_serialize_future(s, a, b):
+    c = await Client(s.address, asynchronous=True)
+    f = await Client(s.address, asynchronous=True)
 
     future = c.submit(lambda: 1)
-    result = yield future
+    result = await future
 
     with temp_default_client(f):
         future2 = pickle.loads(pickle.dumps(future))
         assert future2.client is f
         assert tokey(future2.key) in f.futures
-        result2 = yield future2
+        result2 = await future2
         assert result == result2
 
-    yield c.close()
-    yield f.close()
+    await c.close()
+    await f.close()
 
 
 @gen_cluster(client=False)
-def test_temp_client(s, a, b):
-    c = yield Client(s.address, asynchronous=True)
-    f = yield Client(s.address, asynchronous=True)
+async def test_temp_client(s, a, b):
+    c = await Client(s.address, asynchronous=True)
+    f = await Client(s.address, asynchronous=True)
 
     with temp_default_client(c):
         assert default_client() is c
@@ -3915,13 +3906,13 @@ def test_temp_client(s, a, b):
         assert default_client() is f
         assert default_client(c) is c
 
-    yield c.close()
-    yield f.close()
+    await c.close()
+    await f.close()
 
 
 @nodebug  # test timing is fragile
 @gen_cluster(nthreads=[("127.0.0.1", 1)] * 3, client=True)
-def test_persist_workers(e, s, a, b, c):
+async def test_persist_workers(e, s, a, b, c):
     L1 = [delayed(inc)(i) for i in range(4)]
     total = delayed(sum)(L1)
     L2 = [delayed(add)(i, total) for i in L1]
@@ -3938,7 +3929,7 @@ def test_persist_workers(e, s, a, b, c):
         allow_other_workers=L2 + [total2],
     )
 
-    yield wait(out)
+    await wait(out)
     assert all(v.key in a.data for v in L1)
     assert total.key in b.data
 
@@ -3946,7 +3937,7 @@ def test_persist_workers(e, s, a, b, c):
 
 
 @gen_cluster(nthreads=[("127.0.0.1", 1)] * 3, client=True)
-def test_compute_workers(e, s, a, b, c):
+async def test_compute_workers(e, s, a, b, c):
     L1 = [delayed(inc)(i) for i in range(4)]
     total = delayed(sum)(L1)
     L2 = [delayed(add)(i, total) for i in L1]
@@ -3957,7 +3948,7 @@ def test_compute_workers(e, s, a, b, c):
         allow_other_workers=L1 + [total],
     )
 
-    yield wait(out)
+    await wait(out)
     for v in L1:
         assert s.worker_restrictions[v.key] == {a.address}
     for v in L2:
@@ -3968,13 +3959,13 @@ def test_compute_workers(e, s, a, b, c):
 
 
 @gen_cluster(client=True)
-def test_compute_nested_containers(c, s, a, b):
+async def test_compute_nested_containers(c, s, a, b):
     da = pytest.importorskip("dask.array")
     np = pytest.importorskip("numpy")
     x = da.ones(10, chunks=(5,)) + 1
 
     future = c.compute({"x": [x], "y": 123})
-    result = yield future
+    result = await future
 
     assert isinstance(result, dict)
     assert (result["x"][0] == np.ones(10) + 1).all()
@@ -4004,19 +3995,19 @@ def test_get_restrictions():
 
 
 @gen_cluster(client=True)
-def test_scatter_type(c, s, a, b):
-    [future] = yield c.scatter([1])
+async def test_scatter_type(c, s, a, b):
+    [future] = await c.scatter([1])
     assert future.type == int
 
-    d = yield c.scatter({"x": 1.0})
+    d = await c.scatter({"x": 1.0})
     assert d["x"].type == float
 
 
 @gen_cluster(client=True)
-def test_retire_workers_2(c, s, a, b):
-    [x] = yield c.scatter([1], workers=a.address)
+async def test_retire_workers_2(c, s, a, b):
+    [x] = await c.scatter([1], workers=a.address)
 
-    yield s.retire_workers(workers=[a.address])
+    await s.retire_workers(workers=[a.address])
     assert b.data == {x.key: 1}
     assert s.who_has == {x.key: {b.address}}
     assert s.has_what == {b.address: {x.key}}
@@ -4025,16 +4016,16 @@ def test_retire_workers_2(c, s, a, b):
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 10)
-def test_retire_many_workers(c, s, *workers):
-    futures = yield c.scatter(list(range(100)))
+async def test_retire_many_workers(c, s, *workers):
+    futures = await c.scatter(list(range(100)))
 
-    yield s.retire_workers(workers=[w.address for w in workers[:7]])
+    await s.retire_workers(workers=[w.address for w in workers[:7]])
 
-    results = yield c.gather(futures)
+    results = await c.gather(futures)
     assert results == list(range(100))
 
     while len(s.workers) != 3:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
     assert len(s.has_what) == len(s.nthreads) == 3
 
@@ -4049,19 +4040,19 @@ def test_retire_many_workers(c, s, *workers):
     nthreads=[("127.0.0.1", 3)] * 2,
     config={"distributed.scheduler.default-task-durations": {"f": "10ms"}},
 )
-def test_weight_occupancy_against_data_movement(c, s, a, b):
+async def test_weight_occupancy_against_data_movement(c, s, a, b):
     s.extensions["stealing"]._pc.callback_time = 1000000
 
     def f(x, y=0, z=0):
         sleep(0.01)
         return x
 
-    y = yield c.scatter([[1, 2, 3, 4]], workers=[a.address])
-    z = yield c.scatter([1], workers=[b.address])
+    y = await c.scatter([[1, 2, 3, 4]], workers=[a.address])
+    z = await c.scatter([1], workers=[b.address])
 
     futures = c.map(f, [1, 2, 3, 4], y=y, z=z)
 
-    yield wait(futures)
+    await wait(futures)
 
     assert sum(f.key in a.data for f in futures) >= 2
     assert sum(f.key in b.data for f in futures) >= 1
@@ -4072,24 +4063,24 @@ def f(x, y=0, z=0):
     nthreads=[("127.0.0.1", 1), ("127.0.0.1", 10)],
     config={"distributed.scheduler.default-task-durations": {"f": "10ms"}},
 )
-def test_distribute_tasks_by_nthreads(c, s, a, b):
+async def test_distribute_tasks_by_nthreads(c, s, a, b):
     s.extensions["stealing"]._pc.callback_time = 1000000
 
     def f(x, y=0):
         sleep(0.01)
         return x
 
-    y = yield c.scatter([1], broadcast=True)
+    y = await c.scatter([1], broadcast=True)
 
     futures = c.map(f, range(20), y=y)
 
-    yield wait(futures)
+    await wait(futures)
 
     assert len(b.data) > 2 * len(a.data)
 
 
 @gen_cluster(client=True, clean_kwargs={"threads": False})
-def test_add_done_callback(c, s, a, b):
+async def test_add_done_callback(c, s, a, b):
     S = set()
 
     def f(future):
@@ -4106,19 +4097,19 @@ def g(future):
     v.add_done_callback(f)
     w.add_done_callback(f)
 
-    yield wait((u, v, w, x))
+    await wait((u, v, w, x))
 
     x.add_done_callback(f)
 
     t = time()
     while len(S) < 4 and time() - t < 2.0:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
     assert S == {(f.key, f.status) for f in (u, v, w, x)}
 
 
 @gen_cluster(client=True)
-def test_normalize_collection(c, s, a, b):
+async def test_normalize_collection(c, s, a, b):
     x = delayed(inc)(1)
     y = delayed(inc)(x)
     z = delayed(inc)(y)
@@ -4133,7 +4124,7 @@ def test_normalize_collection(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_normalize_collection_dask_array(c, s, a, b):
+async def test_normalize_collection_dask_array(c, s, a, b):
     da = pytest.importorskip("dask.array")
 
     x = da.ones(10, chunks=(5,))
@@ -4151,8 +4142,8 @@ def test_normalize_collection_dask_array(c, s, a, b):
     for k, v in yy.dask.items():
         assert zz.dask[k].key == v.key
 
-    result1 = yield c.compute(z)
-    result2 = yield c.compute(zz)
+    result1 = await c.compute(z)
+    result2 = await c.compute(zz)
     assert result1 == result2
 
 
@@ -4175,7 +4166,7 @@ def test_normalize_collection_with_released_futures(c):
 
 
 @gen_cluster(client=True)
-def test_auto_normalize_collection(c, s, a, b):
+async def test_auto_normalize_collection(c, s, a, b):
     da = pytest.importorskip("dask.array")
 
     x = da.ones(10, chunks=5)
@@ -4185,17 +4176,17 @@ def test_auto_normalize_collection(c, s, a, b):
         y = x.map_blocks(slowinc, delay=1, dtype=x.dtype)
         yy = c.persist(y)
 
-        yield wait(yy)
+        await wait(yy)
 
         start = time()
         future = c.compute(y.sum())
-        yield future
+        await future
         end = time()
         assert end - start < 1
 
         start = time()
         z = c.persist(y + 1)
-        yield wait(z)
+        await wait(z)
         end = time()
         assert end - start < 1
 
@@ -4224,7 +4215,7 @@ def assert_no_data_loss(scheduler):
 
 
 @gen_cluster(client=True, timeout=None)
-def test_interleave_computations(c, s, a, b):
+async def test_interleave_computations(c, s, a, b):
     import distributed
 
     distributed.g = s
@@ -4238,14 +4229,14 @@ def test_interleave_computations(c, s, a, b):
 
     done = ("memory", "released")
 
-    yield gen.sleep(0.1)
+    await asyncio.sleep(0.1)
 
     x_keys = [x.key for x in xs]
     y_keys = [y.key for y in ys]
     z_keys = [z.key for z in zs]
 
     while not s.tasks or any(w.processing for w in s.workers.values()):
-        yield gen.sleep(0.05)
+        await asyncio.sleep(0.05)
         x_done = sum(state in done for state in s.get_task_status(keys=x_keys).values())
         y_done = sum(state in done for state in s.get_task_status(keys=y_keys).values())
         z_done = sum(state in done for state in s.get_task_status(keys=z_keys).values())
@@ -4259,7 +4250,7 @@ def test_interleave_computations(c, s, a, b):
 
 @pytest.mark.skip(reason="Now prefer first-in-first-out")
 @gen_cluster(client=True, timeout=None)
-def test_interleave_computations_map(c, s, a, b):
+async def test_interleave_computations_map(c, s, a, b):
     xs = c.map(slowinc, range(30), delay=0.02)
     ys = c.map(slowdec, xs, delay=0.02)
     zs = c.map(slowadd, xs, ys, delay=0.02)
@@ -4271,7 +4262,7 @@ def test_interleave_computations_map(c, s, a, b):
     z_keys = [z.key for z in zs]
 
     while not s.tasks or any(w.processing for w in s.workers.values()):
-        yield gen.sleep(0.05)
+        await asyncio.sleep(0.05)
         x_done = sum(state in done for state in s.get_task_status(keys=x_keys).values())
         y_done = sum(state in done for state in s.get_task_status(keys=y_keys).values())
         z_done = sum(state in done for state in s.get_task_status(keys=z_keys).values())
@@ -4282,78 +4273,78 @@ def test_interleave_computations_map(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_scatter_dict_workers(c, s, a, b):
-    yield c.scatter({"a": 10}, workers=[a.address, b.address])
+async def test_scatter_dict_workers(c, s, a, b):
+    await c.scatter({"a": 10}, workers=[a.address, b.address])
     assert "a" in a.data or "a" in b.data
 
 
 @pytest.mark.slow
 @gen_test()
-def test_client_timeout():
+async def test_client_timeout():
     c = Client("127.0.0.1:57484", asynchronous=True)
 
     s = Scheduler(loop=c.loop, port=57484)
-    yield gen.sleep(4)
+    await asyncio.sleep(4)
     try:
-        yield s
+        await s
     except EnvironmentError:  # port in use
-        yield c.close()
+        await c.close()
         return
 
     start = time()
-    yield c
+    await c
     try:
         assert time() < start + 2
     finally:
-        yield c.close()
-        yield s.close()
+        await c.close()
+        await s.close()
 
 
 @gen_cluster(client=True)
-def test_submit_list_kwargs(c, s, a, b):
-    futures = yield c.scatter([1, 2, 3])
+async def test_submit_list_kwargs(c, s, a, b):
+    futures = await c.scatter([1, 2, 3])
 
     def f(L=None):
         return sum(L)
 
     future = c.submit(f, L=futures)
-    result = yield future
+    result = await future
     assert result == 1 + 2 + 3
 
 
 @gen_cluster(client=True)
-def test_map_list_kwargs(c, s, a, b):
-    futures = yield c.scatter([1, 2, 3])
+async def test_map_list_kwargs(c, s, a, b):
+    futures = await c.scatter([1, 2, 3])
 
     def f(i, L=None):
         return i + sum(L)
 
     futures = c.map(f, range(10), L=futures)
-    results = yield c.gather(futures)
+    results = await c.gather(futures)
     assert results == [i + 6 for i in range(10)]
 
 
 @gen_cluster(client=True)
-def test_dont_clear_waiting_data(c, s, a, b):
+async def test_dont_clear_waiting_data(c, s, a, b):
     start = time()
-    x = yield c.scatter(1)
+    x = await c.scatter(1)
     y = c.submit(slowinc, x, delay=0.5)
     while y.key not in s.tasks:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
     key = x.key
     del x
     for i in range(5):
         assert s.waiting_data[key]
-        yield gen.moment
+        await asyncio.sleep(0)
 
 
 @gen_cluster(client=True)
-def test_get_future_error_simple(c, s, a, b):
+async def test_get_future_error_simple(c, s, a, b):
     f = c.submit(div, 1, 0)
-    yield wait(f)
+    await wait(f)
     assert f.status == "error"
 
-    function, args, kwargs, deps = yield c._get_futures_error(f)
+    function, args, kwargs, deps = await c._get_futures_error(f)
     # args contains only solid values, not keys
     assert function.__name__ == "div"
     with pytest.raises(ZeroDivisionError):
@@ -4361,7 +4352,7 @@ def test_get_future_error_simple(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_get_futures_error(c, s, a, b):
+async def test_get_futures_error(c, s, a, b):
     x0 = delayed(dec)(2, dask_key_name="x0")
     y0 = delayed(dec)(1, dask_key_name="y0")
     x = delayed(div)(1, x0, dask_key_name="x")
@@ -4369,16 +4360,16 @@ def test_get_futures_error(c, s, a, b):
     tot = delayed(sum)(x, y, dask_key_name="tot")
 
     f = c.compute(tot)
-    yield wait(f)
+    await wait(f)
     assert f.status == "error"
 
-    function, args, kwargs, deps = yield c._get_futures_error(f)
+    function, args, kwargs, deps = await c._get_futures_error(f)
     assert function.__name__ == "div"
     assert args == (1, y0.key)
 
 
 @gen_cluster(client=True)
-def test_recreate_error_delayed(c, s, a, b):
+async def test_recreate_error_delayed(c, s, a, b):
     x0 = delayed(dec)(2)
     y0 = delayed(dec)(1)
     x = delayed(div)(1, x0)
@@ -4389,7 +4380,7 @@ def test_recreate_error_delayed(c, s, a, b):
 
     assert f.status == "pending"
 
-    function, args, kwargs = yield c._recreate_error_locally(f)
+    function, args, kwargs = await c._recreate_error_locally(f)
     assert f.status == "error"
     assert function.__name__ == "div"
     assert args == (1, 0)
@@ -4398,7 +4389,7 @@ def test_recreate_error_delayed(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_recreate_error_futures(c, s, a, b):
+async def test_recreate_error_futures(c, s, a, b):
     x0 = c.submit(dec, 2)
     y0 = c.submit(dec, 1)
     x = c.submit(div, 1, x0)
@@ -4408,7 +4399,7 @@ def test_recreate_error_futures(c, s, a, b):
 
     assert f.status == "pending"
 
-    function, args, kwargs = yield c._recreate_error_locally(f)
+    function, args, kwargs = await c._recreate_error_locally(f)
     assert f.status == "error"
     assert function.__name__ == "div"
     assert args == (1, 0)
@@ -4417,13 +4408,13 @@ def test_recreate_error_futures(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_recreate_error_collection(c, s, a, b):
+async def test_recreate_error_collection(c, s, a, b):
     b = db.range(10, npartitions=4)
     b = b.map(lambda x: 1 / x)
     b = b.persist()
     f = c.compute(b)
 
-    function, args, kwargs = yield c._recreate_error_locally(f)
+    function, args, kwargs = await c._recreate_error_locally(f)
     with pytest.raises(ZeroDivisionError):
         function(*args, **kwargs)
 
@@ -4440,24 +4431,24 @@ def make_err(x):
 
     df2 = df.a.map(make_err)
     f = c.compute(df2)
-    function, args, kwargs = yield c._recreate_error_locally(f)
+    function, args, kwargs = await c._recreate_error_locally(f)
     with pytest.raises(ValueError):
         function(*args, **kwargs)
 
     # with persist
     df3 = c.persist(df2)
-    function, args, kwargs = yield c._recreate_error_locally(df3)
+    function, args, kwargs = await c._recreate_error_locally(df3)
     with pytest.raises(ValueError):
         function(*args, **kwargs)
 
 
 @gen_cluster(client=True)
-def test_recreate_error_array(c, s, a, b):
+async def test_recreate_error_array(c, s, a, b):
     da = pytest.importorskip("dask.array")
     pytest.importorskip("scipy")
     z = (da.linalg.inv(da.zeros((10, 10), chunks=10)) + 1).sum()
     zz = z.persist()
-    func, args, kwargs = yield c._recreate_error_locally(zz)
+    func, args, kwargs = await c._recreate_error_locally(zz)
     assert "0.,0.,0." in str(args).replace(" ", "")  # args contain actual arrays
 
 
@@ -4481,14 +4472,14 @@ def test_recreate_error_not_error(c):
 
 
 @gen_cluster(client=True)
-def test_retire_workers(c, s, a, b):
+async def test_retire_workers(c, s, a, b):
     assert set(s.workers) == {a.address, b.address}
-    yield c.retire_workers(workers=[a.address], close_workers=True)
+    await c.retire_workers(workers=[a.address], close_workers=True)
     assert set(s.workers) == {b.address}
 
     start = time()
     while a.status != "closed":
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 5
 
 
@@ -4497,7 +4488,7 @@ class MyException(Exception):
 
 
 @gen_cluster(client=True)
-def test_robust_unserializable(c, s, a, b):
+async def test_robust_unserializable(c, s, a, b):
     class Foo:
         def __getstate__(self):
             raise MyException()
@@ -4506,14 +4497,14 @@ def __getstate__(self):
         future = c.submit(identity, Foo())
 
     futures = c.map(inc, range(10))
-    results = yield c.gather(futures)
+    results = await c.gather(futures)
 
     assert results == list(map(inc, range(10)))
     assert a.data and b.data
 
 
 @gen_cluster(client=True)
-def test_robust_undeserializable(c, s, a, b):
+async def test_robust_undeserializable(c, s, a, b):
     class Foo:
         def __getstate__(self):
             return 1
@@ -4523,17 +4514,17 @@ def __setstate__(self, state):
 
     future = c.submit(identity, Foo())
     with pytest.raises(MyException):
-        yield future
+        await future
 
     futures = c.map(inc, range(10))
-    results = yield c.gather(futures)
+    results = await c.gather(futures)
 
     assert results == list(map(inc, range(10)))
     assert a.data and b.data
 
 
 @gen_cluster(client=True)
-def test_robust_undeserializable_function(c, s, a, b):
+async def test_robust_undeserializable_function(c, s, a, b):
     class Foo:
         def __getstate__(self):
             return 1
@@ -4546,17 +4537,17 @@ def __call__(self, *args):
 
     future = c.submit(Foo(), 1)
     with pytest.raises(MyException):
-        yield future
+        await future
 
     futures = c.map(inc, range(10))
-    results = yield c.gather(futures)
+    results = await c.gather(futures)
 
     assert results == list(map(inc, range(10)))
     assert a.data and b.data
 
 
 @gen_cluster(client=True)
-def test_fire_and_forget(c, s, a, b):
+async def test_fire_and_forget(c, s, a, b):
     future = c.submit(slowinc, 1, delay=0.1)
     import distributed
 
@@ -4568,7 +4559,7 @@ def f(x):
 
         start = time()
         while not hasattr(distributed, "foo"):
-            yield gen.sleep(0.01)
+            await asyncio.sleep(0.01)
             assert time() < start + 2
         assert distributed.foo == 123
     finally:
@@ -4576,7 +4567,7 @@ def f(x):
 
     start = time()
     while len(s.tasks) > 1:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 2
 
     assert set(s.who_wants) == {future.key}
@@ -4584,14 +4575,14 @@ def f(x):
 
 
 @gen_cluster(client=True)
-def test_fire_and_forget_err(c, s, a, b):
+async def test_fire_and_forget_err(c, s, a, b):
     fire_and_forget(c.submit(div, 1, 0))
-    yield gen.sleep(0.1)
+    await asyncio.sleep(0.1)
 
     # erred task should clear out quickly
     start = time()
     while s.tasks:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 1
 
 
@@ -4627,16 +4618,16 @@ def test_quiet_client_close_when_cluster_is_closed_before_client(loop):
 
 
 @gen_cluster()
-def test_close(s, a, b):
-    c = yield Client(s.address, asynchronous=True)
+async def test_close(s, a, b):
+    c = await Client(s.address, asynchronous=True)
     future = c.submit(inc, 1)
-    yield wait(future)
+    await wait(future)
     assert c.id in s.wants_what
-    yield c.close()
+    await c.close()
 
     start = time()
     while c.id in s.wants_what or s.tasks:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 5
 
 
@@ -4699,7 +4690,7 @@ def f(_):
 
 
 @gen_cluster(client=True)
-def test_identity(c, s, a, b):
+async def test_identity(c, s, a, b):
     assert c.id.lower().startswith("client")
     assert a.id.lower().startswith("worker")
     assert b.id.lower().startswith("worker")
@@ -4707,7 +4698,7 @@ def test_identity(c, s, a, b):
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 4)] * 2)
-def test_get_client(c, s, a, b):
+async def test_get_client(c, s, a, b):
     assert get_client() is c
     assert c.asynchronous
 
@@ -4725,7 +4716,7 @@ def f(x):
     distributed.tmp_client = c
     try:
         futures = c.map(f, range(5))
-        results = yield c.gather(futures)
+        results = await c.gather(futures)
         assert results == list(map(inc, range(5)))
     finally:
         del distributed.tmp_client
@@ -4742,7 +4733,7 @@ def test_get_client_no_cluster():
 
 
 @gen_cluster(client=True)
-def test_serialize_collections(c, s, a, b):
+async def test_serialize_collections(c, s, a, b):
     da = pytest.importorskip("dask.array")
     x = da.arange(10, chunks=(5,)).persist()
 
@@ -4751,24 +4742,24 @@ def f(x):
         return x.sum().compute()
 
     future = c.submit(f, x)
-    result = yield future
+    result = await future
     assert result == sum(range(10))
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 1, timeout=100)
-def test_secede_simple(c, s, a):
+async def test_secede_simple(c, s, a):
     def f():
         client = get_client()
         secede()
         return client.submit(inc, 1).result()
 
-    result = yield c.submit(f)
+    result = await c.submit(f)
     assert result == 2
 
 
 @pytest.mark.slow
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 2, timeout=60)
-def test_secede_balances(c, s, a, b):
+async def test_secede_balances(c, s, a, b):
     count = threading.active_count()
 
     def f(x):
@@ -4782,24 +4773,24 @@ def f(x):
     futures = c.map(f, range(100))
     start = time()
     while not all(f.status == "finished" for f in futures):
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert threading.active_count() < count + 50
 
     assert len(a.log) < 2 * len(b.log)
     assert len(b.log) < 2 * len(a.log)
 
-    results = yield c.gather(futures)
+    results = await c.gather(futures)
     assert results == [sum(map(inc, range(10)))] * 100
 
 
 @gen_cluster(client=True)
-def test_sub_submit_priority(c, s, a, b):
+async def test_sub_submit_priority(c, s, a, b):
     def f():
         client = get_client()
         client.submit(slowinc, 1, delay=0.2, key="slowinc")
 
     future = c.submit(f, key="f")
-    yield gen.sleep(0.1)
+    await asyncio.sleep(0.1)
     if len(s.tasks) == 2:
         assert (
             s.priorities["f"] > s.priorities["slowinc"]
@@ -4815,17 +4806,17 @@ def test_get_client_sync(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_serialize_collections_of_futures(c, s, a, b):
+async def test_serialize_collections_of_futures(c, s, a, b):
     pd = pytest.importorskip("pandas")
     dd = pytest.importorskip("dask.dataframe")
     from dask.dataframe.utils import assert_eq
 
     df = pd.DataFrame({"x": [1, 2, 3]})
     ddf = dd.from_pandas(df, npartitions=2).persist()
-    future = yield c.scatter(ddf)
+    future = await c.scatter(ddf)
 
-    ddf2 = yield future
-    df2 = yield c.compute(ddf2)
+    ddf2 = await future
+    df2 = await c.compute(ddf2)
 
     assert_eq(df, df2)
 
@@ -4877,10 +4868,10 @@ def test_dynamic_workloads_sync_random(c):
 
 
 @gen_cluster(client=True)
-def test_bytes_keys(c, s, a, b):
+async def test_bytes_keys(c, s, a, b):
     key = b"inc-123"
     future = c.submit(inc, 1, key=key)
-    result = yield future
+    result = await future
     assert type(future.key) is bytes
     assert set(s.tasks) == {key}
     assert key in a.data or key in b.data
@@ -4888,11 +4879,11 @@ def test_bytes_keys(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_unicode_ascii_keys(c, s, a, b):
+async def test_unicode_ascii_keys(c, s, a, b):
     uni_type = type("")
     key = "inc-123"
     future = c.submit(inc, 1, key=key)
-    result = yield future
+    result = await future
     assert type(future.key) is uni_type
     assert set(s.tasks) == {key}
     assert key in a.data or key in b.data
@@ -4900,32 +4891,31 @@ def test_unicode_ascii_keys(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_unicode_keys(c, s, a, b):
+async def test_unicode_keys(c, s, a, b):
     uni_type = type("")
     key = "inc-123\u03bc"
     future = c.submit(inc, 1, key=key)
-    result = yield future
+    result = await future
     assert type(future.key) is uni_type
     assert set(s.tasks) == {key}
     assert key in a.data or key in b.data
     assert result == 2
 
     future2 = c.submit(inc, future)
-    result2 = yield future2
+    result2 = await future2
     assert result2 == 3
 
-    future3 = yield c.scatter({"data-123": 123})
-    result3 = yield future3["data-123"]
+    future3 = await c.scatter({"data-123": 123})
+    result3 = await future3["data-123"]
     assert result3 == 123
 
 
 def test_use_synchronous_client_in_async_context(loop, c):
-    @gen.coroutine
-    def f():
-        x = yield c.scatter(123)
+    async def f():
+        x = await c.scatter(123)
         y = c.submit(inc, x)
-        z = yield c.gather(y)
-        raise gen.Return(z)
+        z = await c.gather(y)
+        return z
 
     z = sync(loop, f)
     assert z == 124
@@ -4956,11 +4946,13 @@ def test_warn_executor(loop, s, a, b):
 
 
 @gen_cluster([("127.0.0.1", 4)] * 2, client=True)
-def test_call_stack_future(c, s, a, b):
+async def test_call_stack_future(c, s, a, b):
     x = c.submit(slowdec, 1, delay=0.5)
     future = c.submit(slowinc, 1, delay=0.5)
-    yield gen.sleep(0.1)
-    results = yield [c.call_stack(future), c.call_stack(keys=[future.key])]
+    await asyncio.sleep(0.1)
+    results = await asyncio.gather(
+        c.call_stack(future), c.call_stack(keys=[future.key])
+    )
     assert all(list(first(result.values())) == [future.key] for result in results)
     assert results[0] == results[1]
     result = results[0]
@@ -4972,11 +4964,11 @@ def test_call_stack_future(c, s, a, b):
 
 
 @gen_cluster([("127.0.0.1", 4)] * 2, client=True)
-def test_call_stack_all(c, s, a, b):
+async def test_call_stack_all(c, s, a, b):
     future = c.submit(slowinc, 1, delay=0.8)
     while not a.executing and not b.executing:
-        yield gen.sleep(0.01)
-    result = yield c.call_stack()
+        await asyncio.sleep(0.01)
+    result = await c.call_stack()
     w = a if a.executing else b
     assert list(result) == [w.address]
     assert list(result[w.address]) == [future.key]
@@ -4984,100 +4976,100 @@ def test_call_stack_all(c, s, a, b):
 
 
 @gen_cluster([("127.0.0.1", 4)] * 2, client=True)
-def test_call_stack_collections(c, s, a, b):
+async def test_call_stack_collections(c, s, a, b):
     da = pytest.importorskip("dask.array")
     x = da.random.random(100, chunks=(10,)).map_blocks(slowinc, delay=0.5).persist()
     while not a.executing and not b.executing:
-        yield gen.sleep(0.001)
-    result = yield c.call_stack(x)
+        await asyncio.sleep(0.001)
+    result = await c.call_stack(x)
     assert result
 
 
 @gen_cluster([("127.0.0.1", 4)] * 2, client=True)
-def test_call_stack_collections_all(c, s, a, b):
+async def test_call_stack_collections_all(c, s, a, b):
     da = pytest.importorskip("dask.array")
     x = da.random.random(100, chunks=(10,)).map_blocks(slowinc, delay=0.5).persist()
     while not a.executing and not b.executing:
-        yield gen.sleep(0.001)
-    result = yield c.call_stack()
+        await asyncio.sleep(0.001)
+    result = await c.call_stack()
     assert result
 
 
 @gen_cluster(client=True, worker_kwargs={"profile_cycle_interval": 100})
-def test_profile(c, s, a, b):
+async def test_profile(c, s, a, b):
     futures = c.map(slowinc, range(10), delay=0.05, workers=a.address)
-    yield wait(futures)
+    await wait(futures)
 
-    x = yield c.profile(start=time() + 10, stop=time() + 20)
+    x = await c.profile(start=time() + 10, stop=time() + 20)
     assert not x["count"]
 
-    x = yield c.profile(start=0, stop=time())
+    x = await c.profile(start=0, stop=time())
     assert (
         x["count"]
         == sum(p["count"] for _, p in a.profile_history) + a.profile_recent["count"]
     )
 
-    y = yield c.profile(start=time() - 0.300, stop=time())
+    y = await c.profile(start=time() - 0.300, stop=time())
     assert 0 < y["count"] < x["count"]
 
     assert not any(p["count"] for _, p in b.profile_history)
-    result = yield c.profile(workers=b.address)
+    result = await c.profile(workers=b.address)
     assert not result["count"]
 
 
 @gen_cluster(client=True, worker_kwargs={"profile_cycle_interval": 100})
-def test_profile_keys(c, s, a, b):
+async def test_profile_keys(c, s, a, b):
     x = c.map(slowinc, range(10), delay=0.05, workers=a.address)
     y = c.map(slowdec, range(10), delay=0.05, workers=a.address)
-    yield wait(x + y)
+    await wait(x + y)
 
-    xp = yield c.profile("slowinc")
-    yp = yield c.profile("slowdec")
-    p = yield c.profile()
+    xp = await c.profile("slowinc")
+    yp = await c.profile("slowdec")
+    p = await c.profile()
 
     assert p["count"] == xp["count"] + yp["count"]
 
     with captured_logger(logging.getLogger("distributed")) as logger:
-        prof = yield c.profile("does-not-exist")
+        prof = await c.profile("does-not-exist")
         assert prof == profile.create()
     out = logger.getvalue()
     assert not out
 
 
 @gen_cluster()
-def test_client_with_name(s, a, b):
+async def test_client_with_name(s, a, b):
     with captured_logger("distributed.scheduler") as sio:
-        client = yield Client(s.address, asynchronous=True, name="foo")
+        client = await Client(s.address, asynchronous=True, name="foo")
         assert "foo" in client.id
-        yield client.close()
+        await client.close()
 
     text = sio.getvalue()
     assert "foo" in text
 
 
 @gen_cluster(client=True)
-def test_future_defaults_to_default_client(c, s, a, b):
+async def test_future_defaults_to_default_client(c, s, a, b):
     x = c.submit(inc, 1)
-    yield wait(x)
+    await wait(x)
 
     future = Future(x.key)
     assert future.client is c
 
 
 @gen_cluster(client=True)
-def test_future_auto_inform(c, s, a, b):
+async def test_future_auto_inform(c, s, a, b):
     x = c.submit(inc, 1)
-    yield wait(x)
+    await wait(x)
 
-    client = yield Client(s.address, asynchronous=True)
+    client = await Client(s.address, asynchronous=True)
     future = Future(x.key, client)
 
     start = time()
     while future.status != "finished":
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 1
 
-    yield client.close()
+    await client.close()
 
 
 def test_client_async_before_loop_starts():
@@ -5089,7 +5081,7 @@ def test_client_async_before_loop_starts():
 
 @pytest.mark.slow
 @gen_cluster(client=True, Worker=Nanny, timeout=60, nthreads=[("127.0.0.1", 3)] * 2)
-def test_nested_compute(c, s, a, b):
+async def test_nested_compute(c, s, a, b):
     def fib(x):
         assert get_worker().get_current_task()
         if x < 2:
@@ -5100,71 +5092,71 @@ def fib(x):
         return c.compute()
 
     future = c.submit(fib, 8)
-    result = yield future
+    result = await future
     assert result == 21
     assert len(s.transition_log) > 50
 
 
 @gen_cluster(client=True)
-def test_task_metadata(c, s, a, b):
-    yield c.set_metadata("x", 1)
-    result = yield c.get_metadata("x")
+async def test_task_metadata(c, s, a, b):
+    await c.set_metadata("x", 1)
+    result = await c.get_metadata("x")
     assert result == 1
 
     future = c.submit(inc, 1)
     key = future.key
-    yield wait(future)
-    yield c.set_metadata(key, 123)
-    result = yield c.get_metadata(key)
+    await wait(future)
+    await c.set_metadata(key, 123)
+    result = await c.get_metadata(key)
     assert result == 123
 
     del future
 
     while key in s.tasks:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
     with pytest.raises(KeyError):
-        yield c.get_metadata(key)
+        await c.get_metadata(key)
 
-    result = yield c.get_metadata(key, None)
+    result = await c.get_metadata(key, None)
     assert result is None
 
-    yield c.set_metadata(["x", "a"], 1)
-    result = yield c.get_metadata("x")
+    await c.set_metadata(["x", "a"], 1)
+    result = await c.get_metadata("x")
     assert result == {"a": 1}
-    yield c.set_metadata(["x", "b"], 2)
-    result = yield c.get_metadata("x")
+    await c.set_metadata(["x", "b"], 2)
+    result = await c.get_metadata("x")
     assert result == {"a": 1, "b": 2}
-    result = yield c.get_metadata(["x", "a"])
+    result = await c.get_metadata(["x", "a"])
     assert result == 1
 
-    yield c.set_metadata(["x", "a", "c", "d"], 1)
-    result = yield c.get_metadata("x")
+    await c.set_metadata(["x", "a", "c", "d"], 1)
+    result = await c.get_metadata("x")
     assert result == {"a": {"c": {"d": 1}}, "b": 2}
 
 
 @gen_cluster(client=True, Worker=Nanny)
-def test_logs(c, s, a, b):
-    yield wait(c.map(inc, range(5)))
-    logs = yield c.get_scheduler_logs(n=5)
+async def test_logs(c, s, a, b):
+    await wait(c.map(inc, range(5)))
+    logs = await c.get_scheduler_logs(n=5)
     assert logs
 
     for _, msg in logs:
         assert "distributed.scheduler" in msg
 
-    w_logs = yield c.get_worker_logs(n=5)
+    w_logs = await c.get_worker_logs(n=5)
     assert set(w_logs.keys()) == {a.worker_address, b.worker_address}
     for log in w_logs.values():
         for _, msg in log:
             assert "distributed.worker" in msg
 
-    n_logs = yield c.get_worker_logs(nanny=True)
+    n_logs = await c.get_worker_logs(nanny=True)
     assert set(n_logs.keys()) == {a.worker_address, b.worker_address}
     for log in n_logs.values():
         for _, msg in log:
             assert "distributed.nanny" in msg
 
-    n_logs = yield c.get_worker_logs(nanny=True, workers=[a.worker_address])
+    n_logs = await c.get_worker_logs(nanny=True, workers=[a.worker_address])
     assert set(n_logs.keys()) == {a.worker_address}
     for log in n_logs.values():
         for _, msg in log:
@@ -5172,29 +5164,29 @@ def test_logs(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_avoid_delayed_finalize(c, s, a, b):
+async def test_avoid_delayed_finalize(c, s, a, b):
     x = delayed(inc)(1)
     future = c.compute(x)
-    result = yield future
+    result = await future
     assert result == 2
     assert list(s.tasks) == [future.key] == [x.key]
 
 
 @gen_cluster()
-def test_config_scheduler_address(s, a, b):
+async def test_config_scheduler_address(s, a, b):
     with dask.config.set({"scheduler-address": s.address}):
         with captured_logger("distributed.client") as sio:
-            c = yield Client(asynchronous=True)
+            c = await Client(asynchronous=True)
             assert c.scheduler.address == s.address
 
         text = sio.getvalue()
         assert s.address in text
 
-        yield c.close()
+        await c.close()
 
 
 @gen_cluster(client=True)
-def test_warn_when_submitting_large_values(c, s, a, b):
+async def test_warn_when_submitting_large_values(c, s, a, b):
     with warnings.catch_warnings(record=True) as record:
         future = c.submit(lambda x: x + 1, b"0" * 2000000)
 
@@ -5215,34 +5207,33 @@ def test_warn_when_submitting_large_values(c, s, a, b):
 
 
 @gen_cluster()
-def test_scatter_direct(s, a, b):
-    c = yield Client(s.address, asynchronous=True, heartbeat_interval=10)
+async def test_scatter_direct(s, a, b):
+    c = await Client(s.address, asynchronous=True, heartbeat_interval=10)
 
     last = s.clients[c.id].last_seen
 
     start = time()
     while s.clients[c.id].last_seen == last:
-        yield gen.sleep(0.10)
+        await asyncio.sleep(0.10)
         assert time() < start + 5
 
-    yield c.close()
+    await c.close()
 
 
-@pytest.mark.skipif(sys.version_info[0] < 3, reason="cloudpickle Py27 issue")
 @gen_cluster(client=True)
-def test_unhashable_function(c, s, a, b):
+async def test_unhashable_function(c, s, a, b):
     d = {"a": 1}
-    result = yield c.submit(d.get, "a")
+    result = await c.submit(d.get, "a")
     assert result == 1
 
 
 @gen_cluster()
-def test_client_name(s, a, b):
+async def test_client_name(s, a, b):
     with dask.config.set({"client-name": "hello-world"}):
-        c = yield Client(s.address, asynchronous=True)
+        c = await Client(s.address, asynchronous=True)
         assert any("hello-world" in name for name in list(s.clients))
 
-    yield c.close()
+    await c.close()
 
 
 def test_client_doesnt_close_given_loop(loop, s, a, b):
@@ -5253,11 +5244,11 @@ def test_client_doesnt_close_given_loop(loop, s, a, b):
 
 
 @gen_cluster(client=True, nthreads=[])
-def test_quiet_scheduler_loss(c, s):
+async def test_quiet_scheduler_loss(c, s):
     c._periodic_callbacks["scheduler-info"].interval = 10
     with captured_logger(logging.getLogger("distributed.client")) as logger:
-        yield s.close()
-        yield c._update_scheduler_info()
+        await s.close()
+        await c._update_scheduler_info()
     text = logger.getvalue()
     assert "BrokenPipeError" not in text
 
@@ -5284,22 +5275,22 @@ async def test_dashboard_link_inproc(cleanup):
 
 
 @gen_test()
-def test_client_timeout_2():
+async def test_client_timeout_2():
     with dask.config.set({"distributed.comm.timeouts.connect": "10ms"}):
         start = time()
         c = Client("127.0.0.1:3755", asynchronous=True)
         with pytest.raises((TimeoutError, IOError)):
-            yield c
+            await c
         stop = time()
 
         assert c.status == "closed"
-        yield c.close()
+        await c.close()
 
         assert stop - start < 1
 
 
 @gen_test()
-def test_client_active_bad_port():
+async def test_client_active_bad_port():
     import tornado.web
     import tornado.httpserver
 
@@ -5309,8 +5300,8 @@ def test_client_active_bad_port():
     with dask.config.set({"distributed.comm.timeouts.connect": "10ms"}):
         c = Client("127.0.0.1:8080", asynchronous=True)
         with pytest.raises((TimeoutError, IOError)):
-            yield c
-        yield c._close(fast=True)
+            await c
+        await c._close(fast=True)
     http_server.stop()
 
 
@@ -5355,10 +5346,10 @@ async def test(s, a, b):
 
 
 @gen_cluster()
-def test_de_serialization(s, a, b):
+async def test_de_serialization(s, a, b):
     import numpy as np
 
-    c = yield Client(
+    c = await Client(
         s.address,
         asynchronous=True,
         serializers=["msgpack", "pickle"],
@@ -5366,35 +5357,35 @@ def test_de_serialization(s, a, b):
     )
     try:
         # Can send complex data
-        future = yield c.scatter(np.ones(5))
+        future = await c.scatter(np.ones(5))
 
         # But can not retrieve it
         with pytest.raises(TypeError):
-            result = yield future
+            result = await future
     finally:
-        yield c.close()
+        await c.close()
 
 
 @gen_cluster()
-def test_de_serialization_none(s, a, b):
+async def test_de_serialization_none(s, a, b):
     import numpy as np
 
-    c = yield Client(s.address, asynchronous=True, deserializers=["msgpack"])
+    c = await Client(s.address, asynchronous=True, deserializers=["msgpack"])
     try:
         # Can send complex data
-        future = yield c.scatter(np.ones(5))
+        future = await c.scatter(np.ones(5))
 
         # But can not retrieve it
         with pytest.raises(TypeError):
-            result = yield future
+            result = await future
     finally:
-        yield c.close()
+        await c.close()
 
 
 @gen_cluster()
-def test_client_repr_closed(s, a, b):
-    c = yield Client(s.address, asynchronous=True)
-    yield c.close()
+async def test_client_repr_closed(s, a, b):
+    c = await Client(s.address, asynchronous=True)
+    await c.close()
     c._repr_html_()
 
 
@@ -5405,7 +5396,7 @@ def test_client_repr_closed_sync(loop):
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)])
-def test_nested_prioritization(c, s, w):
+async def test_nested_prioritization(c, s, w):
     x = delayed(inc)(1, dask_key_name=("a", 2))
     y = delayed(inc)(2, dask_key_name=("a", 10))
 
@@ -5413,7 +5404,7 @@ def test_nested_prioritization(c, s, w):
 
     fx, fy = c.compute([x, y])
 
-    yield wait([fx, fy])
+    await wait([fx, fy])
 
     assert (o[x.key] < o[y.key]) == (
         s.tasks[tokey(fx.key)].priority < s.tasks[tokey(fy.key)].priority
@@ -5421,18 +5412,18 @@ def test_nested_prioritization(c, s, w):
 
 
 @gen_cluster(client=True)
-def test_scatter_error_cancel(c, s, a, b):
+async def test_scatter_error_cancel(c, s, a, b):
     # https://github.com/dask/distributed/issues/2038
     def bad_fn(x):
         raise Exception("lol")
 
-    x = yield c.scatter(1)
+    x = await c.scatter(1)
     y = c.submit(bad_fn, x)
     del x
 
-    yield wait(y)
+    await wait(y)
     assert y.status == "error"
-    yield gen.sleep(0.1)
+    await asyncio.sleep(0.1)
     assert y.status == "error"  # not cancelled
 
 
@@ -5442,14 +5433,14 @@ def test_no_threads_lingering():
 
 
 @gen_cluster()
-def test_direct_async(s, a, b):
-    c = yield Client(s.address, asynchronous=True, direct_to_workers=True)
+async def test_direct_async(s, a, b):
+    c = await Client(s.address, asynchronous=True, direct_to_workers=True)
     assert c.direct_to_workers
-    yield c.close()
+    await c.close()
 
-    c = yield Client(s.address, asynchronous=True, direct_to_workers=False)
+    c = await Client(s.address, asynchronous=True, direct_to_workers=False)
     assert not c.direct_to_workers
-    yield c.close()
+    await c.close()
 
 
 def test_direct_sync(c):
@@ -5462,9 +5453,9 @@ def f():
 
 
 @gen_cluster()
-def test_mixing_clients(s, a, b):
-    c1 = yield Client(s.address, asynchronous=True)
-    c2 = yield Client(s.address, asynchronous=True)
+async def test_mixing_clients(s, a, b):
+    c1 = await Client(s.address, asynchronous=True)
+    c2 = await Client(s.address, asynchronous=True)
 
     future = c1.submit(inc, 1)
     with pytest.raises(ValueError):
@@ -5472,16 +5463,16 @@ def test_mixing_clients(s, a, b):
 
     assert not c2.futures  # Don't create Futures on second Client
 
-    yield c1.close()
-    yield c2.close()
+    await c1.close()
+    await c2.close()
 
 
 @gen_cluster(client=True)
-def test_tuple_keys(c, s, a, b):
+async def test_tuple_keys(c, s, a, b):
     x = dask.delayed(inc)(1, dask_key_name=("x", 1))
     y = dask.delayed(inc)(x, dask_key_name=("y", 1))
     future = c.compute(y)
-    assert (yield future) == 3
+    assert (await future) == 3
 
 
 @gen_cluster(client=True)
@@ -5493,34 +5484,34 @@ async def test_multiple_scatter(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_map_large_kwargs_in_graph(c, s, a, b):
+async def test_map_large_kwargs_in_graph(c, s, a, b):
     np = pytest.importorskip("numpy")
     x = np.random.random(100000)
     futures = c.map(lambda a, b: a + b, range(100), b=x)
     while not s.tasks:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
     assert len(s.tasks) == 101
     assert any(k.startswith("ndarray") for k in s.tasks)
 
 
 @gen_cluster(client=True)
-def test_retry(c, s, a, b):
+async def test_retry(c, s, a, b):
     def f():
         assert dask.config.get("foo")
 
     with dask.config.set(foo=False):
         future = c.submit(f)
         with pytest.raises(AssertionError):
-            yield future
+            await future
 
     with dask.config.set(foo=True):
-        yield future.retry()
-        yield future
+        await future.retry()
+        await future
 
 
 @gen_cluster(client=True)
-def test_retry_dependencies(c, s, a, b):
+async def test_retry_dependencies(c, s, a, b):
     def f():
         return dask.config.get("foo")
 
@@ -5528,21 +5519,21 @@ def f():
     y = c.submit(inc, x)
 
     with pytest.raises(KeyError):
-        yield y
+        await y
 
     with dask.config.set(foo=100):
-        yield y.retry()
-        result = yield y
+        await y.retry()
+        result = await y
         assert result == 101
 
-        yield y.retry()
-        yield x.retry()
-        result = yield y
+        await y.retry()
+        await x.retry()
+        result = await y
         assert result == 101
 
 
 @gen_cluster(client=True)
-def test_released_dependencies(c, s, a, b):
+async def test_released_dependencies(c, s, a, b):
     def f(x):
         return dask.config.get("foo") + 1
 
@@ -5551,26 +5542,26 @@ def f(x):
     del x
 
     with pytest.raises(KeyError):
-        yield y
+        await y
 
     with dask.config.set(foo=100):
-        yield y.retry()
-        result = yield y
+        await y.retry()
+        result = await y
         assert result == 101
 
 
 @gen_cluster(client=True, clean_kwargs={"threads": False})
-def test_profile_bokeh(c, s, a, b):
+async def test_profile_bokeh(c, s, a, b):
     pytest.importorskip("bokeh.plotting")
     from bokeh.model import Model
 
-    yield c.map(slowinc, range(10), delay=0.2)
-    state, figure = yield c.profile(plot=True)
+    await c.gather(c.map(slowinc, range(10), delay=0.2))
+    state, figure = await c.profile(plot=True)
     assert isinstance(figure, Model)
 
     with tmpfile("html") as fn:
         try:
-            yield c.profile(filename=fn)
+            await c.profile(filename=fn)
         except PermissionError:
             if WINDOWS:
                 pytest.xfail()
@@ -5578,7 +5569,7 @@ def test_profile_bokeh(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_get_mix_futures_and_SubgraphCallable(c, s, a, b):
+async def test_get_mix_futures_and_SubgraphCallable(c, s, a, b):
     future = c.submit(add, 1, 2)
 
     subgraph = SubgraphCallable(
@@ -5587,7 +5578,7 @@ def test_get_mix_futures_and_SubgraphCallable(c, s, a, b):
     dsk = {"a": 1, "b": 2, "c": (subgraph, "a", "b"), "d": (subgraph, "c", "b")}
 
     future2 = c.get(dsk, "d", sync=False)
-    result = yield future2
+    result = await future2
     assert result == 11
 
     # Nested subgraphs
@@ -5603,12 +5594,12 @@ def test_get_mix_futures_and_SubgraphCallable(c, s, a, b):
 
     dsk2 = {"e": 1, "f": 2, "g": (subgraph2, "e", "f")}
 
-    result = yield c.get(dsk2, "g", sync=False)
+    result = await c.get(dsk2, "g", sync=False)
     assert result == 22
 
 
 @gen_cluster(client=True)
-def test_get_mix_futures_and_SubgraphCallable_dask_dataframe(c, s, a, b):
+async def test_get_mix_futures_and_SubgraphCallable_dask_dataframe(c, s, a, b):
     dd = pytest.importorskip("dask.dataframe")
     import pandas as pd
 
@@ -5618,7 +5609,7 @@ def test_get_mix_futures_and_SubgraphCallable_dask_dataframe(c, s, a, b):
     ddf["x"] = ddf["x"].astype("f8")
     ddf = ddf.map_partitions(lambda x: x)
     ddf["x"] = ddf["x"].astype("f8")
-    result = yield c.compute(ddf)
+    result = await c.compute(ddf)
     assert result.equals(df.astype("f8"))
 
 
@@ -5631,23 +5622,23 @@ def test_direct_to_workers(s, loop):
 
 
 @gen_cluster(client=True)
-def test_instances(c, s, a, b):
+async def test_instances(c, s, a, b):
     assert list(Client._instances) == [c]
     assert list(Scheduler._instances) == [s]
     assert set(Worker._instances) == {a, b}
 
 
 @gen_cluster(client=True)
-def test_wait_for_workers(c, s, a, b):
+async def test_wait_for_workers(c, s, a, b):
     future = asyncio.ensure_future(c.wait_for_workers(n_workers=3))
-    yield gen.sleep(0.22)  # 2 chances
+    await asyncio.sleep(0.22)  # 2 chances
     assert not future.done()
 
-    w = yield Worker(s.address)
+    w = await Worker(s.address)
     start = time()
-    yield future
+    await future
     assert time() < start + 1
-    yield w.close()
+    await w.close()
 
 
 @pytest.mark.skipif(WINDOWS, reason="num_fds not supported on windows")
@@ -5750,14 +5741,14 @@ async def test_profile_server(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_await_future(c, s, a, b):
+async def test_await_future(c, s, a, b):
     future = c.submit(inc, 1)
 
     async def f():  # flake8: noqa
         result = await future
         assert result == 2
 
-    yield f()
+    await f()
 
     future = c.submit(div, 1, 0)
 
@@ -5765,11 +5756,11 @@ async def f():
         with pytest.raises(ZeroDivisionError):
             await future
 
-    yield f()
+    await f()
 
 
 @gen_cluster(client=True)
-def test_as_completed_async_for(c, s, a, b):
+async def test_as_completed_async_for(c, s, a, b):
     futures = c.map(inc, range(10))
     ac = as_completed(futures)
     results = []
@@ -5779,13 +5770,13 @@ async def f():
             result = await future
             results.append(result)
 
-    yield f()
+    await f()
 
     assert set(results) == set(range(1, 11))
 
 
 @gen_cluster(client=True)
-def test_as_completed_async_for_results(c, s, a, b):
+async def test_as_completed_async_for_results(c, s, a, b):
     futures = c.map(inc, range(10))
     ac = as_completed(futures, with_results=True)
     results = []
@@ -5794,20 +5785,20 @@ async def f():
         async for future, result in ac:
             results.append(result)
 
-    yield f()
+    await f()
 
     assert set(results) == set(range(1, 11))
     assert not s.counters["op"].components[0]["gather"]
 
 
 @gen_cluster(client=True)
-def test_as_completed_async_for_cancel(c, s, a, b):
+async def test_as_completed_async_for_cancel(c, s, a, b):
     x = c.submit(inc, 1)
     y = c.submit(sleep, 0.3)
     ac = as_completed([x, y])
 
     async def _():
-        await gen.sleep(0.1)
+        await asyncio.sleep(0.1)
         await y.cancel(asynchronous=True)
 
     c.loop.add_callback(_)
@@ -5818,7 +5809,7 @@ async def f():
         async for future in ac:
             L.append(future)
 
-    yield f()
+    await f()
 
     assert L == [x, y]
 
@@ -5845,7 +5836,7 @@ async def f():
 
 def test_client_sync_with_async_def(loop):
     async def ff():
-        await gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         return 1
 
     with cluster() as (s, [a, b]):
@@ -5888,13 +5879,13 @@ async def test_dont_hold_on_to_large_messages(c, s, a, b):
                     )
             pytest.fail("array should have been destroyed")
 
-        await gen.sleep(0.200)
+        await asyncio.sleep(0.200)
 
 
 @gen_cluster(client=True)
 async def test_run_scheduler_async_def(c, s, a, b):
     async def f(dask_scheduler):
-        await gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         dask_scheduler.foo = "bar"
 
     await c.run_on_scheduler(f)
@@ -5902,7 +5893,7 @@ async def f(dask_scheduler):
     assert s.foo == "bar"
 
     async def f(dask_worker):
-        await gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         dask_worker.foo = "bar"
 
     await c.run(f)
@@ -5913,23 +5904,23 @@ async def f(dask_worker):
 @gen_cluster(client=True)
 async def test_run_scheduler_async_def_wait(c, s, a, b):
     async def f(dask_scheduler):
-        await gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         dask_scheduler.foo = "bar"
 
     await c.run_on_scheduler(f, wait=False)
 
     while not hasattr(s, "foo"):
-        await gen.sleep(0.01)
+        await asyncio.sleep(0.01)
     assert s.foo == "bar"
 
     async def f(dask_worker):
-        await gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         dask_worker.foo = "bar"
 
     await c.run(f, wait=False)
 
     while not hasattr(a, "foo") or not hasattr(b, "foo"):
-        await gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
     assert a.foo == "bar"
     assert b.foo == "bar"
@@ -5973,7 +5964,7 @@ async def test_client_gather_semaphor_loop(cleanup):
 
 
 @gen_cluster(client=True)
-def test_as_completed_condition_loop(c, s, a, b):
+async def test_as_completed_condition_loop(c, s, a, b):
     seq = c.map(inc, range(5))
     ac = as_completed(seq)
     assert ac.condition._loop == c.loop.asyncio_loop
diff --git a/distributed/tests/test_collections.py b/distributed/tests/test_collections.py
index 61424c68f38..b9af1ef0222 100644
--- a/distributed/tests/test_collections.py
+++ b/distributed/tests/test_collections.py
@@ -34,7 +34,7 @@ def assert_equal(a, b):
 
 
 @gen_cluster(timeout=240, client=True)
-def test_dataframes(c, s, a, b):
+async def test_dataframes(c, s, a, b):
     df = pd.DataFrame(
         {"x": np.random.random(1000), "y": np.random.random(1000)},
         index=np.arange(1000),
@@ -46,7 +46,7 @@ def test_dataframes(c, s, a, b):
     assert rdf.divisions == ldf.divisions
 
     remote = c.compute(rdf)
-    result = yield remote
+    result = await remote
 
     tm.assert_frame_equal(result, ldf.compute(scheduler="sync"))
 
@@ -63,19 +63,19 @@ def test_dataframes(c, s, a, b):
     for f in exprs:
         local = f(ldf).compute(scheduler="sync")
         remote = c.compute(f(rdf))
-        remote = yield remote
+        remote = await remote
         assert_equal(local, remote)
 
 
 @gen_cluster(client=True)
-def test__dask_array_collections(c, s, a, b):
+async def test_dask_array_collections(c, s, a, b):
     import dask.array as da
 
     s.validate = False
     x_dsk = {("x", i, j): np.random.random((3, 3)) for i in range(3) for j in range(2)}
     y_dsk = {("y", i, j): np.random.random((3, 3)) for i in range(2) for j in range(3)}
-    x_futures = yield c.scatter(x_dsk)
-    y_futures = yield c.scatter(y_dsk)
+    x_futures = await c.scatter(x_dsk)
+    y_futures = await c.scatter(y_dsk)
 
     dt = np.random.random(0).dtype
     x_local = da.Array(x_dsk, "x", ((3, 3, 3), (3, 3)), dt)
@@ -95,13 +95,13 @@ def test__dask_array_collections(c, s, a, b):
         local = expr(x_local, y_local).compute(scheduler="sync")
 
         remote = c.compute(expr(x_remote, y_remote))
-        remote = yield remote
+        remote = await remote
 
         assert np.all(local == remote)
 
 
 @gen_cluster(client=True)
-def test_bag_groupby_tasks_default(c, s, a, b):
+async def test_bag_groupby_tasks_default(c, s, a, b):
     b = db.range(100, npartitions=10)
     b2 = b.groupby(lambda x: x % 13)
     assert not any("partd" in k[0] for k in b2.dask)
@@ -147,11 +147,11 @@ def test_rolling_sync(client):
 
 
 @gen_cluster(client=True)
-def test_loc(c, s, a, b):
+async def test_loc(c, s, a, b):
     df = make_time_dataframe()
     ddf = dd.from_pandas(df, npartitions=10)
     future = c.compute(ddf.loc["2000-01-17":"2000-01-24"])
-    yield future
+    await future
 
 
 def test_dataframe_groupby_tasks(client):
@@ -182,7 +182,7 @@ def test_dataframe_groupby_tasks(client):
 
 
 @gen_cluster(client=True)
-def test_sparse_arrays(c, s, a, b):
+async def test_sparse_arrays(c, s, a, b):
     sparse = pytest.importorskip("sparse")
     da = pytest.importorskip("dask.array")
 
@@ -191,13 +191,13 @@ def test_sparse_arrays(c, s, a, b):
     s = x.map_blocks(sparse.COO)
     future = c.compute(s.sum(axis=0)[:10])
 
-    yield future
+    await future
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)])
-def test_delayed_none(c, s, w):
+async def test_delayed_none(c, s, w):
     x = dask.delayed(None)
     y = dask.delayed(123)
     [xx, yy] = c.compute([x, y])
-    assert (yield xx) is None
-    assert (yield yy) == 123
+    assert await xx is None
+    assert await yy == 123
diff --git a/distributed/tests/test_core.py b/distributed/tests/test_core.py
index 49033a6a11e..c75f9c48cf6 100644
--- a/distributed/tests/test_core.py
+++ b/distributed/tests/test_core.py
@@ -732,7 +732,7 @@ async def f():
 
 
 @gen_cluster()
-def test_thread_id(s, a, b):
+async def test_thread_id(s, a, b):
     assert s.thread_id == a.thread_id == b.thread_id == threading.get_ident()
 
 
diff --git a/distributed/tests/test_failed_workers.py b/distributed/tests/test_failed_workers.py
index 99b1b4a42a7..e1556494fe2 100644
--- a/distributed/tests/test_failed_workers.py
+++ b/distributed/tests/test_failed_workers.py
@@ -1,10 +1,10 @@
+import asyncio
 import os
 import random
 from time import sleep
 
 import pytest
 from tlz import partition_all, first
-from tornado import gen
 
 from dask import delayed
 from distributed import Client, Nanny, wait
@@ -35,30 +35,30 @@ def test_submit_after_failed_worker_sync(loop):
 
 
 @gen_cluster(client=True, timeout=60, active_rpc_timeout=10)
-def test_submit_after_failed_worker_async(c, s, a, b):
-    n = yield Nanny(s.address, nthreads=2, loop=s.loop)
+async def test_submit_after_failed_worker_async(c, s, a, b):
+    n = await Nanny(s.address, nthreads=2, loop=s.loop)
     while len(s.workers) < 3:
-        yield gen.sleep(0.1)
+        await asyncio.sleep(0.1)
 
     L = c.map(inc, range(10))
-    yield wait(L)
+    await wait(L)
 
     s.loop.add_callback(n.kill)
     total = c.submit(sum, L)
-    result = yield total
+    result = await total
     assert result == sum(map(inc, range(10)))
 
-    yield n.close()
+    await n.close()
 
 
 @gen_cluster(client=True, timeout=60)
-def test_submit_after_failed_worker(c, s, a, b):
+async def test_submit_after_failed_worker(c, s, a, b):
     L = c.map(inc, range(10))
-    yield wait(L)
-    yield a.close()
+    await wait(L)
+    await a.close()
 
     total = c.submit(sum, L)
-    result = yield total
+    result = await total
     assert result == sum(map(inc, range(10)))
 
 
@@ -78,73 +78,73 @@ def test_gather_after_failed_worker(loop):
     nthreads=[("127.0.0.1", 1)] * 4,
     config={"distributed.comm.timeouts.connect": "1s"},
 )
-def test_gather_then_submit_after_failed_workers(c, s, w, x, y, z):
+async def test_gather_then_submit_after_failed_workers(c, s, w, x, y, z):
     L = c.map(inc, range(20))
-    yield wait(L)
+    await wait(L)
 
     w.process.process._process.terminate()
     total = c.submit(sum, L)
 
     for i in range(3):
-        yield wait(total)
+        await wait(total)
         addr = first(s.tasks[total.key].who_has).address
         for worker in [x, y, z]:
             if worker.worker_address == addr:
                 worker.process.process._process.terminate()
                 break
 
-        result = yield c.gather([total])
+        result = await c.gather([total])
         assert result == [sum(map(inc, range(20)))]
 
 
 @gen_cluster(Worker=Nanny, timeout=60, client=True)
-def test_failed_worker_without_warning(c, s, a, b):
+async def test_failed_worker_without_warning(c, s, a, b):
     L = c.map(inc, range(10))
-    yield wait(L)
+    await wait(L)
 
     original_pid = a.pid
     with ignoring(CommClosedError):
-        yield c._run(os._exit, 1, workers=[a.worker_address])
+        await c._run(os._exit, 1, workers=[a.worker_address])
     start = time()
     while a.pid == original_pid:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() - start < 10
 
-    yield gen.sleep(0.5)
+    await asyncio.sleep(0.5)
 
     start = time()
     while len(s.nthreads) < 2:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() - start < 10
 
-    yield wait(L)
+    await wait(L)
 
     L2 = c.map(inc, range(10, 20))
-    yield wait(L2)
+    await wait(L2)
     assert all(len(keys) > 0 for keys in s.has_what.values())
     nthreads2 = dict(s.nthreads)
 
-    yield c.restart()
+    await c.restart()
 
     L = c.map(inc, range(10))
-    yield wait(L)
+    await wait(L)
     assert all(len(keys) > 0 for keys in s.has_what.values())
 
     assert not (set(nthreads2) & set(s.nthreads))  # no overlap
 
 
 @gen_cluster(Worker=Nanny, client=True, timeout=60)
-def test_restart(c, s, a, b):
+async def test_restart(c, s, a, b):
     assert s.nthreads == {a.worker_address: 1, b.worker_address: 2}
 
     x = c.submit(inc, 1)
     y = c.submit(inc, x)
     z = c.submit(div, 1, 0)
-    yield y
+    await y
 
     assert set(s.who_has) == {x.key, y.key}
 
-    f = yield c.restart()
+    f = await c.restart()
     assert f is c
 
     assert len(s.workers) == 2
@@ -162,12 +162,12 @@ def test_restart(c, s, a, b):
 
 
 @gen_cluster(Worker=Nanny, client=True, timeout=60)
-def test_restart_cleared(c, s, a, b):
+async def test_restart_cleared(c, s, a, b):
     x = 2 * delayed(1) + 1
     f = c.compute(x)
-    yield wait([f])
+    await wait([f])
 
-    yield c.restart()
+    await c.restart()
 
     for coll in [s.tasks, s.unrunnable]:
         assert not coll
@@ -204,18 +204,18 @@ def test_restart_sync(loop):
 
 
 @gen_cluster(Worker=Nanny, client=True, timeout=60)
-def test_restart_fast(c, s, a, b):
+async def test_restart_fast(c, s, a, b):
     L = c.map(sleep, range(10))
 
     start = time()
-    yield c.restart()
+    await c.restart()
     assert time() - start < 10
     assert len(s.nthreads) == 2
 
     assert all(x.status == "cancelled" for x in L)
 
     x = c.submit(inc, 1)
-    result = yield x
+    result = await x
     assert result == 2
 
 
@@ -247,51 +247,51 @@ def test_restart_fast_sync(loop):
 
 
 @gen_cluster(Worker=Nanny, client=True, timeout=60)
-def test_fast_kill(c, s, a, b):
+async def test_fast_kill(c, s, a, b):
     L = c.map(sleep, range(10))
 
     start = time()
-    yield c.restart()
+    await c.restart()
     assert time() - start < 10
 
     assert all(x.status == "cancelled" for x in L)
 
     x = c.submit(inc, 1)
-    result = yield x
+    result = await x
     assert result == 2
 
 
 @gen_cluster(Worker=Nanny, timeout=60)
-def test_multiple_clients_restart(s, a, b):
-    c1 = yield Client(s.address, asynchronous=True)
-    c2 = yield Client(s.address, asynchronous=True)
+async def test_multiple_clients_restart(s, a, b):
+    c1 = await Client(s.address, asynchronous=True)
+    c2 = await Client(s.address, asynchronous=True)
 
     x = c1.submit(inc, 1)
     y = c2.submit(inc, 2)
-    xx = yield x
-    yy = yield y
+    xx = await x
+    yy = await y
     assert xx == 2
     assert yy == 3
 
-    yield c1.restart()
+    await c1.restart()
 
     assert x.cancelled()
     start = time()
     while not y.cancelled():
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 5
 
-    yield c1.close()
-    yield c2.close()
+    await c1.close()
+    await c2.close()
 
 
 @gen_cluster(Worker=Nanny, timeout=60)
-def test_restart_scheduler(s, a, b):
+async def test_restart_scheduler(s, a, b):
     import gc
 
     gc.collect()
     addrs = (a.worker_address, b.worker_address)
-    yield s.restart()
+    await s.restart()
     assert len(s.nthreads) == 2
     addrs2 = (a.worker_address, b.worker_address)
 
@@ -299,26 +299,26 @@ def test_restart_scheduler(s, a, b):
 
 
 @gen_cluster(Worker=Nanny, client=True, timeout=60)
-def test_forgotten_futures_dont_clean_up_new_futures(c, s, a, b):
+async def test_forgotten_futures_dont_clean_up_new_futures(c, s, a, b):
     x = c.submit(inc, 1)
-    yield c.restart()
+    await c.restart()
     y = c.submit(inc, 1)
     del x
     import gc
 
     gc.collect()
-    yield gen.sleep(0.1)
-    yield y
+    await asyncio.sleep(0.1)
+    await y
 
 
 @gen_cluster(client=True, timeout=60, active_rpc_timeout=10)
-def test_broken_worker_during_computation(c, s, a, b):
+async def test_broken_worker_during_computation(c, s, a, b):
     s.allowed_failures = 100
-    n = yield Nanny(s.address, nthreads=2, loop=s.loop)
+    n = await Nanny(s.address, nthreads=2, loop=s.loop)
 
     start = time()
     while len(s.nthreads) < 3:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 5
 
     N = 256
@@ -333,37 +333,37 @@ def test_broken_worker_during_computation(c, s, a, b):
             key=["add-%d-%d" % (i, j) for j in range(len(L) // 2)]
         )
 
-    yield gen.sleep(random.random() / 20)
+    await asyncio.sleep(random.random() / 20)
     with ignoring(CommClosedError):  # comm will be closed abrupty
-        yield c._run(os._exit, 1, workers=[n.worker_address])
+        await c._run(os._exit, 1, workers=[n.worker_address])
 
-    yield gen.sleep(random.random() / 20)
+    await asyncio.sleep(random.random() / 20)
     while len(s.workers) < 3:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
     with ignoring(
         CommClosedError, EnvironmentError
     ):  # perhaps new worker can't be contacted yet
-        yield c._run(os._exit, 1, workers=[n.worker_address])
+        await c._run(os._exit, 1, workers=[n.worker_address])
 
-    [result] = yield c.gather(L)
+    [result] = await c.gather(L)
     assert isinstance(result, int)
     assert result == expected_result
 
-    yield n.close()
+    await n.close()
 
 
 @gen_cluster(client=True, Worker=Nanny, timeout=60)
-def test_restart_during_computation(c, s, a, b):
+async def test_restart_during_computation(c, s, a, b):
     xs = [delayed(slowinc)(i, delay=0.01) for i in range(50)]
     ys = [delayed(slowinc)(i, delay=0.01) for i in xs]
     zs = [delayed(slowadd)(x, y, delay=0.01) for x, y in zip(xs, ys)]
     total = delayed(sum)(zs)
     result = c.compute(total)
 
-    yield gen.sleep(0.5)
+    await asyncio.sleep(0.5)
     assert s.rprocessing
-    yield c.restart()
+    await c.restart()
     assert not s.rprocessing
 
     assert len(s.nthreads) == 2
@@ -371,59 +371,59 @@ def test_restart_during_computation(c, s, a, b):
 
 
 @gen_cluster(client=True, timeout=60)
-def test_worker_who_has_clears_after_failed_connection(c, s, a, b):
-    n = yield Nanny(s.address, nthreads=2, loop=s.loop)
+async def test_worker_who_has_clears_after_failed_connection(c, s, a, b):
+    n = await Nanny(s.address, nthreads=2, loop=s.loop)
 
     start = time()
     while len(s.nthreads) < 3:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 5
 
     futures = c.map(slowinc, range(20), delay=0.01, key=["f%d" % i for i in range(20)])
-    yield wait(futures)
+    await wait(futures)
 
-    result = yield c.submit(sum, futures, workers=a.address)
+    result = await c.submit(sum, futures, workers=a.address)
     for dep in set(a.dep_state) - set(a.task_state):
         a.release_dep(dep, report=True)
 
     n_worker_address = n.worker_address
     with ignoring(CommClosedError):
-        yield c._run(os._exit, 1, workers=[n_worker_address])
+        await c._run(os._exit, 1, workers=[n_worker_address])
 
     while len(s.workers) > 2:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
     total = c.submit(sum, futures, workers=a.address)
-    yield total
+    await total
 
     assert not a.has_what.get(n_worker_address)
     assert not any(n_worker_address in s for s in a.who_has.values())
 
-    yield n.close()
+    await n.close()
 
 
 @pytest.mark.slow
 @gen_cluster(client=True, timeout=60, Worker=Nanny, nthreads=[("127.0.0.1", 1)])
-def test_restart_timeout_on_long_running_task(c, s, a):
+async def test_restart_timeout_on_long_running_task(c, s, a):
     with captured_logger("distributed.scheduler") as sio:
         future = c.submit(sleep, 3600)
-        yield gen.sleep(0.1)
-        yield c.restart(timeout=20)
+        await asyncio.sleep(0.1)
+        await c.restart(timeout=20)
 
     text = sio.getvalue()
     assert "timeout" not in text.lower()
 
 
 @gen_cluster(client=True, scheduler_kwargs={"worker_ttl": "500ms"})
-def test_worker_time_to_live(c, s, a, b):
+async def test_worker_time_to_live(c, s, a, b):
     assert set(s.workers) == {a.address, b.address}
     a.periodic_callbacks["heartbeat"].stop()
-    yield gen.sleep(0.010)
+    await asyncio.sleep(0.010)
     assert set(s.workers) == {a.address, b.address}
 
     start = time()
     while set(s.workers) == {a.address, b.address}:
-        yield gen.sleep(0.050)
+        await asyncio.sleep(0.050)
         assert time() < start + 2
 
     set(s.workers) == {b.address}
diff --git a/distributed/tests/test_locks.py b/distributed/tests/test_locks.py
index 4cf756ef178..0d22fc6cee9 100644
--- a/distributed/tests/test_locks.py
+++ b/distributed/tests/test_locks.py
@@ -10,8 +10,8 @@
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 8)] * 2)
-def test_lock(c, s, a, b):
-    yield c.set_metadata("locked", False)
+async def test_lock(c, s, a, b):
+    await c.set_metadata("locked", False)
 
     def f(x):
         client = get_client()
@@ -23,16 +23,16 @@ def f(x):
             client.set_metadata("locked", False)
 
     futures = c.map(f, range(20))
-    results = yield futures
+    await c.gather(futures)
     assert not s.extensions["locks"].events
     assert not s.extensions["locks"].ids
 
 
 @gen_cluster(client=True)
-def test_timeout(c, s, a, b):
+async def test_timeout(c, s, a, b):
     locks = s.extensions["locks"]
     lock = Lock("x")
-    result = yield lock.acquire()
+    result = await lock.acquire()
     assert result is True
     assert locks.ids["x"] == lock.id
 
@@ -40,35 +40,35 @@ def test_timeout(c, s, a, b):
     assert lock.id != lock2.id
 
     start = time()
-    result = yield lock2.acquire(timeout=0.1)
+    result = await lock2.acquire(timeout=0.1)
     stop = time()
     assert stop - start < 0.3
     assert result is False
     assert locks.ids["x"] == lock.id
     assert not locks.events["x"]
 
-    yield lock.release()
+    await lock.release()
 
 
 @gen_cluster(client=True)
-def test_acquires_with_zero_timeout(c, s, a, b):
+async def test_acquires_with_zero_timeout(c, s, a, b):
     lock = Lock("x")
-    yield lock.acquire(timeout=0)
+    await lock.acquire(timeout=0)
     assert lock.locked()
-    yield lock.release()
+    await lock.release()
 
-    yield lock.acquire(timeout=1)
-    yield lock.release()
-    yield lock.acquire(timeout=1)
-    yield lock.release()
+    await lock.acquire(timeout=1)
+    await lock.release()
+    await lock.acquire(timeout=1)
+    await lock.release()
 
 
 @gen_cluster(client=True)
-def test_acquires_blocking(c, s, a, b):
+async def test_acquires_blocking(c, s, a, b):
     lock = Lock("x")
-    yield lock.acquire(blocking=False)
+    await lock.acquire(blocking=False)
     assert lock.locked()
-    yield lock.release()
+    await lock.release()
     assert not lock.locked()
 
     with pytest.raises(ValueError):
@@ -81,10 +81,10 @@ def test_timeout_sync(client):
 
 
 @gen_cluster(client=True)
-def test_errors(c, s, a, b):
+async def test_errors(c, s, a, b):
     lock = Lock("x")
     with pytest.raises(ValueError):
-        yield lock.release()
+        await lock.release()
 
 
 def test_lock_sync(client):
@@ -103,19 +103,19 @@ def f(x):
 
 
 @gen_cluster(client=True)
-def test_lock_types(c, s, a, b):
+async def test_lock_types(c, s, a, b):
     for name in [1, ("a", 1), ["a", 1], b"123", "123"]:
         lock = Lock(name)
         assert lock.name == name
 
-        yield lock.acquire()
-        yield lock.release()
+        await lock.acquire()
+        await lock.release()
 
     assert not s.extensions["locks"].events
 
 
 @gen_cluster(client=True)
-def test_serializable(c, s, a, b):
+async def test_serializable(c, s, a, b):
     def f(x, lock=None):
         with lock:
             assert lock.name == "x"
@@ -123,7 +123,7 @@ def f(x, lock=None):
 
     lock = Lock("x")
     futures = c.map(f, range(10), lock=lock)
-    yield c.gather(futures)
+    await c.gather(futures)
 
     lock2 = pickle.loads(pickle.dumps(lock))
     assert lock2.name == lock.name
diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
index 2a19bdf8742..2c7a6f83671 100644
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -10,7 +10,6 @@
 
 import pytest
 from tlz import valmap, first
-from tornado import gen
 from tornado.ioloop import IOLoop
 
 import dask
@@ -29,7 +28,8 @@
 )
 
 
-@gen_cluster(nthreads=[])
+# FIXME why does this leave behind unclosed Comm objects?
+@gen_cluster(nthreads=[], allow_unclosed=True)
 async def test_nanny(s):
     async with Nanny(s.address, nthreads=2, loop=s.loop) as n:
         async with rpc(n.address) as nn:
@@ -60,16 +60,16 @@ async def test_nanny(s):
 
 
 @gen_cluster(nthreads=[])
-def test_many_kills(s):
-    n = yield Nanny(s.address, nthreads=2, loop=s.loop)
+async def test_many_kills(s):
+    n = await Nanny(s.address, nthreads=2, loop=s.loop)
     assert n.is_alive()
-    yield [n.kill() for i in range(5)]
-    yield [n.kill() for i in range(5)]
-    yield n.close()
+    await asyncio.gather(*(n.kill() for _ in range(5)))
+    await asyncio.gather(*(n.kill() for _ in range(5)))
+    await n.close()
 
 
 @gen_cluster(Worker=Nanny)
-def test_str(s, a, b):
+async def test_str(s, a, b):
     assert a.worker_address in str(a)
     assert a.worker_address in repr(a)
     assert str(a.nthreads) in str(a)
@@ -77,59 +77,59 @@ def test_str(s, a, b):
 
 
 @gen_cluster(nthreads=[], timeout=20, client=True)
-def test_nanny_process_failure(c, s):
-    n = yield Nanny(s.address, nthreads=2, loop=s.loop)
+async def test_nanny_process_failure(c, s):
+    n = await Nanny(s.address, nthreads=2, loop=s.loop)
     first_dir = n.worker_dir
 
     assert os.path.exists(first_dir)
 
     original_address = n.worker_address
     ww = rpc(n.worker_address)
-    yield ww.update_data(data=valmap(dumps, {"x": 1, "y": 2}))
+    await ww.update_data(data=valmap(dumps, {"x": 1, "y": 2}))
     pid = n.pid
     assert pid is not None
     with ignoring(CommClosedError):
-        yield c.run(os._exit, 0, workers=[n.worker_address])
+        await c.run(os._exit, 0, workers=[n.worker_address])
 
     start = time()
     while n.pid == pid:  # wait while process dies and comes back
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() - start < 5
 
     start = time()
-    yield gen.sleep(1)
+    await asyncio.sleep(1)
     while not n.is_alive():  # wait while process comes back
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() - start < 5
 
     # assert n.worker_address != original_address  # most likely
 
     start = time()
     while n.worker_address not in s.nthreads or n.worker_dir is None:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() - start < 5
 
     second_dir = n.worker_dir
 
-    yield n.close()
+    await n.close()
     assert not os.path.exists(second_dir)
     assert not os.path.exists(first_dir)
     assert first_dir != n.worker_dir
-    yield ww.close_rpc()
+    await ww.close_rpc()
     s.stop()
 
 
 @gen_cluster(nthreads=[])
-def test_run(s):
+async def test_run(s):
     pytest.importorskip("psutil")
-    n = yield Nanny(s.address, nthreads=2, loop=s.loop)
+    n = await Nanny(s.address, nthreads=2, loop=s.loop)
 
     with rpc(n.address) as nn:
-        response = yield nn.run(function=dumps(lambda: 1))
+        response = await nn.run(function=dumps(lambda: 1))
         assert response["status"] == "OK"
         assert response["result"] == 1
 
-    yield n.close()
+    await n.close()
 
 
 @pytest.mark.slow
@@ -150,12 +150,12 @@ async def test_no_hang_when_scheduler_closes(s, a, b):
 @gen_cluster(
     Worker=Nanny, nthreads=[("127.0.0.1", 1)], worker_kwargs={"reconnect": False}
 )
-def test_close_on_disconnect(s, w):
-    yield s.close()
+async def test_close_on_disconnect(s, w):
+    await s.close()
 
     start = time()
     while w.status != "closed":
-        yield gen.sleep(0.05)
+        await asyncio.sleep(0.05)
         assert time() < start + 9
 
 
@@ -165,70 +165,69 @@ class Something(Worker):
 
 
 @gen_cluster(client=True, Worker=Nanny)
-def test_nanny_worker_class(c, s, w1, w2):
-    out = yield c._run(lambda dask_worker=None: str(dask_worker.__class__))
+async def test_nanny_worker_class(c, s, w1, w2):
+    out = await c._run(lambda dask_worker=None: str(dask_worker.__class__))
     assert "Worker" in list(out.values())[0]
     assert w1.Worker is Worker
 
 
 @gen_cluster(client=True, Worker=Nanny, worker_kwargs={"worker_class": Something})
-def test_nanny_alt_worker_class(c, s, w1, w2):
-    out = yield c._run(lambda dask_worker=None: str(dask_worker.__class__))
+async def test_nanny_alt_worker_class(c, s, w1, w2):
+    out = await c._run(lambda dask_worker=None: str(dask_worker.__class__))
     assert "Something" in list(out.values())[0]
     assert w1.Worker is Something
 
 
 @pytest.mark.slow
 @gen_cluster(client=False, nthreads=[])
-def test_nanny_death_timeout(s):
-    yield s.close()
+async def test_nanny_death_timeout(s):
+    await s.close()
     w = Nanny(s.address, death_timeout=1)
     with pytest.raises(TimeoutError):
-        yield w
+        await w
 
     assert w.status == "closed"
 
 
 @gen_cluster(client=True, Worker=Nanny)
-def test_random_seed(c, s, a, b):
-    @gen.coroutine
-    def check_func(func):
+async def test_random_seed(c, s, a, b):
+    async def check_func(func):
         x = c.submit(func, 0, 2 ** 31, pure=False, workers=a.worker_address)
         y = c.submit(func, 0, 2 ** 31, pure=False, workers=b.worker_address)
         assert x.key != y.key
-        x = yield x
-        y = yield y
+        x = await x
+        y = await y
         assert x != y
 
-    yield check_func(lambda a, b: random.randint(a, b))
-    yield check_func(lambda a, b: np.random.randint(a, b))
+    await check_func(lambda a, b: random.randint(a, b))
+    await check_func(lambda a, b: np.random.randint(a, b))
 
 
 @pytest.mark.skipif(
     sys.platform.startswith("win"), reason="num_fds not supported on windows"
 )
 @gen_cluster(client=False, nthreads=[])
-def test_num_fds(s):
+async def test_num_fds(s):
     psutil = pytest.importorskip("psutil")
     proc = psutil.Process()
 
     # Warm up
-    w = yield Nanny(s.address)
-    yield w.close()
+    w = await Nanny(s.address)
+    await w.close()
     del w
     gc.collect()
 
     before = proc.num_fds()
 
     for i in range(3):
-        w = yield Nanny(s.address)
-        yield gen.sleep(0.1)
-        yield w.close()
+        w = await Nanny(s.address)
+        await asyncio.sleep(0.1)
+        await w.close()
 
     start = time()
     while proc.num_fds() > before:
         print("fds:", before, proc.num_fds())
-        yield gen.sleep(0.1)
+        await asyncio.sleep(0.1)
         assert time() < start + 10
 
 
@@ -236,42 +235,42 @@ def test_num_fds(s):
     not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
 )
 @gen_cluster(client=True, nthreads=[])
-def test_worker_uses_same_host_as_nanny(c, s):
+async def test_worker_uses_same_host_as_nanny(c, s):
     for host in ["tcp://0.0.0.0", "tcp://127.0.0.2"]:
-        n = yield Nanny(s.address, host=host)
+        n = await Nanny(s.address, host=host)
 
         def func(dask_worker):
             return dask_worker.listener.listen_address
 
-        result = yield c.run(func)
+        result = await c.run(func)
         assert host in first(result.values())
-        yield n.close()
+        await n.close()
 
 
 @gen_test()
-def test_scheduler_file():
+async def test_scheduler_file():
     with tmpfile() as fn:
-        s = yield Scheduler(scheduler_file=fn, port=8008)
-        w = yield Nanny(scheduler_file=fn)
+        s = await Scheduler(scheduler_file=fn, port=8008)
+        w = await Nanny(scheduler_file=fn)
         assert set(s.workers) == {w.worker_address}
-        yield w.close()
+        await w.close()
         s.stop()
 
 
 @gen_cluster(client=True, Worker=Nanny, nthreads=[("127.0.0.1", 2)])
-def test_nanny_timeout(c, s, a):
-    x = yield c.scatter(123)
+async def test_nanny_timeout(c, s, a):
+    x = await c.scatter(123)
     with captured_logger(
         logging.getLogger("distributed.nanny"), level=logging.ERROR
     ) as logger:
-        response = yield a.restart(timeout=0.1)
+        response = await a.restart(timeout=0.1)
 
     out = logger.getvalue()
     assert "timed out" in out.lower()
 
     start = time()
     while x.status != "cancelled":
-        yield gen.sleep(0.1)
+        await asyncio.sleep(0.1)
         assert time() < start + 7
 
 
@@ -283,7 +282,7 @@ def test_nanny_timeout(c, s, a):
     timeout=20,
     clean_kwargs={"threads": False},
 )
-def test_nanny_terminate(c, s, a):
+async def test_nanny_terminate(c, s, a):
     from time import sleep
 
     def leak():
@@ -297,7 +296,7 @@ def leak():
         future = c.submit(leak)
         start = time()
         while a.process.pid == proc:
-            yield gen.sleep(0.1)
+            await asyncio.sleep(0.1)
             assert time() < start + 10
         out = logger.getvalue()
         assert "restart" in out.lower()
@@ -339,45 +338,45 @@ def pause(dask_worker):
 
 
 @gen_cluster(nthreads=[], client=True)
-def test_avoid_memory_monitor_if_zero_limit(c, s):
-    nanny = yield Nanny(s.address, loop=s.loop, memory_limit=0)
-    typ = yield c.run(lambda dask_worker: type(dask_worker.data))
+async def test_avoid_memory_monitor_if_zero_limit(c, s):
+    nanny = await Nanny(s.address, loop=s.loop, memory_limit=0)
+    typ = await c.run(lambda dask_worker: type(dask_worker.data))
     assert typ == {nanny.worker_address: dict}
-    pcs = yield c.run(lambda dask_worker: list(dask_worker.periodic_callbacks))
+    pcs = await c.run(lambda dask_worker: list(dask_worker.periodic_callbacks))
     assert "memory" not in pcs
     assert "memory" not in nanny.periodic_callbacks
 
     future = c.submit(inc, 1)
-    assert (yield future) == 2
-    yield gen.sleep(0.02)
+    assert await future == 2
+    await asyncio.sleep(0.02)
 
-    yield c.submit(inc, 2)  # worker doesn't pause
+    await c.submit(inc, 2)  # worker doesn't pause
 
-    yield nanny.close()
+    await nanny.close()
 
 
 @gen_cluster(nthreads=[], client=True)
-def test_scheduler_address_config(c, s):
+async def test_scheduler_address_config(c, s):
     with dask.config.set({"scheduler-address": s.address}):
-        nanny = yield Nanny(loop=s.loop)
+        nanny = await Nanny(loop=s.loop)
         assert nanny.scheduler.address == s.address
 
         start = time()
         while not s.workers:
-            yield gen.sleep(0.1)
+            await asyncio.sleep(0.1)
             assert time() < start + 10
 
-    yield nanny.close()
+    await nanny.close()
 
 
 @pytest.mark.slow
 @gen_test(timeout=20)
-def test_wait_for_scheduler():
+async def test_wait_for_scheduler():
     with captured_logger("distributed") as log:
         w = Nanny("127.0.0.1:44737")
         IOLoop.current().add_callback(w.start)
-        yield gen.sleep(6)
-        yield w.close()
+        await asyncio.sleep(6)
+        await w.close()
 
     log = log.getvalue()
     assert "error" not in log.lower(), log
@@ -385,31 +384,31 @@ def test_wait_for_scheduler():
 
 
 @gen_cluster(nthreads=[], client=True)
-def test_environment_variable(c, s):
+async def test_environment_variable(c, s):
     a = Nanny(s.address, loop=s.loop, memory_limit=0, env={"FOO": "123"})
     b = Nanny(s.address, loop=s.loop, memory_limit=0, env={"FOO": "456"})
-    yield [a, b]
-    results = yield c.run(lambda: os.environ["FOO"])
+    await asyncio.gather(a, b)
+    results = await c.run(lambda: os.environ["FOO"])
     assert results == {a.worker_address: "123", b.worker_address: "456"}
-    yield [a.close(), b.close()]
+    await asyncio.gather(a.close(), b.close())
 
 
 @gen_cluster(nthreads=[], client=True)
-def test_data_types(c, s):
-    w = yield Nanny(s.address, data=dict)
-    r = yield c.run(lambda dask_worker: type(dask_worker.data))
+async def test_data_types(c, s):
+    w = await Nanny(s.address, data=dict)
+    r = await c.run(lambda dask_worker: type(dask_worker.data))
     assert r[w.worker_address] == dict
-    yield w.close()
+    await w.close()
 
 
 @gen_cluster(nthreads=[])
-def test_local_directory(s):
+async def test_local_directory(s):
     with tmpfile() as fn:
         with dask.config.set(temporary_directory=fn):
-            w = yield Nanny(s.address)
+            w = await Nanny(s.address)
             assert w.local_directory.startswith(fn)
             assert "dask-worker-space" in w.local_directory
-            yield w.close()
+            await w.close()
 
 
 def _noop(x):
@@ -423,13 +422,13 @@ def _noop(x):
     Worker=Nanny,
     config={"distributed.worker.daemon": False},
 )
-def test_mp_process_worker_no_daemon(c, s, a):
+async def test_mp_process_worker_no_daemon(c, s, a):
     def multiprocessing_worker():
         p = mp.Process(target=_noop, args=(None,))
         p.start()
         p.join()
 
-    yield c.submit(multiprocessing_worker)
+    await c.submit(multiprocessing_worker)
 
 
 @gen_cluster(
@@ -438,12 +437,12 @@ def multiprocessing_worker():
     Worker=Nanny,
     config={"distributed.worker.daemon": False},
 )
-def test_mp_pool_worker_no_daemon(c, s, a):
+async def test_mp_pool_worker_no_daemon(c, s, a):
     def pool_worker(world_size):
         with mp.Pool(processes=world_size) as p:
             p.map(_noop, range(world_size))
 
-    yield c.submit(pool_worker, 4)
+    await c.submit(pool_worker, 4)
 
 
 @pytest.mark.asyncio
@@ -490,7 +489,7 @@ async def test_nanny_closes_cleanly(cleanup):
                     IOLoop.current().add_callback(w.terminate)
                     start = time()
                     while n.status != "closed":
-                        await gen.sleep(0.01)
+                        await asyncio.sleep(0.01)
                         assert time() < start + 5
 
                     assert n.status == "closed"
diff --git a/distributed/tests/test_priorities.py b/distributed/tests/test_priorities.py
index ae96517f1ac..cd4344da840 100644
--- a/distributed/tests/test_priorities.py
+++ b/distributed/tests/test_priorities.py
@@ -1,5 +1,6 @@
+import asyncio
+
 import pytest
-from tornado import gen
 
 from dask.core import flatten
 import dask
@@ -66,29 +67,29 @@ async def test_persist(c, s):
 
 
 @gen_cluster(client=True)
-def test_expand_compute(c, s, a, b):
+async def test_expand_compute(c, s, a, b):
     low = delayed(inc)(1)
     many = [delayed(slowinc)(i, delay=0.1) for i in range(10)]
     high = delayed(inc)(2)
 
     low, many, high = c.compute([low, many, high], priority={low: -1, high: 1})
-    yield wait(high)
+    await wait(high)
     assert s.tasks[low.key].state == "processing"
 
 
 @gen_cluster(client=True)
-def test_expand_persist(c, s, a, b):
+async def test_expand_persist(c, s, a, b):
     low = delayed(inc)(1, dask_key_name="low")
     many = [delayed(slowinc)(i, delay=0.1) for i in range(4)]
     high = delayed(inc)(2, dask_key_name="high")
 
     low, high, x, y, z, w = persist(low, high, *many, priority={low: -1, high: 1})
-    yield wait(high)
+    await wait(high)
     assert s.tasks[low.key].state == "processing"
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)])
-def test_repeated_persists_same_priority(c, s, w):
+async def test_repeated_persists_same_priority(c, s, w):
     xs = [delayed(slowinc)(i, delay=0.05, dask_key_name="x-%d" % i) for i in range(10)]
     ys = [
         delayed(slowinc)(x, delay=0.05, dask_key_name="y-%d" % i)
@@ -105,19 +106,19 @@ def test_repeated_persists_same_priority(c, s, w):
     while (
         sum(t.state == "memory" for t in s.tasks.values()) < 5
     ):  # TODO: reduce this number
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
     assert any(s.tasks[y.key].state == "memory" for y in ys)
     assert any(s.tasks[z.key].state == "memory" for z in zs)
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)])
-def test_last_in_first_out(c, s, w):
+async def test_last_in_first_out(c, s, w):
     xs = [c.submit(slowinc, i, delay=0.05) for i in range(5)]
     ys = [c.submit(slowinc, x, delay=0.05) for x in xs]
     zs = [c.submit(slowinc, y, delay=0.05) for y in ys]
 
     while len(s.tasks) < 15 or not any(s.tasks[z.key].state == "memory" for z in zs):
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
     assert not all(s.tasks[x.key].state == "memory" for x in xs)
diff --git a/distributed/tests/test_publish.py b/distributed/tests/test_publish.py
index dde10b11cf1..ab32d52a112 100644
--- a/distributed/tests/test_publish.py
+++ b/distributed/tests/test_publish.py
@@ -1,3 +1,4 @@
+import asyncio
 import pytest
 
 from dask import delayed
@@ -7,101 +8,93 @@
 from distributed.utils_test import gen_cluster, inc
 from distributed.utils_test import client, cluster_fixture, loop  # noqa F401
 from distributed.protocol import Serialized
-from tornado import gen
 
 
 @gen_cluster(client=False)
-def test_publish_simple(s, a, b):
-    c = yield Client(s.address, asynchronous=True)
-    f = yield Client(s.address, asynchronous=True)
+async def test_publish_simple(s, a, b):
+    c = Client(s.address, asynchronous=True)
+    f = Client(s.address, asynchronous=True)
+    await asyncio.gather(c, f)
 
-    data = yield c.scatter(range(3))
-    out = yield c.publish_dataset(data=data)
+    data = await c.scatter(range(3))
+    await c.publish_dataset(data=data)
     assert "data" in s.extensions["publish"].datasets
     assert isinstance(s.extensions["publish"].datasets["data"]["data"], Serialized)
 
     with pytest.raises(KeyError) as exc_info:
-        out = yield c.publish_dataset(data=data)
+        await c.publish_dataset(data=data)
 
     assert "exists" in str(exc_info.value)
     assert "data" in str(exc_info.value)
 
-    result = yield c.scheduler.publish_list()
+    result = await c.scheduler.publish_list()
     assert result == ("data",)
 
-    result = yield f.scheduler.publish_list()
+    result = await f.scheduler.publish_list()
     assert result == ("data",)
 
-    yield c.close()
-    yield f.close()
+    await asyncio.gather(c.close(), f.close())
 
 
 @gen_cluster(client=False)
-def test_publish_non_string_key(s, a, b):
-    c = yield Client(s.address, asynchronous=True)
-    f = yield Client(s.address, asynchronous=True)
-
-    try:
+async def test_publish_non_string_key(s, a, b):
+    async with Client(s.address, asynchronous=True) as c:
         for name in [("a", "b"), 9.0, 8]:
-            data = yield c.scatter(range(3))
-            out = yield c.publish_dataset(data, name=name)
+            data = await c.scatter(range(3))
+            await c.publish_dataset(data, name=name)
             assert name in s.extensions["publish"].datasets
             assert isinstance(
                 s.extensions["publish"].datasets[name]["data"], Serialized
             )
 
-            datasets = yield c.scheduler.publish_list()
+            datasets = await c.scheduler.publish_list()
             assert name in datasets
 
-    finally:
-        yield c.close()
-        yield f.close()
-
 
 @gen_cluster(client=False)
-def test_publish_roundtrip(s, a, b):
-    c = yield Client(s.address, asynchronous=True)
-    f = yield Client(s.address, asynchronous=True)
+async def test_publish_roundtrip(s, a, b):
+    c = await Client(s.address, asynchronous=True)
+    f = await Client(s.address, asynchronous=True)
 
-    data = yield c.scatter([0, 1, 2])
-    yield c.publish_dataset(data=data)
+    data = await c.scatter([0, 1, 2])
+    await c.publish_dataset(data=data)
 
     assert "published-data" in s.who_wants[data[0].key]
-    result = yield f.get_dataset(name="data")
+    result = await f.get_dataset(name="data")
 
     assert len(result) == len(data)
-    out = yield f.gather(result)
+    out = await f.gather(result)
     assert out == [0, 1, 2]
 
     with pytest.raises(KeyError) as exc_info:
-        result = yield f.get_dataset(name="nonexistent")
+        await f.get_dataset(name="nonexistent")
 
     assert "not found" in str(exc_info.value)
     assert "nonexistent" in str(exc_info.value)
 
-    yield c.close()
-    yield f.close()
+    await c.close()
+    await f.close()
 
 
 @gen_cluster(client=True)
-def test_unpublish(c, s, a, b):
-    data = yield c.scatter([0, 1, 2])
-    yield c.publish_dataset(data=data)
+async def test_unpublish(c, s, a, b):
+    data = await c.scatter([0, 1, 2])
+    await c.publish_dataset(data=data)
 
     key = data[0].key
     del data
 
-    yield c.scheduler.publish_delete(name="data")
+    await c.scheduler.publish_delete(name="data")
 
     assert "data" not in s.extensions["publish"].datasets
 
     start = time()
     while key in s.who_wants:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 5
 
     with pytest.raises(KeyError) as exc_info:
-        result = yield c.get_dataset(name="data")
+        await c.get_dataset(name="data")
 
     assert "not found" in str(exc_info.value)
     assert "data" in str(exc_info.value)
@@ -113,19 +106,19 @@ def test_unpublish_sync(client):
     client.unpublish_dataset(name="data")
 
     with pytest.raises(KeyError) as exc_info:
-        result = client.get_dataset(name="data")
+        client.get_dataset(name="data")
 
     assert "not found" in str(exc_info.value)
     assert "data" in str(exc_info.value)
 
 
 @gen_cluster(client=True)
-def test_publish_multiple_datasets(c, s, a, b):
+async def test_publish_multiple_datasets(c, s, a, b):
     x = delayed(inc)(1)
     y = delayed(inc)(2)
 
-    yield c.publish_dataset(x=x, y=y)
-    datasets = yield c.scheduler.publish_list()
+    await c.publish_dataset(x=x, y=y)
+    datasets = await c.scheduler.publish_list()
     assert set(datasets) == {"x", "y"}
 
 
@@ -136,7 +129,7 @@ def test_unpublish_multiple_datasets_sync(client):
     client.unpublish_dataset(name="x")
 
     with pytest.raises(KeyError) as exc_info:
-        result = client.get_dataset(name="x")
+        client.get_dataset(name="x")
 
     datasets = client.list_datasets()
     assert set(datasets) == {"y"}
@@ -147,17 +140,17 @@ def test_unpublish_multiple_datasets_sync(client):
     client.unpublish_dataset(name="y")
 
     with pytest.raises(KeyError) as exc_info:
-        result = client.get_dataset(name="y")
+        client.get_dataset(name="y")
 
     assert "not found" in str(exc_info.value)
     assert "y" in str(exc_info.value)
 
 
 @gen_cluster(client=False)
-def test_publish_bag(s, a, b):
+async def test_publish_bag(s, a, b):
     db = pytest.importorskip("dask.bag")
-    c = yield Client(s.address, asynchronous=True)
-    f = yield Client(s.address, asynchronous=True)
+    c = await Client(s.address, asynchronous=True)
+    f = await Client(s.address, asynchronous=True)
 
     bag = db.from_sequence([0, 1, 2])
     bagp = c.persist(bag)
@@ -166,19 +159,19 @@ def test_publish_bag(s, a, b):
     keys = {f.key for f in futures_of(bagp)}
     assert keys == set(bag.dask)
 
-    yield c.publish_dataset(data=bagp)
+    await c.publish_dataset(data=bagp)
 
     # check that serialization didn't affect original bag's dask
     assert len(futures_of(bagp)) == 3
 
-    result = yield f.get_dataset("data")
+    result = await f.get_dataset("data")
     assert set(result.dask.keys()) == set(bagp.dask.keys())
     assert {f.key for f in result.dask.values()} == {f.key for f in bagp.dask.values()}
 
-    out = yield f.compute(result)
+    out = await f.compute(result)
     assert out == [0, 1, 2]
-    yield c.close()
-    yield f.close()
+    await c.close()
+    await f.close()
 
 
 def test_datasets_setitem(client):
@@ -223,19 +216,16 @@ def test_datasets_iter(client):
 
 
 @gen_cluster(client=True)
-def test_pickle_safe(c, s, a, b):
-    c2 = yield Client(s.address, asynchronous=True, serializers=["msgpack"])
-    try:
-        yield c2.publish_dataset(x=[1, 2, 3])
-        result = yield c2.get_dataset("x")
+async def test_pickle_safe(c, s, a, b):
+    async with Client(s.address, asynchronous=True, serializers=["msgpack"]) as c2:
+        await c2.publish_dataset(x=[1, 2, 3])
+        result = await c2.get_dataset("x")
         assert result == [1, 2, 3]
 
         with pytest.raises(TypeError):
-            yield c2.publish_dataset(y=lambda x: x)
+            await c2.publish_dataset(y=lambda x: x)
 
-        yield c.publish_dataset(z=lambda x: x)  # this can use pickle
+        await c.publish_dataset(z=lambda x: x)  # this can use pickle
 
         with pytest.raises(TypeError):
-            yield c2.get_dataset("z")
-    finally:
-        yield c2.close()
+            await c2.get_dataset("z")
diff --git a/distributed/tests/test_pubsub.py b/distributed/tests/test_pubsub.py
index 639542df5ca..212d29d4802 100644
--- a/distributed/tests/test_pubsub.py
+++ b/distributed/tests/test_pubsub.py
@@ -2,7 +2,6 @@
 from time import sleep
 
 import pytest
-from tornado import gen
 import tlz as toolz
 
 from distributed import Pub, Sub, wait, get_worker, TimeoutError
@@ -11,7 +10,7 @@
 
 
 @gen_cluster(client=True, timeout=None)
-def test_speed(c, s, a, b):
+async def test_speed(c, s, a, b):
     """
     This tests how quickly we can move messages back and forth
 
@@ -45,13 +44,13 @@ def pingpong(a, b, start=False, n=1000, msg=1):
     y = c.submit(pingpong, "b", "a", n=100)
 
     start = time()
-    yield c.gather([x, y])
+    await c.gather([x, y])
     stop = time()
     # print('duration', stop - start)  # I get around 3ms/roundtrip on my laptop
 
 
 @gen_cluster(client=True, nthreads=[])
-def test_client(c, s):
+async def test_client(c, s):
     with pytest.raises(Exception):
         get_worker()
     sub = Sub("a")
@@ -62,17 +61,17 @@ def test_client(c, s):
 
     start = time()
     while not set(sps.client_subscribers["a"]) == {c.id}:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 3
 
     pub.put(123)
 
-    result = yield sub.__anext__()
+    result = await sub.__anext__()
     assert result == 123
 
 
 @gen_cluster(client=True)
-def test_client_worker(c, s, a, b):
+async def test_client_worker(c, s, a, b):
     sub = Sub("a", client=c, worker=None)
 
     def f(x):
@@ -80,11 +79,11 @@ def f(x):
         pub.put(x)
 
     futures = c.map(f, range(10))
-    yield wait(futures)
+    await wait(futures)
 
     L = []
     for i in range(10):
-        result = yield sub.get()
+        result = await sub.get()
         L.append(result)
 
     assert set(L) == set(range(10))
@@ -101,7 +100,7 @@ def f(x):
         or bps.publishers["a"]
         or len(sps.client_subscribers["a"]) != 1
     ):
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 3
 
     del sub
@@ -112,20 +111,20 @@ def f(x):
         or any(aps.publish_to_scheduler.values())
         or any(bps.publish_to_scheduler.values())
     ):
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 3
 
 
 @gen_cluster(client=True)
-def test_timeouts(c, s, a, b):
+async def test_timeouts(c, s, a, b):
     sub = Sub("a", client=c, worker=None)
     start = time()
     with pytest.raises(TimeoutError):
-        yield sub.get(timeout=0.1)
+        await sub.get(timeout=0.1)
     stop = time()
     assert stop - start < 1
     with pytest.raises(TimeoutError):
-        yield sub.get(timeout=0.01)
+        await sub.get(timeout=0.01)
 
 
 @gen_cluster(client=True)
@@ -140,13 +139,13 @@ async def test_repr(c, s, a, b):
 
 @pytest.mark.xfail(reason="out of order execution")
 @gen_cluster(client=True)
-def test_basic(c, s, a, b):
+async def test_basic(c, s, a, b):
     async def publish():
         pub = Pub("a")
 
         i = 0
         while True:
-            await gen.sleep(0.01)
+            await asyncio.sleep(0.01)
             pub._put(i)
             i += 1
 
@@ -157,7 +156,7 @@ def f(_):
     asyncio.ensure_future(c.run(publish, workers=[a.address]))
 
     tasks = [c.submit(f, i) for i in range(4)]
-    results = yield c.gather(tasks)
+    results = await c.gather(tasks)
 
     for r in results:
         x = r[0]
diff --git a/distributed/tests/test_queues.py b/distributed/tests/test_queues.py
index d797433d6b4..34009602a15 100644
--- a/distributed/tests/test_queues.py
+++ b/distributed/tests/test_queues.py
@@ -1,8 +1,7 @@
-from time import sleep
 import asyncio
+from time import sleep
 
 import pytest
-from tornado import gen
 
 from distributed import Client, Queue, Nanny, worker_client, wait, TimeoutError
 from distributed.metrics import time
@@ -11,47 +10,47 @@
 
 
 @gen_cluster(client=True)
-def test_queue(c, s, a, b):
-    x = yield Queue("x")
-    y = yield Queue("y")
-    xx = yield Queue("x")
+async def test_queue(c, s, a, b):
+    x = await Queue("x")
+    y = await Queue("y")
+    xx = await Queue("x")
     assert x.client is c
 
     future = c.submit(inc, 1)
 
-    yield x.put(future)
-    yield y.put(future)
-    future2 = yield xx.get()
+    await x.put(future)
+    await y.put(future)
+    future2 = await xx.get()
     assert future.key == future2.key
 
     with pytest.raises(TimeoutError):
-        yield x.get(timeout=0.1)
+        await x.get(timeout=0.1)
 
     del future, future2
 
-    yield gen.sleep(0.1)
+    await asyncio.sleep(0.1)
     assert s.tasks  # future still present in y's queue
-    yield y.get()  # burn future
+    await y.get()  # burn future
 
     start = time()
     while s.tasks:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 5
 
 
 @gen_cluster(client=True)
-def test_queue_with_data(c, s, a, b):
-    x = yield Queue("x")
-    xx = yield Queue("x")
+async def test_queue_with_data(c, s, a, b):
+    x = await Queue("x")
+    xx = await Queue("x")
     assert x.client is c
 
-    yield x.put((1, "hello"))
-    data = yield xx.get()
+    await x.put((1, "hello"))
+    data = await xx.get()
 
     assert data == (1, "hello")
 
     with pytest.raises(TimeoutError):
-        yield x.get(timeout=0.1)
+        await x.get(timeout=0.1)
 
 
 def test_sync(client):
@@ -67,35 +66,35 @@ def test_sync(client):
 
 
 @gen_cluster()
-def test_hold_futures(s, a, b):
-    c1 = yield Client(s.address, asynchronous=True)
+async def test_hold_futures(s, a, b):
+    c1 = await Client(s.address, asynchronous=True)
     future = c1.submit(lambda x: x + 1, 10)
-    q1 = yield Queue("q")
-    yield q1.put(future)
+    q1 = await Queue("q")
+    await q1.put(future)
     del q1
-    yield c1.close()
+    await c1.close()
 
-    yield gen.sleep(0.1)
+    await asyncio.sleep(0.1)
 
-    c2 = yield Client(s.address, asynchronous=True)
-    q2 = yield Queue("q")
-    future2 = yield q2.get()
-    result = yield future2
+    c2 = await Client(s.address, asynchronous=True)
+    q2 = await Queue("q")
+    future2 = await q2.get()
+    result = await future2
 
     assert result == 11
-    yield c2.close()
+    await c2.close()
 
 
 @pytest.mark.skip(reason="getting same client from main thread")
 @gen_cluster(client=True)
-def test_picklability(c, s, a, b):
+async def test_picklability(c, s, a, b):
     q = Queue()
 
     def f(x):
         q.put(x + 1)
 
-    yield c.submit(f, 10)
-    result = yield q.get()
+    await c.submit(f, 10)
+    result = await q.get()
     assert result == 11
 
 
@@ -112,7 +111,7 @@ def f(x):
 
 @pytest.mark.slow
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 2)] * 5, Worker=Nanny, timeout=None)
-def test_race(c, s, *workers):
+async def test_race(c, s, *workers):
     def f(i):
         with worker_client() as c:
             q = Queue("x", client=c)
@@ -126,144 +125,144 @@ def f(i):
             return result
 
     q = Queue("x", client=c)
-    L = yield c.scatter(range(5))
+    L = await c.scatter(range(5))
     for future in L:
-        yield q.put(future)
+        await q.put(future)
 
     futures = c.map(f, range(5))
-    results = yield c.gather(futures)
+    results = await c.gather(futures)
     assert all(r > 50 for r in results)
     assert sum(results) == 510
-    qsize = yield q.qsize()
+    qsize = await q.qsize()
     assert not qsize
 
 
 @gen_cluster(client=True)
-def test_same_futures(c, s, a, b):
+async def test_same_futures(c, s, a, b):
     q = Queue("x")
-    future = yield c.scatter(123)
+    future = await c.scatter(123)
 
     for i in range(5):
-        yield q.put(future)
+        await q.put(future)
 
     assert s.wants_what["queue-x"] == {future.key}
 
     for i in range(4):
-        future2 = yield q.get()
+        future2 = await q.get()
         assert s.wants_what["queue-x"] == {future.key}
-        yield gen.sleep(0.05)
+        await asyncio.sleep(0.05)
         assert s.wants_what["queue-x"] == {future.key}
 
-    yield q.get()
+    await q.get()
 
     start = time()
     while s.wants_what["queue-x"]:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() - start < 2
 
 
 @gen_cluster(client=True)
-def test_get_many(c, s, a, b):
-    x = yield Queue("x")
-    xx = yield Queue("x")
+async def test_get_many(c, s, a, b):
+    x = await Queue("x")
+    xx = await Queue("x")
 
-    yield x.put(1)
-    yield x.put(2)
-    yield x.put(3)
+    await x.put(1)
+    await x.put(2)
+    await x.put(3)
 
-    data = yield xx.get(batch=True)
+    data = await xx.get(batch=True)
     assert data == [1, 2, 3]
 
-    yield x.put(1)
-    yield x.put(2)
-    yield x.put(3)
+    await x.put(1)
+    await x.put(2)
+    await x.put(3)
 
-    data = yield xx.get(batch=2)
+    data = await xx.get(batch=2)
     assert data == [1, 2]
 
     with pytest.raises(TimeoutError):
-        data = yield asyncio.wait_for(xx.get(batch=2), 0.1)
+        await asyncio.wait_for(xx.get(batch=2), 0.1)
 
 
 @gen_cluster(client=True)
-def test_Future_knows_status_immediately(c, s, a, b):
-    x = yield c.scatter(123)
-    q = yield Queue("q")
-    yield q.put(x)
-
-    c2 = yield Client(s.address, asynchronous=True)
-    q2 = yield Queue("q", client=c2)
-    future = yield q2.get()
+async def test_Future_knows_status_immediately(c, s, a, b):
+    x = await c.scatter(123)
+    q = await Queue("q")
+    await q.put(x)
+
+    c2 = await Client(s.address, asynchronous=True)
+    q2 = await Queue("q", client=c2)
+    future = await q2.get()
     assert future.status == "finished"
 
     x = c.submit(div, 1, 0)
-    yield wait(x)
-    yield q.put(x)
+    await wait(x)
+    await q.put(x)
 
-    future2 = yield q2.get()
+    future2 = await q2.get()
     assert future2.status == "error"
     with pytest.raises(Exception):
-        yield future2
+        await future2
 
     start = time()
     while True:  # we learn about the true error eventually
         try:
-            yield future2
+            await future2
         except ZeroDivisionError:
             break
         except Exception:
             assert time() < start + 5
-            yield gen.sleep(0.05)
+            await asyncio.sleep(0.05)
 
-    yield c2.close()
+    await c2.close()
 
 
 @gen_cluster(client=True)
-def test_erred_future(c, s, a, b):
+async def test_erred_future(c, s, a, b):
     future = c.submit(div, 1, 0)
-    q = yield Queue()
-    yield q.put(future)
-    yield gen.sleep(0.1)
-    future2 = yield q.get()
+    q = await Queue()
+    await q.put(future)
+    await asyncio.sleep(0.1)
+    future2 = await q.get()
     with pytest.raises(ZeroDivisionError):
-        yield future2.result()
+        await future2.result()
 
-    exc = yield future2.exception()
+    exc = await future2.exception()
     assert isinstance(exc, ZeroDivisionError)
 
 
 @gen_cluster(client=True)
-def test_close(c, s, a, b):
-    q = yield Queue()
+async def test_close(c, s, a, b):
+    q = await Queue()
 
     q.close()
     q.close()
 
     while q.name in s.extensions["queues"].queues:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
 
 @gen_cluster(client=True)
-def test_timeout(c, s, a, b):
-    q = yield Queue("v", maxsize=1)
+async def test_timeout(c, s, a, b):
+    q = await Queue("v", maxsize=1)
 
     start = time()
     with pytest.raises(TimeoutError):
-        yield q.get(timeout=0.3)
+        await q.get(timeout=0.3)
     stop = time()
     assert 0.2 < stop - start < 2.0
 
-    yield q.put(1)
+    await q.put(1)
 
     start = time()
     with pytest.raises(TimeoutError):
-        yield q.put(2, timeout=0.3)
+        await q.put(2, timeout=0.3)
     stop = time()
     assert 0.1 < stop - start < 2.0
 
 
 @gen_cluster(client=True)
-def test_2220(c, s, a, b):
+async def test_2220(c, s, a, b):
     q = Queue()
 
     def put():
@@ -275,4 +274,4 @@ def get():
     fut = c.submit(put)
     res = c.submit(get)
 
-    yield [res, fut]
+    await c.gather([res, fut])
diff --git a/distributed/tests/test_resources.py b/distributed/tests/test_resources.py
index 648a191224e..870b930fdae 100644
--- a/distributed/tests/test_resources.py
+++ b/distributed/tests/test_resources.py
@@ -1,8 +1,8 @@
+import asyncio
 from time import time
 
 from dask import delayed
 import pytest
-from tornado import gen
 
 from distributed import Worker
 from distributed.client import wait
@@ -13,24 +13,23 @@
 
 
 @gen_cluster(client=True, nthreads=[])
-def test_resources(c, s):
+async def test_resources(c, s):
     assert not s.worker_resources
     assert not s.resources
 
     a = Worker(s.address, loop=s.loop, resources={"GPU": 2})
     b = Worker(s.address, loop=s.loop, resources={"GPU": 1, "DB": 1})
-
-    yield [a, b]
+    await asyncio.gather(a, b)
 
     assert s.resources == {"GPU": {a.address: 2, b.address: 1}, "DB": {b.address: 1}}
     assert s.worker_resources == {a.address: {"GPU": 2}, b.address: {"GPU": 1, "DB": 1}}
 
-    yield b.close()
+    await b.close()
 
     assert s.resources == {"GPU": {a.address: 2}, "DB": {}}
     assert s.worker_resources == {a.address: {"GPU": 2}}
 
-    yield a.close()
+    await a.close()
 
 
 @gen_cluster(
@@ -40,25 +39,25 @@ def test_resources(c, s):
         ("127.0.0.1", 1, {"resources": {"A": 1, "B": 1}}),
     ],
 )
-def test_resource_submit(c, s, a, b):
+async def test_resource_submit(c, s, a, b):
     x = c.submit(inc, 1, resources={"A": 3})
     y = c.submit(inc, 2, resources={"B": 1})
     z = c.submit(inc, 3, resources={"C": 2})
 
-    yield wait(x)
+    await wait(x)
     assert x.key in a.data
 
-    yield wait(y)
+    await wait(y)
     assert y.key in b.data
 
     assert s.get_task_status(keys=[z.key]) == {z.key: "no-worker"}
 
-    d = yield Worker(s.address, loop=s.loop, resources={"C": 10})
+    d = await Worker(s.address, loop=s.loop, resources={"C": 10})
 
-    yield wait(z)
+    await wait(z)
     assert z.key in d.data
 
-    yield d.close()
+    await d.close()
 
 
 @gen_cluster(
@@ -68,9 +67,9 @@ def test_resource_submit(c, s, a, b):
         ("127.0.0.1", 1, {"resources": {"B": 1}}),
     ],
 )
-def test_submit_many_non_overlapping(c, s, a, b):
+async def test_submit_many_non_overlapping(c, s, a, b):
     futures = [c.submit(inc, i, resources={"A": 1}) for i in range(5)]
-    yield wait(futures)
+    await wait(futures)
 
     assert len(a.data) == 5
     assert len(b.data) == 0
@@ -83,12 +82,12 @@ def test_submit_many_non_overlapping(c, s, a, b):
         ("127.0.0.1", 1, {"resources": {"B": 1}}),
     ],
 )
-def test_move(c, s, a, b):
-    [x] = yield c._scatter([1], workers=b.address)
+async def test_move(c, s, a, b):
+    [x] = await c._scatter([1], workers=b.address)
 
     future = c.submit(inc, x, resources={"A": 1})
 
-    yield wait(future)
+    await wait(future)
     assert a.data[future.key] == 2
 
 
@@ -99,14 +98,14 @@ def test_move(c, s, a, b):
         ("127.0.0.1", 1, {"resources": {"B": 1}}),
     ],
 )
-def test_dont_work_steal(c, s, a, b):
-    [x] = yield c._scatter([1], workers=a.address)
+async def test_dont_work_steal(c, s, a, b):
+    [x] = await c._scatter([1], workers=a.address)
 
     futures = [
         c.submit(slowadd, x, i, resources={"A": 1}, delay=0.05) for i in range(10)
     ]
 
-    yield wait(futures)
+    await wait(futures)
     assert all(f.key in a.data for f in futures)
 
 
@@ -117,9 +116,9 @@ def test_dont_work_steal(c, s, a, b):
         ("127.0.0.1", 1, {"resources": {"B": 1}}),
     ],
 )
-def test_map(c, s, a, b):
+async def test_map(c, s, a, b):
     futures = c.map(inc, range(10), resources={"B": 1})
-    yield wait(futures)
+    await wait(futures)
     assert set(b.data) == {f.key for f in futures}
     assert not a.data
 
@@ -131,13 +130,13 @@ def test_map(c, s, a, b):
         ("127.0.0.1", 1, {"resources": {"B": 1}}),
     ],
 )
-def test_persist(c, s, a, b):
+async def test_persist(c, s, a, b):
     x = delayed(inc)(1)
     y = delayed(inc)(x)
 
     xx, yy = c.persist([x, y], resources={x: {"A": 1}, y: {"B": 1}})
 
-    yield wait([xx, yy])
+    await wait([xx, yy])
 
     assert x.key in a.data
     assert y.key in b.data
@@ -150,18 +149,18 @@ def test_persist(c, s, a, b):
         ("127.0.0.1", 1, {"resources": {"B": 11}}),
     ],
 )
-def test_compute(c, s, a, b):
+async def test_compute(c, s, a, b):
     x = delayed(inc)(1)
     y = delayed(inc)(x)
 
     yy = c.compute(y, resources={x: {"A": 1}, y: {"B": 1}})
-    yield wait(yy)
+    await wait(yy)
 
     assert b.data
 
     xs = [delayed(inc)(i) for i in range(10, 20)]
     xxs = c.compute(xs, resources={"B": 1})
-    yield wait(xxs)
+    await wait(xxs)
 
     assert len(b.data) > 10
 
@@ -173,10 +172,10 @@ def test_compute(c, s, a, b):
         ("127.0.0.1", 1, {"resources": {"B": 1}}),
     ],
 )
-def test_get(c, s, a, b):
+async def test_get(c, s, a, b):
     dsk = {"x": (inc, 1), "y": (inc, "x")}
 
-    result = yield c.get(dsk, "y", resources={"y": {"A": 1}}, sync=False)
+    result = await c.get(dsk, "y", resources={"y": {"A": 1}}, sync=False)
     assert result == 3
 
 
@@ -187,13 +186,13 @@ def test_get(c, s, a, b):
         ("127.0.0.1", 1, {"resources": {"B": 1}}),
     ],
 )
-def test_persist_tuple(c, s, a, b):
+async def test_persist_tuple(c, s, a, b):
     x = delayed(inc)(1)
     y = delayed(inc)(x)
 
     xx, yy = c.persist([x, y], resources={(x, y): {"A": 1}})
 
-    yield wait([xx, yy])
+    await wait([xx, yy])
 
     assert x.key in a.data
     assert y.key in a.data
@@ -201,16 +200,16 @@ def test_persist_tuple(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_resources_str(c, s, a, b):
+async def test_resources_str(c, s, a, b):
     pd = pytest.importorskip("pandas")
     dd = pytest.importorskip("dask.dataframe")
 
-    yield a.set_resources(MyRes=1)
+    await a.set_resources(MyRes=1)
 
     x = dd.from_pandas(pd.DataFrame({"A": [1, 2], "B": [3, 4]}), npartitions=1)
     y = x.apply(lambda row: row.sum(), axis=1, meta=(None, "int64"))
     yy = y.persist(resources={"MyRes": 1})
-    yield wait(yy)
+    await wait(yy)
 
     ts_first = s.tasks[tokey(y.__dask_keys__()[0])]
     assert ts_first.resource_restrictions == {"MyRes": 1}
@@ -225,38 +224,38 @@ def test_resources_str(c, s, a, b):
         ("127.0.0.1", 4, {"resources": {"A": 1}}),
     ],
 )
-def test_submit_many_non_overlapping(c, s, a, b):
+async def test_submit_many_non_overlapping(c, s, a, b):
     futures = c.map(slowinc, range(100), resources={"A": 1}, delay=0.02)
 
     while len(a.data) + len(b.data) < 100:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert len(a.executing) <= 2
         assert len(b.executing) <= 1
 
-    yield wait(futures)
+    await wait(futures)
     assert a.total_resources == a.available_resources
     assert b.total_resources == b.available_resources
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 4, {"resources": {"A": 2, "B": 1}})])
-def test_minimum_resource(c, s, a):
+async def test_minimum_resource(c, s, a):
     futures = c.map(slowinc, range(30), resources={"A": 1, "B": 1}, delay=0.02)
 
     while len(a.data) < 30:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert len(a.executing) <= 1
 
-    yield wait(futures)
+    await wait(futures)
     assert a.total_resources == a.available_resources
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 2, {"resources": {"A": 1}})])
-def test_prefer_constrained(c, s, a):
+async def test_prefer_constrained(c, s, a):
     futures = c.map(slowinc, range(1000), delay=0.1)
     constrained = c.map(inc, range(10), resources={"A": 1})
 
     start = time()
-    yield wait(constrained)
+    await wait(constrained)
     end = time()
     assert end - start < 4
     has_what = dict(s.has_what)
@@ -273,27 +272,27 @@ def test_prefer_constrained(c, s, a):
         ("127.0.0.1", 2, {"resources": {"A": 1}}),
     ],
 )
-def test_balance_resources(c, s, a, b):
+async def test_balance_resources(c, s, a, b):
     futures = c.map(slowinc, range(100), delay=0.1, workers=a.address)
     constrained = c.map(inc, range(2), resources={"A": 1})
 
-    yield wait(constrained)
+    await wait(constrained)
     assert any(f.key in a.data for f in constrained)  # share
     assert any(f.key in b.data for f in constrained)
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 2)])
-def test_set_resources(c, s, a):
-    yield a.set_resources(A=2)
+async def test_set_resources(c, s, a):
+    await a.set_resources(A=2)
     assert a.total_resources["A"] == 2
     assert a.available_resources["A"] == 2
     assert s.worker_resources[a.address] == {"A": 2}
 
     future = c.submit(slowinc, 1, delay=1, resources={"A": 1})
     while a.available_resources["A"] == 2:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
-    yield a.set_resources(A=3)
+    await a.set_resources(A=3)
     assert a.total_resources["A"] == 3
     assert a.available_resources["A"] == 2
     assert s.worker_resources[a.address] == {"A": 3}
@@ -306,7 +305,7 @@ def test_set_resources(c, s, a):
         ("127.0.0.1", 1, {"resources": {"B": 1}}),
     ],
 )
-def test_persist_collections(c, s, a, b):
+async def test_persist_collections(c, s, a, b):
     da = pytest.importorskip("dask.array")
     x = da.arange(10, chunks=(5,))
     y = x.map_blocks(lambda x: x + 1)
@@ -315,7 +314,7 @@ def test_persist_collections(c, s, a, b):
 
     ww, yy = c.persist([w, y], resources={tuple(y.__dask_keys__()): {"A": 1}})
 
-    yield wait([ww, yy])
+    await wait([ww, yy])
 
     assert all(tokey(key) in a.data for key in y.__dask_keys__())
 
@@ -328,14 +327,14 @@ def test_persist_collections(c, s, a, b):
         ("127.0.0.1", 1, {"resources": {"B": 1}}),
     ],
 )
-def test_dont_optimize_out(c, s, a, b):
+async def test_dont_optimize_out(c, s, a, b):
     da = pytest.importorskip("dask.array")
     x = da.arange(10, chunks=(5,))
     y = x.map_blocks(lambda x: x + 1)
     z = y.map_blocks(lambda x: 2 * x)
     w = z.sum()
 
-    yield c.compute(w, resources={tuple(y.__dask_keys__()): {"A": 1}})
+    await c.compute(w, resources={tuple(y.__dask_keys__()): {"A": 1}})
 
     for key in map(tokey, y.__dask_keys__()):
         assert "executing" in str(a.story(key))
@@ -349,14 +348,14 @@ def test_dont_optimize_out(c, s, a, b):
         ("127.0.0.1", 1, {"resources": {"B": 1}}),
     ],
 )
-def test_full_collections(c, s, a, b):
+async def test_full_collections(c, s, a, b):
     dd = pytest.importorskip("dask.dataframe")
     df = dd.demo.make_timeseries(
         freq="60s", partition_freq="1d", start="2000-01-01", end="2000-01-31"
     )
     z = df.x + df.y  # some extra nodes in the graph
 
-    yield c.compute(z, resources={tuple(z.dask): {"A": 1}})
+    await c.compute(z, resources={tuple(z.dask): {"A": 1}})
     assert a.log
     assert not b.log
 
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index d64c88fceea..5ed8e4e542d 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -1,18 +1,17 @@
 import asyncio
-import cloudpickle
-import pickle
-from collections import defaultdict
 import json
+import logging
+import pickle
 import operator
 import re
 import sys
+from collections import defaultdict
 from time import sleep
-import logging
 
+import cloudpickle
 import dask
 from dask import delayed
 from tlz import merge, concat, valmap, first, frequencies
-from tornado import gen
 
 import pytest
 
@@ -50,7 +49,7 @@
 
 
 @gen_cluster()
-def test_administration(s, a, b):
+async def test_administration(s, a, b):
     assert isinstance(s.address, str)
     assert s.address in str(s)
     assert str(sum(s.nthreads.values())) in repr(s)
@@ -58,11 +57,11 @@ def test_administration(s, a, b):
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)])
-def test_respect_data_in_memory(c, s, a):
+async def test_respect_data_in_memory(c, s, a):
     x = delayed(inc)(1)
     y = delayed(inc)(x)
     f = c.persist(y)
-    yield wait([f])
+    await wait([f])
 
     assert s.tasks[y.key].who_has == {s.workers[a.address]}
 
@@ -70,37 +69,37 @@ def test_respect_data_in_memory(c, s, a):
     f2 = c.persist(z)
     while f2.key not in s.tasks or not s.tasks[f2.key]:
         assert s.tasks[y.key].who_has
-        yield gen.sleep(0.0001)
+        await asyncio.sleep(0.0001)
 
 
 @gen_cluster(client=True)
-def test_recompute_released_results(c, s, a, b):
+async def test_recompute_released_results(c, s, a, b):
     x = delayed(inc)(1)
     y = delayed(inc)(x)
 
     yy = c.persist(y)
-    yield wait(yy)
+    await wait(yy)
 
     while s.tasks[x.key].who_has or x.key in a.data or x.key in b.data:  # let x go away
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
     z = delayed(dec)(x)
     zz = c.compute(z)
-    result = yield zz
+    result = await zz
     assert result == 1
 
 
 @gen_cluster(client=True)
-def test_decide_worker_with_many_independent_leaves(c, s, a, b):
-    xs = yield [
+async def test_decide_worker_with_many_independent_leaves(c, s, a, b):
+    xs = await asyncio.gather(
         c.scatter(list(range(0, 100, 2)), workers=a.address),
         c.scatter(list(range(1, 100, 2)), workers=b.address),
-    ]
+    )
     xs = list(concat(zip(*xs)))
     ys = [delayed(inc)(x) for x in xs]
 
     y2s = c.persist(ys)
-    yield wait(y2s)
+    await wait(y2s)
 
     nhits = sum(y.key in a.data for y in y2s[::2]) + sum(
         y.key in b.data for y in y2s[1::2]
@@ -110,71 +109,70 @@ def test_decide_worker_with_many_independent_leaves(c, s, a, b):
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 3)
-def test_decide_worker_with_restrictions(client, s, a, b, c):
+async def test_decide_worker_with_restrictions(client, s, a, b, c):
     x = client.submit(inc, 1, workers=[a.address, b.address])
-    yield x
+    await x
     assert x.key in a.data or x.key in b.data
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 3)
-def test_move_data_over_break_restrictions(client, s, a, b, c):
-    [x] = yield client.scatter([1], workers=b.address)
+async def test_move_data_over_break_restrictions(client, s, a, b, c):
+    [x] = await client.scatter([1], workers=b.address)
     y = client.submit(inc, x, workers=[a.address, b.address])
-    yield wait(y)
+    await wait(y)
     assert y.key in a.data or y.key in b.data
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 3)
-def test_balance_with_restrictions(client, s, a, b, c):
-    [x], [y] = yield [
+async def test_balance_with_restrictions(client, s, a, b, c):
+    [x], [y] = await asyncio.gather(
         client.scatter([[1, 2, 3]], workers=a.address),
         client.scatter([1], workers=c.address),
-    ]
+    )
     z = client.submit(inc, 1, workers=[a.address, c.address])
-    yield wait(z)
+    await wait(z)
 
     assert s.tasks[z.key].who_has == {s.workers[c.address]}
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 3)
-def test_no_valid_workers(client, s, a, b, c):
+async def test_no_valid_workers(client, s, a, b, c):
     x = client.submit(inc, 1, workers="127.0.0.5:9999")
     while not s.tasks:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
     assert s.tasks[x.key] in s.unrunnable
 
     with pytest.raises(TimeoutError):
-        yield asyncio.wait_for(x, 0.05)
+        await asyncio.wait_for(x, 0.05)
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 3)
-def test_no_valid_workers_loose_restrictions(client, s, a, b, c):
+async def test_no_valid_workers_loose_restrictions(client, s, a, b, c):
     x = client.submit(inc, 1, workers="127.0.0.5:9999", allow_other_workers=True)
-
-    result = yield x
+    result = await x
     assert result == 2
 
 
 @gen_cluster(client=True, nthreads=[])
-def test_no_workers(client, s):
+async def test_no_workers(client, s):
     x = client.submit(inc, 1)
     while not s.tasks:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
     assert s.tasks[x.key] in s.unrunnable
 
     with pytest.raises(TimeoutError):
-        yield asyncio.wait_for(x, 0.05)
+        await asyncio.wait_for(x, 0.05)
 
 
 @gen_cluster(nthreads=[])
-def test_retire_workers_empty(s):
-    yield s.retire_workers(workers=[])
+async def test_retire_workers_empty(s):
+    await s.retire_workers(workers=[])
 
 
 @gen_cluster()
-def test_remove_client(s, a, b):
+async def test_remove_client(s, a, b):
     s.update_graph(
         tasks={"x": dumps_task((inc, 1)), "y": dumps_task((inc, "x"))},
         dependencies={"x": [], "y": ["x"]},
@@ -192,15 +190,15 @@ def test_remove_client(s, a, b):
 
 
 @gen_cluster()
-def test_server_listens_to_other_ops(s, a, b):
+async def test_server_listens_to_other_ops(s, a, b):
     with rpc(s.address) as r:
-        ident = yield r.identity()
+        ident = await r.identity()
         assert ident["type"] == "Scheduler"
         assert ident["id"].lower().startswith("scheduler")
 
 
 @gen_cluster()
-def test_remove_worker_from_scheduler(s, a, b):
+async def test_remove_worker_from_scheduler(s, a, b):
     dsk = {("x-%d" % i): (inc, i) for i in range(20)}
     s.update_graph(
         tasks=valmap(dumps_task, dsk),
@@ -216,7 +214,7 @@ def test_remove_worker_from_scheduler(s, a, b):
 
 
 @gen_cluster()
-def test_remove_worker_by_name_from_scheduler(s, a, b):
+async def test_remove_worker_by_name_from_scheduler(s, a, b):
     assert a.address in s.stream_comms
     assert s.remove_worker(address=a.name) == "OK"
     assert a.address not in s.nthreads
@@ -225,7 +223,7 @@ def test_remove_worker_by_name_from_scheduler(s, a, b):
 
 
 @gen_cluster(config={"distributed.scheduler.events-cleanup-delay": "10 ms"})
-def test_clear_events_worker_removal(s, a, b):
+async def test_clear_events_worker_removal(s, a, b):
     assert a.address in s.events
     assert a.address in s.nthreads
     assert b.address in s.events
@@ -239,7 +237,7 @@ def test_clear_events_worker_removal(s, a, b):
 
     start = time()
     while a.address in s.events:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 2
     assert b.address in s.events
 
@@ -247,7 +245,7 @@ def test_clear_events_worker_removal(s, a, b):
 @gen_cluster(
     config={"distributed.scheduler.events-cleanup-delay": "10 ms"}, client=True
 )
-def test_clear_events_client_removal(c, s, a, b):
+async def test_clear_events_client_removal(c, s, a, b):
     assert c.id in s.events
     s.remove_client(c.id)
 
@@ -259,12 +257,12 @@ def test_clear_events_client_removal(c, s, a, b):
     # If it doesn't reconnect after a given time, the events log should be cleared
     start = time()
     while c.id in s.events:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 2
 
 
 @gen_cluster()
-def test_add_worker(s, a, b):
+async def test_add_worker(s, a, b):
     w = Worker(s.address, nthreads=3)
     w.data["x-5"] = 6
     w.data["y"] = 1
@@ -277,23 +275,23 @@ def test_add_worker(s, a, b):
         dependencies={k: set() for k in dsk},
     )
     s.validate_state()
-    yield w
+    await w
     s.validate_state()
 
     assert w.ip in s.host_info
     assert s.host_info[w.ip]["addresses"] == {a.address, b.address, w.address}
-    yield w.close()
+    await w.close()
 
 
 @gen_cluster(scheduler_kwargs={"blocked_handlers": ["feed"]})
-def test_blocked_handlers_are_respected(s, a, b):
+async def test_blocked_handlers_are_respected(s, a, b):
     def func(scheduler):
         return dumps(dict(scheduler.worker_info))
 
-    comm = yield connect(s.address)
-    yield comm.write({"op": "feed", "function": dumps(func), "interval": 0.01})
+    comm = await connect(s.address)
+    await comm.write({"op": "feed", "function": dumps(func), "interval": 0.01})
 
-    response = yield comm.read()
+    response = await comm.read()
 
     assert "exception" in response
     assert isinstance(response["exception"], ValueError)
@@ -301,7 +299,7 @@ def func(scheduler):
         response["exception"]
     )
 
-    yield comm.close()
+    await comm.close()
 
 
 def test_scheduler_init_pulls_blocked_handlers_from_config():
@@ -311,23 +309,23 @@ def test_scheduler_init_pulls_blocked_handlers_from_config():
 
 
 @gen_cluster()
-def test_feed(s, a, b):
+async def test_feed(s, a, b):
     def func(scheduler):
         return dumps(dict(scheduler.worker_info))
 
-    comm = yield connect(s.address)
-    yield comm.write({"op": "feed", "function": dumps(func), "interval": 0.01})
+    comm = await connect(s.address)
+    await comm.write({"op": "feed", "function": dumps(func), "interval": 0.01})
 
     for i in range(5):
-        response = yield comm.read()
+        response = await comm.read()
         expected = dict(s.worker_info)
         assert cloudpickle.loads(response) == expected
 
-    yield comm.close()
+    await comm.close()
 
 
 @gen_cluster()
-def test_feed_setup_teardown(s, a, b):
+async def test_feed_setup_teardown(s, a, b):
     def setup(scheduler):
         return 1
 
@@ -338,8 +336,8 @@ def func(scheduler, state):
     def teardown(scheduler, state):
         scheduler.flag = "done"
 
-    comm = yield connect(s.address)
-    yield comm.write(
+    comm = await connect(s.address)
+    await comm.write(
         {
             "op": "feed",
             "function": dumps(func),
@@ -350,18 +348,18 @@ def teardown(scheduler, state):
     )
 
     for i in range(5):
-        response = yield comm.read()
+        response = await comm.read()
         assert response == "OK"
 
-    yield comm.close()
+    await comm.close()
     start = time()
     while not hasattr(s, "flag"):
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() - start < 5
 
 
 @gen_cluster()
-def test_feed_large_bytestring(s, a, b):
+async def test_feed_large_bytestring(s, a, b):
     np = pytest.importorskip("numpy")
 
     x = np.ones(10000000)
@@ -370,19 +368,19 @@ def func(scheduler):
         y = x
         return True
 
-    comm = yield connect(s.address)
-    yield comm.write({"op": "feed", "function": dumps(func), "interval": 0.05})
+    comm = await connect(s.address)
+    await comm.write({"op": "feed", "function": dumps(func), "interval": 0.05})
 
     for i in range(5):
-        response = yield comm.read()
+        response = await comm.read()
         assert response is True
 
-    yield comm.close()
+    await comm.close()
 
 
 @gen_cluster(client=True)
-def test_delete_data(c, s, a, b):
-    d = yield c.scatter({"x": 1, "y": 2, "z": 3})
+async def test_delete_data(c, s, a, b):
+    d = await c.scatter({"x": 1, "y": 2, "z": 3})
 
     assert {ts.key for ts in s.tasks.values() if ts.who_has} == {"x", "y", "z"}
     assert set(a.data) | set(b.data) == {"x", "y", "z"}
@@ -393,36 +391,36 @@ def test_delete_data(c, s, a, b):
 
     start = time()
     while set(a.data) | set(b.data) != {"z"}:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 5
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)])
-def test_delete(c, s, a):
+async def test_delete(c, s, a):
     x = c.submit(inc, 1)
-    yield x
+    await x
     assert x.key in a.data
 
-    yield c._cancel(x)
+    await c._cancel(x)
 
     start = time()
     while x.key in a.data:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 5
 
 
 @gen_cluster()
-def test_filtered_communication(s, a, b):
-    c = yield connect(s.address)
-    f = yield connect(s.address)
-    yield c.write({"op": "register-client", "client": "c", "versions": {}})
-    yield f.write({"op": "register-client", "client": "f", "versions": {}})
-    yield c.read()
-    yield f.read()
+async def test_filtered_communication(s, a, b):
+    c = await connect(s.address)
+    f = await connect(s.address)
+    await c.write({"op": "register-client", "client": "c", "versions": {}})
+    await f.write({"op": "register-client", "client": "f", "versions": {}})
+    await c.read()
+    await f.read()
 
     assert set(s.client_comms) == {"c", "f"}
 
-    yield c.write(
+    await c.write(
         {
             "op": "update-graph",
             "tasks": {"x": dumps_task((inc, 1)), "y": dumps_task((inc, "x"))},
@@ -432,7 +430,7 @@ def test_filtered_communication(s, a, b):
         }
     )
 
-    yield f.write(
+    await f.write(
         {
             "op": "update-graph",
             "tasks": {
@@ -444,10 +442,10 @@ def test_filtered_communication(s, a, b):
             "keys": ["z"],
         }
     )
-    (msg,) = yield c.read()
+    (msg,) = await c.read()
     assert msg["op"] == "key-in-memory"
     assert msg["key"] == "y"
-    (msg,) = yield f.read()
+    (msg,) = await f.read()
     assert msg["op"] == "key-in-memory"
     assert msg["key"] == "z"
 
@@ -480,7 +478,7 @@ def test_dumps_task():
 
 
 @gen_cluster()
-def test_ready_remove_worker(s, a, b):
+async def test_ready_remove_worker(s, a, b):
     s.update_graph(
         tasks={"x-%d" % i: dumps_task((inc, i)) for i in range(20)},
         keys=["x-%d" % i for i in range(20)],
@@ -497,11 +495,11 @@ def test_ready_remove_worker(s, a, b):
 
 
 @gen_cluster(client=True, Worker=Nanny)
-def test_restart(c, s, a, b):
+async def test_restart(c, s, a, b):
     futures = c.map(inc, range(20))
-    yield wait(futures)
+    await wait(futures)
 
-    yield s.restart()
+    await s.restart()
 
     assert len(s.workers) == 2
 
@@ -514,56 +512,56 @@ def test_restart(c, s, a, b):
 
 
 @gen_cluster()
-def test_broadcast(s, a, b):
-    result = yield s.broadcast(msg={"op": "ping"})
+async def test_broadcast(s, a, b):
+    result = await s.broadcast(msg={"op": "ping"})
     assert result == {a.address: b"pong", b.address: b"pong"}
 
-    result = yield s.broadcast(msg={"op": "ping"}, workers=[a.address])
+    result = await s.broadcast(msg={"op": "ping"}, workers=[a.address])
     assert result == {a.address: b"pong"}
 
-    result = yield s.broadcast(msg={"op": "ping"}, hosts=[a.ip])
+    result = await s.broadcast(msg={"op": "ping"}, hosts=[a.ip])
     assert result == {a.address: b"pong", b.address: b"pong"}
 
 
 @gen_cluster(Worker=Nanny)
-def test_broadcast_nanny(s, a, b):
-    result1 = yield s.broadcast(msg={"op": "identity"}, nanny=True)
+async def test_broadcast_nanny(s, a, b):
+    result1 = await s.broadcast(msg={"op": "identity"}, nanny=True)
     assert all(d["type"] == "Nanny" for d in result1.values())
 
-    result2 = yield s.broadcast(
+    result2 = await s.broadcast(
         msg={"op": "identity"}, workers=[a.worker_address], nanny=True
     )
     assert len(result2) == 1
     assert first(result2.values())["id"] == a.id
 
-    result3 = yield s.broadcast(msg={"op": "identity"}, hosts=[a.ip], nanny=True)
+    result3 = await s.broadcast(msg={"op": "identity"}, hosts=[a.ip], nanny=True)
     assert result1 == result3
 
 
 @gen_test()
-def test_worker_name():
-    s = yield Scheduler(validate=True, port=0)
-    w = yield Worker(s.address, name="alice")
+async def test_worker_name():
+    s = await Scheduler(validate=True, port=0)
+    w = await Worker(s.address, name="alice")
     assert s.workers[w.address].name == "alice"
     assert s.aliases["alice"] == w.address
 
     with pytest.raises(ValueError):
-        w2 = yield Worker(s.address, name="alice")
-        yield w2.close()
+        w2 = await Worker(s.address, name="alice")
+        await w2.close()
 
-    yield w.close()
-    yield s.close()
+    await w.close()
+    await s.close()
 
 
 @gen_test()
-def test_coerce_address():
+async def test_coerce_address():
     with dask.config.set({"distributed.comm.timeouts.connect": "100ms"}):
-        s = yield Scheduler(validate=True, port=0)
+        s = await Scheduler(validate=True, port=0)
         print("scheduler:", s.address, s.listen_address)
         a = Worker(s.address, name="alice")
         b = Worker(s.address, name=123)
         c = Worker("127.0.0.1", s.port, name="charlie")
-        yield [a, b, c]
+        await asyncio.gather(a, b, c)
 
         assert s.coerce_address("127.0.0.1:8000") == "tcp://127.0.0.1:8000"
         assert s.coerce_address("[::1]:8000") == "tcp://[::1]:8000"
@@ -591,8 +589,8 @@ def test_coerce_address():
 
         assert s.coerce_address("zzzt:8000", resolve=False) == "tcp://zzzt:8000"
 
-        yield s.close()
-        yield [w.close() for w in [a, b, c]]
+        await s.close()
+        await asyncio.gather(a.close(), b.close(), c.close())
 
 
 @pytest.mark.asyncio
@@ -612,24 +610,24 @@ async def test_config_stealing(cleanup):
     sys.platform.startswith("win"), reason="file descriptors not really a thing"
 )
 @gen_cluster(nthreads=[])
-def test_file_descriptors_dont_leak(s):
+async def test_file_descriptors_dont_leak(s):
     psutil = pytest.importorskip("psutil")
     proc = psutil.Process()
     before = proc.num_fds()
 
-    w = yield Worker(s.address)
-    yield w.close()
+    w = await Worker(s.address)
+    await w.close()
 
     during = proc.num_fds()
 
     start = time()
     while proc.num_fds() > before:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 5
 
 
 @gen_cluster()
-def test_update_graph_culls(s, a, b):
+async def test_update_graph_culls(s, a, b):
     s.update_graph(
         tasks={
             "x": dumps_task((inc, 1)),
@@ -650,11 +648,11 @@ def test_io_loop(loop):
 
 
 @gen_cluster(client=True)
-def test_story(c, s, a, b):
+async def test_story(c, s, a, b):
     x = delayed(inc)(1)
     y = delayed(inc)(x)
     f = c.persist(y)
-    yield wait([f])
+    await wait([f])
 
     assert s.transition_log
 
@@ -667,38 +665,38 @@ def test_story(c, s, a, b):
 
 
 @gen_cluster(nthreads=[], client=True)
-def test_scatter_no_workers(c, s):
+async def test_scatter_no_workers(c, s):
     with pytest.raises(TimeoutError):
-        yield s.scatter(data={"x": 1}, client="alice", timeout=0.1)
+        await s.scatter(data={"x": 1}, client="alice", timeout=0.1)
 
     start = time()
     with pytest.raises(TimeoutError):
-        yield c.scatter(123, timeout=0.1)
+        await c.scatter(123, timeout=0.1)
     assert time() < start + 1.5
 
     w = Worker(s.address, nthreads=3)
-    yield [c.scatter(data={"y": 2}, timeout=5), w]
+    await asyncio.gather(c.scatter(data={"y": 2}, timeout=5), w)
 
     assert w.data["y"] == 2
-    yield w.close()
+    await w.close()
 
 
 @gen_cluster(nthreads=[])
-def test_scheduler_sees_memory_limits(s):
-    w = yield Worker(s.address, nthreads=3, memory_limit=12345)
+async def test_scheduler_sees_memory_limits(s):
+    w = await Worker(s.address, nthreads=3, memory_limit=12345)
 
     assert s.workers[w.address].memory_limit == 12345
-    yield w.close()
+    await w.close()
 
 
 @gen_cluster(client=True, timeout=1000)
-def test_retire_workers(c, s, a, b):
-    [x] = yield c.scatter([1], workers=a.address)
-    [y] = yield c.scatter([list(range(1000))], workers=b.address)
+async def test_retire_workers(c, s, a, b):
+    [x] = await c.scatter([1], workers=a.address)
+    [y] = await c.scatter([list(range(1000))], workers=b.address)
 
     assert s.workers_to_close() == [a.address]
 
-    workers = yield s.retire_workers()
+    workers = await s.retire_workers()
     assert list(workers) == [a.address]
     assert workers[a.address]["nthreads"] == a.nthreads
     assert list(s.nthreads) == [b.address]
@@ -707,26 +705,26 @@ def test_retire_workers(c, s, a, b):
 
     assert s.workers[b.address].has_what == {s.tasks[x.key], s.tasks[y.key]}
 
-    workers = yield s.retire_workers()
+    workers = await s.retire_workers()
     assert not workers
 
 
 @gen_cluster(client=True)
-def test_retire_workers_n(c, s, a, b):
-    yield s.retire_workers(n=1, close_workers=True)
+async def test_retire_workers_n(c, s, a, b):
+    await s.retire_workers(n=1, close_workers=True)
     assert len(s.workers) == 1
 
-    yield s.retire_workers(n=0, close_workers=True)
+    await s.retire_workers(n=0, close_workers=True)
     assert len(s.workers) == 1
 
-    yield s.retire_workers(n=1, close_workers=True)
+    await s.retire_workers(n=1, close_workers=True)
     assert len(s.workers) == 0
 
-    yield s.retire_workers(n=0, close_workers=True)
+    await s.retire_workers(n=0, close_workers=True)
     assert len(s.workers) == 0
 
     while not (a.status.startswith("clos") and b.status.startswith("clos")):
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 4)
@@ -736,7 +734,7 @@ async def test_workers_to_close(cl, s, *workers):
     ):
         futures = cl.map(slowinc, [1, 1, 1], key=["a-4", "b-4", "c-1"])
         while sum(len(w.processing) for w in s.workers.values()) < 3:
-            await gen.sleep(0.001)
+            await asyncio.sleep(0.001)
 
         wtc = s.workers_to_close()
         assert all(not s.workers[w].processing for w in wtc)
@@ -744,7 +742,7 @@ async def test_workers_to_close(cl, s, *workers):
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 4)
-def test_workers_to_close_grouped(c, s, *workers):
+async def test_workers_to_close_grouped(c, s, *workers):
     groups = {
         workers[0].address: "a",
         workers[1].address: "a",
@@ -760,30 +758,30 @@ def key(ws):
     # Assert that job in one worker blocks closure of group
     future = c.submit(slowinc, 1, delay=0.2, workers=workers[0].address)
     while len(s.rprocessing) < 1:
-        yield gen.sleep(0.001)
+        await asyncio.sleep(0.001)
 
     assert set(s.workers_to_close(key=key)) == {workers[2].address, workers[3].address}
 
     del future
 
     while len(s.rprocessing) > 0:
-        yield gen.sleep(0.001)
+        await asyncio.sleep(0.001)
 
     # Assert that *total* byte count in group determines group priority
-    av = yield c.scatter("a" * 100, workers=workers[0].address)
-    bv = yield c.scatter("b" * 75, workers=workers[2].address)
-    bv2 = yield c.scatter("b" * 75, workers=workers[3].address)
+    av = await c.scatter("a" * 100, workers=workers[0].address)
+    bv = await c.scatter("b" * 75, workers=workers[2].address)
+    bv2 = await c.scatter("b" * 75, workers=workers[3].address)
 
     assert set(s.workers_to_close(key=key)) == {workers[0].address, workers[1].address}
 
 
 @gen_cluster(client=True)
-def test_retire_workers_no_suspicious_tasks(c, s, a, b):
+async def test_retire_workers_no_suspicious_tasks(c, s, a, b):
     future = c.submit(
         slowinc, 100, delay=0.5, workers=a.address, allow_other_workers=True
     )
-    yield gen.sleep(0.2)
-    yield s.retire_workers(workers=[a.address])
+    await asyncio.sleep(0.2)
+    await s.retire_workers(workers=[a.address])
 
     assert all(ts.suspicious == 0 for ts in s.tasks.values())
     assert all(tp.suspicious == 0 for tp in s.task_prefixes.values())
@@ -793,48 +791,47 @@ def test_retire_workers_no_suspicious_tasks(c, s, a, b):
 @pytest.mark.skipif(
     sys.platform.startswith("win"), reason="file descriptors not really a thing"
 )
-@pytest.mark.skipif(sys.version_info < (3, 6), reason="intermittent failure")
 @gen_cluster(client=True, nthreads=[], timeout=240)
-def test_file_descriptors(c, s):
-    yield gen.sleep(0.1)
+async def test_file_descriptors(c, s):
+    await asyncio.sleep(0.1)
     psutil = pytest.importorskip("psutil")
     da = pytest.importorskip("dask.array")
     proc = psutil.Process()
     num_fds_1 = proc.num_fds()
 
     N = 20
-    nannies = yield [Nanny(s.address, loop=s.loop) for i in range(N)]
+    nannies = await asyncio.gather(*[Nanny(s.address, loop=s.loop) for _ in range(N)])
 
     while len(s.nthreads) < N:
-        yield gen.sleep(0.1)
+        await asyncio.sleep(0.1)
 
     num_fds_2 = proc.num_fds()
 
-    yield gen.sleep(0.2)
+    await asyncio.sleep(0.2)
 
     num_fds_3 = proc.num_fds()
     assert num_fds_3 <= num_fds_2 + N  # add some heartbeats
 
     x = da.random.random(size=(1000, 1000), chunks=(25, 25))
     x = c.persist(x)
-    yield wait(x)
+    await wait(x)
 
     num_fds_4 = proc.num_fds()
     assert num_fds_4 <= num_fds_2 + 2 * N
 
     y = c.persist(x + x.T)
-    yield wait(y)
+    await wait(y)
 
     num_fds_5 = proc.num_fds()
     assert num_fds_5 < num_fds_4 + N
 
-    yield gen.sleep(1)
+    await asyncio.sleep(1)
 
     num_fds_6 = proc.num_fds()
     assert num_fds_6 < num_fds_5 + N
 
-    yield [n.close() for n in nannies]
-    yield c.close()
+    await asyncio.gather(*[n.close() for n in nannies])
+    await c.close()
 
     assert not s.rpc.open
     for addr, occ in c.rpc.occupied.items():
@@ -844,17 +841,17 @@ def test_file_descriptors(c, s):
 
     start = time()
     while proc.num_fds() > num_fds_1 + N:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 3
 
 
 @pytest.mark.slow
 @nodebug
 @gen_cluster(client=True)
-def test_learn_occupancy(c, s, a, b):
+async def test_learn_occupancy(c, s, a, b):
     futures = c.map(slowinc, range(1000), delay=0.2)
     while sum(len(ts.who_has) for ts in s.tasks.values()) < 10:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
     assert 100 < s.total_occupancy < 1000
     for w in [a, b]:
@@ -864,23 +861,23 @@ def test_learn_occupancy(c, s, a, b):
 @pytest.mark.slow
 @nodebug
 @gen_cluster(client=True)
-def test_learn_occupancy_2(c, s, a, b):
+async def test_learn_occupancy_2(c, s, a, b):
     future = c.map(slowinc, range(1000), delay=0.2)
     while not any(ts.who_has for ts in s.tasks.values()):
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
     assert 100 < s.total_occupancy < 1000
 
 
 @gen_cluster(client=True)
-def test_occupancy_cleardown(c, s, a, b):
+async def test_occupancy_cleardown(c, s, a, b):
     s.validate = False
 
     # Inject excess values in s.occupancy
     s.workers[a.address].occupancy = 2
     s.total_occupancy += 2
     futures = c.map(slowinc, range(100), delay=0.01)
-    yield wait(futures)
+    await wait(futures)
 
     # Verify that occupancy values have been zeroed out
     assert abs(s.total_occupancy) < 0.01
@@ -889,28 +886,28 @@ def test_occupancy_cleardown(c, s, a, b):
 
 @nodebug
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 30)
-def test_balance_many_workers(c, s, *workers):
+async def test_balance_many_workers(c, s, *workers):
     futures = c.map(slowinc, range(20), delay=0.2)
-    yield wait(futures)
+    await wait(futures)
     assert {len(w.has_what) for w in s.workers.values()} == {0, 1}
 
 
 @nodebug
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 30)
-def test_balance_many_workers_2(c, s, *workers):
+async def test_balance_many_workers_2(c, s, *workers):
     s.extensions["stealing"]._pc.callback_time = 100000000
     futures = c.map(slowinc, range(90), delay=0.2)
-    yield wait(futures)
+    await wait(futures)
     assert {len(w.has_what) for w in s.workers.values()} == {3}
 
 
 @gen_cluster(client=True)
-def test_learn_occupancy_multiple_workers(c, s, a, b):
+async def test_learn_occupancy_multiple_workers(c, s, a, b):
     x = c.submit(slowinc, 1, delay=0.2, workers=a.address)
-    yield gen.sleep(0.05)
+    await asyncio.sleep(0.05)
     futures = c.map(slowinc, range(100), delay=0.2)
 
-    yield wait(x)
+    await wait(x)
 
     assert not any(v == 0.5 for w in s.workers.values() for v in w.processing.values())
     s.validate_state()
@@ -934,7 +931,7 @@ async def test_include_communication_in_occupancy(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_worker_arrives_with_processing_data(c, s, a, b):
+async def test_worker_arrives_with_processing_data(c, s, a, b):
     x = delayed(slowinc)(1, delay=0.4)
     y = delayed(slowinc)(x, delay=0.4)
     z = delayed(slowinc)(y, delay=0.4)
@@ -942,17 +939,17 @@ def test_worker_arrives_with_processing_data(c, s, a, b):
     yy, zz = c.persist([y, z])
 
     while not any(w.processing for w in s.workers.values()):
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
     w = Worker(s.address, nthreads=1)
     w.put_key_in_memory(y.key, 3)
 
-    yield w
+    await w
 
     start = time()
 
     while len(s.workers) < 3:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
     assert s.get_task_status(keys={x.key, y.key, z.key}) == {
         x.key: "released",
@@ -960,23 +957,23 @@ def test_worker_arrives_with_processing_data(c, s, a, b):
         z.key: "processing",
     }
 
-    yield w.close()
+    await w.close()
 
 
 @pytest.mark.slow
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)])
-def test_worker_breaks_and_returns(c, s, a):
+async def test_worker_breaks_and_returns(c, s, a):
     future = c.submit(slowinc, 1, delay=0.1)
     for i in range(20):
         future = c.submit(slowinc, future, delay=0.1)
 
-    yield wait(future)
+    await wait(future)
 
-    yield a.batched_stream.comm.close()
+    await a.batched_stream.comm.close()
 
-    yield gen.sleep(0.1)
+    await asyncio.sleep(0.1)
     start = time()
-    yield wait(future, timeout=10)
+    await wait(future, timeout=10)
     end = time()
 
     assert end - start < 2
@@ -986,7 +983,7 @@ def test_worker_breaks_and_returns(c, s, a):
 
 
 @gen_cluster(client=True, nthreads=[])
-def test_no_workers_to_memory(c, s):
+async def test_no_workers_to_memory(c, s):
     x = delayed(slowinc)(1, delay=0.4)
     y = delayed(slowinc)(x, delay=0.4)
     z = delayed(slowinc)(y, delay=0.4)
@@ -994,17 +991,17 @@ def test_no_workers_to_memory(c, s):
     yy, zz = c.persist([y, z])
 
     while not s.tasks:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
     w = Worker(s.address, nthreads=1)
     w.put_key_in_memory(y.key, 3)
 
-    yield w
+    await w
 
     start = time()
 
     while not s.workers:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
     assert s.get_task_status(keys={x.key, y.key, z.key}) == {
         x.key: "released",
@@ -1012,11 +1009,11 @@ def test_no_workers_to_memory(c, s):
         z.key: "processing",
     }
 
-    yield w.close()
+    await w.close()
 
 
 @gen_cluster(client=True)
-def test_no_worker_to_memory_restrictions(c, s, a, b):
+async def test_no_worker_to_memory_restrictions(c, s, a, b):
     x = delayed(slowinc)(1, delay=0.4)
     y = delayed(slowinc)(x, delay=0.4)
     z = delayed(slowinc)(y, delay=0.4)
@@ -1024,16 +1021,16 @@ def test_no_worker_to_memory_restrictions(c, s, a, b):
     yy, zz = c.persist([y, z], workers={(x, y, z): "alice"})
 
     while not s.tasks:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
     w = Worker(s.address, nthreads=1, name="alice")
     w.put_key_in_memory(y.key, 3)
 
-    yield w
+    await w
 
     while len(s.workers) < 3:
-        yield gen.sleep(0.01)
-    yield gen.sleep(0.3)
+        await asyncio.sleep(0.01)
+    await asyncio.sleep(0.3)
 
     assert s.get_task_status(keys={x.key, y.key, z.key}) == {
         x.key: "released",
@@ -1041,7 +1038,7 @@ def test_no_worker_to_memory_restrictions(c, s, a, b):
         z.key: "processing",
     }
 
-    yield w.close()
+    await w.close()
 
 
 def test_run_on_scheduler_sync(loop):
@@ -1058,78 +1055,78 @@ def f(dask_scheduler=None):
 
 
 @gen_cluster(client=True)
-def test_run_on_scheduler(c, s, a, b):
+async def test_run_on_scheduler(c, s, a, b):
     def f(dask_scheduler=None):
         return dask_scheduler.address
 
-    response = yield c._run_on_scheduler(f)
+    response = await c._run_on_scheduler(f)
     assert response == s.address
 
 
 @gen_cluster(client=True)
-def test_close_worker(c, s, a, b):
+async def test_close_worker(c, s, a, b):
     assert len(s.workers) == 2
 
-    yield s.close_worker(worker=a.address)
+    await s.close_worker(worker=a.address)
 
     assert len(s.workers) == 1
     assert a.address not in s.workers
 
-    yield gen.sleep(0.5)
+    await asyncio.sleep(0.5)
 
     assert len(s.workers) == 1
 
 
 @pytest.mark.slow
 @gen_cluster(client=True, Worker=Nanny, timeout=20)
-def test_close_nanny(c, s, a, b):
+async def test_close_nanny(c, s, a, b):
     assert len(s.workers) == 2
 
     assert a.process.is_alive()
     a_worker_address = a.worker_address
     start = time()
-    yield s.close_worker(worker=a_worker_address)
+    await s.close_worker(worker=a_worker_address)
 
     assert len(s.workers) == 1
     assert a_worker_address not in s.workers
 
     start = time()
     while a.is_alive():
-        yield gen.sleep(0.1)
+        await asyncio.sleep(0.1)
         assert time() < start + 5
 
     assert not a.is_alive()
     assert a.pid is None
 
     for i in range(10):
-        yield gen.sleep(0.1)
+        await asyncio.sleep(0.1)
         assert len(s.workers) == 1
         assert not a.is_alive()
         assert a.pid is None
 
     while a.status != "closed":
-        yield gen.sleep(0.05)
+        await asyncio.sleep(0.05)
         assert time() < start + 10
 
 
 @gen_cluster(client=True, timeout=20)
-def test_retire_workers_close(c, s, a, b):
-    yield s.retire_workers(close_workers=True)
+async def test_retire_workers_close(c, s, a, b):
+    await s.retire_workers(close_workers=True)
     assert not s.workers
     while a.status != "closed" and b.status != "closed":
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
 
 @gen_cluster(client=True, timeout=20, Worker=Nanny)
-def test_retire_nannies_close(c, s, a, b):
+async def test_retire_nannies_close(c, s, a, b):
     nannies = [a, b]
-    yield s.retire_workers(close_workers=True, remove=True)
+    await s.retire_workers(close_workers=True, remove=True)
     assert not s.workers
 
     start = time()
 
     while any(n.status != "closed" for n in nannies):
-        yield gen.sleep(0.05)
+        await asyncio.sleep(0.05)
         assert time() < start + 10
 
     assert not any(n.is_alive() for n in nannies)
@@ -1137,27 +1134,27 @@ def test_retire_nannies_close(c, s, a, b):
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 2)])
-def test_fifo_submission(c, s, w):
+async def test_fifo_submission(c, s, w):
     futures = []
     for i in range(20):
         future = c.submit(slowinc, i, delay=0.1, key="inc-%02d" % i, fifo_timeout=0.01)
         futures.append(future)
-        yield gen.sleep(0.02)
-    yield wait(futures[-1])
+        await asyncio.sleep(0.02)
+    await wait(futures[-1])
     assert futures[10].status == "finished"
 
 
 @gen_test()
-def test_scheduler_file():
+async def test_scheduler_file():
     with tmpfile() as fn:
-        s = yield Scheduler(scheduler_file=fn, port=0)
+        s = await Scheduler(scheduler_file=fn, port=0)
         with open(fn) as f:
             data = json.load(f)
         assert data["address"] == s.address
 
-        c = yield Client(scheduler_file=fn, loop=s.loop, asynchronous=True)
-        yield c.close()
-        yield s.close()
+        c = await Client(scheduler_file=fn, loop=s.loop, asynchronous=True)
+        await c.close()
+        await s.close()
 
 
 @pytest.mark.xfail(reason="")
@@ -1168,21 +1165,21 @@ async def test_non_existent_worker(c, s):
             address="127.0.0.1:5738", nthreads=2, nbytes={}, host_info={}
         )
         futures = c.map(inc, range(10))
-        await gen.sleep(0.300)
+        await asyncio.sleep(0.300)
         assert not s.workers
         assert all(ts.state == "no-worker" for ts in s.tasks.values())
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 3)
-def test_correct_bad_time_estimate(c, s, *workers):
+async def test_correct_bad_time_estimate(c, s, *workers):
     future = c.submit(slowinc, 1, delay=0)
-    yield wait(future)
+    await wait(future)
 
     futures = [c.submit(slowinc, future, delay=0.1, pure=False) for i in range(20)]
 
-    yield gen.sleep(0.5)
+    await asyncio.sleep(0.5)
 
-    yield wait(futures)
+    await wait(futures)
 
     assert all(w.data for w in workers), [sorted(w.data) for w in workers]
 
@@ -1210,13 +1207,13 @@ async def test_service_hosts():
 
 
 @gen_cluster(client=True, worker_kwargs={"profile_cycle_interval": 100})
-def test_profile_metadata(c, s, a, b):
+async def test_profile_metadata(c, s, a, b):
     start = time() - 1
     futures = c.map(slowinc, range(10), delay=0.05, workers=a.address)
-    yield wait(futures)
-    yield gen.sleep(0.200)
+    await wait(futures)
+    await asyncio.sleep(0.200)
 
-    meta = yield s.get_profile_metadata(profile_cycle_interval=0.100)
+    meta = await s.get_profile_metadata(profile_cycle_interval=0.100)
     now = time() + 1
     assert meta
     assert all(start < t < now for t, count in meta["counts"])
@@ -1225,12 +1222,12 @@ def test_profile_metadata(c, s, a, b):
 
 
 @gen_cluster(client=True, worker_kwargs={"profile_cycle_interval": 100})
-def test_profile_metadata_keys(c, s, a, b):
+async def test_profile_metadata_keys(c, s, a, b):
     x = c.map(slowinc, range(10), delay=0.05)
     y = c.map(slowdec, range(10), delay=0.05)
-    yield wait(x + y)
+    await wait(x + y)
 
-    meta = yield s.get_profile_metadata(profile_cycle_interval=0.100)
+    meta = await s.get_profile_metadata(profile_cycle_interval=0.100)
     assert set(meta["keys"]) == {"slowinc", "slowdec"}
     assert (
         len(meta["counts"]) - 3 <= len(meta["keys"]["slowinc"]) <= len(meta["counts"])
@@ -1238,7 +1235,7 @@ def test_profile_metadata_keys(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_cancel_fire_and_forget(c, s, a, b):
+async def test_cancel_fire_and_forget(c, s, a, b):
     x = delayed(slowinc)(1, delay=0.05)
     y = delayed(slowinc)(x, delay=0.05)
     z = delayed(slowinc)(y, delay=0.05)
@@ -1246,8 +1243,8 @@ def test_cancel_fire_and_forget(c, s, a, b):
     future = c.compute(w)
     fire_and_forget(future)
 
-    yield gen.sleep(0.05)
-    yield future.cancel(force=True)
+    await asyncio.sleep(0.05)
+    await future.cancel(force=True)
     assert future.status == "cancelled"
     assert not s.tasks
 
@@ -1255,34 +1252,34 @@ def test_cancel_fire_and_forget(c, s, a, b):
 @gen_cluster(
     client=True, Worker=Nanny, clean_kwargs={"processes": False, "threads": False}
 )
-def test_log_tasks_during_restart(c, s, a, b):
+async def test_log_tasks_during_restart(c, s, a, b):
     future = c.submit(sys.exit, 0)
-    yield wait(future)
+    await wait(future)
     assert "exit" in str(s.events)
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 2)
-def test_reschedule(c, s, a, b):
-    yield c.submit(slowinc, -1, delay=0.1)  # learn cost
+async def test_reschedule(c, s, a, b):
+    await c.submit(slowinc, -1, delay=0.1)  # learn cost
     x = c.map(slowinc, range(4), delay=0.1)
 
     # add much more work onto worker a
     futures = c.map(slowinc, range(10, 20), delay=0.1, workers=a.address)
 
     while len(s.tasks) < len(x) + len(futures):
-        yield gen.sleep(0.001)
+        await asyncio.sleep(0.001)
 
     for future in x:
         s.reschedule(key=future.key)
 
     # Worker b gets more of the original tasks
-    yield wait(x)
+    await wait(x)
     assert sum(future.key in b.data for future in x) >= 3
     assert sum(future.key in a.data for future in x) <= 1
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 2)
-def test_reschedule_warns(c, s, a, b):
+async def test_reschedule_warns(c, s, a, b):
     with captured_logger(logging.getLogger("distributed.scheduler")) as sched:
         s.reschedule(key="__this-key-does-not-exist__")
 
@@ -1291,11 +1288,11 @@ def test_reschedule_warns(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_get_task_status(c, s, a, b):
+async def test_get_task_status(c, s, a, b):
     future = c.submit(inc, 1)
-    yield wait(future)
+    await wait(future)
 
-    result = yield a.scheduler.get_task_status(keys=[future.key])
+    result = await a.scheduler.get_task_status(keys=[future.key])
     assert result == {future.key: "memory"}
 
 
@@ -1312,29 +1309,29 @@ def test_deque_handler():
 
 
 @gen_cluster(client=True)
-def test_retries(c, s, a, b):
+async def test_retries(c, s, a, b):
     args = [ZeroDivisionError("one"), ZeroDivisionError("two"), 42]
 
     future = c.submit(varying(args), retries=3)
-    result = yield future
+    result = await future
     assert result == 42
     assert s.tasks[future.key].retries == 1
     assert future.key not in s.exceptions
 
     future = c.submit(varying(args), retries=2, pure=False)
-    result = yield future
+    result = await future
     assert result == 42
     assert s.tasks[future.key].retries == 0
     assert future.key not in s.exceptions
 
     future = c.submit(varying(args), retries=1, pure=False)
     with pytest.raises(ZeroDivisionError) as exc_info:
-        res = yield future
+        await future
     exc_info.match("two")
 
     future = c.submit(varying(args), retries=0, pure=False)
     with pytest.raises(ZeroDivisionError) as exc_info:
-        res = yield future
+        await future
     exc_info.match("one")
 
 
@@ -1350,149 +1347,149 @@ async def test_mising_data_errant_worker(c, s, w1, w2, w3):
 
         y = c.submit(len, x, workers=w3.address)
         while not w3.tasks:
-            await gen.sleep(0.001)
+            await asyncio.sleep(0.001)
         await w1.close()
         await wait(y)
 
 
 @gen_cluster(client=True)
-def test_dont_recompute_if_persisted(c, s, a, b):
+async def test_dont_recompute_if_persisted(c, s, a, b):
     x = delayed(inc)(1, dask_key_name="x")
     y = delayed(inc)(x, dask_key_name="y")
 
     yy = y.persist()
-    yield wait(yy)
+    await wait(yy)
 
     old = list(s.transition_log)
 
     yyy = y.persist()
-    yield wait(yyy)
+    await wait(yyy)
 
-    yield gen.sleep(0.100)
+    await asyncio.sleep(0.100)
     assert list(s.transition_log) == old
 
 
 @gen_cluster(client=True)
-def test_dont_recompute_if_persisted_2(c, s, a, b):
+async def test_dont_recompute_if_persisted_2(c, s, a, b):
     x = delayed(inc)(1, dask_key_name="x")
     y = delayed(inc)(x, dask_key_name="y")
     z = delayed(inc)(y, dask_key_name="z")
 
     yy = y.persist()
-    yield wait(yy)
+    await wait(yy)
 
     old = s.story("x", "y")
 
     zz = z.persist()
-    yield wait(zz)
+    await wait(zz)
 
-    yield gen.sleep(0.100)
+    await asyncio.sleep(0.100)
     assert s.story("x", "y") == old
 
 
 @gen_cluster(client=True)
-def test_dont_recompute_if_persisted_3(c, s, a, b):
+async def test_dont_recompute_if_persisted_3(c, s, a, b):
     x = delayed(inc)(1, dask_key_name="x")
     y = delayed(inc)(2, dask_key_name="y")
     z = delayed(inc)(y, dask_key_name="z")
     w = delayed(operator.add)(x, z, dask_key_name="w")
 
     ww = w.persist()
-    yield wait(ww)
+    await wait(ww)
 
     old = list(s.transition_log)
 
     www = w.persist()
-    yield wait(www)
-    yield gen.sleep(0.100)
+    await wait(www)
+    await asyncio.sleep(0.100)
     assert list(s.transition_log) == old
 
 
 @gen_cluster(client=True)
-def test_dont_recompute_if_persisted_4(c, s, a, b):
+async def test_dont_recompute_if_persisted_4(c, s, a, b):
     x = delayed(inc)(1, dask_key_name="x")
     y = delayed(inc)(x, dask_key_name="y")
     z = delayed(inc)(x, dask_key_name="z")
 
     yy = y.persist()
-    yield wait(yy)
+    await wait(yy)
 
     old = s.story("x")
 
     while s.tasks["x"].state == "memory":
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
     yyy, zzz = dask.persist(y, z)
-    yield wait([yyy, zzz])
+    await wait([yyy, zzz])
 
     new = s.story("x")
     assert len(new) > len(old)
 
 
 @gen_cluster(client=True)
-def test_dont_forget_released_keys(c, s, a, b):
+async def test_dont_forget_released_keys(c, s, a, b):
     x = c.submit(inc, 1, key="x")
     y = c.submit(inc, x, key="y")
     z = c.submit(dec, x, key="z")
     del x
-    yield wait([y, z])
+    await wait([y, z])
     del z
 
     while "z" in s.tasks:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
     assert "x" in s.tasks
 
 
 @gen_cluster(client=True)
-def test_dont_recompute_if_erred(c, s, a, b):
+async def test_dont_recompute_if_erred(c, s, a, b):
     x = delayed(inc)(1, dask_key_name="x")
     y = delayed(div)(x, 0, dask_key_name="y")
 
     yy = y.persist()
-    yield wait(yy)
+    await wait(yy)
 
     old = list(s.transition_log)
 
     yyy = y.persist()
-    yield wait(yyy)
+    await wait(yyy)
 
-    yield gen.sleep(0.100)
+    await asyncio.sleep(0.100)
     assert list(s.transition_log) == old
 
 
 @gen_cluster()
-def test_closing_scheduler_closes_workers(s, a, b):
-    yield s.close()
+async def test_closing_scheduler_closes_workers(s, a, b):
+    await s.close()
 
     start = time()
     while a.status != "closed" or b.status != "closed":
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 2
 
 
 @gen_cluster(
     client=True, nthreads=[("127.0.0.1", 1)], worker_kwargs={"resources": {"A": 1}}
 )
-def test_resources_reset_after_cancelled_task(c, s, w):
+async def test_resources_reset_after_cancelled_task(c, s, w):
     future = c.submit(sleep, 0.2, resources={"A": 1})
 
     while not w.executing:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
-    yield future.cancel()
+    await future.cancel()
 
     while w.executing:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
     assert not s.workers[w.address].used_resources["A"]
     assert w.available_resources == {"A": 1}
 
-    yield c.submit(inc, 1, resources={"A": 1})
+    await c.submit(inc, 1, resources={"A": 1})
 
 
 @gen_cluster(client=True)
-def test_gh2187(c, s, a, b):
+async def test_gh2187(c, s, a, b):
     def foo():
         return "foo"
 
@@ -1509,16 +1506,16 @@ def qux(x):
     w = c.submit(foo, key="w")
     x = c.submit(bar, w, key="x")
     y = c.submit(baz, x, key="y")
-    yield y
+    await y
     z = c.submit(qux, y, key="z")
     del y
-    yield gen.sleep(0.1)
+    await asyncio.sleep(0.1)
     f = c.submit(bar, x, key="y")
-    yield f
+    await f
 
 
 @gen_cluster(client=True)
-def test_collect_versions(c, s, a, b):
+async def test_collect_versions(c, s, a, b):
     cs = s.clients[c.id]
     (w1, w2) = s.workers.values()
     assert cs.versions
@@ -1538,12 +1535,12 @@ async def test_idle_timeout(c, s, a, b):
     with captured_logger("distributed.scheduler") as logs:
         start = time()
         while s.status != "closed":
-            await gen.sleep(0.01)
+            await asyncio.sleep(0.01)
             assert time() < start + 3
 
         start = time()
         while not (a.status == "closed" and b.status == "closed"):
-            await gen.sleep(0.01)
+            await asyncio.sleep(0.01)
             assert time() < start + 1
 
     assert "idle" in logs.getvalue()
@@ -1587,7 +1584,7 @@ async def f(dask_worker):
 
 
 @gen_cluster()
-def test_workerstate_clean(s, a, b):
+async def test_workerstate_clean(s, a, b):
     ws = s.workers[a.address].clean()
     assert ws.address == a.address
     b = pickle.dumps(ws)
@@ -1595,16 +1592,16 @@ def test_workerstate_clean(s, a, b):
 
 
 @gen_cluster(client=True)
-def test_result_type(c, s, a, b):
+async def test_result_type(c, s, a, b):
     x = c.submit(lambda: 1)
-    yield x
+    await x
 
     assert "int" in s.tasks[x.key].type
 
 
 @gen_cluster()
-def test_close_workers(s, a, b):
-    yield s.close(close_workers=True)
+async def test_close_workers(s, a, b):
+    await s.close(close_workers=True)
     assert a.status == "closed"
     assert b.status == "closed"
 
@@ -1613,22 +1610,22 @@ def test_close_workers(s, a, b):
     not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
 )
 @gen_test()
-def test_host_address():
-    s = yield Scheduler(host="127.0.0.2", port=0)
+async def test_host_address():
+    s = await Scheduler(host="127.0.0.2", port=0)
     assert "127.0.0.2" in s.address
-    yield s.close()
+    await s.close()
 
 
 @gen_test()
-def test_dashboard_address():
+async def test_dashboard_address():
     pytest.importorskip("bokeh")
-    s = yield Scheduler(dashboard_address="127.0.0.1:8901", port=0)
+    s = await Scheduler(dashboard_address="127.0.0.1:8901", port=0)
     assert s.services["dashboard"].port == 8901
-    yield s.close()
+    await s.close()
 
-    s = yield Scheduler(dashboard_address="127.0.0.1", port=0)
+    s = await Scheduler(dashboard_address="127.0.0.1", port=0)
     assert s.services["dashboard"].port
-    yield s.close()
+    await s.close()
 
 
 @gen_cluster(client=True)
@@ -1644,16 +1641,16 @@ async def test_adaptive_target(c, s, a, b):
         # Long task
         x = c.submit(slowinc, 1, delay=0.5)
         while x.key not in s.tasks:
-            await gen.sleep(0.01)
+            await asyncio.sleep(0.01)
         assert s.adaptive_target(target_duration=".1s") == 1  # still one
 
         L = c.map(slowinc, range(100), delay=0.5)
         while len(s.tasks) < 100:
-            await gen.sleep(0.01)
+            await asyncio.sleep(0.01)
         assert 10 < s.adaptive_target(target_duration=".1s") <= 100
         del x, L
         while s.tasks:
-            await gen.sleep(0.01)
+            await asyncio.sleep(0.01)
         assert s.adaptive_target(target_duration=".1s") == 0
 
 
diff --git a/distributed/tests/test_security.py b/distributed/tests/test_security.py
index 8665ebead33..7bb2fd753c0 100644
--- a/distributed/tests/test_security.py
+++ b/distributed/tests/test_security.py
@@ -1,5 +1,4 @@
 from contextlib import contextmanager
-import sys
 
 try:
     import ssl
@@ -149,15 +148,15 @@ def test_tls_config_for_role():
         sec.get_tls_config_for_role("supervisor")
 
 
+def assert_many_ciphers(ctx):
+    assert len(ctx.get_ciphers()) > 2  # Most likely
+
+
 def test_connection_args():
     def basic_checks(ctx):
         assert ctx.verify_mode == ssl.CERT_REQUIRED
         assert ctx.check_hostname is False
 
-    def many_ciphers(ctx):
-        if sys.version_info >= (3, 6):
-            assert len(ctx.get_ciphers()) > 2  # Most likely
-
     c = {
         "distributed.comm.tls.ca-file": ca_file,
         "distributed.comm.tls.scheduler.key": key1,
@@ -171,12 +170,12 @@ def many_ciphers(ctx):
     assert not d["require_encryption"]
     ctx = d["ssl_context"]
     basic_checks(ctx)
-    many_ciphers(ctx)
+    assert_many_ciphers(ctx)
 
     d = sec.get_connection_args("worker")
     ctx = d["ssl_context"]
     basic_checks(ctx)
-    many_ciphers(ctx)
+    assert_many_ciphers(ctx)
 
     # No cert defined => no TLS
     d = sec.get_connection_args("client")
@@ -193,13 +192,12 @@ def many_ciphers(ctx):
     assert d["require_encryption"]
     ctx = d["ssl_context"]
     basic_checks(ctx)
-    if sys.version_info >= (3, 6):
-        supported_ciphers = ctx.get_ciphers()
-        tls_12_ciphers = [c for c in supported_ciphers if "TLSv1.2" in c["description"]]
-        assert len(tls_12_ciphers) == 1
-        tls_13_ciphers = [c for c in supported_ciphers if "TLSv1.3" in c["description"]]
-        if len(tls_13_ciphers):
-            assert len(tls_13_ciphers) == 3
+
+    supported_ciphers = ctx.get_ciphers()
+    tls_12_ciphers = [c for c in supported_ciphers if "TLSv1.2" in c["description"]]
+    assert len(tls_12_ciphers) == 1
+    tls_13_ciphers = [c for c in supported_ciphers if "TLSv1.3" in c["description"]]
+    assert len(tls_13_ciphers) in (0, 3)
 
 
 def test_listen_args():
@@ -207,10 +205,6 @@ def basic_checks(ctx):
         assert ctx.verify_mode == ssl.CERT_REQUIRED
         assert ctx.check_hostname is False
 
-    def many_ciphers(ctx):
-        if sys.version_info >= (3, 6):
-            assert len(ctx.get_ciphers()) > 2  # Most likely
-
     c = {
         "distributed.comm.tls.ca-file": ca_file,
         "distributed.comm.tls.scheduler.key": key1,
@@ -224,12 +218,12 @@ def many_ciphers(ctx):
     assert not d["require_encryption"]
     ctx = d["ssl_context"]
     basic_checks(ctx)
-    many_ciphers(ctx)
+    assert_many_ciphers(ctx)
 
     d = sec.get_listen_args("worker")
     ctx = d["ssl_context"]
     basic_checks(ctx)
-    many_ciphers(ctx)
+    assert_many_ciphers(ctx)
 
     # No cert defined => no TLS
     d = sec.get_listen_args("client")
@@ -246,13 +240,12 @@ def many_ciphers(ctx):
     assert d["require_encryption"]
     ctx = d["ssl_context"]
     basic_checks(ctx)
-    if sys.version_info >= (3, 6):
-        supported_ciphers = ctx.get_ciphers()
-        tls_12_ciphers = [c for c in supported_ciphers if "TLSv1.2" in c["description"]]
-        assert len(tls_12_ciphers) == 1
-        tls_13_ciphers = [c for c in supported_ciphers if "TLSv1.3" in c["description"]]
-        if len(tls_13_ciphers):
-            assert len(tls_13_ciphers) == 3
+
+    supported_ciphers = ctx.get_ciphers()
+    tls_12_ciphers = [c for c in supported_ciphers if "TLSv1.2" in c["description"]]
+    assert len(tls_12_ciphers) == 1
+    tls_13_ciphers = [c for c in supported_ciphers if "TLSv1.3" in c["description"]]
+    assert len(tls_13_ciphers) in (0, 3)
 
 
 @pytest.mark.asyncio
diff --git a/distributed/tests/test_semaphore.py b/distributed/tests/test_semaphore.py
index 7a36431042f..fc0a6172a85 100644
--- a/distributed/tests/test_semaphore.py
+++ b/distributed/tests/test_semaphore.py
@@ -327,17 +327,15 @@ async def test_oversubscribing_leases(c, s, a, b):
     accept new leases as long as the semaphore is oversubscribed.
 
     Oversubscription may occur if tasks hold the GIL for a longer time than the
-    lease-timeout is configured causing the lease refreshs to go stale and
-    timeout.
+    lease-timeout is configured causing the lease refresh to go stale and timeout.
 
     We cannot protect ourselves entirely from this but we can ensure that while
     a task with a timed out lease is still running, we block further
     acquisitions until we return to normal.
 
     An example would be a task which continuously locks the GIL for a longer
-    time than the lease timeout but this continous lock only makes up a
+    time than the lease timeout but this continuous lock only makes up a
     fraction of the tasks runtime.
-
     """
     # GH3705
 
diff --git a/distributed/tests/test_steal.py b/distributed/tests/test_steal.py
index 1c9fe22e2e8..fb5c96e14e6 100644
--- a/distributed/tests/test_steal.py
+++ b/distributed/tests/test_steal.py
@@ -1,3 +1,4 @@
+import asyncio
 import itertools
 import random
 import sys
@@ -23,7 +24,6 @@
     slowinc,
 )
 from tlz import concat, sliding_window
-from tornado import gen
 
 # Most tests here are timing-dependent
 setup_module = nodebug_setup_module
@@ -34,70 +34,70 @@
     not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
 )
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 2), ("127.0.0.2", 2)], timeout=20)
-def test_work_stealing(c, s, a, b):
-    [x] = yield c._scatter([1], workers=a.address)
+async def test_work_stealing(c, s, a, b):
+    [x] = await c._scatter([1], workers=a.address)
     futures = c.map(slowadd, range(50), [x] * 50)
-    yield wait(futures)
+    await wait(futures)
     assert len(a.data) > 10
     assert len(b.data) > 10
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 2)
-def test_dont_steal_expensive_data_fast_computation(c, s, a, b):
+async def test_dont_steal_expensive_data_fast_computation(c, s, a, b):
     np = pytest.importorskip("numpy")
     x = c.submit(np.arange, 1000000, workers=a.address)
-    yield wait([x])
+    await wait([x])
     future = c.submit(np.sum, [1], workers=a.address)  # learn that sum is fast
-    yield wait([future])
+    await wait([future])
 
     cheap = [
         c.submit(np.sum, x, pure=False, workers=a.address, allow_other_workers=True)
         for i in range(10)
     ]
-    yield wait(cheap)
+    await wait(cheap)
     assert len(s.who_has[x.key]) == 1
     assert len(b.data) == 0
     assert len(a.data) == 12
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 2)
-def test_steal_cheap_data_slow_computation(c, s, a, b):
+async def test_steal_cheap_data_slow_computation(c, s, a, b):
     x = c.submit(slowinc, 100, delay=0.1)  # learn that slowinc is slow
-    yield wait(x)
+    await wait(x)
 
     futures = c.map(
         slowinc, range(10), delay=0.1, workers=a.address, allow_other_workers=True
     )
-    yield wait(futures)
+    await wait(futures)
     assert abs(len(a.data) - len(b.data)) <= 5
 
 
 @pytest.mark.avoid_travis
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 2)
-def test_steal_expensive_data_slow_computation(c, s, a, b):
+async def test_steal_expensive_data_slow_computation(c, s, a, b):
     np = pytest.importorskip("numpy")
 
     x = c.submit(slowinc, 100, delay=0.2, workers=a.address)
-    yield wait(x)  # learn that slowinc is slow
+    await wait(x)  # learn that slowinc is slow
 
     x = c.submit(np.arange, 1000000, workers=a.address)  # put expensive data
-    yield wait(x)
+    await wait(x)
 
     slow = [c.submit(slowinc, x, delay=0.1, pure=False) for i in range(20)]
-    yield wait(slow)
+    await wait(slow)
     assert len(s.who_has[x.key]) > 1
 
     assert b.data  # not empty
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 10)
-def test_worksteal_many_thieves(c, s, *workers):
+async def test_worksteal_many_thieves(c, s, *workers):
     x = c.submit(slowinc, -1, delay=0.1)
-    yield x
+    await x
 
     xs = c.map(slowinc, [x] * 100, pure=False, delay=0.1)
 
-    yield wait(xs)
+    await wait(xs)
 
     for w, keys in s.has_what.items():
         assert 2 < len(keys) < 30
@@ -107,31 +107,30 @@ def test_worksteal_many_thieves(c, s, *workers):
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 2)
-def test_dont_steal_unknown_functions(c, s, a, b):
-    futures = c.map(inc, [1, 2], workers=a.address, allow_other_workers=True)
-    yield wait(futures)
-    assert len(a.data) == 2, [len(a.data), len(b.data)]
-    assert len(b.data) == 0, [len(a.data), len(b.data)]
+async def test_dont_steal_unknown_functions(c, s, a, b):
+    futures = c.map(inc, range(100), workers=a.address, allow_other_workers=True)
+    await wait(futures)
+    assert len(a.data) >= 95, [len(a.data), len(b.data)]
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 2)
-def test_eventually_steal_unknown_functions(c, s, a, b):
+async def test_eventually_steal_unknown_functions(c, s, a, b):
     futures = c.map(
         slowinc, range(10), delay=0.1, workers=a.address, allow_other_workers=True
     )
-    yield wait(futures)
-    assert len(a.data) >= 3
-    assert len(b.data) >= 3
+    await wait(futures)
+    assert len(a.data) >= 3, [len(a.data), len(b.data)]
+    assert len(b.data) >= 3, [len(a.data), len(b.data)]
 
 
 @pytest.mark.skip(reason="")
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 3)
-def test_steal_related_tasks(e, s, a, b, c):
+async def test_steal_related_tasks(e, s, a, b, c):
     futures = e.map(
         slowinc, range(20), delay=0.05, workers=a.address, allow_other_workers=True
     )
 
-    yield wait(futures)
+    await wait(futures)
 
     nearby = 0
     for f1, f2 in sliding_window(2, futures):
@@ -198,17 +197,17 @@ def fast_blacklisted(x, y=None):
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)], timeout=20)
-def test_new_worker_steals(c, s, a):
-    yield wait(c.submit(slowinc, 1, delay=0.01))
+async def test_new_worker_steals(c, s, a):
+    await wait(c.submit(slowinc, 1, delay=0.01))
 
     futures = c.map(slowinc, range(100), delay=0.05)
     total = c.submit(sum, futures)
     while len(a.task_state) < 10:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
-    b = yield Worker(s.address, loop=s.loop, nthreads=1, memory_limit=MEMORY_LIMIT)
+    b = await Worker(s.address, loop=s.loop, nthreads=1, memory_limit=MEMORY_LIMIT)
 
-    result = yield total
+    result = await total
     assert result == sum(map(inc, range(100)))
 
     for w in [a, b]:
@@ -216,44 +215,44 @@ def test_new_worker_steals(c, s, a):
 
     assert b.data
 
-    yield b.close()
+    await b.close()
 
 
 @gen_cluster(client=True, timeout=20)
-def test_work_steal_no_kwargs(c, s, a, b):
-    yield wait(c.submit(slowinc, 1, delay=0.05))
+async def test_work_steal_no_kwargs(c, s, a, b):
+    await wait(c.submit(slowinc, 1, delay=0.05))
 
     futures = c.map(
         slowinc, range(100), workers=a.address, allow_other_workers=True, delay=0.05
     )
 
-    yield wait(futures)
+    await wait(futures)
 
     assert 20 < len(a.data) < 80
     assert 20 < len(b.data) < 80
 
     total = c.submit(sum, futures)
-    result = yield total
+    result = await total
 
     assert result == sum(map(inc, range(100)))
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1), ("127.0.0.1", 2)])
-def test_dont_steal_worker_restrictions(c, s, a, b):
+async def test_dont_steal_worker_restrictions(c, s, a, b):
     future = c.submit(slowinc, 1, delay=0.10, workers=a.address)
-    yield future
+    await future
 
     futures = c.map(slowinc, range(100), delay=0.1, workers=a.address)
 
     while len(a.task_state) + len(b.task_state) < 100:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
     assert len(a.task_state) == 100
     assert len(b.task_state) == 0
 
     result = s.extensions["stealing"].balance()
 
-    yield gen.sleep(0.1)
+    await asyncio.sleep(0.1)
 
     assert len(a.task_state) == 100
     assert len(b.task_state) == 0
@@ -262,15 +261,15 @@ def test_dont_steal_worker_restrictions(c, s, a, b):
 @gen_cluster(
     client=True, nthreads=[("127.0.0.1", 1), ("127.0.0.1", 2), ("127.0.0.1", 2)]
 )
-def test_steal_worker_restrictions(c, s, wa, wb, wc):
+async def test_steal_worker_restrictions(c, s, wa, wb, wc):
     future = c.submit(slowinc, 1, delay=0.1, workers={wa.address, wb.address})
-    yield future
+    await future
 
     ntasks = 100
     futures = c.map(slowinc, range(ntasks), delay=0.1, workers={wa.address, wb.address})
 
     while sum(len(w.task_state) for w in [wa, wb, wc]) < ntasks:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
     assert 0 < len(wa.task_state) < ntasks
     assert 0 < len(wb.task_state) < ntasks
@@ -278,7 +277,7 @@ def test_steal_worker_restrictions(c, s, wa, wb, wc):
 
     s.extensions["stealing"].balance()
 
-    yield gen.sleep(0.1)
+    await asyncio.sleep(0.1)
 
     assert 0 < len(wa.task_state) < ntasks
     assert 0 < len(wb.task_state) < ntasks
@@ -289,19 +288,19 @@ def test_steal_worker_restrictions(c, s, wa, wb, wc):
     not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
 )
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1), ("127.0.0.2", 1)])
-def test_dont_steal_host_restrictions(c, s, a, b):
+async def test_dont_steal_host_restrictions(c, s, a, b):
     future = c.submit(slowinc, 1, delay=0.10, workers=a.address)
-    yield future
+    await future
 
     futures = c.map(slowinc, range(100), delay=0.1, workers="127.0.0.1")
     while len(a.task_state) + len(b.task_state) < 100:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
     assert len(a.task_state) == 100
     assert len(b.task_state) == 0
 
     result = s.extensions["stealing"].balance()
 
-    yield gen.sleep(0.1)
+    await asyncio.sleep(0.1)
     assert len(a.task_state) == 100
     assert len(b.task_state) == 0
 
@@ -310,25 +309,25 @@ def test_dont_steal_host_restrictions(c, s, a, b):
     not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
 )
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1), ("127.0.0.2", 2)])
-def test_steal_host_restrictions(c, s, wa, wb):
+async def test_steal_host_restrictions(c, s, wa, wb):
     future = c.submit(slowinc, 1, delay=0.10, workers=wa.address)
-    yield future
+    await future
 
     ntasks = 100
     futures = c.map(slowinc, range(ntasks), delay=0.1, workers="127.0.0.1")
     while len(wa.task_state) < ntasks:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
     assert len(wa.task_state) == ntasks
     assert len(wb.task_state) == 0
 
-    wc = yield Worker(s.address, nthreads=1)
+    wc = await Worker(s.address, nthreads=1)
 
     start = time()
     while not wc.task_state or len(wa.task_state) == ntasks:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 3
 
-    yield gen.sleep(0.1)
+    await asyncio.sleep(0.1)
     assert 0 < len(wa.task_state) < ntasks
     assert len(wb.task_state) == 0
     assert 0 < len(wc.task_state) < ntasks
@@ -337,19 +336,19 @@ def test_steal_host_restrictions(c, s, wa, wb):
 @gen_cluster(
     client=True, nthreads=[("127.0.0.1", 1, {"resources": {"A": 2}}), ("127.0.0.1", 1)]
 )
-def test_dont_steal_resource_restrictions(c, s, a, b):
+async def test_dont_steal_resource_restrictions(c, s, a, b):
     future = c.submit(slowinc, 1, delay=0.10, workers=a.address)
-    yield future
+    await future
 
     futures = c.map(slowinc, range(100), delay=0.1, resources={"A": 1})
     while len(a.task_state) + len(b.task_state) < 100:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
     assert len(a.task_state) == 100
     assert len(b.task_state) == 0
 
     result = s.extensions["stealing"].balance()
 
-    yield gen.sleep(0.1)
+    await asyncio.sleep(0.1)
     assert len(a.task_state) == 100
     assert len(b.task_state) == 0
 
@@ -357,30 +356,30 @@ def test_dont_steal_resource_restrictions(c, s, a, b):
 @gen_cluster(
     client=True, nthreads=[("127.0.0.1", 1, {"resources": {"A": 2}})], timeout=3
 )
-def test_steal_resource_restrictions(c, s, a):
+async def test_steal_resource_restrictions(c, s, a):
     future = c.submit(slowinc, 1, delay=0.10, workers=a.address)
-    yield future
+    await future
 
     futures = c.map(slowinc, range(100), delay=0.2, resources={"A": 1})
     while len(a.task_state) < 101:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
     assert len(a.task_state) == 101
 
-    b = yield Worker(s.address, loop=s.loop, nthreads=1, resources={"A": 4})
+    b = await Worker(s.address, loop=s.loop, nthreads=1, resources={"A": 4})
 
     start = time()
     while not b.task_state or len(a.task_state) == 101:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 3
 
     assert len(b.task_state) > 0
     assert len(a.task_state) < 101
 
-    yield b.close()
+    await b.close()
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 5, timeout=20)
-def test_balance_without_dependencies(c, s, *workers):
+async def test_balance_without_dependencies(c, s, *workers):
     s.extensions["stealing"]._pc.callback_time = 20
 
     def slow(x):
@@ -389,19 +388,19 @@ def slow(x):
         return y
 
     futures = c.map(slow, range(100))
-    yield wait(futures)
+    await wait(futures)
 
     durations = [sum(w.data.values()) for w in workers]
     assert max(durations) / min(durations) < 3
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 4)] * 2)
-def test_dont_steal_executing_tasks(c, s, a, b):
+async def test_dont_steal_executing_tasks(c, s, a, b):
     futures = c.map(
         slowinc, range(4), delay=0.1, workers=a.address, allow_other_workers=True
     )
 
-    yield wait(futures)
+    await wait(futures)
     assert len(a.data) == 4
     assert len(b.data) == 0
 
@@ -411,14 +410,14 @@ def test_dont_steal_executing_tasks(c, s, a, b):
     nthreads=[("127.0.0.1", 1)] * 10,
     config={"distributed.scheduler.default-task-durations": {"slowidentity": 0.2}},
 )
-def test_dont_steal_few_saturated_tasks_many_workers(c, s, a, *rest):
+async def test_dont_steal_few_saturated_tasks_many_workers(c, s, a, *rest):
     s.extensions["stealing"]._pc.callback_time = 20
     x = c.submit(mul, b"0", 100000000, workers=a.address)  # 100 MB
-    yield wait(x)
+    await wait(x)
 
     futures = [c.submit(slowidentity, x, pure=False, delay=0.2) for i in range(2)]
 
-    yield wait(futures)
+    await wait(futures)
 
     assert len(a.data) == 3
     assert not any(w.task_state for w in rest)
@@ -430,16 +429,16 @@ def test_dont_steal_few_saturated_tasks_many_workers(c, s, a, *rest):
     worker_kwargs={"memory_limit": MEMORY_LIMIT},
     config={"distributed.scheduler.default-task-durations": {"slowidentity": 0.2}},
 )
-def test_steal_when_more_tasks(c, s, a, *rest):
+async def test_steal_when_more_tasks(c, s, a, *rest):
     s.extensions["stealing"]._pc.callback_time = 20
     x = c.submit(mul, b"0", 50000000, workers=a.address)  # 50 MB
-    yield wait(x)
+    await wait(x)
 
     futures = [c.submit(slowidentity, x, pure=False, delay=0.2) for i in range(20)]
 
     start = time()
     while not any(w.task_state for w in rest):
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 1
 
 
@@ -453,20 +452,20 @@ def test_steal_when_more_tasks(c, s, a, *rest):
         }
     },
 )
-def test_steal_more_attractive_tasks(c, s, a, *rest):
+async def test_steal_more_attractive_tasks(c, s, a, *rest):
     def slow2(x):
         sleep(1)
         return x
 
     s.extensions["stealing"]._pc.callback_time = 20
     x = c.submit(mul, b"0", 100000000, workers=a.address)  # 100 MB
-    yield wait(x)
+    await wait(x)
 
     futures = [c.submit(slowidentity, x, pure=False, delay=0.2) for i in range(10)]
     future = c.submit(slow2, x, priority=-1)
 
     while not any(w.task_state for w in rest):
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
     # good future moves first
     assert any(future.key in w.task_state for w in rest)
@@ -476,7 +475,7 @@ def func(x):
     sleep(1)
 
 
-def assert_balanced(inp, expected, c, s, *workers):
+async def assert_balanced(inp, expected, c, s, *workers):
     steal = s.extensions["stealing"]
     steal._pc.stop()
 
@@ -488,7 +487,7 @@ def assert_balanced(inp, expected, c, s, *workers):
     for w, ts in zip(workers, inp):
         for t in sorted(ts, reverse=True):
             if t:
-                [dat] = yield c.scatter([next(data_seq)], workers=w.address)
+                [dat] = await c.scatter([next(data_seq)], workers=w.address)
                 ts = s.tasks[dat.key]
                 # Ensure scheduler state stays consistent
                 old_nbytes = ts.nbytes
@@ -510,13 +509,13 @@ def assert_balanced(inp, expected, c, s, *workers):
             futures.append(f)
 
     while len(s.rprocessing) < len(futures):
-        yield gen.sleep(0.001)
+        await asyncio.sleep(0.001)
 
     for i in range(10):
         steal.balance()
 
         while steal.in_flight:
-            yield gen.sleep(0.001)
+            await asyncio.sleep(0.001)
 
         result = [
             sorted([int(key_split(k)) for k in s.processing[w.address]], reverse=True)
@@ -569,7 +568,9 @@ def assert_balanced(inp, expected, c, s, *workers):
     ],
 )
 def test_balance(inp, expected):
-    test = lambda *args, **kwargs: assert_balanced(inp, expected, *args, **kwargs)
+    async def test(*args, **kwargs):
+        await assert_balanced(inp, expected, *args, **kwargs)
+
     test = gen_cluster(
         client=True,
         nthreads=[("127.0.0.1", 1)] * len(inp),
@@ -583,18 +584,18 @@ def test_balance(inp, expected):
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 2, Worker=Nanny, timeout=20)
-def test_restart(c, s, a, b):
+async def test_restart(c, s, a, b):
     futures = c.map(
         slowinc, range(100), delay=0.1, workers=a.address, allow_other_workers=True
     )
     while not s.processing[b.worker_address]:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
     steal = s.extensions["stealing"]
     assert any(st for st in steal.stealable_all)
     assert any(x for L in steal.stealable.values() for x in L)
 
-    yield c.restart(timeout=10)
+    await c.restart(timeout=10)
 
     assert not any(x for x in steal.stealable_all)
     assert not any(x for L in steal.stealable.values() for x in L)
@@ -604,7 +605,7 @@ def test_restart(c, s, a, b):
     client=True,
     config={"distributed.scheduler.default-task-durations": {"slowadd": 0.001}},
 )
-def test_steal_communication_heavy_tasks(c, s, a, b):
+async def test_steal_communication_heavy_tasks(c, s, a, b):
     steal = s.extensions["stealing"]
     x = c.submit(mul, b"0", int(s.bandwidth), workers=a.address)
     y = c.submit(mul, b"1", int(s.bandwidth), workers=b.address)
@@ -623,29 +624,29 @@ def test_steal_communication_heavy_tasks(c, s, a, b):
     ]
 
     while not any(f.key in s.rprocessing for f in futures):
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
     steal.balance()
     while steal.in_flight:
-        yield gen.sleep(0.001)
+        await asyncio.sleep(0.001)
 
     assert s.processing[b.address]
 
 
 @gen_cluster(client=True)
-def test_steal_twice(c, s, a, b):
+async def test_steal_twice(c, s, a, b):
     x = c.submit(inc, 1, workers=a.address)
-    yield wait(x)
+    await wait(x)
 
     futures = [c.submit(slowadd, x, i, delay=0.2) for i in range(100)]
 
     while len(s.tasks) < 100:  # tasks are all allocated
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
     # Army of new workers arrives to help
-    workers = yield [Worker(s.address, loop=s.loop) for _ in range(20)]
+    workers = await asyncio.gather(*[Worker(s.address, loop=s.loop) for _ in range(20)])
 
-    yield wait(futures)
+    await wait(futures)
 
     has_what = dict(s.has_what)  # take snapshot
     empty_workers = [w for w, keys in has_what.items() if not len(keys)]
@@ -656,42 +657,42 @@ def test_steal_twice(c, s, a, b):
         )
     assert max(map(len, has_what.values())) < 30
 
-    yield c._close()
-    yield [w.close() for w in workers]
+    await c._close()
+    await asyncio.gather(*[w.close() for w in workers])
 
 
 @gen_cluster(client=True)
-def test_dont_steal_executing_tasks(c, s, a, b):
+async def test_dont_steal_executing_tasks(c, s, a, b):
     steal = s.extensions["stealing"]
 
     future = c.submit(slowinc, 1, delay=0.5, workers=a.address)
     while not a.executing:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
     steal.move_task_request(
         s.tasks[future.key], s.workers[a.address], s.workers[b.address]
     )
-    yield gen.sleep(0.1)
+    await asyncio.sleep(0.1)
     assert future.key in a.executing
     assert not b.executing
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 2)
-def test_dont_steal_long_running_tasks(c, s, a, b):
+async def test_dont_steal_long_running_tasks(c, s, a, b):
     def long(delay):
         with worker_client() as c:
             sleep(delay)
 
-    yield c.submit(long, 0.1)  # learn duration
-    yield c.submit(inc, 1)  # learn duration
+    await c.submit(long, 0.1)  # learn duration
+    await c.submit(inc, 1)  # learn duration
 
     long_tasks = c.map(long, [0.5, 0.6], workers=a.address, allow_other_workers=True)
     while sum(map(len, s.processing.values())) < 2:  # let them start
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
     start = time()
     while any(t.key in s.extensions["stealing"].key_stealable for t in long_tasks):
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 1
 
     na = len(a.executing)
@@ -699,9 +700,9 @@ def long(delay):
 
     incs = c.map(inc, range(100), workers=a.address, allow_other_workers=True)
 
-    yield gen.sleep(0.2)
+    await asyncio.sleep(0.2)
 
-    yield wait(long_tasks)
+    await wait(long_tasks)
 
     for t in long_tasks:
         assert (
@@ -716,19 +717,19 @@ def long(delay):
     strict=False,
 )
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 5)] * 2)
-def test_cleanup_repeated_tasks(c, s, a, b):
+async def test_cleanup_repeated_tasks(c, s, a, b):
     class Foo:
         pass
 
     s.extensions["stealing"]._pc.callback_time = 20
-    yield c.submit(slowidentity, -1, delay=0.1)
+    await c.submit(slowidentity, -1, delay=0.1)
     objects = [c.submit(Foo, pure=False, workers=a.address) for _ in range(50)]
 
     x = c.map(
         slowidentity, objects, workers=a.address, allow_other_workers=True, delay=0.05
     )
     del objects
-    yield wait(x)
+    await wait(x)
     assert a.data and b.data
     assert len(a.data) + len(b.data) > 10
     ws = weakref.WeakSet()
@@ -738,7 +739,7 @@ class Foo:
 
     start = time()
     while a.data or b.data:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 1
 
     assert not s.who_has
@@ -748,7 +749,7 @@ class Foo:
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 2)
-def test_lose_task(c, s, a, b):
+async def test_lose_task(c, s, a, b):
     with captured_logger("distributed.stealing") as log:
         s.periodic_callbacks["stealing"].interval = 1
         for i in range(100):
@@ -760,7 +761,7 @@ def test_lose_task(c, s, a, b):
                 workers=a.address,
                 allow_other_workers=True,
             )
-            yield gen.sleep(0.01)
+            await asyncio.sleep(0.01)
             del futures
 
     out = log.getvalue()
@@ -768,7 +769,7 @@ def test_lose_task(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_worker_stealing_interval(c, s, a, b):
+async def test_worker_stealing_interval(c, s, a, b):
     from distributed.scheduler import WorkStealing
 
     ws = WorkStealing(s)
diff --git a/distributed/tests/test_stress.py b/distributed/tests/test_stress.py
index d5e1e62c574..707b93c03cf 100644
--- a/distributed/tests/test_stress.py
+++ b/distributed/tests/test_stress.py
@@ -1,8 +1,8 @@
-from operator import add
+import asyncio
 import random
 import sys
+from operator import add
 from time import sleep
-import asyncio
 
 from dask import delayed
 import pytest
@@ -27,7 +27,6 @@
     nodebug_teardown_module,
 )
 from distributed.client import wait
-from tornado import gen
 
 
 # All tests here are slow in some way
@@ -36,14 +35,14 @@
 
 
 @gen_cluster(client=True)
-def test_stress_1(c, s, a, b):
+async def test_stress_1(c, s, a, b):
     n = 2 ** 6
 
     seq = c.map(inc, range(n))
     while len(seq) > 1:
-        yield gen.sleep(0.1)
+        await asyncio.sleep(0.1)
         seq = [c.submit(add, seq[i], seq[i + 1]) for i in range(0, len(seq), 2)]
-    result = yield seq[0]
+    result = await seq[0]
     assert result == sum(map(inc, range(n)))
 
 
@@ -62,18 +61,18 @@ def test_stress_gc(loop, func, n):
     sys.platform.startswith("win"), reason="test can leave dangling RPC objects"
 )
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 8, timeout=None)
-def test_cancel_stress(c, s, *workers):
+async def test_cancel_stress(c, s, *workers):
     da = pytest.importorskip("dask.array")
     x = da.random.random((50, 50), chunks=(2, 2))
     x = c.persist(x)
-    yield wait([x])
+    await wait([x])
     y = (x.sum(axis=0) + x.sum(axis=1) + 1).std()
     n_todo = len(y.dask) - len(x.dask)
     for i in range(5):
         f = c.compute(y)
         while len(s.waiting) > (random.random() + 1) * 0.5 * n_todo:
-            yield gen.sleep(0.01)
-        yield c._cancel(f)
+            await asyncio.sleep(0.01)
+        await c._cancel(f)
 
 
 def test_cancel_stress_sync(loop):
@@ -91,7 +90,7 @@ def test_cancel_stress_sync(loop):
 
 
 @gen_cluster(nthreads=[], client=True, timeout=None)
-def test_stress_creation_and_deletion(c, s):
+async def test_stress_creation_and_deletion(c, s):
     # Assertions are handled by the validate mechanism in the scheduler
     s.allowed_failures = 100000
     da = pytest.importorskip("dask.array")
@@ -101,28 +100,27 @@ def test_stress_creation_and_deletion(c, s):
 
     z = c.persist(y)
 
-    @gen.coroutine
-    def create_and_destroy_worker(delay):
+    async def create_and_destroy_worker(delay):
         start = time()
         while time() < start + 5:
-            n = yield Nanny(s.address, nthreads=2, loop=s.loop)
-            yield gen.sleep(delay)
-            yield n.close()
+            n = await Nanny(s.address, nthreads=2, loop=s.loop)
+            await asyncio.sleep(delay)
+            await n.close()
             print("Killed nanny")
 
-    yield asyncio.wait_for(
+    await asyncio.wait_for(
         All([create_and_destroy_worker(0.1 * i) for i in range(20)]), 60
     )
 
 
 @gen_cluster(nthreads=[("127.0.0.1", 1)] * 10, client=True, timeout=60)
-def test_stress_scatter_death(c, s, *workers):
+async def test_stress_scatter_death(c, s, *workers):
     import random
 
     s.allowed_failures = 1000
     np = pytest.importorskip("numpy")
-    L = yield c.scatter([np.random.random(10000) for i in range(len(workers))])
-    yield c.replicate(L, n=2)
+    L = await c.scatter([np.random.random(10000) for i in range(len(workers))])
+    await c.replicate(L, n=2)
 
     adds = [
         delayed(slowadd, pure=True)(
@@ -147,7 +145,7 @@ def test_stress_scatter_death(c, s, *workers):
     from distributed.scheduler import logger
 
     for i in range(7):
-        yield gen.sleep(0.1)
+        await asyncio.sleep(0.1)
         try:
             s.validate_state()
         except Exception as c:
@@ -159,11 +157,11 @@ def test_stress_scatter_death(c, s, *workers):
             else:
                 raise
         w = random.choice(alive)
-        yield w.close()
+        await w.close()
         alive.remove(w)
 
     with ignoring(CancelledError):
-        yield c.gather(futures)
+        await c.gather(futures)
 
     futures = None
 
@@ -175,7 +173,7 @@ def vsum(*args):
 @pytest.mark.avoid_travis
 @pytest.mark.slow
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 80, timeout=1000)
-def test_stress_communication(c, s, *workers):
+async def test_stress_communication(c, s, *workers):
     s.validate = False  # very slow otherwise
     da = pytest.importorskip("dask.array")
     # Test consumes many file descriptors and can hang if the limit is too low
@@ -189,13 +187,13 @@ def test_stress_communication(c, s, *workers):
 
     future = c.compute(z.sum())
 
-    result = yield future
+    result = await future
     assert isinstance(result, float)
 
 
 @pytest.mark.skip
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 10, timeout=60)
-def test_stress_steal(c, s, *workers):
+async def test_stress_steal(c, s, *workers):
     s.validate = False
     for w in workers:
         w.validate = False
@@ -209,7 +207,7 @@ def test_stress_steal(c, s, *workers):
     future = c.compute(total)
 
     while future.status != "finished":
-        yield gen.sleep(0.1)
+        await asyncio.sleep(0.1)
         for i in range(3):
             a = random.choice(workers)
             b = random.choice(workers)
@@ -221,7 +219,7 @@ def test_stress_steal(c, s, *workers):
 
 @pytest.mark.slow
 @gen_cluster(nthreads=[("127.0.0.1", 1)] * 10, client=True, timeout=120)
-def test_close_connections(c, s, *workers):
+async def test_close_connections(c, s, *workers):
     da = pytest.importorskip("dask.array")
     x = da.random.random(size=(1000, 1000), chunks=(1000, 1))
     for i in range(3):
@@ -230,7 +228,7 @@ def test_close_connections(c, s, *workers):
 
     future = c.compute(x.sum())
     while any(s.processing.values()):
-        yield gen.sleep(0.5)
+        await asyncio.sleep(0.5)
         worker = random.choice(list(workers))
         for comm in worker._comms:
             comm.abort()
@@ -238,7 +236,7 @@ def test_close_connections(c, s, *workers):
         # for w in workers:
         #     print(w)
 
-    yield wait(future)
+    await wait(future)
 
 
 @pytest.mark.xfail(
@@ -246,7 +244,7 @@ def test_close_connections(c, s, *workers):
     " https://github.com/tornadoweb/tornado/issues/2110"
 )
 @gen_cluster(client=True, timeout=20, nthreads=[("127.0.0.1", 1)])
-def test_no_delay_during_large_transfer(c, s, w):
+async def test_no_delay_during_large_transfer(c, s, w):
     pytest.importorskip("crick")
     np = pytest.importorskip("numpy")
     x = np.random.random(100000000)
@@ -263,8 +261,8 @@ def test_no_delay_during_large_transfer(c, s, w):
         server._last_tick = time()
 
     with ResourceProfiler(dt=0.01) as rprof:
-        future = yield c.scatter(x, direct=True, hash=False)
-        yield gen.sleep(0.5)
+        future = await c.scatter(x, direct=True, hash=False)
+        await asyncio.sleep(0.5)
 
     rprof.close()
     x = None  # lose ref
diff --git a/distributed/tests/test_tls_functional.py b/distributed/tests/test_tls_functional.py
index 3a2bebf790d..67594f42926 100644
--- a/distributed/tests/test_tls_functional.py
+++ b/distributed/tests/test_tls_functional.py
@@ -2,7 +2,7 @@
 Various functional tests for TLS networking.
 Most are taken from other test files and adapted.
 """
-from tornado import gen
+import asyncio
 
 from distributed import Nanny, worker_client, Queue
 from distributed.client import wait
@@ -12,72 +12,72 @@
 
 
 @gen_tls_cluster(client=True)
-def test_basic(c, s, a, b):
+async def test_basic(c, s, a, b):
     pass
 
 
 @gen_tls_cluster(client=True)
-def test_Queue(c, s, a, b):
+async def test_Queue(c, s, a, b):
     assert s.address.startswith("tls://")
 
-    x = yield Queue("x")
-    y = yield Queue("y")
+    x = await Queue("x")
+    y = await Queue("y")
 
-    size = yield x.qsize()
+    size = await x.qsize()
     assert size == 0
 
     future = c.submit(inc, 1)
 
-    yield x.put(future)
+    await x.put(future)
 
-    future2 = yield x.get()
+    future2 = await x.get()
     assert future.key == future2.key
 
 
 @gen_tls_cluster(client=True, timeout=None)
-def test_client_submit(c, s, a, b):
+async def test_client_submit(c, s, a, b):
     assert s.address.startswith("tls://")
 
     x = c.submit(inc, 10)
-    result = yield x
+    result = await x
     assert result == 11
 
     yy = [c.submit(slowinc, i) for i in range(10)]
     results = []
     for y in yy:
-        results.append((yield y))
+        results.append(await y)
     assert results == list(range(1, 11))
 
 
 @gen_tls_cluster(client=True)
-def test_gather(c, s, a, b):
+async def test_gather(c, s, a, b):
     assert s.address.startswith("tls://")
 
     x = c.submit(inc, 10)
     y = c.submit(inc, x)
 
-    result = yield c._gather(x)
+    result = await c._gather(x)
     assert result == 11
-    result = yield c._gather([x])
+    result = await c._gather([x])
     assert result == [11]
-    result = yield c._gather({"x": x, "y": [y]})
+    result = await c._gather({"x": x, "y": [y]})
     assert result == {"x": 11, "y": [12]}
 
 
 @gen_tls_cluster(client=True)
-def test_scatter(c, s, a, b):
+async def test_scatter(c, s, a, b):
     assert s.address.startswith("tls://")
 
-    d = yield c._scatter({"y": 20})
+    d = await c._scatter({"y": 20})
     ts = s.tasks["y"]
     assert ts.who_has
     assert ts.nbytes > 0
-    yy = yield c._gather([d["y"]])
+    yy = await c._gather([d["y"]])
     assert yy == [20]
 
 
 @gen_tls_cluster(client=True, Worker=Nanny)
-def test_nanny(c, s, a, b):
+async def test_nanny(c, s, a, b):
     assert s.address.startswith("tls://")
     for n in [a, b]:
         assert isinstance(n, Nanny)
@@ -86,34 +86,34 @@ def test_nanny(c, s, a, b):
     assert s.nthreads == {n.worker_address: n.nthreads for n in [a, b]}
 
     x = c.submit(inc, 10)
-    result = yield x
+    result = await x
     assert result == 11
 
 
 @gen_tls_cluster(client=True)
-def test_rebalance(c, s, a, b):
-    x, y = yield c._scatter([1, 2], workers=[a.address])
+async def test_rebalance(c, s, a, b):
+    x, y = await c._scatter([1, 2], workers=[a.address])
     assert len(a.data) == 2
     assert len(b.data) == 0
 
-    yield c._rebalance()
+    await c._rebalance()
 
     assert len(a.data) == 1
     assert len(b.data) == 1
 
 
 @gen_tls_cluster(client=True, nthreads=[("tls://127.0.0.1", 2)] * 2)
-def test_work_stealing(c, s, a, b):
-    [x] = yield c._scatter([1], workers=a.address)
+async def test_work_stealing(c, s, a, b):
+    [x] = await c._scatter([1], workers=a.address)
     futures = c.map(slowadd, range(50), [x] * 50, delay=0.1)
-    yield gen.sleep(0.1)
-    yield wait(futures)
+    await asyncio.sleep(0.1)
+    await wait(futures)
     assert len(a.data) > 10
     assert len(b.data) > 10
 
 
 @gen_tls_cluster(client=True)
-def test_worker_client(c, s, a, b):
+async def test_worker_client(c, s, a, b):
     def func(x):
         with worker_client() as c:
             x = c.submit(inc, x)
@@ -122,14 +122,14 @@ def func(x):
             return result
 
     x, y = c.map(func, [10, 20])
-    xx, yy = yield c._gather([x, y])
+    xx, yy = await c._gather([x, y])
 
     assert xx == 10 + 1 + (10 + 1) * 2
     assert yy == 20 + 1 + (20 + 1) * 2
 
 
 @gen_tls_cluster(client=True, nthreads=[("tls://127.0.0.1", 1)] * 2)
-def test_worker_client_gather(c, s, a, b):
+async def test_worker_client_gather(c, s, a, b):
     a_address = a.address
     b_address = b.address
     assert a_address.startswith("tls://")
@@ -145,30 +145,30 @@ def func():
         return xx, yy
 
     future = c.submit(func)
-    result = yield future
+    result = await future
 
     assert result == (2, 3)
 
 
 @gen_tls_cluster(client=True)
-def test_worker_client_executor(c, s, a, b):
+async def test_worker_client_executor(c, s, a, b):
     def mysum():
         with worker_client() as c:
             with c.get_executor() as e:
                 return sum(e.map(double, range(30)))
 
     future = c.submit(mysum)
-    result = yield future
+    result = await future
     assert result == 30 * 29
 
 
 @gen_tls_cluster(client=True, Worker=Nanny)
-def test_retire_workers(c, s, a, b):
+async def test_retire_workers(c, s, a, b):
     assert set(s.workers) == {a.worker_address, b.worker_address}
-    yield c.retire_workers(workers=[a.worker_address], close_workers=True)
+    await c.retire_workers(workers=[a.worker_address], close_workers=True)
     assert set(s.workers) == {b.worker_address}
 
     start = time()
     while a.status != "closed":
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 5
diff --git a/distributed/tests/test_utils.py b/distributed/tests/test_utils.py
index e162b9fc2e1..86f1ca0c208 100644
--- a/distributed/tests/test_utils.py
+++ b/distributed/tests/test_utils.py
@@ -1,3 +1,4 @@
+import asyncio
 import array
 import datetime
 from functools import partial
@@ -11,7 +12,6 @@
 
 import numpy as np
 import pytest
-from tornado import gen
 from tornado.ioloop import IOLoop
 
 import dask
@@ -51,28 +51,23 @@
 
 
 def test_All(loop):
-    @gen.coroutine
-    def throws():
+    async def throws():
         1 / 0
 
-    @gen.coroutine
-    def slow():
-        yield gen.sleep(10)
+    async def slow():
+        await asyncio.sleep(10)
 
-    @gen.coroutine
-    def inc(x):
-        raise gen.Return(x + 1)
+    async def inc(x):
+        return x + 1
 
-    @gen.coroutine
-    def f():
-
-        results = yield All([inc(i) for i in range(10)])
+    async def f():
+        results = await All([inc(i) for i in range(10)])
         assert results == list(range(1, 11))
 
         start = time()
         for tasks in [[throws(), slow()], [slow(), throws()]]:
             try:
-                yield All(tasks)
+                await All(tasks)
                 assert False
             except ZeroDivisionError:
                 pass
@@ -112,7 +107,7 @@ def function2(x):
 def test_sync_timeout(loop_in_thread):
     loop = loop_in_thread
     with pytest.raises(TimeoutError):
-        sync(loop_in_thread, gen.sleep, 0.5, callback_timeout=0.05)
+        sync(loop_in_thread, asyncio.sleep, 0.5, callback_timeout=0.05)
 
 
 def test_sync_closed_loop():
@@ -484,17 +479,17 @@ def test_two_loop_runners(loop_in_thread):
 
 
 @gen_test()
-def test_loop_runner_gen():
+async def test_loop_runner_gen():
     runner = LoopRunner(asynchronous=True)
     assert runner.loop is IOLoop.current()
     assert not runner.is_started()
-    yield gen.sleep(0.01)
+    await asyncio.sleep(0.01)
     runner.start()
     assert runner.is_started()
-    yield gen.sleep(0.01)
+    await asyncio.sleep(0.01)
     runner.stop()
     assert not runner.is_started()
-    yield gen.sleep(0.01)
+    await asyncio.sleep(0.01)
 
 
 def test_parse_bytes():
@@ -537,21 +532,20 @@ def test_parse_timedelta():
 
 
 @gen_test()
-def test_all_exceptions_logging():
-    @gen.coroutine
-    def throws():
+async def test_all_exceptions_logging():
+    async def throws():
         raise Exception("foo1234")
 
     with captured_logger("") as sio:
         try:
-            yield All([throws() for _ in range(5)], quiet_exceptions=Exception)
+            await All([throws() for _ in range(5)], quiet_exceptions=Exception)
         except Exception:
             pass
 
         import gc
 
         gc.collect()
-        yield gen.sleep(0.1)
+        await asyncio.sleep(0.1)
 
     assert "foo1234" not in sio.getvalue()
 
diff --git a/distributed/tests/test_utils_test.py b/distributed/tests/test_utils_test.py
index 6a4a5ceaa5e..502b27b3013 100755
--- a/distributed/tests/test_utils_test.py
+++ b/distributed/tests/test_utils_test.py
@@ -1,10 +1,10 @@
+import asyncio
 from contextlib import contextmanager
 import socket
 import threading
 from time import sleep
 
 import pytest
-from tornado import gen
 
 from distributed import Scheduler, Worker, Client, config, default_client
 from distributed.core import rpc
@@ -43,7 +43,7 @@ def test_cluster(loop):
 
 
 @gen_cluster(client=True)
-def test_gen_cluster(c, s, a, b):
+async def test_gen_cluster(c, s, a, b):
     assert isinstance(c, Client)
     assert isinstance(s, Scheduler)
     for w in [a, b]:
@@ -58,9 +58,9 @@ def test_gen_cluster_cleans_up_client(loop):
     assert not dask.config.get("get", None)
 
     @gen_cluster(client=True)
-    def f(c, s, a, b):
+    async def f(c, s, a, b):
         assert dask.config.get("get", None)
-        yield c.submit(inc, 1)
+        await c.submit(inc, 1)
 
     f()
 
@@ -68,12 +68,17 @@ def f(c, s, a, b):
 
 
 @gen_cluster(client=False)
-def test_gen_cluster_without_client(s, a, b):
+async def test_gen_cluster_without_client(s, a, b):
     assert isinstance(s, Scheduler)
     for w in [a, b]:
         assert isinstance(w, Worker)
     assert s.nthreads == {w.address: w.nthreads for w in [a, b]}
 
+    async with Client(s.address, asynchronous=True) as c:
+        future = c.submit(lambda x: x + 1, 1)
+        result = await future
+        assert result == 2
+
 
 @gen_cluster(
     client=True,
@@ -81,7 +86,7 @@ def test_gen_cluster_without_client(s, a, b):
     nthreads=[("tls://127.0.0.1", 1), ("tls://127.0.0.1", 2)],
     security=tls_only_security(),
 )
-def test_gen_cluster_tls(e, s, a, b):
+async def test_gen_cluster_tls(e, s, a, b):
     assert isinstance(e, Client)
     assert isinstance(s, Scheduler)
     assert s.address.startswith("tls://")
@@ -92,8 +97,8 @@ def test_gen_cluster_tls(e, s, a, b):
 
 
 @gen_test()
-def test_gen_test():
-    yield gen.sleep(0.01)
+async def test_gen_test():
+    await asyncio.sleep(0.01)
 
 
 @contextmanager
@@ -154,8 +159,8 @@ def test_new_config():
 
 def test_lingering_client():
     @gen_cluster()
-    def f(s, a, b):
-        c = yield Client(s.address, asynchronous=True)
+    async def f(s, a, b):
+        await Client(s.address, asynchronous=True)
 
     f()
 
@@ -177,16 +182,3 @@ def test_tls_cluster(tls_client):
 async def test_tls_scheduler(security, cleanup):
     async with Scheduler(security=security, host="localhost") as s:
         assert s.address.startswith("tls")
-
-
-@gen_cluster()
-async def test_gen_cluster_async(s, a, b):  # flake8: noqa
-    async with Client(s.address, asynchronous=True) as c:
-        future = c.submit(lambda x: x + 1, 1)
-        result = await future
-        assert result == 2
-
-
-@gen_test()
-async def test_gen_test_async():  # flake8: noqa
-    await gen.sleep(0.001)
diff --git a/distributed/tests/test_variable.py b/distributed/tests/test_variable.py
index 0e450aa7a02..a60345d0abb 100644
--- a/distributed/tests/test_variable.py
+++ b/distributed/tests/test_variable.py
@@ -1,11 +1,9 @@
 import asyncio
 import random
 from time import sleep
-import sys
 import logging
 
 import pytest
-from tornado import gen
 from tornado.ioloop import IOLoop
 
 from distributed import Client, Variable, worker_client, Nanny, wait, TimeoutError
@@ -17,27 +15,27 @@
 
 
 @gen_cluster(client=True)
-def test_variable(c, s, a, b):
+async def test_variable(c, s, a, b):
     x = Variable("x")
     xx = Variable("x")
     assert x.client is c
 
     future = c.submit(inc, 1)
 
-    yield x.set(future)
-    future2 = yield xx.get()
+    await x.set(future)
+    future2 = await xx.get()
     assert future.key == future2.key
 
     del future, future2
 
-    yield gen.sleep(0.1)
+    await asyncio.sleep(0.1)
     assert s.tasks  # future still present
 
     x.delete()
 
     start = time()
     while s.tasks:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 5
 
 
@@ -53,13 +51,13 @@ async def test_delete_unset_variable(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_queue_with_data(c, s, a, b):
+async def test_queue_with_data(c, s, a, b):
     x = Variable("x")
     xx = Variable("x")
     assert x.client is c
 
-    yield x.set((1, "hello"))
-    data = yield xx.get()
+    await x.set((1, "hello"))
+    data = await xx.get()
 
     assert data == (1, "hello")
 
@@ -75,32 +73,32 @@ def test_sync(client):
 
 
 @gen_cluster()
-def test_hold_futures(s, a, b):
-    c1 = yield Client(s.address, asynchronous=True)
+async def test_hold_futures(s, a, b):
+    c1 = await Client(s.address, asynchronous=True)
     future = c1.submit(lambda x: x + 1, 10)
     x1 = Variable("x")
-    yield x1.set(future)
+    await x1.set(future)
     del x1
-    yield c1.close()
+    await c1.close()
 
-    yield gen.sleep(0.1)
+    await asyncio.sleep(0.1)
 
-    c2 = yield Client(s.address, asynchronous=True)
+    c2 = await Client(s.address, asynchronous=True)
     x2 = Variable("x")
-    future2 = yield x2.get()
-    result = yield future2
+    future2 = await x2.get()
+    result = await future2
 
     assert result == 11
-    yield c2.close()
+    await c2.close()
 
 
 @gen_cluster(client=True)
-def test_timeout(c, s, a, b):
+async def test_timeout(c, s, a, b):
     v = Variable("v")
 
     start = IOLoop.current().time()
     with pytest.raises(TimeoutError):
-        yield v.get(timeout=0.2)
+        await v.get(timeout=0.2)
     stop = IOLoop.current().time()
 
     if WINDOWS:  # timing is weird with asyncio and Windows
@@ -109,7 +107,7 @@ def test_timeout(c, s, a, b):
         assert 0.2 < stop - start < 2.0
 
     with pytest.raises(TimeoutError):
-        yield v.get(timeout=0.01)
+        await v.get(timeout=0.01)
 
 
 def test_timeout_sync(client):
@@ -139,10 +137,10 @@ async def test_cleanup(c, s, a, b):
 
     await v.set(x)
     del x
-    await gen.sleep(0.1)
+    await asyncio.sleep(0.1)
 
     t_future = xx = asyncio.ensure_future(vv._get())
-    await gen.sleep(0)
+    await asyncio.sleep(0)
     asyncio.ensure_future(v.set(y))
 
     future = await t_future
@@ -162,22 +160,21 @@ def f(x):
 
 
 @gen_cluster(client=True)
-def test_timeout_get(c, s, a, b):
+async def test_timeout_get(c, s, a, b):
     v = Variable("v")
 
     tornado_future = v.get()
 
     vv = Variable("v")
-    yield vv.set(1)
+    await vv.set(1)
 
-    result = yield tornado_future
+    result = await tornado_future
     assert result == 1
 
 
-@pytest.mark.skipif(sys.version_info[0] == 2, reason="Multi-client issues")
 @pytest.mark.slow
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 2)] * 5, Worker=Nanny, timeout=None)
-def test_race(c, s, *workers):
+async def test_race(c, s, *workers):
     NITERS = 50
 
     def f(i):
@@ -194,63 +191,63 @@ def f(i):
             return result
 
     v = Variable("x", client=c)
-    x = yield c.scatter(1)
-    yield v.set(x)
+    x = await c.scatter(1)
+    await v.set(x)
 
     futures = c.map(f, range(15))
-    results = yield c.gather(futures)
+    results = await c.gather(futures)
     assert all(r > NITERS * 0.8 for r in results)
 
     start = time()
     while len(s.wants_what["variable-x"]) != 1:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() - start < 2
 
 
 @gen_cluster(client=True)
-def test_Future_knows_status_immediately(c, s, a, b):
-    x = yield c.scatter(123)
+async def test_Future_knows_status_immediately(c, s, a, b):
+    x = await c.scatter(123)
     v = Variable("x")
-    yield v.set(x)
+    await v.set(x)
 
-    c2 = yield Client(s.address, asynchronous=True)
+    c2 = await Client(s.address, asynchronous=True)
     v2 = Variable("x", client=c2)
-    future = yield v2.get()
+    future = await v2.get()
     assert future.status == "finished"
 
     x = c.submit(div, 1, 0)
-    yield wait(x)
-    yield v.set(x)
+    await wait(x)
+    await v.set(x)
 
-    future2 = yield v2.get()
+    future2 = await v2.get()
     assert future2.status == "error"
     with pytest.raises(Exception):
-        yield future2
+        await future2
 
     start = time()
     while True:  # we learn about the true error eventually
         try:
-            yield future2
+            await future2
         except ZeroDivisionError:
             break
         except Exception:
             assert time() < start + 5
-            yield gen.sleep(0.05)
+            await asyncio.sleep(0.05)
 
-    yield c2.close()
+    await c2.close()
 
 
 @gen_cluster(client=True)
-def test_erred_future(c, s, a, b):
+async def test_erred_future(c, s, a, b):
     future = c.submit(div, 1, 0)
     var = Variable()
-    yield var.set(future)
-    yield gen.sleep(0.1)
-    future2 = yield var.get()
+    await var.set(future)
+    await asyncio.sleep(0.1)
+    future2 = await var.get()
     with pytest.raises(ZeroDivisionError):
-        yield future2.result()
+        await future2.result()
 
-    exc = yield future2.exception()
+    exc = await future2.exception()
     assert isinstance(exc, ZeroDivisionError)
 
 
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index a5e364ec0cd..a1f2e46295c 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -16,14 +16,11 @@
 from dask.system import CPU_COUNT
 import pytest
 from tlz import pluck, sliding_window, first
-import tornado
-from tornado import gen
 
 from distributed import (
     Client,
     Nanny,
     get_client,
-    wait,
     default_client,
     get_worker,
     Reschedule,
@@ -65,7 +62,7 @@ async def test_worker_nthreads(cleanup):
 
 
 @gen_cluster()
-def test_str(s, a, b):
+async def test_str(s, a, b):
     assert a.address in str(a)
     assert a.address in repr(a)
     assert str(a.nthreads) in str(a)
@@ -85,7 +82,7 @@ async def test_identity(cleanup):
 
 
 @gen_cluster(client=True)
-def test_worker_bad_args(c, s, a, b):
+async def test_worker_bad_args(c, s, a, b):
     class NoReprObj:
         """ This object cannot be properly represented as a string. """
 
@@ -96,7 +93,7 @@ def __repr__(self):
             raise ValueError("I have no repr representation.")
 
     x = c.submit(NoReprObj, workers=a.address)
-    yield wait(x)
+    await wait(x)
     assert not a.executing
     assert a.data
 
@@ -127,20 +124,17 @@ def reset(self):
     logger.setLevel(logging.DEBUG)
     logger.addHandler(hdlr)
     y = c.submit(bad_func, x, k=x, workers=b.address)
-    yield wait(y)
+    await wait(y)
 
     assert not b.executing
     assert y.status == "error"
     # Make sure job died because of bad func and not because of bad
     # argument.
     with pytest.raises(ZeroDivisionError):
-        yield y
+        await y
 
-    if sys.version_info[0] >= 3:
-        tb = yield y._traceback()
-        assert any(
-            "1 / 0" in line for line in pluck(3, traceback.extract_tb(tb)) if line
-        )
+    tb = await y._traceback()
+    assert any("1 / 0" in line for line in pluck(3, traceback.extract_tb(tb)) if line)
     assert "Compute Failed" in hdlr.messages["warning"][0]
     logger.setLevel(old_level)
 
@@ -149,14 +143,14 @@ def reset(self):
     xx = c.submit(add, 1, 2, workers=a.address)
     yy = c.submit(add, 3, 4, workers=b.address)
 
-    results = yield c._gather([xx, yy])
+    results = await c._gather([xx, yy])
 
     assert tuple(results) == (3, 7)
 
 
 @pytest.mark.slow
 @gen_cluster()
-def dont_test_delete_data_with_missing_worker(c, a, b):
+async def dont_test_delete_data_with_missing_worker(c, a, b):
     bad = "127.0.0.1:9001"  # this worker doesn't exist
     c.who_has["z"].add(bad)
     c.who_has["z"].add(a.address)
@@ -166,26 +160,26 @@ def dont_test_delete_data_with_missing_worker(c, a, b):
 
     cc = rpc(ip=c.ip, port=c.port)
 
-    yield cc.delete_data(keys=["z"])  # TODO: this hangs for a while
+    await cc.delete_data(keys=["z"])  # TODO: this hangs for a while
     assert "z" not in a.data
     assert not c.who_has["z"]
     assert not c.has_what[bad]
     assert not c.has_what[a.address]
 
-    yield cc.close_rpc()
+    await cc.close_rpc()
 
 
 @gen_cluster(client=True)
-def test_upload_file(c, s, a, b):
+async def test_upload_file(c, s, a, b):
     assert not os.path.exists(os.path.join(a.local_directory, "foobar.py"))
     assert not os.path.exists(os.path.join(b.local_directory, "foobar.py"))
     assert a.local_directory != b.local_directory
 
     with rpc(a.address) as aa, rpc(b.address) as bb:
-        yield [
+        await asyncio.gather(
             aa.upload_file(filename="foobar.py", data=b"x = 123"),
             bb.upload_file(filename="foobar.py", data="x = 123"),
-        ]
+        )
 
     assert os.path.exists(os.path.join(a.local_directory, "foobar.py"))
     assert os.path.exists(os.path.join(b.local_directory, "foobar.py"))
@@ -196,17 +190,17 @@ def g():
         return foobar.x
 
     future = c.submit(g, workers=a.address)
-    result = yield future
+    result = await future
     assert result == 123
 
-    yield c.close()
-    yield s.close(close_workers=True)
+    await c.close()
+    await s.close(close_workers=True)
     assert not os.path.exists(os.path.join(a.local_directory, "foobar.py"))
 
 
 @pytest.mark.skip(reason="don't yet support uploading pyc files")
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)])
-def test_upload_file_pyc(c, s, w):
+async def test_upload_file_pyc(c, s, w):
     with tmpfile() as dirname:
         os.mkdir(dirname)
         with open(os.path.join(dirname, "foo.py"), mode="w") as f:
@@ -219,7 +213,7 @@ def test_upload_file_pyc(c, s, w):
             assert foo.f() == 123
             pyc = importlib.util.cache_from_source(os.path.join(dirname, "foo.py"))
             assert os.path.exists(pyc)
-            yield c.upload_file(pyc)
+            await c.upload_file(pyc)
 
             def g():
                 import foo
@@ -227,21 +221,21 @@ def g():
                 return foo.x
 
             future = c.submit(g)
-            result = yield future
+            result = await future
             assert result == 123
         finally:
             sys.path.remove(dirname)
 
 
 @gen_cluster(client=True)
-def test_upload_egg(c, s, a, b):
+async def test_upload_egg(c, s, a, b):
     eggname = "testegg-1.0.0-py3.4.egg"
     local_file = __file__.replace("test_worker.py", eggname)
     assert not os.path.exists(os.path.join(a.local_directory, eggname))
     assert not os.path.exists(os.path.join(b.local_directory, eggname))
     assert a.local_directory != b.local_directory
 
-    yield c.upload_file(filename=local_file)
+    await c.upload_file(filename=local_file)
 
     assert os.path.exists(os.path.join(a.local_directory, eggname))
     assert os.path.exists(os.path.join(b.local_directory, eggname))
@@ -252,25 +246,25 @@ def g(x):
         return testegg.inc(x)
 
     future = c.submit(g, 10, workers=a.address)
-    result = yield future
+    result = await future
     assert result == 10 + 1
 
-    yield c.close()
-    yield s.close()
-    yield a.close()
-    yield b.close()
+    await c.close()
+    await s.close()
+    await a.close()
+    await b.close()
     assert not os.path.exists(os.path.join(a.local_directory, eggname))
 
 
 @gen_cluster(client=True)
-def test_upload_pyz(c, s, a, b):
+async def test_upload_pyz(c, s, a, b):
     pyzname = "mytest.pyz"
     local_file = __file__.replace("test_worker.py", pyzname)
     assert not os.path.exists(os.path.join(a.local_directory, pyzname))
     assert not os.path.exists(os.path.join(b.local_directory, pyzname))
     assert a.local_directory != b.local_directory
 
-    yield c.upload_file(filename=local_file)
+    await c.upload_file(filename=local_file)
 
     assert os.path.exists(os.path.join(a.local_directory, pyzname))
     assert os.path.exists(os.path.join(b.local_directory, pyzname))
@@ -281,42 +275,42 @@ def g(x):
         return mytest.inc(x)
 
     future = c.submit(g, 10, workers=a.address)
-    result = yield future
+    result = await future
     assert result == 10 + 1
 
-    yield c.close()
-    yield s.close()
-    yield a.close()
-    yield b.close()
+    await c.close()
+    await s.close()
+    await a.close()
+    await b.close()
     assert not os.path.exists(os.path.join(a.local_directory, pyzname))
 
 
 @pytest.mark.xfail(reason="Still lose time to network I/O")
 @gen_cluster(client=True)
-def test_upload_large_file(c, s, a, b):
+async def test_upload_large_file(c, s, a, b):
     pytest.importorskip("crick")
-    yield gen.sleep(0.05)
+    await asyncio.sleep(0.05)
     with rpc(a.address) as aa:
-        yield aa.upload_file(filename="myfile.dat", data=b"0" * 100000000)
-        yield gen.sleep(0.05)
+        await aa.upload_file(filename="myfile.dat", data=b"0" * 100000000)
+        await asyncio.sleep(0.05)
         assert a.digests["tick-duration"].components[0].max() < 0.050
 
 
 @gen_cluster()
-def test_broadcast(s, a, b):
+async def test_broadcast(s, a, b):
     with rpc(s.address) as cc:
-        results = yield cc.broadcast(msg={"op": "ping"})
+        results = await cc.broadcast(msg={"op": "ping"})
         assert results == {a.address: b"pong", b.address: b"pong"}
 
 
 @gen_test()
-def test_worker_with_port_zero():
-    s = yield Scheduler(port=8007)
-    w = yield Worker(s.address)
+async def test_worker_with_port_zero():
+    s = await Scheduler(port=8007)
+    w = await Worker(s.address)
     assert isinstance(w.port, int)
     assert w.port > 1024
 
-    yield w.close()
+    await w.close()
 
 
 @pytest.mark.slow
@@ -334,10 +328,10 @@ async def test_worker_waits_for_scheduler(cleanup):
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)])
-def test_worker_task_data(c, s, w):
+async def test_worker_task_data(c, s, w):
     x = delayed(2)
     xx = c.persist(x)
-    yield wait(xx)
+    await wait(xx)
     assert w.data[x.key] == 2
 
 
@@ -370,7 +364,7 @@ def __str__(self):
 
 
 @gen_cluster(client=True)
-def test_chained_error_message(c, s, a, b):
+async def test_chained_error_message(c, s, a, b):
     def chained_exception_fn():
         class MyException(Exception):
             def __init__(self, msg):
@@ -389,18 +383,18 @@ def __str__(self):
 
     f = c.submit(chained_exception_fn)
     try:
-        yield f
+        await f
     except Exception as e:
         assert e.__cause__ is not None
         assert "Bar" in str(e.__cause__)
 
 
 @gen_cluster()
-def test_gather(s, a, b):
+async def test_gather(s, a, b):
     b.data["x"] = 1
     b.data["y"] = 2
     with rpc(a.address) as aa:
-        resp = yield aa.gather(who_has={"x": [b.address], "y": [b.address]})
+        resp = await aa.gather(who_has={"x": [b.address], "y": [b.address]})
         assert resp["status"] == "OK"
 
         assert a.data["x"] == b.data["x"]
@@ -415,9 +409,9 @@ async def test_io_loop(cleanup):
 
 
 @gen_cluster(client=True, nthreads=[])
-def test_spill_to_disk(c, s):
+async def test_spill_to_disk(c, s):
     np = pytest.importorskip("numpy")
-    w = yield Worker(
+    w = await Worker(
         s.address,
         loop=s.loop,
         memory_limit=1200 / 0.6,
@@ -426,79 +420,75 @@ def test_spill_to_disk(c, s):
     )
 
     x = c.submit(np.random.randint, 0, 255, size=500, dtype="u1", key="x")
-    yield wait(x)
+    await wait(x)
     y = c.submit(np.random.randint, 0, 255, size=500, dtype="u1", key="y")
-    yield wait(y)
+    await wait(y)
 
     assert set(w.data) == {x.key, y.key}
     assert set(w.data.memory) == {x.key, y.key}
     assert set(w.data.fast) == set(w.data.memory)
 
     z = c.submit(np.random.randint, 0, 255, size=500, dtype="u1", key="z")
-    yield wait(z)
+    await wait(z)
     assert set(w.data) == {x.key, y.key, z.key}
     assert set(w.data.memory) == {y.key, z.key}
     assert set(w.data.disk) == {x.key} or set(w.data.slow) == {x.key, y.key}
     assert set(w.data.fast) == set(w.data.memory)
     assert set(w.data.slow) == set(w.data.disk)
 
-    yield x
+    await x
     assert set(w.data.memory) == {x.key, z.key}
     assert set(w.data.disk) == {y.key} or set(w.data.slow) == {x.key, y.key}
     assert set(w.data.fast) == set(w.data.memory)
     assert set(w.data.slow) == set(w.data.disk)
-    yield w.close()
+    await w.close()
 
 
 @gen_cluster(client=True)
-def test_access_key(c, s, a, b):
+async def test_access_key(c, s, a, b):
     def f(i):
         from distributed.worker import thread_state
 
         return thread_state.key
 
     futures = [c.submit(f, i, key="x-%d" % i) for i in range(20)]
-    results = yield c._gather(futures)
+    results = await c._gather(futures)
     assert list(results) == ["x-%d" % i for i in range(20)]
 
 
 @gen_cluster(client=True)
-def test_run_dask_worker(c, s, a, b):
+async def test_run_dask_worker(c, s, a, b):
     def f(dask_worker=None):
         return dask_worker.id
 
-    response = yield c._run(f)
+    response = await c._run(f)
     assert response == {a.address: a.id, b.address: b.id}
 
 
 @gen_cluster(client=True)
-def test_run_coroutine_dask_worker(c, s, a, b):
-    if sys.version_info < (3,) and tornado.version_info < (4, 5):
-        pytest.skip("test needs Tornado 4.5+ on Python 2.7")
-
-    @gen.coroutine
-    def f(dask_worker=None):
-        yield gen.sleep(0.001)
-        raise gen.Return(dask_worker.id)
+async def test_run_coroutine_dask_worker(c, s, a, b):
+    async def f(dask_worker=None):
+        await asyncio.sleep(0.001)
+        return dask_worker.id
 
-    response = yield c.run(f)
+    response = await c.run(f)
     assert response == {a.address: a.id, b.address: b.id}
 
 
 @gen_cluster(client=True, nthreads=[])
-def test_Executor(c, s):
+async def test_Executor(c, s):
     with ThreadPoolExecutor(2) as e:
         w = Worker(s.address, executor=e)
         assert w.executor is e
-        w = yield w
+        w = await w
 
         future = c.submit(inc, 1)
-        result = yield future
+        result = await future
         assert result == 2
 
         assert e._threads  # had to do some work
 
-        yield w.close()
+        await w.close()
 
 
 @pytest.mark.skip(
@@ -510,22 +500,22 @@ def test_Executor(c, s):
     timeout=30,
     worker_kwargs={"memory_limit": 10e6},
 )
-def test_spill_by_default(c, s, w):
+async def test_spill_by_default(c, s, w):
     da = pytest.importorskip("dask.array")
     x = da.ones(int(10e6 * 0.7), chunks=1e6, dtype="u1")
     y = c.persist(x)
-    yield wait(y)
+    await wait(y)
     assert len(w.data.disk)  # something is on disk
     del x, y
 
 
 @gen_cluster(nthreads=[("127.0.0.1", 1)], worker_kwargs={"reconnect": False})
-def test_close_on_disconnect(s, w):
-    yield s.close()
+async def test_close_on_disconnect(s, w):
+    await s.close()
 
     start = time()
     while w.status != "closed":
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 5
 
 
@@ -547,20 +537,20 @@ async def test_memory_limit_auto():
 
 
 @gen_cluster(client=True)
-def test_inter_worker_communication(c, s, a, b):
-    [x, y] = yield c._scatter([1, 2], workers=a.address)
+async def test_inter_worker_communication(c, s, a, b):
+    [x, y] = await c._scatter([1, 2], workers=a.address)
 
     future = c.submit(add, x, y, workers=b.address)
-    result = yield future
+    result = await future
     assert result == 3
 
 
 @gen_cluster(client=True)
-def test_clean(c, s, a, b):
+async def test_clean(c, s, a, b):
     x = c.submit(inc, 1, workers=a.address)
     y = c.submit(inc, x, workers=b.address)
 
-    yield y
+    await y
 
     collections = [
         a.tasks,
@@ -580,21 +570,20 @@ def test_clean(c, s, a, b):
     y.release()
 
     while x.key in a.task_state:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
     for c in collections:
         assert not c
 
 
-@pytest.mark.skipif(sys.version_info[:2] == (3, 4), reason="mul bytes fails")
 @gen_cluster(client=True)
-def test_message_breakup(c, s, a, b):
+async def test_message_breakup(c, s, a, b):
     n = 100000
     a.target_message_size = 10 * n
     b.target_message_size = 10 * n
     xs = [c.submit(mul, b"%d" % i, n, workers=a.address) for i in range(30)]
     y = c.submit(lambda *args: None, xs, workers=b.address)
-    yield y
+    await y
 
     assert 2 <= len(b.incoming_transfer_log) <= 20
     assert 2 <= len(a.outgoing_transfer_log) <= 20
@@ -604,29 +593,29 @@ def test_message_breakup(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_types(c, s, a, b):
+async def test_types(c, s, a, b):
     assert not a.types
     assert not b.types
     x = c.submit(inc, 1, workers=a.address)
-    yield wait(x)
+    await wait(x)
     assert a.types[x.key] == int
 
     y = c.submit(inc, x, workers=b.address)
-    yield wait(y)
+    await wait(y)
     assert b.types == {x.key: int, y.key: int}
 
-    yield c._cancel(y)
+    await c._cancel(y)
 
     start = time()
     while y.key in b.data:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 5
 
     assert y.key not in b.types
 
 
 @gen_cluster()
-def test_system_monitor(s, a, b):
+async def test_system_monitor(s, a, b):
     assert b.monitor
     b.monitor.update()
 
@@ -634,38 +623,38 @@ def test_system_monitor(s, a, b):
 @gen_cluster(
     client=True, nthreads=[("127.0.0.1", 2, {"resources": {"A": 1}}), ("127.0.0.1", 1)]
 )
-def test_restrictions(c, s, a, b):
+async def test_restrictions(c, s, a, b):
     # Resource restrictions
     x = c.submit(inc, 1, resources={"A": 1})
-    yield x
+    await x
     assert a.resource_restrictions == {x.key: {"A": 1}}
-    yield c._cancel(x)
+    await c._cancel(x)
 
     while x.key in a.task_state:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
 
     assert a.resource_restrictions == {}
 
 
 @pytest.mark.xfail
 @gen_cluster(client=True)
-def test_clean_nbytes(c, s, a, b):
+async def test_clean_nbytes(c, s, a, b):
     L = [delayed(inc)(i) for i in range(10)]
     for i in range(5):
         L = [delayed(add)(x, y) for x, y in sliding_window(2, L)]
     total = delayed(sum)(L)
 
     future = c.compute(total)
-    yield wait(future)
+    await wait(future)
 
-    yield gen.sleep(1)
+    await asyncio.sleep(1)
     assert len(a.nbytes) + len(b.nbytes) == 1
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 20)
-def test_gather_many_small(c, s, a, *workers):
+async def test_gather_many_small(c, s, a, *workers):
     a.total_out_connections = 2
-    futures = yield c._scatter(list(range(100)))
+    futures = await c._scatter(list(range(100)))
 
     assert all(w.data for w in workers)
 
@@ -673,7 +662,7 @@ def f(*args):
         return 10
 
     future = c.submit(f, *futures, workers=a.address)
-    yield wait(future)
+    await wait(future)
 
     types = list(pluck(0, a.log))
     req = [i for i, t in enumerate(types) if t == "request-dep"]
@@ -684,12 +673,12 @@ def f(*args):
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 3)
-def test_multiple_transfers(c, s, w1, w2, w3):
+async def test_multiple_transfers(c, s, w1, w2, w3):
     x = c.submit(inc, 1, workers=w1.address)
     y = c.submit(inc, 2, workers=w2.address)
     z = c.submit(add, x, y, workers=w3.address)
 
-    yield wait(z)
+    await wait(z)
 
     r = w3.startstops[z.key]
     transfers = [t for t in r if t["action"] == "transfer"]
@@ -697,25 +686,25 @@ def test_multiple_transfers(c, s, w1, w2, w3):
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 3)
-def test_share_communication(c, s, w1, w2, w3):
+async def test_share_communication(c, s, w1, w2, w3):
     x = c.submit(mul, b"1", int(w3.target_message_size + 1), workers=w1.address)
     y = c.submit(mul, b"2", int(w3.target_message_size + 1), workers=w2.address)
-    yield wait([x, y])
-    yield c._replicate([x, y], workers=[w1.address, w2.address])
+    await wait([x, y])
+    await c._replicate([x, y], workers=[w1.address, w2.address])
     z = c.submit(add, x, y, workers=w3.address)
-    yield wait(z)
+    await wait(z)
     assert len(w3.incoming_transfer_log) == 2
     assert w1.outgoing_transfer_log
     assert w2.outgoing_transfer_log
 
 
 @gen_cluster(client=True)
-def test_dont_overlap_communications_to_same_worker(c, s, a, b):
+async def test_dont_overlap_communications_to_same_worker(c, s, a, b):
     x = c.submit(mul, b"1", int(b.target_message_size + 1), workers=a.address)
     y = c.submit(mul, b"2", int(b.target_message_size + 1), workers=a.address)
-    yield wait([x, y])
+    await wait([x, y])
     z = c.submit(add, x, y, workers=b.address)
-    yield wait(z)
+    await wait(z)
     assert len(b.incoming_transfer_log) == 2
     l1, l2 = b.incoming_transfer_log
 
@@ -724,7 +713,7 @@ def test_dont_overlap_communications_to_same_worker(c, s, a, b):
 
 @pytest.mark.avoid_travis
 @gen_cluster(client=True)
-def test_log_exception_on_failed_task(c, s, a, b):
+async def test_log_exception_on_failed_task(c, s, a, b):
     with tmpfile() as fn:
         fh = logging.FileHandler(fn)
         try:
@@ -733,9 +722,9 @@ def test_log_exception_on_failed_task(c, s, a, b):
             logger.addHandler(fh)
 
             future = c.submit(div, 1, 0)
-            yield wait(future)
+            await wait(future)
 
-            yield gen.sleep(0.1)
+            await asyncio.sleep(0.1)
             fh.flush()
             with open(fn) as f:
                 text = f.read()
@@ -747,7 +736,7 @@ def test_log_exception_on_failed_task(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_clean_up_dependencies(c, s, a, b):
+async def test_clean_up_dependencies(c, s, a, b):
     x = delayed(inc)(1)
     y = delayed(inc)(2)
     xx = delayed(inc)(x)
@@ -755,26 +744,26 @@ def test_clean_up_dependencies(c, s, a, b):
     z = delayed(add)(xx, yy)
 
     zz = c.persist(z)
-    yield wait(zz)
+    await wait(zz)
 
     start = time()
     while len(a.data) + len(b.data) > 1:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 2
 
     assert set(a.data) | set(b.data) == {zz.key}
 
 
 @gen_cluster(client=True)
-def test_hold_onto_dependents(c, s, a, b):
+async def test_hold_onto_dependents(c, s, a, b):
     x = c.submit(inc, 1, workers=a.address)
     y = c.submit(inc, x, workers=b.address)
-    yield wait(y)
+    await wait(y)
 
     assert x.key in b.data
 
-    yield c._cancel(y)
-    yield gen.sleep(0.1)
+    await c._cancel(y)
+    await asyncio.sleep(0.1)
 
     assert x.key in b.data
 
@@ -796,20 +785,20 @@ async def test_worker_death_timeout(s):
 
 
 @gen_cluster(client=True)
-def test_stop_doing_unnecessary_work(c, s, a, b):
+async def test_stop_doing_unnecessary_work(c, s, a, b):
     futures = c.map(slowinc, range(1000), delay=0.01)
-    yield gen.sleep(0.1)
+    await asyncio.sleep(0.1)
 
     del futures
 
     start = time()
     while a.executing:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() - start < 0.5
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)])
-def test_priorities(c, s, w):
+async def test_priorities(c, s, w):
     values = []
     for i in range(10):
         a = delayed(slowinc)(i, dask_key_name="a-%d" % i, delay=0.01)
@@ -821,7 +810,7 @@ def test_priorities(c, s, w):
         values.append(b1)
 
     futures = c.compute(values)
-    yield wait(futures)
+    await wait(futures)
 
     log = [
         t[0]
@@ -833,12 +822,12 @@ def test_priorities(c, s, w):
 
 
 @gen_cluster(client=True)
-def test_heartbeats(c, s, a, b):
+async def test_heartbeats(c, s, a, b):
     x = s.workers[a.address].last_seen
     start = time()
-    yield gen.sleep(a.periodic_callbacks["heartbeat"].callback_time / 1000 + 0.1)
+    await asyncio.sleep(a.periodic_callbacks["heartbeat"].callback_time / 1000 + 0.1)
     while s.workers[a.address].last_seen == x:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 2
     assert a.periodic_callbacks["heartbeat"].callback_time < 1000
 
@@ -848,7 +837,7 @@ def test_worker_dir(worker):
     with tmpfile() as fn:
 
         @gen_cluster(client=True, worker_kwargs={"local_directory": fn})
-        def test_worker_dir(c, s, a, b):
+        async def test_worker_dir(c, s, a, b):
             directories = [w.local_directory for w in s.workers.values()]
             assert all(d.startswith(fn) for d in directories)
             assert len(set(directories)) == 2  # distinct
@@ -857,7 +846,7 @@ def test_worker_dir(c, s, a, b):
 
 
 @gen_cluster(client=True)
-def test_dataframe_attribute_error(c, s, a, b):
+async def test_dataframe_attribute_error(c, s, a, b):
     class BadSize:
         def __init__(self, data):
             self.data = data
@@ -866,12 +855,12 @@ def __sizeof__(self):
             raise TypeError("Hello")
 
     future = c.submit(BadSize, 123)
-    result = yield future
+    result = await future
     assert result.data == 123
 
 
 @gen_cluster(client=True)
-def test_fail_write_to_disk(c, s, a, b):
+async def test_fail_write_to_disk(c, s, a, b):
     class Bad:
         def __getstate__(self):
             raise TypeError()
@@ -880,15 +869,15 @@ def __sizeof__(self):
             return int(100e9)
 
     future = c.submit(Bad)
-    yield wait(future)
+    await wait(future)
 
     assert future.status == "error"
 
     with pytest.raises(TypeError):
-        yield future
+        await future
 
     futures = c.map(inc, range(10))
-    results = yield c._gather(futures)
+    results = await c._gather(futures)
     assert results == list(map(inc, range(10)))
 
 
@@ -896,9 +885,9 @@ def __sizeof__(self):
 @gen_cluster(
     nthreads=[("127.0.0.1", 2)], client=True, worker_kwargs={"memory_limit": 10e9}
 )
-def test_fail_write_many_to_disk(c, s, a):
+async def test_fail_write_many_to_disk(c, s, a):
     a.validate = False
-    yield gen.sleep(0.1)
+    await asyncio.sleep(0.1)
     assert not a.paused
 
     class Bad:
@@ -914,23 +903,23 @@ def __sizeof__(self):
     futures = c.map(Bad, range(11))
     future = c.submit(lambda *args: 123, *futures)
 
-    yield wait(future)
+    await wait(future)
 
     with pytest.raises(Exception) as info:
-        yield future
+        await future
 
     # workers still operational
-    result = yield c.submit(inc, 1, workers=a.address)
+    result = await c.submit(inc, 1, workers=a.address)
     assert result == 2
 
 
 @gen_cluster()
-def test_pid(s, a, b):
+async def test_pid(s, a, b):
     assert s.workers[a.address].pid == os.getpid()
 
 
 @gen_cluster(client=True)
-def test_get_client(c, s, a, b):
+async def test_get_client(c, s, a, b):
     def f(x):
         cc = get_client()
         future = cc.submit(inc, x)
@@ -939,7 +928,7 @@ def f(x):
     assert default_client() is c
 
     future = c.submit(f, 10, workers=a.address)
-    result = yield future
+    result = await future
     assert result == 11
 
     assert a._client
@@ -951,7 +940,7 @@ def f(x):
     a_client = a._client
 
     for i in range(10):
-        yield wait(c.submit(f, i))
+        await wait(c.submit(f, i))
 
     assert a._client is a_client
 
@@ -967,32 +956,30 @@ def f(x):
 
 
 @gen_cluster(client=True)
-def test_get_client_coroutine(c, s, a, b):
-    @gen.coroutine
-    def f():
-        client = yield get_client()
+async def test_get_client_coroutine(c, s, a, b):
+    async def f():
+        client = await get_client()
         future = client.submit(inc, 10)
-        result = yield future
-        raise gen.Return(result)
+        result = await future
+        return result
 
-    results = yield c.run(f)
+    results = await c.run(f)
     assert results == {a.address: 11, b.address: 11}
 
 
 def test_get_client_coroutine_sync(client, s, a, b):
-    @gen.coroutine
-    def f():
-        client = yield get_client()
+    async def f():
+        client = await get_client()
         future = client.submit(inc, 10)
-        result = yield future
-        raise gen.Return(result)
+        result = await future
+        return result
 
     results = client.run(f)
     assert results == {a["address"]: 11, b["address"]: 11}
 
 
 @gen_cluster()
-def test_global_workers(s, a, b):
+async def test_global_workers(s, a, b):
     n = len(Worker._instances)
     w = first(Worker._instances)
     assert w is a or w is b
@@ -1000,24 +987,24 @@ def test_global_workers(s, a, b):
 
 @pytest.mark.skipif(WINDOWS, reason="file descriptors")
 @gen_cluster(nthreads=[])
-def test_worker_fds(s):
+async def test_worker_fds(s):
     psutil = pytest.importorskip("psutil")
-    yield gen.sleep(0.05)
+    await asyncio.sleep(0.05)
     start = psutil.Process().num_fds()
 
-    worker = yield Worker(s.address, loop=s.loop)
-    yield gen.sleep(0.1)
+    worker = await Worker(s.address, loop=s.loop)
+    await asyncio.sleep(0.1)
     middle = psutil.Process().num_fds()
     start = time()
     while middle > start:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 1
 
-    yield worker.close()
+    await worker.close()
 
     start = time()
     while psutil.Process().num_fds() > start:
-        yield gen.sleep(0.01)
+        await asyncio.sleep(0.01)
         assert time() < start + 0.5
 
 
@@ -1045,28 +1032,28 @@ async def test_start_services(s):
 
 
 @gen_test()
-def test_scheduler_file():
+async def test_scheduler_file():
     with tmpfile() as fn:
-        s = yield Scheduler(scheduler_file=fn, port=8009)
-        w = yield Worker(scheduler_file=fn)
+        s = await Scheduler(scheduler_file=fn, port=8009)
+        w = await Worker(scheduler_file=fn)
         assert set(s.workers) == {w.address}
-        yield w.close()
+        await w.close()
         s.stop()
 
 
 @gen_cluster(client=True)
-def test_scheduler_delay(c, s, a, b):
+async def test_scheduler_delay(c, s, a, b):
     old = a.scheduler_delay
     assert abs(a.scheduler_delay) < 0.3
     assert abs(b.scheduler_delay) < 0.3
-    yield gen.sleep(a.periodic_callbacks["heartbeat"].callback_time / 1000 + 0.3)
+    await asyncio.sleep(a.periodic_callbacks["heartbeat"].callback_time / 1000 + 0.3)
     assert a.scheduler_delay != old
 
 
 @gen_cluster(client=True)
-def test_statistical_profiling(c, s, a, b):
+async def test_statistical_profiling(c, s, a, b):
     futures = c.map(slowinc, range(10), delay=0.1)
-    yield wait(futures)
+    await wait(futures)
 
     profile = a.profile_keys["slowinc"]
     assert profile["count"]
@@ -1082,14 +1069,14 @@ def test_statistical_profiling(c, s, a, b):
         "distributed.worker.profile.cycle": "100ms",
     },
 )
-def test_statistical_profiling_2(c, s, a, b):
+async def test_statistical_profiling_2(c, s, a, b):
     da = pytest.importorskip("dask.array")
     while True:
         x = da.random.random(1000000, chunks=(10000,))
         y = (x + x * 2) - x.sum().persist()
-        yield wait(y)
+        await wait(y)
 
-        profile = yield a.get_profile()
+        profile = await a.get_profile()
         text = str(profile)
         if profile["count"] and "sum" in text and "random" in text:
             break
@@ -1100,7 +1087,7 @@ def test_statistical_profiling_2(c, s, a, b):
     client=True,
     worker_kwargs={"memory_monitor_interval": 10},
 )
-def test_robust_to_bad_sizeof_estimates(c, s, a):
+async def test_robust_to_bad_sizeof_estimates(c, s, a):
     np = pytest.importorskip("numpy")
     memory = psutil.Process().memory_info().rss
     a.memory_limit = memory / 0.7 + 400e6
@@ -1121,7 +1108,7 @@ def f(n):
 
     start = time()
     while not a.data.disk:
-        yield gen.sleep(0.1)
+        await asyncio.sleep(0.1)
         assert time() < start + 5
 
 
@@ -1142,7 +1129,7 @@ def f(n):
     },
     timeout=20,
 )
-def test_pause_executor(c, s, a):
+async def test_pause_executor(c, s, a):
     memory = psutil.Process().memory_info().rss
     a.memory_limit = memory / 0.5 + 200e6
     np = pytest.importorskip("numpy")
@@ -1157,7 +1144,7 @@ def f():
 
         start = time()
         while not a.paused:
-            yield gen.sleep(0.01)
+            await asyncio.sleep(0.01)
             assert time() < start + 4, (
                 format_bytes(psutil.Process().memory_info().rss),
                 format_bytes(a.memory_limit),
@@ -1169,41 +1156,41 @@ def f():
 
     assert sum(f.status == "finished" for f in futures) < 4
 
-    yield wait(futures)
+    await wait(futures)
 
 
 @gen_cluster(client=True, worker_kwargs={"profile_cycle_interval": "50 ms"})
-def test_statistical_profiling_cycle(c, s, a, b):
+async def test_statistical_profiling_cycle(c, s, a, b):
     futures = c.map(slowinc, range(20), delay=0.05)
-    yield wait(futures)
-    yield gen.sleep(0.01)
+    await wait(futures)
+    await asyncio.sleep(0.01)
     end = time()
     assert len(a.profile_history) > 3
 
-    x = yield a.get_profile(start=time() + 10, stop=time() + 20)
+    x = await a.get_profile(start=time() + 10, stop=time() + 20)
     assert not x["count"]
 
-    x = yield a.get_profile(start=0, stop=time() + 10)
+    x = await a.get_profile(start=0, stop=time() + 10)
     recent = a.profile_recent["count"]
     actual = sum(p["count"] for _, p in a.profile_history) + a.profile_recent["count"]
-    x2 = yield a.get_profile(start=0, stop=time() + 10)
+    x2 = await a.get_profile(start=0, stop=time() + 10)
     assert x["count"] <= actual <= x2["count"]
 
-    y = yield a.get_profile(start=end - 0.300, stop=time())
+    y = await a.get_profile(start=end - 0.300, stop=time())
     assert 0 < y["count"] <= x["count"]
 
 
 @gen_cluster(client=True)
-def test_get_current_task(c, s, a, b):
+async def test_get_current_task(c, s, a, b):
     def some_name():
         return get_worker().get_current_task()
 
-    result = yield c.submit(some_name)
+    result = await c.submit(some_name)
     assert result.startswith("some_name")
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 2)
-def test_reschedule(c, s, a, b):
+async def test_reschedule(c, s, a, b):
     s.extensions["stealing"]._pc.stop()
     a_address = a.address
 
@@ -1214,7 +1201,7 @@ def f(x):
 
     futures = c.map(f, range(4))
     futures2 = c.map(slowinc, range(10), delay=0.1, workers=a.address)
-    yield wait(futures)
+    await wait(futures)
 
     assert all(f.key in b.data for f in futures)
 
@@ -1234,20 +1221,20 @@ async def test_deque_handler(cleanup):
 
 
 @gen_cluster(nthreads=[], client=True)
-def test_avoid_memory_monitor_if_zero_limit(c, s):
-    worker = yield Worker(
+async def test_avoid_memory_monitor_if_zero_limit(c, s):
+    worker = await Worker(
         s.address, loop=s.loop, memory_limit=0, memory_monitor_interval=10
     )
     assert type(worker.data) is dict
     assert "memory" not in worker.periodic_callbacks
 
     future = c.submit(inc, 1)
-    assert (yield future) == 2
-    yield gen.sleep(worker.memory_monitor_interval / 1000)
+    assert (await future) == 2
+    await asyncio.sleep(worker.memory_monitor_interval / 1000)
 
-    yield c.submit(inc, 2)  # worker doesn't pause
+    await c.submit(inc, 2)  # worker doesn't pause
 
-    yield worker.close()
+    await worker.close()
 
 
 @gen_cluster(
@@ -1257,7 +1244,7 @@ def test_avoid_memory_monitor_if_zero_limit(c, s):
         "distributed.worker.memory.target": False,
     },
 )
-def test_dict_data_if_no_spill_to_disk(s, w):
+async def test_dict_data_if_no_spill_to_disk(s, w):
     assert type(w.data) is dict
 
 
@@ -1277,27 +1264,27 @@ def func(dask_scheduler):
 
 
 @gen_cluster(nthreads=[("127.0.0.1", 1)], worker_kwargs={"memory_limit": "2e3 MB"})
-def test_parse_memory_limit(s, w):
+async def test_parse_memory_limit(s, w):
     assert w.memory_limit == 2e9
 
 
 @gen_cluster(nthreads=[], client=True)
-def test_scheduler_address_config(c, s):
+async def test_scheduler_address_config(c, s):
     with dask.config.set({"scheduler-address": s.address}):
-        worker = yield Worker(loop=s.loop)
+        worker = await Worker(loop=s.loop)
         assert worker.scheduler.address == s.address
-    yield worker.close()
+    await worker.close()
 
 
 @pytest.mark.slow
 @gen_cluster(client=True)
-def test_wait_for_outgoing(c, s, a, b):
+async def test_wait_for_outgoing(c, s, a, b):
     np = pytest.importorskip("numpy")
     x = np.random.random(10000000)
-    future = yield c.scatter(x, workers=a.address)
+    future = await c.scatter(x, workers=a.address)
 
     y = c.submit(inc, future, workers=b.address)
-    yield wait(y)
+    await wait(y)
 
     assert len(b.incoming_transfer_log) == len(a.outgoing_transfer_log) == 1
     bb = b.incoming_transfer_log[0]["duration"]
@@ -1313,11 +1300,11 @@ def test_wait_for_outgoing(c, s, a, b):
 @gen_cluster(
     nthreads=[("127.0.0.1", 1), ("127.0.0.1", 1), ("127.0.0.2", 1)], client=True
 )
-def test_prefer_gather_from_local_address(c, s, w1, w2, w3):
-    x = yield c.scatter(123, workers=[w1.address, w3.address], broadcast=True)
+async def test_prefer_gather_from_local_address(c, s, w1, w2, w3):
+    x = await c.scatter(123, workers=[w1.address, w3.address], broadcast=True)
 
     y = c.submit(inc, x, workers=[w2.address])
-    yield wait(y)
+    await wait(y)
 
     assert any(d["who"] == w2.address for d in w1.outgoing_transfer_log)
     assert not any(d["who"] == w2.address for d in w3.outgoing_transfer_log)
@@ -1329,14 +1316,14 @@ def test_prefer_gather_from_local_address(c, s, w1, w2, w3):
     timeout=30,
     config={"distributed.worker.connections.incoming": 1},
 )
-def test_avoid_oversubscription(c, s, *workers):
+async def test_avoid_oversubscription(c, s, *workers):
     np = pytest.importorskip("numpy")
     x = c.submit(np.random.random, 1000000, workers=[workers[0].address])
-    yield wait(x)
+    await wait(x)
 
     futures = [c.submit(len, x, pure=False, workers=[w.address]) for w in workers[1:]]
 
-    yield wait(futures)
+    await wait(futures)
 
     # Original worker not responsible for all transfers
     assert len(workers[0].outgoing_transfer_log) < len(workers) - 2
@@ -1346,13 +1333,13 @@ def test_avoid_oversubscription(c, s, *workers):
 
 
 @gen_cluster(client=True, worker_kwargs={"metrics": {"my_port": lambda w: w.port}})
-def test_custom_metrics(c, s, a, b):
+async def test_custom_metrics(c, s, a, b):
     assert s.workers[a.address].metrics["my_port"] == a.port
     assert s.workers[b.address].metrics["my_port"] == b.port
 
 
 @gen_cluster(client=True)
-def test_register_worker_callbacks(c, s, a, b):
+async def test_register_worker_callbacks(c, s, a, b):
     # preload function to run
     def mystartup(dask_worker):
         dask_worker.init_variable = 1
@@ -1374,81 +1361,81 @@ def test_startup2():
         return os.getenv("MY_ENV_VALUE", None) == "WORKER_ENV_VALUE"
 
     # Nothing has been run yet
-    result = yield c.run(test_import)
+    result = await c.run(test_import)
     assert list(result.values()) == [False] * 2
-    result = yield c.run(test_startup2)
+    result = await c.run(test_startup2)
     assert list(result.values()) == [False] * 2
 
     # Start a worker and check that startup is not run
-    worker = yield Worker(s.address, loop=s.loop)
-    result = yield c.run(test_import, workers=[worker.address])
+    worker = await Worker(s.address, loop=s.loop)
+    result = await c.run(test_import, workers=[worker.address])
     assert list(result.values()) == [False]
-    yield worker.close()
+    await worker.close()
 
     # Add a preload function
-    response = yield c.register_worker_callbacks(setup=mystartup)
+    response = await c.register_worker_callbacks(setup=mystartup)
     assert len(response) == 2
 
     # Check it has been ran on existing worker
-    result = yield c.run(test_import)
+    result = await c.run(test_import)
     assert list(result.values()) == [True] * 2
 
     # Start a worker and check it is ran on it
-    worker = yield Worker(s.address, loop=s.loop)
-    result = yield c.run(test_import, workers=[worker.address])
+    worker = await Worker(s.address, loop=s.loop)
+    result = await c.run(test_import, workers=[worker.address])
     assert list(result.values()) == [True]
-    yield worker.close()
+    await worker.close()
 
     # Register another preload function
-    response = yield c.register_worker_callbacks(setup=mystartup2)
+    response = await c.register_worker_callbacks(setup=mystartup2)
     assert len(response) == 2
 
     # Check it has been run
-    result = yield c.run(test_startup2)
+    result = await c.run(test_startup2)
     assert list(result.values()) == [True] * 2
 
     # Start a worker and check it is ran on it
-    worker = yield Worker(s.address, loop=s.loop)
-    result = yield c.run(test_import, workers=[worker.address])
+    worker = await Worker(s.address, loop=s.loop)
+    result = await c.run(test_import, workers=[worker.address])
     assert list(result.values()) == [True]
-    result = yield c.run(test_startup2, workers=[worker.address])
+    result = await c.run(test_startup2, workers=[worker.address])
     assert list(result.values()) == [True]
-    yield worker.close()
+    await worker.close()
 
 
 @gen_cluster(client=True)
-def test_register_worker_callbacks_err(c, s, a, b):
+async def test_register_worker_callbacks_err(c, s, a, b):
     with pytest.raises(ZeroDivisionError):
-        yield c.register_worker_callbacks(setup=lambda: 1 / 0)
+        await c.register_worker_callbacks(setup=lambda: 1 / 0)
 
 
 @gen_cluster(nthreads=[])
-def test_data_types(s):
-    w = yield Worker(s.address, data=dict)
+async def test_data_types(s):
+    w = await Worker(s.address, data=dict)
     assert isinstance(w.data, dict)
-    yield w.close()
+    await w.close()
 
     data = dict()
-    w = yield Worker(s.address, data=data)
+    w = await Worker(s.address, data=data)
     assert w.data is data
-    yield w.close()
+    await w.close()
 
     class Data(dict):
         def __init__(self, x, y):
             self.x = x
             self.y = y
 
-    w = yield Worker(s.address, data=(Data, {"x": 123, "y": 456}))
+    w = await Worker(s.address, data=(Data, {"x": 123, "y": 456}))
     assert w.data.x == 123
     assert w.data.y == 456
-    yield w.close()
+    await w.close()
 
 
 @gen_cluster(nthreads=[])
-def test_local_directory(s):
+async def test_local_directory(s):
     with tmpfile() as fn:
         with dask.config.set(temporary_directory=fn):
-            w = yield Worker(s.address)
+            w = await Worker(s.address)
             assert w.local_directory.startswith(fn)
             assert "dask-worker-space" in w.local_directory
 
@@ -1457,15 +1444,15 @@ def test_local_directory(s):
     not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
 )
 @gen_cluster(nthreads=[], client=True)
-def test_host_address(c, s):
-    w = yield Worker(s.address, host="127.0.0.2")
+async def test_host_address(c, s):
+    w = await Worker(s.address, host="127.0.0.2")
     assert "127.0.0.2" in w.address
-    yield w.close()
+    await w.close()
 
-    n = yield Nanny(s.address, host="127.0.0.3")
+    n = await Nanny(s.address, host="127.0.0.3")
     assert "127.0.0.3" in n.address
     assert "127.0.0.3" in n.worker_address
-    yield n.close()
+    await n.close()
 
 
 def test_resource_limit(monkeypatch):
@@ -1537,7 +1524,7 @@ async def test_worker_listens_on_same_interface_by_default(Worker):
 async def test_close_gracefully(c, s, a, b):
     futures = c.map(slowinc, range(200), delay=0.1)
     while not b.data:
-        await gen.sleep(0.1)
+        await asyncio.sleep(0.1)
 
     mem = set(b.data)
     proc = set(b.executing)
@@ -1558,7 +1545,7 @@ async def test_lifetime(cleanup):
         async with Worker(s.address) as a, Worker(s.address, lifetime="1 seconds") as b:
             async with Client(s.address, asynchronous=True) as c:
                 futures = c.map(slowinc, range(200), delay=0.1)
-                await gen.sleep(1.5)
+                await asyncio.sleep(1.5)
                 assert b.status != "running"
                 await b.finished()
 
diff --git a/distributed/tests/test_worker_client.py b/distributed/tests/test_worker_client.py
index 14a2d30f7d5..09ae20e8f20 100644
--- a/distributed/tests/test_worker_client.py
+++ b/distributed/tests/test_worker_client.py
@@ -1,3 +1,4 @@
+import asyncio
 import random
 import threading
 from time import sleep
@@ -6,7 +7,6 @@
 import dask
 from dask import delayed
 import pytest
-from tornado import gen
 
 from distributed import (
     worker_client,
@@ -22,7 +22,7 @@
 
 
 @gen_cluster(client=True)
-def test_submit_from_worker(c, s, a, b):
+async def test_submit_from_worker(c, s, a, b):
     def func(x):
         with worker_client() as c:
             x = c.submit(inc, x)
@@ -31,7 +31,7 @@ def func(x):
             return result
 
     x, y = c.map(func, [10, 20])
-    xx, yy = yield c._gather([x, y])
+    xx, yy = await c._gather([x, y])
 
     assert xx == 10 + 1 + (10 + 1) * 2
     assert yy == 20 + 1 + (20 + 1) * 2
@@ -41,7 +41,7 @@ def func(x):
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 2)
-def test_scatter_from_worker(c, s, a, b):
+async def test_scatter_from_worker(c, s, a, b):
     def func():
         with worker_client() as c:
             futures = c.scatter([1, 2, 3, 4, 5])
@@ -56,7 +56,7 @@ def func():
             return total.result()
 
     future = c.submit(func)
-    result = yield future
+    result = await future
     assert result == sum([1, 2, 3, 4, 5])
 
     def func():
@@ -72,17 +72,17 @@ def func():
             return correct
 
     future = c.submit(func)
-    result = yield future
+    result = await future
     assert result is True
 
     start = time()
     while not all(v == 1 for v in s.nthreads.values()):
-        yield gen.sleep(0.1)
+        await asyncio.sleep(0.1)
         assert time() < start + 5
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 2)
-def test_scatter_singleton(c, s, a, b):
+async def test_scatter_singleton(c, s, a, b):
     np = pytest.importorskip("numpy")
 
     def func():
@@ -91,11 +91,11 @@ def func():
             future = c.scatter(x)
             assert future.type == np.ndarray
 
-    yield c.submit(func)
+    await c.submit(func)
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 2)
-def test_gather_multi_machine(c, s, a, b):
+async def test_gather_multi_machine(c, s, a, b):
     a_address = a.address
     b_address = b.address
     assert a_address != b_address
@@ -109,19 +109,19 @@ def func():
         return xx, yy
 
     future = c.submit(func)
-    result = yield future
+    result = await future
 
     assert result == (2, 3)
 
 
 @gen_cluster(client=True)
-def test_same_loop(c, s, a, b):
+async def test_same_loop(c, s, a, b):
     def f():
         with worker_client() as lc:
             return lc.loop is get_worker().loop
 
     future = c.submit(f)
-    result = yield future
+    result = await future
     assert result
 
 
@@ -140,7 +140,7 @@ def mysum():
 
 
 @gen_cluster(client=True)
-def test_async(c, s, a, b):
+async def test_async(c, s, a, b):
     def mysum():
         result = 0
         sub_tasks = [delayed(double)(i) for i in range(100)]
@@ -152,16 +152,16 @@ def mysum():
         return result
 
     future = c.compute(delayed(mysum)())
-    yield future
+    await future
 
     start = time()
     while len(a.data) + len(b.data) > 1:
-        yield gen.sleep(0.1)
+        await asyncio.sleep(0.1)
         assert time() < start + 3
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 3)])
-def test_separate_thread_false(c, s, a):
+async def test_separate_thread_false(c, s, a):
     a.count = 0
 
     def f(i):
@@ -174,19 +174,19 @@ def f(i):
         return i
 
     futures = c.map(f, range(20))
-    results = yield c._gather(futures)
+    results = await c._gather(futures)
     assert list(results) == list(range(20))
 
 
 @gen_cluster(client=True)
-def test_client_executor(c, s, a, b):
+async def test_client_executor(c, s, a, b):
     def mysum():
         with worker_client() as c:
             with c.get_executor() as e:
                 return sum(e.map(double, range(30)))
 
     future = c.submit(mysum)
-    result = yield future
+    result = await future
     assert result == 30 * 29
 
 
@@ -211,7 +211,7 @@ def f(x):
 
 
 @gen_cluster(client=True)
-def test_local_client_warning(c, s, a, b):
+async def test_local_client_warning(c, s, a, b):
     from distributed import local_client
 
     def func(x):
@@ -223,18 +223,18 @@ def func(x):
             return result
 
     future = c.submit(func, 10)
-    result = yield future
+    result = await future
     assert result == 11
 
 
 @gen_cluster(client=True)
-def test_closing_worker_doesnt_close_client(c, s, a, b):
+async def test_closing_worker_doesnt_close_client(c, s, a, b):
     def func(x):
         get_client()
         return
 
-    yield wait(c.map(func, range(10)))
-    yield a.close()
+    await wait(c.map(func, range(10)))
+    await a.close()
     assert c.status == "running"
 
 
@@ -260,15 +260,15 @@ def test_secede_without_stealing_issue_1262():
     # run the loop as an inner function so all workers are closed
     # and exceptions can be examined
     @gen_cluster(client=True, scheduler_kwargs={"extensions": extensions})
-    def secede_test(c, s, a, b):
+    async def secede_test(c, s, a, b):
         def func(x):
             with worker_client() as wc:
                 y = wc.submit(lambda: 1 + x)
                 return wc.gather(y)
 
-        f = yield c.gather(c.submit(func, 1))
+        f = await c.gather(c.submit(func, 1))
 
-        raise gen.Return((c, s, a, b, f))
+        return c, s, a, b, f
 
     c, s, a, b, f = secede_test()
 
@@ -278,40 +278,40 @@ def func(x):
 
 
 @gen_cluster(client=True)
-def test_compute_within_worker_client(c, s, a, b):
+async def test_compute_within_worker_client(c, s, a, b):
     @dask.delayed
     def f():
         with worker_client():
             return dask.delayed(lambda x: x)(1).compute()
 
-    result = yield c.compute(f())
+    result = await c.compute(f())
     assert result == 1
 
 
 @gen_cluster(client=True)
-def test_worker_client_rejoins(c, s, a, b):
+async def test_worker_client_rejoins(c, s, a, b):
     def f():
         with worker_client():
             pass
 
         return threading.current_thread() in get_worker().executor._threads
 
-    result = yield c.submit(f)
+    result = await c.submit(f)
     assert result
 
 
 @gen_cluster()
-def test_submit_different_names(s, a, b):
+async def test_submit_different_names(s, a, b):
     # https://github.com/dask/distributed/issues/2058
     da = pytest.importorskip("dask.array")
-    c = yield Client(
+    c = await Client(
         "localhost:" + s.address.split(":")[-1], loop=s.loop, asynchronous=True
     )
     try:
         X = c.persist(da.random.uniform(size=(100, 10), chunks=50))
-        yield wait(X)
+        await wait(X)
 
-        fut = yield c.submit(lambda x: x.sum().compute(), X)
+        fut = await c.submit(lambda x: x.sum().compute(), X)
         assert fut > 0
     finally:
-        yield c.close()
+        await c.close()
diff --git a/distributed/utils.py b/distributed/utils.py
index adc20d4f368..30af57a25fc 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -202,6 +202,7 @@ def ignoring(*exceptions):
         pass
 
 
+# FIXME: this breaks if changed to async def...
 @gen.coroutine
 def ignore_exceptions(coroutines, *exceptions):
     """ Process list of coroutines, ignoring certain exceptions
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index e1db066b732..e466322eddf 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -1,5 +1,6 @@
 import asyncio
 import collections
+import gc
 from contextlib import contextmanager
 import copy
 import functools
@@ -33,7 +34,6 @@
 
 import dask
 from tlz import merge, memoize, assoc
-from tornado import gen
 from tornado.ioloop import IOLoop
 
 from . import system
@@ -761,18 +761,16 @@ def gen_test(timeout=10):
     """ Coroutine test
 
     @gen_test(timeout=5)
-    def test_foo():
-        yield ...  # use tornado coroutines
+    async def test_foo():
+        await ...  # use tornado coroutines
     """
 
     def _(func):
         def test_func():
             with clean() as loop:
-                if iscoroutinefunction(func):
-                    cor = func
-                else:
-                    cor = gen.coroutine(func)
-                loop.run_sync(cor, timeout=timeout)
+                if not iscoroutinefunction(func):
+                    raise ValueError("@gen_test should wrap async def functions")
+                loop.run_sync(func, timeout=timeout)
 
         return test_func
 
@@ -856,14 +854,15 @@ def gen_cluster(
     active_rpc_timeout=1,
     config={},
     clean_kwargs={},
+    allow_unclosed=False,
 ):
     from distributed import Client
 
     """ Coroutine test with small cluster
 
     @gen_cluster()
-    def test_foo(scheduler, worker1, worker2):
-        yield ...  # use tornado coroutines
+    async def test_foo(scheduler, worker1, worker2):
+        await ...  # use tornado coroutines
 
     See also:
         start
@@ -878,10 +877,10 @@ def test_foo(scheduler, worker1, worker2):
     )
 
     def _(func):
-        if not iscoroutinefunction(func):
-            func = gen.coroutine(func)
-
         def test_func():
+            if not iscoroutinefunction(func):
+                raise ValueError("@gen_cluster should wrap async def functions")
+
             result = None
             workers = []
             with clean(timeout=active_rpc_timeout, **clean_kwargs) as loop:
@@ -905,6 +904,7 @@ async def coro():
                                     "Failed to start gen_cluster, retrying",
                                     exc_info=True,
                                 )
+                                await asyncio.sleep(1)
                             else:
                                 workers[:] = ws
                                 args = [s] + workers
@@ -940,16 +940,28 @@ async def coro():
                         else:
                             await c._close(fast=True)
 
-                        for i in range(5):
-                            if all(c.closed() for c in Comm._instances):
-                                break
-                            else:
+                        def get_unclosed():
+                            return [c for c in Comm._instances if not c.closed()] + [
+                                c
+                                for c in _global_clients.values()
+                                if c.status != "closed"
+                            ]
+
+                        try:
+                            start = time()
+                            while time() < start + 5:
+                                gc.collect()
+                                if not get_unclosed():
+                                    break
                                 await asyncio.sleep(0.05)
-                        else:
-                            L = [c for c in Comm._instances if not c.closed()]
+                            else:
+                                if allow_unclosed:
+                                    print(f"Unclosed Comms: {get_unclosed()}")
+                                else:
+                                    raise RuntimeError("Unclosed Comms", get_unclosed())
+                        finally:
                             Comm._instances.clear()
-                            # raise ValueError("Unclosed Comms", L)
-                            print("Unclosed Comms", L)
+                            _global_clients.clear()
 
                         return result
 
diff --git a/distributed/worker.py b/distributed/worker.py
index a50103bacab..c6734bbce93 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -2310,6 +2310,7 @@ def rescind_key(self, key):
     # Execute Task #
     ################
 
+    # FIXME: this breaks if changed to async def...
     @gen.coroutine
     def executor_submit(self, key, function, args=(), kwargs=None, executor=None):
         """ Safely run function in thread pool executor
diff --git a/docs/source/asynchronous.rst b/docs/source/asynchronous.rst
index a49788e1fbe..342981833db 100644
--- a/docs/source/asynchronous.rst
+++ b/docs/source/asynchronous.rst
@@ -64,18 +64,12 @@ function to run the asynchronous function:
    client.sync(f)
 
 
-Python 2 Compatibility
-----------------------
-
-Everything here works with Python 2 if you replace ``await`` with ``yield``.
-See more extensive comparison in the example below.
-
 Example
 -------
 
 This self-contained example starts an asynchronous client, submits a trivial
-job, waits on the result, and then shuts down the client.  You can see
-implementations for Python 2 and 3 and for Asyncio and Tornado.
+job, waits on the result, and then shuts down the client. You can see
+implementations for Asyncio and Tornado.
 
 Python 3 with Tornado or Asyncio
 ++++++++++++++++++++++++++++++++
@@ -100,25 +94,6 @@ Python 3 with Tornado or Asyncio
    asyncio.get_event_loop().run_until_complete(f())
 
 
-Python 2/3 with Tornado
-+++++++++++++++++++++++
-
-.. code-block:: python
-
-   from dask.distributed import Client
-   from tornado import gen
-
-   @gen.coroutine
-   def f():
-       client = yield Client(asynchronous=True)
-       future = client.submit(lambda x: x + 1, 10)
-       result = yield future
-       yield client.close()
-       raise gen.Return(result)
-
-   from tornado.ioloop import IOLoop
-   IOLoop().run_sync(f)
-
 Use Cases
 ---------
 

From 79546ce43d24924a1b55fdde014b0de290465239 Mon Sep 17 00:00:00 2001
From: Julia Signell <jsignell@gmail.com>
Date: Mon, 20 Apr 2020 11:04:44 -0400
Subject: [PATCH 0798/1550] Fix copy-paste in docs (#3728)

---
 distributed/deploy/spec.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index c6338d3b93f..eb9f0f0043e 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -551,7 +551,7 @@ def adapt(
         minimum_memory : str
             Minimum amount of memory to keep around in the cluster
             Expressed as a string like "100 GiB"
-        maximum_cores : int
+        maximum_memory : str
             Maximum amount of memory to keep around in the cluster
             Expressed as a string like "100 GiB"
 

From 8534e84bba401f61f0339b72db100e637f7b729c Mon Sep 17 00:00:00 2001
From: Julia Signell <jsignell@gmail.com>
Date: Mon, 20 Apr 2020 11:12:23 -0400
Subject: [PATCH 0799/1550] Configurable polling interval for cluster widget
 (#3723)

Customize interval with which to do callbacks to get cluster status for the widget
---
 distributed/client.py         | 8 +++++++-
 distributed/deploy/cluster.py | 7 ++++++-
 distributed/distributed.yaml  | 4 +++-
 3 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 5ba05a84a3b..a48c2367112 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -676,9 +676,15 @@ def __init__(
             heartbeat_interval = dask.config.get("distributed.client.heartbeat")
         heartbeat_interval = parse_timedelta(heartbeat_interval, default="ms")
 
+        scheduler_info_interval = parse_timedelta(
+            dask.config.get("distributed.client.scheduler-info-interval", default="ms")
+        )
+
         self._periodic_callbacks = dict()
         self._periodic_callbacks["scheduler-info"] = PeriodicCallback(
-            self._update_scheduler_info, 2000, io_loop=self.loop
+            self._update_scheduler_info,
+            scheduler_info_interval * 1000,
+            io_loop=self.loop,
         )
         self._periodic_callbacks["heartbeat"] = PeriodicCallback(
             self._heartbeat, heartbeat_interval * 1000, io_loop=self.loop
diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
index 7164b17b076..592195443c1 100644
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -3,6 +3,7 @@
 import threading
 import warnings
 
+import dask.config
 from dask.utils import format_bytes
 
 from .adaptive import Adaptive
@@ -16,6 +17,7 @@
     Logs,
     thread_state,
     format_dashboard_link,
+    parse_timedelta,
 )
 
 
@@ -319,7 +321,10 @@ def scale_cb(b):
         def update():
             status.value = self._widget_status()
 
-        pc = PeriodicCallback(update, 500, io_loop=self.loop)
+        cluster_repr_interval = parse_timedelta(
+            dask.config.get("distributed.deploy.cluster-repr-interval", default="ms")
+        )
+        pc = PeriodicCallback(update, cluster_repr_interval * 1000, io_loop=self.loop)
         self.periodic_callbacks["cluster-repr"] = pc
         pc.start()
 
diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index 71ecd840a10..4f95a179bc3 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -96,10 +96,12 @@ distributed:
     preload-argv: []        # See https://docs.dask.org/en/latest/setup/custom-startup.html
 
   client:
-    heartbeat: 5s  # time between client heartbeats
+    heartbeat: 5s  # Interval between client heartbeats
+    scheduler-info-interval: 2s  # Interval between scheduler-info updates
 
   deploy:
     lost-worker-timeout: 15s  # Interval after which to hard-close a lost worker job
+    cluster-repr-interval: 500ms  # Interval between calls to update cluster-repr for the widget
 
   adaptive:
     interval: 1s         # Interval between scaling evaluations

From 8376f227e288757c3fd9b1e8742ca350e6e57b25 Mon Sep 17 00:00:00 2001
From: Abdulelah Bin Mahfoodh <abduh.bm@gmail.com>
Date: Mon, 20 Apr 2020 22:10:16 +0300
Subject: [PATCH 0800/1550] Add remote_python option in ssh cmd (#3709)

* Add remote_python option in ssh cmd

* Add remote_python option in ssh cmd
---
 distributed/deploy/ssh.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/distributed/deploy/ssh.py b/distributed/deploy/ssh.py
index 4f0e713ffa9..22364ecbacf 100644
--- a/distributed/deploy/ssh.py
+++ b/distributed/deploy/ssh.py
@@ -110,7 +110,7 @@ async def start(self):
         cmd = " ".join(
             [
                 set_env,
-                sys.executable,
+                self.remote_python or sys.executable,
                 "-m",
                 self.worker_module,
                 self.scheduler,
@@ -186,7 +186,12 @@ async def start(self):
                 )
 
         cmd = " ".join(
-            [set_env, sys.executable, "-m", "distributed.cli.dask_scheduler",]
+            [
+                set_env,
+                self.remote_python or sys.executable,
+                "-m",
+                "distributed.cli.dask_scheduler",
+            ]
             + cli_keywords(self.kwargs, cls=_Scheduler)
         )
         self.proc = await self.connection.create_process(cmd)

From 35dc9409f8cf99f82c354b97302846705dfbcc4a Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Mon, 20 Apr 2020 21:18:48 -0500
Subject: [PATCH 0801/1550] Use PeriodicCallback class from tornado (#3725)

---
 distributed/client.py               |  9 +--
 distributed/core.py                 | 15 ++---
 distributed/counter.py              |  4 +-
 distributed/deploy/adaptive_core.py |  4 +-
 distributed/deploy/cluster.py       |  4 +-
 distributed/nanny.py                |  5 +-
 distributed/scheduler.py            |  7 +--
 distributed/semaphore.py            | 13 ++---
 distributed/stealing.py             | 14 ++---
 distributed/utils.py                | 88 ++++++++++++-----------------
 distributed/worker.py               | 33 ++++-------
 11 files changed, 77 insertions(+), 119 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index a48c2367112..52c0e2b420e 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -37,7 +37,7 @@
 except ImportError:
     single_key = first
 from tornado import gen
-from tornado.ioloop import IOLoop
+from tornado.ioloop import IOLoop, PeriodicCallback
 
 from .batched import BatchedSend
 from .utils_comm import (
@@ -72,7 +72,6 @@
     key_split,
     thread_state,
     no_default,
-    PeriodicCallback,
     LoopRunner,
     parse_timedelta,
     shutting_down,
@@ -682,12 +681,10 @@ def __init__(
 
         self._periodic_callbacks = dict()
         self._periodic_callbacks["scheduler-info"] = PeriodicCallback(
-            self._update_scheduler_info,
-            scheduler_info_interval * 1000,
-            io_loop=self.loop,
+            self._update_scheduler_info, scheduler_info_interval * 1000,
         )
         self._periodic_callbacks["heartbeat"] = PeriodicCallback(
-            self._heartbeat, heartbeat_interval * 1000, io_loop=self.loop
+            self._heartbeat, heartbeat_interval * 1000
         )
 
         self._start_arg = address
diff --git a/distributed/core.py b/distributed/core.py
index dd5e18d0007..df0a55780e7 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -12,7 +12,7 @@
 import tblib
 from tlz import merge
 from tornado import gen
-from tornado.ioloop import IOLoop
+from tornado.ioloop import IOLoop, PeriodicCallback
 
 from .comm import (
     connect,
@@ -31,7 +31,6 @@
     truncate_exception,
     ignoring,
     shutting_down,
-    PeriodicCallback,
     parse_timedelta,
     has_keyword,
     CancelledError,
@@ -176,18 +175,14 @@ def stop():
 
         self.periodic_callbacks = dict()
 
-        pc = PeriodicCallback(self.monitor.update, 500, io_loop=self.io_loop)
+        pc = PeriodicCallback(self.monitor.update, 500)
         self.periodic_callbacks["monitor"] = pc
 
         self._last_tick = time()
-        pc = PeriodicCallback(
-            self._measure_tick,
-            parse_timedelta(
-                dask.config.get("distributed.admin.tick.interval"), default="ms"
-            )
-            * 1000,
-            io_loop=self.io_loop,
+        measure_tick_interval = parse_timedelta(
+            dask.config.get("distributed.admin.tick.interval"), default="ms"
         )
+        pc = PeriodicCallback(self._measure_tick, measure_tick_interval * 1000)
         self.periodic_callbacks["tick"] = pc
 
         self.thread_id = 0
diff --git a/distributed/counter.py b/distributed/counter.py
index ebc8cda6104..feffb69ce8c 100644
--- a/distributed/counter.py
+++ b/distributed/counter.py
@@ -1,8 +1,6 @@
 from collections import defaultdict
 
-from tornado.ioloop import IOLoop
-
-from .utils import PeriodicCallback
+from tornado.ioloop import IOLoop, PeriodicCallback
 
 
 try:
diff --git a/distributed/deploy/adaptive_core.py b/distributed/deploy/adaptive_core.py
index 192e244bd08..7d15cb4c2c7 100644
--- a/distributed/deploy/adaptive_core.py
+++ b/distributed/deploy/adaptive_core.py
@@ -1,11 +1,11 @@
 import collections
 import math
 
-from tornado.ioloop import IOLoop
+from tornado.ioloop import IOLoop, PeriodicCallback
 import tlz as toolz
 
 from ..metrics import time
-from ..utils import parse_timedelta, PeriodicCallback
+from ..utils import parse_timedelta
 
 
 class AdaptiveCore:
diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
index 592195443c1..35e0b97c613 100644
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -2,6 +2,7 @@
 import logging
 import threading
 import warnings
+from tornado.ioloop import PeriodicCallback
 
 import dask.config
 from dask.utils import format_bytes
@@ -9,7 +10,6 @@
 from .adaptive import Adaptive
 
 from ..utils import (
-    PeriodicCallback,
     log_errors,
     ignoring,
     sync,
@@ -324,7 +324,7 @@ def update():
         cluster_repr_interval = parse_timedelta(
             dask.config.get("distributed.deploy.cluster-repr-interval", default="ms")
         )
-        pc = PeriodicCallback(update, cluster_repr_interval * 1000, io_loop=self.loop)
+        pc = PeriodicCallback(update, cluster_repr_interval * 1000)
         self.periodic_callbacks["cluster-repr"] = pc
         pc.start()
 
diff --git a/distributed/nanny.py b/distributed/nanny.py
index 3f7c20f98f9..f3a355dca89 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -11,7 +11,7 @@
 
 import dask
 from dask.system import CPU_COUNT
-from tornado.ioloop import IOLoop
+from tornado.ioloop import IOLoop, PeriodicCallback
 from tornado import gen
 
 from .comm import get_address_host, unparse_host_port
@@ -28,7 +28,6 @@
     mp_context,
     silence_logging,
     json_load_robust,
-    PeriodicCallback,
     parse_timedelta,
     ignoring,
     TimeoutError,
@@ -202,7 +201,7 @@ def __init__(
         self.scheduler = self.rpc(self.scheduler_addr)
 
         if self.memory_limit:
-            pc = PeriodicCallback(self.memory_monitor, 100, io_loop=self.loop)
+            pc = PeriodicCallback(self.memory_monitor, 100)
             self.periodic_callbacks["memory"] = pc
 
         if (
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 86dd6b9203e..521415c7c25 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -32,7 +32,7 @@
     groupby,
     concat,
 )
-from tornado.ioloop import IOLoop
+from tornado.ioloop import IOLoop, PeriodicCallback
 
 import dask
 
@@ -64,7 +64,6 @@
     no_default,
     parse_timedelta,
     parse_bytes,
-    PeriodicCallback,
     shutting_down,
     key_split_group,
     empty_context,
@@ -1357,11 +1356,11 @@ def __init__(
         )
 
         if self.worker_ttl:
-            pc = PeriodicCallback(self.check_worker_ttl, self.worker_ttl, io_loop=loop)
+            pc = PeriodicCallback(self.check_worker_ttl, self.worker_ttl)
             self.periodic_callbacks["worker-ttl"] = pc
 
         if self.idle_timeout:
-            pc = PeriodicCallback(self.check_idle, self.idle_timeout / 4, io_loop=loop)
+            pc = PeriodicCallback(self.check_idle, self.idle_timeout / 4)
             self.periodic_callbacks["idle-timeout"] = pc
 
         if extensions is None:
diff --git a/distributed/semaphore.py b/distributed/semaphore.py
index 976f54704c4..263619c9073 100644
--- a/distributed/semaphore.py
+++ b/distributed/semaphore.py
@@ -3,7 +3,8 @@
 import asyncio
 import dask
 from asyncio import TimeoutError
-from .utils import PeriodicCallback, log_errors, parse_timedelta
+from tornado.ioloop import PeriodicCallback
+from .utils import log_errors, parse_timedelta
 from .worker import get_client
 from .metrics import time
 import warnings
@@ -66,14 +67,12 @@ def __init__(self, scheduler):
 
         self.scheduler.extensions["semaphores"] = self
 
-        validation_callback_time = 1000 * parse_timedelta(
+        validation_callback_time = parse_timedelta(
             dask.config.get("distributed.scheduler.locks.lease-validation-interval"),
             default="s",
         )
         self._pc_lease_timeout = PeriodicCallback(
-            self._check_lease_timeout,
-            validation_callback_time,
-            io_loop=self.scheduler.loop,
+            self._check_lease_timeout, validation_callback_time * 1000,
         )
         self._pc_lease_timeout.start()
         self.lease_timeout = parse_timedelta(
@@ -344,9 +343,7 @@ def __init__(self, max_leases=1, name=None, client=None):
         )
         self._refreshing_leases = False
         pc = PeriodicCallback(
-            self._refresh_leases,
-            callback_time=1000 * refresh_leases_interval,
-            io_loop=self.client.io_loop,
+            self._refresh_leases, callback_time=refresh_leases_interval * 1000
         )
         self.refresh_callback = pc
         # Registering the pc to the client here is important for proper cleanup
diff --git a/distributed/stealing.py b/distributed/stealing.py
index 0d552d1689f..874ca98ce77 100644
--- a/distributed/stealing.py
+++ b/distributed/stealing.py
@@ -3,11 +3,13 @@
 from math import log
 from time import time
 
+from tornado.ioloop import PeriodicCallback
+
 import dask
 from .comm.addressing import get_address_host
 from .core import CommClosedError
 from .diagnostics.plugin import SchedulerPlugin
-from .utils import log_errors, parse_timedelta, PeriodicCallback
+from .utils import log_errors, parse_timedelta
 
 from tlz import topk
 
@@ -36,16 +38,12 @@ def __init__(self, scheduler):
         for worker in scheduler.workers:
             self.add_worker(worker=worker)
 
-        # `callback_time` is in milliseconds
-        callback_time = 1000 * parse_timedelta(
+        callback_time = parse_timedelta(
             dask.config.get("distributed.scheduler.work-stealing-interval"),
             default="ms",
         )
-        pc = PeriodicCallback(
-            callback=self.balance,
-            callback_time=callback_time,
-            io_loop=self.scheduler.loop,
-        )
+        # `callback_time` is in milliseconds
+        pc = PeriodicCallback(callback=self.balance, callback_time=callback_time * 1000)
         self._pc = pc
         self.scheduler.periodic_callbacks["stealing"] = pc
         self.scheduler.plugins.append(self)
diff --git a/distributed/utils.py b/distributed/utils.py
index 30af57a25fc..46bd4c245e8 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -48,7 +48,6 @@
 )
 
 import tlz as toolz
-import tornado
 from tornado import gen
 from tornado.ioloop import IOLoop
 
@@ -1118,17 +1117,6 @@ def nbytes(frame, _bytes_like=(bytes, bytearray)):
             return len(frame)
 
 
-def PeriodicCallback(callback, callback_time, io_loop=None):
-    """
-    Wrapper around tornado.IOLoop.PeriodicCallback, for compatibility
-    with removal of the `io_loop` parameter in Tornado 5.0.
-    """
-    if tornado.version_info >= (5,):
-        return tornado.ioloop.PeriodicCallback(callback, callback_time)
-    else:
-        return tornado.ioloop.PeriodicCallback(callback, callback_time, io_loop)
-
-
 @contextmanager
 def time_warn(duration, text):
     start = time()
@@ -1191,49 +1179,47 @@ def reset_logger_locks():
             handler.createLock()
 
 
-if tornado.version_info[0] >= 5:
-
-    is_server_extension = False
+is_server_extension = False
 
-    if "notebook" in sys.modules:
-        import traitlets
-        from notebook.notebookapp import NotebookApp
-
-        is_server_extension = traitlets.config.Application.initialized() and isinstance(
-            traitlets.config.Application.instance(), NotebookApp
-        )
+if "notebook" in sys.modules:
+    import traitlets
+    from notebook.notebookapp import NotebookApp
 
-    if not is_server_extension:
-        is_kernel_and_no_running_loop = False
+    is_server_extension = traitlets.config.Application.initialized() and isinstance(
+        traitlets.config.Application.instance(), NotebookApp
+    )
 
-        if is_kernel():
-            try:
-                get_running_loop()
-            except RuntimeError:
-                is_kernel_and_no_running_loop = True
-
-        if not is_kernel_and_no_running_loop:
-
-            # TODO: Use tornado's AnyThreadEventLoopPolicy, instead of class below,
-            # once tornado > 6.0.3 is available.
-            if WINDOWS and hasattr(asyncio, "WindowsSelectorEventLoopPolicy"):
-                # WindowsProactorEventLoopPolicy is not compatible with tornado 6
-                # fallback to the pre-3.8 default of Selector
-                # https://github.com/tornadoweb/tornado/issues/2608
-                BaseEventLoopPolicy = asyncio.WindowsSelectorEventLoopPolicy
-            else:
-                BaseEventLoopPolicy = asyncio.DefaultEventLoopPolicy
+if not is_server_extension:
+    is_kernel_and_no_running_loop = False
 
-            class AnyThreadEventLoopPolicy(BaseEventLoopPolicy):
-                def get_event_loop(self):
-                    try:
-                        return super().get_event_loop()
-                    except (RuntimeError, AssertionError):
-                        loop = self.new_event_loop()
-                        self.set_event_loop(loop)
-                        return loop
-
-            asyncio.set_event_loop_policy(AnyThreadEventLoopPolicy())
+    if is_kernel():
+        try:
+            get_running_loop()
+        except RuntimeError:
+            is_kernel_and_no_running_loop = True
+
+    if not is_kernel_and_no_running_loop:
+
+        # TODO: Use tornado's AnyThreadEventLoopPolicy, instead of class below,
+        # once tornado > 6.0.3 is available.
+        if WINDOWS and hasattr(asyncio, "WindowsSelectorEventLoopPolicy"):
+            # WindowsProactorEventLoopPolicy is not compatible with tornado 6
+            # fallback to the pre-3.8 default of Selector
+            # https://github.com/tornadoweb/tornado/issues/2608
+            BaseEventLoopPolicy = asyncio.WindowsSelectorEventLoopPolicy
+        else:
+            BaseEventLoopPolicy = asyncio.DefaultEventLoopPolicy
+
+        class AnyThreadEventLoopPolicy(BaseEventLoopPolicy):
+            def get_event_loop(self):
+                try:
+                    return super().get_event_loop()
+                except (RuntimeError, AssertionError):
+                    loop = self.new_event_loop()
+                    self.set_event_loop(loop)
+                    return loop
+
+        asyncio.set_event_loop_policy(AnyThreadEventLoopPolicy())
 
 
 @functools.lru_cache(1000)
diff --git a/distributed/worker.py b/distributed/worker.py
index c6734bbce93..ef95c1f4b7f 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -24,7 +24,7 @@
 
 from tlz import pluck, merge, first, keymap
 from tornado import gen
-from tornado.ioloop import IOLoop
+from tornado.ioloop import IOLoop, PeriodicCallback
 
 from . import profile, comm, system
 from .batched import BatchedSend
@@ -55,7 +55,6 @@
     json_load_robust,
     key_split,
     offload,
-    PeriodicCallback,
     parse_bytes,
     parse_timedelta,
     iscoroutinefunction,
@@ -474,9 +473,6 @@ def __init__(
         self.available_resources = (resources or {}).copy()
         self.death_timeout = parse_timedelta(death_timeout)
 
-        self.memory_monitor_interval = parse_timedelta(
-            memory_monitor_interval, default="ms"
-        )
         self.extensions = dict()
         if silence_logs:
             silence_logging(level=silence_logs)
@@ -659,23 +655,22 @@ def __init__(
             "worker": self,
         }
 
-        pc = PeriodicCallback(self.heartbeat, 1000, io_loop=self.io_loop)
+        pc = PeriodicCallback(self.heartbeat, 1000)
         self.periodic_callbacks["heartbeat"] = pc
         pc = PeriodicCallback(
-            lambda: self.batched_stream.send({"op": "keep-alive"}),
-            60000,
-            io_loop=self.io_loop,
+            lambda: self.batched_stream.send({"op": "keep-alive"}), 60000,
         )
         self.periodic_callbacks["keep-alive"] = pc
 
         self._address = contact_address
 
+        self.memory_monitor_interval = parse_timedelta(
+            memory_monitor_interval, default="ms"
+        )
         if self.memory_limit:
             self._memory_monitoring = False
             pc = PeriodicCallback(
-                self.memory_monitor,
-                self.memory_monitor_interval * 1000,
-                io_loop=self.io_loop,
+                self.memory_monitor, self.memory_monitor_interval * 1000,
             )
             self.periodic_callbacks["memory"] = pc
 
@@ -688,19 +683,13 @@ def __init__(
 
         setproctitle("dask-worker [not started]")
 
-        pc = PeriodicCallback(
-            self.trigger_profile,
-            parse_timedelta(
-                dask.config.get("distributed.worker.profile.interval"), default="ms"
-            )
-            * 1000,
-            io_loop=self.io_loop,
+        profile_trigger_interval = parse_timedelta(
+            dask.config.get("distributed.worker.profile.interval"), default="ms"
         )
+        pc = PeriodicCallback(self.trigger_profile, profile_trigger_interval * 1000)
         self.periodic_callbacks["profile"] = pc
 
-        pc = PeriodicCallback(
-            self.cycle_profile, profile_cycle_interval * 1000, io_loop=self.io_loop
-        )
+        pc = PeriodicCallback(self.cycle_profile, profile_cycle_interval * 1000)
         self.periodic_callbacks["profile-cycle"] = pc
 
         self.plugins = {}

From 83c27fa8dae2cf389b4d8f142721255c9c6ff0e9 Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Tue, 21 Apr 2020 15:05:44 +0100
Subject: [PATCH 0802/1550] Reuse CI scripts for local installation process
 (#3698)

This commit streamlines the CI installation script, with the main purpose of making it easier to replicate on a Linux/MacOSX dev box the exact same environment that exists on CI.

Apply safe defaults for environment variables if the script is not invoked by travis
Do not install miniconda if the 'conda' command is already available
Explicitly state conda channel every time to avoid relying on ~/.condarc
Recommend using conda in developer docs
---
 .github/workflows/ci-windows.yaml             |  4 +-
 .travis.yml                                   |  8 +--
 ...nvironment.yml => environment-windows.yml} | 13 ++---
 continuous_integration/travis/install.sh      | 51 +++++++++++--------
 continuous_integration/travis/run_tests.sh    |  6 +--
 docs/source/develop.rst                       | 27 +++++-----
 6 files changed, 60 insertions(+), 49 deletions(-)
 rename continuous_integration/{environment.yml => environment-windows.yml} (76%)

diff --git a/.github/workflows/ci-windows.yaml b/.github/workflows/ci-windows.yaml
index e0c95d0f234..2e536a79663 100644
--- a/.github/workflows/ci-windows.yaml
+++ b/.github/workflows/ci-windows.yaml
@@ -19,8 +19,8 @@ jobs:
         with:
           miniconda-version: "latest"
           python-version: ${{ matrix.python-version }}
-          environment-file: continuous_integration/environment.yml
-          activate-environment: testenv
+          environment-file: continuous_integration/environment-windows.yml
+          activate-environment: dask-distributed
           auto-activate-base: false
 
       - name: Install tornado
diff --git a/.travis.yml b/.travis.yml
index e8f2afc5057..b7995a2a034 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -6,9 +6,9 @@ dist: trusty
 
 env:
   matrix:
-    - PYTHON=3.6 TESTS=true COVERAGE=true PACKAGES="scikit-learn lz4" TORNADO=5 CRICK=true
-    - PYTHON=3.7 TESTS=true PACKAGES="scikit-learn python-snappy python-blosc" TORNADO=6
-    - PYTHON=3.8 TESTS=true PACKAGES="scikit-learn python-snappy python-blosc" TORNADO=6
+    - PYTHON=3.6 TESTS=true COVERAGE=true PACKAGES="lz4" TORNADO=5 CRICK=true
+    - PYTHON=3.7 TESTS=true PACKAGES="python-snappy python-blosc" TORNADO=6
+    - PYTHON=3.8 TESTS=true PACKAGES="python-snappy python-blosc" TORNADO=6
 
 matrix:
   fast_finish: true
@@ -18,7 +18,7 @@ matrix:
     python: 3.6
     env: LINT=true
   - os: osx
-    env: PYTHON=3.7 TESTS=true PACKAGES="scikit-learn python-snappy python-blosc" TORNADO=6
+    env: PYTHON=3.7 TESTS=true PACKAGES="python-snappy python-blosc" TORNADO=6
     if: type != pull_request OR commit_message =~ test-osx  # Skip on PRs unless the commit message contains "test-osx"
 
   allow_failures:
diff --git a/continuous_integration/environment.yml b/continuous_integration/environment-windows.yml
similarity index 76%
rename from continuous_integration/environment.yml
rename to continuous_integration/environment-windows.yml
index 5f09525caae..2cede561425 100644
--- a/continuous_integration/environment.yml
+++ b/continuous_integration/environment-windows.yml
@@ -1,6 +1,7 @@
-name: testenv
+name: dask-distributed
 channels:
   - conda-forge
+  - defaults
 dependencies:
   - zstandard
   - bokeh!=2.0.0
@@ -16,18 +17,18 @@ dependencies:
   - prometheus_client
   - psutil
   - pytest
+  - pytest-asyncio
+  - pytest-repeat
+  - pytest-timeout
+  - pytest-faulthandler
   - requests
+  - sortedcollections
   - toolz
   - tblib
   - zict
   - fsspec
   - pip
   - pip:
-      - pytest-repeat
-      - pytest-timeout
-      - pytest-faulthandler
-      - sortedcollections
-      - pytest-asyncio
       - git+https://github.com/dask/dask
       - git+https://github.com/joblib/joblib.git
       - git+https://github.com/dask/zict
diff --git a/continuous_integration/travis/install.sh b/continuous_integration/travis/install.sh
index 4ee0790f6c5..e362ea7f079 100644
--- a/continuous_integration/travis/install.sh
+++ b/continuous_integration/travis/install.sh
@@ -5,6 +5,12 @@
 
 # Note we disable progress bars to make Travis log loading much faster
 
+# Set default variable values if unset
+# (useful when this script is not invoked by Travis)
+: ${PYTHON:=3.8}
+: ${TORNADO:=6}
+: ${PACKAGES:=python-snappy python-blosc}
+
 # Install conda
 case "$(uname -s)" in
     'Darwin')
@@ -16,18 +22,17 @@ case "$(uname -s)" in
     *)  ;;
 esac
 
-wget https://repo.continuum.io/miniconda/$MINICONDA_FILENAME -O miniconda.sh
-bash miniconda.sh -b -p $HOME/miniconda
-export PATH="$HOME/miniconda/bin:$PATH"
-conda config --set always_yes yes --set changeps1 no
-conda update -q conda
+if ! which conda; then
+  wget https://repo.continuum.io/miniconda/$MINICONDA_FILENAME -O miniconda.sh
+  bash miniconda.sh -b -p $HOME/miniconda
+  export PATH="$HOME/miniconda/bin:$PATH"
+fi
 
-# Create conda environment
-conda create -q -n test-environment python=$PYTHON
-source activate test-environment
+conda config --set always_yes yes --set quiet yes --set changeps1 no
+conda update conda
 
-# Install dependencies
-conda install -c conda-forge -q \
+# Create conda environment
+conda create -n dask-distributed -c conda-forge -c defaults \
     asyncssh \
     bokeh \
     click \
@@ -39,45 +44,49 @@ conda install -c conda-forge -q \
     ipywidgets \
     joblib \
     jupyter_client \
-    msgpack-python>=0.6.0 \
+    'msgpack-python>=0.6.0' \
     netcdf4 \
     paramiko \
     prometheus_client \
     psutil \
-    pytest>=4 \
+    'pytest>=4' \
+    pytest-asyncio \
+    pytest-faulthandler \
+    pytest-repeat \
     pytest-timeout \
     python=$PYTHON \
     requests \
+    scikit-learn \
     scipy \
-    tblib>=1.5.0 \
+    sortedcollections \
+    'tblib>=1.5.0' \
     toolz \
     tornado=$TORNADO \
     zstandard \
     $PACKAGES
 
+source activate dask-distributed
+
 # stacktrace is not currently avaiable for Python 3.8.
 # Remove the version check block below when it is avaiable.
 if [[ $PYTHON != 3.8 ]]; then
     # For low-level profiler, install libunwind and stacktrace from conda-forge
     # For stacktrace we use --no-deps to avoid upgrade of python
-    conda install -c defaults -c conda-forge libunwind
-    conda install --no-deps -c defaults -c numba -c conda-forge stacktrace
-fi;
-
-python -m pip install -q "pytest>=4" pytest-repeat pytest-faulthandler pytest-asyncio
+    conda install -c conda-forge -c defaults libunwind
+    conda install --no-deps -c conda-forge -c defaults -c numba stacktrace
+fi
 
 python -m pip install -q git+https://github.com/dask/dask.git --upgrade --no-deps
 python -m pip install -q git+https://github.com/joblib/joblib.git --upgrade --no-deps
 python -m pip install -q git+https://github.com/intake/filesystem_spec.git --upgrade --no-deps
 python -m pip install -q git+https://github.com/dask/s3fs.git --upgrade --no-deps
 python -m pip install -q git+https://github.com/dask/zict.git --upgrade --no-deps
-python -m pip install -q sortedcollections --no-deps
 python -m pip install -q keras --upgrade --no-deps
 
 if [[ $CRICK == true ]]; then
-    conda install -q cython
+    conda install -c conda-forge -c defaults cython
     python -m pip install -q git+https://github.com/jcrist/crick.git
-fi;
+fi
 
 # Install distributed
 python -m pip install --no-deps -e .
diff --git a/continuous_integration/travis/run_tests.sh b/continuous_integration/travis/run_tests.sh
index 14c3db7750a..1bf86545cef 100644
--- a/continuous_integration/travis/run_tests.sh
+++ b/continuous_integration/travis/run_tests.sh
@@ -19,7 +19,7 @@ echo "--"
 ulimit -a -H
 
 if [[ $COVERAGE == true ]]; then
-    coverage run $(which py.test) distributed -m "not avoid_travis" $PYTEST_OPTIONS;
+    coverage run $(which py.test) distributed -m "not avoid_travis" $PYTEST_OPTIONS
 else
-    py.test -m "not avoid_travis" distributed $PYTEST_OPTIONS;
-fi;
+    py.test -m "not avoid_travis" distributed $PYTEST_OPTIONS
+fi
diff --git a/docs/source/develop.rst b/docs/source/develop.rst
index 8d0a02fd73d..254eb914aaa 100644
--- a/docs/source/develop.rst
+++ b/docs/source/develop.rst
@@ -12,25 +12,26 @@ guidelines`_ in the main documentation.
 Install
 -------
 
-After setting up an environment as described in the `Dask developer
-guidelines`_ you can clone this repository with git::
+Clone this repository with git::
 
    git clone git@github.com:dask/distributed.git
+   cd distributed
 
-and install it from source::
+Install all dependencies:
 
-   cd distributed
-   python setup.py install
+On Linux / MacOSX::
 
-Using conda, for example::
+    source continuous_integration/travis/install.sh
 
-   git clone git@github.com:{your-fork}/distributed.git
-   cd distributed
-   conda create -y -n distributed python=3.6
-   conda activate distributed
-   python -m pip install -U -r requirements.txt
-   python -m pip install -U -r dev-requirements.txt
-   python -m pip install -e .
+On Windows:
+
+1. Install anaconda or miniconda
+2. ::
+
+    conda create -n dask-distributed -c conda-forge -c defaults python=3.8 tornado=6
+    conda activate dask-distributed
+    conda env update --file continuous_integration/environment-windows.yml
+    python -m pip install .
 
 To keep a fork in sync with the upstream source::
 

From 9cfd06685fa889e9b4705e9943cbc4c4a0fd6643 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 21 Apr 2020 10:34:02 -0700
Subject: [PATCH 0803/1550] Add Configuration Schema (#3696)

* Add Configuration Schema

This adds type and description information to the configuration using
the jsonschema spec.

So far this is only a proof of concept, and touches only a couple of
entries.

* Add schema test to CI

* Try using sphinx-jsonschema

* Add more configuration descriptions

* add more descriptions

* add a bunch more descriptions

* use multi-line text blocks

* add more descriptions

* Test completeness of schema

* add new config value

* Revert changes adding a docpage

* Add informative error when config and schema are out of sync
---
 distributed/distributed-schema.yaml | 796 ++++++++++++++++++++++++++++
 distributed/distributed.yaml        |   2 +-
 distributed/scheduler.py            |   4 +-
 distributed/tests/test_config.py    |  48 ++
 4 files changed, 848 insertions(+), 2 deletions(-)
 create mode 100644 distributed/distributed-schema.yaml

diff --git a/distributed/distributed-schema.yaml b/distributed/distributed-schema.yaml
new file mode 100644
index 00000000000..60dfbf54e05
--- /dev/null
+++ b/distributed/distributed-schema.yaml
@@ -0,0 +1,796 @@
+properties:
+  distributed:
+    type: object
+    properties:
+
+      version:
+        type: integer
+
+      scheduler:
+        type: object
+        properties:
+
+          allowed-failures:
+            type: integer
+            minimum: 0
+            description: |
+              The number of retries before a task is considered bad
+
+              When a worker dies when a task is running that task is rerun elsewhere.
+              If many workers die while running this same task then we call the task bad, and raise a KilledWorker exception.
+              This is the number of workers that are allowed to die before this task is marked as bad.
+
+          bandwidth:
+            type:
+            - integer
+            - string
+            description: |
+              The expected bandwidth between any pair of workers
+
+              This is used when making scheduling decisions.
+              The scheduler will use this value as a baseline, but also learn it over time.
+
+          blocked-handlers:
+            type: array
+            description: |
+              A list of handlers to exclude
+
+              The scheduler operates by receiving messages from various workers and clients
+              and then performing operations based on those messages.
+              Each message has an operation like "close-worker" or "task-finished".
+              In some high security situations administrators may choose to block certain handlers
+              from running.  Those handlers can be listed here.
+
+              For a list of handlers see the `dask.distributed.Scheduler.handlers` attribute.
+
+          default-data-size:
+            type:
+            - string
+            - integer
+            description: |
+              The default size of a piece of data if we don't know anything about it.
+
+              This is used by the scheduler in some scheduling decisions
+
+          events-cleanup-delay:
+            type: string
+            description: |
+              The amount of time to wait until workers or clients are removed from the event log
+              after they have been removed from the scheduler
+
+          idle-timeout:
+            type:
+            - string
+            - "null"
+            description: |
+              Shut down the scheduler after this duration if no activity has occured
+
+              This can be helpful to reduce costs and stop zombie processes from roaming the earth.
+
+          transition-log-length:
+            type: integer
+            minimum: 0
+            description: |
+              How long should we keep the transition log
+
+              Every time a task transitions states (like "waiting", "processing", "memory", "released")
+              we record that transition in a log.
+
+              To make sure that we don't run out of memory
+              we will clear out old entries after a certain length.
+              This is that length.
+
+          work-stealing:
+            type: boolean
+            description: |
+              Whether or not to balance work between workers dynamically
+
+              Some times one worker has more work than we expected.
+              The scheduler will move these tasks around as necessary by default.
+              Set this to false to disable this behavior
+
+          work-stealing-interval:
+            type: string
+            description: |
+              How frequently to balance worker loads
+
+          worker-ttl:
+            type:
+            - string
+            - "null"
+            description: |
+              Time to live for workers.
+
+              If we don't receive a heartbeat faster than this then we assume that the worker has died.
+
+          pickle:
+            type: boolean
+            description: |
+              Is the scheduler allowed to deserialize arbitrary bytestrings?
+
+              The scheduler almost never deserializes user data.
+              However there are some cases where the user can submit functions to run directly on the scheduler.
+              This can be convenient for debugging, but also introduces some security risk.
+              By setting this to false we ensure that the user is unable to run arbitrary code on the scheduler.
+
+          preload:
+            type: array
+            description: |
+              Run custom modules during the lifetime of the scheduler
+
+              You can run custom modules when the scheduler starts up and closes down.
+              See https://docs.dask.org/en/latest/setup/custom-startup.html for more information
+
+          preload-argv:
+            type: array
+            description: |
+              Arguments to pass into the preload scripts described above
+
+              See https://docs.dask.org/en/latest/setup/custom-startup.html for more information
+
+          unknown-task-duration:
+            type: string
+            description: |
+              Default duration for all tasks with unknown durations
+
+              Over time the scheduler learns a duration for tasks.
+              However when it sees a new type of task for the first time it has to make a guess
+              as to how long it will take.  This value is that guess.
+
+          default-task-durations:
+            type: object
+            description: |
+              How long we expect function names to run
+
+              Over time the scheduler will learn these values, but these give it a good starting point.
+
+          validate:
+            type: boolean
+            description: |
+              Whether or not to run consistency checks during execution.
+              This is typically only used for debugging.
+
+          dashboard:
+            type: object
+            description: |
+              Configuration options for Dask's real-time dashboard
+
+            properties:
+              status:
+                type: object
+                description: The main status page of the dashboard
+                properties:
+                  task-stream-length:
+                    type: integer
+                    minimum: 0
+                    description: |
+                      The maximum number of tasks to include in the task stream plot
+              tasks:
+                type: object
+                description: |
+                  The page which includes the full task stream history
+                properties:
+                  task-stream-length:
+                    type: integer
+                    minimum: 0
+                    description: |
+                      The maximum number of tasks to include in the task stream plot
+              tls:
+                type: object
+                description: |
+                  Settings around securing the dashboard
+                properties:
+                  ca-file:
+                    type:
+                    - string
+                    - "null"
+                  key:
+                    type:
+                    - string
+                    - "null"
+                  cert:
+                    type:
+                    - string
+                    - "null"
+              bokeh-application:
+                type: object
+                description: |
+                  Keywords to pass to the BokehTornado application
+          locks:
+            type: object
+            description: |
+              Settings for Dask's distributed Lock object
+
+              See https://docs.dask.org/en/latest/futures.html#locks for more information
+            properties:
+              lease-validation-interval:
+                type: string
+                description: |
+                  The time to wait until an acquired semaphore is released if the Client goes out of scope
+              lease-timeout:
+                type: string
+                description: |
+                  The timeout after which a lease will be released if not refreshed
+
+          http:
+            type: object
+            decription: Settings for Dask's embedded HTTP Server
+            properties:
+              routes:
+                type: array
+                description: |
+                  A list of modules like "prometheus" and "health" that can be included or excluded as desired
+
+                  These modules will have a ``routes`` keyword that gets added to the main HTTP Server.
+                  This is also a list that can be extended with user defined modules.
+
+
+      worker:
+        type: object
+        description: |
+          Configuration settings for Dask Workers
+        properties:
+          blocked-handlers:
+            type: array
+            description: |
+              A list of handlers to exclude
+
+              The scheduler operates by receiving messages from various workers and clients
+              and then performing operations based on those messages.
+              Each message has an operation like "close-worker" or "task-finished".
+              In some high security situations administrators may choose to block certain handlers
+              from running.  Those handlers can be listed here.
+
+              For a list of handlers see the `dask.distributed.Scheduler.handlers` attribute.
+
+          multiprocessing-method:
+            type: string
+            description: |
+              How we create new workers, one of "spawn", "forkserver", or "fork"
+
+              This is passed to the ``multiprocessing.get_context`` function.
+          use-file-locking:
+            type: boolean
+            description: |
+              Whether or not to use lock files when creating workers
+
+              Workers create a local directory in which to place temporary files.
+              When many workers are created on the same process at once
+              these workers can conflict with each other by trying to create this directory all at the same time.
+
+              To avoid this, Dask usually used a file-based lock.
+              However, on some systems file-based locks don't work.
+              This is particularly common on HPC NFS systems, where users may want to set this to false.
+          connections:
+            type: object
+            description: |
+              The number of concurrent connections to allow to other workers
+            properties:
+              incoming:
+                type: integer
+                minimum: 0
+              outgoing:
+                type: integer
+                minimum: 0
+
+          preload:
+            type: array
+            description: |
+              Run custom modules during the lifetime of the worker
+
+              You can run custom modules when the worker starts up and closes down.
+              See https://docs.dask.org/en/latest/setup/custom-startup.html for more information
+
+          preload-argv:
+            type: array
+            description: |
+              Arguments to pass into the preload scripts described above
+
+              See https://docs.dask.org/en/latest/setup/custom-startup.html for more information
+
+          daemon:
+            type: boolean
+            description: |
+              Whether or not to run our process as a daemon process
+
+          validate:
+            type: boolean
+            description: |
+              Whether or not to run consistency checks during execution.
+              This is typically only used for debugging.
+
+          lifetime:
+            type: object
+            description: |
+              The worker may choose to gracefully close itself down after some pre-determined time.
+
+              This is particularly useful if you know that your worker job has a time limit on it.
+              This is particularly common in HPC job schedulers.
+
+              For example if your worker has a walltime of one hour,
+              then you may want to set the lifetime.duration to "55 minutes"
+            properties:
+              duration:
+                type:
+                - string
+                - "null"
+                description: |
+                  The time after creation to close the worker, like "1 hour"
+              stagger:
+                type: string
+                description: |
+                  Random amount by which to stagger lifetimes
+
+                  If you create many workers at the same time,
+                  you may want to avoid having them kill themselves all at the same time.
+                  To avoid this you might want to set a stagger time,
+                  so that they close themselves with some random variation, like "5 minutes"
+
+                  That way some workers can die, new ones can be brought up,
+                  and data can be transferred over smoothly.
+              restart:
+                type: boolean
+                description: |
+                  Do we try to resurrect the worker after the lifetime deadline?
+
+
+          profile:
+            type: object
+            description: |
+              The workers periodically poll every worker thread to see what they are working on.
+              This data gets collected into statistical profiling information,
+              which is then periodically bundled together and sent along to the scheduler.
+            properties:
+              interval:
+                type: string
+                description: |
+                  The time between polling the worker threads, typically short like 10ms
+              cycle:
+                type: string
+                description: |
+                  The time between bundling together this data and sending it to the scheduler
+
+                  This controls the granularity at which people can query the profile information
+                  on the time axis.
+              low-level:
+                type: boolean
+                description: |
+                  Whether or not to use the libunwind and stacktrace libraries
+                  to gather profiling information at the lower level (beneath Python)
+
+                  To get this to work you will need to install the experimental stacktrace library at
+
+                  conda install -c numba stacktrace
+
+                  See https://github.com/numba/stacktrace
+
+          memory:
+            type: object
+            description: |
+              When Dask workers have more data than memory they spill this data to disk.
+              They do this at a few conditions.
+            properties:
+              target:
+                type: number
+                minimum: 0
+                maximum: 1
+                description: |
+                  Target fraction below which to try to keep memory
+
+              spill:
+                type: number
+                minimum: 0
+                maximum: 1
+                description: |
+                  When the process memory (as observed by the operating system) gets above this amount we spill data to disk.
+
+              pause:
+                type: number
+                minimum: 0
+                maximum: 1
+                description: |
+                  When the process memory (as observed by the operating system) gets above this amount
+                  we no longer start new tasks on this worker.
+
+              terminate:
+                type: number
+                minimum: 0
+                maximum: 1
+                description: |
+                  When the process memory reaches this level the nanny process will kill the worker
+                  (if a nanny is present)
+
+          http:
+            type: object
+            decription: Settings for Dask's embedded HTTP Server
+            properties:
+              routes:
+                type: array
+                description: |
+                  A list of modules like "prometheus" and "health" that can be included or excluded as desired
+
+                  These modules will have a ``routes`` keyword that gets added to the main HTTP Server.
+                  This is also a list that can be extended with user defined modules.
+          http:
+            type: object
+            decription: Settings for Dask's embedded HTTP Server
+            properties:
+              routes:
+                type: array
+                description: |
+                  A list of modules like "prometheus" and "health" that can be included or excluded as desired
+
+                  These modules will have a ``routes`` keyword that gets added to the main HTTP Server.
+                  This is also a list that can be extended with user defined modules.
+
+      nanny:
+        type: object
+        description: |
+          Configuration settings for Dask Nannies
+        properties:
+
+          preload:
+            type: array
+            description: |
+              Run custom modules during the lifetime of the scheduler
+
+              You can run custom modules when the scheduler starts up and closes down.
+              See https://docs.dask.org/en/latest/setup/custom-startup.html for more information
+
+          preload-argv:
+            type: array
+            description: |
+              Arguments to pass into the preload scripts described above
+
+              See https://docs.dask.org/en/latest/setup/custom-startup.html for more information
+
+      client:
+        type: object
+        description: |
+          Configuration settings for Dask Clients
+
+        properties:
+          heartbeat:
+            type: string
+            description:
+              This value is the time between heartbeats
+
+              The client sends a periodic heartbeat message to the scheduler.
+              If it misses enough of these then the scheduler assumes that it has gone.
+
+          scheduler-info-interval:
+            type: string
+            description: Interval between scheduler-info updates
+
+      deploy:
+        type: object
+        description: Configuration settings for general Dask deployment
+        properties:
+          lost-worker-timeout:
+            type: string
+            description: |
+              Interval after which to hard-close a lost worker job
+
+              Otherwise we wait for a while to see if a worker will reappear
+
+          cluster-repr-interval:
+            type: string
+            description: Interval between calls to update cluster-repr for the widget
+
+      adaptive:
+        type: object
+        description: Configuration settings for Dask's adaptive scheduling
+        properties:
+          interval:
+            type: string
+            description: |
+              The duration between checking in with adaptive scheduling load
+
+              The adaptive system periodically checks scheduler load and determines
+              if it should scale the cluster up or down.
+              This is the timing between those checks.
+
+          target-duration:
+            type: string
+            description: |
+              The desired time for the entire computation to run
+
+              The adaptive system will try to start up enough workers to run
+              the computation in about this time.
+
+          minimum:
+            type: integer
+            minimum: 0
+            description: |
+              The minimum number of workers to keep around
+
+          maximum:
+            type: number
+            minimum: 0
+            description: |
+              The maximum number of workers to keep around
+
+          wait-count:
+            type: integer
+            minimum: 1
+            description: |
+              The number of times a worker should be suggested for removal before removing it
+
+              This helps to smooth out the number of deployed workers
+
+      comm:
+        type: object
+        description: Configuration settings for Dask communications
+        properties:
+
+          retry:
+            type: object
+            description: |
+              Some operations (such as gathering data) are subject to re-tries with the below parameters
+            properties:
+
+              count:
+                type: integer
+                minimum: 0
+                description: |
+                  The number of times to retry a connection
+
+              delay:
+                type: object
+                properties:
+                  min:
+                    type: string
+                    description: The first non-zero delay between retry attempts
+                  max:
+                    type: string
+                    description: The maximum delay between retries
+
+          compression:
+            type: string
+            description: |
+              The compression algorithm to use
+
+              This could be one of lz4, snappy, zstd, or blosc
+
+          offload:
+            type:
+            - boolean
+            - string
+            description: |
+              The size of message after which we choose to offload serialization to another thread
+
+              In some cases, you may also choose to disable this altogether with the value false
+              This is useful if you want to include serialization in profiling data,
+              or if you have data types that are particularly sensitive to deserialization
+
+          socket-backlog:
+            type: integer
+            description: |
+              When shuffling data between workers, there can
+              really be O(cluster size) connection requests
+              on a single worker socket, make sure the backlog
+              is large enough not to lose any.
+
+          zstd:
+            type: object
+            description: Options for the Z Standard compression scheme
+            properties:
+              level:
+                type: integer
+                minimum: 1
+                maximum: 22
+                description: Compression level, between 1 and 22.
+              threads:
+                type: integer
+                minimum: -1
+                description: |
+                  Number of threads to use.
+
+                  0 for single-threaded, -1 to infer from cpu count.
+
+          timeouts:
+            type: object
+            properties:
+              connect:
+                type: string
+              tcp:
+                type: string
+
+          require-encryption:
+            type: boolean
+            description: |
+              Whether to require encryption on non-local comms
+
+          default-scheme:
+            type: string
+            description: The default protocol to use, like tcp or tls
+
+          recent-messages-log-length:
+            type: integer
+            minimum: 0
+            description: number of messages to keep for debugging
+
+          tls:
+            type: object
+            properties:
+              ciphers:
+                type:
+                - string
+                - "null"
+                descsription: Allowed ciphers, specified as an OpenSSL cipher string.
+
+              ca-file:
+                type:
+                - string
+                - "null"
+                description: Path to a CA file, in pem format
+
+              scheduler:
+                type: object
+                description: TLS information for the scheduler
+                properties:
+                  cert:
+                    type:
+                    - string
+                    - "null"
+                    description: Path to certificate file
+                  key:
+                    type:
+                    - string
+                    - "null"
+                    description: |
+                      Path to key file.
+
+                      Alternatively, the key can be appended to the cert file
+                      above, and this field left blank
+
+              worker:
+                type: object
+                description: TLS information for the worker
+                properties:
+                  cert:
+                    type:
+                    - string
+                    - "null"
+                    description: Path to certificate file
+                  key:
+                    type:
+                    - string
+                    - "null"
+                    description: |
+                      Path to key file.
+
+                      Alternatively, the key can be appended to the cert file
+                      above, and this field left blank
+
+              client:
+                type: object
+                description: TLS information for the client
+                properties:
+                  cert:
+                    type:
+                    - string
+                    - "null"
+                    description: Path to certificate file
+                  key:
+                    type:
+                    - string
+                    - "null"
+                    description: |
+                      Path to key file.
+
+                      Alternatively, the key can be appended to the cert file
+                      above, and this field left blank
+
+      dashboard:
+        type: object
+        properties:
+          link:
+            type: string
+            description: |
+              The form for the dashboard links
+
+              This is used wherever we print out the link for the dashboard
+              It is filled in with relevant information like the schema, host, and port number
+          graph-max-items:
+            type: integer
+            minimum: 0
+            description: maximum number of tasks to try to plot in "graph" view
+
+          export-tool:
+            type: boolean
+
+      admin:
+        type: object
+        description: |
+          Options for logs, event loops, and so on
+        properties:
+          tick:
+            type: object
+            description: |
+              Time between event loop health checks
+
+              We set up a periodic callback to run on the event loop and check in fairly frequently.
+              (by default, this is every 20 milliseconds)
+
+              If this periodic callback sees that the last time it checked in was several seconds ago
+              (by default, this is 3 seconds)
+              then it logs a warning saying that something has been stopping the event loop from smooth operation.
+              This is typically caused by GIL holding operations,
+              but could also be several other things.
+
+            properties:
+              interval:
+                type: string
+                description: The time between ticks, default 20ms
+              limit :
+                type: string
+                description: The time allowed before triggering a warning
+
+          max-error-length:
+            type: integer
+            minimum: 0
+            description: |
+              Maximum length of traceback as text
+
+              Some Python tracebacks can be very very long
+              (particularly in stack overflow errors)
+
+              If the traceback is larger than this size (in bytes) then we truncate it.
+
+          log-length:
+            type: integer
+            minimum: 0
+            description: |
+              Default length of logs to keep in memory
+
+              The scheduler and workers keep the last 10000 or so log entries in memory.
+
+          log-format:
+            type: string
+            description: |
+              The log format to emit.
+
+              See https://docs.python.org/3/library/logging.html#logrecord-attributes
+
+          pdb-on-err:
+            type: boolean
+            description: Enter Python Debugger on scheduling error
+
+  rmm:
+    type: object
+    description: |
+      Configuration options for the RAPIDS Memory Manager
+    properties:
+      pool-size:
+        type:
+        - integer
+        - "null"
+        description:
+          The size of the memory pool in bytes
+  ucx:
+    type: object
+    description: |
+      UCX provides access to other network interconnects like Infiniband and NVLINK
+    properties:
+      tcp:
+        type:
+        - boolean
+        - "null"
+      nvlink:
+        type:
+        - boolean
+        - "null"
+      infiniband:
+        type:
+        - boolean
+        - "null"
+      cuda_copy:
+        type:
+        - boolean
+        - "null"
+      net-devices:
+        type:
+        - string
+        - "null"
+        description: Define which Infiniband device to use
diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index 4f95a179bc3..4103d592e2b 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -12,7 +12,7 @@ distributed:
     allowed-failures: 3     # number of retries before a task is considered bad
     bandwidth: 100000000    # 100 MB/s estimated worker-worker bandwidth
     blocked-handlers: []
-    default-data-size: 1000
+    default-data-size: 1kiB
     # Number of seconds to wait until workers or clients are removed from the events log
     # after they have been removed from the scheduler
     events-cleanup-delay: 1h
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 521415c7c25..82e5f812dd6 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -90,7 +90,9 @@
 
 
 LOG_PDB = dask.config.get("distributed.admin.pdb-on-err")
-DEFAULT_DATA_SIZE = dask.config.get("distributed.scheduler.default-data-size")
+DEFAULT_DATA_SIZE = parse_bytes(
+    dask.config.get("distributed.scheduler.default-data-size")
+)
 
 DEFAULT_EXTENSIONS = [
     LockExtension,
diff --git a/distributed/tests/test_config.py b/distributed/tests/test_config.py
index 2017bb239f7..01cd6eec57b 100644
--- a/distributed/tests/test_config.py
+++ b/distributed/tests/test_config.py
@@ -3,6 +3,7 @@
 import sys
 import tempfile
 import os
+import yaml
 
 import pytest
 
@@ -265,3 +266,50 @@ def test_logging_file_config():
             """
         subprocess.check_call([sys.executable, "-c", code])
     os.remove(logging_config.name)
+
+
+def test_schema():
+    jsonschema = pytest.importorskip("jsonschema")
+    config_fn = os.path.join(os.path.dirname(__file__), "..", "distributed.yaml")
+    schema_fn = os.path.join(os.path.dirname(__file__), "..", "distributed-schema.yaml")
+
+    with open(config_fn) as f:
+        config = yaml.safe_load(f)
+
+    with open(schema_fn) as f:
+        schema = yaml.safe_load(f)
+
+    jsonschema.validate(config, schema)
+
+
+def test_schema_is_complete():
+    config_fn = os.path.join(os.path.dirname(__file__), "..", "distributed.yaml")
+    schema_fn = os.path.join(os.path.dirname(__file__), "..", "distributed-schema.yaml")
+
+    with open(config_fn) as f:
+        config = yaml.safe_load(f)
+
+    with open(schema_fn) as f:
+        schema = yaml.safe_load(f)
+
+    skip = {"default-task-durations", "bokeh-application"}
+
+    def test_matches(c, s):
+        if set(c) != set(s["properties"]):
+            raise ValueError(
+                "\nThe distributed.yaml and distributed-schema.yaml files are not in sync.\n"
+                "This usually happens when we add a new configuration value,\n"
+                "but don't add the schema of that value to the distributed-schema.yaml file\n"
+                "Please modify these files to include the missing values: \n\n"
+                "    distributed.yaml:        {}\n"
+                "    distributed-schema.yaml: {}\n\n"
+                "Examples in these files should be a good start, \n"
+                "even if you are not familiar with the jsonschema spec".format(
+                    sorted(c), sorted(s["properties"])
+                )
+            )
+        for k, v in c.items():
+            if isinstance(v, dict) and k not in skip:
+                test_matches(c[k], s["properties"][k])
+
+    test_matches(config, schema)

From 6db09f32d8ca58a56a50385a49150ef16a5d51b0 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Tue, 21 Apr 2020 16:35:34 -0700
Subject: [PATCH 0804/1550] Relax NumPy requirement in UCX (#3731)

* Make `device_array`'s shape a `tuple`

While it works to have this be a single `int` (as it will be coerced to
a `tuple`), go ahead and make it a `tuple` for clarity and to match more
closely to the Numba case.

* Use `"u1"` to specify `uint8` typed arrays

This is equivalent to using NumPy's `uint8`, but has the added benefit
of not requiring NumPy be imported to work.

* Rename `is_cudas` to `cuda_frames`

Matches the variable name in the `send` case to make things easier to
follow.

* Use `pack`/`unpack` for UCX frame metadata

As `struct.pack` and `struct.unpack` are able to build `bytes` objects
containing the frame metadata needed by UCX easily, just use these
functions instead of creating NumPy arrays each time. Helps soften the
NumPy requirement a bit.

* Rename `cuda_array` to `device_array`

Matches more closely to the name used by RMM and Numba.

* Create function to allocate arrays on host

To relax the NumPy requirement completely, add a function to allocate
arrays on host. If NumPy is not present, this falls back to just
allocating `bytearray` objects, which work just as well.

* Fix formatting with black

* Define `cuda_frames` with other frame definitions

* Store `nframes` for simplicity

Avoids multiple calls to `len(frames)`, is a bit easier to read, and
matches the receive code path more closely.

* Collect sizes along with other frame info

* Use `sizes` to pick out non-trivial frames to send

* Simply call `sum` on `sizes` for bytes sent

* Use `host_array` to make buffers to receive into

* Pack per frame metadata into one message

To send fewer and larger messages, pack both which frames are on device
and how large each frame is into one message.

* Note what `struct` lines are packing/unpacking
---
 distributed/comm/ucx.py | 83 ++++++++++++++++++++++++++---------------
 1 file changed, 52 insertions(+), 31 deletions(-)

diff --git a/distributed/comm/ucx.py b/distributed/comm/ucx.py
index 4e6ca8116c8..7761afef7a1 100644
--- a/distributed/comm/ucx.py
+++ b/distributed/comm/ucx.py
@@ -6,10 +6,10 @@
 .. _UCX: https://github.com/openucx/ucx
 """
 import logging
+import struct
 import weakref
 
 import dask
-import numpy as np
 
 from .addressing import parse_host_port, unparse_host_port
 from .core import Comm, Connector, Listener, CommClosedError
@@ -33,7 +33,8 @@
 # required to ensure Dask configuration gets propagated to UCX, which needs
 # variables to be set before being imported.
 ucp = None
-cuda_array = None
+host_array = None
+device_array = None
 
 
 def synchronize_stream(stream=0):
@@ -46,7 +47,7 @@ def synchronize_stream(stream=0):
 
 
 def init_once():
-    global ucp, cuda_array
+    global ucp, host_array, device_array
     if ucp is not None:
         return
 
@@ -59,34 +60,42 @@ def init_once():
 
     ucp.init(options=ucx_config, env_takes_precedence=True)
 
+    # Find the function, `host_array()`, to use when allocating new host arrays
+    try:
+        import numpy
+
+        host_array = lambda n: numpy.empty((n,), dtype="u1")
+    except ImportError:
+        host_array = lambda n: bytearray(n)
+
     # Find the function, `cuda_array()`, to use when allocating new CUDA arrays
     try:
         import rmm
 
         if hasattr(rmm, "DeviceBuffer"):
-            cuda_array = lambda n: rmm.DeviceBuffer(size=n)
+            device_array = lambda n: rmm.DeviceBuffer(size=n)
         else:  # pre-0.11.0
             import numba.cuda
 
-            def rmm_cuda_array(n):
-                a = rmm.device_array(n, dtype=np.uint8)
+            def rmm_device_array(n):
+                a = rmm.device_array(n, dtype="u1")
                 weakref.finalize(a, numba.cuda.current_context)
                 return a
 
-            cuda_array = rmm_cuda_array
+            device_array = rmm_device_array
     except ImportError:
         try:
             import numba.cuda
 
-            def numba_cuda_array(n):
-                a = numba.cuda.device_array((n,), dtype=np.uint8)
+            def numba_device_array(n):
+                a = numba.cuda.device_array((n,), dtype="u1")
                 weakref.finalize(a, numba.cuda.current_context)
                 return a
 
-            cuda_array = numba_cuda_array
+            device_array = numba_device_array
         except ImportError:
 
-            def cuda_array(n):
+            def device_array(n):
                 raise RuntimeError(
                     "In order to send/recv CUDA arrays, Numba or RMM is required"
                 )
@@ -169,19 +178,25 @@ async def write(
                 frames = await to_frames(
                     msg, serializers=serializers, on_error=on_error
                 )
+                nframes = len(frames)
+                cuda_frames = tuple(
+                    hasattr(f, "__cuda_array_interface__") for f in frames
+                )
+                sizes = tuple(nbytes(f) for f in frames)
                 send_frames = [
-                    each_frame for each_frame in frames if len(each_frame) > 0
+                    each_frame
+                    for each_frame, each_size in zip(frames, sizes)
+                    if each_size
                 ]
 
                 # Send meta data
-                cuda_frames = np.array(
-                    [hasattr(f, "__cuda_array_interface__") for f in frames],
-                    dtype=np.bool,
-                )
-                await self.ep.send(np.array([len(frames)], dtype=np.uint64))
-                await self.ep.send(cuda_frames)
+
+                # Send # of frames (uint64)
+                await self.ep.send(struct.pack("Q", nframes))
+                # Send which frames are CUDA (bool) and
+                # how large each frame is (uint64)
                 await self.ep.send(
-                    np.array([nbytes(f) for f in frames], dtype=np.uint64)
+                    struct.pack(nframes * "?" + nframes * "Q", *cuda_frames, *sizes)
                 )
 
                 # Send frames
@@ -191,12 +206,12 @@ async def write(
                 #  syncing the default stream will wait for other non-blocking CUDA streams.
                 # Note this is only sufficient if the memory being sent is not currently in use on
                 # non-blocking CUDA streams.
-                if cuda_frames.any():
+                if any(cuda_frames):
                     synchronize_stream(0)
 
                 for each_frame in send_frames:
                     await self.ep.send(each_frame)
-                return sum(map(nbytes, send_frames))
+                return sum(sizes)
             except (ucp.exceptions.UCXBaseException):
                 self.abort()
                 raise CommClosedError("While writing, the connection was closed")
@@ -211,22 +226,28 @@ async def read(self, deserializers=("cuda", "dask", "pickle", "error")):
 
             try:
                 # Recv meta data
-                nframes = np.empty(1, dtype=np.uint64)
+
+                # Recv # of frames (uint64)
+                nframes_fmt = "Q"
+                nframes = host_array(struct.calcsize(nframes_fmt))
                 await self.ep.recv(nframes)
-                is_cudas = np.empty(nframes[0], dtype=np.bool)
-                await self.ep.recv(is_cudas)
-                sizes = np.empty(nframes[0], dtype=np.uint64)
-                await self.ep.recv(sizes)
+                (nframes,) = struct.unpack(nframes_fmt, nframes)
+
+                # Recv which frames are CUDA (bool) and
+                # how large each frame is (uint64)
+                header_fmt = nframes * "?" + nframes * "Q"
+                header = host_array(struct.calcsize(header_fmt))
+                await self.ep.recv(header)
+                header = struct.unpack(header_fmt, header)
+                cuda_frames, sizes = header[:nframes], header[nframes:]
             except (ucp.exceptions.UCXBaseException, CancelledError):
                 self.abort()
                 raise CommClosedError("While reading, the connection was closed")
             else:
                 # Recv frames
                 frames = [
-                    cuda_array(each_size)
-                    if is_cuda
-                    else np.empty(each_size, dtype=np.uint8)
-                    for is_cuda, each_size in zip(is_cudas.tolist(), sizes.tolist())
+                    device_array(each_size) if is_cuda else host_array(each_size)
+                    for is_cuda, each_size in zip(cuda_frames, sizes)
                 ]
                 recv_frames = [
                     each_frame for each_frame in frames if len(each_frame) > 0
@@ -234,7 +255,7 @@ async def read(self, deserializers=("cuda", "dask", "pickle", "error")):
 
                 # It is necessary to first populate `frames` with CUDA arrays and synchronize
                 # the default stream before starting receiving to ensure buffers have been allocated
-                if is_cudas.any():
+                if any(cuda_frames):
                     synchronize_stream(0)
 
                 for each_frame in recv_frames:

From ecdcb33720b2794c21b5bd322e4d2ccd8fec2787 Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Fri, 24 Apr 2020 04:51:11 +0100
Subject: [PATCH 0805/1550] Reinstate support for legacy @gen_cluster functions
 (#3738)

* gen_cluster and gen_test to accept legacy coroutines again
* Legacy python cleanup
---
 distributed/_version.py                     |  4 +-
 distributed/comm/tests/test_comms.py        |  7 +---
 distributed/deploy/tests/test_local.py      |  9 +----
 distributed/metrics.py                      |  5 +--
 distributed/protocol/pickle.py              |  6 +--
 distributed/protocol/tests/test_numpy.py    |  3 --
 distributed/protocol/tests/test_protocol.py |  3 --
 distributed/tests/test_steal.py             |  1 +
 distributed/tests/test_utils_test.py        | 34 ++++++++++++++++
 distributed/utils.py                        | 12 +-----
 distributed/utils_test.py                   | 45 ++++++---------------
 11 files changed, 57 insertions(+), 72 deletions(-)

diff --git a/distributed/_version.py b/distributed/_version.py
index f48634810c3..79f2770dd9c 100644
--- a/distributed/_version.py
+++ b/distributed/_version.py
@@ -96,9 +96,7 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env=
         if verbose:
             print("unable to find command, tried %s" % (commands,))
         return None, None
-    stdout = p.communicate()[0].strip()
-    if sys.version_info[0] >= 3:
-        stdout = stdout.decode()
+    stdout = p.communicate()[0].strip().decode()
     if p.returncode != 0:
         if verbose:
             print("unable to run %s (error)" % dispcmd)
diff --git a/distributed/comm/tests/test_comms.py b/distributed/comm/tests/test_comms.py
index 035a95513fb..ac633c3aa86 100644
--- a/distributed/comm/tests/test_comms.py
+++ b/distributed/comm/tests/test_comms.py
@@ -646,8 +646,7 @@ async def handle_comm(comm):
         )
         await comm.write({"x": "foo"})  # TODO: why is this necessary in Tornado 6 ?
 
-    # The wrong error is reported on Python 2, see https://github.com/tornadoweb/tornado/pull/2028
-    if sys.version_info >= (3,) and os.name != "nt":
+    if os.name != "nt":
         try:
             # See https://serverfault.com/questions/793260/what-does-tlsv1-alert-unknown-ca-mean
             assert "unknown ca" in str(excinfo.value)
@@ -670,9 +669,7 @@ async def handle_comm(comm):
         await connect(
             listener.contact_address, timeout=2, ssl_context=cli_ctx,
         )
-    # The wrong error is reported on Python 2, see https://github.com/tornadoweb/tornado/pull/2028
-    if sys.version_info >= (3,):
-        assert "certificate verify failed" in str(excinfo.value)
+    assert "certificate verify failed" in str(excinfo.value)
 
 
 #
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index 94a6016dd2a..0867968a894 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -67,7 +67,6 @@ def test_local_cluster_supports_blocked_handlers(loop):
     )
 
 
-@pytest.mark.skipif("sys.version_info[0] == 2", reason="fork issues")
 def test_close_twice():
     with LocalCluster() as cluster:
         with Client(cluster.scheduler_address) as client:
@@ -81,7 +80,6 @@ def test_close_twice():
         assert not log
 
 
-@pytest.mark.skipif("sys.version_info[0] == 2", reason="multi-loop")
 def test_procs():
     with LocalCluster(
         2,
@@ -173,13 +171,11 @@ def test_transports_tcp_port():
             assert e.submit(inc, 4).result() == 5
 
 
-@pytest.mark.skipif("sys.version_info[0] == 2", reason="")
 class LocalTest(ClusterTest, unittest.TestCase):
     Cluster = partial(LocalCluster, silence_logs=False, dashboard_address=None)
     kwargs = {"dashboard_address": None, "processes": False}
 
 
-@pytest.mark.skipif("sys.version_info[0] == 2", reason="")
 def test_Client_with_local(loop):
     with LocalCluster(
         1, scheduler_port=0, silence_logs=False, dashboard_address=None, loop=loop
@@ -429,7 +425,6 @@ def test_bokeh(loop, processes):
         requests.get(url, timeout=0.2)
 
 
-@pytest.mark.skipif(sys.version_info < (3, 6), reason="Unknown")
 def test_blocks_until_full(loop):
     with Client(loop=loop) as c:
         assert len(c.nthreads()) > 0
@@ -462,7 +457,8 @@ async def test_scale_up_and_down():
 
 @pytest.mark.xfail(
     sys.version_info >= (3, 8) and LooseVersion(tornado.version) < "6.0.3",
-    reason="Known issue with Python 3.8 and Tornado < 6.0.3. See https://github.com/tornadoweb/tornado/pull/2683.",
+    reason="Known issue with Python 3.8 and Tornado < 6.0.3. "
+    "See https://github.com/tornadoweb/tornado/pull/2683.",
     strict=True,
 )
 def test_silent_startup():
@@ -549,7 +545,6 @@ def test_death_timeout_raises(loop):
     LocalCluster._instances.clear()  # ignore test hygiene checks
 
 
-@pytest.mark.skipif(sys.version_info < (3, 6), reason="Unknown")
 @pytest.mark.asyncio
 async def test_bokeh_kwargs(cleanup):
     pytest.importorskip("bokeh")
diff --git a/distributed/metrics.py b/distributed/metrics.py
index f28e9f2ac7f..0f7d78a8129 100755
--- a/distributed/metrics.py
+++ b/distributed/metrics.py
@@ -49,10 +49,7 @@ def __init__(self):
         self.delta = None
         self.last_resync = float("-inf")
 
-    if sys.version_info >= (3,):
-        perf_counter = timemod.perf_counter
-    else:
-        perf_counter = timemod.clock
+    perf_counter = timemod.perf_counter
 
     def time(self):
         delta = self.delta
diff --git a/distributed/protocol/pickle.py b/distributed/protocol/pickle.py
index 629fb962fbf..9a1f135444f 100644
--- a/distributed/protocol/pickle.py
+++ b/distributed/protocol/pickle.py
@@ -1,12 +1,8 @@
 import logging
-import sys
+import pickle
 
 import cloudpickle
 
-if sys.version_info.major == 2:
-    import cPickle as pickle
-else:
-    import pickle
 
 logger = logging.getLogger(__name__)
 
diff --git a/distributed/protocol/tests/test_numpy.py b/distributed/protocol/tests/test_numpy.py
index 08a7c2df244..70ee582fd70 100644
--- a/distributed/protocol/tests/test_numpy.py
+++ b/distributed/protocol/tests/test_numpy.py
@@ -1,4 +1,3 @@
-import sys
 from zlib import crc32
 
 import numpy as np
@@ -189,7 +188,6 @@ def test_itemsize(dt, size):
     assert itemsize(np.dtype(dt)) == size
 
 
-@pytest.mark.skipif(sys.version_info[0] < 3, reason="numpy doesnt use memoryviews")
 def test_compress_numpy():
     pytest.importorskip("lz4")
     x = np.ones(10000000, dtype="i4")
@@ -238,7 +236,6 @@ async def test_dumps_large_blosc(c, s, a, b):
     await x
 
 
-@pytest.mark.skipif(sys.version_info[0] < 3, reason="numpy doesnt use memoryviews")
 def test_compression_takes_advantage_of_itemsize():
     pytest.importorskip("lz4")
     blosc = pytest.importorskip("blosc")
diff --git a/distributed/protocol/tests/test_protocol.py b/distributed/protocol/tests/test_protocol.py
index d3536933a96..6c8296edecb 100644
--- a/distributed/protocol/tests/test_protocol.py
+++ b/distributed/protocol/tests/test_protocol.py
@@ -1,5 +1,3 @@
-import sys
-
 import dask
 import pytest
 
@@ -209,7 +207,6 @@ def test_dumps_loads_Serialized():
     assert result == result3
 
 
-@pytest.mark.skipif(sys.version_info[0] < 3, reason="NumPy doesnt use memoryviews")
 def test_maybe_compress_memoryviews():
     np = pytest.importorskip("numpy")
     pytest.importorskip("lz4")
diff --git a/distributed/tests/test_steal.py b/distributed/tests/test_steal.py
index fb5c96e14e6..5ef3e5330ec 100644
--- a/distributed/tests/test_steal.py
+++ b/distributed/tests/test_steal.py
@@ -106,6 +106,7 @@ async def test_worksteal_many_thieves(c, s, *workers):
     assert sum(map(len, s.has_what.values())) < 150
 
 
+@pytest.mark.xfail(reason="GH#3574")
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 2)
 async def test_dont_steal_unknown_functions(c, s, a, b):
     futures = c.map(inc, range(100), workers=a.address, allow_other_workers=True)
diff --git a/distributed/tests/test_utils_test.py b/distributed/tests/test_utils_test.py
index 502b27b3013..4e9e776b590 100755
--- a/distributed/tests/test_utils_test.py
+++ b/distributed/tests/test_utils_test.py
@@ -5,6 +5,7 @@
 from time import sleep
 
 import pytest
+from tornado import gen
 
 from distributed import Scheduler, Worker, Client, config, default_client
 from distributed.core import rpc
@@ -49,6 +50,28 @@ async def test_gen_cluster(c, s, a, b):
     for w in [a, b]:
         assert isinstance(w, Worker)
     assert s.nthreads == {w.address: w.nthreads for w in [a, b]}
+    assert await c.submit(lambda: 123) == 123
+
+
+@gen_cluster(client=True)
+def test_gen_cluster_legacy_implicit(c, s, a, b):
+    assert isinstance(c, Client)
+    assert isinstance(s, Scheduler)
+    for w in [a, b]:
+        assert isinstance(w, Worker)
+    assert s.nthreads == {w.address: w.nthreads for w in [a, b]}
+    assert (yield c.submit(lambda: 123)) == 123
+
+
+@gen_cluster(client=True)
+@gen.coroutine
+def test_gen_cluster_legacy_explicit(c, s, a, b):
+    assert isinstance(c, Client)
+    assert isinstance(s, Scheduler)
+    for w in [a, b]:
+        assert isinstance(w, Worker)
+    assert s.nthreads == {w.address: w.nthreads for w in [a, b]}
+    assert (yield c.submit(lambda: 123)) == 123
 
 
 @pytest.mark.skip(reason="This hangs on travis")
@@ -101,6 +124,17 @@ async def test_gen_test():
     await asyncio.sleep(0.01)
 
 
+@gen_test()
+def test_gen_test_legacy_implicit():
+    yield asyncio.sleep(0.01)
+
+
+@gen_test()
+@gen.coroutine
+def test_gen_test_legacy_explicit():
+    yield asyncio.sleep(0.01)
+
+
 @contextmanager
 def _listen(delay=0):
     serv = socket.socket()
diff --git a/distributed/utils.py b/distributed/utils.py
index 46bd4c245e8..ea333833b08 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -8,7 +8,6 @@
 import functools
 from hashlib import md5
 import html
-import inspect
 import json
 import logging
 import multiprocessing
@@ -1039,10 +1038,7 @@ def import_file(path):
     if ext in (".egg", ".zip", ".pyz"):
         if path not in sys.path:
             sys.path.insert(0, path)
-        if sys.version_info >= (3, 6):
-            names = (mod_info.name for mod_info in pkgutil.iter_modules([path]))
-        else:
-            names = (mod_info[1] for mod_info in pkgutil.iter_modules([path]))
+        names = (mod_info.name for mod_info in pkgutil.iter_modules([path]))
         names_to_import.extend(names)
 
     loaded = []
@@ -1285,11 +1281,7 @@ def color_of(x, palette=palette):
 
 
 def iscoroutinefunction(f):
-    if gen.is_coroutine_function(f):
-        return True
-    if sys.version_info >= (3, 5) and inspect.iscoroutinefunction(f):
-        return True
-    return False
+    return inspect.iscoroutinefunction(f) or gen.is_coroutine_function(f)
 
 
 @contextmanager
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index e466322eddf..05467bbeb49 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -18,7 +18,6 @@
 import subprocess
 import sys
 import tempfile
-import textwrap
 import threading
 from time import sleep
 import uuid
@@ -34,6 +33,7 @@
 
 import dask
 from tlz import merge, memoize, assoc
+from tornado import gen
 from tornado.ioloop import IOLoop
 
 from . import system
@@ -397,25 +397,9 @@ async def geninc(x, delay=0.02):
     return x + 1
 
 
-def compile_snippet(code, dedent=True):
-    if dedent:
-        code = textwrap.dedent(code)
-    code = compile(code, "<dynamic>", "exec")
-    ns = globals()
-    exec(code, ns, ns)
-
-
-if sys.version_info >= (3, 5):
-    compile_snippet(
-        """
-        async def asyncinc(x, delay=0.02):
-            await asyncio.sleep(delay)
-            return x + 1
-        """
-    )
-    assert asyncinc  # noqa: F821
-else:
-    asyncinc = None
+async def asyncinc(x, delay=0.02):
+    await asyncio.sleep(delay)
+    return x + 1
 
 
 _readone_queues = {}
@@ -768,9 +752,11 @@ async def test_foo():
     def _(func):
         def test_func():
             with clean() as loop:
-                if not iscoroutinefunction(func):
-                    raise ValueError("@gen_test should wrap async def functions")
-                loop.run_sync(func, timeout=timeout)
+                if iscoroutinefunction(func):
+                    cor = func
+                else:
+                    cor = gen.coroutine(func)
+                loop.run_sync(cor, timeout=timeout)
 
         return test_func
 
@@ -877,10 +863,10 @@ async def test_foo(scheduler, worker1, worker2):
     )
 
     def _(func):
-        def test_func():
-            if not iscoroutinefunction(func):
-                raise ValueError("@gen_cluster should wrap async def functions")
+        if not iscoroutinefunction(func):
+            func = gen.coroutine(func)
 
+        def test_func():
             result = None
             workers = []
             with clean(timeout=active_rpc_timeout, **clean_kwargs) as loop:
@@ -1001,12 +987,7 @@ def terminate_process(proc):
         else:
             proc.send_signal(signal.SIGINT)
         try:
-            if sys.version_info[0] == 3:
-                proc.wait(10)
-            else:
-                start = time()
-                while proc.poll() is None and time() < start + 10:
-                    sleep(0.02)
+            proc.wait(10)
         finally:
             # Make sure we don't leave the process lingering around
             with ignoring(OSError):

From 4199c546154a75afa5404a7cfbaa8e864286dcf3 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Fri, 24 Apr 2020 23:22:18 -0500
Subject: [PATCH 0806/1550] bump version to 2.15.0

---
 docs/source/changelog.rst | 42 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index c1bcab71eb5..628a1d2147a 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,46 @@
 Changelog
 =========
 
+2.15.0 - 2020-04-24
+-------------------
+
+- Reinstate support for legacy ``@gen_cluster`` functions (:pr:`3738`) `crusaderky`_
+- Relax NumPy requirement in UCX (:pr:`3731`) `jakirkham`_
+- Add Configuration Schema (:pr:`3696`) `Matthew Rocklin`_
+- Reuse CI scripts for local installation process (:pr:`3698`) `crusaderky`_
+- Use ``PeriodicCallback`` class from tornado (:pr:`3725`) `James Bourbeau`_
+- Add ``remote_python`` option in ssh cmd (:pr:`3709`) `Abdulelah Bin Mahfoodh`_
+- Configurable polling interval for cluster widget (:pr:`3723`) `Julia Signell`_
+- Fix copy-paste in docs (:pr:`3728`) `Julia Signell`_
+- Replace ``gen.coroutine`` with async-await in tests (:pr:`3706`) `crusaderky`_
+- Fix flaky ``test_oversubscribing_leases`` (:pr:`3726`) `Florian Jetter`_
+- Add ``batch_size`` to ``Client.map`` (:pr:`3650`) `Tom Augspurger`_
+- Adjust semaphore test timeouts (:pr:`3720`) `Florian Jetter`_
+- Dask-serialize dicts longer than five elements (:pr:`3689`) `Richard J Zamora`_
+- Force ``threads_per_worker`` (:pr:`3715`) `crusaderky`_
+- Idempotent semaphore acquire with retries (:pr:`3690`) `Florian Jetter`_
+- Always use ``readinto`` in TCP (:pr:`3711`) `jakirkham`_
+- Avoid ``DeprecationWarning`` from pandas (:pr:`3712`) `Tom Augspurger`_
+- Allow modification of ``distributed.comm.retry`` at runtime (:pr:`3705`) `Florian Jetter`_
+- Do not log an error on unset variable delete (:pr:`3652`) `Jonathan J. Helmus`_
+- Add ``remote_python`` keyword to the new ``SSHCluster`` (:pr:`3701`) `Abdulelah Bin Mahfoodh`_
+- Replace Example with Examples in docstrings (:pr:`3697`) `Matthew Rocklin`_
+- Add ``Cluster`` ``__enter__`` and ``__exit__`` methods (:pr:`3699`) `Matthew Rocklin`_
+- Fix propagating inherit config in ``SSHCluster`` for non-bash shells (:pr:`3688`) `Abdulelah Bin Mahfoodh`_
+- Add ``Client.wait_to_workers`` to ``Client`` autosummary table (:pr:`3692`) `James Bourbeau`_
+- Replace Bokeh Server with Tornado HTTPServer (:pr:`3658`) `Matthew Rocklin`_
+- Fix ``dask-ssh`` after removing ``local-directory`` from ``dask_scheduler`` cli (:pr:`3684`) `Abdulelah Bin Mahfoodh`_
+- Support preload modules in ``Nanny`` (:pr:`3678`) `Matthew Rocklin`_
+- Refactor semaphore internals: make ``_get_lease`` synchronous (:pr:`3679`) `Lucas Rademaker`_
+- Don't make task graphs too big (:pr:`3671`) `Martin Durant`_
+- Pass through ``connection``/``listen_args`` as splatted keywords (:pr:`3674`) `Matthew Rocklin`_
+- Run preload at import, start, and teardown (:pr:`3673`) `Matthew Rocklin`_
+- Use relative URL in scheduler dashboard (:pr:`3676`) `Nicholas Smith`_
+- Expose ``Security`` object as public API (:pr:`3675`) `Matthew Rocklin`_
+- Add zoom tools to profile plots (:pr:`3672`) `James Bourbeau`_
+- Update ``Scheduler.rebalance`` return value when data is missing (:pr:`3670`) `James Bourbeau`_
+
+
 2.14.0 - 2020-04-03
 -------------------
 
@@ -1672,3 +1712,5 @@ significantly without many new features.
 .. _`Prasun Anand`: https://github.com/prasunanand
 .. _`Jonathan J. Helmus`: https://github.com/jjhelmus
 .. _`Rami Chowdhury`: https://github.com/necaris
+.. _`crusaderky`: https://github.com/crusaderky
+.. _`Nicholas Smith`: https://github.com/nsmith-

From 2d54ef947a7fe372b1fafc40d2f2b89ff88e9449 Mon Sep 17 00:00:00 2001
From: Dillon Niederhut <deniederhut@users.noreply.github.com>
Date: Mon, 27 Apr 2020 10:21:18 -0500
Subject: [PATCH 0807/1550] BUG: allows logging config under distributed key
 (#2952)

* BUG: allows logging config under distributed key

The logging documentation specifies a configuration path to
logging info like config['distributed']['logging'], but the
config module looks in config['logging']. This commit allows
both by looking first in config['distributed'], then falling
back to config.

Closes https://github.com/dask/distributed/issues/2937
---
 distributed/config.py            | 28 ++++++++++++++++++-----
 distributed/tests/test_config.py | 39 ++++++++++++++++++++++++++++++++
 2 files changed, 61 insertions(+), 6 deletions(-)

diff --git a/distributed/config.py b/distributed/config.py
index a313f18416b..43e545576e5 100644
--- a/distributed/config.py
+++ b/distributed/config.py
@@ -80,7 +80,8 @@ def _initialize_logging_old_style(config):
         "tornado": "critical",
         "tornado.application": "error",
     }
-    loggers.update(config.get("logging", {}))
+    base_config = _find_logging_config(config)
+    loggers.update(base_config.get("logging", {}))
 
     handler = logging.StreamHandler(sys.stderr)
     handler.setFormatter(
@@ -103,7 +104,8 @@ def _initialize_logging_new_style(config):
     Initialize logging using logging's "Configuration dictionary schema".
     (ref.: https://docs.python.org/3/library/logging.config.html#configuration-dictionary-schema)
     """
-    logging.config.dictConfig(config.get("logging"))
+    base_config = _find_logging_config(config)
+    logging.config.dictConfig(base_config.get("logging"))
 
 
 def _initialize_logging_file_config(config):
@@ -111,20 +113,34 @@ def _initialize_logging_file_config(config):
     Initialize logging using logging's "Configuration file format".
     (ref.: https://docs.python.org/3/howto/logging.html#configuring-logging)
     """
+    base_config = _find_logging_config(config)
     logging.config.fileConfig(
-        config.get("logging-file-config"), disable_existing_loggers=False
+        base_config.get("logging-file-config"), disable_existing_loggers=False
     )
 
 
+def _find_logging_config(config):
+    """
+    Look for the dictionary containing logging-specific configurations,
+    starting in the 'distributed' dictionary and then trying the top-level
+    """
+    logging_keys = {"logging-file-config", "logging"}
+    if logging_keys & config.get("distributed", {}).keys():
+        return config["distributed"]
+    else:
+        return config
+
+
 def initialize_logging(config):
-    if "logging-file-config" in config:
-        if "logging" in config:
+    base_config = _find_logging_config(config)
+    if "logging-file-config" in base_config:
+        if "logging" in base_config:
             raise RuntimeError(
                 "Config options 'logging-file-config' and 'logging' are mutually exclusive."
             )
         _initialize_logging_file_config(config)
     else:
-        log_config = config.get("logging", {})
+        log_config = base_config.get("logging", {})
         if "version" in log_config:
             # logging module mandates version to be an int
             log_config["version"] = int(log_config["version"])
diff --git a/distributed/tests/test_config.py b/distributed/tests/test_config.py
index 01cd6eec57b..74b57b1f011 100644
--- a/distributed/tests/test_config.py
+++ b/distributed/tests/test_config.py
@@ -109,6 +109,45 @@ def test_logging_empty_simple():
         test_logging_default()
 
 
+def test_logging_simple_under_distributed():
+    """
+    Test simple ("old-style") logging configuration under the distributed key.
+    """
+    c = {
+        "distributed": {
+            "logging": {"distributed.foo": "info", "distributed.foo.bar": "error"}
+        }
+    }
+    # Must test using a subprocess to avoid wrecking pre-existing configuration
+    with new_config_file(c):
+        code = """if 1:
+            import logging
+            import dask
+
+            from distributed.utils_test import captured_handler
+
+            d = logging.getLogger('distributed')
+            assert len(d.handlers) == 1
+            assert isinstance(d.handlers[0], logging.StreamHandler)
+            df = logging.getLogger('distributed.foo')
+            dfb = logging.getLogger('distributed.foo.bar')
+
+            with captured_handler(d.handlers[0]) as distributed_log:
+                df.info("1: info")
+                dfb.warning("2: warning")
+                dfb.error("3: error")
+
+            distributed_log = distributed_log.getvalue().splitlines()
+
+            assert distributed_log == [
+                "distributed.foo - INFO - 1: info",
+                "distributed.foo.bar - ERROR - 3: error",
+                ], (dask.config.config, distributed_log)
+            """
+
+        subprocess.check_call([sys.executable, "-c", code])
+
+
 def test_logging_simple():
     """
     Test simple ("old-style") logging configuration.

From 3de9973cafaf85809f712a719111b8db3839975d Mon Sep 17 00:00:00 2001
From: Martin Durant <martindurant@users.noreply.github.com>
Date: Mon, 27 Apr 2020 13:00:29 -0400
Subject: [PATCH 0808/1550] Memoryview serialisation (#3743)

* Serialise memview without copy

* black

* Special-case deser memview with one frame
---
 distributed/protocol/serialize.py            | 11 ++++++++++-
 distributed/protocol/tests/test_serialize.py | 10 ++++++++++
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index a1b35ec4463..4d02bc65207 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -556,7 +556,7 @@ def normalize_Serialized(o):
 
 
 # Teach serialize how to handle bytestrings
-@dask_serialize.register((bytes, bytearray))
+@dask_serialize.register((bytes, bytearray, memoryview))
 def _serialize_bytes(obj):
     header = {}  # no special metadata
     frames = [obj]
@@ -568,6 +568,15 @@ def _deserialize_bytes(header, frames):
     return b"".join(frames)
 
 
+@dask_deserialize.register(memoryview)
+def _serialize_memoryview(header, frames):
+    if len(frames) == 1:
+        out = frames[0]
+    else:
+        out = b"".join(frames)
+    return memoryview(out)
+
+
 #########################
 # Descend into __dict__ #
 #########################
diff --git a/distributed/protocol/tests/test_serialize.py b/distributed/protocol/tests/test_serialize.py
index 6a5af842ddd..4cad5a3653b 100644
--- a/distributed/protocol/tests/test_serialize.py
+++ b/distributed/protocol/tests/test_serialize.py
@@ -409,6 +409,7 @@ def _(x):
         ([MyObj([0, 1, 2]), 1], True),
         (tuple([MyObj(None)]), True),
         ({("x", i): MyObj(5) for i in range(100)}, True),
+        (memoryview(b"hello"), True),
     ],
 )
 def test_check_dask_serializable(data, is_serializable):
@@ -428,3 +429,12 @@ def test_serialize_lists(serializers):
     data_out = deserialize(header, frames)
 
     assert data_in == data_out
+
+
+def test_deser_memoryview():
+    data_in = memoryview(b"hello")
+    header, frames = serialize(data_in)
+    assert header["type"] == "builtins.memoryview"
+    assert frames[0] is data_in
+    data_out = deserialize(header, frames)
+    assert data_in == data_out

From 26a9fd6256c24098c4020bbe9deab2e6fc914cca Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 27 Apr 2020 11:50:22 -0700
Subject: [PATCH 0809/1550] Warn if cluster closes before starting (#3735)

Otherwise users get an odd message like the following:

```python-traceback
  File "/home/XXX/.local/lib/python3.6/site-packages/tornado/ioloop.py", line 743, in _run_callback
    ret = callback()
  File "/home/XXX/.local/lib/python3.6/site-packages/tornado/ioloop.py", line 767, in _discard_future_result
    future.result()
  File "/home/XXX/.local/lib/python3.6/site-packages/distributed/deploy/spec.py", line 386, in _close
    await self.scheduler_comm.close(close_workers=True)
AttributeError: 'NoneType' object has no attribute 'close'
```
---
 distributed/deploy/spec.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index eb9f0f0043e..8419659ca3a 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -383,7 +383,10 @@ async def _close(self):
             await future
         async with self._lock:
             with ignoring(CommClosedError):
-                await self.scheduler_comm.close(close_workers=True)
+                if self.scheduler_comm:
+                    await self.scheduler_comm.close(close_workers=True)
+                else:
+                    logger.warning("Cluster closed without starting up")
 
         await self.scheduler.close()
         for w in self._created:

From 7c57f853bf271e83e80983f53cb874b95bddf7b9 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Mon, 27 Apr 2020 19:15:31 -0500
Subject: [PATCH 0810/1550] Ensure BokehTornado uses prefix (#3746)

---
 distributed/dashboard/core.py                 |  6 ++----
 .../dashboard/tests/test_scheduler_bokeh.py   | 19 +++++++++++++++++++
 2 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/distributed/dashboard/core.py b/distributed/dashboard/core.py
index 6843b0659b3..6c3c7e919c2 100644
--- a/distributed/dashboard/core.py
+++ b/distributed/dashboard/core.py
@@ -27,10 +27,7 @@ def BokehApplication(applications, server, prefix="/", template_variables={}):
 
     extra = toolz.merge({"prefix": prefix}, template_variables)
 
-    apps = {
-        prefix + k.lstrip("/"): functools.partial(v, server, extra)
-        for k, v in applications.items()
-    }
+    apps = {k: functools.partial(v, server, extra) for k, v in applications.items()}
     apps = {k: Application(FunctionHandler(v)) for k, v in apps.items()}
     kwargs = dask.config.get("distributed.scheduler.dashboard.bokeh-application").copy()
     extra_websocket_origins = create_hosts_whitelist(
@@ -39,6 +36,7 @@ def BokehApplication(applications, server, prefix="/", template_variables={}):
 
     application = BokehTornado(
         apps,
+        prefix=prefix,
         use_index=False,
         extra_websocket_origins=extra_websocket_origins,
         **kwargs,
diff --git a/distributed/dashboard/tests/test_scheduler_bokeh.py b/distributed/dashboard/tests/test_scheduler_bokeh.py
index 8ed1bb0f8a1..f943807d4df 100644
--- a/distributed/dashboard/tests/test_scheduler_bokeh.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh.py
@@ -8,6 +8,7 @@
 import pytest
 
 pytest.importorskip("bokeh")
+from bokeh.server.server import BokehTornado
 from tlz import first
 from tornado.httpclient import AsyncHTTPClient, HTTPRequest
 
@@ -712,3 +713,21 @@ async def test_memory_by_key(c, s, a, b):
     mbk.update()
     assert mbk.source.data["name"] == ["add", "inc"]
     assert mbk.source.data["nbytes"] == [x.nbytes, sys.getsizeof(1)]
+
+
+@gen_cluster(scheduler_kwargs={"http_prefix": "foo-bar", "dashboard": True})
+async def test_prefix_bokeh(s, a, b):
+    prefix = "foo-bar"
+    http_client = AsyncHTTPClient()
+    response = await http_client.fetch(
+        f"http://localhost:{s.http_server.port}/{prefix}/status"
+    )
+    assert response.code == 200
+    assert (
+        f'<script type="text/javascript" src="/{prefix}/static/'
+        in response.body.decode()
+    )
+
+    bokeh_app = s.http_application.applications[0]
+    assert isinstance(bokeh_app, BokehTornado)
+    assert bokeh_app.prefix == f"/{prefix}"

From 7936dcbb1bae42e47c5af4f91863dd7f7ec3b6c2 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Tue, 28 Apr 2020 10:02:38 -0500
Subject: [PATCH 0811/1550] bump version to 2.15.1

---
 docs/source/changelog.rst | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index 628a1d2147a..73ae07e578d 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,15 @@
 Changelog
 =========
 
+2.15.1 - 2020-04-28
+-------------------
+
+- Ensure ``BokehTornado`` uses prefix (:pr:`3746`) `James Bourbeau`_
+- Warn if cluster closes before starting (:pr:`3735`) `Matthew Rocklin`_
+- Memoryview serialisation (:pr:`3743`) `Martin Durant`_
+- Allows logging config under distributed key (:pr:`2952`) `Dillon Niederhut`_
+
+
 2.15.0 - 2020-04-24
 -------------------
 
@@ -1714,3 +1723,4 @@ significantly without many new features.
 .. _`Rami Chowdhury`: https://github.com/necaris
 .. _`crusaderky`: https://github.com/crusaderky
 .. _`Nicholas Smith`: https://github.com/nsmith-
+.. _`Dillon Niederhut`: https://github.com/deniederhut

From f748d7d539df0285b46b3c966c587dfd76bf3e48 Mon Sep 17 00:00:00 2001
From: Jonas Haag <jonas@lophus.org>
Date: Wed, 29 Apr 2020 17:00:35 +0200
Subject: [PATCH 0812/1550] Fix exception handling typo (#3751)

ignoring() expects the list of catchable exceptions as star arguments rather than as a tuple
---
 distributed/deploy/spec.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index 8419659ca3a..cdde4a8162a 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -606,6 +606,6 @@ async def run_spec(spec: dict, *args):
 @atexit.register
 def close_clusters():
     for cluster in list(SpecCluster._instances):
-        with ignoring((gen.TimeoutError, TimeoutError)):
+        with ignoring(gen.TimeoutError, TimeoutError):
             if cluster.status != "closed":
                 cluster.close(timeout=10)

From c8ee44bebe40e4db4495be1e2e0340fafbb41a37 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 29 Apr 2020 18:30:00 -0700
Subject: [PATCH 0813/1550] Replace OrderedDict with dict in scheduler (#3740)

This is no longer necessary now that we require Python 3.6+
---
 distributed/scheduler.py | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 82e5f812dd6..33bd6373d26 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1,5 +1,5 @@
 import asyncio
-from collections import defaultdict, deque, OrderedDict
+from collections import defaultdict, deque
 from collections.abc import Mapping, Set
 from datetime import timedelta
 from functools import partial
@@ -1973,7 +1973,7 @@ def update_graph(
                 ts.retries = v
 
         # Compute recommendations
-        recommendations = OrderedDict()
+        recommendations = {}
 
         for ts in sorted(runnables, key=operator.attrgetter("priority"), reverse=True):
             if ts.state == "released" and ts.run_spec:
@@ -2107,7 +2107,7 @@ def stimulus_missing_data(
                 return {}
             cts = self.tasks.get(cause)
 
-            recommendations = OrderedDict()
+            recommendations = {}
 
             if cts is not None and cts.state == "memory":  # couldn't find this
                 for ws in cts.who_has:  # TODO: this behavior is extreme
@@ -2206,7 +2206,7 @@ def remove_worker(self, comm=None, address=None, safe=False, close=True):
             ws.status = "closed"
             self.total_occupancy -= ws.occupancy
 
-            recommendations = OrderedDict()
+            recommendations = {}
 
             for ts in list(ws.processing):
                 k = ts.key
@@ -3876,7 +3876,7 @@ def transition_released_waiting(self, key):
 
             ts.state = "waiting"
 
-            recommendations = OrderedDict()
+            recommendations = {}
 
             for dts in ts.dependencies:
                 if dts.exception_blame:
@@ -3926,7 +3926,7 @@ def transition_no_worker_waiting(self, key):
             if ts.has_lost_dependencies:
                 return {key: "forgotten"}
 
-            recommendations = OrderedDict()
+            recommendations = {}
 
             for dts in ts.dependencies:
                 dep = dts.key
@@ -4058,7 +4058,7 @@ def transition_waiting_memory(self, key, nbytes=None, worker=None, **kwargs):
 
             self.check_idle_saturated(ws)
 
-            recommendations = OrderedDict()
+            recommendations = {}
 
             self._add_to_memory(ts, ws, recommendations, **kwargs)
 
@@ -4157,7 +4157,7 @@ def transition_processing_memory(
             if nbytes is not None:
                 ts.set_nbytes(nbytes)
 
-            recommendations = OrderedDict()
+            recommendations = {}
 
             self._remove_from_processing(ts)
 
@@ -4194,7 +4194,7 @@ def transition_memory_released(self, key, safe=False):
                     ts.exception = "Worker holding Actor was lost"
                     return {ts.key: "erred"}  # don't try to recreate
 
-            recommendations = OrderedDict()
+            recommendations = {}
 
             for dts in ts.waiters:
                 if dts.state in ("no-worker", "processing"):
@@ -4288,7 +4288,7 @@ def transition_erred_released(self, key):
                     assert not ts.waiting_on
                     assert not ts.waiters
 
-            recommendations = OrderedDict()
+            recommendations = {}
 
             ts.exception = None
             ts.exception_blame = None
@@ -4362,7 +4362,7 @@ def transition_processing_released(self, key):
 
             ts.state = "released"
 
-            recommendations = OrderedDict()
+            recommendations = {}
 
             if ts.has_lost_dependencies:
                 recommendations[key] = "forgotten"

From a673396ad826354e517839d95e3c0b916fef8b02 Mon Sep 17 00:00:00 2001
From: "Mads R. B. Kristensen" <madsbk@gmail.com>
Date: Thu, 30 Apr 2020 20:16:34 +0200
Subject: [PATCH 0814/1550] Nanny closing worker on KeyboardInterrupt (#3747)

* Nanny: calling do_stop() on KeyboardInterrupt

* Added a test of closing nanny by keyboard interrupt

* reformat: black
---
 distributed/nanny.py            |  4 +++-
 distributed/tests/test_nanny.py | 28 +++++++++++++++++++++++++++-
 2 files changed, 30 insertions(+), 2 deletions(-)

diff --git a/distributed/nanny.py b/distributed/nanny.py
index f3a355dca89..cd11e8ed56f 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -762,4 +762,6 @@ async def run():
             # Loop was stopped before wait_until_closed() returned, ignore
             pass
         except KeyboardInterrupt:
-            pass
+            # At this point the loop is not running thus we have to run
+            # do_stop() explicitly.
+            loop.run_sync(do_stop)
diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
index 2c7a6f83671..4bf8144956f 100644
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -14,7 +14,7 @@
 
 import dask
 from distributed.diagnostics import SchedulerPlugin
-from distributed import Nanny, rpc, Scheduler, Worker, Client, wait
+from distributed import Nanny, rpc, Scheduler, Worker, Client, wait, worker
 from distributed.core import CommClosedError
 from distributed.metrics import time
 from distributed.protocol.pickle import dumps
@@ -502,3 +502,29 @@ async def test_config(cleanup):
             async with Client(s.address, asynchronous=True) as client:
                 config = await client.run(dask.config.get, "foo")
                 assert config[n.worker_address] == "bar"
+
+
+class KeyboardInterruptWorker(worker.Worker):
+    """A Worker that raises KeyboardInterrupt almost immediately"""
+
+    async def heartbeat(self):
+        def raise_err():
+            raise KeyboardInterrupt()
+
+        self.loop.add_callback(raise_err)
+
+
+@pytest.mark.parametrize("protocol", ["tcp", "ucx"])
+@pytest.mark.asyncio
+async def test_nanny_closed_by_keyboard_interrupt(cleanup, protocol):
+    if protocol == "ucx":  # Skip if UCX isn't available
+        pytest.importorskip("ucp")
+
+    async with Scheduler(protocol=protocol) as s:
+        async with Nanny(
+            s.address, nthreads=1, worker_class=KeyboardInterruptWorker
+        ) as n:
+            n.auto_restart = False
+            await n.process.stopped.wait()
+            # Check that the scheduler has been notified about the closed worker
+            assert len(s.workers) == 0

From f7d6daca266509238e68ca3230b1ddba08de74b0 Mon Sep 17 00:00:00 2001
From: Tom Augspurger <TomAugspurger@users.noreply.github.com>
Date: Fri, 1 May 2020 09:32:24 -0500
Subject: [PATCH 0815/1550] Move test_gpu_metrics test (#3721)

gen_cluster doesn't cleanly handle pytest.importorskip

```
Task exception was never retrieved

future: <Task finished coro=<test_gpu_metrics() done, defined at
/home/travis/build/dask/distributed/distributed/tests/test_worker.py:1575>
exception=could not import 'pynvml': No module named 'pynvml'>

Traceback (most recent call last):

  File
  "/home/travis/miniconda/envs/test-environment/lib/python3.6/site-packages/_pytest/runner.py",
  line 244, in from_call
```

This moves the importorskip outside the test.
---
 distributed/tests/test_gpu_metrics.py | 14 ++++++++++++++
 distributed/tests/test_variable.py    |  6 +++---
 distributed/tests/test_worker.py      | 12 ------------
 3 files changed, 17 insertions(+), 15 deletions(-)
 create mode 100644 distributed/tests/test_gpu_metrics.py

diff --git a/distributed/tests/test_gpu_metrics.py b/distributed/tests/test_gpu_metrics.py
new file mode 100644
index 00000000000..584932988dd
--- /dev/null
+++ b/distributed/tests/test_gpu_metrics.py
@@ -0,0 +1,14 @@
+import pytest
+from distributed.utils_test import gen_cluster
+
+pytest.importorskip("pynvml")
+
+
+@gen_cluster()
+async def test_gpu_metrics(s, a, b):
+    from distributed.diagnostics.nvml import handles
+
+    assert "gpu" in a.metrics
+    assert len(s.workers[a.address].metrics["gpu"]["memory-used"]) == len(handles)
+    assert "gpu" in a.startup_information
+    assert len(s.workers[a.address].extra["gpu"]["name"]) == len(handles)
diff --git a/distributed/tests/test_variable.py b/distributed/tests/test_variable.py
index a60345d0abb..e2a4617ea0b 100644
--- a/distributed/tests/test_variable.py
+++ b/distributed/tests/test_variable.py
@@ -1,6 +1,6 @@
 import asyncio
 import random
-from time import sleep
+from time import sleep, monotonic
 import logging
 
 import pytest
@@ -96,10 +96,10 @@ async def test_hold_futures(s, a, b):
 async def test_timeout(c, s, a, b):
     v = Variable("v")
 
-    start = IOLoop.current().time()
+    start = monotonic()
     with pytest.raises(TimeoutError):
         await v.get(timeout=0.2)
-    stop = IOLoop.current().time()
+    stop = monotonic()
 
     if WINDOWS:  # timing is weird with asyncio and Windows
         assert 0.1 < stop - start < 2.0
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index a1f2e46295c..923a138cb60 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -1559,18 +1559,6 @@ async def test_lifetime_stagger(c, s, a, b):
     assert 8 <= b.lifetime <= 12
 
 
-@gen_cluster()
-async def test_gpu_metrics(s, a, b):
-    pytest.importorskip("pynvml")
-    from distributed.diagnostics.nvml import count
-
-    assert "gpu" in a.metrics
-    assert len(s.workers[a.address].metrics["gpu"]["memory-used"]) == count
-
-    assert "gpu" in a.startup_information
-    assert len(s.workers[a.address].extra["gpu"]["name"]) == count
-
-
 @pytest.mark.asyncio
 async def test_bad_metrics(cleanup):
     def bad_metric(w):

From 5cba144ffb8cc9f5252b03851ca7bf6ba51b008f Mon Sep 17 00:00:00 2001
From: Tom Augspurger <TomAugspurger@users.noreply.github.com>
Date: Fri, 1 May 2020 09:46:59 -0500
Subject: [PATCH 0816/1550] Connect to dashboard when address provided (#3758)

It seems like if the caller provides a `dashboard_address` then they
want the dashboard running. I'm not sure why `dashboard` is required,
looking for guidance on that.

Closes https://github.com/dask/dask-gateway/issues/261
---
 distributed/scheduler.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 33bd6373d26..a117583b66d 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1133,14 +1133,14 @@ def __init__(
         )
         self.start_http_server(routes, dashboard_address, default_port=8787)
 
-        if dashboard:
+        if dashboard or (dashboard is None and dashboard_address):
             try:
                 import distributed.dashboard.scheduler
             except ImportError:
                 logger.debug("To start diagnostics web server please install Bokeh")
             else:
                 distributed.dashboard.scheduler.connect(
-                    self.http_application, self.http_server, self, prefix=http_prefix,
+                    self.http_application, self.http_server, self, prefix=http_prefix
                 )
 
         # Communication state

From fdc4327bd236c5cf678d86edb0387ded2dcb8024 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Fri, 1 May 2020 09:56:24 -0500
Subject: [PATCH 0817/1550] bump version to 2.15.2

---
 docs/source/changelog.rst | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index 73ae07e578d..6ef12a686f9 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,16 @@
 Changelog
 =========
 
+2.15.2 - 2020-05-01
+-------------------
+
+- Connect to dashboard when address provided (:pr:`3758`) `Tom Augspurger`_
+- Move ``test_gpu_metrics test`` (:pr:`3721`) `Tom Augspurger`_
+- Nanny closing worker on ``KeyboardInterrupt`` (:pr:`3747`) `Mads R. B. Kristensen`_
+- Replace ``OrderedDict`` with ``dict`` in scheduler (:pr:`3740`) `Matthew Rocklin`_
+- Fix exception handling typo (:pr:`3751`) `Jonas Haag`_
+
+
 2.15.1 - 2020-04-28
 -------------------
 
@@ -1724,3 +1734,4 @@ significantly without many new features.
 .. _`crusaderky`: https://github.com/crusaderky
 .. _`Nicholas Smith`: https://github.com/nsmith-
 .. _`Dillon Niederhut`: https://github.com/deniederhut
+.. _`Jonas Haag`: https://github.com/jonashaag

From e7ba316d496a23d073188466571ce1580336aba4 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 1 May 2020 08:29:10 -0700
Subject: [PATCH 0818/1550] Support web addresses in preload (#3755)

* Support web addresses in preload

This commit does two things:

1.  We support passing web addresses as preloads,
    allowing for for functionality like the following:

    dask-scheduler --preload http://my-web-address/myfile.py

2.  We refactor preloads into a class structure.
    I'm usually against this, but I think that in this case it cleans
    things up.
    I think that it will also make it easier to rewrite how we handle
    preload and preload_argv in configuration later

* fix docstring and type parameters of Preload class

* Separate download_module from import_module

* Return and store actual module rather than dictionary

* typo
---
 distributed/nanny.py              |  22 +++----
 distributed/preloading.py         | 103 +++++++++++++++++-------------
 distributed/scheduler.py          |  10 +--
 distributed/tests/test_preload.py |  22 +++++++
 distributed/worker.py             |  22 +++----
 5 files changed, 107 insertions(+), 72 deletions(-)

diff --git a/distributed/nanny.py b/distributed/nanny.py
index cd11e8ed56f..4807073e08c 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -131,12 +131,10 @@ def __init__(
         if self.preload_argv is None:
             self.preload_argv = dask.config.get("distributed.worker.preload-argv")
 
-        self.preload_nanny = preload_nanny
-        if self.preload_nanny is None:
-            self.preload_nanny = dask.config.get("distributed.nanny.preload")
-        self.preload_nanny_argv = preload_nanny_argv
-        if self.preload_nanny_argv is None:
-            self.preload_nanny_argv = dask.config.get("distributed.nanny.preload-argv")
+        if preload_nanny is None:
+            preload_nanny = dask.config.get("distributed.nanny.preload")
+        if preload_nanny_argv is None:
+            preload_nanny_argv = dask.config.get("distributed.nanny.preload-argv")
 
         self.Worker = Worker if worker_class is None else worker_class
         self.env = env or {}
@@ -168,8 +166,8 @@ def __init__(
 
         self.local_directory = local_directory
 
-        self._preload_modules = preloading.on_creation(
-            self.preload_nanny, file_dir=self.local_directory
+        self.preloads = preloading.process_preloads(
+            self, preload_nanny, preload_nanny_argv, file_dir=self.local_directory
         )
 
         self.services = services
@@ -263,9 +261,8 @@ async def start(self):
         )
         self.ip = get_address_host(self.address)
 
-        await preloading.on_start(
-            self._preload_modules, self, argv=self.preload_nanny_argv,
-        )
+        for preload in self.preloads:
+            await preload.start()
 
         logger.info("        Start Nanny at: %r", self.address)
         response = await self.instantiate()
@@ -465,7 +462,8 @@ async def close(self, comm=None, timeout=5, report=None):
         self.status = "closing"
         logger.info("Closing Nanny at %r", self.address)
 
-        await preloading.on_teardown(self._preload_modules, self)
+        for preload in self.preloads:
+            await preload.teardown()
 
         self.stop()
         try:
diff --git a/distributed/preloading.py b/distributed/preloading.py
index 2e0469419b9..8516bc0947c 100644
--- a/distributed/preloading.py
+++ b/distributed/preloading.py
@@ -3,10 +3,13 @@
 import os
 import shutil
 import sys
+from typing import List
+from types import ModuleType
 import filecmp
 from importlib import import_module
 
 import click
+from tornado.httpclient import AsyncHTTPClient
 
 from dask.utils import tmpfile
 
@@ -34,9 +37,9 @@ def validate_preload_argv(ctx, param, value):
     preload_modules = {name: _import_module(name) for name in ctx.params.get("preload")}
 
     preload_commands = [
-        m["dask_setup"]
+        getattr(m, "dask_setup", None)
         for m in preload_modules.values()
-        if isinstance(m["dask_setup"], click.Command)
+        if isinstance(getattr(m, "dask_setup", None), click.Command)
     ]
 
     if len(preload_commands) > 1:
@@ -60,7 +63,11 @@ def validate_preload_argv(ctx, param, value):
     return value
 
 
-def _import_module(name, file_dir=None):
+def is_webaddress(s: str) -> bool:
+    return any(s.startswith(prefix) for prefix in ("http://", "https://"))
+
+
+def _import_module(name, file_dir=None) -> ModuleType:
     """ Imports module and extract preload interface functions.
 
     Import modules specified by name and extract 'dask_setup'
@@ -105,71 +112,81 @@ def _import_module(name, file_dir=None):
             return _import_module(fn, file_dir=file_dir)
 
     logger.info("Import preload module: %s", name)
-    return {
-        attrname: getattr(module, attrname, None)
-        for attrname in ("dask_setup", "dask_teardown")
-    }
+    return module
 
 
-def on_creation(names, file_dir: str = None) -> dict:
-    """ Imports each of the preload modules
+async def _download_module(url: str) -> ModuleType:
+    assert is_webaddress(url)
 
-    Parameters
-    ----------
-    names: list of strings
-        Module names or file paths
-    file_dir: string
-        Path of a directory where files should be copied
-    """
-    if isinstance(names, str):
-        names = [names]
+    client = AsyncHTTPClient()
+    response = await client.fetch(url)
+    source = response.body.decode()
 
-    return {name: _import_module(name, file_dir=file_dir) for name in names}
+    compiled = compile(source, url, "exec")
+    module = ModuleType(url)
+    exec(compiled, module.__dict__)
+    return module
 
 
-async def on_start(modules: dict, dask_server=None, argv=None):
-    """ Run when the server finishes its start method
+class Preload:
+    """
+    Manage state for setup/teardown of a preload module
 
     Parameters
     ----------
-    modules: Dict[str, module]
-        The imported modules, from on_creation
     dask_server: dask.distributed.Server
         The Worker or Scheduler
+    name: str
+        module name, file name, or web address to load
     argv: [string]
         List of string arguments passed to click-configurable `dask_setup`.
     file_dir: string
         Path of a directory where files should be copied
     """
-    for name, interface in modules.items():
-        dask_setup = interface.get("dask_setup", None)
+
+    def __init__(self, dask_server, name: str, argv: List[str], file_dir: str):
+        self.dask_server = dask_server
+        self.name = name
+        self.argv = argv
+        self.file_dir = file_dir
+
+        if not is_webaddress(name):
+            self.module = _import_module(name, file_dir)
+        else:
+            self.module = None
+
+    async def start(self):
+        """ Run when the server finishes its start method """
+        if is_webaddress(self.name):
+            self.module = await _download_module(self.name)
+
+        dask_setup = getattr(self.module, "dask_setup", None)
 
         if dask_setup:
             if isinstance(dask_setup, click.Command):
                 context = dask_setup.make_context(
-                    "dask_setup", list(argv), allow_extra_args=False
+                    "dask_setup", list(self.argv), allow_extra_args=False
                 )
-                dask_setup.callback(dask_server, *context.args, **context.params)
+                dask_setup.callback(self.dask_server, *context.args, **context.params)
             else:
-                future = dask_setup(dask_server)
+                future = dask_setup(self.dask_server)
                 if inspect.isawaitable(future):
                     await future
-                logger.info("Run preload setup function: %s", name)
-
+                logger.info("Run preload setup function: %s", self.name)
 
-async def on_teardown(modules: dict, dask_server=None):
-    """ Run when the server starts its close method
-
-    Parameters
-    ----------
-    modules: Dict[str, module]
-        The imported modules, from on_creation
-    dask_server: dask.distributed.Server
-        The Worker or Scheduler
-    """
-    for name, interface in modules.items():
-        dask_teardown = interface.get("dask_teardown", None)
+    async def teardown(self):
+        """ Run when the server starts its close method """
+        dask_teardown = getattr(self.module, "dask_teardown", None)
         if dask_teardown:
-            future = dask_teardown(dask_server)
+            future = dask_teardown(self.dask_server)
             if inspect.isawaitable(future):
                 await future
+
+
+def process_preloads(
+    dask_server, preload: List[str], preload_argv: List[List], file_dir: str = None
+) -> List[Preload]:
+    if isinstance(preload, str):
+        preload = [preload]
+
+    return [Preload(dask_server, p, preload_argv, file_dir) for p in preload]
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index a117583b66d..584c0db2b28 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1109,9 +1109,7 @@ def __init__(
             preload = dask.config.get("distributed.scheduler.preload")
         if not preload_argv:
             preload_argv = dask.config.get("distributed.scheduler.preload-argv")
-        self.preload = preload
-        self.preload_argv = preload_argv
-        self._preload_modules = preloading.on_creation(self.preload)
+        self.preloads = preloading.process_preloads(self, preload, preload_argv)
 
         self.security = security or Security()
         assert isinstance(self.security, Security)
@@ -1474,7 +1472,8 @@ def del_scheduler_file():
 
             weakref.finalize(self, del_scheduler_file)
 
-        await preloading.on_start(self._preload_modules, self, argv=self.preload_argv)
+        for preload in self.preloads:
+            await preload.start()
 
         await asyncio.gather(*[plugin.start(self) for plugin in self.plugins])
 
@@ -1498,7 +1497,8 @@ async def close(self, comm=None, fast=False, close_workers=False):
         logger.info("Scheduler closing...")
         setproctitle("dask-scheduler [closing]")
 
-        await preloading.on_teardown(self._preload_modules, self)
+        for preload in self.preloads:
+            await preload.teardown()
 
         if close_workers:
             await self.broadcast(msg={"op": "close_gracefully"}, nanny=True)
diff --git a/distributed/tests/test_preload.py b/distributed/tests/test_preload.py
index 4f60ca586f9..0000cd94206 100644
--- a/distributed/tests/test_preload.py
+++ b/distributed/tests/test_preload.py
@@ -5,6 +5,8 @@
 import tempfile
 import pytest
 
+from tornado import web
+
 import dask
 from distributed import Client, Scheduler, Worker, Nanny
 from distributed.utils_test import cluster
@@ -121,3 +123,23 @@ async def test_preload_import_time(cleanup):
         from distributed.comm.registry import backends
 
         del backends["foo"]
+
+
+@pytest.mark.asyncio
+async def test_web_preload(cleanup):
+    class MyHandler(web.RequestHandler):
+        def get(self):
+            self.write(
+                """
+def dask_setup(dask_server):
+    dask_server.foo = 1
+""".strip()
+            )
+
+    app = web.Application([(r"/preload", MyHandler)])
+    server = app.listen(12345)
+    try:
+        async with Scheduler(preload=["http://localhost:12345/preload"]) as s:
+            assert s.foo == 1
+    finally:
+        server.stop()
diff --git a/distributed/worker.py b/distributed/worker.py
index ef95c1f4b7f..33fc7f5dd10 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -498,14 +498,12 @@ def __init__(
             self._workdir = self._workspace.new_work_dir(prefix="worker-")
             self.local_directory = self._workdir.dir_path
 
-        self.preload = preload
-        if self.preload is None:
-            self.preload = dask.config.get("distributed.worker.preload")
-        self.preload_argv = preload_argv
-        if self.preload_argv is None:
-            self.preload_argv = dask.config.get("distributed.worker.preload-argv")
-        self._preload_modules = preloading.on_creation(
-            self.preload, file_dir=self.local_directory
+        if preload is None:
+            preload = dask.config.get("distributed.worker.preload")
+        if preload_argv is None:
+            preload_argv = dask.config.get("distributed.worker.preload-argv")
+        self.preloads = preloading.process_preloads(
+            self, preload, preload_argv, file_dir=self.local_directory
         )
 
         self.security = security or Security()
@@ -1025,9 +1023,8 @@ async def start(self):
         if self.name is None:
             self.name = self.address
 
-        await preloading.on_start(
-            self._preload_modules, self, argv=self.preload_argv,
-        )
+        for preload in self.preloads:
+            await preload.start()
 
         # Services listen on all addresses
         # Note Nanny is not a "real" service, just some metadata
@@ -1085,7 +1082,8 @@ async def close(
                 logger.info("Closed worker has not yet started: %s", self.status)
             self.status = "closing"
 
-            await preloading.on_teardown(self._preload_modules, self)
+            for preload in self.preloads:
+                await preload.teardown()
 
             if nanny and self.nanny:
                 with self.rpc(self.nanny) as r:

From d8f46d502a561bd86cc5c08eec0fd906a4ba700d Mon Sep 17 00:00:00 2001
From: Peter Andreas Entschev <peter@entschev.com>
Date: Sat, 2 May 2020 00:23:35 +0200
Subject: [PATCH 0819/1550] Add UCX support for RDMACM (#3759)

* Add RDMACM support

* Add UCX's rdmacm to tests

* Add rdmacm UCX config to distributed-schema.yaml
---
 distributed/comm/tests/test_ucx_config.py | 8 +++++++-
 distributed/comm/ucx.py                   | 8 ++++++--
 distributed/distributed-schema.yaml       | 4 ++++
 distributed/distributed.yaml              | 1 +
 4 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/distributed/comm/tests/test_ucx_config.py b/distributed/comm/tests/test_ucx_config.py
index c2e86ed0b49..43eb45acb23 100644
--- a/distributed/comm/tests/test_ucx_config.py
+++ b/distributed/comm/tests/test_ucx_config.py
@@ -23,6 +23,7 @@ async def test_ucx_config(cleanup):
     ucx = {
         "nvlink": True,
         "infiniband": True,
+        "rdmacm": False,
         "net-devices": "",
         "tcp": True,
         "cuda_copy": True,
@@ -31,11 +32,13 @@ async def test_ucx_config(cleanup):
     with dask.config.set(ucx=ucx):
         ucx_config = _scrub_ucx_config()
         assert ucx_config.get("TLS") == "rc,tcp,sockcm,cuda_copy,cuda_ipc"
+        assert ucx_config.get("SOCKADDR_TLS_PRIORITY") == "sockcm"
         assert ucx_config.get("NET_DEVICES") is None
 
     ucx = {
         "nvlink": False,
         "infiniband": True,
+        "rdmacm": False,
         "net-devices": "mlx5_0:1",
         "tcp": True,
         "cuda_copy": False,
@@ -44,11 +47,13 @@ async def test_ucx_config(cleanup):
     with dask.config.set(ucx=ucx):
         ucx_config = _scrub_ucx_config()
         assert ucx_config.get("TLS") == "rc,tcp,sockcm"
+        assert ucx_config.get("SOCKADDR_TLS_PRIORITY") == "sockcm"
         assert ucx_config.get("NET_DEVICES") == "mlx5_0:1"
 
     ucx = {
         "nvlink": False,
         "infiniband": True,
+        "rdmacm": True,
         "net-devices": "all",
         "MEMTYPE_CACHE": "y",
         "tcp": True,
@@ -57,7 +62,8 @@ async def test_ucx_config(cleanup):
 
     with dask.config.set(ucx=ucx):
         ucx_config = _scrub_ucx_config()
-        assert ucx_config.get("TLS") == "rc,tcp,sockcm,cuda_copy"
+        assert ucx_config.get("TLS") == "rc,tcp,rdmacm,cuda_copy"
+        assert ucx_config.get("SOCKADDR_TLS_PRIORITY") == "rdmacm"
         assert ucx_config.get("MEMTYPE_CACHE") == "y"
 
 
diff --git a/distributed/comm/ucx.py b/distributed/comm/ucx.py
index 7761afef7a1..22bf4ad52f3 100644
--- a/distributed/comm/ucx.py
+++ b/distributed/comm/ucx.py
@@ -424,8 +424,12 @@ def _scrub_ucx_config():
             dask.config.get("ucx.infiniband"),
         ]
     ):
-        tls = "tcp,sockcm"
-        tls_priority = "sockcm"
+        if dask.config.get("ucx.rdmacm"):
+            tls = "tcp,rdmacm"
+            tls_priority = "rdmacm"
+        else:
+            tls = "tcp,sockcm"
+            tls_priority = "sockcm"
 
         # CUDA COPY can optionally be used with ucx -- we rely on the user
         # to define when messages will include CUDA objects.  Note:
diff --git a/distributed/distributed-schema.yaml b/distributed/distributed-schema.yaml
index 60dfbf54e05..f97d900e4be 100644
--- a/distributed/distributed-schema.yaml
+++ b/distributed/distributed-schema.yaml
@@ -785,6 +785,10 @@ properties:
         type:
         - boolean
         - "null"
+      rdmacm:
+        type:
+        - boolean
+        - "null"
       cuda_copy:
         type:
         - boolean
diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index 4103d592e2b..db42c21d3d8 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -176,5 +176,6 @@ ucx:
   tcp: null  # enable tcp
   nvlink: null  # enable cuda_ipc
   infiniband: null # enable Infiniband
+  rdmacm: null # enable RDMACM
   cuda_copy: null  # enable cuda-copy
   net-devices: null  # define which Infiniband device to use

From 8c6856ec0e4514ca05d654162ea16482ea9c5e9b Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Mon, 4 May 2020 12:17:37 -0500
Subject: [PATCH 0820/1550] Allow range of ports to be specified for Workers
 (#3704)

---
 distributed/cli/dask_worker.py            | 23 +++---
 distributed/cli/tests/test_dask_worker.py | 62 ++++++++++++++++-
 distributed/nanny.py                      | 45 +++++++++---
 distributed/tests/test_nanny.py           | 30 +++++++-
 distributed/tests/test_utils.py           | 11 +++
 distributed/tests/test_worker.py          | 22 +++++-
 distributed/utils.py                      | 58 ++++++++++++++++
 distributed/worker.py                     | 85 +++++++++++++++--------
 8 files changed, 284 insertions(+), 52 deletions(-)

diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
index 9d73f7af5b2..da809d83ac2 100755
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -52,12 +52,21 @@
 )
 @click.option(
     "--worker-port",
-    type=int,
-    default=0,
-    help="Serving computation port, defaults to random",
+    default=None,
+    help="Serving computation port, defaults to random. "
+    "When creating multiple workers with --nprocs, a sequential range of "
+    "worker ports may be used by specifying the first and last available "
+    "ports like <first-port>:<last-port>. For example, --worker-port=3000:3026 "
+    "will use ports 3000, 3001, ..., 3025, 3026.",
 )
 @click.option(
-    "--nanny-port", type=int, default=0, help="Serving nanny port, defaults to random"
+    "--nanny-port",
+    default=None,
+    help="Serving nanny port, defaults to random. "
+    "When creating multiple nannies with --nprocs, a sequential range of "
+    "nanny ports may be used by specifying the first and last available "
+    "ports like <first-port>:<last-port>. For example, --nanny-port=3000:3026 "
+    "will use ports 3000, 3001, ..., 3025, 3026.",
 )
 @click.option(
     "--bokeh-port", type=int, default=None, help="Deprecated.  See --dashboard-address"
@@ -281,12 +290,6 @@ def main(
         }
     )
 
-    if nprocs > 1 and worker_port != 0:
-        logger.error(
-            "Failed to launch worker.  You cannot use the --port argument when nprocs > 1."
-        )
-        sys.exit(1)
-
     if nprocs > 1 and not nanny:
         logger.error(
             "Failed to launch worker.  You cannot use the --no-nanny argument when nprocs > 1."
diff --git a/distributed/cli/tests/test_dask_worker.py b/distributed/cli/tests/test_dask_worker.py
index 767613f2a26..4d252a51aa1 100644
--- a/distributed/cli/tests/test_dask_worker.py
+++ b/distributed/cli/tests/test_dask_worker.py
@@ -12,7 +12,7 @@
 import distributed.cli.dask_worker
 from distributed import Client, Scheduler
 from distributed.metrics import time
-from distributed.utils import sync, tmpfile
+from distributed.utils import sync, tmpfile, parse_ports
 from distributed.utils_test import popen, terminate_process, wait_for_port
 from distributed.utils_test import loop, cleanup  # noqa: F401
 
@@ -47,6 +47,66 @@ def test_nanny_worker_ports(loop):
                 )
 
 
+def test_nanny_worker_port_range(loop):
+    with popen(["dask-scheduler", "--port", "9359", "--no-dashboard"]) as sched:
+        nprocs = 3
+        worker_port = "9684:9686"
+        nanny_port = "9688:9690"
+        with popen(
+            [
+                "dask-worker",
+                "127.0.0.1:9359",
+                "--nprocs",
+                f"{nprocs}",
+                "--host",
+                "127.0.0.1",
+                "--worker-port",
+                worker_port,
+                "--nanny-port",
+                nanny_port,
+                "--no-dashboard",
+            ]
+        ) as worker:
+            with Client("127.0.0.1:9359", loop=loop) as c:
+                start = time()
+                while len(c.scheduler_info()["workers"]) < nprocs:
+                    sleep(0.1)
+                    assert time() - start < 5
+
+                def get_port(dask_worker):
+                    return dask_worker.port
+
+                expected_worker_ports = set(parse_ports(worker_port))
+                worker_ports = c.run(get_port)
+                assert set(worker_ports.values()) == expected_worker_ports
+
+                expected_nanny_ports = set(parse_ports(nanny_port))
+                nanny_ports = c.run(get_port, nanny=True)
+                assert set(nanny_ports.values()) == expected_nanny_ports
+
+
+def test_nanny_worker_port_range_too_many_workers_raises(loop):
+    with popen(["dask-scheduler", "--port", "9359", "--no-dashboard"]) as sched:
+        with popen(
+            [
+                "dask-worker",
+                "127.0.0.1:9359",
+                "--nprocs",
+                "3",
+                "--host",
+                "127.0.0.1",
+                "--worker-port",
+                "9684:9685",
+                "--nanny-port",
+                "9686:9687",
+                "--no-dashboard",
+            ]
+        ) as worker:
+            assert any(
+                b"Could not start" in worker.stderr.readline() for _ in range(100)
+            )
+
+
 def test_memory_limit(loop):
     with popen(["dask-scheduler", "--no-dashboard"]) as sched:
         with popen(
diff --git a/distributed/nanny.py b/distributed/nanny.py
index 4807073e08c..821031e29eb 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -1,4 +1,5 @@
 import asyncio
+import errno
 import logging
 from multiprocessing.queues import Empty
 import os
@@ -29,6 +30,7 @@
     silence_logging,
     json_load_robust,
     parse_timedelta,
+    parse_ports,
     ignoring,
     TimeoutError,
 )
@@ -90,7 +92,7 @@ def __init__(
         port=None,
         protocol=None,
         config=None,
-        **worker_kwargs
+        **worker_kwargs,
     ):
         self._setup_logging(logger)
         self.loop = loop or IOLoop.current()
@@ -209,13 +211,10 @@ def __init__(
         ):
             host = get_ip(get_address_host(self.scheduler.address))
 
-        self._start_address = address_from_user_args(
-            host=host,
-            port=port,
-            interface=interface,
-            protocol=protocol,
-            security=security,
-        )
+        self._start_port = port
+        self._start_host = host
+        self._interface = interface
+        self._protocol = protocol
 
         self._listen_address = listen_address
         Nanny._instances.add(self)
@@ -256,9 +255,33 @@ async def start(self):
 
         await super().start()
 
-        await self.listen(
-            self._start_address, **self.security.get_listen_args("worker")
-        )
+        ports = parse_ports(self._start_port)
+        for port in ports:
+            start_address = address_from_user_args(
+                host=self._start_host,
+                port=port,
+                interface=self._interface,
+                protocol=self._protocol,
+                security=self.security,
+            )
+            try:
+                await self.listen(
+                    start_address, **self.security.get_listen_args("worker")
+                )
+            except OSError as e:
+                if len(ports) > 1 and e.errno == errno.EADDRINUSE:
+                    continue
+                else:
+                    raise e
+            else:
+                self._start_address = start_address
+                break
+        else:
+            raise ValueError(
+                f"Could not start Nanny on host {self._start_host}"
+                f"with port {self._start_port}"
+            )
+
         self.ip = get_address_host(self.address)
 
         for preload in self.preloads:
diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
index 4bf8144956f..a02fb3c6211 100644
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -18,7 +18,7 @@
 from distributed.core import CommClosedError
 from distributed.metrics import time
 from distributed.protocol.pickle import dumps
-from distributed.utils import ignoring, tmpfile, TimeoutError
+from distributed.utils import ignoring, tmpfile, TimeoutError, parse_ports
 from distributed.utils_test import (  # noqa: F401
     gen_cluster,
     gen_test,
@@ -504,6 +504,34 @@ async def test_config(cleanup):
                 assert config[n.worker_address] == "bar"
 
 
+@pytest.mark.asyncio
+async def test_nanny_port_range(cleanup):
+    async with Scheduler() as s:
+        async with Client(s.address, asynchronous=True) as client:
+            nanny_port = "9867:9868"
+            worker_port = "9869:9870"
+            async with Nanny(s.address, port=nanny_port, worker_port=worker_port) as n1:
+                assert n1.port == 9867  # Selects first port in range
+                async with Nanny(
+                    s.address, port=nanny_port, worker_port=worker_port
+                ) as n2:
+                    assert n2.port == 9868  # Selects next port in range
+                    with pytest.raises(
+                        ValueError, match="Could not start Nanny"
+                    ):  # No more ports left
+                        async with Nanny(
+                            s.address, port=nanny_port, worker_port=worker_port
+                        ):
+                            pass
+
+                    # Ensure Worker ports are in worker_port range
+                    def get_worker_port(dask_worker):
+                        return dask_worker.port
+
+                    worker_ports = await client.run(get_worker_port)
+                    assert list(worker_ports.values()) == parse_ports(worker_port)
+
+
 class KeyboardInterruptWorker(worker.Worker):
     """A Worker that raises KeyboardInterrupt almost immediately"""
 
diff --git a/distributed/tests/test_utils.py b/distributed/tests/test_utils.py
index 86f1ca0c208..74cd051ff81 100644
--- a/distributed/tests/test_utils.py
+++ b/distributed/tests/test_utils.py
@@ -40,6 +40,7 @@
     LoopRunner,
     parse_bytes,
     parse_timedelta,
+    parse_ports,
     warn_on_duration,
     format_dashboard_link,
     LRU,
@@ -597,6 +598,16 @@ def test_format_dashboard_link():
         del os.environ["host"]
 
 
+def test_parse_ports():
+    assert parse_ports(None) == [None]
+    assert parse_ports(23) == [23]
+    assert parse_ports("45") == [45]
+    assert parse_ports("100:103") == [100, 101, 102, 103]
+
+    with pytest.raises(ValueError, match="port_stop must be greater than port_start"):
+        parse_ports("103:100")
+
+
 def test_lru():
 
     l = LRU(maxsize=3)
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 923a138cb60..aabb76c75d6 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -30,7 +30,12 @@
 from distributed.core import rpc, CommClosedError
 from distributed.scheduler import Scheduler
 from distributed.metrics import time
-from distributed.worker import Worker, error_message, logger, parse_memory_limit
+from distributed.worker import (
+    Worker,
+    error_message,
+    logger,
+    parse_memory_limit,
+)
 from distributed.utils import tmpfile, TimeoutError
 from distributed.utils_test import (  # noqa: F401
     cleanup,
@@ -313,6 +318,21 @@ async def test_worker_with_port_zero():
     await w.close()
 
 
+@pytest.mark.asyncio
+async def test_worker_port_range(cleanup):
+    async with Scheduler() as s:
+        port = "9867:9868"
+        async with Worker(s.address, port=port) as w1:
+            assert w1.port == 9867  # Selects first port in range
+            async with Worker(s.address, port=port) as w2:
+                assert w2.port == 9868  # Selects next port in range
+                with pytest.raises(
+                    ValueError, match="Could not start Worker"
+                ):  # No more ports left
+                    async with Worker(s.address, port=port):
+                        pass
+
+
 @pytest.mark.slow
 @pytest.mark.asyncio
 async def test_worker_waits_for_scheduler(cleanup):
diff --git a/distributed/utils.py b/distributed/utils.py
index ea333833b08..ec98617254f 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -1319,6 +1319,64 @@ def format_dashboard_link(host, port):
     )
 
 
+def parse_ports(port):
+    """ Parse input port information into list of ports
+
+    Parameters
+    ----------
+    port : int, str, None
+        Input port or ports. Can be an integer like 8787, a string for a
+        single port like "8787", a string for a sequential range of ports like
+        "8000:8200", or None.
+
+    Returns
+    -------
+    ports : list
+        List of ports
+
+    Examples
+    --------
+    A single port can be specified using an integer:
+
+    >>> parse_ports(8787)
+    >>> [8787]
+
+    or a string:
+
+    >>> parse_ports("8787")
+    >>> [8787]
+
+    A sequential range of ports can be specified by a string which indicates
+    the first and last ports which should be included in the sequence of ports:
+
+    >>> parse_ports("8787:8790")
+    >>> [8787, 8788, 8789, 8790]
+
+    An input of ``None`` is also valid and can be used to indicate that no port
+    has been specified:
+
+    >>> parse_ports(None)
+    >>> [None]
+
+    """
+    if isinstance(port, str) and ":" not in port:
+        port = int(port)
+
+    if isinstance(port, (int, type(None))):
+        ports = [port]
+    else:
+        port_start, port_stop = map(int, port.split(":"))
+        if port_stop <= port_start:
+            raise ValueError(
+                "When specifying a range of ports like port_start:port_stop, "
+                "port_stop must be greater than port_start, but got "
+                f"port_start={port_start} and port_stop={port_stop}"
+            )
+        ports = list(range(port_start, port_stop + 1))
+
+    return ports
+
+
 def is_coroutine_function(f):
     return asyncio.iscoroutinefunction(f) or gen.is_coroutine_function(f)
 
diff --git a/distributed/worker.py b/distributed/worker.py
index 33fc7f5dd10..63fd5c6daa4 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -3,6 +3,7 @@
 from collections import defaultdict, deque, namedtuple
 from collections.abc import MutableMapping
 from datetime import timedelta
+import errno
 from functools import partial
 import heapq
 from inspect import isawaitable
@@ -57,6 +58,7 @@
     offload,
     parse_bytes,
     parse_timedelta,
+    parse_ports,
     iscoroutinefunction,
     warn_on_duration,
     LRU,
@@ -328,7 +330,7 @@ def __init__(
         lifetime=None,
         lifetime_stagger=None,
         lifetime_restart=None,
-        **kwargs
+        **kwargs,
     ):
         self.tasks = dict()
         self.task_state = dict()
@@ -456,13 +458,10 @@ def __init__(
         if not host and not interface and not scheduler_addr.startswith("inproc://"):
             host = get_ip(get_address_host(scheduler_addr.split("://")[-1]))
 
-        self._start_address = address_from_user_args(
-            host=host,
-            port=port,
-            interface=interface,
-            protocol=protocol,
-            security=security,
-        )
+        self._start_port = port
+        self._start_host = host
+        self._interface = interface
+        self._protocol = protocol
 
         if ncores is not None:
             warnings.warn("the ncores= parameter has moved to nthreads=")
@@ -584,22 +583,9 @@ def __init__(
         self.services = {}
         self.service_specs = services or {}
 
-        routes = get_handlers(
-            server=self,
-            modules=dask.config.get("distributed.worker.http.routes"),
-            prefix=http_prefix,
-        )
-        self.start_http_server(routes, dashboard_address)
-
-        if dashboard:
-            try:
-                import distributed.dashboard.worker
-            except ImportError:
-                logger.debug("To start diagnostics web server please install Bokeh")
-            else:
-                distributed.dashboard.worker.connect(
-                    self.http_application, self.http_server, self, prefix=http_prefix,
-                )
+        self._dashboard_address = dashboard_address
+        self._dashboard = dashboard
+        self._http_prefix = http_prefix
 
         self.metrics = dict(metrics) if metrics else {}
         self.startup_information = (
@@ -643,7 +629,7 @@ def __init__(
             stream_handlers=stream_handlers,
             io_loop=self.loop,
             connection_args=self.connection_args,
-            **kwargs
+            **kwargs,
         )
 
         self.scheduler = self.rpc(scheduler_addr)
@@ -1015,9 +1001,52 @@ async def start(self):
         enable_gc_diagnosis()
         thread_state.on_event_loop_thread = True
 
-        await self.listen(
-            self._start_address, **self.security.get_listen_args("worker")
+        ports = parse_ports(self._start_port)
+        for port in ports:
+            start_address = address_from_user_args(
+                host=self._start_host,
+                port=port,
+                interface=self._interface,
+                protocol=self._protocol,
+                security=self.security,
+            )
+            try:
+                await self.listen(
+                    start_address, **self.security.get_listen_args("worker")
+                )
+            except OSError as e:
+                if len(ports) > 1 and e.errno == errno.EADDRINUSE:
+                    continue
+                else:
+                    raise e
+            else:
+                self._start_address = start_address
+                break
+        else:
+            raise ValueError(
+                f"Could not start Worker on host {self._start_host}"
+                f"with port {self._start_port}"
+            )
+
+        # Start HTTP server associated with this Worker node
+        routes = get_handlers(
+            server=self,
+            modules=dask.config.get("distributed.worker.http.routes"),
+            prefix=self._http_prefix,
         )
+        self.start_http_server(routes, self._dashboard_address)
+        if self._dashboard:
+            try:
+                import distributed.dashboard.worker
+            except ImportError:
+                logger.debug("To start diagnostics web server please install Bokeh")
+            else:
+                distributed.dashboard.worker.connect(
+                    self.http_application,
+                    self.http_server,
+                    self,
+                    prefix=self._http_prefix,
+                )
         self.ip = get_address_host(self.address)
 
         if self.name is None:
@@ -1355,7 +1384,7 @@ def add_task(
         duration=None,
         resource_restrictions=None,
         actor=False,
-        **kwargs2
+        **kwargs2,
     ):
         try:
             if key in self.tasks:

From f96d6be114f4c6fed91d4224c91702fd2dda8545 Mon Sep 17 00:00:00 2001
From: Rami Chowdhury <460769+necaris@users.noreply.github.com>
Date: Tue, 5 May 2020 00:24:30 -0400
Subject: [PATCH 0821/1550] Don't validate http preloads locally (#3768)

---
 distributed/cli/tests/test_dask_scheduler.py | 22 ++++++++++++++++++++
 distributed/preloading.py                    |  6 +++++-
 2 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/distributed/cli/tests/test_dask_scheduler.py b/distributed/cli/tests/test_dask_scheduler.py
index 6f4129514b9..3a08f319312 100644
--- a/distributed/cli/tests/test_dask_scheduler.py
+++ b/distributed/cli/tests/test_dask_scheduler.py
@@ -241,6 +241,7 @@ def test_dashboard_port_zero(loop):
 
 def dask_setup(scheduler):
     _scheduler_info['address'] = scheduler.address
+    scheduler.foo = "bar"
 
 def get_scheduler_address():
     return _scheduler_info['address']
@@ -299,6 +300,27 @@ def check_scheduler():
         shutil.rmtree(tmpdir)
 
 
+def test_preload_remote_module(loop, tmpdir):
+    with open(tmpdir / "scheduler_info.py", "w") as f:
+        f.write(PRELOAD_TEXT)
+
+    with popen(["python", "-m", "http.server", "9382"], cwd=tmpdir):
+        with popen(
+            [
+                "dask-scheduler",
+                "--scheduler-file",
+                tmpdir / "scheduler-file.json",
+                "--preload",
+                "http://localhost:9382/scheduler_info.py",
+            ],
+        ) as proc:
+            with Client(scheduler_file=tmpdir / "scheduler-file.json", loop=loop) as c:
+                assert (
+                    c.run_on_scheduler(lambda dask_scheduler: dask_scheduler.foo)
+                    == "bar"
+                )
+
+
 PRELOAD_COMMAND_TEXT = """
 import click
 _config = {}
diff --git a/distributed/preloading.py b/distributed/preloading.py
index 8516bc0947c..112f537e2ca 100644
--- a/distributed/preloading.py
+++ b/distributed/preloading.py
@@ -34,7 +34,11 @@ def validate_preload_argv(ctx, param, value):
             % ("s" if len(value) > 1 else "", " ".join(value))
         )
 
-    preload_modules = {name: _import_module(name) for name in ctx.params.get("preload")}
+    preload_modules = {
+        name: _import_module(name)
+        for name in ctx.params.get("preload")
+        if not is_webaddress(name)
+    }
 
     preload_commands = [
         getattr(m, "dask_setup", None)

From d0f6aec2d8c364075ddf468756f4ac6e238eee4b Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Tue, 5 May 2020 06:27:05 +0200
Subject: [PATCH 0822/1550] Fix broadcast for TLS comms (#3766)

---
 distributed/scheduler.py            | 11 ++++++-----
 distributed/tests/test_scheduler.py | 13 +++++++++++++
 2 files changed, 19 insertions(+), 5 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 584c0db2b28..5c50fd4d0f2 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -45,7 +45,7 @@
     unparse_host_port,
 )
 from .comm.addressing import addresses_from_user_args
-from .core import rpc, connect, send_recv, clean_exception, CommClosedError
+from .core import rpc, send_recv, clean_exception, CommClosedError
 from .diagnostics.plugin import SchedulerPlugin
 
 from .http import get_handlers
@@ -2973,11 +2973,12 @@ async def broadcast(
             addresses = workers
 
         async def send_message(addr):
-            comm = await connect(
-                addr, deserialize=self.deserialize, connection_args=self.connection_args
-            )
+            comm = await self.rpc.connect(addr)
             comm.name = "Scheduler Broadcast"
-            resp = await send_recv(comm, close=True, serializers=serializers, **msg)
+            try:
+                resp = await send_recv(comm, close=True, serializers=serializers, **msg)
+            finally:
+                self.rpc.reuse(addr, comm)
             return resp
 
         results = await All(
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 5ed8e4e542d..89df2757f88 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -37,6 +37,7 @@
     cluster,
     div,
     varying,
+    tls_only_security,
 )
 from distributed.utils_test import loop, nodebug  # noqa: F401
 from dask.compatibility import apply
@@ -523,6 +524,18 @@ async def test_broadcast(s, a, b):
     assert result == {a.address: b"pong", b.address: b"pong"}
 
 
+@gen_cluster(security=tls_only_security(),)
+async def test_broadcast_tls(s, a, b):
+    result = await s.broadcast(msg={"op": "ping"})
+    assert result == {a.address: b"pong", b.address: b"pong"}
+
+    result = await s.broadcast(msg={"op": "ping"}, workers=[a.address])
+    assert result == {a.address: b"pong"}
+
+    result = await s.broadcast(msg={"op": "ping"}, hosts=[a.ip])
+    assert result == {a.address: b"pong", b.address: b"pong"}
+
+
 @gen_cluster(Worker=Nanny)
 async def test_broadcast_nanny(s, a, b):
     result1 = await s.broadcast(msg={"op": "identity"}, nanny=True)

From 3e47fa08e8e4c6a6ab991c8297a826ac541f8a18 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Wed, 6 May 2020 18:31:56 -0500
Subject: [PATCH 0823/1550] Attempt to fix test_preload_remote_module on
 windows (#3775)

---
 distributed/cli/tests/test_dask_scheduler.py | 26 +++++++++++++-------
 1 file changed, 17 insertions(+), 9 deletions(-)

diff --git a/distributed/cli/tests/test_dask_scheduler.py b/distributed/cli/tests/test_dask_scheduler.py
index 3a08f319312..1885cb84a85 100644
--- a/distributed/cli/tests/test_dask_scheduler.py
+++ b/distributed/cli/tests/test_dask_scheduler.py
@@ -300,25 +300,33 @@ def check_scheduler():
         shutil.rmtree(tmpdir)
 
 
-def test_preload_remote_module(loop, tmpdir):
-    with open(tmpdir / "scheduler_info.py", "w") as f:
+def test_preload_remote_module(loop, tmp_path):
+    with open(tmp_path / "scheduler_info.py", "w") as f:
         f.write(PRELOAD_TEXT)
 
-    with popen(["python", "-m", "http.server", "9382"], cwd=tmpdir):
+    with popen([sys.executable, "-m", "http.server", "9382"], cwd=tmp_path):
         with popen(
             [
                 "dask-scheduler",
                 "--scheduler-file",
-                tmpdir / "scheduler-file.json",
+                str(tmp_path / "scheduler-file.json"),
                 "--preload",
                 "http://localhost:9382/scheduler_info.py",
             ],
         ) as proc:
-            with Client(scheduler_file=tmpdir / "scheduler-file.json", loop=loop) as c:
-                assert (
-                    c.run_on_scheduler(lambda dask_scheduler: dask_scheduler.foo)
-                    == "bar"
-                )
+            with Client(
+                scheduler_file=tmp_path / "scheduler-file.json", loop=loop
+            ) as c:
+                for i in range(10):
+                    val = c.run_on_scheduler(
+                        lambda dask_scheduler: getattr(dask_scheduler, "foo", None)
+                    )
+                    if val == "bar":
+                        break
+                    else:
+                        sleep(0.1)
+                else:
+                    raise ValueError(val)
 
 
 PRELOAD_COMMAND_TEXT = """

From f98e267182b0671d788afa8da2e21c860098b286 Mon Sep 17 00:00:00 2001
From: Julia Signell <jsignell@gmail.com>
Date: Thu, 7 May 2020 12:23:34 -0400
Subject: [PATCH 0824/1550] Use bokeh column for /system instead of custom css
 (#3781)

---
 distributed/dashboard/components/scheduler.py |  5 ++--
 distributed/http/static/css/system.css        | 26 -------------------
 distributed/http/templates/system.html        | 18 -------------
 3 files changed, 2 insertions(+), 47 deletions(-)
 delete mode 100644 distributed/http/static/css/system.css
 delete mode 100644 distributed/http/templates/system.html

diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index 26e60c55bce..e1410244f02 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -1667,9 +1667,8 @@ def systemmonitor_doc(scheduler, extra, doc):
         doc.title = "Dask: Scheduler System Monitor"
         add_periodic_callback(doc, sysmon, 500)
 
-        for subdoc in sysmon.root.children:
-            doc.add_root(subdoc)
-        doc.template = env.get_template("system.html")
+        doc.add_root(sysmon.root)
+        doc.template = env.get_template("simple.html")
         doc.template_variables.update(extra)
         doc.theme = BOKEH_THEME
 
diff --git a/distributed/http/static/css/system.css b/distributed/http/static/css/system.css
deleted file mode 100644
index 988897aacc0..00000000000
--- a/distributed/http/static/css/system.css
+++ /dev/null
@@ -1,26 +0,0 @@
-#system-fluid {
-    display: flex;
-    flex-wrap: wrap;
-    height: 100%;
-}
-
-/* Small layout: stack all graphs on top of each other, space split equally */
-@media (min-width: 0px) {
-    #system-fluid {
-        flex-direction: column;
-    }
-    #system-fluid .system-item {
-        flex: 1;
-    }
-}
-
-/* Large layout: as many rows as necessary, each item consuming at least
- * half of the width */
-@media (min-width: 992px) {
-    #system-fluid {
-        flex-direction: row;
-    }
-    #system-fluid .system-item {
-        flex: 1 50%;
-    }
-}
diff --git a/distributed/http/templates/system.html b/distributed/http/templates/system.html
deleted file mode 100644
index dfa8b898451..00000000000
--- a/distributed/http/templates/system.html
+++ /dev/null
@@ -1,18 +0,0 @@
-{% extends "base.html" %}
-
-{% block extra_resources %}
-<link rel="stylesheet" href="statics/css/system.css"/>
-{% endblock %}
-
-{% block content %}
-{% from macros import embed %}
-<div id="system-fluid">
-  {% for plot in roots %}
-    <div class="system-item">
-      {{ embed(plot) }}
-    </div>
-  {% endfor %}
-</div>
-{{ plot_script }}
-{% endblock %}
-

From 6e90128ba214f0be1b1d5e0eb1d2095c235cd0a8 Mon Sep 17 00:00:00 2001
From: Julia Signell <jsignell@gmail.com>
Date: Thu, 7 May 2020 12:41:19 -0400
Subject: [PATCH 0825/1550] Remove dev-requirements since it is unused (#3782)

---
 dev-requirements.txt | 13 -------------
 1 file changed, 13 deletions(-)
 delete mode 100644 dev-requirements.txt

diff --git a/dev-requirements.txt b/dev-requirements.txt
deleted file mode 100644
index a367f706e76..00000000000
--- a/dev-requirements.txt
+++ /dev/null
@@ -1,13 +0,0 @@
-joblib >= 0.10.2
-pandas >= 0.19.2
-numpy >= 1.11.0
-bokeh >= 0.12.3
-requests >= 2.12.4
-pyzmq >= 16.0.2
-ipython >= 5.0.0
-jupyter_client >= 4.4.0
-ipykernel >= 4.5.2
-pytest >= 3.2
-prometheus_client >= 0.6.0
-jupyter-server-proxy >= 1.1.0
-pytest-asyncio

From 77f6c55f2482544871cf38b3730fb6f902bf2682 Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Thu, 7 May 2020 20:14:34 +0100
Subject: [PATCH 0826/1550] Client.get_dataset to always create Futures
 attached to itself (#3729)

Adds a `contextvar` for storing the current client, which allows
calls that load a global client (like deserializing futures) to always
get the correct client for the context.
---
 .github/workflows/ci-windows.yaml        |   7 ++
 continuous_integration/travis/install.sh |   4 +
 distributed/client.py                    |  72 ++++++++---
 distributed/lock.py                      |   8 +-
 distributed/publish.py                   |  59 +++++++--
 distributed/queues.py                    |   6 +-
 distributed/tests/test_client.py         | 154 +++++++++++++++++++----
 distributed/tests/test_publish.py        |  49 ++++++++
 distributed/variable.py                  |   6 +-
 distributed/worker.py                    |   9 +-
 requirements.txt                         |   1 +
 11 files changed, 316 insertions(+), 59 deletions(-)

diff --git a/.github/workflows/ci-windows.yaml b/.github/workflows/ci-windows.yaml
index 2e536a79663..7cff9083064 100644
--- a/.github/workflows/ci-windows.yaml
+++ b/.github/workflows/ci-windows.yaml
@@ -23,6 +23,13 @@ jobs:
           activate-environment: dask-distributed
           auto-activate-base: false
 
+      - name: Install contextvars
+        shell: bash -l {0}
+        run: |
+          if [[ "${{ matrix.python-version }}" = "3.6" ]]; then
+            conda install -c conda-forge contextvars
+          fi
+
       - name: Install tornado
         shell: bash -l {0}
         run: |
diff --git a/continuous_integration/travis/install.sh b/continuous_integration/travis/install.sh
index e362ea7f079..8cc862cfadd 100644
--- a/continuous_integration/travis/install.sh
+++ b/continuous_integration/travis/install.sh
@@ -67,6 +67,10 @@ conda create -n dask-distributed -c conda-forge -c defaults \
 
 source activate dask-distributed
 
+if [[ $PYTHON == 3.6 ]]; then
+  conda install -c conda-forge -c defaults contextvars
+fi
+
 # stacktrace is not currently avaiable for Python 3.8.
 # Remove the version check block below when it is avaiable.
 if [[ $PYTHON != 3.8 ]]; then
diff --git a/distributed/client.py b/distributed/client.py
index 52c0e2b420e..18d77ffab9b 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -5,6 +5,7 @@
 from concurrent.futures import ThreadPoolExecutor
 from concurrent.futures._base import DoneAndNotDoneFutures
 from contextlib import contextmanager
+from contextvars import ContextVar
 import copy
 import errno
 from functools import partial
@@ -89,6 +90,7 @@
 _global_clients = weakref.WeakValueDictionary()
 _global_client_index = [0]
 
+_current_client = ContextVar("_current_client", default=None)
 
 DEFAULT_EXTENSIONS = [PubSubClientExtension]
 
@@ -162,7 +164,7 @@ def __init__(self, key, client=None, inform=True, state=None):
         self.key = key
         self._cleared = False
         tkey = tokey(key)
-        self.client = client or _get_global_client()
+        self.client = client or Client.current()
         self.client._inc_ref(tkey)
         self._generation = self.client.generation
 
@@ -353,11 +355,14 @@ def release(self, _in_destructor=False):
                 pass  # Shutting down, add_callback may be None
 
     def __getstate__(self):
-        return (self.key, self.client.scheduler.address)
+        return self.key, self.client.scheduler.address
 
     def __setstate__(self, state):
         key, address = state
-        c = get_client(address)
+        try:
+            c = Client.current(allow_global=False)
+        except ValueError:
+            c = get_client(address)
         Future.__init__(self, key, c)
         c._send_to_scheduler(
             {
@@ -727,10 +732,41 @@ def __init__(
 
         ReplayExceptionClient(self)
 
+    @contextmanager
+    def as_current(self):
+        """Thread-local, Task-local context manager that causes the Client.current class
+        method to return self. Any Future objects deserialized inside this context
+        manager will be automatically attached to this Client.
+        """
+        # In Python 3.6, contextvars are thread-local but not Task-local.
+        # We can still detect a race condition though.
+        if sys.version_info < (3, 7) and _current_client.get() not in (self, None):
+            raise RuntimeError(
+                "Detected race condition where multiple asynchronous clients tried "
+                "entering the as_current() context manager at the same time. "
+                "Please upgrade to Python 3.7+."
+            )
+
+        tok = _current_client.set(self)
+        try:
+            yield
+        finally:
+            _current_client.reset(tok)
+
     @classmethod
-    def current(cls):
-        """ Return global client if one exists, otherwise raise ValueError """
-        return default_client()
+    def current(cls, allow_global=True):
+        """When running within the context of `as_client`, return the context-local
+        current client. Otherwise, return the latest initialised Client.
+        If no Client instances exist, raise ValueError.
+        If allow_global is set to False, raise ValueError if running outside of the
+        `as_client` context manager.
+        """
+        out = _current_client.get()
+        if out:
+            return out
+        if allow_global:
+            return default_client()
+        raise ValueError("Not running inside the `as_current` context manager")
 
     @property
     def asynchronous(self):
@@ -2178,8 +2214,7 @@ def retry(self, futures, asynchronous=None):
         """
         return self.sync(self._retry, futures, asynchronous=asynchronous)
 
-    @gen.coroutine
-    def _publish_dataset(self, *args, name=None, **kwargs):
+    async def _publish_dataset(self, *args, name=None, **kwargs):
         with log_errors():
             coroutines = []
 
@@ -2205,7 +2240,7 @@ def add_coro(name, data):
             for name, data in kwargs.items():
                 add_coro(name, data)
 
-            yield coroutines
+            await asyncio.gather(*coroutines)
 
     def publish_dataset(self, *args, **kwargs):
         """
@@ -2285,13 +2320,12 @@ def list_datasets(self, **kwargs):
         return self.sync(self.scheduler.publish_list, **kwargs)
 
     async def _get_dataset(self, name):
-        out = await self.scheduler.publish_get(name=name, client=self.id)
-        if out is None:
-            raise KeyError("Dataset '%s' not found" % name)
+        with self.as_current():
+            out = await self.scheduler.publish_get(name=name, client=self.id)
 
-        with temp_default_client(self):
-            data = out["data"]
-        return data
+        if out is None:
+            raise KeyError(f"Dataset '{name}' not found")
+        return out["data"]
 
     def get_dataset(self, name, **kwargs):
         """
@@ -4697,6 +4731,14 @@ def __exit__(self, typ, value, traceback):
 def temp_default_client(c):
     """ Set the default client for the duration of the context
 
+    .. note::
+       This function should be used exclusively for unit testing the default client
+       functionality. In all other cases, please use ``Client.as_current`` instead.
+
+    .. note::
+       Unlike ``Client.as_current``, this context manager is neither thread-local nor
+       task-local.
+
     Parameters
     ----------
     c : Client
diff --git a/distributed/lock.py b/distributed/lock.py
index 3c893a419c2..7a55ccb4413 100644
--- a/distributed/lock.py
+++ b/distributed/lock.py
@@ -3,7 +3,7 @@
 import logging
 import uuid
 
-from .client import _get_global_client
+from .client import Client
 from .utils import log_errors, TimeoutError
 from .worker import get_worker
 
@@ -93,7 +93,11 @@ class Lock:
     """
 
     def __init__(self, name=None, client=None):
-        self.client = client or _get_global_client() or get_worker().client
+        try:
+            self.client = client or Client.current()
+        except ValueError:
+            # Initialise new client
+            self.client = get_worker().client
         self.name = name or "lock-" + uuid.uuid4().hex
         self.id = uuid.uuid4().hex
         self._locked = False
diff --git a/distributed/publish.py b/distributed/publish.py
index 758e5ccc34b..4b30ebde042 100644
--- a/distributed/publish.py
+++ b/distributed/publish.py
@@ -6,10 +6,10 @@
 class PublishExtension:
     """ An extension for the scheduler to manage collections
 
-    *  publish-list
-    *  publish-put
-    *  publish-get
-    *  publish-delete
+    *  publish_list
+    *  publish_put
+    *  publish_get
+    *  publish_delete
     """
 
     def __init__(self, scheduler):
@@ -59,21 +59,60 @@ class Datasets(MutableMapping):
 
     """
 
+    __slots__ = ("_client",)
+
     def __init__(self, client):
-        self.__client = client
+        self._client = client
 
     def __getitem__(self, key):
-        return self.__client.get_dataset(key)
+        # When client is asynchronous, it returns a coroutine
+        return self._client.get_dataset(key)
 
     def __setitem__(self, key, value):
-        self.__client.publish_dataset(value, name=key)
+        if self._client.asynchronous:
+            # 'await obj[key] = value' is not supported by Python as of 3.8
+            raise TypeError(
+                "Can't use 'client.datasets[name] = value' when client is "
+                "asynchronous; please use 'client.publish_dataset(name=value)' instead"
+            )
+        self._client.publish_dataset(value, name=key)
 
     def __delitem__(self, key):
-        self.__client.unpublish_dataset(key)
+        if self._client.asynchronous:
+            # 'await del obj[key]' is not supported by Python as of 3.8
+            raise TypeError(
+                "Can't use 'del client.datasets[name]' when client is asynchronous; "
+                "please use 'client.unpublish_dataset(name)' instead"
+            )
+        return self._client.unpublish_dataset(key)
 
     def __iter__(self):
-        for key in self.__client.list_datasets():
+        if self._client.asynchronous:
+            raise TypeError(
+                "Can't invoke iter() or 'for' on client.datasets when client is "
+                "asynchronous; use 'async for' instead"
+            )
+        for key in self._client.list_datasets():
             yield key
 
+    def __aiter__(self):
+        if not self._client.asynchronous:
+            raise TypeError(
+                "Can't invoke 'async for' on client.datasets when client is "
+                "synchronous; use iter() or 'for' instead"
+            )
+
+        async def _():
+            for key in await self._client.list_datasets():
+                yield key
+
+        return _()
+
     def __len__(self):
-        return len(self.__client.list_datasets())
+        if self._client.asynchronous:
+            # 'await len(obj)' is not supported by Python as of 3.8
+            raise TypeError(
+                "Can't use 'len(client.datasets)' when client is asynchronous; "
+                "please use 'len(await client.list_datasets())' instead"
+            )
+        return len(self._client.list_datasets())
diff --git a/distributed/queues.py b/distributed/queues.py
index 81262703ad4..324fb46c40b 100644
--- a/distributed/queues.py
+++ b/distributed/queues.py
@@ -3,7 +3,7 @@
 import logging
 import uuid
 
-from .client import Future, _get_global_client, Client
+from .client import Future, Client
 from .utils import tokey, sync, thread_state
 from .worker import get_client
 
@@ -148,7 +148,7 @@ class Queue:
         not given, a random name will be generated.
     client: Client (optional)
         Client used for communication with the scheduler. Defaults to the
-        value of ``_get_global_client()``.
+        value of ``Client.current()``.
     maxsize: int (optional)
         Number of items allowed in the queue. If 0 (the default), the queue
         size is unbounded.
@@ -167,7 +167,7 @@ class Queue:
     """
 
     def __init__(self, name=None, client=None, maxsize=0):
-        self.client = client or _get_global_client()
+        self.client = client or Client.current()
         self.name = name or "queue-" + uuid.uuid4().hex
         self._event_started = asyncio.Event()
         if self.client.asynchronous or getattr(
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index fd95895c84e..634194bbae3 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -1201,7 +1201,7 @@ async def test_get_releases_data(c, s, a, b):
         assert time() < start + 2
 
 
-def test_Current(s, a, b):
+def test_current(s, a, b):
     with Client(s["address"]) as c:
         assert Client.current() is c
     with pytest.raises(ValueError):
@@ -3876,38 +3876,148 @@ async def test_scatter_compute_store_lose_processing(c, s, a, b):
 
 @gen_cluster(client=False)
 async def test_serialize_future(s, a, b):
-    c = await Client(s.address, asynchronous=True)
-    f = await Client(s.address, asynchronous=True)
+    c1 = await Client(s.address, asynchronous=True)
+    c2 = await Client(s.address, asynchronous=True)
 
-    future = c.submit(lambda: 1)
+    future = c1.submit(lambda: 1)
     result = await future
 
-    with temp_default_client(f):
-        future2 = pickle.loads(pickle.dumps(future))
-        assert future2.client is f
-        assert tokey(future2.key) in f.futures
-        result2 = await future2
-        assert result == result2
+    for ci in (c1, c2):
+        for ctxman in ci.as_current, lambda: temp_default_client(ci):
+            with ctxman():
+                future2 = pickle.loads(pickle.dumps(future))
+                assert future2.client is ci
+                assert tokey(future2.key) in ci.futures
+                result2 = await future2
+                assert result == result2
 
-    await c.close()
-    await f.close()
+    await c1.close()
+    await c2.close()
 
 
 @gen_cluster(client=False)
-async def test_temp_client(s, a, b):
-    c = await Client(s.address, asynchronous=True)
-    f = await Client(s.address, asynchronous=True)
+async def test_temp_default_client(s, a, b):
+    c1 = await Client(s.address, asynchronous=True)
+    c2 = await Client(s.address, asynchronous=True)
+
+    with temp_default_client(c1):
+        assert default_client() is c1
+        assert default_client(c2) is c2
+
+    with temp_default_client(c2):
+        assert default_client() is c2
+        assert default_client(c1) is c1
+
+    await c1.close()
+    await c2.close()
+
+
+@gen_cluster(client=True)
+async def test_as_current(c, s, a, b):
+    c1 = await Client(s.address, asynchronous=True)
+    c2 = await Client(s.address, asynchronous=True)
 
     with temp_default_client(c):
-        assert default_client() is c
-        assert default_client(f) is f
+        assert Client.current() is c
+        with pytest.raises(ValueError):
+            Client.current(allow_global=False)
+        with c1.as_current():
+            assert Client.current() is c1
+            assert Client.current(allow_global=True) is c1
+        with c2.as_current():
+            assert Client.current() is c2
+            assert Client.current(allow_global=True) is c2
+
+    await c1.close()
+    await c2.close()
 
-    with temp_default_client(f):
-        assert default_client() is f
-        assert default_client(c) is c
 
-    await c.close()
-    await f.close()
+def test_as_current_is_thread_local(s):
+    l1 = threading.Lock()
+    l2 = threading.Lock()
+    l3 = threading.Lock()
+    l4 = threading.Lock()
+    l1.acquire()
+    l2.acquire()
+    l3.acquire()
+    l4.acquire()
+
+    def run1():
+        with Client(s.address) as c:
+            with c.as_current():
+                l1.acquire()
+                l2.release()
+                try:
+                    # This line runs only when both run1 and run2 are inside the
+                    # context manager
+                    assert Client.current(allow_global=False) is c
+                finally:
+                    l3.acquire()
+                    l4.release()
+
+    def run2():
+        with Client(s.address) as c:
+            with c.as_current():
+                l1.release()
+                l2.acquire()
+                try:
+                    # This line runs only when both run1 and run2 are inside the
+                    # context manager
+                    assert Client.current(allow_global=False) is c
+                finally:
+                    l3.release()
+                    l4.acquire()
+
+    t1 = threading.Thread(target=run1)
+    t2 = threading.Thread(target=run2)
+    t1.start()
+    t2.start()
+    t1.join()
+    t2.join()
+
+
+@pytest.mark.xfail(
+    sys.version_info < (3, 7),
+    reason="Python 3.6 contextvars are not copied on Task creation",
+)
+@gen_cluster(client=False)
+async def test_as_current_is_task_local(s, a, b):
+    l1 = asyncio.Lock()
+    l2 = asyncio.Lock()
+    l3 = asyncio.Lock()
+    l4 = asyncio.Lock()
+    await l1.acquire()
+    await l2.acquire()
+    await l3.acquire()
+    await l4.acquire()
+
+    async def run1():
+        async with Client(s.address, asynchronous=True) as c:
+            with c.as_current():
+                await l1.acquire()
+                l2.release()
+                try:
+                    # This line runs only when both run1 and run2 are inside the
+                    # context manager
+                    assert Client.current(allow_global=False) is c
+                finally:
+                    await l3.acquire()
+                    l4.release()
+
+    async def run2():
+        async with Client(s.address, asynchronous=True) as c:
+            with c.as_current():
+                l1.release()
+                await l2.acquire()
+                try:
+                    # This line runs only when both run1 and run2 are inside the
+                    # context manager
+                    assert Client.current(allow_global=False) is c
+                finally:
+                    l3.release()
+                    await l4.acquire()
+
+    await asyncio.gather(run1(), run2())
 
 
 @nodebug  # test timing is fragile
diff --git a/distributed/tests/test_publish.py b/distributed/tests/test_publish.py
index ab32d52a112..a789f5a47f9 100644
--- a/distributed/tests/test_publish.py
+++ b/distributed/tests/test_publish.py
@@ -213,6 +213,23 @@ def test_datasets_iter(client):
     client.publish_dataset(**{str(key): key for key in keys})
     for n, key in enumerate(client.datasets):
         assert key == str(n)
+    with pytest.raises(TypeError):
+        client.datasets.__aiter__()
+
+
+@gen_cluster(client=True)
+async def test_datasets_async(c, s, a, b):
+    await c.publish_dataset(foo=1, bar=2)
+    assert await c.datasets["foo"] == 1
+    assert {k async for k in c.datasets} == {"foo", "bar"}
+    with pytest.raises(TypeError):
+        c.datasets["baz"] = 3
+    with pytest.raises(TypeError):
+        del c.datasets["foo"]
+    with pytest.raises(TypeError):
+        next(iter(c.datasets))
+    with pytest.raises(TypeError):
+        len(c.datasets)
 
 
 @gen_cluster(client=True)
@@ -229,3 +246,35 @@ async def test_pickle_safe(c, s, a, b):
 
         with pytest.raises(TypeError):
             await c2.get_dataset("z")
+
+
+@gen_cluster(client=True)
+async def test_deserialize_client(c, s, a, b):
+    """Test that the client attached to Futures returned by Client.get_dataset is always
+    the instance of the client that invoked the method.
+    Specifically:
+
+    - when the client is defined by hostname, test that it is not accidentally
+      reinitialised by IP;
+    - when multiple clients are connected to the same scheduler, test that they don't
+      interfere with each other.
+
+    See: test_client.test_serialize_future
+    See: https://github.com/dask/distributed/issues/3227
+    """
+    future = await c.scatter("123")
+    await c.publish_dataset(foo=future)
+    future = await c.get_dataset("foo")
+    assert future.client is c
+
+    for addr in (s.address, "localhost:" + s.address.split(":")[-1]):
+        async with Client(addr, asynchronous=True) as c2:
+            future = await c.get_dataset("foo")
+            assert future.client is c
+            future = await c2.get_dataset("foo")
+            assert future.client is c2
+
+    # Ensure cleanup
+    from distributed.client import _current_client
+
+    assert _current_client.get() is None
diff --git a/distributed/variable.py b/distributed/variable.py
index a47064b1397..dc717533a28 100644
--- a/distributed/variable.py
+++ b/distributed/variable.py
@@ -5,7 +5,7 @@
 
 from tlz import merge
 
-from .client import Future, _get_global_client, Client
+from .client import Future, Client
 from .utils import tokey, log_errors, TimeoutError, ignoring
 from .worker import get_client
 
@@ -142,7 +142,7 @@ class Variable:
         If not given, a random name will be generated.
     client: Client (optional)
         Client used for communication with the scheduler. Defaults to the
-        value of ``_get_global_client()``.
+        value of ``Client.current()``.
 
     Examples
     --------
@@ -161,7 +161,7 @@ class Variable:
     """
 
     def __init__(self, name=None, client=None, maxsize=0):
-        self.client = client or _get_global_client()
+        self.client = client or Client.current()
         self.name = name or "variable-" + uuid.uuid4().hex
 
     async def _set(self, value):
diff --git a/distributed/worker.py b/distributed/worker.py
index 63fd5c6daa4..cfdfdb95256 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -3118,14 +3118,15 @@ def get_client(address=None, timeout=3, resolve_address=True):
         if not address or worker.scheduler.address == address:
             return worker._get_client(timeout=timeout)
 
-    from .client import _get_global_client
+    from .client import Client
 
-    client = _get_global_client()  # TODO: assumes the same scheduler
+    try:
+        client = Client.current()  # TODO: assumes the same scheduler
+    except ValueError:
+        client = None
     if client and (not address or client.scheduler.address == address):
         return client
     elif address:
-        from .client import Client
-
         return Client(address, timeout=timeout)
     else:
         raise ValueError("No global client found and no address provided")
diff --git a/requirements.txt b/requirements.txt
index 4cb3ba60ae7..b0d20cdb1eb 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,6 @@
 click >= 6.6
 cloudpickle >= 0.2.2
+contextvars;python_version<'3.7'
 dask >= 2.9.0
 msgpack >= 0.6.0
 psutil >= 5.0

From f66eda86c36e2f0b0496552c496736baf80c1c1e Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Fri, 8 May 2020 15:59:42 -0500
Subject: [PATCH 0827/1550] bump version to 2.16.0

---
 docs/source/changelog.rst | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index 6ef12a686f9..17905db5b6d 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,20 @@
 Changelog
 =========
 
+2.16.0 - 2020-05-08
+-------------------
+
+- ``Client.get_dataset`` to always create ``Futures`` attached to itself (:pr:`3729`) `crusaderky`_
+- Remove dev-requirements since it is unused (:pr:`3782`) `Julia Signell`_
+- Use bokeh column for ``/system`` instead of custom css (:pr:`3781`) `Julia Signell`_
+- Attempt to fix ``test_preload_remote_module`` on windows (:pr:`3775`) `James Bourbeau`_
+- Fix broadcast for TLS comms (:pr:`3766`) `Florian Jetter`_
+- Don't validate http preloads locally (:pr:`3768`) `Rami Chowdhury`_
+- Allow range of ports to be specified for ``Workers`` (:pr:`3704`) `James Bourbeau`_
+- Add UCX support for RDMACM (:pr:`3759`) `Peter Andreas Entschev`_
+- Support web addresses in preload (:pr:`3755`) `Matthew Rocklin`_
+
+
 2.15.2 - 2020-05-01
 -------------------
 

From 9f91653398aee681ba0010b1053f3bd1b415894b Mon Sep 17 00:00:00 2001
From: Scott Sanderson <scott.b.sanderson90@gmail.com>
Date: Fri, 8 May 2020 20:12:57 -0400
Subject: [PATCH 0828/1550] Fix typo in Cluster docstring (#3787)

Co-authored-by: Scott Sanderson <ssanderson@quantopian.com>
---
 distributed/deploy/cluster.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
index 35e0b97c613..7a6bb8669d6 100644
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -36,7 +36,7 @@ class Cluster:
     2.  Implement ``scale``, which takes an integer and scales the cluster to
         that many workers, or else set ``_supports_scaling`` to False
 
-    For that, should should get the following:
+    For that, you should get the following:
 
     1.  A standard ``__repr__``
     2.  A live IPython widget

From f899a994467f72b30edc384b6579b92dc3adbcac Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 12 May 2020 10:46:55 -0700
Subject: [PATCH 0829/1550] Await self before handling comms (#3788)

This stops servers from handling messages until they are finished
starting.

Additionally, we centralize the status management to Server.__await__
---
 distributed/cli/tests/test_dask_scheduler.py  |  21 ++--
 distributed/client.py                         |  26 +++--
 distributed/core.py                           |  52 +++++++++
 distributed/deploy/tests/test_local.py        |   1 -
 distributed/deploy/tests/test_spec_cluster.py |   1 +
 distributed/node.py                           | 106 +-----------------
 distributed/scheduler.py                      |  43 +++----
 distributed/tests/test_client.py              |   5 +-
 distributed/tests/test_core.py                |   1 +
 distributed/worker.py                         |   1 -
 10 files changed, 112 insertions(+), 145 deletions(-)

diff --git a/distributed/cli/tests/test_dask_scheduler.py b/distributed/cli/tests/test_dask_scheduler.py
index 1885cb84a85..6f924e4c8e6 100644
--- a/distributed/cli/tests/test_dask_scheduler.py
+++ b/distributed/cli/tests/test_dask_scheduler.py
@@ -66,6 +66,11 @@ def test_dashboard(loop):
     pytest.importorskip("bokeh")
 
     with popen(["dask-scheduler"]) as proc:
+        for line in proc.stderr:
+            if b"dashboard at" in line:
+                dashboard_port = int(line.decode().split(":")[-1].strip())
+                break
+
         with Client("127.0.0.1:%d" % Scheduler.default_port, loop=loop) as c:
             pass
 
@@ -78,7 +83,7 @@ def test_dashboard(loop):
             try:
                 # All addresses should respond
                 for name in names:
-                    uri = "http://%s:8787/status/" % name
+                    uri = "http://%s:%d/status/" % (name, dashboard_port)
                     response = requests.get(uri)
                     assert response.ok
                 break
@@ -88,7 +93,7 @@ def test_dashboard(loop):
                 assert time() < start + 10
 
     with pytest.raises(Exception):
-        requests.get("http://127.0.0.1:8787/status/")
+        requests.get("http://127.0.0.1:%d/status/" % dashboard_port)
 
 
 def test_dashboard_non_standard_ports(loop):
@@ -317,16 +322,12 @@ def test_preload_remote_module(loop, tmp_path):
             with Client(
                 scheduler_file=tmp_path / "scheduler-file.json", loop=loop
             ) as c:
-                for i in range(10):
-                    val = c.run_on_scheduler(
+                assert (
+                    c.run_on_scheduler(
                         lambda dask_scheduler: getattr(dask_scheduler, "foo", None)
                     )
-                    if val == "bar":
-                        break
-                    else:
-                        sleep(0.1)
-                else:
-                    raise ValueError(val)
+                    == "bar"
+                )
 
 
 PRELOAD_COMMAND_TEXT = """
diff --git a/distributed/client.py b/distributed/client.py
index 18d77ffab9b..4e5e3bff1c1 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -51,9 +51,15 @@
     retry_operation,
 )
 from .cfexecutor import ClientExecutor
-from .core import connect, rpc, clean_exception, CommClosedError, PooledRPCCall
+from .core import (
+    connect,
+    rpc,
+    clean_exception,
+    CommClosedError,
+    PooledRPCCall,
+    ConnectionPool,
+)
 from .metrics import time
-from .node import Node
 from .protocol import to_serialize
 from .protocol.pickle import dumps, loads
 from .publish import Datasets
@@ -492,7 +498,7 @@ class AllExit(Exception):
     """
 
 
-class Client(Node):
+class Client:
     """ Connect to and submit computation to a Dask cluster
 
     The Client connects users to a Dask cluster.  It provides an asynchronous
@@ -590,6 +596,7 @@ def __init__(
         deserializers=None,
         extensions=DEFAULT_EXTENSIONS,
         direct_to_workers=None,
+        connection_limit=512,
         **kwargs,
     ):
         if timeout == no_default:
@@ -671,7 +678,7 @@ def __init__(
         self._asynchronous = asynchronous
         self._should_close_loop = not loop
         self._loop_runner = LoopRunner(loop=loop, asynchronous=asynchronous)
-        self.loop = self._loop_runner.loop
+        self.io_loop = self.loop = self._loop_runner.loop
 
         self._gather_keys = None
         self._gather_future = None
@@ -714,12 +721,14 @@ def __init__(
             "erred": self._handle_task_erred,
         }
 
-        super(Client, self).__init__(
-            connection_args=self.connection_args,
-            io_loop=self.loop,
+        self.rpc = ConnectionPool(
+            limit=connection_limit,
             serializers=serializers,
             deserializers=deserializers,
+            deserialize=True,
+            connection_args=self.connection_args,
             timeout=timeout,
+            server=self,
         )
 
         for ext in extensions:
@@ -961,8 +970,7 @@ def _send_to_scheduler(self, msg):
             )
 
     async def _start(self, timeout=no_default, **kwargs):
-
-        await super().start()
+        await self.rpc.start()
 
         if timeout == no_default:
             timeout = self._timeout
diff --git a/distributed/core.py b/distributed/core.py
index df0a55780e7..f05b85f903e 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -34,6 +34,7 @@
     parse_timedelta,
     has_keyword,
     CancelledError,
+    TimeoutError,
 )
 from . import protocol
 
@@ -106,6 +107,10 @@ def __init__(
         stream_handlers=None,
         connection_limit=512,
         deserialize=True,
+        serializers=None,
+        deserializers=None,
+        connection_args=None,
+        timeout=None,
         io_loop=None,
     ):
         self.handlers = {
@@ -191,6 +196,18 @@ def set_thread_ident():
             self.thread_id = threading.get_ident()
 
         self.io_loop.add_callback(set_thread_ident)
+        self._startup_lock = asyncio.Lock()
+        self.status = None
+
+        self.rpc = ConnectionPool(
+            limit=connection_limit,
+            deserialize=deserialize,
+            serializers=serializers,
+            deserializers=deserializers,
+            connection_args=connection_args,
+            timeout=timeout,
+            server=self,
+        )
 
         self.__stopped = False
 
@@ -198,6 +215,40 @@ async def finished(self):
         """ Wait until the server has finished """
         await self._event_finished.wait()
 
+    def __await__(self):
+        async def _():
+            timeout = getattr(self, "death_timeout", 0)
+            async with self._startup_lock:
+                if self.status == "running":
+                    return self
+                if timeout:
+                    try:
+                        await asyncio.wait_for(self.start(), timeout=timeout)
+                        self.status = "running"
+                    except Exception:
+                        await self.close(timeout=1)
+                        raise TimeoutError(
+                            "{} failed to start in {} seconds".format(
+                                type(self).__name__, timeout
+                            )
+                        )
+                else:
+                    await self.start()
+                    self.status = "running"
+            return self
+
+        return _().__await__()
+
+    async def start(self):
+        await self.rpc.start()
+
+    async def __aenter__(self):
+        await self
+        return self
+
+    async def __aexit__(self, typ, value, traceback):
+        await self.close()
+
     def start_periodic_callbacks(self):
         """ Start Periodic Callbacks consistently
 
@@ -331,6 +382,7 @@ async def handle_comm(self, comm, shutting_down=shutting_down):
 
         logger.debug("Connection from %r to %s", address, type(self).__name__)
         self._comms[comm] = op
+        await self
         try:
             while True:
                 try:
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index 0867968a894..59a2d0c7607 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -806,7 +806,6 @@ def test_local_tls_restart(loop):
         loop=loop,
     ) as c:
         with Client(c.scheduler.address, loop=loop, security=security) as client:
-            print(c.workers, c.workers[0].address)
             workers_before = set(client.scheduler_info()["workers"])
             assert client.submit(inc, 1).result() == 2
             client.restart()
diff --git a/distributed/deploy/tests/test_spec_cluster.py b/distributed/deploy/tests/test_spec_cluster.py
index c9482f5da56..4f232e27aef 100644
--- a/distributed/deploy/tests/test_spec_cluster.py
+++ b/distributed/deploy/tests/test_spec_cluster.py
@@ -399,6 +399,7 @@ def __init__(self, *args, n=1, name=None, nthreads=None, **kwargs):
             )
             for i in range(n)
         ]
+        self._startup_lock = asyncio.Lock()
 
     @property
     def status(self):
diff --git a/distributed/node.py b/distributed/node.py
index 740776bed68..14a8314a8f5 100644
--- a/distributed/node.py
+++ b/distributed/node.py
@@ -1,53 +1,20 @@
-import asyncio
 import logging
 import warnings
 import weakref
 
-from tornado import gen
-from tornado.ioloop import IOLoop
 from tornado.httpserver import HTTPServer
 import tlz
 import dask
 
 from .comm import get_tcp_server_address
 from .comm import get_address_host
-from .core import Server, ConnectionPool
+from .core import Server
 from .http.routing import RoutingApplication
 from .versions import get_versions
-from .utils import DequeHandler, TimeoutError, clean_dashboard_address, ignoring
+from .utils import DequeHandler, clean_dashboard_address, ignoring
 
 
-class Node:
-    """
-    Base class for nodes in a distributed cluster.
-    """
-
-    def __init__(
-        self,
-        connection_limit=512,
-        deserialize=True,
-        connection_args=None,
-        io_loop=None,
-        serializers=None,
-        deserializers=None,
-        timeout=None,
-    ):
-        self.io_loop = io_loop or IOLoop.current()
-        self.rpc = ConnectionPool(
-            limit=connection_limit,
-            deserialize=deserialize,
-            serializers=serializers,
-            deserializers=deserializers,
-            connection_args=connection_args,
-            timeout=timeout,
-            server=self,
-        )
-
-    async def start(self):
-        await self.rpc.start()
-
-
-class ServerNode(Node, Server):
+class ServerNode(Server):
     """
     Base class for server nodes in a distributed cluster.
     """
@@ -57,39 +24,6 @@ class ServerNode(Node, Server):
     # XXX avoid inheriting from Server? there is some large potential for confusion
     # between base and derived attribute namespaces...
 
-    def __init__(
-        self,
-        handlers=None,
-        blocked_handlers=None,
-        stream_handlers=None,
-        connection_limit=512,
-        deserialize=True,
-        connection_args=None,
-        io_loop=None,
-        serializers=None,
-        deserializers=None,
-        timeout=None,
-    ):
-        Node.__init__(
-            self,
-            deserialize=deserialize,
-            connection_limit=connection_limit,
-            connection_args=connection_args,
-            io_loop=io_loop,
-            serializers=serializers,
-            deserializers=deserializers,
-            timeout=timeout,
-        )
-        Server.__init__(
-            self,
-            handlers=handlers,
-            blocked_handlers=blocked_handlers,
-            stream_handlers=stream_handlers,
-            connection_limit=connection_limit,
-            deserialize=deserialize,
-            io_loop=self.io_loop,
-        )
-
     def versions(self, comm=None, packages=None):
         return get_versions(packages=packages)
 
@@ -161,40 +95,6 @@ def get_logs(self, comm=None, n=None):
             L = [L[-i] for i in range(min(n, len(L)))]
         return [(msg.levelname, deque_handler.format(msg)) for msg in L]
 
-    async def __aenter__(self):
-        await self
-        return self
-
-    async def __aexit__(self, typ, value, traceback):
-        await self.close()
-
-    def __await__(self):
-        if self.status == "running":
-            return gen.sleep(0).__await__()
-        else:
-            future = self.start()
-            timeout = getattr(self, "death_timeout", 0)
-            if timeout:
-
-                async def wait_for(future, timeout=None):
-                    try:
-                        await asyncio.wait_for(future, timeout=timeout)
-                    except Exception:
-                        await self.close(timeout=1)
-                        raise TimeoutError(
-                            "{} failed to start in {} seconds".format(
-                                type(self).__name__, timeout
-                            )
-                        )
-
-                future = wait_for(future, timeout=timeout)
-            return future.__await__()
-
-    async def start(self):
-        # subclasses should implement their own start method whichs calls super().start()
-        await Node.start(self)
-        return self
-
     def start_http_server(
         self, routes, dashboard_address, default_port=0, ssl_options=None,
     ):
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 5c50fd4d0f2..69686325598 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1079,7 +1079,6 @@ def __init__(
         if validate is None:
             validate = dask.config.get("distributed.scheduler.validate")
         self.validate = validate
-        self.status = None
         self.proc = psutil.Process()
         self.delete_interval = parse_timedelta(delete_interval, default="ms")
         self.synchronize_worker_interval = parse_timedelta(
@@ -1426,8 +1425,8 @@ def get_worker_service_addr(self, worker, service_name, protocol=False):
 
     async def start(self):
         """ Clear out old state and restart all running coroutines """
-
         await super().start()
+        assert self.status != "running"
 
         enable_gc_diagnosis()
 
@@ -1437,28 +1436,26 @@ async def start(self):
             for c in self._worker_coroutines:
                 c.cancel()
 
-        if self.status != "running":
-            for addr in self._start_address:
-                await self.listen(addr, **self.security.get_listen_args("scheduler"))
-                self.ip = get_address_host(self.listen_address)
-                listen_ip = self.ip
+        for addr in self._start_address:
+            await self.listen(addr, **self.security.get_listen_args("scheduler"))
+            self.ip = get_address_host(self.listen_address)
+            listen_ip = self.ip
 
-                if listen_ip == "0.0.0.0":
-                    listen_ip = ""
+            if listen_ip == "0.0.0.0":
+                listen_ip = ""
 
-            if self.address.startswith("inproc://"):
-                listen_ip = "localhost"
+        if self.address.startswith("inproc://"):
+            listen_ip = "localhost"
 
-            # Services listen on all addresses
-            self.start_services(listen_ip)
+        # Services listen on all addresses
+        self.start_services(listen_ip)
 
-            self.status = "running"
-            for listener in self.listeners:
-                logger.info("  Scheduler at: %25s", listener.contact_address)
-            for k, v in self.services.items():
-                logger.info("%11s at: %25s", k, "%s:%d" % (listen_ip, v.port))
+        for listener in self.listeners:
+            logger.info("  Scheduler at: %25s", listener.contact_address)
+        for k, v in self.services.items():
+            logger.info("%11s at: %25s", k, "%s:%d" % (listen_ip, v.port))
 
-            self.loop.add_callback(self.reevaluate_occupancy)
+        self.loop.add_callback(self.reevaluate_occupancy)
 
         if self.scheduler_file:
             with open(self.scheduler_file, "w") as f:
@@ -2937,7 +2934,13 @@ async def restart(self, client=None, timeout=3):
             finally:
                 await asyncio.gather(*[nanny.close_rpc() for nanny in nannies])
 
-            await self.start()
+            # self.status = None
+            # await self
+            self.clear_task_state()
+
+            with ignoring(AttributeError):
+                for c in self._worker_coroutines:
+                    c.cancel()
 
             self.log_event([client, "all"], {"action": "restart", "client": client})
             start = time()
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 634194bbae3..55cc6c43893 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -3620,8 +3620,8 @@ async def start_worker(sleep, duration, repeat=1):
                     return
                 w = worker(s["address"], loop=loop)
                 running[w] = None
-                workers.add(w)
                 await w
+                workers.add(w)
                 addr = w.worker_address
                 running[w] = addr
                 await asyncio.sleep(duration)
@@ -3649,6 +3649,9 @@ async def start_worker(sleep, duration, repeat=1):
                 sleep(0.2)
                 assert time() < start + 10
 
+            while len(workers) < count * repeat:
+                sleep(0.2)
+
             status = False
 
             [c.sync(w.close) for w in list(workers)]
diff --git a/distributed/tests/test_core.py b/distributed/tests/test_core.py
index c75f9c48cf6..fbdf5677809 100644
--- a/distributed/tests/test_core.py
+++ b/distributed/tests/test_core.py
@@ -88,6 +88,7 @@ async def f():
         await server.listen(8881)
         assert server.port == 8881
         assert server.address == ("tcp://%s:8881" % get_ip())
+        await server
 
         for addr in ("127.0.0.1:8881", "tcp://127.0.0.1:8881", server.address):
             comm = await connect(addr)
diff --git a/distributed/worker.py b/distributed/worker.py
index cfdfdb95256..0729f86332d 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -562,7 +562,6 @@ def __init__(
 
         self.actors = {}
         self.loop = loop or IOLoop.current()
-        self.status = None
         self.reconnect = reconnect
         self.executor = executor or ThreadPoolExecutor(
             self.nthreads, thread_name_prefix="Dask-Worker-Threads'"

From c5d2e4baf0e03585f04acb6d4aa6293c70f6ed04 Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Thu, 14 May 2020 16:32:43 +0200
Subject: [PATCH 0830/1550] Fix profile thread leakage during test teardown on
 some platforms (#3795)

---
 distributed/cli/dask_ssh.py      |  2 +-
 distributed/deploy/old_ssh.py    |  2 +-
 distributed/tests/test_client.py |  2 +-
 distributed/utils_test.py        | 11 ++---------
 4 files changed, 5 insertions(+), 12 deletions(-)

diff --git a/distributed/cli/dask_ssh.py b/distributed/cli/dask_ssh.py
index 7674632807c..f592c572ac5 100755
--- a/distributed/cli/dask_ssh.py
+++ b/distributed/cli/dask_ssh.py
@@ -175,7 +175,7 @@ def main(
             version=distributed.__version__
         )
     )
-    print("Worker nodes:".format(n=len(hostnames)))
+    print("Worker nodes: {n}".format(n=len(hostnames)))
     for i, host in enumerate(hostnames):
         print("  {num}: {host}".format(num=i, host=host))
     print("\nscheduler node: {addr}:{port}".format(addr=scheduler, port=scheduler_port))
diff --git a/distributed/deploy/old_ssh.py b/distributed/deploy/old_ssh.py
index 648d7b80905..fe5703ea5e8 100644
--- a/distributed/deploy/old_ssh.py
+++ b/distributed/deploy/old_ssh.py
@@ -212,7 +212,7 @@ def start_scheduler(
     logdir, addr, port, ssh_username, ssh_port, ssh_private_key, remote_python=None,
 ):
     cmd = "{python} -m distributed.cli.dask_scheduler --port {port}".format(
-        python=remote_python or sys.executable, port=port, logdir=logdir
+        python=remote_python or sys.executable, port=port
     )
 
     # Optionally re-direct stdout and stderr to a logfile
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 55cc6c43893..08dff3bf7bc 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -2889,7 +2889,7 @@ async def test_rebalance_unprepared(c, s, a, b):
 
 @gen_cluster(client=True)
 async def test_rebalance_raises_missing_data(c, s, a, b):
-    with pytest.raises(ValueError, match=f"keys were found to be missing"):
+    with pytest.raises(ValueError, match="keys were found to be missing"):
         futures = await c.scatter(range(100))
         keys = [f.key for f in futures]
         del futures
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index 05467bbeb49..3bb0fc4b2b4 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -1542,12 +1542,5 @@ def null():
 
 @pytest.fixture
 def cleanup():
-    with check_thread_leak():
-        with check_process_leak():
-            with check_instances():
-                reset_config()
-                dask.config.set({"distributed.comm.timeouts.connect": "5s"})
-                for name, level in logging_levels.items():
-                    logging.getLogger(name).setLevel(level)
-
-                yield
+    with clean():
+        yield

From 68e34fccdacac1af826f4e1e58e4b4b8f8ee169d Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Fri, 15 May 2020 17:07:37 +0200
Subject: [PATCH 0831/1550] Fix RuntimeWarning for never awaited coroutine when
 using distributed.Semaphore (#3713)

---
 distributed/semaphore.py | 40 ++++++++++++++++++++++++++++++----------
 1 file changed, 30 insertions(+), 10 deletions(-)

diff --git a/distributed/semaphore.py b/distributed/semaphore.py
index 263619c9073..b853f99002f 100644
--- a/distributed/semaphore.py
+++ b/distributed/semaphore.py
@@ -39,9 +39,9 @@ class SemaphoreExtension:
 
     * semaphore_acquire
     * semaphore_release
-    * semaphore_create
     * semaphore_close
     * semaphore_refresh_leases
+    * semaphore_register
     """
 
     def __init__(self, scheduler):
@@ -56,7 +56,7 @@ def __init__(self, scheduler):
 
         self.scheduler.handlers.update(
             {
-                "semaphore_create": self.create,
+                "semaphore_register": self.create,
                 "semaphore_acquire": self.acquire,
                 "semaphore_release": self.release,
                 "semaphore_close": self.close,
@@ -285,6 +285,12 @@ class Semaphore:
     client: Client (optional)
         Client to use for communication with the scheduler.  If not given, the
         default global client will be used.
+    register: bool
+        If True, register the semaphore with the scheduler. This needs to be
+        done before any leases can be acquired. If not done during
+        initialization, this can also be done by calling the register method of
+        this class.
+        When registering, this needs to be awaited.
 
     Examples
     --------
@@ -320,18 +326,13 @@ class Semaphore:
 
     """
 
-    def __init__(self, max_leases=1, name=None, client=None):
+    def __init__(self, max_leases=1, name=None, client=None, register=True):
         self.client = client or get_client()
         self.name = name or "semaphore-" + uuid.uuid4().hex
         self.max_leases = max_leases
         self.id = uuid.uuid4().hex
         self._leases = deque()
 
-        self._started = self.client.sync(
-            self.client.scheduler.semaphore_create,
-            name=self.name,
-            max_leases=max_leases,
-        )
         # this should give ample time to refresh without introducing another
         # config parameter since this *must* be smaller than the timeout anyhow
         refresh_leases_interval = (
@@ -352,9 +353,28 @@ def __init__(self, max_leases=1, name=None, client=None):
         pc.start()
         self.refresh_leases = True
 
+        self._registered = None
+        if register:
+            self._registered = self.register()
+
+    def register(self):
+        """
+        Register the semaphore on scheduler side
+
+        This will register the semaphore on scheduler side and ensure that all necessary data structures exist.
+        """
+        if self._registered is None:
+            self._registered = self.client.sync(
+                self.client.scheduler.semaphore_register,
+                name=self.name,
+                max_leases=self.max_leases,
+            )
+        return self._registered
+
     def __await__(self):
         async def create_semaphore():
-            await self._started
+            if self._registered:
+                await self._registered
             return self
 
         return create_semaphore().__await__()
@@ -444,7 +464,7 @@ def __getstate__(self):
     def __setstate__(self, state):
         name, max_leases = state
         client = get_client()
-        self.__init__(name=name, client=client, max_leases=max_leases)
+        self.__init__(name=name, client=client, max_leases=max_leases, register=False)
 
     def close(self):
         return self.client.sync(self.client.scheduler.semaphore_close, name=self.name)

From 1bcbaeee4a92d57fb2d399c9580fce6480a6d7c9 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Fri, 15 May 2020 12:46:49 -0500
Subject: [PATCH 0832/1550] Remove commented out lines from scheduler.py
 (#3803)

---
 distributed/scheduler.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 69686325598..029db4ef06c 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -756,7 +756,6 @@ def __init__(self, name):
         self.types = set()
 
     def add(self, ts):
-        # self.tasks.add(ts)
         self.states[ts.state] += 1
         ts.group = self
 
@@ -2934,8 +2933,6 @@ async def restart(self, client=None, timeout=3):
             finally:
                 await asyncio.gather(*[nanny.close_rpc() for nanny in nannies])
 
-            # self.status = None
-            # await self
             self.clear_task_state()
 
             with ignoring(AttributeError):

From f3b241be274a0d75763bf89e4c0106cd47738162 Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Tue, 19 May 2020 08:42:17 +0100
Subject: [PATCH 0833/1550] Add retries to server listen (#3801)

Retry server listens to handle CI failures in cases of OSError: [Errno 98] Address already in use
---
 distributed/node.py | 28 +++++++++++++++++++---------
 1 file changed, 19 insertions(+), 9 deletions(-)

diff --git a/distributed/node.py b/distributed/node.py
index 14a8314a8f5..eeae67ec400 100644
--- a/distributed/node.py
+++ b/distributed/node.py
@@ -96,11 +96,11 @@ def get_logs(self, comm=None, n=None):
         return [(msg.levelname, deque_handler.format(msg)) for msg in L]
 
     def start_http_server(
-        self, routes, dashboard_address, default_port=0, ssl_options=None,
+        self, routes, dashboard_address, default_port=0, ssl_options=None
     ):
         """ This creates an HTTP Server running on this node """
 
-        self.http_application = RoutingApplication(routes,)
+        self.http_application = RoutingApplication(routes)
 
         # TLS configuration
         tls_key = dask.config.get("distributed.scheduler.dashboard.tls.key")
@@ -127,16 +127,26 @@ def start_http_server(
             if address:
                 with ignoring(ValueError):
                     http_address["address"] = get_address_host(address)
-        changed_port = False
-        try:
-            self.http_server.listen(**http_address)
-        except Exception:
-            changed_port = True
-            self.http_server.listen(**tlz.merge(http_address, {"port": 0}))
+
+        change_port = False
+        retries_left = 3
+        while True:
+            try:
+                if not change_port:
+                    self.http_server.listen(**http_address)
+                else:
+                    self.http_server.listen(**tlz.merge(http_address, {"port": 0}))
+                break
+            except Exception:
+                change_port = True
+                retries_left = retries_left - 1
+                if retries_left < 1:
+                    raise
+
         self.http_server.port = get_tcp_server_address(self.http_server)[1]
         self.services["dashboard"] = self.http_server
 
-        if changed_port and dashboard_address:
+        if change_port and dashboard_address:
             warnings.warn(
                 "Port {} is already in use.\n"
                 "Perhaps you already have a cluster running?\n"

From 67abe0a0b5fff68db76a046f3f357f372086f5a8 Mon Sep 17 00:00:00 2001
From: Nick Evans <nick.evans3976@gmail.com>
Date: Tue, 19 May 2020 17:13:08 +0100
Subject: [PATCH 0834/1550] Fix worker plugin called with superseded transition
 (#3812)

---
 .../diagnostics/tests/test_worker_plugin.py      | 16 ++++++++++++++++
 distributed/worker.py                            |  2 +-
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/distributed/diagnostics/tests/test_worker_plugin.py b/distributed/diagnostics/tests/test_worker_plugin.py
index 2ee5a28c780..038924853a7 100644
--- a/distributed/diagnostics/tests/test_worker_plugin.py
+++ b/distributed/diagnostics/tests/test_worker_plugin.py
@@ -85,6 +85,22 @@ def failing(x):
         await c.submit(failing, 1, key="task")
 
 
+@gen_cluster(
+    nthreads=[("127.0.0.1", 1)], client=True, worker_kwargs={"resources": {"X": 1}},
+)
+async def test_superseding_task_transitions_called(c, s, w):
+    expected_transitions = [
+        ("task", "waiting", "constrained"),
+        ("task", "constrained", "executing"),
+        ("task", "executing", "memory"),
+    ]
+
+    plugin = MyPlugin(1, expected_transitions=expected_transitions)
+
+    await c.register_worker_plugin(plugin)
+    await c.submit(lambda x: x, 1, key="task", resources={"X": 1})
+
+
 @gen_cluster(nthreads=[("127.0.0.1", 1)], client=True)
 async def test_empty_plugin(c, s, w):
     class EmptyPlugin:
diff --git a/distributed/worker.py b/distributed/worker.py
index 0729f86332d..3eb698289d3 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -1593,7 +1593,7 @@ def transition(self, key, finish, **kwargs):
         self.task_state[key] = state or finish
         if self.validate:
             self.validate_key(key)
-        self._notify_transition(key, start, finish, **kwargs)
+        self._notify_transition(key, start, state or finish, **kwargs)
 
     def transition_waiting_ready(self, key):
         try:

From caf9939ca31888a468c5fce8f1cdc07b532a7fff Mon Sep 17 00:00:00 2001
From: Lucas Rademaker <44430780+lr4d@users.noreply.github.com>
Date: Tue, 19 May 2020 18:30:36 +0200
Subject: [PATCH 0835/1550] Add prometheus metrics for semaphore (#3757)

* add metrics to SemaphoreExtension

* add prometheus metrics for Semaphore

Co-authored-by: lr4d <lr4d@users.noreply.github.com>
---
 .../{prometheus.py => prometheus/__init__.py} | 27 ++++---
 .../http/scheduler/prometheus/semaphore.py    | 76 ++++++++++++++++++
 .../scheduler/tests/test_semaphore_http.py    | 79 +++++++++++++++++++
 distributed/semaphore.py                      | 49 +++++++++---
 distributed/tests/test_semaphore.py           | 53 ++++++++++++-
 5 files changed, 261 insertions(+), 23 deletions(-)
 rename distributed/http/scheduler/{prometheus.py => prometheus/__init__.py} (79%)
 create mode 100644 distributed/http/scheduler/prometheus/semaphore.py
 create mode 100644 distributed/http/scheduler/tests/test_semaphore_http.py

diff --git a/distributed/http/scheduler/prometheus.py b/distributed/http/scheduler/prometheus/__init__.py
similarity index 79%
rename from distributed/http/scheduler/prometheus.py
rename to distributed/http/scheduler/prometheus/__init__.py
index 0f1f9c3c14f..6d57b41d876 100644
--- a/distributed/http/scheduler/prometheus.py
+++ b/distributed/http/scheduler/prometheus/__init__.py
@@ -1,7 +1,8 @@
 import toolz
 
-from ..utils import RequestHandler
-from ...scheduler import ALL_TASK_STATES
+from distributed.http.utils import RequestHandler
+from distributed.scheduler import ALL_TASK_STATES
+from .semaphore import SemaphoreMetricExtension
 
 
 class _PrometheusCollector:
@@ -68,8 +69,11 @@ def collect(self):
         yield tasks
 
 
+COLLECTORS = [_PrometheusCollector, SemaphoreMetricExtension]
+
+
 class PrometheusHandler(RequestHandler):
-    _collector = None
+    _collectors = None
 
     def __init__(self, *args, dask_server=None, **kwargs):
         import prometheus_client
@@ -78,14 +82,19 @@ def __init__(self, *args, dask_server=None, **kwargs):
             *args, dask_server=dask_server, **kwargs
         )
 
-        if PrometheusHandler._collector:
+        if PrometheusHandler._collectors:
             # Especially during testing, multiple schedulers are started
             # sequentially in the same python process
-            PrometheusHandler._collector.server = self.server
+            for _collector in PrometheusHandler._collectors:
+                _collector.server = self.server
             return
 
-        PrometheusHandler._collector = _PrometheusCollector(self.server)
-        prometheus_client.REGISTRY.register(PrometheusHandler._collector)
+        PrometheusHandler._collectors = tuple(
+            collector(self.server) for collector in COLLECTORS
+        )
+        # Register collectors
+        for instantiated_collector in PrometheusHandler._collectors:
+            prometheus_client.REGISTRY.register(instantiated_collector)
 
     def get(self):
         import prometheus_client
@@ -94,6 +103,4 @@ def get(self):
         self.set_header("Content-Type", "text/plain; version=0.0.4")
 
 
-routes = [
-    ("/metrics", PrometheusHandler, {}),
-]
+routes = [("/metrics", PrometheusHandler, {})]
diff --git a/distributed/http/scheduler/prometheus/semaphore.py b/distributed/http/scheduler/prometheus/semaphore.py
new file mode 100644
index 00000000000..f1df7434019
--- /dev/null
+++ b/distributed/http/scheduler/prometheus/semaphore.py
@@ -0,0 +1,76 @@
+class SemaphoreMetricExtension:
+    def __init__(self, dask_server):
+        self.server = dask_server
+
+    def collect(self):
+        from prometheus_client.core import GaugeMetricFamily, CounterMetricFamily
+
+        sem_ext = self.server.extensions["semaphores"]
+
+        semaphore_max_leases_family = GaugeMetricFamily(
+            "semaphore_max_leases",
+            "Maximum leases allowed per semaphore, this will be constant for each semaphore during its lifetime.",
+            labels=["name"],
+        )
+        semaphore_active_leases_family = GaugeMetricFamily(
+            "semaphore_active_leases",
+            "Amount of currently active leases per semaphore.",
+            labels=["name"],
+        )
+        semaphore_pending_leases = GaugeMetricFamily(
+            "semaphore_pending_leases",
+            "Amount of currently pending leases per semaphore.",
+            labels=["name"],
+        )
+
+        semaphore_acquire_total = CounterMetricFamily(
+            "semaphore_acquire_total",
+            "Total number of leases acquired per semaphore.",
+            labels=["name"],
+        )
+
+        semaphore_release_total = CounterMetricFamily(
+            "semaphore_release_total",
+            "Total number of leases released per semaphore.\n"
+            "Note: if a semaphore is closed while there are still leases active, this count will not equal "
+            "`semaphore_acquired_total` after execution.",
+            labels=["name"],
+        )
+
+        semaphore_average_pending_lease_time = GaugeMetricFamily(
+            "semaphore_average_pending_lease_time",
+            "Exponential moving average of the time it took to acquire a lease per semaphore.\n"
+            "Note: this only includes time spent on scheduler side, "
+            "it does"
+            " not include time spent on communication.\n"
+            "Note: this average is calculated based on order of leases instead of time of lease acquisition.",
+            labels=["name"],
+            unit="s",
+        )
+
+        for semaphore_name, semaphore_max_leases in sem_ext.max_leases.items():
+            semaphore_max_leases_family.add_metric(
+                [semaphore_name], semaphore_max_leases
+            )
+            semaphore_active_leases_family.add_metric(
+                [semaphore_name], len(sem_ext.leases[semaphore_name])
+            )
+            semaphore_pending_leases.add_metric(
+                [semaphore_name], sem_ext.metrics["pending"][semaphore_name]
+            )
+            semaphore_acquire_total.add_metric(
+                [semaphore_name], sem_ext.metrics["acquire_total"][semaphore_name]
+            )
+            semaphore_release_total.add_metric(
+                [semaphore_name], sem_ext.metrics["release_total"][semaphore_name]
+            )
+            semaphore_average_pending_lease_time.add_metric(
+                [semaphore_name],
+                sem_ext.metrics["average_pending_lease_time"][semaphore_name],
+            )
+        yield semaphore_max_leases_family
+        yield semaphore_active_leases_family
+        yield semaphore_pending_leases
+        yield semaphore_acquire_total
+        yield semaphore_release_total
+        yield semaphore_average_pending_lease_time
diff --git a/distributed/http/scheduler/tests/test_semaphore_http.py b/distributed/http/scheduler/tests/test_semaphore_http.py
new file mode 100644
index 00000000000..a5d5e7c37bc
--- /dev/null
+++ b/distributed/http/scheduler/tests/test_semaphore_http.py
@@ -0,0 +1,79 @@
+import pytest
+
+from tornado.httpclient import AsyncHTTPClient
+
+from distributed.utils_test import gen_cluster
+from distributed import Semaphore
+
+
+@gen_cluster(client=True, clean_kwargs={"threads": False})
+async def test_prometheus_collect_task_states(c, s, a, b):
+    pytest.importorskip("prometheus_client")
+    from prometheus_client.parser import text_string_to_metric_families
+
+    http_client = AsyncHTTPClient()
+
+    async def fetch_metrics():
+        port = s.http_server.port
+        response = await http_client.fetch(f"http://localhost:{port}/metrics")
+        txt = response.body.decode("utf8")
+        families = {
+            family.name: family
+            for family in text_string_to_metric_families(txt)
+            if family.name.startswith("semaphore_")
+        }
+        return families
+
+    active_metrics = await fetch_metrics()
+
+    expected_metrics = {
+        "semaphore_max_leases",
+        "semaphore_active_leases",
+        "semaphore_pending_leases",
+        "semaphore_acquire",
+        "semaphore_release",
+        "semaphore_average_pending_lease_time_s",
+    }
+
+    assert active_metrics.keys() == expected_metrics
+    for v in active_metrics.values():  # Not yet any semaphore created
+        assert v.samples == []
+
+    sem = await Semaphore(name="test", max_leases=2)
+
+    active_metrics = await fetch_metrics()
+    assert active_metrics.keys() == expected_metrics
+    # Assert values are set upon intialization
+    for name, v in active_metrics.items():
+        samples = v.samples
+        assert len(samples) == 1
+        sample = samples.pop()
+        assert sample.labels["name"] == "test"
+        if name == "semaphore_max_leases":
+            assert sample.value == 2
+        else:
+            assert sample.value == 0
+
+    assert await sem.acquire()
+    active_metrics = await fetch_metrics()
+    assert active_metrics["semaphore_max_leases"].samples[0].value == 2
+    assert active_metrics["semaphore_active_leases"].samples[0].value == 1
+    assert active_metrics["semaphore_average_pending_lease_time_s"].samples[0].value > 0
+    assert active_metrics["semaphore_acquire"].samples[0].value == 1
+    assert active_metrics["semaphore_release"].samples[0].value == 0
+    assert active_metrics["semaphore_pending_leases"].samples[0].value == 0
+
+    await sem.release()
+    active_metrics = await fetch_metrics()
+    assert active_metrics["semaphore_max_leases"].samples[0].value == 2
+    assert active_metrics["semaphore_active_leases"].samples[0].value == 0
+    assert active_metrics["semaphore_average_pending_lease_time_s"].samples[0].value > 0
+    assert active_metrics["semaphore_acquire"].samples[0].value == 1
+    assert active_metrics["semaphore_release"].samples[0].value == 1
+    assert active_metrics["semaphore_pending_leases"].samples[0].value == 0
+
+    await sem.close()
+    active_metrics = await fetch_metrics()
+    assert active_metrics.keys() == expected_metrics
+    for v in active_metrics.values():
+        assert v.samples == []
diff --git a/distributed/semaphore.py b/distributed/semaphore.py
index b853f99002f..bd7e52f2e40 100644
--- a/distributed/semaphore.py
+++ b/distributed/semaphore.py
@@ -1,15 +1,17 @@
+import asyncio
+import logging
 import uuid
+import warnings
+from asyncio import TimeoutError
 from collections import defaultdict, deque
-import asyncio
+
 import dask
-from asyncio import TimeoutError
 from tornado.ioloop import PeriodicCallback
+
+from distributed.utils_comm import retry_operation
+from .metrics import time
 from .utils import log_errors, parse_timedelta
 from .worker import get_client
-from .metrics import time
-import warnings
-import logging
-from distributed.utils_comm import retry_operation
 
 logger = logging.getLogger(__name__)
 
@@ -67,16 +69,24 @@ def __init__(self, scheduler):
 
         self.scheduler.extensions["semaphores"] = self
 
+        # {metric_name: {semaphore_name: metric}}
+        self.metrics = {
+            "acquire_total": defaultdict(int),  # counter
+            "release_total": defaultdict(int),  # counter
+            "average_pending_lease_time": defaultdict(float),  # gauge
+            "pending": defaultdict(int),  # gauge
+        }
+
         validation_callback_time = parse_timedelta(
             dask.config.get("distributed.scheduler.locks.lease-validation-interval"),
             default="s",
         )
         self._pc_lease_timeout = PeriodicCallback(
-            self._check_lease_timeout, validation_callback_time * 1000,
+            self._check_lease_timeout, validation_callback_time * 1000
         )
         self._pc_lease_timeout.start()
         self.lease_timeout = parse_timedelta(
-            dask.config.get("distributed.scheduler.locks.lease-timeout"), default="s",
+            dask.config.get("distributed.scheduler.locks.lease-timeout"), default="s"
         )
 
     async def get_value(self, comm=None, name=None):
@@ -122,6 +132,7 @@ def _get_lease(self, name, lease_id):
             now = time()
             logger.info("Acquire lease %s for %s at %s", lease_id, name, now)
             self.leases[name][lease_id] = now
+            self.metrics["acquire_total"][name] += 1
         else:
             result = False
         return result
@@ -141,6 +152,7 @@ async def acquire(self, comm=None, name=None, timeout=None, lease_id=None):
             w = _Watch(timeout)
             w.start()
 
+            self.metrics["pending"][name] += 1
             while True:
                 logger.info(
                     "Trying to acquire %s for %s with %ss left.",
@@ -172,6 +184,12 @@ async def acquire(self, comm=None, name=None, timeout=None, lease_id=None):
                     result,
                     w.elapsed(),
                 )
+                # We're about to return, so the lease is no longer "pending"
+                self.metrics["average_pending_lease_time"][name] = (
+                    self.metrics["average_pending_lease_time"][name] + w.elapsed()
+                ) / 2
+                self.metrics["pending"][name] -= 1
+
                 return result
 
     def release(self, comm=None, name=None, lease_id=None):
@@ -196,6 +214,7 @@ def _release_value(self, name, lease_id):
         # Everything needs to be atomic here.
         del self.leases[name][lease_id]
         self.events[name].set()
+        self.metrics["release_total"][name] += 1
 
     def _check_lease_timeout(self):
         now = time()
@@ -235,6 +254,16 @@ def close(self, comm=None, name=None):
                         RuntimeWarning,
                     )
                 del self.leases[name]
+            if name in self.metrics["pending"]:
+                if self.metrics["pending"][name]:
+                    warnings.warn(
+                        f"Closing semaphore {name} but there remain pending leases",
+                        RuntimeWarning,
+                    )
+            # Clean-up state of semaphore metrics
+            for _, metric_dict in self.metrics.items():
+                if name in metric_dict:
+                    del metric_dict[name]
 
 
 class Semaphore:
@@ -394,7 +423,7 @@ async def _refresh_leases(self):
     async def _acquire(self, timeout=None):
         lease_id = uuid.uuid4().hex
         logger.info(
-            "%s requests lease for %s with ID %s", self.client.id, self.name, lease_id,
+            "%s requests lease for %s with ID %s", self.client.id, self.name, lease_id
         )
 
         # Using a unique lease id generated here allows us to retry since the
@@ -433,7 +462,7 @@ def release(self):
         lease_id = self._leases.popleft()
         logger.info("%s releases %s for %s", self.client.id, lease_id, self.name)
         return self.client.sync(
-            self.client.scheduler.semaphore_release, name=self.name, lease_id=lease_id,
+            self.client.scheduler.semaphore_release, name=self.name, lease_id=lease_id
         )
 
     def get_value(self):
diff --git a/distributed/tests/test_semaphore.py b/distributed/tests/test_semaphore.py
index fc0a6172a85..4839f5054f8 100644
--- a/distributed/tests/test_semaphore.py
+++ b/distributed/tests/test_semaphore.py
@@ -1,9 +1,10 @@
+import asyncio
 import pickle
 import dask
 import pytest
 from dask.distributed import Client
-from time import sleep
-from distributed import Semaphore
+from time import time, sleep
+from distributed import Semaphore, fire_and_forget
 from distributed.comm import Comm
 from distributed.core import ConnectionPool
 from distributed.metrics import time
@@ -205,10 +206,26 @@ async def test_close_async(c, s, a, b):
     ):
         await sem.acquire()
 
+    sem2 = await Semaphore(name="t2", max_leases=1)
+    assert await sem2.acquire()
+
+    def f(sem_):
+        return sem_.acquire()
+
     semaphore_object = s.extensions["semaphores"]
+    fire_and_forget(c.submit(f, sem_=sem2))
+    while not semaphore_object.metrics["pending"]["t2"]:  # Wait for the pending lease
+        await asyncio.sleep(0.01)
+    with pytest.warns(
+        RuntimeWarning, match="Closing semaphore .* but there remain pending leases"
+    ):
+        await sem2.close()
+
     assert not semaphore_object.max_leases
     assert not semaphore_object.leases
     assert not semaphore_object.events
+    for metric_dict in semaphore_object.metrics.values():
+        assert not metric_dict
 
 
 def test_close_sync(client):
@@ -420,7 +437,7 @@ def observe_state(sem):
     assert await sem.get_value() == 0
 
 
-@gen_cluster(client=True,)
+@gen_cluster(client=True)
 async def test_timeout_zero(c, s, a, b):
     # Depending on the internals a timeout zero cannot work, e.g. when the
     # initial try already includes a wait. Since some test cases use this, it is
@@ -443,3 +460,33 @@ async def test_getvalue(c, s, a, b):
     assert await sem.get_value() == 1
     await sem.release()
     assert await sem.get_value() == 0
+
+
+@gen_cluster(client=True)
+async def test_metrics(c, s, a, b):
+    from collections import defaultdict
+
+    sem = await Semaphore(name="test", max_leases=5)
+
+    before_acquiring = time()
+
+    assert await sem.acquire()
+    assert await sem.acquire()
+
+    expected_average_pending_lease_time = (time() - before_acquiring) / 2
+    epsilon = max(0.1, 0.5 * expected_average_pending_lease_time)
+
+    sem_ext = s.extensions["semaphores"]
+
+    actual = sem_ext.metrics.copy()
+    assert (
+        expected_average_pending_lease_time - epsilon
+        <= actual.pop("average_pending_lease_time")["test"]
+        <= expected_average_pending_lease_time + epsilon
+    )
+    expected = {
+        "acquire_total": defaultdict(int, {"test": 2}),
+        "release_total": defaultdict(int),
+        "pending": defaultdict(int, {"test": 0}),
+    }
+    assert actual == expected

From c93e8b7abfb813480bb81aab04b838b19bafbdd7 Mon Sep 17 00:00:00 2001
From: Lucas Rademaker <44430780+lr4d@users.noreply.github.com>
Date: Wed, 20 May 2020 11:07:20 +0200
Subject: [PATCH 0836/1550] Make re-creating conda environments from the CI
 output easier (#3816)

Co-authored-by: lr4d <lr4d@users.noreply.github.com>
---
 continuous_integration/travis/install.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/continuous_integration/travis/install.sh b/continuous_integration/travis/install.sh
index 8cc862cfadd..b1dfbc9ce1c 100644
--- a/continuous_integration/travis/install.sh
+++ b/continuous_integration/travis/install.sh
@@ -96,8 +96,8 @@ fi
 python -m pip install --no-deps -e .
 
 # For debugging
-echo -e "--\n--Conda Environment\n--"
-conda list
+echo -e "--\n--Conda Environment (re-create this with \`conda env create --name <name> -f <output_file>\`)\n--"
+conda env export | grep -E -v '^prefix:.*$'
 
 echo -e "--\n--Pip Environment\n--"
 python -m pip list --format=columns

From cc8140a8cd62de08e4a7d00189603547c19a7940 Mon Sep 17 00:00:00 2001
From: Nils Braun <nils-braun@users.noreply.github.com>
Date: Wed, 20 May 2020 23:38:12 +0200
Subject: [PATCH 0837/1550] Replace utils.ignoring with contextlib.suppress
 (#3819)

---
 distributed/cli/dask_worker.py           |  4 ++--
 distributed/client.py                    | 25 ++++++++++++------------
 distributed/comm/core.py                 |  5 +++--
 distributed/core.py                      |  4 ++--
 distributed/deploy/__init__.py           |  4 ++--
 distributed/deploy/cluster.py            | 10 +++++-----
 distributed/deploy/spec.py               | 12 ++++++------
 distributed/diagnostics/__init__.py      |  9 +++++----
 distributed/diagnostics/progressbar.py   |  9 +++++----
 distributed/nanny.py                     |  4 ++--
 distributed/node.py                      |  5 +++--
 distributed/protocol/__init__.py         |  5 ++---
 distributed/protocol/compression.py      | 13 ++++++------
 distributed/pubsub.py                    |  5 +++--
 distributed/scheduler.py                 | 12 ++++++------
 distributed/tests/test_client.py         |  6 +++---
 distributed/tests/test_failed_workers.py | 11 ++++++-----
 distributed/tests/test_nanny.py          |  5 +++--
 distributed/tests/test_stress.py         |  5 +++--
 distributed/utils.py                     | 16 ++++-----------
 distributed/utils_test.py                | 17 ++++++++--------
 distributed/variable.py                  |  9 +++++----
 distributed/versions.py                  |  8 ++++----
 distributed/worker.py                    |  8 ++++----
 24 files changed, 105 insertions(+), 106 deletions(-)

diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
index da809d83ac2..fcb10b51092 100755
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -1,5 +1,6 @@
 import asyncio
 import atexit
+from contextlib import suppress
 import logging
 import gc
 import os
@@ -9,7 +10,6 @@
 
 import click
 import dask
-from dask.utils import ignoring
 from dask.system import CPU_COUNT
 from distributed import Nanny, Security
 from distributed.cli.utils import check_python_3, install_signal_handlers
@@ -381,7 +381,7 @@ def del_pid_file():
             "dask-worker SCHEDULER_ADDRESS:8786"
         )
 
-    with ignoring(TypeError, ValueError):
+    with suppress(TypeError, ValueError):
         name = int(name)
 
     if "DASK_INTERNAL_INHERIT_CONFIG" in os.environ:
diff --git a/distributed/client.py b/distributed/client.py
index 4e5e3bff1c1..8773311194d 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -4,7 +4,7 @@
 from collections.abc import Iterator
 from concurrent.futures import ThreadPoolExecutor
 from concurrent.futures._base import DoneAndNotDoneFutures
-from contextlib import contextmanager
+from contextlib import contextmanager, suppress
 from contextvars import ContextVar
 import copy
 import errno
@@ -72,7 +72,6 @@
 from .utils import (
     All,
     sync,
-    ignoring,
     tokey,
     log_errors,
     str_graph,
@@ -654,7 +653,7 @@ def __init__(
         elif hasattr(address, "scheduler_address"):
             # It's a LocalCluster or LocalCluster-compatible object
             self.cluster = address
-            with ignoring(AttributeError):
+            with suppress(AttributeError):
                 loop = address.loop
             if security is None:
                 security = getattr(self.cluster, "security", None)
@@ -1283,7 +1282,7 @@ def _handle_restart(self):
         for state in self.futures.values():
             state.cancel()
         self.futures.clear()
-        with ignoring(AttributeError):
+        with suppress(AttributeError):
             self._restart_event.set()
 
     def _handle_error(self, exception=None):
@@ -1303,7 +1302,7 @@ async def _close(self, fast=False):
         with log_errors():
             _del_global_client(self)
             self._scheduler_identity = {}
-            with ignoring(AttributeError):
+            with suppress(AttributeError):
                 # clear the dask.config set keys
                 with self._set_config:
                     pass
@@ -1320,7 +1319,7 @@ async def _close(self, fast=False):
 
             # Give the scheduler 'stream-closed' message 100ms to come through
             # This makes the shutdown slightly smoother and quieter
-            with ignoring(AttributeError, asyncio.CancelledError, TimeoutError):
+            with suppress(AttributeError, asyncio.CancelledError, TimeoutError):
                 await asyncio.wait_for(
                     asyncio.shield(self._handle_scheduler_coroutine), 0.1
                 )
@@ -1336,7 +1335,7 @@ async def _close(self, fast=False):
                 self._release_key(key=key)
 
             if self._start_arg is None:
-                with ignoring(AttributeError):
+                with suppress(AttributeError):
                     await self.cluster.close()
 
             await self.rpc.close()
@@ -1350,17 +1349,17 @@ async def _close(self, fast=False):
             for f in self.coroutines:
                 # cancel() works on asyncio futures (Tornado 5)
                 # but is a no-op on Tornado futures
-                with ignoring(RuntimeError):
+                with suppress(RuntimeError):
                     f.cancel()
                 if f.cancelled():
                     coroutines.remove(f)
             del self.coroutines[:]
 
             if not fast:
-                with ignoring(TimeoutError, asyncio.CancelledError):
+                with suppress(TimeoutError, asyncio.CancelledError):
                     await asyncio.wait_for(asyncio.gather(*coroutines), 2)
 
-            with ignoring(AttributeError):
+            with suppress(AttributeError):
                 await self.scheduler.close_rpc()
 
             self.scheduler = None
@@ -1402,7 +1401,7 @@ def close(self, timeout=no_default):
             return future
 
         if self._start_arg is None:
-            with ignoring(AttributeError):
+            with suppress(AttributeError):
                 f = self.cluster.close()
                 if asyncio.iscoroutine(f):
 
@@ -1423,7 +1422,7 @@ async def _shutdown(self):
         if self.cluster:
             await self.cluster.close()
         else:
-            with ignoring(CommClosedError):
+            with suppress(CommClosedError):
                 self.status = "closing"
                 await self.scheduler.terminate(close_workers=True)
 
@@ -1810,7 +1809,7 @@ async def wait(k):
         while True:
             logger.debug("Waiting on futures to clear before gather")
 
-            with ignoring(AllExit):
+            with suppress(AllExit):
                 await All(
                     [wait(key) for key in keys if key in self.futures],
                     quiet_exceptions=AllExit,
diff --git a/distributed/comm/core.py b/distributed/comm/core.py
index bfae9e8dcc0..00a6d87a52a 100644
--- a/distributed/comm/core.py
+++ b/distributed/comm/core.py
@@ -1,5 +1,6 @@
 from abc import ABC, abstractmethod, abstractproperty
 import asyncio
+from contextlib import suppress
 import inspect
 import logging
 import weakref
@@ -7,7 +8,7 @@
 import dask
 
 from ..metrics import time
-from ..utils import parse_timedelta, ignoring, TimeoutError
+from ..utils import parse_timedelta, TimeoutError
 from . import registry
 from .addressing import parse_address
 
@@ -223,7 +224,7 @@ def _raise(error):
                 future = connector.connect(
                     loc, deserialize=deserialize, **connection_args
                 )
-                with ignoring(TimeoutError):
+                with suppress(TimeoutError):
                     comm = await asyncio.wait_for(
                         future, timeout=min(deadline - time(), 1)
                     )
diff --git a/distributed/core.py b/distributed/core.py
index f05b85f903e..829d2095083 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -1,5 +1,6 @@
 import asyncio
 from collections import defaultdict, deque
+from contextlib import suppress
 from functools import partial
 import inspect
 import logging
@@ -29,7 +30,6 @@
     is_coroutine_function,
     get_traceback,
     truncate_exception,
-    ignoring,
     shutting_down,
     parse_timedelta,
     has_keyword,
@@ -167,7 +167,7 @@ def stop():
             )
 
         # Statistics counters for various events
-        with ignoring(ImportError):
+        with suppress(ImportError):
             from .counter import Digest
 
             self.digests = defaultdict(partial(Digest, loop=self.io_loop))
diff --git a/distributed/deploy/__init__.py b/distributed/deploy/__init__.py
index 35fc86fe393..0148328cd4c 100644
--- a/distributed/deploy/__init__.py
+++ b/distributed/deploy/__init__.py
@@ -1,4 +1,4 @@
-from ..utils import ignoring
+from contextlib import suppress
 
 from .cluster import Cluster
 from .local import LocalCluster
@@ -6,5 +6,5 @@
 from .spec import SpecCluster, ProcessInterface
 from .adaptive import Adaptive
 
-with ignoring(ImportError):
+with suppress(ImportError):
     from .ssh import SSHCluster
diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
index 7a6bb8669d6..1f13fd37b10 100644
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -1,4 +1,5 @@
 import asyncio
+from contextlib import suppress
 import logging
 import threading
 import warnings
@@ -11,7 +12,6 @@
 
 from ..utils import (
     log_errors,
-    ignoring,
     sync,
     Log,
     Logs,
@@ -80,12 +80,12 @@ async def _close(self):
         self.status = "closed"
 
     def close(self, timeout=None):
-        with ignoring(RuntimeError):  # loop closed during process shutdown
+        with suppress(RuntimeError):  # loop closed during process shutdown
             return self.sync(self._close, callback_timeout=timeout)
 
     def __del__(self):
         if self.status != "closed":
-            with ignoring(AttributeError, RuntimeError):  # during closing
+            with suppress(AttributeError, RuntimeError):  # during closing
                 self.loop.add_callback(self.close)
 
     async def _watch_worker_status(self, comm):
@@ -121,7 +121,7 @@ def adapt(self, Adaptive=Adaptive, **kwargs) -> Adaptive:
         --------
         >>> cluster.adapt(minimum=0, maximum=10, interval='500ms')
         """
-        with ignoring(AttributeError):
+        with suppress(AttributeError):
             self._adaptive.stop()
         if not hasattr(self, "_adaptive_options"):
             self._adaptive_options = {}
@@ -305,7 +305,7 @@ def adapt_cb(b):
             def scale_cb(b):
                 with log_errors():
                     n = request.value
-                    with ignoring(AttributeError):
+                    with suppress(AttributeError):
                         self._adaptive.stop()
                     self.scale(n)
                     update()
diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index cdde4a8162a..56160592347 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -1,5 +1,6 @@
 import asyncio
 import atexit
+from contextlib import suppress
 import copy
 import logging
 import math
@@ -14,7 +15,6 @@
 from ..utils import (
     LoopRunner,
     silence_logging,
-    ignoring,
     parse_bytes,
     parse_timedelta,
     import_term,
@@ -309,7 +309,7 @@ async def _correct_state_internal(self):
                 tasks = [self.workers[w].close() for w in to_close if w in self.workers]
                 await asyncio.wait(tasks)
                 for task in tasks:  # for tornado gen.coroutine support
-                    with ignoring(RuntimeError):
+                    with suppress(RuntimeError):
                         await task
             for name in to_close:
                 if name in self.workers:
@@ -382,7 +382,7 @@ async def _close(self):
         for future in self._futures:
             await future
         async with self._lock:
-            with ignoring(CommClosedError):
+            with suppress(CommClosedError):
                 if self.scheduler_comm:
                     await self.scheduler_comm.close(close_workers=True)
                 else:
@@ -415,7 +415,7 @@ def _threads_per_worker(self) -> int:
             raise ValueError("To scale by cores= you must specify cores per worker")
 
         for name in ["nthreads", "ncores", "threads", "cores"]:
-            with ignoring(KeyError):
+            with suppress(KeyError):
                 return self.new_spec["options"][name]
 
         if not self.new_spec:
@@ -429,7 +429,7 @@ def _memory_per_worker(self) -> int:
             )
 
         for name in ["memory_limit", "memory"]:
-            with ignoring(KeyError):
+            with suppress(KeyError):
                 return parse_bytes(self.new_spec["options"][name])
 
         raise ValueError(
@@ -606,6 +606,6 @@ async def run_spec(spec: dict, *args):
 @atexit.register
 def close_clusters():
     for cluster in list(SpecCluster._instances):
-        with ignoring(gen.TimeoutError, TimeoutError):
+        with suppress(gen.TimeoutError, TimeoutError):
             if cluster.status != "closed":
                 cluster.close(timeout=10)
diff --git a/distributed/diagnostics/__init__.py b/distributed/diagnostics/__init__.py
index 337f41b7598..390a7b94f39 100644
--- a/distributed/diagnostics/__init__.py
+++ b/distributed/diagnostics/__init__.py
@@ -1,10 +1,11 @@
-from ..utils import ignoring
+from contextlib import suppress
+
 from .graph_layout import GraphLayout
 from .plugin import SchedulerPlugin
 
-with ignoring(ImportError):
+with suppress(ImportError):
     from .progressbar import progress
-with ignoring(ImportError):
+with suppress(ImportError):
     from .resource_monitor import Occupancy
-with ignoring(ImportError):
+with suppress(ImportError):
     from .scheduler_widgets import scheduler_status
diff --git a/distributed/diagnostics/progressbar.py b/distributed/diagnostics/progressbar.py
index 4ef7254f52e..fb663524c5e 100644
--- a/distributed/diagnostics/progressbar.py
+++ b/distributed/diagnostics/progressbar.py
@@ -1,3 +1,4 @@
+from contextlib import suppress
 import logging
 import html
 from timeit import default_timer
@@ -12,7 +13,7 @@
 from ..core import connect, coerce_to_address, CommClosedError
 from ..client import default_client, futures_of
 from ..protocol.pickle import dumps
-from ..utils import ignoring, key_split, is_kernel, LoopRunner, parse_timedelta
+from ..utils import key_split, is_kernel, LoopRunner, parse_timedelta
 
 
 logger = logging.getLogger(__name__)
@@ -98,7 +99,7 @@ def _draw_stop(self, **kwargs):
         pass
 
     def __del__(self):
-        with ignoring(AttributeError):
+        with suppress(AttributeError):
             self.comm.abort()
 
 
@@ -130,7 +131,7 @@ def _draw_bar(self, remaining, all, **kwargs):
         msg = "\r[{0:<{1}}] | {2}% Completed | {3}".format(
             bar, self.width, percent, elapsed
         )
-        with ignoring(ValueError):
+        with suppress(ValueError):
             sys.stdout.write(msg)
             sys.stdout.flush()
 
@@ -286,7 +287,7 @@ def _draw_stop(self, **kwargs):
         pass
 
     def __del__(self):
-        with ignoring(AttributeError):
+        with suppress(AttributeError):
             self.comm.abort()
 
 
diff --git a/distributed/nanny.py b/distributed/nanny.py
index 821031e29eb..b09aa1c2a72 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -1,4 +1,5 @@
 import asyncio
+from contextlib import suppress
 import errno
 import logging
 from multiprocessing.queues import Empty
@@ -31,7 +32,6 @@
     json_load_robust,
     parse_timedelta,
     parse_ports,
-    ignoring,
     TimeoutError,
 )
 from .worker import run, parse_memory_limit, Worker
@@ -231,7 +231,7 @@ async def _unregister(self, timeout=10):
             return
 
         allowed_errors = (TimeoutError, CommClosedError, EnvironmentError, RPCClosed)
-        with ignoring(allowed_errors):
+        with suppress(allowed_errors):
             await asyncio.wait_for(
                 self.scheduler.unregister(address=self.worker_address), timeout
             )
diff --git a/distributed/node.py b/distributed/node.py
index eeae67ec400..52c53f3e9e7 100644
--- a/distributed/node.py
+++ b/distributed/node.py
@@ -1,3 +1,4 @@
+from contextlib import suppress
 import logging
 import warnings
 import weakref
@@ -11,7 +12,7 @@
 from .core import Server
 from .http.routing import RoutingApplication
 from .versions import get_versions
-from .utils import DequeHandler, clean_dashboard_address, ignoring
+from .utils import DequeHandler, clean_dashboard_address
 
 
 class ServerNode(Server):
@@ -125,7 +126,7 @@ def start_http_server(
             if isinstance(address, (list, tuple)):
                 address = address[0]
             if address:
-                with ignoring(ValueError):
+                with suppress(ValueError):
                     http_address["address"] = get_address_host(address)
 
         change_port = False
diff --git a/distributed/protocol/__init__.py b/distributed/protocol/__init__.py
index bb919019e04..79e77e9703b 100644
--- a/distributed/protocol/__init__.py
+++ b/distributed/protocol/__init__.py
@@ -1,3 +1,4 @@
+from contextlib import suppress
 from functools import partial
 from distutils.version import LooseVersion
 
@@ -21,8 +22,6 @@
     register_generic,
 )
 
-from ..utils import ignoring
-
 
 @dask_serialize.register_lazy("numpy")
 @dask_deserialize.register_lazy("numpy")
@@ -115,5 +114,5 @@ def _register_cudf():
 @dask_serialize.register_lazy("cuml")
 @dask_deserialize.register_lazy("cuml")
 def _register_cuml():
-    with ignoring(ImportError):
+    with suppress(ImportError):
         from cuml.comm import serialize
diff --git a/distributed/protocol/compression.py b/distributed/protocol/compression.py
index adb3c888be6..03ebf9d5662 100644
--- a/distributed/protocol/compression.py
+++ b/distributed/protocol/compression.py
@@ -3,6 +3,7 @@
 
 Includes utilities for determining whether or not to compress
 """
+from contextlib import suppress
 from functools import partial
 import logging
 import random
@@ -19,7 +20,7 @@
 except ImportError:
     blosc = False
 
-from ..utils import ignoring, ensure_bytes
+from ..utils import ensure_bytes
 
 
 compressions = {None: {"compress": identity, "decompress": identity}}
@@ -33,12 +34,12 @@
 logger = logging.getLogger(__name__)
 
 
-with ignoring(ImportError):
+with suppress(ImportError):
     import zlib
 
     compressions["zlib"] = {"compress": zlib.compress, "decompress": zlib.decompress}
 
-with ignoring(ImportError):
+with suppress(ImportError):
     import snappy
 
     def _fixed_snappy_decompress(data):
@@ -53,7 +54,7 @@ def _fixed_snappy_decompress(data):
     }
     default_compression = "snappy"
 
-with ignoring(ImportError):
+with suppress(ImportError):
     import lz4
 
     try:
@@ -95,7 +96,7 @@ def _fixed_lz4_decompress(data):
     default_compression = "lz4"
 
 
-with ignoring(ImportError):
+with suppress(ImportError):
     import zstandard
 
     zstd_compressor = zstandard.ZstdCompressor(
@@ -114,7 +115,7 @@ def zstd_decompress(data):
     compressions["zstd"] = {"compress": zstd_compress, "decompress": zstd_decompress}
 
 
-with ignoring(ImportError):
+with suppress(ImportError):
     import blosc
 
     compressions["blosc"] = {
diff --git a/distributed/pubsub.py b/distributed/pubsub.py
index cdff73ffeca..6496a60243d 100644
--- a/distributed/pubsub.py
+++ b/distributed/pubsub.py
@@ -1,12 +1,13 @@
 import asyncio
 from collections import defaultdict, deque
+from contextlib import suppress
 import logging
 import threading
 import weakref
 
 from .core import CommClosedError
 from .metrics import time
-from .utils import sync, TimeoutError, ignoring
+from .utils import sync, TimeoutError
 from .protocol.serialize import to_serialize
 
 logger = logging.getLogger(__name__)
@@ -420,7 +421,7 @@ async def _():
             try:
                 await asyncio.wait_for(_(), timeout2)
             finally:
-                with ignoring(RuntimeError):  # Python 3.6 fails here sometimes
+                with suppress(RuntimeError):  # Python 3.6 fails here sometimes
                     self.condition.release()
 
         return self.buffer.popleft()
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 029db4ef06c..cc3e64fcc8c 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1,6 +1,7 @@
 import asyncio
 from collections import defaultdict, deque
 from collections.abc import Mapping, Set
+from contextlib import suppress
 from datetime import timedelta
 from functools import partial
 import inspect
@@ -56,7 +57,6 @@
 from .security import Security
 from .utils import (
     All,
-    ignoring,
     get_fileno_limit,
     log_errors,
     key_split,
@@ -1431,7 +1431,7 @@ async def start(self):
 
         self.clear_task_state()
 
-        with ignoring(AttributeError):
+        with suppress(AttributeError):
             for c in self._worker_coroutines:
                 c.cancel()
 
@@ -1515,7 +1515,7 @@ async def close(self, comm=None, fast=False, close_workers=False):
         self.stop_services()
 
         for ext in self.extensions.values():
-            with ignoring(AttributeError):
+            with suppress(AttributeError):
                 ext.teardown()
         logger.info("Scheduler closing all comms")
 
@@ -1524,7 +1524,7 @@ async def close(self, comm=None, fast=False, close_workers=False):
             if not comm.closed():
                 comm.send({"op": "close", "report": False})
                 comm.send({"op": "close-stream"})
-            with ignoring(AttributeError):
+            with suppress(AttributeError):
                 futures.append(comm.close())
 
         for future in futures:  # TODO: do all at once
@@ -2181,7 +2181,7 @@ def remove_worker(self, comm=None, address=None, safe=False, close=True):
             )
             logger.info("Remove worker %s", ws)
             if close:
-                with ignoring(AttributeError, CommClosedError):
+                with suppress(AttributeError, CommClosedError):
                     self.stream_comms[address].send({"op": "close", "report": False})
 
             self.remove_resources(address)
@@ -2935,7 +2935,7 @@ async def restart(self, client=None, timeout=3):
 
             self.clear_task_state()
 
-            with ignoring(AttributeError):
+            with suppress(AttributeError):
                 for c in self._worker_coroutines:
                     c.cancel()
 
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 08dff3bf7bc..a347e1d325e 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -1,5 +1,6 @@
 import asyncio
 from collections import deque
+from contextlib import suppress
 from functools import partial
 import gc
 import logging
@@ -57,7 +58,6 @@
 from distributed.scheduler import Scheduler, KilledWorker
 from distributed.sizeof import sizeof
 from distributed.utils import (
-    ignoring,
     mp_context,
     sync,
     tmp_text,
@@ -3391,7 +3391,7 @@ def test_close_idempotent(c):
 @nodebug
 def test_get_returns_early(c):
     start = time()
-    with ignoring(RuntimeError):
+    with suppress(RuntimeError):
         result = c.get({"x": (throws, 1), "y": (sleep, 1)}, ["x", "y"])
     assert time() < start + 0.5
     # Futures should be released and forgotten
@@ -3402,7 +3402,7 @@ def test_get_returns_early(c):
     x = c.submit(inc, 1)
     x.result()
 
-    with ignoring(RuntimeError):
+    with suppress(RuntimeError):
         result = c.get({"x": (throws, 1), x.key: (inc, 1)}, ["x", x.key])
     assert x.key in c.futures
 
diff --git a/distributed/tests/test_failed_workers.py b/distributed/tests/test_failed_workers.py
index e1556494fe2..22d2a2ad91f 100644
--- a/distributed/tests/test_failed_workers.py
+++ b/distributed/tests/test_failed_workers.py
@@ -1,4 +1,5 @@
 import asyncio
+from contextlib import suppress
 import os
 import random
 from time import sleep
@@ -11,7 +12,7 @@
 from distributed.comm import CommClosedError
 from distributed.client import wait
 from distributed.metrics import time
-from distributed.utils import sync, ignoring, CancelledError
+from distributed.utils import sync, CancelledError
 from distributed.utils_test import (
     gen_cluster,
     cluster,
@@ -103,7 +104,7 @@ async def test_failed_worker_without_warning(c, s, a, b):
     await wait(L)
 
     original_pid = a.pid
-    with ignoring(CommClosedError):
+    with suppress(CommClosedError):
         await c._run(os._exit, 1, workers=[a.worker_address])
     start = time()
     while a.pid == original_pid:
@@ -334,14 +335,14 @@ async def test_broken_worker_during_computation(c, s, a, b):
         )
 
     await asyncio.sleep(random.random() / 20)
-    with ignoring(CommClosedError):  # comm will be closed abrupty
+    with suppress(CommClosedError):  # comm will be closed abrupty
         await c._run(os._exit, 1, workers=[n.worker_address])
 
     await asyncio.sleep(random.random() / 20)
     while len(s.workers) < 3:
         await asyncio.sleep(0.01)
 
-    with ignoring(
+    with suppress(
         CommClosedError, EnvironmentError
     ):  # perhaps new worker can't be contacted yet
         await c._run(os._exit, 1, workers=[n.worker_address])
@@ -387,7 +388,7 @@ async def test_worker_who_has_clears_after_failed_connection(c, s, a, b):
         a.release_dep(dep, report=True)
 
     n_worker_address = n.worker_address
-    with ignoring(CommClosedError):
+    with suppress(CommClosedError):
         await c._run(os._exit, 1, workers=[n_worker_address])
 
     while len(s.workers) > 2:
diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
index a02fb3c6211..327649570ed 100644
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -1,4 +1,5 @@
 import asyncio
+from contextlib import suppress
 import gc
 import logging
 import os
@@ -18,7 +19,7 @@
 from distributed.core import CommClosedError
 from distributed.metrics import time
 from distributed.protocol.pickle import dumps
-from distributed.utils import ignoring, tmpfile, TimeoutError, parse_ports
+from distributed.utils import tmpfile, TimeoutError, parse_ports
 from distributed.utils_test import (  # noqa: F401
     gen_cluster,
     gen_test,
@@ -88,7 +89,7 @@ async def test_nanny_process_failure(c, s):
     await ww.update_data(data=valmap(dumps, {"x": 1, "y": 2}))
     pid = n.pid
     assert pid is not None
-    with ignoring(CommClosedError):
+    with suppress(CommClosedError):
         await c.run(os._exit, 0, workers=[n.worker_address])
 
     start = time()
diff --git a/distributed/tests/test_stress.py b/distributed/tests/test_stress.py
index 707b93c03cf..d699ac9452d 100644
--- a/distributed/tests/test_stress.py
+++ b/distributed/tests/test_stress.py
@@ -1,4 +1,5 @@
 import asyncio
+from contextlib import suppress
 import random
 import sys
 from operator import add
@@ -11,7 +12,7 @@
 from distributed import Client, wait, Nanny
 from distributed.config import config
 from distributed.metrics import time
-from distributed.utils import All, ignoring, CancelledError
+from distributed.utils import All, CancelledError
 from distributed.utils_test import (
     gen_cluster,
     cluster,
@@ -160,7 +161,7 @@ async def test_stress_scatter_death(c, s, *workers):
         await w.close()
         alive.remove(w)
 
-    with ignoring(CancelledError):
+    with suppress(CancelledError):
         await c.gather(futures)
 
     futures = None
diff --git a/distributed/utils.py b/distributed/utils.py
index ec98617254f..f43b2f7acc0 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -4,7 +4,7 @@
 import click
 from collections import deque, OrderedDict, UserDict
 from concurrent.futures import ThreadPoolExecutor, CancelledError  # noqa: F401
-from contextlib import contextmanager
+from contextlib import contextmanager, suppress
 import functools
 from hashlib import md5
 import html
@@ -192,14 +192,6 @@ def get_ip_interface(ifname):
     raise ValueError("interface %r doesn't have an IPv4 address" % (ifname,))
 
 
-@contextmanager
-def ignoring(*exceptions):
-    try:
-        yield
-    except exceptions as e:
-        pass
-
-
 # FIXME: this breaks if changed to async def...
 @gen.coroutine
 def ignore_exceptions(coroutines, *exceptions):
@@ -211,7 +203,7 @@ def ignore_exceptions(coroutines, *exceptions):
     wait_iterator = gen.WaitIterator(*coroutines)
     results = []
     while not wait_iterator.done():
-        with ignoring(*exceptions):
+        with suppress(*exceptions):
             result = yield wait_iterator.next()
             results.append(result)
     raise gen.Return(results)
@@ -481,7 +473,7 @@ def _real_stop(self, timeout):
             try:
                 self._loop.add_callback(self._loop.stop)
                 self._loop_thread.join(timeout=timeout)
-                with ignoring(KeyError):  # IOLoop can be missing
+                with suppress(KeyError):  # IOLoop can be missing
                     self._loop.close()
             finally:
                 self._loop_thread = None
@@ -1033,7 +1025,7 @@ def import_file(path):
         names_to_import.append(name)
     if ext == ".py":  # Ensure that no pyc file will be reused
         cache_file = cache_from_source(path)
-        with ignoring(OSError):
+        with suppress(OSError):
             os.remove(cache_file)
     if ext in (".egg", ".zip", ".pyz"):
         if path not in sys.path:
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index 3bb0fc4b2b4..c369c95aa53 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -1,7 +1,7 @@
 import asyncio
 import collections
 import gc
-from contextlib import contextmanager
+from contextlib import contextmanager, suppress
 import copy
 import functools
 from glob import glob
@@ -48,7 +48,6 @@
 from .proctitle import enable_proctitle_on_children
 from .security import Security
 from .utils import (
-    ignoring,
     log_errors,
     mp_context,
     get_ip,
@@ -704,12 +703,12 @@ def cluster(
             for proc in [w["proc"] for w in workers]:
                 proc.join(timeout=2)
 
-            with ignoring(UnboundLocalError):
+            with suppress(UnboundLocalError):
                 del worker, w, proc
             del workers[:]
 
             for fn in glob("_test_worker-*"):
-                with ignoring(OSError):
+                with suppress(OSError):
                     shutil.rmtree(fn)
 
         try:
@@ -730,7 +729,7 @@ async def disconnect(addr, timeout=3, rpc_kwargs=None):
     rpc_kwargs = rpc_kwargs or {}
 
     async def do_disconnect():
-        with ignoring(EnvironmentError, CommClosedError):
+        with suppress(EnvironmentError, CommClosedError):
             with rpc(addr, **rpc_kwargs) as w:
                 await w.terminate(close=True)
 
@@ -818,7 +817,7 @@ async def end_cluster(s, workers):
     logger.debug("Closing out test cluster")
 
     async def end_worker(w):
-        with ignoring(TimeoutError, CommClosedError, EnvironmentError):
+        with suppress(TimeoutError, CommClosedError, EnvironmentError):
             await w.close(report=False)
 
     await asyncio.gather(*[end_worker(w) for w in workers])
@@ -990,7 +989,7 @@ def terminate_process(proc):
             proc.wait(10)
         finally:
             # Make sure we don't leave the process lingering around
-            with ignoring(OSError):
+            with suppress(OSError):
                 proc.kill()
 
 
@@ -1484,7 +1483,7 @@ def check_instances():
     _global_clients.clear()
 
     for w in Worker._instances:
-        with ignoring(RuntimeError):  # closed IOLoop
+        with suppress(RuntimeError):  # closed IOLoop
             w.loop.add_callback(w.close, report=False, executor_wait=False)
             if w.status == "running":
                 w.loop.add_callback(w.close)
@@ -1536,7 +1535,7 @@ def null():
 
                         yield loop
 
-                        with ignoring(AttributeError):
+                        with suppress(AttributeError):
                             del thread_state.on_event_loop_thread
 
 
diff --git a/distributed/variable.py b/distributed/variable.py
index dc717533a28..6729bc2b69c 100644
--- a/distributed/variable.py
+++ b/distributed/variable.py
@@ -1,12 +1,13 @@
 import asyncio
 from collections import defaultdict
+from contextlib import suppress
 import logging
 import uuid
 
 from tlz import merge
 
 from .client import Future, Client
-from .utils import tokey, log_errors, TimeoutError, ignoring
+from .utils import tokey, log_errors, TimeoutError
 from .worker import get_client
 
 logger = logging.getLogger(__name__)
@@ -87,7 +88,7 @@ async def _():  # Python 3.6 is odd and requires special help here
 
                 await asyncio.wait_for(_(), timeout=left)
             finally:
-                with ignoring(RuntimeError):  # Python 3.6 loses lock on finally clause
+                with suppress(RuntimeError):  # Python 3.6 loses lock on finally clause
                     self.started.release()
 
         record = self.variables[name]
@@ -113,9 +114,9 @@ async def delete(self, stream=None, name=None, client=None):
             else:
                 if old["type"] == "Future":
                     await self.release(old["value"], name)
-            with ignoring(KeyError):
+            with suppress(KeyError):
                 del self.waiting_conditions[name]
-            with ignoring(KeyError):
+            with suppress(KeyError):
                 del self.variables[name]
 
 
diff --git a/distributed/versions.py b/distributed/versions.py
index d800f65ec63..2062f48568c 100644
--- a/distributed/versions.py
+++ b/distributed/versions.py
@@ -68,13 +68,13 @@ def get_system_info():
 
 def version_of_package(pkg):
     """ Try a variety of common ways to get the version of a package """
-    from .utils import ignoring
+    from contextlib import suppress
 
-    with ignoring(AttributeError):
+    with suppress(AttributeError):
         return pkg.__version__
-    with ignoring(AttributeError):
+    with suppress(AttributeError):
         return str(pkg.version)
-    with ignoring(AttributeError):
+    with suppress(AttributeError):
         return ".".join(map(str, pkg.version_info))
     return None
 
diff --git a/distributed/worker.py b/distributed/worker.py
index 3eb698289d3..8dff85e653c 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -2,6 +2,7 @@
 import bisect
 from collections import defaultdict, deque, namedtuple
 from collections.abc import MutableMapping
+from contextlib import suppress
 from datetime import timedelta
 import errno
 from functools import partial
@@ -49,7 +50,6 @@
     has_arg,
     _maybe_complex,
     log_errors,
-    ignoring,
     import_file,
     silence_logging,
     thread_state,
@@ -1129,7 +1129,7 @@ async def close(
 
             for pc in self.periodic_callbacks.values():
                 pc.stop()
-            with ignoring(EnvironmentError, TimeoutError):
+            with suppress(EnvironmentError, TimeoutError):
                 if report and self.contact_address is not None:
                     await asyncio.wait_for(
                         self.scheduler.unregister(
@@ -1150,7 +1150,7 @@ async def close(
                 self.batched_stream.send({"op": "close-stream"})
 
             if self.batched_stream:
-                with ignoring(TimeoutError):
+                with suppress(TimeoutError):
                     await self.batched_stream.close(timedelta(seconds=timeout))
 
             self.actor_executor._work_queue.queue.clear()
@@ -3186,7 +3186,7 @@ def parse_memory_limit(memory_limit, nthreads, total_cores=CPU_COUNT):
 
     if memory_limit == "auto":
         memory_limit = int(system.MEMORY_LIMIT * min(1, nthreads / total_cores))
-    with ignoring(ValueError, TypeError):
+    with suppress(ValueError, TypeError):
         memory_limit = float(memory_limit)
         if isinstance(memory_limit, float) and memory_limit <= 1:
             memory_limit = int(memory_limit * system.MEMORY_LIMIT)

From ddc6377b76f957ef4a3a607f1dc2cf2ba772abc7 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Thu, 21 May 2020 03:35:23 -0700
Subject: [PATCH 0838/1550] Support Pickle's protocol 5 (#3784)

* Assign `header` and `frames` before returning

* Import `HIGHEST_PROTOCOL` at top-level

* Collect keyword arguments to `*dumps`

* Assign `result` and `return` once at end

* Support out-of-band buffer serialization

* Require `cloudpickle` version `1.3.0`

Needed for out-of-band buffer handling.

* Test Pickle with out-of-band buffers

* Import `PickleBuffer` (if available)

* Test `serialize`/`deserialize` with `pickle`

* Check serialized header + frames

* Check out-of-band buffers' content

* Take `memoryview` of `PickleBuffer` for testing

* Collect buffers internally first

Before calling the user provided buffer callback, collect buffers in an
internal list. That way if the mechanism of pickling needs to be
changed, the internal list can be purged before handing these to the
user. At the end of pickling, make sure the user's buffer callback is
called on each buffer in order.

* Only collect buffers if `buffer_callback` exists

* Use `elif` instead for simplicity

* Use De Morgan's law to simplify logic

* Check `buffer_callback` before calling it

Co-authored-by: Jim Crist-Harif <jcrist@users.noreply.github.com>

* Use `buffer.clear()` instead of `del buffer[:]`

Co-authored-by: Jim Crist-Harif <jcrist@users.noreply.github.com>
---
 distributed/protocol/pickle.py            | 37 ++++++----
 distributed/protocol/serialize.py         |  9 ++-
 distributed/protocol/tests/test_pickle.py | 82 ++++++++++++++++++++++-
 requirements.txt                          |  2 +-
 4 files changed, 112 insertions(+), 18 deletions(-)

diff --git a/distributed/protocol/pickle.py b/distributed/protocol/pickle.py
index 9a1f135444f..9774202e4fe 100644
--- a/distributed/protocol/pickle.py
+++ b/distributed/protocol/pickle.py
@@ -1,5 +1,6 @@
 import logging
 import pickle
+from pickle import HIGHEST_PROTOCOL
 
 import cloudpickle
 
@@ -23,36 +24,46 @@ def _always_use_pickle_for(x):
         return False
 
 
-def dumps(x):
+def dumps(x, *, buffer_callback=None):
     """ Manage between cloudpickle and pickle
 
     1.  Try pickle
     2.  If it is short then check if it contains __main__
     3.  If it is long, then first check type, then check __main__
     """
+    buffers = []
+    dump_kwargs = {"protocol": HIGHEST_PROTOCOL}
+    if HIGHEST_PROTOCOL >= 5 and buffer_callback is not None:
+        dump_kwargs["buffer_callback"] = buffers.append
     try:
-        result = pickle.dumps(x, protocol=pickle.HIGHEST_PROTOCOL)
+        buffers.clear()
+        result = pickle.dumps(x, **dump_kwargs)
         if len(result) < 1000:
             if b"__main__" in result:
-                return cloudpickle.dumps(x, protocol=pickle.HIGHEST_PROTOCOL)
-            else:
-                return result
-        else:
-            if _always_use_pickle_for(x) or b"__main__" not in result:
-                return result
-            else:
-                return cloudpickle.dumps(x, protocol=pickle.HIGHEST_PROTOCOL)
+                buffers.clear()
+                result = cloudpickle.dumps(x, **dump_kwargs)
+        elif not _always_use_pickle_for(x) and b"__main__" in result:
+            buffers.clear()
+            result = cloudpickle.dumps(x, **dump_kwargs)
     except Exception:
         try:
-            return cloudpickle.dumps(x, protocol=pickle.HIGHEST_PROTOCOL)
+            buffers.clear()
+            result = cloudpickle.dumps(x, **dump_kwargs)
         except Exception as e:
             logger.info("Failed to serialize %s. Exception: %s", x, e)
             raise
+    if buffer_callback is not None:
+        for b in buffers:
+            buffer_callback(b)
+    return result
 
 
-def loads(x):
+def loads(x, *, buffers=()):
     try:
-        return pickle.loads(x)
+        if buffers:
+            return pickle.loads(x, buffers=buffers)
+        else:
+            return pickle.loads(x)
     except Exception:
         logger.info("Failed to deserialize %s", x[:10000], exc_info=True)
         raise
diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index 4d02bc65207..e4fba2b7ba9 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -52,11 +52,16 @@ def dask_loads(header, frames):
 
 
 def pickle_dumps(x):
-    return {"serializer": "pickle"}, [pickle.dumps(x)]
+    header = {"serializer": "pickle"}
+    frames = [None]
+    buffer_callback = lambda f: frames.append(memoryview(f))
+    frames[0] = pickle.dumps(x, buffer_callback=buffer_callback)
+    return header, frames
 
 
 def pickle_loads(header, frames):
-    return pickle.loads(b"".join(frames))
+    x, buffers = frames[0], frames[1:]
+    return pickle.loads(x, buffers=buffers)
 
 
 def msgpack_dumps(x):
diff --git a/distributed/protocol/tests/test_pickle.py b/distributed/protocol/tests/test_pickle.py
index 681992ef844..f4a4ec7f8ee 100644
--- a/distributed/protocol/tests/test_pickle.py
+++ b/distributed/protocol/tests/test_pickle.py
@@ -6,22 +6,93 @@
 
 import pytest
 
-from distributed.protocol.pickle import dumps, loads
+from distributed.protocol import deserialize, serialize
+from distributed.protocol.pickle import HIGHEST_PROTOCOL, dumps, loads
+
+try:
+    from pickle import PickleBuffer
+except ImportError:
+    pass
 
 
 def test_pickle_data():
     data = [1, b"123", "123", [123], {}, set()]
     for d in data:
         assert loads(dumps(d)) == d
+        assert deserialize(*serialize(d, serializers=("pickle",))) == d
+
+
+def test_pickle_out_of_band():
+    class MemoryviewHolder:
+        def __init__(self, mv):
+            self.mv = memoryview(mv)
+
+        def __reduce_ex__(self, protocol):
+            if protocol >= 5:
+                return MemoryviewHolder, (PickleBuffer(self.mv),)
+            else:
+                return MemoryviewHolder, (self.mv.tobytes(),)
+
+    mv = memoryview(b"123")
+    mvh = MemoryviewHolder(mv)
+
+    if HIGHEST_PROTOCOL >= 5:
+        l = []
+        d = dumps(mvh, buffer_callback=l.append)
+        mvh2 = loads(d, buffers=l)
+
+        assert len(l) == 1
+        assert isinstance(l[0], PickleBuffer)
+        assert memoryview(l[0]) == mv
+    else:
+        mvh2 = loads(dumps(mvh))
+
+    assert isinstance(mvh2, MemoryviewHolder)
+    assert isinstance(mvh2.mv, memoryview)
+    assert mvh2.mv == mv
+
+    h, f = serialize(mvh, serializers=("pickle",))
+    mvh3 = deserialize(h, f)
+
+    assert isinstance(mvh3, MemoryviewHolder)
+    assert isinstance(mvh3.mv, memoryview)
+    assert mvh3.mv == mv
+
+    if HIGHEST_PROTOCOL >= 5:
+        assert len(f) == 2
+        assert isinstance(f[0], bytes)
+        assert isinstance(f[1], memoryview)
+        assert f[1] == mv
+    else:
+        assert len(f) == 1
+        assert isinstance(f[0], bytes)
 
 
 def test_pickle_numpy():
     np = pytest.importorskip("numpy")
     x = np.ones(5)
     assert (loads(dumps(x)) == x).all()
+    assert (deserialize(*serialize(x, serializers=("pickle",))) == x).all()
 
     x = np.ones(5000)
     assert (loads(dumps(x)) == x).all()
+    assert (deserialize(*serialize(x, serializers=("pickle",))) == x).all()
+
+    if HIGHEST_PROTOCOL >= 5:
+        x = np.ones(5000)
+
+        l = []
+        d = dumps(x, buffer_callback=l.append)
+        assert len(l) == 1
+        assert isinstance(l[0], PickleBuffer)
+        assert memoryview(l[0]) == memoryview(x)
+        assert (loads(d, buffers=l) == x).all()
+
+        h, f = serialize(x, serializers=("pickle",))
+        assert len(f) == 2
+        assert isinstance(f[0], bytes)
+        assert isinstance(f[1], memoryview)
+        assert (deserialize(h, f) == x).all()
 
 
 @pytest.mark.xfail(
@@ -45,10 +116,17 @@ def funcs():
 
     for func in funcs():
         wr = weakref.ref(func)
+
         func2 = loads(dumps(func))
         wr2 = weakref.ref(func2)
         assert func2(1) == func(1)
-        del func, func2
+
+        func3 = deserialize(*serialize(func, serializers=("pickle",)))
+        wr3 = weakref.ref(func3)
+        assert func3(1) == func(1)
+
+        del func, func2, func3
         gc.collect()
         assert wr() is None
         assert wr2() is None
+        assert wr3() is None
diff --git a/requirements.txt b/requirements.txt
index b0d20cdb1eb..95a681d66c2 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,5 @@
 click >= 6.6
-cloudpickle >= 0.2.2
+cloudpickle >= 1.3.0
 contextvars;python_version<'3.7'
 dask >= 2.9.0
 msgpack >= 0.6.0

From b377f6a944ac4cbcc06a3d6da17cd45b0c33f8d5 Mon Sep 17 00:00:00 2001
From: Abdulelah Bin Mahfoodh <abduh.bm@gmail.com>
Date: Thu, 21 May 2020 17:57:45 +0300
Subject: [PATCH 0839/1550] Enhance VersionMismatchWarning messages (#3786)

* Enhance package version mismatch warning messages

* Update test_versions

* Drop curly braces for single version value

* Update distributed/versions.py

Co-authored-by: Jim Crist-Harif <jcrist@users.noreply.github.com>

* Remove column matching fixture

* Refactor error_message

* Add python version mismatch test

Co-authored-by: Jim Crist-Harif <jcrist@users.noreply.github.com>
---
 distributed/tests/test_versions.py | 36 +++++++++++----
 distributed/versions.py            | 72 ++++++++++++++++++++----------
 2 files changed, 76 insertions(+), 32 deletions(-)

diff --git a/distributed/tests/test_versions.py b/distributed/tests/test_versions.py
index 64b94ba37b1..0a095debeec 100644
--- a/distributed/tests/test_versions.py
+++ b/distributed/tests/test_versions.py
@@ -67,9 +67,9 @@ def kwargs_not_matching(kwargs_matching, node, effect):
 
 @pytest.fixture
 def pattern(effect):
-    """Pattern to match in the right-hand column."""
+    """String to match in the right column."""
     return {
-        "MISMATCHED": r"0\.0\.0\.dev0",
+        "MISMATCHED": "0.0.0.dev0",
         "MISSING": "MISSING",
         "KEY_ERROR": "UNKNOWN",
         "NONE": "UNKNOWN",
@@ -77,11 +77,18 @@ def pattern(effect):
 
 
 def test_version_mismatch(node, effect, kwargs_not_matching, pattern):
+    column_matching = {"client": 1, "scheduler": 2, "workers": 3}
     msg = error_message(**kwargs_not_matching)
-
+    i = column_matching.get(node, 3)
     assert "Mismatched versions found" in msg
     assert "distributed" in msg
-    assert re.search(node + r"\s+\|\s+" + pattern, msg)
+    assert (
+        pattern
+        in re.search(r"distributed\s+(?:(?:\|[^|\r\n]*)+\|(?:\r?\n|\r)?)+", msg)
+        .group(0)
+        .split("|")[i]
+        .strip()
+    )
 
 
 def test_scheduler_mismatched_irrelevant_package(kwargs_matching):
@@ -99,6 +106,20 @@ def test_scheduler_additional_irrelevant_package(kwargs_matching):
     assert error_message(**kwargs_matching) == ""
 
 
+def test_python_mismatch(kwargs_matching):
+    kwargs_matching["client"]["packages"]["python"] = "0.0.0"
+    msg = error_message(**kwargs_matching)
+    assert "Mismatched versions found" in msg
+    assert "python" in msg
+    assert (
+        "0.0.0"
+        in re.search(r"python\s+(?:(?:\|[^|\r\n]*)+\|(?:\r?\n|\r)?)+", msg)
+        .group(0)
+        .split("|")[1]
+        .strip()
+    )
+
+
 @gen_cluster()
 async def test_version_warning_in_cluster(s, a, b):
     s.workers[a.address].versions["packages"]["dask"] = "0.0.0"
@@ -110,14 +131,11 @@ async def test_version_warning_in_cluster(s, a, b):
     assert record
     assert any("dask" in str(r.message) for r in record)
     assert any("0.0.0" in str(r.message) for r in record)
-    assert any(a.address in str(r.message) for r in record)
 
     async with Worker(s.address) as w:
-        assert any("This Worker" in line.message for line in w.logs)
+        assert any("workers" in line.message for line in w.logs)
         assert any("dask" in line.message for line in w.logs)
-        assert any(
-            "0.0.0" in line.message and a.address in line.message for line in w.logs
-        )
+        assert any("0.0.0" in line.message in line.message for line in w.logs)
 
 
 def test_python_version():
diff --git a/distributed/versions.py b/distributed/versions.py
index 2062f48568c..0e49f079d33 100644
--- a/distributed/versions.py
+++ b/distributed/versions.py
@@ -2,7 +2,6 @@
 
 from __future__ import print_function, division, absolute_import
 
-from collections import defaultdict
 import platform
 import struct
 import os
@@ -32,6 +31,14 @@
 )
 
 
+# notes to be displayed for mismatch packages
+notes_mismatch_package = {
+    "msgpack": "Variation is ok, as long as everything is above 0.6",
+    "lz4": "Variation is ok, but missing libraries are not",
+    "python": "Variation is sometimes ok, sometimes not. It depends on your workloads",
+}
+
+
 def get_versions(packages=None):
     """
     Return basic information on our software installation, and our installed versions of packages.
@@ -106,37 +113,56 @@ def get_package_info(pkgs):
 def error_message(scheduler, workers, client, client_name="client"):
     from .utils import asciitable
 
-    nodes = {**{client_name: client}, **{"scheduler": scheduler}, **workers}
-
-    # Hold all versions, e.g. versions["scheduler"]["distributed"] = 2.9.3
-    node_packages = defaultdict(dict)
+    client = client.get("packages") if client else "UNKNOWN"
+    scheduler = scheduler.get("packages") if scheduler else "UNKNOWN"
+    workers = {k: v.get("packages") if v else "UNKNOWN" for k, v in workers.items()}
 
-    # Collect all package versions
     packages = set()
-    for node, info in nodes.items():
-        if info is None or not (isinstance(info, dict)) or "packages" not in info:
-            node_packages[node] = defaultdict(lambda: "UNKNOWN")
-        else:
-            node_packages[node] = defaultdict(lambda: "MISSING")
-            for pkg, version in info["packages"].items():
-                node_packages[node][pkg] = version
-                packages.add(pkg)
+    packages.update(client)
+    packages.update(scheduler)
+    for worker in workers:
+        packages.update(workers.get(worker))
 
     errs = []
+    notes = []
     for pkg in sorted(packages):
-        versions = set(
-            node_packages[node][pkg]
-            for node in nodes
-            if node != "scheduler" or pkg in scheduler_relevant_packages
+        versions = set()
+        scheduler_version = (
+            scheduler.get(pkg, "MISSING") if isinstance(scheduler, dict) else scheduler
+        )
+        if pkg in scheduler_relevant_packages:
+            versions.add(scheduler_version)
+
+        client_version = (
+            client.get(pkg, "MISSING") if isinstance(client, dict) else client
         )
+        versions.add(client_version)
+
+        worker_versions = set(
+            workers[w].get(pkg, "MISSING")
+            if isinstance(workers[w], dict)
+            else workers[w]
+            for w in workers
+        )
+        versions |= worker_versions
+
         if len(versions) <= 1:
             continue
-        rows = [
-            (node_name, node_packages[node_name][pkg]) for node_name in nodes.keys()
-        ]
-        errs.append("%s\n%s" % (pkg, asciitable(["", "version"], rows)))
+        if len(worker_versions) == 1:
+            worker_versions = list(worker_versions)[0]
+        elif len(worker_versions) == 0:
+            worker_versions = None
+
+        errs.append((pkg, client_version, scheduler_version, worker_versions))
+        if pkg in notes_mismatch_package.keys():
+            notes.append(f"-  {pkg}: {notes_mismatch_package[pkg]}")
+
     if errs:
-        return "Mismatched versions found\n" "\n" "%s" % ("\n\n".join(errs))
+        err_table = asciitable(["Package", client_name, "scheduler", "workers"], errs)
+        err_msg = f"Mismatched versions found\n\n{err_table}"
+        if notes:
+            err_msg += "\nNotes: \n{}".format("\n".join(notes))
+        return err_msg
     else:
         return ""
 

From b9e4953583a9a0099cd18997828d17eafae15214 Mon Sep 17 00:00:00 2001
From: Nils Braun <nils-braun@users.noreply.github.com>
Date: Fri, 22 May 2020 20:06:35 +0200
Subject: [PATCH 0840/1550] Add distributed Dask Event that mimics
 threading.Event (#3821)

---
 distributed/__init__.py          |   1 +
 distributed/event.py             | 246 +++++++++++++++++++++++++++++++
 distributed/scheduler.py         |   2 +
 distributed/tests/test_events.py | 220 +++++++++++++++++++++++++++
 docs/source/api.rst              |   3 +
 5 files changed, 472 insertions(+)
 create mode 100644 distributed/event.py
 create mode 100644 distributed/tests/test_events.py

diff --git a/distributed/__init__.py b/distributed/__init__.py
index 2ad25d05093..e7e74c2b2e7 100644
--- a/distributed/__init__.py
+++ b/distributed/__init__.py
@@ -24,6 +24,7 @@
 from .queues import Queue
 from .security import Security
 from .semaphore import Semaphore
+from .event import Event
 from .scheduler import Scheduler
 from .threadpoolexecutor import rejoin
 from .utils import sync, TimeoutError, CancelledError
diff --git a/distributed/event.py b/distributed/event.py
new file mode 100644
index 00000000000..b4507c17600
--- /dev/null
+++ b/distributed/event.py
@@ -0,0 +1,246 @@
+import asyncio
+from collections import defaultdict
+from contextlib import suppress
+import logging
+import uuid
+
+from .client import Client
+from .utils import log_errors, TimeoutError
+from .worker import get_worker
+
+logger = logging.getLogger(__name__)
+
+
+class EventExtension:
+    """ An extension for the scheduler to manage Events
+
+    This adds the following routes to the scheduler
+
+    *  event_wait
+    *  event_set
+    *  event_clear
+    *  event_is_set
+
+    In principle, the implementation logic is quite simple
+    as we can reuse the asyncio.Event as much as possible:
+    we keep a mapping from name to an asyncio.Event and call
+    every function (wait, set, clear, is_set) directly on these
+    events.
+
+    However, this would cause a memory leak: created events in the
+    dictionary are never removed.
+    For this, we also keep a counter for the number of waiters on
+    a specific event.
+    If an event is set, we need to keep track of this state so
+    we can not remove it (the default flag is false).
+    If it is unset but there are waiters, we can also not remove
+    it, as those waiters would then have dangling futures.
+    Therefore the only time we can remove the event from our dict
+    is when the number of waiters is 0 and the event flag is cleared.
+    """
+
+    def __init__(self, scheduler):
+        self.scheduler = scheduler
+        # Keep track of all current events, identified by their name
+        self._events = defaultdict(asyncio.Event)
+        # Keep track on how many waiters are present, so we know when
+        # we can remove the event
+        self._waiter_count = defaultdict(int)
+
+        self.scheduler.handlers.update(
+            {
+                "event_wait": self.event_wait,
+                "event_set": self.event_set,
+                "event_clear": self.event_clear,
+                "event_is_set": self.event_is_set,
+            }
+        )
+
+        self.scheduler.extensions["events"] = self
+
+    async def event_wait(self, comm=None, name=None, timeout=None):
+        """ Wait until the event is set to true.
+        Returns false, when this did not happen in the given time
+        and true otherwise.
+        """
+        with log_errors():
+            name = self._normalize_name(name)
+
+            event = self._events[name]
+            future = event.wait()
+            if timeout is not None:
+                future = asyncio.wait_for(future, timeout)
+
+            self._waiter_count[name] += 1
+            try:
+                await future
+            except TimeoutError:
+                return False
+            finally:
+                self._waiter_count[name] -= 1
+
+                if not self._waiter_count[name] and not event.is_set():
+                    # No one is waiting for this
+                    # and as the default flag for an event is false
+                    # we can safely remove it
+                    self._delete_event(name)
+
+            return True
+
+    def event_set(self, comm=None, name=None):
+        """ Set the event with the given name to true.
+
+        All waiters on this event will be notified.
+        """
+        with log_errors():
+            name = self._normalize_name(name)
+            # No matter if someone is listening or not,
+            # we set the event to true
+            self._events[name].set()
+
+    def event_clear(self, comm=None, name=None):
+        """Set the event with the given name to false."""
+        with log_errors():
+            name = self._normalize_name(name)
+            if not self._waiter_count[name]:
+                # No one is waiting for this
+                # and as the default flag for an event is false
+                # we can safely remove it
+                self._delete_event(name)
+
+            else:
+                # There are waiters
+                # This can happen if an event is "double-cleared"
+                # In principle, the event should be unset at this point
+                # (because if it is set, all waiters should have been
+                # notified). But to prevent race conditions
+                # due to unlucky timing, we clear anyways
+                assert name in self._events
+                event = self._events[name]
+                event.clear()
+
+    def event_is_set(self, comm=None, name=None):
+        with log_errors():
+            name = self._normalize_name(name)
+            # the default flag value is false
+            # we could also let the defaultdict
+            # create a new event for us, but that
+            # could produce many unused events
+            if name not in self._events:
+                return False
+
+            return self._events[name].is_set()
+
+    def _normalize_name(self, name):
+        """ Helper function to normalize an event name """
+        if isinstance(name, list):
+            name = tuple(name)
+
+        return name
+
+    def _delete_event(self, name):
+        """ Helper function to delete an event """
+        # suppress key errors to make calling this method
+        # also possible if we do not even have such an event
+        with suppress(KeyError):
+            del self._waiter_count[name]
+        with suppress(KeyError):
+            del self._events[name]
+
+
+class Event:
+    """ Distributed Centralized Event equivalent to asyncio.Event
+
+    An event stores a single flag, which is set to false on start.
+    The flag can be set to true (using the set() call) or back to false
+    (with the clear() call).
+    Every call to wait() blocks until the event flag is set to true.
+
+    Parameters
+    ----------
+    name: string (optional)
+        Name of the event.  Choosing the same name allows two
+        disconnected processes to coordinate an event.
+        If not given, a random name will be generated.
+    client: Client (optional)
+        Client to use for communication with the scheduler.
+        If not given, the default global client will be used.
+
+    Examples
+    --------
+    >>> event_1 = Event('a')  # doctest: +SKIP
+    >>> event_1.wait(timeout=1)  # doctest: +SKIP
+    >>> # in another process
+    >>> event_2 = Event('a')  # doctest: +SKIP
+    >>> event_2.set() # doctest: +SKIP
+    >>> # now event_1 will stop waiting
+    """
+
+    def __init__(self, name=None, client=None):
+        try:
+            self.client = client or Client.current()
+        except ValueError:
+            # Initialise new client
+            self.client = get_worker().client
+        self.name = name or "event-" + uuid.uuid4().hex
+
+    def __await__(self):
+        """ async constructor
+
+        Make it possible to write
+
+        >>> event = await Event("x") # doctest: +SKIP
+
+        even though no waiting is implied
+        """
+
+        async def _():
+            return self
+
+        return _().__await__()
+
+    def wait(self, timeout=None):
+        """ Wait until the event is set.
+
+        Parameters
+        ----------
+        timeout : number, optional
+            Seconds to wait on the event in the scheduler.  This does not
+            include local coroutine time, network transfer time, etc..
+
+        Examples
+        --------
+        >>> event = Event('a')  # doctest: +SKIP
+        >>> event.wait(timeout=1)  # doctest: +SKIP
+
+        Returns
+        -------
+        True if the event was set of false, if a timeout happend
+        """
+        result = self.client.sync(
+            self.client.scheduler.event_wait, name=self.name, timeout=timeout,
+        )
+        return result
+
+    def clear(self):
+        """ Clear the event (set its flag to false).
+
+        All waiters will now block.
+        """
+        return self.client.sync(self.client.scheduler.event_clear, name=self.name)
+
+    def set(self):
+        """ Set the event (set its flag to false).
+
+        All waiters will now be released.
+        """
+        result = self.client.sync(self.client.scheduler.event_set, name=self.name,)
+        return result
+
+    def is_set(self):
+        """ Check if the event is set """
+        result = self.client.sync(self.client.scheduler.event_is_set, name=self.name,)
+        return result
+
+    def __reduce__(self):
+        return (Event, (self.name,))
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index cc3e64fcc8c..d032086ec79 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -81,6 +81,7 @@
 from .semaphore import SemaphoreExtension
 from .recreate_exceptions import ReplayExceptionScheduler
 from .lock import LockExtension
+from .event import EventExtension
 from .pubsub import PubSubSchedulerExtension
 from .stealing import WorkStealing
 from .variable import VariableExtension
@@ -102,6 +103,7 @@
     VariableExtension,
     PubSubSchedulerExtension,
     SemaphoreExtension,
+    EventExtension,
 ]
 
 ALL_TASK_STATES = {"released", "waiting", "no-worker", "processing", "erred", "memory"}
diff --git a/distributed/tests/test_events.py b/distributed/tests/test_events.py
new file mode 100644
index 00000000000..629d3ddc525
--- /dev/null
+++ b/distributed/tests/test_events.py
@@ -0,0 +1,220 @@
+import pickle
+
+from distributed import Event
+from distributed.utils_test import gen_cluster
+from distributed.utils_test import client, cluster_fixture, loop  # noqa F401
+
+
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 8)] * 2)
+async def test_event_on_workers(c, s, a, b):
+    # Test the "typical" use case of events:
+    # workers set, clear and wait for it
+    def wait_for_it_failing(x):
+        event = Event("x")
+
+        # Event is not set in another task so far
+        assert not event.wait(timeout=0.05)
+        assert not event.is_set()
+
+    def wait_for_it_ok(x):
+        event = Event("x")
+
+        # Event is set in another task
+        assert event.wait(timeout=0.5)
+        assert event.is_set()
+
+    def set_it():
+        event = Event("x")
+        event.set()
+
+    def clear_it():
+        event = Event("x")
+        event.clear()
+
+    wait_futures = c.map(wait_for_it_failing, range(10))
+    await c.gather(wait_futures)
+
+    set_future = c.submit(set_it)
+    await c.gather(set_future)
+
+    wait_futures = c.map(wait_for_it_ok, range(10))
+    await c.gather(wait_futures)
+
+    clear_future = c.submit(clear_it)
+    await c.gather(clear_future)
+
+    wait_futures = c.map(wait_for_it_ok, range(10))
+    await c.gather(wait_futures)
+
+    assert not s.extensions["events"]._events
+    assert not s.extensions["events"]._waiter_count
+
+
+@gen_cluster(client=True)
+async def test_default_event(c, s, a, b):
+    # The default flag for events should be false
+    event = Event("x")
+    assert not await event.is_set()
+
+    await event.clear()
+
+    # Cleanup should have happened
+    assert not s.extensions["events"]._events
+    assert not s.extensions["events"]._waiter_count
+
+
+@gen_cluster(client=True)
+async def test_set_not_set(c, s, a, b):
+    # Set and unset the event and check if the flag is
+    # propagated correctly
+    event = Event("x")
+
+    await event.clear()
+    assert not await event.is_set()
+
+    await event.set()
+    assert await event.is_set()
+
+    await event.set()
+    assert await event.is_set()
+
+    await event.clear()
+    assert not await event.is_set()
+
+    # Cleanup should have happened
+    assert not s.extensions["events"]._events
+    assert not s.extensions["events"]._waiter_count
+
+
+@gen_cluster(client=True)
+async def test_set_not_set_many_events(c, s, a, b):
+    # Set and unset the event and check if the flag is
+    # propagated correctly with many events
+    events = [Event(name) for name in range(100)]
+
+    for event in events:
+        await event.clear()
+        assert not await event.is_set()
+
+    for i, event in enumerate(events):
+        if i % 2 == 0:
+            await event.set()
+            assert await event.is_set()
+        else:
+            assert not await event.is_set()
+
+    for event in events:
+        await event.clear()
+        assert not await event.is_set()
+
+    # Cleanup should have happened
+    assert not s.extensions["events"]._events
+    assert not s.extensions["events"]._waiter_count
+
+
+@gen_cluster(client=True)
+async def test_timeout(c, s, a, b):
+    # The event should not be set and the timeout should happen
+    event = Event("x")
+    assert not await Event("x").wait(timeout=0.1)
+
+    await event.set()
+    assert await Event("x").wait(timeout=0.1)
+
+    await event.clear()
+    assert not await Event("x").wait(timeout=0.1)
+
+
+def test_event_sync(client):
+    # Assert that we call the client.sync correctly
+    def wait_for_it_failing(x):
+        event = Event("x")
+
+        # Event is not set in another task so far
+        assert not event.wait(timeout=0.05)
+        assert not event.is_set()
+
+    def wait_for_it_ok(x):
+        event = Event("x")
+
+        # Event is set in another task
+        assert event.wait(timeout=0.5)
+        assert event.is_set()
+
+    def set_it():
+        event = Event("x")
+        event.set()
+
+    wait_futures = client.map(wait_for_it_failing, range(10))
+    client.gather(wait_futures)
+
+    set_future = client.submit(set_it)
+    client.gather(set_future)
+
+    wait_futures = client.map(wait_for_it_ok, range(10))
+    client.gather(wait_futures)
+
+
+@gen_cluster(client=True)
+async def test_event_types(c, s, a, b):
+    # Event names could be strings, numbers or tuples
+    for name in [1, ("a", 1), ["a", 1], b"123", "123"]:
+        event = Event(name)
+        assert event.name == name
+
+        await event.set()
+        await event.clear()
+        result = await event.is_set()
+        assert not result
+
+    assert not s.extensions["events"]._events
+    assert not s.extensions["events"]._waiter_count
+
+
+@gen_cluster(client=True)
+async def test_serializable(c, s, a, b):
+    # Pickling an event should work properly
+    def f(x, event=None):
+        assert event.name == "x"
+        return x + 1
+
+    event = Event("x")
+    futures = c.map(f, range(10), event=event)
+    await c.gather(futures)
+
+    event2 = pickle.loads(pickle.dumps(event))
+    assert event2.name == event.name
+    assert event2.client is event.client
+
+
+@gen_cluster(client=True)
+async def test_two_events_on_workers(c, s, a, b):
+    # Longer test with multiple events and two workers
+    def event_not_set(event_name):
+        assert not Event(event_name).wait(timeout=0.05)
+
+    def event_is_set(event_name):
+        assert Event(event_name).wait(timeout=0.5)
+
+    await c.gather(c.submit(event_not_set, "first_event"))
+    await c.gather(c.submit(event_not_set, "second_event"))
+
+    await Event("first_event").set()
+
+    await c.gather(c.submit(event_is_set, "first_event"))
+    await c.gather(c.submit(event_not_set, "second_event"))
+
+    await Event("first_event").clear()
+    await Event("second_event").set()
+
+    await c.gather(c.submit(event_not_set, "first_event"))
+    await c.gather(c.submit(event_is_set, "second_event"))
+
+    await Event("first_event").clear()
+    await Event("second_event").clear()
+
+    await c.gather(c.submit(event_not_set, "first_event"))
+    await c.gather(c.submit(event_not_set, "second_event"))
+
+    assert not s.extensions["events"]._events
+    assert not s.extensions["events"]._waiter_count
diff --git a/docs/source/api.rst b/docs/source/api.rst
index c036fdf96db..5a283ed2f5e 100644
--- a/docs/source/api.rst
+++ b/docs/source/api.rst
@@ -83,6 +83,7 @@ API
 .. currentmodule:: distributed
 
 .. autosummary::
+   Event
    Lock
    Queue
    Variable
@@ -193,6 +194,8 @@ Other
 .. autoclass:: distributed.Reschedule
 .. autoclass:: get_task_stream
 
+.. autoclass:: Event
+   :members:
 .. autoclass:: Lock
    :members:
 .. autoclass:: Semaphore

From 3c7b5b1f26371a829f03ee2280790fad5eb9d1e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jakub=20Ber=C3=A1nek?= <berykubik@gmail.com>
Date: Sat, 23 May 2020 17:47:37 +0200
Subject: [PATCH 0841/1550] Don't send empty dependencies (#3423)

---
 distributed/client.py                   |  1 +
 distributed/diagnostics/graph_layout.py | 13 +++++++++----
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 8773311194d..36234ea2e9b 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -2574,6 +2574,7 @@ def _graph_to_futures(
             dependencies = {
                 tokey(k): [tokey(dep) for dep in deps]
                 for k, deps in dependencies.items()
+                if deps
             }
             for k, deps in future_dependencies.items():
                 if deps:
diff --git a/distributed/diagnostics/graph_layout.py b/distributed/diagnostics/graph_layout.py
index a348d2e04ee..ab36bd6978f 100644
--- a/distributed/diagnostics/graph_layout.py
+++ b/distributed/diagnostics/graph_layout.py
@@ -36,16 +36,21 @@ def __init__(self, scheduler):
             }
             priority = {k: ts.priority for k, ts in scheduler.tasks.items()}
             self.update_graph(
-                self.scheduler, dependencies=dependencies, priority=priority
+                self.scheduler,
+                tasks=self.scheduler.tasks,
+                dependencies=dependencies,
+                priority=priority,
             )
 
-    def update_graph(self, scheduler, dependencies=None, priority=None, **kwargs):
-        stack = sorted(dependencies, key=lambda k: priority.get(k, 0), reverse=True)
+    def update_graph(
+        self, scheduler, dependencies=None, priority=None, tasks=None, **kwargs
+    ):
+        stack = sorted(tasks, key=lambda k: priority.get(k, 0), reverse=True)
         while stack:
             key = stack.pop()
             if key in self.x or key not in scheduler.tasks:
                 continue
-            deps = dependencies[key]
+            deps = dependencies.get(key, ())
             if deps:
                 if not all(dep in self.y for dep in deps):
                     stack.append(key)

From 5f5ebaf96c68620e090970fa77867be4bf04eaaa Mon Sep 17 00:00:00 2001
From: Nils Braun <nils-braun@users.noreply.github.com>
Date: Sat, 23 May 2020 17:58:46 +0200
Subject: [PATCH 0842/1550] Parse timeout parameters in Variable/Event/Lock to
 support text timeouts (#3825)

All public-facing API parameters "timeout" are wrapped
with utils.parse_timedelta before passing tem on, to allow
things like timeout="42ms".

Fixes #3823

Co-authored-by: Nils Braun <nilslennartbraun@gmail.com>
---
 distributed/event.py                |  9 +++++++--
 distributed/lock.py                 |  9 +++++++--
 distributed/pubsub.py               | 13 +++++++++++--
 distributed/queues.py               | 19 ++++++++++++++++---
 distributed/semaphore.py            |  9 +++++++++
 distributed/tests/test_events.py    |  5 +++--
 distributed/tests/test_locks.py     |  5 +++--
 distributed/tests/test_pubsub.py    |  5 +++--
 distributed/tests/test_queues.py    |  5 +++--
 distributed/tests/test_semaphore.py |  5 +++--
 distributed/tests/test_variable.py  |  5 +++--
 distributed/variable.py             | 13 +++++++++++--
 12 files changed, 79 insertions(+), 23 deletions(-)

diff --git a/distributed/event.py b/distributed/event.py
index b4507c17600..75661654b86 100644
--- a/distributed/event.py
+++ b/distributed/event.py
@@ -7,6 +7,7 @@
 from .client import Client
 from .utils import log_errors, TimeoutError
 from .worker import get_worker
+from .utils import parse_timedelta
 
 logger = logging.getLogger(__name__)
 
@@ -204,19 +205,23 @@ def wait(self, timeout=None):
 
         Parameters
         ----------
-        timeout : number, optional
+        timeout : number or string or timedelta, optional
             Seconds to wait on the event in the scheduler.  This does not
             include local coroutine time, network transfer time, etc..
+            Instead of number of seconds, it is also possible to specify
+            a timedelta in string format, e.g. "200ms".
 
         Examples
         --------
         >>> event = Event('a')  # doctest: +SKIP
-        >>> event.wait(timeout=1)  # doctest: +SKIP
+        >>> event.wait(timeout="1s")  # doctest: +SKIP
 
         Returns
         -------
         True if the event was set of false, if a timeout happend
         """
+        timeout = parse_timedelta(timeout)
+
         result = self.client.sync(
             self.client.scheduler.event_wait, name=self.name, timeout=timeout,
         )
diff --git a/distributed/lock.py b/distributed/lock.py
index 7a55ccb4413..a9bda6617aa 100644
--- a/distributed/lock.py
+++ b/distributed/lock.py
@@ -6,6 +6,7 @@
 from .client import Client
 from .utils import log_errors, TimeoutError
 from .worker import get_worker
+from .utils import parse_timedelta
 
 logger = logging.getLogger(__name__)
 
@@ -109,20 +110,24 @@ def acquire(self, blocking=True, timeout=None):
         ----------
         blocking : bool, optional
             If false, don't wait on the lock in the scheduler at all.
-        timeout : number, optional
+        timeout : string or number or timedelta, optional
             Seconds to wait on the lock in the scheduler.  This does not
             include local coroutine time, network transfer time, etc..
             It is forbidden to specify a timeout when blocking is false.
+            Instead of number of seconds, it is also possible to specify
+            a timedelta in string format, e.g. "200ms".
 
         Examples
         --------
         >>> lock = Lock('x')  # doctest: +SKIP
-        >>> lock.acquire(timeout=1)  # doctest: +SKIP
+        >>> lock.acquire(timeout="1s")  # doctest: +SKIP
 
         Returns
         -------
         True or False whether or not it sucessfully acquired the lock
         """
+        timeout = parse_timedelta(timeout)
+
         if not blocking:
             if timeout is not None:
                 raise ValueError("can't specify a timeout for a non-blocking call")
diff --git a/distributed/pubsub.py b/distributed/pubsub.py
index 6496a60243d..8234d1a8e19 100644
--- a/distributed/pubsub.py
+++ b/distributed/pubsub.py
@@ -7,7 +7,7 @@
 
 from .core import CommClosedError
 from .metrics import time
-from .utils import sync, TimeoutError
+from .utils import sync, TimeoutError, parse_timedelta
 from .protocol.serialize import to_serialize
 
 logger = logging.getLogger(__name__)
@@ -429,7 +429,16 @@ async def _():
     __anext__ = _get
 
     def get(self, timeout=None):
-        """ Get a single message """
+        """ Get a single message
+
+        Parameters
+        ----------
+        timeout: number or string or timedelta, optional
+            Time in seconds to wait before timing out.
+            Instead of number of seconds, it is also possible to specify
+            a timedelta in string format, e.g. "200ms".
+        """
+        timeout = parse_timedelta(timeout)
         if self.client:
             return self.client.sync(self._get, timeout=timeout)
         elif self.worker.thread_id == threading.get_ident():
diff --git a/distributed/queues.py b/distributed/queues.py
index 324fb46c40b..e7c634c1180 100644
--- a/distributed/queues.py
+++ b/distributed/queues.py
@@ -6,6 +6,7 @@
 from .client import Future, Client
 from .utils import tokey, sync, thread_state
 from .worker import get_client
+from .utils import parse_timedelta
 
 logger = logging.getLogger(__name__)
 
@@ -208,7 +209,16 @@ async def _put(self, value, timeout=None):
             )
 
     def put(self, value, timeout=None, **kwargs):
-        """ Put data into the queue """
+        """ Put data into the queue
+
+        Parameters
+        ----------
+        timeout: number or string or timedelta, optional
+            Time in seconds to wait before timing out.
+            Instead of number of seconds, it is also possible to specify
+            a timedelta in string format, e.g. "200ms".
+        """
+        timeout = parse_timedelta(timeout)
         return self.client.sync(self._put, value, timeout=timeout, **kwargs)
 
     def get(self, timeout=None, batch=False, **kwargs):
@@ -216,13 +226,16 @@ def get(self, timeout=None, batch=False, **kwargs):
 
         Parameters
         ----------
-        timeout: Number (optional)
-            Time in seconds to wait before timing out
+        timeout: number or string or timedelta, optional
+            Time in seconds to wait before timing out.
+            Instead of number of seconds, it is also possible to specify
+            a timedelta in string format, e.g. "200ms".
         batch: boolean, int (optional)
             If True then return all elements currently waiting in the queue.
             If an integer than return that many elements from the queue
             If False (default) then return one item at a time
          """
+        timeout = parse_timedelta(timeout)
         return self.client.sync(self._get, timeout=timeout, batch=batch, **kwargs)
 
     def qsize(self, **kwargs):
diff --git a/distributed/semaphore.py b/distributed/semaphore.py
index bd7e52f2e40..ece69fe5bc2 100644
--- a/distributed/semaphore.py
+++ b/distributed/semaphore.py
@@ -445,7 +445,16 @@ def acquire(self, timeout=None):
 
         If the internal counter is greater than zero, decrement it by one and return True immediately.
         If it is zero, wait until a release() is called and return True.
+
+        Parameters
+        ----------
+        timeout : number or string or timedelta, optional
+            Seconds to wait on acquiring the semaphore.  This does not
+            include local coroutine time, network transfer time, etc..
+            Instead of number of seconds, it is also possible to specify
+            a timedelta in string format, e.g. "200ms".
         """
+        timeout = parse_timedelta(timeout)
         return self.client.sync(self._acquire, timeout=timeout)
 
     def release(self):
diff --git a/distributed/tests/test_events.py b/distributed/tests/test_events.py
index 629d3ddc525..484a01c4e3a 100644
--- a/distributed/tests/test_events.py
+++ b/distributed/tests/test_events.py
@@ -1,4 +1,5 @@
 import pickle
+from datetime import timedelta
 
 from distributed import Event
 from distributed.utils_test import gen_cluster
@@ -119,10 +120,10 @@ async def test_timeout(c, s, a, b):
     assert not await Event("x").wait(timeout=0.1)
 
     await event.set()
-    assert await Event("x").wait(timeout=0.1)
+    assert await Event("x").wait(timeout="100ms")
 
     await event.clear()
-    assert not await Event("x").wait(timeout=0.1)
+    assert not await Event("x").wait(timeout=timedelta(seconds=0.1))
 
 
 def test_event_sync(client):
diff --git a/distributed/tests/test_locks.py b/distributed/tests/test_locks.py
index 0d22fc6cee9..80eaa06a94f 100644
--- a/distributed/tests/test_locks.py
+++ b/distributed/tests/test_locks.py
@@ -1,5 +1,6 @@
 import pickle
 from time import sleep
+from datetime import timedelta
 
 import pytest
 
@@ -57,9 +58,9 @@ async def test_acquires_with_zero_timeout(c, s, a, b):
     assert lock.locked()
     await lock.release()
 
-    await lock.acquire(timeout=1)
+    await lock.acquire(timeout="1s")
     await lock.release()
-    await lock.acquire(timeout=1)
+    await lock.acquire(timeout=timedelta(seconds=1))
     await lock.release()
 
 
diff --git a/distributed/tests/test_pubsub.py b/distributed/tests/test_pubsub.py
index 212d29d4802..6d005a3ae3e 100644
--- a/distributed/tests/test_pubsub.py
+++ b/distributed/tests/test_pubsub.py
@@ -1,4 +1,5 @@
 import asyncio
+from datetime import timedelta
 from time import sleep
 
 import pytest
@@ -120,11 +121,11 @@ async def test_timeouts(c, s, a, b):
     sub = Sub("a", client=c, worker=None)
     start = time()
     with pytest.raises(TimeoutError):
-        await sub.get(timeout=0.1)
+        await sub.get(timeout="100ms")
     stop = time()
     assert stop - start < 1
     with pytest.raises(TimeoutError):
-        await sub.get(timeout=0.01)
+        await sub.get(timeout=timedelta(milliseconds=10))
 
 
 @gen_cluster(client=True)
diff --git a/distributed/tests/test_queues.py b/distributed/tests/test_queues.py
index 34009602a15..94d80c9dbcf 100644
--- a/distributed/tests/test_queues.py
+++ b/distributed/tests/test_queues.py
@@ -1,4 +1,5 @@
 import asyncio
+from datetime import timedelta
 from time import sleep
 
 import pytest
@@ -248,7 +249,7 @@ async def test_timeout(c, s, a, b):
 
     start = time()
     with pytest.raises(TimeoutError):
-        await q.get(timeout=0.3)
+        await q.get(timeout="300ms")
     stop = time()
     assert 0.2 < stop - start < 2.0
 
@@ -256,7 +257,7 @@ async def test_timeout(c, s, a, b):
 
     start = time()
     with pytest.raises(TimeoutError):
-        await q.put(2, timeout=0.3)
+        await q.put(2, timeout=timedelta(seconds=0.3))
     stop = time()
     assert 0.1 < stop - start < 2.0
 
diff --git a/distributed/tests/test_semaphore.py b/distributed/tests/test_semaphore.py
index 4839f5054f8..27af5c4fd96 100644
--- a/distributed/tests/test_semaphore.py
+++ b/distributed/tests/test_semaphore.py
@@ -1,4 +1,5 @@
 import asyncio
+from datetime import timedelta
 import pickle
 import dask
 import pytest
@@ -77,10 +78,10 @@ def f(x, semaphore):
 @gen_cluster(client=True)
 async def test_acquires_with_timeout(c, s, a, b):
     sem = await Semaphore(1, "x")
-    assert await sem.acquire(timeout=0.025)
+    assert await sem.acquire(timeout="25ms")
     assert not await sem.acquire(timeout=0.025)
     await sem.release()
-    assert await sem.acquire(timeout=0.025)
+    assert await sem.acquire(timeout=timedelta(seconds=0.025))
     await sem.release()
 
 
diff --git a/distributed/tests/test_variable.py b/distributed/tests/test_variable.py
index e2a4617ea0b..1e707626235 100644
--- a/distributed/tests/test_variable.py
+++ b/distributed/tests/test_variable.py
@@ -1,5 +1,6 @@
 import asyncio
 import random
+from datetime import timedelta
 from time import sleep, monotonic
 import logging
 
@@ -98,7 +99,7 @@ async def test_timeout(c, s, a, b):
 
     start = monotonic()
     with pytest.raises(TimeoutError):
-        await v.get(timeout=0.2)
+        await v.get(timeout="200ms")
     stop = monotonic()
 
     if WINDOWS:  # timing is weird with asyncio and Windows
@@ -107,7 +108,7 @@ async def test_timeout(c, s, a, b):
         assert 0.2 < stop - start < 2.0
 
     with pytest.raises(TimeoutError):
-        await v.get(timeout=0.01)
+        await v.get(timeout=timedelta(milliseconds=10))
 
 
 def test_timeout_sync(client):
diff --git a/distributed/variable.py b/distributed/variable.py
index 6729bc2b69c..3bf6347fa61 100644
--- a/distributed/variable.py
+++ b/distributed/variable.py
@@ -7,7 +7,7 @@
 from tlz import merge
 
 from .client import Future, Client
-from .utils import tokey, log_errors, TimeoutError
+from .utils import tokey, log_errors, TimeoutError, parse_timedelta
 from .worker import get_client
 
 logger = logging.getLogger(__name__)
@@ -204,7 +204,16 @@ async def _get(self, timeout=None):
         return value
 
     def get(self, timeout=None, **kwargs):
-        """ Get the value of this variable """
+        """ Get the value of this variable
+
+        Parameters
+        ----------
+        timeout: number or string or timedelta, optional
+            Time in seconds to wait before timing out.
+            Instead of number of seconds, it is also possible to specify
+            a timedelta in string format, e.g. "200ms".
+        """
+        timeout = parse_timedelta(timeout)
         return self.client.sync(self._get, timeout=timeout, **kwargs)
 
     def delete(self):

From 0ec78f85b04857a99858ef149f711c9388e680be Mon Sep 17 00:00:00 2001
From: Martin Durant <martindurant@users.noreply.github.com>
Date: Mon, 25 May 2020 13:32:38 -0400
Subject: [PATCH 0843/1550] control de/ser offload (#3793)

* start

* Apply to to_frames too

* Apply to listeners (TCP, UCX)

* listen arg for scheduler only

* black
---
 distributed/comm/core.py  |  1 +
 distributed/comm/tcp.py   | 16 ++++++++++++++--
 distributed/comm/ucx.py   | 19 ++++++++++++++++---
 distributed/comm/utils.py | 30 +++++++++++++++++++++---------
 distributed/core.py       | 11 +++++++++--
 distributed/scheduler.py  |  5 ++++-
 6 files changed, 65 insertions(+), 17 deletions(-)

diff --git a/distributed/comm/core.py b/distributed/comm/core.py
index 00a6d87a52a..852cb991c4b 100644
--- a/distributed/comm/core.py
+++ b/distributed/comm/core.py
@@ -40,6 +40,7 @@ class Comm(ABC):
 
     def __init__(self):
         self._instances.add(self)
+        self.allow_offload = True  # for deserialization in utils.from_frames
         self.name = None
 
     # XXX add set_close_callback()?
diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py
index 769e9132abe..c610bde31ac 100644
--- a/distributed/comm/tcp.py
+++ b/distributed/comm/tcp.py
@@ -200,7 +200,10 @@ async def read(self, deserializers=None):
         else:
             try:
                 msg = await from_frames(
-                    frames, deserialize=self.deserialize, deserializers=deserializers
+                    frames,
+                    deserialize=self.deserialize,
+                    deserializers=deserializers,
+                    allow_offload=self.allow_offload,
                 )
             except EOFError:
                 # Frames possibly garbled or truncated by communication error
@@ -216,6 +219,7 @@ async def write(self, msg, serializers=None, on_error="message"):
 
         frames = await to_frames(
             msg,
+            allow_offload=self.allow_offload,
             serializers=serializers,
             on_error=on_error,
             context={"sender": self._local_addr, "recipient": self._peer_addr},
@@ -378,12 +382,19 @@ def _get_connect_args(self, **connection_args):
 
 class BaseTCPListener(Listener, RequireEncryptionMixin):
     def __init__(
-        self, address, comm_handler, deserialize=True, default_port=0, **connection_args
+        self,
+        address,
+        comm_handler,
+        deserialize=True,
+        allow_offload=True,
+        default_port=0,
+        **connection_args
     ):
         self._check_encryption(address, connection_args)
         self.ip, self.port = parse_host_port(address, default_port)
         self.comm_handler = comm_handler
         self.deserialize = deserialize
+        self.allow_offload = allow_offload
         self.server_args = self._get_server_args(**connection_args)
         self.tcp_server = None
         self.bound_address = None
@@ -432,6 +443,7 @@ async def _handle_stream(self, stream, address):
         logger.debug("Incoming connection from %r to %r", address, self.contact_address)
         local_address = self.prefix + get_stream_address(stream)
         comm = self.comm_class(stream, local_address, address, self.deserialize)
+        comm.allow_offload = self.allow_offload
         await self.comm_handler(comm)
 
     def get_host_port(self):
diff --git a/distributed/comm/ucx.py b/distributed/comm/ucx.py
index 22bf4ad52f3..54a85279ffd 100644
--- a/distributed/comm/ucx.py
+++ b/distributed/comm/ucx.py
@@ -176,7 +176,10 @@ async def write(
                     serializers = ("cuda", "dask", "pickle", "error")
                 # msg can also be a list of dicts when sending batched messages
                 frames = await to_frames(
-                    msg, serializers=serializers, on_error=on_error
+                    msg,
+                    serializers=serializers,
+                    on_error=on_error,
+                    allow_offload=self.allow_offload,
                 )
                 nframes = len(frames)
                 cuda_frames = tuple(
@@ -261,7 +264,10 @@ async def read(self, deserializers=("cuda", "dask", "pickle", "error")):
                 for each_frame in recv_frames:
                     await self.ep.recv(each_frame)
                 msg = await from_frames(
-                    frames, deserialize=self.deserialize, deserializers=deserializers
+                    frames,
+                    deserialize=self.deserialize,
+                    deserializers=deserializers,
+                    allow_offload=self.allow_offload,
                 )
                 return msg
 
@@ -310,13 +316,19 @@ class UCXListener(Listener):
     encrypted = UCXConnector.encrypted
 
     def __init__(
-        self, address: str, comm_handler: None, deserialize=False, **connection_args
+        self,
+        address: str,
+        comm_handler: None,
+        deserialize=False,
+        allow_offload=True,
+        **connection_args
     ):
         if not address.startswith("ucx"):
             address = "ucx://" + address
         self.ip, self._input_port = parse_host_port(address, default_port=0)
         self.comm_handler = comm_handler
         self.deserialize = deserialize
+        self.allow_offload = allow_offload
         self._ep = None  # type: ucp.Endpoint
         self.ucp_server = None
         self.connection_args = connection_args
@@ -337,6 +349,7 @@ async def serve_forever(client_ep):
                 peer_addr=self.address,
                 deserialize=self.deserialize,
             )
+            ucx.allow_offload = self.allow_offload
             if self.comm_handler:
                 await self.comm_handler(ucx)
 
diff --git a/distributed/comm/utils.py b/distributed/comm/utils.py
index b75663a14f2..d1a1a97e63c 100644
--- a/distributed/comm/utils.py
+++ b/distributed/comm/utils.py
@@ -21,7 +21,9 @@
     FRAME_OFFLOAD_THRESHOLD = parse_bytes(FRAME_OFFLOAD_THRESHOLD)
 
 
-async def to_frames(msg, serializers=None, on_error="message", context=None):
+async def to_frames(
+    msg, serializers=None, on_error="message", context=None, allow_offload=True
+):
     """
     Serialize a message into a list of Distributed protocol frames.
     """
@@ -38,22 +40,25 @@ def _to_frames():
             logger.exception(e)
             raise
 
-    try:
-        msg_size = sizeof(msg)
-    except RecursionError:
-        msg_size = math.inf
+    if FRAME_OFFLOAD_THRESHOLD and allow_offload:
+        try:
+            msg_size = sizeof(msg)
+        except RecursionError:
+            msg_size = math.inf
+    else:
+        msg_size = 0
 
-    if FRAME_OFFLOAD_THRESHOLD and msg_size > FRAME_OFFLOAD_THRESHOLD:
+    if allow_offload and FRAME_OFFLOAD_THRESHOLD and msg_size > FRAME_OFFLOAD_THRESHOLD:
         return await offload(_to_frames)
     else:
         return _to_frames()
 
 
-async def from_frames(frames, deserialize=True, deserializers=None):
+async def from_frames(frames, deserialize=True, deserializers=None, allow_offload=True):
     """
     Unserialize a list of Distributed protocol frames.
     """
-    size = sum(map(nbytes, frames))
+    size = False
 
     def _from_frames():
         try:
@@ -69,7 +74,14 @@ def _from_frames():
             logger.error("truncated data stream (%d bytes): %s", size, datastr)
             raise
 
-    if deserialize and FRAME_OFFLOAD_THRESHOLD and size > FRAME_OFFLOAD_THRESHOLD:
+    if allow_offload and deserialize and FRAME_OFFLOAD_THRESHOLD:
+        size = sum(map(nbytes, frames))
+    if (
+        allow_offload
+        and deserialize
+        and FRAME_OFFLOAD_THRESHOLD
+        and size > FRAME_OFFLOAD_THRESHOLD
+    ):
         res = await offload(_from_frames)
     else:
         res = _from_frames()
diff --git a/distributed/core.py b/distributed/core.py
index 829d2095083..1220671d115 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -348,7 +348,7 @@ def port(self):
     def identity(self, comm=None):
         return {"type": type(self).__name__, "id": self.id}
 
-    async def listen(self, port_or_addr=None, **kwargs):
+    async def listen(self, port_or_addr=None, allow_offload=True, **kwargs):
         if port_or_addr is None:
             port_or_addr = self.default_port
         if isinstance(port_or_addr, int):
@@ -359,7 +359,11 @@ async def listen(self, port_or_addr=None, **kwargs):
             addr = port_or_addr
             assert isinstance(addr, str)
         listener = await listen(
-            addr, self.handle_comm, deserialize=self.deserialize, **kwargs,
+            addr,
+            self.handle_comm,
+            deserialize=self.deserialize,
+            allow_offload=allow_offload,
+            **kwargs,
         )
         self.listeners.append(listener)
 
@@ -863,6 +867,7 @@ def __init__(
         limit=512,
         deserialize=True,
         serializers=None,
+        allow_offload=True,
         deserializers=None,
         connection_args=None,
         timeout=None,
@@ -873,6 +878,7 @@ def __init__(
         self.available = defaultdict(set)
         # Invariant: len(occupied) == active
         self.occupied = defaultdict(set)
+        self.allow_offload = allow_offload
         self.deserialize = deserialize
         self.serializers = serializers
         self.deserializers = deserializers if deserializers is not None else serializers
@@ -953,6 +959,7 @@ async def connect(self, addr, timeout=None):
             )
             comm.name = "ConnectionPool"
             comm._pool = weakref.ref(self)
+            comm.allow_offload = self.allow_offload
             self._created.add(comm)
         except Exception:
             self.semaphore.release()
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index d032086ec79..dd96550ed41 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1372,6 +1372,7 @@ def __init__(
 
         setproctitle("dask-scheduler [not started]")
         Scheduler._instances.add(self)
+        self.rpc.allow_offload = False
 
     ##################
     # Administration #
@@ -1438,7 +1439,9 @@ async def start(self):
                 c.cancel()
 
         for addr in self._start_address:
-            await self.listen(addr, **self.security.get_listen_args("scheduler"))
+            await self.listen(
+                addr, allow_offload=False, **self.security.get_listen_args("scheduler")
+            )
             self.ip = get_address_host(self.listen_address)
             listen_ip = self.ip
 

From 342b6a65e30082c45b4072b29217ab4525572894 Mon Sep 17 00:00:00 2001
From: Nils Braun <nils-braun@users.noreply.github.com>
Date: Tue, 26 May 2020 01:46:30 +0200
Subject: [PATCH 0844/1550] Allow to pass in task key strings in the worker
 restrictions (#3826)

The `workers` parameter of Client.compute and Client.persist
allows to restrict the workers where the calculations of the tasks
should take place.
Advanced usages of this keyword might need passing the task key
(string) instead of the dask collection or dask task.
Internally, collections and tasks are turned into string keys
anyways.

This allows now to do things like:

	task = ... long dependencies ...
        all_dependencies = task.dask.dependencies.keys()

        workers = {d: worker_address for d in all_dependencies}
        result = client.compute(last_task, workers=workers)

to have a task and all of its dependencies calculated on a single
worker.
---
 distributed/client.py            | 14 ++++++++------
 distributed/tests/test_client.py | 25 +++++++++++++++++++++++++
 2 files changed, 33 insertions(+), 6 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 36234ea2e9b..170cac93228 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -2762,10 +2762,10 @@ def compute(
             Whether or not to optimize the underlying graphs
         workers: str, list, dict
             Which workers can run which parts of the computation
-            If a string a list then the output collections will run on the listed
+            If a string or list then the output collections will run on the listed
             workers, but other sub-computations can run anywhere
-            If a dict then keys should be (tuples of) collections and values
-            should be addresses or lists.
+            If a dict then keys should be (tuples of) collections or
+            task keys and values should be addresses or lists.
         allow_other_workers: bool, list
             If True then all restrictions in workers= are considered loose
             If a list then only the keys for the listed collections are loose
@@ -2912,10 +2912,10 @@ def persist(
             Whether or not to optimize the underlying graphs
         workers: str, list, dict
             Which workers can run which parts of the computation
-            If a string a list then the output collections will run on the listed
+            If a string or list then the output collections will run on the listed
             workers, but other sub-computations can run anywhere
-            If a dict then keys should be (tuples of) collections and values
-            should be addresses or lists.
+            If a dict then keys should be (tuples of) collections or
+            task keys and values should be addresses or lists.
         allow_other_workers: bool, list
             If True then all restrictions in workers= are considered loose
             If a list then only the keys for the listed collections are loose
@@ -3912,6 +3912,8 @@ def get_restrictions(cls, collections, workers, allow_other_workers):
                     ws = [ws]
                 if dask.is_dask_collection(colls):
                     keys = flatten(colls.__dask_keys__())
+                elif isinstance(colls, str):
+                    keys = [colls]
                 else:
                     keys = list(
                         {k for c in flatten(colls) for k in flatten(c.__dask_keys__())}
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index a347e1d325e..fe1deca9a72 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -913,6 +913,31 @@ async def test_restrictions_ip_port(c, s, a, b):
     assert y.key in b.data
 
 
+@gen_cluster(client=True)
+async def test_restrictions_ip_port_task_key(c, s, a, b):
+    # Create a long dependency list
+    tasks = [delayed(inc)(1)]
+    for _ in range(100):
+        tasks.append(delayed(add)(tasks[-1], random.choice(tasks)))
+
+    last_task = tasks[-1]
+
+    # calculate all dependency keys
+    all_tasks = list(last_task.__dask_graph__())
+    # only restrict to a single worker
+    workers = {d: a.address for d in all_tasks}
+    result = c.compute(last_task, workers=workers)
+    await result
+
+    # all tasks should have been calculated by the first worker
+    for task in tasks:
+        assert s.worker_restrictions[task.key] == {a.address}
+
+    # and the data should also be there
+    assert last_task.key in a.data
+    assert last_task.key not in b.data
+
+
 @pytest.mark.skipif(
     not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
 )

From 0041cc1f23fff4dd64c644c243ba8404122786c8 Mon Sep 17 00:00:00 2001
From: Abdulelah Bin Mahfoodh <abduh.bm@gmail.com>
Date: Tue, 26 May 2020 17:23:08 +0300
Subject: [PATCH 0845/1550] Include total number of tasks in the performance
 report (#3822)

* Include total number of tasks in the performance report

* Include tasks' timings in the performance report
---
 distributed/scheduler.py | 26 +++++++++++++++++++++-----
 1 file changed, 21 insertions(+), 5 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index dd96550ed41..3eae685c355 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1069,7 +1069,7 @@ def __init__(
         preload=None,
         preload_argv=(),
         plugins=(),
-        **kwargs
+        **kwargs,
     ):
         self._setup_logging(logger)
 
@@ -1352,7 +1352,7 @@ def __init__(
             connection_limit=connection_limit,
             deserialize=False,
             connection_args=self.connection_args,
-            **kwargs
+            **kwargs,
         )
 
         if self.worker_ttl:
@@ -2089,7 +2089,7 @@ def stimulus_task_erred(
                     exception=exception,
                     traceback=traceback,
                     worker=worker,
-                    **kwargs
+                    **kwargs,
                 )
         else:
             recommendations = {}
@@ -3389,7 +3389,7 @@ async def retire_workers(
         close_workers=False,
         names=None,
         lock=True,
-        **kwargs
+        **kwargs,
     ):
         """ Gracefully retire workers from cluster
 
@@ -4090,7 +4090,7 @@ def transition_processing_memory(
         typename=None,
         worker=None,
         startstops=None,
-        **kwargs
+        **kwargs,
     ):
         try:
             ts = self.tasks[key]
@@ -5055,6 +5055,15 @@ def profile_to_figure(state):
 
         # Task stream
         task_stream = self.get_task_stream(start=start)
+        total_tasks = len(task_stream)
+        timespent = defaultdict(int)
+        for d in task_stream:
+            for x in d["startstops"]:
+                timespent[x["action"]] += x["stop"] - x["start"]
+        tasks_timings = ""
+        for k in sorted(timespent.keys()):
+            tasks_timings += f"\n<li> {k} time: {format_time(timespent[k])} </li>"
+
         from .diagnostics.task_stream import rectangles
         from .dashboard.components.scheduler import task_stream_figure
 
@@ -5081,6 +5090,11 @@ def profile_to_figure(state):
         <i> Select different tabs on the top for additional information </i>
 
         <h2> Duration: {time} </h2>
+        <h2> Tasks Information </h2>
+        <ul>
+         <li> number of tasks: {ntasks} </li>
+         {tasks_timings}
+        </ul>
 
         <h2> Scheduler Information </h2>
         <ul>
@@ -5096,6 +5110,8 @@ def profile_to_figure(state):
         </pre>
         """.format(
             time=format_time(stop - start),
+            ntasks=total_tasks,
+            tasks_timings=tasks_timings,
             address=self.address,
             nworkers=len(self.workers),
             threads=sum(w.nthreads for w in self.workers.values()),

From 9f85808337367dfcba6104a2b4a280d942f716dc Mon Sep 17 00:00:00 2001
From: Krishan Bhasin <8904718+KrishanBhasin@users.noreply.github.com>
Date: Tue, 26 May 2020 17:35:03 +0100
Subject: [PATCH 0846/1550] Set colour of nbytes pane based on thresholds
 (#3805)

* try-catch missing name error in BandwithWorker

* first cut

* make ordering match

* set yellow at 85% of target (by default ~50% of available RAM)

* code formatting

* remove demo snippet

Co-authored-by: KrishanBhasin <KrishanBhasin@users.noreply.github.com>
---
 distributed/dashboard/components/scheduler.py | 30 ++++++++++++++++++-
 1 file changed, 29 insertions(+), 1 deletion(-)

diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index e1410244f02..466fc36a320 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -664,14 +664,42 @@ def update(self):
                     getattr(self.scheduler.workers[ws.address], "memory_limit", inf)
                     or inf
                 )
+                pause = (
+                    getattr(
+                        self.scheduler.workers[ws.address],
+                        "memory_target_fraction",
+                        inf,
+                    )
+                    or inf
+                )
+                spill = (
+                    getattr(
+                        self.scheduler.workers[ws.address],
+                        "memory_target_fraction",
+                        inf,
+                    )
+                    or inf
+                )
+                target = (
+                    getattr(
+                        self.scheduler.workers[ws.address],
+                        "memory_target_fraction",
+                        inf,
+                    )
+                    or inf
+                )
 
                 if limit > max_limit and limit != inf:
                     max_limit = limit
 
                 if nb > limit:
+                    nbytes_color.append("black")
+                elif nb > pause:
                     nbytes_color.append("red")
-                elif nb > limit / 2:
+                elif nb > target:
                     nbytes_color.append("orange")
+                elif nb > target * 0.85:
+                    nbytes_color.append("yellow")
                 else:
                     nbytes_color.append("blue")
 

From 43ad2f8cfaa216588f15d39dc4ea24c58ac2c633 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 26 May 2020 13:03:05 -0700
Subject: [PATCH 0847/1550] Record the time since the last run task on the
 scheduler (#3830)

This is used for the idle-timeout keyword.
This makes the logic a bit simpler,
and also gives more informative state.
---
 distributed/scheduler.py            | 17 ++++++-----------
 distributed/tests/test_scheduler.py |  4 ++++
 2 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 3eae685c355..dbcd8109ef5 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1099,7 +1099,7 @@ def __init__(
             self.idle_timeout = parse_timedelta(idle_timeout)
         else:
             self.idle_timeout = None
-        self.time_started = time()
+        self.idle_since = time()
         self._lock = asyncio.Lock()
         self.bandwidth = parse_bytes(dask.config.get("distributed.scheduler.bandwidth"))
         self.bandwidth_workers = defaultdict(float)
@@ -5246,18 +5246,13 @@ def check_worker_ttl(self):
                 self.remove_worker(address=ws.address)
 
     def check_idle(self):
-        if any(ws.processing for ws in self.workers.values()):
+        if any(ws.processing for ws in self.workers.values()) or self.unrunnable:
+            self.idle_since = None
             return
-        if self.unrunnable:
-            return
-
-        if not self.transition_log:
-            close = time() > self.time_started + self.idle_timeout
-        else:
-            last_task = self.transition_log[-1][-1]
-            close = time() > last_task + self.idle_timeout
+        elif not self.idle_since:
+            self.idle_since = time()
 
-        if close:
+        if time() > self.idle_since + self.idle_timeout:
             logger.info(
                 "Scheduler closing after being idle for %s",
                 format_time(self.idle_timeout),
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 89df2757f88..da121e2c9d0 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -1540,8 +1540,11 @@ async def test_collect_versions(c, s, a, b):
 
 @gen_cluster(client=True, config={"distributed.scheduler.idle-timeout": "500ms"})
 async def test_idle_timeout(c, s, a, b):
+    beginning = time()
+    assert s.idle_since <= beginning
     future = c.submit(slowinc, 1)
     await future
+    assert s.idle_since is None or s.idle_since > beginning
 
     assert s.status != "closed"
 
@@ -1559,6 +1562,7 @@ async def test_idle_timeout(c, s, a, b):
     assert "idle" in logs.getvalue()
     assert "500" in logs.getvalue()
     assert "ms" in logs.getvalue()
+    assert s.idle_since > beginning
 
 
 @gen_cluster(client=True, config={"distributed.scheduler.bandwidth": "100 GB"})

From 410126467f8dfb7fac3d978a82e456f4d40feb3a Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Tue, 26 May 2020 21:57:17 -0500
Subject: [PATCH 0848/1550] bump version to 2.17.0

---
 docs/source/changelog.rst | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index 17905db5b6d..ed799f5ea2d 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,31 @@
 Changelog
 =========
 
+2.17.0 - 2020-05-26
+-------------------
+
+- Record the time since the last run task on the scheduler (:pr:`3830`) `Matthew Rocklin`_
+- Set colour of ``nbytes`` pane based on thresholds (:pr:`3805`) `Krishan Bhasin`_
+- Include total number of tasks in the performance report (:pr:`3822`) `Abdulelah Bin Mahfoodh`_
+- Allow to pass in task key strings in the worker restrictions (:pr:`3826`) `Nils Braun`_
+- Control de/ser offload (:pr:`3793`) `Martin Durant`_
+- Parse timeout parameters in ``Variable``/``Event``/``Lock`` to support text timeouts (:pr:`3825`) `Nils Braun`_
+- Don't send empty dependencies (:pr:`3423`) `Jakub Beránek`_
+- Add distributed Dask ``Event`` that mimics ``threading.Event`` (:pr:`3821`) `Nils Braun`_
+- Enhance ``VersionMismatchWarning`` messages (:pr:`3786`) `Abdulelah Bin Mahfoodh`_
+- Support Pickle's protocol 5 (:pr:`3784`) `jakirkham`_
+- Replace ``utils.ignoring`` with ``contextlib.suppress`` (:pr:`3819`) `Nils Braun`_
+- Make re-creating conda environments from the CI output easier (:pr:`3816`) `Lucas Rademaker`_
+- Add prometheus metrics for semaphore (:pr:`3757`) `Lucas Rademaker`_
+- Fix worker plugin called with superseded transition (:pr:`3812`) `Nick Evans`_
+- Add retries to server listen (:pr:`3801`) `Jacob Tomlinson`_
+- Remove commented out lines from ``scheduler.py`` (:pr:`3803`) `James Bourbeau`_
+- Fix ``RuntimeWarning`` for never awaited coroutine when using ``distributed.Semaphore`` (:pr:`3713`) `Florian Jetter`_
+- Fix profile thread leakage during test teardown on some platforms (:pr:`3795`) `Florian Jetter`_
+- Await self before handling comms (:pr:`3788`) `Matthew Rocklin`_
+- Fix typo in ``Cluster`` docstring (:pr:`3787`) `Scott Sanderson`_
+
+
 2.16.0 - 2020-05-08
 -------------------
 
@@ -1749,3 +1774,6 @@ significantly without many new features.
 .. _`Nicholas Smith`: https://github.com/nsmith-
 .. _`Dillon Niederhut`: https://github.com/deniederhut
 .. _`Jonas Haag`: https://github.com/jonashaag
+.. _`Nils Braun`: https://github.com/nils-braun
+.. _`Nick Evans`: https://github.com/nre
+.. _`Scott Sanderson`: https://github.com/ssanderson

From 99e70cb6e409377798d60ca0014356e9047d503c Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Wed, 27 May 2020 09:45:11 +0200
Subject: [PATCH 0849/1550] Allow collection of partial profile information in
 case of exceptions (#3773)

---
 distributed/scheduler.py            |  9 ++++-
 distributed/tests/test_client.py    |  4 +-
 distributed/tests/test_scheduler.py | 61 ++++++++++++++++++++++++++++-
 3 files changed, 68 insertions(+), 6 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index dbcd8109ef5..e186ccde9ef 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -4975,9 +4975,12 @@ async def get_profile(
             *(
                 self.rpc(w).profile(start=start, stop=stop, key=key, server=server)
                 for w in workers
-            )
+            ),
+            return_exceptions=True
         )
 
+        results = [r for r in results if not isinstance(r, Exception)]
+
         if merge_workers:
             response = profile.merge(*results)
         else:
@@ -5003,9 +5006,11 @@ async def get_profile_metadata(
         else:
             workers = set(self.workers) & set(workers)
         results = await asyncio.gather(
-            *(self.rpc(w).profile_metadata(start=start, stop=stop) for w in workers)
+            *(self.rpc(w).profile_metadata(start=start, stop=stop) for w in workers),
+            return_exceptions=True
         )
 
+        results = [r for r in results if not isinstance(r, Exception)]
         counts = [v["counts"] for v in results]
         counts = itertools.groupby(merge_sorted(*counts), lambda t: t[0] // dt * dt)
         counts = [(time, sum(pluck(1, group))) for time, group in counts]
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index fe1deca9a72..f940bf45ba3 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -5133,7 +5133,7 @@ async def test_call_stack_collections_all(c, s, a, b):
     assert result
 
 
-@gen_cluster(client=True, worker_kwargs={"profile_cycle_interval": 100})
+@gen_cluster(client=True, worker_kwargs={"profile_cycle_interval": "100ms"})
 async def test_profile(c, s, a, b):
     futures = c.map(slowinc, range(10), delay=0.05, workers=a.address)
     await wait(futures)
@@ -5155,7 +5155,7 @@ async def test_profile(c, s, a, b):
     assert not result["count"]
 
 
-@gen_cluster(client=True, worker_kwargs={"profile_cycle_interval": 100})
+@gen_cluster(client=True, worker_kwargs={"profile_cycle_interval": "100ms"})
 async def test_profile_keys(c, s, a, b):
     x = c.map(slowinc, range(10), delay=0.05, workers=a.address)
     y = c.map(slowdec, range(10), delay=0.05, workers=a.address)
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index da121e2c9d0..a08df22e080 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -1219,7 +1219,7 @@ async def test_service_hosts():
             assert sock.getsockname()[0] == "127.0.0.1"
 
 
-@gen_cluster(client=True, worker_kwargs={"profile_cycle_interval": 100})
+@gen_cluster(client=True, worker_kwargs={"profile_cycle_interval": "100ms"})
 async def test_profile_metadata(c, s, a, b):
     start = time() - 1
     futures = c.map(slowinc, range(10), delay=0.05, workers=a.address)
@@ -1234,7 +1234,28 @@ async def test_profile_metadata(c, s, a, b):
     assert not meta["counts"][-1][1]
 
 
-@gen_cluster(client=True, worker_kwargs={"profile_cycle_interval": 100})
+@gen_cluster(client=True, worker_kwargs={"profile_cycle_interval": "100ms"})
+async def test_profile_metadata_timeout(c, s, a, b):
+    start = time() - 1
+
+    def raise_timeout(*args, **kwargs):
+        raise TimeoutError
+
+    b.handlers["profile_metadata"] = raise_timeout
+
+    futures = c.map(slowinc, range(10), delay=0.05, workers=a.address)
+    await wait(futures)
+    await asyncio.sleep(0.200)
+
+    meta = await s.get_profile_metadata(profile_cycle_interval=0.100)
+    now = time() + 1
+    assert meta
+    assert all(start < t < now for t, count in meta["counts"])
+    assert all(0 <= count < 30 for t, count in meta["counts"][:4])
+    assert not meta["counts"][-1][1]
+
+
+@gen_cluster(client=True, worker_kwargs={"profile_cycle_interval": "100ms"})
 async def test_profile_metadata_keys(c, s, a, b):
     x = c.map(slowinc, range(10), delay=0.05)
     y = c.map(slowdec, range(10), delay=0.05)
@@ -1247,6 +1268,42 @@ async def test_profile_metadata_keys(c, s, a, b):
     )
 
 
+@gen_cluster(
+    client=True,
+    config={
+        "distributed.worker.profile.interval": "1ms",
+        "distributed.worker.profile.cycle": "100ms",
+    },
+)
+async def test_statistical_profiling(c, s, a, b):
+    futures = c.map(slowinc, range(10), delay=0.1)
+
+    await wait(futures)
+
+    profile = await s.get_profile()
+    assert profile["count"]
+
+
+@gen_cluster(
+    client=True,
+    config={
+        "distributed.worker.profile.interval": "1ms",
+        "distributed.worker.profile.cycle": "100ms",
+    },
+)
+async def test_statistical_profiling_failure(c, s, a, b):
+    futures = c.map(slowinc, range(10), delay=0.1)
+
+    def raise_timeout(*args, **kwargs):
+        raise TimeoutError
+
+    b.handlers["profile"] = raise_timeout
+    await wait(futures)
+
+    profile = await s.get_profile()
+    assert profile["count"]
+
+
 @gen_cluster(client=True)
 async def test_cancel_fire_and_forget(c, s, a, b):
     x = delayed(slowinc)(1, delay=0.05)

From e23c93401b6ff1e98557dec6b703b5285f353aa2 Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Thu, 28 May 2020 04:03:57 +0100
Subject: [PATCH 0850/1550] Fix key check in rebalance missing keys (#3834)

---
 distributed/scheduler.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index e186ccde9ef..ed1606c11b7 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -3124,7 +3124,13 @@ async def rebalance(self, comm=None, keys=None, workers=None):
                 if not all(r["status"] == "OK" for r in result):
                     return {
                         "status": "missing-data",
-                        "keys": tuple(concat(r["keys"].keys() for r in result)),
+                        "keys": tuple(
+                            concat(
+                                r["keys"].keys()
+                                for r in result
+                                if r["status"] == "missing-data"
+                            )
+                        ),
                     }
 
                 for sender, recipient, ts in msgs:
@@ -4976,7 +4982,7 @@ async def get_profile(
                 self.rpc(w).profile(start=start, stop=stop, key=key, server=server)
                 for w in workers
             ),
-            return_exceptions=True
+            return_exceptions=True,
         )
 
         results = [r for r in results if not isinstance(r, Exception)]
@@ -5007,7 +5013,7 @@ async def get_profile_metadata(
             workers = set(self.workers) & set(workers)
         results = await asyncio.gather(
             *(self.rpc(w).profile_metadata(start=start, stop=stop) for w in workers),
-            return_exceptions=True
+            return_exceptions=True,
         )
 
         results = [r for r in results if not isinstance(r, Exception)]

From 9db7368bae6f5598324b2b0c8beaf292dfd382fc Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Thu, 28 May 2020 02:00:44 -0700
Subject: [PATCH 0851/1550] Register `dask_cudf` serializers (#3832)

As `dask_cudf` simply relies on the same serialization machinery that is
in `cudf` (plus whatever is implemented by `dask_cudf` objects
themselves), just reuse the `cudf` serialization import to handle
`dask_cudf` object serialization for simplicity.
---
 distributed/protocol/__init__.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/distributed/protocol/__init__.py b/distributed/protocol/__init__.py
index 79e77e9703b..be1c498c35c 100644
--- a/distributed/protocol/__init__.py
+++ b/distributed/protocol/__init__.py
@@ -105,6 +105,10 @@ def _register_rmm():
 @cuda_deserialize.register_lazy("cudf")
 @dask_serialize.register_lazy("cudf")
 @dask_deserialize.register_lazy("cudf")
+@cuda_serialize.register_lazy("dask_cudf")
+@cuda_deserialize.register_lazy("dask_cudf")
+@dask_serialize.register_lazy("dask_cudf")
+@dask_deserialize.register_lazy("dask_cudf")
 def _register_cudf():
     from cudf.comm import serialize
 

From 2d94142be584f9f702cbd23fdd1939d51409a235 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Thu, 28 May 2020 22:11:06 -0500
Subject: [PATCH 0852/1550] Fix typo in journey of a task doc (#3838)

---
 docs/source/journey.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/source/journey.rst b/docs/source/journey.rst
index dd7e60e8d42..c7e6f12719e 100644
--- a/docs/source/journey.rst
+++ b/docs/source/journey.rst
@@ -12,8 +12,8 @@ A user computes the addition of two variables already on the cluster, then pulls
 .. code-block:: python
 
    client = Client('host:port')
-   x = e.submit(...)
-   y = e.submit(...)
+   x = client.submit(...)
+   y = client.submit(...)
 
    z = client.submit(add, x, y)  # we follow z
 

From 1fe50c24792d74643772268c188decd3c1ab76ee Mon Sep 17 00:00:00 2001
From: Matthias Bussonnier <bussonniermatthias@gmail.com>
Date: Fri, 29 May 2020 08:52:04 -0700
Subject: [PATCH 0853/1550] Fix variable name in journey of a task
 documentation (#3840)

Spotted by James Bourbeau during a demo of dask/distributed.
---
 docs/source/journey.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/source/journey.rst b/docs/source/journey.rst
index c7e6f12719e..1b0c9182c9e 100644
--- a/docs/source/journey.rst
+++ b/docs/source/journey.rst
@@ -120,7 +120,7 @@ Nothing blocks.*
 Eventually the computation completes.  The Worker stores this result in its
 local memory::
 
-    data['x'] = ...
+    data['z'] = ...
 
 And transmits back a success, and the number of bytes of the result::
 

From 1313e14767a9b2d4706cc01f87d477fdb5e55004 Mon Sep 17 00:00:00 2001
From: Benjamin Zaitlen <quasiben@users.noreply.github.com>
Date: Thu, 4 Jun 2020 10:35:23 -0400
Subject: [PATCH 0854/1550] Add dashboard plots for the amount of time spent
 per key and for transfer/serialization (#3792)

---
 distributed/dashboard/components/scheduler.py | 250 ++++++++++++++++--
 distributed/dashboard/scheduler.py            |   4 +
 .../dashboard/tests/test_scheduler_bokeh.py   |  48 ++++
 distributed/scheduler.py                      |  23 +-
 4 files changed, 290 insertions(+), 35 deletions(-)

diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index 466fc36a320..2d4bbcccf6d 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -64,6 +64,8 @@
 from distributed.diagnostics.progress_stream import color_of, progress_quads
 from distributed.diagnostics.graph_layout import GraphLayout
 from distributed.diagnostics.task_stream import TaskStreamPlugin
+from distributed.diagnostics.task_stream import color_of as ts_color_of
+from distributed.diagnostics.task_stream import colors as ts_color_lookup
 
 if dask.config.get("distributed.dashboard.export-tool"):
     from distributed.dashboard.export_tool import ExportTool
@@ -438,6 +440,206 @@ def name(address):
             update(self.source, result)
 
 
+class ComputerPerKey(DashboardComponent):
+    """ Bar chart showing time spend in action by key prefix"""
+
+    def __init__(self, scheduler, **kwargs):
+        with log_errors():
+            self.last = 0
+            self.scheduler = scheduler
+
+            es = [p for p in self.scheduler.plugins if isinstance(p, TaskStreamPlugin)]
+            if not es:
+                self.plugin = TaskStreamPlugin(self.scheduler)
+            else:
+                self.plugin = es[0]
+
+            compute_data = {
+                "times": [0.2, 0.1],
+                "color": [ts_color_lookup["transfer"], ts_color_lookup["compute"]],
+                "names": ["sum", "sum_partial"],
+            }
+
+            self.compute_source = ColumnDataSource(data=compute_data)
+
+            fig = figure(
+                title="Compute Time Per Task",
+                tools="",
+                id="bk-Compute-by-key-plot",
+                name="compute_time_per_key",
+                x_range=["a", "b"],
+                **kwargs,
+            )
+
+            rect = fig.vbar(
+                source=self.compute_source,
+                x="names",
+                top="times",
+                width=0.7,
+                color="color",
+                legend_field="names",
+            )
+
+            fig.y_range.start = 0
+            fig.min_border_right = 20
+            fig.min_border_bottom = 60
+            fig.yaxis.axis_label = "Time (s)"
+            fig.yaxis[0].formatter = NumeralTickFormatter(format="0.0s")
+            fig.yaxis.ticker = AdaptiveTicker(**TICKS_1024)
+            fig.xaxis.major_label_orientation = -math.pi / 12
+            rect.nonselection_glyph = None
+
+            fig.xaxis.minor_tick_line_alpha = 0
+            fig.xgrid.visible = False
+
+            fig.toolbar.logo = None
+            fig.toolbar_location = None
+
+            hover = HoverTool()
+            hover.tooltips = """
+            <div>
+                <p><b>Name:</b> @names</p>
+                <p><b>Time:</b> @times s</p>
+            </div>
+            """
+            hover.point_policy = "follow_mouse"
+            fig.add_tools(hover)
+
+            self.fig = fig
+
+    @without_property_validation
+    def update(self):
+        with log_errors():
+            compute_times = defaultdict(float)
+
+            for key, ts in self.scheduler.task_prefixes.items():
+                name = key_split(key)
+                for action, t in ts.all_durations.items():
+                    if action == "compute":
+                        compute_times[name] += t
+
+            # order by largest time first
+            compute_times = sorted(
+                compute_times.items(), key=lambda x: x[1], reverse=True
+            )
+
+            compute_colors = list()
+            compute_names = list()
+            compute_time = list()
+            for name, t in compute_times:
+                compute_names.append(name)
+                compute_colors.append(ts_color_of(name))
+                compute_time.append(t)
+
+            self.fig.x_range.factors = compute_names
+            self.fig.title.text = "Compute Time Per Task"
+
+            compute_result = dict(
+                times=compute_time, color=compute_colors, names=compute_names,
+            )
+
+            update(self.compute_source, compute_result)
+
+
+class AggregateAction(DashboardComponent):
+    """ Bar chart showing time spend in action by key prefix"""
+
+    def __init__(self, scheduler, **kwargs):
+        with log_errors():
+            self.last = 0
+            self.scheduler = scheduler
+
+            es = [p for p in self.scheduler.plugins if isinstance(p, TaskStreamPlugin)]
+            if not es:
+                self.plugin = TaskStreamPlugin(self.scheduler)
+            else:
+                self.plugin = es[0]
+
+            action_data = {
+                "times": [0.2, 0.1],
+                "color": [ts_color_lookup["transfer"], ts_color_lookup["compute"]],
+                "names": ["transfer", "compute"],
+            }
+
+            self.action_source = ColumnDataSource(data=action_data)
+
+            fig = figure(
+                title="Aggregate Per Action",
+                tools="",
+                id="bk-aggregate-per-action-plot",
+                name="aggregate_per_action",
+                x_range=["a", "b"],
+                **kwargs,
+            )
+
+            rect = fig.vbar(
+                source=self.action_source,
+                x="names",
+                top="times",
+                width=0.7,
+                color="color",
+                legend_field="names",
+            )
+
+            fig.y_range.start = 0
+            fig.min_border_right = 20
+            fig.min_border_bottom = 60
+            fig.yaxis[0].formatter = NumeralTickFormatter(format="0.0s")
+            fig.yaxis.axis_label = "Time (s)"
+            fig.yaxis.ticker = AdaptiveTicker(**TICKS_1024)
+            fig.xaxis.major_label_orientation = -math.pi / 12
+            fig.xaxis.major_label_text_font_size = "16px"
+            rect.nonselection_glyph = None
+
+            fig.xaxis.minor_tick_line_alpha = 0
+            fig.xgrid.visible = False
+
+            fig.toolbar.logo = None
+            fig.toolbar_location = None
+
+            hover = HoverTool()
+            hover.tooltips = """
+            <div>
+                <p><b>Name:</b> @names</p>
+                <p><b>Time:</b> @times s</p>
+            </div>
+            """
+            hover.point_policy = "follow_mouse"
+            fig.add_tools(hover)
+
+            self.fig = fig
+
+    @without_property_validation
+    def update(self):
+        with log_errors():
+            agg_times = defaultdict(float)
+
+            for key, ts in self.scheduler.task_prefixes.items():
+                for action, t in ts.all_durations.items():
+                    agg_times[action] += t
+
+            # order by largest time first
+            agg_times = sorted(agg_times.items(), key=lambda x: x[1], reverse=True)
+
+            agg_colors = list()
+            agg_names = list()
+            agg_time = list()
+            for action, t in agg_times:
+                agg_names.append(action)
+                if action == "compute":
+                    agg_colors.append("purple")
+                else:
+                    agg_colors.append(ts_color_lookup[action])
+                agg_time.append(t)
+
+            self.fig.x_range.factors = agg_names
+            self.fig.title.text = "Aggregate Time Per Action"
+
+            action_result = dict(times=agg_time, color=agg_colors, names=agg_names,)
+
+            update(self.action_source, action_result)
+
+
 class MemoryByKey(DashboardComponent):
     """ Bar chart showing memory use by key prefix"""
 
@@ -664,42 +866,14 @@ def update(self):
                     getattr(self.scheduler.workers[ws.address], "memory_limit", inf)
                     or inf
                 )
-                pause = (
-                    getattr(
-                        self.scheduler.workers[ws.address],
-                        "memory_target_fraction",
-                        inf,
-                    )
-                    or inf
-                )
-                spill = (
-                    getattr(
-                        self.scheduler.workers[ws.address],
-                        "memory_target_fraction",
-                        inf,
-                    )
-                    or inf
-                )
-                target = (
-                    getattr(
-                        self.scheduler.workers[ws.address],
-                        "memory_target_fraction",
-                        inf,
-                    )
-                    or inf
-                )
 
                 if limit > max_limit and limit != inf:
                     max_limit = limit
 
                 if nb > limit:
-                    nbytes_color.append("black")
-                elif nb > pause:
                     nbytes_color.append("red")
-                elif nb > target:
+                elif nb > limit / 2:
                     nbytes_color.append("orange")
-                elif nb > target * 0.85:
-                    nbytes_color.append("yellow")
                 else:
                     nbytes_color.append("blue")
 
@@ -1932,6 +2106,24 @@ def individual_memory_by_key_doc(scheduler, extra, doc):
         doc.theme = BOKEH_THEME
 
 
+def individual_compute_time_per_key_doc(scheduler, extra, doc):
+    with log_errors():
+        component = ComputerPerKey(scheduler, sizing_mode="stretch_both")
+        component.update()
+        add_periodic_callback(doc, component, 500)
+        doc.add_root(component.fig)
+        doc.theme = BOKEH_THEME
+
+
+def individual_aggregate_time_per_action_doc(scheduler, extra, doc):
+    with log_errors():
+        component = AggregateAction(scheduler, sizing_mode="stretch_both")
+        component.update()
+        add_periodic_callback(doc, component, 500)
+        doc.add_root(component.fig)
+        doc.theme = BOKEH_THEME
+
+
 def profile_doc(scheduler, extra, doc):
     with log_errors():
         doc.title = "Dask: Profile"
diff --git a/distributed/dashboard/scheduler.py b/distributed/dashboard/scheduler.py
index 825195ecefa..31ecadeca4c 100644
--- a/distributed/dashboard/scheduler.py
+++ b/distributed/dashboard/scheduler.py
@@ -32,6 +32,8 @@
     individual_bandwidth_types_doc,
     individual_bandwidth_workers_doc,
     individual_memory_by_key_doc,
+    individual_compute_time_per_key_doc,
+    individual_aggregate_time_per_action_doc,
 )
 from .worker import counters_doc
 
@@ -83,6 +85,8 @@ def connect(application, http_server, scheduler, prefix=""):
     "/individual-bandwidth-types": individual_bandwidth_types_doc,
     "/individual-bandwidth-workers": individual_bandwidth_workers_doc,
     "/individual-memory-by-key": individual_memory_by_key_doc,
+    "/individual-compute-time-per-key": individual_compute_time_per_key_doc,
+    "/individual-aggregate-time-per-action": individual_aggregate_time_per_action_doc,
 }
 
 try:
diff --git a/distributed/dashboard/tests/test_scheduler_bokeh.py b/distributed/dashboard/tests/test_scheduler_bokeh.py
index f943807d4df..1207b47052a 100644
--- a/distributed/dashboard/tests/test_scheduler_bokeh.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh.py
@@ -35,6 +35,8 @@
     TaskGraph,
     ProfileServer,
     MemoryByKey,
+    AggregateAction,
+    ComputerPerKey,
 )
 from distributed.dashboard import scheduler
 
@@ -715,6 +717,52 @@ async def test_memory_by_key(c, s, a, b):
     assert mbk.source.data["nbytes"] == [x.nbytes, sys.getsizeof(1)]
 
 
+@gen_cluster(client=True, scheduler_kwargs={"dashboard": True})
+async def test_aggregate_action(c, s, a, b):
+    mbk = AggregateAction(s)
+
+    da = pytest.importorskip("dask.array")
+    x = (da.ones((20, 20), chunks=(10, 10)) + 1).persist(optimize_graph=False)
+
+    await x
+    y = await dask.delayed(inc)(1).persist()
+    z = (x + x.T) - x.mean(axis=0)
+    await c.compute(z.sum())
+
+    mbk.update()
+    http_client = AsyncHTTPClient()
+    response = await http_client.fetch(
+        "http://localhost:%d/individual-aggregate-time-per-action" % s.http_server.port
+    )
+    assert response.code == 200
+
+    assert ("transfer") in mbk.action_source.data["names"]
+    assert ("compute") in mbk.action_source.data["names"]
+
+
+@gen_cluster(client=True, scheduler_kwargs={"dashboard": True})
+async def test_computer_per_key(c, s, a, b):
+    mbk = ComputerPerKey(s)
+
+    da = pytest.importorskip("dask.array")
+    x = (da.ones((20, 20), chunks=(10, 10)) + 1).persist(optimize_graph=False)
+
+    await x
+    y = await dask.delayed(inc)(1).persist()
+    z = (x + x.T) - x.mean(axis=0)
+    await c.compute(z.sum())
+
+    mbk.update()
+    http_client = AsyncHTTPClient()
+    response = await http_client.fetch(
+        "http://localhost:%d/individual-compute-time-per-key" % s.http_server.port
+    )
+    assert response.code == 200
+    assert ("sum-aggregate") in mbk.compute_source.data["names"]
+    assert ("inc") in mbk.compute_source.data["names"]
+    assert ("add") in mbk.compute_source.data["names"]
+
+
 @gen_cluster(scheduler_kwargs={"http_prefix": "foo-bar", "dashboard": True})
 async def test_prefix_bokeh(s, a, b):
     prefix = "foo-bar"
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index ed1606c11b7..94adda8175f 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -809,6 +809,10 @@ class TaskPrefix:
     def __init__(self, name):
         self.name = name
         self.groups = []
+
+        # store timings for each prefix-action
+        self.all_durations = defaultdict(float)
+
         if self.name in dask.config.get("distributed.scheduler.default-task-durations"):
             self.duration_average = parse_timedelta(
                 dask.config.get("distributed.scheduler.default-task-durations")[
@@ -4127,12 +4131,19 @@ def transition_processing_memory(
                 return {}
 
             if startstops:
-                L = [
-                    (startstop["start"], startstop["stop"])
-                    for startstop in startstops
-                    if startstop["action"] == "compute"
-                ]
-                if L:
+                L = list()
+                for startstop in startstops:
+                    stop = startstop["stop"]
+                    start = startstop["start"]
+                    action = startstop["action"]
+                    if action == "compute":
+                        L.append((start, stop))
+
+                    # record timings of all actions -- a cheaper way of
+                    # getting timing info compared with get_task_stream()
+                    ts.prefix.all_durations[action] += stop - start
+
+                if len(L) > 0:
                     compute_start, compute_stop = L[0]
                 else:  # This is very rare
                     compute_start = compute_stop = None

From 85620bf43d67807e60cdcb45d18270f131a1cb14 Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Thu, 4 Jun 2020 20:08:50 +0100
Subject: [PATCH 0855/1550] Add favicon to performance report (#3852)

---
 distributed/dashboard/templates/performance_report.html | 6 ++++++
 distributed/scheduler.py                                | 9 ++++++++-
 2 files changed, 14 insertions(+), 1 deletion(-)
 create mode 100644 distributed/dashboard/templates/performance_report.html

diff --git a/distributed/dashboard/templates/performance_report.html b/distributed/dashboard/templates/performance_report.html
new file mode 100644
index 00000000000..183ea0c3c5c
--- /dev/null
+++ b/distributed/dashboard/templates/performance_report.html
@@ -0,0 +1,6 @@
+{% extends "file.html" %}
+{# See https://github.com/bokeh/bokeh/blob/master/bokeh/core/_templates/file.html #}
+
+{% block preamble %}
+<link rel="shortcut icon" href="https://docs.dask.org/en/latest/_static/images/favicon.ico" />
+{% endblock %}
\ No newline at end of file
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 94adda8175f..874f204aeea 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -5165,10 +5165,17 @@ def profile_to_figure(state):
         )
 
         from bokeh.plotting import save, output_file
+        from bokeh.core.templates import get_env
 
         with tmpfile(extension=".html") as fn:
             output_file(filename=fn, title="Dask Performance Report")
-            save(tabs, filename=fn)
+            template_directory = os.path.join(
+                os.path.dirname(os.path.abspath(__file__)), "dashboard", "templates"
+            )
+            template_environment = get_env()
+            template_environment.loader.searchpath.append(template_directory)
+            template = template_environment.get_template("performance_report.html")
+            save(tabs, filename=fn, template=template)
 
             with open(fn) as f:
                 data = f.read()

From a71951bc60f13054d50af4ff479fd1fbc64b13ad Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Fri, 5 Jun 2020 15:05:06 +0100
Subject: [PATCH 0856/1550] Allow SSHCluster to take a list of connect_options
 (#3854)

---
 distributed/deploy/ssh.py            | 21 ++++++++++++++-----
 distributed/deploy/tests/test_ssh.py | 31 ++++++++++++++++++++++++++++
 2 files changed, 47 insertions(+), 5 deletions(-)

diff --git a/distributed/deploy/ssh.py b/distributed/deploy/ssh.py
index 22364ecbacf..ffa9c2ad76e 100644
--- a/distributed/deploy/ssh.py
+++ b/distributed/deploy/ssh.py
@@ -1,6 +1,6 @@
 import logging
 import sys
-from typing import List
+from typing import List, Union
 import warnings
 import weakref
 
@@ -230,7 +230,7 @@ async def start(self):
 
 def SSHCluster(
     hosts: List[str] = None,
-    connect_options: dict = {},
+    connect_options: Union[List[dict], dict] = {},
     worker_options: dict = {},
     scheduler_options: dict = {},
     worker_module: str = "distributed.cli.dask_worker",
@@ -261,8 +261,9 @@ def SSHCluster(
     hosts: List[str]
         List of hostnames or addresses on which to launch our cluster.
         The first will be used for the scheduler and the rest for workers.
-    connect_options: dict, optional
+    connect_options: dict or list of dict, optional
         Keywords to pass through to ``asyncssh.connect``.
+        If a list it must have the same length as ``hosts``.
     worker_options: dict, optional
         Keywords to pass on to workers.
     scheduler_options: dict, optional
@@ -311,11 +312,19 @@ def SSHCluster(
         kwargs.setdefault("worker_addrs", hosts)
         return OldSSHCluster(**kwargs)
 
+    if isinstance(connect_options, list) and len(connect_options) != len(hosts):
+        raise RuntimeError(
+            "When specifying a list of connect_options you must provide a "
+            "dictionary for each address."
+        )
+
     scheduler = {
         "cls": Scheduler,
         "options": {
             "address": hosts[0],
-            "connect_options": connect_options,
+            "connect_options": connect_options
+            if isinstance(connect_options, dict)
+            else connect_options[0],
             "kwargs": scheduler_options,
             "remote_python": remote_python,
         },
@@ -325,7 +334,9 @@ def SSHCluster(
             "cls": Worker,
             "options": {
                 "address": host,
-                "connect_options": connect_options,
+                "connect_options": connect_options
+                if isinstance(connect_options, dict)
+                else connect_options[i + 1],
                 "kwargs": worker_options,
                 "worker_module": worker_module,
                 "remote_python": remote_python,
diff --git a/distributed/deploy/tests/test_ssh.py b/distributed/deploy/tests/test_ssh.py
index 11885dd8612..d52987200b4 100644
--- a/distributed/deploy/tests/test_ssh.py
+++ b/distributed/deploy/tests/test_ssh.py
@@ -115,3 +115,34 @@ async def test_unimplemented_options():
             scheduler_kwargs={"idle_timeout": "5s", "port": 0},
         ) as cluster:
             assert cluster
+
+
+@pytest.mark.asyncio
+async def test_list_of_connect_options():
+    async with SSHCluster(
+        ["127.0.0.1"] * 3,
+        connect_options=[dict(known_hosts=None)] * 3,
+        asynchronous=True,
+        scheduler_options={"port": 0, "idle_timeout": "5s"},
+        worker_options={"death_timeout": "5s"},
+    ) as cluster:
+        assert len(cluster.workers) == 2
+        async with Client(cluster, asynchronous=True) as client:
+            result = await client.submit(lambda x: x + 1, 10)
+            assert result == 11
+        assert not cluster._supports_scaling
+
+        assert "SSH" in repr(cluster)
+
+
+@pytest.mark.asyncio
+async def test_list_of_connect_options_raises():
+    with pytest.raises(RuntimeError):
+        async with SSHCluster(
+            ["127.0.0.1"] * 3,
+            connect_options=[dict(known_hosts=None)] * 4,  # Mismatch in length 4 != 3
+            asynchronous=True,
+            scheduler_options={"port": 0, "idle_timeout": "5s"},
+            worker_options={"death_timeout": "5s"},
+        ) as _:
+            pass

From 6144eb1a2f93274148cf84299f0eb0a2d93fc509 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Fri, 5 Jun 2020 11:43:41 -0700
Subject: [PATCH 0857/1550] Merge frames in `deserialize_bytes` (#3639)

* Test byte serialization in frame splitting case

In some cases where the frames are particularly large, we may opt to
split them into smaller frames. This may be due to performance reasons
when transmitting data or it may be due to limitations like those of
compressors used to compact frames. So include a test case that we know
will get split to make sure it is handled correctly. Or at least make
sure we are catching errors that would cause it to be mishandled.

* Merge frames in `deserialize_bytes`

It appears that we are splitting frames in `serialize_byteslist` so that
we can compress them. However we are not merging them back together
afterwards during deserialization. This can cause an exception to be
raised by serializers that expected their frames to be structured in a
particular way.

To fix this, we make sure to call `merge_frames` after `decompress` in
`deserialize_bytes`. Further we make sure to pack the `lengths` of the
original `frames` in the `header` (if not already present) in
`serialize_byteslist`. This should ensure deserializers get the original
frame structuring back when they operate on them.
---
 distributed/protocol/serialize.py            |  6 +++++-
 distributed/protocol/tests/test_serialize.py | 16 +++++++++++++---
 distributed/protocol/utils.py                |  3 ---
 distributed/utils.py                         |  4 +++-
 4 files changed, 21 insertions(+), 8 deletions(-)

diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index e4fba2b7ba9..4ed27bf278b 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -9,12 +9,13 @@
 import msgpack
 
 from . import pickle
-from ..utils import has_keyword, typename
+from ..utils import has_keyword, nbytes, typename
 from .compression import maybe_compress, decompress
 from .utils import (
     unpack_frames,
     pack_frames_prelude,
     frame_split_size,
+    merge_frames,
     ensure_bytes,
     msgpack_opts,
 )
@@ -473,6 +474,8 @@ def replace_inner(x):
 
 def serialize_bytelist(x, **kwargs):
     header, frames = serialize(x, **kwargs)
+    if "lengths" not in header:
+        header["lengths"] = tuple(map(nbytes, frames))
     frames = sum(map(frame_split_size, frames), [])
     if frames:
         compression, frames = zip(*map(maybe_compress, frames))
@@ -499,6 +502,7 @@ def deserialize_bytes(b):
     else:
         header = {}
     frames = decompress(header, frames)
+    frames = merge_frames(header, frames)
     return deserialize(header, frames)
 
 
diff --git a/distributed/protocol/tests/test_serialize.py b/distributed/protocol/tests/test_serialize.py
index 4cad5a3653b..57fceaea0c9 100644
--- a/distributed/protocol/tests/test_serialize.py
+++ b/distributed/protocol/tests/test_serialize.py
@@ -190,9 +190,19 @@ def test_empty_loads_deep():
     assert isinstance(e2[0][0][0], Empty)
 
 
-def test_serialize_bytes():
-    for x in [1, "abc", np.arange(5), b"ab" * int(40e6)]:
-        b = serialize_bytes(x)
+@pytest.mark.parametrize(
+    "kwargs", [{}, {"serializers": ["pickle"]},],
+)
+def test_serialize_bytes(kwargs):
+    for x in [
+        1,
+        "abc",
+        np.arange(5),
+        b"ab" * int(40e6),
+        int(2 ** 26) * b"ab",
+        (int(2 ** 25) * b"ab", int(2 ** 25) * b"ab"),
+    ]:
+        b = serialize_bytes(x, **kwargs)
         assert isinstance(b, bytes)
         y = deserialize_bytes(b)
         assert str(x) == str(y)
diff --git a/distributed/protocol/utils.py b/distributed/protocol/utils.py
index e58732b881c..fa020dae909 100644
--- a/distributed/protocol/utils.py
+++ b/distributed/protocol/utils.py
@@ -59,9 +59,6 @@ def merge_frames(header, frames):
     """
     lengths = list(header["lengths"])
 
-    if not frames:
-        return frames
-
     assert sum(lengths) == sum(map(nbytes, frames))
 
     if all(len(f) == l for f, l in zip(frames, lengths)):
diff --git a/distributed/utils.py b/distributed/utils.py
index f43b2f7acc0..dec1b6b79d3 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -931,7 +931,9 @@ def ensure_bytes(s):
     >>> ensure_bytes(b'123')
     b'123'
     """
-    if hasattr(s, "encode"):
+    if isinstance(s, bytes):
+        return s
+    elif hasattr(s, "encode"):
         return s.encode()
     else:
         try:

From 5eb3745067fdba2170b8f61882644dab7992d288 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Fri, 5 Jun 2020 20:06:47 -0500
Subject: [PATCH 0858/1550] bump version to 2.18.0

---
 docs/source/changelog.rst | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index ed799f5ea2d..848158b589d 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,20 @@
 Changelog
 =========
 
+2.18.0 - 2020-06-05
+-------------------
+
+- Merge frames in ``deserialize_bytes`` (:pr:`3639`) `John Kirkham`_
+- Allow ``SSHCluster`` to take a list of ``connect_options`` (:pr:`3854`) `Jacob Tomlinson`_
+- Add favicon to performance report (:pr:`3852`) `Jacob Tomlinson`_
+- Add dashboard plots for the amount of time spent per key and for transfer/serialization (:pr:`3792`) `Benjamin Zaitlen`_
+- Fix variable name in journey of a task documentation (:pr:`3840`) `Matthias Bussonnier`_
+- Fix typo in journey of a task doc (:pr:`3838`) `James Bourbeau`_
+- Register ``dask_cudf`` serializers (:pr:`3832`) `John Kirkham`_
+- Fix key check in ``rebalance`` missing keys (:pr:`3834`) `Jacob Tomlinson`_
+- Allow collection of partial profile information in case of exceptions (:pr:`3773`) `Florian Jetter`_
+
+
 2.17.0 - 2020-05-26
 -------------------
 
@@ -1777,3 +1791,4 @@ significantly without many new features.
 .. _`Nils Braun`: https://github.com/nils-braun
 .. _`Nick Evans`: https://github.com/nre
 .. _`Scott Sanderson`: https://github.com/ssanderson
+.. _`Matthias Bussonnier`: https://github.com/Carreau

From 17d65babdd6ddb5cbf3bf3f6d93b6d3039e22878 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Fri, 5 Jun 2020 20:08:35 -0500
Subject: [PATCH 0859/1550] Allow async add_worker remove_worker plugin methods
 (#3847)

---
 distributed/core.py                           |  1 +
 .../tests/test_scheduler_plugin.py            | 35 +++++++++++++++++++
 distributed/scheduler.py                      | 33 ++++++++++-------
 distributed/stealing.py                       |  4 +--
 distributed/tests/test_scheduler.py           | 10 +++---
 5 files changed, 64 insertions(+), 19 deletions(-)

diff --git a/distributed/core.py b/distributed/core.py
index 1220671d115..69f177c6eea 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -516,6 +516,7 @@ async def handle_stream(self, comm, extra=None, every_cycle=[]):
                             handler = self.stream_handlers[op]
                             if is_coroutine_function(handler):
                                 self.loop.add_callback(handler, **merge(extra, msg))
+                                await gen.sleep(0)
                             else:
                                 handler(**merge(extra, msg))
                         else:
diff --git a/distributed/diagnostics/tests/test_scheduler_plugin.py b/distributed/diagnostics/tests/test_scheduler_plugin.py
index 31ada3f9e12..3f1e54f6ed7 100644
--- a/distributed/diagnostics/tests/test_scheduler_plugin.py
+++ b/distributed/diagnostics/tests/test_scheduler_plugin.py
@@ -70,6 +70,41 @@ def remove_worker(self, worker, scheduler):
     assert events == []
 
 
+@gen_cluster(nthreads=[], client=False)
+async def test_async_add_remove_worker(s):
+    events = []
+
+    class MyPlugin(SchedulerPlugin):
+        async def add_worker(self, worker, scheduler):
+            assert scheduler is s
+            events.append(("add_worker", worker))
+
+        async def remove_worker(self, worker, scheduler):
+            assert scheduler is s
+            events.append(("remove_worker", worker))
+
+    plugin = MyPlugin()
+    s.add_plugin(plugin)
+    assert events == []
+
+    async with Worker(s.address) as a:
+        async with Worker(s.address) as b:
+            pass
+
+    assert set(events) == {
+        ("add_worker", a.address),
+        ("add_worker", b.address),
+        ("remove_worker", a.address),
+        ("remove_worker", b.address),
+    }
+
+    events[:] = []
+    s.remove_plugin(plugin)
+    async with Worker(s.address):
+        pass
+    assert events == []
+
+
 @pytest.mark.asyncio
 async def test_lifecycle(cleanup):
     class LifeCycle(SchedulerPlugin):
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 874f204aeea..5a337676155 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1565,7 +1565,7 @@ async def close_worker(self, stream=None, worker=None, safe=None):
             address = nanny_addr or worker
 
             self.worker_send(worker, {"op": "close", "report": False})
-            self.remove_worker(address=worker, safe=safe)
+            await self.remove_worker(address=worker, safe=safe)
 
     ###########
     # Stimuli #
@@ -1724,7 +1724,9 @@ async def add_worker(
 
             for plugin in self.plugins[:]:
                 try:
-                    plugin.add_worker(scheduler=self, worker=address)
+                    result = plugin.add_worker(scheduler=self, worker=address)
+                    if inspect.isawaitable(result):
+                        await result
                 except Exception as e:
                     logger.exception(e)
 
@@ -2159,7 +2161,7 @@ def stimulus_retry(self, comm=None, keys=None, client=None):
 
         return tuple(seen)
 
-    def remove_worker(self, comm=None, address=None, safe=False, close=True):
+    async def remove_worker(self, comm=None, address=None, safe=False, close=True):
         """
         Remove worker from cluster
 
@@ -2240,7 +2242,9 @@ def remove_worker(self, comm=None, address=None, safe=False, close=True):
 
             for plugin in self.plugins[:]:
                 try:
-                    plugin.remove_worker(scheduler=self, worker=address)
+                    result = plugin.remove_worker(scheduler=self, worker=address)
+                    if inspect.isawaitable(result):
+                        await result
                 except Exception as e:
                     logger.exception(e)
 
@@ -2735,7 +2739,7 @@ async def handle_worker(self, comm=None, worker=None):
         finally:
             if worker in self.stream_comms:
                 worker_comm.abort()
-                self.remove_worker(address=worker)
+                await self.remove_worker(address=worker)
 
     def add_plugin(self, plugin=None, idempotent=False, **kwargs):
         """
@@ -2848,8 +2852,12 @@ async def gather(self, comm=None, keys=None, serializers=None):
             with log_errors():
                 # Remove suspicious workers from the scheduler but allow them to
                 # reconnect.
-                for worker in missing_workers:
-                    self.remove_worker(address=worker, close=False)
+                await asyncio.gather(
+                    *[
+                        self.remove_worker(address=worker, close=False)
+                        for worker in missing_workers
+                    ]
+                )
                 for key, workers in missing_keys.items():
                     # Task may already be gone if it was held by a
                     # `missing_worker`
@@ -2897,7 +2905,7 @@ async def restart(self, client=None, timeout=3):
                 try:
                     # Ask the worker to close if it doesn't have a nanny,
                     # otherwise the nanny will kill it anyway
-                    self.remove_worker(address=addr, close=addr not in nannies)
+                    await self.remove_worker(address=addr, close=addr not in nannies)
                 except Exception as e:
                     logger.info(
                         "Exception while restarting.  This is normal", exc_info=True
@@ -3484,8 +3492,9 @@ async def retire_workers(
                         *[self.close_worker(worker=w, safe=True) for w in worker_keys]
                     )
                 if remove:
-                    for w in worker_keys:
-                        self.remove_worker(address=w, safe=True)
+                    await asyncio.gather(
+                        *[self.remove_worker(address=w, safe=True) for w in worker_keys]
+                    )
 
                 self.log_event(
                     "all",
@@ -5263,7 +5272,7 @@ def _reevaluate_occupancy_worker(self, ws):
                 steal.remove_key_from_stealable(ts)
                 steal.put_key_in_stealable(ts)
 
-    def check_worker_ttl(self):
+    async def check_worker_ttl(self):
         now = time()
         for ws in self.workers.values():
             if ws.last_seen < now - self.worker_ttl:
@@ -5272,7 +5281,7 @@ def check_worker_ttl(self):
                     self.worker_ttl,
                     ws,
                 )
-                self.remove_worker(address=ws.address)
+                await self.remove_worker(address=ws.address)
 
     def check_idle(self):
         if any(ws.processing for ws in self.workers.values()) or self.unrunnable:
diff --git a/distributed/stealing.py b/distributed/stealing.py
index 874ca98ce77..88c15809ea7 100644
--- a/distributed/stealing.py
+++ b/distributed/stealing.py
@@ -187,7 +187,7 @@ def move_task_request(self, ts, victim, thief):
                 pdb.set_trace()
             raise
 
-    def move_task_confirm(self, key=None, worker=None, state=None):
+    async def move_task_confirm(self, key=None, worker=None, state=None):
         try:
             try:
                 ts = self.scheduler.tasks[key]
@@ -256,7 +256,7 @@ def move_task_confirm(self, key=None, worker=None, state=None):
                 try:
                     self.scheduler.send_task_to_worker(thief.address, key)
                 except CommClosedError:
-                    self.scheduler.remove_worker(thief.address)
+                    await self.scheduler.remove_worker(thief.address)
                 self.log.append(("confirm", key, victim.address, thief.address))
             else:
                 raise ValueError("Unexpected task state: %s" % state)
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index a08df22e080..ebac68d9387 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -208,7 +208,7 @@ async def test_remove_worker_from_scheduler(s, a, b):
     )
 
     assert a.address in s.stream_comms
-    s.remove_worker(address=a.address)
+    await s.remove_worker(address=a.address)
     assert a.address not in s.nthreads
     assert len(s.workers[b.address].processing) == len(dsk)  # b owns everything
     s.validate_state()
@@ -217,9 +217,9 @@ async def test_remove_worker_from_scheduler(s, a, b):
 @gen_cluster()
 async def test_remove_worker_by_name_from_scheduler(s, a, b):
     assert a.address in s.stream_comms
-    assert s.remove_worker(address=a.name) == "OK"
+    assert await s.remove_worker(address=a.name) == "OK"
     assert a.address not in s.nthreads
-    assert s.remove_worker(address=a.address) == "already-removed"
+    assert await s.remove_worker(address=a.address) == "already-removed"
     s.validate_state()
 
 
@@ -230,7 +230,7 @@ async def test_clear_events_worker_removal(s, a, b):
     assert b.address in s.events
     assert b.address in s.nthreads
 
-    s.remove_worker(address=a.address)
+    await s.remove_worker(address=a.address)
     # Shortly after removal, the events should still be there
     assert a.address in s.events
     assert a.address not in s.nthreads
@@ -489,7 +489,7 @@ async def test_ready_remove_worker(s, a, b):
 
     assert all(len(w.processing) > w.nthreads for w in s.workers.values())
 
-    s.remove_worker(address=a.address)
+    await s.remove_worker(address=a.address)
 
     assert set(s.workers) == {b.address}
     assert all(len(w.processing) > w.nthreads for w in s.workers.values())

From 0acf1c88f1686afbf1ea32db92bc23673e08a99f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lo=C3=AFc=20Est=C3=A8ve?= <loic.esteve@ymail.com>
Date: Sun, 7 Jun 2020 01:20:21 +0200
Subject: [PATCH 0860/1550] Use SpecCluster name in worker names (#3855)

---
 distributed/deploy/spec.py                    | 16 ++++++++---
 distributed/deploy/tests/test_spec_cluster.py | 27 +++++++++++++++++++
 distributed/utils_test.py                     |  2 +-
 3 files changed, 41 insertions(+), 4 deletions(-)

diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index 56160592347..dad59da7179 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -462,6 +462,14 @@ def scale(self, n=0, memory=None, cores=None):
         if self.asynchronous:
             return NoOpAwaitable()
 
+    def _new_worker_name(self, worker_number):
+        """ Returns new worker name.
+
+        This can be overriden in SpecCluster derived classes to customise the
+        worker names.
+        """
+        return worker_number
+
     def new_worker_spec(self):
         """ Return name and spec for the next worker
 
@@ -473,10 +481,12 @@ def new_worker_spec(self):
         --------
         scale
         """
-        while self._i in self.worker_spec:
+        new_worker_name = self._new_worker_name(self._i)
+        while new_worker_name in self.worker_spec:
             self._i += 1
+            new_worker_name = self._new_worker_name(self._i)
 
-        return {self._i: self.new_spec}
+        return {new_worker_name: self.new_spec}
 
     @property
     def _supports_scaling(self):
@@ -535,7 +545,7 @@ def adapt(
         maximum_cores: int = None,
         minimum_memory: str = None,
         maximum_memory: str = None,
-        **kwargs
+        **kwargs,
     ) -> Adaptive:
         """ Turn on adaptivity
 
diff --git a/distributed/deploy/tests/test_spec_cluster.py b/distributed/deploy/tests/test_spec_cluster.py
index 4f232e27aef..12d1c221942 100644
--- a/distributed/deploy/tests/test_spec_cluster.py
+++ b/distributed/deploy/tests/test_spec_cluster.py
@@ -481,3 +481,30 @@ async def test_run_spec(cleanup):
             assert not s.workers
 
             await asyncio.gather(*[w.finished() for w in workers.values()])
+
+
+@pytest.mark.asyncio
+async def test_run_spec_cluster_worker_names(cleanup):
+    worker = {"cls": Worker, "options": {"nthreads": 1}}
+
+    class MyCluster(SpecCluster):
+        def _new_worker_name(self, worker_number):
+            return f"prefix-{self._name }-{worker_number}-suffix"
+
+    async with SpecCluster(
+        asynchronous=True, scheduler=scheduler, worker=worker
+    ) as cluster:
+        cluster.scale(2)
+        await cluster
+        worker_names = [0, 1]
+        assert list(cluster.worker_spec) == worker_names
+        assert sorted(list(cluster.workers)) == worker_names
+
+    async with MyCluster(
+        asynchronous=True, scheduler=scheduler, worker=worker, name="test-name"
+    ) as cluster:
+        worker_names = ["prefix-test-name-0-suffix", "prefix-test-name-1-suffix"]
+        cluster.scale(2)
+        await cluster
+        assert list(cluster.worker_spec) == worker_names
+        assert sorted(list(cluster.workers)) == worker_names
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index c369c95aa53..a2168fedbca 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -1447,7 +1447,7 @@ def check_process_leak(check=True):
     yield
 
     if check:
-        for i in range(100):
+        for i in range(200):
             if not set(mp_context.active_children()):
                 break
             else:

From d77e4c0f885846546821b5c1803e892bc9273f90 Mon Sep 17 00:00:00 2001
From: joshreback <joshreback@gmail.com>
Date: Sun, 7 Jun 2020 13:05:27 -0400
Subject: [PATCH 0861/1550] Validate address parameter in client constructor
 (#3842)

---
 distributed/client.py            | 8 +++++++-
 distributed/tests/test_client.py | 9 +++++++++
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/distributed/client.py b/distributed/client.py
index 170cac93228..75e1be4a03b 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -650,13 +650,19 @@ def __init__(
 
         if isinstance(address, (rpc, PooledRPCCall)):
             self.scheduler = address
-        elif hasattr(address, "scheduler_address"):
+        elif isinstance(getattr(address, "scheduler_address", None), str):
             # It's a LocalCluster or LocalCluster-compatible object
             self.cluster = address
             with suppress(AttributeError):
                 loop = address.loop
             if security is None:
                 security = getattr(self.cluster, "security", None)
+        elif address is not None and not isinstance(address, str):
+            raise TypeError(
+                "Scheduler address must be a string or a Cluster instance, got {}".format(
+                    type(address)
+                )
+            )
 
         if security is None:
             security = Security()
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index f940bf45ba3..33440318a67 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -1847,6 +1847,15 @@ def test_bad_address():
         assert "connect" in str(e).lower()
 
 
+def test_informative_error_on_cluster_type():
+    with pytest.raises(TypeError) as exc_info:
+        Client(LocalCluster)
+
+    assert "Scheduler address must be a string or a Cluster instance" in str(
+        exc_info.value
+    )
+
+
 @gen_cluster(client=True)
 async def test_long_error(c, s, a, b):
     def bad(x):

From b4c0e252f566176047e405bc102b5d0b67045f1b Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Mon, 8 Jun 2020 15:53:45 +0100
Subject: [PATCH 0862/1550] Improve SSHCluster docstring for connect_options
 (#3859)

---
 distributed/deploy/ssh.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/distributed/deploy/ssh.py b/distributed/deploy/ssh.py
index ffa9c2ad76e..ebafaf60183 100644
--- a/distributed/deploy/ssh.py
+++ b/distributed/deploy/ssh.py
@@ -263,6 +263,9 @@ def SSHCluster(
         The first will be used for the scheduler and the rest for workers.
     connect_options: dict or list of dict, optional
         Keywords to pass through to ``asyncssh.connect``.
+        This could include things such as ``port``, ``username``, ``password``
+        or ``known_hosts``. See docs for ``asyncssh.connect`` and
+        ``asyncssh.SSHClientConnectionOptions`` for full information.
         If a list it must have the same length as ``hosts``.
     worker_options: dict, optional
         Keywords to pass on to workers.

From 4c290cdc3e1798864a795d7df50f4fb4648890d6 Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Mon, 8 Jun 2020 16:55:17 +0100
Subject: [PATCH 0863/1550] Change asyncssh objects to sphinx references
 (#3861)

---
 distributed/deploy/ssh.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/distributed/deploy/ssh.py b/distributed/deploy/ssh.py
index ebafaf60183..1f35d7e9672 100644
--- a/distributed/deploy/ssh.py
+++ b/distributed/deploy/ssh.py
@@ -262,10 +262,10 @@ def SSHCluster(
         List of hostnames or addresses on which to launch our cluster.
         The first will be used for the scheduler and the rest for workers.
     connect_options: dict or list of dict, optional
-        Keywords to pass through to ``asyncssh.connect``.
+        Keywords to pass through to :func:asyncssh.connect`.
         This could include things such as ``port``, ``username``, ``password``
-        or ``known_hosts``. See docs for ``asyncssh.connect`` and
-        ``asyncssh.SSHClientConnectionOptions`` for full information.
+        or ``known_hosts``. See docs for :func:`asyncssh.connect` and
+        :class:`asyncssh.SSHClientConnectionOptions` for full information.
         If a list it must have the same length as ``hosts``.
     worker_options: dict, optional
         Keywords to pass on to workers.

From 3cc439d4244c1e6ced2abd5e7235a4d2127ef6ca Mon Sep 17 00:00:00 2001
From: Benjamin Zaitlen <quasiben@users.noreply.github.com>
Date: Mon, 8 Jun 2020 13:30:05 -0400
Subject: [PATCH 0864/1550] handle empty times in task stream (#3862)

---
 distributed/scheduler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 5a337676155..23363c891de 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -5089,7 +5089,7 @@ def profile_to_figure(state):
         total_tasks = len(task_stream)
         timespent = defaultdict(int)
         for d in task_stream:
-            for x in d["startstops"]:
+            for x in d.get("startstops", []):
                 timespent[x["action"]] += x["stop"] - x["start"]
         tasks_timings = ""
         for k in sorted(timespent.keys()):

From 754548b3fc5cd42a413365e79747b87002295eb8 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Mon, 8 Jun 2020 18:15:12 -0700
Subject: [PATCH 0865/1550] Tidy pickle (#3866)

* Assign `HIGHEST_PROTOCOL` to local variable

* Use `import pickle` instead

When using `PickleBuffer`, reference it through the `pickle` module
instead of using a `from ... import ...` to get access to it.
---
 distributed/protocol/pickle.py            |  3 ++-
 distributed/protocol/tests/test_pickle.py | 12 ++++--------
 2 files changed, 6 insertions(+), 9 deletions(-)

diff --git a/distributed/protocol/pickle.py b/distributed/protocol/pickle.py
index 9774202e4fe..6e168947d48 100644
--- a/distributed/protocol/pickle.py
+++ b/distributed/protocol/pickle.py
@@ -1,10 +1,11 @@
 import logging
 import pickle
-from pickle import HIGHEST_PROTOCOL
 
 import cloudpickle
 
 
+HIGHEST_PROTOCOL = pickle.HIGHEST_PROTOCOL
+
 logger = logging.getLogger(__name__)
 
 
diff --git a/distributed/protocol/tests/test_pickle.py b/distributed/protocol/tests/test_pickle.py
index f4a4ec7f8ee..bd784117186 100644
--- a/distributed/protocol/tests/test_pickle.py
+++ b/distributed/protocol/tests/test_pickle.py
@@ -1,6 +1,7 @@
 from functools import partial
 import gc
 from operator import add
+import pickle
 import weakref
 import sys
 
@@ -9,11 +10,6 @@
 from distributed.protocol import deserialize, serialize
 from distributed.protocol.pickle import HIGHEST_PROTOCOL, dumps, loads
 
-try:
-    from pickle import PickleBuffer
-except ImportError:
-    pass
-
 
 def test_pickle_data():
     data = [1, b"123", "123", [123], {}, set()]
@@ -29,7 +25,7 @@ def __init__(self, mv):
 
         def __reduce_ex__(self, protocol):
             if protocol >= 5:
-                return MemoryviewHolder, (PickleBuffer(self.mv),)
+                return MemoryviewHolder, (pickle.PickleBuffer(self.mv),)
             else:
                 return MemoryviewHolder, (self.mv.tobytes(),)
 
@@ -42,7 +38,7 @@ def __reduce_ex__(self, protocol):
         mvh2 = loads(d, buffers=l)
 
         assert len(l) == 1
-        assert isinstance(l[0], PickleBuffer)
+        assert isinstance(l[0], pickle.PickleBuffer)
         assert memoryview(l[0]) == mv
     else:
         mvh2 = loads(dumps(mvh))
@@ -84,7 +80,7 @@ def test_pickle_numpy():
         l = []
         d = dumps(x, buffer_callback=l.append)
         assert len(l) == 1
-        assert isinstance(l[0], PickleBuffer)
+        assert isinstance(l[0], pickle.PickleBuffer)
         assert memoryview(l[0]) == memoryview(x)
         assert (loads(d, buffers=l) == x).all()
 

From 8480b4fde57d16b27b3079f25b95e61b862b79ae Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Mon, 8 Jun 2020 18:15:28 -0700
Subject: [PATCH 0866/1550] Fix typo in `feed`'s log message (#3867)

---
 distributed/scheduler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 23363c891de..0f3d99b05f7 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -3606,7 +3606,7 @@ async def feed(
         """
         if not dask.config.get("distributed.scheduler.pickle"):
             logger.warn(
-                "Tried to call 'feed' route with custom fucntions, but "
+                "Tried to call 'feed' route with custom functions, but "
                 "pickle is disallowed.  Set the 'distributed.scheduler.pickle'"
                 "config value to True to use the 'feed' route (this is mostly "
                 "commonly used with progress bars)"

From 093b609685734d9b8ac16144e501dc61e644226d Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Tue, 9 Jun 2020 06:59:13 -0700
Subject: [PATCH 0867/1550] Drop no longer needed local `import pickle` (#3865)

We already `import pickle` at the top of the module. So there is no need
to import it locally here as well. So drop the local `import pickle`.
---
 distributed/scheduler.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 0f3d99b05f7..238e1a361b0 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -3612,7 +3612,6 @@ async def feed(
                 "commonly used with progress bars)"
             )
             return
-        import pickle
 
         interval = parse_timedelta(interval)
         with log_errors():

From ecba43ce05c4df0eccb2cae7273958dbeccf038d Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Tue, 9 Jun 2020 07:03:26 -0700
Subject: [PATCH 0868/1550] Drop old frame splitting in NumPy serialization
 (#3868)

* Drop old frame splitting in NumPy serialization

This is leftover from when Blosc compression was still being handled
with the NumPy serialization code path itself. These days frame
splitting already happens as part of serialization more generally before
compressing data. Also this cutoff is not relevant for when splitting
occurs as that is determined by a limit in the size of buffers Blosc can
handle, which `frame_split_size` already knows. So go ahead and drop
this special casing in NumPy serialization and rely on these commonly
shared code paths that all serialization passes through.

* Drop `merge_frames` in NumPy deserialization

Since the frames are not being split in serialization and other common
code paths to all deserialization already make sure frames match their
expected length, there should be no need to merge frames in NumPy
deserialization. So drop this as well.

* Unpack `frames` in NumPy deserialization

To simplify the code a bit and assert the number of frames is exactly 1,
unpack `frames` into a single `frame` variable.
---
 distributed/protocol/numpy.py | 13 ++++---------
 1 file changed, 4 insertions(+), 9 deletions(-)

diff --git a/distributed/protocol/numpy.py b/distributed/protocol/numpy.py
index a0df77c8b37..2140c2f0c4e 100644
--- a/distributed/protocol/numpy.py
+++ b/distributed/protocol/numpy.py
@@ -1,7 +1,6 @@
 import math
 import numpy as np
 
-from .utils import frame_split_size, merge_frames
 from .serialize import dask_serialize, dask_deserialize
 from . import pickle
 
@@ -87,10 +86,7 @@ def serialize_numpy_ndarray(x):
     if broadcast_to is not None:
         header["broadcast_to"] = broadcast_to
 
-    if x.nbytes > 1e5:
-        frames = frame_split_size(data)
-    else:
-        frames = [data]
+    frames = [data]
 
     header["lengths"] = [x.nbytes]
 
@@ -100,11 +96,10 @@ def serialize_numpy_ndarray(x):
 @dask_deserialize.register(np.ndarray)
 def deserialize_numpy_ndarray(header, frames):
     with log_errors():
-        if len(frames) > 1:
-            frames = merge_frames(header, frames)
+        (frame,) = frames
 
         if header.get("pickle"):
-            return pickle.loads(frames[0])
+            return pickle.loads(frame)
 
         is_custom, dt = header["dtype"]
         if is_custom:
@@ -117,7 +112,7 @@ def deserialize_numpy_ndarray(header, frames):
         else:
             shape = header["shape"]
 
-        x = np.ndarray(shape, dtype=dt, buffer=frames[0], strides=header["strides"])
+        x = np.ndarray(shape, dtype=dt, buffer=frame, strides=header["strides"])
 
         return x
 

From 4f6e354bc83fd783bb9a0b7fe44386d4847eef4a Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Tue, 9 Jun 2020 07:04:06 -0700
Subject: [PATCH 0869/1550] Drop legacy `buffer_interface` assignment (#3869)

This was part of some Python 2 compatibility code at one point. However
as Python 2 has long since been dropped, we are safe to drop this as
well.
---
 distributed/protocol/tests/test_numpy.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/distributed/protocol/tests/test_numpy.py b/distributed/protocol/tests/test_numpy.py
index 70ee582fd70..830991fd56a 100644
--- a/distributed/protocol/tests/test_numpy.py
+++ b/distributed/protocol/tests/test_numpy.py
@@ -77,9 +77,8 @@ def test_dumps_serialize_numpy(x):
     header, frames = serialize(x)
     if "compression" in header:
         frames = decompress(header, frames)
-    buffer_interface = memoryview
     for frame in frames:
-        assert isinstance(frame, (bytes, buffer_interface))
+        assert isinstance(frame, (bytes, memoryview))
     y = deserialize(header, frames)
 
     np.testing.assert_equal(x, y)

From 1408ab59e691378f5a5c0299c8ac0bb4f42d4b22 Mon Sep 17 00:00:00 2001
From: Matthias Bussonnier <bussonniermatthias@gmail.com>
Date: Tue, 9 Jun 2020 07:47:39 -0700
Subject: [PATCH 0870/1550] Use Enum for worker, scheduler and nanny status.
 (#3853)

Looking at the code it was hard to find all the states a worker, nanny
scheduler could be
in, and in particular it is easy to forget 'closing_gracefully' for
example when skimming over a `self.status.startswith('clos')`, and it
also hard to fine that in some places the status can be "None", "init",
"starting" and "stopped"

So I just took a pass at seeing in how many places status was
assigned/compared in worker, nanny and scheduler and replace that by an
Enum I'm assuming the status each can see is not shared  so we end up

I thus tried to do separate Status Enums for Scheduler/Nanny/Worker.

Note that even if Enums have values for their variants, the variants _do
not_ compare equal to their values. That is to say `Status.running =!
'running'`, but `Status.running.value == 'running'`,

So I implemented a custom `__eq__` method that can compare with
strings/None for backward compatibility. It works both when the LHS or
RHS is str/None as the default impl of str/None forward to our
implementation. This also make sure the value you compare with is also
in the list of possible state.

The main other code changes were updating some of the comparisons to use
the enums instead of the strings directly, and replace the "startswith"
with the `in` operator, which is slightly more verbose.

I believe despite being a bit more verbose, it is a bit more explicit and
may help navigate the codebase and if you want to go this route you
probably want to remove the custom comparison but that would be an API
change that could break other project.

* Consolidate all Status in core.

Status get compared with each other and we likely want to consolidate
all the status.

* add properties to check the states.
---
 distributed/core.py                 | 76 ++++++++++++++++++++++++-----
 distributed/deploy/spec.py          |  6 +--
 distributed/nanny.py                | 55 +++++++++++----------
 distributed/scheduler.py            | 55 +++++++++++++++------
 distributed/tests/test_scheduler.py |  7 ++-
 distributed/utils_test.py           | 10 ++--
 distributed/worker.py               | 40 ++++++++-------
 7 files changed, 172 insertions(+), 77 deletions(-)

diff --git a/distributed/core.py b/distributed/core.py
index 69f177c6eea..250ea076e03 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -1,6 +1,7 @@
 import asyncio
 from collections import defaultdict, deque
 from contextlib import suppress
+from enum import Enum
 from functools import partial
 import inspect
 import logging
@@ -39,6 +40,46 @@
 from . import protocol
 
 
+class Status(Enum):
+    """
+    This Enum contains the various states a worker, scheduler and nanny can be
+    in. Some of the status can only be observed in one of nanny, scheduler or
+    worker but we put them in the same Enum as they are compared with each
+    other.
+    """
+
+    closed = "closed"
+    closing = "closing"
+    closing_gracefully = "closing-gracefully"
+    init = "init"
+    running = "running"
+    starting = "starting"
+    stopped = "stopped"
+    stopping = "stopping"
+    undefined = None
+
+    def __eq__(self, other):
+        """
+        Implement equality checking with backward compatibility.
+
+        If other object instance is string, we compare with the values, but we
+        actually want to make sure the value compared with is in the list of
+        possible Status, this avoid comparison with non-existing status.
+        """
+        if isinstance(other, type(self)):
+            return self.value == other.value
+        elif isinstance(other, str) or (other is None):
+            assert other in [
+                s.value for s in type(self)
+            ], f"comparison with non-existing states {other}"
+            return other == self.value
+        raise TypeError(
+            f"'==' not supported between instances of"
+            f" {type(self).__module__+'.'+type(self).__qualname__!r} and"
+            f" {type(other).__module__+'.'+type(other).__qualname__!r}"
+        )
+
+
 class RPCClosed(IOError):
     pass
 
@@ -197,7 +238,7 @@ def set_thread_ident():
 
         self.io_loop.add_callback(set_thread_ident)
         self._startup_lock = asyncio.Lock()
-        self.status = None
+        self.status = Status.undefined
 
         self.rpc = ConnectionPool(
             limit=connection_limit,
@@ -211,6 +252,19 @@ def set_thread_ident():
 
         self.__stopped = False
 
+    @property
+    def status(self):
+        return self._status
+
+    @status.setter
+    def status(self, new_status):
+        if isinstance(new_status, Status):
+            self._status = new_status
+        elif isinstance(new_status, str) or new_status is None:
+            corresponding_enum_variants = [s for s in Status if s.value == new_status]
+            assert len(corresponding_enum_variants) == 1
+            self._status = corresponding_enum_variants[0]
+
     async def finished(self):
         """ Wait until the server has finished """
         await self._event_finished.wait()
@@ -219,12 +273,12 @@ def __await__(self):
         async def _():
             timeout = getattr(self, "death_timeout", 0)
             async with self._startup_lock:
-                if self.status == "running":
+                if self.status == Status.running:
                     return self
                 if timeout:
                     try:
                         await asyncio.wait_for(self.start(), timeout=timeout)
-                        self.status = "running"
+                        self.status = Status.running
                     except Exception:
                         await self.close(timeout=1)
                         raise TimeoutError(
@@ -234,7 +288,7 @@ async def _():
                         )
                 else:
                     await self.start()
-                    self.status = "running"
+                    self.status = Status.running
             return self
 
         return _().__await__()
@@ -458,7 +512,7 @@ async def handle_comm(self, comm, shutting_down=shutting_down):
                             self._ongoing_coroutines.add(result)
                             result = await result
                     except (CommClosedError, CancelledError) as e:
-                        if self.status == "running":
+                        if self.status == Status.running:
                             logger.info("Lost connection to %r: %s", address, e)
                         break
                     except Exception as e:
@@ -651,7 +705,7 @@ def __init__(
         self.comms = {}
         self.address = coerce_to_address(arg)
         self.timeout = timeout
-        self.status = "running"
+        self.status = Status.running
         self.deserialize = deserialize
         self.serializers = serializers
         self.deserializers = deserializers if deserializers is not None else serializers
@@ -677,7 +731,7 @@ async def live_comm(self):
 
         As is done in __getattr__ below.
         """
-        if self.status == "closed":
+        if self.status == Status.closed:
             raise RPCClosed("RPC Closed")
         to_clear = set()
         open = False
@@ -745,9 +799,9 @@ async def send_recv_from_rpc(**kwargs):
         return send_recv_from_rpc
 
     def close_rpc(self):
-        if self.status != "closed":
+        if self.status != Status.closed:
             rpc.active.discard(self)
-        self.status = "closed"
+        self.status = Status.closed
         return asyncio.gather(*self.close_comms())
 
     def __enter__(self):
@@ -763,9 +817,9 @@ async def __aexit__(self, *args):
         await self.close_rpc()
 
     def __del__(self):
-        if self.status != "closed":
+        if self.status != Status.closed:
             rpc.active.discard(self)
-            self.status = "closed"
+            self.status = Status.closed
             still_open = [comm for comm in self.comms if not comm.closed()]
             if still_open:
                 logger.warning(
diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index dad59da7179..93cf75bd295 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -11,7 +11,7 @@
 
 from .adaptive import Adaptive
 from .cluster import Cluster
-from ..core import rpc, CommClosedError
+from ..core import rpc, CommClosedError, Status
 from ..utils import (
     LoopRunner,
     silence_logging,
@@ -304,7 +304,7 @@ async def _correct_state_internal(self):
             pre = list(set(self.workers))
             to_close = set(self.workers) - set(self.worker_spec)
             if to_close:
-                if self.scheduler.status == "running":
+                if self.scheduler.status == Status.running:
                     await self.scheduler_comm.retire_workers(workers=list(to_close))
                 tasks = [self.workers[w].close() for w in to_close if w in self.workers]
                 await asyncio.wait(tasks)
@@ -390,7 +390,7 @@ async def _close(self):
 
         await self.scheduler.close()
         for w in self._created:
-            assert w.status == "closed", w.status
+            assert w.status == Status.closed, w.status
 
         if hasattr(self, "_old_logging_level"):
             silence_logging(self._old_logging_level)
diff --git a/distributed/nanny.py b/distributed/nanny.py
index b09aa1c2a72..a2d1e5772b5 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -18,7 +18,7 @@
 
 from .comm import get_address_host, unparse_host_port
 from .comm.addressing import address_from_user_args
-from .core import RPCClosed, CommClosedError, coerce_to_address
+from .core import RPCClosed, CommClosedError, coerce_to_address, Status
 from .metrics import time
 from .node import ServerNode
 from . import preloading
@@ -36,6 +36,7 @@
 )
 from .worker import run, parse_memory_limit, Worker
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -56,7 +57,7 @@ class Nanny(ServerNode):
 
     _instances = weakref.WeakSet()
     process = None
-    status = None
+    status = Status.undefined
 
     def __init__(
         self,
@@ -218,7 +219,7 @@ def __init__(
 
         self._listen_address = listen_address
         Nanny._instances.add(self)
-        self.status = "init"
+        self.status = Status.init
 
     def __repr__(self):
         return "<Nanny: %s, threads: %d>" % (self.worker_address, self.nthreads)
@@ -291,7 +292,7 @@ async def start(self):
         response = await self.instantiate()
         if response == "running":
             assert self.worker_address
-            self.status = "running"
+            self.status = Status.running
         else:
             await self.close()
 
@@ -405,7 +406,7 @@ def _psutil_process(self):
 
     def memory_monitor(self):
         """ Track worker's memory.  Restart if it goes above terminate fraction """
-        if self.status != "running":
+        if self.status != Status.running:
             return
         process = self.process.process
         if process is None:
@@ -434,7 +435,7 @@ def _on_exit_sync(self, exitcode):
         self.loop.add_callback(self._on_exit, exitcode)
 
     async def _on_exit(self, exitcode):
-        if self.status not in ("closing", "closed"):
+        if self.status not in (Status.closing, Status.closed):
             try:
                 await self.scheduler.unregister(address=self.worker_address)
             except (EnvironmentError, CommClosedError):
@@ -443,11 +444,15 @@ async def _on_exit(self, exitcode):
                     return
 
             try:
-                if self.status not in ("closing", "closed", "closing-gracefully"):
+                if self.status not in (
+                    Status.closing,
+                    Status.closed,
+                    Status.closing_gracefully,
+                ):
                     if self.auto_restart:
                         logger.warning("Restarting worker")
                         await self.instantiate()
-                elif self.status == "closing-gracefully":
+                elif self.status == Status.closing_gracefully:
                     await self.close()
 
             except Exception:
@@ -469,20 +474,20 @@ def close_gracefully(self, comm=None):
 
         This is used as part of the cluster shutdown process.
         """
-        self.status = "closing-gracefully"
+        self.status = Status.closing_gracefully
 
     async def close(self, comm=None, timeout=5, report=None):
         """
         Close the worker process, stop all comms.
         """
-        if self.status == "closing":
+        if self.status == Status.closing:
             await self.finished()
-            assert self.status == "closed"
+            assert self.status == Status.closed
 
-        if self.status == "closed":
+        if self.status == Status.closed:
             return "OK"
 
-        self.status = "closing"
+        self.status = Status.closing
         logger.info("Closing Nanny at %r", self.address)
 
         for preload in self.preloads:
@@ -496,7 +501,7 @@ async def close(self, comm=None, timeout=5, report=None):
             pass
         self.process = None
         await self.rpc.close()
-        self.status = "closed"
+        self.status = Status.closed
         if comm:
             await comm.write("OK")
         await ServerNode.close(self)
@@ -513,7 +518,7 @@ def __init__(
         env,
         config,
     ):
-        self.status = "init"
+        self.status = Status.init
         self.silence_logs = silence_logs
         self.worker_kwargs = worker_kwargs
         self.worker_start_args = worker_start_args
@@ -532,9 +537,9 @@ async def start(self):
         Ensure the worker process is started.
         """
         enable_proctitle_on_children()
-        if self.status == "running":
+        if self.status == Status.running:
             return self.status
-        if self.status == "starting":
+        if self.status == Status.starting:
             await self.running.wait()
             return self.status
 
@@ -561,7 +566,7 @@ async def start(self):
         self.process.set_exit_callback(self._on_exit)
         self.running = asyncio.Event()
         self.stopped = asyncio.Event()
-        self.status = "starting"
+        self.status = Status.starting
 
         try:
             await self.process.start()
@@ -606,13 +611,13 @@ def pid(self):
         return self.process.pid if self.process and self.process.is_alive() else None
 
     def mark_stopped(self):
-        if self.status != "stopped":
+        if self.status != Status.stopped:
             r = self.process.exitcode
             assert r is not None
             if r != 0:
                 msg = self._death_message(self.process.pid, r)
                 logger.info(msg)
-            self.status = "stopped"
+            self.status = Status.stopped
             self.stopped.set()
             # Release resources
             self.process.close()
@@ -635,13 +640,13 @@ async def kill(self, timeout=2, executor_wait=True):
         loop = IOLoop.current()
         deadline = loop.time() + timeout
 
-        if self.status == "stopped":
+        if self.status == Status.stopped:
             return
-        if self.status == "stopping":
+        if self.status == Status.stopping:
             await self.stopped.wait()
             return
-        assert self.status in ("starting", "running")
-        self.status = "stopping"
+        assert self.status in (Status.starting, Status.running)
+        self.status = Status.stopping
 
         process = self.process
         self.child_stop_q.put(
@@ -669,7 +674,7 @@ async def kill(self, timeout=2, executor_wait=True):
     async def _wait_until_connected(self, uid):
         delay = 0.05
         while True:
-            if self.status != "starting":
+            if self.status != Status.starting:
                 return
             try:
                 msg = self.init_result_q.get_nowait()
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 238e1a361b0..a50407f7966 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -46,7 +46,7 @@
     unparse_host_port,
 )
 from .comm.addressing import addresses_from_user_args
-from .core import rpc, send_recv, clean_exception, CommClosedError
+from .core import rpc, send_recv, clean_exception, CommClosedError, Status
 from .diagnostics.plugin import SchedulerPlugin
 
 from .http import get_handlers
@@ -245,7 +245,7 @@ class WorkerState:
         "processing",
         "resources",
         "services",
-        "status",
+        "_status",
         "time_delay",
         "used_resources",
         "versions",
@@ -274,7 +274,7 @@ def __init__(
         self.versions = versions or {}
         self.nanny = nanny
 
-        self.status = "running"
+        self._status = Status.running
         self.nbytes = 0
         self.occupancy = 0
         self.metrics = {}
@@ -296,6 +296,19 @@ def __hash__(self):
     def __eq__(self, other):
         return type(self) == type(other) and self.address == other.address
 
+    @property
+    def status(self):
+        return self._status
+
+    @status.setter
+    def status(self, new_status):
+        if isinstance(new_status, Status):
+            self._status = new_status
+        elif isinstance(new_status, str) or new_status is None:
+            corresponding_enum_variants = [s for s in Status if s.value == new_status]
+            assert len(corresponding_enum_variants) == 1
+            self._status = corresponding_enum_variants[0]
+
     @property
     def host(self):
         return get_address_host(self.address)
@@ -1377,6 +1390,20 @@ def __init__(
         setproctitle("dask-scheduler [not started]")
         Scheduler._instances.add(self)
         self.rpc.allow_offload = False
+        self.status = Status.undefined
+
+    @property
+    def status(self):
+        return self._status
+
+    @status.setter
+    def status(self, new_status):
+        if isinstance(new_status, Status):
+            self._status = new_status
+        elif isinstance(new_status, str) or new_status is None:
+            corresponding_enum_variants = [s for s in Status if s.value == new_status]
+            assert len(corresponding_enum_variants) == 1
+            self._status = corresponding_enum_variants[0]
 
     ##################
     # Administration #
@@ -1432,7 +1459,7 @@ def get_worker_service_addr(self, worker, service_name, protocol=False):
     async def start(self):
         """ Clear out old state and restart all running coroutines """
         await super().start()
-        assert self.status != "running"
+        assert self.status != Status.running
 
         enable_gc_diagnosis()
 
@@ -1494,10 +1521,10 @@ async def close(self, comm=None, fast=False, close_workers=False):
         --------
         Scheduler.cleanup
         """
-        if self.status.startswith("clos"):
+        if self.status in (Status.closing, Status.closed, Status.closing_gracefully):
             await self.finished()
             return
-        self.status = "closing"
+        self.status = Status.closing
 
         logger.info("Scheduler closing...")
         setproctitle("dask-scheduler [closing]")
@@ -1544,7 +1571,7 @@ async def close(self, comm=None, fast=False, close_workers=False):
 
         await self.rpc.close()
 
-        self.status = "closed"
+        self.status = Status.closed
         self.stop()
         await super(Scheduler, self).close()
 
@@ -2170,7 +2197,7 @@ async def remove_worker(self, comm=None, address=None, safe=False, close=True):
         state.
         """
         with log_errors():
-            if self.status == "closed":
+            if self.status == Status.closed:
                 return
 
             address = self.coerce_address(address)
@@ -2507,7 +2534,7 @@ def report(self, msg, ts=None, client=None):
                 c.send(msg)
                 # logger.debug("Scheduler sends message to client %s", msg)
             except CommClosedError:
-                if self.status == "running":
+                if self.status == Status.running:
                     logger.critical("Tried writing to closed comm: %s", msg)
 
     async def add_client(self, comm, client=None, versions=None):
@@ -2553,14 +2580,14 @@ async def add_client(self, comm, client=None, versions=None):
                 if not shutting_down():
                     await self.client_comms[client].close()
                     del self.client_comms[client]
-                    if self.status == "running":
+                    if self.status == Status.running:
                         logger.info("Close client connection: %s", client)
             except TypeError:  # comm becomes None during GC
                 pass
 
     def remove_client(self, client=None):
         """ Remove client from network """
-        if self.status == "running":
+        if self.status == Status.running:
             logger.info("Remove client %s", client)
         self.log_event(["all", client], {"action": "remove-client", "client": client})
         try:
@@ -3625,7 +3652,7 @@ async def feed(
             if inspect.isawaitable(state):
                 state = await state
             try:
-                while self.status == "running":
+                while self.status == Status.running:
                     if state is None:
                         response = function(self)
                     else:
@@ -4799,7 +4826,7 @@ def check_idle_saturated(self, ws, occ=None):
 
         This is useful for load balancing and adaptivity.
         """
-        if self.total_nthreads == 0 or ws.status == "closed":
+        if self.total_nthreads == 0 or ws.status == Status.closed:
             return
         if occ is None:
             occ = ws.occupancy
@@ -5217,7 +5244,7 @@ def reevaluate_occupancy(self, worker_index=0):
         """
         DELAY = 0.1
         try:
-            if self.status == "closed":
+            if self.status == Status.closed:
                 return
 
             last = time()
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index ebac68d9387..a8cdc18bd15 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -17,7 +17,7 @@
 
 from distributed import Nanny, Worker, Client, wait, fire_and_forget
 from distributed.comm import Comm
-from distributed.core import connect, rpc, ConnectionPool
+from distributed.core import connect, rpc, ConnectionPool, Status
 from distributed.scheduler import Scheduler
 from distributed.client import wait
 from distributed.metrics import time
@@ -736,7 +736,10 @@ async def test_retire_workers_n(c, s, a, b):
     await s.retire_workers(n=0, close_workers=True)
     assert len(s.workers) == 0
 
-    while not (a.status.startswith("clos") and b.status.startswith("clos")):
+    while not (
+        a.status in (Status.closed, Status.closing, Status.closing_gracefully)
+        and b.status in (Status.closed, Status.closing, Status.closing_gracefully)
+    ):
         await asyncio.sleep(0.01)
 
 
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index a2168fedbca..ab1992b5ad9 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -41,7 +41,7 @@
 from .compatibility import WINDOWS
 from .comm import Comm
 from .config import initialize_logging
-from .core import connect, rpc, CommClosedError
+from .core import connect, rpc, CommClosedError, Status
 from .deploy import SpecCluster
 from .metrics import time
 from .process import _cleanup_dangling
@@ -1485,7 +1485,7 @@ def check_instances():
     for w in Worker._instances:
         with suppress(RuntimeError):  # closed IOLoop
             w.loop.add_callback(w.close, report=False, executor_wait=False)
-            if w.status == "running":
+            if w.status == Status.running:
                 w.loop.add_callback(w.close)
     Worker._instances.clear()
 
@@ -1500,9 +1500,9 @@ def check_instances():
         print("Unclosed Comms", L)
         # raise ValueError("Unclosed Comms", L)
 
-    assert all(n.status == "closed" or n.status == "init" for n in Nanny._instances), {
-        n: n.status for n in Nanny._instances
-    }
+    assert all(
+        n.status == Status.closed or n.status == Status.init for n in Nanny._instances
+    ), {n: n.status for n in Nanny._instances}
 
     # assert not list(SpecCluster._instances)  # TODO
     assert all(c.status == "closed" for c in SpecCluster._instances), list(
diff --git a/distributed/worker.py b/distributed/worker.py
index 8dff85e653c..8b4135eb5b8 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -68,6 +68,8 @@
 from .utils_perf import ThrottledGC, enable_gc_diagnosis, disable_gc_diagnosis
 from .versions import get_versions
 
+from .core import Status
+
 logger = logging.getLogger(__name__)
 
 LOG_PDB = dask.config.get("distributed.admin.pdb-on-err")
@@ -832,7 +834,7 @@ async def _register_with_scheduler(self):
                 middle = (_start + _end) / 2
                 self._update_latency(_end - start)
                 self.scheduler_delay = response["time"] - middle
-                self.status = "running"
+                self.status = Status.running
                 break
             except EnvironmentError:
                 logger.info("Waiting to connect to: %26s", self.scheduler.address)
@@ -881,7 +883,7 @@ async def heartbeat(self):
 
                 if response["status"] == "missing":
                     for i in range(10):
-                        if self.status != "running":
+                        if self.status != Status.running:
                             break
                         else:
                             await asyncio.sleep(0.05)
@@ -918,7 +920,7 @@ async def handle_scheduler(self, comm):
             logger.exception(e)
             raise
         finally:
-            if self.reconnect and self.status == "running":
+            if self.reconnect and self.status == Status.running:
                 logger.info("Connection to scheduler broken.  Reconnecting...")
                 self.loop.add_callback(self.heartbeat)
             else:
@@ -991,9 +993,13 @@ async def gather(self, comm=None, who_has=None):
     #############
 
     async def start(self):
-        if self.status and self.status.startswith("clos"):
+        if self.status and self.status in (
+            Status.closed,
+            Status.closing,
+            Status.closing_gracefully,
+        ):
             return
-        assert self.status is None, self.status
+        assert self.status is Status.undefined, self.status
 
         await super().start()
 
@@ -1095,7 +1101,7 @@ async def close(
         self, report=True, timeout=10, nanny=True, executor_wait=True, safe=False
     ):
         with log_errors():
-            if self.status in ("closed", "closing"):
+            if self.status in (Status.closed, Status.closing):
                 await self.finished()
                 return
 
@@ -1106,9 +1112,9 @@ async def close(
                 logger.info("Stopping worker at %s", self.address)
             except ValueError:  # address not available if already closed
                 logger.info("Stopping worker")
-            if self.status not in ("running", "closing-gracefully"):
+            if self.status not in (Status.running, Status.closing_gracefully):
                 logger.info("Closed worker has not yet started: %s", self.status)
-            self.status = "closing"
+            self.status = Status.closing
 
             for preload in self.preloads:
                 await preload.teardown()
@@ -1164,7 +1170,7 @@ async def close(
             self.stop()
             await self.rpc.close()
 
-            self.status = "closed"
+            self.status = Status.closed
             await ServerNode.close(self)
 
             setproctitle("dask-worker [closed]")
@@ -1176,14 +1182,14 @@ async def close_gracefully(self):
         This first informs the scheduler that we're shutting down, and asks it
         to move our data elsewhere.  Afterwards, we close as normal
         """
-        if self.status.startswith("closing"):
+        if self.status in (Status.closing, Status.closing_gracefully):
             await self.finished()
 
-        if self.status == "closed":
+        if self.status == Status.closed:
             return
 
         logger.info("Closing worker gracefully: %s", self.address)
-        self.status = "closing-gracefully"
+        self.status = Status.closing_gracefully
         await self.scheduler.retire_workers(workers=[self.address], remove=False)
         await self.close(safe=True, nanny=not self.lifetime_restart)
 
@@ -1194,7 +1200,7 @@ async def terminate(self, comm=None, report=True, **kwargs):
     async def wait_until_closed(self):
         warnings.warn("wait_until_closed has moved to finished()")
         await self.finished()
-        assert self.status == "closed"
+        assert self.status == Status.closed
 
     ################
     # Worker Peers #
@@ -1707,7 +1713,7 @@ def transition_executing_done(self, key, value=no_value, report=True):
                 if key in self.dep_state:
                     self.transition_dep(key, "memory")
 
-            if report and self.batched_stream and self.status == "running":
+            if report and self.batched_stream and self.status == Status.running:
                 self.send_task_state_to_scheduler(key)
             else:
                 raise CommClosedError
@@ -1960,7 +1966,7 @@ def select_keys_for_gather(self, worker, dep):
         return deps, total_bytes
 
     async def gather_dep(self, worker, dep, deps, total_nbytes, cause=None):
-        if self.status != "running":
+        if self.status != Status.running:
             return
         with log_errors():
             response = {}
@@ -2493,7 +2499,7 @@ def ensure_computing(self):
 
     async def execute(self, key, report=False):
         executor_error = None
-        if self.status in ("closing", "closed", "closing-gracefully"):
+        if self.status in (Status.closing, Status.closed, Status.closing_gracefully):
             return
         try:
             if key not in self.executing or key not in self.task_state:
@@ -2928,7 +2934,7 @@ def validate_dep(self, dep):
             raise
 
     def validate_state(self):
-        if self.status != "running":
+        if self.status != Status.running:
             return
         try:
             for key, workers in self.who_has.items():

From df4206365bbb26196f7349b89bc7eccf3cc5de80 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Tue, 9 Jun 2020 10:47:28 -0700
Subject: [PATCH 0871/1550] Cast any `frame` to `uint8` (same type as `bytes`)
 (#3870)

---
 distributed/protocol/core.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/protocol/core.py b/distributed/protocol/core.py
index 0947b3a6292..c76d84e8a97 100644
--- a/distributed/protocol/core.py
+++ b/distributed/protocol/core.py
@@ -81,7 +81,7 @@ def dumps(msg, serializers=None, on_error="message", context=None):
         for i, frame in enumerate(out_frames):
             if type(frame) is memoryview and frame.strides != (1,):
                 try:
-                    frame = frame.cast("b")
+                    frame = frame.cast("B")
                 except TypeError:
                     frame = frame.tobytes()
                 out_frames[i] = frame

From 6ee7f899d1bc686ba4910c08509c7f2e5345506d Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Tue, 9 Jun 2020 16:22:55 -0700
Subject: [PATCH 0872/1550] Use pickle protocol 5 with NumPy object arrays
 (#3871)

---
 distributed/protocol/numpy.py             | 13 ++++++++-----
 distributed/protocol/tests/test_numpy.py  | 16 +++++++++++++++-
 distributed/protocol/tests/test_pickle.py | 19 +++++++++++++++++++
 3 files changed, 42 insertions(+), 6 deletions(-)

diff --git a/distributed/protocol/numpy.py b/distributed/protocol/numpy.py
index 2140c2f0c4e..caeca147169 100644
--- a/distributed/protocol/numpy.py
+++ b/distributed/protocol/numpy.py
@@ -4,7 +4,7 @@
 from .serialize import dask_serialize, dask_deserialize
 from . import pickle
 
-from ..utils import log_errors
+from ..utils import log_errors, nbytes
 
 
 def itemsize(dt):
@@ -22,7 +22,10 @@ def itemsize(dt):
 def serialize_numpy_ndarray(x):
     if x.dtype.hasobject:
         header = {"pickle": True}
-        frames = [pickle.dumps(x)]
+        frames = [None]
+        buffer_callback = lambda f: frames.append(memoryview(f))
+        frames[0] = pickle.dumps(x, buffer_callback=buffer_callback)
+        header["lengths"] = tuple(map(nbytes, frames))
         return header, frames
 
     # We cannot blindly pickle the dtype as some may fail pickling,
@@ -96,10 +99,10 @@ def serialize_numpy_ndarray(x):
 @dask_deserialize.register(np.ndarray)
 def deserialize_numpy_ndarray(header, frames):
     with log_errors():
-        (frame,) = frames
-
         if header.get("pickle"):
-            return pickle.loads(frame)
+            return pickle.loads(frames[0], buffers=frames[1:])
+
+        (frame,) = frames
 
         is_custom, dt = header["dtype"]
         if is_custom:
diff --git a/distributed/protocol/tests/test_numpy.py b/distributed/protocol/tests/test_numpy.py
index 830991fd56a..0e299632902 100644
--- a/distributed/protocol/tests/test_numpy.py
+++ b/distributed/protocol/tests/test_numpy.py
@@ -14,6 +14,7 @@
 )
 from distributed.protocol.utils import BIG_BYTES_SHARD_SIZE
 from distributed.protocol.numpy import itemsize
+from distributed.protocol.pickle import HIGHEST_PROTOCOL
 from distributed.protocol.compression import maybe_compress
 from distributed.system import MEMORY_LIMIT
 from distributed.utils import tmpfile, nbytes
@@ -57,6 +58,7 @@ def test_serialize():
         np.array(["abc"], dtype="S3"),
         np.array(["abc"], dtype="U3"),
         np.array(["abc"], dtype=object),
+        np.array([np.arange(3), np.arange(4, 6)], dtype=object),
         np.ones(shape=(5,), dtype=("f8", 32)),
         np.ones(shape=(5,), dtype=[("x", "f8", 32)]),
         np.ones(shape=(5,), dtype=np.dtype([("a", "i1"), ("b", "f8")], align=False)),
@@ -79,12 +81,24 @@ def test_dumps_serialize_numpy(x):
         frames = decompress(header, frames)
     for frame in frames:
         assert isinstance(frame, (bytes, memoryview))
+    if x.dtype.char == "O" and any(isinstance(e, np.ndarray) for e in x.flat):
+        if HIGHEST_PROTOCOL >= 5:
+            assert len(frames) > 1
+        else:
+            assert len(frames) == 1
     y = deserialize(header, frames)
 
-    np.testing.assert_equal(x, y)
+    assert x.shape == y.shape
+    assert x.dtype == y.dtype
     if x.flags.c_contiguous or x.flags.f_contiguous:
         assert x.strides == y.strides
 
+    if x.dtype.char == "O":
+        for e_x, e_y in zip(x.flat, y.flat):
+            np.testing.assert_equal(e_x, e_y)
+    else:
+        np.testing.assert_equal(x, y)
+
 
 @pytest.mark.parametrize(
     "x",
diff --git a/distributed/protocol/tests/test_pickle.py b/distributed/protocol/tests/test_pickle.py
index bd784117186..9ee496f5e9f 100644
--- a/distributed/protocol/tests/test_pickle.py
+++ b/distributed/protocol/tests/test_pickle.py
@@ -74,6 +74,25 @@ def test_pickle_numpy():
     assert (loads(dumps(x)) == x).all()
     assert (deserialize(*serialize(x, serializers=("pickle",))) == x).all()
 
+    x = np.array([np.arange(3), np.arange(4, 6)], dtype=object)
+    x2 = loads(dumps(x))
+    assert x.shape == x2.shape
+    assert x.dtype == x2.dtype
+    assert x.strides == x2.strides
+    for e_x, e_x2 in zip(x.flat, x2.flat):
+        np.testing.assert_equal(e_x, e_x2)
+    h, f = serialize(x, serializers=("pickle",))
+    if HIGHEST_PROTOCOL >= 5:
+        assert len(f) == 3
+    else:
+        assert len(f) == 1
+    x3 = deserialize(h, f)
+    assert x.shape == x3.shape
+    assert x.dtype == x3.dtype
+    assert x.strides == x3.strides
+    for e_x, e_x3 in zip(x.flat, x3.flat):
+        np.testing.assert_equal(e_x, e_x3)
+
     if HIGHEST_PROTOCOL >= 5:
         x = np.ones(5000)
 

From f2b09c6b55188f6b633cba5422454b689b0fd7b4 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 10 Jun 2020 12:14:01 -0700
Subject: [PATCH 0873/1550] Allow dictionaries in security= keywords (#3874)

Previously Dask Server and Client objects had to be instantiated with a
Security object for security.  Now we also allow for dictionary inputs
that will be splatted out to the Security object.  This helps when using
systems like dask-spec with Security.
---
 distributed/cli/dask_scheduler.py        | 22 ++++++------
 distributed/cli/dask_worker.py           | 22 ++++++------
 distributed/client.py                    |  2 ++
 distributed/nanny.py                     |  3 ++
 distributed/scheduler.py                 |  2 ++
 distributed/tests/test_tls_functional.py | 45 ++++++++++++++++++++++--
 distributed/worker.py                    |  2 ++
 7 files changed, 72 insertions(+), 26 deletions(-)

diff --git a/distributed/cli/dask_scheduler.py b/distributed/cli/dask_scheduler.py
index 1eeb1e2715f..45d4f484c86 100755
--- a/distributed/cli/dask_scheduler.py
+++ b/distributed/cli/dask_scheduler.py
@@ -11,7 +11,7 @@
 
 from tornado.ioloop import IOLoop
 
-from distributed import Scheduler, Security
+from distributed import Scheduler
 from distributed.preloading import validate_preload_argv
 from distributed.cli.utils import check_python_3, install_signal_handlers
 from distributed.utils import deserialize_for_cli
@@ -157,17 +157,15 @@ def main(
     if port is None and (not host or not re.search(r":\d", host)):
         port = 8786
 
-    sec = Security(
-        **{
-            k: v
-            for k, v in [
-                ("tls_ca_file", tls_ca_file),
-                ("tls_scheduler_cert", tls_cert),
-                ("tls_scheduler_key", tls_key),
-            ]
-            if v is not None
-        }
-    )
+    sec = {
+        k: v
+        for k, v in [
+            ("tls_ca_file", tls_ca_file),
+            ("tls_scheduler_cert", tls_cert),
+            ("tls_scheduler_key", tls_key),
+        ]
+        if v is not None
+    }
 
     if "DASK_INTERNAL_INHERIT_CONFIG" in os.environ:
         config = deserialize_for_cli(os.environ["DASK_INTERNAL_INHERIT_CONFIG"])
diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
index fcb10b51092..ac0d7976b98 100755
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -11,7 +11,7 @@
 import click
 import dask
 from dask.system import CPU_COUNT
-from distributed import Nanny, Security
+from distributed import Nanny
 from distributed.cli.utils import check_python_3, install_signal_handlers
 from distributed.comm import get_address_host_port
 from distributed.preloading import validate_preload_argv
@@ -278,17 +278,15 @@ def main(
         )
         dashboard = bokeh
 
-    sec = Security(
-        **{
-            k: v
-            for k, v in [
-                ("tls_ca_file", tls_ca_file),
-                ("tls_worker_cert", tls_cert),
-                ("tls_worker_key", tls_key),
-            ]
-            if v is not None
-        }
-    )
+    sec = {
+        k: v
+        for k, v in [
+            ("tls_ca_file", tls_ca_file),
+            ("tls_worker_cert", tls_cert),
+            ("tls_worker_key", tls_key),
+        ]
+        if v is not None
+    }
 
     if nprocs > 1 and not nanny:
         logger.error(
diff --git a/distributed/client.py b/distributed/client.py
index 75e1be4a03b..d04ba8a6793 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -666,6 +666,8 @@ def __init__(
 
         if security is None:
             security = Security()
+        elif isinstance(security, dict):
+            security = Security(**security)
         elif security is True:
             security = Security.temporary()
             self._startup_kwargs["security"] = security
diff --git a/distributed/nanny.py b/distributed/nanny.py
index a2d1e5772b5..56b4b9d7883 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -97,6 +97,9 @@ def __init__(
     ):
         self._setup_logging(logger)
         self.loop = loop or IOLoop.current()
+
+        if isinstance(security, dict):
+            security = Security(**security)
         self.security = security or Security()
         assert isinstance(self.security, Security)
         self.connection_args = self.security.get_connection_args("worker")
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index a50407f7966..56332699ecb 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1128,6 +1128,8 @@ def __init__(
             preload_argv = dask.config.get("distributed.scheduler.preload-argv")
         self.preloads = preloading.process_preloads(self, preload, preload_argv)
 
+        if isinstance(security, dict):
+            security = Security(**security)
         self.security = security or Security()
         assert isinstance(self.security, Security)
         self.connection_args = self.security.get_connection_args("scheduler")
diff --git a/distributed/tests/test_tls_functional.py b/distributed/tests/test_tls_functional.py
index 67594f42926..e4152ca2b17 100644
--- a/distributed/tests/test_tls_functional.py
+++ b/distributed/tests/test_tls_functional.py
@@ -3,12 +3,21 @@
 Most are taken from other test files and adapted.
 """
 import asyncio
+import pytest
 
-from distributed import Nanny, worker_client, Queue
+from distributed import Scheduler, Worker, Client, Nanny, worker_client, Queue
 from distributed.client import wait
 from distributed.metrics import time
 from distributed.nanny import Nanny
-from distributed.utils_test import gen_tls_cluster, inc, double, slowinc, slowadd
+from distributed.utils_test import (  # noqa: F401
+    gen_tls_cluster,
+    inc,
+    double,
+    slowinc,
+    slowadd,
+    tls_config,
+    cleanup,
+)
 
 
 @gen_tls_cluster(client=True)
@@ -172,3 +181,35 @@ async def test_retire_workers(c, s, a, b):
     while a.status != "closed":
         await asyncio.sleep(0.01)
         assert time() < start + 5
+
+
+@pytest.mark.asyncio
+async def test_security_dict_input_no_security(cleanup):
+    async with Scheduler(security={}) as s:
+        async with Worker(s.address, security={}) as w:
+            async with Client(s.address, security={}, asynchronous=True) as c:
+                result = await c.submit(inc, 1)
+                assert result == 2
+
+
+@pytest.mark.asyncio
+async def test_security_dict_input(cleanup):
+    conf = tls_config()
+    ca_file = conf["distributed"]["comm"]["tls"]["ca-file"]
+    client = conf["distributed"]["comm"]["tls"]["client"]["cert"]
+    worker = conf["distributed"]["comm"]["tls"]["worker"]["cert"]
+    scheduler = conf["distributed"]["comm"]["tls"]["scheduler"]["cert"]
+
+    async with Scheduler(
+        security={"tls_ca_file": ca_file, "tls_scheduler_cert": scheduler}
+    ) as s:
+        async with Worker(
+            s.address, security={"tls_ca_file": ca_file, "tls_worker_cert": worker}
+        ) as w:
+            async with Client(
+                s.address,
+                security={"tls_ca_file": ca_file, "tls_client_cert": client},
+                asynchronous=True,
+            ) as c:
+                result = await c.submit(inc, 1)
+                assert result == 2
diff --git a/distributed/worker.py b/distributed/worker.py
index 8b4135eb5b8..e51cd064736 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -507,6 +507,8 @@ def __init__(
             self, preload, preload_argv, file_dir=self.local_directory
         )
 
+        if isinstance(security, dict):
+            security = Security(**security)
         self.security = security or Security()
         assert isinstance(self.security, Security)
         self.connection_args = self.security.get_connection_args("worker")

From 1b7734e7feb39a462eea0ef09371f9ddc839fa6f Mon Sep 17 00:00:00 2001
From: Matthias Bussonnier <bussonniermatthias@gmail.com>
Date: Thu, 11 Jun 2020 07:39:44 -0700
Subject: [PATCH 0874/1550] Check hosts for None value in SSH cluster. (#3883)

Then mypy is happy and does not complain in the rest of the function.
If host was None or empty list it would fail anyway either on the next
line `len(None)`, or further down `host[0]`.
---
 distributed/deploy/ssh.py            |  6 +++++-
 distributed/deploy/tests/test_ssh.py | 10 ++++++++++
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/distributed/deploy/ssh.py b/distributed/deploy/ssh.py
index 1f35d7e9672..595e21dbd7a 100644
--- a/distributed/deploy/ssh.py
+++ b/distributed/deploy/ssh.py
@@ -235,7 +235,7 @@ def SSHCluster(
     scheduler_options: dict = {},
     worker_module: str = "distributed.cli.dask_worker",
     remote_python: str = None,
-    **kwargs
+    **kwargs,
 ):
     """ Deploy a Dask cluster using SSH
 
@@ -315,6 +315,10 @@ def SSHCluster(
         kwargs.setdefault("worker_addrs", hosts)
         return OldSSHCluster(**kwargs)
 
+    if not hosts:
+        raise ValueError(
+            f"`hosts` must be a non empty list, value {repr(hosts)!r} found."
+        )
     if isinstance(connect_options, list) and len(connect_options) != len(hosts):
         raise RuntimeError(
             "When specifying a list of connect_options you must provide a "
diff --git a/distributed/deploy/tests/test_ssh.py b/distributed/deploy/tests/test_ssh.py
index d52987200b4..99d5ea15aec 100644
--- a/distributed/deploy/tests/test_ssh.py
+++ b/distributed/deploy/tests/test_ssh.py
@@ -8,6 +8,16 @@
 from distributed.utils_test import loop  # noqa: F401
 
 
+def test_ssh_hosts_None():
+    with pytest.raises(ValueError):
+        SSHCluster(hosts=None)
+
+
+def test_ssh_hosts_empty_list():
+    with pytest.raises(ValueError):
+        SSHCluster(hosts=[])
+
+
 @pytest.mark.asyncio
 async def test_basic():
     async with SSHCluster(

From 7ce0df0aceff4dbf5fd79af2a405a747d7194365 Mon Sep 17 00:00:00 2001
From: Julien Jerphanion <git@jjerphan.xyz>
Date: Thu, 11 Jun 2020 22:23:01 +0100
Subject: [PATCH 0875/1550] Replace stream= with comm= in handlers (#3860)

---
 distributed/comm/tcp.py        | 12 ++++++------
 distributed/lock.py            |  4 ++--
 distributed/publish.py         |  4 ++--
 distributed/queues.py          | 10 +++++-----
 distributed/scheduler.py       | 12 ++++++------
 distributed/tests/test_core.py |  2 +-
 distributed/variable.py        |  6 +++---
 7 files changed, 25 insertions(+), 25 deletions(-)

diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py
index c610bde31ac..cd7490be21a 100644
--- a/distributed/comm/tcp.py
+++ b/distributed/comm/tcp.py
@@ -33,17 +33,17 @@
 MAX_BUFFER_SIZE = MEMORY_LIMIT / 2
 
 
-def set_tcp_timeout(stream):
+def set_tcp_timeout(comm):
     """
     Set kernel-level TCP timeout on the stream.
     """
-    if stream.closed():
+    if comm.closed():
         return
 
     timeout = dask.config.get("distributed.comm.timeouts.tcp")
     timeout = int(parse_timedelta(timeout, default="seconds"))
 
-    sock = stream.socket
+    sock = comm.socket
 
     # Default (unsettable) value on Windows
     # https://msdn.microsoft.com/en-us/library/windows/desktop/dd877220(v=vs.85).aspx
@@ -92,15 +92,15 @@ def set_tcp_timeout(stream):
         logger.warning("Could not set timeout on TCP stream: %s", e)
 
 
-def get_stream_address(stream):
+def get_stream_address(comm):
     """
     Get a stream's local address.
     """
-    if stream.closed():
+    if comm.closed():
         return "<closed>"
 
     try:
-        return unparse_host_port(*stream.socket.getsockname()[:2])
+        return unparse_host_port(*comm.socket.getsockname()[:2])
     except EnvironmentError:
         # Probably EBADF
         return "<closed>"
diff --git a/distributed/lock.py b/distributed/lock.py
index a9bda6617aa..67e926ac281 100644
--- a/distributed/lock.py
+++ b/distributed/lock.py
@@ -31,7 +31,7 @@ def __init__(self, scheduler):
 
         self.scheduler.extensions["locks"] = self
 
-    async def acquire(self, stream=None, name=None, id=None, timeout=None):
+    async def acquire(self, comm=None, name=None, id=None, timeout=None):
         with log_errors():
             if isinstance(name, list):
                 name = tuple(name)
@@ -59,7 +59,7 @@ async def acquire(self, stream=None, name=None, id=None, timeout=None):
                 self.ids[name] = id
             return result
 
-    def release(self, stream=None, name=None, id=None):
+    def release(self, comm=None, name=None, id=None):
         with log_errors():
             if isinstance(name, list):
                 name = tuple(name)
diff --git a/distributed/publish.py b/distributed/publish.py
index 4b30ebde042..021ae5e7acc 100644
--- a/distributed/publish.py
+++ b/distributed/publish.py
@@ -26,7 +26,7 @@ def __init__(self, scheduler):
         self.scheduler.handlers.update(handlers)
         self.scheduler.extensions["publish"] = self
 
-    def put(self, stream=None, keys=None, data=None, name=None, client=None):
+    def put(self, comm=None, keys=None, data=None, name=None, client=None):
         with log_errors():
             if name in self.datasets:
                 raise KeyError("Dataset %s already exists" % name)
@@ -34,7 +34,7 @@ def put(self, stream=None, keys=None, data=None, name=None, client=None):
             self.datasets[name] = {"data": data, "keys": keys}
             return {"status": "OK", "name": name}
 
-    def delete(self, stream=None, name=None):
+    def delete(self, comm=None, name=None):
         with log_errors():
             out = self.datasets.pop(name, {"keys": []})
             self.scheduler.client_releases_keys(
diff --git a/distributed/queues.py b/distributed/queues.py
index e7c634c1180..9b5d1dbf1f3 100644
--- a/distributed/queues.py
+++ b/distributed/queues.py
@@ -44,7 +44,7 @@ def __init__(self, scheduler):
 
         self.scheduler.extensions["queues"] = self
 
-    def create(self, stream=None, name=None, client=None, maxsize=0):
+    def create(self, comm=None, name=None, client=None, maxsize=0):
         logger.debug("Queue name: {}".format(name))
         if name not in self.queues:
             self.queues[name] = asyncio.Queue(maxsize=maxsize)
@@ -52,7 +52,7 @@ def create(self, stream=None, name=None, client=None, maxsize=0):
         else:
             self.client_refcount[name] += 1
 
-    def release(self, stream=None, name=None, client=None):
+    def release(self, comm=None, name=None, client=None):
         if name not in self.queues:
             return
 
@@ -66,7 +66,7 @@ def release(self, stream=None, name=None, client=None):
                 self.scheduler.client_releases_keys(keys=keys, client="queue-%s" % name)
 
     async def put(
-        self, stream=None, name=None, key=None, data=None, client=None, timeout=None
+        self, comm=None, name=None, key=None, data=None, client=None, timeout=None
     ):
         if key is not None:
             record = {"type": "Future", "value": key}
@@ -82,7 +82,7 @@ def future_release(self, name=None, key=None, client=None):
             self.scheduler.client_releases_keys(keys=[key], client="queue-%s" % name)
             del self.future_refcount[name, key]
 
-    async def get(self, stream=None, name=None, client=None, timeout=None, batch=False):
+    async def get(self, comm=None, name=None, client=None, timeout=None, batch=False):
         def process(record):
             """ Add task status if known """
             if record["type"] == "Future":
@@ -122,7 +122,7 @@ def process(record):
             record = process(record)
             return record
 
-    def qsize(self, stream=None, name=None, client=None):
+    def qsize(self, comm=None, name=None, client=None):
         return self.queues[name].qsize()
 
 
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 56332699ecb..1a5232910e9 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1580,7 +1580,7 @@ async def close(self, comm=None, fast=False, close_workers=False):
         setproctitle("dask-scheduler [closed]")
         disable_gc_diagnosis()
 
-    async def close_worker(self, stream=None, worker=None, safe=None):
+    async def close_worker(self, comm=None, worker=None, safe=None):
         """ Remove a worker from the cluster
 
         This both removes the worker from our local state and also sends a
@@ -2703,7 +2703,7 @@ def handle_missing_data(self, key=None, errant_worker=None, **kwargs):
             else:
                 self.transitions({key: "forgotten"})
 
-    def release_worker_data(self, stream=None, keys=None, worker=None):
+    def release_worker_data(self, comm=None, keys=None, worker=None):
         ws = self.workers[worker]
         tasks = {self.tasks[k] for k in keys}
         removed_tasks = tasks & ws.has_what
@@ -3798,7 +3798,7 @@ def run_function(self, stream, function, args=(), kwargs={}, wait=True):
         self.log_event("all", {"action": "run-function", "function": function})
         return run(self, stream, function=function, args=args, kwargs=kwargs, wait=wait)
 
-    def set_metadata(self, stream=None, keys=None, value=None):
+    def set_metadata(self, comm=None, keys=None, value=None):
         try:
             metadata = self.task_metadata
             for key in keys[:-1]:
@@ -3811,7 +3811,7 @@ def set_metadata(self, stream=None, keys=None, value=None):
 
             pdb.set_trace()
 
-    def get_metadata(self, stream=None, keys=None, default=no_default):
+    def get_metadata(self, comm=None, keys=None, default=no_default):
         metadata = self.task_metadata
         for key in keys[:-1]:
             metadata = metadata[key]
@@ -3823,7 +3823,7 @@ def get_metadata(self, stream=None, keys=None, default=no_default):
             else:
                 raise
 
-    def get_task_status(self, stream=None, keys=None):
+    def get_task_status(self, comm=None, keys=None):
         return {
             key: (self.tasks[key].state if key in self.tasks else None) for key in keys
         }
@@ -4912,7 +4912,7 @@ def release_resources(self, ts, ws):
     # Utility functions #
     #####################
 
-    def add_resources(self, stream=None, worker=None, resources=None):
+    def add_resources(self, comm=None, worker=None, resources=None):
         ws = self.workers[worker]
         if resources:
             ws.resources.update(resources)
diff --git a/distributed/tests/test_core.py b/distributed/tests/test_core.py
index fbdf5677809..62b141645c7 100644
--- a/distributed/tests/test_core.py
+++ b/distributed/tests/test_core.py
@@ -471,7 +471,7 @@ async def test_ports(loop):
         server3.stop()
 
 
-def stream_div(stream=None, x=None, y=None):
+def stream_div(comm=None, x=None, y=None):
     return x / y
 
 
diff --git a/distributed/variable.py b/distributed/variable.py
index 3bf6347fa61..82c407a494f 100644
--- a/distributed/variable.py
+++ b/distributed/variable.py
@@ -39,7 +39,7 @@ def __init__(self, scheduler):
 
         self.scheduler.extensions["variables"] = self
 
-    async def set(self, stream=None, name=None, key=None, data=None, client=None):
+    async def set(self, comm=None, name=None, key=None, data=None, client=None):
         if key is not None:
             record = {"type": "Future", "value": key}
             self.scheduler.client_desires_keys(keys=[key], client="variable-%s" % name)
@@ -71,7 +71,7 @@ async def future_release(self, name=None, key=None, token=None, client=None):
             async with self.waiting_conditions[name]:
                 self.waiting_conditions[name].notify_all()
 
-    async def get(self, stream=None, name=None, client=None, timeout=None):
+    async def get(self, comm=None, name=None, client=None, timeout=None):
         start = self.scheduler.loop.time()
         while name not in self.variables:
             if timeout is not None:
@@ -105,7 +105,7 @@ async def _():  # Python 3.6 is odd and requires special help here
             self.waiting[key, name].add(token)
         return record
 
-    async def delete(self, stream=None, name=None, client=None):
+    async def delete(self, comm=None, name=None, client=None):
         with log_errors():
             try:
                 old = self.variables[name]

From 4c4e398b0364ddd5a47bd8efc2fcbc03a7fd4f21 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 11 Jun 2020 15:31:51 -0700
Subject: [PATCH 0876/1550] Skip test_pid_file in older versions of Python
 (#3888)

This has been intermittent for a while, but only seems to be on Python
3.6 .  I'm inclined to ignore it for now.  It's an infrequently used
feature and seems to be fixed in recent Python versions.
---
 distributed/cli/tests/test_dask_scheduler.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/distributed/cli/tests/test_dask_scheduler.py b/distributed/cli/tests/test_dask_scheduler.py
index 6f924e4c8e6..543d4300ff3 100644
--- a/distributed/cli/tests/test_dask_scheduler.py
+++ b/distributed/cli/tests/test_dask_scheduler.py
@@ -185,6 +185,9 @@ def test_interface(loop):
                 assert all("127.0.0.1" == d["host"] for d in info["workers"].values())
 
 
+@pytest.mark.skipif(
+    sys.version_info < (3, 7), reason="Intermittent failure on old Python version"
+)
 def test_pid_file(loop):
     def check_pidfile(proc, pidfile):
         start = time()

From 1d7640b0172febf9ceef37c2c31241c66ac165eb Mon Sep 17 00:00:00 2001
From: DomHudson <dom.hudson@thoughtriver.com>
Date: Fri, 12 Jun 2020 16:19:53 +0100
Subject: [PATCH 0877/1550] Update related-work.rst (#3889)

---
 docs/source/related-work.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/source/related-work.rst b/docs/source/related-work.rst
index 07b08c29787..9e20f2471f4 100644
--- a/docs/source/related-work.rst
+++ b/docs/source/related-work.rst
@@ -198,7 +198,7 @@ concurrent.futures
 ~~~~~~~~~~~~~~~~~~
 
 The :class:`distributed.Client` API is modeled after :mod:`concurrent.futures`
-and :pep:`3184`.  It has a few notable differences:
+and :pep:`3148`.  It has a few notable differences:
 
 *  ``distributed`` accepts :class:`~distributed.client.Future` objects within
    calls to ``submit/map``. When chaining computations, it is preferable to
@@ -216,4 +216,4 @@ API, use the object returned by the
 :meth:`~distributed.client.Client.get_executor` method.
 
 
-.. _PEP-3184: https://www.python.org/dev/peps/pep-3148/
+.. _PEP-3148: https://www.python.org/dev/peps/pep-3148/

From 86b5743ae1be52c80ebe440f5b58e8529fe2817e Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 16 Jun 2020 08:01:27 -0700
Subject: [PATCH 0878/1550] Support Bokeh 2.1 (#3904)

The create_hosts_whitelist function was renamed to create_hosts_allowlist

Fixes https://github.com/dask/distributed/issues/3741
---
 distributed/dashboard/core.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/distributed/dashboard/core.py b/distributed/dashboard/core.py
index 6c3c7e919c2..916504f4d04 100644
--- a/distributed/dashboard/core.py
+++ b/distributed/dashboard/core.py
@@ -4,7 +4,11 @@
 
 import bokeh
 from bokeh.server.server import BokehTornado
-from bokeh.server.util import create_hosts_whitelist
+
+try:
+    from bokeh.server.util import create_hosts_allowlist
+except ImportError:
+    from bokeh.server.util import create_hosts_whitelist as create_hosts_allowlist
 from bokeh.application.handlers.function import FunctionHandler
 from bokeh.application import Application
 import dask
@@ -30,7 +34,7 @@ def BokehApplication(applications, server, prefix="/", template_variables={}):
     apps = {k: functools.partial(v, server, extra) for k, v in applications.items()}
     apps = {k: Application(FunctionHandler(v)) for k, v in apps.items()}
     kwargs = dask.config.get("distributed.scheduler.dashboard.bokeh-application").copy()
-    extra_websocket_origins = create_hosts_whitelist(
+    extra_websocket_origins = create_hosts_allowlist(
         kwargs.pop("allow_websocket_origin"), server.http_server.port
     )
 

From 2407e645cd8ed2e79b5e6a1e9436a838df211cf2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lo=C3=AFc=20Est=C3=A8ve?= <loic.esteve@ymail.com>
Date: Wed, 17 Jun 2020 15:17:06 +0200
Subject: [PATCH 0879/1550] Fix total aggregated metrics in dashboard (#3897)

---
 distributed/dashboard/components/scheduler.py | 27 ++++++++++++++++---
 1 file changed, 24 insertions(+), 3 deletions(-)

diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index 2d4bbcccf6d..6e633708277 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -535,7 +535,7 @@ def update(self):
             self.fig.title.text = "Compute Time Per Task"
 
             compute_result = dict(
-                times=compute_time, color=compute_colors, names=compute_names,
+                times=compute_time, color=compute_colors, names=compute_names
             )
 
             update(self.compute_source, compute_result)
@@ -635,7 +635,7 @@ def update(self):
             self.fig.x_range.factors = agg_names
             self.fig.title.text = "Aggregate Time Per Action"
 
-            action_result = dict(times=agg_time, color=agg_colors, names=agg_names,)
+            action_result = dict(times=agg_time, color=agg_colors, names=agg_names)
 
             update(self.action_source, action_result)
 
@@ -1856,7 +1856,28 @@ def update(self):
                 )
                 continue
             try:
-                data[name].insert(0, sum(data[name]))
+                if len(self.scheduler.workers) == 0:
+                    total_data = None
+                elif name == "memory_percent":
+                    total_data = sum(
+                        ws.metrics["memory"] for ws in self.scheduler.workers.values()
+                    ) / sum(ws.memory_limit for ws in self.scheduler.workers.values())
+                elif name == "cpu":
+                    total_data = (
+                        sum(ws.metrics["cpu"] for ws in self.scheduler.workers.values())
+                        / 100
+                        / len(self.scheduler.workers.values())
+                    )
+                elif name == "cpu_fraction":
+                    total_data = (
+                        sum(ws.metrics["cpu"] for ws in self.scheduler.workers.values())
+                        / 100
+                        / sum(ws.nthreads for ws in self.scheduler.workers.values())
+                    )
+                else:
+                    total_data = sum(data[name])
+
+                data[name].insert(0, total_data)
             except TypeError:
                 data[name].insert(0, None)
 

From 3690ef176538383b403931c23f3c788bf0482723 Mon Sep 17 00:00:00 2001
From: Marco Neumann <57095103+marco-neumann-jdas@users.noreply.github.com>
Date: Wed, 17 Jun 2020 15:24:49 +0200
Subject: [PATCH 0880/1550] Prune virtual client on variable deletion (#3910)

Fixes #3899

Otherwise, setting new data to a an existing variable will remove its
client.
---
 distributed/tests/test_variable.py | 19 +++++++++++++++++++
 distributed/variable.py            |  2 ++
 2 files changed, 21 insertions(+)

diff --git a/distributed/tests/test_variable.py b/distributed/tests/test_variable.py
index 1e707626235..5d9ece6ee54 100644
--- a/distributed/tests/test_variable.py
+++ b/distributed/tests/test_variable.py
@@ -263,3 +263,22 @@ def test_future_erred_sync(client):
 
     with pytest.raises(ZeroDivisionError):
         future2.result()
+
+
+@gen_cluster(client=True)
+async def test_variables_do_not_leak_client(c, s, a, b):
+    # https://github.com/dask/distributed/issues/3899
+    clients_pre = set(s.clients)
+
+    # setup variable with future
+    x = Variable("x")
+    future = c.submit(inc, 1)
+    await x.set(future)
+
+    # complete teardown
+    x.delete()
+
+    start = time()
+    while set(s.clients) != clients_pre:
+        await asyncio.sleep(0.01)
+        assert time() < start + 5
diff --git a/distributed/variable.py b/distributed/variable.py
index 82c407a494f..b20273031ab 100644
--- a/distributed/variable.py
+++ b/distributed/variable.py
@@ -119,6 +119,8 @@ async def delete(self, comm=None, name=None, client=None):
             with suppress(KeyError):
                 del self.variables[name]
 
+            self.scheduler.remove_client("variable-%s" % name)
+
 
 class Variable:
     """ Distributed Global Variable

From 2ceb982bd8c98ed204e28c270f9cc4eaa32c3b86 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lo=C3=AFc=20Est=C3=A8ve?= <loic.esteve@ymail.com>
Date: Wed, 17 Jun 2020 15:30:51 +0200
Subject: [PATCH 0881/1550] Show cpu_fraction on hover for dashboard workers
 circle plot. (#3906)

---
 distributed/dashboard/components/scheduler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index 6e633708277..2cf37a20800 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -1794,7 +1794,7 @@ def __init__(self, scheduler, width=800, **kwargs):
             tooltips="""
                 <div>
                   <span style="font-size: 10px; font-family: Monaco, monospace;">Worker (@name): </span>
-                  <span style="font-size: 10px; font-family: Monaco, monospace;">@cpu</span>
+                  <span style="font-size: 10px; font-family: Monaco, monospace;">@cpu_fraction</span>
                 </div>
                 """,
         )

From acb0f08a126d1988f49450dde6bd9949129d74f9 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 17 Jun 2020 13:05:39 -0700
Subject: [PATCH 0882/1550] Make encryption default if Security is given
 arguments (#3887)

---
 distributed/distributed-schema.yaml      |  4 +++-
 distributed/distributed.yaml             |  2 +-
 distributed/security.py                  | 10 ++++++----
 distributed/tests/test_security.py       |  2 +-
 distributed/tests/test_tls_functional.py |  5 ++++-
 5 files changed, 15 insertions(+), 8 deletions(-)

diff --git a/distributed/distributed-schema.yaml b/distributed/distributed-schema.yaml
index f97d900e4be..1a1f2566a63 100644
--- a/distributed/distributed-schema.yaml
+++ b/distributed/distributed-schema.yaml
@@ -597,7 +597,9 @@ properties:
                 type: string
 
           require-encryption:
-            type: boolean
+            type:
+            - boolean
+            - "null"
             description: |
               Whether to require encryption on non-local comms
 
diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index db42c21d3d8..cbf66dc220d 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -130,7 +130,7 @@ distributed:
       connect: 10s          # time before connecting fails
       tcp: 30s              # time before calling an unresponsive connection dead
 
-    require-encryption: False   # Whether to require encryption on non-local comms
+    require-encryption: null # Whether to require encryption on non-local comms
 
     tls:
       ciphers: null   # Allowed ciphers, specified as an OpenSSL cipher string.
diff --git a/distributed/security.py b/distributed/security.py
index f3430ac7b3e..2cfe952b397 100644
--- a/distributed/security.py
+++ b/distributed/security.py
@@ -60,13 +60,15 @@ class Security:
         "tls_worker_cert",
     )
 
-    def __init__(self, **kwargs):
+    def __init__(self, require_encryption=None, **kwargs):
         extra = set(kwargs).difference(self.__slots__)
         if extra:
             raise TypeError("Unknown parameters: %r" % sorted(extra))
-        self._set_field(
-            kwargs, "require_encryption", "distributed.comm.require-encryption"
-        )
+        if require_encryption is None:
+            require_encryption = dask.config.get("distributed.comm.require-encryption")
+        if require_encryption is None:
+            require_encryption = not not kwargs
+        self.require_encryption = require_encryption
         self._set_field(kwargs, "tls_ciphers", "distributed.comm.tls.ciphers")
         self._set_field(kwargs, "tls_ca_file", "distributed.comm.tls.ca-file")
         self._set_field(kwargs, "tls_client_key", "distributed.comm.tls.client.key")
diff --git a/distributed/tests/test_security.py b/distributed/tests/test_security.py
index 7bb2fd753c0..ab6646f0a12 100644
--- a/distributed/tests/test_security.py
+++ b/distributed/tests/test_security.py
@@ -109,7 +109,7 @@ def test_repr():
     sec = Security(tls_ca_file="ca.pem", tls_scheduler_cert="scert.pem")
     assert (
         repr(sec)
-        == "Security(require_encryption=False, tls_ca_file='ca.pem', tls_scheduler_cert='scert.pem')"
+        == "Security(require_encryption=True, tls_ca_file='ca.pem', tls_scheduler_cert='scert.pem')"
     )
 
 
diff --git a/distributed/tests/test_tls_functional.py b/distributed/tests/test_tls_functional.py
index e4152ca2b17..7e74f74e09c 100644
--- a/distributed/tests/test_tls_functional.py
+++ b/distributed/tests/test_tls_functional.py
@@ -201,11 +201,14 @@ async def test_security_dict_input(cleanup):
     scheduler = conf["distributed"]["comm"]["tls"]["scheduler"]["cert"]
 
     async with Scheduler(
-        security={"tls_ca_file": ca_file, "tls_scheduler_cert": scheduler}
+        host="localhost",
+        security={"tls_ca_file": ca_file, "tls_scheduler_cert": scheduler},
     ) as s:
+        assert s.address.startswith("tls://")
         async with Worker(
             s.address, security={"tls_ca_file": ca_file, "tls_worker_cert": worker}
         ) as w:
+            assert w.address.startswith("tls://")
             async with Client(
                 s.address,
                 security={"tls_ca_file": ca_file, "tls_client_cert": client},

From 920af0faba61416eb1933237cda1830ac43f6e8e Mon Sep 17 00:00:00 2001
From: Benjamin Zaitlen <quasiben@users.noreply.github.com>
Date: Thu, 18 Jun 2020 09:30:26 -0400
Subject: [PATCH 0883/1550] update heartbeat checks in scheduler (#3896)

* update heartbeart checks in scheduler

* scale heartbeart by number of workers
---
 distributed/scheduler.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 1a5232910e9..ec9a5ef05d1 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -5303,7 +5303,9 @@ def _reevaluate_occupancy_worker(self, ws):
     async def check_worker_ttl(self):
         now = time()
         for ws in self.workers.values():
-            if ws.last_seen < now - self.worker_ttl:
+            if (ws.last_seen < now - self.worker_ttl) and (
+                10 * heartbeat_interval(len(self.workers))
+            ):
                 logger.warning(
                     "Worker failed to heartbeat within %s seconds. Closing: %s",
                     self.worker_ttl,
@@ -5571,7 +5573,8 @@ def heartbeat_interval(n):
     elif n < 200:
         return 2
     else:
-        return 5
+        # no more than 200 hearbeats a second scaled by workers
+        return n / 200 + 1
 
 
 class KilledWorker(Exception):

From 5172678da28130ef7f4d86394d30566abc99328e Mon Sep 17 00:00:00 2001
From: Nick Evans <nick.evans3976@gmail.com>
Date: Thu, 18 Jun 2020 14:31:35 +0100
Subject: [PATCH 0884/1550] Notify worker plugins when a task is released
 (#3817)

---
 distributed/client.py                         | 13 ++-
 distributed/diagnostics/plugin.py             | 35 +++++++-
 .../diagnostics/tests/test_worker_plugin.py   | 88 ++++++++++++++-----
 distributed/worker.py                         | 12 ++-
 4 files changed, 116 insertions(+), 32 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index d04ba8a6793..3e6b1e73679 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -4093,10 +4093,11 @@ def register_worker_plugin(self, plugin=None, name=None):
         on all currently connected workers. It will also be run on any worker
         that connects in the future.
 
-        The plugin may include methods ``setup``, ``teardown``, and
-        ``transition``.  See the ``dask.distributed.WorkerPlugin`` class or the
-        examples below for the interface and docstrings.  It must be
-        serializable with the pickle or cloudpickle modules.
+        The plugin may include methods ``setup``, ``teardown``, ``transition``,
+        ``release_key``, and ``release_dep``.  See the
+        ``dask.distributed.WorkerPlugin`` class or the examples below for the
+        interface and docstrings.  It must be serializable with the pickle or
+        cloudpickle modules.
 
         If the plugin has a ``name`` attribute, or if the ``name=`` keyword is
         used then that will control idempotency.  A a plugin with that name has
@@ -4124,6 +4125,10 @@ def register_worker_plugin(self, plugin=None, name=None):
         ...         pass
         ...     def transition(self, key: str, start: str, finish: str, **kwargs):
         ...         pass
+        ...     def release_key(self, key: str, state: str, cause: Optional[str], reason: None, report: bool):
+        ...         pass
+        ...     def release_dep(self, dep: str, state: str, report: bool):
+        ...         pass
 
         >>> plugin = MyPlugin(1, 2, 3)
         >>> client.register_worker_plugin(plugin)
diff --git a/distributed/diagnostics/plugin.py b/distributed/diagnostics/plugin.py
index 12e7ad6ec3f..fb3b2afe203 100644
--- a/distributed/diagnostics/plugin.py
+++ b/distributed/diagnostics/plugin.py
@@ -90,7 +90,8 @@ class WorkerPlugin:
     """ Interface to extend the Worker
 
     A worker plugin enables custom code to run at different stages of the Workers'
-    lifecycle: at setup, during task state transitions and at teardown.
+    lifecycle: at setup, during task state transitions, when a task or dependency
+    is released, and at teardown.
 
     A plugin enables custom code to run at each of step of a Workers's life. Whenever such
     an event happens, the corresponding method on this class will be called. Note that the
@@ -147,3 +148,35 @@ def transition(self, key, start, finish, **kwargs):
             Final state of the transition.
         kwargs: More options passed when transitioning
         """
+
+    def release_key(self, key, state, cause, reason, report):
+        """
+        Called when the worker releases a task.
+
+        Parameters
+        ----------
+        key: string
+        state: string
+            State of the released task.
+            One of waiting, ready, executing, long-running, memory, error.
+        cause: string or None
+            Additional information on what triggered the release of the task.
+        reason: None
+            Not used.
+        report: bool
+            Whether the worker should report the released task to the scheduler.
+        """
+
+    def release_dep(self, dep, state, report):
+        """
+        Called when the worker releases a dependency.
+
+        Parameters
+        ----------
+        dep: string
+        state: string
+            State of the released dependency.
+            One of waiting, flight, memory.
+        report: bool
+            Whether the worker should report the released dependency to the scheduler.
+        """
diff --git a/distributed/diagnostics/tests/test_worker_plugin.py b/distributed/diagnostics/tests/test_worker_plugin.py
index 038924853a7..c9cdbed784c 100644
--- a/distributed/diagnostics/tests/test_worker_plugin.py
+++ b/distributed/diagnostics/tests/test_worker_plugin.py
@@ -1,15 +1,15 @@
 import pytest
 
 from distributed import Worker, WorkerPlugin
-from distributed.utils_test import gen_cluster
+from distributed.utils_test import async_wait_for, gen_cluster, inc
 
 
 class MyPlugin(WorkerPlugin):
     name = "MyPlugin"
 
-    def __init__(self, data, expected_transitions=None):
+    def __init__(self, data, expected_notifications=None):
         self.data = data
-        self.expected_transitions = expected_transitions
+        self.expected_notifications = expected_notifications
 
     def setup(self, worker):
         assert isinstance(worker, Worker)
@@ -17,20 +17,32 @@ def setup(self, worker):
         self.worker._my_plugin_status = "setup"
         self.worker._my_plugin_data = self.data
 
-        self.observed_transitions = []
+        self.observed_notifications = []
 
     def teardown(self, worker):
         self.worker._my_plugin_status = "teardown"
 
-        if self.expected_transitions is not None:
-            assert len(self.observed_transitions) == len(self.expected_transitions)
+        if self.expected_notifications is not None:
+            assert len(self.observed_notifications) == len(self.expected_notifications)
             for expected, real in zip(
-                self.expected_transitions, self.observed_transitions
+                self.expected_notifications, self.observed_notifications
             ):
                 assert expected == real
 
     def transition(self, key, start, finish, **kwargs):
-        self.observed_transitions.append((key, start, finish))
+        self.observed_notifications.append(
+            {"key": key, "start": start, "finish": finish,}
+        )
+
+    def release_key(self, key, state, cause, reason, report):
+        self.observed_notifications.append(
+            {"key": key, "state": state,}
+        )
+
+    def release_dep(self, dep, state, report):
+        self.observed_notifications.append(
+            {"dep": dep, "state": state,}
+        )
 
 
 @gen_cluster(client=True, nthreads=[])
@@ -54,16 +66,18 @@ async def test_create_on_construction(c, s, a, b):
 
 @gen_cluster(nthreads=[("127.0.0.1", 1)], client=True)
 async def test_normal_task_transitions_called(c, s, w):
-    expected_transitions = [
-        ("task", "waiting", "ready"),
-        ("task", "ready", "executing"),
-        ("task", "executing", "memory"),
+    expected_notifications = [
+        {"key": "task", "start": "waiting", "finish": "ready"},
+        {"key": "task", "start": "ready", "finish": "executing"},
+        {"key": "task", "start": "executing", "finish": "memory"},
+        {"key": "task", "state": "memory"},
     ]
 
-    plugin = MyPlugin(1, expected_transitions=expected_transitions)
+    plugin = MyPlugin(1, expected_notifications=expected_notifications)
 
     await c.register_worker_plugin(plugin)
     await c.submit(lambda x: x, 1, key="task")
+    await async_wait_for(lambda: not w.task_state, timeout=10)
 
 
 @gen_cluster(nthreads=[("127.0.0.1", 1)], client=True)
@@ -71,13 +85,13 @@ async def test_failing_task_transitions_called(c, s, w):
     def failing(x):
         raise Exception()
 
-    expected_transitions = [
-        ("task", "waiting", "ready"),
-        ("task", "ready", "executing"),
-        ("task", "executing", "error"),
+    expected_notifications = [
+        {"key": "task", "start": "waiting", "finish": "ready"},
+        {"key": "task", "start": "ready", "finish": "executing"},
+        {"key": "task", "start": "executing", "finish": "error"},
     ]
 
-    plugin = MyPlugin(1, expected_transitions=expected_transitions)
+    plugin = MyPlugin(1, expected_notifications=expected_notifications)
 
     await c.register_worker_plugin(plugin)
 
@@ -89,16 +103,44 @@ def failing(x):
     nthreads=[("127.0.0.1", 1)], client=True, worker_kwargs={"resources": {"X": 1}},
 )
 async def test_superseding_task_transitions_called(c, s, w):
-    expected_transitions = [
-        ("task", "waiting", "constrained"),
-        ("task", "constrained", "executing"),
-        ("task", "executing", "memory"),
+    expected_notifications = [
+        {"key": "task", "start": "waiting", "finish": "constrained"},
+        {"key": "task", "start": "constrained", "finish": "executing"},
+        {"key": "task", "start": "executing", "finish": "memory"},
+        {"key": "task", "state": "memory"},
     ]
 
-    plugin = MyPlugin(1, expected_transitions=expected_transitions)
+    plugin = MyPlugin(1, expected_notifications=expected_notifications)
 
     await c.register_worker_plugin(plugin)
     await c.submit(lambda x: x, 1, key="task", resources={"X": 1})
+    await async_wait_for(lambda: not w.task_state, timeout=10)
+
+
+@gen_cluster(nthreads=[("127.0.0.1", 1)], client=True)
+async def test_release_dep_called(c, s, w):
+    dsk = {
+        "dep": 1,
+        "task": (inc, "dep"),
+    }
+
+    expected_notifications = [
+        {"key": "dep", "start": "waiting", "finish": "ready"},
+        {"key": "dep", "start": "ready", "finish": "executing"},
+        {"key": "dep", "start": "executing", "finish": "memory"},
+        {"key": "task", "start": "waiting", "finish": "ready"},
+        {"key": "task", "start": "ready", "finish": "executing"},
+        {"key": "task", "start": "executing", "finish": "memory"},
+        {"key": "dep", "state": "memory"},
+        {"dep": "dep", "state": "memory"},
+        {"key": "task", "state": "memory"},
+    ]
+
+    plugin = MyPlugin(1, expected_notifications=expected_notifications)
+
+    await c.register_worker_plugin(plugin)
+    await c.get(dsk, "task", sync=False)
+    await async_wait_for(lambda: not (w.task_state or w.dep_state), timeout=10)
 
 
 @gen_cluster(nthreads=[("127.0.0.1", 1)], client=True)
diff --git a/distributed/worker.py b/distributed/worker.py
index e51cd064736..59cd285d49e 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -1601,7 +1601,7 @@ def transition(self, key, finish, **kwargs):
         self.task_state[key] = state or finish
         if self.validate:
             self.validate_key(key)
-        self._notify_transition(key, start, state or finish, **kwargs)
+        self._notify_plugins("transition", key, start, state or finish, **kwargs)
 
     def transition_waiting_ready(self, key):
         try:
@@ -2249,6 +2249,8 @@ def release_key(self, key, cause=None, reason=None, report=True):
 
             if report and state in PROCESSING:  # not finished
                 self.batched_stream.send({"op": "release", "key": key, "cause": cause})
+
+            self._notify_plugins("release_key", key, state, cause, reason, report)
         except CommClosedError:
             pass
         except Exception as e:
@@ -2292,6 +2294,8 @@ def release_dep(self, dep, report=False):
 
             if report and state == "memory":
                 self.batched_stream.send({"op": "release-worker-data", "keys": [dep]})
+
+            self._notify_plugins("release_dep", dep, state, report)
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
@@ -2833,11 +2837,11 @@ def get_call_stack(self, comm=None, keys=None):
         result = {k: profile.call_stack(frame) for k, frame in frames.items()}
         return result
 
-    def _notify_transition(self, key, start, finish, **kwargs):
+    def _notify_plugins(self, method_name, *args, **kwargs):
         for name, plugin in self.plugins.items():
-            if hasattr(plugin, "transition"):
+            if hasattr(plugin, method_name):
                 try:
-                    plugin.transition(key, start, finish, **kwargs)
+                    getattr(plugin, method_name)(*args, **kwargs)
                 except Exception:
                     logger.info(
                         "Plugin '%s' failed with exception" % name, exc_info=True

From 0eacd789cac952a925a12898ae2fa5abe3a5e3d8 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Fri, 19 Jun 2020 13:42:25 -0500
Subject: [PATCH 0885/1550] bump version to 2.19.0

---
 docs/source/changelog.rst | 34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index 848158b589d..d2a233e8ab5 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,37 @@
 Changelog
 =========
 
+2.19.0 - 2020-06-19
+-------------------
+
+- Notify worker plugins when a task is released (:pr:`3817`) `Nick Evans`_
+- Update heartbeat checks in scheduler (:pr:`3896`) `Benjamin Zaitlen`_
+- Make encryption default if ``Security`` is given arguments (:pr:`3887`) `Matthew Rocklin`_
+- Show ``cpu_fraction`` on hover for dashboard workers circle plot. (:pr:`3906`) `Loïc Estève`_
+- Prune virtual client on variable deletion (:pr:`3910`) `Marco Neumann`_
+- Fix total aggregated metrics in dashboard (:pr:`3897`) `Loïc Estève`_
+- Support Bokeh 2.1 (:pr:`3904`) `Matthew Rocklin`_
+- Update ``related-work.rst`` (:pr:`3889`) `DomHudson`_
+- Skip ``test_pid_file`` in older versions of Python (:pr:`3888`) `Matthew Rocklin`_
+- Replace ``stream=`` with ``comm=`` in handlers (:pr:`3860`) `Julien Jerphanion`_
+- Check hosts for ``None`` value in SSH cluster. (:pr:`3883`) `Matthias Bussonnier`_
+- Allow dictionaries in ``security=`` keywords (:pr:`3874`) `Matthew Rocklin`_
+- Use pickle protocol 5 with NumPy object arrays (:pr:`3871`) `jakirkham`_
+- Cast any ``frame`` to ``uint8`` (same type as ``bytes``) (:pr:`3870`) `jakirkham`_
+- Use ``Enum`` for worker, scheduler and nanny status. (:pr:`3853`) `Matthias Bussonnier`_
+- Drop legacy ``buffer_interface`` assignment (:pr:`3869`) `jakirkham`_
+- Drop old frame splitting in NumPy serialization (:pr:`3868`) `jakirkham`_
+- Drop no longer needed local ``import pickle`` (:pr:`3865`) `jakirkham`_
+- Fix typo in ``feed``'s log message (:pr:`3867`) `jakirkham`_
+- Tidy pickle (:pr:`3866`) `jakirkham`_
+- Handle empty times in task stream (:pr:`3862`) `Benjamin Zaitlen`_
+- Change ``asyncssh`` objects to sphinx references (:pr:`3861`) `Jacob Tomlinson`_
+- Improve ``SSHCluster`` docstring for ``connect_options`` (:pr:`3859`) `Jacob Tomlinson`_
+- Validate address parameter in client constructor (:pr:`3842`) `joshreback`_
+- Use ``SpecCluster`` name in worker names (:pr:`3855`) `Loïc Estève`_
+- Allow async ``add_worker`` and ``remove_worker`` plugin methods (:pr:`3847`) `James Bourbeau`_
+
+
 2.18.0 - 2020-06-05
 -------------------
 
@@ -1792,3 +1823,6 @@ significantly without many new features.
 .. _`Nick Evans`: https://github.com/nre
 .. _`Scott Sanderson`: https://github.com/ssanderson
 .. _`Matthias Bussonnier`: https://github.com/Carreau
+.. _`DomHudson`: https://github.com/DomHudson
+.. _`Julien Jerphanion`: https://github.com/jjerphan
+.. _`joshreback`: https://github.com/joshreback

From 4f878b420b349ee725de5ef64fd5e664dedb8aba Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Mon, 22 Jun 2020 15:36:18 +0100
Subject: [PATCH 0886/1550] Fix race condition in repeated calls to
 cluster.adapt() (#3915)

---
 distributed/deploy/adaptive_core.py       |  7 +++++--
 distributed/deploy/tests/test_adaptive.py | 19 +++++++++++++++++++
 2 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/distributed/deploy/adaptive_core.py b/distributed/deploy/adaptive_core.py
index 7d15cb4c2c7..14de96ad396 100644
--- a/distributed/deploy/adaptive_core.py
+++ b/distributed/deploy/adaptive_core.py
@@ -78,10 +78,13 @@ def __init__(
         self.periodic_callback = None
 
         def f():
-            self.periodic_callback = PeriodicCallback(self.adapt, self.interval * 1000)
-            self.periodic_callback.start()
+            try:
+                self.periodic_callback.start()
+            except AttributeError:
+                pass
 
         if self.interval:
+            self.periodic_callback = PeriodicCallback(self.adapt, self.interval * 1000)
             try:
                 self.loop.add_callback(f)
             except AttributeError:
diff --git a/distributed/deploy/tests/test_adaptive.py b/distributed/deploy/tests/test_adaptive.py
index 651717aeee4..bbdfd52c1f0 100644
--- a/distributed/deploy/tests/test_adaptive.py
+++ b/distributed/deploy/tests/test_adaptive.py
@@ -430,3 +430,22 @@ def test_adaptive_config():
         assert adapt.maximum == math.inf
         assert adapt.interval == 5
         assert adapt.wait_count == 8
+
+
+@pytest.mark.asyncio
+async def test_update_adaptive(cleanup):
+    async with LocalCluster(
+        0,
+        threads_per_worker=2,
+        memory_limit="3 GB",
+        scheduler_port=0,
+        silence_logs=False,
+        processes=False,
+        dashboard_address=None,
+        asynchronous=True,
+    ) as cluster:
+        first = cluster.adapt(maxmimum=1)
+        second = cluster.adapt(maxmimum=2)
+        await asyncio.sleep(0.2)
+        assert first.periodic_callback is None
+        assert second.periodic_callback.is_running()

From 44b2358e33a0738c4c70ca96db4242636245e07d Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Mon, 22 Jun 2020 14:48:27 -0700
Subject: [PATCH 0887/1550] Fix `ensure_bytes` import location (#3919)

The `ensure_bytes` function is defined in `distributed.utils`. However
here we are relying on the fact that it is imported into other modules.
Fix this to import `ensure_bytes` from the correct place.
---
 distributed/protocol/serialize.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index 4ed27bf278b..55230b4b3ce 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -9,14 +9,13 @@
 import msgpack
 
 from . import pickle
-from ..utils import has_keyword, nbytes, typename
+from ..utils import has_keyword, nbytes, typename, ensure_bytes
 from .compression import maybe_compress, decompress
 from .utils import (
     unpack_frames,
     pack_frames_prelude,
     frame_split_size,
     merge_frames,
-    ensure_bytes,
     msgpack_opts,
 )
 

From 93701f82c2cef46d4e68696bf48af0fc65ea9159 Mon Sep 17 00:00:00 2001
From: Peter Andreas Entschev <peter@entschev.com>
Date: Fri, 26 Jun 2020 23:03:29 +0200
Subject: [PATCH 0888/1550] Make `Worker.delete_data` sync (#3922)

* Remove call to non-existing scheduler.remove_keys
---
 distributed/worker.py | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/distributed/worker.py b/distributed/worker.py
index 59cd285d49e..7ad84544097 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -1341,7 +1341,7 @@ def update_data(self, comm=None, data=None, report=True, serializers=None):
         info = {"nbytes": {k: sizeof(v) for k, v in data.items()}, "status": "OK"}
         return info
 
-    async def delete_data(self, comm=None, keys=None, report=True):
+    def delete_data(self, comm=None, keys=None, report=True):
         if keys:
             for key in list(keys):
                 self.log.append((key, "delete"))
@@ -1352,12 +1352,6 @@ async def delete_data(self, comm=None, keys=None, report=True):
                     self.release_dep(key)
 
             logger.debug("Deleted %d keys", len(keys))
-            if report:
-                logger.debug("Reporting loss of keys to scheduler")
-                # TODO: this route seems to not exist?
-                await self.scheduler.remove_keys(
-                    address=self.contact_address, keys=list(keys)
-                )
         return "OK"
 
     async def set_resources(self, **resources):

From b4d3a56d8d6e2ee24373e2a3c552246c4664ef74 Mon Sep 17 00:00:00 2001
From: joshreback <joshreback@gmail.com>
Date: Mon, 29 Jun 2020 11:52:12 -0400
Subject: [PATCH 0889/1550] install vanilla status route if bokeh dependency is
 not satisfied (#3844)

* install vanilla status route if bokeh dependency is not satisfied

* ensure root route redirects to proper status route

* as a test: do not manipulate routes and see if that passes the tests

* do not manipulate any routes; do not try importing the scheduler -- basically a noop test

* reinstate code to add routes; but intentionally raise ImportError

* try one more test

* use prints instead?

* avoid double import issue

Co-authored-by: Josh Reback <josh.reback@gradientai.com>
---
 distributed/http/scheduler/missing_bokeh.py | 15 +++++++++++
 distributed/scheduler.py                    | 28 +++++++++++----------
 2 files changed, 30 insertions(+), 13 deletions(-)
 create mode 100644 distributed/http/scheduler/missing_bokeh.py

diff --git a/distributed/http/scheduler/missing_bokeh.py b/distributed/http/scheduler/missing_bokeh.py
new file mode 100644
index 00000000000..3eb68960d53
--- /dev/null
+++ b/distributed/http/scheduler/missing_bokeh.py
@@ -0,0 +1,15 @@
+from ..utils import RequestHandler, redirect
+from ...utils import log_errors
+
+
+class MissingBokeh(RequestHandler):
+    def get(self):
+        with log_errors():
+            self.write(
+                "<p>Dask needs bokeh >= 0.13.0 for the dashboard.</p>"
+                "<p>Install with conda: conda install bokeh>=0.13.0</p>"
+                "<p>Install with pip: pip install bokeh>=0.13.0</p>"
+            )
+
+
+routes = [(r"/", redirect("status"), {}), (r"status", MissingBokeh, {})]
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index ec9a5ef05d1..3d47b88d226 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1143,22 +1143,24 @@ def __init__(
             default_port=self.default_port,
         )
 
-        routes = get_handlers(
-            server=self,
-            modules=dask.config.get("distributed.scheduler.http.routes"),
-            prefix=http_prefix,
-        )
-        self.start_http_server(routes, dashboard_address, default_port=8787)
-
-        if dashboard or (dashboard is None and dashboard_address):
+        http_server_modules = dask.config.get("distributed.scheduler.http.routes")
+        show_dashboard = dashboard or (dashboard is None and dashboard_address)
+        missing_bokeh = False
+        # install vanilla route if show_dashboard but bokeh is not installed
+        if show_dashboard:
             try:
                 import distributed.dashboard.scheduler
             except ImportError:
-                logger.debug("To start diagnostics web server please install Bokeh")
-            else:
-                distributed.dashboard.scheduler.connect(
-                    self.http_application, self.http_server, self, prefix=http_prefix
-                )
+                missing_bokeh = True
+                http_server_modules.append("distributed.http.scheduler.missing_bokeh")
+        routes = get_handlers(
+            server=self, modules=http_server_modules, prefix=http_prefix,
+        )
+        self.start_http_server(routes, dashboard_address, default_port=8787)
+        if show_dashboard and not missing_bokeh:
+            distributed.dashboard.scheduler.connect(
+                self.http_application, self.http_server, self, prefix=http_prefix
+            )
 
         # Communication state
         self.loop = loop or IOLoop.current()

From d4c8bdb062976ad08cb0af6a4521d9a39e781b70 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 29 Jun 2020 09:46:46 -0700
Subject: [PATCH 0890/1550] Make local directory if it does not exist (#3928)

Fixes https://stackoverflow.com/questions/62603609/create-local-directory-for-dask-jobqueue
---
 distributed/tests/test_worker.py | 9 +++++++++
 distributed/worker.py            | 7 ++++---
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index aabb76c75d6..d2efe02196e 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -1460,6 +1460,15 @@ async def test_local_directory(s):
             assert "dask-worker-space" in w.local_directory
 
 
+@gen_cluster(nthreads=[])
+async def test_local_directory_make_new_directory(s):
+    with tmpfile() as fn:
+        w = await Worker(s.address, local_directory=os.path.join(fn, "foo", "bar"))
+        assert w.local_directory.startswith(fn)
+        assert "foo" in w.local_directory
+        assert "dask-worker-space" in w.local_directory
+
+
 @pytest.mark.skipif(
     not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
 )
diff --git a/distributed/worker.py b/distributed/worker.py
index 7ad84544097..187bcc76770 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -484,9 +484,10 @@ def __init__(
 
         if local_directory is None:
             local_directory = dask.config.get("temporary-directory") or os.getcwd()
-            if not os.path.exists(local_directory):
-                os.makedirs(local_directory)
-            local_directory = os.path.join(local_directory, "dask-worker-space")
+
+        if not os.path.exists(local_directory):
+            os.makedirs(local_directory)
+        local_directory = os.path.join(local_directory, "dask-worker-space")
 
         with warn_on_duration(
             "1s",

From 9cfc2ba80584238da9cc65e1f4cedb61c74e6d5d Mon Sep 17 00:00:00 2001
From: Alexander Clausen <alex@gc-web.de>
Date: Mon, 29 Jun 2020 21:34:41 +0200
Subject: [PATCH 0891/1550] Fix exception handling in `_wait_until_connected`
 (#3912)

In the nanny, if there is an exception in the `init_result_q`, the dict was raised, instead of only the exception, leading to a `TypeError: exceptions must derive from BaseException`
---
 distributed/nanny.py            |  2 +-
 distributed/tests/test_nanny.py | 17 +++++++++++++++++
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/distributed/nanny.py b/distributed/nanny.py
index 56b4b9d7883..84ce01ffebb 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -693,7 +693,7 @@ async def _wait_until_connected(self, uid):
                     "Failed while trying to start worker process: %s", msg["exception"]
                 )
                 await self.process.join()
-                raise msg
+                raise msg["exception"]
             else:
                 return msg
 
diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
index 327649570ed..88f401e2cf4 100644
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -557,3 +557,20 @@ async def test_nanny_closed_by_keyboard_interrupt(cleanup, protocol):
             await n.process.stopped.wait()
             # Check that the scheduler has been notified about the closed worker
             assert len(s.workers) == 0
+
+
+class StartException(Exception):
+    pass
+
+
+class BrokenWorker(worker.Worker):
+    async def start(self):
+        raise StartException("broken")
+
+
+@pytest.mark.asyncio
+async def test_worker_start_exception(cleanup):
+    # make sure this raises the right Exception:
+    with pytest.raises(StartException):
+        async with Nanny("tcp://localhost:1", worker_class=BrokenWorker) as n:
+            await n.start()

From 955b5683ec5f19207bdf49cb3522d7d285ffaed8 Mon Sep 17 00:00:00 2001
From: Tom Augspurger <TomAugspurger@users.noreply.github.com>
Date: Mon, 29 Jun 2020 15:18:33 -0500
Subject: [PATCH 0892/1550] Restore Scheduler.time_started for Dask Gateway
 (#3934)

Removed in https://github.com/dask/distributed/pull/3830.
---
 distributed/scheduler.py            | 1 +
 distributed/tests/test_scheduler.py | 5 +++++
 2 files changed, 6 insertions(+)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 3d47b88d226..cbb5b37ea66 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1117,6 +1117,7 @@ def __init__(
         else:
             self.idle_timeout = None
         self.idle_since = time()
+        self.time_started = self.idle_since  # compatibility for dask-gateway
         self._lock = asyncio.Lock()
         self.bandwidth = parse_bytes(dask.config.get("distributed.scheduler.bandwidth"))
         self.bandwidth_workers = defaultdict(float)
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index a8cdc18bd15..189ad107265 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -2147,3 +2147,8 @@ async def test_unknown_task_duration_config(client, s, a, b):
     assert len(s.unknown_durations) == 1
     await wait(future)
     assert len(s.unknown_durations) == 0
+
+
+@gen_cluster()
+async def test_unknown_task_duration_config(s, a, b):
+    assert s.idle_since == s.time_started

From 5c55601210b817cd89ac10bb5d5969c3f663417a Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Tue, 30 Jun 2020 07:27:31 -0700
Subject: [PATCH 0893/1550] Ensure `Worker.run*` handles `kwargs` correctly
 (#3937)

Make sure that `kwargs` is always an empty `dict` in `Worker.run*`
methods.
---
 distributed/worker.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/distributed/worker.py b/distributed/worker.py
index 187bcc76770..bff911e50ae 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -2362,7 +2362,6 @@ def executor_submit(self, key, function, args=(), kwargs=None, executor=None):
         raise gen.Return(result)
 
     def run(self, comm, function, args=(), wait=True, kwargs=None):
-        kwargs = kwargs or {}
         return run(self, comm, function=function, args=args, kwargs=kwargs, wait=wait)
 
     def run_coroutine(self, comm, function, args=(), kwargs=None, wait=True):
@@ -3524,7 +3523,8 @@ def weight(k, v):
     return sizeof(v)
 
 
-async def run(server, comm, function, args=(), kwargs={}, is_coro=None, wait=True):
+async def run(server, comm, function, args=(), kwargs=None, is_coro=None, wait=True):
+    kwargs = kwargs or {}
     function = pickle.loads(function)
     if is_coro is None:
         is_coro = iscoroutinefunction(function)

From e58116cf732031311870e1e3395276acadfad766 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 30 Jun 2020 07:28:18 -0700
Subject: [PATCH 0894/1550] Close a created cluster quietly (#3935)

Without this the user gets some scary warnings.

This happens when a cluster fails to start for some reason
---
 distributed/deploy/cluster.py                 | 12 +++++--
 distributed/deploy/spec.py                    | 31 ++++++++++---------
 distributed/deploy/tests/test_spec_cluster.py | 12 +++++++
 3 files changed, 37 insertions(+), 18 deletions(-)

diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
index 1f13fd37b10..b1f43473f87 100644
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -53,6 +53,8 @@ def __init__(self, asynchronous):
         self.scheduler_info = {"workers": {}}
         self.periodic_callbacks = {}
         self._asynchronous = asynchronous
+        self._watch_worker_status_comm = None
+        self._watch_worker_status_task = None
 
         self.status = "created"
 
@@ -70,12 +72,16 @@ async def _close(self):
         if self.status == "closed":
             return
 
-        await self._watch_worker_status_comm.close()
-        await self._watch_worker_status_task
+        if self._watch_worker_status_comm:
+            await self._watch_worker_status_comm.close()
+        if self._watch_worker_status_task:
+            await self._watch_worker_status_task
 
         for pc in self.periodic_callbacks.values():
             pc.stop()
-        await self.scheduler_comm.close_rpc()
+
+        if self.scheduler_comm:
+            await self.scheduler_comm.close_rpc()
 
         self.status = "closed"
 
diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index 93cf75bd295..325cf58eab7 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -375,22 +375,23 @@ async def _close(self):
             await asyncio.sleep(0.1)
         if self.status == "closed":
             return
-        self.status = "closing"
-
-        self.scale(0)
-        await self._correct_state()
-        for future in self._futures:
-            await future
-        async with self._lock:
-            with suppress(CommClosedError):
-                if self.scheduler_comm:
-                    await self.scheduler_comm.close(close_workers=True)
-                else:
-                    logger.warning("Cluster closed without starting up")
+        if self.status == "running":
+            self.status = "closing"
 
-        await self.scheduler.close()
-        for w in self._created:
-            assert w.status == Status.closed, w.status
+            self.scale(0)
+            await self._correct_state()
+            for future in self._futures:
+                await future
+            async with self._lock:
+                with suppress(CommClosedError):
+                    if self.scheduler_comm:
+                        await self.scheduler_comm.close(close_workers=True)
+                    else:
+                        logger.warning("Cluster closed without starting up")
+
+            await self.scheduler.close()
+            for w in self._created:
+                assert w.status == Status.closed, w.status
 
         if hasattr(self, "_old_logging_level"):
             silence_logging(self._old_logging_level)
diff --git a/distributed/deploy/tests/test_spec_cluster.py b/distributed/deploy/tests/test_spec_cluster.py
index 12d1c221942..f4d6c69827b 100644
--- a/distributed/deploy/tests/test_spec_cluster.py
+++ b/distributed/deploy/tests/test_spec_cluster.py
@@ -1,6 +1,7 @@
 import asyncio
 import re
 from time import sleep
+import warnings
 
 import dask
 from dask.distributed import SpecCluster, Worker, Client, Scheduler, Nanny
@@ -508,3 +509,14 @@ def _new_worker_name(self, worker_number):
         await cluster
         assert list(cluster.worker_spec) == worker_names
         assert sorted(list(cluster.workers)) == worker_names
+
+
+@pytest.mark.asyncio
+async def test_bad_close(cleanup):
+    with warnings.catch_warnings(record=True) as record:
+        cluster = SpecCluster(
+            workers=worker_spec, scheduler=scheduler, asynchronous=True
+        )
+        await cluster.close()
+
+    assert not record

From db5e88cd88a89cdb8420f95359090b94c1d341a3 Mon Sep 17 00:00:00 2001
From: Benjamin Zaitlen <quasiben@users.noreply.github.com>
Date: Wed, 1 Jul 2020 10:08:18 -0400
Subject: [PATCH 0895/1550] update logic for worker removal in check ttl
 (#3927)

---
 distributed/scheduler.py                 |  2 +-
 distributed/tests/test_failed_workers.py | 10 ++++++++--
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index cbb5b37ea66..4d22a320e5e 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -5307,7 +5307,7 @@ async def check_worker_ttl(self):
         now = time()
         for ws in self.workers.values():
             if (ws.last_seen < now - self.worker_ttl) and (
-                10 * heartbeat_interval(len(self.workers))
+                ws.last_seen < now - 10 * heartbeat_interval(len(self.workers))
             ):
                 logger.warning(
                     "Worker failed to heartbeat within %s seconds. Closing: %s",
diff --git a/distributed/tests/test_failed_workers.py b/distributed/tests/test_failed_workers.py
index 22d2a2ad91f..7c7b820a9ec 100644
--- a/distributed/tests/test_failed_workers.py
+++ b/distributed/tests/test_failed_workers.py
@@ -415,16 +415,22 @@ async def test_restart_timeout_on_long_running_task(c, s, a):
     assert "timeout" not in text.lower()
 
 
+@pytest.mark.slow
 @gen_cluster(client=True, scheduler_kwargs={"worker_ttl": "500ms"})
 async def test_worker_time_to_live(c, s, a, b):
+    from distributed.scheduler import heartbeat_interval
+
+    # worker removal is also controlled by 10 * heartbeat
     assert set(s.workers) == {a.address, b.address}
+    interval = 10 * heartbeat_interval(len(s.workers)) + 0.5
+
     a.periodic_callbacks["heartbeat"].stop()
     await asyncio.sleep(0.010)
     assert set(s.workers) == {a.address, b.address}
 
     start = time()
     while set(s.workers) == {a.address, b.address}:
-        await asyncio.sleep(0.050)
-        assert time() < start + 2
+        await asyncio.sleep(interval)
+        assert time() < start + interval + 0.1
 
     set(s.workers) == {b.address}

From d1c42db85181a078527aa1492e927972b997c21f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lo=C3=AFc=20Est=C3=A8ve?= <loic.esteve@ymail.com>
Date: Thu, 2 Jul 2020 16:00:04 +0200
Subject: [PATCH 0896/1550] Make dashboard server listens on all IPs by default
 even when interface is set explicitly. (#3941)

* Make ashboard server listens on all IPs by default.

When dashboard_address was `":8787"` http_address was `""`.

* Fix failing tests.

* Be less stringent in the test.

Some weird failures in py3.7 build ...

* Forgotten test.
---
 distributed/node.py                 | 2 +-
 distributed/tests/test_scheduler.py | 4 ++--
 distributed/tests/test_worker.py    | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/distributed/node.py b/distributed/node.py
index 52c53f3e9e7..997e6a1a988 100644
--- a/distributed/node.py
+++ b/distributed/node.py
@@ -121,7 +121,7 @@ def start_http_server(
         self.http_server = HTTPServer(self.http_application, ssl_options=ssl_options)
         http_address = clean_dashboard_address(dashboard_address or default_port)
 
-        if not http_address["address"]:
+        if http_address["address"] is None:
             address = self._start_address
             if isinstance(address, (list, tuple)):
                 address = address[0]
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 189ad107265..d6e67145136 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -1205,8 +1205,8 @@ async def test_service_hosts():
     port = 0
     for url, expected in [
         ("tcp://0.0.0.0", ("::", "0.0.0.0")),
-        ("tcp://127.0.0.1", "127.0.0.1"),
-        ("tcp://127.0.0.1:38275", "127.0.0.1"),
+        ("tcp://127.0.0.1", ("::", "0.0.0.0")),
+        ("tcp://127.0.0.1:38275", ("::", "0.0.0.0")),
     ]:
         async with Scheduler(host=url) as s:
             sock = first(s.http_server._sockets.values())
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index d2efe02196e..ede268abbcc 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -1042,7 +1042,7 @@ async def test_service_hosts_match_worker(s):
 
     async with Worker(s.address, host="tcp://127.0.0.1") as w:
         sock = first(w.http_server._sockets.values())
-        assert sock.getsockname()[0] == "127.0.0.1"
+        assert sock.getsockname()[0] in ("::", "0.0.0.0")
 
 
 @gen_cluster(nthreads=[])

From bf364ce954a4b1530d4c4d9ccae9a661ea074d2e Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Thu, 2 Jul 2020 12:44:50 -0700
Subject: [PATCH 0897/1550] Link issue on using `async` with `executor_submit`
 (#3939)

---
 distributed/worker.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/distributed/worker.py b/distributed/worker.py
index bff911e50ae..4c0a15870c9 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -2333,6 +2333,7 @@ def rescind_key(self, key):
     ################
 
     # FIXME: this breaks if changed to async def...
+    # xref: https://github.com/dask/distributed/issues/3938
     @gen.coroutine
     def executor_submit(self, key, function, args=(), kwargs=None, executor=None):
         """ Safely run function in thread pool executor

From 08d334e2e18bd977752eeab87e2c09272a2ac829 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Thu, 2 Jul 2020 23:23:16 -0500
Subject: [PATCH 0898/1550] bump version to 2.20.0

---
 docs/source/changelog.rst | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index d2a233e8ab5..9d6732940ee 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,23 @@
 Changelog
 =========
 
+2.20.0 - 2020-07-02
+-------------------
+
+- Link issue on using ``async`` with ``executor_submit`` (:pr:`3939`) `jakirkham`_
+- Make dashboard server listens on all IPs by default even when interface is set explicitly (:pr:`3941`) `Loïc Estève`_
+- Update logic for worker removal in check ttl (:pr:`3927`) `Benjamin Zaitlen`_
+- Close a created cluster quietly (:pr:`3935`) `Matthew Rocklin`_
+- Ensure ``Worker.run*`` handles ``kwargs`` correctly (:pr:`3937`) `jakirkham`_
+- Restore ``Scheduler.time_started`` for Dask Gateway (:pr:`3934`) `Tom Augspurger`_
+- Fix exception handling in ``_wait_until_connected`` (:pr:`3912`) `Alexander Clausen`_
+- Make local directory if it does not exist (:pr:`3928`) `Matthew Rocklin`_
+- Install vanilla status route if bokeh dependency is not satisfied (:pr:`3844`) `joshreback`_
+- Make ``Worker.delete_data`` sync (:pr:`3922`) `Peter Andreas Entschev`_
+- Fix ``ensure_bytes`` import location (:pr:`3919`) `jakirkham`_
+- Fix race condition in repeated calls to ``cluster.adapt()`` (:pr:`3915`) `Jacob Tomlinson`_
+
+
 2.19.0 - 2020-06-19
 -------------------
 
@@ -1826,3 +1843,4 @@ significantly without many new features.
 .. _`DomHudson`: https://github.com/DomHudson
 .. _`Julien Jerphanion`: https://github.com/jjerphan
 .. _`joshreback`: https://github.com/joshreback
+.. _`Alexander Clausen`: https://github.com/sk1p

From 8871c109680c5928144f40ac2d308ed59d291356 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 6 Jul 2020 14:13:08 -0700
Subject: [PATCH 0899/1550] Move scheduler_comm into Cluster.__init__ (#3945)

Following on from https://github.com/dask/distributed/pull/3935
This makes sure that not-yet-started clusters close quietly, even if
they are not SpecClusters
---
 distributed/deploy/cluster.py | 1 +
 distributed/deploy/spec.py    | 1 -
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
index b1f43473f87..3d23e62051b 100644
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -55,6 +55,7 @@ def __init__(self, asynchronous):
         self._asynchronous = asynchronous
         self._watch_worker_status_comm = None
         self._watch_worker_status_task = None
+        self.scheduler_comm = None
 
         self.status = "created"
 
diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index 325cf58eab7..6d9e1677a36 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -233,7 +233,6 @@ def __init__(
         self.workers = {}
         self._i = 0
         self.security = security or Security()
-        self.scheduler_comm = None
         self._futures = set()
 
         if silence_logs:

From 0565f7e5004d60eeda0a452996d774ec4711f5ed Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Mon, 6 Jul 2020 19:46:20 -0700
Subject: [PATCH 0900/1550] Serialize `memoryview`s with `shape` and `format`
 (#3947)

* Standardize frame coercion to `memoryview`

Ensure that `frames` are coerced to `memoryview`s of `uint8` type
whether there are one or multiple `frames`.

* Track `memoryview`'s `format` and `shape`

When serializing a `memoryview`, track its format and shape. Make sure
to reinstantiate the `memoryview` on the other end with that format and
shape.

It's worth noting that `memoryview`s don't actually allow `strides` to
be passed as well. So we skip this. As a result if a F-order
`memoryview` is needed, one will need to perform a copy to fix this.

* Raise if a `memoryview` with `object`s is given

Trying to turn a `memoryview` of `object`s into `bytes` doesn't work
particularly well as it will contain only the memory pointers of the
Python objects without their actual data. So they won't be serialized
correctly when going over to the receiver. There isn't a good
alternative solution in this case. So just raise.

* Test serializing `memoryview` of `object`s
---
 distributed/protocol/serialize.py            | 20 +++++++++++++++-----
 distributed/protocol/tests/test_serialize.py | 13 +++++++++++--
 2 files changed, 26 insertions(+), 7 deletions(-)

diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index 55230b4b3ce..6853d9afe6f 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -564,7 +564,7 @@ def normalize_Serialized(o):
 
 
 # Teach serialize how to handle bytestrings
-@dask_serialize.register((bytes, bytearray, memoryview))
+@dask_serialize.register((bytes, bytearray))
 def _serialize_bytes(obj):
     header = {}  # no special metadata
     frames = [obj]
@@ -576,13 +576,23 @@ def _deserialize_bytes(header, frames):
     return b"".join(frames)
 
 
+@dask_serialize.register(memoryview)
+def _serialize_memoryview(obj):
+    if obj.format == "O":
+        raise ValueError("Cannot serialize `memoryview` containing Python objects")
+    header = {"format": obj.format, "shape": obj.shape}
+    frames = [obj]
+    return header, frames
+
+
 @dask_deserialize.register(memoryview)
-def _serialize_memoryview(header, frames):
+def _deserialize_memoryview(header, frames):
     if len(frames) == 1:
-        out = frames[0]
+        out = memoryview(frames[0]).cast("B")
     else:
-        out = b"".join(frames)
-    return memoryview(out)
+        out = memoryview(b"".join(frames))
+    out = out.cast(header["format"], header["shape"])
+    return out
 
 
 #########################
diff --git a/distributed/protocol/tests/test_serialize.py b/distributed/protocol/tests/test_serialize.py
index 57fceaea0c9..f11186e1c70 100644
--- a/distributed/protocol/tests/test_serialize.py
+++ b/distributed/protocol/tests/test_serialize.py
@@ -420,6 +420,7 @@ def _(x):
         (tuple([MyObj(None)]), True),
         ({("x", i): MyObj(5) for i in range(100)}, True),
         (memoryview(b"hello"), True),
+        (memoryview(np.random.random((3, 4))), True),
     ],
 )
 def test_check_dask_serializable(data, is_serializable):
@@ -441,10 +442,18 @@ def test_serialize_lists(serializers):
     assert data_in == data_out
 
 
-def test_deser_memoryview():
-    data_in = memoryview(b"hello")
+@pytest.mark.parametrize(
+    "data_in", [memoryview(b"hello"), memoryview(np.random.random((3, 4)))],
+)
+def test_deser_memoryview(data_in):
     header, frames = serialize(data_in)
     assert header["type"] == "builtins.memoryview"
     assert frames[0] is data_in
     data_out = deserialize(header, frames)
     assert data_in == data_out
+
+
+def test_ser_memoryview_object():
+    data_in = memoryview(np.array(["hello"], dtype=object))
+    with pytest.raises(TypeError):
+        serialize(data_in, on_error="raise")

From 797981e90c95e2ea4f91c617d169923bff285b64 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Tue, 7 Jul 2020 05:31:01 -0700
Subject: [PATCH 0901/1550] Synchronize on non-trivial CUDA frame transmission
 (#3949)

* Simplify `send_frames` comprehension

Use the same trick we use in `recv_frames` to determine which frames
will be needed for sending.

* Sync when transmitting non-trivial CUDA frames

Previously we would synchronize when any CUDA frames were part of
transmission regardless of whether they would be transmitted or not.
However we know that trivial (`0`-length) frames are not transmitted. So
this change checks whether there are non-trivial CUDA frames that will
be transmitted. Only if non-trivial CUDA frames are found, does
synchronization occur.
---
 distributed/comm/ucx.py | 26 ++++++++++++++++----------
 1 file changed, 16 insertions(+), 10 deletions(-)

diff --git a/distributed/comm/ucx.py b/distributed/comm/ucx.py
index 54a85279ffd..bb48bc6a6f8 100644
--- a/distributed/comm/ucx.py
+++ b/distributed/comm/ucx.py
@@ -186,11 +186,13 @@ async def write(
                     hasattr(f, "__cuda_array_interface__") for f in frames
                 )
                 sizes = tuple(nbytes(f) for f in frames)
-                send_frames = [
-                    each_frame
-                    for each_frame, each_size in zip(frames, sizes)
-                    if each_size
-                ]
+                cuda_send_frames, send_frames = zip(
+                    *(
+                        (is_cuda, each_frame)
+                        for is_cuda, each_frame in zip(cuda_frames, frames)
+                        if len(each_frame) > 0
+                    )
+                )
 
                 # Send meta data
 
@@ -209,7 +211,7 @@ async def write(
                 #  syncing the default stream will wait for other non-blocking CUDA streams.
                 # Note this is only sufficient if the memory being sent is not currently in use on
                 # non-blocking CUDA streams.
-                if any(cuda_frames):
+                if any(cuda_send_frames):
                     synchronize_stream(0)
 
                 for each_frame in send_frames:
@@ -252,13 +254,17 @@ async def read(self, deserializers=("cuda", "dask", "pickle", "error")):
                     device_array(each_size) if is_cuda else host_array(each_size)
                     for is_cuda, each_size in zip(cuda_frames, sizes)
                 ]
-                recv_frames = [
-                    each_frame for each_frame in frames if len(each_frame) > 0
-                ]
+                cuda_recv_frames, recv_frames = zip(
+                    *(
+                        (is_cuda, each_frame)
+                        for is_cuda, each_frame in zip(cuda_frames, frames)
+                        if len(each_frame) > 0
+                    )
+                )
 
                 # It is necessary to first populate `frames` with CUDA arrays and synchronize
                 # the default stream before starting receiving to ensure buffers have been allocated
-                if any(cuda_frames):
+                if any(cuda_recv_frames):
                     synchronize_stream(0)
 
                 for each_frame in recv_frames:

From d332d27973f1bd5ee742be5974c0299a7e42706b Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 8 Jul 2020 05:07:38 -0700
Subject: [PATCH 0902/1550] Parse timeouts in Client.sync (#3952)

This allows for functionality like

```python
x.result(timeout="1 minute")
```
---
 distributed/client.py            | 1 +
 distributed/tests/test_client.py | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/distributed/client.py b/distributed/client.py
index 3e6b1e73679..402fc8e6fcd 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -818,6 +818,7 @@ def dashboard_link(self):
             return format_dashboard_link(host, port)
 
     def sync(self, func, *args, asynchronous=None, callback_timeout=None, **kwargs):
+        callback_timeout = parse_timedelta(callback_timeout)
         if (
             asynchronous
             or self.asynchronous
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 33440318a67..98f731e14de 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -512,7 +512,7 @@ def test_thread(c):
 
     x = c.submit(slowinc, 1, delay=0.3)
     with pytest.raises(TimeoutError):
-        x.result(timeout=0.01)
+        x.result(timeout="10 ms")
     assert x.result() == 2
 
 

From 97358361280034b0aac2b306138d5133ed7963cd Mon Sep 17 00:00:00 2001
From: Matthias Bussonnier <bussonniermatthias@gmail.com>
Date: Wed, 8 Jul 2020 05:24:59 -0700
Subject: [PATCH 0903/1550] Avoid too aggressive retry of connections. (#3944)

* Avoid too aggressive retry of connections.

Originally retrying was implemented to avoid timeout expiring for
addresses not yet existing (#3084). Though giving up after 1 second
maximum can be annoying for slow system and can trigger a thundering
herd-like problem.

Here progressively increase the inner timeout to give a chance to a slow
systems on th other side to respond especially when they are slow.

Here also jitter a bit when we are going to retry, and for how long
before we give up in the inner timeout to try-to again mitigate teh
thundering herd.

This is relatively difficult to test; so no tests are included.

* jitter initial timeout
---
 distributed/comm/core.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/distributed/comm/core.py b/distributed/comm/core.py
index 852cb991c4b..32af20c037d 100644
--- a/distributed/comm/core.py
+++ b/distributed/comm/core.py
@@ -3,6 +3,7 @@
 from contextlib import suppress
 import inspect
 import logging
+import random
 import weakref
 
 import dask
@@ -218,6 +219,8 @@ def _raise(error):
     if timeout and timeout / 20 < backoff:
         backoff = timeout / 20
 
+    retry_timeout_backoff = random.randrange(140, 160) / 100
+
     # This starts a thread
     while True:
         try:
@@ -227,7 +230,7 @@ def _raise(error):
                 )
                 with suppress(TimeoutError):
                     comm = await asyncio.wait_for(
-                        future, timeout=min(deadline - time(), 1)
+                        future, timeout=min(deadline - time(), retry_timeout_backoff)
                     )
                     break
             if not comm:
@@ -239,7 +242,8 @@ def _raise(error):
             if time() < deadline:
                 logger.debug("Could not connect, waiting before retrying")
                 await asyncio.sleep(backoff)
-                backoff *= 1.5
+                backoff *= random.randrange(140, 160) / 100
+                retry_timeout_backoff *= random.randrange(140, 160) / 100
                 backoff = min(backoff, 1)  # wait at most one second
             else:
                 _raise(error)

From 74bb1a73e94522ad0b090f603d864cb669fb2352 Mon Sep 17 00:00:00 2001
From: "Mads R. B. Kristensen" <madsbk@gmail.com>
Date: Wed, 8 Jul 2020 22:22:15 +0200
Subject: [PATCH 0904/1550] UCX: explicit shutdown message (#3950)

* ucx: writer sends -1 to reader on close

* Added closing flag to the initial message

* Update docs

Co-authored-by: Peter Andreas Entschev <peter@entschev.com>

Co-authored-by: Peter Andreas Entschev <peter@entschev.com>
---
 distributed/comm/ucx.py | 23 +++++++++++++----------
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/distributed/comm/ucx.py b/distributed/comm/ucx.py
index bb48bc6a6f8..fecfe062ba9 100644
--- a/distributed/comm/ucx.py
+++ b/distributed/comm/ucx.py
@@ -136,7 +136,7 @@ class UCX(Comm):
 
     The expected read cycle is
 
-    1. Read the frame describing number of frames
+    1. Read the frame describing if connection is closing and number of frames
     2. Read the frame describing whether each data frame is gpu-bound
     3. Read the frame describing whether each data frame is sized
     4. Read all the data frames.
@@ -196,8 +196,8 @@ async def write(
 
                 # Send meta data
 
-                # Send # of frames (uint64)
-                await self.ep.send(struct.pack("Q", nframes))
+                # Send close flag and number of frames (_Bool, int64)
+                await self.ep.send(struct.pack("?Q", False, nframes))
                 # Send which frames are CUDA (bool) and
                 # how large each frame is (uint64)
                 await self.ep.send(
@@ -232,11 +232,13 @@ async def read(self, deserializers=("cuda", "dask", "pickle", "error")):
             try:
                 # Recv meta data
 
-                # Recv # of frames (uint64)
-                nframes_fmt = "Q"
-                nframes = host_array(struct.calcsize(nframes_fmt))
-                await self.ep.recv(nframes)
-                (nframes,) = struct.unpack(nframes_fmt, nframes)
+                # Recv close flag and number of frames (_Bool, int64)
+                msg = host_array(struct.calcsize("?Q"))
+                await self.ep.recv(msg)
+                (shutdown, nframes) = struct.unpack("?Q", msg)
+
+                if shutdown:  # The writer is closing the connection
+                    raise CancelledError("Connection closed by writer")
 
                 # Recv which frames are CUDA (bool) and
                 # how large each frame is (uint64)
@@ -279,7 +281,8 @@ async def read(self, deserializers=("cuda", "dask", "pickle", "error")):
 
     async def close(self):
         if self._ep is not None:
-            await self._ep.close()
+            await self.ep.send(struct.pack("?Q", True, 0))
+            self.abort()
             self._ep = None
 
     def abort(self):
@@ -426,7 +429,7 @@ def _scrub_ucx_config():
 
     # configuration of UCX can happen in two ways:
     # 1) high level on/off flags which correspond to UCX configuration
-    # 2) explicity defined UCX configuration flags
+    # 2) explicitly defined UCX configuration flags
 
     # import does not initialize ucp -- this will occur outside this function
     from ucp import get_config

From a228e69d42031a84f74a221f815dc1cef2c906b7 Mon Sep 17 00:00:00 2001
From: Benjamin Zaitlen <quasiben@users.noreply.github.com>
Date: Wed, 8 Jul 2020 23:04:41 -0400
Subject: [PATCH 0905/1550] Update time per task chart with filtering and pie
 (#3933)

* format times correctly on compute/aggregate per key plots and small name change
* add pie chart
* format times for hover tooltip
* remove pie chart endpoint and use bokeh tab
* filter pie chart as well
* add note on missing tasks
* handle empty compute_times
---
 distributed/dashboard/components/scheduler.py | 135 ++++++++++++++----
 .../dashboard/tests/test_scheduler_bokeh.py   |   8 +-
 2 files changed, 113 insertions(+), 30 deletions(-)

diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index 2cf37a20800..2934b6e9848 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -26,12 +26,15 @@
     BoxSelectTool,
     GroupFilter,
     CDSView,
+    Tabs,
+    Panel,
+    Title,
 )
 from bokeh.models.widgets import DataTable, TableColumn
 from bokeh.plotting import figure
 from bokeh.palettes import Viridis11
 from bokeh.themes import Theme
-from bokeh.transform import factor_cmap, linear_cmap
+from bokeh.transform import factor_cmap, linear_cmap, cumsum
 from bokeh.io import curdoc
 import dask
 from dask import config
@@ -440,7 +443,7 @@ def name(address):
             update(self.source, result)
 
 
-class ComputerPerKey(DashboardComponent):
+class ComputePerKey(DashboardComponent):
     """ Bar chart showing time spend in action by key prefix"""
 
     def __init__(self, scheduler, **kwargs):
@@ -456,6 +459,8 @@ def __init__(self, scheduler, **kwargs):
 
             compute_data = {
                 "times": [0.2, 0.1],
+                "formatted_time": ["0.2 ms", "2.8 us"],
+                "angles": [3.14, 0.785],
                 "color": [ts_color_lookup["transfer"], ts_color_lookup["compute"]],
                 "names": ["sum", "sum_partial"],
             }
@@ -477,14 +482,13 @@ def __init__(self, scheduler, **kwargs):
                 top="times",
                 width=0.7,
                 color="color",
-                legend_field="names",
             )
 
             fig.y_range.start = 0
             fig.min_border_right = 20
             fig.min_border_bottom = 60
             fig.yaxis.axis_label = "Time (s)"
-            fig.yaxis[0].formatter = NumeralTickFormatter(format="0.0s")
+            fig.yaxis[0].formatter = NumeralTickFormatter(format="0")
             fig.yaxis.ticker = AdaptiveTicker(**TICKS_1024)
             fig.xaxis.major_label_orientation = -math.pi / 12
             rect.nonselection_glyph = None
@@ -499,13 +503,76 @@ def __init__(self, scheduler, **kwargs):
             hover.tooltips = """
             <div>
                 <p><b>Name:</b> @names</p>
-                <p><b>Time:</b> @times s</p>
+                <p><b>Time:</b> @formatted_time</p>
             </div>
             """
             hover.point_policy = "follow_mouse"
             fig.add_tools(hover)
 
+            fig.add_layout(
+                Title(
+                    text="Note: tasks less than 2% of max are not displayed",
+                    text_font_style="italic",
+                ),
+                "below",
+            )
+
             self.fig = fig
+            tab1 = Panel(child=fig, title="Bar Chart")
+
+            compute_wedge_data = {
+                "times": [0.2, 0.1],
+                "formatted_time": ["0.2 ms", "2.8 us"],
+                "angles": [1.4, 0.8],
+                "color": [ts_color_lookup["transfer"], ts_color_lookup["compute"]],
+                "names": ["sum", "sum_partial"],
+            }
+
+            fig2 = figure(
+                title="Compute Time Per Task",
+                tools="",
+                id="bk-Compute-by-key-pie",
+                name="compute_time_per_key-pie",
+                x_range=(-0.5, 1.0),
+                **kwargs,
+            )
+
+            wedge = fig2.wedge(
+                x=0,
+                y=1,
+                radius=0.4,
+                start_angle=cumsum("angles", include_zero=True),
+                end_angle=cumsum("angles"),
+                line_color="white",
+                fill_color="color",
+                legend_field="names",
+                source=self.compute_source,
+            )
+
+            fig2.axis.axis_label = None
+            fig2.axis.visible = False
+            fig2.grid.grid_line_color = None
+            fig2.add_layout(
+                Title(
+                    text="Note: tasks less than 2% of max are not displayed",
+                    text_font_style="italic",
+                ),
+                "below",
+            )
+
+            hover = HoverTool()
+            hover.tooltips = """
+            <div>
+                <p><b>Name:</b> @names</p>
+                <p><b>Time:</b> @formatted_time</p>
+            </div>
+            """
+            hover.point_policy = "follow_mouse"
+            fig2.add_tools(hover)
+            self.wedge_fig = fig2
+            tab2 = Panel(child=fig2, title="Pie Chart")
+
+            self.tabs = Tabs(tabs=[tab1, tab2])
 
     @without_property_validation
     def update(self):
@@ -523,22 +590,33 @@ def update(self):
                 compute_times.items(), key=lambda x: x[1], reverse=True
             )
 
-            compute_colors = list()
-            compute_names = list()
-            compute_time = list()
-            for name, t in compute_times:
-                compute_names.append(name)
-                compute_colors.append(ts_color_of(name))
-                compute_time.append(t)
-
-            self.fig.x_range.factors = compute_names
-            self.fig.title.text = "Compute Time Per Task"
-
-            compute_result = dict(
-                times=compute_time, color=compute_colors, names=compute_names
-            )
+            # keep only time which are 2% of max or greater
+            if compute_times:
+                max_time = compute_times[0][1] * 0.02
+                compute_times = [(n, t) for n, t in compute_times if t > max_time]
+                compute_colors = list()
+                compute_names = list()
+                compute_time = list()
+                total_time = 0
+                for name, t in compute_times:
+                    compute_names.append(name)
+                    compute_colors.append(ts_color_of(name))
+                    compute_time.append(t)
+                    total_time += t
+
+                angles = [t / total_time * 2 * math.pi for t in compute_time]
+
+                self.fig.x_range.factors = compute_names
+
+                compute_result = dict(
+                    angles=angles,
+                    times=compute_time,
+                    color=compute_colors,
+                    names=compute_names,
+                    formatted_time=[format_time(t) for t in compute_time],
+                )
 
-            update(self.compute_source, compute_result)
+                update(self.compute_source, compute_result)
 
 
 class AggregateAction(DashboardComponent):
@@ -557,6 +635,7 @@ def __init__(self, scheduler, **kwargs):
 
             action_data = {
                 "times": [0.2, 0.1],
+                "formatted_time": ["0.2 ms", "2.8 us"],
                 "color": [ts_color_lookup["transfer"], ts_color_lookup["compute"]],
                 "names": ["transfer", "compute"],
             }
@@ -578,13 +657,12 @@ def __init__(self, scheduler, **kwargs):
                 top="times",
                 width=0.7,
                 color="color",
-                legend_field="names",
             )
 
             fig.y_range.start = 0
             fig.min_border_right = 20
             fig.min_border_bottom = 60
-            fig.yaxis[0].formatter = NumeralTickFormatter(format="0.0s")
+            fig.yaxis[0].formatter = NumeralTickFormatter(format="0")
             fig.yaxis.axis_label = "Time (s)"
             fig.yaxis.ticker = AdaptiveTicker(**TICKS_1024)
             fig.xaxis.major_label_orientation = -math.pi / 12
@@ -601,7 +679,7 @@ def __init__(self, scheduler, **kwargs):
             hover.tooltips = """
             <div>
                 <p><b>Name:</b> @names</p>
-                <p><b>Time:</b> @times s</p>
+                <p><b>Time:</b> @formatted_time</p>
             </div>
             """
             hover.point_policy = "follow_mouse"
@@ -635,7 +713,12 @@ def update(self):
             self.fig.x_range.factors = agg_names
             self.fig.title.text = "Aggregate Time Per Action"
 
-            action_result = dict(times=agg_time, color=agg_colors, names=agg_names)
+            action_result = dict(
+                times=agg_time,
+                color=agg_colors,
+                names=agg_names,
+                formatted_time=[format_time(t) for t in agg_time],
+            )
 
             update(self.action_source, action_result)
 
@@ -2129,10 +2212,10 @@ def individual_memory_by_key_doc(scheduler, extra, doc):
 
 def individual_compute_time_per_key_doc(scheduler, extra, doc):
     with log_errors():
-        component = ComputerPerKey(scheduler, sizing_mode="stretch_both")
+        component = ComputePerKey(scheduler, sizing_mode="stretch_both")
         component.update()
         add_periodic_callback(doc, component, 500)
-        doc.add_root(component.fig)
+        doc.add_root(component.tabs)
         doc.theme = BOKEH_THEME
 
 
diff --git a/distributed/dashboard/tests/test_scheduler_bokeh.py b/distributed/dashboard/tests/test_scheduler_bokeh.py
index 1207b47052a..0561cd5fbbc 100644
--- a/distributed/dashboard/tests/test_scheduler_bokeh.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh.py
@@ -36,7 +36,7 @@
     ProfileServer,
     MemoryByKey,
     AggregateAction,
-    ComputerPerKey,
+    ComputePerKey,
 )
 from distributed.dashboard import scheduler
 
@@ -741,8 +741,8 @@ async def test_aggregate_action(c, s, a, b):
 
 
 @gen_cluster(client=True, scheduler_kwargs={"dashboard": True})
-async def test_computer_per_key(c, s, a, b):
-    mbk = ComputerPerKey(s)
+async def test_compute_per_key(c, s, a, b):
+    mbk = ComputePerKey(s)
 
     da = pytest.importorskip("dask.array")
     x = (da.ones((20, 20), chunks=(10, 10)) + 1).persist(optimize_graph=False)
@@ -759,8 +759,8 @@ async def test_computer_per_key(c, s, a, b):
     )
     assert response.code == 200
     assert ("sum-aggregate") in mbk.compute_source.data["names"]
-    assert ("inc") in mbk.compute_source.data["names"]
     assert ("add") in mbk.compute_source.data["names"]
+    assert "angles" in mbk.compute_source.data.keys()
 
 
 @gen_cluster(scheduler_kwargs={"http_prefix": "foo-bar", "dashboard": True})

From a8a7586507fb73a08d5a80a1bd79a35ec4af3794 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Fri, 10 Jul 2020 16:53:28 -0700
Subject: [PATCH 0906/1550] Optionally use pickle5 (#3849)

Optionally use pickle5
---
 continuous_integration/travis/install.sh  |  6 ++++--
 distributed/protocol/pickle.py            | 10 +++++++++-
 distributed/protocol/tests/test_pickle.py |  9 ++++++++-
 distributed/scheduler.py                  | 10 +++++++++-
 distributed/tests/test_scheduler.py       |  9 ++++++++-
 requirements.txt                          |  2 +-
 6 files changed, 39 insertions(+), 7 deletions(-)

diff --git a/continuous_integration/travis/install.sh b/continuous_integration/travis/install.sh
index b1dfbc9ce1c..be6711f890d 100644
--- a/continuous_integration/travis/install.sh
+++ b/continuous_integration/travis/install.sh
@@ -71,9 +71,11 @@ if [[ $PYTHON == 3.6 ]]; then
   conda install -c conda-forge -c defaults contextvars
 fi
 
-# stacktrace is not currently avaiable for Python 3.8.
-# Remove the version check block below when it is avaiable.
 if [[ $PYTHON != 3.8 ]]; then
+    # Install backport package for pickle protocol 5 support
+    conda install -c conda-forge -c defaults 'pickle5>=0.0.11'
+    # stacktrace is not currently avaiable for Python 3.8.
+    # Remove the version check block below when it is avaiable.
     # For low-level profiler, install libunwind and stacktrace from conda-forge
     # For stacktrace we use --no-deps to avoid upgrade of python
     conda install -c conda-forge -c defaults libunwind
diff --git a/distributed/protocol/pickle.py b/distributed/protocol/pickle.py
index 6e168947d48..fd2343756a4 100644
--- a/distributed/protocol/pickle.py
+++ b/distributed/protocol/pickle.py
@@ -1,8 +1,16 @@
 import logging
-import pickle
+import sys
 
 import cloudpickle
 
+if sys.version_info < (3, 8):
+    try:
+        import pickle5 as pickle
+    except ImportError:
+        import pickle
+else:
+    import pickle
+
 
 HIGHEST_PROTOCOL = pickle.HIGHEST_PROTOCOL
 
diff --git a/distributed/protocol/tests/test_pickle.py b/distributed/protocol/tests/test_pickle.py
index 9ee496f5e9f..ea2143c5358 100644
--- a/distributed/protocol/tests/test_pickle.py
+++ b/distributed/protocol/tests/test_pickle.py
@@ -1,7 +1,6 @@
 from functools import partial
 import gc
 from operator import add
-import pickle
 import weakref
 import sys
 
@@ -10,6 +9,14 @@
 from distributed.protocol import deserialize, serialize
 from distributed.protocol.pickle import HIGHEST_PROTOCOL, dumps, loads
 
+if sys.version_info < (3, 8):
+    try:
+        import pickle5 as pickle
+    except ImportError:
+        import pickle
+else:
+    import pickle
+
 
 def test_pickle_data():
     data = [1, b"123", "123", [123], {}, set()]
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 4d22a320e5e..f96385656e2 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -12,7 +12,7 @@
 from numbers import Number
 import operator
 import os
-import pickle
+import sys
 import random
 import warnings
 import weakref
@@ -86,6 +86,14 @@
 from .stealing import WorkStealing
 from .variable import VariableExtension
 
+if sys.version_info < (3, 8):
+    try:
+        import pickle5 as pickle
+    except ImportError:
+        import pickle
+else:
+    import pickle
+
 
 logger = logging.getLogger(__name__)
 
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index d6e67145136..7537d0d63e3 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -1,7 +1,6 @@
 import asyncio
 import json
 import logging
-import pickle
 import operator
 import re
 import sys
@@ -42,6 +41,14 @@
 from distributed.utils_test import loop, nodebug  # noqa: F401
 from dask.compatibility import apply
 
+if sys.version_info < (3, 8):
+    try:
+        import pickle5 as pickle
+    except ImportError:
+        import pickle
+else:
+    import pickle
+
 
 alice = "alice:1234"
 bob = "bob:1234"
diff --git a/requirements.txt b/requirements.txt
index 95a681d66c2..516ceb62a8a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,5 @@
 click >= 6.6
-cloudpickle >= 1.3.0
+cloudpickle >= 1.5.0
 contextvars;python_version<'3.7'
 dask >= 2.9.0
 msgpack >= 0.6.0

From d9da4d79cafa1e5f46058b45f2e8209c99590506 Mon Sep 17 00:00:00 2001
From: "Mads R. B. Kristensen" <madsbk@gmail.com>
Date: Mon, 13 Jul 2020 21:05:57 +0200
Subject: [PATCH 0907/1550] UCX: reuse endpoints in order to fix NVLINK issue
 (#3953)

* ucx: writer sends -1 to reader on close

* Added closing flag to the initial message

* Update docs

Co-authored-by: Peter Andreas Entschev <peter@entschev.com>

* Use EndpointReuse from UCX-Py if available

* ucx: added the config option "ucx.reuse-endpoints"

* reformat

* ucx: moved reuse check inside init_once()

* Added reuse-endpoints to config schema

* Rely on default from `dask.config`

Co-authored-by: Peter Andreas Entschev <peter@entschev.com>
Co-authored-by: jakirkham <jakirkham@gmail.com>
---
 distributed/comm/ucx.py             | 21 ++++++++++++++++++---
 distributed/distributed-schema.yaml |  3 +++
 distributed/distributed.yaml        |  1 +
 3 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/distributed/comm/ucx.py b/distributed/comm/ucx.py
index fecfe062ba9..d068ca4e79e 100644
--- a/distributed/comm/ucx.py
+++ b/distributed/comm/ucx.py
@@ -35,6 +35,8 @@
 ucp = None
 host_array = None
 device_array = None
+ucx_create_endpoint = None
+ucx_create_listener = None
 
 
 def synchronize_stream(stream=0):
@@ -47,7 +49,7 @@ def synchronize_stream(stream=0):
 
 
 def init_once():
-    global ucp, host_array, device_array
+    global ucp, host_array, device_array, ucx_create_endpoint, ucx_create_listener
     if ucp is not None:
         return
 
@@ -107,6 +109,19 @@ def device_array(n):
             pool_allocator=True, managed_memory=False, initial_pool_size=pool_size
         )
 
+    try:
+        from ucp.endpoint_reuse import EndpointReuse
+    except ImportError:
+        ucx_create_endpoint = ucp.create_endpoint
+        ucx_create_listener = ucp.create_listener
+    else:
+        if dask.config.get("ucx.reuse-endpoints"):
+            ucx_create_endpoint = EndpointReuse.create_endpoint
+            ucx_create_listener = EndpointReuse.create_listener
+        else:
+            ucx_create_endpoint = ucp.create_endpoint
+            ucx_create_listener = ucp.create_listener
+
 
 class UCX(Comm):
     """Comm object using UCP.
@@ -310,7 +325,7 @@ async def connect(self, address: str, deserialize=True, **connection_args) -> UC
         logger.debug("UCXConnector.connect: %s", address)
         ip, port = parse_host_port(address)
         init_once()
-        ep = await ucp.create_endpoint(ip, port)
+        ep = await ucx_create_endpoint(ip, port)
         return self.comm_class(
             ep,
             local_addr=None,
@@ -363,7 +378,7 @@ async def serve_forever(client_ep):
                 await self.comm_handler(ucx)
 
         init_once()
-        self.ucp_server = ucp.create_listener(serve_forever, port=self._input_port)
+        self.ucp_server = ucx_create_listener(serve_forever, port=self._input_port)
 
     def stop(self):
         self.ucp_server = None
diff --git a/distributed/distributed-schema.yaml b/distributed/distributed-schema.yaml
index 1a1f2566a63..f67cdca84f2 100644
--- a/distributed/distributed-schema.yaml
+++ b/distributed/distributed-schema.yaml
@@ -800,3 +800,6 @@ properties:
         - string
         - "null"
         description: Define which Infiniband device to use
+      reuse-endpoints:
+        type: boolean
+        description: Whether to reuse endpoints or not, default True
diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index cbf66dc220d..f815fadf830 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -179,3 +179,4 @@ ucx:
   rdmacm: null # enable RDMACM
   cuda_copy: null  # enable cuda-copy
   net-devices: null  # define which Infiniband device to use
+  reuse-endpoints: True  # enable endpoint reuse

From 1d921255e211fe5532e45b9fde13afb2c5ee4336 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Tue, 14 Jul 2020 10:05:42 -0700
Subject: [PATCH 0908/1550] Skip coercing to `bytes` in `merge_frames` (#3960)

As the frames we receive are typically mutable, non-`bytes` objects like
`bytearray`s or NumPy `ndarray`s, coercing to `bytes` at this stage
triggers a copy of all frames. As we are going to toss those copied
versions anyways when joining them into a larger `bytes` object, this
ends up being wasteful with memory. Fortunately `bytes.join(...)`
accepts any and all `bytes`-like objects. So instead just pass them all
through as-is to `bytes.join(...)`, which is free and doesn't require a
copy.  Should cutdown on the memory usage in this part of the code.
---
 distributed/protocol/utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/distributed/protocol/utils.py b/distributed/protocol/utils.py
index fa020dae909..defbda2ba4f 100644
--- a/distributed/protocol/utils.py
+++ b/distributed/protocol/utils.py
@@ -1,7 +1,7 @@
 import struct
 import msgpack
 
-from ..utils import ensure_bytes, nbytes
+from ..utils import nbytes
 
 BIG_BYTES_SHARD_SIZE = 2 ** 26
 
@@ -84,7 +84,7 @@ def merge_frames(header, frames):
         if len(L) == 1:  # no work necessary
             out.extend(L)
         else:
-            out.append(b"".join(map(ensure_bytes, L)))
+            out.append(b"".join(L))
     return out
 
 

From 8b96c00578a4fcdf6ed2927766f4d695b17413c5 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 14 Jul 2020 15:12:11 -0700
Subject: [PATCH 0909/1550] Err hard when Dask has mismatched versions or lz4
 installed (#3936)

Fixes https://github.com/dask/distributed/issues/3932
---
 distributed/client.py              | 13 +++++--
 distributed/scheduler.py           |  6 +--
 distributed/tests/test_versions.py | 60 ++++++++++++++++++++++++------
 distributed/versions.py            | 34 ++++++++++++++---
 4 files changed, 90 insertions(+), 23 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 402fc8e6fcd..b6ae437cef4 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -1044,7 +1044,7 @@ async def _start(self, timeout=no_default, **kwargs):
 
         try:
             await self._ensure_connected(timeout=timeout)
-        except OSError:
+        except (OSError, ImportError):
             await self._close()
             raise
 
@@ -1077,6 +1077,9 @@ async def _reconnect(self):
                     # Wait a bit before retrying
                     await asyncio.sleep(0.1)
                     timeout = deadline - self.loop.time()
+                except ImportError:
+                    await self._close()
+                    break
             else:
                 logger.error(
                     "Failed to reconnect to scheduler after %.2f "
@@ -1127,6 +1130,8 @@ async def _ensure_connected(self, timeout=None):
         assert len(msg) == 1
         assert msg[0]["op"] == "stream-start"
 
+        if msg[0].get("error"):
+            raise ImportError(msg[0]["error"])
         if msg[0].get("warning"):
             warnings.warn(version_module.VersionMismatchWarning(msg[0]["warning"]))
 
@@ -3682,8 +3687,10 @@ async def _get_versions(self, check=False, packages=[]):
 
         if check:
             msg = version_module.error_message(scheduler, workers, client)
-            if msg:
-                raise ValueError(msg)
+            if msg["warning"]:
+                warnings.warn(msg["warning"])
+            if msg["error"]:
+                raise ValueError(msg["error"])
 
         return result
 
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index f96385656e2..5081ed426f3 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1812,8 +1812,7 @@ async def add_worker(
                 versions,
                 client_name="This Worker",
             )
-            if version_warning:
-                msg["warning"] = version_warning
+            msg.update(version_warning)
 
             if comm:
                 await comm.write(msg)
@@ -2577,8 +2576,7 @@ async def add_client(self, comm, client=None, versions=None):
                 {w: ws.versions for w, ws in self.workers.items()},
                 versions,
             )
-            if version_warning:
-                msg["warning"] = version_warning
+            msg.update(version_warning)
             bcomm.send(msg)
 
             try:
diff --git a/distributed/tests/test_versions.py b/distributed/tests/test_versions.py
index 0a095debeec..ee313031655 100644
--- a/distributed/tests/test_versions.py
+++ b/distributed/tests/test_versions.py
@@ -2,10 +2,11 @@
 import sys
 
 import pytest
+from toolz import first
 
 from distributed.versions import get_versions, error_message
-from distributed import Client, Worker
-from distributed.utils_test import gen_cluster
+from distributed import Client, Worker, LocalCluster
+from distributed.utils_test import gen_cluster, loop  # noqa: F401
 
 
 # if one of the nodes reports this version, there's a mismatch
@@ -34,7 +35,7 @@ def kwargs_matching():
 
 
 def test_versions_match(kwargs_matching):
-    assert error_message(**kwargs_matching) == ""
+    assert error_message(**kwargs_matching)["warning"] == ""
 
 
 @pytest.fixture(params=["client", "scheduler", "worker-1"])
@@ -80,11 +81,13 @@ def test_version_mismatch(node, effect, kwargs_not_matching, pattern):
     column_matching = {"client": 1, "scheduler": 2, "workers": 3}
     msg = error_message(**kwargs_not_matching)
     i = column_matching.get(node, 3)
-    assert "Mismatched versions found" in msg
-    assert "distributed" in msg
+    assert "Mismatched versions found" in msg["warning"]
+    assert "distributed" in msg["warning"]
     assert (
         pattern
-        in re.search(r"distributed\s+(?:(?:\|[^|\r\n]*)+\|(?:\r?\n|\r)?)+", msg)
+        in re.search(
+            r"distributed\s+(?:(?:\|[^|\r\n]*)+\|(?:\r?\n|\r)?)+", msg["warning"]
+        )
         .group(0)
         .split("|")[i]
         .strip()
@@ -96,24 +99,24 @@ def test_scheduler_mismatched_irrelevant_package(kwargs_matching):
     kwargs_matching["scheduler"]["packages"]["numpy"] = "0.0.0"
     assert "numpy" in kwargs_matching["client"]["packages"]
 
-    assert error_message(**kwargs_matching) == ""
+    assert error_message(**kwargs_matching)["warning"] == ""
 
 
 def test_scheduler_additional_irrelevant_package(kwargs_matching):
     """An irrelevant package on the scheduler does not need to be present elsewhere."""
     kwargs_matching["scheduler"]["packages"]["pyspark"] = "0.0.0"
 
-    assert error_message(**kwargs_matching) == ""
+    assert error_message(**kwargs_matching)["warning"] == ""
 
 
 def test_python_mismatch(kwargs_matching):
     kwargs_matching["client"]["packages"]["python"] = "0.0.0"
     msg = error_message(**kwargs_matching)
-    assert "Mismatched versions found" in msg
-    assert "python" in msg
+    assert "Mismatched versions found" in msg["warning"]
+    assert "python" in msg["warning"]
     assert (
         "0.0.0"
-        in re.search(r"python\s+(?:(?:\|[^|\r\n]*)+\|(?:\r?\n|\r)?)+", msg)
+        in re.search(r"python\s+(?:(?:\|[^|\r\n]*)+\|(?:\r?\n|\r)?)+", msg["warning"])
         .group(0)
         .split("|")[1]
         .strip()
@@ -142,3 +145,38 @@ def test_python_version():
     required = get_versions()["packages"]
     assert "python" in required
     assert required["python"] == ".".join(map(str, sys.version_info))
+
+
+def test_python_version_error(loop):
+
+    with LocalCluster(1, processes=False, silence_logs=False, loop=loop,) as cluster:
+        first(cluster.scheduler.workers.values()).versions["packages"][
+            "python"
+        ] = "3.5.1"
+        with pytest.raises(ImportError) as info:
+            with Client(cluster):
+                pass
+
+    assert "Python" in str(info.value)
+    assert "major" in str(info.value).lower()
+
+
+def test_lz4_version_error(loop):
+
+    with LocalCluster(
+        1, processes=False, silence_logs=False, dashboard_address=None, loop=loop,
+    ) as cluster:
+        try:
+            import lz4  # noqa: F401
+
+            first(cluster.scheduler.workers.values()).versions["packages"]["lz4"] = None
+        except ImportError:
+            first(cluster.scheduler.workers.values()).versions["packages"][
+                "lz4"
+            ] = "1.0.0"
+
+        with pytest.raises(ImportError) as info:
+            with Client(cluster):
+                pass
+
+    assert "lz4" in str(info.value)
diff --git a/distributed/versions.py b/distributed/versions.py
index 0e49f079d33..6f2b92f46f8 100644
--- a/distributed/versions.py
+++ b/distributed/versions.py
@@ -34,8 +34,6 @@
 # notes to be displayed for mismatch packages
 notes_mismatch_package = {
     "msgpack": "Variation is ok, as long as everything is above 0.6",
-    "lz4": "Variation is ok, but missing libraries are not",
-    "python": "Variation is sometimes ok, sometimes not. It depends on your workloads",
 }
 
 
@@ -157,14 +155,40 @@ def error_message(scheduler, workers, client, client_name="client"):
         if pkg in notes_mismatch_package.keys():
             notes.append(f"-  {pkg}: {notes_mismatch_package[pkg]}")
 
+    out = {"warning": "", "error": ""}
+
     if errs:
         err_table = asciitable(["Package", client_name, "scheduler", "workers"], errs)
         err_msg = f"Mismatched versions found\n\n{err_table}"
         if notes:
             err_msg += "\nNotes: \n{}".format("\n".join(notes))
-        return err_msg
-    else:
-        return ""
+        out["warning"] += err_msg
+
+        for name, c, s, ws in errs:
+            if not isinstance(ws, set):
+                ws = {ws}
+
+            if name == "python":
+                majors = [tuple(version.split(".")[:2]) for version in {c, s} | ws]
+                if len(set(majors)) != 1:
+                    err_table = asciitable(
+                        ["Package", client_name, "scheduler", "workers"],
+                        [t for t in errs if t[0] == "python"],
+                    )
+                    out["error"] += f"Python major versions must match\n\n{err_table}\n"
+
+            if name == "lz4":
+                versions = [version for version in {c, s} | ws]
+                if any(versions) and not all(versions):
+                    err_table = asciitable(
+                        ["Package", client_name, "scheduler", "workers"],
+                        [t for t in errs if t[0] == "lz4"],
+                    )
+                    out[
+                        "error"
+                    ] += f"\nLZ4 must be installed everywhere or nowhere\n\n{err_table}\n"
+
+    return out
 
 
 class VersionMismatchWarning(Warning):

From 3c4a98f8941d0a7135711be7ffb4ba415db3eddb Mon Sep 17 00:00:00 2001
From: Gil Forsyth <gforsyth@users.noreply.github.com>
Date: Wed, 15 Jul 2020 10:36:36 -0400
Subject: [PATCH 0910/1550] Unpin `numpydoc` now that 1.1 is released (#3957)

Fix for errant backslashes was fixed upstream and released.
https://numpydoc.readthedocs.io/en/latest/release_notes.html#fixed-bugs
numpy/numpydoc#218
---
 docs/requirements.txt | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/docs/requirements.txt b/docs/requirements.txt
index 6bcd69b284d..965f4fc1f9a 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -1,5 +1,4 @@
-# We pin numpydoc to avoid doubly-escaped *args and **kwargs in rendered docs
-numpydoc==0.8.0
+numpydoc
 tornado
 toolz
 cloudpickle

From 1936af5e3f730b44dbc5477418dfb929912322a9 Mon Sep 17 00:00:00 2001
From: Tom Augspurger <TomAugspurger@users.noreply.github.com>
Date: Wed, 15 Jul 2020 09:37:21 -0500
Subject: [PATCH 0911/1550] Treat falsey local directory as None (#3964)

Closes https://github.com/pangeo-data/pangeo-cloud-federation/issues/655

Dask-Gateway uses local-directory="" to launch workers. I think the
intent is that it's interpreted as False, and should fall back to the
config / current directory.
---
 distributed/tests/test_worker.py | 9 +++++++++
 distributed/worker.py            | 2 +-
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index ede268abbcc..268ade2602f 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -865,6 +865,15 @@ async def test_worker_dir(c, s, a, b):
         test_worker_dir()
 
 
+@gen_cluster(nthreads=[])
+async def test_false_worker_dir(s):
+    async with Worker(s.address, local_directory="") as w:
+        local_directory = w.local_directory
+
+    cwd = os.getcwd()
+    assert os.path.dirname(local_directory) == os.path.join(cwd, "dask-worker-space")
+
+
 @gen_cluster(client=True)
 async def test_dataframe_attribute_error(c, s, a, b):
     class BadSize:
diff --git a/distributed/worker.py b/distributed/worker.py
index 4c0a15870c9..02299dae28e 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -482,7 +482,7 @@ def __init__(
             warnings.warn("The local_dir keyword has moved to local_directory")
             local_directory = local_dir
 
-        if local_directory is None:
+        if not local_directory:
             local_directory = dask.config.get("temporary-directory") or os.getcwd()
 
         if not os.path.exists(local_directory):

From ea282c54d63e333447e9065bc20c5e8c3c20c2b0 Mon Sep 17 00:00:00 2001
From: Andrew Fulton <andrewfulton9@gmail.com>
Date: Thu, 16 Jul 2020 02:35:40 -0600
Subject: [PATCH 0912/1550] Fix data replication error (#3963)

* working test for replication

* debug prints

* adds test for retiring workers that fails because tasks get garbage collected before being replicated to another machine

* fixes replication issue by removing forgotten tasks from task replication list

* remove debugging stuff

* black and flake8

* removes junk I added
---
 distributed/scheduler.py            |  5 ++++-
 distributed/tests/test_scheduler.py | 14 ++++++++++++++
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 5081ed426f3..8c469d2ed38 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -16,7 +16,6 @@
 import random
 import warnings
 import weakref
-
 import psutil
 import sortedcontainers
 
@@ -3266,6 +3265,10 @@ async def replicate(
             while tasks:
                 gathers = defaultdict(dict)
                 for ts in list(tasks):
+                    if ts.state == "forgotten":
+                        # task is no longer needed by any client or dependant task
+                        tasks.remove(ts)
+                        continue
                     n_missing = n - len(ts.who_has & workers)
                     if n_missing <= 0:
                         # Already replicated enough
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 7537d0d63e3..a55a7213919 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -2159,3 +2159,17 @@ async def test_unknown_task_duration_config(client, s, a, b):
 @gen_cluster()
 async def test_unknown_task_duration_config(s, a, b):
     assert s.idle_since == s.time_started
+
+
+@gen_cluster(client=True, timeout=1000)
+async def test_retire_state_change(c, s, a, b):
+    np = pytest.importorskip("numpy")
+    y = c.map(lambda x: x ** 2, range(10))
+    await c.scatter(y)
+    for x in range(2):
+        v = c.map(lambda i: i * np.random.randint(1000), y)
+        k = c.map(lambda i: i * np.random.randint(1000), v)
+        foo = c.map(lambda j: j * 6, k)
+        step = c.compute(foo)
+        c.gather(step)
+    await c.retire_workers(workers=[a.address])

From ef168a471a7b6b4dc4b023196afc229cda109608 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Fri, 17 Jul 2020 17:17:21 -0500
Subject: [PATCH 0913/1550] bump version to 2.21.0

---
 docs/source/changelog.rst | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index 9d6732940ee..a1202c94d83 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,25 @@
 Changelog
 =========
 
+2.21.0 - 2020-07-17
+-------------------
+
+- Fix data replication error (:pr:`3963`) `Andrew Fulton`_
+- Treat falsey local directory as ``None`` (:pr:`3964`) `Tom Augspurger`_
+- Unpin ``numpydoc`` now that 1.1 is released (:pr:`3957`) `Gil Forsyth`_
+- Error hard when Dask has mismatched versions or lz4 installed (:pr:`3936`) `Matthew Rocklin`_
+- Skip coercing to ``bytes`` in ``merge_frames`` (:pr:`3960`) `jakirkham`_
+- UCX: reuse endpoints in order to fix NVLINK issue (:pr:`3953`) `Mads R. B. Kristensen`_
+- Optionally use ``pickle5`` (:pr:`3849`) `jakirkham`_
+- Update time per task chart with filtering and pie (:pr:`3933`) `Benjamin Zaitlen`_
+- UCX: explicit shutdown message (:pr:`3950`) `Mads R. B. Kristensen`_
+- Avoid too aggressive retry of connections (:pr:`3944`) `Matthias Bussonnier`_
+- Parse timeouts in ``Client.sync`` (:pr:`3952`) `Matthew Rocklin`_
+- Synchronize on non-trivial CUDA frame transmission (:pr:`3949`) `jakirkham`_
+- Serialize ``memoryview`` with ``shape`` and ``format`` (:pr:`3947`) `jakirkham`_
+- Move ``scheduler_comm`` into ``Cluster.__init__`` (:pr:`3945`) `Matthew Rocklin`_
+
+
 2.20.0 - 2020-07-02
 -------------------
 
@@ -1844,3 +1863,4 @@ significantly without many new features.
 .. _`Julien Jerphanion`: https://github.com/jjerphan
 .. _`joshreback`: https://github.com/joshreback
 .. _`Alexander Clausen`: https://github.com/sk1p
+.. _`Andrew Fulton`: https://github.com/andrewfulton9

From c67705f3f513de5bc09b897c400011b543ff0f7c Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Fri, 17 Jul 2020 16:21:02 -0700
Subject: [PATCH 0914/1550] Ensure writable frames (#3967)

User code working with NumPy or Pandas objects often expects the objects
to be mutable. However if read-only frames (like `bytes`) objects are
used, this is not true. So add a test to check for this so that we can
make sure this is true and we can catch and fix cases where that may not
be true.
---
 distributed/protocol/tests/test_numpy.py      |  8 +++
 .../protocol/tests/test_protocol_utils.py     | 34 +++++++++----
 distributed/protocol/utils.py                 | 50 ++++++++++---------
 3 files changed, 58 insertions(+), 34 deletions(-)

diff --git a/distributed/protocol/tests/test_numpy.py b/distributed/protocol/tests/test_numpy.py
index 0e299632902..d09b5e435ad 100644
--- a/distributed/protocol/tests/test_numpy.py
+++ b/distributed/protocol/tests/test_numpy.py
@@ -100,6 +100,14 @@ def test_dumps_serialize_numpy(x):
         np.testing.assert_equal(x, y)
 
 
+def test_dumps_numpy_writable():
+    a1 = np.arange(1000)
+    a1.flags.writeable = False
+    (a2,) = loads(dumps([to_serialize(a1)]))
+    assert (a1 == a2).all()
+    assert a2.flags.writeable
+
+
 @pytest.mark.parametrize(
     "x",
     [
diff --git a/distributed/protocol/tests/test_protocol_utils.py b/distributed/protocol/tests/test_protocol_utils.py
index d4250fb3c05..5f18bc1e6e6 100644
--- a/distributed/protocol/tests/test_protocol_utils.py
+++ b/distributed/protocol/tests/test_protocol_utils.py
@@ -1,19 +1,33 @@
+import pytest
+
 from distributed.protocol.utils import merge_frames, pack_frames, unpack_frames
 from distributed.utils import ensure_bytes
 
 
-def test_merge_frames():
-    result = merge_frames({"lengths": [3, 4]}, [b"12", b"34", b"567"])
-    expected = [b"123", b"4567"]
-
+@pytest.mark.parametrize(
+    "lengths,frames",
+    [
+        ([3], [b"123"]),
+        ([3, 3], [b"123", b"456"]),
+        ([2, 3, 2], [b"12345", b"67"]),
+        ([5, 2], [b"123", b"45", b"67"]),
+        ([3, 4], [b"12", b"34", b"567"]),
+    ],
+)
+def test_merge_frames(lengths, frames):
+    header = {"lengths": lengths}
+    result = merge_frames(header, frames)
+
+    data = b"".join(frames)
+    expected = []
+    for i in lengths:
+        expected.append(data[:i])
+        data = data[i:]
+
+    assert all(isinstance(f, memoryview) for f in result)
+    assert all(not f.readonly for f in result)
     assert list(map(ensure_bytes, result)) == expected
 
-    b = b"123"
-    assert merge_frames({"lengths": [3]}, [b])[0] is b
-
-    L = [b"123", b"456"]
-    assert merge_frames({"lengths": [3, 3]}, L) is L
-
 
 def test_pack_frames():
     frames = [b"123", b"asdf"]
diff --git a/distributed/protocol/utils.py b/distributed/protocol/utils.py
index defbda2ba4f..e94d21584d2 100644
--- a/distributed/protocol/utils.py
+++ b/distributed/protocol/utils.py
@@ -58,34 +58,36 @@ def merge_frames(header, frames):
     [b'123456']
     """
     lengths = list(header["lengths"])
+    frames = list(map(memoryview, frames))
 
     assert sum(lengths) == sum(map(nbytes, frames))
 
-    if all(len(f) == l for f, l in zip(frames, lengths)):
-        return frames
-
-    frames = frames[::-1]
-    lengths = lengths[::-1]
-
-    out = []
-    while lengths:
-        l = lengths.pop()
-        L = []
-        while l:
-            frame = frames.pop()
-            if nbytes(frame) <= l:
-                L.append(frame)
-                l -= nbytes(frame)
+    if not all(len(f) == l for f, l in zip(frames, lengths)):
+        frames = frames[::-1]
+        lengths = lengths[::-1]
+
+        out = []
+        while lengths:
+            l = lengths.pop()
+            L = []
+            while l:
+                frame = frames.pop()
+                if nbytes(frame) <= l:
+                    L.append(frame)
+                    l -= nbytes(frame)
+                else:
+                    L.append(frame[:l])
+                    frames.append(frame[l:])
+                    l = 0
+            if len(L) == 1:  # no work necessary
+                out.append(L[0])
             else:
-                mv = memoryview(frame)
-                L.append(mv[:l])
-                frames.append(mv[l:])
-                l = 0
-        if len(L) == 1:  # no work necessary
-            out.extend(L)
-        else:
-            out.append(b"".join(L))
-    return out
+                out.append(memoryview(bytearray().join(L)))
+        frames = out
+
+    frames = [memoryview(bytearray(f)) if f.readonly else f for f in frames]
+
+    return frames
 
 
 def pack_frames_prelude(frames):

From 909c0ace6ea6273f4738c586a722975de5ef7c57 Mon Sep 17 00:00:00 2001
From: Matthias Bussonnier <bussonniermatthias@gmail.com>
Date: Mon, 20 Jul 2020 06:27:59 -0700
Subject: [PATCH 0915/1550] DOC: use continuatino prompt for proper example
 parsing. (#3966)

`>>>` imply that a line is complete, proper continuation needs `...`.
---
 distributed/semaphore.py | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/distributed/semaphore.py b/distributed/semaphore.py
index ece69fe5bc2..eba5908d502 100644
--- a/distributed/semaphore.py
+++ b/distributed/semaphore.py
@@ -324,18 +324,18 @@ class Semaphore:
     Examples
     --------
     >>> from distributed import Semaphore
-    >>> sem = Semaphore(max_leases=2, name='my_database')
-    >>>
-    >>> def access_resource(s, sem):
-    >>>     # This automatically acquires a lease from the semaphore (if available) which will be
-    >>>     # released when leaving the context manager.
-    >>>     with sem:
-    >>>         pass
-    >>>
-    >>> futures = client.map(access_resource, range(10), sem=sem)
-    >>> client.gather(futures)
-    >>> # Once done, close the semaphore to clean up the state on scheduler side.
-    >>> sem.close()
+    ... sem = Semaphore(max_leases=2, name='my_database')
+    ...
+    ... def access_resource(s, sem):
+    ...     # This automatically acquires a lease from the semaphore (if available) which will be
+    ...     # released when leaving the context manager.
+    ...     with sem:
+    ...         pass
+    ...
+    ... futures = client.map(access_resource, range(10), sem=sem)
+    ... client.gather(futures)
+    ... # Once done, close the semaphore to clean up the state on scheduler side.
+    ... sem.close()
 
     Notes
     -----

From 85c35064ec1786055cbc741a126fa581493acbe1 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Mon, 20 Jul 2020 12:03:46 -0700
Subject: [PATCH 0916/1550] Simplify `pack_frames_prelude` (#3961)

* Simplify `pack_frames_prelude`

Reduce to a single line of code.

* Assign variables to clarify meaning
---
 distributed/protocol/utils.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/distributed/protocol/utils.py b/distributed/protocol/utils.py
index e94d21584d2..a73e178fd2b 100644
--- a/distributed/protocol/utils.py
+++ b/distributed/protocol/utils.py
@@ -91,10 +91,9 @@ def merge_frames(header, frames):
 
 
 def pack_frames_prelude(frames):
-    lengths = [struct.pack("Q", len(frames))] + [
-        struct.pack("Q", nbytes(frame)) for frame in frames
-    ]
-    return b"".join(lengths)
+    nframes = len(frames)
+    nbytes_frames = map(nbytes, frames)
+    return struct.pack(f"Q{nframes}Q", nframes, *nbytes_frames)
 
 
 def pack_frames(frames):

From a53ad06dd419d921263cb2f0dbb59a2c8ea7df56 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Mon, 20 Jul 2020 15:01:35 -0700
Subject: [PATCH 0917/1550] Use `merge_frames` with host memory only (#3971)

* Use `merge_frames` with `"dask"` and `"pickle"`

Currently `merge_frames` is only really designed to work with memory on
host. So limit calling `merge_frames` to cases where host memory is used
(as opposed to cases with device memory).

* Check for CUDA frames before merging

As the `header` may not have the serialization mechanism used indicated,
simply check for any frames on device. If any such frames are found,
skip `merge_frames`.
---
 distributed/protocol/core.py      | 3 ++-
 distributed/protocol/serialize.py | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/distributed/protocol/core.py b/distributed/protocol/core.py
index c76d84e8a97..15d9bd24e97 100644
--- a/distributed/protocol/core.py
+++ b/distributed/protocol/core.py
@@ -126,7 +126,8 @@ def loads(frames, deserialize=True, deserializers=None):
             if deserialize or key in bytestrings:
                 if "compression" in head:
                     fs = decompress(head, fs)
-                fs = merge_frames(head, fs)
+                if not any(hasattr(f, "__cuda_array_interface__") for f in fs):
+                    fs = merge_frames(head, fs)
                 value = _deserialize(head, fs, deserializers=deserializers)
             else:
                 value = Serialized(head, fs)
diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index 6853d9afe6f..fb285bd1f00 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -501,7 +501,8 @@ def deserialize_bytes(b):
     else:
         header = {}
     frames = decompress(header, frames)
-    frames = merge_frames(header, frames)
+    if not any(hasattr(f, "__cuda_array_interface__") for f in frames):
+        frames = merge_frames(header, frames)
     return deserialize(header, frames)
 
 

From 8ebd396bc28a829ae9a928ed590ac2a8ef7dc301 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jendrik=20J=C3=B6rdening?=
 <jendrikjoe@users.noreply.github.com>
Date: Tue, 21 Jul 2020 02:49:54 +0200
Subject: [PATCH 0918/1550] Always attempt to create dask-worker-space folder
 and continue if it exists (#3972)

Co-authored-by: Jendrik <jendrik.joerdening@nooxit.com>
---
 distributed/worker.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/distributed/worker.py b/distributed/worker.py
index 02299dae28e..00d69329c40 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -485,8 +485,7 @@ def __init__(
         if not local_directory:
             local_directory = dask.config.get("temporary-directory") or os.getcwd()
 
-        if not os.path.exists(local_directory):
-            os.makedirs(local_directory)
+        os.makedirs(local_directory, exist_ok=True)
         local_directory = os.path.join(local_directory, "dask-worker-space")
 
         with warn_on_duration(

From 1dba1622049dd957dfce138618e4ee4180222b4a Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Tue, 21 Jul 2020 07:21:06 -0700
Subject: [PATCH 0919/1550] Streamline `pack_frames`/`unpack_frames` frames
 (#3973)

* Use `.extend` to collect and append `frames`

Instead of copying `frames` into a new `list`, just use `.extend` to
add them into the `list` already containing the prelude. This is
flexible and handles other non-`list` objects just as well.

* Assign format to a variable for easier reference

* Refactor itemsize from number of elements read

* Compute the size of `"Q"` and use throughout

* Use `struct.unpack_from` to ignore extra bytes

* Unpack frame lengths all at once

* Compute frame `end` once

* Run `black`
---
 distributed/protocol/utils.py | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/distributed/protocol/utils.py b/distributed/protocol/utils.py
index a73e178fd2b..fd50b6ca094 100644
--- a/distributed/protocol/utils.py
+++ b/distributed/protocol/utils.py
@@ -105,12 +105,9 @@ def pack_frames(frames):
     --------
     unpack_frames
     """
-    prelude = [pack_frames_prelude(frames)]
-
-    if not isinstance(frames, list):
-        frames = list(frames)
-
-    return b"".join(prelude + frames)
+    data = [pack_frames_prelude(frames)]
+    data.extend(frames)
+    return b"".join(data)
 
 
 def unpack_frames(b):
@@ -123,14 +120,17 @@ def unpack_frames(b):
     --------
     pack_frames
     """
-    (n_frames,) = struct.unpack("Q", b[:8])
+    fmt = "Q"
+    fmt_size = struct.calcsize(fmt)
+    (n_frames,) = struct.unpack_from(fmt, b)
+    lengths = struct.unpack_from(f"{n_frames}{fmt}", b, fmt_size)
 
     frames = []
-    start = 8 + n_frames * 8
-    for i in range(n_frames):
-        (length,) = struct.unpack("Q", b[(i + 1) * 8 : (i + 2) * 8])
-        frame = b[start : start + length]
+    start = fmt_size * (1 + n_frames)
+    for length in lengths:
+        end = start + length
+        frame = b[start:end]
         frames.append(frame)
-        start += length
+        start = end
 
     return frames

From 6eab3ce464e9ff79e4192de4f1c778d3ef89379b Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 21 Jul 2020 08:05:06 -0700
Subject: [PATCH 0920/1550] Iterate over list of comms (#3959)

Previously this list could change during iteration
---
 distributed/core.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/core.py b/distributed/core.py
index 250ea076e03..16c611142f4 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -606,7 +606,7 @@ def close(self):
                 break
             else:
                 yield asyncio.sleep(0.05)
-        yield [comm.close() for comm in self._comms]  # then forcefully close
+        yield [comm.close() for comm in list(self._comms)]  # then forcefully close
         for cb in self._ongoing_coroutines:
             cb.cancel()
         for i in range(10):

From eb10a5354485707c4825b7807b5d6e0236d87e4a Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Tue, 21 Jul 2020 17:01:51 -0700
Subject: [PATCH 0921/1550] Use `memoryview` in `unpack_frames` (#3980)

* Minor whitespace adjustment

* Coerce input to `memoryview` in `unpack_frames`

Selecting out each frame from the input causes a copy, which increases
memory usage and slows down `unpack_frames`. To fix this, coerce the
input to a `memoryview`. This way slices into the `memoryview` only take
a view onto the underlying data, which is quite fast and doesn't result
in additional memory usage.
---
 distributed/protocol/utils.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/distributed/protocol/utils.py b/distributed/protocol/utils.py
index fd50b6ca094..8dc75a28c63 100644
--- a/distributed/protocol/utils.py
+++ b/distributed/protocol/utils.py
@@ -120,8 +120,11 @@ def unpack_frames(b):
     --------
     pack_frames
     """
+    b = memoryview(b)
+
     fmt = "Q"
     fmt_size = struct.calcsize(fmt)
+
     (n_frames,) = struct.unpack_from(fmt, b)
     lengths = struct.unpack_from(f"{n_frames}{fmt}", b, fmt_size)
 

From ccc9d2a1d5c2ee8a0f37f146b48f64cffa5c373e Mon Sep 17 00:00:00 2001
From: Julia Signell <jsignell@gmail.com>
Date: Fri, 24 Jul 2020 14:45:31 -0400
Subject: [PATCH 0922/1550] Warn when threads_per_worker is set to zero (#3986)

---
 distributed/deploy/local.py            |  7 +++++++
 distributed/deploy/tests/test_local.py | 12 ++++++++++++
 2 files changed, 19 insertions(+)

diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index cd33ad26a12..84b0631c775 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -126,6 +126,13 @@ def __init__(
             )
             dashboard_address = diagnostics_port
 
+        if threads_per_worker == 0:
+            warnings.warn(
+                "Setting `threads_per_worker` to 0 is discouraged. "
+                "Please set to None or to a specific int to get best behavior."
+            )
+            threads_per_worker = None
+
         self.status = None
         self.processes = processes
 
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index 59a2d0c7607..dfeceae7a35 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -997,6 +997,18 @@ async def test_repr(cleanup):
         assert "memory" not in repr(cluster)
 
 
+@pytest.mark.asyncio
+async def test_threads_per_worker_set_to_0(cleanup):
+    with pytest.warns(
+        Warning, match="Setting `threads_per_worker` to 0 is discouraged."
+    ):
+        async with LocalCluster(
+            n_workers=2, processes=False, threads_per_worker=0, asynchronous=True,
+        ) as cluster:
+            assert len(cluster.workers) == 2
+            assert all(w.nthreads < CPU_COUNT for w in cluster.workers.values())
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize("temporary", [True, False])
 async def test_capture_security(cleanup, temporary):

From 12d5dc7926b219e58d346a5caad376876b7d8fcf Mon Sep 17 00:00:00 2001
From: Matthias Bussonnier <bussonniermatthias@gmail.com>
Date: Mon, 27 Jul 2020 07:30:34 -0700
Subject: [PATCH 0923/1550] Add missing backtick in inline directive. (#3988)

This messes-up rst parsing.
---
 distributed/deploy/ssh.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/deploy/ssh.py b/distributed/deploy/ssh.py
index 595e21dbd7a..4d1b8e953a4 100644
--- a/distributed/deploy/ssh.py
+++ b/distributed/deploy/ssh.py
@@ -262,7 +262,7 @@ def SSHCluster(
         List of hostnames or addresses on which to launch our cluster.
         The first will be used for the scheduler and the rest for workers.
     connect_options: dict or list of dict, optional
-        Keywords to pass through to :func:asyncssh.connect`.
+        Keywords to pass through to :func:`asyncssh.connect`.
         This could include things such as ``port``, ``username``, ``password``
         or ``known_hosts``. See docs for :func:`asyncssh.connect` and
         :class:`asyncssh.SSHClientConnectionOptions` for full information.

From c4df1a322e78f168b15b0513626eece4726e4848 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 27 Jul 2020 10:29:11 -0700
Subject: [PATCH 0924/1550] Revert to localhost for local IP if no network
 available (#3991)

Fixes https://github.com/dask/distributed/issues/3989
---
 distributed/comm/inproc.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/distributed/comm/inproc.py b/distributed/comm/inproc.py
index 0642cce7381..c5952e2dc15 100644
--- a/distributed/comm/inproc.py
+++ b/distributed/comm/inproc.py
@@ -36,7 +36,10 @@ def __init__(self):
             # Avoid immediate warning for unreachable network
             # (will still warn for other get_ip() calls when actually used)
             warnings.simplefilter("ignore")
-            self.ip = get_ip()
+            try:
+                self.ip = get_ip()
+            except OSError:
+                self.ip = "127.0.0.1"
         self.lock = threading.Lock()
 
     def add_listener(self, addr, listener):

From 907452b1bc0f7876a01de5ff39cc2dc64eb3c4a4 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Mon, 27 Jul 2020 14:35:30 -0700
Subject: [PATCH 0925/1550] Drop msgpack pre-0.5.2 compat code (#3977)

As we require msgpack 0.6.0+, there is no need for msgpack compat code
for versions prior to 0.6.0. So drop this compat code.
---
 distributed/protocol/utils.py | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/distributed/protocol/utils.py b/distributed/protocol/utils.py
index 8dc75a28c63..578a34e4164 100644
--- a/distributed/protocol/utils.py
+++ b/distributed/protocol/utils.py
@@ -1,5 +1,4 @@
 import struct
-import msgpack
 
 from ..utils import nbytes
 
@@ -10,13 +9,7 @@
     ("max_%s_len" % x): 2 ** 31 - 1 for x in ["str", "bin", "array", "map", "ext"]
 }
 msgpack_opts["strict_map_key"] = False
-
-try:
-    msgpack.loads(msgpack.dumps(""), raw=False, **msgpack_opts)
-    msgpack_opts["raw"] = False
-except TypeError:
-    # Backward compat with old msgpack (prior to 0.5.2)
-    msgpack_opts["encoding"] = "utf-8"
+msgpack_opts["raw"] = False
 
 
 def frame_split_size(frame, n=BIG_BYTES_SHARD_SIZE) -> list:

From 7cdba1d7e17df7be005f4d4b8ca528065a3f83a7 Mon Sep 17 00:00:00 2001
From: Julia Signell <jsignell@gmail.com>
Date: Mon, 27 Jul 2020 20:46:17 -0400
Subject: [PATCH 0926/1550] Handle sum of memory percentage when memory_limit
 is 0 (#3984)

---
 distributed/dashboard/components/scheduler.py   | 17 ++++++++++++++---
 .../dashboard/tests/test_scheduler_bokeh.py     | 11 +++++++++++
 2 files changed, 25 insertions(+), 3 deletions(-)

diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index 2934b6e9848..a10c68b2d16 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -1942,9 +1942,20 @@ def update(self):
                 if len(self.scheduler.workers) == 0:
                     total_data = None
                 elif name == "memory_percent":
-                    total_data = sum(
-                        ws.metrics["memory"] for ws in self.scheduler.workers.values()
-                    ) / sum(ws.memory_limit for ws in self.scheduler.workers.values())
+                    total_mem = sum(
+                        ws.memory_limit for ws in self.scheduler.workers.values()
+                    )
+                    total_data = (
+                        (
+                            sum(
+                                ws.metrics["memory"]
+                                for ws in self.scheduler.workers.values()
+                            )
+                            / total_mem
+                        )
+                        if total_mem
+                        else ""
+                    )
                 elif name == "cpu":
                     total_data = (
                         sum(ws.metrics["cpu"] for ws in self.scheduler.workers.values())
diff --git a/distributed/dashboard/tests/test_scheduler_bokeh.py b/distributed/dashboard/tests/test_scheduler_bokeh.py
index 0561cd5fbbc..04cadeaf043 100644
--- a/distributed/dashboard/tests/test_scheduler_bokeh.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh.py
@@ -430,6 +430,17 @@ def metric(worker):
     assert s.workers[a.address].metrics["metric"] == -999
 
 
+@gen_cluster(client=True, worker_kwargs={"memory_limit": 0})
+async def test_WorkerTable_with_memory_limit_as_0(c, s, a, b):
+
+    wt = WorkerTable(s)
+    wt.update()
+    assert all(wt.source.data.values())
+    assert wt.source.data["name"][0] == "Total (2)"
+    assert wt.source.data["memory_limit"][0] == 0
+    assert wt.source.data["memory_percent"][0] == ""
+
+
 @gen_cluster(client=True)
 async def test_TaskGraph(c, s, a, b):
     gp = TaskGraph(s)

From e5c544bc830d31ee8c5688f0479f0427e5d1e19a Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Fri, 31 Jul 2020 09:17:36 -0700
Subject: [PATCH 0927/1550] Fix failing `test_bandwidth` (#3999)

* Use `<=` instead of `<` in `test_bandwidth`

* Use 1 more byte in `test_bandwidth` data

* update bandwidth test

Co-authored-by: Benjamin Zaitlen <quasiben@gmail.com>
---
 distributed/tests/test_scheduler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index a55a7213919..5743367e0d9 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -1635,7 +1635,7 @@ async def test_idle_timeout(c, s, a, b):
 @gen_cluster(client=True, config={"distributed.scheduler.bandwidth": "100 GB"})
 async def test_bandwidth(c, s, a, b):
     start = s.bandwidth
-    x = c.submit(operator.mul, b"0", 1000000, workers=a.address)
+    x = c.submit(operator.mul, b"0", 1000001, workers=a.address)
     y = c.submit(lambda x: x, x, workers=b.address)
     await y
     await b.heartbeat()

From 80ea63a46ed0997ef28766c63d026d9483866d1d Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Fri, 31 Jul 2020 10:31:41 -0700
Subject: [PATCH 0928/1550] Only call `frame_split_size` when there are frames
 (#3996)

This just becomes a no-op if an empty list is given (returning an empty
list). However there is no need to do this work if we don't need it. So
simply restrict calling `frame_split_size` to the case where frames are
present and compression will occur.
---
 distributed/protocol/serialize.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index fb285bd1f00..8ebac2500f5 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -475,8 +475,8 @@ def serialize_bytelist(x, **kwargs):
     header, frames = serialize(x, **kwargs)
     if "lengths" not in header:
         header["lengths"] = tuple(map(nbytes, frames))
-    frames = sum(map(frame_split_size, frames), [])
     if frames:
+        frames = sum(map(frame_split_size, frames), [])
         compression, frames = zip(*map(maybe_compress, frames))
     else:
         compression = []

From 67fe8208a0a6edc18e02a4c5080d81fb11eab338 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Fri, 31 Jul 2020 18:08:43 -0500
Subject: [PATCH 0929/1550] bump version to 2.22.0

---
 docs/source/changelog.rst | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index a1202c94d83..ebcc5420af8 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,26 @@
 Changelog
 =========
 
+2.22.0 - 2020-07-31
+-------------------
+
+- Only call ``frame_split_size`` when there are frames (:pr:`3996`) `jakirkham`_
+- Fix failing ``test_bandwidth`` (:pr:`3999`) `jakirkham`_
+- Handle sum of memory percentage when ``memory_limit`` is 0 (:pr:`3984`) `Julia Signell`_
+- Drop msgpack pre-0.5.2 compat code (:pr:`3977`) `jakirkham`_
+- Revert to localhost for local IP if no network available (:pr:`3991`) `Matthew Rocklin`_
+- Add missing backtick in inline directive. (:pr:`3988`) `Matthias Bussonnier`_
+- Warn when ``threads_per_worker`` is set to zero (:pr:`3986`) `Julia Signell`_
+- Use ``memoryview`` in ``unpack_frames`` (:pr:`3980`) `jakirkham`_
+- Iterate over list of comms (:pr:`3959`) `Matthew Rocklin`_
+- Streamline ``pack_frames``/``unpack_frames`` frames (:pr:`3973`) `jakirkham`_
+- Always attempt to create ``dask-worker-space`` folder and continue if it exists (:pr:`3972`) `Jendrik Jördening`_
+- Use ``merge_frames`` with host memory only (:pr:`3971`) `jakirkham`_
+- Simplify ``pack_frames_prelude`` (:pr:`3961`) `jakirkham`_
+- Use continuation prompt for proper example parsing (:pr:`3966`) `Matthias Bussonnier`_
+- Ensure writable frames (:pr:`3967`) `jakirkham`_
+
+
 2.21.0 - 2020-07-17
 -------------------
 
@@ -1864,3 +1884,4 @@ significantly without many new features.
 .. _`joshreback`: https://github.com/joshreback
 .. _`Alexander Clausen`: https://github.com/sk1p
 .. _`Andrew Fulton`: https://github.com/andrewfulton9
+.. _`Jendrik Jördening`: https://github.com/jendrikjoe

From b221a63c7aab3ccc9b1e9e1c70f4a793514b2035 Mon Sep 17 00:00:00 2001
From: Willi Rath <willirath@users.noreply.github.com>
Date: Sun, 2 Aug 2020 18:28:14 +0200
Subject: [PATCH 0930/1550] Fix memory histogram values in dashboard (#4006)

---
 distributed/dashboard/components/scheduler.py       | 4 +++-
 distributed/dashboard/tests/test_scheduler_bokeh.py | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index a10c68b2d16..ecc4c91ca8c 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -263,7 +263,9 @@ def __init__(self, scheduler, **kwargs):
 
     @without_property_validation
     def update(self):
-        nbytes = np.asarray([ws.nbytes for ws in self.scheduler.workers.values()])
+        nbytes = np.asarray(
+            [ws.metrics["memory"] for ws in self.scheduler.workers.values()]
+        )
         counts, x = np.histogram(nbytes, bins=40)
         d = {"left": x[:-1], "right": x[1:], "top": counts}
         self.source.data.update(d)
diff --git a/distributed/dashboard/tests/test_scheduler_bokeh.py b/distributed/dashboard/tests/test_scheduler_bokeh.py
index 04cadeaf043..86454328b62 100644
--- a/distributed/dashboard/tests/test_scheduler_bokeh.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh.py
@@ -276,7 +276,7 @@ async def test_ProcessingHistogram(c, s, a, b):
 async def test_NBytesHistogram(c, s, a, b):
     nh = NBytesHistogram(s)
     nh.update()
-    assert (nh.source.data["top"] != 0).sum() == 1
+    assert any(nh.source.data["top"] != 0)
 
     futures = c.map(inc, range(10))
     await wait(futures)

From 4311caf33a6b321b2a56a50264d4d68f78085986 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Mon, 3 Aug 2020 07:40:13 -0700
Subject: [PATCH 0931/1550] Improve `bytes` and `bytearray` serialization
 (#4009)

Handle these two separately to ensure we are creating the right types in
each respective case.

Make sure that `bytes` and `bytearray` types are deserialized correctly
even if the frames are of a different type or more frames are involved.
---
 distributed/protocol/serialize.py            | 27 +++++++++++++++++---
 distributed/protocol/tests/test_serialize.py |  7 +++++
 2 files changed, 30 insertions(+), 4 deletions(-)

diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index 8ebac2500f5..2ce546cea2e 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -564,17 +564,36 @@ def normalize_Serialized(o):
     return [o.header] + o.frames  # for dask.base.tokenize
 
 
-# Teach serialize how to handle bytestrings
-@dask_serialize.register((bytes, bytearray))
+# Teach serialize how to handle bytes
+@dask_serialize.register(bytes)
 def _serialize_bytes(obj):
     header = {}  # no special metadata
     frames = [obj]
     return header, frames
 
 
-@dask_deserialize.register((bytes, bytearray))
+# Teach serialize how to handle bytestrings
+@dask_serialize.register(bytearray)
+def _serialize_bytearray(obj):
+    header = {}  # no special metadata
+    frames = [obj]
+    return header, frames
+
+
+@dask_deserialize.register(bytes)
 def _deserialize_bytes(header, frames):
-    return b"".join(frames)
+    if len(frames) == 1 and isinstance(frames[0], bytes):
+        return frames[0]
+    else:
+        return bytes().join(frames)
+
+
+@dask_deserialize.register(bytearray)
+def _deserialize_bytearray(header, frames):
+    if len(frames) == 1 and isinstance(frames[0], bytearray):
+        return frames[0]
+    else:
+        return bytearray().join(frames)
 
 
 @dask_serialize.register(memoryview)
diff --git a/distributed/protocol/tests/test_serialize.py b/distributed/protocol/tests/test_serialize.py
index f11186e1c70..5ebe615d4e9 100644
--- a/distributed/protocol/tests/test_serialize.py
+++ b/distributed/protocol/tests/test_serialize.py
@@ -69,6 +69,13 @@ def test_serialize_bytestrings():
         header, frames = serialize(b)
         assert frames[0] is b
         bb = deserialize(header, frames)
+        assert type(bb) == type(b)
+        assert bb == b
+        bb = deserialize(header, list(map(memoryview, frames)))
+        assert type(bb) == type(b)
+        assert bb == b
+        bb = deserialize(header, [b"", *frames])
+        assert type(bb) == type(b)
         assert bb == b
 
 

From 5e7285cb200d461034a439274d2a5db87c3df9e7 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Mon, 3 Aug 2020 17:10:42 -0700
Subject: [PATCH 0932/1550] Track mutable frames (#4004)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Flag mutable frames as part of serialization

* Just use `bytes` in `merge_frames`

Keeps logic simple here in exchange for potentially copying later if
this is writeable.

* Only copy mutable frames

* Mark all CUDA frames as non-writeable

We don't need to bother making these frames writeable as they will be
transferred to GPU anyways where they will then be writeable. As a
result this saves us a copy that we don't otherwise need.

* Test NumPy array preserves writeability

* Fix-up `merge_frames` test with `writeable` header

* Relax the `memoryview` requirement of frames

* Drop test for frames being `memoryview`s

* Assert `writeable` and `lengths` have same # items

* Optionally determine `writeable`

Only try to figure out `writeable` if it is not otherwise specified.
This is important for CUDA objects for example, which are not
`memoryview` coercible.

* Rename `m` to `w` for clarity

* Fix Pandas test name

* Test writing to Pandas Series after serialization

* Force Pandas serialized frames to be readonly

Make sure all frames are `bytes` as they are read-only. This should test
whether the deserialization logic is smart enough to coerce them back
into something that is writeable.

* Test `merge_frames` with other `writeable` values

* Use read-only frames for serialized NumPy array

* Use `list` to compare `writeable`

* Drop unneeded CUDA array interface check

As we are only using `"dask"` or `"pickle"` serialization when spilling
here, we can be confident that there will not be any CUDA frames. So
drop this unneeded check.

* Use a `bytearray` to join writeable frames

* Handle singleton frame case as well

* Handle fast-path at the beginning

As we now guarantee frames are readonly or writeable as appropriate when
splitting/merging frames in all cases, there is no need to run this code
in all cases. We simply need it for the fast path. So add it there.

* Fix-up readonly case in fast path

Make sure we force readonly frames to be readonly.

* Mark CUDA frames as neither readonly no writeable

This special case of `None` will bypass both cases.

* Fix-up logic for copying singleton frame

* Test a few more `merge_frames` cases

* Drop unneeded CuPy customization

* Always use `join` path when copying is needed

* Go back to `.extend()`

This is very slightly faster than `.append()`

```python
In [1]: %timeit L = []
17.4 ns ± 0.0668 ns per loop (mean ± std. dev. of 7 runs, 100000000 loops each)

In [2]: L2 = [5]

In [3]: %timeit L = []; L.extend(L2)
73.7 ns ± 1.72 ns per loop (mean ± std. dev. of 7 runs, 10000000 loops each)

In [4]: %timeit L = []; L.append(L2[0])
84.5 ns ± 0.211 ns per loop (mean ± std. dev. of 7 runs, 10000000 loops each)
```

* Skip assignment and return `out`

* One more test with `bytearray`s

* Add function to check whether frame is writeable

Try to coerce the frame through `memoryview` and check the `readonly`
attribute. If the frame cannot be coerced through `memoryview`, use
`None` to indicate the frame is ambiguous, which means we will bypass
any special handling of the frame.

* Use `is_writeable` to detect writeable frames

* Explain return results from `is_writeable`
---
 distributed/protocol/core.py                  |  6 +-
 distributed/protocol/cupy.py                  |  1 +
 distributed/protocol/numba.py                 |  1 +
 distributed/protocol/rmm.py                   |  1 +
 distributed/protocol/serialize.py             |  7 ++-
 distributed/protocol/tests/test_numpy.py      | 14 +++--
 distributed/protocol/tests/test_pandas.py     | 23 ++++++-
 .../protocol/tests/test_protocol_utils.py     | 31 ++++++----
 distributed/protocol/utils.py                 | 60 +++++++++++--------
 distributed/utils.py                          | 13 ++++
 10 files changed, 109 insertions(+), 48 deletions(-)

diff --git a/distributed/protocol/core.py b/distributed/protocol/core.py
index 15d9bd24e97..ab4f0f130b9 100644
--- a/distributed/protocol/core.py
+++ b/distributed/protocol/core.py
@@ -7,7 +7,7 @@
 from .compression import compressions, maybe_compress, decompress
 from .serialize import serialize, deserialize, Serialize, Serialized, extract_serialize
 from .utils import frame_split_size, merge_frames, msgpack_opts
-from ..utils import nbytes
+from ..utils import is_writeable, nbytes
 
 _deserialize = deserialize
 
@@ -46,6 +46,8 @@ def dumps(msg, serializers=None, on_error="message", context=None):
         out_frames = []
 
         for key, (head, frames) in data.items():
+            if "writeable" not in head:
+                head["writeable"] = tuple(map(is_writeable, frames))
             if "lengths" not in head:
                 head["lengths"] = tuple(map(nbytes, frames))
 
@@ -71,6 +73,8 @@ def dumps(msg, serializers=None, on_error="message", context=None):
             out_frames.extend(_out_frames)
 
         for key, (head, frames) in pre.items():
+            if "writeable" not in head:
+                head["writeable"] = tuple(map(is_writeable, frames))
             if "lengths" not in head:
                 head["lengths"] = tuple(map(nbytes, frames))
             head["count"] = len(frames)
diff --git a/distributed/protocol/cupy.py b/distributed/protocol/cupy.py
index 0a2c53be4a5..856fc5adf46 100644
--- a/distributed/protocol/cupy.py
+++ b/distributed/protocol/cupy.py
@@ -47,6 +47,7 @@ def cuda_deserialize_cupy_ndarray(header, frames):
 @dask_serialize.register(cupy.ndarray)
 def dask_serialize_cupy_ndarray(x):
     header, frames = cuda_serialize_cupy_ndarray(x)
+    header["writeable"] = (None,) * len(frames)
     frames = [memoryview(cupy.asnumpy(f)) for f in frames]
     return header, frames
 
diff --git a/distributed/protocol/numba.py b/distributed/protocol/numba.py
index 20eec8e11b6..93d65c0ef6d 100644
--- a/distributed/protocol/numba.py
+++ b/distributed/protocol/numba.py
@@ -51,6 +51,7 @@ def cuda_deserialize_numba_ndarray(header, frames):
 @dask_serialize.register(numba.cuda.devicearray.DeviceNDArray)
 def dask_serialize_numba_ndarray(x):
     header, frames = cuda_serialize_numba_ndarray(x)
+    header["writeable"] = (None,) * len(frames)
     frames = [memoryview(f.copy_to_host()) for f in frames]
     return header, frames
 
diff --git a/distributed/protocol/rmm.py b/distributed/protocol/rmm.py
index 76706d49d89..f4fa9e1075c 100644
--- a/distributed/protocol/rmm.py
+++ b/distributed/protocol/rmm.py
@@ -30,6 +30,7 @@ def cuda_deserialize_rmm_device_buffer(header, frames):
     @dask_serialize.register(rmm.DeviceBuffer)
     def dask_serialize_rmm_device_buffer(x):
         header, frames = cuda_serialize_rmm_device_buffer(x)
+        header["writeable"] = (None,) * len(frames)
         frames = [numba.cuda.as_cuda_array(f).copy_to_host().data for f in frames]
         return header, frames
 
diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index 2ce546cea2e..275c7164df4 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -9,7 +9,7 @@
 import msgpack
 
 from . import pickle
-from ..utils import has_keyword, nbytes, typename, ensure_bytes
+from ..utils import has_keyword, nbytes, typename, ensure_bytes, is_writeable
 from .compression import maybe_compress, decompress
 from .utils import (
     unpack_frames,
@@ -473,6 +473,8 @@ def replace_inner(x):
 
 def serialize_bytelist(x, **kwargs):
     header, frames = serialize(x, **kwargs)
+    if "writeable" not in header:
+        header["writeable"] = tuple(map(is_writeable, frames))
     if "lengths" not in header:
         header["lengths"] = tuple(map(nbytes, frames))
     if frames:
@@ -501,8 +503,7 @@ def deserialize_bytes(b):
     else:
         header = {}
     frames = decompress(header, frames)
-    if not any(hasattr(f, "__cuda_array_interface__") for f in frames):
-        frames = merge_frames(header, frames)
+    frames = merge_frames(header, frames)
     return deserialize(header, frames)
 
 
diff --git a/distributed/protocol/tests/test_numpy.py b/distributed/protocol/tests/test_numpy.py
index d09b5e435ad..ea349692e70 100644
--- a/distributed/protocol/tests/test_numpy.py
+++ b/distributed/protocol/tests/test_numpy.py
@@ -17,7 +17,7 @@
 from distributed.protocol.pickle import HIGHEST_PROTOCOL
 from distributed.protocol.compression import maybe_compress
 from distributed.system import MEMORY_LIMIT
-from distributed.utils import tmpfile, nbytes
+from distributed.utils import ensure_bytes, tmpfile, nbytes
 from distributed.utils_test import gen_cluster
 
 
@@ -100,12 +100,16 @@ def test_dumps_serialize_numpy(x):
         np.testing.assert_equal(x, y)
 
 
-def test_dumps_numpy_writable():
+@pytest.mark.parametrize("writeable", [True, False])
+def test_dumps_numpy_writable(writeable):
     a1 = np.arange(1000)
-    a1.flags.writeable = False
-    (a2,) = loads(dumps([to_serialize(a1)]))
+    a1.flags.writeable = writeable
+    fs = dumps([to_serialize(a1)])
+    # Make all frames read-only
+    fs = list(map(ensure_bytes, fs))
+    (a2,) = loads(fs)
     assert (a1 == a2).all()
-    assert a2.flags.writeable
+    assert a2.flags.writeable == a1.flags.writeable
 
 
 @pytest.mark.parametrize(
diff --git a/distributed/protocol/tests/test_pandas.py b/distributed/protocol/tests/test_pandas.py
index b1f96bfd486..a8134d7e3d0 100644
--- a/distributed/protocol/tests/test_pandas.py
+++ b/distributed/protocol/tests/test_pandas.py
@@ -4,7 +4,15 @@
 
 from dask.dataframe.utils import assert_eq
 
-from distributed.protocol import serialize, deserialize, decompress
+from distributed.protocol import (
+    serialize,
+    deserialize,
+    decompress,
+    dumps,
+    loads,
+    to_serialize,
+)
+from distributed.utils import ensure_bytes
 
 
 dfs = [
@@ -63,10 +71,21 @@
 
 
 @pytest.mark.parametrize("df", dfs)
-def test_dumps_serialize_numpy(df):
+def test_dumps_serialize_pandas(df):
     header, frames = serialize(df)
     if "compression" in header:
         frames = decompress(header, frames)
     df2 = deserialize(header, frames)
 
     assert_eq(df, df2)
+
+
+def test_dumps_pandas_writable():
+    a1 = np.arange(1000)
+    s1 = pd.Series(a1)
+    fs = dumps([to_serialize(s1)])
+    # Make all frames read-only
+    fs = list(map(ensure_bytes, fs))
+    (s2,) = loads(fs)
+    assert (s1 == s2).all()
+    s2[...] = 0
diff --git a/distributed/protocol/tests/test_protocol_utils.py b/distributed/protocol/tests/test_protocol_utils.py
index 5f18bc1e6e6..2132f94008b 100644
--- a/distributed/protocol/tests/test_protocol_utils.py
+++ b/distributed/protocol/tests/test_protocol_utils.py
@@ -1,21 +1,30 @@
 import pytest
 
 from distributed.protocol.utils import merge_frames, pack_frames, unpack_frames
-from distributed.utils import ensure_bytes
+from distributed.utils import ensure_bytes, is_writeable
 
 
 @pytest.mark.parametrize(
-    "lengths,frames",
+    "lengths,writeable,frames",
     [
-        ([3], [b"123"]),
-        ([3, 3], [b"123", b"456"]),
-        ([2, 3, 2], [b"12345", b"67"]),
-        ([5, 2], [b"123", b"45", b"67"]),
-        ([3, 4], [b"12", b"34", b"567"]),
+        ([3], [False], [b"123"]),
+        ([3], [True], [b"123"]),
+        ([3], [None], [b"123"]),
+        ([3], [False], [bytearray(b"123")]),
+        ([3], [True], [bytearray(b"123")]),
+        ([3], [None], [bytearray(b"123")]),
+        ([3, 3], [False, False], [b"123", b"456"]),
+        ([2, 3, 2], [False, True, None], [b"12345", b"67"]),
+        ([2, 3, 2], [False, True, None], [bytearray(b"12345"), bytearray(b"67")]),
+        ([5, 2], [False, True], [b"123", b"45", b"67"]),
+        ([3, 4], [None, False], [b"12", b"34", b"567"]),
     ],
 )
-def test_merge_frames(lengths, frames):
-    header = {"lengths": lengths}
+def test_merge_frames(lengths, writeable, frames):
+    header = {
+        "lengths": lengths,
+        "writeable": writeable,
+    }
     result = merge_frames(header, frames)
 
     data = b"".join(frames)
@@ -24,8 +33,8 @@ def test_merge_frames(lengths, frames):
         expected.append(data[:i])
         data = data[i:]
 
-    assert all(isinstance(f, memoryview) for f in result)
-    assert all(not f.readonly for f in result)
+    writeables = list(map(is_writeable, result))
+    assert (r == e for r, e in zip(writeables, header["writeable"]) if e is not None)
     assert list(map(ensure_bytes, result)) == expected
 
 
diff --git a/distributed/protocol/utils.py b/distributed/protocol/utils.py
index 578a34e4164..66c9687bae4 100644
--- a/distributed/protocol/utils.py
+++ b/distributed/protocol/utils.py
@@ -51,36 +51,44 @@ def merge_frames(header, frames):
     [b'123456']
     """
     lengths = list(header["lengths"])
-    frames = list(map(memoryview, frames))
+    writeables = list(header["writeable"])
 
+    assert len(lengths) == len(writeables)
     assert sum(lengths) == sum(map(nbytes, frames))
 
-    if not all(len(f) == l for f, l in zip(frames, lengths)):
-        frames = frames[::-1]
-        lengths = lengths[::-1]
-
-        out = []
-        while lengths:
-            l = lengths.pop()
-            L = []
-            while l:
-                frame = frames.pop()
-                if nbytes(frame) <= l:
-                    L.append(frame)
-                    l -= nbytes(frame)
-                else:
-                    L.append(frame[:l])
-                    frames.append(frame[l:])
-                    l = 0
-            if len(L) == 1:  # no work necessary
-                out.append(L[0])
-            else:
-                out.append(memoryview(bytearray().join(L)))
-        frames = out
-
-    frames = [memoryview(bytearray(f)) if f.readonly else f for f in frames]
+    if all(len(f) == l for f, l in zip(frames, lengths)):
+        return [
+            (bytearray(f) if w else bytes(f)) if w == memoryview(f).readonly else f
+            for w, f in zip(header["writeable"], frames)
+        ]
 
-    return frames
+    frames = frames[::-1]
+    lengths = lengths[::-1]
+    writeables = writeables[::-1]
+
+    out = []
+    while lengths:
+        l = lengths.pop()
+        w = writeables.pop()
+        L = []
+        while l:
+            frame = frames.pop()
+            if nbytes(frame) <= l:
+                L.append(frame)
+                l -= nbytes(frame)
+            else:
+                frame = memoryview(frame)
+                L.append(frame[:l])
+                frames.append(frame[l:])
+                l = 0
+        if len(L) == 1 and w != memoryview(L[0]).readonly:  # no work necessary
+            out.extend(L)
+        elif w:
+            out.append(bytearray().join(L))
+        else:
+            out.append(bytes().join(L))
+
+    return out
 
 
 def pack_frames_prelude(frames):
diff --git a/distributed/utils.py b/distributed/utils.py
index dec1b6b79d3..fb4ad8f9015 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -1107,6 +1107,19 @@ def nbytes(frame, _bytes_like=(bytes, bytearray)):
             return len(frame)
 
 
+def is_writeable(frame):
+    """
+    Check whether frame is writeable
+
+    Will return ``True`` if writeable, ``False`` if readonly, and
+    ``None`` if undetermined.
+    """
+    try:
+        return not memoryview(frame).readonly
+    except TypeError:
+        return None
+
+
 @contextmanager
 def time_warn(duration, text):
     start = time()

From 586ded30a8f15b91bd4e45e50c832ef21faea79c Mon Sep 17 00:00:00 2001
From: Matthias Bussonnier <bussonniermatthias@gmail.com>
Date: Mon, 3 Aug 2020 19:29:16 -0700
Subject: [PATCH 0933/1550] Emmit warning when assign/comparing string with
 Status Enum. (#3875)

Turn this into an error in the test suite.

We want to make sure of various things:

  1) original behavior of comparing/assign with string still works
  but emit warnings.
  1b) assigning strings convert to proper enum variant.
  2) assign/comparison with invalid strings should fail
  2) warnings are errors in test suite to make sure we don't re-introduce
  strings.

This is the continuation of #3853

Maybe Cluster in cluster.py should also get status as a property
---
 distributed/core.py                           | 24 +++++-
 distributed/deploy/cluster.py                 | 11 +--
 distributed/deploy/spec.py                    | 56 +++++++++-----
 distributed/deploy/ssh.py                     |  3 +-
 distributed/deploy/tests/test_local.py        |  5 +-
 distributed/deploy/tests/test_spec_cluster.py |  9 ++-
 distributed/nanny.py                          |  8 +-
 distributed/protocol/core.py                  | 16 +++-
 distributed/protocol/serialize.py             | 28 +++++++
 distributed/scheduler.py                      | 15 +---
 distributed/tests/test_client.py              | 13 ++--
 distributed/tests/test_core.py                | 77 ++++++++++++++++++-
 distributed/tests/test_nanny.py               | 14 ++--
 distributed/tests/test_scheduler.py           | 22 +++---
 distributed/tests/test_tls_functional.py      |  3 +-
 distributed/tests/test_worker.py              | 16 ++--
 distributed/utils_test.py                     |  4 +-
 distributed/worker.py                         |  4 +-
 setup.cfg                                     |  8 +-
 19 files changed, 241 insertions(+), 95 deletions(-)

diff --git a/distributed/core.py b/distributed/core.py
index 16c611142f4..35f98e25a19 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -9,6 +9,7 @@
 import traceback
 import uuid
 import weakref
+import warnings
 
 import dask
 import tblib
@@ -52,11 +53,13 @@ class Status(Enum):
     closing = "closing"
     closing_gracefully = "closing-gracefully"
     init = "init"
+    created = "created"
     running = "running"
     starting = "starting"
     stopped = "stopped"
     stopping = "stopping"
     undefined = None
+    dont_reply = "dont-reply"
 
     def __eq__(self, other):
         """
@@ -69,6 +72,11 @@ def __eq__(self, other):
         if isinstance(other, type(self)):
             return self.value == other.value
         elif isinstance(other, str) or (other is None):
+            warnings.warn(
+                f"Since distributed 2.19 `.status` is now an Enum, please compare with `Status.{other}`",
+                PendingDeprecationWarning,
+                stacklevel=1,
+            )
             assert other in [
                 s.value for s in type(self)
             ], f"comparison with non-existing states {other}"
@@ -261,9 +269,16 @@ def status(self, new_status):
         if isinstance(new_status, Status):
             self._status = new_status
         elif isinstance(new_status, str) or new_status is None:
+            warnings.warn(
+                f"Since distributed 2.19 `.status` is now an Enum, please assign `Status.{new_status}`",
+                PendingDeprecationWarning,
+                stacklevel=1,
+            )
             corresponding_enum_variants = [s for s in Status if s.value == new_status]
             assert len(corresponding_enum_variants) == 1
             self._status = corresponding_enum_variants[0]
+        else:
+            raise TypeError(f"expected Status or str, got {new_status}")
 
     async def finished(self):
         """ Wait until the server has finished """
@@ -519,7 +534,14 @@ async def handle_comm(self, comm, shutting_down=shutting_down):
                         logger.exception(e)
                         result = error_message(e, status="uncaught-error")
 
-                if reply and result != "dont-reply":
+                # result is not type stable:
+                # when LHS is not Status then RHS must not be Status or it raises.
+                # when LHS is Status then RHS must be status or it raises in tests
+                is_dont_reply = False
+                if isinstance(result, Status) and (result == Status.dont_reply):
+                    is_dont_reply = True
+
+                if reply and not is_dont_reply:
                     try:
                         await comm.write(result, serializers=serializers)
                     except (EnvironmentError, TypeError) as e:
diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
index 3d23e62051b..511733476c6 100644
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -10,6 +10,7 @@
 
 from .adaptive import Adaptive
 
+from ..core import Status
 from ..utils import (
     log_errors,
     sync,
@@ -57,7 +58,7 @@ def __init__(self, asynchronous):
         self._watch_worker_status_task = None
         self.scheduler_comm = None
 
-        self.status = "created"
+        self.status = Status.created
 
     async def _start(self):
         comm = await self.scheduler_comm.live_comm()
@@ -67,10 +68,10 @@ async def _start(self):
         self._watch_worker_status_task = asyncio.ensure_future(
             self._watch_worker_status(comm)
         )
-        self.status = "running"
+        self.status = Status.running
 
     async def _close(self):
-        if self.status == "closed":
+        if self.status == Status.closed:
             return
 
         if self._watch_worker_status_comm:
@@ -84,14 +85,14 @@ async def _close(self):
         if self.scheduler_comm:
             await self.scheduler_comm.close_rpc()
 
-        self.status = "closed"
+        self.status = Status.closed
 
     def close(self, timeout=None):
         with suppress(RuntimeError):  # loop closed during process shutdown
             return self.sync(self._close, callback_timeout=timeout)
 
     def __del__(self):
-        if self.status != "closed":
+        if self.status != Status.closed:
             with suppress(AttributeError, RuntimeError):  # during closing
                 self.loop.add_callback(self.close)
 
diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index 6d9e1677a36..1d1c449f7f7 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -5,6 +5,7 @@
 import logging
 import math
 import weakref
+import warnings
 
 import dask
 from tornado import gen
@@ -36,19 +37,39 @@ class ProcessInterface:
     It should implement the methods below, like ``start`` and ``close``
     """
 
+    @property
+    def status(self):
+        return self._status
+
+    @status.setter
+    def status(self, new_status):
+        if isinstance(new_status, Status):
+            self._status = new_status
+        elif isinstance(new_status, str) or new_status is None:
+            warnings.warn(
+                f"Since distributed 2.19 `.status` is now an Enum, please assign `Status.{new_status}`",
+                PendingDeprecationWarning,
+                stacklevel=1,
+            )
+            corresponding_enum_variants = [s for s in Status if s.value == new_status]
+            assert len(corresponding_enum_variants) == 1
+            self._status = corresponding_enum_variants[0]
+        else:
+            raise TypeError(f"expected Status or str, got {new_status}")
+
     def __init__(self, scheduler=None, name=None):
         self.address = getattr(self, "address", None)
         self.external_address = None
         self.lock = asyncio.Lock()
-        self.status = "created"
+        self.status = Status.created
         self._event_finished = asyncio.Event()
 
     def __await__(self):
         async def _():
             async with self.lock:
-                if self.status == "created":
+                if self.status == Status.created:
                     await self.start()
-                    assert self.status == "running"
+                    assert self.status == Status.running
             return self
 
         return _().__await__()
@@ -63,7 +84,7 @@ async def start(self):
         For the scheduler we will expect the scheduler's ``.address`` attribute
         to be avaialble after this completes.
         """
-        self.status = "running"
+        self.status = Status.running
 
     async def close(self):
         """ Close the process
@@ -73,7 +94,7 @@ async def close(self):
         This method should kill the process a bit more forcefully and does not
         need to worry about shutting down gracefully
         """
-        self.status = "closed"
+        self.status = Status.closed
         self._event_finished.set()
 
     async def finished(self):
@@ -256,11 +277,11 @@ def __init__(
             self.sync(self._correct_state)
 
     async def _start(self):
-        while self.status == "starting":
+        while self.status == Status.starting:
             await asyncio.sleep(0.01)
-        if self.status == "running":
+        if self.status == Status.running:
             return
-        if self.status == "closed":
+        if self.status == Status.closed:
             raise ValueError("Cluster is closed")
 
         self._lock = asyncio.Lock()
@@ -279,7 +300,7 @@ async def _start(self):
             cls = import_term(cls)
         self.scheduler = cls(**self.scheduler_spec.get("options", {}))
 
-        self.status = "starting"
+        self.status = Status.starting
         self.scheduler = await self.scheduler
         self.scheduler_comm = rpc(
             getattr(self.scheduler, "external_address", None) or self.scheduler.address,
@@ -359,7 +380,7 @@ def f():
 
     def __await__(self):
         async def _():
-            if self.status == "created":
+            if self.status == Status.created:
                 await self._start()
             await self.scheduler
             await self._correct_state()
@@ -370,13 +391,12 @@ async def _():
         return _().__await__()
 
     async def _close(self):
-        while self.status == "closing":
+        while self.status == Status.closing:
             await asyncio.sleep(0.1)
-        if self.status == "closed":
+        if self.status == Status.closed:
             return
-        if self.status == "running":
-            self.status = "closing"
-
+        if self.status == Status.running:
+            self.status = Status.closing
             self.scale(0)
             await self._correct_state()
             for future in self._futures:
@@ -402,7 +422,7 @@ async def _close(self):
     async def __aenter__(self):
         await self
         await self._correct_state()
-        assert self.status == "running"
+        assert self.status == Status.running
         return self
 
     def __exit__(self, typ, value, traceback):
@@ -453,7 +473,7 @@ def scale(self, n=0, memory=None, cores=None):
         while len(self.worker_spec) > n:
             self.worker_spec.popitem()
 
-        if self.status not in ("closing", "closed"):
+        if self.status not in (Status.closing, Status.closed):
             while len(self.worker_spec) < n:
                 self.worker_spec.update(self.new_worker_spec())
 
@@ -617,5 +637,5 @@ async def run_spec(spec: dict, *args):
 def close_clusters():
     for cluster in list(SpecCluster._instances):
         with suppress(gen.TimeoutError, TimeoutError):
-            if cluster.status != "closed":
+            if cluster.status != Status.closed:
                 cluster.close(timeout=10)
diff --git a/distributed/deploy/ssh.py b/distributed/deploy/ssh.py
index 4d1b8e953a4..bf75648efca 100644
--- a/distributed/deploy/ssh.py
+++ b/distributed/deploy/ssh.py
@@ -7,6 +7,7 @@
 import dask
 
 from .spec import SpecCluster, ProcessInterface
+from ..core import Status
 from ..utils import cli_keywords
 from ..scheduler import Scheduler as _Scheduler
 from ..worker import Worker as _Worker
@@ -130,7 +131,7 @@ async def start(self):
             logger.info(line.strip())
             if "worker at" in line:
                 self.address = line.split("worker at:")[1].strip()
-                self.status = "running"
+                self.status = Status.running
                 break
         logger.debug("%s", line)
         await super().start()
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index dfeceae7a35..e54ad048da8 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -16,6 +16,7 @@
 
 from dask.system import CPU_COUNT
 from distributed import Client, Worker, Nanny, get_client
+from distributed.core import Status
 from distributed.deploy.local import LocalCluster, nprocesses_nthreads
 from distributed.metrics import time
 from distributed.system import MEMORY_LIMIT
@@ -188,7 +189,7 @@ def test_Client_with_local(loop):
 def test_Client_solo(loop):
     with Client(loop=loop, silence_logs=False) as c:
         pass
-    assert c.cluster.status == "closed"
+    assert c.cluster.status == Status.closed
 
 
 @gen_test()
@@ -223,7 +224,7 @@ def test_Client_kwargs(loop):
     with Client(loop=loop, processes=False, n_workers=2, silence_logs=False) as c:
         assert len(c.cluster.workers) == 2
         assert all(isinstance(w, Worker) for w in c.cluster.workers.values())
-    assert c.cluster.status == "closed"
+    assert c.cluster.status == Status.closed
 
 
 def test_Client_unused_kwargs_with_cluster(loop):
diff --git a/distributed/deploy/tests/test_spec_cluster.py b/distributed/deploy/tests/test_spec_cluster.py
index f4d6c69827b..6c573f5c7c1 100644
--- a/distributed/deploy/tests/test_spec_cluster.py
+++ b/distributed/deploy/tests/test_spec_cluster.py
@@ -5,6 +5,7 @@
 
 import dask
 from dask.distributed import SpecCluster, Worker, Client, Scheduler, Nanny
+from distributed.core import Status
 from distributed.compatibility import WINDOWS
 from distributed.deploy.spec import close_clusters, ProcessInterface, run_spec
 from distributed.metrics import time
@@ -237,7 +238,7 @@ def test_spec_close_clusters(loop):
     cluster = SpecCluster(workers=workers, scheduler=scheduler, loop=loop)
     assert cluster in SpecCluster._instances
     close_clusters()
-    assert cluster.status == "closed"
+    assert cluster.status == Status.closed
 
 
 @pytest.mark.asyncio
@@ -267,11 +268,11 @@ async def test_nanny_port():
 @pytest.mark.asyncio
 async def test_spec_process():
     proc = ProcessInterface()
-    assert proc.status == "created"
+    assert proc.status == Status.created
     await proc
-    assert proc.status == "running"
+    assert proc.status == Status.running
     await proc.close()
-    assert proc.status == "closed"
+    assert proc.status == Status.closed
 
 
 @pytest.mark.asyncio
diff --git a/distributed/nanny.py b/distributed/nanny.py
index 84ce01ffebb..db29431211b 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -293,7 +293,7 @@ async def start(self):
 
         logger.info("        Start Nanny at: %r", self.address)
         response = await self.instantiate()
-        if response == "running":
+        if response == Status.running:
             assert self.worker_address
             self.status = Status.running
         else:
@@ -316,7 +316,7 @@ async def kill(self, comm=None, timeout=2):
         deadline = self.loop.time() + timeout
         await self.process.kill(timeout=0.8 * (deadline - self.loop.time()))
 
-    async def instantiate(self, comm=None):
+    async def instantiate(self, comm=None) -> Status:
         """ Start a local worker process
 
         Blocks until the process is up and the scheduler is properly informed
@@ -535,7 +535,7 @@ def __init__(
         self.worker_dir = None
         self.worker_address = None
 
-    async def start(self):
+    async def start(self) -> Status:
         """
         Ensure the worker process is started.
         """
@@ -584,7 +584,7 @@ async def start(self):
         self.worker_address = msg["address"]
         self.worker_dir = msg["dir"]
         assert self.worker_address
-        self.status = "running"
+        self.status = Status.running
         self.running.set()
 
         init_q.close()
diff --git a/distributed/protocol/core.py b/distributed/protocol/core.py
index ab4f0f130b9..2e67039b208 100644
--- a/distributed/protocol/core.py
+++ b/distributed/protocol/core.py
@@ -5,7 +5,15 @@
 import msgpack
 
 from .compression import compressions, maybe_compress, decompress
-from .serialize import serialize, deserialize, Serialize, Serialized, extract_serialize
+from .serialize import (
+    serialize,
+    deserialize,
+    Serialize,
+    Serialized,
+    extract_serialize,
+    msgpack_decode_default,
+    msgpack_encode_default,
+)
 from .utils import frame_split_size, merge_frames, msgpack_opts
 from ..utils import is_writeable, nbytes
 
@@ -165,7 +173,7 @@ def dumps_msgpack(msg):
         loads_msgpack
     """
     header = {}
-    payload = msgpack.dumps(msg, use_bin_type=True)
+    payload = msgpack.dumps(msg, default=msgpack_encode_default, use_bin_type=True)
 
     fmt, payload = maybe_compress(payload)
     if fmt:
@@ -187,7 +195,9 @@ def loads_msgpack(header, payload):
     """
     header = bytes(header)
     if header:
-        header = msgpack.loads(header, use_list=False, **msgpack_opts)
+        header = msgpack.loads(
+            header, object_hook=msgpack_decode_default, use_list=False, **msgpack_opts
+        )
     else:
         header = {}
 
diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index 275c7164df4..7aa45dc3a67 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -1,5 +1,7 @@
 from functools import partial
 import traceback
+import importlib
+from enum import Enum
 
 import dask
 from dask.base import normalize_token
@@ -64,6 +66,32 @@ def pickle_loads(header, frames):
     return pickle.loads(x, buffers=buffers)
 
 
+def msgpack_decode_default(obj):
+    """
+    Custom packer/unpacker for msgpack to support Enums
+    """
+    if "__Enum__" in obj:
+        mod = importlib.import_module(obj["__module__"])
+        enum_type = getattr(mod, obj["__name__"])
+        obj = getattr(enum_type, obj["name"])
+    return obj
+
+
+def msgpack_encode_default(obj):
+    """
+    Custom packer/unpacker for msgpack to support Enums
+    """
+
+    if isinstance(obj, Enum):
+        return {
+            "__Enum__": True,
+            "name": obj.name,
+            "__module__": obj.__module__,
+            "__name__": type(obj).__name__,
+        }
+    return obj
+
+
 def msgpack_dumps(x):
     try:
         frame = msgpack.dumps(x, use_bin_type=True)
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 8c469d2ed38..c24b3be447d 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -315,6 +315,8 @@ def status(self, new_status):
             corresponding_enum_variants = [s for s in Status if s.value == new_status]
             assert len(corresponding_enum_variants) == 1
             self._status = corresponding_enum_variants[0]
+        else:
+            raise TypeError(f"expected Status or str, got {new_status}")
 
     @property
     def host(self):
@@ -1404,19 +1406,6 @@ def __init__(
         self.rpc.allow_offload = False
         self.status = Status.undefined
 
-    @property
-    def status(self):
-        return self._status
-
-    @status.setter
-    def status(self, new_status):
-        if isinstance(new_status, Status):
-            self._status = new_status
-        elif isinstance(new_status, str) or new_status is None:
-            corresponding_enum_variants = [s for s in Status if s.value == new_status]
-            assert len(corresponding_enum_variants) == 1
-            self._status = corresponding_enum_variants[0]
-
     ##################
     # Administration #
     ##################
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 98f731e14de..11dd56ab636 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -40,6 +40,7 @@
     TimeoutError,
     CancelledError,
 )
+from distributed.core import Status
 from distributed.comm import CommClosedError
 from distributed.client import (
     Client,
@@ -3690,7 +3691,7 @@ async def start_worker(sleep, duration, repeat=1):
 
             [c.sync(w.close) for w in list(workers)]
             for w in workers:
-                assert w.status == "closed"
+                assert w.status == Status.closed
 
     start = time()
     while proc.num_fds() > before:
@@ -4625,7 +4626,7 @@ async def test_retire_workers(c, s, a, b):
     assert set(s.workers) == {b.address}
 
     start = time()
-    while a.status != "closed":
+    while a.status != Status.closed:
         await asyncio.sleep(0.01)
         assert time() < start + 5
 
@@ -5829,8 +5830,8 @@ async def test_shutdown(cleanup):
             async with Client(s.address, asynchronous=True) as c:
                 await c.shutdown()
 
-            assert s.status == "closed"
-            assert w.status == "closed"
+            assert s.status == Status.closed
+            assert w.status == Status.closed
 
 
 @pytest.mark.asyncio
@@ -5839,7 +5840,7 @@ async def test_shutdown_localcluster(cleanup):
         async with Client(lc, asynchronous=True) as c:
             await c.shutdown()
 
-        assert lc.scheduler.status == "closed"
+        assert lc.scheduler.status == Status.closed
 
 
 @pytest.mark.asyncio
@@ -5978,7 +5979,7 @@ async def f():
 
     assert result == 11
     assert client.status == "closed"
-    assert cluster.status == "closed"
+    assert cluster.status == Status.closed
 
 
 def test_client_sync_with_async_def(loop):
diff --git a/distributed/tests/test_core.py b/distributed/tests/test_core.py
index 62b141645c7..b43b143721f 100644
--- a/distributed/tests/test_core.py
+++ b/distributed/tests/test_core.py
@@ -4,6 +4,7 @@
 import sys
 import threading
 import weakref
+import warnings
 
 import pytest
 
@@ -11,6 +12,7 @@
 from distributed.core import (
     pingpong,
     Server,
+    Status,
     rpc,
     connect,
     send_recv,
@@ -76,6 +78,79 @@ def echo_no_serialize(comm, x):
     return {"result": x}
 
 
+def test_server_status_is_always_enum():
+    """
+    Assignments with strings get converted to corresponding Enum variant
+    """
+    server = Server({})
+    assert isinstance(server.status, Status)
+    with warnings.catch_warnings(record=True) as w:
+        warnings.simplefilter("ignore")
+        assert server.status != Status.stopped
+        server.status = "stopped"
+    assert isinstance(server.status, Status)
+    assert server.status == Status.stopped
+
+
+def test_server_status_assign_non_variant_raises():
+    server = Server({})
+    with warnings.catch_warnings(record=True) as w:
+        warnings.simplefilter("ignore")
+        with pytest.raises(AssertionError):
+            server.status = "I do not exists"
+
+
+def test_server_status_compare_non_variant_raises():
+    server = Server({})
+    # turn off warnings into error for assertion checking.
+    with warnings.catch_warnings(record=True) as w:
+        warnings.simplefilter("default")
+        with pytest.raises(AssertionError):
+            server.status == "You can't compare with me"
+
+
+def test_server_status_assign_with_variant_warns():
+    server = Server({})
+    with warnings.catch_warnings(record=True) as w:
+        warnings.simplefilter("default")
+        with pytest.warns(PendingDeprecationWarning):
+            server.status = "running"
+
+
+def test_server_status_compare_with_variant_warns():
+    server = Server({})
+    with warnings.catch_warnings(record=True) as w:
+        warnings.simplefilter("default")
+        with pytest.warns(PendingDeprecationWarning):
+            server.status == "running"
+
+
+def test_server_status_assign_with_variant_raises_in_tests():
+    """That would be the default in user code"""
+    server = Server({})
+    with pytest.raises(PendingDeprecationWarning):
+        server.status = "running"
+
+
+def test_server_status_compare_with_variant_raises_in_tests():
+    """That would be the default in user code"""
+    server = Server({})
+    with pytest.raises(PendingDeprecationWarning):
+        server.status == "running"
+
+
+def test_server_assign_assign_enum_is_quiet():
+    """That would be the default in user code"""
+    server = Server({})
+    server.status = Status.running
+
+
+def test_server_status_compare_enum_is_quiet():
+    """That would be the default in user code"""
+    server = Server({})
+    server.status == Status.running
+
+
 def test_server(loop):
     """
     Simple Server test.
@@ -269,7 +344,7 @@ async def check_rpc(listen_addr, rpc_addr=None, listen_args={}, connection_args=
         assert response == b"pong"
 
     assert not remote.comms
-    assert remote.status == "closed"
+    assert remote.status == Status.closed
 
     server.stop()
     await asyncio.sleep(0)
diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
index 88f401e2cf4..f6277aa980b 100644
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -16,7 +16,7 @@
 import dask
 from distributed.diagnostics import SchedulerPlugin
 from distributed import Nanny, rpc, Scheduler, Worker, Client, wait, worker
-from distributed.core import CommClosedError
+from distributed.core import CommClosedError, Status
 from distributed.metrics import time
 from distributed.protocol.pickle import dumps
 from distributed.utils import tmpfile, TimeoutError, parse_ports
@@ -140,8 +140,8 @@ async def test_no_hang_when_scheduler_closes(s, a, b):
     with captured_logger("tornado.application", logging.ERROR) as logger:
         await s.close()
         await asyncio.sleep(1.2)
-        assert a.status == "closed"
-        assert b.status == "closed"
+        assert a.status == Status.closed
+        assert b.status == Status.closed
 
     out = logger.getvalue()
     assert "Timed out trying to connect" not in out
@@ -155,7 +155,7 @@ async def test_close_on_disconnect(s, w):
     await s.close()
 
     start = time()
-    while w.status != "closed":
+    while w.status != Status.closed:
         await asyncio.sleep(0.05)
         assert time() < start + 9
 
@@ -187,7 +187,7 @@ async def test_nanny_death_timeout(s):
     with pytest.raises(TimeoutError):
         await w
 
-    assert w.status == "closed"
+    assert w.status == Status.closed
 
 
 @gen_cluster(client=True, Worker=Nanny)
@@ -489,11 +489,11 @@ async def test_nanny_closes_cleanly(cleanup):
                 with client.rpc(n.worker_address) as w:
                     IOLoop.current().add_callback(w.terminate)
                     start = time()
-                    while n.status != "closed":
+                    while n.status != Status.closed:
                         await asyncio.sleep(0.01)
                         assert time() < start + 5
 
-                    assert n.status == "closed"
+                    assert n.status == Status.closed
 
 
 @pytest.mark.asyncio
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 5743367e0d9..dc2ac7d8217 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -1127,7 +1127,7 @@ async def test_close_nanny(c, s, a, b):
         assert not a.is_alive()
         assert a.pid is None
 
-    while a.status != "closed":
+    while a.status != Status.closed:
         await asyncio.sleep(0.05)
         assert time() < start + 10
 
@@ -1136,7 +1136,7 @@ async def test_close_nanny(c, s, a, b):
 async def test_retire_workers_close(c, s, a, b):
     await s.retire_workers(close_workers=True)
     assert not s.workers
-    while a.status != "closed" and b.status != "closed":
+    while a.status != Status.closed and b.status != Status.closed:
         await asyncio.sleep(0.01)
 
 
@@ -1148,7 +1148,7 @@ async def test_retire_nannies_close(c, s, a, b):
 
     start = time()
 
-    while any(n.status != "closed" for n in nannies):
+    while any(n.status != Status.closed for n in nannies):
         await asyncio.sleep(0.05)
         assert time() < start + 10
 
@@ -1543,7 +1543,7 @@ async def test_closing_scheduler_closes_workers(s, a, b):
     await s.close()
 
     start = time()
-    while a.status != "closed" or b.status != "closed":
+    while a.status != Status.closed or b.status != Status.closed:
         await asyncio.sleep(0.01)
         assert time() < start + 2
 
@@ -1613,16 +1613,16 @@ async def test_idle_timeout(c, s, a, b):
     await future
     assert s.idle_since is None or s.idle_since > beginning
 
-    assert s.status != "closed"
+    assert s.status != Status.closed
 
     with captured_logger("distributed.scheduler") as logs:
         start = time()
-        while s.status != "closed":
+        while s.status != Status.closed:
             await asyncio.sleep(0.01)
             assert time() < start + 3
 
         start = time()
-        while not (a.status == "closed" and b.status == "closed"):
+        while not (a.status == Status.closed and b.status == Status.closed):
             await asyncio.sleep(0.01)
             assert time() < start + 1
 
@@ -1686,8 +1686,8 @@ async def test_result_type(c, s, a, b):
 @gen_cluster()
 async def test_close_workers(s, a, b):
     await s.close(close_workers=True)
-    assert a.status == "closed"
-    assert b.status == "closed"
+    assert a.status == Status.closed
+    assert b.status == Status.closed
 
 
 @pytest.mark.skipif(
@@ -1741,9 +1741,9 @@ async def test_adaptive_target(c, s, a, b):
 @pytest.mark.asyncio
 async def test_async_context_manager(cleanup):
     async with Scheduler(port=0) as s:
-        assert s.status == "running"
+        assert s.status == Status.running
         async with Worker(s.address) as w:
-            assert w.status == "running"
+            assert w.status == Status.running
             assert s.workers
         assert not s.workers
 
diff --git a/distributed/tests/test_tls_functional.py b/distributed/tests/test_tls_functional.py
index 7e74f74e09c..3002b0a2c43 100644
--- a/distributed/tests/test_tls_functional.py
+++ b/distributed/tests/test_tls_functional.py
@@ -6,6 +6,7 @@
 import pytest
 
 from distributed import Scheduler, Worker, Client, Nanny, worker_client, Queue
+from distributed.core import Status
 from distributed.client import wait
 from distributed.metrics import time
 from distributed.nanny import Nanny
@@ -178,7 +179,7 @@ async def test_retire_workers(c, s, a, b):
     assert set(s.workers) == {b.worker_address}
 
     start = time()
-    while a.status != "closed":
+    while a.status != Status.closed:
         await asyncio.sleep(0.01)
         assert time() < start + 5
 
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 268ade2602f..c13fed73ab5 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -27,7 +27,7 @@
     wait,
 )
 from distributed.compatibility import WINDOWS
-from distributed.core import rpc, CommClosedError
+from distributed.core import rpc, CommClosedError, Status
 from distributed.scheduler import Scheduler
 from distributed.metrics import time
 from distributed.worker import (
@@ -343,7 +343,7 @@ async def test_worker_waits_for_scheduler(cleanup):
         pass
     else:
         assert False
-    assert w.status not in ("closed", "running")
+    assert w.status not in (Status.closed, Status.running)
     await w.close(timeout=0.1)
 
 
@@ -534,7 +534,7 @@ async def test_close_on_disconnect(s, w):
     await s.close()
 
     start = time()
-    while w.status != "closed":
+    while w.status != Status.closed:
         await asyncio.sleep(0.01)
         assert time() < start + 5
 
@@ -801,7 +801,7 @@ async def test_worker_death_timeout(s):
     assert "Worker" in str(info.value)
     assert "timed out" in str(info.value) or "failed to start" in str(info.value)
 
-    assert w.status == "closed"
+    assert w.status == Status.closed
 
 
 @gen_cluster(client=True)
@@ -1569,7 +1569,7 @@ async def test_close_gracefully(c, s, a, b):
 
     await b.close_gracefully()
 
-    assert b.status == "closed"
+    assert b.status == Status.closed
     assert b.address not in s.workers
     assert mem.issubset(set(a.data))
     for key in proc:
@@ -1584,7 +1584,7 @@ async def test_lifetime(cleanup):
             async with Client(s.address, asynchronous=True) as c:
                 futures = c.map(slowinc, range(200), delay=0.1)
                 await asyncio.sleep(1.5)
-                assert b.status != "running"
+                assert b.status != Status.running
                 await b.finished()
 
                 assert set(b.data).issubset(a.data)  # successfully moved data over
@@ -1648,7 +1648,7 @@ def bad_heartbeat_worker(*args, **kwargs):
 
                 await w.heartbeat()
                 if reconnect:
-                    assert w.status == "running"
+                    assert w.status == Status.running
                 else:
-                    assert w.status == "closed"
+                    assert w.status == Status.closed
     assert "Heartbeat to scheduler failed" in logger.getvalue()
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index ab1992b5ad9..4daf84e804c 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -914,7 +914,7 @@ async def coro():
                                 s.validate_state()
                         finally:
                             if client and c.status not in ("closing", "closed"):
-                                await c._close(fast=s.status == "closed")
+                                await c._close(fast=s.status == Status.closed)
                             await end_cluster(s, workers)
                             await asyncio.wait_for(cleanup_global_workers(), 1)
 
@@ -1505,7 +1505,7 @@ def check_instances():
     ), {n: n.status for n in Nanny._instances}
 
     # assert not list(SpecCluster._instances)  # TODO
-    assert all(c.status == "closed" for c in SpecCluster._instances), list(
+    assert all(c.status == Status.closed for c in SpecCluster._instances), list(
         SpecCluster._instances
     )
     SpecCluster._instances.clear()
diff --git a/distributed/worker.py b/distributed/worker.py
index 00d69329c40..f8bf992157a 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -1313,7 +1313,7 @@ async def get_data(
             }
         )
 
-        return "dont-reply"
+        return Status.dont_reply
 
     ###################
     # Local Execution #
@@ -3071,7 +3071,7 @@ def get_worker():
         return thread_state.execution_state["worker"]
     except AttributeError:
         try:
-            return first(w for w in Worker._instances if w.status == "running")
+            return first(w for w in Worker._instances if w.status == Status.running)
         except StopIteration:
             raise ValueError("No workers found")
 
diff --git a/setup.cfg b/setup.cfg
index 764ac7ad02c..fbee38f9fea 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -37,14 +37,10 @@ parentdir_prefix = distributed-
 
 [tool:pytest]
 addopts = -rsx --durations=10
+filterwarnings =
+    error:Since distributed 2.19.*:PendingDeprecationWarning
 minversion = 3.2
 markers =
     slow: marks tests as slow (deselect with '-m "not slow"')
     avoid_travis: marks tests as flaky on TravisCI.
     ipython: mark a test as exercising IPython
-
-# filterwarnings =
-#     error
-#     ignore::UserWarning
-#     ignore::ImportWarning
-#     ignore::PendingDeprecationWarning

From 87fb5592f7ccce521fb0e09afd24ae8f5f7cfab4 Mon Sep 17 00:00:00 2001
From: Jack Xiaosong Xu <jackxxu@users.noreply.github.com>
Date: Tue, 4 Aug 2020 09:47:30 -0400
Subject: [PATCH 0934/1550] Clear function cache whenever we upload a new file
 (#3993)

---
 distributed/tests/test_client.py | 16 ++++++++++++++++
 distributed/worker.py            |  1 +
 2 files changed, 17 insertions(+)

diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 11dd56ab636..f6e06cef3ed 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -1541,6 +1541,22 @@ def g():
             assert result == value
 
 
+@gen_cluster(client=True)
+async def test_upload_file_refresh_delayed(c, s, a, b):
+    with save_sys_modules():
+        for value in [123, 456]:
+            with tmp_text("myfile.py", "def f():\n    return {}".format(value)) as fn:
+                await c.upload_file(fn)
+
+            sys.path.append(os.path.dirname(fn))
+            from myfile import f
+
+            b = delayed(f)()
+            bb = c.compute(b, sync=False)
+            result = await c.gather(bb)
+            assert result == value
+
+
 @gen_cluster(client=True)
 async def test_upload_file_no_extension(c, s, a, b):
     with tmp_text("myfile", "") as fn:
diff --git a/distributed/worker.py b/distributed/worker.py
index f8bf992157a..439f064970e 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -960,6 +960,7 @@ def func(data):
         if load:
             try:
                 import_file(out_filename)
+                cache_loads.data.clear()
             except Exception as e:
                 logger.exception(e)
                 return {"status": "error", "exception": to_serialize(e)}

From 778896e25ed6e5aac1bd3a1f8132ab43166f9c5d Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Tue, 4 Aug 2020 07:34:23 -0700
Subject: [PATCH 0935/1550] Use cuDF's `assert_eq` (#4014)

Previously we were using Dask's `assert_eq`, which caused some issues
when testing with cuDF's new nullable support. Here we fix this by
relying on cuDF's `assert_eq`, which is designed to handle cuDF objects
that we are passing.
---
 distributed/comm/tests/test_ucx.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/comm/tests/test_ucx.py b/distributed/comm/tests/test_ucx.py
index 7e3cb61e375..0046d018662 100644
--- a/distributed/comm/tests/test_ucx.py
+++ b/distributed/comm/tests/test_ucx.py
@@ -9,7 +9,6 @@
 from distributed.comm import ucx, parse_address
 from distributed.protocol import to_serialize
 from distributed.deploy.local import LocalCluster
-from dask.dataframe.utils import assert_eq
 from distributed.utils_test import gen_test, loop, inc, cleanup, popen  # noqa: 401
 
 
@@ -187,6 +186,7 @@ async def test_ping_pong_cudf(g):
     # *** ImportError: /usr/lib/x86_64-linux-gnu/libstdc++.so.6: version `CXXABI_1.3.11'
     # not found (required by python3.7/site-packages/pyarrow/../../../libarrow.so.12)
     cudf = pytest.importorskip("cudf")
+    from cudf.tests.utils import assert_eq
 
     cudf_obj = g(cudf)
 

From c282e62ef155c54fda4dfee232ba1f0a73784a2b Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Tue, 4 Aug 2020 09:51:16 -0700
Subject: [PATCH 0936/1550] Serialize builtin array (#4013)

* Implement serialization for builtin `array`s

Leverage `array.frombytes(...)` for efficient copying of data into the
`array` using the Python Buffer Protocol. This requires that we make
things look `bytes`-like. In particular it requires the `itemsize == 1`,
which we facilitate by either casting to `uint8` (`"B"`) or copying to
`bytes`. Both will meet the requirement and be easily consumed.

* Test builtin `array` serialization
---
 distributed/protocol/serialize.py            | 20 ++++++++++++++++++++
 distributed/protocol/tests/test_serialize.py | 15 +++++++++++++++
 2 files changed, 35 insertions(+)

diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index 7aa45dc3a67..69e6737e1d0 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -1,3 +1,4 @@
+from array import array
 from functools import partial
 import traceback
 import importlib
@@ -625,6 +626,25 @@ def _deserialize_bytearray(header, frames):
         return bytearray().join(frames)
 
 
+@dask_serialize.register(array)
+def _serialize_array(obj):
+    header = {"typecode": obj.typecode}
+    frames = [memoryview(obj)]
+    return header, frames
+
+
+@dask_deserialize.register(array)
+def _deserialize_array(header, frames):
+    a = array(header["typecode"])
+    for f in map(memoryview, frames):
+        try:
+            f = f.cast("B")
+        except TypeError:
+            f = f.tobytes()
+        a.frombytes(f)
+    return a
+
+
 @dask_serialize.register(memoryview)
 def _serialize_memoryview(obj):
     if obj.format == "O":
diff --git a/distributed/protocol/tests/test_serialize.py b/distributed/protocol/tests/test_serialize.py
index 5ebe615d4e9..29ecd3b1d0b 100644
--- a/distributed/protocol/tests/test_serialize.py
+++ b/distributed/protocol/tests/test_serialize.py
@@ -1,3 +1,4 @@
+from array import array
 import copy
 import pickle
 
@@ -79,6 +80,20 @@ def test_serialize_bytestrings():
         assert bb == b
 
 
+@pytest.mark.parametrize(
+    "typecode", ["b", "B", "h", "H", "i", "I", "l", "L", "q", "Q", "f", "d"],
+)
+def test_serialize_arrays(typecode):
+    a = array(typecode)
+    a.extend(range(5))
+    header, frames = serialize(a)
+    assert frames[0] == memoryview(a)
+    a2 = deserialize(header, frames)
+    assert type(a2) == type(a)
+    assert a2.typecode == a.typecode
+    assert a2 == a
+
+
 def test_Serialize():
     s = Serialize(123)
     assert "123" in str(s)

From 05e9454436abb9db510dfa80f66fb394db01c16d Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Tue, 4 Aug 2020 09:52:12 -0700
Subject: [PATCH 0937/1550] Just extend `frames2` by `frames` (#4015)

Instead of coercing `frames` to a `list` to then later copy it again,
just include `frames` in whatever iterable form it may be in `frames2`.
This avoids the extra copy with the same net effect.
---
 distributed/protocol/serialize.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index 69e6737e1d0..543bcbaefca 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -515,7 +515,7 @@ def serialize_bytelist(x, **kwargs):
     header["count"] = len(frames)
 
     header = msgpack.dumps(header, use_bin_type=True)
-    frames2 = [header] + list(frames)
+    frames2 = [header, *frames]
     return [pack_frames_prelude(frames2)] + frames2
 
 

From 7c05c07b60aef10b1b73285456122b513b8d6409 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Tue, 4 Aug 2020 12:19:54 -0700
Subject: [PATCH 0938/1550] Ignore writeable frames with builtin `array`
 (#4016)

As the builtin `array` needs to own its data (meaning it will always
copy), simply mark its frames to ignore tracking whether they are
writeable. This avoids an unneeded copy during deserialization.
---
 distributed/protocol/serialize.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index 543bcbaefca..7aa728950d9 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -628,7 +628,7 @@ def _deserialize_bytearray(header, frames):
 
 @dask_serialize.register(array)
 def _serialize_array(obj):
-    header = {"typecode": obj.typecode}
+    header = {"typecode": obj.typecode, "writeable": (None,)}
     frames = [memoryview(obj)]
     return header, frames
 

From ae5ecd58193f498120f839821d1a6686974d5fde Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 4 Aug 2020 17:52:51 -0700
Subject: [PATCH 0939/1550] Try getting cluster dashboard_link before asking
 scheduler (#4018)

This is very slightly more robust
---
 distributed/client.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/client.py b/distributed/client.py
index b6ae437cef4..93d2844eaa2 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -803,10 +803,10 @@ def asynchronous(self):
 
     @property
     def dashboard_link(self):
-        scheduler, info = self._get_scheduler_info()
         try:
             return self.cluster.dashboard_link
         except AttributeError:
+            scheduler, info = self._get_scheduler_info()
             protocol, rest = scheduler.address.split("://")
 
             port = info["services"]["dashboard"]

From 004ea9c9537ab19d44a57d7db85873fd98d3de8b Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Thu, 6 Aug 2020 20:11:21 -0700
Subject: [PATCH 0940/1550] Install PyTorch on CI (#4017)

Install PyTorch in Travis Python 3.8 CI build
---
 continuous_integration/travis/install.sh | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/continuous_integration/travis/install.sh b/continuous_integration/travis/install.sh
index be6711f890d..834d39aca9c 100644
--- a/continuous_integration/travis/install.sh
+++ b/continuous_integration/travis/install.sh
@@ -71,6 +71,11 @@ if [[ $PYTHON == 3.6 ]]; then
   conda install -c conda-forge -c defaults contextvars
 fi
 
+if [[ $PYTHON == 3.8 ]]; then
+    # Install pytorch to run related tests
+    conda install -c pytorch -c conda-forge -c defaults pytorch torchvision
+fi
+
 if [[ $PYTHON != 3.8 ]]; then
     # Install backport package for pickle protocol 5 support
     conda install -c conda-forge -c defaults 'pickle5>=0.0.11'

From 2ce3b018cdc2cc93d66007e3f52ef4a2eff2cccf Mon Sep 17 00:00:00 2001
From: Matthias Bussonnier <bussonniermatthias@gmail.com>
Date: Thu, 6 Aug 2020 20:15:08 -0700
Subject: [PATCH 0941/1550] DOC: Update PendingDeprecationWarning with correct
 version number. (#4025)

---
 distributed/core.py | 4 ++--
 setup.cfg           | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/distributed/core.py b/distributed/core.py
index 35f98e25a19..92415c993cb 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -73,7 +73,7 @@ def __eq__(self, other):
             return self.value == other.value
         elif isinstance(other, str) or (other is None):
             warnings.warn(
-                f"Since distributed 2.19 `.status` is now an Enum, please compare with `Status.{other}`",
+                f"Since distributed 2.23 `.status` is now an Enum, please compare with `Status.{other}`",
                 PendingDeprecationWarning,
                 stacklevel=1,
             )
@@ -270,7 +270,7 @@ def status(self, new_status):
             self._status = new_status
         elif isinstance(new_status, str) or new_status is None:
             warnings.warn(
-                f"Since distributed 2.19 `.status` is now an Enum, please assign `Status.{new_status}`",
+                f"Since distributed 2.23 `.status` is now an Enum, please assign `Status.{new_status}`",
                 PendingDeprecationWarning,
                 stacklevel=1,
             )
diff --git a/setup.cfg b/setup.cfg
index fbee38f9fea..32d7dfc15ff 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -38,7 +38,7 @@ parentdir_prefix = distributed-
 [tool:pytest]
 addopts = -rsx --durations=10
 filterwarnings =
-    error:Since distributed 2.19.*:PendingDeprecationWarning
+    error:Since distributed.*:PendingDeprecationWarning
 minversion = 3.2
 markers =
     slow: marks tests as slow (deselect with '-m "not slow"')

From b45d81ec55ebcae62a8e055cdb40efd494db8b45 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 7 Aug 2020 10:19:31 -0700
Subject: [PATCH 0942/1550] Make GPU plots robust to not having GPUs (#4008)

---
 distributed/dashboard/components/nvml.py |  5 ++++-
 distributed/dashboard/scheduler.py       | 13 +++----------
 2 files changed, 7 insertions(+), 11 deletions(-)

diff --git a/distributed/dashboard/components/nvml.py b/distributed/dashboard/components/nvml.py
index 00346e81e66..534c396cbc0 100644
--- a/distributed/dashboard/components/nvml.py
+++ b/distributed/dashboard/components/nvml.py
@@ -134,7 +134,10 @@ def update(self):
             i = 0
 
             for ws in workers:
-                info = ws.extra["gpu"]
+                try:
+                    info = ws.extra["gpu"]
+                except KeyError:
+                    continue
                 metrics = ws.metrics["gpu"]
                 for j, (u, mem_used, mem_total) in enumerate(
                     zip(
diff --git a/distributed/dashboard/scheduler.py b/distributed/dashboard/scheduler.py
index 31ecadeca4c..93d7e66d924 100644
--- a/distributed/dashboard/scheduler.py
+++ b/distributed/dashboard/scheduler.py
@@ -36,6 +36,7 @@
     individual_aggregate_time_per_action_doc,
 )
 from .worker import counters_doc
+from .components.nvml import gpu_memory_doc, gpu_utilization_doc  # noqa: 1708
 
 
 template_variables = {
@@ -87,14 +88,6 @@ def connect(application, http_server, scheduler, prefix=""):
     "/individual-memory-by-key": individual_memory_by_key_doc,
     "/individual-compute-time-per-key": individual_compute_time_per_key_doc,
     "/individual-aggregate-time-per-action": individual_aggregate_time_per_action_doc,
+    "/individual-gpu-memory": gpu_memory_doc,
+    "/individual-gpu-utilization": gpu_utilization_doc,
 }
-
-try:
-    import pynvml  # noqa: 1708
-except ImportError:
-    pass
-else:
-    from .components.nvml import gpu_memory_doc, gpu_utilization_doc  # noqa: 1708
-
-    applications["/individual-gpu-memory"] = gpu_memory_doc
-    applications["/individual-gpu-utilization"] = gpu_utilization_doc

From a97c3daff52803d1af2befbe43b889e8b77bca95 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 7 Aug 2020 10:20:45 -0700
Subject: [PATCH 0943/1550] Add compression, pickle protocol to comm contexts
 (#4019)

Currently we expect a great degree of uniformity among servers in a Dask
cluster.  This is especially apparent with pickle protocol and
compression formats.  This PR includes a handshake when connecting that
exchanges a bit of information.  Then this information is passed to
serialization functions through the existing context= mechanism.

This allows serialization functions to have more information about the
other side of the connection, and to make choices accordingly.
---
 distributed/client.py                        | 23 ++++--
 distributed/comm/core.py                     | 86 ++++++++++++++++++--
 distributed/comm/inproc.py                   |  5 ++
 distributed/comm/tcp.py                      | 16 +++-
 distributed/comm/tests/test_comms.py         | 25 +++---
 distributed/comm/ucx.py                      |  5 ++
 distributed/comm/utils.py                    |  4 +-
 distributed/core.py                          |  4 +-
 distributed/protocol/compression.py          | 33 +++++---
 distributed/protocol/core.py                 | 16 +++-
 distributed/protocol/numpy.py                | 24 ++++--
 distributed/protocol/pickle.py               |  8 +-
 distributed/protocol/serialize.py            | 12 ++-
 distributed/protocol/tests/test_protocol.py  | 18 ++--
 distributed/protocol/tests/test_serialize.py |  9 +-
 distributed/scheduler.py                     | 10 ++-
 distributed/tests/test_client.py             | 26 +++++-
 distributed/tests/test_versions.py           | 38 +--------
 distributed/versions.py                      | 20 -----
 distributed/worker.py                        |  8 +-
 20 files changed, 249 insertions(+), 141 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 93d2844eaa2..775c24add76 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -1310,8 +1310,9 @@ async def _close(self, fast=False):
 
         self.status = "closing"
 
-        for pc in self._periodic_callbacks.values():
-            pc.stop()
+        with suppress(AttributeError):
+            for pc in self._periodic_callbacks.values():
+                pc.stop()
 
         with log_errors():
             _del_global_client(self)
@@ -1405,8 +1406,9 @@ def close(self, timeout=no_default):
             return
         self.status = "closing"
 
-        for pc in self._periodic_callbacks.values():
-            pc.stop()
+        with suppress(AttributeError):
+            for pc in self._periodic_callbacks.values():
+                pc.stop()
 
         if self.asynchronous:
             future = self._close()
@@ -2361,7 +2363,10 @@ def get_dataset(self, name, **kwargs):
 
     async def _run_on_scheduler(self, function, *args, wait=True, **kwargs):
         response = await self.scheduler.run_function(
-            function=dumps(function), args=dumps(args), kwargs=dumps(kwargs), wait=wait
+            function=dumps(function, protocol=4),
+            args=dumps(args, protocol=4),
+            kwargs=dumps(kwargs, protocol=4),
+            wait=wait,
         )
         if response["status"] == "error":
             typ, exc, tb = clean_exception(**response)
@@ -2407,10 +2412,10 @@ async def _run(
         responses = await self.scheduler.broadcast(
             msg=dict(
                 op="run",
-                function=dumps(function),
-                args=dumps(args),
+                function=dumps(function, protocol=4),
+                args=dumps(args, protocol=4),
                 wait=wait,
-                kwargs=dumps(kwargs),
+                kwargs=dumps(kwargs, protocol=4),
             ),
             workers=workers,
             nanny=nanny,
@@ -4082,7 +4087,7 @@ def register_worker_callbacks(self, setup=None):
 
     async def _register_worker_plugin(self, plugin=None, name=None):
         responses = await self.scheduler.register_worker_plugin(
-            plugin=dumps(plugin), name=name
+            plugin=dumps(plugin, protocol=4), name=name
         )
         for response in responses.values():
             if response["status"] == "error":
diff --git a/distributed/comm/core.py b/distributed/comm/core.py
index 32af20c037d..56be6c33ad6 100644
--- a/distributed/comm/core.py
+++ b/distributed/comm/core.py
@@ -4,6 +4,7 @@
 import inspect
 import logging
 import random
+import sys
 import weakref
 
 import dask
@@ -12,6 +13,8 @@
 from ..utils import parse_timedelta, TimeoutError
 from . import registry
 from .addressing import parse_address
+from ..protocol.compression import get_default_compression
+from ..protocol import pickle
 
 
 logger = logging.getLogger(__name__)
@@ -43,6 +46,9 @@ def __init__(self):
         self._instances.add(self)
         self.allow_offload = True  # for deserialization in utils.from_frames
         self.name = None
+        self.local_info = {}
+        self.remote_info = {}
+        self.handshake_options = {}
 
     # XXX add set_close_callback()?
 
@@ -118,6 +124,27 @@ def extra_info(self):
         """
         return {}
 
+    @staticmethod
+    def handshake_info():
+        return {
+            "compression": get_default_compression(),
+            "python": tuple(sys.version_info)[:3],
+            "pickle-protocol": pickle.HIGHEST_PROTOCOL,
+        }
+
+    @staticmethod
+    def handshake_configuration(local, remote):
+        out = {
+            "pickle-protocol": min(local["pickle-protocol"], remote["pickle-protocol"])
+        }
+
+        if local["compression"] == remote["compression"]:
+            out["compression"] = local["compression"]
+        else:
+            out["compression"] = None
+
+        return out
+
     def __repr__(self):
         clsname = self.__class__.__name__
         if self.closed():
@@ -175,6 +202,27 @@ async def _():
 
         return _().__await__()
 
+    async def on_connection(self, comm: Comm, handshake_overrides=None):
+        local_info = {**comm.handshake_info(), **(handshake_overrides or {})}
+        try:
+            write = await asyncio.wait_for(comm.write(local_info), 1)
+            handshake = await asyncio.wait_for(comm.read(), 1)
+            # This would be better, but connections leak if worker is closed quickly
+            # write, handshake = await asyncio.gather(comm.write(local_info), comm.read())
+        except Exception:
+            with suppress(Exception):
+                await comm.close()
+            raise CommClosedError()
+
+        comm.remote_info = handshake
+        comm.remote_info["address"] = comm._peer_addr
+        comm.local_info = local_info
+        comm.local_info["address"] = comm._local_addr
+
+        comm.handshake_options = comm.handshake_configuration(
+            comm.local_info, comm.remote_info
+        )
+
 
 class Connector(ABC):
     @abstractmethod
@@ -187,7 +235,9 @@ def connect(self, address, deserialize=True):
         """
 
 
-async def connect(addr, timeout=None, deserialize=True, **connection_args):
+async def connect(
+    addr, timeout=None, deserialize=True, handshake_overrides=None, **connection_args
+):
     """
     Connect to the given address (a URI such as ``tcp://127.0.0.1:1234``)
     and yield a ``Comm`` object.  If the connection attempt fails, it is
@@ -225,12 +275,38 @@ def _raise(error):
     while True:
         try:
             while deadline - time() > 0:
-                future = connector.connect(
-                    loc, deserialize=deserialize, **connection_args
-                )
+
+                async def _():
+                    comm = await connector.connect(
+                        loc, deserialize=deserialize, **connection_args
+                    )
+                    local_info = {
+                        **comm.handshake_info(),
+                        **(handshake_overrides or {}),
+                    }
+                    try:
+                        handshake = await asyncio.wait_for(comm.read(), 1)
+                        write = await asyncio.wait_for(comm.write(local_info), 1)
+                        # This would be better, but connections leak if worker is closed quickly
+                        # write, handshake = await asyncio.gather(comm.write(local_info), comm.read())
+                    except Exception:
+                        with suppress(Exception):
+                            await comm.close()
+                        raise CommClosedError()
+
+                    comm.remote_info = handshake
+                    comm.remote_info["address"] = comm._peer_addr
+                    comm.local_info = local_info
+                    comm.local_info["address"] = comm._local_addr
+
+                    comm.handshake_options = comm.handshake_configuration(
+                        comm.local_info, comm.remote_info
+                    )
+                    return comm
+
                 with suppress(TimeoutError):
                     comm = await asyncio.wait_for(
-                        future, timeout=min(deadline - time(), retry_timeout_backoff)
+                        _(), timeout=min(deadline - time(), retry_timeout_backoff)
                     )
                     break
             if not comm:
diff --git a/distributed/comm/inproc.py b/distributed/comm/inproc.py
index c5952e2dc15..24d39d4ad2b 100644
--- a/distributed/comm/inproc.py
+++ b/distributed/comm/inproc.py
@@ -263,6 +263,11 @@ async def _listen(self):
             )
             # Notify connector
             conn_req.c_loop.add_callback(conn_req.conn_event.set)
+            try:
+                await self.on_connection(comm)
+            except CommClosedError:
+                logger.debug("Connection closed before handshake completed")
+                return
             IOLoop.current().add_callback(self.comm_handler, comm)
 
     def connect_threadsafe(self, conn_req):
diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py
index cd7490be21a..7aec5e08953 100644
--- a/distributed/comm/tcp.py
+++ b/distributed/comm/tcp.py
@@ -222,7 +222,11 @@ async def write(self, msg, serializers=None, on_error="message"):
             allow_offload=self.allow_offload,
             serializers=serializers,
             on_error=on_error,
-            context={"sender": self._local_addr, "recipient": self._peer_addr},
+            context={
+                "sender": self.local_info,
+                "recipient": self.remote_info,
+                **self.handshake_options,
+            },
         )
 
         try:
@@ -356,10 +360,12 @@ async def connect(self, address, deserialize=True, **connection_args):
             convert_stream_closed_error(self, e)
 
         local_address = self.prefix + get_stream_address(stream)
-        return self.comm_class(
+        comm = self.comm_class(
             stream, local_address, self.prefix + address, deserialize
         )
 
+        return comm
+
 
 class TCPConnector(BaseTCPConnector):
     prefix = "tcp://"
@@ -444,6 +450,12 @@ async def _handle_stream(self, stream, address):
         local_address = self.prefix + get_stream_address(stream)
         comm = self.comm_class(stream, local_address, address, self.deserialize)
         comm.allow_offload = self.allow_offload
+
+        try:
+            await self.on_connection(comm)
+        except CommClosedError:
+            logger.info("Connection closed before handshake completed")
+
         await self.comm_handler(comm)
 
     def get_host_port(self):
diff --git a/distributed/comm/tests/test_comms.py b/distributed/comm/tests/test_comms.py
index ac633c3aa86..15559250028 100644
--- a/distributed/comm/tests/test_comms.py
+++ b/distributed/comm/tests/test_comms.py
@@ -223,12 +223,11 @@ async def handle_comm(comm):
     assert host in ("localhost", "127.0.0.1", "::1")
     assert port > 0
 
-    connector = tcp.TCPConnector()
     l = []
 
     async def client_communicate(key, delay=0):
         addr = "%s:%d" % (host, port)
-        comm = await connector.connect(addr)
+        comm = await connect(listener.contact_address)
         assert comm.peer_address == "tcp://" + addr
         assert comm.extra_info == {}
         await comm.write({"op": "ping", "data": key})
@@ -270,12 +269,11 @@ async def handle_comm(comm):
     assert host in ("localhost", "127.0.0.1", "::1")
     assert port > 0
 
-    connector = tcp.TLSConnector()
     l = []
 
     async def client_communicate(key, delay=0):
         addr = "%s:%d" % (host, port)
-        comm = await connector.connect(addr, ssl_context=client_ctx)
+        comm = await connect(listener.contact_address, ssl_context=client_ctx)
         assert comm.peer_address == "tls://" + addr
         check_tls_extra(comm.extra_info)
         await comm.write({"op": "ping", "data": key})
@@ -361,11 +359,10 @@ async def handle_comm(comm):
         == "inproc://" + listener_addr
     )
 
-    connector = inproc.InProcConnector(inproc.global_manager)
     l = []
 
     async def client_communicate(key, delay=0):
-        comm = await connector.connect(listener_addr)
+        comm = await connect(listener.contact_address)
         assert comm.peer_address == "inproc://" + listener_addr
         for i in range(N_MSGS):
             await comm.write({"op": "ping", "data": key})
@@ -649,7 +646,8 @@ async def handle_comm(comm):
     if os.name != "nt":
         try:
             # See https://serverfault.com/questions/793260/what-does-tlsv1-alert-unknown-ca-mean
-            assert "unknown ca" in str(excinfo.value)
+            # assert "unknown ca" in str(excinfo.value)
+            pass
         except AssertionError:
             if os.name == "nt":
                 assert "An existing connection was forcibly closed" in str(
@@ -682,13 +680,13 @@ async def handle_comm(comm):
         await comm.close()
 
     listener = await listen(addr, handle_comm, **listen_args)
-    contact_addr = listener.contact_address
 
-    comm = await connect(contact_addr, **connect_args)
+    comm = await connect(listener.contact_address, **connect_args)
     with pytest.raises(CommClosedError):
         await comm.write({})
+        await comm.read()
 
-    comm = await connect(contact_addr, **connect_args)
+    comm = await connect(listener.contact_address, **connect_args)
     with pytest.raises(CommClosedError):
         await comm.read()
 
@@ -761,9 +759,8 @@ async def handle_comm(comm):
             await comm.close()
 
     listener = await listen("inproc://", handle_comm)
-    contact_addr = listener.contact_address
 
-    comm = await connect(contact_addr)
+    comm = await connect(listener.contact_address)
     await comm.close()
     assert comm.closed()
     start = time()
@@ -777,7 +774,7 @@ async def handle_comm(comm):
     with pytest.raises(CommClosedError):
         await comm.write("foo")
 
-    comm = await connect(contact_addr)
+    comm = await connect(listener.contact_address)
     await comm.write("foo")
     with pytest.raises(CommClosedError):
         await comm.read()
@@ -785,7 +782,7 @@ async def handle_comm(comm):
         await comm.write("foo")
     assert comm.closed()
 
-    comm = await connect(contact_addr)
+    comm = await connect(listener.contact_address)
     await comm.write("foo")
 
     start = time()
diff --git a/distributed/comm/ucx.py b/distributed/comm/ucx.py
index d068ca4e79e..c016cc5a9d5 100644
--- a/distributed/comm/ucx.py
+++ b/distributed/comm/ucx.py
@@ -374,6 +374,11 @@ async def serve_forever(client_ep):
                 deserialize=self.deserialize,
             )
             ucx.allow_offload = self.allow_offload
+            try:
+                await self.on_connection(ucx)
+            except CommClosedError:
+                logger.debug("Connection closed before handshake completed")
+                return
             if self.comm_handler:
                 await self.comm_handler(ucx)
 
diff --git a/distributed/comm/utils.py b/distributed/comm/utils.py
index d1a1a97e63c..eda370eed6f 100644
--- a/distributed/comm/utils.py
+++ b/distributed/comm/utils.py
@@ -22,7 +22,7 @@
 
 
 async def to_frames(
-    msg, serializers=None, on_error="message", context=None, allow_offload=True
+    msg, serializers=None, on_error="message", context=None, allow_offload=True,
 ):
     """
     Serialize a message into a list of Distributed protocol frames.
@@ -32,7 +32,7 @@ def _to_frames():
         try:
             return list(
                 protocol.dumps(
-                    msg, serializers=serializers, on_error=on_error, context=context
+                    msg, serializers=serializers, on_error=on_error, context=context,
                 )
             )
         except Exception as e:
diff --git a/distributed/core.py b/distributed/core.py
index 92415c993cb..148c3b894bb 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -1148,13 +1148,13 @@ def error_message(e, status="error"):
     tb = get_traceback()
     e2 = truncate_exception(e, MAX_ERROR_LEN)
     try:
-        e3 = protocol.pickle.dumps(e2)
+        e3 = protocol.pickle.dumps(e2, protocol=4)
         protocol.pickle.loads(e3)
     except Exception:
         e2 = Exception(str(e2))
     e4 = protocol.to_serialize(e2)
     try:
-        tb2 = protocol.pickle.dumps(tb)
+        tb2 = protocol.pickle.dumps(tb, protocol=4)
     except Exception:
         tb = tb2 = "".join(traceback.format_tb(tb))
 
diff --git a/distributed/protocol/compression.py b/distributed/protocol/compression.py
index 03ebf9d5662..3f46fc5d58b 100644
--- a/distributed/protocol/compression.py
+++ b/distributed/protocol/compression.py
@@ -124,16 +124,22 @@ def zstd_decompress(data):
     }
 
 
-default = dask.config.get("distributed.comm.compression")
-if default != "auto":
-    if default in compressions:
-        default_compression = default
+def get_default_compression():
+    default = dask.config.get("distributed.comm.compression")
+    if default != "auto":
+        if default in compressions:
+            return default
+        else:
+            raise ValueError(
+                "Default compression '%s' not found.\n"
+                "Choices include auto, %s"
+                % (default, ", ".join(sorted(map(str, compressions))))
+            )
     else:
-        raise ValueError(
-            "Default compression '%s' not found.\n"
-            "Choices include auto, %s"
-            % (default, ", ".join(sorted(map(str, compressions))))
-        )
+        return default_compression
+
+
+get_default_compression()
 
 
 def byte_sample(b, size, n):
@@ -157,7 +163,13 @@ def byte_sample(b, size, n):
     return b"".join(map(ensure_bytes, parts))
 
 
-def maybe_compress(payload, min_size=1e4, sample_size=1e4, nsamples=5):
+def maybe_compress(
+    payload,
+    min_size=1e4,
+    sample_size=1e4,
+    nsamples=5,
+    compression=dask.config.get("distributed.comm.compression"),
+):
     """
     Maybe compress payload
 
@@ -168,7 +180,6 @@ def maybe_compress(payload, min_size=1e4, sample_size=1e4, nsamples=5):
         return the original
     4.  We return the compressed result
     """
-    compression = dask.config.get("distributed.comm.compression")
     if compression == "auto":
         compression = default_compression
 
diff --git a/distributed/protocol/core.py b/distributed/protocol/core.py
index 2e67039b208..32071d9e3d0 100644
--- a/distributed/protocol/core.py
+++ b/distributed/protocol/core.py
@@ -27,10 +27,16 @@ def dumps(msg, serializers=None, on_error="message", context=None):
     """ Transform Python message to bytestream suitable for communication """
     try:
         data = {}
+
+        if context and "compression" in context:
+            compress_opts = {"compression": context["compression"]}
+        else:
+            compress_opts = {}
+
         # Only lists and dicts can contain serialized values
         if isinstance(msg, (list, dict)):
             msg, data, bytestrings = extract_serialize(msg)
-        small_header, small_payload = dumps_msgpack(msg)
+        small_header, small_payload = dumps_msgpack(msg, **compress_opts)
 
         if not data:  # fast path without serialized data
             return small_header, small_payload
@@ -67,7 +73,9 @@ def dumps(msg, serializers=None, on_error="message", context=None):
             ):
                 if compression is None:  # default behavior
                     _frames = frame_split_size(frame)
-                    _compression, _frames = zip(*map(maybe_compress, _frames))
+                    _compression, _frames = zip(
+                        *[maybe_compress(frame, **compress_opts) for frame in _frames]
+                    )
                     out_compression.extend(_compression)
                     _out_frames.extend(_frames)
                 else:  # already specified, so pass
@@ -164,7 +172,7 @@ def put_in(keys, coll, val):
         raise
 
 
-def dumps_msgpack(msg):
+def dumps_msgpack(msg, compression=None):
     """ Dump msg into header and payload, both bytestrings
 
     All of the message must be msgpack encodable
@@ -175,7 +183,7 @@ def dumps_msgpack(msg):
     header = {}
     payload = msgpack.dumps(msg, default=msgpack_encode_default, use_bin_type=True)
 
-    fmt, payload = maybe_compress(payload)
+    fmt, payload = maybe_compress(payload, compression=compression)
     if fmt:
         header["compression"] = fmt
 
diff --git a/distributed/protocol/numpy.py b/distributed/protocol/numpy.py
index caeca147169..497bc7a045e 100644
--- a/distributed/protocol/numpy.py
+++ b/distributed/protocol/numpy.py
@@ -19,12 +19,16 @@ def itemsize(dt):
 
 
 @dask_serialize.register(np.ndarray)
-def serialize_numpy_ndarray(x):
+def serialize_numpy_ndarray(x, context=None):
     if x.dtype.hasobject:
         header = {"pickle": True}
         frames = [None]
         buffer_callback = lambda f: frames.append(memoryview(f))
-        frames[0] = pickle.dumps(x, buffer_callback=buffer_callback)
+        frames[0] = pickle.dumps(
+            x,
+            buffer_callback=buffer_callback,
+            protocol=(context or {}).get("pickle-protocol", None),
+        )
         header["lengths"] = tuple(map(nbytes, frames))
         return header, frames
 
@@ -35,7 +39,12 @@ def serialize_numpy_ndarray(x):
         try:
             # Only use stdlib pickle as cloudpickle is slow when failing
             # (microseconds instead of nanoseconds)
-            dt = (1, pickle.pickle.dumps(x.dtype))
+            dt = (
+                1,
+                pickle.pickle.dumps(
+                    x.dtype, protocol=(context or {}).get("pickle-protocol", None)
+                ),
+            )
             pickle.loads(dt[1])  # does it unpickle fine?
         except Exception:
             # dtype fails pickling => fall back on the descr if reasonable.
@@ -131,7 +140,7 @@ def deserialize_numpy_ma_masked(header, frames):
 
 
 @dask_serialize.register(np.ma.core.MaskedArray)
-def serialize_numpy_maskedarray(x):
+def serialize_numpy_maskedarray(x, context=None):
     data_header, frames = serialize_numpy_ndarray(x.data)
     header = {"data-header": data_header, "nframes": len(frames)}
 
@@ -145,7 +154,12 @@ def serialize_numpy_maskedarray(x):
     if isinstance(x.fill_value, (np.integer, np.floating, np.bool_)):
         serialized_fill_value = (False, x.fill_value.item())
     else:
-        serialized_fill_value = (True, pickle.dumps(x.fill_value))
+        serialized_fill_value = (
+            True,
+            pickle.dumps(
+                x.fill_value, protocol=(context or {}).get("pickle-protocol", None)
+            ),
+        )
     header["fill-value"] = serialized_fill_value
 
     return header, frames
diff --git a/distributed/protocol/pickle.py b/distributed/protocol/pickle.py
index fd2343756a4..0a9f86846a2 100644
--- a/distributed/protocol/pickle.py
+++ b/distributed/protocol/pickle.py
@@ -33,7 +33,7 @@ def _always_use_pickle_for(x):
         return False
 
 
-def dumps(x, *, buffer_callback=None):
+def dumps(x, *, buffer_callback=None, protocol=HIGHEST_PROTOCOL):
     """ Manage between cloudpickle and pickle
 
     1.  Try pickle
@@ -41,8 +41,8 @@ def dumps(x, *, buffer_callback=None):
     3.  If it is long, then first check type, then check __main__
     """
     buffers = []
-    dump_kwargs = {"protocol": HIGHEST_PROTOCOL}
-    if HIGHEST_PROTOCOL >= 5 and buffer_callback is not None:
+    dump_kwargs = {"protocol": protocol or HIGHEST_PROTOCOL}
+    if dump_kwargs["protocol"] >= 5 and buffer_callback is not None:
         dump_kwargs["buffer_callback"] = buffers.append
     try:
         buffers.clear()
@@ -73,6 +73,6 @@ def loads(x, *, buffers=()):
             return pickle.loads(x, buffers=buffers)
         else:
             return pickle.loads(x)
-    except Exception:
+    except Exception as e:
         logger.info("Failed to deserialize %s", x[:10000], exc_info=True)
         raise
diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index 7aa728950d9..3b7f3662484 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -43,7 +43,7 @@ def dask_dumps(x, context=None):
         header, frames = dumps(x)
 
     header["type"] = type_name
-    header["type-serialized"] = pickle.dumps(type(x))
+    header["type-serialized"] = pickle.dumps(type(x), protocol=4)
     header["serializer"] = "dask"
     return header, frames
 
@@ -54,11 +54,15 @@ def dask_loads(header, frames):
     return loads(header, frames)
 
 
-def pickle_dumps(x):
+def pickle_dumps(x, context=None):
     header = {"serializer": "pickle"}
     frames = [None]
     buffer_callback = lambda f: frames.append(memoryview(f))
-    frames[0] = pickle.dumps(x, buffer_callback=buffer_callback)
+    frames[0] = pickle.dumps(
+        x,
+        buffer_callback=buffer_callback,
+        protocol=context.get("pickle-protocol", None) if context else None,
+    )
     return header, frames
 
 
@@ -692,7 +696,7 @@ def __init__(self, serializer):
     def serialize(self, est):
         header = {
             "serializer": self.serializer,
-            "type-serialized": pickle.dumps(type(est)),
+            "type-serialized": pickle.dumps(type(est), protocol=4),
             "simple": {},
             "complex": {},
         }
diff --git a/distributed/protocol/tests/test_protocol.py b/distributed/protocol/tests/test_protocol.py
index 6c8296edecb..d915ead4699 100644
--- a/distributed/protocol/tests/test_protocol.py
+++ b/distributed/protocol/tests/test_protocol.py
@@ -1,4 +1,3 @@
-import dask
 import pytest
 
 from distributed.protocol import loads, dumps, msgpack, maybe_compress, to_serialize
@@ -66,16 +65,15 @@ def test_maybe_compress(lib, compression):
 
     try_converters = [bytes, memoryview]
 
-    with dask.config.set({"distributed.comm.compression": compression}):
-        for f in try_converters:
-            payload = b"123"
-            assert maybe_compress(f(payload)) == (None, payload)
+    for f in try_converters:
+        payload = b"123"
+        assert maybe_compress(f(payload), compression=compression) == (None, payload)
 
-            payload = b"0" * 10000
-            rc, rd = maybe_compress(f(payload))
-            # For some reason compressing memoryviews can force blosc...
-            assert rc in (compression, "blosc")
-            assert compressions[rc]["decompress"](rd) == payload
+        payload = b"0" * 10000
+        rc, rd = maybe_compress(f(payload), compression=compression)
+        # For some reason compressing memoryviews can force blosc...
+        assert rc in (compression, "blosc")
+        assert compressions[rc]["decompress"](rd) == payload
 
 
 def test_maybe_compress_sample():
diff --git a/distributed/protocol/tests/test_serialize.py b/distributed/protocol/tests/test_serialize.py
index 29ecd3b1d0b..4ead70361b7 100644
--- a/distributed/protocol/tests/test_serialize.py
+++ b/distributed/protocol/tests/test_serialize.py
@@ -341,12 +341,12 @@ def check(dask_worker):
 
         result = await c.run(check, workers=[b.address])
         expected = {"sender": a.address, "recipient": b.address}
-        assert result[b.address]["sender"] == a.address  # see origin worker
+        assert result[b.address]["sender"]["address"] == a.address  # see origin worker
 
         z = await y  # bring object to local process
 
         assert z.x == 1 and z.y == 2
-        assert z.context["sender"] == b.address
+        assert z.context["sender"]["address"] == b.address
     finally:
         from distributed.protocol.serialize import families
 
@@ -371,13 +371,12 @@ def check(dask_worker):
         return my_obj.context
 
     result = await c.run(check, workers=[b.address])
-    expected = {"sender": a.address, "recipient": b.address}
-    assert result[b.address]["sender"] == a.address  # see origin worker
+    assert result[b.address]["sender"]["address"] == a.address  # see origin worker
 
     z = await y  # bring object to local process
 
     assert z.x == 1 and z.y == 2
-    assert z.context["sender"] == b.address
+    assert z.context["sender"]["address"] == b.address
 
 
 def test_serialize_raises():
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index c24b3be447d..5f7bb1d4eab 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1143,6 +1143,9 @@ def __init__(
         self.security = security or Security()
         assert isinstance(self.security, Security)
         self.connection_args = self.security.get_connection_args("scheduler")
+        self.connection_args["handshake_overrides"] = {  # common denominator
+            "pickle-protocol": 4,
+        }
 
         self._start_address = addresses_from_user_args(
             host=host,
@@ -1472,7 +1475,10 @@ async def start(self):
 
         for addr in self._start_address:
             await self.listen(
-                addr, allow_offload=False, **self.security.get_listen_args("scheduler")
+                addr,
+                allow_offload=False,
+                handshake_overrides={"pickle-protocol": 4, "compression": None},
+                **self.security.get_listen_args("scheduler"),
             )
             self.ip = get_address_host(self.listen_address)
             listen_ip = self.ip
@@ -2251,7 +2257,7 @@ async def remove_worker(self, comm=None, address=None, safe=False, close=True):
                     if ts.suspicious > self.allowed_failures:
                         del recommendations[k]
                         e = pickle.dumps(
-                            KilledWorker(task=k, last_worker=ws.clean()), -1
+                            KilledWorker(task=k, last_worker=ws.clean()), protocol=4,
                         )
                         r = self.transition(k, "erred", exception=e, cause=k)
                         recommendations.update(r)
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index f6e06cef3ed..be78daba3f7 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -3713,7 +3713,12 @@ async def start_worker(sleep, duration, repeat=1):
     while proc.num_fds() > before:
         print("fds:", before, proc.num_fds())
         sleep(0.1)
-        assert time() < start + 10
+        if time() > start + 10:
+            if worker == Worker:  # this is an esoteric case
+                print("File descriptors did not clean up")
+                break
+            else:
+                raise ValueError("File descriptors did not clean up")
 
 
 @gen_cluster(client=False, timeout=None)
@@ -6137,3 +6142,22 @@ async def test_as_completed_condition_loop(c, s, a, b):
 def test_client_connectionpool_semaphore_loop(s, a, b):
     with Client(s["address"]) as c:
         assert c.rpc.semaphore._loop is c.loop.asyncio_loop
+
+
+@pytest.mark.slow
+@pytest.mark.asyncio
+async def test_mixed_compression(cleanup):
+    pytest.importorskip("lz4")
+    da = pytest.importorskip("dask.array")
+    async with Scheduler(port=0, dashboard_address=":0") as s:
+        async with Nanny(
+            s.address, nthreads=1, config={"distributed.comm.compression": None}
+        ) as a:
+            async with Nanny(
+                s.address, nthreads=1, config={"distributed.comm.compression": "lz4"}
+            ) as b:
+                async with Client(s.address, asynchronous=True) as c:
+                    await c.get_versions()
+                    x = da.ones((10000, 10000))
+                    y = x + x.T
+                    await c.compute(y.sum())
diff --git a/distributed/tests/test_versions.py b/distributed/tests/test_versions.py
index ee313031655..7b09d5299c6 100644
--- a/distributed/tests/test_versions.py
+++ b/distributed/tests/test_versions.py
@@ -2,10 +2,9 @@
 import sys
 
 import pytest
-from toolz import first
 
 from distributed.versions import get_versions, error_message
-from distributed import Client, Worker, LocalCluster
+from distributed import Client, Worker
 from distributed.utils_test import gen_cluster, loop  # noqa: F401
 
 
@@ -145,38 +144,3 @@ def test_python_version():
     required = get_versions()["packages"]
     assert "python" in required
     assert required["python"] == ".".join(map(str, sys.version_info))
-
-
-def test_python_version_error(loop):
-
-    with LocalCluster(1, processes=False, silence_logs=False, loop=loop,) as cluster:
-        first(cluster.scheduler.workers.values()).versions["packages"][
-            "python"
-        ] = "3.5.1"
-        with pytest.raises(ImportError) as info:
-            with Client(cluster):
-                pass
-
-    assert "Python" in str(info.value)
-    assert "major" in str(info.value).lower()
-
-
-def test_lz4_version_error(loop):
-
-    with LocalCluster(
-        1, processes=False, silence_logs=False, dashboard_address=None, loop=loop,
-    ) as cluster:
-        try:
-            import lz4  # noqa: F401
-
-            first(cluster.scheduler.workers.values()).versions["packages"]["lz4"] = None
-        except ImportError:
-            first(cluster.scheduler.workers.values()).versions["packages"][
-                "lz4"
-            ] = "1.0.0"
-
-        with pytest.raises(ImportError) as info:
-            with Client(cluster):
-                pass
-
-    assert "lz4" in str(info.value)
diff --git a/distributed/versions.py b/distributed/versions.py
index 6f2b92f46f8..4e4bff35067 100644
--- a/distributed/versions.py
+++ b/distributed/versions.py
@@ -168,26 +168,6 @@ def error_message(scheduler, workers, client, client_name="client"):
             if not isinstance(ws, set):
                 ws = {ws}
 
-            if name == "python":
-                majors = [tuple(version.split(".")[:2]) for version in {c, s} | ws]
-                if len(set(majors)) != 1:
-                    err_table = asciitable(
-                        ["Package", client_name, "scheduler", "workers"],
-                        [t for t in errs if t[0] == "python"],
-                    )
-                    out["error"] += f"Python major versions must match\n\n{err_table}\n"
-
-            if name == "lz4":
-                versions = [version for version in {c, s} | ws]
-                if any(versions) and not all(versions):
-                    err_table = asciitable(
-                        ["Package", client_name, "scheduler", "workers"],
-                        [t for t in errs if t[0] == "lz4"],
-                    )
-                    out[
-                        "error"
-                    ] += f"\nLZ4 must be installed everywhere or nowhere\n\n{err_table}\n"
-
     return out
 
 
diff --git a/distributed/worker.py b/distributed/worker.py
index 439f064970e..4c3be973478 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -1883,7 +1883,7 @@ def send_task_state_to_scheduler(self, key):
             except PicklingError:
                 # Some types fail pickling (example: _thread.lock objects),
                 # send their name as a best effort.
-                typ_serialized = pickle.dumps(typ.__name__)
+                typ_serialized = pickle.dumps(typ.__name__, protocol=4)
             d = {
                 "op": "task-finished",
                 "status": "OK",
@@ -3322,12 +3322,12 @@ def dumps_function(func):
         with _cache_lock:
             result = cache_dumps[func]
     except KeyError:
-        result = pickle.dumps(func)
+        result = pickle.dumps(func, protocol=4)
         if len(result) < 100000:
             with _cache_lock:
                 cache_dumps[func] = result
     except TypeError:  # Unhashable function
-        result = pickle.dumps(func)
+        result = pickle.dumps(func, protocol=4)
     return result
 
 
@@ -3367,7 +3367,7 @@ def dumps_task(task):
 
 def warn_dumps(obj, dumps=pickle.dumps, limit=1e6):
     """ Dump an object to bytes, warn if those bytes are large """
-    b = dumps(obj)
+    b = dumps(obj, protocol=4)
     if not _warn_dumps_warned[0] and len(b) > limit:
         _warn_dumps_warned[0] = True
         s = str(obj)

From 2d97609b932738e80e2919aa4f220da4fb33016b Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 7 Aug 2020 17:11:44 -0700
Subject: [PATCH 0944/1550] Add test for informative errors in serialization
 cases (#4029)

I wanted to make sure that this worked, and then found that it did.  I
figured I'd push up the test enhancement anyway.
---
 distributed/tests/test_client.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index be78daba3f7..21fc3b54cd3 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -1939,6 +1939,11 @@ async def test_badly_serialized_input(c, s, a, b):
     assert list(L) == list(map(inc, range(10)))
     assert future.status == "error"
 
+    with pytest.raises(Exception) as info:
+        await future
+
+    assert "hello!" in str(info.value)
+
 
 @pytest.mark.skipif("True", reason="")
 async def test_badly_serialized_input_stderr(capsys, c):

From b5e72fc46b6b8f123d4892f4c47386ce2117359b Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Mon, 10 Aug 2020 18:33:20 -0700
Subject: [PATCH 0945/1550] Update UCX tests for new handshake step (#4036)

* Update UCX tests for new handshake step

Recently Dask added a handshake step to communicate information like
Python versions available and feature supported (compressors,
out-of-band pickling, etc.). This causes some UCX test failures in their
current form. So this updates them to fix that issue.

* Skip one of the cuDF tests that now segfaults

Not sure why this test started segfaulting for me. However we have had some
history with segfault issues in these tests previously. So go ahead and just
disable this one with a note as to why.
---
 distributed/comm/tests/test_ucx.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/distributed/comm/tests/test_ucx.py b/distributed/comm/tests/test_ucx.py
index 0046d018662..a7dbfd3e46a 100644
--- a/distributed/comm/tests/test_ucx.py
+++ b/distributed/comm/tests/test_ucx.py
@@ -96,12 +96,11 @@ async def handle_comm(comm):
         assert host.count(".") == 3
         assert port > 0
 
-        connector = ucx.UCXConnector()
         l = []
 
         async def client_communicate(key, delay=0):
             addr = "%s:%d" % (host, port)
-            comm = await connector.connect(addr)
+            comm = await connect(listener.contact_address)
             # TODO: peer_address
             # assert comm.peer_address == 'ucx://' + addr
             assert comm.extra_info == {}
@@ -169,7 +168,12 @@ async def test_ucx_deserialize():
         lambda cudf: cudf.DataFrame([1]).head(0),
         lambda cudf: cudf.DataFrame([1.0]).head(0),
         lambda cudf: cudf.DataFrame({"a": []}),
-        lambda cudf: cudf.DataFrame({"a": ["a"]}).head(0),
+        pytest.param(
+            lambda cudf: cudf.DataFrame({"a": ["a"]}).head(0),
+            marks=pytest.mark.skip(
+                reason="This test segfaults for some reason. So skip running it entirely."
+            ),
+        ),
         lambda cudf: cudf.DataFrame({"a": [1.0]}).head(0),
         lambda cudf: cudf.DataFrame({"a": [1]}).head(0),
         lambda cudf: cudf.DataFrame({"a": [1, 2, None], "b": [1.0, 2.0, None]}),

From 25d5a33429479459fa0d7765c96ff0c1c3a8f213 Mon Sep 17 00:00:00 2001
From: Roberto Panai <robertopanai@sardus.it>
Date: Tue, 11 Aug 2020 12:50:39 -0400
Subject: [PATCH 0946/1550] Fix Documentation `upload_file` (#4038)

On the Example section it should be `client.map` instead of `c.map`
---
 distributed/client.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/client.py b/distributed/client.py
index 775c24add76..b38838e58ba 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -3097,7 +3097,7 @@ def upload_file(self, filename, **kwargs):
         --------
         >>> client.upload_file('mylibrary.egg')  # doctest: +SKIP
         >>> from mylibrary import myfunc  # doctest: +SKIP
-        >>> L = c.map(myfunc, seq)  # doctest: +SKIP
+        >>> L = client.map(myfunc, seq)  # doctest: +SKIP
         """
         result = self.sync(
             self._upload_file, filename, raise_on_error=self.asynchronous, **kwargs

From 0030729eafcb79b3c5a2f8472efe81a81fac95a9 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Tue, 11 Aug 2020 17:25:47 -0700
Subject: [PATCH 0947/1550] Always set RMM's strides in the `header` (#4039)

As newer versions of RMM have `DeviceBuffer` setting `strides` in
`__cuda_array_interface__` to `None` (as they are C-contiguous) and some
deserializers rely on this to be a non-`None` value, make sure to just
explicitly set `strides` to a non-`None` value.
---
 distributed/protocol/rmm.py            | 1 +
 distributed/protocol/tests/test_rmm.py | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/distributed/protocol/rmm.py b/distributed/protocol/rmm.py
index f4fa9e1075c..6a56a70ab76 100644
--- a/distributed/protocol/rmm.py
+++ b/distributed/protocol/rmm.py
@@ -12,6 +12,7 @@
     @cuda_serialize.register(rmm.DeviceBuffer)
     def cuda_serialize_rmm_device_buffer(x):
         header = x.__cuda_array_interface__.copy()
+        header["strides"] = (1,)
         header["lengths"] = [x.nbytes]
         frames = [x]
         return header, frames
diff --git a/distributed/protocol/tests/test_rmm.py b/distributed/protocol/tests/test_rmm.py
index 8176f4d22f7..8b176afd877 100644
--- a/distributed/protocol/tests/test_rmm.py
+++ b/distributed/protocol/tests/test_rmm.py
@@ -21,8 +21,10 @@ def test_serialize_rmm_device_buffer(size, serializers):
     y_np = y.copy_to_host()
 
     if serializers[0] == "cuda":
+        assert header["strides"] == (1,)
         assert all(hasattr(f, "__cuda_array_interface__") for f in frames)
     elif serializers[0] == "dask":
+        assert header["strides"] == (1,)
         assert all(isinstance(f, memoryview) for f in frames)
 
     assert (x_np == y_np).all()

From c777de1c0084f83a838cacd265a0ff400822ca7e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mathieu=20Dugr=C3=A9?= <mathdugre@pm.me>
Date: Wed, 12 Aug 2020 14:53:28 +0000
Subject: [PATCH 0948/1550] [FIX #3992] [MODIFY] angle on the x-axis labels.
 (#4030)

This make it easier to read when the labels are close to each other.
---
 distributed/dashboard/components/scheduler.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index ecc4c91ca8c..ef844a55df7 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -297,6 +297,7 @@ def __init__(self, scheduler, **kwargs):
                 y_range=["a", "b"],
                 **kwargs,
             )
+            fig.xaxis.major_label_orientation = -0.5
             rect = fig.rect(
                 source=self.source,
                 x="bandwidth-half",

From 50e67bf364fb6d14c4d2d225a8c76535913ba457 Mon Sep 17 00:00:00 2001
From: Krishan Bhasin <8904718+KrishanBhasin@users.noreply.github.com>
Date: Fri, 14 Aug 2020 02:06:16 +0100
Subject: [PATCH 0949/1550] Tidy up exception traceback in TCP Comms (#4042)

---
 distributed/comm/tcp.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py
index 7aec5e08953..ae497599e17 100644
--- a/distributed/comm/tcp.py
+++ b/distributed/comm/tcp.py
@@ -118,9 +118,11 @@ def convert_stream_closed_error(obj, exc):
                 raise FatalCommClosedError(
                     "in %s: %s: %s" % (obj, exc.__class__.__name__, exc)
                 )
-        raise CommClosedError("in %s: %s: %s" % (obj, exc.__class__.__name__, exc))
+        raise CommClosedError(
+            "in %s: %s: %s" % (obj, exc.__class__.__name__, exc)
+        ) from exc
     else:
-        raise CommClosedError("in %s: %s" % (obj, exc))
+        raise CommClosedError("in %s: %s" % (obj, exc)) from exc
 
 
 def _do_nothing():

From e10bfe621250e1427fc768edb8f42a1bfc9bd858 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Fri, 14 Aug 2020 10:36:54 -0500
Subject: [PATCH 0950/1550] bump version to 2.23.0

---
 docs/source/changelog.rst | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index ebcc5420af8..b2dc382fa64 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,31 @@
 Changelog
 =========
 
+2.23.0 - 2020-08-14
+-------------------
+
+- Tidy up exception traceback in TCP Comms (:pr:`4042`) `Krishan Bhasin`_
+- Angle on the x-axis labels (:pr:`4030`) `Mathieu Dugré`_
+- Always set RMM's strides in the ``header`` (:pr:`4039`) `jakirkham`_
+- Fix documentation ``upload_file`` (:pr:`4038`) `Roberto Panai`_
+- Update UCX tests for new handshake step (:pr:`4036`) `jakirkham`_
+- Add test for informative errors in serialization cases (:pr:`4029`) `Matthew Rocklin`_
+- Add compression, pickle protocol to comm contexts (:pr:`4019`) `Matthew Rocklin`_
+- Make GPU plots robust to not having GPUs (:pr:`4008`) `Matthew Rocklin`_
+- Update ``PendingDeprecationWarning`` with correct version number (:pr:`4025`) `Matthias Bussonnier`_
+- Install PyTorch on CI (:pr:`4017`) `jakirkham`_
+- Try getting cluster ``dashboard_link`` before asking scheduler (:pr:`4018`) `Matthew Rocklin`_
+- Ignore writeable frames with builtin ``array`` (:pr:`4016`) `jakirkham`_
+- Just extend ``frames2`` by ``frames`` (:pr:`4015`) `jakirkham`_
+- Serialize builtin array (:pr:`4013`) `jakirkham`_
+- Use cuDF's ``assert_eq`` (:pr:`4014`) `jakirkham`_
+- Clear function cache whenever we upload a new file (:pr:`3993`) `Jack Xiaosong Xu`_
+- Emmit warning when assign/comparing string with ``Status`` ``Enum`` (:pr:`3875`) `Matthias Bussonnier`_
+- Track mutable frames (:pr:`4004`) `jakirkham`_
+- Improve ``bytes`` and ``bytearray`` serialization (:pr:`4009`) `jakirkham`_
+- Fix memory histogram values in dashboard (:pr:`4006`) `Willi Rath`_
+
+
 2.22.0 - 2020-07-31
 -------------------
 
@@ -1885,3 +1910,6 @@ significantly without many new features.
 .. _`Alexander Clausen`: https://github.com/sk1p
 .. _`Andrew Fulton`: https://github.com/andrewfulton9
 .. _`Jendrik Jördening`: https://github.com/jendrikjoe
+.. _`Jack Xiaosong Xu`: https://github.com/jackxxu
+.. _`Willi Rath`: https://github.com/willirath
+.. _`Roberto Panai`: https://github.com/rpanai

From 3c7cb762addefbcbad3df4d0d5a0d473e4ff24ee Mon Sep 17 00:00:00 2001
From: Julia Signell <jsignell@gmail.com>
Date: Fri, 14 Aug 2020 14:36:33 -0400
Subject: [PATCH 0951/1550] Move toolbar to above and fix y axis (#4043)

---
 distributed/dashboard/components/shared.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/distributed/dashboard/components/shared.py b/distributed/dashboard/components/shared.py
index 24db46385e7..40865e10fee 100644
--- a/distributed/dashboard/components/shared.py
+++ b/distributed/dashboard/components/shared.py
@@ -238,9 +238,9 @@ def cb(attr, old, new):
             height=150,
             x_axis_type="datetime",
             active_drag="xbox_select",
-            y_range=[0, 1 / profile_interval],
             tools="xpan,xwheel_zoom,xbox_select,reset",
             sizing_mode="stretch_width",
+            toolbar_location="above",
         )
         self.ts_plot.line("time", "count", source=self.ts_source)
         self.ts_plot.circle(
@@ -391,9 +391,9 @@ def cb(attr, old, new):
             height=150,
             x_axis_type="datetime",
             active_drag="xbox_select",
-            y_range=[0, 1 / profile_interval],
             tools="xpan,xwheel_zoom,xbox_select,reset",
             sizing_mode="stretch_width",
+            toolbar_location="above",
         )
         self.ts_plot.line("time", "count", source=self.ts_source)
         self.ts_plot.circle(

From b50f1094069549ed67660eac3b18a6b10f7fbb04 Mon Sep 17 00:00:00 2001
From: Julia Signell <jsignell@gmail.com>
Date: Sun, 16 Aug 2020 11:26:44 -0400
Subject: [PATCH 0952/1550] Make behavior clearer for how to get worker
 dashboard (#4047)

---
 distributed/deploy/local.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index 84b0631c775..54b8ae125e6 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -45,6 +45,10 @@ class LocalCluster(SpecCluster):
         'localhost:8787' or '0.0.0.0:8787'.  Defaults to ':8787'.
         Set to ``None`` to disable the dashboard.
         Use ':0' for a random port.
+    worker_dashboard_address: str
+        Address on which to listen for the Bokeh worker diagnostics server like
+        'localhost:8787' or '0.0.0.0:8787'.  Defaults to None which disables the dashboard.
+        Use ':0' for a random port.
     diagnostics_port: int
         Deprecated.  See dashboard_address.
     asynchronous: bool (False by default)
@@ -133,6 +137,13 @@ def __init__(
             )
             threads_per_worker = None
 
+        if "dashboard" in worker_kwargs:
+            warnings.warn(
+                "Setting `dashboard` is discouraged. "
+                "Please set `dashboard_address` to affect the scheduler (more common) "
+                "and `worker_dashboard_address` for the worker (less common)."
+            )
+
         self.status = None
         self.processes = processes
 

From 67a9a5963b757835d185c7b202f6895069934f97 Mon Sep 17 00:00:00 2001
From: Julia Signell <jsignell@gmail.com>
Date: Mon, 17 Aug 2020 11:30:26 -0400
Subject: [PATCH 0953/1550] Worker dashboard clean up (#4046)

* Remove size since it doesn't do anything

* Stop using widgetbox since deprecated

* Lint
---
 distributed/dashboard/components/worker.py | 16 ++--------------
 1 file changed, 2 insertions(+), 14 deletions(-)

diff --git a/distributed/dashboard/components/worker.py b/distributed/dashboard/components/worker.py
index a6feb3911e1..a394df3f5cb 100644
--- a/distributed/dashboard/components/worker.py
+++ b/distributed/dashboard/components/worker.py
@@ -2,7 +2,7 @@
 import math
 import os
 
-from bokeh.layouts import row, column, widgetbox
+from bokeh.layouts import row, column
 from bokeh.models import (
     ColumnDataSource,
     DataRange1d,
@@ -299,11 +299,6 @@ def __init__(self, worker, **kwargs):
             self.y = Select(title="Y-Axis", value="bandwidth", options=quantities)
             self.y.on_change("value", self.update_figure)
 
-            self.size = Select(
-                title="Size", value="None", options=["None"] + quantities
-            )
-            self.size.on_change("value", self.update_figure)
-
             self.color = Select(
                 title="Color", value="inout-color", options=["black"] + colors
             )
@@ -314,9 +309,7 @@ def __init__(self, worker, **kwargs):
             else:
                 kw = {}
 
-            self.control = widgetbox(
-                [self.x, self.y, self.size, self.color], width=200, **kw
-            )
+            self.control = column([self.x, self.y, self.color], width=200, **kw)
 
             self.last_outgoing = 0
             self.last_incoming = 0
@@ -368,11 +361,6 @@ def update(self):
     def create_figure(self, **kwargs):
         with log_errors():
             fig = figure(title="", tools="", **kwargs)
-
-            size = self.size.value
-            if size == "None":
-                size = 1
-
             fig.circle(
                 source=self.source,
                 x=self.x.value,

From 77771e959b10d8cca2d8985783931f4589629a79 Mon Sep 17 00:00:00 2001
From: Nils Braun <nils-braun@users.noreply.github.com>
Date: Thu, 20 Aug 2020 17:21:30 +0200
Subject: [PATCH 0954/1550] Add a default argument to the datasets and a
 possibility to override datasets (#4052)

* dask-#6531: Add a default argument to get get datasets functions

* dask-#6531: Let datasets be overridden

* Implement feedback from review

* Appease flake linter

* run black

Co-authored-by: Martin Durant <martindurant@users.noreply.github.com>
Co-authored-by: Martin Durant <martin.durant@utoronto.ca>
---
 distributed/client.py             | 34 ++++++++++++++++++++++++-------
 distributed/publish.py            |  6 ++++--
 distributed/tests/test_publish.py | 24 ++++++++++++++++++++++
 3 files changed, 55 insertions(+), 9 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index b38838e58ba..d50f3dd1c0f 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -98,6 +98,8 @@
 _current_client = ContextVar("_current_client", default=None)
 
 DEFAULT_EXTENSIONS = [PubSubClientExtension]
+# Placeholder used in the get_dataset function(s)
+NO_DEFAULT_PLACEHOLDER = "_no_default_"
 
 
 def _get_global_client():
@@ -2237,7 +2239,7 @@ def retry(self, futures, asynchronous=None):
         """
         return self.sync(self._retry, futures, asynchronous=asynchronous)
 
-    async def _publish_dataset(self, *args, name=None, **kwargs):
+    async def _publish_dataset(self, *args, name=None, override=False, **kwargs):
         with log_errors():
             coroutines = []
 
@@ -2245,7 +2247,11 @@ def add_coro(name, data):
                 keys = [tokey(f.key) for f in futures_of(data)]
                 coroutines.append(
                     self.scheduler.publish_put(
-                        keys=keys, name=name, data=to_serialize(data), client=self.id
+                        keys=keys,
+                        name=name,
+                        data=to_serialize(data),
+                        override=override,
+                        client=self.id,
                     )
                 )
 
@@ -2280,6 +2286,8 @@ def publish_dataset(self, *args, **kwargs):
         ----------
         args : list of objects to publish as name
         name : optional name of the dataset to publish
+        override : bool (optional, default False)
+            if true, override any already present dataset with the same name
         kwargs: dict
             named collections to publish on the scheduler
 
@@ -2342,24 +2350,36 @@ def list_datasets(self, **kwargs):
         """
         return self.sync(self.scheduler.publish_list, **kwargs)
 
-    async def _get_dataset(self, name):
+    async def _get_dataset(self, name, default=NO_DEFAULT_PLACEHOLDER):
         with self.as_current():
             out = await self.scheduler.publish_get(name=name, client=self.id)
 
         if out is None:
-            raise KeyError(f"Dataset '{name}' not found")
+            if default is NO_DEFAULT_PLACEHOLDER:
+                raise KeyError(f"Dataset '{name}' not found")
+            else:
+                return default
         return out["data"]
 
-    def get_dataset(self, name, **kwargs):
+    def get_dataset(self, name, default=NO_DEFAULT_PLACEHOLDER, **kwargs):
         """
-        Get named dataset from the scheduler
+        Get named dataset from the scheduler if present.
+        Return the default or raise a KeyError if not present.
+
+        Parameters
+        ----------
+        name : name of the dataset to retrieve
+        default : optional, not set by default
+            If set, do not raise a KeyError if the name is not present but return this default
+        kwargs: dict
+            additional arguments to _get_dataset
 
         See Also
         --------
         Client.publish_dataset
         Client.list_datasets
         """
-        return self.sync(self._get_dataset, name, **kwargs)
+        return self.sync(self._get_dataset, name, default=default, **kwargs)
 
     async def _run_on_scheduler(self, function, *args, wait=True, **kwargs):
         response = await self.scheduler.run_function(
diff --git a/distributed/publish.py b/distributed/publish.py
index 021ae5e7acc..44f34b68a72 100644
--- a/distributed/publish.py
+++ b/distributed/publish.py
@@ -26,9 +26,11 @@ def __init__(self, scheduler):
         self.scheduler.handlers.update(handlers)
         self.scheduler.extensions["publish"] = self
 
-    def put(self, comm=None, keys=None, data=None, name=None, client=None):
+    def put(
+        self, comm=None, keys=None, data=None, name=None, override=False, client=None
+    ):
         with log_errors():
-            if name in self.datasets:
+            if not override and name in self.datasets:
                 raise KeyError("Dataset %s already exists" % name)
             self.scheduler.client_desires_keys(keys, "published-%s" % tokey(name))
             self.datasets[name] = {"data": data, "keys": keys}
diff --git a/distributed/tests/test_publish.py b/distributed/tests/test_publish.py
index a789f5a47f9..82bcc7c996f 100644
--- a/distributed/tests/test_publish.py
+++ b/distributed/tests/test_publish.py
@@ -179,6 +179,7 @@ def test_datasets_setitem(client):
         value = "value"
         client.datasets[key] = value
         assert client.get_dataset(key) == value
+        assert client.get_dataset(key, default="something else") == value
 
 
 def test_datasets_getitem(client):
@@ -186,6 +187,17 @@ def test_datasets_getitem(client):
         value = "value"
         client.publish_dataset(value, name=key)
         assert client.datasets[key] == value
+        assert client.datasets.get(key) == value
+        assert client.datasets.get(key, default="something else") == value
+
+
+def test_datasets_getitem_default(client):
+    with pytest.raises(KeyError) as exc_info:
+        client.get_dataset("key")
+
+    assert client.datasets.get("key", default="value") == "value"
+    assert client.datasets.get("key", default=None) is None
+    assert client.get_dataset("key", default="value") == "value"
 
 
 def test_datasets_delitem(client):
@@ -208,6 +220,18 @@ def test_datasets_contains(client):
     assert key in client.datasets
 
 
+def test_datasets_republish(client):
+    key, value, value2 = "key", "value", "value2"
+    client.publish_dataset(key=value)
+    assert client.get_dataset(key) == value
+
+    with pytest.raises(KeyError) as exc_info:
+        client.publish_dataset(key=value)
+
+    client.publish_dataset(key=value2, override=True)
+    assert client.get_dataset(key) == value2
+
+
 def test_datasets_iter(client):
     keys = [n for n in range(10)]
     client.publish_dataset(**{str(key): key for key in keys})

From 6a32208717c04aa9ccb4defdf253b8a00cab32b7 Mon Sep 17 00:00:00 2001
From: Martin Durant <martindurant@users.noreply.github.com>
Date: Thu, 20 Aug 2020 11:23:32 -0400
Subject: [PATCH 0955/1550] Discover HTTP endpoints (#3744)

* Start writing out HTTP endpoints

* Add some worker stubs

* Add autogenerated list and move known list to docs

(docs page is not linked anywehere for the moment)

* special-case bokeh

(because it exposes many more paths than are expected to be accessed)

* Add tests

* typo

* update and link endpoint doc

* Add statics

* black

* for win

* Try again

* black

* alternate

* fix the fix
---
 distributed/http/routing.py                   | 38 +++++++++
 .../scheduler/tests/test_scheduler_http.py    | 15 ++++
 .../http/worker/tests/test_worker_http.py     | 16 +++-
 docs/source/http_services.rst                 | 79 +++++++++++++++++++
 docs/source/index.rst                         |  1 +
 5 files changed, 148 insertions(+), 1 deletion(-)
 create mode 100644 docs/source/http_services.rst

diff --git a/distributed/http/routing.py b/distributed/http/routing.py
index ac51086493d..ecd14da28e4 100644
--- a/distributed/http/routing.py
+++ b/distributed/http/routing.py
@@ -1,11 +1,49 @@
+import os
 from tornado import web
 import tornado.httputil
+import tornado.routing
+
+
+def _descend_routes(router, routers=set(), out=set()):
+    if router in routers:
+        return
+    routers.add(router)
+    for rule in list(router.named_rules.values()) + router.rules:
+        if isinstance(rule.matcher, tornado.routing.PathMatches):
+            if issubclass(rule.target, tornado.web.StaticFileHandler):
+                prefix = rule.matcher.regex.pattern.rstrip("(.*)$").rstrip("/")
+                path = rule.target_kwargs["path"]
+                for d, dirs, files in os.walk(path):
+                    for fn in files:
+                        fullpath = d + "/" + fn
+                        ourpath = fullpath.replace(path, prefix).replace("\\", "/")
+                        out.add(ourpath)
+            else:
+                out.add(rule.matcher.regex.pattern.rstrip("$"))
+        if isinstance(rule.target, tornado.routing.RuleRouter):
+            _descend_routes(rule.target, routers, out)
+
+
+class DirectoryHandler(web.RequestHandler):
+    """Crawls the HTTP application to find all routes"""
+
+    def get(self):
+        out = set()
+        routers = set()
+        for app in self.application.applications + [self.application]:
+            if "bokeh" in str(app):
+                out.update(set(app.app_paths))
+            else:
+                _descend_routes(app.default_router, routers, out)
+                _descend_routes(app.wildcard_router, routers, out)
+        self.write({"paths": sorted(out)})
 
 
 class RoutingApplication(web.Application):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.applications = []
+        self.add_handlers(".*$", [(r"/sitemap.json", DirectoryHandler),])
 
     def find_handler(self, request: tornado.httputil.HTTPServerRequest, **kwargs):
         handler = super().find_handler(request, **kwargs)
diff --git a/distributed/http/scheduler/tests/test_scheduler_http.py b/distributed/http/scheduler/tests/test_scheduler_http.py
index f1c3a8ed064..62328df4474 100644
--- a/distributed/http/scheduler/tests/test_scheduler_http.py
+++ b/distributed/http/scheduler/tests/test_scheduler_http.py
@@ -31,6 +31,7 @@ async def test_connect(c, s, a, b):
         "json/identity.json",
         "json/index.html",
         "individual-plots.json",
+        "sitemap.json",
     ]:
         response = await http_client.fetch(
             "http://localhost:%d/%s" % (s.http_server.port, suffix)
@@ -162,6 +163,20 @@ async def test_health(c, s, a, b):
     assert txt == "ok"
 
 
+@gen_cluster()
+async def test_sitemap(s, a, b):
+    http_client = AsyncHTTPClient()
+
+    response = await http_client.fetch(
+        "http://localhost:%d/sitemap.json" % s.http_server.port
+    )
+    out = json.loads(response.body.decode())
+    assert "paths" in out
+    assert "/sitemap.json" in out["paths"]
+    assert "/health" in out["paths"]
+    assert "/statics/css/base.css" in out["paths"]
+
+
 @gen_cluster(client=True)
 async def test_task_page(c, s, a, b):
     future = c.submit(lambda x: x + 1, 1, workers=a.address)
diff --git a/distributed/http/worker/tests/test_worker_http.py b/distributed/http/worker/tests/test_worker_http.py
index 2282a4daa66..5bc0a5debc9 100644
--- a/distributed/http/worker/tests/test_worker_http.py
+++ b/distributed/http/worker/tests/test_worker_http.py
@@ -1,5 +1,5 @@
 import pytest
-
+import json
 from tornado.httpclient import AsyncHTTPClient
 from distributed.utils_test import gen_cluster
 
@@ -37,3 +37,17 @@ async def test_health(c, s, a, b):
 
     txt = response.body.decode("utf8")
     assert txt == "ok"
+
+
+@gen_cluster()
+async def test_sitemap(s, a, b):
+    http_client = AsyncHTTPClient()
+
+    response = await http_client.fetch(
+        "http://localhost:%d/sitemap.json" % a.http_server.port
+    )
+    out = json.loads(response.body.decode())
+    assert "paths" in out
+    assert "/sitemap.json" in out["paths"]
+    assert "/health" in out["paths"]
+    assert "/statics/css/base.css" in out["paths"]
diff --git a/docs/source/http_services.rst b/docs/source/http_services.rst
new file mode 100644
index 00000000000..ef5b7527957
--- /dev/null
+++ b/docs/source/http_services.rst
@@ -0,0 +1,79 @@
+HTTP endpoints
+==============
+
+A subset of the following pages will be available from the scheduler or
+workers of a running cluster. The list of currently available endpoins can
+be found by examining ``/sitemap.json``.
+
+
+Main dashboard links
+--------------------
+
+
+Dynamic bokeh pages. The root redirects to /status, and each page links to the
+others via a header navbar.
+
+- ``/status``: entry point to the dashboards, shows cluster-wide memory and tasks
+- ``/workers``: currently connected workers and their CPU/memory usage
+- ``/tasks``: task block view with longer look-back than on /status
+- ``/system``: global stats for the cluster, CPU, memory, bandwidth, file descriptors
+- ``/profile``: flamegraph statistical profiling across the cluster
+- ``/graph``: currently processing graphs in a dependnecy tree view
+- ``/info``: redirect to ``/info/main/workers.html``
+
+Scheduler HTTP
+--------------
+
+Pages and JSON endpoints served by the scheduler
+
+- ``/health``: check server is alive
+- ``/info/main/workers.html`` basic info about workers and links to their dashboards and logs
+- ``info/worker/(worker).html``: more detail about given worker, keyed by TCP address; links to tasks
+- ``info/task/(task).html``: details about a task on the cluster, by dask key; links to worker,
+            related tasks, and client
+- ``/info/call-stacks/(worker).html``: tasks currently handled by given worker
+- ``/info/call-stack/(task).html``: state of task (where it is running)
+- ``/info/main/logs.html``: scheduler logs
+- ``/info/logs/(worker).html``: logs of given worker
+- ``/individual-plots.json``: map of path to description for available individual
+            (i.e,. one-pane, non-dashboard) plots
+- ``/eventstream``: scheduler events on a websocket
+- ``/proxy/(port)/(address)/(path): proxy to worker HTTP locations
+            (if you have jupyter-server-proxy)
+- ``/metrics``: prometheus endpoint
+- ``/json/counts.json``: cluster count stats
+- ``/json/identity.json``: scheduler information
+- ``/json/index.html``: link to the above two endpoints
+- ``/sitemap.json``: list of available endpoints
+- ``/statics/()``: static file content (CSS, etc)
+- ``/stealing``: worker occupancy metrics, to evaluate task stealing
+
+Individual bokeh plots
+----------------------
+
+- ``/individual-task-stream``
+- ``/individual-progress``
+- ``/individual-graph``
+- ``/individual-profile``
+- ``/individual-profile-server``
+- ``/individual-nbytes``
+- ``/individual-cpu``
+- ``/individual-nprocessing``
+- ``/individual-workers``
+- ``/individual-bandwidth-types``
+- ``/individual-bandwidth-workers``
+- ``/individual-memory-by-key``
+- ``/individual-gpu-memory`` (GPU only)
+- ``/individual-gpu-utilization`` (GPU only)
+
+Worker HTTP
+-----------
+
+- ``/status``: 
+- ``/counters``: 
+- ``/crossfilter``:
+- ``/sitemap.json``: list of available endpoints
+- ``/system``: 
+- ``/health``: check server is alive
+- ``/metrics``: prometheus endpoint
+- ``/statics/()``: static file content (CSS, etc)
diff --git a/docs/source/index.rst b/docs/source/index.rst
index 249f9eb9faf..b0def11f56c 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -111,6 +111,7 @@ Contents
    configuration
    ipython
    prometheus
+   http_services
    publish
    resources
    task-launch

From 93a8d28933ebe4b82d715e9f6948be3c38c49953 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sat, 22 Aug 2020 10:52:44 -0700
Subject: [PATCH 0956/1550] bump version to 2.24.0

---
 docs/source/changelog.rst | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index b2dc382fa64..af5f5b2296f 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,16 @@
 Changelog
 =========
 
+2.24.0 - 2020-08-22
+-------------------
+
+-   Move toolbar to above and fix y axis (#4043) `Julia Signell`_
+-   Make behavior clearer for how to get worker dashboard (#4047) `Julia Signell`_
+-   Worker dashboard clean up (#4046) `Julia Signell`_
+-   Add a default argument to the datasets and a possibility to override datasets (#4052) `Nils Braun`_
+-   Discover HTTP endpoints (#3744) `Martin Durant`_
+
+
 2.23.0 - 2020-08-14
 -------------------
 

From 05d21f5bdf2ff620e8f548ec9515c3355c31b19a Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Sat, 22 Aug 2020 21:26:33 +0200
Subject: [PATCH 0957/1550] Improve documentation of scheduler.locks options
 (#4062)

---
 distributed/distributed-schema.yaml | 4 ++--
 distributed/distributed.yaml        | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/distributed/distributed-schema.yaml b/distributed/distributed-schema.yaml
index f67cdca84f2..912a81920f2 100644
--- a/distributed/distributed-schema.yaml
+++ b/distributed/distributed-schema.yaml
@@ -206,11 +206,11 @@ properties:
               lease-validation-interval:
                 type: string
                 description: |
-                  The time to wait until an acquired semaphore is released if the Client goes out of scope
+                  The interval in which the scheduler validates staleness of all acquired leases. Must always be smaller than the lease-timeout itself.
               lease-timeout:
                 type: string
                 description: |
-                  The timeout after which a lease will be released if not refreshed
+                  Maximum interval to wait for a Client refresh before a lease is invalidated and released.
 
           http:
             type: object
diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index f815fadf830..79ed89003c9 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -43,8 +43,8 @@ distributed:
         keep_alive_milliseconds: 500
         check_unused_sessions_milliseconds: 500
     locks:
-      lease-validation-interval: 10s  # The time to wait until an acquired semaphore is released if the Client goes out of scope
-      lease-timeout: 30s  # The timeout after which a lease will be released if not refreshed
+      lease-validation-interval: 10s  # The interval in which the scheduler validates staleness of all acquired leases. Must always be smaller than the lease-timeout itself.
+      lease-timeout: 30s  # Maximum interval to wait for a Client refresh before a lease is invalidated and released.
 
     http:
       routes:

From 7755811fbf43e27d0fedc18161a91bb9f5b6c421 Mon Sep 17 00:00:00 2001
From: Krishan Bhasin <8904718+KrishanBhasin@users.noreply.github.com>
Date: Sat, 22 Aug 2020 20:26:57 +0100
Subject: [PATCH 0958/1550] Cleanup more exception tracebacks (#4054)

---
 distributed/comm/core.py | 8 ++++----
 distributed/core.py      | 4 ++--
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/distributed/comm/core.py b/distributed/comm/core.py
index 56be6c33ad6..b8598c738fb 100644
--- a/distributed/comm/core.py
+++ b/distributed/comm/core.py
@@ -209,10 +209,10 @@ async def on_connection(self, comm: Comm, handshake_overrides=None):
             handshake = await asyncio.wait_for(comm.read(), 1)
             # This would be better, but connections leak if worker is closed quickly
             # write, handshake = await asyncio.gather(comm.write(local_info), comm.read())
-        except Exception:
+        except Exception as e:
             with suppress(Exception):
                 await comm.close()
-            raise CommClosedError()
+            raise CommClosedError() from e
 
         comm.remote_info = handshake
         comm.remote_info["address"] = comm._peer_addr
@@ -289,10 +289,10 @@ async def _():
                         write = await asyncio.wait_for(comm.write(local_info), 1)
                         # This would be better, but connections leak if worker is closed quickly
                         # write, handshake = await asyncio.gather(comm.write(local_info), comm.read())
-                    except Exception:
+                    except Exception as e:
                         with suppress(Exception):
                             await comm.close()
-                        raise CommClosedError()
+                        raise CommClosedError() from e
 
                     comm.remote_info = handshake
                     comm.remote_info["address"] = comm._peer_addr
diff --git a/distributed/core.py b/distributed/core.py
index 148c3b894bb..7d244896dd9 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -482,10 +482,10 @@ async def handle_comm(self, comm, shutting_down=shutting_down):
 
                 try:
                     op = msg.pop("op")
-                except KeyError:
+                except KeyError as e:
                     raise ValueError(
                         "Received unexpected message without 'op' key: " + str(msg)
-                    )
+                    ) from e
                 if self.counters is not None:
                     self.counters["op"].add(op)
                 self._comms[comm] = op

From da699e9e7e8edd736f5ecafe476922981aa72966 Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Mon, 24 Aug 2020 17:43:40 +0100
Subject: [PATCH 0959/1550] Add cluster log method (#4051)

---
 distributed/deploy/cluster.py                 | 38 ++++++++++++++++---
 distributed/deploy/tests/test_spec_cluster.py | 11 ++++--
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
index 511733476c6..05d7880a5bc 100644
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -1,4 +1,5 @@
 import asyncio
+import datetime
 from contextlib import suppress
 import logging
 import threading
@@ -50,12 +51,14 @@ class Cluster:
 
     _supports_scaling = True
 
-    def __init__(self, asynchronous):
+    def __init__(self, asynchronous, quiet=False):
         self.scheduler_info = {"workers": {}}
         self.periodic_callbacks = {}
         self._asynchronous = asynchronous
         self._watch_worker_status_comm = None
         self._watch_worker_status_task = None
+        self._cluster_manager_logs = []
+        self.quiet = quiet
         self.scheduler_comm = None
 
         self.status = Status.created
@@ -170,9 +173,30 @@ def sync(self, func, *args, asynchronous=None, callback_timeout=None, **kwargs):
         else:
             return sync(self.loop, func, *args, **kwargs)
 
-    async def _get_logs(self, scheduler=True, workers=True):
+    def _log(self, log):
+        """Log a message.
+
+        Output a message to the user and also store for future retrieval.
+
+        For use in subclasses where initialisation may take a while and it would
+        be beneficial to feed back to the user.
+
+        Examples
+        --------
+        >>> self._log("Submitted job X to batch scheduler")
+        """
+        self._cluster_manager_logs.append((datetime.datetime.now(), log))
+        if not self.quiet:
+            print(log)
+
+    async def _get_logs(self, cluster=True, scheduler=True, workers=True):
         logs = Logs()
 
+        if cluster:
+            logs["Cluster"] = Log(
+                "\n".join(line[1] for line in self._cluster_manager_logs)
+            )
+
         if scheduler:
             L = await self.scheduler_comm.get_logs()
             logs["Scheduler"] = Log("\n".join(line for level, line in L))
@@ -184,11 +208,13 @@ async def _get_logs(self, scheduler=True, workers=True):
 
         return logs
 
-    def get_logs(self, scheduler=True, workers=True):
-        """ Return logs for the scheduler and workers
+    def get_logs(self, cluster=True, scheduler=True, workers=True):
+        """ Return logs for the cluster, scheduler and workers
 
         Parameters
         ----------
+        cluster : boolean
+            Whether or not to collect logs for the cluster manager
         scheduler : boolean
             Whether or not to collect logs for the scheduler
         workers : boolean or Iterable[str], optional
@@ -201,7 +227,9 @@ def get_logs(self, scheduler=True, workers=True):
             A dictionary of logs, with one item for the scheduler and one for
             each worker
         """
-        return self.sync(self._get_logs, scheduler=scheduler, workers=workers)
+        return self.sync(
+            self._get_logs, cluster=cluster, scheduler=scheduler, workers=workers
+        )
 
     def logs(self, *args, **kwargs):
         warnings.warn("logs is deprecated, use get_logs instead", DeprecationWarning)
diff --git a/distributed/deploy/tests/test_spec_cluster.py b/distributed/deploy/tests/test_spec_cluster.py
index 6c573f5c7c1..5d5c4f7553a 100644
--- a/distributed/deploy/tests/test_spec_cluster.py
+++ b/distributed/deploy/tests/test_spec_cluster.py
@@ -292,17 +292,20 @@ async def test_logs(cleanup):
 
         assert "Registered" in str(logs)
 
-        logs = await cluster.get_logs(scheduler=True, workers=False)
+        logs = await cluster.get_logs(cluster=True, scheduler=False, workers=False)
+        assert list(logs) == ["Cluster"]
+
+        logs = await cluster.get_logs(cluster=False, scheduler=True, workers=False)
         assert list(logs) == ["Scheduler"]
 
-        logs = await cluster.get_logs(scheduler=False, workers=False)
+        logs = await cluster.get_logs(cluster=False, scheduler=False, workers=False)
         assert list(logs) == []
 
-        logs = await cluster.get_logs(scheduler=False, workers=True)
+        logs = await cluster.get_logs(cluster=False, scheduler=False, workers=True)
         assert set(logs) == set(cluster.scheduler.workers)
 
         w = toolz.first(cluster.scheduler.workers)
-        logs = await cluster.get_logs(scheduler=False, workers=[w])
+        logs = await cluster.get_logs(cluster=False, scheduler=False, workers=[w])
         assert set(logs) == {w}
 
 

From e08aa22a4a7e955f247033ed8403a908461896df Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Mon, 24 Aug 2020 18:54:27 +0200
Subject: [PATCH 0960/1550] Ensure semaphore picks correct IOLoop for
 threadpool workers (#4060)

---
 distributed/semaphore.py            | 15 +++++++++------
 distributed/tests/test_semaphore.py | 27 +++++++++++++++++++++++++++
 2 files changed, 36 insertions(+), 6 deletions(-)

diff --git a/distributed/semaphore.py b/distributed/semaphore.py
index eba5908d502..b1c3c029177 100644
--- a/distributed/semaphore.py
+++ b/distributed/semaphore.py
@@ -362,6 +362,12 @@ def __init__(self, max_leases=1, name=None, client=None, register=True):
         self.id = uuid.uuid4().hex
         self._leases = deque()
 
+        self.refresh_leases = True
+
+        self._registered = None
+        if register:
+            self._registered = self.register()
+
         # this should give ample time to refresh without introducing another
         # config parameter since this *must* be smaller than the timeout anyhow
         refresh_leases_interval = (
@@ -371,7 +377,6 @@ def __init__(self, max_leases=1, name=None, client=None, register=True):
             )
             / 5
         )
-        self._refreshing_leases = False
         pc = PeriodicCallback(
             self._refresh_leases, callback_time=refresh_leases_interval * 1000
         )
@@ -379,12 +384,10 @@ def __init__(self, max_leases=1, name=None, client=None, register=True):
         # Registering the pc to the client here is important for proper cleanup
         self._periodic_callback_name = f"refresh_semaphores_{self.id}"
         self.client._periodic_callbacks[self._periodic_callback_name] = pc
-        pc.start()
-        self.refresh_leases = True
 
-        self._registered = None
-        if register:
-            self._registered = self.register()
+        # Need to start the callback using IOLoop.add_callback to ensure that the
+        # PC uses the correct event lopp.
+        self.client.io_loop.add_callback(pc.start)
 
     def register(self):
         """
diff --git a/distributed/tests/test_semaphore.py b/distributed/tests/test_semaphore.py
index 27af5c4fd96..7803d513962 100644
--- a/distributed/tests/test_semaphore.py
+++ b/distributed/tests/test_semaphore.py
@@ -11,6 +11,7 @@
 from distributed.metrics import time
 from distributed.utils_test import (  # noqa: F401
     client,
+    cleanup,
     cluster,
     async_wait_for,
     captured_logger,
@@ -491,3 +492,29 @@ async def test_metrics(c, s, a, b):
         "pending": defaultdict(int, {"test": 0}),
     }
     assert actual == expected
+
+
+def test_threadpoolworkers_pick_correct_ioloop(cleanup):
+    # gh4057
+
+    with dask.config.set(
+        {
+            "distributed.scheduler.locks.lease-validation-interval": 0.01,
+            "distributed.scheduler.locks.lease-timeout": 0.05,
+        }
+    ):
+        with Client(processes=False, threads_per_worker=4) as client:
+            sem = Semaphore(max_leases=1, name="database")
+            protected_ressource = []
+
+            def access_limited(val, sem):
+                import time
+
+                with sem:
+                    assert len(protected_ressource) == 0
+                    protected_ressource.append(val)
+                    # Interact with the DB
+                    time.sleep(0.1)
+                    protected_ressource.remove(val)
+
+            client.gather(client.map(access_limited, range(10), sem=sem))

From 5a7eded0ed314c68b8695b2b34863cdef1b69d60 Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Mon, 24 Aug 2020 22:47:13 +0200
Subject: [PATCH 0961/1550] Graph dashboard: Reset container data if task
 number is too large (#4056)

---
 distributed/dashboard/components/scheduler.py | 37 ++++++++++---------
 .../dashboard/tests/test_scheduler_bokeh.py   |  3 +-
 2 files changed, 22 insertions(+), 18 deletions(-)

diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index ef844a55df7..eae767e2134 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -1467,21 +1467,28 @@ def __init__(self, scheduler, **kwargs):
     @without_property_validation
     def update(self):
         with log_errors():
-            # occasionally reset the column data source to remove old nodes
-            if self.invisible_count > len(self.node_source.data["x"]) / 2:
-                self.layout.reset_index()
-                self.invisible_count = 0
-                update = True
+            # If there are too many tasks in the scheduler we'll disable this
+            # compoonents to not overload scheduler or client. Once we drop
+            # below the threshold, the data is filled up again as usual
+            if len(self.scheduler.tasks) > self.max_items:
+                for container in [self.node_source, self.edge_source]:
+                    container.data = {col: [] for col in container.column_names}
             else:
-                update = False
+                # occasionally reset the column data source to remove old nodes
+                if self.invisible_count > len(self.node_source.data["x"]) / 2:
+                    self.layout.reset_index()
+                    self.invisible_count = 0
+                    update = True
+                else:
+                    update = False
 
-            new, self.layout.new = self.layout.new, []
-            new_edges = self.layout.new_edges
-            self.layout.new_edges = []
+                new, self.layout.new = self.layout.new, []
+                new_edges = self.layout.new_edges
+                self.layout.new_edges = []
 
-            self.add_new_nodes_edges(new, new_edges, update=update)
+                self.add_new_nodes_edges(new, new_edges, update=update)
 
-            self.patch_updates()
+                self.patch_updates()
 
     @without_property_validation
     def add_new_nodes_edges(self, new, new_edges, update=False):
@@ -1498,10 +1505,6 @@ def add_new_nodes_edges(self, new, new_edges, update=False):
             y = self.layout.y
 
             tasks = self.scheduler.tasks
-            if len(tasks) > self.max_items:
-                # graph to big - no update, reset for next time
-                self.invisible_count = len(tasks)
-                return
             for key in new:
                 try:
                     task = tasks[key]
@@ -1557,14 +1560,14 @@ def patch_updates(self):
         if self.layout.visible_updates:
             updates = self.layout.visible_updates
             updates = [(i, c) for i, c in updates if i < n]
-            self.visible_updates = []
+            self.layout.visible_updates = []
             self.node_source.patch({"visible": updates})
             self.invisible_count += len(updates)
 
         if self.layout.visible_edge_updates:
             updates = self.layout.visible_edge_updates
             updates = [(i, c) for i, c in updates if i < m]
-            self.visible_updates = []
+            self.layout.visible_edge_updates = []
             self.edge_source.patch({"visible": updates})
 
     def __del__(self):
diff --git a/distributed/dashboard/tests/test_scheduler_bokeh.py b/distributed/dashboard/tests/test_scheduler_bokeh.py
index 86454328b62..2d4f5a6bc16 100644
--- a/distributed/dashboard/tests/test_scheduler_bokeh.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh.py
@@ -526,8 +526,9 @@ def func(x):
     assert len(gp.node_source.data["x"]) == 2
     f3 = c.submit(func, 3)
     await wait(f3)
+    # Breached task limit, clearing graph
     gp.update()
-    assert len(gp.node_source.data["x"]) == 2
+    assert len(gp.node_source.data["x"]) == 0
 
 
 @gen_cluster(client=True, timeout=30)

From 055bd64fa7bc3dd11ca218689cdbba460fd8579e Mon Sep 17 00:00:00 2001
From: Julia Signell <jsignell@gmail.com>
Date: Mon, 24 Aug 2020 17:43:15 -0400
Subject: [PATCH 0962/1550] Graph helper text (#4064)

* Graph dashboard: Reset container data if task number is too large

* Add subtitle to relay empty or too many to the user

Co-authored-by: fjetter <fjetter@users.noreply.github.com>
Co-authored-by: Benjamin Zaitlen <quasiben@gmail.com>
---
 distributed/dashboard/components/scheduler.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index eae767e2134..34c6f5a8aa4 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -1433,6 +1433,9 @@ def __init__(self, scheduler, **kwargs):
         )
 
         self.root = figure(title="Task Graph", **kwargs)
+        self.subtitle = Title(text=" ", text_font_style="italic")
+        self.root.add_layout(self.subtitle, "above")
+
         self.root.multi_line(
             xs="x",
             ys="y",
@@ -1471,10 +1474,12 @@ def update(self):
             # compoonents to not overload scheduler or client. Once we drop
             # below the threshold, the data is filled up again as usual
             if len(self.scheduler.tasks) > self.max_items:
+                self.subtitle.text = "Scheduler has too many tasks to display."
                 for container in [self.node_source, self.edge_source]:
                     container.data = {col: [] for col in container.column_names}
             else:
                 # occasionally reset the column data source to remove old nodes
+                self.subtitle.text = " "
                 if self.invisible_count > len(self.node_source.data["x"]) / 2:
                     self.layout.reset_index()
                     self.invisible_count = 0
@@ -1490,6 +1495,9 @@ def update(self):
 
                 self.patch_updates()
 
+                if len(self.scheduler.tasks) == 0:
+                    self.subtitle.text = "Scheduler is empty."
+
     @without_property_validation
     def add_new_nodes_edges(self, new, new_edges, update=False):
         if new or update:

From 554370b4c9970137e161ae370d8fef50dd7862b2 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Mon, 24 Aug 2020 18:27:07 -0700
Subject: [PATCH 0963/1550] Use `list.insert` to add prelude up front (#4066)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Avoids creating a temporary `list` by simply modifying the existing
`list` in-place. This can result in significant savings particularly for
large `list`s.

```python
In [1]: %%timeit L = list(range(1_000_000))
   ...: L = [None] + L
   ...:
   ...:
7.1 ms ± 45.7 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)

In [2]: %%timeit L = list(range(1_000_000))
   ...: L.insert(0, None)
   ...:
   ...:
634 µs ± 4.57 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)
```
---
 distributed/protocol/serialize.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index 3b7f3662484..abbbb9a5302 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -520,7 +520,8 @@ def serialize_bytelist(x, **kwargs):
 
     header = msgpack.dumps(header, use_bin_type=True)
     frames2 = [header, *frames]
-    return [pack_frames_prelude(frames2)] + frames2
+    frames2.insert(0, pack_frames_prelude(frames2))
+    return frames2
 
 
 def serialize_bytes(x, **kwargs):

From 7ea4e24287a1e57836081b23f49e65e95eabc4da Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Mon, 24 Aug 2020 18:31:34 -0700
Subject: [PATCH 0964/1550] Simplify `frame_split_size` (#4067)

* Drop 2nd check of `frame` in `frame_split_size`

As we already check `nbytes` and `return`ed if it was `<= n`, we already
know `nbytes(frame) > n`. So there is no need to check it again. Thus we
get rid of this second check.

* Store `nbytes(frame)` in `frame_split_size`

To avoid calling `nbytes(frame)` repeatedly, just store the result as a
variable and access that.

* Assign other computed values in `frame_split_size`

Should also make it a bit clearer what is going on in this code.

* Run `black` on `distributed/protocol/utils.py`

* Always use `memoryview` in `frame_split_size`

As slicing a `memoryview` is free (it just views the underlying data)
and it is able to easily access `itemsize`, just always use
`memoryview`. This keeps `frame_split_size` performing in a reasonable
way. Also simplifies the code a bit.

* Get `memoryview` of `frame` to start

* Skip assigning `itemsize`

This is fast and easy to access from `memoryview` and we only use it as
part of computing the next two variables. So just skip assigning it and
use this `memoryview` property directly.
---
 distributed/protocol/utils.py | 18 ++++++------------
 1 file changed, 6 insertions(+), 12 deletions(-)

diff --git a/distributed/protocol/utils.py b/distributed/protocol/utils.py
index 66c9687bae4..d5694a1b3f4 100644
--- a/distributed/protocol/utils.py
+++ b/distributed/protocol/utils.py
@@ -23,21 +23,15 @@ def frame_split_size(frame, n=BIG_BYTES_SHARD_SIZE) -> list:
     >>> frame_split_size([b'12345', b'678'], n=3)  # doctest: +SKIP
     [b'123', b'45', b'678']
     """
-    if nbytes(frame) <= n:
+    frame = memoryview(frame)
+
+    if frame.nbytes <= n:
         return [frame]
 
-    if nbytes(frame) > n:
-        if isinstance(frame, (bytes, bytearray)):
-            frame = memoryview(frame)
-        try:
-            itemsize = frame.itemsize
-        except AttributeError:
-            itemsize = 1
+    nitems = frame.nbytes // frame.itemsize
+    items_per_shard = n // frame.itemsize
 
-        return [
-            frame[i : i + n // itemsize]
-            for i in range(0, nbytes(frame) // itemsize, n // itemsize)
-        ]
+    return [frame[i : i + items_per_shard] for i in range(0, nitems, items_per_shard)]
 
 
 def merge_frames(header, frames):

From a848ee0c20d4b3df261da562c4c66cb762d2a3ec Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Mon, 24 Aug 2020 18:32:50 -0700
Subject: [PATCH 0965/1550] Simplify `pack_frames` (#4068)

* Unpack `frames` directly into `data`

* Inline `data` into `bytes.join(...)` call
---
 distributed/protocol/utils.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/distributed/protocol/utils.py b/distributed/protocol/utils.py
index d5694a1b3f4..32fdfd7af1b 100644
--- a/distributed/protocol/utils.py
+++ b/distributed/protocol/utils.py
@@ -100,9 +100,7 @@ def pack_frames(frames):
     --------
     unpack_frames
     """
-    data = [pack_frames_prelude(frames)]
-    data.extend(frames)
-    return b"".join(data)
+    return b"".join([pack_frames_prelude(frames), *frames])
 
 
 def unpack_frames(b):

From 750a5144f83e1c4103ccf9b40f2df61ba60830e2 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 25 Aug 2020 14:19:36 -0700
Subject: [PATCH 0966/1550] Provide informative error when connecting an older
 version of Dask (#4076)

This seems to be coming up with the new protocol changes.
In the future we'll know which versions are present and so will be able
to give better messages.  This should stop the bleeding short term
though.
---
 distributed/comm/core.py | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/distributed/comm/core.py b/distributed/comm/core.py
index b8598c738fb..05439504a1f 100644
--- a/distributed/comm/core.py
+++ b/distributed/comm/core.py
@@ -134,9 +134,18 @@ def handshake_info():
 
     @staticmethod
     def handshake_configuration(local, remote):
-        out = {
-            "pickle-protocol": min(local["pickle-protocol"], remote["pickle-protocol"])
-        }
+        try:
+            out = {
+                "pickle-protocol": min(
+                    local["pickle-protocol"], remote["pickle-protocol"]
+                )
+            }
+        except KeyError:
+            raise ValueError(
+                "Your Dask versions may not be in sync. "
+                "Please ensure that you have the same version of dask "
+                "and distributed on your client, scheduler, and worker machines"
+            )
 
         if local["compression"] == remote["compression"]:
             out["compression"] = local["compression"]

From 44bf0b981039ef5e474fbc5ccc6cd5a98b42e5e4 Mon Sep 17 00:00:00 2001
From: Tom Augspurger <TomAugspurger@users.noreply.github.com>
Date: Fri, 28 Aug 2020 09:40:00 -0500
Subject: [PATCH 0967/1550] Update for black (#4081)

Update for black-20.8b1
---
 distributed/actor.py                          |   4 +-
 distributed/batched.py                        |   6 +-
 distributed/cli/tests/test_dask_scheduler.py  |   2 +-
 distributed/client.py                         | 111 +++---
 distributed/comm/addressing.py                |   2 +-
 distributed/comm/tests/test_comms.py          |  10 +-
 distributed/comm/tests/test_ucx.py            |   4 +-
 distributed/comm/utils.py                     |   4 +-
 distributed/core.py                           |  20 +-
 distributed/dashboard/components/__init__.py  |   4 +-
 distributed/dashboard/components/scheduler.py |   2 +-
 distributed/dashboard/components/shared.py    |   8 +-
 .../dashboard/tests/test_scheduler_bokeh.py   |   8 +-
 .../dashboard/tests/test_worker_bokeh.py      |  10 +-
 distributed/dashboard/utils.py                |   2 +-
 distributed/deploy/cluster.py                 |   8 +-
 distributed/deploy/local.py                   |   2 +-
 distributed/deploy/old_ssh.py                 |   2 +-
 distributed/deploy/spec.py                    |  12 +-
 distributed/deploy/ssh.py                     |   8 +-
 distributed/deploy/tests/test_adaptive.py     |   4 +-
 distributed/deploy/tests/test_local.py        |   2 +-
 distributed/diagnostics/eventstream.py        |   2 +-
 distributed/diagnostics/graph_layout.py       |   4 +-
 distributed/diagnostics/plugin.py             |  10 +-
 distributed/diagnostics/progress.py           |   4 +-
 distributed/diagnostics/progress_stream.py    |   2 +-
 distributed/diagnostics/progressbar.py        |  16 +-
 .../diagnostics/tests/test_worker_plugin.py   |  17 +-
 distributed/diagnostics/websocket.py          |   2 +-
 distributed/event.py                          |  22 +-
 distributed/http/health.py                    |   4 +-
 distributed/http/proxy.py                     |   7 +-
 distributed/http/routing.py                   |   2 +-
 distributed/http/statics.py                   |   2 +-
 distributed/http/tests/test_routing.py        |   4 +-
 distributed/http/worker/prometheus.py         |   4 +-
 distributed/lock.py                           |   6 +-
 distributed/nanny.py                          |   6 +-
 distributed/preloading.py                     |   2 +-
 distributed/process.py                        |   2 +-
 distributed/profile.py                        |  16 +-
 distributed/protocol/compression.py           |   2 +-
 distributed/protocol/core.py                  |   4 +-
 distributed/protocol/numba.py                 |   2 +-
 distributed/protocol/numpy.py                 |   2 +-
 distributed/protocol/pickle.py                |   2 +-
 distributed/protocol/serialize.py             |   8 +-
 distributed/protocol/tests/test_cupy.py       |   4 +-
 .../protocol/tests/test_protocol_utils.py     |   5 +-
 distributed/protocol/tests/test_scipy.py      |   2 +-
 distributed/protocol/tests/test_serialize.py  |   8 +-
 distributed/protocol/utils.py                 |   6 +-
 distributed/publish.py                        |   2 +-
 distributed/pubsub.py                         |   6 +-
 distributed/queues.py                         |  10 +-
 distributed/recreate_exceptions.py            |   2 +-
 distributed/scheduler.py                      | 368 +++++++++---------
 distributed/semaphore.py                      |   4 +-
 distributed/sizeof.py                         |   2 +-
 distributed/stealing.py                       |   2 +-
 distributed/tests/test_asyncprocess.py        |   6 +-
 distributed/tests/test_client.py              |   9 +-
 distributed/tests/test_preload.py             |   2 +-
 distributed/tests/test_scheduler.py           |   7 +-
 distributed/tests/test_security.py            |  18 +-
 distributed/tests/test_worker.py              |   7 +-
 distributed/threadpoolexecutor.py             |   4 +-
 distributed/utils.py                          |  42 +-
 distributed/utils_comm.py                     |  12 +-
 distributed/utils_test.py                     |  12 +-
 distributed/variable.py                       |  10 +-
 distributed/versions.py                       |   2 +-
 distributed/worker.py                         |  61 ++-
 distributed/worker_client.py                  |   2 +-
 75 files changed, 473 insertions(+), 531 deletions(-)

diff --git a/distributed/actor.py b/distributed/actor.py
index 69172bf23ec..dc49571d1db 100644
--- a/distributed/actor.py
+++ b/distributed/actor.py
@@ -11,7 +11,7 @@
 
 
 class Actor(WrappedKey):
-    """ Controls an object on a remote worker
+    """Controls an object on a remote worker
 
     An actor allows remote control of a stateful object living on a remote
     worker.  Method calls on this object trigger operations on the remote
@@ -195,7 +195,7 @@ async def func(**msg):
 
 
 class ActorFuture:
-    """ Future to an actor's method call
+    """Future to an actor's method call
 
     Whenever you call a method on an Actor you get an ActorFuture immediately
     while the computation happens in the background.  You can call ``.result``
diff --git a/distributed/batched.py b/distributed/batched.py
index 07eb8e41014..eab57c420ef 100644
--- a/distributed/batched.py
+++ b/distributed/batched.py
@@ -13,7 +13,7 @@
 
 
 class BatchedSend:
-    """ Batch messages in batches on a stream
+    """Batch messages in batches on a stream
 
     This takes an IOStream and an interval (in ms) and ensures that we send no
     more than one message every interval milliseconds.  We send lists of
@@ -109,7 +109,7 @@ def _background_send(self):
         self.stopped.set()
 
     def send(self, msg):
-        """ Schedule a message for sending to the other side
+        """Schedule a message for sending to the other side
 
         This completes quickly and synchronously
         """
@@ -124,7 +124,7 @@ def send(self, msg):
 
     @gen.coroutine
     def close(self, timeout=None):
-        """ Flush existing messages and then close comm
+        """Flush existing messages and then close comm
 
         If set, raises `tornado.util.TimeoutError` after a timeout.
         """
diff --git a/distributed/cli/tests/test_dask_scheduler.py b/distributed/cli/tests/test_dask_scheduler.py
index 543d4300ff3..470bfddf81d 100644
--- a/distributed/cli/tests/test_dask_scheduler.py
+++ b/distributed/cli/tests/test_dask_scheduler.py
@@ -320,7 +320,7 @@ def test_preload_remote_module(loop, tmp_path):
                 str(tmp_path / "scheduler-file.json"),
                 "--preload",
                 "http://localhost:9382/scheduler_info.py",
-            ],
+            ]
         ) as proc:
             with Client(
                 scheduler_file=tmp_path / "scheduler-file.json", loop=loop
diff --git a/distributed/client.py b/distributed/client.py
index d50f3dd1c0f..27cfced05d8 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -129,7 +129,7 @@ def _del_global_client(c):
 
 
 class Future(WrappedKey):
-    """ A remotely running computation
+    """A remotely running computation
 
     A Future is a local proxy to a result running on a remote worker.  A user
     manages future objects in the local Python process to determine what
@@ -210,7 +210,7 @@ def done(self):
         return self._state.done()
 
     def result(self, timeout=None):
-        """ Wait until computation completes, gather result to local process.
+        """Wait until computation completes, gather result to local process.
 
         If *timeout* seconds are elapsed before returning, a
         ``dask.distributed.TimeoutError`` is raised.
@@ -255,7 +255,7 @@ async def _exception(self):
             return None
 
     def exception(self, timeout=None, **kwargs):
-        """ Return the exception of a failed task
+        """Return the exception of a failed task
 
         If *timeout* seconds are elapsed before returning, a
         ``dask.distributed.TimeoutError`` is raised.
@@ -267,7 +267,7 @@ def exception(self, timeout=None, **kwargs):
         return self.client.sync(self._exception, callback_timeout=timeout, **kwargs)
 
     def add_done_callback(self, fn):
-        """ Call callback on future when callback has finished
+        """Call callback on future when callback has finished
 
         The callback ``fn`` should take the future as its only argument.  This
         will be called regardless of if the future completes successfully,
@@ -296,7 +296,7 @@ def execute_callback(fut):
         )
 
     def cancel(self, **kwargs):
-        """ Cancel request to run this future
+        """Cancel request to run this future
 
         See Also
         --------
@@ -305,7 +305,7 @@ def cancel(self, **kwargs):
         return self.client.cancel([self], **kwargs)
 
     def retry(self, **kwargs):
-        """ Retry this future if it has failed
+        """Retry this future if it has failed
 
         See Also
         --------
@@ -325,7 +325,7 @@ async def _traceback(self):
             return None
 
     def traceback(self, timeout=None, **kwargs):
-        """ Return the traceback of a failed task
+        """Return the traceback of a failed task
 
         This returns a traceback object.  You can inspect this object using the
         ``traceback`` module.  Alternatively if you call ``future.result()``
@@ -495,12 +495,11 @@ def normalize_future(f):
 
 
 class AllExit(Exception):
-    """Custom exception class to exit All(...) early.
-    """
+    """Custom exception class to exit All(...) early."""
 
 
 class Client:
-    """ Connect to and submit computation to a Dask cluster
+    """Connect to and submit computation to a Dask cluster
 
     The Client connects users to a Dask cluster.  It provides an asynchronous
     user interface around functions and futures.  This class resembles
@@ -702,7 +701,7 @@ def __init__(
 
         self._periodic_callbacks = dict()
         self._periodic_callbacks["scheduler-info"] = PeriodicCallback(
-            self._update_scheduler_info, scheduler_info_interval * 1000,
+            self._update_scheduler_info, scheduler_info_interval * 1000
         )
         self._periodic_callbacks["heartbeat"] = PeriodicCallback(
             self._heartbeat, heartbeat_interval * 1000
@@ -788,7 +787,7 @@ def current(cls, allow_global=True):
 
     @property
     def asynchronous(self):
-        """ Are we running in the event loop?
+        """Are we running in the event loop?
 
         This is true if the user signaled that we might be when creating the
         client as in the following::
@@ -1386,7 +1385,7 @@ async def _close(self, fast=False):
     _shutdown = _close
 
     def close(self, timeout=no_default):
-        """ Close this client
+        """Close this client
 
         Clients will also close automatically when your Python session ends
 
@@ -1445,7 +1444,7 @@ async def _shutdown(self):
                 await self.scheduler.terminate(close_workers=True)
 
     def shutdown(self):
-        """ Shut down the connected scheduler and workers
+        """Shut down the connected scheduler and workers
 
         Note, this may disrupt other clients that may be using the same
         scheduler and workers.
@@ -1490,7 +1489,7 @@ def submit(
         **kwargs,
     ):
 
-        """ Submit a function application to the scheduler
+        """Submit a function application to the scheduler
 
         Parameters
         ----------
@@ -1610,7 +1609,7 @@ def map(
         batch_size=None,
         **kwargs,
     ):
-        """ Map a function on a sequence of arguments
+        """Map a function on a sequence of arguments
 
         Arguments can be normal objects or Futures
 
@@ -1904,7 +1903,7 @@ async def wait(k):
         return result
 
     async def _gather_remote(self, direct, local_worker):
-        """ Perform gather with workers or scheduler
+        """Perform gather with workers or scheduler
 
         This method exists to limit and batch many concurrent gathers into a
         few.  In controls access using a Tornado semaphore, and picks up keys
@@ -1933,7 +1932,7 @@ async def _gather_remote(self, direct, local_worker):
         return response
 
     def gather(self, futures, errors="raise", direct=None, asynchronous=None):
-        """ Gather futures from distributed memory
+        """Gather futures from distributed memory
 
         Accepts a future, nested container of futures, iterator, or queue.
         The return type will match the input type.
@@ -2111,7 +2110,7 @@ def scatter(
         timeout=no_default,
         asynchronous=None,
     ):
-        """ Scatter data into distributed memory
+        """Scatter data into distributed memory
 
         This moves data from the local client process into the workers of the
         distributed scheduler.  Note that it is often better to submit jobs to
@@ -2395,7 +2394,7 @@ async def _run_on_scheduler(self, function, *args, wait=True, **kwargs):
             return response["result"]
 
     def run_on_scheduler(self, function, *args, **kwargs):
-        """ Run a function on the scheduler process
+        """Run a function on the scheduler process
 
         This is typically used for live debugging.  The function should take a
         keyword argument ``dask_scheduler=``, which will be given the scheduler
@@ -2658,7 +2657,7 @@ def get(
         actors=None,
         **kwargs,
     ):
-        """ Compute dask graph
+        """Compute dask graph
 
         Parameters
         ----------
@@ -2720,7 +2719,7 @@ def get(
         return packed
 
     def _optimize_insert_futures(self, dsk, keys):
-        """ Replace known keys in dask graph with Futures
+        """Replace known keys in dask graph with Futures
 
         When given a Dask graph that might have overlapping keys with our known
         results we replace the values of that graph with futures.  This can be
@@ -2789,7 +2788,7 @@ def compute(
         traverse=True,
         **kwargs,
     ):
-        """ Compute dask collections on cluster
+        """Compute dask collections on cluster
 
         Parameters
         ----------
@@ -2937,7 +2936,7 @@ def persist(
         actors=None,
         **kwargs,
     ):
-        """ Persist dask collections on cluster
+        """Persist dask collections on cluster
 
         Starts computation of the collection on the cluster in the background.
         Provides a new dask collection that is semantically identical to the
@@ -3052,7 +3051,7 @@ async def _restart(self, timeout=no_default):
         return self
 
     def restart(self, **kwargs):
-        """ Restart the distributed network
+        """Restart the distributed network
 
         This kills all active work, deletes all data on the network, and
         restarts the worker processes.
@@ -3102,7 +3101,7 @@ def dump_to_file(dask_worker=None):
         assert all(len(data) == v for v in response.values())
 
     def upload_file(self, filename, **kwargs):
-        """ Upload local package to workers
+        """Upload local package to workers
 
         This sends a local file up to all worker nodes.  This file is placed
         into a temporary directory on Python's system path so any .py,  .egg
@@ -3138,7 +3137,7 @@ async def _rebalance(self, futures=None, workers=None):
         assert result["status"] == "OK"
 
     def rebalance(self, futures=None, workers=None, **kwargs):
-        """ Rebalance data within network
+        """Rebalance data within network
 
         Move data between workers to roughly balance memory burden.  This
         either affects a subset of the keys/workers or the entire network,
@@ -3166,7 +3165,7 @@ async def _replicate(self, futures, n=None, workers=None, branching_factor=2):
         )
 
     def replicate(self, futures, n=None, workers=None, branching_factor=2, **kwargs):
-        """ Set replication of futures within network
+        """Set replication of futures within network
 
         Copy data onto many workers.  This helps to broadcast frequently
         accessed data and it helps to improve resilience.
@@ -3211,7 +3210,7 @@ def replicate(self, futures, n=None, workers=None, branching_factor=2, **kwargs)
         )
 
     def nthreads(self, workers=None, **kwargs):
-        """ The number of threads/cores available on each worker node
+        """The number of threads/cores available on each worker node
 
         Parameters
         ----------
@@ -3243,7 +3242,7 @@ def nthreads(self, workers=None, **kwargs):
     ncores = nthreads
 
     def who_has(self, futures=None, **kwargs):
-        """ The workers storing each future's data
+        """The workers storing each future's data
 
         Parameters
         ----------
@@ -3276,7 +3275,7 @@ def who_has(self, futures=None, **kwargs):
         return self.sync(self.scheduler.who_has, keys=keys, **kwargs)
 
     def has_what(self, workers=None, **kwargs):
-        """ Which keys are held by which workers
+        """Which keys are held by which workers
 
         This returns the keys of the data that are held in each worker's
         memory.
@@ -3310,7 +3309,7 @@ def has_what(self, workers=None, **kwargs):
         return self.sync(self.scheduler.has_what, workers=workers, **kwargs)
 
     def processing(self, workers=None):
-        """ The tasks currently running on each worker
+        """The tasks currently running on each worker
 
         Parameters
         ----------
@@ -3340,7 +3339,7 @@ def processing(self, workers=None):
         return self.sync(self.scheduler.processing, workers=workers)
 
     def nbytes(self, keys=None, summary=True, **kwargs):
-        """ The bytes taken up by each key on the cluster
+        """The bytes taken up by each key on the cluster
 
         This is as measured by ``sys.getsizeof`` which may not accurately
         reflect the true cost.
@@ -3370,7 +3369,7 @@ def nbytes(self, keys=None, summary=True, **kwargs):
         return self.sync(self.scheduler.nbytes, keys=keys, summary=summary, **kwargs)
 
     def call_stack(self, futures=None, keys=None):
-        """ The actively running call stack of all relevant keys
+        """The actively running call stack of all relevant keys
 
         You can specify data of interest either by providing futures or
         collections in the ``futures=`` keyword or a list of explicit keys in
@@ -3409,7 +3408,7 @@ def profile(
         server=False,
         scheduler=False,
     ):
-        """ Collect statistical profiling information about recent work
+        """Collect statistical profiling information about recent work
 
         Parameters
         ----------
@@ -3499,7 +3498,7 @@ async def _profile(
             return state
 
     def scheduler_info(self, **kwargs):
-        """ Basic information about the workers in the cluster
+        """Basic information about the workers in the cluster
 
         Examples
         --------
@@ -3519,7 +3518,7 @@ def scheduler_info(self, **kwargs):
         return self._scheduler_identity
 
     def write_scheduler_file(self, scheduler_file):
-        """ Write the scheduler information to a json file.
+        """Write the scheduler information to a json file.
 
         This facilitates easy sharing of scheduler information using a file
         system. The scheduler file can be used to instantiate a second Client
@@ -3546,7 +3545,7 @@ def write_scheduler_file(self, scheduler_file):
             json.dump(self.scheduler_info(), f, indent=2)
 
     def get_metadata(self, keys, default=no_default):
-        """ Get arbitrary metadata from scheduler
+        """Get arbitrary metadata from scheduler
 
         See set_metadata for the full docstring with examples
 
@@ -3568,7 +3567,7 @@ def get_metadata(self, keys, default=no_default):
         return self.sync(self.scheduler.get_metadata, keys=keys, default=default)
 
     def get_scheduler_logs(self, n=None):
-        """ Get logs from scheduler
+        """Get logs from scheduler
 
         Parameters
         ----------
@@ -3583,7 +3582,7 @@ def get_scheduler_logs(self, n=None):
         return self.sync(self.scheduler.logs, n=n)
 
     def get_worker_logs(self, n=None, workers=None, nanny=False):
-        """ Get logs from workers
+        """Get logs from workers
 
         Parameters
         ----------
@@ -3605,7 +3604,7 @@ def get_worker_logs(self, n=None, workers=None, nanny=False):
         return self.sync(self.scheduler.worker_logs, n=n, workers=workers, nanny=nanny)
 
     def retire_workers(self, workers=None, close_workers=True, **kwargs):
-        """ Retire certain workers on the scheduler
+        """Retire certain workers on the scheduler
 
         See dask.distributed.Scheduler.retire_workers for the full docstring.
 
@@ -3631,7 +3630,7 @@ def retire_workers(self, workers=None, close_workers=True, **kwargs):
         )
 
     def set_metadata(self, key, value):
-        """ Set arbitrary metadata in the scheduler
+        """Set arbitrary metadata in the scheduler
 
         This allows you to store small amounts of data on the central scheduler
         process for administrative purposes.  Data should be msgpack
@@ -3678,7 +3677,7 @@ def set_metadata(self, key, value):
         return self.sync(self.scheduler.set_metadata, keys=key, value=value)
 
     def get_versions(self, check=False, packages=[]):
-        """ Return version info for the scheduler, all workers and myself
+        """Return version info for the scheduler, all workers and myself
 
         Parameters
         ----------
@@ -3737,7 +3736,7 @@ async def _start_ipython_workers(self, workers):
     def start_ipython_workers(
         self, workers=None, magic_names=False, qtconsole=False, qtconsole_args=None
     ):
-        """ Start IPython kernels on workers
+        """Start IPython kernels on workers
 
         Parameters
         ----------
@@ -3819,7 +3818,7 @@ def start_ipython_workers(
     def start_ipython_scheduler(
         self, magic_name="scheduler_if_ipython", qtconsole=False, qtconsole_args=None
     ):
-        """ Start IPython kernel on the scheduler
+        """Start IPython kernel on the scheduler
 
         Parameters
         ----------
@@ -3987,7 +3986,7 @@ def get_task_stream(
         filename="task-stream.html",
         bokeh_resources=None,
     ):
-        """ Get task stream data from scheduler
+        """Get task stream data from scheduler
 
         This collects the data present in the diagnostic "Task Stream" plot on
         the dashboard.  It includes the start, stop, transfer, and
@@ -4247,7 +4246,7 @@ async def _wait(fs, timeout=None, return_when=ALL_COMPLETED):
 
 
 def wait(fs, timeout=None, return_when=ALL_COMPLETED):
-    """ Wait until all/any futures are finished
+    """Wait until all/any futures are finished
 
     Parameters
     ----------
@@ -4283,7 +4282,7 @@ async def _as_completed(fs, queue):
 
 
 async def _first_completed(futures):
-    """ Return a single completed future
+    """Return a single completed future
 
     See Also:
         _as_completed
@@ -4399,7 +4398,7 @@ async def _track_future(self, future):
                     self.thread_condition.notify()
 
     def update(self, futures):
-        """ Add multiple futures to the collection.
+        """Add multiple futures to the collection.
 
         The added futures will emit from the iterator once they finish"""
         with self.lock:
@@ -4410,7 +4409,7 @@ def update(self, futures):
                 self.loop.add_callback(self._track_future, f)
 
     def add(self, future):
-        """ Add a future to the collection
+        """Add a future to the collection
 
         This future will emit from the iterator once it finishes
         """
@@ -4425,7 +4424,7 @@ def has_ready(self):
         return not self.queue.empty()
 
     def count(self):
-        """ Return the number of futures yet to be returned
+        """Return the number of futures yet to be returned
 
         This includes both the number of futures still computing, as well as
         those that are finished, but have not yet been returned from this
@@ -4476,7 +4475,7 @@ async def __anext__(self):
     next = __next__
 
     def next_batch(self, block=True):
-        """ Get the next batch of completed futures.
+        """Get the next batch of completed futures.
 
         Parameters
         ----------
@@ -4573,7 +4572,7 @@ def redict_collection(c, dsk):
 
 
 def futures_of(o, client=None):
-    """ Future objects in a collection
+    """Future objects in a collection
 
     Parameters
     ----------
@@ -4618,7 +4617,7 @@ def futures_of(o, client=None):
 
 
 def fire_and_forget(obj):
-    """ Run tasks at least once, even if we release the futures
+    """Run tasks at least once, even if we release the futures
 
     Under normal operation Dask will not run any tasks for which there is not
     an active future (this avoids unnecessary work in many situations).
@@ -4733,7 +4732,7 @@ async def __aexit__(self, typ, value, traceback):
 
 
 class performance_report:
-    """ Gather performance report
+    """Gather performance report
 
     This creates a static HTML file that includes many of the same plots of the
     dashboard for later viewing.
@@ -4785,7 +4784,7 @@ def __exit__(self, typ, value, traceback):
 
 @contextmanager
 def temp_default_client(c):
-    """ Set the default client for the duration of the context
+    """Set the default client for the duration of the context
 
     .. note::
        This function should be used exclusively for unit testing the default client
diff --git a/distributed/comm/addressing.py b/distributed/comm/addressing.py
index 2b1c4717407..537ecd4ef23 100644
--- a/distributed/comm/addressing.py
+++ b/distributed/comm/addressing.py
@@ -216,7 +216,7 @@ def addresses_from_user_args(
     security=None,
     default_port=0,
 ) -> list:
-    """ Get a list of addresses if the inputs are lists
+    """Get a list of addresses if the inputs are lists
 
     This is like ``address_from_user_args`` except that it also accepts lists
     for some of the arguments.  If these arguments are lists then it will map
diff --git a/distributed/comm/tests/test_comms.py b/distributed/comm/tests/test_comms.py
index 15559250028..0a8f5250485 100644
--- a/distributed/comm/tests/test_comms.py
+++ b/distributed/comm/tests/test_comms.py
@@ -326,7 +326,7 @@ async def sleep_for_60ms():
     sleep_future = sleep_for_60ms()
     with pytest.raises(IOError):
         await connect(
-            "tls://localhost:28400", 0.052, ssl_context=get_client_ssl_context(),
+            "tls://localhost:28400", 0.052, ssl_context=get_client_ssl_context()
         )
     max_thread_count = await sleep_future
     assert max_thread_count <= 2 + original_thread_count
@@ -639,7 +639,7 @@ async def handle_comm(comm):
 
     with pytest.raises(EnvironmentError) as excinfo:
         comm = await connect(
-            listener.contact_address, timeout=0.5, ssl_context=bad_cli_ctx,
+            listener.contact_address, timeout=0.5, ssl_context=bad_cli_ctx
         )
         await comm.write({"x": "foo"})  # TODO: why is this necessary in Tornado 6 ?
 
@@ -657,16 +657,14 @@ async def handle_comm(comm):
                 raise
 
     # Sanity check
-    comm = await connect(listener.contact_address, timeout=2, ssl_context=cli_ctx,)
+    comm = await connect(listener.contact_address, timeout=2, ssl_context=cli_ctx)
     await comm.close()
 
     # Connector refuses a listener not signed by the CA
     listener = await listen("tls://", handle_comm, ssl_context=bad_serv_ctx)
 
     with pytest.raises(EnvironmentError) as excinfo:
-        await connect(
-            listener.contact_address, timeout=2, ssl_context=cli_ctx,
-        )
+        await connect(listener.contact_address, timeout=2, ssl_context=cli_ctx)
     assert "certificate verify failed" in str(excinfo.value)
 
 
diff --git a/distributed/comm/tests/test_ucx.py b/distributed/comm/tests/test_ucx.py
index a7dbfd3e46a..aa5095e2f3c 100644
--- a/distributed/comm/tests/test_ucx.py
+++ b/distributed/comm/tests/test_ucx.py
@@ -228,9 +228,7 @@ async def test_ping_pong_cupy(shape):
 
 @pytest.mark.slow
 @pytest.mark.asyncio
-@pytest.mark.parametrize(
-    "n", [int(1e9), int(2.5e9),],
-)
+@pytest.mark.parametrize("n", [int(1e9), int(2.5e9)])
 async def test_large_cupy(n, cleanup):
     cupy = pytest.importorskip("cupy")
     com, serv_com = await get_comm_pair()
diff --git a/distributed/comm/utils.py b/distributed/comm/utils.py
index eda370eed6f..d1a1a97e63c 100644
--- a/distributed/comm/utils.py
+++ b/distributed/comm/utils.py
@@ -22,7 +22,7 @@
 
 
 async def to_frames(
-    msg, serializers=None, on_error="message", context=None, allow_offload=True,
+    msg, serializers=None, on_error="message", context=None, allow_offload=True
 ):
     """
     Serialize a message into a list of Distributed protocol frames.
@@ -32,7 +32,7 @@ def _to_frames():
         try:
             return list(
                 protocol.dumps(
-                    msg, serializers=serializers, on_error=on_error, context=context,
+                    msg, serializers=serializers, on_error=on_error, context=context
                 )
             )
         except Exception as e:
diff --git a/distributed/core.py b/distributed/core.py
index 7d244896dd9..5ede25d7a05 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -110,7 +110,7 @@ def _raise(*args, **kwargs):
 
 
 class Server:
-    """ Dask Distributed Server
+    """Dask Distributed Server
 
     Superclass for endpoints in a distributed cluster, such as Worker
     and Scheduler objects.
@@ -319,7 +319,7 @@ async def __aexit__(self, typ, value, traceback):
         await self.close()
 
     def start_periodic_callbacks(self):
-        """ Start Periodic Callbacks consistently
+        """Start Periodic Callbacks consistently
 
         This starts all PeriodicCallbacks stored in self.periodic_callbacks if
         they are not yet running.  It does this safely on the IOLoop.
@@ -437,7 +437,7 @@ async def listen(self, port_or_addr=None, allow_offload=True, **kwargs):
         self.listeners.append(listener)
 
     async def handle_comm(self, comm, shutting_down=shutting_down):
-        """ Dispatch new communications to coroutine-handlers
+        """Dispatch new communications to coroutine-handlers
 
         Handlers is a dictionary mapping operation names to functions or
         coroutines.
@@ -645,7 +645,7 @@ def pingpong(comm):
 
 
 async def send_recv(comm, reply=True, serializers=None, deserializers=None, **kwargs):
-    """ Send and recv with a Comm.
+    """Send and recv with a Comm.
 
     Keyword arguments turn into the message
 
@@ -696,7 +696,7 @@ def addr_from_args(addr=None, ip=None, port=None):
 
 
 class rpc:
-    """ Conveniently interact with a remote server
+    """Conveniently interact with a remote server
 
     >>> remote = rpc(address)  # doctest: +SKIP
     >>> response = yield remote.add(x=10, y=20)  # doctest: +SKIP
@@ -736,7 +736,7 @@ def __init__(
         rpc.active.add(self)
 
     async def live_comm(self):
-        """ Get an open communication
+        """Get an open communication
 
         Some comms to the ip/port target may be in current use by other
         coroutines.  We track this with the `comms` dict
@@ -855,7 +855,7 @@ def __repr__(self):
 
 
 class PooledRPCCall:
-    """ The result of ConnectionPool()('host:port')
+    """The result of ConnectionPool()('host:port')
 
     See Also:
         ConnectionPool
@@ -904,7 +904,7 @@ def __repr__(self):
 
 
 class ConnectionPool:
-    """ A maximum sized pool of Comm objects.
+    """A maximum sized pool of Comm objects.
 
     This provides a connect method that mirrors the normal distributed.connect
     method, but provides connection sharing and tracks connection limits.
@@ -1129,7 +1129,7 @@ def collect_causes(e):
 
 
 def error_message(e, status="error"):
-    """ Produce message to send back given an exception has occurred
+    """Produce message to send back given an exception has occurred
 
     This does the following:
 
@@ -1167,7 +1167,7 @@ def error_message(e, status="error"):
 
 
 def clean_exception(exception, traceback, **kwargs):
-    """ Reraise exception and traceback. Deserialize if necessary
+    """Reraise exception and traceback. Deserialize if necessary
 
     See Also
     --------
diff --git a/distributed/dashboard/components/__init__.py b/distributed/dashboard/components/__init__.py
index f6159e83bcf..78d60108c8e 100644
--- a/distributed/dashboard/components/__init__.py
+++ b/distributed/dashboard/components/__init__.py
@@ -42,7 +42,7 @@
 
 
 class DashboardComponent:
-    """ Base class for Dask.distributed UI dashboard components.
+    """Base class for Dask.distributed UI dashboard components.
 
     This class must have two attributes, ``root`` and ``source``, and one
     method ``update``:
@@ -62,7 +62,7 @@ def update(self, messages):
 
 
 def add_periodic_callback(doc, component, interval):
-    """ Add periodic callback to doc in a way that avoids reference cycles
+    """Add periodic callback to doc in a way that avoids reference cycles
 
     If we instead use ``doc.add_periodic_callback(component.update, 100)`` then
     the component stays in memory as a reference cycle because its method is
diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index 34c6f5a8aa4..8c43d243673 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -1759,7 +1759,7 @@ def update(self):
 
 
 class WorkerTable(DashboardComponent):
-    """ Status of the current workers
+    """Status of the current workers
 
     This is two plots, a text-based table for each host and a thin horizontal
     plot laying out hosts by their current memory use.
diff --git a/distributed/dashboard/components/shared.py b/distributed/dashboard/components/shared.py
index 40865e10fee..9360a6b77ac 100644
--- a/distributed/dashboard/components/shared.py
+++ b/distributed/dashboard/components/shared.py
@@ -38,7 +38,7 @@
 
 
 class Processing(DashboardComponent):
-    """ Processing and distribution per core
+    """Processing and distribution per core
 
     This shows how many tasks are actively running on each worker and how many
     tasks are enqueued for each worker and how many are in the common pool
@@ -128,7 +128,7 @@ def processing_update(msg):
 
 
 class ProfilePlot(DashboardComponent):
-    """ Time plots of the current resource usage on the cluster
+    """Time plots of the current resource usage on the cluster
 
     This is two plots, one for CPU and Memory and another for Network I/O
     """
@@ -171,7 +171,7 @@ def update(self, state):
 
 
 class ProfileTimePlot(DashboardComponent):
-    """ Time plots of the current resource usage on the cluster
+    """Time plots of the current resource usage on the cluster
 
     This is two plots, one for CPU and Memory and another for Network I/O
     """
@@ -336,7 +336,7 @@ async def cb():
 
 
 class ProfileServer(DashboardComponent):
-    """ Time plots of the current resource usage on the cluster
+    """Time plots of the current resource usage on the cluster
 
     This is two plots, one for CPU and Memory and another for Network I/O
     """
diff --git a/distributed/dashboard/tests/test_scheduler_bokeh.py b/distributed/dashboard/tests/test_scheduler_bokeh.py
index 2d4f5a6bc16..e79f3494279 100644
--- a/distributed/dashboard/tests/test_scheduler_bokeh.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh.py
@@ -507,9 +507,7 @@ async def test_TaskGraph_clear(c, s, a, b):
         assert time() < start + 5
 
 
-@gen_cluster(
-    client=True, config={"distributed.dashboard.graph-max-items": 2,},
-)
+@gen_cluster(client=True, config={"distributed.dashboard.graph-max-items": 2})
 async def test_TaskGraph_limit(c, s, a, b):
     gp = TaskGraph(s)
 
@@ -591,9 +589,7 @@ async def test_profile_server(c, s, a, b):
         assert time() < start + 2
 
 
-@gen_cluster(
-    client=True, scheduler_kwargs={"dashboard": True},
-)
+@gen_cluster(client=True, scheduler_kwargs={"dashboard": True})
 async def test_root_redirect(c, s, a, b):
     http_client = AsyncHTTPClient()
     response = await http_client.fetch("http://localhost:%d/" % s.http_server.port)
diff --git a/distributed/dashboard/tests/test_worker_bokeh.py b/distributed/dashboard/tests/test_worker_bokeh.py
index 47ac89c6b0a..6143e837529 100644
--- a/distributed/dashboard/tests/test_worker_bokeh.py
+++ b/distributed/dashboard/tests/test_worker_bokeh.py
@@ -24,9 +24,7 @@
 
 
 @gen_cluster(
-    client=True,
-    worker_kwargs={"dashboard": True},
-    scheduler_kwargs={"dashboard": True},
+    client=True, worker_kwargs={"dashboard": True}, scheduler_kwargs={"dashboard": True}
 )
 async def test_routes(c, s, a, b):
     port = a.http_server.port
@@ -64,9 +62,7 @@ async def test_simple(c, s, a, b):
         assert "bokeh" in response.body.decode().lower()
 
 
-@gen_cluster(
-    client=True, worker_kwargs={"dashboard": True},
-)
+@gen_cluster(client=True, worker_kwargs={"dashboard": True})
 async def test_services_kwargs(c, s, a, b):
     assert s.workers[a.address].services == {"dashboard": a.http_server.port}
 
@@ -157,7 +153,7 @@ async def test_CommunicatingStream(c, s, a, b):
 
 
 @gen_cluster(
-    client=True, clean_kwargs={"threads": False}, worker_kwargs={"dashboard": True},
+    client=True, clean_kwargs={"threads": False}, worker_kwargs={"dashboard": True}
 )
 async def test_prometheus(c, s, a, b):
     pytest.importorskip("prometheus_client")
diff --git a/distributed/dashboard/utils.py b/distributed/dashboard/utils.py
index 0de536a6050..1c54e8b478e 100644
--- a/distributed/dashboard/utils.py
+++ b/distributed/dashboard/utils.py
@@ -44,7 +44,7 @@ def transpose(lod):
 
 @without_property_validation
 def update(source, data):
-    """ Update source with data
+    """Update source with data
 
     This checks a few things first
 
diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
index 05d7880a5bc..5b9c598be9c 100644
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -27,7 +27,7 @@
 
 
 class Cluster:
-    """ Superclass for cluster objects
+    """Superclass for cluster objects
 
     This class contains common functionality for Dask Cluster manager classes.
 
@@ -124,7 +124,7 @@ def _update_worker_status(self, op, msg):
             raise ValueError("Invalid op", op, msg)
 
     def adapt(self, Adaptive=Adaptive, **kwargs) -> Adaptive:
-        """ Turn on adaptivity
+        """Turn on adaptivity
 
         For keyword arguments see dask.distributed.Adaptive
 
@@ -141,7 +141,7 @@ def adapt(self, Adaptive=Adaptive, **kwargs) -> Adaptive:
         return self._adaptive
 
     def scale(self, n: int) -> None:
-        """ Scale cluster to n workers
+        """Scale cluster to n workers
 
         Parameters
         ----------
@@ -209,7 +209,7 @@ async def _get_logs(self, cluster=True, scheduler=True, workers=True):
         return logs
 
     def get_logs(self, cluster=True, scheduler=True, workers=True):
-        """ Return logs for the cluster, scheduler and workers
+        """Return logs for the cluster, scheduler and workers
 
         Parameters
         ----------
diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index 54b8ae125e6..66cfe70ee6d 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -18,7 +18,7 @@
 
 
 class LocalCluster(SpecCluster):
-    """ Create local Scheduler and Workers
+    """Create local Scheduler and Workers
 
     This creates a "cluster" of a scheduler and workers running on the local
     machine.
diff --git a/distributed/deploy/old_ssh.py b/distributed/deploy/old_ssh.py
index fe5703ea5e8..6d62e0cfd8b 100644
--- a/distributed/deploy/old_ssh.py
+++ b/distributed/deploy/old_ssh.py
@@ -209,7 +209,7 @@ def communicate():
 
 
 def start_scheduler(
-    logdir, addr, port, ssh_username, ssh_port, ssh_private_key, remote_python=None,
+    logdir, addr, port, ssh_username, ssh_port, ssh_private_key, remote_python=None
 ):
     cmd = "{python} -m distributed.cli.dask_scheduler --port {port}".format(
         python=remote_python or sys.executable, port=port
diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index 1d1c449f7f7..0ba24c43632 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -75,7 +75,7 @@ async def _():
         return _().__await__()
 
     async def start(self):
-        """ Submit the process to the resource manager
+        """Submit the process to the resource manager
 
         For workers this doesn't have to wait until the process actually starts,
         but can return once the resource manager has the request, and will work
@@ -87,7 +87,7 @@ async def start(self):
         self.status = Status.running
 
     async def close(self):
-        """ Close the process
+        """Close the process
 
         This will be called by the Cluster object when we scale down a node,
         but only after we ask the Scheduler to close the worker gracefully.
@@ -126,7 +126,7 @@ async def f():
 
 
 class SpecCluster(Cluster):
-    """ Cluster that requires a full specification of workers
+    """Cluster that requires a full specification of workers
 
     The SpecCluster class expects a full specification of the Scheduler and
     Workers to use.  It removes any handling of user inputs (like threads vs
@@ -483,7 +483,7 @@ def scale(self, n=0, memory=None, cores=None):
             return NoOpAwaitable()
 
     def _new_worker_name(self, worker_number):
-        """ Returns new worker name.
+        """Returns new worker name.
 
         This can be overriden in SpecCluster derived classes to customise the
         worker names.
@@ -491,7 +491,7 @@ def _new_worker_name(self, worker_number):
         return worker_number
 
     def new_worker_spec(self):
-        """ Return name and spec for the next worker
+        """Return name and spec for the next worker
 
         Returns
         -------
@@ -567,7 +567,7 @@ def adapt(
         maximum_memory: str = None,
         **kwargs,
     ) -> Adaptive:
-        """ Turn on adaptivity
+        """Turn on adaptivity
 
         This scales Dask clusters automatically based on scheduler activity.
 
diff --git a/distributed/deploy/ssh.py b/distributed/deploy/ssh.py
index bf75648efca..a6020998fec 100644
--- a/distributed/deploy/ssh.py
+++ b/distributed/deploy/ssh.py
@@ -17,7 +17,7 @@
 
 
 class Process(ProcessInterface):
-    """ A superclass for SSH Workers and Nannies
+    """A superclass for SSH Workers and Nannies
 
     See Also
     --------
@@ -47,7 +47,7 @@ def __repr__(self):
 
 
 class Worker(Process):
-    """ A Remote Dask Worker controled by SSH
+    """A Remote Dask Worker controled by SSH
 
     Parameters
     ----------
@@ -138,7 +138,7 @@ async def start(self):
 
 
 class Scheduler(Process):
-    """ A Remote Dask Scheduler controlled by SSH
+    """A Remote Dask Scheduler controlled by SSH
 
     Parameters
     ----------
@@ -238,7 +238,7 @@ def SSHCluster(
     remote_python: str = None,
     **kwargs,
 ):
-    """ Deploy a Dask cluster using SSH
+    """Deploy a Dask cluster using SSH
 
     The SSHCluster function deploys a Dask Scheduler and Workers for you on a
     set of machine addresses that you provide.  The first address will be used
diff --git a/distributed/deploy/tests/test_adaptive.py b/distributed/deploy/tests/test_adaptive.py
index bbdfd52c1f0..17f1b175a27 100644
--- a/distributed/deploy/tests/test_adaptive.py
+++ b/distributed/deploy/tests/test_adaptive.py
@@ -195,7 +195,7 @@ async def test_min_max():
 
 @pytest.mark.asyncio
 async def test_avoid_churn(cleanup):
-    """ We want to avoid creating and deleting workers frequently
+    """We want to avoid creating and deleting workers frequently
 
     Instead we want to wait a few beats before removing a worker in case the
     user is taking a brief pause between work
@@ -220,7 +220,7 @@ async def test_avoid_churn(cleanup):
 
 @pytest.mark.asyncio
 async def test_adapt_quickly():
-    """ We want to avoid creating and deleting workers frequently
+    """We want to avoid creating and deleting workers frequently
 
     Instead we want to wait a few beats before removing a worker in case the
     user is taking a brief pause between work
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index e54ad048da8..b46fdd065de 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -1004,7 +1004,7 @@ async def test_threads_per_worker_set_to_0(cleanup):
         Warning, match="Setting `threads_per_worker` to 0 is discouraged."
     ):
         async with LocalCluster(
-            n_workers=2, processes=False, threads_per_worker=0, asynchronous=True,
+            n_workers=2, processes=False, threads_per_worker=0, asynchronous=True
         ) as cluster:
             assert len(cluster.workers) == 2
             assert all(w.nthreads < CPU_COUNT for w in cluster.workers.values())
diff --git a/distributed/diagnostics/eventstream.py b/distributed/diagnostics/eventstream.py
index c0fde24470b..f1f70f458af 100644
--- a/distributed/diagnostics/eventstream.py
+++ b/distributed/diagnostics/eventstream.py
@@ -34,7 +34,7 @@ def teardown(scheduler, es):
 
 
 async def eventstream(address, interval):
-    """ Open a TCP connection to scheduler, receive batched task messages
+    """Open a TCP connection to scheduler, receive batched task messages
 
     The messages coming back are lists of dicts.  Each dict is of the following
     form::
diff --git a/distributed/diagnostics/graph_layout.py b/distributed/diagnostics/graph_layout.py
index ab36bd6978f..4404aa6e111 100644
--- a/distributed/diagnostics/graph_layout.py
+++ b/distributed/diagnostics/graph_layout.py
@@ -2,7 +2,7 @@
 
 
 class GraphLayout(SchedulerPlugin):
-    """ Dynamic graph layout during computation
+    """Dynamic graph layout during computation
 
     This assigns (x, y) locations to all tasks quickly and dynamically as new
     tasks are added.  This scales to a few thousand nodes.
@@ -116,7 +116,7 @@ def transition(self, key, start, finish, *args, **kwargs):
                 del collection[key]
 
     def reset_index(self):
-        """ Reset the index and refill new and new_edges
+        """Reset the index and refill new and new_edges
 
         From time to time TaskGraph wants to remove invisible nodes and reset
         all of its indices.  This helps.
diff --git a/distributed/diagnostics/plugin.py b/distributed/diagnostics/plugin.py
index fb3b2afe203..2e9cb31ba14 100644
--- a/distributed/diagnostics/plugin.py
+++ b/distributed/diagnostics/plugin.py
@@ -4,7 +4,7 @@
 
 
 class SchedulerPlugin:
-    """ Interface to extend the Scheduler
+    """Interface to extend the Scheduler
 
     The scheduler operates by triggering and responding to events like
     ``task_finished``, ``update_graph``, ``task_erred``, etc..
@@ -38,14 +38,14 @@ class SchedulerPlugin:
     """
 
     async def start(self, scheduler):
-        """ Run when the scheduler starts up
+        """Run when the scheduler starts up
 
         This runs at the end of the Scheduler startup process
         """
         pass
 
     async def close(self):
-        """ Run when the scheduler closes down
+        """Run when the scheduler closes down
 
         This runs at the beginning of the Scheduler shutdown process, but after
         workers have been asked to shut down gracefully
@@ -59,7 +59,7 @@ def restart(self, scheduler, **kwargs):
         """ Run when the scheduler restarts itself """
 
     def transition(self, key, start, finish, *args, **kwargs):
-        """ Run whenever a task changes state
+        """Run whenever a task changes state
 
         Parameters
         ----------
@@ -87,7 +87,7 @@ def remove_client(self, scheduler=None, client=None, **kwargs):
 
 
 class WorkerPlugin:
-    """ Interface to extend the Worker
+    """Interface to extend the Worker
 
     A worker plugin enables custom code to run at different stages of the Workers'
     lifecycle: at setup, during task state transitions, when a task or dependency
diff --git a/distributed/diagnostics/progress.py b/distributed/diagnostics/progress.py
index 2aeba986839..4cb1188a010 100644
--- a/distributed/diagnostics/progress.py
+++ b/distributed/diagnostics/progress.py
@@ -40,7 +40,7 @@ def dependent_keys(tasks, complete=False):
 
 
 class Progress(SchedulerPlugin):
-    """ Tracks progress of a set of keys or futures
+    """Tracks progress of a set of keys or futures
 
     On creation we provide a set of keys or futures that interest us as well as
     a scheduler.  We traverse through the scheduler's dependencies to find all
@@ -130,7 +130,7 @@ def stop(self, exception=None, key=None):
 
 
 class MultiProgress(Progress):
-    """ Progress variant that keeps track of different groups of keys
+    """Progress variant that keeps track of different groups of keys
 
     See Progress for most details.  This only adds a function ``func=``
     that splits keys.  This defaults to ``key_split`` which aligns with naming
diff --git a/distributed/diagnostics/progress_stream.py b/distributed/diagnostics/progress_stream.py
index c5e74a30f34..f40da0495bc 100644
--- a/distributed/diagnostics/progress_stream.py
+++ b/distributed/diagnostics/progress_stream.py
@@ -24,7 +24,7 @@ def counts(scheduler, allprogress):
 
 
 async def progress_stream(address, interval):
-    """ Open a TCP connection to scheduler, receive progress messages
+    """Open a TCP connection to scheduler, receive progress messages
 
     The messages coming back are dicts containing counts of key groups::
 
diff --git a/distributed/diagnostics/progressbar.py b/distributed/diagnostics/progressbar.py
index fb663524c5e..af1c0b7f26c 100644
--- a/distributed/diagnostics/progressbar.py
+++ b/distributed/diagnostics/progressbar.py
@@ -64,7 +64,7 @@ def function(scheduler, p):
             return result
 
         self.comm = await connect(
-            self.scheduler, **(self.client().connection_args if self.client else {}),
+            self.scheduler, **(self.client().connection_args if self.client else {})
         )
         logger.debug("Progressbar Connected to scheduler")
 
@@ -113,7 +113,7 @@ def __init__(
         loop=None,
         complete=True,
         start=True,
-        **kwargs
+        **kwargs,
     ):
         super(TextProgressBar, self).__init__(keys, scheduler, interval, complete)
         self.width = width
@@ -141,7 +141,7 @@ def _draw_stop(self, **kwargs):
 
 
 class ProgressWidget(ProgressBar):
-    """ ProgressBar that uses an IPython ProgressBar widget for the notebook
+    """ProgressBar that uses an IPython ProgressBar widget for the notebook
 
     See Also
     --------
@@ -156,7 +156,7 @@ def __init__(
         interval="100ms",
         complete=False,
         loop=None,
-        **kwargs
+        **kwargs,
     ):
         super(ProgressWidget, self).__init__(keys, scheduler, interval, complete)
 
@@ -215,7 +215,7 @@ def __init__(
         func=key_split,
         interval="100ms",
         complete=False,
-        **kwargs
+        **kwargs,
     ):
         self.scheduler = get_scheduler(scheduler)
 
@@ -292,7 +292,7 @@ def __del__(self):
 
 
 class MultiProgressWidget(MultiProgressBar):
-    """ Multiple progress bar Widget suitable for the notebook
+    """Multiple progress bar Widget suitable for the notebook
 
     Displays multiple progress bars for a computation, split on computation
     type.
@@ -312,7 +312,7 @@ def __init__(
         interval=0.1,
         func=key_split,
         complete=False,
-        **kwargs
+        **kwargs,
     ):
         super(MultiProgressWidget, self).__init__(
             keys, scheduler, func, interval, complete
@@ -399,7 +399,7 @@ def _draw_bar(self, remaining, all, status, **kwargs):
 
 
 def progress(*futures, notebook=None, multi=True, complete=True, **kwargs):
-    """ Track progress of futures
+    """Track progress of futures
 
     This operates differently in the notebook and the console
 
diff --git a/distributed/diagnostics/tests/test_worker_plugin.py b/distributed/diagnostics/tests/test_worker_plugin.py
index c9cdbed784c..9bcc0c58ca4 100644
--- a/distributed/diagnostics/tests/test_worker_plugin.py
+++ b/distributed/diagnostics/tests/test_worker_plugin.py
@@ -31,18 +31,14 @@ def teardown(self, worker):
 
     def transition(self, key, start, finish, **kwargs):
         self.observed_notifications.append(
-            {"key": key, "start": start, "finish": finish,}
+            {"key": key, "start": start, "finish": finish}
         )
 
     def release_key(self, key, state, cause, reason, report):
-        self.observed_notifications.append(
-            {"key": key, "state": state,}
-        )
+        self.observed_notifications.append({"key": key, "state": state})
 
     def release_dep(self, dep, state, report):
-        self.observed_notifications.append(
-            {"dep": dep, "state": state,}
-        )
+        self.observed_notifications.append({"dep": dep, "state": state})
 
 
 @gen_cluster(client=True, nthreads=[])
@@ -100,7 +96,7 @@ def failing(x):
 
 
 @gen_cluster(
-    nthreads=[("127.0.0.1", 1)], client=True, worker_kwargs={"resources": {"X": 1}},
+    nthreads=[("127.0.0.1", 1)], client=True, worker_kwargs={"resources": {"X": 1}}
 )
 async def test_superseding_task_transitions_called(c, s, w):
     expected_notifications = [
@@ -119,10 +115,7 @@ async def test_superseding_task_transitions_called(c, s, w):
 
 @gen_cluster(nthreads=[("127.0.0.1", 1)], client=True)
 async def test_release_dep_called(c, s, w):
-    dsk = {
-        "dep": 1,
-        "task": (inc, "dep"),
-    }
+    dsk = {"dep": 1, "task": (inc, "dep")}
 
     expected_notifications = [
         {"key": "dep", "start": "waiting", "finish": "ready"},
diff --git a/distributed/diagnostics/websocket.py b/distributed/diagnostics/websocket.py
index 641730faf54..f8704487c14 100644
--- a/distributed/diagnostics/websocket.py
+++ b/distributed/diagnostics/websocket.py
@@ -33,7 +33,7 @@ def update_graph(self, scheduler, client=None, **kwargs):
         self.socket.send("update_graph", {"client": client})
 
     def transition(self, key, start, finish, *args, **kwargs):
-        """ Run whenever a task changes state
+        """Run whenever a task changes state
 
         Parameters
         ----------
diff --git a/distributed/event.py b/distributed/event.py
index 75661654b86..0136d35ef26 100644
--- a/distributed/event.py
+++ b/distributed/event.py
@@ -13,7 +13,7 @@
 
 
 class EventExtension:
-    """ An extension for the scheduler to manage Events
+    """An extension for the scheduler to manage Events
 
     This adds the following routes to the scheduler
 
@@ -60,7 +60,7 @@ def __init__(self, scheduler):
         self.scheduler.extensions["events"] = self
 
     async def event_wait(self, comm=None, name=None, timeout=None):
-        """ Wait until the event is set to true.
+        """Wait until the event is set to true.
         Returns false, when this did not happen in the given time
         and true otherwise.
         """
@@ -89,7 +89,7 @@ async def event_wait(self, comm=None, name=None, timeout=None):
             return True
 
     def event_set(self, comm=None, name=None):
-        """ Set the event with the given name to true.
+        """Set the event with the given name to true.
 
         All waiters on this event will be notified.
         """
@@ -150,7 +150,7 @@ def _delete_event(self, name):
 
 
 class Event:
-    """ Distributed Centralized Event equivalent to asyncio.Event
+    """Distributed Centralized Event equivalent to asyncio.Event
 
     An event stores a single flag, which is set to false on start.
     The flag can be set to true (using the set() call) or back to false
@@ -186,7 +186,7 @@ def __init__(self, name=None, client=None):
         self.name = name or "event-" + uuid.uuid4().hex
 
     def __await__(self):
-        """ async constructor
+        """async constructor
 
         Make it possible to write
 
@@ -201,7 +201,7 @@ async def _():
         return _().__await__()
 
     def wait(self, timeout=None):
-        """ Wait until the event is set.
+        """Wait until the event is set.
 
         Parameters
         ----------
@@ -223,28 +223,28 @@ def wait(self, timeout=None):
         timeout = parse_timedelta(timeout)
 
         result = self.client.sync(
-            self.client.scheduler.event_wait, name=self.name, timeout=timeout,
+            self.client.scheduler.event_wait, name=self.name, timeout=timeout
         )
         return result
 
     def clear(self):
-        """ Clear the event (set its flag to false).
+        """Clear the event (set its flag to false).
 
         All waiters will now block.
         """
         return self.client.sync(self.client.scheduler.event_clear, name=self.name)
 
     def set(self):
-        """ Set the event (set its flag to false).
+        """Set the event (set its flag to false).
 
         All waiters will now be released.
         """
-        result = self.client.sync(self.client.scheduler.event_set, name=self.name,)
+        result = self.client.sync(self.client.scheduler.event_set, name=self.name)
         return result
 
     def is_set(self):
         """ Check if the event is set """
-        result = self.client.sync(self.client.scheduler.event_is_set, name=self.name,)
+        result = self.client.sync(self.client.scheduler.event_is_set, name=self.name)
         return result
 
     def __reduce__(self):
diff --git a/distributed/http/health.py b/distributed/http/health.py
index 2a45c4abf77..0fab7dea7c8 100644
--- a/distributed/http/health.py
+++ b/distributed/http/health.py
@@ -7,6 +7,4 @@ def get(self):
         self.set_header("Content-Type", "text/plain")
 
 
-routes = [
-    ("/health", HealthHandler, {}),
-]
+routes = [("/health", HealthHandler, {})]
diff --git a/distributed/http/proxy.py b/distributed/http/proxy.py
index c1f437d9b5f..6e39a999990 100644
--- a/distributed/http/proxy.py
+++ b/distributed/http/proxy.py
@@ -74,8 +74,7 @@ def proxy(self, port, proxied_path):
     )
 
     class GlobalProxyHandler(web.RequestHandler):
-        """Minimal Proxy handler when jupyter-server-proxy is not installed
-        """
+        """Minimal Proxy handler when jupyter-server-proxy is not installed"""
 
         def initialize(self, dask_server=None, extra=None):
             self.server = dask_server
@@ -130,6 +129,4 @@ def check_worker_dashboard_exits(scheduler, worker):
     return False
 
 
-routes = [
-    (r"proxy/(\d+)/(.*?)/(.*)", GlobalProxyHandler, {}),
-]
+routes = [(r"proxy/(\d+)/(.*?)/(.*)", GlobalProxyHandler, {})]
diff --git a/distributed/http/routing.py b/distributed/http/routing.py
index ecd14da28e4..8a1d90d5490 100644
--- a/distributed/http/routing.py
+++ b/distributed/http/routing.py
@@ -43,7 +43,7 @@ class RoutingApplication(web.Application):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.applications = []
-        self.add_handlers(".*$", [(r"/sitemap.json", DirectoryHandler),])
+        self.add_handlers(".*$", [(r"/sitemap.json", DirectoryHandler)])
 
     def find_handler(self, request: tornado.httputil.HTTPServerRequest, **kwargs):
         handler = super().find_handler(request, **kwargs)
diff --git a/distributed/http/statics.py b/distributed/http/statics.py
index 4a8a60298fe..e1c7a98e9a2 100644
--- a/distributed/http/statics.py
+++ b/distributed/http/statics.py
@@ -6,5 +6,5 @@
         r"/statics/(.*)",
         web.StaticFileHandler,
         {"path": os.path.join(os.path.dirname(__file__), "static")},
-    ),
+    )
 ]
diff --git a/distributed/http/tests/test_routing.py b/distributed/http/tests/test_routing.py
index 481cfb3a209..ca7d071d256 100644
--- a/distributed/http/tests/test_routing.py
+++ b/distributed/http/tests/test_routing.py
@@ -17,8 +17,8 @@ def get(self):
 
 @pytest.mark.asyncio
 async def test_basic():
-    application = RoutingApplication([(r"/one", OneHandler),])
-    two = web.Application([(r"/two", TwoHandler),])
+    application = RoutingApplication([(r"/one", OneHandler)])
+    two = web.Application([(r"/two", TwoHandler)])
     server = application.listen(1234)
 
     client = AsyncHTTPClient("http://localhost:1234")
diff --git a/distributed/http/worker/prometheus.py b/distributed/http/worker/prometheus.py
index a60de3a6b64..69e3439df9b 100644
--- a/distributed/http/worker/prometheus.py
+++ b/distributed/http/worker/prometheus.py
@@ -93,6 +93,4 @@ def get(self):
         self.set_header("Content-Type", "text/plain; version=0.0.4")
 
 
-routes = [
-    (r"metrics", PrometheusHandler, {}),
-]
+routes = [(r"metrics", PrometheusHandler, {})]
diff --git a/distributed/lock.py b/distributed/lock.py
index 67e926ac281..7d1c1a4af57 100644
--- a/distributed/lock.py
+++ b/distributed/lock.py
@@ -12,7 +12,7 @@
 
 
 class LockExtension:
-    """ An extension for the scheduler to manage Locks
+    """An extension for the scheduler to manage Locks
 
     This adds the following routes to the scheduler
 
@@ -73,7 +73,7 @@ def release(self, comm=None, name=None, id=None):
 
 
 class Lock:
-    """ Distributed Centralized Lock
+    """Distributed Centralized Lock
 
     Parameters
     ----------
@@ -104,7 +104,7 @@ def __init__(self, name=None, client=None):
         self._locked = False
 
     def acquire(self, blocking=True, timeout=None):
-        """ Acquire the lock
+        """Acquire the lock
 
         Parameters
         ----------
diff --git a/distributed/nanny.py b/distributed/nanny.py
index db29431211b..fd7078982a7 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -41,7 +41,7 @@
 
 
 class Nanny(ServerNode):
-    """ A process to manage worker processes
+    """A process to manage worker processes
 
     The nanny spins up Worker processes, watches then, and kills or restarts
     them as necessary. It is necessary if you want to use the
@@ -304,7 +304,7 @@ async def start(self):
         return self
 
     async def kill(self, comm=None, timeout=2):
-        """ Kill the local worker process
+        """Kill the local worker process
 
         Blocks until both the process is down and the scheduler is properly
         informed
@@ -317,7 +317,7 @@ async def kill(self, comm=None, timeout=2):
         await self.process.kill(timeout=0.8 * (deadline - self.loop.time()))
 
     async def instantiate(self, comm=None) -> Status:
-        """ Start a local worker process
+        """Start a local worker process
 
         Blocks until the process is up and the scheduler is properly informed
         """
diff --git a/distributed/preloading.py b/distributed/preloading.py
index 112f537e2ca..f1e125b7b13 100644
--- a/distributed/preloading.py
+++ b/distributed/preloading.py
@@ -72,7 +72,7 @@ def is_webaddress(s: str) -> bool:
 
 
 def _import_module(name, file_dir=None) -> ModuleType:
-    """ Imports module and extract preload interface functions.
+    """Imports module and extract preload interface functions.
 
     Import modules specified by name and extract 'dask_setup'
     and 'dask_teardown' if present.
diff --git a/distributed/process.py b/distributed/process.py
index b070342b340..a0d462d4074 100644
--- a/distributed/process.py
+++ b/distributed/process.py
@@ -162,7 +162,7 @@ def monitor_parent():
 
     @staticmethod
     def reset_logger_locks():
-        """ Python 2's logger's locks don't survive a fork event
+        """Python 2's logger's locks don't survive a fork event
 
         https://github.com/dask/distributed/issues/1491
         """
diff --git a/distributed/profile.py b/distributed/profile.py
index 33eba502ef9..2b3363e6945 100644
--- a/distributed/profile.py
+++ b/distributed/profile.py
@@ -38,7 +38,7 @@
 
 
 def identifier(frame):
-    """ A string identifier from a frame
+    """A string identifier from a frame
 
     Strings are cheaper to use as indexes into dicts than tuples or dicts
     """
@@ -74,7 +74,7 @@ def info_frame(frame):
 
 
 def process(frame, child, state, stop=None, omit=None):
-    """ Add counts from a frame stack onto existing state
+    """Add counts from a frame stack onto existing state
 
     This recursively adds counts to the existing state dictionary and creates
     new entries for new functions.
@@ -157,7 +157,7 @@ def create():
 
 
 def call_stack(frame):
-    """ Create a call text stack from a frame
+    """Create a call text stack from a frame
 
     Returns
     -------
@@ -171,7 +171,7 @@ def call_stack(frame):
 
 
 def plot_data(state, profile_interval=0.010):
-    """ Convert a profile state into data useful by Bokeh
+    """Convert a profile state into data useful by Bokeh
 
     See Also
     --------
@@ -277,7 +277,7 @@ def watch(
     omit=None,
     stop=lambda: False,
 ):
-    """ Gather profile information on a particular thread
+    """Gather profile information on a particular thread
 
     This starts a new thread to watch a particular thread and returns a deque
     that holds periodic profile information.
@@ -324,7 +324,7 @@ def watch(
 
 
 def get_profile(history, recent=None, start=None, stop=None, key=None):
-    """ Collect profile information from a sequence of profile states
+    """Collect profile information from a sequence of profile states
 
     Parameters
     ----------
@@ -366,7 +366,7 @@ def get_profile(history, recent=None, start=None, stop=None, key=None):
 
 
 def plot_figure(data, **kwargs):
-    """ Plot profile data using Bokeh
+    """Plot profile data using Bokeh
 
     This takes the output from the function ``plot_data`` and produces a Bokeh
     figure
@@ -444,7 +444,7 @@ def _remove_py_stack(frames):
 
 
 def llprocess(frames, child, state):
-    """ Add counts from low level profile information onto existing state
+    """Add counts from low level profile information onto existing state
 
     This uses the ``stacktrace`` module to collect low level stack trace
     information and place it onto the given sttate.
diff --git a/distributed/protocol/compression.py b/distributed/protocol/compression.py
index 3f46fc5d58b..82cfbeec3c2 100644
--- a/distributed/protocol/compression.py
+++ b/distributed/protocol/compression.py
@@ -143,7 +143,7 @@ def get_default_compression():
 
 
 def byte_sample(b, size, n):
-    """ Sample a bytestring from many locations
+    """Sample a bytestring from many locations
 
     Parameters
     ----------
diff --git a/distributed/protocol/core.py b/distributed/protocol/core.py
index 32071d9e3d0..ceee3982941 100644
--- a/distributed/protocol/core.py
+++ b/distributed/protocol/core.py
@@ -173,7 +173,7 @@ def put_in(keys, coll, val):
 
 
 def dumps_msgpack(msg, compression=None):
-    """ Dump msg into header and payload, both bytestrings
+    """Dump msg into header and payload, both bytestrings
 
     All of the message must be msgpack encodable
 
@@ -196,7 +196,7 @@ def dumps_msgpack(msg, compression=None):
 
 
 def loads_msgpack(header, payload):
-    """ Read msgpack header and payload back to Python object
+    """Read msgpack header and payload back to Python object
 
     See Also:
         dumps_msgpack
diff --git a/distributed/protocol/numba.py b/distributed/protocol/numba.py
index 93d65c0ef6d..e1915251f6f 100644
--- a/distributed/protocol/numba.py
+++ b/distributed/protocol/numba.py
@@ -26,7 +26,7 @@ def cuda_serialize_numba_ndarray(x):
     header["lengths"] = [x.nbytes]
     frames = [
         numba.cuda.cudadrv.devicearray.DeviceNDArray(
-            shape=(x.nbytes,), strides=(1,), dtype=np.dtype("u1"), gpu_data=x.gpu_data,
+            shape=(x.nbytes,), strides=(1,), dtype=np.dtype("u1"), gpu_data=x.gpu_data
         )
     ]
 
diff --git a/distributed/protocol/numpy.py b/distributed/protocol/numpy.py
index 497bc7a045e..4ae9298f142 100644
--- a/distributed/protocol/numpy.py
+++ b/distributed/protocol/numpy.py
@@ -8,7 +8,7 @@
 
 
 def itemsize(dt):
-    """ Itemsize of dtype
+    """Itemsize of dtype
 
     Try to return the itemsize of the base element, return 8 as a fallback
     """
diff --git a/distributed/protocol/pickle.py b/distributed/protocol/pickle.py
index 0a9f86846a2..843f6a28d9a 100644
--- a/distributed/protocol/pickle.py
+++ b/distributed/protocol/pickle.py
@@ -34,7 +34,7 @@ def _always_use_pickle_for(x):
 
 
 def dumps(x, *, buffer_callback=None, protocol=HIGHEST_PROTOCOL):
-    """ Manage between cloudpickle and pickle
+    """Manage between cloudpickle and pickle
 
     1.  Try pickle
     2.  If it is short then check if it contains __main__
diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index abbbb9a5302..97b962a3f67 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -336,7 +336,7 @@ def deserialize(header, frames, deserializers=None):
 
 
 class Serialize:
-    """ Mark an object that should be serialized
+    """Mark an object that should be serialized
 
     Examples
     --------
@@ -408,7 +408,7 @@ def container_copy(c):
 
 
 def extract_serialize(x):
-    """ Pull out Serialize objects from message
+    """Pull out Serialize objects from message
 
     This also remove large bytestrings from the message into a second
     dictionary.
@@ -547,7 +547,7 @@ def deserialize_bytes(b):
 
 
 def register_serialization(cls, serialize, deserialize):
-    """ Register a new class for dask-custom serialization
+    """Register a new class for dask-custom serialization
 
     Parameters
     ----------
@@ -752,7 +752,7 @@ def register_generic(
     serialize_func=dask_serialize,
     deserialize_func=dask_deserialize,
 ):
-    """ Register (de)serialize to traverse through __dict__
+    """Register (de)serialize to traverse through __dict__
 
     Normally when registering new classes for Dask's custom serialization you
     need to manage headers and frames, which can be tedious.  If all you want
diff --git a/distributed/protocol/tests/test_cupy.py b/distributed/protocol/tests/test_cupy.py
index 95cb530c4db..520693fb5c1 100644
--- a/distributed/protocol/tests/test_cupy.py
+++ b/distributed/protocol/tests/test_cupy.py
@@ -65,11 +65,11 @@ def test_serialize_cupy_from_rmm(size):
 
 
 @pytest.mark.parametrize(
-    "sparse_name", ["coo_matrix", "csc_matrix", "csr_matrix", "dia_matrix",],
+    "sparse_name", ["coo_matrix", "csc_matrix", "csr_matrix", "dia_matrix"]
 )
 @pytest.mark.parametrize(
     "dtype",
-    [numpy.dtype("<f4"), numpy.dtype(">f4"), numpy.dtype("<f8"), numpy.dtype(">f8"),],
+    [numpy.dtype("<f4"), numpy.dtype(">f4"), numpy.dtype("<f8"), numpy.dtype(">f8")],
 )
 @pytest.mark.parametrize("serializer", ["cuda", "dask", "pickle"])
 def test_serialize_cupy_sparse(sparse_name, dtype, serializer):
diff --git a/distributed/protocol/tests/test_protocol_utils.py b/distributed/protocol/tests/test_protocol_utils.py
index 2132f94008b..847dec1ac3d 100644
--- a/distributed/protocol/tests/test_protocol_utils.py
+++ b/distributed/protocol/tests/test_protocol_utils.py
@@ -21,10 +21,7 @@
     ],
 )
 def test_merge_frames(lengths, writeable, frames):
-    header = {
-        "lengths": lengths,
-        "writeable": writeable,
-    }
+    header = {"lengths": lengths, "writeable": writeable}
     result = merge_frames(header, frames)
 
     data = b"".join(frames)
diff --git a/distributed/protocol/tests/test_scipy.py b/distributed/protocol/tests/test_scipy.py
index 2cb5d7477e5..4e5eb8423cf 100644
--- a/distributed/protocol/tests/test_scipy.py
+++ b/distributed/protocol/tests/test_scipy.py
@@ -20,7 +20,7 @@
 )
 @pytest.mark.parametrize(
     "dtype",
-    [numpy.dtype("<f4"), numpy.dtype(">f4"), numpy.dtype("<f8"), numpy.dtype(">f8"),],
+    [numpy.dtype("<f4"), numpy.dtype(">f4"), numpy.dtype("<f8"), numpy.dtype(">f8")],
 )
 def test_serialize_scipy_sparse(sparse_type, dtype):
     a = numpy.array([[0, 1, 0], [2, 0, 3], [0, 4, 0]], dtype=dtype)
diff --git a/distributed/protocol/tests/test_serialize.py b/distributed/protocol/tests/test_serialize.py
index 4ead70361b7..72bc9764a22 100644
--- a/distributed/protocol/tests/test_serialize.py
+++ b/distributed/protocol/tests/test_serialize.py
@@ -81,7 +81,7 @@ def test_serialize_bytestrings():
 
 
 @pytest.mark.parametrize(
-    "typecode", ["b", "B", "h", "H", "i", "I", "l", "L", "q", "Q", "f", "d"],
+    "typecode", ["b", "B", "h", "H", "i", "I", "l", "L", "q", "Q", "f", "d"]
 )
 def test_serialize_arrays(typecode):
     a = array(typecode)
@@ -212,9 +212,7 @@ def test_empty_loads_deep():
     assert isinstance(e2[0][0][0], Empty)
 
 
-@pytest.mark.parametrize(
-    "kwargs", [{}, {"serializers": ["pickle"]},],
-)
+@pytest.mark.parametrize("kwargs", [{}, {"serializers": ["pickle"]}])
 def test_serialize_bytes(kwargs):
     for x in [
         1,
@@ -464,7 +462,7 @@ def test_serialize_lists(serializers):
 
 
 @pytest.mark.parametrize(
-    "data_in", [memoryview(b"hello"), memoryview(np.random.random((3, 4)))],
+    "data_in", [memoryview(b"hello"), memoryview(np.random.random((3, 4)))]
 )
 def test_deser_memoryview(data_in):
     header, frames = serialize(data_in)
diff --git a/distributed/protocol/utils.py b/distributed/protocol/utils.py
index 32fdfd7af1b..2b96fba4c61 100644
--- a/distributed/protocol/utils.py
+++ b/distributed/protocol/utils.py
@@ -35,7 +35,7 @@ def frame_split_size(frame, n=BIG_BYTES_SHARD_SIZE) -> list:
 
 
 def merge_frames(header, frames):
-    """ Merge frames into original lengths
+    """Merge frames into original lengths
 
     Examples
     --------
@@ -92,7 +92,7 @@ def pack_frames_prelude(frames):
 
 
 def pack_frames(frames):
-    """ Pack frames into a byte-like object
+    """Pack frames into a byte-like object
 
     This prepends length information to the front of the bytes-like object
 
@@ -104,7 +104,7 @@ def pack_frames(frames):
 
 
 def unpack_frames(b):
-    """ Unpack bytes into a sequence of frames
+    """Unpack bytes into a sequence of frames
 
     This assumes that length information is at the front of the bytestring,
     as performed by pack_frames
diff --git a/distributed/publish.py b/distributed/publish.py
index 44f34b68a72..9dbab4ff3dd 100644
--- a/distributed/publish.py
+++ b/distributed/publish.py
@@ -4,7 +4,7 @@
 
 
 class PublishExtension:
-    """ An extension for the scheduler to manage collections
+    """An extension for the scheduler to manage collections
 
     *  publish_list
     *  publish_put
diff --git a/distributed/pubsub.py b/distributed/pubsub.py
index 8234d1a8e19..3aeec084df1 100644
--- a/distributed/pubsub.py
+++ b/distributed/pubsub.py
@@ -199,7 +199,7 @@ def cleanup(self):
 
 
 class Pub:
-    """ Publish data with Publish-Subscribe pattern
+    """Publish data with Publish-Subscribe pattern
 
     This allows clients and workers to directly communicate data between each
     other with a typical Publish-Subscribe pattern.  This involves two
@@ -355,7 +355,7 @@ def __repr__(self):
 
 
 class Sub:
-    """ Subscribe to a Publish/Subscribe topic
+    """Subscribe to a Publish/Subscribe topic
 
     See Also
     --------
@@ -429,7 +429,7 @@ async def _():
     __anext__ = _get
 
     def get(self, timeout=None):
-        """ Get a single message
+        """Get a single message
 
         Parameters
         ----------
diff --git a/distributed/queues.py b/distributed/queues.py
index 9b5d1dbf1f3..6646676cce3 100644
--- a/distributed/queues.py
+++ b/distributed/queues.py
@@ -12,7 +12,7 @@
 
 
 class QueueExtension:
-    """ An extension for the scheduler to manage queues
+    """An extension for the scheduler to manage queues
 
     This adds the following routes to the scheduler
 
@@ -127,7 +127,7 @@ def qsize(self, comm=None, name=None, client=None):
 
 
 class Queue:
-    """ Distributed Queue
+    """Distributed Queue
 
     This allows multiple clients to share futures or small bits of data between
     each other with a multi-producer/multi-consumer queue.  All metadata is
@@ -209,7 +209,7 @@ async def _put(self, value, timeout=None):
             )
 
     def put(self, value, timeout=None, **kwargs):
-        """ Put data into the queue
+        """Put data into the queue
 
         Parameters
         ----------
@@ -222,7 +222,7 @@ def put(self, value, timeout=None, **kwargs):
         return self.client.sync(self._put, value, timeout=timeout, **kwargs)
 
     def get(self, timeout=None, batch=False, **kwargs):
-        """ Get data from the queue
+        """Get data from the queue
 
         Parameters
         ----------
@@ -234,7 +234,7 @@ def get(self, timeout=None, batch=False, **kwargs):
             If True then return all elements currently waiting in the queue.
             If an integer than return that many elements from the queue
             If False (default) then return one item at a time
-         """
+        """
         timeout = parse_timedelta(timeout)
         return self.client.sync(self._get, timeout=timeout, batch=batch, **kwargs)
 
diff --git a/distributed/recreate_exceptions.py b/distributed/recreate_exceptions.py
index 4aaa851ee23..51a30e52e1c 100644
--- a/distributed/recreate_exceptions.py
+++ b/distributed/recreate_exceptions.py
@@ -8,7 +8,7 @@
 
 
 class ReplayExceptionScheduler:
-    """ A plugin for the scheduler to recreate exceptions locally
+    """A plugin for the scheduler to recreate exceptions locally
 
     This adds the following routes to the scheduler
 
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 5f7bb1d4eab..3f787a1f92e 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -371,235 +371,235 @@ def ncores(self):
 
 class TaskState:
     """
-    A simple object holding information about a task.
+        A simple object holding information about a task.
 
-    .. attribute:: key: str
+        .. attribute:: key: str
 
-       The key is the unique identifier of a task, generally formed
-       from the name of the function, followed by a hash of the function
-       and arguments, like ``'inc-ab31c010444977004d656610d2d421ec'``.
+           The key is the unique identifier of a task, generally formed
+           from the name of the function, followed by a hash of the function
+           and arguments, like ``'inc-ab31c010444977004d656610d2d421ec'``.
 
-    .. attribute:: prefix: TaskPrefix
+        .. attribute:: prefix: TaskPrefix
 
-       The broad class of tasks to which this task belongs like "inc" or
-       "read_csv"
+           The broad class of tasks to which this task belongs like "inc" or
+           "read_csv"
 
-    .. attribute:: run_spec: object
+        .. attribute:: run_spec: object
 
-       A specification of how to run the task.  The type and meaning of this
-       value is opaque to the scheduler, as it is only interpreted by the
-       worker to which the task is sent for executing.
+           A specification of how to run the task.  The type and meaning of this
+           value is opaque to the scheduler, as it is only interpreted by the
+           worker to which the task is sent for executing.
 
-       As a special case, this attribute may also be ``None``, in which case
-       the task is "pure data" (such as, for example, a piece of data loaded
-       in the scheduler using :meth:`Client.scatter`).  A "pure data" task
-       cannot be computed again if its value is lost.
+           As a special case, this attribute may also be ``None``, in which case
+           the task is "pure data" (such as, for example, a piece of data loaded
+           in the scheduler using :meth:`Client.scatter`).  A "pure data" task
+           cannot be computed again if its value is lost.
 
-    .. attribute:: priority: tuple
+        .. attribute:: priority: tuple
 
-       The priority provides each task with a relative ranking which is used
-       to break ties when many tasks are being considered for execution.
+           The priority provides each task with a relative ranking which is used
+           to break ties when many tasks are being considered for execution.
 
-       This ranking is generally a 2-item tuple.  The first (and dominant)
-       item corresponds to when it was submitted.  Generally, earlier tasks
-       take precedence.  The second item is determined by the client, and is
-       a way to prioritize tasks within a large graph that may be important,
-       such as if they are on the critical path, or good to run in order to
-       release many dependencies.  This is explained further in
-       :doc:`Scheduling Policy <scheduling-policies>`.
+           This ranking is generally a 2-item tuple.  The first (and dominant)
+           item corresponds to when it was submitted.  Generally, earlier tasks
+           take precedence.  The second item is determined by the client, and is
+           a way to prioritize tasks within a large graph that may be important,
+           such as if they are on the critical path, or good to run in order to
+           release many dependencies.  This is explained further in
+           :doc:`Scheduling Policy <scheduling-policies>`.
 
-    .. attribute:: state: str
+        .. attribute:: state: str
 
-       This task's current state.  Valid states include ``released``,
-       ``waiting``, ``no-worker``, ``processing``, ``memory``, ``erred``
-       and ``forgotten``.  If it is ``forgotten``, the task isn't stored
-       in the ``tasks`` dictionary anymore and will probably disappear
-       soon from memory.
+           This task's current state.  Valid states include ``released``,
+           ``waiting``, ``no-worker``, ``processing``, ``memory``, ``erred``
+           and ``forgotten``.  If it is ``forgotten``, the task isn't stored
+           in the ``tasks`` dictionary anymore and will probably disappear
+           soon from memory.
 
-    .. attribute:: dependencies: {TaskState}
+        .. attribute:: dependencies: {TaskState}
 
-       The set of tasks this task depends on for proper execution.  Only
-       tasks still alive are listed in this set.  If, for whatever reason,
-       this task also depends on a forgotten task, the
-       :attr:`has_lost_dependencies` flag is set.
+           The set of tasks this task depends on for proper execution.  Only
+           tasks still alive are listed in this set.  If, for whatever reason,
+           this task also depends on a forgotten task, the
+           :attr:`has_lost_dependencies` flag is set.
 
-       A task can only be executed once all its dependencies have already
-       been successfully executed and have their result stored on at least
-       one worker.  This is tracked by progressively draining the
-       :attr:`waiting_on` set.
+           A task can only be executed once all its dependencies have already
+           been successfully executed and have their result stored on at least
+           one worker.  This is tracked by progressively draining the
+           :attr:`waiting_on` set.
 
-    .. attribute:: dependents: {TaskState}
+        .. attribute:: dependents: {TaskState}
 
-       The set of tasks which depend on this task.  Only tasks still alive
-       are listed in this set.
+           The set of tasks which depend on this task.  Only tasks still alive
+           are listed in this set.
 
-       This is the reverse mapping of :attr:`dependencies`.
+           This is the reverse mapping of :attr:`dependencies`.
 
-    .. attribute:: has_lost_dependencies: bool
+        .. attribute:: has_lost_dependencies: bool
 
-       Whether any of the dependencies of this task has been forgotten.
-       For memory consumption reasons, forgotten tasks are not kept in
-       memory even though they may have dependent tasks.  When a task is
-       forgotten, therefore, each of its dependents has their
-       :attr:`has_lost_dependencies` attribute set to ``True``.
+           Whether any of the dependencies of this task has been forgotten.
+           For memory consumption reasons, forgotten tasks are not kept in
+           memory even though they may have dependent tasks.  When a task is
+           forgotten, therefore, each of its dependents has their
+           :attr:`has_lost_dependencies` attribute set to ``True``.
 
-       If :attr:`has_lost_dependencies` is true, this task cannot go
-       into the "processing" state anymore.
+           If :attr:`has_lost_dependencies` is true, this task cannot go
+           into the "processing" state anymore.
 
-    .. attribute:: waiting_on: {TaskState}
+        .. attribute:: waiting_on: {TaskState}
 
-       The set of tasks this task is waiting on *before* it can be executed.
-       This is always a subset of :attr:`dependencies`.  Each time one of the
-       dependencies has finished processing, it is removed from the
-       :attr:`waiting_on` set.
+           The set of tasks this task is waiting on *before* it can be executed.
+           This is always a subset of :attr:`dependencies`.  Each time one of the
+           dependencies has finished processing, it is removed from the
+           :attr:`waiting_on` set.
 
-       Once :attr:`waiting_on` becomes empty, this task can move from the
-       "waiting" state to the "processing" state (unless one of the
-       dependencies errored out, in which case this task is instead
-       marked "erred").
+           Once :attr:`waiting_on` becomes empty, this task can move from the
+           "waiting" state to the "processing" state (unless one of the
+           dependencies errored out, in which case this task is instead
+           marked "erred").
 
-    .. attribute:: waiters: {TaskState}
+        .. attribute:: waiters: {TaskState}
 
-       The set of tasks which need this task to remain alive.  This is always
-       a subset of :attr:`dependents`.  Each time one of the dependents
-       has finished processing, it is removed from the :attr:`waiters`
-       set.
+           The set of tasks which need this task to remain alive.  This is always
+           a subset of :attr:`dependents`.  Each time one of the dependents
+           has finished processing, it is removed from the :attr:`waiters`
+           set.
 
-       Once both :attr:`waiters` and :attr:`who_wants` become empty, this
-       task can be released (if it has a non-empty :attr:`run_spec`) or
-       forgotten (otherwise) by the scheduler, and by any workers
-       in :attr:`who_has`.
+           Once both :attr:`waiters` and :attr:`who_wants` become empty, this
+           task can be released (if it has a non-empty :attr:`run_spec`) or
+           forgotten (otherwise) by the scheduler, and by any workers
+           in :attr:`who_has`.
 
-       .. note:: Counter-intuitively, :attr:`waiting_on` and
-          :attr:`waiters` are not reverse mappings of each other.
+           .. note:: Counter-intuitively, :attr:`waiting_on` and
+              :attr:`waiters` are not reverse mappings of each other.
 
-    .. attribute:: who_wants: {ClientState}
+        .. attribute:: who_wants: {ClientState}
 
-       The set of clients who want this task's result to remain alive.
-       This is the reverse mapping of :attr:`ClientState.wants_what`.
+           The set of clients who want this task's result to remain alive.
+           This is the reverse mapping of :attr:`ClientState.wants_what`.
 
-       When a client submits a graph to the scheduler it also specifies
-       which output tasks it desires, such that their results are not released
-       from memory.
+           When a client submits a graph to the scheduler it also specifies
+           which output tasks it desires, such that their results are not released
+           from memory.
 
-       Once a task has finished executing (i.e. moves into the "memory"
-       or "erred" state), the clients in :attr:`who_wants` are notified.
+           Once a task has finished executing (i.e. moves into the "memory"
+           or "erred" state), the clients in :attr:`who_wants` are notified.
 
-       Once both :attr:`waiters` and :attr:`who_wants` become empty, this
-       task can be released (if it has a non-empty :attr:`run_spec`) or
-       forgotten (otherwise) by the scheduler, and by any workers
-       in :attr:`who_has`.
+           Once both :attr:`waiters` and :attr:`who_wants` become empty, this
+           task can be released (if it has a non-empty :attr:`run_spec`) or
+           forgotten (otherwise) by the scheduler, and by any workers
+           in :attr:`who_has`.
 
-    .. attribute:: who_has: {WorkerState}
+        .. attribute:: who_has: {WorkerState}
 
-       The set of workers who have this task's result in memory.
-       It is non-empty iff the task is in the "memory" state.  There can be
-       more than one worker in this set if, for example, :meth:`Client.scatter`
-       or :meth:`Client.replicate` was used.
+           The set of workers who have this task's result in memory.
+           It is non-empty iff the task is in the "memory" state.  There can be
+           more than one worker in this set if, for example, :meth:`Client.scatter`
+           or :meth:`Client.replicate` was used.
 
-       This is the reverse mapping of :attr:`WorkerState.has_what`.
+           This is the reverse mapping of :attr:`WorkerState.has_what`.
 
-    .. attribute:: processing_on: WorkerState (or None)
+        .. attribute:: processing_on: WorkerState (or None)
 
-       If this task is in the "processing" state, which worker is currently
-       processing it.  Otherwise this is ``None``.
+           If this task is in the "processing" state, which worker is currently
+           processing it.  Otherwise this is ``None``.
 
-       This attribute is kept in sync with :attr:`WorkerState.processing`.
+           This attribute is kept in sync with :attr:`WorkerState.processing`.
 
-    .. attribute:: retries: int
+        .. attribute:: retries: int
 
-       The number of times this task can automatically be retried in case
-       of failure.  If a task fails executing (the worker returns with
-       an error), its :attr:`retries` attribute is checked.  If it is
-       equal to 0, the task is marked "erred".  If it is greater than 0,
-       the :attr:`retries` attribute is decremented and execution is
-       attempted again.
+           The number of times this task can automatically be retried in case
+           of failure.  If a task fails executing (the worker returns with
+           an error), its :attr:`retries` attribute is checked.  If it is
+           equal to 0, the task is marked "erred".  If it is greater than 0,
+           the :attr:`retries` attribute is decremented and execution is
+           attempted again.
 
-    .. attribute:: nbytes: int (or None)
+        .. attribute:: nbytes: int (or None)
 
-       The number of bytes, as determined by ``sizeof``, of the result
-       of a finished task.  This number is used for diagnostics and to
-       help prioritize work.
+           The number of bytes, as determined by ``sizeof``, of the result
+           of a finished task.  This number is used for diagnostics and to
+           help prioritize work.
 
-    .. attribute:: type: str
+        .. attribute:: type: str
 
-       The type of the object as a string.  Only present for tasks that have
-       been computed.
+           The type of the object as a string.  Only present for tasks that have
+           been computed.
 
-    .. attribute:: exception: object
+        .. attribute:: exception: object
 
-       If this task failed executing, the exception object is stored here.
-       Otherwise this is ``None``.
+           If this task failed executing, the exception object is stored here.
+           Otherwise this is ``None``.
 
-    .. attribute:: traceback: object
+        .. attribute:: traceback: object
 
-       If this task failed executing, the traceback object is stored here.
-       Otherwise this is ``None``.
+           If this task failed executing, the traceback object is stored here.
+           Otherwise this is ``None``.
 
-    .. attribute:: exception_blame: TaskState (or None)
+        .. attribute:: exception_blame: TaskState (or None)
 
-       If this task or one of its dependencies failed executing, the
-       failed task is stored here (possibly itself).  Otherwise this
-       is ``None``.
+           If this task or one of its dependencies failed executing, the
+           failed task is stored here (possibly itself).  Otherwise this
+           is ``None``.
 
-    .. attribute:: suspicious: int
+        .. attribute:: suspicious: int
 
-       The number of times this task has been involved in a worker death.
+           The number of times this task has been involved in a worker death.
 
-       Some tasks may cause workers to die (such as calling ``os._exit(0)``).
-       When a worker dies, all of the tasks on that worker are reassigned
-       to others.  This combination of behaviors can cause a bad task to
-       catastrophically destroy all workers on the cluster, one after
-       another.  Whenever a worker dies, we mark each task currently
-       processing on that worker (as recorded by
-       :attr:`WorkerState.processing`) as suspicious.
+           Some tasks may cause workers to die (such as calling ``os._exit(0)``).
+           When a worker dies, all of the tasks on that worker are reassigned
+           to others.  This combination of behaviors can cause a bad task to
+           catastrophically destroy all workers on the cluster, one after
+           another.  Whenever a worker dies, we mark each task currently
+           processing on that worker (as recorded by
+           :attr:`WorkerState.processing`) as suspicious.
 
-       If a task is involved in three deaths (or some other fixed constant)
-       then we mark the task as ``erred``.
+           If a task is involved in three deaths (or some other fixed constant)
+           then we mark the task as ``erred``.
 
-    .. attribute:: host_restrictions: {hostnames}
+        .. attribute:: host_restrictions: {hostnames}
 
-       A set of hostnames where this task can be run (or ``None`` if empty).
-       Usually this is empty unless the task has been specifically restricted
-       to only run on certain hosts.  A hostname may correspond to one or
-       several connected workers.
+           A set of hostnames where this task can be run (or ``None`` if empty).
+           Usually this is empty unless the task has been specifically restricted
+           to only run on certain hosts.  A hostname may correspond to one or
+           several connected workers.
 
-    .. attribute:: worker_restrictions: {worker addresses}
+        .. attribute:: worker_restrictions: {worker addresses}
 
-       A set of complete worker addresses where this can be run (or ``None``
-       if empty).  Usually this is empty unless the task has been specifically
-       restricted to only run on certain workers.
+           A set of complete worker addresses where this can be run (or ``None``
+           if empty).  Usually this is empty unless the task has been specifically
+           restricted to only run on certain workers.
 
-       Note this is tracking worker addresses, not worker states, since
-       the specific workers may not be connected at this time.
+           Note this is tracking worker addresses, not worker states, since
+           the specific workers may not be connected at this time.
 
-    .. attribute:: resource_restrictions: {resource: quantity}
+        .. attribute:: resource_restrictions: {resource: quantity}
 
-       Resources required by this task, such as ``{'gpu': 1}`` or
-       ``{'memory': 1e9}`` (or ``None`` if empty).  These are user-defined
-       names and are matched against the contents of each
-       :attr:`WorkerState.resources` dictionary.
+           Resources required by this task, such as ``{'gpu': 1}`` or
+           ``{'memory': 1e9}`` (or ``None`` if empty).  These are user-defined
+           names and are matched against the contents of each
+           :attr:`WorkerState.resources` dictionary.
 
-    .. attribute:: loose_restrictions: bool
+        .. attribute:: loose_restrictions: bool
 
-       If ``False``, each of :attr:`host_restrictions`,
-       :attr:`worker_restrictions` and :attr:`resource_restrictions` is
-       a hard constraint: if no worker is available satisfying those
-       restrictions, the task cannot go into the "processing" state and
-       will instead go into the "no-worker" state.
+           If ``False``, each of :attr:`host_restrictions`,
+           :attr:`worker_restrictions` and :attr:`resource_restrictions` is
+           a hard constraint: if no worker is available satisfying those
+           restrictions, the task cannot go into the "processing" state and
+           will instead go into the "no-worker" state.
 
-       If ``True``, the above restrictions are mere preferences: if no worker
-       is available satisfying those restrictions, the task can still go
-       into the "processing" state and be sent for execution to another
-       connected worker.
+           If ``True``, the above restrictions are mere preferences: if no worker
+           is available satisfying those restrictions, the task can still go
+           into the "processing" state and be sent for execution to another
+           connected worker.
 
-    .. attribute: actor: bool
+        .. attribute: actor: bool
 
-       Whether or not this task is an Actor.
+           Whether or not this task is an Actor.
 
-    .. attribute: group: TaskGroup
+        .. attribute: group: TaskGroup
 
-:      The group of tasks to which this one belongs.
+    :      The group of tasks to which this one belongs.
     """
 
     __slots__ = (
@@ -729,7 +729,7 @@ def validate(self):
 
 
 class TaskGroup:
-    """ Collection tracking all tasks within a group
+    """Collection tracking all tasks within a group
 
     Keys often have a structure like ``("x-123", 0)``
     A group takes the first section, like ``"x-123"``
@@ -799,7 +799,7 @@ def __len__(self):
 
 
 class TaskPrefix:
-    """ Collection tracking all tasks within a group
+    """Collection tracking all tasks within a group
 
     Keys often have a structure like ``("x-123", 0)``
     A group takes the first section, like ``"x"``
@@ -999,7 +999,7 @@ def _task_key_or_none(task):
 
 
 class Scheduler(ServerNode):
-    """ Dynamic distributed task scheduler
+    """Dynamic distributed task scheduler
 
     The scheduler tracks the current state of workers, data, and computations.
     The scheduler listens for events and responds by controlling workers
@@ -1144,7 +1144,7 @@ def __init__(
         assert isinstance(self.security, Security)
         self.connection_args = self.security.get_connection_args("scheduler")
         self.connection_args["handshake_overrides"] = {  # common denominator
-            "pickle-protocol": 4,
+            "pickle-protocol": 4
         }
 
         self._start_address = addresses_from_user_args(
@@ -1167,7 +1167,7 @@ def __init__(
                 missing_bokeh = True
                 http_server_modules.append("distributed.http.scheduler.missing_bokeh")
         routes = get_handlers(
-            server=self, modules=http_server_modules, prefix=http_prefix,
+            server=self, modules=http_server_modules, prefix=http_prefix
         )
         self.start_http_server(routes, dashboard_address, default_port=8787)
         if show_dashboard and not missing_bokeh:
@@ -1522,7 +1522,7 @@ def del_scheduler_file():
         return self
 
     async def close(self, comm=None, fast=False, close_workers=False):
-        """ Send cleanup signal to all coroutines then wait until finished
+        """Send cleanup signal to all coroutines then wait until finished
 
         See Also
         --------
@@ -1586,7 +1586,7 @@ async def close(self, comm=None, fast=False, close_workers=False):
         disable_gc_diagnosis()
 
     async def close_worker(self, comm=None, worker=None, safe=None):
-        """ Remove a worker from the cluster
+        """Remove a worker from the cluster
 
         This both removes the worker from our local state and also sends a
         signal to the worker to shut down.  This works regardless of whether or
@@ -2257,7 +2257,7 @@ async def remove_worker(self, comm=None, address=None, safe=False, close=True):
                     if ts.suspicious > self.allowed_failures:
                         del recommendations[k]
                         e = pickle.dumps(
-                            KilledWorker(task=k, last_worker=ws.clean()), protocol=4,
+                            KilledWorker(task=k, last_worker=ws.clean()), protocol=4
                         )
                         r = self.transition(k, "erred", exception=e, cause=k)
                         recommendations.update(r)
@@ -2544,7 +2544,7 @@ def report(self, msg, ts=None, client=None):
                     logger.critical("Tried writing to closed comm: %s", msg)
 
     async def add_client(self, comm, client=None, versions=None):
-        """ Add client to network
+        """Add client to network
 
         We listen to all future messages from this Comm.
         """
@@ -2723,7 +2723,7 @@ def release_worker_data(self, comm=None, keys=None, worker=None):
             self.transitions(recommendations)
 
     def handle_long_running(self, key=None, worker=None, compute_duration=None):
-        """ A task has seceded from the thread pool
+        """A task has seceded from the thread pool
 
         We stop the task from being stolen in the future, and change task
         duration accounting as if the task has stopped.
@@ -2792,7 +2792,7 @@ def remove_plugin(self, plugin):
         self.plugins.remove(plugin)
 
     def worker_send(self, worker, msg):
-        """ Send message to worker
+        """Send message to worker
 
         This also handles connection failures by adding a callback to remove
         the worker on the next cycle.
@@ -2815,7 +2815,7 @@ async def scatter(
         broadcast=False,
         timeout=2,
     ):
-        """ Send data out to workers
+        """Send data out to workers
 
         See also
         --------
@@ -3044,7 +3044,7 @@ async def proxy(self, comm=None, msg=None, worker=None, serializers=None):
         return d[worker]
 
     async def _delete_worker_data(self, worker_address, keys):
-        """ Delete data from a worker and update the corresponding worker/task states
+        """Delete data from a worker and update the corresponding worker/task states
 
         Parameters
         ----------
@@ -3066,7 +3066,7 @@ async def _delete_worker_data(self, worker_address, keys):
         self.log_event(ws.address, {"action": "remove-worker-data", "keys": keys})
 
     async def rebalance(self, comm=None, keys=None, workers=None):
-        """ Rebalance keys so that each worker stores roughly equal bytes
+        """Rebalance keys so that each worker stores roughly equal bytes
 
         **Policy**
 
@@ -3202,7 +3202,7 @@ async def replicate(
         delete=True,
         lock=True,
     ):
-        """ Replicate data throughout cluster
+        """Replicate data throughout cluster
 
         This performs a tree copy of the data throughout the network
         individually on each piece of data.
@@ -3445,7 +3445,7 @@ async def retire_workers(
         lock=True,
         **kwargs,
     ):
-        """ Gracefully retire workers from cluster
+        """Gracefully retire workers from cluster
 
         Parameters
         ----------
@@ -3794,7 +3794,7 @@ def get_task_duration(self, ts, default=None):
         return duration
 
     def run_function(self, stream, function, args=(), kwargs={}, wait=True):
-        """ Run a function within this process
+        """Run a function within this process
 
         See Also
         --------
@@ -4679,7 +4679,7 @@ def transition_released_forgotten(self, key):
             raise
 
     def transition(self, key, finish, *args, **kwargs):
-        """ Transition a key from its current state to the finish state
+        """Transition a key from its current state to the finish state
 
         Examples
         --------
@@ -4771,7 +4771,7 @@ def transition(self, key, finish, *args, **kwargs):
             raise
 
     def transitions(self, recommendations):
-        """ Process transitions until none are left
+        """Process transitions until none are left
 
         This includes feedback from previous transitions and continues until we
         reach a steady state
@@ -4798,7 +4798,7 @@ def story(self, *keys):
     transition_story = story
 
     def reschedule(self, key=None, worker=None):
-        """ Reschedule a task
+        """Reschedule a task
 
         Things may have shifted and this task may now be better suited to run
         elsewhere
@@ -4822,7 +4822,7 @@ def reschedule(self, key=None, worker=None):
     ##############################
 
     def check_idle_saturated(self, ws, occ=None):
-        """ Update the status of the idle and saturated state
+        """Update the status of the idle and saturated state
 
         The scheduler keeps track of workers that are ..
 
@@ -4857,7 +4857,7 @@ def check_idle_saturated(self, ws, occ=None):
                 self.saturated.discard(ws)
 
     def valid_workers(self, ts):
-        """ Return set of currently valid workers for key
+        """Return set of currently valid workers for key
 
         If all workers are valid then this returns ``True``.
         This checks tracks the following state:
@@ -5237,7 +5237,7 @@ async def get_worker_logs(self, comm=None, n=None, workers=None, nanny=False):
     ###########
 
     def reevaluate_occupancy(self, worker_index=0):
-        """ Periodically reassess task duration time
+        """Periodically reassess task duration time
 
         The expected duration of a task can change over time.  Unfortunately we
         don't have a good constant-time way to propagate the effects of these
@@ -5335,7 +5335,7 @@ def check_idle(self):
             self.loop.add_callback(self.close)
 
     def adaptive_target(self, comm=None, target_duration=None):
-        """ Desired number of workers based on the current workload
+        """Desired number of workers based on the current workload
 
         This looks at the current running tasks and memory use, and returns a
         number of desired workers.  This is often used by adaptive scheduling.
diff --git a/distributed/semaphore.py b/distributed/semaphore.py
index b1c3c029177..5d86bd8ea59 100644
--- a/distributed/semaphore.py
+++ b/distributed/semaphore.py
@@ -35,7 +35,7 @@ def leftover(self):
 
 
 class SemaphoreExtension:
-    """ An extension for the scheduler to manage Semaphores
+    """An extension for the scheduler to manage Semaphores
 
     This adds the following routes to the scheduler
 
@@ -267,7 +267,7 @@ def close(self, comm=None, name=None):
 
 
 class Semaphore:
-    """ Semaphore
+    """Semaphore
 
     This `semaphore <https://en.wikipedia.org/wiki/Semaphore_(programming)>`_
     will track leases on the scheduler which can be acquired and
diff --git a/distributed/sizeof.py b/distributed/sizeof.py
index bc51b3603ae..699db3e8fb3 100644
--- a/distributed/sizeof.py
+++ b/distributed/sizeof.py
@@ -6,7 +6,7 @@
 
 
 def safe_sizeof(obj, default_size=1e6):
-    """ Safe variant of sizeof that captures and logs exceptions
+    """Safe variant of sizeof that captures and logs exceptions
 
     This returns a default size of 1e6 if the sizeof function fails
     """
diff --git a/distributed/stealing.py b/distributed/stealing.py
index 88c15809ea7..0ebffcba6d1 100644
--- a/distributed/stealing.py
+++ b/distributed/stealing.py
@@ -109,7 +109,7 @@ def remove_key_from_stealable(self, ts):
             pass
 
     def steal_time_ratio(self, ts):
-        """ The compute to communication time ratio of a key
+        """The compute to communication time ratio of a key
 
         Returns
         -------
diff --git a/distributed/tests/test_asyncprocess.py b/distributed/tests/test_asyncprocess.py
index 3923d81cf2c..f5cfab17895 100644
--- a/distributed/tests/test_asyncprocess.py
+++ b/distributed/tests/test_asyncprocess.py
@@ -339,10 +339,10 @@ def _worker_process(worker_ready, child_pipe):
 
 
 def _parent_process(child_pipe):
-    """ Simulate starting an AsyncProcess and then dying.
+    """Simulate starting an AsyncProcess and then dying.
 
     The child_alive pipe is held open for as long as the child is alive, and can
-    be used to determine if it exited correctly. """
+    be used to determine if it exited correctly."""
 
     async def parent_process_coroutine():
         worker_ready = mp_context.Event()
@@ -369,7 +369,7 @@ async def parent_process_coroutine():
 
 
 def test_asyncprocess_child_teardown_on_parent_exit():
-    r""" Check that a child process started by AsyncProcess exits if its parent
+    r"""Check that a child process started by AsyncProcess exits if its parent
     exits.
 
     The motivation is to ensure that if an AsyncProcess is created and the
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 21fc3b54cd3..39580bc6857 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -58,14 +58,7 @@
 from distributed.metrics import time
 from distributed.scheduler import Scheduler, KilledWorker
 from distributed.sizeof import sizeof
-from distributed.utils import (
-    mp_context,
-    sync,
-    tmp_text,
-    tokey,
-    tmpfile,
-    is_valid_xml,
-)
+from distributed.utils import mp_context, sync, tmp_text, tokey, tmpfile, is_valid_xml
 from distributed.utils_test import (
     cluster,
     slowinc,
diff --git a/distributed/tests/test_preload.py b/distributed/tests/test_preload.py
index 0000cd94206..2c35a59e06c 100644
--- a/distributed/tests/test_preload.py
+++ b/distributed/tests/test_preload.py
@@ -69,7 +69,7 @@ def dask_teardown(worker):
     worker.foo = 'teardown'
 """
     with dask.config.set(
-        {"distributed.worker.preload": text, "distributed.nanny.preload": text,}
+        {"distributed.worker.preload": text, "distributed.nanny.preload": text}
     ):
         async with Scheduler(port=0) as s:
             async with Nanny(s.address) as w:
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index dc2ac7d8217..00d6631c9c5 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -531,7 +531,7 @@ async def test_broadcast(s, a, b):
     assert result == {a.address: b"pong", b.address: b"pong"}
 
 
-@gen_cluster(security=tls_only_security(),)
+@gen_cluster(security=tls_only_security())
 async def test_broadcast_tls(s, a, b):
     result = await s.broadcast(msg={"op": "ping"})
     assert result == {a.address: b"pong", b.address: b"pong"}
@@ -1918,8 +1918,7 @@ async def test_task_group_non_tuple_key(c, s, a, b):
 
 @gen_cluster(client=True)
 async def test_task_unique_groups(c, s, a, b):
-    """ This test ensure that task groups remain unique when using submit
-    """
+    """This test ensure that task groups remain unique when using submit"""
     x = c.submit(sum, [1, 2])
     y = c.submit(len, [1, 2])
     z = c.submit(sum, [3, 4])
@@ -2044,7 +2043,7 @@ def reducer(x, y):
     s.rpc = await FlakyConnectionPool(failing_connections=4)
 
     with dask.config.set(
-        {"distributed.comm.retry.delay_min": 0.5, "distributed.comm.retry.count": 3,}
+        {"distributed.comm.retry.delay_min": 0.5, "distributed.comm.retry.count": 3}
     ):
         with captured_logger(
             logging.getLogger("distributed.scheduler")
diff --git a/distributed/tests/test_security.py b/distributed/tests/test_security.py
index ab6646f0a12..dc1aee9e9b6 100644
--- a/distributed/tests/test_security.py
+++ b/distributed/tests/test_security.py
@@ -285,13 +285,11 @@ async def handle_comm(comm):
 
         # No SSL context for client
         with pytest.raises(TypeError):
-            await connect(
-                listener.contact_address, **sec.get_connection_args("client"),
-            )
+            await connect(listener.contact_address, **sec.get_connection_args("client"))
 
         # Check forced cipher
         comm = await connect(
-            listener.contact_address, **forced_cipher_sec.get_connection_args("worker"),
+            listener.contact_address, **forced_cipher_sec.get_connection_args("worker")
         )
         cipher, _, _ = comm.extra_info["cipher"]
         assert cipher in [FORCED_CIPHER] + TLS_13_CIPHERS
@@ -325,7 +323,7 @@ async def handle_comm(comm):
             listen_addr, handle_comm, **sec.get_listen_args("scheduler")
         ) as listener:
             comm = await connect(
-                listener.contact_address, **sec2.get_connection_args("worker"),
+                listener.contact_address, **sec2.get_connection_args("worker")
             )
             comm.abort()
 
@@ -333,7 +331,7 @@ async def handle_comm(comm):
             listen_addr, handle_comm, **sec2.get_listen_args("scheduler")
         ) as listener:
             comm = await connect(
-                listener.contact_address, **sec2.get_connection_args("worker"),
+                listener.contact_address, **sec2.get_connection_args("worker")
             )
             comm.abort()
 
@@ -348,19 +346,17 @@ def check_encryption_error():
             listen_addr, handle_comm, **sec.get_listen_args("scheduler")
         ) as listener:
             comm = await connect(
-                listener.contact_address, **sec.get_connection_args("worker"),
+                listener.contact_address, **sec.get_connection_args("worker")
             )
             comm.abort()
 
             with pytest.raises(RuntimeError):
                 await connect(
-                    listener.contact_address, **sec2.get_connection_args("worker"),
+                    listener.contact_address, **sec2.get_connection_args("worker")
                 )
 
         with pytest.raises(RuntimeError):
-            listen(
-                listen_addr, handle_comm, **sec2.get_listen_args("scheduler"),
-            )
+            listen(listen_addr, handle_comm, **sec2.get_listen_args("scheduler"))
 
 
 def test_temporary_credentials():
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index c13fed73ab5..796eca82e3f 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -30,12 +30,7 @@
 from distributed.core import rpc, CommClosedError, Status
 from distributed.scheduler import Scheduler
 from distributed.metrics import time
-from distributed.worker import (
-    Worker,
-    error_message,
-    logger,
-    parse_memory_limit,
-)
+from distributed.worker import Worker, error_message, logger, parse_memory_limit
 from distributed.utils import tmpfile, TimeoutError
 from distributed.utils_test import (  # noqa: F401
     cleanup,
diff --git a/distributed/threadpoolexecutor.py b/distributed/threadpoolexecutor.py
index 44770900028..c3112b3056b 100644
--- a/distributed/threadpoolexecutor.py
+++ b/distributed/threadpoolexecutor.py
@@ -106,7 +106,7 @@ def shutdown(self, wait=True, timeout=None):
 
 
 def secede(adjust=True):
-    """ Have this thread secede from the ThreadPoolExecutor
+    """Have this thread secede from the ThreadPoolExecutor
 
     See Also
     --------
@@ -120,7 +120,7 @@ def secede(adjust=True):
 
 
 def rejoin():
-    """ Have this thread rejoin the ThreadPoolExecutor
+    """Have this thread rejoin the ThreadPoolExecutor
 
     This will block until a new slot opens up in the executor.  The next thread
     to finish a task will leave the pool to allow this one to join.
diff --git a/distributed/utils.py b/distributed/utils.py
index fb4ad8f9015..237b19ea37b 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -195,7 +195,7 @@ def get_ip_interface(ifname):
 # FIXME: this breaks if changed to async def...
 @gen.coroutine
 def ignore_exceptions(coroutines, *exceptions):
-    """ Process list of coroutines, ignoring certain exceptions
+    """Process list of coroutines, ignoring certain exceptions
 
     >>> coroutines = [cor(...) for ...]  # doctest: +SKIP
     >>> x = yield ignore_exceptions(coroutines, TypeError)  # doctest: +SKIP
@@ -210,7 +210,7 @@ def ignore_exceptions(coroutines, *exceptions):
 
 
 async def All(args, quiet_exceptions=()):
-    """ Wait on many tasks at the same time
+    """Wait on many tasks at the same time
 
     Err once any of the tasks err.
 
@@ -231,7 +231,7 @@ async def All(args, quiet_exceptions=()):
 
             @gen.coroutine
             def quiet():
-                """ Watch unfinished tasks
+                """Watch unfinished tasks
 
                 Otherwise if they err they get logged in a way that is hard to
                 control.  They need some other task to watch them so that they
@@ -250,7 +250,7 @@ def quiet():
 
 
 async def Any(args, quiet_exceptions=()):
-    """ Wait on many tasks at the same time and return when any is finished
+    """Wait on many tasks at the same time and return when any is finished
 
     Err once any of the tasks err.
 
@@ -269,7 +269,7 @@ async def Any(args, quiet_exceptions=()):
 
             @gen.coroutine
             def quiet():
-                """ Watch unfinished tasks
+                """Watch unfinished tasks
 
                 Otherwise if they err they get logged in a way that is hard to
                 control.  They need some other task to watch them so that they
@@ -544,7 +544,7 @@ def clear_queue(q):
 
 
 def is_kernel():
-    """ Determine if we're running within an IPython kernel
+    """Determine if we're running within an IPython kernel
 
     >>> is_kernel()
     False
@@ -687,7 +687,7 @@ def silence_logging(level, root="distributed"):
 
 @toolz.memoize
 def ensure_ip(hostname):
-    """ Ensure that address is an IP address
+    """Ensure that address is an IP address
 
     Examples
     --------
@@ -741,7 +741,7 @@ def truncate_exception(e, n=10000):
 
 
 def tokey(o):
-    """ Convert an object to a string.
+    """Convert an object to a string.
 
     Examples
     --------
@@ -761,8 +761,7 @@ def tokey(o):
 
 
 def validate_key(k):
-    """Validate a key as received on a stream.
-    """
+    """Validate a key as received on a stream."""
     typ = type(k)
     if typ is not str and typ is not bytes:
         raise TypeError("Unexpected key type %s (value: %r)" % (typ, k))
@@ -799,7 +798,7 @@ def str_graph(dsk, extra_values=()):
 
 
 def seek_delimiter(file, delimiter, blocksize):
-    """ Seek current file to next byte after a delimiter bytestring
+    """Seek current file to next byte after a delimiter bytestring
 
     This seeks the file to the next byte following the delimiter.  It does
     not return anything.  Use ``file.tell()`` to see location afterwards.
@@ -832,7 +831,7 @@ def seek_delimiter(file, delimiter, blocksize):
 
 
 def read_block(f, offset, length, delimiter=None):
-    """ Read a block of bytes from a file
+    """Read a block of bytes from a file
 
     Parameters
     ----------
@@ -1000,7 +999,7 @@ def shutting_down(l=_shutting_down):
 
 
 def open_port(host=""):
-    """ Return a probably-open port
+    """Return a probably-open port
 
     There is a chance that this port will be taken by the operating system soon
     after returning from this function.
@@ -1173,7 +1172,7 @@ def clear_all_instances(cls):
 
 
 def reset_logger_locks():
-    """ Python 2's logger's locks don't survive a fork event
+    """Python 2's logger's locks don't survive a fork event
 
     https://github.com/dask/distributed/issues/1491
     """
@@ -1301,7 +1300,7 @@ def warn_on_duration(duration, msg):
 
 
 def typename(typ):
-    """ Return name of type
+    """Return name of type
 
     Examples
     --------
@@ -1327,7 +1326,7 @@ def format_dashboard_link(host, port):
 
 
 def parse_ports(port):
-    """ Parse input port information into list of ports
+    """Parse input port information into list of ports
 
     Parameters
     ----------
@@ -1412,7 +1411,7 @@ def _repr_html_(self):
 
 
 def cli_keywords(d: dict, cls=None, cmd=None):
-    """ Convert a kwargs dictionary into a list of CLI keywords
+    """Convert a kwargs dictionary into a list of CLI keywords
 
     Parameters
     ----------
@@ -1481,7 +1480,7 @@ def is_valid_xml(text):
 
 
 def import_term(name: str):
-    """ Return the fully qualified term
+    """Return the fully qualified term
 
     Examples
     --------
@@ -1503,7 +1502,7 @@ async def offload(fn, *args, **kwargs):
 
 
 def serialize_for_cli(data):
-    """ Serialize data into a string that can be passthrough cli
+    """Serialize data into a string that can be passthrough cli
 
     Parameters
     ----------
@@ -1518,7 +1517,7 @@ def serialize_for_cli(data):
 
 
 def deserialize_for_cli(data):
-    """ De-serialize data into the original object
+    """De-serialize data into the original object
 
     Parameters
     ----------
@@ -1550,8 +1549,7 @@ async def __aexit__(self, *args):
 
 
 class LRU(UserDict):
-    """ Limited size mapping, evicting the least recently looked-up key when full
-    """
+    """Limited size mapping, evicting the least recently looked-up key when full"""
 
     def __init__(self, maxsize):
         super().__init__()
diff --git a/distributed/utils_comm.py b/distributed/utils_comm.py
index b7e33656ab8..814ebb7bce8 100644
--- a/distributed/utils_comm.py
+++ b/distributed/utils_comm.py
@@ -17,7 +17,7 @@
 
 
 async def gather_from_workers(who_has, rpc, close=True, serializers=None, who=None):
-    """ Gather data directly from peers
+    """Gather data directly from peers
 
     Parameters
     ----------
@@ -97,7 +97,7 @@ async def gather_from_workers(who_has, rpc, close=True, serializers=None, who=No
 
 
 class WrappedKey:
-    """ Interface for a key in a dask graph.
+    """Interface for a key in a dask graph.
 
     Subclasses must have .key attribute that refers to a key in a dask graph.
 
@@ -118,7 +118,7 @@ def __repr__(self):
 
 
 async def scatter_to_workers(nthreads, data, rpc=rpc, report=True, serializers=None):
-    """ Scatter data directly to workers
+    """Scatter data directly to workers
 
     This distributes data in a round-robin fashion to a set of workers based on
     how many cores they have.  nthreads should be a dictionary mapping worker
@@ -164,7 +164,7 @@ async def scatter_to_workers(nthreads, data, rpc=rpc, report=True, serializers=N
 
 
 def unpack_remotedata(o, byte_keys=False, myset=None):
-    """ Unpack WrappedKey objects from collection
+    """Unpack WrappedKey objects from collection
 
     Returns original collection and set of all found WrappedKey objects
 
@@ -246,7 +246,7 @@ def unpack_remotedata(o, byte_keys=False, myset=None):
 
 
 def pack_data(o, d, key_types=object):
-    """ Merge known data into tuple or dict
+    """Merge known data into tuple or dict
 
     Parameters
     ----------
@@ -281,7 +281,7 @@ def pack_data(o, d, key_types=object):
 
 
 def subs_multiple(o, d):
-    """ Perform substitutions on a tasks
+    """Perform substitutions on a tasks
 
     Parameters
     ----------
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index 4daf84e804c..1a373aeac2c 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -741,7 +741,7 @@ async def disconnect_all(addresses, timeout=3, rpc_kwargs=None):
 
 
 def gen_test(timeout=10):
-    """ Coroutine test
+    """Coroutine test
 
     @gen_test(timeout=5)
     async def test_foo():
@@ -1137,9 +1137,7 @@ async def assert_can_connect_from_everywhere_4_6(port, protocol="tcp", **kwargs)
     await asyncio.gather(*futures)
 
 
-async def assert_can_connect_from_everywhere_4(
-    port, protocol="tcp", **kwargs,
-):
+async def assert_can_connect_from_everywhere_4(port, protocol="tcp", **kwargs):
     """
     Check that the local *port* is reachable from all IPv4 addresses.
     """
@@ -1205,8 +1203,7 @@ async def assert_can_connect_locally_6(port, **kwargs):
 
 @contextmanager
 def captured_logger(logger, level=logging.INFO, propagate=None):
-    """Capture output from the given Logger.
-    """
+    """Capture output from the given Logger."""
     if isinstance(logger, str):
         logger = logging.getLogger(logger)
     orig_level = logger.level
@@ -1228,8 +1225,7 @@ def captured_logger(logger, level=logging.INFO, propagate=None):
 
 @contextmanager
 def captured_handler(handler):
-    """Capture output from the given logging.StreamHandler.
-    """
+    """Capture output from the given logging.StreamHandler."""
     assert isinstance(handler, logging.StreamHandler)
     orig_stream = handler.stream
     handler.stream = io.StringIO()
diff --git a/distributed/variable.py b/distributed/variable.py
index b20273031ab..26a56466588 100644
--- a/distributed/variable.py
+++ b/distributed/variable.py
@@ -14,7 +14,7 @@
 
 
 class VariableExtension:
-    """ An extension for the scheduler to manage queues
+    """An extension for the scheduler to manage queues
 
     This adds the following routes to the scheduler
 
@@ -123,7 +123,7 @@ async def delete(self, comm=None, name=None, client=None):
 
 
 class Variable:
-    """ Distributed Global Variable
+    """Distributed Global Variable
 
     This allows multiple clients to share futures and data between each other
     with a single mutable variable.  All metadata is sequentialized through the
@@ -176,7 +176,7 @@ async def _set(self, value):
             await self.client.scheduler.variable_set(data=value, name=self.name)
 
     def set(self, value, **kwargs):
-        """ Set the value of this variable
+        """Set the value of this variable
 
         Parameters
         ----------
@@ -206,7 +206,7 @@ async def _get(self, timeout=None):
         return value
 
     def get(self, timeout=None, **kwargs):
-        """ Get the value of this variable
+        """Get the value of this variable
 
         Parameters
         ----------
@@ -219,7 +219,7 @@ def get(self, timeout=None, **kwargs):
         return self.client.sync(self._get, timeout=timeout, **kwargs)
 
     def delete(self):
-        """ Delete this variable
+        """Delete this variable
 
         Caution, this affects all clients currently pointing to this variable.
         """
diff --git a/distributed/versions.py b/distributed/versions.py
index 4e4bff35067..a276eb418ca 100644
--- a/distributed/versions.py
+++ b/distributed/versions.py
@@ -33,7 +33,7 @@
 
 # notes to be displayed for mismatch packages
 notes_mismatch_package = {
-    "msgpack": "Variation is ok, as long as everything is above 0.6",
+    "msgpack": "Variation is ok, as long as everything is above 0.6"
 }
 
 
diff --git a/distributed/worker.py b/distributed/worker.py
index 4c3be973478..848b00d564d 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -92,7 +92,7 @@
 
 
 class Worker(ServerNode):
-    """ Worker node in a Dask distributed cluster
+    """Worker node in a Dask distributed cluster
 
     Workers perform two functions:
 
@@ -645,7 +645,7 @@ def __init__(
         pc = PeriodicCallback(self.heartbeat, 1000)
         self.periodic_callbacks["heartbeat"] = pc
         pc = PeriodicCallback(
-            lambda: self.batched_stream.send({"op": "keep-alive"}), 60000,
+            lambda: self.batched_stream.send({"op": "keep-alive"}), 60000
         )
         self.periodic_callbacks["keep-alive"] = pc
 
@@ -657,7 +657,7 @@ def __init__(
         if self.memory_limit:
             self._memory_monitoring = False
             pc = PeriodicCallback(
-                self.memory_monitor, self.memory_monitor_interval * 1000,
+                self.memory_monitor, self.memory_monitor_interval * 1000
             )
             self.periodic_callbacks["memory"] = pc
 
@@ -706,20 +706,17 @@ def __init__(
     ##################
 
     def __repr__(self):
-        return (
-            "<%s: %r, %s, %s, stored: %d, running: %d/%d, ready: %d, comm: %d, waiting: %d>"
-            % (
-                self.__class__.__name__,
-                self.address,
-                self.name,
-                self.status,
-                len(self.data),
-                len(self.executing),
-                self.nthreads,
-                len(self.ready),
-                len(self.in_flight_tasks),
-                len(self.waiting_for_data),
-            )
+        return "<%s: %r, %s, %s, stored: %d, running: %d/%d, ready: %d, comm: %d, waiting: %d>" % (
+            self.__class__.__name__,
+            self.address,
+            self.name,
+            self.status,
+            len(self.data),
+            len(self.executing),
+            self.nthreads,
+            len(self.ready),
+            len(self.in_flight_tasks),
+            len(self.waiting_for_data),
         )
 
     @property
@@ -1180,7 +1177,7 @@ async def close(
         return "OK"
 
     async def close_gracefully(self):
-        """ Gracefully shut down a worker
+        """Gracefully shut down a worker
 
         This first informs the scheduler that we're shutting down, and asks it
         to move our data elsewhere.  Afterwards, we close as normal
@@ -2336,7 +2333,7 @@ def rescind_key(self, key):
     # xref: https://github.com/dask/distributed/issues/3938
     @gen.coroutine
     def executor_submit(self, key, function, args=(), kwargs=None, executor=None):
-        """ Safely run function in thread pool executor
+        """Safely run function in thread pool executor
 
         We've run into issues running concurrent.future futures within
         tornado.  Apparently it's advantageous to use timeouts and periodic
@@ -2616,7 +2613,7 @@ async def execute(self, key, report=False):
     ##################
 
     async def memory_monitor(self):
-        """ Track this process's memory usage and act accordingly
+        """Track this process's memory usage and act accordingly
 
         If we rise above 70% memory use, start dumping data to disk.
 
@@ -2988,7 +2985,7 @@ def client(self):
                 return self._get_client()
 
     def _get_client(self, timeout=3):
-        """ Get local client attached to this worker
+        """Get local client attached to this worker
 
         If no such client exists, create one
 
@@ -3029,7 +3026,7 @@ def _get_client(self, timeout=3):
         return self._client
 
     def get_current_task(self):
-        """ Get the key of the task we are currently running
+        """Get the key of the task we are currently running
 
         This only makes sense to run within a task
 
@@ -3051,7 +3048,7 @@ def get_current_task(self):
 
 
 def get_worker():
-    """ Get the worker currently running this task
+    """Get the worker currently running this task
 
     Examples
     --------
@@ -3172,7 +3169,7 @@ def secede():
 
 
 class Reschedule(Exception):
-    """ Reschedule this task
+    """Reschedule this task
 
     Raising this exception will stop the current execution of the task and ask
     the scheduler to reschedule this task, possibly on a different machine.
@@ -3215,7 +3212,7 @@ async def get_data_from_worker(
     serializers=None,
     deserializers=None,
 ):
-    """ Get keys from worker
+    """Get keys from worker
 
     The worker has a two step handshake to acknowledge when data has been fully
     delivered.  This function implements that handshake.
@@ -3294,7 +3291,7 @@ def _deserialize(function=None, args=None, kwargs=None, task=no_value):
 
 
 def execute_task(task):
-    """ Evaluate a nested task
+    """Evaluate a nested task
 
     >>> inc = lambda x: x + 1
     >>> execute_task((inc, 1))
@@ -3332,7 +3329,7 @@ def dumps_function(func):
 
 
 def dumps_task(task):
-    """ Serialize a dask task
+    """Serialize a dask task
 
     Returns a dict of bytestrings that can each be loaded with ``loads``
 
@@ -3397,7 +3394,7 @@ def apply_function(
     active_threads_lock,
     time_delay,
 ):
-    """ Run a function, collect information
+    """Run a function, collect information
 
     Returns
     -------
@@ -3437,7 +3434,7 @@ def apply_function(
 def apply_function_actor(
     function, args, kwargs, execution_state, key, active_threads, active_threads_lock
 ):
-    """ Run a function, collect information
+    """Run a function, collect information
 
     Returns
     -------
@@ -3460,7 +3457,7 @@ def apply_function_actor(
 
 
 def get_msg_safe_str(msg):
-    """ Make a worker msg, which contains args and kwargs, safe to cast to str:
+    """Make a worker msg, which contains args and kwargs, safe to cast to str:
     allowing for some arguments to raise exceptions during conversion and
     ignoring them.
     """
@@ -3482,7 +3479,7 @@ def __repr__(self):
 
 
 def convert_args_to_str(args, max_len=None):
-    """ Convert args to a string, allowing for some arguments to raise
+    """Convert args to a string, allowing for some arguments to raise
     exceptions during conversion and ignoring them.
     """
     length = 0
@@ -3501,7 +3498,7 @@ def convert_args_to_str(args, max_len=None):
 
 
 def convert_kwargs_to_str(kwargs, max_len=None):
-    """ Convert kwargs to a string, allowing for some arguments to raise
+    """Convert kwargs to a string, allowing for some arguments to raise
     exceptions during conversion and ignoring them.
     """
     length = 0
diff --git a/distributed/worker_client.py b/distributed/worker_client.py
index a45eb891f7d..b39c6236df9 100644
--- a/distributed/worker_client.py
+++ b/distributed/worker_client.py
@@ -7,7 +7,7 @@
 
 @contextmanager
 def worker_client(timeout=3, separate_thread=True):
-    """ Get client for this thread
+    """Get client for this thread
 
     This context manager is intended to be called within functions that we run
     on workers.  When run as a context manager it delivers a client

From b862f14cf338cac1c31d9f5ee604aea9c59c9935 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Fri, 28 Aug 2020 17:36:26 -0500
Subject: [PATCH 0968/1550] bump version to 2.25.0

---
 docs/source/changelog.rst | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index af5f5b2296f..9a0cda5bf1f 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,22 @@
 Changelog
 =========
 
+2.25.0 - 2020-08-28
+-------------------
+
+- Update for black (:pr:`4081`) `Tom Augspurger`_
+- Provide informative error when connecting an older version of Dask (:pr:`4076`) `Matthew Rocklin`_
+- Simplify ``pack_frames`` (:pr:`4068`) `jakirkham`_
+- Simplify ``frame_split_size`` (:pr:`4067`) `jakirkham`_
+- Use ``list.insert`` to add prelude up front (:pr:`4066`) `jakirkham`_
+- Graph helper text (:pr:`4064`) `Julia Signell`_
+- Graph dashboard: Reset container data if task number is too large (:pr:`4056`) `Florian Jetter`_
+- Ensure semaphore picks correct ``IOLoop`` for threadpool workers (:pr:`4060`) `Florian Jetter`_
+- Add cluster log method (:pr:`4051`) `Jacob Tomlinson`_
+- Cleanup more exception tracebacks (:pr:`4054`) `Krishan Bhasin`_
+- Improve documentation of ``scheduler.locks`` options (:pr:`4062`) `Florian Jetter`_
+
+
 2.24.0 - 2020-08-22
 -------------------
 

From 73e7f99e51818e268dfed203121b584fd46f933c Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sat, 29 Aug 2020 04:34:19 -0700
Subject: [PATCH 0969/1550] Add timeout= keyword to Client.wait_for_workers
 method (#4087)

---
 distributed/client.py            | 15 ++++++++++++---
 distributed/tests/test_client.py |  6 ++++++
 2 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 27cfced05d8..3284d3285a1 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -1157,15 +1157,24 @@ async def _update_scheduler_info(self):
         except EnvironmentError:
             logger.debug("Not able to query scheduler for identity")
 
-    async def _wait_for_workers(self, n_workers=0):
+    async def _wait_for_workers(self, n_workers=0, timeout=None):
         info = await self.scheduler.identity()
+        if timeout:
+            deadline = time() + parse_timedelta(timeout)
+        else:
+            deadline = None
         while n_workers and len(info["workers"]) < n_workers:
+            if deadline and time() > deadline:
+                raise TimeoutError(
+                    "Only %d/%d workers arrived after %s"
+                    % (len(info["workers"]), n_workers, timeout)
+                )
             await asyncio.sleep(0.1)
             info = await self.scheduler.identity()
 
-    def wait_for_workers(self, n_workers=0):
+    def wait_for_workers(self, n_workers=0, timeout=None):
         """Blocking call to wait for n workers before continuing"""
-        return self.sync(self._wait_for_workers, n_workers)
+        return self.sync(self._wait_for_workers, n_workers, timeout=timeout)
 
     def _heartbeat(self):
         if self.scheduler_comm:
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 39580bc6857..15005ecf837 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -5807,6 +5807,12 @@ async def test_wait_for_workers(c, s, a, b):
     assert time() < start + 1
     await w.close()
 
+    with pytest.raises(TimeoutError) as info:
+        await c.wait_for_workers(n_workers=10, timeout="1 ms")
+
+    assert "2/10" in str(info.value).replace(" ", "")
+    assert "1 ms" in str(info.value)
+
 
 @pytest.mark.skipif(WINDOWS, reason="num_fds not supported on windows")
 @pytest.mark.asyncio

From 6b259c441121c1f86f1165252b5140218a408609 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Sun, 30 Aug 2020 15:14:09 -0700
Subject: [PATCH 0970/1550] Drop `Serialized.deserialize()` method (#4073)

This method appears not to be used. So go ahead and drop it.
---
 distributed/protocol/serialize.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index 97b962a3f67..bc659735445 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -381,12 +381,6 @@ def __init__(self, header, frames):
         self.header = header
         self.frames = frames
 
-    def deserialize(self):
-        from .core import decompress
-
-        frames = decompress(self.header, self.frames)
-        return deserialize(self.header, frames)
-
     def __eq__(self, other):
         return (
             isinstance(other, Serialized)

From f761dd21e8cbba62273ab850f5f83e8a45a2fffa Mon Sep 17 00:00:00 2001
From: Tom Augspurger <TomAugspurger@users.noreply.github.com>
Date: Mon, 31 Aug 2020 09:00:48 -0500
Subject: [PATCH 0971/1550] Exclude 'fire-and-forget' client from metrics
 (#4078)

Right now we include the "fire-and-forget" client in our prometheus
metrics. I think it's best to leave that out, to only have "user"
clients in the reported metrics.
---
 distributed/http/scheduler/prometheus/__init__.py       | 2 +-
 distributed/http/scheduler/tests/test_scheduler_http.py | 7 ++++++-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/distributed/http/scheduler/prometheus/__init__.py b/distributed/http/scheduler/prometheus/__init__.py
index 6d57b41d876..c746f86f938 100644
--- a/distributed/http/scheduler/prometheus/__init__.py
+++ b/distributed/http/scheduler/prometheus/__init__.py
@@ -15,7 +15,7 @@ def collect(self):
         yield GaugeMetricFamily(
             "dask_scheduler_clients",
             "Number of clients connected.",
-            value=len(self.server.clients),
+            value=len([k for k in self.server.clients if k != "fire-and-forget"]),
         )
 
         yield GaugeMetricFamily(
diff --git a/distributed/http/scheduler/tests/test_scheduler_http.py b/distributed/http/scheduler/tests/test_scheduler_http.py
index 62328df4474..2aa4f58c4ea 100644
--- a/distributed/http/scheduler/tests/test_scheduler_http.py
+++ b/distributed/http/scheduler/tests/test_scheduler_http.py
@@ -92,9 +92,14 @@ async def test_prometheus(c, s, a, b):
         assert response.headers["Content-Type"] == "text/plain; version=0.0.4"
 
         txt = response.body.decode("utf8")
-        families = {familiy.name for familiy in text_string_to_metric_families(txt)}
+        families = {
+            family.name: family for family in text_string_to_metric_families(txt)
+        }
         assert "dask_scheduler_workers" in families
 
+        client = families["dask_scheduler_clients"]
+        assert client.samples[0].value == 1.0
+
 
 @gen_cluster(client=True, clean_kwargs={"threads": False})
 async def test_prometheus_collect_task_states(c, s, a, b):

From b2cfd852eee0ba6e8b5e40e95a6e4fed4b363ec4 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 1 Sep 2020 06:50:16 -0700
Subject: [PATCH 0972/1550] Support zero-worker clients (#4090)

Previously we used to wait until workers arrived before handing back
control.  However this stopped zero-worker clients from starting up

I don't think that we need this any more.  I'm guessing that SpecCluster
handles this now.
---
 distributed/client.py                  | 7 -------
 distributed/deploy/tests/test_local.py | 8 ++++++++
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 3284d3285a1..069a2cb0ada 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -1028,13 +1028,6 @@ async def _start(self, timeout=no_default, **kwargs):
                     **self._startup_kwargs,
                 )
 
-            # Wait for all workers to be ready
-            # XXX should be a LocalCluster method instead
-            while not self.cluster.workers or len(self.cluster.scheduler.workers) < len(
-                self.cluster.workers
-            ):
-                await asyncio.sleep(0.01)
-
             address = self.cluster.scheduler_address
 
         self._gather_semaphore = asyncio.Semaphore(5)
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index b46fdd065de..2b63f28b10f 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -1050,3 +1050,11 @@ async def test_async_with():
         assert w
 
     assert not w
+
+
+@pytest.mark.asyncio
+async def test_no_workers(cleanup):
+    async with Client(
+        n_workers=0, silence_logs=False, dashboard_address=None, asynchronous=True
+    ) as c:
+        pass

From 0333b9a4967f7bde571513f90ad028a647154c9f Mon Sep 17 00:00:00 2001
From: Dror Speiser <dror.mastershin@gmail.com>
Date: Tue, 1 Sep 2020 16:51:04 +0300
Subject: [PATCH 0973/1550] dask-worker --nprocs accepts negative values
 (#4089)

* dask-worker --nprocs accepts negative values

* Update distributed/cli/tests/test_dask_worker.py

Co-authored-by: Matthew Rocklin <mrocklin@gmail.com>
---
 distributed/cli/dask_worker.py            | 12 +++++++++++-
 distributed/cli/tests/test_dask_worker.py |  8 ++++++++
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
index ac0d7976b98..707a410348e 100755
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -127,7 +127,8 @@
     type=int,
     default=1,
     show_default=True,
-    help="Number of worker processes to launch.",
+    help="Number of worker processes to launch. "
+    "If negative, then (CPU_COUNT + 1 + nprocs) is used.",
 )
 @click.option(
     "--name",
@@ -288,6 +289,15 @@ def main(
         if v is not None
     }
 
+    if nprocs < 0:
+        nprocs = CPU_COUNT + 1 + nprocs
+
+    if nprocs <= 0:
+        logger.error(
+            "Failed to launch worker. Must specify --nprocs so that there's at least one process."
+        )
+        sys.exit(1)
+
     if nprocs > 1 and not nanny:
         logger.error(
             "Failed to launch worker.  You cannot use the --no-nanny argument when nprocs > 1."
diff --git a/distributed/cli/tests/test_dask_worker.py b/distributed/cli/tests/test_dask_worker.py
index 4d252a51aa1..f877ff9209b 100644
--- a/distributed/cli/tests/test_dask_worker.py
+++ b/distributed/cli/tests/test_dask_worker.py
@@ -8,6 +8,7 @@
 import sys
 import os
 from time import sleep
+from multiprocessing import cpu_count
 
 import distributed.cli.dask_worker
 from distributed import Client, Scheduler
@@ -238,6 +239,13 @@ def test_nprocs_requires_nanny(loop):
             )
 
 
+def test_nprocs_negative(loop):
+    with popen(["dask-scheduler", "--no-dashboard"]) as sched:
+        with popen(["dask-worker", "127.0.0.1:8786", "--nprocs=-1"]) as worker:
+            with Client("tcp://127.0.0.1:8786", loop=loop) as c:
+                c.wait_for_workers(cpu_count(), timeout="10 seconds")
+
+
 def test_nprocs_expands_name(loop):
     with popen(["dask-scheduler", "--no-dashboard"]) as sched:
         with popen(

From 73fa9bd1bd7dcb4ceed72cdbdc6dd4b92f887521 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 2 Sep 2020 11:52:43 -0700
Subject: [PATCH 0974/1550] Log when downloading a preload script (#4094)

---
 distributed/preloading.py         | 1 +
 distributed/tests/test_preload.py | 8 +++++---
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/distributed/preloading.py b/distributed/preloading.py
index f1e125b7b13..b3d6a8e6d13 100644
--- a/distributed/preloading.py
+++ b/distributed/preloading.py
@@ -120,6 +120,7 @@ def _import_module(name, file_dir=None) -> ModuleType:
 
 
 async def _download_module(url: str) -> ModuleType:
+    logger.info("Downloading preload at %s", url)
     assert is_webaddress(url)
 
     client = AsyncHTTPClient()
diff --git a/distributed/tests/test_preload.py b/distributed/tests/test_preload.py
index 2c35a59e06c..989e3d629ae 100644
--- a/distributed/tests/test_preload.py
+++ b/distributed/tests/test_preload.py
@@ -9,7 +9,7 @@
 
 import dask
 from distributed import Client, Scheduler, Worker, Nanny
-from distributed.utils_test import cluster
+from distributed.utils_test import cluster, captured_logger
 from distributed.utils_test import loop, cleanup  # noqa F401
 
 
@@ -139,7 +139,9 @@ def dask_setup(dask_server):
     app = web.Application([(r"/preload", MyHandler)])
     server = app.listen(12345)
     try:
-        async with Scheduler(preload=["http://localhost:12345/preload"]) as s:
-            assert s.foo == 1
+        with captured_logger("distributed.preloading") as log:
+            async with Scheduler(preload=["http://localhost:12345/preload"]) as s:
+                assert s.foo == 1
+        assert "12345/preload" in log.getvalue()
     finally:
         server.stop()

From a803253492ab13aafee22e4a997996315aca63d0 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sat, 5 Sep 2020 09:06:02 -0700
Subject: [PATCH 0975/1550] Respect timeouts when closing clients synchronously
 (#4096)

---
 distributed/client.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 069a2cb0ada..b3b780fce4e 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -1429,7 +1429,7 @@ async def _():
 
                     self.sync(_)
 
-        sync(self.loop, self._close, fast=True)
+        sync(self.loop, self._close, fast=True, callback_timeout=timeout)
 
         assert self.status == "closed"
 
@@ -4817,7 +4817,8 @@ def _close_global_client():
     c = _get_global_client()
     if c is not None:
         c._should_close_loop = False
-        c.close(timeout=2)
+        with suppress(TimeoutError):
+            c.close(timeout=2)
 
 
 atexit.register(_close_global_client)

From a31ae12e517254348b8566848f4b2a5aebd376ee Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 9 Sep 2020 14:19:33 -0700
Subject: [PATCH 0976/1550] Don't close a nannied worker if it hasn't yet
 started (#4093)

Fixes #4075

This error has been coming up a bunch recently.
Hopefully this helps the real issues to rise to the front
---
 distributed/nanny.py             |  2 +-
 distributed/tests/test_worker.py | 17 +++++++++++++++++
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/distributed/nanny.py b/distributed/nanny.py
index fd7078982a7..87cf54ed839 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -438,7 +438,7 @@ def _on_exit_sync(self, exitcode):
         self.loop.add_callback(self._on_exit, exitcode)
 
     async def _on_exit(self, exitcode):
-        if self.status not in (Status.closing, Status.closed):
+        if self.status not in (Status.init, Status.closing, Status.closed):
             try:
                 await self.scheduler.unregister(address=self.worker_address)
             except (EnvironmentError, CommClosedError):
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 796eca82e3f..42f9236f2d5 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -1647,3 +1647,20 @@ def bad_heartbeat_worker(*args, **kwargs):
                 else:
                     assert w.status == Status.closed
     assert "Heartbeat to scheduler failed" in logger.getvalue()
+
+
+@pytest.mark.asyncio
+async def test_bad_local_directory(cleanup):
+    async with await Scheduler() as s:
+        try:
+            async with Worker(s.address, local_directory="/not/a/valid-directory"):
+                pass
+        except PermissionError:
+            pass
+        else:
+            if WINDOWS:
+                pass
+            else:
+                assert False
+
+        assert not any("error" in log for log in s.get_logs())

From 4115f55c655000b04a8132c9ca62fa07bd6df6f5 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 10 Sep 2020 09:58:24 -0700
Subject: [PATCH 0977/1550] add logging for adaptive start and stop (#4101)

---
 distributed/deploy/adaptive.py      | 2 ++
 distributed/deploy/adaptive_core.py | 6 ++++++
 2 files changed, 8 insertions(+)

diff --git a/distributed/deploy/adaptive.py b/distributed/deploy/adaptive.py
index 1c53155de15..3d5271ed98a 100644
--- a/distributed/deploy/adaptive.py
+++ b/distributed/deploy/adaptive.py
@@ -103,6 +103,8 @@ def __init__(
 
         self.target_duration = parse_timedelta(target_duration)
 
+        logger.info("Adaptive scaling started: minimum=%s maximum=%s", minimum, maximum)
+
         super().__init__(
             minimum=minimum, maximum=maximum, wait_count=wait_count, interval=interval
         )
diff --git a/distributed/deploy/adaptive_core.py b/distributed/deploy/adaptive_core.py
index 14de96ad396..b15aa496345 100644
--- a/distributed/deploy/adaptive_core.py
+++ b/distributed/deploy/adaptive_core.py
@@ -1,4 +1,5 @@
 import collections
+import logging
 import math
 
 from tornado.ioloop import IOLoop, PeriodicCallback
@@ -8,6 +9,9 @@
 from ..utils import parse_timedelta
 
 
+logger = logging.getLogger(__name__)
+
+
 class AdaptiveCore:
     """
     The core logic for adaptive deployments, with none of the cluster details
@@ -103,6 +107,8 @@ def f():
         self.log = collections.deque(maxlen=10000)
 
     def stop(self):
+        logger.info("Adaptive stop")
+
         if self.periodic_callback:
             self.periodic_callback.stop()
             self.periodic_callback = None

From 73d381a29906bf6e08fad013c921922bd73fd9d4 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Fri, 11 Sep 2020 16:27:23 -0500
Subject: [PATCH 0978/1550] bump version to 2.26.0

---
 docs/source/changelog.rst | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index 9a0cda5bf1f..d04d8103d58 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,20 @@
 Changelog
 =========
 
+2.26.0 - 2020-09-11
+-------------------
+
+- Add logging for adaptive start and stop (:pr:`4101`) `Matthew Rocklin`_
+- Don't close a nannied worker if it hasn't yet started (:pr:`4093`) `Matthew Rocklin`_
+- Respect timeouts when closing clients synchronously (:pr:`4096`) `Matthew Rocklin`_
+- Log when downloading a preload script (:pr:`4094`) `Matthew Rocklin`_
+- ``dask-worker --nprocs`` accepts negative values (:pr:`4089`) `Dror Speiser`_
+- Support zero-worker clients (:pr:`4090`) `Matthew Rocklin`_
+- Exclude ``fire-and-forget`` client from metrics (:pr:`4078`) `Tom Augspurger`_
+- Drop ``Serialized.deserialize()`` method (:pr:`4073`) `jakirkham`_
+- Add ``timeout=`` keyword to ``Client.wait_for_workers`` method (:pr:`4087`) `Matthew Rocklin`_
+
+
 2.25.0 - 2020-08-28
 -------------------
 
@@ -1939,3 +1953,4 @@ significantly without many new features.
 .. _`Jack Xiaosong Xu`: https://github.com/jackxxu
 .. _`Willi Rath`: https://github.com/willirath
 .. _`Roberto Panai`: https://github.com/rpanai
+.. _`Dror Speiser`: https://github.com/drorspei

From f081ad0f51239bfeb27391fd72c62fbfa1bd7b0b Mon Sep 17 00:00:00 2001
From: Poruri Sai Rahul <rahul.poruri@gmail.com>
Date: Sun, 13 Sep 2020 02:48:07 +0000
Subject: [PATCH 0979/1550] Update super usage (#4110)

On Python 3, super can be called with no arguments but on Python 2,
super needs to be called with the class name and an instance.

Because distributed does not support Python 2 on master, super usage can
be updated.

Note that this was an automated regex-based search and replace. After
the automated replace, each change was individually checked and
committed.
---
 distributed/client.py                             | 2 +-
 distributed/deploy/adaptive.py                    | 2 +-
 distributed/deploy/local.py                       | 2 +-
 distributed/diagnostics/progressbar.py            | 8 +++-----
 distributed/http/scheduler/prometheus/__init__.py | 4 +---
 distributed/http/worker/prometheus.py             | 2 +-
 distributed/nanny.py                              | 2 +-
 distributed/scheduler.py                          | 6 +++---
 distributed/tests/test_scheduler.py               | 4 ++--
 distributed/threadpoolexecutor.py                 | 2 +-
 distributed/utils.py                              | 2 +-
 distributed/worker.py                             | 2 +-
 12 files changed, 17 insertions(+), 21 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index b3b780fce4e..899fd16167f 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -4202,7 +4202,7 @@ class Executor(Client):
 
     def __init__(self, *args, **kwargs):
         warnings.warn("Executor has been renamed to Client")
-        super(Executor, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
 
 
 def CompatibleExecutor(*args, **kwargs):
diff --git a/distributed/deploy/adaptive.py b/distributed/deploy/adaptive.py
index 3d5271ed98a..d3e9ddb1fbf 100644
--- a/distributed/deploy/adaptive.py
+++ b/distributed/deploy/adaptive.py
@@ -152,7 +152,7 @@ async def recommendations(self, target: int) -> dict:
             # are in sync before making recommendations.
             await self.cluster
 
-        return await super(Adaptive, self).recommendations(target)
+        return await super().recommendations(target)
 
     async def workers_to_close(self, target: int):
         """
diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index 66cfe70ee6d..f06892a1820 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -226,7 +226,7 @@ def __init__(
 
         workers = {i: worker for i in range(n_workers)}
 
-        super(LocalCluster, self).__init__(
+        super().__init__(
             scheduler=scheduler,
             workers=workers,
             worker=worker,
diff --git a/distributed/diagnostics/progressbar.py b/distributed/diagnostics/progressbar.py
index af1c0b7f26c..692e14bcd68 100644
--- a/distributed/diagnostics/progressbar.py
+++ b/distributed/diagnostics/progressbar.py
@@ -115,7 +115,7 @@ def __init__(
         start=True,
         **kwargs,
     ):
-        super(TextProgressBar, self).__init__(keys, scheduler, interval, complete)
+        super().__init__(keys, scheduler, interval, complete)
         self.width = width
         self.loop = loop or IOLoop()
 
@@ -158,7 +158,7 @@ def __init__(
         loop=None,
         **kwargs,
     ):
-        super(ProgressWidget, self).__init__(keys, scheduler, interval, complete)
+        super().__init__(keys, scheduler, interval, complete)
 
         from ipywidgets import FloatProgress, HBox, VBox, HTML
 
@@ -314,9 +314,7 @@ def __init__(
         complete=False,
         **kwargs,
     ):
-        super(MultiProgressWidget, self).__init__(
-            keys, scheduler, func, interval, complete
-        )
+        super().__init__(keys, scheduler, func, interval, complete)
         from ipywidgets import VBox
 
         self.widget = VBox([])
diff --git a/distributed/http/scheduler/prometheus/__init__.py b/distributed/http/scheduler/prometheus/__init__.py
index c746f86f938..2016ca32512 100644
--- a/distributed/http/scheduler/prometheus/__init__.py
+++ b/distributed/http/scheduler/prometheus/__init__.py
@@ -78,9 +78,7 @@ class PrometheusHandler(RequestHandler):
     def __init__(self, *args, dask_server=None, **kwargs):
         import prometheus_client
 
-        super(PrometheusHandler, self).__init__(
-            *args, dask_server=dask_server, **kwargs
-        )
+        super().__init__(*args, dask_server=dask_server, **kwargs)
 
         if PrometheusHandler._collectors:
             # Especially during testing, multiple schedulers are started
diff --git a/distributed/http/worker/prometheus.py b/distributed/http/worker/prometheus.py
index 69e3439df9b..08781bfd151 100644
--- a/distributed/http/worker/prometheus.py
+++ b/distributed/http/worker/prometheus.py
@@ -77,7 +77,7 @@ class PrometheusHandler(RequestHandler):
     def __init__(self, *args, **kwargs):
         import prometheus_client
 
-        super(PrometheusHandler, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
 
         if PrometheusHandler._initialized:
             return
diff --git a/distributed/nanny.py b/distributed/nanny.py
index 87cf54ed839..85c814272c5 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -198,7 +198,7 @@ def __init__(
             "run": self.run,
         }
 
-        super(Nanny, self).__init__(
+        super().__init__(
             handlers=handlers, io_loop=self.loop, connection_args=self.connection_args
         )
 
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 3f787a1f92e..b57366be48b 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1379,7 +1379,7 @@ def __init__(
 
         connection_limit = get_fileno_limit() / 2
 
-        super(Scheduler, self).__init__(
+        super().__init__(
             handlers=self.handlers,
             stream_handlers=merge(worker_handlers, client_handlers),
             io_loop=self.loop,
@@ -1580,7 +1580,7 @@ async def close(self, comm=None, fast=False, close_workers=False):
 
         self.status = Status.closed
         self.stop()
-        await super(Scheduler, self).close()
+        await super().close()
 
         setproctitle("dask-scheduler [closed]")
         disable_gc_diagnosis()
@@ -5586,7 +5586,7 @@ def heartbeat_interval(n):
 
 class KilledWorker(Exception):
     def __init__(self, task, last_worker):
-        super(KilledWorker, self).__init__(task, last_worker)
+        super().__init__(task, last_worker)
         self.task = task
         self.last_worker = last_worker
 
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 00d6631c9c5..4efc40d85d4 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -1963,12 +1963,12 @@ class FlakyConnectionPool(ConnectionPool):
     def __init__(self, *args, failing_connections=0, **kwargs):
         self.cnn_count = 0
         self.failing_connections = failing_connections
-        super(FlakyConnectionPool, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
 
     async def connect(self, *args, **kwargs):
         self.cnn_count += 1
         if self.cnn_count > self.failing_connections:
-            return await super(FlakyConnectionPool, self).connect(*args, **kwargs)
+            return await super().connect(*args, **kwargs)
         else:
             return BrokenComm()
 
diff --git a/distributed/threadpoolexecutor.py b/distributed/threadpoolexecutor.py
index c3112b3056b..57791809f60 100644
--- a/distributed/threadpoolexecutor.py
+++ b/distributed/threadpoolexecutor.py
@@ -70,7 +70,7 @@ class ThreadPoolExecutor(thread.ThreadPoolExecutor):
     _counter = itertools.count()
 
     def __init__(self, *args, **kwargs):
-        super(ThreadPoolExecutor, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self._rejoin_list = []
         self._rejoin_lock = threading.Lock()
         self._thread_name_prefix = kwargs.get(
diff --git a/distributed/utils.py b/distributed/utils.py
index 237b19ea37b..bbe01de96ff 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -1150,7 +1150,7 @@ class DequeHandler(logging.Handler):
 
     def __init__(self, *args, n=10000, **kwargs):
         self.deque = deque(maxlen=n)
-        super(DequeHandler, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self._instances.add(self)
 
     def emit(self, record):
diff --git a/distributed/worker.py b/distributed/worker.py
index 848b00d564d..d5cefcc5b7d 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -627,7 +627,7 @@ def __init__(
             "steal-request": self.steal_request,
         }
 
-        super(Worker, self).__init__(
+        super().__init__(
             handlers=handlers,
             stream_handlers=stream_handlers,
             io_loop=self.loop,

From ea2f88bc61d7f34e418ce7e872bf371fe10a3db4 Mon Sep 17 00:00:00 2001
From: Julia Signell <jsignell@gmail.com>
Date: Wed, 16 Sep 2020 14:04:26 -0400
Subject: [PATCH 0980/1550] Match pre-commit in dask (#4049)

---
 .github/workflows/ci-pre-commit.yml | 15 +++++++++++++++
 .pre-commit-config.yaml             | 20 +++++++++++---------
 .travis.yml                         |  6 ------
 setup.cfg                           |  2 +-
 4 files changed, 27 insertions(+), 16 deletions(-)
 create mode 100644 .github/workflows/ci-pre-commit.yml

diff --git a/.github/workflows/ci-pre-commit.yml b/.github/workflows/ci-pre-commit.yml
new file mode 100644
index 00000000000..630ae98fac4
--- /dev/null
+++ b/.github/workflows/ci-pre-commit.yml
@@ -0,0 +1,15 @@
+name: pre-commit
+
+on:
+  push:
+    branches: master
+  pull_request:
+    branches: master
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-python@v2
+    - uses: pre-commit/action@v2.0.0
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 2b64eddd06a..a934af611e1 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,10 +1,12 @@
 repos:
--   repo: https://github.com/ambv/black
-    rev: stable
-    hooks:
-    - id: black
-      language_version: python3.7
--   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v2.3.0
-    hooks:
-    - id: flake8
+  -   repo: https://github.com/psf/black
+      rev: stable
+      hooks:
+      - id: black
+        language_version: python3
+        exclude: versioneer.py
+  -   repo: https://gitlab.com/pycqa/flake8
+      rev: 3.8.3
+      hooks:
+      - id: flake8
+        language_version: python3
diff --git a/.travis.yml b/.travis.yml
index b7995a2a034..850fcf23d53 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -13,10 +13,6 @@ env:
 matrix:
   fast_finish: true
   include:
-  - os: linux
-    language: python
-    python: 3.6
-    env: LINT=true
   - os: osx
     env: PYTHON=3.7 TESTS=true PACKAGES="python-snappy python-blosc" TORNADO=6
     if: type != pull_request OR commit_message =~ test-osx  # Skip on PRs unless the commit message contains "test-osx"
@@ -30,8 +26,6 @@ install:
 
 script:
   - if [[ $TESTS == true ]]; then source continuous_integration/travis/run_tests.sh ; fi
-  - if [[ $LINT == true ]]; then python -m pip install flake8 ; flake8 distributed ; fi
-  - if [[ $LINT == true ]]; then python -m pip install black ; black distributed --check; fi
 
 after_success:
   - if [[ $COVERAGE == true ]]; then coverage report; python -m pip install -q coveralls ; coveralls ; fi
diff --git a/setup.cfg b/setup.cfg
index 32d7dfc15ff..ca4417fabdb 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -4,7 +4,7 @@
 # https://flake8.readthedocs.io/en/latest/user/error-codes.html
 
 # Note: there cannot be spaces after comma's here
-exclude = __init__.py,distributed/_concurrent_futures_thread.py
+exclude = __init__.py,versioneer.py,distributed/_concurrent_futures_thread.py
 ignore =
     E20,        # Extra space in brackets
     E231,E241,  # Multiple spaces around ","

From 1c9d8646fa8b747fbcdffd412ba1919979e88e2b Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 17 Sep 2020 07:32:11 -0700
Subject: [PATCH 0981/1550] Allow RuntimeErrors when closing global clients
 (#4115)

These tend to be due to the event loop being closed.
---
 distributed/client.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 899fd16167f..4115b4409b5 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -4817,8 +4817,8 @@ def _close_global_client():
     c = _get_global_client()
     if c is not None:
         c._should_close_loop = False
-        with suppress(TimeoutError):
-            c.close(timeout=2)
+        with suppress(TimeoutError, RuntimeError):
+            c.close(timeout=3)
 
 
 atexit.register(_close_global_client)

From 755e1055559c2a5a7b14e2134b09e6ef10bdaa23 Mon Sep 17 00:00:00 2001
From: Abdulelah Bin Mahfoodh <abduh.bm@gmail.com>
Date: Thu, 17 Sep 2020 17:34:30 +0300
Subject: [PATCH 0982/1550] Support different remote_python paths on cluster
 nodes (#4085)

---
 distributed/deploy/ssh.py            | 28 +++++++++++++++----
 distributed/deploy/tests/test_ssh.py | 41 ++++++++++++++++++++++++++++
 2 files changed, 63 insertions(+), 6 deletions(-)

diff --git a/distributed/deploy/ssh.py b/distributed/deploy/ssh.py
index a6020998fec..763fe779c3a 100644
--- a/distributed/deploy/ssh.py
+++ b/distributed/deploy/ssh.py
@@ -108,10 +108,13 @@ async def start(self):
                     "Worker failed to set DASK_INTERNAL_INHERIT_CONFIG variable "
                 )
 
+        if not self.remote_python:
+            self.remote_python = sys.executable
+
         cmd = " ".join(
             [
                 set_env,
-                self.remote_python or sys.executable,
+                self.remote_python,
                 "-m",
                 self.worker_module,
                 self.scheduler,
@@ -186,10 +189,13 @@ async def start(self):
                     "Scheduler failed to set DASK_INTERNAL_INHERIT_CONFIG variable "
                 )
 
+        if not self.remote_python:
+            self.remote_python = sys.executable
+
         cmd = " ".join(
             [
                 set_env,
-                self.remote_python or sys.executable,
+                self.remote_python,
                 "-m",
                 "distributed.cli.dask_scheduler",
             ]
@@ -235,7 +241,7 @@ def SSHCluster(
     worker_options: dict = {},
     scheduler_options: dict = {},
     worker_module: str = "distributed.cli.dask_worker",
-    remote_python: str = None,
+    remote_python: Union[str, List[str]] = None,
     **kwargs,
 ):
     """Deploy a Dask cluster using SSH
@@ -274,7 +280,7 @@ def SSHCluster(
         Keywords to pass on to scheduler.
     worker_module: str, optional
         Python module to call to start the worker.
-    remote_python: str, optional
+    remote_python: str or list of str, optional
         Path to Python on remote nodes.
 
     Examples
@@ -326,6 +332,12 @@ def SSHCluster(
             "dictionary for each address."
         )
 
+    if isinstance(remote_python, list) and len(remote_python) != len(hosts):
+        raise RuntimeError(
+            "When specifying a list of remote_python you must provide a "
+            "path for each address."
+        )
+
     scheduler = {
         "cls": Scheduler,
         "options": {
@@ -334,7 +346,9 @@ def SSHCluster(
             if isinstance(connect_options, dict)
             else connect_options[0],
             "kwargs": scheduler_options,
-            "remote_python": remote_python,
+            "remote_python": remote_python[0]
+            if isinstance(remote_python, list)
+            else remote_python,
         },
     }
     workers = {
@@ -347,7 +361,9 @@ def SSHCluster(
                 else connect_options[i + 1],
                 "kwargs": worker_options,
                 "worker_module": worker_module,
-                "remote_python": remote_python,
+                "remote_python": remote_python[i + 1]
+                if isinstance(remote_python, list)
+                else remote_python,
             },
         }
         for i, host in enumerate(hosts[1:])
diff --git a/distributed/deploy/tests/test_ssh.py b/distributed/deploy/tests/test_ssh.py
index 99d5ea15aec..2bed638ff49 100644
--- a/distributed/deploy/tests/test_ssh.py
+++ b/distributed/deploy/tests/test_ssh.py
@@ -2,6 +2,7 @@
 
 pytest.importorskip("asyncssh")
 
+import sys
 import dask
 from dask.distributed import Client
 from distributed.deploy.ssh import SSHCluster
@@ -156,3 +157,43 @@ async def test_list_of_connect_options_raises():
             worker_options={"death_timeout": "5s"},
         ) as _:
             pass
+
+
+@pytest.mark.asyncio
+async def test_remote_python():
+    async with SSHCluster(
+        ["127.0.0.1"] * 3,
+        connect_options=[dict(known_hosts=None)] * 3,
+        asynchronous=True,
+        scheduler_options={"port": 0, "idle_timeout": "5s"},
+        worker_options={"death_timeout": "5s"},
+        remote_python=sys.executable,
+    ) as cluster:
+        assert cluster.workers[0].remote_python == sys.executable
+
+
+@pytest.mark.asyncio
+async def test_remote_python_as_dict():
+    async with SSHCluster(
+        ["127.0.0.1"] * 3,
+        connect_options=[dict(known_hosts=None)] * 3,
+        asynchronous=True,
+        scheduler_options={"port": 0, "idle_timeout": "5s"},
+        worker_options={"death_timeout": "5s"},
+        remote_python=[sys.executable] * 3,
+    ) as cluster:
+        assert cluster.workers[0].remote_python == sys.executable
+
+
+@pytest.mark.asyncio
+async def test_list_of_remote_python_raises():
+    with pytest.raises(RuntimeError):
+        async with SSHCluster(
+            ["127.0.0.1"] * 3,
+            connect_options=[dict(known_hosts=None)] * 3,
+            asynchronous=True,
+            scheduler_options={"port": 0, "idle_timeout": "5s"},
+            worker_options={"death_timeout": "5s"},
+            remote_python=[sys.executable] * 4,  # Mismatch in length 4 != 3
+        ) as _:
+            pass

From 92a8884ecadecb47805ec920b37fdc26c6049324 Mon Sep 17 00:00:00 2001
From: Nick Evans <nick.evans3976@gmail.com>
Date: Thu, 17 Sep 2020 15:48:45 +0100
Subject: [PATCH 0983/1550] Fix registering a worker plugin with name arg
 (#4105)

---
 .../diagnostics/tests/test_worker_plugin.py     | 17 +++++++++++++++++
 distributed/scheduler.py                        |  2 +-
 distributed/worker.py                           |  4 ++--
 3 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/distributed/diagnostics/tests/test_worker_plugin.py b/distributed/diagnostics/tests/test_worker_plugin.py
index 9bcc0c58ca4..2f17a86a8e7 100644
--- a/distributed/diagnostics/tests/test_worker_plugin.py
+++ b/distributed/diagnostics/tests/test_worker_plugin.py
@@ -136,6 +136,23 @@ async def test_release_dep_called(c, s, w):
     await async_wait_for(lambda: not (w.task_state or w.dep_state), timeout=10)
 
 
+@gen_cluster(nthreads=[("127.0.0.1", 1)], client=True)
+async def test_registering_with_name_arg(c, s, w):
+    class FooWorkerPlugin:
+        def setup(self, worker):
+            if hasattr(worker, "foo"):
+                raise RuntimeError(f"Worker {worker.address} already has foo!")
+
+            worker.foo = True
+
+    responses = await c.register_worker_plugin(FooWorkerPlugin(), name="foo")
+    assert list(responses.values()) == [{"status": "OK"}]
+
+    async with Worker(s.address, loop=s.loop):
+        responses = await c.register_worker_plugin(FooWorkerPlugin(), name="foo")
+        assert list(responses.values()) == [{"status": "repeat"}] * 2
+
+
 @gen_cluster(nthreads=[("127.0.0.1", 1)], client=True)
 async def test_empty_plugin(c, s, w):
     class EmptyPlugin:
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index b57366be48b..f2d4de8b16f 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -3844,7 +3844,7 @@ def get_task_stream(self, comm=None, start=None, stop=None, count=None):
 
     async def register_worker_plugin(self, comm, plugin, name=None):
         """ Registers a setup function, and call it on every worker """
-        self.worker_plugins.append(plugin)
+        self.worker_plugins.append({"plugin": plugin, "name": name})
 
         responses = await self.broadcast(
             msg=dict(op="plugin-add", plugin=plugin, name=name)
diff --git a/distributed/worker.py b/distributed/worker.py
index d5cefcc5b7d..2fabf256ddf 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -845,8 +845,8 @@ async def _register_with_scheduler(self):
         else:
             await asyncio.gather(
                 *[
-                    self.plugin_add(plugin=plugin)
-                    for plugin in response["worker-plugins"]
+                    self.plugin_add(**plugin_kwargs)
+                    for plugin_kwargs in response["worker-plugins"]
                 ]
             )
 

From ecaf14097f5e69e5b884e9c87b708c85d181a9ef Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Fri, 18 Sep 2020 21:33:59 -0500
Subject: [PATCH 0984/1550] bump version to 2.27.0

---
 docs/source/changelog.rst | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index d04d8103d58..aa1f8ca2ca9 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,16 @@
 Changelog
 =========
 
+2.27.0 - 2020-09-18
+-------------------
+
+- Fix registering a worker plugin with ``name`` arg (:pr:`4105`) `Nick Evans`_
+- Support different ``remote_python`` paths on cluster nodes (:pr:`4085`) `Abdulelah Bin Mahfoodh`_
+- Allow ``RuntimeError`` s when closing global clients (:pr:`4115`) `Matthew Rocklin`_
+- Match ``pre-commit`` in dask (:pr:`4049`) `Julia Signell`_
+- Update ``super`` usage (:pr:`4110`) `Poruri Sai Rahul`_
+
+
 2.26.0 - 2020-09-11
 -------------------
 
@@ -1954,3 +1964,4 @@ significantly without many new features.
 .. _`Willi Rath`: https://github.com/willirath
 .. _`Roberto Panai`: https://github.com/rpanai
 .. _`Dror Speiser`: https://github.com/drorspei
+.. _`Poruri Sai Rahul`: https://github.com/rahulporuri

From 8aefbf364b27afedcd492947dd209cc7c26e020d Mon Sep 17 00:00:00 2001
From: jennalc <66142035+jennalc@users.noreply.github.com>
Date: Tue, 22 Sep 2020 15:32:36 -0400
Subject: [PATCH 0985/1550] Fix SSL connection_args for progressbar connect
 (#4122)

---
 distributed/diagnostics/progressbar.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/distributed/diagnostics/progressbar.py b/distributed/diagnostics/progressbar.py
index 692e14bcd68..28aa135a649 100644
--- a/distributed/diagnostics/progressbar.py
+++ b/distributed/diagnostics/progressbar.py
@@ -256,8 +256,7 @@ def function(scheduler, p):
             return result
 
         self.comm = await connect(
-            self.scheduler,
-            connection_args=self.client().connection_args if self.client else None,
+            self.scheduler, **(self.client().connection_args if self.client else {})
         )
         logger.debug("Progressbar Connected to scheduler")
 

From 34fb932855df7fe2fb18e1076ede2dd0eb3d95d0 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Fri, 25 Sep 2020 22:47:22 -0500
Subject: [PATCH 0986/1550] bump version to 2.28.0

---
 docs/source/changelog.rst | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index aa1f8ca2ca9..ec8b0378267 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,12 @@
 Changelog
 =========
 
+2.28.0 - 2020-09-25
+-------------------
+
+- Fix SSL ``connection_args`` for ``progressbar`` connect (:pr:`4122`) `jennalc`_
+
+
 2.27.0 - 2020-09-18
 -------------------
 
@@ -1965,3 +1971,4 @@ significantly without many new features.
 .. _`Roberto Panai`: https://github.com/rpanai
 .. _`Dror Speiser`: https://github.com/drorspei
 .. _`Poruri Sai Rahul`: https://github.com/rahulporuri
+.. _`jennalc`: https://github.com/jennalc

From 4dd0e1ba2ddcbc8178cd1113e1120f311d4080e0 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 28 Sep 2020 08:59:08 -0700
Subject: [PATCH 0987/1550] Stop writing config files by default (#4123)

Follows on #6647 upstream
---
 distributed/config.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/distributed/config.py b/distributed/config.py
index 43e545576e5..5601c307558 100644
--- a/distributed/config.py
+++ b/distributed/config.py
@@ -12,7 +12,6 @@
 
 
 fn = os.path.join(os.path.dirname(__file__), "distributed.yaml")
-dask.config.ensure_file(source=fn)
 
 with open(fn) as f:
     defaults = yaml.safe_load(f)

From a968158416a790483a2a9a66a60fa270c2fdfa9e Mon Sep 17 00:00:00 2001
From: Krishan Bhasin <8904718+KrishanBhasin@users.noreply.github.com>
Date: Mon, 28 Sep 2020 16:09:35 +0000
Subject: [PATCH 0988/1550] Cleanup new exception traceback (#4125)

---
 distributed/comm/core.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/distributed/comm/core.py b/distributed/comm/core.py
index 05439504a1f..a0141e4d816 100644
--- a/distributed/comm/core.py
+++ b/distributed/comm/core.py
@@ -140,12 +140,12 @@ def handshake_configuration(local, remote):
                     local["pickle-protocol"], remote["pickle-protocol"]
                 )
             }
-        except KeyError:
+        except KeyError as e:
             raise ValueError(
                 "Your Dask versions may not be in sync. "
                 "Please ensure that you have the same version of dask "
                 "and distributed on your client, scheduler, and worker machines"
-            )
+            ) from e
 
         if local["compression"] == remote["compression"]:
             out["compression"] = local["compression"]

From 0131b6bff1af076d250fb8cfe464f7f4c7ed8e10 Mon Sep 17 00:00:00 2001
From: "Mads R. B. Kristensen" <madsbk@gmail.com>
Date: Mon, 28 Sep 2020 18:37:42 +0200
Subject: [PATCH 0989/1550] Simplify Client._graph_to_futures() (#4127)

* Removed extra_keys, which is redundant.

extra_keys is only used by str_graph() to find keys to tokenize
but since extra_keys are already tokenized, it is redundant to
to tokenize the keys it finds!

* Removed dsk2 and dsk3

* Avoid empty keys in future_dependencies

* Postponed the tokey() calls to the end of _graph_to_futures()

* Eliminated the need to juggle two graphs `dsk` and `d`
---
 distributed/client.py                     | 70 +++++++++++++----------
 distributed/deploy/tests/test_adaptive.py |  2 +-
 2 files changed, 42 insertions(+), 30 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 4115b4409b5..a9c8306c0a1 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -2569,6 +2569,13 @@ def _graph_to_futures(
             if actors is not None and actors is not True and actors is not False:
                 actors = list(self._expand_key(actors))
 
+            if restrictions:
+                restrictions = keymap(tokey, restrictions)
+                restrictions = valmap(list, restrictions)
+
+            if loose_restrictions is not None:
+                loose_restrictions = list(map(tokey, loose_restrictions))
+
             keyset = set(keys)
 
             values = {
@@ -2579,55 +2586,60 @@ def _graph_to_futures(
             if values:
                 dsk = subs_multiple(dsk, values)
 
-            d = {k: unpack_remotedata(v, byte_keys=True) for k, v in dsk.items()}
-            extra_futures = set.union(*[v[1] for v in d.values()]) if d else set()
-            extra_keys = {tokey(future.key) for future in extra_futures}
-            dsk2 = str_graph({k: v[0] for k, v in d.items()}, extra_keys)
-            dsk3 = {k: v for k, v in dsk2.items() if k is not v}
-            for future in extra_futures:
+            # Unpack remote data in `dsk`, which are "WrappedKeys" that are
+            # unknown to `dsk` but known to the scheduler.
+            dsk = {k: unpack_remotedata(v) for k, v in dsk.items()}
+            unpacked_futures = (
+                set.union(*[v[1] for v in dsk.values()]) if dsk else set()
+            )
+            for future in unpacked_futures:
                 if future.client is not self:
                     msg = "Inputs contain futures that were created by another client."
                     raise ValueError(msg)
-
-            if restrictions:
-                restrictions = keymap(tokey, restrictions)
-                restrictions = valmap(list, restrictions)
-
-            if loose_restrictions is not None:
-                loose_restrictions = list(map(tokey, loose_restrictions))
-
-            future_dependencies = {
-                tokey(k): {tokey(f.key) for f in v[1]} for k, v in d.items()
-            }
-
-            for s in future_dependencies.values():
-                for v in s:
-                    if v not in self.futures:
-                        raise CancelledError(v)
-
+                if tokey(future.key) not in self.futures:
+                    raise CancelledError(tokey(future.key))
+            unpacked_futures_deps = {}
+            for k, v in dsk.items():
+                if len(v[1]):
+                    unpacked_futures_deps[k] = {f.key for f in v[1]}
+            dsk = {k: v[0] for k, v in dsk.items()}
+
+            # Find dependencies for the scheduler,
             dependencies = {k: get_dependencies(dsk, k) for k in dsk}
 
             if priority is None:
-                priority = dask.order.order(dsk, dependencies=dependencies)
+                # Removing all unpacked futures before calling order()
+                unpacked_keys = {future.key for future in unpacked_futures}
+                stripped_dsk = {k: v for k, v in dsk.items() if k not in unpacked_keys}
+                stripped_deps = {
+                    k: v - unpacked_keys
+                    for k, v in dependencies.items()
+                    if k not in unpacked_keys
+                }
+                priority = dask.order.order(stripped_dsk, dependencies=stripped_deps)
                 priority = keymap(tokey, priority)
 
+            # Append the dependencies of unpacked futures.
+            for k, v in unpacked_futures_deps.items():
+                dependencies[k] = set(dependencies.get(k, ())) | v
+
+            # The scheduler expect all keys to be strings
             dependencies = {
                 tokey(k): [tokey(dep) for dep in deps]
                 for k, deps in dependencies.items()
                 if deps
             }
-            for k, deps in future_dependencies.items():
-                if deps:
-                    dependencies[k] = list(set(dependencies.get(k, ())) | deps)
+            dsk = str_graph(dsk, extra_values={f.key for f in unpacked_futures})
 
             if isinstance(retries, Number) and retries > 0:
-                retries = {k: retries for k in dsk3}
+                retries = {k: retries for k in dsk}
 
+            # Create futures before sending graph (helps avoid contention)
             futures = {key: Future(key, self, inform=False) for key in keyset}
             self._send_to_scheduler(
                 {
                     "op": "update-graph",
-                    "tasks": valmap(dumps_task, dsk3),
+                    "tasks": valmap(dumps_task, dsk),
                     "dependencies": dependencies,
                     "keys": list(map(tokey, keys)),
                     "restrictions": restrictions or {},
diff --git a/distributed/deploy/tests/test_adaptive.py b/distributed/deploy/tests/test_adaptive.py
index 17f1b175a27..c8b588ef140 100644
--- a/distributed/deploy/tests/test_adaptive.py
+++ b/distributed/deploy/tests/test_adaptive.py
@@ -298,7 +298,7 @@ async def test_adapt_down():
             start = time()
             while len(cluster.scheduler.workers) != 2:
                 await asyncio.sleep(0.1)
-                assert time() < start + 1
+                assert time() < start + 3
 
 
 @gen_test(timeout=30)

From b2f594e69751e0733ac595026e04181c55119ae9 Mon Sep 17 00:00:00 2001
From: Sergey Kozlov <skozlovf@gmail.com>
Date: Tue, 29 Sep 2020 04:25:46 +0600
Subject: [PATCH 0990/1550] Correctly tear down LoopRunner in Client (#4112)

See #4072

LoopRunner is designed to stop loops it started.
`loop` fixture doesn't start its IOLoop but always stops it.
This leads to conflict in the test: both LoopRunner and `loop` fixture
stop the same loop. So we need to use `loop_in_thread` fixture
since it starts and stops its IOLoop.

`_should_close_loop` flag is not used, moreover, LoopRunner detects
already started loop in the `_start_unlocked()` and doesn't stop it
in the `_stop_unlocked()`.
---
 distributed/client.py                 |  3 +--
 distributed/tests/test_client.py      |  6 ++---
 distributed/tests/test_client_loop.py | 34 +++++++++++++++++++++++++++
 distributed/utils.py                  |  2 --
 4 files changed, 38 insertions(+), 7 deletions(-)
 create mode 100644 distributed/tests/test_client_loop.py

diff --git a/distributed/client.py b/distributed/client.py
index a9c8306c0a1..e4a31acbbff 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -684,7 +684,6 @@ def __init__(
 
         self._connecting_to_scheduler = False
         self._asynchronous = asynchronous
-        self._should_close_loop = not loop
         self._loop_runner = LoopRunner(loop=loop, asynchronous=asynchronous)
         self.io_loop = self.loop = self._loop_runner.loop
 
@@ -1433,7 +1432,7 @@ async def _():
 
         assert self.status == "closed"
 
-        if self._should_close_loop and not shutting_down():
+        if not shutting_down():
             self._loop_runner.stop()
 
     async def _shutdown(self):
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 15005ecf837..e2251d82b53 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -5403,10 +5403,10 @@ async def test_client_name(s, a, b):
     await c.close()
 
 
-def test_client_doesnt_close_given_loop(loop, s, a, b):
-    with Client(s["address"], loop=loop) as c:
+def test_client_doesnt_close_given_loop(loop_in_thread, s, a, b):
+    with Client(s["address"], loop=loop_in_thread) as c:
         assert c.submit(inc, 1).result() == 2
-    with Client(s["address"], loop=loop) as c:
+    with Client(s["address"], loop=loop_in_thread) as c:
         assert c.submit(inc, 2).result() == 3
 
 
diff --git a/distributed/tests/test_client_loop.py b/distributed/tests/test_client_loop.py
new file mode 100644
index 00000000000..ce50498f7e4
--- /dev/null
+++ b/distributed/tests/test_client_loop.py
@@ -0,0 +1,34 @@
+import pytest
+from distributed import LocalCluster, Client
+from distributed.utils import LoopRunner
+
+
+# Test if Client stops LoopRunner on close.
+@pytest.mark.parametrize("with_own_loop", [True, False])
+def test_close_loop_sync(with_own_loop):
+    loop_runner = loop = None
+
+    # Setup simple cluster with one threaded worker.
+    # Complex setup is not required here since we test only IO loop teardown.
+    cluster_params = dict(n_workers=1, dashboard_address=None, processes=False)
+
+    loops_before = LoopRunner._all_loops.copy()
+
+    # Start own loop or use current thread's one.
+    if with_own_loop:
+        loop_runner = LoopRunner()
+        loop_runner.start()
+        loop = loop_runner.loop
+
+    with LocalCluster(loop=loop, **cluster_params) as cluster:
+        with Client(cluster, loop=loop) as client:
+            client.run(max, 1, 2)
+
+    # own loop must be explicitly stopped.
+    if with_own_loop:
+        loop_runner.stop()
+
+    # Internal loops registry must the same as before cluster running.
+    # This means loop runners in LocalCluster and Client correctly stopped.
+    # See LoopRunner._stop_unlocked().
+    assert loops_before == LoopRunner._all_loops
diff --git a/distributed/utils.py b/distributed/utils.py
index bbe01de96ff..f641a1bf2a0 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -371,10 +371,8 @@ def __init__(self, loop=None, asynchronous=False):
                 # We're expecting the loop to run in another thread,
                 # avoid re-using this thread's assigned loop
                 self._loop = IOLoop()
-            self._should_close_loop = True
         else:
             self._loop = loop
-            self._should_close_loop = False
         self._asynchronous = asynchronous
         self._loop_thread = None
         self._started = False

From 13098e1970d0bb4da7112d2a3aeab8a3ab691a08 Mon Sep 17 00:00:00 2001
From: Benjamin Zaitlen <quasiben@users.noreply.github.com>
Date: Tue, 29 Sep 2020 12:03:21 -0400
Subject: [PATCH 0991/1550] update async doc with example on .compute() vs
 client.compute() (#4137)

---
 docs/source/asynchronous.rst | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/docs/source/asynchronous.rst b/docs/source/asynchronous.rst
index 342981833db..b3c5c0cbb38 100644
--- a/docs/source/asynchronous.rst
+++ b/docs/source/asynchronous.rst
@@ -64,6 +64,18 @@ function to run the asynchronous function:
    client.sync(f)
 
 
+.. note: Blocking operations like the .compute() method aren’t ok to use in
+         asynchronous mode. Instead you’ll have to use the Client.compute
+         method
+
+
+.. code-block:: python
+
+    async with Client(asynchronous=True) as client:
+        arr = da.random.random((1000, 1000), chunks=(1000, 100))
+        await client.compute(arr.mean())
+
+
 Example
 -------
 

From bd5bff09417901ef55aed580302326b0b16db07c Mon Sep 17 00:00:00 2001
From: Tom Rochette <roctom@gmail.com>
Date: Tue, 29 Sep 2020 13:52:07 -0400
Subject: [PATCH 0992/1550] Return right away in Cluster.close if cluster is
 already closed (#4116)

If the cluster encountered an exception for example it may already
be closed. Calling close again should not at this point do anything.
---
 distributed/deploy/cluster.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
index 5b9c598be9c..075867cd0b0 100644
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -91,6 +91,15 @@ async def _close(self):
         self.status = Status.closed
 
     def close(self, timeout=None):
+        # If the cluster is already closed, we're already done
+        if self.status == Status.closed:
+            if self.asynchronous:
+                future = asyncio.Future()
+                future.set_result(None)
+                return future
+            else:
+                return
+
         with suppress(RuntimeError):  # loop closed during process shutdown
             return self.sync(self._close, callback_timeout=timeout)
 

From 111029d53536d596043feee2a8c914fb680774bd Mon Sep 17 00:00:00 2001
From: Pav A <rootsumsquared@gmail.com>
Date: Tue, 29 Sep 2020 18:56:09 +0100
Subject: [PATCH 0993/1550] Fix a few typos (#4131)

---
 distributed/semaphore.py         | 2 +-
 distributed/tests/test_client.py | 2 +-
 distributed/tests/test_steal.py  | 6 +++---
 docs/source/http_services.rst    | 2 +-
 4 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/distributed/semaphore.py b/distributed/semaphore.py
index 5d86bd8ea59..6ff66eac811 100644
--- a/distributed/semaphore.py
+++ b/distributed/semaphore.py
@@ -386,7 +386,7 @@ def __init__(self, max_leases=1, name=None, client=None, register=True):
         self.client._periodic_callbacks[self._periodic_callback_name] = pc
 
         # Need to start the callback using IOLoop.add_callback to ensure that the
-        # PC uses the correct event lopp.
+        # PC uses the correct event loop.
         self.client.io_loop.add_callback(pc.start)
 
     def register(self):
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index e2251d82b53..846e098a11d 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -6130,7 +6130,7 @@ async def f():
 
 
 @pytest.mark.asyncio
-async def test_client_gather_semaphor_loop(cleanup):
+async def test_client_gather_semaphore_loop(cleanup):
     async with Scheduler(port=0) as s:
         async with Client(s.address, asynchronous=True) as c:
             assert c._gather_semaphore._loop is c.loop.asyncio_loop
diff --git a/distributed/tests/test_steal.py b/distributed/tests/test_steal.py
index 5ef3e5330ec..f6fb91b1ce2 100644
--- a/distributed/tests/test_steal.py
+++ b/distributed/tests/test_steal.py
@@ -149,7 +149,7 @@ async def test_dont_steal_fast_tasks_compute_time(c, s, *workers):
     def do_nothing(x, y=None):
         pass
 
-    # execute and meassure runtime once
+    # execute and measure runtime once
     await wait(c.submit(do_nothing, 1))
 
     futures = c.map(do_nothing, range(1000), y=x)
@@ -166,7 +166,7 @@ async def test_dont_steal_fast_tasks_blacklist(c, s, a, b):
     x = c.submit(slowinc, 1, workers=[b.address])
 
     # If the blacklist of fast tasks is tracked somewhere else, this needs to be
-    # changed. This test requies *any* key which is blacklisted.
+    # changed. This test requires *any* key which is blacklisted.
     from distributed.stealing import fast_tasks
 
     blacklisted_key = next(iter(fast_tasks))
@@ -174,7 +174,7 @@ async def test_dont_steal_fast_tasks_blacklist(c, s, a, b):
     def fast_blacklisted(x, y=None):
         # The task should observe a certain computation time such that we can
         # ensure that it is not stolen due to the blacklisting. If it is too
-        # fast, the standard mechansim shouldn't allow stealing
+        # fast, the standard mechanism shouldn't allow stealing
         import time
 
         time.sleep(0.01)
diff --git a/docs/source/http_services.rst b/docs/source/http_services.rst
index ef5b7527957..d94485319a9 100644
--- a/docs/source/http_services.rst
+++ b/docs/source/http_services.rst
@@ -2,7 +2,7 @@ HTTP endpoints
 ==============
 
 A subset of the following pages will be available from the scheduler or
-workers of a running cluster. The list of currently available endpoins can
+workers of a running cluster. The list of currently available endpoints can
 be found by examining ``/sitemap.json``.
 
 

From 0c2c9f8e81cb488f97d85d61cc1eeb8e83b1a355 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Fri, 2 Oct 2020 08:09:12 -0700
Subject: [PATCH 0994/1550] Use `pandas.testing` (#4138)

`pandas.util.testing` is deprecated. The preferred way forward is to use
`pandas.testing`. So this updates our usage accordingly.
---
 distributed/tests/test_collections.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/distributed/tests/test_collections.py b/distributed/tests/test_collections.py
index b9af1ef0222..c0aae5d5ea6 100644
--- a/distributed/tests/test_collections.py
+++ b/distributed/tests/test_collections.py
@@ -1,3 +1,5 @@
+from distutils.version import LooseVersion
+
 import pytest
 
 pytest.importorskip("numpy")
@@ -11,7 +13,14 @@
 from distributed.utils_test import client, cluster_fixture, loop  # noqa F401
 import numpy as np
 import pandas as pd
-import pandas.testing as tm
+
+PANDAS_VERSION = LooseVersion(pd.__version__)
+PANDAS_GT_100 = PANDAS_VERSION >= LooseVersion("1.0.0")
+
+if PANDAS_GT_100:
+    import pandas.testing as tm  # noqa: F401
+else:
+    import pandas.util.testing as tm  # noqa: F401
 
 
 dfs = [

From a80b867cf40b05aa423a19aea1a077764ffba0f4 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Fri, 2 Oct 2020 18:22:26 -0500
Subject: [PATCH 0995/1550] bump version to 2.29.0

---
 docs/source/changelog.rst | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index ec8b0378267..3ae42669b91 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,19 @@
 Changelog
 =========
 
+2.29.0 - 2020-10-02
+-------------------
+
+- Use ``pandas.testing`` (:pr:`4138`) `jakirkham`_
+- Fix a few typos (:pr:`4131`) `Pav A`_
+- Return right away in ``Cluster.close`` if cluster is already closed (:pr:`4116`) `Tom Rochette`_
+- Update async doc with example on ``.compute()`` vs ``client.compute()`` (:pr:`4137`) `Benjamin Zaitlen`_
+- Correctly tear down ``LoopRunner`` in ``Client`` (:pr:`4112`) `Sergey Kozlov`_
+- Simplify ``Client._graph_to_futures()`` (:pr:`4127`) `Mads R. B. Kristensen`_
+- Cleanup new exception traceback (:pr:`4125`) `Krishan Bhasin`_
+- Stop writing config files by default (:pr:`4123`) `Matthew Rocklin`_
+
+
 2.28.0 - 2020-09-25
 -------------------
 
@@ -1972,3 +1985,4 @@ significantly without many new features.
 .. _`Dror Speiser`: https://github.com/drorspei
 .. _`Poruri Sai Rahul`: https://github.com/rahulporuri
 .. _`jennalc`: https://github.com/jennalc
+.. _`Sergey Kozlov`: https://github.com/skozlovf

From 6efd5b6147d29ec613bd83c72cdbd6a5a45a1438 Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Tue, 6 Oct 2020 15:25:05 +0100
Subject: [PATCH 0996/1550] Limit GPU metrics to visible devices only (#3810)

* Limit GPU metrics to visible devices only

* Move importorskip

* init nvmInit once rather than handles

* ws object has data for all workers

* match pid from worker process with process from GPUs

* send nvml data per worker/gpu not all gpus

* remove notebook

Co-authored-by: Benjamin Zaitlen <quasiben@gmail.com>
---
 distributed/dashboard/components/nvml.py   | 28 ++++++-------
 distributed/diagnostics/nvml.py            | 46 ++++++++++++++++------
 distributed/diagnostics/tests/test_nvml.py | 34 ++++++++++++++++
 3 files changed, 78 insertions(+), 30 deletions(-)
 create mode 100644 distributed/diagnostics/tests/test_nvml.py

diff --git a/distributed/dashboard/components/nvml.py b/distributed/dashboard/components/nvml.py
index 534c396cbc0..34cce3c4bc7 100644
--- a/distributed/dashboard/components/nvml.py
+++ b/distributed/dashboard/components/nvml.py
@@ -131,29 +131,23 @@ def update(self):
             memory_total = 0
             memory_max = 0
             worker = []
-            i = 0
 
-            for ws in workers:
+            for idx, ws in enumerate(workers):
                 try:
                     info = ws.extra["gpu"]
                 except KeyError:
                     continue
                 metrics = ws.metrics["gpu"]
-                for j, (u, mem_used, mem_total) in enumerate(
-                    zip(
-                        metrics["utilization"],
-                        metrics["memory-used"],
-                        info["memory-total"],
-                    )
-                ):
-                    memory_max = max(memory_max, mem_total)
-                    memory_total += mem_total
-                    utilization.append(int(u))
-                    memory.append(mem_used)
-                    worker.append(ws.address)
-                    gpu_index.append(j)
-                    y.append(i)
-                    i += 1
+                u = metrics["utilization"]
+                mem_used = metrics["memory-used"]
+                mem_total = info["memory-total"]
+                memory_max = max(memory_max, mem_total)
+                memory_total += mem_total
+                utilization.append(int(u))
+                memory.append(mem_used)
+                worker.append(ws.address)
+                gpu_index.append(idx)
+                y.append(idx)
 
             memory_text = [format_bytes(m) for m in memory]
 
diff --git a/distributed/diagnostics/nvml.py b/distributed/diagnostics/nvml.py
index a96a5547598..c1bbb4161a8 100644
--- a/distributed/diagnostics/nvml.py
+++ b/distributed/diagnostics/nvml.py
@@ -1,28 +1,48 @@
+import os
 import pynvml
 
-handles = None
+nvmlInit = None
+
+
+def init_once():
+    global nvmlInit
+    if nvmlInit is not None:
+        return
+
+    from pynvml import nvmlInit as _nvmlInit
+
+    nvmlInit = _nvmlInit
+    nvmlInit()
 
 
 def _pynvml_handles():
-    global handles
-    if handles is None:
-        pynvml.nvmlInit()
-        count = pynvml.nvmlDeviceGetCount()
-        handles = [pynvml.nvmlDeviceGetHandleByIndex(i) for i in range(count)]
-    return handles
+    count = pynvml.nvmlDeviceGetCount()
+    try:
+        cuda_visible_devices = [
+            int(idx) for idx in os.environ.get("CUDA_VISIBLE_DEVICES", "").split(",")
+        ]
+    except ValueError:
+        # CUDA_VISIBLE_DEVICES is not set
+        cuda_visible_devices = False
+    if not cuda_visible_devices:
+        cuda_visible_devices = list(range(count))
+    gpu_idx = cuda_visible_devices[0]
+    return pynvml.nvmlDeviceGetHandleByIndex(gpu_idx)
 
 
 def real_time():
-    handles = _pynvml_handles()
+    init_once()
+    h = _pynvml_handles()
     return {
-        "utilization": [pynvml.nvmlDeviceGetUtilizationRates(h).gpu for h in handles],
-        "memory-used": [pynvml.nvmlDeviceGetMemoryInfo(h).used for h in handles],
+        "utilization": pynvml.nvmlDeviceGetUtilizationRates(h).gpu,
+        "memory-used": pynvml.nvmlDeviceGetMemoryInfo(h).used,
     }
 
 
 def one_time():
-    handles = _pynvml_handles()
+    init_once()
+    h = _pynvml_handles()
     return {
-        "memory-total": [pynvml.nvmlDeviceGetMemoryInfo(h).total for h in handles],
-        "name": [pynvml.nvmlDeviceGetName(h).decode() for h in handles],
+        "memory-total": pynvml.nvmlDeviceGetMemoryInfo(h).total,
+        "name": pynvml.nvmlDeviceGetName(h).decode(),
     }
diff --git a/distributed/diagnostics/tests/test_nvml.py b/distributed/diagnostics/tests/test_nvml.py
new file mode 100644
index 00000000000..820ba57a8a5
--- /dev/null
+++ b/distributed/diagnostics/tests/test_nvml.py
@@ -0,0 +1,34 @@
+import pytest
+import os
+
+pynvml = pytest.importorskip("pynvml")
+
+from distributed.diagnostics import nvml
+
+
+def test_one_time():
+    output = nvml.one_time()
+    assert "memory-total" in output
+    assert "name" in output
+
+    assert len(output["name"]) > 0
+
+
+def test_1_visible_devices():
+    os.environ["CUDA_VISIBLE_DEVICES"] = "0"
+    output = nvml.one_time()
+    assert len(output["memory-total"]) == 1
+
+
+@pytest.mark.parametrize("CVD", ["1,0", "0,1"])
+def test_2_visible_devices(CVD):
+    os.environ["CUDA_VISIBLE_DEVICES"] = CVD
+    idx = int(CVD.split(",")[0])
+
+    h = nvml._pynvml_handles()
+    h2 = pynvml.nvmlDeviceGetHandleByIndex(idx)
+
+    s = pynvml.nvmlDeviceGetSerial(h)
+    s2 = pynvml.nvmlDeviceGetSerial(h2)
+
+    assert s == s2

From 3d409dfd1d3fee77317b15ff7b35cd94d9716d99 Mon Sep 17 00:00:00 2001
From: Gil Forsyth <gforsyth@users.noreply.github.com>
Date: Tue, 6 Oct 2020 10:54:28 -0400
Subject: [PATCH 0997/1550] Fix for missing `:` in autosummary docs (#4143)

Same fix as in dask/dask#6700

Shouldn't be merged until dask/dask-sphinx-theme#43 is merged and a
release is cut
---
 docs/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/requirements.txt b/docs/requirements.txt
index 965f4fc1f9a..ae537b5c474 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -4,5 +4,5 @@ toolz
 cloudpickle
 dask
 sphinx
-dask_sphinx_theme
+dask-sphinx-theme>=1.3.5
 sphinx-click

From b9dd003c0b969cb67177909301c9bd84dc374c34 Mon Sep 17 00:00:00 2001
From: Tom Augspurger <TomAugspurger@users.noreply.github.com>
Date: Tue, 6 Oct 2020 09:56:28 -0500
Subject: [PATCH 0998/1550] Handle exceptions in BatchedSend (#4135)

This changes exception handling in BatchedSend to be more robust to failures.
---
 distributed/batched.py            | 33 +++++++++++++++++--
 distributed/compatibility.py      |  4 +++
 distributed/tests/test_batched.py | 54 +++++++++++++++++++++++++++++++
 3 files changed, 89 insertions(+), 2 deletions(-)

diff --git a/distributed/batched.py b/distributed/batched.py
index eab57c420ef..ad9a26a8113 100644
--- a/distributed/batched.py
+++ b/distributed/batched.py
@@ -54,6 +54,7 @@ def __init__(self, interval, loop=None, serializers=None):
             maxlen=dask.config.get("distributed.comm.recent-messages-log-length")
         )
         self.serializers = serializers
+        self._consecutive_failures = 0
 
     def start(self, comm):
         self.comm = comm
@@ -98,15 +99,43 @@ def _background_send(self):
                     self.recent_message_log.append("large-message")
                 self.byte_count += nbytes
             except CommClosedError as e:
+                # If the comm is known to be closed, we'll immediately
+                # give up.
                 logger.info("Batched Comm Closed: %s", e)
                 break
             except Exception:
-                logger.exception("Error in batched write")
-                break
+                # In other cases we'll retry a few times.
+                # https://github.com/pangeo-data/pangeo/issues/788
+                if self._consecutive_failures <= 5:
+                    logger.warning("Error in batched write, retrying")
+                    yield gen.sleep(0.100 * 1.5 ** self._consecutive_failures)
+                    self._consecutive_failures += 1
+                    # Exponential backoff for retries.
+                    # Ensure we don't drop any messages.
+                    if self.buffer:
+                        # Someone could call send while we yielded above?
+                        self.buffer = payload + self.buffer
+                    else:
+                        self.buffer = payload
+                    continue
+                else:
+                    logger.exception("Error in batched write")
+                    break
             finally:
                 payload = None  # lose ref
+        else:
+            # nobreak. We've been gracefully closed.
+            self.stopped.set()
+            return
 
+        # If we've reached here, it means our comm is known to be closed or
+        # we've repeatedly failed to send a message. We can't close gracefully
+        # via `.close()` since we can't send messages. So we just abort.
+        # This means that any messages in our buffer our lost.
+        # To propagate exceptions, we rely on subsequent `BatchedSend.send`
+        # calls to raise CommClosedErrors.
         self.stopped.set()
+        self.abort()
 
     def send(self, msg):
         """Schedule a message for sending to the other side
diff --git a/distributed/compatibility.py b/distributed/compatibility.py
index 0dca141e0e9..fc64b7568b5 100644
--- a/distributed/compatibility.py
+++ b/distributed/compatibility.py
@@ -2,11 +2,15 @@
 import platform
 import sys
 
+import tornado
+
 logging_names = logging._levelToName.copy()
 logging_names.update(logging._nameToLevel)
 
 PYPY = platform.python_implementation().lower() == "pypy"
 WINDOWS = sys.platform.startswith("win")
+TORNADO6 = tornado.version_info[0] >= 6
+PY37 = sys.version_info[:2] >= (3, 7)
 
 if sys.version_info[:2] >= (3, 7):
     from asyncio import get_running_loop
diff --git a/distributed/tests/test_batched.py b/distributed/tests/test_batched.py
index a288a25bbb9..93abdd98159 100644
--- a/distributed/tests/test_batched.py
+++ b/distributed/tests/test_batched.py
@@ -1,5 +1,6 @@
 import asyncio
 import random
+from unittest import mock
 
 import pytest
 from tlz import assoc
@@ -10,6 +11,7 @@
 from distributed.utils import All, TimeoutError
 from distributed.utils_test import captured_logger
 from distributed.protocol import to_serialize
+from distributed.compatibility import WINDOWS, PY37, TORNADO6
 
 
 class EchoServer:
@@ -253,3 +255,55 @@ async def test_serializers():
 
         with pytest.raises(TimeoutError):
             msg = await asyncio.wait_for(comm.read(), 0.1)
+
+
+@pytest.mark.asyncio
+@pytest.mark.skipif(
+    WINDOWS and not PY37 and not TORNADO6, reason="failing on windows, py36, tornado 5."
+)
+async def test_handles_exceptions():
+    # Ensure that we properly handle exceptions in BatchedSend.
+    # https://github.com/pangeo-data/pangeo/issues/788
+    # mentioned in https://github.com/dask/distributed/issues/4080, but
+    # possibly distinct.
+    #
+    # The reported issues (https://github.com/tornadoweb/tornado/pull/2008)
+    # claim that the BufferError *should* only happen when the application
+    # is incorrectly using threads. I haven't been able to construct an
+    # actual example, so we mock IOStream.write to raise and ensure that
+    # BufferedSend handles things correctly. We don't (yet) test that
+    # any *users* of BatchedSend correctly handle BatchedSend dropping
+    # messages.
+    async with EchoServer() as e:
+        comm = await connect(e.address)
+        b = BatchedSend(interval=10)
+        b.start(comm)
+        await asyncio.sleep(0.020)
+        orig = comm.stream.write
+
+        n = 0
+
+        def raise_buffererror(*args, **kwargs):
+            nonlocal n
+            n += 1
+
+            if n == 1:
+                raise BufferError("bad!")
+            elif n == 2:
+                orig(*args, **kwargs)
+            else:
+                raise CommClosedError
+
+        with mock.patch.object(comm.stream, "write", wraps=raise_buffererror):
+            b.send("hello")
+            b.send("hello")
+            b.send("world")
+            await asyncio.sleep(0.020)
+            result = await comm.read()
+            assert result == ("hello", "hello", "world")
+
+            b.send("raises when flushed")
+            await asyncio.sleep(0.020)  # CommClosedError hit in callback
+
+            with pytest.raises(CommClosedError):
+                b.send("raises when sent")

From a7c2895c267ccc931ea79fd5de90a9869d5a7c7e Mon Sep 17 00:00:00 2001
From: "Mads R. B. Kristensen" <madsbk@gmail.com>
Date: Tue, 6 Oct 2020 18:59:16 +0200
Subject: [PATCH 0999/1550] Support SubgraphCallable in str_graph() (#4148)

Fixes #4145
---
 distributed/tests/test_client.py | 25 +++++++++++++++++++++++++
 distributed/tests/test_utils.py  |  6 ++++++
 distributed/utils.py             | 23 +++++++++++++++++++----
 3 files changed, 50 insertions(+), 4 deletions(-)

diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 846e098a11d..2230077b8c2 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -6165,3 +6165,28 @@ async def test_mixed_compression(cleanup):
                     x = da.ones((10000, 10000))
                     y = x + x.T
                     await c.compute(y.sum())
+
+
+@gen_cluster(client=True)
+async def test_futures_in_subgraphs(c, s, a, b):
+    """Regression test of <https://github.com/dask/distributed/issues/4145>"""
+
+    dd = pytest.importorskip("dask.dataframe")
+    import pandas as pd
+
+    ddf = dd.from_pandas(
+        pd.DataFrame(
+            dict(
+                uid=range(50),
+                enter_time=pd.date_range(
+                    start="2020-01-01", end="2020-09-01", periods=50, tz="UTC"
+                ),
+            )
+        ),
+        npartitions=5,
+    )
+
+    ddf = ddf[ddf.uid.isin(range(29))].persist()
+    ddf["local_time"] = ddf.enter_time.dt.tz_convert("US/Central")
+    ddf["day"] = ddf.enter_time.dt.day_name()
+    ddf = await c.submit(dd.categorical.categorize, ddf, columns=["day"], index=False)
diff --git a/distributed/tests/test_utils.py b/distributed/tests/test_utils.py
index 74cd051ff81..2d96a63d048 100644
--- a/distributed/tests/test_utils.py
+++ b/distributed/tests/test_utils.py
@@ -49,6 +49,7 @@
 )
 from distributed.utils_test import loop, loop_in_thread  # noqa: F401
 from distributed.utils_test import div, has_ipv6, inc, throws, gen_test, captured_logger
+from dask.optimization import SubgraphCallable
 
 
 def test_All(loop):
@@ -217,6 +218,11 @@ def test_str_graph():
         assert all(isinstance(k, str) for k in sdsk)
         assert dask.get(dsk, keys) == dask.get(sdsk, skeys)
 
+    dsk = {("y", 1): (SubgraphCallable({"x": ("y", 1)}, "x", (("y", 1),)), (("z", 1),))}
+    dsk = str_graph(dsk, extra_values=(("z", 1),))
+    assert dsk["('y', 1)"][0].dsk["x"] == "('y', 1)"
+    assert dsk["('y', 1)"][1][0] == "('z', 1)"
+
 
 def test_maybe_complex():
     assert not _maybe_complex(1)
diff --git a/distributed/utils.py b/distributed/utils.py
index f641a1bf2a0..77487f8cec6 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -36,6 +36,7 @@
 
 import dask
 from dask import istask
+from dask.optimization import SubgraphCallable
 
 # provide format_bytes here for backwards compatibility
 from dask.utils import (  # noqa
@@ -777,17 +778,31 @@ def _maybe_complex(task):
 
 
 def convert(task, dsk, extra_values):
-    if type(task) is list:
+    typ = type(task)
+    if typ is tuple and task:
+        if type(task[0]) is SubgraphCallable:
+            sc = task[0]
+            return (
+                SubgraphCallable(
+                    convert(sc.dsk, dsk, extra_values),
+                    sc.outkey,
+                    convert(sc.inkeys, dsk, extra_values),
+                    sc.name,
+                ),
+            ) + tuple(convert(x, dsk, extra_values) for x in task[1:])
+        elif callable(task[0]):
+            return (task[0],) + tuple(convert(x, dsk, extra_values) for x in task[1:])
+    if typ is list:
         return [convert(v, dsk, extra_values) for v in task]
-    if type(task) is dict:
+    if typ is dict:
         return {k: convert(v, dsk, extra_values) for k, v in task.items()}
-    if istask(task):
-        return (task[0],) + tuple(convert(x, dsk, extra_values) for x in task[1:])
     try:
         if task in dsk or task in extra_values:
             return tokey(task)
     except TypeError:
         pass
+    if typ is tuple:  # If the tuple itself isn't a key, check its elements
+        return tuple(convert(v, dsk, extra_values) for v in task)
     return task
 
 

From a1dc5f437b39c1b35a9b05cbc048e3a793b89715 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Tue, 6 Oct 2020 17:35:34 -0500
Subject: [PATCH 1000/1550] bump version to 2.30.0

---
 docs/source/changelog.rst | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index 3ae42669b91..dcfb8b455f3 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,15 @@
 Changelog
 =========
 
+2.30.0 - 2020-10-06
+-------------------
+
+- Support ``SubgraphCallable`` in ``str_graph()`` (:pr:`4148`) `Mads R. B. Kristensen`_
+- Handle exceptions in ``BatchedSend`` (:pr:`4135`) `Tom Augspurger`_
+- Fix for missing ``:`` in autosummary docs (:pr:`4143`) `Gil Forsyth`_
+- Limit GPU metrics to visible devices only (:pr:`3810`) `Jacob Tomlinson`_
+
+
 2.29.0 - 2020-10-02
 -------------------
 

From d35628ea33092a2a0add89ae7ce9dbaa7c82c842 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Wed, 7 Oct 2020 05:54:59 -0700
Subject: [PATCH 1001/1550] Fix `test_gpu_metrics` failure (#4154)

* Import `_pynvml_handles`

There is no `handles` in `distributed.diagnostics.nvml`. Though there is
`_pynvml_handles`. So try `import`ing that.

* Test attributes of single handle

* Run `black`

* Consolidate NVML tests

* Access `_pynvml_handles` via `nvml`

* Fix NVML memory total test as well

* Skip test needing 2 GPUs on single GPU

* Workaround pyNVML initialization

Skip inside the test to make sure pyNVML is already initialized first.
---
 distributed/diagnostics/tests/test_nvml.py | 23 +++++++++++++++++++++-
 distributed/tests/test_gpu_metrics.py      | 14 -------------
 2 files changed, 22 insertions(+), 15 deletions(-)
 delete mode 100644 distributed/tests/test_gpu_metrics.py

diff --git a/distributed/diagnostics/tests/test_nvml.py b/distributed/diagnostics/tests/test_nvml.py
index 820ba57a8a5..6182049fe40 100644
--- a/distributed/diagnostics/tests/test_nvml.py
+++ b/distributed/diagnostics/tests/test_nvml.py
@@ -4,6 +4,7 @@
 pynvml = pytest.importorskip("pynvml")
 
 from distributed.diagnostics import nvml
+from distributed.utils_test import gen_cluster
 
 
 def test_one_time():
@@ -17,11 +18,15 @@ def test_one_time():
 def test_1_visible_devices():
     os.environ["CUDA_VISIBLE_DEVICES"] = "0"
     output = nvml.one_time()
-    assert len(output["memory-total"]) == 1
+    h = nvml._pynvml_handles()
+    assert output["memory-total"] == pynvml.nvmlDeviceGetMemoryInfo(h).total
 
 
 @pytest.mark.parametrize("CVD", ["1,0", "0,1"])
 def test_2_visible_devices(CVD):
+    if pynvml.nvmlDeviceGetCount() <= 1:
+        pytest.skip("Machine only has a single GPU")
+
     os.environ["CUDA_VISIBLE_DEVICES"] = CVD
     idx = int(CVD.split(",")[0])
 
@@ -32,3 +37,19 @@ def test_2_visible_devices(CVD):
     s2 = pynvml.nvmlDeviceGetSerial(h2)
 
     assert s == s2
+
+
+@gen_cluster()
+async def test_gpu_metrics(s, a, b):
+    h = nvml._pynvml_handles()
+
+    assert "gpu" in a.metrics
+    assert (
+        s.workers[a.address].metrics["gpu"]["memory-used"]
+        == pynvml.nvmlDeviceGetMemoryInfo(h).used
+    )
+    assert "gpu" in a.startup_information
+    assert (
+        s.workers[a.address].extra["gpu"]["name"]
+        == pynvml.nvmlDeviceGetName(h).decode()
+    )
diff --git a/distributed/tests/test_gpu_metrics.py b/distributed/tests/test_gpu_metrics.py
deleted file mode 100644
index 584932988dd..00000000000
--- a/distributed/tests/test_gpu_metrics.py
+++ /dev/null
@@ -1,14 +0,0 @@
-import pytest
-from distributed.utils_test import gen_cluster
-
-pytest.importorskip("pynvml")
-
-
-@gen_cluster()
-async def test_gpu_metrics(s, a, b):
-    from distributed.diagnostics.nvml import handles
-
-    assert "gpu" in a.metrics
-    assert len(s.workers[a.address].metrics["gpu"]["memory-used"]) == len(handles)
-    assert "gpu" in a.startup_information
-    assert len(s.workers[a.address].extra["gpu"]["name"]) == len(handles)

From d51ed56e45488d3ec923bda25faa676375736dcc Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 8 Oct 2020 09:57:34 -0700
Subject: [PATCH 1002/1550] Log KilledWorker events in the scheduler (#4157)

---
 distributed/scheduler.py         | 6 ++++++
 distributed/tests/test_client.py | 8 ++++++--
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index f2d4de8b16f..bdae660d9af 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -2261,6 +2261,12 @@ async def remove_worker(self, comm=None, address=None, safe=False, close=True):
                         )
                         r = self.transition(k, "erred", exception=e, cause=k)
                         recommendations.update(r)
+                        logger.info(
+                            "Task %s marked as failed because %d workers died"
+                            " while trying to run it",
+                            ts.key,
+                            self.allowed_failures,
+                        )
 
             for ts in ws.has_what:
                 ts.who_has.remove(ws)
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 2230077b8c2..a9e2429f9f2 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -3404,8 +3404,12 @@ async def test_get_foo_lost_keys(c, s, u, v, w):
 )
 async def test_bad_tasks_fail(c, s, a, b):
     f = c.submit(sys.exit, 0)
-    with pytest.raises(KilledWorker) as info:
-        await f
+    with captured_logger(logging.getLogger("distributed.scheduler")) as logger:
+        with pytest.raises(KilledWorker) as info:
+            await f
+
+    text = logger.getvalue()
+    assert f.key in text
 
     assert info.value.last_worker.nanny in {a.address, b.address}
     await asyncio.gather(a.close(), b.close())

From 946c6a46abb04615051537f1bf39b81163aa91f0 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 9 Oct 2020 10:53:06 -0700
Subject: [PATCH 1003/1550] Add PipInstall WorkerPlugin (#3216)

---
 distributed/__init__.py           |  2 +-
 distributed/diagnostics/plugin.py | 78 +++++++++++++++++++++++++++++++
 distributed/tests/test_worker.py  | 49 +++++++++++++++++++
 distributed/worker.py             |  7 ++-
 docs/source/plugins.rst           | 11 +++++
 5 files changed, 144 insertions(+), 3 deletions(-)

diff --git a/distributed/__init__.py b/distributed/__init__.py
index e7e74c2b2e7..0f6858fc9c9 100644
--- a/distributed/__init__.py
+++ b/distributed/__init__.py
@@ -4,7 +4,7 @@
 from .core import connect, rpc
 from .deploy import LocalCluster, Adaptive, SpecCluster, SSHCluster
 from .diagnostics.progressbar import progress
-from .diagnostics.plugin import WorkerPlugin, SchedulerPlugin
+from .diagnostics.plugin import WorkerPlugin, SchedulerPlugin, PipInstall
 from .client import (
     Client,
     Executor,
diff --git a/distributed/diagnostics/plugin.py b/distributed/diagnostics/plugin.py
index 2e9cb31ba14..b45e9f25684 100644
--- a/distributed/diagnostics/plugin.py
+++ b/distributed/diagnostics/plugin.py
@@ -1,4 +1,7 @@
 import logging
+import socket
+import subprocess
+import sys
 
 logger = logging.getLogger(__name__)
 
@@ -180,3 +183,78 @@ def release_dep(self, dep, state, report):
         report: bool
             Whether the worker should report the released dependency to the scheduler.
         """
+
+
+class PipInstall(WorkerPlugin):
+    """A Worker Plugin to pip install a set of packages
+
+    This accepts a set of packages to install on all workers.
+    You can also optionally ask for the worker to restart itself after
+    performing this installation.
+
+    .. note::
+
+       This will increase the time it takes to start up
+       each worker. If possible, we recommend including the
+       libraries in the worker environment or image. This is
+       primarily intended for experimentation and debugging.
+
+       Additional issues may arise if multiple workers share the same
+       file system. Each worker might try to install the packages
+       simultaneously.
+
+    Parameters
+    ----------
+    packages : List[str]
+        A list of strings to place after "pip install" command
+    pip_options : List[str]
+        Additional options to pass to pip.
+    restart : bool, default False
+        Whether or not to restart the worker after pip installing
+        Only functions if the worker has an attached nanny process
+
+    Examples
+    --------
+    >>> from dask.distributed import PipInstall
+    >>> plugin = PipInstall(packages=["scikit-learn"], pip_options=["--upgrade"])
+
+    >>> client.register_worker_plugin(plugin)
+    """
+
+    name = "pip"
+
+    def __init__(self, packages, pip_options=None, restart=False):
+        self.packages = packages
+        self.restart = restart
+        if pip_options is None:
+            pip_options = []
+        self.pip_options = pip_options
+
+    async def setup(self, worker):
+        from ..lock import Lock
+
+        async with Lock(socket.gethostname()):  # don't clobber one installation
+            logger.info("Pip installing the following packages: %s", self.packages)
+            proc = subprocess.Popen(
+                [sys.executable, "-m", "pip"]
+                + self.pip_options
+                + ["install"]
+                + self.packages,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+            )
+            stdout, stderr = proc.communicate()
+            returncode = proc.wait()
+
+            if returncode:
+                logger.error("Pip install failed with '%s'", stderr.decode().strip())
+                return
+
+            if self.restart and worker.nanny:
+                lines = stdout.strip().split(b"\n")
+                if not all(
+                    line.startswith(b"Requirement already satisfied") for line in lines
+                ):
+                    worker.loop.add_callback(
+                        worker.close_gracefully, restart=True
+                    )  # restart
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 42f9236f2d5..68f1958f5f7 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -8,6 +8,7 @@
 import sys
 from time import sleep
 import traceback
+from unittest import mock
 import asyncio
 
 import dask
@@ -26,6 +27,7 @@
     Reschedule,
     wait,
 )
+from distributed.diagnostics.plugin import PipInstall
 from distributed.compatibility import WINDOWS
 from distributed.core import rpc, CommClosedError, Status
 from distributed.scheduler import Scheduler
@@ -1614,6 +1616,53 @@ def bad_startup(w):
             pytest.fail("Startup exception was raised")
 
 
+@gen_cluster(client=True)
+async def test_pip_install(c, s, a, b):
+    with mock.patch(
+        "distributed.diagnostics.plugin.subprocess.Popen.communicate",
+        return_value=(b"", b""),
+    ) as p1:
+        with mock.patch(
+            "distributed.diagnostics.plugin.subprocess.Popen", return_value=p1
+        ) as p2:
+            p1.communicate.return_value = b"", b""
+            p1.wait.return_value = 0
+            await c.register_worker_plugin(
+                PipInstall(packages=["requests"], pip_options=["--upgrade"])
+            )
+
+            args = p2.call_args[0][0]
+            assert "python" in args[0]
+            assert args[1:] == ["-m", "pip", "--upgrade", "install", "requests"]
+
+
+@gen_cluster(client=True)
+async def test_pip_install_fails(c, s, a, b):
+    with captured_logger(
+        "distributed.diagnostics.plugin", level=logging.ERROR
+    ) as logger:
+        with mock.patch(
+            "distributed.diagnostics.plugin.subprocess.Popen.communicate",
+            return_value=(b"", b"error"),
+        ) as p1:
+            with mock.patch(
+                "distributed.diagnostics.plugin.subprocess.Popen", return_value=p1
+            ) as p2:
+                p1.communicate.return_value = (
+                    b"",
+                    b"Could not find a version that satisfies the requirement not-a-package",
+                )
+                p1.wait.return_value = 1
+                await c.register_worker_plugin(PipInstall(packages=["not-a-package"]))
+
+                assert "not-a-package" in logger.getvalue()
+
+
+#             args = p2.call_args[0][0]
+#             assert "python" in args[0]
+#             assert args[1:] == ["-m", "pip", "--upgrade", "install", "requests"]
+
+
 @pytest.mark.asyncio
 async def test_update_latency(cleanup):
     async with await Scheduler() as s:
diff --git a/distributed/worker.py b/distributed/worker.py
index 2fabf256ddf..16a70db9663 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -1176,7 +1176,7 @@ async def close(
             setproctitle("dask-worker [closed]")
         return "OK"
 
-    async def close_gracefully(self):
+    async def close_gracefully(self, restart=None):
         """Gracefully shut down a worker
 
         This first informs the scheduler that we're shutting down, and asks it
@@ -1188,10 +1188,13 @@ async def close_gracefully(self):
         if self.status == Status.closed:
             return
 
+        if restart is None:
+            restart = self.lifetime_restart
+
         logger.info("Closing worker gracefully: %s", self.address)
         self.status = Status.closing_gracefully
         await self.scheduler.retire_workers(workers=[self.address], remove=False)
-        await self.close(safe=True, nanny=not self.lifetime_restart)
+        await self.close(safe=True, nanny=not restart)
 
     async def terminate(self, comm=None, report=True, **kwargs):
         await self.close(report=report, **kwargs)
diff --git a/docs/source/plugins.rst b/docs/source/plugins.rst
index 5c831fc167e..0ed63e01d21 100644
--- a/docs/source/plugins.rst
+++ b/docs/source/plugins.rst
@@ -77,5 +77,16 @@ the scheduler as so:
 Worker Plugins
 =================
 
+:class:`distributed.diagnostics.plugin.WorkerPlugin` provides a base class
+for creating your own worker plugins. In addition, Dask provides some
+:ref:`built-in plugins <plugins.builtin>`.
+
 .. autoclass:: distributed.diagnostics.plugin.WorkerPlugin
    :members:
+
+.. _plugins.builtin:
+
+Built-In Worker Plugins
+-----------------------
+
+.. autoclass:: distributed.diagnostics.plugin.PipInstall

From 785500b6ff1244e22f08051b391e425262c1ac51 Mon Sep 17 00:00:00 2001
From: Krishan Bhasin <8904718+KrishanBhasin@users.noreply.github.com>
Date: Mon, 12 Oct 2020 18:28:40 +0000
Subject: [PATCH 1004/1550] Set Zict buffer target to maxsize when
 `memory_target_fraction` is False (#4156)

---
 distributed/worker.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/distributed/worker.py b/distributed/worker.py
index 16a70db9663..96cdf9e87e0 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -557,7 +557,10 @@ def __init__(
                 deserialize_bytes,
                 File(path),
             )
-            target = int(float(self.memory_limit) * self.memory_target_fraction)
+            target = (
+                int(float(self.memory_limit) * self.memory_target_fraction)
+                or sys.maxsize
+            )
             self.data = Buffer({}, storage, target, weight)
             self.data.memory = self.data.fast
             self.data.disk = self.data.slow

From 849650e49c84b2499c78b04926acf4af460fb46d Mon Sep 17 00:00:00 2001
From: Tom Augspurger <TomAugspurger@users.noreply.github.com>
Date: Wed, 14 Oct 2020 12:56:16 -0500
Subject: [PATCH 1005/1550] TST/CI: Skip batched comm test win / tornado5
 (#4166)

Previously we only skipped on Python 3.6. Now we skip on any Python
version on Windows with Tornado 5.

I can reproduce the issue if people want me to look more, but I don't
think this is high priority.

Closes https://github.com/dask/distributed/issues/4163.
---
 distributed/tests/test_batched.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/distributed/tests/test_batched.py b/distributed/tests/test_batched.py
index 93abdd98159..fa12f9649e9 100644
--- a/distributed/tests/test_batched.py
+++ b/distributed/tests/test_batched.py
@@ -11,7 +11,7 @@
 from distributed.utils import All, TimeoutError
 from distributed.utils_test import captured_logger
 from distributed.protocol import to_serialize
-from distributed.compatibility import WINDOWS, PY37, TORNADO6
+from distributed.compatibility import WINDOWS, TORNADO6
 
 
 class EchoServer:
@@ -258,9 +258,7 @@ async def test_serializers():
 
 
 @pytest.mark.asyncio
-@pytest.mark.skipif(
-    WINDOWS and not PY37 and not TORNADO6, reason="failing on windows, py36, tornado 5."
-)
+@pytest.mark.skipif(WINDOWS and not TORNADO6, reason="failing on windows, tornado 5.")
 async def test_handles_exceptions():
     # Ensure that we properly handle exceptions in BatchedSend.
     # https://github.com/pangeo-data/pangeo/issues/788

From 2b43b402c19b278a1e2fe7e1bf7718452b2fe007 Mon Sep 17 00:00:00 2001
From: Lucas Rademaker <44430780+lr4d@users.noreply.github.com>
Date: Thu, 15 Oct 2020 09:56:01 +0200
Subject: [PATCH 1006/1550] Increase robustness of Semaphore.release (#4151)

This adds retries to the semaphore release method, which must be
configured using "distributed.comm.retry.count". This may come to
use when the scheduler is under high load and some release
calls may fail.
This PR also avoids raising an exception in case a release fails,
instead a warning is logged at ERROR level, and the Semaphore.release
method will return `False`. The lease will eventually be cleaned up
during the semaphore lease validation check, which may be configured using
"distributed.scheduler.locks.lease-validation-interval" and
""distributed.scheduler.locks.lease-timeout".


Co-authored-by: lr4d <lr4d@users.noreply.github.com>
---
 .../scheduler/tests/test_semaphore_http.py    |   2 +-
 distributed/semaphore.py                      |  46 ++++++--
 distributed/tests/test_semaphore.py           | 100 ++++++++++++++++--
 3 files changed, 126 insertions(+), 22 deletions(-)

diff --git a/distributed/http/scheduler/tests/test_semaphore_http.py b/distributed/http/scheduler/tests/test_semaphore_http.py
index a5d5e7c37bc..4c66165b985 100644
--- a/distributed/http/scheduler/tests/test_semaphore_http.py
+++ b/distributed/http/scheduler/tests/test_semaphore_http.py
@@ -63,7 +63,7 @@ async def fetch_metrics():
     assert active_metrics["semaphore_release"].samples[0].value == 0
     assert active_metrics["semaphore_pending_leases"].samples[0].value == 0
 
-    await sem.release()
+    assert await sem.release() is True
     active_metrics = await fetch_metrics()
     assert active_metrics["semaphore_max_leases"].samples[0].value == 2
     assert active_metrics["semaphore_active_leases"].samples[0].value == 0
diff --git a/distributed/semaphore.py b/distributed/semaphore.py
index 6ff66eac811..1ece16276de 100644
--- a/distributed/semaphore.py
+++ b/distributed/semaphore.py
@@ -437,6 +437,8 @@ async def _acquire(self, timeout=None):
             name=self.name,
             timeout=timeout,
             lease_id=lease_id,
+            operation="semaphore acquire: client=%s, lease_id=%s, name=%s"
+            % (self.client.id, lease_id, self.name),
         )
         if result:
             self._leases.append(lease_id)
@@ -460,22 +462,44 @@ def acquire(self, timeout=None):
         timeout = parse_timedelta(timeout)
         return self.client.sync(self._acquire, timeout=timeout)
 
+    async def _release(self):
+        # popleft to release the oldest lease first
+        lease_id = self._leases.popleft()
+        logger.info("%s releases %s for %s", self.client.id, lease_id, self.name)
+
+        try:
+            await retry_operation(
+                self.client.scheduler.semaphore_release,
+                name=self.name,
+                lease_id=lease_id,
+                operation="semaphore release: client=%s, lease_id=%s, name=%s"
+                % (self.client.id, lease_id, self.name),
+            )
+            return True
+        except Exception:  # Release fails for whatever reason
+            logger.error(
+                "Release failed for client=%s, lease_id=%s, name=%s. Cluster network might be unstable?"
+                % (self.client.id, lease_id, self.name),
+                exc_info=True,
+            )
+            return False
+
     def release(self):
         """
-        Release a semaphore.
-
-        Increment the internal counter by one.
+        Release the semaphore.
+
+        Returns
+        -------
+        bool
+            This value indicates whether a lease was released immediately or not. Note that a user should  *not* retry
+            this operation. Under certain circumstances (e.g. scheduler overload) the lease may not be released
+            immediately, but it will always be automatically released after a specific interval configured using
+            "distributed.scheduler.locks.lease-validation-interval" and "distributed.scheduler.locks.lease-timeout".
         """
-
-        """ Release the lock if already acquired """
         if not self._leases:
             raise RuntimeError("Released too often")
-        # popleft to release the oldest lease first
-        lease_id = self._leases.popleft()
-        logger.info("%s releases %s for %s", self.client.id, lease_id, self.name)
-        return self.client.sync(
-            self.client.scheduler.semaphore_release, name=self.name, lease_id=lease_id
-        )
+
+        return self.client.sync(self._release)
 
     def get_value(self):
         """
diff --git a/distributed/tests/test_semaphore.py b/distributed/tests/test_semaphore.py
index 7803d513962..5df270a0a5a 100644
--- a/distributed/tests/test_semaphore.py
+++ b/distributed/tests/test_semaphore.py
@@ -20,6 +20,7 @@
     slowidentity,
     loop,
 )
+import logging
 
 
 @gen_cluster(client=True)
@@ -81,9 +82,9 @@ async def test_acquires_with_timeout(c, s, a, b):
     sem = await Semaphore(1, "x")
     assert await sem.acquire(timeout="25ms")
     assert not await sem.acquire(timeout=0.025)
-    await sem.release()
+    assert await sem.release() is True
     assert await sem.acquire(timeout=timedelta(seconds=0.025))
-    await sem.release()
+    assert await sem.release() is True
 
 
 def test_timeout_sync(client):
@@ -166,7 +167,7 @@ def f(x, release=True):
         if not sem.acquire(timeout=0.1):
             return False
         if release:
-            sem.release()
+            assert sem.release() is True
 
         return True
 
@@ -182,7 +183,7 @@ def f(x, release=True):
     assert len(s.extensions["semaphores"].leases["x"]) == 1
     futures = c.map(f, list(range(10)))
     assert not any(await c.gather(futures))
-    await sem.release()
+    assert await sem.release() is True
 
     del futures
 
@@ -242,20 +243,20 @@ def test_close_sync(client):
 async def test_release_once_too_many(c, s, a, b):
     sem = await Semaphore(name="x")
     assert await sem.acquire()
-    await sem.release()
+    assert await sem.release() is True
 
     with pytest.raises(RuntimeError, match="Released too often"):
-        await sem.release()
+        sem.release()
 
     assert await sem.acquire()
-    await sem.release()
+    assert await sem.release() is True
 
 
 @gen_cluster(client=True)
 async def test_release_once_too_many_resilience(c, s, a, b):
     def f(x, sem):
         sem.acquire()
-        sem.release()
+        assert sem.release() is True
         with pytest.raises(RuntimeError, match="Released too often"):
             sem.release()
         return x
@@ -301,6 +302,9 @@ def __init__(self, *args, failing_connections=0, **kwargs):
     def activate(self):
         self._flaky_active = True
 
+    def deactivate(self):
+        self._flaky_active = False
+
     async def connect(self, *args, **kwargs):
         if self.cnn_count >= self.failing_connections or not self._flaky_active:
             return await super().connect(*args, **kwargs)
@@ -449,7 +453,7 @@ async def test_timeout_zero(c, s, a, b):
 
     assert await sem.acquire(timeout=0)
     assert not await sem.acquire(timeout=0)
-    await sem.release()
+    assert await sem.release() is True
 
 
 @gen_cluster(client=True)
@@ -460,7 +464,7 @@ async def test_getvalue(c, s, a, b):
     assert await sem.get_value() == 0
     await sem.acquire()
     assert await sem.get_value() == 1
-    await sem.release()
+    assert await sem.release() is True
     assert await sem.get_value() == 0
 
 
@@ -518,3 +522,79 @@ def access_limited(val, sem):
                     protected_ressource.remove(val)
 
             client.gather(client.map(access_limited, range(10), sem=sem))
+
+
+@gen_cluster(client=True)
+async def test_release_retry(c, s, a, b):
+    """Verify that we can properly retry a semaphore release operation"""
+    with dask.config.set({"distributed.comm.retry.count": 1}):
+        pool = await FlakyConnectionPool(failing_connections=1)
+        rpc = pool(s.address)
+        c.scheduler = rpc
+        semaphore = await Semaphore(
+            max_leases=2, name="resource_we_want_to_limit", client=c
+        )
+        await semaphore.acquire()
+        pool.activate()  # Comm chaos starts
+        with captured_logger("distributed.utils_comm") as caplog:
+            assert await semaphore.release() is True
+        logs = caplog.getvalue().split("\n")
+        log = logs[0]
+        assert log.startswith("Retrying semaphore release:") and log.endswith(
+            "after exception in attempt 0/1: "
+        )
+
+        assert await semaphore.acquire() is True
+        assert await semaphore.release() is True
+
+
+@gen_cluster(
+    client=True,
+    config={
+        "distributed.scheduler.locks.lease-timeout": "100ms",
+        "distributed.scheduler.locks.lease-validation-interval": "100ms",
+    },
+)
+async def test_release_failure(c, s, a, b):
+    """Don't raise even if release fails: lease will be cleaned up by the lease-validation after
+    a specified interval anyways (see config parameters used)."""
+
+    with dask.config.set({"distributed.comm.retry.count": 1}):
+        pool = await FlakyConnectionPool(failing_connections=5)
+        rpc = pool(s.address)
+        c.scheduler = rpc
+        semaphore = await Semaphore(
+            max_leases=2, name="resource_we_want_to_limit", client=c
+        )
+        await semaphore.acquire()
+        pool.activate()  # Comm chaos starts
+
+        # Release fails (after a single retry) because of broken connections
+        with captured_logger(
+            "distributed.semaphore", level=logging.ERROR
+        ) as semaphore_log:
+            with captured_logger("distributed.utils_comm") as retry_log:
+                assert await semaphore.release() is False
+
+        with captured_logger("distributed.semaphore") as semaphore_cleanup_log:
+            pool.deactivate()  # comm chaos stops
+            assert await semaphore.get_value() == 1  # lease is still registered
+            await asyncio.sleep(0.2)  # Wait for lease to be cleaned up
+
+        # Check release was retried
+        retry_log = retry_log.getvalue().split("\n")[0]
+        assert retry_log.startswith(
+            "Retrying semaphore release:"
+        ) and retry_log.endswith("after exception in attempt 0/1: ")
+        # Check release failed
+        semaphore_log = semaphore_log.getvalue().split("\n")[0]
+        assert semaphore_log.startswith(
+            "Release failed for client="
+        ) and semaphore_log.endswith("Cluster network might be unstable?")
+
+        # Check lease has timed out
+        assert any(
+            log.startswith("Lease") and "timed out after" in log
+            for log in semaphore_cleanup_log.getvalue().split("\n")
+        )
+        assert await semaphore.get_value() == 0

From dcb46d07318282d8a8976aadb0e4003144a42ae7 Mon Sep 17 00:00:00 2001
From: Gil Forsyth <gforsyth@users.noreply.github.com>
Date: Thu, 15 Oct 2020 13:15:38 -0400
Subject: [PATCH 1007/1550] Add TaskState to worker.py (#4107)

Fixes #4097

This replaces various state dictionaries with a single TaskState class
---
 distributed/dashboard/components/worker.py    |    6 +-
 .../dashboard/tests/test_scheduler_bokeh.py   |    4 +-
 .../diagnostics/tests/test_worker_plugin.py   |    7 +-
 distributed/http/worker/prometheus.py         |    4 +-
 distributed/tests/test_client.py              |   15 +-
 distributed/tests/test_failed_workers.py      |    7 +-
 distributed/tests/test_resources.py           |    6 +-
 distributed/tests/test_scheduler.py           |   10 +-
 distributed/tests/test_steal.py               |   88 +-
 distributed/tests/test_worker.py              |   47 +-
 distributed/worker.py                         | 1153 ++++++++---------
 distributed/worker_client.py                  |    4 +-
 docs/source/worker.rst                        |    1 +
 13 files changed, 648 insertions(+), 704 deletions(-)

diff --git a/distributed/dashboard/components/worker.py b/distributed/dashboard/components/worker.py
index a394df3f5cb..ee9ad65d2e1 100644
--- a/distributed/dashboard/components/worker.py
+++ b/distributed/dashboard/components/worker.py
@@ -72,9 +72,9 @@ def update(self):
             w = self.worker
             d = {
                 "Stored": [len(w.data)],
-                "Executing": ["%d / %d" % (len(w.executing), w.nthreads)],
+                "Executing": ["%d / %d" % (w.executing_count, w.nthreads)],
                 "Ready": [len(w.ready)],
-                "Waiting": [len(w.waiting_for_data)],
+                "Waiting": [w.waiting_for_data_count],
                 "Connections": [len(w.in_flight_workers)],
                 "Serving": [len(w._comms)],
             }
@@ -265,7 +265,7 @@ def __init__(self, worker, **kwargs):
     def update(self):
         with log_errors():
             self.source.stream(
-                {"x": [time() * 1000], "y": [len(self.worker.executing)]}, 1000
+                {"x": [time() * 1000], "y": [self.worker.executing_count]}, 1000
             )
 
 
diff --git a/distributed/dashboard/tests/test_scheduler_bokeh.py b/distributed/dashboard/tests/test_scheduler_bokeh.py
index e79f3494279..5f192a75b87 100644
--- a/distributed/dashboard/tests/test_scheduler_bokeh.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh.py
@@ -101,7 +101,7 @@ async def test_stealing_events(c, s, a, b):
         slowinc, range(100), delay=0.1, workers=a.address, allow_other_workers=True
     )
 
-    while not b.task_state:  # will steal soon
+    while not b.tasks:  # will steal soon
         await asyncio.sleep(0.01)
 
     se.update()
@@ -117,7 +117,7 @@ async def test_events(c, s, a, b):
         slowinc, range(100), delay=0.1, workers=a.address, allow_other_workers=True
     )
 
-    while not b.task_state:
+    while not b.tasks:
         await asyncio.sleep(0.01)
 
     e.update()
diff --git a/distributed/diagnostics/tests/test_worker_plugin.py b/distributed/diagnostics/tests/test_worker_plugin.py
index 2f17a86a8e7..bc34b61e5a9 100644
--- a/distributed/diagnostics/tests/test_worker_plugin.py
+++ b/distributed/diagnostics/tests/test_worker_plugin.py
@@ -73,7 +73,7 @@ async def test_normal_task_transitions_called(c, s, w):
 
     await c.register_worker_plugin(plugin)
     await c.submit(lambda x: x, 1, key="task")
-    await async_wait_for(lambda: not w.task_state, timeout=10)
+    await async_wait_for(lambda: not w.tasks, timeout=10)
 
 
 @gen_cluster(nthreads=[("127.0.0.1", 1)], client=True)
@@ -110,7 +110,7 @@ async def test_superseding_task_transitions_called(c, s, w):
 
     await c.register_worker_plugin(plugin)
     await c.submit(lambda x: x, 1, key="task", resources={"X": 1})
-    await async_wait_for(lambda: not w.task_state, timeout=10)
+    await async_wait_for(lambda: not w.tasks, timeout=10)
 
 
 @gen_cluster(nthreads=[("127.0.0.1", 1)], client=True)
@@ -125,7 +125,6 @@ async def test_release_dep_called(c, s, w):
         {"key": "task", "start": "ready", "finish": "executing"},
         {"key": "task", "start": "executing", "finish": "memory"},
         {"key": "dep", "state": "memory"},
-        {"dep": "dep", "state": "memory"},
         {"key": "task", "state": "memory"},
     ]
 
@@ -133,7 +132,7 @@ async def test_release_dep_called(c, s, w):
 
     await c.register_worker_plugin(plugin)
     await c.get(dsk, "task", sync=False)
-    await async_wait_for(lambda: not (w.task_state or w.dep_state), timeout=10)
+    await async_wait_for(lambda: not w.tasks, timeout=10)
 
 
 @gen_cluster(nthreads=[("127.0.0.1", 1)], client=True)
diff --git a/distributed/http/worker/prometheus.py b/distributed/http/worker/prometheus.py
index 08781bfd151..b354cad3ea9 100644
--- a/distributed/http/worker/prometheus.py
+++ b/distributed/http/worker/prometheus.py
@@ -23,9 +23,9 @@ def collect(self):
             "dask_worker_tasks", "Number of tasks at worker.", labels=["state"]
         )
         tasks.add_metric(["stored"], len(self.worker.data))
-        tasks.add_metric(["executing"], len(self.worker.executing))
+        tasks.add_metric(["executing"], self.worker.executing_count)
         tasks.add_metric(["ready"], len(self.worker.ready))
-        tasks.add_metric(["waiting"], len(self.worker.waiting_for_data))
+        tasks.add_metric(["waiting"], self.worker.waiting_for_data_count)
         tasks.add_metric(["serving"], len(self.worker._comms))
         yield tasks
 
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index a9e2429f9f2..a38292f23a4 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -5127,7 +5127,12 @@ async def test_call_stack_future(c, s, a, b):
     assert all(list(first(result.values())) == [future.key] for result in results)
     assert results[0] == results[1]
     result = results[0]
-    w = a if future.key in a.executing else b
+    ts = a.tasks.get(future.key)
+    if ts is not None and ts.state == "executing":
+        w = a
+    else:
+        w = b
+
     assert list(result) == [w.address]
     assert list(result[w.address]) == [future.key]
     assert "slowinc" in str(result)
@@ -5137,10 +5142,10 @@ async def test_call_stack_future(c, s, a, b):
 @gen_cluster([("127.0.0.1", 4)] * 2, client=True)
 async def test_call_stack_all(c, s, a, b):
     future = c.submit(slowinc, 1, delay=0.8)
-    while not a.executing and not b.executing:
+    while not a.executing_count and not b.executing_count:
         await asyncio.sleep(0.01)
     result = await c.call_stack()
-    w = a if a.executing else b
+    w = a if a.executing_count else b
     assert list(result) == [w.address]
     assert list(result[w.address]) == [future.key]
     assert "slowinc" in str(result)
@@ -5150,7 +5155,7 @@ async def test_call_stack_all(c, s, a, b):
 async def test_call_stack_collections(c, s, a, b):
     da = pytest.importorskip("dask.array")
     x = da.random.random(100, chunks=(10,)).map_blocks(slowinc, delay=0.5).persist()
-    while not a.executing and not b.executing:
+    while not a.executing_count and not b.executing_count:
         await asyncio.sleep(0.001)
     result = await c.call_stack(x)
     assert result
@@ -5160,7 +5165,7 @@ async def test_call_stack_collections(c, s, a, b):
 async def test_call_stack_collections_all(c, s, a, b):
     da = pytest.importorskip("dask.array")
     x = da.random.random(100, chunks=(10,)).map_blocks(slowinc, delay=0.5).persist()
-    while not a.executing and not b.executing:
+    while not a.executing_count and not b.executing_count:
         await asyncio.sleep(0.001)
     result = await c.call_stack()
     assert result
diff --git a/distributed/tests/test_failed_workers.py b/distributed/tests/test_failed_workers.py
index 7c7b820a9ec..32addfa68bc 100644
--- a/distributed/tests/test_failed_workers.py
+++ b/distributed/tests/test_failed_workers.py
@@ -384,8 +384,9 @@ async def test_worker_who_has_clears_after_failed_connection(c, s, a, b):
     await wait(futures)
 
     result = await c.submit(sum, futures, workers=a.address)
-    for dep in set(a.dep_state) - set(a.task_state):
-        a.release_dep(dep, report=True)
+    deps = [dep for dep in a.tasks.values() if dep.key not in a.data_needed]
+    for dep in deps:
+        a.release_key(dep.key, report=True)
 
     n_worker_address = n.worker_address
     with suppress(CommClosedError):
@@ -398,7 +399,7 @@ async def test_worker_who_has_clears_after_failed_connection(c, s, a, b):
     await total
 
     assert not a.has_what.get(n_worker_address)
-    assert not any(n_worker_address in s for s in a.who_has.values())
+    assert not any(n_worker_address in s for ts in a.tasks.values() for s in ts.who_has)
 
     await n.close()
 
diff --git a/distributed/tests/test_resources.py b/distributed/tests/test_resources.py
index 870b930fdae..4404ab7cb38 100644
--- a/distributed/tests/test_resources.py
+++ b/distributed/tests/test_resources.py
@@ -229,8 +229,8 @@ async def test_submit_many_non_overlapping(c, s, a, b):
 
     while len(a.data) + len(b.data) < 100:
         await asyncio.sleep(0.01)
-        assert len(a.executing) <= 2
-        assert len(b.executing) <= 1
+        assert a.executing_count <= 2
+        assert b.executing_count <= 1
 
     await wait(futures)
     assert a.total_resources == a.available_resources
@@ -243,7 +243,7 @@ async def test_minimum_resource(c, s, a):
 
     while len(a.data) < 30:
         await asyncio.sleep(0.01)
-        assert len(a.executing) <= 1
+        assert a.executing_count <= 1
 
     await wait(futures)
     assert a.total_resources == a.available_resources
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 4efc40d85d4..41ecb48990b 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -965,7 +965,7 @@ async def test_worker_arrives_with_processing_data(c, s, a, b):
         await asyncio.sleep(0.01)
 
     w = Worker(s.address, nthreads=1)
-    w.put_key_in_memory(y.key, 3)
+    w.update_data(data={y.key: 3})
 
     await w
 
@@ -1017,7 +1017,7 @@ async def test_no_workers_to_memory(c, s):
         await asyncio.sleep(0.01)
 
     w = Worker(s.address, nthreads=1)
-    w.put_key_in_memory(y.key, 3)
+    w.update_data(data={y.key: 3})
 
     await w
 
@@ -1047,7 +1047,7 @@ async def test_no_worker_to_memory_restrictions(c, s, a, b):
         await asyncio.sleep(0.01)
 
     w = Worker(s.address, nthreads=1, name="alice")
-    w.put_key_in_memory(y.key, 3)
+    w.update_data(data={y.key: 3})
 
     await w
 
@@ -1554,12 +1554,12 @@ async def test_closing_scheduler_closes_workers(s, a, b):
 async def test_resources_reset_after_cancelled_task(c, s, w):
     future = c.submit(sleep, 0.2, resources={"A": 1})
 
-    while not w.executing:
+    while not w.executing_count:
         await asyncio.sleep(0.01)
 
     await future.cancel()
 
-    while w.executing:
+    while w.executing_count:
         await asyncio.sleep(0.01)
 
     assert not s.workers[w.address].used_resources["A"]
diff --git a/distributed/tests/test_steal.py b/distributed/tests/test_steal.py
index f6fb91b1ce2..15a8db9c22d 100644
--- a/distributed/tests/test_steal.py
+++ b/distributed/tests/test_steal.py
@@ -203,7 +203,7 @@ async def test_new_worker_steals(c, s, a):
 
     futures = c.map(slowinc, range(100), delay=0.05)
     total = c.submit(sum, futures)
-    while len(a.task_state) < 10:
+    while len(a.tasks) < 10:
         await asyncio.sleep(0.01)
 
     b = await Worker(s.address, loop=s.loop, nthreads=1, memory_limit=MEMORY_LIMIT)
@@ -245,18 +245,18 @@ async def test_dont_steal_worker_restrictions(c, s, a, b):
 
     futures = c.map(slowinc, range(100), delay=0.1, workers=a.address)
 
-    while len(a.task_state) + len(b.task_state) < 100:
+    while len(a.tasks) + len(b.tasks) < 100:
         await asyncio.sleep(0.01)
 
-    assert len(a.task_state) == 100
-    assert len(b.task_state) == 0
+    assert len(a.tasks) == 100
+    assert len(b.tasks) == 0
 
     result = s.extensions["stealing"].balance()
 
     await asyncio.sleep(0.1)
 
-    assert len(a.task_state) == 100
-    assert len(b.task_state) == 0
+    assert len(a.tasks) == 100
+    assert len(b.tasks) == 0
 
 
 @gen_cluster(
@@ -269,20 +269,20 @@ async def test_steal_worker_restrictions(c, s, wa, wb, wc):
     ntasks = 100
     futures = c.map(slowinc, range(ntasks), delay=0.1, workers={wa.address, wb.address})
 
-    while sum(len(w.task_state) for w in [wa, wb, wc]) < ntasks:
+    while sum(len(w.tasks) for w in [wa, wb, wc]) < ntasks:
         await asyncio.sleep(0.01)
 
-    assert 0 < len(wa.task_state) < ntasks
-    assert 0 < len(wb.task_state) < ntasks
-    assert len(wc.task_state) == 0
+    assert 0 < len(wa.tasks) < ntasks
+    assert 0 < len(wb.tasks) < ntasks
+    assert len(wc.tasks) == 0
 
     s.extensions["stealing"].balance()
 
     await asyncio.sleep(0.1)
 
-    assert 0 < len(wa.task_state) < ntasks
-    assert 0 < len(wb.task_state) < ntasks
-    assert len(wc.task_state) == 0
+    assert 0 < len(wa.tasks) < ntasks
+    assert 0 < len(wb.tasks) < ntasks
+    assert len(wc.tasks) == 0
 
 
 @pytest.mark.skipif(
@@ -294,16 +294,16 @@ async def test_dont_steal_host_restrictions(c, s, a, b):
     await future
 
     futures = c.map(slowinc, range(100), delay=0.1, workers="127.0.0.1")
-    while len(a.task_state) + len(b.task_state) < 100:
+    while len(a.tasks) + len(b.tasks) < 100:
         await asyncio.sleep(0.01)
-    assert len(a.task_state) == 100
-    assert len(b.task_state) == 0
+    assert len(a.tasks) == 100
+    assert len(b.tasks) == 0
 
     result = s.extensions["stealing"].balance()
 
     await asyncio.sleep(0.1)
-    assert len(a.task_state) == 100
-    assert len(b.task_state) == 0
+    assert len(a.tasks) == 100
+    assert len(b.tasks) == 0
 
 
 @pytest.mark.skipif(
@@ -316,22 +316,22 @@ async def test_steal_host_restrictions(c, s, wa, wb):
 
     ntasks = 100
     futures = c.map(slowinc, range(ntasks), delay=0.1, workers="127.0.0.1")
-    while len(wa.task_state) < ntasks:
+    while len(wa.tasks) < ntasks:
         await asyncio.sleep(0.01)
-    assert len(wa.task_state) == ntasks
-    assert len(wb.task_state) == 0
+    assert len(wa.tasks) == ntasks
+    assert len(wb.tasks) == 0
 
     wc = await Worker(s.address, nthreads=1)
 
     start = time()
-    while not wc.task_state or len(wa.task_state) == ntasks:
+    while not wc.tasks or len(wa.tasks) == ntasks:
         await asyncio.sleep(0.01)
         assert time() < start + 3
 
     await asyncio.sleep(0.1)
-    assert 0 < len(wa.task_state) < ntasks
-    assert len(wb.task_state) == 0
-    assert 0 < len(wc.task_state) < ntasks
+    assert 0 < len(wa.tasks) < ntasks
+    assert len(wb.tasks) == 0
+    assert 0 < len(wc.tasks) < ntasks
 
 
 @gen_cluster(
@@ -342,16 +342,16 @@ async def test_dont_steal_resource_restrictions(c, s, a, b):
     await future
 
     futures = c.map(slowinc, range(100), delay=0.1, resources={"A": 1})
-    while len(a.task_state) + len(b.task_state) < 100:
+    while len(a.tasks) + len(b.tasks) < 100:
         await asyncio.sleep(0.01)
-    assert len(a.task_state) == 100
-    assert len(b.task_state) == 0
+    assert len(a.tasks) == 100
+    assert len(b.tasks) == 0
 
     result = s.extensions["stealing"].balance()
 
     await asyncio.sleep(0.1)
-    assert len(a.task_state) == 100
-    assert len(b.task_state) == 0
+    assert len(a.tasks) == 100
+    assert len(b.tasks) == 0
 
 
 @gen_cluster(
@@ -362,19 +362,19 @@ async def test_steal_resource_restrictions(c, s, a):
     await future
 
     futures = c.map(slowinc, range(100), delay=0.2, resources={"A": 1})
-    while len(a.task_state) < 101:
+    while len(a.tasks) < 101:
         await asyncio.sleep(0.01)
-    assert len(a.task_state) == 101
+    assert len(a.tasks) == 101
 
     b = await Worker(s.address, loop=s.loop, nthreads=1, resources={"A": 4})
 
     start = time()
-    while not b.task_state or len(a.task_state) == 101:
+    while not b.tasks or len(a.tasks) == 101:
         await asyncio.sleep(0.01)
         assert time() < start + 3
 
-    assert len(b.task_state) > 0
-    assert len(a.task_state) < 101
+    assert len(b.tasks) > 0
+    assert len(a.tasks) < 101
 
     await b.close()
 
@@ -421,7 +421,7 @@ async def test_dont_steal_few_saturated_tasks_many_workers(c, s, a, *rest):
     await wait(futures)
 
     assert len(a.data) == 3
-    assert not any(w.task_state for w in rest)
+    assert not any(w.tasks for w in rest)
 
 
 @gen_cluster(
@@ -438,7 +438,7 @@ async def test_steal_when_more_tasks(c, s, a, *rest):
     futures = [c.submit(slowidentity, x, pure=False, delay=0.2) for i in range(20)]
 
     start = time()
-    while not any(w.task_state for w in rest):
+    while not any(w.tasks for w in rest):
         await asyncio.sleep(0.01)
         assert time() < start + 1
 
@@ -465,11 +465,11 @@ def slow2(x):
     futures = [c.submit(slowidentity, x, pure=False, delay=0.2) for i in range(10)]
     future = c.submit(slow2, x, priority=-1)
 
-    while not any(w.task_state for w in rest):
+    while not any(w.tasks for w in rest):
         await asyncio.sleep(0.01)
 
     # good future moves first
-    assert any(future.key in w.task_state for w in rest)
+    assert any(future.key in w.tasks for w in rest)
 
 
 def func(x):
@@ -667,15 +667,15 @@ async def test_dont_steal_executing_tasks(c, s, a, b):
     steal = s.extensions["stealing"]
 
     future = c.submit(slowinc, 1, delay=0.5, workers=a.address)
-    while not a.executing:
+    while not a.executing_count:
         await asyncio.sleep(0.01)
 
     steal.move_task_request(
         s.tasks[future.key], s.workers[a.address], s.workers[b.address]
     )
     await asyncio.sleep(0.1)
-    assert future.key in a.executing
-    assert not b.executing
+    assert a.tasks[future.key].state == "executing"
+    assert not b.executing_count
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 2)
@@ -696,8 +696,8 @@ def long(delay):
         await asyncio.sleep(0.01)
         assert time() < start + 1
 
-    na = len(a.executing)
-    nb = len(b.executing)
+    na = a.executing_count
+    nb = b.executing_count
 
     incs = c.map(inc, range(100), workers=a.address, allow_other_workers=True)
 
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 68f1958f5f7..6119c8fb094 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -69,7 +69,7 @@ async def test_str(s, a, b):
     assert a.address in repr(a)
     assert str(a.nthreads) in str(a)
     assert str(a.nthreads) in repr(a)
-    assert str(len(a.executing)) in repr(a)
+    assert str(a.executing_count) in repr(a)
 
 
 @pytest.mark.asyncio
@@ -96,7 +96,7 @@ def __repr__(self):
 
     x = c.submit(NoReprObj, workers=a.address)
     await wait(x)
-    assert not a.executing
+    assert not a.executing_count
     assert a.data
 
     def bad_func(*args, **kwargs):
@@ -128,7 +128,7 @@ def reset(self):
     y = c.submit(bad_func, x, k=x, workers=b.address)
     await wait(y)
 
-    assert not b.executing
+    assert not b.executing_count
     assert y.status == "error"
     # Make sure job died because of bad func and not because of bad
     # argument.
@@ -571,13 +571,8 @@ async def test_clean(c, s, a, b):
 
     collections = [
         a.tasks,
-        a.task_state,
-        a.startstops,
         a.data,
         a.nbytes,
-        a.durations,
-        a.priorities,
-        a.types,
         a.threads,
     ]
     for c in collections:
@@ -586,7 +581,7 @@ async def test_clean(c, s, a, b):
     x.release()
     y.release()
 
-    while x.key in a.task_state:
+    while x.key in a.tasks:
         await asyncio.sleep(0.01)
 
     for c in collections:
@@ -611,15 +606,16 @@ async def test_message_breakup(c, s, a, b):
 
 @gen_cluster(client=True)
 async def test_types(c, s, a, b):
-    assert not a.types
-    assert not b.types
+    assert all(ts.type is None for ts in a.tasks.values())
+    assert all(ts.type is None for ts in b.tasks.values())
     x = c.submit(inc, 1, workers=a.address)
     await wait(x)
-    assert a.types[x.key] == int
+    assert a.tasks[x.key].type == int
 
     y = c.submit(inc, x, workers=b.address)
     await wait(y)
-    assert b.types == {x.key: int, y.key: int}
+    assert b.tasks[x.key].type == int
+    assert b.tasks[y.key].type == int
 
     await c._cancel(y)
 
@@ -628,7 +624,7 @@ async def test_types(c, s, a, b):
         await asyncio.sleep(0.01)
         assert time() < start + 5
 
-    assert y.key not in b.types
+    assert y.key not in b.tasks
 
 
 @gen_cluster()
@@ -641,19 +637,24 @@ async def test_system_monitor(s, a, b):
     client=True, nthreads=[("127.0.0.1", 2, {"resources": {"A": 1}}), ("127.0.0.1", 1)]
 )
 async def test_restrictions(c, s, a, b):
+    # Worker has resource available
+    assert a.available_resources == {"A": 1}
     # Resource restrictions
     x = c.submit(inc, 1, resources={"A": 1})
     await x
-    assert a.resource_restrictions == {x.key: {"A": 1}}
+    ts = a.tasks[x.key]
+    assert ts.resource_restrictions == {"A": 1}
     await c._cancel(x)
 
-    while x.key in a.task_state:
+    while ts.state != "memory":
+        # Resource should be unavailable while task isn't finished
+        assert a.available_resources == {"A": 0}
         await asyncio.sleep(0.01)
 
-    assert a.resource_restrictions == {}
+    # Resource restored after task is in memory
+    assert a.available_resources["A"] == 1
 
 
-@pytest.mark.xfail
 @gen_cluster(client=True)
 async def test_clean_nbytes(c, s, a, b):
     L = [delayed(inc)(i) for i in range(10)]
@@ -697,7 +698,7 @@ async def test_multiple_transfers(c, s, w1, w2, w3):
 
     await wait(z)
 
-    r = w3.startstops[z.key]
+    r = w3.tasks[z.key].startstops
     transfers = [t for t in r if t["action"] == "transfer"]
     assert len(transfers) == 2
 
@@ -809,7 +810,7 @@ async def test_stop_doing_unnecessary_work(c, s, a, b):
     del futures
 
     start = time()
-    while a.executing:
+    while a.executing_count:
         await asyncio.sleep(0.01)
         assert time() - start < 0.5
 
@@ -1562,15 +1563,15 @@ async def test_close_gracefully(c, s, a, b):
         await asyncio.sleep(0.1)
 
     mem = set(b.data)
-    proc = set(b.executing)
+    proc = [ts for ts in b.tasks.values() if ts.state == "executing"]
 
     await b.close_gracefully()
 
     assert b.status == Status.closed
     assert b.address not in s.workers
     assert mem.issubset(set(a.data))
-    for key in proc:
-        assert s.tasks[key].state in ("processing", "memory")
+    for ts in proc:
+        assert ts.state in ("processing", "memory")
 
 
 @pytest.mark.slow
diff --git a/distributed/worker.py b/distributed/worker.py
index 96cdf9e87e0..19d77c40646 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -91,6 +91,76 @@
 SerializedTask = namedtuple("SerializedTask", ["function", "args", "kwargs", "task"])
 
 
+class TaskState:
+    """Holds volatile state relating to an individual Dask task
+
+
+    * **dependencies**: ``set(TaskState instances)``
+        The data needed by this key to run
+    * **dependents**: ``set(TaskState instances)``
+        The keys that use this dependency
+    * **duration**: ``float``
+        Expected duration the a task
+    * **priority**: ``tuple``
+        The priority this task given by the scheduler.  Determines run order.
+    * **state**: ``str``
+        The current state of the task. One of ["waiting", "ready", "executing",
+        "memory", "flight", "executing", "error", "long-running",
+        "rescheduled", "error"]
+    * **who_has**: ``set(worker)``
+        Workers that we believe have this data
+    * **coming_from**: ``str``
+        The worker that current task data is coming from if task is in flight
+    * **waiting_for_data**: ``set(keys of dependencies)``
+        A dynamic verion of dependencies.  All dependencies that we still don't
+        have for a particular key.
+    * **resource_restrictions**: ``{str: number}``
+        Abstract resources required to run a task
+    * **exception**: ``str``
+        The exception caused by running a task if it erred
+    * **traceback**: ``str``
+        The exception caused by running a task if it erred
+    * **type**: ``type``
+        The type of a particular piece of data
+    * **suspicious_count**: ``int``
+        The number of times a dependency has not been where we expected it
+    * **startstops**: ``[{startstop}]``
+        Log of transfer, load, and compute times for a task
+
+    Parameters
+    ----------
+    key: str
+    runspec: SerializedTask
+        A named tuple containing the ``function``, ``args``, ``kwargs`` and
+        ``task`` associated with this `TaskState` instance. This defaults to
+        ``None`` and can remain empty if it is a dependency that this worker
+        will receive from another worker.
+
+    """
+
+    def __init__(self, key, runspec=None):
+        assert key is not None
+        self.key = key
+        self.runspec = runspec
+        self.dependencies = set()
+        self.dependents = set()
+        self.duration = None
+        self.priority = None
+        self.state = None
+        self.who_has = set()
+        self.coming_from = None
+        self.waiting_for_data = set()
+        self.resource_restrictions = None
+        self.exception = None
+        self.traceback = None
+        self.type = None
+        self.suspicious_count = 0
+        self.startstops = list()
+
+    def __repr__(self):
+        return "<Task %r %s>" % (self.key, self.state)
+
+
 class Worker(ServerNode):
     """Worker node in a Dask distributed cluster
 
@@ -144,11 +214,13 @@ class Worker(ServerNode):
 
     **Volatile State**
 
-    This attributes track the progress of tasks that this worker is trying to
+    These attributes track the progress of tasks that this worker is trying to
     complete.  In the descriptions below a ``key`` is the name of a task that
     we want to compute and ``dep`` is the name of a piece of dependent data
     that we want to collect from others.
 
+    * **tasks**: ``{key: TaskState}``
+        The tasks currently executing on this worker (and any dependencies of those tasks)
     * **data:** ``{key: object}``:
         Prefer using the **host** attribute instead of this, unless
         memory_limit and at least one of memory_target_fraction or
@@ -160,74 +232,41 @@ class Worker(ServerNode):
     * **data.disk:** ``{key: object}``:
         Dictionary mapping keys to actual values stored on disk. Only
         available if condition for **data** being a zict.Buffer is met.
-    * **task_state**: ``{key: string}``:
-        The state of all tasks that the scheduler has asked us to compute.
-        Valid states include waiting, constrained, executing, memory, erred
-    * **tasks**: ``{key: dict}``
-        The function, args, kwargs of a task.  We run this when appropriate
-    * **dependencies**: ``{key: {deps}}``
-        The data needed by this key to run
-    * **dependents**: ``{dep: {keys}}``
-        The keys that use this dependency
     * **data_needed**: deque(keys)
         The keys whose data we still lack, arranged in a deque
-    * **waiting_for_data**: ``{kep: {deps}}``
-        A dynamic verion of dependencies.  All dependencies that we still don't
-        have for a particular key.
     * **ready**: [keys]
         Keys that are ready to run.  Stored in a LIFO stack
     * **constrained**: [keys]
         Keys for which we have the data to run, but are waiting on abstract
         resources like GPUs.  Stored in a FIFO deque
-    * **executing**: {keys}
-        Keys that are currently executing
+    * **executing_count**: ``int``
+        A count of tasks currently executing on this worker
     * **executed_count**: int
         A number of tasks that this worker has run in its lifetime
     * **long_running**: {keys}
         A set of keys of tasks that are running and have started their own
         long-running clients.
-
-    * **dep_state**: ``{dep: string}``:
-        The state of all dependencies required by our tasks
-        Valid states include waiting, flight, and memory
-    * **who_has**: ``{dep: {worker}}``
-        Workers that we believe have this data
     * **has_what**: ``{worker: {deps}}``
         The data that we care about that we think a worker has
     * **pending_data_per_worker**: ``{worker: [dep]}``
         The data on each worker that we still want, prioritized as a deque
-    * **in_flight_tasks**: ``{task: worker}``
-        All dependencies that are coming to us in current peer-to-peer
-        connections and the workers from which they are coming.
+    * **in_flight_tasks**: ``int``
+        A count of the number of tasks that are coming to us in current
+        peer-to-peer connections
     * **in_flight_workers**: ``{worker: {task}}``
         The workers from which we are currently gathering data and the
         dependencies we expect from those connections
     * **comm_bytes**: ``int``
         The total number of bytes in flight
-    * **suspicious_deps**: ``{dep: int}``
-        The number of times a dependency has not been where we expected it
-
     * **nbytes**: ``{key: int}``
         The size of a particular piece of data
-    * **types**: ``{key: type}``
-        The type of a particular piece of data
     * **threads**: ``{key: int}``
         The ID of the thread on which the task ran
     * **active_threads**: ``{int: key}``
         The keys currently running on active threads
-    * **exceptions**: ``{key: exception}``
-        The exception caused by running a task if it erred
-    * **tracebacks**: ``{key: traceback}``
-        The exception caused by running a task if it erred
-    * **startstops**: ``{key: [{startstop}]}``
-        Log of transfer, load, and compute times for a task
+    * **waiting_for_data_count**: ``int``
+        A count of how many tasks are currently waiting for data
 
-    * **priorities**: ``{key: tuple}``
-        The priority of a key given by the scheduler.  Determines run order.
-    * **durations**: ``{key: float}``
-        Expected duration of a task
-    * **resource_restrictions**: ``{key: {str: number}}``
-        Abstract resources required to run a task
 
     Parameters
     ----------
@@ -335,12 +374,7 @@ def __init__(
         **kwargs,
     ):
         self.tasks = dict()
-        self.task_state = dict()
-        self.dep_state = dict()
-        self.dependencies = dict()
-        self.dependents = dict()
-        self.waiting_for_data = dict()
-        self.who_has = dict()
+        self.waiting_for_data_count = 0
         self.has_what = defaultdict(set)
         self.pending_data_per_worker = defaultdict(deque)
         self.nanny = nanny
@@ -348,7 +382,7 @@ def __init__(
 
         self.data_needed = deque()  # TODO: replace with heap?
 
-        self.in_flight_tasks = dict()
+        self.in_flight_tasks = 0
         self.in_flight_workers = dict()
         self.total_out_connections = dask.config.get(
             "distributed.worker.connections.outgoing"
@@ -358,14 +392,10 @@ def __init__(
         )
         self.total_comm_nbytes = 10e6
         self.comm_nbytes = 0
-        self.suspicious_deps = defaultdict(lambda: 0)
         self._missing_dep_flight = set()
 
         self.nbytes = dict()
-        self.types = dict()
         self.threads = dict()
-        self.exceptions = dict()
-        self.tracebacks = dict()
 
         self.active_threads_lock = threading.Lock()
         self.active_threads = dict()
@@ -374,15 +404,11 @@ def __init__(
         self.profile_recent = profile.create()
         self.profile_history = deque(maxlen=3600)
 
-        self.priorities = dict()
         self.generation = 0
-        self.durations = dict()
-        self.startstops = defaultdict(list)
-        self.resource_restrictions = dict()
 
         self.ready = list()
         self.constrained = deque()
-        self.executing = set()
+        self.executing_count = 0
         self.executed_count = 0
         self.long_running = set()
 
@@ -400,6 +426,7 @@ def __init__(
             ("waiting", "ready"): self.transition_waiting_ready,
             ("waiting", "memory"): self.transition_waiting_done,
             ("waiting", "error"): self.transition_waiting_done,
+            ("waiting", "flight"): self.transition_waiting_flight,
             ("ready", "executing"): self.transition_ready_executing,
             ("ready", "memory"): self.transition_ready_memory,
             ("constrained", "executing"): self.transition_constrained_executing,
@@ -410,13 +437,9 @@ def __init__(
             ("long-running", "error"): self.transition_executing_done,
             ("long-running", "memory"): self.transition_executing_done,
             ("long-running", "rescheduled"): self.transition_executing_done,
-        }
-
-        self._dep_transitions = {
-            ("waiting", "flight"): self.transition_dep_waiting_flight,
-            ("waiting", "memory"): self.transition_dep_waiting_memory,
-            ("flight", "waiting"): self.transition_dep_flight_waiting,
-            ("flight", "memory"): self.transition_dep_flight_memory,
+            ("flight", "memory"): self.transition_flight_memory,
+            ("flight", "ready"): self.transition_flight_memory,
+            ("flight", "waiting"): self.transition_flight_waiting,
         }
 
         self.incoming_transfer_log = deque(maxlen=100000)
@@ -715,11 +738,11 @@ def __repr__(self):
             self.name,
             self.status,
             len(self.data),
-            len(self.executing),
+            self.executing_count,
             self.nthreads,
             len(self.ready),
-            len(self.in_flight_tasks),
-            len(self.waiting_for_data),
+            self.in_flight_tasks,
+            self.waiting_for_data_count,
         )
 
     @property
@@ -741,10 +764,10 @@ def local_dir(self):
 
     async def get_metrics(self):
         core = dict(
-            executing=len(self.executing),
+            executing=self.executing_count,
             in_memory=len(self.data),
             ready=len(self.ready),
-            in_flight=len(self.in_flight_tasks),
+            in_flight=self.in_flight_tasks,
             bandwidth={
                 "total": self.bandwidth,
                 "workers": dict(self.bandwidth_workers),
@@ -1325,18 +1348,14 @@ async def get_data(
 
     def update_data(self, comm=None, data=None, report=True, serializers=None):
         for key, value in data.items():
-            if key in self.task_state:
-                self.transition(key, "memory", value=value)
+            ts = self.tasks.get(key)
+            if getattr(ts, "state", None) is not None:
+                self.transition(ts, "memory", value=value)
             else:
-                self.put_key_in_memory(key, value)
-                self.task_state[key] = "memory"
-                self.tasks[key] = None
-                self.priorities[key] = None
-                self.durations[key] = None
-                self.dependencies[key] = set()
-
-            if key in self.dep_state:
-                self.transition_dep(key, "memory", value=value)
+                self.tasks[key] = ts = TaskState(key)
+                self.put_key_in_memory(ts, value)
+                ts.priority = None
+                ts.duration = None
 
             self.log.append((key, "receive-from-scatter"))
 
@@ -1349,13 +1368,9 @@ def delete_data(self, comm=None, keys=None, report=True):
         if keys:
             for key in list(keys):
                 self.log.append((key, "delete"))
-                if key in self.task_state:
-                    self.release_key(key)
+                self.release_key(key)
 
-                if key in self.dep_state:
-                    self.release_dep(key)
-
-            logger.debug("Deleted %d keys", len(keys))
+            logger.debug("Worker %s -- Deleted %d keys", self.name, len(keys))
         return "OK"
 
     async def set_resources(self, **resources):
@@ -1393,89 +1408,79 @@ def add_task(
     ):
         try:
             if key in self.tasks:
-                state = self.task_state[key]
-                if state == "memory":
+                ts = self.tasks[key]
+                if ts.state == "memory":
                     assert key in self.data or key in self.actors
                     logger.debug(
-                        "Asked to compute pre-existing result: %s: %s", key, state
+                        "Asked to compute pre-existing result: %s: %s", key, ts.state
                     )
-                    self.send_task_state_to_scheduler(key)
+                    self.send_task_state_to_scheduler(ts)
                     return
-                if state in IN_PLAY:
+                if ts.state in IN_PLAY:
                     return
-                if state == "erred":
-                    del self.exceptions[key]
-                    del self.tracebacks[key]
+                if ts.state == "erred":
+                    ts.exception = None
+                    ts.traceback = None
+            else:
+                self.log.append((key, "new"))
+                self.tasks[key] = ts = TaskState(
+                    key=key, runspec=SerializedTask(function, args, kwargs, task)
+                )
+                ts.state = "waiting"
 
             if priority is not None:
                 priority = tuple(priority) + (self.generation,)
                 self.generation -= 1
 
-            if self.dep_state.get(key) == "memory":
-                self.task_state[key] = "memory"
-                self.send_task_state_to_scheduler(key)
-                self.tasks[key] = None
-                self.log.append((key, "new-task-already-in-memory"))
-                self.priorities[key] = priority
-                self.durations[key] = duration
-                return
-
-            self.log.append((key, "new"))
-            self.tasks[key] = SerializedTask(function, args, kwargs, task)
             if actor:
-                self.actors[key] = None
+                self.actors[ts.key] = None
 
-            self.priorities[key] = priority
-            self.durations[key] = duration
+            ts.priority = priority
+            ts.duration = duration
             if resource_restrictions:
-                self.resource_restrictions[key] = resource_restrictions
-            self.task_state[key] = "waiting"
+                ts.resource_restrictions = resource_restrictions
+            ts.state = "waiting"
 
             if nbytes is not None:
                 self.nbytes.update(nbytes)
 
             who_has = who_has or {}
-            self.dependencies[key] = set(who_has)
-            self.waiting_for_data[key] = set()
 
-            for dep in who_has:
-                if dep not in self.dependents:
-                    self.dependents[dep] = set()
-                self.dependents[dep].add(key)
+            for dependency, workers in who_has.items():
+                assert workers
+                if dependency not in self.tasks:
+                    self.tasks[dependency] = dep_ts = TaskState(key=dependency)
+                    dep_ts.state = "waiting"
 
-                if dep not in self.dep_state:
-                    if self.task_state.get(dep) == "memory":
-                        state = "memory"
-                    else:
-                        state = "waiting"
-                    self.dep_state[dep] = state
-                    self.log.append((dep, "new-dep", state))
+                dep_ts = self.tasks[dependency]
+                self.log.append((dependency, "new-dep", dep_ts.state))
 
-                if self.dep_state[dep] != "memory":
-                    self.waiting_for_data[key].add(dep)
+                if dep_ts.state != "memory":
+                    ts.waiting_for_data.add(dep_ts.key)
+                    self.waiting_for_data_count += 1
 
-            for dep, workers in who_has.items():
-                assert workers
-                if dep not in self.who_has:
-                    self.who_has[dep] = set(workers)
-                self.who_has[dep].update(workers)
+                dep_ts.who_has.update(workers)
+
+                ts.dependencies.add(dep_ts)
+                dep_ts.dependents.add(ts)
 
                 for worker in workers:
-                    self.has_what[worker].add(dep)
-                    if self.dep_state[dep] != "memory":
-                        self.pending_data_per_worker[worker].append(dep)
+                    self.has_what[worker].add(dep_ts.key)
+                    if dep_ts.state != "memory":
+                        self.pending_data_per_worker[worker].append(dep_ts.key)
 
-            if self.waiting_for_data[key]:
-                self.data_needed.append(key)
+            if ts.waiting_for_data:
+                self.data_needed.append(ts.key)
             else:
-                self.transition(key, "ready")
+                self.transition(ts, "ready")
             if self.validate:
                 if who_has:
-                    assert all(dep in self.dep_state for dep in who_has)
+                    assert all(self.tasks[dep] in ts.dependencies for dep in who_has)
                     assert all(dep in self.nbytes for dep in who_has)
-                    for dep in who_has:
-                        self.validate_dep(dep)
-                    self.validate_key(key)
+                    assert all(self.tasks[dep.key] for dep in ts.dependencies)
+                    for dependency in ts.dependencies:
+                        self.validate_task(dependency)
+                    self.validate_task(ts)
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
@@ -1484,28 +1489,28 @@ def add_task(
                 pdb.set_trace()
             raise
 
-    def transition_dep(self, dep, finish, **kwargs):
-        try:
-            start = self.dep_state[dep]
-        except KeyError:
+    def transition(self, ts, finish, **kwargs):
+        if ts is None:
             return
+        start = ts.state
         if start == finish:
             return
-        func = self._dep_transitions[start, finish]
-        state = func(dep, **kwargs)
-        self.log.append(("dep", dep, start, state or finish))
-        if dep in self.dep_state:
-            self.dep_state[dep] = state or finish
-            if self.validate:
-                self.validate_dep(dep)
+        func = self._transitions[start, finish]
+        state = func(ts, **kwargs)
+        self.log.append((ts.key, start, state or finish))
+        ts.state = state or finish
+        if self.validate:
+            self.validate_task(ts)
+        self._notify_plugins("transition", ts.key, start, state or finish, **kwargs)
 
-    def transition_dep_waiting_flight(self, dep, worker=None):
+    def transition_waiting_flight(self, ts, worker=None):
         try:
             if self.validate:
-                assert dep not in self.in_flight_tasks
-                assert self.dependents[dep]
+                assert ts.state != "flight"
+                assert ts.dependents
 
-            self.in_flight_tasks[dep] = worker
+            ts.coming_from = worker
+            self.in_flight_tasks += 1
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
@@ -1514,35 +1519,33 @@ def transition_dep_waiting_flight(self, dep, worker=None):
                 pdb.set_trace()
             raise
 
-    def transition_dep_flight_waiting(self, dep, worker=None, remove=True):
+    def transition_flight_waiting(self, ts, worker=None, remove=True):
         try:
             if self.validate:
-                assert dep in self.in_flight_tasks
+                assert ts.state == "flight"
 
-            del self.in_flight_tasks[dep]
+            self.in_flight_tasks -= 1
+            ts.coming_from = None
             if remove:
                 try:
-                    self.who_has[dep].remove(worker)
-                except KeyError:
-                    pass
-                try:
-                    self.has_what[worker].remove(dep)
+                    ts.who_has.remove(worker)
+                    self.has_what[worker].remove(ts.key)
                 except KeyError:
                     pass
 
-            if not self.who_has.get(dep):
-                if dep not in self._missing_dep_flight:
-                    self._missing_dep_flight.add(dep)
-                    self.loop.add_callback(self.handle_missing_dep, dep)
-            for key in self.dependents.get(dep, ()):
-                if self.task_state[key] == "waiting":
+            if not ts.who_has:
+                if ts.key not in self._missing_dep_flight:
+                    self._missing_dep_flight.add(ts.key)
+                    self.loop.add_callback(self.handle_missing_dep, ts)
+            for dependent in ts.dependents:
+                if dependent.state == "waiting":
                     if remove:  # try a new worker immediately
-                        self.data_needed.appendleft(key)
+                        self.data_needed.appendleft(dependent.key)
                     else:  # worker was probably busy, wait a while
-                        self.data_needed.append(key)
+                        self.data_needed.append(dependent.key)
 
-            if not self.dependents[dep]:
-                self.release_dep(dep)
+            if not ts.dependents:
+                self.release_key(ts.key)
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
@@ -1551,34 +1554,26 @@ def transition_dep_flight_waiting(self, dep, worker=None, remove=True):
                 pdb.set_trace()
             raise
 
-    def transition_dep_flight_memory(self, dep, value=None):
+    def transition_flight_memory(self, ts, value=None):
         try:
             if self.validate:
-                assert dep in self.in_flight_tasks
-
-            del self.in_flight_tasks[dep]
-            if self.dependents[dep]:
-                self.dep_state[dep] = "memory"
-                self.put_key_in_memory(dep, value)
-                self.batched_stream.send({"op": "add-keys", "keys": [dep]})
-            else:
-                self.release_dep(dep)
+                assert ts.state == "flight"
 
-        except Exception as e:
-            logger.exception(e)
-            if LOG_PDB:
-                import pdb
+            self.in_flight_tasks -= 1
+            ts.coming_from = None
+            if ts.dependents:
+                self.put_key_in_memory(ts, value)
+                for dependent in ts.dependents:
+                    try:
+                        dependent.waiting_for_data.remove(ts.key)
+                        self.waiting_for_data_count -= 1
+                    except KeyError:
+                        pass
 
-                pdb.set_trace()
-            raise
+                self.batched_stream.send({"op": "add-keys", "keys": [ts.key]})
+            else:
+                self.release_key(ts.key)
 
-    def transition_dep_waiting_memory(self, dep, value=None):
-        try:
-            if self.validate:
-                assert dep in self.data
-                assert dep in self.nbytes
-                assert dep in self.types
-                assert self.task_state[dep] == "memory"
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
@@ -1586,41 +1581,26 @@ def transition_dep_waiting_memory(self, dep, value=None):
 
                 pdb.set_trace()
             raise
-        if value is not no_value and dep not in self.data:
-            self.put_key_in_memory(dep, value, transition=False)
 
-    def transition(self, key, finish, **kwargs):
-        start = self.task_state[key]
-        if start == finish:
-            return
-        func = self._transitions[start, finish]
-        state = func(key, **kwargs)
-        self.log.append((key, start, state or finish))
-        self.task_state[key] = state or finish
-        if self.validate:
-            self.validate_key(key)
-        self._notify_plugins("transition", key, start, state or finish, **kwargs)
-
-    def transition_waiting_ready(self, key):
+    def transition_waiting_ready(self, ts):
         try:
             if self.validate:
-                assert self.task_state[key] == "waiting"
-                assert key in self.waiting_for_data
-                assert not self.waiting_for_data[key]
+                assert ts.state == "waiting"
+                assert not ts.waiting_for_data
                 assert all(
-                    dep in self.data or dep in self.actors
-                    for dep in self.dependencies[key]
+                    dep.key in self.data or dep.key in self.actors
+                    for dep in ts.dependencies
                 )
-                assert key not in self.executing
-                assert key not in self.ready
+                assert all(dep.state == "memory" for dep in ts.dependencies)
+                assert ts.key not in self.ready
 
-            self.waiting_for_data.pop(key, None)
+            ts.waiting_for_data.clear()
 
-            if key in self.resource_restrictions:
-                self.constrained.append(key)
+            if ts.resource_restrictions is not None:
+                self.constrained.append(ts.key)
                 return "constrained"
             else:
-                heapq.heappush(self.ready, (self.priorities[key], key))
+                heapq.heappush(self.ready, (ts.priority, ts.key))
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
@@ -1629,16 +1609,17 @@ def transition_waiting_ready(self, key):
                 pdb.set_trace()
             raise
 
-    def transition_waiting_done(self, key, value=None):
+    def transition_waiting_done(self, ts, value=None):
         try:
             if self.validate:
-                assert self.task_state[key] == "waiting"
-                assert key in self.waiting_for_data
-                assert key not in self.executing
-                assert key not in self.ready
-
-            del self.waiting_for_data[key]
-            self.send_task_state_to_scheduler(key)
+                assert ts.state == "waiting"
+                assert ts.key not in self.ready
+
+            self.waiting_for_data_count -= len(ts.waiting_for_data)
+            ts.waiting_for_data.clear()
+            if value is not None:
+                self.put_key_in_memory(ts, value)
+            self.send_task_state_to_scheduler(ts)
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
@@ -1647,20 +1628,20 @@ def transition_waiting_done(self, key, value=None):
                 pdb.set_trace()
             raise
 
-    def transition_ready_executing(self, key):
+    def transition_ready_executing(self, ts):
         try:
             if self.validate:
-                assert key not in self.waiting_for_data
-                # assert key not in self.data
-                assert self.task_state[key] in READY
-                assert key not in self.ready
+                assert not ts.waiting_for_data
+                assert ts.key not in self.data
+                assert ts.state in READY
+                assert ts.key not in self.ready
                 assert all(
-                    dep in self.data or dep in self.actors
-                    for dep in self.dependencies[key]
+                    dep.key in self.data or dep.key in self.actors
+                    for dep in ts.dependencies
                 )
 
-            self.executing.add(key)
-            self.loop.add_callback(self.execute, key)
+            self.executing_count += 1
+            self.loop.add_callback(self.execute, ts.key)
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
@@ -1669,52 +1650,53 @@ def transition_ready_executing(self, key):
                 pdb.set_trace()
             raise
 
-    def transition_ready_memory(self, key, value=None):
-        self.send_task_state_to_scheduler(key)
+    def transition_ready_memory(self, ts, value=None):
+        self.send_task_state_to_scheduler(ts)
 
-    def transition_constrained_executing(self, key):
-        self.transition_ready_executing(key)
-        for resource, quantity in self.resource_restrictions[key].items():
+    def transition_constrained_executing(self, ts):
+        self.transition_ready_executing(ts)
+        for resource, quantity in ts.resource_restrictions.items():
             self.available_resources[resource] -= quantity
 
         if self.validate:
             assert all(v >= 0 for v in self.available_resources.values())
 
-    def transition_executing_done(self, key, value=no_value, report=True):
+    def transition_executing_done(self, ts, value=no_value, report=True):
         try:
             if self.validate:
-                assert key in self.executing or key in self.long_running
-                assert key not in self.waiting_for_data
-                assert key not in self.ready
+                assert ts.state == "executing" or ts.key in self.long_running
+                assert not ts.waiting_for_data
+                assert ts.key not in self.ready
 
             out = None
-            if key in self.resource_restrictions:
-                for resource, quantity in self.resource_restrictions[key].items():
+            if ts.resource_restrictions is not None:
+                for resource, quantity in ts.resource_restrictions.items():
                     self.available_resources[resource] += quantity
 
-            if self.task_state[key] == "executing":
-                self.executing.remove(key)
+            if ts.state == "executing":
+                self.executing_count -= 1
                 self.executed_count += 1
-            elif self.task_state[key] == "long-running":
-                self.long_running.remove(key)
+            elif ts.state == "long-running":
+                self.long_running.remove(ts.key)
 
             if value is not no_value:
                 try:
-                    self.task_state[key] = "memory"
-                    self.put_key_in_memory(key, value, transition=False)
+                    self.put_key_in_memory(ts, value, transition=False)
                 except Exception as e:
                     logger.info("Failed to put key in memory", exc_info=True)
                     msg = error_message(e)
-                    self.exceptions[key] = msg["exception"]
-                    self.tracebacks[key] = msg["traceback"]
-                    self.task_state[key] = "error"
+                    ts.exception = msg["exception"]
+                    ts.traceback = msg["traceback"]
+                    ts.state = "error"
                     out = "error"
 
-                if key in self.dep_state:
-                    self.transition_dep(key, "memory")
+                # Don't release the dependency keys, but do remove them from `dependents`
+                for dependency in ts.dependencies:
+                    dependency.dependents.discard(ts)
+                ts.dependencies.clear()
 
             if report and self.batched_stream and self.status == Status.running:
-                self.send_task_state_to_scheduler(key)
+                self.send_task_state_to_scheduler(ts)
             else:
                 raise CommClosedError
 
@@ -1730,15 +1712,19 @@ def transition_executing_done(self, key, value=no_value, report=True):
                 pdb.set_trace()
             raise
 
-    def transition_executing_long_running(self, key, compute_duration=None):
+    def transition_executing_long_running(self, ts, compute_duration=None):
         try:
             if self.validate:
-                assert key in self.executing
+                assert ts.state == "executing"
 
-            self.executing.remove(key)
-            self.long_running.add(key)
+            self.executing_count -= 1
+            self.long_running.add(ts.key)
             self.batched_stream.send(
-                {"op": "long-running", "key": key, "compute_duration": compute_duration}
+                {
+                    "op": "long-running",
+                    "key": ts.key,
+                    "compute_duration": compute_duration,
+                }
             )
 
             self.ensure_computing()
@@ -1750,14 +1736,15 @@ def transition_executing_long_running(self, key, compute_duration=None):
                 pdb.set_trace()
             raise
 
-    def maybe_transition_long_running(self, key, compute_duration=None):
-        if self.task_state.get(key) == "executing":
-            self.transition(key, "long-running", compute_duration=compute_duration)
+    def maybe_transition_long_running(self, ts, compute_duration=None):
+        if ts.state == "executing":
+            self.transition(ts, "long-running", compute_duration=compute_duration)
 
     def stateof(self, key):
+        ts = self.tasks[key]
         return {
-            "executing": key in self.executing,
-            "waiting_for_data": key in self.waiting_for_data,
+            "executing": ts.state == "executing",
+            "waiting_for_data": bool(ts.waiting_for_data),
             "heap": key in pluck(1, self.ready),
             "data": key in self.data,
         }
@@ -1798,28 +1785,29 @@ def ensure_communicating(self):
                     changed = True
                     continue
 
-                if self.task_state.get(key) != "waiting":
+                ts = self.tasks[key]
+                if ts.state != "waiting":
                     self.log.append((key, "communication pass"))
                     self.data_needed.popleft()
                     changed = True
                     continue
 
-                deps = self.dependencies[key]
+                deps = ts.dependencies
                 if self.validate:
-                    assert all(dep in self.dep_state for dep in deps)
+                    assert all(dep.key in self.tasks for dep in deps)
 
-                deps = [dep for dep in deps if self.dep_state[dep] == "waiting"]
+                deps = [dep for dep in deps if dep.state == "waiting"]
 
-                missing_deps = {dep for dep in deps if not self.who_has.get(dep)}
+                missing_deps = {dep for dep in deps if not dep.who_has}
                 if missing_deps:
                     logger.info("Can't find dependencies for key %s", key)
                     missing_deps2 = {
                         dep
                         for dep in missing_deps
-                        if dep not in self._missing_dep_flight
+                        if dep.key not in self._missing_dep_flight
                     }
                     for dep in missing_deps2:
-                        self._missing_dep_flight.add(dep)
+                        self._missing_dep_flight.add(dep.key)
                     self.loop.add_callback(self.handle_missing_dep, *missing_deps2)
 
                     deps = [dep for dep in deps if dep not in missing_deps]
@@ -1833,12 +1821,12 @@ def ensure_communicating(self):
                     or self.comm_nbytes < self.total_comm_nbytes
                 ):
                     dep = deps.pop()
-                    if self.dep_state[dep] != "waiting":
+                    if dep.state != "waiting":
                         continue
-                    if dep not in self.who_has:
+                    if not dep.who_has:
                         continue
                     workers = [
-                        w for w in self.who_has[dep] if w not in self.in_flight_workers
+                        w for w in dep.who_has if w not in self.in_flight_workers
                     ]
                     if not workers:
                         in_flight = True
@@ -1849,11 +1837,13 @@ def ensure_communicating(self):
                         worker = random.choice(local)
                     else:
                         worker = random.choice(list(workers))
-                    to_gather, total_nbytes = self.select_keys_for_gather(worker, dep)
+                    to_gather, total_nbytes = self.select_keys_for_gather(
+                        worker, dep.key
+                    )
                     self.comm_nbytes += total_nbytes
                     self.in_flight_workers[worker] = to_gather
                     for d in to_gather:
-                        self.transition_dep(d, "flight", worker=worker)
+                        self.transition(self.tasks[d], "flight", worker=worker)
                     self.loop.add_callback(
                         self.gather_dep, worker, dep, to_gather, total_nbytes, cause=key
                     )
@@ -1861,6 +1851,7 @@ def ensure_communicating(self):
 
                 if not deps and not in_flight:
                     self.data_needed.popleft()
+
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
@@ -1869,17 +1860,17 @@ def ensure_communicating(self):
                 pdb.set_trace()
             raise
 
-    def send_task_state_to_scheduler(self, key):
-        if key in self.data or self.actors.get(key):
-            nbytes = self.nbytes.get(key)
-            typ = self.types.get(key)
+    def send_task_state_to_scheduler(self, ts):
+        if ts.key in self.data or self.actors.get(ts.key):
+            nbytes = self.nbytes.get(ts.key)
+            typ = ts.type
             if nbytes is None or typ is None:
                 try:
-                    value = self.data[key]
+                    value = self.data[ts.key]
                 except KeyError:
-                    value = self.actors[key]
-                nbytes = self.nbytes[key] = sizeof(value)
-                typ = self.types[key] = type(value)
+                    value = self.actors[ts.key]
+                nbytes = self.nbytes[ts.key] = sizeof(value)
+                typ = ts.type = type(value)
                 del value
             try:
                 typ_serialized = dumps_function(typ)
@@ -1890,65 +1881,65 @@ def send_task_state_to_scheduler(self, key):
             d = {
                 "op": "task-finished",
                 "status": "OK",
-                "key": key,
+                "key": ts.key,
                 "nbytes": nbytes,
-                "thread": self.threads.get(key),
+                "thread": self.threads.get(ts.key),
                 "type": typ_serialized,
                 "typename": typename(typ),
             }
-        elif key in self.exceptions:
+        elif ts.exception is not None:
             d = {
                 "op": "task-erred",
                 "status": "error",
-                "key": key,
-                "thread": self.threads.get(key),
-                "exception": self.exceptions[key],
-                "traceback": self.tracebacks[key],
+                "key": ts.key,
+                "thread": self.threads.get(ts.key),
+                "exception": ts.exception,
+                "traceback": ts.traceback,
             }
         else:
-            logger.error(
-                "Key not ready to send to worker, %s: %s", key, self.task_state[key]
-            )
+            logger.error("Key not ready to send to worker, %s: %s", ts.key, ts.state)
             return
 
-        if key in self.startstops:
-            d["startstops"] = self.startstops[key]
+        if ts.startstops:
+            d["startstops"] = ts.startstops
         self.batched_stream.send(d)
 
-    def put_key_in_memory(self, key, value, transition=True):
-        if key in self.data:
+    def put_key_in_memory(self, ts, value, transition=True):
+        if ts.key in self.data:
+            ts.state = "memory"
             return
 
-        if key in self.actors:
-            self.actors[key] = value
+        if ts.key in self.actors:
+            self.actors[ts.key] = value
 
         else:
             start = time()
-            self.data[key] = value
+            self.data[ts.key] = value
+            ts.state = "memory"
             stop = time()
             if stop - start > 0.020:
-                self.startstops[key].append(
+                ts.startstops.append(
                     {"action": "disk-write", "start": start, "stop": stop}
                 )
 
-        if key not in self.nbytes:
-            self.nbytes[key] = sizeof(value)
-
-        self.types[key] = type(value)
+        if ts.key not in self.nbytes:
+            self.nbytes[ts.key] = sizeof(value)
 
-        for dep in self.dependents.get(key, ()):
-            if dep in self.waiting_for_data:
-                if key in self.waiting_for_data[dep]:
-                    self.waiting_for_data[dep].remove(key)
-                if not self.waiting_for_data[dep]:
-                    self.transition(dep, "ready")
+        ts.type = type(value)
 
-        if transition and key in self.task_state:
-            self.transition(key, "memory")
+        for dep in ts.dependents:
+            try:
+                dep.waiting_for_data.remove(ts.key)
+                self.waiting_for_data_count -= 1
+            except KeyError:
+                pass
+            if not dep.waiting_for_data:
+                self.transition(dep, "ready")
 
-        self.log.append((key, "put-in-memory"))
+        self.log.append((ts.key, "put-in-memory"))
 
     def select_keys_for_gather(self, worker, dep):
+        assert isinstance(dep, str)
         deps = {dep}
 
         total_bytes = self.nbytes[dep]
@@ -1956,7 +1947,8 @@ def select_keys_for_gather(self, worker, dep):
 
         while L:
             d = L.popleft()
-            if self.dep_state.get(d) != "waiting":
+            ts = self.tasks.get(d)
+            if ts is None or ts.state != "waiting":
                 continue
             if total_bytes + self.nbytes[d] > self.target_message_size:
                 break
@@ -1966,6 +1958,19 @@ def select_keys_for_gather(self, worker, dep):
         return deps, total_bytes
 
     async def gather_dep(self, worker, dep, deps, total_nbytes, cause=None):
+        """Gather dependencies for a task from a worker who has them
+
+        Parameters
+        ----------
+        worker : str
+            address of worker to gather dependency from
+        dep : TaskState
+            task we want to gather dependencies for
+        deps : list
+            keys of dependencies to gather from worker -- this is not
+            necessarily equivalent to the full list of dependencies of ``dep``
+            as some dependencies may already be present on this worker.
+        """
         if self.status != Status.running:
             return
         with log_errors():
@@ -1976,9 +1981,11 @@ async def gather_dep(self, worker, dep, deps, total_nbytes, cause=None):
 
                 # dep states may have changed before gather_dep runs
                 # if a dep is no longer in-flight then don't fetch it
-                deps = tuple(dep for dep in deps if self.dep_state.get(dep) == "flight")
+                deps_ts = [self.tasks[key] for key in deps]
+                deps_ts = tuple(ts for ts in deps_ts if ts.state == "flight")
+                deps = [d.key for d in deps_ts]
 
-                self.log.append(("request-dep", dep, worker, deps))
+                self.log.append(("request-dep", dep.key, worker, deps))
                 logger.debug("Request %d keys", len(deps))
 
                 start = time()
@@ -1989,13 +1996,14 @@ async def gather_dep(self, worker, dep, deps, total_nbytes, cause=None):
 
                 if response["status"] == "busy":
                     self.log.append(("busy-gather", worker, deps))
-                    for dep in deps:
-                        if self.dep_state.get(dep, None) == "flight":
-                            self.transition_dep(dep, "waiting")
+                    for ts in deps_ts:
+                        if ts.state == "flight":
+                            self.transition(ts, "waiting")
                     return
 
                 if cause:
-                    self.startstops[cause].append(
+                    cause_ts = self.tasks.get(cause, TaskState(key=cause))
+                    cause_ts.startstops.append(
                         {
                             "action": "transfer",
                             "start": start + self.scheduler_delay,
@@ -2004,7 +2012,7 @@ async def gather_dep(self, worker, dep, deps, total_nbytes, cause=None):
                         }
                     )
 
-                total_bytes = sum(self.nbytes.get(dep, 0) for dep in response["data"])
+                total_bytes = sum(self.nbytes.get(key, 0) for key in response["data"])
                 duration = (stop - start) or 0.010
                 bandwidth = total_bytes / duration
                 self.incoming_transfer_log.append(
@@ -2014,7 +2022,7 @@ async def gather_dep(self, worker, dep, deps, total_nbytes, cause=None):
                         "middle": (start + stop) / 2.0 + self.scheduler_delay,
                         "duration": duration,
                         "keys": {
-                            dep: self.nbytes.get(dep, None) for dep in response["data"]
+                            key: self.nbytes.get(key, None) for key in response["data"]
                         },
                         "total": total_bytes,
                         "bandwidth": bandwidth,
@@ -2043,9 +2051,7 @@ async def gather_dep(self, worker, dep, deps, total_nbytes, cause=None):
                 logger.exception("Worker stream died during communication: %s", worker)
                 self.log.append(("receive-dep-failed", worker))
                 for d in self.has_what.pop(worker):
-                    self.who_has[d].remove(worker)
-                    if not self.who_has[d]:
-                        del self.who_has[d]
+                    self.tasks[d].who_has.remove(worker)
 
             except Exception as e:
                 logger.exception(e)
@@ -2060,14 +2066,18 @@ async def gather_dep(self, worker, dep, deps, total_nbytes, cause=None):
                 data = response.get("data", {})
 
                 for d in self.in_flight_workers.pop(worker):
+
+                    ts = self.tasks.get(d)
+
                     if not busy and d in data:
-                        self.transition_dep(d, "memory", value=data[d])
-                    elif self.dep_state.get(d) != "memory":
-                        self.transition_dep(
-                            d, "waiting", worker=worker, remove=not busy
-                        )
+                        self.transition(ts, "memory", value=data[d])
+                    elif ts is None or ts.state == "executing":
+                        self.release_key(d)
+                        continue
+                    elif ts.state not in ("ready", "memory"):
+                        self.transition(ts, "waiting", worker=worker, remove=not busy)
 
-                    if not busy and d not in data and d in self.dependents:
+                    if not busy and d not in data and ts.dependents:
                         self.log.append(("missing-dep", d))
                         self.batched_stream.send(
                             {"op": "missing-data", "errant_worker": worker, "key": d}
@@ -2087,29 +2097,29 @@ async def gather_dep(self, worker, dep, deps, total_nbytes, cause=None):
                     await asyncio.sleep(0.100 * 1.5 ** self.repetitively_busy)
 
                     # See if anyone new has the data
-                    await self.query_who_has(dep)
+                    await self.query_who_has(dep.key)
                     self.ensure_communicating()
 
     def bad_dep(self, dep):
-        exc = ValueError("Could not find dependent %s.  Check worker logs" % str(dep))
-        for key in self.dependents[dep]:
+        exc = ValueError(
+            "Could not find dependent %s.  Check worker logs" % str(dep.ts)
+        )
+        for ts in dep.dependents:
             msg = error_message(exc)
-            self.exceptions[key] = msg["exception"]
-            self.tracebacks[key] = msg["traceback"]
-            self.transition(key, "error")
-        self.release_dep(dep)
+            ts.exception = msg["exception"]
+            ts.traceback = msg["traceback"]
+            self.transition(ts, "error")
+        self.release_key(dep.key)
 
     async def handle_missing_dep(self, *deps, **kwargs):
-        original_deps = list(deps)
         self.log.append(("handle-missing", deps))
         try:
-            deps = {dep for dep in deps if dep in self.dependents}
+            deps = {dep for dep in deps if dep.dependents}
             if not deps:
                 return
 
-            for dep in list(deps):
-                suspicious = self.suspicious_deps[dep]
-                if suspicious > 5:
+            for dep in deps:
+                if dep.suspicious_count > 5:
                     deps.remove(dep)
                     self.bad_dep(dep)
             if not deps:
@@ -2118,24 +2128,26 @@ async def handle_missing_dep(self, *deps, **kwargs):
             for dep in deps:
                 logger.info(
                     "Dependent not found: %s %s .  Asking scheduler",
-                    dep,
-                    self.suspicious_deps[dep],
+                    dep.key,
+                    dep.suspicious_count,
                 )
 
-            who_has = await retry_operation(self.scheduler.who_has, keys=list(deps))
+            who_has = await retry_operation(
+                self.scheduler.who_has, keys=list(dep.key for dep in deps)
+            )
             who_has = {k: v for k, v in who_has.items() if v}
             self.update_who_has(who_has)
             for dep in deps:
-                self.suspicious_deps[dep] += 1
+                dep.suspicious_count += 1
 
-                if not who_has.get(dep):
-                    self.log.append((dep, "no workers found", self.dependents.get(dep)))
-                    self.release_dep(dep)
+                if not who_has.get(dep.key):
+                    self.log.append((dep.key, "no workers found", dep.dependents))
+                    self.release_key(dep.key)
                 else:
-                    self.log.append((dep, "new workers found"))
-                    for key in self.dependents.get(dep, ()):
-                        if key in self.waiting_for_data:
-                            self.data_needed.append(key)
+                    self.log.append((dep.key, "new workers found"))
+                    for dependent in dep.dependents:
+                        if dependent.key in dep.waiting_for_data:
+                            self.data_needed.append(dependent.key)
 
         except Exception:
             logger.error("Handle missing dep failed, retrying", exc_info=True)
@@ -2147,8 +2159,8 @@ async def handle_missing_dep(self, *deps, **kwargs):
                 raise
         finally:
             try:
-                for dep in original_deps:
-                    self._missing_dep_flight.remove(dep)
+                for dep in deps:
+                    self._missing_dep_flight.remove(dep.key)
             except KeyError:
                 pass
 
@@ -2165,10 +2177,8 @@ def update_who_has(self, who_has):
             for dep, workers in who_has.items():
                 if not workers:
                     continue
-                if dep in self.who_has:
-                    self.who_has[dep].update(workers)
-                else:
-                    self.who_has[dep] = set(workers)
+
+                self.tasks[dep].who_has.update(workers)
 
                 for worker in workers:
                     self.has_what[worker].add(dep)
@@ -2181,7 +2191,7 @@ def update_who_has(self, who_has):
             raise
 
     def steal_request(self, key):
-        state = self.task_state.get(key, None)
+        state = self.tasks[key].state
 
         response = {"op": "steal-response", "key": key, "state": state}
         self.batched_stream.send(response)
@@ -2191,64 +2201,51 @@ def steal_request(self, key):
 
     def release_key(self, key, cause=None, reason=None, report=True):
         try:
-            if key not in self.task_state:
-                return
-            state = self.task_state.pop(key)
+            ts = self.tasks.get(key, TaskState(key=key))
             if cause:
                 self.log.append((key, "release-key", {"cause": cause}))
             else:
                 self.log.append((key, "release-key"))
-            del self.tasks[key]
-            if key in self.data and key not in self.dep_state:
+            if key in self.data and not ts.dependents:
                 try:
                     del self.data[key]
                 except FileNotFoundError:
                     logger.error("Tried to delete %s but no file found", exc_info=True)
                 del self.nbytes[key]
-                del self.types[key]
-            if key in self.actors and key not in self.dep_state:
+            if key in self.actors and not ts.dependents:
                 del self.actors[key]
                 del self.nbytes[key]
-                del self.types[key]
 
-            if key in self.waiting_for_data:
-                del self.waiting_for_data[key]
+            # for any dependencies of key we are releasing remove task as dependent
+            for dependency in ts.dependencies:
+                dependency.dependents.discard(ts)
+                if not dependency.dependents and dependency.state in (
+                    "waiting",
+                    "flight",
+                ):
+                    self.release_key(dependency.key)
 
-            for dep in self.dependencies.pop(key, ()):
-                if dep in self.dependents:
-                    self.dependents[dep].discard(key)
-                    if not self.dependents[dep] and self.dep_state[dep] in (
-                        "waiting",
-                        "flight",
-                    ):
-                        self.release_dep(dep)
+            for worker in ts.who_has:
+                self.has_what[worker].discard(ts.key)
 
             if key in self.threads:
                 del self.threads[key]
-            del self.priorities[key]
-            del self.durations[key]
-
-            if key in self.exceptions:
-                del self.exceptions[key]
-            if key in self.tracebacks:
-                del self.tracebacks[key]
-
-            if key in self.startstops:
-                del self.startstops[key]
 
-            if key in self.executing:
-                self.executing.remove(key)
+            if ts.state == "executing":
+                self.executing_count -= 1
 
-            if key in self.resource_restrictions:
-                if state == "executing":
-                    for resource, quantity in self.resource_restrictions[key].items():
+            if ts.resource_restrictions is not None:
+                if ts.state == "executing":
+                    for resource, quantity in ts.resource_restrictions.items():
                         self.available_resources[resource] += quantity
-                del self.resource_restrictions[key]
 
-            if report and state in PROCESSING:  # not finished
+            if report and ts.state in PROCESSING:  # not finished
                 self.batched_stream.send({"op": "release", "key": key, "cause": cause})
 
-            self._notify_plugins("release_key", key, state, cause, reason, report)
+            self._notify_plugins("release_key", key, ts.state, cause, reason, report)
+            if key in self.tasks:
+                self.tasks.pop(key)
+            del ts
         except CommClosedError:
             pass
         except Exception as e:
@@ -2259,70 +2256,24 @@ def release_key(self, key, cause=None, reason=None, report=True):
                 pdb.set_trace()
             raise
 
-    def release_dep(self, dep, report=False):
+    def rescind_key(self, key):
         try:
-            if dep not in self.dep_state:
+            if self.tasks[key].state not in PENDING:
                 return
-            self.log.append((dep, "release-dep"))
-            state = self.dep_state.pop(dep)
-
-            if dep in self.suspicious_deps:
-                del self.suspicious_deps[dep]
-
-            if dep in self.who_has:
-                for worker in self.who_has.pop(dep):
-                    self.has_what[worker].remove(dep)
 
-            if dep not in self.task_state:
-                if dep in self.data:
-                    del self.data[dep]
-                    del self.types[dep]
-                if dep in self.actors:
-                    del self.actors[dep]
-                    del self.types[dep]
-                del self.nbytes[dep]
-
-            if dep in self.in_flight_tasks:
-                worker = self.in_flight_tasks.pop(dep)
-                self.in_flight_workers[worker].remove(dep)
-
-            for key in self.dependents.pop(dep, ()):
-                if self.task_state[key] != "memory":
-                    self.release_key(key, cause=dep)
-
-            if report and state == "memory":
-                self.batched_stream.send({"op": "release-worker-data", "keys": [dep]})
-
-            self._notify_plugins("release_dep", dep, state, report)
-        except Exception as e:
-            logger.exception(e)
-            if LOG_PDB:
-                import pdb
-
-                pdb.set_trace()
-            raise
+            ts = self.tasks.pop(key)
+
+            # Task has been rescinded
+            # For every task that it required
+            for dependency in ts.dependencies:
+                # Remove it as a dependent
+                dependency.dependents.remove(key)
+                # If the dependent is now without purpose (no dependencies), remove it
+                if not dependency.dependents:
+                    self.release_key(
+                        dependency.key, reason="All dependent keys rescinded"
+                    )
 
-    def rescind_key(self, key):
-        try:
-            if self.task_state.get(key) not in PENDING:
-                return
-            del self.task_state[key]
-            del self.tasks[key]
-            if key in self.waiting_for_data:
-                del self.waiting_for_data[key]
-
-            for dep in self.dependencies.pop(key, ()):
-                self.dependents[dep].remove(key)
-                if not self.dependents[dep]:
-                    del self.dependents[dep]
-
-            if key not in self.dependents:
-                # if key in self.nbytes:
-                #     del self.nbytes[key]
-                if key in self.priorities:
-                    del self.priorities[key]
-                if key in self.durations:
-                    del self.durations[key]
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
@@ -2435,64 +2386,74 @@ def actor_attribute(self, comm=None, actor=None, attribute=None):
         return {"status": "OK", "result": to_serialize(value)}
 
     def meets_resource_constraints(self, key):
-        if key not in self.resource_restrictions:
+        ts = self.tasks[key]
+        if not ts.resource_restrictions:
             return True
-        for resource, needed in self.resource_restrictions[key].items():
+        for resource, needed in ts.resource_restrictions.items():
             if self.available_resources[resource] < needed:
                 return False
 
         return True
 
-    def _maybe_deserialize_task(self, key):
-        if not isinstance(self.tasks[key], SerializedTask):
-            return self.tasks[key]
+    def _maybe_deserialize_task(self, ts):
+        if not isinstance(ts.runspec, SerializedTask):
+            return ts.runspec
         try:
             start = time()
-            function, args, kwargs = _deserialize(*self.tasks[key])
+            function, args, kwargs = _deserialize(*ts.runspec)
             stop = time()
 
             if stop - start > 0.010:
-                self.startstops[key].append(
+                ts.startstops.append(
                     {"action": "deserialize", "start": start, "stop": stop}
                 )
             return function, args, kwargs
         except Exception as e:
             logger.warning("Could not deserialize task", exc_info=True)
             emsg = error_message(e)
-            emsg["key"] = key
+            emsg["key"] = ts.key
             emsg["op"] = "task-erred"
             self.batched_stream.send(emsg)
-            self.log.append((key, "deserialize-error"))
+            self.log.append((ts.key, "deserialize-error"))
             raise
 
     def ensure_computing(self):
         if self.paused:
             return
         try:
-            while self.constrained and len(self.executing) < self.nthreads:
+            while self.constrained and self.executing_count < self.nthreads:
                 key = self.constrained[0]
-                if self.task_state.get(key) != "constrained":
+                ts = self.tasks[key]
+                if ts.state != "constrained":
                     self.constrained.popleft()
                     continue
                 if self.meets_resource_constraints(key):
                     self.constrained.popleft()
                     try:
                         # Ensure task is deserialized prior to execution
-                        self.tasks[key] = self._maybe_deserialize_task(key)
+                        ts.runspec = self._maybe_deserialize_task(ts)
                     except Exception:
                         continue
-                    self.transition(key, "executing")
+                    self.transition(ts, "executing")
                 else:
                     break
-            while self.ready and len(self.executing) < self.nthreads:
-                _, key = heapq.heappop(self.ready)
-                if self.task_state.get(key) in READY:
+            while self.ready and self.executing_count < self.nthreads:
+                priority, key = heapq.heappop(self.ready)
+                ts = self.tasks.get(key)
+                if ts is None:
+                    # It is possible for tasks to be released while still remaining on `ready`
+                    # The scheduler might have re-routed to a new worker and told this worker
+                    # to release.  If the task has "disappeared" just continue through the heap
+                    continue
+                elif ts.key in self.data:
+                    self.transition(ts, "memory")
+                elif ts.state in READY:
                     try:
                         # Ensure task is deserialized prior to execution
-                        self.tasks[key] = self._maybe_deserialize_task(key)
+                        ts.runspec = self._maybe_deserialize_task(ts)
                     except Exception:
                         continue
-                    self.transition(key, "executing")
+                    self.transition(ts, "executing")
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
@@ -2506,17 +2467,21 @@ async def execute(self, key, report=False):
         if self.status in (Status.closing, Status.closed, Status.closing_gracefully):
             return
         try:
-            if key not in self.executing or key not in self.task_state:
+            if key not in self.tasks:
+                return
+            ts = self.tasks[key]
+            if ts.state != "executing" or ts.runspec is None:
                 return
             if self.validate:
-                assert key not in self.waiting_for_data
-                assert self.task_state[key] == "executing"
+                assert not ts.waiting_for_data
+                assert ts.state == "executing"
 
-            function, args, kwargs = self.tasks[key]
+            function, args, kwargs = ts.runspec
 
             start = time()
             data = {}
-            for k in self.dependencies[key]:
+            for dep in ts.dependencies:
+                k = dep.key
                 try:
                     data[k] = self.data[k]
                 except KeyError:
@@ -2527,25 +2492,26 @@ async def execute(self, key, report=False):
             kwargs2 = pack_data(kwargs, data, key_types=(bytes, str))
             stop = time()
             if stop - start > 0.005:
-                self.startstops[key].append(
+                ts.startstops.append(
                     {"action": "disk-read", "start": start, "stop": stop}
                 )
                 if self.digests is not None:
                     self.digests["disk-load-duration"].add(stop - start)
 
             logger.debug(
-                "Execute key: %s worker: %s", key, self.address
+                "Execute key: %s worker: %s", ts.key, self.address
             )  # TODO: comment out?
+            assert key == ts.key
             try:
                 result = await self.executor_submit(
-                    key,
+                    ts.key,
                     apply_function,
                     args=(
                         function,
                         args2,
                         kwargs2,
                         self.execution_state,
-                        key,
+                        ts.key,
                         self.active_threads,
                         self.active_threads_lock,
                         self.scheduler_delay,
@@ -2555,30 +2521,30 @@ async def execute(self, key, report=False):
                 executor_error = e
                 raise
 
-            if self.task_state.get(key) not in ("executing", "long-running"):
+            if ts.state not in ("executing", "long-running"):
                 return
 
-            result["key"] = key
+            result["key"] = ts.key
             value = result.pop("result", None)
-            self.startstops[key].append(
+            ts.startstops.append(
                 {"action": "compute", "start": result["start"], "stop": result["stop"]}
             )
-            self.threads[key] = result["thread"]
+            self.threads[ts.key] = result["thread"]
 
             if result["op"] == "task-finished":
-                self.nbytes[key] = result["nbytes"]
-                self.types[key] = result["type"]
-                self.transition(key, "memory", value=value)
+                self.nbytes[ts.key] = result["nbytes"]
+                ts.type = result["type"]
+                self.transition(ts, "memory", value=value)
                 if self.digests is not None:
                     self.digests["task-duration"].add(result["stop"] - result["start"])
             else:
                 if isinstance(result.pop("actual-exception"), Reschedule):
-                    self.batched_stream.send({"op": "reschedule", "key": key})
-                    self.transition(key, "rescheduled", report=False)
-                    self.release_key(key, report=False)
+                    self.batched_stream.send({"op": "reschedule", "key": ts.key})
+                    self.transition(ts, "rescheduled", report=False)
+                    self.release_key(ts.key, report=False)
                 else:
-                    self.exceptions[key] = result["exception"]
-                    self.tracebacks[key] = result["traceback"]
+                    ts.exception = result["exception"]
+                    ts.traceback = result["traceback"]
                     logger.warning(
                         " Compute Failed\n"
                         "Function:  %s\n"
@@ -2590,13 +2556,13 @@ async def execute(self, key, report=False):
                         convert_kwargs_to_str(kwargs2, max_len=1000),
                         repr(result["exception"].data),
                     )
-                    self.transition(key, "error")
+                    self.transition(ts, "error")
 
-            logger.debug("Send compute response to scheduler: %s, %s", key, result)
+            logger.debug("Send compute response to scheduler: %s, %s", ts.key, result)
 
             if self.validate:
-                assert key not in self.executing
-                assert key not in self.waiting_for_data
+                assert ts.state != "executing"
+                assert not ts.waiting_for_data
 
             self.ensure_computing()
             self.ensure_communicating()
@@ -2610,9 +2576,6 @@ async def execute(self, key, report=False):
 
                     pdb.set_trace()
                 raise
-        finally:
-            if key in self.executing:
-                self.executing.remove(key)
 
     ##################
     # Administrative #
@@ -2849,86 +2812,53 @@ def _notify_plugins(self, method_name, *args, **kwargs):
     # Validation #
     ##############
 
-    def validate_key_memory(self, key):
-        assert key in self.data or key in self.actors
-        assert key in self.nbytes
-        assert key not in self.waiting_for_data
-        assert key not in self.executing
-        assert key not in self.ready
-        if key in self.dep_state:
-            assert self.dep_state[key] == "memory"
-
-    def validate_key_executing(self, key):
-        assert key in self.executing
-        assert key not in self.data
-        assert key not in self.waiting_for_data
+    def validate_task_memory(self, ts):
+        assert ts.key in self.data or ts.key in self.actors
+        assert ts.key in self.nbytes
+        assert not ts.waiting_for_data
+        assert ts.key not in self.ready
+        assert ts.state == "memory"
+
+    def validate_task_executing(self, ts):
+        assert ts.state == "executing"
+        assert ts.key not in self.data
+        assert not ts.waiting_for_data
         assert all(
-            dep in self.data or dep in self.actors for dep in self.dependencies[key]
+            dep.key in self.data or dep.key in self.actors for dep in ts.dependencies
         )
 
-    def validate_key_ready(self, key):
-        assert key in pluck(1, self.ready)
-        assert key not in self.data
-        assert key not in self.executing
-        assert key not in self.waiting_for_data
+    def validate_task_ready(self, ts):
+        assert ts.key in pluck(1, self.ready)
+        assert ts.key not in self.data
+        assert ts.state != "executing"
+        assert not ts.waiting_for_data
         assert all(
-            dep in self.data or dep in self.actors for dep in self.dependencies[key]
+            dep.key in self.data or dep.key in self.actors for dep in ts.dependencies
         )
 
-    def validate_key_waiting(self, key):
-        assert key not in self.data
-        assert not all(dep in self.data for dep in self.dependencies[key])
-
-    def validate_key(self, key):
-        try:
-            state = self.task_state[key]
-            if state == "memory":
-                self.validate_key_memory(key)
-            elif state == "waiting":
-                self.validate_key_waiting(key)
-            elif state == "ready":
-                self.validate_key_ready(key)
-            elif state == "executing":
-                self.validate_key_executing(key)
-        except Exception as e:
-            logger.exception(e)
-            if LOG_PDB:
-                import pdb
+    def validate_task_waiting(self, ts):
+        assert ts.key not in self.data
+        assert ts.state == "waiting"
+        if ts.dependencies:
+            assert not all(dep.key in self.data for dep in ts.dependencies)
 
-                pdb.set_trace()
-            raise
+    def validate_task_flight(self, ts):
+        assert ts.key not in self.data
+        assert not any(dep.key in self.ready for dep in ts.dependents)
+        assert ts.key in self.in_flight_workers[ts.coming_from]
 
-    def validate_dep_waiting(self, dep):
-        assert dep not in self.data
-        assert dep in self.nbytes
-        assert self.dependents[dep]
-        assert not any(key in self.ready for key in self.dependents[dep])
-
-    def validate_dep_flight(self, dep):
-        assert dep not in self.data
-        assert dep in self.nbytes
-        assert not any(key in self.ready for key in self.dependents[dep])
-        peer = self.in_flight_tasks[dep]
-        assert dep in self.in_flight_workers[peer]
-
-    def validate_dep_memory(self, dep):
-        assert dep in self.data or dep in self.actors
-        assert dep in self.nbytes
-        assert dep in self.types
-        if dep in self.task_state:
-            assert self.task_state[dep] == "memory"
-
-    def validate_dep(self, dep):
+    def validate_task(self, ts):
         try:
-            state = self.dep_state[dep]
-            if state == "waiting":
-                self.validate_dep_waiting(dep)
-            elif state == "flight":
-                self.validate_dep_flight(dep)
-            elif state == "memory":
-                self.validate_dep_memory(dep)
-            else:
-                raise ValueError("Unknown dependent state", state)
+            if ts.state == "memory":
+                self.validate_task_memory(ts)
+            elif ts.state == "waiting":
+                self.validate_task_waiting(ts)
+            elif ts.state == "ready":
+                self.validate_task_ready(ts)
+            elif ts.state == "executing":
+                self.validate_task_executing(ts)
+            elif ts.state == "flight":
+                self.validate_task_flight(ts)
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
@@ -2941,34 +2871,39 @@ def validate_state(self):
         if self.status != Status.running:
             return
         try:
-            for key, workers in self.who_has.items():
-                for w in workers:
-                    assert key in self.has_what[w]
+            for ts in self.tasks.values():
+                assert ts.state is not None
+                # check that worker has task
+                for worker in ts.who_has:
+                    assert ts.key in self.has_what[worker]
+                # check that deps have a set state and that dependency<->dependent links are there
+                for dep in ts.dependencies:
+                    # self.tasks was just a dict of tasks
+                    # and this check was originally that the key was in `task_state`
+                    # so we may have popped the key out of `self.tasks` but the
+                    # dependency can still be in `memory` before GC grabs it...?
+                    # Might need better bookkeeping
+                    assert dep.state is not None
+                    assert ts in dep.dependents
+                for key in ts.waiting_for_data:
+                    ts_wait = self.tasks[key]
+                    assert (
+                        ts_wait.state == "flight"
+                        or ts_wait.state == "waiting"
+                        or ts.wait.key in self._missing_dep_flight
+                        or ts_wait.who_has.issubset(self.in_flight_workers)
+                    )
+                if ts.state == "memory":
+                    assert isinstance(self.nbytes[ts.key], int)
+                    assert not ts.waiting_for_data
+                    assert ts.key in self.data or ts.key in self.actors
 
             for worker, keys in self.has_what.items():
                 for k in keys:
-                    assert worker in self.who_has[k]
-
-            for key in self.task_state:
-                self.validate_key(key)
+                    assert worker in self.tasks[k].who_has
 
-            for dep in self.dep_state:
-                self.validate_dep(dep)
-
-            for key, deps in self.waiting_for_data.items():
-                if key not in self.data_needed:
-                    for dep in deps:
-                        assert (
-                            dep in self.in_flight_tasks
-                            or dep in self._missing_dep_flight
-                            or self.who_has[dep].issubset(self.in_flight_workers)
-                        )
-
-            for key in self.tasks:
-                if self.task_state[key] == "memory":
-                    assert isinstance(self.nbytes[key], int)
-                    assert key not in self.waiting_for_data
-                    assert key in self.data or key in self.actors
+            for ts in self.tasks.values():
+                self.validate_task(ts)
 
         except Exception as e:
             logger.exception(e)
@@ -3169,7 +3104,7 @@ def secede():
     duration = time() - thread_state.start_time
     worker.loop.add_callback(
         worker.maybe_transition_long_running,
-        thread_state.key,
+        worker.tasks[thread_state.key],
         compute_duration=duration,
     )
 
diff --git a/distributed/worker_client.py b/distributed/worker_client.py
index b39c6236df9..7f506cbf1a1 100644
--- a/distributed/worker_client.py
+++ b/distributed/worker_client.py
@@ -42,7 +42,9 @@ def worker_client(timeout=3, separate_thread=True):
     client = get_client(timeout=timeout)
     if separate_thread:
         secede()  # have this thread secede from the thread pool
-        worker.loop.add_callback(worker.transition, thread_state.key, "long-running")
+        worker.loop.add_callback(
+            worker.transition, worker.tasks[thread_state.key], "long-running"
+        )
 
     yield client
 
diff --git a/docs/source/worker.rst b/docs/source/worker.rst
index dc4e56d9ac7..adf1db9b5d3 100644
--- a/docs/source/worker.rst
+++ b/docs/source/worker.rst
@@ -246,4 +246,5 @@ process.
 API Documentation
 -----------------
 
+.. autoclass:: distributed.worker.TaskState
 .. autoclass:: distributed.worker.Worker

From a0b2a605b496a0459789e72b00989e17a83ed56c Mon Sep 17 00:00:00 2001
From: marwan116 <sarieddine.marwan@gmail.com>
Date: Mon, 19 Oct 2020 12:01:07 -0400
Subject: [PATCH 1008/1550] doc: dask-worker cli memory limit option doc fix
 (#4172)

---
 distributed/cli/dask_worker.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
index 707a410348e..b3b546ea3cc 100755
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -142,11 +142,14 @@
     "--memory-limit",
     default="auto",
     show_default=True,
-    help="Bytes of memory per process that the worker can use. "
-    "This can be an integer (bytes), "
-    "float (fraction of total system memory), "
-    "string (like 5GB or 5000M), "
-    "'auto', or zero for no memory management",
+    help="""\b
+    Bytes of memory per process that the worker can use.
+    This can be:
+    - an integer (bytes), note 0 is a special case for no memory management.
+    - a float (fraction of total system memory).
+    - a string (like 5GB or 5000M).
+    - 'auto' for automatically computing the memory limit.
+    """,
 )
 @click.option(
     "--reconnect/--no-reconnect",

From 3e5b506ed94dea47c2ce565fca91709d964a8716 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Tue, 20 Oct 2020 12:04:35 -0500
Subject: [PATCH 1009/1550] Support async preload click commands (#4170)

---
 distributed/preloading.py         |  7 ++++++-
 distributed/tests/test_preload.py | 30 ++++++++++++++++++++++++++++++
 2 files changed, 36 insertions(+), 1 deletion(-)

diff --git a/distributed/preloading.py b/distributed/preloading.py
index b3d6a8e6d13..fa5d19a0c1a 100644
--- a/distributed/preloading.py
+++ b/distributed/preloading.py
@@ -172,7 +172,12 @@ async def start(self):
                 context = dask_setup.make_context(
                     "dask_setup", list(self.argv), allow_extra_args=False
                 )
-                dask_setup.callback(self.dask_server, *context.args, **context.params)
+                result = dask_setup.callback(
+                    self.dask_server, *context.args, **context.params
+                )
+                if inspect.isawaitable(result):
+                    await result
+                logger.info("Run preload setup click command: %s", self.name)
             else:
                 future = dask_setup(self.dask_server)
                 if inspect.isawaitable(future):
diff --git a/distributed/tests/test_preload.py b/distributed/tests/test_preload.py
index 989e3d629ae..ae13137d847 100644
--- a/distributed/tests/test_preload.py
+++ b/distributed/tests/test_preload.py
@@ -106,6 +106,36 @@ def check_worker():
         shutil.rmtree(tmpdir)
 
 
+@pytest.mark.asyncio
+async def test_worker_preload_click(cleanup, tmpdir):
+    CLICK_PRELOAD_TEXT = """
+import click
+
+@click.command()
+def dask_setup(worker):
+    worker.foo = 'setup'
+"""
+    async with Scheduler(port=0) as s:
+        async with Worker(s.address, preload=CLICK_PRELOAD_TEXT) as w:
+            assert w.foo == "setup"
+
+
+@pytest.mark.asyncio
+async def test_worker_preload_click_async(cleanup, tmpdir):
+    # Ensure we allow for click commands wrapping coroutines
+    # https://github.com/dask/distributed/issues/4169
+    CLICK_PRELOAD_TEXT = """
+import click
+
+@click.command()
+async def dask_setup(worker):
+    worker.foo = 'setup'
+"""
+    async with Scheduler(port=0) as s:
+        async with Worker(s.address, preload=CLICK_PRELOAD_TEXT) as w:
+            assert w.foo == "setup"
+
+
 @pytest.mark.asyncio
 async def test_preload_import_time(cleanup):
     text = """

From dd28f7af1446e71091e079f7fdceacf489899911 Mon Sep 17 00:00:00 2001
From: "Mads R. B. Kristensen" <madsbk@gmail.com>
Date: Tue, 20 Oct 2020 22:37:20 +0200
Subject: [PATCH 1010/1550] Implement pass HighLevelGraphs through
 _graph_to_futures (#4139)

* Performing alias substitution using map_basic_layers()

* Performing future unpacking through map_tasks()

* get_dependencies -> get_all_dependencies

* Bump minimum dask version

Co-authored-by: James Bourbeau <jrbourbeau@gmail.com>
---
 distributed/client.py | 63 ++++++++++++++++++++++++++-----------------
 requirements.txt      |  4 ++-
 2 files changed, 42 insertions(+), 25 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index e4a31acbbff..b6b0db0549a 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -26,10 +26,11 @@
 
 import dask
 from dask.base import tokenize, normalize_token, collections_to_dsk
-from dask.core import flatten, get_dependencies
+from dask.core import flatten
 from dask.optimization import SubgraphCallable
 from dask.compatibility import apply
 from dask.utils import ensure_dict, format_bytes, funcname
+from dask.highlevelgraph import HighLevelGraph
 
 from tlz import first, groupby, merge, valmap, keymap, partition_all
 
@@ -2577,34 +2578,52 @@ def _graph_to_futures(
 
             keyset = set(keys)
 
-            values = {
-                k: v
-                for k, v in dsk.items()
-                if isinstance(v, Future) and k not in keyset
-            }
-            if values:
-                dsk = subs_multiple(dsk, values)
+            # Make sure `dsk` is a high level graph
+            if not isinstance(dsk, HighLevelGraph):
+                dsk = HighLevelGraph.from_collections(id(dsk), dsk, dependencies=())
+
+            def substitute_future_aliases(dsk):
+                # Find aliases not in `keyset`
+                values = {
+                    k: v
+                    for k, v in dsk.items()
+                    if isinstance(v, Future) and k not in keyset
+                }
+                # And substitute all matching keys with its Future
+                if values:
+                    dsk = subs_multiple(dsk, values)
+                return dsk
+
+            # Notice, we only have to do the substitution on already materialized layers
+            dsk = dsk.map_basic_layers(substitute_future_aliases)
+
+            # We need to track all futures unpack_remotedata() unpacks
+            unpacked_futures = set()
 
             # Unpack remote data in `dsk`, which are "WrappedKeys" that are
-            # unknown to `dsk` but known to the scheduler.
-            dsk = {k: unpack_remotedata(v) for k, v in dsk.items()}
-            unpacked_futures = (
-                set.union(*[v[1] for v in dsk.values()]) if dsk else set()
-            )
+            # unknown to `dsk` but known to the scheduler
+            def unpack_remote_data(tasks):
+                tasks, futures = unpack_remotedata(tasks)
+                unpacked_futures.update(futures)
+                return tasks
+
+            dsk = dsk.map_tasks(unpack_remote_data)
+
             for future in unpacked_futures:
                 if future.client is not self:
                     msg = "Inputs contain futures that were created by another client."
                     raise ValueError(msg)
                 if tokey(future.key) not in self.futures:
                     raise CancelledError(tokey(future.key))
-            unpacked_futures_deps = {}
-            for k, v in dsk.items():
-                if len(v[1]):
-                    unpacked_futures_deps[k] = {f.key for f in v[1]}
-            dsk = {k: v[0] for k, v in dsk.items()}
 
-            # Find dependencies for the scheduler,
-            dependencies = {k: get_dependencies(dsk, k) for k in dsk}
+            # HACK: currently when submitting work to the scheduler, the client need to
+            # send all key dependencies along with the task graph. Since `dsk` doesn't
+            # know about the unpacked futures, we add them manually to `dsk` before
+            # calculating dependencies. This hack shouldn't be necessary when the
+            # scheduler accepts high level graphs.
+            dsk.keyset()
+            dsk._keys.update({f.key for f in unpacked_futures})
+            dependencies = dsk.get_all_dependencies()
 
             if priority is None:
                 # Removing all unpacked futures before calling order()
@@ -2618,10 +2637,6 @@ def _graph_to_futures(
                 priority = dask.order.order(stripped_dsk, dependencies=stripped_deps)
                 priority = keymap(tokey, priority)
 
-            # Append the dependencies of unpacked futures.
-            for k, v in unpacked_futures_deps.items():
-                dependencies[k] = set(dependencies.get(k, ())) | v
-
             # The scheduler expect all keys to be strings
             dependencies = {
                 tokey(k): [tokey(dep) for dep in deps]
diff --git a/requirements.txt b/requirements.txt
index 516ceb62a8a..d268861c45a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,7 +1,9 @@
 click >= 6.6
 cloudpickle >= 1.5.0
 contextvars;python_version<'3.7'
-dask >= 2.9.0
+# TODO: Update our dask version requirement to
+# the latest dask release before releasing distributed
+dask @ git+https://github.com/dask/dask.git@master
 msgpack >= 0.6.0
 psutil >= 5.0
 sortedcontainers !=2.0.0, !=2.0.1

From 18fff8b0ca0297e100695519931c78e579cb55a6 Mon Sep 17 00:00:00 2001
From: "Mads R. B. Kristensen" <madsbk@gmail.com>
Date: Thu, 22 Oct 2020 16:11:42 +0200
Subject: [PATCH 1011/1550] High level graph dumps/loads support (#4174)

---
 distributed/distributed-schema.yaml          |   6 ++
 distributed/distributed.yaml                 |   4 +
 distributed/protocol/core.py                 |   4 +-
 distributed/protocol/serialize.py            | 102 +++++++++++++++++--
 distributed/protocol/tests/test_serialize.py |  61 ++++++++++-
 docs/requirements.txt                        |   4 +-
 6 files changed, 168 insertions(+), 13 deletions(-)

diff --git a/distributed/distributed-schema.yaml b/distributed/distributed-schema.yaml
index 912a81920f2..de3a21ce9d1 100644
--- a/distributed/distributed-schema.yaml
+++ b/distributed/distributed-schema.yaml
@@ -224,6 +224,12 @@ properties:
                   These modules will have a ``routes`` keyword that gets added to the main HTTP Server.
                   This is also a list that can be extended with user defined modules.
 
+          allowed-imports:
+            type: array
+            description: |
+              A list of trusted root modules the schedular is allowed to import (incl. submodules). For security reasons, the
+              scheduler does not import arbitrary Python modules.
+
 
       worker:
         type: object
diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index 79ed89003c9..bc7e3eb0be6 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -55,6 +55,10 @@ distributed:
         - distributed.http.proxy
         - distributed.http.statics
 
+    allowed-imports:
+      - dask
+      - distributed
+
   worker:
     blocked-handlers: []
     multiprocessing-method: spawn
diff --git a/distributed/protocol/core.py b/distributed/protocol/core.py
index ceee3982941..6df5931e22a 100644
--- a/distributed/protocol/core.py
+++ b/distributed/protocol/core.py
@@ -219,4 +219,6 @@ def loads_msgpack(header, payload):
                 " installed" % str(header["compression"])
             )
 
-    return msgpack.loads(payload, use_list=False, **msgpack_opts)
+    return msgpack.loads(
+        payload, object_hook=msgpack_decode_default, use_list=False, **msgpack_opts
+    )
diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index bc659735445..4ae416075c7 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -6,6 +6,8 @@
 
 import dask
 from dask.base import normalize_token
+from dask.highlevelgraph import HighLevelGraph, Layer, BasicLayer
+from dask.optimization import SubgraphCallable
 
 from tlz import valmap, get_in
 
@@ -25,10 +27,11 @@
 
 lazy_registrations = {}
 
-
 dask_serialize = dask.utils.Dispatch("dask_serialize")
 dask_deserialize = dask.utils.Dispatch("dask_deserialize")
 
+_cached_allowed_modules = {}
+
 
 def dask_dumps(x, context=None):
     """Serialise object using the class-based registry"""
@@ -71,22 +74,80 @@ def pickle_loads(header, frames):
     return pickle.loads(x, buffers=buffers)
 
 
+def import_allowed_module(name):
+    if name in _cached_allowed_modules:
+        return _cached_allowed_modules[name]
+
+    # Check for non-ASCII characters
+    name = name.encode("ascii").decode()
+    # We only compare the root module
+    root = name.split(".", 1)[0]
+
+    # Note, if an empty string creeps into allowed-imports it is disallowed explicitly
+    if root and root in dask.config.get("distributed.scheduler.allowed-imports"):
+        _cached_allowed_modules[name] = importlib.import_module(name)
+        return _cached_allowed_modules[name]
+    else:
+        raise RuntimeError(
+            f"Importing {repr(name)} is not allowed, please add it to the list of "
+            "allowed modules the scheduler can import via the "
+            "distributed.scheduler.allowed-imports configuration setting."
+        )
+
+
 def msgpack_decode_default(obj):
     """
-    Custom packer/unpacker for msgpack to support Enums
+    Custom packer/unpacker for msgpack
     """
     if "__Enum__" in obj:
-        mod = importlib.import_module(obj["__module__"])
-        enum_type = getattr(mod, obj["__name__"])
-        obj = getattr(enum_type, obj["name"])
+        mod = import_allowed_module(obj["__module__"])
+        typ = getattr(mod, obj["__name__"])
+        return getattr(typ, obj["name"])
+
+    if "__Set__" in obj:
+        return set(obj["as-list"])
+
+    if "__SubgraphCallable__" in obj:
+        mod = import_allowed_module(obj["__module__"])
+        typ = getattr(mod, obj["__name__"])
+        return typ(*obj["args"])
+
+    if "__Layer__" in obj:
+        obj_name = obj["__name__"]
+        if obj_name == "BasicLayer":
+            # The default implemention of Layer returns a BasicLayer, which might
+            # not be defined in `mod` therefore we import it explicitly here
+            typ = BasicLayer
+        else:
+            mod = import_allowed_module(obj["__module__"])
+            typ = getattr(mod, obj["__name__"])
+        return typ(*obj["args"])
+
+    if "__HighLevelGraph__" in obj:
+        return HighLevelGraph(
+            obj["layers"],
+            obj["dependencies"],
+        )
+
+    if "__Serialized__" in obj:
+        # Notice, the data here is marked a Serialized rather than deserialized. This
+        # is because deserialization requires Pickle which the Scheduler cannot run
+        # because of security reasons.
+        # By marking it Serialized, the data is passed through to the workers that
+        # eventually will deserialize it.
+        return Serialized(*obj["data"])
+
     return obj
 
 
 def msgpack_encode_default(obj):
     """
-    Custom packer/unpacker for msgpack to support Enums
+    Custom packer/unpacker for msgpack
     """
 
+    if isinstance(obj, Serialize):
+        return {"__Serialized__": True, "data": serialize(obj.data)}
+
     if isinstance(obj, Enum):
         return {
             "__Enum__": True,
@@ -94,6 +155,35 @@ def msgpack_encode_default(obj):
             "__module__": obj.__module__,
             "__name__": type(obj).__name__,
         }
+
+    if isinstance(obj, set):
+        return {"__Set__": True, "as-list": list(obj)}
+
+    if isinstance(obj, SubgraphCallable):
+        cls, args = obj.__reduce__()
+        return {
+            "__SubgraphCallable__": True,
+            "__module__": obj.__module__,
+            "__name__": cls.__name__,
+            "args": args,
+        }
+
+    if isinstance(obj, Layer):
+        cls, args = obj.__reduce__()
+        return {
+            "__Layer__": True,
+            "__module__": obj.__module__,
+            "__name__": cls.__name__,
+            "args": args,
+        }
+
+    if isinstance(obj, HighLevelGraph):
+        return {
+            "__HighLevelGraph__": True,
+            "layers": obj.layers,
+            "dependencies": obj.dependencies,
+        }
+
     return obj
 
 
diff --git a/distributed/protocol/tests/test_serialize.py b/distributed/protocol/tests/test_serialize.py
index 72bc9764a22..4be344488c5 100644
--- a/distributed/protocol/tests/test_serialize.py
+++ b/distributed/protocol/tests/test_serialize.py
@@ -5,7 +5,12 @@
 import msgpack
 import numpy as np
 import pytest
-from tlz import identity
+from tlz import identity, valmap
+
+from dask.dataframe.io.parquet.core import ParquetSubgraph
+from dask.highlevelgraph import HighLevelGraph, BasicLayer
+from dask.blockwise import Blockwise
+from dask.utils_test import inc
 
 from distributed import wait
 from distributed.protocol import (
@@ -21,11 +26,14 @@
     serialize_bytelist,
     register_serialization_family,
     dask_serialize,
+    dumps,
+    loads,
 )
 from distributed.protocol.serialize import check_dask_serializable
 from distributed.utils import nbytes
 from distributed.utils_test import inc, gen_test
 from distributed.comm.utils import to_frames, from_frames
+from distributed.worker import dumps_task
 
 
 class MyObj:
@@ -197,16 +205,12 @@ def test_empty():
 
 
 def test_empty_loads():
-    from distributed.protocol import loads, dumps
-
     e = Empty()
     e2 = loads(dumps([to_serialize(e)]))
     assert isinstance(e2[0], Empty)
 
 
 def test_empty_loads_deep():
-    from distributed.protocol import loads, dumps
-
     e = Empty()
     e2 = loads(dumps([[[to_serialize(e)]]]))
     assert isinstance(e2[0][0][0], Empty)
@@ -476,3 +480,50 @@ def test_ser_memoryview_object():
     data_in = memoryview(np.array(["hello"], dtype=object))
     with pytest.raises(TypeError):
         serialize(data_in, on_error="raise")
+
+
+def test_highlevelgraphs():
+    """Check dumps/loads of a HLG"""
+
+    # Create a HLG with different types of Layers
+    layers = {
+        "basic": BasicLayer({"a key": 42}),
+        "blockwise": Blockwise(
+            output="z",
+            output_indices=("i",),
+            dsk=valmap(dumps_task, {"z": (inc, ["x"])}),
+            indices=(("x", ("i",)),),
+            numblocks={"x": (3,)},
+            concatenate=False,
+            new_axes=None,
+        ),
+        "parquet": ParquetSubgraph(
+            "",
+            None,
+            "",
+            None,
+            [],
+            [],
+            [],
+            {},
+        ),
+        "Serialize": BasicLayer({"Serialize key": Serialize(42)}),
+    }
+    dependencies = {"basic": set(), "blockwise": {"basic"}}
+    hlg = HighLevelGraph(layers, dependencies)
+
+    # Dump and load the HLG
+    header, frames = dumps(hlg)
+    res = loads([header, frames])
+
+    # Check the loaded result
+    assert isinstance(res, HighLevelGraph)
+    assert hlg.dependencies == res.dependencies
+    assert hlg.layers.keys() == res.layers.keys()
+    for l1, l2 in zip(hlg.layers.values(), res.layers.values()):
+        if "Serialize key" in l1:
+            # `Serialize` wrapped values are not deserialized by `loads()`
+            ser = l2["Serialize key"]
+            assert deserialize(ser.header, ser.frames) == 42
+        else:
+            assert dict(l1) == dict(l2)
diff --git a/docs/requirements.txt b/docs/requirements.txt
index ae537b5c474..a1f19502f04 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -2,7 +2,9 @@ numpydoc
 tornado
 toolz
 cloudpickle
-dask
+# TODO: Update our dask version requirement to
+# the latest dask release before releasing distributed
+dask @ git+https://github.com/dask/dask.git@master
 sphinx
 dask-sphinx-theme>=1.3.5
 sphinx-click

From 23ceba98b4830af006a21036f11891f09090eff2 Mon Sep 17 00:00:00 2001
From: "Richard (Rick) Zamora" <rzamora217@gmail.com>
Date: Fri, 23 Oct 2020 16:25:53 -0500
Subject: [PATCH 1012/1550] Fix _graph_to_futures bug for futures-based
 dependencies (#4178)

---
 distributed/client.py                 | 26 +++++++++++++++++++++-----
 distributed/tests/test_collections.py | 16 ++++++++++++++++
 2 files changed, 37 insertions(+), 5 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index b6b0db0549a..2a66681db2f 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -2607,7 +2607,21 @@ def unpack_remote_data(tasks):
                 unpacked_futures.update(futures)
                 return tasks
 
-            dsk = dsk.map_tasks(unpack_remote_data)
+            # HACK: Need to loop through each layer manually to collect
+            # futures-related dependencies (for now).  This should be removed
+            # once the scheduler doesn't need these dependencies.
+            _dsk = {}
+            future_deps = defaultdict(set)
+            for key, layer in dsk.layers.items():
+                _dsk[key] = layer.map_tasks(unpack_remote_data)
+                # Here is the "manual" loop where the full graph
+                # needs to be materialized.
+                if unpacked_futures:
+                    for k, v in dict(layer).items():
+                        futures = unpack_remotedata(v)[1]
+                        future_deps[k].update({f.key for f in futures})
+            if _dsk:
+                dsk = HighLevelGraph(_dsk, dsk.dependencies, dsk.key_dependencies)
 
             for future in unpacked_futures:
                 if future.client is not self:
@@ -2618,12 +2632,14 @@ def unpack_remote_data(tasks):
 
             # HACK: currently when submitting work to the scheduler, the client need to
             # send all key dependencies along with the task graph. Since `dsk` doesn't
-            # know about the unpacked futures, we add them manually to `dsk` before
-            # calculating dependencies. This hack shouldn't be necessary when the
-            # scheduler accepts high level graphs.
+            # know about the unpacked futures, we add futures-related dependencies from
+            # `future_deps`. This hack shouldn't be necessary when the scheduler accepts
+            # high level graphs.
             dsk.keyset()
-            dsk._keys.update({f.key for f in unpacked_futures})
             dependencies = dsk.get_all_dependencies()
+            if future_deps:
+                for key in dependencies.keys():
+                    dependencies[key] |= future_deps.get(key, set())
 
             if priority is None:
                 # Removing all unpacked futures before calling order()
diff --git a/distributed/tests/test_collections.py b/distributed/tests/test_collections.py
index c0aae5d5ea6..022db4080dd 100644
--- a/distributed/tests/test_collections.py
+++ b/distributed/tests/test_collections.py
@@ -210,3 +210,19 @@ async def test_delayed_none(c, s, w):
     [xx, yy] = c.compute([x, y])
     assert await xx is None
     assert await yy == 123
+
+
+@pytest.mark.parametrize("typ", [tuple, list])
+def test_tuple_futures_arg(client, typ):
+    x = client.submit(
+        make_time_dataframe,
+    )
+    df2 = client.submit(
+        pd.concat,
+        typ(
+            [
+                x,
+            ]
+        ),
+    )
+    dd.assert_eq(df2.result().iloc[:0], make_time_dataframe().iloc[:0])

From b7a2274f7ee8ac6a370604fbdd538c7a56d447af Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Tue, 27 Oct 2020 22:28:11 +0100
Subject: [PATCH 1013/1550] Fix regression in task stealing for already
 released keys (#4182)

---
 distributed/stealing.py         |  2 +-
 distributed/tests/test_steal.py | 25 +++++++++++++++++++++++++
 distributed/worker.py           |  8 +++++++-
 3 files changed, 33 insertions(+), 2 deletions(-)

diff --git a/distributed/stealing.py b/distributed/stealing.py
index 0ebffcba6d1..3d9593006d8 100644
--- a/distributed/stealing.py
+++ b/distributed/stealing.py
@@ -142,7 +142,7 @@ def steal_time_ratio(self, ts):
     def move_task_request(self, ts, victim, thief):
         try:
             if self.scheduler.validate:
-                if victim is not ts.processing_on:
+                if victim is not ts.processing_on and LOG_PDB:
                     import pdb
 
                     pdb.set_trace()
diff --git a/distributed/tests/test_steal.py b/distributed/tests/test_steal.py
index 15a8db9c22d..8db31aa8b62 100644
--- a/distributed/tests/test_steal.py
+++ b/distributed/tests/test_steal.py
@@ -1,5 +1,6 @@
 import asyncio
 import itertools
+import logging
 import random
 import sys
 import weakref
@@ -678,6 +679,30 @@ async def test_dont_steal_executing_tasks(c, s, a, b):
     assert not b.executing_count
 
 
+@gen_cluster(client=True)
+async def test_dont_steal_already_released(c, s, a, b):
+    future = c.submit(slowinc, 1, delay=0.05, workers=a.address)
+    key = future.key
+    await asyncio.sleep(0.05)
+    assert key in a.tasks
+    del future
+    await asyncio.sleep(0.05)
+    # In case the system is slow (e.g. network) ensure that nothing bad happens
+    # if the key was already released
+    assert key not in a.tasks
+    a.steal_request(key)
+    assert a.batched_stream.buffer == [
+        {"op": "steal-response", "key": key, "state": None}
+    ]
+    with captured_logger(
+        logging.getLogger("distributed.stealing"), level=logging.DEBUG
+    ) as stealing_logs:
+        await asyncio.sleep(0.05)
+
+    logs = stealing_logs.getvalue()
+    assert f"Key released between request and confirm: {key}" in logs
+
+
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 2)
 async def test_dont_steal_long_running_tasks(c, s, a, b):
     def long(delay):
diff --git a/distributed/worker.py b/distributed/worker.py
index 19d77c40646..b2ff7027743 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -2191,7 +2191,13 @@ def update_who_has(self, who_has):
             raise
 
     def steal_request(self, key):
-        state = self.tasks[key].state
+        # There may be a race condition between stealing and releasing a task.
+        # In this case the self.tasks is already cleared. The `None` will be
+        # registered as `already-computing` on the other end
+        if key in self.tasks:
+            state = self.tasks[key].state
+        else:
+            state = None
 
         response = {"op": "steal-response", "key": key, "state": state}
         self.batched_stream.send(response)

From 24007c2b81e12d98fb229504f858c8f28b6063ba Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Wed, 28 Oct 2020 16:45:50 -0500
Subject: [PATCH 1014/1550] Add TaskState metadata (#4191)

---
 distributed/http/templates/task.html |  2 +-
 distributed/scheduler.py             |  7 ++++++
 distributed/tests/test_worker.py     | 35 +++++++++++++++++++++++++++-
 distributed/worker.py                |  5 ++++
 4 files changed, 47 insertions(+), 2 deletions(-)

diff --git a/distributed/http/templates/task.html b/distributed/http/templates/task.html
index bcc0d17c0a8..0b5c10695e0 100644
--- a/distributed/http/templates/task.html
+++ b/distributed/http/templates/task.html
@@ -43,7 +43,7 @@ <h1 class="title"> Task: {{ ts.key }} </h1>
               <th> Priority </th>
               <td>{{ts.priority}}</td>
           </tr>
-          {% for attr in ['has_lost_dependencies', 'host_restrictions', 'worker_restrictions', 'resource_restrictions', 'loose_restrictions', 'suspicious', 'retries'] %}
+          {% for attr in ['has_lost_dependencies', 'host_restrictions', 'worker_restrictions', 'resource_restrictions', 'loose_restrictions', 'suspicious', 'retries', 'metadata'] %}
           {% if getattr(ts, attr) %}
           <tr>
               <th> {{attr.replace('_', ' ').title()}} </th>
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index bdae660d9af..d0637b680fb 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -593,6 +593,10 @@ class TaskState:
            into the "processing" state and be sent for execution to another
            connected worker.
 
+        .. attribute: metadata: dict
+
+           Metadata related to task.
+
         .. attribute: actor: bool
 
            Whether or not this task is an Actor.
@@ -646,6 +650,7 @@ class TaskState:
         "type",
         "group_key",
         "group",
+        "metadata",
     )
 
     def __init__(self, key, run_spec):
@@ -672,6 +677,7 @@ def __init__(self, key, run_spec):
         self.type = None
         self.group_key = key_split_group(key)
         self.group = None
+        self.metadata = {}
 
     @property
     def state(self) -> str:
@@ -2084,6 +2090,7 @@ def stimulus_task_finished(self, key=None, worker=None, **kwargs):
         if ts is None:
             return {}
         ws = self.workers[worker]
+        ts.metadata.update(kwargs["metadata"])
 
         if ts.state == "processing":
             recommendations = self.transition(key, "memory", worker=worker, **kwargs)
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 6119c8fb094..0a441e118fa 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -27,7 +27,7 @@
     Reschedule,
     wait,
 )
-from distributed.diagnostics.plugin import PipInstall
+from distributed.diagnostics.plugin import PipInstall, WorkerPlugin
 from distributed.compatibility import WINDOWS
 from distributed.core import rpc, CommClosedError, Status
 from distributed.scheduler import Scheduler
@@ -1714,3 +1714,36 @@ async def test_bad_local_directory(cleanup):
                 assert False
 
         assert not any("error" in log for log in s.get_logs())
+
+
+@pytest.mark.asyncio
+async def test_taskstate_metadata(cleanup):
+    class MyPlugin(WorkerPlugin):
+        """WorkPlugin to populate TaskState.metadata"""
+
+        def setup(self, worker):
+            self.worker = worker
+
+        def transition(self, key, start, finish, **kwargs):
+            ts = self.worker.tasks[key]
+
+            if start == "ready" and finish == "executing":
+                ts.metadata["start_time"] = time()
+            elif start == "executing" and finish == "memory":
+                ts.metadata["stop_time"] = time()
+
+    async with await Scheduler() as s:
+        async with await Worker(s.address) as w:
+            async with Client(s.address, asynchronous=True) as c:
+                await c.register_worker_plugin(MyPlugin())
+
+                f = c.submit(inc, 1)
+                await f
+
+                ts = w.tasks[f.key]
+                assert "start_time" in ts.metadata
+                assert "stop_time" in ts.metadata
+                assert ts.metadata["stop_time"] > ts.metadata["start_time"]
+
+                # Check that Scheduler TaskState.metadata was also updated
+                assert s.tasks[f.key].metadata == ts.metadata
diff --git a/distributed/worker.py b/distributed/worker.py
index b2ff7027743..a682beadcd7 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -126,6 +126,9 @@ class TaskState:
         The number of times a dependency has not been where we expected it
     * **startstops**: ``[{startstop}]``
         Log of transfer, load, and compute times for a task
+    * **metadata**: ``dict``
+        Metadata related to task. Stored metadata should be msgpack
+        serializable (e.g. int, string, list, dict).
 
     Parameters
     ----------
@@ -156,6 +159,7 @@ def __init__(self, key, runspec=None):
         self.type = None
         self.suspicious_count = 0
         self.startstops = list()
+        self.metadata = {}
 
     def __repr__(self):
         return "<Task %r %s>" % (self.key, self.state)
@@ -1886,6 +1890,7 @@ def send_task_state_to_scheduler(self, ts):
                 "thread": self.threads.get(ts.key),
                 "type": typ_serialized,
                 "typename": typename(typ),
+                "metadata": ts.metadata,
             }
         elif ts.exception is not None:
             d = {

From d2e63c515f6293da626116530cf7118829d5e15c Mon Sep 17 00:00:00 2001
From: "Mads R. B. Kristensen" <madsbk@gmail.com>
Date: Fri, 30 Oct 2020 15:45:35 +0100
Subject: [PATCH 1015/1550] Avoid materializing layers in Client.compute()
 (#4196)

---
 distributed/client.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/distributed/client.py b/distributed/client.py
index 2a66681db2f..0fbd6fcc1f0 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -2936,8 +2936,19 @@ def compute(
         if not isinstance(priority, Number):
             priority = {k: p for c, p in priority.items() for k in self._expand_key(c)}
 
+        if not isinstance(dsk, HighLevelGraph):
+            dsk = HighLevelGraph.from_collections(id(dsk), dsk, dependencies=())
+
+        # Let's append the finalize graph to dsk
+        finalize_name = tokenize(names)
+        layers = {finalize_name: dsk2}
+        layers.update(dsk.layers)
+        dependencies = {finalize_name: set(dsk.layers.keys())}
+        dependencies.update(dsk.dependencies)
+        dsk = HighLevelGraph(layers, dependencies)
+
         futures_dict = self._graph_to_futures(
-            merge(dsk2, dsk),
+            dsk,
             names,
             restrictions,
             loose_restrictions,

From 8612473a1c289d469e209891244bd8efb7d0f4d8 Mon Sep 17 00:00:00 2001
From: Peter Andreas Entschev <peter@entschev.com>
Date: Fri, 30 Oct 2020 17:12:04 +0100
Subject: [PATCH 1016/1550] Replace `AsyncProcess` exit handler by
 weakref.finalize (#4184)

---
 distributed/process.py    | 30 +++++++++++++-----------------
 distributed/utils_test.py |  2 --
 2 files changed, 13 insertions(+), 19 deletions(-)

diff --git a/distributed/process.py b/distributed/process.py
index a0d462d4074..1c11dd2e3d7 100644
--- a/distributed/process.py
+++ b/distributed/process.py
@@ -1,4 +1,3 @@
-import atexit
 import logging
 import os
 from queue import Queue as PyQueue
@@ -81,8 +80,10 @@ def __init__(self, loop=None, target=None, name=None, args=(), kwargs={}):
                 dask.config.global_config,
             ),
         )
-        _dangling.add(self._process)
         self._name = self._process.name
+        self._proc_finalizer = weakref.finalize(
+            self, _asyncprocess_finalizer, self._process
+        )
         self._watch_q = PyQueue()
         self._exit_future = Future()
         self._exit_callback = None
@@ -118,8 +119,8 @@ def stop_thread(q):
             # We don't join the thread here as a finalizer can be called
             # asynchronously from anywhere
 
-        self._finalizer = weakref.finalize(self, stop_thread, q=self._watch_q)
-        self._finalizer.atexit = False
+        self._thread_finalizer = weakref.finalize(self, stop_thread, q=self._watch_q)
+        self._thread_finalizer.atexit = False
 
     def _on_exit(self, exitcode):
         # Called from the event loop when the child process exited
@@ -292,7 +293,7 @@ def close(self):
         immediately and does not ensure the child process has exited.
         """
         if not self._closed:
-            self._finalizer()
+            self._thread_finalizer()
             self._process = None
             self._closed = True
 
@@ -334,15 +335,10 @@ def daemon(self, value):
         self._process.daemon = value
 
 
-_dangling = weakref.WeakSet()
-
-
-@atexit.register
-def _cleanup_dangling():
-    for proc in list(_dangling):
-        if proc.is_alive():
-            try:
-                logger.info("reaping stray process %s" % (proc,))
-                proc.terminate()
-            except OSError:
-                pass
+def _asyncprocess_finalizer(proc):
+    if proc.is_alive():
+        try:
+            logger.info("reaping stray process %s" % (proc,))
+            proc.terminate()
+        except OSError:
+            pass
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index 1a373aeac2c..85b4f17d60e 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -44,7 +44,6 @@
 from .core import connect, rpc, CommClosedError, Status
 from .deploy import SpecCluster
 from .metrics import time
-from .process import _cleanup_dangling
 from .proctitle import enable_proctitle_on_children
 from .security import Security
 from .utils import (
@@ -1451,7 +1450,6 @@ def check_process_leak(check=True):
         else:
             assert not mp_context.active_children()
 
-    _cleanup_dangling()
     for proc in mp_context.active_children():
         proc.terminate()
 

From cc7eab821fbdfd72ad797228f094c7dae1b31c2d Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Fri, 30 Oct 2020 11:26:54 -0500
Subject: [PATCH 1017/1550] Release notes for 2.30.1 (#4205)

---
 docs/source/changelog.rst | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index dcfb8b455f3..2933ada363d 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,13 @@
 Changelog
 =========
 
+2.30.1 - 2020-10-30
+-------------------
+
+- Replace ``AsyncProcess`` exit handler by ``weakref.finalize`` (:pr:`4184`) `Peter Andreas Entschev`_
+- Remove hard coded connect handshake timeouts (:pr:`4176`) `Florian Jetter`_
+
+
 2.30.0 - 2020-10-06
 -------------------
 

From da4d5bdd91554b1faa1a23ec569dab67f4a9a65e Mon Sep 17 00:00:00 2001
From: Gil Forsyth <gforsyth@users.noreply.github.com>
Date: Fri, 30 Oct 2020 12:55:33 -0400
Subject: [PATCH 1018/1550] Fix state check in test_close_gracefully (#4203)

Also adding a typo fix (I had two duplicate states in the `TaskState`
docstring) and a very small change to the way state is assigned in
`add_task` that has been bugging me for a few days.
---
 distributed/tests/test_worker.py | 2 +-
 distributed/worker.py            | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 0a441e118fa..ddc5dfcb94e 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -1571,7 +1571,7 @@ async def test_close_gracefully(c, s, a, b):
     assert b.address not in s.workers
     assert mem.issubset(set(a.data))
     for ts in proc:
-        assert ts.state in ("processing", "memory")
+        assert ts.state in ("executing", "memory")
 
 
 @pytest.mark.slow
diff --git a/distributed/worker.py b/distributed/worker.py
index a682beadcd7..9fc6b3303ff 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -105,8 +105,7 @@ class TaskState:
         The priority this task given by the scheduler.  Determines run order.
     * **state**: ``str``
         The current state of the task. One of ["waiting", "ready", "executing",
-        "memory", "flight", "executing", "error", "long-running",
-        "rescheduled", "error"]
+        "memory", "flight", "long-running", "rescheduled", "error"]
     * **who_has**: ``set(worker)``
         Workers that we believe have this data
     * **coming_from**: ``str``
@@ -1425,6 +1424,8 @@ def add_task(
                 if ts.state == "erred":
                     ts.exception = None
                     ts.traceback = None
+                else:
+                    ts.state = "waiting"
             else:
                 self.log.append((key, "new"))
                 self.tasks[key] = ts = TaskState(
@@ -1443,7 +1444,6 @@ def add_task(
             ts.duration = duration
             if resource_restrictions:
                 ts.resource_restrictions = resource_restrictions
-            ts.state = "waiting"
 
             if nbytes is not None:
                 self.nbytes.update(nbytes)

From cee4e3c99c34c1e515b9e80032d19c24b5f390ed Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Fri, 30 Oct 2020 12:16:31 -0500
Subject: [PATCH 1019/1550] Send active task durations from worker to scheduler
 (#4192)

---
 distributed/tests/test_worker.py | 17 +++++++++++++++++
 distributed/worker.py            | 17 ++++++++++++++++-
 2 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index ddc5dfcb94e..55e0e8c387e 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -1676,6 +1676,23 @@ async def test_update_latency(cleanup):
                 assert w.digests["latency"].size() > 0
 
 
+@pytest.mark.asyncio
+async def test_heartbeat_executing(cleanup):
+    async with await Scheduler() as s:
+        async with await Worker(s.address) as w:
+            async with Client(s.address, asynchronous=True) as c:
+                ws = s.workers[w.address]
+                # Initially there are no active tasks
+                assert not ws.metrics["executing"]
+                # Submit a task and ensure the worker's heartbeat includes the task
+                # in it's executing
+                f = c.submit(slowinc, 1, delay=1)
+                while not ws.metrics["executing"]:
+                    await w.heartbeat()
+                assert f.key in ws.metrics["executing"]
+                await f
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize("reconnect", [True, False])
 async def test_heartbeat_comm_closed(cleanup, monkeypatch, reconnect):
diff --git a/distributed/worker.py b/distributed/worker.py
index 9fc6b3303ff..830bc458ac6 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -125,6 +125,10 @@ class TaskState:
         The number of times a dependency has not been where we expected it
     * **startstops**: ``[{startstop}]``
         Log of transfer, load, and compute times for a task
+    * **start_time**: ``float``
+        Time at which task begins running
+    * **stop_time**: ``float``
+        Time at which task finishes running
     * **metadata**: ``dict``
         Metadata related to task. Stored metadata should be msgpack
         serializable (e.g. int, string, list, dict).
@@ -158,6 +162,8 @@ def __init__(self, key, runspec=None):
         self.type = None
         self.suspicious_count = 0
         self.startstops = list()
+        self.start_time = None
+        self.stop_time = None
         self.metadata = {}
 
     def __repr__(self):
@@ -766,8 +772,8 @@ def local_dir(self):
         return self.local_directory
 
     async def get_metrics(self):
+        now = time()
         core = dict(
-            executing=self.executing_count,
             in_memory=len(self.data),
             ready=len(self.ready),
             in_flight=self.in_flight_tasks,
@@ -776,6 +782,10 @@ async def get_metrics(self):
                 "workers": dict(self.bandwidth_workers),
                 "types": keymap(typename, self.bandwidth_types),
             },
+            executing={
+                key: now - self.tasks[key].start_time
+                for key in self.active_threads.values()
+            },
         )
         custom = {}
         for k, metric in self.metrics.items():
@@ -2316,11 +2326,16 @@ def executor_submit(self, key, function, args=(), kwargs=None, executor=None):
         pc = PeriodicCallback(
             lambda: logger.debug("future state: %s - %s", key, future._state), 1000
         )
+        ts = self.tasks.get(key)
+        if ts is not None:
+            ts.start_time = time()
         pc.start()
         try:
             yield future
         finally:
             pc.stop()
+            if ts is not None:
+                ts.stop_time = time()
 
         result = future.result()
 

From debe1b80e74e43ed88001c40a9cf07cfefa2fc47 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Mon, 2 Nov 2020 17:56:58 -0600
Subject: [PATCH 1020/1550] Pin pytest-asyncio version (#4212)

---
 continuous_integration/environment-windows.yml | 2 +-
 continuous_integration/travis/install.sh       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/continuous_integration/environment-windows.yml b/continuous_integration/environment-windows.yml
index 2cede561425..5bf1c4903d1 100644
--- a/continuous_integration/environment-windows.yml
+++ b/continuous_integration/environment-windows.yml
@@ -17,7 +17,7 @@ dependencies:
   - prometheus_client
   - psutil
   - pytest
-  - pytest-asyncio
+  - pytest-asyncio<0.14.0
   - pytest-repeat
   - pytest-timeout
   - pytest-faulthandler
diff --git a/continuous_integration/travis/install.sh b/continuous_integration/travis/install.sh
index 834d39aca9c..d7a54093ce0 100644
--- a/continuous_integration/travis/install.sh
+++ b/continuous_integration/travis/install.sh
@@ -50,7 +50,7 @@ conda create -n dask-distributed -c conda-forge -c defaults \
     prometheus_client \
     psutil \
     'pytest>=4' \
-    pytest-asyncio \
+    'pytest-asyncio<0.14.0' \
     pytest-faulthandler \
     pytest-repeat \
     pytest-timeout \

From 4205280f3f6584c59b5bc35af5f1938e328654f9 Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Tue, 3 Nov 2020 16:44:20 +0100
Subject: [PATCH 1021/1550] Remove hard coded connect handshake timeouts
 (#4176)

---
 distributed/comm/core.py             | 136 +++++++++++++-------------
 distributed/comm/tcp.py              |   4 +-
 distributed/comm/tests/test_comms.py | 140 +++++++++++++++++++++------
 3 files changed, 180 insertions(+), 100 deletions(-)

diff --git a/distributed/comm/core.py b/distributed/comm/core.py
index a0141e4d816..21f9446725b 100644
--- a/distributed/comm/core.py
+++ b/distributed/comm/core.py
@@ -213,9 +213,15 @@ async def _():
 
     async def on_connection(self, comm: Comm, handshake_overrides=None):
         local_info = {**comm.handshake_info(), **(handshake_overrides or {})}
+
+        timeout = dask.config.get("distributed.comm.timeouts.connect")
+        timeout = parse_timedelta(timeout, default="seconds")
         try:
-            write = await asyncio.wait_for(comm.write(local_info), 1)
-            handshake = await asyncio.wait_for(comm.read(), 1)
+            # Timeout is to ensure that we'll terminate connections eventually.
+            # Connector side will employ smaller timeouts and we should only
+            # reach this if the comm is dead anyhow.
+            write = await asyncio.wait_for(comm.write(local_info), timeout=timeout)
+            handshake = await asyncio.wait_for(comm.read(), timeout=timeout)
             # This would be better, but connections leak if worker is closed quickly
             # write, handshake = await asyncio.gather(comm.write(local_info), comm.read())
         except Exception as e:
@@ -262,79 +268,71 @@ async def connect(
     comm = None
 
     start = time()
-    deadline = start + timeout
-    error = None
-
-    def _raise(error):
-        error = error or "connect() didn't finish in time"
-        msg = "Timed out trying to connect to %r after %s s: %s" % (
-            addr,
-            timeout,
-            error,
-        )
-        raise IOError(msg)
 
-    backoff = 0.01
-    if timeout and timeout / 20 < backoff:
-        backoff = timeout / 20
+    def time_left():
+        deadline = start + timeout
+        return max(0, deadline - time())
 
-    retry_timeout_backoff = random.randrange(140, 160) / 100
+    backoff_base = 0.01
+    attempt = 0
 
-    # This starts a thread
-    while True:
+    # Prefer multiple small attempts than one long attempt. This should protect
+    # primarily from DNS race conditions
+    # gh3104, gh4176, gh4167
+    intermediate_cap = timeout / 5
+    active_exception = None
+    while time_left() > 0:
         try:
-            while deadline - time() > 0:
-
-                async def _():
-                    comm = await connector.connect(
-                        loc, deserialize=deserialize, **connection_args
-                    )
-                    local_info = {
-                        **comm.handshake_info(),
-                        **(handshake_overrides or {}),
-                    }
-                    try:
-                        handshake = await asyncio.wait_for(comm.read(), 1)
-                        write = await asyncio.wait_for(comm.write(local_info), 1)
-                        # This would be better, but connections leak if worker is closed quickly
-                        # write, handshake = await asyncio.gather(comm.write(local_info), comm.read())
-                    except Exception as e:
-                        with suppress(Exception):
-                            await comm.close()
-                        raise CommClosedError() from e
-
-                    comm.remote_info = handshake
-                    comm.remote_info["address"] = comm._peer_addr
-                    comm.local_info = local_info
-                    comm.local_info["address"] = comm._local_addr
-
-                    comm.handshake_options = comm.handshake_configuration(
-                        comm.local_info, comm.remote_info
-                    )
-                    return comm
-
-                with suppress(TimeoutError):
-                    comm = await asyncio.wait_for(
-                        _(), timeout=min(deadline - time(), retry_timeout_backoff)
-                    )
-                    break
-            if not comm:
-                _raise(error)
+            comm = await asyncio.wait_for(
+                connector.connect(loc, deserialize=deserialize, **connection_args),
+                timeout=min(intermediate_cap, time_left()),
+            )
+            break
         except FatalCommClosedError:
             raise
-        except EnvironmentError as e:
-            error = str(e)
-            if time() < deadline:
-                logger.debug("Could not connect, waiting before retrying")
-                await asyncio.sleep(backoff)
-                backoff *= random.randrange(140, 160) / 100
-                retry_timeout_backoff *= random.randrange(140, 160) / 100
-                backoff = min(backoff, 1)  # wait at most one second
-            else:
-                _raise(error)
-        else:
-            break
-
+        # CommClosed, EnvironmentError inherit from OSError
+        except (TimeoutError, OSError) as exc:
+            active_exception = exc
+
+            # The intermediate capping is mostly relevant for the initial
+            # connect. Afterwards we should be more forgiving
+            intermediate_cap = intermediate_cap * 1.5
+            # FullJitter see https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/
+
+            upper_cap = min(time_left(), backoff_base * (2 ** attempt))
+            backoff = random.uniform(0, upper_cap)
+            attempt += 1
+            logger.debug("Could not connect, waiting for %s before retrying", backoff)
+            await asyncio.sleep(backoff)
+    else:
+        raise IOError(
+            f"Timed out trying to connect to {addr} after {timeout} s"
+        ) from active_exception
+
+    local_info = {
+        **comm.handshake_info(),
+        **(handshake_overrides or {}),
+    }
+    try:
+        # This would be better, but connections leak if worker is closed quickly
+        # write, handshake = await asyncio.gather(comm.write(local_info), comm.read())
+        handshake = await asyncio.wait_for(comm.read(), time_left())
+        await asyncio.wait_for(comm.write(local_info), time_left())
+    except Exception as exc:
+        with suppress(Exception):
+            await comm.close()
+        raise IOError(
+            f"Timed out during handshake while connecting to {addr} after {timeout} s"
+        ) from exc
+
+    comm.remote_info = handshake
+    comm.remote_info["address"] = comm._peer_addr
+    comm.local_info = local_info
+    comm.local_info["address"] = comm._local_addr
+
+    comm.handshake_options = comm.handshake_configuration(
+        comm.local_info, comm.remote_info
+    )
     return comm
 
 
diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py
index ae497599e17..d92f83fa445 100644
--- a/distributed/comm/tcp.py
+++ b/distributed/comm/tcp.py
@@ -1,6 +1,7 @@
 import errno
 import logging
 import socket
+from ssl import SSLError
 import struct
 import sys
 from tornado import gen
@@ -349,7 +350,6 @@ async def connect(self, address, deserialize=True, **connection_args):
             stream = await self.client.connect(
                 ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs
             )
-
             # Under certain circumstances tornado will have a closed connnection with an error and not raise
             # a StreamClosedError.
             #
@@ -360,6 +360,8 @@ async def connect(self, address, deserialize=True, **connection_args):
         except StreamClosedError as e:
             # The socket connect() call failed
             convert_stream_closed_error(self, e)
+        except SSLError as err:
+            raise FatalCommClosedError() from err
 
         local_address = self.prefix + get_stream_address(stream)
         comm = self.comm_class(
diff --git a/distributed/comm/tests/test_comms.py b/distributed/comm/tests/test_comms.py
index 0a8f5250485..707819ba596 100644
--- a/distributed/comm/tests/test_comms.py
+++ b/distributed/comm/tests/test_comms.py
@@ -1,45 +1,42 @@
 import asyncio
-import types
-from functools import partial
 import os
 import sys
 import threading
+import types
 import warnings
+from functools import partial
 
+import distributed
 import pkg_resources
 import pytest
-
-from tornado import ioloop
-from tornado.concurrent import Future
-
-import distributed
-from distributed.metrics import time
-from distributed.utils import get_ip, get_ipv6
-from distributed.utils_test import (
-    requires_ipv6,
-    has_ipv6,
-    get_cert,
-    get_server_ssl_context,
-    get_client_ssl_context,
-)
-from distributed.utils_test import loop  # noqa: F401
-
-from distributed.protocol import to_serialize, Serialized, serialize, deserialize
-
-from distributed.comm.registry import backends, get_backend
 from distributed.comm import (
-    tcp,
-    inproc,
+    CommClosedError,
     connect,
+    get_address_host,
+    get_local_address_for,
+    inproc,
     listen,
-    CommClosedError,
     parse_address,
     parse_host_port,
-    unparse_host_port,
     resolve_address,
-    get_address_host,
-    get_local_address_for,
+    tcp,
+    unparse_host_port,
+)
+from distributed.comm.registry import backends, get_backend
+from distributed.comm.tcp import TCP, TCPBackend, TCPConnector
+from distributed.metrics import time
+from distributed.protocol import Serialized, deserialize, serialize, to_serialize
+from distributed.utils import get_ip, get_ipv6
+from distributed.utils_test import loop  # noqa: F401
+from distributed.utils_test import (
+    get_cert,
+    get_client_ssl_context,
+    get_server_ssl_context,
+    has_ipv6,
+    requires_ipv6,
 )
+from tornado import ioloop
+from tornado.concurrent import Future
 
 EXTERNAL_IP4 = get_ip()
 if has_ipv6():
@@ -218,7 +215,7 @@ async def handle_comm(comm):
         await comm.write(msg)
         await comm.close()
 
-    listener = await tcp.TCPListener("localhost", handle_comm)
+    listener = await tcp.TCPListener("127.0.0.1", handle_comm)
     host, port = listener.get_host_port()
     assert host in ("localhost", "127.0.0.1", "::1")
     assert port > 0
@@ -264,7 +261,7 @@ async def handle_comm(comm):
     server_ctx = get_server_ssl_context()
     client_ctx = get_client_ssl_context()
 
-    listener = await tcp.TLSListener("localhost", handle_comm, ssl_context=server_ctx)
+    listener = await tcp.TLSListener("127.0.0.1", handle_comm, ssl_context=server_ctx)
     host, port = listener.get_host_port()
     assert host in ("localhost", "127.0.0.1", "::1")
     assert port > 0
@@ -665,7 +662,8 @@ async def handle_comm(comm):
 
     with pytest.raises(EnvironmentError) as excinfo:
         await connect(listener.contact_address, timeout=2, ssl_context=cli_ctx)
-    assert "certificate verify failed" in str(excinfo.value)
+
+    assert "certificate verify failed" in str(excinfo.value.__cause__)
 
 
 #
@@ -797,6 +795,88 @@ async def handle_comm(comm):
 #
 
 
+async def echo(comm):
+    message = await comm.read()
+    await comm.write(message)
+
+
+@pytest.mark.asyncio
+async def test_retry_connect(monkeypatch):
+    async def echo(comm):
+        message = await comm.read()
+        await comm.write(message)
+
+    class UnreliableConnector(TCPConnector):
+        def __init__(self):
+
+            self.num_failures = 2
+            self.failures = 0
+            super().__init__()
+
+        async def connect(self, address, deserialize=True, **connection_args):
+            if self.failures > self.num_failures:
+                return await super().connect(address, deserialize, **connection_args)
+            else:
+                self.failures += 1
+                raise IOError()
+
+    class UnreliableBackend(TCPBackend):
+        _connector_class = UnreliableConnector
+
+    monkeypatch.setitem(backends, "tcp", UnreliableBackend())
+
+    listener = await listen("tcp://127.0.0.1:1234", echo)
+    try:
+        comm = await connect(listener.contact_address)
+        await comm.write(b"test")
+        msg = await comm.read()
+        assert msg == b"test"
+    finally:
+        listener.stop()
+
+
+@pytest.mark.asyncio
+async def test_handshake_slow_comm(monkeypatch):
+    class SlowComm(TCP):
+        def __init__(self, *args, delay_in_comm=0.5, **kwargs):
+            super().__init__(*args, **kwargs)
+            self.delay_in_comm = delay_in_comm
+
+        async def read(self, *args, **kwargs):
+            await asyncio.sleep(self.delay_in_comm)
+            return await super().read(*args, **kwargs)
+
+        async def write(self, *args, **kwargs):
+            await asyncio.sleep(self.delay_in_comm)
+            res = await super(type(self), self).write(*args, **kwargs)
+            return res
+
+    class SlowConnector(TCPConnector):
+        comm_class = SlowComm
+
+    class SlowBackend(TCPBackend):
+        _connector_class = SlowConnector
+
+    monkeypatch.setitem(backends, "tcp", SlowBackend())
+
+    listener = await listen("tcp://127.0.0.1:1234", echo)
+    try:
+        comm = await connect(listener.contact_address)
+        await comm.write(b"test")
+        msg = await comm.read()
+        assert msg == b"test"
+
+        import dask
+
+        with dask.config.set({"distributed.comm.timeouts.connect": "100ms"}):
+            with pytest.raises(
+                IOError, match="Timed out during handshake while connecting to"
+            ):
+                await connect(listener.contact_address)
+    finally:
+        listener.stop()
+
+
 async def check_connect_timeout(addr):
     t1 = time()
     with pytest.raises(IOError):

From 87041b615fd34c2556d3bf070f38dd6c768a6ce3 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Tue, 3 Nov 2020 21:25:46 -0600
Subject: [PATCH 1022/1550] Update changelog for 2.30.1 (#4217)

---
 docs/source/changelog.rst | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index 2933ada363d..6671e52c7b5 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,9 +1,10 @@
 Changelog
 =========
 
-2.30.1 - 2020-10-30
+2.30.1 - 2020-11-03
 -------------------
 
+- Pin ``pytest-asyncio`` version (:pr:`4212`) `James Bourbeau`_
 - Replace ``AsyncProcess`` exit handler by ``weakref.finalize`` (:pr:`4184`) `Peter Andreas Entschev`_
 - Remove hard coded connect handshake timeouts (:pr:`4176`) `Florian Jetter`_
 

From d7f532caa1564ef09d456d60125c03200fa60fef Mon Sep 17 00:00:00 2001
From: Gil Forsyth <gforsyth@users.noreply.github.com>
Date: Wed, 4 Nov 2020 09:41:19 -0500
Subject: [PATCH 1023/1550] Task state logs and data fix (#4206)

This fixes the sporadic error that was showing up in
test_failed_workers.py::test_worker_who_has_clears_after_failed_connection

The tests would occasionally hit a validation error when trying to
transition from waiting -> executing because the task key already
existed in `self.data`.

I thought this was because of an extra call to `release_key` somewhere,
and it _is_, but that call is _in the test_ -- it's called explicitly as
part of that test.

So when it's called but the given task still has dependent tasks, we
remove the TaskState object but leave the data in `self.data` (because
the dependents probably need it) -- but then when the task is recreated
as part of the call to `add_task` (on behalf of that depedent task), the
state is set to "waiting", because this is almost always the correct
state.

But not always, as in this case. If the data already exists on the
worker because something called `release_key` and then the task is
recreated, the state should be `memory` because the output is already in `self.data`

Co-authored-by: fjetter <fjetter@users.noreply.github.com>
---
 distributed/worker.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/distributed/worker.py b/distributed/worker.py
index 830bc458ac6..b9bc9b63375 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -1464,7 +1464,9 @@ def add_task(
                 assert workers
                 if dependency not in self.tasks:
                     self.tasks[dependency] = dep_ts = TaskState(key=dependency)
-                    dep_ts.state = "waiting"
+                    dep_ts.state = (
+                        "waiting" if dependency not in self.data else "memory"
+                    )
 
                 dep_ts = self.tasks[dependency]
                 self.log.append((dependency, "new-dep", dep_ts.state))
@@ -2117,7 +2119,7 @@ async def gather_dep(self, worker, dep, deps, total_nbytes, cause=None):
 
     def bad_dep(self, dep):
         exc = ValueError(
-            "Could not find dependent %s.  Check worker logs" % str(dep.ts)
+            "Could not find dependent %s.  Check worker logs" % str(dep.key)
         )
         for ts in dep.dependents:
             msg = error_message(exc)
@@ -2169,7 +2171,7 @@ async def handle_missing_dep(self, *deps, **kwargs):
             retries = kwargs.get("retries", 5)
             self.log.append(("handle-missing-failed", retries, deps))
             if retries > 0:
-                await self.handle_missing_dep(self, *deps, retries=retries - 1)
+                await self.handle_missing_dep(*deps, retries=retries - 1)
             else:
                 raise
         finally:

From 96516fb6f566f492eb88b1e769836dc864abb51c Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Thu, 5 Nov 2020 11:50:35 -0600
Subject: [PATCH 1024/1550] Add get_task_metadata context manager (#4216)

---
 distributed/__init__.py          |  1 +
 distributed/client.py            | 39 +++++++++++++++++++++++
 distributed/scheduler.py         | 43 ++++++++++++++++++++++++++
 distributed/tests/test_client.py | 53 +++++++++++++++++++++++++++++++-
 distributed/tests/test_worker.py | 18 ++---------
 distributed/utils_test.py        | 16 ++++++++++
 docs/source/api.rst              |  2 ++
 7 files changed, 156 insertions(+), 16 deletions(-)

diff --git a/distributed/__init__.py b/distributed/__init__.py
index 0f6858fc9c9..bca3195b39c 100644
--- a/distributed/__init__.py
+++ b/distributed/__init__.py
@@ -17,6 +17,7 @@
     futures_of,
     get_task_stream,
     performance_report,
+    get_task_metadata,
 )
 from .lock import Lock
 from .nanny import Nanny
diff --git a/distributed/client.py b/distributed/client.py
index 0fbd6fcc1f0..b5fda6b87c8 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -4837,6 +4837,45 @@ def __exit__(self, typ, value, traceback):
         get_client().sync(self.__aexit__, type, value, traceback, code=code)
 
 
+class get_task_metadata:
+    """Collect task metadata within a context block
+
+    This gathers ``TaskState`` metadata and final state from the scheduler
+    for tasks which are submitted and finished within the scope of this
+    context manager.
+
+    Examples
+    --------
+    >>> with get_task_metadata() as tasks:
+    ...     x.compute()
+    >>> tasks.metadata
+    {...}
+    >>> tasks.state
+    {...}
+    """
+
+    def __init__(self):
+        self.name = f"task-metadata-{uuid.uuid4().hex}"
+        self.keys = set()
+        self.metadata = None
+        self.state = None
+
+    async def __aenter__(self):
+        await get_client().scheduler.start_task_metadata(name=self.name)
+        return self
+
+    async def __aexit__(self, typ, value, traceback):
+        response = await get_client().scheduler.stop_task_metadata(name=self.name)
+        self.metadata = response["metadata"]
+        self.state = response["state"]
+
+    def __enter__(self):
+        return get_client().sync(self.__aenter__)
+
+    def __exit__(self, typ, value, traceback):
+        return get_client().sync(self.__aexit__, type, value, traceback)
+
+
 @contextmanager
 def temp_default_client(c):
     """Set the default client for the duration of the context
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index d0637b680fb..1b5d7f46b4b 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1363,6 +1363,8 @@ def __init__(
             "adaptive_target": self.adaptive_target,
             "workers_to_close": self.workers_to_close,
             "subscribe_worker_status": self.subscribe_worker_status,
+            "start_task_metadata": self.start_task_metadata,
+            "stop_task_metadata": self.stop_task_metadata,
         }
 
         self._transitions = {
@@ -3855,6 +3857,27 @@ def get_task_stream(self, comm=None, start=None, stop=None, count=None):
         ts = [p for p in self.plugins if isinstance(p, TaskStreamPlugin)][0]
         return ts.collect(start=start, stop=stop, count=count)
 
+    def start_task_metadata(self, comm=None, name=None):
+        plugin = CollectTaskMetaDataPlugin(scheduler=self, name=name)
+
+        self.add_plugin(plugin)
+
+    def stop_task_metadata(self, comm=None, name=None):
+        plugins = [
+            p
+            for p in self.plugins
+            if isinstance(p, CollectTaskMetaDataPlugin) and p.name == name
+        ]
+        if len(plugins) != 1:
+            raise ValueError(
+                "Expected to find exactly one CollectTaskMetaDataPlugin "
+                f"with name {name} but found {len(plugins)}."
+            )
+
+        plugin = plugins[0]
+        self.remove_plugin(plugin)
+        return {"metadata": plugin.metadata, "state": plugin.state}
+
     async def register_worker_plugin(self, comm, plugin, name=None):
         """ Registers a setup function, and call it on every worker """
         self.worker_plugins.append({"plugin": plugin, "name": name})
@@ -5636,3 +5659,23 @@ def remove_worker(self, worker=None, **kwargs):
 
     def teardown(self):
         self.bcomm.close()
+
+
+class CollectTaskMetaDataPlugin(SchedulerPlugin):
+    def __init__(self, scheduler, name):
+        self.scheduler = scheduler
+        self.name = name
+        self.keys = set()
+        self.metadata = {}
+        self.state = {}
+
+    def update_graph(self, scheduler, dsk=None, keys=None, restrictions=None, **kwargs):
+        self.keys.update(keys)
+
+    def transition(self, key, start, finish, *args, **kwargs):
+        if finish == "memory" or finish == "erred":
+            ts = self.scheduler.tasks.get(key)
+            if ts is not None and ts.key in self.keys:
+                self.metadata[key] = ts.metadata
+                self.state[key] = finish
+                self.keys.discard(key)
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index a38292f23a4..1de8f2acfbd 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -52,11 +52,12 @@
     default_client,
     futures_of,
     temp_default_client,
+    get_task_metadata,
 )
 from distributed.compatibility import WINDOWS
 
 from distributed.metrics import time
-from distributed.scheduler import Scheduler, KilledWorker
+from distributed.scheduler import Scheduler, KilledWorker, CollectTaskMetaDataPlugin
 from distributed.sizeof import sizeof
 from distributed.utils import mp_context, sync, tmp_text, tokey, tmpfile, is_valid_xml
 from distributed.utils_test import (
@@ -82,6 +83,7 @@
     async_wait_for,
     pristine_loop,
     save_sys_modules,
+    TaskStateMetadataPlugin,
 )
 from distributed.utils_test import (  # noqa: F401
     client as c,
@@ -6199,3 +6201,52 @@ async def test_futures_in_subgraphs(c, s, a, b):
     ddf["local_time"] = ddf.enter_time.dt.tz_convert("US/Central")
     ddf["day"] = ddf.enter_time.dt.day_name()
     ddf = await c.submit(dd.categorical.categorize, ddf, columns=["day"], index=False)
+
+
+@gen_cluster(client=True)
+async def test_get_task_metadata(c, s, a, b):
+
+    # Populate task metadata
+    await c.register_worker_plugin(TaskStateMetadataPlugin())
+
+    async with get_task_metadata() as tasks:
+        f = c.submit(slowinc, 1)
+        await f
+
+    metadata = tasks.metadata
+    assert f.key in metadata
+    assert metadata[f.key] == s.tasks.get(f.key).metadata
+
+    state = tasks.state
+    assert f.key in state
+    assert state[f.key] == "memory"
+
+    assert not any(isinstance(p, CollectTaskMetaDataPlugin) for p in s.plugins)
+
+
+@gen_cluster(client=True)
+async def test_get_task_metadata_multiple(c, s, a, b):
+
+    # Populate task metadata
+    await c.register_worker_plugin(TaskStateMetadataPlugin())
+
+    # Ensure that get_task_metadata only collects metadata for
+    # tasks which are submitted and completed within its context
+    async with get_task_metadata() as tasks1:
+        f1 = c.submit(slowinc, 1)
+        await f1
+        async with get_task_metadata() as tasks2:
+            f2 = c.submit(slowinc, 2)
+            await f2
+
+    metadata1 = tasks1.metadata
+    metadata2 = tasks2.metadata
+
+    assert len(metadata1) == 2
+    assert sorted(metadata1.keys()) == sorted([f1.key, f2.key])
+    assert metadata1[f1.key] == s.tasks.get(f1.key).metadata
+    assert metadata1[f2.key] == s.tasks.get(f2.key).metadata
+
+    assert len(metadata2) == 1
+    assert list(metadata2.keys()) == [f2.key]
+    assert metadata2[f2.key] == s.tasks.get(f2.key).metadata
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 55e0e8c387e..6bb3360d310 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -27,7 +27,7 @@
     Reschedule,
     wait,
 )
-from distributed.diagnostics.plugin import PipInstall, WorkerPlugin
+from distributed.diagnostics.plugin import PipInstall
 from distributed.compatibility import WINDOWS
 from distributed.core import rpc, CommClosedError, Status
 from distributed.scheduler import Scheduler
@@ -53,6 +53,7 @@
     s,
     a,
     b,
+    TaskStateMetadataPlugin,
 )
 
 
@@ -1735,24 +1736,11 @@ async def test_bad_local_directory(cleanup):
 
 @pytest.mark.asyncio
 async def test_taskstate_metadata(cleanup):
-    class MyPlugin(WorkerPlugin):
-        """WorkPlugin to populate TaskState.metadata"""
-
-        def setup(self, worker):
-            self.worker = worker
-
-        def transition(self, key, start, finish, **kwargs):
-            ts = self.worker.tasks[key]
-
-            if start == "ready" and finish == "executing":
-                ts.metadata["start_time"] = time()
-            elif start == "executing" and finish == "memory":
-                ts.metadata["stop_time"] = time()
 
     async with await Scheduler() as s:
         async with await Worker(s.address) as w:
             async with Client(s.address, asynchronous=True) as c:
-                await c.register_worker_plugin(MyPlugin())
+                await c.register_worker_plugin(TaskStateMetadataPlugin())
 
                 f = c.submit(inc, 1)
                 await f
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index 85b4f17d60e..ace82d976d0 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -61,6 +61,7 @@
 )
 from .worker import Worker
 from .nanny import Nanny
+from .diagnostics.plugin import WorkerPlugin
 
 try:
     import dask.array  # register config
@@ -1537,3 +1538,18 @@ def null():
 def cleanup():
     with clean():
         yield
+
+
+class TaskStateMetadataPlugin(WorkerPlugin):
+    """WorkPlugin to populate TaskState.metadata"""
+
+    def setup(self, worker):
+        self.worker = worker
+
+    def transition(self, key, start, finish, **kwargs):
+        ts = self.worker.tasks[key]
+
+        if start == "ready" and finish == "executing":
+            ts.metadata["start_time"] = time()
+        elif start == "executing" and finish == "memory":
+            ts.metadata["stop_time"] = time()
diff --git a/docs/source/api.rst b/docs/source/api.rst
index 5a283ed2f5e..e92617a5634 100644
--- a/docs/source/api.rst
+++ b/docs/source/api.rst
@@ -98,6 +98,7 @@ API
    fire_and_forget
    futures_of
    get_task_stream
+   get_task_metadata
 
 
 Asynchronous methods
@@ -193,6 +194,7 @@ Other
 .. autofunction:: distributed.rejoin
 .. autoclass:: distributed.Reschedule
 .. autoclass:: get_task_stream
+.. autoclass:: get_task_metadata
 
 .. autoclass:: Event
    :members:

From 09d9799e401da695981600fe98ab4d4de4ec419e Mon Sep 17 00:00:00 2001
From: "Mads R. B. Kristensen" <madsbk@gmail.com>
Date: Thu, 5 Nov 2020 21:58:06 +0100
Subject: [PATCH 1025/1550] Communicate HighLevelGraphs directly to the
 Scheduler (#4140)

---
 distributed/client.py                         |  91 +-----------
 distributed/protocol/highlevelgraph.py        | 137 ++++++++++++++++++
 distributed/protocol/serialize.py             |  49 -------
 .../protocol/tests/test_highlevelgraph.py     |  84 +++++++++++
 distributed/protocol/tests/test_serialize.py  |  53 +------
 distributed/scheduler.py                      |  58 +++++++-
 distributed/tests/test_client.py              |   1 +
 7 files changed, 280 insertions(+), 193 deletions(-)
 create mode 100644 distributed/protocol/highlevelgraph.py
 create mode 100644 distributed/protocol/tests/test_highlevelgraph.py

diff --git a/distributed/client.py b/distributed/client.py
index b5fda6b87c8..8c36af4c37b 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -46,7 +46,6 @@
     WrappedKey,
     unpack_remotedata,
     pack_data,
-    subs_multiple,
     scatter_to_workers,
     gather_from_workers,
     retry_operation,
@@ -63,19 +62,19 @@
 from .metrics import time
 from .protocol import to_serialize
 from .protocol.pickle import dumps, loads
+from .protocol.highlevelgraph import highlevelgraph_pack
 from .publish import Datasets
 from .pubsub import PubSubClientExtension
 from .security import Security
 from .sizeof import sizeof
 from .threadpoolexecutor import rejoin
-from .worker import dumps_task, get_client, get_worker, secede
+from .worker import get_client, get_worker, secede
 from .diagnostics.plugin import WorkerPlugin
 from .utils import (
     All,
     sync,
     tokey,
     log_errors,
-    str_graph,
     key_split,
     thread_state,
     no_default,
@@ -2516,7 +2515,7 @@ def run_coroutine(self, function, *args, **kwargs):
         """
         Spawn a coroutine on all workers.
 
-        This spaws a coroutine on all currently known workers and then waits
+        This spawns a coroutine on all currently known workers and then waits
         for the coroutine on each worker.  The coroutines' results are returned
         as a dictionary keyed by worker address.
 
@@ -2582,84 +2581,7 @@ def _graph_to_futures(
             if not isinstance(dsk, HighLevelGraph):
                 dsk = HighLevelGraph.from_collections(id(dsk), dsk, dependencies=())
 
-            def substitute_future_aliases(dsk):
-                # Find aliases not in `keyset`
-                values = {
-                    k: v
-                    for k, v in dsk.items()
-                    if isinstance(v, Future) and k not in keyset
-                }
-                # And substitute all matching keys with its Future
-                if values:
-                    dsk = subs_multiple(dsk, values)
-                return dsk
-
-            # Notice, we only have to do the substitution on already materialized layers
-            dsk = dsk.map_basic_layers(substitute_future_aliases)
-
-            # We need to track all futures unpack_remotedata() unpacks
-            unpacked_futures = set()
-
-            # Unpack remote data in `dsk`, which are "WrappedKeys" that are
-            # unknown to `dsk` but known to the scheduler
-            def unpack_remote_data(tasks):
-                tasks, futures = unpack_remotedata(tasks)
-                unpacked_futures.update(futures)
-                return tasks
-
-            # HACK: Need to loop through each layer manually to collect
-            # futures-related dependencies (for now).  This should be removed
-            # once the scheduler doesn't need these dependencies.
-            _dsk = {}
-            future_deps = defaultdict(set)
-            for key, layer in dsk.layers.items():
-                _dsk[key] = layer.map_tasks(unpack_remote_data)
-                # Here is the "manual" loop where the full graph
-                # needs to be materialized.
-                if unpacked_futures:
-                    for k, v in dict(layer).items():
-                        futures = unpack_remotedata(v)[1]
-                        future_deps[k].update({f.key for f in futures})
-            if _dsk:
-                dsk = HighLevelGraph(_dsk, dsk.dependencies, dsk.key_dependencies)
-
-            for future in unpacked_futures:
-                if future.client is not self:
-                    msg = "Inputs contain futures that were created by another client."
-                    raise ValueError(msg)
-                if tokey(future.key) not in self.futures:
-                    raise CancelledError(tokey(future.key))
-
-            # HACK: currently when submitting work to the scheduler, the client need to
-            # send all key dependencies along with the task graph. Since `dsk` doesn't
-            # know about the unpacked futures, we add futures-related dependencies from
-            # `future_deps`. This hack shouldn't be necessary when the scheduler accepts
-            # high level graphs.
-            dsk.keyset()
-            dependencies = dsk.get_all_dependencies()
-            if future_deps:
-                for key in dependencies.keys():
-                    dependencies[key] |= future_deps.get(key, set())
-
-            if priority is None:
-                # Removing all unpacked futures before calling order()
-                unpacked_keys = {future.key for future in unpacked_futures}
-                stripped_dsk = {k: v for k, v in dsk.items() if k not in unpacked_keys}
-                stripped_deps = {
-                    k: v - unpacked_keys
-                    for k, v in dependencies.items()
-                    if k not in unpacked_keys
-                }
-                priority = dask.order.order(stripped_dsk, dependencies=stripped_deps)
-                priority = keymap(tokey, priority)
-
-            # The scheduler expect all keys to be strings
-            dependencies = {
-                tokey(k): [tokey(dep) for dep in deps]
-                for k, deps in dependencies.items()
-                if deps
-            }
-            dsk = str_graph(dsk, extra_values={f.key for f in unpacked_futures})
+            dsk = highlevelgraph_pack(dsk, keyset, self, self.futures)
 
             if isinstance(retries, Number) and retries > 0:
                 retries = {k: retries for k in dsk}
@@ -2668,9 +2590,8 @@ def unpack_remote_data(tasks):
             futures = {key: Future(key, self, inform=False) for key in keyset}
             self._send_to_scheduler(
                 {
-                    "op": "update-graph",
-                    "tasks": valmap(dumps_task, dsk),
-                    "dependencies": dependencies,
+                    "op": "update-graph-hlg",
+                    "hlg": dsk,
                     "keys": list(map(tokey, keys)),
                     "restrictions": restrictions or {},
                     "loose_restrictions": loose_restrictions,
diff --git a/distributed/protocol/highlevelgraph.py b/distributed/protocol/highlevelgraph.py
new file mode 100644
index 00000000000..77de669ebba
--- /dev/null
+++ b/distributed/protocol/highlevelgraph.py
@@ -0,0 +1,137 @@
+import msgpack
+from tlz import valmap
+
+from dask.core import keys_in_tasks
+from dask.highlevelgraph import HighLevelGraph, Layer
+
+from ..utils_comm import unpack_remotedata, subs_multiple
+from ..worker import dumps_task
+
+from ..utils import (
+    str_graph,
+    tokey,
+    CancelledError,
+)
+
+from .utils import (
+    msgpack_opts,
+)
+from .serialize import (
+    import_allowed_module,
+    msgpack_encode_default,
+    msgpack_decode_default,
+)
+
+
+def _materialized_layer_pack(
+    layer: Layer,
+    all_keys,
+    known_key_dependencies,
+    client_keys,
+    allowed_client,
+    allowed_futures,
+):
+    from ..client import Future
+
+    dsk = dict(layer)
+
+    # Find aliases not in `client_keys` and substitute all matching keys
+    # with its Future
+    values = {
+        k: v for k, v in dsk.items() if isinstance(v, Future) and k not in client_keys
+    }
+    if values:
+        dsk = subs_multiple(dsk, values)
+
+    # Unpack remote data re record its dependencies
+    dsk = {k: unpack_remotedata(v, byte_keys=True) for k, v in layer.items()}
+    unpacked_futures = set.union(*[v[1] for v in dsk.values()]) if dsk else set()
+    for future in unpacked_futures:
+        if future.client is not allowed_client:
+            raise ValueError(
+                "Inputs contain futures that were created by another client."
+            )
+        if tokey(future.key) not in allowed_futures:
+            raise CancelledError(tokey(future.key))
+    unpacked_futures_deps = {}
+    for k, v in dsk.items():
+        if len(v[1]):
+            unpacked_futures_deps[k] = {f.key for f in v[1]}
+    dsk = {k: v[0] for k, v in dsk.items()}
+
+    # Calculate dependencies without re-calculating already known dependencies
+    missing_keys = set(dsk.keys()).difference(known_key_dependencies.keys())
+    dependencies = {
+        k: keys_in_tasks(all_keys, [dsk[k]], as_list=False) for k in missing_keys
+    }
+    for k, v in unpacked_futures_deps.items():
+        dependencies[k] = set(dependencies.get(k, ())) | v
+
+    # The scheduler expect all keys to be strings
+    dependencies = {
+        tokey(k): [tokey(dep) for dep in deps] for k, deps in dependencies.items()
+    }
+    dsk = str_graph(dsk, extra_values=all_keys)
+    dsk = valmap(dumps_task, dsk)
+    return {"dsk": dsk, "dependencies": dependencies}
+
+
+def highlevelgraph_pack(
+    hlg: HighLevelGraph, client_keys, allowed_client, allowed_futures
+):
+    layers = []
+
+    # Dump each layer (in topological order)
+    for layer in (hlg.layers[name] for name in hlg._toposort_layers()):
+        if not layer.is_materialized():
+            state = layer.__dask_distributed_pack__()
+            if state is not None:
+                layers.append(
+                    {
+                        "__module__": layer.__module__,
+                        "__name__": type(layer).__name__,
+                        "state": state,
+                    }
+                )
+                continue
+
+        # Falling back to the default serialization, which will materialize the layer
+        layers.append(
+            {
+                "__module__": None,
+                "__name__": None,
+                "state": _materialized_layer_pack(
+                    layer,
+                    hlg.get_all_external_keys(),
+                    hlg.key_dependencies,
+                    client_keys,
+                    allowed_client,
+                    allowed_futures,
+                ),
+            }
+        )
+
+    return msgpack.dumps({"layers": layers}, default=msgpack_encode_default)
+
+
+def _materialized_layer_unpack(state, dsk, dependencies):
+    dsk.update(state["dsk"])
+    for k, v in state["dependencies"].items():
+        dependencies[k] = list(set(dependencies.get(k, ())) | set(v))
+
+
+def highlevelgraph_unpack(dumped_hlg):
+    # Notice, we set `use_list=False`, which makes msgpack convert lists to tuples
+    hlg = msgpack.loads(
+        dumped_hlg, object_hook=msgpack_decode_default, use_list=False, **msgpack_opts
+    )
+    dsk = {}
+    deps = {}
+    for layer in hlg["layers"]:
+        if layer["__module__"] is None:  # Default implementation
+            unpack_func = _materialized_layer_unpack
+        else:
+            mod = import_allowed_module(layer["__module__"])
+            unpack_func = getattr(mod, layer["__name__"]).__dask_distributed_unpack__
+        unpack_func(layer["state"], dsk, deps)
+    return dsk, deps
diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index 4ae416075c7..5e1faeeb029 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -6,8 +6,6 @@
 
 import dask
 from dask.base import normalize_token
-from dask.highlevelgraph import HighLevelGraph, Layer, BasicLayer
-from dask.optimization import SubgraphCallable
 
 from tlz import valmap, get_in
 
@@ -107,28 +105,6 @@ def msgpack_decode_default(obj):
     if "__Set__" in obj:
         return set(obj["as-list"])
 
-    if "__SubgraphCallable__" in obj:
-        mod = import_allowed_module(obj["__module__"])
-        typ = getattr(mod, obj["__name__"])
-        return typ(*obj["args"])
-
-    if "__Layer__" in obj:
-        obj_name = obj["__name__"]
-        if obj_name == "BasicLayer":
-            # The default implemention of Layer returns a BasicLayer, which might
-            # not be defined in `mod` therefore we import it explicitly here
-            typ = BasicLayer
-        else:
-            mod = import_allowed_module(obj["__module__"])
-            typ = getattr(mod, obj["__name__"])
-        return typ(*obj["args"])
-
-    if "__HighLevelGraph__" in obj:
-        return HighLevelGraph(
-            obj["layers"],
-            obj["dependencies"],
-        )
-
     if "__Serialized__" in obj:
         # Notice, the data here is marked a Serialized rather than deserialized. This
         # is because deserialization requires Pickle which the Scheduler cannot run
@@ -159,31 +135,6 @@ def msgpack_encode_default(obj):
     if isinstance(obj, set):
         return {"__Set__": True, "as-list": list(obj)}
 
-    if isinstance(obj, SubgraphCallable):
-        cls, args = obj.__reduce__()
-        return {
-            "__SubgraphCallable__": True,
-            "__module__": obj.__module__,
-            "__name__": cls.__name__,
-            "args": args,
-        }
-
-    if isinstance(obj, Layer):
-        cls, args = obj.__reduce__()
-        return {
-            "__Layer__": True,
-            "__module__": obj.__module__,
-            "__name__": cls.__name__,
-            "args": args,
-        }
-
-    if isinstance(obj, HighLevelGraph):
-        return {
-            "__HighLevelGraph__": True,
-            "layers": obj.layers,
-            "dependencies": obj.dependencies,
-        }
-
     return obj
 
 
diff --git a/distributed/protocol/tests/test_highlevelgraph.py b/distributed/protocol/tests/test_highlevelgraph.py
new file mode 100644
index 00000000000..25fdef25ba0
--- /dev/null
+++ b/distributed/protocol/tests/test_highlevelgraph.py
@@ -0,0 +1,84 @@
+import pytest
+
+import dask.array as da
+import dask.dataframe as dd
+
+from distributed.utils_test import gen_cluster
+
+np = pytest.importorskip("numpy")
+pd = pytest.importorskip("pandas")
+
+
+@gen_cluster(client=True)
+async def test_combo_of_layer_types(c, s, a, b):
+    """Check pack/unpack of a HLG that has everything!"""
+
+    def add(x, y, z, extra_arg):
+        return x + y + z + extra_arg
+
+    y = c.submit(lambda x: x, 2)
+    z = c.submit(lambda x: x, 3)
+    x = da.blockwise(
+        add,
+        "x",
+        da.zeros((3,), chunks=(1,)),
+        "x",
+        da.ones((3,), chunks=(1,)),
+        "x",
+        y,
+        None,
+        concatenate=False,
+        dtype=int,
+        extra_arg=z,
+    )
+
+    df = dd.from_pandas(pd.DataFrame({"a": np.arange(3)}), npartitions=3)
+    df = df.shuffle("a", shuffle="tasks")
+    df = df["a"].to_dask_array()
+
+    res = x.sum() + df.sum()
+    res = await c.compute(res, optimize_graph=False)
+    assert res == 21
+
+
+@gen_cluster(client=True)
+async def test_blockwise(c, s, a, b):
+    """Check pack/unpack of blockwise layer"""
+
+    def add(x, y, z, extra_arg):
+        return x + y + z + extra_arg
+
+    y = c.submit(lambda x: x, 10)
+    z = c.submit(lambda x: x, 3)
+    x = da.blockwise(
+        add,
+        "x",
+        da.zeros((3,), chunks=(1,)),
+        "x",
+        da.ones((3,), chunks=(1,)),
+        "x",
+        y,
+        None,
+        concatenate=False,
+        dtype=int,
+        extra_arg=z,
+    )
+    res = await c.compute(x.sum(), optimize_graph=False)
+    assert res == 42
+
+
+@gen_cluster(client=True)
+async def test_shuffle(c, s, a, b):
+    """Check pack/unpack of a shuffled dataframe"""
+
+    df = dd.from_pandas(
+        pd.DataFrame(
+            {"a": np.arange(10, dtype=int), "b": np.arange(10, 0, -1, dtype=float)}
+        ),
+        npartitions=5,
+    )
+    df = df.shuffle("a", shuffle="tasks", max_branch=2)
+    df = df["a"] + df["b"]
+    res = await c.compute(df, optimize_graph=False)
+    assert res.dtypes == np.float64
+    assert (res == 10.0).all()
diff --git a/distributed/protocol/tests/test_serialize.py b/distributed/protocol/tests/test_serialize.py
index 4be344488c5..735dffb5c19 100644
--- a/distributed/protocol/tests/test_serialize.py
+++ b/distributed/protocol/tests/test_serialize.py
@@ -5,11 +5,8 @@
 import msgpack
 import numpy as np
 import pytest
-from tlz import identity, valmap
+from tlz import identity
 
-from dask.dataframe.io.parquet.core import ParquetSubgraph
-from dask.highlevelgraph import HighLevelGraph, BasicLayer
-from dask.blockwise import Blockwise
 from dask.utils_test import inc
 
 from distributed import wait
@@ -33,7 +30,6 @@
 from distributed.utils import nbytes
 from distributed.utils_test import inc, gen_test
 from distributed.comm.utils import to_frames, from_frames
-from distributed.worker import dumps_task
 
 
 class MyObj:
@@ -480,50 +476,3 @@ def test_ser_memoryview_object():
     data_in = memoryview(np.array(["hello"], dtype=object))
     with pytest.raises(TypeError):
         serialize(data_in, on_error="raise")
-
-
-def test_highlevelgraphs():
-    """Check dumps/loads of a HLG"""
-
-    # Create a HLG with different types of Layers
-    layers = {
-        "basic": BasicLayer({"a key": 42}),
-        "blockwise": Blockwise(
-            output="z",
-            output_indices=("i",),
-            dsk=valmap(dumps_task, {"z": (inc, ["x"])}),
-            indices=(("x", ("i",)),),
-            numblocks={"x": (3,)},
-            concatenate=False,
-            new_axes=None,
-        ),
-        "parquet": ParquetSubgraph(
-            "",
-            None,
-            "",
-            None,
-            [],
-            [],
-            [],
-            {},
-        ),
-        "Serialize": BasicLayer({"Serialize key": Serialize(42)}),
-    }
-    dependencies = {"basic": set(), "blockwise": {"basic"}}
-    hlg = HighLevelGraph(layers, dependencies)
-
-    # Dump and load the HLG
-    header, frames = dumps(hlg)
-    res = loads([header, frames])
-
-    # Check the loaded result
-    assert isinstance(res, HighLevelGraph)
-    assert hlg.dependencies == res.dependencies
-    assert hlg.layers.keys() == res.layers.keys()
-    for l1, l2 in zip(hlg.layers.values(), res.layers.values()):
-        if "Serialize key" in l1:
-            # `Serialize` wrapped values are not deserialized by `loads()`
-            ser = l2["Serialize key"]
-            assert deserialize(ser.header, ser.frames) == 42
-        else:
-            assert dict(l1) == dict(l2)
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 1b5d7f46b4b..6c74cf833e8 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -84,6 +84,7 @@
 from .pubsub import PubSubSchedulerExtension
 from .stealing import WorkStealing
 from .variable import VariableExtension
+from .protocol.highlevelgraph import highlevelgraph_unpack
 
 if sys.version_info < (3, 8):
     try:
@@ -1313,6 +1314,7 @@ def __init__(
 
         client_handlers = {
             "update-graph": self.update_graph,
+            "update-graph-hlg": self.update_graph_hlg,
             "client-desires-keys": self.client_desires_keys,
             "update-data": self.update_data,
             "report-key": self.report_on_key,
@@ -1820,6 +1822,55 @@ async def add_worker(
                 await comm.write(msg)
             await self.handle_worker(comm=comm, worker=address)
 
+    def update_graph_hlg(
+        self,
+        client=None,
+        hlg=None,
+        keys=None,
+        dependencies=None,
+        restrictions=None,
+        priority=None,
+        loose_restrictions=None,
+        resources=None,
+        submitting_task=None,
+        retries=None,
+        user_priority=0,
+        actors=None,
+        fifo_timeout=0,
+    ):
+
+        dsk, dependencies = highlevelgraph_unpack(hlg)
+
+        # Remove any self-dependencies (happens on test_publish_bag() and others)
+        for k, v in dependencies.items():
+            deps = set(v)
+            if k in deps:
+                deps.remove(k)
+            dependencies[k] = deps
+
+        if priority is None:
+            # Removing all non-local keys before calling order()
+            stripped_deps = {
+                k: v.intersection(dsk) for k, v in dependencies.items() if k in dsk
+            }
+            priority = dask.order.order(dsk, dependencies=stripped_deps)
+
+        return self.update_graph(
+            client,
+            dsk,
+            keys,
+            dependencies,
+            restrictions,
+            priority,
+            loose_restrictions,
+            resources,
+            submitting_task,
+            retries,
+            user_priority,
+            actors,
+            fifo_timeout,
+        )
+
     def update_graph(
         self,
         client=None,
@@ -1871,13 +1922,6 @@ def update_graph(
                     self.report({"op": "cancelled-key", "key": k}, client=client)
                     self.client_releases_keys(keys=[k], client=client)
 
-        # Remove any self-dependencies (happens on test_publish_bag() and others)
-        for k, v in dependencies.items():
-            deps = set(v)
-            if k in deps:
-                deps.remove(k)
-            dependencies[k] = deps
-
         # Avoid computation that is already finished
         already_in_memory = set()  # tasks that are already done
         for k, v in dependencies.items():
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 1de8f2acfbd..73dcb72b3d0 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -5573,6 +5573,7 @@ def test_client_repr_closed_sync(loop):
         c._repr_html_()
 
 
+@pytest.mark.xfail(reason="https://github.com/dask/dask/pull/6807")
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)])
 async def test_nested_prioritization(c, s, w):
     x = delayed(inc)(1, dask_key_name=("a", 2))

From 49044c3c3092d0bc3c98663a57c9f95b3ef38fd8 Mon Sep 17 00:00:00 2001
From: "Mads R. B. Kristensen" <madsbk@gmail.com>
Date: Mon, 9 Nov 2020 15:36:25 +0100
Subject: [PATCH 1026/1550] Use intersection() on a set instead of dict_keys in
 update_graph (#4227)

This is significantly faster
---
 distributed/scheduler.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 6c74cf833e8..1890f7c1c13 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1850,8 +1850,11 @@ def update_graph_hlg(
 
         if priority is None:
             # Removing all non-local keys before calling order()
+            dsk_keys = set(dsk)  # intersection() of sets is much faster than dict_keys
             stripped_deps = {
-                k: v.intersection(dsk) for k, v in dependencies.items() if k in dsk
+                k: v.intersection(dsk_keys)
+                for k, v in dependencies.items()
+                if k in dsk_keys
             }
             priority = dask.order.order(dsk, dependencies=stripped_deps)
 

From aa8e2dd87f58a29e3f4efb65e0ffcd83239ccc68 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Mon, 9 Nov 2020 18:55:16 -0600
Subject: [PATCH 1027/1550] Special case profile thread in leaked thread check
 (#4229)

---
 distributed/utils_test.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index ace82d976d0..9de0e342e7e 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -1421,6 +1421,9 @@ def check_thread_leak():
             and "Threaded" not in v.name
             and "watch message" not in v.name
             and "TCP-Executor" not in v.name
+            # TODO: Make sure profile thread is cleaned up
+            # and remove the line below
+            and "Profile" not in v.name
         ]
         if not bad:
             break

From 7d769b8d87197211f6bd7c91335f9712fc4da949 Mon Sep 17 00:00:00 2001
From: Martin Durant <martindurant@users.noreply.github.com>
Date: Tue, 10 Nov 2020 13:07:37 -0500
Subject: [PATCH 1028/1550] Allow actors to call actors on the same worker
 (#4225)

---
 distributed/actor.py            | 32 +++++++++++++++++++++++++++++---
 distributed/tests/test_actor.py | 33 ++++++++++++++++++++++++++++++++-
 distributed/worker.py           |  1 +
 3 files changed, 62 insertions(+), 4 deletions(-)

diff --git a/distributed/actor.py b/distributed/actor.py
index dc49571d1db..54e9000bdaf 100644
--- a/distributed/actor.py
+++ b/distributed/actor.py
@@ -1,11 +1,12 @@
 import asyncio
 import functools
+from inspect import iscoroutinefunction
 import threading
 from queue import Queue
 
 from .client import Future, default_client
 from .protocol import to_serialize
-from .utils import sync
+from .utils import thread_state, sync
 from .utils_comm import WrappedKey
 from .worker import get_worker
 
@@ -118,13 +119,33 @@ def __dir__(self):
         return sorted(o)
 
     def __getattr__(self, key):
-        attr = getattr(self._cls, key)
 
         if self._future and self._future.status not in ("finished", "pending"):
             raise ValueError(
                 "Worker holding Actor was lost.  Status: " + self._future.status
             )
 
+        if (
+            self._worker
+            and self._worker.address == self._address
+            and getattr(thread_state, "actor", False)
+        ):
+            # actor calls actor on same worker
+            actor = self._worker.actors[self.key]
+            attr = getattr(actor, key)
+
+            if iscoroutinefunction(attr):
+                return attr
+
+            elif callable(attr):
+                return lambda *args, **kwargs: ActorFuture(
+                    None, None, result=attr(*args, **kwargs)
+                )
+            else:
+                return attr
+
+        attr = getattr(self._cls, key)
+
         if callable(attr):
 
             @functools.wraps(attr)
@@ -206,9 +227,14 @@ class ActorFuture:
     Actor
     """
 
-    def __init__(self, q, io_loop):
+    def __init__(self, q, io_loop, result=None):
         self.q = q
         self.io_loop = io_loop
+        if result:
+            self._cached_result = result
+
+    def __await__(self):
+        return self.result()
 
     def result(self, timeout=None):
         try:
diff --git a/distributed/tests/test_actor.py b/distributed/tests/test_actor.py
index 89233eaca24..26421d0385c 100644
--- a/distributed/tests/test_actor.py
+++ b/distributed/tests/test_actor.py
@@ -6,7 +6,7 @@
 
 import dask
 from distributed import Actor, ActorFuture, Client, Future, wait, Nanny
-from distributed.utils_test import gen_cluster
+from distributed.utils_test import cluster, gen_cluster
 from distributed.utils_test import client, cluster_fixture, loop  # noqa: F401
 from distributed.metrics import time
 
@@ -21,11 +21,25 @@ def increment(self):
         self.n += 1
         return self.n
 
+    async def ainc(self):
+        self.n += 1
+        return self.n
+
     def add(self, x):
         self.n += x
         return self.n
 
 
+class UsesCounter:
+    # An actor whose method argument is another actor
+
+    def do_inc(self, ac):
+        return ac.increment().result()
+
+    async def ado_inc(self, ac):
+        return await ac.ainc()
+
+
 class List:
     L = []
 
@@ -550,3 +564,20 @@ async def wait(self):
     await waiter.set()
 
     await c.gather(futures)
+
+
+def test_one_thread_deadlock():
+    with cluster(nworkers=2) as (cl, w):
+        client = Client(cl["address"])
+        ac = client.submit(Counter, actor=True).result()
+        ac2 = client.submit(UsesCounter, actor=True, workers=[ac._address]).result()
+
+        assert ac2.do_inc(ac).result() == 1
+
+
+@gen_cluster(client=True)
+async def test_async_deadlock(client, s, a, b):
+    ac = await client.submit(Counter, actor=True)
+    ac2 = await client.submit(UsesCounter, actor=True, workers=[ac._address])
+
+    assert (await ac2.ado_inc(ac)) == 1
diff --git a/distributed/worker.py b/distributed/worker.py
index b9bc9b63375..d247fb42119 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -3416,6 +3416,7 @@ def apply_function_actor(
 
     thread_state.execution_state = execution_state
     thread_state.key = key
+    thread_state.actor = True
 
     result = function(*args, **kwargs)
 

From d5c8af6aad32a846fa19a392d78100cf06c3b566 Mon Sep 17 00:00:00 2001
From: Julia Signell <jsignell@gmail.com>
Date: Wed, 11 Nov 2020 11:14:10 -0500
Subject: [PATCH 1029/1550] Instantiate plugin if needed in
 register_worker_plugin (#4198)

---
 distributed/client.py                               | 10 ++++++++--
 distributed/diagnostics/tests/test_worker_plugin.py | 13 +++++++++++++
 2 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 8c36af4c37b..537b8a6a74c 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -4092,7 +4092,7 @@ async def _register_worker_plugin(self, plugin=None, name=None):
                 raise exc.with_traceback(tb)
         return responses
 
-    def register_worker_plugin(self, plugin=None, name=None):
+    def register_worker_plugin(self, plugin=None, name=None, **kwargs):
         """
         Registers a lifecycle worker plugin for all current and future workers.
 
@@ -4108,7 +4108,7 @@ def register_worker_plugin(self, plugin=None, name=None):
         cloudpickle modules.
 
         If the plugin has a ``name`` attribute, or if the ``name=`` keyword is
-        used then that will control idempotency.  A a plugin with that name has
+        used then that will control idempotency.  If a plugin with that name has
         already registered then any future plugins will not run.
 
         For alternatives to plugins, you may also wish to look into preload
@@ -4121,6 +4121,9 @@ def register_worker_plugin(self, plugin=None, name=None):
         name: str, optional
             A name for the plugin.
             Registering a plugin with the same name will have no effect.
+        **kwargs: optional
+            If you pass a class as the plugin, instead of a class instance, then the
+            class will be instantiated with any extra keyword arguments.
 
         Examples
         --------
@@ -4155,6 +4158,9 @@ def register_worker_plugin(self, plugin=None, name=None):
         --------
         distributed.WorkerPlugin
         """
+        if isinstance(plugin, type):
+            plugin = plugin(**kwargs)
+
         return self.sync(self._register_worker_plugin, plugin=plugin, name=name)
 
 
diff --git a/distributed/diagnostics/tests/test_worker_plugin.py b/distributed/diagnostics/tests/test_worker_plugin.py
index bc34b61e5a9..79d180c360e 100644
--- a/distributed/diagnostics/tests/test_worker_plugin.py
+++ b/distributed/diagnostics/tests/test_worker_plugin.py
@@ -53,6 +53,19 @@ async def test_create_with_client(c, s):
     assert worker._my_plugin_status == "teardown"
 
 
+@gen_cluster(client=True, nthreads=[])
+async def test_create_with_client_and_plugin_from_class(c, s):
+    await c.register_worker_plugin(MyPlugin, data=456)
+
+    worker = await Worker(s.address, loop=s.loop)
+    assert worker._my_plugin_status == "setup"
+    assert worker._my_plugin_data == 456
+
+    # Give the plugin a new name so that it registers
+    await c.register_worker_plugin(MyPlugin, name="new", data=789)
+    assert worker._my_plugin_data == 789
+
+
 @gen_cluster(client=True, worker_kwargs={"plugins": [MyPlugin(5)]})
 async def test_create_on_construction(c, s, a, b):
     assert len(a.plugins) == len(b.plugins) == 1

From 9181f9ef46999c93d141cab59e4536f1df72a521 Mon Sep 17 00:00:00 2001
From: Gil Forsyth <gforsyth@users.noreply.github.com>
Date: Thu, 12 Nov 2020 10:41:57 -0500
Subject: [PATCH 1030/1550] Create dependency TaskState as needed in gather_dep
 (#4241)

Fixes #4231

As raised in #4231 there are a few cases where we try to gather
dependencies that don't have a corresponding TaskState object -- I think
this is probably because those objects have been recently released (or
it may be due to work-stealing, still trying to work that out
definitively).

This prevents the worker from bailing out -- and in either of the cases
above, we'll still need to ask the scheduler for more information, so we
don't need to add this particular TaskState object to the tasks dict.
---
 distributed/worker.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/worker.py b/distributed/worker.py
index d247fb42119..5a9a7ed1cf4 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -1998,7 +1998,7 @@ async def gather_dep(self, worker, dep, deps, total_nbytes, cause=None):
 
                 # dep states may have changed before gather_dep runs
                 # if a dep is no longer in-flight then don't fetch it
-                deps_ts = [self.tasks[key] for key in deps]
+                deps_ts = [self.tasks.get(key, None) or TaskState(key) for key in deps]
                 deps_ts = tuple(ts for ts in deps_ts if ts.state == "flight")
                 deps = [d.key for d in deps_ts]
 

From 0f4c1dabb23016db14966929406138db8ee29cf8 Mon Sep 17 00:00:00 2001
From: Ian Rose <ian.r.rose@gmail.com>
Date: Thu, 12 Nov 2020 20:58:36 -0800
Subject: [PATCH 1031/1550] Upload file worker plugin (#4238)

---
 distributed/client.py             | 30 +++++----------------------
 distributed/diagnostics/plugin.py | 34 +++++++++++++++++++++++++++++++
 distributed/tests/test_client.py  | 15 ++++++++++++++
 distributed/worker.py             |  2 +-
 docs/source/plugins.rst           |  1 +
 5 files changed, 56 insertions(+), 26 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 537b8a6a74c..bbda46acf4a 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -69,7 +69,7 @@
 from .sizeof import sizeof
 from .threadpoolexecutor import rejoin
 from .worker import get_client, get_worker, secede
-from .diagnostics.plugin import WorkerPlugin
+from .diagnostics.plugin import UploadFile, WorkerPlugin
 from .utils import (
     All,
     sync,
@@ -3034,23 +3034,6 @@ def restart(self, **kwargs):
         """
         return self.sync(self._restart, **kwargs)
 
-    async def _upload_file(self, filename, raise_on_error=True):
-        with open(filename, "rb") as f:
-            data = f.read()
-        _, fn = os.path.split(filename)
-        d = await self.scheduler.broadcast(
-            msg={"op": "upload_file", "filename": fn, "data": to_serialize(data)}
-        )
-
-        if any(v["status"] == "error" for v in d.values()):
-            exceptions = [v["exception"] for v in d.values() if v["status"] == "error"]
-            if raise_on_error:
-                raise exceptions[0]
-            else:
-                return exceptions[0]
-
-        assert all(len(data) == v["nbytes"] for v in d.values())
-
     async def _upload_large_file(self, local_filename, remote_filename=None):
         if remote_filename is None:
             remote_filename = os.path.split(local_filename)[1]
@@ -3094,13 +3077,10 @@ def upload_file(self, filename, **kwargs):
         >>> from mylibrary import myfunc  # doctest: +SKIP
         >>> L = client.map(myfunc, seq)  # doctest: +SKIP
         """
-        result = self.sync(
-            self._upload_file, filename, raise_on_error=self.asynchronous, **kwargs
+        return self.register_worker_plugin(
+            UploadFile(filename),
+            name=filename + str(uuid.uuid4()),
         )
-        if isinstance(result, Exception):
-            raise result
-        else:
-            return result
 
     async def _rebalance(self, futures=None, workers=None):
         await _wait(futures)
@@ -4109,7 +4089,7 @@ def register_worker_plugin(self, plugin=None, name=None, **kwargs):
 
         If the plugin has a ``name`` attribute, or if the ``name=`` keyword is
         used then that will control idempotency.  If a plugin with that name has
-        already registered then any future plugins will not run.
+        already been registered then any future plugins will not run.
 
         For alternatives to plugins, you may also wish to look into preload
         scripts.
diff --git a/distributed/diagnostics/plugin.py b/distributed/diagnostics/plugin.py
index b45e9f25684..cdb811d71cc 100644
--- a/distributed/diagnostics/plugin.py
+++ b/distributed/diagnostics/plugin.py
@@ -1,4 +1,5 @@
 import logging
+import os
 import socket
 import subprocess
 import sys
@@ -258,3 +259,36 @@ async def setup(self, worker):
                     worker.loop.add_callback(
                         worker.close_gracefully, restart=True
                     )  # restart
+
+
+# Adapted from https://github.com/dask/distributed/issues/3560#issuecomment-596138522
+class UploadFile(WorkerPlugin):
+    """A WorkerPlugin to upload a local file to workers.
+
+    Parameters
+    ----------
+    filepath: str
+        A path to the file (.py, egg, or zip) to upload
+
+    Examples
+    --------
+    >>> from distributed.diagnostics.plugin import UploadFile
+
+    >>> client.register_worker_plugin(UploadFile("/path/to/file.py"))  # doctest: +SKIP
+    """
+
+    name = "upload_file"
+
+    def __init__(self, filepath):
+        """
+        Initialize the plugin by reading in the data from the given file.
+        """
+        self.filename = os.path.basename(filepath)
+        with open(filepath, "rb") as f:
+            self.data = f.read()
+
+    async def setup(self, worker):
+        response = await worker.upload_file(
+            comm=None, filename=self.filename, data=self.data, load=True
+        )
+        assert len(self.data) == response["nbytes"]
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 73dcb72b3d0..100b5b13354 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -1684,6 +1684,21 @@ def test_upload_file_exception_sync(c):
             c.upload_file(fn)
 
 
+@gen_cluster(client=True, nthreads=[])
+async def test_upload_file_new_worker(c, s):
+    def g():
+        import myfile
+
+        return myfile.x
+
+    with tmp_text("myfile.py", "x = 123") as fn:
+        await c.upload_file(fn)
+        async with Worker(s.address):
+            x = await c.submit(g)
+
+        assert x == 123
+
+
 @pytest.mark.skip
 @gen_cluster()
 async def test_multiple_clients(s, a, b):
diff --git a/distributed/worker.py b/distributed/worker.py
index 5a9a7ed1cf4..f063cfc0f97 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -999,7 +999,7 @@ def func(data):
                 cache_loads.data.clear()
             except Exception as e:
                 logger.exception(e)
-                return {"status": "error", "exception": to_serialize(e)}
+                raise e
 
         return {"status": "OK", "nbytes": len(data)}
 
diff --git a/docs/source/plugins.rst b/docs/source/plugins.rst
index 0ed63e01d21..add010f862b 100644
--- a/docs/source/plugins.rst
+++ b/docs/source/plugins.rst
@@ -90,3 +90,4 @@ Built-In Worker Plugins
 -----------------------
 
 .. autoclass:: distributed.diagnostics.plugin.PipInstall
+.. autoclass:: distributed.diagnostics.plugin.UploadFile

From 62eb3372320c5178bc1a033d68a37f4c2d196d0a Mon Sep 17 00:00:00 2001
From: Timost <Timost@users.noreply.github.com>
Date: Mon, 16 Nov 2020 19:38:39 +0100
Subject: [PATCH 1032/1550] Redirect to dask-worker cli documentation (#4247)

The doc on the `dask-worker` cli was not up-to-date on `distributed`
docs. It's kept up-to-date automatically on the main `dask` docs, so
this commit makes the `distributed` docs redirect to the main `dask`
docs for the `dask-worker` cli section.
---
 docs/source/worker.rst | 26 +++-----------------------
 1 file changed, 3 insertions(+), 23 deletions(-)

diff --git a/docs/source/worker.rst b/docs/source/worker.rst
index adf1db9b5d3..77a361dc686 100644
--- a/docs/source/worker.rst
+++ b/docs/source/worker.rst
@@ -81,29 +81,9 @@ code then this topic probably doesn't apply.
 Command Line tool
 -----------------
 
-Use the ``dask-worker`` command line tool to start an individual worker.  Here
-are the available options::
-
-   $ dask-worker --help
-   Usage: dask-worker [OPTIONS] SCHEDULER
-
-   Options:
-     --worker-port INTEGER  Serving worker port, defaults to randomly assigned
-     --http-port INTEGER    Serving http port, defaults to randomly assigned
-     --nanny-port INTEGER   Serving nanny port, defaults to randomly assigned
-     --port INTEGER         Deprecated, see --nanny-port
-     --host TEXT            Serving host. Defaults to an ip address that can
-                            hopefully be visible from the scheduler network.
-     --nthreads INTEGER     Number of threads per process. Defaults to number of
-                            cores
-     --nprocs INTEGER       Number of worker processes to launch.  Defaults to one.
-     --name TEXT            Alias
-     --memory-limit TEXT    Maximum bytes of memory that this worker should use.
-                            Use 0 for unlimited, or 'auto' for
-                            TOTAL_MEMORY * min(1, nthreads / total_nthreads)
-     --no-nanny
-     --help                 Show this message and exit.
-
+Use the ``dask-worker`` command line tool to start an individual worker. For
+more details on the command line options, please have a look at the
+`command line tools documentation <https://docs.dask.org/en/latest/setup/cli.html#dask-worker>`_.
 
 Internal Scheduling
 -------------------

From d268735ce22082d5ab06e7f7d520fc66bfd0c62f Mon Sep 17 00:00:00 2001
From: "Mads R. B. Kristensen" <madsbk@gmail.com>
Date: Tue, 17 Nov 2020 01:05:28 +0100
Subject: [PATCH 1033/1550] UCX closing ignore error (#4236)

---
 distributed/comm/ucx.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/distributed/comm/ucx.py b/distributed/comm/ucx.py
index c016cc5a9d5..0d91b404ee2 100644
--- a/distributed/comm/ucx.py
+++ b/distributed/comm/ucx.py
@@ -296,7 +296,13 @@ async def read(self, deserializers=("cuda", "dask", "pickle", "error")):
 
     async def close(self):
         if self._ep is not None:
-            await self.ep.send(struct.pack("?Q", True, 0))
+            try:
+                await self.ep.send(struct.pack("?Q", True, 0))
+            except ucp.exceptions.UCXError:
+                # If the other end is in the process of closing,
+                # UCX will sometimes raise a `Input/output` error,
+                # which we can ignore.
+                pass
             self.abort()
             self._ep = None
 

From 98e17ccf339aad6fc771647d4f28308376871b32 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Tue, 17 Nov 2020 14:21:00 -0600
Subject: [PATCH 1034/1550] Update miniconda GitHub action (#4250)

---
 .github/workflows/ci-windows.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci-windows.yaml b/.github/workflows/ci-windows.yaml
index 7cff9083064..3cf6554a709 100644
--- a/.github/workflows/ci-windows.yaml
+++ b/.github/workflows/ci-windows.yaml
@@ -15,7 +15,7 @@ jobs:
         uses: actions/checkout@v2
 
       - name: Setup Conda Environment
-        uses: goanpeca/setup-miniconda@v1
+        uses: conda-incubator/setup-miniconda@v2
         with:
           miniconda-version: "latest"
           python-version: ${{ matrix.python-version }}

From dfbe171905b349fdd933424ae7af2b6217a28133 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 17 Nov 2020 15:15:05 -0800
Subject: [PATCH 1035/1550] Add dask/distributed versions to performance_report
 (#4249)

Co-authored-by: James Bourbeau <jrbourbeau@gmail.com>
---
 distributed/scheduler.py         | 5 +++++
 distributed/tests/test_client.py | 1 +
 2 files changed, 6 insertions(+)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 1890f7c1c13..451d9fca64b 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -5231,6 +5231,7 @@ def profile_to_figure(state):
         bandwidth_types.update()
 
         from bokeh.models import Panel, Tabs, Div
+        import distributed
 
         # HTML
         html = """
@@ -5251,6 +5252,8 @@ def profile_to_figure(state):
           <li> Workers: {nworkers} </li>
           <li> Threads: {threads} </li>
           <li> Memory: {memory} </li>
+          <li> Dask Version: {dask_version} </li>
+          <li> Dask.Distributed Version: {distributed_version} </li>
         </ul>
 
         <h2> Calling Code </h2>
@@ -5266,6 +5269,8 @@ def profile_to_figure(state):
             threads=sum(w.nthreads for w in self.workers.values()),
             memory=format_bytes(sum(w.memory_limit for w in self.workers.values())),
             code=code,
+            dask_version=dask.__version__,
+            distributed_version=distributed.__version__,
         )
         html = Div(text=html)
 
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 100b5b13354..f34b8b4c742 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -6154,6 +6154,7 @@ async def f():
     assert "Dask Performance Report" in data
     assert "x = da.random" in data
     assert "Threads: 4" in data
+    assert dask.__version__ in data
 
 
 @pytest.mark.asyncio

From 04a6b784cff1db9e1da68c44e4cfc7a99f7fefab Mon Sep 17 00:00:00 2001
From: GeethanjaliEswaran <geethanjali.eswaran@gmail.com>
Date: Wed, 18 Nov 2020 08:37:10 +0530
Subject: [PATCH 1036/1550] Configurable timeouts for worker_client and
 get_client. (#4146)

Issue: https://github.com/dask/distributed/issues/4114

Previously, connection timeout for worker_client and get_client was
hard-coded to 3s by default.
With this change, the default timeout value is fetched from the
dask config 'distributed.comm.timeouts.connect'.
---
 distributed/worker.py        | 24 +++++++++++++++++++-----
 distributed/worker_client.py | 17 +++++++++++++----
 2 files changed, 32 insertions(+), 9 deletions(-)

diff --git a/distributed/worker.py b/distributed/worker.py
index f063cfc0f97..e4a08557c0c 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -2953,7 +2953,7 @@ def client(self):
             else:
                 return self._get_client()
 
-    def _get_client(self, timeout=3):
+    def _get_client(self, timeout=None):
         """Get local client attached to this worker
 
         If no such client exists, create one
@@ -2962,6 +2962,12 @@ def _get_client(self, timeout=3):
         --------
         get_client
         """
+
+        if timeout is None:
+            timeout = dask.config.get("distributed.comm.timeouts.connect")
+
+        timeout = parse_timedelta(timeout, "s")
+
         try:
             from .client import default_client
 
@@ -2992,6 +2998,7 @@ def _get_client(self, timeout=3):
             )
             if not asynchronous:
                 assert self._client.status == "running"
+
         return self._client
 
     def get_current_task(self):
@@ -3043,7 +3050,7 @@ def get_worker():
             raise ValueError("No workers found")
 
 
-def get_client(address=None, timeout=3, resolve_address=True):
+def get_client(address=None, timeout=None, resolve_address=True):
     """Get a client while within a task.
 
     This client connects to the same scheduler to which the worker is connected
@@ -3053,8 +3060,9 @@ def get_client(address=None, timeout=3, resolve_address=True):
     address : str, optional
         The address of the scheduler to connect to. Defaults to the scheduler
         the worker is connected to.
-    timeout : int, default 3
-        Timeout (in seconds) for getting the Client
+    timeout : int or str
+        Timeout (in seconds) for getting the Client. Defaults to the
+        ``distributed.comm.timeouts.connect`` configuration value.
     resolve_address : bool, default True
         Whether to resolve `address` to its canonical form.
 
@@ -3065,7 +3073,7 @@ def get_client(address=None, timeout=3, resolve_address=True):
     Examples
     --------
     >>> def f():
-    ...     client = get_client()
+    ...     client = get_client(timeout="10s")
     ...     futures = client.map(lambda x: x + 1, range(10))  # spawn many tasks
     ...     results = client.gather(futures)
     ...     return sum(results)
@@ -3080,6 +3088,12 @@ def get_client(address=None, timeout=3, resolve_address=True):
     worker_client
     secede
     """
+
+    if timeout is None:
+        timeout = dask.config.get("distributed.comm.timeouts.connect")
+
+    timeout = parse_timedelta(timeout, "s")
+
     if address and resolve_address:
         address = comm.resolve_address(address)
     try:
diff --git a/distributed/worker_client.py b/distributed/worker_client.py
index 7f506cbf1a1..1c4e1378905 100644
--- a/distributed/worker_client.py
+++ b/distributed/worker_client.py
@@ -1,12 +1,14 @@
 from contextlib import contextmanager
 import warnings
 
+import dask
 from .threadpoolexecutor import secede, rejoin
 from .worker import thread_state, get_client, get_worker
+from .utils import parse_timedelta
 
 
 @contextmanager
-def worker_client(timeout=3, separate_thread=True):
+def worker_client(timeout=None, separate_thread=True):
     """Get client for this thread
 
     This context manager is intended to be called within functions that we run
@@ -15,8 +17,9 @@ def worker_client(timeout=3, separate_thread=True):
 
     Parameters
     ----------
-    timeout: Number
-        Timeout after which to err
+    timeout: Number or String
+        Timeout after which to error out. Defaults to the
+        ``distributed.comm.timeouts.connect`` configuration value.
     separate_thread: bool, optional
         Whether to run this function outside of the normal thread pool
         defaults to True
@@ -24,7 +27,7 @@ def worker_client(timeout=3, separate_thread=True):
     Examples
     --------
     >>> def func(x):
-    ...     with worker_client() as c:  # connect from worker back to scheduler
+    ...     with worker_client(timeout="10s") as c:  # connect from worker back to scheduler
     ...         a = c.submit(inc, x)     # this task can submit more tasks
     ...         b = c.submit(dec, x)
     ...         result = c.gather([a, b])  # and gather results
@@ -38,6 +41,12 @@ def worker_client(timeout=3, separate_thread=True):
     get_client
     secede
     """
+
+    if timeout is None:
+        timeout = dask.config.get("distributed.comm.timeouts.connect")
+
+    timeout = parse_timedelta(timeout, "s")
+
     worker = get_worker()
     client = get_client(timeout=timeout)
     if separate_thread:

From eda9bcca8baa502481bf929f39340857da1ebf09 Mon Sep 17 00:00:00 2001
From: "Mads R. B. Kristensen" <madsbk@gmail.com>
Date: Wed, 18 Nov 2020 04:29:33 +0100
Subject: [PATCH 1037/1550] __dask_distributed_pack__(): client argument
 (#4248)

---
 distributed/client.py                  |  2 +-
 distributed/protocol/highlevelgraph.py | 16 ++++++----------
 2 files changed, 7 insertions(+), 11 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index bbda46acf4a..5ce872c0641 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -2581,7 +2581,7 @@ def _graph_to_futures(
             if not isinstance(dsk, HighLevelGraph):
                 dsk = HighLevelGraph.from_collections(id(dsk), dsk, dependencies=())
 
-            dsk = highlevelgraph_pack(dsk, keyset, self, self.futures)
+            dsk = highlevelgraph_pack(dsk, self, keyset)
 
             if isinstance(retries, Number) and retries > 0:
                 retries = {k: retries for k in dsk}
diff --git a/distributed/protocol/highlevelgraph.py b/distributed/protocol/highlevelgraph.py
index 77de669ebba..512a7abd4b4 100644
--- a/distributed/protocol/highlevelgraph.py
+++ b/distributed/protocol/highlevelgraph.py
@@ -27,9 +27,8 @@ def _materialized_layer_pack(
     layer: Layer,
     all_keys,
     known_key_dependencies,
+    client,
     client_keys,
-    allowed_client,
-    allowed_futures,
 ):
     from ..client import Future
 
@@ -47,11 +46,11 @@ def _materialized_layer_pack(
     dsk = {k: unpack_remotedata(v, byte_keys=True) for k, v in layer.items()}
     unpacked_futures = set.union(*[v[1] for v in dsk.values()]) if dsk else set()
     for future in unpacked_futures:
-        if future.client is not allowed_client:
+        if future.client is not client:
             raise ValueError(
                 "Inputs contain futures that were created by another client."
             )
-        if tokey(future.key) not in allowed_futures:
+        if tokey(future.key) not in client.futures:
             raise CancelledError(tokey(future.key))
     unpacked_futures_deps = {}
     for k, v in dsk.items():
@@ -76,15 +75,13 @@ def _materialized_layer_pack(
     return {"dsk": dsk, "dependencies": dependencies}
 
 
-def highlevelgraph_pack(
-    hlg: HighLevelGraph, client_keys, allowed_client, allowed_futures
-):
+def highlevelgraph_pack(hlg: HighLevelGraph, client, client_keys):
     layers = []
 
     # Dump each layer (in topological order)
     for layer in (hlg.layers[name] for name in hlg._toposort_layers()):
         if not layer.is_materialized():
-            state = layer.__dask_distributed_pack__()
+            state = layer.__dask_distributed_pack__(client)
             if state is not None:
                 layers.append(
                     {
@@ -104,9 +101,8 @@ def highlevelgraph_pack(
                     layer,
                     hlg.get_all_external_keys(),
                     hlg.key_dependencies,
+                    client,
                     client_keys,
-                    allowed_client,
-                    allowed_futures,
                 ),
             }
         )

From e42a3a8b092051ce79cf5b51f7c94e999c55a4e4 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Wed, 18 Nov 2020 09:46:33 -0600
Subject: [PATCH 1038/1550] Document task priority tie breaking (#4252)

---
 docs/source/priority.rst | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/source/priority.rst b/docs/source/priority.rst
index 9cef3b95e96..3b47dc4e9da 100644
--- a/docs/source/priority.rst
+++ b/docs/source/priority.rst
@@ -57,3 +57,7 @@ Dask uses the following priorities, in order:
     call) Dask orders tasks in such a way as to minimize the memory-footprint
     of the computation.  This is discussed in more depth in the
     `task ordering documentation <https://github.com/dask/dask/blob/master/dask/order.py>`_.
+
+If multiple tasks each have exactly the same priorities outlined above, then
+the order in which tasks arrive at a worker, in a last in first out manner,
+is used to determine the order in which tasks run.
\ No newline at end of file

From 565608f26a5cfede9a9aa6d5158720dcf781d710 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Wed, 18 Nov 2020 22:14:49 -0800
Subject: [PATCH 1039/1550] Drop `test_sklearn` (#4253)

It seems that we no longer run or use this test. So perhaps it is best
if we drop it.
---
 distributed/protocol/tests/test_sklearn.py | 24 ----------------------
 1 file changed, 24 deletions(-)
 delete mode 100644 distributed/protocol/tests/test_sklearn.py

diff --git a/distributed/protocol/tests/test_sklearn.py b/distributed/protocol/tests/test_sklearn.py
deleted file mode 100644
index 2a3835168ee..00000000000
--- a/distributed/protocol/tests/test_sklearn.py
+++ /dev/null
@@ -1,24 +0,0 @@
-import pytest
-
-pytest.importorskip("sklearn")
-
-import sklearn.linear_model
-
-from distributed.protocol import serialize, deserialize
-
-
-@pytest.mark.xfail(
-    reason="We no longer special-case the BaseEstimator "
-    "super class. It's hard to guarantee support for all subclasseses"
-)
-def test_basic():
-    est = sklearn.linear_model.LinearRegression()
-    est.fit([[0, 0], [1, 1], [2, 2]], [0, 1, 2])
-
-    header, frames = serialize(est)
-    assert header["serializer"] == "dask"
-
-    est2 = deserialize(header, frames)
-
-    inp = [[2, 3], [-1, 3]]
-    assert (est.predict(inp) == est2.predict(inp)).all()

From 7e2fb2ff6a8b19487be5d8b9ba086e1560a75ed5 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Wed, 18 Nov 2020 22:15:43 -0800
Subject: [PATCH 1040/1550] Fuse steps in `extract_serialize` (#4254)

* Join `list` & `dict` code in `_extract_serialize`

* Flip order of `dict` and `list` check

This matches the ordering we have elsewhere.

* Call `type(x)` once

* Assign `path + (k,)` in `_extract_serialize`

To avoid having to reconstruct this value in every case, simply assign
`path_k` with the result of `path + (k,)`.

* Rename `typ` to `typ_v` for clarity

* Use `is` and `or` instead of `in`

This is a bit faster than using `in`, which needs to constructing a
`tuple` and then check for a type match. Also matches what the other
conditions do.

* Avoid unneeded copying of message to serialize

* Collect `bytestrings` when recursing too
---
 distributed/protocol/serialize.py | 80 +++++++++++--------------------
 1 file changed, 27 insertions(+), 53 deletions(-)

diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index 5e1faeeb029..4fa85d66a57 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -1,5 +1,6 @@
 from array import array
 from functools import partial
+from itertools import repeat
 import traceback
 import importlib
 from enum import Enum
@@ -7,8 +8,6 @@
 import dask
 from dask.base import normalize_token
 
-from tlz import valmap, get_in
-
 import msgpack
 
 from . import pickle
@@ -433,15 +432,6 @@ def __ne__(self, other):
         return not (self == other)
 
 
-def container_copy(c):
-    typ = type(c)
-    if typ is list:
-        return list(map(container_copy, c))
-    if typ is dict:
-        return valmap(container_copy, c)
-    return c
-
-
 def extract_serialize(x):
     """Pull out Serialize objects from message
 
@@ -455,50 +445,34 @@ def extract_serialize(x):
     >>> extract_serialize(msg)
     ({'op': 'update'}, {('data',): <Serialize: 123>}, set())
     """
+    x2 = type(x)()
     ser = {}
-    _extract_serialize(x, ser)
-    if ser:
-        x = container_copy(x)
-        for path in ser:
-            t = get_in(path[:-1], x)
-            if isinstance(t, dict):
-                del t[path[-1]]
-            else:
-                t[path[-1]] = None
-
     bytestrings = set()
-    for k, v in ser.items():
-        if type(v) in (bytes, bytearray):
-            ser[k] = to_serialize(v)
-            bytestrings.add(k)
-    return x, ser, bytestrings
-
-
-def _extract_serialize(x, ser, path=()):
-    if type(x) is dict:
-        for k, v in x.items():
-            typ = type(v)
-            if typ is list or typ is dict:
-                _extract_serialize(v, ser, path + (k,))
-            elif (
-                typ is Serialize
-                or typ is Serialized
-                or typ in (bytes, bytearray)
-                and len(v) > 2 ** 16
-            ):
-                ser[path + (k,)] = v
-    elif type(x) is list:
-        for k, v in enumerate(x):
-            typ = type(v)
-            if typ is list or typ is dict:
-                _extract_serialize(v, ser, path + (k,))
-            elif (
-                typ is Serialize
-                or typ is Serialized
-                or typ in (bytes, bytearray)
-                and len(v) > 2 ** 16
-            ):
-                ser[path + (k,)] = v
+    _extract_serialize(x, x2, ser, bytestrings)
+    return x2, ser, bytestrings
+
+
+def _extract_serialize(x, x2, ser, bytestrings, path=()):
+    typ_x = type(x)
+    if typ_x is dict:
+        x_items = x.items()
+    elif typ_x is list:
+        x_items = enumerate(x)
+        x2.extend(repeat(None, len(x)))
+
+    for k, v in x_items:
+        path_k = path + (k,)
+        typ_v = type(v)
+        if typ_v is dict or typ_v is list:
+            x2[k] = v2 = typ_v()
+            _extract_serialize(v, v2, ser, bytestrings, path_k)
+        elif typ_v is Serialize or typ_v is Serialized:
+            ser[path_k] = v
+        elif (typ_v is bytes or typ_v is bytearray) and len(v) > 2 ** 16:
+            ser[path_k] = to_serialize(v)
+            bytestrings.add(path_k)
+        else:
+            x2[k] = v
 
 
 def nested_deserialize(x):

From 2ba2731855cddf32a8981c7ae64cf650d559510a Mon Sep 17 00:00:00 2001
From: jochen-ott-by <jochen.ott@blue-yonder.com>
Date: Fri, 20 Nov 2020 04:37:42 +0100
Subject: [PATCH 1041/1550] comm/tcp listener: do not pass comm with failed
 handshake to comm_handler (#4240)

---
 distributed/comm/tcp.py              |  1 +
 distributed/comm/tests/test_comms.py | 25 +++++++++++++++++++++++++
 2 files changed, 26 insertions(+)

diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py
index d92f83fa445..3c8f75fd9fc 100644
--- a/distributed/comm/tcp.py
+++ b/distributed/comm/tcp.py
@@ -459,6 +459,7 @@ async def _handle_stream(self, stream, address):
             await self.on_connection(comm)
         except CommClosedError:
             logger.info("Connection closed before handshake completed")
+            return
 
         await self.comm_handler(comm)
 
diff --git a/distributed/comm/tests/test_comms.py b/distributed/comm/tests/test_comms.py
index 707819ba596..f166b29486e 100644
--- a/distributed/comm/tests/test_comms.py
+++ b/distributed/comm/tests/test_comms.py
@@ -6,6 +6,8 @@
 import warnings
 from functools import partial
 
+import dask
+
 import distributed
 import pkg_resources
 import pytest
@@ -201,6 +203,29 @@ def test_get_local_address_for():
 #
 
 
+@pytest.mark.asyncio
+async def test_tcp_listener_does_not_call_handler_on_handshake_error():
+    handle_comm_called = False
+
+    async def handle_comm(comm):
+        nonlocal handle_comm_called
+        handle_comm_called = True
+
+    with dask.config.set({"distributed.comm.timeouts.connect": 0.01}):
+        listener = await tcp.TCPListener("127.0.0.1", handle_comm)
+        host, port = listener.get_host_port()
+        # connect without handshake:
+        reader, writer = await asyncio.open_connection(host=host, port=port)
+        # wait a bit to let the listener side hit the timeout on the handshake:
+        await asyncio.sleep(0.02)
+
+    assert not handle_comm_called
+
+    writer.close()
+    if hasattr(writer, "wait_closed"):  # always true for python >= 3.7, but not for 3.6
+        await writer.wait_closed()
+
+
 @pytest.mark.asyncio
 async def test_tcp_specific():
     """

From 48648fb8a49779330d09dafbc554798f02ba21af Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Fri, 20 Nov 2020 07:29:23 -0800
Subject: [PATCH 1042/1550] Simplify frame length packing in TCP write (#4257)

---
 distributed/comm/tcp.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py
index 3c8f75fd9fc..4d8289c904c 100644
--- a/distributed/comm/tcp.py
+++ b/distributed/comm/tcp.py
@@ -233,15 +233,15 @@ async def write(self, msg, serializers=None, on_error="message"):
         )
 
         try:
+            nframes = len(frames)
             lengths = [nbytes(frame) for frame in frames]
-            length_bytes = [struct.pack("Q", len(frames))] + [
-                struct.pack("Q", x) for x in lengths
-            ]
+            length_bytes = struct.pack(f"Q{nframes}Q", nframes, *lengths)
             if sum(lengths) < 2 ** 17:  # 128kiB
-                b = b"".join(length_bytes + frames)  # small enough, send in one go
-                stream.write(b)
+                # small enough, send in one go
+                stream.write(b"".join([length_bytes, *frames]))
             else:
-                stream.write(b"".join(length_bytes))  # avoid large memcpy, send in many
+                # avoid large memcpy, send in many
+                stream.write(length_bytes)
 
                 for frame, frame_bytes in zip(frames, lengths):
                     # Can't wait for the write() Future as it may be lost
@@ -398,7 +398,7 @@ def __init__(
         deserialize=True,
         allow_offload=True,
         default_port=0,
-        **connection_args
+        **connection_args,
     ):
         self._check_encryption(address, connection_args)
         self.ip, self.port = parse_host_port(address, default_port)

From 95f0cca39e95b4bb51b78c09abec1aa28a6fd28d Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Fri, 20 Nov 2020 12:56:47 -0800
Subject: [PATCH 1043/1550] Use `.items()` to walk through keys and values
 (#4261)

This code is likely some Python 2 holdover code from before one could
use generator expressions in `dict`s. As we are Python 3 only (and 3.6+
at that), rewrite this to be a generator expression that populates the
full `dict` in one pass.
---
 distributed/utils_comm.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/distributed/utils_comm.py b/distributed/utils_comm.py
index 814ebb7bce8..c1218eadeab 100644
--- a/distributed/utils_comm.py
+++ b/distributed/utils_comm.py
@@ -231,8 +231,7 @@ def unpack_remotedata(o, byte_keys=False, myset=None):
         return typ(outs)
     elif typ is dict:
         if o:
-            values = [unpack_remotedata(v, byte_keys, myset) for v in o.values()]
-            return dict(zip(o.keys(), values))
+            return {k: unpack_remotedata(v, byte_keys, myset) for k, v in o.items()}
         else:
             return o
     elif issubclass(typ, WrappedKey):  # TODO use type is Future

From 96ecbcdca3179b23358ef0dc656b651c9f156035 Mon Sep 17 00:00:00 2001
From: "Mads R. B. Kristensen" <madsbk@gmail.com>
Date: Mon, 23 Nov 2020 21:28:43 +0100
Subject: [PATCH 1044/1550] Use dask.utils.stringify() instead of
 distributed.utils.tokey() (#4255)

---
 distributed/client.py                         | 53 +++++++++---------
 .../dashboard/tests/test_scheduler_bokeh.py   |  5 +-
 distributed/diagnostics/progress.py           |  5 +-
 distributed/protocol/highlevelgraph.py        | 19 ++++---
 distributed/publish.py                        |  8 +--
 distributed/queues.py                         |  6 ++-
 distributed/recreate_exceptions.py            |  5 +-
 distributed/tests/test_client.py              | 11 ++--
 distributed/tests/test_priorities.py          |  6 +--
 distributed/tests/test_resources.py           | 10 ++--
 distributed/tests/test_utils.py               | 33 ------------
 distributed/utils.py                          | 54 -------------------
 distributed/utils_comm.py                     |  8 +--
 distributed/variable.py                       |  5 +-
 14 files changed, 74 insertions(+), 154 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 5ce872c0641..9c6e54f55e6 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -29,7 +29,7 @@
 from dask.core import flatten
 from dask.optimization import SubgraphCallable
 from dask.compatibility import apply
-from dask.utils import ensure_dict, format_bytes, funcname
+from dask.utils import ensure_dict, format_bytes, funcname, stringify
 from dask.highlevelgraph import HighLevelGraph
 
 from tlz import first, groupby, merge, valmap, keymap, partition_all
@@ -73,7 +73,6 @@
 from .utils import (
     All,
     sync,
-    tokey,
     log_errors,
     key_split,
     thread_state,
@@ -170,7 +169,7 @@ class Future(WrappedKey):
     def __init__(self, key, client=None, inform=True, state=None):
         self.key = key
         self._cleared = False
-        tkey = tokey(key)
+        tkey = stringify(key)
         self.client = client or Client.current()
         self.client._inc_ref(tkey)
         self._generation = self.client.generation
@@ -184,7 +183,7 @@ def __init__(self, key, client=None, inform=True, state=None):
             self.client._send_to_scheduler(
                 {
                     "op": "client-desires-keys",
-                    "keys": [tokey(key)],
+                    "keys": [stringify(key)],
                     "client": self.client.id,
                 }
             )
@@ -357,7 +356,7 @@ def release(self, _in_destructor=False):
         if not self._cleared and self.client.generation == self._generation:
             self._cleared = True
             try:
-                self.client.loop.add_callback(self.client._dec_ref, tokey(self.key))
+                self.client.loop.add_callback(self.client._dec_ref, stringify(self.key))
             except TypeError:
                 pass  # Shutting down, add_callback may be None
 
@@ -375,7 +374,7 @@ def __setstate__(self, state):
             {
                 "op": "update-graph",
                 "tasks": {},
-                "keys": [tokey(self.key)],
+                "keys": [stringify(self.key)],
                 "client": c.id,
             }
         )
@@ -1553,7 +1552,7 @@ def submit(
             else:
                 key = funcname(func) + "-" + str(uuid.uuid4())
 
-        skey = tokey(key)
+        skey = stringify(key)
 
         with self._refcount_lock:
             if skey in self.futures:
@@ -1798,11 +1797,11 @@ def map(
         )
         logger.debug("map(%s, ...)", funcname(func))
 
-        return [futures[tokey(k)] for k in keys]
+        return [futures[stringify(k)] for k in keys]
 
     async def _gather(self, futures, errors="raise", direct=None, local_worker=None):
         unpacked, future_set = unpack_remotedata(futures, byte_keys=True)
-        keys = [tokey(future.key) for future in future_set]
+        keys = [stringify(future.key) for future in future_set]
         bad_data = dict()
         data = {}
 
@@ -2009,8 +2008,8 @@ async def _scatter(
         if isinstance(data, dict) and not all(
             isinstance(k, (bytes, str)) for k in data
         ):
-            d = await self._scatter(keymap(tokey, data), workers, broadcast)
-            return {k: d[tokey(k)] for k in data}
+            d = await self._scatter(keymap(stringify, data), workers, broadcast)
+            return {k: d[stringify(k)] for k in data}
 
         if isinstance(data, type(range(0))):
             data = list(data)
@@ -2199,7 +2198,7 @@ def scatter(
         )
 
     async def _cancel(self, futures, force=False):
-        keys = list({tokey(f.key) for f in futures_of(futures)})
+        keys = list({stringify(f.key) for f in futures_of(futures)})
         await self.scheduler.cancel(keys=keys, client=self.id, force=force)
         for k in keys:
             st = self.futures.pop(k, None)
@@ -2223,7 +2222,7 @@ def cancel(self, futures, asynchronous=None, force=False):
         return self.sync(self._cancel, futures, asynchronous=asynchronous, force=force)
 
     async def _retry(self, futures):
-        keys = list({tokey(f.key) for f in futures_of(futures)})
+        keys = list({stringify(f.key) for f in futures_of(futures)})
         response = await self.scheduler.retry(keys=keys, client=self.id)
         for key in response:
             st = self.futures[key]
@@ -2244,7 +2243,7 @@ async def _publish_dataset(self, *args, name=None, override=False, **kwargs):
             coroutines = []
 
             def add_coro(name, data):
-                keys = [tokey(f.key) for f in futures_of(data)]
+                keys = [stringify(f.key) for f in futures_of(data)]
                 coroutines.append(
                     self.scheduler.publish_put(
                         keys=keys,
@@ -2558,7 +2557,7 @@ def _graph_to_futures(
                 resources = self._expand_resources(
                     resources, all_keys=itertools.chain(dsk, keys)
                 )
-                resources = {tokey(k): v for k, v in resources.items()}
+                resources = {stringify(k): v for k, v in resources.items()}
 
             if retries:
                 retries = self._expand_retries(
@@ -2569,11 +2568,11 @@ def _graph_to_futures(
                 actors = list(self._expand_key(actors))
 
             if restrictions:
-                restrictions = keymap(tokey, restrictions)
+                restrictions = keymap(stringify, restrictions)
                 restrictions = valmap(list, restrictions)
 
             if loose_restrictions is not None:
-                loose_restrictions = list(map(tokey, loose_restrictions))
+                loose_restrictions = list(map(stringify, loose_restrictions))
 
             keyset = set(keys)
 
@@ -2592,7 +2591,7 @@ def _graph_to_futures(
                 {
                     "op": "update-graph-hlg",
                     "hlg": dsk,
-                    "keys": list(map(tokey, keys)),
+                    "keys": list(map(stringify, keys)),
                     "restrictions": restrictions or {},
                     "loose_restrictions": loose_restrictions,
                     "priority": priority,
@@ -2696,7 +2695,7 @@ def _optimize_insert_futures(self, dsk, keys):
         with self._refcount_lock:
             changed = False
             for key in list(dsk):
-                if tokey(key) in self.futures:
+                if stringify(key) in self.futures:
                     if not changed:
                         changed = True
                         dsk = ensure_dict(dsk)
@@ -3084,7 +3083,7 @@ def upload_file(self, filename, **kwargs):
 
     async def _rebalance(self, futures=None, workers=None):
         await _wait(futures)
-        keys = list({tokey(f.key) for f in self.futures_of(futures)})
+        keys = list({stringify(f.key) for f in self.futures_of(futures)})
         result = await self.scheduler.rebalance(keys=keys, workers=workers)
         if result["status"] == "missing-data":
             raise ValueError(
@@ -3115,7 +3114,7 @@ def rebalance(self, futures=None, workers=None, **kwargs):
     async def _replicate(self, futures, n=None, workers=None, branching_factor=2):
         futures = self.futures_of(futures)
         await _wait(futures)
-        keys = {tokey(f.key) for f in futures}
+        keys = {stringify(f.key) for f in futures}
         await self.scheduler.replicate(
             keys=list(keys), n=n, workers=workers, branching_factor=branching_factor
         )
@@ -3225,7 +3224,7 @@ def who_has(self, futures=None, **kwargs):
         """
         if futures is not None:
             futures = self.futures_of(futures)
-            keys = list(map(tokey, {f.key for f in futures}))
+            keys = list(map(stringify, {f.key for f in futures}))
         else:
             keys = None
         return self.sync(self.scheduler.who_has, keys=keys, **kwargs)
@@ -3349,7 +3348,7 @@ def call_stack(self, futures=None, keys=None):
         keys = keys or []
         if futures is not None:
             futures = self.futures_of(futures)
-            keys += list(map(tokey, {f.key for f in futures}))
+            keys += list(map(stringify, {f.key for f in futures}))
         return self.sync(self.scheduler.call_stack, keys=keys or None)
 
     def profile(
@@ -3841,9 +3840,9 @@ def _expand_key(cls, k):
         for kk in k:
             if dask.is_dask_collection(kk):
                 for kkk in kk.__dask_keys__():
-                    yield tokey(kkk)
+                    yield stringify(kkk)
             else:
-                yield tokey(kk)
+                yield stringify(kk)
 
     @classmethod
     def _expand_retries(cls, retries, all_keys):
@@ -3864,7 +3863,7 @@ def _expand_retries(cls, retries, all_keys):
             raise TypeError(
                 "`retries` should be an integer or dict, got %r" % (type(retries))
             )
-        return keymap(tokey, result)
+        return keymap(stringify, result)
 
     def _expand_resources(cls, resources, all_keys):
         """
@@ -4606,7 +4605,7 @@ def fire_and_forget(obj):
         future.client._send_to_scheduler(
             {
                 "op": "client-desires-keys",
-                "keys": [tokey(future.key)],
+                "keys": [stringify(future.key)],
                 "client": "fire-and-forget",
             }
         )
diff --git a/distributed/dashboard/tests/test_scheduler_bokeh.py b/distributed/dashboard/tests/test_scheduler_bokeh.py
index 5f192a75b87..22ff8d6ecc1 100644
--- a/distributed/dashboard/tests/test_scheduler_bokeh.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh.py
@@ -14,7 +14,8 @@
 
 import dask
 from dask.core import flatten
-from distributed.utils import tokey, format_dashboard_link
+from dask.utils import stringify
+from distributed.utils import format_dashboard_link
 from distributed.client import wait
 from distributed.metrics import time
 from distributed.utils_test import gen_cluster, inc, dec, slowinc, div, get_cert
@@ -556,7 +557,7 @@ async def test_TaskGraph_complex(c, s, a, b):
     gp.update()
     assert set(gp.layout.index.values()) == set(range(len(gp.layout.index)))
     visible = gp.node_source.data["visible"]
-    keys = list(map(tokey, flatten(y.__dask_keys__())))
+    keys = list(map(stringify, flatten(y.__dask_keys__())))
     assert all(visible[gp.layout.index[key]] == "True" for key in keys)
 
 
diff --git a/distributed/diagnostics/progress.py b/distributed/diagnostics/progress.py
index 4cb1188a010..eaad51a2747 100644
--- a/distributed/diagnostics/progress.py
+++ b/distributed/diagnostics/progress.py
@@ -5,8 +5,9 @@
 
 from tlz import groupby, valmap
 
+from dask.utils import stringify
 from .plugin import SchedulerPlugin
-from ..utils import key_split, key_split_group, log_errors, tokey
+from ..utils import key_split, key_split_group, log_errors
 
 
 logger = logging.getLogger(__name__)
@@ -61,7 +62,7 @@ class Progress(SchedulerPlugin):
 
     def __init__(self, keys, scheduler, minimum=0, dt=0.1, complete=False):
         self.keys = {k.key if hasattr(k, "key") else k for k in keys}
-        self.keys = {tokey(k) for k in self.keys}
+        self.keys = {stringify(k) for k in self.keys}
         self.scheduler = scheduler
         self.complete = complete
         self._minimum = minimum
diff --git a/distributed/protocol/highlevelgraph.py b/distributed/protocol/highlevelgraph.py
index 512a7abd4b4..ca515823520 100644
--- a/distributed/protocol/highlevelgraph.py
+++ b/distributed/protocol/highlevelgraph.py
@@ -3,15 +3,12 @@
 
 from dask.core import keys_in_tasks
 from dask.highlevelgraph import HighLevelGraph, Layer
+from dask.utils import stringify
 
 from ..utils_comm import unpack_remotedata, subs_multiple
 from ..worker import dumps_task
 
-from ..utils import (
-    str_graph,
-    tokey,
-    CancelledError,
-)
+from ..utils import CancelledError
 
 from .utils import (
     msgpack_opts,
@@ -42,7 +39,7 @@ def _materialized_layer_pack(
     if values:
         dsk = subs_multiple(dsk, values)
 
-    # Unpack remote data re record its dependencies
+    # Unpack remote data and record its dependencies
     dsk = {k: unpack_remotedata(v, byte_keys=True) for k, v in layer.items()}
     unpacked_futures = set.union(*[v[1] for v in dsk.values()]) if dsk else set()
     for future in unpacked_futures:
@@ -50,8 +47,8 @@ def _materialized_layer_pack(
             raise ValueError(
                 "Inputs contain futures that were created by another client."
             )
-        if tokey(future.key) not in client.futures:
-            raise CancelledError(tokey(future.key))
+        if stringify(future.key) not in client.futures:
+            raise CancelledError(stringify(future.key))
     unpacked_futures_deps = {}
     for k, v in dsk.items():
         if len(v[1]):
@@ -68,9 +65,11 @@ def _materialized_layer_pack(
 
     # The scheduler expect all keys to be strings
     dependencies = {
-        tokey(k): [tokey(dep) for dep in deps] for k, deps in dependencies.items()
+        stringify(k): [stringify(dep) for dep in deps]
+        for k, deps in dependencies.items()
     }
-    dsk = str_graph(dsk, extra_values=all_keys)
+    all_keys = all_keys.union(dsk)
+    dsk = {stringify(k): stringify(v, exclusive=all_keys) for k, v in dsk.items()}
     dsk = valmap(dumps_task, dsk)
     return {"dsk": dsk, "dependencies": dependencies}
 
diff --git a/distributed/publish.py b/distributed/publish.py
index 9dbab4ff3dd..485b874d5f3 100644
--- a/distributed/publish.py
+++ b/distributed/publish.py
@@ -1,6 +1,8 @@
 from collections.abc import MutableMapping
 
-from .utils import log_errors, tokey
+from dask.utils import stringify
+
+from .utils import log_errors
 
 
 class PublishExtension:
@@ -32,7 +34,7 @@ def put(
         with log_errors():
             if not override and name in self.datasets:
                 raise KeyError("Dataset %s already exists" % name)
-            self.scheduler.client_desires_keys(keys, "published-%s" % tokey(name))
+            self.scheduler.client_desires_keys(keys, "published-%s" % stringify(name))
             self.datasets[name] = {"data": data, "keys": keys}
             return {"status": "OK", "name": name}
 
@@ -40,7 +42,7 @@ def delete(self, comm=None, name=None):
         with log_errors():
             out = self.datasets.pop(name, {"keys": []})
             self.scheduler.client_releases_keys(
-                out["keys"], "published-%s" % tokey(name)
+                out["keys"], "published-%s" % stringify(name)
             )
 
     def list(self, *args):
diff --git a/distributed/queues.py b/distributed/queues.py
index 6646676cce3..15d6c8adca5 100644
--- a/distributed/queues.py
+++ b/distributed/queues.py
@@ -3,8 +3,10 @@
 import logging
 import uuid
 
+from dask.utils import stringify
+
 from .client import Future, Client
-from .utils import tokey, sync, thread_state
+from .utils import sync, thread_state
 from .worker import get_client
 from .utils import parse_timedelta
 
@@ -201,7 +203,7 @@ async def _():
     async def _put(self, value, timeout=None):
         if isinstance(value, Future):
             await self.client.scheduler.queue_put(
-                key=tokey(value.key), timeout=timeout, name=self.name
+                key=stringify(value.key), timeout=timeout, name=self.name
             )
         else:
             await self.client.scheduler.queue_put(
diff --git a/distributed/recreate_exceptions.py b/distributed/recreate_exceptions.py
index 51a30e52e1c..a046b8c846b 100644
--- a/distributed/recreate_exceptions.py
+++ b/distributed/recreate_exceptions.py
@@ -1,6 +1,7 @@
 import logging
+from dask.utils import stringify
 from .client import futures_of, wait
-from .utils import sync, tokey
+from .utils import sync
 from .utils_comm import pack_data
 from .worker import _deserialize
 
@@ -38,7 +39,7 @@ def cause_of_failure(self, *args, keys=(), **kwargs):
         for key in keys:
             if isinstance(key, list):
                 key = tuple(key)  # ensure not a list from msgpack
-            key = tokey(key)
+            key = stringify(key)
             ts = self.scheduler.tasks.get(key)
             if ts is not None and ts.exception_blame is not None:
                 cause = ts.exception_blame
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index f34b8b4c742..33f02b950b2 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -25,6 +25,7 @@
 import dask
 from dask import delayed
 from dask.optimization import SubgraphCallable
+from dask.utils import stringify
 import dask.bag as db
 from distributed import (
     Worker,
@@ -59,7 +60,7 @@
 from distributed.metrics import time
 from distributed.scheduler import Scheduler, KilledWorker, CollectTaskMetaDataPlugin
 from distributed.sizeof import sizeof
-from distributed.utils import mp_context, sync, tmp_text, tokey, tmpfile, is_valid_xml
+from distributed.utils import mp_context, sync, tmp_text, tmpfile, is_valid_xml
 from distributed.utils_test import (
     cluster,
     slowinc,
@@ -3574,7 +3575,7 @@ async def test_persist_optimize_graph(c, s, a, b):
         b4 = method(b3, optimize_graph=False)
         await wait(b4)
 
-        assert set(map(tokey, b3.__dask_keys__())).issubset(s.tasks)
+        assert set(map(stringify, b3.__dask_keys__())).issubset(s.tasks)
 
         b = db.range(i, npartitions=2)
         i += 1
@@ -3584,7 +3585,7 @@ async def test_persist_optimize_graph(c, s, a, b):
         b4 = method(b3, optimize_graph=True)
         await wait(b4)
 
-        assert not any(tokey(k) in s.tasks for k in b2.__dask_keys__())
+        assert not any(stringify(k) in s.tasks for k in b2.__dask_keys__())
 
 
 @gen_cluster(client=True, nthreads=[])
@@ -3965,7 +3966,7 @@ async def test_serialize_future(s, a, b):
             with ctxman():
                 future2 = pickle.loads(pickle.dumps(future))
                 assert future2.client is ci
-                assert tokey(future2.key) in ci.futures
+                assert stringify(future2.key) in ci.futures
                 result2 = await future2
                 assert result == result2
 
@@ -5601,7 +5602,7 @@ async def test_nested_prioritization(c, s, w):
     await wait([fx, fy])
 
     assert (o[x.key] < o[y.key]) == (
-        s.tasks[tokey(fx.key)].priority < s.tasks[tokey(fy.key)].priority
+        s.tasks[stringify(fx.key)].priority < s.tasks[stringify(fy.key)].priority
     )
 
 
diff --git a/distributed/tests/test_priorities.py b/distributed/tests/test_priorities.py
index cd4344da840..b3cd6ace2d9 100644
--- a/distributed/tests/test_priorities.py
+++ b/distributed/tests/test_priorities.py
@@ -5,10 +5,10 @@
 from dask.core import flatten
 import dask
 from dask import delayed, persist
+from dask.utils import stringify
 
 from distributed.utils_test import gen_cluster, inc, slowinc, slowdec
 from distributed import wait, Worker
-from distributed.utils import tokey
 
 
 @gen_cluster(client=True, nthreads=[])
@@ -45,7 +45,7 @@ async def test_compute(c, s):
     async with Worker(s.address, nthreads=1):
         await wait(high)
         assert all(s.processing.values())
-        assert s.tasks[tokey(low.key)].state in ("processing", "waiting")
+        assert s.tasks[stringify(low.key)].state in ("processing", "waiting")
 
 
 @gen_cluster(client=True, nthreads=[])
@@ -61,7 +61,7 @@ async def test_persist(c, s):
         await wait(high)
         assert all(s.processing.values())
         assert all(
-            s.tasks[tokey(k)].state in ("processing", "waiting")
+            s.tasks[stringify(k)].state in ("processing", "waiting")
             for k in flatten(low.__dask_keys__())
         )
 
diff --git a/distributed/tests/test_resources.py b/distributed/tests/test_resources.py
index 4404ab7cb38..ac7c06f07eb 100644
--- a/distributed/tests/test_resources.py
+++ b/distributed/tests/test_resources.py
@@ -2,12 +2,12 @@
 from time import time
 
 from dask import delayed
+from dask.utils import stringify
 import pytest
 
 from distributed import Worker
 from distributed.client import wait
 from distributed.compatibility import WINDOWS
-from distributed.utils import tokey
 from distributed.utils_test import inc, gen_cluster, slowinc, slowadd
 from distributed.utils_test import client, cluster_fixture, loop, s, a, b  # noqa: F401
 
@@ -211,9 +211,9 @@ async def test_resources_str(c, s, a, b):
     yy = y.persist(resources={"MyRes": 1})
     await wait(yy)
 
-    ts_first = s.tasks[tokey(y.__dask_keys__()[0])]
+    ts_first = s.tasks[stringify(y.__dask_keys__()[0])]
     assert ts_first.resource_restrictions == {"MyRes": 1}
-    ts_last = s.tasks[tokey(y.__dask_keys__()[-1])]
+    ts_last = s.tasks[stringify(y.__dask_keys__()[-1])]
     assert ts_last.resource_restrictions == {"MyRes": 1}
 
 
@@ -316,7 +316,7 @@ async def test_persist_collections(c, s, a, b):
 
     await wait([ww, yy])
 
-    assert all(tokey(key) in a.data for key in y.__dask_keys__())
+    assert all(stringify(key) in a.data for key in y.__dask_keys__())
 
 
 @pytest.mark.skip(reason="Should protect resource keys from optimization")
@@ -336,7 +336,7 @@ async def test_dont_optimize_out(c, s, a, b):
 
     await c.compute(w, resources={tuple(y.__dask_keys__()): {"A": 1}})
 
-    for key in map(tokey, y.__dask_keys__()):
+    for key in map(stringify, y.__dask_keys__()):
         assert "executing" in str(a.story(key))
 
 
diff --git a/distributed/tests/test_utils.py b/distributed/tests/test_utils.py
index 2d96a63d048..87ae97336d1 100644
--- a/distributed/tests/test_utils.py
+++ b/distributed/tests/test_utils.py
@@ -24,7 +24,6 @@
     is_kernel,
     is_valid_xml,
     ensure_ip,
-    str_graph,
     truncate_exception,
     get_traceback,
     _maybe_complex,
@@ -49,7 +48,6 @@
 )
 from distributed.utils_test import loop, loop_in_thread  # noqa: F401
 from distributed.utils_test import div, has_ipv6, inc, throws, gen_test, captured_logger
-from dask.optimization import SubgraphCallable
 
 
 def test_All(loop):
@@ -193,37 +191,6 @@ def c(x):
         assert type(tb).__name__ == "traceback"
 
 
-def test_str_graph():
-    dsk = {"x": 1}
-    assert str_graph(dsk) == dsk
-
-    dsk = {("x", 1): (inc, 1)}
-    assert str_graph(dsk) == {str(("x", 1)): (inc, 1)}
-
-    dsk = {("x", 1): (inc, 1), ("x", 2): (inc, ("x", 1))}
-    assert str_graph(dsk) == {
-        str(("x", 1)): (inc, 1),
-        str(("x", 2)): (inc, str(("x", 1))),
-    }
-
-    dsks = [
-        {"x": 1},
-        {("x", 1): (inc, 1), ("x", 2): (inc, ("x", 1))},
-        {("x", 1): (sum, [1, 2, 3]), ("x", 2): (sum, [("x", 1), ("x", 1)])},
-    ]
-    for dsk in dsks:
-        sdsk = str_graph(dsk)
-        keys = list(dsk)
-        skeys = [str(k) for k in keys]
-        assert all(isinstance(k, str) for k in sdsk)
-        assert dask.get(dsk, keys) == dask.get(sdsk, skeys)
-
-    dsk = {("y", 1): (SubgraphCallable({"x": ("y", 1)}, "x", (("y", 1),)), (("z", 1),))}
-    dsk = str_graph(dsk, extra_values=(("z", 1),))
-    assert dsk["('y', 1)"][0].dsk["x"] == "('y', 1)"
-    assert dsk["('y', 1)"][1][0] == "('z', 1)"
-
-
 def test_maybe_complex():
     assert not _maybe_complex(1)
     assert not _maybe_complex("x")
diff --git a/distributed/utils.py b/distributed/utils.py
index 77487f8cec6..3c2daad0215 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -36,7 +36,6 @@
 
 import dask
 from dask import istask
-from dask.optimization import SubgraphCallable
 
 # provide format_bytes here for backwards compatibility
 from dask.utils import (  # noqa
@@ -739,26 +738,6 @@ def truncate_exception(e, n=10000):
         return e
 
 
-def tokey(o):
-    """Convert an object to a string.
-
-    Examples
-    --------
-
-    >>> tokey(b'x')
-    b'x'
-    >>> tokey('x')
-    'x'
-    >>> tokey(1)
-    '1'
-    """
-    typ = type(o)
-    if typ is str or typ is bytes:
-        return o
-    else:
-        return str(o)
-
-
 def validate_key(k):
     """Validate a key as received on a stream."""
     typ = type(k)
@@ -777,39 +756,6 @@ def _maybe_complex(task):
     )
 
 
-def convert(task, dsk, extra_values):
-    typ = type(task)
-    if typ is tuple and task:
-        if type(task[0]) is SubgraphCallable:
-            sc = task[0]
-            return (
-                SubgraphCallable(
-                    convert(sc.dsk, dsk, extra_values),
-                    sc.outkey,
-                    convert(sc.inkeys, dsk, extra_values),
-                    sc.name,
-                ),
-            ) + tuple(convert(x, dsk, extra_values) for x in task[1:])
-        elif callable(task[0]):
-            return (task[0],) + tuple(convert(x, dsk, extra_values) for x in task[1:])
-    if typ is list:
-        return [convert(v, dsk, extra_values) for v in task]
-    if typ is dict:
-        return {k: convert(v, dsk, extra_values) for k, v in task.items()}
-    try:
-        if task in dsk or task in extra_values:
-            return tokey(task)
-    except TypeError:
-        pass
-    if typ is tuple:  # If the tuple itself isn't a key, check its elements
-        return tuple(convert(v, dsk, extra_values) for v in task)
-    return task
-
-
-def str_graph(dsk, extra_values=()):
-    return {tokey(k): convert(v, dsk, extra_values) for k, v in dsk.items()}
-
-
 def seek_delimiter(file, delimiter, blocksize):
     """Seek current file to next byte after a delimiter bytestring
 
diff --git a/distributed/utils_comm.py b/distributed/utils_comm.py
index c1218eadeab..d1e6431d373 100644
--- a/distributed/utils_comm.py
+++ b/distributed/utils_comm.py
@@ -7,11 +7,11 @@
 
 from dask.optimization import SubgraphCallable
 import dask.config
-from dask.utils import parse_timedelta
+from dask.utils import parse_timedelta, stringify
 from tlz import merge, concat, groupby, drop
 
 from .core import rpc
-from .utils import All, tokey
+from .utils import All
 
 logger = logging.getLogger(__name__)
 
@@ -210,7 +210,7 @@ def unpack_remotedata(o, byte_keys=False, myset=None):
             if futures:
                 myset.update(futures)
                 futures = (
-                    tuple(tokey(f.key) for f in futures)
+                    tuple(stringify(f.key) for f in futures)
                     if byte_keys
                     else tuple(f.key for f in futures)
                 )
@@ -237,7 +237,7 @@ def unpack_remotedata(o, byte_keys=False, myset=None):
     elif issubclass(typ, WrappedKey):  # TODO use type is Future
         k = o.key
         if byte_keys:
-            k = tokey(k)
+            k = stringify(k)
         myset.add(o)
         return k
     else:
diff --git a/distributed/variable.py b/distributed/variable.py
index 26a56466588..db8da76e44c 100644
--- a/distributed/variable.py
+++ b/distributed/variable.py
@@ -6,8 +6,9 @@
 
 from tlz import merge
 
+from dask.utils import stringify
 from .client import Future, Client
-from .utils import tokey, log_errors, TimeoutError, parse_timedelta
+from .utils import log_errors, TimeoutError, parse_timedelta
 from .worker import get_client
 
 logger = logging.getLogger(__name__)
@@ -170,7 +171,7 @@ def __init__(self, name=None, client=None, maxsize=0):
     async def _set(self, value):
         if isinstance(value, Future):
             await self.client.scheduler.variable_set(
-                key=tokey(value.key), name=self.name
+                key=stringify(value.key), name=self.name
             )
         else:
             await self.client.scheduler.variable_set(data=value, name=self.name)

From d01304812cde00a44c2ffddc841b8bebf5b9cec2 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Mon, 23 Nov 2020 15:15:39 -0600
Subject: [PATCH 1045/1550] Support string timeouts in sync (#4266)

---
 distributed/tests/test_utils.py | 3 +++
 distributed/utils.py            | 1 +
 2 files changed, 4 insertions(+)

diff --git a/distributed/tests/test_utils.py b/distributed/tests/test_utils.py
index 87ae97336d1..01332f38834 100644
--- a/distributed/tests/test_utils.py
+++ b/distributed/tests/test_utils.py
@@ -109,6 +109,9 @@ def test_sync_timeout(loop_in_thread):
     with pytest.raises(TimeoutError):
         sync(loop_in_thread, asyncio.sleep, 0.5, callback_timeout=0.05)
 
+    with pytest.raises(TimeoutError):
+        sync(loop_in_thread, asyncio.sleep, 0.5, callback_timeout="50ms")
+
 
 def test_sync_closed_loop():
     loop = IOLoop.current()
diff --git a/distributed/utils.py b/distributed/utils.py
index 3c2daad0215..80124a89dcb 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -293,6 +293,7 @@ def sync(loop, func, *args, callback_timeout=None, **kwargs):
     """
     Run coroutine in loop running in separate thread.
     """
+    callback_timeout = parse_timedelta(callback_timeout, "s")
     # Tornado's PollIOLoop doesn't raise when using closed, do it ourselves
     if PollIOLoop and (
         (isinstance(loop, PollIOLoop) and getattr(loop, "_closing", False))

From f1ade2ff0e2bba8bd648e4930738766b47ba1a0b Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Mon, 23 Nov 2020 17:02:02 -0800
Subject: [PATCH 1046/1550] Use `set.update` to include other `comms` (#4268)

Instead of coercing other collections to `set`s before performing a
union with the `set` of `comms`, which can take a bit longer, simply
call `comms.update(...)` on the provided collection. This avoids the
need to build a separate `set` before performing the union and still has
the same effect of including new values in `comms` where relevant.

Included a link to relevant Python docs for this method (for those
unfamiliar). Note that both `frozenset` and `set` are documented
together hence the reference to `frozenset` in the link even though it
applies to `set` also.

https://docs.python.org/3/library/stdtypes.html#frozenset.update
---
 distributed/scheduler.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 451d9fca64b..691e91e0f9a 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -2589,14 +2589,14 @@ def report(self, msg, ts=None, client=None):
             ts = self.tasks.get(msg["key"])
         if ts is None:
             # Notify all clients
-            comms |= set(self.client_comms.values())
+            comms.update(self.client_comms.values())
         else:
             # Notify clients interested in key
-            comms |= {
+            comms.update(
                 self.client_comms[c.client_key]
                 for c in ts.who_wants
                 if c.client_key in self.client_comms
-            }
+            )
         for c in comms:
             try:
                 c.send(msg)

From 4f2130c9a2e2d0af2cae671dc4837f52959f374b Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Mon, 23 Nov 2020 17:02:52 -0800
Subject: [PATCH 1047/1550] Find the set of workers instead of their frequency
 (#4267)

Since we don't use the frequency of workers in any way and this takes a
bit of time to compute, simply determine the set of workers instead.
This gives us the information we use in the end without any loss of
functionality and speeds up a bottleneck here.
---
 distributed/scheduler.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 691e91e0f9a..ab92fab8fb6 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -20,7 +20,6 @@
 import sortedcontainers
 
 from tlz import (
-    frequencies,
     merge,
     pluck,
     merge_sorted,
@@ -5497,14 +5496,14 @@ def decide_worker(ts, all_workers, valid_workers, objective):
     deps = ts.dependencies
     assert all(dts.who_has for dts in deps)
     if ts.actor:
-        candidates = all_workers
+        candidates = set(all_workers)
     else:
-        candidates = frequencies([ws for dts in deps for ws in dts.who_has])
+        candidates = {ws for dts in deps for ws in dts.who_has}
     if valid_workers is True:
         if not candidates:
-            candidates = all_workers
+            candidates = set(all_workers)
     else:
-        candidates = valid_workers & set(candidates)
+        candidates &= valid_workers
         if not candidates:
             candidates = valid_workers
             if not candidates:

From 8dbe8a28d3c0d4343d26c62852fe86d348ec4afe Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Mon, 23 Nov 2020 23:01:18 -0600
Subject: [PATCH 1048/1550] Add aggregated topic logs and log_event method
 (#4230)

This PR adds the ability to log events under a specified topic to a centralized ledger of events that lives on the scheduler. Currently the centralized ledger contains some administrative events which were previously logged using `Scheduler.log_event`. With the changes in this PR, the client and workers now also have a `log_event` method to add their own events to the ledger. Additionally, the client also has a `get_events` method which allows users to retrieve events from the ledger.

Closes https://github.com/dask/distributed/issues/4222
---
 distributed/client.py                         | 29 ++++++++
 distributed/core.py                           | 16 +----
 distributed/dashboard/components/scheduler.py |  8 +--
 distributed/scheduler.py                      | 24 +++++++
 distributed/stealing.py                       | 19 +++--
 distributed/tests/test_client.py              | 29 ++++++++
 distributed/worker.py                         |  9 +++
 docs/source/api.rst                           |  2 +
 docs/source/index.rst                         |  1 +
 docs/source/logging.rst                       | 70 +++++++++++++++++++
 docs/source/scheduling-state.rst              |  2 +
 11 files changed, 179 insertions(+), 30 deletions(-)
 create mode 100644 docs/source/logging.rst

diff --git a/distributed/client.py b/distributed/client.py
index 9c6e54f55e6..156460c022e 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -3558,6 +3558,35 @@ def get_worker_logs(self, n=None, workers=None, nanny=False):
         """
         return self.sync(self.scheduler.worker_logs, n=n, workers=workers, nanny=nanny)
 
+    def log_event(self, topic, msg):
+        """Log an event under a given topic
+
+        Parameters
+        ----------
+        topic: str, list
+            Name of the topic under which to log an event. To log the same
+            event under multiple topics, pass a list of topic names.
+        msg
+            Event message to log. Note this must be msgpack serializable.
+
+        Examples
+        --------
+        >>> from time import time
+        >>> client.log_event("current-time", time())
+        """
+        return self.sync(self.scheduler.log_event, topic=topic, msg=msg)
+
+    def get_events(self, topic: str = None):
+        """Retrieve structured topic logs
+
+        Parameters
+        ----------
+        topic: str, optional
+            Name of topic log to retrieve events for. If no ``topic`` is
+            provided, then logs for all topics will be returned.
+        """
+        return self.sync(self.scheduler.events, topic=topic)
+
     def retire_workers(self, workers=None, close_workers=True, **kwargs):
         """Retire certain workers on the scheduler
 
diff --git a/distributed/core.py b/distributed/core.py
index 5ede25d7a05..91125788900 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -1,5 +1,5 @@
 import asyncio
-from collections import defaultdict, deque
+from collections import defaultdict
 from contextlib import suppress
 from enum import Enum
 from functools import partial
@@ -184,8 +184,6 @@ def __init__(
         self.monitor = SystemMonitor()
         self.counters = None
         self.digests = None
-        self.events = None
-        self.event_counts = None
         self._ongoing_coroutines = weakref.WeakSet()
         self._event_finished = asyncio.Event()
 
@@ -224,8 +222,6 @@ def stop():
         from .counter import Counter
 
         self.counters = defaultdict(partial(Counter, loop=self.io_loop))
-        self.events = defaultdict(lambda: deque(maxlen=10000))
-        self.event_counts = defaultdict(lambda: 0)
 
         self.periodic_callbacks = dict()
 
@@ -369,16 +365,6 @@ def _measure_tick(self):
         if self.digests is not None:
             self.digests["tick-duration"].add(diff)
 
-    def log_event(self, name, msg):
-        msg["time"] = time()
-        if isinstance(name, list):
-            for n in name:
-                self.events[n].append(msg)
-                self.event_counts[n] += 1
-        else:
-            self.events[name].append(msg)
-            self.event_counts[name] += 1
-
     @property
     def address(self):
         """
diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index 8c43d243673..11c47390864 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -1121,9 +1121,9 @@ def convert(self, msgs):
     @without_property_validation
     def update(self):
         with log_errors():
-            log = self.steal.log
+            log = self.scheduler.get_events(topic="stealing")
             n = self.steal.count - self.last
-            log = [log[-i] for i in range(1, n + 1) if isinstance(log[-i], list)]
+            log = [log[-i][1] for i in range(1, n + 1) if isinstance(log[-i][1], list)]
             self.last = self.steal.count
 
             if log:
@@ -1205,8 +1205,8 @@ def update(self):
                 hovers = []
                 ys = []
                 colors = []
-                for msg in log:
-                    times.append(msg["time"] * 1000)
+                for msg_time, msg in log:
+                    times.append(msg_time * 1000)
                     action = msg["action"]
                     actions.append(action)
                     try:
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index ab92fab8fb6..8c74e3ad51a 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1297,6 +1297,8 @@ def __init__(
         self.log = deque(
             maxlen=dask.config.get("distributed.scheduler.transition-log-length")
         )
+        self.events = defaultdict(lambda: deque(maxlen=100000))
+        self.event_counts = defaultdict(int)
         self.worker_plugins = []
 
         worker_handlers = {
@@ -1309,6 +1311,7 @@ def __init__(
             "long-running": self.handle_long_running,
             "reschedule": self.reschedule,
             "keep-alive": lambda *args, **kwargs: None,
+            "log-event": self.log_worker_event,
         }
 
         client_handlers = {
@@ -1345,6 +1348,8 @@ def __init__(
             "get_logs": self.get_logs,
             "logs": self.get_logs,
             "worker_logs": self.get_worker_logs,
+            "log_event": self.log_worker_event,
+            "events": self.get_events,
             "nbytes": self.get_nbytes,
             "versions": self.versions,
             "add_keys": self.add_keys,
@@ -3735,6 +3740,9 @@ async def feed(
                 if teardown:
                     teardown(self, state)
 
+    def log_worker_event(self, worker=None, topic=None, msg=None):
+        self.log_event(topic, msg)
+
     def subscribe_worker_status(self, comm=None):
         WorkerStatusPlugin(self, comm)
         ident = self.identity()
@@ -5319,6 +5327,22 @@ async def get_worker_logs(self, comm=None, n=None, workers=None, nanny=False):
         )
         return results
 
+    def log_event(self, name, msg):
+        event = (time(), msg)
+        if isinstance(name, list):
+            for n in name:
+                self.events[n].append(event)
+                self.event_counts[n] += 1
+        else:
+            self.events[name].append(event)
+            self.event_counts[name] += 1
+
+    def get_events(self, comm=None, topic=None):
+        if topic is not None:
+            return tuple(self.events[topic])
+        else:
+            return valmap(tuple, self.events)
+
     ###########
     # Cleanup #
     ###########
diff --git a/distributed/stealing.py b/distributed/stealing.py
index 3d9593006d8..1fc26de4b55 100644
--- a/distributed/stealing.py
+++ b/distributed/stealing.py
@@ -57,9 +57,8 @@ def __init__(self, scheduler):
 
         self.scheduler.stream_handlers["steal-response"] = self.move_task_confirm
 
-    @property
-    def log(self):
-        return self.scheduler.events["stealing"]
+    def log(self, msg):
+        return self.scheduler.log_event("stealing", msg)
 
     def add_worker(self, scheduler=None, worker=None):
         self.stealable[worker] = [set() for i in range(15)]
@@ -86,7 +85,7 @@ def put_key_in_stealable(self, ts):
         ws = ts.processing_on
         worker = ws.address
         cost_multiplier, level = self.steal_time_ratio(ts)
-        self.log.append(("add-stealable", ts.key, worker, level))
+        self.log(("add-stealable", ts.key, worker, level))
         if cost_multiplier is not None:
             self.stealable_all[level].add(ts)
             self.stealable[worker][level].add(ts)
@@ -98,7 +97,7 @@ def remove_key_from_stealable(self, ts):
             return
 
         worker, level = result
-        self.log.append(("remove-stealable", ts.key, worker, level))
+        self.log(("remove-stealable", ts.key, worker, level))
         try:
             self.stealable[worker][level].remove(ts)
         except KeyError:
@@ -232,9 +231,7 @@ async def move_task_confirm(self, key=None, worker=None, state=None):
 
             # Victim had already started execution, reverse stealing
             if state in ("memory", "executing", "long-running", None):
-                self.log.append(
-                    ("already-computing", key, victim.address, thief.address)
-                )
+                self.log(("already-computing", key, victim.address, thief.address))
                 self.scheduler.check_idle_saturated(thief)
                 self.scheduler.check_idle_saturated(victim)
 
@@ -257,7 +254,7 @@ async def move_task_confirm(self, key=None, worker=None, state=None):
                     self.scheduler.send_task_to_worker(thief.address, key)
                 except CommClosedError:
                     await self.scheduler.remove_worker(thief.address)
-                self.log.append(("confirm", key, victim.address, thief.address))
+                self.log(("confirm", key, victim.address, thief.address))
             else:
                 raise ValueError("Unexpected task state: %s" % state)
         except Exception as e:
@@ -392,7 +389,7 @@ def maybe_move_task(level, ts, sat, idl, duration, cost_multiplier):
                         )
 
             if log:
-                self.log.append(log)
+                self.log(log)
                 self.count += 1
             stop = time()
             if s.digests:
@@ -410,7 +407,7 @@ def restart(self, scheduler):
     def story(self, *keys):
         keys = set(keys)
         out = []
-        for L in self.log:
+        for _, L in self.scheduler.get_event("stealing"):
             if not isinstance(L, list):
                 L = [L]
             for t in L:
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 33f02b950b2..5712c6a9315 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -6268,3 +6268,32 @@ async def test_get_task_metadata_multiple(c, s, a, b):
     assert len(metadata2) == 1
     assert list(metadata2.keys()) == [f2.key]
     assert metadata2[f2.key] == s.tasks.get(f2.key).metadata
+
+
+@gen_cluster(client=True)
+async def test_log_event(c, s, a, b):
+
+    # Log an event from inside a task
+    def foo():
+        get_worker().log_event("topic1", {"foo": "bar"})
+
+    assert not await c.get_events("topic1")
+    await c.submit(foo)
+    events = await c.get_events("topic1")
+    assert len(events) == 1
+    assert events[0][1] == {"foo": "bar"}
+
+    # Log an event while on the scheduler
+    def log_scheduler(dask_scheduler):
+        dask_scheduler.log_event("topic2", {"woo": "hoo"})
+
+    await c.run_on_scheduler(log_scheduler)
+    events = await c.get_events("topic2")
+    assert len(events) == 1
+    assert events[0][1] == {"woo": "hoo"}
+
+    # Log an event from the client process
+    await c.log_event("topic2", ("alice", "bob"))
+    events = await c.get_events("topic2")
+    assert len(events) == 2
+    assert events[1][1] == ("alice", "bob")
diff --git a/distributed/worker.py b/distributed/worker.py
index e4a08557c0c..be929da4e8d 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -758,6 +758,15 @@ def __repr__(self):
     def logs(self):
         return self._deque_handler.deque
 
+    def log_event(self, topic, msg):
+        self.batched_stream.send(
+            {
+                "op": "log-event",
+                "topic": topic,
+                "msg": msg,
+            }
+        )
+
     @property
     def worker_address(self):
         """ For API compatibility with Nanny """
diff --git a/docs/source/api.rst b/docs/source/api.rst
index e92617a5634..43bd953c9ca 100644
--- a/docs/source/api.rst
+++ b/docs/source/api.rst
@@ -16,6 +16,7 @@ API
    Client.gather
    Client.get
    Client.get_dataset
+   Client.get_events
    Client.get_executor
    Client.get_metadata
    Client.get_scheduler_logs
@@ -23,6 +24,7 @@ API
    Client.get_task_stream
    Client.has_what
    Client.list_datasets
+   Client.log_event
    Client.map
    Client.nthreads
    Client.persist
diff --git a/docs/source/index.rst b/docs/source/index.rst
index b0def11f56c..4364a97c954 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -91,6 +91,7 @@ Contents
    efficiency
    limitations
    locality
+   logging
    manage-computation
    memory
    priority
diff --git a/docs/source/logging.rst b/docs/source/logging.rst
new file mode 100644
index 00000000000..a210fe526c2
--- /dev/null
+++ b/docs/source/logging.rst
@@ -0,0 +1,70 @@
+
+Logging
+=======
+
+.. currentmodule:: distributed
+
+There are several ways in which state and other activities are logged throughout
+a Dask cluster.
+
+
+Logs
+----
+
+The scheduler, workers, and client all log various administrative events using Python's standard
+logging module. Both the logging level and logging handlers are customizable.
+See the `Debugging docs <https://docs.dask.org/en/latest/debugging.html#logs>`_ for more information.
+
+
+Task transition logs
+--------------------
+
+The scheduler keeps track of all :ref:`state transitions <scheduler-task-state>` for each task.
+This gives insight into how tasks progressed through their computation and can be particularly
+valuable when debugging.
+To retrieve the transition logs for a given task, pass the task's key to the :meth:`Scheduler.story` method.
+
+.. code-block:: Python
+
+    >>> f = client.submit(inc, 123)
+    >>> f
+    <Future: finished, type: builtins.int, key: inc-aad7bbea25dc61c8e53d929c7ec50bed>
+    >>> s.story(f.key)
+    [('inc-aad7bbea25dc61c8e53d929c7ec50bed', 'released', 'waiting', {'inc-aad7bbea25dc61c8e53d929c7ec50bed': 'processing'}, 1605143345.7283862),
+     ('inc-aad7bbea25dc61c8e53d929c7ec50bed', 'waiting', 'processing', {}, 1605143345.7284858),
+     ('inc-aad7bbea25dc61c8e53d929c7ec50bed', 'processing', 'memory', {}, 1605143345.731495)]
+
+
+Structured logs
+---------------
+
+The scheduler, workers, and client all support logging structured events to a centralized ledger,
+which is indexed by topic. By default, Dask will log a few administrative events to this system
+(e.g. when workers enter and leave the cluster) but custom events can be logged using the
+:meth:`Scheduler.log_event`, :meth:`Worker.log_event`, or :meth:`Client.log_event` methods.
+
+For example, below we log start and stop times to the ``"runtimes"`` topic using the worker's
+``log_event`` method:
+
+.. code-block:: python
+
+    >>> def myfunc(x):
+    ...     start = time()
+    ...     ...
+    ...     stop = time()
+    ...     dask.distributed.get_worker().log_event("runtimes", {"start": start, "stop": stop})
+    >>> futures = client.map(myfunc, range(10))
+    >>> client.get_events("runtimes")
+    ((1605207481.77175, {'start': 1605207481.769397, 'stop': 1605207481.769397}),
+     (1605207481.772021, {'start': 1605207481.770036, 'stop': 1605207481.770037}),
+     ...
+    )
+
+Events for a given topic can be retrieved using the :meth:`Client.get_events` method.
+In the above example, we retrieved the logged start and stop times with
+``client.get_events("runtimes")``. Note that ``Client.get_events`` returns a tuple for
+each logged event which contains the logged message along with a timestamp for when
+the event was logged.
+
+When combined with scheduler and worker plugins, the structured events system can produce
+rich logging / diagnostic systems.
\ No newline at end of file
diff --git a/docs/source/scheduling-state.rst b/docs/source/scheduling-state.rst
index 4bffd182439..74a44573d59 100644
--- a/docs/source/scheduling-state.rst
+++ b/docs/source/scheduling-state.rst
@@ -46,6 +46,8 @@ The scheduler keeps internal state about several kinds of entities:
    in user code (including on any APIs explained here).
 
 
+.. _scheduler-task-state:
+
 Task State
 ----------
 

From bbb23eb4a68ee0301db35dd18a916cb6b1cbafe4 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Tue, 24 Nov 2020 09:12:04 -0800
Subject: [PATCH 1049/1550] Use `Status` `Enum` in `remove_worker` (#4269)

Looks like this can use the `Status` `Enum`, which we already use
elsewhere. So make use of it here too.
---
 distributed/scheduler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 8c74e3ad51a..4d9b3b2bcb3 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -2303,7 +2303,7 @@ async def remove_worker(self, comm=None, address=None, safe=False, close=True):
             self.idle.discard(ws)
             self.saturated.discard(ws)
             del self.workers[address]
-            ws.status = "closed"
+            ws.status = Status.closed
             self.total_occupancy -= ws.occupancy
 
             recommendations = {}

From f3488988c6ad31a888b66f6ce7de69afd66ee1ce Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Tue, 24 Nov 2020 15:14:26 -0800
Subject: [PATCH 1050/1550] Precompute `hash` for `WorkerState` (#4271)

Despite `self.address` being a simple `str`. It appears calling
`hash(...)` on it results in a bit of overhead. As we know that
`self.address` should not be changing, we can precompute this once in
`__init__` and simply `return` that result in `__hash__`. In all a
simple change that should cutdown on the time spent selecting a `Worker`
in `decide_worker`.
---
 distributed/scheduler.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 4d9b3b2bcb3..6848110e43f 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -239,6 +239,7 @@ class WorkerState:
         "bandwidth",
         "extra",
         "has_what",
+        "_hash",
         "last_seen",
         "local_directory",
         "memory_limit",
@@ -281,6 +282,7 @@ def __init__(
         self.versions = versions or {}
         self.nanny = nanny
 
+        self._hash = hash(address)
         self._status = Status.running
         self.nbytes = 0
         self.occupancy = 0
@@ -298,7 +300,7 @@ def __init__(
         self.extra = extra or {}
 
     def __hash__(self):
-        return hash(self.address)
+        return self._hash
 
     def __eq__(self, other):
         return type(self) == type(other) and self.address == other.address

From 724e70bfbf9c778114f88a7a9aba3321c4ef2983 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Tue, 24 Nov 2020 19:52:26 -0800
Subject: [PATCH 1051/1550] Collect `report`'s `client_key`s in a `list`
 (#4275)

* Collect `comms` in `dict`

Instead of collecting a `set` of `comms` where `BatchedSend` objects
will need to be `hash`ed, use a `dict` with keys based on `Client` IDs
and their `BatchedSend` object as values.  This should still ensure that
we only pick up each `Client`'s `BatchedSend` object once. Though this
will hash the `Client` IDs (`str`s) instead of `BatchedSend` objects,
which should be more performant.

* Only check for `c.client_key` once

Previously we looped through all `Client`s interested in a `TaskState`
and collected their `BatchedSend` comms object (if we had it). This
results in checking for the `Client` in our collection of `BatchedSend`
once and then retrieving it. IOW this is two calls to `__getitem__`. We
can cut this down to one call of `__getitem__` by simply trying to grab
the `Client` (whether it exists or not) and simply skipping over it if
we don't have it. The end result is the same, but the number of steps
performed is reduced.

* Add blank line for readability

* Collect `Client` keys in a `set` instead

Per James' suggestion try collecting a `set` of `Client` keys instead.
This simplifies a bit of the logic earlier on. Plus we can fold the
handling of `Client`s whose comms we don't have into our existing
`for`-loop below.

* Use a `list` collect `client_keys`

As `self.client_comms` is a `dict`, we know none of its keys will
repeat.  Similarly as we know `TaskState`'s `who_wants` is a set, none
of its values will repeat. So these can be added to a `list` without
creating duplicates. The only possible duplicate is the `client` key
passed into the function, which we can just check against the
`client_keys` once produced. So just do this instead thus avoiding a
bunch of checks for collisions that won't occur.

* Add blank line for readability

* Assign `client_keys` directly

As `client_keys` is blank to begin with, just assign directly to it
instead of `extend`ing it.

* Only add `client` when it may not be included

In the case where all `Client`s are notified, there is no need to add
the `client` argument given to `report` as it is either in
`self.client_comms`, from which we included all keys, or it is not, in
which case we skip it. Thus we only need to try and added `client` when
only some of the `Client`s may have been notified, which may or may not
include the `client` argument given. Hence this branch is moved into the
`else` case.

* Only add keys that don't duplicate `client`

* Use `list` comprehensions instead of `for`-loops

* Flatten branch structure

* Run `black`

* Flip `client` cases
---
 distributed/scheduler.py | 29 ++++++++++++++---------------
 1 file changed, 14 insertions(+), 15 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 6848110e43f..10bca0a50f7 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -2584,26 +2584,25 @@ def report(self, msg, ts=None, client=None):
         If the message contains a key then we only send the message to those
         comms that care about the key.
         """
-        comms = set()
-        if client is not None:
-            try:
-                comms.add(self.client_comms[client])
-            except KeyError:
-                pass
-
         if ts is None and "key" in msg:
             ts = self.tasks.get(msg["key"])
+
         if ts is None:
             # Notify all clients
-            comms.update(self.client_comms.values())
-        else:
+            client_keys = list(self.client_comms)
+        elif client is None:
             # Notify clients interested in key
-            comms.update(
-                self.client_comms[c.client_key]
-                for c in ts.who_wants
-                if c.client_key in self.client_comms
-            )
-        for c in comms:
+            client_keys = [c.client_key for c in ts.who_wants]
+        else:
+            # Notify clients interested in key (including `client`)
+            client_keys = [c.client_key for c in ts.who_wants if c.client_key != client]
+            client_keys.append(client)
+
+        for k in client_keys:
+            try:
+                c = self.client_comms[k]
+            except KeyError:
+                continue
             try:
                 c.send(msg)
                 # logger.debug("Scheduler sends message to client %s", msg)

From 5b82d1cabd041ffe23b92123bc49a69c2f18b157 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Tue, 24 Nov 2020 20:33:49 -0800
Subject: [PATCH 1052/1550] Add `__hash__` and `__eq__` for `ClientState`
 (#4276)

Since we periodically collect `ClientState` objects in `set`s or as keys
in `dict`, we need to make sure they are hashable, which means
implementing these methods. Of course there is a fallback if we don't,
but it is preferable to have these implemented as we would expect
instead.
---
 distributed/scheduler.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 10bca0a50f7..67276734692 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -137,14 +137,21 @@ class ClientState:
 
     """
 
-    __slots__ = ("client_key", "wants_what", "last_seen", "versions")
+    __slots__ = ("client_key", "_hash", "wants_what", "last_seen", "versions")
 
     def __init__(self, client, versions=None):
         self.client_key = client
+        self._hash = hash(client)
         self.wants_what = set()
         self.last_seen = time()
         self.versions = versions or {}
 
+    def __hash__(self):
+        return self._hash
+
+    def __eq__(self, other):
+        return type(self) == type(other) and self.client_key == other.client_key
+
     def __repr__(self):
         return "<Client %r>" % (self.client_key,)
 

From 9460e3fe1e0bcdb2daf6ebafe5335d536fa4f492 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Tue, 24 Nov 2020 22:19:37 -0800
Subject: [PATCH 1053/1550] Add `__hash__` and `__eq__` for `TaskState` (#4278)

Implemented to make `TaskState` objects hashable.
---
 distributed/scheduler.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 67276734692..3ff499d876f 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -620,6 +620,8 @@ class TaskState:
         "actor",
         # Key name
         "key",
+        # Hash of the key name
+        "_hash",
         # Key prefix (see key_split())
         "prefix",
         # How to run the task (None if pure data)
@@ -664,6 +666,7 @@ class TaskState:
 
     def __init__(self, key, run_spec):
         self.key = key
+        self._hash = hash(key)
         self.run_spec = run_spec
         self._state = None
         self.exception = self.traceback = self.exception_blame = None
@@ -688,6 +691,12 @@ def __init__(self, key, run_spec):
         self.group = None
         self.metadata = {}
 
+    def __hash__(self):
+        return self._hash
+
+    def __eq__(self, other):
+        return type(self) == type(other) and self.key == other.key
+
     @property
     def state(self) -> str:
         return self._state

From 723f4814e81ea84e571a6e08256563451bdafb85 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 25 Nov 2020 08:06:16 -0800
Subject: [PATCH 1054/1550] Move Status to top-level import (#4280)

---
 distributed/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/__init__.py b/distributed/__init__.py
index bca3195b39c..bd21b701022 100644
--- a/distributed/__init__.py
+++ b/distributed/__init__.py
@@ -1,7 +1,7 @@
 from . import config
 from dask.config import config
 from .actor import Actor, ActorFuture
-from .core import connect, rpc
+from .core import connect, rpc, Status
 from .deploy import LocalCluster, Adaptive, SpecCluster, SSHCluster
 from .diagnostics.progressbar import progress
 from .diagnostics.plugin import WorkerPlugin, SchedulerPlugin, PipInstall

From 0c8a9f4c50d387f04329d009cdcd15d01d53c7b0 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Wed, 25 Nov 2020 11:59:09 -0800
Subject: [PATCH 1055/1550] Drop extra type check in `_extract_serialize`
 (#4281)

---
 distributed/protocol/serialize.py | 30 +++++++++++++++++-------------
 1 file changed, 17 insertions(+), 13 deletions(-)

diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index 4fa85d66a57..5395f393384 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -1,6 +1,5 @@
 from array import array
 from functools import partial
-from itertools import repeat
 import traceback
 import importlib
 from enum import Enum
@@ -445,27 +444,32 @@ def extract_serialize(x):
     >>> extract_serialize(msg)
     ({'op': 'update'}, {('data',): <Serialize: 123>}, set())
     """
-    x2 = type(x)()
-    ser = {}
-    bytestrings = set()
-    _extract_serialize(x, x2, ser, bytestrings)
-    return x2, ser, bytestrings
-
-
-def _extract_serialize(x, x2, ser, bytestrings, path=()):
     typ_x = type(x)
     if typ_x is dict:
         x_items = x.items()
+        x2 = {}
     elif typ_x is list:
         x_items = enumerate(x)
-        x2.extend(repeat(None, len(x)))
+        x2 = len(x) * [None]
+
+    ser = {}
+    bytestrings = set()
+    _extract_serialize(x_items, x2, ser, bytestrings)
+    return x2, ser, bytestrings
+
 
+def _extract_serialize(x_items, x2, ser, bytestrings, path=()):
     for k, v in x_items:
         path_k = path + (k,)
         typ_v = type(v)
-        if typ_v is dict or typ_v is list:
-            x2[k] = v2 = typ_v()
-            _extract_serialize(v, v2, ser, bytestrings, path_k)
+        if typ_v is dict:
+            v_items = v.items()
+            x2[k] = v2 = {}
+            _extract_serialize(v_items, v2, ser, bytestrings, path_k)
+        elif typ_v is list:
+            v_items = enumerate(v)
+            x2[k] = v2 = len(v) * [None]
+            _extract_serialize(v_items, v2, ser, bytestrings, path_k)
         elif typ_v is Serialize or typ_v is Serialized:
             ser[path_k] = v
         elif (typ_v is bytes or typ_v is bytearray) and len(v) > 2 ** 16:

From b4dfc925bac32a488be2016a5930a9b7dd95cec5 Mon Sep 17 00:00:00 2001
From: Gil Forsyth <gforsyth@users.noreply.github.com>
Date: Wed, 25 Nov 2020 15:46:50 -0500
Subject: [PATCH 1056/1550] Move `nbytes` from Worker's state to `TaskState`
 (#4274)

Removing one less extra dictionary to track -- this was the source of
some tricky-to-debug issues over in #4264 and I thought I'd break it out
into a separate PR instead of jamming everything into one enormous PR.
---
 distributed/tests/test_worker.py |  7 ++--
 distributed/worker.py            | 57 ++++++++++++++++++--------------
 2 files changed, 37 insertions(+), 27 deletions(-)

diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 6bb3360d310..b41e174bf5f 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -573,7 +573,6 @@ async def test_clean(c, s, a, b):
     collections = [
         a.tasks,
         a.data,
-        a.nbytes,
         a.threads,
     ]
     for c in collections:
@@ -667,7 +666,11 @@ async def test_clean_nbytes(c, s, a, b):
     await wait(future)
 
     await asyncio.sleep(1)
-    assert len(a.nbytes) + len(b.nbytes) == 1
+    assert (
+        len(list(filter(None, [ts.nbytes for ts in a.tasks.values()])))
+        + len(list(filter(None, [ts.nbytes for ts in b.tasks.values()])))
+        == 1
+    )
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 20)
diff --git a/distributed/worker.py b/distributed/worker.py
index be929da4e8d..447f591b1fd 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -88,6 +88,10 @@
 
 DEFAULT_STARTUP_INFORMATION = {}
 
+DEFAULT_DATA_SIZE = parse_bytes(
+    dask.config.get("distributed.scheduler.default-data-size")
+)
+
 SerializedTask = namedtuple("SerializedTask", ["function", "args", "kwargs", "task"])
 
 
@@ -132,6 +136,8 @@ class TaskState:
     * **metadata**: ``dict``
         Metadata related to task. Stored metadata should be msgpack
         serializable (e.g. int, string, list, dict).
+    * **nbytes**: ``int``
+        The size of a particular piece of data
 
     Parameters
     ----------
@@ -165,10 +171,15 @@ def __init__(self, key, runspec=None):
         self.start_time = None
         self.stop_time = None
         self.metadata = {}
+        self.nbytes = None
 
     def __repr__(self):
         return "<Task %r %s>" % (self.key, self.state)
 
+    def get_nbytes(self) -> int:
+        nbytes = self.nbytes
+        return nbytes if nbytes is not None else DEFAULT_DATA_SIZE
+
 
 class Worker(ServerNode):
     """Worker node in a Dask distributed cluster
@@ -267,8 +278,6 @@ class Worker(ServerNode):
         dependencies we expect from those connections
     * **comm_bytes**: ``int``
         The total number of bytes in flight
-    * **nbytes**: ``{key: int}``
-        The size of a particular piece of data
     * **threads**: ``{key: int}``
         The ID of the thread on which the task ran
     * **active_threads**: ``{int: key}``
@@ -403,7 +412,6 @@ def __init__(
         self.comm_nbytes = 0
         self._missing_dep_flight = set()
 
-        self.nbytes = dict()
         self.threads = dict()
 
         self.active_threads_lock = threading.Lock()
@@ -856,7 +864,7 @@ async def _register_with_scheduler(self):
                         keys=list(self.data),
                         nthreads=self.nthreads,
                         name=self.name,
-                        nbytes=self.nbytes,
+                        nbytes={ts.key: ts.get_nbytes() for ts in self.tasks.values()},
                         types={k: typename(v) for k, v in self.data.items()},
                         now=time(),
                         resources=self.total_resources,
@@ -1322,7 +1330,7 @@ async def get_data(
                     data[k] = Actor(type(self.actors[k]), self.address, k)
 
         msg = {"status": "OK", "data": {k: to_serialize(v) for k, v in data.items()}}
-        nbytes = {k: self.nbytes.get(k) for k in data}
+        nbytes = {k: self.tasks[k].nbytes for k in data if k in self.tasks}
         stop = time()
         if self.digests is not None:
             self.digests["get-data-load-duration"].add(stop - start)
@@ -1464,9 +1472,6 @@ def add_task(
             if resource_restrictions:
                 ts.resource_restrictions = resource_restrictions
 
-            if nbytes is not None:
-                self.nbytes.update(nbytes)
-
             who_has = who_has or {}
 
             for dependency, workers in who_has.items():
@@ -1494,6 +1499,10 @@ def add_task(
                     if dep_ts.state != "memory":
                         self.pending_data_per_worker[worker].append(dep_ts.key)
 
+            if nbytes is not None:
+                for key, value in nbytes.items():
+                    self.tasks[key].nbytes = value
+
             if ts.waiting_for_data:
                 self.data_needed.append(ts.key)
             else:
@@ -1501,7 +1510,6 @@ def add_task(
             if self.validate:
                 if who_has:
                     assert all(self.tasks[dep] in ts.dependencies for dep in who_has)
-                    assert all(dep in self.nbytes for dep in who_has)
                     assert all(self.tasks[dep.key] for dep in ts.dependencies)
                     for dependency in ts.dependencies:
                         self.validate_task(dependency)
@@ -1887,14 +1895,13 @@ def ensure_communicating(self):
 
     def send_task_state_to_scheduler(self, ts):
         if ts.key in self.data or self.actors.get(ts.key):
-            nbytes = self.nbytes.get(ts.key)
             typ = ts.type
-            if nbytes is None or typ is None:
+            if ts.nbytes is None or typ is None:
                 try:
                     value = self.data[ts.key]
                 except KeyError:
                     value = self.actors[ts.key]
-                nbytes = self.nbytes[ts.key] = sizeof(value)
+                nbytes = ts.nbytes = sizeof(value)
                 typ = ts.type = type(value)
                 del value
             try:
@@ -1907,7 +1914,7 @@ def send_task_state_to_scheduler(self, ts):
                 "op": "task-finished",
                 "status": "OK",
                 "key": ts.key,
-                "nbytes": nbytes,
+                "nbytes": ts.nbytes,
                 "thread": self.threads.get(ts.key),
                 "type": typ_serialized,
                 "typename": typename(typ),
@@ -1948,8 +1955,8 @@ def put_key_in_memory(self, ts, value, transition=True):
                     {"action": "disk-write", "start": start, "stop": stop}
                 )
 
-        if ts.key not in self.nbytes:
-            self.nbytes[ts.key] = sizeof(value)
+        if ts.nbytes is None:
+            ts.nbytes = sizeof(value)
 
         ts.type = type(value)
 
@@ -1968,7 +1975,7 @@ def select_keys_for_gather(self, worker, dep):
         assert isinstance(dep, str)
         deps = {dep}
 
-        total_bytes = self.nbytes[dep]
+        total_bytes = self.tasks[dep].get_nbytes()
         L = self.pending_data_per_worker[worker]
 
         while L:
@@ -1976,10 +1983,10 @@ def select_keys_for_gather(self, worker, dep):
             ts = self.tasks.get(d)
             if ts is None or ts.state != "waiting":
                 continue
-            if total_bytes + self.nbytes[d] > self.target_message_size:
+            if total_bytes + ts.get_nbytes() > self.target_message_size:
                 break
             deps.add(d)
-            total_bytes += self.nbytes[d]
+            total_bytes += ts.get_nbytes()
 
         return deps, total_bytes
 
@@ -2038,7 +2045,9 @@ async def gather_dep(self, worker, dep, deps, total_nbytes, cause=None):
                         }
                     )
 
-                total_bytes = sum(self.nbytes.get(key, 0) for key in response["data"])
+                total_bytes = sum(
+                    self.tasks[key].nbytes or 0 for key in response["data"]
+                )
                 duration = (stop - start) or 0.010
                 bandwidth = total_bytes / duration
                 self.incoming_transfer_log.append(
@@ -2048,7 +2057,7 @@ async def gather_dep(self, worker, dep, deps, total_nbytes, cause=None):
                         "middle": (start + stop) / 2.0 + self.scheduler_delay,
                         "duration": duration,
                         "keys": {
-                            key: self.nbytes.get(key, None) for key in response["data"]
+                            key: self.tasks[key].nbytes for key in response["data"]
                         },
                         "total": total_bytes,
                         "bandwidth": bandwidth,
@@ -2243,10 +2252,8 @@ def release_key(self, key, cause=None, reason=None, report=True):
                     del self.data[key]
                 except FileNotFoundError:
                     logger.error("Tried to delete %s but no file found", exc_info=True)
-                del self.nbytes[key]
             if key in self.actors and not ts.dependents:
                 del self.actors[key]
-                del self.nbytes[key]
 
             # for any dependencies of key we are releasing remove task as dependent
             for dependency in ts.dependencies:
@@ -2569,7 +2576,7 @@ async def execute(self, key, report=False):
             self.threads[ts.key] = result["thread"]
 
             if result["op"] == "task-finished":
-                self.nbytes[ts.key] = result["nbytes"]
+                ts.nbytes = result["nbytes"]
                 ts.type = result["type"]
                 self.transition(ts, "memory", value=value)
                 if self.digests is not None:
@@ -2851,7 +2858,7 @@ def _notify_plugins(self, method_name, *args, **kwargs):
 
     def validate_task_memory(self, ts):
         assert ts.key in self.data or ts.key in self.actors
-        assert ts.key in self.nbytes
+        assert ts.nbytes is not None
         assert not ts.waiting_for_data
         assert ts.key not in self.ready
         assert ts.state == "memory"
@@ -2931,7 +2938,7 @@ def validate_state(self):
                         or ts_wait.who_has.issubset(self.in_flight_workers)
                     )
                 if ts.state == "memory":
-                    assert isinstance(self.nbytes[ts.key], int)
+                    assert isinstance(ts.nbytes, int)
                     assert not ts.waiting_for_data
                     assert ts.key in self.data or ts.key in self.actors
 

From e9cd97f8fef88268f26d855b1f15c955fe7b37b5 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Mon, 30 Nov 2020 12:05:21 -0800
Subject: [PATCH 1057/1550] Annotate `extract_serialize` (for Cythonization)
 (#4283)

Provides some unintrusive type annotations of variables in
`extract_serialize`. Cython is able to parse these type annotations and
optimize the code. Though as the type annotations are already supported
in Python normally, this remains valid Python code that is otherwise
unaffected.
---
 distributed/protocol/serialize.py | 44 +++++++++++++++++++++----------
 1 file changed, 30 insertions(+), 14 deletions(-)

diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index 5395f393384..1bea7ca496a 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -431,7 +431,7 @@ def __ne__(self, other):
         return not (self == other)
 
 
-def extract_serialize(x):
+def extract_serialize(x) -> tuple:
     """Pull out Serialize objects from message
 
     This also remove large bytestrings from the message into a second
@@ -444,37 +444,53 @@ def extract_serialize(x):
     >>> extract_serialize(msg)
     ({'op': 'update'}, {('data',): <Serialize: 123>}, set())
     """
-    typ_x = type(x)
+    typ_x: type = type(x)
     if typ_x is dict:
-        x_items = x.items()
+        x_d: dict = x
+        x_items = x_d.items()
         x2 = {}
     elif typ_x is list:
-        x_items = enumerate(x)
-        x2 = len(x) * [None]
+        x_l: list = x
+        x_items = enumerate(x_l)
+        x2 = len(x_l) * [None]
 
     ser = {}
     bytestrings = set()
-    _extract_serialize(x_items, x2, ser, bytestrings)
+    path = ()
+    _extract_serialize(x_items, x2, ser, bytestrings, path)
     return x2, ser, bytestrings
 
 
-def _extract_serialize(x_items, x2, ser, bytestrings, path=()):
+def _extract_serialize(x_items, x2, ser: dict, bytestrings: set, path: tuple) -> None:
     for k, v in x_items:
         path_k = path + (k,)
-        typ_v = type(v)
+        typ_v: type = type(v)
         if typ_v is dict:
-            v_items = v.items()
+            v_d: dict = v
+            v_items = v_d.items()
             x2[k] = v2 = {}
             _extract_serialize(v_items, v2, ser, bytestrings, path_k)
         elif typ_v is list:
-            v_items = enumerate(v)
-            x2[k] = v2 = len(v) * [None]
+            v_l: list = v
+            v_items = enumerate(v_l)
+            x2[k] = v2 = len(v_l) * [None]
             _extract_serialize(v_items, v2, ser, bytestrings, path_k)
         elif typ_v is Serialize or typ_v is Serialized:
             ser[path_k] = v
-        elif (typ_v is bytes or typ_v is bytearray) and len(v) > 2 ** 16:
-            ser[path_k] = to_serialize(v)
-            bytestrings.add(path_k)
+        elif typ_v is bytes:
+            v_b: bytes = v
+            if len(v_b) > 2 ** 16:
+                ser[path_k] = to_serialize(v_b)
+                bytestrings.add(path_k)
+            else:
+                x2[k] = v_b
+        elif typ_v is bytearray:
+            v_ba: bytearray = v
+            if len(v_ba) > 2 ** 16:
+                ser[path_k] = to_serialize(v_ba)
+                bytestrings.add(path_k)
+            else:
+                x2[k] = v_ba
         else:
             x2[k] = v
 

From 9ed9ffec9855d8d33482184979c4c78f33fe2f5a Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 30 Nov 2020 17:12:36 -0800
Subject: [PATCH 1058/1550] Make gather_dep robust to missing tasks (#4285)

Previously the gather_dep function assumed that tasks valid when
starting the function were still valid after an await call.  This may
not be the case.  This was fine, the rest of the work in this functions
was just about book-keeping for diagnostics, so errors didn't stop
things from halting.

Now we don't count the data sent for keys that we no longer know about.
This isn't entirely accurate, but should be fine.

Co-authored-by: James Bourbeau <jrbourbeau@users.noreply.github.com>
---
 distributed/worker.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/distributed/worker.py b/distributed/worker.py
index 447f591b1fd..83e63e4a737 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -2046,7 +2046,9 @@ async def gather_dep(self, worker, dep, deps, total_nbytes, cause=None):
                     )
 
                 total_bytes = sum(
-                    self.tasks[key].nbytes or 0 for key in response["data"]
+                    self.tasks[key].get_nbytes()
+                    for key in response["data"]
+                    if key in self.tasks
                 )
                 duration = (stop - start) or 0.010
                 bandwidth = total_bytes / duration
@@ -2057,7 +2059,9 @@ async def gather_dep(self, worker, dep, deps, total_nbytes, cause=None):
                         "middle": (start + stop) / 2.0 + self.scheduler_delay,
                         "duration": duration,
                         "keys": {
-                            key: self.tasks[key].nbytes for key in response["data"]
+                            key: self.tasks[key].nbytes
+                            for key in response["data"]
+                            if key in self.tasks
                         },
                         "total": total_bytes,
                         "bandwidth": bandwidth,

From 863b010554d37d44d7038ac5402a285869f6b14b Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 30 Nov 2020 17:13:02 -0800
Subject: [PATCH 1059/1550] Avoid flicker in TaskStream with "Scheduler is
 empty" message (#4284)

---
 distributed/dashboard/components/scheduler.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index 11c47390864..c462068bb7b 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -1479,7 +1479,6 @@ def update(self):
                     container.data = {col: [] for col in container.column_names}
             else:
                 # occasionally reset the column data source to remove old nodes
-                self.subtitle.text = " "
                 if self.invisible_count > len(self.node_source.data["x"]) / 2:
                     self.layout.reset_index()
                     self.invisible_count = 0
@@ -1497,6 +1496,8 @@ def update(self):
 
                 if len(self.scheduler.tasks) == 0:
                     self.subtitle.text = "Scheduler is empty."
+                else:
+                    self.subtitle.text = " "
 
     @without_property_validation
     def add_new_nodes_edges(self, new, new_edges, update=False):

From db7de3032dad4cbd27740a660c0adbddb93b0f93 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Tue, 1 Dec 2020 12:46:02 -0800
Subject: [PATCH 1060/1550] Annotate `check_idle_saturated` for Cythonization
 (#4289)

In our profiling of the scheduler, we identified to main transitions that took a good chunk of time. These are `transition_processing_memory` and `transition_waiting_processing`. The former takes slightly longer than the other, but both are easily 2x slower than any transition that follows them. Both of them directly or indirectly make a call to `check_idle_saturated`. While this is not necessarily the worst bottleneck for either of them, it does stick out on the callgraph and stands a good chance of improving both transitions runtimes at once. Additionally `check_idle_saturated` includes a fair bit of code that simply crunches numbers and does not touch Python objects as much. It also isn't as dependent on the Cythonization of other classes as other functions in the profile are. So this makes it easier to Cythonize this piece of code without needing to touch too much other code.

Here we go through and annotate the local variables with types. Also we assign non-local variables accessed through attributes to local variables, which we type. Additionally we changed default argument values to be more friendly with C-style types. Initially we tried to type `self.idle`. However as [`self.idle` is a `sortedcontainers.SortedSet`]( https://github.com/dask/distributed/blob/9460e3fe1e0bcdb2daf6ebafe5335d536fa4f492/distributed/scheduler.py#L1292 ), this didn't work (as we need an actual Python `set` to type it). So we left this as untyped.

Should add as a good chunk of the time in `check_idle_saturated` is just spent doing `ws.status == Status.closed`, we still need PR ( https://github.com/dask/distributed/pull/4270 ) to cutdown on the time spent in this method.



Combine assignments and separate them from `if` blocks.

* Change `occ` default to not be `None`

To make it easier to type `occ` later, define this as a non-`None`
value, which is also clearly bogus (namely `-1.0`). That way we can
still replace this value when it hasn't been otherwise set while also
including a type that includes the default value.

* Annotate `check_idle_saturated` for Cythonization

* Hackily type `set`s through local assignment

For now to just see what is possible, assign these attributes to local
variables typed as `set`s. This will allow Cython to use the
corresponding Python C APIs with these objects; thus, optimizing how
they are handled. This saves us temporarily from trying to more
generally Cythonize this class while exploring optimizations here.

* Drop typing of `self.idle`

As `self.idle` is actually a `SortedSet` instead of a `set`, we can't
type it. So revert the typing of `idle`. Though leave all other
`set`-based typing for other optimizations to be applied where possible.
Also leave the assignment of `self.idle` to `idle` as this generates the
attribute access code only once and we need this for both branches
anyways.
---
 distributed/scheduler.py | 36 ++++++++++++++++++++++++------------
 1 file changed, 24 insertions(+), 12 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 3ff499d876f..f5680dfa7da 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -85,6 +85,14 @@
 from .variable import VariableExtension
 from .protocol.highlevelgraph import highlevelgraph_unpack
 
+try:
+    from cython import double, Py_ssize_t
+except ImportError:
+    from ctypes import (
+        c_double as double,
+        c_ssize_t as Py_ssize_t,
+    )
+
 if sys.version_info < (3, 8):
     try:
         import pickle5 as pickle
@@ -4928,7 +4936,7 @@ def reschedule(self, key=None, worker=None):
     # Assigning Tasks to Workers #
     ##############################
 
-    def check_idle_saturated(self, ws, occ=None):
+    def check_idle_saturated(self, ws, occ: double = -1.0):
         """Update the status of the idle and saturated state
 
         The scheduler keeps track of workers that are ..
@@ -4942,26 +4950,30 @@ def check_idle_saturated(self, ws, occ=None):
 
         This is useful for load balancing and adaptivity.
         """
-        if self.total_nthreads == 0 or ws.status == Status.closed:
+        total_nthreads: Py_ssize_t = self.total_nthreads
+        if total_nthreads == 0 or ws.status == Status.closed:
             return
-        if occ is None:
+        if occ < 0:
             occ = ws.occupancy
-        nc = ws.nthreads
-        p = len(ws.processing)
 
-        avg = self.total_occupancy / self.total_nthreads
+        nc: Py_ssize_t = ws.nthreads
+        p: Py_ssize_t = len(ws.processing)
+        total_occupancy: double = self.total_occupancy
+        avg: double = total_occupancy / total_nthreads
 
+        idle = self.idle
+        saturated: set = self.saturated
         if p < nc or occ / nc < avg / 2:
-            self.idle.add(ws)
-            self.saturated.discard(ws)
+            idle.add(ws)
+            saturated.discard(ws)
         else:
-            self.idle.discard(ws)
+            idle.discard(ws)
 
-            pending = occ * (p - nc) / p / nc
+            pending: double = occ * (p - nc) / p / nc
             if p > nc and pending > 0.4 and pending > 1.9 * avg:
-                self.saturated.add(ws)
+                saturated.add(ws)
             else:
-                self.saturated.discard(ws)
+                saturated.discard(ws)
 
     def valid_workers(self, ts):
         """Return set of currently valid workers for key

From a192b22dece37b9e151b71aaaa129a409f5bc630 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Tue, 1 Dec 2020 17:46:08 -0800
Subject: [PATCH 1061/1550] Annotate `ClientState` for Cythonization (#4290)

* Use `cs` variable for `ClientState`

* Run `black`

* Assign `cs` first to avoid retrieval cost

* Assign `ClientState` instance to local variable

* Check `type`s exactly

* Check `other` is `ClientState` first

* Annotate `ClientState` for Cythonization

* Simplify `ClientState.__repr__`

Avoid building a `tuple` and simply inject the `client_key` directly
into the `str`.

* Annotate `ClientState` methods

* Type all `ClientState` variables

* Type `c` as `str` as well

* Add `_` before all `ClientState` attributes

* Add Python-level `property`s for attributes

* Revert "Add Python-level `property`s for attributes"

This reverts commit a0e0c1c35e6b57989a117cfdd5acaff832d16f09.

* Revert "Add `_` before all `ClientState` attributes"

This reverts commit 33f2b0098d3e2e5d6dd98485dd4916fe908ff9b7.

* Show `declare` approach

* Revert "Show `declare` approach"

This reverts commit 9000361f9b714e6564870d90e7d0afe1ca32d1b5.

* Add `_` before all `ClientState` attributes

* Add Python-level `property`s for attributes

* Add missing `versions` property
---
 distributed/scheduler.py | 122 +++++++++++++++++++++++++++------------
 1 file changed, 84 insertions(+), 38 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index f5680dfa7da..8fe7079f7a3 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -86,13 +86,18 @@
 from .protocol.highlevelgraph import highlevelgraph_unpack
 
 try:
-    from cython import double, Py_ssize_t
+    from cython import cclass, double, Py_hash_t, Py_ssize_t
 except ImportError:
     from ctypes import (
         c_double as double,
+        c_ssize_t as Py_hash_t,
         c_ssize_t as Py_ssize_t,
     )
 
+    def cclass(cls):
+        return cls
+
+
 if sys.version_info < (3, 8):
     try:
         import pickle5 as pickle
@@ -124,6 +129,7 @@
 ALL_TASK_STATES = {"released", "waiting", "no-worker", "processing", "erred", "memory"}
 
 
+@cclass
 class ClientState:
     """
     A simple object holding information about a client.
@@ -145,26 +151,54 @@ class ClientState:
 
     """
 
-    __slots__ = ("client_key", "_hash", "wants_what", "last_seen", "versions")
+    _client_key: str
+    _hash: Py_hash_t
+    _wants_what: set
+    _last_seen: double
+    _versions: dict
 
-    def __init__(self, client, versions=None):
-        self.client_key = client
+    __slots__ = ("_client_key", "_hash", "_wants_what", "_last_seen", "_versions")
+
+    def __init__(self, client: str, versions: dict = None):
+        self._client_key = client
         self._hash = hash(client)
-        self.wants_what = set()
-        self.last_seen = time()
-        self.versions = versions or {}
+        self._wants_what = set()
+        self._last_seen = time()
+        self._versions = versions or {}
 
     def __hash__(self):
         return self._hash
 
     def __eq__(self, other):
-        return type(self) == type(other) and self.client_key == other.client_key
+        typ_self: type = type(self)
+        typ_other: type = type(other)
+        if typ_self == typ_other:
+            other_cs: ClientState = other
+            return self._client_key == other_cs._client_key
+        else:
+            return False
 
     def __repr__(self):
-        return "<Client %r>" % (self.client_key,)
+        return "<Client '%s'>" % self._client_key
 
     def __str__(self):
-        return self.client_key
+        return self._client_key
+
+    @property
+    def client_key(self):
+        return self._client_key
+
+    @property
+    def wants_what(self):
+        return self._wants_what
+
+    @property
+    def last_seen(self):
+        return self._last_seen
+
+    @property
+    def versions(self):
+        return self._versions
 
 
 class WorkerState:
@@ -1009,7 +1043,8 @@ def _legacy_client_key_set(clients):
     """
     Transform a set of client states into a set of client keys.
     """
-    return {cs.client_key for cs in clients}
+    cs: ClientState
+    return {cs._client_key for cs in clients}
 
 
 def _legacy_worker_key_set(workers):
@@ -1837,11 +1872,12 @@ async def add_worker(
                 "worker-plugins": self.worker_plugins,
             }
 
+            cs: ClientState
             version_warning = version_module.error_message(
                 version_module.get_versions(),
                 merge(
                     {w: ws.versions for w, ws in self.workers.items()},
-                    {c: cs.versions for c, cs in self.clients.items() if cs.versions},
+                    {c: cs._versions for c, cs in self.clients.items() if cs._versions},
                 ),
                 versions,
                 client_name="This Worker",
@@ -2408,7 +2444,7 @@ def cancel_key(self, key, client, retries=5, force=False):
         # TODO: this should be converted to use the transition mechanism
         ts = self.tasks.get(key)
         try:
-            cs = self.clients[client]
+            cs: ClientState = self.clients[client]
         except KeyError:
             return
         if ts is None or not ts.who_wants:  # no key yet, lets try again in a moment
@@ -2423,21 +2459,21 @@ def cancel_key(self, key, client, retries=5, force=False):
         logger.info("Scheduler cancels key %s.  Force=%s", key, force)
         self.report({"op": "cancelled-key", "key": key})
         clients = list(ts.who_wants) if force else [cs]
-        for c in clients:
-            self.client_releases_keys(keys=[key], client=c.client_key)
+        for cs in clients:
+            self.client_releases_keys(keys=[key], client=cs._client_key)
 
     def client_desires_keys(self, keys=None, client=None):
-        cs = self.clients.get(client)
+        cs: ClientState = self.clients.get(client)
         if cs is None:
             # For publish, queues etc.
-            cs = self.clients[client] = ClientState(client)
+            self.clients[client] = cs = ClientState(client)
         for k in keys:
             ts = self.tasks.get(k)
             if ts is None:
                 # For publish, queues etc.
                 ts = self.new_task(k, None, "released")
             ts.who_wants.add(cs)
-            cs.wants_what.add(ts)
+            cs._wants_what.add(ts)
 
             if ts.state in ("memory", "erred"):
                 self.report_on_key(k, client=client)
@@ -2445,12 +2481,12 @@ def client_desires_keys(self, keys=None, client=None):
     def client_releases_keys(self, keys=None, client=None):
         """ Remove keys from client desired list """
         logger.debug("Client %s releases keys: %s", client, keys)
-        cs = self.clients[client]
+        cs: ClientState = self.clients[client]
         tasks2 = set()
         for key in list(keys):
             ts = self.tasks.get(key)
-            if ts is not None and ts in cs.wants_what:
-                cs.wants_what.remove(ts)
+            if ts is not None and ts in cs._wants_what:
+                cs._wants_what.remove(ts)
                 s = ts.who_wants
                 s.remove(cs)
                 if not s:
@@ -2468,7 +2504,8 @@ def client_releases_keys(self, keys=None, client=None):
 
     def client_heartbeat(self, client=None):
         """ Handle heartbeats from Client """
-        self.clients[client].last_seen = time()
+        cs: ClientState = self.clients[client]
+        cs._last_seen = time()
 
     ###################
     # Task Validation #
@@ -2574,11 +2611,13 @@ def validate_state(self, allow_overlap=False):
             assert ts.key == k
             self.validate_key(k, ts)
 
+        c: str
+        cs: ClientState
         for c, cs in self.clients.items():
             # client=None is often used in tests...
-            assert c is None or isinstance(c, str), (type(c), c)
-            assert isinstance(cs, ClientState), (type(cs), cs)
-            assert cs.client_key == c
+            assert c is None or type(c) == str, (type(c), c)
+            assert type(cs) == ClientState, (type(cs), cs)
+            assert cs._client_key == c
 
         a = {w: ws.nbytes for w, ws in self.workers.items()}
         b = {
@@ -2611,15 +2650,18 @@ def report(self, msg, ts=None, client=None):
         if ts is None and "key" in msg:
             ts = self.tasks.get(msg["key"])
 
+        cs: ClientState
         if ts is None:
             # Notify all clients
             client_keys = list(self.client_comms)
         elif client is None:
             # Notify clients interested in key
-            client_keys = [c.client_key for c in ts.who_wants]
+            client_keys = [cs._client_key for cs in ts.who_wants]
         else:
             # Notify clients interested in key (including `client`)
-            client_keys = [c.client_key for c in ts.who_wants if c.client_key != client]
+            client_keys = [
+                cs._client_key for cs in ts.who_wants if cs._client_key != client
+            ]
             client_keys.append(client)
 
         for k in client_keys:
@@ -2687,13 +2729,13 @@ def remove_client(self, client=None):
             logger.info("Remove client %s", client)
         self.log_event(["all", client], {"action": "remove-client", "client": client})
         try:
-            cs = self.clients[client]
+            cs: ClientState = self.clients[client]
         except KeyError:
             # XXX is this a legitimate condition?
             pass
         else:
             self.client_releases_keys(
-                keys=[ts.key for ts in cs.wants_what], client=cs.client_key
+                keys=[ts.key for ts in cs._wants_what], client=cs._client_key
             )
             del self.clients[client]
 
@@ -3017,9 +3059,10 @@ async def restart(self, client=None, timeout=3):
             n_workers = len(self.workers)
 
             logger.info("Send lost future signal to clients")
+            cs: ClientState
             for cs in self.clients.values():
                 self.client_releases_keys(
-                    keys=[ts.key for ts in cs.wants_what], client=cs.client_key
+                    keys=[ts.key for ts in cs._wants_what], client=cs._client_key
                 )
 
             nannies = {addr: ws.nanny for addr, ws in self.workers.items()}
@@ -4031,8 +4074,8 @@ def _add_to_memory(
         ts.type = typename
         ts.group.types.add(typename)
 
-        cs = self.clients["fire-and-forget"]
-        if ts in cs.wants_what:
+        cs: ClientState = self.clients["fire-and-forget"]
+        if ts in cs._wants_what:
             self.client_releases_keys(client="fire-and-forget", keys=[ts.key])
 
     def transition_released_waiting(self, key):
@@ -4625,8 +4668,8 @@ def transition_processing_erred(
                 }
             )
 
-            cs = self.clients["fire-and-forget"]
-            if ts in cs.wants_what:
+            cs: ClientState = self.clients["fire-and-forget"]
+            if ts in cs._wants_what:
                 self.client_releases_keys(client="fire-and-forget", keys=[key])
 
             if self.validate:
@@ -4671,8 +4714,9 @@ def remove_key(self, key):
         ts = self.tasks.pop(key)
         assert ts.state == "forgotten"
         self.unrunnable.discard(ts)
+        cs: ClientState
         for cs in ts.who_wants:
-            cs.wants_what.remove(ts)
+            cs._wants_what.remove(ts)
         ts.who_wants.clear()
         ts.processing_on = None
         ts.exception_blame = ts.exception = ts.traceback = None
@@ -5655,12 +5699,13 @@ def validate_task_state(ts):
             )
 
     if ts.who_wants:
+        cs: ClientState
         for cs in ts.who_wants:
-            assert ts in cs.wants_what, (
+            assert ts in cs._wants_what, (
                 "not in who_wants' wants_what",
                 str(ts),
                 str(cs),
-                str(cs.wants_what),
+                str(cs._wants_what),
             )
 
     if ts.actor:
@@ -5696,8 +5741,9 @@ def validate_state(tasks, workers, clients):
     for ws in workers.values():
         validate_worker_state(ws)
 
+    cs: ClientState
     for cs in clients.values():
-        for ts in cs.wants_what:
+        for ts in cs._wants_what:
             assert cs in ts.who_wants, (
                 "not in wants_what' who_wants",
                 str(cs),

From f018007fba3b5d82e7b1a484ba758d2c302abd25 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Wed, 2 Dec 2020 06:55:15 -0800
Subject: [PATCH 1062/1550] Set `WorkerState.processing` w/`dict` in `clean`
 (#4295)

Based on the documentation about `processing`, this should be a `dict`.
However here we are assigning it a `set`. Here we simply pass along the
values along with the keys.
---
 distributed/scheduler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 8fe7079f7a3..a5a1e6ac2c2 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -386,7 +386,7 @@ def clean(self):
             nanny=self.nanny,
             extra=self.extra,
         )
-        ws.processing = {ts.key for ts in self.processing}
+        ws.processing = {ts.key: cost for ts, cost in self.processing.items()}
         return ws
 
     def __repr__(self):

From d114d3a0cc401a8aea0e7a0832aca5b4404f3c66 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Wed, 2 Dec 2020 11:56:42 -0800
Subject: [PATCH 1063/1550] Provide option to build Distributed with Cython
 (#4292)

* Provide option to build Distributed with Cython

* Add `cython` to our environment

* Cythonize Distributed on Python 3.7

* Set some generally useful Cython directives
---
 .github/workflows/ci-windows.yaml             |  7 ++++
 .../environment-windows.yml                   |  1 +
 setup.py                                      | 38 +++++++++++++++++++
 3 files changed, 46 insertions(+)

diff --git a/.github/workflows/ci-windows.yaml b/.github/workflows/ci-windows.yaml
index 3cf6554a709..6d5923eb187 100644
--- a/.github/workflows/ci-windows.yaml
+++ b/.github/workflows/ci-windows.yaml
@@ -47,6 +47,13 @@ jobs:
         shell: bash -l {0}
         run: conda list
 
+      - name: Optionally Cythonize
+        shell: bash -l {0}
+        run: |
+          if [[ "${{ matrix.python-version }}" = "3.7" ]]; then
+            python setup.py build_ext --with-cython
+          fi
+
       - name: Run tests
         shell: bash -l {0}
         env:
diff --git a/continuous_integration/environment-windows.yml b/continuous_integration/environment-windows.yml
index 5bf1c4903d1..5f553204077 100644
--- a/continuous_integration/environment-windows.yml
+++ b/continuous_integration/environment-windows.yml
@@ -7,6 +7,7 @@ dependencies:
   - bokeh!=2.0.0
   - click
   - cloudpickle
+  - cython
   - dask
   - lz4
   - ipykernel
diff --git a/setup.py b/setup.py
index 6b70d0638ce..6cc1e471cbf 100755
--- a/setup.py
+++ b/setup.py
@@ -1,10 +1,13 @@
 #!/usr/bin/env python
 
 import os
+import sys
 from setuptools import setup, find_packages
+from setuptools.extension import Extension
 import versioneer
 
 requires = open("requirements.txt").read().strip().split("\n")
+setup_requires = []
 install_requires = []
 extras_require = {}
 for r in requires:
@@ -18,6 +21,39 @@
     else:
         install_requires.append(r)
 
+try:
+    sys.argv.remove("--with-cython")
+    cython = True
+except ValueError:
+    cython = False
+
+ext_modules = []
+if cython:
+    try:
+        import cython
+    except ImportError:
+        setup_requires.append("cython")
+
+    cyext_modules = [
+        Extension(
+            "distributed.scheduler",
+            sources=["distributed/scheduler.py"],
+        ),
+        Extension(
+            "distributed.protocol.serialize",
+            sources=["distributed/protocol/serialize.py"],
+        ),
+    ]
+    for e in cyext_modules:
+        e.cython_directives = {
+            "annotation_typing": True,
+            "binding": False,
+            "embedsignature": True,
+            "language_level": 3,
+        }
+    ext_modules.extend(cyext_modules)
+
+
 setup(
     name="distributed",
     version=versioneer.get_version(),
@@ -33,9 +69,11 @@
         "distributed": ["http/templates/*.html"],
     },
     include_package_data=True,
+    setup_requires=setup_requires,
     install_requires=install_requires,
     extras_require=extras_require,
     packages=find_packages(exclude=["*tests*"]),
+    ext_modules=ext_modules,
     long_description=(
         open("README.rst").read() if os.path.exists("README.rst") else ""
     ),

From c960db12a5d2a63204d91a4708b14b330a0f8762 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 2 Dec 2020 16:25:25 -0800
Subject: [PATCH 1064/1550] Protect against missing key in get_metrics (#4300)

---
 distributed/worker.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/distributed/worker.py b/distributed/worker.py
index 83e63e4a737..f1896d40c81 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -802,6 +802,7 @@ async def get_metrics(self):
             executing={
                 key: now - self.tasks[key].start_time
                 for key in self.active_threads.values()
+                if key in self.tasks
             },
         )
         custom = {}

From 9bb7179e50b596c3ee3385f66b4dd9e30af0977c Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Wed, 2 Dec 2020 17:14:25 -0800
Subject: [PATCH 1065/1550] Ignores any compiled files generated by Cython
 (#4301)

---
 .gitignore | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.gitignore b/.gitignore
index cf6732eaa70..1c704bb6e77 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,8 @@
 *.pyc
+*.pyd
 *.py~
+*.c
+*.so
 build/
 dist/
 *.egg-info

From dcdb4651eb0331bb99caf3362c84d4878d94ba5b Mon Sep 17 00:00:00 2001
From: Simon Perkins <simon.perkins@gmail.com>
Date: Thu, 3 Dec 2020 19:32:12 +0200
Subject: [PATCH 1066/1550] Transmit Layer annotations to scheduler (#4279)

See also https://github.com/dask/dask/pull/6889
---
 .../environment-windows.yml                   |   2 +-
 distributed/protocol/highlevelgraph.py        |  22 +-
 .../protocol/tests/test_highlevelgraph.py     | 105 ++++-
 distributed/scheduler.py                      | 364 ++++++++++--------
 distributed/tests/test_client.py              | 100 +++++
 distributed/tests/test_highgraph.py           |   0
 6 files changed, 421 insertions(+), 172 deletions(-)
 create mode 100644 distributed/tests/test_highgraph.py

diff --git a/continuous_integration/environment-windows.yml b/continuous_integration/environment-windows.yml
index 5f553204077..81e0f93de37 100644
--- a/continuous_integration/environment-windows.yml
+++ b/continuous_integration/environment-windows.yml
@@ -30,6 +30,6 @@ dependencies:
   - fsspec
   - pip
   - pip:
-      - git+https://github.com/dask/dask
+      - git+https://github.com/dask/dask.git
       - git+https://github.com/joblib/joblib.git
       - git+https://github.com/dask/zict
diff --git a/distributed/protocol/highlevelgraph.py b/distributed/protocol/highlevelgraph.py
index ca515823520..cd0d8316fe0 100644
--- a/distributed/protocol/highlevelgraph.py
+++ b/distributed/protocol/highlevelgraph.py
@@ -10,9 +10,7 @@
 
 from ..utils import CancelledError
 
-from .utils import (
-    msgpack_opts,
-)
+from .utils import msgpack_opts
 from .serialize import (
     import_allowed_module,
     msgpack_encode_default,
@@ -68,10 +66,12 @@ def _materialized_layer_pack(
         stringify(k): [stringify(dep) for dep in deps]
         for k, deps in dependencies.items()
     }
+
+    annotations = layer.pack_annotations()
     all_keys = all_keys.union(dsk)
     dsk = {stringify(k): stringify(v, exclusive=all_keys) for k, v in dsk.items()}
     dsk = valmap(dumps_task, dsk)
-    return {"dsk": dsk, "dependencies": dependencies}
+    return {"dsk": dsk, "dependencies": dependencies, "annotations": annotations}
 
 
 def highlevelgraph_pack(hlg: HighLevelGraph, client, client_keys):
@@ -109,24 +109,32 @@ def highlevelgraph_pack(hlg: HighLevelGraph, client, client_keys):
     return msgpack.dumps({"layers": layers}, default=msgpack_encode_default)
 
 
-def _materialized_layer_unpack(state, dsk, dependencies):
+def _materialized_layer_unpack(state, dsk, dependencies, annotations):
     dsk.update(state["dsk"])
     for k, v in state["dependencies"].items():
         dependencies[k] = list(set(dependencies.get(k, ())) | set(v))
 
+    if state["annotations"]:
+        annotations.update(
+            Layer.expand_annotations(state["annotations"], state["dsk"].keys())
+        )
+
 
 def highlevelgraph_unpack(dumped_hlg):
     # Notice, we set `use_list=False`, which makes msgpack convert lists to tuples
     hlg = msgpack.loads(
         dumped_hlg, object_hook=msgpack_decode_default, use_list=False, **msgpack_opts
     )
+
     dsk = {}
     deps = {}
+    annotations = {}
     for layer in hlg["layers"]:
         if layer["__module__"] is None:  # Default implementation
             unpack_func = _materialized_layer_unpack
         else:
             mod = import_allowed_module(layer["__module__"])
             unpack_func = getattr(mod, layer["__name__"]).__dask_distributed_unpack__
-        unpack_func(layer["state"], dsk, deps)
-    return dsk, deps
+        unpack_func(layer["state"], dsk, deps, annotations)
+
+    return dsk, deps, annotations
diff --git a/distributed/protocol/tests/test_highlevelgraph.py b/distributed/protocol/tests/test_highlevelgraph.py
index 25fdef25ba0..1fcb339f721 100644
--- a/distributed/protocol/tests/test_highlevelgraph.py
+++ b/distributed/protocol/tests/test_highlevelgraph.py
@@ -1,13 +1,20 @@
-import pytest
+import ast
+
+import dask
 
 import dask.array as da
 import dask.dataframe as dd
 
 from distributed.utils_test import gen_cluster
+from distributed.diagnostics import SchedulerPlugin
+
+import pytest
 
 np = pytest.importorskip("numpy")
 pd = pytest.importorskip("pandas")
 
+from numpy.testing import assert_array_equal
+
 
 @gen_cluster(client=True)
 async def test_combo_of_layer_types(c, s, a, b):
@@ -82,3 +89,99 @@ async def test_shuffle(c, s, a, b):
     res = await c.compute(df, optimize_graph=False)
     assert res.dtypes == np.float64
     assert (res == 10.0).all()
+
+
+class TestAnnotationPlugin(SchedulerPlugin):
+    def __init__(self, priority_fn=None, qux="", resource="", retries=0):
+        self.priority_fn = priority_fn or (lambda k: 0)
+        self.qux = qux
+        self.resource = resource
+        self.retries = retries
+
+        self.priority_matches = 0
+        self.resource_matches = 0
+        self.retry_matches = 0
+        self.qux_matches = 0
+
+    def update_graph(self, scheduler, dsk=None, keys=None, restrictions=None, **kwargs):
+        annots = kwargs["annotations"]
+
+        if "priority" in annots:
+            self.priority_matches = sum(
+                int(self.priority_fn(ast.literal_eval(k)) == p)
+                for k, p in annots["priority"].items()
+            )
+
+        if "qux" in annots:
+            self.qux_matches = sum(int(self.qux == v) for v in annots["qux"].values())
+
+        if "custom_resource" in annots:
+            self.resource_matches = sum(
+                int(self.resource == v) for v in annots["custom_resource"].values()
+            )
+
+        if "retries" in annots:
+            self.retry_matches = sum(
+                int(self.retries == v) for v in annots["retries"].values()
+            )
+
+
+@gen_cluster(client=True)
+async def test_array_annotations(c, s, a, b):
+    def fn(k):
+        return k[1] * 5 + k[2]
+
+    qux = "baz"
+    resource = "widget"
+
+    plugin = TestAnnotationPlugin(priority_fn=fn, qux=qux, resource=resource)
+    s.add_plugin(plugin)
+
+    assert plugin in s.plugins
+
+    with dask.annotate(priority=fn, qux=qux):
+        A = da.ones((10, 10), chunks=(2, 2))
+
+    with dask.annotate(custom_resource=resource):
+        B = A + 1
+
+    with dask.config.set(optimization__fuse__active=False):
+        result = await c.compute(B)
+
+    assert_array_equal(result, 2)
+
+    # There are annotation matches per array chunk (i.e. task)
+    assert plugin.qux_matches == A.npartitions
+    assert plugin.priority_matches == A.npartitions
+    assert plugin.resource_matches == B.npartitions
+
+
+@gen_cluster(client=True)
+async def test_dataframe_annotations(c, s, a, b):
+    retries = 5
+    plugin = TestAnnotationPlugin(retries=retries)
+    s.add_plugin(plugin)
+
+    assert plugin in s.plugins
+
+    df = dd.from_pandas(
+        pd.DataFrame(
+            {"a": np.arange(10, dtype=int), "b": np.arange(10, 0, -1, dtype=float)}
+        ),
+        npartitions=5,
+    )
+    df = df.shuffle("a", shuffle="tasks", max_branch=2)
+    acol = df["a"]
+    bcol = df["b"]
+
+    with dask.annotate(retries=retries):
+        df = acol + bcol
+
+    with dask.config.set(optimization__fuse__active=False):
+        rdf = await c.compute(df)
+
+    assert rdf.dtypes == np.float64
+    assert (rdf == 10.0).all()
+
+    # There is an annotation match per partition (i.e. task)
+    assert plugin.retry_matches == df.npartitions
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index a5a1e6ac2c2..88583e7e47b 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -422,239 +422,243 @@ def ncores(self):
 
 class TaskState:
     """
-        A simple object holding information about a task.
+    A simple object holding information about a task.
 
-        .. attribute:: key: str
+    .. attribute:: key: str
 
-           The key is the unique identifier of a task, generally formed
-           from the name of the function, followed by a hash of the function
-           and arguments, like ``'inc-ab31c010444977004d656610d2d421ec'``.
+       The key is the unique identifier of a task, generally formed
+       from the name of the function, followed by a hash of the function
+       and arguments, like ``'inc-ab31c010444977004d656610d2d421ec'``.
 
-        .. attribute:: prefix: TaskPrefix
+    .. attribute:: prefix: TaskPrefix
 
-           The broad class of tasks to which this task belongs like "inc" or
-           "read_csv"
+       The broad class of tasks to which this task belongs like "inc" or
+       "read_csv"
 
-        .. attribute:: run_spec: object
+    .. attribute:: run_spec: object
 
-           A specification of how to run the task.  The type and meaning of this
-           value is opaque to the scheduler, as it is only interpreted by the
-           worker to which the task is sent for executing.
+       A specification of how to run the task.  The type and meaning of this
+       value is opaque to the scheduler, as it is only interpreted by the
+       worker to which the task is sent for executing.
 
-           As a special case, this attribute may also be ``None``, in which case
-           the task is "pure data" (such as, for example, a piece of data loaded
-           in the scheduler using :meth:`Client.scatter`).  A "pure data" task
-           cannot be computed again if its value is lost.
+       As a special case, this attribute may also be ``None``, in which case
+       the task is "pure data" (such as, for example, a piece of data loaded
+       in the scheduler using :meth:`Client.scatter`).  A "pure data" task
+       cannot be computed again if its value is lost.
 
-        .. attribute:: priority: tuple
+    .. attribute:: priority: tuple
 
-           The priority provides each task with a relative ranking which is used
-           to break ties when many tasks are being considered for execution.
+       The priority provides each task with a relative ranking which is used
+       to break ties when many tasks are being considered for execution.
 
-           This ranking is generally a 2-item tuple.  The first (and dominant)
-           item corresponds to when it was submitted.  Generally, earlier tasks
-           take precedence.  The second item is determined by the client, and is
-           a way to prioritize tasks within a large graph that may be important,
-           such as if they are on the critical path, or good to run in order to
-           release many dependencies.  This is explained further in
-           :doc:`Scheduling Policy <scheduling-policies>`.
+       This ranking is generally a 2-item tuple.  The first (and dominant)
+       item corresponds to when it was submitted.  Generally, earlier tasks
+       take precedence.  The second item is determined by the client, and is
+       a way to prioritize tasks within a large graph that may be important,
+       such as if they are on the critical path, or good to run in order to
+       release many dependencies.  This is explained further in
+       :doc:`Scheduling Policy <scheduling-policies>`.
 
-        .. attribute:: state: str
+    .. attribute:: state: str
 
-           This task's current state.  Valid states include ``released``,
-           ``waiting``, ``no-worker``, ``processing``, ``memory``, ``erred``
-           and ``forgotten``.  If it is ``forgotten``, the task isn't stored
-           in the ``tasks`` dictionary anymore and will probably disappear
-           soon from memory.
+       This task's current state.  Valid states include ``released``,
+       ``waiting``, ``no-worker``, ``processing``, ``memory``, ``erred``
+       and ``forgotten``.  If it is ``forgotten``, the task isn't stored
+       in the ``tasks`` dictionary anymore and will probably disappear
+       soon from memory.
 
-        .. attribute:: dependencies: {TaskState}
+    .. attribute:: dependencies: {TaskState}
 
-           The set of tasks this task depends on for proper execution.  Only
-           tasks still alive are listed in this set.  If, for whatever reason,
-           this task also depends on a forgotten task, the
-           :attr:`has_lost_dependencies` flag is set.
+       The set of tasks this task depends on for proper execution.  Only
+       tasks still alive are listed in this set.  If, for whatever reason,
+       this task also depends on a forgotten task, the
+       :attr:`has_lost_dependencies` flag is set.
 
-           A task can only be executed once all its dependencies have already
-           been successfully executed and have their result stored on at least
-           one worker.  This is tracked by progressively draining the
-           :attr:`waiting_on` set.
+       A task can only be executed once all its dependencies have already
+       been successfully executed and have their result stored on at least
+       one worker.  This is tracked by progressively draining the
+       :attr:`waiting_on` set.
 
-        .. attribute:: dependents: {TaskState}
+    .. attribute:: dependents: {TaskState}
 
-           The set of tasks which depend on this task.  Only tasks still alive
-           are listed in this set.
+       The set of tasks which depend on this task.  Only tasks still alive
+       are listed in this set.
 
-           This is the reverse mapping of :attr:`dependencies`.
+       This is the reverse mapping of :attr:`dependencies`.
 
-        .. attribute:: has_lost_dependencies: bool
+    .. attribute:: has_lost_dependencies: bool
 
-           Whether any of the dependencies of this task has been forgotten.
-           For memory consumption reasons, forgotten tasks are not kept in
-           memory even though they may have dependent tasks.  When a task is
-           forgotten, therefore, each of its dependents has their
-           :attr:`has_lost_dependencies` attribute set to ``True``.
+       Whether any of the dependencies of this task has been forgotten.
+       For memory consumption reasons, forgotten tasks are not kept in
+       memory even though they may have dependent tasks.  When a task is
+       forgotten, therefore, each of its dependents has their
+       :attr:`has_lost_dependencies` attribute set to ``True``.
 
-           If :attr:`has_lost_dependencies` is true, this task cannot go
-           into the "processing" state anymore.
+       If :attr:`has_lost_dependencies` is true, this task cannot go
+       into the "processing" state anymore.
 
-        .. attribute:: waiting_on: {TaskState}
+    .. attribute:: waiting_on: {TaskState}
 
-           The set of tasks this task is waiting on *before* it can be executed.
-           This is always a subset of :attr:`dependencies`.  Each time one of the
-           dependencies has finished processing, it is removed from the
-           :attr:`waiting_on` set.
+       The set of tasks this task is waiting on *before* it can be executed.
+       This is always a subset of :attr:`dependencies`.  Each time one of the
+       dependencies has finished processing, it is removed from the
+       :attr:`waiting_on` set.
 
-           Once :attr:`waiting_on` becomes empty, this task can move from the
-           "waiting" state to the "processing" state (unless one of the
-           dependencies errored out, in which case this task is instead
-           marked "erred").
+       Once :attr:`waiting_on` becomes empty, this task can move from the
+       "waiting" state to the "processing" state (unless one of the
+       dependencies errored out, in which case this task is instead
+       marked "erred").
 
-        .. attribute:: waiters: {TaskState}
+    .. attribute:: waiters: {TaskState}
 
-           The set of tasks which need this task to remain alive.  This is always
-           a subset of :attr:`dependents`.  Each time one of the dependents
-           has finished processing, it is removed from the :attr:`waiters`
-           set.
+       The set of tasks which need this task to remain alive.  This is always
+       a subset of :attr:`dependents`.  Each time one of the dependents
+       has finished processing, it is removed from the :attr:`waiters`
+       set.
 
-           Once both :attr:`waiters` and :attr:`who_wants` become empty, this
-           task can be released (if it has a non-empty :attr:`run_spec`) or
-           forgotten (otherwise) by the scheduler, and by any workers
-           in :attr:`who_has`.
+       Once both :attr:`waiters` and :attr:`who_wants` become empty, this
+       task can be released (if it has a non-empty :attr:`run_spec`) or
+       forgotten (otherwise) by the scheduler, and by any workers
+       in :attr:`who_has`.
 
-           .. note:: Counter-intuitively, :attr:`waiting_on` and
-              :attr:`waiters` are not reverse mappings of each other.
+       .. note:: Counter-intuitively, :attr:`waiting_on` and
+          :attr:`waiters` are not reverse mappings of each other.
 
-        .. attribute:: who_wants: {ClientState}
+    .. attribute:: who_wants: {ClientState}
 
-           The set of clients who want this task's result to remain alive.
-           This is the reverse mapping of :attr:`ClientState.wants_what`.
+       The set of clients who want this task's result to remain alive.
+       This is the reverse mapping of :attr:`ClientState.wants_what`.
 
-           When a client submits a graph to the scheduler it also specifies
-           which output tasks it desires, such that their results are not released
-           from memory.
+       When a client submits a graph to the scheduler it also specifies
+       which output tasks it desires, such that their results are not released
+       from memory.
 
-           Once a task has finished executing (i.e. moves into the "memory"
-           or "erred" state), the clients in :attr:`who_wants` are notified.
+       Once a task has finished executing (i.e. moves into the "memory"
+       or "erred" state), the clients in :attr:`who_wants` are notified.
 
-           Once both :attr:`waiters` and :attr:`who_wants` become empty, this
-           task can be released (if it has a non-empty :attr:`run_spec`) or
-           forgotten (otherwise) by the scheduler, and by any workers
-           in :attr:`who_has`.
+       Once both :attr:`waiters` and :attr:`who_wants` become empty, this
+       task can be released (if it has a non-empty :attr:`run_spec`) or
+       forgotten (otherwise) by the scheduler, and by any workers
+       in :attr:`who_has`.
 
-        .. attribute:: who_has: {WorkerState}
+    .. attribute:: who_has: {WorkerState}
 
-           The set of workers who have this task's result in memory.
-           It is non-empty iff the task is in the "memory" state.  There can be
-           more than one worker in this set if, for example, :meth:`Client.scatter`
-           or :meth:`Client.replicate` was used.
+       The set of workers who have this task's result in memory.
+       It is non-empty iff the task is in the "memory" state.  There can be
+       more than one worker in this set if, for example, :meth:`Client.scatter`
+       or :meth:`Client.replicate` was used.
 
-           This is the reverse mapping of :attr:`WorkerState.has_what`.
+       This is the reverse mapping of :attr:`WorkerState.has_what`.
 
-        .. attribute:: processing_on: WorkerState (or None)
+    .. attribute:: processing_on: WorkerState (or None)
 
-           If this task is in the "processing" state, which worker is currently
-           processing it.  Otherwise this is ``None``.
+       If this task is in the "processing" state, which worker is currently
+       processing it.  Otherwise this is ``None``.
 
-           This attribute is kept in sync with :attr:`WorkerState.processing`.
+       This attribute is kept in sync with :attr:`WorkerState.processing`.
 
-        .. attribute:: retries: int
+    .. attribute:: retries: int
 
-           The number of times this task can automatically be retried in case
-           of failure.  If a task fails executing (the worker returns with
-           an error), its :attr:`retries` attribute is checked.  If it is
-           equal to 0, the task is marked "erred".  If it is greater than 0,
-           the :attr:`retries` attribute is decremented and execution is
-           attempted again.
+       The number of times this task can automatically be retried in case
+       of failure.  If a task fails executing (the worker returns with
+       an error), its :attr:`retries` attribute is checked.  If it is
+       equal to 0, the task is marked "erred".  If it is greater than 0,
+       the :attr:`retries` attribute is decremented and execution is
+       attempted again.
 
-        .. attribute:: nbytes: int (or None)
+    .. attribute:: nbytes: int (or None)
 
-           The number of bytes, as determined by ``sizeof``, of the result
-           of a finished task.  This number is used for diagnostics and to
-           help prioritize work.
+       The number of bytes, as determined by ``sizeof``, of the result
+       of a finished task.  This number is used for diagnostics and to
+       help prioritize work.
 
-        .. attribute:: type: str
+    .. attribute:: type: str
 
-           The type of the object as a string.  Only present for tasks that have
-           been computed.
+       The type of the object as a string.  Only present for tasks that have
+       been computed.
 
-        .. attribute:: exception: object
+    .. attribute:: exception: object
 
-           If this task failed executing, the exception object is stored here.
-           Otherwise this is ``None``.
+       If this task failed executing, the exception object is stored here.
+       Otherwise this is ``None``.
 
-        .. attribute:: traceback: object
+    .. attribute:: traceback: object
 
-           If this task failed executing, the traceback object is stored here.
-           Otherwise this is ``None``.
+       If this task failed executing, the traceback object is stored here.
+       Otherwise this is ``None``.
 
-        .. attribute:: exception_blame: TaskState (or None)
+    .. attribute:: exception_blame: TaskState (or None)
 
-           If this task or one of its dependencies failed executing, the
-           failed task is stored here (possibly itself).  Otherwise this
-           is ``None``.
+       If this task or one of its dependencies failed executing, the
+       failed task is stored here (possibly itself).  Otherwise this
+       is ``None``.
 
-        .. attribute:: suspicious: int
+    .. attribute:: suspicious: int
+
+       The number of times this task has been involved in a worker death.
+
+       Some tasks may cause workers to die (such as calling ``os._exit(0)``).
+       When a worker dies, all of the tasks on that worker are reassigned
+       to others.  This combination of behaviors can cause a bad task to
+       catastrophically destroy all workers on the cluster, one after
+       another.  Whenever a worker dies, we mark each task currently
+       processing on that worker (as recorded by
+       :attr:`WorkerState.processing`) as suspicious.
 
-           The number of times this task has been involved in a worker death.
+       If a task is involved in three deaths (or some other fixed constant)
+       then we mark the task as ``erred``.
 
-           Some tasks may cause workers to die (such as calling ``os._exit(0)``).
-           When a worker dies, all of the tasks on that worker are reassigned
-           to others.  This combination of behaviors can cause a bad task to
-           catastrophically destroy all workers on the cluster, one after
-           another.  Whenever a worker dies, we mark each task currently
-           processing on that worker (as recorded by
-           :attr:`WorkerState.processing`) as suspicious.
+    .. attribute:: host_restrictions: {hostnames}
 
-           If a task is involved in three deaths (or some other fixed constant)
-           then we mark the task as ``erred``.
+       A set of hostnames where this task can be run (or ``None`` if empty).
+       Usually this is empty unless the task has been specifically restricted
+       to only run on certain hosts.  A hostname may correspond to one or
+       several connected workers.
 
-        .. attribute:: host_restrictions: {hostnames}
+    .. attribute:: worker_restrictions: {worker addresses}
 
-           A set of hostnames where this task can be run (or ``None`` if empty).
-           Usually this is empty unless the task has been specifically restricted
-           to only run on certain hosts.  A hostname may correspond to one or
-           several connected workers.
+       A set of complete worker addresses where this can be run (or ``None``
+       if empty).  Usually this is empty unless the task has been specifically
+       restricted to only run on certain workers.
 
-        .. attribute:: worker_restrictions: {worker addresses}
+       Note this is tracking worker addresses, not worker states, since
+       the specific workers may not be connected at this time.
 
-           A set of complete worker addresses where this can be run (or ``None``
-           if empty).  Usually this is empty unless the task has been specifically
-           restricted to only run on certain workers.
+    .. attribute:: resource_restrictions: {resource: quantity}
 
-           Note this is tracking worker addresses, not worker states, since
-           the specific workers may not be connected at this time.
+       Resources required by this task, such as ``{'gpu': 1}`` or
+       ``{'memory': 1e9}`` (or ``None`` if empty).  These are user-defined
+       names and are matched against the contents of each
+       :attr:`WorkerState.resources` dictionary.
 
-        .. attribute:: resource_restrictions: {resource: quantity}
+    .. attribute:: loose_restrictions: bool
 
-           Resources required by this task, such as ``{'gpu': 1}`` or
-           ``{'memory': 1e9}`` (or ``None`` if empty).  These are user-defined
-           names and are matched against the contents of each
-           :attr:`WorkerState.resources` dictionary.
+       If ``False``, each of :attr:`host_restrictions`,
+       :attr:`worker_restrictions` and :attr:`resource_restrictions` is
+       a hard constraint: if no worker is available satisfying those
+       restrictions, the task cannot go into the "processing" state and
+       will instead go into the "no-worker" state.
 
-        .. attribute:: loose_restrictions: bool
+       If ``True``, the above restrictions are mere preferences: if no worker
+       is available satisfying those restrictions, the task can still go
+       into the "processing" state and be sent for execution to another
+       connected worker.
 
-           If ``False``, each of :attr:`host_restrictions`,
-           :attr:`worker_restrictions` and :attr:`resource_restrictions` is
-           a hard constraint: if no worker is available satisfying those
-           restrictions, the task cannot go into the "processing" state and
-           will instead go into the "no-worker" state.
+    .. attribute: metadata: dict
 
-           If ``True``, the above restrictions are mere preferences: if no worker
-           is available satisfying those restrictions, the task can still go
-           into the "processing" state and be sent for execution to another
-           connected worker.
+       Metadata related to task.
 
-        .. attribute: metadata: dict
+    .. attribute: actor: bool
 
-           Metadata related to task.
+       Whether or not this task is an Actor.
 
-        .. attribute: actor: bool
+    .. attribute: group: TaskGroup
 
-           Whether or not this task is an Actor.
+        The group of tasks to which this one belongs.
 
-        .. attribute: group: TaskGroup
+    .. attribute: annotations: dict
 
-    :      The group of tasks to which this one belongs.
+        Task annotations
     """
 
     __slots__ = (
@@ -704,6 +708,7 @@ class TaskState:
         "group_key",
         "group",
         "metadata",
+        "annotations",
     )
 
     def __init__(self, key, run_spec):
@@ -732,6 +737,7 @@ def __init__(self, key, run_spec):
         self.group_key = key_split_group(key)
         self.group = None
         self.metadata = {}
+        self.annotations = {}
 
     def __hash__(self):
         return self._hash
@@ -1905,7 +1911,7 @@ def update_graph_hlg(
         fifo_timeout=0,
     ):
 
-        dsk, dependencies = highlevelgraph_unpack(hlg)
+        dsk, dependencies, annotations = highlevelgraph_unpack(hlg)
 
         # Remove any self-dependencies (happens on test_publish_bag() and others)
         for k, v in dependencies.items():
@@ -1938,6 +1944,7 @@ def update_graph_hlg(
             user_priority,
             actors,
             fifo_timeout,
+            annotations,
         )
 
     def update_graph(
@@ -1955,6 +1962,7 @@ def update_graph(
         user_priority=0,
         actors=None,
         fifo_timeout=0,
+        annotations=None,
     ):
         """
         Add new computations to the internal dask graph
@@ -2056,6 +2064,35 @@ def update_graph(
         if isinstance(user_priority, Number):
             user_priority = {k: user_priority for k in tasks}
 
+        annotations = annotations or {}
+        restrictions = restrictions or {}
+        loose_restrictions = loose_restrictions or []
+        resources = resources or {}
+        retries = retries or {}
+
+        # Override existing taxonomy with per task annotations
+        if annotations:
+            if "priority" in annotations:
+                priority.update(annotations["priority"])
+
+            if "workers" in annotations:
+                restrictions.update(annotations["workers"])
+
+            if "allow_other_workers" in annotations:
+                loose_restrictions.extend(
+                    k for k, v in annotations["allow_other_workers"].items() if v
+                )
+
+            if "retries" in annotations:
+                retries.update(annotations["retries"])
+
+            if "resources" in annotations:
+                resources.update(annotations["resources"])
+
+            for a, kv in annotations.items():
+                for k, v in kv.items():
+                    self.tasks[k].annotations[a] = v
+
         # Add actors
         if actors is True:
             actors = list(keys)
@@ -2159,6 +2196,7 @@ def update_graph(
                     priority=priority,
                     loose_restrictions=loose_restrictions,
                     resources=resources,
+                    annotations=annotations,
                 )
             except Exception as e:
                 logger.exception(e)
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 5712c6a9315..606d0ddc571 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -6297,3 +6297,103 @@ def log_scheduler(dask_scheduler):
     events = await c.get_events("topic2")
     assert len(events) == 2
     assert events[1][1] == ("alice", "bob")
+
+
+@gen_cluster(client=True)
+async def test_annotations_task_state(c, s, a, b):
+    da = pytest.importorskip("dask.array")
+
+    with dask.annotate(qux="bar", priority=100):
+        x = da.ones(10, chunks=(5,))
+
+    with dask.config.set(optimization__fuse__active=False):
+        x = await x.persist()
+
+    assert all(
+        {"qux": "bar", "priority": 100} == ts.annotations for ts in s.tasks.values()
+    )
+
+
+@gen_cluster(client=True)
+async def test_annotations_priorities(c, s, a, b):
+    da = pytest.importorskip("dask.array")
+
+    with dask.annotate(priority=15):
+        x = da.ones(10, chunks=(5,))
+
+    with dask.config.set(optimization__fuse__active=False):
+        x = await x.persist()
+
+    assert all("15" in str(ts.priority) for ts in s.tasks.values())
+    assert all({"priority": 15} == ts.annotations for ts in s.tasks.values())
+
+
+@gen_cluster(client=True)
+async def test_annotations_workers(c, s, a, b):
+    da = pytest.importorskip("dask.array")
+
+    with dask.annotate(workers=[a.address]):
+        x = da.ones(10, chunks=(5,))
+
+    with dask.config.set(optimization__fuse__active=False):
+        x = await x.persist()
+
+    assert all({"workers": (a.address,)} == ts.annotations for ts in s.tasks.values())
+    assert all({a.address} == ts.worker_restrictions for ts in s.tasks.values())
+    assert a.data
+    assert not b.data
+
+
+@gen_cluster(client=True)
+async def test_annotations_retries(c, s, a, b):
+    da = pytest.importorskip("dask.array")
+
+    with dask.annotate(retries=2):
+        x = da.ones(10, chunks=(5,))
+
+    with dask.config.set(optimization__fuse__active=False):
+        x = await x.persist()
+
+    assert all(ts.retries == 2 for ts in s.tasks.values())
+    assert all(ts.annotations == {"retries": 2} for ts in s.tasks.values())
+
+
+@gen_cluster(
+    client=True,
+    nthreads=[
+        ("127.0.0.1", 1),
+        ("127.0.0.1", 1, {"resources": {"GPU": 1}}),
+    ],
+)
+async def test_annotations_resources(c, s, a, b):
+    da = pytest.importorskip("dask.array")
+
+    with dask.annotate(resources={"GPU": 1}):
+        x = da.ones(10, chunks=(5,))
+
+    with dask.config.set(optimization__fuse__active=False):
+        x = await x.persist()
+
+    assert all([{"GPU": 1} == ts.resource_restrictions for ts in s.tasks.values()])
+    assert all([{"resources": {"GPU": 1}} == ts.annotations for ts in s.tasks.values()])
+
+
+@gen_cluster(client=True)
+async def test_annotations_loose_restrictions(c, s, a, b):
+    da = pytest.importorskip("dask.array")
+
+    # Eventually fails if allow_other_workers=False
+    with dask.annotate(workers=["fake"], allow_other_workers=True):
+        x = da.ones(10, chunks=(5,))
+
+    with dask.config.set(optimization__fuse__active=False):
+        x = await x.persist()
+
+    assert all(not ts.worker_restrictions for ts in s.tasks.values())
+    assert all({"fake"} == ts.host_restrictions for ts in s.tasks.values())
+    assert all(
+        [
+            {"workers": ("fake",), "allow_other_workers": True} == ts.annotations
+            for ts in s.tasks.values()
+        ]
+    )
diff --git a/distributed/tests/test_highgraph.py b/distributed/tests/test_highgraph.py
new file mode 100644
index 00000000000..e69de29bb2d

From 1c86be3208d73df3074a153fbad59324a804831e Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Thu, 3 Dec 2020 13:31:03 -0800
Subject: [PATCH 1067/1550] Fix `pip` install issue on CI (#4310)

---
 continuous_integration/environment-windows.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/continuous_integration/environment-windows.yml b/continuous_integration/environment-windows.yml
index 81e0f93de37..9b05bd0d0ff 100644
--- a/continuous_integration/environment-windows.yml
+++ b/continuous_integration/environment-windows.yml
@@ -30,6 +30,6 @@ dependencies:
   - fsspec
   - pip
   - pip:
-      - git+https://github.com/dask/dask.git
-      - git+https://github.com/joblib/joblib.git
-      - git+https://github.com/dask/zict
+      - git+https://github.com/dask/dask.git@master
+      - git+https://github.com/joblib/joblib.git@master
+      - git+https://github.com/dask/zict.git@master

From 3407aa31be08ce100bd96e42acb99e1d7daf996e Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Thu, 3 Dec 2020 15:36:46 -0600
Subject: [PATCH 1068/1550] Remove offload try/except for thread_name_prefix
 keyword (#4308)

---
 distributed/utils.py | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/distributed/utils.py b/distributed/utils.py
index 80124a89dcb..30044740f64 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -1429,13 +1429,7 @@ def is_valid_xml(text):
     return xml.etree.ElementTree.fromstring(text) is not None
 
 
-try:
-    _offload_executor = ThreadPoolExecutor(
-        max_workers=1, thread_name_prefix="Dask-Offload"
-    )
-except TypeError:
-    _offload_executor = ThreadPoolExecutor(max_workers=1)
-
+_offload_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="Dask-Offload")
 weakref.finalize(_offload_executor, _offload_executor.shutdown)
 
 

From 7ea9696cc6861e5a0d524025c2f0ff251e881bf6 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Thu, 3 Dec 2020 14:12:17 -0800
Subject: [PATCH 1069/1550] Coerce new `TaskState.nbytes` value to `int`
 (#4311)

Make sure that whatever value we assign to `nbytes` is an `int`. This is
important as `Schedule.bandwidth` may be a `float`. So the value here
could be a `float`. Though we only measure `nbytes` in whole values.
---
 distributed/tests/test_steal.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/tests/test_steal.py b/distributed/tests/test_steal.py
index 8db31aa8b62..8277ede1833 100644
--- a/distributed/tests/test_steal.py
+++ b/distributed/tests/test_steal.py
@@ -493,7 +493,7 @@ async def assert_balanced(inp, expected, c, s, *workers):
                 ts = s.tasks[dat.key]
                 # Ensure scheduler state stays consistent
                 old_nbytes = ts.nbytes
-                ts.nbytes = s.bandwidth * t
+                ts.nbytes = int(s.bandwidth * t)
                 for ws in ts.who_has:
                     ws.nbytes += ts.nbytes - old_nbytes
             else:

From b524ac53719494a2dca778671488f3775b138198 Mon Sep 17 00:00:00 2001
From: jochen-ott-by <jochen.ott@blue-yonder.com>
Date: Fri, 4 Dec 2020 15:42:36 +0100
Subject: [PATCH 1070/1550] Close comm on low-level errors (#4239)

Close comm.stream on low-level errors, such as BufferErrors. In such cases, we do not really know what was written to / read from the underlying socket, so the only safe way forward is to immediately close the connection.

See also #4133
---
 distributed/batched.py               | 34 ++++++----------
 distributed/comm/tcp.py              | 22 +++++++---
 distributed/comm/tests/test_comms.py | 19 +++++++++
 distributed/core.py                  | 16 +++++---
 distributed/tests/test_batched.py    | 60 +---------------------------
 5 files changed, 61 insertions(+), 90 deletions(-)

diff --git a/distributed/batched.py b/distributed/batched.py
index ad9a26a8113..313aab67b56 100644
--- a/distributed/batched.py
+++ b/distributed/batched.py
@@ -99,28 +99,17 @@ def _background_send(self):
                     self.recent_message_log.append("large-message")
                 self.byte_count += nbytes
             except CommClosedError as e:
-                # If the comm is known to be closed, we'll immediately
-                # give up.
                 logger.info("Batched Comm Closed: %s", e)
                 break
             except Exception:
-                # In other cases we'll retry a few times.
-                # https://github.com/pangeo-data/pangeo/issues/788
-                if self._consecutive_failures <= 5:
-                    logger.warning("Error in batched write, retrying")
-                    yield gen.sleep(0.100 * 1.5 ** self._consecutive_failures)
-                    self._consecutive_failures += 1
-                    # Exponential backoff for retries.
-                    # Ensure we don't drop any messages.
-                    if self.buffer:
-                        # Someone could call send while we yielded above?
-                        self.buffer = payload + self.buffer
-                    else:
-                        self.buffer = payload
-                    continue
-                else:
-                    logger.exception("Error in batched write")
-                    break
+                # We cannot safely retry self.comm.write, as we have no idea
+                # what (if anything) was actually written to the underlying stream.
+                # Re-writing messages could result in complete garbage (e.g. if a frame
+                # header has been written, but not the frame payload), therefore
+                # the only safe thing to do here is to abort the stream without
+                # any attempt to re-try `write`.
+                logger.exception("Error in batched write")
+                break
             finally:
                 payload = None  # lose ref
         else:
@@ -128,9 +117,10 @@ def _background_send(self):
             self.stopped.set()
             return
 
-        # If we've reached here, it means our comm is known to be closed or
-        # we've repeatedly failed to send a message. We can't close gracefully
-        # via `.close()` since we can't send messages. So we just abort.
+        # If we've reached here, it means `break` was hit above and
+        # there was an exception when using `comm`.
+        # We can't close gracefully via `.close()` since we can't send messages.
+        # So we just abort.
         # This means that any messages in our buffer our lost.
         # To propagate exceptions, we rely on subsequent `BatchedSend.send`
         # calls to raise CommClosedErrors.
diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py
index 4d8289c904c..b9721b99546 100644
--- a/distributed/comm/tcp.py
+++ b/distributed/comm/tcp.py
@@ -200,6 +200,13 @@ async def read(self, deserializers=None):
             self.stream = None
             if not shutting_down():
                 convert_stream_closed_error(self, e)
+        except Exception:
+            # Some OSError or a another "low-level" exception. We do not really know what
+            # was already read from the underlying socket, so it is not even safe to retry
+            # here using the same stream. The only safe thing to do is to abort.
+            # (See also GitHub #4133).
+            self.abort()
+            raise
         else:
             try:
                 msg = await from_frames(
@@ -253,13 +260,18 @@ async def write(self, msg, serializers=None, on_error="message"):
                         await future
                         bytes_since_last_yield = 0
         except StreamClosedError as e:
-            stream = None
-            convert_stream_closed_error(self, e)
-        except TypeError as e:
+            self.stream = None
+            if not shutting_down():
+                convert_stream_closed_error(self, e)
+        except Exception:
+            # Some OSError or a another "low-level" exception. We do not really know what
+            # was already written to the underlying socket, so it is not even safe to retry
+            # here using the same stream. The only safe thing to do is to abort.
+            # (See also GitHub #4133).
             if stream._write_buffer is None:
                 logger.info("tried to write message %s on closed stream", msg)
-            else:
-                raise
+            self.abort()
+            raise
 
         return sum(lengths)
 
diff --git a/distributed/comm/tests/test_comms.py b/distributed/comm/tests/test_comms.py
index f166b29486e..e94767de2df 100644
--- a/distributed/comm/tests/test_comms.py
+++ b/distributed/comm/tests/test_comms.py
@@ -815,6 +815,25 @@ async def handle_comm(comm):
     await comm.close()
 
 
+@pytest.mark.asyncio
+async def test_comm_closed_on_buffer_error():
+    # Internal errors from comm.stream.write, such as
+    # BufferError should lead to the stream being closed
+    # and not re-used. See GitHub #4133
+    reader, writer = await get_tcp_comm_pair()
+
+    def _write(data):
+        raise BufferError
+
+    writer.stream.write = _write
+    with pytest.raises(BufferError):
+        await writer.write("x")
+    assert writer.stream is None
+
+    await reader.close()
+    await writer.close()
+
+
 #
 # Various stress tests
 #
diff --git a/distributed/core.py b/distributed/core.py
index 91125788900..efb2f7a0ca5 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -458,9 +458,12 @@ async def handle_comm(self, comm, shutting_down=shutting_down):
                         )
                     break
                 except Exception as e:
-                    logger.exception(e)
-                    await comm.write(error_message(e, status="uncaught-error"))
-                    continue
+                    logger.exception("Exception while reading from %s", address)
+                    if comm.closed():
+                        raise
+                    else:
+                        await comm.write(error_message(e, status="uncaught-error"))
+                        continue
                 if not isinstance(msg, dict):
                     raise TypeError(
                         "Bad message type.  Expected dict, got\n  " + str(msg)
@@ -517,8 +520,11 @@ async def handle_comm(self, comm, shutting_down=shutting_down):
                             logger.info("Lost connection to %r: %s", address, e)
                         break
                     except Exception as e:
-                        logger.exception(e)
-                        result = error_message(e, status="uncaught-error")
+                        logger.exception("Exception while handling op %s", op)
+                        if comm.closed():
+                            raise
+                        else:
+                            result = error_message(e, status="uncaught-error")
 
                 # result is not type stable:
                 # when LHS is not Status then RHS must not be Status or it raises.
diff --git a/distributed/tests/test_batched.py b/distributed/tests/test_batched.py
index fa12f9649e9..a1342178a71 100644
--- a/distributed/tests/test_batched.py
+++ b/distributed/tests/test_batched.py
@@ -1,6 +1,5 @@
 import asyncio
 import random
-from unittest import mock
 
 import pytest
 from tlz import assoc
@@ -8,10 +7,9 @@
 from distributed.batched import BatchedSend
 from distributed.core import listen, connect, CommClosedError
 from distributed.metrics import time
-from distributed.utils import All, TimeoutError
+from distributed.utils import All
 from distributed.utils_test import captured_logger
 from distributed.protocol import to_serialize
-from distributed.compatibility import WINDOWS, TORNADO6
 
 
 class EchoServer:
@@ -250,58 +248,4 @@ async def test_serializers():
         assert "type" in value
         assert "function" in value
 
-        msg = await comm.read()
-        assert list(msg) == [{"x": 123}, {"x": "hello"}]
-
-        with pytest.raises(TimeoutError):
-            msg = await asyncio.wait_for(comm.read(), 0.1)
-
-
-@pytest.mark.asyncio
-@pytest.mark.skipif(WINDOWS and not TORNADO6, reason="failing on windows, tornado 5.")
-async def test_handles_exceptions():
-    # Ensure that we properly handle exceptions in BatchedSend.
-    # https://github.com/pangeo-data/pangeo/issues/788
-    # mentioned in https://github.com/dask/distributed/issues/4080, but
-    # possibly distinct.
-    #
-    # The reported issues (https://github.com/tornadoweb/tornado/pull/2008)
-    # claim that the BufferError *should* only happen when the application
-    # is incorrectly using threads. I haven't been able to construct an
-    # actual example, so we mock IOStream.write to raise and ensure that
-    # BufferedSend handles things correctly. We don't (yet) test that
-    # any *users* of BatchedSend correctly handle BatchedSend dropping
-    # messages.
-    async with EchoServer() as e:
-        comm = await connect(e.address)
-        b = BatchedSend(interval=10)
-        b.start(comm)
-        await asyncio.sleep(0.020)
-        orig = comm.stream.write
-
-        n = 0
-
-        def raise_buffererror(*args, **kwargs):
-            nonlocal n
-            n += 1
-
-            if n == 1:
-                raise BufferError("bad!")
-            elif n == 2:
-                orig(*args, **kwargs)
-            else:
-                raise CommClosedError
-
-        with mock.patch.object(comm.stream, "write", wraps=raise_buffererror):
-            b.send("hello")
-            b.send("hello")
-            b.send("world")
-            await asyncio.sleep(0.020)
-            result = await comm.read()
-            assert result == ("hello", "hello", "world")
-
-            b.send("raises when flushed")
-            await asyncio.sleep(0.020)  # CommClosedError hit in callback
-
-            with pytest.raises(CommClosedError):
-                b.send("raises when sent")
+        assert comm.closed()

From 5d086633093205111d64275f4532ee0d5907ead0 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Fri, 4 Dec 2020 06:49:46 -0800
Subject: [PATCH 1071/1550] Annotate `WorkerState` for Cythonization (#4294)

Analogous to PR ( https://github.com/dask/distributed/pull/4290 ) except instead of `ClientState`, this is annotating `WorkerState` and usages thereof. This is a bit longer simply because of how many attributes `WorkerState` has. That said, this just uses the same strategy here as was used for `ClientState`.

* Coerce `bandwidth` to `float`

Typically `bandwidth` seems to be a `float` elsewhere in the code. So
coerce it to a `float` here as well to align with that practice.

* Use `ws` variable name for `WorkerState` objects

* Name `WorkerState` variable distinctly in closure

Avoids confusion with the variable name used outside this closure. Also
bypasses any potential leakage from the external scope.

* Assign selected `WorkerState` to variable

* Annotate `WorkerState` for Cythonization

* Annotate all `WorkerState` variables

* Prefix `WorkerState` attributes with `_`

Since these are effectively private given they are only available in
Cython/C, go ahead and mark them as such. We can then add Python
`@property`s on top of them to handle Python access to the original
attribute names with minimal disruption.

* Add Python-level `property`s for attributes

* Run `black`

* Add some `property.setter`s

Includes `setter`s for `occupancy` and `nbytes`.

* Create `list` from generator

This ensures Cython still uses `WorkerState` to annotate the variable
iterated over. Otherwise it constructs a generator with its own scope
where this is ignored.

* Relax `_name` to `object`

Apparently `_name` is sometimes an `int`. So relax typing around it to
just an `object`.
---
 distributed/scheduler.py | 643 ++++++++++++++++++++++++---------------
 1 file changed, 401 insertions(+), 242 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 88583e7e47b..fe155ccce7d 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -201,11 +201,12 @@ def versions(self):
         return self._versions
 
 
+@cclass
 class WorkerState:
     """
     A simple object holding information about a worker.
 
-    .. attribute:: address
+    .. attribute:: address: str
 
        This worker's unique key.  This can be its connected address
        (such as ``'tcp://127.0.0.1:8891'``) or an alias (such as ``'alice'``).
@@ -253,7 +254,7 @@ class WorkerState:
        The numbers in this dictionary can only be less or equal than
        those in this worker's :attr:`resources`.
 
-    .. attribute:: occupancy: Number
+    .. attribute:: occupancy: double
 
        The total expected runtime, in seconds, of all tasks currently
        processing on this worker.  This is the sum of all the costs in
@@ -267,7 +268,7 @@ class WorkerState:
 
        Address of the associated Nanny, if present
 
-    .. attribute:: last_seen: Number
+    .. attribute:: last_seen: Py_ssize_t
 
        The last time we received a heartbeat from this worker, in local
        scheduler time.
@@ -282,71 +283,97 @@ class WorkerState:
 
     # XXX need a state field to signal active/removed?
 
+    _actors: set
+    _address: str
+    _bandwidth: double
+    _extra: dict
+    _has_what: set
+    _hash: Py_hash_t
+    _last_seen: double
+    _local_directory: str
+    _memory_limit: Py_ssize_t
+    _metrics: dict
+    _name: object
+    _nanny: str
+    _nbytes: Py_ssize_t
+    _nthreads: Py_ssize_t
+    _occupancy: double
+    _pid: Py_ssize_t
+    _processing: dict
+    _resources: dict
+    _services: dict
+    _status: Status
+    _time_delay: double
+    _used_resources: dict
+    _versions: dict
+
     __slots__ = (
-        "actors",
-        "address",
-        "bandwidth",
-        "extra",
-        "has_what",
+        "_actors",
+        "_address",
+        "_bandwidth",
+        "_extra",
+        "_has_what",
         "_hash",
-        "last_seen",
-        "local_directory",
-        "memory_limit",
-        "metrics",
-        "name",
-        "nanny",
-        "nbytes",
-        "nthreads",
-        "occupancy",
-        "pid",
-        "processing",
-        "resources",
-        "services",
+        "_last_seen",
+        "_local_directory",
+        "_memory_limit",
+        "_metrics",
+        "_name",
+        "_nanny",
+        "_nbytes",
+        "_nthreads",
+        "_occupancy",
+        "_pid",
+        "_processing",
+        "_resources",
+        "_services",
         "_status",
-        "time_delay",
-        "used_resources",
-        "versions",
+        "_time_delay",
+        "_used_resources",
+        "_versions",
     )
 
     def __init__(
         self,
-        address=None,
-        pid=0,
-        name=None,
-        nthreads=0,
-        memory_limit=0,
-        local_directory=None,
-        services=None,
-        versions=None,
-        nanny=None,
-        extra=None,
+        address: str = None,
+        pid: Py_ssize_t = 0,
+        name: object = None,
+        nthreads: Py_ssize_t = 0,
+        memory_limit: Py_ssize_t = 0,
+        local_directory: str = None,
+        services: dict = None,
+        versions: dict = None,
+        nanny: str = None,
+        extra: dict = None,
     ):
-        self.address = address
-        self.pid = pid
-        self.name = name
-        self.nthreads = nthreads
-        self.memory_limit = memory_limit
-        self.local_directory = local_directory
-        self.services = services or {}
-        self.versions = versions or {}
-        self.nanny = nanny
+        self._address = address
+        self._pid = pid
+        self._name = name
+        self._nthreads = nthreads
+        self._memory_limit = memory_limit
+        self._local_directory = local_directory
+        self._services = services or {}
+        self._versions = versions or {}
+        self._nanny = nanny
 
         self._hash = hash(address)
         self._status = Status.running
-        self.nbytes = 0
-        self.occupancy = 0
-        self.metrics = {}
-        self.last_seen = 0
-        self.time_delay = 0
-        self.bandwidth = parse_bytes(dask.config.get("distributed.scheduler.bandwidth"))
+        self._nbytes = 0
+        self._occupancy = 0
+        self._metrics = {}
+        self._last_seen = 0
+        self._time_delay = 0
+        self._bandwidth = float(
+            parse_bytes(dask.config.get("distributed.scheduler.bandwidth"))
+        )
 
-        self.actors = set()
-        self.has_what = set()
-        self.processing = {}
-        self.resources = {}
-        self.used_resources = {}
+        self._actors = set()
+        self._has_what = set()
+        self._processing = {}
+        self._resources = {}
+        self._used_resources = {}
 
-        self.extra = extra or {}
+        self._extra = extra or {}
 
     def __hash__(self):
         return self._hash
@@ -354,6 +381,90 @@ def __hash__(self):
     def __eq__(self, other):
         return type(self) == type(other) and self.address == other.address
 
+    @property
+    def actors(self):
+        return self._actors
+
+    @property
+    def address(self):
+        return self._address
+
+    @property
+    def bandwidth(self):
+        return self._bandwidth
+
+    @property
+    def extra(self):
+        return self._extra
+
+    @property
+    def has_what(self):
+        return self._has_what
+
+    @property
+    def host(self):
+        return get_address_host(self.address)
+
+    @property
+    def last_seen(self):
+        return self._last_seen
+
+    @property
+    def local_directory(self):
+        return self._local_directory
+
+    @property
+    def memory_limit(self):
+        return self._memory_limit
+
+    @property
+    def metrics(self):
+        return self._metrics
+
+    @property
+    def name(self):
+        return self._name
+
+    @property
+    def nanny(self):
+        return self._nanny
+
+    @property
+    def nbytes(self):
+        return self._nbytes
+
+    @nbytes.setter
+    def nbytes(self, v: Py_ssize_t):
+        self._nbytes = v
+
+    @property
+    def nthreads(self):
+        return self._nthreads
+
+    @property
+    def occupancy(self):
+        return self._occupancy
+
+    @occupancy.setter
+    def occupancy(self, v: double):
+        self._occupancy = v
+
+    @property
+    def pid(self):
+        return self._pid
+
+    @property
+    def processing(self):
+        return self._processing
+
+    @property
+    def resources(self):
+        return self._resources
+
+    @property
+    def services(self):
+        return self._services
+
     @property
     def status(self):
         return self._status
@@ -370,54 +481,62 @@ def status(self, new_status):
             raise TypeError(f"expected Status or str, got {new_status}")
 
     @property
-    def host(self):
-        return get_address_host(self.address)
+    def time_delay(self):
+        return self._time_delay
+
+    @property
+    def used_resources(self):
+        return self._used_resources
+
+    @property
+    def versions(self):
+        return self._versions
 
     def clean(self):
         """ Return a version of this object that is appropriate for serialization """
-        ws = WorkerState(
-            address=self.address,
-            pid=self.pid,
-            name=self.name,
-            nthreads=self.nthreads,
-            memory_limit=self.memory_limit,
-            local_directory=self.local_directory,
-            services=self.services,
-            nanny=self.nanny,
-            extra=self.extra,
+        ws: WorkerState = WorkerState(
+            address=self._address,
+            pid=self._pid,
+            name=self._name,
+            nthreads=self._nthreads,
+            memory_limit=self._memory_limit,
+            local_directory=self._local_directory,
+            services=self._services,
+            nanny=self._nanny,
+            extra=self._extra,
         )
-        ws.processing = {ts.key: cost for ts, cost in self.processing.items()}
+        ws._processing = {ts.key: cost for ts, cost in self._processing.items()}
         return ws
 
     def __repr__(self):
         return "<Worker %r, name: %s, memory: %d, processing: %d>" % (
-            self.address,
-            self.name,
-            len(self.has_what),
-            len(self.processing),
+            self._address,
+            self._name,
+            len(self._has_what),
+            len(self._processing),
         )
 
     def identity(self):
         return {
             "type": "Worker",
-            "id": self.name,
+            "id": self._name,
             "host": self.host,
-            "resources": self.resources,
-            "local_directory": self.local_directory,
-            "name": self.name,
-            "nthreads": self.nthreads,
-            "memory_limit": self.memory_limit,
-            "last_seen": self.last_seen,
-            "services": self.services,
-            "metrics": self.metrics,
-            "nanny": self.nanny,
-            **self.extra,
+            "resources": self._resources,
+            "local_directory": self._local_directory,
+            "name": self._name,
+            "nthreads": self._nthreads,
+            "memory_limit": self._memory_limit,
+            "last_seen": self._last_seen,
+            "services": self._services,
+            "metrics": self._metrics,
+            "nanny": self._nanny,
+            **self._extra,
         }
 
     @property
     def ncores(self):
         warnings.warn("WorkerState.ncores has moved to WorkerState.nthreads")
-        return self.nthreads
+        return self._nthreads
 
 
 class TaskState:
@@ -774,8 +893,9 @@ def set_nbytes(self, nbytes: int):
         diff = nbytes - (old_nbytes or 0)
         self.group.nbytes_total += diff
         self.group.nbytes_in_memory += diff
+        ws: WorkerState
         for ws in self.who_has:
-            ws.nbytes += diff
+            ws._nbytes += diff
         self.nbytes = nbytes
 
     def __repr__(self):
@@ -1057,7 +1177,8 @@ def _legacy_worker_key_set(workers):
     """
     Transform a set of worker states into a set of worker keys.
     """
-    return {ws.address for ws in workers}
+    ws: WorkerState
+    return {ws._address for ws in workers}
 
 
 def _legacy_task_key_dict(task_dict):
@@ -1528,13 +1649,13 @@ def get_worker_service_addr(self, worker, service_name, protocol=False):
             Whether or not to include a full address with protocol (True)
             or just a (host, port) pair
         """
-        ws = self.workers[worker]
-        port = ws.services.get(service_name)
+        ws: WorkerState = self.workers[worker]
+        port = ws._services.get(service_name)
         if port is None:
             return None
         elif protocol:
             return "%(protocol)s://%(host)s:%(port)d" % {
-                "protocol": ws.address.split("://")[0],
+                "protocol": ws._address.split("://")[0],
                 "host": ws.host,
                 "port": port,
             }
@@ -1676,7 +1797,8 @@ async def close_worker(self, comm=None, worker=None, safe=None):
         logger.info("Closing worker %s", worker)
         with log_errors():
             self.log_event(worker, {"action": "close-worker"})
-            nanny_addr = self.workers[worker].nanny
+            ws: WorkerState = self.workers[worker]
+            nanny_addr = ws._nanny
             address = nanny_addr or worker
 
             self.worker_send(worker, {"op": "close", "report": False})
@@ -1729,18 +1851,18 @@ def heartbeat_worker(
                     1 - alpha
                 )
 
-        ws = self.workers[address]
+        ws: WorkerState = self.workers[address]
 
-        ws.last_seen = time()
+        ws._last_seen = time()
 
         if metrics:
-            ws.metrics = metrics
+            ws._metrics = metrics
 
         if host_info:
             self.host_info[host].update(host_info)
 
         delay = time() - now
-        ws.time_delay = delay
+        ws._time_delay = delay
 
         if resources:
             self.add_resources(worker=address, resources=resources)
@@ -1781,7 +1903,7 @@ async def add_worker(
             address = normalize_address(address)
             host = get_address_host(address)
 
-            ws = self.workers.get(address)
+            ws: WorkerState = self.workers.get(address)
             if ws is not None:
                 raise ValueError("Worker already exists %s" % ws)
 
@@ -1834,7 +1956,7 @@ async def add_worker(
 
             self.stream_comms[address] = BatchedSend(interval="5ms", loop=self.loop)
 
-            if ws.nthreads > len(ws.processing):
+            if ws._nthreads > len(ws._processing):
                 self.idle.add(ws)
 
             for plugin in self.plugins[:]:
@@ -1882,7 +2004,7 @@ async def add_worker(
             version_warning = version_module.error_message(
                 version_module.get_versions(),
                 merge(
-                    {w: ws.versions for w, ws in self.workers.items()},
+                    {w: ws._versions for w, ws in self.workers.items()},
                     {c: cs._versions for c, cs in self.clients.items() if cs._versions},
                 ),
                 versions,
@@ -2242,7 +2364,7 @@ def stimulus_task_finished(self, key=None, worker=None, **kwargs):
         ts = self.tasks.get(key)
         if ts is None:
             return {}
-        ws = self.workers[worker]
+        ws: WorkerState = self.workers[worker]
         ts.metadata.update(kwargs["metadata"])
 
         if ts.state == "processing":
@@ -2310,9 +2432,10 @@ def stimulus_missing_data(
             recommendations = {}
 
             if cts is not None and cts.state == "memory":  # couldn't find this
+                ws: WorkerState
                 for ws in cts.who_has:  # TODO: this behavior is extreme
-                    ws.has_what.remove(cts)
-                    ws.nbytes -= cts.get_nbytes()
+                    ws._has_what.remove(cts)
+                    ws._nbytes -= cts.get_nbytes()
                 cts.who_has.clear()
                 recommendations[cause] = "released"
 
@@ -2373,14 +2496,14 @@ async def remove_worker(self, comm=None, address=None, safe=False, close=True):
 
             host = get_address_host(address)
 
-            ws = self.workers[address]
+            ws: WorkerState = self.workers[address]
 
             self.log_event(
                 ["all", address],
                 {
                     "action": "remove-worker",
                     "worker": address,
-                    "processing-tasks": dict(ws.processing),
+                    "processing-tasks": dict(ws._processing),
                 },
             )
             logger.info("Remove worker %s", ws)
@@ -2390,25 +2513,25 @@ async def remove_worker(self, comm=None, address=None, safe=False, close=True):
 
             self.remove_resources(address)
 
-            self.host_info[host]["nthreads"] -= ws.nthreads
+            self.host_info[host]["nthreads"] -= ws._nthreads
             self.host_info[host]["addresses"].remove(address)
-            self.total_nthreads -= ws.nthreads
+            self.total_nthreads -= ws._nthreads
 
             if not self.host_info[host]["addresses"]:
                 del self.host_info[host]
 
             self.rpc.remove(address)
             del self.stream_comms[address]
-            del self.aliases[ws.name]
+            del self.aliases[ws._name]
             self.idle.discard(ws)
             self.saturated.discard(ws)
             del self.workers[address]
             ws.status = Status.closed
-            self.total_occupancy -= ws.occupancy
+            self.total_occupancy -= ws._occupancy
 
             recommendations = {}
 
-            for ts in list(ws.processing):
+            for ts in list(ws._processing):
                 k = ts.key
                 recommendations[k] = "released"
                 if not safe:
@@ -2428,14 +2551,14 @@ async def remove_worker(self, comm=None, address=None, safe=False, close=True):
                             self.allowed_failures,
                         )
 
-            for ts in ws.has_what:
+            for ts in ws._has_what:
                 ts.who_has.remove(ws)
                 if not ts.who_has:
                     if ts.run_spec:
                         recommendations[ts.key] = "released"
                     else:  # pure data
                         recommendations[ts.key] = "forgotten"
-            ws.has_what.clear()
+            ws._has_what.clear()
 
             self.transitions(recommendations)
 
@@ -2573,9 +2696,9 @@ def validate_waiting(self, key):
     def validate_processing(self, key):
         ts = self.tasks[key]
         assert not ts.waiting_on
-        ws = ts.processing_on
+        ws: WorkerState = ts.processing_on
         assert ws
-        assert ts in ws.processing
+        assert ts in ws._processing
         assert not ts.who_has
         for dts in ts.dependencies:
             assert dts.who_has
@@ -2636,12 +2759,13 @@ def validate_state(self, allow_overlap=False):
         if not (set(self.workers) == set(self.stream_comms)):
             raise ValueError("Workers not the same in all collections")
 
+        ws: WorkerState
         for w, ws in self.workers.items():
             assert isinstance(w, str), (type(w), w)
             assert isinstance(ws, WorkerState), (type(ws), ws)
-            assert ws.address == w
-            if not ws.processing:
-                assert not ws.occupancy
+            assert ws._address == w
+            if not ws._processing:
+                assert not ws._occupancy
                 assert ws in self.idle
 
         for k, ts in self.tasks.items():
@@ -2657,17 +2781,17 @@ def validate_state(self, allow_overlap=False):
             assert type(cs) == ClientState, (type(cs), cs)
             assert cs._client_key == c
 
-        a = {w: ws.nbytes for w, ws in self.workers.items()}
+        a = {w: ws._nbytes for w, ws in self.workers.items()}
         b = {
-            w: sum(ts.get_nbytes() for ts in ws.has_what)
+            w: sum(ts.get_nbytes() for ts in ws._has_what)
             for w, ws in self.workers.items()
         }
         assert a == b, (a, b)
 
         actual_total_occupancy = 0
         for worker, ws in self.workers.items():
-            assert abs(sum(ws.processing.values()) - ws.occupancy) < 1e-8
-            actual_total_occupancy += ws.occupancy
+            assert abs(sum(ws._processing.values()) - ws._occupancy) < 1e-8
+            actual_total_occupancy += ws._occupancy
 
         assert abs(actual_total_occupancy - self.total_occupancy) < 1e-8, (
             actual_total_occupancy,
@@ -2736,9 +2860,10 @@ async def add_client(self, comm, client=None, versions=None):
             bcomm.start(comm)
             self.client_comms[client] = bcomm
             msg = {"op": "stream-start"}
+            ws: WorkerState
             version_warning = version_module.error_message(
                 version_module.get_versions(),
-                {w: ws.versions for w, ws in self.workers.items()},
+                {w: ws._versions for w, ws in self.workers.items()},
                 versions,
             )
             msg.update(version_warning)
@@ -2811,8 +2936,9 @@ def send_task_to_worker(self, worker, key):
 
             deps = ts.dependencies
             if deps:
+                ws: WorkerState
                 msg["who_has"] = {
-                    dep.key: [ws.address for ws in dep.who_has] for dep in deps
+                    dep.key: [ws._address for ws in dep.who_has] for dep in deps
                 }
                 msg["nbytes"] = {dep.key: dep.nbytes for dep in deps}
 
@@ -2852,7 +2978,7 @@ def handle_release_data(self, key=None, worker=None, client=None, **msg):
         ts = self.tasks.get(key)
         if ts is None:
             return
-        ws = self.workers[worker]
+        ws: WorkerState = self.workers[worker]
         if ts.processing_on != ws:
             return
         r = self.stimulus_missing_data(key=key, ensure=False, **msg)
@@ -2866,11 +2992,11 @@ def handle_missing_data(self, key=None, errant_worker=None, **kwargs):
         if ts is None or not ts.who_has:
             return
         if errant_worker in self.workers:
-            ws = self.workers[errant_worker]
+            ws: WorkerState = self.workers[errant_worker]
             if ws in ts.who_has:
                 ts.who_has.remove(ws)
-                ws.has_what.remove(ts)
-                ws.nbytes -= ts.get_nbytes()
+                ws._has_what.remove(ts)
+                ws._nbytes -= ts.get_nbytes()
         if not ts.who_has:
             if ts.run_spec:
                 self.transitions({key: "released"})
@@ -2878,14 +3004,14 @@ def handle_missing_data(self, key=None, errant_worker=None, **kwargs):
                 self.transitions({key: "forgotten"})
 
     def release_worker_data(self, comm=None, keys=None, worker=None):
-        ws = self.workers[worker]
+        ws: WorkerState = self.workers[worker]
         tasks = {self.tasks[k] for k in keys}
-        removed_tasks = tasks & ws.has_what
-        ws.has_what -= removed_tasks
+        removed_tasks = tasks & ws._has_what
+        ws._has_what -= removed_tasks
 
         recommendations = {}
         for ts in removed_tasks:
-            ws.nbytes -= ts.get_nbytes()
+            ws._nbytes -= ts.get_nbytes()
             wh = ts.who_has
             wh.remove(ws)
             if not wh:
@@ -2903,7 +3029,7 @@ def handle_long_running(self, key=None, worker=None, compute_duration=None):
         if "stealing" in self.extensions:
             self.extensions["stealing"].remove_key_from_stealable(ts)
 
-        ws = ts.processing_on
+        ws: WorkerState = ts.processing_on
         if ws is None:
             logger.debug("Received long-running signal from duplicate task. Ignoring.")
             return
@@ -2918,9 +3044,9 @@ def handle_long_running(self, key=None, worker=None, compute_duration=None):
 
             ts.prefix.duration_average = avg_duration
 
-        ws.occupancy -= ws.processing[ts]
-        self.total_occupancy -= ws.processing[ts]
-        ws.processing[ts] = 0
+        ws._occupancy -= ws._processing[ts]
+        self.total_occupancy -= ws._processing[ts]
+        ws._processing[ts] = 0
         self.check_idle_saturated(ws)
 
     async def handle_worker(self, comm=None, worker=None):
@@ -2999,7 +3125,8 @@ async def scatter(
                 raise TimeoutError("No workers found")
 
         if workers is None:
-            nthreads = {w: ws.nthreads for w, ws in self.workers.items()}
+            ws: WorkerState
+            nthreads = {w: ws._nthreads for w, ws in self.workers.items()}
         else:
             workers = [self.coerce_address(w) for w in workers]
             nthreads = {w: self.workers[w].nthreads for w in workers}
@@ -3026,12 +3153,13 @@ async def scatter(
 
     async def gather(self, comm=None, keys=None, serializers=None):
         """ Collect data in from workers """
+        ws: WorkerState
         keys = list(keys)
         who_has = {}
         for key in keys:
             ts = self.tasks.get(key)
             if ts is not None:
-                who_has[key] = [ws.address for ws in ts.who_has]
+                who_has[key] = [ws._address for ws in ts.who_has]
             else:
                 who_has[key] = []
 
@@ -3074,10 +3202,10 @@ async def gather(self, comm=None, keys=None, serializers=None):
                         continue
                     for worker in workers:
                         ws = self.workers.get(worker)
-                        if ws is not None and ts in ws.has_what:
-                            ws.has_what.remove(ts)
+                        if ws is not None and ts in ws._has_what:
+                            ws._has_what.remove(ts)
                             ts.who_has.remove(ws)
-                            ws.nbytes -= ts.get_nbytes()
+                            ws._nbytes -= ts.get_nbytes()
                             self.transitions({key: "released"})
 
         self.log_event("all", {"action": "gather", "count": len(keys)})
@@ -3103,7 +3231,8 @@ async def restart(self, client=None, timeout=3):
                     keys=[ts.key for ts in cs._wants_what], client=cs._client_key
                 )
 
-            nannies = {addr: ws.nanny for addr, ws in self.workers.items()}
+            ws: WorkerState
+            nannies = {addr: ws._nanny for addr, ws in self.workers.items()}
 
             for addr in list(self.workers):
                 try:
@@ -3229,13 +3358,13 @@ async def _delete_worker_data(self, worker_address, keys):
             self.rpc(addr=worker_address).delete_data, keys=list(keys), report=False
         )
 
-        ws = self.workers[worker_address]
+        ws: WorkerState = self.workers[worker_address]
         tasks = {self.tasks[key] for key in keys}
-        ws.has_what -= tasks
+        ws._has_what -= tasks
         for ts in tasks:
             ts.who_has.remove(ws)
-            ws.nbytes -= ts.get_nbytes()
-        self.log_event(ws.address, {"action": "remove-worker-data", "keys": keys})
+            ws._nbytes -= ts.get_nbytes()
+        self.log_event(ws._address, {"action": "remove-worker-data", "keys": keys})
 
     async def rebalance(self, comm=None, keys=None, workers=None):
         """Rebalance keys so that each worker stores roughly equal bytes
@@ -3265,6 +3394,7 @@ async def rebalance(self, comm=None, keys=None, workers=None):
                     workers = set(self.workers.values())
                     workers_by_task = {ts: ts.who_has for ts in tasks}
 
+                ws: WorkerState
                 tasks_by_worker = {ws: set() for ws in workers}
 
                 for k, v in workers_by_task.items():
@@ -3395,6 +3525,9 @@ async def replicate(
         --------
         Scheduler.rebalance
         """
+        ws: WorkerState
+        wws: WorkerState
+
         assert branching_factor > 0
         async with self._lock if lock else empty_context:
             workers = {self.workers[w] for w in self.workers_list(workers)}
@@ -3423,7 +3556,7 @@ async def replicate(
 
                 await asyncio.gather(
                     *(
-                        self._delete_worker_data(ws.address, [t.key for t in tasks])
+                        self._delete_worker_data(ws._address, [t.key for t in tasks])
                         for ws, tasks in del_worker_tasks.items()
                     )
                 )
@@ -3446,8 +3579,8 @@ async def replicate(
                     assert count > 0
 
                     for ws in random.sample(workers - ts.who_has, count):
-                        gathers[ws.address][ts.key] = [
-                            wws.address for wws in ts.who_has
+                        gathers[ws._address][ts.key] = [
+                            wws._address for wws in ts.who_has
                         ]
 
                 results = await asyncio.gather(
@@ -3551,8 +3684,9 @@ def workers_to_close(
         if n is None and memory_ratio is None:
             memory_ratio = 2
 
+        ws: WorkerState
         with log_errors():
-            if not n and all(ws.processing for ws in self.workers.values()):
+            if not n and all([ws._processing for ws in self.workers.values()]):
                 return []
 
             if key is None:
@@ -3565,15 +3699,16 @@ def workers_to_close(
             groups = groupby(key, self.workers.values())
 
             limit_bytes = {
-                k: sum(ws.memory_limit for ws in v) for k, v in groups.items()
+                k: sum([ws._memory_limit for ws in v]) for k, v in groups.items()
             }
-            group_bytes = {k: sum(ws.nbytes for ws in v) for k, v in groups.items()}
+            group_bytes = {k: sum([ws._nbytes for ws in v]) for k, v in groups.items()}
 
             limit = sum(limit_bytes.values())
             total = sum(group_bytes.values())
 
             def _key(group):
-                is_idle = not any(ws.processing for ws in groups[group])
+                wws: WorkerState
+                is_idle = not any([wws._processing for wws in groups[group]])
                 bytes = -group_bytes[group]
                 return (is_idle, bytes)
 
@@ -3584,7 +3719,7 @@ def _key(group):
 
             while idle:
                 group = idle.pop()
-                if n is None and any(ws.processing for ws in groups[group]):
+                if n is None and any([ws._processing for ws in groups[group]]):
                     break
 
                 if minimum and n_remain - len(groups[group]) < minimum:
@@ -3646,6 +3781,7 @@ async def retire_workers(
         --------
         Scheduler.workers_to_close
         """
+        ws: WorkerState
         with log_errors():
             async with self._lock if lock else empty_context:
                 if names is not None:
@@ -3653,9 +3789,9 @@ async def retire_workers(
                         logger.info("Retire worker names %s", names)
                     names = set(map(str, names))
                     workers = [
-                        ws.address
+                        ws._address
                         for ws in self.workers.values()
-                        if str(ws.name) in names
+                        if str(ws._name) in names
                     ]
                 if workers is None:
                     while True:
@@ -3686,7 +3822,7 @@ async def retire_workers(
                         logger.info("Moving %d keys to other workers", len(keys))
                         await self.replicate(
                             keys=keys,
-                            workers=[ws.address for ws in other_workers],
+                            workers=[ws._address for ws in other_workers],
                             n=1,
                             delete=False,
                             lock=False,
@@ -3694,7 +3830,7 @@ async def retire_workers(
                     else:
                         return []
 
-                worker_keys = {ws.address: ws.identity() for ws in workers}
+                worker_keys = {ws._address: ws.identity() for ws in workers}
                 if close_workers and worker_keys:
                     await asyncio.gather(
                         *[self.close_worker(worker=w, safe=True) for w in worker_keys]
@@ -3725,13 +3861,13 @@ def add_keys(self, comm=None, worker=None, keys=()):
         """
         if worker not in self.workers:
             return "not found"
-        ws = self.workers[worker]
+        ws: WorkerState = self.workers[worker]
         for key in keys:
             ts = self.tasks.get(key)
             if ts is not None and ts.state == "memory":
-                if ts not in ws.has_what:
-                    ws.nbytes += ts.get_nbytes()
-                    ws.has_what.add(ts)
+                if ts not in ws._has_what:
+                    ws._nbytes += ts.get_nbytes()
+                    ws._has_what.add(ts)
                     ts.who_has.add(ws)
             else:
                 self.worker_send(
@@ -3764,10 +3900,10 @@ def update_data(
                 if key in nbytes:
                     ts.set_nbytes(nbytes[key])
                 for w in workers:
-                    ws = self.workers[w]
-                    if ts not in ws.has_what:
-                        ws.nbytes += ts.get_nbytes()
-                        ws.has_what.add(ts)
+                    ws: WorkerState = self.workers[w]
+                    if ts not in ws._has_what:
+                        ws._nbytes += ts.get_nbytes()
+                        ws._has_what.add(ts)
                         ts.who_has.add(ws)
                 self.report(
                     {"op": "key-in-memory", "key": key, "workers": list(workers)}
@@ -3858,28 +3994,32 @@ def subscribe_worker_status(self, comm=None):
         return ident
 
     def get_processing(self, comm=None, workers=None):
+        ws: WorkerState
         if workers is not None:
             workers = set(map(self.coerce_address, workers))
             return {w: [ts.key for ts in self.workers[w].processing] for w in workers}
         else:
             return {
-                w: [ts.key for ts in ws.processing] for w, ws in self.workers.items()
+                w: [ts.key for ts in ws._processing] for w, ws in self.workers.items()
             }
 
     def get_who_has(self, comm=None, keys=None):
+        ws: WorkerState
         if keys is not None:
             return {
-                k: [ws.address for ws in self.tasks[k].who_has]
+                k: [ws._address for ws in self.tasks[k].who_has]
                 if k in self.tasks
                 else []
                 for k in keys
             }
         else:
             return {
-                key: [ws.address for ws in ts.who_has] for key, ts in self.tasks.items()
+                key: [ws._address for ws in ts.who_has]
+                for key, ts in self.tasks.items()
             }
 
     def get_has_what(self, comm=None, workers=None):
+        ws: WorkerState
         if workers is not None:
             workers = map(self.coerce_address, workers)
             return {
@@ -3889,14 +4029,17 @@ def get_has_what(self, comm=None, workers=None):
                 for w in workers
             }
         else:
-            return {w: [ts.key for ts in ws.has_what] for w, ws in self.workers.items()}
+            return {
+                w: [ts.key for ts in ws._has_what] for w, ws in self.workers.items()
+            }
 
     def get_ncores(self, comm=None, workers=None):
+        ws: WorkerState
         if workers is not None:
             workers = map(self.coerce_address, workers)
             return {w: self.workers[w].nthreads for w in workers if w in self.workers}
         else:
-            return {w: ws.nthreads for w, ws in self.workers.items()}
+            return {w: ws._nthreads for w, ws in self.workers.items()}
 
     async def get_call_stack(self, comm=None, keys=None):
         if keys is not None:
@@ -3945,12 +4088,14 @@ def get_nbytes(self, comm=None, keys=None, summary=True):
 
             return result
 
-    def get_comm_cost(self, ts, ws):
+    def get_comm_cost(self, ts, ws: WorkerState):
         """
         Get the estimated communication cost (in s.) to compute the task
         on the given worker.
         """
-        return sum(dts.nbytes for dts in ts.dependencies - ws.has_what) / self.bandwidth
+        return (
+            sum(dts.nbytes for dts in ts.dependencies - ws._has_what) / self.bandwidth
+        )
 
     def get_task_duration(self, ts, default=None):
         """
@@ -4055,34 +4200,34 @@ def _remove_from_processing(self, ts, send_worker_msg=None):
         """
         Remove *ts* from the set of processing tasks.
         """
-        ws = ts.processing_on
+        ws: WorkerState = ts.processing_on
         ts.processing_on = None
-        w = ws.address
+        w = ws._address
         if w in self.workers:  # may have been removed
-            duration = ws.processing.pop(ts)
-            if not ws.processing:
-                self.total_occupancy -= ws.occupancy
-                ws.occupancy = 0
+            duration = ws._processing.pop(ts)
+            if not ws._processing:
+                self.total_occupancy -= ws._occupancy
+                ws._occupancy = 0
             else:
                 self.total_occupancy -= duration
-                ws.occupancy -= duration
+                ws._occupancy -= duration
             self.check_idle_saturated(ws)
             self.release_resources(ts, ws)
             if send_worker_msg:
                 self.worker_send(w, send_worker_msg)
 
     def _add_to_memory(
-        self, ts, ws, recommendations, type=None, typename=None, **kwargs
+        self, ts, ws: WorkerState, recommendations, type=None, typename=None, **kwargs
     ):
         """
         Add *ts* to the set of in-memory tasks.
         """
         if self.validate:
-            assert ts not in ws.has_what
+            assert ts not in ws._has_what
 
         ts.who_has.add(ws)
-        ws.has_what.add(ts)
-        ws.nbytes += ts.get_nbytes()
+        ws._has_what.add(ts)
+        ws._nbytes += ts.get_nbytes()
 
         deps = ts.dependents
         if len(deps) > 1:
@@ -4264,17 +4409,17 @@ def transition_waiting_processing(self, key):
                 assert ts not in self.unrunnable
                 assert all(dts.who_has for dts in ts.dependencies)
 
-            ws = self.decide_worker(ts)
+            ws: WorkerState = self.decide_worker(ts)
             if ws is None:
                 return {}
-            worker = ws.address
+            worker = ws._address
 
             duration = self.get_task_duration(ts)
             comm = self.get_comm_cost(ts, ws)
 
-            ws.processing[ts] = duration + comm
+            ws._processing[ts] = duration + comm
             ts.processing_on = ws
-            ws.occupancy += duration + comm
+            ws._occupancy += duration + comm
             self.total_occupancy += duration + comm
             ts.state = "processing"
             self.consume_resources(ts, ws)
@@ -4282,7 +4427,7 @@ def transition_waiting_processing(self, key):
             self.n_tasks += 1
 
             if ts.actor:
-                ws.actors.add(ts)
+                ws._actors.add(ts)
 
             # logger.debug("Send job to worker: %s, %s", worker, key)
 
@@ -4299,7 +4444,7 @@ def transition_waiting_processing(self, key):
 
     def transition_waiting_memory(self, key, nbytes=None, worker=None, **kwargs):
         try:
-            ws = self.workers[worker]
+            ws: WorkerState = self.workers[worker]
             ts = self.tasks[key]
 
             if self.validate:
@@ -4342,6 +4487,8 @@ def transition_processing_memory(
         startstops=None,
         **kwargs,
     ):
+        ws: WorkerState
+        wws: WorkerState
         try:
             ts = self.tasks[key]
             assert worker
@@ -4350,7 +4497,7 @@ def transition_processing_memory(
             if self.validate:
                 assert ts.processing_on
                 ws = ts.processing_on
-                assert ts in ws.processing
+                assert ts in ws._processing
                 assert not ts.waiting_on
                 assert not ts.who_has, (ts, ts.who_has)
                 assert not ts.exception_blame
@@ -4393,7 +4540,7 @@ def transition_processing_memory(
             #############################
             # Update Timing Information #
             #############################
-            if compute_start and ws.processing.get(ts, True):
+            if compute_start and ws._processing.get(ts, True):
                 # Update average task duration for worker
                 old_duration = ts.prefix.duration_average or 0
                 new_duration = compute_stop - compute_start
@@ -4408,10 +4555,10 @@ def transition_processing_memory(
                 for tts in self.unknown_durations.pop(ts.prefix.name, ()):
                     if tts.processing_on:
                         wws = tts.processing_on
-                        old = wws.processing[tts]
+                        old = wws._processing[tts]
                         comm = self.get_comm_cost(tts, wws)
-                        wws.processing[tts] = avg_duration + comm
-                        wws.occupancy += avg_duration + comm - old
+                        wws._processing[tts] = avg_duration + comm
+                        wws._occupancy += avg_duration + comm - old
                         self.total_occupancy += avg_duration + comm - old
 
             ############################
@@ -4440,6 +4587,7 @@ def transition_processing_memory(
             raise
 
     def transition_memory_released(self, key, safe=False):
+        ws: WorkerState
         try:
             ts = self.tasks[key]
 
@@ -4451,7 +4599,7 @@ def transition_memory_released(self, key, safe=False):
 
             if ts.actor:
                 for ws in ts.who_has:
-                    ws.actors.discard(ts)
+                    ws._actors.discard(ts)
                 if ts.who_wants:
                     ts.exception_blame = ts
                     ts.exception = "Worker holding Actor was lost"
@@ -4467,11 +4615,11 @@ def transition_memory_released(self, key, safe=False):
 
             # XXX factor this out?
             for ws in ts.who_has:
-                ws.has_what.remove(ts)
-                ws.nbytes -= ts.get_nbytes()
+                ws._has_what.remove(ts)
+                ws._nbytes -= ts.get_nbytes()
                 ts.group.nbytes_in_memory -= ts.get_nbytes()
                 self.worker_send(
-                    ws.address, {"op": "delete-data", "keys": [key], "report": False}
+                    ws._address, {"op": "delete-data", "keys": [key], "report": False}
                 )
             ts.who_has.clear()
 
@@ -4656,6 +4804,7 @@ def transition_processing_released(self, key):
     def transition_processing_erred(
         self, key, cause=None, exception=None, traceback=None, **kwargs
     ):
+        ws: WorkerState
         try:
             ts = self.tasks[key]
 
@@ -4667,7 +4816,7 @@ def transition_processing_erred(
 
             if ts.actor:
                 ws = ts.processing_on
-                ws.actors.remove(ts)
+                ws._actors.remove(ts)
 
             self._remove_from_processing(ts)
 
@@ -4789,10 +4938,11 @@ def _propagate_forgotten(self, ts, recommendations):
         if ts.who_has:
             ts.group.nbytes_in_memory -= ts.get_nbytes()
 
+        ws: WorkerState
         for ws in ts.who_has:
-            ws.has_what.remove(ts)
-            ws.nbytes -= ts.get_nbytes()
-            w = ws.address
+            ws._has_what.remove(ts)
+            ws._nbytes -= ts.get_nbytes()
+            w = ws._address
             if w in self.workers:  # in case worker has died
                 self.worker_send(
                     w, {"op": "delete-data", "keys": [key], "report": False}
@@ -4800,6 +4950,7 @@ def _propagate_forgotten(self, ts, recommendations):
         ts.who_has.clear()
 
     def transition_memory_forgotten(self, key):
+        ws: WorkerState
         try:
             ts = self.tasks[key]
 
@@ -4823,7 +4974,7 @@ def transition_memory_forgotten(self, key):
 
             if ts.actor:
                 for ws in ts.who_has:
-                    ws.actors.discard(ts)
+                    ws._actors.discard(ts)
 
             self._propagate_forgotten(ts, recommendations)
 
@@ -5018,7 +5169,7 @@ def reschedule(self, key=None, worker=None):
     # Assigning Tasks to Workers #
     ##############################
 
-    def check_idle_saturated(self, ws, occ: double = -1.0):
+    def check_idle_saturated(self, ws: WorkerState, occ: double = -1.0):
         """Update the status of the idle and saturated state
 
         The scheduler keeps track of workers that are ..
@@ -5036,10 +5187,10 @@ def check_idle_saturated(self, ws, occ: double = -1.0):
         if total_nthreads == 0 or ws.status == Status.closed:
             return
         if occ < 0:
-            occ = ws.occupancy
+            occ = ws._occupancy
 
-        nc: Py_ssize_t = ws.nthreads
-        p: Py_ssize_t = len(ws.processing)
+        nc: Py_ssize_t = ws._nthreads
+        p: Py_ssize_t = len(ws._processing)
         total_occupancy: double = self.total_occupancy
         avg: double = total_occupancy / total_nthreads
 
@@ -5106,33 +5257,33 @@ def valid_workers(self, ts):
         else:
             return {self.workers[w] for w in s}
 
-    def consume_resources(self, ts, ws):
+    def consume_resources(self, ts, ws: WorkerState):
         if ts.resource_restrictions:
             for r, required in ts.resource_restrictions.items():
-                ws.used_resources[r] += required
+                ws._used_resources[r] += required
 
-    def release_resources(self, ts, ws):
+    def release_resources(self, ts, ws: WorkerState):
         if ts.resource_restrictions:
             for r, required in ts.resource_restrictions.items():
-                ws.used_resources[r] -= required
+                ws._used_resources[r] -= required
 
     #####################
     # Utility functions #
     #####################
 
     def add_resources(self, comm=None, worker=None, resources=None):
-        ws = self.workers[worker]
+        ws: WorkerState = self.workers[worker]
         if resources:
-            ws.resources.update(resources)
-        ws.used_resources = {}
-        for resource, quantity in ws.resources.items():
-            ws.used_resources[resource] = 0
+            ws._resources.update(resources)
+        ws._used_resources = {}
+        for resource, quantity in ws._resources.items():
+            ws._used_resources[resource] = 0
             self.resources[resource][worker] = quantity
         return "OK"
 
     def remove_resources(self, worker):
-        ws = self.workers[worker]
-        for resource, quantity in ws.resources.items():
+        ws: WorkerState = self.workers[worker]
+        for resource, quantity in ws._resources.items():
             del self.resources[resource][worker]
 
     def coerce_address(self, addr, resolve=True):
@@ -5197,7 +5348,7 @@ def start_ipython(self, comm=None):
             )
         return self._ipython_kernel.get_connection_info()
 
-    def worker_objective(self, ts, ws):
+    def worker_objective(self, ts, ws: WorkerState):
         """
         Objective function to determine which worker should get the task
 
@@ -5206,13 +5357,13 @@ def worker_objective(self, ts, ws):
         comm_bytes = sum(
             [dts.get_nbytes() for dts in ts.dependencies if ws not in dts.who_has]
         )
-        stack_time = ws.occupancy / ws.nthreads
+        stack_time = ws._occupancy / ws._nthreads
         start_time = comm_bytes / self.bandwidth + stack_time
 
         if ts.actor:
-            return (len(ws.actors), start_time, ws.nbytes)
+            return (len(ws._actors), start_time, ws._nbytes)
         else:
-            return (start_time, ws.nbytes)
+            return (start_time, ws._nbytes)
 
     async def get_profile(
         self,
@@ -5352,6 +5503,7 @@ def profile_to_figure(state):
         import distributed
 
         # HTML
+        ws: WorkerState
         html = """
         <h1> Dask Performance Report </h1>
 
@@ -5384,8 +5536,8 @@ def profile_to_figure(state):
             tasks_timings=tasks_timings,
             address=self.address,
             nworkers=len(self.workers),
-            threads=sum(w.nthreads for w in self.workers.values()),
-            memory=format_bytes(sum(w.memory_limit for w in self.workers.values())),
+            threads=sum(ws._nthreads for ws in self.workers.values()),
+            memory=format_bytes(sum(ws._memory_limit for ws in self.workers.values())),
             code=code,
             dask_version=dask.__version__,
             distributed_version=distributed.__version__,
@@ -5484,10 +5636,10 @@ def reevaluate_occupancy(self, worker_index=0):
             if self.proc.cpu_percent() < 50:
                 workers = list(self.workers.values())
                 for i in range(len(workers)):
-                    ws = workers[worker_index % len(workers)]
+                    ws: WorkerState = workers[worker_index % len(workers)]
                     worker_index += 1
                     try:
-                        if ws is None or not ws.processing:
+                        if ws is None or not ws._processing:
                             continue
                         self._reevaluate_occupancy_worker(ws)
                     finally:
@@ -5506,44 +5658,46 @@ def reevaluate_occupancy(self, worker_index=0):
             logger.error("Error in reevaluate occupancy", exc_info=True)
             raise
 
-    def _reevaluate_occupancy_worker(self, ws):
+    def _reevaluate_occupancy_worker(self, ws: WorkerState):
         """ See reevaluate_occupancy """
-        old = ws.occupancy
+        old = ws._occupancy
 
         new = 0
         nbytes = 0
-        for ts in ws.processing:
+        for ts in ws._processing:
             duration = self.get_task_duration(ts)
             comm = self.get_comm_cost(ts, ws)
-            ws.processing[ts] = duration + comm
+            ws._processing[ts] = duration + comm
             new += duration + comm
 
-        ws.occupancy = new
+        ws._occupancy = new
         self.total_occupancy += new - old
         self.check_idle_saturated(ws)
 
         # significant increase in duration
         if (new > old * 1.3) and ("stealing" in self.extensions):
             steal = self.extensions["stealing"]
-            for ts in ws.processing:
+            for ts in ws._processing:
                 steal.remove_key_from_stealable(ts)
                 steal.put_key_in_stealable(ts)
 
     async def check_worker_ttl(self):
+        ws: WorkerState
         now = time()
         for ws in self.workers.values():
-            if (ws.last_seen < now - self.worker_ttl) and (
-                ws.last_seen < now - 10 * heartbeat_interval(len(self.workers))
+            if (ws._last_seen < now - self.worker_ttl) and (
+                ws._last_seen < now - 10 * heartbeat_interval(len(self.workers))
             ):
                 logger.warning(
                     "Worker failed to heartbeat within %s seconds. Closing: %s",
                     self.worker_ttl,
                     ws,
                 )
-                await self.remove_worker(address=ws.address)
+                await self.remove_worker(address=ws._address)
 
     def check_idle(self):
-        if any(ws.processing for ws in self.workers.values()) or self.unrunnable:
+        ws: WorkerState
+        if any([ws._processing for ws in self.workers.values()]) or self.unrunnable:
             self.idle_since = None
             return
         elif not self.idle_since:
@@ -5582,9 +5736,10 @@ def adaptive_target(self, comm=None, target_duration=None):
         )  # TODO: threads per worker
 
         # Avoid a few long tasks from asking for many cores
+        ws: WorkerState
         tasks_processing = 0
         for ws in self.workers.values():
-            tasks_processing += len(ws.processing)
+            tasks_processing += len(ws._processing)
 
             if tasks_processing > cpu:
                 break
@@ -5595,8 +5750,8 @@ def adaptive_target(self, comm=None, target_duration=None):
             cpu = max(1, cpu)
 
         # Memory
-        limit_bytes = {addr: ws.memory_limit for addr, ws in self.workers.items()}
-        worker_bytes = [ws.nbytes for ws in self.workers.values()]
+        limit_bytes = {addr: ws._memory_limit for addr, ws in self.workers.items()}
+        worker_bytes = [ws._nbytes for ws in self.workers.values()]
         limit = sum(limit_bytes.values())
         total = sum(worker_bytes)
         if total > 0.6 * limit:
@@ -5633,6 +5788,7 @@ def decide_worker(ts, all_workers, valid_workers, objective):
     if ts.actor:
         candidates = set(all_workers)
     else:
+        ws: WorkerState
         candidates = {ws for dts in deps for ws in dts.who_has}
     if valid_workers is True:
         if not candidates:
@@ -5659,6 +5815,8 @@ def validate_task_state(ts):
     """
     Validate the given TaskState.
     """
+    ws: WorkerState
+
     assert ts.state in ALL_TASK_STATES or ts.state == "forgotten", ts
 
     if ts.waiting_on:
@@ -5729,11 +5887,11 @@ def validate_task_state(ts):
             assert isinstance(ts.type, str)
         assert not any(ts in dts.waiting_on for dts in ts.dependents)
         for ws in ts.who_has:
-            assert ts in ws.has_what, (
+            assert ts in ws._has_what, (
                 "not in who_has' has_what",
                 str(ts),
                 str(ws),
-                str(ws.has_what),
+                str(ws._has_what),
             )
 
     if ts.who_wants:
@@ -5748,13 +5906,13 @@ def validate_task_state(ts):
 
     if ts.actor:
         if ts.state == "memory":
-            assert sum([ts in ws.actors for ws in ts.who_has]) == 1
+            assert sum([ts in ws._actors for ws in ts.who_has]) == 1
         if ts.state == "processing":
             assert ts in ts.processing_on.actors
 
 
-def validate_worker_state(ws):
-    for ts in ws.has_what:
+def validate_worker_state(ws: WorkerState):
+    for ts in ws._has_what:
         assert ws in ts.who_has, (
             "not in has_what' who_has",
             str(ws),
@@ -5762,7 +5920,7 @@ def validate_worker_state(ws):
             str(ts.who_has),
         )
 
-    for ts in ws.actors:
+    for ts in ws._actors:
         assert ts.state in ("memory", "processing")
 
 
@@ -5776,6 +5934,7 @@ def validate_state(tasks, workers, clients):
     for ts in tasks.values():
         validate_task_state(ts)
 
+    ws: WorkerState
     for ws in workers.values():
         validate_worker_state(ws)
 

From 26e98049709c94c1eba3199ba32a89ebee4682d2 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 4 Dec 2020 08:05:50 -0800
Subject: [PATCH 1072/1550] Ensure that retire_workers returns a dict (#4315)

A long while ago we changed retire_workers to return a dict, but didn't
change all of the cases, particularly those for empty results, which
still returned a list.  Now we reliably return a dict in all cases.
---
 distributed/scheduler.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index fe155ccce7d..661ab484b94 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -3751,7 +3751,7 @@ async def retire_workers(
         names=None,
         lock=True,
         **kwargs,
-    ):
+    ) -> dict:
         """Gracefully retire workers from cluster
 
         Parameters
@@ -3809,7 +3809,7 @@ async def retire_workers(
                             pass
                 workers = {self.workers[w] for w in workers if w in self.workers}
                 if not workers:
-                    return []
+                    return {}
                 logger.info("Retire workers %s", workers)
 
                 # Keys orphaned by retiring those workers
@@ -3828,7 +3828,7 @@ async def retire_workers(
                             lock=False,
                         )
                     else:
-                        return []
+                        return {}
 
                 worker_keys = {ws._address: ws.identity() for ws in workers}
                 if close_workers and worker_keys:

From c89e9bad76fe5895ae85ce6f52a66a05646291c4 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Fri, 4 Dec 2020 12:42:42 -0600
Subject: [PATCH 1073/1550] Remove empty test_highgraph.py file (#4313)

---
 distributed/tests/test_highgraph.py | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 delete mode 100644 distributed/tests/test_highgraph.py

diff --git a/distributed/tests/test_highgraph.py b/distributed/tests/test_highgraph.py
deleted file mode 100644
index e69de29bb2d..00000000000

From b23f7478e08e7022ec6102082eb8b7144731526d Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Fri, 4 Dec 2020 15:59:05 -0800
Subject: [PATCH 1074/1550] Move `TaskGroup` & `TaskPrefix` before `TaskState`
 (#4318)

This makes it easier to use these definitions in `TaskState` then if
they are defined afterwards.
---
 distributed/scheduler.py | 328 +++++++++++++++++++--------------------
 1 file changed, 164 insertions(+), 164 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 661ab484b94..f2ff043f7ee 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -539,6 +539,170 @@ def ncores(self):
         return self._nthreads
 
 
+class TaskGroup:
+    """Collection tracking all tasks within a group
+
+    Keys often have a structure like ``("x-123", 0)``
+    A group takes the first section, like ``"x-123"``
+
+    .. attribute:: name: str
+
+       The name of a group of tasks.
+       For a task like ``("x-123", 0)`` this is the text ``"x-123"``
+
+    .. attribute:: states: Dict[str, int]
+
+       The number of tasks in each state,
+       like ``{"memory": 10, "processing": 3, "released": 4, ...}``
+
+    .. attribute:: dependencies: Set[TaskGroup]
+
+       The other TaskGroups on which this one depends
+
+    .. attribute:: nbytes_total: int
+
+       The total number of bytes that this task group has produced
+
+    .. attribute:: nbytes_in_memory: int
+
+       The number of bytes currently stored by this TaskGroup
+
+    .. attribute:: duration: float
+
+       The total amount of time spent on all tasks in this TaskGroup
+
+    .. attribute:: types: Set[str]
+
+       The result types of this TaskGroup
+
+    See also
+    --------
+    TaskPrefix
+    """
+
+    def __init__(self, name):
+        self.name = name
+        self.states = {state: 0 for state in ALL_TASK_STATES}
+        self.states["forgotten"] = 0
+        self.dependencies = set()
+        self.nbytes_total = 0
+        self.nbytes_in_memory = 0
+        self.duration = 0
+        self.types = set()
+
+    def add(self, ts):
+        self.states[ts.state] += 1
+        ts.group = self
+
+    def __repr__(self):
+        return (
+            "<"
+            + (self.name or "no-group")
+            + ": "
+            + ", ".join(
+                "%s: %d" % (k, v) for (k, v) in sorted(self.states.items()) if v
+            )
+            + ">"
+        )
+
+    def __len__(self):
+        return sum(self.states.values())
+
+
+class TaskPrefix:
+    """Collection tracking all tasks within a group
+
+    Keys often have a structure like ``("x-123", 0)``
+    A group takes the first section, like ``"x"``
+
+    .. attribute:: name: str
+
+       The name of a group of tasks.
+       For a task like ``("x-123", 0)`` this is the text ``"x"``
+
+    .. attribute:: states: Dict[str, int]
+
+       The number of tasks in each state,
+       like ``{"memory": 10, "processing": 3, "released": 4, ...}``
+
+    .. attribute:: duration_average: float
+
+       An exponentially weighted moving average duration of all tasks with this prefix
+
+    .. attribute:: suspicious: int
+
+       Numbers of times a task was marked as suspicious with this prefix
+
+
+    See Also
+    --------
+    TaskGroup
+    """
+
+    def __init__(self, name):
+        self.name = name
+        self.groups = []
+
+        # store timings for each prefix-action
+        self.all_durations = defaultdict(float)
+
+        if self.name in dask.config.get("distributed.scheduler.default-task-durations"):
+            self.duration_average = parse_timedelta(
+                dask.config.get("distributed.scheduler.default-task-durations")[
+                    self.name
+                ]
+            )
+        else:
+            self.duration_average = None
+        self.suspicious = 0
+
+    @property
+    def states(self):
+        return merge_with(sum, [g.states for g in self.groups])
+
+    @property
+    def active(self):
+        return [
+            g
+            for g in self.groups
+            if any(v != 0 for k, v in g.states.items() if k != "forgotten")
+        ]
+
+    @property
+    def active_states(self):
+        return merge_with(sum, [g.states for g in self.active])
+
+    def __repr__(self):
+        return (
+            "<"
+            + self.name
+            + ": "
+            + ", ".join(
+                "%s: %d" % (k, v) for (k, v) in sorted(self.states.items()) if v
+            )
+            + ">"
+        )
+
+    @property
+    def nbytes_in_memory(self):
+        return sum(tg.nbytes_in_memory for tg in self.groups)
+
+    @property
+    def nbytes_total(self):
+        return sum(tg.nbytes_total for tg in self.groups)
+
+    def __len__(self):
+        return sum(map(len, self.groups))
+
+    @property
+    def duration(self):
+        return sum(tg.duration for tg in self.groups)
+
+    @property
+    def types(self):
+        return set().union(*[tg.types for tg in self.groups])
+
+
 class TaskState:
     """
     A simple object holding information about a task.
@@ -920,170 +1084,6 @@ def validate(self):
                 pdb.set_trace()
 
 
-class TaskGroup:
-    """Collection tracking all tasks within a group
-
-    Keys often have a structure like ``("x-123", 0)``
-    A group takes the first section, like ``"x-123"``
-
-    .. attribute:: name: str
-
-       The name of a group of tasks.
-       For a task like ``("x-123", 0)`` this is the text ``"x-123"``
-
-    .. attribute:: states: Dict[str, int]
-
-       The number of tasks in each state,
-       like ``{"memory": 10, "processing": 3, "released": 4, ...}``
-
-    .. attribute:: dependencies: Set[TaskGroup]
-
-       The other TaskGroups on which this one depends
-
-    .. attribute:: nbytes_total: int
-
-       The total number of bytes that this task group has produced
-
-    .. attribute:: nbytes_in_memory: int
-
-       The number of bytes currently stored by this TaskGroup
-
-    .. attribute:: duration: float
-
-       The total amount of time spent on all tasks in this TaskGroup
-
-    .. attribute:: types: Set[str]
-
-       The result types of this TaskGroup
-
-    See also
-    --------
-    TaskPrefix
-    """
-
-    def __init__(self, name):
-        self.name = name
-        self.states = {state: 0 for state in ALL_TASK_STATES}
-        self.states["forgotten"] = 0
-        self.dependencies = set()
-        self.nbytes_total = 0
-        self.nbytes_in_memory = 0
-        self.duration = 0
-        self.types = set()
-
-    def add(self, ts):
-        self.states[ts.state] += 1
-        ts.group = self
-
-    def __repr__(self):
-        return (
-            "<"
-            + (self.name or "no-group")
-            + ": "
-            + ", ".join(
-                "%s: %d" % (k, v) for (k, v) in sorted(self.states.items()) if v
-            )
-            + ">"
-        )
-
-    def __len__(self):
-        return sum(self.states.values())
-
-
-class TaskPrefix:
-    """Collection tracking all tasks within a group
-
-    Keys often have a structure like ``("x-123", 0)``
-    A group takes the first section, like ``"x"``
-
-    .. attribute:: name: str
-
-       The name of a group of tasks.
-       For a task like ``("x-123", 0)`` this is the text ``"x"``
-
-    .. attribute:: states: Dict[str, int]
-
-       The number of tasks in each state,
-       like ``{"memory": 10, "processing": 3, "released": 4, ...}``
-
-    .. attribute:: duration_average: float
-
-       An exponentially weighted moving average duration of all tasks with this prefix
-
-    .. attribute:: suspicious: int
-
-       Numbers of times a task was marked as suspicious with this prefix
-
-
-    See Also
-    --------
-    TaskGroup
-    """
-
-    def __init__(self, name):
-        self.name = name
-        self.groups = []
-
-        # store timings for each prefix-action
-        self.all_durations = defaultdict(float)
-
-        if self.name in dask.config.get("distributed.scheduler.default-task-durations"):
-            self.duration_average = parse_timedelta(
-                dask.config.get("distributed.scheduler.default-task-durations")[
-                    self.name
-                ]
-            )
-        else:
-            self.duration_average = None
-        self.suspicious = 0
-
-    @property
-    def states(self):
-        return merge_with(sum, [g.states for g in self.groups])
-
-    @property
-    def active(self):
-        return [
-            g
-            for g in self.groups
-            if any(v != 0 for k, v in g.states.items() if k != "forgotten")
-        ]
-
-    @property
-    def active_states(self):
-        return merge_with(sum, [g.states for g in self.active])
-
-    def __repr__(self):
-        return (
-            "<"
-            + self.name
-            + ": "
-            + ", ".join(
-                "%s: %d" % (k, v) for (k, v) in sorted(self.states.items()) if v
-            )
-            + ">"
-        )
-
-    @property
-    def nbytes_in_memory(self):
-        return sum(tg.nbytes_in_memory for tg in self.groups)
-
-    @property
-    def nbytes_total(self):
-        return sum(tg.nbytes_total for tg in self.groups)
-
-    def __len__(self):
-        return sum(map(len, self.groups))
-
-    @property
-    def duration(self):
-        return sum(tg.duration for tg in self.groups)
-
-    @property
-    def types(self):
-        return set().union(*[tg.types for tg in self.groups])
-
-
 class _StateLegacyMapping(Mapping):
     """
     A mapping interface mimicking the former Scheduler state dictionaries.

From c32f157d46562ba66568c3cda36f4546906591e6 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Fri, 4 Dec 2020 16:49:47 -0800
Subject: [PATCH 1075/1550] Check traceback object can be unpickled (#4299)

Just as we check if the exception can be round-tripped through pickle,
check the traceback can as well. If that does, work proceed to fallback
handling as before for the traceback.
---
 distributed/core.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/distributed/core.py b/distributed/core.py
index efb2f7a0ca5..3eedef8faab 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -1147,6 +1147,7 @@ def error_message(e, status="error"):
     e4 = protocol.to_serialize(e2)
     try:
         tb2 = protocol.pickle.dumps(tb, protocol=4)
+        protocol.pickle.loads(tb2)
     except Exception:
         tb = tb2 = "".join(traceback.format_tb(tb))
 

From 95ccaf8284a3faebbd609c1bbb93780317e45186 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Fri, 4 Dec 2020 17:43:52 -0800
Subject: [PATCH 1076/1550] Swap order of `TaskGroup` and `TaskPrefix` (#4319)

---
 distributed/scheduler.py | 140 +++++++++++++++++++--------------------
 1 file changed, 70 insertions(+), 70 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index f2ff043f7ee..cf82573d2c5 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -539,76 +539,6 @@ def ncores(self):
         return self._nthreads
 
 
-class TaskGroup:
-    """Collection tracking all tasks within a group
-
-    Keys often have a structure like ``("x-123", 0)``
-    A group takes the first section, like ``"x-123"``
-
-    .. attribute:: name: str
-
-       The name of a group of tasks.
-       For a task like ``("x-123", 0)`` this is the text ``"x-123"``
-
-    .. attribute:: states: Dict[str, int]
-
-       The number of tasks in each state,
-       like ``{"memory": 10, "processing": 3, "released": 4, ...}``
-
-    .. attribute:: dependencies: Set[TaskGroup]
-
-       The other TaskGroups on which this one depends
-
-    .. attribute:: nbytes_total: int
-
-       The total number of bytes that this task group has produced
-
-    .. attribute:: nbytes_in_memory: int
-
-       The number of bytes currently stored by this TaskGroup
-
-    .. attribute:: duration: float
-
-       The total amount of time spent on all tasks in this TaskGroup
-
-    .. attribute:: types: Set[str]
-
-       The result types of this TaskGroup
-
-    See also
-    --------
-    TaskPrefix
-    """
-
-    def __init__(self, name):
-        self.name = name
-        self.states = {state: 0 for state in ALL_TASK_STATES}
-        self.states["forgotten"] = 0
-        self.dependencies = set()
-        self.nbytes_total = 0
-        self.nbytes_in_memory = 0
-        self.duration = 0
-        self.types = set()
-
-    def add(self, ts):
-        self.states[ts.state] += 1
-        ts.group = self
-
-    def __repr__(self):
-        return (
-            "<"
-            + (self.name or "no-group")
-            + ": "
-            + ", ".join(
-                "%s: %d" % (k, v) for (k, v) in sorted(self.states.items()) if v
-            )
-            + ">"
-        )
-
-    def __len__(self):
-        return sum(self.states.values())
-
-
 class TaskPrefix:
     """Collection tracking all tasks within a group
 
@@ -703,6 +633,76 @@ def types(self):
         return set().union(*[tg.types for tg in self.groups])
 
 
+class TaskGroup:
+    """Collection tracking all tasks within a group
+
+    Keys often have a structure like ``("x-123", 0)``
+    A group takes the first section, like ``"x-123"``
+
+    .. attribute:: name: str
+
+       The name of a group of tasks.
+       For a task like ``("x-123", 0)`` this is the text ``"x-123"``
+
+    .. attribute:: states: Dict[str, int]
+
+       The number of tasks in each state,
+       like ``{"memory": 10, "processing": 3, "released": 4, ...}``
+
+    .. attribute:: dependencies: Set[TaskGroup]
+
+       The other TaskGroups on which this one depends
+
+    .. attribute:: nbytes_total: int
+
+       The total number of bytes that this task group has produced
+
+    .. attribute:: nbytes_in_memory: int
+
+       The number of bytes currently stored by this TaskGroup
+
+    .. attribute:: duration: float
+
+       The total amount of time spent on all tasks in this TaskGroup
+
+    .. attribute:: types: Set[str]
+
+       The result types of this TaskGroup
+
+    See also
+    --------
+    TaskPrefix
+    """
+
+    def __init__(self, name):
+        self.name = name
+        self.states = {state: 0 for state in ALL_TASK_STATES}
+        self.states["forgotten"] = 0
+        self.dependencies = set()
+        self.nbytes_total = 0
+        self.nbytes_in_memory = 0
+        self.duration = 0
+        self.types = set()
+
+    def add(self, ts):
+        self.states[ts.state] += 1
+        ts.group = self
+
+    def __repr__(self):
+        return (
+            "<"
+            + (self.name or "no-group")
+            + ": "
+            + ", ".join(
+                "%s: %d" % (k, v) for (k, v) in sorted(self.states.items()) if v
+            )
+            + ">"
+        )
+
+    def __len__(self):
+        return sum(self.states.values())
+
+
 class TaskState:
     """
     A simple object holding information about a task.

From 82557d735fcccdb46963bef2c0438aef8be85ae5 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Sat, 5 Dec 2020 07:49:32 -0800
Subject: [PATCH 1077/1550] Optimize `WorkerState.__eq__` (#4320)

Make sure we are using the C-level attribute for comparison. Also cast
`other` to `WorkerState` to perform the comparison.
---
 distributed/scheduler.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index cf82573d2c5..cce448c9a58 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -379,7 +379,13 @@ def __hash__(self):
         return self._hash
 
     def __eq__(self, other):
-        return type(self) == type(other) and self.address == other.address
+        typ_self: type = type(self)
+        typ_other: type = type(other)
+        if typ_self == typ_other:
+            other_ws: WorkerState = other
+            return self._address == other_ws._address
+        else:
+            return False
 
     @property
     def actors(self):

From 6641feabb69dcefee0875895b0b7400ead5c945a Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Sat, 5 Dec 2020 07:49:54 -0800
Subject: [PATCH 1078/1550] Some Cython fixes for `WorkerState` (#4321)

* Use `0` for `memory_limit` instead of `None`

We need an integer to pass to `WorkerState` so set this to `0` instead
of `None`.

* Use `list` comprehension to bind `WorkerState`

As Cython may not type things based on outer scope variables when using
a generator, convert the generator to a `list` comprehension, which does
respect the type of external variables used within the loop. This should
fix some attribute access errors seen here.
---
 distributed/scheduler.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index cce448c9a58..46a38c1ac8f 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1894,7 +1894,7 @@ async def add_worker(
         now=None,
         resources=None,
         host_info=None,
-        memory_limit=None,
+        memory_limit=0,
         metrics=None,
         pid=0,
         services=None,
@@ -3561,10 +3561,10 @@ async def replicate(
                             del_worker_tasks[ws].add(ts)
 
                 await asyncio.gather(
-                    *(
+                    *[
                         self._delete_worker_data(ws._address, [t.key for t in tasks])
                         for ws, tasks in del_worker_tasks.items()
-                    )
+                    ]
                 )
 
             # Copy not-yet-filled data
@@ -5542,8 +5542,10 @@ def profile_to_figure(state):
             tasks_timings=tasks_timings,
             address=self.address,
             nworkers=len(self.workers),
-            threads=sum(ws._nthreads for ws in self.workers.values()),
-            memory=format_bytes(sum(ws._memory_limit for ws in self.workers.values())),
+            threads=sum([ws._nthreads for ws in self.workers.values()]),
+            memory=format_bytes(
+                sum([ws._memory_limit for ws in self.workers.values()])
+            ),
             code=code,
             dask_version=dask.__version__,
             distributed_version=distributed.__version__,

From b39fddca5489fc0fac2e85922a2cf797f8ac57c7 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Mon, 7 Dec 2020 10:53:49 -0800
Subject: [PATCH 1079/1550] Ensure `retire_workers` always `return`s a `dict`
 (#4323)

As `workers_to_close` actually `return`s a `list` and that `list` could
be empty, we can wind up `return`ing an empty `list` instead of
`return`ing a `dict`. To fix that, we only return the `workers` object
when it is non-trivial (so will be replaced by a `dict`). In the trivial
case we just `return` an empty `dict`. This ensures we only `return` a
`dict` in this case, which fixes an issue that was cropping up when
compiling the scheduler with Cython.
---
 distributed/scheduler.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 46a38c1ac8f..86b76516e9d 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -3810,7 +3810,9 @@ async def retire_workers(
                                     close_workers=close_workers,
                                     lock=False,
                                 )
-                            return workers
+                                return workers
+                            else:
+                                return {}
                         except KeyError:  # keys left during replicate
                             pass
                 workers = {self.workers[w] for w in workers if w in self.workers}

From 0d4a4993d0e1c949c5c8c1b27884e168e4b58b5f Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Mon, 7 Dec 2020 12:20:53 -0800
Subject: [PATCH 1080/1550] Annotate `Task*` objects for Cythonization (#4302)

* Use `tsp` variable name for `TaskStreamPlugin`s

* Use `dts` for iterated `TaskState` variables

* Create `list` from generator

This ensures Cython still uses `TaskState` to annotate the variable
iterated over. Otherwise it constructs a generator with its own scope
where this is ignored.

* Use `-1` as `TaskState.nbytes` default

Ideally we would want to type `nbytes` as some kind of integer for
Cython optimization. However as we assign and work with `nbytes` being
`None` in some cases, this is not currently possible since a C integer
cannot be `None`. To fix that change the default value of `nbytes` to be
`-1`. This way it is still a valid integer and one we would never use
for `nbytes` unless it wasn't set. Thus it remains easy to check for and
won't overlap. Also it is something a C integer can be assigned.

* Assign `TaskState` instances to variables

* Annotate `TaskState` for Cythonization

* Annotate all `TaskState` variables

* Use closure to access `TaskState.priority`

This way we are able to inform Cython of the type and it is able to do
the work necessary to retrieve it. We will wind up reverting this later.

* Add `_` before all `TaskState` attributes

* Use `_` prefixed `TaskState` attributes throughout

* Add Python-level `property`s for attributes

* Add some `property.setter`s

Includes `setter`s for `processing_on` and `nbytes`.

* Drop recently added `TaskState.priority` closures

Now that we have properties accessible from `TaskState`, drop the
closures we added previously to access the typed values internally. This
should be equivalently performant and cut out a little bit of
boilerplate.

* Swap assignment order with `TaskPrefix`

Make sure to assign to the `TaskPrefix` variable, `tp`, first before
assigning to the `dict`. This should avoid the admittedly likely low
overhead of looking up the result in the dictionary when we already have
the value available.

* Assign `dask.config` query to a variable

Saves us need to fetch this twice. Also makes the code a bit more
readable. Finally may allow Cython optimizations on the variable later.

* Set `TaskPrefix.duration_average` to `-1`

Instead of using `None` for `TaskPrefix.duration_average`, set it `-1`.
This works better when typing `TaskPrefix.duration_average` as it can
always be floating point. This also works logically with this value as
it can't actually be negative unless it wasn't defined. Rework the logic
around this variable to ensure it is positive semi-definite.

* Annotate `TaskPrefix` for Cythonization

* Annotate `TaskPrefix` constructor as well

* Annotate all `TaskPrefix` variables

* Add `_` before all `TaskPrefix` attributes

* Add Python-level `property`s for attributes

* Use `Py_ssize_t` in `set_nbytes`

This allows Cython to perform C-level optimizations on these variables
and usages thereof.

* Use `tg` for `TaskGroup` variables

* Swap assignment order with `TaskGroup`

Make sure to assign to the `TaskGroup` variable, `tg`, first before
assigning to the `dict`. This should avoid the admittedly likely low
overhead of looking up the result in the dictionary when we already have
the value available.

* Create `list` from generator

This ensures Cython still uses `TaskGroup` to annotate the variable
iterated over. Otherwise it constructs a generator with its own scope
where this is ignored.

* Annotate `TaskGroup` for Cythonization

* Annotate `TaskGroup` constructor as well

* Set `TaskGroup.prefix` to `None` initially

* Annotate all `TaskGroup` variables

* Add `_` before all `TaskGroup` attributes

* Add Python-level `property`s for attributes
---
 distributed/diagnostics/progress.py |   16 +-
 distributed/scheduler.py            | 1523 ++++++++++++++++-----------
 2 files changed, 922 insertions(+), 617 deletions(-)

diff --git a/distributed/diagnostics/progress.py b/distributed/diagnostics/progress.py
index eaad51a2747..eeb3c8a2817 100644
--- a/distributed/diagnostics/progress.py
+++ b/distributed/diagnostics/progress.py
@@ -250,7 +250,7 @@ def __init__(self, scheduler):
             prefix = ts.prefix.name
             self.all[prefix].add(key)
             self.state[ts.state][prefix].add(key)
-            if ts.nbytes is not None:
+            if ts.nbytes >= 0:
                 self.nbytes[prefix] += ts.nbytes
 
         scheduler.add_plugin(self)
@@ -264,11 +264,11 @@ def transition(self, key, start, finish, *args, **kwargs):
         except KeyError:  # TODO: remove me once we have a new or clean state
             pass
 
-        if start == "memory":
+        if start == "memory" and ts.nbytes >= 0:
             # XXX why not respect DEFAULT_DATA_SIZE?
-            self.nbytes[prefix] -= ts.nbytes or 0
-        if finish == "memory":
-            self.nbytes[prefix] += ts.nbytes or 0
+            self.nbytes[prefix] -= ts.nbytes
+        if finish == "memory" and ts.nbytes >= 0:
+            self.nbytes[prefix] += ts.nbytes
 
         if finish != "forgotten":
             self.state[finish][prefix].add(key)
@@ -304,7 +304,7 @@ def __init__(self, scheduler):
                 self.create(key, k)
             self.keys[k].add(key)
             self.groups[k][ts.state] += 1
-            if ts.state == "memory" and ts.nbytes is not None:
+            if ts.state == "memory" and ts.nbytes >= 0:
                 self.nbytes[k] += ts.nbytes
 
         scheduler.add_plugin(self)
@@ -347,9 +347,9 @@ def transition(self, key, start, finish, *args, **kwargs):
                     for dep in self.dependencies.pop(k):
                         self.dependents[key_split_group(dep)].remove(k)
 
-            if start == "memory" and ts.nbytes is not None:
+            if start == "memory" and ts.nbytes >= 0:
                 self.nbytes[k] -= ts.nbytes
-            if finish == "memory" and ts.nbytes is not None:
+            if finish == "memory" and ts.nbytes >= 0:
                 self.nbytes[k] += ts.nbytes
 
     def restart(self, scheduler):
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 86b76516e9d..b61aa5f07a7 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -511,7 +511,8 @@ def clean(self):
             nanny=self._nanny,
             extra=self._extra,
         )
-        ws._processing = {ts.key: cost for ts, cost in self._processing.items()}
+        ts: TaskState
+        ws._processing = {ts._key: cost for ts, cost in self._processing.items()}
         return ws
 
     def __repr__(self):
@@ -545,6 +546,7 @@ def ncores(self):
         return self._nthreads
 
 
+@cclass
 class TaskPrefix:
     """Collection tracking all tasks within a group
 
@@ -575,43 +577,68 @@ class TaskPrefix:
     TaskGroup
     """
 
-    def __init__(self, name):
-        self.name = name
-        self.groups = []
+    _name: str
+    _all_durations: object
+    _duration_average: double
+    _suspicious: Py_ssize_t
+    _groups: list
+
+    def __init__(self, name: str):
+        self._name = name
+        self._groups = []
 
         # store timings for each prefix-action
-        self.all_durations = defaultdict(float)
+        self._all_durations = defaultdict(float)
 
-        if self.name in dask.config.get("distributed.scheduler.default-task-durations"):
-            self.duration_average = parse_timedelta(
-                dask.config.get("distributed.scheduler.default-task-durations")[
-                    self.name
-                ]
-            )
+        task_durations = dask.config.get("distributed.scheduler.default-task-durations")
+        if self._name in task_durations:
+            self._duration_average = parse_timedelta(task_durations[self._name])
         else:
-            self.duration_average = None
-        self.suspicious = 0
+            self._duration_average = -1
+        self._suspicious = 0
+
+    @property
+    def name(self):
+        return self._name
+
+    @property
+    def all_durations(self):
+        return self._all_durations
+
+    @property
+    def duration_average(self):
+        return self._duration_average
+
+    @property
+    def suspicious(self):
+        return self._suspicious
+
+    @property
+    def groups(self):
+        return self._groups
 
     @property
     def states(self):
-        return merge_with(sum, [g.states for g in self.groups])
+        tg: TaskGroup
+        return merge_with(sum, [tg._states for tg in self._groups])
 
     @property
     def active(self):
+        tg: TaskGroup
         return [
-            g
-            for g in self.groups
-            if any(v != 0 for k, v in g.states.items() if k != "forgotten")
+            tg
+            for tg in self._groups
+            if any(v != 0 for k, v in tg._states.items() if k != "forgotten")
         ]
 
     @property
     def active_states(self):
-        return merge_with(sum, [g.states for g in self.active])
+        return merge_with(sum, [tg._states for tg in self.active])
 
     def __repr__(self):
         return (
             "<"
-            + self.name
+            + self._name
             + ": "
             + ", ".join(
                 "%s: %d" % (k, v) for (k, v) in sorted(self.states.items()) if v
@@ -621,24 +648,29 @@ def __repr__(self):
 
     @property
     def nbytes_in_memory(self):
-        return sum(tg.nbytes_in_memory for tg in self.groups)
+        tg: TaskGroup
+        return sum([tg._nbytes_in_memory for tg in self._groups])
 
     @property
     def nbytes_total(self):
-        return sum(tg.nbytes_total for tg in self.groups)
+        tg: TaskGroup
+        return sum([tg._nbytes_total for tg in self._groups])
 
     def __len__(self):
-        return sum(map(len, self.groups))
+        return sum(map(len, self._groups))
 
     @property
     def duration(self):
-        return sum(tg.duration for tg in self.groups)
+        tg: TaskGroup
+        return sum([tg._duration for tg in self._groups])
 
     @property
     def types(self):
-        return set().union(*[tg.types for tg in self.groups])
+        tg: TaskGroup
+        return set().union(*[tg._types for tg in self._groups])
 
 
+@cclass
 class TaskGroup:
     """Collection tracking all tasks within a group
 
@@ -680,35 +712,79 @@ class TaskGroup:
     TaskPrefix
     """
 
-    def __init__(self, name):
-        self.name = name
-        self.states = {state: 0 for state in ALL_TASK_STATES}
-        self.states["forgotten"] = 0
-        self.dependencies = set()
-        self.nbytes_total = 0
-        self.nbytes_in_memory = 0
-        self.duration = 0
-        self.types = set()
-
-    def add(self, ts):
-        self.states[ts.state] += 1
-        ts.group = self
+    _name: str
+    _prefix: TaskPrefix
+    _states: dict
+    _dependencies: set
+    _nbytes_total: Py_ssize_t
+    _nbytes_in_memory: Py_ssize_t
+    _duration: double
+    _types: set
+
+    def __init__(self, name: str):
+        self._name = name
+        self._prefix = None
+        self._states = {state: 0 for state in ALL_TASK_STATES}
+        self._states["forgotten"] = 0
+        self._dependencies = set()
+        self._nbytes_total = 0
+        self._nbytes_in_memory = 0
+        self._duration = 0
+        self._types = set()
+
+    @property
+    def name(self):
+        return self._name
+
+    @property
+    def prefix(self):
+        return self._prefix
+
+    @property
+    def states(self):
+        return self._states
+
+    @property
+    def dependencies(self):
+        return self._dependencies
+
+    @property
+    def nbytes_total(self):
+        return self._nbytes_total
+
+    @property
+    def nbytes_in_memory(self):
+        return self._nbytes_in_memory
+
+    @property
+    def duration(self):
+        return self._duration
+
+    @property
+    def types(self):
+        return self._types
+
+    def add(self, o):
+        ts: TaskState = o
+        self._states[ts.state] += 1
+        ts._group = self
 
     def __repr__(self):
         return (
             "<"
-            + (self.name or "no-group")
+            + (self._name or "no-group")
             + ": "
             + ", ".join(
-                "%s: %d" % (k, v) for (k, v) in sorted(self.states.items()) if v
+                "%s: %d" % (k, v) for (k, v) in sorted(self._states.items()) if v
             )
             + ">"
         )
 
     def __len__(self):
-        return sum(self.states.values())
+        return sum(self._states.values())
 
 
+@cclass
 class TaskState:
     """
     A simple object holding information about a task.
@@ -950,137 +1026,296 @@ class TaskState:
         Task annotations
     """
 
+    _key: str
+    _hash: Py_hash_t
+    _prefix: TaskPrefix
+    _run_spec: object
+    _priority: tuple
+    _state: str
+    _dependencies: set
+    _dependents: set
+    _has_lost_dependencies: bool
+    _waiting_on: set
+    _waiters: set
+    _who_wants: set
+    _who_has: set
+    _processing_on: WorkerState
+    _retries: Py_ssize_t
+    _nbytes: Py_ssize_t
+    _type: str
+    _exception: object
+    _traceback: object
+    _exception_blame: object
+    _suspicious: Py_ssize_t
+    _host_restrictions: set
+    _worker_restrictions: set
+    _resource_restrictions: dict
+    _loose_restrictions: bool
+    _metadata: dict
+    _annotations: dict
+    _actor: bool
+    _group: TaskGroup
+    _group_key: str
+
     __slots__ = (
         # === General description ===
-        "actor",
+        "_actor",
         # Key name
-        "key",
+        "_key",
         # Hash of the key name
         "_hash",
         # Key prefix (see key_split())
-        "prefix",
+        "_prefix",
         # How to run the task (None if pure data)
-        "run_spec",
+        "_run_spec",
         # Alive dependents and dependencies
-        "dependencies",
-        "dependents",
+        "_dependencies",
+        "_dependents",
         # Compute priority
-        "priority",
+        "_priority",
         # Restrictions
-        "host_restrictions",
-        "worker_restrictions",  # not WorkerStates but addresses
-        "resource_restrictions",
-        "loose_restrictions",
+        "_host_restrictions",
+        "_worker_restrictions",  # not WorkerStates but addresses
+        "_resource_restrictions",
+        "_loose_restrictions",
         # === Task state ===
         "_state",
         # Whether some dependencies were forgotten
-        "has_lost_dependencies",
+        "_has_lost_dependencies",
         # If in 'waiting' state, which tasks need to complete
         # before we can run
-        "waiting_on",
+        "_waiting_on",
         # If in 'waiting' or 'processing' state, which tasks needs us
         # to complete before they can run
-        "waiters",
+        "_waiters",
         # In in 'processing' state, which worker we are processing on
-        "processing_on",
+        "_processing_on",
         # If in 'memory' state, Which workers have us
-        "who_has",
+        "_who_has",
         # Which clients want us
-        "who_wants",
-        "exception",
-        "traceback",
-        "exception_blame",
-        "suspicious",
-        "retries",
-        "nbytes",
-        "type",
-        "group_key",
-        "group",
-        "metadata",
-        "annotations",
+        "_who_wants",
+        "_exception",
+        "_traceback",
+        "_exception_blame",
+        "_suspicious",
+        "_retries",
+        "_nbytes",
+        "_type",
+        "_group_key",
+        "_group",
+        "_metadata",
+        "_annotations",
     )
 
-    def __init__(self, key, run_spec):
-        self.key = key
+    def __init__(self, key: str, run_spec: object):
+        self._key = key
         self._hash = hash(key)
-        self.run_spec = run_spec
+        self._run_spec = run_spec
         self._state = None
-        self.exception = self.traceback = self.exception_blame = None
-        self.suspicious = self.retries = 0
-        self.nbytes = None
-        self.priority = None
-        self.who_wants = set()
-        self.dependencies = set()
-        self.dependents = set()
-        self.waiting_on = set()
-        self.waiters = set()
-        self.who_has = set()
-        self.processing_on = None
-        self.has_lost_dependencies = False
-        self.host_restrictions = None
-        self.worker_restrictions = None
-        self.resource_restrictions = None
-        self.loose_restrictions = False
-        self.actor = None
-        self.type = None
-        self.group_key = key_split_group(key)
-        self.group = None
-        self.metadata = {}
-        self.annotations = {}
+        self._exception = self._traceback = self._exception_blame = None
+        self._suspicious = self._retries = 0
+        self._nbytes = -1
+        self._priority = None
+        self._who_wants = set()
+        self._dependencies = set()
+        self._dependents = set()
+        self._waiting_on = set()
+        self._waiters = set()
+        self._who_has = set()
+        self._processing_on = None
+        self._has_lost_dependencies = False
+        self._host_restrictions = None
+        self._worker_restrictions = None
+        self._resource_restrictions = None
+        self._loose_restrictions = False
+        self._actor = None
+        self._type = None
+        self._group_key = key_split_group(key)
+        self._group = None
+        self._metadata = {}
+        self._annotations = {}
 
     def __hash__(self):
         return self._hash
 
     def __eq__(self, other):
-        return type(self) == type(other) and self.key == other.key
+        typ_self: type = type(self)
+        typ_other: type = type(other)
+        if typ_self == typ_other:
+            other_ts: TaskState = other
+            return self._key == other_ts._key
+        else:
+            return False
 
     @property
-    def state(self) -> str:
-        return self._state
+    def key(self):
+        return self._key
 
     @property
-    def prefix_key(self):
-        return self.prefix.name
+    def prefix(self):
+        return self._prefix
+
+    @property
+    def run_spec(self):
+        return self._run_spec
+
+    @property
+    def priority(self):
+        return self._priority
+
+    @property
+    def state(self) -> str:
+        return self._state
 
     @state.setter
     def state(self, value: str):
-        self.group.states[self._state] -= 1
-        self.group.states[value] += 1
+        self._group._states[self._state] -= 1
+        self._group._states[value] += 1
         self._state = value
 
+    @property
+    def dependencies(self):
+        return self._dependencies
+
+    @property
+    def dependents(self):
+        return self._dependents
+
+    @property
+    def has_lost_dependencies(self):
+        return self._has_lost_dependencies
+
+    @property
+    def waiting_on(self):
+        return self._waiting_on
+
+    @property
+    def waiters(self):
+        return self._waiters
+
+    @property
+    def who_wants(self):
+        return self._who_wants
+
+    @property
+    def who_has(self):
+        return self._who_has
+
+    @property
+    def processing_on(self):
+        return self._processing_on
+
+    @processing_on.setter
+    def processing_on(self, v: WorkerState):
+        self._processing_on = v
+
+    @property
+    def retries(self):
+        return self._retries
+
+    @property
+    def nbytes(self):
+        return self._nbytes
+
+    @nbytes.setter
+    def nbytes(self, v: Py_ssize_t):
+        self._nbytes = v
+
+    @property
+    def type(self):
+        return self._type
+
+    @property
+    def exception(self):
+        return self._exception
+
+    @property
+    def traceback(self):
+        return self._traceback
+
+    @property
+    def exception_blame(self):
+        return self._exception_blame
+
+    @property
+    def suspicious(self):
+        return self._suspicious
+
+    @property
+    def host_restrictions(self):
+        return self._host_restrictions
+
+    @property
+    def worker_restrictions(self):
+        return self._worker_restrictions
+
+    @property
+    def resource_restrictions(self):
+        return self._resource_restrictions
+
+    @property
+    def loose_restrictions(self):
+        return self._loose_restrictions
+
+    @property
+    def metadata(self):
+        return self._metadata
+
+    @property
+    def annotations(self):
+        return self._annotations
+
+    @property
+    def actor(self):
+        return self._actor
+
+    @property
+    def group(self):
+        return self._group
+
+    @property
+    def group_key(self):
+        return self._group_key
+
+    @property
+    def prefix_key(self):
+        return self._prefix._name
+
     def add_dependency(self, other: "TaskState"):
         """ Add another task as a dependency of this task """
-        self.dependencies.add(other)
-        self.group.dependencies.add(other.group)
-        other.dependents.add(self)
+        self._dependencies.add(other)
+        self._group._dependencies.add(other._group)
+        other._dependents.add(self)
 
     def get_nbytes(self) -> int:
-        nbytes = self.nbytes
-        return nbytes if nbytes is not None else DEFAULT_DATA_SIZE
-
-    def set_nbytes(self, nbytes: int):
-        old_nbytes = self.nbytes
-        diff = nbytes - (old_nbytes or 0)
-        self.group.nbytes_total += diff
-        self.group.nbytes_in_memory += diff
+        nbytes = self._nbytes
+        return nbytes if nbytes >= 0 else DEFAULT_DATA_SIZE
+
+    def set_nbytes(self, nbytes: Py_ssize_t):
+        diff: Py_ssize_t = nbytes
+        old_nbytes: Py_ssize_t = self._nbytes
+        if old_nbytes >= 0:
+            diff -= old_nbytes
+        self._group._nbytes_total += diff
+        self._group._nbytes_in_memory += diff
         ws: WorkerState
-        for ws in self.who_has:
+        for ws in self._who_has:
             ws._nbytes += diff
-        self.nbytes = nbytes
+        self._nbytes = nbytes
 
     def __repr__(self):
-        return "<Task %r %s>" % (self.key, self.state)
+        return "<Task %r %s>" % (self._key, self._state)
 
     def validate(self):
         try:
-            for cs in self.who_wants:
-                assert isinstance(cs, ClientState), (repr(cs), self.who_wants)
-            for ws in self.who_has:
-                assert isinstance(ws, WorkerState), (repr(ws), self.who_has)
-            for ts in self.dependencies:
-                assert isinstance(ts, TaskState), (repr(ts), self.dependencies)
-            for ts in self.dependents:
-                assert isinstance(ts, TaskState), (repr(ts), self.dependents)
+            for cs in self._who_wants:
+                assert isinstance(cs, ClientState), (repr(cs), self._who_wants)
+            for ws in self._who_has:
+                assert isinstance(ws, WorkerState), (repr(ws), self._who_has)
+            for ts in self._dependencies:
+                assert isinstance(ts, TaskState), (repr(ts), self._dependencies)
+            for ts in self._dependents:
+                assert isinstance(ts, TaskState), (repr(ts), self._dependents)
             validate_task_state(self)
         except Exception as e:
             logger.exception(e)
@@ -1168,7 +1403,8 @@ def _legacy_task_key_set(tasks):
     """
     Transform a set of task states into a set of task keys.
     """
-    return {ts.key for ts in tasks}
+    ts: TaskState
+    return {ts._key for ts in tasks}
 
 
 def _legacy_client_key_set(clients):
@@ -1191,7 +1427,8 @@ def _legacy_task_key_dict(task_dict):
     """
     Transform a dict of {task state: value} into a dict of {task key: value}.
     """
-    return {ts.key: value for ts, value in task_dict.items()}
+    ts: TaskState
+    return {ts._key: value for ts, value in task_dict.items()}
 
 
 def _task_key_or_none(task):
@@ -1975,7 +2212,7 @@ async def add_worker(
 
             if nbytes:
                 for key in nbytes:
-                    ts = self.tasks.get(key)
+                    ts: TaskState = self.tasks.get(key)
                     if ts is not None and ts.state in ("processing", "waiting"):
                         recommendations = self.transition(
                             key,
@@ -1990,7 +2227,7 @@ async def add_worker(
             for ts in list(self.unrunnable):
                 valid = self.valid_workers(ts)
                 if valid is True or ws in valid:
-                    recommendations[ts.key] = "waiting"
+                    recommendations[ts._key] = "waiting"
 
             if recommendations:
                 self.transitions(recommendations)
@@ -2128,11 +2365,15 @@ def update_graph(
                     self.client_releases_keys(keys=[k], client=client)
 
         # Avoid computation that is already finished
+        ts: TaskState
         already_in_memory = set()  # tasks that are already done
         for k, v in dependencies.items():
-            if v and k in self.tasks and self.tasks[k].state in ("memory", "erred"):
-                already_in_memory.add(k)
+            if v and k in self.tasks:
+                ts = self.tasks[k]
+                if ts.state in ("memory", "erred"):
+                    already_in_memory.add(k)
 
+        dts: TaskState
         if already_in_memory:
             dependents = dask.core.reverse_dict(dependencies)
             stack = list(already_in_memory)
@@ -2170,8 +2411,8 @@ def update_graph(
             ts = self.tasks.get(k)
             if ts is None:
                 ts = self.new_task(k, tasks.get(k), "released")
-            elif not ts.run_spec:
-                ts.run_spec = tasks.get(k)
+            elif not ts._run_spec:
+                ts._run_spec = tasks.get(k)
 
             touched_keys.add(k)
             touched_tasks.append(ts)
@@ -2182,7 +2423,7 @@ def update_graph(
         # Add dependencies
         for key, deps in dependencies.items():
             ts = self.tasks.get(key)
-            if ts is None or ts.dependencies:
+            if ts is None or ts._dependencies:
                 continue
             for dep in deps:
                 dts = self.tasks[dep]
@@ -2219,13 +2460,15 @@ def update_graph(
 
             for a, kv in annotations.items():
                 for k, v in kv.items():
-                    self.tasks[k].annotations[a] = v
+                    ts = self.tasks[k]
+                    ts._annotations[a] = v
 
         # Add actors
         if actors is True:
             actors = list(keys)
         for actor in actors or []:
-            self.tasks[actor].actor = True
+            ts = self.tasks[actor]
+            ts._actor = True
 
         priority = priority or dask.order.order(
             tasks
@@ -2234,7 +2477,7 @@ def update_graph(
         if submitting_task:  # sub-tasks get better priority than parent tasks
             ts = self.tasks.get(submitting_task)
             if ts is not None:
-                generation = ts.priority[0] - 0.01
+                generation = ts._priority[0] - 0.01
             else:  # super-task already cleaned up
                 generation = self.generation
         elif self._last_time + fifo_timeout < start:
@@ -2246,14 +2489,14 @@ def update_graph(
 
         for key in set(priority) & touched_keys:
             ts = self.tasks[key]
-            if ts.priority is None:
-                ts.priority = (-(user_priority.get(key, 0)), generation, priority[key])
+            if ts._priority is None:
+                ts._priority = (-(user_priority.get(key, 0)), generation, priority[key])
 
         # Ensure all runnables have a priority
-        runnables = [ts for ts in touched_tasks if ts.run_spec]
+        runnables = [ts for ts in touched_tasks if ts._run_spec]
         for ts in runnables:
-            if ts.priority is None and ts.run_spec:
-                ts.priority = (self.generation, 0)
+            if ts._priority is None and ts._run_spec:
+                ts._priority = (self.generation, 0)
 
         if restrictions:
             # *restrictions* is a dict keying task ids to lists of
@@ -2264,21 +2507,21 @@ def update_graph(
                 ts = self.tasks.get(k)
                 if ts is None:
                     continue
-                ts.host_restrictions = set()
-                ts.worker_restrictions = set()
+                ts._host_restrictions = set()
+                ts._worker_restrictions = set()
                 for w in v:
                     try:
                         w = self.coerce_address(w)
                     except ValueError:
                         # Not a valid address, but perhaps it's a hostname
-                        ts.host_restrictions.add(w)
+                        ts._host_restrictions.add(w)
                     else:
-                        ts.worker_restrictions.add(w)
+                        ts._worker_restrictions.add(w)
 
             if loose_restrictions:
                 for k in loose_restrictions:
                     ts = self.tasks[k]
-                    ts.loose_restrictions = True
+                    ts._loose_restrictions = True
 
         if resources:
             for k, v in resources.items():
@@ -2288,7 +2531,7 @@ def update_graph(
                 ts = self.tasks.get(k)
                 if ts is None:
                     continue
-                ts.resource_restrictions = v
+                ts._resource_restrictions = v
 
         if retries:
             for k, v in retries.items():
@@ -2296,20 +2539,20 @@ def update_graph(
                 ts = self.tasks.get(k)
                 if ts is None:
                     continue
-                ts.retries = v
+                ts._retries = v
 
         # Compute recommendations
         recommendations = {}
 
         for ts in sorted(runnables, key=operator.attrgetter("priority"), reverse=True):
-            if ts.state == "released" and ts.run_spec:
-                recommendations[ts.key] = "waiting"
+            if ts.state == "released" and ts._run_spec:
+                recommendations[ts._key] = "waiting"
 
         for ts in touched_tasks:
-            for dts in ts.dependencies:
-                if dts.exception_blame:
-                    ts.exception_blame = dts.exception_blame
-                    recommendations[ts.key] = "erred"
+            for dts in ts._dependencies:
+                if dts._exception_blame:
+                    ts._exception_blame = dts._exception_blame
+                    recommendations[ts._key] = "erred"
                     break
 
         for plugin in self.plugins[:]:
@@ -2333,7 +2576,7 @@ def update_graph(
 
         for ts in touched_tasks:
             if ts.state in ("memory", "erred"):
-                self.report_on_key(ts.key, client=client)
+                self.report_on_key(ts._key, client=client)
 
         end = time()
         if self.digests is not None:
@@ -2343,22 +2586,24 @@ def update_graph(
 
     def new_task(self, key, spec, state):
         """ Create a new task, and associated states """
-        ts = TaskState(key, spec)
+        ts: TaskState = TaskState(key, spec)
+        tp: TaskPrefix
+        tg: TaskGroup
         ts._state = state
         prefix_key = key_split(key)
         try:
             tp = self.task_prefixes[prefix_key]
         except KeyError:
-            tp = self.task_prefixes[prefix_key] = TaskPrefix(prefix_key)
-        ts.prefix = tp
+            self.task_prefixes[prefix_key] = tp = TaskPrefix(prefix_key)
+        ts._prefix = tp
 
-        group_key = ts.group_key
+        group_key = ts._group_key
         try:
             tg = self.task_groups[group_key]
         except KeyError:
-            tg = self.task_groups[group_key] = TaskGroup(group_key)
-            tg.prefix = tp
-            tp.groups.append(tg)
+            self.task_groups[group_key] = tg = TaskGroup(group_key)
+            tg._prefix = tp
+            tp._groups.append(tg)
         tg.add(ts)
         self.tasks[key] = ts
         return ts
@@ -2367,17 +2612,17 @@ def stimulus_task_finished(self, key=None, worker=None, **kwargs):
         """ Mark that a task has finished execution on a particular worker """
         logger.debug("Stimulus task finished %s, %s", key, worker)
 
-        ts = self.tasks.get(key)
+        ts: TaskState = self.tasks.get(key)
         if ts is None:
             return {}
         ws: WorkerState = self.workers[worker]
-        ts.metadata.update(kwargs["metadata"])
+        ts._metadata.update(kwargs["metadata"])
 
         if ts.state == "processing":
             recommendations = self.transition(key, "memory", worker=worker, **kwargs)
 
             if ts.state == "memory":
-                assert ws in ts.who_has
+                assert ws in ts._who_has
         else:
             logger.debug(
                 "Received already computed task, worker: %s, state: %s"
@@ -2385,9 +2630,9 @@ def stimulus_task_finished(self, key=None, worker=None, **kwargs):
                 worker,
                 ts.state,
                 key,
-                ts.who_has,
+                ts._who_has,
             )
-            if ws not in ts.who_has:
+            if ws not in ts._who_has:
                 self.worker_send(worker, {"op": "release-task", "key": key})
             recommendations = {}
 
@@ -2399,14 +2644,14 @@ def stimulus_task_erred(
         """ Mark that a task has erred on a particular worker """
         logger.debug("Stimulus task erred %s, %s", key, worker)
 
-        ts = self.tasks.get(key)
+        ts: TaskState = self.tasks.get(key)
         if ts is None:
             return {}
 
         if ts.state == "processing":
-            retries = ts.retries
+            retries = ts._retries
             if retries > 0:
-                ts.retries = retries - 1
+                ts._retries = retries - 1
                 recommendations = self.transition(key, "waiting")
             else:
                 recommendations = self.transition(
@@ -2430,19 +2675,19 @@ def stimulus_missing_data(
         with log_errors():
             logger.debug("Stimulus missing data %s, %s", key, worker)
 
-            ts = self.tasks.get(key)
+            ts: TaskState = self.tasks.get(key)
             if ts is None or ts.state == "memory":
                 return {}
-            cts = self.tasks.get(cause)
+            cts: TaskState = self.tasks.get(cause)
 
             recommendations = {}
 
             if cts is not None and cts.state == "memory":  # couldn't find this
                 ws: WorkerState
-                for ws in cts.who_has:  # TODO: this behavior is extreme
+                for ws in cts._who_has:  # TODO: this behavior is extreme
                     ws._has_what.remove(cts)
                     ws._nbytes -= cts.get_nbytes()
-                cts.who_has.clear()
+                cts._who_has.clear()
                 recommendations[cause] = "released"
 
             if key:
@@ -2463,12 +2708,13 @@ def stimulus_retry(self, comm=None, keys=None, client=None):
         stack = list(keys)
         seen = set()
         roots = []
+        ts: TaskState
+        dts: TaskState
         while stack:
             key = stack.pop()
             seen.add(key)
-            erred_deps = [
-                dts.key for dts in self.tasks[key].dependencies if dts.state == "erred"
-            ]
+            ts = self.tasks[key]
+            erred_deps = [dts._key for dts in ts._dependencies if dts.state == "erred"]
             if erred_deps:
                 stack.extend(erred_deps)
             else:
@@ -2537,13 +2783,14 @@ async def remove_worker(self, comm=None, address=None, safe=False, close=True):
 
             recommendations = {}
 
+            ts: TaskState
             for ts in list(ws._processing):
-                k = ts.key
+                k = ts._key
                 recommendations[k] = "released"
                 if not safe:
-                    ts.suspicious += 1
-                    ts.prefix.suspicious += 1
-                    if ts.suspicious > self.allowed_failures:
+                    ts._suspicious += 1
+                    ts._prefix._suspicious += 1
+                    if ts._suspicious > self.allowed_failures:
                         del recommendations[k]
                         e = pickle.dumps(
                             KilledWorker(task=k, last_worker=ws.clean()), protocol=4
@@ -2553,17 +2800,17 @@ async def remove_worker(self, comm=None, address=None, safe=False, close=True):
                         logger.info(
                             "Task %s marked as failed because %d workers died"
                             " while trying to run it",
-                            ts.key,
+                            ts._key,
                             self.allowed_failures,
                         )
 
             for ts in ws._has_what:
-                ts.who_has.remove(ws)
-                if not ts.who_has:
-                    if ts.run_spec:
-                        recommendations[ts.key] = "released"
+                ts._who_has.remove(ws)
+                if not ts._who_has:
+                    if ts._run_spec:
+                        recommendations[ts._key] = "released"
                     else:  # pure data
-                        recommendations[ts.key] = "forgotten"
+                        recommendations[ts._key] = "forgotten"
             ws._has_what.clear()
 
             self.transitions(recommendations)
@@ -2609,23 +2856,24 @@ def stimulus_cancel(self, comm, keys=None, client=None, force=False):
     def cancel_key(self, key, client, retries=5, force=False):
         """ Cancel a particular key and all dependents """
         # TODO: this should be converted to use the transition mechanism
-        ts = self.tasks.get(key)
+        ts: TaskState = self.tasks.get(key)
+        dts: TaskState
         try:
             cs: ClientState = self.clients[client]
         except KeyError:
             return
-        if ts is None or not ts.who_wants:  # no key yet, lets try again in a moment
+        if ts is None or not ts._who_wants:  # no key yet, lets try again in a moment
             if retries:
                 self.loop.call_later(
                     0.2, lambda: self.cancel_key(key, client, retries - 1)
                 )
             return
-        if force or ts.who_wants == {cs}:  # no one else wants this key
-            for dts in list(ts.dependents):
-                self.cancel_key(dts.key, client, force=force)
+        if force or ts._who_wants == {cs}:  # no one else wants this key
+            for dts in list(ts._dependents):
+                self.cancel_key(dts._key, client, force=force)
         logger.info("Scheduler cancels key %s.  Force=%s", key, force)
         self.report({"op": "cancelled-key", "key": key})
-        clients = list(ts.who_wants) if force else [cs]
+        clients = list(ts._who_wants) if force else [cs]
         for cs in clients:
             self.client_releases_keys(keys=[key], client=cs._client_key)
 
@@ -2634,12 +2882,13 @@ def client_desires_keys(self, keys=None, client=None):
         if cs is None:
             # For publish, queues etc.
             self.clients[client] = cs = ClientState(client)
+        ts: TaskState
         for k in keys:
             ts = self.tasks.get(k)
             if ts is None:
                 # For publish, queues etc.
                 ts = self.new_task(k, None, "released")
-            ts.who_wants.add(cs)
+            ts._who_wants.add(cs)
             cs._wants_what.add(ts)
 
             if ts.state in ("memory", "erred"):
@@ -2649,23 +2898,24 @@ def client_releases_keys(self, keys=None, client=None):
         """ Remove keys from client desired list """
         logger.debug("Client %s releases keys: %s", client, keys)
         cs: ClientState = self.clients[client]
+        ts: TaskState
         tasks2 = set()
         for key in list(keys):
             ts = self.tasks.get(key)
             if ts is not None and ts in cs._wants_what:
                 cs._wants_what.remove(ts)
-                s = ts.who_wants
+                s = ts._who_wants
                 s.remove(cs)
                 if not s:
                     tasks2.add(ts)
 
         recommendations = {}
         for ts in tasks2:
-            if not ts.dependents:
+            if not ts._dependents:
                 # No live dependents, can forget
-                recommendations[ts.key] = "forgotten"
-            elif ts.state != "erred" and not ts.waiters:
-                recommendations[ts.key] = "released"
+                recommendations[ts._key] = "forgotten"
+            elif ts.state != "erred" and not ts._waiters:
+                recommendations[ts._key] = "released"
 
         self.transitions(recommendations)
 
@@ -2679,63 +2929,68 @@ def client_heartbeat(self, client=None):
     ###################
 
     def validate_released(self, key):
-        ts = self.tasks[key]
+        ts: TaskState = self.tasks[key]
+        dts: TaskState
         assert ts.state == "released"
-        assert not ts.waiters
-        assert not ts.waiting_on
-        assert not ts.who_has
-        assert not ts.processing_on
-        assert not any(ts in dts.waiters for dts in ts.dependencies)
+        assert not ts._waiters
+        assert not ts._waiting_on
+        assert not ts._who_has
+        assert not ts._processing_on
+        assert not any([ts in dts._waiters for dts in ts._dependencies])
         assert ts not in self.unrunnable
 
     def validate_waiting(self, key):
-        ts = self.tasks[key]
-        assert ts.waiting_on
-        assert not ts.who_has
-        assert not ts.processing_on
+        ts: TaskState = self.tasks[key]
+        dts: TaskState
+        assert ts._waiting_on
+        assert not ts._who_has
+        assert not ts._processing_on
         assert ts not in self.unrunnable
-        for dts in ts.dependencies:
+        for dts in ts._dependencies:
             # We are waiting on a dependency iff it's not stored
-            assert bool(dts.who_has) + (dts in ts.waiting_on) == 1
-            assert ts in dts.waiters  # XXX even if dts.who_has?
+            assert bool(dts._who_has) + (dts in ts._waiting_on) == 1
+            assert ts in dts._waiters  # XXX even if dts._who_has?
 
     def validate_processing(self, key):
-        ts = self.tasks[key]
-        assert not ts.waiting_on
-        ws: WorkerState = ts.processing_on
+        ts: TaskState = self.tasks[key]
+        dts: TaskState
+        assert not ts._waiting_on
+        ws: WorkerState = ts._processing_on
         assert ws
         assert ts in ws._processing
-        assert not ts.who_has
-        for dts in ts.dependencies:
-            assert dts.who_has
-            assert ts in dts.waiters
+        assert not ts._who_has
+        for dts in ts._dependencies:
+            assert dts._who_has
+            assert ts in dts._waiters
 
     def validate_memory(self, key):
-        ts = self.tasks[key]
-        assert ts.who_has
-        assert not ts.processing_on
-        assert not ts.waiting_on
+        ts: TaskState = self.tasks[key]
+        dts: TaskState
+        assert ts._who_has
+        assert not ts._processing_on
+        assert not ts._waiting_on
         assert ts not in self.unrunnable
-        for dts in ts.dependents:
-            assert (dts in ts.waiters) == (dts.state in ("waiting", "processing"))
-            assert ts not in dts.waiting_on
+        for dts in ts._dependents:
+            assert (dts in ts._waiters) == (dts.state in ("waiting", "processing"))
+            assert ts not in dts._waiting_on
 
     def validate_no_worker(self, key):
-        ts = self.tasks[key]
+        ts: TaskState = self.tasks[key]
+        dts: TaskState
         assert ts in self.unrunnable
-        assert not ts.waiting_on
+        assert not ts._waiting_on
         assert ts in self.unrunnable
-        assert not ts.processing_on
-        assert not ts.who_has
-        for dts in ts.dependencies:
-            assert dts.who_has
+        assert not ts._processing_on
+        assert not ts._who_has
+        for dts in ts._dependencies:
+            assert dts._who_has
 
     def validate_erred(self, key):
-        ts = self.tasks[key]
-        assert ts.exception_blame
-        assert not ts.who_has
+        ts: TaskState = self.tasks[key]
+        assert ts._exception_blame
+        assert not ts._who_has
 
-    def validate_key(self, key, ts=None):
+    def validate_key(self, key, ts: TaskState = None):
         try:
             if ts is None:
                 ts = self.tasks.get(key)
@@ -2774,9 +3029,10 @@ def validate_state(self, allow_overlap=False):
                 assert not ws._occupancy
                 assert ws in self.idle
 
+        ts: TaskState
         for k, ts in self.tasks.items():
             assert isinstance(ts, TaskState), (type(ts), ts)
-            assert ts.key == k
+            assert ts._key == k
             self.validate_key(k, ts)
 
         c: str
@@ -2808,7 +3064,7 @@ def validate_state(self, allow_overlap=False):
     # Manage Messages #
     ###################
 
-    def report(self, msg, ts=None, client=None):
+    def report(self, msg, ts: TaskState = None, client=None):
         """
         Publish updates to all listening Queues and Comms
 
@@ -2824,11 +3080,11 @@ def report(self, msg, ts=None, client=None):
             client_keys = list(self.client_comms)
         elif client is None:
             # Notify clients interested in key
-            client_keys = [cs._client_key for cs in ts.who_wants]
+            client_keys = [cs._client_key for cs in ts._who_wants]
         else:
             # Notify clients interested in key (including `client`)
             client_keys = [
-                cs._client_key for cs in ts.who_wants if cs._client_key != client
+                cs._client_key for cs in ts._who_wants if cs._client_key != client
             ]
             client_keys.append(client)
 
@@ -2903,8 +3159,9 @@ def remove_client(self, client=None):
             # XXX is this a legitimate condition?
             pass
         else:
+            ts: TaskState
             self.client_releases_keys(
-                keys=[ts.key for ts in cs._wants_what], client=cs._client_key
+                keys=[ts._key for ts in cs._wants_what], client=cs._client_key
             )
             del self.clients[client]
 
@@ -2927,31 +3184,32 @@ def remove_client_from_events():
     def send_task_to_worker(self, worker, key):
         """ Send a single computational task to a worker """
         try:
-            ts = self.tasks[key]
+            ts: TaskState = self.tasks[key]
+            dts: TaskState
 
             msg = {
                 "op": "compute-task",
                 "key": key,
-                "priority": ts.priority,
+                "priority": ts._priority,
                 "duration": self.get_task_duration(ts),
             }
-            if ts.resource_restrictions:
-                msg["resource_restrictions"] = ts.resource_restrictions
-            if ts.actor:
+            if ts._resource_restrictions:
+                msg["resource_restrictions"] = ts._resource_restrictions
+            if ts._actor:
                 msg["actor"] = True
 
-            deps = ts.dependencies
+            deps = ts._dependencies
             if deps:
                 ws: WorkerState
                 msg["who_has"] = {
-                    dep.key: [ws._address for ws in dep.who_has] for dep in deps
+                    dts._key: [ws._address for ws in dts._who_has] for dts in deps
                 }
-                msg["nbytes"] = {dep.key: dep.nbytes for dep in deps}
+                msg["nbytes"] = {dts._key: dts._nbytes for dts in deps}
 
             if self.validate and deps:
                 assert all(msg["who_has"].values())
 
-            task = ts.run_spec
+            task = ts._run_spec
             if type(task) is dict:
                 msg.update(task)
             else:
@@ -2981,11 +3239,11 @@ def handle_task_erred(self, key=None, **msg):
         self.transitions(r)
 
     def handle_release_data(self, key=None, worker=None, client=None, **msg):
-        ts = self.tasks.get(key)
+        ts: TaskState = self.tasks.get(key)
         if ts is None:
             return
         ws: WorkerState = self.workers[worker]
-        if ts.processing_on != ws:
+        if ts._processing_on != ws:
             return
         r = self.stimulus_missing_data(key=key, ensure=False, **msg)
         self.transitions(r)
@@ -2994,17 +3252,17 @@ def handle_missing_data(self, key=None, errant_worker=None, **kwargs):
         logger.debug("handle missing data key=%s worker=%s", key, errant_worker)
         self.log.append(("missing", key, errant_worker))
 
-        ts = self.tasks.get(key)
-        if ts is None or not ts.who_has:
+        ts: TaskState = self.tasks.get(key)
+        if ts is None or not ts._who_has:
             return
         if errant_worker in self.workers:
             ws: WorkerState = self.workers[errant_worker]
-            if ws in ts.who_has:
-                ts.who_has.remove(ws)
+            if ws in ts._who_has:
+                ts._who_has.remove(ws)
                 ws._has_what.remove(ts)
                 ws._nbytes -= ts.get_nbytes()
-        if not ts.who_has:
-            if ts.run_spec:
+        if not ts._who_has:
+            if ts._run_spec:
                 self.transitions({key: "released"})
             else:
                 self.transitions({key: "forgotten"})
@@ -3015,13 +3273,14 @@ def release_worker_data(self, comm=None, keys=None, worker=None):
         removed_tasks = tasks & ws._has_what
         ws._has_what -= removed_tasks
 
+        ts: TaskState
         recommendations = {}
         for ts in removed_tasks:
             ws._nbytes -= ts.get_nbytes()
-            wh = ts.who_has
+            wh = ts._who_has
             wh.remove(ws)
             if not wh:
-                recommendations[ts.key] = "released"
+                recommendations[ts._key] = "released"
         if recommendations:
             self.transitions(recommendations)
 
@@ -3031,24 +3290,24 @@ def handle_long_running(self, key=None, worker=None, compute_duration=None):
         We stop the task from being stolen in the future, and change task
         duration accounting as if the task has stopped.
         """
-        ts = self.tasks[key]
+        ts: TaskState = self.tasks[key]
         if "stealing" in self.extensions:
             self.extensions["stealing"].remove_key_from_stealable(ts)
 
-        ws: WorkerState = ts.processing_on
+        ws: WorkerState = ts._processing_on
         if ws is None:
             logger.debug("Received long-running signal from duplicate task. Ignoring.")
             return
 
         if compute_duration:
-            old_duration = ts.prefix.duration_average or 0
+            old_duration = ts._prefix._duration_average
             new_duration = compute_duration
-            if not old_duration:
+            if old_duration < 0:
                 avg_duration = new_duration
             else:
                 avg_duration = 0.5 * old_duration + 0.5 * new_duration
 
-            ts.prefix.duration_average = avg_duration
+            ts._prefix._duration_average = avg_duration
 
         ws._occupancy -= ws._processing[ts]
         self.total_occupancy -= ws._processing[ts]
@@ -3163,9 +3422,9 @@ async def gather(self, comm=None, keys=None, serializers=None):
         keys = list(keys)
         who_has = {}
         for key in keys:
-            ts = self.tasks.get(key)
+            ts: TaskState = self.tasks.get(key)
             if ts is not None:
-                who_has[key] = [ws._address for ws in ts.who_has]
+                who_has[key] = [ws._address for ws in ts._who_has]
             else:
                 who_has[key] = []
 
@@ -3198,7 +3457,7 @@ async def gather(self, comm=None, keys=None, serializers=None):
                 for key, workers in missing_keys.items():
                     # Task may already be gone if it was held by a
                     # `missing_worker`
-                    ts = self.tasks.get(key)
+                    ts: TaskState = self.tasks.get(key)
                     logger.exception(
                         "Workers don't have promised key: %s, %s",
                         str(workers),
@@ -3210,7 +3469,7 @@ async def gather(self, comm=None, keys=None, serializers=None):
                         ws = self.workers.get(worker)
                         if ws is not None and ts in ws._has_what:
                             ws._has_what.remove(ts)
-                            ts.who_has.remove(ws)
+                            ts._who_has.remove(ws)
                             ws._nbytes -= ts.get_nbytes()
                             self.transitions({key: "released"})
 
@@ -3232,9 +3491,10 @@ async def restart(self, client=None, timeout=3):
 
             logger.info("Send lost future signal to clients")
             cs: ClientState
+            ts: TaskState
             for cs in self.clients.values():
                 self.client_releases_keys(
-                    keys=[ts.key for ts in cs._wants_what], client=cs._client_key
+                    keys=[ts._key for ts in cs._wants_what], client=cs._client_key
                 )
 
             ws: WorkerState
@@ -3365,10 +3625,11 @@ async def _delete_worker_data(self, worker_address, keys):
         )
 
         ws: WorkerState = self.workers[worker_address]
+        ts: TaskState
         tasks = {self.tasks[key] for key in keys}
         ws._has_what -= tasks
         for ts in tasks:
-            ts.who_has.remove(ws)
+            ts._who_has.remove(ws)
             ws._nbytes -= ts.get_nbytes()
         self.log_event(ws._address, {"action": "remove-worker-data", "keys": keys})
 
@@ -3383,11 +3644,12 @@ async def rebalance(self, comm=None, keys=None, workers=None):
         occupied worker until either the sender or the recipient are at the
         average expected load.
         """
+        ts: TaskState
         with log_errors():
             async with self._lock:
                 if keys:
                     tasks = {self.tasks[k] for k in keys}
-                    missing_data = [ts.key for ts in tasks if not ts.who_has]
+                    missing_data = [ts._key for ts in tasks if not ts._who_has]
                     if missing_data:
                         return {"status": "missing-data", "keys": missing_data}
                 else:
@@ -3395,10 +3657,10 @@ async def rebalance(self, comm=None, keys=None, workers=None):
 
                 if workers:
                     workers = {self.workers[w] for w in workers}
-                    workers_by_task = {ts: ts.who_has & workers for ts in tasks}
+                    workers_by_task = {ts: ts._who_has & workers for ts in tasks}
                 else:
                     workers = set(self.workers.values())
-                    workers_by_task = {ts: ts.who_has for ts in tasks}
+                    workers_by_task = {ts: ts._who_has for ts in tasks}
 
                 ws: WorkerState
                 tasks_by_worker = {ws: set() for ws in workers}
@@ -3450,8 +3712,8 @@ async def rebalance(self, comm=None, keys=None, workers=None):
                 to_recipients = defaultdict(lambda: defaultdict(list))
                 to_senders = defaultdict(list)
                 for sender, recipient, ts in msgs:
-                    to_recipients[recipient.address][ts.key].append(sender.address)
-                    to_senders[sender.address].append(ts.key)
+                    to_recipients[recipient.address][ts._key].append(sender.address)
+                    to_senders[sender.address].append(ts._key)
 
                 result = await asyncio.gather(
                     *(
@@ -3487,11 +3749,17 @@ async def rebalance(self, comm=None, keys=None, workers=None):
 
                 for sender, recipient, ts in msgs:
                     assert ts.state == "memory"
-                    ts.who_has.add(recipient)
+                    ts._who_has.add(recipient)
                     recipient.has_what.add(ts)
                     recipient.nbytes += ts.get_nbytes()
                     self.log.append(
-                        ("rebalance", ts.key, time(), sender.address, recipient.address)
+                        (
+                            "rebalance",
+                            ts._key,
+                            time(),
+                            sender.address,
+                            recipient.address,
+                        )
                     )
 
                 await asyncio.gather(
@@ -3533,6 +3801,7 @@ async def replicate(
         """
         ws: WorkerState
         wws: WorkerState
+        ts: TaskState
 
         assert branching_factor > 0
         async with self._lock if lock else empty_context:
@@ -3545,7 +3814,7 @@ async def replicate(
                 raise ValueError("Can not use replicate to delete data")
 
             tasks = {self.tasks[k] for k in keys}
-            missing_data = [ts.key for ts in tasks if not ts.who_has]
+            missing_data = [ts._key for ts in tasks if not ts._who_has]
             if missing_data:
                 return {"status": "missing-data", "keys": missing_data}
 
@@ -3553,7 +3822,7 @@ async def replicate(
             if delete:
                 del_worker_tasks = defaultdict(set)
                 for ts in tasks:
-                    del_candidates = ts.who_has & workers
+                    del_candidates = ts._who_has & workers
                     if len(del_candidates) > n:
                         for ws in random.sample(
                             del_candidates, len(del_candidates) - n
@@ -3575,18 +3844,18 @@ async def replicate(
                         # task is no longer needed by any client or dependant task
                         tasks.remove(ts)
                         continue
-                    n_missing = n - len(ts.who_has & workers)
+                    n_missing = n - len(ts._who_has & workers)
                     if n_missing <= 0:
                         # Already replicated enough
                         tasks.remove(ts)
                         continue
 
-                    count = min(n_missing, branching_factor * len(ts.who_has))
+                    count = min(n_missing, branching_factor * len(ts._who_has))
                     assert count > 0
 
-                    for ws in random.sample(workers - ts.who_has, count):
-                        gathers[ws._address][ts.key] = [
-                            wws._address for wws in ts.who_has
+                    for ws in random.sample(workers - ts._who_has, count):
+                        gathers[ws._address][ts._key] = [
+                            wws._address for wws in ts._who_has
                         ]
 
                 results = await asyncio.gather(
@@ -3788,6 +4057,7 @@ async def retire_workers(
         Scheduler.workers_to_close
         """
         ws: WorkerState
+        ts: TaskState
         with log_errors():
             async with self._lock if lock else empty_context:
                 if names is not None:
@@ -3822,7 +4092,7 @@ async def retire_workers(
 
                 # Keys orphaned by retiring those workers
                 keys = set.union(*[w.has_what for w in workers])
-                keys = {ts.key for ts in keys if ts.who_has.issubset(workers)}
+                keys = {ts._key for ts in keys if ts._who_has.issubset(workers)}
 
                 other_workers = set(self.workers.values()) - workers
                 if keys:
@@ -3871,12 +4141,12 @@ def add_keys(self, comm=None, worker=None, keys=()):
             return "not found"
         ws: WorkerState = self.workers[worker]
         for key in keys:
-            ts = self.tasks.get(key)
+            ts: TaskState = self.tasks.get(key)
             if ts is not None and ts.state == "memory":
                 if ts not in ws._has_what:
                     ws._nbytes += ts.get_nbytes()
                     ws._has_what.add(ts)
-                    ts.who_has.add(ws)
+                    ts._who_has.add(ws)
             else:
                 self.worker_send(
                     worker, {"op": "delete-data", "keys": [key], "report": False}
@@ -3901,9 +4171,9 @@ def update_data(
             logger.debug("Update data %s", who_has)
 
             for key, workers in who_has.items():
-                ts = self.tasks.get(key)
+                ts: TaskState = self.tasks.get(key)
                 if ts is None:
-                    ts = self.new_task(key, None, "memory")
+                    ts: TaskState = self.new_task(key, None, "memory")
                 ts.state = "memory"
                 if key in nbytes:
                     ts.set_nbytes(nbytes[key])
@@ -3912,7 +4182,7 @@ def update_data(
                     if ts not in ws._has_what:
                         ws._nbytes += ts.get_nbytes()
                         ws._has_what.add(ts)
-                        ts.who_has.add(ws)
+                        ts._who_has.add(ws)
                 self.report(
                     {"op": "key-in-memory", "key": key, "workers": list(workers)}
                 )
@@ -3920,7 +4190,7 @@ def update_data(
             if client:
                 self.client_desires_keys(keys=list(who_has), client=client)
 
-    def report_on_key(self, key=None, ts=None, client=None):
+    def report_on_key(self, key=None, ts: TaskState = None, client=None):
         assert (key is None) + (ts is None) == 1, (key, ts)
         if ts is None:
             try:
@@ -3929,19 +4199,19 @@ def report_on_key(self, key=None, ts=None, client=None):
                 self.report({"op": "cancelled-key", "key": key}, client=client)
                 return
         else:
-            key = ts.key
+            key = ts._key
         if ts.state == "forgotten":
             self.report({"op": "cancelled-key", "key": key}, ts=ts, client=client)
         elif ts.state == "memory":
             self.report({"op": "key-in-memory", "key": key}, ts=ts, client=client)
         elif ts.state == "erred":
-            failing_ts = ts.exception_blame
+            failing_ts: TaskState = ts._exception_blame
             self.report(
                 {
                     "op": "task-erred",
                     "key": key,
-                    "exception": failing_ts.exception,
-                    "traceback": failing_ts.traceback,
+                    "exception": failing_ts._exception,
+                    "traceback": failing_ts._traceback,
                 },
                 ts=ts,
                 client=client,
@@ -4003,16 +4273,18 @@ def subscribe_worker_status(self, comm=None):
 
     def get_processing(self, comm=None, workers=None):
         ws: WorkerState
+        ts: TaskState
         if workers is not None:
             workers = set(map(self.coerce_address, workers))
-            return {w: [ts.key for ts in self.workers[w].processing] for w in workers}
+            return {w: [ts._key for ts in self.workers[w].processing] for w in workers}
         else:
             return {
-                w: [ts.key for ts in ws._processing] for w, ws in self.workers.items()
+                w: [ts._key for ts in ws._processing] for w, ws in self.workers.items()
             }
 
     def get_who_has(self, comm=None, keys=None):
         ws: WorkerState
+        ts: TaskState
         if keys is not None:
             return {
                 k: [ws._address for ws in self.tasks[k].who_has]
@@ -4022,23 +4294,24 @@ def get_who_has(self, comm=None, keys=None):
             }
         else:
             return {
-                key: [ws._address for ws in ts.who_has]
+                key: [ws._address for ws in ts._who_has]
                 for key, ts in self.tasks.items()
             }
 
     def get_has_what(self, comm=None, workers=None):
         ws: WorkerState
+        ts: TaskState
         if workers is not None:
             workers = map(self.coerce_address, workers)
             return {
-                w: [ts.key for ts in self.workers[w].has_what]
+                w: [ts._key for ts in self.workers[w].has_what]
                 if w in self.workers
                 else []
                 for w in workers
             }
         else:
             return {
-                w: [ts.key for ts in ws._has_what] for w, ws in self.workers.items()
+                w: [ts._key for ts in ws._has_what] for w, ws in self.workers.items()
             }
 
     def get_ncores(self, comm=None, workers=None):
@@ -4050,6 +4323,8 @@ def get_ncores(self, comm=None, workers=None):
             return {w: ws._nthreads for w, ws in self.workers.items()}
 
     async def get_call_stack(self, comm=None, keys=None):
+        ts: TaskState
+        dts: TaskState
         if keys is not None:
             stack = list(keys)
             processing = set()
@@ -4057,14 +4332,14 @@ async def get_call_stack(self, comm=None, keys=None):
                 key = stack.pop()
                 ts = self.tasks[key]
                 if ts.state == "waiting":
-                    stack.extend(dts.key for dts in ts.dependencies)
+                    stack.extend([dts._key for dts in ts._dependencies])
                 elif ts.state == "processing":
                     processing.add(ts)
 
             workers = defaultdict(list)
             for ts in processing:
-                if ts.processing_on:
-                    workers[ts.processing_on.address].append(ts.key)
+                if ts._processing_on:
+                    workers[ts._processing_on.address].append(ts._key)
         else:
             workers = {w: None for w in self.workers}
 
@@ -4078,14 +4353,13 @@ async def get_call_stack(self, comm=None, keys=None):
         return response
 
     def get_nbytes(self, comm=None, keys=None, summary=True):
+        ts: TaskState
         with log_errors():
             if keys is not None:
                 result = {k: self.tasks[k].nbytes for k in keys}
             else:
                 result = {
-                    k: ts.nbytes
-                    for k, ts in self.tasks.items()
-                    if ts.nbytes is not None
+                    k: ts._nbytes for k, ts in self.tasks.items() if ts._nbytes >= 0
                 }
 
             if summary:
@@ -4096,23 +4370,25 @@ def get_nbytes(self, comm=None, keys=None, summary=True):
 
             return result
 
-    def get_comm_cost(self, ts, ws: WorkerState):
+    def get_comm_cost(self, ts: TaskState, ws: WorkerState):
         """
         Get the estimated communication cost (in s.) to compute the task
         on the given worker.
         """
+        dts: TaskState
         return (
-            sum(dts.nbytes for dts in ts.dependencies - ws._has_what) / self.bandwidth
+            sum([dts._nbytes for dts in ts._dependencies - ws._has_what])
+            / self.bandwidth
         )
 
-    def get_task_duration(self, ts, default=None):
+    def get_task_duration(self, ts: TaskState, default=None):
         """
         Get the estimated computation cost of the given task
         (not including any communication cost).
         """
-        duration = ts.prefix.duration_average
-        if duration is None:
-            self.unknown_durations[ts.prefix.name].add(ts)
+        duration = ts._prefix._duration_average
+        if duration < 0:
+            self.unknown_durations[ts._prefix._name].add(ts)
             if default is None:
                 default = parse_timedelta(
                     dask.config.get("distributed.scheduler.unknown-task-duration")
@@ -4167,8 +4443,8 @@ def get_task_stream(self, comm=None, start=None, stop=None, count=None):
         from distributed.diagnostics.task_stream import TaskStreamPlugin
 
         self.add_plugin(TaskStreamPlugin, idempotent=True)
-        ts = [p for p in self.plugins if isinstance(p, TaskStreamPlugin)][0]
-        return ts.collect(start=start, stop=stop, count=count)
+        tsp = [p for p in self.plugins if isinstance(p, TaskStreamPlugin)][0]
+        return tsp.collect(start=start, stop=stop, count=count)
 
     def start_task_metadata(self, comm=None, name=None):
         plugin = CollectTaskMetaDataPlugin(scheduler=self, name=name)
@@ -4204,12 +4480,12 @@ async def register_worker_plugin(self, comm, plugin, name=None):
     # State Transitions #
     #####################
 
-    def _remove_from_processing(self, ts, send_worker_msg=None):
+    def _remove_from_processing(self, ts: TaskState, send_worker_msg=None):
         """
         Remove *ts* from the set of processing tasks.
         """
-        ws: WorkerState = ts.processing_on
-        ts.processing_on = None
+        ws: WorkerState = ts._processing_on
+        ts._processing_on = None
         w = ws._address
         if w in self.workers:  # may have been removed
             duration = ws._processing.pop(ts)
@@ -4225,7 +4501,13 @@ def _remove_from_processing(self, ts, send_worker_msg=None):
                 self.worker_send(w, send_worker_msg)
 
     def _add_to_memory(
-        self, ts, ws: WorkerState, recommendations, type=None, typename=None, **kwargs
+        self,
+        ts: TaskState,
+        ws: WorkerState,
+        recommendations,
+        type=None,
+        typename=None,
+        **kwargs,
     ):
         """
         Add *ts* to the set of in-memory tasks.
@@ -4233,78 +4515,81 @@ def _add_to_memory(
         if self.validate:
             assert ts not in ws._has_what
 
-        ts.who_has.add(ws)
+        ts._who_has.add(ws)
         ws._has_what.add(ts)
         ws._nbytes += ts.get_nbytes()
 
-        deps = ts.dependents
+        deps = ts._dependents
         if len(deps) > 1:
             deps = sorted(deps, key=operator.attrgetter("priority"), reverse=True)
+        dts: TaskState
         for dts in deps:
-            s = dts.waiting_on
+            s = dts._waiting_on
             if ts in s:
                 s.discard(ts)
                 if not s:  # new task ready to run
-                    recommendations[dts.key] = "processing"
+                    recommendations[dts._key] = "processing"
 
-        for dts in ts.dependencies:
-            s = dts.waiters
+        for dts in ts._dependencies:
+            s = dts._waiters
             s.discard(ts)
-            if not s and not dts.who_wants:
-                recommendations[dts.key] = "released"
+            if not s and not dts._who_wants:
+                recommendations[dts._key] = "released"
 
-        if not ts.waiters and not ts.who_wants:
-            recommendations[ts.key] = "released"
+        if not ts._waiters and not ts._who_wants:
+            recommendations[ts._key] = "released"
         else:
-            msg = {"op": "key-in-memory", "key": ts.key}
+            msg = {"op": "key-in-memory", "key": ts._key}
             if type is not None:
                 msg["type"] = type
             self.report(msg)
 
         ts.state = "memory"
-        ts.type = typename
-        ts.group.types.add(typename)
+        ts._type = typename
+        ts._group._types.add(typename)
 
         cs: ClientState = self.clients["fire-and-forget"]
         if ts in cs._wants_what:
-            self.client_releases_keys(client="fire-and-forget", keys=[ts.key])
+            self.client_releases_keys(client="fire-and-forget", keys=[ts._key])
 
     def transition_released_waiting(self, key):
         try:
-            ts = self.tasks[key]
+            ts: TaskState = self.tasks[key]
+            dts: TaskState
 
             if self.validate:
-                assert ts.run_spec
-                assert not ts.waiting_on
-                assert not ts.who_has
-                assert not ts.processing_on
-                assert not any(dts.state == "forgotten" for dts in ts.dependencies)
+                assert ts._run_spec
+                assert not ts._waiting_on
+                assert not ts._who_has
+                assert not ts._processing_on
+                assert not any([dts.state == "forgotten" for dts in ts._dependencies])
 
-            if ts.has_lost_dependencies:
+            if ts._has_lost_dependencies:
                 return {key: "forgotten"}
 
             ts.state = "waiting"
 
             recommendations = {}
 
-            for dts in ts.dependencies:
-                if dts.exception_blame:
-                    ts.exception_blame = dts.exception_blame
+            dts: TaskState
+            for dts in ts._dependencies:
+                if dts._exception_blame:
+                    ts._exception_blame = dts._exception_blame
                     recommendations[key] = "erred"
                     return recommendations
 
-            for dts in ts.dependencies:
-                dep = dts.key
-                if not dts.who_has:
-                    ts.waiting_on.add(dts)
+            for dts in ts._dependencies:
+                dep = dts._key
+                if not dts._who_has:
+                    ts._waiting_on.add(dts)
                 if dts.state == "released":
                     recommendations[dep] = "waiting"
                 else:
-                    dts.waiters.add(ts)
+                    dts._waiters.add(ts)
 
-            ts.waiters = {dts for dts in ts.dependents if dts.state == "waiting"}
+            ts._waiters = {dts for dts in ts._dependents if dts.state == "waiting"}
 
-            if not ts.waiting_on:
+            if not ts._waiting_on:
                 if self.workers:
                     recommendations[key] = "processing"
                 else:
@@ -4322,33 +4607,34 @@ def transition_released_waiting(self, key):
 
     def transition_no_worker_waiting(self, key):
         try:
-            ts = self.tasks[key]
+            ts: TaskState = self.tasks[key]
+            dts: TaskState
 
             if self.validate:
                 assert ts in self.unrunnable
-                assert not ts.waiting_on
-                assert not ts.who_has
-                assert not ts.processing_on
+                assert not ts._waiting_on
+                assert not ts._who_has
+                assert not ts._processing_on
 
             self.unrunnable.remove(ts)
 
-            if ts.has_lost_dependencies:
+            if ts._has_lost_dependencies:
                 return {key: "forgotten"}
 
             recommendations = {}
 
-            for dts in ts.dependencies:
-                dep = dts.key
-                if not dts.who_has:
-                    ts.waiting_on.add(dts)
+            for dts in ts._dependencies:
+                dep = dts._key
+                if not dts._who_has:
+                    ts._waiting_on.add(dts)
                 if dts.state == "released":
                     recommendations[dep] = "waiting"
                 else:
-                    dts.waiters.add(ts)
+                    dts._waiters.add(ts)
 
             ts.state = "waiting"
 
-            if not ts.waiting_on:
+            if not ts._waiting_on:
                 if self.workers:
                     recommendations[key] = "processing"
                 else:
@@ -4364,18 +4650,18 @@ def transition_no_worker_waiting(self, key):
                 pdb.set_trace()
             raise
 
-    def decide_worker(self, ts):
+    def decide_worker(self, ts: TaskState):
         """
         Decide on a worker for task *ts*.  Return a WorkerState.
         """
         valid_workers = self.valid_workers(ts)
 
-        if not valid_workers and not ts.loose_restrictions and self.workers:
+        if not valid_workers and not ts._loose_restrictions and self.workers:
             self.unrunnable.add(ts)
             ts.state = "no-worker"
             return None
 
-        if ts.dependencies or valid_workers is not True:
+        if ts._dependencies or valid_workers is not True:
             worker = decide_worker(
                 ts,
                 self.workers.values(),
@@ -4406,16 +4692,17 @@ def decide_worker(self, ts):
 
     def transition_waiting_processing(self, key):
         try:
-            ts = self.tasks[key]
+            ts: TaskState = self.tasks[key]
+            dts: TaskState
 
             if self.validate:
-                assert not ts.waiting_on
-                assert not ts.who_has
-                assert not ts.exception_blame
-                assert not ts.processing_on
-                assert not ts.has_lost_dependencies
+                assert not ts._waiting_on
+                assert not ts._who_has
+                assert not ts._exception_blame
+                assert not ts._processing_on
+                assert not ts._has_lost_dependencies
                 assert ts not in self.unrunnable
-                assert all(dts.who_has for dts in ts.dependencies)
+                assert all([dts._who_has for dts in ts._dependencies])
 
             ws: WorkerState = self.decide_worker(ts)
             if ws is None:
@@ -4426,7 +4713,7 @@ def transition_waiting_processing(self, key):
             comm = self.get_comm_cost(ts, ws)
 
             ws._processing[ts] = duration + comm
-            ts.processing_on = ws
+            ts._processing_on = ws
             ws._occupancy += duration + comm
             self.total_occupancy += duration + comm
             ts.state = "processing"
@@ -4434,7 +4721,7 @@ def transition_waiting_processing(self, key):
             self.check_idle_saturated(ws)
             self.n_tasks += 1
 
-            if ts.actor:
+            if ts._actor:
                 ws._actors.add(ts)
 
             # logger.debug("Send job to worker: %s, %s", worker, key)
@@ -4453,14 +4740,14 @@ def transition_waiting_processing(self, key):
     def transition_waiting_memory(self, key, nbytes=None, worker=None, **kwargs):
         try:
             ws: WorkerState = self.workers[worker]
-            ts = self.tasks[key]
+            ts: TaskState = self.tasks[key]
 
             if self.validate:
-                assert not ts.processing_on
-                assert ts.waiting_on
+                assert not ts._processing_on
+                assert ts._waiting_on
                 assert ts.state == "waiting"
 
-            ts.waiting_on.clear()
+            ts._waiting_on.clear()
 
             if nbytes is not None:
                 ts.set_nbytes(nbytes)
@@ -4472,9 +4759,9 @@ def transition_waiting_memory(self, key, nbytes=None, worker=None, **kwargs):
             self._add_to_memory(ts, ws, recommendations, **kwargs)
 
             if self.validate:
-                assert not ts.processing_on
-                assert not ts.waiting_on
-                assert ts.who_has
+                assert not ts._processing_on
+                assert not ts._waiting_on
+                assert ts._who_has
 
             return recommendations
         except Exception as e:
@@ -4498,28 +4785,28 @@ def transition_processing_memory(
         ws: WorkerState
         wws: WorkerState
         try:
-            ts = self.tasks[key]
+            ts: TaskState = self.tasks[key]
             assert worker
             assert isinstance(worker, str)
 
             if self.validate:
-                assert ts.processing_on
-                ws = ts.processing_on
+                assert ts._processing_on
+                ws = ts._processing_on
                 assert ts in ws._processing
-                assert not ts.waiting_on
-                assert not ts.who_has, (ts, ts.who_has)
-                assert not ts.exception_blame
+                assert not ts._waiting_on
+                assert not ts._who_has, (ts, ts._who_has)
+                assert not ts._exception_blame
                 assert ts.state == "processing"
 
             ws = self.workers.get(worker)
             if ws is None:
                 return {key: "released"}
 
-            if ws != ts.processing_on:  # someone else has this task
+            if ws != ts._processing_on:  # someone else has this task
                 logger.info(
                     "Unexpected worker completed task, likely due to"
                     " work stealing.  Expected: %s, Got: %s, Key: %s",
-                    ts.processing_on,
+                    ts._processing_on,
                     ws,
                     key,
                 )
@@ -4536,7 +4823,7 @@ def transition_processing_memory(
 
                     # record timings of all actions -- a cheaper way of
                     # getting timing info compared with get_task_stream()
-                    ts.prefix.all_durations[action] += stop - start
+                    ts._prefix._all_durations[action] += stop - start
 
                 if len(L) > 0:
                     compute_start, compute_stop = L[0]
@@ -4550,19 +4837,20 @@ def transition_processing_memory(
             #############################
             if compute_start and ws._processing.get(ts, True):
                 # Update average task duration for worker
-                old_duration = ts.prefix.duration_average or 0
+                old_duration = ts._prefix._duration_average
                 new_duration = compute_stop - compute_start
-                if not old_duration:
+                if old_duration < 0:
                     avg_duration = new_duration
                 else:
                     avg_duration = 0.5 * old_duration + 0.5 * new_duration
 
-                ts.prefix.duration_average = avg_duration
-                ts.group.duration += new_duration
+                ts._prefix._duration_average = avg_duration
+                ts._group._duration += new_duration
 
-                for tts in self.unknown_durations.pop(ts.prefix.name, ()):
-                    if tts.processing_on:
-                        wws = tts.processing_on
+                tts: TaskState
+                for tts in self.unknown_durations.pop(ts._prefix._name, ()):
+                    if tts._processing_on:
+                        wws = tts._processing_on
                         old = wws._processing[tts]
                         comm = self.get_comm_cost(tts, wws)
                         wws._processing[tts] = avg_duration + comm
@@ -4582,8 +4870,8 @@ def transition_processing_memory(
             self._add_to_memory(ts, ws, recommendations, type=type, typename=typename)
 
             if self.validate:
-                assert not ts.processing_on
-                assert not ts.waiting_on
+                assert not ts._processing_on
+                assert not ts._waiting_on
 
             return recommendations
         except Exception as e:
@@ -4597,53 +4885,54 @@ def transition_processing_memory(
     def transition_memory_released(self, key, safe=False):
         ws: WorkerState
         try:
-            ts = self.tasks[key]
+            ts: TaskState = self.tasks[key]
+            dts: TaskState
 
             if self.validate:
-                assert not ts.waiting_on
-                assert not ts.processing_on
+                assert not ts._waiting_on
+                assert not ts._processing_on
                 if safe:
-                    assert not ts.waiters
+                    assert not ts._waiters
 
-            if ts.actor:
-                for ws in ts.who_has:
+            if ts._actor:
+                for ws in ts._who_has:
                     ws._actors.discard(ts)
-                if ts.who_wants:
-                    ts.exception_blame = ts
-                    ts.exception = "Worker holding Actor was lost"
-                    return {ts.key: "erred"}  # don't try to recreate
+                if ts._who_wants:
+                    ts._exception_blame = ts
+                    ts._exception = "Worker holding Actor was lost"
+                    return {ts._key: "erred"}  # don't try to recreate
 
             recommendations = {}
 
-            for dts in ts.waiters:
+            for dts in ts._waiters:
                 if dts.state in ("no-worker", "processing"):
-                    recommendations[dts.key] = "waiting"
+                    recommendations[dts._key] = "waiting"
                 elif dts.state == "waiting":
-                    dts.waiting_on.add(ts)
+                    dts._waiting_on.add(ts)
 
             # XXX factor this out?
-            for ws in ts.who_has:
+            for ws in ts._who_has:
                 ws._has_what.remove(ts)
                 ws._nbytes -= ts.get_nbytes()
-                ts.group.nbytes_in_memory -= ts.get_nbytes()
+                ts._group._nbytes_in_memory -= ts.get_nbytes()
                 self.worker_send(
                     ws._address, {"op": "delete-data", "keys": [key], "report": False}
                 )
-            ts.who_has.clear()
+            ts._who_has.clear()
 
             ts.state = "released"
 
             self.report({"op": "lost-data", "key": key})
 
-            if not ts.run_spec:  # pure data
+            if not ts._run_spec:  # pure data
                 recommendations[key] = "forgotten"
-            elif ts.has_lost_dependencies:
+            elif ts._has_lost_dependencies:
                 recommendations[key] = "forgotten"
-            elif ts.who_wants or ts.waiters:
+            elif ts._who_wants or ts._waiters:
                 recommendations[key] = "waiting"
 
             if self.validate:
-                assert not ts.waiting_on
+                assert not ts._waiting_on
 
             return recommendations
         except Exception as e:
@@ -4656,30 +4945,32 @@ def transition_memory_released(self, key, safe=False):
 
     def transition_released_erred(self, key):
         try:
-            ts = self.tasks[key]
+            ts: TaskState = self.tasks[key]
+            dts: TaskState
+            failing_ts: TaskState
 
             if self.validate:
                 with log_errors(pdb=LOG_PDB):
-                    assert ts.exception_blame
-                    assert not ts.who_has
-                    assert not ts.waiting_on
-                    assert not ts.waiters
+                    assert ts._exception_blame
+                    assert not ts._who_has
+                    assert not ts._waiting_on
+                    assert not ts._waiters
 
             recommendations = {}
 
-            failing_ts = ts.exception_blame
+            failing_ts = ts._exception_blame
 
-            for dts in ts.dependents:
-                dts.exception_blame = failing_ts
-                if not dts.who_has:
-                    recommendations[dts.key] = "erred"
+            for dts in ts._dependents:
+                dts._exception_blame = failing_ts
+                if not dts._who_has:
+                    recommendations[dts._key] = "erred"
 
             self.report(
                 {
                     "op": "task-erred",
                     "key": key,
-                    "exception": failing_ts.exception,
-                    "traceback": failing_ts.traceback,
+                    "exception": failing_ts._exception,
+                    "traceback": failing_ts._traceback,
                 }
             )
 
@@ -4697,25 +4988,26 @@ def transition_released_erred(self, key):
 
     def transition_erred_released(self, key):
         try:
-            ts = self.tasks[key]
+            ts: TaskState = self.tasks[key]
+            dts: TaskState
 
             if self.validate:
                 with log_errors(pdb=LOG_PDB):
-                    assert all(dts.state != "erred" for dts in ts.dependencies)
-                    assert ts.exception_blame
-                    assert not ts.who_has
-                    assert not ts.waiting_on
-                    assert not ts.waiters
+                    assert all([dts.state != "erred" for dts in ts._dependencies])
+                    assert ts._exception_blame
+                    assert not ts._who_has
+                    assert not ts._waiting_on
+                    assert not ts._waiters
 
             recommendations = {}
 
-            ts.exception = None
-            ts.exception_blame = None
-            ts.traceback = None
+            ts._exception = None
+            ts._exception_blame = None
+            ts._traceback = None
 
-            for dep in ts.dependents:
-                if dep.state == "erred":
-                    recommendations[dep.key] = "waiting"
+            for dts in ts._dependents:
+                if dts.state == "erred":
+                    recommendations[dts._key] = "waiting"
 
             self.report({"op": "task-retried", "key": key})
             ts.state = "released"
@@ -4731,30 +5023,31 @@ def transition_erred_released(self, key):
 
     def transition_waiting_released(self, key):
         try:
-            ts = self.tasks[key]
+            ts: TaskState = self.tasks[key]
 
             if self.validate:
-                assert not ts.who_has
-                assert not ts.processing_on
+                assert not ts._who_has
+                assert not ts._processing_on
 
             recommendations = {}
 
-            for dts in ts.dependencies:
-                s = dts.waiters
+            dts: TaskState
+            for dts in ts._dependencies:
+                s = dts._waiters
                 if ts in s:
                     s.discard(ts)
-                    if not s and not dts.who_wants:
-                        recommendations[dts.key] = "released"
-            ts.waiting_on.clear()
+                    if not s and not dts._who_wants:
+                        recommendations[dts._key] = "released"
+            ts._waiting_on.clear()
 
             ts.state = "released"
 
-            if ts.has_lost_dependencies:
+            if ts._has_lost_dependencies:
                 recommendations[key] = "forgotten"
-            elif not ts.exception_blame and (ts.who_wants or ts.waiters):
+            elif not ts._exception_blame and (ts._who_wants or ts._waiters):
                 recommendations[key] = "waiting"
             else:
-                ts.waiters.clear()
+                ts._waiters.clear()
 
             return recommendations
         except Exception as e:
@@ -4767,12 +5060,13 @@ def transition_waiting_released(self, key):
 
     def transition_processing_released(self, key):
         try:
-            ts = self.tasks[key]
+            ts: TaskState = self.tasks[key]
+            dts: TaskState
 
             if self.validate:
-                assert ts.processing_on
-                assert not ts.who_has
-                assert not ts.waiting_on
+                assert ts._processing_on
+                assert not ts._who_has
+                assert not ts._waiting_on
                 assert self.tasks[key].state == "processing"
 
             self._remove_from_processing(
@@ -4783,22 +5077,22 @@ def transition_processing_released(self, key):
 
             recommendations = {}
 
-            if ts.has_lost_dependencies:
+            if ts._has_lost_dependencies:
                 recommendations[key] = "forgotten"
-            elif ts.waiters or ts.who_wants:
+            elif ts._waiters or ts._who_wants:
                 recommendations[key] = "waiting"
 
             if recommendations.get(key) != "waiting":
-                for dts in ts.dependencies:
+                for dts in ts._dependencies:
                     if dts.state != "released":
-                        s = dts.waiters
+                        s = dts._waiters
                         s.discard(ts)
-                        if not s and not dts.who_wants:
-                            recommendations[dts.key] = "released"
-                ts.waiters.clear()
+                        if not s and not dts._who_wants:
+                            recommendations[dts._key] = "released"
+                ts._waiters.clear()
 
             if self.validate:
-                assert not ts.processing_on
+                assert not ts._processing_on
 
             return recommendations
         except Exception as e:
@@ -4814,43 +5108,45 @@ def transition_processing_erred(
     ):
         ws: WorkerState
         try:
-            ts = self.tasks[key]
+            ts: TaskState = self.tasks[key]
+            dts: TaskState
+            failing_ts: TaskState
 
             if self.validate:
-                assert cause or ts.exception_blame
-                assert ts.processing_on
-                assert not ts.who_has
-                assert not ts.waiting_on
+                assert cause or ts._exception_blame
+                assert ts._processing_on
+                assert not ts._who_has
+                assert not ts._waiting_on
 
-            if ts.actor:
-                ws = ts.processing_on
+            if ts._actor:
+                ws = ts._processing_on
                 ws._actors.remove(ts)
 
             self._remove_from_processing(ts)
 
             if exception is not None:
-                ts.exception = exception
+                ts._exception = exception
             if traceback is not None:
-                ts.traceback = traceback
+                ts._traceback = traceback
             if cause is not None:
                 failing_ts = self.tasks[cause]
-                ts.exception_blame = failing_ts
+                ts._exception_blame = failing_ts
             else:
-                failing_ts = ts.exception_blame
+                failing_ts = ts._exception_blame
 
             recommendations = {}
 
-            for dts in ts.dependents:
-                dts.exception_blame = failing_ts
-                recommendations[dts.key] = "erred"
+            for dts in ts._dependents:
+                dts._exception_blame = failing_ts
+                recommendations[dts._key] = "erred"
 
-            for dts in ts.dependencies:
-                s = dts.waiters
+            for dts in ts._dependencies:
+                s = dts._waiters
                 s.discard(ts)
-                if not s and not dts.who_wants:
-                    recommendations[dts.key] = "released"
+                if not s and not dts._who_wants:
+                    recommendations[dts._key] = "released"
 
-            ts.waiters.clear()  # do anything with this?
+            ts._waiters.clear()  # do anything with this?
 
             ts.state = "erred"
 
@@ -4858,8 +5154,8 @@ def transition_processing_erred(
                 {
                     "op": "task-erred",
                     "key": key,
-                    "exception": failing_ts.exception,
-                    "traceback": failing_ts.traceback,
+                    "exception": failing_ts._exception,
+                    "traceback": failing_ts._traceback,
                 }
             )
 
@@ -4868,7 +5164,7 @@ def transition_processing_erred(
                 self.client_releases_keys(client="fire-and-forget", keys=[key])
 
             if self.validate:
-                assert not ts.processing_on
+                assert not ts._processing_on
 
             return recommendations
         except Exception as e:
@@ -4881,20 +5177,21 @@ def transition_processing_erred(
 
     def transition_no_worker_released(self, key):
         try:
-            ts = self.tasks[key]
+            ts: TaskState = self.tasks[key]
+            dts: TaskState
 
             if self.validate:
                 assert self.tasks[key].state == "no-worker"
-                assert not ts.who_has
-                assert not ts.waiting_on
+                assert not ts._who_has
+                assert not ts._waiting_on
 
             self.unrunnable.remove(ts)
             ts.state = "released"
 
-            for dts in ts.dependencies:
-                dts.waiters.discard(ts)
+            for dts in ts._dependencies:
+                dts._waiters.discard(ts)
 
-            ts.waiters.clear()
+            ts._waiters.clear()
 
             return {}
         except Exception as e:
@@ -4906,48 +5203,49 @@ def transition_no_worker_released(self, key):
             raise
 
     def remove_key(self, key):
-        ts = self.tasks.pop(key)
+        ts: TaskState = self.tasks.pop(key)
         assert ts.state == "forgotten"
         self.unrunnable.discard(ts)
         cs: ClientState
-        for cs in ts.who_wants:
+        for cs in ts._who_wants:
             cs._wants_what.remove(ts)
-        ts.who_wants.clear()
-        ts.processing_on = None
-        ts.exception_blame = ts.exception = ts.traceback = None
+        ts._who_wants.clear()
+        ts._processing_on = None
+        ts._exception_blame = ts._exception = ts._traceback = None
 
         if key in self.task_metadata:
             del self.task_metadata[key]
 
-    def _propagate_forgotten(self, ts, recommendations):
+    def _propagate_forgotten(self, ts: TaskState, recommendations):
         ts.state = "forgotten"
-        key = ts.key
-        for dts in ts.dependents:
-            dts.has_lost_dependencies = True
-            dts.dependencies.remove(ts)
-            dts.waiting_on.discard(ts)
+        key = ts._key
+        dts: TaskState
+        for dts in ts._dependents:
+            dts._has_lost_dependencies = True
+            dts._dependencies.remove(ts)
+            dts._waiting_on.discard(ts)
             if dts.state not in ("memory", "erred"):
                 # Cannot compute task anymore
-                recommendations[dts.key] = "forgotten"
-        ts.dependents.clear()
-        ts.waiters.clear()
+                recommendations[dts._key] = "forgotten"
+        ts._dependents.clear()
+        ts._waiters.clear()
 
-        for dts in ts.dependencies:
-            dts.dependents.remove(ts)
-            s = dts.waiters
+        for dts in ts._dependencies:
+            dts._dependents.remove(ts)
+            s = dts._waiters
             s.discard(ts)
-            if not dts.dependents and not dts.who_wants:
+            if not dts._dependents and not dts._who_wants:
                 # Task not needed anymore
                 assert dts is not ts
-                recommendations[dts.key] = "forgotten"
-        ts.dependencies.clear()
-        ts.waiting_on.clear()
+                recommendations[dts._key] = "forgotten"
+        ts._dependencies.clear()
+        ts._waiting_on.clear()
 
-        if ts.who_has:
-            ts.group.nbytes_in_memory -= ts.get_nbytes()
+        if ts._who_has:
+            ts._group._nbytes_in_memory -= ts.get_nbytes()
 
         ws: WorkerState
-        for ws in ts.who_has:
+        for ws in ts._who_has:
             ws._has_what.remove(ts)
             ws._nbytes -= ts.get_nbytes()
             w = ws._address
@@ -4955,24 +5253,24 @@ def _propagate_forgotten(self, ts, recommendations):
                 self.worker_send(
                     w, {"op": "delete-data", "keys": [key], "report": False}
                 )
-        ts.who_has.clear()
+        ts._who_has.clear()
 
     def transition_memory_forgotten(self, key):
         ws: WorkerState
         try:
-            ts = self.tasks[key]
+            ts: TaskState = self.tasks[key]
 
             if self.validate:
                 assert ts.state == "memory"
-                assert not ts.processing_on
-                assert not ts.waiting_on
-                if not ts.run_spec:
+                assert not ts._processing_on
+                assert not ts._waiting_on
+                if not ts._run_spec:
                     # It's ok to forget a pure data task
                     pass
-                elif ts.has_lost_dependencies:
+                elif ts._has_lost_dependencies:
                     # It's ok to forget a task with forgotten dependencies
                     pass
-                elif not ts.who_wants and not ts.waiters and not ts.dependents:
+                elif not ts._who_wants and not ts._waiters and not ts._dependents:
                     # It's ok to forget a task that nobody needs
                     pass
                 else:
@@ -4980,8 +5278,8 @@ def transition_memory_forgotten(self, key):
 
             recommendations = {}
 
-            if ts.actor:
-                for ws in ts.who_has:
+            if ts._actor:
+                for ws in ts._who_has:
                     ws._actors.discard(ts)
 
             self._propagate_forgotten(ts, recommendations)
@@ -5000,20 +5298,20 @@ def transition_memory_forgotten(self, key):
 
     def transition_released_forgotten(self, key):
         try:
-            ts = self.tasks[key]
+            ts: TaskState = self.tasks[key]
 
             if self.validate:
                 assert ts.state in ("released", "erred")
-                assert not ts.who_has
-                assert not ts.processing_on
-                assert not ts.waiting_on, (ts, ts.waiting_on)
-                if not ts.run_spec:
+                assert not ts._who_has
+                assert not ts._processing_on
+                assert not ts._waiting_on, (ts, ts._waiting_on)
+                if not ts._run_spec:
                     # It's ok to forget a pure data task
                     pass
-                elif ts.has_lost_dependencies:
+                elif ts._has_lost_dependencies:
                     # It's ok to forget a task with forgotten dependencies
                     pass
-                elif not ts.who_wants and not ts.waiters and not ts.dependents:
+                elif not ts._who_wants and not ts._waiters and not ts._dependents:
                     # It's ok to forget a task that nobody needs
                     pass
                 else:
@@ -5050,6 +5348,7 @@ def transition(self, key, finish, *args, **kwargs):
         --------
         Scheduler.transitions: transitive version of this function
         """
+        ts: TaskState
         try:
             try:
                 ts = self.tasks[key]
@@ -5060,8 +5359,8 @@ def transition(self, key, finish, *args, **kwargs):
                 return {}
 
             if self.plugins:
-                dependents = set(ts.dependents)
-                dependencies = set(ts.dependencies)
+                dependents = set(ts._dependents)
+                dependencies = set(ts._dependencies)
 
             if (start, finish) in self._transitions:
                 func = self._transitions[start, finish]
@@ -5097,25 +5396,25 @@ def transition(self, key, finish, *args, **kwargs):
                 # Temporarily put back forgotten key for plugin to retrieve it
                 if ts.state == "forgotten":
                     try:
-                        ts.dependents = dependents
-                        ts.dependencies = dependencies
+                        ts._dependents = dependents
+                        ts._dependencies = dependencies
                     except KeyError:
                         pass
-                    self.tasks[ts.key] = ts
+                    self.tasks[ts._key] = ts
                 for plugin in list(self.plugins):
                     try:
                         plugin.transition(key, start, finish2, *args, **kwargs)
                     except Exception:
                         logger.info("Plugin failed with exception", exc_info=True)
                 if ts.state == "forgotten":
-                    del self.tasks[ts.key]
+                    del self.tasks[ts._key]
 
-            if ts.state == "forgotten" and ts.group.name in self.task_groups:
+            if ts.state == "forgotten" and ts._group._name in self.task_groups:
                 # Remove TaskGroup if all tasks are in the forgotten state
-                tg = ts.group
-                if not any(tg.states.get(s) for s in ALL_TASK_STATES):
-                    ts.prefix.groups.remove(tg)
-                    del self.task_groups[tg.name]
+                tg: TaskGroup = ts._group
+                if not any(tg._states.get(s) for s in ALL_TASK_STATES):
+                    ts._prefix._groups.remove(tg)
+                    del self.task_groups[tg._name]
 
             return recommendations
         except Exception as e:
@@ -5159,6 +5458,7 @@ def reschedule(self, key=None, worker=None):
         Things may have shifted and this task may now be better suited to run
         elsewhere
         """
+        ts: TaskState
         try:
             ts = self.tasks[key]
         except KeyError:
@@ -5169,7 +5469,7 @@ def reschedule(self, key=None, worker=None):
             return
         if ts.state != "processing":
             return
-        if worker and ts.processing_on.address != worker:
+        if worker and ts._processing_on.address != worker:
             return
         self.transitions({key: "released"})
 
@@ -5216,7 +5516,7 @@ def check_idle_saturated(self, ws: WorkerState, occ: double = -1.0):
             else:
                 saturated.discard(ws)
 
-    def valid_workers(self, ts):
+    def valid_workers(self, ts: TaskState):
         """Return set of currently valid workers for key
 
         If all workers are valid then this returns ``True``.
@@ -5228,13 +5528,13 @@ def valid_workers(self, ts):
         """
         s = True
 
-        if ts.worker_restrictions:
-            s = {w for w in ts.worker_restrictions if w in self.workers}
+        if ts._worker_restrictions:
+            s = {w for w in ts._worker_restrictions if w in self.workers}
 
-        if ts.host_restrictions:
+        if ts._host_restrictions:
             # Resolve the alias here rather than early, for the worker
             # may not be connected when host_restrictions is populated
-            hr = [self.coerce_hostname(h) for h in ts.host_restrictions]
+            hr = [self.coerce_hostname(h) for h in ts._host_restrictions]
             # XXX need HostState?
             ss = [self.host_info[h]["addresses"] for h in hr if h in self.host_info]
             ss = set.union(*ss) if ss else set()
@@ -5243,14 +5543,14 @@ def valid_workers(self, ts):
             else:
                 s |= ss
 
-        if ts.resource_restrictions:
+        if ts._resource_restrictions:
             w = {
                 resource: {
                     w
                     for w, supplied in self.resources[resource].items()
                     if supplied >= required
                 }
-                for resource, required in ts.resource_restrictions.items()
+                for resource, required in ts._resource_restrictions.items()
             }
 
             ww = set.intersection(*w.values())
@@ -5265,14 +5565,14 @@ def valid_workers(self, ts):
         else:
             return {self.workers[w] for w in s}
 
-    def consume_resources(self, ts, ws: WorkerState):
-        if ts.resource_restrictions:
-            for r, required in ts.resource_restrictions.items():
+    def consume_resources(self, ts: TaskState, ws: WorkerState):
+        if ts._resource_restrictions:
+            for r, required in ts._resource_restrictions.items():
                 ws._used_resources[r] += required
 
-    def release_resources(self, ts, ws: WorkerState):
-        if ts.resource_restrictions:
-            for r, required in ts.resource_restrictions.items():
+    def release_resources(self, ts: TaskState, ws: WorkerState):
+        if ts._resource_restrictions:
+            for r, required in ts._resource_restrictions.items():
                 ws._used_resources[r] -= required
 
     #####################
@@ -5356,19 +5656,20 @@ def start_ipython(self, comm=None):
             )
         return self._ipython_kernel.get_connection_info()
 
-    def worker_objective(self, ts, ws: WorkerState):
+    def worker_objective(self, ts: TaskState, ws: WorkerState):
         """
         Objective function to determine which worker should get the task
 
         Minimize expected start time.  If a tie then break with data storage.
         """
+        dts: TaskState
         comm_bytes = sum(
-            [dts.get_nbytes() for dts in ts.dependencies if ws not in dts.who_has]
+            [dts.get_nbytes() for dts in ts._dependencies if ws not in dts._who_has]
         )
         stack_time = ws._occupancy / ws._nthreads
         start_time = comm_bytes / self.bandwidth + stack_time
 
-        if ts.actor:
+        if ts._actor:
             return (len(ws._actors), start_time, ws._nbytes)
         else:
             return (start_time, ws._nbytes)
@@ -5777,7 +6078,7 @@ def adaptive_target(self, comm=None, target_duration=None):
             return len(self.workers) - len(to_close)
 
 
-def decide_worker(ts, all_workers, valid_workers, objective):
+def decide_worker(ts: TaskState, all_workers, valid_workers, objective):
     """
     Decide which worker should take task *ts*.
 
@@ -5793,13 +6094,14 @@ def decide_worker(ts, all_workers, valid_workers, objective):
     of bytes sent between workers.  This is determined by calling the
     *objective* function.
     """
-    deps = ts.dependencies
-    assert all(dts.who_has for dts in deps)
-    if ts.actor:
+    dts: TaskState
+    deps = ts._dependencies
+    assert all([dts._who_has for dts in deps])
+    if ts._actor:
         candidates = set(all_workers)
     else:
         ws: WorkerState
-        candidates = {ws for dts in deps for ws in dts.who_has}
+        candidates = {ws for dts in deps for ws in dts._who_has}
     if valid_workers is True:
         if not candidates:
             candidates = set(all_workers)
@@ -5808,7 +6110,7 @@ def decide_worker(ts, all_workers, valid_workers, objective):
         if not candidates:
             candidates = valid_workers
             if not candidates:
-                if ts.loose_restrictions:
+                if ts._loose_restrictions:
                     return decide_worker(ts, all_workers, True, objective)
                 else:
                     return None
@@ -5821,82 +6123,83 @@ def decide_worker(ts, all_workers, valid_workers, objective):
     return min(candidates, key=objective)
 
 
-def validate_task_state(ts):
+def validate_task_state(ts: TaskState):
     """
     Validate the given TaskState.
     """
     ws: WorkerState
+    dts: TaskState
 
     assert ts.state in ALL_TASK_STATES or ts.state == "forgotten", ts
 
-    if ts.waiting_on:
-        assert ts.waiting_on.issubset(ts.dependencies), (
+    if ts._waiting_on:
+        assert ts._waiting_on.issubset(ts._dependencies), (
             "waiting not subset of dependencies",
-            str(ts.waiting_on),
-            str(ts.dependencies),
+            str(ts._waiting_on),
+            str(ts._dependencies),
         )
-    if ts.waiters:
-        assert ts.waiters.issubset(ts.dependents), (
+    if ts._waiters:
+        assert ts._waiters.issubset(ts._dependents), (
             "waiters not subset of dependents",
-            str(ts.waiters),
-            str(ts.dependents),
+            str(ts._waiters),
+            str(ts._dependents),
         )
 
-    for dts in ts.waiting_on:
-        assert not dts.who_has, ("waiting on in-memory dep", str(ts), str(dts))
+    for dts in ts._waiting_on:
+        assert not dts._who_has, ("waiting on in-memory dep", str(ts), str(dts))
         assert dts.state != "released", ("waiting on released dep", str(ts), str(dts))
-    for dts in ts.dependencies:
-        assert ts in dts.dependents, (
+    for dts in ts._dependencies:
+        assert ts in dts._dependents, (
             "not in dependency's dependents",
             str(ts),
             str(dts),
-            str(dts.dependents),
+            str(dts._dependents),
         )
         if ts.state in ("waiting", "processing"):
-            assert dts in ts.waiting_on or dts.who_has, (
+            assert dts in ts._waiting_on or dts._who_has, (
                 "dep missing",
                 str(ts),
                 str(dts),
             )
         assert dts.state != "forgotten"
 
-    for dts in ts.waiters:
+    for dts in ts._waiters:
         assert dts.state in ("waiting", "processing"), (
             "waiter not in play",
             str(ts),
             str(dts),
         )
-    for dts in ts.dependents:
-        assert ts in dts.dependencies, (
+    for dts in ts._dependents:
+        assert ts in dts._dependencies, (
             "not in dependent's dependencies",
             str(ts),
             str(dts),
-            str(dts.dependencies),
+            str(dts._dependencies),
         )
         assert dts.state != "forgotten"
 
-    assert (ts.processing_on is not None) == (ts.state == "processing")
-    assert bool(ts.who_has) == (ts.state == "memory"), (ts, ts.who_has)
+    assert (ts._processing_on is not None) == (ts.state == "processing")
+    assert bool(ts._who_has) == (ts.state == "memory"), (ts, ts._who_has)
 
     if ts.state == "processing":
-        assert all(dts.who_has for dts in ts.dependencies), (
+        assert all([dts._who_has for dts in ts._dependencies]), (
             "task processing without all deps",
             str(ts),
-            str(ts.dependencies),
+            str(ts._dependencies),
         )
-        assert not ts.waiting_on
+        assert not ts._waiting_on
 
-    if ts.who_has:
-        assert ts.waiters or ts.who_wants, (
+    if ts._who_has:
+        assert ts._waiters or ts._who_wants, (
             "unneeded task in memory",
             str(ts),
-            str(ts.who_has),
+            str(ts._who_has),
         )
-        if ts.run_spec:  # was computed
-            assert ts.type
-            assert isinstance(ts.type, str)
-        assert not any(ts in dts.waiting_on for dts in ts.dependents)
-        for ws in ts.who_has:
+        if ts._run_spec:  # was computed
+            assert ts._type
+            assert isinstance(ts._type, str)
+        assert not any([ts in dts._waiting_on for dts in ts._dependents])
+        for ws in ts._who_has:
             assert ts in ws._has_what, (
                 "not in who_has' has_what",
                 str(ts),
@@ -5904,9 +6207,9 @@ def validate_task_state(ts):
                 str(ws._has_what),
             )
 
-    if ts.who_wants:
+    if ts._who_wants:
         cs: ClientState
-        for cs in ts.who_wants:
+        for cs in ts._who_wants:
             assert ts in cs._wants_what, (
                 "not in who_wants' wants_what",
                 str(ts),
@@ -5914,20 +6217,21 @@ def validate_task_state(ts):
                 str(cs._wants_what),
             )
 
-    if ts.actor:
+    if ts._actor:
         if ts.state == "memory":
-            assert sum([ts in ws._actors for ws in ts.who_has]) == 1
+            assert sum([ts in ws._actors for ws in ts._who_has]) == 1
         if ts.state == "processing":
-            assert ts in ts.processing_on.actors
+            assert ts in ts._processing_on.actors
 
 
 def validate_worker_state(ws: WorkerState):
+    ts: TaskState
     for ts in ws._has_what:
-        assert ws in ts.who_has, (
+        assert ws in ts._who_has, (
             "not in has_what' who_has",
             str(ws),
             str(ts),
-            str(ts.who_has),
+            str(ts._who_has),
         )
 
     for ts in ws._actors:
@@ -5941,6 +6245,7 @@ def validate_state(tasks, workers, clients):
     This performs a sequence of checks on the entire graph, running in about
     linear time.  This raises assert errors if anything doesn't check out.
     """
+    ts: TaskState
     for ts in tasks.values():
         validate_task_state(ts)
 
@@ -5951,11 +6256,11 @@ def validate_state(tasks, workers, clients):
     cs: ClientState
     for cs in clients.values():
         for ts in cs._wants_what:
-            assert cs in ts.who_wants, (
+            assert cs in ts._who_wants, (
                 "not in wants_what' who_wants",
                 str(cs),
                 str(ts),
-                str(ts.who_wants),
+                str(ts._who_wants),
             )
 
 
@@ -6031,8 +6336,8 @@ def update_graph(self, scheduler, dsk=None, keys=None, restrictions=None, **kwar
 
     def transition(self, key, start, finish, *args, **kwargs):
         if finish == "memory" or finish == "erred":
-            ts = self.scheduler.tasks.get(key)
-            if ts is not None and ts.key in self.keys:
-                self.metadata[key] = ts.metadata
+            ts: TaskState = self.scheduler.tasks.get(key)
+            if ts is not None and ts._key in self.keys:
+                self.metadata[key] = ts._metadata
                 self.state[key] = finish
                 self.keys.discard(key)

From 06043e10374ed7c31ceac75b221258204f29221f Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Mon, 7 Dec 2020 12:34:21 -0800
Subject: [PATCH 1081/1550] Use `operator.attrgetter` on `WorkerState.address`
 (#4324)

Make use of `operator.attrgetter` to get `WorkerState`'s `address`
attribute instead of a `lambda`. This winds up being a bit faster. Also
`operator.attrgetter` is honed for this purpose. Not too mention it is
already used frequently for this purpose in this code.
---
 distributed/scheduler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index b61aa5f07a7..595501c1d2b 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -3965,7 +3965,7 @@ def workers_to_close(
                 return []
 
             if key is None:
-                key = lambda ws: ws.address
+                key = operator.attrgetter("address")
             if isinstance(key, bytes) and dask.config.get(
                 "distributed.scheduler.pickle"
             ):

From 2764e87d33139c35c520552761aa79f773c77137 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Tue, 8 Dec 2020 06:30:30 -0800
Subject: [PATCH 1082/1550] Use `.pop(...)` to remove `key` (#4327)

Instead of checking for the `key` and then using `del` to remove it,
simply do it in one operation with `.pop(...)`. If the key exists, it
will be removed. If not, nothing happens.
---
 distributed/scheduler.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 595501c1d2b..03a90cff3cb 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -5212,9 +5212,7 @@ def remove_key(self, key):
         ts._who_wants.clear()
         ts._processing_on = None
         ts._exception_blame = ts._exception = ts._traceback = None
-
-        if key in self.task_metadata:
-            del self.task_metadata[key]
+        self.task_metadata.pop(key, None)
 
     def _propagate_forgotten(self, ts: TaskState, recommendations):
         ts.state = "forgotten"

From 9f5426e030b5c4f7cd9db08d5c73f9aca3db4830 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Tue, 8 Dec 2020 12:57:14 -0800
Subject: [PATCH 1083/1550] Optimize `get_comm_cost` (#4328)

Instead of using a generator expression to build out a `list` and `sum`
it, simply iterate through each object and accumulate the total
ourselves. In Cython this works out to be a `for`-loop roughly with the
addition itself occurring just the same in C.
---
 distributed/scheduler.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 03a90cff3cb..5ee9da2bfb2 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -4376,10 +4376,11 @@ def get_comm_cost(self, ts: TaskState, ws: WorkerState):
         on the given worker.
         """
         dts: TaskState
-        return (
-            sum([dts._nbytes for dts in ts._dependencies - ws._has_what])
-            / self.bandwidth
-        )
+        deps: set = ts._dependencies - ws._has_what
+        nbytes: Py_ssize_t = 0
+        for dts in deps:
+            nbytes += dts._nbytes
+        return nbytes / self.bandwidth
 
     def get_task_duration(self, ts: TaskState, default=None):
         """

From 47876b64c11ff175b044214dcbf14465a6352c7d Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Tue, 8 Dec 2020 14:49:45 -0800
Subject: [PATCH 1084/1550] Store occupancy in `transition_waiting_processing`
 (#4330)

Instead of computing the sum of the task duration and communication cost
repeatedly, simply compute it once and assign it to a variable. Then
reuse it in later steps.
---
 distributed/scheduler.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 5ee9da2bfb2..66a04536dca 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -4712,11 +4712,12 @@ def transition_waiting_processing(self, key):
 
             duration = self.get_task_duration(ts)
             comm = self.get_comm_cost(ts, ws)
+            occupancy = duration + comm
 
-            ws._processing[ts] = duration + comm
+            ws._processing[ts] = occupancy
             ts._processing_on = ws
-            ws._occupancy += duration + comm
-            self.total_occupancy += duration + comm
+            ws._occupancy += occupancy
+            self.total_occupancy += occupancy
             ts.state = "processing"
             self.consume_resources(ts, ws)
             self.check_idle_saturated(ws)

From 881f532bfff0b8b58b6f5b930cd8ca09edf2fb8c Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Tue, 8 Dec 2020 16:16:21 -0800
Subject: [PATCH 1085/1550] Optimize `valid_workers` (#4329)

* Drop extra blank line

* Use different variables for `list` and `set`

* Distinguish iterated variable from result

* Annotate intermediate variables

* Assign `s` `None` instead of `True`

This works better if we want to type `s` as `set`. Also this is a more
typical default value when initializing a variable unlike `True`.

* Always return `set` from `valid_workers`
---
 distributed/scheduler.py | 50 ++++++++++++++++++++++------------------
 1 file changed, 28 insertions(+), 22 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 66a04536dca..f7ad5916e1e 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -2225,8 +2225,8 @@ async def add_worker(
 
             recommendations = {}
             for ts in list(self.unrunnable):
-                valid = self.valid_workers(ts)
-                if valid is True or ws in valid:
+                valid: set = self.valid_workers(ts)
+                if valid is None or ws in valid:
                     recommendations[ts._key] = "waiting"
 
             if recommendations:
@@ -4655,14 +4655,19 @@ def decide_worker(self, ts: TaskState):
         """
         Decide on a worker for task *ts*.  Return a WorkerState.
         """
-        valid_workers = self.valid_workers(ts)
-
-        if not valid_workers and not ts._loose_restrictions and self.workers:
+        valid_workers: set = self.valid_workers(ts)
+
+        if (
+            valid_workers is not None
+            and not valid_workers
+            and not ts._loose_restrictions
+            and self.workers
+        ):
             self.unrunnable.add(ts)
             ts.state = "no-worker"
             return None
 
-        if ts._dependencies or valid_workers is not True:
+        if ts._dependencies or valid_workers is not None:
             worker = decide_worker(
                 ts,
                 self.workers.values(),
@@ -5516,17 +5521,17 @@ def check_idle_saturated(self, ws: WorkerState, occ: double = -1.0):
             else:
                 saturated.discard(ws)
 
-    def valid_workers(self, ts: TaskState):
+    def valid_workers(self, ts: TaskState) -> set:
         """Return set of currently valid workers for key
 
-        If all workers are valid then this returns ``True``.
+        If all workers are valid then this returns ``None``.
         This checks tracks the following state:
 
         *  worker_restrictions
         *  host_restrictions
         *  resource_restrictions
         """
-        s = True
+        s: set = None
 
         if ts._worker_restrictions:
             s = {w for w in ts._worker_restrictions if w in self.workers}
@@ -5534,17 +5539,19 @@ def valid_workers(self, ts: TaskState):
         if ts._host_restrictions:
             # Resolve the alias here rather than early, for the worker
             # may not be connected when host_restrictions is populated
-            hr = [self.coerce_hostname(h) for h in ts._host_restrictions]
+            hr: list = [self.coerce_hostname(h) for h in ts._host_restrictions]
             # XXX need HostState?
-            ss = [self.host_info[h]["addresses"] for h in hr if h in self.host_info]
-            ss = set.union(*ss) if ss else set()
-            if s is True:
+            sl: list = [
+                self.host_info[h]["addresses"] for h in hr if h in self.host_info
+            ]
+            ss: set = set.union(*sl) if sl else set()
+            if s is None:
                 s = ss
             else:
                 s |= ss
 
         if ts._resource_restrictions:
-            w = {
+            dw: dict = {
                 resource: {
                     w
                     for w, supplied in self.resources[resource].items()
@@ -5553,14 +5560,13 @@ def valid_workers(self, ts: TaskState):
                 for resource, required in ts._resource_restrictions.items()
             }
 
-            ww = set.intersection(*w.values())
-
-            if s is True:
+            ww: set = set.intersection(*dw.values())
+            if s is None:
                 s = ww
             else:
                 s &= ww
 
-        if s is True:
+        if s is None:
             return s
         else:
             return {self.workers[w] for w in s}
@@ -6078,7 +6084,7 @@ def adaptive_target(self, comm=None, target_duration=None):
             return len(self.workers) - len(to_close)
 
 
-def decide_worker(ts: TaskState, all_workers, valid_workers, objective):
+def decide_worker(ts: TaskState, all_workers, valid_workers: set, objective):
     """
     Decide which worker should take task *ts*.
 
@@ -6087,7 +6093,7 @@ def decide_worker(ts: TaskState, all_workers, valid_workers, objective):
     If several workers have dependencies then we choose the less-busy worker.
 
     Optionally provide *valid_workers* of where jobs are allowed to occur
-    (if all workers are allowed to take the task, pass True instead).
+    (if all workers are allowed to take the task, pass None instead).
 
     If the task requires data communication because no eligible worker has
     all the dependencies already, then we choose to minimize the number
@@ -6102,7 +6108,7 @@ def decide_worker(ts: TaskState, all_workers, valid_workers, objective):
     else:
         ws: WorkerState
         candidates = {ws for dts in deps for ws in dts._who_has}
-    if valid_workers is True:
+    if valid_workers is None:
         if not candidates:
             candidates = set(all_workers)
     else:
@@ -6111,7 +6117,7 @@ def decide_worker(ts: TaskState, all_workers, valid_workers, objective):
             candidates = valid_workers
             if not candidates:
                 if ts._loose_restrictions:
-                    return decide_worker(ts, all_workers, True, objective)
+                    return decide_worker(ts, all_workers, None, objective)
                 else:
                     return None
     if not candidates:

From 13bbc7fa58f616c801c8a62710a7ca6a9c0d7609 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Tue, 8 Dec 2020 16:18:33 -0800
Subject: [PATCH 1086/1550] Optimize `send_task_to_worker` (#4331)

* Move validation block with dependency handling

As both of these branches check that `deps` is non-trivial, we can skip
the check in the second branch if we include it within the branch where
the check already happened. Should simplify the work needed for this
check.

* Group `WorkerState` annotation with others

* Annotate `msg` and `deps` as `dict` and `set`

* Use `_key` attribute from `TaskState`

* Take `TaskState` object in `send_task_to_worker`

All that is needed in `send_task_to_worker` can already be gotten from
the `TaskState` object, which we retrieve in
`transition_waiting_processing` anyways. So just pass the `TaskState`
object directly instead of the `key`, which is also in the `TaskState`
object.

* Add optional `duration` to `send_task_to_worker`

In some cases we may have already computed the `duration`, so provide an
optional parameter to supply this if it is available. Otherwise go ahead
and compute it here.

Since we already compute the `duration` in
`transition_waiting_processing`, go ahead and pass that along to
`send_task_to_worker` to avoid needing to compute it twice.
---
 distributed/scheduler.py | 22 ++++++++++++----------
 distributed/stealing.py  |  2 +-
 2 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index f7ad5916e1e..c2743dc9310 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -3181,33 +3181,35 @@ def remove_client_from_events():
         )
         self.loop.call_later(cleanup_delay, remove_client_from_events)
 
-    def send_task_to_worker(self, worker, key):
+    def send_task_to_worker(self, worker, ts: TaskState, duration=None):
         """ Send a single computational task to a worker """
         try:
-            ts: TaskState = self.tasks[key]
+            ws: WorkerState
             dts: TaskState
 
-            msg = {
+            if duration is None:
+                duration = self.get_task_duration(ts)
+
+            msg: dict = {
                 "op": "compute-task",
-                "key": key,
+                "key": ts._key,
                 "priority": ts._priority,
-                "duration": self.get_task_duration(ts),
+                "duration": duration,
             }
             if ts._resource_restrictions:
                 msg["resource_restrictions"] = ts._resource_restrictions
             if ts._actor:
                 msg["actor"] = True
 
-            deps = ts._dependencies
+            deps: set = ts._dependencies
             if deps:
-                ws: WorkerState
                 msg["who_has"] = {
                     dts._key: [ws._address for ws in dts._who_has] for dts in deps
                 }
                 msg["nbytes"] = {dts._key: dts._nbytes for dts in deps}
 
-            if self.validate and deps:
-                assert all(msg["who_has"].values())
+                if self.validate:
+                    assert all(msg["who_has"].values())
 
             task = ts._run_spec
             if type(task) is dict:
@@ -4733,7 +4735,7 @@ def transition_waiting_processing(self, key):
 
             # logger.debug("Send job to worker: %s, %s", worker, key)
 
-            self.send_task_to_worker(worker, key)
+            self.send_task_to_worker(worker, ts, duration)
 
             return {}
         except Exception as e:
diff --git a/distributed/stealing.py b/distributed/stealing.py
index 1fc26de4b55..f2f95e4b133 100644
--- a/distributed/stealing.py
+++ b/distributed/stealing.py
@@ -251,7 +251,7 @@ async def move_task_confirm(self, key=None, worker=None, state=None):
                 self.put_key_in_stealable(ts)
 
                 try:
-                    self.scheduler.send_task_to_worker(thief.address, key)
+                    self.scheduler.send_task_to_worker(thief.address, ts)
                 except CommClosedError:
                     await self.scheduler.remove_worker(thief.address)
                 self.log(("confirm", key, victim.address, thief.address))

From d54388c92f5cb6643180749f7d30f580609379a2 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 8 Dec 2020 16:23:14 -0800
Subject: [PATCH 1087/1550] Optionally use offload executor in worker (#4307)

Normally we have one separate thread pool executor for deserializing data
and another thread pool executor in a worker for execution.
Sometimes this presents a problem because some data types want to be
used in the thread in which they were created.  One example of this is
TensorFlow graphs, but there are others.

One way to resolve this is to reuse the same executor in both
situations, and ensure that it has only one thread.  This means that
execution and deserialization will block each other (not great) but that
user data will always be operated on in one thread only.

Co-authored-by: James Bourbeau <jrbourbeau@gmail.com>
---
 distributed/comm/utils.py        | 21 ++++++------------
 distributed/core.py              |  5 ++++-
 distributed/tests/test_worker.py | 28 +++++++++++++++++++++++
 distributed/worker.py            | 38 ++++++++++++++++++++------------
 4 files changed, 63 insertions(+), 29 deletions(-)

diff --git a/distributed/comm/utils.py b/distributed/comm/utils.py
index d1a1a97e63c..b3ac85feed8 100644
--- a/distributed/comm/utils.py
+++ b/distributed/comm/utils.py
@@ -14,11 +14,9 @@
 
 
 # Offload (de)serializing large frames to improve event loop responsiveness.
-# We use at most 4 threads to allow for parallel processing of large messages.
-
-FRAME_OFFLOAD_THRESHOLD = dask.config.get("distributed.comm.offload")
-if isinstance(FRAME_OFFLOAD_THRESHOLD, str):
-    FRAME_OFFLOAD_THRESHOLD = parse_bytes(FRAME_OFFLOAD_THRESHOLD)
+OFFLOAD_THRESHOLD = dask.config.get("distributed.comm.offload")
+if isinstance(OFFLOAD_THRESHOLD, str):
+    OFFLOAD_THRESHOLD = parse_bytes(OFFLOAD_THRESHOLD)
 
 
 async def to_frames(
@@ -40,7 +38,7 @@ def _to_frames():
             logger.exception(e)
             raise
 
-    if FRAME_OFFLOAD_THRESHOLD and allow_offload:
+    if OFFLOAD_THRESHOLD and allow_offload:
         try:
             msg_size = sizeof(msg)
         except RecursionError:
@@ -48,7 +46,7 @@ def _to_frames():
     else:
         msg_size = 0
 
-    if allow_offload and FRAME_OFFLOAD_THRESHOLD and msg_size > FRAME_OFFLOAD_THRESHOLD:
+    if allow_offload and OFFLOAD_THRESHOLD and msg_size > OFFLOAD_THRESHOLD:
         return await offload(_to_frames)
     else:
         return _to_frames()
@@ -74,14 +72,9 @@ def _from_frames():
             logger.error("truncated data stream (%d bytes): %s", size, datastr)
             raise
 
-    if allow_offload and deserialize and FRAME_OFFLOAD_THRESHOLD:
+    if allow_offload and deserialize and OFFLOAD_THRESHOLD:
         size = sum(map(nbytes, frames))
-    if (
-        allow_offload
-        and deserialize
-        and FRAME_OFFLOAD_THRESHOLD
-        and size > FRAME_OFFLOAD_THRESHOLD
-    ):
+    if allow_offload and deserialize and OFFLOAD_THRESHOLD and size > OFFLOAD_THRESHOLD:
         res = await offload(_from_frames)
     else:
         res = _from_frames()
diff --git a/distributed/core.py b/distributed/core.py
index 3eedef8faab..001f3bac31f 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -592,7 +592,10 @@ async def handle_stream(self, comm, extra=None, every_cycle=[]):
                     await asyncio.sleep(0)
 
                 for func in every_cycle:
-                    func()
+                    if is_coroutine_function(func):
+                        self.loop.add_callback(func)
+                    else:
+                        func()
 
         except (CommClosedError, EnvironmentError) as e:
             io_error = e
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index b41e174bf5f..fc758da2338 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -7,6 +7,7 @@
 import psutil
 import sys
 from time import sleep
+import threading
 import traceback
 from unittest import mock
 import asyncio
@@ -1755,3 +1756,30 @@ async def test_taskstate_metadata(cleanup):
 
                 # Check that Scheduler TaskState.metadata was also updated
                 assert s.tasks[f.key].metadata == ts.metadata
+
+
+@pytest.mark.asyncio
+async def test_executor_offload(cleanup, monkeypatch):
+    class SameThreadClass:
+        def __getstate__(self):
+            return ()
+
+        def __setstate__(self, state):
+            self._thread_ident = threading.get_ident()
+            return self
+
+    monkeypatch.setattr("distributed.worker.OFFLOAD_THRESHOLD", 1)
+
+    async with Scheduler() as s:
+        async with Worker(s.address, executor="offload") as w:
+            from distributed.utils import _offload_executor
+
+            assert w.executor is _offload_executor
+
+            async with Client(s.address, asynchronous=True) as c:
+                x = SameThreadClass()
+
+                def f(x):
+                    return threading.get_ident() == x._thread_ident
+
+                assert await c.submit(f, x)
diff --git a/distributed/worker.py b/distributed/worker.py
index f1896d40c81..205369623d1 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -32,6 +32,7 @@
 from .batched import BatchedSend
 from .comm import get_address_host, connect
 from .comm.addressing import address_from_user_args
+from .comm.utils import OFFLOAD_THRESHOLD
 from .core import error_message, CommClosedError, send_recv, pingpong, coerce_to_address
 from .diskutils import WorkSpace
 from .http import get_handlers
@@ -213,6 +214,9 @@ class Worker(ServerNode):
         Number of nthreads used by this worker process
     * **executor:** ``concurrent.futures.ThreadPoolExecutor``:
         Executor used to perform computation
+        This can also be the string "offload" in which case this uses the same
+        thread pool used for offloading communications.  This results in the
+        same thread being used for deserialization and computation.
     * **local_directory:** ``path``:
         Path on local machine to store temporary files
     * **scheduler:** ``rpc``:
@@ -610,6 +614,8 @@ def __init__(
         self.actors = {}
         self.loop = loop or IOLoop.current()
         self.reconnect = reconnect
+        if executor == "offload":
+            from distributed.utils import _offload_executor as executor
         self.executor = executor or ThreadPoolExecutor(
             self.nthreads, thread_name_prefix="Dask-Worker-Threads'"
         )
@@ -1761,7 +1767,7 @@ def transition_executing_long_running(self, ts, compute_duration=None):
                 }
             )
 
-            self.ensure_computing()
+            self.io_loop.add_callback(self.ensure_computing)
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
@@ -2126,7 +2132,7 @@ async def gather_dep(self, worker, dep, deps, total_nbytes, cause=None):
                 if self.validate:
                     self.validate_state()
 
-                self.ensure_computing()
+                await self.ensure_computing()
 
                 if not busy:
                     self.repetitively_busy = 0
@@ -2444,12 +2450,16 @@ def meets_resource_constraints(self, key):
 
         return True
 
-    def _maybe_deserialize_task(self, ts):
+    async def _maybe_deserialize_task(self, ts):
         if not isinstance(ts.runspec, SerializedTask):
             return ts.runspec
         try:
             start = time()
-            function, args, kwargs = _deserialize(*ts.runspec)
+            # Offload deserializing large tasks
+            if sizeof(ts.runspec) > OFFLOAD_THRESHOLD:
+                function, args, kwargs = await offload(_deserialize, *ts.runspec)
+            else:
+                function, args, kwargs = _deserialize(*ts.runspec)
             stop = time()
 
             if stop - start > 0.010:
@@ -2466,21 +2476,21 @@ def _maybe_deserialize_task(self, ts):
             self.log.append((ts.key, "deserialize-error"))
             raise
 
-    def ensure_computing(self):
+    async def ensure_computing(self):
         if self.paused:
             return
         try:
             while self.constrained and self.executing_count < self.nthreads:
                 key = self.constrained[0]
-                ts = self.tasks[key]
-                if ts.state != "constrained":
+                ts = self.tasks.get(key, None)
+                if ts is None or ts.state != "constrained":
                     self.constrained.popleft()
                     continue
                 if self.meets_resource_constraints(key):
                     self.constrained.popleft()
                     try:
                         # Ensure task is deserialized prior to execution
-                        ts.runspec = self._maybe_deserialize_task(ts)
+                        ts.runspec = await self._maybe_deserialize_task(ts)
                     except Exception:
                         continue
                     self.transition(ts, "executing")
@@ -2499,7 +2509,7 @@ def ensure_computing(self):
                 elif ts.state in READY:
                     try:
                         # Ensure task is deserialized prior to execution
-                        ts.runspec = self._maybe_deserialize_task(ts)
+                        ts.runspec = await self._maybe_deserialize_task(ts)
                     except Exception:
                         continue
                     self.transition(ts, "executing")
@@ -2613,7 +2623,7 @@ async def execute(self, key, report=False):
                 assert ts.state != "executing"
                 assert not ts.waiting_for_data
 
-            self.ensure_computing()
+            await self.ensure_computing()
             self.ensure_communicating()
         except Exception as e:
             if executor_error is e:
@@ -2646,7 +2656,7 @@ async def memory_monitor(self):
         memory = proc.memory_info().rss
         frac = memory / self.memory_limit
 
-        def check_pause(memory):
+        async def check_pause(memory):
             frac = memory / self.memory_limit
             # Pause worker threads if above 80% memory use
             if self.memory_pause_fraction and frac > self.memory_pause_fraction:
@@ -2674,9 +2684,9 @@ def check_pause(memory):
                     else "None",
                 )
                 self.paused = False
-                self.ensure_computing()
+                await self.ensure_computing()
 
-        check_pause(memory)
+        await check_pause(memory)
         # Dump data to disk if above 70%
         if self.memory_spill_fraction and frac > self.memory_spill_fraction:
             logger.debug(
@@ -2718,7 +2728,7 @@ def check_pause(memory):
                     # before trying to evict even more data.
                     self._throttled_gc.collect()
                     memory = proc.memory_info().rss
-            check_pause(memory)
+            await check_pause(memory)
             if count:
                 logger.debug(
                     "Moved %d pieces of data data and %s to disk",

From e78870caa61d2d81790d20a35c0c706b7550be46 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Wed, 9 Dec 2020 07:04:50 -0800
Subject: [PATCH 1088/1550] Use `bint` to annotate boolean attributes (#4334)

* Set `_actor` attribute to `False` not `None`

As `_actor` is expected to be a boolean, make sure it is set to `False`
instead of `None`.

* Use `bint` to annotate boolean attributes

As Cython treats `bool` as a Python object, use `bint` instead, which
Cython optimizes to effectively a boolean at the C level.
---
 distributed/scheduler.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index c2743dc9310..b5c5e5a5302 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -86,7 +86,7 @@
 from .protocol.highlevelgraph import highlevelgraph_unpack
 
 try:
-    from cython import cclass, double, Py_hash_t, Py_ssize_t
+    from cython import bint, cclass, double, Py_hash_t, Py_ssize_t
 except ImportError:
     from ctypes import (
         c_double as double,
@@ -94,6 +94,8 @@
         c_ssize_t as Py_ssize_t,
     )
 
+    bint = bool
+
     def cclass(cls):
         return cls
 
@@ -1034,7 +1036,7 @@ class TaskState:
     _state: str
     _dependencies: set
     _dependents: set
-    _has_lost_dependencies: bool
+    _has_lost_dependencies: bint
     _waiting_on: set
     _waiters: set
     _who_wants: set
@@ -1050,10 +1052,10 @@ class TaskState:
     _host_restrictions: set
     _worker_restrictions: set
     _resource_restrictions: dict
-    _loose_restrictions: bool
+    _loose_restrictions: bint
     _metadata: dict
     _annotations: dict
-    _actor: bool
+    _actor: bint
     _group: TaskGroup
     _group_key: str
 
@@ -1128,7 +1130,7 @@ def __init__(self, key: str, run_spec: object):
         self._worker_restrictions = None
         self._resource_restrictions = None
         self._loose_restrictions = False
-        self._actor = None
+        self._actor = False
         self._type = None
         self._group_key = key_split_group(key)
         self._group = None

From 427ee844f8d285f6efe26e5f209a57b5ca761921 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Wed, 9 Dec 2020 07:05:35 -0800
Subject: [PATCH 1089/1550] Handle `WorkerState` `memory_limit` of `None`
 (#4335)

If a `memory_limit` of `None` is passed to `add_worker`, make sure to
replace it with `0` before constructing `WorkerState` as `memory_limit`
in `WorkerState` is typed as `Py_ssize_t` so `None` is not permissible.
---
 distributed/scheduler.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index b5c5e5a5302..1b39d1d4d71 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -2133,7 +2133,7 @@ async def add_worker(
         now=None,
         resources=None,
         host_info=None,
-        memory_limit=0,
+        memory_limit=None,
         metrics=None,
         pid=0,
         services=None,
@@ -2166,7 +2166,7 @@ async def add_worker(
                 address=address,
                 pid=pid,
                 nthreads=nthreads,
-                memory_limit=memory_limit,
+                memory_limit=memory_limit or 0,
                 name=name,
                 local_directory=local_directory,
                 services=services,

From ee179eb586fcda4ce68e13e1b5270b376ee1d629 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Wed, 9 Dec 2020 07:08:38 -0800
Subject: [PATCH 1090/1550] Store occupancy in `_reevaluate_occupancy_worker`
 (#4337)

Instead of computing the sum of the task duration and communication cost
repeatedly, simply compute it once and assign it to a variable. Then
reuse it in later steps.
---
 distributed/scheduler.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 1b39d1d4d71..8c4e0be218d 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -5988,8 +5988,9 @@ def _reevaluate_occupancy_worker(self, ws: WorkerState):
         for ts in ws._processing:
             duration = self.get_task_duration(ts)
             comm = self.get_comm_cost(ts, ws)
-            ws._processing[ts] = duration + comm
-            new += duration + comm
+            occupancy = duration + comm
+            ws._processing[ts] = occupancy
+            new += occupancy
 
         ws._occupancy = new
         self.total_occupancy += new - old

From 3d53801f1ba7b2d9b59804548162efc3bc57f857 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Wed, 9 Dec 2020 07:25:06 -0800
Subject: [PATCH 1091/1550] Optimize `decide_worker` (#4332)

---
 distributed/scheduler.py | 49 ++++++++++++++++++++--------------------
 1 file changed, 25 insertions(+), 24 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 8c4e0be218d..5692f014ed3 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -4655,10 +4655,11 @@ def transition_no_worker_waiting(self, key):
                 pdb.set_trace()
             raise
 
-    def decide_worker(self, ts: TaskState):
+    def decide_worker(self, ts: TaskState) -> WorkerState:
         """
         Decide on a worker for task *ts*.  Return a WorkerState.
         """
+        ws: WorkerState = None
         valid_workers: set = self.valid_workers(ts)
 
         if (
@@ -4669,36 +4670,31 @@ def decide_worker(self, ts: TaskState):
         ):
             self.unrunnable.add(ts)
             ts.state = "no-worker"
-            return None
+            return ws
 
         if ts._dependencies or valid_workers is not None:
-            worker = decide_worker(
+            ws = decide_worker(
                 ts,
                 self.workers.values(),
                 valid_workers,
                 partial(self.worker_objective, ts),
             )
-        elif self.idle:
-            if len(self.idle) < 20:  # smart but linear in small case
-                worker = min(self.idle, key=operator.attrgetter("occupancy"))
-            else:  # dumb but fast in large case
-                worker = self.idle[self.n_tasks % len(self.idle)]
         else:
-            if len(self.workers) < 20:  # smart but linear in small case
-                worker = min(
-                    self.workers.values(), key=operator.attrgetter("occupancy")
-                )
+            worker_pool = self.idle or self.workers.values()
+            n_workers = len(worker_pool)
+            if n_workers < 20:  # smart but linear in small case
+                ws = min(worker_pool, key=operator.attrgetter("occupancy"))
             else:  # dumb but fast in large case
-                worker = self.workers.values()[self.n_tasks % len(self.workers)]
+                ws = worker_pool[self.n_tasks % n_workers]
 
         if self.validate:
-            assert worker is None or isinstance(worker, WorkerState), (
-                type(worker),
-                worker,
+            assert ws is None or isinstance(ws, WorkerState), (
+                type(ws),
+                ws,
             )
-            assert worker.address in self.workers
+            assert ws.address in self.workers
 
-        return worker
+        return ws
 
     def transition_waiting_processing(self, key):
         try:
@@ -6089,7 +6085,9 @@ def adaptive_target(self, comm=None, target_duration=None):
             return len(self.workers) - len(to_close)
 
 
-def decide_worker(ts: TaskState, all_workers, valid_workers: set, objective):
+def decide_worker(
+    ts: TaskState, all_workers, valid_workers: set, objective
+) -> WorkerState:
     """
     Decide which worker should take task *ts*.
 
@@ -6105,13 +6103,14 @@ def decide_worker(ts: TaskState, all_workers, valid_workers: set, objective):
     of bytes sent between workers.  This is determined by calling the
     *objective* function.
     """
+    ws: WorkerState
     dts: TaskState
-    deps = ts._dependencies
+    deps: set = ts._dependencies
+    candidates: set
     assert all([dts._who_has for dts in deps])
     if ts._actor:
         candidates = set(all_workers)
     else:
-        ws: WorkerState
         candidates = {ws for dts in deps for ws in dts._who_has}
     if valid_workers is None:
         if not candidates:
@@ -6129,9 +6128,11 @@ def decide_worker(ts: TaskState, all_workers, valid_workers: set, objective):
         return None
 
     if len(candidates) == 1:
-        return first(candidates)
-
-    return min(candidates, key=objective)
+        for ws in candidates:
+            break
+    else:
+        ws = min(candidates, key=objective)
+    return ws
 
 
 def validate_task_state(ts: TaskState):

From 64930284e3260299b7ccf8d55916c4b3be07acdf Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Wed, 9 Dec 2020 18:39:33 -0800
Subject: [PATCH 1092/1550] Optimize `assert` in `validate_waiting` (#4342)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Use `!=` between `bool`s for xor in Python

* Use `not not` instead of `bool`

This winds up being a faster way to test truthiness (even in pure
Python). Plus when Cython looks at this code, it can skip constructing a
Python `bool` object and simply operate on the results in C directly.

```python
In [1]: s0 = set()

In [2]: s1 = {1, 2, 3}

In [3]: %timeit bool(s0)
66.1 ns ± 0.123 ns per loop (mean ± std. dev. of 7 runs, 10000000 loops each)

In [4]: %timeit bool(s1)
70.5 ns ± 0.203 ns per loop (mean ± std. dev. of 7 runs, 10000000 loops each)

In [5]: %timeit (not not s0)
22.6 ns ± 0.25 ns per loop (mean ± std. dev. of 7 runs, 10000000 loops each)

In [6]: %timeit (not not s1)
28.3 ns ± 1.35 ns per loop (mean ± std. dev. of 7 runs, 10000000 loops each)
```
---
 distributed/scheduler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 5692f014ed3..0798d4d831a 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -2950,7 +2950,7 @@ def validate_waiting(self, key):
         assert ts not in self.unrunnable
         for dts in ts._dependencies:
             # We are waiting on a dependency iff it's not stored
-            assert bool(dts._who_has) + (dts in ts._waiting_on) == 1
+            assert (not not dts._who_has) != (dts in ts._waiting_on)
             assert ts in dts._waiters  # XXX even if dts._who_has?
 
     def validate_processing(self, key):

From 9771408af3a67ec7ef8986316e61d91737bec4d9 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Wed, 9 Dec 2020 18:41:09 -0800
Subject: [PATCH 1093/1550] Annotation of some comm related methods in the
 Scheduler (#4341)

* Annotate `stream_comms` in `worker_send`

Provides type annotation for `stream_comms` in the form of a locally
stored variable in `worker_send`. This should allow the Python C API to
used for lookups when the scheduler is Cythonized.

* Just use `msg.get("key")`

Instead of checking for `"key"` in `msg`, just use `.get(...)` to try
and retrieve the `key`. If it fails, we get `None`, which we can check
and ignore quickly. If not, we get the value and can proceed to use it.
Should avoid needing to check for the presence of `"key"` in `msg`
twice.

* Store `self.tasks` to locally annotated variable

Should allow more efficient access of `tasks` in Cython as it is known
to be a `dict`.

* Annotate `client_keys` as a `list` in `report`

* Set locally annotated var with `self.client_comms`

Should allow more efficient access of `client_comms` in Cython as it is
known to be a `dict`.

* Annotate `k` as `str`

* Use `dict.get(...)` instead of catching error

This ends up being a bit faster than catching the error (especially in
Cython/C where it knows this error can just be cleared instead of
handled). Plus the comparison to `None` is fast in both (in Cython/C
this is a simple pointer value comparison similar to comparing to
`NULL`).

* Annotate `report_on_key`'s function arguments

* Use `!=` to emulate `xor` in Python

* Store `self.tasks` to locally annotated variable

Should allow more efficient access of `tasks` in Cython as it is known
to be a `dict`.

* Use `dict.get(...)` instead of catching error

This ends up being a bit faster than catching the error (especially in
Cython/C where it knows this error can just be cleared instead of
handled). Plus the comparison to `None` is fast in both (in Cython/C
this is a simple pointer value comparison similar to comparing to
`NULL`).
---
 distributed/scheduler.py | 32 +++++++++++++++++++-------------
 1 file changed, 19 insertions(+), 13 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 0798d4d831a..e1867e60d67 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -3066,20 +3066,25 @@ def validate_state(self, allow_overlap=False):
     # Manage Messages #
     ###################
 
-    def report(self, msg, ts: TaskState = None, client=None):
+    def report(self, msg: dict, ts: TaskState = None, client: str = None):
         """
         Publish updates to all listening Queues and Comms
 
         If the message contains a key then we only send the message to those
         comms that care about the key.
         """
-        if ts is None and "key" in msg:
-            ts = self.tasks.get(msg["key"])
+        if ts is None:
+            msg_key = msg.get("key")
+            if msg_key is not None:
+                tasks: dict = self.tasks
+                ts = tasks.get(msg_key)
 
         cs: ClientState
+        client_comms: dict = self.client_comms
+        client_keys: list
         if ts is None:
             # Notify all clients
-            client_keys = list(self.client_comms)
+            client_keys = list(client_comms)
         elif client is None:
             # Notify clients interested in key
             client_keys = [cs._client_key for cs in ts._who_wants]
@@ -3090,10 +3095,10 @@ def report(self, msg, ts: TaskState = None, client=None):
             ]
             client_keys.append(client)
 
+        k: str
         for k in client_keys:
-            try:
-                c = self.client_comms[k]
-            except KeyError:
+            c = client_comms.get(k)
+            if c is None:
                 continue
             try:
                 c.send(msg)
@@ -3363,8 +3368,9 @@ def worker_send(self, worker, msg):
         This also handles connection failures by adding a callback to remove
         the worker on the next cycle.
         """
+        stream_comms: dict = self.stream_comms
         try:
-            self.stream_comms[worker].send(msg)
+            stream_comms[worker].send(msg)
         except (CommClosedError, AttributeError):
             self.loop.add_callback(self.remove_worker, address=worker)
 
@@ -4194,12 +4200,12 @@ def update_data(
             if client:
                 self.client_desires_keys(keys=list(who_has), client=client)
 
-    def report_on_key(self, key=None, ts: TaskState = None, client=None):
-        assert (key is None) + (ts is None) == 1, (key, ts)
+    def report_on_key(self, key: str = None, ts: TaskState = None, client: str = None):
+        assert (key is None) != (ts is None), (key, ts)
         if ts is None:
-            try:
-                ts = self.tasks[key]
-            except KeyError:
+            tasks: dict = self.tasks
+            ts = tasks.get(key)
+            if ts is None:
                 self.report({"op": "cancelled-key", "key": key}, client=client)
                 return
         else:

From 997157aae7d8d49ca54665f8d905a85f19a4d8ca Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 9 Dec 2020 18:42:04 -0800
Subject: [PATCH 1094/1550] Log duplciate workers in scheduler (#4338)

---
 distributed/scheduler.py            | 3 +++
 distributed/tests/test_scheduler.py | 9 +++++++--
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index e1867e60d67..3c2f7d97498 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -2153,6 +2153,9 @@ async def add_worker(
                 raise ValueError("Worker already exists %s" % ws)
 
             if name in self.aliases:
+                logger.warning(
+                    "Worker tried to connect with a duplicate name: %s", name
+                )
                 msg = {
                     "status": "error",
                     "message": "name taken, %s" % name,
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 41ecb48990b..ec44ca85560 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -2136,13 +2136,18 @@ async def test_worker_name_collision(s, a):
     # test that a name collision for workers produces the expected respsone
     # and leaves the data structures of Scheduler in a good state
     # is not updated by the second worker
-    with pytest.raises(ValueError, match=f"name taken, {a.name!r}"):
-        await Worker(s.address, name=a.name, loop=s.loop, host="127.0.0.1")
+    with captured_logger(logging.getLogger("distributed.scheduler")) as log:
+        with pytest.raises(ValueError, match=f"name taken, {a.name!r}"):
+            await Worker(s.address, name=a.name, loop=s.loop, host="127.0.0.1")
 
     s.validate_state()
     assert set(s.workers) == {a.address}
     assert s.aliases == {a.name: a.address}
 
+    log = log.getvalue()
+    assert "duplicate" in log
+    assert str(a.name) in log
+
 
 @gen_cluster(client=True, config={"distributed.scheduler.unknown-task-duration": "1h"})
 async def test_unknown_task_duration_config(client, s, a, b):

From 216b53ff5aebd1b5e2d71f52d6a687a1bafa8947 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Thu, 10 Dec 2020 04:08:20 -0800
Subject: [PATCH 1095/1550] Drop serialize extension (#4344)

This disables Cythonizing `serialize`. While we have annotated one
function in that module, most of `serialize` is not Cythonized.
Additionally there are some `ImportError`s in development mode that have
been getting in the way. So suggest we just drop this for now and
revisit if this becomes more important in the future.
---
 setup.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/setup.py b/setup.py
index 6cc1e471cbf..7c91a44bdde 100755
--- a/setup.py
+++ b/setup.py
@@ -39,10 +39,6 @@
             "distributed.scheduler",
             sources=["distributed/scheduler.py"],
         ),
-        Extension(
-            "distributed.protocol.serialize",
-            sources=["distributed/protocol/serialize.py"],
-        ),
     ]
     for e in cyext_modules:
         e.cython_directives = {

From c6d6fff1369e7d4bf300542a8c410745cb946aaf Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Thu, 10 Dec 2020 15:27:27 -0800
Subject: [PATCH 1096/1550] Some common optimizations across transitions
 (#4348)

* Overwrite `s` in non-`None` `valid_workers` case

* Annotate some variables in `_propagate_forgotten`

* Type `recommendations` as `dict`

* Assign `task` to locally typed variable

* Use `not not` for faster, C amenable truthiness

As shown before `not not` is faster than `bool`. Plus Cython is able to
use the result from `not not` directly in C without having to convert
back to a Python object as is the case with `bool`. Thus resulting in
faster comparisons in the C generated by Cython.

* Annotate local variables in `_add_to_memory`

* Coerce `ts._dependents` to a `list`

This is what `sorted` will do anyways. Plus it simplifies the logic on
Cython later when constructing the `for`-loop over `deps`.

* Use faster `state` attribute access on `TaskState`
---
 distributed/scheduler.py | 237 +++++++++++++++++++++------------------
 1 file changed, 130 insertions(+), 107 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 3c2f7d97498..55baa274395 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -768,7 +768,7 @@ def types(self):
 
     def add(self, o):
         ts: TaskState = o
-        self._states[ts.state] += 1
+        self._states[ts._state] += 1
         ts._group = self
 
     def __repr__(self):
@@ -2215,10 +2215,12 @@ async def add_worker(
                 except Exception as e:
                     logger.exception(e)
 
+            recommendations: dict
             if nbytes:
                 for key in nbytes:
-                    ts: TaskState = self.tasks.get(key)
-                    if ts is not None and ts.state in ("processing", "waiting"):
+                    tasks: dict = self.tasks
+                    ts: TaskState = tasks.get(key)
+                    if ts is not None and ts._state in ("processing", "waiting"):
                         recommendations = self.transition(
                             key,
                             "memory",
@@ -2375,7 +2377,7 @@ def update_graph(
         for k, v in dependencies.items():
             if v and k in self.tasks:
                 ts = self.tasks[k]
-                if ts.state in ("memory", "erred"):
+                if ts._state in ("memory", "erred"):
                     already_in_memory.add(k)
 
         dts: TaskState
@@ -2547,10 +2549,10 @@ def update_graph(
                 ts._retries = v
 
         # Compute recommendations
-        recommendations = {}
+        recommendations: dict = {}
 
         for ts in sorted(runnables, key=operator.attrgetter("priority"), reverse=True):
-            if ts.state == "released" and ts._run_spec:
+            if ts._state == "released" and ts._run_spec:
                 recommendations[ts._key] = "waiting"
 
         for ts in touched_tasks:
@@ -2580,7 +2582,7 @@ def update_graph(
         self.transitions(recommendations)
 
         for ts in touched_tasks:
-            if ts.state in ("memory", "erred"):
+            if ts._state in ("memory", "erred"):
                 self.report_on_key(ts._key, client=client)
 
         end = time()
@@ -2623,17 +2625,18 @@ def stimulus_task_finished(self, key=None, worker=None, **kwargs):
         ws: WorkerState = self.workers[worker]
         ts._metadata.update(kwargs["metadata"])
 
-        if ts.state == "processing":
+        recommendations: dict
+        if ts._state == "processing":
             recommendations = self.transition(key, "memory", worker=worker, **kwargs)
 
-            if ts.state == "memory":
+            if ts._state == "memory":
                 assert ws in ts._who_has
         else:
             logger.debug(
                 "Received already computed task, worker: %s, state: %s"
                 ", key: %s, who_has: %s",
                 worker,
-                ts.state,
+                ts._state,
                 key,
                 ts._who_has,
             )
@@ -2653,7 +2656,8 @@ def stimulus_task_erred(
         if ts is None:
             return {}
 
-        if ts.state == "processing":
+        recommendations: dict
+        if ts._state == "processing":
             retries = ts._retries
             if retries > 0:
                 ts._retries = retries - 1
@@ -2681,13 +2685,13 @@ def stimulus_missing_data(
             logger.debug("Stimulus missing data %s, %s", key, worker)
 
             ts: TaskState = self.tasks.get(key)
-            if ts is None or ts.state == "memory":
+            if ts is None or ts._state == "memory":
                 return {}
             cts: TaskState = self.tasks.get(cause)
 
-            recommendations = {}
+            recommendations: dict = {}
 
-            if cts is not None and cts.state == "memory":  # couldn't find this
+            if cts is not None and cts._state == "memory":  # couldn't find this
                 ws: WorkerState
                 for ws in cts._who_has:  # TODO: this behavior is extreme
                     ws._has_what.remove(cts)
@@ -2719,13 +2723,13 @@ def stimulus_retry(self, comm=None, keys=None, client=None):
             key = stack.pop()
             seen.add(key)
             ts = self.tasks[key]
-            erred_deps = [dts._key for dts in ts._dependencies if dts.state == "erred"]
+            erred_deps = [dts._key for dts in ts._dependencies if dts._state == "erred"]
             if erred_deps:
                 stack.extend(erred_deps)
             else:
                 roots.append(key)
 
-        recommendations = {key: "waiting" for key in roots}
+        recommendations: dict = {key: "waiting" for key in roots}
         self.transitions(recommendations)
 
         if self.validate:
@@ -2786,7 +2790,7 @@ async def remove_worker(self, comm=None, address=None, safe=False, close=True):
             ws.status = Status.closed
             self.total_occupancy -= ws._occupancy
 
-            recommendations = {}
+            recommendations: dict = {}
 
             ts: TaskState
             for ts in list(ws._processing):
@@ -2896,7 +2900,7 @@ def client_desires_keys(self, keys=None, client=None):
             ts._who_wants.add(cs)
             cs._wants_what.add(ts)
 
-            if ts.state in ("memory", "erred"):
+            if ts._state in ("memory", "erred"):
                 self.report_on_key(k, client=client)
 
     def client_releases_keys(self, keys=None, client=None):
@@ -2914,12 +2918,12 @@ def client_releases_keys(self, keys=None, client=None):
                 if not s:
                     tasks2.add(ts)
 
-        recommendations = {}
+        recommendations: dict = {}
         for ts in tasks2:
             if not ts._dependents:
                 # No live dependents, can forget
                 recommendations[ts._key] = "forgotten"
-            elif ts.state != "erred" and not ts._waiters:
+            elif ts._state != "erred" and not ts._waiters:
                 recommendations[ts._key] = "released"
 
         self.transitions(recommendations)
@@ -2936,7 +2940,7 @@ def client_heartbeat(self, client=None):
     def validate_released(self, key):
         ts: TaskState = self.tasks[key]
         dts: TaskState
-        assert ts.state == "released"
+        assert ts._state == "released"
         assert not ts._waiters
         assert not ts._waiting_on
         assert not ts._who_has
@@ -2976,7 +2980,7 @@ def validate_memory(self, key):
         assert not ts._waiting_on
         assert ts not in self.unrunnable
         for dts in ts._dependents:
-            assert (dts in ts._waiters) == (dts.state in ("waiting", "processing"))
+            assert (dts in ts._waiters) == (dts._state in ("waiting", "processing"))
             assert ts not in dts._waiting_on
 
     def validate_no_worker(self, key):
@@ -3004,10 +3008,10 @@ def validate_key(self, key, ts: TaskState = None):
             else:
                 ts.validate()
                 try:
-                    func = getattr(self, "validate_" + ts.state.replace("-", "_"))
+                    func = getattr(self, "validate_" + ts._state.replace("-", "_"))
                 except AttributeError:
                     logger.error(
-                        "self.validate_%s not found", ts.state.replace("-", "_")
+                        "self.validate_%s not found", ts._state.replace("-", "_")
                     )
                 else:
                     func(key)
@@ -3286,7 +3290,7 @@ def release_worker_data(self, comm=None, keys=None, worker=None):
         ws._has_what -= removed_tasks
 
         ts: TaskState
-        recommendations = {}
+        recommendations: dict = {}
         for ts in removed_tasks:
             ws._nbytes -= ts.get_nbytes()
             wh = ts._who_has
@@ -3761,7 +3765,7 @@ async def rebalance(self, comm=None, keys=None, workers=None):
                     }
 
                 for sender, recipient, ts in msgs:
-                    assert ts.state == "memory"
+                    assert ts._state == "memory"
                     ts._who_has.add(recipient)
                     recipient.has_what.add(ts)
                     recipient.nbytes += ts.get_nbytes()
@@ -3853,7 +3857,7 @@ async def replicate(
             while tasks:
                 gathers = defaultdict(dict)
                 for ts in list(tasks):
-                    if ts.state == "forgotten":
+                    if ts._state == "forgotten":
                         # task is no longer needed by any client or dependant task
                         tasks.remove(ts)
                         continue
@@ -4155,7 +4159,7 @@ def add_keys(self, comm=None, worker=None, keys=()):
         ws: WorkerState = self.workers[worker]
         for key in keys:
             ts: TaskState = self.tasks.get(key)
-            if ts is not None and ts.state == "memory":
+            if ts is not None and ts._state == "memory":
                 if ts not in ws._has_what:
                     ws._nbytes += ts.get_nbytes()
                     ws._has_what.add(ts)
@@ -4213,11 +4217,11 @@ def report_on_key(self, key: str = None, ts: TaskState = None, client: str = Non
                 return
         else:
             key = ts._key
-        if ts.state == "forgotten":
+        if ts._state == "forgotten":
             self.report({"op": "cancelled-key", "key": key}, ts=ts, client=client)
-        elif ts.state == "memory":
+        elif ts._state == "memory":
             self.report({"op": "key-in-memory", "key": key}, ts=ts, client=client)
-        elif ts.state == "erred":
+        elif ts._state == "erred":
             failing_ts: TaskState = ts._exception_blame
             self.report(
                 {
@@ -4344,9 +4348,9 @@ async def get_call_stack(self, comm=None, keys=None):
             while stack:
                 key = stack.pop()
                 ts = self.tasks[key]
-                if ts.state == "waiting":
+                if ts._state == "waiting":
                     stack.extend([dts._key for dts in ts._dependencies])
-                elif ts.state == "processing":
+                elif ts._state == "processing":
                     processing.add(ts)
 
             workers = defaultdict(list)
@@ -4518,7 +4522,7 @@ def _add_to_memory(
         self,
         ts: TaskState,
         ws: WorkerState,
-        recommendations,
+        recommendations: dict,
         type=None,
         typename=None,
         **kwargs,
@@ -4533,10 +4537,12 @@ def _add_to_memory(
         ws._has_what.add(ts)
         ws._nbytes += ts.get_nbytes()
 
-        deps = ts._dependents
+        deps: list = list(ts._dependents)
         if len(deps) > 1:
-            deps = sorted(deps, key=operator.attrgetter("priority"), reverse=True)
+            deps.sort(key=operator.attrgetter("priority"), reverse=True)
+
         dts: TaskState
+        s: set
         for dts in deps:
             s = dts._waiting_on
             if ts in s:
@@ -4553,7 +4559,7 @@ def _add_to_memory(
         if not ts._waiters and not ts._who_wants:
             recommendations[ts._key] = "released"
         else:
-            msg = {"op": "key-in-memory", "key": ts._key}
+            msg: dict = {"op": "key-in-memory", "key": ts._key}
             if type is not None:
                 msg["type"] = type
             self.report(msg)
@@ -4568,7 +4574,8 @@ def _add_to_memory(
 
     def transition_released_waiting(self, key):
         try:
-            ts: TaskState = self.tasks[key]
+            tasks: dict = self.tasks
+            ts: TaskState = tasks[key]
             dts: TaskState
 
             if self.validate:
@@ -4576,14 +4583,14 @@ def transition_released_waiting(self, key):
                 assert not ts._waiting_on
                 assert not ts._who_has
                 assert not ts._processing_on
-                assert not any([dts.state == "forgotten" for dts in ts._dependencies])
+                assert not any([dts._state == "forgotten" for dts in ts._dependencies])
 
             if ts._has_lost_dependencies:
                 return {key: "forgotten"}
 
             ts.state = "waiting"
 
-            recommendations = {}
+            recommendations: dict = {}
 
             dts: TaskState
             for dts in ts._dependencies:
@@ -4596,12 +4603,12 @@ def transition_released_waiting(self, key):
                 dep = dts._key
                 if not dts._who_has:
                     ts._waiting_on.add(dts)
-                if dts.state == "released":
+                if dts._state == "released":
                     recommendations[dep] = "waiting"
                 else:
                     dts._waiters.add(ts)
 
-            ts._waiters = {dts for dts in ts._dependents if dts.state == "waiting"}
+            ts._waiters = {dts for dts in ts._dependents if dts._state == "waiting"}
 
             if not ts._waiting_on:
                 if self.workers:
@@ -4621,7 +4628,8 @@ def transition_released_waiting(self, key):
 
     def transition_no_worker_waiting(self, key):
         try:
-            ts: TaskState = self.tasks[key]
+            tasks: dict = self.tasks
+            ts: TaskState = tasks[key]
             dts: TaskState
 
             if self.validate:
@@ -4635,13 +4643,13 @@ def transition_no_worker_waiting(self, key):
             if ts._has_lost_dependencies:
                 return {key: "forgotten"}
 
-            recommendations = {}
+            recommendations: dict = {}
 
             for dts in ts._dependencies:
                 dep = dts._key
                 if not dts._who_has:
                     ts._waiting_on.add(dts)
-                if dts.state == "released":
+                if dts._state == "released":
                     recommendations[dep] = "waiting"
                 else:
                     dts._waiters.add(ts)
@@ -4707,7 +4715,8 @@ def decide_worker(self, ts: TaskState) -> WorkerState:
 
     def transition_waiting_processing(self, key):
         try:
-            ts: TaskState = self.tasks[key]
+            tasks: dict = self.tasks
+            ts: TaskState = tasks[key]
             dts: TaskState
 
             if self.validate:
@@ -4756,12 +4765,13 @@ def transition_waiting_processing(self, key):
     def transition_waiting_memory(self, key, nbytes=None, worker=None, **kwargs):
         try:
             ws: WorkerState = self.workers[worker]
-            ts: TaskState = self.tasks[key]
+            tasks: dict = self.tasks
+            ts: TaskState = tasks[key]
 
             if self.validate:
                 assert not ts._processing_on
                 assert ts._waiting_on
-                assert ts.state == "waiting"
+                assert ts._state == "waiting"
 
             ts._waiting_on.clear()
 
@@ -4770,7 +4780,7 @@ def transition_waiting_memory(self, key, nbytes=None, worker=None, **kwargs):
 
             self.check_idle_saturated(ws)
 
-            recommendations = {}
+            recommendations: dict = {}
 
             self._add_to_memory(ts, ws, recommendations, **kwargs)
 
@@ -4801,7 +4811,8 @@ def transition_processing_memory(
         ws: WorkerState
         wws: WorkerState
         try:
-            ts: TaskState = self.tasks[key]
+            tasks: dict = self.tasks
+            ts: TaskState = tasks[key]
             assert worker
             assert isinstance(worker, str)
 
@@ -4812,7 +4823,7 @@ def transition_processing_memory(
                 assert not ts._waiting_on
                 assert not ts._who_has, (ts, ts._who_has)
                 assert not ts._exception_blame
-                assert ts.state == "processing"
+                assert ts._state == "processing"
 
             ws = self.workers.get(worker)
             if ws is None:
@@ -4879,7 +4890,7 @@ def transition_processing_memory(
             if nbytes is not None:
                 ts.set_nbytes(nbytes)
 
-            recommendations = {}
+            recommendations: dict = {}
 
             self._remove_from_processing(ts)
 
@@ -4901,7 +4912,8 @@ def transition_processing_memory(
     def transition_memory_released(self, key, safe=False):
         ws: WorkerState
         try:
-            ts: TaskState = self.tasks[key]
+            tasks: dict = self.tasks
+            ts: TaskState = tasks[key]
             dts: TaskState
 
             if self.validate:
@@ -4918,12 +4930,12 @@ def transition_memory_released(self, key, safe=False):
                     ts._exception = "Worker holding Actor was lost"
                     return {ts._key: "erred"}  # don't try to recreate
 
-            recommendations = {}
+            recommendations: dict = {}
 
             for dts in ts._waiters:
-                if dts.state in ("no-worker", "processing"):
+                if dts._state in ("no-worker", "processing"):
                     recommendations[dts._key] = "waiting"
-                elif dts.state == "waiting":
+                elif dts._state == "waiting":
                     dts._waiting_on.add(ts)
 
             # XXX factor this out?
@@ -4961,7 +4973,8 @@ def transition_memory_released(self, key, safe=False):
 
     def transition_released_erred(self, key):
         try:
-            ts: TaskState = self.tasks[key]
+            tasks: dict = self.tasks
+            ts: TaskState = tasks[key]
             dts: TaskState
             failing_ts: TaskState
 
@@ -4972,7 +4985,7 @@ def transition_released_erred(self, key):
                     assert not ts._waiting_on
                     assert not ts._waiters
 
-            recommendations = {}
+            recommendations: dict = {}
 
             failing_ts = ts._exception_blame
 
@@ -5004,25 +5017,26 @@ def transition_released_erred(self, key):
 
     def transition_erred_released(self, key):
         try:
-            ts: TaskState = self.tasks[key]
+            tasks: dict = self.tasks
+            ts: TaskState = tasks[key]
             dts: TaskState
 
             if self.validate:
                 with log_errors(pdb=LOG_PDB):
-                    assert all([dts.state != "erred" for dts in ts._dependencies])
+                    assert all([dts._state != "erred" for dts in ts._dependencies])
                     assert ts._exception_blame
                     assert not ts._who_has
                     assert not ts._waiting_on
                     assert not ts._waiters
 
-            recommendations = {}
+            recommendations: dict = {}
 
             ts._exception = None
             ts._exception_blame = None
             ts._traceback = None
 
             for dts in ts._dependents:
-                if dts.state == "erred":
+                if dts._state == "erred":
                     recommendations[dts._key] = "waiting"
 
             self.report({"op": "task-retried", "key": key})
@@ -5039,13 +5053,14 @@ def transition_erred_released(self, key):
 
     def transition_waiting_released(self, key):
         try:
-            ts: TaskState = self.tasks[key]
+            tasks: dict = self.tasks
+            ts: TaskState = tasks[key]
 
             if self.validate:
                 assert not ts._who_has
                 assert not ts._processing_on
 
-            recommendations = {}
+            recommendations: dict = {}
 
             dts: TaskState
             for dts in ts._dependencies:
@@ -5076,7 +5091,8 @@ def transition_waiting_released(self, key):
 
     def transition_processing_released(self, key):
         try:
-            ts: TaskState = self.tasks[key]
+            tasks: dict = self.tasks
+            ts: TaskState = tasks[key]
             dts: TaskState
 
             if self.validate:
@@ -5091,7 +5107,7 @@ def transition_processing_released(self, key):
 
             ts.state = "released"
 
-            recommendations = {}
+            recommendations: dict = {}
 
             if ts._has_lost_dependencies:
                 recommendations[key] = "forgotten"
@@ -5100,7 +5116,7 @@ def transition_processing_released(self, key):
 
             if recommendations.get(key) != "waiting":
                 for dts in ts._dependencies:
-                    if dts.state != "released":
+                    if dts._state != "released":
                         s = dts._waiters
                         s.discard(ts)
                         if not s and not dts._who_wants:
@@ -5124,7 +5140,8 @@ def transition_processing_erred(
     ):
         ws: WorkerState
         try:
-            ts: TaskState = self.tasks[key]
+            tasks: dict = self.tasks
+            ts: TaskState = tasks[key]
             dts: TaskState
             failing_ts: TaskState
 
@@ -5150,7 +5167,7 @@ def transition_processing_erred(
             else:
                 failing_ts = ts._exception_blame
 
-            recommendations = {}
+            recommendations: dict = {}
 
             for dts in ts._dependents:
                 dts._exception_blame = failing_ts
@@ -5193,7 +5210,8 @@ def transition_processing_erred(
 
     def transition_no_worker_released(self, key):
         try:
-            ts: TaskState = self.tasks[key]
+            tasks: dict = self.tasks
+            ts: TaskState = tasks[key]
             dts: TaskState
 
             if self.validate:
@@ -5219,8 +5237,9 @@ def transition_no_worker_released(self, key):
             raise
 
     def remove_key(self, key):
-        ts: TaskState = self.tasks.pop(key)
-        assert ts.state == "forgotten"
+        tasks: dict = self.tasks
+        ts: TaskState = tasks.pop(key)
+        assert ts._state == "forgotten"
         self.unrunnable.discard(ts)
         cs: ClientState
         for cs in ts._who_wants:
@@ -5230,15 +5249,15 @@ def remove_key(self, key):
         ts._exception_blame = ts._exception = ts._traceback = None
         self.task_metadata.pop(key, None)
 
-    def _propagate_forgotten(self, ts: TaskState, recommendations):
+    def _propagate_forgotten(self, ts: TaskState, recommendations: dict):
         ts.state = "forgotten"
-        key = ts._key
+        key: str = ts._key
         dts: TaskState
         for dts in ts._dependents:
             dts._has_lost_dependencies = True
             dts._dependencies.remove(ts)
             dts._waiting_on.discard(ts)
-            if dts.state not in ("memory", "erred"):
+            if dts._state not in ("memory", "erred"):
                 # Cannot compute task anymore
                 recommendations[dts._key] = "forgotten"
         ts._dependents.clear()
@@ -5246,7 +5265,7 @@ def _propagate_forgotten(self, ts: TaskState, recommendations):
 
         for dts in ts._dependencies:
             dts._dependents.remove(ts)
-            s = dts._waiters
+            s: set = dts._waiters
             s.discard(ts)
             if not dts._dependents and not dts._who_wants:
                 # Task not needed anymore
@@ -5262,7 +5281,7 @@ def _propagate_forgotten(self, ts: TaskState, recommendations):
         for ws in ts._who_has:
             ws._has_what.remove(ts)
             ws._nbytes -= ts.get_nbytes()
-            w = ws._address
+            w: str = ws._address
             if w in self.workers:  # in case worker has died
                 self.worker_send(
                     w, {"op": "delete-data", "keys": [key], "report": False}
@@ -5270,12 +5289,14 @@ def _propagate_forgotten(self, ts: TaskState, recommendations):
         ts._who_has.clear()
 
     def transition_memory_forgotten(self, key):
+        tasks: dict
         ws: WorkerState
         try:
-            ts: TaskState = self.tasks[key]
+            tasks = self.tasks
+            ts: TaskState = tasks[key]
 
             if self.validate:
-                assert ts.state == "memory"
+                assert ts._state == "memory"
                 assert not ts._processing_on
                 assert not ts._waiting_on
                 if not ts._run_spec:
@@ -5290,7 +5311,7 @@ def transition_memory_forgotten(self, key):
                 else:
                     assert 0, (ts,)
 
-            recommendations = {}
+            recommendations: dict = {}
 
             if ts._actor:
                 for ws in ts._who_has:
@@ -5312,10 +5333,11 @@ def transition_memory_forgotten(self, key):
 
     def transition_released_forgotten(self, key):
         try:
-            ts: TaskState = self.tasks[key]
+            tasks: dict = self.tasks
+            ts: TaskState = tasks[key]
 
             if self.validate:
-                assert ts.state in ("released", "erred")
+                assert ts._state in ("released", "erred")
                 assert not ts._who_has
                 assert not ts._processing_on
                 assert not ts._waiting_on, (ts, ts._waiting_on)
@@ -5331,7 +5353,7 @@ def transition_released_forgotten(self, key):
                 else:
                     assert 0, (ts,)
 
-            recommendations = {}
+            recommendations: dict = {}
             self._propagate_forgotten(ts, recommendations)
 
             self.report_on_key(ts=ts)
@@ -5368,7 +5390,7 @@ def transition(self, key, finish, *args, **kwargs):
                 ts = self.tasks[key]
             except KeyError:
                 return {}
-            start = ts.state
+            start = ts._state
             if start == finish:
                 return {}
 
@@ -5376,6 +5398,7 @@ def transition(self, key, finish, *args, **kwargs):
                 dependents = set(ts._dependents)
                 dependencies = set(ts._dependencies)
 
+            recommendations: dict = {}
             if (start, finish) in self._transitions:
                 func = self._transitions[start, finish]
                 recommendations = func(key, *args, **kwargs)
@@ -5395,7 +5418,7 @@ def transition(self, key, finish, *args, **kwargs):
                     "Impossible transition from %r to %r" % (start, finish)
                 )
 
-            finish2 = ts.state
+            finish2 = ts._state
             self.transition_log.append((key, start, finish2, recommendations, time()))
             if self.validate:
                 logger.debug(
@@ -5403,12 +5426,12 @@ def transition(self, key, finish, *args, **kwargs):
                     key,
                     start,
                     finish2,
-                    ts.state,
+                    ts._state,
                     dict(recommendations),
                 )
             if self.plugins:
                 # Temporarily put back forgotten key for plugin to retrieve it
-                if ts.state == "forgotten":
+                if ts._state == "forgotten":
                     try:
                         ts._dependents = dependents
                         ts._dependencies = dependencies
@@ -5420,10 +5443,10 @@ def transition(self, key, finish, *args, **kwargs):
                         plugin.transition(key, start, finish2, *args, **kwargs)
                     except Exception:
                         logger.info("Plugin failed with exception", exc_info=True)
-                if ts.state == "forgotten":
+                if ts._state == "forgotten":
                     del self.tasks[ts._key]
 
-            if ts.state == "forgotten" and ts._group._name in self.task_groups:
+            if ts._state == "forgotten" and ts._group._name in self.task_groups:
                 # Remove TaskGroup if all tasks are in the forgotten state
                 tg: TaskGroup = ts._group
                 if not any(tg._states.get(s) for s in ALL_TASK_STATES):
@@ -5439,7 +5462,7 @@ def transition(self, key, finish, *args, **kwargs):
                 pdb.set_trace()
             raise
 
-    def transitions(self, recommendations):
+    def transitions(self, recommendations: dict):
         """Process transitions until none are left
 
         This includes feedback from previous transitions and continues until we
@@ -5481,7 +5504,7 @@ def reschedule(self, key=None, worker=None):
                 "found on the scheduler. Aborting reschedule.".format(key)
             )
             return
-        if ts.state != "processing":
+        if ts._state != "processing":
             return
         if worker and ts._processing_on.address != worker:
             return
@@ -5575,10 +5598,10 @@ def valid_workers(self, ts: TaskState) -> set:
             else:
                 s &= ww
 
-        if s is None:
-            return s
-        else:
-            return {self.workers[w] for w in s}
+        if s is not None:
+            s = {self.workers[w] for w in s}
+
+        return s
 
     def consume_resources(self, ts: TaskState, ws: WorkerState):
         if ts._resource_restrictions:
@@ -6151,7 +6174,7 @@ def validate_task_state(ts: TaskState):
     ws: WorkerState
     dts: TaskState
 
-    assert ts.state in ALL_TASK_STATES or ts.state == "forgotten", ts
+    assert ts._state in ALL_TASK_STATES or ts._state == "forgotten", ts
 
     if ts._waiting_on:
         assert ts._waiting_on.issubset(ts._dependencies), (
@@ -6168,7 +6191,7 @@ def validate_task_state(ts: TaskState):
 
     for dts in ts._waiting_on:
         assert not dts._who_has, ("waiting on in-memory dep", str(ts), str(dts))
-        assert dts.state != "released", ("waiting on released dep", str(ts), str(dts))
+        assert dts._state != "released", ("waiting on released dep", str(ts), str(dts))
     for dts in ts._dependencies:
         assert ts in dts._dependents, (
             "not in dependency's dependents",
@@ -6176,16 +6199,16 @@ def validate_task_state(ts: TaskState):
             str(dts),
             str(dts._dependents),
         )
-        if ts.state in ("waiting", "processing"):
+        if ts._state in ("waiting", "processing"):
             assert dts in ts._waiting_on or dts._who_has, (
                 "dep missing",
                 str(ts),
                 str(dts),
             )
-        assert dts.state != "forgotten"
+        assert dts._state != "forgotten"
 
     for dts in ts._waiters:
-        assert dts.state in ("waiting", "processing"), (
+        assert dts._state in ("waiting", "processing"), (
             "waiter not in play",
             str(ts),
             str(dts),
@@ -6197,12 +6220,12 @@ def validate_task_state(ts: TaskState):
             str(dts),
             str(dts._dependencies),
         )
-        assert dts.state != "forgotten"
+        assert dts._state != "forgotten"
 
-    assert (ts._processing_on is not None) == (ts.state == "processing")
-    assert bool(ts._who_has) == (ts.state == "memory"), (ts, ts._who_has)
+    assert (ts._processing_on is not None) == (ts._state == "processing")
+    assert (not not ts._who_has) == (ts._state == "memory"), (ts, ts._who_has)
 
-    if ts.state == "processing":
+    if ts._state == "processing":
         assert all([dts._who_has for dts in ts._dependencies]), (
             "task processing without all deps",
             str(ts),
@@ -6239,9 +6262,9 @@ def validate_task_state(ts: TaskState):
             )
 
     if ts._actor:
-        if ts.state == "memory":
+        if ts._state == "memory":
             assert sum([ts in ws._actors for ws in ts._who_has]) == 1
-        if ts.state == "processing":
+        if ts._state == "processing":
             assert ts in ts._processing_on.actors
 
 
@@ -6256,7 +6279,7 @@ def validate_worker_state(ws: WorkerState):
         )
 
     for ts in ws._actors:
-        assert ts.state in ("memory", "processing")
+        assert ts._state in ("memory", "processing")
 
 
 def validate_state(tasks, workers, clients):

From e9bd27aa4def9a4af46fd879c401e848b40c299a Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Thu, 10 Dec 2020 20:08:13 -0600
Subject: [PATCH 1097/1550] 2020.12.0 release (#4349)

---
 docs/requirements.txt     |   4 +-
 docs/source/changelog.rst | 118 ++++++++++++++++++++++++++++++++++++++
 requirements.txt          |   4 +-
 3 files changed, 120 insertions(+), 6 deletions(-)

diff --git a/docs/requirements.txt b/docs/requirements.txt
index a1f19502f04..e5cbc5caaca 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -2,9 +2,7 @@ numpydoc
 tornado
 toolz
 cloudpickle
-# TODO: Update our dask version requirement to
-# the latest dask release before releasing distributed
-dask @ git+https://github.com/dask/dask.git@master
+dask>=2020.12.0
 sphinx
 dask-sphinx-theme>=1.3.5
 sphinx-click
diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index 6671e52c7b5..a1b36f831c6 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,118 @@
 Changelog
 =========
 
+2020.12.0
+---------
+
+Released on December 10, 2020
+
+Highlights
+^^^^^^^^^^
+
+- Switched to `CalVer <https://calver.org/>`_ for versioning scheme.
+- The scheduler can now receives Dask ``HighLevelGraph`` s instead of raw dictionary task graphs.
+  This allows for a much more efficient communication of task graphs from the client to the scheduler.
+- Added support for using custom ``Layer``-level annotations like ``priority``, ``retries``,
+  etc. with the ``dask.annotations`` context manager.
+- Updated minimum supported version of Dask to 2020.12.0.
+- Added many type annotations and updates to allow for gradually Cythonizing the scheduler.
+
+All changes
+^^^^^^^^^^^
+
+- Some common optimizations across transitions (:pr:`4348`) `jakirkham`_
+- Drop serialize extension (:pr:`4344`) `jakirkham`_
+- Log duplciate workers in scheduler (:pr:`4338`) `Matthew Rocklin`_
+- Annotation of some comm related methods in the ``Scheduler`` (:pr:`4341`) `jakirkham`_
+- Optimize ``assert`` in ``validate_waiting`` (:pr:`4342`) `jakirkham`_
+- Optimize ``decide_worker`` (:pr:`4332`) `jakirkham`_
+- Store occupancy in ``_reevaluate_occupancy_worker`` (:pr:`4337`) `jakirkham`_
+- Handle ``WorkerState`` ``memory_limit`` of ``None`` (:pr:`4335`) `jakirkham`_
+- Use ``bint`` to annotate boolean attributes (:pr:`4334`) `jakirkham`_
+- Optionally use offload executor in worker (:pr:`4307`) `Matthew Rocklin`_
+- Optimize ``send_task_to_worker`` (:pr:`4331`) `jakirkham`_
+- Optimize ``valid_workers`` (:pr:`4329`) `jakirkham`_
+- Store occupancy in ``transition_waiting_processing`` (:pr:`4330`) `jakirkham`_
+- Optimize ``get_comm_cost`` (:pr:`4328`) `jakirkham`_
+- Use ``.pop(...)`` to remove ``key`` (:pr:`4327`) `jakirkham`_
+- Use ``operator.attrgetter`` on ``WorkerState.address`` (:pr:`4324`) `jakirkham`_
+- Annotate ``Task*`` objects for Cythonization (:pr:`4302`) `jakirkham`_
+- Ensure ``retire_workers`` always ``return`` a ``dict`` (:pr:`4323`) `jakirkham`_
+- Some Cython fixes for ``WorkerState`` (:pr:`4321`) `jakirkham`_
+- Optimize ``WorkerState.__eq__`` (:pr:`4320`) `jakirkham`_
+- Swap order of ``TaskGroup`` and ``TaskPrefix`` (:pr:`4319`) `jakirkham`_
+- Check traceback object can be unpickled (:pr:`4299`) `jakirkham`_
+- Move ``TaskGroup`` & ``TaskPrefix`` before `TaskState` (:pr:`4318`) `jakirkham`_
+- Remove empty ``test_highgraph.py`` file (:pr:`4313`) `James Bourbeau`_
+- Ensure that ``retire_workers`` returns a ``dict`` (:pr:`4315`) `Matthew Rocklin`_
+- Annotate ``WorkerState`` for Cythonization (:pr:`4294`) `jakirkham`_
+- Close ``comm`` on low-level errors (:pr:`4239`) `jochen-ott-by`_
+- Coerce new ``TaskState.nbytes`` value to ``int`` (:pr:`4311`) `jakirkham`_
+- Remove offload ``try``/``except`` for ``thread_name_prefix`` keyword (:pr:`4308`) `James Bourbeau`_
+- Fix ``pip`` install issue on CI (:pr:`4310`) `jakirkham`_
+- Transmit ``Layer`` annotations to scheduler (:pr:`4279`) `Simon Perkins`_
+- Ignores any compiled files generated by Cython (:pr:`4301`) `jakirkham`_
+- Protect against missing key in ``get_metrics`` (:pr:`4300`) `Matthew Rocklin`_
+- Provide option to build Distributed with Cython (:pr:`4292`) `jakirkham`_
+- Set ``WorkerState.processing`` w/``dict`` in ``clean`` (:pr:`4295`) `jakirkham`_
+- Annotate ``ClientState`` for Cythonization (:pr:`4290`) `jakirkham`_
+- Annotate ``check_idle_saturated`` for Cythonization (:pr:`4289`) `jakirkham`_
+- Avoid flicker in ``TaskStream`` with "Scheduler is empty" message (:pr:`4284`) `Matthew Rocklin`_
+- Make ``gather_dep`` robust to missing tasks (:pr:`4285`) `Matthew Rocklin`_
+- Annotate ``extract_serialize`` (for Cythonization) (:pr:`4283`) `jakirkham`_
+- Move ``nbytes`` from Worker's state to ``TaskState`` (:pr:`4274`) `Gil Forsyth`_
+- Drop extra type check in ``_extract_serialize`` (:pr:`4281`) `jakirkham`_
+- Move Status to top-level import (:pr:`4280`) `Matthew Rocklin`_
+- Add ``__hash__`` and ``__eq__`` for ``TaskState`` (:pr:`4278`) `jakirkham`_
+- Add ``__hash__`` and ``__eq__`` for ``ClientState`` (:pr:`4276`) `jakirkham`_
+- Collect ``report``'s ``client_key``s in a ``list`` (:pr:`4275`) `jakirkham`_
+- Precompute ``hash`` for ``WorkerState`` (:pr:`4271`) `jakirkham`_
+- Use ``Status`` ``Enum`` in ``remove_worker`` (:pr:`4269`) `jakirkham`_
+- Add aggregated topic logs and ``log_event`` method (:pr:`4230`) `James Bourbeau`_
+- Find the set of workers instead of their frequency (:pr:`4267`) `jakirkham`_
+- Use ``set.update`` to include other ``comms`` (:pr:`4268`) `jakirkham`_
+- Support string timeouts in ``sync`` (:pr:`4266`) `James Bourbeau`_
+- Use ``dask.utils.stringify()`` instead of ``distributed.utils.tokey()`` (:pr:`4255`) `Mads R. B. Kristensen`_
+- Use ``.items()`` to walk through keys and values (:pr:`4261`) `jakirkham`_
+- Simplify frame length packing in TCP write (:pr:`4257`) `jakirkham`_
+- Comm/tcp listener: do not pass comm with failed handshake to ``comm_handler`` (:pr:`4240`) `jochen-ott-by`_
+- Fuse steps in ``extract_serialize`` (:pr:`4254`) `jakirkham`_
+- Drop ``test_sklearn`` (:pr:`4253`) `jakirkham`_
+- Document task priority tie breaking (:pr:`4252`) `James Bourbeau`_
+- ``__dask_distributed_pack__()``: client argument (:pr:`4248`) `Mads R. B. Kristensen`_
+- Configurable timeouts for ``worker_client`` and ``get_client`` (:pr:`4146`) `GeethanjaliEswaran`_
+- Add dask/distributed versions to ``performance_report`` (:pr:`4249`) `Matthew Rocklin`_
+- Update miniconda GitHub action (:pr:`4250`) `James Bourbeau`_
+- UCX closing ignore error (:pr:`4236`) `Mads R. B. Kristensen`_
+- Redirect to ``dask-worker`` cli documentation (:pr:`4247`) `Timost`_
+- Upload file worker plugin (:pr:`4238`) `Ian Rose`_
+- Create dependency ``TaskState`` as needed in ``gather_dep`` (:pr:`4241`) `Gil Forsyth`_
+- Instantiate plugin if needed in ``register_worker_plugin`` (:pr:`4198`) `Julia Signell`_
+- Allow actors to call actors on the same worker (:pr:`4225`) `Martin Durant`_
+- Special case profile thread in leaked thread check (:pr:`4229`) `James Bourbeau`_
+- Use ``intersection()`` on a set instead of ``dict_keys`` in ``update_graph`` (:pr:`4227`) `Mads R. B. Kristensen`_
+- Communicate ``HighLevelGraphs`` directly to the ``Scheduler`` (:pr:`4140`) `Mads R. B. Kristensen`_
+- Add ``get_task_metadata`` context manager (:pr:`4216`) `James Bourbeau`_
+- Task state logs and data fix (:pr:`4206`) `Gil Forsyth`_
+- Send active task durations from worker to scheduler (:pr:`4192`) `James Bourbeau`_
+- Fix state check in ``test_close_gracefully`` (:pr:`4203`) `Gil Forsyth`_
+- Avoid materializing layers in ``Client.compute()`` (:pr:`4196`) `Mads R. B. Kristensen`_
+- Add ``TaskState`` metadata (:pr:`4191`) `James Bourbeau`_
+- Fix regression in task stealing for already released keys (:pr:`4182`) `Florian Jetter`_
+- Fix ``_graph_to_futures`` bug for futures-based dependencies (:pr:`4178`) `Richard J Zamora`_
+- High level graph ``dumps``/``loads`` support (:pr:`4174`) `Mads R. B. Kristensen`_
+- Implement pass HighLevelGraphs through ``_graph_to_futures`` (:pr:`4139`) `Mads R. B. Kristensen`_
+- Support ``async`` preload click commands (:pr:`4170`) `James Bourbeau`_
+- ``dask-worker`` cli memory limit option doc fix (:pr:`4172`) `marwan116`_
+- Add ``TaskState`` to ``worker.py`` (:pr:`4107`) `Gil Forsyth`_
+- Increase robustness of ``Semaphore.release`` (:pr:`4151`) `Lucas Rademaker`_
+- Skip batched comm test win / tornado5 (:pr:`4166`) `Tom Augspurger`_
+- Set Zict buffer target to maxsize when ``memory_target_fraction`` is ``False`` (:pr:`4156`) `Krishan Bhasin`_
+- Add ``PipInstall`` ``WorkerPlugin`` (:pr:`3216`) `Matthew Rocklin`_
+- Log ``KilledWorker`` events in the scheduler (:pr:`4157`) `Matthew Rocklin`_
+- Fix ``test_gpu_metrics`` failure (:pr:`4154`) `jakirkham`_
+
+
 2.30.1 - 2020-11-03
 -------------------
 
@@ -2003,3 +2115,9 @@ significantly without many new features.
 .. _`Poruri Sai Rahul`: https://github.com/rahulporuri
 .. _`jennalc`: https://github.com/jennalc
 .. _`Sergey Kozlov`: https://github.com/skozlovf
+.. _`jochen-ott-by`: https://github.com/jochen-ott-by
+.. _`Simon Perkins`: https://github.com/sjperkins
+.. _`GeethanjaliEswaran`: https://github.com/geethanjalieswaran
+.. _`Timost`: https://github.com/Timost
+.. _`Ian Rose`: https://github.com/ian-r-rose
+.. _`marwan116`: https://github.com/marwan116
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index d268861c45a..91a18f6346b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,9 +1,7 @@
 click >= 6.6
 cloudpickle >= 1.5.0
 contextvars;python_version<'3.7'
-# TODO: Update our dask version requirement to
-# the latest dask release before releasing distributed
-dask @ git+https://github.com/dask/dask.git@master
+dask>=2020.12.0
 msgpack >= 0.6.0
 psutil >= 5.0
 sortedcontainers !=2.0.0, !=2.0.1

From d421adc836e33bd7fa22402bd1227452337b976b Mon Sep 17 00:00:00 2001
From: Benjamin Zaitlen <quasiben@users.noreply.github.com>
Date: Fri, 11 Dec 2020 09:47:15 -0500
Subject: [PATCH 1098/1550] Silence exception when releasing futures on process
 shutdown (#4309)

Fixes #4305
---
 distributed/client.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/distributed/client.py b/distributed/client.py
index 156460c022e..a7887ef4b9d 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -382,6 +382,11 @@ def __setstate__(self, state):
     def __del__(self):
         try:
             self.release()
+        except AttributeError:
+            # Ocassionally we see this error when shutting down the client
+            # https://github.com/dask/distributed/issues/4305
+            if not shutting_down():
+                raise
         except RuntimeError:  # closed event loop
             pass
 

From 2e4ec3a90f4d3ffd42d24d5bd8148d935cccc5f4 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Fri, 11 Dec 2020 06:48:24 -0800
Subject: [PATCH 1099/1550] Optimizations in notable functions used by
 transitions (#4351)

* Type `w` as `str` to match `ws._address`

* Use `TaskState` w/`report_on_key` in Scheduler

As `report_on_key` needs the `TaskState` regardless and will just fetch
it if given the `key`, go ahead and just pass the `TaskState` object
instead of the `key` when available. After all `TaskState` is a typed
object that has fast attribute access. So we can fetch anything else we
need (including the key) anyways.

* Inline `self._nbytes` usage in `get_nbytes`

* Make `sum` of `list` comprehension a `for`-loop

Avoids some extra boilerplate that Cython would generate to build the
`list` and then loop through it with `sum`. Instead this builds a single
`for`-loop over `ts._dependencies` and handles the addition in C. Should
improve the performance here a bit.

* Add `stack_time` first

This is slightly easier to read. Also Cython will translate this into
doing some work first before checking for zero in `self.bandwidth`.

* Annotate and compute times in C

Make sure that variables like `self.bandwidth` are cast to C variables.
Perform computations with these in straight C.

* Merge `assert` handling into argument handling

As we already need to go through and handle `ts is None` and the second
branch should already be `key is None` (and is easy enough to check), go
ahead and move the `assert` into the `else` case at the end. All of
these comparisons are really fast pointer comparisons that we were
already doing before more-or-less. So finishing it out with an `assert
False`, doesn't really cost us anything and is more optimized for both
pure Python and Cython.

* Break out 2nd handling of `ts is None`

As the `key` may be missing from `TaskState`, in which case `ts` will
still be `None` after trying to retrieve it from `self.tasks`, handle
that `if` as well, but handle it like the other `report` conditions and
not separately. Simplifies the code a bit and improves readability.
---
 distributed/scheduler.py | 38 ++++++++++++++++++++++----------------
 1 file changed, 22 insertions(+), 16 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 55baa274395..c13cf08927a 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1290,8 +1290,7 @@ def add_dependency(self, other: "TaskState"):
         other._dependents.add(self)
 
     def get_nbytes(self) -> int:
-        nbytes = self._nbytes
-        return nbytes if nbytes >= 0 else DEFAULT_DATA_SIZE
+        return self._nbytes if self._nbytes >= 0 else DEFAULT_DATA_SIZE
 
     def set_nbytes(self, nbytes: Py_ssize_t):
         diff: Py_ssize_t = nbytes
@@ -2583,7 +2582,7 @@ def update_graph(
 
         for ts in touched_tasks:
             if ts._state in ("memory", "erred"):
-                self.report_on_key(ts._key, client=client)
+                self.report_on_key(ts=ts, client=client)
 
         end = time()
         if self.digests is not None:
@@ -2901,7 +2900,7 @@ def client_desires_keys(self, keys=None, client=None):
             cs._wants_what.add(ts)
 
             if ts._state in ("memory", "erred"):
-                self.report_on_key(k, client=client)
+                self.report_on_key(ts=ts, client=client)
 
     def client_releases_keys(self, keys=None, client=None):
         """ Remove keys from client desired list """
@@ -4208,16 +4207,18 @@ def update_data(
                 self.client_desires_keys(keys=list(who_has), client=client)
 
     def report_on_key(self, key: str = None, ts: TaskState = None, client: str = None):
-        assert (key is None) != (ts is None), (key, ts)
         if ts is None:
             tasks: dict = self.tasks
             ts = tasks.get(key)
-            if ts is None:
-                self.report({"op": "cancelled-key", "key": key}, client=client)
-                return
-        else:
+        elif key is None:
             key = ts._key
-        if ts._state == "forgotten":
+        else:
+            assert False, (key, ts)
+            return
+
+        if ts is None:
+            self.report({"op": "cancelled-key", "key": key}, client=client)
+        elif ts._state == "forgotten":
             self.report({"op": "cancelled-key", "key": key}, ts=ts, client=client)
         elif ts._state == "memory":
             self.report({"op": "key-in-memory", "key": key}, ts=ts, client=client)
@@ -4504,7 +4505,7 @@ def _remove_from_processing(self, ts: TaskState, send_worker_msg=None):
         """
         ws: WorkerState = ts._processing_on
         ts._processing_on = None
-        w = ws._address
+        w: str = ws._address
         if w in self.workers:  # may have been removed
             duration = ws._processing.pop(ts)
             if not ws._processing:
@@ -5701,11 +5702,16 @@ def worker_objective(self, ts: TaskState, ws: WorkerState):
         Minimize expected start time.  If a tie then break with data storage.
         """
         dts: TaskState
-        comm_bytes = sum(
-            [dts.get_nbytes() for dts in ts._dependencies if ws not in dts._who_has]
-        )
-        stack_time = ws._occupancy / ws._nthreads
-        start_time = comm_bytes / self.bandwidth + stack_time
+        nbytes: Py_ssize_t
+        comm_bytes: Py_ssize_t = 0
+        for dts in ts._dependencies:
+            if ws not in dts._who_has:
+                nbytes = dts.get_nbytes()
+                comm_bytes += nbytes
+
+        bandwidth: double = self.bandwidth
+        stack_time: double = ws._occupancy / ws._nthreads
+        start_time: double = stack_time + comm_bytes / bandwidth
 
         if ts._actor:
             return (len(ws._actors), start_time, ws._nbytes)

From e5e2c55b14d734c023dba1b50c8dcaa22f8825c9 Mon Sep 17 00:00:00 2001
From: "Mads R. B. Kristensen" <madsbk@gmail.com>
Date: Fri, 11 Dec 2020 16:56:48 +0100
Subject: [PATCH 1100/1550] Added docs to highlevelgraph pack/unpack (#4352)

---
 distributed/protocol/highlevelgraph.py | 43 ++++++++++++++++++++++++++
 1 file changed, 43 insertions(+)

diff --git a/distributed/protocol/highlevelgraph.py b/distributed/protocol/highlevelgraph.py
index cd0d8316fe0..6e637717435 100644
--- a/distributed/protocol/highlevelgraph.py
+++ b/distributed/protocol/highlevelgraph.py
@@ -75,6 +75,27 @@ def _materialized_layer_pack(
 
 
 def highlevelgraph_pack(hlg: HighLevelGraph, client, client_keys):
+    """Pack the high level graph for Scheduler -> Worker communication
+
+    The approach is to delegate the packaging to each layer in the high
+    level graph by calling .__dask_distributed_pack__() on each layer.
+    If the layer doesn't implement packaging, we materialize the layer
+    and pack it.
+
+    Parameters
+    ----------
+    hlg: HighLevelGraph
+        The high level graph to pack
+    client: distributed.Client
+        The client calling this function.
+    client_keys: Iterable
+        List of keys requested by the client.
+
+    Returns
+    -------
+    data: bytes
+        Packed high level graph serialized by msgpack
+    """
     layers = []
 
     # Dump each layer (in topological order)
@@ -121,6 +142,28 @@ def _materialized_layer_unpack(state, dsk, dependencies, annotations):
 
 
 def highlevelgraph_unpack(dumped_hlg):
+    """Unpack the high level graph for Scheduler -> Worker communication
+
+    The approach is to delegate the packaging to each layer in the high
+    level graph by calling .__dask_distributed_pack__() on each layer.
+    If the layer doesn't implement a specialized packaging, we materialize
+    the layer can pack it.
+
+    Parameters
+    ----------
+    dumped_hlg: bytes
+        Packed high level graph serialized by msgpack
+
+    Returns
+    -------
+    dsk: dict
+        Materialized graph of all nodes in the high level graph
+    deps: dict
+        Dependencies of each key in `dsk`
+    annotations: dict
+        Annotations for `dsk`
+    """
+
     # Notice, we set `use_list=False`, which makes msgpack convert lists to tuples
     hlg = msgpack.loads(
         dumped_hlg, object_hook=msgpack_decode_default, use_list=False, **msgpack_opts

From 6fa9b1849a2ed43d123a9ae6221354cb420ce333 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Fri, 11 Dec 2020 18:28:12 -0600
Subject: [PATCH 1101/1550] Use task annotation priotities for user-level
 priorities (#4354)

---
 distributed/scheduler.py         | 2 +-
 distributed/tests/test_client.py | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index c13cf08927a..34a14da048e 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -2448,7 +2448,7 @@ def update_graph(
         # Override existing taxonomy with per task annotations
         if annotations:
             if "priority" in annotations:
-                priority.update(annotations["priority"])
+                user_priority.update(annotations["priority"])
 
             if "workers" in annotations:
                 restrictions.update(annotations["workers"])
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 606d0ddc571..2a477a49427 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -6325,6 +6325,7 @@ async def test_annotations_priorities(c, s, a, b):
         x = await x.persist()
 
     assert all("15" in str(ts.priority) for ts in s.tasks.values())
+    assert all(ts.priority[0] == -15 for ts in s.tasks.values())
     assert all({"priority": 15} == ts.annotations for ts in s.tasks.values())
 
 

From 07970728386788eace213487b54562b3af077b84 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Sat, 12 Dec 2020 08:00:44 -0800
Subject: [PATCH 1102/1550] Cast `SortedDict`s to `dict`s in a few key places &
 other minor changes (#4355)

* Use `WorkerState` `address` attribute

Allows for faster access via C as opposed to via Python.

* Annotate `n_workers` and `n_tasks`

Moves the comparisons and math related to these from Python into C when
Cython is used. Otherwise has no real effect.

* Use `SortedDict` for `self.idle`

Instead of using `SortedSet` for `self.idle`, use `SortedDict`. This
avoids needing `operator` to retrieve `WorkerState.address`, which we
can retrieve on our own (and do so faster!). Plus `SortedDict` inherits
from `dict`, which allows us to use some Python C API functions on it.
Something that `SortedSet` does not.

* Cast `SortedDict` to `dict` when viewing

As `SortedDict` inherits from `dict`, we can ask Cython to cast
`SortedDict` to `dict`. This is helpful as Cython will make use of
efficient Python C APIs for `dict`s if it knows the value is typed that
way. So should improve performance of things like item retrieval, checks
for inclusion of keys, iteration, etc..

That said, it's worth noting that operations that manipulate the
contents of the `SortedDict` should not use the Python C API as
`SortedDict` overrides all of these methods to provide the `sorted`
behavior we expect. So when adding or removing values, we use the actual
`SortedDict` object not the cast representation.
---
 distributed/scheduler.py | 57 ++++++++++++++++++++++++----------------
 distributed/stealing.py  |  2 +-
 2 files changed, 36 insertions(+), 23 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 34a14da048e..6692d7867ac 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -86,7 +86,7 @@
 from .protocol.highlevelgraph import highlevelgraph_unpack
 
 try:
-    from cython import bint, cclass, double, Py_hash_t, Py_ssize_t
+    from cython import bint, cast, cclass, double, Py_hash_t, Py_ssize_t
 except ImportError:
     from ctypes import (
         c_double as double,
@@ -96,6 +96,9 @@
 
     bint = bool
 
+    def cast(T, v, *a, **k):
+        return v
+
     def cclass(cls):
         return cls
 
@@ -1703,7 +1706,7 @@ def __init__(
                 func = compose(wrap, func)
             setattr(self, old_attr, _StateLegacyMapping(self.workers, func))
 
-        self.idle = sortedcontainers.SortedSet(key=operator.attrgetter("address"))
+        self.idle = sortedcontainers.SortedDict()
         self.saturated = set()
 
         self.total_nthreads = 0
@@ -2204,7 +2207,7 @@ async def add_worker(
             self.stream_comms[address] = BatchedSend(interval="5ms", loop=self.loop)
 
             if ws._nthreads > len(ws._processing):
-                self.idle.add(ws)
+                self.idle[ws._address] = ws
 
             for plugin in self.plugins[:]:
                 try:
@@ -2783,7 +2786,7 @@ async def remove_worker(self, comm=None, address=None, safe=False, close=True):
             self.rpc.remove(address)
             del self.stream_comms[address]
             del self.aliases[ws._name]
-            self.idle.discard(ws)
+            self.idle.pop(ws._address, None)
             self.saturated.discard(ws)
             del self.workers[address]
             ws.status = Status.closed
@@ -3035,7 +3038,7 @@ def validate_state(self, allow_overlap=False):
             assert ws._address == w
             if not ws._processing:
                 assert not ws._occupancy
-                assert ws in self.idle
+                assert ws._address in cast(dict, self.idle)
 
         ts: TaskState
         for k, ts in self.tasks.items():
@@ -4503,10 +4506,11 @@ def _remove_from_processing(self, ts: TaskState, send_worker_msg=None):
         """
         Remove *ts* from the set of processing tasks.
         """
+        workers: dict = cast(dict, self.workers)
         ws: WorkerState = ts._processing_on
         ts._processing_on = None
         w: str = ws._address
-        if w in self.workers:  # may have been removed
+        if w in workers:  # may have been removed
             duration = ws._processing.pop(ts)
             if not ws._processing:
                 self.total_occupancy -= ws._occupancy
@@ -4576,6 +4580,7 @@ def _add_to_memory(
     def transition_released_waiting(self, key):
         try:
             tasks: dict = self.tasks
+            workers: dict = cast(dict, self.workers)
             ts: TaskState = tasks[key]
             dts: TaskState
 
@@ -4612,7 +4617,7 @@ def transition_released_waiting(self, key):
             ts._waiters = {dts for dts in ts._dependents if dts._state == "waiting"}
 
             if not ts._waiting_on:
-                if self.workers:
+                if workers:
                     recommendations[key] = "processing"
                 else:
                     self.unrunnable.add(ts)
@@ -4630,6 +4635,7 @@ def transition_released_waiting(self, key):
     def transition_no_worker_waiting(self, key):
         try:
             tasks: dict = self.tasks
+            workers: dict = cast(dict, self.workers)
             ts: TaskState = tasks[key]
             dts: TaskState
 
@@ -4658,7 +4664,7 @@ def transition_no_worker_waiting(self, key):
             ts.state = "waiting"
 
             if not ts._waiting_on:
-                if self.workers:
+                if workers:
                     recommendations[key] = "processing"
                 else:
                     self.unrunnable.add(ts)
@@ -4677,6 +4683,7 @@ def decide_worker(self, ts: TaskState) -> WorkerState:
         """
         Decide on a worker for task *ts*.  Return a WorkerState.
         """
+        workers: dict = cast(dict, self.workers)
         ws: WorkerState = None
         valid_workers: set = self.valid_workers(ts)
 
@@ -4684,7 +4691,7 @@ def decide_worker(self, ts: TaskState) -> WorkerState:
             valid_workers is not None
             and not valid_workers
             and not ts._loose_restrictions
-            and self.workers
+            and workers
         ):
             self.unrunnable.add(ts)
             ts.state = "no-worker"
@@ -4693,24 +4700,26 @@ def decide_worker(self, ts: TaskState) -> WorkerState:
         if ts._dependencies or valid_workers is not None:
             ws = decide_worker(
                 ts,
-                self.workers.values(),
+                workers.values(),
                 valid_workers,
                 partial(self.worker_objective, ts),
             )
         else:
-            worker_pool = self.idle or self.workers.values()
-            n_workers = len(worker_pool)
+            worker_pool = self.idle or self.workers
+            worker_pool_dv = cast(dict, worker_pool)
+            n_workers: Py_ssize_t = len(worker_pool_dv)
             if n_workers < 20:  # smart but linear in small case
-                ws = min(worker_pool, key=operator.attrgetter("occupancy"))
+                ws = min(worker_pool.values(), key=operator.attrgetter("occupancy"))
             else:  # dumb but fast in large case
-                ws = worker_pool[self.n_tasks % n_workers]
+                n_tasks: Py_ssize_t = self.n_tasks
+                ws = worker_pool.values()[n_tasks % n_workers]
 
         if self.validate:
             assert ws is None or isinstance(ws, WorkerState), (
                 type(ws),
                 ws,
             )
-            assert ws.address in self.workers
+            assert ws._address in workers
 
         return ws
 
@@ -4765,7 +4774,8 @@ def transition_waiting_processing(self, key):
 
     def transition_waiting_memory(self, key, nbytes=None, worker=None, **kwargs):
         try:
-            ws: WorkerState = self.workers[worker]
+            workers: dict = cast(dict, self.workers)
+            ws: WorkerState = workers[worker]
             tasks: dict = self.tasks
             ts: TaskState = tasks[key]
 
@@ -4826,7 +4836,8 @@ def transition_processing_memory(
                 assert not ts._exception_blame
                 assert ts._state == "processing"
 
-            ws = self.workers.get(worker)
+            workers: dict = cast(dict, self.workers)
+            ws = workers.get(worker)
             if ws is None:
                 return {key: "released"}
 
@@ -5251,6 +5262,7 @@ def remove_key(self, key):
         self.task_metadata.pop(key, None)
 
     def _propagate_forgotten(self, ts: TaskState, recommendations: dict):
+        workers: dict = cast(dict, self.workers)
         ts.state = "forgotten"
         key: str = ts._key
         dts: TaskState
@@ -5283,7 +5295,7 @@ def _propagate_forgotten(self, ts: TaskState, recommendations: dict):
             ws._has_what.remove(ts)
             ws._nbytes -= ts.get_nbytes()
             w: str = ws._address
-            if w in self.workers:  # in case worker has died
+            if w in workers:  # in case worker has died
                 self.worker_send(
                     w, {"op": "delete-data", "keys": [key], "report": False}
                 )
@@ -5543,10 +5555,10 @@ def check_idle_saturated(self, ws: WorkerState, occ: double = -1.0):
         idle = self.idle
         saturated: set = self.saturated
         if p < nc or occ / nc < avg / 2:
-            idle.add(ws)
+            idle[ws._address] = ws
             saturated.discard(ws)
         else:
-            idle.discard(ws)
+            idle.pop(ws._address, None)
 
             pending: double = occ * (p - nc) / p / nc
             if p > nc and pending > 0.4 and pending > 1.9 * avg:
@@ -5564,10 +5576,11 @@ def valid_workers(self, ts: TaskState) -> set:
         *  host_restrictions
         *  resource_restrictions
         """
+        workers: dict = cast(dict, self.workers)
         s: set = None
 
         if ts._worker_restrictions:
-            s = {w for w in ts._worker_restrictions if w in self.workers}
+            s = {w for w in ts._worker_restrictions if w in workers}
 
         if ts._host_restrictions:
             # Resolve the alias here rather than early, for the worker
@@ -5600,7 +5613,7 @@ def valid_workers(self, ts: TaskState) -> set:
                 s &= ww
 
         if s is not None:
-            s = {self.workers[w] for w in s}
+            s = {workers[w] for w in s}
 
         return s
 
diff --git a/distributed/stealing.py b/distributed/stealing.py
index f2f95e4b133..90cb4010612 100644
--- a/distributed/stealing.py
+++ b/distributed/stealing.py
@@ -303,7 +303,7 @@ def maybe_move_task(level, ts, sat, idl, duration, cost_multiplier):
 
         with log_errors():
             i = 0
-            idle = s.idle
+            idle = s.idle.values()
             saturated = s.saturated
             if not idle or len(idle) == len(s.workers):
                 return

From e0980309e2ac5f0e5c69a057d8a96b0f30f99ea2 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Mon, 14 Dec 2020 07:07:23 -0800
Subject: [PATCH 1103/1550] Optimizations of `*State` and `Task*` objects and
 stealing (#4358)

* Cast `workers` in `stimulus_task_finished`

* Assign `self.tasks` to `dict` typed variable

* Multiply `p` & `nc` before dividing

Avoids an extra check for zero division.

* Multiply `nc` & `avg`

This avoids a division by zero check that seems unlikely in this case
(we should have some threads).

* Delay computing `pending` until needed

* Combine comparisons of `pending`

Cython will pipeline this sort of check more effectively in C. So
leverage that behavior here.

* Simplify branch structure

* Type `tasks` as `set`

* Type `bandwidth` in `get_comm_cost`

* Type `duration` in `get_task_duration`

* Assign `set` to local variable for Cython opts

* Type `default` in `get_task_duration`

* Extract & assign `UNKNOWN_TASK_DURATION` globally

* Overwrite `duration` in `get_task_duration`

* Annotate `identity` return type as `dict`

* Type `get_nbytes` return type as `Py_ssize_t`

* Add `ccall` wrapper for `cpdef` style functions

Should allow us to make faster calls to Python functions decorated with
`ccall`.

* Add `exceptval` to tweak function exception checks

Normally the default exception check behavior from Cythonized functions
is fine. However in a few cases we need to tweak this slightly. so
import `exceptval` so that we can make those tweaks.

* Decorate all `*State` & `Task*` methods w/ `ccall`

In addition to the Python interface these methods have, this adds C
interface, which Cython will opt to use in Cython code with access to
that interface (currently just `scheduler.py`). Should speed up function
calls in these cases.

* Add `final` wrapper for classes w/o inheritance

By default Cython assumes that extension type objects may be subclassed.
So it creates a virtual table for all methods of the base class in the
object `struct`. This way these function pointers can be changed to
point to some other method for subclasses. However if we know a class
will not be subclassed, we can indicate that by marking it as `final`.
This allows Cython to do away with the virtual table and instead just
have a collection of functions to call on the objects. These functions
can in turn be inlined or otherwise optimized. Should be useful for the
`*State` and `Task*` objects, which are not subclassed and can benefit
from the faster function calls (like `get_nbytes`).

* Mark all `*State` & `Task*` objects `final`

Should eliminate virtual tables for these objects and allow inlining of
their methods.

* Add `declare` for typing variables

Ideally we would just use type annotations at the module level,
unfortunately that doesn't seem to be something Cython translates into C
typed variables. Fortunately `declare` seems to work just fine for this
purpose. So go ahead and include it so we can type module variables.
Should make it easier to use these in other functions without Cython
needing to convert them every time.

* Type module global variables with `declare`

This tells Cython that these variables have the type provided and should
be assigned the value given. By doing this, we should now have `static`
variables in C with these values, which we can use locally and Cython
will allow C operations to occur on them.

* Add `inline` to suggest inlining of functions

This can be handy when trying to suggest to the C compiler to just go
ahead an inline a function (like `get_nbytes`) to save on the cost of a
function call.

* Add `inline` to `get_nbytes`

Should encourage the compiler to just go ahead and include the code of
this method directly where it is used.

* Add `nogil` wrapper for functions

Should help us mark functions that don't require the GIL (like
`get_nbytes`) to avoid some overhead in them.

* Decorate `get_nbytes` as `nogil`

Should avoid some extra overhead when calling this method. Also should
make it easier for the compiler to optimize and inline as needed.

* Add `get_nbytes_deps`

Provides a utility function for getting the total number of bytes of all
dependencies of this task. Should be useful in stealing where there is
a bottleneck computing this.

* Use `TaskState`'s `get_nbytes_deps` in stealing

Now that we have Cythonized this expression in this method, just use it
here to remove this bottleneck.
---
 distributed/scheduler.py | 114 +++++++++++++++++++++++++++++++--------
 distributed/stealing.py  |   2 +-
 2 files changed, 92 insertions(+), 24 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 6692d7867ac..ed3f7150e9e 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -86,7 +86,20 @@
 from .protocol.highlevelgraph import highlevelgraph_unpack
 
 try:
-    from cython import bint, cast, cclass, double, Py_hash_t, Py_ssize_t
+    from cython import (
+        bint,
+        cast,
+        ccall,
+        cclass,
+        declare,
+        double,
+        exceptval,
+        final,
+        inline,
+        nogil,
+        Py_hash_t,
+        Py_ssize_t,
+    )
 except ImportError:
     from ctypes import (
         c_double as double,
@@ -99,9 +112,33 @@
     def cast(T, v, *a, **k):
         return v
 
+    def ccall(func):
+        return func
+
     def cclass(cls):
         return cls
 
+    def declare(*a, **k):
+        if len(a) == 2:
+            return a[1]
+        else:
+            pass
+
+    def exceptval(*a, **k):
+        def wrapper(func):
+            return func
+
+        return wrapper
+
+    def final(cls):
+        return cls
+
+    def inline(func):
+        return func
+
+    def nogil(func):
+        return func
+
 
 if sys.version_info < (3, 8):
     try:
@@ -116,8 +153,12 @@ def cclass(cls):
 
 
 LOG_PDB = dask.config.get("distributed.admin.pdb-on-err")
-DEFAULT_DATA_SIZE = parse_bytes(
-    dask.config.get("distributed.scheduler.default-data-size")
+DEFAULT_DATA_SIZE = declare(
+    Py_ssize_t, parse_bytes(dask.config.get("distributed.scheduler.default-data-size"))
+)
+UNKNOWN_TASK_DURATION = declare(
+    double,
+    parse_timedelta(dask.config.get("distributed.scheduler.unknown-task-duration")),
 )
 
 DEFAULT_EXTENSIONS = [
@@ -134,6 +175,7 @@ def cclass(cls):
 ALL_TASK_STATES = {"released", "waiting", "no-worker", "processing", "erred", "memory"}
 
 
+@final
 @cclass
 class ClientState:
     """
@@ -206,6 +248,7 @@ def versions(self):
         return self._versions
 
 
+@final
 @cclass
 class WorkerState:
     """
@@ -503,6 +546,7 @@ def used_resources(self):
     def versions(self):
         return self._versions
 
+    @ccall
     def clean(self):
         """ Return a version of this object that is appropriate for serialization """
         ws: WorkerState = WorkerState(
@@ -528,7 +572,9 @@ def __repr__(self):
             len(self._processing),
         )
 
-    def identity(self):
+    @ccall
+    @exceptval(check=False)
+    def identity(self) -> dict:
         return {
             "type": "Worker",
             "id": self._name,
@@ -551,6 +597,7 @@ def ncores(self):
         return self._nthreads
 
 
+@final
 @cclass
 class TaskPrefix:
     """Collection tracking all tasks within a group
@@ -675,6 +722,7 @@ def types(self):
         return set().union(*[tg._types for tg in self._groups])
 
 
+@final
 @cclass
 class TaskGroup:
     """Collection tracking all tasks within a group
@@ -769,6 +817,7 @@ def duration(self):
     def types(self):
         return self._types
 
+    @ccall
     def add(self, o):
         ts: TaskState = o
         self._states[ts._state] += 1
@@ -789,6 +838,7 @@ def __len__(self):
         return sum(self._states.values())
 
 
+@final
 @cclass
 class TaskState:
     """
@@ -1286,15 +1336,20 @@ def group_key(self):
     def prefix_key(self):
         return self._prefix._name
 
+    @ccall
     def add_dependency(self, other: "TaskState"):
         """ Add another task as a dependency of this task """
         self._dependencies.add(other)
         self._group._dependencies.add(other._group)
         other._dependents.add(self)
 
-    def get_nbytes(self) -> int:
+    @ccall
+    @inline
+    @nogil
+    def get_nbytes(self) -> Py_ssize_t:
         return self._nbytes if self._nbytes >= 0 else DEFAULT_DATA_SIZE
 
+    @ccall
     def set_nbytes(self, nbytes: Py_ssize_t):
         diff: Py_ssize_t = nbytes
         old_nbytes: Py_ssize_t = self._nbytes
@@ -1310,6 +1365,7 @@ def set_nbytes(self, nbytes: Py_ssize_t):
     def __repr__(self):
         return "<Task %r %s>" % (self._key, self._state)
 
+    @ccall
     def validate(self):
         try:
             for cs in self._who_wants:
@@ -1328,6 +1384,13 @@ def validate(self):
 
                 pdb.set_trace()
 
+    def get_nbytes_deps(self):
+        nbytes: Py_ssize_t = 0
+        ts: TaskState
+        for ts in self._dependencies:
+            nbytes += ts.get_nbytes()
+        return nbytes
+
 
 class _StateLegacyMapping(Mapping):
     """
@@ -2621,10 +2684,12 @@ def stimulus_task_finished(self, key=None, worker=None, **kwargs):
         """ Mark that a task has finished execution on a particular worker """
         logger.debug("Stimulus task finished %s, %s", key, worker)
 
-        ts: TaskState = self.tasks.get(key)
+        tasks: dict = self.tasks
+        ts: TaskState = tasks.get(key)
         if ts is None:
             return {}
-        ws: WorkerState = self.workers[worker]
+        workers: dict = cast(dict, self.workers)
+        ws: WorkerState = workers[worker]
         ts._metadata.update(kwargs["metadata"])
 
         recommendations: dict
@@ -3645,7 +3710,7 @@ async def _delete_worker_data(self, worker_address, keys):
 
         ws: WorkerState = self.workers[worker_address]
         ts: TaskState
-        tasks = {self.tasks[key] for key in keys}
+        tasks: set = {self.tasks[key] for key in keys}
         ws._has_what -= tasks
         for ts in tasks:
             ts._who_has.remove(ws)
@@ -4399,23 +4464,24 @@ def get_comm_cost(self, ts: TaskState, ws: WorkerState):
         dts: TaskState
         deps: set = ts._dependencies - ws._has_what
         nbytes: Py_ssize_t = 0
+        bandwidth: double = self.bandwidth
         for dts in deps:
             nbytes += dts._nbytes
-        return nbytes / self.bandwidth
+        return nbytes / bandwidth
 
-    def get_task_duration(self, ts: TaskState, default=None):
+    def get_task_duration(self, ts: TaskState, default: double = -1):
         """
         Get the estimated computation cost of the given task
         (not including any communication cost).
         """
-        duration = ts._prefix._duration_average
+        duration: double = ts._prefix._duration_average
         if duration < 0:
-            self.unknown_durations[ts._prefix._name].add(ts)
-            if default is None:
-                default = parse_timedelta(
-                    dask.config.get("distributed.scheduler.unknown-task-duration")
-                )
-            return default
+            s: set = self.unknown_durations[ts._prefix._name]
+            s.add(ts)
+            if default < 0:
+                duration = UNKNOWN_TASK_DURATION
+            else:
+                duration = default
 
         return duration
 
@@ -5554,17 +5620,19 @@ def check_idle_saturated(self, ws: WorkerState, occ: double = -1.0):
 
         idle = self.idle
         saturated: set = self.saturated
-        if p < nc or occ / nc < avg / 2:
+        if p < nc or occ < nc * avg / 2:
             idle[ws._address] = ws
             saturated.discard(ws)
         else:
             idle.pop(ws._address, None)
 
-            pending: double = occ * (p - nc) / p / nc
-            if p > nc and pending > 0.4 and pending > 1.9 * avg:
-                saturated.add(ws)
-            else:
-                saturated.discard(ws)
+            if p > nc:
+                pending: double = occ * (p - nc) / (p * nc)
+                if 0.4 < pending > 1.9 * avg:
+                    saturated.add(ws)
+                    return
+
+            saturated.discard(ws)
 
     def valid_workers(self, ts: TaskState) -> set:
         """Return set of currently valid workers for key
diff --git a/distributed/stealing.py b/distributed/stealing.py
index 90cb4010612..1f62b6f6726 100644
--- a/distributed/stealing.py
+++ b/distributed/stealing.py
@@ -120,7 +120,7 @@ def steal_time_ratio(self, ts):
         if not ts.dependencies:  # no dependencies fast path
             return 0, 0
 
-        nbytes = sum(dep.get_nbytes() for dep in ts.dependencies)
+        nbytes = ts.get_nbytes_deps()
 
         transfer_time = nbytes / self.scheduler.bandwidth + LATENCY
         split = ts.prefix.name

From c2d87738828ddad421e4766cc9521b9e2c471857 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Mon, 14 Dec 2020 13:23:24 -0800
Subject: [PATCH 1104/1550] Provide `setup.py` option to profile Cython code
 (#4362)

As it can be useful for development to profile the Cythonized code and
this requires an argument be provided to the extensions to enable, go
ahead and add some logic to `setup.py` to parse this option if provided.
If it is not provided, profiling is turned off, which is preferable as
this would impact the performance of the compiled code.
---
 setup.py | 23 ++++++++++++++++-------
 1 file changed, 16 insertions(+), 7 deletions(-)

diff --git a/setup.py b/setup.py
index 7c91a44bdde..9160e1ffdc9 100755
--- a/setup.py
+++ b/setup.py
@@ -21,19 +21,27 @@
     else:
         install_requires.append(r)
 
-try:
-    sys.argv.remove("--with-cython")
-    cython = True
-except ValueError:
-    cython = False
+cython_arg = None
+for i in range(len(sys.argv)):
+    if sys.argv[i].startswith("--with-cython"):
+        cython_arg = sys.argv[i]
+        del sys.argv[i]
+        break
 
 ext_modules = []
-if cython:
+if cython_arg:
     try:
-        import cython
+        import cython  # noqa: F401
     except ImportError:
         setup_requires.append("cython")
 
+    profile = False
+    try:
+        _, param = cython_arg.split("=")
+        profile = param == "profile"
+    except ValueError:
+        pass
+
     cyext_modules = [
         Extension(
             "distributed.scheduler",
@@ -46,6 +54,7 @@
             "binding": False,
             "embedsignature": True,
             "language_level": 3,
+            "profile": profile,
         }
     ext_modules.extend(cyext_modules)
 

From 3fb3d8c8c14800ff8f1cd76d0052985f355ecfcc Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Thu, 17 Dec 2020 09:46:11 +0000
Subject: [PATCH 1105/1550] Dask internal inherit config (#4364)

* Change update method to ensure config is applied

* Add test for serialization methods

* Remove note

* Implement DASK_INTERNAL_INHERIT_CONFIG for dask-spec
---
 distributed/cli/dask_scheduler.py |  3 +--
 distributed/cli/dask_spec.py      |  8 ++++++++
 distributed/cli/dask_worker.py    |  3 +--
 distributed/tests/test_utils.py   | 18 ++++++++++++++++++
 4 files changed, 28 insertions(+), 4 deletions(-)

diff --git a/distributed/cli/dask_scheduler.py b/distributed/cli/dask_scheduler.py
index 45d4f484c86..335fcd63b5c 100755
--- a/distributed/cli/dask_scheduler.py
+++ b/distributed/cli/dask_scheduler.py
@@ -169,8 +169,7 @@ def main(
 
     if "DASK_INTERNAL_INHERIT_CONFIG" in os.environ:
         config = deserialize_for_cli(os.environ["DASK_INTERNAL_INHERIT_CONFIG"])
-        # Update the global config given priority to the existing global config
-        dask.config.update(dask.config.global_config, config, priority="old")
+        dask.config.update(dask.config.global_config, config)
 
     if not host and (tls_ca_file or tls_cert or tls_key):
         host = "tls://"
diff --git a/distributed/cli/dask_spec.py b/distributed/cli/dask_spec.py
index 0a224e5b37c..299878a3a46 100644
--- a/distributed/cli/dask_spec.py
+++ b/distributed/cli/dask_spec.py
@@ -1,10 +1,13 @@
 import asyncio
 import click
 import json
+import os
 import sys
 import yaml
 
+import dask.config
 from distributed.deploy.spec import run_spec
+from distributed.utils import deserialize_for_cli
 
 
 @click.command(context_settings=dict(ignore_unknown_options=True))
@@ -13,6 +16,11 @@
 @click.option("--spec-file", type=str, default=None, help="")
 @click.version_option()
 def main(args, spec: str, spec_file: str):
+
+    if "DASK_INTERNAL_INHERIT_CONFIG" in os.environ:
+        config = deserialize_for_cli(os.environ["DASK_INTERNAL_INHERIT_CONFIG"])
+        dask.config.update(dask.config.global_config, config)
+
     if spec and spec_file or not spec and not spec_file:
         print("Must specify exactly one of --spec and --spec-file")
         sys.exit(1)
diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
index b3b546ea3cc..140d0d1ab8f 100755
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -397,8 +397,7 @@ def del_pid_file():
 
     if "DASK_INTERNAL_INHERIT_CONFIG" in os.environ:
         config = deserialize_for_cli(os.environ["DASK_INTERNAL_INHERIT_CONFIG"])
-        # Update the global config given priority to the existing global config
-        dask.config.update(dask.config.global_config, config, priority="old")
+        dask.config.update(dask.config.global_config, config)
 
     nannies = [
         t(
diff --git a/distributed/tests/test_utils.py b/distributed/tests/test_utils.py
index 01332f38834..022e9925445 100644
--- a/distributed/tests/test_utils.py
+++ b/distributed/tests/test_utils.py
@@ -45,6 +45,8 @@
     LRU,
     offload,
     TimeoutError,
+    deserialize_for_cli,
+    serialize_for_cli,
 )
 from distributed.utils_test import loop, loop_in_thread  # noqa: F401
 from distributed.utils_test import div, has_ipv6, inc, throws, gen_test, captured_logger
@@ -606,3 +608,19 @@ def test_lru():
 async def test_offload():
     assert (await offload(inc, 1)) == 2
     assert (await offload(lambda x, y: x + y, 1, y=2)) == 3
+
+
+def test_cli_serialization():
+    # Use context manager without changing the value to ensure test side effects are restored
+    with dask.config.set(
+        {
+            "distributed.comm.default-scheme": dask.config.get(
+                "distributed.comm.default-scheme"
+            )
+        }
+    ):
+        config = deserialize_for_cli(
+            serialize_for_cli({"distributed": {"comm": {"default-scheme": "tls"}}})
+        )  # Take a round trip through the serialization
+        dask.config.update(dask.config.global_config, config)
+        assert dask.config.get("distributed.comm.default-scheme") == "tls"

From 976e02eb09efa630f061af7beafadec4da14deaf Mon Sep 17 00:00:00 2001
From: selshowk <67599884+selshowk@users.noreply.github.com>
Date: Mon, 21 Dec 2020 23:52:45 +0100
Subject: [PATCH 1106/1550] Better task duration estimates for outliers (#4213)

Rather than always using the average task duration as an estimate we
flag "outliers" (tasks that are taking 2x longer than expected duration)
and we set expected duration to be twice their current running time.
---
 distributed/scheduler.py         | 55 ++++++++++++++++++++++++--------
 distributed/tests/test_steal.py  | 16 ++++++++++
 distributed/tests/test_worker.py | 14 ++++----
 distributed/worker.py            | 11 ++++---
 4 files changed, 71 insertions(+), 25 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index ed3f7150e9e..c66c7183d53 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -274,6 +274,12 @@ class WorkerState:
 
        This attribute is kept in sync with :attr:`TaskState.processing_on`.
 
+    .. attribute:: executing: {TaskState: duration}
+
+       A dictionary of tasks that are currently being run on this worker.
+       Each task state is asssociated with the duration in seconds which
+       the task has been running.
+
     .. attribute:: has_what: {TaskState}
 
        The set of tasks which currently reside on this worker.
@@ -334,6 +340,7 @@ class WorkerState:
     _actors: set
     _address: str
     _bandwidth: double
+    _executing: dict
     _extra: dict
     _has_what: set
     _hash: Py_hash_t
@@ -360,6 +367,7 @@ class WorkerState:
         "_address",
         "_bandwidth",
         "_extra",
+        "_executing",
         "_has_what",
         "_hash",
         "_last_seen",
@@ -418,6 +426,7 @@ def __init__(
         self._actors = set()
         self._has_what = set()
         self._processing = {}
+        self._executing = {}
         self._resources = {}
         self._used_resources = {}
 
@@ -447,6 +456,10 @@ def address(self):
     def bandwidth(self):
         return self._bandwidth
 
+    @property
+    def executing(self):
+        return self._executing
+
     @property
     def extra(self):
         return self._extra
@@ -562,6 +575,7 @@ def clean(self):
         )
         ts: TaskState
         ws._processing = {ts._key: cost for ts, cost in self._processing.items()}
+        ws._executing = {ts._key: duration for ts, duration in self._executing.items()}
         return ws
 
     def __repr__(self):
@@ -2127,6 +2141,7 @@ def heartbeat_worker(
         resources=None,
         host_info=None,
         metrics=None,
+        executing=None,
     ):
         address = self.coerce_address(address, resolve_address)
         address = normalize_address(address)
@@ -2165,6 +2180,11 @@ def heartbeat_worker(
 
         ws._last_seen = time()
 
+        if executing is not None:
+            ws._executing = {
+                self.tasks[key]: duration for key, duration in executing.items()
+            }
+
         if metrics:
             ws._metrics = metrics
 
@@ -4789,6 +4809,23 @@ def decide_worker(self, ts: TaskState) -> WorkerState:
 
         return ws
 
+    def set_duration_estimate(self, ts: TaskState, ws: WorkerState):
+        """Estimate task duration using worker state and task state.
+
+        If a task takes longer than twice the current average duration we
+        estimate the task duration to be 2x current-runtime, otherwise we set it
+        to be the average duration.
+        """
+        duration: double = self.get_task_duration(ts)
+        comm: double = self.get_comm_cost(ts, ws)
+        total_duration: double = duration + comm
+        if ts in ws._executing:
+            exec_time: double = ws._executing[ts]
+            if exec_time > 2 * duration:
+                total_duration = 2 * exec_time
+        ws._processing[ts] = total_duration
+        return ws._processing[ts]
+
     def transition_waiting_processing(self, key):
         try:
             tasks: dict = self.tasks
@@ -4809,14 +4846,10 @@ def transition_waiting_processing(self, key):
                 return {}
             worker = ws._address
 
-            duration = self.get_task_duration(ts)
-            comm = self.get_comm_cost(ts, ws)
-            occupancy = duration + comm
-
-            ws._processing[ts] = occupancy
+            duration_estimate = self.set_duration_estimate(ts, ws)
             ts._processing_on = ws
-            ws._occupancy += occupancy
-            self.total_occupancy += occupancy
+            ws._occupancy += duration_estimate
+            self.total_occupancy += duration_estimate
             ts.state = "processing"
             self.consume_resources(ts, ws)
             self.check_idle_saturated(ws)
@@ -4827,7 +4860,7 @@ def transition_waiting_processing(self, key):
 
             # logger.debug("Send job to worker: %s, %s", worker, key)
 
-            self.send_task_to_worker(worker, ts, duration)
+            self.send_task_to_worker(worker, ts)
 
             return {}
         except Exception as e:
@@ -6101,11 +6134,7 @@ def _reevaluate_occupancy_worker(self, ws: WorkerState):
         new = 0
         nbytes = 0
         for ts in ws._processing:
-            duration = self.get_task_duration(ts)
-            comm = self.get_comm_cost(ts, ws)
-            occupancy = duration + comm
-            ws._processing[ts] = occupancy
-            new += occupancy
+            new += self.set_duration_estimate(ts, ws)
 
         ws._occupancy = new
         self.total_occupancy += new - old
diff --git a/distributed/tests/test_steal.py b/distributed/tests/test_steal.py
index 8277ede1833..bee0f06544e 100644
--- a/distributed/tests/test_steal.py
+++ b/distributed/tests/test_steal.py
@@ -809,3 +809,19 @@ async def test_worker_stealing_interval(c, s, a, b):
     with dask.config.set({"distributed.scheduler.work-stealing-interval": 2}):
         ws = WorkStealing(s)
     assert ws._pc.callback_time == 2
+
+
+@gen_cluster(client=True)
+async def test_balance_with_longer_task(c, s, a, b):
+    np = pytest.importorskip("numpy")
+
+    await c.submit(slowinc, 0, delay=0)  # scheduler learns that slowinc is very fast
+    x = await c.scatter(np.arange(10000), workers=[a.address])
+    y = c.submit(
+        slowinc, 1, delay=5, workers=[a.address], priority=1
+    )  # a surprisingly long task
+    z = c.submit(
+        inc, x, workers=[a.address], allow_other_workers=True, priority=0
+    )  # a task after y, suggesting a, but open to b
+    await z
+    assert z.key in b.data
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index fc758da2338..112263ecd7d 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -1682,19 +1682,19 @@ async def test_update_latency(cleanup):
 
 
 @pytest.mark.asyncio
-async def test_heartbeat_executing(cleanup):
+async def test_workerstate_executing(cleanup):
     async with await Scheduler() as s:
         async with await Worker(s.address) as w:
             async with Client(s.address, asynchronous=True) as c:
                 ws = s.workers[w.address]
                 # Initially there are no active tasks
-                assert not ws.metrics["executing"]
-                # Submit a task and ensure the worker's heartbeat includes the task
-                # in it's executing
+                assert not ws.executing
+                # Submit a task and ensure the WorkerState is updated with the task
+                # it's executing
                 f = c.submit(slowinc, 1, delay=1)
-                while not ws.metrics["executing"]:
-                    await w.heartbeat()
-                assert f.key in ws.metrics["executing"]
+                while not ws.executing:
+                    await asyncio.sleep(0.01)
+                assert s.tasks[f.key] in ws.executing
                 await f
 
 
diff --git a/distributed/worker.py b/distributed/worker.py
index 205369623d1..87ee7fe368d 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -797,6 +797,7 @@ def local_dir(self):
     async def get_metrics(self):
         now = time()
         core = dict(
+            executing=self.executing_count,
             in_memory=len(self.data),
             ready=len(self.ready),
             in_flight=self.in_flight_tasks,
@@ -805,11 +806,6 @@ async def get_metrics(self):
                 "workers": dict(self.bandwidth_workers),
                 "types": keymap(typename, self.bandwidth_types),
             },
-            executing={
-                key: now - self.tasks[key].start_time
-                for key in self.active_threads.values()
-                if key in self.tasks
-            },
         )
         custom = {}
         for k, metric in self.metrics.items():
@@ -937,6 +933,11 @@ async def heartbeat(self):
                     address=self.contact_address,
                     now=time(),
                     metrics=await self.get_metrics(),
+                    executing={
+                        key: start - self.tasks[key].start_time
+                        for key in self.active_threads.values()
+                        if key in self.tasks
+                    },
                 )
                 end = time()
                 middle = (start + end) / 2

From 3c06883d8a2157315d8abaa00c06d88fd468a9a4 Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Tue, 22 Dec 2020 21:41:33 +0100
Subject: [PATCH 1107/1550] Nanny now respects dask settings from ctx mgr
 (#4378)

Co-authored-by: James Bourbeau <jrbourbeau@gmail.com>
---
 distributed/nanny.py                 |  2 +-
 distributed/tests/test_utils_test.py | 19 ++++++++++++++++++-
 2 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/distributed/nanny.py b/distributed/nanny.py
index 85c814272c5..5ddc28b0f1d 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -144,7 +144,7 @@ def __init__(
 
         self.Worker = Worker if worker_class is None else worker_class
         self.env = env or {}
-        self.config = config or {}
+        self.config = config or dask.config.config
         worker_kwargs.update(
             {
                 "port": worker_port,
diff --git a/distributed/tests/test_utils_test.py b/distributed/tests/test_utils_test.py
index 4e9e776b590..6093d2eaea7 100755
--- a/distributed/tests/test_utils_test.py
+++ b/distributed/tests/test_utils_test.py
@@ -7,7 +7,7 @@
 import pytest
 from tornado import gen
 
-from distributed import Scheduler, Worker, Client, config, default_client
+from distributed import Scheduler, Worker, Nanny, Client, config, default_client
 from distributed.core import rpc
 from distributed.metrics import time
 from distributed.utils_test import (  # noqa: F401
@@ -53,6 +53,23 @@ async def test_gen_cluster(c, s, a, b):
     assert await c.submit(lambda: 123) == 123
 
 
+@gen_cluster(
+    client=True,
+    Worker=Nanny,
+    config={"distributed.comm.timeouts.connect": "1s", "new.config.value": "foo"},
+)
+async def test_gen_cluster_set_config_nanny(c, s, a, b):
+    def assert_config():
+        import dask
+
+        assert dask.config.get("distributed.comm.timeouts.connect") == "1s"
+        assert dask.config.get("new.config.value") == "foo"
+        return dask.config
+
+    await c.run(assert_config)
+    await c.run_on_scheduler(assert_config)
+
+
 @gen_cluster(client=True)
 def test_gen_cluster_legacy_implicit(c, s, a, b):
     assert isinstance(c, Client)

From 4386b7566ee57f5c224bdeddf4f5fc60776ff47a Mon Sep 17 00:00:00 2001
From: Bruno Pagani <bruno.n.pagani@gmail.com>
Date: Wed, 23 Dec 2020 21:47:44 +0400
Subject: [PATCH 1108/1550] test_performance_report: skip without bokeh (#4388)

This test is failing if bokeh is not available.
---
 distributed/tests/test_client.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 2a477a49427..0625bf8a0fa 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -6129,6 +6129,7 @@ async def f(dask_worker):
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 2)] * 2)
 async def test_performance_report(c, s, a, b):
+    pytest.importorskip("bokeh")
     da = pytest.importorskip("dask.array")
 
     async def f():

From 607cfd2ce00edd44c99da3273de0763a426dda7d Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Mon, 4 Jan 2021 09:37:27 -0800
Subject: [PATCH 1109/1550] Drop custom `__eq__` from `Status` (#4270)

* Drop custom `__eq__` from `Status`

This appears to eat up a good chunk of time in places where the `Status`
needs to be checked. So try dropping this method in favor of the default
implementation.

* Drop tests that would be no longer needed

If we drop the special `__eq__` method, then these tests are no longer
relevant as they would error instead. So go ahead and drop them for now
should we decide to go ahead with this change.
---
 distributed/core.py            | 26 --------------------------
 distributed/tests/test_core.py | 24 ------------------------
 2 files changed, 50 deletions(-)

diff --git a/distributed/core.py b/distributed/core.py
index 001f3bac31f..1d98241bb1f 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -61,32 +61,6 @@ class Status(Enum):
     undefined = None
     dont_reply = "dont-reply"
 
-    def __eq__(self, other):
-        """
-        Implement equality checking with backward compatibility.
-
-        If other object instance is string, we compare with the values, but we
-        actually want to make sure the value compared with is in the list of
-        possible Status, this avoid comparison with non-existing status.
-        """
-        if isinstance(other, type(self)):
-            return self.value == other.value
-        elif isinstance(other, str) or (other is None):
-            warnings.warn(
-                f"Since distributed 2.23 `.status` is now an Enum, please compare with `Status.{other}`",
-                PendingDeprecationWarning,
-                stacklevel=1,
-            )
-            assert other in [
-                s.value for s in type(self)
-            ], f"comparison with non-existing states {other}"
-            return other == self.value
-        raise TypeError(
-            f"'==' not supported between instances of"
-            f" {type(self).__module__+'.'+type(self).__qualname__!r} and"
-            f" {type(other).__module__+'.'+type(other).__qualname__!r}"
-        )
-
 
 class RPCClosed(IOError):
     pass
diff --git a/distributed/tests/test_core.py b/distributed/tests/test_core.py
index b43b143721f..07dc6c29434 100644
--- a/distributed/tests/test_core.py
+++ b/distributed/tests/test_core.py
@@ -100,15 +100,6 @@ def test_server_status_assign_non_variant_raises():
             server.status = "I do not exists"
 
 
-def test_server_status_compare_non_variant_raises():
-    server = Server({})
-    # turn off warnings into error for assertion checking.
-    with warnings.catch_warnings(record=True) as w:
-        warnings.simplefilter("default")
-        with pytest.raises(AssertionError):
-            server.status == "You can't compare with me"
-
-
 def test_server_status_assign_with_variant_warns():
     server = Server({})
     with warnings.catch_warnings(record=True) as w:
@@ -117,14 +108,6 @@ def test_server_status_assign_with_variant_warns():
             server.status = "running"
 
 
-def test_server_status_compare_with_variant_warns():
-    server = Server({})
-    with warnings.catch_warnings(record=True) as w:
-        warnings.simplefilter("default")
-        with pytest.warns(PendingDeprecationWarning):
-            server.status == "running"
-
-
 def test_server_status_assign_with_variant_raises_in_tests():
     """That would be the default in user code"""
     server = Server({})
@@ -132,13 +115,6 @@ def test_server_status_assign_with_variant_raises_in_tests():
         server.status = "running"
 
 
-def test_server_status_compare_with_variant_raises_in_tests():
-    """That would be the default in user code"""
-    server = Server({})
-    with pytest.raises(PendingDeprecationWarning):
-        server.status == "running"
-
-
 def test_server_assign_assign_enum_is_quiet():
     """That would be the default in user code"""
     server = Server({})

From 76ef459fbb0af0bf0f3c6f16c86c489c1bbafbb0 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Tue, 5 Jan 2021 10:07:52 -0800
Subject: [PATCH 1110/1550] Fix compilation warnings, `decide_worker` now a C
 func, stealing improvements (#4375)

Fix compilation warnings, `decide_worker` now a C func, stealing improvements
---
 distributed/scheduler.py | 13 ++++++++++++-
 distributed/stealing.py  | 17 +++++++++--------
 2 files changed, 21 insertions(+), 9 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index c66c7183d53..a29560617d9 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -86,11 +86,17 @@
 from .protocol.highlevelgraph import highlevelgraph_unpack
 
 try:
+    from cython import compiled
+except ImportError:
+    compiled = False
+
+if compiled:
     from cython import (
         bint,
         cast,
         ccall,
         cclass,
+        cfunc,
         declare,
         double,
         exceptval,
@@ -100,7 +106,7 @@
         Py_hash_t,
         Py_ssize_t,
     )
-except ImportError:
+else:
     from ctypes import (
         c_double as double,
         c_ssize_t as Py_hash_t,
@@ -118,6 +124,9 @@ def ccall(func):
     def cclass(cls):
         return cls
 
+    def cfunc(func):
+        return func
+
     def declare(*a, **k):
         if len(a) == 2:
             return a[1]
@@ -6233,6 +6242,8 @@ def adaptive_target(self, comm=None, target_duration=None):
             return len(self.workers) - len(to_close)
 
 
+@cfunc
+@exceptval(check=False)
 def decide_worker(
     ts: TaskState, all_workers, valid_workers: set, objective
 ) -> WorkerState:
diff --git a/distributed/stealing.py b/distributed/stealing.py
index 1f62b6f6726..c1f07a8adfc 100644
--- a/distributed/stealing.py
+++ b/distributed/stealing.py
@@ -1,6 +1,6 @@
 from collections import defaultdict, deque
 import logging
-from math import log
+from math import log2
 from time import time
 
 from tornado.ioloop import PeriodicCallback
@@ -14,7 +14,6 @@
 from tlz import topk
 
 LATENCY = 10e-3
-log_2 = log(2)
 
 logger = logging.getLogger(__name__)
 
@@ -85,7 +84,6 @@ def put_key_in_stealable(self, ts):
         ws = ts.processing_on
         worker = ws.address
         cost_multiplier, level = self.steal_time_ratio(ts)
-        self.log(("add-stealable", ts.key, worker, level))
         if cost_multiplier is not None:
             self.stealable_all[level].add(ts)
             self.stealable[worker][level].add(ts)
@@ -120,22 +118,25 @@ def steal_time_ratio(self, ts):
         if not ts.dependencies:  # no dependencies fast path
             return 0, 0
 
-        nbytes = ts.get_nbytes_deps()
-
-        transfer_time = nbytes / self.scheduler.bandwidth + LATENCY
         split = ts.prefix.name
         if split in fast_tasks:
             return None, None
+
         ws = ts.processing_on
         compute_time = ws.processing[ts]
         if compute_time < 0.005:  # 5ms, just give up
             return None, None
+
+        nbytes = ts.get_nbytes_deps()
+        transfer_time = nbytes / self.scheduler.bandwidth + LATENCY
         cost_multiplier = transfer_time / compute_time
         if cost_multiplier > 100:
             return None, None
 
-        level = int(round(log(cost_multiplier) / log_2 + 6, 0))
-        level = max(1, level)
+        level = int(round(log2(cost_multiplier) + 6))
+        if level < 1:
+            level = 1
+
         return cost_multiplier, level
 
     def move_task_request(self, ts, victim, thief):

From a07fc398afb4c87949ab82a176989a2e1c855e54 Mon Sep 17 00:00:00 2001
From: "Bernhard M. Wiedemann" <githubbmw2020@lsmod.de>
Date: Wed, 6 Jan 2021 17:24:10 +0100
Subject: [PATCH 1111/1550] Make tests pass after 2028 (#4403)

This PR was done while working on reproducible builds for openSUSE.
---
 distributed/tests/make_tls_certs.py        |  10 +-
 distributed/tests/tls-ca-cert.pem          | 110 +++++------
 distributed/tests/tls-ca-key.pem           |  52 ++---
 distributed/tests/tls-cert-chain.pem       | 214 ++++++++++-----------
 distributed/tests/tls-cert.pem             | 104 +++++-----
 distributed/tests/tls-key-cert.pem         | 156 +++++++--------
 distributed/tests/tls-key.pem              |  52 ++---
 distributed/tests/tls-self-signed-cert.pem |  34 ++--
 distributed/tests/tls-self-signed-key.pem  |  52 ++---
 9 files changed, 392 insertions(+), 392 deletions(-)

diff --git a/distributed/tests/make_tls_certs.py b/distributed/tests/make_tls_certs.py
index 0c1c5876134..02817cbdf5e 100644
--- a/distributed/tests/make_tls_certs.py
+++ b/distributed/tests/make_tls_certs.py
@@ -34,8 +34,8 @@
     database  = $dir/index.txt
     crlnumber = $dir/crl.txt
     default_md = sha256
-    default_days = 3600
-    default_crl_days = 3600
+    default_days = 360000
+    default_crl_days = 360000
     certificate = tls-ca-cert.pem
     private_key = tls-ca-key.pem
     serial    = $dir/serial
@@ -84,7 +84,7 @@ def make_cert_key(hostname, sign=False):
             "req",
             "-new",
             "-days",
-            "3650",
+            "365242",
             "-nodes",
             "-newkey",
             "rsa:2048",
@@ -154,7 +154,7 @@ def make_ca():
                 "req",
                 "-new",
                 "-days",
-                "3650",
+                "365242",
                 "-extensions",
                 "v3_ca",
                 "-nodes",
@@ -181,7 +181,7 @@ def make_ca():
                 "-keyfile",
                 "tls-ca-key.pem",
                 "-days",
-                "3650",
+                "365242",
                 "-selfsign",
                 "-extensions",
                 "v3_ca",
diff --git a/distributed/tests/tls-ca-cert.pem b/distributed/tests/tls-ca-cert.pem
index 5441bacbc78..2ba3b04dc03 100644
--- a/distributed/tests/tls-ca-cert.pem
+++ b/distributed/tests/tls-ca-cert.pem
@@ -2,77 +2,77 @@ Certificate:
     Data:
         Version: 3 (0x2)
         Serial Number:
-            3f:d8:8a:ed:26:a6:7e:6f:53:9f:fe:85:a4:a3:62:44:65:9f:ff:46
+            3d:ba:1c:2e:06:59:1d:b8:7d:f0:67:cd:e2:68:3d:e4:70:ac:81:85
         Signature Algorithm: sha256WithRSAEncryption
         Issuer: C=XY, O=Dask CA, CN=our-ca-server
         Validity
-            Not Before: Dec 13 02:52:10 2018 GMT
-            Not After : Dec 10 02:52:10 2028 GMT
+            Not Before: Jan  6 07:46:08 2021 GMT
+            Not After : Jan  6 07:46:08 3021 GMT
         Subject: C=XY, O=Dask CA, CN=our-ca-server
         Subject Public Key Info:
             Public Key Algorithm: rsaEncryption
                 RSA Public-Key: (2048 bit)
                 Modulus:
-                    00:d0:49:58:02:6d:4c:db:00:c2:bf:90:d7:95:f6:
-                    87:c0:8d:9c:1e:3b:c4:ed:2c:33:02:2d:be:58:82:
-                    59:2b:7e:4c:91:23:8b:73:2c:97:89:07:2d:ac:93:
-                    5e:39:11:25:75:11:7a:1a:d0:d6:e3:80:e6:e3:5b:
-                    93:db:3d:97:be:bb:96:3f:ff:1d:26:c8:fe:e2:a8:
-                    cb:28:cb:3b:5f:88:05:df:27:81:b1:38:29:d5:77:
-                    6d:f7:e5:9a:19:60:4b:f8:a9:53:00:90:c8:a3:32:
-                    0e:73:0a:3b:51:a9:ab:92:1e:6b:2f:28:82:01:18:
-                    6e:13:fd:69:1a:79:bd:a9:40:cb:70:b7:92:75:1e:
-                    19:0c:b4:3c:c1:7e:55:80:05:be:8d:f0:80:d9:25:
-                    89:74:1a:92:1d:ba:26:a1:65:39:96:50:46:95:e4:
-                    be:29:47:8b:77:0d:11:eb:a1:7a:87:19:14:bf:82:
-                    0a:4d:ca:d4:6d:40:66:4a:5f:12:cd:a3:16:14:19:
-                    a2:8d:7a:07:80:36:04:56:e6:f3:7d:38:c6:75:97:
-                    c9:7c:95:9a:f9:ff:79:6d:c7:40:7a:c2:92:2d:ef:
-                    7a:52:07:de:6b:03:9f:af:17:72:63:53:31:88:57:
-                    76:d5:45:e1:2f:c5:aa:ed:93:d6:46:7e:07:00:cf:
-                    49:eb
+                    00:98:f6:94:70:e6:f7:65:a9:fd:85:9c:a2:e2:ae:
+                    08:3b:c4:93:4a:62:95:c3:19:f9:65:dd:f1:df:28:
+                    ee:0b:e4:a1:a8:7d:68:97:b5:40:45:94:c7:a7:79:
+                    62:0f:dd:ff:87:ea:12:76:ee:91:f1:3f:0e:0e:4f:
+                    bd:b6:8f:26:40:63:08:fb:47:71:8a:0b:ae:71:d9:
+                    86:10:f0:fc:02:e1:37:bb:87:68:3f:cb:67:44:6e:
+                    60:04:64:38:26:40:96:dc:00:94:e0:2a:e5:8d:c8:
+                    a2:9b:d1:12:d0:fa:43:ca:29:e7:42:0c:33:3a:f9:
+                    cf:0a:01:54:3a:bf:02:b2:b4:49:46:6c:3c:d9:b5:
+                    2c:0a:5c:24:0b:5a:0f:66:6e:31:4b:f5:33:4a:67:
+                    61:e5:71:02:02:cb:16:1e:00:ad:bd:a5:8a:60:6b:
+                    ad:09:25:14:8b:d0:ab:60:eb:d5:6b:70:af:9a:b6:
+                    66:36:44:08:6a:6d:c4:92:72:ce:9b:83:a3:54:04:
+                    3e:08:13:8d:0a:0b:4e:7a:4c:da:8e:ba:a0:34:76:
+                    3b:ce:56:9c:d9:6e:19:ac:e9:3a:50:66:90:56:90:
+                    bc:ff:a9:f2:1e:ac:b2:9f:01:7c:96:50:6f:bf:84:
+                    df:22:9c:f6:b1:21:94:8a:79:3c:6c:dc:94:c2:e7:
+                    a7:f7
                 Exponent: 65537 (0x10001)
         X509v3 extensions:
             X509v3 Subject Key Identifier: 
-                59:66:3D:F8:65:B7:98:7E:C1:E4:31:19:34:E5:A3:0D:2E:3D:F9:38
+                EB:0C:96:1D:60:5D:3A:39:00:EA:D8:C3:73:B0:4E:C9:D6:B9:5D:5B
             X509v3 Authority Key Identifier: 
-                keyid:59:66:3D:F8:65:B7:98:7E:C1:E4:31:19:34:E5:A3:0D:2E:3D:F9:38
+                keyid:EB:0C:96:1D:60:5D:3A:39:00:EA:D8:C3:73:B0:4E:C9:D6:B9:5D:5B
 
             X509v3 Basic Constraints: 
                 CA:TRUE
     Signature Algorithm: sha256WithRSAEncryption
-         a2:bd:c7:e4:23:24:4e:da:db:d3:bd:7f:33:87:5f:1f:3b:e8:
-         fa:19:54:17:eb:60:86:ac:d6:18:47:35:90:88:47:d6:f1:5a:
-         31:ed:e1:44:b4:16:6a:04:c3:6f:9b:ba:cf:99:14:0a:c8:32:
-         0a:69:fd:41:c3:67:25:3d:f7:84:31:81:8b:2e:b0:9c:04:74:
-         2c:0b:87:b2:a8:48:a2:c7:42:bb:90:31:82:a5:51:38:03:df:
-         e7:aa:76:f7:1a:ae:b9:69:5c:94:62:7f:19:3e:5c:6b:97:99:
-         9e:4a:9e:4b:34:f5:39:15:fe:12:72:88:7b:1f:be:db:3a:3b:
-         7f:14:06:7b:ef:b2:fc:c4:ad:82:51:4a:95:6c:23:a5:bb:65:
-         38:0e:55:70:e1:96:88:bc:29:82:38:48:63:ac:5a:73:20:3c:
-         d7:39:25:e9:35:46:9b:6b:b1:a0:77:56:a0:a6:30:0f:7e:0e:
-         12:10:11:41:28:6a:f4:ca:b5:c3:a9:fd:fe:9f:cf:29:5e:26:
-         d7:a8:4c:28:a5:31:10:c2:07:25:c2:0a:5d:35:73:f7:db:3c:
-         6b:5d:2f:1a:f1:b0:e7:81:26:22:fe:38:63:2f:3e:5b:a7:d5:
-         b5:a8:ef:fe:87:b4:6f:2d:5e:7e:48:6c:c0:b9:ca:7f:ab:28:
-         c3:31:ae:93
+         08:9b:4d:5a:f0:f0:3d:ef:2e:d2:20:fc:9a:91:41:ba:f6:12:
+         5b:49:7e:2c:84:4d:f7:cf:2b:54:9c:d3:31:d5:54:da:b5:80:
+         0f:04:e5:ca:79:81:19:d4:89:c6:55:fa:b0:0d:9d:3a:f7:34:
+         39:00:3b:db:18:66:45:25:9a:1c:11:02:c0:e3:46:b9:72:37:
+         a8:49:dc:0a:7f:fd:1b:fb:78:6c:e3:ae:2f:ac:99:dc:bd:3e:
+         14:12:f2:c3:f9:75:97:7d:d3:74:61:2b:fa:07:78:e5:99:ed:
+         58:31:87:ee:07:d0:d1:31:b3:b6:e0:90:1b:93:ac:e2:f2:71:
+         ce:c8:81:56:be:9c:2e:22:a7:8e:6c:29:dc:e8:69:4e:26:af:
+         41:4d:57:b2:f7:62:a5:d5:33:0e:0d:3d:fb:bb:35:a9:bf:e0:
+         0a:4e:75:85:d5:39:aa:02:29:d7:a7:20:24:97:2a:48:0f:33:
+         57:24:8d:11:02:88:c7:19:bb:22:95:e7:7b:09:51:ef:79:71:
+         40:8e:36:f6:40:4a:df:3c:a4:ca:f3:ac:60:ee:a8:8f:58:38:
+         88:07:06:78:5b:6c:cd:38:e4:1f:aa:69:d4:a1:ba:dc:59:21:
+         3b:e5:5c:50:ac:0b:5c:3f:c3:c9:5a:6a:fb:c3:46:5f:97:93:
+         7f:f1:a7:0f
 -----BEGIN CERTIFICATE-----
-MIIDTDCCAjSgAwIBAgIUP9iK7Samfm9Tn/6FpKNiRGWf/0YwDQYJKoZIhvcNAQEL
+MIIDTjCCAjagAwIBAgIUPbocLgZZHbh98GfN4mg95HCsgYUwDQYJKoZIhvcNAQEL
 BQAwNzELMAkGA1UEBhMCWFkxEDAOBgNVBAoMB0Rhc2sgQ0ExFjAUBgNVBAMMDW91
-ci1jYS1zZXJ2ZXIwHhcNMTgxMjEzMDI1MjEwWhcNMjgxMjEwMDI1MjEwWjA3MQsw
-CQYDVQQGEwJYWTEQMA4GA1UECgwHRGFzayBDQTEWMBQGA1UEAwwNb3VyLWNhLXNl
-cnZlcjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANBJWAJtTNsAwr+Q
-15X2h8CNnB47xO0sMwItvliCWSt+TJEji3Msl4kHLayTXjkRJXURehrQ1uOA5uNb
-k9s9l767lj//HSbI/uKoyyjLO1+IBd8ngbE4KdV3bfflmhlgS/ipUwCQyKMyDnMK
-O1Gpq5Ieay8oggEYbhP9aRp5valAy3C3knUeGQy0PMF+VYAFvo3wgNkliXQakh26
-JqFlOZZQRpXkvilHi3cNEeuheocZFL+CCk3K1G1AZkpfEs2jFhQZoo16B4A2BFbm
-8304xnWXyXyVmvn/eW3HQHrCki3velIH3msDn68XcmNTMYhXdtVF4S/Fqu2T1kZ+
-BwDPSesCAwEAAaNQME4wHQYDVR0OBBYEFFlmPfhlt5h+weQxGTTlow0uPfk4MB8G
-A1UdIwQYMBaAFFlmPfhlt5h+weQxGTTlow0uPfk4MAwGA1UdEwQFMAMBAf8wDQYJ
-KoZIhvcNAQELBQADggEBAKK9x+QjJE7a29O9fzOHXx876PoZVBfrYIas1hhHNZCI
-R9bxWjHt4US0FmoEw2+bus+ZFArIMgpp/UHDZyU994QxgYsusJwEdCwLh7KoSKLH
-QruQMYKlUTgD3+eqdvcarrlpXJRifxk+XGuXmZ5Knks09TkV/hJyiHsfvts6O38U
-BnvvsvzErYJRSpVsI6W7ZTgOVXDhloi8KYI4SGOsWnMgPNc5Jek1RptrsaB3VqCm
-MA9+DhIQEUEoavTKtcOp/f6fzyleJteoTCilMRDCByXCCl01c/fbPGtdLxrxsOeB
-JiL+OGMvPlun1bWo7/6HtG8tXn5IbMC5yn+rKMMxrpM=
+ci1jYS1zZXJ2ZXIwIBcNMjEwMTA2MDc0NjA4WhgPMzAyMTAxMDYwNzQ2MDhaMDcx
+CzAJBgNVBAYTAlhZMRAwDgYDVQQKDAdEYXNrIENBMRYwFAYDVQQDDA1vdXItY2Et
+c2VydmVyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAmPaUcOb3Zan9
+hZyi4q4IO8STSmKVwxn5Zd3x3yjuC+ShqH1ol7VARZTHp3liD93/h+oSdu6R8T8O
+Dk+9to8mQGMI+0dxiguucdmGEPD8AuE3u4doP8tnRG5gBGQ4JkCW3ACU4Crljcii
+m9ES0PpDyinnQgwzOvnPCgFUOr8CsrRJRmw82bUsClwkC1oPZm4xS/UzSmdh5XEC
+AssWHgCtvaWKYGutCSUUi9CrYOvVa3CvmrZmNkQIam3EknLOm4OjVAQ+CBONCgtO
+ekzajrqgNHY7zlac2W4ZrOk6UGaQVpC8/6nyHqyynwF8llBvv4TfIpz2sSGUink8
+bNyUwuen9wIDAQABo1AwTjAdBgNVHQ4EFgQU6wyWHWBdOjkA6tjDc7BOyda5XVsw
+HwYDVR0jBBgwFoAU6wyWHWBdOjkA6tjDc7BOyda5XVswDAYDVR0TBAUwAwEB/zAN
+BgkqhkiG9w0BAQsFAAOCAQEACJtNWvDwPe8u0iD8mpFBuvYSW0l+LIRN988rVJzT
+MdVU2rWADwTlynmBGdSJxlX6sA2dOvc0OQA72xhmRSWaHBECwONGuXI3qEncCn/9
+G/t4bOOuL6yZ3L0+FBLyw/l1l33TdGEr+gd45ZntWDGH7gfQ0TGztuCQG5Os4vJx
+zsiBVr6cLiKnjmwp3OhpTiavQU1XsvdipdUzDg09+7s1qb/gCk51hdU5qgIp16cg
+JJcqSA8zVySNEQKIxxm7IpXnewlR73lxQI429kBK3zykyvOsYO6oj1g4iAcGeFts
+zTjkH6pp1KG63FkhO+VcUKwLXD/DyVpq+8NGX5eTf/GnDw==
 -----END CERTIFICATE-----
diff --git a/distributed/tests/tls-ca-key.pem b/distributed/tests/tls-ca-key.pem
index a5807f29c14..d60ef29ed31 100644
--- a/distributed/tests/tls-ca-key.pem
+++ b/distributed/tests/tls-ca-key.pem
@@ -1,28 +1,28 @@
 -----BEGIN PRIVATE KEY-----
-MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDQSVgCbUzbAMK/
-kNeV9ofAjZweO8TtLDMCLb5YglkrfkyRI4tzLJeJBy2sk145ESV1EXoa0NbjgObj
-W5PbPZe+u5Y//x0myP7iqMsoyztfiAXfJ4GxOCnVd2335ZoZYEv4qVMAkMijMg5z
-CjtRqauSHmsvKIIBGG4T/Wkaeb2pQMtwt5J1HhkMtDzBflWABb6N8IDZJYl0GpId
-uiahZTmWUEaV5L4pR4t3DRHroXqHGRS/ggpNytRtQGZKXxLNoxYUGaKNegeANgRW
-5vN9OMZ1l8l8lZr5/3ltx0B6wpIt73pSB95rA5+vF3JjUzGIV3bVReEvxartk9ZG
-fgcAz0nrAgMBAAECggEBAIve1PU8RQJEroNKVxvjpCIQdqhu5r2GkvKBmXOChTMJ
-640D25+KcHCam3dqtzmHZCoLhH4BhGzeoS5Izy/JWt/AxRNQ1Fi5XUEErPlGEScE
-Bz1mIFfMKXuqafSxARzNZ/cibhTRJYtCECX81vzt/cNwHfZbM4Jcz38pC62110GS
-hSQOAhZv5TFskzlNrmNEVF0vo48v5CS17H2V6qTqQ1bWfLIl8KgMVdvExxrqeIcf
-0wq6QcDZ2vuuTnz8fBMl8q87SoB5oBSqQvkRi67Wv4+8XqmJH06IN4i54IJAG3MG
-ROcvjma+v0zJJsskngE0FtB51axo2gg478A/qpehkjECgYEA8LyC4XEJHL1FykRP
-uzu9HnH50clAUBTGFVGmavY7yytWiEO2bUIES77fZjMv2sWh38emSo3efjCp+s8I
-WYhfrgN4hGLHeHd58tU4wzuQC9kj+8vhuwXVLP7ppVNjDQtuwKV3cHb4Nz5kQ5Iw
-2SOsWGZ5GK3rSvP/Rha4esqi768CgYEA3X4gKDldUIfeMmw88uMLfgxrwuGSuL8Z
-Me/iICTqQjz9DNVS+xfW3qWnYK9NXW8qqu5PqiOyPTEk5aPUgFZXat+a4jsxgK0p
-5I7J4k7uP+5F0CVAIERY+/r8T9kWFAB8yyOz0lm9Z1N8K2Id8+P8wKloJpN5nVwc
-v2mhKgbwfIUCgYAW2u7aFG83CUks8NtY/UzxLQT4w9CVScOclonkMchve4fVnAgf
-y1q5DkALZeSvMoAxzLSe82UiOeCJbUpvpaDf1niEqRQi/cTYJyIidu8wEMd3yl19
-8gnRIcpw5b/Lp0A2jf8O8FyUchn2yFRW6SodomzwyOJD1XJw5wtQrRhXBQKBgDOq
-MugPEXQybSMH2mOxkEGztgM8SNQtRzw9v1pOIhKXVVOEAxos4mMG8vb7jSlJQAOl
-GRIPx4i8iOXEBFJ3UWbNh7xToGVqfwYgRHk0fiCNfN1jgwTDzJniXq5lbd/2iSV9
-S3m6kxKiB6m69k+ok5GBMpkk9048aTxA8GX0Pm5dAoGBAN+yHAq72nXT9ZrjI5gA
-Fhqn7lnZJddLr31/2y6vO4h4a3Uf/pUFqsu5ku/BVwglwJxHx3AeUqlWrHliLqkC
-VQC5hUkaG7jtKumLgZKXuLoAFWsqu2pImNLTYOiEcf3gUXkopa3kbIxfHgspLYXV
-XrYnjJSpTav4WIESxs1SqICB
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCY9pRw5vdlqf2F
+nKLirgg7xJNKYpXDGfll3fHfKO4L5KGofWiXtUBFlMeneWIP3f+H6hJ27pHxPw4O
+T722jyZAYwj7R3GKC65x2YYQ8PwC4Te7h2g/y2dEbmAEZDgmQJbcAJTgKuWNyKKb
+0RLQ+kPKKedCDDM6+c8KAVQ6vwKytElGbDzZtSwKXCQLWg9mbjFL9TNKZ2HlcQIC
+yxYeAK29pYpga60JJRSL0Ktg69VrcK+atmY2RAhqbcSScs6bg6NUBD4IE40KC056
+TNqOuqA0djvOVpzZbhms6TpQZpBWkLz/qfIerLKfAXyWUG+/hN8inPaxIZSKeTxs
+3JTC56f3AgMBAAECggEASZYWQ1pZmib1N9nq4nIms9oVoatbsMhGGqNzQJBINKCA
+xLXQI2umx9KTWhYi/s8lpxUMXPrYat6SvH/QHIsxM7hxEbGFBYwpF8w1muDT2+7N
+CQJTPJ11yI3w0ywDdyOLb7s347+/Pz5OO/eQvcgn5JHUFwoXez/CQmXxR3Gsspsa
+Xt2JSUt7NKyS5L+wvAJa7q02iBTKnFPoJgXHOL2FDr/FCJmQLE7oLlJjagGN6UZs
+LLikGagJw1TOLz0FbfmD/mxB4x+doC977i0eAgpvdZ/vUS3FMF85thbe6lw37hVa
+ubRoE4njGEnTgk0XWnLLQjaHWGLej2ixjs2pjeYGgQKBgQDG8yAa/h/g6lKEfCMb
+RIbN52JIIGbjXopCV8XFr5gotUhJ2zdB8Nv3cH0YciVrPkL0alJJ9Hs3BR/timJn
+u1e+vlonlZjKvheNPNMR/hEh38UIWpfFgZ5J3f/Qp3a3m5Rs7v9kev3x5tfdATtO
+8HYhZff1FIiEQCf8PTpf9enxbwKBgQDE05coXPDpr8qurxaPWjKNrBd0s9DsTA+z
+cLSHVTshvfKwopPFRidMjHZxP6wsHlU/F+19ews+Cxa+P+brsNNWyBHjZSah0bl0
+OAK/Behs/IyGQ8PbMM0RDErfzwq4fhOw7OGfkXp4/OnNu+ww7BOREeuJxdQN3kxO
+WdzcTIHd+QKBgQDBYE4DjqDDyF8eoV6qQWhaOR6BhuUa+aifpZr5dJ0MbHBwQWCr
+eocEMtWU7lZKsED6O2Hyum0NZLDM/eZ6d1FpeLI+qZzS7AQOZ2GtqmtSpqNoZE4D
+TuLhX6QoVq4A93tEiuW48Adnmn6VNX5FU1svSWLDWAr9qTJ0bbsQzQFwhwKBgCXJ
+kY2YOlBDBaB/0M74vBfi4CXxl7yl5aXHj4Ej5dRWiU9MBx6ql0Tk+qBHXyChOjCA
+wYGDq+bfbiBcWFMLgad5haWod8A9aWNclXpjAfpaiggE3ocoWVZai81oDGDbI6Cl
++bMO1PqBY8mxuGi0ChWh4Rx0NH/tG603J2XL8mrRAoGARGkODhSZHcu+5zNomQMX
+kQK+KfwGLMQQDOOuNC99sQPsKtS3FGg3LMKMF27wN7VI2jjCehdycd9NG1JWYVlF
+e4mMnb+M/Z4i34bxAMUL2ueBFmAm5cTWr4qT/VH/z7/Va/R7H4GfE47ewQBou56p
+ezqOR6ZwulVgcYQqTJhdv5U=
 -----END PRIVATE KEY-----
diff --git a/distributed/tests/tls-cert-chain.pem b/distributed/tests/tls-cert-chain.pem
index d967b0c25ed..7a1ab123156 100644
--- a/distributed/tests/tls-cert-chain.pem
+++ b/distributed/tests/tls-cert-chain.pem
@@ -2,146 +2,146 @@ Certificate:
     Data:
         Version: 1 (0x0)
         Serial Number:
-            3f:d8:8a:ed:26:a6:7e:6f:53:9f:fe:85:a4:a3:62:44:65:9f:ff:47
+            3d:ba:1c:2e:06:59:1d:b8:7d:f0:67:cd:e2:68:3d:e4:70:ac:81:86
         Signature Algorithm: sha256WithRSAEncryption
         Issuer: C=XY, O=Dask CA, CN=our-ca-server
         Validity
-            Not Before: Dec 13 02:52:10 2018 GMT
-            Not After : Oct 21 02:52:10 2028 GMT
+            Not Before: Jan  6 07:46:08 2021 GMT
+            Not After : Aug 31 07:46:08 3006 GMT
         Subject: C=XY, L=Dask-distributed, O=Dask, CN=localhost
         Subject Public Key Info:
             Public Key Algorithm: rsaEncryption
                 RSA Public-Key: (2048 bit)
                 Modulus:
-                    00:e5:73:ee:b0:34:96:b9:f2:12:3e:15:8a:e9:76:
-                    c7:11:29:e0:23:78:84:03:ec:eb:a8:1e:eb:b6:02:
-                    bf:34:95:43:29:1e:fb:27:41:fc:7d:55:f8:74:ea:
-                    dc:e2:da:88:d1:8d:4e:ee:78:bf:ca:4c:da:5a:01:
-                    24:e7:4e:95:c6:b2:8d:55:e0:b1:6d:70:a4:c7:0e:
-                    a5:35:77:91:75:06:42:74:6a:a4:2c:7f:8f:d4:0a:
-                    f0:9e:af:b1:97:1d:0e:37:4e:fa:e8:ba:c1:a9:59:
-                    38:f5:5d:67:d1:1f:61:37:e5:ca:72:89:6f:3b:61:
-                    49:45:7f:fa:e7:b4:17:76:8e:4a:77:ef:6a:2e:6b:
-                    b4:ba:24:27:7c:45:d7:af:d4:20:53:e2:ec:da:4f:
-                    bd:c2:e0:fc:a2:61:8a:d8:96:f0:68:67:41:21:7e:
-                    79:25:17:7b:96:bb:f8:59:c0:9c:e5:e5:03:58:26:
-                    2d:e4:b3:61:27:94:38:b8:55:ff:65:fb:22:10:e8:
-                    63:56:2b:1e:26:63:cc:7d:43:16:1e:1b:49:56:38:
-                    1b:79:f7:03:e8:b2:f5:02:fa:ed:15:06:d3:fc:af:
-                    84:d8:34:a6:df:27:15:08:59:83:c7:83:34:65:ca:
-                    c6:d5:8c:86:9a:f0:30:83:20:24:41:10:18:6c:20:
-                    06:5d
+                    00:ab:2b:90:e9:59:22:80:c4:78:58:c5:fc:27:dd:
+                    5a:1f:8e:af:db:f4:b6:e6:07:1f:53:90:4a:c8:d6:
+                    69:43:a5:53:0e:18:7e:8b:23:78:ce:9f:63:d6:cb:
+                    3b:6d:8a:f0:58:2c:46:b4:02:78:49:37:a0:ab:1d:
+                    a7:15:7e:e8:06:fa:e1:de:07:d9:7c:86:b5:cf:36:
+                    a1:22:e2:8d:6a:c7:85:4b:cc:a9:eb:a0:e9:c3:d7:
+                    ce:45:bc:c6:80:46:9d:d0:7d:f6:bd:94:ec:09:d6:
+                    ae:5e:94:d2:28:3e:0b:5a:99:0e:10:24:8e:eb:fd:
+                    50:46:1f:8f:ba:2b:ae:66:4b:57:d6:20:9a:32:98:
+                    8f:4a:b2:bb:c4:5a:86:b2:fd:46:f5:44:8c:22:f9:
+                    69:d1:9d:96:06:47:e7:c9:6b:f0:df:59:2a:f0:72:
+                    ba:31:9a:fd:81:d8:85:27:01:fd:5c:d4:10:a1:5f:
+                    92:95:ec:45:92:23:d7:b6:78:89:99:f9:b6:15:4a:
+                    81:53:d2:b2:7f:73:d8:09:9f:2c:f9:34:62:32:e6:
+                    24:3e:19:3f:bd:c9:7b:07:40:82:7d:b0:26:c1:14:
+                    2d:31:52:4c:74:cd:59:db:fa:a6:eb:65:5c:79:9b:
+                    61:ca:76:2c:72:af:c7:ba:ad:3e:8f:86:38:d8:0a:
+                    45:c5
                 Exponent: 65537 (0x10001)
     Signature Algorithm: sha256WithRSAEncryption
-         6f:78:94:3b:66:82:b5:6f:c5:59:91:fe:54:91:da:5d:04:28:
-         37:63:53:d3:76:16:21:b4:4f:18:57:c6:ee:11:ef:c1:5f:d1:
-         9a:a4:a0:15:60:33:94:52:2f:be:3d:60:9c:9f:51:79:aa:36:
-         28:02:cf:0d:60:a9:e9:54:b8:a5:d4:49:d1:10:e6:f4:a6:a2:
-         1d:27:55:c7:28:63:9e:48:99:ec:d2:88:d7:da:34:03:b6:d9:
-         32:a0:4f:0f:6d:b5:a3:36:ac:54:cc:60:fd:6a:e8:86:c8:57:
-         6e:61:de:66:b3:44:ab:18:95:1c:7f:2d:6e:53:a3:66:17:6e:
-         8c:c8:0c:e1:3c:da:82:d6:c6:28:f8:4c:fa:7f:ca:71:0a:9d:
-         de:7e:19:40:17:e9:40:2f:b3:18:aa:e8:31:bc:0f:5e:61:b5:
-         50:e6:14:3b:e2:7c:dd:70:b3:dc:b9:d0:18:af:87:e4:db:29:
-         1b:f1:a1:6d:2a:44:e5:bf:d7:8d:18:77:dc:f5:25:6a:b3:8a:
-         19:00:3e:15:2b:28:ed:40:c1:45:ad:66:db:74:a0:53:e8:0c:
-         2f:fb:ae:d6:35:9c:3f:c8:81:f9:13:dd:49:06:77:26:82:7d:
-         6d:66:a1:87:a9:9d:ca:4b:2f:c5:19:91:a6:3c:4a:f2:b7:bf:
-         af:8c:97:90
+         59:86:0c:cb:0a:2f:ec:58:91:55:1b:ee:81:15:c3:a0:d7:4c:
+         74:b3:3f:4e:14:06:16:2c:bf:37:3d:ea:12:92:c2:80:1d:af:
+         65:ce:00:b0:fe:b4:d7:83:cb:ef:d7:da:78:5b:db:88:6f:ab:
+         02:de:c1:fe:3d:d7:b1:2f:c7:9d:0f:45:08:18:1e:e8:12:83:
+         78:e7:3f:67:de:89:fe:70:90:81:fa:66:64:33:c9:e7:41:de:
+         44:9a:aa:cf:fc:c8:49:93:7d:f4:a9:90:1f:59:ef:ce:72:48:
+         39:d1:54:46:eb:fe:5b:06:67:d9:44:fd:a2:0a:3a:7f:6b:a9:
+         ab:6b:1f:ee:6e:ec:1a:cc:63:96:d9:e8:d2:11:d7:5a:68:d6:
+         42:69:d6:cb:c9:d9:0a:35:81:16:34:70:76:a1:69:8a:88:6c:
+         57:c7:70:f5:23:1c:3f:6b:12:04:ff:44:5e:a5:f5:94:f8:e9:
+         f4:20:de:d8:32:33:2f:e3:b4:69:39:5c:e9:9a:5e:44:7d:ae:
+         94:00:d8:c2:88:15:a4:ed:3f:5c:d6:4f:bc:19:a0:54:ab:97:
+         15:94:01:93:9e:f8:a5:dc:06:4d:02:75:f4:3a:eb:93:16:24:
+         03:a1:12:85:e2:96:3b:b6:d7:b8:0b:99:00:f9:f0:f5:bc:7c:
+         a0:59:0f:06
 -----BEGIN CERTIFICATE-----
-MIIDCTCCAfECFD/Yiu0mpn5vU5/+haSjYkRln/9HMA0GCSqGSIb3DQEBCwUAMDcx
+MIIDCzCCAfMCFD26HC4GWR24ffBnzeJoPeRwrIGGMA0GCSqGSIb3DQEBCwUAMDcx
 CzAJBgNVBAYTAlhZMRAwDgYDVQQKDAdEYXNrIENBMRYwFAYDVQQDDA1vdXItY2Et
-c2VydmVyMB4XDTE4MTIxMzAyNTIxMFoXDTI4MTAyMTAyNTIxMFowSzELMAkGA1UE
-BhMCWFkxGTAXBgNVBAcMEERhc2stZGlzdHJpYnV0ZWQxDTALBgNVBAoMBERhc2sx
-EjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC
-ggEBAOVz7rA0lrnyEj4Viul2xxEp4CN4hAPs66ge67YCvzSVQyke+ydB/H1V+HTq
-3OLaiNGNTu54v8pM2loBJOdOlcayjVXgsW1wpMcOpTV3kXUGQnRqpCx/j9QK8J6v
-sZcdDjdO+ui6walZOPVdZ9EfYTflynKJbzthSUV/+ue0F3aOSnfvai5rtLokJ3xF
-16/UIFPi7NpPvcLg/KJhitiW8GhnQSF+eSUXe5a7+FnAnOXlA1gmLeSzYSeUOLhV
-/2X7IhDoY1YrHiZjzH1DFh4bSVY4G3n3A+iy9QL67RUG0/yvhNg0pt8nFQhZg8eD
-NGXKxtWMhprwMIMgJEEQGGwgBl0CAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAb3iU
-O2aCtW/FWZH+VJHaXQQoN2NT03YWIbRPGFfG7hHvwV/RmqSgFWAzlFIvvj1gnJ9R
-eao2KALPDWCp6VS4pdRJ0RDm9KaiHSdVxyhjnkiZ7NKI19o0A7bZMqBPD221ozas
-VMxg/WrohshXbmHeZrNEqxiVHH8tblOjZhdujMgM4TzagtbGKPhM+n/KcQqd3n4Z
-QBfpQC+zGKroMbwPXmG1UOYUO+J83XCz3LnQGK+H5NspG/GhbSpE5b/XjRh33PUl
-arOKGQA+FSso7UDBRa1m23SgU+gML/uu1jWcP8iB+RPdSQZ3JoJ9bWahh6mdyksv
-xRmRpjxK8re/r4yXkA==
+c2VydmVyMCAXDTIxMDEwNjA3NDYwOFoYDzMwMDYwODMxMDc0NjA4WjBLMQswCQYD
+VQQGEwJYWTEZMBcGA1UEBwwQRGFzay1kaXN0cmlidXRlZDENMAsGA1UECgwERGFz
+azESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB
+CgKCAQEAqyuQ6VkigMR4WMX8J91aH46v2/S25gcfU5BKyNZpQ6VTDhh+iyN4zp9j
+1ss7bYrwWCxGtAJ4STegqx2nFX7oBvrh3gfZfIa1zzahIuKNaseFS8yp66Dpw9fO
+RbzGgEad0H32vZTsCdauXpTSKD4LWpkOECSO6/1QRh+PuiuuZktX1iCaMpiPSrK7
+xFqGsv1G9USMIvlp0Z2WBkfnyWvw31kq8HK6MZr9gdiFJwH9XNQQoV+SlexFkiPX
+tniJmfm2FUqBU9Kyf3PYCZ8s+TRiMuYkPhk/vcl7B0CCfbAmwRQtMVJMdM1Z2/qm
+62VceZthynYscq/Huq0+j4Y42ApFxQIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQBZ
+hgzLCi/sWJFVG+6BFcOg10x0sz9OFAYWLL83PeoSksKAHa9lzgCw/rTXg8vv19p4
+W9uIb6sC3sH+PdexL8edD0UIGB7oEoN45z9n3on+cJCB+mZkM8nnQd5EmqrP/MhJ
+k330qZAfWe/Ockg50VRG6/5bBmfZRP2iCjp/a6mrax/ubuwazGOW2ejSEddaaNZC
+adbLydkKNYEWNHB2oWmKiGxXx3D1Ixw/axIE/0RepfWU+On0IN7YMjMv47RpOVzp
+ml5Efa6UANjCiBWk7T9c1k+8GaBUq5cVlAGTnvil3AZNAnX0OuuTFiQDoRKF4pY7
+tte4C5kA+fD1vHygWQ8G
 -----END CERTIFICATE-----
 Certificate:
     Data:
         Version: 3 (0x2)
         Serial Number:
-            3f:d8:8a:ed:26:a6:7e:6f:53:9f:fe:85:a4:a3:62:44:65:9f:ff:46
+            3d:ba:1c:2e:06:59:1d:b8:7d:f0:67:cd:e2:68:3d:e4:70:ac:81:85
         Signature Algorithm: sha256WithRSAEncryption
         Issuer: C=XY, O=Dask CA, CN=our-ca-server
         Validity
-            Not Before: Dec 13 02:52:10 2018 GMT
-            Not After : Dec 10 02:52:10 2028 GMT
+            Not Before: Jan  6 07:46:08 2021 GMT
+            Not After : Jan  6 07:46:08 3021 GMT
         Subject: C=XY, O=Dask CA, CN=our-ca-server
         Subject Public Key Info:
             Public Key Algorithm: rsaEncryption
                 RSA Public-Key: (2048 bit)
                 Modulus:
-                    00:d0:49:58:02:6d:4c:db:00:c2:bf:90:d7:95:f6:
-                    87:c0:8d:9c:1e:3b:c4:ed:2c:33:02:2d:be:58:82:
-                    59:2b:7e:4c:91:23:8b:73:2c:97:89:07:2d:ac:93:
-                    5e:39:11:25:75:11:7a:1a:d0:d6:e3:80:e6:e3:5b:
-                    93:db:3d:97:be:bb:96:3f:ff:1d:26:c8:fe:e2:a8:
-                    cb:28:cb:3b:5f:88:05:df:27:81:b1:38:29:d5:77:
-                    6d:f7:e5:9a:19:60:4b:f8:a9:53:00:90:c8:a3:32:
-                    0e:73:0a:3b:51:a9:ab:92:1e:6b:2f:28:82:01:18:
-                    6e:13:fd:69:1a:79:bd:a9:40:cb:70:b7:92:75:1e:
-                    19:0c:b4:3c:c1:7e:55:80:05:be:8d:f0:80:d9:25:
-                    89:74:1a:92:1d:ba:26:a1:65:39:96:50:46:95:e4:
-                    be:29:47:8b:77:0d:11:eb:a1:7a:87:19:14:bf:82:
-                    0a:4d:ca:d4:6d:40:66:4a:5f:12:cd:a3:16:14:19:
-                    a2:8d:7a:07:80:36:04:56:e6:f3:7d:38:c6:75:97:
-                    c9:7c:95:9a:f9:ff:79:6d:c7:40:7a:c2:92:2d:ef:
-                    7a:52:07:de:6b:03:9f:af:17:72:63:53:31:88:57:
-                    76:d5:45:e1:2f:c5:aa:ed:93:d6:46:7e:07:00:cf:
-                    49:eb
+                    00:98:f6:94:70:e6:f7:65:a9:fd:85:9c:a2:e2:ae:
+                    08:3b:c4:93:4a:62:95:c3:19:f9:65:dd:f1:df:28:
+                    ee:0b:e4:a1:a8:7d:68:97:b5:40:45:94:c7:a7:79:
+                    62:0f:dd:ff:87:ea:12:76:ee:91:f1:3f:0e:0e:4f:
+                    bd:b6:8f:26:40:63:08:fb:47:71:8a:0b:ae:71:d9:
+                    86:10:f0:fc:02:e1:37:bb:87:68:3f:cb:67:44:6e:
+                    60:04:64:38:26:40:96:dc:00:94:e0:2a:e5:8d:c8:
+                    a2:9b:d1:12:d0:fa:43:ca:29:e7:42:0c:33:3a:f9:
+                    cf:0a:01:54:3a:bf:02:b2:b4:49:46:6c:3c:d9:b5:
+                    2c:0a:5c:24:0b:5a:0f:66:6e:31:4b:f5:33:4a:67:
+                    61:e5:71:02:02:cb:16:1e:00:ad:bd:a5:8a:60:6b:
+                    ad:09:25:14:8b:d0:ab:60:eb:d5:6b:70:af:9a:b6:
+                    66:36:44:08:6a:6d:c4:92:72:ce:9b:83:a3:54:04:
+                    3e:08:13:8d:0a:0b:4e:7a:4c:da:8e:ba:a0:34:76:
+                    3b:ce:56:9c:d9:6e:19:ac:e9:3a:50:66:90:56:90:
+                    bc:ff:a9:f2:1e:ac:b2:9f:01:7c:96:50:6f:bf:84:
+                    df:22:9c:f6:b1:21:94:8a:79:3c:6c:dc:94:c2:e7:
+                    a7:f7
                 Exponent: 65537 (0x10001)
         X509v3 extensions:
             X509v3 Subject Key Identifier: 
-                59:66:3D:F8:65:B7:98:7E:C1:E4:31:19:34:E5:A3:0D:2E:3D:F9:38
+                EB:0C:96:1D:60:5D:3A:39:00:EA:D8:C3:73:B0:4E:C9:D6:B9:5D:5B
             X509v3 Authority Key Identifier: 
-                keyid:59:66:3D:F8:65:B7:98:7E:C1:E4:31:19:34:E5:A3:0D:2E:3D:F9:38
+                keyid:EB:0C:96:1D:60:5D:3A:39:00:EA:D8:C3:73:B0:4E:C9:D6:B9:5D:5B
 
             X509v3 Basic Constraints: 
                 CA:TRUE
     Signature Algorithm: sha256WithRSAEncryption
-         a2:bd:c7:e4:23:24:4e:da:db:d3:bd:7f:33:87:5f:1f:3b:e8:
-         fa:19:54:17:eb:60:86:ac:d6:18:47:35:90:88:47:d6:f1:5a:
-         31:ed:e1:44:b4:16:6a:04:c3:6f:9b:ba:cf:99:14:0a:c8:32:
-         0a:69:fd:41:c3:67:25:3d:f7:84:31:81:8b:2e:b0:9c:04:74:
-         2c:0b:87:b2:a8:48:a2:c7:42:bb:90:31:82:a5:51:38:03:df:
-         e7:aa:76:f7:1a:ae:b9:69:5c:94:62:7f:19:3e:5c:6b:97:99:
-         9e:4a:9e:4b:34:f5:39:15:fe:12:72:88:7b:1f:be:db:3a:3b:
-         7f:14:06:7b:ef:b2:fc:c4:ad:82:51:4a:95:6c:23:a5:bb:65:
-         38:0e:55:70:e1:96:88:bc:29:82:38:48:63:ac:5a:73:20:3c:
-         d7:39:25:e9:35:46:9b:6b:b1:a0:77:56:a0:a6:30:0f:7e:0e:
-         12:10:11:41:28:6a:f4:ca:b5:c3:a9:fd:fe:9f:cf:29:5e:26:
-         d7:a8:4c:28:a5:31:10:c2:07:25:c2:0a:5d:35:73:f7:db:3c:
-         6b:5d:2f:1a:f1:b0:e7:81:26:22:fe:38:63:2f:3e:5b:a7:d5:
-         b5:a8:ef:fe:87:b4:6f:2d:5e:7e:48:6c:c0:b9:ca:7f:ab:28:
-         c3:31:ae:93
+         08:9b:4d:5a:f0:f0:3d:ef:2e:d2:20:fc:9a:91:41:ba:f6:12:
+         5b:49:7e:2c:84:4d:f7:cf:2b:54:9c:d3:31:d5:54:da:b5:80:
+         0f:04:e5:ca:79:81:19:d4:89:c6:55:fa:b0:0d:9d:3a:f7:34:
+         39:00:3b:db:18:66:45:25:9a:1c:11:02:c0:e3:46:b9:72:37:
+         a8:49:dc:0a:7f:fd:1b:fb:78:6c:e3:ae:2f:ac:99:dc:bd:3e:
+         14:12:f2:c3:f9:75:97:7d:d3:74:61:2b:fa:07:78:e5:99:ed:
+         58:31:87:ee:07:d0:d1:31:b3:b6:e0:90:1b:93:ac:e2:f2:71:
+         ce:c8:81:56:be:9c:2e:22:a7:8e:6c:29:dc:e8:69:4e:26:af:
+         41:4d:57:b2:f7:62:a5:d5:33:0e:0d:3d:fb:bb:35:a9:bf:e0:
+         0a:4e:75:85:d5:39:aa:02:29:d7:a7:20:24:97:2a:48:0f:33:
+         57:24:8d:11:02:88:c7:19:bb:22:95:e7:7b:09:51:ef:79:71:
+         40:8e:36:f6:40:4a:df:3c:a4:ca:f3:ac:60:ee:a8:8f:58:38:
+         88:07:06:78:5b:6c:cd:38:e4:1f:aa:69:d4:a1:ba:dc:59:21:
+         3b:e5:5c:50:ac:0b:5c:3f:c3:c9:5a:6a:fb:c3:46:5f:97:93:
+         7f:f1:a7:0f
 -----BEGIN CERTIFICATE-----
-MIIDTDCCAjSgAwIBAgIUP9iK7Samfm9Tn/6FpKNiRGWf/0YwDQYJKoZIhvcNAQEL
+MIIDTjCCAjagAwIBAgIUPbocLgZZHbh98GfN4mg95HCsgYUwDQYJKoZIhvcNAQEL
 BQAwNzELMAkGA1UEBhMCWFkxEDAOBgNVBAoMB0Rhc2sgQ0ExFjAUBgNVBAMMDW91
-ci1jYS1zZXJ2ZXIwHhcNMTgxMjEzMDI1MjEwWhcNMjgxMjEwMDI1MjEwWjA3MQsw
-CQYDVQQGEwJYWTEQMA4GA1UECgwHRGFzayBDQTEWMBQGA1UEAwwNb3VyLWNhLXNl
-cnZlcjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANBJWAJtTNsAwr+Q
-15X2h8CNnB47xO0sMwItvliCWSt+TJEji3Msl4kHLayTXjkRJXURehrQ1uOA5uNb
-k9s9l767lj//HSbI/uKoyyjLO1+IBd8ngbE4KdV3bfflmhlgS/ipUwCQyKMyDnMK
-O1Gpq5Ieay8oggEYbhP9aRp5valAy3C3knUeGQy0PMF+VYAFvo3wgNkliXQakh26
-JqFlOZZQRpXkvilHi3cNEeuheocZFL+CCk3K1G1AZkpfEs2jFhQZoo16B4A2BFbm
-8304xnWXyXyVmvn/eW3HQHrCki3velIH3msDn68XcmNTMYhXdtVF4S/Fqu2T1kZ+
-BwDPSesCAwEAAaNQME4wHQYDVR0OBBYEFFlmPfhlt5h+weQxGTTlow0uPfk4MB8G
-A1UdIwQYMBaAFFlmPfhlt5h+weQxGTTlow0uPfk4MAwGA1UdEwQFMAMBAf8wDQYJ
-KoZIhvcNAQELBQADggEBAKK9x+QjJE7a29O9fzOHXx876PoZVBfrYIas1hhHNZCI
-R9bxWjHt4US0FmoEw2+bus+ZFArIMgpp/UHDZyU994QxgYsusJwEdCwLh7KoSKLH
-QruQMYKlUTgD3+eqdvcarrlpXJRifxk+XGuXmZ5Knks09TkV/hJyiHsfvts6O38U
-BnvvsvzErYJRSpVsI6W7ZTgOVXDhloi8KYI4SGOsWnMgPNc5Jek1RptrsaB3VqCm
-MA9+DhIQEUEoavTKtcOp/f6fzyleJteoTCilMRDCByXCCl01c/fbPGtdLxrxsOeB
-JiL+OGMvPlun1bWo7/6HtG8tXn5IbMC5yn+rKMMxrpM=
+ci1jYS1zZXJ2ZXIwIBcNMjEwMTA2MDc0NjA4WhgPMzAyMTAxMDYwNzQ2MDhaMDcx
+CzAJBgNVBAYTAlhZMRAwDgYDVQQKDAdEYXNrIENBMRYwFAYDVQQDDA1vdXItY2Et
+c2VydmVyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAmPaUcOb3Zan9
+hZyi4q4IO8STSmKVwxn5Zd3x3yjuC+ShqH1ol7VARZTHp3liD93/h+oSdu6R8T8O
+Dk+9to8mQGMI+0dxiguucdmGEPD8AuE3u4doP8tnRG5gBGQ4JkCW3ACU4Crljcii
+m9ES0PpDyinnQgwzOvnPCgFUOr8CsrRJRmw82bUsClwkC1oPZm4xS/UzSmdh5XEC
+AssWHgCtvaWKYGutCSUUi9CrYOvVa3CvmrZmNkQIam3EknLOm4OjVAQ+CBONCgtO
+ekzajrqgNHY7zlac2W4ZrOk6UGaQVpC8/6nyHqyynwF8llBvv4TfIpz2sSGUink8
+bNyUwuen9wIDAQABo1AwTjAdBgNVHQ4EFgQU6wyWHWBdOjkA6tjDc7BOyda5XVsw
+HwYDVR0jBBgwFoAU6wyWHWBdOjkA6tjDc7BOyda5XVswDAYDVR0TBAUwAwEB/zAN
+BgkqhkiG9w0BAQsFAAOCAQEACJtNWvDwPe8u0iD8mpFBuvYSW0l+LIRN988rVJzT
+MdVU2rWADwTlynmBGdSJxlX6sA2dOvc0OQA72xhmRSWaHBECwONGuXI3qEncCn/9
+G/t4bOOuL6yZ3L0+FBLyw/l1l33TdGEr+gd45ZntWDGH7gfQ0TGztuCQG5Os4vJx
+zsiBVr6cLiKnjmwp3OhpTiavQU1XsvdipdUzDg09+7s1qb/gCk51hdU5qgIp16cg
+JJcqSA8zVySNEQKIxxm7IpXnewlR73lxQI429kBK3zykyvOsYO6oj1g4iAcGeFts
+zTjkH6pp1KG63FkhO+VcUKwLXD/DyVpq+8NGX5eTf/GnDw==
 -----END CERTIFICATE-----
diff --git a/distributed/tests/tls-cert.pem b/distributed/tests/tls-cert.pem
index eb510c57061..8452e3f1b1b 100644
--- a/distributed/tests/tls-cert.pem
+++ b/distributed/tests/tls-cert.pem
@@ -2,68 +2,68 @@ Certificate:
     Data:
         Version: 1 (0x0)
         Serial Number:
-            3f:d8:8a:ed:26:a6:7e:6f:53:9f:fe:85:a4:a3:62:44:65:9f:ff:47
+            3d:ba:1c:2e:06:59:1d:b8:7d:f0:67:cd:e2:68:3d:e4:70:ac:81:86
         Signature Algorithm: sha256WithRSAEncryption
         Issuer: C=XY, O=Dask CA, CN=our-ca-server
         Validity
-            Not Before: Dec 13 02:52:10 2018 GMT
-            Not After : Oct 21 02:52:10 2028 GMT
+            Not Before: Jan  6 07:46:08 2021 GMT
+            Not After : Aug 31 07:46:08 3006 GMT
         Subject: C=XY, L=Dask-distributed, O=Dask, CN=localhost
         Subject Public Key Info:
             Public Key Algorithm: rsaEncryption
                 RSA Public-Key: (2048 bit)
                 Modulus:
-                    00:e5:73:ee:b0:34:96:b9:f2:12:3e:15:8a:e9:76:
-                    c7:11:29:e0:23:78:84:03:ec:eb:a8:1e:eb:b6:02:
-                    bf:34:95:43:29:1e:fb:27:41:fc:7d:55:f8:74:ea:
-                    dc:e2:da:88:d1:8d:4e:ee:78:bf:ca:4c:da:5a:01:
-                    24:e7:4e:95:c6:b2:8d:55:e0:b1:6d:70:a4:c7:0e:
-                    a5:35:77:91:75:06:42:74:6a:a4:2c:7f:8f:d4:0a:
-                    f0:9e:af:b1:97:1d:0e:37:4e:fa:e8:ba:c1:a9:59:
-                    38:f5:5d:67:d1:1f:61:37:e5:ca:72:89:6f:3b:61:
-                    49:45:7f:fa:e7:b4:17:76:8e:4a:77:ef:6a:2e:6b:
-                    b4:ba:24:27:7c:45:d7:af:d4:20:53:e2:ec:da:4f:
-                    bd:c2:e0:fc:a2:61:8a:d8:96:f0:68:67:41:21:7e:
-                    79:25:17:7b:96:bb:f8:59:c0:9c:e5:e5:03:58:26:
-                    2d:e4:b3:61:27:94:38:b8:55:ff:65:fb:22:10:e8:
-                    63:56:2b:1e:26:63:cc:7d:43:16:1e:1b:49:56:38:
-                    1b:79:f7:03:e8:b2:f5:02:fa:ed:15:06:d3:fc:af:
-                    84:d8:34:a6:df:27:15:08:59:83:c7:83:34:65:ca:
-                    c6:d5:8c:86:9a:f0:30:83:20:24:41:10:18:6c:20:
-                    06:5d
+                    00:ab:2b:90:e9:59:22:80:c4:78:58:c5:fc:27:dd:
+                    5a:1f:8e:af:db:f4:b6:e6:07:1f:53:90:4a:c8:d6:
+                    69:43:a5:53:0e:18:7e:8b:23:78:ce:9f:63:d6:cb:
+                    3b:6d:8a:f0:58:2c:46:b4:02:78:49:37:a0:ab:1d:
+                    a7:15:7e:e8:06:fa:e1:de:07:d9:7c:86:b5:cf:36:
+                    a1:22:e2:8d:6a:c7:85:4b:cc:a9:eb:a0:e9:c3:d7:
+                    ce:45:bc:c6:80:46:9d:d0:7d:f6:bd:94:ec:09:d6:
+                    ae:5e:94:d2:28:3e:0b:5a:99:0e:10:24:8e:eb:fd:
+                    50:46:1f:8f:ba:2b:ae:66:4b:57:d6:20:9a:32:98:
+                    8f:4a:b2:bb:c4:5a:86:b2:fd:46:f5:44:8c:22:f9:
+                    69:d1:9d:96:06:47:e7:c9:6b:f0:df:59:2a:f0:72:
+                    ba:31:9a:fd:81:d8:85:27:01:fd:5c:d4:10:a1:5f:
+                    92:95:ec:45:92:23:d7:b6:78:89:99:f9:b6:15:4a:
+                    81:53:d2:b2:7f:73:d8:09:9f:2c:f9:34:62:32:e6:
+                    24:3e:19:3f:bd:c9:7b:07:40:82:7d:b0:26:c1:14:
+                    2d:31:52:4c:74:cd:59:db:fa:a6:eb:65:5c:79:9b:
+                    61:ca:76:2c:72:af:c7:ba:ad:3e:8f:86:38:d8:0a:
+                    45:c5
                 Exponent: 65537 (0x10001)
     Signature Algorithm: sha256WithRSAEncryption
-         6f:78:94:3b:66:82:b5:6f:c5:59:91:fe:54:91:da:5d:04:28:
-         37:63:53:d3:76:16:21:b4:4f:18:57:c6:ee:11:ef:c1:5f:d1:
-         9a:a4:a0:15:60:33:94:52:2f:be:3d:60:9c:9f:51:79:aa:36:
-         28:02:cf:0d:60:a9:e9:54:b8:a5:d4:49:d1:10:e6:f4:a6:a2:
-         1d:27:55:c7:28:63:9e:48:99:ec:d2:88:d7:da:34:03:b6:d9:
-         32:a0:4f:0f:6d:b5:a3:36:ac:54:cc:60:fd:6a:e8:86:c8:57:
-         6e:61:de:66:b3:44:ab:18:95:1c:7f:2d:6e:53:a3:66:17:6e:
-         8c:c8:0c:e1:3c:da:82:d6:c6:28:f8:4c:fa:7f:ca:71:0a:9d:
-         de:7e:19:40:17:e9:40:2f:b3:18:aa:e8:31:bc:0f:5e:61:b5:
-         50:e6:14:3b:e2:7c:dd:70:b3:dc:b9:d0:18:af:87:e4:db:29:
-         1b:f1:a1:6d:2a:44:e5:bf:d7:8d:18:77:dc:f5:25:6a:b3:8a:
-         19:00:3e:15:2b:28:ed:40:c1:45:ad:66:db:74:a0:53:e8:0c:
-         2f:fb:ae:d6:35:9c:3f:c8:81:f9:13:dd:49:06:77:26:82:7d:
-         6d:66:a1:87:a9:9d:ca:4b:2f:c5:19:91:a6:3c:4a:f2:b7:bf:
-         af:8c:97:90
+         59:86:0c:cb:0a:2f:ec:58:91:55:1b:ee:81:15:c3:a0:d7:4c:
+         74:b3:3f:4e:14:06:16:2c:bf:37:3d:ea:12:92:c2:80:1d:af:
+         65:ce:00:b0:fe:b4:d7:83:cb:ef:d7:da:78:5b:db:88:6f:ab:
+         02:de:c1:fe:3d:d7:b1:2f:c7:9d:0f:45:08:18:1e:e8:12:83:
+         78:e7:3f:67:de:89:fe:70:90:81:fa:66:64:33:c9:e7:41:de:
+         44:9a:aa:cf:fc:c8:49:93:7d:f4:a9:90:1f:59:ef:ce:72:48:
+         39:d1:54:46:eb:fe:5b:06:67:d9:44:fd:a2:0a:3a:7f:6b:a9:
+         ab:6b:1f:ee:6e:ec:1a:cc:63:96:d9:e8:d2:11:d7:5a:68:d6:
+         42:69:d6:cb:c9:d9:0a:35:81:16:34:70:76:a1:69:8a:88:6c:
+         57:c7:70:f5:23:1c:3f:6b:12:04:ff:44:5e:a5:f5:94:f8:e9:
+         f4:20:de:d8:32:33:2f:e3:b4:69:39:5c:e9:9a:5e:44:7d:ae:
+         94:00:d8:c2:88:15:a4:ed:3f:5c:d6:4f:bc:19:a0:54:ab:97:
+         15:94:01:93:9e:f8:a5:dc:06:4d:02:75:f4:3a:eb:93:16:24:
+         03:a1:12:85:e2:96:3b:b6:d7:b8:0b:99:00:f9:f0:f5:bc:7c:
+         a0:59:0f:06
 -----BEGIN CERTIFICATE-----
-MIIDCTCCAfECFD/Yiu0mpn5vU5/+haSjYkRln/9HMA0GCSqGSIb3DQEBCwUAMDcx
+MIIDCzCCAfMCFD26HC4GWR24ffBnzeJoPeRwrIGGMA0GCSqGSIb3DQEBCwUAMDcx
 CzAJBgNVBAYTAlhZMRAwDgYDVQQKDAdEYXNrIENBMRYwFAYDVQQDDA1vdXItY2Et
-c2VydmVyMB4XDTE4MTIxMzAyNTIxMFoXDTI4MTAyMTAyNTIxMFowSzELMAkGA1UE
-BhMCWFkxGTAXBgNVBAcMEERhc2stZGlzdHJpYnV0ZWQxDTALBgNVBAoMBERhc2sx
-EjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC
-ggEBAOVz7rA0lrnyEj4Viul2xxEp4CN4hAPs66ge67YCvzSVQyke+ydB/H1V+HTq
-3OLaiNGNTu54v8pM2loBJOdOlcayjVXgsW1wpMcOpTV3kXUGQnRqpCx/j9QK8J6v
-sZcdDjdO+ui6walZOPVdZ9EfYTflynKJbzthSUV/+ue0F3aOSnfvai5rtLokJ3xF
-16/UIFPi7NpPvcLg/KJhitiW8GhnQSF+eSUXe5a7+FnAnOXlA1gmLeSzYSeUOLhV
-/2X7IhDoY1YrHiZjzH1DFh4bSVY4G3n3A+iy9QL67RUG0/yvhNg0pt8nFQhZg8eD
-NGXKxtWMhprwMIMgJEEQGGwgBl0CAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAb3iU
-O2aCtW/FWZH+VJHaXQQoN2NT03YWIbRPGFfG7hHvwV/RmqSgFWAzlFIvvj1gnJ9R
-eao2KALPDWCp6VS4pdRJ0RDm9KaiHSdVxyhjnkiZ7NKI19o0A7bZMqBPD221ozas
-VMxg/WrohshXbmHeZrNEqxiVHH8tblOjZhdujMgM4TzagtbGKPhM+n/KcQqd3n4Z
-QBfpQC+zGKroMbwPXmG1UOYUO+J83XCz3LnQGK+H5NspG/GhbSpE5b/XjRh33PUl
-arOKGQA+FSso7UDBRa1m23SgU+gML/uu1jWcP8iB+RPdSQZ3JoJ9bWahh6mdyksv
-xRmRpjxK8re/r4yXkA==
+c2VydmVyMCAXDTIxMDEwNjA3NDYwOFoYDzMwMDYwODMxMDc0NjA4WjBLMQswCQYD
+VQQGEwJYWTEZMBcGA1UEBwwQRGFzay1kaXN0cmlidXRlZDENMAsGA1UECgwERGFz
+azESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB
+CgKCAQEAqyuQ6VkigMR4WMX8J91aH46v2/S25gcfU5BKyNZpQ6VTDhh+iyN4zp9j
+1ss7bYrwWCxGtAJ4STegqx2nFX7oBvrh3gfZfIa1zzahIuKNaseFS8yp66Dpw9fO
+RbzGgEad0H32vZTsCdauXpTSKD4LWpkOECSO6/1QRh+PuiuuZktX1iCaMpiPSrK7
+xFqGsv1G9USMIvlp0Z2WBkfnyWvw31kq8HK6MZr9gdiFJwH9XNQQoV+SlexFkiPX
+tniJmfm2FUqBU9Kyf3PYCZ8s+TRiMuYkPhk/vcl7B0CCfbAmwRQtMVJMdM1Z2/qm
+62VceZthynYscq/Huq0+j4Y42ApFxQIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQBZ
+hgzLCi/sWJFVG+6BFcOg10x0sz9OFAYWLL83PeoSksKAHa9lzgCw/rTXg8vv19p4
+W9uIb6sC3sH+PdexL8edD0UIGB7oEoN45z9n3on+cJCB+mZkM8nnQd5EmqrP/MhJ
+k330qZAfWe/Ockg50VRG6/5bBmfZRP2iCjp/a6mrax/ubuwazGOW2ejSEddaaNZC
+adbLydkKNYEWNHB2oWmKiGxXx3D1Ixw/axIE/0RepfWU+On0IN7YMjMv47RpOVzp
+ml5Efa6UANjCiBWk7T9c1k+8GaBUq5cVlAGTnvil3AZNAnX0OuuTFiQDoRKF4pY7
+tte4C5kA+fD1vHygWQ8G
 -----END CERTIFICATE-----
diff --git a/distributed/tests/tls-key-cert.pem b/distributed/tests/tls-key-cert.pem
index 3b52d8d8b40..44e1287c927 100644
--- a/distributed/tests/tls-key-cert.pem
+++ b/distributed/tests/tls-key-cert.pem
@@ -1,97 +1,97 @@
 -----BEGIN PRIVATE KEY-----
-MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDlc+6wNJa58hI+
-FYrpdscRKeAjeIQD7OuoHuu2Ar80lUMpHvsnQfx9Vfh06tzi2ojRjU7ueL/KTNpa
-ASTnTpXGso1V4LFtcKTHDqU1d5F1BkJ0aqQsf4/UCvCer7GXHQ43TvrousGpWTj1
-XWfRH2E35cpyiW87YUlFf/rntBd2jkp372oua7S6JCd8Rdev1CBT4uzaT73C4Pyi
-YYrYlvBoZ0EhfnklF3uWu/hZwJzl5QNYJi3ks2EnlDi4Vf9l+yIQ6GNWKx4mY8x9
-QxYeG0lWOBt59wPosvUC+u0VBtP8r4TYNKbfJxUIWYPHgzRlysbVjIaa8DCDICRB
-EBhsIAZdAgMBAAECggEBAJlC8bjSt6bmUVcUzp+Yzejh/qvSsONqWhIG7Uy60vqN
-hLmvzvRu9/bD+NwnkHkanW7TI1jYYMsAwIyD3SF/dT2uQAePuROvvPCVELcA8bCo
-CLV4EccDQY/R6t0j38xfBQLeh2vyaq7yQMBrro+ZlTdgDvSoZhrUpbt9e5tOyKlP
-p8EwVW74251XSfONrbsbEvTq5GsNSgerAJT9pC05FhpXp92WPYrDFV0906hlt6z2
-vyLw/PVaQR3YLeNcZ7Fp/AEOTht3M4vfaMq1ibEOi5DueBV0RSGcFzKaP1FFnqsD
-a5x3NGpY5q8ngCcwlWWEsBLnVzDIjI7FJWOnX5F87GUCgYEA/dWuKo09GD6g+oeC
-R6LsSxZdqLl5jopUFgKMpdFWs07rZD1HoH+8Dpl0bYSvSBnXza7d5BmE5HwpA3ts
-Y9OinVu9UCgqToO3PZbbyqQ9AhqLW7YlVjZ2G+U8BM6Z1j0+C3aZ8jRXRRXQj9V1
-pUDuDjCK7WEp3Z3fyuqFpMLy+msCgYEA52kB539F7hRkxoob0gMidC3+qiMDjC6t
-uwbiovyWnO38t+ggQR27AZHfVHqW+iXSt2oiTjZ8pNhrBzpWcLZLvOVXzlTeJL+B
-YL2I/rjnRlplBDAfoUyLRCfQQ9FpaBvvoxooySNeRAIZ+9GgeiqTRYz91/CZX5h/
-OIgNV6Y5xFcCgYAo6ny8ly+17MJ+H8G8UmLMZ7qJK/yQIXQUaCzLvca/+ISxD7+v
-+2cQ42OFE9I9SQ0kNn/F+OcW/XyO9yItpTudKeCGPzMynMC4yMx9CaRb5nN4QRAb
-qYO4U2ctTzA0/II3ztJZ3F4E4NNRnBLupFy7JNAAG51n/77HMq6ANHx+KQKBgQDi
-d4CWlpbQVYL+TiDF2a15N54X0SZipTf+ZdGcokefksSVEoWmVDs0YRbH7SUd77AV
-MbgbBPXcubOcojoAGldqqKPS2HmCSyyJiVOtvTURQ9vCSnpZ97f/XltSU3mtmlFe
-NHafyY4yTkAxkkhT5qC0GzQ9hUgEJxCyOWudUJ8OBwKBgGd8n3UchJ+Ll3calcHO
-GKlWdqNPI6yLtY0wTqZGCZIWGjskm6YoYj4kEyBten/ERBArSuUeucj5qCxij97Y
-ue27ZP2xDX7ONdd5D36UPck3mfqBiW+BlUI9vGPhBDlA9sRhT2wIkDuu8W9MkyKd
-7Y9YAmBiswDTAj5TYYeWqWcM
+MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCrK5DpWSKAxHhY
+xfwn3Vofjq/b9LbmBx9TkErI1mlDpVMOGH6LI3jOn2PWyzttivBYLEa0AnhJN6Cr
+HacVfugG+uHeB9l8hrXPNqEi4o1qx4VLzKnroOnD185FvMaARp3Qffa9lOwJ1q5e
+lNIoPgtamQ4QJI7r/VBGH4+6K65mS1fWIJoymI9KsrvEWoay/Ub1RIwi+WnRnZYG
+R+fJa/DfWSrwcroxmv2B2IUnAf1c1BChX5KV7EWSI9e2eImZ+bYVSoFT0rJ/c9gJ
+nyz5NGIy5iQ+GT+9yXsHQIJ9sCbBFC0xUkx0zVnb+qbrZVx5m2HKdixyr8e6rT6P
+hjjYCkXFAgMBAAECggEAEF7uUSMRRbeEuxjUcQuenv4Km8Lf60grGtbmECdRkgKx
+2ttE3xHZcAmWhuiRGnz7V7OEX5sM3hPP7g27+TZWArTj/VRK3LoVkWfsKWX5mIjU
+mtdMORj+wphVRh4z7fjTBynE+4TOvnIl4y3gm7CzbUrOKYUVlkR39Pz3xhp/Z5EN
+ATFquJ4fCCtTMLiLmtwsIeLfqdzbnYy20W5ILdYmDhRG1OSaC9ioFax9VUHZ4/LB
+CpWry0fA4J4ulk2oMKo7MNVTh+uubUT8/xyBULx/u9mj9LLPzVLi3FBDsgg4Uglt
+K/pcAmMXUbwa4pxm5YMX2Rg+cM56/kH7EZnDjMCdgQKBgQDh4W84ClPbrVsk/+yX
+y2HxnCjMqc3r5OtAR7cFnv5BX+4urVF4pal720QmHGLTzo9Jiib172WvXPWwNc8a
+x7Knv3hXM1Z40J1E04VqwZx7fg/6DHRDhJ1j/sRb0VNStChKakOeVgvqJriapdUZ
+L/sCsUEWZqr3qYYj/Lkm3xGzeQKBgQDB/o8+IgaisarFKZDhiF8XWNJ+v7lmTRI3
+wpryVaBVopxJKVc2F61iWhllA8skvpmGqx+E7E2QeYHU+y3S3z+MASyoCKpc7pOA
+QaN2Mxu4nzc211n6a7T8b8UtRxO0wTXC3PfPFx0uKcOTGqBf9mPehN02WeiO2jiH
+uKs0qt+lrQKBgGSohlyN1aLelPBfD/rxCluWAGR8AyTtJZzGmJvrp1hQvJ6WY+kE
+wgtMwgjNu4ix0w6tfwE3xrOoFGCLChjgBxBT9fe+C/nY1Uy3igvx7Z7mpY4DgMgZ
+z92gybY1CRGg8ReAVVO4+qREzx0za0xHIm+nt96Yr/Wf2QQuegAFhijZAoGAE4WN
+XqBH8KlWWP7k0H1LrGrIy+55Nr08vxW7R/Wr69IwTgB2AUCCtyn/8jVehPUVOAMS
+aerpp0dC9G7OXrEOkQUxPDubAT0U5Mw2Kdfcy0hiYwdqy5N5JAoPWo4TgpYK1fAs
+k89spU+Ymb4NM3r089iE9ko35+A33hiMSgGD/gUCgYBNgBAUPrRG+si3Hub0xtmj
+3fbcG423kUEptjr5oouH4KGa3M0o3gY4KX9GfQ+cTd/zO7kju64lGrbQIk8V25fy
+Dc0C6kQCxAA1cBZrHC3mo/p/bg+oDX2bvDA7SSnxI18awkPL5Rq8BRNyJHVIvE9P
+1LrREdObq7nOuU18HkRKfA==
 -----END PRIVATE KEY-----
 Certificate:
     Data:
         Version: 1 (0x0)
         Serial Number:
-            3f:d8:8a:ed:26:a6:7e:6f:53:9f:fe:85:a4:a3:62:44:65:9f:ff:47
+            3d:ba:1c:2e:06:59:1d:b8:7d:f0:67:cd:e2:68:3d:e4:70:ac:81:86
         Signature Algorithm: sha256WithRSAEncryption
         Issuer: C=XY, O=Dask CA, CN=our-ca-server
         Validity
-            Not Before: Dec 13 02:52:10 2018 GMT
-            Not After : Oct 21 02:52:10 2028 GMT
+            Not Before: Jan  6 07:46:08 2021 GMT
+            Not After : Aug 31 07:46:08 3006 GMT
         Subject: C=XY, L=Dask-distributed, O=Dask, CN=localhost
         Subject Public Key Info:
             Public Key Algorithm: rsaEncryption
                 RSA Public-Key: (2048 bit)
                 Modulus:
-                    00:e5:73:ee:b0:34:96:b9:f2:12:3e:15:8a:e9:76:
-                    c7:11:29:e0:23:78:84:03:ec:eb:a8:1e:eb:b6:02:
-                    bf:34:95:43:29:1e:fb:27:41:fc:7d:55:f8:74:ea:
-                    dc:e2:da:88:d1:8d:4e:ee:78:bf:ca:4c:da:5a:01:
-                    24:e7:4e:95:c6:b2:8d:55:e0:b1:6d:70:a4:c7:0e:
-                    a5:35:77:91:75:06:42:74:6a:a4:2c:7f:8f:d4:0a:
-                    f0:9e:af:b1:97:1d:0e:37:4e:fa:e8:ba:c1:a9:59:
-                    38:f5:5d:67:d1:1f:61:37:e5:ca:72:89:6f:3b:61:
-                    49:45:7f:fa:e7:b4:17:76:8e:4a:77:ef:6a:2e:6b:
-                    b4:ba:24:27:7c:45:d7:af:d4:20:53:e2:ec:da:4f:
-                    bd:c2:e0:fc:a2:61:8a:d8:96:f0:68:67:41:21:7e:
-                    79:25:17:7b:96:bb:f8:59:c0:9c:e5:e5:03:58:26:
-                    2d:e4:b3:61:27:94:38:b8:55:ff:65:fb:22:10:e8:
-                    63:56:2b:1e:26:63:cc:7d:43:16:1e:1b:49:56:38:
-                    1b:79:f7:03:e8:b2:f5:02:fa:ed:15:06:d3:fc:af:
-                    84:d8:34:a6:df:27:15:08:59:83:c7:83:34:65:ca:
-                    c6:d5:8c:86:9a:f0:30:83:20:24:41:10:18:6c:20:
-                    06:5d
+                    00:ab:2b:90:e9:59:22:80:c4:78:58:c5:fc:27:dd:
+                    5a:1f:8e:af:db:f4:b6:e6:07:1f:53:90:4a:c8:d6:
+                    69:43:a5:53:0e:18:7e:8b:23:78:ce:9f:63:d6:cb:
+                    3b:6d:8a:f0:58:2c:46:b4:02:78:49:37:a0:ab:1d:
+                    a7:15:7e:e8:06:fa:e1:de:07:d9:7c:86:b5:cf:36:
+                    a1:22:e2:8d:6a:c7:85:4b:cc:a9:eb:a0:e9:c3:d7:
+                    ce:45:bc:c6:80:46:9d:d0:7d:f6:bd:94:ec:09:d6:
+                    ae:5e:94:d2:28:3e:0b:5a:99:0e:10:24:8e:eb:fd:
+                    50:46:1f:8f:ba:2b:ae:66:4b:57:d6:20:9a:32:98:
+                    8f:4a:b2:bb:c4:5a:86:b2:fd:46:f5:44:8c:22:f9:
+                    69:d1:9d:96:06:47:e7:c9:6b:f0:df:59:2a:f0:72:
+                    ba:31:9a:fd:81:d8:85:27:01:fd:5c:d4:10:a1:5f:
+                    92:95:ec:45:92:23:d7:b6:78:89:99:f9:b6:15:4a:
+                    81:53:d2:b2:7f:73:d8:09:9f:2c:f9:34:62:32:e6:
+                    24:3e:19:3f:bd:c9:7b:07:40:82:7d:b0:26:c1:14:
+                    2d:31:52:4c:74:cd:59:db:fa:a6:eb:65:5c:79:9b:
+                    61:ca:76:2c:72:af:c7:ba:ad:3e:8f:86:38:d8:0a:
+                    45:c5
                 Exponent: 65537 (0x10001)
     Signature Algorithm: sha256WithRSAEncryption
-         6f:78:94:3b:66:82:b5:6f:c5:59:91:fe:54:91:da:5d:04:28:
-         37:63:53:d3:76:16:21:b4:4f:18:57:c6:ee:11:ef:c1:5f:d1:
-         9a:a4:a0:15:60:33:94:52:2f:be:3d:60:9c:9f:51:79:aa:36:
-         28:02:cf:0d:60:a9:e9:54:b8:a5:d4:49:d1:10:e6:f4:a6:a2:
-         1d:27:55:c7:28:63:9e:48:99:ec:d2:88:d7:da:34:03:b6:d9:
-         32:a0:4f:0f:6d:b5:a3:36:ac:54:cc:60:fd:6a:e8:86:c8:57:
-         6e:61:de:66:b3:44:ab:18:95:1c:7f:2d:6e:53:a3:66:17:6e:
-         8c:c8:0c:e1:3c:da:82:d6:c6:28:f8:4c:fa:7f:ca:71:0a:9d:
-         de:7e:19:40:17:e9:40:2f:b3:18:aa:e8:31:bc:0f:5e:61:b5:
-         50:e6:14:3b:e2:7c:dd:70:b3:dc:b9:d0:18:af:87:e4:db:29:
-         1b:f1:a1:6d:2a:44:e5:bf:d7:8d:18:77:dc:f5:25:6a:b3:8a:
-         19:00:3e:15:2b:28:ed:40:c1:45:ad:66:db:74:a0:53:e8:0c:
-         2f:fb:ae:d6:35:9c:3f:c8:81:f9:13:dd:49:06:77:26:82:7d:
-         6d:66:a1:87:a9:9d:ca:4b:2f:c5:19:91:a6:3c:4a:f2:b7:bf:
-         af:8c:97:90
+         59:86:0c:cb:0a:2f:ec:58:91:55:1b:ee:81:15:c3:a0:d7:4c:
+         74:b3:3f:4e:14:06:16:2c:bf:37:3d:ea:12:92:c2:80:1d:af:
+         65:ce:00:b0:fe:b4:d7:83:cb:ef:d7:da:78:5b:db:88:6f:ab:
+         02:de:c1:fe:3d:d7:b1:2f:c7:9d:0f:45:08:18:1e:e8:12:83:
+         78:e7:3f:67:de:89:fe:70:90:81:fa:66:64:33:c9:e7:41:de:
+         44:9a:aa:cf:fc:c8:49:93:7d:f4:a9:90:1f:59:ef:ce:72:48:
+         39:d1:54:46:eb:fe:5b:06:67:d9:44:fd:a2:0a:3a:7f:6b:a9:
+         ab:6b:1f:ee:6e:ec:1a:cc:63:96:d9:e8:d2:11:d7:5a:68:d6:
+         42:69:d6:cb:c9:d9:0a:35:81:16:34:70:76:a1:69:8a:88:6c:
+         57:c7:70:f5:23:1c:3f:6b:12:04:ff:44:5e:a5:f5:94:f8:e9:
+         f4:20:de:d8:32:33:2f:e3:b4:69:39:5c:e9:9a:5e:44:7d:ae:
+         94:00:d8:c2:88:15:a4:ed:3f:5c:d6:4f:bc:19:a0:54:ab:97:
+         15:94:01:93:9e:f8:a5:dc:06:4d:02:75:f4:3a:eb:93:16:24:
+         03:a1:12:85:e2:96:3b:b6:d7:b8:0b:99:00:f9:f0:f5:bc:7c:
+         a0:59:0f:06
 -----BEGIN CERTIFICATE-----
-MIIDCTCCAfECFD/Yiu0mpn5vU5/+haSjYkRln/9HMA0GCSqGSIb3DQEBCwUAMDcx
+MIIDCzCCAfMCFD26HC4GWR24ffBnzeJoPeRwrIGGMA0GCSqGSIb3DQEBCwUAMDcx
 CzAJBgNVBAYTAlhZMRAwDgYDVQQKDAdEYXNrIENBMRYwFAYDVQQDDA1vdXItY2Et
-c2VydmVyMB4XDTE4MTIxMzAyNTIxMFoXDTI4MTAyMTAyNTIxMFowSzELMAkGA1UE
-BhMCWFkxGTAXBgNVBAcMEERhc2stZGlzdHJpYnV0ZWQxDTALBgNVBAoMBERhc2sx
-EjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC
-ggEBAOVz7rA0lrnyEj4Viul2xxEp4CN4hAPs66ge67YCvzSVQyke+ydB/H1V+HTq
-3OLaiNGNTu54v8pM2loBJOdOlcayjVXgsW1wpMcOpTV3kXUGQnRqpCx/j9QK8J6v
-sZcdDjdO+ui6walZOPVdZ9EfYTflynKJbzthSUV/+ue0F3aOSnfvai5rtLokJ3xF
-16/UIFPi7NpPvcLg/KJhitiW8GhnQSF+eSUXe5a7+FnAnOXlA1gmLeSzYSeUOLhV
-/2X7IhDoY1YrHiZjzH1DFh4bSVY4G3n3A+iy9QL67RUG0/yvhNg0pt8nFQhZg8eD
-NGXKxtWMhprwMIMgJEEQGGwgBl0CAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAb3iU
-O2aCtW/FWZH+VJHaXQQoN2NT03YWIbRPGFfG7hHvwV/RmqSgFWAzlFIvvj1gnJ9R
-eao2KALPDWCp6VS4pdRJ0RDm9KaiHSdVxyhjnkiZ7NKI19o0A7bZMqBPD221ozas
-VMxg/WrohshXbmHeZrNEqxiVHH8tblOjZhdujMgM4TzagtbGKPhM+n/KcQqd3n4Z
-QBfpQC+zGKroMbwPXmG1UOYUO+J83XCz3LnQGK+H5NspG/GhbSpE5b/XjRh33PUl
-arOKGQA+FSso7UDBRa1m23SgU+gML/uu1jWcP8iB+RPdSQZ3JoJ9bWahh6mdyksv
-xRmRpjxK8re/r4yXkA==
+c2VydmVyMCAXDTIxMDEwNjA3NDYwOFoYDzMwMDYwODMxMDc0NjA4WjBLMQswCQYD
+VQQGEwJYWTEZMBcGA1UEBwwQRGFzay1kaXN0cmlidXRlZDENMAsGA1UECgwERGFz
+azESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB
+CgKCAQEAqyuQ6VkigMR4WMX8J91aH46v2/S25gcfU5BKyNZpQ6VTDhh+iyN4zp9j
+1ss7bYrwWCxGtAJ4STegqx2nFX7oBvrh3gfZfIa1zzahIuKNaseFS8yp66Dpw9fO
+RbzGgEad0H32vZTsCdauXpTSKD4LWpkOECSO6/1QRh+PuiuuZktX1iCaMpiPSrK7
+xFqGsv1G9USMIvlp0Z2WBkfnyWvw31kq8HK6MZr9gdiFJwH9XNQQoV+SlexFkiPX
+tniJmfm2FUqBU9Kyf3PYCZ8s+TRiMuYkPhk/vcl7B0CCfbAmwRQtMVJMdM1Z2/qm
+62VceZthynYscq/Huq0+j4Y42ApFxQIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQBZ
+hgzLCi/sWJFVG+6BFcOg10x0sz9OFAYWLL83PeoSksKAHa9lzgCw/rTXg8vv19p4
+W9uIb6sC3sH+PdexL8edD0UIGB7oEoN45z9n3on+cJCB+mZkM8nnQd5EmqrP/MhJ
+k330qZAfWe/Ockg50VRG6/5bBmfZRP2iCjp/a6mrax/ubuwazGOW2ejSEddaaNZC
+adbLydkKNYEWNHB2oWmKiGxXx3D1Ixw/axIE/0RepfWU+On0IN7YMjMv47RpOVzp
+ml5Efa6UANjCiBWk7T9c1k+8GaBUq5cVlAGTnvil3AZNAnX0OuuTFiQDoRKF4pY7
+tte4C5kA+fD1vHygWQ8G
 -----END CERTIFICATE-----
diff --git a/distributed/tests/tls-key.pem b/distributed/tests/tls-key.pem
index ffd84eac31f..27083dc7b63 100644
--- a/distributed/tests/tls-key.pem
+++ b/distributed/tests/tls-key.pem
@@ -1,28 +1,28 @@
 -----BEGIN PRIVATE KEY-----
-MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDlc+6wNJa58hI+
-FYrpdscRKeAjeIQD7OuoHuu2Ar80lUMpHvsnQfx9Vfh06tzi2ojRjU7ueL/KTNpa
-ASTnTpXGso1V4LFtcKTHDqU1d5F1BkJ0aqQsf4/UCvCer7GXHQ43TvrousGpWTj1
-XWfRH2E35cpyiW87YUlFf/rntBd2jkp372oua7S6JCd8Rdev1CBT4uzaT73C4Pyi
-YYrYlvBoZ0EhfnklF3uWu/hZwJzl5QNYJi3ks2EnlDi4Vf9l+yIQ6GNWKx4mY8x9
-QxYeG0lWOBt59wPosvUC+u0VBtP8r4TYNKbfJxUIWYPHgzRlysbVjIaa8DCDICRB
-EBhsIAZdAgMBAAECggEBAJlC8bjSt6bmUVcUzp+Yzejh/qvSsONqWhIG7Uy60vqN
-hLmvzvRu9/bD+NwnkHkanW7TI1jYYMsAwIyD3SF/dT2uQAePuROvvPCVELcA8bCo
-CLV4EccDQY/R6t0j38xfBQLeh2vyaq7yQMBrro+ZlTdgDvSoZhrUpbt9e5tOyKlP
-p8EwVW74251XSfONrbsbEvTq5GsNSgerAJT9pC05FhpXp92WPYrDFV0906hlt6z2
-vyLw/PVaQR3YLeNcZ7Fp/AEOTht3M4vfaMq1ibEOi5DueBV0RSGcFzKaP1FFnqsD
-a5x3NGpY5q8ngCcwlWWEsBLnVzDIjI7FJWOnX5F87GUCgYEA/dWuKo09GD6g+oeC
-R6LsSxZdqLl5jopUFgKMpdFWs07rZD1HoH+8Dpl0bYSvSBnXza7d5BmE5HwpA3ts
-Y9OinVu9UCgqToO3PZbbyqQ9AhqLW7YlVjZ2G+U8BM6Z1j0+C3aZ8jRXRRXQj9V1
-pUDuDjCK7WEp3Z3fyuqFpMLy+msCgYEA52kB539F7hRkxoob0gMidC3+qiMDjC6t
-uwbiovyWnO38t+ggQR27AZHfVHqW+iXSt2oiTjZ8pNhrBzpWcLZLvOVXzlTeJL+B
-YL2I/rjnRlplBDAfoUyLRCfQQ9FpaBvvoxooySNeRAIZ+9GgeiqTRYz91/CZX5h/
-OIgNV6Y5xFcCgYAo6ny8ly+17MJ+H8G8UmLMZ7qJK/yQIXQUaCzLvca/+ISxD7+v
-+2cQ42OFE9I9SQ0kNn/F+OcW/XyO9yItpTudKeCGPzMynMC4yMx9CaRb5nN4QRAb
-qYO4U2ctTzA0/II3ztJZ3F4E4NNRnBLupFy7JNAAG51n/77HMq6ANHx+KQKBgQDi
-d4CWlpbQVYL+TiDF2a15N54X0SZipTf+ZdGcokefksSVEoWmVDs0YRbH7SUd77AV
-MbgbBPXcubOcojoAGldqqKPS2HmCSyyJiVOtvTURQ9vCSnpZ97f/XltSU3mtmlFe
-NHafyY4yTkAxkkhT5qC0GzQ9hUgEJxCyOWudUJ8OBwKBgGd8n3UchJ+Ll3calcHO
-GKlWdqNPI6yLtY0wTqZGCZIWGjskm6YoYj4kEyBten/ERBArSuUeucj5qCxij97Y
-ue27ZP2xDX7ONdd5D36UPck3mfqBiW+BlUI9vGPhBDlA9sRhT2wIkDuu8W9MkyKd
-7Y9YAmBiswDTAj5TYYeWqWcM
+MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCrK5DpWSKAxHhY
+xfwn3Vofjq/b9LbmBx9TkErI1mlDpVMOGH6LI3jOn2PWyzttivBYLEa0AnhJN6Cr
+HacVfugG+uHeB9l8hrXPNqEi4o1qx4VLzKnroOnD185FvMaARp3Qffa9lOwJ1q5e
+lNIoPgtamQ4QJI7r/VBGH4+6K65mS1fWIJoymI9KsrvEWoay/Ub1RIwi+WnRnZYG
+R+fJa/DfWSrwcroxmv2B2IUnAf1c1BChX5KV7EWSI9e2eImZ+bYVSoFT0rJ/c9gJ
+nyz5NGIy5iQ+GT+9yXsHQIJ9sCbBFC0xUkx0zVnb+qbrZVx5m2HKdixyr8e6rT6P
+hjjYCkXFAgMBAAECggEAEF7uUSMRRbeEuxjUcQuenv4Km8Lf60grGtbmECdRkgKx
+2ttE3xHZcAmWhuiRGnz7V7OEX5sM3hPP7g27+TZWArTj/VRK3LoVkWfsKWX5mIjU
+mtdMORj+wphVRh4z7fjTBynE+4TOvnIl4y3gm7CzbUrOKYUVlkR39Pz3xhp/Z5EN
+ATFquJ4fCCtTMLiLmtwsIeLfqdzbnYy20W5ILdYmDhRG1OSaC9ioFax9VUHZ4/LB
+CpWry0fA4J4ulk2oMKo7MNVTh+uubUT8/xyBULx/u9mj9LLPzVLi3FBDsgg4Uglt
+K/pcAmMXUbwa4pxm5YMX2Rg+cM56/kH7EZnDjMCdgQKBgQDh4W84ClPbrVsk/+yX
+y2HxnCjMqc3r5OtAR7cFnv5BX+4urVF4pal720QmHGLTzo9Jiib172WvXPWwNc8a
+x7Knv3hXM1Z40J1E04VqwZx7fg/6DHRDhJ1j/sRb0VNStChKakOeVgvqJriapdUZ
+L/sCsUEWZqr3qYYj/Lkm3xGzeQKBgQDB/o8+IgaisarFKZDhiF8XWNJ+v7lmTRI3
+wpryVaBVopxJKVc2F61iWhllA8skvpmGqx+E7E2QeYHU+y3S3z+MASyoCKpc7pOA
+QaN2Mxu4nzc211n6a7T8b8UtRxO0wTXC3PfPFx0uKcOTGqBf9mPehN02WeiO2jiH
+uKs0qt+lrQKBgGSohlyN1aLelPBfD/rxCluWAGR8AyTtJZzGmJvrp1hQvJ6WY+kE
+wgtMwgjNu4ix0w6tfwE3xrOoFGCLChjgBxBT9fe+C/nY1Uy3igvx7Z7mpY4DgMgZ
+z92gybY1CRGg8ReAVVO4+qREzx0za0xHIm+nt96Yr/Wf2QQuegAFhijZAoGAE4WN
+XqBH8KlWWP7k0H1LrGrIy+55Nr08vxW7R/Wr69IwTgB2AUCCtyn/8jVehPUVOAMS
+aerpp0dC9G7OXrEOkQUxPDubAT0U5Mw2Kdfcy0hiYwdqy5N5JAoPWo4TgpYK1fAs
+k89spU+Ymb4NM3r089iE9ko35+A33hiMSgGD/gUCgYBNgBAUPrRG+si3Hub0xtmj
+3fbcG423kUEptjr5oouH4KGa3M0o3gY4KX9GfQ+cTd/zO7kju64lGrbQIk8V25fy
+Dc0C6kQCxAA1cBZrHC3mo/p/bg+oDX2bvDA7SSnxI18awkPL5Rq8BRNyJHVIvE9P
+1LrREdObq7nOuU18HkRKfA==
 -----END PRIVATE KEY-----
diff --git a/distributed/tests/tls-self-signed-cert.pem b/distributed/tests/tls-self-signed-cert.pem
index 9348cd3ed76..c77863bdbc0 100644
--- a/distributed/tests/tls-self-signed-cert.pem
+++ b/distributed/tests/tls-self-signed-cert.pem
@@ -1,20 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIIDPDCCAiSgAwIBAgIURCSGaBiclxOXPJYUmnDRNUjo1ggwDQYJKoZIhvcNAQEL
+MIIDPjCCAiagAwIBAgIUSd9/JbzkBSXo1da/uhRAok2k2EowDQYJKoZIhvcNAQEL
 BQAwSzELMAkGA1UEBhMCWFkxGTAXBgNVBAcMEERhc2stZGlzdHJpYnV0ZWQxDTAL
-BgNVBAoMBERhc2sxEjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0xODEyMTMwMjUyMTBa
-Fw0yODEyMTAwMjUyMTBaMEsxCzAJBgNVBAYTAlhZMRkwFwYDVQQHDBBEYXNrLWRp
-c3RyaWJ1dGVkMQ0wCwYDVQQKDAREYXNrMRIwEAYDVQQDDAlsb2NhbGhvc3QwggEi
-MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDD5bauoB8+u6oFQwPjQ8OAgMGr
-qUV6jkvfnhBpjUVhX/Y6CjZAGwFUye6twYzGXrwMrrwFtV4ApfcefAv4naCLvs1F
-yhAM8H8Sgqt7dChg/ejAnEvbxJW7vFnnpVmaiKfvkv36uLyLNjncumZ6e/jvtGgT
-Wr2rgrkjjT9zwRXt+be7zONht2Hx2P3iJQU/d4CTvU2c0DfnJqRI4B4vQ1sCeHHn
-nnKMjd+qYqQ3T5bmfH2gqqAESRbwfaRdf1k3S+zLpqWbqg7zW8cY9kysNZIvyfCg
-brHFzgvjyhw4enJWQiRxEHSfiHEza9TWqwm3gaETdhxryK+1KlQ2Q/fLuBJXAgMB
-AAGjGDAWMBQGA1UdEQQNMAuCCWxvY2FsaG9zdDANBgkqhkiG9w0BAQsFAAOCAQEA
-Ai9eW0p8XK26NF14jYZi830Qs0sUQ2uqO5glZbBGJz2U9L7kID68JDU+W2DsspYo
-xfj7aSFNo/1bYFMcWR/iJq0SzHX7ROwJ2GBpQwHWN/dMvmFVBreBPN2ggrV+wOPD
-iWHkitYEZGdViRQkMoyPeIBbkKcq3BPwNDcNzWQLhwbcxJNUk9PeEUG1c98Zcbvl
-H1YQtGh6LO/wPL8b+zqVxG0klYfe77uZHVFt92RiQA+KnTXAi+W+iWpQgDIdC3YA
-f3Jd/17CirqWtD+9u9kLV7vknL8TUCRO28as/2fLOjv7LjKliu23L2mfWp5Vhn0G
-A2ql2SoZUyYUWt/ZgWX0JA==
+BgNVBAoMBERhc2sxEjAQBgNVBAMMCWxvY2FsaG9zdDAgFw0yMTAxMDYwNzQ2MDha
+GA8zMDIxMDEwNjA3NDYwOFowSzELMAkGA1UEBhMCWFkxGTAXBgNVBAcMEERhc2st
+ZGlzdHJpYnV0ZWQxDTALBgNVBAoMBERhc2sxEjAQBgNVBAMMCWxvY2FsaG9zdDCC
+ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANtynllQbxI0l+nkM8WGezqA
+R0CKDBwVCaPogBfDlUi6K4DVh1GRFfO93N2I+lsTvPLg8m0JRt3Kt8ERVM66tqyC
+kL9YsBvlhlA+CzL54ae7hGtvyPlge9nQSRRVDDcn4J998uqh9hBbe9JdtwRrRZ5q
+/ElKNxNA8d12HPuNF6gOdKd3KenWV2NghyBi55LqlwiubSwPrhoAFmFzSeUohlr1
+o+kKSlm+RSQhqm8UMV8YFqGRqMrkLcY9y5GOpVKXoNorvwnVfwbKc31yGGGyRvAd
++OZ9d0250wt0aZrWV7Xkp7aiwvW1q0Y/D+iljXzmn/TAL6pH9SsY3kSWDwj4VzUC
+AwEAAaMYMBYwFAYDVR0RBA0wC4IJbG9jYWxob3N0MA0GCSqGSIb3DQEBCwUAA4IB
+AQCQ7tREbMu6WbBD0QspBNjU6zB5owFrNoAuXp1VAFarCHTFFayy0MFvcWbl6FAY
+hmyshjDV7ugWKEE1KB2oHRY7OUqz2VEwvnILfeQqN5AFStWgSUgrlfwH15kTGTpg
+ikzt1wqmYMDqoVf4RJxm9tJx7k0tQEoqXHELwMKSExEY0EXtV2HI+cUjUL86FGi+
+SbRWGVc5WkNeNG4vY4PcSNnJriOzdIchZlpqTfzYwwsTz9nsOx/jUoTESP/bRsso
+VrHXW5eZ8HaX/nIgNC8JGtwidrqdmt4PEV5PBvF9zjiYt6v44SIc2Vs5jjzHmC9k
+p2hyqM6bfq326WpQxgtKABTu
 -----END CERTIFICATE-----
diff --git a/distributed/tests/tls-self-signed-key.pem b/distributed/tests/tls-self-signed-key.pem
index 78c5705552e..8c322e6a65b 100644
--- a/distributed/tests/tls-self-signed-key.pem
+++ b/distributed/tests/tls-self-signed-key.pem
@@ -1,28 +1,28 @@
 -----BEGIN PRIVATE KEY-----
-MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDD5bauoB8+u6oF
-QwPjQ8OAgMGrqUV6jkvfnhBpjUVhX/Y6CjZAGwFUye6twYzGXrwMrrwFtV4Apfce
-fAv4naCLvs1FyhAM8H8Sgqt7dChg/ejAnEvbxJW7vFnnpVmaiKfvkv36uLyLNjnc
-umZ6e/jvtGgTWr2rgrkjjT9zwRXt+be7zONht2Hx2P3iJQU/d4CTvU2c0DfnJqRI
-4B4vQ1sCeHHnnnKMjd+qYqQ3T5bmfH2gqqAESRbwfaRdf1k3S+zLpqWbqg7zW8cY
-9kysNZIvyfCgbrHFzgvjyhw4enJWQiRxEHSfiHEza9TWqwm3gaETdhxryK+1KlQ2
-Q/fLuBJXAgMBAAECggEAYcPL1vyce09m+7dFQm94KzKW9RfGT/KiFRiqcjMJ1B3z
-IWYlSvrvQCSDFH5KxllTGjU+Ks8n+cTaPoGD6/CmxfdUNKcyOXLwyAkkkbkXycgO
-F8oo5aYHYCkQ6VWREVlwDlVaoXz+w68tU2ZHbItgYiiz5CzeCbF0ujRs3VUYn6mo
-CqGk979FS6Q+q48aoGYtrWqslP9RT9oB4wsrSawxiRTnSFdjqpmmMLpVheC5f0u9
-szF0P9Uv/gkX+TZXGU4h099LPav7jRDhDX8jQ7RJbDG7QdUT0HWFiix+DexdPN5T
-ZBqoSTIdncSmLF0DlcHxe45sXkrAQ1KsH+9G9KM4WQKBgQDn3OJKibhnp2cPbO6F
-kT2r46V681xZ1YMPioTD3vPquOtBUbzyIqdAZcAGn9IjtksQXv5riq/Y2yzxHYja
-ZmouBhu8BgZyxQujpAHeduKuE1xYWK9uGrFKt3t3YNtUUum7bISPyzgSzldNIaqG
-fTjCZ+fC/2OC4w6k0arTePOR9QKBgQDYSlps8EoT/t7Vu9KZs57Kph5dyGjSL6M9
-Gn0lwZR0wyJA5Snw64nFcBJETIM5MIMaMDeUAo2i5upqfKzNEuK8+q3+E3khfb8i
-HZBnBDZiRZbnECobSswuxyZfMZko2guZ5PrA4XnWiJDpGWP6u7F8C45Ji2W191GM
-KpiyVJ0HmwKBgG+d8L3LMJ+Ngdp2KRjZg2PE8fuPPcztMOuBVkpogOlk3bOTIq85
-qWQh12AY/fM9JUObR0UD/YUadbcY7bGdjc3cGOvZxq7nVqQPJusuNg9wuIq7HrCV
-XvXGAETGtq8QPv5/zgze2n6LIJG7/DhD/dfwZyyC/yeaIImNeXvWBQMlAoGBAImK
-3u4f8Du+/F3WzxpHC1e7Q9xHscDX4mrdCXCVJZ8K6CF0mbNHVZlkp3kfXuNjYjup
-Je7NoNxTFAcbuZe/xsh+cV79ay5NFNrRCh84oozjzA6sTNqGOkoVAuXOWKON5Yq7
-/mnh+OTge6aAKwArk7ZacuV56/hHimgkPm5JOIK5AoGBAKDmXI2DeziBvKu4GoXf
-ZjsAULxm09vnVqJTD+R8a//HTGBH2WTp3/tiEilOtDQ3eWhfhwqS5VFsqYSw/asS
-HTahl1q4aYQbYM+B83zlrBglV89GpN7ZBlFXXbasDAKh8mkg3nRad6Zy2Vz99Nts
-m1W0vRoaqKJIcxM25x9Vc16f
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDbcp5ZUG8SNJfp
+5DPFhns6gEdAigwcFQmj6IAXw5VIuiuA1YdRkRXzvdzdiPpbE7zy4PJtCUbdyrfB
+EVTOurasgpC/WLAb5YZQPgsy+eGnu4Rrb8j5YHvZ0EkUVQw3J+CfffLqofYQW3vS
+XbcEa0WeavxJSjcTQPHddhz7jReoDnSndynp1ldjYIcgYueS6pcIrm0sD64aABZh
+c0nlKIZa9aPpCkpZvkUkIapvFDFfGBahkajK5C3GPcuRjqVSl6DaK78J1X8GynN9
+chhhskbwHfjmfXdNudMLdGma1le15Ke2osL1tatGPw/opY185p/0wC+qR/UrGN5E
+lg8I+Fc1AgMBAAECggEAWZdd1pDkB8a5DqBAErBotuWQMWUqtHxSRzywrwMGe36m
+LB8A8+gL0kr9VOSLcYuJZI856hITgJE2m/f7a7sGwT884SHQJFN8ll3+LH2gBt6j
+nUkx/73IUpCahVaDejEpn3PILMqarurqjmP8SC2Taq26+iD2g0HMfOGvy3M66Tzb
+8p68Zmx+PKQUq1h0YvyxkBCnWWk+tJhcLKjr6UQ/nsgYSuRNwTstAmdlneAmTuPI
+tsbI/T/VPqfJELgPPhPzcYmrNP4L1S8tlZ59uFSAKreqEDW6shQ/umrnKcVXucGP
+7/qoe0v1gIlutAk1H9iGdrg3yp8x6eSf8U0geKg+gQKBgQD1s4HVvvNLG67oCGt0
+gVHbDIMTDMmngnu/2nv7q7TmOMWx2T+m6GkHiYewIOTCoTHvNf537SRrSVUaHiRw
+Unoj6JG5PHYLfX13MKMx5V+VbvTU15GAE2ZDzrDVvyu3Fw7JqYu/LNSiC3dez0Nx
+2bIY+TeKXAKPU8ZSzUS+s/2SGQKBgQDkpWYgB76u9muvtYVTWMWVZlozytcc7Muh
+h7iSmF06YM17KTLFEaUEEPjkP6nS8k+2HHjS6JxCBiRI9sqJmVMrOCA8FZ41ixvl
+Be2ZwxD52H+grCTstHY3apahoSTkN5DY3qQx6/hV/4HaYV+SJ7l/6NTb7kHBwnCl
+qfDj5BIpfQKBgEg12pm0RhKz7yK4nsQd4a3MLpWhs35spewLzVZ473CeDNQpMkDe
+m0egzuUftbvR3jgcmU4rvwFdCwrjAviT9Nuq1XEoaNUVq5MlE0Ft1Tq44nHPbS7E
+BNE25NfdGH714t23259zEIcTcaDGr169iU83p3AdfI8dIvNg6qxVf9e5AoGATHe8
+he3RKfaj4SnWIx/HRMXsWPH3q2bOW4FmizCFzWZeklge8KWMCKYcd0mLZE9BtWgk
+SHUe/T3MmNJgvlAW7pdgwHeCZ+nUNG4dHpWMceKkHHo2Byv7dWEia3/XNLQYr08W
+ntOKREilhwGV0u2SLmMR/VPbjg+XWPBS7gRyxokCgYEAtLfy3zhbXlqK0d6ADwlk
+RExdzVJkmQXr6VfKhcM1Cv+DRPCRH/1tSy45JAXL3ahW6BoTU2lV5bwEhaEmR1WC
+4B0n+fNmtK+6iu7XOc/tSWKaR8jXf1kdx9XAE9zemREp/1e1uzA1teV2iGh2iknH
+MUpprCMh5ulG6wymy3vfgcg=
 -----END PRIVATE KEY-----

From eeb411e37f6ce754b800358462bf6c2a8a37f437 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Wed, 6 Jan 2021 09:36:32 -0800
Subject: [PATCH 1112/1550] Use `list` comprehensions to bind `TaskGroup` type
 (#4401)

As Cython doesn't track typed variables in generator expressions well,
use `list` comprehensions, which Cython does type correctly.
---
 distributed/scheduler.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index a29560617d9..af282263884 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -703,7 +703,7 @@ def active(self):
         return [
             tg
             for tg in self._groups
-            if any(v != 0 for k, v in tg._states.items() if k != "forgotten")
+            if any([v != 0 for k, v in tg._states.items() if k != "forgotten"])
         ]
 
     @property
@@ -5570,7 +5570,7 @@ def transition(self, key, finish, *args, **kwargs):
             if ts._state == "forgotten" and ts._group._name in self.task_groups:
                 # Remove TaskGroup if all tasks are in the forgotten state
                 tg: TaskGroup = ts._group
-                if not any(tg._states.get(s) for s in ALL_TASK_STATES):
+                if not any([tg._states.get(s) for s in ALL_TASK_STATES]):
                     ts._prefix._groups.remove(tg)
                     del self.task_groups[tg._name]
 

From 7fb5d4d691df91cb08702509aeebc366bac6f5a0 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Wed, 6 Jan 2021 10:51:29 -0800
Subject: [PATCH 1113/1550] Fix `test_as_current_is_thread_local` (#4402)

---
 distributed/tests/test_client.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 0625bf8a0fa..7aab958f3a7 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -4022,7 +4022,7 @@ def test_as_current_is_thread_local(s):
     l4.acquire()
 
     def run1():
-        with Client(s.address) as c:
+        with Client(s["address"]) as c:
             with c.as_current():
                 l1.acquire()
                 l2.release()
@@ -4035,7 +4035,7 @@ def run1():
                     l4.release()
 
     def run2():
-        with Client(s.address) as c:
+        with Client(s["address"]) as c:
             with c.as_current():
                 l1.release()
                 l2.acquire()

From 49e19c010b101c629691c73d62caee7140bb92d9 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Wed, 6 Jan 2021 19:23:51 -0800
Subject: [PATCH 1114/1550] Type `TaskGroup` in `active_states` (#4408)

This was missing a type annotation before, which results in issues when
accessing `_states` when compiled. This fixes that issue.
---
 distributed/scheduler.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index af282263884..fe8d1c61f8c 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -708,6 +708,7 @@ def active(self):
 
     @property
     def active_states(self):
+        tg: TaskGroup
         return merge_with(sum, [tg._states for tg in self.active])
 
     def __repr__(self):

From 8f33b9ed6f7f1ac27a4cc855f26188b2fe6ea6c5 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Thu, 7 Jan 2021 08:33:42 -0800
Subject: [PATCH 1115/1550] Type annotation of `_reevaluate_occupancy_worker`
 (#4398)

* Return `total_duration` in `set_duration_estimate`

There is no need to retrieve this value when we already have it. So just
go ahead and return the variable directly.

* Group `old` and `new` variables

* Drop unused variable `nbytes`

* Use `.get(...)` to grab stealing extension

This way if it is `None`, we can check that and handle it much faster
(both in Python and Cython/C).

* Type iterated variable

* Assign `new - old` to `diff`

Hopefully should order computation to avoid a round off error.

* Type occupancy related variables as `double`

This should speed up mathematical operations with these variables.
---
 distributed/scheduler.py | 28 ++++++++++++++++------------
 1 file changed, 16 insertions(+), 12 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index fe8d1c61f8c..77a9be2f6dc 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -4834,7 +4834,7 @@ def set_duration_estimate(self, ts: TaskState, ws: WorkerState):
             if exec_time > 2 * duration:
                 total_duration = 2 * exec_time
         ws._processing[ts] = total_duration
-        return ws._processing[ts]
+        return total_duration
 
     def transition_waiting_processing(self, key):
         try:
@@ -6139,23 +6139,27 @@ def reevaluate_occupancy(self, worker_index=0):
 
     def _reevaluate_occupancy_worker(self, ws: WorkerState):
         """ See reevaluate_occupancy """
-        old = ws._occupancy
-
-        new = 0
-        nbytes = 0
+        old: double = ws._occupancy
+        new: double = 0
+        diff: double
+        ts: TaskState
+        est: double
         for ts in ws._processing:
-            new += self.set_duration_estimate(ts, ws)
+            est = self.set_duration_estimate(ts, ws)
+            new += est
 
         ws._occupancy = new
-        self.total_occupancy += new - old
+        diff = new - old
+        self.total_occupancy += diff
         self.check_idle_saturated(ws)
 
         # significant increase in duration
-        if (new > old * 1.3) and ("stealing" in self.extensions):
-            steal = self.extensions["stealing"]
-            for ts in ws._processing:
-                steal.remove_key_from_stealable(ts)
-                steal.put_key_in_stealable(ts)
+        if new > old * 1.3:
+            steal = self.extensions.get("stealing")
+            if steal is not None:
+                for ts in ws._processing:
+                    steal.remove_key_from_stealable(ts)
+                    steal.put_key_in_stealable(ts)
 
     async def check_worker_ttl(self):
         ws: WorkerState

From 5812314e7a3081d8e95808c12724f035c6a01ffd Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Thu, 7 Jan 2021 17:50:45 +0000
Subject: [PATCH 1116/1550] Add nprocs auto option to dask-worker CLI (#4377)

---
 distributed/cli/dask_worker.py                | 11 +++++--
 distributed/cli/tests/test_dask_worker.py     |  9 ++++++
 distributed/deploy/local.py                   | 30 +----------------
 distributed/deploy/tests/test_deploy_utils.py | 14 ++++++++
 distributed/deploy/tests/test_local.py        | 15 +--------
 distributed/deploy/utils.py                   | 32 +++++++++++++++++++
 6 files changed, 66 insertions(+), 45 deletions(-)
 create mode 100644 distributed/deploy/tests/test_deploy_utils.py
 create mode 100644 distributed/deploy/utils.py

diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
index 140d0d1ab8f..3d60ce35603 100755
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -14,6 +14,7 @@
 from distributed import Nanny
 from distributed.cli.utils import check_python_3, install_signal_handlers
 from distributed.comm import get_address_host_port
+from distributed.deploy.utils import nprocesses_nthreads
 from distributed.preloading import validate_preload_argv
 from distributed.proctitle import (
     enable_proctitle_on_children,
@@ -124,11 +125,12 @@
 @click.option("--nthreads", type=int, default=0, help="Number of threads per process.")
 @click.option(
     "--nprocs",
-    type=int,
+    type=str,
     default=1,
     show_default=True,
     help="Number of worker processes to launch. "
-    "If negative, then (CPU_COUNT + 1 + nprocs) is used.",
+    "If negative, then (CPU_COUNT + 1 + nprocs) is used. "
+    "Set to 'auto' to set nprocs and nthreads dynamically based on CPU_COUNT",
 )
 @click.option(
     "--name",
@@ -292,6 +294,11 @@ def main(
         if v is not None
     }
 
+    if nprocs == "auto":
+        nprocs, nthreads = nprocesses_nthreads()
+    else:
+        nprocs = int(nprocs)
+
     if nprocs < 0:
         nprocs = CPU_COUNT + 1 + nprocs
 
diff --git a/distributed/cli/tests/test_dask_worker.py b/distributed/cli/tests/test_dask_worker.py
index f877ff9209b..0055c38e6dd 100644
--- a/distributed/cli/tests/test_dask_worker.py
+++ b/distributed/cli/tests/test_dask_worker.py
@@ -12,6 +12,7 @@
 
 import distributed.cli.dask_worker
 from distributed import Client, Scheduler
+from distributed.deploy.utils import nprocesses_nthreads
 from distributed.metrics import time
 from distributed.utils import sync, tmpfile, parse_ports
 from distributed.utils_test import popen, terminate_process, wait_for_port
@@ -246,6 +247,14 @@ def test_nprocs_negative(loop):
                 c.wait_for_workers(cpu_count(), timeout="10 seconds")
 
 
+def test_nprocs_auto(loop):
+    with popen(["dask-scheduler", "--no-dashboard"]) as sched:
+        with popen(["dask-worker", "127.0.0.1:8786", "--nprocs=auto"]) as worker:
+            with Client("tcp://127.0.0.1:8786", loop=loop) as c:
+                procs, _ = nprocesses_nthreads()
+                c.wait_for_workers(procs, timeout="10 seconds")
+
+
 def test_nprocs_expands_name(loop):
     with popen(["dask-scheduler", "--no-dashboard"]) as sched:
         with popen(
diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index f06892a1820..fa83ffae607 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -4,11 +4,11 @@
 import warnings
 import weakref
 
-from dask.utils import factors
 from dask.system import CPU_COUNT
 import toolz
 
 from .spec import SpecCluster
+from .utils import nprocesses_nthreads
 from ..nanny import Nanny
 from ..scheduler import Scheduler
 from ..security import Security
@@ -243,34 +243,6 @@ def start_worker(self, *args, **kwargs):
         )
 
 
-def nprocesses_nthreads(n=CPU_COUNT):
-    """
-    The default breakdown of processes and threads for a given number of cores
-
-    Parameters
-    ----------
-    n: int
-        Number of available cores
-
-    Examples
-    --------
-    >>> nprocesses_nthreads(4)
-    (4, 1)
-    >>> nprocesses_nthreads(32)
-    (8, 4)
-
-    Returns
-    -------
-    nprocesses, nthreads
-    """
-    if n <= 4:
-        processes = n
-    else:
-        processes = min(f for f in factors(n) if f >= math.sqrt(n))
-    threads = n // processes
-    return (processes, threads)
-
-
 clusters_to_close = weakref.WeakSet()
 
 
diff --git a/distributed/deploy/tests/test_deploy_utils.py b/distributed/deploy/tests/test_deploy_utils.py
new file mode 100644
index 00000000000..871aca3fd30
--- /dev/null
+++ b/distributed/deploy/tests/test_deploy_utils.py
@@ -0,0 +1,14 @@
+from distributed.deploy.utils import nprocesses_nthreads
+
+
+def test_default_process_thread_breakdown():
+    assert nprocesses_nthreads(1) == (1, 1)
+    assert nprocesses_nthreads(4) == (4, 1)
+    assert nprocesses_nthreads(5) == (5, 1)
+    assert nprocesses_nthreads(8) == (4, 2)
+    assert nprocesses_nthreads(12) in ((6, 2), (4, 3))
+    assert nprocesses_nthreads(20) == (5, 4)
+    assert nprocesses_nthreads(24) in ((6, 4), (8, 3))
+    assert nprocesses_nthreads(32) == (8, 4)
+    assert nprocesses_nthreads(40) in ((8, 5), (10, 4))
+    assert nprocesses_nthreads(80) in ((10, 8), (16, 5))
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index 2b63f28b10f..4f88d1fc3d6 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -17,7 +17,7 @@
 from dask.system import CPU_COUNT
 from distributed import Client, Worker, Nanny, get_client
 from distributed.core import Status
-from distributed.deploy.local import LocalCluster, nprocesses_nthreads
+from distributed.deploy.local import LocalCluster
 from distributed.metrics import time
 from distributed.system import MEMORY_LIMIT
 from distributed.utils_test import (  # noqa: F401
@@ -815,19 +815,6 @@ def test_local_tls_restart(loop):
             assert workers_before != workers_after
 
 
-def test_default_process_thread_breakdown():
-    assert nprocesses_nthreads(1) == (1, 1)
-    assert nprocesses_nthreads(4) == (4, 1)
-    assert nprocesses_nthreads(5) == (5, 1)
-    assert nprocesses_nthreads(8) == (4, 2)
-    assert nprocesses_nthreads(12) in ((6, 2), (4, 3))
-    assert nprocesses_nthreads(20) == (5, 4)
-    assert nprocesses_nthreads(24) in ((6, 4), (8, 3))
-    assert nprocesses_nthreads(32) == (8, 4)
-    assert nprocesses_nthreads(40) in ((8, 5), (10, 4))
-    assert nprocesses_nthreads(80) in ((10, 8), (16, 5))
-
-
 def test_asynchronous_property(loop):
     with LocalCluster(
         4,
diff --git a/distributed/deploy/utils.py b/distributed/deploy/utils.py
new file mode 100644
index 00000000000..4bfa32419bc
--- /dev/null
+++ b/distributed/deploy/utils.py
@@ -0,0 +1,32 @@
+import math
+
+from dask.system import CPU_COUNT
+from dask.utils import factors
+
+
+def nprocesses_nthreads(n=CPU_COUNT):
+    """
+    The default breakdown of processes and threads for a given number of cores
+
+    Parameters
+    ----------
+    n: int
+        Number of available cores
+
+    Examples
+    --------
+    >>> nprocesses_nthreads(4)
+    (4, 1)
+    >>> nprocesses_nthreads(32)
+    (8, 4)
+
+    Returns
+    -------
+    nprocesses, nthreads
+    """
+    if n <= 4:
+        processes = n
+    else:
+        processes = min(f for f in factors(n) if f >= math.sqrt(n))
+    threads = n // processes
+    return (processes, threads)

From f33edf8a3acfecf2a95cfde4ed682d2c4eb7e20b Mon Sep 17 00:00:00 2001
From: "Mads R. B. Kristensen" <madsbk@gmail.com>
Date: Thu, 7 Jan 2021 19:38:24 +0100
Subject: [PATCH 1117/1550] Use dumps_msgpack and loads_msgpack when packing
 high level graphs (#4409)

---
 distributed/client.py                  |  4 ++--
 distributed/protocol/highlevelgraph.py | 25 ++++++++-----------------
 2 files changed, 10 insertions(+), 19 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index a7887ef4b9d..ca747305c71 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -2585,11 +2585,11 @@ def _graph_to_futures(
             if not isinstance(dsk, HighLevelGraph):
                 dsk = HighLevelGraph.from_collections(id(dsk), dsk, dependencies=())
 
-            dsk = highlevelgraph_pack(dsk, self, keyset)
-
             if isinstance(retries, Number) and retries > 0:
                 retries = {k: retries for k in dsk}
 
+            dsk = highlevelgraph_pack(dsk, self, keyset)
+
             # Create futures before sending graph (helps avoid contention)
             futures = {key: Future(key, self, inform=False) for key in keyset}
             self._send_to_scheduler(
diff --git a/distributed/protocol/highlevelgraph.py b/distributed/protocol/highlevelgraph.py
index 6e637717435..7895c465c95 100644
--- a/distributed/protocol/highlevelgraph.py
+++ b/distributed/protocol/highlevelgraph.py
@@ -1,4 +1,3 @@
-import msgpack
 from tlz import valmap
 
 from dask.core import keys_in_tasks
@@ -10,12 +9,8 @@
 
 from ..utils import CancelledError
 
-from .utils import msgpack_opts
-from .serialize import (
-    import_allowed_module,
-    msgpack_encode_default,
-    msgpack_decode_default,
-)
+from .core import dumps_msgpack, loads_msgpack
+from .serialize import import_allowed_module
 
 
 def _materialized_layer_pack(
@@ -93,8 +88,8 @@ def highlevelgraph_pack(hlg: HighLevelGraph, client, client_keys):
 
     Returns
     -------
-    data: bytes
-        Packed high level graph serialized by msgpack
+    data: list of header and payload
+        Packed high level graph serialized by dumps_msgpack
     """
     layers = []
 
@@ -126,8 +121,7 @@ def highlevelgraph_pack(hlg: HighLevelGraph, client, client_keys):
                 ),
             }
         )
-
-    return msgpack.dumps({"layers": layers}, default=msgpack_encode_default)
+    return dumps_msgpack({"layers": layers})
 
 
 def _materialized_layer_unpack(state, dsk, dependencies, annotations):
@@ -151,8 +145,8 @@ def highlevelgraph_unpack(dumped_hlg):
 
     Parameters
     ----------
-    dumped_hlg: bytes
-        Packed high level graph serialized by msgpack
+    dumped_hlg: list of header and payload
+        Packed high level graph serialized by dumps_msgpack
 
     Returns
     -------
@@ -164,10 +158,7 @@ def highlevelgraph_unpack(dumped_hlg):
         Annotations for `dsk`
     """
 
-    # Notice, we set `use_list=False`, which makes msgpack convert lists to tuples
-    hlg = msgpack.loads(
-        dumped_hlg, object_hook=msgpack_decode_default, use_list=False, **msgpack_opts
-    )
+    hlg = loads_msgpack(*dumped_hlg)
 
     dsk = {}
     deps = {}

From 658fccc3268bd16226cc5815bf00eed349fe1553 Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Thu, 7 Jan 2021 23:15:44 +0000
Subject: [PATCH 1118/1550] Xfail normalization tests (#4411)

---
 distributed/tests/test_client.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 7aab958f3a7..724a738777c 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -4354,6 +4354,7 @@ def test_normalize_collection_with_released_futures(c):
     assert res == sol
 
 
+@pytest.mark.xfail(reason="https://github.com/dask/distributed/issues/4404")
 @gen_cluster(client=True)
 async def test_auto_normalize_collection(c, s, a, b):
     da = pytest.importorskip("dask.array")
@@ -4380,6 +4381,7 @@ async def test_auto_normalize_collection(c, s, a, b):
         assert end - start < 1
 
 
+@pytest.mark.xfail(reason="https://github.com/dask/distributed/issues/4404")
 def test_auto_normalize_collection_sync(c):
     da = pytest.importorskip("dask.array")
     x = da.ones(10, chunks=5)

From 74e8dc64ef0436147a88ba225604ed7f86b0d569 Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Tue, 12 Jan 2021 18:22:52 +0000
Subject: [PATCH 1119/1550] Log adaptive error (#4422)

---
 distributed/deploy/adaptive_core.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/distributed/deploy/adaptive_core.py b/distributed/deploy/adaptive_core.py
index b15aa496345..b74e013c947 100644
--- a/distributed/deploy/adaptive_core.py
+++ b/distributed/deploy/adaptive_core.py
@@ -201,7 +201,8 @@ async def adapt(self) -> None:
                 await self.scale_up(**recommendations)
             if status == "down":
                 await self.scale_down(**recommendations)
-        except OSError:
+        except OSError as e:
+            logger.error("Adaptive stopping due to error %s", str(e))
             self.stop()
         finally:
             self._adapting = False

From da3d2a4e0145286620c7e3dad901351164ab3702 Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Wed, 13 Jan 2021 18:12:30 +0000
Subject: [PATCH 1120/1550] Add time started to scheduler info (#4425)

---
 distributed/scheduler.py         | 1 +
 distributed/tests/test_client.py | 1 +
 2 files changed, 2 insertions(+)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 77a9be2f6dc..43e54e47c27 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1963,6 +1963,7 @@ def identity(self, comm=None):
             "id": str(self.id),
             "address": self.address,
             "services": {key: v.port for (key, v) in self.services.items()},
+            "started": self.time_started,
             "workers": {
                 worker.address: worker.identity() for worker in self.workers.values()
             },
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 724a738777c..6fd59e49d72 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -3789,6 +3789,7 @@ def test_scheduler_info(c):
     info = c.scheduler_info()
     assert isinstance(info, dict)
     assert len(info["workers"]) == 2
+    assert isinstance(info["started"], float)
 
 
 def test_write_scheduler_file(c):

From 467bbc2c7d1c23328571a418f4d19f0110a3c18c Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Fri, 15 Jan 2021 15:35:04 -0600
Subject: [PATCH 1121/1550] bump version to 2021.01.0

---
 docs/source/changelog.rst | 35 ++++++++++++++++++++++++++++++++++-
 1 file changed, 34 insertions(+), 1 deletion(-)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index a1b36f831c6..de8d3ded2d4 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,36 @@
 Changelog
 =========
 
+2021.01.0
+---------
+
+Released on January 15, 2021
+
+- Add time started to scheduler info (:pr:`4425`) `Jacob Tomlinson`_
+- Log adaptive error (:pr:`4422`) `Jacob Tomlinson`_
+- Xfail normalization tests (:pr:`4411`) `Jacob Tomlinson`_
+- Use ``dumps_msgpack`` and ``loads_msgpack`` when packing high level graphs (:pr:`4409`) `Mads R. B. Kristensen`_
+- Add ``nprocs`` auto option to ``dask-worker`` CLI (:pr:`4377`) `Jacob Tomlinson`_
+- Type annotation of ``_reevaluate_occupancy_worker`` (:pr:`4398`) `jakirkham`_
+- Type ``TaskGroup`` in ``active_states`` (:pr:`4408`) `jakirkham`_
+- Fix ``test_as_current_is_thread_local`` (:pr:`4402`) `jakirkham`_
+- Use ``list`` comprehensions to bind ``TaskGroup`` type (:pr:`4401`) `jakirkham`_
+- Make tests pass after 2028 (:pr:`4403`) `Bernhard M. Wiedemann`_
+- Fix compilation warnings, ``decide_worker`` now a C func, stealing improvements (:pr:`4375`) `jakirkham`_
+- Drop custom ``__eq__`` from ``Status`` (:pr:`4270`) `jakirkham`_
+- ``test_performance_report``: skip without bokeh (:pr:`4388`) `Bruno Pagani`_
+- ``Nanny`` now respects dask settings from ctx mgr (:pr:`4378`) `Florian Jetter`_
+- Better task duration estimates for outliers (:pr:`4213`) `selshowk`_
+- Dask internal inherit config (:pr:`4364`) `Jacob Tomlinson`_
+- Provide ``setup.py`` option to profile Cython code (:pr:`4362`) `jakirkham`_
+- Optimizations of ``*State`` and ``Task*`` objects and stealing (:pr:`4358`) `jakirkham`_
+- Cast ``SortedDict`` s to ``dict`` s in a few key places & other minor changes (:pr:`4355`) `jakirkham`_
+- Use task annotation priorities for user-level priorities (:pr:`4354`) `James Bourbeau`_
+- Added docs to highlevelgraph pack/unpack (:pr:`4352`) `Mads R. B. Kristensen`_
+- Optimizations in notable functions used by transitions (:pr:`4351`) `jakirkham`_
+- Silence exception when releasing futures on process shutdown (:pr:`4309`) `Benjamin Zaitlen`_
+
+
 2020.12.0
 ---------
 
@@ -2120,4 +2150,7 @@ significantly without many new features.
 .. _`GeethanjaliEswaran`: https://github.com/geethanjalieswaran
 .. _`Timost`: https://github.com/Timost
 .. _`Ian Rose`: https://github.com/ian-r-rose
-.. _`marwan116`: https://github.com/marwan116
\ No newline at end of file
+.. _`marwan116`: https://github.com/marwan116
+.. _`Bernhard M. Wiedemann`: https://github.com/bmwiedemann
+.. _`Bruno Pagani`: https://github.com/ArchangeGabriel
+.. _`selshowk`: https://github.com/selshowk
\ No newline at end of file

From a7268f0df170a0b225b16e7c11904e683b9ce84c Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 18 Jan 2021 09:09:26 -0800
Subject: [PATCH 1122/1550] Don't require network to inspect tests (#4433)

Previously the get_ipv6 function was called at import time in test_core.py
This stopped any tests from running if a network adapter was not
attached

The results of this function were used only in one test,
so we move this call within that test and skip if it fails
---
 distributed/tests/test_core.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/distributed/tests/test_core.py b/distributed/tests/test_core.py
index 07dc6c29434..8b702b4c8ff 100644
--- a/distributed/tests/test_core.py
+++ b/distributed/tests/test_core.py
@@ -42,11 +42,6 @@
 from distributed.utils_test import loop  # noqa F401
 
 
-EXTERNAL_IP4 = get_ip()
-if has_ipv6():
-    EXTERNAL_IP6 = get_ipv6()
-
-
 def echo(comm, x):
     return x
 
@@ -194,6 +189,15 @@ async def test_server_listen():
     """
     Test various Server.listen() arguments and their effect.
     """
+    import socket
+
+    try:
+        EXTERNAL_IP4 = get_ip()
+        if has_ipv6():
+            EXTERNAL_IP6 = get_ipv6()
+    except socket.gaierror:
+        raise pytest.skip(reason="no network access")
+
     from contextlib import asynccontextmanager
 
     @asynccontextmanager

From d2f52c39a0ac48b6b081419f959ad3d09d297d7b Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 18 Jan 2021 11:39:57 -0800
Subject: [PATCH 1123/1550] Support missing event loop in Client.asynchronous
 (#4436)

* Support missing event loop in Client.asynchronous

* fixup test with late arriving worker

* Provide slightly more informative error message in cli test
---
 distributed/cli/tests/test_dask_scheduler.py | 2 ++
 distributed/client.py                        | 5 ++++-
 distributed/deploy/tests/test_local.py       | 2 ++
 3 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/distributed/cli/tests/test_dask_scheduler.py b/distributed/cli/tests/test_dask_scheduler.py
index 470bfddf81d..301f46fdd14 100644
--- a/distributed/cli/tests/test_dask_scheduler.py
+++ b/distributed/cli/tests/test_dask_scheduler.py
@@ -70,6 +70,8 @@ def test_dashboard(loop):
             if b"dashboard at" in line:
                 dashboard_port = int(line.decode().split(":")[-1].strip())
                 break
+        else:
+            raise Exception("dashboard not found")
 
         with Client("127.0.0.1:%d" % Scheduler.default_port, loop=loop) as c:
             pass
diff --git a/distributed/client.py b/distributed/client.py
index ca747305c71..82f9076c1f0 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -803,7 +803,10 @@ def asynchronous(self):
         though the client was originally created in asynchronous mode we may
         find ourselves in contexts when it is better to operate synchronously.
         """
-        return self._asynchronous and self.loop is IOLoop.current()
+        try:
+            return self._asynchronous and self.loop is IOLoop.current()
+        except RuntimeError:
+            return False
 
     @property
     def dashboard_link(self):
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index 4f88d1fc3d6..4a3c22b4383 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -973,6 +973,8 @@ async def test_repr(cleanup):
         memory_limit="2GB",
         asynchronous=True,
     ) as cluster:
+        async with Client(cluster, asynchronous=True) as client:
+            await client.wait_for_workers(2)
         text = repr(cluster)
         assert "workers=2" in text
         assert cluster.scheduler_address in text

From b21eac39b468f0c79fd4f130b7baccbc480f5905 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 18 Jan 2021 12:45:54 -0800
Subject: [PATCH 1124/1550] Support TaskState objects in story methods (#4434)

---
 distributed/scheduler.py            | 3 ++-
 distributed/tests/test_scheduler.py | 2 ++
 distributed/tests/test_worker.py    | 9 +++++++++
 distributed/worker.py               | 2 ++
 4 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 43e54e47c27..7529f76a216 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1,5 +1,6 @@
 import asyncio
 from collections import defaultdict, deque
+
 from collections.abc import Mapping, Set
 from contextlib import suppress
 from datetime import timedelta
@@ -5605,7 +5606,7 @@ def transitions(self, recommendations: dict):
 
     def story(self, *keys):
         """ Get all transitions that touch one of the input keys """
-        keys = set(keys)
+        keys = {key.key if isinstance(key, TaskState) else key for key in keys}
         return [
             t for t in self.transition_log if t[0] in keys or keys.intersection(t[3])
         ]
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index ec44ca85560..f6e0be29bbf 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -683,6 +683,8 @@ async def test_story(c, s, a, b):
 
     assert len(s.story(x.key, y.key)) > len(story)
 
+    assert s.story(x.key) == s.story(s.tasks[x.key])
+
 
 @gen_cluster(nthreads=[], client=True)
 async def test_scatter_no_workers(c, s):
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 112263ecd7d..59685511d6f 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -1783,3 +1783,12 @@ def f(x):
                     return threading.get_ident() == x._thread_ident
 
                 assert await c.submit(f, x)
+
+
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)])
+async def test_story(c, s, w):
+    future = c.submit(inc, 1)
+    await future
+    ts = w.tasks[future.key]
+    assert ts.state in str(w.story(ts))
+    assert w.story(ts) == w.story(ts.key)
diff --git a/distributed/worker.py b/distributed/worker.py
index 87ee7fe368d..32f92898fa1 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -1,5 +1,6 @@
 import asyncio
 import bisect
+
 from collections import defaultdict, deque, namedtuple
 from collections.abc import MutableMapping
 from contextlib import suppress
@@ -1791,6 +1792,7 @@ def stateof(self, key):
         }
 
     def story(self, *keys):
+        keys = [key.key if isinstance(key, TaskState) else key for key in keys]
         return [
             msg
             for msg in self.log

From 9442d9b3f2847bf6d0252a8ed671d342a5379501 Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Mon, 18 Jan 2021 21:47:24 +0100
Subject: [PATCH 1125/1550] Set runspec on all new tasks to avoid deadlocks
 (#4432)

* Set runspec in add_task
* Don't transition task to waiting prior to releasing
* Also, check that we don't release keys with dependents
* transition instead of release
* Clear who_has on released task
* Allow constrained error transition
* remove unnecessary definition for transition_ready_waiting
---
 distributed/worker.py | 79 ++++++++++++++++++++++++++++++++++++-------
 1 file changed, 67 insertions(+), 12 deletions(-)

diff --git a/distributed/worker.py b/distributed/worker.py
index 32f92898fa1..4d45fa46743 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -451,6 +451,10 @@ def __init__(
             ("waiting", "flight"): self.transition_waiting_flight,
             ("ready", "executing"): self.transition_ready_executing,
             ("ready", "memory"): self.transition_ready_memory,
+            ("ready", "error"): self.transition_ready_error,
+            ("ready", "waiting"): self.transition_ready_waiting,
+            ("constrained", "waiting"): self.transition_ready_waiting,
+            ("constrained", "error"): self.transition_ready_error,
             ("constrained", "executing"): self.transition_constrained_executing,
             ("executing", "memory"): self.transition_executing_done,
             ("executing", "error"): self.transition_executing_done,
@@ -1446,8 +1450,10 @@ def add_task(
         **kwargs2,
     ):
         try:
+            runspec = SerializedTask(function, args, kwargs, task)
             if key in self.tasks:
                 ts = self.tasks[key]
+                ts.runspec = runspec
                 if ts.state == "memory":
                     assert key in self.data or key in self.actors
                     logger.debug(
@@ -1476,6 +1482,7 @@ def add_task(
             if actor:
                 self.actors[ts.key] = None
 
+            ts.runspec = runspec
             ts.priority = priority
             ts.duration = duration
             if resource_restrictions:
@@ -1561,13 +1568,14 @@ def transition_waiting_flight(self, ts, worker=None):
                 pdb.set_trace()
             raise
 
-    def transition_flight_waiting(self, ts, worker=None, remove=True):
+    def transition_flight_waiting(self, ts, worker=None, remove=True, runspec=None):
         try:
             if self.validate:
                 assert ts.state == "flight"
 
             self.in_flight_tasks -= 1
             ts.coming_from = None
+            ts.runspec = runspec or ts.runspec
             if remove:
                 try:
                     ts.who_has.remove(worker)
@@ -1692,9 +1700,23 @@ def transition_ready_executing(self, ts):
                 pdb.set_trace()
             raise
 
+    def transition_ready_error(self, ts):
+        if self.validate:
+            assert ts.exception is not None
+            assert ts.traceback is not None
+        self.send_task_state_to_scheduler(ts)
+
     def transition_ready_memory(self, ts, value=None):
+        if value:
+            self.put_key_in_memory(ts, value=value)
         self.send_task_state_to_scheduler(ts)
 
+    def transition_ready_waiting(self, ts):
+        """
+        This transition is common for work stealing
+        """
+        pass
+
     def transition_constrained_executing(self, ts):
         self.transition_ready_executing(ts)
         for resource, quantity in ts.resource_restrictions.items():
@@ -2145,7 +2167,6 @@ async def gather_dep(self, worker, dep, deps, total_nbytes, cause=None):
                     self.repetitively_busy += 1
                     await asyncio.sleep(0.100 * 1.5 ** self.repetitively_busy)
 
-                    # See if anyone new has the data
                     await self.query_who_has(dep.key)
                     self.ensure_communicating()
 
@@ -2227,10 +2248,11 @@ def update_who_has(self, who_has):
                 if not workers:
                     continue
 
-                self.tasks[dep].who_has.update(workers)
+                if dep in self.tasks:
+                    self.tasks[dep].who_has.update(workers)
 
-                for worker in workers:
-                    self.has_what[worker].add(dep)
+                    for worker in workers:
+                        self.has_what[worker].add(dep)
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
@@ -2243,8 +2265,9 @@ def steal_request(self, key):
         # There may be a race condition between stealing and releasing a task.
         # In this case the self.tasks is already cleared. The `None` will be
         # registered as `already-computing` on the other end
+        ts = self.tasks.get(key)
         if key in self.tasks:
-            state = self.tasks[key].state
+            state = ts.state
         else:
             state = None
 
@@ -2252,11 +2275,24 @@ def steal_request(self, key):
         self.batched_stream.send(response)
 
         if state in ("ready", "waiting", "constrained"):
-            self.release_key(key)
+            # Resetting the runspec should be reset by the transition. However,
+            # the waiting->waiting transition results in a no-op which would not
+            # reset.
+            ts.runspec = None
+            self.transition(ts, "waiting")
+            if not ts.dependents:
+                self.release_key(ts.key)
+                if self.validate:
+                    assert ts.key not in self.tasks
+            if self.validate:
+                assert ts.runspec is None
 
     def release_key(self, key, cause=None, reason=None, report=True):
         try:
+            if self.validate:
+                assert isinstance(key, str)
             ts = self.tasks.get(key, TaskState(key=key))
+
             if cause:
                 self.log.append((key, "release-key", {"cause": cause}))
             else:
@@ -2280,6 +2316,7 @@ def release_key(self, key, cause=None, reason=None, report=True):
 
             for worker in ts.who_has:
                 self.has_what[worker].discard(ts.key)
+            ts.who_has.clear()
 
             if key in self.threads:
                 del self.threads[key]
@@ -2296,7 +2333,7 @@ def release_key(self, key, cause=None, reason=None, report=True):
                 self.batched_stream.send({"op": "release", "key": key, "cause": cause})
 
             self._notify_plugins("release_key", key, ts.state, cause, reason, report)
-            if key in self.tasks:
+            if key in self.tasks and not ts.dependents:
                 self.tasks.pop(key)
             del ts
         except CommClosedError:
@@ -2532,8 +2569,15 @@ async def execute(self, key, report=False):
             if key not in self.tasks:
                 return
             ts = self.tasks[key]
-            if ts.state != "executing" or ts.runspec is None:
+            if ts.state != "executing":
+                # This might happen if keys are canceled
+                logger.debug(
+                    "Trying to execute a task %s which is not in executing state anymore"
+                    % ts
+                )
                 return
+            if ts.runspec is None:
+                logger.critical("No runspec available for task %s." % ts)
             if self.validate:
                 assert not ts.waiting_for_data
                 assert ts.state == "executing"
@@ -2583,7 +2627,17 @@ async def execute(self, key, report=False):
                 executor_error = e
                 raise
 
-            if ts.state not in ("executing", "long-running"):
+            # We'll need to check again for the task state since it may have
+            # changed since the execution was kicked off. In particular, it may
+            # have been canceled and released already in which case we'll have
+            # to drop the result immediately
+            key = ts.key
+            ts = self.tasks.get(key)
+
+            if ts is None:
+                logger.debug(
+                    "Dropping result for %s since task has already been released." % key
+                )
                 return
 
             result["key"] = ts.key
@@ -2876,13 +2930,14 @@ def _notify_plugins(self, method_name, *args, **kwargs):
 
     def validate_task_memory(self, ts):
         assert ts.key in self.data or ts.key in self.actors
-        assert ts.nbytes is not None
+        assert isinstance(ts.nbytes, int)
         assert not ts.waiting_for_data
         assert ts.key not in self.ready
         assert ts.state == "memory"
 
     def validate_task_executing(self, ts):
         assert ts.state == "executing"
+        assert ts.runspec is not None
         assert ts.key not in self.data
         assert not ts.waiting_for_data
         assert all(
@@ -2901,7 +2956,7 @@ def validate_task_ready(self, ts):
     def validate_task_waiting(self, ts):
         assert ts.key not in self.data
         assert ts.state == "waiting"
-        if ts.dependencies:
+        if ts.dependencies and ts.runspec:
             assert not all(dep.key in self.data for dep in ts.dependencies)
 
     def validate_task_flight(self, ts):

From e736c0beee25584bbb7310c369c97b11b88ec8dc Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Wed, 20 Jan 2021 16:48:32 +0100
Subject: [PATCH 1126/1550] Use worker comm pool in Semaphore (#4195)

* Semaphore uses worker comm pool
* Switch semaphore logging to debug level
* Align usage of loop and scheduler attribute names in Semaphore
---
 distributed/semaphore.py            | 155 +++++++++++++++++-----------
 distributed/tests/test_semaphore.py |  67 ++++++------
 2 files changed, 134 insertions(+), 88 deletions(-)

diff --git a/distributed/semaphore.py b/distributed/semaphore.py
index 1ece16276de..4b2bff3f74c 100644
--- a/distributed/semaphore.py
+++ b/distributed/semaphore.py
@@ -6,12 +6,13 @@
 from collections import defaultdict, deque
 
 import dask
-from tornado.ioloop import PeriodicCallback
+from tornado.ioloop import IOLoop, PeriodicCallback
 
 from distributed.utils_comm import retry_operation
+
 from .metrics import time
-from .utils import log_errors, parse_timedelta
-from .worker import get_client
+from .utils import log_errors, parse_timedelta, sync, thread_state
+from .worker import get_client, get_worker
 
 logger = logging.getLogger(__name__)
 
@@ -130,7 +131,7 @@ def _get_lease(self, name, lease_id):
             or len(self.leases[name]) < self.max_leases[name]
         ):
             now = time()
-            logger.info("Acquire lease %s for %s at %s", lease_id, name, now)
+            logger.debug("Acquire lease %s for %s at %s", lease_id, name, now)
             self.leases[name][lease_id] = now
             self.metrics["acquire_total"][name] += 1
         else:
@@ -154,8 +155,8 @@ async def acquire(self, comm=None, name=None, timeout=None, lease_id=None):
 
             self.metrics["pending"][name] += 1
             while True:
-                logger.info(
-                    "Trying to acquire %s for %s with %ss left.",
+                logger.debug(
+                    "Trying to acquire %s for %s with %s seconds left.",
                     lease_id,
                     name,
                     w.leftover(),
@@ -177,7 +178,7 @@ async def acquire(self, comm=None, name=None, timeout=None, lease_id=None):
                         continue
                     except TimeoutError:
                         result = False
-                logger.info(
+                logger.debug(
                     "Acquisition of lease %s for %s is %s after waiting for %ss.",
                     lease_id,
                     name,
@@ -210,7 +211,7 @@ def release(self, comm=None, name=None, lease_id=None):
                 )
 
     def _release_value(self, name, lease_id):
-        logger.info("Releasing %s for %s", lease_id, name)
+        logger.debug("Releasing %s for %s", lease_id, name)
         # Everything needs to be atomic here.
         del self.leases[name][lease_id]
         self.events[name].set()
@@ -230,7 +231,7 @@ def _check_lease_timeout(self):
             for _id in ids:
                 time_since_refresh = now - self.leases[name][_id]
                 if time_since_refresh > self.lease_timeout:
-                    logger.info(
+                    logger.debug(
                         "Lease %s for %s timed out after %ss.",
                         _id,
                         name,
@@ -311,15 +312,19 @@ class Semaphore:
         Name of the semaphore to acquire.  Choosing the same name allows two
         disconnected processes to coordinate.  If not given, a random
         name will be generated.
-    client: Client (optional)
-        Client to use for communication with the scheduler.  If not given, the
-        default global client will be used.
     register: bool
         If True, register the semaphore with the scheduler. This needs to be
         done before any leases can be acquired. If not done during
         initialization, this can also be done by calling the register method of
         this class.
         When registering, this needs to be awaited.
+    scheduler_rpc: ConnectionPool
+        The ConnectionPool to connect to the scheduler. If None is provided, it
+        uses the worker or client pool. This paramter is mostly used for
+        testing.
+    loop: IOLoop
+        The event loop this instance is using. If None is provided, reuse the
+        loop of the active worker or client.
 
     Examples
     --------
@@ -355,8 +360,25 @@ class Semaphore:
 
     """
 
-    def __init__(self, max_leases=1, name=None, client=None, register=True):
-        self.client = client or get_client()
+    def __init__(
+        self,
+        max_leases=1,
+        name=None,
+        register=True,
+        scheduler_rpc=None,
+        loop=None,
+    ):
+
+        try:
+            worker = get_worker()
+            self.scheduler = scheduler_rpc or worker.scheduler
+            self.loop = loop or worker.loop
+
+        except ValueError:
+            client = get_client()
+            self.scheduler = scheduler_rpc or client.scheduler
+            self.loop = loop or client.io_loop
+
         self.name = name or "semaphore-" + uuid.uuid4().hex
         self.max_leases = max_leases
         self.id = uuid.uuid4().hex
@@ -381,27 +403,25 @@ def __init__(self, max_leases=1, name=None, client=None, register=True):
             self._refresh_leases, callback_time=refresh_leases_interval * 1000
         )
         self.refresh_callback = pc
-        # Registering the pc to the client here is important for proper cleanup
-        self._periodic_callback_name = f"refresh_semaphores_{self.id}"
-        self.client._periodic_callbacks[self._periodic_callback_name] = pc
 
         # Need to start the callback using IOLoop.add_callback to ensure that the
         # PC uses the correct event loop.
-        self.client.io_loop.add_callback(pc.start)
+        self.loop.add_callback(pc.start)
 
-    def register(self):
-        """
-        Register the semaphore on scheduler side
+    @property
+    def asynchronous(self):
+        return self.loop is IOLoop.current()
 
-        This will register the semaphore on scheduler side and ensure that all necessary data structures exist.
-        """
-        if self._registered is None:
-            self._registered = self.client.sync(
-                self.client.scheduler.semaphore_register,
-                name=self.name,
-                max_leases=self.max_leases,
-            )
-        return self._registered
+    async def _register(self):
+        await retry_operation(
+            self.scheduler.semaphore_register,
+            name=self.name,
+            max_leases=self.max_leases,
+            operation=f"semaphore register id={self.id} name={self.name}",
+        )
+
+    def register(self, **kwargs):
+        return self.sync(self._register)
 
     def __await__(self):
         async def create_semaphore():
@@ -411,34 +431,53 @@ async def create_semaphore():
 
         return create_semaphore().__await__()
 
+    def sync(self, func, *args, asynchronous=None, callback_timeout=None, **kwargs):
+        callback_timeout = parse_timedelta(callback_timeout)
+        if (
+            asynchronous
+            or self.asynchronous
+            or getattr(thread_state, "asynchronous", False)
+        ):
+            future = func(*args, **kwargs)
+            if callback_timeout is not None:
+                future = asyncio.wait_for(future, callback_timeout)
+            return future
+        else:
+            return sync(
+                self.loop, func, *args, callback_timeout=callback_timeout, **kwargs
+            )
+
     async def _refresh_leases(self):
         if self.refresh_leases and self._leases:
             logger.debug(
                 "%s refreshing leases for %s with IDs %s",
-                self.client.id,
+                self.id,
                 self.name,
                 self._leases,
             )
-            await self.client.scheduler.semaphore_refresh_leases(
-                lease_ids=list(self._leases), name=self.name
+            await retry_operation(
+                self.scheduler.semaphore_refresh_leases,
+                lease_ids=list(self._leases),
+                name=self.name,
+                operation="semaphore refresh leases: id=%s, lease_ids=%s, name=%s"
+                % (self.id, list(self._leases), self.name),
             )
 
     async def _acquire(self, timeout=None):
         lease_id = uuid.uuid4().hex
-        logger.info(
-            "%s requests lease for %s with ID %s", self.client.id, self.name, lease_id
+        logger.debug(
+            "%s requests lease for %s with ID %s", self.id, self.name, lease_id
         )
 
         # Using a unique lease id generated here allows us to retry since the
         # server handle is idempotent
-
         result = await retry_operation(
-            self.client.scheduler.semaphore_acquire,
+            self.scheduler.semaphore_acquire,
             name=self.name,
             timeout=timeout,
             lease_id=lease_id,
-            operation="semaphore acquire: client=%s, lease_id=%s, name=%s"
-            % (self.client.id, lease_id, self.name),
+            operation="semaphore acquire: id=%s, lease_id=%s, name=%s"
+            % (self.id, lease_id, self.name),
         )
         if result:
             self._leases.append(lease_id)
@@ -460,26 +499,22 @@ def acquire(self, timeout=None):
             a timedelta in string format, e.g. "200ms".
         """
         timeout = parse_timedelta(timeout)
-        return self.client.sync(self._acquire, timeout=timeout)
-
-    async def _release(self):
-        # popleft to release the oldest lease first
-        lease_id = self._leases.popleft()
-        logger.info("%s releases %s for %s", self.client.id, lease_id, self.name)
+        return self.sync(self._acquire, timeout=timeout)
 
+    async def _release(self, lease_id):
         try:
             await retry_operation(
-                self.client.scheduler.semaphore_release,
+                self.scheduler.semaphore_release,
                 name=self.name,
                 lease_id=lease_id,
-                operation="semaphore release: client=%s, lease_id=%s, name=%s"
-                % (self.client.id, lease_id, self.name),
+                operation="semaphore release: id=%s, lease_id=%s, name=%s"
+                % (self.id, lease_id, self.name),
             )
             return True
         except Exception:  # Release fails for whatever reason
             logger.error(
-                "Release failed for client=%s, lease_id=%s, name=%s. Cluster network might be unstable?"
-                % (self.client.id, lease_id, self.name),
+                "Release failed for id=%s, lease_id=%s, name=%s. Cluster network might be unstable?"
+                % (self.id, lease_id, self.name),
                 exc_info=True,
             )
             return False
@@ -499,13 +534,16 @@ def release(self):
         if not self._leases:
             raise RuntimeError("Released too often")
 
-        return self.client.sync(self._release)
+        # popleft to release the oldest lease first
+        lease_id = self._leases.popleft()
+        logger.debug("%s releases %s for %s", self.id, lease_id, self.name)
+        return self.sync(self._release, lease_id=lease_id)
 
     def get_value(self):
         """
         Return the number of currently registered leases.
         """
-        return self.client.sync(self.client.scheduler.semaphore_value, name=self.name)
+        return self.sync(self.scheduler.semaphore_value, name=self.name)
 
     def __enter__(self):
         self.acquire()
@@ -528,13 +566,14 @@ def __getstate__(self):
 
     def __setstate__(self, state):
         name, max_leases = state
-        client = get_client()
-        self.__init__(name=name, client=client, max_leases=max_leases, register=False)
+        self.__init__(
+            name=name,
+            max_leases=max_leases,
+            register=False,
+        )
 
     def close(self):
-        return self.client.sync(self.client.scheduler.semaphore_close, name=self.name)
+        return self.sync(self.scheduler.semaphore_close, name=self.name)
 
     def __del__(self):
-        if self._periodic_callback_name in self.client._periodic_callbacks:
-            self.client._periodic_callbacks[self._periodic_callback_name].stop()
-            del self.client._periodic_callbacks[self._periodic_callback_name]
+        self.refresh_callback.stop()
diff --git a/distributed/tests/test_semaphore.py b/distributed/tests/test_semaphore.py
index 5df270a0a5a..af2d59f3ea1 100644
--- a/distributed/tests/test_semaphore.py
+++ b/distributed/tests/test_semaphore.py
@@ -47,7 +47,7 @@ async def test_serializable(c, s, a, b):
     assert res
     sem2 = pickle.loads(pickle.dumps(sem))
     assert sem2.name == sem.name
-    assert sem2.client.scheduler.address == sem.client.scheduler.address
+    # assert sem2.client.scheduler.address == sem.client.scheduler.address
 
     # actual leases didn't change
     assert len(s.extensions["semaphores"].leases["x"]) == 1
@@ -73,6 +73,8 @@ def f(x, semaphore):
             return x + 1
 
     sem = await Semaphore(max_leases=2, name="x")
+
+    assert s.extensions["semaphores"]._semaphore_exists("x")
     futures = c.map(f, list(range(10)), semaphore=sem)
     await c.gather(futures)
 
@@ -98,33 +100,30 @@ def test_timeout_sync(client):
     client=True,
     timeout=20,
     config={
-        "distributed.scheduler.locks.lease-validation-interval": "500ms",
-        "distributed.scheduler.locks.lease-timeout": "500ms",
+        "distributed.scheduler.locks.lease-validation-interval": "200ms",
+        "distributed.scheduler.locks.lease-timeout": "200ms",
     },
 )
 async def test_release_semaphore_after_timeout(c, s, a, b):
     sem = await Semaphore(name="x", max_leases=2)
     await sem.acquire()  # leases: 2 - 1 = 1
-    semY = await Semaphore(name="y")
 
-    async with Client(s.address, asynchronous=True, name="ClientB") as clientB:
-        semB = await Semaphore(name="x", max_leases=2, client=clientB)
-        semYB = await Semaphore(name="y", client=clientB)
+    semB = await Semaphore(name="x", max_leases=2)
 
-        assert await semB.acquire()  # leases: 1 - 1 = 0
-        assert await semYB.acquire()
+    assert await semB.acquire()  # leases: 1 - 1 = 0
 
-        assert not (await sem.acquire(timeout=0.01))
-        assert not (await semB.acquire(timeout=0.01))
-        assert not (await semYB.acquire(timeout=0.01))
+    assert not (await sem.acquire(timeout=0.01))
+    assert not (await semB.acquire(timeout=0.01))
 
-    # `ClientB` goes out of scope, leases should be released
-    # At this point, we should be able to acquire x and y once
-    assert await sem.acquire()
-    assert await semY.acquire()
+    # B goes out of scope / cannot refresh anymore. For instance, because its
+    # worker died
+
+    semB.refresh_callback.stop()
+    del semB
 
-    assert not (await semY.acquire(timeout=0.5))
-    assert not (await sem.acquire(timeout=0.5))
+    assert await sem.acquire(timeout=1)
+
+    assert not (await sem.acquire(timeout=0.1))
 
 
 @gen_cluster()
@@ -312,16 +311,20 @@ async def connect(self, *args, **kwargs):
             self.cnn_count += 1
             return BrokenComm()
 
+    def reuse(self, addr, comm):
+        pass
+
 
 @gen_cluster(client=True)
 async def test_retry_acquire(c, s, a, b):
     with dask.config.set({"distributed.comm.retry.count": 1}):
 
         pool = await FlakyConnectionPool(failing_connections=1)
-        rpc = pool(s.address)
-        c.scheduler = rpc
+
         semaphore = await Semaphore(
-            max_leases=2, name="resource_we_want_to_limit", client=c
+            max_leases=2,
+            name="resource_we_want_to_limit",
+            scheduler_rpc=pool(s.address),
         )
         pool.activate()
 
@@ -429,7 +432,7 @@ def observe_state(sem):
     )
     fut_observe = c.submit(observe_state, sem=sem, workers=[observer.address])
 
-    with captured_logger("distributed.semaphore") as caplog:
+    with captured_logger("distributed.semaphore", level=logging.DEBUG) as caplog:
         payload, observer = await c.gather([futures, fut_observe])
 
     logs = caplog.getvalue().split("\n")
@@ -529,10 +532,11 @@ async def test_release_retry(c, s, a, b):
     """Verify that we can properly retry a semaphore release operation"""
     with dask.config.set({"distributed.comm.retry.count": 1}):
         pool = await FlakyConnectionPool(failing_connections=1)
-        rpc = pool(s.address)
-        c.scheduler = rpc
+
         semaphore = await Semaphore(
-            max_leases=2, name="resource_we_want_to_limit", client=c
+            max_leases=2,
+            name="resource_we_want_to_limit",
+            scheduler_rpc=pool(s.address),
         )
         await semaphore.acquire()
         pool.activate()  # Comm chaos starts
@@ -561,10 +565,11 @@ async def test_release_failure(c, s, a, b):
 
     with dask.config.set({"distributed.comm.retry.count": 1}):
         pool = await FlakyConnectionPool(failing_connections=5)
-        rpc = pool(s.address)
-        c.scheduler = rpc
+
         semaphore = await Semaphore(
-            max_leases=2, name="resource_we_want_to_limit", client=c
+            max_leases=2,
+            name="resource_we_want_to_limit",
+            scheduler_rpc=pool(s.address),
         )
         await semaphore.acquire()
         pool.activate()  # Comm chaos starts
@@ -576,7 +581,9 @@ async def test_release_failure(c, s, a, b):
             with captured_logger("distributed.utils_comm") as retry_log:
                 assert await semaphore.release() is False
 
-        with captured_logger("distributed.semaphore") as semaphore_cleanup_log:
+        with captured_logger(
+            "distributed.semaphore", level=logging.DEBUG
+        ) as semaphore_cleanup_log:
             pool.deactivate()  # comm chaos stops
             assert await semaphore.get_value() == 1  # lease is still registered
             await asyncio.sleep(0.2)  # Wait for lease to be cleaned up
@@ -589,7 +596,7 @@ async def test_release_failure(c, s, a, b):
         # Check release failed
         semaphore_log = semaphore_log.getvalue().split("\n")[0]
         assert semaphore_log.startswith(
-            "Release failed for client="
+            "Release failed for id="
         ) and semaphore_log.endswith("Cluster network might be unstable?")
 
         # Check lease has timed out

From e98d57dcad72d4a87fb177c4b2b9c98b2b2c50e1 Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Wed, 20 Jan 2021 17:32:27 +0000
Subject: [PATCH 1127/1550] Give clusters names (#4426)

* Give clusters names

* Add tests

* Add shutdown on close option

* Make n_workers kwargs explicit

* Missed one

* Move uuid setting into init
---
 distributed/deploy/cluster.py                 | 17 +++++++++++--
 distributed/deploy/local.py                   |  2 ++
 distributed/deploy/spec.py                    | 19 +++++++++++---
 distributed/deploy/tests/test_adaptive.py     | 14 +++++++----
 distributed/deploy/tests/test_local.py        | 25 ++++++++++++++++---
 distributed/deploy/tests/test_spec_cluster.py |  2 +-
 6 files changed, 63 insertions(+), 16 deletions(-)

diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
index 075867cd0b0..8838bd13d32 100644
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -4,6 +4,7 @@
 import logging
 import threading
 import warnings
+import uuid
 from tornado.ioloop import PeriodicCallback
 
 import dask.config
@@ -50,8 +51,9 @@ class Cluster:
     """
 
     _supports_scaling = True
+    name = None
 
-    def __init__(self, asynchronous, quiet=False):
+    def __init__(self, asynchronous, quiet=False, name=None):
         self.scheduler_info = {"workers": {}}
         self.periodic_callbacks = {}
         self._asynchronous = asynchronous
@@ -61,6 +63,10 @@ def __init__(self, asynchronous, quiet=False):
         self.quiet = quiet
         self.scheduler_comm = None
 
+        if name is not None:
+            self.name = name
+        elif self.name is None:
+            self.name = str(uuid.uuid4())[:8]
         self.status = Status.created
 
     async def _start(self):
@@ -424,8 +430,9 @@ def _cluster_class_name(self):
         return getattr(self, "_name", type(self).__name__)
 
     def __repr__(self):
-        text = "%s(%r, workers=%d, threads=%d" % (
+        text = "%s(%s, %r, workers=%d, threads=%d" % (
             self._cluster_class_name,
+            self.name,
             self.scheduler_address,
             len(self.scheduler_info["workers"]),
             sum(w["nthreads"] for w in self.scheduler_info["workers"].values()),
@@ -449,3 +456,9 @@ def requested(self):
     @property
     def observed(self):
         return {d["name"] for d in self.scheduler_info["workers"].values()}
+
+    def __eq__(self, other):
+        return self.name == other.name
+
+    def __hash__(self):
+        return id(self)
diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index fa83ffae607..a30a6e410f5 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -94,6 +94,7 @@ class LocalCluster(SpecCluster):
 
     def __init__(
         self,
+        name=None,
         n_workers=None,
         threads_per_worker=None,
         processes=True,
@@ -227,6 +228,7 @@ def __init__(
         workers = {i: worker for i in range(n_workers)}
 
         super().__init__(
+            name=name,
             scheduler=scheduler,
             workers=workers,
             worker=worker,
diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index 0ba24c43632..ac0dd68bf2c 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -245,6 +245,7 @@ def __init__(
         security=None,
         silence_logs=False,
         name=None,
+        shutdown_on_close=True,
     ):
         self._created = weakref.WeakSet()
 
@@ -268,8 +269,12 @@ def __init__(
         self._instances.add(self)
         self._correct_state_waiting = None
         self._name = name or type(self).__name__
+        self.shutdown_on_close = shutdown_on_close
 
-        super().__init__(asynchronous=asynchronous)
+        super().__init__(
+            asynchronous=asynchronous,
+            name=name,
+        )
 
         if not self.asynchronous:
             self._loop_runner.start()
@@ -617,6 +622,11 @@ def adapt(
 
         return super().adapt(*args, minimum=minimum, maximum=maximum, **kwargs)
 
+    @classmethod
+    def from_name(cls, name: str):
+        """Create an instance of this class to represent an existing cluster by name."""
+        raise NotImplementedError()
+
 
 async def run_spec(spec: dict, *args):
     workers = {}
@@ -636,6 +646,7 @@ async def run_spec(spec: dict, *args):
 @atexit.register
 def close_clusters():
     for cluster in list(SpecCluster._instances):
-        with suppress(gen.TimeoutError, TimeoutError):
-            if cluster.status != Status.closed:
-                cluster.close(timeout=10)
+        if cluster.shutdown_on_close:
+            with suppress(gen.TimeoutError, TimeoutError):
+                if cluster.status != Status.closed:
+                    cluster.close(timeout=10)
diff --git a/distributed/deploy/tests/test_adaptive.py b/distributed/deploy/tests/test_adaptive.py
index c8b588ef140..5f84b3165b4 100644
--- a/distributed/deploy/tests/test_adaptive.py
+++ b/distributed/deploy/tests/test_adaptive.py
@@ -51,7 +51,11 @@ def scale_down(self, workers):
 
 def test_adaptive_local_cluster(loop):
     with LocalCluster(
-        0, scheduler_port=0, silence_logs=False, dashboard_address=None, loop=loop
+        n_workers=0,
+        scheduler_port=0,
+        silence_logs=False,
+        dashboard_address=None,
+        loop=loop,
     ) as cluster:
         alc = cluster.adapt(interval="100 ms")
         with Client(cluster, loop=loop) as c:
@@ -76,7 +80,7 @@ def test_adaptive_local_cluster(loop):
 @pytest.mark.asyncio
 async def test_adaptive_local_cluster_multi_workers(cleanup):
     async with LocalCluster(
-        0,
+        n_workers=0,
         scheduler_port=0,
         silence_logs=False,
         processes=False,
@@ -146,7 +150,7 @@ def scale_up(self, n, **kwargs):
 @gen_test()
 async def test_min_max():
     cluster = await LocalCluster(
-        0,
+        n_workers=0,
         scheduler_port=0,
         silence_logs=False,
         processes=False,
@@ -201,7 +205,7 @@ async def test_avoid_churn(cleanup):
     user is taking a brief pause between work
     """
     async with LocalCluster(
-        0,
+        n_workers=0,
         asynchronous=True,
         processes=False,
         scheduler_port=0,
@@ -226,7 +230,7 @@ async def test_adapt_quickly():
     user is taking a brief pause between work
     """
     cluster = await LocalCluster(
-        0,
+        n_workers=0,
         asynchronous=True,
         processes=False,
         scheduler_port=0,
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index 4a3c22b4383..5a35f36695c 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -83,7 +83,7 @@ def test_close_twice():
 
 def test_procs():
     with LocalCluster(
-        2,
+        n_workers=2,
         scheduler_port=0,
         processes=False,
         threads_per_worker=3,
@@ -98,7 +98,7 @@ def test_procs():
         repr(c)
 
     with LocalCluster(
-        2,
+        n_workers=2,
         scheduler_port=0,
         processes=True,
         threads_per_worker=3,
@@ -434,7 +434,7 @@ def test_blocks_until_full(loop):
 @pytest.mark.asyncio
 async def test_scale_up_and_down():
     async with LocalCluster(
-        0,
+        n_workers=0,
         scheduler_port=0,
         processes=False,
         silence_logs=False,
@@ -763,7 +763,7 @@ def scale_down(self, *args, **kwargs):
 
     loop = IOLoop.current()
     cluster = await MyCluster(
-        0,
+        n_workers=0,
         scheduler_port=0,
         processes=False,
         silence_logs=False,
@@ -1047,3 +1047,20 @@ async def test_no_workers(cleanup):
         n_workers=0, silence_logs=False, dashboard_address=None, asynchronous=True
     ) as c:
         pass
+
+
+@pytest.mark.asyncio
+async def test_cluster_names():
+    async with LocalCluster(processes=False, asynchronous=True) as unnamed_cluster:
+        async with LocalCluster(
+            processes=False, asynchronous=True, name="mycluster"
+        ) as named_cluster:
+            assert isinstance(unnamed_cluster.name, str)
+            assert isinstance(named_cluster.name, str)
+            assert named_cluster.name == "mycluster"
+            assert unnamed_cluster == unnamed_cluster
+            assert named_cluster == named_cluster
+            assert unnamed_cluster != named_cluster
+
+        async with LocalCluster(processes=False, asynchronous=True) as unnamed_cluster2:
+            assert unnamed_cluster2 != unnamed_cluster
diff --git a/distributed/deploy/tests/test_spec_cluster.py b/distributed/deploy/tests/test_spec_cluster.py
index 5d5c4f7553a..0bd157d1546 100644
--- a/distributed/deploy/tests/test_spec_cluster.py
+++ b/distributed/deploy/tests/test_spec_cluster.py
@@ -494,7 +494,7 @@ async def test_run_spec_cluster_worker_names(cleanup):
 
     class MyCluster(SpecCluster):
         def _new_worker_name(self, worker_number):
-            return f"prefix-{self._name }-{worker_number}-suffix"
+            return f"prefix-{self.name}-{worker_number}-suffix"
 
     async with SpecCluster(
         asynchronous=True, scheduler=scheduler, worker=worker

From 2bc8138948bd30d7d11a977d3f806ffc401a7671 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Thu, 21 Jan 2021 11:15:06 -0800
Subject: [PATCH 1128/1550] Additional optimizations to stealing (#4445)

* Delay getting worker info until needed

* Construct default `next_time` only when needed

* Determine `last` only when used

* Assign `len(workers)` to `nworkers`

* Annotate variables in `reevaluate_occupancy`

* Inline `DELAY` for simplicity

* Go back to initializing `last` and `next_time`

There are some cases where we enter the `if`, but wouldn't set
`next_time`. So go back to giving this a default value.

* Avoid extra `"processing"` check if not needed

* Only retrieve `ts` when used

* Simplify `set_duration_estimate` for unknown `ts`

* Compute `total_duration` only as needed

* Define `total_duration` last

* Determine `comm` cost only when it is used

* Swap `exec_time` & `duration` definition
---
 distributed/scheduler.py | 26 ++++++++++++++------------
 distributed/stealing.py  | 10 +++++-----
 2 files changed, 19 insertions(+), 17 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 7529f76a216..9b4a5b85a3b 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -4828,13 +4828,14 @@ def set_duration_estimate(self, ts: TaskState, ws: WorkerState):
         estimate the task duration to be 2x current-runtime, otherwise we set it
         to be the average duration.
         """
+        exec_time: double = ws._executing.get(ts, 0)
         duration: double = self.get_task_duration(ts)
-        comm: double = self.get_comm_cost(ts, ws)
-        total_duration: double = duration + comm
-        if ts in ws._executing:
-            exec_time: double = ws._executing[ts]
-            if exec_time > 2 * duration:
-                total_duration = 2 * exec_time
+        total_duration: double
+        if exec_time > 2 * duration:
+            total_duration = 2 * exec_time
+        else:
+            comm: double = self.get_comm_cost(ts, ws)
+            total_duration = duration + comm
         ws._processing[ts] = total_duration
         return total_duration
 
@@ -6091,7 +6092,7 @@ def get_events(self, comm=None, topic=None):
     # Cleanup #
     ###########
 
-    def reevaluate_occupancy(self, worker_index=0):
+    def reevaluate_occupancy(self, worker_index: Py_ssize_t = 0):
         """Periodically reassess task duration time
 
         The expected duration of a task can change over time.  Unfortunately we
@@ -6106,18 +6107,19 @@ def reevaluate_occupancy(self, worker_index=0):
         lets us avoid this fringe optimization when we have better things to
         think about.
         """
-        DELAY = 0.1
         try:
             if self.status == Status.closed:
                 return
 
             last = time()
-            next_time = timedelta(seconds=DELAY)
+            next_time = timedelta(seconds=0.1)
 
             if self.proc.cpu_percent() < 50:
-                workers = list(self.workers.values())
-                for i in range(len(workers)):
-                    ws: WorkerState = workers[worker_index % len(workers)]
+                workers: list = list(self.workers.values())
+                nworkers: Py_ssize_t = len(workers)
+                i: Py_ssize_t
+                for i in range(nworkers):
+                    ws: WorkerState = workers[worker_index % nworkers]
                     worker_index += 1
                     try:
                         if ws is None or not ws._processing:
diff --git a/distributed/stealing.py b/distributed/stealing.py
index c1f07a8adfc..1e8428854ff 100644
--- a/distributed/stealing.py
+++ b/distributed/stealing.py
@@ -71,20 +71,20 @@ def teardown(self):
     def transition(
         self, key, start, finish, compute_start=None, compute_stop=None, *args, **kwargs
     ):
-        ts = self.scheduler.tasks[key]
         if finish == "processing":
+            ts = self.scheduler.tasks[key]
             self.put_key_in_stealable(ts)
-
-        if start == "processing":
+        elif start == "processing":
+            ts = self.scheduler.tasks[key]
             self.remove_key_from_stealable(ts)
             if finish != "memory":
                 self.in_flight.pop(ts, None)
 
     def put_key_in_stealable(self, ts):
-        ws = ts.processing_on
-        worker = ws.address
         cost_multiplier, level = self.steal_time_ratio(ts)
         if cost_multiplier is not None:
+            ws = ts.processing_on
+            worker = ws.address
             self.stealable_all[level].add(ts)
             self.stealable[worker][level].add(ts)
             self.key_stealable[ts] = (worker, level)

From 9b486882a609ba4aab5b30c9c59ff6e609255c2e Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 21 Jan 2021 13:48:36 -0800
Subject: [PATCH 1129/1550] Add uvloop config value (#4448)

---
 distributed/__init__.py             | 13 +++++++++++++
 distributed/distributed-schema.yaml |  6 ++++++
 distributed/distributed.yaml        |  1 +
 3 files changed, 20 insertions(+)

diff --git a/distributed/__init__.py b/distributed/__init__.py
index bd21b701022..448e52cd28b 100644
--- a/distributed/__init__.py
+++ b/distributed/__init__.py
@@ -1,4 +1,5 @@
 from . import config
+import dask
 from dask.config import config
 from .actor import Actor, ActorFuture
 from .core import connect, rpc, Status
@@ -39,3 +40,15 @@
 __version__ = versions["version"]
 __git_revision__ = versions["full-revisionid"]
 del get_versions, versions
+
+if dask.config.get("distributed.admin.event-loop") in ("asyncio", "tornado"):
+    pass
+elif dask.config.get("distributed.admin.event-loop") == "uvloop":
+    import uvloop
+
+    uvloop.install()
+else:
+    raise ValueError(
+        "Expected distributed.admin.event-loop to be in ('asyncio', 'tornado', 'uvloop'), got %s"
+        % dask.config.get("distributed.admin.event-loop")
+    )
diff --git a/distributed/distributed-schema.yaml b/distributed/distributed-schema.yaml
index de3a21ce9d1..e2bb3916447 100644
--- a/distributed/distributed-schema.yaml
+++ b/distributed/distributed-schema.yaml
@@ -760,6 +760,12 @@ properties:
               The log format to emit.
 
               See https://docs.python.org/3/library/logging.html#logrecord-attributes
+          event-loop:
+            type: string
+            description: |
+              The event loop to use,
+
+              Must be one of tornado, asyncio, or uvloop
 
           pdb-on-err:
             type: boolean
diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index bc7e3eb0be6..0d1d1ccf579 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -174,6 +174,7 @@ distributed:
     log-length: 10000  # default length of logs to keep in memory
     log-format: '%(name)s - %(levelname)s - %(message)s'
     pdb-on-err: False       # enter debug mode on scheduling error
+    event-loop: tornado
 rmm:
   pool-size: null
 ucx:

From 6be2013d9ba0aea27ea04faece45b1fdebee05a5 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 21 Jan 2021 16:57:15 -0800
Subject: [PATCH 1130/1550] Make system monitor interval configurable (#4447)

Sometimes we want to reduce this to improve performance
---
 distributed/core.py                 | 8 +++++++-
 distributed/distributed-schema.yaml | 9 +++++++++
 distributed/distributed.yaml        | 2 ++
 3 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/distributed/core.py b/distributed/core.py
index 1d98241bb1f..4efa17680f1 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -199,7 +199,13 @@ def stop():
 
         self.periodic_callbacks = dict()
 
-        pc = PeriodicCallback(self.monitor.update, 500)
+        pc = PeriodicCallback(
+            self.monitor.update,
+            parse_timedelta(
+                dask.config.get("distributed.admin.system-monitor.interval")
+            )
+            * 1000,
+        )
         self.periodic_callbacks["monitor"] = pc
 
         self._last_tick = time()
diff --git a/distributed/distributed-schema.yaml b/distributed/distributed-schema.yaml
index e2bb3916447..89730d83d2c 100644
--- a/distributed/distributed-schema.yaml
+++ b/distributed/distributed-schema.yaml
@@ -771,6 +771,15 @@ properties:
             type: boolean
             description: Enter Python Debugger on scheduling error
 
+          system-monitor:
+            type: object
+            description: |
+              Options for the periodic system monitor
+            properties:
+              interval:
+                type: string
+                description: Polling time to query cpu/memory statistics default 500ms
+
   rmm:
     type: object
     description: |
diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index 0d1d1ccf579..764478a8824 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -174,6 +174,8 @@ distributed:
     log-length: 10000  # default length of logs to keep in memory
     log-format: '%(name)s - %(levelname)s - %(message)s'
     pdb-on-err: False       # enter debug mode on scheduling error
+    system-monitor:
+      interval: 500ms
     event-loop: tornado
 rmm:
   pool-size: null

From a2308319d3030da311d41813b6921b74717e5f5c Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Fri, 22 Jan 2021 12:28:06 -0600
Subject: [PATCH 1131/1550] bump version to 2021.01.1

---
 docs/source/changelog.rst | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index de8d3ded2d4..0fbc53cd3f2 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,22 @@
 Changelog
 =========
 
+2021.01.1
+---------
+
+Released on January 22, 2021
+
+- Make system monitor interval configurable (:pr:`4447`) `Matthew Rocklin`_
+- Add ``uvloop`` config value (:pr:`4448`) `Matthew Rocklin`_
+- Additional optimizations to stealing (:pr:`4445`) `jakirkham`_
+- Give clusters names (:pr:`4426`) `Jacob Tomlinson`_
+- Use worker comm pool in ``Semaphore`` (:pr:`4195`) `Florian Jetter`_
+- Set ``runspec`` on all new tasks to avoid deadlocks (:pr:`4432`) `Florian Jetter`_
+- Support ``TaskState`` objects in story methods (:pr:`4434`) `Matthew Rocklin`_
+- Support missing event loop in ``Client.asynchronous`` (:pr:`4436`) `Matthew Rocklin`_
+- Don't require network to inspect tests (:pr:`4433`) `Matthew Rocklin`_
+
+
 2021.01.0
 ---------
 

From a3b1ab1fb11f3494a2a2a5e7219c9e387017934e Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Fri, 22 Jan 2021 12:48:44 -0800
Subject: [PATCH 1132/1550] Refactor `SchedulerState` from `Scheduler` (#4365)

* Move `worker_send` into transition functions

* Refactor `_task_to_msg` from `send_task_to_worker`

Provides a way for callers to simply construct the message if they are
not wanting to send it yet.

* Move `report` out of `_add_to_memory`

* Refactor out `_client_releases_keys`

This provides us a way to effectively call `client_releases_keys` from
other transitions without starting a new transition of its own.

* Collect client recs in `_add_to_memory`

* Use `_client_releases_keys` in transitions

* Refactor out `_task_to_report_msg`

Separates out the code needed to build a message for `report` based on
the `TaskState` in question from the actual call to `self.report`.

* Collect and send worker messages from transitions

* Handle `report` in `transition`

* Add method to send a message to a specific client

* Add `_task_to_client_msgs`

This converts a `TaskState` into a `dict` of messages with the keys
being the Clients to notify and the message being the report message.
Allows us to think of messages simply in terms of the message and where
it needs to be delivered without needing to know anything about the
`TaskState` it came from or the `ClientState`s involved.

* Replace `report_msg` with `client_msgs`

Instead of collecting a message to pass to `report` and letting the
relevant Clients be collected from the `TaskState` information later, go
ahead and collect that immediately while handling that `TaskState`.
These Clients then form the keys of `client_msgs` where the message
contains what was in `report_msg`. This allows us to keep all the
`TaskState` work contained to where it is relevant and can be handled
efficiently. Then the messaging out to Clients only needs be concerned
with the messages and where they go without needing to worry about what
they pertain to.

* Create empty `SchedulerState` class

This class should ultimately handle everything related to task graph
state and performing transitions currently done in the `Scheduler`. This
is intended to be a base class of the `Scheduler`, which we will use to
manage related state. As this is a separate class that does not inherit
from other Python classes, it should be a good target for Cythonization.
This should allow us to more thoroughly optimize these components of the
`Scheduler`.  Afterwards the `Scheduler` should ideally be left with
communication part, `async` methods, user facing APIs, etc., used to
interact with other elements that are not really targets of the
Cythonization effort.

* Move `transition*` methods into `SchedulerState`

Grabs all of the `transition` methods and methods they call and moves
them to `SchedulerState`.

* Add attributes for `SchedulerState`

Provides annotations for some attributes used by `SchedulerState`. Also
makes sure to provide an `_` prefix for these as they are accessible in
Cython only.

* Initialize attributes in `SchedulerState`

* Pass arguments to `super` class

* Use `SchedulerState` attributes

Now that we have typed attributes available through Cython. Use them
throughout the scheduler to improve performance. Also drop any duplicate
attribute assignments in `Scheduler`'s `__init__`.

* Use `cast` to access parent class attributes

Apparently Cython does not allow C-style access of attributes from
Python subclasses of extension types. However we can work around this by
`cast`ing `self` as the parent class type. To avoid muddying the code
too much, do this once per method and assign the result to `parent`.
Then use `parent` for all of the parent attribute access needed. Should
fix this issue while still allowing fast C-style attribute access in the
`Scheduler` subclass.

* Drop no longer needed `cast`s & local assignments

* Use `dict` views onto `SortedDict` where possible

As Cython will leverage the Python C API on `dict` objects, it is faster
to use a `dict`. This works well with `SortedDict` as it is a subclass
of `dict`. However we have to keep in mind the `dict` will not have the
same order as the `SortedDict`. Also the `dict` won't handle
modifications correctly (unlike the `SortedDict`). So we need to be
mindful of how the object is being used before using the `dict` view.
That said, this works well in a lot of the code if order doesn't matter
as happens when things are placed in new `dict`s, `set`s, etc. or when
ordered objects like `list` end up being used in a way where order is
irrelevant like `sum`ming or where the iteration order is matched by
everything else. In cases where it is not obvious the `dict` can be
used, we simply skip it and stick with the `SortedDict`.

* Add `@property`s for `SchedulerState` attributes

Ensures these Cython attributes can be retrieved by Python code.

* Take `worker_msgs` arg in `_propagate_forgotten`

Matches the behavior around `recommendations`. Also simplifies the
handling of `worker_msgs` (it's created only once).

* Use `parent` for private methods

As these will become `cfunc` decorated functions (only accessible from
the C API), we will need to `cast` to `SchedulerState` to get access to
them, which is what we do here in preparation for that.

* Annotate function arguments and return types

* Use `cfunc` on private `SchedulerState` methods

* Decorate `SchedulerState` methods with `@ccall`

Should allow faster C API calls to these methods in addition to their
existing Python APIs. Also disable exception checking when Python
objects are `return`ed since these already can and are checked for an
exception. Note that `@ccall` is not permitted on functions taking
`**kwargs` so those have been skipped.

* Add optional args to `transition_waiting_memory`

* Drop `**kwargs` from `_add_to_memory`

* Use `cfunc` to decorate `_add_to_memory`

* Make `SchedulerState` private methods functions

Gets them out of the `SchedulerState` virtual table so they can be used
like normal C functions. Should allow inlining and other optimizations
that apply.

* Collect `@property`s in `__pdict__`

As `@property`s are not placed in `__dict__`, but we need these in some
kind of `dict` for the dashboard. Collect them in `__pdict__`. This
should allow us to easily access them and place them in the dashboard.

https://stackoverflow.com/q/47432613/3877089

* Use `__pdict__` with `__dict__` in dashboard

Should give us the `@property`s as well.

* Refactor `consume_resources` & `release_resources`

* Refactor `aliases` into `SchedulerState`

* Refactor `coerce_hostname` into `SchedulerState`

* Refactor `task_metadata` into `SchedulerState`

* Refactor `remove_key` into `SchedulerState`
---
 distributed/core.py                |    3 +
 distributed/http/scheduler/info.py |   24 +-
 distributed/scheduler.py           | 7542 +++++++++++++++-------------
 3 files changed, 4017 insertions(+), 3552 deletions(-)

diff --git a/distributed/core.py b/distributed/core.py
index 4efa17680f1..15205f4f72c 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -135,6 +135,7 @@ def __init__(
         connection_args=None,
         timeout=None,
         io_loop=None,
+        **kwargs,
     ):
         self.handlers = {
             "identity": self.identity,
@@ -236,6 +237,8 @@ def set_thread_ident():
 
         self.__stopped = False
 
+        super().__init__(**kwargs)
+
     @property
     def status(self):
         return self._status
diff --git a/distributed/http/scheduler/info.py b/distributed/http/scheduler/info.py
index 6e5a222dd23..96199faba38 100644
--- a/distributed/http/scheduler/info.py
+++ b/distributed/http/scheduler/info.py
@@ -33,7 +33,13 @@ def get(self):
                 "workers.html",
                 title="Workers",
                 scheduler=self.server,
-                **merge(self.server.__dict__, ns, self.extra, rel_path_statics),
+                **merge(
+                    self.server.__dict__,
+                    self.server.__pdict__,
+                    ns,
+                    self.extra,
+                    rel_path_statics,
+                ),
             )
 
 
@@ -49,7 +55,13 @@ def get(self, worker):
                 title="Worker: " + worker,
                 scheduler=self.server,
                 Worker=worker,
-                **merge(self.server.__dict__, ns, self.extra, rel_path_statics),
+                **merge(
+                    self.server.__dict__,
+                    self.server.__pdict__,
+                    ns,
+                    self.extra,
+                    rel_path_statics,
+                ),
             )
 
 
@@ -65,7 +77,13 @@ def get(self, task):
                 title="Task: " + task,
                 Task=task,
                 scheduler=self.server,
-                **merge(self.server.__dict__, ns, self.extra, rel_path_statics),
+                **merge(
+                    self.server.__dict__,
+                    self.server.__pdict__,
+                    ns,
+                    self.extra,
+                    rel_path_statics,
+                ),
             )
 
 
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 9b4a5b85a3b..896b10e1380 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1527,43 +1527,33 @@ def _task_key_or_none(task):
     return task.key if task is not None else None
 
 
-class Scheduler(ServerNode):
-    """Dynamic distributed task scheduler
-
-    The scheduler tracks the current state of workers, data, and computations.
-    The scheduler listens for events and responds by controlling workers
-    appropriately.  It continuously tries to use the workers to execute an ever
-    growing dask graph.
-
-    All events are handled quickly, in linear time with respect to their input
-    (which is often of constant size) and generally within a millisecond.  To
-    accomplish this the scheduler tracks a lot of state.  Every operation
-    maintains the consistency of this state.
-
-    The scheduler communicates with the outside world through Comm objects.
-    It maintains a consistent and valid view of the world even when listening
-    to several clients at once.
-
-    A Scheduler is typically started either with the ``dask-scheduler``
-    executable::
+@cclass
+class SchedulerState:
+    """Underlying task state of dynamic scheduler
 
-         $ dask-scheduler
-         Scheduler started at 127.0.0.1:8786
+    Tracks the current state of workers, data, and computations.
 
-    Or within a LocalCluster a Client starts up without connection
-    information::
+    Handles transitions between different task states. Notifies the
+    Scheduler of changes by messaging passing through Queues, which the
+    Scheduler listens to responds accordingly.
 
-        >>> c = Client()  # doctest: +SKIP
-        >>> c.cluster.scheduler  # doctest: +SKIP
-        Scheduler(...)
+    All events are handled quickly, in linear time with respect to their
+    input (which is often of constant size) and generally within a
+    millisecond. Additionally when Cythonized, this can be faster still.
+    To accomplish this the scheduler tracks a lot of state.  Every
+    operation maintains the consistency of this state.
 
-    Users typically do not interact with the scheduler directly but rather with
-    the client object ``Client``.
+    Users typically do not interact with ``Transitions`` directly. Instead
+    users interact with the ``Client``, which in turn engages the
+    ``Scheduler`` affecting different transitions here under-the-hood. In
+    the background ``Worker``s also engage with the ``Scheduler``
+    affecting these state transitions as well.
 
     **State**
 
-    The scheduler contains the following state variables.  Each variable is
-    listed along with what it stores and a brief description.
+    The ``Transitions`` object contains the following state variables.
+    Each variable is listed along with what it stores and a brief
+    description.
 
     * **tasks:** ``{task key: TaskState}``
         Tasks currently known to the scheduler
@@ -1577,3921 +1567,4241 @@ class Scheduler(ServerNode):
     * **saturated:** ``{WorkerState}``:
         Set of workers that are not over-utilized
 
-    * **host_info:** ``{hostname: dict}``:
-        Information about each worker host
-
     * **clients:** ``{client key: ClientState}``
         Clients currently connected to the scheduler
 
-    * **services:** ``{str: port}``:
-        Other services running on this scheduler, like Bokeh
-    * **loop:** ``IOLoop``:
-        The running Tornado IOLoop
-    * **client_comms:** ``{client key: Comm}``
-        For each client, a Comm object used to receive task requests and
-        report task status updates.
-    * **stream_comms:** ``{worker key: Comm}``
-        For each worker, a Comm object from which we both accept stimuli and
-        report results
     * **task_duration:** ``{key-prefix: time}``
         Time we expect certain functions to take, e.g. ``{'sum': 0.25}``
     """
 
-    default_port = 8786
-    _instances = weakref.WeakSet()
+    _aliases: dict
+    _bandwidth: double
+    _clients: dict
+    _extensions: dict
+    _host_info: object
+    _idle: object
+    _idle_dv: dict
+    _n_tasks: Py_ssize_t
+    _resources: object
+    _saturated: set
+    _tasks: dict
+    _task_metadata: dict
+    _total_nthreads: Py_ssize_t
+    _total_occupancy: double
+    _unknown_durations: object
+    _unrunnable: set
+    _validate: bint
+    _workers: object
+    _workers_dv: dict
 
     def __init__(
         self,
-        loop=None,
-        delete_interval="500ms",
-        synchronize_worker_interval="60s",
-        services=None,
-        service_kwargs=None,
-        allowed_failures=None,
-        extensions=None,
-        validate=None,
-        scheduler_file=None,
-        security=None,
-        worker_ttl=None,
-        idle_timeout=None,
-        interface=None,
-        host=None,
-        port=0,
-        protocol=None,
-        dashboard_address=None,
-        dashboard=None,
-        http_prefix="/",
-        preload=None,
-        preload_argv=(),
-        plugins=(),
+        aliases: dict = None,
+        clients: dict = None,
+        workers=None,
+        host_info=None,
+        resources=None,
+        tasks: dict = None,
+        unrunnable: set = None,
+        validate: bint = False,
         **kwargs,
     ):
-        self._setup_logging(logger)
-
-        # Attributes
-        if allowed_failures is None:
-            allowed_failures = dask.config.get("distributed.scheduler.allowed-failures")
-        self.allowed_failures = allowed_failures
-        if validate is None:
-            validate = dask.config.get("distributed.scheduler.validate")
-        self.validate = validate
-        self.proc = psutil.Process()
-        self.delete_interval = parse_timedelta(delete_interval, default="ms")
-        self.synchronize_worker_interval = parse_timedelta(
-            synchronize_worker_interval, default="ms"
-        )
-        self.digests = None
-        self.service_specs = services or {}
-        self.service_kwargs = service_kwargs or {}
-        self.services = {}
-        self.scheduler_file = scheduler_file
-        worker_ttl = worker_ttl or dask.config.get("distributed.scheduler.worker-ttl")
-        self.worker_ttl = parse_timedelta(worker_ttl) if worker_ttl else None
-        idle_timeout = idle_timeout or dask.config.get(
-            "distributed.scheduler.idle-timeout"
+        if aliases is not None:
+            self._aliases = aliases
+        else:
+            self._aliases = dict()
+        self._bandwidth = parse_bytes(
+            dask.config.get("distributed.scheduler.bandwidth")
         )
-        if idle_timeout:
-            self.idle_timeout = parse_timedelta(idle_timeout)
+        if clients is not None:
+            self._clients = clients
         else:
-            self.idle_timeout = None
-        self.idle_since = time()
-        self.time_started = self.idle_since  # compatibility for dask-gateway
-        self._lock = asyncio.Lock()
-        self.bandwidth = parse_bytes(dask.config.get("distributed.scheduler.bandwidth"))
-        self.bandwidth_workers = defaultdict(float)
-        self.bandwidth_types = defaultdict(float)
+            self._clients = dict()
+        self._clients["fire-and-forget"] = ClientState("fire-and-forget")
+        self._extensions = dict()
+        if host_info is not None:
+            self._host_info = host_info
+        else:
+            self._host_info = defaultdict(dict)
+        self._idle = sortedcontainers.SortedDict()
+        self._idle_dv: dict = cast(dict, self._idle)
+        self._n_tasks = 0
+        if resources is not None:
+            self._resources = resources
+        else:
+            self._resources = defaultdict(dict)
+        self._saturated = set()
+        if tasks is not None:
+            self._tasks = tasks
+        else:
+            self._tasks = dict()
+        self._task_metadata = dict()
+        self._total_nthreads = 0
+        self._total_occupancy = 0
+        self._unknown_durations = defaultdict(set)
+        if unrunnable is not None:
+            self._unrunnable = unrunnable
+        else:
+            self._unrunnable = set()
+        self._validate = validate
+        if workers is not None:
+            self._workers = workers
+        else:
+            self._workers = sortedcontainers.SortedDict()
+        self._workers_dv: dict = cast(dict, self._workers)
+        super().__init__(**kwargs)
 
-        if not preload:
-            preload = dask.config.get("distributed.scheduler.preload")
-        if not preload_argv:
-            preload_argv = dask.config.get("distributed.scheduler.preload-argv")
-        self.preloads = preloading.process_preloads(self, preload, preload_argv)
+    @property
+    def aliases(self):
+        return self._aliases
 
-        if isinstance(security, dict):
-            security = Security(**security)
-        self.security = security or Security()
-        assert isinstance(self.security, Security)
-        self.connection_args = self.security.get_connection_args("scheduler")
-        self.connection_args["handshake_overrides"] = {  # common denominator
-            "pickle-protocol": 4
-        }
+    @property
+    def bandwidth(self):
+        return self._bandwidth
 
-        self._start_address = addresses_from_user_args(
-            host=host,
-            port=port,
-            interface=interface,
-            protocol=protocol,
-            security=security,
-            default_port=self.default_port,
-        )
+    @property
+    def clients(self):
+        return self._clients
 
-        http_server_modules = dask.config.get("distributed.scheduler.http.routes")
-        show_dashboard = dashboard or (dashboard is None and dashboard_address)
-        missing_bokeh = False
-        # install vanilla route if show_dashboard but bokeh is not installed
-        if show_dashboard:
-            try:
-                import distributed.dashboard.scheduler
-            except ImportError:
-                missing_bokeh = True
-                http_server_modules.append("distributed.http.scheduler.missing_bokeh")
-        routes = get_handlers(
-            server=self, modules=http_server_modules, prefix=http_prefix
-        )
-        self.start_http_server(routes, dashboard_address, default_port=8787)
-        if show_dashboard and not missing_bokeh:
-            distributed.dashboard.scheduler.connect(
-                self.http_application, self.http_server, self, prefix=http_prefix
-            )
+    @property
+    def extensions(self):
+        return self._extensions
 
-        # Communication state
-        self.loop = loop or IOLoop.current()
-        self.client_comms = dict()
-        self.stream_comms = dict()
-        self._worker_coroutines = []
-        self._ipython_kernel = None
+    @property
+    def host_info(self):
+        return self._host_info
 
-        # Task state
-        self.tasks = dict()
-        self.task_groups = dict()
-        self.task_prefixes = dict()
-        for old_attr, new_attr, wrap in [
-            ("priority", "priority", None),
-            ("dependencies", "dependencies", _legacy_task_key_set),
-            ("dependents", "dependents", _legacy_task_key_set),
-            ("retries", "retries", None),
-        ]:
-            func = operator.attrgetter(new_attr)
-            if wrap is not None:
-                func = compose(wrap, func)
-            setattr(self, old_attr, _StateLegacyMapping(self.tasks, func))
+    @property
+    def idle(self):
+        return self._idle
 
-        for old_attr, new_attr, wrap in [
-            ("nbytes", "nbytes", None),
-            ("who_wants", "who_wants", _legacy_client_key_set),
-            ("who_has", "who_has", _legacy_worker_key_set),
-            ("waiting", "waiting_on", _legacy_task_key_set),
-            ("waiting_data", "waiters", _legacy_task_key_set),
-            ("rprocessing", "processing_on", None),
-            ("host_restrictions", "host_restrictions", None),
-            ("worker_restrictions", "worker_restrictions", None),
-            ("resource_restrictions", "resource_restrictions", None),
-            ("suspicious_tasks", "suspicious", None),
-            ("exceptions", "exception", None),
-            ("tracebacks", "traceback", None),
-            ("exceptions_blame", "exception_blame", _task_key_or_none),
-        ]:
-            func = operator.attrgetter(new_attr)
-            if wrap is not None:
-                func = compose(wrap, func)
-            setattr(self, old_attr, _OptionalStateLegacyMapping(self.tasks, func))
+    @property
+    def n_tasks(self):
+        return self._n_tasks
 
-        for old_attr, new_attr, wrap in [
-            ("loose_restrictions", "loose_restrictions", None)
-        ]:
-            func = operator.attrgetter(new_attr)
-            if wrap is not None:
-                func = compose(wrap, func)
-            setattr(self, old_attr, _StateLegacySet(self.tasks, func))
+    @property
+    def resources(self):
+        return self._resources
 
-        self.generation = 0
-        self._last_client = None
-        self._last_time = 0
-        self.unrunnable = set()
+    @property
+    def saturated(self):
+        return self._saturated
 
-        self.n_tasks = 0
-        self.task_metadata = dict()
-        self.datasets = dict()
+    @property
+    def tasks(self):
+        return self._tasks
 
-        # Prefix-keyed containers
-        self.unknown_durations = defaultdict(set)
+    @property
+    def task_metadata(self):
+        return self._task_metadata
 
-        # Client state
-        self.clients = dict()
-        for old_attr, new_attr, wrap in [
-            ("wants_what", "wants_what", _legacy_task_key_set)
-        ]:
-            func = operator.attrgetter(new_attr)
-            if wrap is not None:
-                func = compose(wrap, func)
-            setattr(self, old_attr, _StateLegacyMapping(self.clients, func))
-        self.clients["fire-and-forget"] = ClientState("fire-and-forget")
+    @property
+    def total_nthreads(self):
+        return self._total_nthreads
 
-        # Worker state
-        self.workers = sortedcontainers.SortedDict()
-        for old_attr, new_attr, wrap in [
-            ("nthreads", "nthreads", None),
-            ("worker_bytes", "nbytes", None),
-            ("worker_resources", "resources", None),
-            ("used_resources", "used_resources", None),
-            ("occupancy", "occupancy", None),
-            ("worker_info", "metrics", None),
-            ("processing", "processing", _legacy_task_key_dict),
-            ("has_what", "has_what", _legacy_task_key_set),
-        ]:
-            func = operator.attrgetter(new_attr)
-            if wrap is not None:
-                func = compose(wrap, func)
-            setattr(self, old_attr, _StateLegacyMapping(self.workers, func))
+    @property
+    def total_occupancy(self):
+        return self._total_occupancy
 
-        self.idle = sortedcontainers.SortedDict()
-        self.saturated = set()
+    @total_occupancy.setter
+    def total_occupancy(self, v: double):
+        self._total_occupancy = v
 
-        self.total_nthreads = 0
-        self.total_occupancy = 0
-        self.host_info = defaultdict(dict)
-        self.resources = defaultdict(dict)
-        self.aliases = dict()
+    @property
+    def unknown_durations(self):
+        return self._unknown_durations
 
-        self._task_state_collections = [self.unrunnable]
+    @property
+    def unrunnable(self):
+        return self._unrunnable
 
-        self._worker_collections = [
-            self.workers,
-            self.host_info,
-            self.resources,
-            self.aliases,
-        ]
+    @property
+    def validate(self):
+        return self._validate
 
-        self.extensions = {}
-        self.plugins = list(plugins)
-        self.transition_log = deque(
-            maxlen=dask.config.get("distributed.scheduler.transition-log-length")
-        )
-        self.log = deque(
-            maxlen=dask.config.get("distributed.scheduler.transition-log-length")
-        )
-        self.events = defaultdict(lambda: deque(maxlen=100000))
-        self.event_counts = defaultdict(int)
-        self.worker_plugins = []
+    @validate.setter
+    def validate(self, v: bint):
+        self._validate = v
 
-        worker_handlers = {
-            "task-finished": self.handle_task_finished,
-            "task-erred": self.handle_task_erred,
-            "release": self.handle_release_data,
-            "release-worker-data": self.release_worker_data,
-            "add-keys": self.add_keys,
-            "missing-data": self.handle_missing_data,
-            "long-running": self.handle_long_running,
-            "reschedule": self.reschedule,
-            "keep-alive": lambda *args, **kwargs: None,
-            "log-event": self.log_worker_event,
-        }
+    @property
+    def workers(self):
+        return self._workers
 
-        client_handlers = {
-            "update-graph": self.update_graph,
-            "update-graph-hlg": self.update_graph_hlg,
-            "client-desires-keys": self.client_desires_keys,
-            "update-data": self.update_data,
-            "report-key": self.report_on_key,
-            "client-releases-keys": self.client_releases_keys,
-            "heartbeat-client": self.client_heartbeat,
-            "close-client": self.remove_client,
-            "restart": self.restart,
+    @property
+    def __pdict__(self):
+        return {
+            "bandwidth": self._bandwidth,
+            "resources": self._resources,
+            "saturated": self._saturated,
+            "unrunnable": self._unrunnable,
+            "n_tasks": self._n_tasks,
+            "unknown_durations": self._unknown_durations,
+            "validate": self._validate,
+            "tasks": self._tasks,
+            "total_nthreads": self._total_nthreads,
+            "total_occupancy": self._total_occupancy,
+            "extensions": self._extensions,
+            "clients": self._clients,
+            "workers": self._workers,
+            "idle": self._idle,
+            "host_info": self._host_info,
         }
 
-        self.handlers = {
-            "register-client": self.add_client,
-            "scatter": self.scatter,
-            "register-worker": self.add_worker,
-            "unregister": self.remove_worker,
-            "gather": self.gather,
-            "cancel": self.stimulus_cancel,
-            "retry": self.stimulus_retry,
-            "feed": self.feed,
-            "terminate": self.close,
-            "broadcast": self.broadcast,
-            "proxy": self.proxy,
-            "ncores": self.get_ncores,
-            "has_what": self.get_has_what,
-            "who_has": self.get_who_has,
-            "processing": self.get_processing,
-            "call_stack": self.get_call_stack,
-            "profile": self.get_profile,
-            "performance_report": self.performance_report,
-            "get_logs": self.get_logs,
-            "logs": self.get_logs,
-            "worker_logs": self.get_worker_logs,
-            "log_event": self.log_worker_event,
-            "events": self.get_events,
-            "nbytes": self.get_nbytes,
-            "versions": self.versions,
-            "add_keys": self.add_keys,
-            "rebalance": self.rebalance,
-            "replicate": self.replicate,
-            "start_ipython": self.start_ipython,
-            "run_function": self.run_function,
-            "update_data": self.update_data,
-            "set_resources": self.add_resources,
-            "retire_workers": self.retire_workers,
-            "get_metadata": self.get_metadata,
-            "set_metadata": self.set_metadata,
-            "heartbeat_worker": self.heartbeat_worker,
-            "get_task_status": self.get_task_status,
-            "get_task_stream": self.get_task_stream,
-            "register_worker_plugin": self.register_worker_plugin,
-            "adaptive_target": self.adaptive_target,
-            "workers_to_close": self.workers_to_close,
-            "subscribe_worker_status": self.subscribe_worker_status,
-            "start_task_metadata": self.start_task_metadata,
-            "stop_task_metadata": self.stop_task_metadata,
-        }
+    def transition_released_waiting(self, key):
+        try:
+            ts: TaskState = self._tasks[key]
+            dts: TaskState
+            worker_msgs: dict = {}
+            client_msgs: dict = {}
 
-        self._transitions = {
-            ("released", "waiting"): self.transition_released_waiting,
-            ("waiting", "released"): self.transition_waiting_released,
-            ("waiting", "processing"): self.transition_waiting_processing,
-            ("waiting", "memory"): self.transition_waiting_memory,
-            ("processing", "released"): self.transition_processing_released,
-            ("processing", "memory"): self.transition_processing_memory,
-            ("processing", "erred"): self.transition_processing_erred,
-            ("no-worker", "released"): self.transition_no_worker_released,
-            ("no-worker", "waiting"): self.transition_no_worker_waiting,
-            ("released", "forgotten"): self.transition_released_forgotten,
-            ("memory", "forgotten"): self.transition_memory_forgotten,
-            ("erred", "forgotten"): self.transition_released_forgotten,
-            ("erred", "released"): self.transition_erred_released,
-            ("memory", "released"): self.transition_memory_released,
-            ("released", "erred"): self.transition_released_erred,
-        }
+            if self._validate:
+                assert ts._run_spec
+                assert not ts._waiting_on
+                assert not ts._who_has
+                assert not ts._processing_on
+                assert not any([dts._state == "forgotten" for dts in ts._dependencies])
 
-        connection_limit = get_fileno_limit() / 2
+            if ts._has_lost_dependencies:
+                return {key: "forgotten"}, worker_msgs, client_msgs
 
-        super().__init__(
-            handlers=self.handlers,
-            stream_handlers=merge(worker_handlers, client_handlers),
-            io_loop=self.loop,
-            connection_limit=connection_limit,
-            deserialize=False,
-            connection_args=self.connection_args,
-            **kwargs,
-        )
+            ts.state = "waiting"
 
-        if self.worker_ttl:
-            pc = PeriodicCallback(self.check_worker_ttl, self.worker_ttl)
-            self.periodic_callbacks["worker-ttl"] = pc
+            recommendations: dict = {}
 
-        if self.idle_timeout:
-            pc = PeriodicCallback(self.check_idle, self.idle_timeout / 4)
-            self.periodic_callbacks["idle-timeout"] = pc
+            dts: TaskState
+            for dts in ts._dependencies:
+                if dts._exception_blame:
+                    ts._exception_blame = dts._exception_blame
+                    recommendations[key] = "erred"
+                    return recommendations, worker_msgs, client_msgs
 
-        if extensions is None:
-            extensions = list(DEFAULT_EXTENSIONS)
-            if dask.config.get("distributed.scheduler.work-stealing"):
-                extensions.append(WorkStealing)
-        for ext in extensions:
-            ext(self)
+            for dts in ts._dependencies:
+                dep = dts._key
+                if not dts._who_has:
+                    ts._waiting_on.add(dts)
+                if dts._state == "released":
+                    recommendations[dep] = "waiting"
+                else:
+                    dts._waiters.add(ts)
 
-        setproctitle("dask-scheduler [not started]")
-        Scheduler._instances.add(self)
-        self.rpc.allow_offload = False
-        self.status = Status.undefined
+            ts._waiters = {dts for dts in ts._dependents if dts._state == "waiting"}
 
-    ##################
-    # Administration #
-    ##################
+            if not ts._waiting_on:
+                if self._workers_dv:
+                    recommendations[key] = "processing"
+                else:
+                    self._unrunnable.add(ts)
+                    ts.state = "no-worker"
 
-    def __repr__(self):
-        return '<Scheduler: "%s" processes: %d cores: %d>' % (
-            self.address,
-            len(self.workers),
-            self.total_nthreads,
-        )
+            return recommendations, worker_msgs, client_msgs
+        except Exception as e:
+            logger.exception(e)
+            if LOG_PDB:
+                import pdb
 
-    def identity(self, comm=None):
-        """ Basic information about ourselves and our cluster """
-        d = {
-            "type": type(self).__name__,
-            "id": str(self.id),
-            "address": self.address,
-            "services": {key: v.port for (key, v) in self.services.items()},
-            "started": self.time_started,
-            "workers": {
-                worker.address: worker.identity() for worker in self.workers.values()
-            },
-        }
-        return d
+                pdb.set_trace()
+            raise
 
-    def get_worker_service_addr(self, worker, service_name, protocol=False):
-        """
-        Get the (host, port) address of the named service on the *worker*.
-        Returns None if the service doesn't exist.
+    def transition_no_worker_waiting(self, key):
+        try:
+            ts: TaskState = self._tasks[key]
+            dts: TaskState
+            worker_msgs: dict = {}
+            client_msgs: dict = {}
 
-        Parameters
-        ----------
-        worker : address
-        service_name : str
-            Common services include 'bokeh' and 'nanny'
-        protocol : boolean
-            Whether or not to include a full address with protocol (True)
-            or just a (host, port) pair
-        """
-        ws: WorkerState = self.workers[worker]
-        port = ws._services.get(service_name)
-        if port is None:
-            return None
-        elif protocol:
-            return "%(protocol)s://%(host)s:%(port)d" % {
-                "protocol": ws._address.split("://")[0],
-                "host": ws.host,
-                "port": port,
-            }
-        else:
-            return ws.host, port
+            if self._validate:
+                assert ts in self._unrunnable
+                assert not ts._waiting_on
+                assert not ts._who_has
+                assert not ts._processing_on
 
-    async def start(self):
-        """ Clear out old state and restart all running coroutines """
-        await super().start()
-        assert self.status != Status.running
+            self._unrunnable.remove(ts)
 
-        enable_gc_diagnosis()
+            if ts._has_lost_dependencies:
+                return {key: "forgotten"}, worker_msgs, client_msgs
 
-        self.clear_task_state()
+            recommendations: dict = {}
 
-        with suppress(AttributeError):
-            for c in self._worker_coroutines:
-                c.cancel()
+            for dts in ts._dependencies:
+                dep = dts._key
+                if not dts._who_has:
+                    ts._waiting_on.add(dts)
+                if dts._state == "released":
+                    recommendations[dep] = "waiting"
+                else:
+                    dts._waiters.add(ts)
 
-        for addr in self._start_address:
-            await self.listen(
-                addr,
-                allow_offload=False,
-                handshake_overrides={"pickle-protocol": 4, "compression": None},
-                **self.security.get_listen_args("scheduler"),
-            )
-            self.ip = get_address_host(self.listen_address)
-            listen_ip = self.ip
+            ts.state = "waiting"
 
-            if listen_ip == "0.0.0.0":
-                listen_ip = ""
+            if not ts._waiting_on:
+                if self._workers_dv:
+                    recommendations[key] = "processing"
+                else:
+                    self._unrunnable.add(ts)
+                    ts.state = "no-worker"
 
-        if self.address.startswith("inproc://"):
-            listen_ip = "localhost"
+            return recommendations, worker_msgs, client_msgs
+        except Exception as e:
+            logger.exception(e)
+            if LOG_PDB:
+                import pdb
 
-        # Services listen on all addresses
-        self.start_services(listen_ip)
+                pdb.set_trace()
+            raise
 
-        for listener in self.listeners:
-            logger.info("  Scheduler at: %25s", listener.contact_address)
-        for k, v in self.services.items():
-            logger.info("%11s at: %25s", k, "%s:%d" % (listen_ip, v.port))
+    @ccall
+    @exceptval(check=False)
+    def decide_worker(self, ts: TaskState) -> WorkerState:
+        """
+        Decide on a worker for task *ts*.  Return a WorkerState.
+        """
+        ws: WorkerState = None
+        valid_workers: set = self.valid_workers(ts)
 
-        self.loop.add_callback(self.reevaluate_occupancy)
+        if (
+            valid_workers is not None
+            and not valid_workers
+            and not ts._loose_restrictions
+            and self._workers_dv
+        ):
+            self._unrunnable.add(ts)
+            ts.state = "no-worker"
+            return ws
 
-        if self.scheduler_file:
-            with open(self.scheduler_file, "w") as f:
-                json.dump(self.identity(), f, indent=2)
-
-            fn = self.scheduler_file  # remove file when we close the process
-
-            def del_scheduler_file():
-                if os.path.exists(fn):
-                    os.remove(fn)
-
-            weakref.finalize(self, del_scheduler_file)
+        if ts._dependencies or valid_workers is not None:
+            ws = decide_worker(
+                ts,
+                self._workers_dv.values(),
+                valid_workers,
+                partial(self.worker_objective, ts),
+            )
+        else:
+            worker_pool = self._idle or self._workers
+            worker_pool_dv = cast(dict, worker_pool)
+            n_workers: Py_ssize_t = len(worker_pool_dv)
+            if n_workers < 20:  # smart but linear in small case
+                ws = min(worker_pool.values(), key=operator.attrgetter("occupancy"))
+            else:  # dumb but fast in large case
+                ws = worker_pool.values()[self._n_tasks % n_workers]
 
-        for preload in self.preloads:
-            await preload.start()
+        if self._validate:
+            assert ws is None or isinstance(ws, WorkerState), (
+                type(ws),
+                ws,
+            )
+            assert ws._address in self._workers_dv
 
-        await asyncio.gather(*[plugin.start(self) for plugin in self.plugins])
+        return ws
 
-        self.start_periodic_callbacks()
+    @ccall
+    def set_duration_estimate(self, ts: TaskState, ws: WorkerState) -> double:
+        """Estimate task duration using worker state and task state.
 
-        setproctitle("dask-scheduler [%s]" % (self.address,))
-        return self
+        If a task takes longer than twice the current average duration we
+        estimate the task duration to be 2x current-runtime, otherwise we set it
+        to be the average duration.
+        """
+        exec_time: double = ws._executing.get(ts, 0)
+        duration: double = self.get_task_duration(ts)
+        total_duration: double
+        if exec_time > 2 * duration:
+            total_duration = 2 * exec_time
+        else:
+            comm: double = self.get_comm_cost(ts, ws)
+            total_duration = duration + comm
+        ws._processing[ts] = total_duration
+        return total_duration
 
-    async def close(self, comm=None, fast=False, close_workers=False):
-        """Send cleanup signal to all coroutines then wait until finished
+    def transition_waiting_processing(self, key):
+        try:
+            ts: TaskState = self._tasks[key]
+            dts: TaskState
+            worker_msgs: dict = {}
+            client_msgs: dict = {}
 
-        See Also
-        --------
-        Scheduler.cleanup
-        """
-        if self.status in (Status.closing, Status.closed, Status.closing_gracefully):
-            await self.finished()
-            return
-        self.status = Status.closing
+            if self._validate:
+                assert not ts._waiting_on
+                assert not ts._who_has
+                assert not ts._exception_blame
+                assert not ts._processing_on
+                assert not ts._has_lost_dependencies
+                assert ts not in self._unrunnable
+                assert all([dts._who_has for dts in ts._dependencies])
 
-        logger.info("Scheduler closing...")
-        setproctitle("dask-scheduler [closing]")
+            ws: WorkerState = self.decide_worker(ts)
+            if ws is None:
+                return {}, worker_msgs, client_msgs
+            worker = ws._address
 
-        for preload in self.preloads:
-            await preload.teardown()
+            duration_estimate = self.set_duration_estimate(ts, ws)
+            ts._processing_on = ws
+            ws._occupancy += duration_estimate
+            self._total_occupancy += duration_estimate
+            ts.state = "processing"
+            self.consume_resources(ts, ws)
+            self.check_idle_saturated(ws)
+            self._n_tasks += 1
 
-        if close_workers:
-            await self.broadcast(msg={"op": "close_gracefully"}, nanny=True)
-            for worker in self.workers:
-                self.worker_send(worker, {"op": "close"})
-            for i in range(20):  # wait a second for send signals to clear
-                if self.workers:
-                    await asyncio.sleep(0.05)
-                else:
-                    break
+            if ts._actor:
+                ws._actors.add(ts)
 
-        await asyncio.gather(*[plugin.close() for plugin in self.plugins])
+            # logger.debug("Send job to worker: %s, %s", worker, key)
 
-        for pc in self.periodic_callbacks.values():
-            pc.stop()
-        self.periodic_callbacks.clear()
+            worker_msgs[worker] = _task_to_msg(self, ts)
 
-        self.stop_services()
+            return {}, worker_msgs, client_msgs
+        except Exception as e:
+            logger.exception(e)
+            if LOG_PDB:
+                import pdb
 
-        for ext in self.extensions.values():
-            with suppress(AttributeError):
-                ext.teardown()
-        logger.info("Scheduler closing all comms")
+                pdb.set_trace()
+            raise
 
-        futures = []
-        for w, comm in list(self.stream_comms.items()):
-            if not comm.closed():
-                comm.send({"op": "close", "report": False})
-                comm.send({"op": "close-stream"})
-            with suppress(AttributeError):
-                futures.append(comm.close())
+    def transition_waiting_memory(
+        self, key, nbytes=None, type=None, typename: str = None, worker=None, **kwargs
+    ):
+        try:
+            ws: WorkerState = self._workers_dv[worker]
+            ts: TaskState = self._tasks[key]
+            worker_msgs: dict = {}
+            client_msgs: dict = {}
 
-        for future in futures:  # TODO: do all at once
-            await future
+            if self._validate:
+                assert not ts._processing_on
+                assert ts._waiting_on
+                assert ts._state == "waiting"
 
-        for comm in self.client_comms.values():
-            comm.abort()
+            ts._waiting_on.clear()
 
-        await self.rpc.close()
+            if nbytes is not None:
+                ts.set_nbytes(nbytes)
 
-        self.status = Status.closed
-        self.stop()
-        await super().close()
+            self.check_idle_saturated(ws)
 
-        setproctitle("dask-scheduler [closed]")
-        disable_gc_diagnosis()
+            recommendations: dict = {}
+            client_msgs: dict = {}
 
-    async def close_worker(self, comm=None, worker=None, safe=None):
-        """Remove a worker from the cluster
+            _add_to_memory(
+                self, ts, ws, recommendations, client_msgs, type=type, typename=typename
+            )
 
-        This both removes the worker from our local state and also sends a
-        signal to the worker to shut down.  This works regardless of whether or
-        not the worker has a nanny process restarting it
-        """
-        logger.info("Closing worker %s", worker)
-        with log_errors():
-            self.log_event(worker, {"action": "close-worker"})
-            ws: WorkerState = self.workers[worker]
-            nanny_addr = ws._nanny
-            address = nanny_addr or worker
+            if self._validate:
+                assert not ts._processing_on
+                assert not ts._waiting_on
+                assert ts._who_has
 
-            self.worker_send(worker, {"op": "close", "report": False})
-            await self.remove_worker(address=worker, safe=safe)
+            return recommendations, worker_msgs, client_msgs
+        except Exception as e:
+            logger.exception(e)
+            if LOG_PDB:
+                import pdb
 
-    ###########
-    # Stimuli #
-    ###########
+                pdb.set_trace()
+            raise
 
-    def heartbeat_worker(
+    def transition_processing_memory(
         self,
-        comm=None,
-        address=None,
-        resolve_address=True,
-        now=None,
-        resources=None,
-        host_info=None,
-        metrics=None,
-        executing=None,
+        key,
+        nbytes=None,
+        type=None,
+        typename: str = None,
+        worker=None,
+        startstops=None,
+        **kwargs,
     ):
-        address = self.coerce_address(address, resolve_address)
-        address = normalize_address(address)
-        if address not in self.workers:
-            return {"status": "missing"}
+        ws: WorkerState
+        wws: WorkerState
+        worker_msgs: dict = {}
+        client_msgs: dict = {}
+        try:
+            ts: TaskState = self._tasks[key]
+            assert worker
+            assert isinstance(worker, str)
 
-        host = get_address_host(address)
-        local_now = time()
-        now = now or time()
-        assert metrics
-        host_info = host_info or {}
+            if self._validate:
+                assert ts._processing_on
+                ws = ts._processing_on
+                assert ts in ws._processing
+                assert not ts._waiting_on
+                assert not ts._who_has, (ts, ts._who_has)
+                assert not ts._exception_blame
+                assert ts._state == "processing"
 
-        self.host_info[host]["last-seen"] = local_now
-        frac = 1 / len(self.workers)
-        self.bandwidth = (
-            self.bandwidth * (1 - frac) + metrics["bandwidth"]["total"] * frac
-        )
-        for other, (bw, count) in metrics["bandwidth"]["workers"].items():
-            if (address, other) not in self.bandwidth_workers:
-                self.bandwidth_workers[address, other] = bw / count
-            else:
-                alpha = (1 - frac) ** count
-                self.bandwidth_workers[address, other] = self.bandwidth_workers[
-                    address, other
-                ] * alpha + bw * (1 - alpha)
-        for typ, (bw, count) in metrics["bandwidth"]["types"].items():
-            if typ not in self.bandwidth_types:
-                self.bandwidth_types[typ] = bw / count
-            else:
-                alpha = (1 - frac) ** count
-                self.bandwidth_types[typ] = self.bandwidth_types[typ] * alpha + bw * (
-                    1 - alpha
+            ws = self._workers_dv.get(worker)
+            if ws is None:
+                return {key: "released"}, worker_msgs, client_msgs
+
+            if ws != ts._processing_on:  # someone else has this task
+                logger.info(
+                    "Unexpected worker completed task, likely due to"
+                    " work stealing.  Expected: %s, Got: %s, Key: %s",
+                    ts._processing_on,
+                    ws,
+                    key,
                 )
+                return {}, worker_msgs, client_msgs
 
-        ws: WorkerState = self.workers[address]
+            if startstops:
+                L = list()
+                for startstop in startstops:
+                    stop = startstop["stop"]
+                    start = startstop["start"]
+                    action = startstop["action"]
+                    if action == "compute":
+                        L.append((start, stop))
 
-        ws._last_seen = time()
+                    # record timings of all actions -- a cheaper way of
+                    # getting timing info compared with get_task_stream()
+                    ts._prefix._all_durations[action] += stop - start
 
-        if executing is not None:
-            ws._executing = {
-                self.tasks[key]: duration for key, duration in executing.items()
-            }
+                if len(L) > 0:
+                    compute_start, compute_stop = L[0]
+                else:  # This is very rare
+                    compute_start = compute_stop = None
+            else:
+                compute_start = compute_stop = None
 
-        if metrics:
-            ws._metrics = metrics
+            #############################
+            # Update Timing Information #
+            #############################
+            if compute_start and ws._processing.get(ts, True):
+                # Update average task duration for worker
+                old_duration = ts._prefix._duration_average
+                new_duration = compute_stop - compute_start
+                if old_duration < 0:
+                    avg_duration = new_duration
+                else:
+                    avg_duration = 0.5 * old_duration + 0.5 * new_duration
 
-        if host_info:
-            self.host_info[host].update(host_info)
-
-        delay = time() - now
-        ws._time_delay = delay
+                ts._prefix._duration_average = avg_duration
+                ts._group._duration += new_duration
 
-        if resources:
-            self.add_resources(worker=address, resources=resources)
+                tts: TaskState
+                for tts in self._unknown_durations.pop(ts._prefix._name, ()):
+                    if tts._processing_on:
+                        wws = tts._processing_on
+                        old = wws._processing[tts]
+                        comm = self.get_comm_cost(tts, wws)
+                        wws._processing[tts] = avg_duration + comm
+                        wws._occupancy += avg_duration + comm - old
+                        self._total_occupancy += avg_duration + comm - old
 
-        self.log_event(address, merge({"action": "heartbeat"}, metrics))
+            ############################
+            # Update State Information #
+            ############################
+            if nbytes is not None:
+                ts.set_nbytes(nbytes)
 
-        return {
-            "status": "OK",
-            "time": time(),
-            "heartbeat-interval": heartbeat_interval(len(self.workers)),
-        }
+            recommendations: dict = {}
+            client_msgs: dict = {}
 
-    async def add_worker(
-        self,
-        comm=None,
-        address=None,
-        keys=(),
-        nthreads=None,
-        name=None,
-        resolve_address=True,
-        nbytes=None,
-        types=None,
-        now=None,
-        resources=None,
-        host_info=None,
-        memory_limit=None,
-        metrics=None,
-        pid=0,
-        services=None,
-        local_directory=None,
-        versions=None,
-        nanny=None,
-        extra=None,
-    ):
-        """ Add a new worker to the cluster """
-        with log_errors():
-            address = self.coerce_address(address, resolve_address)
-            address = normalize_address(address)
-            host = get_address_host(address)
+            _remove_from_processing(self, ts)
 
-            ws: WorkerState = self.workers.get(address)
-            if ws is not None:
-                raise ValueError("Worker already exists %s" % ws)
+            _add_to_memory(
+                self, ts, ws, recommendations, client_msgs, type=type, typename=typename
+            )
 
-            if name in self.aliases:
-                logger.warning(
-                    "Worker tried to connect with a duplicate name: %s", name
-                )
-                msg = {
-                    "status": "error",
-                    "message": "name taken, %s" % name,
-                    "time": time(),
-                }
-                if comm:
-                    await comm.write(msg)
-                return
+            if self._validate:
+                assert not ts._processing_on
+                assert not ts._waiting_on
 
-            self.workers[address] = ws = WorkerState(
-                address=address,
-                pid=pid,
-                nthreads=nthreads,
-                memory_limit=memory_limit or 0,
-                name=name,
-                local_directory=local_directory,
-                services=services,
-                versions=versions,
-                nanny=nanny,
-                extra=extra,
-            )
+            return recommendations, worker_msgs, client_msgs
+        except Exception as e:
+            logger.exception(e)
+            if LOG_PDB:
+                import pdb
 
-            if "addresses" not in self.host_info[host]:
-                self.host_info[host].update({"addresses": set(), "nthreads": 0})
+                pdb.set_trace()
+            raise
 
-            self.host_info[host]["addresses"].add(address)
-            self.host_info[host]["nthreads"] += nthreads
+    def transition_memory_released(self, key, safe: bint = False):
+        ws: WorkerState
+        try:
+            ts: TaskState = self._tasks[key]
+            dts: TaskState
+            worker_msgs: dict = {}
+            client_msgs: dict = {}
 
-            self.total_nthreads += nthreads
-            self.aliases[name] = address
+            if self._validate:
+                assert not ts._waiting_on
+                assert not ts._processing_on
+                if safe:
+                    assert not ts._waiters
 
-            response = self.heartbeat_worker(
-                address=address,
-                resolve_address=resolve_address,
-                now=now,
-                resources=resources,
-                host_info=host_info,
-                metrics=metrics,
-            )
+            if ts._actor:
+                for ws in ts._who_has:
+                    ws._actors.discard(ts)
+                if ts._who_wants:
+                    ts._exception_blame = ts
+                    ts._exception = "Worker holding Actor was lost"
+                    return (
+                        {ts._key: "erred"},
+                        worker_msgs,
+                        client_msgs,
+                    )  # don't try to recreate
 
-            # Do not need to adjust self.total_occupancy as self.occupancy[ws] cannot exist before this.
-            self.check_idle_saturated(ws)
+            recommendations: dict = {}
 
-            # for key in keys:  # TODO
-            #     self.mark_key_in_memory(key, [address])
+            for dts in ts._waiters:
+                if dts._state in ("no-worker", "processing"):
+                    recommendations[dts._key] = "waiting"
+                elif dts._state == "waiting":
+                    dts._waiting_on.add(ts)
 
-            self.stream_comms[address] = BatchedSend(interval="5ms", loop=self.loop)
+            # XXX factor this out?
+            for ws in ts._who_has:
+                ws._has_what.remove(ts)
+                ws._nbytes -= ts.get_nbytes()
+                ts._group._nbytes_in_memory -= ts.get_nbytes()
+                worker_msgs[ws._address] = {
+                    "op": "delete-data",
+                    "keys": [key],
+                    "report": False,
+                }
 
-            if ws._nthreads > len(ws._processing):
-                self.idle[ws._address] = ws
+            ts._who_has.clear()
 
-            for plugin in self.plugins[:]:
-                try:
-                    result = plugin.add_worker(scheduler=self, worker=address)
-                    if inspect.isawaitable(result):
-                        await result
-                except Exception as e:
-                    logger.exception(e)
+            ts.state = "released"
 
-            recommendations: dict
-            if nbytes:
-                for key in nbytes:
-                    tasks: dict = self.tasks
-                    ts: TaskState = tasks.get(key)
-                    if ts is not None and ts._state in ("processing", "waiting"):
-                        recommendations = self.transition(
-                            key,
-                            "memory",
-                            worker=address,
-                            nbytes=nbytes[key],
-                            typename=types[key],
-                        )
-                        self.transitions(recommendations)
+            report_msg = {"op": "lost-data", "key": key}
+            cs: ClientState
+            for cs in ts._who_wants:
+                client_msgs[cs._client_key] = report_msg
 
-            recommendations = {}
-            for ts in list(self.unrunnable):
-                valid: set = self.valid_workers(ts)
-                if valid is None or ws in valid:
-                    recommendations[ts._key] = "waiting"
+            if not ts._run_spec:  # pure data
+                recommendations[key] = "forgotten"
+            elif ts._has_lost_dependencies:
+                recommendations[key] = "forgotten"
+            elif ts._who_wants or ts._waiters:
+                recommendations[key] = "waiting"
 
-            if recommendations:
-                self.transitions(recommendations)
+            if self._validate:
+                assert not ts._waiting_on
 
-            self.log_event(address, {"action": "add-worker"})
-            self.log_event("all", {"action": "add-worker", "worker": address})
-            logger.info("Register worker %s", ws)
+            return recommendations, worker_msgs, client_msgs
+        except Exception as e:
+            logger.exception(e)
+            if LOG_PDB:
+                import pdb
 
-            msg = {
-                "status": "OK",
-                "time": time(),
-                "heartbeat-interval": heartbeat_interval(len(self.workers)),
-                "worker-plugins": self.worker_plugins,
-            }
+                pdb.set_trace()
+            raise
 
-            cs: ClientState
-            version_warning = version_module.error_message(
-                version_module.get_versions(),
-                merge(
-                    {w: ws._versions for w, ws in self.workers.items()},
-                    {c: cs._versions for c, cs in self.clients.items() if cs._versions},
-                ),
-                versions,
-                client_name="This Worker",
-            )
-            msg.update(version_warning)
+    def transition_released_erred(self, key):
+        try:
+            ts: TaskState = self._tasks[key]
+            dts: TaskState
+            failing_ts: TaskState
+            worker_msgs: dict = {}
+            client_msgs: dict = {}
 
-            if comm:
-                await comm.write(msg)
-            await self.handle_worker(comm=comm, worker=address)
+            if self._validate:
+                with log_errors(pdb=LOG_PDB):
+                    assert ts._exception_blame
+                    assert not ts._who_has
+                    assert not ts._waiting_on
+                    assert not ts._waiters
 
-    def update_graph_hlg(
-        self,
-        client=None,
-        hlg=None,
-        keys=None,
-        dependencies=None,
-        restrictions=None,
-        priority=None,
-        loose_restrictions=None,
-        resources=None,
-        submitting_task=None,
-        retries=None,
-        user_priority=0,
-        actors=None,
-        fifo_timeout=0,
-    ):
+            recommendations: dict = {}
 
-        dsk, dependencies, annotations = highlevelgraph_unpack(hlg)
+            failing_ts = ts._exception_blame
 
-        # Remove any self-dependencies (happens on test_publish_bag() and others)
-        for k, v in dependencies.items():
-            deps = set(v)
-            if k in deps:
-                deps.remove(k)
-            dependencies[k] = deps
+            for dts in ts._dependents:
+                dts._exception_blame = failing_ts
+                if not dts._who_has:
+                    recommendations[dts._key] = "erred"
 
-        if priority is None:
-            # Removing all non-local keys before calling order()
-            dsk_keys = set(dsk)  # intersection() of sets is much faster than dict_keys
-            stripped_deps = {
-                k: v.intersection(dsk_keys)
-                for k, v in dependencies.items()
-                if k in dsk_keys
+            report_msg = {
+                "op": "task-erred",
+                "key": key,
+                "exception": failing_ts._exception,
+                "traceback": failing_ts._traceback,
             }
-            priority = dask.order.order(dsk, dependencies=stripped_deps)
+            cs: ClientState
+            for cs in ts._who_wants:
+                client_msgs[cs._client_key] = report_msg
 
-        return self.update_graph(
-            client,
-            dsk,
-            keys,
-            dependencies,
-            restrictions,
-            priority,
-            loose_restrictions,
-            resources,
-            submitting_task,
-            retries,
-            user_priority,
-            actors,
-            fifo_timeout,
-            annotations,
-        )
+            ts.state = "erred"
 
-    def update_graph(
-        self,
-        client=None,
-        tasks=None,
-        keys=None,
-        dependencies=None,
-        restrictions=None,
-        priority=None,
-        loose_restrictions=None,
-        resources=None,
-        submitting_task=None,
-        retries=None,
-        user_priority=0,
-        actors=None,
-        fifo_timeout=0,
-        annotations=None,
-    ):
-        """
-        Add new computations to the internal dask graph
+            # TODO: waiting data?
+            return recommendations, worker_msgs, client_msgs
+        except Exception as e:
+            logger.exception(e)
+            if LOG_PDB:
+                import pdb
 
-        This happens whenever the Client calls submit, map, get, or compute.
-        """
-        start = time()
-        fifo_timeout = parse_timedelta(fifo_timeout)
-        keys = set(keys)
-        if len(tasks) > 1:
-            self.log_event(
-                ["all", client], {"action": "update_graph", "count": len(tasks)}
-            )
+                pdb.set_trace()
+            raise
 
-        # Remove aliases
-        for k in list(tasks):
-            if tasks[k] is k:
-                del tasks[k]
+    def transition_erred_released(self, key):
+        try:
+            ts: TaskState = self._tasks[key]
+            dts: TaskState
+            worker_msgs: dict = {}
+            client_msgs: dict = {}
 
-        dependencies = dependencies or {}
+            if self._validate:
+                with log_errors(pdb=LOG_PDB):
+                    assert all([dts._state != "erred" for dts in ts._dependencies])
+                    assert ts._exception_blame
+                    assert not ts._who_has
+                    assert not ts._waiting_on
+                    assert not ts._waiters
 
-        n = 0
-        while len(tasks) != n:  # walk through new tasks, cancel any bad deps
-            n = len(tasks)
-            for k, deps in list(dependencies.items()):
-                if any(
-                    dep not in self.tasks and dep not in tasks for dep in deps
-                ):  # bad key
-                    logger.info("User asked for computation on lost data, %s", k)
-                    del tasks[k]
-                    del dependencies[k]
-                    if k in keys:
-                        keys.remove(k)
-                    self.report({"op": "cancelled-key", "key": k}, client=client)
-                    self.client_releases_keys(keys=[k], client=client)
+            recommendations: dict = {}
 
-        # Avoid computation that is already finished
-        ts: TaskState
-        already_in_memory = set()  # tasks that are already done
-        for k, v in dependencies.items():
-            if v and k in self.tasks:
-                ts = self.tasks[k]
-                if ts._state in ("memory", "erred"):
-                    already_in_memory.add(k)
+            ts._exception = None
+            ts._exception_blame = None
+            ts._traceback = None
 
-        dts: TaskState
-        if already_in_memory:
-            dependents = dask.core.reverse_dict(dependencies)
-            stack = list(already_in_memory)
-            done = set(already_in_memory)
-            while stack:  # remove unnecessary dependencies
-                key = stack.pop()
-                ts = self.tasks[key]
-                try:
-                    deps = dependencies[key]
-                except KeyError:
-                    deps = self.dependencies[key]
-                for dep in deps:
-                    if dep in dependents:
-                        child_deps = dependents[dep]
-                    else:
-                        child_deps = self.dependencies[dep]
-                    if all(d in done for d in child_deps):
-                        if dep in self.tasks and dep not in done:
-                            done.add(dep)
-                            stack.append(dep)
+            for dts in ts._dependents:
+                if dts._state == "erred":
+                    recommendations[dts._key] = "waiting"
 
-            for d in done:
-                tasks.pop(d, None)
-                dependencies.pop(d, None)
+            report_msg = {"op": "task-retried", "key": key}
+            cs: ClientState
+            for cs in ts._who_wants:
+                client_msgs[cs._client_key] = report_msg
 
-        # Get or create task states
-        stack = list(keys)
-        touched_keys = set()
-        touched_tasks = []
-        while stack:
-            k = stack.pop()
-            if k in touched_keys:
-                continue
-            # XXX Have a method get_task_state(self, k) ?
-            ts = self.tasks.get(k)
-            if ts is None:
-                ts = self.new_task(k, tasks.get(k), "released")
-            elif not ts._run_spec:
-                ts._run_spec = tasks.get(k)
+            ts.state = "released"
 
-            touched_keys.add(k)
-            touched_tasks.append(ts)
-            stack.extend(dependencies.get(k, ()))
+            return recommendations, worker_msgs, client_msgs
+        except Exception as e:
+            logger.exception(e)
+            if LOG_PDB:
+                import pdb
 
-        self.client_desires_keys(keys=keys, client=client)
+                pdb.set_trace()
+            raise
 
-        # Add dependencies
-        for key, deps in dependencies.items():
-            ts = self.tasks.get(key)
-            if ts is None or ts._dependencies:
-                continue
-            for dep in deps:
-                dts = self.tasks[dep]
-                ts.add_dependency(dts)
+    def transition_waiting_released(self, key):
+        try:
+            ts: TaskState = self._tasks[key]
+            worker_msgs: dict = {}
+            client_msgs: dict = {}
 
-        # Compute priorities
-        if isinstance(user_priority, Number):
-            user_priority = {k: user_priority for k in tasks}
+            if self._validate:
+                assert not ts._who_has
+                assert not ts._processing_on
 
-        annotations = annotations or {}
-        restrictions = restrictions or {}
-        loose_restrictions = loose_restrictions or []
-        resources = resources or {}
-        retries = retries or {}
+            recommendations: dict = {}
 
-        # Override existing taxonomy with per task annotations
-        if annotations:
-            if "priority" in annotations:
-                user_priority.update(annotations["priority"])
+            dts: TaskState
+            for dts in ts._dependencies:
+                s = dts._waiters
+                if ts in s:
+                    s.discard(ts)
+                    if not s and not dts._who_wants:
+                        recommendations[dts._key] = "released"
+            ts._waiting_on.clear()
 
-            if "workers" in annotations:
-                restrictions.update(annotations["workers"])
+            ts.state = "released"
 
-            if "allow_other_workers" in annotations:
-                loose_restrictions.extend(
-                    k for k, v in annotations["allow_other_workers"].items() if v
-                )
+            if ts._has_lost_dependencies:
+                recommendations[key] = "forgotten"
+            elif not ts._exception_blame and (ts._who_wants or ts._waiters):
+                recommendations[key] = "waiting"
+            else:
+                ts._waiters.clear()
 
-            if "retries" in annotations:
-                retries.update(annotations["retries"])
+            return recommendations, worker_msgs, client_msgs
+        except Exception as e:
+            logger.exception(e)
+            if LOG_PDB:
+                import pdb
 
-            if "resources" in annotations:
-                resources.update(annotations["resources"])
+                pdb.set_trace()
+            raise
 
-            for a, kv in annotations.items():
-                for k, v in kv.items():
-                    ts = self.tasks[k]
-                    ts._annotations[a] = v
+    def transition_processing_released(self, key):
+        try:
+            ts: TaskState = self._tasks[key]
+            dts: TaskState
+            worker_msgs: dict = {}
+            client_msgs: dict = {}
 
-        # Add actors
-        if actors is True:
-            actors = list(keys)
-        for actor in actors or []:
-            ts = self.tasks[actor]
-            ts._actor = True
+            if self._validate:
+                assert ts._processing_on
+                assert not ts._who_has
+                assert not ts._waiting_on
+                assert self._tasks[key].state == "processing"
 
-        priority = priority or dask.order.order(
-            tasks
-        )  # TODO: define order wrt old graph
+            w: str = _remove_from_processing(self, ts)
+            if w:
+                worker_msgs[w] = {"op": "release-task", "key": key}
 
-        if submitting_task:  # sub-tasks get better priority than parent tasks
-            ts = self.tasks.get(submitting_task)
-            if ts is not None:
-                generation = ts._priority[0] - 0.01
-            else:  # super-task already cleaned up
-                generation = self.generation
-        elif self._last_time + fifo_timeout < start:
-            self.generation += 1  # older graph generations take precedence
-            generation = self.generation
-            self._last_time = start
-        else:
-            generation = self.generation
+            ts.state = "released"
 
-        for key in set(priority) & touched_keys:
-            ts = self.tasks[key]
-            if ts._priority is None:
-                ts._priority = (-(user_priority.get(key, 0)), generation, priority[key])
+            recommendations: dict = {}
 
-        # Ensure all runnables have a priority
-        runnables = [ts for ts in touched_tasks if ts._run_spec]
-        for ts in runnables:
-            if ts._priority is None and ts._run_spec:
-                ts._priority = (self.generation, 0)
+            if ts._has_lost_dependencies:
+                recommendations[key] = "forgotten"
+            elif ts._waiters or ts._who_wants:
+                recommendations[key] = "waiting"
 
-        if restrictions:
-            # *restrictions* is a dict keying task ids to lists of
-            # restriction specifications (either worker names or addresses)
-            for k, v in restrictions.items():
-                if v is None:
-                    continue
-                ts = self.tasks.get(k)
-                if ts is None:
-                    continue
-                ts._host_restrictions = set()
-                ts._worker_restrictions = set()
-                for w in v:
-                    try:
-                        w = self.coerce_address(w)
-                    except ValueError:
-                        # Not a valid address, but perhaps it's a hostname
-                        ts._host_restrictions.add(w)
-                    else:
-                        ts._worker_restrictions.add(w)
+            if recommendations.get(key) != "waiting":
+                for dts in ts._dependencies:
+                    if dts._state != "released":
+                        s = dts._waiters
+                        s.discard(ts)
+                        if not s and not dts._who_wants:
+                            recommendations[dts._key] = "released"
+                ts._waiters.clear()
 
-            if loose_restrictions:
-                for k in loose_restrictions:
-                    ts = self.tasks[k]
-                    ts._loose_restrictions = True
+            if self._validate:
+                assert not ts._processing_on
 
-        if resources:
-            for k, v in resources.items():
-                if v is None:
-                    continue
-                assert isinstance(v, dict)
-                ts = self.tasks.get(k)
-                if ts is None:
-                    continue
-                ts._resource_restrictions = v
+            return recommendations, worker_msgs, client_msgs
+        except Exception as e:
+            logger.exception(e)
+            if LOG_PDB:
+                import pdb
 
-        if retries:
-            for k, v in retries.items():
-                assert isinstance(v, int)
-                ts = self.tasks.get(k)
-                if ts is None:
-                    continue
-                ts._retries = v
+                pdb.set_trace()
+            raise
 
-        # Compute recommendations
-        recommendations: dict = {}
+    def transition_processing_erred(
+        self, key, cause=None, exception=None, traceback=None, **kwargs
+    ):
+        ws: WorkerState
+        try:
+            ts: TaskState = self._tasks[key]
+            dts: TaskState
+            failing_ts: TaskState
+            worker_msgs: dict = {}
+            client_msgs: dict = {}
 
-        for ts in sorted(runnables, key=operator.attrgetter("priority"), reverse=True):
-            if ts._state == "released" and ts._run_spec:
-                recommendations[ts._key] = "waiting"
+            if self._validate:
+                assert cause or ts._exception_blame
+                assert ts._processing_on
+                assert not ts._who_has
+                assert not ts._waiting_on
 
-        for ts in touched_tasks:
-            for dts in ts._dependencies:
-                if dts._exception_blame:
-                    ts._exception_blame = dts._exception_blame
-                    recommendations[ts._key] = "erred"
-                    break
+            if ts._actor:
+                ws = ts._processing_on
+                ws._actors.remove(ts)
 
-        for plugin in self.plugins[:]:
-            try:
-                plugin.update_graph(
-                    self,
-                    client=client,
-                    tasks=tasks,
-                    keys=keys,
-                    restrictions=restrictions or {},
-                    dependencies=dependencies,
-                    priority=priority,
-                    loose_restrictions=loose_restrictions,
-                    resources=resources,
-                    annotations=annotations,
-                )
-            except Exception as e:
-                logger.exception(e)
+            _remove_from_processing(self, ts)
 
-        self.transitions(recommendations)
+            if exception is not None:
+                ts._exception = exception
+            if traceback is not None:
+                ts._traceback = traceback
+            if cause is not None:
+                failing_ts = self._tasks[cause]
+                ts._exception_blame = failing_ts
+            else:
+                failing_ts = ts._exception_blame
 
-        for ts in touched_tasks:
-            if ts._state in ("memory", "erred"):
-                self.report_on_key(ts=ts, client=client)
+            recommendations: dict = {}
 
-        end = time()
-        if self.digests is not None:
-            self.digests["update-graph-duration"].add(end - start)
+            for dts in ts._dependents:
+                dts._exception_blame = failing_ts
+                recommendations[dts._key] = "erred"
 
-        # TODO: balance workers
+            for dts in ts._dependencies:
+                s = dts._waiters
+                s.discard(ts)
+                if not s and not dts._who_wants:
+                    recommendations[dts._key] = "released"
 
-    def new_task(self, key, spec, state):
-        """ Create a new task, and associated states """
-        ts: TaskState = TaskState(key, spec)
-        tp: TaskPrefix
-        tg: TaskGroup
-        ts._state = state
-        prefix_key = key_split(key)
-        try:
-            tp = self.task_prefixes[prefix_key]
-        except KeyError:
-            self.task_prefixes[prefix_key] = tp = TaskPrefix(prefix_key)
-        ts._prefix = tp
+            ts._waiters.clear()  # do anything with this?
 
-        group_key = ts._group_key
-        try:
-            tg = self.task_groups[group_key]
-        except KeyError:
-            self.task_groups[group_key] = tg = TaskGroup(group_key)
-            tg._prefix = tp
-            tp._groups.append(tg)
-        tg.add(ts)
-        self.tasks[key] = ts
-        return ts
+            ts.state = "erred"
 
-    def stimulus_task_finished(self, key=None, worker=None, **kwargs):
-        """ Mark that a task has finished execution on a particular worker """
-        logger.debug("Stimulus task finished %s, %s", key, worker)
+            report_msg = {
+                "op": "task-erred",
+                "key": key,
+                "exception": failing_ts._exception,
+                "traceback": failing_ts._traceback,
+            }
+            cs: ClientState
+            for cs in ts._who_wants:
+                client_msgs[cs._client_key] = report_msg
 
-        tasks: dict = self.tasks
-        ts: TaskState = tasks.get(key)
-        if ts is None:
-            return {}
-        workers: dict = cast(dict, self.workers)
-        ws: WorkerState = workers[worker]
-        ts._metadata.update(kwargs["metadata"])
+            cs = self._clients["fire-and-forget"]
+            if ts in cs._wants_what:
+                _client_releases_keys(
+                    self,
+                    cs=cs,
+                    keys=[key],
+                    recommendations=recommendations,
+                )
 
-        recommendations: dict
-        if ts._state == "processing":
-            recommendations = self.transition(key, "memory", worker=worker, **kwargs)
+            if self._validate:
+                assert not ts._processing_on
 
-            if ts._state == "memory":
-                assert ws in ts._who_has
-        else:
-            logger.debug(
-                "Received already computed task, worker: %s, state: %s"
-                ", key: %s, who_has: %s",
-                worker,
-                ts._state,
-                key,
-                ts._who_has,
-            )
-            if ws not in ts._who_has:
-                self.worker_send(worker, {"op": "release-task", "key": key})
-            recommendations = {}
+            return recommendations, worker_msgs, client_msgs
+        except Exception as e:
+            logger.exception(e)
+            if LOG_PDB:
+                import pdb
 
-        return recommendations
+                pdb.set_trace()
+            raise
 
-    def stimulus_task_erred(
-        self, key=None, worker=None, exception=None, traceback=None, **kwargs
-    ):
-        """ Mark that a task has erred on a particular worker """
-        logger.debug("Stimulus task erred %s, %s", key, worker)
+    def transition_no_worker_released(self, key):
+        try:
+            ts: TaskState = self._tasks[key]
+            dts: TaskState
+            worker_msgs: dict = {}
+            client_msgs: dict = {}
 
-        ts: TaskState = self.tasks.get(key)
-        if ts is None:
-            return {}
+            if self._validate:
+                assert self._tasks[key].state == "no-worker"
+                assert not ts._who_has
+                assert not ts._waiting_on
 
-        recommendations: dict
-        if ts._state == "processing":
-            retries = ts._retries
-            if retries > 0:
-                ts._retries = retries - 1
-                recommendations = self.transition(key, "waiting")
-            else:
-                recommendations = self.transition(
-                    key,
-                    "erred",
-                    cause=key,
-                    exception=exception,
-                    traceback=traceback,
-                    worker=worker,
-                    **kwargs,
-                )
-        else:
-            recommendations = {}
+            self._unrunnable.remove(ts)
+            ts.state = "released"
 
-        return recommendations
+            for dts in ts._dependencies:
+                dts._waiters.discard(ts)
 
-    def stimulus_missing_data(
-        self, cause=None, key=None, worker=None, ensure=True, **kwargs
-    ):
-        """ Mark that certain keys have gone missing.  Recover. """
-        with log_errors():
-            logger.debug("Stimulus missing data %s, %s", key, worker)
+            ts._waiters.clear()
 
-            ts: TaskState = self.tasks.get(key)
-            if ts is None or ts._state == "memory":
-                return {}
-            cts: TaskState = self.tasks.get(cause)
+            return {}, worker_msgs, client_msgs
+        except Exception as e:
+            logger.exception(e)
+            if LOG_PDB:
+                import pdb
 
-            recommendations: dict = {}
+                pdb.set_trace()
+            raise
 
-            if cts is not None and cts._state == "memory":  # couldn't find this
-                ws: WorkerState
-                for ws in cts._who_has:  # TODO: this behavior is extreme
-                    ws._has_what.remove(cts)
-                    ws._nbytes -= cts.get_nbytes()
-                cts._who_has.clear()
-                recommendations[cause] = "released"
+    @ccall
+    def remove_key(self, key):
+        ts: TaskState = self._tasks.pop(key)
+        assert ts._state == "forgotten"
+        self._unrunnable.discard(ts)
+        cs: ClientState
+        for cs in ts._who_wants:
+            cs._wants_what.remove(ts)
+        ts._who_wants.clear()
+        ts._processing_on = None
+        ts._exception_blame = ts._exception = ts._traceback = None
+        self._task_metadata.pop(key, None)
 
-            if key:
-                recommendations[key] = "released"
+    def transition_memory_forgotten(self, key):
+        ws: WorkerState
+        try:
+            ts: TaskState = self._tasks[key]
+            worker_msgs: dict = {}
+            client_msgs: dict = {}
 
-            self.transitions(recommendations)
+            if self._validate:
+                assert ts._state == "memory"
+                assert not ts._processing_on
+                assert not ts._waiting_on
+                if not ts._run_spec:
+                    # It's ok to forget a pure data task
+                    pass
+                elif ts._has_lost_dependencies:
+                    # It's ok to forget a task with forgotten dependencies
+                    pass
+                elif not ts._who_wants and not ts._waiters and not ts._dependents:
+                    # It's ok to forget a task that nobody needs
+                    pass
+                else:
+                    assert 0, (ts,)
 
-            if self.validate:
-                assert cause not in self.who_has
+            recommendations: dict = {}
 
-            return {}
+            if ts._actor:
+                for ws in ts._who_has:
+                    ws._actors.discard(ts)
 
-    def stimulus_retry(self, comm=None, keys=None, client=None):
-        logger.info("Client %s requests to retry %d keys", client, len(keys))
-        if client:
-            self.log_event(client, {"action": "retry", "count": len(keys)})
+            _propagate_forgotten(self, ts, recommendations, worker_msgs)
 
-        stack = list(keys)
-        seen = set()
-        roots = []
-        ts: TaskState
-        dts: TaskState
-        while stack:
-            key = stack.pop()
-            seen.add(key)
-            ts = self.tasks[key]
-            erred_deps = [dts._key for dts in ts._dependencies if dts._state == "erred"]
-            if erred_deps:
-                stack.extend(erred_deps)
-            else:
-                roots.append(key)
+            client_msgs = _task_to_client_msgs(self, ts)
+            self.remove_key(key)
 
-        recommendations: dict = {key: "waiting" for key in roots}
-        self.transitions(recommendations)
+            return recommendations, worker_msgs, client_msgs
+        except Exception as e:
+            logger.exception(e)
+            if LOG_PDB:
+                import pdb
 
-        if self.validate:
-            for key in seen:
-                assert not self.tasks[key].exception_blame
+                pdb.set_trace()
+            raise
 
-        return tuple(seen)
+    def transition_released_forgotten(self, key):
+        try:
+            ts: TaskState = self._tasks[key]
+            worker_msgs: dict = {}
+            client_msgs: dict = {}
 
-    async def remove_worker(self, comm=None, address=None, safe=False, close=True):
-        """
-        Remove worker from cluster
+            if self._validate:
+                assert ts._state in ("released", "erred")
+                assert not ts._who_has
+                assert not ts._processing_on
+                assert not ts._waiting_on, (ts, ts._waiting_on)
+                if not ts._run_spec:
+                    # It's ok to forget a pure data task
+                    pass
+                elif ts._has_lost_dependencies:
+                    # It's ok to forget a task with forgotten dependencies
+                    pass
+                elif not ts._who_wants and not ts._waiters and not ts._dependents:
+                    # It's ok to forget a task that nobody needs
+                    pass
+                else:
+                    assert 0, (ts,)
 
-        We do this when a worker reports that it plans to leave or when it
-        appears to be unresponsive.  This may send its tasks back to a released
-        state.
-        """
-        with log_errors():
-            if self.status == Status.closed:
-                return
+            recommendations: dict = {}
+            _propagate_forgotten(self, ts, recommendations, worker_msgs)
 
-            address = self.coerce_address(address)
+            client_msgs = _task_to_client_msgs(self, ts)
+            self.remove_key(key)
 
-            if address not in self.workers:
-                return "already-removed"
+            return recommendations, worker_msgs, client_msgs
+        except Exception as e:
+            logger.exception(e)
+            if LOG_PDB:
+                import pdb
 
-            host = get_address_host(address)
+                pdb.set_trace()
+            raise
 
-            ws: WorkerState = self.workers[address]
+    ##############################
+    # Assigning Tasks to Workers #
+    ##############################
 
-            self.log_event(
-                ["all", address],
-                {
-                    "action": "remove-worker",
-                    "worker": address,
-                    "processing-tasks": dict(ws._processing),
-                },
-            )
-            logger.info("Remove worker %s", ws)
-            if close:
-                with suppress(AttributeError, CommClosedError):
-                    self.stream_comms[address].send({"op": "close", "report": False})
+    @ccall
+    @exceptval(check=False)
+    def check_idle_saturated(self, ws: WorkerState, occ: double = -1.0):
+        """Update the status of the idle and saturated state
 
-            self.remove_resources(address)
+        The scheduler keeps track of workers that are ..
 
-            self.host_info[host]["nthreads"] -= ws._nthreads
-            self.host_info[host]["addresses"].remove(address)
-            self.total_nthreads -= ws._nthreads
+        -  Saturated: have enough work to stay busy
+        -  Idle: do not have enough work to stay busy
 
-            if not self.host_info[host]["addresses"]:
-                del self.host_info[host]
+        They are considered saturated if they both have enough tasks to occupy
+        all of their threads, and if the expected runtime of those tasks is
+        large enough.
 
-            self.rpc.remove(address)
-            del self.stream_comms[address]
-            del self.aliases[ws._name]
-            self.idle.pop(ws._address, None)
-            self.saturated.discard(ws)
-            del self.workers[address]
-            ws.status = Status.closed
-            self.total_occupancy -= ws._occupancy
+        This is useful for load balancing and adaptivity.
+        """
+        total_nthreads: Py_ssize_t = self._total_nthreads
+        if total_nthreads == 0 or ws.status == Status.closed:
+            return
+        if occ < 0:
+            occ = ws._occupancy
 
-            recommendations: dict = {}
+        nc: Py_ssize_t = ws._nthreads
+        p: Py_ssize_t = len(ws._processing)
+        total_occupancy: double = self._total_occupancy
+        avg: double = total_occupancy / total_nthreads
 
-            ts: TaskState
-            for ts in list(ws._processing):
-                k = ts._key
-                recommendations[k] = "released"
-                if not safe:
-                    ts._suspicious += 1
-                    ts._prefix._suspicious += 1
-                    if ts._suspicious > self.allowed_failures:
-                        del recommendations[k]
-                        e = pickle.dumps(
-                            KilledWorker(task=k, last_worker=ws.clean()), protocol=4
-                        )
-                        r = self.transition(k, "erred", exception=e, cause=k)
-                        recommendations.update(r)
-                        logger.info(
-                            "Task %s marked as failed because %d workers died"
-                            " while trying to run it",
-                            ts._key,
-                            self.allowed_failures,
-                        )
+        idle = self._idle
+        saturated: set = self._saturated
+        if p < nc or occ < nc * avg / 2:
+            idle[ws._address] = ws
+            saturated.discard(ws)
+        else:
+            idle.pop(ws._address, None)
 
-            for ts in ws._has_what:
-                ts._who_has.remove(ws)
-                if not ts._who_has:
-                    if ts._run_spec:
-                        recommendations[ts._key] = "released"
-                    else:  # pure data
-                        recommendations[ts._key] = "forgotten"
-            ws._has_what.clear()
+            if p > nc:
+                pending: double = occ * (p - nc) / (p * nc)
+                if 0.4 < pending > 1.9 * avg:
+                    saturated.add(ws)
+                    return
 
-            self.transitions(recommendations)
+            saturated.discard(ws)
 
-            for plugin in self.plugins[:]:
-                try:
-                    result = plugin.remove_worker(scheduler=self, worker=address)
-                    if inspect.isawaitable(result):
-                        await result
-                except Exception as e:
-                    logger.exception(e)
+    @ccall
+    def get_comm_cost(self, ts: TaskState, ws: WorkerState) -> double:
+        """
+        Get the estimated communication cost (in s.) to compute the task
+        on the given worker.
+        """
+        dts: TaskState
+        deps: set = ts._dependencies - ws._has_what
+        nbytes: Py_ssize_t = 0
+        bandwidth: double = self._bandwidth
+        for dts in deps:
+            nbytes += dts._nbytes
+        return nbytes / bandwidth
 
-            if not self.workers:
-                logger.info("Lost all workers")
+    @ccall
+    def get_task_duration(self, ts: TaskState, default: double = -1) -> double:
+        """
+        Get the estimated computation cost of the given task
+        (not including any communication cost).
+        """
+        duration: double = ts._prefix._duration_average
+        if duration < 0:
+            s: set = self._unknown_durations[ts._prefix._name]
+            s.add(ts)
+            if default < 0:
+                duration = UNKNOWN_TASK_DURATION
+            else:
+                duration = default
 
-            for w in self.workers:
-                self.bandwidth_workers.pop((address, w), None)
-                self.bandwidth_workers.pop((w, address), None)
+        return duration
 
-            def remove_worker_from_events():
-                # If the worker isn't registered anymore after the delay, remove from events
-                if address not in self.workers and address in self.events:
-                    del self.events[address]
+    @ccall
+    @exceptval(check=False)
+    def valid_workers(self, ts: TaskState) -> set:
+        """Return set of currently valid workers for key
 
-            cleanup_delay = parse_timedelta(
-                dask.config.get("distributed.scheduler.events-cleanup-delay")
-            )
-            self.loop.call_later(cleanup_delay, remove_worker_from_events)
-            logger.debug("Removed worker %s", ws)
+        If all workers are valid then this returns ``None``.
+        This checks tracks the following state:
 
-        return "OK"
+        *  worker_restrictions
+        *  host_restrictions
+        *  resource_restrictions
+        """
+        s: set = None
 
-    def stimulus_cancel(self, comm, keys=None, client=None, force=False):
-        """ Stop execution on a list of keys """
-        logger.info("Client %s requests to cancel %d keys", client, len(keys))
-        if client:
-            self.log_event(
-                client, {"action": "cancel", "count": len(keys), "force": force}
-            )
-        for key in keys:
-            self.cancel_key(key, client, force=force)
+        if ts._worker_restrictions:
+            s = {w for w in ts._worker_restrictions if w in self._workers_dv}
 
-    def cancel_key(self, key, client, retries=5, force=False):
-        """ Cancel a particular key and all dependents """
-        # TODO: this should be converted to use the transition mechanism
-        ts: TaskState = self.tasks.get(key)
-        dts: TaskState
-        try:
-            cs: ClientState = self.clients[client]
-        except KeyError:
-            return
-        if ts is None or not ts._who_wants:  # no key yet, lets try again in a moment
-            if retries:
-                self.loop.call_later(
-                    0.2, lambda: self.cancel_key(key, client, retries - 1)
-                )
-            return
-        if force or ts._who_wants == {cs}:  # no one else wants this key
-            for dts in list(ts._dependents):
-                self.cancel_key(dts._key, client, force=force)
-        logger.info("Scheduler cancels key %s.  Force=%s", key, force)
-        self.report({"op": "cancelled-key", "key": key})
-        clients = list(ts._who_wants) if force else [cs]
-        for cs in clients:
-            self.client_releases_keys(keys=[key], client=cs._client_key)
+        if ts._host_restrictions:
+            # Resolve the alias here rather than early, for the worker
+            # may not be connected when host_restrictions is populated
+            hr: list = [self.coerce_hostname(h) for h in ts._host_restrictions]
+            # XXX need HostState?
+            sl: list = [
+                self._host_info[h]["addresses"] for h in hr if h in self._host_info
+            ]
+            ss: set = set.union(*sl) if sl else set()
+            if s is None:
+                s = ss
+            else:
+                s |= ss
 
-    def client_desires_keys(self, keys=None, client=None):
-        cs: ClientState = self.clients.get(client)
-        if cs is None:
-            # For publish, queues etc.
-            self.clients[client] = cs = ClientState(client)
-        ts: TaskState
-        for k in keys:
-            ts = self.tasks.get(k)
-            if ts is None:
-                # For publish, queues etc.
-                ts = self.new_task(k, None, "released")
-            ts._who_wants.add(cs)
-            cs._wants_what.add(ts)
+        if ts._resource_restrictions:
+            dw: dict = {
+                resource: {
+                    w
+                    for w, supplied in self._resources[resource].items()
+                    if supplied >= required
+                }
+                for resource, required in ts._resource_restrictions.items()
+            }
 
-            if ts._state in ("memory", "erred"):
-                self.report_on_key(ts=ts, client=client)
+            ww: set = set.intersection(*dw.values())
+            if s is None:
+                s = ww
+            else:
+                s &= ww
 
-    def client_releases_keys(self, keys=None, client=None):
-        """ Remove keys from client desired list """
-        logger.debug("Client %s releases keys: %s", client, keys)
-        cs: ClientState = self.clients[client]
-        ts: TaskState
-        tasks2 = set()
-        for key in list(keys):
-            ts = self.tasks.get(key)
-            if ts is not None and ts in cs._wants_what:
-                cs._wants_what.remove(ts)
-                s = ts._who_wants
-                s.remove(cs)
-                if not s:
-                    tasks2.add(ts)
+        if s is not None:
+            s = {self._workers_dv[w] for w in s}
 
-        recommendations: dict = {}
-        for ts in tasks2:
-            if not ts._dependents:
-                # No live dependents, can forget
-                recommendations[ts._key] = "forgotten"
-            elif ts._state != "erred" and not ts._waiters:
-                recommendations[ts._key] = "released"
+        return s
 
-        self.transitions(recommendations)
+    @ccall
+    def consume_resources(self, ts: TaskState, ws: WorkerState):
+        if ts._resource_restrictions:
+            for r, required in ts._resource_restrictions.items():
+                ws._used_resources[r] += required
 
-    def client_heartbeat(self, client=None):
-        """ Handle heartbeats from Client """
-        cs: ClientState = self.clients[client]
-        cs._last_seen = time()
+    @ccall
+    def release_resources(self, ts: TaskState, ws: WorkerState):
+        if ts._resource_restrictions:
+            for r, required in ts._resource_restrictions.items():
+                ws._used_resources[r] -= required
 
-    ###################
-    # Task Validation #
-    ###################
-
-    def validate_released(self, key):
-        ts: TaskState = self.tasks[key]
-        dts: TaskState
-        assert ts._state == "released"
-        assert not ts._waiters
-        assert not ts._waiting_on
-        assert not ts._who_has
-        assert not ts._processing_on
-        assert not any([ts in dts._waiters for dts in ts._dependencies])
-        assert ts not in self.unrunnable
+    @ccall
+    def coerce_hostname(self, host):
+        """
+        Coerce the hostname of a worker.
+        """
+        if host in self._aliases:
+            return self._workers_dv[self._aliases[host]].host
+        else:
+            return host
 
-    def validate_waiting(self, key):
-        ts: TaskState = self.tasks[key]
-        dts: TaskState
-        assert ts._waiting_on
-        assert not ts._who_has
-        assert not ts._processing_on
-        assert ts not in self.unrunnable
-        for dts in ts._dependencies:
-            # We are waiting on a dependency iff it's not stored
-            assert (not not dts._who_has) != (dts in ts._waiting_on)
-            assert ts in dts._waiters  # XXX even if dts._who_has?
+    @ccall
+    @exceptval(check=False)
+    def worker_objective(self, ts: TaskState, ws: WorkerState) -> tuple:
+        """
+        Objective function to determine which worker should get the task
 
-    def validate_processing(self, key):
-        ts: TaskState = self.tasks[key]
+        Minimize expected start time.  If a tie then break with data storage.
+        """
         dts: TaskState
-        assert not ts._waiting_on
-        ws: WorkerState = ts._processing_on
-        assert ws
-        assert ts in ws._processing
-        assert not ts._who_has
+        nbytes: Py_ssize_t
+        comm_bytes: Py_ssize_t = 0
         for dts in ts._dependencies:
-            assert dts._who_has
-            assert ts in dts._waiters
+            if ws not in dts._who_has:
+                nbytes = dts.get_nbytes()
+                comm_bytes += nbytes
 
-    def validate_memory(self, key):
-        ts: TaskState = self.tasks[key]
-        dts: TaskState
-        assert ts._who_has
-        assert not ts._processing_on
-        assert not ts._waiting_on
-        assert ts not in self.unrunnable
-        for dts in ts._dependents:
-            assert (dts in ts._waiters) == (dts._state in ("waiting", "processing"))
-            assert ts not in dts._waiting_on
+        bandwidth: double = self._bandwidth
+        stack_time: double = ws._occupancy / ws._nthreads
+        start_time: double = stack_time + comm_bytes / bandwidth
 
-    def validate_no_worker(self, key):
-        ts: TaskState = self.tasks[key]
-        dts: TaskState
-        assert ts in self.unrunnable
-        assert not ts._waiting_on
-        assert ts in self.unrunnable
-        assert not ts._processing_on
-        assert not ts._who_has
-        for dts in ts._dependencies:
-            assert dts._who_has
+        if ts._actor:
+            return (len(ws._actors), start_time, ws._nbytes)
+        else:
+            return (start_time, ws._nbytes)
 
-    def validate_erred(self, key):
-        ts: TaskState = self.tasks[key]
-        assert ts._exception_blame
-        assert not ts._who_has
 
-    def validate_key(self, key, ts: TaskState = None):
-        try:
-            if ts is None:
-                ts = self.tasks.get(key)
-            if ts is None:
-                logger.debug("Key lost: %s", key)
-            else:
-                ts.validate()
-                try:
-                    func = getattr(self, "validate_" + ts._state.replace("-", "_"))
-                except AttributeError:
-                    logger.error(
-                        "self.validate_%s not found", ts._state.replace("-", "_")
-                    )
-                else:
-                    func(key)
-        except Exception as e:
-            logger.exception(e)
-            if LOG_PDB:
-                import pdb
+class Scheduler(SchedulerState, ServerNode):
+    """Dynamic distributed task scheduler
 
-                pdb.set_trace()
-            raise
+    The scheduler tracks the current state of workers, data, and computations.
+    The scheduler listens for events and responds by controlling workers
+    appropriately.  It continuously tries to use the workers to execute an ever
+    growing dask graph.
 
-    def validate_state(self, allow_overlap=False):
-        validate_state(self.tasks, self.workers, self.clients)
+    All events are handled quickly, in linear time with respect to their input
+    (which is often of constant size) and generally within a millisecond.  To
+    accomplish this the scheduler tracks a lot of state.  Every operation
+    maintains the consistency of this state.
 
-        if not (set(self.workers) == set(self.stream_comms)):
-            raise ValueError("Workers not the same in all collections")
+    The scheduler communicates with the outside world through Comm objects.
+    It maintains a consistent and valid view of the world even when listening
+    to several clients at once.
 
-        ws: WorkerState
-        for w, ws in self.workers.items():
-            assert isinstance(w, str), (type(w), w)
-            assert isinstance(ws, WorkerState), (type(ws), ws)
-            assert ws._address == w
-            if not ws._processing:
-                assert not ws._occupancy
-                assert ws._address in cast(dict, self.idle)
+    A Scheduler is typically started either with the ``dask-scheduler``
+    executable::
 
-        ts: TaskState
-        for k, ts in self.tasks.items():
-            assert isinstance(ts, TaskState), (type(ts), ts)
-            assert ts._key == k
-            self.validate_key(k, ts)
+         $ dask-scheduler
+         Scheduler started at 127.0.0.1:8786
 
-        c: str
-        cs: ClientState
-        for c, cs in self.clients.items():
-            # client=None is often used in tests...
-            assert c is None or type(c) == str, (type(c), c)
-            assert type(cs) == ClientState, (type(cs), cs)
-            assert cs._client_key == c
+    Or within a LocalCluster a Client starts up without connection
+    information::
 
-        a = {w: ws._nbytes for w, ws in self.workers.items()}
-        b = {
-            w: sum(ts.get_nbytes() for ts in ws._has_what)
-            for w, ws in self.workers.items()
-        }
-        assert a == b, (a, b)
+        >>> c = Client()  # doctest: +SKIP
+        >>> c.cluster.scheduler  # doctest: +SKIP
+        Scheduler(...)
 
-        actual_total_occupancy = 0
-        for worker, ws in self.workers.items():
-            assert abs(sum(ws._processing.values()) - ws._occupancy) < 1e-8
-            actual_total_occupancy += ws._occupancy
+    Users typically do not interact with the scheduler directly but rather with
+    the client object ``Client``.
 
-        assert abs(actual_total_occupancy - self.total_occupancy) < 1e-8, (
-            actual_total_occupancy,
-            self.total_occupancy,
-        )
+    **State**
 
-    ###################
-    # Manage Messages #
-    ###################
+    The scheduler contains the following state variables.  Each variable is
+    listed along with what it stores and a brief description.
 
-    def report(self, msg: dict, ts: TaskState = None, client: str = None):
-        """
-        Publish updates to all listening Queues and Comms
+    * **tasks:** ``{task key: TaskState}``
+        Tasks currently known to the scheduler
+    * **unrunnable:** ``{TaskState}``
+        Tasks in the "no-worker" state
 
-        If the message contains a key then we only send the message to those
-        comms that care about the key.
-        """
-        if ts is None:
-            msg_key = msg.get("key")
-            if msg_key is not None:
-                tasks: dict = self.tasks
-                ts = tasks.get(msg_key)
+    * **workers:** ``{worker key: WorkerState}``
+        Workers currently connected to the scheduler
+    * **idle:** ``{WorkerState}``:
+        Set of workers that are not fully utilized
+    * **saturated:** ``{WorkerState}``:
+        Set of workers that are not over-utilized
 
-        cs: ClientState
-        client_comms: dict = self.client_comms
-        client_keys: list
-        if ts is None:
-            # Notify all clients
-            client_keys = list(client_comms)
-        elif client is None:
-            # Notify clients interested in key
-            client_keys = [cs._client_key for cs in ts._who_wants]
-        else:
-            # Notify clients interested in key (including `client`)
-            client_keys = [
-                cs._client_key for cs in ts._who_wants if cs._client_key != client
-            ]
-            client_keys.append(client)
+    * **host_info:** ``{hostname: dict}``:
+        Information about each worker host
 
-        k: str
-        for k in client_keys:
-            c = client_comms.get(k)
-            if c is None:
-                continue
-            try:
-                c.send(msg)
-                # logger.debug("Scheduler sends message to client %s", msg)
-            except CommClosedError:
-                if self.status == Status.running:
-                    logger.critical("Tried writing to closed comm: %s", msg)
+    * **clients:** ``{client key: ClientState}``
+        Clients currently connected to the scheduler
 
-    async def add_client(self, comm, client=None, versions=None):
-        """Add client to network
+    * **services:** ``{str: port}``:
+        Other services running on this scheduler, like Bokeh
+    * **loop:** ``IOLoop``:
+        The running Tornado IOLoop
+    * **client_comms:** ``{client key: Comm}``
+        For each client, a Comm object used to receive task requests and
+        report task status updates.
+    * **stream_comms:** ``{worker key: Comm}``
+        For each worker, a Comm object from which we both accept stimuli and
+        report results
+    * **task_duration:** ``{key-prefix: time}``
+        Time we expect certain functions to take, e.g. ``{'sum': 0.25}``
+    """
 
-        We listen to all future messages from this Comm.
-        """
-        assert client is not None
-        comm.name = "Scheduler->Client"
-        logger.info("Receive client connection: %s", client)
-        self.log_event(["all", client], {"action": "add-client", "client": client})
-        self.clients[client] = ClientState(client, versions=versions)
+    default_port = 8786
+    _instances = weakref.WeakSet()
 
-        for plugin in self.plugins[:]:
-            try:
-                plugin.add_client(scheduler=self, client=client)
-            except Exception as e:
-                logger.exception(e)
+    def __init__(
+        self,
+        loop=None,
+        delete_interval="500ms",
+        synchronize_worker_interval="60s",
+        services=None,
+        service_kwargs=None,
+        allowed_failures=None,
+        extensions=None,
+        validate=None,
+        scheduler_file=None,
+        security=None,
+        worker_ttl=None,
+        idle_timeout=None,
+        interface=None,
+        host=None,
+        port=0,
+        protocol=None,
+        dashboard_address=None,
+        dashboard=None,
+        http_prefix="/",
+        preload=None,
+        preload_argv=(),
+        plugins=(),
+        **kwargs,
+    ):
+        self._setup_logging(logger)
 
-        try:
-            bcomm = BatchedSend(interval="2ms", loop=self.loop)
-            bcomm.start(comm)
-            self.client_comms[client] = bcomm
-            msg = {"op": "stream-start"}
-            ws: WorkerState
-            version_warning = version_module.error_message(
-                version_module.get_versions(),
-                {w: ws._versions for w, ws in self.workers.items()},
-                versions,
-            )
-            msg.update(version_warning)
-            bcomm.send(msg)
-
-            try:
-                await self.handle_stream(comm=comm, extra={"client": client})
-            finally:
-                self.remove_client(client=client)
-                logger.debug("Finished handling client %s", client)
-        finally:
-            if not comm.closed():
-                self.client_comms[client].send({"op": "stream-closed"})
-            try:
-                if not shutting_down():
-                    await self.client_comms[client].close()
-                    del self.client_comms[client]
-                    if self.status == Status.running:
-                        logger.info("Close client connection: %s", client)
-            except TypeError:  # comm becomes None during GC
-                pass
-
-    def remove_client(self, client=None):
-        """ Remove client from network """
-        if self.status == Status.running:
-            logger.info("Remove client %s", client)
-        self.log_event(["all", client], {"action": "remove-client", "client": client})
-        try:
-            cs: ClientState = self.clients[client]
-        except KeyError:
-            # XXX is this a legitimate condition?
-            pass
+        # Attributes
+        if allowed_failures is None:
+            allowed_failures = dask.config.get("distributed.scheduler.allowed-failures")
+        self.allowed_failures = allowed_failures
+        if validate is None:
+            validate = dask.config.get("distributed.scheduler.validate")
+        self.proc = psutil.Process()
+        self.delete_interval = parse_timedelta(delete_interval, default="ms")
+        self.synchronize_worker_interval = parse_timedelta(
+            synchronize_worker_interval, default="ms"
+        )
+        self.digests = None
+        self.service_specs = services or {}
+        self.service_kwargs = service_kwargs or {}
+        self.services = {}
+        self.scheduler_file = scheduler_file
+        worker_ttl = worker_ttl or dask.config.get("distributed.scheduler.worker-ttl")
+        self.worker_ttl = parse_timedelta(worker_ttl) if worker_ttl else None
+        idle_timeout = idle_timeout or dask.config.get(
+            "distributed.scheduler.idle-timeout"
+        )
+        if idle_timeout:
+            self.idle_timeout = parse_timedelta(idle_timeout)
         else:
-            ts: TaskState
-            self.client_releases_keys(
-                keys=[ts._key for ts in cs._wants_what], client=cs._client_key
-            )
-            del self.clients[client]
+            self.idle_timeout = None
+        self.idle_since = time()
+        self.time_started = self.idle_since  # compatibility for dask-gateway
+        self._lock = asyncio.Lock()
+        self.bandwidth_workers = defaultdict(float)
+        self.bandwidth_types = defaultdict(float)
 
-            for plugin in self.plugins[:]:
-                try:
-                    plugin.remove_client(scheduler=self, client=client)
-                except Exception as e:
-                    logger.exception(e)
+        if not preload:
+            preload = dask.config.get("distributed.scheduler.preload")
+        if not preload_argv:
+            preload_argv = dask.config.get("distributed.scheduler.preload-argv")
+        self.preloads = preloading.process_preloads(self, preload, preload_argv)
 
-        def remove_client_from_events():
-            # If the client isn't registered anymore after the delay, remove from events
-            if client not in self.clients and client in self.events:
-                del self.events[client]
+        if isinstance(security, dict):
+            security = Security(**security)
+        self.security = security or Security()
+        assert isinstance(self.security, Security)
+        self.connection_args = self.security.get_connection_args("scheduler")
+        self.connection_args["handshake_overrides"] = {  # common denominator
+            "pickle-protocol": 4
+        }
 
-        cleanup_delay = parse_timedelta(
-            dask.config.get("distributed.scheduler.events-cleanup-delay")
+        self._start_address = addresses_from_user_args(
+            host=host,
+            port=port,
+            interface=interface,
+            protocol=protocol,
+            security=security,
+            default_port=self.default_port,
         )
-        self.loop.call_later(cleanup_delay, remove_client_from_events)
-
-    def send_task_to_worker(self, worker, ts: TaskState, duration=None):
-        """ Send a single computational task to a worker """
-        try:
-            ws: WorkerState
-            dts: TaskState
 
-            if duration is None:
-                duration = self.get_task_duration(ts)
+        http_server_modules = dask.config.get("distributed.scheduler.http.routes")
+        show_dashboard = dashboard or (dashboard is None and dashboard_address)
+        missing_bokeh = False
+        # install vanilla route if show_dashboard but bokeh is not installed
+        if show_dashboard:
+            try:
+                import distributed.dashboard.scheduler
+            except ImportError:
+                missing_bokeh = True
+                http_server_modules.append("distributed.http.scheduler.missing_bokeh")
+        routes = get_handlers(
+            server=self, modules=http_server_modules, prefix=http_prefix
+        )
+        self.start_http_server(routes, dashboard_address, default_port=8787)
+        if show_dashboard and not missing_bokeh:
+            distributed.dashboard.scheduler.connect(
+                self.http_application, self.http_server, self, prefix=http_prefix
+            )
 
-            msg: dict = {
-                "op": "compute-task",
-                "key": ts._key,
-                "priority": ts._priority,
-                "duration": duration,
-            }
-            if ts._resource_restrictions:
-                msg["resource_restrictions"] = ts._resource_restrictions
-            if ts._actor:
-                msg["actor"] = True
+        # Communication state
+        self.loop = loop or IOLoop.current()
+        self.client_comms = dict()
+        self.stream_comms = dict()
+        self._worker_coroutines = []
+        self._ipython_kernel = None
 
-            deps: set = ts._dependencies
-            if deps:
-                msg["who_has"] = {
-                    dts._key: [ws._address for ws in dts._who_has] for dts in deps
-                }
-                msg["nbytes"] = {dts._key: dts._nbytes for dts in deps}
+        # Task state
+        tasks = dict()
+        self.task_groups = dict()
+        self.task_prefixes = dict()
+        for old_attr, new_attr, wrap in [
+            ("priority", "priority", None),
+            ("dependencies", "dependencies", _legacy_task_key_set),
+            ("dependents", "dependents", _legacy_task_key_set),
+            ("retries", "retries", None),
+        ]:
+            func = operator.attrgetter(new_attr)
+            if wrap is not None:
+                func = compose(wrap, func)
+            setattr(self, old_attr, _StateLegacyMapping(tasks, func))
 
-                if self.validate:
-                    assert all(msg["who_has"].values())
+        for old_attr, new_attr, wrap in [
+            ("nbytes", "nbytes", None),
+            ("who_wants", "who_wants", _legacy_client_key_set),
+            ("who_has", "who_has", _legacy_worker_key_set),
+            ("waiting", "waiting_on", _legacy_task_key_set),
+            ("waiting_data", "waiters", _legacy_task_key_set),
+            ("rprocessing", "processing_on", None),
+            ("host_restrictions", "host_restrictions", None),
+            ("worker_restrictions", "worker_restrictions", None),
+            ("resource_restrictions", "resource_restrictions", None),
+            ("suspicious_tasks", "suspicious", None),
+            ("exceptions", "exception", None),
+            ("tracebacks", "traceback", None),
+            ("exceptions_blame", "exception_blame", _task_key_or_none),
+        ]:
+            func = operator.attrgetter(new_attr)
+            if wrap is not None:
+                func = compose(wrap, func)
+            setattr(self, old_attr, _OptionalStateLegacyMapping(tasks, func))
 
-            task = ts._run_spec
-            if type(task) is dict:
-                msg.update(task)
-            else:
-                msg["task"] = task
+        for old_attr, new_attr, wrap in [
+            ("loose_restrictions", "loose_restrictions", None)
+        ]:
+            func = operator.attrgetter(new_attr)
+            if wrap is not None:
+                func = compose(wrap, func)
+            setattr(self, old_attr, _StateLegacySet(tasks, func))
 
-            self.worker_send(worker, msg)
-        except Exception as e:
-            logger.exception(e)
-            if LOG_PDB:
-                import pdb
+        self.generation = 0
+        self._last_client = None
+        self._last_time = 0
+        unrunnable = set()
 
-                pdb.set_trace()
-            raise
+        self.datasets = dict()
 
-    def handle_uncaught_error(self, **msg):
-        logger.exception(clean_exception(**msg)[1])
+        # Prefix-keyed containers
 
-    def handle_task_finished(self, key=None, worker=None, **msg):
-        if worker not in self.workers:
-            return
-        validate_key(key)
-        r = self.stimulus_task_finished(key=key, worker=worker, **msg)
-        self.transitions(r)
+        # Client state
+        clients = dict()
+        for old_attr, new_attr, wrap in [
+            ("wants_what", "wants_what", _legacy_task_key_set)
+        ]:
+            func = operator.attrgetter(new_attr)
+            if wrap is not None:
+                func = compose(wrap, func)
+            setattr(self, old_attr, _StateLegacyMapping(clients, func))
 
-    def handle_task_erred(self, key=None, **msg):
-        r = self.stimulus_task_erred(key=key, **msg)
-        self.transitions(r)
+        # Worker state
+        workers = sortedcontainers.SortedDict()
+        for old_attr, new_attr, wrap in [
+            ("nthreads", "nthreads", None),
+            ("worker_bytes", "nbytes", None),
+            ("worker_resources", "resources", None),
+            ("used_resources", "used_resources", None),
+            ("occupancy", "occupancy", None),
+            ("worker_info", "metrics", None),
+            ("processing", "processing", _legacy_task_key_dict),
+            ("has_what", "has_what", _legacy_task_key_set),
+        ]:
+            func = operator.attrgetter(new_attr)
+            if wrap is not None:
+                func = compose(wrap, func)
+            setattr(self, old_attr, _StateLegacyMapping(workers, func))
 
-    def handle_release_data(self, key=None, worker=None, client=None, **msg):
-        ts: TaskState = self.tasks.get(key)
-        if ts is None:
-            return
-        ws: WorkerState = self.workers[worker]
-        if ts._processing_on != ws:
-            return
-        r = self.stimulus_missing_data(key=key, ensure=False, **msg)
-        self.transitions(r)
+        host_info = defaultdict(dict)
+        resources = defaultdict(dict)
+        aliases = dict()
 
-    def handle_missing_data(self, key=None, errant_worker=None, **kwargs):
-        logger.debug("handle missing data key=%s worker=%s", key, errant_worker)
-        self.log.append(("missing", key, errant_worker))
+        self._task_state_collections = [unrunnable]
 
-        ts: TaskState = self.tasks.get(key)
-        if ts is None or not ts._who_has:
-            return
-        if errant_worker in self.workers:
-            ws: WorkerState = self.workers[errant_worker]
-            if ws in ts._who_has:
-                ts._who_has.remove(ws)
-                ws._has_what.remove(ts)
-                ws._nbytes -= ts.get_nbytes()
-        if not ts._who_has:
-            if ts._run_spec:
-                self.transitions({key: "released"})
-            else:
-                self.transitions({key: "forgotten"})
+        self._worker_collections = [
+            workers,
+            host_info,
+            resources,
+            aliases,
+        ]
 
-    def release_worker_data(self, comm=None, keys=None, worker=None):
-        ws: WorkerState = self.workers[worker]
-        tasks = {self.tasks[k] for k in keys}
-        removed_tasks = tasks & ws._has_what
-        ws._has_what -= removed_tasks
+        self.plugins = list(plugins)
+        self.transition_log = deque(
+            maxlen=dask.config.get("distributed.scheduler.transition-log-length")
+        )
+        self.log = deque(
+            maxlen=dask.config.get("distributed.scheduler.transition-log-length")
+        )
+        self.events = defaultdict(lambda: deque(maxlen=100000))
+        self.event_counts = defaultdict(int)
+        self.worker_plugins = []
 
-        ts: TaskState
-        recommendations: dict = {}
-        for ts in removed_tasks:
-            ws._nbytes -= ts.get_nbytes()
-            wh = ts._who_has
-            wh.remove(ws)
-            if not wh:
-                recommendations[ts._key] = "released"
-        if recommendations:
-            self.transitions(recommendations)
-
-    def handle_long_running(self, key=None, worker=None, compute_duration=None):
-        """A task has seceded from the thread pool
-
-        We stop the task from being stolen in the future, and change task
-        duration accounting as if the task has stopped.
-        """
-        ts: TaskState = self.tasks[key]
-        if "stealing" in self.extensions:
-            self.extensions["stealing"].remove_key_from_stealable(ts)
-
-        ws: WorkerState = ts._processing_on
-        if ws is None:
-            logger.debug("Received long-running signal from duplicate task. Ignoring.")
-            return
+        worker_handlers = {
+            "task-finished": self.handle_task_finished,
+            "task-erred": self.handle_task_erred,
+            "release": self.handle_release_data,
+            "release-worker-data": self.release_worker_data,
+            "add-keys": self.add_keys,
+            "missing-data": self.handle_missing_data,
+            "long-running": self.handle_long_running,
+            "reschedule": self.reschedule,
+            "keep-alive": lambda *args, **kwargs: None,
+            "log-event": self.log_worker_event,
+        }
 
-        if compute_duration:
-            old_duration = ts._prefix._duration_average
-            new_duration = compute_duration
-            if old_duration < 0:
-                avg_duration = new_duration
-            else:
-                avg_duration = 0.5 * old_duration + 0.5 * new_duration
+        client_handlers = {
+            "update-graph": self.update_graph,
+            "update-graph-hlg": self.update_graph_hlg,
+            "client-desires-keys": self.client_desires_keys,
+            "update-data": self.update_data,
+            "report-key": self.report_on_key,
+            "client-releases-keys": self.client_releases_keys,
+            "heartbeat-client": self.client_heartbeat,
+            "close-client": self.remove_client,
+            "restart": self.restart,
+        }
 
-            ts._prefix._duration_average = avg_duration
+        self.handlers = {
+            "register-client": self.add_client,
+            "scatter": self.scatter,
+            "register-worker": self.add_worker,
+            "unregister": self.remove_worker,
+            "gather": self.gather,
+            "cancel": self.stimulus_cancel,
+            "retry": self.stimulus_retry,
+            "feed": self.feed,
+            "terminate": self.close,
+            "broadcast": self.broadcast,
+            "proxy": self.proxy,
+            "ncores": self.get_ncores,
+            "has_what": self.get_has_what,
+            "who_has": self.get_who_has,
+            "processing": self.get_processing,
+            "call_stack": self.get_call_stack,
+            "profile": self.get_profile,
+            "performance_report": self.performance_report,
+            "get_logs": self.get_logs,
+            "logs": self.get_logs,
+            "worker_logs": self.get_worker_logs,
+            "log_event": self.log_worker_event,
+            "events": self.get_events,
+            "nbytes": self.get_nbytes,
+            "versions": self.versions,
+            "add_keys": self.add_keys,
+            "rebalance": self.rebalance,
+            "replicate": self.replicate,
+            "start_ipython": self.start_ipython,
+            "run_function": self.run_function,
+            "update_data": self.update_data,
+            "set_resources": self.add_resources,
+            "retire_workers": self.retire_workers,
+            "get_metadata": self.get_metadata,
+            "set_metadata": self.set_metadata,
+            "heartbeat_worker": self.heartbeat_worker,
+            "get_task_status": self.get_task_status,
+            "get_task_stream": self.get_task_stream,
+            "register_worker_plugin": self.register_worker_plugin,
+            "adaptive_target": self.adaptive_target,
+            "workers_to_close": self.workers_to_close,
+            "subscribe_worker_status": self.subscribe_worker_status,
+            "start_task_metadata": self.start_task_metadata,
+            "stop_task_metadata": self.stop_task_metadata,
+        }
 
-        ws._occupancy -= ws._processing[ts]
-        self.total_occupancy -= ws._processing[ts]
-        ws._processing[ts] = 0
-        self.check_idle_saturated(ws)
+        self._transitions = {
+            ("released", "waiting"): self.transition_released_waiting,
+            ("waiting", "released"): self.transition_waiting_released,
+            ("waiting", "processing"): self.transition_waiting_processing,
+            ("waiting", "memory"): self.transition_waiting_memory,
+            ("processing", "released"): self.transition_processing_released,
+            ("processing", "memory"): self.transition_processing_memory,
+            ("processing", "erred"): self.transition_processing_erred,
+            ("no-worker", "released"): self.transition_no_worker_released,
+            ("no-worker", "waiting"): self.transition_no_worker_waiting,
+            ("released", "forgotten"): self.transition_released_forgotten,
+            ("memory", "forgotten"): self.transition_memory_forgotten,
+            ("erred", "forgotten"): self.transition_released_forgotten,
+            ("erred", "released"): self.transition_erred_released,
+            ("memory", "released"): self.transition_memory_released,
+            ("released", "erred"): self.transition_released_erred,
+        }
 
-    async def handle_worker(self, comm=None, worker=None):
-        """
-        Listen to responses from a single worker
+        connection_limit = get_fileno_limit() / 2
 
-        This is the main loop for scheduler-worker interaction
+        super().__init__(
+            aliases=aliases,
+            handlers=self.handlers,
+            stream_handlers=merge(worker_handlers, client_handlers),
+            io_loop=self.loop,
+            connection_limit=connection_limit,
+            deserialize=False,
+            connection_args=self.connection_args,
+            clients=clients,
+            workers=workers,
+            host_info=host_info,
+            resources=resources,
+            tasks=tasks,
+            unrunnable=unrunnable,
+            validate=validate,
+            **kwargs,
+        )
 
-        See Also
-        --------
-        Scheduler.handle_client: Equivalent coroutine for clients
-        """
-        comm.name = "Scheduler connection to worker"
-        worker_comm = self.stream_comms[worker]
-        worker_comm.start(comm)
-        logger.info("Starting worker compute stream, %s", worker)
-        try:
-            await self.handle_stream(comm=comm, extra={"worker": worker})
-        finally:
-            if worker in self.stream_comms:
-                worker_comm.abort()
-                await self.remove_worker(address=worker)
+        if self.worker_ttl:
+            pc = PeriodicCallback(self.check_worker_ttl, self.worker_ttl)
+            self.periodic_callbacks["worker-ttl"] = pc
 
-    def add_plugin(self, plugin=None, idempotent=False, **kwargs):
-        """
-        Add external plugin to scheduler
+        if self.idle_timeout:
+            pc = PeriodicCallback(self.check_idle, self.idle_timeout / 4)
+            self.periodic_callbacks["idle-timeout"] = pc
 
-        See https://distributed.readthedocs.io/en/latest/plugins.html
-        """
-        if isinstance(plugin, type):
-            plugin = plugin(self, **kwargs)
+        if extensions is None:
+            extensions = list(DEFAULT_EXTENSIONS)
+            if dask.config.get("distributed.scheduler.work-stealing"):
+                extensions.append(WorkStealing)
+        for ext in extensions:
+            ext(self)
 
-        if idempotent and any(isinstance(p, type(plugin)) for p in self.plugins):
-            return
+        setproctitle("dask-scheduler [not started]")
+        Scheduler._instances.add(self)
+        self.rpc.allow_offload = False
+        self.status = Status.undefined
 
-        self.plugins.append(plugin)
+    ##################
+    # Administration #
+    ##################
 
-    def remove_plugin(self, plugin):
-        """ Remove external plugin from scheduler """
-        self.plugins.remove(plugin)
+    def __repr__(self):
+        parent: SchedulerState = cast(SchedulerState, self)
+        return '<Scheduler: "%s" processes: %d cores: %d>' % (
+            self.address,
+            len(parent._workers),
+            parent._total_nthreads,
+        )
 
-    def worker_send(self, worker, msg):
-        """Send message to worker
+    def identity(self, comm=None):
+        """ Basic information about ourselves and our cluster """
+        parent: SchedulerState = cast(SchedulerState, self)
+        d = {
+            "type": type(self).__name__,
+            "id": str(self.id),
+            "address": self.address,
+            "services": {key: v.port for (key, v) in self.services.items()},
+            "started": self.time_started,
+            "workers": {
+                worker.address: worker.identity() for worker in parent._workers.values()
+            },
+        }
+        return d
 
-        This also handles connection failures by adding a callback to remove
-        the worker on the next cycle.
+    def get_worker_service_addr(self, worker, service_name, protocol=False):
         """
-        stream_comms: dict = self.stream_comms
-        try:
-            stream_comms[worker].send(msg)
-        except (CommClosedError, AttributeError):
-            self.loop.add_callback(self.remove_worker, address=worker)
-
-    ############################
-    # Less common interactions #
-    ############################
-
-    async def scatter(
-        self,
-        comm=None,
-        data=None,
-        workers=None,
-        client=None,
-        broadcast=False,
-        timeout=2,
-    ):
-        """Send data out to workers
+        Get the (host, port) address of the named service on the *worker*.
+        Returns None if the service doesn't exist.
 
-        See also
-        --------
-        Scheduler.broadcast:
+        Parameters
+        ----------
+        worker : address
+        service_name : str
+            Common services include 'bokeh' and 'nanny'
+        protocol : boolean
+            Whether or not to include a full address with protocol (True)
+            or just a (host, port) pair
         """
-        start = time()
-        while not self.workers:
-            await asyncio.sleep(0.2)
-            if time() > start + timeout:
-                raise TimeoutError("No workers found")
-
-        if workers is None:
-            ws: WorkerState
-            nthreads = {w: ws._nthreads for w, ws in self.workers.items()}
+        parent: SchedulerState = cast(SchedulerState, self)
+        ws: WorkerState = parent._workers[worker]
+        port = ws._services.get(service_name)
+        if port is None:
+            return None
+        elif protocol:
+            return "%(protocol)s://%(host)s:%(port)d" % {
+                "protocol": ws._address.split("://")[0],
+                "host": ws.host,
+                "port": port,
+            }
         else:
-            workers = [self.coerce_address(w) for w in workers]
-            nthreads = {w: self.workers[w].nthreads for w in workers}
+            return ws.host, port
 
-        assert isinstance(data, dict)
+    async def start(self):
+        """ Clear out old state and restart all running coroutines """
+        await super().start()
+        assert self.status != Status.running
 
-        keys, who_has, nbytes = await scatter_to_workers(
-            nthreads, data, rpc=self.rpc, report=False
-        )
+        enable_gc_diagnosis()
 
-        self.update_data(who_has=who_has, nbytes=nbytes, client=client)
+        self.clear_task_state()
 
-        if broadcast:
-            if broadcast == True:  # noqa: E712
-                n = len(nthreads)
-            else:
-                n = broadcast
-            await self.replicate(keys=keys, workers=workers, n=n)
+        with suppress(AttributeError):
+            for c in self._worker_coroutines:
+                c.cancel()
 
-        self.log_event(
-            [client, "all"], {"action": "scatter", "client": client, "count": len(data)}
-        )
-        return keys
+        for addr in self._start_address:
+            await self.listen(
+                addr,
+                allow_offload=False,
+                handshake_overrides={"pickle-protocol": 4, "compression": None},
+                **self.security.get_listen_args("scheduler"),
+            )
+            self.ip = get_address_host(self.listen_address)
+            listen_ip = self.ip
 
-    async def gather(self, comm=None, keys=None, serializers=None):
-        """ Collect data in from workers """
-        ws: WorkerState
-        keys = list(keys)
-        who_has = {}
-        for key in keys:
-            ts: TaskState = self.tasks.get(key)
-            if ts is not None:
-                who_has[key] = [ws._address for ws in ts._who_has]
-            else:
-                who_has[key] = []
+            if listen_ip == "0.0.0.0":
+                listen_ip = ""
 
-        data, missing_keys, missing_workers = await gather_from_workers(
-            who_has, rpc=self.rpc, close=False, serializers=serializers
-        )
-        if not missing_keys:
-            result = {"status": "OK", "data": data}
-        else:
-            missing_states = [
-                (self.tasks[key].state if key in self.tasks else None)
-                for key in missing_keys
-            ]
-            logger.exception(
-                "Couldn't gather keys %s state: %s workers: %s",
-                missing_keys,
-                missing_states,
-                missing_workers,
-            )
-            result = {"status": "error", "keys": missing_keys}
-            with log_errors():
-                # Remove suspicious workers from the scheduler but allow them to
-                # reconnect.
-                await asyncio.gather(
-                    *[
-                        self.remove_worker(address=worker, close=False)
-                        for worker in missing_workers
-                    ]
-                )
-                for key, workers in missing_keys.items():
-                    # Task may already be gone if it was held by a
-                    # `missing_worker`
-                    ts: TaskState = self.tasks.get(key)
-                    logger.exception(
-                        "Workers don't have promised key: %s, %s",
-                        str(workers),
-                        str(key),
-                    )
-                    if not workers or ts is None:
-                        continue
-                    for worker in workers:
-                        ws = self.workers.get(worker)
-                        if ws is not None and ts in ws._has_what:
-                            ws._has_what.remove(ts)
-                            ts._who_has.remove(ws)
-                            ws._nbytes -= ts.get_nbytes()
-                            self.transitions({key: "released"})
+        if self.address.startswith("inproc://"):
+            listen_ip = "localhost"
 
-        self.log_event("all", {"action": "gather", "count": len(keys)})
-        return result
+        # Services listen on all addresses
+        self.start_services(listen_ip)
 
-    def clear_task_state(self):
-        # XXX what about nested state such as ClientState.wants_what
-        # (see also fire-and-forget...)
-        logger.info("Clear task state")
-        for collection in self._task_state_collections:
-            collection.clear()
+        for listener in self.listeners:
+            logger.info("  Scheduler at: %25s", listener.contact_address)
+        for k, v in self.services.items():
+            logger.info("%11s at: %25s", k, "%s:%d" % (listen_ip, v.port))
 
-    async def restart(self, client=None, timeout=3):
-        """ Restart all workers.  Reset local state. """
-        with log_errors():
+        self.loop.add_callback(self.reevaluate_occupancy)
 
-            n_workers = len(self.workers)
+        if self.scheduler_file:
+            with open(self.scheduler_file, "w") as f:
+                json.dump(self.identity(), f, indent=2)
 
-            logger.info("Send lost future signal to clients")
-            cs: ClientState
-            ts: TaskState
-            for cs in self.clients.values():
-                self.client_releases_keys(
-                    keys=[ts._key for ts in cs._wants_what], client=cs._client_key
-                )
+            fn = self.scheduler_file  # remove file when we close the process
 
-            ws: WorkerState
-            nannies = {addr: ws._nanny for addr, ws in self.workers.items()}
+            def del_scheduler_file():
+                if os.path.exists(fn):
+                    os.remove(fn)
 
-            for addr in list(self.workers):
-                try:
-                    # Ask the worker to close if it doesn't have a nanny,
-                    # otherwise the nanny will kill it anyway
-                    await self.remove_worker(address=addr, close=addr not in nannies)
-                except Exception as e:
-                    logger.info(
-                        "Exception while restarting.  This is normal", exc_info=True
-                    )
+            weakref.finalize(self, del_scheduler_file)
 
-            self.clear_task_state()
+        for preload in self.preloads:
+            await preload.start()
 
-            for plugin in self.plugins[:]:
-                try:
-                    plugin.restart(self)
-                except Exception as e:
-                    logger.exception(e)
+        await asyncio.gather(*[plugin.start(self) for plugin in self.plugins])
 
-            logger.debug("Send kill signal to nannies: %s", nannies)
+        self.start_periodic_callbacks()
 
-            nannies = [
-                rpc(nanny_address, connection_args=self.connection_args)
-                for nanny_address in nannies.values()
-                if nanny_address is not None
-            ]
+        setproctitle("dask-scheduler [%s]" % (self.address,))
+        return self
 
-            resps = All(
-                [
-                    nanny.restart(
-                        close=True, timeout=timeout * 0.8, executor_wait=False
-                    )
-                    for nanny in nannies
-                ]
-            )
-            try:
-                resps = await asyncio.wait_for(resps, timeout)
-            except TimeoutError:
-                logger.error(
-                    "Nannies didn't report back restarted within "
-                    "timeout.  Continuuing with restart process"
-                )
-            else:
-                if not all(resp == "OK" for resp in resps):
-                    logger.error(
-                        "Not all workers responded positively: %s", resps, exc_info=True
-                    )
-            finally:
-                await asyncio.gather(*[nanny.close_rpc() for nanny in nannies])
+    async def close(self, comm=None, fast=False, close_workers=False):
+        """Send cleanup signal to all coroutines then wait until finished
 
-            self.clear_task_state()
+        See Also
+        --------
+        Scheduler.cleanup
+        """
+        parent: SchedulerState = cast(SchedulerState, self)
+        if self.status in (Status.closing, Status.closed, Status.closing_gracefully):
+            await self.finished()
+            return
+        self.status = Status.closing
 
-            with suppress(AttributeError):
-                for c in self._worker_coroutines:
-                    c.cancel()
+        logger.info("Scheduler closing...")
+        setproctitle("dask-scheduler [closing]")
 
-            self.log_event([client, "all"], {"action": "restart", "client": client})
-            start = time()
-            while time() < start + 10 and len(self.workers) < n_workers:
-                await asyncio.sleep(0.01)
+        for preload in self.preloads:
+            await preload.teardown()
 
-            self.report({"op": "restart"})
+        if close_workers:
+            await self.broadcast(msg={"op": "close_gracefully"}, nanny=True)
+            for worker in parent._workers:
+                self.worker_send(worker, {"op": "close"})
+            for i in range(20):  # wait a second for send signals to clear
+                if parent._workers:
+                    await asyncio.sleep(0.05)
+                else:
+                    break
 
-    async def broadcast(
-        self,
-        comm=None,
-        msg=None,
-        workers=None,
-        hosts=None,
-        nanny=False,
-        serializers=None,
-    ):
-        """ Broadcast message to workers, return all results """
-        if workers is None or workers is True:
-            if hosts is None:
-                workers = list(self.workers)
-            else:
-                workers = []
-        if hosts is not None:
-            for host in hosts:
-                if host in self.host_info:
-                    workers.extend(self.host_info[host]["addresses"])
-        # TODO replace with worker_list
+        await asyncio.gather(*[plugin.close() for plugin in self.plugins])
 
-        if nanny:
-            addresses = [self.workers[w].nanny for w in workers]
-        else:
-            addresses = workers
+        for pc in self.periodic_callbacks.values():
+            pc.stop()
+        self.periodic_callbacks.clear()
 
-        async def send_message(addr):
-            comm = await self.rpc.connect(addr)
-            comm.name = "Scheduler Broadcast"
-            try:
-                resp = await send_recv(comm, close=True, serializers=serializers, **msg)
-            finally:
-                self.rpc.reuse(addr, comm)
-            return resp
+        self.stop_services()
 
-        results = await All(
-            [send_message(address) for address in addresses if address is not None]
-        )
+        for ext in parent._extensions.values():
+            with suppress(AttributeError):
+                ext.teardown()
+        logger.info("Scheduler closing all comms")
 
-        return dict(zip(workers, results))
+        futures = []
+        for w, comm in list(self.stream_comms.items()):
+            if not comm.closed():
+                comm.send({"op": "close", "report": False})
+                comm.send({"op": "close-stream"})
+            with suppress(AttributeError):
+                futures.append(comm.close())
 
-    async def proxy(self, comm=None, msg=None, worker=None, serializers=None):
-        """ Proxy a communication through the scheduler to some other worker """
-        d = await self.broadcast(
-            comm=comm, msg=msg, workers=[worker], serializers=serializers
-        )
-        return d[worker]
+        for future in futures:  # TODO: do all at once
+            await future
 
-    async def _delete_worker_data(self, worker_address, keys):
-        """Delete data from a worker and update the corresponding worker/task states
+        for comm in self.client_comms.values():
+            comm.abort()
 
-        Parameters
-        ----------
-        worker_address: str
-            Worker address to delete keys from
-        keys: List[str]
-            List of keys to delete on the specified worker
-        """
-        await retry_operation(
-            self.rpc(addr=worker_address).delete_data, keys=list(keys), report=False
-        )
+        await self.rpc.close()
 
-        ws: WorkerState = self.workers[worker_address]
-        ts: TaskState
-        tasks: set = {self.tasks[key] for key in keys}
-        ws._has_what -= tasks
-        for ts in tasks:
-            ts._who_has.remove(ws)
-            ws._nbytes -= ts.get_nbytes()
-        self.log_event(ws._address, {"action": "remove-worker-data", "keys": keys})
+        self.status = Status.closed
+        self.stop()
+        await super().close()
 
-    async def rebalance(self, comm=None, keys=None, workers=None):
-        """Rebalance keys so that each worker stores roughly equal bytes
+        setproctitle("dask-scheduler [closed]")
+        disable_gc_diagnosis()
 
-        **Policy**
+    async def close_worker(self, comm=None, worker=None, safe=None):
+        """Remove a worker from the cluster
 
-        This orders the workers by what fraction of bytes of the existing keys
-        they have.  It walks down this list from most-to-least.  At each worker
-        it sends the largest results it can find and sends them to the least
-        occupied worker until either the sender or the recipient are at the
-        average expected load.
+        This both removes the worker from our local state and also sends a
+        signal to the worker to shut down.  This works regardless of whether or
+        not the worker has a nanny process restarting it
         """
-        ts: TaskState
+        parent: SchedulerState = cast(SchedulerState, self)
+        logger.info("Closing worker %s", worker)
         with log_errors():
-            async with self._lock:
-                if keys:
-                    tasks = {self.tasks[k] for k in keys}
-                    missing_data = [ts._key for ts in tasks if not ts._who_has]
-                    if missing_data:
-                        return {"status": "missing-data", "keys": missing_data}
-                else:
-                    tasks = set(self.tasks.values())
-
-                if workers:
-                    workers = {self.workers[w] for w in workers}
-                    workers_by_task = {ts: ts._who_has & workers for ts in tasks}
-                else:
-                    workers = set(self.workers.values())
-                    workers_by_task = {ts: ts._who_has for ts in tasks}
+            self.log_event(worker, {"action": "close-worker"})
+            ws: WorkerState = parent._workers[worker]
+            nanny_addr = ws._nanny
+            address = nanny_addr or worker
 
-                ws: WorkerState
-                tasks_by_worker = {ws: set() for ws in workers}
+            self.worker_send(worker, {"op": "close", "report": False})
+            await self.remove_worker(address=worker, safe=safe)
 
-                for k, v in workers_by_task.items():
-                    for vv in v:
-                        tasks_by_worker[vv].add(k)
+    ###########
+    # Stimuli #
+    ###########
 
-                worker_bytes = {
-                    ws: sum(ts.get_nbytes() for ts in v)
-                    for ws, v in tasks_by_worker.items()
-                }
+    def heartbeat_worker(
+        self,
+        comm=None,
+        address=None,
+        resolve_address=True,
+        now=None,
+        resources=None,
+        host_info=None,
+        metrics=None,
+        executing=None,
+    ):
+        parent: SchedulerState = cast(SchedulerState, self)
+        address = self.coerce_address(address, resolve_address)
+        address = normalize_address(address)
+        if address not in parent._workers:
+            return {"status": "missing"}
 
-                avg = sum(worker_bytes.values()) / len(worker_bytes)
+        host = get_address_host(address)
+        local_now = time()
+        now = now or time()
+        assert metrics
+        host_info = host_info or {}
 
-                sorted_workers = list(
-                    map(first, sorted(worker_bytes.items(), key=second, reverse=True))
+        parent._host_info[host]["last-seen"] = local_now
+        frac = 1 / len(parent._workers)
+        parent._bandwidth = (
+            parent._bandwidth * (1 - frac) + metrics["bandwidth"]["total"] * frac
+        )
+        for other, (bw, count) in metrics["bandwidth"]["workers"].items():
+            if (address, other) not in self.bandwidth_workers:
+                self.bandwidth_workers[address, other] = bw / count
+            else:
+                alpha = (1 - frac) ** count
+                self.bandwidth_workers[address, other] = self.bandwidth_workers[
+                    address, other
+                ] * alpha + bw * (1 - alpha)
+        for typ, (bw, count) in metrics["bandwidth"]["types"].items():
+            if typ not in self.bandwidth_types:
+                self.bandwidth_types[typ] = bw / count
+            else:
+                alpha = (1 - frac) ** count
+                self.bandwidth_types[typ] = self.bandwidth_types[typ] * alpha + bw * (
+                    1 - alpha
                 )
 
-                recipients = iter(reversed(sorted_workers))
-                recipient = next(recipients)
-                msgs = []  # (sender, recipient, key)
-                for sender in sorted_workers[: len(workers) // 2]:
-                    sender_keys = {
-                        ts: ts.get_nbytes() for ts in tasks_by_worker[sender]
-                    }
-                    sender_keys = iter(
-                        sorted(sender_keys.items(), key=second, reverse=True)
-                    )
+        ws: WorkerState = parent._workers[address]
 
-                    try:
-                        while worker_bytes[sender] > avg:
-                            while (
-                                worker_bytes[recipient] < avg
-                                and worker_bytes[sender] > avg
-                            ):
-                                ts, nb = next(sender_keys)
-                                if ts not in tasks_by_worker[recipient]:
-                                    tasks_by_worker[recipient].add(ts)
-                                    # tasks_by_worker[sender].remove(ts)
-                                    msgs.append((sender, recipient, ts))
-                                    worker_bytes[sender] -= nb
-                                    worker_bytes[recipient] += nb
-                            if worker_bytes[sender] > avg:
-                                recipient = next(recipients)
-                    except StopIteration:
-                        break
+        ws._last_seen = time()
 
-                to_recipients = defaultdict(lambda: defaultdict(list))
-                to_senders = defaultdict(list)
-                for sender, recipient, ts in msgs:
-                    to_recipients[recipient.address][ts._key].append(sender.address)
-                    to_senders[sender.address].append(ts._key)
+        if executing is not None:
+            ws._executing = {
+                self.tasks[key]: duration for key, duration in executing.items()
+            }
 
-                result = await asyncio.gather(
-                    *(
-                        retry_operation(self.rpc(addr=r).gather, who_has=v)
-                        for r, v in to_recipients.items()
-                    )
-                )
-                for r, v in to_recipients.items():
-                    self.log_event(r, {"action": "rebalance", "who_has": v})
+        if metrics:
+            ws._metrics = metrics
 
-                self.log_event(
-                    "all",
-                    {
-                        "action": "rebalance",
-                        "total-keys": len(tasks),
-                        "senders": valmap(len, to_senders),
-                        "recipients": valmap(len, to_recipients),
-                        "moved_keys": len(msgs),
-                    },
-                )
+        if host_info:
+            parent._host_info[host].update(host_info)
 
-                if not all(r["status"] == "OK" for r in result):
-                    return {
-                        "status": "missing-data",
-                        "keys": tuple(
-                            concat(
-                                r["keys"].keys()
-                                for r in result
-                                if r["status"] == "missing-data"
-                            )
-                        ),
-                    }
+        delay = time() - now
+        ws._time_delay = delay
 
-                for sender, recipient, ts in msgs:
-                    assert ts._state == "memory"
-                    ts._who_has.add(recipient)
-                    recipient.has_what.add(ts)
-                    recipient.nbytes += ts.get_nbytes()
-                    self.log.append(
-                        (
-                            "rebalance",
-                            ts._key,
-                            time(),
-                            sender.address,
-                            recipient.address,
-                        )
-                    )
+        if resources:
+            self.add_resources(worker=address, resources=resources)
 
-                await asyncio.gather(
-                    *(self._delete_worker_data(r, v) for r, v in to_senders.items())
-                )
+        self.log_event(address, merge({"action": "heartbeat"}, metrics))
 
-                return {"status": "OK"}
+        return {
+            "status": "OK",
+            "time": time(),
+            "heartbeat-interval": heartbeat_interval(len(parent._workers)),
+        }
 
-    async def replicate(
+    async def add_worker(
         self,
         comm=None,
-        keys=None,
-        n=None,
-        workers=None,
-        branching_factor=2,
-        delete=True,
-        lock=True,
+        address=None,
+        keys=(),
+        nthreads=None,
+        name=None,
+        resolve_address=True,
+        nbytes=None,
+        types=None,
+        now=None,
+        resources=None,
+        host_info=None,
+        memory_limit=None,
+        metrics=None,
+        pid=0,
+        services=None,
+        local_directory=None,
+        versions=None,
+        nanny=None,
+        extra=None,
     ):
-        """Replicate data throughout cluster
-
-        This performs a tree copy of the data throughout the network
-        individually on each piece of data.
+        """ Add a new worker to the cluster """
+        parent: SchedulerState = cast(SchedulerState, self)
+        with log_errors():
+            address = self.coerce_address(address, resolve_address)
+            address = normalize_address(address)
+            host = get_address_host(address)
 
-        Parameters
-        ----------
-        keys: Iterable
-            list of keys to replicate
-        n: int
-            Number of replications we expect to see within the cluster
-        branching_factor: int, optional
-            The number of workers that can copy data in each generation.
-            The larger the branching factor, the more data we copy in
-            a single step, but the more a given worker risks being
-            swamped by data requests.
+            ws: WorkerState = parent._workers.get(address)
+            if ws is not None:
+                raise ValueError("Worker already exists %s" % ws)
 
-        See also
-        --------
-        Scheduler.rebalance
-        """
-        ws: WorkerState
-        wws: WorkerState
-        ts: TaskState
+            if name in parent._aliases:
+                logger.warning(
+                    "Worker tried to connect with a duplicate name: %s", name
+                )
+                msg = {
+                    "status": "error",
+                    "message": "name taken, %s" % name,
+                    "time": time(),
+                }
+                if comm:
+                    await comm.write(msg)
+                return
 
-        assert branching_factor > 0
-        async with self._lock if lock else empty_context:
-            workers = {self.workers[w] for w in self.workers_list(workers)}
-            if n is None:
-                n = len(workers)
-            else:
-                n = min(n, len(workers))
-            if n == 0:
-                raise ValueError("Can not use replicate to delete data")
-
-            tasks = {self.tasks[k] for k in keys}
-            missing_data = [ts._key for ts in tasks if not ts._who_has]
-            if missing_data:
-                return {"status": "missing-data", "keys": missing_data}
-
-            # Delete extraneous data
-            if delete:
-                del_worker_tasks = defaultdict(set)
-                for ts in tasks:
-                    del_candidates = ts._who_has & workers
-                    if len(del_candidates) > n:
-                        for ws in random.sample(
-                            del_candidates, len(del_candidates) - n
-                        ):
-                            del_worker_tasks[ws].add(ts)
-
-                await asyncio.gather(
-                    *[
-                        self._delete_worker_data(ws._address, [t.key for t in tasks])
-                        for ws, tasks in del_worker_tasks.items()
-                    ]
-                )
-
-            # Copy not-yet-filled data
-            while tasks:
-                gathers = defaultdict(dict)
-                for ts in list(tasks):
-                    if ts._state == "forgotten":
-                        # task is no longer needed by any client or dependant task
-                        tasks.remove(ts)
-                        continue
-                    n_missing = n - len(ts._who_has & workers)
-                    if n_missing <= 0:
-                        # Already replicated enough
-                        tasks.remove(ts)
-                        continue
-
-                    count = min(n_missing, branching_factor * len(ts._who_has))
-                    assert count > 0
+            parent._workers[address] = ws = WorkerState(
+                address=address,
+                pid=pid,
+                nthreads=nthreads,
+                memory_limit=memory_limit or 0,
+                name=name,
+                local_directory=local_directory,
+                services=services,
+                versions=versions,
+                nanny=nanny,
+                extra=extra,
+            )
 
-                    for ws in random.sample(workers - ts._who_has, count):
-                        gathers[ws._address][ts._key] = [
-                            wws._address for wws in ts._who_has
-                        ]
+            if "addresses" not in parent._host_info[host]:
+                parent._host_info[host].update({"addresses": set(), "nthreads": 0})
 
-                results = await asyncio.gather(
-                    *(
-                        retry_operation(self.rpc(addr=w).gather, who_has=who_has)
-                        for w, who_has in gathers.items()
-                    )
-                )
-                for w, v in zip(gathers, results):
-                    if v["status"] == "OK":
-                        self.add_keys(worker=w, keys=list(gathers[w]))
-                    else:
-                        logger.warning("Communication failed during replication: %s", v)
+            parent._host_info[host]["addresses"].add(address)
+            parent._host_info[host]["nthreads"] += nthreads
 
-                    self.log_event(w, {"action": "replicate-add", "keys": gathers[w]})
+            parent._total_nthreads += nthreads
+            parent._aliases[name] = address
 
-            self.log_event(
-                "all",
-                {
-                    "action": "replicate",
-                    "workers": list(workers),
-                    "key-count": len(keys),
-                    "branching-factor": branching_factor,
-                },
+            response = self.heartbeat_worker(
+                address=address,
+                resolve_address=resolve_address,
+                now=now,
+                resources=resources,
+                host_info=host_info,
+                metrics=metrics,
             )
 
-    def workers_to_close(
-        self,
-        comm=None,
-        memory_ratio=None,
-        n=None,
-        key=None,
-        minimum=None,
-        target=None,
-        attribute="address",
-    ):
-        """
-        Find workers that we can close with low cost
-
-        This returns a list of workers that are good candidates to retire.
-        These workers are not running anything and are storing
-        relatively little data relative to their peers.  If all workers are
-        idle then we still maintain enough workers to have enough RAM to store
-        our data, with a comfortable buffer.
-
-        This is for use with systems like ``distributed.deploy.adaptive``.
+            # Do not need to adjust parent._total_occupancy as self.occupancy[ws] cannot exist before this.
+            self.check_idle_saturated(ws)
 
-        Parameters
-        ----------
-        memory_factor: Number
-            Amount of extra space we want to have for our stored data.
-            Defaults two 2, or that we want to have twice as much memory as we
-            currently have data.
-        n: int
-            Number of workers to close
-        minimum: int
-            Minimum number of workers to keep around
-        key: Callable(WorkerState)
-            An optional callable mapping a WorkerState object to a group
-            affiliation.  Groups will be closed together.  This is useful when
-            closing workers must be done collectively, such as by hostname.
-        target: int
-            Target number of workers to have after we close
-        attribute : str
-            The attribute of the WorkerState object to return, like "address"
-            or "name".  Defaults to "address".
+            # for key in keys:  # TODO
+            #     self.mark_key_in_memory(key, [address])
 
-        Examples
-        --------
-        >>> scheduler.workers_to_close()
-        ['tcp://192.168.0.1:1234', 'tcp://192.168.0.2:1234']
+            self.stream_comms[address] = BatchedSend(interval="5ms", loop=self.loop)
 
-        Group workers by hostname prior to closing
+            if ws._nthreads > len(ws._processing):
+                parent._idle[ws._address] = ws
 
-        >>> scheduler.workers_to_close(key=lambda ws: ws.host)
-        ['tcp://192.168.0.1:1234', 'tcp://192.168.0.1:4567']
+            for plugin in self.plugins[:]:
+                try:
+                    result = plugin.add_worker(scheduler=self, worker=address)
+                    if inspect.isawaitable(result):
+                        await result
+                except Exception as e:
+                    logger.exception(e)
 
-        Remove two workers
+            recommendations: dict
+            if nbytes:
+                for key in nbytes:
+                    tasks: dict = parent._tasks
+                    ts: TaskState = tasks.get(key)
+                    if ts is not None and ts._state in ("processing", "waiting"):
+                        recommendations = self.transition(
+                            key,
+                            "memory",
+                            worker=address,
+                            nbytes=nbytes[key],
+                            typename=types[key],
+                        )
+                        self.transitions(recommendations)
 
-        >>> scheduler.workers_to_close(n=2)
+            recommendations = {}
+            for ts in list(parent._unrunnable):
+                valid: set = self.valid_workers(ts)
+                if valid is None or ws in valid:
+                    recommendations[ts._key] = "waiting"
 
-        Keep enough workers to have twice as much memory as we we need.
+            if recommendations:
+                self.transitions(recommendations)
 
-        >>> scheduler.workers_to_close(memory_ratio=2)
+            self.log_event(address, {"action": "add-worker"})
+            self.log_event("all", {"action": "add-worker", "worker": address})
+            logger.info("Register worker %s", ws)
 
-        Returns
-        -------
-        to_close: list of worker addresses that are OK to close
+            msg = {
+                "status": "OK",
+                "time": time(),
+                "heartbeat-interval": heartbeat_interval(len(parent._workers)),
+                "worker-plugins": self.worker_plugins,
+            }
 
-        See Also
-        --------
-        Scheduler.retire_workers
-        """
-        if target is not None and n is None:
-            n = len(self.workers) - target
-        if n is not None:
-            if n < 0:
-                n = 0
-            target = len(self.workers) - n
+            cs: ClientState
+            version_warning = version_module.error_message(
+                version_module.get_versions(),
+                merge(
+                    {w: ws._versions for w, ws in parent._workers.items()},
+                    {
+                        c: cs._versions
+                        for c, cs in parent._clients.items()
+                        if cs._versions
+                    },
+                ),
+                versions,
+                client_name="This Worker",
+            )
+            msg.update(version_warning)
 
-        if n is None and memory_ratio is None:
-            memory_ratio = 2
+            if comm:
+                await comm.write(msg)
+            await self.handle_worker(comm=comm, worker=address)
 
-        ws: WorkerState
-        with log_errors():
-            if not n and all([ws._processing for ws in self.workers.values()]):
-                return []
+    def update_graph_hlg(
+        self,
+        client=None,
+        hlg=None,
+        keys=None,
+        dependencies=None,
+        restrictions=None,
+        priority=None,
+        loose_restrictions=None,
+        resources=None,
+        submitting_task=None,
+        retries=None,
+        user_priority=0,
+        actors=None,
+        fifo_timeout=0,
+    ):
 
-            if key is None:
-                key = operator.attrgetter("address")
-            if isinstance(key, bytes) and dask.config.get(
-                "distributed.scheduler.pickle"
-            ):
-                key = pickle.loads(key)
+        dsk, dependencies, annotations = highlevelgraph_unpack(hlg)
 
-            groups = groupby(key, self.workers.values())
+        # Remove any self-dependencies (happens on test_publish_bag() and others)
+        for k, v in dependencies.items():
+            deps = set(v)
+            if k in deps:
+                deps.remove(k)
+            dependencies[k] = deps
 
-            limit_bytes = {
-                k: sum([ws._memory_limit for ws in v]) for k, v in groups.items()
+        if priority is None:
+            # Removing all non-local keys before calling order()
+            dsk_keys = set(dsk)  # intersection() of sets is much faster than dict_keys
+            stripped_deps = {
+                k: v.intersection(dsk_keys)
+                for k, v in dependencies.items()
+                if k in dsk_keys
             }
-            group_bytes = {k: sum([ws._nbytes for ws in v]) for k, v in groups.items()}
-
-            limit = sum(limit_bytes.values())
-            total = sum(group_bytes.values())
-
-            def _key(group):
-                wws: WorkerState
-                is_idle = not any([wws._processing for wws in groups[group]])
-                bytes = -group_bytes[group]
-                return (is_idle, bytes)
+            priority = dask.order.order(dsk, dependencies=stripped_deps)
 
-            idle = sorted(groups, key=_key)
+        return self.update_graph(
+            client,
+            dsk,
+            keys,
+            dependencies,
+            restrictions,
+            priority,
+            loose_restrictions,
+            resources,
+            submitting_task,
+            retries,
+            user_priority,
+            actors,
+            fifo_timeout,
+            annotations,
+        )
+
+    def update_graph(
+        self,
+        client=None,
+        tasks=None,
+        keys=None,
+        dependencies=None,
+        restrictions=None,
+        priority=None,
+        loose_restrictions=None,
+        resources=None,
+        submitting_task=None,
+        retries=None,
+        user_priority=0,
+        actors=None,
+        fifo_timeout=0,
+        annotations=None,
+    ):
+        """
+        Add new computations to the internal dask graph
+
+        This happens whenever the Client calls submit, map, get, or compute.
+        """
+        parent: SchedulerState = cast(SchedulerState, self)
+        start = time()
+        fifo_timeout = parse_timedelta(fifo_timeout)
+        keys = set(keys)
+        if len(tasks) > 1:
+            self.log_event(
+                ["all", client], {"action": "update_graph", "count": len(tasks)}
+            )
+
+        # Remove aliases
+        for k in list(tasks):
+            if tasks[k] is k:
+                del tasks[k]
+
+        dependencies = dependencies or {}
+
+        n = 0
+        while len(tasks) != n:  # walk through new tasks, cancel any bad deps
+            n = len(tasks)
+            for k, deps in list(dependencies.items()):
+                if any(
+                    dep not in parent._tasks and dep not in tasks for dep in deps
+                ):  # bad key
+                    logger.info("User asked for computation on lost data, %s", k)
+                    del tasks[k]
+                    del dependencies[k]
+                    if k in keys:
+                        keys.remove(k)
+                    self.report({"op": "cancelled-key", "key": k}, client=client)
+                    self.client_releases_keys(keys=[k], client=client)
+
+        # Avoid computation that is already finished
+        ts: TaskState
+        already_in_memory = set()  # tasks that are already done
+        for k, v in dependencies.items():
+            if v and k in parent._tasks:
+                ts = parent._tasks[k]
+                if ts._state in ("memory", "erred"):
+                    already_in_memory.add(k)
+
+        dts: TaskState
+        if already_in_memory:
+            dependents = dask.core.reverse_dict(dependencies)
+            stack = list(already_in_memory)
+            done = set(already_in_memory)
+            while stack:  # remove unnecessary dependencies
+                key = stack.pop()
+                ts = parent._tasks[key]
+                try:
+                    deps = dependencies[key]
+                except KeyError:
+                    deps = self.dependencies[key]
+                for dep in deps:
+                    if dep in dependents:
+                        child_deps = dependents[dep]
+                    else:
+                        child_deps = self.dependencies[dep]
+                    if all(d in done for d in child_deps):
+                        if dep in parent._tasks and dep not in done:
+                            done.add(dep)
+                            stack.append(dep)
+
+            for d in done:
+                tasks.pop(d, None)
+                dependencies.pop(d, None)
+
+        # Get or create task states
+        stack = list(keys)
+        touched_keys = set()
+        touched_tasks = []
+        while stack:
+            k = stack.pop()
+            if k in touched_keys:
+                continue
+            # XXX Have a method get_task_state(self, k) ?
+            ts = parent._tasks.get(k)
+            if ts is None:
+                ts = self.new_task(k, tasks.get(k), "released")
+            elif not ts._run_spec:
+                ts._run_spec = tasks.get(k)
+
+            touched_keys.add(k)
+            touched_tasks.append(ts)
+            stack.extend(dependencies.get(k, ()))
+
+        self.client_desires_keys(keys=keys, client=client)
+
+        # Add dependencies
+        for key, deps in dependencies.items():
+            ts = parent._tasks.get(key)
+            if ts is None or ts._dependencies:
+                continue
+            for dep in deps:
+                dts = parent._tasks[dep]
+                ts.add_dependency(dts)
+
+        # Compute priorities
+        if isinstance(user_priority, Number):
+            user_priority = {k: user_priority for k in tasks}
+
+        annotations = annotations or {}
+        restrictions = restrictions or {}
+        loose_restrictions = loose_restrictions or []
+        resources = resources or {}
+        retries = retries or {}
+
+        # Override existing taxonomy with per task annotations
+        if annotations:
+            if "priority" in annotations:
+                user_priority.update(annotations["priority"])
+
+            if "workers" in annotations:
+                restrictions.update(annotations["workers"])
+
+            if "allow_other_workers" in annotations:
+                loose_restrictions.extend(
+                    k for k, v in annotations["allow_other_workers"].items() if v
+                )
+
+            if "retries" in annotations:
+                retries.update(annotations["retries"])
+
+            if "resources" in annotations:
+                resources.update(annotations["resources"])
+
+            for a, kv in annotations.items():
+                for k, v in kv.items():
+                    ts = parent._tasks[k]
+                    ts._annotations[a] = v
+
+        # Add actors
+        if actors is True:
+            actors = list(keys)
+        for actor in actors or []:
+            ts = parent._tasks[actor]
+            ts._actor = True
+
+        priority = priority or dask.order.order(
+            tasks
+        )  # TODO: define order wrt old graph
+
+        if submitting_task:  # sub-tasks get better priority than parent tasks
+            ts = parent._tasks.get(submitting_task)
+            if ts is not None:
+                generation = ts._priority[0] - 0.01
+            else:  # super-task already cleaned up
+                generation = self.generation
+        elif self._last_time + fifo_timeout < start:
+            self.generation += 1  # older graph generations take precedence
+            generation = self.generation
+            self._last_time = start
+        else:
+            generation = self.generation
+
+        for key in set(priority) & touched_keys:
+            ts = parent._tasks[key]
+            if ts._priority is None:
+                ts._priority = (-(user_priority.get(key, 0)), generation, priority[key])
+
+        # Ensure all runnables have a priority
+        runnables = [ts for ts in touched_tasks if ts._run_spec]
+        for ts in runnables:
+            if ts._priority is None and ts._run_spec:
+                ts._priority = (self.generation, 0)
+
+        if restrictions:
+            # *restrictions* is a dict keying task ids to lists of
+            # restriction specifications (either worker names or addresses)
+            for k, v in restrictions.items():
+                if v is None:
+                    continue
+                ts = parent._tasks.get(k)
+                if ts is None:
+                    continue
+                ts._host_restrictions = set()
+                ts._worker_restrictions = set()
+                for w in v:
+                    try:
+                        w = self.coerce_address(w)
+                    except ValueError:
+                        # Not a valid address, but perhaps it's a hostname
+                        ts._host_restrictions.add(w)
+                    else:
+                        ts._worker_restrictions.add(w)
+
+            if loose_restrictions:
+                for k in loose_restrictions:
+                    ts = parent._tasks[k]
+                    ts._loose_restrictions = True
+
+        if resources:
+            for k, v in resources.items():
+                if v is None:
+                    continue
+                assert isinstance(v, dict)
+                ts = parent._tasks.get(k)
+                if ts is None:
+                    continue
+                ts._resource_restrictions = v
+
+        if retries:
+            for k, v in retries.items():
+                assert isinstance(v, int)
+                ts = parent._tasks.get(k)
+                if ts is None:
+                    continue
+                ts._retries = v
+
+        # Compute recommendations
+        recommendations: dict = {}
+
+        for ts in sorted(runnables, key=operator.attrgetter("priority"), reverse=True):
+            if ts._state == "released" and ts._run_spec:
+                recommendations[ts._key] = "waiting"
+
+        for ts in touched_tasks:
+            for dts in ts._dependencies:
+                if dts._exception_blame:
+                    ts._exception_blame = dts._exception_blame
+                    recommendations[ts._key] = "erred"
+                    break
+
+        for plugin in self.plugins[:]:
+            try:
+                plugin.update_graph(
+                    self,
+                    client=client,
+                    tasks=tasks,
+                    keys=keys,
+                    restrictions=restrictions or {},
+                    dependencies=dependencies,
+                    priority=priority,
+                    loose_restrictions=loose_restrictions,
+                    resources=resources,
+                    annotations=annotations,
+                )
+            except Exception as e:
+                logger.exception(e)
+
+        self.transitions(recommendations)
+
+        for ts in touched_tasks:
+            if ts._state in ("memory", "erred"):
+                self.report_on_key(ts=ts, client=client)
+
+        end = time()
+        if self.digests is not None:
+            self.digests["update-graph-duration"].add(end - start)
+
+        # TODO: balance workers
+
+    def new_task(self, key, spec, state):
+        """ Create a new task, and associated states """
+        parent: SchedulerState = cast(SchedulerState, self)
+        ts: TaskState = TaskState(key, spec)
+        tp: TaskPrefix
+        tg: TaskGroup
+        ts._state = state
+        prefix_key = key_split(key)
+        try:
+            tp = self.task_prefixes[prefix_key]
+        except KeyError:
+            self.task_prefixes[prefix_key] = tp = TaskPrefix(prefix_key)
+        ts._prefix = tp
+
+        group_key = ts._group_key
+        try:
+            tg = self.task_groups[group_key]
+        except KeyError:
+            self.task_groups[group_key] = tg = TaskGroup(group_key)
+            tg._prefix = tp
+            tp._groups.append(tg)
+        tg.add(ts)
+        parent._tasks[key] = ts
+        return ts
+
+    def stimulus_task_finished(self, key=None, worker=None, **kwargs):
+        """ Mark that a task has finished execution on a particular worker """
+        parent: SchedulerState = cast(SchedulerState, self)
+        logger.debug("Stimulus task finished %s, %s", key, worker)
+
+        tasks: dict = parent._tasks
+        ts: TaskState = tasks.get(key)
+        if ts is None:
+            return {}
+        ws: WorkerState = parent._workers_dv[worker]
+        ts._metadata.update(kwargs["metadata"])
+
+        recommendations: dict
+        if ts._state == "processing":
+            recommendations = self.transition(key, "memory", worker=worker, **kwargs)
+
+            if ts._state == "memory":
+                assert ws in ts._who_has
+        else:
+            logger.debug(
+                "Received already computed task, worker: %s, state: %s"
+                ", key: %s, who_has: %s",
+                worker,
+                ts._state,
+                key,
+                ts._who_has,
+            )
+            if ws not in ts._who_has:
+                self.worker_send(worker, {"op": "release-task", "key": key})
+            recommendations = {}
 
-            to_close = []
-            n_remain = len(self.workers)
+        return recommendations
 
-            while idle:
-                group = idle.pop()
-                if n is None and any([ws._processing for ws in groups[group]]):
-                    break
+    def stimulus_task_erred(
+        self, key=None, worker=None, exception=None, traceback=None, **kwargs
+    ):
+        """ Mark that a task has erred on a particular worker """
+        parent: SchedulerState = cast(SchedulerState, self)
+        logger.debug("Stimulus task erred %s, %s", key, worker)
 
-                if minimum and n_remain - len(groups[group]) < minimum:
-                    break
+        ts: TaskState = parent._tasks.get(key)
+        if ts is None:
+            return {}
 
-                limit -= limit_bytes[group]
+        recommendations: dict
+        if ts._state == "processing":
+            retries = ts._retries
+            if retries > 0:
+                ts._retries = retries - 1
+                recommendations = self.transition(key, "waiting")
+            else:
+                recommendations = self.transition(
+                    key,
+                    "erred",
+                    cause=key,
+                    exception=exception,
+                    traceback=traceback,
+                    worker=worker,
+                    **kwargs,
+                )
+        else:
+            recommendations = {}
 
-                if (n is not None and n_remain - len(groups[group]) >= target) or (
-                    memory_ratio is not None and limit >= memory_ratio * total
-                ):
-                    to_close.append(group)
-                    n_remain -= len(groups[group])
+        return recommendations
 
-                else:
-                    break
+    def stimulus_missing_data(
+        self, cause=None, key=None, worker=None, ensure=True, **kwargs
+    ):
+        """ Mark that certain keys have gone missing.  Recover. """
+        parent: SchedulerState = cast(SchedulerState, self)
+        with log_errors():
+            logger.debug("Stimulus missing data %s, %s", key, worker)
 
-            result = [getattr(ws, attribute) for g in to_close for ws in groups[g]]
-            if result:
-                logger.debug("Suggest closing workers: %s", result)
+            ts: TaskState = parent._tasks.get(key)
+            if ts is None or ts._state == "memory":
+                return {}
+            cts: TaskState = parent._tasks.get(cause)
 
-            return result
+            recommendations: dict = {}
 
-    async def retire_workers(
-        self,
-        comm=None,
-        workers=None,
-        remove=True,
-        close_workers=False,
-        names=None,
-        lock=True,
-        **kwargs,
-    ) -> dict:
-        """Gracefully retire workers from cluster
+            if cts is not None and cts._state == "memory":  # couldn't find this
+                ws: WorkerState
+                for ws in cts._who_has:  # TODO: this behavior is extreme
+                    ws._has_what.remove(cts)
+                    ws._nbytes -= cts.get_nbytes()
+                cts._who_has.clear()
+                recommendations[cause] = "released"
 
-        Parameters
-        ----------
-        workers: list (optional)
-            List of worker addresses to retire.
-            If not provided we call ``workers_to_close`` which finds a good set
-        workers_names: list (optional)
-            List of worker names to retire.
-        remove: bool (defaults to True)
-            Whether or not to remove the worker metadata immediately or else
-            wait for the worker to contact us
-        close_workers: bool (defaults to False)
-            Whether or not to actually close the worker explicitly from here.
-            Otherwise we expect some external job scheduler to finish off the
-            worker.
-        **kwargs: dict
-            Extra options to pass to workers_to_close to determine which
-            workers we should drop
+            if key:
+                recommendations[key] = "released"
 
-        Returns
-        -------
-        Dictionary mapping worker ID/address to dictionary of information about
-        that worker for each retired worker.
+            self.transitions(recommendations)
 
-        See Also
-        --------
-        Scheduler.workers_to_close
-        """
-        ws: WorkerState
+            if parent._validate:
+                assert cause not in self.who_has
+
+            return {}
+
+    def stimulus_retry(self, comm=None, keys=None, client=None):
+        parent: SchedulerState = cast(SchedulerState, self)
+        logger.info("Client %s requests to retry %d keys", client, len(keys))
+        if client:
+            self.log_event(client, {"action": "retry", "count": len(keys)})
+
+        stack = list(keys)
+        seen = set()
+        roots = []
         ts: TaskState
+        dts: TaskState
+        while stack:
+            key = stack.pop()
+            seen.add(key)
+            ts = parent._tasks[key]
+            erred_deps = [dts._key for dts in ts._dependencies if dts._state == "erred"]
+            if erred_deps:
+                stack.extend(erred_deps)
+            else:
+                roots.append(key)
+
+        recommendations: dict = {key: "waiting" for key in roots}
+        self.transitions(recommendations)
+
+        if parent._validate:
+            for key in seen:
+                assert not parent._tasks[key].exception_blame
+
+        return tuple(seen)
+
+    async def remove_worker(self, comm=None, address=None, safe=False, close=True):
+        """
+        Remove worker from cluster
+
+        We do this when a worker reports that it plans to leave or when it
+        appears to be unresponsive.  This may send its tasks back to a released
+        state.
+        """
+        parent: SchedulerState = cast(SchedulerState, self)
         with log_errors():
-            async with self._lock if lock else empty_context:
-                if names is not None:
-                    if names:
-                        logger.info("Retire worker names %s", names)
-                    names = set(map(str, names))
-                    workers = [
-                        ws._address
-                        for ws in self.workers.values()
-                        if str(ws._name) in names
-                    ]
-                if workers is None:
-                    while True:
-                        try:
-                            workers = self.workers_to_close(**kwargs)
-                            if workers:
-                                workers = await self.retire_workers(
-                                    workers=workers,
-                                    remove=remove,
-                                    close_workers=close_workers,
-                                    lock=False,
-                                )
-                                return workers
-                            else:
-                                return {}
-                        except KeyError:  # keys left during replicate
-                            pass
-                workers = {self.workers[w] for w in workers if w in self.workers}
-                if not workers:
-                    return {}
-                logger.info("Retire workers %s", workers)
+            if self.status == Status.closed:
+                return
 
-                # Keys orphaned by retiring those workers
-                keys = set.union(*[w.has_what for w in workers])
-                keys = {ts._key for ts in keys if ts._who_has.issubset(workers)}
+            address = self.coerce_address(address)
 
-                other_workers = set(self.workers.values()) - workers
-                if keys:
-                    if other_workers:
-                        logger.info("Moving %d keys to other workers", len(keys))
-                        await self.replicate(
-                            keys=keys,
-                            workers=[ws._address for ws in other_workers],
-                            n=1,
-                            delete=False,
-                            lock=False,
+            if address not in parent._workers_dv:
+                return "already-removed"
+
+            host = get_address_host(address)
+
+            ws: WorkerState = parent._workers_dv[address]
+
+            self.log_event(
+                ["all", address],
+                {
+                    "action": "remove-worker",
+                    "worker": address,
+                    "processing-tasks": dict(ws._processing),
+                },
+            )
+            logger.info("Remove worker %s", ws)
+            if close:
+                with suppress(AttributeError, CommClosedError):
+                    self.stream_comms[address].send({"op": "close", "report": False})
+
+            self.remove_resources(address)
+
+            parent._host_info[host]["nthreads"] -= ws._nthreads
+            parent._host_info[host]["addresses"].remove(address)
+            parent._total_nthreads -= ws._nthreads
+
+            if not parent._host_info[host]["addresses"]:
+                del parent._host_info[host]
+
+            self.rpc.remove(address)
+            del self.stream_comms[address]
+            del parent._aliases[ws._name]
+            parent._idle.pop(ws._address, None)
+            parent._saturated.discard(ws)
+            del parent._workers[address]
+            ws.status = Status.closed
+            parent._total_occupancy -= ws._occupancy
+
+            recommendations: dict = {}
+
+            ts: TaskState
+            for ts in list(ws._processing):
+                k = ts._key
+                recommendations[k] = "released"
+                if not safe:
+                    ts._suspicious += 1
+                    ts._prefix._suspicious += 1
+                    if ts._suspicious > self.allowed_failures:
+                        del recommendations[k]
+                        e = pickle.dumps(
+                            KilledWorker(task=k, last_worker=ws.clean()), protocol=4
                         )
-                    else:
-                        return {}
+                        r = self.transition(k, "erred", exception=e, cause=k)
+                        recommendations.update(r)
+                        logger.info(
+                            "Task %s marked as failed because %d workers died"
+                            " while trying to run it",
+                            ts._key,
+                            self.allowed_failures,
+                        )
+
+            for ts in ws._has_what:
+                ts._who_has.remove(ws)
+                if not ts._who_has:
+                    if ts._run_spec:
+                        recommendations[ts._key] = "released"
+                    else:  # pure data
+                        recommendations[ts._key] = "forgotten"
+            ws._has_what.clear()
+
+            self.transitions(recommendations)
+
+            for plugin in self.plugins[:]:
+                try:
+                    result = plugin.remove_worker(scheduler=self, worker=address)
+                    if inspect.isawaitable(result):
+                        await result
+                except Exception as e:
+                    logger.exception(e)
 
-                worker_keys = {ws._address: ws.identity() for ws in workers}
-                if close_workers and worker_keys:
-                    await asyncio.gather(
-                        *[self.close_worker(worker=w, safe=True) for w in worker_keys]
-                    )
-                if remove:
-                    await asyncio.gather(
-                        *[self.remove_worker(address=w, safe=True) for w in worker_keys]
-                    )
+            if not parent._workers_dv:
+                logger.info("Lost all workers")
 
-                self.log_event(
-                    "all",
-                    {
-                        "action": "retire-workers",
-                        "workers": worker_keys,
-                        "moved-keys": len(keys),
-                    },
-                )
-                self.log_event(list(worker_keys), {"action": "retired"})
+            for w in parent._workers_dv:
+                self.bandwidth_workers.pop((address, w), None)
+                self.bandwidth_workers.pop((w, address), None)
 
-                return worker_keys
+            def remove_worker_from_events():
+                # If the worker isn't registered anymore after the delay, remove from events
+                if address not in parent._workers_dv and address in self.events:
+                    del self.events[address]
 
-    def add_keys(self, comm=None, worker=None, keys=()):
-        """
-        Learn that a worker has certain keys
+            cleanup_delay = parse_timedelta(
+                dask.config.get("distributed.scheduler.events-cleanup-delay")
+            )
+            self.loop.call_later(cleanup_delay, remove_worker_from_events)
+            logger.debug("Removed worker %s", ws)
 
-        This should not be used in practice and is mostly here for legacy
-        reasons.  However, it is sent by workers from time to time.
-        """
-        if worker not in self.workers:
-            return "not found"
-        ws: WorkerState = self.workers[worker]
+        return "OK"
+
+    def stimulus_cancel(self, comm, keys=None, client=None, force=False):
+        """ Stop execution on a list of keys """
+        logger.info("Client %s requests to cancel %d keys", client, len(keys))
+        if client:
+            self.log_event(
+                client, {"action": "cancel", "count": len(keys), "force": force}
+            )
         for key in keys:
-            ts: TaskState = self.tasks.get(key)
-            if ts is not None and ts._state == "memory":
-                if ts not in ws._has_what:
-                    ws._nbytes += ts.get_nbytes()
-                    ws._has_what.add(ts)
-                    ts._who_has.add(ws)
-            else:
-                self.worker_send(
-                    worker, {"op": "delete-data", "keys": [key], "report": False}
+            self.cancel_key(key, client, force=force)
+
+    def cancel_key(self, key, client, retries=5, force=False):
+        """ Cancel a particular key and all dependents """
+        # TODO: this should be converted to use the transition mechanism
+        parent: SchedulerState = cast(SchedulerState, self)
+        ts: TaskState = parent._tasks.get(key)
+        dts: TaskState
+        try:
+            cs: ClientState = parent._clients[client]
+        except KeyError:
+            return
+        if ts is None or not ts._who_wants:  # no key yet, lets try again in a moment
+            if retries:
+                self.loop.call_later(
+                    0.2, lambda: self.cancel_key(key, client, retries - 1)
                 )
+            return
+        if force or ts._who_wants == {cs}:  # no one else wants this key
+            for dts in list(ts._dependents):
+                self.cancel_key(dts._key, client, force=force)
+        logger.info("Scheduler cancels key %s.  Force=%s", key, force)
+        self.report({"op": "cancelled-key", "key": key})
+        clients = list(ts._who_wants) if force else [cs]
+        for cs in clients:
+            self.client_releases_keys(keys=[key], client=cs._client_key)
 
-        return "OK"
+    def client_desires_keys(self, keys=None, client=None):
+        parent: SchedulerState = cast(SchedulerState, self)
+        cs: ClientState = parent._clients.get(client)
+        if cs is None:
+            # For publish, queues etc.
+            parent._clients[client] = cs = ClientState(client)
+        ts: TaskState
+        for k in keys:
+            ts = parent._tasks.get(k)
+            if ts is None:
+                # For publish, queues etc.
+                ts = self.new_task(k, None, "released")
+            ts._who_wants.add(cs)
+            cs._wants_what.add(ts)
 
-    def update_data(
-        self, comm=None, who_has=None, nbytes=None, client=None, serializers=None
-    ):
-        """
-        Learn that new data has entered the network from an external source
+            if ts._state in ("memory", "erred"):
+                self.report_on_key(ts=ts, client=client)
 
-        See Also
-        --------
-        Scheduler.mark_key_in_memory
-        """
-        with log_errors():
-            who_has = {
-                k: [self.coerce_address(vv) for vv in v] for k, v in who_has.items()
-            }
-            logger.debug("Update data %s", who_has)
+    def client_releases_keys(self, keys=None, client=None):
+        """ Remove keys from client desired list """
 
-            for key, workers in who_has.items():
-                ts: TaskState = self.tasks.get(key)
-                if ts is None:
-                    ts: TaskState = self.new_task(key, None, "memory")
-                ts.state = "memory"
-                if key in nbytes:
-                    ts.set_nbytes(nbytes[key])
-                for w in workers:
-                    ws: WorkerState = self.workers[w]
-                    if ts not in ws._has_what:
-                        ws._nbytes += ts.get_nbytes()
-                        ws._has_what.add(ts)
-                        ts._who_has.add(ws)
-                self.report(
-                    {"op": "key-in-memory", "key": key, "workers": list(workers)}
-                )
+        parent: SchedulerState = cast(SchedulerState, self)
+        if not isinstance(keys, list):
+            keys = list(keys)
+        cs: ClientState = parent._clients[client]
+        recommendations: dict = {}
 
-            if client:
-                self.client_desires_keys(keys=list(who_has), client=client)
+        _client_releases_keys(parent, keys=keys, cs=cs, recommendations=recommendations)
+        self.transitions(recommendations)
 
-    def report_on_key(self, key: str = None, ts: TaskState = None, client: str = None):
-        if ts is None:
-            tasks: dict = self.tasks
-            ts = tasks.get(key)
-        elif key is None:
-            key = ts._key
-        else:
-            assert False, (key, ts)
-            return
+    def client_heartbeat(self, client=None):
+        """ Handle heartbeats from Client """
+        parent: SchedulerState = cast(SchedulerState, self)
+        cs: ClientState = parent._clients[client]
+        cs._last_seen = time()
 
-        if ts is None:
-            self.report({"op": "cancelled-key", "key": key}, client=client)
-        elif ts._state == "forgotten":
-            self.report({"op": "cancelled-key", "key": key}, ts=ts, client=client)
-        elif ts._state == "memory":
-            self.report({"op": "key-in-memory", "key": key}, ts=ts, client=client)
-        elif ts._state == "erred":
-            failing_ts: TaskState = ts._exception_blame
-            self.report(
-                {
-                    "op": "task-erred",
-                    "key": key,
-                    "exception": failing_ts._exception,
-                    "traceback": failing_ts._traceback,
-                },
-                ts=ts,
-                client=client,
-            )
+    ###################
+    # Task Validation #
+    ###################
 
-    async def feed(
-        self, comm, function=None, setup=None, teardown=None, interval="1s", **kwargs
-    ):
-        """
-        Provides a data Comm to external requester
+    def validate_released(self, key):
+        parent: SchedulerState = cast(SchedulerState, self)
+        ts: TaskState = parent._tasks[key]
+        dts: TaskState
+        assert ts._state == "released"
+        assert not ts._waiters
+        assert not ts._waiting_on
+        assert not ts._who_has
+        assert not ts._processing_on
+        assert not any([ts in dts._waiters for dts in ts._dependencies])
+        assert ts not in parent._unrunnable
 
-        Caution: this runs arbitrary Python code on the scheduler.  This should
-        eventually be phased out.  It is mostly used by diagnostics.
-        """
-        if not dask.config.get("distributed.scheduler.pickle"):
-            logger.warn(
-                "Tried to call 'feed' route with custom functions, but "
-                "pickle is disallowed.  Set the 'distributed.scheduler.pickle'"
-                "config value to True to use the 'feed' route (this is mostly "
-                "commonly used with progress bars)"
-            )
-            return
+    def validate_waiting(self, key):
+        parent: SchedulerState = cast(SchedulerState, self)
+        ts: TaskState = parent._tasks[key]
+        dts: TaskState
+        assert ts._waiting_on
+        assert not ts._who_has
+        assert not ts._processing_on
+        assert ts not in parent._unrunnable
+        for dts in ts._dependencies:
+            # We are waiting on a dependency iff it's not stored
+            assert (not not dts._who_has) != (dts in ts._waiting_on)
+            assert ts in dts._waiters  # XXX even if dts._who_has?
 
-        interval = parse_timedelta(interval)
-        with log_errors():
-            if function:
-                function = pickle.loads(function)
-            if setup:
-                setup = pickle.loads(setup)
-            if teardown:
-                teardown = pickle.loads(teardown)
-            state = setup(self) if setup else None
-            if inspect.isawaitable(state):
-                state = await state
-            try:
-                while self.status == Status.running:
-                    if state is None:
-                        response = function(self)
-                    else:
-                        response = function(self, state)
-                    await comm.write(response)
-                    await asyncio.sleep(interval)
-            except (EnvironmentError, CommClosedError):
-                pass
-            finally:
-                if teardown:
-                    teardown(self, state)
+    def validate_processing(self, key):
+        parent: SchedulerState = cast(SchedulerState, self)
+        ts: TaskState = parent._tasks[key]
+        dts: TaskState
+        assert not ts._waiting_on
+        ws: WorkerState = ts._processing_on
+        assert ws
+        assert ts in ws._processing
+        assert not ts._who_has
+        for dts in ts._dependencies:
+            assert dts._who_has
+            assert ts in dts._waiters
 
-    def log_worker_event(self, worker=None, topic=None, msg=None):
-        self.log_event(topic, msg)
+    def validate_memory(self, key):
+        parent: SchedulerState = cast(SchedulerState, self)
+        ts: TaskState = parent._tasks[key]
+        dts: TaskState
+        assert ts._who_has
+        assert not ts._processing_on
+        assert not ts._waiting_on
+        assert ts not in parent._unrunnable
+        for dts in ts._dependents:
+            assert (dts in ts._waiters) == (dts._state in ("waiting", "processing"))
+            assert ts not in dts._waiting_on
 
-    def subscribe_worker_status(self, comm=None):
-        WorkerStatusPlugin(self, comm)
-        ident = self.identity()
-        for v in ident["workers"].values():
-            del v["metrics"]
-            del v["last_seen"]
-        return ident
+    def validate_no_worker(self, key):
+        parent: SchedulerState = cast(SchedulerState, self)
+        ts: TaskState = parent._tasks[key]
+        dts: TaskState
+        assert ts in parent._unrunnable
+        assert not ts._waiting_on
+        assert ts in parent._unrunnable
+        assert not ts._processing_on
+        assert not ts._who_has
+        for dts in ts._dependencies:
+            assert dts._who_has
+
+    def validate_erred(self, key):
+        parent: SchedulerState = cast(SchedulerState, self)
+        ts: TaskState = parent._tasks[key]
+        assert ts._exception_blame
+        assert not ts._who_has
+
+    def validate_key(self, key, ts: TaskState = None):
+        parent: SchedulerState = cast(SchedulerState, self)
+        try:
+            if ts is None:
+                ts = parent._tasks.get(key)
+            if ts is None:
+                logger.debug("Key lost: %s", key)
+            else:
+                ts.validate()
+                try:
+                    func = getattr(self, "validate_" + ts._state.replace("-", "_"))
+                except AttributeError:
+                    logger.error(
+                        "self.validate_%s not found", ts._state.replace("-", "_")
+                    )
+                else:
+                    func(key)
+        except Exception as e:
+            logger.exception(e)
+            if LOG_PDB:
+                import pdb
 
-    def get_processing(self, comm=None, workers=None):
-        ws: WorkerState
-        ts: TaskState
-        if workers is not None:
-            workers = set(map(self.coerce_address, workers))
-            return {w: [ts._key for ts in self.workers[w].processing] for w in workers}
-        else:
-            return {
-                w: [ts._key for ts in ws._processing] for w, ws in self.workers.items()
-            }
+                pdb.set_trace()
+            raise
 
-    def get_who_has(self, comm=None, keys=None):
-        ws: WorkerState
-        ts: TaskState
-        if keys is not None:
-            return {
-                k: [ws._address for ws in self.tasks[k].who_has]
-                if k in self.tasks
-                else []
-                for k in keys
-            }
-        else:
-            return {
-                key: [ws._address for ws in ts._who_has]
-                for key, ts in self.tasks.items()
-            }
+    def validate_state(self, allow_overlap=False):
+        parent: SchedulerState = cast(SchedulerState, self)
+        validate_state(parent._tasks, parent._workers, parent._clients)
 
-    def get_has_what(self, comm=None, workers=None):
-        ws: WorkerState
-        ts: TaskState
-        if workers is not None:
-            workers = map(self.coerce_address, workers)
-            return {
-                w: [ts._key for ts in self.workers[w].has_what]
-                if w in self.workers
-                else []
-                for w in workers
-            }
-        else:
-            return {
-                w: [ts._key for ts in ws._has_what] for w, ws in self.workers.items()
-            }
+        if not (set(parent._workers_dv) == set(self.stream_comms)):
+            raise ValueError("Workers not the same in all collections")
 
-    def get_ncores(self, comm=None, workers=None):
         ws: WorkerState
-        if workers is not None:
-            workers = map(self.coerce_address, workers)
-            return {w: self.workers[w].nthreads for w in workers if w in self.workers}
-        else:
-            return {w: ws._nthreads for w, ws in self.workers.items()}
+        for w, ws in parent._workers_dv.items():
+            assert isinstance(w, str), (type(w), w)
+            assert isinstance(ws, WorkerState), (type(ws), ws)
+            assert ws._address == w
+            if not ws._processing:
+                assert not ws._occupancy
+                assert ws._address in parent._idle_dv
 
-    async def get_call_stack(self, comm=None, keys=None):
         ts: TaskState
-        dts: TaskState
-        if keys is not None:
-            stack = list(keys)
-            processing = set()
-            while stack:
-                key = stack.pop()
-                ts = self.tasks[key]
-                if ts._state == "waiting":
-                    stack.extend([dts._key for dts in ts._dependencies])
-                elif ts._state == "processing":
-                    processing.add(ts)
-
-            workers = defaultdict(list)
-            for ts in processing:
-                if ts._processing_on:
-                    workers[ts._processing_on.address].append(ts._key)
-        else:
-            workers = {w: None for w in self.workers}
+        for k, ts in parent._tasks.items():
+            assert isinstance(ts, TaskState), (type(ts), ts)
+            assert ts._key == k
+            self.validate_key(k, ts)
 
-        if not workers:
-            return {}
+        c: str
+        cs: ClientState
+        for c, cs in parent._clients.items():
+            # client=None is often used in tests...
+            assert c is None or type(c) == str, (type(c), c)
+            assert type(cs) == ClientState, (type(cs), cs)
+            assert cs._client_key == c
 
-        results = await asyncio.gather(
-            *(self.rpc(w).call_stack(keys=v) for w, v in workers.items())
-        )
-        response = {w: r for w, r in zip(workers, results) if r}
-        return response
+        a = {w: ws._nbytes for w, ws in parent._workers_dv.items()}
+        b = {
+            w: sum(ts.get_nbytes() for ts in ws._has_what)
+            for w, ws in parent._workers_dv.items()
+        }
+        assert a == b, (a, b)
 
-    def get_nbytes(self, comm=None, keys=None, summary=True):
-        ts: TaskState
-        with log_errors():
-            if keys is not None:
-                result = {k: self.tasks[k].nbytes for k in keys}
-            else:
-                result = {
-                    k: ts._nbytes for k, ts in self.tasks.items() if ts._nbytes >= 0
-                }
+        actual_total_occupancy = 0
+        for worker, ws in parent._workers_dv.items():
+            assert abs(sum(ws._processing.values()) - ws._occupancy) < 1e-8
+            actual_total_occupancy += ws._occupancy
 
-            if summary:
-                out = defaultdict(lambda: 0)
-                for k, v in result.items():
-                    out[key_split(k)] += v
-                result = dict(out)
+        assert abs(actual_total_occupancy - parent._total_occupancy) < 1e-8, (
+            actual_total_occupancy,
+            parent._total_occupancy,
+        )
 
-            return result
+    ###################
+    # Manage Messages #
+    ###################
 
-    def get_comm_cost(self, ts: TaskState, ws: WorkerState):
-        """
-        Get the estimated communication cost (in s.) to compute the task
-        on the given worker.
+    def report(self, msg: dict, ts: TaskState = None, client: str = None):
         """
-        dts: TaskState
-        deps: set = ts._dependencies - ws._has_what
-        nbytes: Py_ssize_t = 0
-        bandwidth: double = self.bandwidth
-        for dts in deps:
-            nbytes += dts._nbytes
-        return nbytes / bandwidth
+        Publish updates to all listening Queues and Comms
 
-    def get_task_duration(self, ts: TaskState, default: double = -1):
-        """
-        Get the estimated computation cost of the given task
-        (not including any communication cost).
+        If the message contains a key then we only send the message to those
+        comms that care about the key.
         """
-        duration: double = ts._prefix._duration_average
-        if duration < 0:
-            s: set = self.unknown_durations[ts._prefix._name]
-            s.add(ts)
-            if default < 0:
-                duration = UNKNOWN_TASK_DURATION
-            else:
-                duration = default
+        parent: SchedulerState = cast(SchedulerState, self)
+        if ts is None:
+            msg_key = msg.get("key")
+            if msg_key is not None:
+                tasks: dict = parent._tasks
+                ts = tasks.get(msg_key)
 
-        return duration
+        cs: ClientState
+        client_comms: dict = self.client_comms
+        client_keys: list
+        if ts is None:
+            # Notify all clients
+            client_keys = list(client_comms)
+        elif client is None:
+            # Notify clients interested in key
+            client_keys = [cs._client_key for cs in ts._who_wants]
+        else:
+            # Notify clients interested in key (including `client`)
+            client_keys = [
+                cs._client_key for cs in ts._who_wants if cs._client_key != client
+            ]
+            client_keys.append(client)
 
-    def run_function(self, stream, function, args=(), kwargs={}, wait=True):
-        """Run a function within this process
+        k: str
+        for k in client_keys:
+            c = client_comms.get(k)
+            if c is None:
+                continue
+            try:
+                c.send(msg)
+                # logger.debug("Scheduler sends message to client %s", msg)
+            except CommClosedError:
+                if self.status == Status.running:
+                    logger.critical("Tried writing to closed comm: %s", msg)
 
-        See Also
-        --------
-        Client.run_on_scheduler:
+    async def add_client(self, comm, client=None, versions=None):
+        """Add client to network
+
+        We listen to all future messages from this Comm.
         """
-        from .worker import run
+        parent: SchedulerState = cast(SchedulerState, self)
+        assert client is not None
+        comm.name = "Scheduler->Client"
+        logger.info("Receive client connection: %s", client)
+        self.log_event(["all", client], {"action": "add-client", "client": client})
+        parent._clients[client] = ClientState(client, versions=versions)
 
-        self.log_event("all", {"action": "run-function", "function": function})
-        return run(self, stream, function=function, args=args, kwargs=kwargs, wait=wait)
+        for plugin in self.plugins[:]:
+            try:
+                plugin.add_client(scheduler=self, client=client)
+            except Exception as e:
+                logger.exception(e)
 
-    def set_metadata(self, comm=None, keys=None, value=None):
         try:
-            metadata = self.task_metadata
-            for key in keys[:-1]:
-                if key not in metadata or not isinstance(metadata[key], (dict, list)):
-                    metadata[key] = dict()
-                metadata = metadata[key]
-            metadata[keys[-1]] = value
-        except Exception as e:
-            import pdb
+            bcomm = BatchedSend(interval="2ms", loop=self.loop)
+            bcomm.start(comm)
+            self.client_comms[client] = bcomm
+            msg = {"op": "stream-start"}
+            ws: WorkerState
+            version_warning = version_module.error_message(
+                version_module.get_versions(),
+                {w: ws._versions for w, ws in parent._workers_dv.items()},
+                versions,
+            )
+            msg.update(version_warning)
+            bcomm.send(msg)
 
-            pdb.set_trace()
+            try:
+                await self.handle_stream(comm=comm, extra={"client": client})
+            finally:
+                self.remove_client(client=client)
+                logger.debug("Finished handling client %s", client)
+        finally:
+            if not comm.closed():
+                self.client_comms[client].send({"op": "stream-closed"})
+            try:
+                if not shutting_down():
+                    await self.client_comms[client].close()
+                    del self.client_comms[client]
+                    if self.status == Status.running:
+                        logger.info("Close client connection: %s", client)
+            except TypeError:  # comm becomes None during GC
+                pass
 
-    def get_metadata(self, comm=None, keys=None, default=no_default):
-        metadata = self.task_metadata
-        for key in keys[:-1]:
-            metadata = metadata[key]
+    def remove_client(self, client=None):
+        """ Remove client from network """
+        parent: SchedulerState = cast(SchedulerState, self)
+        if self.status == Status.running:
+            logger.info("Remove client %s", client)
+        self.log_event(["all", client], {"action": "remove-client", "client": client})
         try:
-            return metadata[keys[-1]]
+            cs: ClientState = parent._clients[client]
         except KeyError:
-            if default != no_default:
-                return default
-            else:
-                raise
+            # XXX is this a legitimate condition?
+            pass
+        else:
+            ts: TaskState
+            self.client_releases_keys(
+                keys=[ts._key for ts in cs._wants_what], client=cs._client_key
+            )
+            del parent._clients[client]
+
+            for plugin in self.plugins[:]:
+                try:
+                    plugin.remove_client(scheduler=self, client=client)
+                except Exception as e:
+                    logger.exception(e)
+
+        def remove_client_from_events():
+            # If the client isn't registered anymore after the delay, remove from events
+            if client not in parent._clients and client in self.events:
+                del self.events[client]
+
+        cleanup_delay = parse_timedelta(
+            dask.config.get("distributed.scheduler.events-cleanup-delay")
+        )
+        self.loop.call_later(cleanup_delay, remove_client_from_events)
+
+    def send_task_to_worker(self, worker, ts: TaskState, duration=None):
+        """ Send a single computational task to a worker """
+        parent: SchedulerState = cast(SchedulerState, self)
+        try:
+            msg: dict = _task_to_msg(parent, ts, duration)
+            self.worker_send(worker, msg)
+        except Exception as e:
+            logger.exception(e)
+            if LOG_PDB:
+                import pdb
 
-    def get_task_status(self, comm=None, keys=None):
-        return {
-            key: (self.tasks[key].state if key in self.tasks else None) for key in keys
-        }
+                pdb.set_trace()
+            raise
 
-    def get_task_stream(self, comm=None, start=None, stop=None, count=None):
-        from distributed.diagnostics.task_stream import TaskStreamPlugin
+    def handle_uncaught_error(self, **msg):
+        logger.exception(clean_exception(**msg)[1])
 
-        self.add_plugin(TaskStreamPlugin, idempotent=True)
-        tsp = [p for p in self.plugins if isinstance(p, TaskStreamPlugin)][0]
-        return tsp.collect(start=start, stop=stop, count=count)
+    def handle_task_finished(self, key=None, worker=None, **msg):
+        parent: SchedulerState = cast(SchedulerState, self)
+        if worker not in parent._workers_dv:
+            return
+        validate_key(key)
+        r = self.stimulus_task_finished(key=key, worker=worker, **msg)
+        self.transitions(r)
 
-    def start_task_metadata(self, comm=None, name=None):
-        plugin = CollectTaskMetaDataPlugin(scheduler=self, name=name)
+    def handle_task_erred(self, key=None, **msg):
+        r = self.stimulus_task_erred(key=key, **msg)
+        self.transitions(r)
 
-        self.add_plugin(plugin)
+    def handle_release_data(self, key=None, worker=None, client=None, **msg):
+        parent: SchedulerState = cast(SchedulerState, self)
+        ts: TaskState = parent._tasks.get(key)
+        if ts is None:
+            return
+        ws: WorkerState = parent._workers_dv[worker]
+        if ts._processing_on != ws:
+            return
+        r = self.stimulus_missing_data(key=key, ensure=False, **msg)
+        self.transitions(r)
 
-    def stop_task_metadata(self, comm=None, name=None):
-        plugins = [
-            p
-            for p in self.plugins
-            if isinstance(p, CollectTaskMetaDataPlugin) and p.name == name
-        ]
-        if len(plugins) != 1:
-            raise ValueError(
-                "Expected to find exactly one CollectTaskMetaDataPlugin "
-                f"with name {name} but found {len(plugins)}."
-            )
+    def handle_missing_data(self, key=None, errant_worker=None, **kwargs):
+        parent: SchedulerState = cast(SchedulerState, self)
+        logger.debug("handle missing data key=%s worker=%s", key, errant_worker)
+        self.log.append(("missing", key, errant_worker))
 
-        plugin = plugins[0]
-        self.remove_plugin(plugin)
-        return {"metadata": plugin.metadata, "state": plugin.state}
+        ts: TaskState = parent._tasks.get(key)
+        if ts is None or not ts._who_has:
+            return
+        if errant_worker in parent._workers_dv:
+            ws: WorkerState = parent._workers_dv[errant_worker]
+            if ws in ts._who_has:
+                ts._who_has.remove(ws)
+                ws._has_what.remove(ts)
+                ws._nbytes -= ts.get_nbytes()
+        if not ts._who_has:
+            if ts._run_spec:
+                self.transitions({key: "released"})
+            else:
+                self.transitions({key: "forgotten"})
 
-    async def register_worker_plugin(self, comm, plugin, name=None):
-        """ Registers a setup function, and call it on every worker """
-        self.worker_plugins.append({"plugin": plugin, "name": name})
+    def release_worker_data(self, comm=None, keys=None, worker=None):
+        parent: SchedulerState = cast(SchedulerState, self)
+        ws: WorkerState = parent._workers_dv[worker]
+        tasks = {parent._tasks[k] for k in keys}
+        removed_tasks = tasks & ws._has_what
+        ws._has_what -= removed_tasks
 
-        responses = await self.broadcast(
-            msg=dict(op="plugin-add", plugin=plugin, name=name)
-        )
-        return responses
+        ts: TaskState
+        recommendations: dict = {}
+        for ts in removed_tasks:
+            ws._nbytes -= ts.get_nbytes()
+            wh = ts._who_has
+            wh.remove(ws)
+            if not wh:
+                recommendations[ts._key] = "released"
+        if recommendations:
+            self.transitions(recommendations)
 
-    #####################
-    # State Transitions #
-    #####################
+    def handle_long_running(self, key=None, worker=None, compute_duration=None):
+        """A task has seceded from the thread pool
 
-    def _remove_from_processing(self, ts: TaskState, send_worker_msg=None):
-        """
-        Remove *ts* from the set of processing tasks.
+        We stop the task from being stolen in the future, and change task
+        duration accounting as if the task has stopped.
         """
-        workers: dict = cast(dict, self.workers)
+        parent: SchedulerState = cast(SchedulerState, self)
+        ts: TaskState = parent._tasks[key]
+        if "stealing" in self._extensions:
+            self._extensions["stealing"].remove_key_from_stealable(ts)
+
         ws: WorkerState = ts._processing_on
-        ts._processing_on = None
-        w: str = ws._address
-        if w in workers:  # may have been removed
-            duration = ws._processing.pop(ts)
-            if not ws._processing:
-                self.total_occupancy -= ws._occupancy
-                ws._occupancy = 0
-            else:
-                self.total_occupancy -= duration
-                ws._occupancy -= duration
-            self.check_idle_saturated(ws)
-            self.release_resources(ts, ws)
-            if send_worker_msg:
-                self.worker_send(w, send_worker_msg)
+        if ws is None:
+            logger.debug("Received long-running signal from duplicate task. Ignoring.")
+            return
 
-    def _add_to_memory(
-        self,
-        ts: TaskState,
-        ws: WorkerState,
-        recommendations: dict,
-        type=None,
-        typename=None,
-        **kwargs,
-    ):
-        """
-        Add *ts* to the set of in-memory tasks.
-        """
-        if self.validate:
-            assert ts not in ws._has_what
+        if compute_duration:
+            old_duration = ts._prefix._duration_average
+            new_duration = compute_duration
+            if old_duration < 0:
+                avg_duration = new_duration
+            else:
+                avg_duration = 0.5 * old_duration + 0.5 * new_duration
 
-        ts._who_has.add(ws)
-        ws._has_what.add(ts)
-        ws._nbytes += ts.get_nbytes()
+            ts._prefix._duration_average = avg_duration
 
-        deps: list = list(ts._dependents)
-        if len(deps) > 1:
-            deps.sort(key=operator.attrgetter("priority"), reverse=True)
+        ws._occupancy -= ws._processing[ts]
+        parent._total_occupancy -= ws._processing[ts]
+        ws._processing[ts] = 0
+        self.check_idle_saturated(ws)
 
-        dts: TaskState
-        s: set
-        for dts in deps:
-            s = dts._waiting_on
-            if ts in s:
-                s.discard(ts)
-                if not s:  # new task ready to run
-                    recommendations[dts._key] = "processing"
+    async def handle_worker(self, comm=None, worker=None):
+        """
+        Listen to responses from a single worker
 
-        for dts in ts._dependencies:
-            s = dts._waiters
-            s.discard(ts)
-            if not s and not dts._who_wants:
-                recommendations[dts._key] = "released"
+        This is the main loop for scheduler-worker interaction
 
-        if not ts._waiters and not ts._who_wants:
-            recommendations[ts._key] = "released"
-        else:
-            msg: dict = {"op": "key-in-memory", "key": ts._key}
-            if type is not None:
-                msg["type"] = type
-            self.report(msg)
+        See Also
+        --------
+        Scheduler.handle_client: Equivalent coroutine for clients
+        """
+        comm.name = "Scheduler connection to worker"
+        worker_comm = self.stream_comms[worker]
+        worker_comm.start(comm)
+        logger.info("Starting worker compute stream, %s", worker)
+        try:
+            await self.handle_stream(comm=comm, extra={"worker": worker})
+        finally:
+            if worker in self.stream_comms:
+                worker_comm.abort()
+                await self.remove_worker(address=worker)
 
-        ts.state = "memory"
-        ts._type = typename
-        ts._group._types.add(typename)
+    def add_plugin(self, plugin=None, idempotent=False, **kwargs):
+        """
+        Add external plugin to scheduler
 
-        cs: ClientState = self.clients["fire-and-forget"]
-        if ts in cs._wants_what:
-            self.client_releases_keys(client="fire-and-forget", keys=[ts._key])
+        See https://distributed.readthedocs.io/en/latest/plugins.html
+        """
+        if isinstance(plugin, type):
+            plugin = plugin(self, **kwargs)
 
-    def transition_released_waiting(self, key):
-        try:
-            tasks: dict = self.tasks
-            workers: dict = cast(dict, self.workers)
-            ts: TaskState = tasks[key]
-            dts: TaskState
+        if idempotent and any(isinstance(p, type(plugin)) for p in self.plugins):
+            return
 
-            if self.validate:
-                assert ts._run_spec
-                assert not ts._waiting_on
-                assert not ts._who_has
-                assert not ts._processing_on
-                assert not any([dts._state == "forgotten" for dts in ts._dependencies])
+        self.plugins.append(plugin)
 
-            if ts._has_lost_dependencies:
-                return {key: "forgotten"}
+    def remove_plugin(self, plugin):
+        """ Remove external plugin from scheduler """
+        self.plugins.remove(plugin)
 
-            ts.state = "waiting"
+    def worker_send(self, worker, msg):
+        """Send message to worker
 
-            recommendations: dict = {}
+        This also handles connection failures by adding a callback to remove
+        the worker on the next cycle.
+        """
+        stream_comms: dict = self.stream_comms
+        try:
+            stream_comms[worker].send(msg)
+        except (CommClosedError, AttributeError):
+            self.loop.add_callback(self.remove_worker, address=worker)
 
-            dts: TaskState
-            for dts in ts._dependencies:
-                if dts._exception_blame:
-                    ts._exception_blame = dts._exception_blame
-                    recommendations[key] = "erred"
-                    return recommendations
+    def client_send(self, client, msg):
+        """Send message to client"""
+        client_comms: dict = self.client_comms
+        c = client_comms.get(client)
+        if c is None:
+            return
+        try:
+            c.send(msg)
+        except CommClosedError:
+            if self.status == Status.running:
+                logger.critical("Tried writing to closed comm: %s", msg)
 
-            for dts in ts._dependencies:
-                dep = dts._key
-                if not dts._who_has:
-                    ts._waiting_on.add(dts)
-                if dts._state == "released":
-                    recommendations[dep] = "waiting"
-                else:
-                    dts._waiters.add(ts)
+    ############################
+    # Less common interactions #
+    ############################
 
-            ts._waiters = {dts for dts in ts._dependents if dts._state == "waiting"}
+    async def scatter(
+        self,
+        comm=None,
+        data=None,
+        workers=None,
+        client=None,
+        broadcast=False,
+        timeout=2,
+    ):
+        """Send data out to workers
 
-            if not ts._waiting_on:
-                if workers:
-                    recommendations[key] = "processing"
-                else:
-                    self.unrunnable.add(ts)
-                    ts.state = "no-worker"
+        See also
+        --------
+        Scheduler.broadcast:
+        """
+        parent: SchedulerState = cast(SchedulerState, self)
+        start = time()
+        while not parent._workers_dv:
+            await asyncio.sleep(0.2)
+            if time() > start + timeout:
+                raise TimeoutError("No workers found")
 
-            return recommendations
-        except Exception as e:
-            logger.exception(e)
-            if LOG_PDB:
-                import pdb
+        if workers is None:
+            ws: WorkerState
+            nthreads = {w: ws._nthreads for w, ws in parent._workers_dv.items()}
+        else:
+            workers = [self.coerce_address(w) for w in workers]
+            nthreads = {w: parent._workers_dv[w].nthreads for w in workers}
 
-                pdb.set_trace()
-            raise
+        assert isinstance(data, dict)
 
-    def transition_no_worker_waiting(self, key):
-        try:
-            tasks: dict = self.tasks
-            workers: dict = cast(dict, self.workers)
-            ts: TaskState = tasks[key]
-            dts: TaskState
+        keys, who_has, nbytes = await scatter_to_workers(
+            nthreads, data, rpc=self.rpc, report=False
+        )
 
-            if self.validate:
-                assert ts in self.unrunnable
-                assert not ts._waiting_on
-                assert not ts._who_has
-                assert not ts._processing_on
+        self.update_data(who_has=who_has, nbytes=nbytes, client=client)
 
-            self.unrunnable.remove(ts)
+        if broadcast:
+            if broadcast == True:  # noqa: E712
+                n = len(nthreads)
+            else:
+                n = broadcast
+            await self.replicate(keys=keys, workers=workers, n=n)
 
-            if ts._has_lost_dependencies:
-                return {key: "forgotten"}
+        self.log_event(
+            [client, "all"], {"action": "scatter", "client": client, "count": len(data)}
+        )
+        return keys
 
-            recommendations: dict = {}
+    async def gather(self, comm=None, keys=None, serializers=None):
+        """ Collect data in from workers """
+        parent: SchedulerState = cast(SchedulerState, self)
+        ws: WorkerState
+        keys = list(keys)
+        who_has = {}
+        for key in keys:
+            ts: TaskState = parent._tasks.get(key)
+            if ts is not None:
+                who_has[key] = [ws._address for ws in ts._who_has]
+            else:
+                who_has[key] = []
 
-            for dts in ts._dependencies:
-                dep = dts._key
-                if not dts._who_has:
-                    ts._waiting_on.add(dts)
-                if dts._state == "released":
-                    recommendations[dep] = "waiting"
-                else:
-                    dts._waiters.add(ts)
+        data, missing_keys, missing_workers = await gather_from_workers(
+            who_has, rpc=self.rpc, close=False, serializers=serializers
+        )
+        if not missing_keys:
+            result = {"status": "OK", "data": data}
+        else:
+            missing_states = [
+                (parent._tasks[key].state if key in parent._tasks else None)
+                for key in missing_keys
+            ]
+            logger.exception(
+                "Couldn't gather keys %s state: %s workers: %s",
+                missing_keys,
+                missing_states,
+                missing_workers,
+            )
+            result = {"status": "error", "keys": missing_keys}
+            with log_errors():
+                # Remove suspicious workers from the scheduler but allow them to
+                # reconnect.
+                await asyncio.gather(
+                    *[
+                        self.remove_worker(address=worker, close=False)
+                        for worker in missing_workers
+                    ]
+                )
+                for key, workers in missing_keys.items():
+                    # Task may already be gone if it was held by a
+                    # `missing_worker`
+                    ts: TaskState = parent._tasks.get(key)
+                    logger.exception(
+                        "Workers don't have promised key: %s, %s",
+                        str(workers),
+                        str(key),
+                    )
+                    if not workers or ts is None:
+                        continue
+                    for worker in workers:
+                        ws = parent._workers_dv.get(worker)
+                        if ws is not None and ts in ws._has_what:
+                            ws._has_what.remove(ts)
+                            ts._who_has.remove(ws)
+                            ws._nbytes -= ts.get_nbytes()
+                            self.transitions({key: "released"})
 
-            ts.state = "waiting"
+        self.log_event("all", {"action": "gather", "count": len(keys)})
+        return result
 
-            if not ts._waiting_on:
-                if workers:
-                    recommendations[key] = "processing"
-                else:
-                    self.unrunnable.add(ts)
-                    ts.state = "no-worker"
+    def clear_task_state(self):
+        # XXX what about nested state such as ClientState.wants_what
+        # (see also fire-and-forget...)
+        logger.info("Clear task state")
+        for collection in self._task_state_collections:
+            collection.clear()
 
-            return recommendations
-        except Exception as e:
-            logger.exception(e)
-            if LOG_PDB:
-                import pdb
+    async def restart(self, client=None, timeout=3):
+        """ Restart all workers.  Reset local state. """
+        parent: SchedulerState = cast(SchedulerState, self)
+        with log_errors():
 
-                pdb.set_trace()
-            raise
+            n_workers = len(parent._workers_dv)
 
-    def decide_worker(self, ts: TaskState) -> WorkerState:
-        """
-        Decide on a worker for task *ts*.  Return a WorkerState.
-        """
-        workers: dict = cast(dict, self.workers)
-        ws: WorkerState = None
-        valid_workers: set = self.valid_workers(ts)
+            logger.info("Send lost future signal to clients")
+            cs: ClientState
+            ts: TaskState
+            for cs in parent._clients.values():
+                self.client_releases_keys(
+                    keys=[ts._key for ts in cs._wants_what], client=cs._client_key
+                )
 
-        if (
-            valid_workers is not None
-            and not valid_workers
-            and not ts._loose_restrictions
-            and workers
-        ):
-            self.unrunnable.add(ts)
-            ts.state = "no-worker"
-            return ws
+            ws: WorkerState
+            nannies = {addr: ws._nanny for addr, ws in parent._workers_dv.items()}
 
-        if ts._dependencies or valid_workers is not None:
-            ws = decide_worker(
-                ts,
-                workers.values(),
-                valid_workers,
-                partial(self.worker_objective, ts),
-            )
-        else:
-            worker_pool = self.idle or self.workers
-            worker_pool_dv = cast(dict, worker_pool)
-            n_workers: Py_ssize_t = len(worker_pool_dv)
-            if n_workers < 20:  # smart but linear in small case
-                ws = min(worker_pool.values(), key=operator.attrgetter("occupancy"))
-            else:  # dumb but fast in large case
-                n_tasks: Py_ssize_t = self.n_tasks
-                ws = worker_pool.values()[n_tasks % n_workers]
+            for addr in list(parent._workers_dv):
+                try:
+                    # Ask the worker to close if it doesn't have a nanny,
+                    # otherwise the nanny will kill it anyway
+                    await self.remove_worker(address=addr, close=addr not in nannies)
+                except Exception as e:
+                    logger.info(
+                        "Exception while restarting.  This is normal", exc_info=True
+                    )
 
-        if self.validate:
-            assert ws is None or isinstance(ws, WorkerState), (
-                type(ws),
-                ws,
-            )
-            assert ws._address in workers
+            self.clear_task_state()
 
-        return ws
+            for plugin in self.plugins[:]:
+                try:
+                    plugin.restart(self)
+                except Exception as e:
+                    logger.exception(e)
 
-    def set_duration_estimate(self, ts: TaskState, ws: WorkerState):
-        """Estimate task duration using worker state and task state.
+            logger.debug("Send kill signal to nannies: %s", nannies)
 
-        If a task takes longer than twice the current average duration we
-        estimate the task duration to be 2x current-runtime, otherwise we set it
-        to be the average duration.
-        """
-        exec_time: double = ws._executing.get(ts, 0)
-        duration: double = self.get_task_duration(ts)
-        total_duration: double
-        if exec_time > 2 * duration:
-            total_duration = 2 * exec_time
-        else:
-            comm: double = self.get_comm_cost(ts, ws)
-            total_duration = duration + comm
-        ws._processing[ts] = total_duration
-        return total_duration
+            nannies = [
+                rpc(nanny_address, connection_args=self.connection_args)
+                for nanny_address in nannies.values()
+                if nanny_address is not None
+            ]
 
-    def transition_waiting_processing(self, key):
-        try:
-            tasks: dict = self.tasks
-            ts: TaskState = tasks[key]
-            dts: TaskState
+            resps = All(
+                [
+                    nanny.restart(
+                        close=True, timeout=timeout * 0.8, executor_wait=False
+                    )
+                    for nanny in nannies
+                ]
+            )
+            try:
+                resps = await asyncio.wait_for(resps, timeout)
+            except TimeoutError:
+                logger.error(
+                    "Nannies didn't report back restarted within "
+                    "timeout.  Continuuing with restart process"
+                )
+            else:
+                if not all(resp == "OK" for resp in resps):
+                    logger.error(
+                        "Not all workers responded positively: %s", resps, exc_info=True
+                    )
+            finally:
+                await asyncio.gather(*[nanny.close_rpc() for nanny in nannies])
 
-            if self.validate:
-                assert not ts._waiting_on
-                assert not ts._who_has
-                assert not ts._exception_blame
-                assert not ts._processing_on
-                assert not ts._has_lost_dependencies
-                assert ts not in self.unrunnable
-                assert all([dts._who_has for dts in ts._dependencies])
+            self.clear_task_state()
 
-            ws: WorkerState = self.decide_worker(ts)
-            if ws is None:
-                return {}
-            worker = ws._address
+            with suppress(AttributeError):
+                for c in self._worker_coroutines:
+                    c.cancel()
 
-            duration_estimate = self.set_duration_estimate(ts, ws)
-            ts._processing_on = ws
-            ws._occupancy += duration_estimate
-            self.total_occupancy += duration_estimate
-            ts.state = "processing"
-            self.consume_resources(ts, ws)
-            self.check_idle_saturated(ws)
-            self.n_tasks += 1
+            self.log_event([client, "all"], {"action": "restart", "client": client})
+            start = time()
+            while time() < start + 10 and len(parent._workers_dv) < n_workers:
+                await asyncio.sleep(0.01)
 
-            if ts._actor:
-                ws._actors.add(ts)
+            self.report({"op": "restart"})
 
-            # logger.debug("Send job to worker: %s, %s", worker, key)
+    async def broadcast(
+        self,
+        comm=None,
+        msg=None,
+        workers=None,
+        hosts=None,
+        nanny=False,
+        serializers=None,
+    ):
+        """ Broadcast message to workers, return all results """
+        parent: SchedulerState = cast(SchedulerState, self)
+        if workers is None or workers is True:
+            if hosts is None:
+                workers = list(parent._workers_dv)
+            else:
+                workers = []
+        if hosts is not None:
+            for host in hosts:
+                if host in parent._host_info:
+                    workers.extend(parent._host_info[host]["addresses"])
+        # TODO replace with worker_list
 
-            self.send_task_to_worker(worker, ts)
+        if nanny:
+            addresses = [parent._workers_dv[w].nanny for w in workers]
+        else:
+            addresses = workers
 
-            return {}
-        except Exception as e:
-            logger.exception(e)
-            if LOG_PDB:
-                import pdb
+        async def send_message(addr):
+            comm = await self.rpc.connect(addr)
+            comm.name = "Scheduler Broadcast"
+            try:
+                resp = await send_recv(comm, close=True, serializers=serializers, **msg)
+            finally:
+                self.rpc.reuse(addr, comm)
+            return resp
 
-                pdb.set_trace()
-            raise
+        results = await All(
+            [send_message(address) for address in addresses if address is not None]
+        )
 
-    def transition_waiting_memory(self, key, nbytes=None, worker=None, **kwargs):
-        try:
-            workers: dict = cast(dict, self.workers)
-            ws: WorkerState = workers[worker]
-            tasks: dict = self.tasks
-            ts: TaskState = tasks[key]
+        return dict(zip(workers, results))
 
-            if self.validate:
-                assert not ts._processing_on
-                assert ts._waiting_on
-                assert ts._state == "waiting"
+    async def proxy(self, comm=None, msg=None, worker=None, serializers=None):
+        """ Proxy a communication through the scheduler to some other worker """
+        d = await self.broadcast(
+            comm=comm, msg=msg, workers=[worker], serializers=serializers
+        )
+        return d[worker]
 
-            ts._waiting_on.clear()
+    async def _delete_worker_data(self, worker_address, keys):
+        """Delete data from a worker and update the corresponding worker/task states
 
-            if nbytes is not None:
-                ts.set_nbytes(nbytes)
+        Parameters
+        ----------
+        worker_address: str
+            Worker address to delete keys from
+        keys: List[str]
+            List of keys to delete on the specified worker
+        """
+        parent: SchedulerState = cast(SchedulerState, self)
+        await retry_operation(
+            self.rpc(addr=worker_address).delete_data, keys=list(keys), report=False
+        )
 
-            self.check_idle_saturated(ws)
+        ws: WorkerState = parent._workers_dv[worker_address]
+        ts: TaskState
+        tasks: set = {parent._tasks[key] for key in keys}
+        ws._has_what -= tasks
+        for ts in tasks:
+            ts._who_has.remove(ws)
+            ws._nbytes -= ts.get_nbytes()
+        self.log_event(ws._address, {"action": "remove-worker-data", "keys": keys})
 
-            recommendations: dict = {}
+    async def rebalance(self, comm=None, keys=None, workers=None):
+        """Rebalance keys so that each worker stores roughly equal bytes
 
-            self._add_to_memory(ts, ws, recommendations, **kwargs)
+        **Policy**
 
-            if self.validate:
-                assert not ts._processing_on
-                assert not ts._waiting_on
-                assert ts._who_has
+        This orders the workers by what fraction of bytes of the existing keys
+        they have.  It walks down this list from most-to-least.  At each worker
+        it sends the largest results it can find and sends them to the least
+        occupied worker until either the sender or the recipient are at the
+        average expected load.
+        """
+        parent: SchedulerState = cast(SchedulerState, self)
+        ts: TaskState
+        with log_errors():
+            async with self._lock:
+                if keys:
+                    tasks = {parent._tasks[k] for k in keys}
+                    missing_data = [ts._key for ts in tasks if not ts._who_has]
+                    if missing_data:
+                        return {"status": "missing-data", "keys": missing_data}
+                else:
+                    tasks = set(parent._tasks.values())
 
-            return recommendations
-        except Exception as e:
-            logger.exception(e)
-            if LOG_PDB:
-                import pdb
+                if workers:
+                    workers = {parent._workers_dv[w] for w in workers}
+                    workers_by_task = {ts: ts._who_has & workers for ts in tasks}
+                else:
+                    workers = set(parent._workers_dv.values())
+                    workers_by_task = {ts: ts._who_has for ts in tasks}
 
-                pdb.set_trace()
-            raise
+                ws: WorkerState
+                tasks_by_worker = {ws: set() for ws in workers}
 
-    def transition_processing_memory(
-        self,
-        key,
-        nbytes=None,
-        type=None,
-        typename=None,
-        worker=None,
-        startstops=None,
-        **kwargs,
-    ):
-        ws: WorkerState
-        wws: WorkerState
-        try:
-            tasks: dict = self.tasks
-            ts: TaskState = tasks[key]
-            assert worker
-            assert isinstance(worker, str)
+                for k, v in workers_by_task.items():
+                    for vv in v:
+                        tasks_by_worker[vv].add(k)
 
-            if self.validate:
-                assert ts._processing_on
-                ws = ts._processing_on
-                assert ts in ws._processing
-                assert not ts._waiting_on
-                assert not ts._who_has, (ts, ts._who_has)
-                assert not ts._exception_blame
-                assert ts._state == "processing"
+                worker_bytes = {
+                    ws: sum(ts.get_nbytes() for ts in v)
+                    for ws, v in tasks_by_worker.items()
+                }
 
-            workers: dict = cast(dict, self.workers)
-            ws = workers.get(worker)
-            if ws is None:
-                return {key: "released"}
+                avg = sum(worker_bytes.values()) / len(worker_bytes)
 
-            if ws != ts._processing_on:  # someone else has this task
-                logger.info(
-                    "Unexpected worker completed task, likely due to"
-                    " work stealing.  Expected: %s, Got: %s, Key: %s",
-                    ts._processing_on,
-                    ws,
-                    key,
+                sorted_workers = list(
+                    map(first, sorted(worker_bytes.items(), key=second, reverse=True))
                 )
-                return {}
-
-            if startstops:
-                L = list()
-                for startstop in startstops:
-                    stop = startstop["stop"]
-                    start = startstop["start"]
-                    action = startstop["action"]
-                    if action == "compute":
-                        L.append((start, stop))
-
-                    # record timings of all actions -- a cheaper way of
-                    # getting timing info compared with get_task_stream()
-                    ts._prefix._all_durations[action] += stop - start
 
-                if len(L) > 0:
-                    compute_start, compute_stop = L[0]
-                else:  # This is very rare
-                    compute_start = compute_stop = None
-            else:
-                compute_start = compute_stop = None
+                recipients = iter(reversed(sorted_workers))
+                recipient = next(recipients)
+                msgs = []  # (sender, recipient, key)
+                for sender in sorted_workers[: len(workers) // 2]:
+                    sender_keys = {
+                        ts: ts.get_nbytes() for ts in tasks_by_worker[sender]
+                    }
+                    sender_keys = iter(
+                        sorted(sender_keys.items(), key=second, reverse=True)
+                    )
 
-            #############################
-            # Update Timing Information #
-            #############################
-            if compute_start and ws._processing.get(ts, True):
-                # Update average task duration for worker
-                old_duration = ts._prefix._duration_average
-                new_duration = compute_stop - compute_start
-                if old_duration < 0:
-                    avg_duration = new_duration
-                else:
-                    avg_duration = 0.5 * old_duration + 0.5 * new_duration
+                    try:
+                        while worker_bytes[sender] > avg:
+                            while (
+                                worker_bytes[recipient] < avg
+                                and worker_bytes[sender] > avg
+                            ):
+                                ts, nb = next(sender_keys)
+                                if ts not in tasks_by_worker[recipient]:
+                                    tasks_by_worker[recipient].add(ts)
+                                    # tasks_by_worker[sender].remove(ts)
+                                    msgs.append((sender, recipient, ts))
+                                    worker_bytes[sender] -= nb
+                                    worker_bytes[recipient] += nb
+                            if worker_bytes[sender] > avg:
+                                recipient = next(recipients)
+                    except StopIteration:
+                        break
 
-                ts._prefix._duration_average = avg_duration
-                ts._group._duration += new_duration
+                to_recipients = defaultdict(lambda: defaultdict(list))
+                to_senders = defaultdict(list)
+                for sender, recipient, ts in msgs:
+                    to_recipients[recipient.address][ts._key].append(sender.address)
+                    to_senders[sender.address].append(ts._key)
 
-                tts: TaskState
-                for tts in self.unknown_durations.pop(ts._prefix._name, ()):
-                    if tts._processing_on:
-                        wws = tts._processing_on
-                        old = wws._processing[tts]
-                        comm = self.get_comm_cost(tts, wws)
-                        wws._processing[tts] = avg_duration + comm
-                        wws._occupancy += avg_duration + comm - old
-                        self.total_occupancy += avg_duration + comm - old
+                result = await asyncio.gather(
+                    *(
+                        retry_operation(self.rpc(addr=r).gather, who_has=v)
+                        for r, v in to_recipients.items()
+                    )
+                )
+                for r, v in to_recipients.items():
+                    self.log_event(r, {"action": "rebalance", "who_has": v})
 
-            ############################
-            # Update State Information #
-            ############################
-            if nbytes is not None:
-                ts.set_nbytes(nbytes)
+                self.log_event(
+                    "all",
+                    {
+                        "action": "rebalance",
+                        "total-keys": len(tasks),
+                        "senders": valmap(len, to_senders),
+                        "recipients": valmap(len, to_recipients),
+                        "moved_keys": len(msgs),
+                    },
+                )
 
-            recommendations: dict = {}
+                if not all(r["status"] == "OK" for r in result):
+                    return {
+                        "status": "missing-data",
+                        "keys": tuple(
+                            concat(
+                                r["keys"].keys()
+                                for r in result
+                                if r["status"] == "missing-data"
+                            )
+                        ),
+                    }
 
-            self._remove_from_processing(ts)
+                for sender, recipient, ts in msgs:
+                    assert ts._state == "memory"
+                    ts._who_has.add(recipient)
+                    recipient.has_what.add(ts)
+                    recipient.nbytes += ts.get_nbytes()
+                    self.log.append(
+                        (
+                            "rebalance",
+                            ts._key,
+                            time(),
+                            sender.address,
+                            recipient.address,
+                        )
+                    )
 
-            self._add_to_memory(ts, ws, recommendations, type=type, typename=typename)
+                await asyncio.gather(
+                    *(self._delete_worker_data(r, v) for r, v in to_senders.items())
+                )
 
-            if self.validate:
-                assert not ts._processing_on
-                assert not ts._waiting_on
+                return {"status": "OK"}
 
-            return recommendations
-        except Exception as e:
-            logger.exception(e)
-            if LOG_PDB:
-                import pdb
+    async def replicate(
+        self,
+        comm=None,
+        keys=None,
+        n=None,
+        workers=None,
+        branching_factor=2,
+        delete=True,
+        lock=True,
+    ):
+        """Replicate data throughout cluster
 
-                pdb.set_trace()
-            raise
+        This performs a tree copy of the data throughout the network
+        individually on each piece of data.
 
-    def transition_memory_released(self, key, safe=False):
-        ws: WorkerState
-        try:
-            tasks: dict = self.tasks
-            ts: TaskState = tasks[key]
-            dts: TaskState
+        Parameters
+        ----------
+        keys: Iterable
+            list of keys to replicate
+        n: int
+            Number of replications we expect to see within the cluster
+        branching_factor: int, optional
+            The number of workers that can copy data in each generation.
+            The larger the branching factor, the more data we copy in
+            a single step, but the more a given worker risks being
+            swamped by data requests.
 
-            if self.validate:
-                assert not ts._waiting_on
-                assert not ts._processing_on
-                if safe:
-                    assert not ts._waiters
+        See also
+        --------
+        Scheduler.rebalance
+        """
+        parent: SchedulerState = cast(SchedulerState, self)
+        ws: WorkerState
+        wws: WorkerState
+        ts: TaskState
 
-            if ts._actor:
-                for ws in ts._who_has:
-                    ws._actors.discard(ts)
-                if ts._who_wants:
-                    ts._exception_blame = ts
-                    ts._exception = "Worker holding Actor was lost"
-                    return {ts._key: "erred"}  # don't try to recreate
+        assert branching_factor > 0
+        async with self._lock if lock else empty_context:
+            workers = {parent._workers_dv[w] for w in self.workers_list(workers)}
+            if n is None:
+                n = len(workers)
+            else:
+                n = min(n, len(workers))
+            if n == 0:
+                raise ValueError("Can not use replicate to delete data")
 
-            recommendations: dict = {}
+            tasks = {parent._tasks[k] for k in keys}
+            missing_data = [ts._key for ts in tasks if not ts._who_has]
+            if missing_data:
+                return {"status": "missing-data", "keys": missing_data}
 
-            for dts in ts._waiters:
-                if dts._state in ("no-worker", "processing"):
-                    recommendations[dts._key] = "waiting"
-                elif dts._state == "waiting":
-                    dts._waiting_on.add(ts)
+            # Delete extraneous data
+            if delete:
+                del_worker_tasks = defaultdict(set)
+                for ts in tasks:
+                    del_candidates = ts._who_has & workers
+                    if len(del_candidates) > n:
+                        for ws in random.sample(
+                            del_candidates, len(del_candidates) - n
+                        ):
+                            del_worker_tasks[ws].add(ts)
 
-            # XXX factor this out?
-            for ws in ts._who_has:
-                ws._has_what.remove(ts)
-                ws._nbytes -= ts.get_nbytes()
-                ts._group._nbytes_in_memory -= ts.get_nbytes()
-                self.worker_send(
-                    ws._address, {"op": "delete-data", "keys": [key], "report": False}
+                await asyncio.gather(
+                    *[
+                        self._delete_worker_data(ws._address, [t.key for t in tasks])
+                        for ws, tasks in del_worker_tasks.items()
+                    ]
                 )
-            ts._who_has.clear()
-
-            ts.state = "released"
 
-            self.report({"op": "lost-data", "key": key})
+            # Copy not-yet-filled data
+            while tasks:
+                gathers = defaultdict(dict)
+                for ts in list(tasks):
+                    if ts._state == "forgotten":
+                        # task is no longer needed by any client or dependant task
+                        tasks.remove(ts)
+                        continue
+                    n_missing = n - len(ts._who_has & workers)
+                    if n_missing <= 0:
+                        # Already replicated enough
+                        tasks.remove(ts)
+                        continue
 
-            if not ts._run_spec:  # pure data
-                recommendations[key] = "forgotten"
-            elif ts._has_lost_dependencies:
-                recommendations[key] = "forgotten"
-            elif ts._who_wants or ts._waiters:
-                recommendations[key] = "waiting"
+                    count = min(n_missing, branching_factor * len(ts._who_has))
+                    assert count > 0
 
-            if self.validate:
-                assert not ts._waiting_on
+                    for ws in random.sample(workers - ts._who_has, count):
+                        gathers[ws._address][ts._key] = [
+                            wws._address for wws in ts._who_has
+                        ]
 
-            return recommendations
-        except Exception as e:
-            logger.exception(e)
-            if LOG_PDB:
-                import pdb
+                results = await asyncio.gather(
+                    *(
+                        retry_operation(self.rpc(addr=w).gather, who_has=who_has)
+                        for w, who_has in gathers.items()
+                    )
+                )
+                for w, v in zip(gathers, results):
+                    if v["status"] == "OK":
+                        self.add_keys(worker=w, keys=list(gathers[w]))
+                    else:
+                        logger.warning("Communication failed during replication: %s", v)
 
-                pdb.set_trace()
-            raise
+                    self.log_event(w, {"action": "replicate-add", "keys": gathers[w]})
 
-    def transition_released_erred(self, key):
-        try:
-            tasks: dict = self.tasks
-            ts: TaskState = tasks[key]
-            dts: TaskState
-            failing_ts: TaskState
+            self.log_event(
+                "all",
+                {
+                    "action": "replicate",
+                    "workers": list(workers),
+                    "key-count": len(keys),
+                    "branching-factor": branching_factor,
+                },
+            )
 
-            if self.validate:
-                with log_errors(pdb=LOG_PDB):
-                    assert ts._exception_blame
-                    assert not ts._who_has
-                    assert not ts._waiting_on
-                    assert not ts._waiters
+    def workers_to_close(
+        self,
+        comm=None,
+        memory_ratio=None,
+        n=None,
+        key=None,
+        minimum=None,
+        target=None,
+        attribute="address",
+    ):
+        """
+        Find workers that we can close with low cost
 
-            recommendations: dict = {}
+        This returns a list of workers that are good candidates to retire.
+        These workers are not running anything and are storing
+        relatively little data relative to their peers.  If all workers are
+        idle then we still maintain enough workers to have enough RAM to store
+        our data, with a comfortable buffer.
 
-            failing_ts = ts._exception_blame
+        This is for use with systems like ``distributed.deploy.adaptive``.
 
-            for dts in ts._dependents:
-                dts._exception_blame = failing_ts
-                if not dts._who_has:
-                    recommendations[dts._key] = "erred"
+        Parameters
+        ----------
+        memory_factor: Number
+            Amount of extra space we want to have for our stored data.
+            Defaults two 2, or that we want to have twice as much memory as we
+            currently have data.
+        n: int
+            Number of workers to close
+        minimum: int
+            Minimum number of workers to keep around
+        key: Callable(WorkerState)
+            An optional callable mapping a WorkerState object to a group
+            affiliation.  Groups will be closed together.  This is useful when
+            closing workers must be done collectively, such as by hostname.
+        target: int
+            Target number of workers to have after we close
+        attribute : str
+            The attribute of the WorkerState object to return, like "address"
+            or "name".  Defaults to "address".
 
-            self.report(
-                {
-                    "op": "task-erred",
-                    "key": key,
-                    "exception": failing_ts._exception,
-                    "traceback": failing_ts._traceback,
-                }
-            )
+        Examples
+        --------
+        >>> scheduler.workers_to_close()
+        ['tcp://192.168.0.1:1234', 'tcp://192.168.0.2:1234']
 
-            ts.state = "erred"
+        Group workers by hostname prior to closing
 
-            # TODO: waiting data?
-            return recommendations
-        except Exception as e:
-            logger.exception(e)
-            if LOG_PDB:
-                import pdb
+        >>> scheduler.workers_to_close(key=lambda ws: ws.host)
+        ['tcp://192.168.0.1:1234', 'tcp://192.168.0.1:4567']
 
-                pdb.set_trace()
-            raise
+        Remove two workers
 
-    def transition_erred_released(self, key):
-        try:
-            tasks: dict = self.tasks
-            ts: TaskState = tasks[key]
-            dts: TaskState
+        >>> scheduler.workers_to_close(n=2)
 
-            if self.validate:
-                with log_errors(pdb=LOG_PDB):
-                    assert all([dts._state != "erred" for dts in ts._dependencies])
-                    assert ts._exception_blame
-                    assert not ts._who_has
-                    assert not ts._waiting_on
-                    assert not ts._waiters
+        Keep enough workers to have twice as much memory as we we need.
 
-            recommendations: dict = {}
+        >>> scheduler.workers_to_close(memory_ratio=2)
 
-            ts._exception = None
-            ts._exception_blame = None
-            ts._traceback = None
+        Returns
+        -------
+        to_close: list of worker addresses that are OK to close
 
-            for dts in ts._dependents:
-                if dts._state == "erred":
-                    recommendations[dts._key] = "waiting"
+        See Also
+        --------
+        Scheduler.retire_workers
+        """
+        parent: SchedulerState = cast(SchedulerState, self)
+        if target is not None and n is None:
+            n = len(parent._workers_dv) - target
+        if n is not None:
+            if n < 0:
+                n = 0
+            target = len(parent._workers_dv) - n
 
-            self.report({"op": "task-retried", "key": key})
-            ts.state = "released"
+        if n is None and memory_ratio is None:
+            memory_ratio = 2
 
-            return recommendations
-        except Exception as e:
-            logger.exception(e)
-            if LOG_PDB:
-                import pdb
+        ws: WorkerState
+        with log_errors():
+            if not n and all([ws._processing for ws in parent._workers_dv.values()]):
+                return []
 
-                pdb.set_trace()
-            raise
+            if key is None:
+                key = operator.attrgetter("address")
+            if isinstance(key, bytes) and dask.config.get(
+                "distributed.scheduler.pickle"
+            ):
+                key = pickle.loads(key)
 
-    def transition_waiting_released(self, key):
-        try:
-            tasks: dict = self.tasks
-            ts: TaskState = tasks[key]
+            groups = groupby(key, parent._workers.values())
 
-            if self.validate:
-                assert not ts._who_has
-                assert not ts._processing_on
+            limit_bytes = {
+                k: sum([ws._memory_limit for ws in v]) for k, v in groups.items()
+            }
+            group_bytes = {k: sum([ws._nbytes for ws in v]) for k, v in groups.items()}
 
-            recommendations: dict = {}
+            limit = sum(limit_bytes.values())
+            total = sum(group_bytes.values())
 
-            dts: TaskState
-            for dts in ts._dependencies:
-                s = dts._waiters
-                if ts in s:
-                    s.discard(ts)
-                    if not s and not dts._who_wants:
-                        recommendations[dts._key] = "released"
-            ts._waiting_on.clear()
+            def _key(group):
+                wws: WorkerState
+                is_idle = not any([wws._processing for wws in groups[group]])
+                bytes = -group_bytes[group]
+                return (is_idle, bytes)
 
-            ts.state = "released"
+            idle = sorted(groups, key=_key)
 
-            if ts._has_lost_dependencies:
-                recommendations[key] = "forgotten"
-            elif not ts._exception_blame and (ts._who_wants or ts._waiters):
-                recommendations[key] = "waiting"
-            else:
-                ts._waiters.clear()
+            to_close = []
+            n_remain = len(parent._workers_dv)
 
-            return recommendations
-        except Exception as e:
-            logger.exception(e)
-            if LOG_PDB:
-                import pdb
+            while idle:
+                group = idle.pop()
+                if n is None and any([ws._processing for ws in groups[group]]):
+                    break
 
-                pdb.set_trace()
-            raise
+                if minimum and n_remain - len(groups[group]) < minimum:
+                    break
 
-    def transition_processing_released(self, key):
-        try:
-            tasks: dict = self.tasks
-            ts: TaskState = tasks[key]
-            dts: TaskState
+                limit -= limit_bytes[group]
 
-            if self.validate:
-                assert ts._processing_on
-                assert not ts._who_has
-                assert not ts._waiting_on
-                assert self.tasks[key].state == "processing"
+                if (n is not None and n_remain - len(groups[group]) >= target) or (
+                    memory_ratio is not None and limit >= memory_ratio * total
+                ):
+                    to_close.append(group)
+                    n_remain -= len(groups[group])
 
-            self._remove_from_processing(
-                ts, send_worker_msg={"op": "release-task", "key": key}
-            )
+                else:
+                    break
 
-            ts.state = "released"
+            result = [getattr(ws, attribute) for g in to_close for ws in groups[g]]
+            if result:
+                logger.debug("Suggest closing workers: %s", result)
 
-            recommendations: dict = {}
+            return result
 
-            if ts._has_lost_dependencies:
-                recommendations[key] = "forgotten"
-            elif ts._waiters or ts._who_wants:
-                recommendations[key] = "waiting"
+    async def retire_workers(
+        self,
+        comm=None,
+        workers=None,
+        remove=True,
+        close_workers=False,
+        names=None,
+        lock=True,
+        **kwargs,
+    ) -> dict:
+        """Gracefully retire workers from cluster
 
-            if recommendations.get(key) != "waiting":
-                for dts in ts._dependencies:
-                    if dts._state != "released":
-                        s = dts._waiters
-                        s.discard(ts)
-                        if not s and not dts._who_wants:
-                            recommendations[dts._key] = "released"
-                ts._waiters.clear()
+        Parameters
+        ----------
+        workers: list (optional)
+            List of worker addresses to retire.
+            If not provided we call ``workers_to_close`` which finds a good set
+        workers_names: list (optional)
+            List of worker names to retire.
+        remove: bool (defaults to True)
+            Whether or not to remove the worker metadata immediately or else
+            wait for the worker to contact us
+        close_workers: bool (defaults to False)
+            Whether or not to actually close the worker explicitly from here.
+            Otherwise we expect some external job scheduler to finish off the
+            worker.
+        **kwargs: dict
+            Extra options to pass to workers_to_close to determine which
+            workers we should drop
 
-            if self.validate:
-                assert not ts._processing_on
+        Returns
+        -------
+        Dictionary mapping worker ID/address to dictionary of information about
+        that worker for each retired worker.
 
-            return recommendations
-        except Exception as e:
-            logger.exception(e)
-            if LOG_PDB:
-                import pdb
+        See Also
+        --------
+        Scheduler.workers_to_close
+        """
+        parent: SchedulerState = cast(SchedulerState, self)
+        ws: WorkerState
+        ts: TaskState
+        with log_errors():
+            async with self._lock if lock else empty_context:
+                if names is not None:
+                    if names:
+                        logger.info("Retire worker names %s", names)
+                    names = set(map(str, names))
+                    workers = [
+                        ws._address
+                        for ws in parent._workers_dv.values()
+                        if str(ws._name) in names
+                    ]
+                if workers is None:
+                    while True:
+                        try:
+                            workers = self.workers_to_close(**kwargs)
+                            if workers:
+                                workers = await self.retire_workers(
+                                    workers=workers,
+                                    remove=remove,
+                                    close_workers=close_workers,
+                                    lock=False,
+                                )
+                                return workers
+                            else:
+                                return {}
+                        except KeyError:  # keys left during replicate
+                            pass
+                workers = {
+                    parent._workers_dv[w] for w in workers if w in parent._workers_dv
+                }
+                if not workers:
+                    return {}
+                logger.info("Retire workers %s", workers)
 
-                pdb.set_trace()
-            raise
+                # Keys orphaned by retiring those workers
+                keys = set.union(*[w.has_what for w in workers])
+                keys = {ts._key for ts in keys if ts._who_has.issubset(workers)}
 
-    def transition_processing_erred(
-        self, key, cause=None, exception=None, traceback=None, **kwargs
-    ):
-        ws: WorkerState
-        try:
-            tasks: dict = self.tasks
-            ts: TaskState = tasks[key]
-            dts: TaskState
-            failing_ts: TaskState
+                other_workers = set(parent._workers_dv.values()) - workers
+                if keys:
+                    if other_workers:
+                        logger.info("Moving %d keys to other workers", len(keys))
+                        await self.replicate(
+                            keys=keys,
+                            workers=[ws._address for ws in other_workers],
+                            n=1,
+                            delete=False,
+                            lock=False,
+                        )
+                    else:
+                        return {}
 
-            if self.validate:
-                assert cause or ts._exception_blame
-                assert ts._processing_on
-                assert not ts._who_has
-                assert not ts._waiting_on
+                worker_keys = {ws._address: ws.identity() for ws in workers}
+                if close_workers and worker_keys:
+                    await asyncio.gather(
+                        *[self.close_worker(worker=w, safe=True) for w in worker_keys]
+                    )
+                if remove:
+                    await asyncio.gather(
+                        *[self.remove_worker(address=w, safe=True) for w in worker_keys]
+                    )
 
-            if ts._actor:
-                ws = ts._processing_on
-                ws._actors.remove(ts)
+                self.log_event(
+                    "all",
+                    {
+                        "action": "retire-workers",
+                        "workers": worker_keys,
+                        "moved-keys": len(keys),
+                    },
+                )
+                self.log_event(list(worker_keys), {"action": "retired"})
 
-            self._remove_from_processing(ts)
+                return worker_keys
 
-            if exception is not None:
-                ts._exception = exception
-            if traceback is not None:
-                ts._traceback = traceback
-            if cause is not None:
-                failing_ts = self.tasks[cause]
-                ts._exception_blame = failing_ts
+    def add_keys(self, comm=None, worker=None, keys=()):
+        """
+        Learn that a worker has certain keys
+
+        This should not be used in practice and is mostly here for legacy
+        reasons.  However, it is sent by workers from time to time.
+        """
+        parent: SchedulerState = cast(SchedulerState, self)
+        if worker not in parent._workers_dv:
+            return "not found"
+        ws: WorkerState = parent._workers_dv[worker]
+        for key in keys:
+            ts: TaskState = parent._tasks.get(key)
+            if ts is not None and ts._state == "memory":
+                if ts not in ws._has_what:
+                    ws._nbytes += ts.get_nbytes()
+                    ws._has_what.add(ts)
+                    ts._who_has.add(ws)
             else:
-                failing_ts = ts._exception_blame
+                self.worker_send(
+                    worker, {"op": "delete-data", "keys": [key], "report": False}
+                )
 
-            recommendations: dict = {}
+        return "OK"
 
-            for dts in ts._dependents:
-                dts._exception_blame = failing_ts
-                recommendations[dts._key] = "erred"
+    def update_data(
+        self, comm=None, who_has=None, nbytes=None, client=None, serializers=None
+    ):
+        """
+        Learn that new data has entered the network from an external source
 
-            for dts in ts._dependencies:
-                s = dts._waiters
-                s.discard(ts)
-                if not s and not dts._who_wants:
-                    recommendations[dts._key] = "released"
+        See Also
+        --------
+        Scheduler.mark_key_in_memory
+        """
+        parent: SchedulerState = cast(SchedulerState, self)
+        with log_errors():
+            who_has = {
+                k: [self.coerce_address(vv) for vv in v] for k, v in who_has.items()
+            }
+            logger.debug("Update data %s", who_has)
+
+            for key, workers in who_has.items():
+                ts: TaskState = parent._tasks.get(key)
+                if ts is None:
+                    ts: TaskState = self.new_task(key, None, "memory")
+                ts.state = "memory"
+                if key in nbytes:
+                    ts.set_nbytes(nbytes[key])
+                for w in workers:
+                    ws: WorkerState = parent._workers_dv[w]
+                    if ts not in ws._has_what:
+                        ws._nbytes += ts.get_nbytes()
+                        ws._has_what.add(ts)
+                        ts._who_has.add(ws)
+                self.report(
+                    {"op": "key-in-memory", "key": key, "workers": list(workers)}
+                )
+
+            if client:
+                self.client_desires_keys(keys=list(who_has), client=client)
+
+    def report_on_key(self, key: str = None, ts: TaskState = None, client: str = None):
+        parent: SchedulerState = cast(SchedulerState, self)
+        if ts is None:
+            tasks: dict = parent._tasks
+            ts = tasks.get(key)
+        elif key is None:
+            key = ts._key
+        else:
+            assert False, (key, ts)
+            return
 
-            ts._waiters.clear()  # do anything with this?
+        report_msg: dict = _task_to_report_msg(parent, ts)
+        if report_msg is not None:
+            self.report(report_msg, ts=ts, client=client)
 
-            ts.state = "erred"
+    async def feed(
+        self, comm, function=None, setup=None, teardown=None, interval="1s", **kwargs
+    ):
+        """
+        Provides a data Comm to external requester
 
-            self.report(
-                {
-                    "op": "task-erred",
-                    "key": key,
-                    "exception": failing_ts._exception,
-                    "traceback": failing_ts._traceback,
-                }
+        Caution: this runs arbitrary Python code on the scheduler.  This should
+        eventually be phased out.  It is mostly used by diagnostics.
+        """
+        if not dask.config.get("distributed.scheduler.pickle"):
+            logger.warn(
+                "Tried to call 'feed' route with custom functions, but "
+                "pickle is disallowed.  Set the 'distributed.scheduler.pickle'"
+                "config value to True to use the 'feed' route (this is mostly "
+                "commonly used with progress bars)"
             )
+            return
 
-            cs: ClientState = self.clients["fire-and-forget"]
-            if ts in cs._wants_what:
-                self.client_releases_keys(client="fire-and-forget", keys=[key])
+        interval = parse_timedelta(interval)
+        with log_errors():
+            if function:
+                function = pickle.loads(function)
+            if setup:
+                setup = pickle.loads(setup)
+            if teardown:
+                teardown = pickle.loads(teardown)
+            state = setup(self) if setup else None
+            if inspect.isawaitable(state):
+                state = await state
+            try:
+                while self.status == Status.running:
+                    if state is None:
+                        response = function(self)
+                    else:
+                        response = function(self, state)
+                    await comm.write(response)
+                    await asyncio.sleep(interval)
+            except (EnvironmentError, CommClosedError):
+                pass
+            finally:
+                if teardown:
+                    teardown(self, state)
 
-            if self.validate:
-                assert not ts._processing_on
+    def log_worker_event(self, worker=None, topic=None, msg=None):
+        self.log_event(topic, msg)
 
-            return recommendations
-        except Exception as e:
-            logger.exception(e)
-            if LOG_PDB:
-                import pdb
+    def subscribe_worker_status(self, comm=None):
+        WorkerStatusPlugin(self, comm)
+        ident = self.identity()
+        for v in ident["workers"].values():
+            del v["metrics"]
+            del v["last_seen"]
+        return ident
 
-                pdb.set_trace()
-            raise
+    def get_processing(self, comm=None, workers=None):
+        parent: SchedulerState = cast(SchedulerState, self)
+        ws: WorkerState
+        ts: TaskState
+        if workers is not None:
+            workers = set(map(self.coerce_address, workers))
+            return {
+                w: [ts._key for ts in parent._workers_dv[w].processing] for w in workers
+            }
+        else:
+            return {
+                w: [ts._key for ts in ws._processing]
+                for w, ws in parent._workers_dv.items()
+            }
 
-    def transition_no_worker_released(self, key):
-        try:
-            tasks: dict = self.tasks
-            ts: TaskState = tasks[key]
-            dts: TaskState
+    def get_who_has(self, comm=None, keys=None):
+        parent: SchedulerState = cast(SchedulerState, self)
+        ws: WorkerState
+        ts: TaskState
+        if keys is not None:
+            return {
+                k: [ws._address for ws in parent._tasks[k].who_has]
+                if k in parent._tasks
+                else []
+                for k in keys
+            }
+        else:
+            return {
+                key: [ws._address for ws in ts._who_has]
+                for key, ts in parent._tasks.items()
+            }
 
-            if self.validate:
-                assert self.tasks[key].state == "no-worker"
-                assert not ts._who_has
-                assert not ts._waiting_on
+    def get_has_what(self, comm=None, workers=None):
+        parent: SchedulerState = cast(SchedulerState, self)
+        ws: WorkerState
+        ts: TaskState
+        if workers is not None:
+            workers = map(self.coerce_address, workers)
+            return {
+                w: [ts._key for ts in parent._workers_dv[w].has_what]
+                if w in parent._workers_dv
+                else []
+                for w in workers
+            }
+        else:
+            return {
+                w: [ts._key for ts in ws._has_what]
+                for w, ws in parent._workers_dv.items()
+            }
 
-            self.unrunnable.remove(ts)
-            ts.state = "released"
+    def get_ncores(self, comm=None, workers=None):
+        parent: SchedulerState = cast(SchedulerState, self)
+        ws: WorkerState
+        if workers is not None:
+            workers = map(self.coerce_address, workers)
+            return {
+                w: parent._workers_dv[w].nthreads
+                for w in workers
+                if w in parent._workers_dv
+            }
+        else:
+            return {w: ws._nthreads for w, ws in parent._workers_dv.items()}
 
-            for dts in ts._dependencies:
-                dts._waiters.discard(ts)
+    async def get_call_stack(self, comm=None, keys=None):
+        parent: SchedulerState = cast(SchedulerState, self)
+        ts: TaskState
+        dts: TaskState
+        if keys is not None:
+            stack = list(keys)
+            processing = set()
+            while stack:
+                key = stack.pop()
+                ts = parent._tasks[key]
+                if ts._state == "waiting":
+                    stack.extend([dts._key for dts in ts._dependencies])
+                elif ts._state == "processing":
+                    processing.add(ts)
 
-            ts._waiters.clear()
+            workers = defaultdict(list)
+            for ts in processing:
+                if ts._processing_on:
+                    workers[ts._processing_on.address].append(ts._key)
+        else:
+            workers = {w: None for w in parent._workers_dv}
 
+        if not workers:
             return {}
-        except Exception as e:
-            logger.exception(e)
-            if LOG_PDB:
-                import pdb
-
-                pdb.set_trace()
-            raise
-
-    def remove_key(self, key):
-        tasks: dict = self.tasks
-        ts: TaskState = tasks.pop(key)
-        assert ts._state == "forgotten"
-        self.unrunnable.discard(ts)
-        cs: ClientState
-        for cs in ts._who_wants:
-            cs._wants_what.remove(ts)
-        ts._who_wants.clear()
-        ts._processing_on = None
-        ts._exception_blame = ts._exception = ts._traceback = None
-        self.task_metadata.pop(key, None)
 
-    def _propagate_forgotten(self, ts: TaskState, recommendations: dict):
-        workers: dict = cast(dict, self.workers)
-        ts.state = "forgotten"
-        key: str = ts._key
-        dts: TaskState
-        for dts in ts._dependents:
-            dts._has_lost_dependencies = True
-            dts._dependencies.remove(ts)
-            dts._waiting_on.discard(ts)
-            if dts._state not in ("memory", "erred"):
-                # Cannot compute task anymore
-                recommendations[dts._key] = "forgotten"
-        ts._dependents.clear()
-        ts._waiters.clear()
+        results = await asyncio.gather(
+            *(self.rpc(w).call_stack(keys=v) for w, v in workers.items())
+        )
+        response = {w: r for w, r in zip(workers, results) if r}
+        return response
 
-        for dts in ts._dependencies:
-            dts._dependents.remove(ts)
-            s: set = dts._waiters
-            s.discard(ts)
-            if not dts._dependents and not dts._who_wants:
-                # Task not needed anymore
-                assert dts is not ts
-                recommendations[dts._key] = "forgotten"
-        ts._dependencies.clear()
-        ts._waiting_on.clear()
+    def get_nbytes(self, comm=None, keys=None, summary=True):
+        parent: SchedulerState = cast(SchedulerState, self)
+        ts: TaskState
+        with log_errors():
+            if keys is not None:
+                result = {k: parent._tasks[k].nbytes for k in keys}
+            else:
+                result = {
+                    k: ts._nbytes for k, ts in parent._tasks.items() if ts._nbytes >= 0
+                }
 
-        if ts._who_has:
-            ts._group._nbytes_in_memory -= ts.get_nbytes()
+            if summary:
+                out = defaultdict(lambda: 0)
+                for k, v in result.items():
+                    out[key_split(k)] += v
+                result = dict(out)
 
-        ws: WorkerState
-        for ws in ts._who_has:
-            ws._has_what.remove(ts)
-            ws._nbytes -= ts.get_nbytes()
-            w: str = ws._address
-            if w in workers:  # in case worker has died
-                self.worker_send(
-                    w, {"op": "delete-data", "keys": [key], "report": False}
-                )
-        ts._who_has.clear()
+            return result
 
-    def transition_memory_forgotten(self, key):
-        tasks: dict
-        ws: WorkerState
-        try:
-            tasks = self.tasks
-            ts: TaskState = tasks[key]
+    def run_function(self, stream, function, args=(), kwargs={}, wait=True):
+        """Run a function within this process
 
-            if self.validate:
-                assert ts._state == "memory"
-                assert not ts._processing_on
-                assert not ts._waiting_on
-                if not ts._run_spec:
-                    # It's ok to forget a pure data task
-                    pass
-                elif ts._has_lost_dependencies:
-                    # It's ok to forget a task with forgotten dependencies
-                    pass
-                elif not ts._who_wants and not ts._waiters and not ts._dependents:
-                    # It's ok to forget a task that nobody needs
-                    pass
-                else:
-                    assert 0, (ts,)
+        See Also
+        --------
+        Client.run_on_scheduler:
+        """
+        from .worker import run
 
-            recommendations: dict = {}
+        self.log_event("all", {"action": "run-function", "function": function})
+        return run(self, stream, function=function, args=args, kwargs=kwargs, wait=wait)
 
-            if ts._actor:
-                for ws in ts._who_has:
-                    ws._actors.discard(ts)
+    def set_metadata(self, comm=None, keys=None, value=None):
+        parent: SchedulerState = cast(SchedulerState, self)
+        try:
+            metadata = parent._task_metadata
+            for key in keys[:-1]:
+                if key not in metadata or not isinstance(metadata[key], (dict, list)):
+                    metadata[key] = dict()
+                metadata = metadata[key]
+            metadata[keys[-1]] = value
+        except Exception as e:
+            import pdb
 
-            self._propagate_forgotten(ts, recommendations)
+            pdb.set_trace()
 
-            self.report_on_key(ts=ts)
-            self.remove_key(key)
+    def get_metadata(self, comm=None, keys=None, default=no_default):
+        parent: SchedulerState = cast(SchedulerState, self)
+        metadata = parent._task_metadata
+        for key in keys[:-1]:
+            metadata = metadata[key]
+        try:
+            return metadata[keys[-1]]
+        except KeyError:
+            if default != no_default:
+                return default
+            else:
+                raise
 
-            return recommendations
-        except Exception as e:
-            logger.exception(e)
-            if LOG_PDB:
-                import pdb
+    def get_task_status(self, comm=None, keys=None):
+        parent: SchedulerState = cast(SchedulerState, self)
+        return {
+            key: (parent._tasks[key].state if key in parent._tasks else None)
+            for key in keys
+        }
 
-                pdb.set_trace()
-            raise
+    def get_task_stream(self, comm=None, start=None, stop=None, count=None):
+        from distributed.diagnostics.task_stream import TaskStreamPlugin
 
-    def transition_released_forgotten(self, key):
-        try:
-            tasks: dict = self.tasks
-            ts: TaskState = tasks[key]
+        self.add_plugin(TaskStreamPlugin, idempotent=True)
+        tsp = [p for p in self.plugins if isinstance(p, TaskStreamPlugin)][0]
+        return tsp.collect(start=start, stop=stop, count=count)
 
-            if self.validate:
-                assert ts._state in ("released", "erred")
-                assert not ts._who_has
-                assert not ts._processing_on
-                assert not ts._waiting_on, (ts, ts._waiting_on)
-                if not ts._run_spec:
-                    # It's ok to forget a pure data task
-                    pass
-                elif ts._has_lost_dependencies:
-                    # It's ok to forget a task with forgotten dependencies
-                    pass
-                elif not ts._who_wants and not ts._waiters and not ts._dependents:
-                    # It's ok to forget a task that nobody needs
-                    pass
-                else:
-                    assert 0, (ts,)
+    def start_task_metadata(self, comm=None, name=None):
+        plugin = CollectTaskMetaDataPlugin(scheduler=self, name=name)
 
-            recommendations: dict = {}
-            self._propagate_forgotten(ts, recommendations)
+        self.add_plugin(plugin)
 
-            self.report_on_key(ts=ts)
-            self.remove_key(key)
+    def stop_task_metadata(self, comm=None, name=None):
+        plugins = [
+            p
+            for p in self.plugins
+            if isinstance(p, CollectTaskMetaDataPlugin) and p.name == name
+        ]
+        if len(plugins) != 1:
+            raise ValueError(
+                "Expected to find exactly one CollectTaskMetaDataPlugin "
+                f"with name {name} but found {len(plugins)}."
+            )
 
-            return recommendations
-        except Exception as e:
-            logger.exception(e)
-            if LOG_PDB:
-                import pdb
+        plugin = plugins[0]
+        self.remove_plugin(plugin)
+        return {"metadata": plugin.metadata, "state": plugin.state}
 
-                pdb.set_trace()
-            raise
+    async def register_worker_plugin(self, comm, plugin, name=None):
+        """ Registers a setup function, and call it on every worker """
+        self.worker_plugins.append({"plugin": plugin, "name": name})
+
+        responses = await self.broadcast(
+            msg=dict(op="plugin-add", plugin=plugin, name=name)
+        )
+        return responses
+
+    #####################
+    # State Transitions #
+    #####################
 
     def transition(self, key, finish, *args, **kwargs):
         """Transition a key from its current state to the finish state
@@ -5509,10 +5819,13 @@ def transition(self, key, finish, *args, **kwargs):
         --------
         Scheduler.transitions: transitive version of this function
         """
+        parent: SchedulerState = cast(SchedulerState, self)
         ts: TaskState
+        worker_msgs: dict
+        client_msgs: dict
         try:
             try:
-                ts = self.tasks[key]
+                ts = parent._tasks[key]
             except KeyError:
                 return {}
             start = ts._state
@@ -5524,16 +5837,18 @@ def transition(self, key, finish, *args, **kwargs):
                 dependencies = set(ts._dependencies)
 
             recommendations: dict = {}
+            worker_msgs = {}
+            client_msgs = {}
             if (start, finish) in self._transitions:
                 func = self._transitions[start, finish]
-                recommendations = func(key, *args, **kwargs)
+                recommendations, worker_msgs, client_msgs = func(key, *args, **kwargs)
             elif "released" not in (start, finish):
                 func = self._transitions["released", finish]
                 assert not args and not kwargs
                 a = self.transition(key, "released")
                 if key in a:
                     func = self._transitions["released", a[key]]
-                b = func(key)
+                b, worker_msgs, client_msgs = func(key)
                 a = a.copy()
                 a.update(b)
                 recommendations = a
@@ -5543,9 +5858,14 @@ def transition(self, key, finish, *args, **kwargs):
                     "Impossible transition from %r to %r" % (start, finish)
                 )
 
+            for worker, msg in worker_msgs.items():
+                self.worker_send(worker, msg)
+            for client, msg in client_msgs.items():
+                self.client_send(client, msg)
+
             finish2 = ts._state
             self.transition_log.append((key, start, finish2, recommendations, time()))
-            if self.validate:
+            if parent._validate:
                 logger.debug(
                     "Transitioned %r %s->%s (actual: %s).  Consequence: %s",
                     key,
@@ -5562,14 +5882,14 @@ def transition(self, key, finish, *args, **kwargs):
                         ts._dependencies = dependencies
                     except KeyError:
                         pass
-                    self.tasks[ts._key] = ts
+                    parent._tasks[ts._key] = ts
                 for plugin in list(self.plugins):
                     try:
                         plugin.transition(key, start, finish2, *args, **kwargs)
                     except Exception:
                         logger.info("Plugin failed with exception", exc_info=True)
                 if ts._state == "forgotten":
-                    del self.tasks[ts._key]
+                    del parent._tasks[ts._key]
 
             if ts._state == "forgotten" and ts._group._name in self.task_groups:
                 # Remove TaskGroup if all tasks are in the forgotten state
@@ -5593,6 +5913,7 @@ def transitions(self, recommendations: dict):
         This includes feedback from previous transitions and continues until we
         reach a steady state
         """
+        parent: SchedulerState = cast(SchedulerState, self)
         keys = set()
         recommendations = recommendations.copy()
         while recommendations:
@@ -5601,7 +5922,7 @@ def transitions(self, recommendations: dict):
             new = self.transition(key, finish)
             recommendations.update(new)
 
-        if self.validate:
+        if parent._validate:
             for key in keys:
                 self.validate_key(key)
 
@@ -5620,9 +5941,10 @@ def reschedule(self, key=None, worker=None):
         Things may have shifted and this task may now be better suited to run
         elsewhere
         """
+        parent: SchedulerState = cast(SchedulerState, self)
         ts: TaskState
         try:
-            ts = self.tasks[key]
+            ts = parent._tasks[key]
         except KeyError:
             logger.warning(
                 "Attempting to reschedule task {}, which was not "
@@ -5635,130 +5957,26 @@ def reschedule(self, key=None, worker=None):
             return
         self.transitions({key: "released"})
 
-    ##############################
-    # Assigning Tasks to Workers #
-    ##############################
-
-    def check_idle_saturated(self, ws: WorkerState, occ: double = -1.0):
-        """Update the status of the idle and saturated state
-
-        The scheduler keeps track of workers that are ..
-
-        -  Saturated: have enough work to stay busy
-        -  Idle: do not have enough work to stay busy
-
-        They are considered saturated if they both have enough tasks to occupy
-        all of their threads, and if the expected runtime of those tasks is
-        large enough.
-
-        This is useful for load balancing and adaptivity.
-        """
-        total_nthreads: Py_ssize_t = self.total_nthreads
-        if total_nthreads == 0 or ws.status == Status.closed:
-            return
-        if occ < 0:
-            occ = ws._occupancy
-
-        nc: Py_ssize_t = ws._nthreads
-        p: Py_ssize_t = len(ws._processing)
-        total_occupancy: double = self.total_occupancy
-        avg: double = total_occupancy / total_nthreads
-
-        idle = self.idle
-        saturated: set = self.saturated
-        if p < nc or occ < nc * avg / 2:
-            idle[ws._address] = ws
-            saturated.discard(ws)
-        else:
-            idle.pop(ws._address, None)
-
-            if p > nc:
-                pending: double = occ * (p - nc) / (p * nc)
-                if 0.4 < pending > 1.9 * avg:
-                    saturated.add(ws)
-                    return
-
-            saturated.discard(ws)
-
-    def valid_workers(self, ts: TaskState) -> set:
-        """Return set of currently valid workers for key
-
-        If all workers are valid then this returns ``None``.
-        This checks tracks the following state:
-
-        *  worker_restrictions
-        *  host_restrictions
-        *  resource_restrictions
-        """
-        workers: dict = cast(dict, self.workers)
-        s: set = None
-
-        if ts._worker_restrictions:
-            s = {w for w in ts._worker_restrictions if w in workers}
-
-        if ts._host_restrictions:
-            # Resolve the alias here rather than early, for the worker
-            # may not be connected when host_restrictions is populated
-            hr: list = [self.coerce_hostname(h) for h in ts._host_restrictions]
-            # XXX need HostState?
-            sl: list = [
-                self.host_info[h]["addresses"] for h in hr if h in self.host_info
-            ]
-            ss: set = set.union(*sl) if sl else set()
-            if s is None:
-                s = ss
-            else:
-                s |= ss
-
-        if ts._resource_restrictions:
-            dw: dict = {
-                resource: {
-                    w
-                    for w, supplied in self.resources[resource].items()
-                    if supplied >= required
-                }
-                for resource, required in ts._resource_restrictions.items()
-            }
-
-            ww: set = set.intersection(*dw.values())
-            if s is None:
-                s = ww
-            else:
-                s &= ww
-
-        if s is not None:
-            s = {workers[w] for w in s}
-
-        return s
-
-    def consume_resources(self, ts: TaskState, ws: WorkerState):
-        if ts._resource_restrictions:
-            for r, required in ts._resource_restrictions.items():
-                ws._used_resources[r] += required
-
-    def release_resources(self, ts: TaskState, ws: WorkerState):
-        if ts._resource_restrictions:
-            for r, required in ts._resource_restrictions.items():
-                ws._used_resources[r] -= required
-
     #####################
     # Utility functions #
     #####################
 
     def add_resources(self, comm=None, worker=None, resources=None):
-        ws: WorkerState = self.workers[worker]
+        parent: SchedulerState = cast(SchedulerState, self)
+        ws: WorkerState = parent._workers_dv[worker]
         if resources:
             ws._resources.update(resources)
         ws._used_resources = {}
         for resource, quantity in ws._resources.items():
             ws._used_resources[resource] = 0
-            self.resources[resource][worker] = quantity
+            parent._resources[resource][worker] = quantity
         return "OK"
 
     def remove_resources(self, worker):
-        ws: WorkerState = self.workers[worker]
+        parent: SchedulerState = cast(SchedulerState, self)
+        ws: WorkerState = parent._workers_dv[worker]
         for resource, quantity in ws._resources.items():
-            del self.resources[resource][worker]
+            del parent._resources[resource][worker]
 
     def coerce_address(self, addr, resolve=True):
         """
@@ -5768,8 +5986,9 @@ def coerce_address(self, addr, resolve=True):
         Handles strings, tuples, or aliases.
         """
         # XXX how many address-parsing routines do we have?
-        if addr in self.aliases:
-            addr = self.aliases[addr]
+        parent: SchedulerState = cast(SchedulerState, self)
+        if addr in parent._aliases:
+            addr = parent._aliases[addr]
         if isinstance(addr, tuple):
             addr = unparse_host_port(*addr)
         if not isinstance(addr, str):
@@ -5782,15 +6001,6 @@ def coerce_address(self, addr, resolve=True):
 
         return addr
 
-    def coerce_hostname(self, host):
-        """
-        Coerce the hostname of a worker.
-        """
-        if host in self.aliases:
-            return self.workers[self.aliases[host]].host
-        else:
-            return host
-
     def workers_list(self, workers):
         """
         List of qualifying workers
@@ -5798,15 +6008,16 @@ def workers_list(self, workers):
         Takes a list of worker addresses or hostnames.
         Returns a list of all worker addresses that match
         """
+        parent: SchedulerState = cast(SchedulerState, self)
         if workers is None:
-            return list(self.workers)
+            return list(parent._workers)
 
         out = set()
         for w in workers:
             if ":" in w:
                 out.add(w)
             else:
-                out.update({ww for ww in self.workers if w in ww})  # TODO: quadratic
+                out.update({ww for ww in parent._workers if w in ww})  # TODO: quadratic
         return list(out)
 
     def start_ipython(self, comm=None):
@@ -5822,29 +6033,6 @@ def start_ipython(self, comm=None):
             )
         return self._ipython_kernel.get_connection_info()
 
-    def worker_objective(self, ts: TaskState, ws: WorkerState):
-        """
-        Objective function to determine which worker should get the task
-
-        Minimize expected start time.  If a tie then break with data storage.
-        """
-        dts: TaskState
-        nbytes: Py_ssize_t
-        comm_bytes: Py_ssize_t = 0
-        for dts in ts._dependencies:
-            if ws not in dts._who_has:
-                nbytes = dts.get_nbytes()
-                comm_bytes += nbytes
-
-        bandwidth: double = self.bandwidth
-        stack_time: double = ws._occupancy / ws._nthreads
-        start_time: double = stack_time + comm_bytes / bandwidth
-
-        if ts._actor:
-            return (len(ws._actors), start_time, ws._nbytes)
-        else:
-            return (start_time, ws._nbytes)
-
     async def get_profile(
         self,
         comm=None,
@@ -5856,10 +6044,11 @@ async def get_profile(
         stop=None,
         key=None,
     ):
+        parent: SchedulerState = cast(SchedulerState, self)
         if workers is None:
-            workers = self.workers
+            workers = parent._workers_dv
         else:
-            workers = set(self.workers) & set(workers)
+            workers = set(parent._workers_dv) & set(workers)
 
         if scheduler:
             return profile.get_profile(self.io_loop.profile, start=start, stop=stop)
@@ -5889,15 +6078,16 @@ async def get_profile_metadata(
         stop=None,
         profile_cycle_interval=None,
     ):
+        parent: SchedulerState = cast(SchedulerState, self)
         dt = profile_cycle_interval or dask.config.get(
             "distributed.worker.profile.cycle"
         )
         dt = parse_timedelta(dt, default="ms")
 
         if workers is None:
-            workers = self.workers
+            workers = parent._workers_dv
         else:
-            workers = set(self.workers) & set(workers)
+            workers = set(parent._workers_dv) & set(workers)
         results = await asyncio.gather(
             *(self.rpc(w).profile_metadata(start=start, stop=stop) for w in workers),
             return_exceptions=True,
@@ -5931,6 +6121,7 @@ async def get_profile_metadata(
         return {"counts": counts, "keys": keys}
 
     async def performance_report(self, comm=None, start=None, code=""):
+        parent: SchedulerState = cast(SchedulerState, self)
         stop = time()
         # Profiles
         compute, scheduler, workers = await asyncio.gather(
@@ -6015,10 +6206,10 @@ def profile_to_figure(state):
             ntasks=total_tasks,
             tasks_timings=tasks_timings,
             address=self.address,
-            nworkers=len(self.workers),
-            threads=sum([ws._nthreads for ws in self.workers.values()]),
+            nworkers=len(parent._workers_dv),
+            threads=sum([ws._nthreads for ws in parent._workers_dv.values()]),
             memory=format_bytes(
-                sum([ws._memory_limit for ws in self.workers.values()])
+                sum([ws._memory_limit for ws in parent._workers_dv.values()])
             ),
             code=code,
             dask_version=dask.__version__,
@@ -6107,6 +6298,7 @@ def reevaluate_occupancy(self, worker_index: Py_ssize_t = 0):
         lets us avoid this fringe optimization when we have better things to
         think about.
         """
+        parent: SchedulerState = cast(SchedulerState, self)
         try:
             if self.status == Status.closed:
                 return
@@ -6115,7 +6307,7 @@ def reevaluate_occupancy(self, worker_index: Py_ssize_t = 0):
             next_time = timedelta(seconds=0.1)
 
             if self.proc.cpu_percent() < 50:
-                workers: list = list(self.workers.values())
+                workers: list = list(parent._workers.values())
                 nworkers: Py_ssize_t = len(workers)
                 i: Py_ssize_t
                 for i in range(nworkers):
@@ -6124,7 +6316,7 @@ def reevaluate_occupancy(self, worker_index: Py_ssize_t = 0):
                     try:
                         if ws is None or not ws._processing:
                             continue
-                        self._reevaluate_occupancy_worker(ws)
+                        _reevaluate_occupancy_worker(parent, ws)
                     finally:
                         del ws  # lose ref
 
@@ -6141,36 +6333,13 @@ def reevaluate_occupancy(self, worker_index: Py_ssize_t = 0):
             logger.error("Error in reevaluate occupancy", exc_info=True)
             raise
 
-    def _reevaluate_occupancy_worker(self, ws: WorkerState):
-        """ See reevaluate_occupancy """
-        old: double = ws._occupancy
-        new: double = 0
-        diff: double
-        ts: TaskState
-        est: double
-        for ts in ws._processing:
-            est = self.set_duration_estimate(ts, ws)
-            new += est
-
-        ws._occupancy = new
-        diff = new - old
-        self.total_occupancy += diff
-        self.check_idle_saturated(ws)
-
-        # significant increase in duration
-        if new > old * 1.3:
-            steal = self.extensions.get("stealing")
-            if steal is not None:
-                for ts in ws._processing:
-                    steal.remove_key_from_stealable(ts)
-                    steal.put_key_in_stealable(ts)
-
     async def check_worker_ttl(self):
+        parent: SchedulerState = cast(SchedulerState, self)
         ws: WorkerState
         now = time()
-        for ws in self.workers.values():
+        for ws in parent._workers_dv.values():
             if (ws._last_seen < now - self.worker_ttl) and (
-                ws._last_seen < now - 10 * heartbeat_interval(len(self.workers))
+                ws._last_seen < now - 10 * heartbeat_interval(len(parent._workers_dv))
             ):
                 logger.warning(
                     "Worker failed to heartbeat within %s seconds. Closing: %s",
@@ -6180,8 +6349,12 @@ async def check_worker_ttl(self):
                 await self.remove_worker(address=ws._address)
 
     def check_idle(self):
+        parent: SchedulerState = cast(SchedulerState, self)
         ws: WorkerState
-        if any([ws._processing for ws in self.workers.values()]) or self.unrunnable:
+        if (
+            any([ws._processing for ws in parent._workers_dv.values()])
+            or parent._unrunnable
+        ):
             self.idle_since = None
             return
         elif not self.idle_since:
@@ -6210,19 +6383,20 @@ def adaptive_target(self, comm=None, target_duration=None):
         --------
         distributed.deploy.Adaptive
         """
+        parent: SchedulerState = cast(SchedulerState, self)
         if target_duration is None:
             target_duration = dask.config.get("distributed.adaptive.target-duration")
         target_duration = parse_timedelta(target_duration)
 
         # CPU
         cpu = math.ceil(
-            self.total_occupancy / target_duration
+            parent._total_occupancy / target_duration
         )  # TODO: threads per worker
 
         # Avoid a few long tasks from asking for many cores
         ws: WorkerState
         tasks_processing = 0
-        for ws in self.workers.values():
+        for ws in parent._workers_dv.values():
             tasks_processing += len(ws._processing)
 
             if tasks_processing > cpu:
@@ -6230,25 +6404,295 @@ def adaptive_target(self, comm=None, target_duration=None):
         else:
             cpu = min(tasks_processing, cpu)
 
-        if self.unrunnable and not self.workers:
+        if parent._unrunnable and not parent._workers_dv:
             cpu = max(1, cpu)
 
         # Memory
-        limit_bytes = {addr: ws._memory_limit for addr, ws in self.workers.items()}
-        worker_bytes = [ws._nbytes for ws in self.workers.values()]
+        limit_bytes = {
+            addr: ws._memory_limit for addr, ws in parent._workers_dv.items()
+        }
+        worker_bytes = [ws._nbytes for ws in parent._workers_dv.values()]
         limit = sum(limit_bytes.values())
         total = sum(worker_bytes)
         if total > 0.6 * limit:
-            memory = 2 * len(self.workers)
+            memory = 2 * len(parent._workers_dv)
         else:
             memory = 0
 
         target = max(memory, cpu)
-        if target >= len(self.workers):
+        if target >= len(parent._workers_dv):
             return target
         else:  # Scale down?
             to_close = self.workers_to_close()
-            return len(self.workers) - len(to_close)
+            return len(parent._workers_dv) - len(to_close)
+
+
+@cfunc
+@exceptval(check=False)
+def _remove_from_processing(state: SchedulerState, ts: TaskState) -> str:
+    """
+    Remove *ts* from the set of processing tasks.
+    """
+    ws: WorkerState = ts._processing_on
+    ts._processing_on = None
+    w: str = ws._address
+    if w in state._workers_dv:  # may have been removed
+        duration = ws._processing.pop(ts)
+        if not ws._processing:
+            state._total_occupancy -= ws._occupancy
+            ws._occupancy = 0
+        else:
+            state._total_occupancy -= duration
+            ws._occupancy -= duration
+        state.check_idle_saturated(ws)
+        state.release_resources(ts, ws)
+        return w
+    else:
+        return None
+
+
+@cfunc
+@exceptval(check=False)
+def _add_to_memory(
+    state: SchedulerState,
+    ts: TaskState,
+    ws: WorkerState,
+    recommendations: dict,
+    client_msgs: dict,
+    type=None,
+    typename: str = None,
+):
+    """
+    Add *ts* to the set of in-memory tasks.
+    """
+    if state._validate:
+        assert ts not in ws._has_what
+
+    ts._who_has.add(ws)
+    ws._has_what.add(ts)
+    ws._nbytes += ts.get_nbytes()
+
+    deps: list = list(ts._dependents)
+    if len(deps) > 1:
+        deps.sort(key=operator.attrgetter("priority"), reverse=True)
+
+    dts: TaskState
+    s: set
+    for dts in deps:
+        s = dts._waiting_on
+        if ts in s:
+            s.discard(ts)
+            if not s:  # new task ready to run
+                recommendations[dts._key] = "processing"
+
+    for dts in ts._dependencies:
+        s = dts._waiters
+        s.discard(ts)
+        if not s and not dts._who_wants:
+            recommendations[dts._key] = "released"
+
+    report_msg: dict = {}
+    cs: ClientState
+    if not ts._waiters and not ts._who_wants:
+        recommendations[ts._key] = "released"
+    else:
+        report_msg["op"] = "key-in-memory"
+        report_msg["key"] = ts._key
+        if type is not None:
+            report_msg["type"] = type
+
+        for cs in ts._who_wants:
+            client_msgs[cs._client_key] = report_msg
+
+    ts.state = "memory"
+    ts._type = typename
+    ts._group._types.add(typename)
+
+    cs = state._clients["fire-and-forget"]
+    if ts in cs._wants_what:
+        _client_releases_keys(
+            state,
+            cs=cs,
+            keys=[ts._key],
+            recommendations=recommendations,
+        )
+
+
+@cfunc
+@exceptval(check=False)
+def _propagate_forgotten(
+    state: SchedulerState, ts: TaskState, recommendations: dict, worker_msgs: dict
+):
+    ts.state = "forgotten"
+    key: str = ts._key
+    dts: TaskState
+    for dts in ts._dependents:
+        dts._has_lost_dependencies = True
+        dts._dependencies.remove(ts)
+        dts._waiting_on.discard(ts)
+        if dts._state not in ("memory", "erred"):
+            # Cannot compute task anymore
+            recommendations[dts._key] = "forgotten"
+    ts._dependents.clear()
+    ts._waiters.clear()
+
+    for dts in ts._dependencies:
+        dts._dependents.remove(ts)
+        s: set = dts._waiters
+        s.discard(ts)
+        if not dts._dependents and not dts._who_wants:
+            # Task not needed anymore
+            assert dts is not ts
+            recommendations[dts._key] = "forgotten"
+    ts._dependencies.clear()
+    ts._waiting_on.clear()
+
+    if ts._who_has:
+        ts._group._nbytes_in_memory -= ts.get_nbytes()
+
+    ws: WorkerState
+    for ws in ts._who_has:
+        ws._has_what.remove(ts)
+        ws._nbytes -= ts.get_nbytes()
+        w: str = ws._address
+        if w in state._workers_dv:  # in case worker has died
+            worker_msgs[w] = {"op": "delete-data", "keys": [key], "report": False}
+    ts._who_has.clear()
+
+
+@cfunc
+@exceptval(check=False)
+def _client_releases_keys(
+    state: SchedulerState, keys: list, cs: ClientState, recommendations: dict
+):
+    """ Remove keys from client desired list """
+    logger.debug("Client %s releases keys: %s", cs._client_key, keys)
+    ts: TaskState
+    tasks2: set = set()
+    for key in keys:
+        ts = state._tasks.get(key)
+        if ts is not None and ts in cs._wants_what:
+            cs._wants_what.remove(ts)
+            s: set = ts._who_wants
+            s.remove(cs)
+            if not s:
+                tasks2.add(ts)
+
+    for ts in tasks2:
+        if not ts._dependents:
+            # No live dependents, can forget
+            recommendations[ts._key] = "forgotten"
+        elif ts._state != "erred" and not ts._waiters:
+            recommendations[ts._key] = "released"
+
+
+@cfunc
+@exceptval(check=False)
+def _task_to_msg(state: SchedulerState, ts: TaskState, duration=None) -> dict:
+    """ Convert a single computational task to a message """
+    ws: WorkerState
+    dts: TaskState
+
+    if duration is None:
+        duration = state.get_task_duration(ts)
+
+    msg: dict = {
+        "op": "compute-task",
+        "key": ts._key,
+        "priority": ts._priority,
+        "duration": duration,
+    }
+    if ts._resource_restrictions:
+        msg["resource_restrictions"] = ts._resource_restrictions
+    if ts._actor:
+        msg["actor"] = True
+
+    deps: set = ts._dependencies
+    if deps:
+        msg["who_has"] = {
+            dts._key: [ws._address for ws in dts._who_has] for dts in deps
+        }
+        msg["nbytes"] = {dts._key: dts._nbytes for dts in deps}
+
+        if state._validate:
+            assert all(msg["who_has"].values())
+
+    task = ts._run_spec
+    if type(task) is dict:
+        msg.update(task)
+    else:
+        msg["task"] = task
+
+    return msg
+
+
+@cfunc
+@exceptval(check=False)
+def _task_to_report_msg(state: SchedulerState, ts: TaskState) -> dict:
+    if ts is None:
+        return {"op": "cancelled-key", "key": ts._key}
+    elif ts._state == "forgotten":
+        return {"op": "cancelled-key", "key": ts._key}
+    elif ts._state == "memory":
+        return {"op": "key-in-memory", "key": ts._key}
+    elif ts._state == "erred":
+        failing_ts: TaskState = ts._exception_blame
+        return {
+            "op": "task-erred",
+            "key": ts._key,
+            "exception": failing_ts._exception,
+            "traceback": failing_ts._traceback,
+        }
+    else:
+        return None
+
+
+@cfunc
+@exceptval(check=False)
+def _task_to_client_msgs(state: SchedulerState, ts: TaskState) -> dict:
+    cs: ClientState
+    client_keys: list
+    if ts is None:
+        # Notify all clients
+        client_keys = list(state._clients)
+    else:
+        # Notify clients interested in key
+        client_keys = [cs._client_key for cs in ts._who_wants]
+
+    report_msg: dict = _task_to_report_msg(state, ts)
+
+    client_msgs: dict = {}
+    for k in client_keys:
+        client_msgs[k] = report_msg
+
+    return client_msgs
+
+
+@cfunc
+@exceptval(check=False)
+def _reevaluate_occupancy_worker(state: SchedulerState, ws: WorkerState):
+    """ See reevaluate_occupancy """
+    old: double = ws._occupancy
+    new: double = 0
+    diff: double
+    ts: TaskState
+    est: double
+    for ts in ws._processing:
+        est = state.set_duration_estimate(ts, ws)
+        new += est
+
+    ws._occupancy = new
+    diff = new - old
+    state._total_occupancy += diff
+    state.check_idle_saturated(ws)
+
+    # significant increase in duration
+    if new > old * 1.3:
+        steal = state._extensions.get("stealing")
+        if steal is not None:
+            for ts in ws._processing:
+                steal.remove_key_from_stealable(ts)
+                steal.put_key_in_stealable(ts)
 
 
 @cfunc

From 777d48e977021253f071bce038a7eac40f468c82 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Fri, 22 Jan 2021 15:59:54 -0800
Subject: [PATCH 1133/1550] Use `parent._tasks` in heartbeat (#4450)

Make sure we grab the typed `parent._tasks` in heartbeat. This benefits
from the type annotation of this attribute.
---
 distributed/scheduler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 896b10e1380..5a60eac5f92 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -3411,7 +3411,7 @@ def heartbeat_worker(
 
         if executing is not None:
             ws._executing = {
-                self.tasks[key]: duration for key, duration in executing.items()
+                parent._tasks[key]: duration for key, duration in executing.items()
             }
 
         if metrics:

From 2089e79b3afbc3e974981f3e3411bbcbb9c4d0a6 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Sat, 23 Jan 2021 08:25:26 -0800
Subject: [PATCH 1134/1550] Refactor `task_groups` & `task_prefixes` (#4452)

* Refactor `task_groups` & `task_prefixes`

Moves `task_groups` and `task_prefixes` to `SchedulerState` where they
are type annotated. Then uses them through `parent` within `Scheduler`.
Allows Cython to recognize these are Python `dict`s and optimize calls
and operations on them.

* Use `.get(...)` instead of `try...except...`

Instead of using `try...except...` to catch and handle `KeyError`s, just
use `.get(...), which gets the key needed or returns `None`. This has
less overhead. Also the following `None` check is a quick pointer
comparison. Otherwise the code is unchanged.

* Assign `TaskGroup` before `if`

Since it is used in the check as well, go ahead and assign it beforehand
for simplicity.
---
 distributed/scheduler.py | 36 +++++++++++++++++++++++-------------
 1 file changed, 23 insertions(+), 13 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 5a60eac5f92..f2126a80e41 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1585,6 +1585,8 @@ class SchedulerState:
     _resources: object
     _saturated: set
     _tasks: dict
+    _task_groups: dict
+    _task_prefixes: dict
     _task_metadata: dict
     _total_nthreads: Py_ssize_t
     _total_occupancy: double
@@ -1635,6 +1637,8 @@ def __init__(
             self._tasks = tasks
         else:
             self._tasks = dict()
+        self._task_groups = dict()
+        self._task_prefixes = dict()
         self._task_metadata = dict()
         self._total_nthreads = 0
         self._total_occupancy = 0
@@ -1691,6 +1695,14 @@ def saturated(self):
     def tasks(self):
         return self._tasks
 
+    @property
+    def task_groups(self):
+        return self._task_groups
+
+    @property
+    def task_prefixes(self):
+        return self._task_prefixes
+
     @property
     def task_metadata(self):
         return self._task_metadata
@@ -1738,6 +1750,8 @@ def __pdict__(self):
             "unknown_durations": self._unknown_durations,
             "validate": self._validate,
             "tasks": self._tasks,
+            "task_groups": self._task_groups,
+            "task_prefixes": self._task_prefixes,
             "total_nthreads": self._total_nthreads,
             "total_occupancy": self._total_occupancy,
             "extensions": self._extensions,
@@ -2926,8 +2940,6 @@ def __init__(
 
         # Task state
         tasks = dict()
-        self.task_groups = dict()
-        self.task_prefixes = dict()
         for old_attr, new_attr, wrap in [
             ("priority", "priority", None),
             ("dependencies", "dependencies", _legacy_task_key_set),
@@ -3919,17 +3931,15 @@ def new_task(self, key, spec, state):
         tg: TaskGroup
         ts._state = state
         prefix_key = key_split(key)
-        try:
-            tp = self.task_prefixes[prefix_key]
-        except KeyError:
-            self.task_prefixes[prefix_key] = tp = TaskPrefix(prefix_key)
+        tp = parent._task_prefixes.get(prefix_key)
+        if tp is None:
+            parent._task_prefixes[prefix_key] = tp = TaskPrefix(prefix_key)
         ts._prefix = tp
 
         group_key = ts._group_key
-        try:
-            tg = self.task_groups[group_key]
-        except KeyError:
-            self.task_groups[group_key] = tg = TaskGroup(group_key)
+        tg = parent._task_groups.get(group_key)
+        if tg is None:
+            parent._task_groups[group_key] = tg = TaskGroup(group_key)
             tg._prefix = tp
             tp._groups.append(tg)
         tg.add(ts)
@@ -5891,12 +5901,12 @@ def transition(self, key, finish, *args, **kwargs):
                 if ts._state == "forgotten":
                     del parent._tasks[ts._key]
 
-            if ts._state == "forgotten" and ts._group._name in self.task_groups:
+            tg: TaskGroup = ts._group
+            if ts._state == "forgotten" and tg._name in parent._task_groups:
                 # Remove TaskGroup if all tasks are in the forgotten state
-                tg: TaskGroup = ts._group
                 if not any([tg._states.get(s) for s in ALL_TASK_STATES]):
                     ts._prefix._groups.remove(tg)
-                    del self.task_groups[tg._name]
+                    del parent._task_groups[tg._name]
 
             return recommendations
         except Exception as e:

From 3a94ebc12b3922bf05f5d7ca4bc6fd117986504d Mon Sep 17 00:00:00 2001
From: Gil Forsyth <gforsyth@users.noreply.github.com>
Date: Mon, 25 Jan 2021 17:00:31 -0500
Subject: [PATCH 1135/1550] Fix var name in worker validation func (#4457)

---
 distributed/worker.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/worker.py b/distributed/worker.py
index 4d45fa46743..31f16d30d54 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -3007,7 +3007,7 @@ def validate_state(self):
                     assert (
                         ts_wait.state == "flight"
                         or ts_wait.state == "waiting"
-                        or ts.wait.key in self._missing_dep_flight
+                        or ts_wait.key in self._missing_dep_flight
                         or ts_wait.who_has.issubset(self.in_flight_workers)
                     )
                 if ts.state == "memory":

From b5c36b587f0e3295fe59db330603d5c78b39331f Mon Sep 17 00:00:00 2001
From: Tom Augspurger <1312546+TomAugspurger@users.noreply.github.com>
Date: Mon, 25 Jan 2021 17:30:07 -0600
Subject: [PATCH 1136/1550] Added worker resources from config (#4456)

---
 distributed/distributed-schema.yaml |  8 ++++++
 distributed/distributed.yaml        |  1 +
 distributed/tests/test_resources.py | 18 ++++++++++++
 distributed/worker.py               |  3 ++
 docs/source/resources.rst           | 43 +++++++++++++++++++++++++++++
 5 files changed, 73 insertions(+)

diff --git a/distributed/distributed-schema.yaml b/distributed/distributed-schema.yaml
index 89730d83d2c..7e8cba0a3c2 100644
--- a/distributed/distributed-schema.yaml
+++ b/distributed/distributed-schema.yaml
@@ -305,6 +305,14 @@ properties:
               Whether or not to run consistency checks during execution.
               This is typically only used for debugging.
 
+          resources:
+            type: object
+            description: |
+              A dictionary specifying resources for workers.
+
+              See https://distributed.dask.org/en/latest/resources.html for more information.
+            properties: {}
+
           lifetime:
             type: object
             description: |
diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index 764478a8824..36fc7baead1 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -70,6 +70,7 @@ distributed:
     preload-argv: []        # See https://docs.dask.org/en/latest/setup/custom-startup.html
     daemon: True
     validate: False         # Check worker state at every step for debugging
+    resources: {}           # Key: value pairs specifying worker resources.
     lifetime:
       duration: null        # Time after which to gracefully shutdown the worker
       stagger: 0 seconds    # Random amount by which to stagger lifetimes
diff --git a/distributed/tests/test_resources.py b/distributed/tests/test_resources.py
index ac7c06f07eb..e055a4b362f 100644
--- a/distributed/tests/test_resources.py
+++ b/distributed/tests/test_resources.py
@@ -392,3 +392,21 @@ def g(dask_worker):
     logs = client.run(g)
     assert logs[a["address"]]
     assert not logs[b["address"]]
+
+
+@gen_cluster(config={"distributed.worker.resources.my_resources": 1}, client=True)
+async def test_resources_from_config(c, s, a, b):
+    info = c.scheduler_info()
+    for worker in [a, b]:
+        assert info["workers"][worker.address]["resources"] == {"my_resources": 1}
+
+
+@gen_cluster(
+    worker_kwargs=dict(resources={"my_resources": 10}),
+    config={"distributed.worker.resources.my_resources": 1},
+    client=True,
+)
+async def test_resources_from_python_override_config(c, s, a, b):
+    info = c.scheduler_info()
+    for worker in [a, b]:
+        assert info["workers"][worker.address]["resources"] == {"my_resources": 10}
diff --git a/distributed/worker.py b/distributed/worker.py
index 31f16d30d54..4cf60b597e9 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -519,6 +519,9 @@ def __init__(
             nthreads = ncores
 
         self.nthreads = nthreads or CPU_COUNT
+        if resources is None:
+            resources = dask.config.get("distributed.worker.resources", None)
+
         self.total_resources = resources or {}
         self.available_resources = (resources or {}).copy()
         self.death_timeout = parse_timedelta(death_timeout)
diff --git a/docs/source/resources.rst b/docs/source/resources.rst
index 7931b980d03..f0c7c0ead60 100644
--- a/docs/source/resources.rst
+++ b/docs/source/resources.rst
@@ -47,6 +47,49 @@ When we submit tasks to the cluster we specify constraints per task
    processed = [client.submit(process, d, resources={'GPU': 1}) for d in data]
    final = client.submit(aggregate, processed, resources={'MEMORY': 70e9})
 
+Specifying Resources
+--------------------
+
+Resources can be specifed in several ways. The easiest option will depend on exactly
+how your cluster is being created.
+
+**From the command line**
+
+Resources can be provided when starting the worker process, as shown above:
+
+.. code-block:: console
+
+   dask-worker scheduler:8786 --resources "GPU=2"
+
+The keys are used as the resource name and the values are parsed into a numeric value.
+
+**From Dask's configuration system**
+
+Alternatively, resources can be specified using Dask's
+`configuration system <https://docs.dask.org/en/latest/configuration.html>`_.
+
+.. code-block:: python
+
+   from distributed import LocalCluster
+
+   with dask.config.set({"distributed.worker.resources.GPU": 2}):
+       cluster = LocalCluster()
+
+The configuration will need to be set in the process that's spawning the actual worker.
+This might be easiest to achieve by specifying resources as an environment variable
+(shown in the next section).
+
+**From environment variables**
+
+Like any other Dask config value, resources can be specified as environment variables
+before starting the process. Using Bash syntax
+
+.. code-block:: console
+
+   $ DASK_DISTRIBUTED__WORKER__RESOURCES__GPU=2 dask-worker
+   ...
+
+This might be the easiest solution if you aren't able to pass options to the :class:`distributed.Worker` class.
 
 Resources are applied separately to each worker process
 -------------------------------------------------------

From 1297b18ff09276f6ad1553d40ab3ce77acf0fc0e Mon Sep 17 00:00:00 2001
From: Ian Rose <ian.r.rose@gmail.com>
Date: Thu, 28 Jan 2021 16:39:57 -0800
Subject: [PATCH 1137/1550] Unify annotations (#4406)

Previously we had two systems to send per-task metadata like retries or
workers or priorities to the scheduler.

1.  Older system with explicit workers= keywords and expand_foo functions
2.  Newer system with annotations

The annotations system is nicer for a few reasons:

1.  It's more generic
2.  It's more consistent (there were some bugs in the expand foo
    functions, especially when dealing with collections)
3.  We ship values up on a per-layer basis rather than a per-key basis

This rips out the old system and uses the new system.
---
 distributed/client.py                  | 282 +++++++------------------
 distributed/protocol/highlevelgraph.py |  31 ++-
 distributed/scheduler.py               |   3 +-
 distributed/tests/test_client.py       | 261 +++++++++++++++--------
 distributed/tests/test_priorities.py   |  20 +-
 distributed/tests/test_resources.py    |  41 ++--
 distributed/tests/test_scheduler.py    |   9 +-
 docs/source/priority.rst               |  17 +-
 docs/source/resources.rst              |  31 +--
 9 files changed, 342 insertions(+), 353 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 82f9076c1f0..7c1ea57c119 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -11,10 +11,9 @@
 from functools import partial
 import html
 import inspect
-import itertools
 import json
 import logging
-from numbers import Number, Integral
+from numbers import Number
 import os
 import sys
 import uuid
@@ -1507,13 +1506,13 @@ def submit(
         pure: bool (defaults to True)
             Whether or not the function is pure.  Set ``pure=False`` for
             impure functions like ``np.random.random``.
-        workers: set, iterable of sets
-            A set of worker hostnames on which computations may be performed.
-            Leave empty to default to all workers (common case)
+        workers: string or iterable of strings
+            A set of worker addresses or hostnames on which computations may be
+            performed. Leave empty to default to all workers (common case)
         key: str
             Unique identifier for the task.  Defaults to function-name and hash
         allow_other_workers: bool (defaults to False)
-            Used with `workers`. Indicates whether or not the computations
+            Used with ``workers``. Indicates whether or not the computations
             may be performed on workers that are not in the `workers` set(s).
         retries: int (default to 0)
             Number of allowed automatic retries if the task fails
@@ -1523,9 +1522,10 @@ def submit(
         fifo_timeout: str timedelta (default '100ms')
             Allowed amount of time between calls to consider the same priority
         resources: dict (defaults to {})
-            Defines the `resources` this job requires on the worker; e.g.
-            ``{'GPU': 2}``. See :doc:`worker resources <resources>` for details
-            on defining resources.
+            Defines the ``resources`` each instance of this mapped task requires
+            on the worker; e.g. ``{'GPU': 2}``.
+            See :doc:`worker resources <resources>` for details on defining
+            resources.
         actor: bool (default False)
             Whether this task should exist on the worker as a stateful actor.
             See :doc:`actors` for additional details.
@@ -1571,12 +1571,6 @@ def submit(
 
         if isinstance(workers, (str, Number)):
             workers = [workers]
-        if workers is not None:
-            restrictions = {skey: workers}
-            loose_restrictions = [skey] if allow_other_workers else []
-        else:
-            restrictions = {}
-            loose_restrictions = []
 
         if kwargs:
             dsk = {skey: (apply, func, list(args), kwargs)}
@@ -1586,11 +1580,11 @@ def submit(
         futures = self._graph_to_futures(
             dsk,
             [skey],
-            restrictions,
-            loose_restrictions,
+            workers=workers,
+            allow_other_workers=allow_other_workers,
             priority={skey: 0},
             user_priority=priority,
-            resources={skey: resources} if resources else None,
+            resources=resources,
             retries=retries,
             fifo_timeout=fifo_timeout,
             actors=actor,
@@ -1631,7 +1625,7 @@ def map(
         pure: bool (defaults to True)
             Whether or not the function is pure.  Set ``pure=False`` for
             impure functions like ``np.random.random``.
-        workers: set, iterable of sets
+        workers: string or iterable of strings
             A set of worker hostnames on which computations may be performed.
             Leave empty to default to all workers (common case)
         allow_other_workers: bool (defaults to False)
@@ -1646,8 +1640,8 @@ def map(
             Allowed amount of time between calls to consider the same priority
         resources: dict (defaults to {})
             Defines the `resources` each instance of this mapped task requires
-            on the worker; e.g. ``{'GPU': 2}``. See
-            :doc:`worker resources <resources>` for details on defining
+            on the worker; e.g. ``{'GPU': 2}``.
+            See :doc:`worker resources <resources>` for details on defining
             resources.
         actor: bool (default False)
             Whether these tasks should exist on the worker as stateful actors.
@@ -1763,39 +1757,16 @@ def map(
 
         if isinstance(workers, (str, Number)):
             workers = [workers]
-        if isinstance(workers, (list, set)):
-            if workers and isinstance(first(workers), (list, set)):
-                if len(workers) != len(keys):
-                    raise ValueError(
-                        "You only provided %d worker restrictions"
-                        " for a sequence of length %d" % (len(workers), len(keys))
-                    )
-                restrictions = dict(zip(keys, workers))
-            else:
-                restrictions = {k: workers for k in keys}
-        elif workers is None:
-            restrictions = {}
-        else:
+        if workers is not None and not isinstance(workers, (list, set)):
             raise TypeError("Workers must be a list or set of workers or None")
-        if allow_other_workers not in (True, False, None):
-            raise TypeError("allow_other_workers= must be True or False")
-        if allow_other_workers is True:
-            loose_restrictions = set(keys)
-        else:
-            loose_restrictions = set()
 
         internal_priority = dict(zip(keys, range(len(keys))))
 
-        if resources:
-            resources = {k: resources for k in keys}
-        else:
-            resources = None
-
         futures = self._graph_to_futures(
             dsk,
             keys,
-            restrictions,
-            loose_restrictions,
+            workers=workers,
+            allow_other_workers=allow_other_workers,
             priority=internal_priority,
             resources=resources,
             retries=retries,
@@ -2551,8 +2522,8 @@ def _graph_to_futures(
         self,
         dsk,
         keys,
-        restrictions=None,
-        loose_restrictions=None,
+        workers=None,
+        allow_other_workers=None,
         priority=None,
         user_priority=0,
         resources=None,
@@ -2561,38 +2532,35 @@ def _graph_to_futures(
         actors=None,
     ):
         with self._refcount_lock:
-            if resources:
-                resources = self._expand_resources(
-                    resources, all_keys=itertools.chain(dsk, keys)
-                )
-                resources = {stringify(k): v for k, v in resources.items()}
-
-            if retries:
-                retries = self._expand_retries(
-                    retries, all_keys=itertools.chain(dsk, keys)
-                )
-
             if actors is not None and actors is not True and actors is not False:
                 actors = list(self._expand_key(actors))
 
-            if restrictions:
-                restrictions = keymap(stringify, restrictions)
-                restrictions = valmap(list, restrictions)
-
-            if loose_restrictions is not None:
-                loose_restrictions = list(map(stringify, loose_restrictions))
-
             keyset = set(keys)
 
             # Make sure `dsk` is a high level graph
             if not isinstance(dsk, HighLevelGraph):
                 dsk = HighLevelGraph.from_collections(id(dsk), dsk, dependencies=())
 
-            if isinstance(retries, Number) and retries > 0:
-                retries = {k: retries for k in dsk}
-
             dsk = highlevelgraph_pack(dsk, self, keyset)
 
+            annotations = {}
+            if user_priority:
+                annotations["priority"] = user_priority
+            if workers:
+                if not isinstance(workers, (list, tuple, set)):
+                    workers = [workers]
+                annotations["workers"] = workers
+            if retries:
+                annotations["retries"] = retries
+            if allow_other_workers not in (True, False, None):
+                raise TypeError("allow_other_workers= must be True, False, or None")
+            if allow_other_workers:
+                annotations["allow_other_workers"] = allow_other_workers
+            if resources:
+                annotations["resources"] = resources
+
+            annotations = merge(dask.config.get("annotations", {}), annotations)
+
             # Create futures before sending graph (helps avoid contention)
             futures = {key: Future(key, self, inform=False) for key in keyset}
             self._send_to_scheduler(
@@ -2600,14 +2568,10 @@ def _graph_to_futures(
                     "op": "update-graph-hlg",
                     "hlg": dsk,
                     "keys": list(map(stringify, keys)),
-                    "restrictions": restrictions or {},
-                    "loose_restrictions": loose_restrictions,
                     "priority": priority,
-                    "user_priority": user_priority,
-                    "resources": resources,
                     "submitting_task": getattr(thread_state, "key", None),
-                    "retries": retries,
                     "fifo_timeout": fifo_timeout,
+                    "annotations": annotations,
                     "actors": actors,
                 }
             )
@@ -2617,8 +2581,8 @@ def get(
         self,
         dsk,
         keys,
-        restrictions=None,
-        loose_restrictions=None,
+        workers=None,
+        allow_other_workers=None,
         resources=None,
         sync=True,
         asynchronous=None,
@@ -2635,14 +2599,22 @@ def get(
         ----------
         dsk: dict
         keys: object, or nested lists of objects
-        restrictions: dict (optional)
-            A mapping of {key: {set of worker hostnames}} that restricts where
-            jobs can take place
+        workers: string or iterable of strings
+            A set of worker addresses or hostnames on which computations may be
+            performed. Leave empty to default to all workers (common case)
+        allow_other_workers: bool (defaults to False)
+            Used with ``workers``. Indicates whether or not the computations
+            may be performed on workers that are not in the `workers` set(s).
         retries: int (default to 0)
             Number of allowed automatic retries if computing a result fails
         priority: Number
             Optional prioritization of task.  Zero is default.
             Higher priorities take precedence
+        resources: dict (defaults to {})
+            Defines the ``resources`` each instance of this mapped task requires
+            on the worker; e.g. ``{'GPU': 2}``.
+            See :doc:`worker resources <resources>` for details on defining
+            resources.
         sync: bool (optional)
             Returns Futures if False or concrete values if True (default).
         direct: bool
@@ -2664,8 +2636,8 @@ def get(
         futures = self._graph_to_futures(
             dsk,
             keys=set(flatten([keys])),
-            restrictions=restrictions,
-            loose_restrictions=loose_restrictions,
+            workers=workers,
+            allow_other_workers=allow_other_workers,
             resources=resources,
             fifo_timeout=fifo_timeout,
             retries=retries,
@@ -2770,15 +2742,12 @@ def compute(
             Returns Futures if False (default) or concrete values if True
         optimize_graph: bool
             Whether or not to optimize the underlying graphs
-        workers: str, list, dict
-            Which workers can run which parts of the computation
-            If a string or list then the output collections will run on the listed
-            workers, but other sub-computations can run anywhere
-            If a dict then keys should be (tuples of) collections or
-            task keys and values should be addresses or lists.
-        allow_other_workers: bool, list
-            If True then all restrictions in workers= are considered loose
-            If a list then only the keys for the listed collections are loose
+        workers: string or iterable of strings
+            A set of worker hostnames on which computations may be performed.
+            Leave empty to default to all workers (common case)
+        allow_other_workers: bool (defaults to False)
+            Used with `workers`. Indicates whether or not the computations
+            may be performed on workers that are not in the `workers` set(s).
         retries: int (default to 0)
             Number of allowed automatic retries if computing a result fails
         priority: Number
@@ -2792,9 +2761,8 @@ def compute(
             be expensive. If none of the arguments contain any dask objects,
             set ``traverse=False`` to avoid doing this traversal.
         resources: dict (defaults to {})
-            Defines the `resources` these tasks require on the worker. Can
-            specify global resources (``{'GPU': 2}``), or per-task resources
-            (``{'x': {'GPU': 1}, 'y': {'SSD': 4}}``), but not both.
+            Defines the `resources` each instance of this mapped task requires
+            on the worker; e.g. ``{'GPU': 2}``.
             See :doc:`worker resources <resources>` for details on defining
             resources.
         actors: bool or dict (default None)
@@ -2857,13 +2825,6 @@ def compute(
             else:
                 dsk2[name] = (func, keys) + extra_args
 
-        restrictions, loose_restrictions = self.get_restrictions(
-            collections, workers, allow_other_workers
-        )
-
-        if not isinstance(priority, Number):
-            priority = {k: p for c, p in priority.items() for k in self._expand_key(c)}
-
         if not isinstance(dsk, HighLevelGraph):
             dsk = HighLevelGraph.from_collections(id(dsk), dsk, dependencies=())
 
@@ -2878,8 +2839,8 @@ def compute(
         futures_dict = self._graph_to_futures(
             dsk,
             names,
-            restrictions,
-            loose_restrictions,
+            workers=workers,
+            allow_other_workers=allow_other_workers,
             resources=resources,
             retries=retries,
             user_priority=priority,
@@ -2931,15 +2892,12 @@ def persist(
             Collections like dask.array or dataframe or dask.value objects
         optimize_graph: bool
             Whether or not to optimize the underlying graphs
-        workers: str, list, dict
-            Which workers can run which parts of the computation
-            If a string or list then the output collections will run on the listed
-            workers, but other sub-computations can run anywhere
-            If a dict then keys should be (tuples of) collections or
-            task keys and values should be addresses or lists.
-        allow_other_workers: bool, list
-            If True then all restrictions in workers= are considered loose
-            If a list then only the keys for the listed collections are loose
+        workers: string or iterable of strings
+            A set of worker hostnames on which computations may be performed.
+            Leave empty to default to all workers (common case)
+        allow_other_workers: bool (defaults to False)
+            Used with `workers`. Indicates whether or not the computations
+            may be performed on workers that are not in the `workers` set(s).
         retries: int (default to 0)
             Number of allowed automatic retries if computing a result fails
         priority: Number
@@ -2948,9 +2906,8 @@ def persist(
         fifo_timeout: timedelta str (defaults to '60s')
             Allowed amount of time between calls to consider the same priority
         resources: dict (defaults to {})
-            Defines the `resources` these tasks require on the worker. Can
-            specify global resources (``{'GPU': 2}``), or per-task resources
-            (``{'x': {'GPU': 1}, 'y': {'SSD': 4}}``), but not both.
+            Defines the `resources` each instance of this mapped task requires
+            on the worker; e.g. ``{'GPU': 2}``.
             See :doc:`worker resources <resources>` for details on defining
             resources.
         actors: bool or dict (default None)
@@ -2985,18 +2942,11 @@ def persist(
 
         names = {k for c in collections for k in flatten(c.__dask_keys__())}
 
-        restrictions, loose_restrictions = self.get_restrictions(
-            collections, workers, allow_other_workers
-        )
-
-        if not isinstance(priority, Number):
-            priority = {k: p for c, p in priority.items() for k in self._expand_key(c)}
-
         futures = self._graph_to_futures(
             dsk,
             names,
-            restrictions,
-            loose_restrictions,
+            workers=workers,
+            allow_other_workers=allow_other_workers,
             resources=resources,
             retries=retries,
             user_priority=priority,
@@ -3881,90 +3831,6 @@ def _expand_key(cls, k):
             else:
                 yield stringify(kk)
 
-    @classmethod
-    def _expand_retries(cls, retries, all_keys):
-        """
-        Expand the user-provided "retries" specification
-        to a {task key: Integral} dictionary.
-        """
-        if retries and isinstance(retries, dict):
-            result = {
-                name: value
-                for key, value in retries.items()
-                for name in cls._expand_key(key)
-            }
-        elif isinstance(retries, Integral):
-            # Each task unit may potentially fail, allow retrying all of them
-            result = {name: retries for name in all_keys}
-        else:
-            raise TypeError(
-                "`retries` should be an integer or dict, got %r" % (type(retries))
-            )
-        return keymap(stringify, result)
-
-    def _expand_resources(cls, resources, all_keys):
-        """
-        Expand the user-provided "resources" specification
-        to a {task key: {resource name: Number}} dictionary.
-        """
-        # Resources can either be a single dict such as {'GPU': 2},
-        # indicating a requirement for all keys, or a nested dict
-        # such as {'x': {'GPU': 1}, 'y': {'SSD': 4}} indicating
-        # per-key requirements
-        if not isinstance(resources, dict):
-            raise TypeError("`resources` should be a dict, got %r" % (type(resources)))
-
-        per_key_reqs = {}
-        global_reqs = {}
-        all_keys = list(all_keys)
-        for k, v in resources.items():
-            if isinstance(v, dict):
-                # It's a per-key requirement
-                per_key_reqs.update((kk, v) for kk in cls._expand_key(k))
-            else:
-                # It's a global requirement
-                global_reqs.update((kk, {k: v}) for kk in all_keys)
-
-        if global_reqs and per_key_reqs:
-            raise ValueError(
-                "cannot have both per-key and all-key requirements "
-                "in resources dict %r" % (resources,)
-            )
-        return global_reqs or per_key_reqs
-
-    @classmethod
-    def get_restrictions(cls, collections, workers, allow_other_workers):
-        """ Get restrictions from inputs to compute/persist """
-        if isinstance(workers, (str, tuple, list)):
-            workers = {tuple(collections): workers}
-        if isinstance(workers, dict):
-            restrictions = {}
-            for colls, ws in workers.items():
-                if isinstance(ws, str):
-                    ws = [ws]
-                if dask.is_dask_collection(colls):
-                    keys = flatten(colls.__dask_keys__())
-                elif isinstance(colls, str):
-                    keys = [colls]
-                else:
-                    keys = list(
-                        {k for c in flatten(colls) for k in flatten(c.__dask_keys__())}
-                    )
-                restrictions.update({k: ws for k in keys})
-        else:
-            restrictions = {}
-
-        if allow_other_workers is True:
-            loose_restrictions = list(restrictions)
-        elif allow_other_workers:
-            loose_restrictions = list(
-                {k for c in flatten(allow_other_workers) for k in c.__dask_keys__()}
-            )
-        else:
-            loose_restrictions = []
-
-        return restrictions, loose_restrictions
-
     @staticmethod
     def collections_to_dsk(collections, *args, **kwargs):
         return collections_to_dsk(collections, *args, **kwargs)
diff --git a/distributed/protocol/highlevelgraph.py b/distributed/protocol/highlevelgraph.py
index 7895c465c95..a1419fb4f5a 100644
--- a/distributed/protocol/highlevelgraph.py
+++ b/distributed/protocol/highlevelgraph.py
@@ -124,18 +124,7 @@ def highlevelgraph_pack(hlg: HighLevelGraph, client, client_keys):
     return dumps_msgpack({"layers": layers})
 
 
-def _materialized_layer_unpack(state, dsk, dependencies, annotations):
-    dsk.update(state["dsk"])
-    for k, v in state["dependencies"].items():
-        dependencies[k] = list(set(dependencies.get(k, ())) | set(v))
-
-    if state["annotations"]:
-        annotations.update(
-            Layer.expand_annotations(state["annotations"], state["dsk"].keys())
-        )
-
-
-def highlevelgraph_unpack(dumped_hlg):
+def highlevelgraph_unpack(dumped_hlg, annotations: dict):
     """Unpack the high level graph for Scheduler -> Worker communication
 
     The approach is to delegate the packaging to each layer in the high
@@ -148,6 +137,11 @@ def highlevelgraph_unpack(dumped_hlg):
     dumped_hlg: list of header and payload
         Packed high level graph serialized by dumps_msgpack
 
+    annotations: dict
+        A top-level annotations object which may be partially populated,
+        and which may be further filled by annotations from the layers
+        of the dumped_hlg.
+
     Returns
     -------
     dsk: dict
@@ -157,18 +151,21 @@ def highlevelgraph_unpack(dumped_hlg):
     annotations: dict
         Annotations for `dsk`
     """
-
     hlg = loads_msgpack(*dumped_hlg)
 
     dsk = {}
     deps = {}
-    annotations = {}
+    out_annotations = {}
     for layer in hlg["layers"]:
+        if annotations:
+            if layer["state"]["annotations"] is None:
+                layer["state"]["annotations"] = {}
+            layer["state"]["annotations"].update(annotations)
         if layer["__module__"] is None:  # Default implementation
-            unpack_func = _materialized_layer_unpack
+            unpack_func = Layer.__dask_distributed_unpack__
         else:
             mod = import_allowed_module(layer["__module__"])
             unpack_func = getattr(mod, layer["__name__"]).__dask_distributed_unpack__
-        unpack_func(layer["state"], dsk, deps, annotations)
+        unpack_func(layer["state"], dsk, deps, out_annotations)
 
-    return dsk, deps, annotations
+    return dsk, deps, out_annotations
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index f2126a80e41..bbd8f01aeda 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -3612,9 +3612,10 @@ def update_graph_hlg(
         user_priority=0,
         actors=None,
         fifo_timeout=0,
+        annotations=None,
     ):
 
-        dsk, dependencies, annotations = highlevelgraph_unpack(hlg)
+        dsk, dependencies, annotations = highlevelgraph_unpack(hlg, annotations)
 
         # Remove any self-dependencies (happens on test_publish_bag() and others)
         for k, v in dependencies.items():
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 6fd59e49d72..b4e23ed7ea0 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -250,21 +250,31 @@ async def test_compute_retries(c, s, a, b):
     x = c.compute(delayed(varying(args))(), retries=2)
     assert await x == 3
 
+
+@gen_cluster(client=True)
+async def test_compute_retries_annotations(c, s, a, b):
     # Per-future retries
     xargs = [ZeroDivisionError("one"), ZeroDivisionError("two"), 30, 40]
     yargs = [ZeroDivisionError("five"), ZeroDivisionError("six"), 70]
     zargs = [80, 90, 100]
 
-    x, y = [delayed(varying(args))() for args in (xargs, yargs)]
-    x, y = c.compute([x, y], retries={x: 2})
+    with dask.annotate(retries=2):
+        x = delayed(varying(xargs))()
+    y = delayed(varying(yargs))()
+
+    x, y = c.compute([x, y], optimize_graph=False)
     gc.collect()
 
     assert await x == 30
     with pytest.raises(ZeroDivisionError, match="five"):
         await y
 
-    x, y, z = [delayed(varying(args))() for args in (xargs, yargs, zargs)]
-    x, y, z = c.compute([x, y, z], retries={(y, z): 2})
+    x = delayed(varying(xargs))()
+    with dask.annotate(retries=2):
+        y = delayed(varying(yargs))()
+        z = delayed(varying(zargs))()
+
+    x, y, z = c.compute([x, y, z], optimize_graph=False)
 
     with pytest.raises(ZeroDivisionError, match="one"):
         await x
@@ -322,13 +332,20 @@ async def test_persist_retries(c, s, a, b):
     x = c.compute(x)
     assert await x == 3
 
+
+@gen_cluster(client=True)
+async def test_persist_retries_annotations(c, s, a, b):
     # Per-key retries
     xargs = [ZeroDivisionError("one"), ZeroDivisionError("two"), 30, 40]
     yargs = [ZeroDivisionError("five"), ZeroDivisionError("six"), 70]
     zargs = [80, 90, 100]
 
-    x, y, z = [delayed(varying(args))() for args in (xargs, yargs, zargs)]
-    x, y, z = c.persist([x, y, z], retries={(y, z): 2})
+    x = delayed(varying(xargs))()
+    with dask.annotate(retries=2):
+        y = delayed(varying(yargs))()
+        z = delayed(varying(zargs))()
+
+    x, y, z = c.persist([x, y, z], optimize_graph=False)
     x, y, z = c.compute([x, y, z])
 
     with pytest.raises(ZeroDivisionError, match="one"):
@@ -910,31 +927,6 @@ async def test_restrictions_ip_port(c, s, a, b):
     assert y.key in b.data
 
 
-@gen_cluster(client=True)
-async def test_restrictions_ip_port_task_key(c, s, a, b):
-    # Create a long dependency list
-    tasks = [delayed(inc)(1)]
-    for _ in range(100):
-        tasks.append(delayed(add)(tasks[-1], random.choice(tasks)))
-
-    last_task = tasks[-1]
-
-    # calculate all dependency keys
-    all_tasks = list(last_task.__dask_graph__())
-    # only restrict to a single worker
-    workers = {d: a.address for d in all_tasks}
-    result = c.compute(last_task, workers=workers)
-    await result
-
-    # all tasks should have been calculated by the first worker
-    for task in tasks:
-        assert s.worker_restrictions[task.key] == {a.address}
-
-    # and the data should also be there
-    assert last_task.key in a.data
-    assert last_task.key not in b.data
-
-
 @pytest.mark.skipif(
     not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
 )
@@ -948,16 +940,6 @@ async def test_restrictions_map(c, s, a, b):
     for x in L:
         assert s.host_restrictions[x.key] == {a.ip}
 
-    L = c.map(inc, [10, 11, 12], workers=[{a.ip}, {a.ip, b.ip}, {b.ip}])
-    await wait(L)
-
-    assert s.host_restrictions[L[0].key] == {a.ip}
-    assert s.host_restrictions[L[1].key] == {a.ip, b.ip}
-    assert s.host_restrictions[L[2].key] == {b.ip}
-
-    with pytest.raises(ValueError):
-        c.map(inc, [10, 11, 12], workers=[{a.ip}])
-
 
 @pytest.mark.skipif(
     not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
@@ -965,13 +947,28 @@ async def test_restrictions_map(c, s, a, b):
 @gen_cluster([("127.0.0.1", 1), ("127.0.0.2", 2)], client=True)
 async def test_restrictions_get(c, s, a, b):
     dsk = {"x": 1, "y": (inc, "x"), "z": (inc, "y")}
-    restrictions = {"y": {a.ip}, "z": {b.ip}}
 
-    futures = c.get(dsk, ["y", "z"], restrictions, sync=False)
+    futures = c.get(dsk, ["y", "z"], workers=a.ip, sync=False)
     result = await c.gather(futures)
     assert result == [2, 3]
     assert "y" in a.data
-    assert "z" in b.data
+    assert "z" in a.data
+    assert len(b.data) == 0
+
+
+@gen_cluster(client=True)
+async def test_restrictions_get_annotate(c, s, a, b):
+    x = 1
+    with dask.annotate(workers=a.address):
+        y = delayed(inc)(x)
+    with dask.annotate(workers=b.address):
+        z = delayed(inc)(y)
+
+    futures = c.get(z.__dask_graph__(), [y.key, z.key], sync=False)
+    result = await c.gather(futures)
+    assert result == [2, 3]
+    assert y.key in a.data
+    assert z.key in b.data
 
 
 @gen_cluster(client=True)
@@ -4100,6 +4097,29 @@ async def run2():
     await asyncio.gather(run1(), run2())
 
 
+@nodebug  # test timing is fragile
+@gen_cluster(nthreads=[("127.0.0.1", 1)] * 3, client=True)
+async def test_persist_workers_annotate(e, s, a, b, c):
+    with dask.annotate(workers=a.address, allow_other_workers=False):
+        L1 = [delayed(inc)(i) for i in range(4)]
+    with dask.annotate(workers=b.address, allow_other_workers=False):
+        total = delayed(sum)(L1)
+    with dask.annotate(workers=c.address, allow_other_workers=True):
+        L2 = [delayed(add)(i, total) for i in L1]
+    with dask.annotate(workers=b.address, allow_other_workers=True):
+        total2 = delayed(sum)(L2)
+
+    # TODO: once annotations are faithfully forwarded upon graph optimization,
+    # we shouldn't need to disable that here.
+    out = e.persist(L1 + L2 + [total, total2], optimize_graph=False)
+
+    await wait(out)
+    assert all(v.key in a.data for v in L1)
+    assert total.key in b.data
+
+    assert s.loose_restrictions == {total2.key} | {v.key for v in L2}
+
+
 @nodebug  # test timing is fragile
 @gen_cluster(nthreads=[("127.0.0.1", 1)] * 3, client=True)
 async def test_persist_workers(e, s, a, b, c):
@@ -4110,20 +4130,40 @@ async def test_persist_workers(e, s, a, b, c):
 
     out = e.persist(
         L1 + L2 + [total, total2],
-        workers={
-            tuple(L1): a.address,
-            total: b.address,
-            tuple(L2): [c.address],
-            total2: b.address,
-        },
-        allow_other_workers=L2 + [total2],
+        workers=[a.address, b.address],
+        allow_other_workers=True,
     )
 
     await wait(out)
-    assert all(v.key in a.data for v in L1)
-    assert total.key in b.data
 
-    assert s.loose_restrictions == {total2.key} | {v.key for v in L2}
+    for v in L1 + L2 + [total, total2]:
+        assert s.worker_restrictions[v.key] == {a.address, b.address}
+    assert not any(c.address in r for r in s.worker_restrictions)
+
+    assert s.loose_restrictions == {total.key, total2.key} | {v.key for v in L1 + L2}
+
+
+@gen_cluster(nthreads=[("127.0.0.1", 1)] * 3, client=True)
+async def test_compute_workers_annotate(e, s, a, b, c):
+    with dask.annotate(workers=a.address, allow_other_workers=True):
+        L1 = [delayed(inc)(i) for i in range(4)]
+    with dask.annotate(workers=b.address, allow_other_workers=True):
+        total = delayed(sum)(L1)
+    with dask.annotate(workers=[c.address]):
+        L2 = [delayed(add)(i, total) for i in L1]
+
+    # TODO: once annotations are faithfully forwarded upon graph optimization,
+    # we shouldn't need to disable that here.
+    out = e.compute(L1 + L2 + [total], optimize_graph=False)
+
+    await wait(out)
+    for v in L1:
+        assert s.worker_restrictions[v.key] == {a.address}
+    for v in L2:
+        assert s.worker_restrictions[v.key] == {c.address}
+    assert s.worker_restrictions[total.key] == {b.address}
+
+    assert s.loose_restrictions == {total.key} | {v.key for v in L1}
 
 
 @gen_cluster(nthreads=[("127.0.0.1", 1)] * 3, client=True)
@@ -4134,18 +4174,16 @@ async def test_compute_workers(e, s, a, b, c):
 
     out = e.compute(
         L1 + L2 + [total],
-        workers={tuple(L1): a.address, total: b.address, tuple(L2): [c.address]},
-        allow_other_workers=L1 + [total],
+        workers=[a.address, b.address],
+        allow_other_workers=True,
     )
 
     await wait(out)
-    for v in L1:
-        assert s.worker_restrictions[v.key] == {a.address}
-    for v in L2:
-        assert s.worker_restrictions[v.key] == {c.address}
-    assert s.worker_restrictions[total.key] == {b.address}
+    for v in L1 + L2 + [total]:
+        assert s.worker_restrictions[v.key] == {a.address, b.address}
+    assert not any(c.address in r for r in s.worker_restrictions)
 
-    assert s.loose_restrictions == {total.key} | {v.key for v in L1}
+    assert s.loose_restrictions == {total.key} | {v.key for v in L1 + L2}
 
 
 @gen_cluster(client=True)
@@ -4162,28 +4200,6 @@ async def test_compute_nested_containers(c, s, a, b):
     assert result["y"] == 123
 
 
-def test_get_restrictions():
-    L1 = [delayed(inc)(i) for i in range(4)]
-    total = delayed(sum)(L1)
-    L2 = [delayed(add)(i, total) for i in L1]
-
-    r1, loose = Client.get_restrictions(L2, "127.0.0.1", False)
-    assert r1 == {d.key: ["127.0.0.1"] for d in L2}
-    assert not loose
-
-    r1, loose = Client.get_restrictions(L2, ["127.0.0.1"], True)
-    assert r1 == {d.key: ["127.0.0.1"] for d in L2}
-    assert set(loose) == {d.key for d in L2}
-
-    r1, loose = Client.get_restrictions(L2, {total: "127.0.0.1"}, True)
-    assert r1 == {total.key: ["127.0.0.1"]}
-    assert loose == [total.key]
-
-    r1, loose = Client.get_restrictions(L2, {(total,): "127.0.0.1"}, True)
-    assert r1 == {total.key: ["127.0.0.1"]}
-    assert loose == [total.key]
-
-
 @gen_cluster(client=True)
 async def test_scatter_type(c, s, a, b):
     [future] = await c.scatter([1])
@@ -6318,6 +6334,43 @@ async def test_annotations_task_state(c, s, a, b):
     )
 
 
+@pytest.mark.parametrize("fn", ["compute", "persist"])
+def test_annotations_compute_time(fn):
+    da = pytest.importorskip("dask.array")
+
+    @gen_cluster(client=True)
+    async def test(c, s, a, b):
+        x = da.ones(10, chunks=(5,))
+
+        with dask.annotate(foo="bar"):
+            # Turn off optimization to avoid rewriting layers and picking up annotations
+            # that way. Instead, we want `compute`/`persist` to be able to pick them up.
+            x = await getattr(c, fn)(x, optimize_graph=False)
+
+        assert all({"foo": "bar"} == ts.annotations for ts in s.tasks.values())
+
+    test()
+
+
+@pytest.mark.xfail(reason="https://github.com/dask/dask/issues/7036")
+@gen_cluster(client=True)
+async def test_annotations_survive_optimization(c, s, a, b):
+    da = pytest.importorskip("dask.array")
+
+    with dask.annotate(foo="bar"):
+        x = da.ones(10, chunks=(5,))
+
+    ann = x.__dask_graph__().layers[x.name].annotations
+    assert ann is not None
+    assert ann.get("foo", None) == "bar"
+
+    (xx,) = dask.optimize(x)
+
+    ann = xx.__dask_graph__().layers[x.name].annotations
+    assert ann is not None
+    assert ann.get("foo", None) == "bar"
+
+
 @gen_cluster(client=True)
 async def test_annotations_priorities(c, s, a, b):
     da = pytest.importorskip("dask.array")
@@ -6363,6 +6416,37 @@ async def test_annotations_retries(c, s, a, b):
     assert all(ts.annotations == {"retries": 2} for ts in s.tasks.values())
 
 
+@gen_cluster(client=True)
+async def test_annotations_blockwise_unpack(c, s, a, b):
+    da = pytest.importorskip("dask.array")
+    from dask.array.utils import assert_eq
+    import numpy as np
+
+    # A flaky doubling function -- need extra args because it is called before
+    # application to establish dtype/meta.
+    scale = varying([ZeroDivisionError("one"), ZeroDivisionError("two"), 2, 2])
+
+    def flaky_double(x):
+        return scale() * x
+
+    # A reliable double function.
+    def reliable_double(x):
+        return 2 * x
+
+    x = da.ones(10, chunks=(5,))
+
+    # The later annotations should not override the earlier annotations
+    with dask.annotate(retries=2):
+        y = x.map_blocks(flaky_double, meta=np.array((), dtype=np.float))
+    with dask.annotate(retries=0):
+        z = y.map_blocks(reliable_double, meta=np.array((), dtype=np.float))
+
+    with dask.config.set(optimization__fuse__active=False):
+        z = await c.compute(z)
+
+    assert_eq(z, np.ones(10) * 4.0)
+
+
 @gen_cluster(
     client=True,
     nthreads=[
@@ -6402,3 +6486,12 @@ async def test_annotations_loose_restrictions(c, s, a, b):
             for ts in s.tasks.values()
         ]
     )
+
+
+@gen_cluster(client=True)
+async def test_workers_collection_restriction(c, s, a, b):
+    da = pytest.importorskip("dask.array")
+
+    future = c.compute(da.arange(10), workers=a.address)
+    await future
+    assert a.data and not b.data
diff --git a/distributed/tests/test_priorities.py b/distributed/tests/test_priorities.py
index b3cd6ace2d9..d3ba8a1a026 100644
--- a/distributed/tests/test_priorities.py
+++ b/distributed/tests/test_priorities.py
@@ -67,23 +67,27 @@ async def test_persist(c, s):
 
 
 @gen_cluster(client=True)
-async def test_expand_compute(c, s, a, b):
-    low = delayed(inc)(1)
+async def test_annotate_compute(c, s, a, b):
+    with dask.annotate(priority=-1):
+        low = delayed(inc)(1)
+    with dask.annotate(priority=1):
+        high = delayed(inc)(2)
     many = [delayed(slowinc)(i, delay=0.1) for i in range(10)]
-    high = delayed(inc)(2)
 
-    low, many, high = c.compute([low, many, high], priority={low: -1, high: 1})
+    low, many, high = c.compute([low, many, high], optimize_graph=False)
     await wait(high)
     assert s.tasks[low.key].state == "processing"
 
 
 @gen_cluster(client=True)
-async def test_expand_persist(c, s, a, b):
-    low = delayed(inc)(1, dask_key_name="low")
+async def test_annotate_persist(c, s, a, b):
+    with dask.annotate(priority=-1):
+        low = delayed(inc)(1, dask_key_name="low")
+    with dask.annotate(priority=1):
+        high = delayed(inc)(2, dask_key_name="high")
     many = [delayed(slowinc)(i, delay=0.1) for i in range(4)]
-    high = delayed(inc)(2, dask_key_name="high")
 
-    low, high, x, y, z, w = persist(low, high, *many, priority={low: -1, high: 1})
+    low, high, x, y, z, w = persist(low, high, *many, optimize_graph=False)
     await wait(high)
     assert s.tasks[low.key].state == "processing"
 
diff --git a/distributed/tests/test_resources.py b/distributed/tests/test_resources.py
index e055a4b362f..a7332c4a92e 100644
--- a/distributed/tests/test_resources.py
+++ b/distributed/tests/test_resources.py
@@ -1,6 +1,7 @@
 import asyncio
 from time import time
 
+import dask
 from dask import delayed
 from dask.utils import stringify
 import pytest
@@ -131,10 +132,12 @@ async def test_map(c, s, a, b):
     ],
 )
 async def test_persist(c, s, a, b):
-    x = delayed(inc)(1)
-    y = delayed(inc)(x)
+    with dask.annotate(resources={"A": 1}):
+        x = delayed(inc)(1)
+    with dask.annotate(resources={"B": 1}):
+        y = delayed(inc)(x)
 
-    xx, yy = c.persist([x, y], resources={x: {"A": 1}, y: {"B": 1}})
+    xx, yy = c.persist([x, y], optimize_graph=False)
 
     await wait([xx, yy])
 
@@ -150,10 +153,12 @@ async def test_persist(c, s, a, b):
     ],
 )
 async def test_compute(c, s, a, b):
-    x = delayed(inc)(1)
-    y = delayed(inc)(x)
+    with dask.annotate(resources={"A": 1}):
+        x = delayed(inc)(1)
+    with dask.annotate(resources={"B": 1}):
+        y = delayed(inc)(x)
 
-    yy = c.compute(y, resources={x: {"A": 1}, y: {"B": 1}})
+    yy = c.compute(y, optimize_graph=False)
     await wait(yy)
 
     assert b.data
@@ -175,8 +180,10 @@ async def test_compute(c, s, a, b):
 async def test_get(c, s, a, b):
     dsk = {"x": (inc, 1), "y": (inc, "x")}
 
-    result = await c.get(dsk, "y", resources={"y": {"A": 1}}, sync=False)
+    result = await c.get(dsk, "y", resources={"A": 1}, sync=False)
     assert result == 3
+    assert "y" in a.data
+    assert not b.data
 
 
 @gen_cluster(
@@ -186,11 +193,12 @@ async def test_get(c, s, a, b):
         ("127.0.0.1", 1, {"resources": {"B": 1}}),
     ],
 )
-async def test_persist_tuple(c, s, a, b):
-    x = delayed(inc)(1)
-    y = delayed(inc)(x)
+async def test_persist_multiple_collections(c, s, a, b):
+    with dask.annotate(resources={"A": 1}):
+        x = delayed(inc)(1)
+        y = delayed(inc)(x)
 
-    xx, yy = c.persist([x, y], resources={(x, y): {"A": 1}})
+    xx, yy = c.persist([x, y], optimize_graph=False)
 
     await wait([xx, yy])
 
@@ -308,11 +316,12 @@ async def test_set_resources(c, s, a):
 async def test_persist_collections(c, s, a, b):
     da = pytest.importorskip("dask.array")
     x = da.arange(10, chunks=(5,))
-    y = x.map_blocks(lambda x: x + 1)
+    with dask.annotate(resources={"A": 1}):
+        y = x.map_blocks(lambda x: x + 1)
     z = y.map_blocks(lambda x: 2 * x)
     w = z.sum()
 
-    ww, yy = c.persist([w, y], resources={tuple(y.__dask_keys__()): {"A": 1}})
+    ww, yy = c.persist([w, y], optimize_graph=False)
 
     await wait([ww, yy])
 
@@ -360,6 +369,7 @@ async def test_full_collections(c, s, a, b):
     assert not b.log
 
 
+@pytest.mark.xfail(reason="atop fusion seemed to break this")
 @pytest.mark.parametrize(
     "optimize_graph",
     [
@@ -382,9 +392,10 @@ async def f(dask_worker):
 
     client.run(f, workers=[a["address"]])
 
-    x = da.random.random(100, chunks=(10,)) + 1
+    with dask.annotate(resources={"A": 1}):
+        x = da.random.random(100, chunks=(10,)) + 1
 
-    x.compute(resources={tuple(x.dask): {"A": 1}}, optimize_graph=optimize_graph)
+    x.compute(optimize_graph=optimize_graph)
 
     def g(dask_worker):
         return len(dask_worker.log)
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index f6e0be29bbf..49f488d9673 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -1039,11 +1039,12 @@ async def test_no_workers_to_memory(c, s):
 
 @gen_cluster(client=True)
 async def test_no_worker_to_memory_restrictions(c, s, a, b):
-    x = delayed(slowinc)(1, delay=0.4)
-    y = delayed(slowinc)(x, delay=0.4)
-    z = delayed(slowinc)(y, delay=0.4)
+    with dask.annotate(workers="alice"):
+        x = delayed(slowinc)(1, delay=0.4)
+        y = delayed(slowinc)(x, delay=0.4)
+        z = delayed(slowinc)(y, delay=0.4)
 
-    yy, zz = c.persist([y, z], workers={(x, y, z): "alice"})
+    yy, zz = c.persist([y, z], optimize_graph=False)
 
     while not s.tasks:
         await asyncio.sleep(0.01)
diff --git a/docs/source/priority.rst b/docs/source/priority.rst
index 3b47dc4e9da..8b4729cbb27 100644
--- a/docs/source/priority.rst
+++ b/docs/source/priority.rst
@@ -8,7 +8,7 @@ their needs.
 
 Dask uses the following priorities, in order:
 
-1.  **User priorities**: A user defined priority, provided by the ``priority=`` keyword argument
+1.  **User priorities**: A user defined priority is provided by the ``priority=`` keyword argument
     to functions like ``compute()``, ``persist()``, ``submit()``, or ``map()``.
     Tasks with higher priorities run before tasks with lower priorities with
     the default priority being zero.
@@ -20,6 +20,19 @@ Dask uses the following priorities, in order:
 
        df = df.persist(priority=10)  # high priority computation
 
+    Priorities can also be specified using the dask annotations machinery:
+
+    .. code-block:: python
+
+       with dask.annotate(priority=10):
+           future = client.submit(func, *args)  # high priority task
+       with dask.annotate(priority=-10):
+           future = client.submit(func, *args)  # low priority task
+
+       with dask.annotate(priority=10):
+           df = df.persist()  # high priority computation
+
+
 2.  **First in first out chronologically**: Dask prefers computations that were
     submitted early.  Because users can submit computations asynchronously it
     may be that several different computations are running on the workers at
@@ -60,4 +73,4 @@ Dask uses the following priorities, in order:
 
 If multiple tasks each have exactly the same priorities outlined above, then
 the order in which tasks arrive at a worker, in a last in first out manner,
-is used to determine the order in which tasks run.
\ No newline at end of file
+is used to determine the order in which tasks run.
diff --git a/docs/source/resources.rst b/docs/source/resources.rst
index f0c7c0ead60..1e66e081144 100644
--- a/docs/source/resources.rst
+++ b/docs/source/resources.rst
@@ -47,6 +47,15 @@ When we submit tasks to the cluster we specify constraints per task
    processed = [client.submit(process, d, resources={'GPU': 1}) for d in data]
    final = client.submit(aggregate, processed, resources={'MEMORY': 70e9})
 
+Equivalently, we can specify resource constraints using the dask annotations machinery:
+
+.. code-block:: python
+
+   with dask.annotate(resources={'GPU': 1}):
+       processed = [client.submit(process, d) for d in data]
+   with dask.annotate(resources={'MEMORY': 70e9}):
+       final = client.submit(aggregate, processed)
+
 Specifying Resources
 --------------------
 
@@ -135,24 +144,18 @@ Resources with collections
 --------------------------
 
 You can also use resources with Dask collections, like arrays, dataframes, and
-delayed objects.  You can pass a dictionary mapping keys of the collection to
-resource requirements during compute or persist calls.
+delayed objects. You can annotate operations on collections with specific resources
+that should be required perform the computation using the dask annotations machinery.
 
 .. code-block:: python
 
-    from dask import core
-    
     x = dd.read_csv(...)
-    y = x.map_partitions(func1)
+    with dask.annotate(resources={'GPU': 1}):
+        y = x.map_partitions(func1)
     z = y.map_partitions(func2)
 
-    z.compute(resources={tuple(core.flatten(y.__dask_keys__())): {'GPU': 1}})
-
-In some cases (such as the case above) the keys for ``y`` may be optimized away
-before execution.  You can avoid that either by requiring them as an explicit
-output, or by passing the ``optimize_graph=False`` keyword.
-
-
-.. code-block:: python
+    z.compute(optimize_graph=False)
 
-    z.compute(resources={tuple(core.flatten(y.__dask_keys__())): {'GPU': 1}}, optimize_graph=False)
+In most cases (such as the case above) the annotations for ``y`` may be lost during
+graph optimization before execution. You can avoid that by passing the
+``optimize_graph=False`` keyword.

From 08ea96890674d48b90f4e1f92959957e5e362a18 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Fri, 29 Jan 2021 11:24:51 -0600
Subject: [PATCH 1138/1550] Update pytest.skip usage in test_server_listen
 (#4467)

---
 distributed/tests/test_core.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/tests/test_core.py b/distributed/tests/test_core.py
index 8b702b4c8ff..e9b77c0b1a3 100644
--- a/distributed/tests/test_core.py
+++ b/distributed/tests/test_core.py
@@ -196,7 +196,7 @@ async def test_server_listen():
         if has_ipv6():
             EXTERNAL_IP6 = get_ipv6()
     except socket.gaierror:
-        raise pytest.skip(reason="no network access")
+        raise pytest.skip("no network access")
 
     from contextlib import asynccontextmanager
 

From 24e61eaa5825bc7855f011aa8ef8a32572d19693 Mon Sep 17 00:00:00 2001
From: Matthias Bussonnier <bussonniermatthias@gmail.com>
Date: Mon, 1 Feb 2021 11:12:57 -0800
Subject: [PATCH 1139/1550] DOC: Proper numpydoc syntax for
 distributed/protocol/*.py (#4473)

Numpydoc return semantically different informations when there is no
space in front of :, in particular it will put the whole line as the
name of the parameter.

This might not be visible in the sphinx theme depending on the theme
chosen.
---
 distributed/protocol/compression.py    |  6 +++---
 distributed/protocol/highlevelgraph.py | 11 +++++------
 distributed/protocol/serialize.py      | 16 ++++++++--------
 3 files changed, 16 insertions(+), 17 deletions(-)

diff --git a/distributed/protocol/compression.py b/distributed/protocol/compression.py
index 82cfbeec3c2..b067ae0e526 100644
--- a/distributed/protocol/compression.py
+++ b/distributed/protocol/compression.py
@@ -147,10 +147,10 @@ def byte_sample(b, size, n):
 
     Parameters
     ----------
-    b: bytes or memoryview
-    size: int
+    b : bytes or memoryview
+    size : int
         size of each sample to collect
-    n: int
+    n : int
         number of samples to collect
     """
     starts = [random.randint(0, len(b) - size) for j in range(n)]
diff --git a/distributed/protocol/highlevelgraph.py b/distributed/protocol/highlevelgraph.py
index a1419fb4f5a..16eb4f9ff2e 100644
--- a/distributed/protocol/highlevelgraph.py
+++ b/distributed/protocol/highlevelgraph.py
@@ -79,11 +79,11 @@ def highlevelgraph_pack(hlg: HighLevelGraph, client, client_keys):
 
     Parameters
     ----------
-    hlg: HighLevelGraph
+    hlg : HighLevelGraph
         The high level graph to pack
-    client: distributed.Client
+    client : distributed.Client
         The client calling this function.
-    client_keys: Iterable
+    client_keys : Iterable
         List of keys requested by the client.
 
     Returns
@@ -134,10 +134,9 @@ def highlevelgraph_unpack(dumped_hlg, annotations: dict):
 
     Parameters
     ----------
-    dumped_hlg: list of header and payload
+    dumped_hlg : list of header and payload
         Packed high level graph serialized by dumps_msgpack
-
-    annotations: dict
+    annotations : dict
         A top-level annotations object which may be partially populated,
         and which may be further filled by annotations from the layers
         of the dumped_hlg.
diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index 1bea7ca496a..b99e7692e43 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -213,9 +213,9 @@ def serialize(x, serializers=None, on_error="message", context=None):
 
     See Also
     --------
-    deserialize: Convert header and frames back to object
-    to_serialize: Mark that data in a message should be serialized
-    register_serialization: Register custom serialization functions
+    deserialize : Convert header and frames back to object
+    to_serialize : Mark that data in a message should be serialized
+    register_serialization : Register custom serialization functions
     """
     if serializers is None:
         serializers = ("dask", "pickle")  # TODO: get from configuration
@@ -322,8 +322,8 @@ def deserialize(header, frames, deserializers=None):
 
     Parameters
     ----------
-    header: dict
-    frames: list of bytes
+    header : dict
+    frames : list of bytes
     deserializers : Optional[Dict[str, Tuple[Callable, Callable, bool]]]
         An optional dict mapping a name to a (de)serializer.
         See `dask_serialize` and `dask_deserialize` for more.
@@ -580,9 +580,9 @@ def register_serialization(cls, serialize, deserialize):
 
     Parameters
     ----------
-    cls: type
-    serialize: callable(cls) -> Tuple[Dict, List[bytes]]
-    deserialize: callable(header: Dict, frames: List[bytes]) -> cls
+    cls : type
+    serialize : callable(cls) -> Tuple[Dict, List[bytes]]
+    deserialize : callable(header: Dict, frames: List[bytes]) -> cls
 
     Examples
     --------

From 0e996b2bf1e2f5272077ea4297005b6f6c1208ba Mon Sep 17 00:00:00 2001
From: Matthias Bussonnier <bussonniermatthias@gmail.com>
Date: Mon, 1 Feb 2021 14:48:02 -0800
Subject: [PATCH 1140/1550] DOC: typo, directives ends with 2 colons `::`.
 (#4472)

---
 distributed/scheduler.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index bbd8f01aeda..4ea7ce98ef1 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1089,19 +1089,19 @@ class TaskState:
        into the "processing" state and be sent for execution to another
        connected worker.
 
-    .. attribute: metadata: dict
+    .. attribute:: metadata: dict
 
        Metadata related to task.
 
-    .. attribute: actor: bool
+    .. attribute:: actor: bool
 
        Whether or not this task is an Actor.
 
-    .. attribute: group: TaskGroup
+    .. attribute:: group: TaskGroup
 
         The group of tasks to which this one belongs.
 
-    .. attribute: annotations: dict
+    .. attribute:: annotations: dict
 
         Task annotations
     """

From 5eaba1a92b92c3d2b12396b2f0c2e920b8dd0013 Mon Sep 17 00:00:00 2001
From: Ray Bell <rayjohnbell0@gmail.com>
Date: Mon, 1 Feb 2021 23:03:37 -0500
Subject: [PATCH 1141/1550] Create PULL_REQUEST_TEMPLATE.md (#4476)

---
 .github/PULL_REQUEST_TEMPLATE.md | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 .github/PULL_REQUEST_TEMPLATE.md

diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
new file mode 100644
index 00000000000..c75019cffff
--- /dev/null
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -0,0 +1,3 @@
+- [ ] Closes #xxxx
+- [ ] Tests added / passed
+- [ ] Passes `black distributed` / `flake8 distributed`

From 98570fbcd662bc0fe468ccba9d57d582d2a3c678 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Tue, 2 Feb 2021 07:56:52 -0800
Subject: [PATCH 1142/1550] Optimize transitions (#4451)

* Annotate `keys` & `new`

* Combine `recommendations` annotation with others

* Annotate `dependents` & `dependencies`

* Annotate `start` & `finish`

* Create empty `dict` for `recommendations` once

Instead of letting Cython generate empty `dict`s for each of these
cases, just create an empty `dict` once and assign it to
`recommendations`. That way we can just `return` it simply and avoid the
C boilerplate that would otherwise be needed.

* Use `.get(...)` to retrieve `TaskState`

* Assign `start, finish` to a variable

* Just use `.get(...)` to retrieve transition func

Avoids checking for the presence of the key and then retrieving the
function corresponding the key by simply trying to get the function in
the first place or `None` if it is absent. As it is pretty quick to
check if something is `None` both in Python and Cython, this should
speed up the check and function retrieval time.

* Annotate `a` & `b`

* Use `.get(...)` to get `key` from `a`

Avoids looking up `key` twice. Once to see if it is there and a second
time to grab it. This way we just grab the value corresponding to `key`
or `None` if it is missing. The following `None` check is quite fast in
both Python and Cython.

* Just `update` `recommendations` with `a` & `b`

* Drop unneeded `KeyError` handling

Neither of these statements should raise a `KeyError`. So just drop this
`try...except...`.

* Annotate `finish2`

* Replace generator with simple `for`-loop

This avoids building a `list`, which makes it easier for Cython to
optimize.

* Bind `tuple` results to typed variable

This should simplify the C code generated by Cython to unpack the
`tuple` as it no longer needs to check if it is a `list` or some other
sequence that needs to be unpacked and can simply use the `tuple`
unpacking logic.

* Collect `list` of messages for clients and workers

* Extend `BatchedSend`'s `send` to take many msgs

* Add `send_all` method and use in `transition`

This allows us to batch all worker and client sends into a single
function.

* Deliver all messages to batched send

* Refactor out private `_transition` function

* Send all messages after processing all transitions

* `declare` `ALL_TASK_STATES` a `set`
---
 distributed/batched.py   |   6 +-
 distributed/scheduler.py | 225 +++++++++++++++++++++++++++++----------
 2 files changed, 174 insertions(+), 57 deletions(-)

diff --git a/distributed/batched.py b/distributed/batched.py
index 313aab67b56..89e99719e9c 100644
--- a/distributed/batched.py
+++ b/distributed/batched.py
@@ -127,7 +127,7 @@ def _background_send(self):
         self.stopped.set()
         self.abort()
 
-    def send(self, msg):
+    def send(self, *msgs):
         """Schedule a message for sending to the other side
 
         This completes quickly and synchronously
@@ -135,8 +135,8 @@ def send(self, msg):
         if self.comm is not None and self.comm.closed():
             raise CommClosedError
 
-        self.message_count += 1
-        self.buffer.append(msg)
+        self.message_count += len(msgs)
+        self.buffer.extend(msgs)
         # Avoid spurious wakeups if possible
         if self.next_deadline is None:
             self.waker.set()
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 4ea7ce98ef1..2f4526b483a 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -182,7 +182,10 @@ def nogil(func):
     EventExtension,
 ]
 
-ALL_TASK_STATES = {"released", "waiting", "no-worker", "processing", "erred", "memory"}
+ALL_TASK_STATES = declare(
+    set, {"released", "waiting", "no-worker", "processing", "erred", "memory"}
+)
+globals()["ALL_TASK_STATES"] = ALL_TASK_STATES
 
 
 @final
@@ -1961,7 +1964,7 @@ def transition_waiting_processing(self, key):
 
             # logger.debug("Send job to worker: %s, %s", worker, key)
 
-            worker_msgs[worker] = _task_to_msg(self, ts)
+            worker_msgs[worker] = [_task_to_msg(self, ts)]
 
             return {}, worker_msgs, client_msgs
         except Exception as e:
@@ -2168,11 +2171,13 @@ def transition_memory_released(self, key, safe: bint = False):
                 ws._has_what.remove(ts)
                 ws._nbytes -= ts.get_nbytes()
                 ts._group._nbytes_in_memory -= ts.get_nbytes()
-                worker_msgs[ws._address] = {
-                    "op": "delete-data",
-                    "keys": [key],
-                    "report": False,
-                }
+                worker_msgs[ws._address] = [
+                    {
+                        "op": "delete-data",
+                        "keys": [key],
+                        "report": False,
+                    }
+                ]
 
             ts._who_has.clear()
 
@@ -2181,7 +2186,7 @@ def transition_memory_released(self, key, safe: bint = False):
             report_msg = {"op": "lost-data", "key": key}
             cs: ClientState
             for cs in ts._who_wants:
-                client_msgs[cs._client_key] = report_msg
+                client_msgs[cs._client_key] = [report_msg]
 
             if not ts._run_spec:  # pure data
                 recommendations[key] = "forgotten"
@@ -2234,7 +2239,7 @@ def transition_released_erred(self, key):
             }
             cs: ClientState
             for cs in ts._who_wants:
-                client_msgs[cs._client_key] = report_msg
+                client_msgs[cs._client_key] = [report_msg]
 
             ts.state = "erred"
 
@@ -2276,7 +2281,7 @@ def transition_erred_released(self, key):
             report_msg = {"op": "task-retried", "key": key}
             cs: ClientState
             for cs in ts._who_wants:
-                client_msgs[cs._client_key] = report_msg
+                client_msgs[cs._client_key] = [report_msg]
 
             ts.state = "released"
 
@@ -2343,7 +2348,7 @@ def transition_processing_released(self, key):
 
             w: str = _remove_from_processing(self, ts)
             if w:
-                worker_msgs[w] = {"op": "release-task", "key": key}
+                worker_msgs[w] = [{"op": "release-task", "key": key}]
 
             ts.state = "released"
 
@@ -2432,7 +2437,7 @@ def transition_processing_erred(
             }
             cs: ClientState
             for cs in ts._who_wants:
-                client_msgs[cs._client_key] = report_msg
+                client_msgs[cs._client_key] = [report_msg]
 
             cs = self._clients["fire-and-forget"]
             if ts in cs._wants_what:
@@ -4706,6 +4711,29 @@ def client_send(self, client, msg):
             if self.status == Status.running:
                 logger.critical("Tried writing to closed comm: %s", msg)
 
+    def send_all(self, client_msgs: dict, worker_msgs: dict):
+        """Send messages to client and workers"""
+        stream_comms: dict = self.stream_comms
+        client_comms: dict = self.client_comms
+        msgs: list
+
+        for worker, msgs in worker_msgs.items():
+            try:
+                w = stream_comms[worker]
+                w.send(*msgs)
+            except (CommClosedError, AttributeError):
+                self.loop.add_callback(self.remove_worker, address=worker)
+
+        for client, msgs in client_msgs.items():
+            c = client_comms.get(client)
+            if c is None:
+                continue
+            try:
+                c.send(*msgs)
+            except CommClosedError:
+                if self.status == Status.running:
+                    logger.critical("Tried writing to closed comm: %s", msgs)
+
     ############################
     # Less common interactions #
     ############################
@@ -5814,12 +5842,12 @@ async def register_worker_plugin(self, comm, plugin, name=None):
     # State Transitions #
     #####################
 
-    def transition(self, key, finish, *args, **kwargs):
+    def _transition(self, key, finish: str, *args, **kwargs):
         """Transition a key from its current state to the finish state
 
         Examples
         --------
-        >>> self.transition('x', 'waiting')
+        >>> self._transition('x', 'waiting')
         {'x': 'processing'}
 
         Returns
@@ -5832,47 +5860,85 @@ def transition(self, key, finish, *args, **kwargs):
         """
         parent: SchedulerState = cast(SchedulerState, self)
         ts: TaskState
+        start: str
+        start_finish: tuple
+        finish2: str
+        recommendations: dict
         worker_msgs: dict
         client_msgs: dict
+        msgs: list
+        new_msgs: list
+        dependents: set
+        dependencies: set
         try:
-            try:
-                ts = parent._tasks[key]
-            except KeyError:
-                return {}
+            recommendations = {}
+            worker_msgs = {}
+            client_msgs = {}
+
+            ts = parent._tasks.get(key)
+            if ts is None:
+                return recommendations, worker_msgs, client_msgs
             start = ts._state
             if start == finish:
-                return {}
+                return recommendations, worker_msgs, client_msgs
 
             if self.plugins:
                 dependents = set(ts._dependents)
                 dependencies = set(ts._dependencies)
 
-            recommendations: dict = {}
-            worker_msgs = {}
-            client_msgs = {}
-            if (start, finish) in self._transitions:
-                func = self._transitions[start, finish]
-                recommendations, worker_msgs, client_msgs = func(key, *args, **kwargs)
-            elif "released" not in (start, finish):
+            start_finish = (start, finish)
+            func = self._transitions.get(start_finish)
+            if func is not None:
+                a: tuple = func(key, *args, **kwargs)
+                recommendations, worker_msgs, client_msgs = a
+            elif "released" not in start_finish:
                 func = self._transitions["released", finish]
                 assert not args and not kwargs
-                a = self.transition(key, "released")
-                if key in a:
-                    func = self._transitions["released", a[key]]
-                b, worker_msgs, client_msgs = func(key)
-                a = a.copy()
-                a.update(b)
-                recommendations = a
+                a_recs: dict
+                a_wmsgs: dict
+                a_cmsgs: dict
+                a: tuple = self._transition(key, "released")
+                a_recs, a_wmsgs, a_cmsgs = a
+                v = a_recs.get(key)
+                if v is not None:
+                    func = self._transitions["released", v]
+                b_recs: dict
+                b_wmsgs: dict
+                b_cmsgs: dict
+                b: tuple = func(key)
+                b_recs, b_wmsgs, b_cmsgs = b
+
+                recommendations.update(a_recs)
+                for w, new_msgs in a_wmsgs.items():
+                    msgs = worker_msgs.get(w)
+                    if msgs is not None:
+                        msgs.extend(new_msgs)
+                    else:
+                        worker_msgs[w] = new_msgs
+                for c, new_msgs in a_cmsgs.items():
+                    msgs = client_msgs.get(c)
+                    if msgs is not None:
+                        msgs.extend(new_msgs)
+                    else:
+                        client_msgs[c] = new_msgs
+
+                recommendations.update(b_recs)
+                for w, new_msgs in b_wmsgs.items():
+                    msgs = worker_msgs.get(w)
+                    if msgs is not None:
+                        msgs.extend(new_msgs)
+                    else:
+                        worker_msgs[w] = new_msgs
+                for c, new_msgs in b_cmsgs.items():
+                    msgs = client_msgs.get(c)
+                    if msgs is not None:
+                        msgs.extend(new_msgs)
+                    else:
+                        client_msgs[c] = new_msgs
+
                 start = "released"
             else:
-                raise RuntimeError(
-                    "Impossible transition from %r to %r" % (start, finish)
-                )
-
-            for worker, msg in worker_msgs.items():
-                self.worker_send(worker, msg)
-            for client, msg in client_msgs.items():
-                self.client_send(client, msg)
+                raise RuntimeError("Impossible transition from %r to %r" % start_finish)
 
             finish2 = ts._state
             self.transition_log.append((key, start, finish2, recommendations, time()))
@@ -5888,11 +5954,8 @@ def transition(self, key, finish, *args, **kwargs):
             if self.plugins:
                 # Temporarily put back forgotten key for plugin to retrieve it
                 if ts._state == "forgotten":
-                    try:
-                        ts._dependents = dependents
-                        ts._dependencies = dependencies
-                    except KeyError:
-                        pass
+                    ts._dependents = dependents
+                    ts._dependencies = dependencies
                     parent._tasks[ts._key] = ts
                 for plugin in list(self.plugins):
                     try:
@@ -5905,11 +5968,16 @@ def transition(self, key, finish, *args, **kwargs):
             tg: TaskGroup = ts._group
             if ts._state == "forgotten" and tg._name in parent._task_groups:
                 # Remove TaskGroup if all tasks are in the forgotten state
-                if not any([tg._states.get(s) for s in ALL_TASK_STATES]):
+                all_forgotten: bint = True
+                for s in ALL_TASK_STATES:
+                    if tg._states.get(s):
+                        all_forgotten = False
+                        break
+                if all_forgotten:
                     ts._prefix._groups.remove(tg)
                     del parent._task_groups[tg._name]
 
-            return recommendations
+            return recommendations, worker_msgs, client_msgs
         except Exception as e:
             logger.exception("Error transitioning %r from %r to %r", key, start, finish)
             if LOG_PDB:
@@ -5918,6 +5986,30 @@ def transition(self, key, finish, *args, **kwargs):
                 pdb.set_trace()
             raise
 
+    def transition(self, key, finish: str, *args, **kwargs):
+        """Transition a key from its current state to the finish state
+
+        Examples
+        --------
+        >>> self.transition('x', 'waiting')
+        {'x': 'processing'}
+
+        Returns
+        -------
+        Dictionary of recommendations for future transitions
+
+        See Also
+        --------
+        Scheduler.transitions: transitive version of this function
+        """
+        recommendations: dict
+        worker_msgs: dict
+        client_msgs: dict
+        a: tuple = self._transition(key, finish, *args, **kwargs)
+        recommendations, worker_msgs, client_msgs = a
+        self.send_all(client_msgs, worker_msgs)
+        return recommendations
+
     def transitions(self, recommendations: dict):
         """Process transitions until none are left
 
@@ -5925,13 +6017,38 @@ def transitions(self, recommendations: dict):
         reach a steady state
         """
         parent: SchedulerState = cast(SchedulerState, self)
-        keys = set()
+        keys: set = set()
         recommendations = recommendations.copy()
+        worker_msgs: dict = {}
+        client_msgs: dict = {}
+        msgs: list
+        new_msgs: list
+        new: tuple
+        new_recs: dict
+        new_wmsgs: dict
+        new_cmsgs: dict
         while recommendations:
             key, finish = recommendations.popitem()
             keys.add(key)
-            new = self.transition(key, finish)
-            recommendations.update(new)
+
+            new = self._transition(key, finish)
+            new_recs, new_wmsgs, new_cmsgs = new
+
+            recommendations.update(new_recs)
+            for w, new_msgs in new_wmsgs.items():
+                msgs = worker_msgs.get(w)
+                if msgs is not None:
+                    msgs.extend(new_msgs)
+                else:
+                    worker_msgs[w] = new_msgs
+            for c, new_msgs in new_cmsgs.items():
+                msgs = client_msgs.get(c)
+                if msgs is not None:
+                    msgs.extend(new_msgs)
+                else:
+                    client_msgs[c] = new_msgs
+
+        self.send_all(client_msgs, worker_msgs)
 
         if parent._validate:
             for key in keys:
@@ -6513,7 +6630,7 @@ def _add_to_memory(
             report_msg["type"] = type
 
         for cs in ts._who_wants:
-            client_msgs[cs._client_key] = report_msg
+            client_msgs[cs._client_key] = [report_msg]
 
     ts.state = "memory"
     ts._type = typename
@@ -6567,7 +6684,7 @@ def _propagate_forgotten(
         ws._nbytes -= ts.get_nbytes()
         w: str = ws._address
         if w in state._workers_dv:  # in case worker has died
-            worker_msgs[w] = {"op": "delete-data", "keys": [key], "report": False}
+            worker_msgs[w] = [{"op": "delete-data", "keys": [key], "report": False}]
     ts._who_has.clear()
 
 
@@ -6674,7 +6791,7 @@ def _task_to_client_msgs(state: SchedulerState, ts: TaskState) -> dict:
 
     client_msgs: dict = {}
     for k in client_keys:
-        client_msgs[k] = report_msg
+        client_msgs[k] = [report_msg]
 
     return client_msgs
 

From 54bfc2fd0275b9643fa36669b559082638b75ed6 Mon Sep 17 00:00:00 2001
From: Ian Rose <ian.r.rose@gmail.com>
Date: Wed, 3 Feb 2021 18:10:50 -0800
Subject: [PATCH 1143/1550] Use cached version of is_coroutine_function in
 stream handling to (#4481)

mitigate re-checking the same function many times.

Decorate function definition, cache all function definitions.

Oops, cache is python 3.9-only

Alias duplicate function.

Use distributed's version of this util.

bad rebase.
---
 distributed/actor.py | 3 +--
 distributed/utils.py | 4 ++--
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/distributed/actor.py b/distributed/actor.py
index 54e9000bdaf..b9149fec7f4 100644
--- a/distributed/actor.py
+++ b/distributed/actor.py
@@ -1,12 +1,11 @@
 import asyncio
 import functools
-from inspect import iscoroutinefunction
 import threading
 from queue import Queue
 
 from .client import Future, default_client
 from .protocol import to_serialize
-from .utils import thread_state, sync
+from .utils import iscoroutinefunction, thread_state, sync
 from .utils_comm import WrappedKey
 from .worker import get_worker
 
diff --git a/distributed/utils.py b/distributed/utils.py
index 30044740f64..3565b4ceb9a 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -1246,6 +1246,7 @@ def color_of(x, palette=palette):
     return palette[n % len(palette)]
 
 
+@functools.lru_cache(None)
 def iscoroutinefunction(f):
     return inspect.iscoroutinefunction(f) or gen.is_coroutine_function(f)
 
@@ -1343,8 +1344,7 @@ def parse_ports(port):
     return ports
 
 
-def is_coroutine_function(f):
-    return asyncio.iscoroutinefunction(f) or gen.is_coroutine_function(f)
+is_coroutine_function = iscoroutinefunction
 
 
 class Log(str):

From f3f4bffea0640c01fc54f49c3219cf5807d14c66 Mon Sep 17 00:00:00 2001
From: Matthias Bussonnier <bussonniermatthias@gmail.com>
Date: Wed, 3 Feb 2021 18:32:44 -0800
Subject: [PATCH 1144/1550] DOC: Autoreformat all functions docstrings. (#4475)

Prompted by comment in #4473
This uses my prototype package `velin` to auto format all the functions
docstrings, from the root of the repo:

    $ velin distributed/ --write

This should fix some common issues/mistakes with numpydoc format often
around the presence/absence of space and/or colon, that can have
semantic imlications.

Note that this easy to redo per-file / folder; so let me know if this is
too big
---
 distributed/_ipython_utils.py              |   7 +-
 distributed/cfexecutor.py                  |  10 +-
 distributed/client.py                      | 300 ++++++++++-----------
 distributed/comm/core.py                   |   2 +-
 distributed/core.py                        |   4 +-
 distributed/deploy/cluster.py              |   2 +-
 distributed/deploy/ssh.py                  |  12 +-
 distributed/deploy/utils.py                |   2 +-
 distributed/diagnostics/plugin.py          |  32 +--
 distributed/diagnostics/progress_stream.py |   1 -
 distributed/diagnostics/progressbar.py     |   8 +-
 distributed/diagnostics/websocket.py       |   8 +-
 distributed/diskutils.py                   |   4 +-
 distributed/preloading.py                  |   4 +-
 distributed/profile.py                     |  20 +-
 distributed/pubsub.py                      |   2 +-
 distributed/queues.py                      |   6 +-
 distributed/recreate_exceptions.py         |   3 +-
 distributed/scheduler.py                   |  16 +-
 distributed/stealing.py                    |   1 -
 distributed/threadpoolexecutor.py          |   4 +-
 distributed/utils.py                       |  17 +-
 distributed/utils_comm.py                  |   8 +-
 distributed/variable.py                    |   4 +-
 distributed/worker_client.py               |   4 +-
 25 files changed, 234 insertions(+), 247 deletions(-)

diff --git a/distributed/_ipython_utils.py b/distributed/_ipython_utils.py
index 1a999833786..0fe97c7280e 100644
--- a/distributed/_ipython_utils.py
+++ b/distributed/_ipython_utils.py
@@ -187,12 +187,11 @@ def start_ipython(ip=None, ns=None, log=None):
 
     Parameters
     ----------
-
-    ip: str
+    ip : str
         The IP address to listen on (likely the parent object's ip).
-    ns: dict
+    ns : dict
         Any names that should be injected into the IPython namespace.
-    log: logger instance
+    log : logger instance
         Hook up IPython's logging to an existing logger instead of the default.
     """
     from IPython import get_ipython
diff --git a/distributed/cfexecutor.py b/distributed/cfexecutor.py
index 545dbbced09..e11c96c2821 100644
--- a/distributed/cfexecutor.py
+++ b/distributed/cfexecutor.py
@@ -98,12 +98,12 @@ def map(self, fn, *iterables, **kwargs):
 
         Parameters
         ----------
-        fn: A callable that will take as many arguments as there are
+        fn : A callable that will take as many arguments as there are
             passed iterables.
-        iterables: One iterable for each parameter to *fn*.
-        timeout: The maximum number of seconds to wait. If None, then there
+        iterables : One iterable for each parameter to *fn*.
+        timeout : The maximum number of seconds to wait. If None, then there
             is no limit on the wait time.
-        chunksize: ignored.
+        chunksize : ignored.
 
         Returns
         -------
@@ -155,7 +155,7 @@ def shutdown(self, wait=True):
 
         Parameters
         ----------
-        wait: If True then shutdown will not return until all running
+        wait : If True then shutdown will not return until all running
             futures have finished executing.  If False then all running
             futures are cancelled immediately.
         """
diff --git a/distributed/client.py b/distributed/client.py
index 7c1ea57c119..1883439f4c9 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -1456,9 +1456,9 @@ def shutdown(self):
         Note, this may disrupt other clients that may be using the same
         scheduler and workers.
 
-        See also
+        See Also
         --------
-        Client.close: close only this client
+        Client.close : close only this client
         """
         return self.sync(self._shutdown)
 
@@ -1468,7 +1468,7 @@ def get_executor(self, **kwargs):
 
         Parameters
         ----------
-        **kwargs:
+        **kwargs
             Any submit()- or map()- compatible arguments, such as
             `workers` or `resources`.
 
@@ -1500,36 +1500,36 @@ def submit(
 
         Parameters
         ----------
-        func: callable
-        *args:
-        **kwargs:
-        pure: bool (defaults to True)
+        func : callable
+        *args
+        **kwargs
+        pure : bool (defaults to True)
             Whether or not the function is pure.  Set ``pure=False`` for
             impure functions like ``np.random.random``.
-        workers: string or iterable of strings
+        workers : string or iterable of strings
             A set of worker addresses or hostnames on which computations may be
             performed. Leave empty to default to all workers (common case)
-        key: str
+        key : str
             Unique identifier for the task.  Defaults to function-name and hash
-        allow_other_workers: bool (defaults to False)
+        allow_other_workers : bool (defaults to False)
             Used with ``workers``. Indicates whether or not the computations
             may be performed on workers that are not in the `workers` set(s).
-        retries: int (default to 0)
+        retries : int (default to 0)
             Number of allowed automatic retries if the task fails
-        priority: Number
+        priority : Number
             Optional prioritization of task.  Zero is default.
             Higher priorities take precedence
-        fifo_timeout: str timedelta (default '100ms')
+        fifo_timeout : str timedelta (default '100ms')
             Allowed amount of time between calls to consider the same priority
-        resources: dict (defaults to {})
+        resources : dict (defaults to {})
             Defines the ``resources`` each instance of this mapped task requires
             on the worker; e.g. ``{'GPU': 2}``.
             See :doc:`worker resources <resources>` for details on defining
             resources.
-        actor: bool (default False)
+        actor : bool (default False)
             Whether this task should exist on the worker as a stateful actor.
             See :doc:`actors` for additional details.
-        actors: bool (default False)
+        actors : bool (default False)
             Alias for `actor`
 
         Examples
@@ -1542,7 +1542,7 @@ def submit(
 
         See Also
         --------
-        Client.map: Submit on many arguments at once
+        Client.map : Submit on many arguments at once
         """
         if not callable(func):
             raise TypeError("First input to submit must be a callable function")
@@ -1617,43 +1617,43 @@ def map(
 
         Parameters
         ----------
-        func: callable
-        iterables: Iterables
+        func : callable
+        iterables : Iterables
             List-like objects to map over.  They should have the same length.
-        key: str, list
+        key : str, list
             Prefix for task names if string.  Explicit names if list.
-        pure: bool (defaults to True)
+        pure : bool (defaults to True)
             Whether or not the function is pure.  Set ``pure=False`` for
             impure functions like ``np.random.random``.
-        workers: string or iterable of strings
+        workers : string or iterable of strings
             A set of worker hostnames on which computations may be performed.
             Leave empty to default to all workers (common case)
-        allow_other_workers: bool (defaults to False)
+        allow_other_workers : bool (defaults to False)
             Used with `workers`. Indicates whether or not the computations
             may be performed on workers that are not in the `workers` set(s).
-        retries: int (default to 0)
+        retries : int (default to 0)
             Number of allowed automatic retries if a task fails
-        priority: Number
+        priority : Number
             Optional prioritization of task.  Zero is default.
             Higher priorities take precedence
-        fifo_timeout: str timedelta (default '100ms')
+        fifo_timeout : str timedelta (default '100ms')
             Allowed amount of time between calls to consider the same priority
-        resources: dict (defaults to {})
+        resources : dict (defaults to {})
             Defines the `resources` each instance of this mapped task requires
             on the worker; e.g. ``{'GPU': 2}``.
             See :doc:`worker resources <resources>` for details on defining
             resources.
-        actor: bool (default False)
+        actor : bool (default False)
             Whether these tasks should exist on the worker as stateful actors.
             See :doc:`actors` for additional details.
-        actors: bool (default False)
+        actors : bool (default False)
             Alias for `actor`
         batch_size : int, optional
             Submit tasks to the scheduler in batches of (at most) ``batch_size``.
             Larger batch sizes can be useful for very large ``iterables``,
             as the cluster can start processing tasks while later ones are
             submitted asynchronously.
-        **kwargs: dict
+        **kwargs : dict
             Extra keywords to send to the function.
             Large values will be included explicitly in the task graph.
 
@@ -1666,9 +1666,9 @@ def map(
         List, iterator, or Queue of futures, depending on the type of the
         inputs.
 
-        See also
+        See Also
         --------
-        Client.submit: Submit a single function
+        Client.submit : Submit a single function
         """
         if not callable(func):
             raise TypeError("First input to map must be a callable function")
@@ -1918,13 +1918,13 @@ def gather(self, futures, errors="raise", direct=None, asynchronous=None):
 
         Parameters
         ----------
-        futures: Collection of futures
+        futures : Collection of futures
             This can be a possibly nested collection of Future objects.
             Collections can be lists, sets, or dictionaries
-        errors: string
+        errors : string
             Either 'raise' or 'skip' if we should raise if a future has erred
             or skip its inclusion in the output collection
-        direct: boolean
+        direct : boolean
             Whether or not to connect directly to the workers, or to ask
             the scheduler to serve as intermediary.  This can also be set when
             creating the Client.
@@ -1946,7 +1946,7 @@ def gather(self, futures, errors="raise", direct=None, asynchronous=None):
 
         See Also
         --------
-        Client.scatter: Send data out to cluster
+        Client.scatter : Send data out to cluster
         """
         if isinstance(futures, pyQueue):
             raise TypeError(
@@ -2098,19 +2098,19 @@ def scatter(
 
         Parameters
         ----------
-        data: list, dict, or object
+        data : list, dict, or object
             Data to scatter out to workers.  Output type matches input type.
-        workers: list of tuples (optional)
+        workers : list of tuples (optional)
             Optionally constrain locations of data.
             Specify workers as hostname/port pairs, e.g. ``('127.0.0.1', 8787)``.
-        broadcast: bool (defaults to False)
+        broadcast : bool (defaults to False)
             Whether to send each data element to all workers.
             By default we round-robin based on number of cores.
-        direct: bool (defaults to automatically check)
+        direct : bool (defaults to automatically check)
             Whether or not to connect directly to the workers, or to ask
             the scheduler to serve as intermediary.  This can also be set when
             creating the Client.
-        hash: bool (optional)
+        hash : bool (optional)
             Whether or not to hash data to determine key.
             If False then this uses a random key
 
@@ -2150,7 +2150,7 @@ def scatter(
 
         See Also
         --------
-        Client.gather: Gather data back to local process
+        Client.gather : Gather data back to local process
         """
         if timeout == no_default:
             timeout = self._timeout
@@ -2194,8 +2194,8 @@ def cancel(self, futures, asynchronous=None, force=False):
 
         Parameters
         ----------
-        futures: list of Futures
-        force: boolean (False)
+        futures : list of Futures
+        force : boolean (False)
             Cancel this future even if other clients desire it
         """
         return self.sync(self._cancel, futures, asynchronous=asynchronous, force=force)
@@ -2213,7 +2213,7 @@ def retry(self, futures, asynchronous=None):
 
         Parameters
         ----------
-        futures: list of Futures
+        futures : list of Futures
         """
         return self.sync(self._retry, futures, asynchronous=asynchronous)
 
@@ -2266,7 +2266,7 @@ def publish_dataset(self, *args, **kwargs):
         name : optional name of the dataset to publish
         override : bool (optional, default False)
             if true, override any already present dataset with the same name
-        kwargs: dict
+        kwargs : dict
             named collections to publish on the scheduler
 
         Examples
@@ -2349,7 +2349,7 @@ def get_dataset(self, name, default=NO_DEFAULT_PLACEHOLDER, **kwargs):
         name : name of the dataset to retrieve
         default : optional, not set by default
             If set, do not raise a KeyError if the name is not present but return this default
-        kwargs: dict
+        kwargs : dict
             additional arguments to _get_dataset
 
         See Also
@@ -2381,7 +2381,6 @@ def run_on_scheduler(self, function, *args, **kwargs):
 
         Examples
         --------
-
         >>> def get_number_of_tasks(dask_scheduler=None):
         ...     return len(dask_scheduler.tasks)
 
@@ -2399,8 +2398,8 @@ def run_on_scheduler(self, function, *args, **kwargs):
 
         See Also
         --------
-        Client.run: Run a function on all workers
-        Client.start_ipython_scheduler: Start an IPython session on scheduler
+        Client.run : Run a function on all workers
+        Client.start_ipython_scheduler : Start an IPython session on scheduler
         """
         return self.sync(self._run_on_scheduler, function, *args, **kwargs)
 
@@ -2443,12 +2442,12 @@ def run(self, function, *args, **kwargs):
 
         Parameters
         ----------
-        function: callable
-        *args: arguments for remote function
-        **kwargs: keyword arguments for remote function
-        workers: list
+        function : callable
+        *args : arguments for remote function
+        **kwargs : keyword arguments for remote function
+        workers : list
             Workers on which to run the function. Defaults to all known workers.
-        wait: boolean (optional)
+        wait : boolean (optional)
             If the function is asynchronous whether or not to wait until that
             function finishes.
         nanny : bool, defualt False
@@ -2499,14 +2498,14 @@ def run_coroutine(self, function, *args, **kwargs):
 
         Parameters
         ----------
-        function: a coroutine function
+        function : a coroutine function
             (typically a function wrapped in gen.coroutine or
              a Python 3.5+ async function)
-        *args: arguments for remote function
-        **kwargs: keyword arguments for remote function
-        wait: boolean (default True)
+        *args : arguments for remote function
+        **kwargs : keyword arguments for remote function
+        wait : boolean (default True)
             Whether to wait for coroutines to end.
-        workers: list
+        workers : list
             Workers on which to run the function. Defaults to all known workers.
 
         """
@@ -2597,27 +2596,27 @@ def get(
 
         Parameters
         ----------
-        dsk: dict
-        keys: object, or nested lists of objects
-        workers: string or iterable of strings
+        dsk : dict
+        keys : object, or nested lists of objects
+        workers : string or iterable of strings
             A set of worker addresses or hostnames on which computations may be
             performed. Leave empty to default to all workers (common case)
-        allow_other_workers: bool (defaults to False)
+        allow_other_workers : bool (defaults to False)
             Used with ``workers``. Indicates whether or not the computations
             may be performed on workers that are not in the `workers` set(s).
-        retries: int (default to 0)
+        retries : int (default to 0)
             Number of allowed automatic retries if computing a result fails
-        priority: Number
+        priority : Number
             Optional prioritization of task.  Zero is default.
             Higher priorities take precedence
-        resources: dict (defaults to {})
+        resources : dict (defaults to {})
             Defines the ``resources`` each instance of this mapped task requires
             on the worker; e.g. ``{'GPU': 2}``.
             See :doc:`worker resources <resources>` for details on defining
             resources.
-        sync: bool (optional)
+        sync : bool (optional)
             Returns Futures if False or concrete values if True (default).
-        direct: bool
+        direct : bool
             Whether or not to connect directly to the workers, or to ask
             the scheduler to serve as intermediary.  This can also be set when
             creating the Client.
@@ -2631,7 +2630,7 @@ def get(
 
         See Also
         --------
-        Client.compute: Compute asynchronous collections
+        Client.compute : Compute asynchronous collections
         """
         futures = self._graph_to_futures(
             dsk,
@@ -2707,7 +2706,7 @@ def normalize_collection(self, collection):
 
         See Also
         --------
-        Client.persist: trigger computation of collection's tasks
+        Client.persist : trigger computation of collection's tasks
         """
         dsk_orig = collection.__dask_graph__()
         dsk = self._optimize_insert_futures(dsk_orig, collection.__dask_keys__())
@@ -2736,40 +2735,40 @@ def compute(
 
         Parameters
         ----------
-        collections: iterable of dask objects or single dask object
+        collections : iterable of dask objects or single dask object
             Collections like dask.array or dataframe or dask.value objects
-        sync: bool (optional)
+        sync : bool (optional)
             Returns Futures if False (default) or concrete values if True
-        optimize_graph: bool
+        optimize_graph : bool
             Whether or not to optimize the underlying graphs
-        workers: string or iterable of strings
+        workers : string or iterable of strings
             A set of worker hostnames on which computations may be performed.
             Leave empty to default to all workers (common case)
-        allow_other_workers: bool (defaults to False)
+        allow_other_workers : bool (defaults to False)
             Used with `workers`. Indicates whether or not the computations
             may be performed on workers that are not in the `workers` set(s).
-        retries: int (default to 0)
+        retries : int (default to 0)
             Number of allowed automatic retries if computing a result fails
-        priority: Number
+        priority : Number
             Optional prioritization of task.  Zero is default.
             Higher priorities take precedence
-        fifo_timeout: timedelta str (defaults to '60s')
+        fifo_timeout : timedelta str (defaults to '60s')
             Allowed amount of time between calls to consider the same priority
-        traverse: bool (defaults to True)
+        traverse : bool (defaults to True)
             By default dask traverses builtin python collections looking for
             dask objects passed to ``compute``. For large collections this can
             be expensive. If none of the arguments contain any dask objects,
             set ``traverse=False`` to avoid doing this traversal.
-        resources: dict (defaults to {})
+        resources : dict (defaults to {})
             Defines the `resources` each instance of this mapped task requires
             on the worker; e.g. ``{'GPU': 2}``.
             See :doc:`worker resources <resources>` for details on defining
             resources.
-        actors: bool or dict (default None)
+        actors : bool or dict (default None)
             Whether these tasks should exist on the worker as stateful actors.
             Specified on a global (True/False) or per-task (``{'x': True,
             'y': False}``) basis. See :doc:`actors` for additional details.
-        **kwargs:
+        **kwargs
             Options to pass to the graph optimize calls
 
         Returns
@@ -2796,7 +2795,7 @@ def compute(
 
         See Also
         --------
-        Client.get: Normal synchronous dask.get function
+        Client.get : Normal synchronous dask.get function
         """
         if isinstance(collections, (list, tuple, set, frozenset)):
             singleton = False
@@ -2888,33 +2887,33 @@ def persist(
 
         Parameters
         ----------
-        collections: sequence or single dask object
+        collections : sequence or single dask object
             Collections like dask.array or dataframe or dask.value objects
-        optimize_graph: bool
+        optimize_graph : bool
             Whether or not to optimize the underlying graphs
-        workers: string or iterable of strings
+        workers : string or iterable of strings
             A set of worker hostnames on which computations may be performed.
             Leave empty to default to all workers (common case)
-        allow_other_workers: bool (defaults to False)
+        allow_other_workers : bool (defaults to False)
             Used with `workers`. Indicates whether or not the computations
             may be performed on workers that are not in the `workers` set(s).
-        retries: int (default to 0)
+        retries : int (default to 0)
             Number of allowed automatic retries if computing a result fails
-        priority: Number
+        priority : Number
             Optional prioritization of task.  Zero is default.
             Higher priorities take precedence
-        fifo_timeout: timedelta str (defaults to '60s')
+        fifo_timeout : timedelta str (defaults to '60s')
             Allowed amount of time between calls to consider the same priority
-        resources: dict (defaults to {})
+        resources : dict (defaults to {})
             Defines the `resources` each instance of this mapped task requires
             on the worker; e.g. ``{'GPU': 2}``.
             See :doc:`worker resources <resources>` for details on defining
             resources.
-        actors: bool or dict (default None)
+        actors : bool or dict (default None)
             Whether these tasks should exist on the worker as stateful actors.
             Specified on a global (True/False) or per-task (``{'x': True,
             'y': False}``) basis. See :doc:`actors` for additional details.
-        **kwargs:
+        **kwargs
             Options to pass to the graph optimize calls
 
         Returns
@@ -3025,7 +3024,7 @@ def upload_file(self, filename, **kwargs):
 
         Parameters
         ----------
-        filename: string
+        filename : string
             Filename of .py, .egg or .zip file to send to workers
 
         Examples
@@ -3062,9 +3061,9 @@ def rebalance(self, futures=None, workers=None, **kwargs):
 
         Parameters
         ----------
-        futures: list, optional
+        futures : list, optional
             A list of futures to balance, defaults all data
-        workers: list, optional
+        workers : list, optional
             A list of workers on which to balance, defaults to all workers
         """
         return self.sync(self._rebalance, futures, workers, **kwargs)
@@ -3089,15 +3088,15 @@ def replicate(self, futures, n=None, workers=None, branching_factor=2, **kwargs)
 
         Parameters
         ----------
-        futures: list of futures
+        futures : list of futures
             Futures we wish to replicate
-        n: int, optional
+        n : int, optional
             Number of processes on the cluster on which to replicate the data.
             Defaults to all.
-        workers: list of worker addresses
+        workers : list of worker addresses
             Workers on which we want to restrict the replication.
             Defaults to all.
-        branching_factor: int, optional
+        branching_factor : int, optional
             The number of workers that can copy data in each generation
 
         Examples
@@ -3109,7 +3108,7 @@ def replicate(self, futures, n=None, workers=None, branching_factor=2, **kwargs)
         >>> c.replicate([x], n=1, workers=['alice', 'bob'])  # send to one of specific workers  # doctest: +SKIP
         >>> c.replicate([x], n=1)  # reduce replications # doctest: +SKIP
 
-        See also
+        See Also
         --------
         Client.rebalance
         """
@@ -3127,7 +3126,7 @@ def nthreads(self, workers=None, **kwargs):
 
         Parameters
         ----------
-        workers: list (optional)
+        workers : list (optional)
             A list of workers that we care about specifically.
             Leave empty to receive information about all workers.
 
@@ -3159,7 +3158,7 @@ def who_has(self, futures=None, **kwargs):
 
         Parameters
         ----------
-        futures: list (optional)
+        futures : list (optional)
             A list of futures, defaults to all data
 
         Examples
@@ -3195,7 +3194,7 @@ def has_what(self, workers=None, **kwargs):
 
         Parameters
         ----------
-        workers: list (optional)
+        workers : list (optional)
             A list of worker addresses, defaults to all
 
         Examples
@@ -3226,7 +3225,7 @@ def processing(self, workers=None):
 
         Parameters
         ----------
-        workers: list (optional)
+        workers : list (optional)
             A list of worker addresses, defaults to all
 
         Examples
@@ -3259,9 +3258,9 @@ def nbytes(self, keys=None, summary=True, **kwargs):
 
         Parameters
         ----------
-        keys: list (optional)
+        keys : list (optional)
             A list of keys, defaults to all keys
-        summary: boolean, (optional)
+        summary : boolean, (optional)
             Summarize keys into key types
 
         Examples
@@ -3291,9 +3290,9 @@ def call_stack(self, futures=None, keys=None):
 
         Parameters
         ----------
-        futures: list (optional)
+        futures : list (optional)
             List of futures, defaults to all data
-        keys: list (optional)
+        keys : list (optional)
             List of key names, defaults to all data
 
         Examples
@@ -3325,24 +3324,24 @@ def profile(
 
         Parameters
         ----------
-        key: str
+        key : str
             Key prefix to select, this is typically a function name like 'inc'
             Leave as None to collect all data
-        start: time
-        stop: time
-        workers: list
+        start : time
+        stop : time
+        workers : list
             List of workers to restrict profile information
         server : bool
             If true, return the profile of the worker's administrative thread
             rather than the worker threads.
             This is useful when profiling Dask itself, rather than user code.
-        scheduler: bool
+        scheduler : bool
             If true, return the profile information from the scheduler's
             administrative thread rather than the workers.
             This is useful when profiling Dask's scheduling itself.
-        plot: boolean or string
+        plot : boolean or string
             Whether or not to return a plot object
-        filename: str
+        filename : str
             Filename to save the plot
 
         Examples
@@ -3439,7 +3438,7 @@ def write_scheduler_file(self, scheduler_file):
 
         Parameters
         ----------
-        scheduler_file: str
+        scheduler_file : str
             Path to a write the scheduler file.
 
         Examples
@@ -3464,14 +3463,14 @@ def get_metadata(self, keys, default=no_default):
 
         Parameters
         ----------
-        keys: key or list
+        keys : key or list
             Key to access.  If a list then gets within a nested collection
-        default: optional
+        default : optional
             If the key does not exist then return this value instead.
             If not provided then this raises a KeyError if the key is not
             present
 
-        See also
+        See Also
         --------
         Client.set_metadata
         """
@@ -3521,7 +3520,7 @@ def log_event(self, topic, msg):
 
         Parameters
         ----------
-        topic: str, list
+        topic : str, list
             Name of the topic under which to log an event. To log the same
             event under multiple topics, pass a list of topic names.
         msg
@@ -3539,7 +3538,7 @@ def get_events(self, topic: str = None):
 
         Parameters
         ----------
-        topic: str, optional
+        topic : str, optional
             Name of topic log to retrieve events for. If no ``topic`` is
             provided, then logs for all topics will be returned.
         """
@@ -3682,18 +3681,15 @@ def start_ipython_workers(
 
         Parameters
         ----------
-        workers: list (optional)
+        workers : list (optional)
             A list of worker addresses, defaults to all
-
-        magic_names: str or list(str) (optional)
+        magic_names : str or list(str) (optional)
             If defined, register IPython magics with these names for
             executing code on the workers.  If string has asterix then expand
             asterix into 0, 1, ..., n for n workers
-
-        qtconsole: bool (optional)
+        qtconsole : bool (optional)
             If True, launch a Jupyter QtConsole connected to the worker(s).
-
-        qtconsole_args: list(str) (optional)
+        qtconsole_args : list(str) (optional)
             Additional arguments to pass to the qtconsole on startup.
 
         Examples
@@ -3724,7 +3720,7 @@ def start_ipython_workers(
 
         See Also
         --------
-        Client.start_ipython_scheduler: start ipython on the scheduler
+        Client.start_ipython_scheduler : start ipython on the scheduler
         """
         if isinstance(workers, (str, Number)):
             workers = [workers]
@@ -3764,15 +3760,13 @@ def start_ipython_scheduler(
 
         Parameters
         ----------
-        magic_name: str or None (optional)
+        magic_name : str or None (optional)
             If defined, register IPython magic with this name for
             executing code on the scheduler.
             If not defined, register %scheduler magic if IPython is running.
-
-        qtconsole: bool (optional)
+        qtconsole : bool (optional)
             If True, launch a Jupyter QtConsole connected to the worker(s).
-
-        qtconsole_args: list(str) (optional)
+        qtconsole_args : list(str) (optional)
             Additional arguments to pass to the qtconsole on startup.
 
         Examples
@@ -3792,7 +3786,7 @@ def start_ipython_scheduler(
 
         See Also
         --------
-        Client.start_ipython_workers: Start IPython on the workers
+        Client.start_ipython_workers : Start IPython on the workers
         """
         info = sync(self.loop, self.scheduler.start_ipython)
         if magic_name == "scheduler_if_ipython":
@@ -3856,22 +3850,22 @@ def get_task_stream(
 
         Parameters
         ----------
-        start: Number or string
+        start : Number or string
             When you want to start recording
             If a number it should be the result of calling time()
             If a string then it should be a time difference before now,
             like '60s' or '500 ms'
-        stop: Number or string
+        stop : Number or string
             When you want to stop recording
-        count: int
+        count : int
             The number of desired records, ignored if both start and stop are
             specified
-        plot: boolean, str
+        plot : boolean, str
             If true then also return a Bokeh figure
             If plot == 'save' then save the figure to a file
-        filename: str (optional)
+        filename : str (optional)
             The filename to save to if you set ``plot='save'``
-        bokeh_resources: bokeh.resources.Resources (optional)
+        bokeh_resources : bokeh.resources.Resources (optional)
             Specifies if the resource component is INLINE or CDN
 
         Examples
@@ -3903,7 +3897,7 @@ def get_task_stream(
 
         See Also
         --------
-        get_task_stream: a context manager version of this method
+        get_task_stream : a context manager version of this method
         """
         return self.sync(
             self._get_task_stream,
@@ -3998,12 +3992,12 @@ def register_worker_plugin(self, plugin=None, name=None, **kwargs):
 
         Parameters
         ----------
-        plugin: WorkerPlugin
+        plugin : WorkerPlugin
             The plugin object to pass to the workers
-        name: str, optional
+        name : str, optional
             A name for the plugin.
             Registering a plugin with the same name will have no effect.
-        **kwargs: optional
+        **kwargs : optional
             If you pass a class as the plugin, instead of a class instance, then the
             class will be instantiated with any extra keyword arguments.
 
@@ -4114,10 +4108,10 @@ def wait(fs, timeout=None, return_when=ALL_COMPLETED):
 
     Parameters
     ----------
-    fs: list of futures
-    timeout: number, optional
+    fs : list of futures
+    timeout : number, optional
         Time in seconds after which to raise a ``dask.distributed.TimeoutError``
-    return_when: str, optional
+    return_when : str, optional
         One of `ALL_COMPLETED` or `FIRST_COMPLETED`
 
     Returns
@@ -4343,7 +4337,7 @@ def next_batch(self, block=True):
 
         Parameters
         ----------
-        block: bool, optional
+        block : bool, optional
             If True then wait until we have some result, otherwise return
             immediately, even with an empty list.  Defaults to True.
 
@@ -4440,7 +4434,7 @@ def futures_of(o, client=None):
 
     Parameters
     ----------
-    o: collection
+    o : collection
         A possibly nested collection of Dask objects
 
     Examples
@@ -4496,7 +4490,7 @@ def fire_and_forget(obj):
 
     Parameters
     ----------
-    obj: Future, list, dict, dask collection
+    obj : Future, list, dict, dask collection
         The futures that you want to run at least once
 
     Examples
diff --git a/distributed/comm/core.py b/distributed/comm/core.py
index 21f9446725b..2a01ec3d728 100644
--- a/distributed/comm/core.py
+++ b/distributed/comm/core.py
@@ -75,7 +75,7 @@ def write(self, msg, serializers=None, on_error=None):
 
         Parameters
         ----------
-        msg :
+        msg
         on_error : Optional[str]
             The behavior when serialization fails. See
             ``distributed.protocol.core.dumps`` for valid values.
diff --git a/distributed/core.py b/distributed/core.py
index 15205f4f72c..6e7090c852c 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -1119,7 +1119,7 @@ def error_message(e, status="error"):
 
     See Also
     --------
-    clean_exception: deserialize and unpack message into exception/traceback
+    clean_exception : deserialize and unpack message into exception/traceback
     """
     MAX_ERROR_LEN = dask.config.get("distributed.admin.max-error-length")
     tblib.pickling_support.install(e, *collect_causes(e))
@@ -1150,7 +1150,7 @@ def clean_exception(exception, traceback, **kwargs):
 
     See Also
     --------
-    error_message: create and serialize errors into message
+    error_message : create and serialize errors into message
     """
     if isinstance(exception, bytes) or isinstance(exception, bytearray):
         try:
diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
index 8838bd13d32..93a40cfa0f3 100644
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -160,7 +160,7 @@ def scale(self, n: int) -> None:
 
         Parameters
         ----------
-        n: int
+        n : int
             Target number of workers
 
         Examples
diff --git a/distributed/deploy/ssh.py b/distributed/deploy/ssh.py
index 763fe779c3a..66003a49572 100644
--- a/distributed/deploy/ssh.py
+++ b/distributed/deploy/ssh.py
@@ -265,22 +265,22 @@ def SSHCluster(
 
     Parameters
     ----------
-    hosts: List[str]
+    hosts : List[str]
         List of hostnames or addresses on which to launch our cluster.
         The first will be used for the scheduler and the rest for workers.
-    connect_options: dict or list of dict, optional
+    connect_options : dict or list of dict, optional
         Keywords to pass through to :func:`asyncssh.connect`.
         This could include things such as ``port``, ``username``, ``password``
         or ``known_hosts``. See docs for :func:`asyncssh.connect` and
         :class:`asyncssh.SSHClientConnectionOptions` for full information.
         If a list it must have the same length as ``hosts``.
-    worker_options: dict, optional
+    worker_options : dict, optional
         Keywords to pass on to workers.
-    scheduler_options: dict, optional
+    scheduler_options : dict, optional
         Keywords to pass on to scheduler.
-    worker_module: str, optional
+    worker_module : str, optional
         Python module to call to start the worker.
-    remote_python: str or list of str, optional
+    remote_python : str or list of str, optional
         Path to Python on remote nodes.
 
     Examples
diff --git a/distributed/deploy/utils.py b/distributed/deploy/utils.py
index 4bfa32419bc..dad23f1eb8c 100644
--- a/distributed/deploy/utils.py
+++ b/distributed/deploy/utils.py
@@ -10,7 +10,7 @@ def nprocesses_nthreads(n=CPU_COUNT):
 
     Parameters
     ----------
-    n: int
+    n : int
         Number of available cores
 
     Examples
diff --git a/distributed/diagnostics/plugin.py b/distributed/diagnostics/plugin.py
index cdb811d71cc..1425b89b63d 100644
--- a/distributed/diagnostics/plugin.py
+++ b/distributed/diagnostics/plugin.py
@@ -67,13 +67,13 @@ def transition(self, key, start, finish, *args, **kwargs):
 
         Parameters
         ----------
-        key: string
-        start: string
+        key : string
+        start : string
             Start state of the transition.
             One of released, waiting, processing, memory, error.
-        finish: string
+        finish : string
             Final state of the transition.
-        *args, **kwargs: More options passed when transitioning
+        *args, **kwargs : More options passed when transitioning
             This may include worker ID, compute time, etc.
         """
 
@@ -144,13 +144,13 @@ def transition(self, key, start, finish, **kwargs):
 
         Parameters
         ----------
-        key: string
-        start: string
+        key : string
+        start : string
             Start state of the transition.
             One of waiting, ready, executing, long-running, memory, error.
-        finish: string
+        finish : string
             Final state of the transition.
-        kwargs: More options passed when transitioning
+        kwargs : More options passed when transitioning
         """
 
     def release_key(self, key, state, cause, reason, report):
@@ -159,15 +159,15 @@ def release_key(self, key, state, cause, reason, report):
 
         Parameters
         ----------
-        key: string
-        state: string
+        key : string
+        state : string
             State of the released task.
             One of waiting, ready, executing, long-running, memory, error.
-        cause: string or None
+        cause : string or None
             Additional information on what triggered the release of the task.
-        reason: None
+        reason : None
             Not used.
-        report: bool
+        report : bool
             Whether the worker should report the released task to the scheduler.
         """
 
@@ -177,11 +177,11 @@ def release_dep(self, dep, state, report):
 
         Parameters
         ----------
-        dep: string
-        state: string
+        dep : string
+        state : string
             State of the released dependency.
             One of waiting, flight, memory.
-        report: bool
+        report : bool
             Whether the worker should report the released dependency to the scheduler.
         """
 
diff --git a/distributed/diagnostics/progress_stream.py b/distributed/diagnostics/progress_stream.py
index f40da0495bc..b17c38fc1b6 100644
--- a/distributed/diagnostics/progress_stream.py
+++ b/distributed/diagnostics/progress_stream.py
@@ -57,7 +57,6 @@ async def progress_stream(address, interval):
 
 def progress_quads(msg, nrows=8, ncols=3):
     """
-
     >>> msg = {'all': {'inc': 5, 'dec': 1, 'add': 4},
     ...        'memory': {'inc': 2, 'dec': 0, 'add': 1},
     ...        'erred': {'inc': 0, 'dec': 1, 'add': 0},
diff --git a/distributed/diagnostics/progressbar.py b/distributed/diagnostics/progressbar.py
index 28aa135a649..45884bd5b1c 100644
--- a/distributed/diagnostics/progressbar.py
+++ b/distributed/diagnostics/progressbar.py
@@ -405,13 +405,13 @@ def progress(*futures, notebook=None, multi=True, complete=True, **kwargs):
 
     Parameters
     ----------
-    futures: Futures
+    futures : Futures
         A list of futures or keys to track
-    notebook: bool (optional)
+    notebook : bool (optional)
         Running in the notebook or not (defaults to guess)
-    multi: bool (optional)
+    multi : bool (optional)
         Track different functions independently (defaults to True)
-    complete: bool (optional)
+    complete : bool (optional)
         Track all keys (True) or only keys that have not yet run (False)
         (defaults to True)
 
diff --git a/distributed/diagnostics/websocket.py b/distributed/diagnostics/websocket.py
index f8704487c14..e34961bfeff 100644
--- a/distributed/diagnostics/websocket.py
+++ b/distributed/diagnostics/websocket.py
@@ -37,13 +37,13 @@ def transition(self, key, start, finish, *args, **kwargs):
 
         Parameters
         ----------
-        key: string
-        start: string
+        key : string
+        start : string
             Start state of the transition.
             One of released, waiting, processing, memory, error.
-        finish: string
+        finish : string
             Final state of the transition.
-        *args, **kwargs: More options passed when transitioning
+        *args, **kwargs : More options passed when transitioning
             This may include worker ID, compute time, etc.
         """
         if key not in self.scheduler.tasks:
diff --git a/distributed/diskutils.py b/distributed/diskutils.py
index 64124b753a8..e9dbb7b25bf 100644
--- a/distributed/diskutils.py
+++ b/distributed/diskutils.py
@@ -228,9 +228,9 @@ def new_work_dir(self, **kwargs):
 
         Parameters
         ----------
-        prefix: str (optional)
+        prefix : str (optional)
             The prefix of the temporary subdirectory name for the workdir
-        name: str (optional)
+        name : str (optional)
             The subdirectory name for the workdir
         """
         try:
diff --git a/distributed/preloading.py b/distributed/preloading.py
index fa5d19a0c1a..dbbe42d57d0 100644
--- a/distributed/preloading.py
+++ b/distributed/preloading.py
@@ -79,9 +79,9 @@ def _import_module(name, file_dir=None) -> ModuleType:
 
     Parameters
     ----------
-    name: str
+    name : str
         Module name, file path, or text of module or script
-    file_dir: string
+    file_dir : string
         Path of a directory where files should be copied
 
     Returns
diff --git a/distributed/profile.py b/distributed/profile.py
index 2b3363e6945..78e1765d041 100644
--- a/distributed/profile.py
+++ b/distributed/profile.py
@@ -284,16 +284,16 @@ def watch(
 
     Parameters
     ----------
-    thread_id: int
-    interval: str
+    thread_id : int
+    interval : str
         Time per sample
-    cycle: str
+    cycle : str
         Time per refreshing to a new profile state
-    maxlen: int
+    maxlen : int
         Passed onto deque, maximum number of periods
-    omit: str
+    omit : str
         Don't include entries that start with this filename
-    stop: callable
+    stop : callable
         Function to call to see if we should stop
 
     Returns
@@ -328,12 +328,12 @@ def get_profile(history, recent=None, start=None, stop=None, key=None):
 
     Parameters
     ----------
-    history: Sequence[Tuple[time, Dict]]
+    history : Sequence[Tuple[time, Dict]]
         A list or deque of profile states
-    recent: dict
+    recent : dict
         The most recent accumulating state
-    start: time
-    stop: time
+    start : time
+    stop : time
     """
     now = time()
     if start is None:
diff --git a/distributed/pubsub.py b/distributed/pubsub.py
index 3aeec084df1..3b580d6180a 100644
--- a/distributed/pubsub.py
+++ b/distributed/pubsub.py
@@ -433,7 +433,7 @@ def get(self, timeout=None):
 
         Parameters
         ----------
-        timeout: number or string or timedelta, optional
+        timeout : number or string or timedelta, optional
             Time in seconds to wait before timing out.
             Instead of number of seconds, it is also possible to specify
             a timedelta in string format, e.g. "200ms".
diff --git a/distributed/queues.py b/distributed/queues.py
index 15d6c8adca5..d022b010e1c 100644
--- a/distributed/queues.py
+++ b/distributed/queues.py
@@ -215,7 +215,7 @@ def put(self, value, timeout=None, **kwargs):
 
         Parameters
         ----------
-        timeout: number or string or timedelta, optional
+        timeout : number or string or timedelta, optional
             Time in seconds to wait before timing out.
             Instead of number of seconds, it is also possible to specify
             a timedelta in string format, e.g. "200ms".
@@ -228,11 +228,11 @@ def get(self, timeout=None, batch=False, **kwargs):
 
         Parameters
         ----------
-        timeout: number or string or timedelta, optional
+        timeout : number or string or timedelta, optional
             Time in seconds to wait before timing out.
             Instead of number of seconds, it is also possible to specify
             a timedelta in string format, e.g. "200ms".
-        batch: boolean, int (optional)
+        batch : boolean, int (optional)
             If True then return all elements currently waiting in the queue.
             If an integer than return that many elements from the queue
             If False (default) then return one item at a time
diff --git a/distributed/recreate_exceptions.py b/distributed/recreate_exceptions.py
index a046b8c846b..7e966f270a0 100644
--- a/distributed/recreate_exceptions.py
+++ b/distributed/recreate_exceptions.py
@@ -27,7 +27,7 @@ def cause_of_failure(self, *args, keys=(), **kwargs):
 
         Parameters
         ----------
-        keys: list of keys known to the scheduler
+        keys : list of keys known to the scheduler
 
         Returns
         -------
@@ -106,7 +106,6 @@ def get_futures_error(self, future):
 
         Returns
         -------
-
         Tuple:
         - the function that raised an exception
         - argument list (a tuple), may include values and keys
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 2f4526b483a..325d9cac940 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -5282,19 +5282,19 @@ def workers_to_close(
 
         Parameters
         ----------
-        memory_factor: Number
+        memory_factor : Number
             Amount of extra space we want to have for our stored data.
             Defaults two 2, or that we want to have twice as much memory as we
             currently have data.
-        n: int
+        n : int
             Number of workers to close
-        minimum: int
+        minimum : int
             Minimum number of workers to keep around
-        key: Callable(WorkerState)
+        key : Callable(WorkerState)
             An optional callable mapping a WorkerState object to a group
             affiliation.  Groups will be closed together.  This is useful when
             closing workers must be done collectively, such as by hostname.
-        target: int
+        target : int
             Target number of workers to have after we close
         attribute : str
             The attribute of the WorkerState object to return, like "address"
@@ -5760,7 +5760,7 @@ def run_function(self, stream, function, args=(), kwargs={}, wait=True):
 
         See Also
         --------
-        Client.run_on_scheduler:
+        Client.run_on_scheduler
         """
         from .worker import run
 
@@ -5856,7 +5856,7 @@ def _transition(self, key, finish: str, *args, **kwargs):
 
         See Also
         --------
-        Scheduler.transitions: transitive version of this function
+        Scheduler.transitions : transitive version of this function
         """
         parent: SchedulerState = cast(SchedulerState, self)
         ts: TaskState
@@ -6503,7 +6503,7 @@ def adaptive_target(self, comm=None, target_duration=None):
 
         Parameters
         ----------
-        target_duration: str
+        target_duration : str
             A desired duration of time for computations to take.  This affects
             how rapidly the scheduler will ask to scale.
 
diff --git a/distributed/stealing.py b/distributed/stealing.py
index 1e8428854ff..7a336777f26 100644
--- a/distributed/stealing.py
+++ b/distributed/stealing.py
@@ -110,7 +110,6 @@ def steal_time_ratio(self, ts):
 
         Returns
         -------
-
         cost_multiplier: The increased cost from moving this task as a factor.
         For example a result of zero implies a task without dependencies.
         level: The location within a stealable list to place this value
diff --git a/distributed/threadpoolexecutor.py b/distributed/threadpoolexecutor.py
index 57791809f60..cc4a9894823 100644
--- a/distributed/threadpoolexecutor.py
+++ b/distributed/threadpoolexecutor.py
@@ -110,7 +110,7 @@ def secede(adjust=True):
 
     See Also
     --------
-    rejoin: rejoin the thread pool
+    rejoin : rejoin the thread pool
     """
     thread_state.proceed = False
     with threads_lock:
@@ -127,7 +127,7 @@ def rejoin():
 
     See Also
     --------
-    secede: leave the thread pool
+    secede : leave the thread pool
     """
     thread = threading.current_thread()
     event = threading.Event()
diff --git a/distributed/utils.py b/distributed/utils.py
index 3565b4ceb9a..d7d4db6b2c5 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -884,7 +884,6 @@ def ensure_bytes(s):
 
     Examples
     --------
-
     >>> ensure_bytes('123')
     b'123'
     >>> ensure_bytes(b'123')
@@ -905,7 +904,6 @@ def ensure_bytes(s):
 
 def divide_n_among_bins(n, bins):
     """
-
     >>> divide_n_among_bins(12, [1, 1])
     [6, 6]
     >>> divide_n_among_bins(12, [1, 2])
@@ -1375,11 +1373,11 @@ def cli_keywords(d: dict, cls=None, cmd=None):
 
     Parameters
     ----------
-    d: dict
+    d : dict
         The keywords to convert
-    cls: callable
+    cls : callable
         The callable that consumes these terms to check them for validity
-    cmd: string or object
+    cmd : string or object
         A string with the name of a module, or the module containing a
         click-generated command with a "main" function, or the function itself.
         It may be used to parse a module's custom arguments (i.e., arguments that
@@ -1460,11 +1458,11 @@ def serialize_for_cli(data):
 
     Parameters
     ----------
-    data: json-serializable object
+    data : json-serializable object
         The data to serialize
     Returns
     -------
-    serialized_data: str
+    serialized_data : str
         The serialized data as a string
     """
     return base64.urlsafe_b64encode(json.dumps(data).encode()).decode()
@@ -1475,11 +1473,11 @@ def deserialize_for_cli(data):
 
     Parameters
     ----------
-    data: str
+    data : str
         String serialied by serialize_for_cli()
     Returns
     -------
-    deserialized_data: obj
+    deserialized_data : obj
         The de-serialized data
     """
     return json.loads(base64.urlsafe_b64decode(data.encode()).decode())
@@ -1523,7 +1521,6 @@ def __setitem__(self, key, value):
 
 def clean_dashboard_address(addr, default_listen_ip=""):
     """
-
     Examples
     --------
     >>> clean_dashboard_address(8787)
diff --git a/distributed/utils_comm.py b/distributed/utils_comm.py
index d1e6431d373..4175e0ee495 100644
--- a/distributed/utils_comm.py
+++ b/distributed/utils_comm.py
@@ -249,9 +249,9 @@ def pack_data(o, d, key_types=object):
 
     Parameters
     ----------
-    o:
+    o
         core data structures containing literals and keys
-    d: dict
+    d : dict
         mapping of keys to data
 
     Examples
@@ -284,9 +284,9 @@ def subs_multiple(o, d):
 
     Parameters
     ----------
-    o:
+    o
         Core data structures containing literals and keys
-    d: dict
+    d : dict
         Mapping of keys to values
 
     Examples
diff --git a/distributed/variable.py b/distributed/variable.py
index db8da76e44c..19fbd2bb031 100644
--- a/distributed/variable.py
+++ b/distributed/variable.py
@@ -181,7 +181,7 @@ def set(self, value, **kwargs):
 
         Parameters
         ----------
-        value: Future or object
+        value : Future or object
             Must be either a Future or a msgpack-encodable value
         """
         return self.client.sync(self._set, value, **kwargs)
@@ -211,7 +211,7 @@ def get(self, timeout=None, **kwargs):
 
         Parameters
         ----------
-        timeout: number or string or timedelta, optional
+        timeout : number or string or timedelta, optional
             Time in seconds to wait before timing out.
             Instead of number of seconds, it is also possible to specify
             a timedelta in string format, e.g. "200ms".
diff --git a/distributed/worker_client.py b/distributed/worker_client.py
index 1c4e1378905..2f6eec73c62 100644
--- a/distributed/worker_client.py
+++ b/distributed/worker_client.py
@@ -17,10 +17,10 @@ def worker_client(timeout=None, separate_thread=True):
 
     Parameters
     ----------
-    timeout: Number or String
+    timeout : Number or String
         Timeout after which to error out. Defaults to the
         ``distributed.comm.timeouts.connect`` configuration value.
-    separate_thread: bool, optional
+    separate_thread : bool, optional
         Whether to run this function outside of the normal thread pool
         defaults to True
 

From a0d60e7b2ad16304128c7989a111539a4d80ed38 Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Thu, 4 Feb 2021 17:35:30 +0100
Subject: [PATCH 1145/1550] Update TaskState documentation about dependents
 attribute (#4440)

---
 distributed/worker.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/distributed/worker.py b/distributed/worker.py
index 4cf60b597e9..7e282c2e46a 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -104,7 +104,10 @@ class TaskState:
     * **dependencies**: ``set(TaskState instances)``
         The data needed by this key to run
     * **dependents**: ``set(TaskState instances)``
-        The keys that use this dependency
+        The keys that use this dependency. Only keys which are not available
+        already are tracked in this structure and dependents made available are
+        actively removed. Only after all dependents have been removed, this task
+        is allowed to be forgotten
     * **duration**: ``float``
         Expected duration the a task
     * **priority**: ``tuple``

From 5586ac18ffd9fc9e149c39a1e3fe822b31647e81 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Fri, 5 Feb 2021 13:31:58 -0600
Subject: [PATCH 1146/1550] Bump minimum Dask to 2021.02.0 (#4486)

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 91a18f6346b..c79072fb88d 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,7 +1,7 @@
 click >= 6.6
 cloudpickle >= 1.5.0
 contextvars;python_version<'3.7'
-dask>=2020.12.0
+dask>=2021.02.0
 msgpack >= 0.6.0
 psutil >= 5.0
 sortedcontainers !=2.0.0, !=2.0.1

From cbb88bd43288a7e25a03e11f17e1c9a2f5e6f31d Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Fri, 5 Feb 2021 13:39:20 -0600
Subject: [PATCH 1147/1550] bump version to 2021.02.0

---
 docs/source/changelog.rst | 25 ++++++++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index 0fbc53cd3f2..a3c04b1bdd7 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,28 @@
 Changelog
 =========
 
+2021.02.0
+---------
+
+Released on February 5, 2021
+
+- Bump minimum Dask to 2021.02.0 (:pr:`4486`) `James Bourbeau`_
+- Update ``TaskState`` documentation about dependents attribute (:pr:`4440`) `Florian Jetter`_
+- DOC: Autoreformat all functions docstrings (:pr:`4475`) `Matthias Bussonnier`_
+- Use cached version of ``is_coroutine_function`` in stream handling to (:pr:`4481`) `Ian Rose`_
+- Optimize ``transitions`` (:pr:`4451`) `jakirkham`_
+- Create ``PULL_REQUEST_TEMPLATE.md`` (:pr:`4476`) `Ray Bell`_
+- DOC: typo, directives ends with 2 colons ``::`` (:pr:`4472`) `Matthias Bussonnier`_
+- DOC: Proper numpydoc syntax for ``distributed/protocol/*.py`` (:pr:`4473`) `Matthias Bussonnier`_
+- Update ``pytest.skip`` usage in ``test_server_listen`` (:pr:`4467`) `James Bourbeau`_
+- Unify annotations (:pr:`4406`) `Ian Rose`_
+- Added worker resources from config (:pr:`4456`) `Tom Augspurger`_
+- Fix var name in worker validation func (:pr:`4457`) `Gil Forsyth`_
+- Refactor ``task_groups`` & ``task_prefixes`` (:pr:`4452`) `jakirkham`_
+- Use ``parent._tasks`` in ``heartbeat`` (:pr:`4450`) `jakirkham`_
+- Refactor ``SchedulerState`` from ``Scheduler`` (:pr:`4365`) `jakirkham`_
+
+
 2021.01.1
 ---------
 
@@ -2169,4 +2191,5 @@ significantly without many new features.
 .. _`marwan116`: https://github.com/marwan116
 .. _`Bernhard M. Wiedemann`: https://github.com/bmwiedemann
 .. _`Bruno Pagani`: https://github.com/ArchangeGabriel
-.. _`selshowk`: https://github.com/selshowk
\ No newline at end of file
+.. _`selshowk`: https://github.com/selshowk
+.. _`Ray Bell`: https://github.com/raybellwaves
\ No newline at end of file

From 7d2a22f94332974825e599dd779b4d98f44dbca2 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 5 Feb 2021 11:42:11 -0800
Subject: [PATCH 1148/1550] Use a callback to close TCP Comms, rather than
 check every time (#4453)

In a recent trace this relieves about 30ms of a 3s shuffle computation
resulting in around a 1% overall speedup
---
 distributed/comm/tcp.py | 34 +++++++++++++++++++++-------------
 1 file changed, 21 insertions(+), 13 deletions(-)

diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py
index b9721b99546..ff283dcf3f3 100644
--- a/distributed/comm/tcp.py
+++ b/distributed/comm/tcp.py
@@ -1,4 +1,5 @@
 import errno
+import functools
 import logging
 import socket
 from ssl import SSLError
@@ -126,8 +127,16 @@ def convert_stream_closed_error(obj, exc):
         raise CommClosedError("in %s: %s" % (obj, exc)) from exc
 
 
-def _do_nothing():
-    pass
+def _close_comm(ref):
+    """Callback to close Dask Comm when Tornado Stream closes
+
+    Parameters
+    ----------
+        ref: weak reference to a Dask comm
+    """
+    comm = ref()
+    if comm:
+        comm._closed = True
 
 
 class TCP(Comm):
@@ -136,6 +145,7 @@ class TCP(Comm):
     """
 
     def __init__(self, stream, local_addr, peer_addr, deserialize=True):
+        self._closed = False
         Comm.__init__(self)
         self._local_addr = local_addr
         self._peer_addr = peer_addr
@@ -145,18 +155,12 @@ def __init__(self, stream, local_addr, peer_addr, deserialize=True):
         self._finalizer.atexit = False
         self._extra = {}
 
+        ref = weakref.ref(self)
+
+        stream.set_close_callback(functools.partial(_close_comm, ref))
+
         stream.set_nodelay(True)
         set_tcp_timeout(stream)
-        # set a close callback, to make `self.stream.closed()` more reliable.
-        # Background: if `stream` is unused (e.g. because it's in `ConnectionPool.available`),
-        # the underlying fd is not watched for changes. In this case, even if the
-        # connection is actively closed by the remote end, `self.closed()` would still return `False`.
-        # Registering a closed callback will make tornado register the underlying fd
-        # for changes, and this would be reflected in `self.closed()` even without reading/writing.
-        #
-        # Use a global method (instead of a lambda) to avoid creating a reference
-        # to the local scope.
-        stream.set_close_callback(_do_nothing)
         self._read_extra()
 
     def _read_extra(self):
@@ -198,6 +202,7 @@ async def read(self, deserializers=None):
                 frames.append(frame)
         except StreamClosedError as e:
             self.stream = None
+            self._closed = True
             if not shutting_down():
                 convert_stream_closed_error(self, e)
         except Exception:
@@ -261,6 +266,7 @@ async def write(self, msg, serializers=None, on_error="message"):
                         bytes_since_last_yield = 0
         except StreamClosedError as e:
             self.stream = None
+            self._closed = True
             if not shutting_down():
                 convert_stream_closed_error(self, e)
         except Exception:
@@ -281,6 +287,7 @@ def close(self):
         # Task was destroyed but it is pending!
         # Triggered by distributed.deploy.tests.test_local::test_silent_startup
         stream, self.stream = self.stream, None
+        self._closed = True
         if stream is not None and not stream.closed():
             try:
                 # Flush the stream's write buffer by waiting for a last write.
@@ -295,12 +302,13 @@ def close(self):
 
     def abort(self):
         stream, self.stream = self.stream, None
+        self._closed = True
         if stream is not None and not stream.closed():
             self._finalizer.detach()
             stream.close()
 
     def closed(self):
-        return self.stream is None or self.stream.closed()
+        return self._closed
 
     @property
     def extra_info(self):

From 2b9ba97a1692e9cb04a8571aa8896911b1748592 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Tue, 9 Feb 2021 19:18:25 -0800
Subject: [PATCH 1149/1550] Use `main` branch for zict (#4499)

This was recently changed from `master`. Though it appears `pip` uses
`master` by default. So code it as `main` where it is not specified and
fix the branch name where `master` is still used.
---
 continuous_integration/environment-windows.yml | 2 +-
 continuous_integration/travis/install.sh       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/continuous_integration/environment-windows.yml b/continuous_integration/environment-windows.yml
index 9b05bd0d0ff..5dd3043ccdc 100644
--- a/continuous_integration/environment-windows.yml
+++ b/continuous_integration/environment-windows.yml
@@ -32,4 +32,4 @@ dependencies:
   - pip:
       - git+https://github.com/dask/dask.git@master
       - git+https://github.com/joblib/joblib.git@master
-      - git+https://github.com/dask/zict.git@master
+      - git+https://github.com/dask/zict.git@main
diff --git a/continuous_integration/travis/install.sh b/continuous_integration/travis/install.sh
index d7a54093ce0..223c32ddbaa 100644
--- a/continuous_integration/travis/install.sh
+++ b/continuous_integration/travis/install.sh
@@ -91,7 +91,7 @@ python -m pip install -q git+https://github.com/dask/dask.git --upgrade --no-dep
 python -m pip install -q git+https://github.com/joblib/joblib.git --upgrade --no-deps
 python -m pip install -q git+https://github.com/intake/filesystem_spec.git --upgrade --no-deps
 python -m pip install -q git+https://github.com/dask/s3fs.git --upgrade --no-deps
-python -m pip install -q git+https://github.com/dask/zict.git --upgrade --no-deps
+python -m pip install -q git+https://github.com/dask/zict.git@main --upgrade --no-deps
 python -m pip install -q keras --upgrade --no-deps
 
 if [[ $CRICK == true ]]; then

From 725f001996712ec291248ad3011567e13cac46db Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Wed, 10 Feb 2021 19:45:32 -0600
Subject: [PATCH 1150/1550] Attempt to get client from worker in Queue and
 Variable (#4490)

---
 distributed/queues.py              | 12 ++++++++----
 distributed/tests/test_queues.py   | 21 ++++++++++++++++++++-
 distributed/tests/test_variable.py | 22 ++++++++++++++++++++--
 distributed/variable.py            | 14 +++++++++-----
 4 files changed, 57 insertions(+), 12 deletions(-)

diff --git a/distributed/queues.py b/distributed/queues.py
index d022b010e1c..e368d329d03 100644
--- a/distributed/queues.py
+++ b/distributed/queues.py
@@ -7,7 +7,7 @@
 
 from .client import Future, Client
 from .utils import sync, thread_state
-from .worker import get_client
+from .worker import get_client, get_worker
 from .utils import parse_timedelta
 
 logger = logging.getLogger(__name__)
@@ -150,8 +150,8 @@ class Queue:
         Name used by other clients and the scheduler to identify the queue. If
         not given, a random name will be generated.
     client: Client (optional)
-        Client used for communication with the scheduler. Defaults to the
-        value of ``Client.current()``.
+        Client used for communication with the scheduler.
+        If not given, the default global client will be used.
     maxsize: int (optional)
         Number of items allowed in the queue. If 0 (the default), the queue
         size is unbounded.
@@ -170,7 +170,11 @@ class Queue:
     """
 
     def __init__(self, name=None, client=None, maxsize=0):
-        self.client = client or Client.current()
+        try:
+            self.client = client or Client.current()
+        except ValueError:
+            # Initialise new client
+            self.client = get_worker().client
         self.name = name or "queue-" + uuid.uuid4().hex
         self._event_started = asyncio.Event()
         if self.client.asynchronous or getattr(
diff --git a/distributed/tests/test_queues.py b/distributed/tests/test_queues.py
index 94d80c9dbcf..8f400498854 100644
--- a/distributed/tests/test_queues.py
+++ b/distributed/tests/test_queues.py
@@ -6,7 +6,7 @@
 
 from distributed import Client, Queue, Nanny, worker_client, wait, TimeoutError
 from distributed.metrics import time
-from distributed.utils_test import gen_cluster, inc, div
+from distributed.utils_test import gen_cluster, inc, div, popen
 from distributed.utils_test import client, cluster_fixture, loop  # noqa: F401
 
 
@@ -276,3 +276,22 @@ def get():
     res = c.submit(get)
 
     await c.gather([res, fut])
+
+
+def test_queue_in_task(loop):
+    # Ensure that we can create a Queue inside a task on a
+    # worker in a separate Python process than the client
+    with popen(["dask-scheduler", "--no-dashboard"]):
+        with popen(["dask-worker", "127.0.0.1:8786"]):
+            with Client("tcp://127.0.0.1:8786", loop=loop) as c:
+                c.wait_for_workers(1)
+
+                x = Queue("x")
+                x.put(123)
+
+                def foo():
+                    y = Queue("x")
+                    return y.get()
+
+                result = c.submit(foo).result()
+                assert result == 123
diff --git a/distributed/tests/test_variable.py b/distributed/tests/test_variable.py
index 5d9ece6ee54..37b3c756be7 100644
--- a/distributed/tests/test_variable.py
+++ b/distributed/tests/test_variable.py
@@ -10,9 +10,8 @@
 from distributed import Client, Variable, worker_client, Nanny, wait, TimeoutError
 from distributed.metrics import time
 from distributed.compatibility import WINDOWS
-from distributed.utils_test import gen_cluster, inc, div
+from distributed.utils_test import gen_cluster, inc, div, captured_logger, popen
 from distributed.utils_test import client, cluster_fixture, loop  # noqa: F401
-from distributed.utils_test import captured_logger
 
 
 @gen_cluster(client=True)
@@ -40,6 +39,25 @@ async def test_variable(c, s, a, b):
         assert time() < start + 5
 
 
+def test_variable_in_task(loop):
+    # Ensure that we can create a Variable inside a task on a
+    # worker in a separate Python process than the client
+    with popen(["dask-scheduler", "--no-dashboard"]):
+        with popen(["dask-worker", "127.0.0.1:8786"]):
+            with Client("tcp://127.0.0.1:8786", loop=loop) as c:
+                c.wait_for_workers(1)
+
+                x = Variable("x")
+                x.set(123)
+
+                def foo():
+                    y = Variable("x")
+                    return y.get()
+
+                result = c.submit(foo).result()
+                assert result == 123
+
+
 @gen_cluster(client=True)
 async def test_delete_unset_variable(c, s, a, b):
     x = Variable()
diff --git a/distributed/variable.py b/distributed/variable.py
index 19fbd2bb031..c3fdc94d0d7 100644
--- a/distributed/variable.py
+++ b/distributed/variable.py
@@ -9,13 +9,13 @@
 from dask.utils import stringify
 from .client import Future, Client
 from .utils import log_errors, TimeoutError, parse_timedelta
-from .worker import get_client
+from .worker import get_client, get_worker
 
 logger = logging.getLogger(__name__)
 
 
 class VariableExtension:
-    """An extension for the scheduler to manage queues
+    """An extension for the scheduler to manage Variables
 
     This adds the following routes to the scheduler
 
@@ -145,8 +145,8 @@ class Variable:
         Name used by other clients and the scheduler to identify the variable.
         If not given, a random name will be generated.
     client: Client (optional)
-        Client used for communication with the scheduler. Defaults to the
-        value of ``Client.current()``.
+        Client used for communication with the scheduler.
+        If not given, the default global client will be used.
 
     Examples
     --------
@@ -165,7 +165,11 @@ class Variable:
     """
 
     def __init__(self, name=None, client=None, maxsize=0):
-        self.client = client or Client.current()
+        try:
+            self.client = client or Client.current()
+        except ValueError:
+            # Initialise new client
+            self.client = get_worker().client
         self.name = name or "variable-" + uuid.uuid4().hex
 
     async def _set(self, value):

From de7cf0a58c177d10beb84b0851fd1f8865eb4fc3 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Thu, 11 Feb 2021 08:57:14 -0800
Subject: [PATCH 1151/1550] `transition_processing_memory` optimizations, etc.
 (#4487)

* Use `.get` to retrieve alias

Avoids checking for the value and then retrieving it by simply trying to
get the associated value. If it is not found, it is `None`, which is
fast to check after and handle.

* Assign `WorkerState` object

Allows Cython to optimize operations on this object.

* Use `_address` attribute in `host`

This should allow faster access (particularly in Cython).

* Drop `bandwidth` intermediate variable

This is a holdover from before the `SchedulerState` refactor. Now this
is properly typed and can be used efficiently in Cython anyways.

* Drop `total_nthreads` intermediate variable

* Drop `total_occupancy` intermediate variable

* Drop `bandwidth` intermediate variable

* Annotate `s` as a `set`

* Drop extra `client_msgs` definitions

* Define `recommendations` up top

* Annotate variables involved in `startstops`

* Assign `compute_start` and `compute_stop` `0`

If we don't have `startstops`, just assign `compute_start` and
`compute_stop` `0` instead of `None`.

* Type `compute_start` & `compute_stop` as `double`

* Skip collecting unused startstop list, `L`

We only ever grab the first value from this `list`. The rest of the
values are ignored. So simply alter the loop to collect this first match
and skip collecting the rest of the `list`. Should speed this code up
considerably.

* Type durations as `double`

Should speed up computations using these.

* Type various durations

* Use temporary variables to group some work

* Type `set` iterated over

* Just check `_processing_on is not None`

We already know this is a `WorkerState` object or `None`. Also there is
not really a concept of a `False`y `WorkerState`. So the only way this
expression is `False` is if it is `None`. To speed up the check and
avoid unnecessary work. Just check that it `is not None`, which is a
fast pointer check.

* Swap `comm` & `old` order

* Relax `startstops` typing

Apparently this can be a `tuple` in some cases.

* Type `duration` as `double`

* Simplify branching in `_remove_from_processing`

* Fuse `for`-loops in `_client_releases_keys`

Instead of collecting `TaskState`s for special handling in another loop,
go ahead and handle them as they come up within the check where they
were previously collected. Avoids a second loop and build up another
collection for it.

* Use `.get` in `handle_missing_data`

Avoid checking for the key and then getting it. Instead just use `.get`
knowing it will return `None` if the key is not found. Then handle the
case where `None` is returned as the missing key case.

* Annotate variables in `release_worker_data`

Should allow the more efficient Python C APIs to be used.

* Use `.get` to grab stealing extension

* Assign `double` to variable

Avoids duplicate retrieval. Also handles the coercion to `double` once.

* Use `parent` to get stealing extension quickly

* Move `ClientState` annotation into `else`

* Use `dict` comprehension in `_task_to_client_msgs`

* Create client msgs from keys directly

Avoid collecting an intermediate list of client keys and instead use
them to produce messages directly.

* Annotate couple variables in attribute functions

* Type `duration` as `double` w/`-1` default

This makes it easier to type this value and thus for Cython to optimize
this value throughout.

* Tidy up `new_task`

* Annotate arguments in `new_task`

Should knock out type checking when the function is called, which should
simplify the work needed in later steps.

* Use `None` with `pop` to defer object creation

This allows us to only create the `set` `s` when we know we need it.
Otherwise we just get the `set` previously contained in the `dict`.

* Just check `s` is non-trivial

Instead of creating an empty `set` when `s` is not defined, just check
that `s` is non-trivial (either `None` or empty). This should be a bit
faster and avoid the unnecessary creation step.

* Make `_unknown_durations` an ordinary `dict`

As `defaultdict`s are difficult for Cython to optimize and usually a
`dict` will suffice, change `_unknown_duractions` to a `dict` to allow
Cython to use Python C API specific to `dict`s.

* Assign messages to `client_msgs` for clarity

* Annotate `for`-loops in `valid_workers` for perf

* Use `dict`s for `_host_info` & `_resources`

This makes it easier to annotate these more accurately and benefit from
Cython's optimizations around these `dict`-typed variables. Requires a
very small amount of checking for keys and setting their values if not
present. Though this remains efficient in both Python & Cython as well
as compact.

* Annotate `"addresses"` field as `set`

* Assign to `dw` after `sw` is filled

* Drop some unneeded intermediate variables

These are annotated local variables, which are holdovers from before the
`SchedulerState` refactor. We can now drop these and use
`SchedulerState` to access these directly.

* Use `wws` for a looping variable

* Assign `ws` `None` and `return`

* Assign `ws` result and `return`

* Join `if`s and cleanup spacing

* Get `len(candidates)` and branch based on that

Cython will turn this into a very efficient `switch...case` statement.
So this cuts down on the overhead of comparisons and checks (paying them
once when computing the length). Then focuses on just checking this C
typed integral value for which `case` to run.
---
 distributed/scheduler.py | 370 +++++++++++++++++++++------------------
 1 file changed, 199 insertions(+), 171 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 325d9cac940..c13f5292538 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -483,7 +483,7 @@ def has_what(self):
 
     @property
     def host(self):
-        return get_address_host(self.address)
+        return get_address_host(self._address)
 
     @property
     def last_seen(self):
@@ -1518,7 +1518,7 @@ def _legacy_worker_key_set(workers):
     return {ws._address for ws in workers}
 
 
-def _legacy_task_key_dict(task_dict):
+def _legacy_task_key_dict(task_dict: dict):
     """
     Transform a dict of {task state: value} into a dict of {task key: value}.
     """
@@ -1526,8 +1526,8 @@ def _legacy_task_key_dict(task_dict):
     return {ts._key: value for ts, value in task_dict.items()}
 
 
-def _task_key_or_none(task):
-    return task.key if task is not None else None
+def _task_key_or_none(task: TaskState):
+    return task._key if task is not None else None
 
 
 @cclass
@@ -1581,11 +1581,11 @@ class SchedulerState:
     _bandwidth: double
     _clients: dict
     _extensions: dict
-    _host_info: object
+    _host_info: dict
     _idle: object
     _idle_dv: dict
     _n_tasks: Py_ssize_t
-    _resources: object
+    _resources: dict
     _saturated: set
     _tasks: dict
     _task_groups: dict
@@ -1593,7 +1593,7 @@ class SchedulerState:
     _task_metadata: dict
     _total_nthreads: Py_ssize_t
     _total_occupancy: double
-    _unknown_durations: object
+    _unknown_durations: dict
     _unrunnable: set
     _validate: bint
     _workers: object
@@ -1627,14 +1627,14 @@ def __init__(
         if host_info is not None:
             self._host_info = host_info
         else:
-            self._host_info = defaultdict(dict)
+            self._host_info = dict()
         self._idle = sortedcontainers.SortedDict()
         self._idle_dv: dict = cast(dict, self._idle)
         self._n_tasks = 0
         if resources is not None:
             self._resources = resources
         else:
-            self._resources = defaultdict(dict)
+            self._resources = dict()
         self._saturated = set()
         if tasks is not None:
             self._tasks = tasks
@@ -1645,7 +1645,7 @@ def __init__(
         self._task_metadata = dict()
         self._total_nthreads = 0
         self._total_occupancy = 0
-        self._unknown_durations = defaultdict(set)
+        self._unknown_durations = dict()
         if unrunnable is not None:
             self._unrunnable = unrunnable
         else:
@@ -1768,6 +1768,7 @@ def transition_released_waiting(self, key):
         try:
             ts: TaskState = self._tasks[key]
             dts: TaskState
+            recommendations: dict = {}
             worker_msgs: dict = {}
             client_msgs: dict = {}
 
@@ -1779,12 +1780,11 @@ def transition_released_waiting(self, key):
                 assert not any([dts._state == "forgotten" for dts in ts._dependencies])
 
             if ts._has_lost_dependencies:
-                return {key: "forgotten"}, worker_msgs, client_msgs
+                recommendations[key] = "forgotten"
+                return recommendations, worker_msgs, client_msgs
 
             ts.state = "waiting"
 
-            recommendations: dict = {}
-
             dts: TaskState
             for dts in ts._dependencies:
                 if dts._exception_blame:
@@ -1823,6 +1823,7 @@ def transition_no_worker_waiting(self, key):
         try:
             ts: TaskState = self._tasks[key]
             dts: TaskState
+            recommendations: dict = {}
             worker_msgs: dict = {}
             client_msgs: dict = {}
 
@@ -1835,9 +1836,8 @@ def transition_no_worker_waiting(self, key):
             self._unrunnable.remove(ts)
 
             if ts._has_lost_dependencies:
-                return {key: "forgotten"}, worker_msgs, client_msgs
-
-            recommendations: dict = {}
+                recommendations[key] = "forgotten"
+                return recommendations, worker_msgs, client_msgs
 
             for dts in ts._dependencies:
                 dep = dts._key
@@ -1933,6 +1933,7 @@ def transition_waiting_processing(self, key):
         try:
             ts: TaskState = self._tasks[key]
             dts: TaskState
+            recommendations: dict = {}
             worker_msgs: dict = {}
             client_msgs: dict = {}
 
@@ -1947,7 +1948,7 @@ def transition_waiting_processing(self, key):
 
             ws: WorkerState = self.decide_worker(ts)
             if ws is None:
-                return {}, worker_msgs, client_msgs
+                return recommendations, worker_msgs, client_msgs
             worker = ws._address
 
             duration_estimate = self.set_duration_estimate(ts, ws)
@@ -1966,7 +1967,7 @@ def transition_waiting_processing(self, key):
 
             worker_msgs[worker] = [_task_to_msg(self, ts)]
 
-            return {}, worker_msgs, client_msgs
+            return recommendations, worker_msgs, client_msgs
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
@@ -1981,6 +1982,7 @@ def transition_waiting_memory(
         try:
             ws: WorkerState = self._workers_dv[worker]
             ts: TaskState = self._tasks[key]
+            recommendations: dict = {}
             worker_msgs: dict = {}
             client_msgs: dict = {}
 
@@ -1996,9 +1998,6 @@ def transition_waiting_memory(
 
             self.check_idle_saturated(ws)
 
-            recommendations: dict = {}
-            client_msgs: dict = {}
-
             _add_to_memory(
                 self, ts, ws, recommendations, client_msgs, type=type, typename=typename
             )
@@ -2029,6 +2028,7 @@ def transition_processing_memory(
     ):
         ws: WorkerState
         wws: WorkerState
+        recommendations: dict = {}
         worker_msgs: dict = {}
         client_msgs: dict = {}
         try:
@@ -2047,7 +2047,8 @@ def transition_processing_memory(
 
             ws = self._workers_dv.get(worker)
             if ws is None:
-                return {key: "released"}, worker_msgs, client_msgs
+                recommendations[key] = "released"
+                return recommendations, worker_msgs, client_msgs
 
             if ws != ts._processing_on:  # someone else has this task
                 logger.info(
@@ -2057,35 +2058,34 @@ def transition_processing_memory(
                     ws,
                     key,
                 )
-                return {}, worker_msgs, client_msgs
+                return recommendations, worker_msgs, client_msgs
 
+            has_compute_startstop: bool = False
+            compute_start: double
+            compute_stop: double
             if startstops:
-                L = list()
+                startstop: dict
                 for startstop in startstops:
                     stop = startstop["stop"]
                     start = startstop["start"]
                     action = startstop["action"]
-                    if action == "compute":
-                        L.append((start, stop))
+                    if not has_compute_startstop and action == "compute":
+                        compute_start = start
+                        compute_stop = stop
+                        has_compute_startstop = True
 
                     # record timings of all actions -- a cheaper way of
                     # getting timing info compared with get_task_stream()
                     ts._prefix._all_durations[action] += stop - start
 
-                if len(L) > 0:
-                    compute_start, compute_stop = L[0]
-                else:  # This is very rare
-                    compute_start = compute_stop = None
-            else:
-                compute_start = compute_stop = None
-
             #############################
             # Update Timing Information #
             #############################
-            if compute_start and ws._processing.get(ts, True):
+            if has_compute_startstop and ws._processing.get(ts, True):
                 # Update average task duration for worker
-                old_duration = ts._prefix._duration_average
-                new_duration = compute_stop - compute_start
+                old_duration: double = ts._prefix._duration_average
+                new_duration: double = compute_stop - compute_start
+                avg_duration: double
                 if old_duration < 0:
                     avg_duration = new_duration
                 else:
@@ -2094,15 +2094,19 @@ def transition_processing_memory(
                 ts._prefix._duration_average = avg_duration
                 ts._group._duration += new_duration
 
+                s: set = self._unknown_durations.pop(ts._prefix._name, None)
                 tts: TaskState
-                for tts in self._unknown_durations.pop(ts._prefix._name, ()):
-                    if tts._processing_on:
-                        wws = tts._processing_on
-                        old = wws._processing[tts]
-                        comm = self.get_comm_cost(tts, wws)
-                        wws._processing[tts] = avg_duration + comm
-                        wws._occupancy += avg_duration + comm - old
-                        self._total_occupancy += avg_duration + comm - old
+                if s:
+                    for tts in s:
+                        if tts._processing_on is not None:
+                            wws = tts._processing_on
+                            comm: double = self.get_comm_cost(tts, wws)
+                            old: double = wws._processing[tts]
+                            new: double = avg_duration + comm
+                            diff: double = new - old
+                            wws._processing[tts] = new
+                            wws._occupancy += diff
+                            self._total_occupancy += diff
 
             ############################
             # Update State Information #
@@ -2110,9 +2114,6 @@ def transition_processing_memory(
             if nbytes is not None:
                 ts.set_nbytes(nbytes)
 
-            recommendations: dict = {}
-            client_msgs: dict = {}
-
             _remove_from_processing(self, ts)
 
             _add_to_memory(
@@ -2137,6 +2138,7 @@ def transition_memory_released(self, key, safe: bint = False):
         try:
             ts: TaskState = self._tasks[key]
             dts: TaskState
+            recommendations: dict = {}
             worker_msgs: dict = {}
             client_msgs: dict = {}
 
@@ -2152,14 +2154,13 @@ def transition_memory_released(self, key, safe: bint = False):
                 if ts._who_wants:
                     ts._exception_blame = ts
                     ts._exception = "Worker holding Actor was lost"
+                    recommendations[ts._key] = "erred"
                     return (
-                        {ts._key: "erred"},
+                        recommendations,
                         worker_msgs,
                         client_msgs,
                     )  # don't try to recreate
 
-            recommendations: dict = {}
-
             for dts in ts._waiters:
                 if dts._state in ("no-worker", "processing"):
                     recommendations[dts._key] = "waiting"
@@ -2212,6 +2213,7 @@ def transition_released_erred(self, key):
             ts: TaskState = self._tasks[key]
             dts: TaskState
             failing_ts: TaskState
+            recommendations: dict = {}
             worker_msgs: dict = {}
             client_msgs: dict = {}
 
@@ -2222,8 +2224,6 @@ def transition_released_erred(self, key):
                     assert not ts._waiting_on
                     assert not ts._waiters
 
-            recommendations: dict = {}
-
             failing_ts = ts._exception_blame
 
             for dts in ts._dependents:
@@ -2257,6 +2257,7 @@ def transition_erred_released(self, key):
         try:
             ts: TaskState = self._tasks[key]
             dts: TaskState
+            recommendations: dict = {}
             worker_msgs: dict = {}
             client_msgs: dict = {}
 
@@ -2268,8 +2269,6 @@ def transition_erred_released(self, key):
                     assert not ts._waiting_on
                     assert not ts._waiters
 
-            recommendations: dict = {}
-
             ts._exception = None
             ts._exception_blame = None
             ts._traceback = None
@@ -2297,6 +2296,7 @@ def transition_erred_released(self, key):
     def transition_waiting_released(self, key):
         try:
             ts: TaskState = self._tasks[key]
+            recommendations: dict = {}
             worker_msgs: dict = {}
             client_msgs: dict = {}
 
@@ -2304,11 +2304,9 @@ def transition_waiting_released(self, key):
                 assert not ts._who_has
                 assert not ts._processing_on
 
-            recommendations: dict = {}
-
             dts: TaskState
             for dts in ts._dependencies:
-                s = dts._waiters
+                s: set = dts._waiters
                 if ts in s:
                     s.discard(ts)
                     if not s and not dts._who_wants:
@@ -2337,6 +2335,7 @@ def transition_processing_released(self, key):
         try:
             ts: TaskState = self._tasks[key]
             dts: TaskState
+            recommendations: dict = {}
             worker_msgs: dict = {}
             client_msgs: dict = {}
 
@@ -2352,8 +2351,6 @@ def transition_processing_released(self, key):
 
             ts.state = "released"
 
-            recommendations: dict = {}
-
             if ts._has_lost_dependencies:
                 recommendations[key] = "forgotten"
             elif ts._waiters or ts._who_wants:
@@ -2362,7 +2359,7 @@ def transition_processing_released(self, key):
             if recommendations.get(key) != "waiting":
                 for dts in ts._dependencies:
                     if dts._state != "released":
-                        s = dts._waiters
+                        s: set = dts._waiters
                         s.discard(ts)
                         if not s and not dts._who_wants:
                             recommendations[dts._key] = "released"
@@ -2388,6 +2385,7 @@ def transition_processing_erred(
             ts: TaskState = self._tasks[key]
             dts: TaskState
             failing_ts: TaskState
+            recommendations: dict = {}
             worker_msgs: dict = {}
             client_msgs: dict = {}
 
@@ -2413,14 +2411,12 @@ def transition_processing_erred(
             else:
                 failing_ts = ts._exception_blame
 
-            recommendations: dict = {}
-
             for dts in ts._dependents:
                 dts._exception_blame = failing_ts
                 recommendations[dts._key] = "erred"
 
             for dts in ts._dependencies:
-                s = dts._waiters
+                s: set = dts._waiters
                 s.discard(ts)
                 if not s and not dts._who_wants:
                     recommendations[dts._key] = "released"
@@ -2464,6 +2460,7 @@ def transition_no_worker_released(self, key):
         try:
             ts: TaskState = self._tasks[key]
             dts: TaskState
+            recommendations: dict = {}
             worker_msgs: dict = {}
             client_msgs: dict = {}
 
@@ -2480,7 +2477,7 @@ def transition_no_worker_released(self, key):
 
             ts._waiters.clear()
 
-            return {}, worker_msgs, client_msgs
+            return recommendations, worker_msgs, client_msgs
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
@@ -2506,6 +2503,7 @@ def transition_memory_forgotten(self, key):
         ws: WorkerState
         try:
             ts: TaskState = self._tasks[key]
+            recommendations: dict = {}
             worker_msgs: dict = {}
             client_msgs: dict = {}
 
@@ -2525,8 +2523,6 @@ def transition_memory_forgotten(self, key):
                 else:
                     assert 0, (ts,)
 
-            recommendations: dict = {}
-
             if ts._actor:
                 for ws in ts._who_has:
                     ws._actors.discard(ts)
@@ -2548,6 +2544,7 @@ def transition_memory_forgotten(self, key):
     def transition_released_forgotten(self, key):
         try:
             ts: TaskState = self._tasks[key]
+            recommendations: dict = {}
             worker_msgs: dict = {}
             client_msgs: dict = {}
 
@@ -2568,7 +2565,6 @@ def transition_released_forgotten(self, key):
                 else:
                     assert 0, (ts,)
 
-            recommendations: dict = {}
             _propagate_forgotten(self, ts, recommendations, worker_msgs)
 
             client_msgs = _task_to_client_msgs(self, ts)
@@ -2603,16 +2599,14 @@ def check_idle_saturated(self, ws: WorkerState, occ: double = -1.0):
 
         This is useful for load balancing and adaptivity.
         """
-        total_nthreads: Py_ssize_t = self._total_nthreads
-        if total_nthreads == 0 or ws.status == Status.closed:
+        if self._total_nthreads == 0 or ws.status == Status.closed:
             return
         if occ < 0:
             occ = ws._occupancy
 
         nc: Py_ssize_t = ws._nthreads
         p: Py_ssize_t = len(ws._processing)
-        total_occupancy: double = self._total_occupancy
-        avg: double = total_occupancy / total_nthreads
+        avg: double = self._total_occupancy / self._total_nthreads
 
         idle = self._idle
         saturated: set = self._saturated
@@ -2639,10 +2633,9 @@ def get_comm_cost(self, ts: TaskState, ws: WorkerState) -> double:
         dts: TaskState
         deps: set = ts._dependencies - ws._has_what
         nbytes: Py_ssize_t = 0
-        bandwidth: double = self._bandwidth
         for dts in deps:
             nbytes += dts._nbytes
-        return nbytes / bandwidth
+        return nbytes / self._bandwidth
 
     @ccall
     def get_task_duration(self, ts: TaskState, default: double = -1) -> double:
@@ -2652,8 +2645,11 @@ def get_task_duration(self, ts: TaskState, default: double = -1) -> double:
         """
         duration: double = ts._prefix._duration_average
         if duration < 0:
-            s: set = self._unknown_durations[ts._prefix._name]
+            s: set = self._unknown_durations.get(ts._prefix._name)
+            if s is None:
+                self._unknown_durations[ts._prefix._name] = s = set()
             s.add(ts)
+
             if default < 0:
                 duration = UNKNOWN_TASK_DURATION
             else:
@@ -2683,9 +2679,12 @@ def valid_workers(self, ts: TaskState) -> set:
             # may not be connected when host_restrictions is populated
             hr: list = [self.coerce_hostname(h) for h in ts._host_restrictions]
             # XXX need HostState?
-            sl: list = [
-                self._host_info[h]["addresses"] for h in hr if h in self._host_info
-            ]
+            sl: list = []
+            for h in hr:
+                dh: dict = self._host_info.get(h)
+                if dh is not None:
+                    sl.append(dh["addresses"])
+
             ss: set = set.union(*sl) if sl else set()
             if s is None:
                 s = ss
@@ -2693,14 +2692,18 @@ def valid_workers(self, ts: TaskState) -> set:
                 s |= ss
 
         if ts._resource_restrictions:
-            dw: dict = {
-                resource: {
-                    w
-                    for w, supplied in self._resources[resource].items()
-                    if supplied >= required
-                }
-                for resource, required in ts._resource_restrictions.items()
-            }
+            dw: dict = {}
+            for resource, required in ts._resource_restrictions.items():
+                dr: dict = self._resources.get(resource)
+                if dr is None:
+                    self._resources[resource] = dr = dict()
+
+                sw: set = set()
+                for w, supplied in dr.items():
+                    if supplied >= required:
+                        sw.add(w)
+
+                dw[resource] = sw
 
             ww: set = set.intersection(*dw.values())
             if s is None:
@@ -2730,8 +2733,10 @@ def coerce_hostname(self, host):
         """
         Coerce the hostname of a worker.
         """
-        if host in self._aliases:
-            return self._workers_dv[self._aliases[host]].host
+        alias = self._aliases.get(host)
+        if alias is not None:
+            ws: WorkerState = self._workers_dv[alias]
+            return ws.host
         else:
             return host
 
@@ -2751,9 +2756,8 @@ def worker_objective(self, ts: TaskState, ws: WorkerState) -> tuple:
                 nbytes = dts.get_nbytes()
                 comm_bytes += nbytes
 
-        bandwidth: double = self._bandwidth
         stack_time: double = ws._occupancy / ws._nthreads
-        start_time: double = stack_time + comm_bytes / bandwidth
+        start_time: double = stack_time + comm_bytes / self._bandwidth
 
         if ts._actor:
             return (len(ws._actors), start_time, ws._nbytes)
@@ -3020,8 +3024,8 @@ def __init__(
                 func = compose(wrap, func)
             setattr(self, old_attr, _StateLegacyMapping(workers, func))
 
-        host_info = defaultdict(dict)
-        resources = defaultdict(dict)
+        host_info = dict()
+        resources = dict()
         aliases = dict()
 
         self._task_state_collections = [unrunnable]
@@ -3400,7 +3404,11 @@ def heartbeat_worker(
         assert metrics
         host_info = host_info or {}
 
-        parent._host_info[host]["last-seen"] = local_now
+        dh: dict = parent._host_info.get(host)
+        if dh is None:
+            parent._host_info[host] = dh = dict()
+        dh["last-seen"] = local_now
+
         frac = 1 / len(parent._workers)
         parent._bandwidth = (
             parent._bandwidth * (1 - frac) + metrics["bandwidth"]["total"] * frac
@@ -3435,7 +3443,10 @@ def heartbeat_worker(
             ws._metrics = metrics
 
         if host_info:
-            parent._host_info[host].update(host_info)
+            dh: dict = parent._host_info.get(host)
+            if dh is None:
+                parent._host_info[host] = dh = dict()
+            dh.update(host_info)
 
         delay = time() - now
         ws._time_delay = delay
@@ -3510,11 +3521,17 @@ async def add_worker(
                 extra=extra,
             )
 
-            if "addresses" not in parent._host_info[host]:
-                parent._host_info[host].update({"addresses": set(), "nthreads": 0})
+            dh: dict = parent._host_info.get(host)
+            if dh is None:
+                parent._host_info[host] = dh = dict()
+
+            dh_addresses: set = dh.get("addresses")
+            if dh_addresses is None:
+                dh["addresses"] = dh_addresses = set()
+                dh["nthreads"] = 0
 
-            parent._host_info[host]["addresses"].add(address)
-            parent._host_info[host]["nthreads"] += nthreads
+            dh_addresses.add(address)
+            dh["nthreads"] += nthreads
 
             parent._total_nthreads += nthreads
             parent._aliases[name] = address
@@ -3550,8 +3567,7 @@ async def add_worker(
             recommendations: dict
             if nbytes:
                 for key in nbytes:
-                    tasks: dict = parent._tasks
-                    ts: TaskState = tasks.get(key)
+                    ts: TaskState = parent._tasks.get(key)
                     if ts is not None and ts._state in ("processing", "waiting"):
                         recommendations = self.transition(
                             key,
@@ -3929,19 +3945,21 @@ def update_graph(
 
         # TODO: balance workers
 
-    def new_task(self, key, spec, state):
+    def new_task(self, key: str, spec: object, state: str):
         """ Create a new task, and associated states """
         parent: SchedulerState = cast(SchedulerState, self)
+
         ts: TaskState = TaskState(key, spec)
-        tp: TaskPrefix
-        tg: TaskGroup
         ts._state = state
+
+        tp: TaskPrefix
         prefix_key = key_split(key)
         tp = parent._task_prefixes.get(prefix_key)
         if tp is None:
             parent._task_prefixes[prefix_key] = tp = TaskPrefix(prefix_key)
         ts._prefix = tp
 
+        tg: TaskGroup
         group_key = ts._group_key
         tg = parent._task_groups.get(group_key)
         if tg is None:
@@ -3949,7 +3967,9 @@ def new_task(self, key, spec, state):
             tg._prefix = tp
             tp._groups.append(tg)
         tg.add(ts)
+
         parent._tasks[key] = ts
+
         return ts
 
     def stimulus_task_finished(self, key=None, worker=None, **kwargs):
@@ -3957,8 +3977,7 @@ def stimulus_task_finished(self, key=None, worker=None, **kwargs):
         parent: SchedulerState = cast(SchedulerState, self)
         logger.debug("Stimulus task finished %s, %s", key, worker)
 
-        tasks: dict = parent._tasks
-        ts: TaskState = tasks.get(key)
+        ts: TaskState = parent._tasks.get(key)
         if ts is None:
             return {}
         ws: WorkerState = parent._workers_dv[worker]
@@ -4117,11 +4136,18 @@ async def remove_worker(self, comm=None, address=None, safe=False, close=True):
 
             self.remove_resources(address)
 
-            parent._host_info[host]["nthreads"] -= ws._nthreads
-            parent._host_info[host]["addresses"].remove(address)
+            dh: dict = parent._host_info.get(host)
+            if dh is None:
+                parent._host_info[host] = dh = dict()
+
+            dh_addresses: set = dh["addresses"]
+            dh_addresses.remove(address)
+            dh["nthreads"] -= ws._nthreads
             parent._total_nthreads -= ws._nthreads
 
-            if not parent._host_info[host]["addresses"]:
+            if not dh_addresses:
+                dh = None
+                dh_addresses = None
                 del parent._host_info[host]
 
             self.rpc.remove(address)
@@ -4539,7 +4565,7 @@ def remove_client_from_events():
         )
         self.loop.call_later(cleanup_delay, remove_client_from_events)
 
-    def send_task_to_worker(self, worker, ts: TaskState, duration=None):
+    def send_task_to_worker(self, worker, ts: TaskState, duration: double = -1):
         """ Send a single computational task to a worker """
         parent: SchedulerState = cast(SchedulerState, self)
         try:
@@ -4587,12 +4613,11 @@ def handle_missing_data(self, key=None, errant_worker=None, **kwargs):
         ts: TaskState = parent._tasks.get(key)
         if ts is None or not ts._who_has:
             return
-        if errant_worker in parent._workers_dv:
-            ws: WorkerState = parent._workers_dv[errant_worker]
-            if ws in ts._who_has:
-                ts._who_has.remove(ws)
-                ws._has_what.remove(ts)
-                ws._nbytes -= ts.get_nbytes()
+        ws: WorkerState = parent._workers_dv.get(errant_worker)
+        if ws is not None and ws in ts._who_has:
+            ts._who_has.remove(ws)
+            ws._has_what.remove(ts)
+            ws._nbytes -= ts.get_nbytes()
         if not ts._who_has:
             if ts._run_spec:
                 self.transitions({key: "released"})
@@ -4602,15 +4627,15 @@ def handle_missing_data(self, key=None, errant_worker=None, **kwargs):
     def release_worker_data(self, comm=None, keys=None, worker=None):
         parent: SchedulerState = cast(SchedulerState, self)
         ws: WorkerState = parent._workers_dv[worker]
-        tasks = {parent._tasks[k] for k in keys}
-        removed_tasks = tasks & ws._has_what
+        tasks: set = {parent._tasks[k] for k in keys}
+        removed_tasks: set = tasks & ws._has_what
         ws._has_what -= removed_tasks
 
         ts: TaskState
         recommendations: dict = {}
         for ts in removed_tasks:
             ws._nbytes -= ts.get_nbytes()
-            wh = ts._who_has
+            wh: set = ts._who_has
             wh.remove(ws)
             if not wh:
                 recommendations[ts._key] = "released"
@@ -4625,8 +4650,9 @@ def handle_long_running(self, key=None, worker=None, compute_duration=None):
         """
         parent: SchedulerState = cast(SchedulerState, self)
         ts: TaskState = parent._tasks[key]
-        if "stealing" in self._extensions:
-            self._extensions["stealing"].remove_key_from_stealable(ts)
+        steal = parent._extensions.get("stealing")
+        if steal is not None:
+            steal.remove_key_from_stealable(ts)
 
         ws: WorkerState = ts._processing_on
         if ws is None:
@@ -4634,8 +4660,9 @@ def handle_long_running(self, key=None, worker=None, compute_duration=None):
             return
 
         if compute_duration:
-            old_duration = ts._prefix._duration_average
-            new_duration = compute_duration
+            old_duration: double = ts._prefix._duration_average
+            new_duration: double = compute_duration
+            avg_duration: double
             if old_duration < 0:
                 avg_duration = new_duration
             else:
@@ -4643,8 +4670,9 @@ def handle_long_running(self, key=None, worker=None, compute_duration=None):
 
             ts._prefix._duration_average = avg_duration
 
-        ws._occupancy -= ws._processing[ts]
-        parent._total_occupancy -= ws._processing[ts]
+        occ: double = ws._processing[ts]
+        ws._occupancy -= occ
+        parent._total_occupancy -= occ
         ws._processing[ts] = 0
         self.check_idle_saturated(ws)
 
@@ -4953,8 +4981,9 @@ async def broadcast(
                 workers = []
         if hosts is not None:
             for host in hosts:
-                if host in parent._host_info:
-                    workers.extend(parent._host_info[host]["addresses"])
+                dh: dict = parent._host_info.get(host)
+                if dh is not None:
+                    workers.extend(dh["addresses"])
         # TODO replace with worker_list
 
         if nanny:
@@ -5576,8 +5605,7 @@ def update_data(
     def report_on_key(self, key: str = None, ts: TaskState = None, client: str = None):
         parent: SchedulerState = cast(SchedulerState, self)
         if ts is None:
-            tasks: dict = parent._tasks
-            ts = tasks.get(key)
+            ts = parent._tasks.get(key)
         elif key is None:
             key = ts._key
         else:
@@ -6097,14 +6125,20 @@ def add_resources(self, comm=None, worker=None, resources=None):
         ws._used_resources = {}
         for resource, quantity in ws._resources.items():
             ws._used_resources[resource] = 0
-            parent._resources[resource][worker] = quantity
+            dr: dict = parent._resources.get(resource, None)
+            if dr is None:
+                parent._resources[resource] = dr = dict()
+            dr[worker] = quantity
         return "OK"
 
     def remove_resources(self, worker):
         parent: SchedulerState = cast(SchedulerState, self)
         ws: WorkerState = parent._workers_dv[worker]
         for resource, quantity in ws._resources.items():
-            del parent._resources[resource][worker]
+            dr: dict = parent._resources.get(resource, None)
+            if dr is None:
+                parent._resources[resource] = dr = dict()
+            del dr[worker]
 
     def coerce_address(self, addr, resolve=True):
         """
@@ -6564,20 +6598,23 @@ def _remove_from_processing(state: SchedulerState, ts: TaskState) -> str:
     ws: WorkerState = ts._processing_on
     ts._processing_on = None
     w: str = ws._address
-    if w in state._workers_dv:  # may have been removed
-        duration = ws._processing.pop(ts)
-        if not ws._processing:
-            state._total_occupancy -= ws._occupancy
-            ws._occupancy = 0
-        else:
-            state._total_occupancy -= duration
-            ws._occupancy -= duration
-        state.check_idle_saturated(ws)
-        state.release_resources(ts, ws)
-        return w
-    else:
+
+    if w not in state._workers_dv:  # may have been removed
         return None
 
+    duration: double = ws._processing.pop(ts)
+    if not ws._processing:
+        state._total_occupancy -= ws._occupancy
+        ws._occupancy = 0
+    else:
+        state._total_occupancy -= duration
+        ws._occupancy -= duration
+
+    state.check_idle_saturated(ws)
+    state.release_resources(ts, ws)
+
+    return w
+
 
 @cfunc
 @exceptval(check=False)
@@ -6696,7 +6733,6 @@ def _client_releases_keys(
     """ Remove keys from client desired list """
     logger.debug("Client %s releases keys: %s", cs._client_key, keys)
     ts: TaskState
-    tasks2: set = set()
     for key in keys:
         ts = state._tasks.get(key)
         if ts is not None and ts in cs._wants_what:
@@ -6704,24 +6740,21 @@ def _client_releases_keys(
             s: set = ts._who_wants
             s.remove(cs)
             if not s:
-                tasks2.add(ts)
-
-    for ts in tasks2:
-        if not ts._dependents:
-            # No live dependents, can forget
-            recommendations[ts._key] = "forgotten"
-        elif ts._state != "erred" and not ts._waiters:
-            recommendations[ts._key] = "released"
+                if not ts._dependents:
+                    # No live dependents, can forget
+                    recommendations[ts._key] = "forgotten"
+                elif ts._state != "erred" and not ts._waiters:
+                    recommendations[ts._key] = "released"
 
 
 @cfunc
 @exceptval(check=False)
-def _task_to_msg(state: SchedulerState, ts: TaskState, duration=None) -> dict:
+def _task_to_msg(state: SchedulerState, ts: TaskState, duration: double = -1) -> dict:
     """ Convert a single computational task to a message """
     ws: WorkerState
     dts: TaskState
 
-    if duration is None:
+    if duration < 0:
         duration = state.get_task_duration(ts)
 
     msg: dict = {
@@ -6778,21 +6811,15 @@ def _task_to_report_msg(state: SchedulerState, ts: TaskState) -> dict:
 @cfunc
 @exceptval(check=False)
 def _task_to_client_msgs(state: SchedulerState, ts: TaskState) -> dict:
-    cs: ClientState
-    client_keys: list
+    report_msg: dict = _task_to_report_msg(state, ts)
+    client_msgs: dict
     if ts is None:
         # Notify all clients
-        client_keys = list(state._clients)
+        client_msgs = {k: [report_msg] for k in state._clients}
     else:
         # Notify clients interested in key
-        client_keys = [cs._client_key for cs in ts._who_wants]
-
-    report_msg: dict = _task_to_report_msg(state, ts)
-
-    client_msgs: dict = {}
-    for k in client_keys:
-        client_msgs[k] = [report_msg]
-
+        cs: ClientState
+        client_msgs = {cs._client_key: [report_msg] for cs in ts._who_wants}
     return client_msgs
 
 
@@ -6843,7 +6870,8 @@ def decide_worker(
     of bytes sent between workers.  This is determined by calling the
     *objective* function.
     """
-    ws: WorkerState
+    ws: WorkerState = None
+    wws: WorkerState
     dts: TaskState
     deps: set = ts._dependencies
     candidates: set
@@ -6851,7 +6879,7 @@ def decide_worker(
     if ts._actor:
         candidates = set(all_workers)
     else:
-        candidates = {ws for dts in deps for ws in dts._who_has}
+        candidates = {wws for dts in deps for wws in dts._who_has}
     if valid_workers is None:
         if not candidates:
             candidates = set(all_workers)
@@ -6861,13 +6889,13 @@ def decide_worker(
             candidates = valid_workers
             if not candidates:
                 if ts._loose_restrictions:
-                    return decide_worker(ts, all_workers, None, objective)
-                else:
-                    return None
-    if not candidates:
-        return None
+                    ws = decide_worker(ts, all_workers, None, objective)
+                return ws
 
-    if len(candidates) == 1:
+    ncandidates: Py_ssize_t = len(candidates)
+    if ncandidates == 0:
+        pass
+    elif ncandidates == 1:
         for ws in candidates:
             break
     else:

From ff56bd1ebf7ab1da30b3ec314651f78b25118dd8 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Fri, 12 Feb 2021 13:56:37 -0800
Subject: [PATCH 1152/1550] Prepend frame metadata header (#4505)

---
 distributed/comm/tcp.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py
index ff283dcf3f3..477438c64c5 100644
--- a/distributed/comm/tcp.py
+++ b/distributed/comm/tcp.py
@@ -248,13 +248,15 @@ async def write(self, msg, serializers=None, on_error="message"):
             nframes = len(frames)
             lengths = [nbytes(frame) for frame in frames]
             length_bytes = struct.pack(f"Q{nframes}Q", nframes, *lengths)
+
+            frames = [length_bytes, *frames]
+            lengths = [len(length_bytes), *lengths]
+
             if sum(lengths) < 2 ** 17:  # 128kiB
                 # small enough, send in one go
-                stream.write(b"".join([length_bytes, *frames]))
+                stream.write(b"".join(frames))
             else:
                 # avoid large memcpy, send in many
-                stream.write(length_bytes)
-
                 for frame, frame_bytes in zip(frames, lengths):
                     # Can't wait for the write() Future as it may be lost
                     # ("If write is called again before that Future has resolved,

From 02a85f7891b5872aa8e7a0fc62c2891f503a7963 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Mon, 15 Feb 2021 07:52:06 -0800
Subject: [PATCH 1153/1550] Skip `stream.write` call for empty frames (#4507)

Sometimes a frame winds up have `0` bytes. In these cases, make sure to
skip calling `stream.write` to avoid incurring the overhead. After all
it is trivial to create an empty frame on the receiving end as long as
we know it exists (the size of all frames are tracked) without involving
IO.
---
 distributed/comm/tcp.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py
index 477438c64c5..616109a26e5 100644
--- a/distributed/comm/tcp.py
+++ b/distributed/comm/tcp.py
@@ -261,11 +261,12 @@ async def write(self, msg, serializers=None, on_error="message"):
                     # Can't wait for the write() Future as it may be lost
                     # ("If write is called again before that Future has resolved,
                     #   the previous future will be orphaned and will never resolve")
-                    future = stream.write(frame)
-                    bytes_since_last_yield += frame_bytes
-                    if bytes_since_last_yield > 32e6:
-                        await future
-                        bytes_since_last_yield = 0
+                    if frame_bytes:
+                        future = stream.write(frame)
+                        bytes_since_last_yield += frame_bytes
+                        if bytes_since_last_yield > 32e6:
+                            await future
+                            bytes_since_last_yield = 0
         except StreamClosedError as e:
             self.stream = None
             self._closed = True

From 383ea0326ae103b5d5e0b62ed9c3cb18510c5b9e Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Wed, 17 Feb 2021 07:35:25 -0800
Subject: [PATCH 1154/1550] Read & write all frames in one pass (#4506)

* Overwrite `frames` variable w/`join`ed result

Ensures that the separate `frames` are freed from memory before
proceeding to the next step of sending them out.

* Read all frames in one pass

Instead of doing multiple reads with `async`, just allocate one big
chunk of memory for all of the frames and read into it. Should cutdown
on the number of passes through Tornado needed to fill these frames.

* Transmit total frame size in initial message

This should allow us to allocate space for the entirety of the rest of
the message including the size of each frames and all following frames.
We can then unpack all of this information once received. By doing this
we are able to cutdown on addition send/recv calls that would otherwise
occur and spend less time in Tornado's IO handling.

* Handle `nframes` with rest of message

* Drop unneeded variable assignment

* Use packing and unpacking utility functions

Simplifies the code in the TCP path by leveraging existing utility
functions.

* Fix-up variable names

Make it a little easier to follow how the variables relate to each
other.

* Move packing/unpacking steps out of `try`

These are just binary serialization steps that are not really depended
on communication or issues that may come from sockets. So go ahead and
move them out of the `try` block.

* Group header building steps together

Also group `frames` and `frames_nbytes` steps together. Finally rewrites
the code to avoid use of constant for size of `"Q"`, which should make
it invariant to changes in that size.

* Use `struct.calcsize` to get size

Should avoid issues on platforms where this may not be the exact size.

* Concatenate small frames beforehand

To simplify the logic, just concatenate small frames before doing any
sends. This way we can use the same code path for all sends.

* Use trick to enqueue frames to write beforehand

Tornado has an internal queue that uses to hold frames before writing
them. To avoid needing to track and wait on various `Future`s and the
amount of data sent, we can just enqueue all of the frames we want to
send before a send even happens and then start the write. This way
Tornado already has all of the data we plan to send once it starts
working. In the meantime, we are able to carry on with other tasks while
this gets handled in the background.

https://github.com/tornadoweb/tornado/blob/6cdf82e927d962290165ba7c4cccb3e974b541c3/tornado/iostream.py#L537-L538
---
 distributed/comm/tcp.py       | 66 +++++++++++++++++------------------
 distributed/protocol/utils.py |  3 +-
 2 files changed, 33 insertions(+), 36 deletions(-)

diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py
index 616109a26e5..2ad6fee9b33 100644
--- a/distributed/comm/tcp.py
+++ b/distributed/comm/tcp.py
@@ -27,6 +27,7 @@
 from .addressing import parse_host_port, unparse_host_port
 from .core import Comm, Connector, Listener, CommClosedError, FatalCommClosedError
 from .utils import to_frames, from_frames, get_tcp_server_address, ensure_concrete_host
+from ..protocol.utils import pack_frames_prelude, unpack_frames
 
 
 logger = logging.getLogger(__name__)
@@ -187,19 +188,16 @@ async def read(self, deserializers=None):
         if stream is None:
             raise CommClosedError
 
+        fmt = "Q"
+        fmt_size = struct.calcsize(fmt)
+
         try:
-            n_frames = await stream.read_bytes(8)
-            n_frames = struct.unpack("Q", n_frames)[0]
-            lengths = await stream.read_bytes(8 * n_frames)
-            lengths = struct.unpack("Q" * n_frames, lengths)
-
-            frames = []
-            for length in lengths:
-                frame = bytearray(length)
-                if length:
-                    n = await stream.read_into(frame)
-                    assert n == length, (n, length)
-                frames.append(frame)
+            frames_nbytes = await stream.read_bytes(fmt_size)
+            (frames_nbytes,) = struct.unpack(fmt, frames_nbytes)
+
+            frames = bytearray(frames_nbytes)
+            n = await stream.read_into(frames)
+            assert n == frames_nbytes, (n, frames_nbytes)
         except StreamClosedError as e:
             self.stream = None
             self._closed = True
@@ -214,6 +212,8 @@ async def read(self, deserializers=None):
             raise
         else:
             try:
+                frames = unpack_frames(frames)
+
                 msg = await from_frames(
                     frames,
                     deserialize=self.deserialize,
@@ -243,30 +243,28 @@ async def write(self, msg, serializers=None, on_error="message"):
                 **self.handshake_options,
             },
         )
+        frames_nbytes = sum(map(nbytes, frames))
 
-        try:
-            nframes = len(frames)
-            lengths = [nbytes(frame) for frame in frames]
-            length_bytes = struct.pack(f"Q{nframes}Q", nframes, *lengths)
+        header = pack_frames_prelude(frames)
+        header = struct.pack("Q", nbytes(header) + frames_nbytes) + header
 
-            frames = [length_bytes, *frames]
-            lengths = [len(length_bytes), *lengths]
+        frames = [header, *frames]
+        frames_nbytes += nbytes(header)
 
-            if sum(lengths) < 2 ** 17:  # 128kiB
-                # small enough, send in one go
-                stream.write(b"".join(frames))
-            else:
-                # avoid large memcpy, send in many
-                for frame, frame_bytes in zip(frames, lengths):
-                    # Can't wait for the write() Future as it may be lost
-                    # ("If write is called again before that Future has resolved,
-                    #   the previous future will be orphaned and will never resolve")
-                    if frame_bytes:
-                        future = stream.write(frame)
-                        bytes_since_last_yield += frame_bytes
-                        if bytes_since_last_yield > 32e6:
-                            await future
-                            bytes_since_last_yield = 0
+        if frames_nbytes < 2 ** 17:  # 128kiB
+            # small enough, send in one go
+            frames = [b"".join(frames)]
+
+        try:
+            # trick to enque all frames for writing beforehand
+            for each_frame in frames:
+                each_frame_nbytes = nbytes(each_frame)
+                if each_frame_nbytes:
+                    stream._write_buffer.append(each_frame)
+                    stream._total_write_index += each_frame_nbytes
+
+            # start writing frames
+            stream.write(b"")
         except StreamClosedError as e:
             self.stream = None
             self._closed = True
@@ -282,7 +280,7 @@ async def write(self, msg, serializers=None, on_error="message"):
             self.abort()
             raise
 
-        return sum(lengths)
+        return frames_nbytes
 
     @gen.coroutine
     def close(self):
diff --git a/distributed/protocol/utils.py b/distributed/protocol/utils.py
index 2b96fba4c61..45cb1466a89 100644
--- a/distributed/protocol/utils.py
+++ b/distributed/protocol/utils.py
@@ -125,8 +125,7 @@ def unpack_frames(b):
     start = fmt_size * (1 + n_frames)
     for length in lengths:
         end = start + length
-        frame = b[start:end]
-        frames.append(frame)
+        frames.append(b[start:end])
         start = end
 
     return frames

From fdeca218134dbfe6c2c46f947413c7d0e1d2acab Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Mon, 22 Feb 2021 17:33:41 -0600
Subject: [PATCH 1155/1550] Pin black pre-commit (#4533)

---
 .pre-commit-config.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index a934af611e1..ca1c26a7d87 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,6 +1,6 @@
 repos:
   -   repo: https://github.com/psf/black
-      rev: stable
+      rev: 20.8b1
       hooks:
       - id: black
         language_version: python3

From 7146449173c0329463ca39b98ba51087278853d5 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Mon, 22 Feb 2021 16:12:20 -0800
Subject: [PATCH 1156/1550] `transition_memory_released` and `get_nbytes()`
 optimizations (#4516)

* Annotate `nbytes` in `update_data` as a `dict`

The one case where this variable is used it is a `dict`. So go ahead and
annotate it that way. Should speed up usage of this variable when
cythonized.

* Assign `ts.get_nbytes()` to a variable

To avoid calling this method repeatedly in a few cases, assign the
result to a variable and reuse it.

* Create msg once in `transition_memory_released`
---
 distributed/scheduler.py | 45 +++++++++++++++++++++++++---------------
 1 file changed, 28 insertions(+), 17 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index c13f5292538..911bf4fba3d 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -2168,17 +2168,17 @@ def transition_memory_released(self, key, safe: bint = False):
                     dts._waiting_on.add(ts)
 
             # XXX factor this out?
+            ts_nbytes: Py_ssize_t = ts.get_nbytes()
+            worker_msg = {
+                "op": "delete-data",
+                "keys": [key],
+                "report": False,
+            }
             for ws in ts._who_has:
                 ws._has_what.remove(ts)
-                ws._nbytes -= ts.get_nbytes()
-                ts._group._nbytes_in_memory -= ts.get_nbytes()
-                worker_msgs[ws._address] = [
-                    {
-                        "op": "delete-data",
-                        "keys": [key],
-                        "report": False,
-                    }
-                ]
+                ws._nbytes -= ts_nbytes
+                ts._group._nbytes_in_memory -= ts_nbytes
+                worker_msgs[ws._address] = [worker_msg]
 
             ts._who_has.clear()
 
@@ -4053,9 +4053,10 @@ def stimulus_missing_data(
 
             if cts is not None and cts._state == "memory":  # couldn't find this
                 ws: WorkerState
+                cts_nbytes: Py_ssize_t = cts.get_nbytes()
                 for ws in cts._who_has:  # TODO: this behavior is extreme
                     ws._has_what.remove(cts)
-                    ws._nbytes -= cts.get_nbytes()
+                    ws._nbytes -= cts_nbytes
                 cts._who_has.clear()
                 recommendations[cause] = "released"
 
@@ -4865,12 +4866,13 @@ async def gather(self, comm=None, keys=None, serializers=None):
                     )
                     if not workers or ts is None:
                         continue
+                    ts_nbytes: Py_ssize_t = ts.get_nbytes()
                     for worker in workers:
                         ws = parent._workers_dv.get(worker)
                         if ws is not None and ts in ws._has_what:
                             ws._has_what.remove(ts)
                             ts._who_has.remove(ws)
-                            ws._nbytes -= ts.get_nbytes()
+                            ws._nbytes -= ts_nbytes
                             self.transitions({key: "released"})
 
         self.log_event("all", {"action": "gather", "count": len(keys)})
@@ -5566,7 +5568,12 @@ def add_keys(self, comm=None, worker=None, keys=()):
         return "OK"
 
     def update_data(
-        self, comm=None, who_has=None, nbytes=None, client=None, serializers=None
+        self,
+        comm=None,
+        who_has=None,
+        nbytes: dict = None,
+        client=None,
+        serializers=None,
     ):
         """
         Learn that new data has entered the network from an external source
@@ -5587,12 +5594,15 @@ def update_data(
                 if ts is None:
                     ts: TaskState = self.new_task(key, None, "memory")
                 ts.state = "memory"
-                if key in nbytes:
-                    ts.set_nbytes(nbytes[key])
+                ts_nbytes: Py_ssize_t = nbytes.get(key, -1)
+                if ts_nbytes >= 0:
+                    ts.set_nbytes(ts_nbytes)
+                else:
+                    ts_nbytes = ts.get_nbytes()
                 for w in workers:
                     ws: WorkerState = parent._workers_dv[w]
                     if ts not in ws._has_what:
-                        ws._nbytes += ts.get_nbytes()
+                        ws._nbytes += ts_nbytes
                         ws._has_what.add(ts)
                         ts._who_has.add(ws)
                 self.report(
@@ -6712,13 +6722,14 @@ def _propagate_forgotten(
     ts._dependencies.clear()
     ts._waiting_on.clear()
 
+    ts_nbytes: Py_ssize_t = ts.get_nbytes()
     if ts._who_has:
-        ts._group._nbytes_in_memory -= ts.get_nbytes()
+        ts._group._nbytes_in_memory -= ts_nbytes
 
     ws: WorkerState
     for ws in ts._who_has:
         ws._has_what.remove(ts)
-        ws._nbytes -= ts.get_nbytes()
+        ws._nbytes -= ts_nbytes
         w: str = ws._address
         if w in state._workers_dv:  # in case worker has died
             worker_msgs[w] = [{"op": "delete-data", "keys": [key], "report": False}]

From f774f1ea21d110b2b66643080e1e99d648efc55a Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Tue, 23 Feb 2021 15:37:40 -0800
Subject: [PATCH 1157/1550] Batch more Scheduler sends (#4526)

* Annotate `retries` as `Py_ssize_t`

* Assign and use `recommendations` throughout

* Reuse `recommendations` throughout

* Validate before sending messages in `transitions`

* Assign `recommendations` an empty `dict` to start

Also clear it out after running `transitions` as that should have
handled all of them.

* Rename `_transitions` to `_transitions_table`

* Swap `client_msg` and `worker_msg` order

* Factor out `send_all` from `_transitions`

* Batch communication from `handle_task_finished`

Track messages to send from transition calls in `handle_task_finished`
and `stimulus_task_finished` to allow sending all of these in one go.
Should allow the transition work to happen with fewer interruptions and
send more data at once.

* Batch all messages from `add_worker`

* Batch all messages from `handle_task_erred`

* Batch all messages from `handle_release_data`

* Batch all messages from `gather`
---
 distributed/scheduler.py | 269 +++++++++++++++++++++++----------------
 1 file changed, 160 insertions(+), 109 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 911bf4fba3d..0ddd630b71f 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1769,8 +1769,8 @@ def transition_released_waiting(self, key):
             ts: TaskState = self._tasks[key]
             dts: TaskState
             recommendations: dict = {}
-            worker_msgs: dict = {}
             client_msgs: dict = {}
+            worker_msgs: dict = {}
 
             if self._validate:
                 assert ts._run_spec
@@ -1781,7 +1781,7 @@ def transition_released_waiting(self, key):
 
             if ts._has_lost_dependencies:
                 recommendations[key] = "forgotten"
-                return recommendations, worker_msgs, client_msgs
+                return recommendations, client_msgs, worker_msgs
 
             ts.state = "waiting"
 
@@ -1790,7 +1790,7 @@ def transition_released_waiting(self, key):
                 if dts._exception_blame:
                     ts._exception_blame = dts._exception_blame
                     recommendations[key] = "erred"
-                    return recommendations, worker_msgs, client_msgs
+                    return recommendations, client_msgs, worker_msgs
 
             for dts in ts._dependencies:
                 dep = dts._key
@@ -1810,7 +1810,7 @@ def transition_released_waiting(self, key):
                     self._unrunnable.add(ts)
                     ts.state = "no-worker"
 
-            return recommendations, worker_msgs, client_msgs
+            return recommendations, client_msgs, worker_msgs
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
@@ -1824,8 +1824,8 @@ def transition_no_worker_waiting(self, key):
             ts: TaskState = self._tasks[key]
             dts: TaskState
             recommendations: dict = {}
-            worker_msgs: dict = {}
             client_msgs: dict = {}
+            worker_msgs: dict = {}
 
             if self._validate:
                 assert ts in self._unrunnable
@@ -1837,7 +1837,7 @@ def transition_no_worker_waiting(self, key):
 
             if ts._has_lost_dependencies:
                 recommendations[key] = "forgotten"
-                return recommendations, worker_msgs, client_msgs
+                return recommendations, client_msgs, worker_msgs
 
             for dts in ts._dependencies:
                 dep = dts._key
@@ -1857,7 +1857,7 @@ def transition_no_worker_waiting(self, key):
                     self._unrunnable.add(ts)
                     ts.state = "no-worker"
 
-            return recommendations, worker_msgs, client_msgs
+            return recommendations, client_msgs, worker_msgs
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
@@ -1934,8 +1934,8 @@ def transition_waiting_processing(self, key):
             ts: TaskState = self._tasks[key]
             dts: TaskState
             recommendations: dict = {}
-            worker_msgs: dict = {}
             client_msgs: dict = {}
+            worker_msgs: dict = {}
 
             if self._validate:
                 assert not ts._waiting_on
@@ -1948,7 +1948,7 @@ def transition_waiting_processing(self, key):
 
             ws: WorkerState = self.decide_worker(ts)
             if ws is None:
-                return recommendations, worker_msgs, client_msgs
+                return recommendations, client_msgs, worker_msgs
             worker = ws._address
 
             duration_estimate = self.set_duration_estimate(ts, ws)
@@ -1967,7 +1967,7 @@ def transition_waiting_processing(self, key):
 
             worker_msgs[worker] = [_task_to_msg(self, ts)]
 
-            return recommendations, worker_msgs, client_msgs
+            return recommendations, client_msgs, worker_msgs
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
@@ -1983,8 +1983,8 @@ def transition_waiting_memory(
             ws: WorkerState = self._workers_dv[worker]
             ts: TaskState = self._tasks[key]
             recommendations: dict = {}
-            worker_msgs: dict = {}
             client_msgs: dict = {}
+            worker_msgs: dict = {}
 
             if self._validate:
                 assert not ts._processing_on
@@ -2007,7 +2007,7 @@ def transition_waiting_memory(
                 assert not ts._waiting_on
                 assert ts._who_has
 
-            return recommendations, worker_msgs, client_msgs
+            return recommendations, client_msgs, worker_msgs
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
@@ -2029,8 +2029,8 @@ def transition_processing_memory(
         ws: WorkerState
         wws: WorkerState
         recommendations: dict = {}
-        worker_msgs: dict = {}
         client_msgs: dict = {}
+        worker_msgs: dict = {}
         try:
             ts: TaskState = self._tasks[key]
             assert worker
@@ -2048,7 +2048,7 @@ def transition_processing_memory(
             ws = self._workers_dv.get(worker)
             if ws is None:
                 recommendations[key] = "released"
-                return recommendations, worker_msgs, client_msgs
+                return recommendations, client_msgs, worker_msgs
 
             if ws != ts._processing_on:  # someone else has this task
                 logger.info(
@@ -2058,7 +2058,7 @@ def transition_processing_memory(
                     ws,
                     key,
                 )
-                return recommendations, worker_msgs, client_msgs
+                return recommendations, client_msgs, worker_msgs
 
             has_compute_startstop: bool = False
             compute_start: double
@@ -2124,7 +2124,7 @@ def transition_processing_memory(
                 assert not ts._processing_on
                 assert not ts._waiting_on
 
-            return recommendations, worker_msgs, client_msgs
+            return recommendations, client_msgs, worker_msgs
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
@@ -2139,8 +2139,8 @@ def transition_memory_released(self, key, safe: bint = False):
             ts: TaskState = self._tasks[key]
             dts: TaskState
             recommendations: dict = {}
-            worker_msgs: dict = {}
             client_msgs: dict = {}
+            worker_msgs: dict = {}
 
             if self._validate:
                 assert not ts._waiting_on
@@ -2157,8 +2157,8 @@ def transition_memory_released(self, key, safe: bint = False):
                     recommendations[ts._key] = "erred"
                     return (
                         recommendations,
-                        worker_msgs,
                         client_msgs,
+                        worker_msgs,
                     )  # don't try to recreate
 
             for dts in ts._waiters:
@@ -2199,7 +2199,7 @@ def transition_memory_released(self, key, safe: bint = False):
             if self._validate:
                 assert not ts._waiting_on
 
-            return recommendations, worker_msgs, client_msgs
+            return recommendations, client_msgs, worker_msgs
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
@@ -2214,8 +2214,8 @@ def transition_released_erred(self, key):
             dts: TaskState
             failing_ts: TaskState
             recommendations: dict = {}
-            worker_msgs: dict = {}
             client_msgs: dict = {}
+            worker_msgs: dict = {}
 
             if self._validate:
                 with log_errors(pdb=LOG_PDB):
@@ -2244,7 +2244,7 @@ def transition_released_erred(self, key):
             ts.state = "erred"
 
             # TODO: waiting data?
-            return recommendations, worker_msgs, client_msgs
+            return recommendations, client_msgs, worker_msgs
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
@@ -2258,8 +2258,8 @@ def transition_erred_released(self, key):
             ts: TaskState = self._tasks[key]
             dts: TaskState
             recommendations: dict = {}
-            worker_msgs: dict = {}
             client_msgs: dict = {}
+            worker_msgs: dict = {}
 
             if self._validate:
                 with log_errors(pdb=LOG_PDB):
@@ -2284,7 +2284,7 @@ def transition_erred_released(self, key):
 
             ts.state = "released"
 
-            return recommendations, worker_msgs, client_msgs
+            return recommendations, client_msgs, worker_msgs
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
@@ -2297,8 +2297,8 @@ def transition_waiting_released(self, key):
         try:
             ts: TaskState = self._tasks[key]
             recommendations: dict = {}
-            worker_msgs: dict = {}
             client_msgs: dict = {}
+            worker_msgs: dict = {}
 
             if self._validate:
                 assert not ts._who_has
@@ -2322,7 +2322,7 @@ def transition_waiting_released(self, key):
             else:
                 ts._waiters.clear()
 
-            return recommendations, worker_msgs, client_msgs
+            return recommendations, client_msgs, worker_msgs
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
@@ -2336,8 +2336,8 @@ def transition_processing_released(self, key):
             ts: TaskState = self._tasks[key]
             dts: TaskState
             recommendations: dict = {}
-            worker_msgs: dict = {}
             client_msgs: dict = {}
+            worker_msgs: dict = {}
 
             if self._validate:
                 assert ts._processing_on
@@ -2368,7 +2368,7 @@ def transition_processing_released(self, key):
             if self._validate:
                 assert not ts._processing_on
 
-            return recommendations, worker_msgs, client_msgs
+            return recommendations, client_msgs, worker_msgs
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
@@ -2386,8 +2386,8 @@ def transition_processing_erred(
             dts: TaskState
             failing_ts: TaskState
             recommendations: dict = {}
-            worker_msgs: dict = {}
             client_msgs: dict = {}
+            worker_msgs: dict = {}
 
             if self._validate:
                 assert cause or ts._exception_blame
@@ -2447,7 +2447,7 @@ def transition_processing_erred(
             if self._validate:
                 assert not ts._processing_on
 
-            return recommendations, worker_msgs, client_msgs
+            return recommendations, client_msgs, worker_msgs
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
@@ -2461,8 +2461,8 @@ def transition_no_worker_released(self, key):
             ts: TaskState = self._tasks[key]
             dts: TaskState
             recommendations: dict = {}
-            worker_msgs: dict = {}
             client_msgs: dict = {}
+            worker_msgs: dict = {}
 
             if self._validate:
                 assert self._tasks[key].state == "no-worker"
@@ -2477,7 +2477,7 @@ def transition_no_worker_released(self, key):
 
             ts._waiters.clear()
 
-            return recommendations, worker_msgs, client_msgs
+            return recommendations, client_msgs, worker_msgs
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
@@ -2504,8 +2504,8 @@ def transition_memory_forgotten(self, key):
         try:
             ts: TaskState = self._tasks[key]
             recommendations: dict = {}
-            worker_msgs: dict = {}
             client_msgs: dict = {}
+            worker_msgs: dict = {}
 
             if self._validate:
                 assert ts._state == "memory"
@@ -2532,7 +2532,7 @@ def transition_memory_forgotten(self, key):
             client_msgs = _task_to_client_msgs(self, ts)
             self.remove_key(key)
 
-            return recommendations, worker_msgs, client_msgs
+            return recommendations, client_msgs, worker_msgs
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
@@ -2545,8 +2545,8 @@ def transition_released_forgotten(self, key):
         try:
             ts: TaskState = self._tasks[key]
             recommendations: dict = {}
-            worker_msgs: dict = {}
             client_msgs: dict = {}
+            worker_msgs: dict = {}
 
             if self._validate:
                 assert ts._state in ("released", "erred")
@@ -2570,7 +2570,7 @@ def transition_released_forgotten(self, key):
             client_msgs = _task_to_client_msgs(self, ts)
             self.remove_key(key)
 
-            return recommendations, worker_msgs, client_msgs
+            return recommendations, client_msgs, worker_msgs
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
@@ -3120,7 +3120,7 @@ def __init__(
             "stop_task_metadata": self.stop_task_metadata,
         }
 
-        self._transitions = {
+        self._transitions_table = {
             ("released", "waiting"): self.transition_released_waiting,
             ("waiting", "released"): self.transition_waiting_released,
             ("waiting", "processing"): self.transition_waiting_processing,
@@ -3564,28 +3564,34 @@ async def add_worker(
                 except Exception as e:
                     logger.exception(e)
 
-            recommendations: dict
+            recommendations: dict = {}
+            client_msgs: dict = {}
+            worker_msgs: dict = {}
             if nbytes:
                 for key in nbytes:
                     ts: TaskState = parent._tasks.get(key)
                     if ts is not None and ts._state in ("processing", "waiting"):
-                        recommendations = self.transition(
+                        t: tuple = self._transition(
                             key,
                             "memory",
                             worker=address,
                             nbytes=nbytes[key],
                             typename=types[key],
                         )
-                        self.transitions(recommendations)
+                        recommendations, client_msgs, worker_msgs = t
+                        self._transitions(recommendations, client_msgs, worker_msgs)
+                        recommendations = {}
 
-            recommendations = {}
             for ts in list(parent._unrunnable):
                 valid: set = self.valid_workers(ts)
                 if valid is None or ws in valid:
                     recommendations[ts._key] = "waiting"
 
             if recommendations:
-                self.transitions(recommendations)
+                self._transitions(recommendations, client_msgs, worker_msgs)
+                recommendations = {}
+
+            self.send_all(client_msgs, worker_msgs)
 
             self.log_event(address, {"action": "add-worker"})
             self.log_event("all", {"action": "add-worker", "worker": address})
@@ -3977,15 +3983,19 @@ def stimulus_task_finished(self, key=None, worker=None, **kwargs):
         parent: SchedulerState = cast(SchedulerState, self)
         logger.debug("Stimulus task finished %s, %s", key, worker)
 
+        recommendations: dict = {}
+        client_msgs: dict = {}
+        worker_msgs: dict = {}
+
         ts: TaskState = parent._tasks.get(key)
         if ts is None:
-            return {}
+            return recommendations, client_msgs, worker_msgs
         ws: WorkerState = parent._workers_dv[worker]
         ts._metadata.update(kwargs["metadata"])
 
-        recommendations: dict
         if ts._state == "processing":
-            recommendations = self.transition(key, "memory", worker=worker, **kwargs)
+            r: tuple = self._transition(key, "memory", worker=worker, **kwargs)
+            recommendations, client_msgs, worker_msgs = r
 
             if ts._state == "memory":
                 assert ws in ts._who_has
@@ -3999,10 +4009,9 @@ def stimulus_task_finished(self, key=None, worker=None, **kwargs):
                 ts._who_has,
             )
             if ws not in ts._who_has:
-                self.worker_send(worker, {"op": "release-task", "key": key})
-            recommendations = {}
+                worker_msgs[worker] = [{"op": "release-task", "key": key}]
 
-        return recommendations
+        return recommendations, client_msgs, worker_msgs
 
     def stimulus_task_erred(
         self, key=None, worker=None, exception=None, traceback=None, **kwargs
@@ -4011,18 +4020,22 @@ def stimulus_task_erred(
         parent: SchedulerState = cast(SchedulerState, self)
         logger.debug("Stimulus task erred %s, %s", key, worker)
 
+        recommendations: dict = {}
+        client_msgs: dict = {}
+        worker_msgs: dict = {}
+
         ts: TaskState = parent._tasks.get(key)
         if ts is None:
-            return {}
+            return recommendations, client_msgs, worker_msgs
 
-        recommendations: dict
         if ts._state == "processing":
-            retries = ts._retries
+            retries: Py_ssize_t = ts._retries
+            r: tuple
             if retries > 0:
                 ts._retries = retries - 1
-                recommendations = self.transition(key, "waiting")
+                r = self._transition(key, "waiting")
             else:
-                recommendations = self.transition(
+                r = self._transition(
                     key,
                     "erred",
                     cause=key,
@@ -4031,10 +4044,9 @@ def stimulus_task_erred(
                     worker=worker,
                     **kwargs,
                 )
-        else:
-            recommendations = {}
+            recommendations, client_msgs, worker_msgs = r
 
-        return recommendations
+        return recommendations, client_msgs, worker_msgs
 
     def stimulus_missing_data(
         self, cause=None, key=None, worker=None, ensure=True, **kwargs
@@ -4044,13 +4056,15 @@ def stimulus_missing_data(
         with log_errors():
             logger.debug("Stimulus missing data %s, %s", key, worker)
 
+            recommendations: dict = {}
+            client_msgs: dict = {}
+            worker_msgs: dict = {}
+
             ts: TaskState = parent._tasks.get(key)
             if ts is None or ts._state == "memory":
-                return {}
+                return recommendations, client_msgs, worker_msgs
             cts: TaskState = parent._tasks.get(cause)
 
-            recommendations: dict = {}
-
             if cts is not None and cts._state == "memory":  # couldn't find this
                 ws: WorkerState
                 cts_nbytes: Py_ssize_t = cts.get_nbytes()
@@ -4063,12 +4077,13 @@ def stimulus_missing_data(
             if key:
                 recommendations[key] = "released"
 
-            self.transitions(recommendations)
+            self._transitions(recommendations, client_msgs, worker_msgs)
+            recommendations = {}
 
             if parent._validate:
                 assert cause not in self.who_has
 
-            return {}
+            return recommendations, client_msgs, worker_msgs
 
     def stimulus_retry(self, comm=None, keys=None, client=None):
         parent: SchedulerState = cast(SchedulerState, self)
@@ -4588,12 +4603,27 @@ def handle_task_finished(self, key=None, worker=None, **msg):
         if worker not in parent._workers_dv:
             return
         validate_key(key)
-        r = self.stimulus_task_finished(key=key, worker=worker, **msg)
-        self.transitions(r)
+
+        recommendations: dict
+        client_msgs: dict
+        worker_msgs: dict
+
+        r: tuple = self.stimulus_task_finished(key=key, worker=worker, **msg)
+        recommendations, client_msgs, worker_msgs = r
+        self._transitions(recommendations, client_msgs, worker_msgs)
+
+        self.send_all(client_msgs, worker_msgs)
 
     def handle_task_erred(self, key=None, **msg):
-        r = self.stimulus_task_erred(key=key, **msg)
-        self.transitions(r)
+        recommendations: dict
+        client_msgs: dict
+        worker_msgs: dict
+
+        r: tuple = self.stimulus_task_erred(key=key, **msg)
+        recommendations, client_msgs, worker_msgs = r
+        self._transitions(recommendations, client_msgs, worker_msgs)
+
+        self.send_all(client_msgs, worker_msgs)
 
     def handle_release_data(self, key=None, worker=None, client=None, **msg):
         parent: SchedulerState = cast(SchedulerState, self)
@@ -4603,8 +4633,16 @@ def handle_release_data(self, key=None, worker=None, client=None, **msg):
         ws: WorkerState = parent._workers_dv[worker]
         if ts._processing_on != ws:
             return
-        r = self.stimulus_missing_data(key=key, ensure=False, **msg)
-        self.transitions(r)
+
+        recommendations: dict
+        client_msgs: dict
+        worker_msgs: dict
+
+        r: tuple = self.stimulus_missing_data(key=key, ensure=False, **msg)
+        recommendations, client_msgs, worker_msgs = r
+        self._transitions(recommendations, client_msgs, worker_msgs)
+
+        self.send_all(client_msgs, worker_msgs)
 
     def handle_missing_data(self, key=None, errant_worker=None, **kwargs):
         parent: SchedulerState = cast(SchedulerState, self)
@@ -4742,17 +4780,10 @@ def client_send(self, client, msg):
 
     def send_all(self, client_msgs: dict, worker_msgs: dict):
         """Send messages to client and workers"""
-        stream_comms: dict = self.stream_comms
         client_comms: dict = self.client_comms
+        stream_comms: dict = self.stream_comms
         msgs: list
 
-        for worker, msgs in worker_msgs.items():
-            try:
-                w = stream_comms[worker]
-                w.send(*msgs)
-            except (CommClosedError, AttributeError):
-                self.loop.add_callback(self.remove_worker, address=worker)
-
         for client, msgs in client_msgs.items():
             c = client_comms.get(client)
             if c is None:
@@ -4763,6 +4794,13 @@ def send_all(self, client_msgs: dict, worker_msgs: dict):
                 if self.status == Status.running:
                     logger.critical("Tried writing to closed comm: %s", msgs)
 
+        for worker, msgs in worker_msgs.items():
+            try:
+                w = stream_comms[worker]
+                w.send(*msgs)
+            except (CommClosedError, AttributeError):
+                self.loop.add_callback(self.remove_worker, address=worker)
+
     ############################
     # Less common interactions #
     ############################
@@ -4855,6 +4893,10 @@ async def gather(self, comm=None, keys=None, serializers=None):
                         for worker in missing_workers
                     ]
                 )
+
+                recommendations: dict
+                client_msgs: dict = {}
+                worker_msgs: dict = {}
                 for key, workers in missing_keys.items():
                     # Task may already be gone if it was held by a
                     # `missing_worker`
@@ -4867,13 +4909,15 @@ async def gather(self, comm=None, keys=None, serializers=None):
                     if not workers or ts is None:
                         continue
                     ts_nbytes: Py_ssize_t = ts.get_nbytes()
+                    recommendations: dict = {key: "released"}
                     for worker in workers:
                         ws = parent._workers_dv.get(worker)
                         if ws is not None and ts in ws._has_what:
                             ws._has_what.remove(ts)
                             ts._who_has.remove(ws)
                             ws._nbytes -= ts_nbytes
-                            self.transitions({key: "released"})
+                            self._transitions(recommendations, client_msgs, worker_msgs)
+                self.send_all(client_msgs, worker_msgs)
 
         self.log_event("all", {"action": "gather", "count": len(keys)})
         return result
@@ -5915,64 +5959,64 @@ def _transition(self, key, finish: str, *args, **kwargs):
 
             ts = parent._tasks.get(key)
             if ts is None:
-                return recommendations, worker_msgs, client_msgs
+                return recommendations, client_msgs, worker_msgs
             start = ts._state
             if start == finish:
-                return recommendations, worker_msgs, client_msgs
+                return recommendations, client_msgs, worker_msgs
 
             if self.plugins:
                 dependents = set(ts._dependents)
                 dependencies = set(ts._dependencies)
 
             start_finish = (start, finish)
-            func = self._transitions.get(start_finish)
+            func = self._transitions_table.get(start_finish)
             if func is not None:
                 a: tuple = func(key, *args, **kwargs)
-                recommendations, worker_msgs, client_msgs = a
+                recommendations, client_msgs, worker_msgs = a
             elif "released" not in start_finish:
-                func = self._transitions["released", finish]
+                func = self._transitions_table["released", finish]
                 assert not args and not kwargs
                 a_recs: dict
-                a_wmsgs: dict
                 a_cmsgs: dict
+                a_wmsgs: dict
                 a: tuple = self._transition(key, "released")
-                a_recs, a_wmsgs, a_cmsgs = a
+                a_recs, a_cmsgs, a_wmsgs = a
                 v = a_recs.get(key)
                 if v is not None:
-                    func = self._transitions["released", v]
+                    func = self._transitions_table["released", v]
                 b_recs: dict
-                b_wmsgs: dict
                 b_cmsgs: dict
+                b_wmsgs: dict
                 b: tuple = func(key)
-                b_recs, b_wmsgs, b_cmsgs = b
+                b_recs, b_cmsgs, b_wmsgs = b
 
                 recommendations.update(a_recs)
-                for w, new_msgs in a_wmsgs.items():
-                    msgs = worker_msgs.get(w)
-                    if msgs is not None:
-                        msgs.extend(new_msgs)
-                    else:
-                        worker_msgs[w] = new_msgs
                 for c, new_msgs in a_cmsgs.items():
                     msgs = client_msgs.get(c)
                     if msgs is not None:
                         msgs.extend(new_msgs)
                     else:
                         client_msgs[c] = new_msgs
-
-                recommendations.update(b_recs)
-                for w, new_msgs in b_wmsgs.items():
+                for w, new_msgs in a_wmsgs.items():
                     msgs = worker_msgs.get(w)
                     if msgs is not None:
                         msgs.extend(new_msgs)
                     else:
                         worker_msgs[w] = new_msgs
+
+                recommendations.update(b_recs)
                 for c, new_msgs in b_cmsgs.items():
                     msgs = client_msgs.get(c)
                     if msgs is not None:
                         msgs.extend(new_msgs)
                     else:
                         client_msgs[c] = new_msgs
+                for w, new_msgs in b_wmsgs.items():
+                    msgs = worker_msgs.get(w)
+                    if msgs is not None:
+                        msgs.extend(new_msgs)
+                    else:
+                        worker_msgs[w] = new_msgs
 
                 start = "released"
             else:
@@ -6015,7 +6059,7 @@ def _transition(self, key, finish: str, *args, **kwargs):
                     ts._prefix._groups.remove(tg)
                     del parent._task_groups[tg._name]
 
-            return recommendations, worker_msgs, client_msgs
+            return recommendations, client_msgs, worker_msgs
         except Exception as e:
             logger.exception("Error transitioning %r from %r to %r", key, start, finish)
             if LOG_PDB:
@@ -6044,11 +6088,11 @@ def transition(self, key, finish: str, *args, **kwargs):
         worker_msgs: dict
         client_msgs: dict
         a: tuple = self._transition(key, finish, *args, **kwargs)
-        recommendations, worker_msgs, client_msgs = a
+        recommendations, client_msgs, worker_msgs = a
         self.send_all(client_msgs, worker_msgs)
         return recommendations
 
-    def transitions(self, recommendations: dict):
+    def _transitions(self, recommendations: dict, client_msgs: dict, worker_msgs: dict):
         """Process transitions until none are left
 
         This includes feedback from previous transitions and continues until we
@@ -6057,41 +6101,48 @@ def transitions(self, recommendations: dict):
         parent: SchedulerState = cast(SchedulerState, self)
         keys: set = set()
         recommendations = recommendations.copy()
-        worker_msgs: dict = {}
-        client_msgs: dict = {}
         msgs: list
         new_msgs: list
         new: tuple
         new_recs: dict
-        new_wmsgs: dict
         new_cmsgs: dict
+        new_wmsgs: dict
         while recommendations:
             key, finish = recommendations.popitem()
             keys.add(key)
 
             new = self._transition(key, finish)
-            new_recs, new_wmsgs, new_cmsgs = new
+            new_recs, new_cmsgs, new_wmsgs = new
 
             recommendations.update(new_recs)
-            for w, new_msgs in new_wmsgs.items():
-                msgs = worker_msgs.get(w)
-                if msgs is not None:
-                    msgs.extend(new_msgs)
-                else:
-                    worker_msgs[w] = new_msgs
             for c, new_msgs in new_cmsgs.items():
                 msgs = client_msgs.get(c)
                 if msgs is not None:
                     msgs.extend(new_msgs)
                 else:
                     client_msgs[c] = new_msgs
-
-        self.send_all(client_msgs, worker_msgs)
+            for w, new_msgs in new_wmsgs.items():
+                msgs = worker_msgs.get(w)
+                if msgs is not None:
+                    msgs.extend(new_msgs)
+                else:
+                    worker_msgs[w] = new_msgs
 
         if parent._validate:
             for key in keys:
                 self.validate_key(key)
 
+    def transitions(self, recommendations: dict):
+        """Process transitions until none are left
+
+        This includes feedback from previous transitions and continues until we
+        reach a steady state
+        """
+        client_msgs: dict = {}
+        worker_msgs: dict = {}
+        self._transitions(recommendations, client_msgs, worker_msgs)
+        self.send_all(client_msgs, worker_msgs)
+
     def story(self, *keys):
         """ Get all transitions that touch one of the input keys """
         keys = {key.key if isinstance(key, TaskState) else key for key in keys}

From 7e8cbf134747dc9f913c445517a609f4cf639d10 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Tue, 23 Feb 2021 15:37:55 -0800
Subject: [PATCH 1158/1550] Move `new_task` to `SchedulerState` (#4527)

---
 distributed/scheduler.py | 60 ++++++++++++++++++++--------------------
 1 file changed, 30 insertions(+), 30 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 0ddd630b71f..77fb63ff0d7 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1764,6 +1764,33 @@ def __pdict__(self):
             "host_info": self._host_info,
         }
 
+    @ccall
+    @exceptval(check=False)
+    def new_task(self, key: str, spec: object, state: str) -> TaskState:
+        """ Create a new task, and associated states """
+        ts: TaskState = TaskState(key, spec)
+        ts._state = state
+
+        tp: TaskPrefix
+        prefix_key = key_split(key)
+        tp = self._task_prefixes.get(prefix_key)
+        if tp is None:
+            self._task_prefixes[prefix_key] = tp = TaskPrefix(prefix_key)
+        ts._prefix = tp
+
+        tg: TaskGroup
+        group_key = ts._group_key
+        tg = self._task_groups.get(group_key)
+        if tg is None:
+            self._task_groups[group_key] = tg = TaskGroup(group_key)
+            tg._prefix = tp
+            tp._groups.append(tg)
+        tg.add(ts)
+
+        self._tasks[key] = ts
+
+        return ts
+
     def transition_released_waiting(self, key):
         try:
             ts: TaskState = self._tasks[key]
@@ -3777,7 +3804,7 @@ def update_graph(
             # XXX Have a method get_task_state(self, k) ?
             ts = parent._tasks.get(k)
             if ts is None:
-                ts = self.new_task(k, tasks.get(k), "released")
+                ts = parent.new_task(k, tasks.get(k), "released")
             elif not ts._run_spec:
                 ts._run_spec = tasks.get(k)
 
@@ -3951,33 +3978,6 @@ def update_graph(
 
         # TODO: balance workers
 
-    def new_task(self, key: str, spec: object, state: str):
-        """ Create a new task, and associated states """
-        parent: SchedulerState = cast(SchedulerState, self)
-
-        ts: TaskState = TaskState(key, spec)
-        ts._state = state
-
-        tp: TaskPrefix
-        prefix_key = key_split(key)
-        tp = parent._task_prefixes.get(prefix_key)
-        if tp is None:
-            parent._task_prefixes[prefix_key] = tp = TaskPrefix(prefix_key)
-        ts._prefix = tp
-
-        tg: TaskGroup
-        group_key = ts._group_key
-        tg = parent._task_groups.get(group_key)
-        if tg is None:
-            parent._task_groups[group_key] = tg = TaskGroup(group_key)
-            tg._prefix = tp
-            tp._groups.append(tg)
-        tg.add(ts)
-
-        parent._tasks[key] = ts
-
-        return ts
-
     def stimulus_task_finished(self, key=None, worker=None, **kwargs):
         """ Mark that a task has finished execution on a particular worker """
         parent: SchedulerState = cast(SchedulerState, self)
@@ -4283,7 +4283,7 @@ def client_desires_keys(self, keys=None, client=None):
             ts = parent._tasks.get(k)
             if ts is None:
                 # For publish, queues etc.
-                ts = self.new_task(k, None, "released")
+                ts = parent.new_task(k, None, "released")
             ts._who_wants.add(cs)
             cs._wants_what.add(ts)
 
@@ -5636,7 +5636,7 @@ def update_data(
             for key, workers in who_has.items():
                 ts: TaskState = parent._tasks.get(key)
                 if ts is None:
-                    ts: TaskState = self.new_task(key, None, "memory")
+                    ts: TaskState = parent.new_task(key, None, "memory")
                 ts.state = "memory"
                 ts_nbytes: Py_ssize_t = nbytes.get(key, -1)
                 if ts_nbytes >= 0:

From d24d62f1dafa482b04209c1ef25a2305d6c41539 Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Wed, 24 Feb 2021 18:39:47 +0000
Subject: [PATCH 1159/1550] Migrate from travis to github actions (#4504)

---
 .github/workflows/ci-docs.yaml                |   6 +-
 .github/workflows/ci-pre-commit.yml           |  11 +-
 .github/workflows/ci-windows.yaml             |  61 ----------
 .github/workflows/ssh_debug.yaml              |  47 ++++++++
 .github/workflows/tests-macos.yaml            |  70 +++++++++++
 .github/workflows/tests.yaml                  |  83 +++++++++++++
 .gitignore                                    |   2 +
 .travis.yml                                   |  34 ------
 README.rst                                    |  10 +-
 continuous_integration/environment-3.6.yaml   |  47 ++++++++
 continuous_integration/environment-3.7.yaml   |  45 +++++++
 continuous_integration/environment-3.8.yaml   |  52 +++++++++
 .../environment-windows.yml                   |  35 ------
 continuous_integration/scripts/set_ulimit.sh  |   9 ++
 continuous_integration/scripts/setup_ssh.sh   |  13 +++
 continuous_integration/travis/install.sh      | 110 ------------------
 continuous_integration/travis/run_tests.sh    |  25 ----
 continuous_integration/travis/setup-ssh.sh    |   2 -
 distributed/batched.py                        |   2 +-
 distributed/cli/tests/test_dask_scheduler.py  |   8 +-
 distributed/cli/tests/test_dask_worker.py     |   2 +-
 distributed/comm/inproc.py                    |   6 +-
 distributed/comm/tcp.py                       |  17 ++-
 distributed/comm/tests/test_comms.py          |   3 +-
 distributed/compatibility.py                  |   1 +
 .../dashboard/tests/test_scheduler_bokeh.py   |   2 +-
 distributed/deploy/tests/test_old_ssh.py      |   2 +-
 distributed/deploy/tests/test_spec_cluster.py |   8 +-
 distributed/deploy/tests/test_ssh.py          |  12 +-
 distributed/diagnostics/tests/test_widgets.py |   2 +
 distributed/protocol/tests/test_pickle.py     |   6 +-
 distributed/tests/test_actor.py               |   6 +-
 distributed/tests/test_as_completed.py        |   3 +-
 distributed/tests/test_asyncprocess.py        |  15 +--
 distributed/tests/test_client.py              |   6 +-
 distributed/tests/test_client_executor.py     |   3 +-
 distributed/tests/test_diskutils.py           |   4 +-
 distributed/tests/test_failed_workers.py      |   7 +-
 distributed/tests/test_nanny.py               |   4 +-
 distributed/tests/test_pubsub.py              |   2 +-
 distributed/tests/test_resources.py           |   3 +-
 distributed/tests/test_scheduler.py           |  33 +++---
 distributed/tests/test_semaphore.py           |   5 +-
 distributed/tests/test_steal.py               |  21 ++--
 distributed/tests/test_stress.py              |  10 +-
 distributed/tests/test_threadpoolexecutor.py  |  12 +-
 distributed/tests/test_utils_perf.py          |   2 +-
 distributed/tests/test_worker.py              |  26 +++--
 distributed/utils_test.py                     |  18 +--
 setup.cfg                                     |  10 +-
 50 files changed, 530 insertions(+), 393 deletions(-)
 delete mode 100644 .github/workflows/ci-windows.yaml
 create mode 100644 .github/workflows/ssh_debug.yaml
 create mode 100644 .github/workflows/tests-macos.yaml
 create mode 100644 .github/workflows/tests.yaml
 delete mode 100644 .travis.yml
 create mode 100644 continuous_integration/environment-3.6.yaml
 create mode 100644 continuous_integration/environment-3.7.yaml
 create mode 100644 continuous_integration/environment-3.8.yaml
 delete mode 100644 continuous_integration/environment-windows.yml
 create mode 100644 continuous_integration/scripts/set_ulimit.sh
 create mode 100644 continuous_integration/scripts/setup_ssh.sh
 delete mode 100644 continuous_integration/travis/install.sh
 delete mode 100644 continuous_integration/travis/run_tests.sh
 delete mode 100644 continuous_integration/travis/setup-ssh.sh

diff --git a/.github/workflows/ci-docs.yaml b/.github/workflows/ci-docs.yaml
index c519427f140..625e496cad3 100644
--- a/.github/workflows/ci-docs.yaml
+++ b/.github/workflows/ci-docs.yaml
@@ -1,4 +1,4 @@
-name: Documentation CI
+name: Documentation
 
 on: [push, pull_request]
 
@@ -24,5 +24,5 @@ jobs:
 
       - name: Build docs
         run: |
-            cd docs
-            make html
+          cd docs
+          make html
diff --git a/.github/workflows/ci-pre-commit.yml b/.github/workflows/ci-pre-commit.yml
index 630ae98fac4..3d1e66571f3 100644
--- a/.github/workflows/ci-pre-commit.yml
+++ b/.github/workflows/ci-pre-commit.yml
@@ -1,4 +1,4 @@
-name: pre-commit
+name: Linting
 
 on:
   push:
@@ -7,9 +7,10 @@ on:
     branches: master
 
 jobs:
-  build:
+  checks:
+    name: pre-commit hooks
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
-    - uses: actions/setup-python@v2
-    - uses: pre-commit/action@v2.0.0
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+      - uses: pre-commit/action@v2.0.0
diff --git a/.github/workflows/ci-windows.yaml b/.github/workflows/ci-windows.yaml
deleted file mode 100644
index 6d5923eb187..00000000000
--- a/.github/workflows/ci-windows.yaml
+++ /dev/null
@@ -1,61 +0,0 @@
-name: Windows CI
-
-on: [push, pull_request]
-
-jobs:
-  build:
-    runs-on: windows-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version: ["3.6", "3.7", "3.8"]
-
-    steps:
-      - name: Checkout source
-        uses: actions/checkout@v2
-
-      - name: Setup Conda Environment
-        uses: conda-incubator/setup-miniconda@v2
-        with:
-          miniconda-version: "latest"
-          python-version: ${{ matrix.python-version }}
-          environment-file: continuous_integration/environment-windows.yml
-          activate-environment: dask-distributed
-          auto-activate-base: false
-
-      - name: Install contextvars
-        shell: bash -l {0}
-        run: |
-          if [[ "${{ matrix.python-version }}" = "3.6" ]]; then
-            conda install -c conda-forge contextvars
-          fi
-
-      - name: Install tornado
-        shell: bash -l {0}
-        run: |
-          if [[ "${{ matrix.python-version }}" = "3.8" ]]; then
-            conda install -c conda-forge tornado=6
-          else
-            conda install -c conda-forge tornado=5
-          fi
-
-      - name: Install distributed from source
-        shell: bash -l {0}
-        run: python -m pip install -q --no-deps -e .
-
-      - name: List packages in environment
-        shell: bash -l {0}
-        run: conda list
-
-      - name: Optionally Cythonize
-        shell: bash -l {0}
-        run: |
-          if [[ "${{ matrix.python-version }}" = "3.7" ]]; then
-            python setup.py build_ext --with-cython
-          fi
-
-      - name: Run tests
-        shell: bash -l {0}
-        env:
-          PYTHONFAULTHANDLER: 1
-        run: py.test -m "not avoid_travis" distributed -r s --timeout-method=thread --timeout=300 --durations=20
diff --git a/.github/workflows/ssh_debug.yaml b/.github/workflows/ssh_debug.yaml
new file mode 100644
index 00000000000..473e98ae973
--- /dev/null
+++ b/.github/workflows/ssh_debug.yaml
@@ -0,0 +1,47 @@
+name: Debug passwordless `ssh localhost`
+
+on: []
+# on: [pull_request]  # Uncomment to enable
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - ubuntu-latest
+          - macos-latest
+          # - windows-latest  # FIXME https://github.com/dask/distributed/issues/4509
+
+    steps:
+      - name: Setup SSH
+        shell: bash -l {0}
+        run: bash continuous_integration/scripts/setup_ssh.sh
+
+      - name: Dump SSH config
+        shell: bash -l {0}
+        run: |
+          ls -ld ~ ~/.ssh ~/.ssh/*
+          for f in ~/.ssh/* /etc/ssh/sshd_config; do
+            echo ==================================
+            echo $f
+            echo ==================================
+            cat $f
+          done
+
+      - name: Test SSH vs. localhost
+        shell: bash -l {0}
+        run: ssh -vvv localhost 'echo hello world'
+
+      - name: Test SSH vs. 127.0.0.1
+        shell: bash -l {0}
+        run: ssh -vvv 127.0.0.1 'echo hello world'
+
+      - name: Test SSH vs. hostname
+        shell: bash -l {0}
+        run: ssh -vvv $(hostname) 'echo hello world'
+
+      - name: Debug with tmate on failure
+        if: ${{ failure() }}
+        uses: mxschmitt/action-tmate@v3
diff --git a/.github/workflows/tests-macos.yaml b/.github/workflows/tests-macos.yaml
new file mode 100644
index 00000000000..a89ce4e616e
--- /dev/null
+++ b/.github/workflows/tests-macos.yaml
@@ -0,0 +1,70 @@
+name: MacOS tests
+
+on:
+  - push
+  # MacOS tests are currently afflicted by a very high level of flakiness; do not run on
+  # PR unless this line is uncommented.
+  # - pull_request
+
+jobs:
+  test:
+    runs-on: macos-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        # Many deterministic test failures on Python 3.6
+        python-version: ["3.7", "3.8"]
+
+        # Uncomment to stress-test the test suite for random failures
+        # This will take a LONG time and delay all PRs across the whole github.com/dask!
+        # run: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]
+
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v2
+
+      - name: Setup Conda Environment
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          miniconda-version: latest
+          python-version: ${{ matrix.python-version }}
+          environment-file: continuous_integration/environment-${{ matrix.python-version }}.yaml
+          activate-environment: dask-distributed
+          auto-activate-base: false
+
+      - name: Cythonize
+        shell: bash -l {0}
+        if: ${{ matrix.python-version == '3.7' }}
+        run: python setup.py build_ext --with-cython
+
+      - name: Install
+        shell: bash -l {0}
+        run: python -m pip install --no-deps -e .
+
+      - name: conda list
+        shell: bash -l {0}
+        run: conda list
+
+      - name: conda env export
+        shell: bash -l {0}
+        run: |
+          echo -e "--\n--Conda Environment (re-create this with \`conda env create --name <name> -f <output_file>\`)\n--"
+          conda env export | grep -E -v '^prefix:.*$'
+
+      - name: Setup SSH
+        shell: bash -l {0}
+        run: bash continuous_integration/scripts/setup_ssh.sh
+
+      - name: Test
+        shell: bash -l {0}
+        env:
+          PYTHONFAULTHANDLER: 1
+          MARKERS: not avoid_ci
+        run: |
+          source continuous_integration/scripts/set_ulimit.sh
+          pytest distributed -m "not avoid_ci" --runslow
+
+      # - name: Debug with tmate on failure
+      #   if: ${{ failure() }}
+      #   uses: mxschmitt/action-tmate@v3
diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
new file mode 100644
index 00000000000..d189e291374
--- /dev/null
+++ b/.github/workflows/tests.yaml
@@ -0,0 +1,83 @@
+name: Tests
+
+on: [push, pull_request]
+
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        # MacOS tests are currently afflicted by a very high level of flakiness. On
+        # Linux and Windows, run on merge to master and on all pull requests.
+        # On MacOS, run only on merge to master (see twin workflow tests-macos.yaml).
+        os: [ubuntu-latest, windows-latest]
+        python-version: ["3.6", "3.7", "3.8"]
+
+        # Uncomment to stress-test the test suite for random failures
+        # This will take a LONG time and delay all PRs across the whole github.com/dask!
+        # run: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]
+
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v2
+
+      - name: Setup Conda Environment
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          miniconda-version: latest
+          python-version: ${{ matrix.python-version }}
+          environment-file: continuous_integration/environment-${{ matrix.python-version }}.yaml
+          activate-environment: dask-distributed
+          auto-activate-base: false
+
+      - name: Install stacktrace
+        shell: bash -l {0}
+        # stacktrace for Python 3.8 has not been released at the moment of writing
+        if: ${{ matrix.os == 'ubuntu-latest' && matrix.python-version < '3.8' }}
+        run: conda install -c conda-forge -c defaults -c numba libunwind stacktrace
+
+      - name: Cythonize
+        shell: bash -l {0}
+        if: ${{ matrix.python-version == '3.7' }}
+        run: python setup.py build_ext --with-cython
+
+      - name: Install
+        shell: bash -l {0}
+        run: python -m pip install --no-deps -e .
+
+      - name: conda list
+        shell: bash -l {0}
+        run: conda list
+
+      - name: conda env export
+        shell: bash -l {0}
+        run: |
+          echo -e "--\n--Conda Environment (re-create this with \`conda env create --name <name> -f <output_file>\`)\n--"
+          conda env export | grep -E -v '^prefix:.*$'
+
+      - name: Setup SSH
+        shell: bash -l {0}
+        # FIXME no SSH available on Windows
+        # https://github.com/dask/distributed/issues/4509
+        if: ${{ matrix.os != 'windows-latest' }}
+        run: bash continuous_integration/scripts/setup_ssh.sh
+
+      - name: Test
+        shell: bash -l {0}
+        env:
+          PYTHONFAULTHANDLER: 1
+        run: |
+          if [[ "${{ matrix.os }}" = "ubuntu-latest" ]]; then
+              # FIXME ipv6-related failures on Ubuntu github actions CI
+              # https://github.com/dask/distributed/issues/4514
+              export DISABLE_IPV6=1
+          fi
+
+          source continuous_integration/scripts/set_ulimit.sh
+          pytest distributed -m "not avoid_ci" --runslow
+
+      # - name: Debug with tmate on failure
+      #   if: ${{ failure() }}
+      #   uses: mxschmitt/action-tmate@v3
diff --git a/.gitignore b/.gitignore
index 1c704bb6e77..ea068b3a900 100644
--- a/.gitignore
+++ b/.gitignore
@@ -10,8 +10,10 @@ docs/build
 continuous_integration/hdfs-initialized
 .DS_Store
 .cache
+.coverage
 *.lock
 .#*
+.eggs/
 .idea/
 .vscode/
 .pytest_cache/
diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index 850fcf23d53..00000000000
--- a/.travis.yml
+++ /dev/null
@@ -1,34 +0,0 @@
-language: generic
-# sudo shouldn't be required, but currently tests fail when run in a container
-# on travis instead of a vm. See https://github.com/dask/distributed/pull/1563.
-sudo: required
-dist: trusty
-
-env:
-  matrix:
-    - PYTHON=3.6 TESTS=true COVERAGE=true PACKAGES="lz4" TORNADO=5 CRICK=true
-    - PYTHON=3.7 TESTS=true PACKAGES="python-snappy python-blosc" TORNADO=6
-    - PYTHON=3.8 TESTS=true PACKAGES="python-snappy python-blosc" TORNADO=6
-
-matrix:
-  fast_finish: true
-  include:
-  - os: osx
-    env: PYTHON=3.7 TESTS=true PACKAGES="python-snappy python-blosc" TORNADO=6
-    if: type != pull_request OR commit_message =~ test-osx  # Skip on PRs unless the commit message contains "test-osx"
-
-  allow_failures:
-    - os: osx
-
-install:
-  - if [[ $TESTS == true ]]; then source continuous_integration/travis/install.sh ; fi
-  - if [[ $TESTS == true ]]; then source continuous_integration/travis/setup-ssh.sh ; fi
-
-script:
-  - if [[ $TESTS == true ]]; then source continuous_integration/travis/run_tests.sh ; fi
-
-after_success:
-  - if [[ $COVERAGE == true ]]; then coverage report; python -m pip install -q coveralls ; coveralls ; fi
-
-notifications:
-  email: false
diff --git a/README.rst b/README.rst
index eb4c601bc8b..4317f34a29e 100644
--- a/README.rst
+++ b/README.rst
@@ -1,15 +1,15 @@
 Distributed
 ===========
 
-|Linux Build Status| |Windows Build Status| |Doc Status| |Gitter| |Version Status| |NumFOCUS|
+|Linux/Windows Build Status| |MacOS Build Status| |Doc Status| |Gitter| |Version Status| |NumFOCUS|
 
 A library for distributed computation.  See documentation_ for more details.
 
 .. _documentation: https://distributed.dask.org
-.. |Linux Build Status| image:: https://travis-ci.org/dask/distributed.svg?branch=master
-   :target: https://travis-ci.org/dask/distributed
-.. |Windows Build Status| image:: https://github.com/dask/distributed/workflows/Windows%20CI/badge.svg?branch=master
-   :target: https://github.com/dask/distributed/actions?query=workflow%3A%22Windows+CI%22
+.. |Linux/Windows Build Status| image:: https://github.com/dask/distributed/workflows/Tests/badge.svg?branch=master
+   :target: https://github.com/dask/distributed/actions?query=workflow%3A%22Tests%22
+.. |MacOS Build Status| image:: https://github.com/dask/distributed/workflows/MacOS%20tests/badge.svg?branch=master
+   :target: https://github.com/dask/distributed/actions?query=workflow%3A%22MacOS+tests%22
 .. |Doc Status| image:: https://readthedocs.org/projects/distributed/badge/?version=latest
    :target: https://distributed.dask.org
    :alt: Documentation Status
diff --git a/continuous_integration/environment-3.6.yaml b/continuous_integration/environment-3.6.yaml
new file mode 100644
index 00000000000..7a13f63b232
--- /dev/null
+++ b/continuous_integration/environment-3.6.yaml
@@ -0,0 +1,47 @@
+name: dask-distributed
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  - python=3.6
+  - pip
+  - asyncssh
+  - bokeh
+  - click
+  - cloudpickle
+  - contextvars  # Backport from Python 3.7
+  - dask  # overridden by git tip below
+  - filesystem-spec
+  - h5py
+  - ipykernel
+  - ipywidgets
+  - joblib
+  - jupyter_client
+  - msgpack-python
+  - netcdf4
+  - paramiko
+  - pickle5  # Backport from Python 3.8
+  - prometheus_client
+  - psutil
+  - pytest
+  - pytest-asyncio<0.14.0
+  - pytest-faulthandler
+  - pytest-repeat
+  - pytest-rerunfailures
+  - pytest-timeout
+  - requests
+  - s3fs
+  - scikit-learn
+  - scipy
+  - sortedcollections
+  - tblib
+  - toolz
+  - tornado=5  # Only tested here
+  - zict
+  - zstandard
+  - pip:
+      # The '.git@master' is a hack needed to work around the failure
+      #     ERROR: Package 'dask' requires a different Python: 3.6.12 not in '>=3.6'
+      # It only happens in CI; cannot reproduce locally
+      - git+https://github.com/dask/dask.git@master
+      - keras
diff --git a/continuous_integration/environment-3.7.yaml b/continuous_integration/environment-3.7.yaml
new file mode 100644
index 00000000000..7057155b470
--- /dev/null
+++ b/continuous_integration/environment-3.7.yaml
@@ -0,0 +1,45 @@
+name: dask-distributed
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  - python=3.7
+  - pip
+  - asyncssh
+  - bokeh
+  - click
+  - cloudpickle
+  - cython  # Only tested here; also a dependency of crick
+  - dask  # overridden by git tip below
+  - filesystem-spec
+  - h5py
+  - ipykernel
+  - ipywidgets
+  - joblib
+  - jupyter_client
+  - msgpack-python
+  - netcdf4
+  - paramiko
+  - pickle5  # Backport from Python 3.8
+  - prometheus_client
+  - psutil
+  - pytest
+  - pytest-asyncio<0.14.0
+  - pytest-faulthandler
+  - pytest-repeat
+  - pytest-rerunfailures
+  - pytest-timeout
+  - requests
+  - s3fs
+  - scikit-learn
+  - scipy
+  - sortedcollections
+  - tblib
+  - toolz
+  - tornado=6
+  - zict
+  - zstandard
+  - pip:
+      - git+https://github.com/dask/dask
+      - git+https://github.com/jcrist/crick  # Only tested here
+      - keras
diff --git a/continuous_integration/environment-3.8.yaml b/continuous_integration/environment-3.8.yaml
new file mode 100644
index 00000000000..153a47d8756
--- /dev/null
+++ b/continuous_integration/environment-3.8.yaml
@@ -0,0 +1,52 @@
+name: dask-distributed
+channels:
+  - conda-forge
+  - defaults
+  - pytorch
+dependencies:
+  - python=3.8
+  - pip
+  - asyncssh
+  - bokeh
+  - click
+  - cloudpickle
+  - dask  # overridden by git tip below
+  - filesystem-spec  # overridden by git tip below
+  - h5py
+  - ipykernel
+  - ipywidgets
+  - joblib  # overridden by git tip below
+  - jupyter_client
+  - lz4  # Only tested here
+  - msgpack-python
+  - netcdf4
+  - paramiko
+  - prometheus_client
+  - psutil
+  - pytest
+  - pytest-asyncio<0.14.0
+  - pytest-faulthandler
+  - pytest-repeat
+  - pytest-rerunfailures
+  - pytest-timeout
+  - python-blosc  # Only tested here
+  - python-snappy  # Only tested here
+  - pytorch  # Only tested here
+  - requests
+  - s3fs  # overridden by git tip below
+  - scikit-learn
+  - scipy
+  - sortedcollections
+  - tblib
+  - toolz
+  - torchvision  # Only tested here
+  - tornado=6
+  - zict  # overridden by git tip below
+  - zstandard
+  - pip:
+      - git+https://github.com/dask/dask
+      - git+https://github.com/dask/s3fs
+      - git+https://github.com/dask/zict
+      - git+https://github.com/intake/filesystem_spec
+      - git+https://github.com/joblib/joblib
+      - keras
diff --git a/continuous_integration/environment-windows.yml b/continuous_integration/environment-windows.yml
deleted file mode 100644
index 5dd3043ccdc..00000000000
--- a/continuous_integration/environment-windows.yml
+++ /dev/null
@@ -1,35 +0,0 @@
-name: dask-distributed
-channels:
-  - conda-forge
-  - defaults
-dependencies:
-  - zstandard
-  - bokeh!=2.0.0
-  - click
-  - cloudpickle
-  - cython
-  - dask
-  - lz4
-  - ipykernel
-  - ipywidgets
-  - joblib
-  - jupyter_client
-  - msgpack-python>=0.6.0
-  - prometheus_client
-  - psutil
-  - pytest
-  - pytest-asyncio<0.14.0
-  - pytest-repeat
-  - pytest-timeout
-  - pytest-faulthandler
-  - requests
-  - sortedcollections
-  - toolz
-  - tblib
-  - zict
-  - fsspec
-  - pip
-  - pip:
-      - git+https://github.com/dask/dask.git@master
-      - git+https://github.com/joblib/joblib.git@master
-      - git+https://github.com/dask/zict.git@main
diff --git a/continuous_integration/scripts/set_ulimit.sh b/continuous_integration/scripts/set_ulimit.sh
new file mode 100644
index 00000000000..d9aea5d3395
--- /dev/null
+++ b/continuous_integration/scripts/set_ulimit.sh
@@ -0,0 +1,9 @@
+# On OS X builders, the default open files limit is too small (256)
+if [[ "$OSTYPE" == "darwin"* ]]; then
+    ulimit -n 8192
+fi
+
+echo "-- Soft limits"
+ulimit -a -S
+echo "-- Hard limits"
+ulimit -a -H
diff --git a/continuous_integration/scripts/setup_ssh.sh b/continuous_integration/scripts/setup_ssh.sh
new file mode 100644
index 00000000000..b85f2e2b1ad
--- /dev/null
+++ b/continuous_integration/scripts/setup_ssh.sh
@@ -0,0 +1,13 @@
+# Enable `ssh localhost` without a password.
+# If you change this script, make sure to retest both on Ubuntu and on MacOSX!
+# See .github/workflows/ssh_debug.yaml
+
+set -o errexit
+set -o nounset
+set -o xtrace
+
+mkdir -p ~/.ssh
+chmod 700 ~ ~/.ssh
+ssh-keygen -t rsa -f ~/.ssh/id_rsa -N "" -q
+cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
+ssh-keyscan -H localhost 127.0.0.1 $(hostname) >> ~/.ssh/known_hosts
diff --git a/continuous_integration/travis/install.sh b/continuous_integration/travis/install.sh
deleted file mode 100644
index 223c32ddbaa..00000000000
--- a/continuous_integration/travis/install.sh
+++ /dev/null
@@ -1,110 +0,0 @@
-#
-# This file should be source'd, so as to update the caller's environment
-# (such as the PATH variable)
-#
-
-# Note we disable progress bars to make Travis log loading much faster
-
-# Set default variable values if unset
-# (useful when this script is not invoked by Travis)
-: ${PYTHON:=3.8}
-: ${TORNADO:=6}
-: ${PACKAGES:=python-snappy python-blosc}
-
-# Install conda
-case "$(uname -s)" in
-    'Darwin')
-        MINICONDA_FILENAME="Miniconda3-latest-MacOSX-x86_64.sh"
-        ;;
-    'Linux')
-        MINICONDA_FILENAME="Miniconda3-latest-Linux-x86_64.sh"
-        ;;
-    *)  ;;
-esac
-
-if ! which conda; then
-  wget https://repo.continuum.io/miniconda/$MINICONDA_FILENAME -O miniconda.sh
-  bash miniconda.sh -b -p $HOME/miniconda
-  export PATH="$HOME/miniconda/bin:$PATH"
-fi
-
-conda config --set always_yes yes --set quiet yes --set changeps1 no
-conda update conda
-
-# Create conda environment
-conda create -n dask-distributed -c conda-forge -c defaults \
-    asyncssh \
-    bokeh \
-    click \
-    coverage \
-    dask \
-    flake8 \
-    h5py \
-    ipykernel \
-    ipywidgets \
-    joblib \
-    jupyter_client \
-    'msgpack-python>=0.6.0' \
-    netcdf4 \
-    paramiko \
-    prometheus_client \
-    psutil \
-    'pytest>=4' \
-    'pytest-asyncio<0.14.0' \
-    pytest-faulthandler \
-    pytest-repeat \
-    pytest-timeout \
-    python=$PYTHON \
-    requests \
-    scikit-learn \
-    scipy \
-    sortedcollections \
-    'tblib>=1.5.0' \
-    toolz \
-    tornado=$TORNADO \
-    zstandard \
-    $PACKAGES
-
-source activate dask-distributed
-
-if [[ $PYTHON == 3.6 ]]; then
-  conda install -c conda-forge -c defaults contextvars
-fi
-
-if [[ $PYTHON == 3.8 ]]; then
-    # Install pytorch to run related tests
-    conda install -c pytorch -c conda-forge -c defaults pytorch torchvision
-fi
-
-if [[ $PYTHON != 3.8 ]]; then
-    # Install backport package for pickle protocol 5 support
-    conda install -c conda-forge -c defaults 'pickle5>=0.0.11'
-    # stacktrace is not currently avaiable for Python 3.8.
-    # Remove the version check block below when it is avaiable.
-    # For low-level profiler, install libunwind and stacktrace from conda-forge
-    # For stacktrace we use --no-deps to avoid upgrade of python
-    conda install -c conda-forge -c defaults libunwind
-    conda install --no-deps -c conda-forge -c defaults -c numba stacktrace
-fi
-
-python -m pip install -q git+https://github.com/dask/dask.git --upgrade --no-deps
-python -m pip install -q git+https://github.com/joblib/joblib.git --upgrade --no-deps
-python -m pip install -q git+https://github.com/intake/filesystem_spec.git --upgrade --no-deps
-python -m pip install -q git+https://github.com/dask/s3fs.git --upgrade --no-deps
-python -m pip install -q git+https://github.com/dask/zict.git@main --upgrade --no-deps
-python -m pip install -q keras --upgrade --no-deps
-
-if [[ $CRICK == true ]]; then
-    conda install -c conda-forge -c defaults cython
-    python -m pip install -q git+https://github.com/jcrist/crick.git
-fi
-
-# Install distributed
-python -m pip install --no-deps -e .
-
-# For debugging
-echo -e "--\n--Conda Environment (re-create this with \`conda env create --name <name> -f <output_file>\`)\n--"
-conda env export | grep -E -v '^prefix:.*$'
-
-echo -e "--\n--Pip Environment\n--"
-python -m pip list --format=columns
diff --git a/continuous_integration/travis/run_tests.sh b/continuous_integration/travis/run_tests.sh
deleted file mode 100644
index 1bf86545cef..00000000000
--- a/continuous_integration/travis/run_tests.sh
+++ /dev/null
@@ -1,25 +0,0 @@
-export PYTEST_OPTIONS="-r s --timeout-method=thread --timeout=300 --durations=20"
-if [[ $RUNSLOW != false ]]; then
-    export PYTEST_OPTIONS="$PYTEST_OPTIONS --runslow"
-fi
-
-# On OS X builders, the default open files limit is too small (256)
-if [[ $TRAVIS_OS_NAME == osx ]]; then
-    ulimit -n 8192
-fi
-
-echo "--"
-echo "-- Soft limits"
-echo "--"
-ulimit -a -S
-
-echo "--"
-echo "-- Hard limits"
-echo "--"
-ulimit -a -H
-
-if [[ $COVERAGE == true ]]; then
-    coverage run $(which py.test) distributed -m "not avoid_travis" $PYTEST_OPTIONS
-else
-    py.test -m "not avoid_travis" distributed $PYTEST_OPTIONS
-fi
diff --git a/continuous_integration/travis/setup-ssh.sh b/continuous_integration/travis/setup-ssh.sh
deleted file mode 100644
index f102612bc96..00000000000
--- a/continuous_integration/travis/setup-ssh.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-ssh-keygen -t rsa -f ~/.ssh/id_rsa -N "" -q
-cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
\ No newline at end of file
diff --git a/distributed/batched.py b/distributed/batched.py
index 89e99719e9c..7b2523fc1e0 100644
--- a/distributed/batched.py
+++ b/distributed/batched.py
@@ -133,7 +133,7 @@ def send(self, *msgs):
         This completes quickly and synchronously
         """
         if self.comm is not None and self.comm.closed():
-            raise CommClosedError
+            raise CommClosedError()
 
         self.message_count += len(msgs)
         self.buffer.extend(msgs)
diff --git a/distributed/cli/tests/test_dask_scheduler.py b/distributed/cli/tests/test_dask_scheduler.py
index 301f46fdd14..6be7f3c365d 100644
--- a/distributed/cli/tests/test_dask_scheduler.py
+++ b/distributed/cli/tests/test_dask_scheduler.py
@@ -187,9 +187,7 @@ def test_interface(loop):
                 assert all("127.0.0.1" == d["host"] for d in info["workers"].values())
 
 
-@pytest.mark.skipif(
-    sys.version_info < (3, 7), reason="Intermittent failure on old Python version"
-)
+@pytest.mark.flaky(reruns=10, reruns_delay=5)
 def test_pid_file(loop):
     def check_pidfile(proc, pidfile):
         start = time()
@@ -416,9 +414,7 @@ def test_version_option():
 def test_idle_timeout(loop):
     start = time()
     runner = CliRunner()
-    result = runner.invoke(
-        distributed.cli.dask_scheduler.main, ["--idle-timeout", "1s"]
-    )
+    runner.invoke(distributed.cli.dask_scheduler.main, ["--idle-timeout", "1s"])
     stop = time()
     assert 1 < stop - start < 10
 
diff --git a/distributed/cli/tests/test_dask_worker.py b/distributed/cli/tests/test_dask_worker.py
index 0055c38e6dd..f8bbeedfb97 100644
--- a/distributed/cli/tests/test_dask_worker.py
+++ b/distributed/cli/tests/test_dask_worker.py
@@ -73,7 +73,7 @@ def test_nanny_worker_port_range(loop):
                 start = time()
                 while len(c.scheduler_info()["workers"]) < nprocs:
                     sleep(0.1)
-                    assert time() - start < 5
+                    assert time() - start < 60
 
                 def get_port(dask_worker):
                     return dask_worker.port
diff --git a/distributed/comm/inproc.py b/distributed/comm/inproc.py
index 24d39d4ad2b..d93377975ba 100644
--- a/distributed/comm/inproc.py
+++ b/distributed/comm/inproc.py
@@ -187,13 +187,13 @@ def peer_address(self):
 
     async def read(self, deserializers="ignored"):
         if self._closed:
-            raise CommClosedError
+            raise CommClosedError()
 
         msg = await self._read_q.get()
         if msg is _EOF:
             self._closed = True
             self._finalizer.detach()
-            raise CommClosedError
+            raise CommClosedError()
 
         if self.deserialize:
             msg = nested_deserialize(msg)
@@ -201,7 +201,7 @@ async def read(self, deserializers="ignored"):
 
     async def write(self, msg, serializers=None, on_error=None):
         if self.closed():
-            raise CommClosedError
+            raise CommClosedError()
 
         # Ensure we feed the queue in the same thread it is read from.
         self._write_loop.add_callback(self._write_q.put_nowait, msg)
diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py
index 2ad6fee9b33..11a0a228f45 100644
--- a/distributed/comm/tcp.py
+++ b/distributed/comm/tcp.py
@@ -186,7 +186,7 @@ def peer_address(self):
     async def read(self, deserializers=None):
         stream = self.stream
         if stream is None:
-            raise CommClosedError
+            raise CommClosedError()
 
         fmt = "Q"
         fmt_size = struct.calcsize(fmt)
@@ -228,9 +228,8 @@ async def read(self, deserializers=None):
 
     async def write(self, msg, serializers=None, on_error="message"):
         stream = self.stream
-        bytes_since_last_yield = 0
         if stream is None:
-            raise CommClosedError
+            raise CommClosedError()
 
         frames = await to_frames(
             msg,
@@ -260,6 +259,8 @@ async def write(self, msg, serializers=None, on_error="message"):
             for each_frame in frames:
                 each_frame_nbytes = nbytes(each_frame)
                 if each_frame_nbytes:
+                    if stream._write_buffer is None:
+                        raise StreamClosedError()
                     stream._write_buffer.append(each_frame)
                     stream._total_write_index += each_frame_nbytes
 
@@ -271,12 +272,10 @@ async def write(self, msg, serializers=None, on_error="message"):
             if not shutting_down():
                 convert_stream_closed_error(self, e)
         except Exception:
-            # Some OSError or a another "low-level" exception. We do not really know what
-            # was already written to the underlying socket, so it is not even safe to retry
-            # here using the same stream. The only safe thing to do is to abort.
-            # (See also GitHub #4133).
-            if stream._write_buffer is None:
-                logger.info("tried to write message %s on closed stream", msg)
+            # Some OSError or a another "low-level" exception. We do not really know
+            # what was already written to the underlying socket, so it is not even safe
+            # to retry here using the same stream. The only safe thing to do is to
+            # abort. (See also GitHub #4133).
             self.abort()
             raise
 
diff --git a/distributed/comm/tests/test_comms.py b/distributed/comm/tests/test_comms.py
index e94767de2df..b7a2061af48 100644
--- a/distributed/comm/tests/test_comms.py
+++ b/distributed/comm/tests/test_comms.py
@@ -26,6 +26,7 @@
 )
 from distributed.comm.registry import backends, get_backend
 from distributed.comm.tcp import TCP, TCPBackend, TCPConnector
+from distributed.compatibility import WINDOWS
 from distributed.metrics import time
 from distributed.protocol import Serialized, deserialize, serialize, to_serialize
 from distributed.utils import get_ip, get_ipv6
@@ -1104,7 +1105,7 @@ def check_out(deserialize_flag, out_value):
     await check_connector_deserialize(addr, True, msg, partial(check_out, True))
 
 
-@pytest.mark.xfail(reason="intermittent failure on windows")
+@pytest.mark.flaky(reruns=10, reruns_delay=5, condition=WINDOWS)
 @pytest.mark.asyncio
 async def test_tcp_deserialize():
     await check_deserialize("tcp://")
diff --git a/distributed/compatibility.py b/distributed/compatibility.py
index fc64b7568b5..6141f3559a7 100644
--- a/distributed/compatibility.py
+++ b/distributed/compatibility.py
@@ -8,6 +8,7 @@
 logging_names.update(logging._nameToLevel)
 
 PYPY = platform.python_implementation().lower() == "pypy"
+MACOS = sys.platform == "darwin"
 WINDOWS = sys.platform.startswith("win")
 TORNADO6 = tornado.version_info[0] >= 6
 PY37 = sys.version_info[:2] >= (3, 7)
diff --git a/distributed/dashboard/tests/test_scheduler_bokeh.py b/distributed/dashboard/tests/test_scheduler_bokeh.py
index 22ff8d6ecc1..ed7cc299dec 100644
--- a/distributed/dashboard/tests/test_scheduler_bokeh.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh.py
@@ -530,7 +530,7 @@ def func(x):
     assert len(gp.node_source.data["x"]) == 0
 
 
-@gen_cluster(client=True, timeout=30)
+@gen_cluster(client=True)
 async def test_TaskGraph_complex(c, s, a, b):
     da = pytest.importorskip("dask.array")
     gp = TaskGraph(s)
diff --git a/distributed/deploy/tests/test_old_ssh.py b/distributed/deploy/tests/test_old_ssh.py
index e6960b3392d..7f3aaeadbbf 100644
--- a/distributed/deploy/tests/test_old_ssh.py
+++ b/distributed/deploy/tests/test_old_ssh.py
@@ -10,7 +10,7 @@
 from distributed.utils_test import loop  # noqa: F401
 
 
-@pytest.mark.avoid_travis
+@pytest.mark.avoid_ci
 def test_cluster(loop):
     with SSHCluster(
         scheduler_addr="127.0.0.1",
diff --git a/distributed/deploy/tests/test_spec_cluster.py b/distributed/deploy/tests/test_spec_cluster.py
index 0bd157d1546..7a0f60ba011 100644
--- a/distributed/deploy/tests/test_spec_cluster.py
+++ b/distributed/deploy/tests/test_spec_cluster.py
@@ -209,11 +209,11 @@ async def test_restart(cleanup):
                 cluster.scale(2)
                 await cluster
                 assert len(cluster.workers) == 2
-
                 await client.restart()
-                await asyncio.sleep(3)
-
-                assert len(cluster.workers) == 2
+                start = time()
+                while len(cluster.workers) < 2:
+                    await asyncio.sleep(0.5)
+                    assert time() < start + 60
 
 
 @pytest.mark.skipif(WINDOWS, reason="HTTP Server doesn't close out")
diff --git a/distributed/deploy/tests/test_ssh.py b/distributed/deploy/tests/test_ssh.py
index 2bed638ff49..376194549ce 100644
--- a/distributed/deploy/tests/test_ssh.py
+++ b/distributed/deploy/tests/test_ssh.py
@@ -4,10 +4,16 @@
 
 import sys
 import dask
-from dask.distributed import Client
+from distributed import Client
+from distributed.compatibility import MACOS, WINDOWS
 from distributed.deploy.ssh import SSHCluster
 from distributed.utils_test import loop  # noqa: F401
 
+pytestmark = [
+    pytest.mark.xfail(MACOS, reason="very high flakiness; see distributed/issues/4543"),
+    pytest.mark.skipif(WINDOWS, reason="no CI support; see distributed/issues/4509"),
+]
+
 
 def test_ssh_hosts_None():
     with pytest.raises(ValueError):
@@ -61,7 +67,7 @@ async def test_keywords():
             assert all(v["nthreads"] == 2 for v in d.values())
 
 
-@pytest.mark.avoid_travis
+@pytest.mark.avoid_ci
 def test_defer_to_old(loop):
     with pytest.warns(Warning):
         with SSHCluster(
@@ -74,7 +80,7 @@ def test_defer_to_old(loop):
             assert isinstance(c, OldSSHCluster)
 
 
-@pytest.mark.avoid_travis
+@pytest.mark.avoid_ci
 def test_old_ssh_wih_local_dir(loop):
     with pytest.warns(Warning):
         from distributed.deploy.old_ssh import SSHCluster as OldSSHCluster
diff --git a/distributed/diagnostics/tests/test_widgets.py b/distributed/diagnostics/tests/test_widgets.py
index 6064462d893..2505f986406 100644
--- a/distributed/diagnostics/tests/test_widgets.py
+++ b/distributed/diagnostics/tests/test_widgets.py
@@ -2,6 +2,7 @@
 
 pytest.importorskip("ipywidgets")
 
+from distributed.compatibility import WINDOWS
 from ipykernel.comm import Comm
 from ipywidgets import Widget
 
@@ -144,6 +145,7 @@ async def test_multi_progressbar_widget(c, s, a, b):
     assert sorted(capacities, reverse=True) == capacities
 
 
+@pytest.mark.flaky(reruns=10, reruns_delay=5, condition=WINDOWS)
 @gen_cluster()
 async def test_multi_progressbar_widget_after_close(s, a, b):
     s.update_graph(
diff --git a/distributed/protocol/tests/test_pickle.py b/distributed/protocol/tests/test_pickle.py
index ea2143c5358..d7a2ad6d1ad 100644
--- a/distributed/protocol/tests/test_pickle.py
+++ b/distributed/protocol/tests/test_pickle.py
@@ -117,11 +117,7 @@ def test_pickle_numpy():
         assert (deserialize(h, f) == x).all()
 
 
-@pytest.mark.xfail(
-    sys.version_info[:2] == (3, 8),
-    reason="Sporadic failure on Python 3.8",
-    strict=False,
-)
+@pytest.mark.flaky(reruns=10, reruns_delay=5, condition=sys.version_info[:2] == (3, 8))
 def test_pickle_functions():
     def make_closure():
         value = 1
diff --git a/distributed/tests/test_actor.py b/distributed/tests/test_actor.py
index 26421d0385c..7b91b3da1c3 100644
--- a/distributed/tests/test_actor.py
+++ b/distributed/tests/test_actor.py
@@ -465,7 +465,7 @@ def f(block, ps=None):
     print(format_time(end - start))
 
 
-@pytest.mark.xfail(reason="unknown")
+@pytest.mark.flaky(reruns=10, reruns_delay=5)
 @gen_cluster(client=True)
 async def test_compute(c, s, a, b):
     @dask.delayed
@@ -488,7 +488,7 @@ def check(counter, blanks):
     start = time()
     while a.data or b.data:
         await asyncio.sleep(0.01)
-        assert time() < start + 5
+        assert time() < start + 30
 
 
 def test_compute_sync(client):
@@ -515,7 +515,7 @@ def check(dask_worker):
     start = time()
     while any(client.run(check).values()):
         sleep(0.01)
-        assert time() < start + 2
+        assert time() < start + 30
 
 
 @gen_cluster(
diff --git a/distributed/tests/test_as_completed.py b/distributed/tests/test_as_completed.py
index ae257f9bb8e..3efa759fa36 100644
--- a/distributed/tests/test_as_completed.py
+++ b/distributed/tests/test_as_completed.py
@@ -202,6 +202,7 @@ async def test_as_completed_with_results_async(c, s, a, b):
     assert str(exc.value) == "hello!"
 
 
+@pytest.mark.flaky(reruns=10, reruns_delay=5)
 def test_as_completed_with_results_no_raise(client):
     x = client.submit(throws, 1)
     y = client.submit(inc, 5)
@@ -212,7 +213,7 @@ def test_as_completed_with_results_no_raise(client):
     res = list(ac)
 
     dd = {r[0]: r[1:] for r in res}
-    assert set(dd.keys()) == {y, x, z}
+    assert dd.keys() == {x, y, z}
     assert x.status == "error"
     assert y.status == "cancelled"
     assert z.status == "finished"
diff --git a/distributed/tests/test_asyncprocess.py b/distributed/tests/test_asyncprocess.py
index f5cfab17895..288ba58be86 100644
--- a/distributed/tests/test_asyncprocess.py
+++ b/distributed/tests/test_asyncprocess.py
@@ -12,6 +12,7 @@
 from tornado import gen
 from tornado.locks import Event
 
+from distributed.compatibility import WINDOWS
 from distributed.metrics import time
 from distributed.process import AsyncProcess
 from distributed.utils import mp_context
@@ -48,7 +49,7 @@ def threads_info(q):
     q.put(threading.current_thread().name)
 
 
-@pytest.mark.xfail(reason="Intermittent failure")
+@pytest.mark.xfail()
 @nodebug
 @gen_test()
 async def test_simple():
@@ -92,7 +93,7 @@ async def test_simple():
 
     # child should be stopping now
     t1 = time()
-    await proc.join(timeout=10)
+    await proc.join(timeout=30)
     dt = time() - t1
     assert dt <= 1.0
     assert not proc.is_alive()
@@ -154,12 +155,12 @@ async def test_exitcode():
     assert proc.exitcode is None
 
     q.put(5)
-    await proc.join(timeout=3.0)
+    await proc.join(timeout=30)
     assert not proc.is_alive()
     assert proc.exitcode == 5
 
 
-@pytest.mark.skipif(os.name == "nt", reason="POSIX only")
+@pytest.mark.skipif(WINDOWS, reason="POSIX only")
 @gen_test()
 async def test_signal():
     proc = AsyncProcess(target=exit_with_signal, args=(signal.SIGINT,))
@@ -168,7 +169,7 @@ async def test_signal():
     assert proc.exitcode is None
 
     await proc.start()
-    await proc.join(timeout=3.0)
+    await proc.join(timeout=30)
 
     assert not proc.is_alive()
     # Can be 255 with forkserver, see https://bugs.python.org/issue30589
@@ -177,7 +178,7 @@ async def test_signal():
     proc = AsyncProcess(target=wait)
     await proc.start()
     os.kill(proc.pid, signal.SIGTERM)
-    await proc.join(timeout=3.0)
+    await proc.join(timeout=30)
 
     assert not proc.is_alive()
     assert proc.exitcode in (-signal.SIGTERM, 255)
@@ -190,7 +191,7 @@ async def test_terminate():
     await proc.start()
     await proc.terminate()
 
-    await proc.join(timeout=3.0)
+    await proc.join(timeout=30)
     assert not proc.is_alive()
     assert proc.exitcode in (-signal.SIGTERM, 255)
 
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index b4e23ed7ea0..be77accc47d 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -55,7 +55,7 @@
     temp_default_client,
     get_task_metadata,
 )
-from distributed.compatibility import WINDOWS
+from distributed.compatibility import MACOS, WINDOWS
 
 from distributed.metrics import time
 from distributed.scheduler import Scheduler, KilledWorker, CollectTaskMetaDataPlugin
@@ -3665,6 +3665,7 @@ async def test_reconnect_timeout(c, s):
     assert "Failed to reconnect" in text
 
 
+@pytest.mark.avoid_ci(reason="hangs on github actions ubuntu-latest CI")
 @pytest.mark.slow
 @pytest.mark.skipif(WINDOWS, reason="num_fds not supported on windows")
 @pytest.mark.skipif(sys.version_info < (3, 7), reason="TODO: intermittent failures")
@@ -4486,6 +4487,7 @@ async def test_scatter_dict_workers(c, s, a, b):
     assert "a" in a.data or "a" in b.data
 
 
+@pytest.mark.flaky(reruns=10, reruns_delay=5, condition=MACOS)
 @pytest.mark.slow
 @gen_test()
 async def test_client_timeout():
@@ -4965,6 +4967,7 @@ def f():
     assert result == 2
 
 
+@pytest.mark.flaky(reruns=10, reruns_delay=5)
 @pytest.mark.slow
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 2, timeout=60)
 async def test_secede_balances(c, s, a, b):
@@ -4983,6 +4986,7 @@ def f(x):
     while not all(f.status == "finished" for f in futures):
         await asyncio.sleep(0.01)
         assert threading.active_count() < count + 50
+        assert time() < start + 60
 
     assert len(a.log) < 2 * len(b.log)
     assert len(b.log) < 2 * len(a.log)
diff --git a/distributed/tests/test_client_executor.py b/distributed/tests/test_client_executor.py
index e7e3fc24c7d..b55b86ce7c8 100644
--- a/distributed/tests/test_client_executor.py
+++ b/distributed/tests/test_client_executor.py
@@ -95,6 +95,7 @@ def test_wait(client):
         assert "hello" in str(errors[0])
 
 
+@pytest.mark.flaky(reruns=10, reruns_delay=5)
 def test_cancellation(client):
     with client.get_executor(pure=False) as e:
         fut = e.submit(time.sleep, 2.0)
@@ -228,7 +229,7 @@ def test_retries(client):
 
 
 def test_shutdown(loop):
-    with cluster(disconnect_timeout=10) as (s, [a, b]):
+    with cluster() as (s, [a, b]):
         with Client(s["address"], loop=loop) as client:
             # shutdown(wait=True) waits for pending tasks to finish
             e = client.get_executor()
diff --git a/distributed/tests/test_diskutils.py b/distributed/tests/test_diskutils.py
index f69485cfa46..f4afb6d66d4 100644
--- a/distributed/tests/test_diskutils.py
+++ b/distributed/tests/test_diskutils.py
@@ -11,7 +11,7 @@
 import pytest
 
 import dask
-from distributed.compatibility import WINDOWS
+from distributed.compatibility import MACOS, WINDOWS
 from distributed.diskutils import WorkSpace
 from distributed.metrics import time
 from distributed.utils import mp_context
@@ -272,6 +272,7 @@ def _test_workspace_concurrency(tmpdir, timeout, max_procs):
     return n_created, n_purged
 
 
+@pytest.mark.flaky(reruns=10, reruns_delay=5, condition=MACOS)
 @pytest.mark.slow
 def test_workspace_concurrency(tmpdir):
     if WINDOWS:
@@ -281,6 +282,7 @@ def test_workspace_concurrency(tmpdir):
     _test_workspace_concurrency(tmpdir, 5.0, 6)
 
 
+@pytest.mark.flaky(reruns=10, reruns_delay=5, condition=MACOS)
 @pytest.mark.slow
 def test_workspace_concurrency_intense(tmpdir):
     n_created, n_purged = _test_workspace_concurrency(tmpdir, 8.0, 16)
diff --git a/distributed/tests/test_failed_workers.py b/distributed/tests/test_failed_workers.py
index 32addfa68bc..717fe01c80d 100644
--- a/distributed/tests/test_failed_workers.py
+++ b/distributed/tests/test_failed_workers.py
@@ -10,7 +10,7 @@
 from dask import delayed
 from distributed import Client, Nanny, wait
 from distributed.comm import CommClosedError
-from distributed.client import wait
+from distributed.compatibility import MACOS
 from distributed.metrics import time
 from distributed.utils import sync, CancelledError
 from distributed.utils_test import (
@@ -26,7 +26,7 @@
 
 
 def test_submit_after_failed_worker_sync(loop):
-    with cluster(active_rpc_timeout=10, disconnect_timeout=10) as (s, [a, b]):
+    with cluster() as (s, [a, b]):
         with Client(s["address"], loop=loop) as c:
             L = c.map(inc, range(10))
             wait(L)
@@ -64,7 +64,7 @@ async def test_submit_after_failed_worker(c, s, a, b):
 
 
 def test_gather_after_failed_worker(loop):
-    with cluster(active_rpc_timeout=10, disconnect_timeout=10) as (s, [a, b]):
+    with cluster() as (s, [a, b]):
         with Client(s["address"], loop=loop) as c:
             L = c.map(inc, range(10))
             wait(L)
@@ -286,6 +286,7 @@ async def test_multiple_clients_restart(s, a, b):
     await c2.close()
 
 
+@pytest.mark.flaky(reruns=10, reruns_timeout=5, condition=MACOS)
 @gen_cluster(Worker=Nanny, timeout=60)
 async def test_restart_scheduler(s, a, b):
     import gc
diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
index f6277aa980b..d11e4b50d50 100644
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -16,6 +16,7 @@
 import dask
 from distributed.diagnostics import SchedulerPlugin
 from distributed import Nanny, rpc, Scheduler, Worker, Client, wait, worker
+from distributed.compatibility import MACOS
 from distributed.core import CommClosedError, Status
 from distributed.metrics import time
 from distributed.protocol.pickle import dumps
@@ -77,7 +78,7 @@ async def test_str(s, a, b):
     assert str(a.nthreads) in repr(a)
 
 
-@gen_cluster(nthreads=[], timeout=20, client=True)
+@gen_cluster(nthreads=[], client=True)
 async def test_nanny_process_failure(c, s):
     n = await Nanny(s.address, nthreads=2, loop=s.loop)
     first_dir = n.worker_dir
@@ -568,6 +569,7 @@ async def start(self):
         raise StartException("broken")
 
 
+@pytest.mark.flaky(reruns=10, reruns_delay=5, condition=MACOS)
 @pytest.mark.asyncio
 async def test_worker_start_exception(cleanup):
     # make sure this raises the right Exception:
diff --git a/distributed/tests/test_pubsub.py b/distributed/tests/test_pubsub.py
index 6d005a3ae3e..8f8a3b734e7 100644
--- a/distributed/tests/test_pubsub.py
+++ b/distributed/tests/test_pubsub.py
@@ -138,7 +138,7 @@ async def test_repr(c, s, a, b):
     assert "Sub" in str(sub)
 
 
-@pytest.mark.xfail(reason="out of order execution")
+@pytest.mark.xfail(reason="flaky and re-fails on reruns; out of order execution")
 @gen_cluster(client=True)
 async def test_basic(c, s, a, b):
     async def publish():
diff --git a/distributed/tests/test_resources.py b/distributed/tests/test_resources.py
index a7332c4a92e..68837e30a14 100644
--- a/distributed/tests/test_resources.py
+++ b/distributed/tests/test_resources.py
@@ -369,7 +369,6 @@ async def test_full_collections(c, s, a, b):
     assert not b.log
 
 
-@pytest.mark.xfail(reason="atop fusion seemed to break this")
 @pytest.mark.parametrize(
     "optimize_graph",
     [
@@ -380,7 +379,7 @@ async def test_full_collections(c, s, a, b):
             ),
         ),
         pytest.param(
-            False, marks=pytest.mark.skipif(WINDOWS, reason="intermittent failure")
+            False, marks=pytest.mark.flaky(reruns=10, reruns_delay=5, condition=WINDOWS)
         ),
     ],
 )
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 49f488d9673..0d4cad348c2 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -16,6 +16,7 @@
 
 from distributed import Nanny, Worker, Client, wait, fire_and_forget
 from distributed.comm import Comm
+from distributed.compatibility import MACOS
 from distributed.core import connect, rpc, ConnectionPool, Status
 from distributed.scheduler import Scheduler
 from distributed.client import wait
@@ -1104,7 +1105,7 @@ async def test_close_worker(c, s, a, b):
 
 
 @pytest.mark.slow
-@gen_cluster(client=True, Worker=Nanny, timeout=20)
+@gen_cluster(client=True, Worker=Nanny)
 async def test_close_nanny(c, s, a, b):
     assert len(s.workers) == 2
 
@@ -1135,7 +1136,7 @@ async def test_close_nanny(c, s, a, b):
         assert time() < start + 10
 
 
-@gen_cluster(client=True, timeout=20)
+@gen_cluster(client=True)
 async def test_retire_workers_close(c, s, a, b):
     await s.retire_workers(close_workers=True)
     assert not s.workers
@@ -1143,7 +1144,7 @@ async def test_retire_workers_close(c, s, a, b):
         await asyncio.sleep(0.01)
 
 
-@gen_cluster(client=True, timeout=20, Worker=Nanny)
+@gen_cluster(client=True, Worker=Nanny)
 async def test_retire_nannies_close(c, s, a, b):
     nannies = [a, b]
     await s.retire_workers(close_workers=True, remove=True)
@@ -1183,7 +1184,7 @@ async def test_scheduler_file():
         await s.close()
 
 
-@pytest.mark.xfail(reason="")
+@pytest.mark.xfail()
 @gen_cluster(client=True, nthreads=[])
 async def test_non_existent_worker(c, s):
     with dask.config.set({"distributed.comm.timeouts.connect": "100ms"}):
@@ -1418,9 +1419,11 @@ async def test_retries(c, s, a, b):
     exc_info.match("one")
 
 
-@pytest.mark.xfail(reason="second worker also errant for some reason")
-@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 3, timeout=5)
-async def test_mising_data_errant_worker(c, s, w1, w2, w3):
+@pytest.mark.flaky(
+    reruns=10, reruns_delay=5, reason="second worker also errant for some reason"
+)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 3)
+async def test_missing_data_errant_worker(c, s, w1, w2, w3):
     with dask.config.set({"distributed.comm.timeouts.connect": "1s"}):
         np = pytest.importorskip("numpy")
 
@@ -1608,6 +1611,7 @@ async def test_collect_versions(c, s, a, b):
     assert cs.versions == w1.versions == w2.versions
 
 
+@pytest.mark.xfail(reason="flaky and re-fails on rerun")
 @gen_cluster(client=True, config={"distributed.scheduler.idle-timeout": "500ms"})
 async def test_idle_timeout(c, s, a, b):
     beginning = time()
@@ -2011,6 +2015,7 @@ async def test_gather_no_workers(c, s, a, b):
     assert list(res["keys"]) == ["x"]
 
 
+@pytest.mark.flaky(reruns=10, reruns_delay=5, condition=MACOS)
 @gen_cluster(client=True, client_kwargs={"direct_to_workers": False})
 async def test_gather_allow_worker_reconnect(c, s, a, b):
     """
@@ -2022,7 +2027,7 @@ async def test_gather_allow_worker_reconnect(c, s, a, b):
     its results instead of recomputing them.
     """
     # GH3246
-    ALREADY_CALCULATED = []
+    already_calculated = []
 
     import time
 
@@ -2030,9 +2035,9 @@ def inc_slow(x):
         # Once the graph below is rescheduled this computation runs again. We
         # need to sleep for at least 0.5 seconds to give the worker a chance to
         # reconnect (Heartbeat timing)
-        if x in ALREADY_CALCULATED:
+        if x in already_calculated:
             time.sleep(1)
-        ALREADY_CALCULATED.append(x)
+        already_calculated.append(x)
         return x + 1
 
     x = c.submit(inc_slow, 1)
@@ -2065,12 +2070,14 @@ def reducer(x, y):
     client_logger = client_logger.getvalue()
     utils_comm_logger = utils_comm_logger.getvalue()
 
-    # Ensure that the communication was done via the scheduler, i.e. we actually hit a bad connection
+    # Ensure that the communication was done via the scheduler, i.e. we actually hit a
+    # bad connection
     assert s.rpc.cnn_count > 0
 
     assert "Retrying get_data_from_worker after exception" in utils_comm_logger
 
-    # The reducer task was actually not found upon first collection. The client will reschedule the graph
+    # The reducer task was actually not found upon first collection. The client will
+    # reschedule the graph
     assert "Couldn't gather 1 keys, rescheduling" in client_logger
     # There will also be a `Unexpected worker completed task` message but this
     # is rather an artifact and not the intention
@@ -2168,7 +2175,7 @@ async def test_unknown_task_duration_config(s, a, b):
     assert s.idle_since == s.time_started
 
 
-@gen_cluster(client=True, timeout=1000)
+@gen_cluster(client=True, timeout=None)
 async def test_retire_state_change(c, s, a, b):
     np = pytest.importorskip("numpy")
     y = c.map(lambda x: x ** 2, range(10))
diff --git a/distributed/tests/test_semaphore.py b/distributed/tests/test_semaphore.py
index af2d59f3ea1..e23864d811f 100644
--- a/distributed/tests/test_semaphore.py
+++ b/distributed/tests/test_semaphore.py
@@ -7,6 +7,7 @@
 from time import time, sleep
 from distributed import Semaphore, fire_and_forget
 from distributed.comm import Comm
+from distributed.compatibility import WINDOWS
 from distributed.core import ConnectionPool
 from distributed.metrics import time
 from distributed.utils_test import (  # noqa: F401
@@ -136,7 +137,7 @@ async def test_async_ctx(s, a, b):
 
 @pytest.mark.slow
 def test_worker_dies():
-    with cluster(disconnect_timeout=10) as (scheduler, workers):
+    with cluster() as (scheduler, workers):
         with Client(scheduler["address"]) as client:
             sem = Semaphore(name="x", max_leases=1)
 
@@ -340,6 +341,7 @@ async def test_retry_acquire(c, s, a, b):
         assert result is False
 
 
+@pytest.mark.flaky(reruns=10, reruns_delay=5, condition=WINDOWS)
 @gen_cluster(
     client=True,
     config={
@@ -552,6 +554,7 @@ async def test_release_retry(c, s, a, b):
         assert await semaphore.release() is True
 
 
+@pytest.mark.flaky(reruns=10, reruns_delay=5, condition=WINDOWS)
 @gen_cluster(
     client=True,
     config={
diff --git a/distributed/tests/test_steal.py b/distributed/tests/test_steal.py
index bee0f06544e..ac37c0d5acf 100644
--- a/distributed/tests/test_steal.py
+++ b/distributed/tests/test_steal.py
@@ -34,7 +34,7 @@
 @pytest.mark.skipif(
     not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
 )
-@gen_cluster(client=True, nthreads=[("127.0.0.1", 2), ("127.0.0.2", 2)], timeout=20)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 2), ("127.0.0.2", 2)])
 async def test_work_stealing(c, s, a, b):
     [x] = await c._scatter([1], workers=a.address)
     futures = c.map(slowadd, range(50), [x] * 50)
@@ -73,7 +73,7 @@ async def test_steal_cheap_data_slow_computation(c, s, a, b):
     assert abs(len(a.data) - len(b.data)) <= 5
 
 
-@pytest.mark.avoid_travis
+@pytest.mark.avoid_ci
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 2)
 async def test_steal_expensive_data_slow_computation(c, s, a, b):
     np = pytest.importorskip("numpy")
@@ -107,7 +107,7 @@ async def test_worksteal_many_thieves(c, s, *workers):
     assert sum(map(len, s.has_what.values())) < 150
 
 
-@pytest.mark.xfail(reason="GH#3574")
+@pytest.mark.flaky(reruns=10, reruns_delay=5, reason="GH#3574")
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 2)
 async def test_dont_steal_unknown_functions(c, s, a, b):
     futures = c.map(inc, range(100), workers=a.address, allow_other_workers=True)
@@ -198,7 +198,7 @@ def fast_blacklisted(x, y=None):
     assert len(s.has_what[b.address]) == 1
 
 
-@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)], timeout=20)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)])
 async def test_new_worker_steals(c, s, a):
     await wait(c.submit(slowinc, 1, delay=0.01))
 
@@ -220,7 +220,7 @@ async def test_new_worker_steals(c, s, a):
     await b.close()
 
 
-@gen_cluster(client=True, timeout=20)
+@gen_cluster(client=True)
 async def test_work_steal_no_kwargs(c, s, a, b):
     await wait(c.submit(slowinc, 1, delay=0.05))
 
@@ -380,7 +380,7 @@ async def test_steal_resource_restrictions(c, s, a):
     await b.close()
 
 
-@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 5, timeout=20)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 5)
 async def test_balance_without_dependencies(c, s, *workers):
     s.extensions["stealing"]._pc.callback_time = 20
 
@@ -563,6 +563,7 @@ async def assert_balanced(inp, expected, c, s, *workers):
         pytest.param(
             [[1, 1, 1, 1, 1, 1, 1], [1, 1], [1, 1], [1, 1], []],
             [[1, 1, 1, 1, 1], [1, 1], [1, 1], [1, 1], [1, 1]],
+            # Can't mark as flaky as when it fails it does so every time for some reason
             marks=pytest.mark.xfail(
                 reason="Some uncertainty based on executing stolen task"
             ),
@@ -585,7 +586,7 @@ async def test(*args, **kwargs):
     test()
 
 
-@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 2, Worker=Nanny, timeout=20)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 2, Worker=Nanny)
 async def test_restart(c, s, a, b):
     futures = c.map(
         slowinc, range(100), delay=0.1, workers=a.address, allow_other_workers=True
@@ -737,11 +738,7 @@ def long(delay):
         ) <= 1
 
 
-@pytest.mark.xfail(
-    sys.version_info[:2] == (3, 8),
-    reason="Sporadic failure on Python 3.8",
-    strict=False,
-)
+@pytest.mark.flaky(reruns=10, reruns_delay=5, condition=sys.version_info[:2] == (3, 8))
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 5)] * 2)
 async def test_cleanup_repeated_tasks(c, s, a, b):
     class Foo:
diff --git a/distributed/tests/test_stress.py b/distributed/tests/test_stress.py
index d699ac9452d..2cc65b29ae0 100644
--- a/distributed/tests/test_stress.py
+++ b/distributed/tests/test_stress.py
@@ -171,7 +171,7 @@ def vsum(*args):
     return sum(args)
 
 
-@pytest.mark.avoid_travis
+@pytest.mark.avoid_ci
 @pytest.mark.slow
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 80, timeout=1000)
 async def test_stress_communication(c, s, *workers):
@@ -240,11 +240,13 @@ async def test_close_connections(c, s, *workers):
     await wait(future)
 
 
+@pytest.mark.slow
 @pytest.mark.xfail(
-    reason="IOStream._handle_write blocks on large write_buffer"
-    " https://github.com/tornadoweb/tornado/issues/2110"
+    reason="flaky and re-fails on rerun; "
+    "IOStream._handle_write blocks on large write_buffer"
+    " https://github.com/tornadoweb/tornado/issues/2110",
 )
-@gen_cluster(client=True, timeout=20, nthreads=[("127.0.0.1", 1)])
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)])
 async def test_no_delay_during_large_transfer(c, s, w):
     pytest.importorskip("crick")
     np = pytest.importorskip("numpy")
diff --git a/distributed/tests/test_threadpoolexecutor.py b/distributed/tests/test_threadpoolexecutor.py
index 8b807512168..6da81e39288 100644
--- a/distributed/tests/test_threadpoolexecutor.py
+++ b/distributed/tests/test_threadpoolexecutor.py
@@ -62,11 +62,16 @@ def test_shutdown_wait():
 
 def test_secede_rejoin_busy():
     with ThreadPoolExecutor(2) as e:
+        # Prime threads
+        f1 = e.submit(sleep, 0.1)
+        f2 = e.submit(sleep, 0.1)
+        f1.result()
+        f2.result()
 
         def f():
             assert threading.current_thread() in e._threads
             secede()
-            sleep(0.1)
+            sleep(0.2)
             assert threading.current_thread() not in e._threads
             rejoin()
             assert len(e._threads) == 2
@@ -74,12 +79,13 @@ def f():
             return threading.current_thread()
 
         future = e.submit(f)
-        L = [e.submit(sleep, 0.2) for i in range(10)]
+        for _ in range(6):
+            e.submit(sleep, 0.4)
         start = time()
         special_thread = future.result()
         stop = time()
 
-        assert 0.1 < stop - start < 0.3
+        assert 0.2 < stop - start < 0.6
 
         assert len(e._threads) == 2
         assert special_thread in e._threads
diff --git a/distributed/tests/test_utils_perf.py b/distributed/tests/test_utils_perf.py
index a1591df0280..82dfac5653e 100644
--- a/distributed/tests/test_utils_perf.py
+++ b/distributed/tests/test_utils_perf.py
@@ -110,7 +110,7 @@ def test_gc_diagnosis_cpu_time():
         assert not sio.getvalue()
 
 
-@pytest.mark.xfail(reason="unknown")
+@pytest.mark.xfail(reason="flaky and re-fails on rerun")
 def test_gc_diagnosis_rss_win():
     diag = GCDiagnosis(info_over_rss_win=10e6)
 
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 59685511d6f..46a9195a869 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -29,7 +29,7 @@
     wait,
 )
 from distributed.diagnostics.plugin import PipInstall
-from distributed.compatibility import WINDOWS
+from distributed.compatibility import MACOS, WINDOWS
 from distributed.core import rpc, CommClosedError, Status
 from distributed.scheduler import Scheduler
 from distributed.metrics import time
@@ -708,6 +708,7 @@ async def test_multiple_transfers(c, s, w1, w2, w3):
     assert len(transfers) == 2
 
 
+@pytest.mark.xfail(reason="very high flakiness")
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 3)
 async def test_share_communication(c, s, w1, w2, w3):
     x = c.submit(mul, b"1", int(w3.target_message_size + 1), workers=w1.address)
@@ -721,6 +722,7 @@ async def test_share_communication(c, s, w1, w2, w3):
     assert w2.outgoing_transfer_log
 
 
+@pytest.mark.xfail(reason="very high flakiness")
 @gen_cluster(client=True)
 async def test_dont_overlap_communications_to_same_worker(c, s, a, b):
     x = c.submit(mul, b"1", int(b.target_message_size + 1), workers=a.address)
@@ -734,7 +736,7 @@ async def test_dont_overlap_communications_to_same_worker(c, s, a, b):
     assert l1["stop"] < l2["start"]
 
 
-@pytest.mark.avoid_travis
+@pytest.mark.avoid_ci
 @gen_cluster(client=True)
 async def test_log_exception_on_failed_task(c, s, a, b):
     with tmpfile() as fn:
@@ -758,6 +760,7 @@ async def test_log_exception_on_failed_task(c, s, a, b):
             logger.removeHandler(fh)
 
 
+@pytest.mark.flaky(reruns=10, reruns_delay=5)
 @gen_cluster(client=True)
 async def test_clean_up_dependencies(c, s, a, b):
     x = delayed(inc)(1)
@@ -777,6 +780,7 @@ async def test_clean_up_dependencies(c, s, a, b):
     assert set(a.data) | set(b.data) == {zz.key}
 
 
+@pytest.mark.flaky(reruns=10, reruns_delay=5)
 @gen_cluster(client=True)
 async def test_hold_onto_dependents(c, s, a, b):
     x = c.submit(inc, 1, workers=a.address)
@@ -1082,6 +1086,7 @@ async def test_scheduler_delay(c, s, a, b):
     assert a.scheduler_delay != old
 
 
+@pytest.mark.flaky(reruns=10, reruns_delay=5, condition=MACOS)
 @gen_cluster(client=True)
 async def test_statistical_profiling(c, s, a, b):
     futures = c.map(slowinc, range(10), delay=0.1)
@@ -1145,11 +1150,7 @@ def f(n):
 
 
 @pytest.mark.slow
-@pytest.mark.xfail(
-    sys.version_info[:2] == (3, 8),
-    reason="Sporadic failure on Python 3.8",
-    strict=False,
-)
+@pytest.mark.flaky(reruns=10, reruns_delay=5, condition=sys.version_info[:2] == (3, 8))
 @gen_cluster(
     nthreads=[("127.0.0.1", 2)],
     client=True,
@@ -1308,6 +1309,7 @@ async def test_scheduler_address_config(c, s):
     await worker.close()
 
 
+@pytest.mark.xfail(reason="very high flakiness")
 @pytest.mark.slow
 @gen_cluster(client=True)
 async def test_wait_for_outgoing(c, s, a, b):
@@ -1681,6 +1683,7 @@ async def test_update_latency(cleanup):
                 assert w.digests["latency"].size() > 0
 
 
+@pytest.mark.skipif(MACOS, reason="frequently hangs")
 @pytest.mark.asyncio
 async def test_workerstate_executing(cleanup):
     async with await Scheduler() as s:
@@ -1727,13 +1730,12 @@ async def test_bad_local_directory(cleanup):
         try:
             async with Worker(s.address, local_directory="/not/a/valid-directory"):
                 pass
-        except PermissionError:
+        except OSError:
+            # On Linux: [Errno 13] Permission denied: '/not'
+            # On MacOSX: [Errno 30] Read-only file system: '/not'
             pass
         else:
-            if WINDOWS:
-                pass
-            else:
-                assert False
+            assert WINDOWS
 
         assert not any("error" in log for log in s.get_logs())
 
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index 9de0e342e7e..e253d7760c7 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -591,8 +591,8 @@ def cluster(
     nworkers=2,
     nanny=False,
     worker_kwargs={},
-    active_rpc_timeout=1,
-    disconnect_timeout=3,
+    active_rpc_timeout=10,
+    disconnect_timeout=20,
     scheduler_kwargs={},
 ):
     ws = weakref.WeakSet()
@@ -701,7 +701,7 @@ def cluster(
             scheduler.join(2)
             del scheduler
             for proc in [w["proc"] for w in workers]:
-                proc.join(timeout=2)
+                proc.join(timeout=30)
 
             with suppress(UnboundLocalError):
                 del worker, w, proc
@@ -829,7 +829,7 @@ def gen_cluster(
     nthreads=[("127.0.0.1", 1), ("127.0.0.1", 2)],
     ncores=None,
     scheduler="127.0.0.1",
-    timeout=10,
+    timeout=30,
     security=None,
     Worker=Worker,
     client=False,
@@ -873,7 +873,7 @@ def test_func():
                 async def coro():
                     with dask.config.set(config):
                         s = False
-                        for i in range(5):
+                        for _ in range(60):
                             try:
                                 s, ws = await start_cluster(
                                     nthreads,
@@ -886,7 +886,8 @@ async def coro():
                                 )
                             except Exception as e:
                                 logger.error(
-                                    "Failed to start gen_cluster, retrying",
+                                    "Failed to start gen_cluster: "
+                                    f"{e.__class__.__name__}: {e}; retrying",
                                     exc_info=True,
                                 )
                                 await asyncio.sleep(1)
@@ -934,7 +935,7 @@ def get_unclosed():
 
                         try:
                             start = time()
-                            while time() < start + 5:
+                            while time() < start + 60:
                                 gc.collect()
                                 if not get_unclosed():
                                     break
@@ -1073,6 +1074,9 @@ def has_ipv6():
     Return whether IPv6 is locally functional.  This doesn't guarantee IPv6
     is properly configured outside of localhost.
     """
+    if os.getenv("DISABLE_IPV6") == "1":
+        return False
+
     serv = cli = None
     try:
         serv = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
diff --git a/setup.cfg b/setup.cfg
index ca4417fabdb..0689b58795d 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -36,11 +36,13 @@ tag_prefix =
 parentdir_prefix = distributed-
 
 [tool:pytest]
-addopts = -rsx --durations=10
+addopts = -v -r s --durations=20
 filterwarnings =
     error:Since distributed.*:PendingDeprecationWarning
-minversion = 3.2
+minversion = 4
 markers =
     slow: marks tests as slow (deselect with '-m "not slow"')
-    avoid_travis: marks tests as flaky on TravisCI.
-    ipython: mark a test as exercising IPython
+    avoid_ci: marks tests as flaky on CI on all OSs
+    ipython: marks tests as exercising IPython
+timeout_method = thread
+timeout = 300

From cf9abafd5478a5f15001f999fe616b580f323da5 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Thu, 25 Feb 2021 16:40:21 -0800
Subject: [PATCH 1160/1550] Move `_transition*` to `SchedulerState` (#4545)

Refactors the `_transition*` methods out of the `Scheduler` and into
`SchedulerState`. These are primarily focused on graph updates. So make
more sense next to other task graph state. This also should make it
easier to optimize these further in the future.
---
 distributed/scheduler.py | 444 ++++++++++++++++++++-------------------
 1 file changed, 224 insertions(+), 220 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 77fb63ff0d7..7a50d847ba8 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1593,6 +1593,7 @@ class SchedulerState:
     _task_metadata: dict
     _total_nthreads: Py_ssize_t
     _total_occupancy: double
+    _transitions_table: dict
     _unknown_durations: dict
     _unrunnable: set
     _validate: bint
@@ -1645,6 +1646,23 @@ def __init__(
         self._task_metadata = dict()
         self._total_nthreads = 0
         self._total_occupancy = 0
+        self._transitions_table = {
+            ("released", "waiting"): self.transition_released_waiting,
+            ("waiting", "released"): self.transition_waiting_released,
+            ("waiting", "processing"): self.transition_waiting_processing,
+            ("waiting", "memory"): self.transition_waiting_memory,
+            ("processing", "released"): self.transition_processing_released,
+            ("processing", "memory"): self.transition_processing_memory,
+            ("processing", "erred"): self.transition_processing_erred,
+            ("no-worker", "released"): self.transition_no_worker_released,
+            ("no-worker", "waiting"): self.transition_no_worker_waiting,
+            ("released", "forgotten"): self.transition_released_forgotten,
+            ("memory", "forgotten"): self.transition_memory_forgotten,
+            ("erred", "forgotten"): self.transition_released_forgotten,
+            ("erred", "released"): self.transition_erred_released,
+            ("memory", "released"): self.transition_memory_released,
+            ("released", "erred"): self.transition_released_erred,
+        }
         self._unknown_durations = dict()
         if unrunnable is not None:
             self._unrunnable = unrunnable
@@ -1791,6 +1809,194 @@ def new_task(self, key: str, spec: object, state: str) -> TaskState:
 
         return ts
 
+    #####################
+    # State Transitions #
+    #####################
+
+    def _transition(self, key, finish: str, *args, **kwargs):
+        """Transition a key from its current state to the finish state
+
+        Examples
+        --------
+        >>> self._transition('x', 'waiting')
+        {'x': 'processing'}
+
+        Returns
+        -------
+        Dictionary of recommendations for future transitions
+
+        See Also
+        --------
+        Scheduler.transitions : transitive version of this function
+        """
+        parent: SchedulerState = cast(SchedulerState, self)
+        ts: TaskState
+        start: str
+        start_finish: tuple
+        finish2: str
+        recommendations: dict
+        worker_msgs: dict
+        client_msgs: dict
+        msgs: list
+        new_msgs: list
+        dependents: set
+        dependencies: set
+        try:
+            recommendations = {}
+            worker_msgs = {}
+            client_msgs = {}
+
+            ts = parent._tasks.get(key)
+            if ts is None:
+                return recommendations, client_msgs, worker_msgs
+            start = ts._state
+            if start == finish:
+                return recommendations, client_msgs, worker_msgs
+
+            if self.plugins:
+                dependents = set(ts._dependents)
+                dependencies = set(ts._dependencies)
+
+            start_finish = (start, finish)
+            func = self._transitions_table.get(start_finish)
+            if func is not None:
+                a: tuple = func(key, *args, **kwargs)
+                recommendations, client_msgs, worker_msgs = a
+            elif "released" not in start_finish:
+                func = self._transitions_table["released", finish]
+                assert not args and not kwargs
+                a_recs: dict
+                a_cmsgs: dict
+                a_wmsgs: dict
+                a: tuple = self._transition(key, "released")
+                a_recs, a_cmsgs, a_wmsgs = a
+                v = a_recs.get(key)
+                if v is not None:
+                    func = self._transitions_table["released", v]
+                b_recs: dict
+                b_cmsgs: dict
+                b_wmsgs: dict
+                b: tuple = func(key)
+                b_recs, b_cmsgs, b_wmsgs = b
+
+                recommendations.update(a_recs)
+                for c, new_msgs in a_cmsgs.items():
+                    msgs = client_msgs.get(c)
+                    if msgs is not None:
+                        msgs.extend(new_msgs)
+                    else:
+                        client_msgs[c] = new_msgs
+                for w, new_msgs in a_wmsgs.items():
+                    msgs = worker_msgs.get(w)
+                    if msgs is not None:
+                        msgs.extend(new_msgs)
+                    else:
+                        worker_msgs[w] = new_msgs
+
+                recommendations.update(b_recs)
+                for c, new_msgs in b_cmsgs.items():
+                    msgs = client_msgs.get(c)
+                    if msgs is not None:
+                        msgs.extend(new_msgs)
+                    else:
+                        client_msgs[c] = new_msgs
+                for w, new_msgs in b_wmsgs.items():
+                    msgs = worker_msgs.get(w)
+                    if msgs is not None:
+                        msgs.extend(new_msgs)
+                    else:
+                        worker_msgs[w] = new_msgs
+
+                start = "released"
+            else:
+                raise RuntimeError("Impossible transition from %r to %r" % start_finish)
+
+            finish2 = ts._state
+            self.transition_log.append((key, start, finish2, recommendations, time()))
+            if parent._validate:
+                logger.debug(
+                    "Transitioned %r %s->%s (actual: %s).  Consequence: %s",
+                    key,
+                    start,
+                    finish2,
+                    ts._state,
+                    dict(recommendations),
+                )
+            if self.plugins:
+                # Temporarily put back forgotten key for plugin to retrieve it
+                if ts._state == "forgotten":
+                    ts._dependents = dependents
+                    ts._dependencies = dependencies
+                    parent._tasks[ts._key] = ts
+                for plugin in list(self.plugins):
+                    try:
+                        plugin.transition(key, start, finish2, *args, **kwargs)
+                    except Exception:
+                        logger.info("Plugin failed with exception", exc_info=True)
+                if ts._state == "forgotten":
+                    del parent._tasks[ts._key]
+
+            tg: TaskGroup = ts._group
+            if ts._state == "forgotten" and tg._name in parent._task_groups:
+                # Remove TaskGroup if all tasks are in the forgotten state
+                all_forgotten: bint = True
+                for s in ALL_TASK_STATES:
+                    if tg._states.get(s):
+                        all_forgotten = False
+                        break
+                if all_forgotten:
+                    ts._prefix._groups.remove(tg)
+                    del parent._task_groups[tg._name]
+
+            return recommendations, client_msgs, worker_msgs
+        except Exception as e:
+            logger.exception("Error transitioning %r from %r to %r", key, start, finish)
+            if LOG_PDB:
+                import pdb
+
+                pdb.set_trace()
+            raise
+
+    def _transitions(self, recommendations: dict, client_msgs: dict, worker_msgs: dict):
+        """Process transitions until none are left
+
+        This includes feedback from previous transitions and continues until we
+        reach a steady state
+        """
+        parent: SchedulerState = cast(SchedulerState, self)
+        keys: set = set()
+        recommendations = recommendations.copy()
+        msgs: list
+        new_msgs: list
+        new: tuple
+        new_recs: dict
+        new_cmsgs: dict
+        new_wmsgs: dict
+        while recommendations:
+            key, finish = recommendations.popitem()
+            keys.add(key)
+
+            new = self._transition(key, finish)
+            new_recs, new_cmsgs, new_wmsgs = new
+
+            recommendations.update(new_recs)
+            for c, new_msgs in new_cmsgs.items():
+                msgs = client_msgs.get(c)
+                if msgs is not None:
+                    msgs.extend(new_msgs)
+                else:
+                    client_msgs[c] = new_msgs
+            for w, new_msgs in new_wmsgs.items():
+                msgs = worker_msgs.get(w)
+                if msgs is not None:
+                    msgs.extend(new_msgs)
+                else:
+                    worker_msgs[w] = new_msgs
+
+        if parent._validate:
+            for key in keys:
+                self.validate_key(key)
+
     def transition_released_waiting(self, key):
         try:
             ts: TaskState = self._tasks[key]
@@ -3147,24 +3353,6 @@ def __init__(
             "stop_task_metadata": self.stop_task_metadata,
         }
 
-        self._transitions_table = {
-            ("released", "waiting"): self.transition_released_waiting,
-            ("waiting", "released"): self.transition_waiting_released,
-            ("waiting", "processing"): self.transition_waiting_processing,
-            ("waiting", "memory"): self.transition_waiting_memory,
-            ("processing", "released"): self.transition_processing_released,
-            ("processing", "memory"): self.transition_processing_memory,
-            ("processing", "erred"): self.transition_processing_erred,
-            ("no-worker", "released"): self.transition_no_worker_released,
-            ("no-worker", "waiting"): self.transition_no_worker_waiting,
-            ("released", "forgotten"): self.transition_released_forgotten,
-            ("memory", "forgotten"): self.transition_memory_forgotten,
-            ("erred", "forgotten"): self.transition_released_forgotten,
-            ("erred", "released"): self.transition_erred_released,
-            ("memory", "released"): self.transition_memory_released,
-            ("released", "erred"): self.transition_released_erred,
-        }
-
         connection_limit = get_fileno_limit() / 2
 
         super().__init__(
@@ -3598,7 +3786,7 @@ async def add_worker(
                 for key in nbytes:
                     ts: TaskState = parent._tasks.get(key)
                     if ts is not None and ts._state in ("processing", "waiting"):
-                        t: tuple = self._transition(
+                        t: tuple = parent._transition(
                             key,
                             "memory",
                             worker=address,
@@ -3606,7 +3794,7 @@ async def add_worker(
                             typename=types[key],
                         )
                         recommendations, client_msgs, worker_msgs = t
-                        self._transitions(recommendations, client_msgs, worker_msgs)
+                        parent._transitions(recommendations, client_msgs, worker_msgs)
                         recommendations = {}
 
             for ts in list(parent._unrunnable):
@@ -3615,7 +3803,7 @@ async def add_worker(
                     recommendations[ts._key] = "waiting"
 
             if recommendations:
-                self._transitions(recommendations, client_msgs, worker_msgs)
+                parent._transitions(recommendations, client_msgs, worker_msgs)
                 recommendations = {}
 
             self.send_all(client_msgs, worker_msgs)
@@ -3994,7 +4182,7 @@ def stimulus_task_finished(self, key=None, worker=None, **kwargs):
         ts._metadata.update(kwargs["metadata"])
 
         if ts._state == "processing":
-            r: tuple = self._transition(key, "memory", worker=worker, **kwargs)
+            r: tuple = parent._transition(key, "memory", worker=worker, **kwargs)
             recommendations, client_msgs, worker_msgs = r
 
             if ts._state == "memory":
@@ -4033,9 +4221,9 @@ def stimulus_task_erred(
             r: tuple
             if retries > 0:
                 ts._retries = retries - 1
-                r = self._transition(key, "waiting")
+                r = parent._transition(key, "waiting")
             else:
-                r = self._transition(
+                r = parent._transition(
                     key,
                     "erred",
                     cause=key,
@@ -4077,7 +4265,7 @@ def stimulus_missing_data(
             if key:
                 recommendations[key] = "released"
 
-            self._transitions(recommendations, client_msgs, worker_msgs)
+            parent._transitions(recommendations, client_msgs, worker_msgs)
             recommendations = {}
 
             if parent._validate:
@@ -4610,18 +4798,18 @@ def handle_task_finished(self, key=None, worker=None, **msg):
 
         r: tuple = self.stimulus_task_finished(key=key, worker=worker, **msg)
         recommendations, client_msgs, worker_msgs = r
-        self._transitions(recommendations, client_msgs, worker_msgs)
+        parent._transitions(recommendations, client_msgs, worker_msgs)
 
         self.send_all(client_msgs, worker_msgs)
 
     def handle_task_erred(self, key=None, **msg):
+        parent: SchedulerState = cast(SchedulerState, self)
         recommendations: dict
         client_msgs: dict
         worker_msgs: dict
-
         r: tuple = self.stimulus_task_erred(key=key, **msg)
         recommendations, client_msgs, worker_msgs = r
-        self._transitions(recommendations, client_msgs, worker_msgs)
+        parent._transitions(recommendations, client_msgs, worker_msgs)
 
         self.send_all(client_msgs, worker_msgs)
 
@@ -4640,7 +4828,7 @@ def handle_release_data(self, key=None, worker=None, client=None, **msg):
 
         r: tuple = self.stimulus_missing_data(key=key, ensure=False, **msg)
         recommendations, client_msgs, worker_msgs = r
-        self._transitions(recommendations, client_msgs, worker_msgs)
+        parent._transitions(recommendations, client_msgs, worker_msgs)
 
         self.send_all(client_msgs, worker_msgs)
 
@@ -4916,7 +5104,9 @@ async def gather(self, comm=None, keys=None, serializers=None):
                             ws._has_what.remove(ts)
                             ts._who_has.remove(ws)
                             ws._nbytes -= ts_nbytes
-                            self._transitions(recommendations, client_msgs, worker_msgs)
+                            parent._transitions(
+                                recommendations, client_msgs, worker_msgs
+                            )
                 self.send_all(client_msgs, worker_msgs)
 
         self.log_event("all", {"action": "gather", "count": len(keys)})
@@ -5920,154 +6110,6 @@ async def register_worker_plugin(self, comm, plugin, name=None):
         )
         return responses
 
-    #####################
-    # State Transitions #
-    #####################
-
-    def _transition(self, key, finish: str, *args, **kwargs):
-        """Transition a key from its current state to the finish state
-
-        Examples
-        --------
-        >>> self._transition('x', 'waiting')
-        {'x': 'processing'}
-
-        Returns
-        -------
-        Dictionary of recommendations for future transitions
-
-        See Also
-        --------
-        Scheduler.transitions : transitive version of this function
-        """
-        parent: SchedulerState = cast(SchedulerState, self)
-        ts: TaskState
-        start: str
-        start_finish: tuple
-        finish2: str
-        recommendations: dict
-        worker_msgs: dict
-        client_msgs: dict
-        msgs: list
-        new_msgs: list
-        dependents: set
-        dependencies: set
-        try:
-            recommendations = {}
-            worker_msgs = {}
-            client_msgs = {}
-
-            ts = parent._tasks.get(key)
-            if ts is None:
-                return recommendations, client_msgs, worker_msgs
-            start = ts._state
-            if start == finish:
-                return recommendations, client_msgs, worker_msgs
-
-            if self.plugins:
-                dependents = set(ts._dependents)
-                dependencies = set(ts._dependencies)
-
-            start_finish = (start, finish)
-            func = self._transitions_table.get(start_finish)
-            if func is not None:
-                a: tuple = func(key, *args, **kwargs)
-                recommendations, client_msgs, worker_msgs = a
-            elif "released" not in start_finish:
-                func = self._transitions_table["released", finish]
-                assert not args and not kwargs
-                a_recs: dict
-                a_cmsgs: dict
-                a_wmsgs: dict
-                a: tuple = self._transition(key, "released")
-                a_recs, a_cmsgs, a_wmsgs = a
-                v = a_recs.get(key)
-                if v is not None:
-                    func = self._transitions_table["released", v]
-                b_recs: dict
-                b_cmsgs: dict
-                b_wmsgs: dict
-                b: tuple = func(key)
-                b_recs, b_cmsgs, b_wmsgs = b
-
-                recommendations.update(a_recs)
-                for c, new_msgs in a_cmsgs.items():
-                    msgs = client_msgs.get(c)
-                    if msgs is not None:
-                        msgs.extend(new_msgs)
-                    else:
-                        client_msgs[c] = new_msgs
-                for w, new_msgs in a_wmsgs.items():
-                    msgs = worker_msgs.get(w)
-                    if msgs is not None:
-                        msgs.extend(new_msgs)
-                    else:
-                        worker_msgs[w] = new_msgs
-
-                recommendations.update(b_recs)
-                for c, new_msgs in b_cmsgs.items():
-                    msgs = client_msgs.get(c)
-                    if msgs is not None:
-                        msgs.extend(new_msgs)
-                    else:
-                        client_msgs[c] = new_msgs
-                for w, new_msgs in b_wmsgs.items():
-                    msgs = worker_msgs.get(w)
-                    if msgs is not None:
-                        msgs.extend(new_msgs)
-                    else:
-                        worker_msgs[w] = new_msgs
-
-                start = "released"
-            else:
-                raise RuntimeError("Impossible transition from %r to %r" % start_finish)
-
-            finish2 = ts._state
-            self.transition_log.append((key, start, finish2, recommendations, time()))
-            if parent._validate:
-                logger.debug(
-                    "Transitioned %r %s->%s (actual: %s).  Consequence: %s",
-                    key,
-                    start,
-                    finish2,
-                    ts._state,
-                    dict(recommendations),
-                )
-            if self.plugins:
-                # Temporarily put back forgotten key for plugin to retrieve it
-                if ts._state == "forgotten":
-                    ts._dependents = dependents
-                    ts._dependencies = dependencies
-                    parent._tasks[ts._key] = ts
-                for plugin in list(self.plugins):
-                    try:
-                        plugin.transition(key, start, finish2, *args, **kwargs)
-                    except Exception:
-                        logger.info("Plugin failed with exception", exc_info=True)
-                if ts._state == "forgotten":
-                    del parent._tasks[ts._key]
-
-            tg: TaskGroup = ts._group
-            if ts._state == "forgotten" and tg._name in parent._task_groups:
-                # Remove TaskGroup if all tasks are in the forgotten state
-                all_forgotten: bint = True
-                for s in ALL_TASK_STATES:
-                    if tg._states.get(s):
-                        all_forgotten = False
-                        break
-                if all_forgotten:
-                    ts._prefix._groups.remove(tg)
-                    del parent._task_groups[tg._name]
-
-            return recommendations, client_msgs, worker_msgs
-        except Exception as e:
-            logger.exception("Error transitioning %r from %r to %r", key, start, finish)
-            if LOG_PDB:
-                import pdb
-
-                pdb.set_trace()
-            raise
-
     def transition(self, key, finish: str, *args, **kwargs):
         """Transition a key from its current state to the finish state
 
@@ -6084,63 +6126,25 @@ def transition(self, key, finish: str, *args, **kwargs):
         --------
         Scheduler.transitions: transitive version of this function
         """
+        parent: SchedulerState = cast(SchedulerState, self)
         recommendations: dict
         worker_msgs: dict
         client_msgs: dict
-        a: tuple = self._transition(key, finish, *args, **kwargs)
+        a: tuple = parent._transition(key, finish, *args, **kwargs)
         recommendations, client_msgs, worker_msgs = a
         self.send_all(client_msgs, worker_msgs)
         return recommendations
 
-    def _transitions(self, recommendations: dict, client_msgs: dict, worker_msgs: dict):
-        """Process transitions until none are left
-
-        This includes feedback from previous transitions and continues until we
-        reach a steady state
-        """
-        parent: SchedulerState = cast(SchedulerState, self)
-        keys: set = set()
-        recommendations = recommendations.copy()
-        msgs: list
-        new_msgs: list
-        new: tuple
-        new_recs: dict
-        new_cmsgs: dict
-        new_wmsgs: dict
-        while recommendations:
-            key, finish = recommendations.popitem()
-            keys.add(key)
-
-            new = self._transition(key, finish)
-            new_recs, new_cmsgs, new_wmsgs = new
-
-            recommendations.update(new_recs)
-            for c, new_msgs in new_cmsgs.items():
-                msgs = client_msgs.get(c)
-                if msgs is not None:
-                    msgs.extend(new_msgs)
-                else:
-                    client_msgs[c] = new_msgs
-            for w, new_msgs in new_wmsgs.items():
-                msgs = worker_msgs.get(w)
-                if msgs is not None:
-                    msgs.extend(new_msgs)
-                else:
-                    worker_msgs[w] = new_msgs
-
-        if parent._validate:
-            for key in keys:
-                self.validate_key(key)
-
     def transitions(self, recommendations: dict):
         """Process transitions until none are left
 
         This includes feedback from previous transitions and continues until we
         reach a steady state
         """
+        parent: SchedulerState = cast(SchedulerState, self)
         client_msgs: dict = {}
         worker_msgs: dict = {}
-        self._transitions(recommendations, client_msgs, worker_msgs)
+        parent._transitions(recommendations, client_msgs, worker_msgs)
         self.send_all(client_msgs, worker_msgs)
 
     def story(self, *keys):

From 4cebe6d4b541e5e657604359d86f5bb518f6739b Mon Sep 17 00:00:00 2001
From: Casey Clements <caseyclements@users.noreply.github.com>
Date: Fri, 26 Feb 2021 14:16:16 +0000
Subject: [PATCH 1161/1550] Make sphinx autosummary and autoclass consistent
 (#3693) (#4367)

* First commit of sphinx-autoautosummary

* Added a couple docstrings

* Fixed pre-commit checks (black and flake)

* Removed try/except pass that was in borrowed template

Co-authored-by: Casey Clements <caclemen@blackrock.com>
---
 distributed/client.py |  2 ++
 docs/source/api.rst   | 52 ++++--------------------------
 docs/source/conf.py   | 75 +++++++++++++++++++++++++++++++++++++++----
 3 files changed, 76 insertions(+), 53 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 1883439f4c9..d71fc309ecb 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -3663,6 +3663,7 @@ def futures_of(self, futures):
         return futures_of(futures, client=self)
 
     def start_ipython(self, *args, **kwargs):
+        """Deprecated - Method moved to start_ipython_workers"""
         raise Exception("Method moved to start_ipython_workers")
 
     async def _start_ipython_workers(self, workers):
@@ -3827,6 +3828,7 @@ def _expand_key(cls, k):
 
     @staticmethod
     def collections_to_dsk(collections, *args, **kwargs):
+        """Convert many collections into a single dask graph, after optimization"""
         return collections_to_dsk(collections, *args, **kwargs)
 
     def get_task_stream(
diff --git a/docs/source/api.rst b/docs/source/api.rst
index 43bd953c9ca..0f918565849 100644
--- a/docs/source/api.rst
+++ b/docs/source/api.rst
@@ -9,44 +9,9 @@ API
 
 .. autosummary::
    Client
-   Client.call_stack
-   Client.cancel
-   Client.close
-   Client.compute
-   Client.gather
-   Client.get
-   Client.get_dataset
-   Client.get_events
-   Client.get_executor
-   Client.get_metadata
-   Client.get_scheduler_logs
-   Client.get_worker_logs
-   Client.get_task_stream
-   Client.has_what
-   Client.list_datasets
-   Client.log_event
-   Client.map
-   Client.nthreads
-   Client.persist
-   Client.publish_dataset
-   Client.profile
-   Client.rebalance
-   Client.replicate
-   Client.restart
-   Client.retry
-   Client.run
-   Client.run_on_scheduler
-   Client.scatter
-   Client.scheduler_info
-   Client.write_scheduler_file
-   Client.set_metadata
-   Client.start_ipython_workers
-   Client.start_ipython_scheduler
-   Client.submit
-   Client.unpublish_dataset
-   Client.upload_file
-   Client.wait_for_workers
-   Client.who_has
+
+.. autoautosummary:: distributed.Client
+   :methods:
 
 .. currentmodule:: distributed
 
@@ -71,14 +36,9 @@ API
 
 .. autosummary::
    Future
-   Future.add_done_callback
-   Future.cancel
-   Future.cancelled
-   Future.done
-   Future.exception
-   Future.result
-   Future.retry
-   Future.traceback
+
+.. autoautosummary:: distributed.Future
+   :methods:
 
 **Client Coordination**
 
diff --git a/docs/source/conf.py b/docs/source/conf.py
index 9bda8cb1a14..d83c6b9d2c6 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -55,9 +55,9 @@
 master_doc = "index"
 
 # General information about the project.
-project = u"Dask.distributed"
-copyright = u"2016, Anaconda, Inc."
-author = u"Anaconda, Inc."
+project = "Dask.distributed"
+copyright = "2016, Anaconda, Inc."
+author = "Anaconda, Inc."
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
@@ -234,8 +234,8 @@
     (
         master_doc,
         "distributed.tex",
-        u"Dask.distributed Documentation",
-        u"Matthew Rocklin",
+        "Dask.distributed Documentation",
+        "Matthew Rocklin",
         "manual",
     )
 ]
@@ -266,7 +266,7 @@
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
 man_pages = [
-    (master_doc, "Dask.distributed", u"Dask.distributed Documentation", [author], 1)
+    (master_doc, "Dask.distributed", "Dask.distributed Documentation", [author], 1)
 ]
 
 # If true, show URL addresses after external links.
@@ -282,7 +282,7 @@
     (
         master_doc,
         "Dask.distributed",
-        u"Dask.distributed Documentation",
+        "Dask.distributed Documentation",
         author,
         "Dask.distributed",
         "One line description of project.",
@@ -425,5 +425,66 @@ def copy_legacy_redirects(app, docname):
                 f.write(page)
 
 
+# -- Configuration to keep autosummary in sync with autoclass::members ----------------------------------------------
+# Fixes issues/3693
+# See https://stackoverflow.com/questions/20569011/python-sphinx-autosummary-automated-listing-of-member-functions
+from sphinx.ext.autosummary import Autosummary
+from sphinx.ext.autosummary import get_documenter
+from docutils.parsers.rst import directives
+from sphinx.util.inspect import safe_getattr
+
+
+class AutoAutoSummary(Autosummary):
+    """Create a summary for methods and attributes (autosummary).
+
+    See https://stackoverflow.com/questions/20569011/python-sphinx-autosummary-automated-listing-of-member-functions
+    """
+
+    option_spec = {
+        "methods": directives.unchanged,
+        "attributes": directives.unchanged,
+    }
+
+    required_arguments = 1
+
+    @staticmethod
+    def get_members(app, obj, typ, include_public=None):
+        if not include_public:
+            include_public = []
+        items = []
+        for name in sorted(obj.__dict__.keys()):
+            try:
+                documenter = get_documenter(app, safe_getattr(obj, name), obj)
+            except AttributeError:
+                continue
+            if documenter.objtype in typ:
+                items.append(name)
+        public = [x for x in items if x in include_public or not x.startswith("_")]
+        return public, items
+
+    def run(self):
+        clazz = str(self.arguments[0])
+        (module_name, class_name) = clazz.rsplit(".", 1)
+        m = __import__(module_name, globals(), locals(), [class_name])
+        c = getattr(m, class_name)
+        app = self.state.document.settings.env.app
+        if "methods" in self.options:
+            _, methods = self.get_members(app, c, ["method"], ["__init__"])
+            self.content = [
+                "%s.%s" % (class_name, method)
+                for method in methods
+                if not method.startswith("_")
+            ]
+        if "attributes" in self.options:
+            _, attribs = self.get_members(app, c, ["attribute", "property"])
+            self.content = [
+                "~%s.%s" % (clazz, attrib)
+                for attrib in attribs
+                if not attrib.startswith("_")
+            ]
+        return super().run()
+
+
 def setup(app):
+    app.add_directive("autoautosummary", AutoAutoSummary)
     app.connect("build-finished", copy_legacy_redirects)

From 3e1fd2a65908641f758fed89a0b18c98ab7f29ff Mon Sep 17 00:00:00 2001
From: Tom Augspurger <tom.w.augspurger@gmail.com>
Date: Fri, 26 Feb 2021 09:33:21 -0600
Subject: [PATCH 1162/1550] Handle annotations for culled tasks (#4544)

Handle annotations for culled tasks
---
 distributed/scheduler.py         |  7 +++++--
 distributed/tests/test_client.py | 21 +++++++++++++++++++++
 2 files changed, 26 insertions(+), 2 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 7a50d847ba8..5bf1116db69 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -4042,8 +4042,11 @@ def update_graph(
 
             for a, kv in annotations.items():
                 for k, v in kv.items():
-                    ts = parent._tasks[k]
-                    ts._annotations[a] = v
+                    # Tasks might have been culled, in which case
+                    # we have nothing to annotate.
+                    ts = parent._tasks.get(k)
+                    if ts is not None:
+                        ts._annotations[a] = v
 
         # Add actors
         if actors is True:
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index be77accc47d..100b52eaf81 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -6471,6 +6471,27 @@ async def test_annotations_resources(c, s, a, b):
     assert all([{"resources": {"GPU": 1}} == ts.annotations for ts in s.tasks.values()])
 
 
+@gen_cluster(
+    client=True,
+    nthreads=[
+        ("127.0.0.1", 1),
+        ("127.0.0.1", 1, {"resources": {"GPU": 1}}),
+    ],
+)
+async def test_annotations_resources_culled(c, s, a, b):
+    da = pytest.importorskip("dask.array")
+
+    x = da.ones((2, 2, 2), chunks=1)
+    with dask.annotate(resources={"GPU": 1}):
+        y = x.map_blocks(lambda x0: x0, meta=x._meta)
+
+    z = y[0, 0, 0]
+
+    (z,) = c.compute([z], optimize_graph=False)
+    await z
+    # it worked!
+
+
 @gen_cluster(client=True)
 async def test_annotations_loose_restrictions(c, s, a, b):
     da = pytest.importorskip("dask.array")

From 31119c49536303dbb877ab5420d2feafdcbb60bb Mon Sep 17 00:00:00 2001
From: "Mads R. B. Kristensen" <madsbk@gmail.com>
Date: Fri, 26 Feb 2021 17:29:29 +0100
Subject: [PATCH 1163/1550] Use the new HLG pack/unpack API in Dask (#4489)

---
 distributed/client.py                  |   3 +-
 distributed/protocol/highlevelgraph.py | 170 -------------------------
 distributed/scheduler.py               |   8 +-
 requirements.txt                       |   5 +-
 4 files changed, 10 insertions(+), 176 deletions(-)
 delete mode 100644 distributed/protocol/highlevelgraph.py

diff --git a/distributed/client.py b/distributed/client.py
index d71fc309ecb..fa17a6b710d 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -61,7 +61,6 @@
 from .metrics import time
 from .protocol import to_serialize
 from .protocol.pickle import dumps, loads
-from .protocol.highlevelgraph import highlevelgraph_pack
 from .publish import Datasets
 from .pubsub import PubSubClientExtension
 from .security import Security
@@ -2540,7 +2539,7 @@ def _graph_to_futures(
             if not isinstance(dsk, HighLevelGraph):
                 dsk = HighLevelGraph.from_collections(id(dsk), dsk, dependencies=())
 
-            dsk = highlevelgraph_pack(dsk, self, keyset)
+            dsk = dsk.__dask_distributed_pack__(self, keyset)
 
             annotations = {}
             if user_priority:
diff --git a/distributed/protocol/highlevelgraph.py b/distributed/protocol/highlevelgraph.py
deleted file mode 100644
index 16eb4f9ff2e..00000000000
--- a/distributed/protocol/highlevelgraph.py
+++ /dev/null
@@ -1,170 +0,0 @@
-from tlz import valmap
-
-from dask.core import keys_in_tasks
-from dask.highlevelgraph import HighLevelGraph, Layer
-from dask.utils import stringify
-
-from ..utils_comm import unpack_remotedata, subs_multiple
-from ..worker import dumps_task
-
-from ..utils import CancelledError
-
-from .core import dumps_msgpack, loads_msgpack
-from .serialize import import_allowed_module
-
-
-def _materialized_layer_pack(
-    layer: Layer,
-    all_keys,
-    known_key_dependencies,
-    client,
-    client_keys,
-):
-    from ..client import Future
-
-    dsk = dict(layer)
-
-    # Find aliases not in `client_keys` and substitute all matching keys
-    # with its Future
-    values = {
-        k: v for k, v in dsk.items() if isinstance(v, Future) and k not in client_keys
-    }
-    if values:
-        dsk = subs_multiple(dsk, values)
-
-    # Unpack remote data and record its dependencies
-    dsk = {k: unpack_remotedata(v, byte_keys=True) for k, v in layer.items()}
-    unpacked_futures = set.union(*[v[1] for v in dsk.values()]) if dsk else set()
-    for future in unpacked_futures:
-        if future.client is not client:
-            raise ValueError(
-                "Inputs contain futures that were created by another client."
-            )
-        if stringify(future.key) not in client.futures:
-            raise CancelledError(stringify(future.key))
-    unpacked_futures_deps = {}
-    for k, v in dsk.items():
-        if len(v[1]):
-            unpacked_futures_deps[k] = {f.key for f in v[1]}
-    dsk = {k: v[0] for k, v in dsk.items()}
-
-    # Calculate dependencies without re-calculating already known dependencies
-    missing_keys = set(dsk.keys()).difference(known_key_dependencies.keys())
-    dependencies = {
-        k: keys_in_tasks(all_keys, [dsk[k]], as_list=False) for k in missing_keys
-    }
-    for k, v in unpacked_futures_deps.items():
-        dependencies[k] = set(dependencies.get(k, ())) | v
-
-    # The scheduler expect all keys to be strings
-    dependencies = {
-        stringify(k): [stringify(dep) for dep in deps]
-        for k, deps in dependencies.items()
-    }
-
-    annotations = layer.pack_annotations()
-    all_keys = all_keys.union(dsk)
-    dsk = {stringify(k): stringify(v, exclusive=all_keys) for k, v in dsk.items()}
-    dsk = valmap(dumps_task, dsk)
-    return {"dsk": dsk, "dependencies": dependencies, "annotations": annotations}
-
-
-def highlevelgraph_pack(hlg: HighLevelGraph, client, client_keys):
-    """Pack the high level graph for Scheduler -> Worker communication
-
-    The approach is to delegate the packaging to each layer in the high
-    level graph by calling .__dask_distributed_pack__() on each layer.
-    If the layer doesn't implement packaging, we materialize the layer
-    and pack it.
-
-    Parameters
-    ----------
-    hlg : HighLevelGraph
-        The high level graph to pack
-    client : distributed.Client
-        The client calling this function.
-    client_keys : Iterable
-        List of keys requested by the client.
-
-    Returns
-    -------
-    data: list of header and payload
-        Packed high level graph serialized by dumps_msgpack
-    """
-    layers = []
-
-    # Dump each layer (in topological order)
-    for layer in (hlg.layers[name] for name in hlg._toposort_layers()):
-        if not layer.is_materialized():
-            state = layer.__dask_distributed_pack__(client)
-            if state is not None:
-                layers.append(
-                    {
-                        "__module__": layer.__module__,
-                        "__name__": type(layer).__name__,
-                        "state": state,
-                    }
-                )
-                continue
-
-        # Falling back to the default serialization, which will materialize the layer
-        layers.append(
-            {
-                "__module__": None,
-                "__name__": None,
-                "state": _materialized_layer_pack(
-                    layer,
-                    hlg.get_all_external_keys(),
-                    hlg.key_dependencies,
-                    client,
-                    client_keys,
-                ),
-            }
-        )
-    return dumps_msgpack({"layers": layers})
-
-
-def highlevelgraph_unpack(dumped_hlg, annotations: dict):
-    """Unpack the high level graph for Scheduler -> Worker communication
-
-    The approach is to delegate the packaging to each layer in the high
-    level graph by calling .__dask_distributed_pack__() on each layer.
-    If the layer doesn't implement a specialized packaging, we materialize
-    the layer can pack it.
-
-    Parameters
-    ----------
-    dumped_hlg : list of header and payload
-        Packed high level graph serialized by dumps_msgpack
-    annotations : dict
-        A top-level annotations object which may be partially populated,
-        and which may be further filled by annotations from the layers
-        of the dumped_hlg.
-
-    Returns
-    -------
-    dsk: dict
-        Materialized graph of all nodes in the high level graph
-    deps: dict
-        Dependencies of each key in `dsk`
-    annotations: dict
-        Annotations for `dsk`
-    """
-    hlg = loads_msgpack(*dumped_hlg)
-
-    dsk = {}
-    deps = {}
-    out_annotations = {}
-    for layer in hlg["layers"]:
-        if annotations:
-            if layer["state"]["annotations"] is None:
-                layer["state"]["annotations"] = {}
-            layer["state"]["annotations"].update(annotations)
-        if layer["__module__"] is None:  # Default implementation
-            unpack_func = Layer.__dask_distributed_unpack__
-        else:
-            mod = import_allowed_module(layer["__module__"])
-            unpack_func = getattr(mod, layer["__name__"]).__dask_distributed_unpack__
-        unpack_func(layer["state"], dsk, deps, out_annotations)
-
-    return dsk, deps, out_annotations
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 5bf1116db69..733b98b620d 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -35,6 +35,7 @@
 from tornado.ioloop import IOLoop, PeriodicCallback
 
 import dask
+from dask.highlevelgraph import HighLevelGraph
 
 from . import profile
 from .batched import BatchedSend
@@ -84,7 +85,6 @@
 from .pubsub import PubSubSchedulerExtension
 from .stealing import WorkStealing
 from .variable import VariableExtension
-from .protocol.highlevelgraph import highlevelgraph_unpack
 
 try:
     from cython import compiled
@@ -3856,8 +3856,10 @@ def update_graph_hlg(
         fifo_timeout=0,
         annotations=None,
     ):
-
-        dsk, dependencies, annotations = highlevelgraph_unpack(hlg, annotations)
+        unpacked_graph = HighLevelGraph.__dask_distributed_unpack__(hlg, annotations)
+        dsk = unpacked_graph["dsk"]
+        dependencies = unpacked_graph["deps"]
+        annotations = unpacked_graph["annotations"]
 
         # Remove any self-dependencies (happens on test_publish_bag() and others)
         for k, v in dependencies.items():
diff --git a/requirements.txt b/requirements.txt
index c79072fb88d..bf1dd021dd8 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,7 +1,10 @@
 click >= 6.6
 cloudpickle >= 1.5.0
 contextvars;python_version<'3.7'
-dask>=2021.02.0
+# TODO: Update our dask version requirement to
+# the latest dask release before releasing distributed
+# xref https://github.com/dask/distributed/pull/4489
+dask @ git+https://github.com/dask/dask.git@master
 msgpack >= 0.6.0
 psutil >= 5.0
 sortedcontainers !=2.0.0, !=2.0.1

From 7f8bb81701c747f042dc8e5d19c5c7fec09de67a Mon Sep 17 00:00:00 2001
From: "Mads R. B. Kristensen" <madsbk@gmail.com>
Date: Fri, 26 Feb 2021 18:16:09 +0100
Subject: [PATCH 1164/1550] Serialize and split (#4541)

* Minor clean up

* serialize numpy handles the writeable flag

* pickle handles the writeable flag

* serialize_and_split() and merge_and_deserialize()

* docstrings

* Use numpy require() to make it writeable

Co-authored-by: jakirkham <jakirkham@gmail.com>

* removed merge_frames()

* Removed obsolete writeable and lengths header

* use tuples to match msgpack's implicit convertion to tuples

* Make sure compression in header is extended when splitting frames

* pickle_loads(): cast shape and type

Co-authored-by: jakirkham <jakirkham@gmail.com>

Co-authored-by: jakirkham <jakirkham@gmail.com>
---
 distributed/protocol/core.py                  | 43 +++------
 distributed/protocol/cupy.py                  |  2 -
 distributed/protocol/numba.py                 |  2 -
 distributed/protocol/numpy.py                 | 18 ++--
 distributed/protocol/rmm.py                   |  2 -
 distributed/protocol/serialize.py             | 94 ++++++++++++++++---
 distributed/protocol/tests/test_numpy.py      |  6 --
 .../protocol/tests/test_protocol_utils.py     | 36 +------
 distributed/protocol/utils.py                 | 51 ----------
 9 files changed, 106 insertions(+), 148 deletions(-)

diff --git a/distributed/protocol/core.py b/distributed/protocol/core.py
index 6df5931e22a..5818a7f23a9 100644
--- a/distributed/protocol/core.py
+++ b/distributed/protocol/core.py
@@ -6,18 +6,15 @@
 
 from .compression import compressions, maybe_compress, decompress
 from .serialize import (
-    serialize,
-    deserialize,
     Serialize,
     Serialized,
     extract_serialize,
     msgpack_decode_default,
     msgpack_encode_default,
+    merge_and_deserialize,
+    serialize_and_split,
 )
-from .utils import frame_split_size, merge_frames, msgpack_opts
-from ..utils import is_writeable, nbytes
-
-_deserialize = deserialize
+from .utils import msgpack_opts
 
 
 logger = logging.getLogger(__name__)
@@ -48,7 +45,7 @@ def dumps(msg, serializers=None, on_error="message", context=None):
         }
 
         data = {
-            key: serialize(
+            key: serialize_and_split(
                 value.data, serializers=serializers, on_error=on_error, context=context
             )
             for key, value in data.items()
@@ -60,39 +57,23 @@ def dumps(msg, serializers=None, on_error="message", context=None):
         out_frames = []
 
         for key, (head, frames) in data.items():
-            if "writeable" not in head:
-                head["writeable"] = tuple(map(is_writeable, frames))
-            if "lengths" not in head:
-                head["lengths"] = tuple(map(nbytes, frames))
-
             # Compress frames that are not yet compressed
             out_compression = []
-            _out_frames = []
             for frame, compression in zip(
                 frames, head.get("compression") or [None] * len(frames)
             ):
-                if compression is None:  # default behavior
-                    _frames = frame_split_size(frame)
-                    _compression, _frames = zip(
-                        *[maybe_compress(frame, **compress_opts) for frame in _frames]
-                    )
-                    out_compression.extend(_compression)
-                    _out_frames.extend(_frames)
-                else:  # already specified, so pass
-                    out_compression.append(compression)
-                    _out_frames.append(frame)
+                if compression is None:
+                    compression, frame = maybe_compress(frame, **compress_opts)
+
+                out_compression.append(compression)
+                out_frames.append(frame)
 
             head["compression"] = out_compression
-            head["count"] = len(_out_frames)
+            head["count"] = len(frames)
             header["headers"][key] = head
             header["keys"].append(key)
-            out_frames.extend(_out_frames)
 
         for key, (head, frames) in pre.items():
-            if "writeable" not in head:
-                head["writeable"] = tuple(map(is_writeable, frames))
-            if "lengths" not in head:
-                head["lengths"] = tuple(map(nbytes, frames))
             head["count"] = len(frames)
             header["headers"][key] = head
             header["keys"].append(key)
@@ -146,9 +127,7 @@ def loads(frames, deserialize=True, deserializers=None):
             if deserialize or key in bytestrings:
                 if "compression" in head:
                     fs = decompress(head, fs)
-                if not any(hasattr(f, "__cuda_array_interface__") for f in fs):
-                    fs = merge_frames(head, fs)
-                value = _deserialize(head, fs, deserializers=deserializers)
+                value = merge_and_deserialize(head, fs, deserializers=deserializers)
             else:
                 value = Serialized(head, fs)
 
diff --git a/distributed/protocol/cupy.py b/distributed/protocol/cupy.py
index 856fc5adf46..eeeae687557 100644
--- a/distributed/protocol/cupy.py
+++ b/distributed/protocol/cupy.py
@@ -22,7 +22,6 @@ def cuda_serialize_cupy_ndarray(x):
 
     header = x.__cuda_array_interface__.copy()
     header["strides"] = tuple(x.strides)
-    header["lengths"] = [x.nbytes]
     frames = [
         cupy.ndarray(
             shape=(x.nbytes,), dtype=cupy.dtype("u1"), memptr=x.data, strides=(1,)
@@ -47,7 +46,6 @@ def cuda_deserialize_cupy_ndarray(header, frames):
 @dask_serialize.register(cupy.ndarray)
 def dask_serialize_cupy_ndarray(x):
     header, frames = cuda_serialize_cupy_ndarray(x)
-    header["writeable"] = (None,) * len(frames)
     frames = [memoryview(cupy.asnumpy(f)) for f in frames]
     return header, frames
 
diff --git a/distributed/protocol/numba.py b/distributed/protocol/numba.py
index e1915251f6f..668f07e0926 100644
--- a/distributed/protocol/numba.py
+++ b/distributed/protocol/numba.py
@@ -23,7 +23,6 @@ def cuda_serialize_numba_ndarray(x):
 
     header = x.__cuda_array_interface__.copy()
     header["strides"] = tuple(x.strides)
-    header["lengths"] = [x.nbytes]
     frames = [
         numba.cuda.cudadrv.devicearray.DeviceNDArray(
             shape=(x.nbytes,), strides=(1,), dtype=np.dtype("u1"), gpu_data=x.gpu_data
@@ -51,7 +50,6 @@ def cuda_deserialize_numba_ndarray(header, frames):
 @dask_serialize.register(numba.cuda.devicearray.DeviceNDArray)
 def dask_serialize_numba_ndarray(x):
     header, frames = cuda_serialize_numba_ndarray(x)
-    header["writeable"] = (None,) * len(frames)
     frames = [memoryview(f.copy_to_host()) for f in frames]
     return header, frames
 
diff --git a/distributed/protocol/numpy.py b/distributed/protocol/numpy.py
index 4ae9298f142..65f7e2f4076 100644
--- a/distributed/protocol/numpy.py
+++ b/distributed/protocol/numpy.py
@@ -4,7 +4,7 @@
 from .serialize import dask_serialize, dask_deserialize
 from . import pickle
 
-from ..utils import log_errors, nbytes
+from ..utils import log_errors
 
 
 def itemsize(dt):
@@ -29,7 +29,6 @@ def serialize_numpy_ndarray(x, context=None):
             buffer_callback=buffer_callback,
             protocol=(context or {}).get("pickle-protocol", None),
         )
-        header["lengths"] = tuple(map(nbytes, frames))
         return header, frames
 
     # We cannot blindly pickle the dtype as some may fail pickling,
@@ -93,15 +92,17 @@ def serialize_numpy_ndarray(x, context=None):
         # "ValueError: cannot include dtype 'M' in a buffer"
         data = data.view("u%d" % math.gcd(x.dtype.itemsize, 8)).data
 
-    header = {"dtype": dt, "shape": x.shape, "strides": strides}
+    header = {
+        "dtype": dt,
+        "shape": x.shape,
+        "strides": strides,
+        "writeable": [x.flags.writeable],
+    }
 
     if broadcast_to is not None:
         header["broadcast_to"] = broadcast_to
 
     frames = [data]
-
-    header["lengths"] = [x.nbytes]
-
     return header, frames
 
 
@@ -112,6 +113,7 @@ def deserialize_numpy_ndarray(header, frames):
             return pickle.loads(frames[0], buffers=frames[1:])
 
         (frame,) = frames
+        (writeable,) = header["writeable"]
 
         is_custom, dt = header["dtype"]
         if is_custom:
@@ -125,6 +127,10 @@ def deserialize_numpy_ndarray(header, frames):
             shape = header["shape"]
 
         x = np.ndarray(shape, dtype=dt, buffer=frame, strides=header["strides"])
+        if not writeable:
+            x.flags.writeable = False
+        else:
+            x = np.require(x, requirements=["W"])
 
         return x
 
diff --git a/distributed/protocol/rmm.py b/distributed/protocol/rmm.py
index 6a56a70ab76..e25919c0fbf 100644
--- a/distributed/protocol/rmm.py
+++ b/distributed/protocol/rmm.py
@@ -13,7 +13,6 @@
     def cuda_serialize_rmm_device_buffer(x):
         header = x.__cuda_array_interface__.copy()
         header["strides"] = (1,)
-        header["lengths"] = [x.nbytes]
         frames = [x]
         return header, frames
 
@@ -31,7 +30,6 @@ def cuda_deserialize_rmm_device_buffer(header, frames):
     @dask_serialize.register(rmm.DeviceBuffer)
     def dask_serialize_rmm_device_buffer(x):
         header, frames = cuda_serialize_rmm_device_buffer(x)
-        header["writeable"] = (None,) * len(frames)
         frames = [numba.cuda.as_cuda_array(f).copy_to_host().data for f in frames]
         return header, frames
 
diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index b99e7692e43..1a447468bf3 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -10,13 +10,12 @@
 import msgpack
 
 from . import pickle
-from ..utils import has_keyword, nbytes, typename, ensure_bytes, is_writeable
+from ..utils import has_keyword, typename, ensure_bytes
 from .compression import maybe_compress, decompress
 from .utils import (
     unpack_frames,
     pack_frames_prelude,
     frame_split_size,
-    merge_frames,
     msgpack_opts,
 )
 
@@ -30,7 +29,7 @@
 
 
 def dask_dumps(x, context=None):
-    """Serialise object using the class-based registry"""
+    """Serialize object using the class-based registry"""
     type_name = typename(type(x))
     try:
         dumps = dask_serialize.dispatch(type(x))
@@ -54,7 +53,6 @@ def dask_loads(header, frames):
 
 
 def pickle_dumps(x, context=None):
-    header = {"serializer": "pickle"}
     frames = [None]
     buffer_callback = lambda f: frames.append(memoryview(f))
     frames[0] = pickle.dumps(
@@ -62,11 +60,23 @@ def pickle_dumps(x, context=None):
         buffer_callback=buffer_callback,
         protocol=context.get("pickle-protocol", None) if context else None,
     )
+    header = {
+        "serializer": "pickle",
+        "writeable": tuple(not f.readonly for f in frames[1:]),
+    }
     return header, frames
 
 
 def pickle_loads(header, frames):
     x, buffers = frames[0], frames[1:]
+    writeable = header["writeable"]
+    for i in range(len(buffers)):
+        mv = memoryview(buffers[i])
+        if writeable[i] == mv.readonly:
+            if mv.readonly:
+                buffers[i] = memoryview(bytearray(mv)).cast(mv.format, mv.shape)
+            else:
+                buffers[i] = memoryview(bytes(mv)).cast(mv.format, mv.shape)
     return pickle.loads(x, buffers=buffers)
 
 
@@ -374,6 +384,72 @@ def deserialize(header, frames, deserializers=None):
     return loads(header, frames)
 
 
+def serialize_and_split(x, serializers=None, on_error="message", context=None):
+    """Serialize and split compressable frames
+
+    This function is a drop-in replacement of `serialize()` that calls `serialize()`
+    followed by `frame_split_size()` on frames that should be compressed.
+
+    Use `merge_and_deserialize()` to merge and deserialize the frames back.
+
+    See Also
+    --------
+    serialize
+    merge_and_deserialize
+    """
+    header, frames = serialize(x, serializers, on_error, context)
+    num_sub_frames = []
+    offsets = []
+    out_frames = []
+    out_compression = []
+    for frame, compression in zip(
+        frames, header.get("compression") or [None] * len(frames)
+    ):
+        if compression is None:  # default behavior
+            sub_frames = frame_split_size(frame)
+            num_sub_frames.append(len(sub_frames))
+            offsets.append(len(out_frames))
+            out_frames.extend(sub_frames)
+            out_compression.extend([None] * len(sub_frames))
+        else:
+            num_sub_frames.append(1)
+            offsets.append(len(out_frames))
+            out_frames.append(frame)
+            out_compression.append(compression)
+    assert len(out_compression) == len(out_frames)
+
+    # Notice, in order to match msgpack's implicit convertion to tuples,
+    # we convert to tuples here as well.
+    header["split-num-sub-frames"] = tuple(num_sub_frames)
+    header["split-offsets"] = tuple(offsets)
+    header["compression"] = tuple(out_compression)
+    return header, out_frames
+
+
+def merge_and_deserialize(header, frames, deserializers=None):
+    """Merge and deserialize frames
+
+    This function is a drop-in replacement of `deserialize()` that merges
+    frames that were split by `serialize_and_split()`
+
+    See Also
+    --------
+    deserialize
+    serialize_and_split
+    """
+    merged_frames = []
+    if "split-num-sub-frames" not in header:
+        merged_frames = frames
+    else:
+        for n, offset in zip(header["split-num-sub-frames"], header["split-offsets"]):
+            if n == 1:
+                merged_frames.append(frames[offset])
+            else:
+                merged_frames.append(bytearray().join(frames[offset : offset + n]))
+
+    return deserialize(header, merged_frames, deserializers=deserializers)
+
+
 class Serialize:
     """Mark an object that should be serialized
 
@@ -534,13 +610,8 @@ def replace_inner(x):
 
 
 def serialize_bytelist(x, **kwargs):
-    header, frames = serialize(x, **kwargs)
-    if "writeable" not in header:
-        header["writeable"] = tuple(map(is_writeable, frames))
-    if "lengths" not in header:
-        header["lengths"] = tuple(map(nbytes, frames))
+    header, frames = serialize_and_split(x, **kwargs)
     if frames:
-        frames = sum(map(frame_split_size, frames), [])
         compression, frames = zip(*map(maybe_compress, frames))
     else:
         compression = []
@@ -566,8 +637,7 @@ def deserialize_bytes(b):
     else:
         header = {}
     frames = decompress(header, frames)
-    frames = merge_frames(header, frames)
-    return deserialize(header, frames)
+    return merge_and_deserialize(header, frames)
 
 
 ################################
diff --git a/distributed/protocol/tests/test_numpy.py b/distributed/protocol/tests/test_numpy.py
index ea349692e70..c52e4f5b402 100644
--- a/distributed/protocol/tests/test_numpy.py
+++ b/distributed/protocol/tests/test_numpy.py
@@ -278,12 +278,6 @@ def test_compression_takes_advantage_of_itemsize():
     assert sum(map(nbytes, aa)) < sum(map(nbytes, bb))
 
 
-def test_large_numpy_array():
-    x = np.ones((100000000,), dtype="u4")
-    header, frames = serialize(x)
-    assert sum(header["lengths"]) == sum(map(nbytes, frames))
-
-
 @pytest.mark.parametrize(
     "x",
     [
diff --git a/distributed/protocol/tests/test_protocol_utils.py b/distributed/protocol/tests/test_protocol_utils.py
index 847dec1ac3d..aed16dd0146 100644
--- a/distributed/protocol/tests/test_protocol_utils.py
+++ b/distributed/protocol/tests/test_protocol_utils.py
@@ -1,38 +1,4 @@
-import pytest
-
-from distributed.protocol.utils import merge_frames, pack_frames, unpack_frames
-from distributed.utils import ensure_bytes, is_writeable
-
-
-@pytest.mark.parametrize(
-    "lengths,writeable,frames",
-    [
-        ([3], [False], [b"123"]),
-        ([3], [True], [b"123"]),
-        ([3], [None], [b"123"]),
-        ([3], [False], [bytearray(b"123")]),
-        ([3], [True], [bytearray(b"123")]),
-        ([3], [None], [bytearray(b"123")]),
-        ([3, 3], [False, False], [b"123", b"456"]),
-        ([2, 3, 2], [False, True, None], [b"12345", b"67"]),
-        ([2, 3, 2], [False, True, None], [bytearray(b"12345"), bytearray(b"67")]),
-        ([5, 2], [False, True], [b"123", b"45", b"67"]),
-        ([3, 4], [None, False], [b"12", b"34", b"567"]),
-    ],
-)
-def test_merge_frames(lengths, writeable, frames):
-    header = {"lengths": lengths, "writeable": writeable}
-    result = merge_frames(header, frames)
-
-    data = b"".join(frames)
-    expected = []
-    for i in lengths:
-        expected.append(data[:i])
-        data = data[i:]
-
-    writeables = list(map(is_writeable, result))
-    assert (r == e for r, e in zip(writeables, header["writeable"]) if e is not None)
-    assert list(map(ensure_bytes, result)) == expected
+from distributed.protocol.utils import pack_frames, unpack_frames
 
 
 def test_pack_frames():
diff --git a/distributed/protocol/utils.py b/distributed/protocol/utils.py
index 45cb1466a89..25ccce7c9f6 100644
--- a/distributed/protocol/utils.py
+++ b/distributed/protocol/utils.py
@@ -34,57 +34,6 @@ def frame_split_size(frame, n=BIG_BYTES_SHARD_SIZE) -> list:
     return [frame[i : i + items_per_shard] for i in range(0, nitems, items_per_shard)]
 
 
-def merge_frames(header, frames):
-    """Merge frames into original lengths
-
-    Examples
-    --------
-    >>> merge_frames({'lengths': [3, 3]}, [b'123456'])
-    [b'123', b'456']
-    >>> merge_frames({'lengths': [6]}, [b'123', b'456'])
-    [b'123456']
-    """
-    lengths = list(header["lengths"])
-    writeables = list(header["writeable"])
-
-    assert len(lengths) == len(writeables)
-    assert sum(lengths) == sum(map(nbytes, frames))
-
-    if all(len(f) == l for f, l in zip(frames, lengths)):
-        return [
-            (bytearray(f) if w else bytes(f)) if w == memoryview(f).readonly else f
-            for w, f in zip(header["writeable"], frames)
-        ]
-
-    frames = frames[::-1]
-    lengths = lengths[::-1]
-    writeables = writeables[::-1]
-
-    out = []
-    while lengths:
-        l = lengths.pop()
-        w = writeables.pop()
-        L = []
-        while l:
-            frame = frames.pop()
-            if nbytes(frame) <= l:
-                L.append(frame)
-                l -= nbytes(frame)
-            else:
-                frame = memoryview(frame)
-                L.append(frame[:l])
-                frames.append(frame[l:])
-                l = 0
-        if len(L) == 1 and w != memoryview(L[0]).readonly:  # no work necessary
-            out.extend(L)
-        elif w:
-            out.append(bytearray().join(L))
-        else:
-            out.append(bytes().join(L))
-
-    return out
-
-
 def pack_frames_prelude(frames):
     nframes = len(frames)
     nbytes_frames = map(nbytes, frames)

From de121eb6e8a7b8aa6ef907183f1e4502a100bc23 Mon Sep 17 00:00:00 2001
From: Martin Durant <martindurant@users.noreply.github.com>
Date: Sat, 27 Feb 2021 03:57:39 -0500
Subject: [PATCH 1165/1550] Pass on original temp dir from nanny to worker
 (#4549)

Pass on original temp dir from nanny to worker
---
 distributed/nanny.py            | 5 ++++-
 distributed/tests/test_nanny.py | 1 +
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/distributed/nanny.py b/distributed/nanny.py
index 5ddc28b0f1d..f93f18522ca 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -168,7 +168,10 @@ def __init__(
             local_directory = dask.config.get("temporary-directory") or os.getcwd()
             if not os.path.exists(local_directory):
                 os.makedirs(local_directory)
+            self._original_local_dir = local_directory
             local_directory = os.path.join(local_directory, "dask-worker-space")
+        else:
+            self._original_local_dir = local_directory
 
         self.local_directory = local_directory
 
@@ -333,7 +336,7 @@ async def instantiate(self, comm=None) -> Status:
             worker_kwargs = dict(
                 scheduler_ip=self.scheduler_addr,
                 nthreads=self.nthreads,
-                local_directory=self.local_directory,
+                local_directory=self._original_local_dir,
                 services=self.services,
                 nanny=self.address,
                 name=self.name,
diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
index d11e4b50d50..1993f11a377 100644
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -410,6 +410,7 @@ async def test_local_directory(s):
             w = await Nanny(s.address)
             assert w.local_directory.startswith(fn)
             assert "dask-worker-space" in w.local_directory
+            assert w.process.worker_dir.count("dask-worker-space") == 1
             await w.close()
 
 

From 2c5d2cf814f13b0efc0fb21acc890476158468da Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Mon, 1 Mar 2021 10:59:55 +0000
Subject: [PATCH 1166/1550] Expose system monitor dashboard as individual plot
 for lab extension (#4540)

---
 distributed/dashboard/components/scheduler.py | 10 ++++++++++
 distributed/dashboard/scheduler.py            |  2 ++
 2 files changed, 12 insertions(+)

diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index c462068bb7b..089d041fa8e 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -2184,6 +2184,16 @@ def individual_graph_doc(scheduler, extra, doc):
         doc.theme = BOKEH_THEME
 
 
+def individual_systemmonitor_doc(scheduler, extra, doc):
+    with log_errors():
+        sysmon = SystemMonitor(scheduler, sizing_mode="stretch_both")
+        doc.title = "Dask: Scheduler System Monitor"
+        add_periodic_callback(doc, sysmon, 500)
+
+        doc.add_root(sysmon.root)
+        doc.theme = BOKEH_THEME
+
+
 def individual_profile_doc(scheduler, extra, doc):
     with log_errors():
         prof = ProfileTimePlot(scheduler, sizing_mode="scale_width", doc=doc)
diff --git a/distributed/dashboard/scheduler.py b/distributed/dashboard/scheduler.py
index 93d7e66d924..09a4339b50a 100644
--- a/distributed/dashboard/scheduler.py
+++ b/distributed/dashboard/scheduler.py
@@ -34,6 +34,7 @@
     individual_memory_by_key_doc,
     individual_compute_time_per_key_doc,
     individual_aggregate_time_per_action_doc,
+    individual_systemmonitor_doc,
 )
 from .worker import counters_doc
 from .components.nvml import gpu_memory_doc, gpu_utilization_doc  # noqa: 1708
@@ -90,4 +91,5 @@ def connect(application, http_server, scheduler, prefix=""):
     "/individual-aggregate-time-per-action": individual_aggregate_time_per_action_doc,
     "/individual-gpu-memory": gpu_memory_doc,
     "/individual-gpu-utilization": gpu_utilization_doc,
+    "/individual-scheduler-system": individual_systemmonitor_doc,
 }

From 79c5284a9caaeed747a09dd715d449052e30d80b Mon Sep 17 00:00:00 2001
From: Benjamin Zaitlen <quasiben@users.noreply.github.com>
Date: Tue, 2 Mar 2021 15:03:00 -0500
Subject: [PATCH 1167/1550] Better bokeh defaults for Dashboard (#4554)

---
 distributed/dashboard/components/scheduler.py | 2 +-
 distributed/dashboard/components/shared.py    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index 089d041fa8e..49032665150 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -870,7 +870,7 @@ def __init__(self, scheduler, width=600, **kwargs):
                 id="bk-cpu-worker-plot",
                 width=int(width / 2),
                 name="cpu_hist",
-                x_range=(0, None),
+                x_range=(0, 100),
                 **kwargs,
             )
             rect = cpu.rect(
diff --git a/distributed/dashboard/components/shared.py b/distributed/dashboard/components/shared.py
index 9360a6b77ac..037b256b188 100644
--- a/distributed/dashboard/components/shared.py
+++ b/distributed/dashboard/components/shared.py
@@ -187,7 +187,7 @@ def __init__(self, server, doc=None, **kwargs):
                 self.key = self.key[0]
             if isinstance(self.key, bytes):
                 self.key = self.key.decode()
-            self.task_names = ["All", self.key]
+            self.task_names = ["All", self.key] if self.key else ["All"]
         else:
             self.key = None
             self.task_names = ["All"]

From ff937b9ca65765d39493816d28f12d13dc347c3c Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Wed, 3 Mar 2021 23:11:34 +0000
Subject: [PATCH 1168/1550] Python 3.9 (#4460)

---
 .github/workflows/tests-macos.yaml            | 70 -------------------
 .github/workflows/tests.yaml                  | 11 +--
 continuous_integration/environment-3.8.yaml   | 18 ++---
 continuous_integration/environment-3.9.yaml   | 52 ++++++++++++++
 distributed/cli/tests/test_dask_worker.py     |  2 +-
 .../dashboard/tests/test_scheduler_bokeh.py   |  4 +-
 distributed/deploy/tests/test_adaptive.py     |  2 +-
 distributed/deploy/tests/test_spec_cluster.py |  1 +
 distributed/deploy/tests/test_ssh.py          |  2 +-
 distributed/utils.py                          |  6 ++
 setup.py                                      |  1 +
 11 files changed, 76 insertions(+), 93 deletions(-)
 delete mode 100644 .github/workflows/tests-macos.yaml
 create mode 100644 continuous_integration/environment-3.9.yaml

diff --git a/.github/workflows/tests-macos.yaml b/.github/workflows/tests-macos.yaml
deleted file mode 100644
index a89ce4e616e..00000000000
--- a/.github/workflows/tests-macos.yaml
+++ /dev/null
@@ -1,70 +0,0 @@
-name: MacOS tests
-
-on:
-  - push
-  # MacOS tests are currently afflicted by a very high level of flakiness; do not run on
-  # PR unless this line is uncommented.
-  # - pull_request
-
-jobs:
-  test:
-    runs-on: macos-latest
-
-    strategy:
-      fail-fast: false
-      matrix:
-        # Many deterministic test failures on Python 3.6
-        python-version: ["3.7", "3.8"]
-
-        # Uncomment to stress-test the test suite for random failures
-        # This will take a LONG time and delay all PRs across the whole github.com/dask!
-        # run: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]
-
-    steps:
-      - name: Checkout source
-        uses: actions/checkout@v2
-
-      - name: Setup Conda Environment
-        uses: conda-incubator/setup-miniconda@v2
-        with:
-          miniconda-version: latest
-          python-version: ${{ matrix.python-version }}
-          environment-file: continuous_integration/environment-${{ matrix.python-version }}.yaml
-          activate-environment: dask-distributed
-          auto-activate-base: false
-
-      - name: Cythonize
-        shell: bash -l {0}
-        if: ${{ matrix.python-version == '3.7' }}
-        run: python setup.py build_ext --with-cython
-
-      - name: Install
-        shell: bash -l {0}
-        run: python -m pip install --no-deps -e .
-
-      - name: conda list
-        shell: bash -l {0}
-        run: conda list
-
-      - name: conda env export
-        shell: bash -l {0}
-        run: |
-          echo -e "--\n--Conda Environment (re-create this with \`conda env create --name <name> -f <output_file>\`)\n--"
-          conda env export | grep -E -v '^prefix:.*$'
-
-      - name: Setup SSH
-        shell: bash -l {0}
-        run: bash continuous_integration/scripts/setup_ssh.sh
-
-      - name: Test
-        shell: bash -l {0}
-        env:
-          PYTHONFAULTHANDLER: 1
-          MARKERS: not avoid_ci
-        run: |
-          source continuous_integration/scripts/set_ulimit.sh
-          pytest distributed -m "not avoid_ci" --runslow
-
-      # - name: Debug with tmate on failure
-      #   if: ${{ failure() }}
-      #   uses: mxschmitt/action-tmate@v3
diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
index d189e291374..c27ae784ea8 100644
--- a/.github/workflows/tests.yaml
+++ b/.github/workflows/tests.yaml
@@ -9,11 +9,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # MacOS tests are currently afflicted by a very high level of flakiness. On
-        # Linux and Windows, run on merge to master and on all pull requests.
-        # On MacOS, run only on merge to master (see twin workflow tests-macos.yaml).
-        os: [ubuntu-latest, windows-latest]
-        python-version: ["3.6", "3.7", "3.8"]
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        python-version: ["3.6", "3.7", "3.8", "3.9"]
+        exclude:
+          # Several failures; Python 3.6 to be phased out soon
+          - os: macos-latest
+            python-version: "3.6"
 
         # Uncomment to stress-test the test suite for random failures
         # This will take a LONG time and delay all PRs across the whole github.com/dask!
diff --git a/continuous_integration/environment-3.8.yaml b/continuous_integration/environment-3.8.yaml
index 153a47d8756..118d4231e4b 100644
--- a/continuous_integration/environment-3.8.yaml
+++ b/continuous_integration/environment-3.8.yaml
@@ -2,7 +2,6 @@ name: dask-distributed
 channels:
   - conda-forge
   - defaults
-  - pytorch
 dependencies:
   - python=3.8
   - pip
@@ -11,13 +10,12 @@ dependencies:
   - click
   - cloudpickle
   - dask  # overridden by git tip below
-  - filesystem-spec  # overridden by git tip below
+  - filesystem-spec
   - h5py
   - ipykernel
   - ipywidgets
-  - joblib  # overridden by git tip below
+  - joblib
   - jupyter_client
-  - lz4  # Only tested here
   - msgpack-python
   - netcdf4
   - paramiko
@@ -29,24 +27,16 @@ dependencies:
   - pytest-repeat
   - pytest-rerunfailures
   - pytest-timeout
-  - python-blosc  # Only tested here
-  - python-snappy  # Only tested here
-  - pytorch  # Only tested here
   - requests
-  - s3fs  # overridden by git tip below
+  - s3fs
   - scikit-learn
   - scipy
   - sortedcollections
   - tblib
   - toolz
-  - torchvision  # Only tested here
   - tornado=6
-  - zict  # overridden by git tip below
+  - zict
   - zstandard
   - pip:
       - git+https://github.com/dask/dask
-      - git+https://github.com/dask/s3fs
-      - git+https://github.com/dask/zict
-      - git+https://github.com/intake/filesystem_spec
-      - git+https://github.com/joblib/joblib
       - keras
diff --git a/continuous_integration/environment-3.9.yaml b/continuous_integration/environment-3.9.yaml
new file mode 100644
index 00000000000..fa0a661a96b
--- /dev/null
+++ b/continuous_integration/environment-3.9.yaml
@@ -0,0 +1,52 @@
+name: dask-distributed
+channels:
+  - conda-forge
+  - defaults
+  - pytorch
+dependencies:
+  - python=3.9
+  - pip
+  - asyncssh
+  - bokeh
+  - click
+  - cloudpickle
+  - dask  # overridden by git tip below
+  - filesystem-spec  # overridden by git tip below
+  - h5py
+  - ipykernel
+  - ipywidgets
+  - joblib  # overridden by git tip below
+  - jupyter_client
+  - lz4  # Only tested here
+  - msgpack-python
+  - netcdf4
+  - paramiko
+  - prometheus_client
+  - psutil
+  - pytest
+  - pytest-asyncio<0.14.0
+  - pytest-faulthandler
+  - pytest-repeat
+  - pytest-rerunfailures
+  - pytest-timeout
+  - python-blosc  # Only tested here
+  - python-snappy  # Only tested here
+  - pytorch  # Only tested here
+  - requests
+  - s3fs  # overridden by git tip below
+  - scikit-learn
+  - scipy
+  - sortedcollections
+  - tblib
+  - toolz
+  - torchvision  # Only tested here
+  - tornado=6
+  - zict  # overridden by git tip below
+  - zstandard
+  - pip:
+      - git+https://github.com/dask/dask
+      - git+https://github.com/dask/s3fs
+      - git+https://github.com/dask/zict
+      - git+https://github.com/intake/filesystem_spec
+      - git+https://github.com/joblib/joblib
+      - keras
diff --git a/distributed/cli/tests/test_dask_worker.py b/distributed/cli/tests/test_dask_worker.py
index f8bbeedfb97..853b9964128 100644
--- a/distributed/cli/tests/test_dask_worker.py
+++ b/distributed/cli/tests/test_dask_worker.py
@@ -41,7 +41,7 @@ def test_nanny_worker_ports(loop):
                     if d["workers"]:
                         break
                     else:
-                        assert time() - start < 5
+                        assert time() - start < 60
                         sleep(0.1)
                 assert (
                     d["workers"]["tcp://127.0.0.1:9684"]["nanny"]
diff --git a/distributed/dashboard/tests/test_scheduler_bokeh.py b/distributed/dashboard/tests/test_scheduler_bokeh.py
index ed7cc299dec..9c50ae50434 100644
--- a/distributed/dashboard/tests/test_scheduler_bokeh.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh.py
@@ -15,9 +15,10 @@
 import dask
 from dask.core import flatten
 from dask.utils import stringify
-from distributed.utils import format_dashboard_link
 from distributed.client import wait
+from distributed.compatibility import MACOS
 from distributed.metrics import time
+from distributed.utils import format_dashboard_link
 from distributed.utils_test import gen_cluster, inc, dec, slowinc, div, get_cert
 from distributed.dashboard.components.worker import Counters
 from distributed.dashboard.scheduler import applications
@@ -749,6 +750,7 @@ async def test_aggregate_action(c, s, a, b):
     assert ("compute") in mbk.action_source.data["names"]
 
 
+@pytest.mark.flaky(reruns=10, reruns_delay=5, condition=MACOS)
 @gen_cluster(client=True, scheduler_kwargs={"dashboard": True})
 async def test_compute_per_key(c, s, a, b):
     mbk = ComputePerKey(s)
diff --git a/distributed/deploy/tests/test_adaptive.py b/distributed/deploy/tests/test_adaptive.py
index 5f84b3165b4..e747cf95a20 100644
--- a/distributed/deploy/tests/test_adaptive.py
+++ b/distributed/deploy/tests/test_adaptive.py
@@ -302,7 +302,7 @@ async def test_adapt_down():
             start = time()
             while len(cluster.scheduler.workers) != 2:
                 await asyncio.sleep(0.1)
-                assert time() < start + 3
+                assert time() < start + 60
 
 
 @gen_test(timeout=30)
diff --git a/distributed/deploy/tests/test_spec_cluster.py b/distributed/deploy/tests/test_spec_cluster.py
index 7a0f60ba011..afdc10cbe04 100644
--- a/distributed/deploy/tests/test_spec_cluster.py
+++ b/distributed/deploy/tests/test_spec_cluster.py
@@ -196,6 +196,7 @@ async def test_unexpected_closed_worker(cleanup):
             assert len(cluster.workers) == 2
 
 
+@pytest.mark.flaky(reruns=10, reruns_delay=5)
 @pytest.mark.slow
 @pytest.mark.asyncio
 async def test_restart(cleanup):
diff --git a/distributed/deploy/tests/test_ssh.py b/distributed/deploy/tests/test_ssh.py
index 376194549ce..0bea6f7dc75 100644
--- a/distributed/deploy/tests/test_ssh.py
+++ b/distributed/deploy/tests/test_ssh.py
@@ -81,7 +81,7 @@ def test_defer_to_old(loop):
 
 
 @pytest.mark.avoid_ci
-def test_old_ssh_wih_local_dir(loop):
+def test_old_ssh_with_local_dir(loop):
     with pytest.warns(Warning):
         from distributed.deploy.old_ssh import SSHCluster as OldSSHCluster
 
diff --git a/distributed/utils.py b/distributed/utils.py
index d7d4db6b2c5..502118218c7 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -64,6 +64,12 @@
 except ImportError:
     thread_state = threading.local()
 
+# For some reason this is required in python >= 3.9
+if WINDOWS:
+    import multiprocessing.popen_spawn_win32
+else:
+    import multiprocessing.popen_spawn_posix
+
 logger = _logger = logging.getLogger(__name__)
 
 
diff --git a/setup.py b/setup.py
index 9160e1ffdc9..5483e0b963d 100755
--- a/setup.py
+++ b/setup.py
@@ -92,6 +92,7 @@
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
         "Topic :: Scientific/Engineering",
         "Topic :: System :: Distributed Computing",
     ],

From 03c5c1e0295a642f10db6ea3a82c2abbdc2185bb Mon Sep 17 00:00:00 2001
From: "Mads R. B. Kristensen" <madsbk@gmail.com>
Date: Thu, 4 Mar 2021 03:36:27 +0100
Subject: [PATCH 1169/1550] Refcount the thread_state.asynchronous flag (#4557)

---
 distributed/utils.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/distributed/utils.py b/distributed/utils.py
index 502118218c7..dd6ca2fad02 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -319,11 +319,15 @@ def sync(loop, func, *args, callback_timeout=None, **kwargs):
 
     @gen.coroutine
     def f():
+        # We flag the thread state asynchronous, which will make sync() call
+        # within `func` use async semantic. In order to support concurrent
+        # calls to sync(), `asynchronous` is used as a ref counter.
+        thread_state.asynchronous = getattr(thread_state, "asynchronous", 0)
+        thread_state.asynchronous += 1
         try:
             if main_tid == threading.get_ident():
                 raise RuntimeError("sync() called from thread of running loop")
             yield gen.moment
-            thread_state.asynchronous = True
             future = func(*args, **kwargs)
             if callback_timeout is not None:
                 future = asyncio.wait_for(future, callback_timeout)
@@ -331,7 +335,8 @@ def f():
         except Exception as exc:
             error[0] = sys.exc_info()
         finally:
-            thread_state.asynchronous = False
+            assert thread_state.asynchronous > 0
+            thread_state.asynchronous -= 1
             e.set()
 
     loop.add_callback(f)

From 074bc4809480a88dfdf4d05e33169fc9c19a69d1 Mon Sep 17 00:00:00 2001
From: "Mads R. B. Kristensen" <madsbk@gmail.com>
Date: Thu, 4 Mar 2021 03:37:28 +0100
Subject: [PATCH 1170/1550] tcp.write(): cast memoryview to 1byte itemsize
 (#4555)

---
 distributed/comm/tcp.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py
index 11a0a228f45..056e0216fbc 100644
--- a/distributed/comm/tcp.py
+++ b/distributed/comm/tcp.py
@@ -261,6 +261,12 @@ async def write(self, msg, serializers=None, on_error="message"):
                 if each_frame_nbytes:
                     if stream._write_buffer is None:
                         raise StreamClosedError()
+
+                    if isinstance(each_frame, memoryview):
+                        # Make sure that len(data) == data.nbytes`
+                        # See <https://github.com/tornadoweb/tornado/pull/2996>
+                        each_frame = memoryview(each_frame).cast("B")
+
                     stream._write_buffer.append(each_frame)
                     stream._total_write_index += each_frame_nbytes
 

From 5ea74cd7ec3cf5b90c5e6d5a31fb7dc3888ea242 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Fri, 5 Mar 2021 17:39:48 -0600
Subject: [PATCH 1171/1550] bump version to 2021.03.0

---
 docs/source/changelog.rst | 38 +++++++++++++++++++++++++++++++++++++-
 requirements.txt          |  5 +----
 2 files changed, 38 insertions(+), 5 deletions(-)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index a3c04b1bdd7..30b4f1c77d1 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,41 @@
 Changelog
 =========
 
+2021.03.0
+---------
+
+Released on March 5, 2021
+
+.. note::
+
+    This is the first release with support for Python 3.9 and the
+    last release with support for Python 3.6
+
+- ``tcp.write()``: cast ``memoryview`` to byte itemsize (:pr:`4555`) `Mads R. B. Kristensen`_
+- Refcount the ``thread_state.asynchronous`` flag (:pr:`4557`) `Mads R. B. Kristensen`_
+- Python 3.9 (:pr:`4460`) `crusaderky`_
+- Better bokeh defaults for dashboard (:pr:`4554`) `Benjamin Zaitlen`_
+- Expose system monitor dashboard as individual plot for lab extension (:pr:`4540`) `Jacob Tomlinson`_
+- Pass on original temp dir from nanny to worker (:pr:`4549`) `Martin Durant`_
+- Serialize and split (:pr:`4541`) `Mads R. B. Kristensen`_
+- Use the new HLG pack/unpack API in Dask (:pr:`4489`) `Mads R. B. Kristensen`_
+- Handle annotations for culled tasks (:pr:`4544`) `Tom Augspurger`_
+- Make sphinx autosummary and autoclass consistent (:pr:`4367`) `Casey Clements`_
+- Move ``_transition*`` to ``SchedulerState`` (:pr:`4545`) `jakirkham`_
+- Migrate from travis to GitHub actions (:pr:`4504`) `crusaderky`_
+- Move ``new_task`` to ``SchedulerState`` (:pr:`4527`) `jakirkham`_
+- Batch more Scheduler sends (:pr:`4526`) `jakirkham`_
+- ``transition_memory_released`` and ``get_nbytes()`` optimizations (:pr:`4516`) `jakirkham`_
+- Pin ``black`` pre-commit (:pr:`4533`) `James Bourbeau`_
+- Read & write all frames in one pass (:pr:`4506`) `jakirkham`_
+- Skip ``stream.write`` call for empty frames (:pr:`4507`) `jakirkham`_
+- Prepend frame metadata header (:pr:`4505`) `jakirkham`_
+- ``transition_processing_memory`` optimizations, etc. (:pr:`4487`) `jakirkham`_
+- Attempt to get client from worker in ``Queue`` and ``Variable`` (:pr:`4490`) `James Bourbeau`_
+- Use ``main`` branch for ``zict`` (:pr:`4499`) `jakirkham`_
+- Use a callback to close TCP Comms, rather than check every time (:pr:`4453`) `Matthew Rocklin`_
+
+
 2021.02.0
 ---------
 
@@ -2192,4 +2227,5 @@ significantly without many new features.
 .. _`Bernhard M. Wiedemann`: https://github.com/bmwiedemann
 .. _`Bruno Pagani`: https://github.com/ArchangeGabriel
 .. _`selshowk`: https://github.com/selshowk
-.. _`Ray Bell`: https://github.com/raybellwaves
\ No newline at end of file
+.. _`Ray Bell`: https://github.com/raybellwaves
+.. _`Casey Clements`: https://github.com/caseyclements
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index bf1dd021dd8..37d23c45e69 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,10 +1,7 @@
 click >= 6.6
 cloudpickle >= 1.5.0
 contextvars;python_version<'3.7'
-# TODO: Update our dask version requirement to
-# the latest dask release before releasing distributed
-# xref https://github.com/dask/distributed/pull/4489
-dask @ git+https://github.com/dask/dask.git@master
+dask >= 2021.03.0
 msgpack >= 0.6.0
 psutil >= 5.0
 sortedcontainers !=2.0.0, !=2.0.1

From 2231f9028e8e9ab596460edf24d146d60c1d2c0d Mon Sep 17 00:00:00 2001
From: "Mads R. B. Kristensen" <madsbk@gmail.com>
Date: Mon, 8 Mar 2021 19:39:36 +0100
Subject: [PATCH 1172/1550] [REVIEW] Msgpack handles extract serialize (#4531)

* tcp.write(): cast memoryview to 1byte itemsize

* dumps and loads now extract Serialize and Serialized

* test_numpy: now use the new dumps scheme

* Currently not handling bytes in dumps()

* Removing extract_serialize()

* test_comms: handle implicit list => tuple conversion by msgpack

* Workaround a buffer length bug

* Fixed some tuple/list mismatch in test_comms

* test_compression_takes_advantage_of_itemsize(): xfail

* to_serialize tests using bytearrays

* Removing "B" cast of memoryviews hack

Now that we depend on https://github.com/dask/distributed/pull/4555,
we don't need this hack.

* TCP.write(): use nbytes() to determine if memoryview 0-size

Co-authored-by: John Kirkham <jakirkham@gmail.com>
---
 distributed/comm/tests/test_comms.py          |   9 +-
 distributed/protocol/core.py                  | 179 +++++++-----------
 distributed/protocol/serialize.py             |  64 -------
 distributed/protocol/tests/test_collection.py |   2 +-
 distributed/protocol/tests/test_numpy.py      |   4 +-
 distributed/protocol/tests/test_protocol.py   |  10 +-
 6 files changed, 83 insertions(+), 185 deletions(-)

diff --git a/distributed/comm/tests/test_comms.py b/distributed/comm/tests/test_comms.py
index b7a2061af48..e3c46984092 100644
--- a/distributed/comm/tests/test_comms.py
+++ b/distributed/comm/tests/test_comms.py
@@ -1037,7 +1037,7 @@ def check_out_false(out_value):
         assert isinstance(ser, Serialized)
         assert deserialize(ser.header, ser.frames) == 456
 
-        assert isinstance(to_ser, list)
+        assert isinstance(to_ser, (tuple, list)) and len(to_ser) == 1
         (to_ser,) = to_ser
         # The to_serialize() value could have been actually serialized
         # or not (it's a transport-specific optimization)
@@ -1051,6 +1051,8 @@ def check_out_true(out_value):
         expected_msg = msg.copy()
         expected_msg["ser"] = 456
         expected_msg["to_ser"] = [123]
+        # Notice, we allow "to_ser" to be a tuple or a list
+        assert list(out_value.pop("to_ser")) == expected_msg.pop("to_ser")
         assert out_value == expected_msg
 
     await check_listener_deserialize(addr, False, msg, check_out_false)
@@ -1061,13 +1063,14 @@ def check_out_true(out_value):
 
     # Test with long bytestrings, large enough to be transferred
     # as a separate payload
+    # TODO: currently bytestrings are not transferred as a separate payload
 
     _uncompressible = os.urandom(1024 ** 2) * 4  # end size: 8 MB
 
     msg = {
         "op": "update",
         "x": _uncompressible,
-        "to_ser": [to_serialize(_uncompressible)],
+        "to_ser": (to_serialize(_uncompressible),),
         "ser": Serialized(*serialize(_uncompressible)),
     }
     msg_orig = msg.copy()
@@ -1089,7 +1092,7 @@ def check_out(deserialize_flag, out_value):
         else:
             assert isinstance(ser, Serialized)
             assert deserialize(ser.header, ser.frames) == _uncompressible
-            assert isinstance(to_ser, list)
+            assert isinstance(to_ser, tuple) and len(to_ser) == 1
             (to_ser,) = to_ser
             # The to_serialize() value could have been actually serialized
             # or not (it's a transport-specific optimization)
diff --git a/distributed/protocol/core.py b/distributed/protocol/core.py
index 5818a7f23a9..4235dca9f9c 100644
--- a/distributed/protocol/core.py
+++ b/distributed/protocol/core.py
@@ -1,14 +1,10 @@
-from functools import reduce
 import logging
-import operator
-
 import msgpack
 
 from .compression import compressions, maybe_compress, decompress
 from .serialize import (
     Serialize,
     Serialized,
-    extract_serialize,
     msgpack_decode_default,
     msgpack_encode_default,
     merge_and_deserialize,
@@ -20,78 +16,59 @@
 logger = logging.getLogger(__name__)
 
 
-def dumps(msg, serializers=None, on_error="message", context=None):
-    """ Transform Python message to bytestream suitable for communication """
-    try:
-        data = {}
+def dumps(msg, serializers=None, on_error="message", context=None) -> list:
+    """Transform Python message to bytestream suitable for communication
 
+    Developer Notes
+    ---------------
+    The approach here is to use `msgpack.dumps()` to serialize `msg` and
+    write the result to the first output frame. If `msgpack.dumps()`
+    encounters an object it cannot serialize like a NumPy array, it is handled
+    out-of-band by `_encode_default()` and appended to the output frame list.
+    """
+    try:
         if context and "compression" in context:
             compress_opts = {"compression": context["compression"]}
         else:
             compress_opts = {}
 
-        # Only lists and dicts can contain serialized values
-        if isinstance(msg, (list, dict)):
-            msg, data, bytestrings = extract_serialize(msg)
-        small_header, small_payload = dumps_msgpack(msg, **compress_opts)
-
-        if not data:  # fast path without serialized data
-            return small_header, small_payload
+        def _inplace_compress_frames(header, frames):
+            compression = list(header.get("compression", [None] * len(frames)))
+
+            for i in range(len(frames)):
+                if compression[i] is None:
+                    compression[i], frames[i] = maybe_compress(
+                        frames[i], **compress_opts
+                    )
+
+            header["compression"] = tuple(compression)
+
+        frames = [None]
+
+        def _encode_default(obj):
+            typ = type(obj)
+            if typ is Serialize or typ is Serialized:
+                if typ is Serialize:
+                    obj = obj.data
+                offset = len(frames)
+                sub_header, sub_frames = serialize_and_split(
+                    obj, serializers=serializers, on_error=on_error, context=context
+                )
+                sub_header["num-sub-frames"] = len(sub_frames)
+                _inplace_compress_frames(sub_header, sub_frames)
+                frames.append(
+                    msgpack.dumps(
+                        sub_header, default=msgpack_encode_default, use_bin_type=True
+                    )
+                )
+                frames.extend(sub_frames)
+                return {"__Serialized__": offset}
+            else:
+                return msgpack_encode_default(obj)
 
-        pre = {
-            key: (value.header, value.frames)
-            for key, value in data.items()
-            if type(value) is Serialized
-        }
+        frames[0] = msgpack.dumps(msg, default=_encode_default, use_bin_type=True)
+        return frames
 
-        data = {
-            key: serialize_and_split(
-                value.data, serializers=serializers, on_error=on_error, context=context
-            )
-            for key, value in data.items()
-            if type(value) is Serialize
-        }
-
-        header = {"headers": {}, "keys": [], "bytestrings": list(bytestrings)}
-
-        out_frames = []
-
-        for key, (head, frames) in data.items():
-            # Compress frames that are not yet compressed
-            out_compression = []
-            for frame, compression in zip(
-                frames, head.get("compression") or [None] * len(frames)
-            ):
-                if compression is None:
-                    compression, frame = maybe_compress(frame, **compress_opts)
-
-                out_compression.append(compression)
-                out_frames.append(frame)
-
-            head["compression"] = out_compression
-            head["count"] = len(frames)
-            header["headers"][key] = head
-            header["keys"].append(key)
-
-        for key, (head, frames) in pre.items():
-            head["count"] = len(frames)
-            header["headers"][key] = head
-            header["keys"].append(key)
-            out_frames.extend(frames)
-
-        for i, frame in enumerate(out_frames):
-            if type(frame) is memoryview and frame.strides != (1,):
-                try:
-                    frame = frame.cast("B")
-                except TypeError:
-                    frame = frame.tobytes()
-                out_frames[i] = frame
-
-        return [
-            small_header,
-            small_payload,
-            msgpack.dumps(header, use_bin_type=True),
-        ] + out_frames
     except Exception:
         logger.critical("Failed to Serialize", exc_info=True)
         raise
@@ -99,53 +76,35 @@ def dumps(msg, serializers=None, on_error="message", context=None):
 
 def loads(frames, deserialize=True, deserializers=None):
     """ Transform bytestream back into Python value """
-    frames = frames[::-1]  # reverse order to improve pop efficiency
-    if not isinstance(frames, list):
-        frames = list(frames)
+
     try:
-        small_header = frames.pop()
-        small_payload = frames.pop()
-        msg = loads_msgpack(small_header, small_payload)
-        if not frames:
-            return msg
-
-        header = frames.pop()
-        header = msgpack.loads(header, use_list=False, **msgpack_opts)
-        keys = header["keys"]
-        headers = header["headers"]
-        bytestrings = set(header["bytestrings"])
-
-        for key in keys:
-            head = headers[key]
-            count = head["count"]
-            if count:
-                fs = frames[-count::][::-1]
-                del frames[-count:]
-            else:
-                fs = []
 
-            if deserialize or key in bytestrings:
-                if "compression" in head:
-                    fs = decompress(head, fs)
-                value = merge_and_deserialize(head, fs, deserializers=deserializers)
-            else:
-                value = Serialized(head, fs)
-
-            def put_in(keys, coll, val):
-                """Inverse of get_in, but does type promotion in the case of lists"""
-                if keys:
-                    holder = reduce(operator.getitem, keys[:-1], coll)
-                    if isinstance(holder, tuple):
-                        holder = list(holder)
-                        coll = put_in(keys[:-1], coll, holder)
-                    holder[keys[-1]] = val
+        def _decode_default(obj):
+            offset = obj.get("__Serialized__", 0)
+            if offset > 0:
+                sub_header = msgpack.loads(
+                    frames[offset],
+                    object_hook=msgpack_decode_default,
+                    use_list=False,
+                    **msgpack_opts
+                )
+                offset += 1
+                sub_frames = frames[offset : offset + sub_header["num-sub-frames"]]
+                if deserialize:
+                    if "compression" in sub_header:
+                        sub_frames = decompress(sub_header, sub_frames)
+                    return merge_and_deserialize(
+                        sub_header, sub_frames, deserializers=deserializers
+                    )
                 else:
-                    coll = val
-                return coll
+                    return Serialized(sub_header, sub_frames)
+            else:
+                return msgpack_decode_default(obj)
 
-            msg = put_in(key, msg, value)
+        return msgpack.loads(
+            frames[0], object_hook=_decode_default, use_list=False, **msgpack_opts
+        )
 
-        return msg
     except Exception:
         logger.critical("Failed to deserialize", exc_info=True)
         raise
diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index 1a447468bf3..228f02d78ca 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -507,70 +507,6 @@ def __ne__(self, other):
         return not (self == other)
 
 
-def extract_serialize(x) -> tuple:
-    """Pull out Serialize objects from message
-
-    This also remove large bytestrings from the message into a second
-    dictionary.
-
-    Examples
-    --------
-    >>> from distributed.protocol import to_serialize
-    >>> msg = {'op': 'update', 'data': to_serialize(123)}
-    >>> extract_serialize(msg)
-    ({'op': 'update'}, {('data',): <Serialize: 123>}, set())
-    """
-    typ_x: type = type(x)
-    if typ_x is dict:
-        x_d: dict = x
-        x_items = x_d.items()
-        x2 = {}
-    elif typ_x is list:
-        x_l: list = x
-        x_items = enumerate(x_l)
-        x2 = len(x_l) * [None]
-
-    ser = {}
-    bytestrings = set()
-    path = ()
-    _extract_serialize(x_items, x2, ser, bytestrings, path)
-    return x2, ser, bytestrings
-
-
-def _extract_serialize(x_items, x2, ser: dict, bytestrings: set, path: tuple) -> None:
-    for k, v in x_items:
-        path_k = path + (k,)
-        typ_v: type = type(v)
-        if typ_v is dict:
-            v_d: dict = v
-            v_items = v_d.items()
-            x2[k] = v2 = {}
-            _extract_serialize(v_items, v2, ser, bytestrings, path_k)
-        elif typ_v is list:
-            v_l: list = v
-            v_items = enumerate(v_l)
-            x2[k] = v2 = len(v_l) * [None]
-            _extract_serialize(v_items, v2, ser, bytestrings, path_k)
-        elif typ_v is Serialize or typ_v is Serialized:
-            ser[path_k] = v
-        elif typ_v is bytes:
-            v_b: bytes = v
-            if len(v_b) > 2 ** 16:
-                ser[path_k] = to_serialize(v_b)
-                bytestrings.add(path_k)
-            else:
-                x2[k] = v_b
-        elif typ_v is bytearray:
-            v_ba: bytearray = v
-            if len(v_ba) > 2 ** 16:
-                ser[path_k] = to_serialize(v_ba)
-                bytestrings.add(path_k)
-            else:
-                x2[k] = v_ba
-        else:
-            x2[k] = v
-
-
 def nested_deserialize(x):
     """
     Replace all Serialize and Serialized values nested in *x*
diff --git a/distributed/protocol/tests/test_collection.py b/distributed/protocol/tests/test_collection.py
index ddb8a44bd44..fd112b6c792 100644
--- a/distributed/protocol/tests/test_collection.py
+++ b/distributed/protocol/tests/test_collection.py
@@ -47,4 +47,4 @@ def test_nested_types():
     header, frames = serialize([[[x]]])
     assert "dask" in str(header)
     assert len(frames) == 1
-    assert x.data in frames
+    assert x.data == np.frombuffer(frames[0]).data
diff --git a/distributed/protocol/tests/test_numpy.py b/distributed/protocol/tests/test_numpy.py
index c52e4f5b402..9096748b5d1 100644
--- a/distributed/protocol/tests/test_numpy.py
+++ b/distributed/protocol/tests/test_numpy.py
@@ -219,13 +219,13 @@ def test_compress_numpy():
     frames = dumps({"x": to_serialize(x)})
     assert sum(map(nbytes, frames)) < x.nbytes
 
-    header = msgpack.loads(frames[2], raw=False, use_list=False, strict_map_key=False)
+    header = msgpack.loads(frames[1], raw=False, use_list=False, strict_map_key=False)
     try:
         import blosc  # noqa: F401
     except ImportError:
         pass
     else:
-        assert all(c == "blosc" for c in header["headers"][("x",)]["compression"])
+        assert all(c == "blosc" for c in header["compression"])
 
 
 def test_compress_memoryview():
diff --git a/distributed/protocol/tests/test_protocol.py b/distributed/protocol/tests/test_protocol.py
index d915ead4699..e088f146bfb 100644
--- a/distributed/protocol/tests/test_protocol.py
+++ b/distributed/protocol/tests/test_protocol.py
@@ -26,8 +26,9 @@ def test_compression_2():
     pytest.importorskip("lz4")
     np = pytest.importorskip("numpy")
     x = np.random.random(10000)
-    header, payload = dumps(x.tobytes())
-    assert not header or not msgpack.loads(header, encoding="utf8").get("compression")
+    msg = dumps(to_serialize(x.tobytes()))
+    compression = msgpack.loads(msg[1]).get("compression")
+    assert all(c is None for c in compression)
 
 
 def test_compression_without_deserialization():
@@ -87,14 +88,13 @@ def test_maybe_compress_sample():
 
 def test_large_bytes():
     for tp in (bytes, bytearray):
-        msg = {"x": tp(b"0" * 1000000), "y": 1}
+        msg = {"x": to_serialize(tp(b"0" * 1000000)), "y": 1}
         frames = dumps(msg)
+        msg["x"] = msg["x"].data
         assert loads(frames) == msg
         assert len(frames[0]) < 1000
         assert len(frames[1]) < 1000
 
-        assert loads(frames, deserialize=False) == msg
-
 
 @pytest.mark.slow
 def test_large_messages():

From 6f95c6323454825f1a64b554ffc7267faeb27de5 Mon Sep 17 00:00:00 2001
From: Julia Signell <jsignell@gmail.com>
Date: Mon, 8 Mar 2021 14:03:08 -0500
Subject: [PATCH 1173/1550] Change default branch from master to main (#4495)

---
 .github/workflows/ci-pre-commit.yml                |  4 ++--
 README.rst                                         |  4 ++--
 distributed/_version.py                            |  2 +-
 .../dashboard/templates/performance_report.html    |  4 ++--
 docs/release-procedure.md                          |  2 +-
 docs/source/develop.rst                            | 14 +++++++-------
 docs/source/diagnosing-performance.rst             |  4 ++--
 versioneer.py                                      |  4 ++--
 8 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/.github/workflows/ci-pre-commit.yml b/.github/workflows/ci-pre-commit.yml
index 3d1e66571f3..82e8821447e 100644
--- a/.github/workflows/ci-pre-commit.yml
+++ b/.github/workflows/ci-pre-commit.yml
@@ -2,9 +2,9 @@ name: Linting
 
 on:
   push:
-    branches: master
+    branches: main
   pull_request:
-    branches: master
+    branches: main
 
 jobs:
   checks:
diff --git a/README.rst b/README.rst
index 4317f34a29e..77cb2985809 100644
--- a/README.rst
+++ b/README.rst
@@ -6,9 +6,9 @@ Distributed
 A library for distributed computation.  See documentation_ for more details.
 
 .. _documentation: https://distributed.dask.org
-.. |Linux/Windows Build Status| image:: https://github.com/dask/distributed/workflows/Tests/badge.svg?branch=master
+.. |Linux/Windows Build Status| image:: https://github.com/dask/distributed/workflows/Tests/badge.svg?branch=main
    :target: https://github.com/dask/distributed/actions?query=workflow%3A%22Tests%22
-.. |MacOS Build Status| image:: https://github.com/dask/distributed/workflows/MacOS%20tests/badge.svg?branch=master
+.. |MacOS Build Status| image:: https://github.com/dask/distributed/workflows/MacOS%20tests/badge.svg?branch=main
    :target: https://github.com/dask/distributed/actions?query=workflow%3A%22MacOS+tests%22
 .. |Doc Status| image:: https://readthedocs.org/projects/distributed/badge/?version=latest
    :target: https://distributed.dask.org
diff --git a/distributed/_version.py b/distributed/_version.py
index 79f2770dd9c..5530c657430 100644
--- a/distributed/_version.py
+++ b/distributed/_version.py
@@ -196,7 +196,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose):
         # refs/heads/ and refs/tags/ prefixes that would let us distinguish
         # between branches and tags. By ignoring refnames without digits, we
         # filter out many common branch names like "release" and
-        # "stabilization", as well as "HEAD" and "master".
+        # "stabilization", as well as "HEAD" and "main".
         tags = set([r for r in refs if re.search(r"\d", r)])
         if verbose:
             print("discarding '%s', no digits" % ",".join(refs - tags))
diff --git a/distributed/dashboard/templates/performance_report.html b/distributed/dashboard/templates/performance_report.html
index 183ea0c3c5c..adf46647b81 100644
--- a/distributed/dashboard/templates/performance_report.html
+++ b/distributed/dashboard/templates/performance_report.html
@@ -1,6 +1,6 @@
 {% extends "file.html" %}
-{# See https://github.com/bokeh/bokeh/blob/master/bokeh/core/_templates/file.html #}
+{# See https://github.com/bokeh/bokeh/blob/main/bokeh/core/_templates/file.html #}
 
 {% block preamble %}
 <link rel="shortcut icon" href="https://docs.dask.org/en/latest/_static/images/favicon.ico" />
-{% endblock %}
\ No newline at end of file
+{% endblock %}
diff --git a/docs/release-procedure.md b/docs/release-procedure.md
index f9efd6a0ab1..bf1b904c186 100644
--- a/docs/release-procedure.md
+++ b/docs/release-procedure.md
@@ -1,3 +1,3 @@
 Distributed follows a similar procedure for releasing as the core Dask project.
 
-See https://github.com/dask/dask/blob/master/docs/release-procedure.md for instructions.
\ No newline at end of file
+See https://github.com/dask/dask/blob/master/docs/release-procedure.md for instructions.
diff --git a/docs/source/develop.rst b/docs/source/develop.rst
index 254eb914aaa..da593cf4b2b 100644
--- a/docs/source/develop.rst
+++ b/docs/source/develop.rst
@@ -39,9 +39,9 @@ To keep a fork in sync with the upstream source::
    git remote add upstream git@github.com:dask/distributed.git
    git remote -v
    git fetch -a upstream
-   git checkout master
-   git pull upstream master
-   git push origin master
+   git checkout main
+   git pull upstream main
+   git push origin main
 
 Test
 ----
@@ -92,7 +92,7 @@ The test suite contains three kinds of tests
     same event loop in the main thread.  These are good for testing complex
     logic and inspecting the state of the system directly.  They are also
     easier to debug and cause the fewest problems with shutdowns.
-2.  ``def test_foo(client)``: Tests with multiple processes forked from the master
+2.  ``def test_foo(client)``: Tests with multiple processes forked from the main
     process.  These are good for testing the synchronous (normal user) API and
     when triggering hard failures for resilience tests.
 3.  ``popen``: Tests that call out to the command line to start the system.
@@ -114,15 +114,15 @@ using the ``@gen_cluster`` style of test, e.g.
         assert isinstance(s, Scheduler)
         assert isinstance(a, Worker)
         assert isinstance(b, Worker)
-    
+
         future = c.submit(inc, 1)
         assert isinstance(future, Future)
         assert future.key in c.futures
-    
+
         # result = future.result()  # This synchronous API call would block
         result = await future
         assert result == 2
-    
+
         assert future.key in s.tasks
         assert future.key in a.data or future.key in b.data
 
diff --git a/docs/source/diagnosing-performance.rst b/docs/source/diagnosing-performance.rst
index 330194076d3..3b367b9b309 100644
--- a/docs/source/diagnosing-performance.rst
+++ b/docs/source/diagnosing-performance.rst
@@ -39,7 +39,7 @@ The main way to observe these times is with the task stream plot on the
 scheduler's ``/status`` page where the colors of the bars correspond to the
 colors listed above.
 
-.. image:: https://raw.githubusercontent.com/dask/dask-org/master/images/bokeh-task-stream.gif
+.. image:: https://raw.githubusercontent.com/dask/dask-org/main/images/bokeh-task-stream.gif
    :alt: Dask task stream
    :width: 50%
 
@@ -73,7 +73,7 @@ any of the bars in the profile will zoom the user into just that section, as is
 typical with most profiling tools.  There is a timeline at the bottom of the
 page to allow users to select different periods in time.
 
-.. image:: https://raw.githubusercontent.com/dask/dask-org/master/images/daskboard-profile.gif
+.. image:: https://raw.githubusercontent.com/dask/dask-org/main/images/daskboard-profile.gif
    :alt: Dask profiler
    :width: 70%
 
diff --git a/versioneer.py b/versioneer.py
index f250cde55b8..fa805b12b0f 100644
--- a/versioneer.py
+++ b/versioneer.py
@@ -606,7 +606,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose):
         # refs/heads/ and refs/tags/ prefixes that would let us distinguish
         # between branches and tags. By ignoring refnames without digits, we
         # filter out many common branch names like "release" and
-        # "stabilization", as well as "HEAD" and "master".
+        # "stabilization", as well as "HEAD" and "main".
         tags = set([r for r in refs if re.search(r'\d', r)])
         if verbose:
             print("discarding '%%s', no digits" %% ",".join(refs - tags))
@@ -998,7 +998,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose):
         # refs/heads/ and refs/tags/ prefixes that would let us distinguish
         # between branches and tags. By ignoring refnames without digits, we
         # filter out many common branch names like "release" and
-        # "stabilization", as well as "HEAD" and "master".
+        # "stabilization", as well as "HEAD" and "main".
         tags = set([r for r in refs if re.search(r'\d', r)])
         if verbose:
             print("discarding '%s', no digits" % ",".join(refs - tags))

From 10e0c6fa8c5e7395e67742f66479a90b40db7ce9 Mon Sep 17 00:00:00 2001
From: Doug Davis <ddavis@ddavis.io>
Date: Tue, 9 Mar 2021 12:23:28 -0500
Subject: [PATCH 1174/1550] Add docstring for dashboard_link property (#4572)

---
 distributed/client.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/distributed/client.py b/distributed/client.py
index fa17a6b710d..95347b1a663 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -808,6 +808,23 @@ def asynchronous(self):
 
     @property
     def dashboard_link(self):
+        """Link to the scheduler's dashboard.
+
+        Returns
+        -------
+        str
+            Dashboard URL.
+
+        Examples
+        --------
+        Opening the dashboard in your default web browser:
+
+        >>> import webbrowser
+        >>> from distributed import Client
+        >>> client = Client()
+        >>> webbrowser.open(client.dashboard_link)
+
+        """
         try:
             return self.cluster.dashboard_link
         except AttributeError:

From 51541d88607c9af8e6d250b96ed9885a5e69c385 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Tue, 9 Mar 2021 13:06:56 -0600
Subject: [PATCH 1175/1550] Drop support for Python 3.6 (#4390)

---
 .github/workflows/tests.yaml                |  8 +--
 continuous_integration/environment-3.6.yaml | 47 ---------------
 continuous_integration/environment-3.7.yaml |  2 +-
 distributed/client.py                       |  9 ---
 distributed/comm/tests/test_comms.py        |  3 +-
 distributed/compatibility.py                | 13 ----
 distributed/deploy/tests/test_local.py      |  3 -
 distributed/metrics.py                      | 67 +++------------------
 distributed/pubsub.py                       |  4 +-
 distributed/tests/test_client.py            |  5 --
 distributed/tests/test_core.py              |  5 --
 distributed/tests/test_diskutils.py         |  2 -
 distributed/tests/test_scheduler.py         |  3 -
 distributed/utils.py                        |  4 +-
 distributed/variable.py                     |  3 +-
 docs/source/install.rst                     |  4 +-
 requirements.txt                            |  1 -
 setup.py                                    |  3 +-
 18 files changed, 22 insertions(+), 164 deletions(-)
 delete mode 100644 continuous_integration/environment-3.6.yaml

diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
index c27ae784ea8..9d6b6a526c4 100644
--- a/.github/workflows/tests.yaml
+++ b/.github/workflows/tests.yaml
@@ -10,11 +10,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python-version: ["3.6", "3.7", "3.8", "3.9"]
-        exclude:
-          # Several failures; Python 3.6 to be phased out soon
-          - os: macos-latest
-            python-version: "3.6"
+        python-version: ["3.7", "3.8", "3.9"]
 
         # Uncomment to stress-test the test suite for random failures
         # This will take a LONG time and delay all PRs across the whole github.com/dask!
@@ -23,6 +19,8 @@ jobs:
     steps:
       - name: Checkout source
         uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
 
       - name: Setup Conda Environment
         uses: conda-incubator/setup-miniconda@v2
diff --git a/continuous_integration/environment-3.6.yaml b/continuous_integration/environment-3.6.yaml
deleted file mode 100644
index 7a13f63b232..00000000000
--- a/continuous_integration/environment-3.6.yaml
+++ /dev/null
@@ -1,47 +0,0 @@
-name: dask-distributed
-channels:
-  - conda-forge
-  - defaults
-dependencies:
-  - python=3.6
-  - pip
-  - asyncssh
-  - bokeh
-  - click
-  - cloudpickle
-  - contextvars  # Backport from Python 3.7
-  - dask  # overridden by git tip below
-  - filesystem-spec
-  - h5py
-  - ipykernel
-  - ipywidgets
-  - joblib
-  - jupyter_client
-  - msgpack-python
-  - netcdf4
-  - paramiko
-  - pickle5  # Backport from Python 3.8
-  - prometheus_client
-  - psutil
-  - pytest
-  - pytest-asyncio<0.14.0
-  - pytest-faulthandler
-  - pytest-repeat
-  - pytest-rerunfailures
-  - pytest-timeout
-  - requests
-  - s3fs
-  - scikit-learn
-  - scipy
-  - sortedcollections
-  - tblib
-  - toolz
-  - tornado=5  # Only tested here
-  - zict
-  - zstandard
-  - pip:
-      # The '.git@master' is a hack needed to work around the failure
-      #     ERROR: Package 'dask' requires a different Python: 3.6.12 not in '>=3.6'
-      # It only happens in CI; cannot reproduce locally
-      - git+https://github.com/dask/dask.git@master
-      - keras
diff --git a/continuous_integration/environment-3.7.yaml b/continuous_integration/environment-3.7.yaml
index 7057155b470..bb7838fe205 100644
--- a/continuous_integration/environment-3.7.yaml
+++ b/continuous_integration/environment-3.7.yaml
@@ -36,7 +36,7 @@ dependencies:
   - sortedcollections
   - tblib
   - toolz
-  - tornado=6
+  - tornado=5 # Only tested here
   - zict
   - zstandard
   - pip:
diff --git a/distributed/client.py b/distributed/client.py
index 95347b1a663..7d0c72e3f90 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -756,15 +756,6 @@ def as_current(self):
         method to return self. Any Future objects deserialized inside this context
         manager will be automatically attached to this Client.
         """
-        # In Python 3.6, contextvars are thread-local but not Task-local.
-        # We can still detect a race condition though.
-        if sys.version_info < (3, 7) and _current_client.get() not in (self, None):
-            raise RuntimeError(
-                "Detected race condition where multiple asynchronous clients tried "
-                "entering the as_current() context manager at the same time. "
-                "Please upgrade to Python 3.7+."
-            )
-
         tok = _current_client.set(self)
         try:
             yield
diff --git a/distributed/comm/tests/test_comms.py b/distributed/comm/tests/test_comms.py
index e3c46984092..56f3c7a1b11 100644
--- a/distributed/comm/tests/test_comms.py
+++ b/distributed/comm/tests/test_comms.py
@@ -223,8 +223,7 @@ async def handle_comm(comm):
     assert not handle_comm_called
 
     writer.close()
-    if hasattr(writer, "wait_closed"):  # always true for python >= 3.7, but not for 3.6
-        await writer.wait_closed()
+    await writer.wait_closed()
 
 
 @pytest.mark.asyncio
diff --git a/distributed/compatibility.py b/distributed/compatibility.py
index 6141f3559a7..982b7017951 100644
--- a/distributed/compatibility.py
+++ b/distributed/compatibility.py
@@ -11,16 +11,3 @@
 MACOS = sys.platform == "darwin"
 WINDOWS = sys.platform.startswith("win")
 TORNADO6 = tornado.version_info[0] >= 6
-PY37 = sys.version_info[:2] >= (3, 7)
-
-if sys.version_info[:2] >= (3, 7):
-    from asyncio import get_running_loop
-else:
-
-    def get_running_loop():
-        from asyncio import _get_running_loop
-
-        loop = _get_running_loop()
-        if loop is None:
-            raise RuntimeError("no running event loop")
-        return loop
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index 5a35f36695c..8eaea2ba83f 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -1020,9 +1020,6 @@ async def test_capture_security(cleanup, temporary):
 
 
 @pytest.mark.asyncio
-@pytest.mark.skipif(
-    sys.version_info < (3, 7), reason="asyncio.all_tasks not implemented"
-)
 async def test_no_danglng_asyncio_tasks(cleanup):
     start = asyncio.all_tasks()
     async with LocalCluster(asynchronous=True, processes=False):
diff --git a/distributed/metrics.py b/distributed/metrics.py
index 0f7d78a8129..163a982b792 100755
--- a/distributed/metrics.py
+++ b/distributed/metrics.py
@@ -84,60 +84,13 @@ def resync(self):
     # Under modern Unices, time.time() should be good enough
     time = timemod.time
 
-
-def _native_thread_time():
-    # Python 3.7+, not all platforms
-    return timemod.thread_time()
-
-
-def _linux_thread_time():
-    # Use hardcoded CLOCK_THREAD_CPUTIME_ID on Python 3 <= 3.6
-    if sys.platform != "linux":
-        raise OSError
-    return timemod.clock_gettime(3)
-
-
-def _native_process_time():
-    # Python 3, should work everywhere
-    return timemod.process_time()
-
-
-def _native_clock_func():
-    # time.clock() unfortunately has different semantics depending on the
-    # platform.  On POSIX it's a per-process CPU timer (with possibly
-    # poor resolution).  On Windows it's a high-resolution wall clock timer.
-    return timemod.clock()
-
-
-def _detect_process_time():
-    """
-    Return a per-process CPU timer function if possible, otherwise
-    a wall-clock timer.
-    """
-    for func in [_native_process_time]:
-        try:
-            func()
-            return func
-        except (AttributeError, OSError):
-            pass
-    # Only Python 2?
-    return _native_clock_func
-
-
-def _detect_thread_time():
-    """
-    Return a per-thread CPU timer function if possible, otherwise
-    a per-process CPU timer function, or at worse a wall-clock timer.
-    """
-    for func in [_native_thread_time, _linux_thread_time, _native_process_time]:
-        try:
-            func()
-            return func
-        except (AttributeError, OSError):
-            pass
-    # Only Python 2?
-    return time
-
-
-process_time = _detect_process_time()
-thread_time = _detect_thread_time()
+process_time = timemod.process_time
+
+# Get a per-thread CPU timer function if possible, otherwise
+# use a per-process CPU timer function.
+try:
+    # thread_time is supported on Python 3.7+ but not all platforms
+    thread_time = timemod.thread_time
+except (AttributeError, OSError):
+    # process_time is supported on Python 3.3+ everywhere
+    thread_time = process_time
diff --git a/distributed/pubsub.py b/distributed/pubsub.py
index 3b580d6180a..91200be06eb 100644
--- a/distributed/pubsub.py
+++ b/distributed/pubsub.py
@@ -1,6 +1,5 @@
 import asyncio
 from collections import defaultdict, deque
-from contextlib import suppress
 import logging
 import threading
 import weakref
@@ -421,8 +420,7 @@ async def _():
             try:
                 await asyncio.wait_for(_(), timeout2)
             finally:
-                with suppress(RuntimeError):  # Python 3.6 fails here sometimes
-                    self.condition.release()
+                self.condition.release()
 
         return self.buffer.popleft()
 
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 100b52eaf81..9caa3ccb08f 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -3668,7 +3668,6 @@ async def test_reconnect_timeout(c, s):
 @pytest.mark.avoid_ci(reason="hangs on github actions ubuntu-latest CI")
 @pytest.mark.slow
 @pytest.mark.skipif(WINDOWS, reason="num_fds not supported on windows")
-@pytest.mark.skipif(sys.version_info < (3, 7), reason="TODO: intermittent failures")
 @pytest.mark.parametrize("worker,count,repeat", [(Worker, 100, 5), (Nanny, 10, 20)])
 def test_open_close_many_workers(loop, worker, count, repeat):
     psutil = pytest.importorskip("psutil")
@@ -4054,10 +4053,6 @@ def run2():
     t2.join()
 
 
-@pytest.mark.xfail(
-    sys.version_info < (3, 7),
-    reason="Python 3.6 contextvars are not copied on Task creation",
-)
 @gen_cluster(client=False)
 async def test_as_current_is_task_local(s, a, b):
     l1 = asyncio.Lock()
diff --git a/distributed/tests/test_core.py b/distributed/tests/test_core.py
index e9b77c0b1a3..071b6b039c7 100644
--- a/distributed/tests/test_core.py
+++ b/distributed/tests/test_core.py
@@ -1,7 +1,6 @@
 import asyncio
 import os
 import socket
-import sys
 import threading
 import weakref
 import warnings
@@ -180,10 +179,6 @@ class MyServer(Server):
     default_port = 8756
 
 
-@pytest.mark.skipif(
-    sys.version_info < (3, 7),
-    reason="asynccontextmanager not avaiable before Python 3.7",
-)
 @pytest.mark.asyncio
 async def test_server_listen():
     """
diff --git a/distributed/tests/test_diskutils.py b/distributed/tests/test_diskutils.py
index f4afb6d66d4..3b97fde1498 100644
--- a/distributed/tests/test_diskutils.py
+++ b/distributed/tests/test_diskutils.py
@@ -277,8 +277,6 @@ def _test_workspace_concurrency(tmpdir, timeout, max_procs):
 def test_workspace_concurrency(tmpdir):
     if WINDOWS:
         raise pytest.xfail.Exception("TODO: unknown failure on windows")
-    if sys.version_info < (3, 7):
-        raise pytest.xfail.Exception("TODO: unknown failure on Python 3.6")
     _test_workspace_concurrency(tmpdir, 5.0, 6)
 
 
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 0d4cad348c2..701a874fbcc 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -1818,9 +1818,6 @@ async def test_get_task_duration(c, s, a, b):
 
 
 @pytest.mark.asyncio
-@pytest.mark.skipif(
-    sys.version_info < (3, 7), reason="asyncio.all_tasks not implemented"
-)
 async def test_no_danglng_asyncio_tasks(cleanup):
     start = asyncio.all_tasks()
     async with Scheduler(port=0) as s:
diff --git a/distributed/utils.py b/distributed/utils.py
index dd6ca2fad02..115e9578b6b 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -55,7 +55,7 @@
 except ImportError:
     PollIOLoop = None  # dropped in tornado 6.0
 
-from .compatibility import PYPY, WINDOWS, get_running_loop
+from .compatibility import PYPY, WINDOWS
 from .metrics import time
 
 
@@ -1165,7 +1165,7 @@ def reset_logger_locks():
 
     if is_kernel():
         try:
-            get_running_loop()
+            asyncio.get_running_loop()
         except RuntimeError:
             is_kernel_and_no_running_loop = True
 
diff --git a/distributed/variable.py b/distributed/variable.py
index c3fdc94d0d7..83bea566b45 100644
--- a/distributed/variable.py
+++ b/distributed/variable.py
@@ -89,8 +89,7 @@ async def _():  # Python 3.6 is odd and requires special help here
 
                 await asyncio.wait_for(_(), timeout=left)
             finally:
-                with suppress(RuntimeError):  # Python 3.6 loses lock on finally clause
-                    self.started.release()
+                self.started.release()
 
         record = self.variables[name]
         if record["type"] == "Future":
diff --git a/docs/source/install.rst b/docs/source/install.rst
index db1bf316400..695cd3e777f 100644
--- a/docs/source/install.rst
+++ b/docs/source/install.rst
@@ -38,6 +38,6 @@ Notes
 <https://trac.macports.org/ticket/50058>`_.  with Python from macports that
 makes executables be placed in a location that is not available by default. A
 simple solution is to extend the ``PATH`` environment variable to the location
-where Python from macports install the binaries. For example, for Python 3.6::
+where Python from macports install the binaries. For example, for Python 3.7::
 
-    $ export PATH=/opt/local/Library/Frameworks/Python.framework/Versions/3.6/bin:$PATH
+    $ export PATH=/opt/local/Library/Frameworks/Python.framework/Versions/3.7/bin:$PATH
diff --git a/requirements.txt b/requirements.txt
index 37d23c45e69..fd8159cfefe 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,5 @@
 click >= 6.6
 cloudpickle >= 1.5.0
-contextvars;python_version<'3.7'
 dask >= 2021.03.0
 msgpack >= 0.6.0
 psutil >= 5.0
diff --git a/setup.py b/setup.py
index 5483e0b963d..8aaaeee5de8 100755
--- a/setup.py
+++ b/setup.py
@@ -67,7 +67,7 @@
     url="https://distributed.dask.org",
     maintainer="Matthew Rocklin",
     maintainer_email="mrocklin@gmail.com",
-    python_requires=">=3.6",
+    python_requires=">=3.7",
     license="BSD",
     package_data={
         "": ["templates/index.html", "template.html"],
@@ -89,7 +89,6 @@
         "License :: OSI Approved :: BSD License",
         "Operating System :: OS Independent",
         "Programming Language :: Python",
-        "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",

From 93bcc943bf380d7a774fd96ea64c2ba554e22301 Mon Sep 17 00:00:00 2001
From: Julia Signell <jsignell@gmail.com>
Date: Tue, 9 Mar 2021 14:09:55 -0500
Subject: [PATCH 1176/1550] Dask master -> main (#4569)

---
 docs/release-procedure.md           | 2 +-
 docs/source/priority.rst            | 2 +-
 docs/source/scheduling-policies.rst | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/release-procedure.md b/docs/release-procedure.md
index bf1b904c186..30792206bd8 100644
--- a/docs/release-procedure.md
+++ b/docs/release-procedure.md
@@ -1,3 +1,3 @@
 Distributed follows a similar procedure for releasing as the core Dask project.
 
-See https://github.com/dask/dask/blob/master/docs/release-procedure.md for instructions.
+See https://github.com/dask/dask/blob/main/docs/release-procedure.md for instructions.
diff --git a/docs/source/priority.rst b/docs/source/priority.rst
index 8b4729cbb27..374f04628e8 100644
--- a/docs/source/priority.rst
+++ b/docs/source/priority.rst
@@ -69,7 +69,7 @@ Dask uses the following priorities, in order:
 3.  **Graph Structure**: Within any given computation (a compute or persist
     call) Dask orders tasks in such a way as to minimize the memory-footprint
     of the computation.  This is discussed in more depth in the
-    `task ordering documentation <https://github.com/dask/dask/blob/master/dask/order.py>`_.
+    `task ordering documentation <https://github.com/dask/dask/blob/main/dask/order.py>`_.
 
 If multiple tasks each have exactly the same priorities outlined above, then
 the order in which tasks arrive at a worker, in a last in first out manner,
diff --git a/docs/source/scheduling-policies.rst b/docs/source/scheduling-policies.rst
index e01197c0ccc..cf93fb612ff 100644
--- a/docs/source/scheduling-policies.rst
+++ b/docs/source/scheduling-policies.rst
@@ -78,7 +78,7 @@ can be used to break ties and helps us to prioritize nodes with longer critical
 paths and nodes with many children.  The actual algorithms used are somewhat
 more complex and are described in detail in `dask/order.py`_
 
-.. _`dask/order.py`: https://github.com/dask/dask/blob/master/dask/order.py
+.. _`dask/order.py`: https://github.com/dask/dask/blob/main/dask/order.py
 
 Initial Task Placement
 ~~~~~~~~~~~~~~~~~~~~~~
@@ -122,7 +122,7 @@ scheduler, and workers at various points in the computation.
     computing deeply before broadly, preferring critical paths, preferring
     nodes with many dependencies, etc..  This is the same logic used by the
     single-machine scheduler and lives in `dask/order.py
-    <https://github.com/dask/dask/blob/master/dask/order.py>`_.
+    <https://github.com/dask/dask/blob/main/dask/order.py>`_.
 2.  When the graph reaches the scheduler the scheduler changes each of these
     numeric priorities into a tuple of two numbers, the first of which is an
     increasing counter, the second of which is the client-generated priority

From e734fc3de5d6c3a938413fb0a3dfa08d298e2fbe Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Wed, 10 Mar 2021 14:30:28 -0600
Subject: [PATCH 1177/1550] Remove outdated macOS build badge from README
 (#4576)

---
 README.rst | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/README.rst b/README.rst
index 77cb2985809..9d76f73165e 100644
--- a/README.rst
+++ b/README.rst
@@ -1,15 +1,13 @@
 Distributed
 ===========
 
-|Linux/Windows Build Status| |MacOS Build Status| |Doc Status| |Gitter| |Version Status| |NumFOCUS|
+|Test Status| |Doc Status| |Gitter| |Version Status| |NumFOCUS|
 
 A library for distributed computation.  See documentation_ for more details.
 
 .. _documentation: https://distributed.dask.org
-.. |Linux/Windows Build Status| image:: https://github.com/dask/distributed/workflows/Tests/badge.svg?branch=main
+.. |Test Status| image:: https://github.com/dask/distributed/workflows/Tests/badge.svg?branch=main
    :target: https://github.com/dask/distributed/actions?query=workflow%3A%22Tests%22
-.. |MacOS Build Status| image:: https://github.com/dask/distributed/workflows/MacOS%20tests/badge.svg?branch=main
-   :target: https://github.com/dask/distributed/actions?query=workflow%3A%22MacOS+tests%22
 .. |Doc Status| image:: https://readthedocs.org/projects/distributed/badge/?version=latest
    :target: https://distributed.dask.org
    :alt: Documentation Status

From bd65bef6dcdea1f057ac782a28db02cfe77de4e3 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Thu, 11 Mar 2021 13:30:34 -0600
Subject: [PATCH 1178/1550] Add GitHub actions workflow to cancel duplicate
 builds (#4581)

---
 .github/workflows/cancel.yml | 15 +++++++++++++++
 1 file changed, 15 insertions(+)
 create mode 100644 .github/workflows/cancel.yml

diff --git a/.github/workflows/cancel.yml b/.github/workflows/cancel.yml
new file mode 100644
index 00000000000..2f079bac01d
--- /dev/null
+++ b/.github/workflows/cancel.yml
@@ -0,0 +1,15 @@
+name: Cancel
+
+on:
+  workflow_run:
+    workflows: ["Tests"]
+    types:
+      - requested
+
+jobs:
+  cancel:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: styfle/cancel-workflow-action@0.8.0
+      with:
+        workflow_id: ${{ github.event.workflow.id }}
\ No newline at end of file

From 38b0f05b0eccdd42c96af66e23e4db338a5298e3 Mon Sep 17 00:00:00 2001
From: Min RK <benjaminrk@gmail.com>
Date: Fri, 12 Mar 2021 19:25:38 +0100
Subject: [PATCH 1179/1550] IPython magics: remove deprecated ioloop
 workarounds (#4530)

---
 distributed/_ipython_utils.py     | 15 ++-------------
 distributed/tests/test_ipython.py |  7 +------
 2 files changed, 3 insertions(+), 19 deletions(-)

diff --git a/distributed/_ipython_utils.py b/distributed/_ipython_utils.py
index 0fe97c7280e..8aa1fe7ad7b 100644
--- a/distributed/_ipython_utils.py
+++ b/distributed/_ipython_utils.py
@@ -199,16 +199,8 @@ def start_ipython(ip=None, ns=None, log=None):
     if get_ipython() is not None:
         raise RuntimeError("Cannot start IPython, it's already running.")
 
-    from zmq.eventloop.ioloop import ZMQIOLoop
     from ipykernel.kernelapp import IPKernelApp
 
-    # save the global IOLoop instance
-    # since IPython relies on it, but we are going to put it in a thread.
-    save_inst = IOLoop.instance()
-    IOLoop.clear_instance()
-    zmq_loop = ZMQIOLoop()
-    zmq_loop.install()
-
     # start IPython, disabling its signal handlers that won't work due to running in a thread:
     app = IPKernelApp.instance(log=log)
     # Don't connect to the history database
@@ -234,20 +226,17 @@ def _start():
         app.kernel.pre_handler_hook = noop
         app.kernel.post_handler_hook = noop
         app.kernel.start()
-        app.kernel.loop = IOLoop.instance()
         # save self in the IPython namespace as 'worker'
         # inject things into the IPython namespace
         if ns:
             app.kernel.shell.user_ns.update(ns)
         evt.set()
-        zmq_loop.start()
+        # start the app's IOLoop in its thread
+        IOLoop.current().start()
 
     zmq_loop_thread = Thread(target=_start)
     zmq_loop_thread.daemon = True
     zmq_loop_thread.start()
     assert evt.wait(timeout=5), "IPython didn't start in a reasonable amount of time."
 
-    # put the global IOLoop instance back:
-    IOLoop.clear_instance()
-    save_inst.install()
     return app
diff --git a/distributed/tests/test_ipython.py b/distributed/tests/test_ipython.py
index a6d387589e6..51d7ae85f26 100644
--- a/distributed/tests/test_ipython.py
+++ b/distributed/tests/test_ipython.py
@@ -2,7 +2,6 @@
 
 import pytest
 from tlz import first
-import tornado
 
 from distributed import Client
 from distributed.utils_test import cluster, mock_ipython
@@ -15,11 +14,7 @@ def need_functional_ipython(func):
         import jupyter_client  # noqa: F401
     except ImportError:
         return pytest.mark.skip("need ipykernel and jupyter_client installed")(func)
-    if tornado.version_info >= (5,):
-        # https://github.com/ipython/ipykernel/issues/277
-        return pytest.mark.skip("IPython kernel broken with Tornado 5")(func)
-    else:
-        return func
+    return func
 
 
 @pytest.mark.ipython

From 4ce8c65039e9b531878f9d77cf884920690a3745 Mon Sep 17 00:00:00 2001
From: Min RK <benjaminrk@gmail.com>
Date: Fri, 12 Mar 2021 19:26:13 +0100
Subject: [PATCH 1180/1550] IPython magics: remove deprecated ioloop
 workarounds (#4530)


From 2d6736b3e7fbe77fc0986714322ce2b91fe83cdc Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Mon, 15 Mar 2021 09:43:54 -0500
Subject: [PATCH 1181/1550] Update PipInstall plugin command (#4584)

* Update command PipInstall uses

* Update test
---
 distributed/diagnostics/plugin.py | 3 +--
 distributed/tests/test_worker.py  | 2 +-
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/distributed/diagnostics/plugin.py b/distributed/diagnostics/plugin.py
index 1425b89b63d..58b5adbc585 100644
--- a/distributed/diagnostics/plugin.py
+++ b/distributed/diagnostics/plugin.py
@@ -237,9 +237,8 @@ async def setup(self, worker):
         async with Lock(socket.gethostname()):  # don't clobber one installation
             logger.info("Pip installing the following packages: %s", self.packages)
             proc = subprocess.Popen(
-                [sys.executable, "-m", "pip"]
+                [sys.executable, "-m", "pip", "install"]
                 + self.pip_options
-                + ["install"]
                 + self.packages,
                 stdout=subprocess.PIPE,
                 stderr=subprocess.PIPE,
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 46a9195a869..b15030d3599 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -1641,7 +1641,7 @@ async def test_pip_install(c, s, a, b):
 
             args = p2.call_args[0][0]
             assert "python" in args[0]
-            assert args[1:] == ["-m", "pip", "--upgrade", "install", "requests"]
+            assert args[1:] == ["-m", "pip", "install", "--upgrade", "requests"]
 
 
 @gen_cluster(client=True)

From 7c6709fd8d2ee6356ba43ceff64cd04e9803b2a1 Mon Sep 17 00:00:00 2001
From: "Mads R. B. Kristensen" <madsbk@gmail.com>
Date: Mon, 15 Mar 2021 16:59:56 +0100
Subject: [PATCH 1182/1550] Multi-lock extension (#4503)

---
 distributed/__init__.py               |   1 +
 distributed/multi_lock.py             | 238 ++++++++++++++++++++++++++
 distributed/scheduler.py              |   2 +
 distributed/tests/test_multi_locks.py | 210 +++++++++++++++++++++++
 docs/source/api.rst                   |   3 +
 5 files changed, 454 insertions(+)
 create mode 100644 distributed/multi_lock.py
 create mode 100644 distributed/tests/test_multi_locks.py

diff --git a/distributed/__init__.py b/distributed/__init__.py
index 448e52cd28b..475288a3b32 100644
--- a/distributed/__init__.py
+++ b/distributed/__init__.py
@@ -21,6 +21,7 @@
     get_task_metadata,
 )
 from .lock import Lock
+from .multi_lock import MultiLock
 from .nanny import Nanny
 from .pubsub import Pub, Sub
 from .queues import Queue
diff --git a/distributed/multi_lock.py b/distributed/multi_lock.py
new file mode 100644
index 00000000000..6d1df68c2f4
--- /dev/null
+++ b/distributed/multi_lock.py
@@ -0,0 +1,238 @@
+import asyncio
+from collections import defaultdict
+import logging
+from typing import Hashable, List
+import uuid
+
+from .client import Client
+from .utils import log_errors, TimeoutError
+from .worker import get_worker
+from .utils import parse_timedelta
+
+logger = logging.getLogger(__name__)
+
+
+class MultiLockExtension:
+    """An extension for the scheduler to manage MultiLocks
+
+    This adds the following routes to the scheduler
+
+    *  multi_lock_acquire
+    *  multi_lock_release
+
+    The approach is to maintain `self.locks` that maps a lock (unique name given to
+    `MultiLock(names=, ...)` at creation) to a list of users (instances of `MultiLock`)
+    that "requests" the lock. Additionally, `self.requests` maps a user to its requested
+    locks and `self.requests_left` maps a user to the number of locks still need.
+
+    Every time a user `x` gets to the front in `self.locks[name] = [x, ...]` it means
+    that `x` now holds the lock `name` and when it holds all the requested locks
+    `acquire()` can return.
+
+    Finally, `self.events` contains all the events users are waiting on to finish.
+    """
+
+    def __init__(self, scheduler):
+        self.scheduler = scheduler
+        self.locks = defaultdict(list)  # lock -> users
+        self.requests = {}  # user -> locks
+        self.requests_left = {}  # user -> locks still needed
+        self.events = {}
+
+        self.scheduler.handlers.update(
+            {"multi_lock_acquire": self.acquire, "multi_lock_release": self.release}
+        )
+
+        self.scheduler.extensions["multi_locks"] = self
+
+    def _request_locks(self, locks: List[str], id: Hashable, num_locks: int):
+        """Request locks
+
+        Parameters
+        ----------
+        locks: List[str]
+            Names of the locks to request.
+        id: Hashable
+            Identifier of the `MultiLock` instance requesting the locks.
+        num_locks: int
+            Number of locks in `locks` requesting
+
+        Return
+        ------
+        result: bool
+            Whether `num_locks` requested locks are free immediately or not.
+        """
+        assert id not in self.requests
+        self.requests[id] = set(locks)
+        assert len(locks) >= num_locks and num_locks > 0
+        self.requests_left[id] = num_locks
+
+        locks = sorted(locks, key=lambda x: len(self.locks[x]))
+        for i, lock in enumerate(locks):
+            self.locks[lock].append(id)
+            if len(self.locks[lock]) == 1:  # The lock was free
+                self.requests_left[id] -= 1
+                if self.requests_left[id] == 0:  # Got all locks needed
+                    # Since we got all locks need, we can remove the rest of the requests
+                    self.requests[id] -= set(locks[i + 1 :])
+                    return True
+        return False
+
+    def _refain_locks(self, locks, id):
+        """Cancel/release previously requested/acquired locks
+
+        Parameters
+        ----------
+        locks: List[str]
+            Names of the locks to refain.
+        id: Hashable
+            Identifier of the `MultiLock` instance refraining the locks.
+        """
+        waiters_ready = set()
+        for lock in locks:
+            if self.locks[lock][0] == id:
+                self.locks[lock].pop(0)
+                if self.locks[lock]:
+                    new_first = self.locks[lock][0]
+                    self.requests_left[new_first] -= 1
+                    if self.requests_left[new_first] <= 0:
+                        # Notice, `self.requests_left[new_first]` might go below zero
+                        # if more locks are freed than requested.
+                        self.requests_left[new_first] = 0
+                        waiters_ready.add(new_first)
+            else:
+                self.locks[lock].remove(id)
+            assert id not in self.locks[lock]
+        del self.requests[id]
+        del self.requests_left[id]
+
+        for waiter in waiters_ready:
+            self.scheduler.loop.add_callback(self.events[waiter].set)
+
+    async def acquire(
+        self, comm=None, locks=None, id=None, timeout=None, num_locks=None
+    ):
+        with log_errors():
+            if not self._request_locks(locks, id, num_locks):
+                assert id not in self.events
+                event = asyncio.Event()
+                self.events[id] = event
+                future = event.wait()
+                if timeout is not None:
+                    future = asyncio.wait_for(future, timeout)
+                try:
+                    await future
+                except TimeoutError:
+                    self._refain_locks(locks, id)
+                    return False
+                finally:
+                    del self.events[id]
+            # At this point `id` acquired all `locks`
+            assert self.requests_left[id] == 0
+            return True
+
+    def release(self, comm=None, id=None):
+        with log_errors():
+            self._refain_locks(self.requests[id], id)
+
+
+class MultiLock:
+    """Distributed Centralized Lock
+
+    Parameters
+    ----------
+    names: List[str]
+        Names of the locks to acquire. Choosing the same name allows two
+        disconnected processes to coordinate a lock.
+    client: Client (optional)
+        Client to use for communication with the scheduler.  If not given, the
+        default global client will be used.
+
+    Examples
+    --------
+    >>> lock = MultiLock(['x', 'y'])  # doctest: +SKIP
+    >>> lock.acquire(timeout=1)  # doctest: +SKIP
+    >>> # do things with protected resource 'x' and 'y'
+    >>> lock.release()  # doctest: +SKIP
+    """
+
+    def __init__(self, names=[], client=None):
+        try:
+            self.client = client or Client.current()
+        except ValueError:
+            # Initialise new client
+            self.client = get_worker().client
+
+        self.names = names
+        self.id = uuid.uuid4().hex
+        self._locked = False
+
+    def acquire(self, blocking=True, timeout=None, num_locks=None):
+        """Acquire the lock
+
+        Parameters
+        ----------
+        blocking : bool, optional
+            If false, don't wait on the lock in the scheduler at all.
+        timeout : string or number or timedelta, optional
+            Seconds to wait on the lock in the scheduler.  This does not
+            include local coroutine time, network transfer time, etc..
+            It is forbidden to specify a timeout when blocking is false.
+            Instead of number of seconds, it is also possible to specify
+            a timedelta in string format, e.g. "200ms".
+        num_locks : int, optional
+            Number of locks needed. If None, all locks are needed
+
+        Examples
+        --------
+        >>> lock = MultiLock(['x', 'y'])  # doctest: +SKIP
+        >>> lock.acquire(timeout="1s")  # doctest: +SKIP
+
+        Returns
+        -------
+        True or False whether or not it successfully acquired the lock
+        """
+        timeout = parse_timedelta(timeout)
+
+        if not blocking:
+            if timeout is not None:
+                raise ValueError("can't specify a timeout for a non-blocking call")
+            timeout = 0
+
+        result = self.client.sync(
+            self.client.scheduler.multi_lock_acquire,
+            locks=self.names,
+            id=self.id,
+            timeout=timeout,
+            num_locks=num_locks or len(self.names),
+        )
+        self._locked = True
+        return result
+
+    def release(self):
+        """ Release the lock if already acquired """
+        if not self.locked():
+            raise ValueError("Lock is not yet acquired")
+        ret = self.client.sync(self.client.scheduler.multi_lock_release, id=self.id)
+        self._locked = False
+        return ret
+
+    def locked(self):
+        return self._locked
+
+    def __enter__(self):
+        self.acquire()
+        return self
+
+    def __exit__(self, *args, **kwargs):
+        self.release()
+
+    async def __aenter__(self):
+        await self.acquire()
+        return self
+
+    async def __aexit__(self, *args, **kwargs):
+        await self.release()
+
+    def __reduce__(self):
+        return (type(self), (self.names,))
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 733b98b620d..d4e0b20dcb8 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -81,6 +81,7 @@
 from .semaphore import SemaphoreExtension
 from .recreate_exceptions import ReplayExceptionScheduler
 from .lock import LockExtension
+from .multi_lock import MultiLockExtension
 from .event import EventExtension
 from .pubsub import PubSubSchedulerExtension
 from .stealing import WorkStealing
@@ -173,6 +174,7 @@ def nogil(func):
 
 DEFAULT_EXTENSIONS = [
     LockExtension,
+    MultiLockExtension,
     PublishExtension,
     ReplayExceptionScheduler,
     QueueExtension,
diff --git a/distributed/tests/test_multi_locks.py b/distributed/tests/test_multi_locks.py
new file mode 100644
index 00000000000..c1a7cc55af5
--- /dev/null
+++ b/distributed/tests/test_multi_locks.py
@@ -0,0 +1,210 @@
+import asyncio
+from distributed.multi_lock import MultiLockExtension
+from time import sleep
+
+
+from distributed import MultiLock, get_client
+from distributed.metrics import time
+from distributed.utils_test import gen_cluster
+from distributed.utils_test import client, cluster_fixture, loop  # noqa F401
+
+
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 8)] * 2)
+async def test_single_lock(c, s, a, b):
+    await c.set_metadata("locked", False)
+
+    def f(_):
+        client = get_client()
+        with MultiLock(names=["x"]):
+            assert client.get_metadata("locked") is False
+            client.set_metadata("locked", True)
+            sleep(0.05)
+            assert client.get_metadata("locked") is True
+            client.set_metadata("locked", False)
+
+    futures = c.map(f, range(20))
+    await c.gather(futures)
+    ext: MultiLockExtension = s.extensions["multi_locks"]
+    assert not ext.events
+    assert not ext.requests
+    assert not ext.requests_left
+    assert all(len(l) == 0 for l in ext.locks.values())
+
+
+@gen_cluster(client=True)
+async def test_timeout(c, s, a, b):
+    ext: MultiLockExtension = s.extensions["multi_locks"]
+    lock1 = MultiLock(names=["x"])
+    result = await lock1.acquire()
+    assert result is True
+    assert ext.requests_left[lock1.id] == 0
+    assert ext.locks["x"] == [lock1.id]
+    assert not ext.events
+
+    lock2 = MultiLock(names=["x"])
+    assert lock1.id != lock2.id
+
+    start = time()
+    result = await lock2.acquire(timeout=0.1)
+    stop = time()
+    assert stop - start < 0.3
+    assert result is False
+    assert ext.locks["x"] == [lock1.id]
+    assert not ext.events
+
+    await lock1.release()
+
+
+@gen_cluster(client=True)
+async def test_timeout_wake_waiter(c, s, a, b):
+    ext: MultiLockExtension = s.extensions["multi_locks"]
+    l1 = MultiLock(names=["x"])
+    l2 = MultiLock(names=["x", "y"])
+    l3 = MultiLock(names=["y"])
+    await l1.acquire()
+
+    l2_acquire = asyncio.ensure_future(l2.acquire(timeout=1))
+    try:
+        await asyncio.wait_for(asyncio.shield(l2_acquire), 0.1)
+    except asyncio.TimeoutError:
+        pass
+    else:
+        assert False
+
+    l3_acquire = asyncio.ensure_future(l3.acquire())
+    try:
+        await asyncio.wait_for(asyncio.shield(l3_acquire), 0.1)
+    except asyncio.TimeoutError:
+        pass
+    else:
+        assert False
+
+    assert await l2_acquire is False
+    assert await l3_acquire
+    l1.release()
+    l3.release()
+
+
+@gen_cluster(client=True)
+async def test_multiple_locks(c, s, a, b):
+    ext: MultiLockExtension = s.extensions["multi_locks"]
+    l1 = MultiLock(names=["l1"])
+    l2 = MultiLock(names=["l2"])
+    l3 = MultiLock(names=["l1", "l2"])
+
+    # Both `l1` and `l2` are free to acquire
+    assert await l1.acquire()
+    assert await l2.acquire()
+    assert list(ext.locks.keys()) == ["l1", "l2"]
+    assert list(ext.locks.values()) == [[l1.id], [l2.id]]
+    assert list(ext.requests.keys()) == [l1.id, l2.id]
+    assert list(ext.requests_left.values()) == [0, 0]
+    assert not ext.events
+
+    # Since `l3` requires both `l1` and `l2`, it isn't available immediately
+    l3_acquire = asyncio.ensure_future(l3.acquire())
+    try:
+        await asyncio.wait_for(asyncio.shield(l3_acquire), 0.1)
+    except asyncio.TimeoutError:
+        assert list(ext.locks.keys()) == ["l1", "l2"]
+        assert list(ext.locks.values()) == [[l1.id, l3.id], [l2.id, l3.id]]
+        assert ext.requests[l3.id] == {"l1", "l2"}
+        assert ext.requests_left[l3.id] == 2
+        assert l3.id in ext.events
+    else:
+        assert False  # We except a TimeoutError since `l3` isn't availabe
+
+    # Releasing `l1` isn't enough since `l3` also requires `l2`
+    await l1.release()
+    try:
+        await asyncio.wait_for(asyncio.shield(l3_acquire), 0.1)
+    except asyncio.TimeoutError:
+        # `l3` now only wait on `l2`
+        assert list(ext.locks.keys()) == ["l1", "l2"]
+        assert list(ext.locks.values()) == [[l3.id], [l2.id, l3.id]]
+        assert ext.requests[l3.id] == {"l1", "l2"}
+        assert ext.requests_left[l3.id] == 1
+        assert l3.id in ext.events
+    else:
+        assert False
+
+    # Releasing `l2` should make `l3` available
+    await l2.release()
+    assert list(ext.locks.keys()) == ["l1", "l2"]
+    assert list(ext.locks.values()) == [[l3.id], [l3.id]]
+    assert ext.requests[l3.id] == {"l1", "l2"}
+    assert ext.requests_left[l3.id] == 0
+
+    await l3.release()
+    assert not ext.events
+    assert not ext.requests
+    assert not ext.requests_left
+    assert all(len(l) == 0 for l in ext.locks.values())
+
+
+@gen_cluster(client=True)
+async def test_num_locks(c, s, a, b):
+    ext: MultiLockExtension = s.extensions["multi_locks"]
+    l1 = MultiLock(names=["l1", "l2", "l3"])
+    l2 = MultiLock(names=["l1", "l2", "l3"])
+    l3 = MultiLock(names=["l1", "l2", "l3", "l4"])
+
+    # Even though `l1` and `l2` uses the same lock names they
+    # only requires a subset of the locks
+    assert await l1.acquire(num_locks=1)
+    assert await l2.acquire(num_locks=2)
+    assert list(ext.locks.keys()) == ["l1", "l2", "l3"]
+    assert list(ext.locks.values()) == [[l1.id], [l2.id], [l2.id]]
+    assert list(ext.requests.keys()) == [l1.id, l2.id]
+    assert list(ext.requests_left.values()) == [0, 0]
+    assert not ext.events
+
+    # Since `l3` requires three out of four locks it has to wait
+    l3_acquire = asyncio.ensure_future(l3.acquire(num_locks=3))
+    try:
+        await asyncio.wait_for(asyncio.shield(l3_acquire), 0.1)
+    except asyncio.TimeoutError:
+        assert list(ext.locks.keys()) == ["l1", "l2", "l3", "l4"]
+        assert list(ext.locks.values()) == [
+            [l1.id, l3.id],
+            [l2.id, l3.id],
+            [l2.id, l3.id],
+            [l3.id],
+        ]
+        assert list(ext.requests_left.values()) == [0, 0, 2]
+        assert l3.id in ext.events
+    else:
+        assert False  # We except a TimeoutError since `l3` isn't availabe
+
+    # Releasing `l1` isn't enough since `l3` also requires three locks
+    await l1.release()
+    try:
+        await asyncio.wait_for(asyncio.shield(l3_acquire), 0.1)
+    except asyncio.TimeoutError:
+        assert list(ext.locks.keys()) == ["l1", "l2", "l3", "l4"]
+        assert list(ext.locks.values()) == [
+            [l3.id],
+            [l2.id, l3.id],
+            [l2.id, l3.id],
+            [l3.id],
+        ]
+        assert list(ext.requests.keys()) == [l2.id, l3.id]
+        assert list(ext.requests_left.values()) == [0, 1]
+        assert l3.id in ext.events
+    else:
+        assert False
+
+    # Releasing `l2` is enough to release `l3`
+    await l2.release()
+    await asyncio.sleep(0.1)  # Give `l3` a change to wake up and acquire its locks
+    assert list(ext.locks.keys()) == ["l1", "l2", "l3", "l4"]
+    assert list(ext.locks.values()) == [[l3.id], [l3.id], [l3.id], [l3.id]]
+    assert list(ext.requests.keys()) == [l3.id]
+    assert list(ext.requests_left.values()) == [0]
+    assert l3.id not in ext.events
+
+    await l3.release()
+    assert not ext.events
+    assert not ext.requests
+    assert not ext.requests_left
+    assert all(len(l) == 0 for l in ext.locks.values())
diff --git a/docs/source/api.rst b/docs/source/api.rst
index 0f918565849..472e0b9616a 100644
--- a/docs/source/api.rst
+++ b/docs/source/api.rst
@@ -47,6 +47,7 @@ API
 .. autosummary::
    Event
    Lock
+   MultiLock
    Queue
    Variable
 
@@ -162,6 +163,8 @@ Other
    :members:
 .. autoclass:: Lock
    :members:
+.. autoclass:: MultiLock
+   :members:
 .. autoclass:: Semaphore
    :members:
 .. autoclass:: Queue

From b57d7239b94d733c768b7c6597a0c06e35b079ee Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Tue, 16 Mar 2021 15:23:19 +0000
Subject: [PATCH 1183/1550] If SpecCluster fails to start attempt to gracefully
 close out again. (#4590)

* If SpecCluster fails to start attempt to gracefully close out again.

* Add failed status

* Small refactor of change
---
 distributed/core.py        | 5 +++--
 distributed/deploy/spec.py | 9 +++++++--
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/distributed/core.py b/distributed/core.py
index 6e7090c852c..bc2930a4f78 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -43,8 +43,8 @@
 
 class Status(Enum):
     """
-    This Enum contains the various states a worker, scheduler and nanny can be
-    in. Some of the status can only be observed in one of nanny, scheduler or
+    This Enum contains the various states a cluster, worker, scheduler and nanny can be
+    in. Some of the status can only be observed in one of cluster, nanny, scheduler or
     worker but we put them in the same Enum as they are compared with each
     other.
     """
@@ -52,6 +52,7 @@ class Status(Enum):
     closed = "closed"
     closing = "closing"
     closing_gracefully = "closing-gracefully"
+    failed = "failed"
     init = "init"
     created = "created"
     running = "running"
diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index ac0dd68bf2c..ef1d8a4c6fa 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -311,7 +311,12 @@ async def _start(self):
             getattr(self.scheduler, "external_address", None) or self.scheduler.address,
             connection_args=self.security.get_connection_args("client"),
         )
-        await super()._start()
+        try:
+            await super()._start()
+        except Exception as e:
+            self.status = Status.failed
+            await self._close()
+            raise RuntimeError(f"Cluster failed to start. {str(e)}") from e
 
     def _correct_state(self):
         if self._correct_state_waiting:
@@ -400,7 +405,7 @@ async def _close(self):
             await asyncio.sleep(0.1)
         if self.status == Status.closed:
             return
-        if self.status == Status.running:
+        if self.status == Status.running or self.status == Status.failed:
             self.status = Status.closing
             self.scale(0)
             await self._correct_state()

From 4898d39e603b490deebeca4e692faa282bbdcd88 Mon Sep 17 00:00:00 2001
From: Sultan Orazbayev <contact@econpoint.com>
Date: Wed, 17 Mar 2021 04:35:02 +0600
Subject: [PATCH 1184/1550] Add support for a list of keys when using
 batch_size in client.map (#4592)

---
 distributed/client.py            |  6 +++++-
 distributed/tests/test_client.py | 14 ++++++++++++++
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/distributed/client.py b/distributed/client.py
index 7d0c72e3f90..2669a25473b 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -1693,6 +1693,10 @@ def map(
             batches = list(
                 zip(*[partition_all(batch_size, iterable) for iterable in iterables])
             )
+            if isinstance(key, list):
+                keys = [list(element) for element in partition_all(batch_size, key)]
+            else:
+                keys = [key for _ in range(len(batches))]
             return sum(
                 [
                     self.map(
@@ -1710,7 +1714,7 @@ def map(
                         pure=pure,
                         **kwargs,
                     )
-                    for batch in batches
+                    for key, batch in zip(keys, batches)
                 ],
                 [],
             )
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 9caa3ccb08f..7a9ca27352a 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -229,6 +229,20 @@ async def test_map_batch_size(c, s, a, b):
     assert result == list(range(100, 120, 2))
 
 
+@gen_cluster(client=True)
+async def test_custom_key_with_batches(c, s, a, b):
+    """ Test of <https://github.com/dask/distributed/issues/4588>"""
+
+    futs = c.map(
+        lambda x: x ** 2,
+        range(10),
+        batch_size=5,
+        key=[str(x) for x in range(10)],
+    )
+    assert len(futs) == 10
+    await wait(futs)
+
+
 @gen_cluster(client=True)
 async def test_compute_retries(c, s, a, b):
     args = [ZeroDivisionError("one"), ZeroDivisionError("two"), 3]

From aacdd78fd2f48481ec1c24b281f399ebb0f215f8 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Wed, 17 Mar 2021 07:20:11 -0700
Subject: [PATCH 1185/1550] Track frame sizes along with frames (#4593)

* Add missing backtick

* Track frame sizes along with frames

Should cutdown on the number of `nbytes` calls to 1 per frame as opposed
to a couple as before.
---
 distributed/comm/tcp.py | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py
index 056e0216fbc..b79a5105a0d 100644
--- a/distributed/comm/tcp.py
+++ b/distributed/comm/tcp.py
@@ -242,28 +242,30 @@ async def write(self, msg, serializers=None, on_error="message"):
                 **self.handshake_options,
             },
         )
-        frames_nbytes = sum(map(nbytes, frames))
+        frames_nbytes = [nbytes(f) for f in frames]
+        frames_nbytes_total = sum(frames_nbytes)
 
         header = pack_frames_prelude(frames)
-        header = struct.pack("Q", nbytes(header) + frames_nbytes) + header
+        header = struct.pack("Q", nbytes(header) + frames_nbytes_total) + header
 
         frames = [header, *frames]
-        frames_nbytes += nbytes(header)
+        frames_nbytes = [nbytes(header), *frames_nbytes]
+        frames_nbytes_total += frames_nbytes[0]
 
-        if frames_nbytes < 2 ** 17:  # 128kiB
+        if frames_nbytes_total < 2 ** 17:  # 128kiB
             # small enough, send in one go
             frames = [b"".join(frames)]
+            frames_nbytes = [frames_nbytes_total]
 
         try:
             # trick to enque all frames for writing beforehand
-            for each_frame in frames:
-                each_frame_nbytes = nbytes(each_frame)
+            for each_frame_nbytes, each_frame in zip(frames_nbytes, frames):
                 if each_frame_nbytes:
                     if stream._write_buffer is None:
                         raise StreamClosedError()
 
                     if isinstance(each_frame, memoryview):
-                        # Make sure that len(data) == data.nbytes`
+                        # Make sure that `len(data) == data.nbytes`
                         # See <https://github.com/tornadoweb/tornado/pull/2996>
                         each_frame = memoryview(each_frame).cast("B")
 
@@ -285,7 +287,7 @@ async def write(self, msg, serializers=None, on_error="message"):
             self.abort()
             raise
 
-        return frames_nbytes
+        return frames_nbytes_total
 
     @gen.coroutine
     def close(self):

From de60c90d968c3e5adeada80c90e864d7651acc6f Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Wed, 17 Mar 2021 12:43:45 -0500
Subject: [PATCH 1186/1550] Switch documentation builds for PRs to readthedocs
 (#4599)

---
 .github/workflows/ci-docs.yaml | 28 ----------------------------
 1 file changed, 28 deletions(-)
 delete mode 100644 .github/workflows/ci-docs.yaml

diff --git a/.github/workflows/ci-docs.yaml b/.github/workflows/ci-docs.yaml
deleted file mode 100644
index 625e496cad3..00000000000
--- a/.github/workflows/ci-docs.yaml
+++ /dev/null
@@ -1,28 +0,0 @@
-name: Documentation
-
-on: [push, pull_request]
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-
-      - name: Set up Python 3.7
-        uses: actions/setup-python@v1
-        with:
-          python-version: 3.7
-
-      - name: Install Distributed
-        run: |
-          python -m pip install --upgrade pip
-          python -m pip install -e .
-
-      - name: Install doc dependencies
-        run: python -m pip install -r docs/requirements.txt
-
-      - name: Build docs
-        run: |
-          cd docs
-          make html

From a7df3ca8866b406ff0c3b67e7c1fd7a32b5c4040 Mon Sep 17 00:00:00 2001
From: "Mads R. B. Kristensen" <madsbk@gmail.com>
Date: Thu, 18 Mar 2021 06:55:54 +0100
Subject: [PATCH 1187/1550] pickle_loads(): Handle empty memoryview (#4595)

Fixes https://github.com/dask/distributed/issues/4594
---
 distributed/protocol/serialize.py         |  8 ++++++--
 distributed/protocol/tests/test_pickle.py | 11 +++++++++++
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index 228f02d78ca..49a2fdab09d 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -74,9 +74,13 @@ def pickle_loads(header, frames):
         mv = memoryview(buffers[i])
         if writeable[i] == mv.readonly:
             if mv.readonly:
-                buffers[i] = memoryview(bytearray(mv)).cast(mv.format, mv.shape)
+                buf = memoryview(bytearray(mv))
             else:
-                buffers[i] = memoryview(bytes(mv)).cast(mv.format, mv.shape)
+                buf = memoryview(bytes(mv))
+            if buf.nbytes > 0:
+                buffers[i] = buf.cast(mv.format, mv.shape)
+            else:
+                buffers[i] = buf.cast(mv.format)
     return pickle.loads(x, buffers=buffers)
 
 
diff --git a/distributed/protocol/tests/test_pickle.py b/distributed/protocol/tests/test_pickle.py
index d7a2ad6d1ad..6430ea352b8 100644
--- a/distributed/protocol/tests/test_pickle.py
+++ b/distributed/protocol/tests/test_pickle.py
@@ -8,6 +8,7 @@
 
 from distributed.protocol import deserialize, serialize
 from distributed.protocol.pickle import HIGHEST_PROTOCOL, dumps, loads
+from distributed.protocol.serialize import pickle_dumps
 
 if sys.version_info < (3, 8):
     try:
@@ -71,6 +72,16 @@ def __reduce_ex__(self, protocol):
         assert isinstance(f[0], bytes)
 
 
+def test_pickle_empty():
+    np = pytest.importorskip("numpy")
+    x = np.arange(2)[0:0]  # Empty view
+    header, frames = pickle_dumps(x)
+    header["writeable"] = [False] * len(frames)
+    y = deserialize(header, frames)
+    assert memoryview(y).nbytes == 0
+    assert memoryview(y).readonly
+
+
 def test_pickle_numpy():
     np = pytest.importorskip("numpy")
     x = np.ones(5)

From 206c6f8411f770f6536ea5c171fa7299a1e120c5 Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Thu, 18 Mar 2021 13:50:06 +0100
Subject: [PATCH 1188/1550] Update develop.rst (#4603)

Co-authored-by: fjetter <florian.jetter@blueyonder.com>
---
 docs/source/develop.rst | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/docs/source/develop.rst b/docs/source/develop.rst
index da593cf4b2b..4d30f038b17 100644
--- a/docs/source/develop.rst
+++ b/docs/source/develop.rst
@@ -19,19 +19,15 @@ Clone this repository with git::
 
 Install all dependencies:
 
-On Linux / MacOSX::
-
-    source continuous_integration/travis/install.sh
-
-On Windows:
+All OS::
 
 1. Install anaconda or miniconda
 2. ::
 
-    conda create -n dask-distributed -c conda-forge -c defaults python=3.8 tornado=6
+    conda env create --file continuous_integration/environment-3.8.yaml
     conda activate dask-distributed
-    conda env update --file continuous_integration/environment-windows.yml
-    python -m pip install .
+    python -m pip install -e .
+
 
 To keep a fork in sync with the upstream source::
 

From 98148cbfee818889f683c4be0e591d33a59ffdf8 Mon Sep 17 00:00:00 2001
From: Gil Forsyth <gforsyth@users.noreply.github.com>
Date: Thu, 18 Mar 2021 10:56:50 -0400
Subject: [PATCH 1189/1550] Add explicit `fetch` state to worker TaskState
 (#4470)

---
 .../diagnostics/tests/test_worker_plugin.py   |   5 +
 distributed/pytest_resourceleaks.py           |   2 +-
 distributed/tests/test_steal.py               |   3 +
 distributed/worker.py                         | 282 ++++++++++++------
 docs/source/images/worker-dep-state.dot       |   4 +-
 docs/source/images/worker-dep-state.svg       |  65 ++--
 docs/source/images/worker-task-state.dot      |   1 +
 docs/source/images/worker-task-state.svg      | 111 ++++---
 docs/source/worker.rst                        |  32 +-
 9 files changed, 344 insertions(+), 161 deletions(-)

diff --git a/distributed/diagnostics/tests/test_worker_plugin.py b/distributed/diagnostics/tests/test_worker_plugin.py
index 79d180c360e..858f8feedc2 100644
--- a/distributed/diagnostics/tests/test_worker_plugin.py
+++ b/distributed/diagnostics/tests/test_worker_plugin.py
@@ -76,6 +76,7 @@ async def test_create_on_construction(c, s, a, b):
 @gen_cluster(nthreads=[("127.0.0.1", 1)], client=True)
 async def test_normal_task_transitions_called(c, s, w):
     expected_notifications = [
+        {"key": "task", "start": "new", "finish": "waiting"},
         {"key": "task", "start": "waiting", "finish": "ready"},
         {"key": "task", "start": "ready", "finish": "executing"},
         {"key": "task", "start": "executing", "finish": "memory"},
@@ -95,6 +96,7 @@ def failing(x):
         raise Exception()
 
     expected_notifications = [
+        {"key": "task", "start": "new", "finish": "waiting"},
         {"key": "task", "start": "waiting", "finish": "ready"},
         {"key": "task", "start": "ready", "finish": "executing"},
         {"key": "task", "start": "executing", "finish": "error"},
@@ -113,6 +115,7 @@ def failing(x):
 )
 async def test_superseding_task_transitions_called(c, s, w):
     expected_notifications = [
+        {"key": "task", "start": "new", "finish": "waiting"},
         {"key": "task", "start": "waiting", "finish": "constrained"},
         {"key": "task", "start": "constrained", "finish": "executing"},
         {"key": "task", "start": "executing", "finish": "memory"},
@@ -131,9 +134,11 @@ async def test_release_dep_called(c, s, w):
     dsk = {"dep": 1, "task": (inc, "dep")}
 
     expected_notifications = [
+        {"key": "dep", "start": "new", "finish": "waiting"},
         {"key": "dep", "start": "waiting", "finish": "ready"},
         {"key": "dep", "start": "ready", "finish": "executing"},
         {"key": "dep", "start": "executing", "finish": "memory"},
+        {"key": "task", "start": "new", "finish": "waiting"},
         {"key": "task", "start": "waiting", "finish": "ready"},
         {"key": "task", "start": "ready", "finish": "executing"},
         {"key": "task", "start": "executing", "finish": "memory"},
diff --git a/distributed/pytest_resourceleaks.py b/distributed/pytest_resourceleaks.py
index 348472892d6..f86117c7862 100644
--- a/distributed/pytest_resourceleaks.py
+++ b/distributed/pytest_resourceleaks.py
@@ -30,7 +30,7 @@ def pytest_addoption(parser):
     group.addoption(
         "--leaks-timeout",
         action="store",
-        type="float",
+        type=float,
         dest="leaks_timeout",
         default=0.5,
         help="""\
diff --git a/distributed/tests/test_steal.py b/distributed/tests/test_steal.py
index ac37c0d5acf..de7ffda8f83 100644
--- a/distributed/tests/test_steal.py
+++ b/distributed/tests/test_steal.py
@@ -660,6 +660,9 @@ async def test_steal_twice(c, s, a, b):
         )
     assert max(map(len, has_what.values())) < 30
 
+    assert a.in_flight_tasks == 0
+    assert b.in_flight_tasks == 0
+
     await c._close()
     await asyncio.gather(*[w.close() for w in workers])
 
diff --git a/distributed/worker.py b/distributed/worker.py
index 7e282c2e46a..276b41fd853 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -114,13 +114,13 @@ class TaskState:
         The priority this task given by the scheduler.  Determines run order.
     * **state**: ``str``
         The current state of the task. One of ["waiting", "ready", "executing",
-        "memory", "flight", "long-running", "rescheduled", "error"]
+        "fetch", "memory", "flight", "long-running", "rescheduled", "error"]
     * **who_has**: ``set(worker)``
         Workers that we believe have this data
     * **coming_from**: ``str``
         The worker that current task data is coming from if task is in flight
     * **waiting_for_data**: ``set(keys of dependencies)``
-        A dynamic verion of dependencies.  All dependencies that we still don't
+        A dynamic version of dependencies.  All dependencies that we still don't
         have for a particular key.
     * **resource_restrictions**: ``{str: number}``
         Abstract resources required to run a task
@@ -163,7 +163,7 @@ def __init__(self, key, runspec=None):
         self.dependents = set()
         self.duration = None
         self.priority = None
-        self.state = None
+        self.state = "new"
         self.who_has = set()
         self.coming_from = None
         self.waiting_for_data = set()
@@ -261,7 +261,7 @@ class Worker(ServerNode):
         Dictionary mapping keys to actual values stored on disk. Only
         available if condition for **data** being a zict.Buffer is met.
     * **data_needed**: deque(keys)
-        The keys whose data we still lack, arranged in a deque
+        The keys which still require data in order to execute, arranged in a deque
     * **ready**: [keys]
         Keys that are ready to run.  Stored in a LIFO stack
     * **constrained**: [keys]
@@ -448,27 +448,27 @@ def __init__(
         self.validate = validate
 
         self._transitions = {
+            # Basic state transitions
+            ("new", "waiting"): self.transition_new_waiting,
+            ("new", "fetch"): self.transition_new_fetch,
             ("waiting", "ready"): self.transition_waiting_ready,
-            ("waiting", "memory"): self.transition_waiting_done,
-            ("waiting", "error"): self.transition_waiting_done,
-            ("waiting", "flight"): self.transition_waiting_flight,
+            ("fetch", "flight"): self.transition_fetch_flight,
             ("ready", "executing"): self.transition_ready_executing,
-            ("ready", "memory"): self.transition_ready_memory,
-            ("ready", "error"): self.transition_ready_error,
-            ("ready", "waiting"): self.transition_ready_waiting,
-            ("constrained", "waiting"): self.transition_ready_waiting,
-            ("constrained", "error"): self.transition_ready_error,
-            ("constrained", "executing"): self.transition_constrained_executing,
             ("executing", "memory"): self.transition_executing_done,
+            ("flight", "memory"): self.transition_flight_memory,
+            ("flight", "fetch"): self.transition_flight_fetch,
+            # Scheduler intercession (re-assignment)
+            ("fetch", "waiting"): self.transition_fetch_waiting,
+            ("flight", "waiting"): self.transition_flight_waiting,
+            # Errors, long-running, constrained
+            ("waiting", "error"): self.transition_waiting_done,
+            ("constrained", "executing"): self.transition_constrained_executing,
             ("executing", "error"): self.transition_executing_done,
             ("executing", "rescheduled"): self.transition_executing_done,
             ("executing", "long-running"): self.transition_executing_long_running,
             ("long-running", "error"): self.transition_executing_done,
             ("long-running", "memory"): self.transition_executing_done,
             ("long-running", "rescheduled"): self.transition_executing_done,
-            ("flight", "memory"): self.transition_flight_memory,
-            ("flight", "ready"): self.transition_flight_memory,
-            ("flight", "waiting"): self.transition_flight_waiting,
         }
 
         self.incoming_transfer_log = deque(maxlen=100000)
@@ -1417,7 +1417,7 @@ def delete_data(self, comm=None, keys=None, report=True):
         if keys:
             for key in list(keys):
                 self.log.append((key, "delete"))
-                self.release_key(key)
+                self.release_key(key, cause="delete data")
 
             logger.debug("Worker %s -- Deleted %d keys", self.name, len(keys))
         return "OK"
@@ -1459,7 +1459,6 @@ def add_task(
             runspec = SerializedTask(function, args, kwargs, task)
             if key in self.tasks:
                 ts = self.tasks[key]
-                ts.runspec = runspec
                 if ts.state == "memory":
                     assert key in self.data or key in self.actors
                     logger.debug(
@@ -1473,13 +1472,20 @@ def add_task(
                     ts.exception = None
                     ts.traceback = None
                 else:
-                    ts.state = "waiting"
+                    # This is a scheduler re-assignment
+                    # Either `fetch` -> `waiting` or `flight` -> `waiting`
+                    self.log.append((ts.key, "re-adding key, new TaskState"))
+                    self.transition(ts, "waiting", runspec=runspec)
             else:
                 self.log.append((key, "new"))
                 self.tasks[key] = ts = TaskState(
                     key=key, runspec=SerializedTask(function, args, kwargs, task)
                 )
-                ts.state = "waiting"
+                self.transition(ts, "waiting")
+
+            # TODO: move transition of `ts` to end of `add_task`
+            # This will require a chained recommendation transition system like
+            # the scheduler
 
             if priority is not None:
                 priority = tuple(priority) + (self.generation,)
@@ -1499,26 +1505,42 @@ def add_task(
             for dependency, workers in who_has.items():
                 assert workers
                 if dependency not in self.tasks:
+                    # initial state is "new"
+                    # this dependency does not already exist on worker
                     self.tasks[dependency] = dep_ts = TaskState(key=dependency)
-                    dep_ts.state = (
-                        "waiting" if dependency not in self.data else "memory"
-                    )
 
-                dep_ts = self.tasks[dependency]
-                self.log.append((dependency, "new-dep", dep_ts.state))
+                    # link up to child / parents
+                    ts.dependencies.add(dep_ts)
+                    dep_ts.dependents.add(ts)
 
-                if dep_ts.state != "memory":
-                    ts.waiting_for_data.add(dep_ts.key)
-                    self.waiting_for_data_count += 1
+                    # check to ensure task wasn't already executed and partially released
+                    # # TODO: make this less bad
+                    state = "fetch" if dependency not in self.data else "memory"
+
+                    # transition from new -> fetch handles adding dependency
+                    # to waiting_for_data
+                    self.transition(dep_ts, state)
+
+                    self.log.append(
+                        (dependency, "new-dep", dep_ts.state, f"requested by {ts.key}")
+                    )
+
+                else:
+                    # task was already present on worker
+                    dep_ts = self.tasks[dependency]
 
-                dep_ts.who_has.update(workers)
+                    # link up to child / parents
+                    ts.dependencies.add(dep_ts)
+                    dep_ts.dependents.add(ts)
 
-                ts.dependencies.add(dep_ts)
-                dep_ts.dependents.add(ts)
+                if dep_ts.state in ("fetch", "flight"):
+                    # if we _need_ to grab data or are in the process
+                    ts.waiting_for_data.add(dep_ts.key)
+                    # Ensure we know which workers to grab data from
+                    dep_ts.who_has.update(workers)
 
-                for worker in workers:
-                    self.has_what[worker].add(dep_ts.key)
-                    if dep_ts.state != "memory":
+                    for worker in workers:
+                        self.has_what[worker].add(dep_ts.key)
                         self.pending_data_per_worker[worker].append(dep_ts.key)
 
             if nbytes is not None:
@@ -1530,6 +1552,9 @@ def add_task(
             else:
                 self.transition(ts, "ready")
             if self.validate:
+                for worker, keys in self.has_what.items():
+                    for k in keys:
+                        assert worker in self.tasks[k].who_has
                 if who_has:
                     assert all(self.tasks[dep] in ts.dependencies for dep in who_has)
                     assert all(self.tasks[dep.key] for dep in ts.dependencies)
@@ -1558,10 +1583,104 @@ def transition(self, ts, finish, **kwargs):
             self.validate_task(ts)
         self._notify_plugins("transition", ts.key, start, state or finish, **kwargs)
 
-    def transition_waiting_flight(self, ts, worker=None):
+    def transition_new_waiting(self, ts):
+        try:
+            if self.validate:
+                assert ts.state == "new"
+                assert ts.runspec is not None
+                assert not ts.who_has
+        except Exception as e:
+            logger.exception(e)
+            if LOG_PDB:
+                import pdb
+
+                pdb.set_trace()
+            raise
+
+    def transition_new_fetch(self, ts):
+        try:
+            if self.validate:
+                assert ts.state == "new"
+                assert ts.runspec is None
+
+            for dependent in ts.dependents:
+                dependent.waiting_for_data.add(ts.key)
+
+        except Exception as e:
+            logger.exception(e)
+            if LOG_PDB:
+                import pdb
+
+                pdb.set_trace()
+            raise
+
+    def transition_fetch_waiting(self, ts, runspec):
+        """This is a rescheduling transition that occurs after a worker failure.
+        A task was available from another worker but that worker died and the
+        scheduler reassigned the task for computation here.
+        """
+        try:
+            if self.validate:
+                assert ts.state == "fetch"
+                assert ts.runspec is None
+                assert runspec is not None
+
+            ts.runspec = runspec
+
+            # remove any stale entries in `has_what`
+            for worker in self.has_what.keys():
+                self.has_what[worker].discard(ts.key)
+
+            # clear `who_has` of stale info
+            ts.who_has.clear()
+
+            # remove entry from dependents to avoid a spurious `gather_dep` call``
+            for dependent in ts.dependents:
+                dependent.waiting_for_data.discard(ts.key)
+        except Exception as e:
+            logger.exception(e)
+            if LOG_PDB:
+                import pdb
+
+                pdb.set_trace()
+            raise
+
+    def transition_flight_waiting(self, ts, runspec):
+        """This is a rescheduling transition that occurs after
+        a worker failure.  A task was in flight from another worker to this
+        worker when that worker died and the scheduler reassigned the task for
+        computation here.
+        """
         try:
             if self.validate:
-                assert ts.state != "flight"
+                assert ts.state == "flight"
+                assert ts.runspec is None
+                assert runspec is not None
+
+            ts.runspec = runspec
+
+            # remove any stale entries in `has_what`
+            for worker in self.has_what.keys():
+                self.has_what[worker].discard(ts.key)
+
+            # clear `who_has` of stale info
+            ts.who_has.clear()
+
+            # remove entry from dependents to avoid a spurious `gather_dep` call``
+            for dependent in ts.dependents:
+                dependent.waiting_for_data.discard(ts.key)
+        except Exception as e:
+            logger.exception(e)
+            if LOG_PDB:
+                import pdb
+
+                pdb.set_trace()
+            raise
+
+    def transition_fetch_flight(self, ts, worker=None):
+        try:
+            if self.validate:
+                assert ts.state == "fetch"
                 assert ts.dependents
 
             ts.coming_from = worker
@@ -1574,7 +1693,7 @@ def transition_waiting_flight(self, ts, worker=None):
                 pdb.set_trace()
             raise
 
-    def transition_flight_waiting(self, ts, worker=None, remove=True, runspec=None):
+    def transition_flight_fetch(self, ts, worker=None, runspec=None):
         try:
             if self.validate:
                 assert ts.state == "flight"
@@ -1582,26 +1701,16 @@ def transition_flight_waiting(self, ts, worker=None, remove=True, runspec=None):
             self.in_flight_tasks -= 1
             ts.coming_from = None
             ts.runspec = runspec or ts.runspec
-            if remove:
-                try:
-                    ts.who_has.remove(worker)
-                    self.has_what[worker].remove(ts.key)
-                except KeyError:
-                    pass
 
             if not ts.who_has:
                 if ts.key not in self._missing_dep_flight:
                     self._missing_dep_flight.add(ts.key)
                     self.loop.add_callback(self.handle_missing_dep, ts)
             for dependent in ts.dependents:
+                dependent.waiting_for_data.add(ts.key)
                 if dependent.state == "waiting":
-                    if remove:  # try a new worker immediately
-                        self.data_needed.appendleft(dependent.key)
-                    else:  # worker was probably busy, wait a while
-                        self.data_needed.append(dependent.key)
+                    self.data_needed.append(dependent.key)
 
-            if not ts.dependents:
-                self.release_key(ts.key)
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
@@ -1617,18 +1726,15 @@ def transition_flight_memory(self, ts, value=None):
 
             self.in_flight_tasks -= 1
             ts.coming_from = None
-            if ts.dependents:
-                self.put_key_in_memory(ts, value)
-                for dependent in ts.dependents:
-                    try:
-                        dependent.waiting_for_data.remove(ts.key)
-                        self.waiting_for_data_count -= 1
-                    except KeyError:
-                        pass
+            self.put_key_in_memory(ts, value)
+            for dependent in ts.dependents:
+                try:
+                    dependent.waiting_for_data.remove(ts.key)
+                    self.waiting_for_data_count -= 1
+                except KeyError:
+                    pass
 
-                self.batched_stream.send({"op": "add-keys", "keys": [ts.key]})
-            else:
-                self.release_key(ts.key)
+            self.batched_stream.send({"op": "add-keys", "keys": [ts.key]})
 
         except Exception as e:
             logger.exception(e)
@@ -1650,7 +1756,7 @@ def transition_waiting_ready(self, ts):
                 assert all(dep.state == "memory" for dep in ts.dependencies)
                 assert ts.key not in self.ready
 
-            ts.waiting_for_data.clear()
+            self.has_what[self.address].discard(ts.key)
 
             if ts.resource_restrictions is not None:
                 self.constrained.append(ts.key)
@@ -1717,12 +1823,6 @@ def transition_ready_memory(self, ts, value=None):
             self.put_key_in_memory(ts, value=value)
         self.send_task_state_to_scheduler(ts)
 
-    def transition_ready_waiting(self, ts):
-        """
-        This transition is common for work stealing
-        """
-        pass
-
     def transition_constrained_executing(self, ts):
         self.transition_ready_executing(ts)
         for resource, quantity in ts.resource_restrictions.items():
@@ -1867,7 +1967,7 @@ def ensure_communicating(self):
                 if self.validate:
                     assert all(dep.key in self.tasks for dep in deps)
 
-                deps = [dep for dep in deps if dep.state == "waiting"]
+                deps = {dep for dep in deps if dep.state == "fetch"}
 
                 missing_deps = {dep for dep in deps if not dep.who_has}
                 if missing_deps:
@@ -1892,7 +1992,7 @@ def ensure_communicating(self):
                     or self.comm_nbytes < self.total_comm_nbytes
                 ):
                     dep = deps.pop()
-                    if dep.state != "waiting":
+                    if dep.state != "fetch":
                         continue
                     if not dep.who_has:
                         continue
@@ -2019,7 +2119,7 @@ def select_keys_for_gather(self, worker, dep):
         while L:
             d = L.popleft()
             ts = self.tasks.get(d)
-            if ts is None or ts.state != "waiting":
+            if ts is None or ts.state != "fetch":
                 continue
             if total_bytes + ts.get_nbytes() > self.target_message_size:
                 break
@@ -2069,7 +2169,7 @@ async def gather_dep(self, worker, dep, deps, total_nbytes, cause=None):
                     self.log.append(("busy-gather", worker, deps))
                     for ts in deps_ts:
                         if ts.state == "flight":
-                            self.transition(ts, "waiting")
+                            self.transition(ts, "fetch")
                     return
 
                 if cause:
@@ -2149,10 +2249,10 @@ async def gather_dep(self, worker, dep, deps, total_nbytes, cause=None):
                     if not busy and d in data:
                         self.transition(ts, "memory", value=data[d])
                     elif ts is None or ts.state == "executing":
-                        self.release_key(d)
+                        self.release_key(d, cause="already executing at gather")
                         continue
                     elif ts.state not in ("ready", "memory"):
-                        self.transition(ts, "waiting", worker=worker, remove=not busy)
+                        self.transition(ts, "fetch", worker=worker)
 
                     if not busy and d not in data and ts.dependents:
                         self.log.append(("missing-dep", d))
@@ -2185,7 +2285,7 @@ def bad_dep(self, dep):
             ts.exception = msg["exception"]
             ts.traceback = msg["traceback"]
             self.transition(ts, "error")
-        self.release_key(dep.key)
+        self.release_key(dep.key, cause="bad dep")
 
     async def handle_missing_dep(self, *deps, **kwargs):
         self.log.append(("handle-missing", deps))
@@ -2281,17 +2381,12 @@ def steal_request(self, key):
         self.batched_stream.send(response)
 
         if state in ("ready", "waiting", "constrained"):
-            # Resetting the runspec should be reset by the transition. However,
-            # the waiting->waiting transition results in a no-op which would not
-            # reset.
-            ts.runspec = None
-            self.transition(ts, "waiting")
-            if not ts.dependents:
-                self.release_key(ts.key)
-                if self.validate:
-                    assert ts.key not in self.tasks
+            # If task is marked as "constrained" we haven't yet assigned it an
+            # `available_resources` to run on, that happens in
+            # `transition_constrained_executing`
+            self.release_key(ts.key, cause="stolen")
             if self.validate:
-                assert ts.runspec is None
+                assert ts.key not in self.tasks
 
     def release_key(self, key, cause=None, reason=None, report=True):
         try:
@@ -2314,11 +2409,14 @@ def release_key(self, key, cause=None, reason=None, report=True):
             # for any dependencies of key we are releasing remove task as dependent
             for dependency in ts.dependencies:
                 dependency.dependents.discard(ts)
+                # don't boot keys that are in flight
+                # we don't know if they're already queued up for transit
+                # in a gather_dep callback
                 if not dependency.dependents and dependency.state in (
                     "waiting",
-                    "flight",
+                    "fetch",
                 ):
-                    self.release_key(dependency.key)
+                    self.release_key(dependency.key, cause=f"Dependent {ts} released")
 
             for worker in ts.who_has:
                 self.has_what[worker].discard(ts.key)
@@ -2335,7 +2433,9 @@ def release_key(self, key, cause=None, reason=None, report=True):
                     for resource, quantity in ts.resource_restrictions.items():
                         self.available_resources[resource] += quantity
 
-            if report and ts.state in PROCESSING:  # not finished
+            # Inform the scheduler of keys which will have gone missing
+            # We are releasing them before they have completed
+            if report and ts.state in PROCESSING:
                 self.batched_stream.send({"op": "release", "key": key, "cause": cause})
 
             self._notify_plugins("release_key", key, ts.state, cause, reason, report)
@@ -2970,6 +3070,10 @@ def validate_task_flight(self, ts):
         assert not any(dep.key in self.ready for dep in ts.dependents)
         assert ts.key in self.in_flight_workers[ts.coming_from]
 
+    def validate_task_fetch(self, ts):
+        assert ts.runspec is None
+        assert ts.key not in self.data
+
     def validate_task(self, ts):
         try:
             if ts.state == "memory":
@@ -2982,6 +3086,8 @@ def validate_task(self, ts):
                 self.validate_task_executing(ts)
             elif ts.state == "flight":
                 self.validate_task_flight(ts)
+            elif ts.state == "fetch":
+                self.validate_task_fetch(ts)
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
@@ -3012,7 +3118,7 @@ def validate_state(self):
                     ts_wait = self.tasks[key]
                     assert (
                         ts_wait.state == "flight"
-                        or ts_wait.state == "waiting"
+                        or ts_wait.state == "fetch"
                         or ts_wait.key in self._missing_dep_flight
                         or ts_wait.who_has.issubset(self.in_flight_workers)
                     )
diff --git a/docs/source/images/worker-dep-state.dot b/docs/source/images/worker-dep-state.dot
index 8da78a1c789..18a5e40cfac 100644
--- a/docs/source/images/worker-dep-state.dot
+++ b/docs/source/images/worker-dep-state.dot
@@ -3,6 +3,8 @@ digraph{
         bgcolor="#FFFFFFF00",
         rankdir=LR,
         ];
-    waiting -> flight;
+    new -> fetch;
+    fetch -> flight;
+    flight -> fetch;
     flight -> memory;
 }
diff --git a/docs/source/images/worker-dep-state.svg b/docs/source/images/worker-dep-state.svg
index 412d4dc7d3a..9b7b04e8d99 100644
--- a/docs/source/images/worker-dep-state.svg
+++ b/docs/source/images/worker-dep-state.svg
@@ -1,38 +1,61 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
 <!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
  "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
-<!-- Generated by graphviz version 2.36.0 (20140111.2315)
+<!-- Generated by graphviz version 2.42.3 (20191010.1750)
  -->
 <!-- Title: %3 Pages: 1 -->
-<svg width="296pt" height="44pt"
- viewBox="0.00 0.00 296.00 44.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+<svg width="369pt" height="44pt"
+ viewBox="0.00 0.00 368.88 44.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
 <g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 40)">
 <title>%3</title>
-<polygon fill="#ffffff" fill-opacity="0.941176" stroke="none" points="-4,4 -4,-40 292,-40 292,4 -4,4"/>
-<!-- waiting -->
-<g id="node1" class="node"><title>waiting</title>
-<ellipse fill="none" stroke="black" cx="37" cy="-18" rx="37.0935" ry="18"/>
-<text text-anchor="middle" x="37" y="-14.3" font-family="Times,serif" font-size="14.00">waiting</text>
+<polygon fill="#ffffff" fill-opacity="0.941176" stroke="transparent" points="-4,4 -4,-40 364.88,-40 364.88,4 -4,4"/>
+<!-- new -->
+<g id="node1" class="node">
+<title>new</title>
+<ellipse fill="none" stroke="black" cx="27" cy="-18" rx="27" ry="18"/>
+<text text-anchor="middle" x="27" y="-14.3" font-family="Times,serif" font-size="14.00">new</text>
+</g>
+<!-- fetch -->
+<g id="node2" class="node">
+<title>fetch</title>
+<ellipse fill="none" stroke="black" cx="118.6" cy="-18" rx="28.7" ry="18"/>
+<text text-anchor="middle" x="118.6" y="-14.3" font-family="Times,serif" font-size="14.00">fetch</text>
+</g>
+<!-- new&#45;&gt;fetch -->
+<g id="edge1" class="edge">
+<title>new&#45;&gt;fetch</title>
+<path fill="none" stroke="black" d="M54.4,-18C62.32,-18 71.18,-18 79.7,-18"/>
+<polygon fill="black" stroke="black" points="79.83,-21.5 89.83,-18 79.83,-14.5 79.83,-21.5"/>
 </g>
 <!-- flight -->
-<g id="node2" class="node"><title>flight</title>
-<ellipse fill="none" stroke="black" cx="140" cy="-18" rx="29.795" ry="18"/>
-<text text-anchor="middle" x="140" y="-14.3" font-family="Times,serif" font-size="14.00">flight</text>
+<g id="node3" class="node">
+<title>flight</title>
+<ellipse fill="none" stroke="black" cx="213.09" cy="-18" rx="29.8" ry="18"/>
+<text text-anchor="middle" x="213.09" y="-14.3" font-family="Times,serif" font-size="14.00">flight</text>
+</g>
+<!-- fetch&#45;&gt;flight -->
+<g id="edge2" class="edge">
+<title>fetch&#45;&gt;flight</title>
+<path fill="none" stroke="black" d="M145.86,-11.84C154.8,-11.28 164.98,-11.12 174.62,-11.36"/>
+<polygon fill="black" stroke="black" points="174.67,-14.86 184.8,-11.78 174.96,-7.87 174.67,-14.86"/>
 </g>
-<!-- waiting&#45;&gt;flight -->
-<g id="edge1" class="edge"><title>waiting&#45;&gt;flight</title>
-<path fill="none" stroke="black" d="M74.178,-18C82.5482,-18 91.4907,-18 99.9767,-18"/>
-<polygon fill="black" stroke="black" points="100.032,-21.5001 110.032,-18 100.032,-14.5001 100.032,-21.5001"/>
+<!-- flight&#45;&gt;fetch -->
+<g id="edge3" class="edge">
+<title>flight&#45;&gt;fetch</title>
+<path fill="none" stroke="black" d="M184.8,-24.22C175.77,-24.75 165.57,-24.88 155.96,-24.61"/>
+<polygon fill="black" stroke="black" points="156,-21.11 145.86,-24.16 155.69,-28.1 156,-21.11"/>
 </g>
 <!-- memory -->
-<g id="node3" class="node"><title>memory</title>
-<ellipse fill="none" stroke="black" cx="247" cy="-18" rx="40.8928" ry="18"/>
-<text text-anchor="middle" x="247" y="-14.3" font-family="Times,serif" font-size="14.00">memory</text>
+<g id="node4" class="node">
+<title>memory</title>
+<ellipse fill="none" stroke="black" cx="319.94" cy="-18" rx="40.89" ry="18"/>
+<text text-anchor="middle" x="319.94" y="-14.3" font-family="Times,serif" font-size="14.00">memory</text>
 </g>
 <!-- flight&#45;&gt;memory -->
-<g id="edge2" class="edge"><title>flight&#45;&gt;memory</title>
-<path fill="none" stroke="black" d="M169.97,-18C177.91,-18 186.756,-18 195.527,-18"/>
-<polygon fill="black" stroke="black" points="195.729,-21.5001 205.729,-18 195.729,-14.5001 195.729,-21.5001"/>
+<g id="edge4" class="edge">
+<title>flight&#45;&gt;memory</title>
+<path fill="none" stroke="black" d="M243.02,-18C250.95,-18 259.78,-18 268.54,-18"/>
+<polygon fill="black" stroke="black" points="268.73,-21.5 278.73,-18 268.73,-14.5 268.73,-21.5"/>
 </g>
 </g>
 </svg>
diff --git a/docs/source/images/worker-task-state.dot b/docs/source/images/worker-task-state.dot
index 5bd9ac93a8f..4a6fc8cbacf 100644
--- a/docs/source/images/worker-task-state.dot
+++ b/docs/source/images/worker-task-state.dot
@@ -3,6 +3,7 @@ digraph{
         bgcolor="#FFFFFFF00",
         rankdir=LR,
         ];
+    new -> waiting;
     waiting -> ready;
     ready -> executing;
     executing -> "long-running";
diff --git a/docs/source/images/worker-task-state.svg b/docs/source/images/worker-task-state.svg
index 2b307d80d64..edc4b834044 100644
--- a/docs/source/images/worker-task-state.svg
+++ b/docs/source/images/worker-task-state.svg
@@ -1,78 +1,103 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
 <!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
  "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
-<!-- Generated by graphviz version 2.38.0 (20140413.2041)
+<!-- Generated by graphviz version 2.42.3 (20191010.1750)
  -->
 <!-- Title: %3 Pages: 1 -->
-<svg width="572pt" height="107pt"
- viewBox="0.00 0.00 571.86 107.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+<svg width="671pt" height="107pt"
+ viewBox="0.00 0.00 670.96 107.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
 <g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 103)">
 <title>%3</title>
-<polygon fill="#ffffff" fill-opacity="0.941176" stroke="none" points="-4,4 -4,-103 567.863,-103 567.863,4 -4,4"/>
+<polygon fill="#ffffff" fill-opacity="0.941176" stroke="transparent" points="-4,4 -4,-103 666.96,-103 666.96,4 -4,4"/>
+<!-- new -->
+<g id="node1" class="node">
+<title>new</title>
+<ellipse fill="none" stroke="black" cx="27" cy="-45" rx="27" ry="18"/>
+<text text-anchor="middle" x="27" y="-41.3" font-family="Times,serif" font-size="14.00">new</text>
+</g>
 <!-- waiting -->
-<g id="node1" class="node"><title>waiting</title>
-<ellipse fill="none" stroke="black" cx="37.0467" cy="-45" rx="37.0935" ry="18"/>
-<text text-anchor="middle" x="37.0467" y="-41.3" font-family="Times,serif" font-size="14.00">waiting</text>
+<g id="node2" class="node">
+<title>waiting</title>
+<ellipse fill="none" stroke="black" cx="127.7" cy="-45" rx="37.89" ry="18"/>
+<text text-anchor="middle" x="127.7" y="-41.3" font-family="Times,serif" font-size="14.00">waiting</text>
+</g>
+<!-- new&#45;&gt;waiting -->
+<g id="edge1" class="edge">
+<title>new&#45;&gt;waiting</title>
+<path fill="none" stroke="black" d="M54.19,-45C62.07,-45 70.97,-45 79.77,-45"/>
+<polygon fill="black" stroke="black" points="79.98,-48.5 89.98,-45 79.98,-41.5 79.98,-48.5"/>
 </g>
 <!-- ready -->
-<g id="node2" class="node"><title>ready</title>
-<ellipse fill="none" stroke="black" cx="140.641" cy="-45" rx="30.5947" ry="18"/>
-<text text-anchor="middle" x="140.641" y="-41.3" font-family="Times,serif" font-size="14.00">ready</text>
+<g id="node3" class="node">
+<title>ready</title>
+<ellipse fill="none" stroke="black" cx="232.59" cy="-45" rx="31.4" ry="18"/>
+<text text-anchor="middle" x="232.59" y="-41.3" font-family="Times,serif" font-size="14.00">ready</text>
 </g>
 <!-- waiting&#45;&gt;ready -->
-<g id="edge1" class="edge"><title>waiting&#45;&gt;ready</title>
-<path fill="none" stroke="black" d="M74.151,-45C82.5037,-45 91.4363,-45 99.9337,-45"/>
-<polygon fill="black" stroke="black" points="100.014,-48.5001 110.014,-45 100.014,-41.5001 100.014,-48.5001"/>
+<g id="edge2" class="edge">
+<title>waiting&#45;&gt;ready</title>
+<path fill="none" stroke="black" d="M165.55,-45C173.83,-45 182.67,-45 191.09,-45"/>
+<polygon fill="black" stroke="black" points="191.09,-48.5 201.09,-45 191.09,-41.5 191.09,-48.5"/>
 </g>
 <!-- executing -->
-<g id="node3" class="node"><title>executing</title>
-<ellipse fill="none" stroke="black" cx="252.034" cy="-45" rx="44.6926" ry="18"/>
-<text text-anchor="middle" x="252.034" y="-41.3" font-family="Times,serif" font-size="14.00">executing</text>
+<g id="node4" class="node">
+<title>executing</title>
+<ellipse fill="none" stroke="black" cx="345.93" cy="-45" rx="46.29" ry="18"/>
+<text text-anchor="middle" x="345.93" y="-41.3" font-family="Times,serif" font-size="14.00">executing</text>
 </g>
 <!-- ready&#45;&gt;executing -->
-<g id="edge2" class="edge"><title>ready&#45;&gt;executing</title>
-<path fill="none" stroke="black" d="M171.244,-45C179.158,-45 187.966,-45 196.759,-45"/>
-<polygon fill="black" stroke="black" points="197.021,-48.5001 207.021,-45 197.021,-41.5001 197.021,-48.5001"/>
+<g id="edge3" class="edge">
+<title>ready&#45;&gt;executing</title>
+<path fill="none" stroke="black" d="M264.02,-45C271.88,-45 280.6,-45 289.32,-45"/>
+<polygon fill="black" stroke="black" points="289.51,-48.5 299.51,-45 289.51,-41.5 289.51,-48.5"/>
 </g>
 <!-- long&#45;running -->
-<g id="node4" class="node"><title>long&#45;running</title>
-<ellipse fill="none" stroke="black" cx="389.425" cy="-45" rx="56.59" ry="18"/>
-<text text-anchor="middle" x="389.425" y="-41.3" font-family="Times,serif" font-size="14.00">long&#45;running</text>
+<g id="node5" class="node">
+<title>long&#45;running</title>
+<ellipse fill="none" stroke="black" cx="486.57" cy="-45" rx="58.49" ry="18"/>
+<text text-anchor="middle" x="486.57" y="-41.3" font-family="Times,serif" font-size="14.00">long&#45;running</text>
 </g>
 <!-- executing&#45;&gt;long&#45;running -->
-<g id="edge3" class="edge"><title>executing&#45;&gt;long&#45;running</title>
-<path fill="none" stroke="black" d="M296.992,-45C305.205,-45 313.953,-45 322.657,-45"/>
-<polygon fill="black" stroke="black" points="322.822,-48.5001 332.822,-45 322.822,-41.5001 322.822,-48.5001"/>
+<g id="edge4" class="edge">
+<title>executing&#45;&gt;long&#45;running</title>
+<path fill="none" stroke="black" d="M392.33,-45C400.5,-45 409.18,-45 417.83,-45"/>
+<polygon fill="black" stroke="black" points="417.93,-48.5 427.93,-45 417.93,-41.5 417.93,-48.5"/>
 </g>
 <!-- memory -->
-<g id="node5" class="node"><title>memory</title>
-<ellipse fill="none" stroke="black" cx="522.916" cy="-81" rx="40.8928" ry="18"/>
-<text text-anchor="middle" x="522.916" y="-77.3" font-family="Times,serif" font-size="14.00">memory</text>
+<g id="node6" class="node">
+<title>memory</title>
+<ellipse fill="none" stroke="black" cx="622.02" cy="-81" rx="40.89" ry="18"/>
+<text text-anchor="middle" x="622.02" y="-77.3" font-family="Times,serif" font-size="14.00">memory</text>
 </g>
 <!-- executing&#45;&gt;memory -->
-<g id="edge4" class="edge"><title>executing&#45;&gt;memory</title>
-<path fill="none" stroke="black" d="M283.97,-57.7049C298.553,-63.1083 316.345,-68.8743 332.88,-72 379.21,-80.7579 433.001,-82.5251 471.329,-82.387"/>
-<polygon fill="black" stroke="black" points="471.74,-85.8839 481.711,-82.3018 471.682,-78.8841 471.74,-85.8839"/>
+<g id="edge5" class="edge">
+<title>executing&#45;&gt;memory</title>
+<path fill="none" stroke="black" d="M378.78,-57.85C393.53,-63.21 411.44,-68.9 428.08,-72 476.05,-80.92 531.81,-82.61 571.02,-82.4"/>
+<polygon fill="black" stroke="black" points="571.15,-85.9 581.11,-82.31 571.08,-78.9 571.15,-85.9"/>
 </g>
 <!-- error -->
-<g id="node6" class="node"><title>error</title>
-<ellipse fill="none" stroke="black" cx="522.916" cy="-18" rx="27.8951" ry="18"/>
-<text text-anchor="middle" x="522.916" y="-14.3" font-family="Times,serif" font-size="14.00">error</text>
+<g id="node7" class="node">
+<title>error</title>
+<ellipse fill="none" stroke="black" cx="622.02" cy="-18" rx="28.7" ry="18"/>
+<text text-anchor="middle" x="622.02" y="-14.3" font-family="Times,serif" font-size="14.00">error</text>
 </g>
 <!-- executing&#45;&gt;error -->
-<g id="edge5" class="edge"><title>executing&#45;&gt;error</title>
-<path fill="none" stroke="black" d="M283.527,-32.0792C298.166,-26.5573 316.131,-20.7553 332.88,-18 385.125,-9.40567 446.597,-11.6437 484.917,-14.4742"/>
-<polygon fill="black" stroke="black" points="484.839,-17.9789 495.084,-15.2797 485.392,-11.0008 484.839,-17.9789"/>
+<g id="edge6" class="edge">
+<title>executing&#45;&gt;error</title>
+<path fill="none" stroke="black" d="M378.33,-31.94C393.14,-26.46 411.23,-20.73 428.08,-18 481.54,-9.34 544.43,-11.62 583.51,-14.47"/>
+<polygon fill="black" stroke="black" points="583.39,-17.97 593.64,-15.26 583.94,-11 583.39,-17.97"/>
 </g>
 <!-- long&#45;running&#45;&gt;memory -->
-<g id="edge6" class="edge"><title>long&#45;running&#45;&gt;memory</title>
-<path fill="none" stroke="black" d="M432.754,-56.5888C447.149,-60.5301 463.268,-64.9432 477.801,-68.9219"/>
-<polygon fill="black" stroke="black" points="476.998,-72.3308 487.567,-71.5958 478.846,-65.5793 476.998,-72.3308"/>
+<g id="edge7" class="edge">
+<title>long&#45;running&#45;&gt;memory</title>
+<path fill="none" stroke="black" d="M531.27,-56.79C545.84,-60.72 562.09,-65.1 576.71,-69.05"/>
+<polygon fill="black" stroke="black" points="575.97,-72.47 586.53,-71.7 577.79,-65.71 575.97,-72.47"/>
 </g>
 <!-- long&#45;running&#45;&gt;error -->
-<g id="edge7" class="edge"><title>long&#45;running&#45;&gt;error</title>
-<path fill="none" stroke="black" d="M437.525,-35.3286C453.571,-32.0337 471.233,-28.4071 486.234,-25.3269"/>
-<polygon fill="black" stroke="black" points="487.05,-28.7325 496.141,-23.2926 485.642,-21.8756 487.05,-28.7325"/>
+<g id="edge8" class="edge">
+<title>long&#45;running&#45;&gt;error</title>
+<path fill="none" stroke="black" d="M536.13,-35.18C552.11,-31.94 569.6,-28.4 584.53,-25.38"/>
+<polygon fill="black" stroke="black" points="585.3,-28.8 594.41,-23.38 583.91,-21.94 585.3,-28.8"/>
 </g>
 </g>
 </svg>
diff --git a/docs/source/worker.rst b/docs/source/worker.rst
index 77a361dc686..eaf547a8321 100644
--- a/docs/source/worker.rst
+++ b/docs/source/worker.rst
@@ -89,13 +89,18 @@ Internal Scheduling
 -------------------
 
 Internally tasks that come to the scheduler proceed through the following
-pipeline:
+pipeline as :py:class:`distributed.worker.TaskState` objects.  Tasks which
+follow this path have a :py:attr:`distributed.worker.TaskState.runspec` defined
+which instructs the worker how to execute them.
 
 .. image:: images/worker-task-state.svg
     :alt: Dask worker task states
 
-The worker also tracks data dependencies that are required to run the tasks
-above.  These follow through a simpler pipeline:
+Data dependencies are also represented as
+:py:class:`distributed.worker.TaskState` objects and follow a simpler path
+through the execution pipeline.  These tasks do not have a
+:py:attr:`distributed.worker.TaskState.runspec` defined and instead contain a
+listing of workers to collect their result from.
 
 
 .. image:: images/worker-dep-state.svg
@@ -108,9 +113,12 @@ dependency from that worker.  To improve bandwidth we opportunistically gather
 other dependencies of other tasks that are known to be on that worker, up to a
 maximum of 200MB of data (too little data and bandwidth suffers, too much data
 and responsiveness suffers).  We use a fixed number of connections (around
-10-50) so as to avoid overly-fragmenting our network bandwidth.  After all
-dependencies for a task are in memory we transition the task to the ready state
-and put the task again into a heap of tasks that are ready to run.
+10-50) so as to avoid overly-fragmenting our network bandwidth. In the event
+that the network comms between two workers are saturated, a dependency task may
+cycle between ``fetch`` and ``flight`` until it is successfully collected.
+
+After all dependencies for a task are in memory we transition the task to the
+ready state and put the task again into a heap of tasks that are ready to run.
 
 We collect from this heap and put the task into a thread from a local thread
 pool to execute.
@@ -122,7 +130,17 @@ thread pool.
 A task either errs or its result is put into memory.  In either case a response
 is sent back to the scheduler.
 
-.. _memman::
+Tasks slated for execution and tasks marked for collection from other workers
+must follow their respective transition paths as defined above. The only
+exceptions to this are when:
+
+* A task is `stolen <work-stealing>`_, in which case a task which might have
+  been collected will instead be executed on the thieving worker
+* Scheduler intercession, in which the scheduler reassigns a task that was
+  previously assigned to a separate worker to a new worker.  This most commonly
+  occurs when a `worker dies <killed>`_ during computation.
+
+.. _memman:
 
 Memory Management
 -----------------

From 5ce305ed603638c2f09dee4c816ac07638936674 Mon Sep 17 00:00:00 2001
From: Fabian Gebhart <fabian.gebhart@gmail.com>
Date: Fri, 19 Mar 2021 16:52:21 +0100
Subject: [PATCH 1190/1550] Make length of events deque configurable (#4604)

---
 distributed/scheduler.py            |  6 +++++-
 distributed/tests/test_scheduler.py | 16 ++++++++++++++++
 2 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index d4e0b20dcb8..4a5806b12a8 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -3279,7 +3279,11 @@ def __init__(
         self.log = deque(
             maxlen=dask.config.get("distributed.scheduler.transition-log-length")
         )
-        self.events = defaultdict(lambda: deque(maxlen=100000))
+        self.events = defaultdict(
+            lambda: deque(
+                maxlen=dask.config.get("distributed.scheduler.transition-log-length")
+            )
+        )
         self.event_counts = defaultdict(int)
         self.worker_plugins = []
 
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 701a874fbcc..826842bffb4 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -2184,3 +2184,19 @@ async def test_retire_state_change(c, s, a, b):
         step = c.compute(foo)
         c.gather(step)
     await c.retire_workers(workers=[a.address])
+
+
+@gen_cluster(client=True, config={"distributed.scheduler.transition-log-length": 3})
+async def test_configurable_events_log_length(c, s, a, b):
+    s.log_event("test", "dummy message 1")
+    assert len(s.events["test"]) == 1
+    s.log_event("test", "dummy message 2")
+    s.log_event("test", "dummy message 3")
+    assert len(s.events["test"]) == 3
+
+    # adding a forth message will drop the first one and length stays at 3
+    s.log_event("test", "dummy message 4")
+    assert len(s.events["test"]) == 3
+    assert s.events["test"][0][1] == "dummy message 2"
+    assert s.events["test"][1][1] == "dummy message 3"
+    assert s.events["test"][2][1] == "dummy message 4"

From 3e2ea59bf9ff8dbf53d4eccaec95e99a222d30f0 Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Fri, 19 Mar 2021 15:56:22 +0000
Subject: [PATCH 1191/1550] Add check to scheduler creation in SpecCluster
 (#4605)

---
 distributed/deploy/spec.py | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index ef1d8a4c6fa..d9b1fe90e58 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -252,6 +252,7 @@ def __init__(
         self.scheduler_spec = copy.copy(scheduler)
         self.worker_spec = copy.copy(workers) or {}
         self.new_spec = copy.copy(worker)
+        self.scheduler = None
         self.workers = {}
         self._i = 0
         self.security = security or Security()
@@ -290,6 +291,7 @@ async def _start(self):
             raise ValueError("Cluster is closed")
 
         self._lock = asyncio.Lock()
+        self.status = Status.starting
 
         if self.scheduler_spec is None:
             try:
@@ -300,13 +302,13 @@ async def _start(self):
                 options = {"dashboard": True}
             self.scheduler_spec = {"cls": Scheduler, "options": options}
 
-        cls = self.scheduler_spec["cls"]
-        if isinstance(cls, str):
-            cls = import_term(cls)
-        self.scheduler = cls(**self.scheduler_spec.get("options", {}))
-
-        self.status = Status.starting
-        self.scheduler = await self.scheduler
+        # Check if scheduler has already been created by a subclass
+        if self.scheduler is None:
+            cls = self.scheduler_spec["cls"]
+            if isinstance(cls, str):
+                cls = import_term(cls)
+            self.scheduler = cls(**self.scheduler_spec.get("options", {}))
+            self.scheduler = await self.scheduler
         self.scheduler_comm = rpc(
             getattr(self.scheduler, "external_address", None) or self.scheduler.address,
             connection_args=self.security.get_connection_args("client"),

From 42894418a7c63499eeb240a8cbd3c7cc548bea11 Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Sat, 20 Mar 2021 18:24:25 +0000
Subject: [PATCH 1192/1550] Upstream config serialization and inheritance
 (#4372)

---
 distributed/cli/dask_scheduler.py |  6 ------
 distributed/cli/dask_spec.py      |  7 -------
 distributed/cli/dask_worker.py    |  6 +-----
 distributed/deploy/ssh.py         | 10 ++++-----
 distributed/tests/test_utils.py   | 18 ----------------
 distributed/utils.py              | 35 ++++---------------------------
 6 files changed, 10 insertions(+), 72 deletions(-)

diff --git a/distributed/cli/dask_scheduler.py b/distributed/cli/dask_scheduler.py
index 335fcd63b5c..37e8c497555 100755
--- a/distributed/cli/dask_scheduler.py
+++ b/distributed/cli/dask_scheduler.py
@@ -7,14 +7,12 @@
 import warnings
 
 import click
-import dask
 
 from tornado.ioloop import IOLoop
 
 from distributed import Scheduler
 from distributed.preloading import validate_preload_argv
 from distributed.cli.utils import check_python_3, install_signal_handlers
-from distributed.utils import deserialize_for_cli
 from distributed.proctitle import (
     enable_proctitle_on_children,
     enable_proctitle_on_current,
@@ -167,10 +165,6 @@ def main(
         if v is not None
     }
 
-    if "DASK_INTERNAL_INHERIT_CONFIG" in os.environ:
-        config = deserialize_for_cli(os.environ["DASK_INTERNAL_INHERIT_CONFIG"])
-        dask.config.update(dask.config.global_config, config)
-
     if not host and (tls_ca_file or tls_cert or tls_key):
         host = "tls://"
 
diff --git a/distributed/cli/dask_spec.py b/distributed/cli/dask_spec.py
index 299878a3a46..3311031763f 100644
--- a/distributed/cli/dask_spec.py
+++ b/distributed/cli/dask_spec.py
@@ -1,13 +1,10 @@
 import asyncio
 import click
 import json
-import os
 import sys
 import yaml
 
-import dask.config
 from distributed.deploy.spec import run_spec
-from distributed.utils import deserialize_for_cli
 
 
 @click.command(context_settings=dict(ignore_unknown_options=True))
@@ -17,10 +14,6 @@
 @click.version_option()
 def main(args, spec: str, spec_file: str):
 
-    if "DASK_INTERNAL_INHERIT_CONFIG" in os.environ:
-        config = deserialize_for_cli(os.environ["DASK_INTERNAL_INHERIT_CONFIG"])
-        dask.config.update(dask.config.global_config, config)
-
     if spec and spec_file or not spec and not spec_file:
         print("Must specify exactly one of --spec and --spec-file")
         sys.exit(1)
diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
index 3d60ce35603..a7d2798a2e5 100755
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -20,7 +20,7 @@
     enable_proctitle_on_children,
     enable_proctitle_on_current,
 )
-from distributed.utils import deserialize_for_cli, import_term
+from distributed.utils import import_term
 
 from tlz import valmap
 from tornado.ioloop import IOLoop, TimeoutError
@@ -402,10 +402,6 @@ def del_pid_file():
     with suppress(TypeError, ValueError):
         name = int(name)
 
-    if "DASK_INTERNAL_INHERIT_CONFIG" in os.environ:
-        config = deserialize_for_cli(os.environ["DASK_INTERNAL_INHERIT_CONFIG"])
-        dask.config.update(dask.config.global_config, config)
-
     nannies = [
         t(
             scheduler,
diff --git a/distributed/deploy/ssh.py b/distributed/deploy/ssh.py
index 66003a49572..edf44b83660 100644
--- a/distributed/deploy/ssh.py
+++ b/distributed/deploy/ssh.py
@@ -5,13 +5,13 @@
 import weakref
 
 import dask
+import dask.config
 
 from .spec import SpecCluster, ProcessInterface
 from ..core import Status
 from ..utils import cli_keywords
 from ..scheduler import Scheduler as _Scheduler
 from ..worker import Worker as _Worker
-from ..utils import serialize_for_cli
 
 logger = logging.getLogger(__name__)
 
@@ -95,13 +95,13 @@ async def start(self):
         result = await self.connection.run("uname")
         if result.exit_status == 0:
             set_env = 'env DASK_INTERNAL_INHERIT_CONFIG="{}"'.format(
-                serialize_for_cli(dask.config.global_config)
+                dask.config.serialize(dask.config.global_config)
             )
         else:
             result = await self.connection.run("cmd /c ver")
             if result.exit_status == 0:
                 set_env = "set DASK_INTERNAL_INHERIT_CONFIG={} &&".format(
-                    serialize_for_cli(dask.config.global_config)
+                    dask.config.serialize(dask.config.global_config)
                 )
             else:
                 raise Exception(
@@ -176,13 +176,13 @@ async def start(self):
         result = await self.connection.run("uname")
         if result.exit_status == 0:
             set_env = 'env DASK_INTERNAL_INHERIT_CONFIG="{}"'.format(
-                serialize_for_cli(dask.config.global_config)
+                dask.config.serialize(dask.config.global_config)
             )
         else:
             result = await self.connection.run("cmd /c ver")
             if result.exit_status == 0:
                 set_env = "set DASK_INTERNAL_INHERIT_CONFIG={} &&".format(
-                    serialize_for_cli(dask.config.global_config)
+                    dask.config.serialize(dask.config.global_config)
                 )
             else:
                 raise Exception(
diff --git a/distributed/tests/test_utils.py b/distributed/tests/test_utils.py
index 022e9925445..01332f38834 100644
--- a/distributed/tests/test_utils.py
+++ b/distributed/tests/test_utils.py
@@ -45,8 +45,6 @@
     LRU,
     offload,
     TimeoutError,
-    deserialize_for_cli,
-    serialize_for_cli,
 )
 from distributed.utils_test import loop, loop_in_thread  # noqa: F401
 from distributed.utils_test import div, has_ipv6, inc, throws, gen_test, captured_logger
@@ -608,19 +606,3 @@ def test_lru():
 async def test_offload():
     assert (await offload(inc, 1)) == 2
     assert (await offload(lambda x, y: x + y, 1, y=2)) == 3
-
-
-def test_cli_serialization():
-    # Use context manager without changing the value to ensure test side effects are restored
-    with dask.config.set(
-        {
-            "distributed.comm.default-scheme": dask.config.get(
-                "distributed.comm.default-scheme"
-            )
-        }
-    ):
-        config = deserialize_for_cli(
-            serialize_for_cli({"distributed": {"comm": {"default-scheme": "tls"}}})
-        )  # Take a round trip through the serialization
-        dask.config.update(dask.config.global_config, config)
-        assert dask.config.get("distributed.comm.default-scheme") == "tls"
diff --git a/distributed/utils.py b/distributed/utils.py
index 115e9578b6b..00dd4680442 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -25,7 +25,6 @@
 import warnings
 import weakref
 import pkgutil
-import base64
 import tblib.pickling_support
 import xml.etree.ElementTree
 
@@ -37,6 +36,10 @@
 import dask
 from dask import istask
 
+# Import config serialization functions here for backward compatibility
+from dask.config import serialize as serialize_for_cli  # noqa
+from dask.config import deserialize as deserialize_for_cli  # noqa
+
 # provide format_bytes here for backwards compatibility
 from dask.utils import (  # noqa
     format_bytes,
@@ -1464,36 +1467,6 @@ async def offload(fn, *args, **kwargs):
     return await loop.run_in_executor(_offload_executor, lambda: fn(*args, **kwargs))
 
 
-def serialize_for_cli(data):
-    """Serialize data into a string that can be passthrough cli
-
-    Parameters
-    ----------
-    data : json-serializable object
-        The data to serialize
-    Returns
-    -------
-    serialized_data : str
-        The serialized data as a string
-    """
-    return base64.urlsafe_b64encode(json.dumps(data).encode()).decode()
-
-
-def deserialize_for_cli(data):
-    """De-serialize data into the original object
-
-    Parameters
-    ----------
-    data : str
-        String serialied by serialize_for_cli()
-    Returns
-    -------
-    deserialized_data : obj
-        The de-serialized data
-    """
-    return json.loads(base64.urlsafe_b64decode(data.encode()).decode())
-
-
 class EmptyContext:
     def __enter__(self):
         pass

From 2ca82c010f054e5dafd430406993286a1130b90b Mon Sep 17 00:00:00 2001
From: Fabian Gebhart <fabian.gebhart@gmail.com>
Date: Mon, 22 Mar 2021 19:08:19 +0100
Subject: [PATCH 1193/1550] Introduce events log length config option (#4615)

---
 distributed/distributed-schema.yaml | 12 ++++++++++++
 distributed/distributed.yaml        |  1 +
 distributed/scheduler.py            |  2 +-
 distributed/tests/test_scheduler.py |  2 +-
 4 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/distributed/distributed-schema.yaml b/distributed/distributed-schema.yaml
index 7e8cba0a3c2..331fdd10ef0 100644
--- a/distributed/distributed-schema.yaml
+++ b/distributed/distributed-schema.yaml
@@ -80,6 +80,18 @@ properties:
               we will clear out old entries after a certain length.
               This is that length.
 
+          events-log-length:
+            type: integer
+            minimum: 0
+            description: |
+              How long should we keep the events log
+
+              All events (e.g. worker heartbeat) are stored in the events log.
+
+              To make sure that we don't run out of memory
+              we will clear out old entries after a certain length.
+              This is that length.
+
           work-stealing:
             type: boolean
             description: |
diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index 36fc7baead1..fb01a168d38 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -18,6 +18,7 @@ distributed:
     events-cleanup-delay: 1h
     idle-timeout: null      # Shut down after this duration, like "1h" or "30 minutes"
     transition-log-length: 100000
+    events-log-length: 100000
     work-stealing: True     # workers should steal tasks from each other
     work-stealing-interval: 100ms  # Callback time for work stealing
     worker-ttl: null        # like '60s'. Time to live for workers.  They must heartbeat faster than this
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 4a5806b12a8..86ffd1982d2 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -3281,7 +3281,7 @@ def __init__(
         )
         self.events = defaultdict(
             lambda: deque(
-                maxlen=dask.config.get("distributed.scheduler.transition-log-length")
+                maxlen=dask.config.get("distributed.scheduler.events-log-length")
             )
         )
         self.event_counts = defaultdict(int)
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 826842bffb4..14cc084f5dc 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -2186,7 +2186,7 @@ async def test_retire_state_change(c, s, a, b):
     await c.retire_workers(workers=[a.address])
 
 
-@gen_cluster(client=True, config={"distributed.scheduler.transition-log-length": 3})
+@gen_cluster(client=True, config={"distributed.scheduler.events-log-length": 3})
 async def test_configurable_events_log_length(c, s, a, b):
     s.log_event("test", "dummy message 1")
     assert len(s.events["test"]) == 1

From 7ccf8378ffae08b96653dde3210e095109e70a77 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Tue, 23 Mar 2021 07:15:41 -0700
Subject: [PATCH 1194/1550] Drop `log` from `remove_key_from_stealable` (#4609)

We already dropped the `log` line from `put_key_in_stealable` because it
was too granular of logging and was slowing things down there. This does
the same thing for `remove_key_from_stealable` for the same reason.
---
 distributed/stealing.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/distributed/stealing.py b/distributed/stealing.py
index 7a336777f26..8dc8c218f87 100644
--- a/distributed/stealing.py
+++ b/distributed/stealing.py
@@ -95,7 +95,6 @@ def remove_key_from_stealable(self, ts):
             return
 
         worker, level = result
-        self.log(("remove-stealable", ts.key, worker, level))
         try:
             self.stealable[worker][level].remove(ts)
         except KeyError:

From 842801820d2b7fbef5165b217da2c19887c3ad21 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 23 Mar 2021 15:37:29 -0500
Subject: [PATCH 1195/1550] Handle async clients when closing (#4623)

---
 distributed/client.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/distributed/client.py b/distributed/client.py
index 2669a25473b..83dd4b85cf3 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -4726,7 +4726,10 @@ def _close_global_client():
     if c is not None:
         c._should_close_loop = False
         with suppress(TimeoutError, RuntimeError):
-            c.close(timeout=3)
+            if c.asynchronous:
+                c.loop.add_callback(c.close, timeout=3)
+            else:
+                c.close(timeout=3)
 
 
 atexit.register(_close_global_client)

From bef0308962345303123aba7ec6730757a61a4dfc Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Tue, 23 Mar 2021 15:52:44 -0500
Subject: [PATCH 1196/1550] Remove `utils.shutting_down` in favor of
 `sys.is_finalizing` (#4624)

---
 distributed/client.py    |  5 ++---
 distributed/comm/tcp.py  |  6 +++---
 distributed/core.py      |  8 ++++----
 distributed/scheduler.py |  3 +--
 distributed/utils.py     | 27 ---------------------------
 5 files changed, 10 insertions(+), 39 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 83dd4b85cf3..625d1fb1040 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -77,7 +77,6 @@
     no_default,
     LoopRunner,
     parse_timedelta,
-    shutting_down,
     Any,
     has_keyword,
     format_dashboard_link,
@@ -383,7 +382,7 @@ def __del__(self):
         except AttributeError:
             # Ocassionally we see this error when shutting down the client
             # https://github.com/dask/distributed/issues/4305
-            if not shutting_down():
+            if not sys.is_finalizing():
                 raise
         except RuntimeError:  # closed event loop
             pass
@@ -1445,7 +1444,7 @@ async def _():
 
         assert self.status == "closed"
 
-        if not shutting_down():
+        if not sys.is_finalizing():
             self._loop_runner.stop()
 
     async def _shutdown(self):
diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py
index b79a5105a0d..6cff9ad8f50 100644
--- a/distributed/comm/tcp.py
+++ b/distributed/comm/tcp.py
@@ -21,7 +21,7 @@
 
 from ..system import MEMORY_LIMIT
 from ..threadpoolexecutor import ThreadPoolExecutor
-from ..utils import ensure_ip, get_ip, get_ipv6, nbytes, parse_timedelta, shutting_down
+from ..utils import ensure_ip, get_ip, get_ipv6, nbytes, parse_timedelta
 
 from .registry import Backend, backends
 from .addressing import parse_host_port, unparse_host_port
@@ -201,7 +201,7 @@ async def read(self, deserializers=None):
         except StreamClosedError as e:
             self.stream = None
             self._closed = True
-            if not shutting_down():
+            if not sys.is_finalizing():
                 convert_stream_closed_error(self, e)
         except Exception:
             # Some OSError or a another "low-level" exception. We do not really know what
@@ -277,7 +277,7 @@ async def write(self, msg, serializers=None, on_error="message"):
         except StreamClosedError as e:
             self.stream = None
             self._closed = True
-            if not shutting_down():
+            if not sys.is_finalizing():
                 convert_stream_closed_error(self, e)
         except Exception:
             # Some OSError or a another "low-level" exception. We do not really know
diff --git a/distributed/core.py b/distributed/core.py
index bc2930a4f78..bcb5d8fe92f 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -5,6 +5,7 @@
 from functools import partial
 import inspect
 import logging
+import sys
 import threading
 import traceback
 import uuid
@@ -32,7 +33,6 @@
     is_coroutine_function,
     get_traceback,
     truncate_exception,
-    shutting_down,
     parse_timedelta,
     has_keyword,
     CancelledError,
@@ -406,7 +406,7 @@ async def listen(self, port_or_addr=None, allow_offload=True, **kwargs):
         )
         self.listeners.append(listener)
 
-    async def handle_comm(self, comm, shutting_down=shutting_down):
+    async def handle_comm(self, comm):
         """Dispatch new communications to coroutine-handlers
 
         Handlers is a dictionary mapping operation names to functions or
@@ -432,7 +432,7 @@ async def handle_comm(self, comm, shutting_down=shutting_down):
                     msg = await comm.read()
                     logger.debug("Message from %r: %s", address, msg)
                 except EnvironmentError as e:
-                    if not shutting_down():
+                    if not sys.is_finalizing():
                         logger.debug(
                             "Lost connection to %r while reading message: %s."
                             " Last operation: %s",
@@ -536,7 +536,7 @@ async def handle_comm(self, comm, shutting_down=shutting_down):
 
         finally:
             del self._comms[comm]
-            if not shutting_down() and not comm.closed():
+            if not sys.is_finalizing() and not comm.closed():
                 try:
                     comm.abort()
                 except Exception as e:
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 86ffd1982d2..190bc7c30c9 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -64,7 +64,6 @@
     no_default,
     parse_timedelta,
     parse_bytes,
-    shutting_down,
     key_split_group,
     empty_context,
     tmpfile,
@@ -4738,7 +4737,7 @@ async def add_client(self, comm, client=None, versions=None):
             if not comm.closed():
                 self.client_comms[client].send({"op": "stream-closed"})
             try:
-                if not shutting_down():
+                if not sys.is_finalizing():
                     await self.client_comms[client].close()
                     del self.client_comms[client]
                     if self.status == Status.running:
diff --git a/distributed/utils.py b/distributed/utils.py
index 00dd4680442..165b13ecc21 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -1,6 +1,5 @@
 import asyncio
 from asyncio import TimeoutError
-import atexit
 import click
 from collections import deque, OrderedDict, UserDict
 from concurrent.futures import ThreadPoolExecutor, CancelledError  # noqa: F401
@@ -944,32 +943,6 @@ def mean(seq):
     return sum(seq) / len(seq)
 
 
-if hasattr(sys, "is_finalizing"):
-
-    def shutting_down(is_finalizing=sys.is_finalizing):
-        return is_finalizing()
-
-
-else:
-    _shutting_down = [False]
-
-    def _at_shutdown(l=_shutting_down):
-        l[0] = True
-
-    def shutting_down(l=_shutting_down):
-        return l[0]
-
-    atexit.register(_at_shutdown)
-
-
-shutting_down.__doc__ = """
-    Whether the interpreter is currently shutting down.
-    For use in finalizers, __del__ methods, and similar; it is advised
-    to early bind this function rather than look it up when calling it,
-    since at shutdown module globals may be cleared.
-    """
-
-
 def open_port(host=""):
     """Return a probably-open port
 

From 89b70acd1963014590207a8d82727df13dc03c5f Mon Sep 17 00:00:00 2001
From: Ben Greiner <code@bnavigator.de>
Date: Wed, 24 Mar 2021 01:15:49 +0100
Subject: [PATCH 1197/1550] Skip numpy and pandas tests if not importable
 (#4563)

---
 distributed/protocol/tests/test_arrow.py      |  2 +-
 distributed/protocol/tests/test_collection.py |  5 ++--
 .../protocol/tests/test_collection_cuda.py    |  7 ++---
 distributed/protocol/tests/test_keras.py      |  5 ++--
 distributed/protocol/tests/test_numpy.py      |  3 +-
 distributed/protocol/tests/test_pandas.py     |  5 ++--
 distributed/protocol/tests/test_serialize.py  | 28 +++++++++++++++++--
 distributed/protocol/tests/test_sparse.py     |  5 ++--
 distributed/tests/test_actor.py               |  2 +-
 distributed/tests/test_client.py              |  8 +++---
 distributed/tests/test_collections.py         |  6 ++--
 distributed/tests/test_nanny.py               |  3 +-
 distributed/tests/test_pubsub.py              |  2 +-
 distributed/tests/test_utils.py               |  4 ++-
 14 files changed, 55 insertions(+), 30 deletions(-)

diff --git a/distributed/protocol/tests/test_arrow.py b/distributed/protocol/tests/test_arrow.py
index e86bfa6f827..3d2c66afbb4 100644
--- a/distributed/protocol/tests/test_arrow.py
+++ b/distributed/protocol/tests/test_arrow.py
@@ -1,7 +1,7 @@
-import pandas as pd
 import pytest
 
 pa = pytest.importorskip("pyarrow")
+pd = pytest.importorskip("pandas")
 
 import distributed
 from distributed.utils_test import gen_cluster
diff --git a/distributed/protocol/tests/test_collection.py b/distributed/protocol/tests/test_collection.py
index fd112b6c792..4bd10a9928c 100644
--- a/distributed/protocol/tests/test_collection.py
+++ b/distributed/protocol/tests/test_collection.py
@@ -1,7 +1,8 @@
 import pytest
 from distributed.protocol import serialize, deserialize
-import pandas as pd
-import numpy as np
+
+np = pytest.importorskip("numpy")
+pd = pytest.importorskip("pandas")
 
 
 @pytest.mark.parametrize("collection", [tuple, dict, list])
diff --git a/distributed/protocol/tests/test_collection_cuda.py b/distributed/protocol/tests/test_collection_cuda.py
index e2602795782..fe66b18bd68 100644
--- a/distributed/protocol/tests/test_collection_cuda.py
+++ b/distributed/protocol/tests/test_collection_cuda.py
@@ -2,7 +2,6 @@
 
 from distributed.protocol import serialize, deserialize
 from dask.dataframe.utils import assert_eq
-import pandas as pd
 
 
 @pytest.mark.parametrize("collection", [tuple, dict])
@@ -37,14 +36,14 @@ def test_serialize_cupy(collection, y, y_serializer):
 @pytest.mark.parametrize("collection", [tuple, dict])
 @pytest.mark.parametrize(
     "df2,df2_serializer",
-    [(pd.DataFrame({"C": [3, 4, 5], "D": [2.5, 3.5, 4.5]}), "cuda"), (None, "pickle")],
+    [({"C": [3, 4, 5], "D": [2.5, 3.5, 4.5]}, "cuda"), (None, "pickle")],
 )
 def test_serialize_pandas_pandas(collection, df2, df2_serializer):
     cudf = pytest.importorskip("cudf")
-
+    pd = pytest.importorskip("pandas")
     df1 = cudf.DataFrame({"A": [1, 2, None], "B": [1.0, 2.0, None]})
     if df2 is not None:
-        df2 = cudf.from_pandas(df2)
+        df2 = cudf.from_pandas(pd.DataFrame(df2))
     if issubclass(collection, dict):
         header, frames = serialize(
             {"df1": df1, "df2": df2}, serializers=("cuda", "dask", "pickle")
diff --git a/distributed/protocol/tests/test_keras.py b/distributed/protocol/tests/test_keras.py
index da8cdf6374a..79dec609ccb 100644
--- a/distributed/protocol/tests/test_keras.py
+++ b/distributed/protocol/tests/test_keras.py
@@ -1,13 +1,14 @@
-import numpy as np
-from numpy.testing import assert_allclose
 import pytest
 
 keras = pytest.importorskip("keras")
+np = pytest.importorskip("numpy")
 
 from distributed.protocol import serialize, deserialize, dumps, loads, to_serialize
 
 
 def test_serialize_deserialize_model():
+    from numpy.testing import assert_allclose
+
     model = keras.models.Sequential()
     model.add(keras.layers.Dense(5, input_dim=3))
     model.add(keras.layers.Dense(2))
diff --git a/distributed/protocol/tests/test_numpy.py b/distributed/protocol/tests/test_numpy.py
index 9096748b5d1..7b8c9236bec 100644
--- a/distributed/protocol/tests/test_numpy.py
+++ b/distributed/protocol/tests/test_numpy.py
@@ -1,8 +1,9 @@
 from zlib import crc32
 
-import numpy as np
 import pytest
 
+np = pytest.importorskip("numpy")
+
 from distributed.protocol import (
     serialize,
     deserialize,
diff --git a/distributed/protocol/tests/test_pandas.py b/distributed/protocol/tests/test_pandas.py
index a8134d7e3d0..ae338200312 100644
--- a/distributed/protocol/tests/test_pandas.py
+++ b/distributed/protocol/tests/test_pandas.py
@@ -1,7 +1,8 @@
-import numpy as np
-import pandas as pd
 import pytest
 
+pd = pytest.importorskip("pandas")
+np = pytest.importorskip("numpy")
+
 from dask.dataframe.utils import assert_eq
 
 from distributed.protocol import (
diff --git a/distributed/protocol/tests/test_serialize.py b/distributed/protocol/tests/test_serialize.py
index 735dffb5c19..a05bedd7e50 100644
--- a/distributed/protocol/tests/test_serialize.py
+++ b/distributed/protocol/tests/test_serialize.py
@@ -3,10 +3,14 @@
 import pickle
 
 import msgpack
-import numpy as np
 import pytest
 from tlz import identity
 
+try:
+    import numpy as np
+except ImportError:
+    np = None
+
 from dask.utils_test import inc
 
 from distributed import wait
@@ -212,6 +216,7 @@ def test_empty_loads_deep():
     assert isinstance(e2[0][0][0], Empty)
 
 
+@pytest.mark.skipif(np is None, reason="Test needs numpy")
 @pytest.mark.parametrize("kwargs", [{}, {"serializers": ["pickle"]}])
 def test_serialize_bytes(kwargs):
     for x in [
@@ -228,6 +233,7 @@ def test_serialize_bytes(kwargs):
         assert str(x) == str(y)
 
 
+@pytest.mark.skipif(np is None, reason="Test needs numpy")
 def test_serialize_list_compress():
     pytest.importorskip("lz4")
     x = np.ones(1000000)
@@ -439,7 +445,13 @@ def _(x):
         (tuple([MyObj(None)]), True),
         ({("x", i): MyObj(5) for i in range(100)}, True),
         (memoryview(b"hello"), True),
-        (memoryview(np.random.random((3, 4))), True),
+        pytest.param(
+            memoryview(
+                np.random.random((3, 4)) if np is not None else b"skip np.random"
+            ),
+            True,
+            marks=pytest.mark.skipif(np is None, reason="Test needs numpy"),
+        ),
     ],
 )
 def test_check_dask_serializable(data, is_serializable):
@@ -462,7 +474,16 @@ def test_serialize_lists(serializers):
 
 
 @pytest.mark.parametrize(
-    "data_in", [memoryview(b"hello"), memoryview(np.random.random((3, 4)))]
+    "data_in",
+    [
+        memoryview(b"hello"),
+        pytest.param(
+            memoryview(
+                np.random.random((3, 4)) if np is not None else b"skip np.random"
+            ),
+            marks=pytest.mark.skipif(np is None, reason="Test needs numpy"),
+        ),
+    ],
 )
 def test_deser_memoryview(data_in):
     header, frames = serialize(data_in)
@@ -472,6 +493,7 @@ def test_deser_memoryview(data_in):
     assert data_in == data_out
 
 
+@pytest.mark.skipif(np is None, reason="Test needs numpy")
 def test_ser_memoryview_object():
     data_in = memoryview(np.array(["hello"], dtype=object))
     with pytest.raises(TypeError):
diff --git a/distributed/protocol/tests/test_sparse.py b/distributed/protocol/tests/test_sparse.py
index 89f9da09bc2..f414e2f3b6f 100644
--- a/distributed/protocol/tests/test_sparse.py
+++ b/distributed/protocol/tests/test_sparse.py
@@ -1,13 +1,14 @@
-import numpy as np
-from numpy.testing import assert_allclose
 import pytest
 
+np = pytest.importorskip("numpy")
 sparse = pytest.importorskip("sparse")
 
 from distributed.protocol import deserialize, serialize
 
 
 def test_serialize_deserialize_sparse():
+    from numpy.testing import assert_allclose
+
     x = np.random.random((2, 3, 4, 5))
     x[x < 0.8] = 0
 
diff --git a/distributed/tests/test_actor.py b/distributed/tests/test_actor.py
index 7b91b3da1c3..72123a14dc4 100644
--- a/distributed/tests/test_actor.py
+++ b/distributed/tests/test_actor.py
@@ -425,8 +425,8 @@ def __init__(self, x, y=None):
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 4, Worker=Nanny)
 async def bench_param_server(c, s, *workers):
+    np = pytest.importorskip("numpy")
     import dask.array as da
-    import numpy as np
 
     x = da.random.random((500000, 1000), chunks=(1000, 1000))
     x = x.persist()
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 7a9ca27352a..b36b7fbf3a4 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -5535,7 +5535,7 @@ async def test_client_active_bad_port():
 def test_turn_off_pickle(direct):
     @gen_cluster()
     async def test(s, a, b):
-        import numpy as np
+        np = pytest.importorskip("numpy")
 
         async with Client(
             s.address, asynchronous=True, serializers=["dask", "msgpack"]
@@ -5573,7 +5573,7 @@ async def test(s, a, b):
 
 @gen_cluster()
 async def test_de_serialization(s, a, b):
-    import numpy as np
+    np = pytest.importorskip("numpy")
 
     c = await Client(
         s.address,
@@ -5594,7 +5594,7 @@ async def test_de_serialization(s, a, b):
 
 @gen_cluster()
 async def test_de_serialization_none(s, a, b):
-    import numpy as np
+    np = pytest.importorskip("numpy")
 
     c = await Client(s.address, asynchronous=True, deserializers=["msgpack"])
     try:
@@ -6432,8 +6432,8 @@ async def test_annotations_retries(c, s, a, b):
 @gen_cluster(client=True)
 async def test_annotations_blockwise_unpack(c, s, a, b):
     da = pytest.importorskip("dask.array")
+    np = pytest.importorskip("numpy")
     from dask.array.utils import assert_eq
-    import numpy as np
 
     # A flaky doubling function -- need extra args because it is called before
     # application to establish dtype/meta.
diff --git a/distributed/tests/test_collections.py b/distributed/tests/test_collections.py
index 022db4080dd..8b3e6cd5a38 100644
--- a/distributed/tests/test_collections.py
+++ b/distributed/tests/test_collections.py
@@ -2,8 +2,8 @@
 
 import pytest
 
-pytest.importorskip("numpy")
-pytest.importorskip("pandas")
+np = pytest.importorskip("numpy")
+pd = pytest.importorskip("pandas")
 
 import dask
 import dask.dataframe as dd
@@ -11,8 +11,6 @@
 from distributed.client import wait
 from distributed.utils_test import gen_cluster
 from distributed.utils_test import client, cluster_fixture, loop  # noqa F401
-import numpy as np
-import pandas as pd
 
 PANDAS_VERSION = LooseVersion(pd.__version__)
 PANDAS_GT_100 = PANDAS_VERSION >= LooseVersion("1.0.0")
diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
index 1993f11a377..83c3de18c43 100644
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -7,8 +7,6 @@
 import sys
 import multiprocessing as mp
 
-import numpy as np
-
 import pytest
 from tlz import valmap, first
 from tornado.ioloop import IOLoop
@@ -202,6 +200,7 @@ async def check_func(func):
         assert x != y
 
     await check_func(lambda a, b: random.randint(a, b))
+    np = pytest.importorskip("numpy")
     await check_func(lambda a, b: np.random.randint(a, b))
 
 
diff --git a/distributed/tests/test_pubsub.py b/distributed/tests/test_pubsub.py
index 8f8a3b734e7..09ab4dcd342 100644
--- a/distributed/tests/test_pubsub.py
+++ b/distributed/tests/test_pubsub.py
@@ -37,7 +37,7 @@ def pingpong(a, b, start=False, n=1000, msg=1):
             #     print(a, b, i)
         return n
 
-    import numpy as np
+    np = pytest.importorskip("numpy")
 
     x = np.random.random(1000)
 
diff --git a/distributed/tests/test_utils.py b/distributed/tests/test_utils.py
index 01332f38834..8b13b6b27fb 100644
--- a/distributed/tests/test_utils.py
+++ b/distributed/tests/test_utils.py
@@ -10,7 +10,6 @@
 from time import sleep
 import traceback
 
-import numpy as np
 import pytest
 from tornado.ioloop import IOLoop
 
@@ -269,6 +268,7 @@ def test_ensure_bytes():
 
 
 def test_ensure_bytes_ndarray():
+    np = pytest.importorskip("numpy")
     result = ensure_bytes(np.arange(12))
     assert isinstance(result, bytes)
 
@@ -281,6 +281,8 @@ def test_ensure_bytes_pyarrow_buffer():
 
 
 def test_nbytes():
+    np = pytest.importorskip("numpy")
+
     def check(obj, expected):
         assert nbytes(obj) == expected
         assert nbytes(memoryview(obj)) == expected

From df8f8923ea39c71fdaa3056dbe3726a993abebab Mon Sep 17 00:00:00 2001
From: "Mads R. B. Kristensen" <madsbk@gmail.com>
Date: Wed, 24 Mar 2021 15:08:57 +0100
Subject: [PATCH 1198/1550] UCX use nbytes instead of len (#4621)

---
 distributed/comm/ucx.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/distributed/comm/ucx.py b/distributed/comm/ucx.py
index 0d91b404ee2..efa74254745 100644
--- a/distributed/comm/ucx.py
+++ b/distributed/comm/ucx.py
@@ -205,7 +205,7 @@ async def write(
                     *(
                         (is_cuda, each_frame)
                         for is_cuda, each_frame in zip(cuda_frames, frames)
-                        if len(each_frame) > 0
+                        if nbytes(each_frame) > 0
                     )
                 )
 
@@ -275,7 +275,7 @@ async def read(self, deserializers=("cuda", "dask", "pickle", "error")):
                     *(
                         (is_cuda, each_frame)
                         for is_cuda, each_frame in zip(cuda_frames, frames)
-                        if len(each_frame) > 0
+                        if nbytes(each_frame) > 0
                     )
                 )
 

From 8942856bb83781c50c1307a3c8e8ed75f43191db Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Wed, 24 Mar 2021 10:23:47 -0500
Subject: [PATCH 1199/1550] Rename annotation plugin in test_highlevelgraph.py
 (#4618)

---
 distributed/protocol/tests/test_highlevelgraph.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/distributed/protocol/tests/test_highlevelgraph.py b/distributed/protocol/tests/test_highlevelgraph.py
index 1fcb339f721..e974c00fdbc 100644
--- a/distributed/protocol/tests/test_highlevelgraph.py
+++ b/distributed/protocol/tests/test_highlevelgraph.py
@@ -91,7 +91,7 @@ async def test_shuffle(c, s, a, b):
     assert (res == 10.0).all()
 
 
-class TestAnnotationPlugin(SchedulerPlugin):
+class ExampleAnnotationPlugin(SchedulerPlugin):
     def __init__(self, priority_fn=None, qux="", resource="", retries=0):
         self.priority_fn = priority_fn or (lambda k: 0)
         self.qux = qux
@@ -134,7 +134,7 @@ def fn(k):
     qux = "baz"
     resource = "widget"
 
-    plugin = TestAnnotationPlugin(priority_fn=fn, qux=qux, resource=resource)
+    plugin = ExampleAnnotationPlugin(priority_fn=fn, qux=qux, resource=resource)
     s.add_plugin(plugin)
 
     assert plugin in s.plugins
@@ -159,7 +159,7 @@ def fn(k):
 @gen_cluster(client=True)
 async def test_dataframe_annotations(c, s, a, b):
     retries = 5
-    plugin = TestAnnotationPlugin(retries=retries)
+    plugin = ExampleAnnotationPlugin(retries=retries)
     s.add_plugin(plugin)
 
     assert plugin in s.plugins

From 3b8b97e31ed10598531fb458b53db16683c92c47 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Thu, 25 Mar 2021 01:52:43 -0500
Subject: [PATCH 1200/1550] Include LIST_PICKLE in NumPy array serialization
 (#4632)

---
 distributed/protocol/numpy.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/protocol/numpy.py b/distributed/protocol/numpy.py
index 65f7e2f4076..03064581d8b 100644
--- a/distributed/protocol/numpy.py
+++ b/distributed/protocol/numpy.py
@@ -20,7 +20,7 @@ def itemsize(dt):
 
 @dask_serialize.register(np.ndarray)
 def serialize_numpy_ndarray(x, context=None):
-    if x.dtype.hasobject:
+    if x.dtype.hasobject or (x.dtype.flags & np.core.multiarray.LIST_PICKLE):
         header = {"pickle": True}
         frames = [None]
         buffer_callback = lambda f: frames.append(memoryview(f))

From 43859e25d850d903028e5aa3de8368c12b1e4fdc Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Thu, 25 Mar 2021 08:55:07 -0500
Subject: [PATCH 1201/1550] Handle stream is None case in TCP comm finalizer
 (#4631)

---
 distributed/comm/tcp.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py
index 6cff9ad8f50..530dbfbcf1a 100644
--- a/distributed/comm/tcp.py
+++ b/distributed/comm/tcp.py
@@ -169,8 +169,9 @@ def _read_extra(self):
 
     def _get_finalizer(self):
         def finalize(stream=self.stream, r=repr(self)):
-            if not stream.closed():
-                logger.warning("Closing dangling stream in %s" % (r,))
+            # stream is None if a StreamClosedError is raised during interpreter shutdown
+            if stream is not None and not stream.closed():
+                logger.warning(f"Closing dangling stream in {r}")
                 stream.close()
 
         return finalize

From bf9ddab3fd67c33b0d87da4f18bcdc7e47262393 Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Thu, 25 Mar 2021 17:29:40 +0000
Subject: [PATCH 1202/1550] Add standalone dashboard page for GPU usage (#4556)

---
 distributed/dashboard/components/nvml.py | 42 +++++++++++++++++-------
 distributed/dashboard/scheduler.py       | 11 ++++++-
 distributed/http/static/css/gpu.css      | 16 +++++++++
 distributed/http/templates/gpu.html      | 22 +++++++++++++
 4 files changed, 78 insertions(+), 13 deletions(-)
 create mode 100644 distributed/http/static/css/gpu.css
 create mode 100644 distributed/http/templates/gpu.html

diff --git a/distributed/dashboard/components/nvml.py b/distributed/dashboard/components/nvml.py
index 34cce3c4bc7..8be3b406bcb 100644
--- a/distributed/dashboard/components/nvml.py
+++ b/distributed/dashboard/components/nvml.py
@@ -14,7 +14,7 @@
 from tornado import escape
 from dask.utils import format_bytes
 from distributed.utils import log_errors
-from distributed.dashboard.components.scheduler import BOKEH_THEME, TICKS_1024
+from distributed.dashboard.components.scheduler import BOKEH_THEME, TICKS_1024, env
 from distributed.dashboard.utils import without_property_validation, update
 
 
@@ -22,8 +22,10 @@
     import pynvml
 
     pynvml.nvmlInit()
+
+    NVML_ENABLED = True
 except Exception:
-    pass
+    NVML_ENABLED = False
 
 
 class GPUCurrentLoad(DashboardComponent):
@@ -173,16 +175,32 @@ def update(self):
 
 
 def gpu_memory_doc(scheduler, extra, doc):
-    gpu_load = GPUCurrentLoad(scheduler, sizing_mode="stretch_both")
-    gpu_load.update()
-    add_periodic_callback(doc, gpu_load, 100)
-    doc.add_root(gpu_load.memory_figure)
-    doc.theme = BOKEH_THEME
+    with log_errors():
+        gpu_load = GPUCurrentLoad(scheduler, sizing_mode="stretch_both")
+        gpu_load.update()
+        add_periodic_callback(doc, gpu_load, 100)
+        doc.add_root(gpu_load.memory_figure)
+        doc.theme = BOKEH_THEME
 
 
 def gpu_utilization_doc(scheduler, extra, doc):
-    gpu_load = GPUCurrentLoad(scheduler, sizing_mode="stretch_both")
-    gpu_load.update()
-    add_periodic_callback(doc, gpu_load, 100)
-    doc.add_root(gpu_load.utilization_figure)
-    doc.theme = BOKEH_THEME
+    with log_errors():
+        gpu_load = GPUCurrentLoad(scheduler, sizing_mode="stretch_both")
+        gpu_load.update()
+        add_periodic_callback(doc, gpu_load, 100)
+        doc.add_root(gpu_load.utilization_figure)
+        doc.theme = BOKEH_THEME
+
+
+def gpu_doc(scheduler, extra, doc):
+    with log_errors():
+        gpu_load = GPUCurrentLoad(scheduler, sizing_mode="stretch_both")
+        gpu_load.update()
+        add_periodic_callback(doc, gpu_load, 100)
+        doc.add_root(gpu_load.memory_figure)
+        doc.add_root(gpu_load.utilization_figure)
+
+        doc.title = "Dask: GPU"
+        doc.theme = BOKEH_THEME
+        doc.template = env.get_template("gpu.html")
+        doc.template_variables.update(extra)
diff --git a/distributed/dashboard/scheduler.py b/distributed/dashboard/scheduler.py
index 09a4339b50a..413dde57301 100644
--- a/distributed/dashboard/scheduler.py
+++ b/distributed/dashboard/scheduler.py
@@ -37,13 +37,21 @@
     individual_systemmonitor_doc,
 )
 from .worker import counters_doc
-from .components.nvml import gpu_memory_doc, gpu_utilization_doc  # noqa: 1708
+from .components.nvml import (
+    NVML_ENABLED,
+    gpu_memory_doc,
+    gpu_utilization_doc,
+    gpu_doc,
+)  # noqa: 1708
 
 
 template_variables = {
     "pages": ["status", "workers", "tasks", "system", "profile", "graph", "info"]
 }
 
+if NVML_ENABLED:
+    template_variables["pages"].insert(4, "gpu")
+
 
 def connect(application, http_server, scheduler, prefix=""):
     bokeh_app = BokehApplication(
@@ -75,6 +83,7 @@ def connect(application, http_server, scheduler, prefix=""):
     "/profile": profile_doc,
     "/profile-server": profile_server_doc,
     "/graph": graph_doc,
+    "/gpu": gpu_doc,
     "/individual-task-stream": individual_task_stream_doc,
     "/individual-progress": individual_progress_doc,
     "/individual-graph": individual_graph_doc,
diff --git a/distributed/http/static/css/gpu.css b/distributed/http/static/css/gpu.css
new file mode 100644
index 00000000000..44d66163a62
--- /dev/null
+++ b/distributed/http/static/css/gpu.css
@@ -0,0 +1,16 @@
+#status-fluid {
+    display: grid;
+    height: 100%;
+  }
+#status-fluid {
+    grid-template-columns: 1fr 1fr;
+    grid-template-rows: 1fr;
+}
+#gpu-memory {
+    grid-column: 2;
+    grid-row: 1;
+}
+#gpu-utilization {
+    grid-column: 1;
+    grid-row: 1;
+}
\ No newline at end of file
diff --git a/distributed/http/templates/gpu.html b/distributed/http/templates/gpu.html
new file mode 100644
index 00000000000..3c80f21d332
--- /dev/null
+++ b/distributed/http/templates/gpu.html
@@ -0,0 +1,22 @@
+{% extends "base.html" %}
+
+{% block extra_resources %}
+<link rel="stylesheet" href="statics/css/gpu.css">
+{% endblock %}
+
+{% block content %}
+{% from macros import embed %}
+<div id="status-fluid">
+
+  <div id="gpu-utilization">
+    {{ embed(roots.gpu_utilization_histogram) }}
+  </div>
+
+  <div id="gpu-memory">
+    {{ embed(roots.gpu_memory_histogram) }}
+  </div>
+
+</div>
+{{ plot_script }}
+
+{% endblock %}
\ No newline at end of file

From 25a2867e8e950d9cb51b071337ffc43ddaf0bb52 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Fri, 26 Mar 2021 15:19:25 -0500
Subject: [PATCH 1203/1550] bump version to 2021.03.1

---
 docs/source/changelog.rst | 43 ++++++++++++++++++++++++++++++++++++++-
 1 file changed, 42 insertions(+), 1 deletion(-)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index 30b4f1c77d1..984e3e936df 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,43 @@
 Changelog
 =========
 
+2021.03.1
+---------
+
+Released on March 26, 2021
+
+- Add standalone dashboard page for GPU usage (:pr:`4556`) `Jacob Tomlinson`_
+- Handle ``stream is None`` case in TCP comm finalizer (:pr:`4631`) `James Bourbeau`_
+- Include ``LIST_PICKLE`` in NumPy array serialization (:pr:`4632`) `James Bourbeau`_
+- Rename annotation plugin in ``test_highlevelgraph.py`` (:pr:`4618`) `James Bourbeau`_
+- UCX use ``nbytes`` instead of ``len`` (:pr:`4621`) `Mads R. B. Kristensen`_
+- Skip NumPy and pandas tests if not importable (:pr:`4563`) `Ben Greiner`_
+- Remove ``utils.shutting_down`` in favor of ``sys.is_finalizing`` (:pr:`4624`) `James Bourbeau`_
+- Handle ``async`` clients when closing (:pr:`4623`) `Matthew Rocklin`_
+- Drop ``log`` from ``remove_key_from_stealable`` (:pr:`4609`) `jakirkham`_
+- Introduce events log length config option (:pr:`4615`) `Fabian Gebhart`_
+- Upstream config serialization and inheritance (:pr:`4372`) `Jacob Tomlinson`_
+- Add check to scheduler creation in ``SpecCluster`` (:pr:`4605`) `Jacob Tomlinson`_
+- Make length of events ``deque`` configurable (:pr:`4604`) `Fabian Gebhart`_
+- Add explicit ``fetch`` state to worker ``TaskState`` (:pr:`4470`) `Gil Forsyth`_
+- Update ``develop.rst`` (:pr:`4603`) `Florian Jetter`_
+- ``pickle_loads()``: Handle empty ``memoryview`` (:pr:`4595`) `Mads R. B. Kristensen`_
+- Switch documentation builds for PRs to readthedocs (:pr:`4599`) `James Bourbeau`_
+- Track frame sizes along with frames (:pr:`4593`) `jakirkham`_
+- Add support for a list of keys when using ``batch_size`` in ``client.map`` (:pr:`4592`) `Sultan Orazbayev`_
+- If ``SpecCluster`` fails to start attempt to gracefully close out again (:pr:`4590`) `Jacob Tomlinson`_
+- Multi-lock extension (:pr:`4503`) `Mads R. B. Kristensen`_
+- Update ``PipInstall`` plugin command (:pr:`4584`) `James Bourbeau`_
+- IPython magics: remove deprecated ``ioloop`` workarounds (:pr:`4530`) `Min RK`_
+- Add GitHub actions workflow to cancel duplicate builds (:pr:`4581`) `James Bourbeau`_
+- Remove outdated macOS build badge from ``README`` (:pr:`4576`) `James Bourbeau`_
+- Dask master -> main (:pr:`4569`) `Julia Signell`_
+- Drop support for Python 3.6 (:pr:`4390`) `James Bourbeau`_
+- Add docstring for ``dashboard_link`` property (:pr:`4572`) `Doug Davis`_
+- Change default branch from master to main (:pr:`4495`) `Julia Signell`_
+- Msgpack handles extract serialize (:pr:`4531`) `Mads R. B. Kristensen`_
+
+
 2021.03.0
 ---------
 
@@ -2228,4 +2265,8 @@ significantly without many new features.
 .. _`Bruno Pagani`: https://github.com/ArchangeGabriel
 .. _`selshowk`: https://github.com/selshowk
 .. _`Ray Bell`: https://github.com/raybellwaves
-.. _`Casey Clements`: https://github.com/caseyclements
\ No newline at end of file
+.. _`Casey Clements`: https://github.com/caseyclements
+.. _`Ben Greiner`: https://github.com/bnavigator
+.. _`Fabian Gebhart`: https://github.com/fgebhart
+.. _`Sultan Orazbayev`: https://github.com/SultanOrazbayev
+.. _`Doug Davis`: https://github.com/douglasdavis
\ No newline at end of file

From 892c673bec08716a2bb5a2688b4ceaec612060a5 Mon Sep 17 00:00:00 2001
From: James Lamb <jaylamb20@gmail.com>
Date: Mon, 29 Mar 2021 13:49:53 -0500
Subject: [PATCH 1204/1550] Expand documentation on pure functions (#4644)

---
 distributed/client.py  | 2 ++
 docs/source/client.rst | 9 +++++++--
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 625d1fb1040..462806261a4 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -1512,6 +1512,7 @@ def submit(
         pure : bool (defaults to True)
             Whether or not the function is pure.  Set ``pure=False`` for
             impure functions like ``np.random.random``.
+            See :ref:`pure functions` for more details.
         workers : string or iterable of strings
             A set of worker addresses or hostnames on which computations may be
             performed. Leave empty to default to all workers (common case)
@@ -1631,6 +1632,7 @@ def map(
         pure : bool (defaults to True)
             Whether or not the function is pure.  Set ``pure=False`` for
             impure functions like ``np.random.random``.
+            See :ref:`pure functions` for more details.
         workers : string or iterable of strings
             A set of worker hostnames on which computations may be performed.
             Leave empty to default to all workers (common case)
diff --git a/docs/source/client.rst b/docs/source/client.rst
index 444b651681e..df3b2e98be3 100644
--- a/docs/source/client.rst
+++ b/docs/source/client.rst
@@ -108,12 +108,17 @@ which are used for larger and smaller result sets respectively.
 
 For more information see the page on :doc:`Managing Computation <manage-computation>`.
 
+.. _pure functions:
 
 Pure Functions by Default
 -------------------------
 
-By default we assume that all functions are pure_.  If this is not the case we
-should use the ``pure=False`` keyword argument.
+By default, ``distributed`` assumes that all functions are pure_. Pure functions:
+
+* always return the same output for a given set of inputs
+* do not have side effects, like modifying global state or creating files
+
+If this is not the case, you should use the ``pure=False`` keyword argument in methods like ``Client.map()`` and ``Client.submit()``.
 
 The client associates a key to all computations.  This key is accessible on
 the Future object.

From 2c89ac32e54e27aeb17d0d304b95af570fd4b825 Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Mon, 29 Mar 2021 19:57:54 +0100
Subject: [PATCH 1205/1550] Replace conda with mamba (#4585)

---
 .github/workflows/tests.yaml | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
index 9d6b6a526c4..5b8eece7739 100644
--- a/.github/workflows/tests.yaml
+++ b/.github/workflows/tests.yaml
@@ -25,7 +25,11 @@ jobs:
       - name: Setup Conda Environment
         uses: conda-incubator/setup-miniconda@v2
         with:
-          miniconda-version: latest
+          miniforge-variant: Mambaforge
+          miniforge-version: latest
+          use-mamba: true
+          channels: conda-forge,defaults
+          channel-priority: true
           python-version: ${{ matrix.python-version }}
           environment-file: continuous_integration/environment-${{ matrix.python-version }}.yaml
           activate-environment: dask-distributed
@@ -35,7 +39,7 @@ jobs:
         shell: bash -l {0}
         # stacktrace for Python 3.8 has not been released at the moment of writing
         if: ${{ matrix.os == 'ubuntu-latest' && matrix.python-version < '3.8' }}
-        run: conda install -c conda-forge -c defaults -c numba libunwind stacktrace
+        run: mamba install -c conda-forge -c defaults -c numba libunwind stacktrace
 
       - name: Cythonize
         shell: bash -l {0}
@@ -46,15 +50,15 @@ jobs:
         shell: bash -l {0}
         run: python -m pip install --no-deps -e .
 
-      - name: conda list
+      - name: mamba list
         shell: bash -l {0}
-        run: conda list
+        run: mamba list
 
-      - name: conda env export
+      - name: mamba env export
         shell: bash -l {0}
         run: |
-          echo -e "--\n--Conda Environment (re-create this with \`conda env create --name <name> -f <output_file>\`)\n--"
-          conda env export | grep -E -v '^prefix:.*$'
+          echo -e "--\n--Conda Environment (re-create this with \`mamba env create --name <name> -f <output_file>\`)\n--"
+          mamba env export | grep -E -v '^prefix:.*$'
 
       - name: Setup SSH
         shell: bash -l {0}

From 43b20c22a6d5de5fc98db6f7c5129cf1c811b7a3 Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Mon, 29 Mar 2021 21:49:56 +0100
Subject: [PATCH 1206/1550] Suppress OSError on SpecCluster shutdown (#4567)

---
 distributed/deploy/spec.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index d9b1fe90e58..d16dd189bb9 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -414,7 +414,7 @@ async def _close(self):
             for future in self._futures:
                 await future
             async with self._lock:
-                with suppress(CommClosedError):
+                with suppress(CommClosedError, OSError):
                     if self.scheduler_comm:
                         await self.scheduler_comm.close(close_workers=True)
                     else:

From 93f6b58e328b12641069776dcf9a0d194c0ae2bd Mon Sep 17 00:00:00 2001
From: Hristo Georgiev <hristog@users.noreply.github.com>
Date: Tue, 30 Mar 2021 00:32:18 +0100
Subject: [PATCH 1207/1550] Update out-of-date references to `config.yaml`
 (#4643)

---
 distributed/client.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 462806261a4..f327f1df890 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -3497,7 +3497,8 @@ def get_scheduler_logs(self, n=None):
         ----------
         n : int
             Number of logs to retrive.  Maxes out at 10000 by default,
-            confiruable in config.yaml::log-length
+            configurable via the ``distributed.admin.log-length``
+            configuration value.
 
         Returns
         -------
@@ -3512,7 +3513,8 @@ def get_worker_logs(self, n=None, workers=None, nanny=False):
         ----------
         n : int
             Number of logs to retrive.  Maxes out at 10000 by default,
-            confiruable in config.yaml::log-length
+            configurable via the ``distributed.admin.log-length``
+            configuration value.
         workers : iterable
             List of worker addresses to retrieve.  Gets all workers by default.
         nanny : bool, default False

From 77a1fd19a17b1e70d412db9134316ccb154eaead Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Tue, 30 Mar 2021 03:03:25 +0100
Subject: [PATCH 1208/1550] scheduler.py / worker.py code cleanup (#4626)

---
 distributed/scheduler.py | 107 +++++++++++++----------------
 distributed/worker.py    | 145 +++++++++++++++++++--------------------
 2 files changed, 120 insertions(+), 132 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 190bc7c30c9..4affb556bdf 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -3604,29 +3604,27 @@ async def close_worker(self, comm=None, worker=None, safe=None):
     def heartbeat_worker(
         self,
         comm=None,
-        address=None,
-        resolve_address=True,
-        now=None,
-        resources=None,
-        host_info=None,
-        metrics=None,
-        executing=None,
+        *,
+        address,
+        resolve_address: bool = True,
+        now: float = None,
+        resources: dict = None,
+        host_info: dict = None,
+        metrics: dict,
+        executing: dict = None,
     ):
         parent: SchedulerState = cast(SchedulerState, self)
         address = self.coerce_address(address, resolve_address)
         address = normalize_address(address)
-        if address not in parent._workers:
+        ws: WorkerState = parent._workers.get(address)
+        if ws is None:
             return {"status": "missing"}
 
         host = get_address_host(address)
         local_now = time()
-        now = now or time()
-        assert metrics
         host_info = host_info or {}
 
-        dh: dict = parent._host_info.get(host)
-        if dh is None:
-            parent._host_info[host] = dh = dict()
+        dh: dict = parent._host_info.setdefault(host, {})
         dh["last-seen"] = local_now
 
         frac = 1 / len(parent._workers)
@@ -3650,26 +3648,20 @@ def heartbeat_worker(
                     1 - alpha
                 )
 
-        ws: WorkerState = parent._workers[address]
-
-        ws._last_seen = time()
-
+        ws._last_seen = local_now
         if executing is not None:
             ws._executing = {
                 parent._tasks[key]: duration for key, duration in executing.items()
             }
 
-        if metrics:
-            ws._metrics = metrics
+        ws._metrics = metrics
 
         if host_info:
-            dh: dict = parent._host_info.get(host)
-            if dh is None:
-                parent._host_info[host] = dh = dict()
+            dh: dict = parent._host_info.setdefault(host, {})
             dh.update(host_info)
 
-        delay = time() - now
-        ws._time_delay = delay
+        if now:
+            ws._time_delay = local_now - now
 
         if resources:
             self.add_resources(worker=address, resources=resources)
@@ -3678,7 +3670,7 @@ def heartbeat_worker(
 
         return {
             "status": "OK",
-            "time": time(),
+            "time": local_now,
             "heartbeat-interval": heartbeat_interval(len(parent._workers)),
         }
 
@@ -3756,7 +3748,7 @@ async def add_worker(
             parent._total_nthreads += nthreads
             parent._aliases[name] = address
 
-            response = self.heartbeat_worker(
+            self.heartbeat_worker(
                 address=address,
                 resolve_address=resolve_address,
                 now=now,
@@ -5331,7 +5323,7 @@ async def rebalance(self, comm=None, keys=None, workers=None):
                     map(first, sorted(worker_bytes.items(), key=second, reverse=True))
                 )
 
-                recipients = iter(reversed(sorted_workers))
+                recipients = reversed(sorted_workers)
                 recipient = next(recipients)
                 msgs = []  # (sender, recipient, key)
                 for sender in sorted_workers[: len(workers) // 2]:
@@ -5343,11 +5335,8 @@ async def rebalance(self, comm=None, keys=None, workers=None):
                     )
 
                     try:
-                        while worker_bytes[sender] > avg:
-                            while (
-                                worker_bytes[recipient] < avg
-                                and worker_bytes[sender] > avg
-                            ):
+                        while avg < worker_bytes[sender]:
+                            while worker_bytes[recipient] < avg < worker_bytes[sender]:
                                 ts, nb = next(sender_keys)
                                 if ts not in tasks_by_worker[recipient]:
                                     tasks_by_worker[recipient].add(ts)
@@ -5355,7 +5344,7 @@ async def rebalance(self, comm=None, keys=None, workers=None):
                                     msgs.append((sender, recipient, ts))
                                     worker_bytes[sender] -= nb
                                     worker_bytes[recipient] += nb
-                            if worker_bytes[sender] > avg:
+                            if avg < worker_bytes[sender]:
                                 recipient = next(recipients)
                     except StopIteration:
                         break
@@ -5386,7 +5375,7 @@ async def rebalance(self, comm=None, keys=None, workers=None):
                     },
                 )
 
-                if not all(r["status"] == "OK" for r in result):
+                if any(r["status"] != "OK" for r in result):
                     return {
                         "status": "missing-data",
                         "keys": tuple(
@@ -5687,7 +5676,7 @@ async def retire_workers(
         workers: list (optional)
             List of worker addresses to retire.
             If not provided we call ``workers_to_close`` which finds a good set
-        workers_names: list (optional)
+        names: list (optional)
             List of worker names to retire.
         remove: bool (defaults to True)
             Whether or not to remove the worker metadata immediately or else
@@ -5715,30 +5704,31 @@ async def retire_workers(
         with log_errors():
             async with self._lock if lock else empty_context:
                 if names is not None:
+                    if workers is not None:
+                        raise TypeError("names and workers are mutually exclusive")
                     if names:
                         logger.info("Retire worker names %s", names)
                     names = set(map(str, names))
-                    workers = [
+                    workers = {
                         ws._address
                         for ws in parent._workers_dv.values()
                         if str(ws._name) in names
-                    ]
-                if workers is None:
+                    }
+                elif workers is None:
                     while True:
                         try:
                             workers = self.workers_to_close(**kwargs)
-                            if workers:
-                                workers = await self.retire_workers(
-                                    workers=workers,
-                                    remove=remove,
-                                    close_workers=close_workers,
-                                    lock=False,
-                                )
-                                return workers
-                            else:
+                            if not workers:
                                 return {}
+                            return await self.retire_workers(
+                                workers=workers,
+                                remove=remove,
+                                close_workers=close_workers,
+                                lock=False,
+                            )
                         except KeyError:  # keys left during replicate
                             pass
+
                 workers = {
                     parent._workers_dv[w] for w in workers if w in parent._workers_dv
                 }
@@ -5750,22 +5740,21 @@ async def retire_workers(
                 keys = set.union(*[w.has_what for w in workers])
                 keys = {ts._key for ts in keys if ts._who_has.issubset(workers)}
 
-                other_workers = set(parent._workers_dv.values()) - workers
                 if keys:
-                    if other_workers:
-                        logger.info("Moving %d keys to other workers", len(keys))
-                        await self.replicate(
-                            keys=keys,
-                            workers=[ws._address for ws in other_workers],
-                            n=1,
-                            delete=False,
-                            lock=False,
-                        )
-                    else:
+                    other_workers = set(parent._workers_dv.values()) - workers
+                    if not other_workers:
                         return {}
+                    logger.info("Moving %d keys to other workers", len(keys))
+                    await self.replicate(
+                        keys=keys,
+                        workers=[ws._address for ws in other_workers],
+                        n=1,
+                        delete=False,
+                        lock=False,
+                    )
 
                 worker_keys = {ws._address: ws.identity() for ws in workers}
-                if close_workers and worker_keys:
+                if close_workers:
                     await asyncio.gather(
                         *[self.close_worker(worker=w, safe=True) for w in worker_keys]
                     )
diff --git a/distributed/worker.py b/distributed/worker.py
index 276b41fd853..f448f9730e1 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -25,7 +25,7 @@
 from dask.utils import format_bytes, funcname
 from dask.system import CPU_COUNT
 
-from tlz import pluck, merge, first, keymap
+from tlz import pluck, first, keymap
 from tornado import gen
 from tornado.ioloop import IOLoop, PeriodicCallback
 
@@ -806,8 +806,7 @@ def local_dir(self):
         return self.local_directory
 
     async def get_metrics(self):
-        now = time()
-        core = dict(
+        out = dict(
             executing=self.executing_count,
             in_memory=len(self.data),
             ready=len(self.ready),
@@ -818,17 +817,19 @@ async def get_metrics(self):
                 "types": keymap(typename, self.bandwidth_types),
             },
         )
-        custom = {}
+        out.update(self.monitor.recent())
+
         for k, metric in self.metrics.items():
             try:
                 result = metric(self)
                 if isawaitable(result):
                     result = await result
-                custom[k] = result
+                # In case of collision, prefer core metrics
+                out.setdefault(k, result)
             except Exception:  # TODO: log error once
                 pass
 
-        return merge(custom, self.monitor.recent(), core)
+        return out
 
     async def get_startup_information(self):
         result = {}
@@ -934,56 +935,57 @@ def _update_latency(self, latency):
             self.digests["latency"].add(latency)
 
     async def heartbeat(self):
-        if not self.heartbeat_active:
-            self.heartbeat_active = True
-            logger.debug("Heartbeat: %s" % self.address)
-            try:
-                start = time()
-                response = await retry_operation(
-                    self.scheduler.heartbeat_worker,
-                    address=self.contact_address,
-                    now=time(),
-                    metrics=await self.get_metrics(),
-                    executing={
-                        key: start - self.tasks[key].start_time
-                        for key in self.active_threads.values()
-                        if key in self.tasks
-                    },
-                )
-                end = time()
-                middle = (start + end) / 2
+        if self.heartbeat_active:
+            logger.debug("Heartbeat skipped: channel busy")
+            return
 
-                self._update_latency(end - start)
+        self.heartbeat_active = True
+        logger.debug("Heartbeat: %s", self.address)
+        try:
+            start = time()
+            response = await retry_operation(
+                self.scheduler.heartbeat_worker,
+                address=self.contact_address,
+                now=start,
+                metrics=await self.get_metrics(),
+                executing={
+                    key: start - self.tasks[key].start_time
+                    for key in self.active_threads.values()
+                    if key in self.tasks
+                },
+            )
+            end = time()
+            middle = (start + end) / 2
+
+            self._update_latency(end - start)
 
-                if response["status"] == "missing":
-                    for i in range(10):
-                        if self.status != Status.running:
-                            break
-                        else:
-                            await asyncio.sleep(0.05)
+            if response["status"] == "missing":
+                for i in range(10):
+                    if self.status != Status.running:
+                        break
                     else:
-                        await self._register_with_scheduler()
-                    return
-                self.scheduler_delay = response["time"] - middle
-                self.periodic_callbacks["heartbeat"].callback_time = (
-                    response["heartbeat-interval"] * 1000
-                )
-                self.bandwidth_workers.clear()
-                self.bandwidth_types.clear()
-            except CommClosedError:
-                logger.warning("Heartbeat to scheduler failed")
-                if not self.reconnect:
-                    await self.close(report=False)
-            except IOError as e:
-                # Scheduler is gone. Respect distributed.comm.timeouts.connect
-                if "Timed out trying to connect" in str(e):
-                    await self.close(report=False)
+                        await asyncio.sleep(0.05)
                 else:
-                    raise e
-            finally:
-                self.heartbeat_active = False
-        else:
-            logger.debug("Heartbeat skipped: channel busy")
+                    await self._register_with_scheduler()
+                return
+            self.scheduler_delay = response["time"] - middle
+            self.periodic_callbacks["heartbeat"].callback_time = (
+                response["heartbeat-interval"] * 1000
+            )
+            self.bandwidth_workers.clear()
+            self.bandwidth_types.clear()
+        except CommClosedError:
+            logger.warning("Heartbeat to scheduler failed")
+            if not self.reconnect:
+                await self.close(report=False)
+        except IOError as e:
+            # Scheduler is gone. Respect distributed.comm.timeouts.connect
+            if "Timed out trying to connect" in str(e):
+                await self.close(report=False)
+            else:
+                raise e
+        finally:
+            self.heartbeat_active = False
 
     async def handle_scheduler(self, comm):
         try:
@@ -2759,26 +2761,25 @@ async def execute(self, key, report=False):
                 self.transition(ts, "memory", value=value)
                 if self.digests is not None:
                     self.digests["task-duration"].add(result["stop"] - result["start"])
+            elif isinstance(result.pop("actual-exception"), Reschedule):
+                self.batched_stream.send({"op": "reschedule", "key": ts.key})
+                self.transition(ts, "rescheduled", report=False)
+                self.release_key(ts.key, report=False)
             else:
-                if isinstance(result.pop("actual-exception"), Reschedule):
-                    self.batched_stream.send({"op": "reschedule", "key": ts.key})
-                    self.transition(ts, "rescheduled", report=False)
-                    self.release_key(ts.key, report=False)
-                else:
-                    ts.exception = result["exception"]
-                    ts.traceback = result["traceback"]
-                    logger.warning(
-                        " Compute Failed\n"
-                        "Function:  %s\n"
-                        "args:      %s\n"
-                        "kwargs:    %s\n"
-                        "Exception: %s\n",
-                        str(funcname(function))[:1000],
-                        convert_args_to_str(args2, max_len=1000),
-                        convert_kwargs_to_str(kwargs2, max_len=1000),
-                        repr(result["exception"].data),
-                    )
-                    self.transition(ts, "error")
+                ts.exception = result["exception"]
+                ts.traceback = result["traceback"]
+                logger.warning(
+                    "Compute Failed\n"
+                    "Function:  %s\n"
+                    "args:      %s\n"
+                    "kwargs:    %s\n"
+                    "Exception: %r\n",
+                    str(funcname(function))[:1000],
+                    convert_args_to_str(args2, max_len=1000),
+                    convert_kwargs_to_str(kwargs2, max_len=1000),
+                    result["exception"].data,
+                )
+                self.transition(ts, "error")
 
             logger.debug("Send compute response to scheduler: %s, %s", ts.key, result)
 
@@ -3365,8 +3366,6 @@ class Reschedule(Exception):
     the task.
     """
 
-    pass
-
 
 def parse_memory_limit(memory_limit, nthreads, total_cores=CPU_COUNT):
     if memory_limit is None:

From e8c8e351c38c429bbdd53be72c26b9f8141eea97 Mon Sep 17 00:00:00 2001
From: Marcos Moyano <marcos@anue.biz>
Date: Tue, 30 Mar 2021 10:38:50 -0300
Subject: [PATCH 1209/1550] Support Websocket communication protocols (#4396)

This PR adds a websocket Comm.

This allows for Dask to communicate over normal HTTP/HTTPS ports, which is useful in some network constrained situations.

Additionally, this also allows Dask to serve both the dashboard and computation on a single port

```python
from dask.distributed import Client
client = Client(protocol="ws")

>>> client
<Client: 'ws://192.168.0.2:8787' processes=4 threads=12, memory=16.44 GB>
```
---
 distributed/cfexecutor.py         |   3 +-
 distributed/comm/__init__.py      |   1 +
 distributed/comm/tests/test_ws.py | 188 ++++++++++++++
 distributed/comm/ws.py            | 409 ++++++++++++++++++++++++++++++
 4 files changed, 600 insertions(+), 1 deletion(-)
 create mode 100644 distributed/comm/tests/test_ws.py
 create mode 100644 distributed/comm/ws.py

diff --git a/distributed/cfexecutor.py b/distributed/cfexecutor.py
index e11c96c2821..4096181581f 100644
--- a/distributed/cfexecutor.py
+++ b/distributed/cfexecutor.py
@@ -6,7 +6,7 @@
 from tornado import gen
 
 from .metrics import time
-from .utils import sync, TimeoutError
+from .utils import sync, TimeoutError, parse_timedelta
 
 
 @gen.coroutine
@@ -118,6 +118,7 @@ def map(self, fn, *iterables, **kwargs):
         """
         timeout = kwargs.pop("timeout", None)
         if timeout is not None:
+            timeout = parse_timedelta(timeout)
             end_time = timeout + time()
         if "chunksize" in kwargs:
             del kwargs["chunksize"]
diff --git a/distributed/comm/__init__.py b/distributed/comm/__init__.py
index 2ff679ada3d..7eb820d3f54 100644
--- a/distributed/comm/__init__.py
+++ b/distributed/comm/__init__.py
@@ -16,6 +16,7 @@
 def _register_transports():
     from . import inproc
     from . import tcp
+    from . import ws
 
     try:
         from . import ucx
diff --git a/distributed/comm/tests/test_ws.py b/distributed/comm/tests/test_ws.py
new file mode 100644
index 00000000000..1d4a5222b2e
--- /dev/null
+++ b/distributed/comm/tests/test_ws.py
@@ -0,0 +1,188 @@
+import os
+import warnings
+import pytest
+import tempfile
+
+import numpy as np
+import dask
+from distributed import Client, Scheduler, Worker
+from distributed.security import Security
+from distributed.comm.registry import backends, get_backend
+from distributed.comm import ws, listen, connect
+from distributed.utils_test import (  # noqa: F401
+    get_client_ssl_context,
+    get_server_ssl_context,
+    cleanup,
+    inc,
+    gen_cluster,
+)
+
+from .test_comms import check_tls_extra
+
+security = Security.temporary()
+
+
+def test_registered():
+    assert "ws" in backends
+    backend = get_backend("ws")
+    assert isinstance(backend, ws.WSBackend)
+
+
+@pytest.mark.asyncio
+async def test_listen_connect(cleanup):
+    async def handle_comm(comm):
+        while True:
+            msg = await comm.read()
+            await comm.write(msg)
+
+    async with listen("ws://", handle_comm) as listener:
+        comm = await connect(listener.contact_address)
+        await comm.write(b"Hello!")
+        result = await comm.read()
+        assert result == b"Hello!"
+
+        await comm.close()
+
+
+@pytest.mark.asyncio
+async def test_listen_connect_wss(cleanup):
+    async def handle_comm(comm):
+        while True:
+            msg = await comm.read()
+            await comm.write(msg)
+
+    server_ctx = get_server_ssl_context()
+    client_ctx = get_client_ssl_context()
+
+    async with listen("wss://", handle_comm, ssl_context=server_ctx) as listener:
+        comm = await connect(listener.contact_address, ssl_context=client_ctx)
+        assert comm.peer_address.startswith("wss://")
+        check_tls_extra(comm.extra_info)
+        await comm.write(b"Hello!")
+        result = await comm.read()
+        assert result == b"Hello!"
+        await comm.close()
+
+
+@pytest.mark.asyncio
+async def test_expect_ssl_context(cleanup):
+    server_ctx = get_server_ssl_context()
+
+    async with listen("wss://", lambda comm: comm, ssl_context=server_ctx) as listener:
+        with pytest.raises(TypeError):
+            comm = await connect(listener.contact_address)
+
+
+@pytest.mark.asyncio
+async def test_expect_scheduler_ssl_when_sharing_server(cleanup):
+    with tempfile.TemporaryDirectory() as tempdir:
+        key_path = os.path.join(tempdir, "dask.pem")
+        cert_path = os.path.join(tempdir, "dask.crt")
+        with open(key_path, "w") as f:
+            f.write(security.tls_scheduler_key)
+        with open(cert_path, "w") as f:
+            f.write(security.tls_scheduler_cert)
+        c = {
+            "distributed.scheduler.dashboard.tls.key": key_path,
+            "distributed.scheduler.dashboard.tls.cert": cert_path,
+        }
+        with dask.config.set(c):
+            with pytest.raises(RuntimeError):
+                async with Scheduler(protocol="ws://", dashboard=True, port=8787) as s:
+                    pass
+
+
+@pytest.mark.asyncio
+async def test_roundtrip(cleanup):
+    async with Scheduler(protocol="ws://") as s:
+        async with Worker(s.address) as w:
+            async with Client(s.address, asynchronous=True) as c:
+                assert c.scheduler.address.startswith("ws://")
+                assert w.address.startswith("ws://")
+                future = c.submit(inc, 1)
+                result = await future
+                assert result == 2
+
+
+@pytest.mark.asyncio
+async def test_collections(cleanup):
+    da = pytest.importorskip("dask.array")
+    async with Scheduler(protocol="ws://") as s:
+        async with Worker(s.address) as a:
+            async with Worker(s.address) as b:
+                async with Client(s.address, asynchronous=True) as c:
+                    x = da.random.random((1000, 1000), chunks=(100, 100))
+                    x = x + x.T
+                    await x.persist()
+
+
+@pytest.mark.asyncio
+async def test_large_transfer(cleanup):
+    np = pytest.importorskip("numpy")
+    async with Scheduler(protocol="ws://") as s:
+        async with Worker(s.address, protocol="ws://") as w:
+            async with Client(s.address, asynchronous=True) as c:
+                future = await c.scatter(np.random.random(1000000))
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "dashboard,protocol,security,port",
+    [
+        (True, "ws://", None, 8787),
+        (True, "wss://", security, 8787),
+        (False, "ws://", None, 8787),
+        (False, "wss://", security, 8787),
+        (True, "ws://", None, 8786),
+        (True, "wss://", security, 8786),
+        (False, "ws://", None, 8786),
+        (False, "wss://", security, 8786),
+    ],
+)
+async def test_http_and_comm_server(cleanup, dashboard, protocol, security, port):
+    async with Scheduler(
+        protocol=protocol, dashboard=dashboard, port=port, security=security
+    ) as s:
+        if port == 8787:
+            assert s.http_server is s.listener.server
+        else:
+            assert s.http_server is not s.listener.server
+        async with Worker(s.address, protocol=protocol, security=security) as w:
+            async with Client(s.address, asynchronous=True, security=security) as c:
+                result = await c.submit(lambda x: x + 1, 10)
+                assert result == 11
+
+
+@pytest.mark.asyncio
+async def test_quiet_close(cleanup):
+    with warnings.catch_warnings(record=True) as record:
+        async with Client(protocol="ws", processes=False, asynchronous=True) as c:
+            pass
+
+    # For some reason unrelated @coroutine warnings are showing up
+    record = [warning for warning in record if "coroutine" not in str(warning.message)]
+
+    assert not record, record[0].message
+
+
+@gen_cluster(
+    client=True,
+    scheduler_kwargs={"protocol": "ws://"},
+)
+async def test_ws_roundtrip(c, s, a, b):
+    x = np.arange(100)
+    future = await c.scatter(x)
+    y = await future
+    assert (x == y).all()
+
+
+@gen_cluster(
+    client=True,
+    security=security,
+    scheduler_kwargs={"protocol": "wss://"},
+)
+async def test_wss_roundtrip(c, s, a, b):
+    x = np.arange(100)
+    future = await c.scatter(x)
+    y = await future
+    assert (x == y).all()
diff --git a/distributed/comm/ws.py b/distributed/comm/ws.py
new file mode 100644
index 00000000000..ac76df3c5d4
--- /dev/null
+++ b/distributed/comm/ws.py
@@ -0,0 +1,409 @@
+import asyncio
+import logging
+import struct
+import weakref
+import warnings
+
+from typing import Callable
+from ssl import SSLError
+
+from tornado.iostream import StreamClosedError
+from tornado.httpserver import HTTPServer
+from tornado.httpclient import HTTPRequest
+from tornado import web
+from tornado.websocket import WebSocketClosedError, websocket_connect, WebSocketHandler
+
+from ..utils import ensure_bytes, nbytes
+
+from .registry import backends
+from .addressing import parse_host_port, unparse_host_port
+from .core import Comm, CommClosedError, Listener, Connector, FatalCommClosedError
+from .tcp import BaseTCPBackend, _expect_tls_context, convert_stream_closed_error
+from .utils import ensure_concrete_host, to_frames, from_frames, get_tcp_server_address
+
+
+logger = logging.getLogger(__name__)
+
+
+class WSHandler(WebSocketHandler):
+    def __init__(
+        self,
+        application,
+        request,
+        handler=None,
+        deserialize: bool = True,
+        allow_offload: bool = True,
+        listener=None,
+        **kwargs,
+    ):
+        self.handler = handler
+        self.deserialize = deserialize
+        self.allow_offload = allow_offload
+        self.request = request
+        self.listener = listener()
+        super().__init__(application, request, **kwargs)
+
+    async def open(self):
+        self.set_nodelay(True)
+        self.q = asyncio.Queue()
+        self.closed = False
+        self.comm = WSHandlerComm(
+            self, deserialize=self.deserialize, allow_offload=self.allow_offload
+        )
+        # `on_message` won't get called until `open` returns
+        # we need `open` to return to finish the handshake
+        asyncio.ensure_future(self.on_open())
+
+    async def on_open(self):
+        try:
+            await self.listener.on_connection(self.comm)
+        except CommClosedError:
+            logger.debug("Connection closed before handshake completed")
+        await self.handler(self.comm)
+
+    async def on_message(self, msg):
+        await self.q.put(msg)
+
+    def on_close(self):
+        self.closed = True
+        self.q.put_nowait(CommClosedError)
+
+    def close(self):
+        super().close()
+        self.closed = True
+
+
+class WSHandlerComm(Comm):
+    def __init__(self, handler, deserialize=True, allow_offload=True):
+        self.handler = handler
+        self.deserialize = deserialize
+        self.allow_offload = allow_offload
+        super().__init__()
+
+    async def read(self, deserializers=None):
+        try:
+            n_frames = await self.handler.q.get()
+        except RuntimeError:  # Event loop is closed
+            raise CommClosedError()
+
+        if n_frames is CommClosedError:
+            raise CommClosedError()
+        else:
+            n_frames = struct.unpack("Q", n_frames)[0]
+        frames = [(await self.handler.q.get()) for _ in range(n_frames)]
+        return await from_frames(
+            frames,
+            deserialize=self.deserialize,
+            deserializers=deserializers,
+            allow_offload=self.allow_offload,
+        )
+
+    async def write(self, msg, serializers=None, on_error=None):
+        frames = await to_frames(
+            msg,
+            allow_offload=self.allow_offload,
+            serializers=serializers,
+            on_error=on_error,
+            context={
+                "sender": self.local_info,
+                "recipient": self.remote_info,
+                **self.handshake_options,
+            },
+        )
+        n = struct.pack("Q", len(frames))
+        try:
+            await self.handler.write_message(n, binary=True)
+            for frame in frames:
+                await self.handler.write_message(ensure_bytes(frame), binary=True)
+        except WebSocketClosedError as e:
+            raise CommClosedError(str(e))
+
+        return sum(map(nbytes, frames))
+
+    def abort(self):
+        self.handler.close()
+
+    @property
+    def local_address(self):
+        return self.handler.request.host
+
+    @property
+    def peer_address(self):
+        return self.handler.request.remote_ip + ":0"
+
+    @property
+    def _local_addr(self):
+        return self.handler.request.host
+
+    @property
+    def _peer_addr(self):
+        return self.handler.request.remote_ip + ":0"
+
+    def closed(self):
+        return (
+            self.handler.closed
+            or not self.handler.ws_connection
+            or self.handler.request.connection.stream
+            and self.handler.request.connection.stream.closed
+        )
+
+    async def close(self):
+        self.handler.close()
+
+
+class WS(Comm):
+    prefix = "ws://"
+
+    def __init__(self, sock, deserialize=True, allow_offload=True):
+        self._closed = False
+        Comm.__init__(self)
+        self.sock = sock
+        self._peer_addr = f"{self.prefix}{self.sock.parsed.netloc}"
+        self._local_addr = f"{self.prefix}{self.sock.parsed.netloc}"
+        self.deserialize = deserialize
+        self.allow_offload = allow_offload
+        self._finalizer = weakref.finalize(self, self._get_finalizer())
+        self._extra = {}
+        self._read_extra()
+
+    def _get_finalizer(self):
+        def finalize(sock=self.sock, r=repr(self)):
+            if not sock.close_code:
+                logger.info("Closing dangling websocket in %s", r)
+                sock.close()
+
+        return finalize
+
+    async def read(self, deserializers=None):
+        try:
+            n_frames = await self.sock.read_message()
+            if n_frames is None:
+                # Connection is closed
+                self.abort()
+                raise CommClosedError()
+            n_frames = struct.unpack("Q", n_frames)[0]
+        except WebSocketClosedError as e:
+            raise CommClosedError(e)
+
+        frames = [(await self.sock.read_message()) for _ in range(n_frames)]
+
+        msg = await from_frames(
+            frames,
+            deserialize=self.deserialize,
+            deserializers=deserializers,
+            allow_offload=self.allow_offload,
+        )
+        return msg
+
+    async def write(self, msg, serializers=None, on_error=None):
+        frames = await to_frames(
+            msg,
+            allow_offload=self.allow_offload,
+            serializers=serializers,
+            on_error=on_error,
+            context={
+                "sender": self.local_info,
+                "recipient": self.remote_info,
+                **self.handshake_options,
+            },
+        )
+        n = struct.pack("Q", len(frames))
+        try:
+            await self.sock.write_message(n, binary=True)
+            for frame in frames:
+                await self.sock.write_message(ensure_bytes(frame), binary=True)
+        except WebSocketClosedError as e:
+            raise CommClosedError(e)
+
+        return sum(map(nbytes, frames))
+
+    async def close(self):
+        if not self.sock.close_code:
+            self._finalizer.detach()
+            self.sock.close()
+        self._closed = True
+
+    def abort(self):
+        if not self.sock.close_code:
+            self._finalizer.detach()
+            self.sock.close()
+        self._closed = True
+
+    def closed(self):
+        return not self.sock or self.sock.close_code or self._closed
+
+    @property
+    def local_address(self):
+        return f"{self.prefix}{self.sock.parsed.netloc}"
+
+    @property
+    def peer_address(self):
+        return f"{self.prefix}{self.sock.parsed.netloc}"
+
+    def _read_extra(self):
+        pass
+
+    @property
+    def extra_info(self):
+        return self._extra
+
+
+class WSS(WS):
+    prefix = "wss://"
+
+    def _read_extra(self):
+        WS._read_extra(self)
+        sock = self.sock.stream.socket
+        if sock is not None:
+            self._extra.update(peercert=sock.getpeercert(), cipher=sock.cipher())
+            cipher, proto, bits = self._extra["cipher"]
+            logger.debug(
+                "TLS connection with %r: protocol=%s, cipher=%s, bits=%d",
+                self._peer_addr,
+                proto,
+                cipher,
+                bits,
+            )
+
+
+class WSListener(Listener):
+    prefix = "ws://"
+
+    def __init__(
+        self,
+        address: str,
+        handler: Callable,
+        deserialize=True,
+        allow_offload=False,
+        **connection_args,
+    ):
+        if not address.startswith(self.prefix):
+            address = f"{self.prefix}{address}"
+
+        self.ip, self.port = parse_host_port(address, default_port=0)
+        self.handler = handler
+        self.deserialize = deserialize
+        self.allow_offload = allow_offload
+        self.connection_args = connection_args
+        self.bound_address = None
+        self.new_comm_server = True
+        self.server_args = self._get_server_args(**connection_args)
+
+    def _get_server_args(self, **connection_args):
+        return {}
+
+    @property
+    def address(self) -> str:
+        return f"{self.prefix}{self.ip}:{self.port}"
+
+    async def start(self):
+        routes = [
+            (
+                r"/",
+                WSHandler,
+                {
+                    "handler": self.handler,
+                    "deserialize": self.deserialize,
+                    "allow_offload": self.allow_offload,
+                    "listener": weakref.ref(self),
+                },
+            )
+        ]
+        try:
+            self.server = self.handler.__self__.http_server
+            if self.server.port == self.port:
+                self.new_comm_server = False
+                logger.debug(f"Sharing the same server on port {self.port}")
+                ssl_options = self.server_args.get("ssl_options")
+                if self.server.ssl_options and ssl_options is None:
+                    raise RuntimeError("No ssl context found for the Scheduler")
+                if ssl_options:
+                    warnings.warn(
+                        "Dashboard and Scheduler are using "
+                        f"the same server on port {self.port}, "
+                        "defaulting to the Scheduler's ssl context. "
+                        "Your dashboard could become inaccessible",
+                        RuntimeWarning,
+                    )
+                    self.server.ssl_options = ssl_options
+                self.handler.__self__.http_application.add_handlers(r".*", routes)
+        except AttributeError:
+            logger.debug("No server available. Creating a new one")
+        finally:
+            if self.new_comm_server:
+                self.server = HTTPServer(web.Application(routes), **self.server_args)
+                self.server.listen(self.port)
+
+    async def stop(self):
+        self.server.stop()
+
+    def get_host_port(self):
+        """
+        The listening address as a (host, port) tuple.
+        """
+        if self.bound_address is None:
+            self.bound_address = get_tcp_server_address(self.server)
+        # IPv6 getsockname() can return more a 4-len tuple
+        return self.bound_address[:2]
+
+    @property
+    def listen_address(self) -> str:
+        return self.prefix + unparse_host_port(*self.get_host_port())
+
+    @property
+    def contact_address(self) -> str:
+        host, port = self.get_host_port()
+        host = ensure_concrete_host(host)
+        return self.prefix + unparse_host_port(host, port)
+
+
+class WSSListener(WSListener):
+    prefix = "wss://"
+
+    def _get_server_args(self, **connection_args):
+        ctx = _expect_tls_context(connection_args)
+        return {"ssl_options": ctx}
+
+
+class WSConnector(Connector):
+    prefix = "ws://"
+    comm_class = WS
+
+    async def connect(self, address, deserialize=True, **connection_args):
+        kwargs = self._get_connect_args(**connection_args)
+        try:
+            request = HTTPRequest(f"{self.prefix}{address}", **kwargs)
+            sock = await websocket_connect(request, max_message_size=10_000_000_000)
+            if sock.stream.closed() and sock.stream.error:
+                raise StreamClosedError(sock.stream.error)
+        except StreamClosedError as e:
+            convert_stream_closed_error(self, e)
+        except SSLError as err:
+            raise FatalCommClosedError() from err
+        return self.comm_class(sock, deserialize=deserialize)
+
+    def _get_connect_args(self, **connection_args):
+        return {}
+
+
+class WSSConnector(WSConnector):
+    prefix = "wss://"
+    comm_class = WSS
+
+    def _get_connect_args(self, **connection_args):
+        ctx = _expect_tls_context(connection_args)
+        return {"ssl_options": ctx}
+
+
+class WSBackend(BaseTCPBackend):
+    _connector_class = WSConnector
+    _listener_class = WSListener
+
+
+class WSSBackend(BaseTCPBackend):
+    _connector_class = WSSConnector
+    _listener_class = WSSListener
+
+
+backends["ws"] = WSBackend()
+backends["wss"] = WSSBackend()

From bb067a16de8659a783ebfa1f35aabc978216fdb8 Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Wed, 31 Mar 2021 00:06:10 +0100
Subject: [PATCH 1210/1550] Use powers-of-two when displaying RAM (#4649)

---
 distributed/dashboard/components/scheduler.py | 4 ++--
 distributed/deploy/tests/test_local.py        | 2 +-
 distributed/deploy/tests/test_spec_cluster.py | 5 +++--
 distributed/tests/test_client.py              | 7 +++----
 4 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index 49032665150..2fd68e8d6f6 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -1817,8 +1817,8 @@ def __init__(self, scheduler, width=800, **kwargs):
         formatters = {
             "cpu": NumberFormatter(format="0.0 %"),
             "memory_percent": NumberFormatter(format="0.0 %"),
-            "memory": NumberFormatter(format="0 b"),
-            "memory_limit": NumberFormatter(format="0 b"),
+            "memory": NumberFormatter(format="0.00 b"),
+            "memory_limit": NumberFormatter(format="0.00 b"),
             "read_bytes": NumberFormatter(format="0 b"),
             "write_bytes": NumberFormatter(format="0 b"),
             "num_fds": NumberFormatter(format="0"),
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index 8eaea2ba83f..6e24d77f120 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -979,7 +979,7 @@ async def test_repr(cleanup):
         assert "workers=2" in text
         assert cluster.scheduler_address in text
         assert "cores=4" in text or "threads=4" in text
-        assert "GB" in text and "4" in text
+        assert "4.00 GB" in text or "3.73 GiB" in text
 
     async with LocalCluster(
         n_workers=2, processes=False, memory_limit=None, asynchronous=True
diff --git a/distributed/deploy/tests/test_spec_cluster.py b/distributed/deploy/tests/test_spec_cluster.py
index afdc10cbe04..41e9e9158f9 100644
--- a/distributed/deploy/tests/test_spec_cluster.py
+++ b/distributed/deploy/tests/test_spec_cluster.py
@@ -361,8 +361,9 @@ async def test_widget(cleanup):
             await asyncio.sleep(0.01)
             assert time() < start + 1
 
-        assert "3" in cluster._widget_status()
-        assert "GB" in cluster._widget_status()
+        text = cluster._widget_status()
+        assert "3" in text
+        assert "GB" in text or "GiB" in text
 
         cluster.scale(5)
         assert "3 / 5" in cluster._widget_status()
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index b36b7fbf3a4..9a4ea4e188c 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -1989,9 +1989,8 @@ def test_repr(loop):
             for func in funcs:
                 text = func(c)
                 assert c.scheduler.address in text
-                assert "3" in text
-                assert "6" in text
-                assert "GB" in text
+                assert "threads=3" in text or "Cores: </b>3" in text
+                assert "6.00 GB" in text or "5.59 GiB" in text
                 if "<table" not in text:
                     assert len(text) < 80
 
@@ -5417,7 +5416,7 @@ async def test_warn_when_submitting_large_values(c, s, a, b):
         future = c.submit(lambda x: x + 1, b"0" * 2000000)
 
     text = str(record[0].message)
-    assert "2.00 MB" in text
+    assert "2.00 MB" in text or "1.91 MiB" in text
     assert "large" in text
     assert "..." in text
     assert "'000" in text

From 20a55e91b52d8e51a62ba1b47ccc1ece07adb72e Mon Sep 17 00:00:00 2001
From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com>
Date: Tue, 30 Mar 2021 22:10:44 -0400
Subject: [PATCH 1211/1550] Add isort to pre-commit hooks, package resorting
 (#4647)

---
 .github/PULL_REQUEST_TEMPLATE.md              |   2 +-
 .pre-commit-config.yaml                       |   5 +
 conftest.py                                   |   1 -
 distributed/__init__.py                       |  30 ++---
 distributed/_concurrent_futures_thread.py     |   5 +-
 distributed/_ipython_utils.py                 |  12 +-
 distributed/actor.py                          |   2 +-
 distributed/batched.py                        |   6 +-
 distributed/cfexecutor.py                     |   3 +-
 distributed/cli/dask_scheduler.py             |   5 +-
 distributed/cli/dask_spec.py                  |   3 +-
 distributed/cli/dask_ssh.py                   |   2 +-
 distributed/cli/dask_worker.py                |  11 +-
 distributed/cli/tests/test_dask_scheduler.py  |  14 +--
 distributed/cli/tests/test_dask_spec.py       |   5 +-
 distributed/cli/tests/test_dask_ssh.py        |   1 +
 distributed/cli/tests/test_dask_worker.py     |  19 ++-
 distributed/cli/tests/test_tls_cli.py         |   9 +-
 distributed/cli/utils.py                      |   1 -
 distributed/client.py                         |  83 +++++++------
 distributed/comm/__init__.py                  |  18 ++-
 distributed/comm/addressing.py                |   4 +-
 distributed/comm/core.py                      |  11 +-
 distributed/comm/inproc.py                    |   8 +-
 distributed/comm/tcp.py                       |  18 +--
 distributed/comm/tests/test_comms.py          |   9 +-
 distributed/comm/tests/test_ucx.py            |  11 +-
 distributed/comm/tests/test_ucx_config.py     |   8 +-
 distributed/comm/tests/test_ws.py             |  14 ++-
 distributed/comm/ucx.py                       |  12 +-
 distributed/comm/utils.py                     |   1 -
 distributed/comm/ws.py                        |  21 ++--
 distributed/config.py                         |   3 +-
 distributed/core.py                           |  30 ++---
 distributed/counter.py                        |   1 -
 distributed/dashboard/components/__init__.py  |  27 +++--
 distributed/dashboard/components/nvml.py      |  18 +--
 distributed/dashboard/components/scheduler.py |  59 +++++-----
 distributed/dashboard/components/shared.py    |  17 +--
 distributed/dashboard/components/worker.py    |  23 ++--
 distributed/dashboard/core.py                 |   9 +-
 distributed/dashboard/export_tool.py          |   1 -
 distributed/dashboard/scheduler.py            |  51 ++++----
 .../dashboard/tests/test_components.py        |   2 +-
 .../dashboard/tests/test_scheduler_bokeh.py   |  37 +++---
 .../dashboard/tests/test_worker_bokeh.py      |  12 +-
 distributed/dashboard/worker.py               |  10 +-
 distributed/deploy/__init__.py                |   4 +-
 distributed/deploy/adaptive.py                |   7 +-
 distributed/deploy/adaptive_core.py           |   3 +-
 distributed/deploy/cluster.py                 |  17 ++-
 distributed/deploy/local.py                   |   7 +-
 distributed/deploy/old_ssh.py                 |   6 +-
 distributed/deploy/spec.py                    |  24 ++--
 distributed/deploy/ssh.py                     |   6 +-
 distributed/deploy/tests/test_adaptive.py     |  15 ++-
 .../deploy/tests/test_adaptive_core.py        |   1 +
 distributed/deploy/tests/test_local.py        |  32 ++---
 .../deploy/tests/test_slow_adaptive.py        |   6 +-
 distributed/deploy/tests/test_spec_cluster.py |  16 +--
 distributed/deploy/tests/test_ssh.py          |   2 +
 distributed/deploy/utils_test.py              |   4 +-
 distributed/diagnostics/eventstream.py        |   6 +-
 distributed/diagnostics/nvml.py               |   1 +
 distributed/diagnostics/progress.py           |   6 +-
 distributed/diagnostics/progress_stream.py    |  10 +-
 distributed/diagnostics/progressbar.py        |  18 ++-
 distributed/diagnostics/task_stream.py        |  10 +-
 .../diagnostics/tests/test_graph_layout.py    |   4 +-
 distributed/diagnostics/tests/test_nvml.py    |   3 +-
 .../diagnostics/tests/test_progress.py        |   8 +-
 .../diagnostics/tests/test_progress_stream.py |   1 +
 .../diagnostics/tests/test_progressbar.py     |  10 +-
 .../tests/test_scheduler_plugin.py            |   5 +-
 .../diagnostics/tests/test_task_stream.py     |  11 +-
 distributed/diagnostics/tests/test_widgets.py |  21 +++-
 distributed/diagnostics/websocket.py          |   2 +-
 distributed/diskutils.py                      |   1 -
 distributed/event.py                          |   7 +-
 distributed/http/routing.py                   |   3 +-
 distributed/http/scheduler/info.py            |  12 +-
 distributed/http/scheduler/json.py            |   2 +-
 distributed/http/scheduler/missing_bokeh.py   |   2 +-
 .../http/scheduler/prometheus/__init__.py     |   3 +-
 .../http/scheduler/prometheus/semaphore.py    |   2 +-
 .../scheduler/tests/test_scheduler_http.py    |   3 +-
 .../scheduler/tests/test_semaphore_http.py    |   3 +-
 distributed/http/statics.py                   |   3 +-
 distributed/http/tests/test_core.py           |   3 +-
 distributed/http/tests/test_routing.py        |   2 +-
 distributed/http/utils.py                     |   3 +-
 distributed/http/worker/prometheus.py         |   4 +-
 .../http/worker/tests/test_worker_http.py     |   4 +-
 distributed/lock.py                           |   5 +-
 distributed/locket.py                         |   2 +-
 distributed/metrics.py                        |   3 +-
 distributed/multi_lock.py                     |   7 +-
 distributed/nanny.py                          |  26 ++--
 distributed/node.py                           |  10 +-
 distributed/preloading.py                     |   6 +-
 distributed/process.py                        |  10 +-
 distributed/profile.py                        |   6 +-
 distributed/protocol/__init__.py              |  24 ++--
 distributed/protocol/arrow.py                 |   4 +-
 distributed/protocol/compression.py           |   8 +-
 distributed/protocol/core.py                  |   6 +-
 distributed/protocol/cuda.py                  |   2 +-
 distributed/protocol/h5py.py                  |   4 +-
 distributed/protocol/keras.py                 |   4 +-
 distributed/protocol/netcdf4.py               |   4 +-
 distributed/protocol/numpy.py                 |   6 +-
 distributed/protocol/serialize.py             |  22 ++--
 distributed/protocol/sparse.py                |   4 +-
 distributed/protocol/tests/test_arrow.py      |   2 +-
 distributed/protocol/tests/test_collection.py |   3 +-
 .../protocol/tests/test_collection_cuda.py    |   3 +-
 distributed/protocol/tests/test_cupy.py       |   1 +
 distributed/protocol/tests/test_h5py.py       |   6 +-
 .../protocol/tests/test_highlevelgraph.py     |   7 +-
 distributed/protocol/tests/test_keras.py      |   2 +-
 distributed/protocol/tests/test_netcdf4.py    |   6 +-
 distributed/protocol/tests/test_numba.py      |   4 +-
 distributed/protocol/tests/test_numpy.py      |  12 +-
 distributed/protocol/tests/test_pandas.py     |   5 +-
 distributed/protocol/tests/test_pickle.py     |   6 +-
 distributed/protocol/tests/test_protocol.py   |   4 +-
 distributed/protocol/tests/test_rmm.py        |   3 +-
 distributed/protocol/tests/test_scipy.py      |   1 +
 distributed/protocol/tests/test_serialize.py  |  27 +++--
 distributed/protocol/tests/test_torch.py      |   3 +-
 distributed/protocol/torch.py                 |   6 +-
 distributed/pubsub.py                         |   8 +-
 distributed/pytest_resourceleaks.py           |   2 +-
 distributed/queues.py                         |   7 +-
 distributed/recreate_exceptions.py            |   2 +
 distributed/scheduler.py                      |  99 ++++++++--------
 distributed/security.py                       |   6 +-
 distributed/semaphore.py                      |   3 +-
 distributed/stealing.py                       |   6 +-
 distributed/system_monitor.py                 |   1 +
 distributed/tests/make_tls_certs.py           |   2 +-
 distributed/tests/test_actor.py               |  12 +-
 distributed/tests/test_as_completed.py        |  16 ++-
 distributed/tests/test_asyncprocess.py        |   2 +-
 distributed/tests/test_batched.py             |   4 +-
 distributed/tests/test_client.py              | 111 +++++++++---------
 distributed/tests/test_client_executor.py     |  24 ++--
 distributed/tests/test_client_loop.py         |   3 +-
 distributed/tests/test_collections.py         |  11 +-
 distributed/tests/test_config.py              |   6 +-
 distributed/tests/test_core.py                |  24 ++--
 distributed/tests/test_diskutils.py           |   1 +
 distributed/tests/test_events.py              |   8 +-
 distributed/tests/test_failed_workers.py      |  17 +--
 distributed/tests/test_ipython.py             |   3 +-
 distributed/tests/test_locks.py               |  12 +-
 distributed/tests/test_multi_locks.py         |  11 +-
 distributed/tests/test_nanny.py               |  22 ++--
 distributed/tests/test_preload.py             |   8 +-
 distributed/tests/test_priorities.py          |   6 +-
 distributed/tests/test_profile.py             |  17 +--
 distributed/tests/test_publish.py             |  11 +-
 distributed/tests/test_pubsub.py              |   4 +-
 distributed/tests/test_queues.py              |  13 +-
 distributed/tests/test_resources.py           |  17 ++-
 distributed/tests/test_scheduler.py           |  33 +++---
 distributed/tests/test_security.py            |   4 +-
 distributed/tests/test_semaphore.py           |  19 +--
 distributed/tests/test_sizeof.py              |   4 +-
 distributed/tests/test_steal.py               |   6 +-
 distributed/tests/test_stress.py              |  28 ++---
 distributed/tests/test_threadpoolexecutor.py  |   4 +-
 distributed/tests/test_tls_functional.py      |  11 +-
 distributed/tests/test_utils.py               |  55 +++++----
 distributed/tests/test_utils_comm.py          |  10 +-
 distributed/tests/test_utils_test.py          |  17 ++-
 distributed/tests/test_variable.py            |  20 +++-
 distributed/tests/test_versions.py            |   3 +-
 distributed/tests/test_worker.py              |  57 +++++----
 distributed/tests/test_worker_client.py       |  19 ++-
 distributed/threadpoolexecutor.py             |   6 +-
 distributed/utils.py                          |  42 +++----
 distributed/utils_comm.py                     |   9 +-
 distributed/utils_perf.py                     |   3 +-
 distributed/utils_test.py                     |  37 +++---
 distributed/variable.py                       |   9 +-
 distributed/versions.py                       |   7 +-
 distributed/worker.py                         |  72 ++++++------
 distributed/worker_client.py                  |   7 +-
 docs/source/conf.py                           |   6 +-
 setup.cfg                                     |   9 ++
 setup.py                                      |   4 +-
 versioneer.py                                 |   3 +
 193 files changed, 1187 insertions(+), 1057 deletions(-)

diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index c75019cffff..0c693bd7f72 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -1,3 +1,3 @@
 - [ ] Closes #xxxx
 - [ ] Tests added / passed
-- [ ] Passes `black distributed` / `flake8 distributed`
+- [ ] Passes `black distributed` / `flake8 distributed` / `isort distributed`
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index ca1c26a7d87..222d7c1b466 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,4 +1,9 @@
 repos:
+  -   repo: https://github.com/pycqa/isort
+      rev: 5.8.0
+      hooks:
+      - id: isort
+        language_version: python3
   -   repo: https://github.com/psf/black
       rev: 20.8b1
       hooks:
diff --git a/conftest.py b/conftest.py
index 07adc4982f6..e8a159fa4a3 100644
--- a/conftest.py
+++ b/conftest.py
@@ -1,7 +1,6 @@
 # https://pytest.org/latest/example/simple.html#control-skipping-of-tests-according-to-command-line-option
 import pytest
 
-
 # Uncomment to enable more logging and checks
 # (https://docs.python.org/3/library/asyncio-dev.html)
 # Note this makes things slower and might consume much memory.
diff --git a/distributed/__init__.py b/distributed/__init__.py
index 475288a3b32..ad1e4756141 100644
--- a/distributed/__init__.py
+++ b/distributed/__init__.py
@@ -1,42 +1,42 @@
-from . import config
+from . import config  # isort:skip; load distributed configuration first
 import dask
 from dask.config import config
+
+from ._version import get_versions
 from .actor import Actor, ActorFuture
-from .core import connect, rpc, Status
-from .deploy import LocalCluster, Adaptive, SpecCluster, SSHCluster
-from .diagnostics.progressbar import progress
-from .diagnostics.plugin import WorkerPlugin, SchedulerPlugin, PipInstall
 from .client import (
     Client,
-    Executor,
     CompatibleExecutor,
-    wait,
+    Executor,
+    Future,
     as_completed,
     default_client,
     fire_and_forget,
-    Future,
     futures_of,
+    get_task_metadata,
     get_task_stream,
     performance_report,
-    get_task_metadata,
+    wait,
 )
+from .core import Status, connect, rpc
+from .deploy import Adaptive, LocalCluster, SpecCluster, SSHCluster
+from .diagnostics.plugin import PipInstall, SchedulerPlugin, WorkerPlugin
+from .diagnostics.progressbar import progress
+from .event import Event
 from .lock import Lock
 from .multi_lock import MultiLock
 from .nanny import Nanny
 from .pubsub import Pub, Sub
 from .queues import Queue
+from .scheduler import Scheduler
 from .security import Security
 from .semaphore import Semaphore
-from .event import Event
-from .scheduler import Scheduler
 from .threadpoolexecutor import rejoin
-from .utils import sync, TimeoutError, CancelledError
+from .utils import CancelledError, TimeoutError, sync
 from .variable import Variable
-from .worker import Worker, get_worker, get_client, secede, Reschedule
+from .worker import Reschedule, Worker, get_client, get_worker, secede
 from .worker_client import local_client, worker_client
 
-from ._version import get_versions
-
 versions = get_versions()
 __version__ = versions["version"]
 __git_revision__ = versions["full-revisionid"]
diff --git a/distributed/_concurrent_futures_thread.py b/distributed/_concurrent_futures_thread.py
index b26da12cb7a..1b6f328991f 100644
--- a/distributed/_concurrent_futures_thread.py
+++ b/distributed/_concurrent_futures_thread.py
@@ -8,16 +8,17 @@
 __author__ = "Brian Quinlan (brian@sweetapp.com)"
 
 import atexit
-from concurrent.futures import _base
 import itertools
+from concurrent.futures import _base
 
 try:
     import queue
 except ImportError:
     import Queue as queue
+
+import os
 import threading
 import weakref
-import os
 
 # Workers are created as daemon threads. This is done to allow the interpreter
 # to exit when there are still idle threads in a ThreadPoolExecutor's thread
diff --git a/distributed/_ipython_utils.py b/distributed/_ipython_utils.py
index 8aa1fe7ad7b..8e83a78ff4f 100644
--- a/distributed/_ipython_utils.py
+++ b/distributed/_ipython_utils.py
@@ -12,19 +12,17 @@
 except ImportError:
     # Python 2
     import Queue as queue
-from subprocess import Popen
+
 import sys
-from threading import Thread
+from subprocess import Popen
+from threading import Event, Thread
 from uuid import uuid4
 
-from tornado.gen import TimeoutError
-from tornado.ioloop import IOLoop
-from threading import Event
-
 from IPython import get_ipython
 from jupyter_client import BlockingKernelClient, write_connection_file
 from jupyter_core.paths import jupyter_runtime_dir
-
+from tornado.gen import TimeoutError
+from tornado.ioloop import IOLoop
 
 OUTPUT_TIMEOUT = 10
 
diff --git a/distributed/actor.py b/distributed/actor.py
index b9149fec7f4..0facdda4cb8 100644
--- a/distributed/actor.py
+++ b/distributed/actor.py
@@ -5,7 +5,7 @@
 
 from .client import Future, default_client
 from .protocol import to_serialize
-from .utils import iscoroutinefunction, thread_state, sync
+from .utils import iscoroutinefunction, sync, thread_state
 from .utils_comm import WrappedKey
 from .worker import get_worker
 
diff --git a/distributed/batched.py b/distributed/batched.py
index 7b2523fc1e0..b932f1b24c7 100644
--- a/distributed/batched.py
+++ b/distributed/batched.py
@@ -1,14 +1,14 @@
-from collections import deque
 import logging
+from collections import deque
 
-import dask
 from tornado import gen, locks
 from tornado.ioloop import IOLoop
 
+import dask
+
 from .core import CommClosedError
 from .utils import parse_timedelta
 
-
 logger = logging.getLogger(__name__)
 
 
diff --git a/distributed/cfexecutor.py b/distributed/cfexecutor.py
index 4096181581f..f54db42b9a7 100644
--- a/distributed/cfexecutor.py
+++ b/distributed/cfexecutor.py
@@ -2,11 +2,10 @@
 import weakref
 
 from tlz import merge
-
 from tornado import gen
 
 from .metrics import time
-from .utils import sync, TimeoutError, parse_timedelta
+from .utils import TimeoutError, parse_timedelta, sync
 
 
 @gen.coroutine
diff --git a/distributed/cli/dask_scheduler.py b/distributed/cli/dask_scheduler.py
index 37e8c497555..51ac13f2907 100755
--- a/distributed/cli/dask_scheduler.py
+++ b/distributed/cli/dask_scheduler.py
@@ -1,18 +1,17 @@
 import atexit
-import logging
 import gc
+import logging
 import os
 import re
 import sys
 import warnings
 
 import click
-
 from tornado.ioloop import IOLoop
 
 from distributed import Scheduler
-from distributed.preloading import validate_preload_argv
 from distributed.cli.utils import check_python_3, install_signal_handlers
+from distributed.preloading import validate_preload_argv
 from distributed.proctitle import (
     enable_proctitle_on_children,
     enable_proctitle_on_current,
diff --git a/distributed/cli/dask_spec.py b/distributed/cli/dask_spec.py
index 3311031763f..ef82ca3f493 100644
--- a/distributed/cli/dask_spec.py
+++ b/distributed/cli/dask_spec.py
@@ -1,7 +1,8 @@
 import asyncio
-import click
 import json
 import sys
+
+import click
 import yaml
 
 from distributed.deploy.spec import run_spec
diff --git a/distributed/cli/dask_ssh.py b/distributed/cli/dask_ssh.py
index f592c572ac5..f81cd73d495 100755
--- a/distributed/cli/dask_ssh.py
+++ b/distributed/cli/dask_ssh.py
@@ -1,7 +1,7 @@
-from distributed.deploy.old_ssh import SSHCluster
 import click
 
 from distributed.cli.utils import check_python_3
+from distributed.deploy.old_ssh import SSHCluster
 
 
 @click.command(
diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
index a7d2798a2e5..cc004baf631 100755
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -1,16 +1,20 @@
 import asyncio
 import atexit
-from contextlib import suppress
-import logging
 import gc
+import logging
 import os
 import signal
 import sys
 import warnings
+from contextlib import suppress
 
 import click
+from tlz import valmap
+from tornado.ioloop import IOLoop, TimeoutError
+
 import dask
 from dask.system import CPU_COUNT
+
 from distributed import Nanny
 from distributed.cli.utils import check_python_3, install_signal_handlers
 from distributed.comm import get_address_host_port
@@ -22,9 +26,6 @@
 )
 from distributed.utils import import_term
 
-from tlz import valmap
-from tornado.ioloop import IOLoop, TimeoutError
-
 logger = logging.getLogger("distributed.dask_worker")
 
 
diff --git a/distributed/cli/tests/test_dask_scheduler.py b/distributed/cli/tests/test_dask_scheduler.py
index 6be7f3c365d..2e938db558f 100644
--- a/distributed/cli/tests/test_dask_scheduler.py
+++ b/distributed/cli/tests/test_dask_scheduler.py
@@ -3,26 +3,26 @@
 pytest.importorskip("requests")
 
 import os
-import requests
-import socket
 import shutil
+import socket
 import sys
 import tempfile
 from time import sleep
 
+import requests
 from click.testing import CliRunner
 
 import distributed
-from distributed import Scheduler, Client
+import distributed.cli.dask_scheduler
+from distributed import Client, Scheduler
+from distributed.metrics import time
 from distributed.utils import get_ip, get_ip_interface, tmpfile
+from distributed.utils_test import loop  # noqa: F401
 from distributed.utils_test import (
-    popen,
     assert_can_connect_from_everywhere_4_6,
     assert_can_connect_locally_4,
+    popen,
 )
-from distributed.utils_test import loop  # noqa: F401
-from distributed.metrics import time
-import distributed.cli.dask_scheduler
 
 
 def test_defaults(loop):
diff --git a/distributed/cli/tests/test_dask_spec.py b/distributed/cli/tests/test_dask_spec.py
index a18b9fb383a..0a5f64fc484 100644
--- a/distributed/cli/tests/test_dask_spec.py
+++ b/distributed/cli/tests/test_dask_spec.py
@@ -1,10 +1,11 @@
-import pytest
 import sys
+
+import pytest
 import yaml
 
 from distributed import Client
-from distributed.utils_test import popen
 from distributed.utils_test import cleanup  # noqa: F401
+from distributed.utils_test import popen
 
 
 @pytest.mark.asyncio
diff --git a/distributed/cli/tests/test_dask_ssh.py b/distributed/cli/tests/test_dask_ssh.py
index 9be8cb06f62..b73cd66d914 100644
--- a/distributed/cli/tests/test_dask_ssh.py
+++ b/distributed/cli/tests/test_dask_ssh.py
@@ -1,4 +1,5 @@
 from click.testing import CliRunner
+
 from distributed.cli.dask_ssh import main
 
 
diff --git a/distributed/cli/tests/test_dask_worker.py b/distributed/cli/tests/test_dask_worker.py
index 853b9964128..98724c6754b 100644
--- a/distributed/cli/tests/test_dask_worker.py
+++ b/distributed/cli/tests/test_dask_worker.py
@@ -1,22 +1,29 @@
 import asyncio
+
 import pytest
 from click.testing import CliRunner
 
 pytest.importorskip("requests")
 
-import requests
-import sys
 import os
-from time import sleep
+import sys
 from multiprocessing import cpu_count
+from time import sleep
+
+import requests
 
 import distributed.cli.dask_worker
 from distributed import Client, Scheduler
 from distributed.deploy.utils import nprocesses_nthreads
 from distributed.metrics import time
-from distributed.utils import sync, tmpfile, parse_ports
-from distributed.utils_test import popen, terminate_process, wait_for_port
-from distributed.utils_test import loop, cleanup  # noqa: F401
+from distributed.utils import parse_ports, sync, tmpfile
+from distributed.utils_test import (  # noqa: F401
+    cleanup,
+    loop,
+    popen,
+    terminate_process,
+    wait_for_port,
+)
 
 
 def test_nanny_worker_ports(loop):
diff --git a/distributed/cli/tests/test_tls_cli.py b/distributed/cli/tests/test_tls_cli.py
index def31bc244d..9301b47ab3e 100644
--- a/distributed/cli/tests/test_tls_cli.py
+++ b/distributed/cli/tests/test_tls_cli.py
@@ -1,16 +1,15 @@
 from time import sleep
 
 from distributed import Client
+from distributed.metrics import time
+from distributed.utils_test import loop  # noqa: F401
 from distributed.utils_test import (
-    popen,
     get_cert,
     new_config_file,
-    tls_security,
+    popen,
     tls_only_config,
+    tls_security,
 )
-from distributed.utils_test import loop  # noqa: F401
-from distributed.metrics import time
-
 
 ca_file = get_cert("tls-ca-cert.pem")
 cert = get_cert("tls-cert.pem")
diff --git a/distributed/cli/utils.py b/distributed/cli/utils.py
index c1bff051534..b2515faff11 100644
--- a/distributed/cli/utils.py
+++ b/distributed/cli/utils.py
@@ -1,6 +1,5 @@
 from tornado.ioloop import IOLoop
 
-
 py3_err_msg = """
 Warning: Your terminal does not set locales.
 
diff --git a/distributed/client.py b/distributed/client.py
index f327f1df890..14a5e0c8091 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -1,37 +1,37 @@
 import asyncio
 import atexit
-from collections import defaultdict
-from collections.abc import Iterator
-from concurrent.futures import ThreadPoolExecutor
-from concurrent.futures._base import DoneAndNotDoneFutures
-from contextlib import contextmanager, suppress
-from contextvars import ContextVar
 import copy
 import errno
-from functools import partial
 import html
 import inspect
 import json
 import logging
-from numbers import Number
 import os
+import socket
 import sys
-import uuid
 import threading
-import socket
-from queue import Queue as pyQueue
+import uuid
 import warnings
 import weakref
+from collections import defaultdict
+from collections.abc import Iterator
+from concurrent.futures import ThreadPoolExecutor
+from concurrent.futures._base import DoneAndNotDoneFutures
+from contextlib import contextmanager, suppress
+from contextvars import ContextVar
+from functools import partial
+from numbers import Number
+from queue import Queue as pyQueue
+
+from tlz import first, groupby, keymap, merge, partition_all, valmap
 
 import dask
-from dask.base import tokenize, normalize_token, collections_to_dsk
+from dask.base import collections_to_dsk, normalize_token, tokenize
+from dask.compatibility import apply
 from dask.core import flatten
+from dask.highlevelgraph import HighLevelGraph
 from dask.optimization import SubgraphCallable
-from dask.compatibility import apply
 from dask.utils import ensure_dict, format_bytes, funcname, stringify
-from dask.highlevelgraph import HighLevelGraph
-
-from tlz import first, groupby, merge, valmap, keymap, partition_all
 
 try:
     from dask.delayed import single_key
@@ -40,24 +40,18 @@
 from tornado import gen
 from tornado.ioloop import IOLoop, PeriodicCallback
 
+from . import versions as version_module
 from .batched import BatchedSend
-from .utils_comm import (
-    WrappedKey,
-    unpack_remotedata,
-    pack_data,
-    scatter_to_workers,
-    gather_from_workers,
-    retry_operation,
-)
 from .cfexecutor import ClientExecutor
 from .core import (
-    connect,
-    rpc,
-    clean_exception,
     CommClosedError,
-    PooledRPCCall,
     ConnectionPool,
+    PooledRPCCall,
+    clean_exception,
+    connect,
+    rpc,
 )
+from .diagnostics.plugin import UploadFile, WorkerPlugin
 from .metrics import time
 from .protocol import to_serialize
 from .protocol.pickle import dumps, loads
@@ -66,25 +60,30 @@
 from .security import Security
 from .sizeof import sizeof
 from .threadpoolexecutor import rejoin
-from .worker import get_client, get_worker, secede
-from .diagnostics.plugin import UploadFile, WorkerPlugin
 from .utils import (
     All,
-    sync,
-    log_errors,
+    Any,
+    CancelledError,
+    LoopRunner,
+    TimeoutError,
+    format_dashboard_link,
+    has_keyword,
     key_split,
-    thread_state,
+    log_errors,
     no_default,
-    LoopRunner,
     parse_timedelta,
-    Any,
-    has_keyword,
-    format_dashboard_link,
-    TimeoutError,
-    CancelledError,
+    sync,
+    thread_state,
 )
-from . import versions as version_module
-
+from .utils_comm import (
+    WrappedKey,
+    gather_from_workers,
+    pack_data,
+    retry_operation,
+    scatter_to_workers,
+    unpack_remotedata,
+)
+from .worker import get_client, get_worker, secede
 
 logger = logging.getLogger(__name__)
 
@@ -3944,7 +3943,7 @@ async def _get_task_stream(
             source, figure = task_stream_figure(sizing_mode="stretch_both")
             source.data.update(rects)
             if plot == "save":
-                from bokeh.plotting import save, output_file
+                from bokeh.plotting import output_file, save
 
                 output_file(filename=filename, title="Dask Task Stream")
                 save(figure, filename=filename, resources=bokeh_resources)
diff --git a/distributed/comm/__init__.py b/distributed/comm/__init__.py
index 7eb820d3f54..fe227ec1ff4 100644
--- a/distributed/comm/__init__.py
+++ b/distributed/comm/__init__.py
@@ -1,22 +1,20 @@
 from .addressing import (
-    parse_address,
-    unparse_address,
+    get_address_host,
+    get_address_host_port,
+    get_local_address_for,
     normalize_address,
+    parse_address,
     parse_host_port,
-    unparse_host_port,
     resolve_address,
-    get_address_host_port,
-    get_address_host,
-    get_local_address_for,
+    unparse_address,
+    unparse_host_port,
 )
-from .core import connect, listen, Comm, CommClosedError
+from .core import Comm, CommClosedError, connect, listen
 from .utils import get_tcp_server_address
 
 
 def _register_transports():
-    from . import inproc
-    from . import tcp
-    from . import ws
+    from . import inproc, tcp, ws
 
     try:
         from . import ucx
diff --git a/distributed/comm/addressing.py b/distributed/comm/addressing.py
index 537ecd4ef23..949fa31bcd1 100644
--- a/distributed/comm/addressing.py
+++ b/distributed/comm/addressing.py
@@ -1,9 +1,9 @@
 import itertools
+
 import dask
 
-from . import registry
 from ..utils import get_ip_interface
-
+from . import registry
 
 DEFAULT_SCHEME = dask.config.get("distributed.comm.default-scheme")
 
diff --git a/distributed/comm/core.py b/distributed/comm/core.py
index 2a01ec3d728..6ee79723736 100644
--- a/distributed/comm/core.py
+++ b/distributed/comm/core.py
@@ -1,21 +1,20 @@
-from abc import ABC, abstractmethod, abstractproperty
 import asyncio
-from contextlib import suppress
 import inspect
 import logging
 import random
 import sys
 import weakref
+from abc import ABC, abstractmethod, abstractproperty
+from contextlib import suppress
 
 import dask
 
 from ..metrics import time
-from ..utils import parse_timedelta, TimeoutError
+from ..protocol import pickle
+from ..protocol.compression import get_default_compression
+from ..utils import TimeoutError, parse_timedelta
 from . import registry
 from .addressing import parse_address
-from ..protocol.compression import get_default_compression
-from ..protocol import pickle
-
 
 logger = logging.getLogger(__name__)
 
diff --git a/distributed/comm/inproc.py b/distributed/comm/inproc.py
index d93377975ba..7374fba188d 100644
--- a/distributed/comm/inproc.py
+++ b/distributed/comm/inproc.py
@@ -1,21 +1,19 @@
 import asyncio
-from collections import deque, namedtuple
 import itertools
 import logging
 import os
 import threading
-import weakref
 import warnings
+import weakref
+from collections import deque, namedtuple
 
 from tornado.concurrent import Future
 from tornado.ioloop import IOLoop
 
 from ..protocol import nested_deserialize
 from ..utils import get_ip
-
+from .core import Comm, CommClosedError, Connector, Listener
 from .registry import Backend, backends
-from .core import Comm, Connector, Listener, CommClosedError
-
 
 logger = logging.getLogger(__name__)
 
diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py
index 530dbfbcf1a..258f4e88a5a 100644
--- a/distributed/comm/tcp.py
+++ b/distributed/comm/tcp.py
@@ -2,33 +2,33 @@
 import functools
 import logging
 import socket
-from ssl import SSLError
 import struct
 import sys
-from tornado import gen
 import weakref
+from ssl import SSLError
+
+from tornado import gen
 
 try:
     import ssl
 except ImportError:
     ssl = None
 
-import dask
 from tornado import netutil
 from tornado.iostream import StreamClosedError
 from tornado.tcpclient import TCPClient
 from tornado.tcpserver import TCPServer
 
+import dask
+
+from ..protocol.utils import pack_frames_prelude, unpack_frames
 from ..system import MEMORY_LIMIT
 from ..threadpoolexecutor import ThreadPoolExecutor
 from ..utils import ensure_ip, get_ip, get_ipv6, nbytes, parse_timedelta
-
-from .registry import Backend, backends
 from .addressing import parse_host_port, unparse_host_port
-from .core import Comm, Connector, Listener, CommClosedError, FatalCommClosedError
-from .utils import to_frames, from_frames, get_tcp_server_address, ensure_concrete_host
-from ..protocol.utils import pack_frames_prelude, unpack_frames
-
+from .core import Comm, CommClosedError, Connector, FatalCommClosedError, Listener
+from .registry import Backend, backends
+from .utils import ensure_concrete_host, from_frames, get_tcp_server_address, to_frames
 
 logger = logging.getLogger(__name__)
 
diff --git a/distributed/comm/tests/test_comms.py b/distributed/comm/tests/test_comms.py
index 56f3c7a1b11..8b78f7f37d4 100644
--- a/distributed/comm/tests/test_comms.py
+++ b/distributed/comm/tests/test_comms.py
@@ -6,11 +6,14 @@
 import warnings
 from functools import partial
 
+import pkg_resources
+import pytest
+from tornado import ioloop
+from tornado.concurrent import Future
+
 import dask
 
 import distributed
-import pkg_resources
-import pytest
 from distributed.comm import (
     CommClosedError,
     connect,
@@ -38,8 +41,6 @@
     has_ipv6,
     requires_ipv6,
 )
-from tornado import ioloop
-from tornado.concurrent import Future
 
 EXTERNAL_IP4 = get_ip()
 if has_ipv6():
diff --git a/distributed/comm/tests/test_ucx.py b/distributed/comm/tests/test_ucx.py
index aa5095e2f3c..2a388904475 100644
--- a/distributed/comm/tests/test_ucx.py
+++ b/distributed/comm/tests/test_ucx.py
@@ -1,16 +1,15 @@
 import asyncio
+
 import pytest
 
 ucp = pytest.importorskip("ucp")
 
-from distributed import Client, Worker, Scheduler, wait
-from distributed.comm import ucx, listen, connect
+from distributed import Client, Scheduler, Worker, wait
+from distributed.comm import connect, listen, parse_address, ucx
 from distributed.comm.registry import backends, get_backend
-from distributed.comm import ucx, parse_address
-from distributed.protocol import to_serialize
 from distributed.deploy.local import LocalCluster
-from distributed.utils_test import gen_test, loop, inc, cleanup, popen  # noqa: 401
-
+from distributed.protocol import to_serialize
+from distributed.utils_test import cleanup, gen_test, inc, loop, popen  # noqa: 401
 
 try:
     HOST = ucp.get_address()
diff --git a/distributed/comm/tests/test_ucx_config.py b/distributed/comm/tests/test_ucx_config.py
index 43eb45acb23..891793ae972 100644
--- a/distributed/comm/tests/test_ucx_config.py
+++ b/distributed/comm/tests/test_ucx_config.py
@@ -1,12 +1,14 @@
-import pytest
 from time import sleep
 
+import pytest
+
 import dask
 from dask.utils import format_bytes
+
 from distributed import Client
-from distributed.utils_test import gen_test, loop, inc, cleanup, popen  # noqa: 401
-from distributed.utils import get_ip
 from distributed.comm.ucx import _scrub_ucx_config
+from distributed.utils import get_ip
+from distributed.utils_test import cleanup, gen_test, inc, loop, popen  # noqa: 401
 
 try:
     HOST = get_ip()
diff --git a/distributed/comm/tests/test_ws.py b/distributed/comm/tests/test_ws.py
index 1d4a5222b2e..f63c9484958 100644
--- a/distributed/comm/tests/test_ws.py
+++ b/distributed/comm/tests/test_ws.py
@@ -1,20 +1,22 @@
 import os
-import warnings
-import pytest
 import tempfile
+import warnings
 
 import numpy as np
+import pytest
+
 import dask
+
 from distributed import Client, Scheduler, Worker
-from distributed.security import Security
+from distributed.comm import connect, listen, ws
 from distributed.comm.registry import backends, get_backend
-from distributed.comm import ws, listen, connect
+from distributed.security import Security
 from distributed.utils_test import (  # noqa: F401
+    cleanup,
+    gen_cluster,
     get_client_ssl_context,
     get_server_ssl_context,
-    cleanup,
     inc,
-    gen_cluster,
 )
 
 from .test_comms import check_tls_extra
diff --git a/distributed/comm/ucx.py b/distributed/comm/ucx.py
index efa74254745..92b457307ac 100644
--- a/distributed/comm/ucx.py
+++ b/distributed/comm/ucx.py
@@ -11,19 +11,19 @@
 
 import dask
 
-from .addressing import parse_host_port, unparse_host_port
-from .core import Comm, Connector, Listener, CommClosedError
-from .registry import Backend, backends
-from .utils import ensure_concrete_host, to_frames, from_frames
 from ..utils import (
+    CancelledError,
     ensure_ip,
     get_ip,
     get_ipv6,
-    nbytes,
     log_errors,
-    CancelledError,
+    nbytes,
     parse_bytes,
 )
+from .addressing import parse_host_port, unparse_host_port
+from .core import Comm, CommClosedError, Connector, Listener
+from .registry import Backend, backends
+from .utils import ensure_concrete_host, from_frames, to_frames
 
 logger = logging.getLogger(__name__)
 
diff --git a/distributed/comm/utils.py b/distributed/comm/utils.py
index b3ac85feed8..15b9244329a 100644
--- a/distributed/comm/utils.py
+++ b/distributed/comm/utils.py
@@ -9,7 +9,6 @@
 from .. import protocol
 from ..utils import get_ip, get_ipv6, nbytes, offload
 
-
 logger = logging.getLogger(__name__)
 
 
diff --git a/distributed/comm/ws.py b/distributed/comm/ws.py
index ac76df3c5d4..b9e0afdd7ec 100644
--- a/distributed/comm/ws.py
+++ b/distributed/comm/ws.py
@@ -1,26 +1,23 @@
 import asyncio
 import logging
 import struct
-import weakref
 import warnings
-
-from typing import Callable
+import weakref
 from ssl import SSLError
+from typing import Callable
 
-from tornado.iostream import StreamClosedError
-from tornado.httpserver import HTTPServer
-from tornado.httpclient import HTTPRequest
 from tornado import web
-from tornado.websocket import WebSocketClosedError, websocket_connect, WebSocketHandler
+from tornado.httpclient import HTTPRequest
+from tornado.httpserver import HTTPServer
+from tornado.iostream import StreamClosedError
+from tornado.websocket import WebSocketClosedError, WebSocketHandler, websocket_connect
 
 from ..utils import ensure_bytes, nbytes
-
-from .registry import backends
 from .addressing import parse_host_port, unparse_host_port
-from .core import Comm, CommClosedError, Listener, Connector, FatalCommClosedError
+from .core import Comm, CommClosedError, Connector, FatalCommClosedError, Listener
+from .registry import backends
 from .tcp import BaseTCPBackend, _expect_tls_context, convert_stream_closed_error
-from .utils import ensure_concrete_host, to_frames, from_frames, get_tcp_server_address
-
+from .utils import ensure_concrete_host, from_frames, get_tcp_server_address, to_frames
 
 logger = logging.getLogger(__name__)
 
diff --git a/distributed/config.py b/distributed/config.py
index 5601c307558..4b1cfb35042 100644
--- a/distributed/config.py
+++ b/distributed/config.py
@@ -3,9 +3,10 @@
 import os
 import sys
 
-import dask
 import yaml
 
+import dask
+
 from .compatibility import logging_names
 
 config = dask.config.config
diff --git a/distributed/core.py b/distributed/core.py
index bcb5d8fe92f..0987564c4dc 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -1,44 +1,44 @@
 import asyncio
-from collections import defaultdict
-from contextlib import suppress
-from enum import Enum
-from functools import partial
 import inspect
 import logging
 import sys
 import threading
 import traceback
 import uuid
-import weakref
 import warnings
+import weakref
+from collections import defaultdict
+from contextlib import suppress
+from enum import Enum
+from functools import partial
 
-import dask
 import tblib
 from tlz import merge
 from tornado import gen
 from tornado.ioloop import IOLoop, PeriodicCallback
 
+import dask
+
+from . import profile, protocol
 from .comm import (
+    CommClosedError,
     connect,
+    get_address_host_port,
     listen,
-    CommClosedError,
     normalize_address,
     unparse_host_port,
-    get_address_host_port,
 )
 from .metrics import time
-from . import profile
 from .system_monitor import SystemMonitor
 from .utils import (
-    is_coroutine_function,
-    get_traceback,
-    truncate_exception,
-    parse_timedelta,
-    has_keyword,
     CancelledError,
     TimeoutError,
+    get_traceback,
+    has_keyword,
+    is_coroutine_function,
+    parse_timedelta,
+    truncate_exception,
 )
-from . import protocol
 
 
 class Status(Enum):
diff --git a/distributed/counter.py b/distributed/counter.py
index feffb69ce8c..d6c1dad9ecc 100644
--- a/distributed/counter.py
+++ b/distributed/counter.py
@@ -2,7 +2,6 @@
 
 from tornado.ioloop import IOLoop, PeriodicCallback
 
-
 try:
     from crick import TDigest
 except ImportError:
diff --git a/distributed/dashboard/components/__init__.py b/distributed/dashboard/components/__init__.py
index 78d60108c8e..c80372faa08 100644
--- a/distributed/dashboard/components/__init__.py
+++ b/distributed/dashboard/components/__init__.py
@@ -1,34 +1,35 @@
 import asyncio
+import weakref
 from bisect import bisect
 from operator import add
 from time import time
-import weakref
 
-from bokeh.layouts import row, column
+from bokeh.layouts import column, row
 from bokeh.models import (
+    BoxZoomTool,
+    Button,
     ColumnDataSource,
-    Plot,
     DataRange1d,
-    LinearAxis,
     HoverTool,
-    BoxZoomTool,
-    ResetTool,
+    LinearAxis,
+    OpenURL,
     PanTool,
-    WheelZoomTool,
-    Range1d,
+    Plot,
     Quad,
-    TapTool,
-    OpenURL,
-    Button,
+    Range1d,
+    ResetTool,
     Select,
+    TapTool,
+    WheelZoomTool,
 )
 from bokeh.palettes import Spectral9
 from bokeh.plotting import figure
-import dask
 from tornado import gen
 
-from distributed.dashboard.utils import without_property_validation, BOKEH_VERSION
+import dask
+
 from distributed import profile
+from distributed.dashboard.utils import BOKEH_VERSION, without_property_validation
 from distributed.utils import log_errors, parse_timedelta
 
 if dask.config.get("distributed.dashboard.export-tool"):
diff --git a/distributed/dashboard/components/nvml.py b/distributed/dashboard/components/nvml.py
index 8be3b406bcb..6a544f1b9d9 100644
--- a/distributed/dashboard/components/nvml.py
+++ b/distributed/dashboard/components/nvml.py
@@ -1,22 +1,22 @@
 import math
 
-from distributed.dashboard.components import DashboardComponent, add_periodic_callback
-
-from bokeh.plotting import figure
 from bokeh.models import (
-    ColumnDataSource,
     BasicTicker,
+    ColumnDataSource,
+    HoverTool,
     NumeralTickFormatter,
-    TapTool,
     OpenURL,
-    HoverTool,
+    TapTool,
 )
+from bokeh.plotting import figure
 from tornado import escape
+
 from dask.utils import format_bytes
-from distributed.utils import log_errors
-from distributed.dashboard.components.scheduler import BOKEH_THEME, TICKS_1024, env
-from distributed.dashboard.utils import without_property_validation, update
 
+from distributed.dashboard.components import DashboardComponent, add_periodic_callback
+from distributed.dashboard.components.scheduler import BOKEH_THEME, TICKS_1024, env
+from distributed.dashboard.utils import update, without_property_validation
+from distributed.utils import log_errors
 
 try:
     import pynvml
diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index 2fd68e8d6f6..eae544d3fac 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -1,47 +1,48 @@
-from collections import defaultdict
 import logging
 import math
-from numbers import Number
 import operator
 import os
+from collections import defaultdict
+from numbers import Number
 
+from bokeh.io import curdoc
 from bokeh.layouts import column, row
 from bokeh.models import (
-    ColumnDataSource,
+    AdaptiveTicker,
+    BasicTicker,
+    BoxSelectTool,
+    BoxZoomTool,
+    CDSView,
     ColorBar,
+    ColumnDataSource,
     DataRange1d,
+    GroupFilter,
     HoverTool,
-    ResetTool,
-    PanTool,
-    WheelZoomTool,
-    TapTool,
+    NumberFormatter,
+    NumeralTickFormatter,
     OpenURL,
+    Panel,
+    PanTool,
     Range1d,
-    value,
-    NumeralTickFormatter,
-    BoxZoomTool,
-    AdaptiveTicker,
-    BasicTicker,
-    NumberFormatter,
-    BoxSelectTool,
-    GroupFilter,
-    CDSView,
+    ResetTool,
     Tabs,
-    Panel,
+    TapTool,
     Title,
+    WheelZoomTool,
+    value,
 )
 from bokeh.models.widgets import DataTable, TableColumn
-from bokeh.plotting import figure
 from bokeh.palettes import Viridis11
+from bokeh.plotting import figure
 from bokeh.themes import Theme
-from bokeh.transform import factor_cmap, linear_cmap, cumsum
-from bokeh.io import curdoc
+from bokeh.transform import cumsum, factor_cmap, linear_cmap
+from tlz import pipe
+from tlz.curried import concat, groupby, map
+from tornado import escape
+
 import dask
 from dask import config
 from dask.utils import format_bytes, key_split
-from tlz import pipe
-from tlz.curried import map, concat, groupby
-from tornado import escape
 
 try:
     import numpy as np
@@ -51,24 +52,24 @@
 from distributed.dashboard.components import add_periodic_callback
 from distributed.dashboard.components.shared import (
     DashboardComponent,
-    ProfileTimePlot,
     ProfileServer,
+    ProfileTimePlot,
     SystemMonitor,
 )
 from distributed.dashboard.utils import (
-    transpose,
     BOKEH_VERSION,
     PROFILING,
-    without_property_validation,
+    transpose,
     update,
+    without_property_validation,
 )
-from distributed.metrics import time
-from distributed.utils import log_errors, format_time, parse_timedelta
-from distributed.diagnostics.progress_stream import color_of, progress_quads
 from distributed.diagnostics.graph_layout import GraphLayout
+from distributed.diagnostics.progress_stream import color_of, progress_quads
 from distributed.diagnostics.task_stream import TaskStreamPlugin
 from distributed.diagnostics.task_stream import color_of as ts_color_of
 from distributed.diagnostics.task_stream import colors as ts_color_lookup
+from distributed.metrics import time
+from distributed.utils import format_time, log_errors, parse_timedelta
 
 if dask.config.get("distributed.dashboard.export-tool"):
     from distributed.dashboard.export_tool import ExportTool
diff --git a/distributed/dashboard/components/shared.py b/distributed/dashboard/components/shared.py
index 037b256b188..4aa2561c831 100644
--- a/distributed/dashboard/components/shared.py
+++ b/distributed/dashboard/components/shared.py
@@ -1,31 +1,32 @@
 import asyncio
 import weakref
 
-from bokeh.layouts import row, column
+import tlz as toolz
+from bokeh.layouts import column, row
 from bokeh.models import (
+    Button,
     ColumnDataSource,
     DataRange1d,
     HoverTool,
+    NumeralTickFormatter,
     Range1d,
-    Button,
     Select,
-    NumeralTickFormatter,
 )
 from bokeh.palettes import Spectral9
 from bokeh.plotting import figure
-import dask
 from tornado import gen
-import tlz as toolz
 
+import dask
+
+from distributed import profile
+from distributed.compatibility import WINDOWS
 from distributed.dashboard.components import DashboardComponent
 from distributed.dashboard.utils import (
-    without_property_validation,
     BOKEH_VERSION,
     update,
+    without_property_validation,
 )
-from distributed import profile
 from distributed.utils import log_errors, parse_timedelta
-from distributed.compatibility import WINDOWS
 
 if dask.config.get("distributed.dashboard.export-tool"):
     from distributed.dashboard.export_tool import ExportTool
diff --git a/distributed/dashboard/components/worker.py b/distributed/dashboard/components/worker.py
index ee9ad65d2e1..83f8c3736f8 100644
--- a/distributed/dashboard/components/worker.py
+++ b/distributed/dashboard/components/worker.py
@@ -2,38 +2,37 @@
 import math
 import os
 
-from bokeh.layouts import row, column
+from bokeh.layouts import column, row
 from bokeh.models import (
+    BoxZoomTool,
     ColumnDataSource,
     DataRange1d,
     HoverTool,
-    BoxZoomTool,
-    ResetTool,
-    PanTool,
-    WheelZoomTool,
     NumeralTickFormatter,
+    PanTool,
+    ResetTool,
     Select,
+    WheelZoomTool,
 )
-
 from bokeh.models.widgets import DataTable, TableColumn
-from bokeh.plotting import figure
 from bokeh.palettes import RdBu
+from bokeh.plotting import figure
 from bokeh.themes import Theme
-from dask.utils import format_bytes
 from tlz import merge, partition_all
 
+from dask.utils import format_bytes
+
 from distributed.dashboard.components import add_periodic_callback
 from distributed.dashboard.components.shared import (
     DashboardComponent,
-    ProfileTimePlot,
     ProfileServer,
+    ProfileTimePlot,
     SystemMonitor,
 )
-from distributed.dashboard.utils import transpose, without_property_validation, update
+from distributed.dashboard.utils import transpose, update, without_property_validation
 from distributed.diagnostics.progress_stream import color_of
 from distributed.metrics import time
-from distributed.utils import log_errors, key_split, format_time
-
+from distributed.utils import format_time, key_split, log_errors
 
 logger = logging.getLogger(__name__)
 
diff --git a/distributed/dashboard/core.py b/distributed/dashboard/core.py
index 916504f4d04..3b6ea0b23a4 100644
--- a/distributed/dashboard/core.py
+++ b/distributed/dashboard/core.py
@@ -1,6 +1,6 @@
-from distutils.version import LooseVersion
 import functools
 import warnings
+from distutils.version import LooseVersion
 
 import bokeh
 from bokeh.server.server import BokehTornado
@@ -9,11 +9,12 @@
     from bokeh.server.util import create_hosts_allowlist
 except ImportError:
     from bokeh.server.util import create_hosts_whitelist as create_hosts_allowlist
-from bokeh.application.handlers.function import FunctionHandler
-from bokeh.application import Application
-import dask
+
 import toolz
+from bokeh.application import Application
+from bokeh.application.handlers.function import FunctionHandler
 
+import dask
 
 if LooseVersion(bokeh.__version__) < LooseVersion("0.13.0"):
     warnings.warn(
diff --git a/distributed/dashboard/export_tool.py b/distributed/dashboard/export_tool.py
index d93d21b881b..a9be2d7a4bf 100644
--- a/distributed/dashboard/export_tool.py
+++ b/distributed/dashboard/export_tool.py
@@ -6,7 +6,6 @@
 from bokeh.resources import CDN
 from bokeh.util.compiler import JavaScript
 
-
 fn = __file__
 fn = os.path.join(os.path.dirname(fn), "export_tool.js")
 with open(fn) as f:
diff --git a/distributed/dashboard/scheduler.py b/distributed/dashboard/scheduler.py
index 413dde57301..f0d85b04db9 100644
--- a/distributed/dashboard/scheduler.py
+++ b/distributed/dashboard/scheduler.py
@@ -1,49 +1,44 @@
 from urllib.parse import urljoin
 
-from tornado.ioloop import IOLoop
 from tornado import web
+from tornado.ioloop import IOLoop
 
 try:
     import numpy as np
 except ImportError:
     np = False
 
-from .core import BokehApplication
-from .components.worker import counters_doc
+from .components.nvml import gpu_doc  # noqa: 1708
+from .components.nvml import NVML_ENABLED, gpu_memory_doc, gpu_utilization_doc
 from .components.scheduler import (
-    systemmonitor_doc,
-    stealing_doc,
-    workers_doc,
     events_doc,
-    tasks_doc,
-    status_doc,
-    profile_doc,
-    profile_server_doc,
     graph_doc,
-    individual_task_stream_doc,
-    individual_progress_doc,
-    individual_graph_doc,
-    individual_profile_doc,
-    individual_profile_server_doc,
-    individual_nbytes_doc,
-    individual_cpu_doc,
-    individual_nprocessing_doc,
-    individual_workers_doc,
+    individual_aggregate_time_per_action_doc,
     individual_bandwidth_types_doc,
     individual_bandwidth_workers_doc,
-    individual_memory_by_key_doc,
     individual_compute_time_per_key_doc,
-    individual_aggregate_time_per_action_doc,
+    individual_cpu_doc,
+    individual_graph_doc,
+    individual_memory_by_key_doc,
+    individual_nbytes_doc,
+    individual_nprocessing_doc,
+    individual_profile_doc,
+    individual_profile_server_doc,
+    individual_progress_doc,
     individual_systemmonitor_doc,
+    individual_task_stream_doc,
+    individual_workers_doc,
+    profile_doc,
+    profile_server_doc,
+    status_doc,
+    stealing_doc,
+    systemmonitor_doc,
+    tasks_doc,
+    workers_doc,
 )
+from .components.worker import counters_doc
+from .core import BokehApplication
 from .worker import counters_doc
-from .components.nvml import (
-    NVML_ENABLED,
-    gpu_memory_doc,
-    gpu_utilization_doc,
-    gpu_doc,
-)  # noqa: 1708
-
 
 template_variables = {
     "pages": ["status", "workers", "tasks", "system", "profile", "graph", "info"]
diff --git a/distributed/dashboard/tests/test_components.py b/distributed/dashboard/tests/test_components.py
index a3e444e17e6..bc9f6c74849 100644
--- a/distributed/dashboard/tests/test_components.py
+++ b/distributed/dashboard/tests/test_components.py
@@ -6,12 +6,12 @@
 
 from bokeh.models import ColumnDataSource, Model
 
-from distributed.utils_test import slowinc, gen_cluster
 from distributed.dashboard.components.shared import (
     Processing,
     ProfilePlot,
     ProfileTimePlot,
 )
+from distributed.utils_test import gen_cluster, slowinc
 
 
 @pytest.mark.parametrize("Component", [Processing])
diff --git a/distributed/dashboard/tests/test_scheduler_bokeh.py b/distributed/dashboard/tests/test_scheduler_bokeh.py
index 9c50ae50434..3f8a1f2ce71 100644
--- a/distributed/dashboard/tests/test_scheduler_bokeh.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh.py
@@ -15,32 +15,33 @@
 import dask
 from dask.core import flatten
 from dask.utils import stringify
+
 from distributed.client import wait
 from distributed.compatibility import MACOS
-from distributed.metrics import time
-from distributed.utils import format_dashboard_link
-from distributed.utils_test import gen_cluster, inc, dec, slowinc, div, get_cert
-from distributed.dashboard.components.worker import Counters
-from distributed.dashboard.scheduler import applications
+from distributed.dashboard import scheduler
 from distributed.dashboard.components.scheduler import (
-    SystemMonitor,
+    AggregateAction,
+    ComputePerKey,
+    CurrentLoad,
+    Events,
+    MemoryByKey,
+    NBytesHistogram,
     Occupancy,
-    StealingTimeSeries,
+    ProcessingHistogram,
+    ProfileServer,
     StealingEvents,
-    Events,
-    TaskStream,
+    StealingTimeSeries,
+    SystemMonitor,
+    TaskGraph,
     TaskProgress,
-    CurrentLoad,
-    ProcessingHistogram,
-    NBytesHistogram,
+    TaskStream,
     WorkerTable,
-    TaskGraph,
-    ProfileServer,
-    MemoryByKey,
-    AggregateAction,
-    ComputePerKey,
 )
-from distributed.dashboard import scheduler
+from distributed.dashboard.components.worker import Counters
+from distributed.dashboard.scheduler import applications
+from distributed.metrics import time
+from distributed.utils import format_dashboard_link
+from distributed.utils_test import dec, div, gen_cluster, get_cert, inc, slowinc
 
 scheduler.PROFILING = False
 
diff --git a/distributed/dashboard/tests/test_worker_bokeh.py b/distributed/dashboard/tests/test_worker_bokeh.py
index 6143e837529..b6f5476f618 100644
--- a/distributed/dashboard/tests/test_worker_bokeh.py
+++ b/distributed/dashboard/tests/test_worker_bokeh.py
@@ -10,17 +10,17 @@
 from tornado.httpclient import AsyncHTTPClient
 
 from distributed.client import wait
-from distributed.metrics import time
-from distributed.utils_test import gen_cluster, inc, dec
 from distributed.dashboard.components.worker import (
-    StateTable,
-    CrossFilter,
     CommunicatingStream,
-    ExecutingTimeSeries,
     CommunicatingTimeSeries,
-    SystemMonitor,
     Counters,
+    CrossFilter,
+    ExecutingTimeSeries,
+    StateTable,
+    SystemMonitor,
 )
+from distributed.metrics import time
+from distributed.utils_test import dec, gen_cluster, inc
 
 
 @gen_cluster(
diff --git a/distributed/dashboard/worker.py b/distributed/dashboard/worker.py
index ff9ae3b2f7d..1e65cfcc765 100644
--- a/distributed/dashboard/worker.py
+++ b/distributed/dashboard/worker.py
@@ -1,14 +1,14 @@
+from tornado.ioloop import IOLoop
+
 from .components.worker import (
-    status_doc,
-    crossfilter_doc,
-    systemmonitor_doc,
     counters_doc,
+    crossfilter_doc,
     profile_doc,
     profile_server_doc,
+    status_doc,
+    systemmonitor_doc,
 )
 from .core import BokehApplication
-from tornado.ioloop import IOLoop
-
 
 template_variables = {
     "pages": ["status", "system", "profile", "crossfilter", "profile-server"]
diff --git a/distributed/deploy/__init__.py b/distributed/deploy/__init__.py
index 0148328cd4c..1518942dc4c 100644
--- a/distributed/deploy/__init__.py
+++ b/distributed/deploy/__init__.py
@@ -1,10 +1,10 @@
 from contextlib import suppress
 
+from .adaptive import Adaptive
 from .cluster import Cluster
 from .local import LocalCluster
+from .spec import ProcessInterface, SpecCluster
 from .ssh import SSHCluster
-from .spec import SpecCluster, ProcessInterface
-from .adaptive import Adaptive
 
 with suppress(ImportError):
     from .ssh import SSHCluster
diff --git a/distributed/deploy/adaptive.py b/distributed/deploy/adaptive.py
index d3e9ddb1fbf..ff73015a40a 100644
--- a/distributed/deploy/adaptive.py
+++ b/distributed/deploy/adaptive.py
@@ -1,10 +1,11 @@
-from inspect import isawaitable
 import logging
+from inspect import isawaitable
+
 import dask.config
 
-from .adaptive_core import AdaptiveCore
-from ..utils import log_errors, parse_timedelta
 from ..protocol import pickle
+from ..utils import log_errors, parse_timedelta
+from .adaptive_core import AdaptiveCore
 
 logger = logging.getLogger(__name__)
 
diff --git a/distributed/deploy/adaptive_core.py b/distributed/deploy/adaptive_core.py
index b74e013c947..7c95096956a 100644
--- a/distributed/deploy/adaptive_core.py
+++ b/distributed/deploy/adaptive_core.py
@@ -2,13 +2,12 @@
 import logging
 import math
 
-from tornado.ioloop import IOLoop, PeriodicCallback
 import tlz as toolz
+from tornado.ioloop import IOLoop, PeriodicCallback
 
 from ..metrics import time
 from ..utils import parse_timedelta
 
-
 logger = logging.getLogger(__name__)
 
 
diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
index 93a40cfa0f3..b872e266176 100644
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -1,28 +1,27 @@
 import asyncio
 import datetime
-from contextlib import suppress
 import logging
 import threading
-import warnings
 import uuid
+import warnings
+from contextlib import suppress
+
 from tornado.ioloop import PeriodicCallback
 
 import dask.config
 from dask.utils import format_bytes
 
-from .adaptive import Adaptive
-
 from ..core import Status
 from ..utils import (
-    log_errors,
-    sync,
     Log,
     Logs,
-    thread_state,
     format_dashboard_link,
+    log_errors,
     parse_timedelta,
+    sync,
+    thread_state,
 )
-
+from .adaptive import Adaptive
 
 logger = logging.getLogger(__name__)
 
@@ -310,7 +309,7 @@ def _widget(self):
             pass
 
         try:
-            from ipywidgets import Layout, VBox, HBox, IntText, Button, HTML, Accordion
+            from ipywidgets import HTML, Accordion, Button, HBox, IntText, Layout, VBox
         except ImportError:
             self._cached_widget = None
             return None
diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index a30a6e410f5..7e2ed49b5fc 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -4,15 +4,16 @@
 import warnings
 import weakref
 
-from dask.system import CPU_COUNT
 import toolz
 
-from .spec import SpecCluster
-from .utils import nprocesses_nthreads
+from dask.system import CPU_COUNT
+
 from ..nanny import Nanny
 from ..scheduler import Scheduler
 from ..security import Security
 from ..worker import Worker, parse_memory_limit
+from .spec import SpecCluster
+from .utils import nprocesses_nthreads
 
 logger = logging.getLogger(__name__)
 
diff --git a/distributed/deploy/old_ssh.py b/distributed/deploy/old_ssh.py
index 6d62e0cfd8b..77b01e2388f 100644
--- a/distributed/deploy/old_ssh.py
+++ b/distributed/deploy/old_ssh.py
@@ -1,6 +1,6 @@
 import logging
-import socket
 import os
+import socket
 import sys
 import time
 import traceback
@@ -13,10 +13,8 @@
 from threading import Thread
 
 from tlz import merge
-
 from tornado import gen
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -36,7 +34,7 @@ class bcolors:
 def async_ssh(cmd_dict):
     import paramiko
     from paramiko.buffered_pipe import PipeTimeout
-    from paramiko.ssh_exception import SSHException, PasswordRequiredException
+    from paramiko.ssh_exception import PasswordRequiredException, SSHException
 
     ssh = paramiko.SSHClient()
     ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index d16dd189bb9..3ba421e93f2 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -1,29 +1,29 @@
 import asyncio
 import atexit
-from contextlib import suppress
 import copy
 import logging
 import math
-import weakref
 import warnings
+import weakref
+from contextlib import suppress
 
-import dask
 from tornado import gen
 
-from .adaptive import Adaptive
-from .cluster import Cluster
-from ..core import rpc, CommClosedError, Status
+import dask
+
+from ..core import CommClosedError, Status, rpc
+from ..scheduler import Scheduler
+from ..security import Security
 from ..utils import (
     LoopRunner,
-    silence_logging,
+    TimeoutError,
+    import_term,
     parse_bytes,
     parse_timedelta,
-    import_term,
-    TimeoutError,
+    silence_logging,
 )
-from ..scheduler import Scheduler
-from ..security import Security
-
+from .adaptive import Adaptive
+from .cluster import Cluster
 
 logger = logging.getLogger(__name__)
 
diff --git a/distributed/deploy/ssh.py b/distributed/deploy/ssh.py
index edf44b83660..a70a7ca50c5 100644
--- a/distributed/deploy/ssh.py
+++ b/distributed/deploy/ssh.py
@@ -1,17 +1,17 @@
 import logging
 import sys
-from typing import List, Union
 import warnings
 import weakref
+from typing import List, Union
 
 import dask
 import dask.config
 
-from .spec import SpecCluster, ProcessInterface
 from ..core import Status
-from ..utils import cli_keywords
 from ..scheduler import Scheduler as _Scheduler
+from ..utils import cli_keywords
 from ..worker import Worker as _Worker
+from .spec import ProcessInterface, SpecCluster
 
 logger = logging.getLogger(__name__)
 
diff --git a/distributed/deploy/tests/test_adaptive.py b/distributed/deploy/tests/test_adaptive.py
index e747cf95a20..2e460e5e150 100644
--- a/distributed/deploy/tests/test_adaptive.py
+++ b/distributed/deploy/tests/test_adaptive.py
@@ -3,13 +3,20 @@
 import math
 from time import sleep
 
-import dask
 import pytest
 
-from distributed import Client, wait, Adaptive, LocalCluster, SpecCluster, Worker
-from distributed.utils_test import gen_test, slowinc, clean
-from distributed.utils_test import loop, nodebug, cleanup  # noqa: F401
+import dask
+
+from distributed import Adaptive, Client, LocalCluster, SpecCluster, Worker, wait
 from distributed.metrics import time
+from distributed.utils_test import (  # noqa: F401
+    clean,
+    cleanup,
+    gen_test,
+    loop,
+    nodebug,
+    slowinc,
+)
 
 
 @pytest.mark.asyncio
diff --git a/distributed/deploy/tests/test_adaptive_core.py b/distributed/deploy/tests/test_adaptive_core.py
index a073314223d..b4fc5768f82 100644
--- a/distributed/deploy/tests/test_adaptive_core.py
+++ b/distributed/deploy/tests/test_adaptive_core.py
@@ -1,4 +1,5 @@
 import asyncio
+
 import pytest
 
 from distributed.deploy.adaptive_core import AdaptiveCore
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index 6e24d77f120..bd67581385e 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -1,42 +1,42 @@
 import asyncio
-from functools import partial
 import gc
 import subprocess
 import sys
-from time import sleep
-from threading import Lock
 import unittest
 import weakref
 from distutils.version import LooseVersion
+from functools import partial
+from threading import Lock
+from time import sleep
 
-from tornado.ioloop import IOLoop
+import pytest
 import tornado
 from tornado.httpclient import AsyncHTTPClient
-import pytest
+from tornado.ioloop import IOLoop
 
 from dask.system import CPU_COUNT
-from distributed import Client, Worker, Nanny, get_client
+
+from distributed import Client, Nanny, Worker, get_client
 from distributed.core import Status
 from distributed.deploy.local import LocalCluster
+from distributed.deploy.utils_test import ClusterTest
 from distributed.metrics import time
 from distributed.system import MEMORY_LIMIT
+from distributed.utils import TimeoutError, sync
 from distributed.utils_test import (  # noqa: F401
+    assert_can_connect_from_everywhere_4,
+    assert_can_connect_from_everywhere_4_6,
+    assert_can_connect_locally_4,
+    assert_cannot_connect,
+    captured_logger,
     clean,
     cleanup,
-    inc,
     gen_test,
+    inc,
+    loop,
     slowinc,
-    assert_cannot_connect,
-    assert_can_connect_locally_4,
-    assert_can_connect_from_everywhere_4,
-    assert_can_connect_from_everywhere_4_6,
-    captured_logger,
     tls_only_security,
 )
-from distributed.utils_test import loop  # noqa: F401
-from distributed.utils import sync, TimeoutError
-
-from distributed.deploy.utils_test import ClusterTest
 
 
 def test_simple(loop):
diff --git a/distributed/deploy/tests/test_slow_adaptive.py b/distributed/deploy/tests/test_slow_adaptive.py
index e7021fc854a..8a8919c7eea 100644
--- a/distributed/deploy/tests/test_slow_adaptive.py
+++ b/distributed/deploy/tests/test_slow_adaptive.py
@@ -1,9 +1,11 @@
 import asyncio
+
 import pytest
 
-from dask.distributed import Worker, Scheduler, SpecCluster, Client
-from distributed.utils_test import slowinc, cleanup  # noqa: F401
+from dask.distributed import Client, Scheduler, SpecCluster, Worker
+
 from distributed.metrics import time
+from distributed.utils_test import cleanup, slowinc  # noqa: F401
 
 
 class SlowWorker:
diff --git a/distributed/deploy/tests/test_spec_cluster.py b/distributed/deploy/tests/test_spec_cluster.py
index 41e9e9158f9..c1798fd580b 100644
--- a/distributed/deploy/tests/test_spec_cluster.py
+++ b/distributed/deploy/tests/test_spec_cluster.py
@@ -1,18 +1,20 @@
 import asyncio
 import re
-from time import sleep
 import warnings
+from time import sleep
+
+import pytest
+import tlz as toolz
 
 import dask
-from dask.distributed import SpecCluster, Worker, Client, Scheduler, Nanny
-from distributed.core import Status
+from dask.distributed import Client, Nanny, Scheduler, SpecCluster, Worker
+
 from distributed.compatibility import WINDOWS
-from distributed.deploy.spec import close_clusters, ProcessInterface, run_spec
+from distributed.core import Status
+from distributed.deploy.spec import ProcessInterface, close_clusters, run_spec
 from distributed.metrics import time
-from distributed.utils_test import loop, cleanup  # noqa: F401
 from distributed.utils import is_valid_xml
-import tlz as toolz
-import pytest
+from distributed.utils_test import cleanup, loop  # noqa: F401
 
 
 class MyWorker(Worker):
diff --git a/distributed/deploy/tests/test_ssh.py b/distributed/deploy/tests/test_ssh.py
index 0bea6f7dc75..6c330383d7d 100644
--- a/distributed/deploy/tests/test_ssh.py
+++ b/distributed/deploy/tests/test_ssh.py
@@ -3,7 +3,9 @@
 pytest.importorskip("asyncssh")
 
 import sys
+
 import dask
+
 from distributed import Client
 from distributed.compatibility import MACOS, WINDOWS
 from distributed.deploy.ssh import SSHCluster
diff --git a/distributed/deploy/utils_test.py b/distributed/deploy/utils_test.py
index fd6ba03aae9..543020ec21b 100644
--- a/distributed/deploy/utils_test.py
+++ b/distributed/deploy/utils_test.py
@@ -1,7 +1,7 @@
-from ..client import Client
-
 import pytest
 
+from ..client import Client
+
 
 class ClusterTest:
     Cluster = None
diff --git a/distributed/diagnostics/eventstream.py b/distributed/diagnostics/eventstream.py
index f1f70f458af..1f98e47c64b 100644
--- a/distributed/diagnostics/eventstream.py
+++ b/distributed/diagnostics/eventstream.py
@@ -1,10 +1,8 @@
 import logging
 
-from .plugin import SchedulerPlugin
-
-from ..core import connect, coerce_to_address
+from ..core import coerce_to_address, connect
 from ..worker import dumps_function
-
+from .plugin import SchedulerPlugin
 
 logger = logging.getLogger(__name__)
 
diff --git a/distributed/diagnostics/nvml.py b/distributed/diagnostics/nvml.py
index c1bbb4161a8..cd39f9e04c5 100644
--- a/distributed/diagnostics/nvml.py
+++ b/distributed/diagnostics/nvml.py
@@ -1,4 +1,5 @@
 import os
+
 import pynvml
 
 nvmlInit = None
diff --git a/distributed/diagnostics/progress.py b/distributed/diagnostics/progress.py
index eeb3c8a2817..f71d56d8bf4 100644
--- a/distributed/diagnostics/progress.py
+++ b/distributed/diagnostics/progress.py
@@ -1,14 +1,14 @@
 import asyncio
-from collections import defaultdict
 import logging
+from collections import defaultdict
 from timeit import default_timer
 
 from tlz import groupby, valmap
 
 from dask.utils import stringify
-from .plugin import SchedulerPlugin
-from ..utils import key_split, key_split_group, log_errors
 
+from ..utils import key_split, key_split_group, log_errors
+from .plugin import SchedulerPlugin
 
 logger = logging.getLogger(__name__)
 
diff --git a/distributed/diagnostics/progress_stream.py b/distributed/diagnostics/progress_stream.py
index b17c38fc1b6..2ee2b1c5000 100644
--- a/distributed/diagnostics/progress_stream.py
+++ b/distributed/diagnostics/progress_stream.py
@@ -1,14 +1,12 @@
 import logging
 
-from tlz import valmap, merge
+from tlz import merge, valmap
 
-from .progress import AllProgress
-
-from ..core import connect, coerce_to_address
+from ..core import coerce_to_address, connect
 from ..scheduler import Scheduler
-from ..utils import key_split, color_of
+from ..utils import color_of, key_split
 from ..worker import dumps_function
-
+from .progress import AllProgress
 
 logger = logging.getLogger(__name__)
 
diff --git a/distributed/diagnostics/progressbar.py b/distributed/diagnostics/progressbar.py
index 45884bd5b1c..851910e45d4 100644
--- a/distributed/diagnostics/progressbar.py
+++ b/distributed/diagnostics/progressbar.py
@@ -1,20 +1,18 @@
-from contextlib import suppress
-import logging
 import html
-from timeit import default_timer
+import logging
 import sys
 import weakref
+from contextlib import suppress
+from timeit import default_timer
 
 from tlz import valmap
 from tornado.ioloop import IOLoop
 
-from .progress import format_time, Progress, MultiProgress
-
-from ..core import connect, coerce_to_address, CommClosedError
 from ..client import default_client, futures_of
+from ..core import CommClosedError, coerce_to_address, connect
 from ..protocol.pickle import dumps
-from ..utils import key_split, is_kernel, LoopRunner, parse_timedelta
-
+from ..utils import LoopRunner, is_kernel, key_split, parse_timedelta
+from .progress import MultiProgress, Progress, format_time
 
 logger = logging.getLogger(__name__)
 
@@ -160,7 +158,7 @@ def __init__(
     ):
         super().__init__(keys, scheduler, interval, complete)
 
-        from ipywidgets import FloatProgress, HBox, VBox, HTML
+        from ipywidgets import HTML, FloatProgress, HBox, VBox
 
         self.elapsed_time = HTML("")
         self.bar = FloatProgress(min=0, max=1, description="")
@@ -319,7 +317,7 @@ def __init__(
         self.widget = VBox([])
 
     def make_widget(self, all):
-        from ipywidgets import FloatProgress, HBox, VBox, HTML
+        from ipywidgets import HTML, FloatProgress, HBox, VBox
 
         self.elapsed_time = HTML("")
         self.bars = {key: FloatProgress(min=0, max=1, description="") for key in all}
diff --git a/distributed/diagnostics/task_stream.py b/distributed/diagnostics/task_stream.py
index c319ca73d69..e276c477776 100644
--- a/distributed/diagnostics/task_stream.py
+++ b/distributed/diagnostics/task_stream.py
@@ -1,12 +1,12 @@
-from collections import deque
 import logging
+from collections import deque
 
 import dask
-from .progress_stream import color_of
-from .plugin import SchedulerPlugin
-from ..utils import key_split, format_time, parse_timedelta
-from ..metrics import time
 
+from ..metrics import time
+from ..utils import format_time, key_split, parse_timedelta
+from .plugin import SchedulerPlugin
+from .progress_stream import color_of
 
 logger = logging.getLogger(__name__)
 
diff --git a/distributed/diagnostics/tests/test_graph_layout.py b/distributed/diagnostics/tests/test_graph_layout.py
index b63311f8432..b714b261a4c 100644
--- a/distributed/diagnostics/tests/test_graph_layout.py
+++ b/distributed/diagnostics/tests/test_graph_layout.py
@@ -1,9 +1,9 @@
 import asyncio
 import operator
 
-from distributed.utils_test import gen_cluster, inc
-from distributed.diagnostics import GraphLayout
 from distributed import wait
+from distributed.diagnostics import GraphLayout
+from distributed.utils_test import gen_cluster, inc
 
 
 @gen_cluster(client=True)
diff --git a/distributed/diagnostics/tests/test_nvml.py b/distributed/diagnostics/tests/test_nvml.py
index 6182049fe40..6938a86850f 100644
--- a/distributed/diagnostics/tests/test_nvml.py
+++ b/distributed/diagnostics/tests/test_nvml.py
@@ -1,6 +1,7 @@
-import pytest
 import os
 
+import pytest
+
 pynvml = pytest.importorskip("pynvml")
 
 from distributed.diagnostics import nvml
diff --git a/distributed/diagnostics/tests/test_progress.py b/distributed/diagnostics/tests/test_progress.py
index 871dcb0c5a5..fa74bfa3b46 100644
--- a/distributed/diagnostics/tests/test_progress.py
+++ b/distributed/diagnostics/tests/test_progress.py
@@ -4,15 +4,15 @@
 
 from distributed import Nanny
 from distributed.client import wait
-from distributed.metrics import time
-from distributed.utils_test import gen_cluster, inc, dec, div, nodebug
 from distributed.diagnostics.progress import (
-    Progress,
-    SchedulerPlugin,
     AllProgress,
     GroupProgress,
     MultiProgress,
+    Progress,
+    SchedulerPlugin,
 )
+from distributed.metrics import time
+from distributed.utils_test import dec, div, gen_cluster, inc, nodebug
 
 
 def f(*args):
diff --git a/distributed/diagnostics/tests/test_progress_stream.py b/distributed/diagnostics/tests/test_progress_stream.py
index 8f506b7a7bb..d65d953e6b5 100644
--- a/distributed/diagnostics/tests/test_progress_stream.py
+++ b/distributed/diagnostics/tests/test_progress_stream.py
@@ -3,6 +3,7 @@
 pytest.importorskip("bokeh")
 
 from dask import delayed
+
 from distributed.client import wait
 from distributed.diagnostics.progress_stream import progress_quads, progress_stream
 from distributed.utils_test import div, gen_cluster, inc
diff --git a/distributed/diagnostics/tests/test_progressbar.py b/distributed/diagnostics/tests/test_progressbar.py
index f19dbd2df26..36db0d7b45b 100644
--- a/distributed/diagnostics/tests/test_progressbar.py
+++ b/distributed/diagnostics/tests/test_progressbar.py
@@ -5,8 +5,14 @@
 from distributed import Scheduler, Worker
 from distributed.diagnostics.progressbar import TextProgressBar, progress
 from distributed.metrics import time
-from distributed.utils_test import inc, div, gen_cluster
-from distributed.utils_test import client, loop, cluster_fixture  # noqa: F401
+from distributed.utils_test import (  # noqa: F401
+    client,
+    cluster_fixture,
+    div,
+    gen_cluster,
+    inc,
+    loop,
+)
 
 
 def test_text_progressbar(capsys, client):
diff --git a/distributed/diagnostics/tests/test_scheduler_plugin.py b/distributed/diagnostics/tests/test_scheduler_plugin.py
index 3f1e54f6ed7..465b674971f 100644
--- a/distributed/diagnostics/tests/test_scheduler_plugin.py
+++ b/distributed/diagnostics/tests/test_scheduler_plugin.py
@@ -1,6 +1,7 @@
 import pytest
-from distributed import Scheduler, Worker, SchedulerPlugin
-from distributed.utils_test import inc, gen_cluster, cleanup  # noqa: F401
+
+from distributed import Scheduler, SchedulerPlugin, Worker
+from distributed.utils_test import cleanup, gen_cluster, inc  # noqa: F401
 
 
 @gen_cluster(client=True)
diff --git a/distributed/diagnostics/tests/test_task_stream.py b/distributed/diagnostics/tests/test_task_stream.py
index 4b57d18ee7a..642277e81dc 100644
--- a/distributed/diagnostics/tests/test_task_stream.py
+++ b/distributed/diagnostics/tests/test_task_stream.py
@@ -5,11 +5,18 @@
 from tlz import frequencies
 
 from distributed import get_task_stream
-from distributed.utils_test import gen_cluster, div, inc, slowinc
-from distributed.utils_test import client, loop, cluster_fixture  # noqa: F401
 from distributed.client import wait
 from distributed.diagnostics.task_stream import TaskStreamPlugin
 from distributed.metrics import time
+from distributed.utils_test import (  # noqa: F401
+    client,
+    cluster_fixture,
+    div,
+    gen_cluster,
+    inc,
+    loop,
+    slowinc,
+)
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 3)
diff --git a/distributed/diagnostics/tests/test_widgets.py b/distributed/diagnostics/tests/test_widgets.py
index 2505f986406..bc86e436b24 100644
--- a/distributed/diagnostics/tests/test_widgets.py
+++ b/distributed/diagnostics/tests/test_widgets.py
@@ -2,10 +2,11 @@
 
 pytest.importorskip("ipywidgets")
 
-from distributed.compatibility import WINDOWS
 from ipykernel.comm import Comm
 from ipywidgets import Widget
 
+from distributed.compatibility import WINDOWS
+
 #################
 # Utility stuff #
 #################
@@ -72,20 +73,28 @@ def record_display(*args):
 # Distributed stuff #
 #####################
 
-from operator import add
 import re
+from operator import add
 
 from tlz import valmap
 
 from distributed.client import wait
-from distributed.worker import dumps_task
-from distributed.utils_test import inc, dec, throws, gen_cluster, gen_tls_cluster
-from distributed.utils_test import client, loop, cluster_fixture  # noqa: F401
 from distributed.diagnostics.progressbar import (
-    ProgressWidget,
     MultiProgressWidget,
+    ProgressWidget,
     progress,
 )
+from distributed.utils_test import (  # noqa: F401
+    client,
+    cluster_fixture,
+    dec,
+    gen_cluster,
+    gen_tls_cluster,
+    inc,
+    loop,
+    throws,
+)
+from distributed.worker import dumps_task
 
 
 @gen_cluster(client=True)
diff --git a/distributed/diagnostics/websocket.py b/distributed/diagnostics/websocket.py
index e34961bfeff..51282c1e621 100644
--- a/distributed/diagnostics/websocket.py
+++ b/distributed/diagnostics/websocket.py
@@ -1,5 +1,5 @@
-from .plugin import SchedulerPlugin
 from ..utils import key_split
+from .plugin import SchedulerPlugin
 from .task_stream import colors
 
 
diff --git a/distributed/diskutils.py b/distributed/diskutils.py
index e9dbb7b25bf..49d0a26222c 100644
--- a/distributed/diskutils.py
+++ b/distributed/diskutils.py
@@ -11,7 +11,6 @@
 
 from . import locket
 
-
 logger = logging.getLogger(__name__)
 
 DIR_LOCK_EXT = ".dirlock"
diff --git a/distributed/event.py b/distributed/event.py
index 0136d35ef26..a3e2a1b7eeb 100644
--- a/distributed/event.py
+++ b/distributed/event.py
@@ -1,13 +1,12 @@
 import asyncio
-from collections import defaultdict
-from contextlib import suppress
 import logging
 import uuid
+from collections import defaultdict
+from contextlib import suppress
 
 from .client import Client
-from .utils import log_errors, TimeoutError
+from .utils import TimeoutError, log_errors, parse_timedelta
 from .worker import get_worker
-from .utils import parse_timedelta
 
 logger = logging.getLogger(__name__)
 
diff --git a/distributed/http/routing.py b/distributed/http/routing.py
index 8a1d90d5490..7de870faa72 100644
--- a/distributed/http/routing.py
+++ b/distributed/http/routing.py
@@ -1,7 +1,8 @@
 import os
-from tornado import web
+
 import tornado.httputil
 import tornado.routing
+from tornado import web
 
 
 def _descend_routes(router, routers=set(), out=set()):
diff --git a/distributed/http/scheduler/info.py b/distributed/http/scheduler/info.py
index 96199faba38..588fbc1505f 100644
--- a/distributed/http/scheduler/info.py
+++ b/distributed/http/scheduler/info.py
@@ -1,19 +1,19 @@
-from datetime import datetime
 import json
 import logging
 import os
 import os.path
+from datetime import datetime
 
-from dask.utils import format_bytes
-
+from tlz import first, merge
 from tornado import escape
 from tornado.websocket import WebSocketHandler
-from tlz import first, merge
 
-from ..utils import RequestHandler, redirect
+from dask.utils import format_bytes
+
 from ...diagnostics.websocket import WebsocketPlugin
 from ...metrics import time
-from ...utils import log_errors, format_time
+from ...utils import format_time, log_errors
+from ..utils import RequestHandler, redirect
 
 ns = {
     func.__name__: func
diff --git a/distributed/http/scheduler/json.py b/distributed/http/scheduler/json.py
index 5dc09b4b6fe..61801a00d34 100644
--- a/distributed/http/scheduler/json.py
+++ b/distributed/http/scheduler/json.py
@@ -1,5 +1,5 @@
-from ..utils import RequestHandler
 from ...utils import log_errors
+from ..utils import RequestHandler
 
 
 class CountsJSON(RequestHandler):
diff --git a/distributed/http/scheduler/missing_bokeh.py b/distributed/http/scheduler/missing_bokeh.py
index 3eb68960d53..917e79f610b 100644
--- a/distributed/http/scheduler/missing_bokeh.py
+++ b/distributed/http/scheduler/missing_bokeh.py
@@ -1,5 +1,5 @@
-from ..utils import RequestHandler, redirect
 from ...utils import log_errors
+from ..utils import RequestHandler, redirect
 
 
 class MissingBokeh(RequestHandler):
diff --git a/distributed/http/scheduler/prometheus/__init__.py b/distributed/http/scheduler/prometheus/__init__.py
index 2016ca32512..120a01dab58 100644
--- a/distributed/http/scheduler/prometheus/__init__.py
+++ b/distributed/http/scheduler/prometheus/__init__.py
@@ -2,6 +2,7 @@
 
 from distributed.http.utils import RequestHandler
 from distributed.scheduler import ALL_TASK_STATES
+
 from .semaphore import SemaphoreMetricExtension
 
 
@@ -10,7 +11,7 @@ def __init__(self, dask_server):
         self.server = dask_server
 
     def collect(self):
-        from prometheus_client.core import GaugeMetricFamily, CounterMetricFamily
+        from prometheus_client.core import CounterMetricFamily, GaugeMetricFamily
 
         yield GaugeMetricFamily(
             "dask_scheduler_clients",
diff --git a/distributed/http/scheduler/prometheus/semaphore.py b/distributed/http/scheduler/prometheus/semaphore.py
index f1df7434019..aac467b66cc 100644
--- a/distributed/http/scheduler/prometheus/semaphore.py
+++ b/distributed/http/scheduler/prometheus/semaphore.py
@@ -3,7 +3,7 @@ def __init__(self, dask_server):
         self.server = dask_server
 
     def collect(self):
-        from prometheus_client.core import GaugeMetricFamily, CounterMetricFamily
+        from prometheus_client.core import CounterMetricFamily, GaugeMetricFamily
 
         sem_ext = self.server.extensions["semaphores"]
 
diff --git a/distributed/http/scheduler/tests/test_scheduler_http.py b/distributed/http/scheduler/tests/test_scheduler_http.py
index 2aa4f58c4ea..0582b65dccd 100644
--- a/distributed/http/scheduler/tests/test_scheduler_http.py
+++ b/distributed/http/scheduler/tests/test_scheduler_http.py
@@ -9,8 +9,9 @@
 from tornado.httpclient import AsyncHTTPClient, HTTPClientError
 
 from dask.sizeof import sizeof
+
 from distributed.utils import is_valid_xml
-from distributed.utils_test import gen_cluster, slowinc, inc
+from distributed.utils_test import gen_cluster, inc, slowinc
 
 
 @gen_cluster(client=True)
diff --git a/distributed/http/scheduler/tests/test_semaphore_http.py b/distributed/http/scheduler/tests/test_semaphore_http.py
index 4c66165b985..21996cb35f4 100644
--- a/distributed/http/scheduler/tests/test_semaphore_http.py
+++ b/distributed/http/scheduler/tests/test_semaphore_http.py
@@ -1,9 +1,8 @@
 import pytest
-
 from tornado.httpclient import AsyncHTTPClient
 
-from distributed.utils_test import gen_cluster
 from distributed import Semaphore
+from distributed.utils_test import gen_cluster
 
 
 @gen_cluster(client=True, clean_kwargs={"threads": False})
diff --git a/distributed/http/statics.py b/distributed/http/statics.py
index e1c7a98e9a2..f838ac8522e 100644
--- a/distributed/http/statics.py
+++ b/distributed/http/statics.py
@@ -1,6 +1,7 @@
-from tornado import web
 import os
 
+from tornado import web
+
 routes = [
     (
         r"/statics/(.*)",
diff --git a/distributed/http/tests/test_core.py b/distributed/http/tests/test_core.py
index c1bffedb72e..61cb713fcf2 100644
--- a/distributed/http/tests/test_core.py
+++ b/distributed/http/tests/test_core.py
@@ -1,6 +1,7 @@
-from distributed.utils_test import gen_cluster
 from tornado.httpclient import AsyncHTTPClient
 
+from distributed.utils_test import gen_cluster
+
 
 @gen_cluster(client=True)
 async def test_scheduler(c, s, a, b):
diff --git a/distributed/http/tests/test_routing.py b/distributed/http/tests/test_routing.py
index ca7d071d256..764475ce4ed 100644
--- a/distributed/http/tests/test_routing.py
+++ b/distributed/http/tests/test_routing.py
@@ -1,6 +1,6 @@
+import pytest
 from tornado import web
 from tornado.httpclient import AsyncHTTPClient, HTTPClientError
-import pytest
 
 from distributed.http.routing import RoutingApplication
 
diff --git a/distributed/http/utils.py b/distributed/http/utils.py
index 5977ccd5bad..c0de926c99b 100644
--- a/distributed/http/utils.py
+++ b/distributed/http/utils.py
@@ -2,12 +2,11 @@
 import os
 from typing import List
 
-from tornado import web
 import toolz
+from tornado import web
 
 from ..utils import has_keyword
 
-
 dirname = os.path.dirname(__file__)
 
 
diff --git a/distributed/http/worker/prometheus.py b/distributed/http/worker/prometheus.py
index b354cad3ea9..4d0c0a55e60 100644
--- a/distributed/http/worker/prometheus.py
+++ b/distributed/http/worker/prometheus.py
@@ -1,7 +1,7 @@
-from ..utils import RequestHandler
-
 import logging
 
+from ..utils import RequestHandler
+
 
 class _PrometheusCollector:
     def __init__(self, server):
diff --git a/distributed/http/worker/tests/test_worker_http.py b/distributed/http/worker/tests/test_worker_http.py
index 5bc0a5debc9..e464c484ebe 100644
--- a/distributed/http/worker/tests/test_worker_http.py
+++ b/distributed/http/worker/tests/test_worker_http.py
@@ -1,6 +1,8 @@
-import pytest
 import json
+
+import pytest
 from tornado.httpclient import AsyncHTTPClient
+
 from distributed.utils_test import gen_cluster
 
 
diff --git a/distributed/lock.py b/distributed/lock.py
index 7d1c1a4af57..7ffe8c8daf9 100644
--- a/distributed/lock.py
+++ b/distributed/lock.py
@@ -1,12 +1,11 @@
 import asyncio
-from collections import defaultdict, deque
 import logging
 import uuid
+from collections import defaultdict, deque
 
 from .client import Client
-from .utils import log_errors, TimeoutError
+from .utils import TimeoutError, log_errors, parse_timedelta
 from .worker import get_worker
-from .utils import parse_timedelta
 
 logger = logging.getLogger(__name__)
 
diff --git a/distributed/locket.py b/distributed/locket.py
index 65a10f195f7..906938e6085 100644
--- a/distributed/locket.py
+++ b/distributed/locket.py
@@ -3,9 +3,9 @@
 
 # flake8: noqa
 
-import time
 import errno
 import threading
+import time
 import weakref
 
 __all__ = ["lock_file"]
diff --git a/distributed/metrics.py b/distributed/metrics.py
index 163a982b792..c52b40c5afd 100755
--- a/distributed/metrics.py
+++ b/distributed/metrics.py
@@ -1,8 +1,7 @@
 import collections
-from functools import wraps
 import sys
 import time as timemod
-
+from functools import wraps
 
 _empty_namedtuple = collections.namedtuple("_empty_namedtuple", ())
 
diff --git a/distributed/multi_lock.py b/distributed/multi_lock.py
index 6d1df68c2f4..aaa21999a19 100644
--- a/distributed/multi_lock.py
+++ b/distributed/multi_lock.py
@@ -1,13 +1,12 @@
 import asyncio
-from collections import defaultdict
 import logging
-from typing import Hashable, List
 import uuid
+from collections import defaultdict
+from typing import Hashable, List
 
 from .client import Client
-from .utils import log_errors, TimeoutError
+from .utils import TimeoutError, log_errors, parse_timedelta
 from .worker import get_worker
-from .utils import parse_timedelta
 
 logger = logging.getLogger(__name__)
 
diff --git a/distributed/nanny.py b/distributed/nanny.py
index f93f18522ca..5c25c3297ea 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -1,41 +1,41 @@
 import asyncio
-from contextlib import suppress
 import errno
 import logging
-from multiprocessing.queues import Empty
 import os
-import psutil
 import shutil
 import threading
 import uuid
 import warnings
 import weakref
+from contextlib import suppress
+from multiprocessing.queues import Empty
+
+import psutil
+from tornado import gen
+from tornado.ioloop import IOLoop, PeriodicCallback
 
 import dask
 from dask.system import CPU_COUNT
-from tornado.ioloop import IOLoop, PeriodicCallback
-from tornado import gen
 
+from . import preloading
 from .comm import get_address_host, unparse_host_port
 from .comm.addressing import address_from_user_args
-from .core import RPCClosed, CommClosedError, coerce_to_address, Status
+from .core import CommClosedError, RPCClosed, Status, coerce_to_address
 from .metrics import time
 from .node import ServerNode
-from . import preloading
 from .process import AsyncProcess
 from .proctitle import enable_proctitle_on_children
 from .security import Security
 from .utils import (
+    TimeoutError,
     get_ip,
-    mp_context,
-    silence_logging,
     json_load_robust,
-    parse_timedelta,
+    mp_context,
     parse_ports,
-    TimeoutError,
+    parse_timedelta,
+    silence_logging,
 )
-from .worker import run, parse_memory_limit, Worker
-
+from .worker import Worker, parse_memory_limit, run
 
 logger = logging.getLogger(__name__)
 
diff --git a/distributed/node.py b/distributed/node.py
index 997e6a1a988..21000a04733 100644
--- a/distributed/node.py
+++ b/distributed/node.py
@@ -1,18 +1,18 @@
-from contextlib import suppress
 import logging
 import warnings
 import weakref
+from contextlib import suppress
 
-from tornado.httpserver import HTTPServer
 import tlz
+from tornado.httpserver import HTTPServer
+
 import dask
 
-from .comm import get_tcp_server_address
-from .comm import get_address_host
+from .comm import get_address_host, get_tcp_server_address
 from .core import Server
 from .http.routing import RoutingApplication
-from .versions import get_versions
 from .utils import DequeHandler, clean_dashboard_address
+from .versions import get_versions
 
 
 class ServerNode(Server):
diff --git a/distributed/preloading.py b/distributed/preloading.py
index dbbe42d57d0..9108a5c9090 100644
--- a/distributed/preloading.py
+++ b/distributed/preloading.py
@@ -1,12 +1,12 @@
+import filecmp
 import inspect
 import logging
 import os
 import shutil
 import sys
-from typing import List
-from types import ModuleType
-import filecmp
 from importlib import import_module
+from types import ModuleType
+from typing import List
 
 import click
 from tornado.httpclient import AsyncHTTPClient
diff --git a/distributed/process.py b/distributed/process.py
index 1c11dd2e3d7..862ea4edeb8 100644
--- a/distributed/process.py
+++ b/distributed/process.py
@@ -1,18 +1,18 @@
+import asyncio
 import logging
 import os
-from queue import Queue as PyQueue
 import re
 import threading
 import weakref
-import asyncio
-import dask
-
-from .utils import mp_context, TimeoutError
+from queue import Queue as PyQueue
 
 from tornado import gen
 from tornado.concurrent import Future
 from tornado.ioloop import IOLoop
 
+import dask
+
+from .utils import TimeoutError, mp_context
 
 logger = logging.getLogger(__name__)
 
diff --git a/distributed/profile.py b/distributed/profile.py
index 78e1765d041..fbaf6ea6599 100644
--- a/distributed/profile.py
+++ b/distributed/profile.py
@@ -25,16 +25,16 @@
     }
 """
 import bisect
-from collections import defaultdict, deque
 import linecache
 import sys
 import threading
+from collections import defaultdict, deque
 from time import sleep
 
 import tlz as toolz
 
 from .metrics import time
-from .utils import format_time, color_of, parse_timedelta
+from .utils import color_of, format_time, parse_timedelta
 
 
 def identifier(frame):
@@ -375,8 +375,8 @@ def plot_figure(data, **kwargs):
     --------
     plot_data
     """
-    from bokeh.plotting import ColumnDataSource, figure
     from bokeh.models import HoverTool
+    from bokeh.plotting import ColumnDataSource, figure
 
     if "states" in data:
         data = toolz.dissoc(data, "states")
diff --git a/distributed/protocol/__init__.py b/distributed/protocol/__init__.py
index be1c498c35c..36aa3f42c12 100644
--- a/distributed/protocol/__init__.py
+++ b/distributed/protocol/__init__.py
@@ -1,25 +1,25 @@
 from contextlib import suppress
-from functools import partial
 from distutils.version import LooseVersion
+from functools import partial
 
 from .compression import compressions, default_compression
-from .core import dumps, loads, maybe_compress, decompress, msgpack
-from .cuda import cuda_serialize, cuda_deserialize
+from .core import decompress, dumps, loads, maybe_compress, msgpack
+from .cuda import cuda_deserialize, cuda_serialize
 from .serialize import (
-    serialize,
-    deserialize,
-    nested_deserialize,
     Serialize,
     Serialized,
-    to_serialize,
-    register_serialization,
-    dask_serialize,
     dask_deserialize,
-    serialize_bytes,
+    dask_serialize,
+    deserialize,
     deserialize_bytes,
-    serialize_bytelist,
-    register_serialization_family,
+    nested_deserialize,
     register_generic,
+    register_serialization,
+    register_serialization_family,
+    serialize,
+    serialize_bytelist,
+    serialize_bytes,
+    to_serialize,
 )
 
 
diff --git a/distributed/protocol/arrow.py b/distributed/protocol/arrow.py
index 1f2b4e83e9a..2850c47466e 100644
--- a/distributed/protocol/arrow.py
+++ b/distributed/protocol/arrow.py
@@ -1,7 +1,7 @@
-from .serialize import dask_serialize, dask_deserialize
-
 import pyarrow
 
+from .serialize import dask_deserialize, dask_serialize
+
 if pyarrow.__version__ < "0.10":
     raise ImportError(
         "Need pyarrow >= 0.10 . "
diff --git a/distributed/protocol/compression.py b/distributed/protocol/compression.py
index b067ae0e526..800f79adb90 100644
--- a/distributed/protocol/compression.py
+++ b/distributed/protocol/compression.py
@@ -3,14 +3,15 @@
 
 Includes utilities for determining whether or not to compress
 """
-from contextlib import suppress
-from functools import partial
 import logging
 import random
+from contextlib import suppress
+from functools import partial
 
-import dask
 from tlz import identity
 
+import dask
+
 try:
     import blosc
 
@@ -22,7 +23,6 @@
 
 from ..utils import ensure_bytes
 
-
 compressions = {None: {"compress": identity, "decompress": identity}}
 
 compressions[False] = compressions[None]  # alias
diff --git a/distributed/protocol/core.py b/distributed/protocol/core.py
index 4235dca9f9c..05a804d3b52 100644
--- a/distributed/protocol/core.py
+++ b/distributed/protocol/core.py
@@ -1,18 +1,18 @@
 import logging
+
 import msgpack
 
-from .compression import compressions, maybe_compress, decompress
+from .compression import compressions, decompress, maybe_compress
 from .serialize import (
     Serialize,
     Serialized,
+    merge_and_deserialize,
     msgpack_decode_default,
     msgpack_encode_default,
-    merge_and_deserialize,
     serialize_and_split,
 )
 from .utils import msgpack_opts
 
-
 logger = logging.getLogger(__name__)
 
 
diff --git a/distributed/protocol/cuda.py b/distributed/protocol/cuda.py
index 44ed6a033df..572b63547f5 100644
--- a/distributed/protocol/cuda.py
+++ b/distributed/protocol/cuda.py
@@ -1,8 +1,8 @@
 import dask
+from dask.utils import typename
 
 from . import pickle
 from .serialize import ObjectDictSerializer, register_serialization_family
-from dask.utils import typename
 
 cuda_serialize = dask.utils.Dispatch("cuda_serialize")
 cuda_deserialize = dask.utils.Dispatch("cuda_deserialize")
diff --git a/distributed/protocol/h5py.py b/distributed/protocol/h5py.py
index e129c166683..8a47c7abdc6 100644
--- a/distributed/protocol/h5py.py
+++ b/distributed/protocol/h5py.py
@@ -1,7 +1,7 @@
-from .serialize import dask_serialize, dask_deserialize
-
 import h5py
 
+from .serialize import dask_deserialize, dask_serialize
+
 
 @dask_serialize.register(h5py.File)
 def serialize_h5py_file(f):
diff --git a/distributed/protocol/keras.py b/distributed/protocol/keras.py
index 121aa0c4700..c2c24e3992e 100644
--- a/distributed/protocol/keras.py
+++ b/distributed/protocol/keras.py
@@ -1,7 +1,7 @@
-from .serialize import dask_serialize, dask_deserialize, serialize, deserialize
-
 import keras
 
+from .serialize import dask_deserialize, dask_serialize, deserialize, serialize
+
 
 @dask_serialize.register(keras.Model)
 def serialize_keras_model(model):
diff --git a/distributed/protocol/netcdf4.py b/distributed/protocol/netcdf4.py
index eb83461eddc..d3d0b1e2c0c 100644
--- a/distributed/protocol/netcdf4.py
+++ b/distributed/protocol/netcdf4.py
@@ -1,7 +1,7 @@
-from .serialize import dask_serialize, dask_deserialize, serialize, deserialize
-
 import netCDF4
 
+from .serialize import dask_deserialize, dask_serialize, deserialize, serialize
+
 
 @dask_serialize.register(netCDF4.Dataset)
 def serialize_netcdf4_dataset(ds):
diff --git a/distributed/protocol/numpy.py b/distributed/protocol/numpy.py
index 03064581d8b..a416297afa8 100644
--- a/distributed/protocol/numpy.py
+++ b/distributed/protocol/numpy.py
@@ -1,10 +1,10 @@
 import math
-import numpy as np
 
-from .serialize import dask_serialize, dask_deserialize
-from . import pickle
+import numpy as np
 
 from ..utils import log_errors
+from . import pickle
+from .serialize import dask_deserialize, dask_serialize
 
 
 def itemsize(dt):
diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index 49a2fdab09d..3179784c2a8 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -1,24 +1,18 @@
-from array import array
-from functools import partial
-import traceback
 import importlib
+import traceback
+from array import array
 from enum import Enum
+from functools import partial
+
+import msgpack
 
 import dask
 from dask.base import normalize_token
 
-import msgpack
-
+from ..utils import ensure_bytes, has_keyword, typename
 from . import pickle
-from ..utils import has_keyword, typename, ensure_bytes
-from .compression import maybe_compress, decompress
-from .utils import (
-    unpack_frames,
-    pack_frames_prelude,
-    frame_split_size,
-    msgpack_opts,
-)
-
+from .compression import decompress, maybe_compress
+from .utils import frame_split_size, msgpack_opts, pack_frames_prelude, unpack_frames
 
 lazy_registrations = {}
 
diff --git a/distributed/protocol/sparse.py b/distributed/protocol/sparse.py
index a22d661f849..42d625b0df2 100644
--- a/distributed/protocol/sparse.py
+++ b/distributed/protocol/sparse.py
@@ -1,7 +1,7 @@
-from .serialize import dask_serialize, dask_deserialize, serialize, deserialize
-
 import sparse
 
+from .serialize import dask_deserialize, dask_serialize, deserialize, serialize
+
 
 @dask_serialize.register(sparse.COO)
 def serialize_sparse(x):
diff --git a/distributed/protocol/tests/test_arrow.py b/distributed/protocol/tests/test_arrow.py
index 3d2c66afbb4..b36b5550247 100644
--- a/distributed/protocol/tests/test_arrow.py
+++ b/distributed/protocol/tests/test_arrow.py
@@ -4,8 +4,8 @@
 pd = pytest.importorskip("pandas")
 
 import distributed
-from distributed.utils_test import gen_cluster
 from distributed.protocol import deserialize, serialize, to_serialize
+from distributed.utils_test import gen_cluster
 
 df = pd.DataFrame({"A": list("abc"), "B": [1, 2, 3]})
 tbl = pa.Table.from_pandas(df, preserve_index=False)
diff --git a/distributed/protocol/tests/test_collection.py b/distributed/protocol/tests/test_collection.py
index 4bd10a9928c..46b88a3a691 100644
--- a/distributed/protocol/tests/test_collection.py
+++ b/distributed/protocol/tests/test_collection.py
@@ -1,5 +1,6 @@
 import pytest
-from distributed.protocol import serialize, deserialize
+
+from distributed.protocol import deserialize, serialize
 
 np = pytest.importorskip("numpy")
 pd = pytest.importorskip("pandas")
diff --git a/distributed/protocol/tests/test_collection_cuda.py b/distributed/protocol/tests/test_collection_cuda.py
index fe66b18bd68..a50fb7e2bb8 100644
--- a/distributed/protocol/tests/test_collection_cuda.py
+++ b/distributed/protocol/tests/test_collection_cuda.py
@@ -1,8 +1,9 @@
 import pytest
 
-from distributed.protocol import serialize, deserialize
 from dask.dataframe.utils import assert_eq
 
+from distributed.protocol import deserialize, serialize
+
 
 @pytest.mark.parametrize("collection", [tuple, dict])
 @pytest.mark.parametrize("y,y_serializer", [(50, "cuda"), (None, "pickle")])
diff --git a/distributed/protocol/tests/test_cupy.py b/distributed/protocol/tests/test_cupy.py
index 520693fb5c1..5c684e46d62 100644
--- a/distributed/protocol/tests/test_cupy.py
+++ b/distributed/protocol/tests/test_cupy.py
@@ -1,6 +1,7 @@
 import pickle
 
 import pytest
+
 from distributed.protocol import deserialize, serialize
 
 cupy = pytest.importorskip("cupy")
diff --git a/distributed/protocol/tests/test_h5py.py b/distributed/protocol/tests/test_h5py.py
index 80eeb2c05f5..0ab3f864281 100644
--- a/distributed/protocol/tests/test_h5py.py
+++ b/distributed/protocol/tests/test_h5py.py
@@ -6,7 +6,6 @@
 h5py = pytest.importorskip("h5py")
 
 from distributed.protocol import deserialize, serialize
-
 from distributed.utils import tmpfile
 
 
@@ -82,11 +81,10 @@ def test_raise_error_on_serialize_write_permissions():
                 deserialize(*serialize(f))
 
 
-from distributed.utils_test import gen_cluster
-
-
 import dask.array as da
 
+from distributed.utils_test import gen_cluster
+
 
 @silence_h5py_issue775
 @gen_cluster(client=True)
diff --git a/distributed/protocol/tests/test_highlevelgraph.py b/distributed/protocol/tests/test_highlevelgraph.py
index e974c00fdbc..df9f2942179 100644
--- a/distributed/protocol/tests/test_highlevelgraph.py
+++ b/distributed/protocol/tests/test_highlevelgraph.py
@@ -1,14 +1,13 @@
 import ast
 
-import dask
+import pytest
 
+import dask
 import dask.array as da
 import dask.dataframe as dd
 
-from distributed.utils_test import gen_cluster
 from distributed.diagnostics import SchedulerPlugin
-
-import pytest
+from distributed.utils_test import gen_cluster
 
 np = pytest.importorskip("numpy")
 pd = pytest.importorskip("pandas")
diff --git a/distributed/protocol/tests/test_keras.py b/distributed/protocol/tests/test_keras.py
index 79dec609ccb..6b350699a8d 100644
--- a/distributed/protocol/tests/test_keras.py
+++ b/distributed/protocol/tests/test_keras.py
@@ -3,7 +3,7 @@
 keras = pytest.importorskip("keras")
 np = pytest.importorskip("numpy")
 
-from distributed.protocol import serialize, deserialize, dumps, loads, to_serialize
+from distributed.protocol import deserialize, dumps, loads, serialize, to_serialize
 
 
 def test_serialize_deserialize_model():
diff --git a/distributed/protocol/tests/test_netcdf4.py b/distributed/protocol/tests/test_netcdf4.py
index 1ed78508156..b37c116ec43 100644
--- a/distributed/protocol/tests/test_netcdf4.py
+++ b/distributed/protocol/tests/test_netcdf4.py
@@ -4,7 +4,6 @@
 np = pytest.importorskip("numpy")
 
 from distributed.protocol import deserialize, serialize
-
 from distributed.utils import tmpfile
 
 
@@ -75,11 +74,10 @@ def test_serialize_deserialize_group():
                 assert (x[:] == y[:]).all()
 
 
-from distributed.utils_test import gen_cluster
-
-
 import dask.array as da
 
+from distributed.utils_test import gen_cluster
+
 
 @gen_cluster(client=True)
 async def test_netcdf4_serialize(c, s, a, b):
diff --git a/distributed/protocol/tests/test_numba.py b/distributed/protocol/tests/test_numba.py
index 61213640715..b34d4be25ab 100644
--- a/distributed/protocol/tests/test_numba.py
+++ b/distributed/protocol/tests/test_numba.py
@@ -1,7 +1,9 @@
-from distributed.protocol import serialize, deserialize
 import pickle
+
 import pytest
 
+from distributed.protocol import deserialize, serialize
+
 cuda = pytest.importorskip("numba.cuda")
 np = pytest.importorskip("numpy")
 
diff --git a/distributed/protocol/tests/test_numpy.py b/distributed/protocol/tests/test_numpy.py
index 7b8c9236bec..effcb609c96 100644
--- a/distributed/protocol/tests/test_numpy.py
+++ b/distributed/protocol/tests/test_numpy.py
@@ -5,20 +5,20 @@
 np = pytest.importorskip("numpy")
 
 from distributed.protocol import (
-    serialize,
-    deserialize,
     decompress,
+    deserialize,
     dumps,
     loads,
-    to_serialize,
     msgpack,
+    serialize,
+    to_serialize,
 )
-from distributed.protocol.utils import BIG_BYTES_SHARD_SIZE
+from distributed.protocol.compression import maybe_compress
 from distributed.protocol.numpy import itemsize
 from distributed.protocol.pickle import HIGHEST_PROTOCOL
-from distributed.protocol.compression import maybe_compress
+from distributed.protocol.utils import BIG_BYTES_SHARD_SIZE
 from distributed.system import MEMORY_LIMIT
-from distributed.utils import ensure_bytes, tmpfile, nbytes
+from distributed.utils import ensure_bytes, nbytes, tmpfile
 from distributed.utils_test import gen_cluster
 
 
diff --git a/distributed/protocol/tests/test_pandas.py b/distributed/protocol/tests/test_pandas.py
index ae338200312..58bfb90f75e 100644
--- a/distributed/protocol/tests/test_pandas.py
+++ b/distributed/protocol/tests/test_pandas.py
@@ -6,16 +6,15 @@
 from dask.dataframe.utils import assert_eq
 
 from distributed.protocol import (
-    serialize,
-    deserialize,
     decompress,
+    deserialize,
     dumps,
     loads,
+    serialize,
     to_serialize,
 )
 from distributed.utils import ensure_bytes
 
-
 dfs = [
     pd.DataFrame({}),
     pd.DataFrame({"x": [1, 2, 3]}),
diff --git a/distributed/protocol/tests/test_pickle.py b/distributed/protocol/tests/test_pickle.py
index 6430ea352b8..25989bfcde4 100644
--- a/distributed/protocol/tests/test_pickle.py
+++ b/distributed/protocol/tests/test_pickle.py
@@ -1,8 +1,8 @@
-from functools import partial
 import gc
-from operator import add
-import weakref
 import sys
+import weakref
+from functools import partial
+from operator import add
 
 import pytest
 
diff --git a/distributed/protocol/tests/test_protocol.py b/distributed/protocol/tests/test_protocol.py
index e088f146bfb..411fcc0c4e4 100644
--- a/distributed/protocol/tests/test_protocol.py
+++ b/distributed/protocol/tests/test_protocol.py
@@ -1,8 +1,8 @@
 import pytest
 
-from distributed.protocol import loads, dumps, msgpack, maybe_compress, to_serialize
+from distributed.protocol import dumps, loads, maybe_compress, msgpack, to_serialize
 from distributed.protocol.compression import compressions
-from distributed.protocol.serialize import Serialize, Serialized, serialize, deserialize
+from distributed.protocol.serialize import Serialize, Serialized, deserialize, serialize
 from distributed.system import MEMORY_LIMIT
 from distributed.utils import nbytes
 
diff --git a/distributed/protocol/tests/test_rmm.py b/distributed/protocol/tests/test_rmm.py
index 8b176afd877..bd9d7f4cab9 100644
--- a/distributed/protocol/tests/test_rmm.py
+++ b/distributed/protocol/tests/test_rmm.py
@@ -1,6 +1,7 @@
-from distributed.protocol import serialize, deserialize
 import pytest
 
+from distributed.protocol import deserialize, serialize
+
 numpy = pytest.importorskip("numpy")
 cuda = pytest.importorskip("numba.cuda")
 rmm = pytest.importorskip("rmm")
diff --git a/distributed/protocol/tests/test_scipy.py b/distributed/protocol/tests/test_scipy.py
index 4e5eb8423cf..0904f92002a 100644
--- a/distributed/protocol/tests/test_scipy.py
+++ b/distributed/protocol/tests/test_scipy.py
@@ -1,4 +1,5 @@
 import pytest
+
 from distributed.protocol import deserialize, serialize
 
 numpy = pytest.importorskip("numpy")
diff --git a/distributed/protocol/tests/test_serialize.py b/distributed/protocol/tests/test_serialize.py
index a05bedd7e50..cd2efa3bb28 100644
--- a/distributed/protocol/tests/test_serialize.py
+++ b/distributed/protocol/tests/test_serialize.py
@@ -1,6 +1,6 @@
-from array import array
 import copy
 import pickle
+from array import array
 
 import msgpack
 import pytest
@@ -14,26 +14,26 @@
 from dask.utils_test import inc
 
 from distributed import wait
+from distributed.comm.utils import from_frames, to_frames
 from distributed.protocol import (
-    register_serialization,
-    serialize,
-    deserialize,
-    nested_deserialize,
     Serialize,
     Serialized,
-    to_serialize,
-    serialize_bytes,
-    deserialize_bytes,
-    serialize_bytelist,
-    register_serialization_family,
     dask_serialize,
+    deserialize,
+    deserialize_bytes,
     dumps,
     loads,
+    nested_deserialize,
+    register_serialization,
+    register_serialization_family,
+    serialize,
+    serialize_bytelist,
+    serialize_bytes,
+    to_serialize,
 )
 from distributed.protocol.serialize import check_dask_serializable
 from distributed.utils import nbytes
-from distributed.utils_test import inc, gen_test
-from distributed.comm.utils import to_frames, from_frames
+from distributed.utils_test import gen_test, inc
 
 
 class MyObj:
@@ -145,9 +145,10 @@ def test_nested_deserialize():
     assert x == x_orig  # x wasn't mutated
 
 
-from distributed.utils_test import gen_cluster
 from dask import delayed
 
+from distributed.utils_test import gen_cluster
+
 
 @gen_cluster(client=True)
 async def test_object_in_graph(c, s, a, b):
diff --git a/distributed/protocol/tests/test_torch.py b/distributed/protocol/tests/test_torch.py
index efb5fa6610a..3e212c0e858 100644
--- a/distributed/protocol/tests/test_torch.py
+++ b/distributed/protocol/tests/test_torch.py
@@ -1,6 +1,7 @@
-from distributed.protocol import serialize, deserialize
 import pytest
 
+from distributed.protocol import deserialize, serialize
+
 np = pytest.importorskip("numpy")
 torch = pytest.importorskip("torch")
 
diff --git a/distributed/protocol/torch.py b/distributed/protocol/torch.py
index 3b4c6d19c8d..f8b6acb13b0 100644
--- a/distributed/protocol/torch.py
+++ b/distributed/protocol/torch.py
@@ -1,7 +1,7 @@
-from .serialize import serialize, dask_serialize, dask_deserialize, register_generic
-
-import torch
 import numpy as np
+import torch
+
+from .serialize import dask_deserialize, dask_serialize, register_generic, serialize
 
 
 @dask_serialize.register(torch.Tensor)
diff --git a/distributed/pubsub.py b/distributed/pubsub.py
index 91200be06eb..5dba5b679c6 100644
--- a/distributed/pubsub.py
+++ b/distributed/pubsub.py
@@ -1,13 +1,13 @@
 import asyncio
-from collections import defaultdict, deque
 import logging
 import threading
 import weakref
+from collections import defaultdict, deque
 
 from .core import CommClosedError
 from .metrics import time
-from .utils import sync, TimeoutError, parse_timedelta
 from .protocol.serialize import to_serialize
+from .utils import TimeoutError, parse_timedelta, sync
 
 logger = logging.getLogger(__name__)
 
@@ -283,7 +283,7 @@ class Pub:
 
     def __init__(self, name, worker=None, client=None):
         if worker is None and client is None:
-            from distributed import get_worker, get_client
+            from distributed import get_client, get_worker
 
             try:
                 worker = get_worker()
@@ -363,7 +363,7 @@ class Sub:
 
     def __init__(self, name, worker=None, client=None):
         if worker is None and client is None:
-            from distributed.worker import get_worker, get_client
+            from distributed.worker import get_client, get_worker
 
             try:
                 worker = get_worker()
diff --git a/distributed/pytest_resourceleaks.py b/distributed/pytest_resourceleaks.py
index f86117c7862..6d9959ac46e 100644
--- a/distributed/pytest_resourceleaks.py
+++ b/distributed/pytest_resourceleaks.py
@@ -4,10 +4,10 @@
 """
 import collections
 import gc
-import time
 import os
 import sys
 import threading
+import time
 
 import pytest
 
diff --git a/distributed/queues.py b/distributed/queues.py
index e368d329d03..cadaf358f0a 100644
--- a/distributed/queues.py
+++ b/distributed/queues.py
@@ -1,14 +1,13 @@
 import asyncio
-from collections import defaultdict
 import logging
 import uuid
+from collections import defaultdict
 
 from dask.utils import stringify
 
-from .client import Future, Client
-from .utils import sync, thread_state
+from .client import Client, Future
+from .utils import parse_timedelta, sync, thread_state
 from .worker import get_client, get_worker
-from .utils import parse_timedelta
 
 logger = logging.getLogger(__name__)
 
diff --git a/distributed/recreate_exceptions.py b/distributed/recreate_exceptions.py
index 7e966f270a0..6b498113b5e 100644
--- a/distributed/recreate_exceptions.py
+++ b/distributed/recreate_exceptions.py
@@ -1,5 +1,7 @@
 import logging
+
 from dask.utils import stringify
+
 from .client import futures_of, wait
 from .utils import sync
 from .utils_comm import pack_data
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 4affb556bdf..6d0da803bac 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1,89 +1,85 @@
 import asyncio
-from collections import defaultdict, deque
-
-from collections.abc import Mapping, Set
-from contextlib import suppress
-from datetime import timedelta
-from functools import partial
 import inspect
 import itertools
 import json
 import logging
 import math
-from numbers import Number
 import operator
 import os
-import sys
 import random
+import sys
 import warnings
 import weakref
+from collections import defaultdict, deque
+from collections.abc import Mapping, Set
+from contextlib import suppress
+from datetime import timedelta
+from functools import partial
+from numbers import Number
+
 import psutil
 import sortedcontainers
-
 from tlz import (
+    compose,
+    concat,
+    first,
+    groupby,
     merge,
-    pluck,
     merge_sorted,
-    first,
     merge_with,
-    valmap,
+    pluck,
     second,
-    compose,
-    groupby,
-    concat,
+    valmap,
 )
 from tornado.ioloop import IOLoop, PeriodicCallback
 
 import dask
 from dask.highlevelgraph import HighLevelGraph
 
-from . import profile
+from . import preloading, profile
+from . import versions as version_module
 from .batched import BatchedSend
 from .comm import (
+    get_address_host,
     normalize_address,
     resolve_address,
-    get_address_host,
     unparse_host_port,
 )
 from .comm.addressing import addresses_from_user_args
-from .core import rpc, send_recv, clean_exception, CommClosedError, Status
+from .core import CommClosedError, Status, clean_exception, rpc, send_recv
 from .diagnostics.plugin import SchedulerPlugin
-
+from .event import EventExtension
 from .http import get_handlers
+from .lock import LockExtension
 from .metrics import time
+from .multi_lock import MultiLockExtension
 from .node import ServerNode
-from . import preloading
 from .proctitle import setproctitle
+from .publish import PublishExtension
+from .pubsub import PubSubSchedulerExtension
+from .queues import QueueExtension
+from .recreate_exceptions import ReplayExceptionScheduler
 from .security import Security
+from .semaphore import SemaphoreExtension
+from .stealing import WorkStealing
 from .utils import (
     All,
+    TimeoutError,
+    empty_context,
+    format_bytes,
+    format_time,
     get_fileno_limit,
-    log_errors,
     key_split,
-    validate_key,
+    key_split_group,
+    log_errors,
     no_default,
-    parse_timedelta,
     parse_bytes,
-    key_split_group,
-    empty_context,
+    parse_timedelta,
     tmpfile,
-    format_bytes,
-    format_time,
-    TimeoutError,
+    validate_key,
 )
-from .utils_comm import scatter_to_workers, gather_from_workers, retry_operation
-from .utils_perf import enable_gc_diagnosis, disable_gc_diagnosis
-from . import versions as version_module
-
-from .publish import PublishExtension
-from .queues import QueueExtension
-from .semaphore import SemaphoreExtension
-from .recreate_exceptions import ReplayExceptionScheduler
-from .lock import LockExtension
-from .multi_lock import MultiLockExtension
-from .event import EventExtension
-from .pubsub import PubSubSchedulerExtension
-from .stealing import WorkStealing
+from .utils_comm import gather_from_workers, retry_operation, scatter_to_workers
+from .utils_perf import disable_gc_diagnosis, enable_gc_diagnosis
 from .variable import VariableExtension
 
 try:
@@ -93,6 +89,8 @@
 
 if compiled:
     from cython import (
+        Py_hash_t,
+        Py_ssize_t,
         bint,
         cast,
         ccall,
@@ -104,15 +102,11 @@
         final,
         inline,
         nogil,
-        Py_hash_t,
-        Py_ssize_t,
     )
 else:
-    from ctypes import (
-        c_double as double,
-        c_ssize_t as Py_hash_t,
-        c_ssize_t as Py_ssize_t,
-    )
+    from ctypes import c_double as double
+    from ctypes import c_ssize_t as Py_hash_t
+    from ctypes import c_ssize_t as Py_ssize_t
 
     bint = bool
 
@@ -6379,16 +6373,16 @@ def profile_to_figure(state):
         for k in sorted(timespent.keys()):
             tasks_timings += f"\n<li> {k} time: {format_time(timespent[k])} </li>"
 
-        from .diagnostics.task_stream import rectangles
         from .dashboard.components.scheduler import task_stream_figure
+        from .diagnostics.task_stream import rectangles
 
         rects = rectangles(task_stream)
         source, task_stream = task_stream_figure(sizing_mode="stretch_both")
         source.data.update(rects)
 
         from distributed.dashboard.components.scheduler import (
-            BandwidthWorkers,
             BandwidthTypes,
+            BandwidthWorkers,
         )
 
         bandwidth_workers = BandwidthWorkers(self, sizing_mode="stretch_both")
@@ -6396,7 +6390,8 @@ def profile_to_figure(state):
         bandwidth_types = BandwidthTypes(self, sizing_mode="stretch_both")
         bandwidth_types.update()
 
-        from bokeh.models import Panel, Tabs, Div
+        from bokeh.models import Div, Panel, Tabs
+
         import distributed
 
         # HTML
@@ -6465,8 +6460,8 @@ def profile_to_figure(state):
             ]
         )
 
-        from bokeh.plotting import save, output_file
         from bokeh.core.templates import get_env
+        from bokeh.plotting import output_file, save
 
         with tmpfile(extension=".html") as fn:
             output_file(filename=fn, title="Dask Performance Report")
diff --git a/distributed/security.py b/distributed/security.py
index 2cfe952b397..551e29c8652 100644
--- a/distributed/security.py
+++ b/distributed/security.py
@@ -1,6 +1,6 @@
 import datetime
-import tempfile
 import os
+import tempfile
 
 try:
     import ssl
@@ -9,7 +9,6 @@
 
 import dask
 
-
 __all__ = ("Security",)
 
 
@@ -95,8 +94,7 @@ def temporary(cls):
         try:
             from cryptography import x509
             from cryptography.hazmat.backends import default_backend
-            from cryptography.hazmat.primitives import hashes
-            from cryptography.hazmat.primitives import serialization
+            from cryptography.hazmat.primitives import hashes, serialization
             from cryptography.hazmat.primitives.asymmetric import rsa
             from cryptography.x509.oid import NameOID
         except ImportError:
diff --git a/distributed/semaphore.py b/distributed/semaphore.py
index 4b2bff3f74c..2de45c42f34 100644
--- a/distributed/semaphore.py
+++ b/distributed/semaphore.py
@@ -5,9 +5,10 @@
 from asyncio import TimeoutError
 from collections import defaultdict, deque
 
-import dask
 from tornado.ioloop import IOLoop, PeriodicCallback
 
+import dask
+
 from distributed.utils_comm import retry_operation
 
 from .metrics import time
diff --git a/distributed/stealing.py b/distributed/stealing.py
index 8dc8c218f87..b352040bff7 100644
--- a/distributed/stealing.py
+++ b/distributed/stealing.py
@@ -1,18 +1,18 @@
-from collections import defaultdict, deque
 import logging
+from collections import defaultdict, deque
 from math import log2
 from time import time
 
+from tlz import topk
 from tornado.ioloop import PeriodicCallback
 
 import dask
+
 from .comm.addressing import get_address_host
 from .core import CommClosedError
 from .diagnostics.plugin import SchedulerPlugin
 from .utils import log_errors, parse_timedelta
 
-from tlz import topk
-
 LATENCY = 10e-3
 
 logger = logging.getLogger(__name__)
diff --git a/distributed/system_monitor.py b/distributed/system_monitor.py
index cf305869a8c..2f8e64c6640 100644
--- a/distributed/system_monitor.py
+++ b/distributed/system_monitor.py
@@ -1,4 +1,5 @@
 from collections import deque
+
 import psutil
 
 from .compatibility import WINDOWS
diff --git a/distributed/tests/make_tls_certs.py b/distributed/tests/make_tls_certs.py
index 02817cbdf5e..7286b780449 100644
--- a/distributed/tests/make_tls_certs.py
+++ b/distributed/tests/make_tls_certs.py
@@ -5,8 +5,8 @@
 
 import os
 import shutil
-import tempfile
 import subprocess
+import tempfile
 
 req_template = """
     [req]
diff --git a/distributed/tests/test_actor.py b/distributed/tests/test_actor.py
index 72123a14dc4..e639769ae17 100644
--- a/distributed/tests/test_actor.py
+++ b/distributed/tests/test_actor.py
@@ -5,10 +5,16 @@
 import pytest
 
 import dask
-from distributed import Actor, ActorFuture, Client, Future, wait, Nanny
-from distributed.utils_test import cluster, gen_cluster
-from distributed.utils_test import client, cluster_fixture, loop  # noqa: F401
+
+from distributed import Actor, ActorFuture, Client, Future, Nanny, wait
 from distributed.metrics import time
+from distributed.utils_test import (  # noqa: F401
+    client,
+    cluster,
+    cluster_fixture,
+    gen_cluster,
+    loop,
+)
 
 
 class Counter:
diff --git a/distributed/tests/test_as_completed.py b/distributed/tests/test_as_completed.py
index 3efa759fa36..1002259391b 100644
--- a/distributed/tests/test_as_completed.py
+++ b/distributed/tests/test_as_completed.py
@@ -1,17 +1,23 @@
 import asyncio
-from collections.abc import Iterator
-from operator import add
 import queue
 import random
+from collections.abc import Iterator
+from operator import add
 from time import sleep
 
 import pytest
 
-from distributed.client import _as_completed, as_completed, _first_completed, wait
+from distributed.client import _as_completed, _first_completed, as_completed, wait
 from distributed.metrics import time
 from distributed.utils import CancelledError
-from distributed.utils_test import gen_cluster, inc, throws
-from distributed.utils_test import client, cluster_fixture, loop  # noqa: F401
+from distributed.utils_test import (  # noqa: F401
+    client,
+    cluster_fixture,
+    gen_cluster,
+    inc,
+    loop,
+    throws,
+)
 
 
 @gen_cluster(client=True)
diff --git a/distributed/tests/test_asyncprocess.py b/distributed/tests/test_asyncprocess.py
index 288ba58be86..32c4e5c5589 100644
--- a/distributed/tests/test_asyncprocess.py
+++ b/distributed/tests/test_asyncprocess.py
@@ -16,7 +16,7 @@
 from distributed.metrics import time
 from distributed.process import AsyncProcess
 from distributed.utils import mp_context
-from distributed.utils_test import gen_test, pristine_loop, nodebug
+from distributed.utils_test import gen_test, nodebug, pristine_loop
 
 
 def feed(in_q, out_q):
diff --git a/distributed/tests/test_batched.py b/distributed/tests/test_batched.py
index a1342178a71..ee84ec3224e 100644
--- a/distributed/tests/test_batched.py
+++ b/distributed/tests/test_batched.py
@@ -5,11 +5,11 @@
 from tlz import assoc
 
 from distributed.batched import BatchedSend
-from distributed.core import listen, connect, CommClosedError
+from distributed.core import CommClosedError, connect, listen
 from distributed.metrics import time
+from distributed.protocol import to_serialize
 from distributed.utils import All
 from distributed.utils_test import captured_logger
-from distributed.protocol import to_serialize
 
 
 class EchoServer:
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 9a4ea4e188c..1bdfb15ab6e 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -1,102 +1,102 @@
 import asyncio
-from collections import deque
-from contextlib import suppress
-from functools import partial
 import gc
 import logging
-from operator import add
 import os
 import pickle
-import psutil
 import random
 import subprocess
 import sys
 import threading
-from threading import Semaphore
-from time import sleep
 import traceback
 import warnings
 import weakref
 import zipfile
+from collections import deque
+from contextlib import suppress
+from functools import partial
+from operator import add
+from threading import Semaphore
+from time import sleep
 
+import psutil
 import pytest
-from tlz import identity, isdistinct, concat, pluck, valmap, first, merge
+from tlz import concat, first, identity, isdistinct, merge, pluck, valmap
 
 import dask
+import dask.bag as db
 from dask import delayed
 from dask.optimization import SubgraphCallable
 from dask.utils import stringify
-import dask.bag as db
+
 from distributed import (
-    Worker,
+    CancelledError,
+    Executor,
+    LocalCluster,
     Nanny,
+    TimeoutError,
+    Worker,
     fire_and_forget,
-    LocalCluster,
     get_client,
-    secede,
     get_worker,
-    Executor,
-    profile,
     performance_report,
-    TimeoutError,
-    CancelledError,
+    profile,
+    secede,
 )
-from distributed.core import Status
-from distributed.comm import CommClosedError
 from distributed.client import (
     Client,
     Future,
-    wait,
-    as_completed,
-    tokenize,
     _get_global_client,
+    as_completed,
     default_client,
     futures_of,
-    temp_default_client,
     get_task_metadata,
+    temp_default_client,
+    tokenize,
+    wait,
 )
+from distributed.comm import CommClosedError
 from distributed.compatibility import MACOS, WINDOWS
-
+from distributed.core import Status
 from distributed.metrics import time
-from distributed.scheduler import Scheduler, KilledWorker, CollectTaskMetaDataPlugin
+from distributed.scheduler import CollectTaskMetaDataPlugin, KilledWorker, Scheduler
 from distributed.sizeof import sizeof
-from distributed.utils import mp_context, sync, tmp_text, tmpfile, is_valid_xml
-from distributed.utils_test import (
-    cluster,
-    slowinc,
-    slowadd,
-    slowdec,
-    randominc,
-    inc,
-    dec,
-    div,
-    throws,
-    geninc,
+from distributed.utils import is_valid_xml, mp_context, sync, tmp_text, tmpfile
+from distributed.utils_test import (  # noqa: F401
+    TaskStateMetadataPlugin,
+    a,
+    async_wait_for,
     asyncinc,
-    gen_cluster,
-    gen_test,
-    double,
-    popen,
+    b,
     captured_logger,
-    varying,
-    map_varying,
-    wait_for,
-    async_wait_for,
-    pristine_loop,
-    save_sys_modules,
-    TaskStateMetadataPlugin,
+    cleanup,
 )
+from distributed.utils_test import client as c  # noqa: F401
+from distributed.utils_test import client_secondary as c2  # noqa: F401
 from distributed.utils_test import (  # noqa: F401
-    client as c,
-    client_secondary as c2,
-    cleanup,
+    cluster,
     cluster_fixture,
+    dec,
+    div,
+    double,
+    gen_cluster,
+    gen_test,
+    geninc,
+    inc,
     loop,
     loop_in_thread,
+    map_varying,
     nodebug,
+    popen,
+    pristine_loop,
+    randominc,
     s,
-    a,
-    b,
+    save_sys_modules,
+    slowadd,
+    slowdec,
+    slowinc,
+    throws,
+    varying,
+    wait_for,
 )
 
 
@@ -1598,7 +1598,8 @@ def g():
 @gen_cluster(client=True)
 async def test_upload_file_egg(c, s, a, b):
     def g():
-        import package_1, package_2
+        import package_1
+        import package_2
 
         return package_1.a, package_2.b
 
@@ -2450,7 +2451,7 @@ def test_Future_exception_sync_2(loop, capsys):
 
 @gen_cluster(timeout=60, client=True)
 async def test_async_persist(c, s, a, b):
-    from dask.delayed import delayed, Delayed
+    from dask.delayed import Delayed, delayed
 
     x = delayed(1)
     y = delayed(inc)(x)
@@ -5516,8 +5517,8 @@ async def test_client_timeout_2():
 
 @gen_test()
 async def test_client_active_bad_port():
-    import tornado.web
     import tornado.httpserver
+    import tornado.web
 
     application = tornado.web.Application([(r"/", tornado.web.RequestHandler)])
     http_server = tornado.httpserver.HTTPServer(application)
diff --git a/distributed/tests/test_client_executor.py b/distributed/tests/test_client_executor.py
index b55b86ce7c8..555bcb86fbe 100644
--- a/distributed/tests/test_client_executor.py
+++ b/distributed/tests/test_client_executor.py
@@ -1,13 +1,12 @@
 import random
 import time
-
 from concurrent.futures import (
-    TimeoutError,
-    Future,
-    wait,
-    as_completed,
     FIRST_COMPLETED,
     FIRST_EXCEPTION,
+    Future,
+    TimeoutError,
+    as_completed,
+    wait,
 )
 
 import pytest
@@ -15,16 +14,21 @@
 
 from distributed import Client
 from distributed.utils import CancelledError
-from distributed.utils_test import (
-    slowinc,
+from distributed.utils_test import (  # noqa: F401
+    a,
+    b,
+    client,
+    cluster,
+    cluster_fixture,
+    inc,
+    loop,
+    s,
     slowadd,
     slowdec,
-    inc,
+    slowinc,
     throws,
     varying,
-    cluster,
 )
-from distributed.utils_test import client, cluster_fixture, loop, s, a, b  # noqa: F401
 
 
 def number_of_processing_tasks(client):
diff --git a/distributed/tests/test_client_loop.py b/distributed/tests/test_client_loop.py
index ce50498f7e4..63a08cb1639 100644
--- a/distributed/tests/test_client_loop.py
+++ b/distributed/tests/test_client_loop.py
@@ -1,5 +1,6 @@
 import pytest
-from distributed import LocalCluster, Client
+
+from distributed import Client, LocalCluster
 from distributed.utils import LoopRunner
 
 
diff --git a/distributed/tests/test_collections.py b/distributed/tests/test_collections.py
index 8b3e6cd5a38..ea045bb49a5 100644
--- a/distributed/tests/test_collections.py
+++ b/distributed/tests/test_collections.py
@@ -6,11 +6,16 @@
 pd = pytest.importorskip("pandas")
 
 import dask
-import dask.dataframe as dd
 import dask.bag as db
+import dask.dataframe as dd
+
 from distributed.client import wait
-from distributed.utils_test import gen_cluster
-from distributed.utils_test import client, cluster_fixture, loop  # noqa F401
+from distributed.utils_test import (  # noqa F401
+    client,
+    cluster_fixture,
+    gen_cluster,
+    loop,
+)
 
 PANDAS_VERSION = LooseVersion(pd.__version__)
 PANDAS_GT_100 = PANDAS_VERSION >= LooseVersion("1.0.0")
diff --git a/distributed/tests/test_config.py b/distributed/tests/test_config.py
index 74b57b1f011..200493a822c 100644
--- a/distributed/tests/test_config.py
+++ b/distributed/tests/test_config.py
@@ -1,19 +1,19 @@
 import logging
+import os
 import subprocess
 import sys
 import tempfile
-import os
-import yaml
 
 import pytest
+import yaml
 
+from distributed.config import initialize_logging
 from distributed.utils_test import (
     captured_handler,
     captured_logger,
     new_config,
     new_config_file,
 )
-from distributed.config import initialize_logging
 
 
 def dump_logger_list():
diff --git a/distributed/tests/test_core.py b/distributed/tests/test_core.py
index 071b6b039c7..9502d9d4d15 100644
--- a/distributed/tests/test_core.py
+++ b/distributed/tests/test_core.py
@@ -2,43 +2,43 @@
 import os
 import socket
 import threading
-import weakref
 import warnings
+import weakref
 
 import pytest
 
 import dask
+
 from distributed.core import (
-    pingpong,
+    ConnectionPool,
     Server,
     Status,
-    rpc,
+    coerce_to_address,
     connect,
+    pingpong,
+    rpc,
     send_recv,
-    coerce_to_address,
-    ConnectionPool,
 )
-from distributed.protocol.compression import compressions
-
 from distributed.metrics import time
 from distributed.protocol import to_serialize
+from distributed.protocol.compression import compressions
 from distributed.utils import get_ip, get_ipv6
+from distributed.utils_test import loop  # noqa F401
 from distributed.utils_test import (
-    gen_cluster,
-    has_ipv6,
     assert_can_connect,
-    assert_cannot_connect,
     assert_can_connect_from_everywhere_4,
     assert_can_connect_from_everywhere_4_6,
     assert_can_connect_from_everywhere_6,
     assert_can_connect_locally_4,
     assert_can_connect_locally_6,
-    tls_security,
+    assert_cannot_connect,
     captured_logger,
+    gen_cluster,
+    has_ipv6,
     inc,
     throws,
+    tls_security,
 )
-from distributed.utils_test import loop  # noqa F401
 
 
 def echo(comm, x):
diff --git a/distributed/tests/test_diskutils.py b/distributed/tests/test_diskutils.py
index 3b97fde1498..8a9d99e5844 100644
--- a/distributed/tests/test_diskutils.py
+++ b/distributed/tests/test_diskutils.py
@@ -11,6 +11,7 @@
 import pytest
 
 import dask
+
 from distributed.compatibility import MACOS, WINDOWS
 from distributed.diskutils import WorkSpace
 from distributed.metrics import time
diff --git a/distributed/tests/test_events.py b/distributed/tests/test_events.py
index 484a01c4e3a..9f27a19a65d 100644
--- a/distributed/tests/test_events.py
+++ b/distributed/tests/test_events.py
@@ -2,8 +2,12 @@
 from datetime import timedelta
 
 from distributed import Event
-from distributed.utils_test import gen_cluster
-from distributed.utils_test import client, cluster_fixture, loop  # noqa F401
+from distributed.utils_test import (  # noqa F401
+    client,
+    cluster_fixture,
+    gen_cluster,
+    loop,
+)
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 8)] * 2)
diff --git a/distributed/tests/test_failed_workers.py b/distributed/tests/test_failed_workers.py
index 717fe01c80d..4bd3e77eb81 100644
--- a/distributed/tests/test_failed_workers.py
+++ b/distributed/tests/test_failed_workers.py
@@ -1,28 +1,29 @@
 import asyncio
-from contextlib import suppress
 import os
 import random
+from contextlib import suppress
 from time import sleep
 
 import pytest
-from tlz import partition_all, first
+from tlz import first, partition_all
 
 from dask import delayed
+
 from distributed import Client, Nanny, wait
 from distributed.comm import CommClosedError
 from distributed.compatibility import MACOS
 from distributed.metrics import time
-from distributed.utils import sync, CancelledError
+from distributed.utils import CancelledError, sync
+from distributed.utils_test import loop  # noqa: F401
 from distributed.utils_test import (
-    gen_cluster,
+    captured_logger,
     cluster,
-    inc,
     div,
-    slowinc,
+    gen_cluster,
+    inc,
     slowadd,
-    captured_logger,
+    slowinc,
 )
-from distributed.utils_test import loop  # noqa: F401
 
 
 def test_submit_after_failed_worker_sync(loop):
diff --git a/distributed/tests/test_ipython.py b/distributed/tests/test_ipython.py
index 51d7ae85f26..fbe5cfc8f9d 100644
--- a/distributed/tests/test_ipython.py
+++ b/distributed/tests/test_ipython.py
@@ -4,8 +4,7 @@
 from tlz import first
 
 from distributed import Client
-from distributed.utils_test import cluster, mock_ipython
-from distributed.utils_test import loop, zmq_ctx  # noqa F401
+from distributed.utils_test import cluster, loop, mock_ipython, zmq_ctx  # noqa F401
 
 
 def need_functional_ipython(func):
diff --git a/distributed/tests/test_locks.py b/distributed/tests/test_locks.py
index 80eaa06a94f..54c25c49099 100644
--- a/distributed/tests/test_locks.py
+++ b/distributed/tests/test_locks.py
@@ -1,13 +1,17 @@
 import pickle
-from time import sleep
 from datetime import timedelta
+from time import sleep
 
 import pytest
 
-from distributed import Lock, get_client, Client
+from distributed import Client, Lock, get_client
 from distributed.metrics import time
-from distributed.utils_test import gen_cluster
-from distributed.utils_test import client, cluster_fixture, loop  # noqa F401
+from distributed.utils_test import (  # noqa F401
+    client,
+    cluster_fixture,
+    gen_cluster,
+    loop,
+)
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 8)] * 2)
diff --git a/distributed/tests/test_multi_locks.py b/distributed/tests/test_multi_locks.py
index c1a7cc55af5..26f4a6dbdb7 100644
--- a/distributed/tests/test_multi_locks.py
+++ b/distributed/tests/test_multi_locks.py
@@ -1,12 +1,15 @@
 import asyncio
-from distributed.multi_lock import MultiLockExtension
 from time import sleep
 
-
 from distributed import MultiLock, get_client
 from distributed.metrics import time
-from distributed.utils_test import gen_cluster
-from distributed.utils_test import client, cluster_fixture, loop  # noqa F401
+from distributed.multi_lock import MultiLockExtension
+from distributed.utils_test import (  # noqa F401
+    client,
+    cluster_fixture,
+    gen_cluster,
+    loop,
+)
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 8)] * 2)
diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
index 83c3de18c43..fd07e308f7b 100644
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -1,31 +1,27 @@
 import asyncio
-from contextlib import suppress
 import gc
 import logging
+import multiprocessing as mp
 import os
 import random
 import sys
-import multiprocessing as mp
+from contextlib import suppress
 
 import pytest
-from tlz import valmap, first
+from tlz import first, valmap
 from tornado.ioloop import IOLoop
 
 import dask
-from distributed.diagnostics import SchedulerPlugin
-from distributed import Nanny, rpc, Scheduler, Worker, Client, wait, worker
+
+from distributed import Client, Nanny, Scheduler, Worker, rpc, wait, worker
 from distributed.compatibility import MACOS
 from distributed.core import CommClosedError, Status
+from distributed.diagnostics import SchedulerPlugin
 from distributed.metrics import time
 from distributed.protocol.pickle import dumps
-from distributed.utils import tmpfile, TimeoutError, parse_ports
-from distributed.utils_test import (  # noqa: F401
-    gen_cluster,
-    gen_test,
-    inc,
-    captured_logger,
-    cleanup,
-)
+from distributed.utils import TimeoutError, parse_ports, tmpfile
+from distributed.utils_test import cleanup  # noqa: F401
+from distributed.utils_test import captured_logger, gen_cluster, gen_test, inc
 
 
 # FIXME why does this leave behind unclosed Comm objects?
diff --git a/distributed/tests/test_preload.py b/distributed/tests/test_preload.py
index ae13137d847..7915aafab15 100644
--- a/distributed/tests/test_preload.py
+++ b/distributed/tests/test_preload.py
@@ -1,17 +1,15 @@
 import os
-import pytest
 import shutil
 import sys
 import tempfile
-import pytest
 
+import pytest
 from tornado import web
 
 import dask
-from distributed import Client, Scheduler, Worker, Nanny
-from distributed.utils_test import cluster, captured_logger
-from distributed.utils_test import loop, cleanup  # noqa F401
 
+from distributed import Client, Nanny, Scheduler, Worker
+from distributed.utils_test import captured_logger, cleanup, cluster, loop  # noqa F401
 
 PRELOAD_TEXT = """
 _worker_info = {}
diff --git a/distributed/tests/test_priorities.py b/distributed/tests/test_priorities.py
index d3ba8a1a026..00a046b457e 100644
--- a/distributed/tests/test_priorities.py
+++ b/distributed/tests/test_priorities.py
@@ -2,13 +2,13 @@
 
 import pytest
 
-from dask.core import flatten
 import dask
 from dask import delayed, persist
+from dask.core import flatten
 from dask.utils import stringify
 
-from distributed.utils_test import gen_cluster, inc, slowinc, slowdec
-from distributed import wait, Worker
+from distributed import Worker, wait
+from distributed.utils_test import gen_cluster, inc, slowdec, slowinc
 
 
 @gen_cluster(client=True, nthreads=[])
diff --git a/distributed/tests/test_profile.py b/distributed/tests/test_profile.py
index 9f673e8caaf..5fd6f6f2dab 100644
--- a/distributed/tests/test_profile.py
+++ b/distributed/tests/test_profile.py
@@ -1,21 +1,22 @@
-import pytest
 import sys
+import threading
 import time
+
+import pytest
 from tlz import first
-import threading
 
-from distributed.compatibility import WINDOWS
 from distributed import metrics
+from distributed.compatibility import WINDOWS
 from distributed.profile import (
-    process,
-    merge,
-    create,
     call_stack,
+    create,
     identifier,
-    watch,
-    llprocess,
     ll_get_stack,
+    llprocess,
+    merge,
     plot_data,
+    process,
+    watch,
 )
 
 
diff --git a/distributed/tests/test_publish.py b/distributed/tests/test_publish.py
index 82bcc7c996f..7f4c03fe71a 100644
--- a/distributed/tests/test_publish.py
+++ b/distributed/tests/test_publish.py
@@ -1,13 +1,20 @@
 import asyncio
+
 import pytest
 
 from dask import delayed
+
 from distributed import Client
 from distributed.client import futures_of
 from distributed.metrics import time
-from distributed.utils_test import gen_cluster, inc
-from distributed.utils_test import client, cluster_fixture, loop  # noqa F401
 from distributed.protocol import Serialized
+from distributed.utils_test import (  # noqa F401
+    client,
+    cluster_fixture,
+    gen_cluster,
+    inc,
+    loop,
+)
 
 
 @gen_cluster(client=False)
diff --git a/distributed/tests/test_pubsub.py b/distributed/tests/test_pubsub.py
index 09ab4dcd342..6f555c09fc6 100644
--- a/distributed/tests/test_pubsub.py
+++ b/distributed/tests/test_pubsub.py
@@ -5,9 +5,9 @@
 import pytest
 import tlz as toolz
 
-from distributed import Pub, Sub, wait, get_worker, TimeoutError
-from distributed.utils_test import gen_cluster
+from distributed import Pub, Sub, TimeoutError, get_worker, wait
 from distributed.metrics import time
+from distributed.utils_test import gen_cluster
 
 
 @gen_cluster(client=True, timeout=None)
diff --git a/distributed/tests/test_queues.py b/distributed/tests/test_queues.py
index 8f400498854..c7efa9a0d23 100644
--- a/distributed/tests/test_queues.py
+++ b/distributed/tests/test_queues.py
@@ -4,10 +4,17 @@
 
 import pytest
 
-from distributed import Client, Queue, Nanny, worker_client, wait, TimeoutError
+from distributed import Client, Nanny, Queue, TimeoutError, wait, worker_client
 from distributed.metrics import time
-from distributed.utils_test import gen_cluster, inc, div, popen
-from distributed.utils_test import client, cluster_fixture, loop  # noqa: F401
+from distributed.utils_test import (  # noqa: F401
+    client,
+    cluster_fixture,
+    div,
+    gen_cluster,
+    inc,
+    loop,
+    popen,
+)
 
 
 @gen_cluster(client=True)
diff --git a/distributed/tests/test_resources.py b/distributed/tests/test_resources.py
index 68837e30a14..15e9f344a83 100644
--- a/distributed/tests/test_resources.py
+++ b/distributed/tests/test_resources.py
@@ -1,16 +1,27 @@
 import asyncio
 from time import time
 
+import pytest
+
 import dask
 from dask import delayed
 from dask.utils import stringify
-import pytest
 
 from distributed import Worker
 from distributed.client import wait
 from distributed.compatibility import WINDOWS
-from distributed.utils_test import inc, gen_cluster, slowinc, slowadd
-from distributed.utils_test import client, cluster_fixture, loop, s, a, b  # noqa: F401
+from distributed.utils_test import (  # noqa: F401
+    a,
+    b,
+    client,
+    cluster_fixture,
+    gen_cluster,
+    inc,
+    loop,
+    s,
+    slowadd,
+    slowinc,
+)
 
 
 @gen_cluster(client=True, nthreads=[])
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 14cc084f5dc..930239e5c9e 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -8,39 +8,40 @@
 from time import sleep
 
 import cloudpickle
+import pytest
+from tlz import concat, first, frequencies, merge, valmap
+
 import dask
 from dask import delayed
-from tlz import merge, concat, valmap, first, frequencies
-
-import pytest
+from dask.compatibility import apply
 
-from distributed import Nanny, Worker, Client, wait, fire_and_forget
+from distributed import Client, Nanny, Worker, fire_and_forget, wait
+from distributed.client import wait
 from distributed.comm import Comm
 from distributed.compatibility import MACOS
-from distributed.core import connect, rpc, ConnectionPool, Status
-from distributed.scheduler import Scheduler
-from distributed.client import wait
+from distributed.core import ConnectionPool, Status, connect, rpc
 from distributed.metrics import time
 from distributed.protocol.pickle import dumps
-from distributed.worker import dumps_function, dumps_task
-from distributed.utils import tmpfile, typename, TimeoutError
+from distributed.scheduler import Scheduler
+from distributed.utils import TimeoutError, tmpfile, typename
 from distributed.utils_test import (  # noqa: F401
     captured_logger,
     cleanup,
-    inc,
+    cluster,
     dec,
+    div,
     gen_cluster,
     gen_test,
-    slowinc,
+    inc,
+    loop,
+    nodebug,
     slowadd,
     slowdec,
-    cluster,
-    div,
-    varying,
+    slowinc,
     tls_only_security,
+    varying,
 )
-from distributed.utils_test import loop, nodebug  # noqa: F401
-from dask.compatibility import apply
+from distributed.worker import dumps_function, dumps_task
 
 if sys.version_info < (3, 8):
     try:
diff --git a/distributed/tests/test_security.py b/distributed/tests/test_security.py
index dc1aee9e9b6..aab34f0f2b1 100644
--- a/distributed/tests/test_security.py
+++ b/distributed/tests/test_security.py
@@ -7,12 +7,12 @@
 
 import pytest
 
+import dask
+
 from distributed.comm import connect, listen
 from distributed.security import Security
 from distributed.utils_test import get_cert
 
-import dask
-
 ca_file = get_cert("tls-ca-cert.pem")
 
 cert1 = get_cert("tls-cert.pem")
diff --git a/distributed/tests/test_semaphore.py b/distributed/tests/test_semaphore.py
index e23864d811f..caf4f106815 100644
--- a/distributed/tests/test_semaphore.py
+++ b/distributed/tests/test_semaphore.py
@@ -1,27 +1,30 @@
 import asyncio
-from datetime import timedelta
+import logging
 import pickle
-import dask
+from datetime import timedelta
+from time import sleep, time
+
 import pytest
+
+import dask
 from dask.distributed import Client
-from time import time, sleep
+
 from distributed import Semaphore, fire_and_forget
 from distributed.comm import Comm
 from distributed.compatibility import WINDOWS
 from distributed.core import ConnectionPool
 from distributed.metrics import time
 from distributed.utils_test import (  # noqa: F401
-    client,
-    cleanup,
-    cluster,
     async_wait_for,
     captured_logger,
+    cleanup,
+    client,
+    cluster,
     cluster_fixture,
     gen_cluster,
-    slowidentity,
     loop,
+    slowidentity,
 )
-import logging
 
 
 @gen_cluster(client=True)
diff --git a/distributed/tests/test_sizeof.py b/distributed/tests/test_sizeof.py
index a92d2900ae8..49947a0d50e 100644
--- a/distributed/tests/test_sizeof.py
+++ b/distributed/tests/test_sizeof.py
@@ -1,5 +1,7 @@
-import pytest
 import logging
+
+import pytest
+
 from dask.sizeof import sizeof
 
 from distributed.sizeof import safe_sizeof
diff --git a/distributed/tests/test_steal.py b/distributed/tests/test_steal.py
index de7ffda8f83..b5abb67d35a 100644
--- a/distributed/tests/test_steal.py
+++ b/distributed/tests/test_steal.py
@@ -7,8 +7,11 @@
 from operator import mul
 from time import sleep
 
-import dask
 import pytest
+from tlz import concat, sliding_window
+
+import dask
+
 from distributed import Nanny, Worker, wait, worker_client
 from distributed.config import config
 from distributed.metrics import time
@@ -24,7 +27,6 @@
     slowidentity,
     slowinc,
 )
-from tlz import concat, sliding_window
 
 # Most tests here are timing-dependent
 setup_module = nodebug_setup_module
diff --git a/distributed/tests/test_stress.py b/distributed/tests/test_stress.py
index 2cc65b29ae0..b3424ed4c62 100644
--- a/distributed/tests/test_stress.py
+++ b/distributed/tests/test_stress.py
@@ -1,34 +1,32 @@
 import asyncio
-from contextlib import suppress
 import random
 import sys
+from contextlib import suppress
 from operator import add
 from time import sleep
 
-from dask import delayed
 import pytest
 from tlz import concat, sliding_window
 
-from distributed import Client, wait, Nanny
+from dask import delayed
+
+from distributed import Client, Nanny, wait
+from distributed.client import wait
 from distributed.config import config
 from distributed.metrics import time
 from distributed.utils import All, CancelledError
-from distributed.utils_test import (
-    gen_cluster,
+from distributed.utils_test import (  # noqa: F401
+    bump_rlimit,
     cluster,
+    gen_cluster,
     inc,
-    slowinc,
-    slowadd,
-    slowsum,
-    bump_rlimit,
-)
-from distributed.utils_test import (  # noqa: F401
     loop,
     nodebug_setup_module,
     nodebug_teardown_module,
+    slowadd,
+    slowinc,
+    slowsum,
 )
-from distributed.client import wait
-
 
 # All tests here are slow in some way
 setup_module = nodebug_setup_module
@@ -254,11 +252,13 @@ async def test_no_delay_during_large_transfer(c, s, w):
     x_nbytes = x.nbytes
 
     # Reset digests
-    from distributed.counter import Digest
     from collections import defaultdict
     from functools import partial
+
     from dask.diagnostics import ResourceProfiler
 
+    from distributed.counter import Digest
+
     for server in [s, w]:
         server.digests = defaultdict(partial(Digest, loop=server.io_loop))
         server._last_tick = time()
diff --git a/distributed/tests/test_threadpoolexecutor.py b/distributed/tests/test_threadpoolexecutor.py
index 6da81e39288..ec26be4cd19 100644
--- a/distributed/tests/test_threadpoolexecutor.py
+++ b/distributed/tests/test_threadpoolexecutor.py
@@ -1,8 +1,8 @@
-from time import sleep
 import threading
+from time import sleep
 
 from distributed.metrics import time
-from distributed.threadpoolexecutor import ThreadPoolExecutor, secede, rejoin
+from distributed.threadpoolexecutor import ThreadPoolExecutor, rejoin, secede
 
 
 def test_tpe():
diff --git a/distributed/tests/test_tls_functional.py b/distributed/tests/test_tls_functional.py
index 3002b0a2c43..59e0bbb429d 100644
--- a/distributed/tests/test_tls_functional.py
+++ b/distributed/tests/test_tls_functional.py
@@ -3,21 +3,22 @@
 Most are taken from other test files and adapted.
 """
 import asyncio
+
 import pytest
 
-from distributed import Scheduler, Worker, Client, Nanny, worker_client, Queue
-from distributed.core import Status
+from distributed import Client, Nanny, Queue, Scheduler, Worker, worker_client
 from distributed.client import wait
+from distributed.core import Status
 from distributed.metrics import time
 from distributed.nanny import Nanny
 from distributed.utils_test import (  # noqa: F401
+    cleanup,
+    double,
     gen_tls_cluster,
     inc,
-    double,
-    slowinc,
     slowadd,
+    slowinc,
     tls_config,
-    cleanup,
 )
 
 
diff --git a/distributed/tests/test_utils.py b/distributed/tests/test_utils.py
index 8b13b6b27fb..669368c70fd 100644
--- a/distributed/tests/test_utils.py
+++ b/distributed/tests/test_utils.py
@@ -1,52 +1,61 @@
-import asyncio
 import array
+import asyncio
 import datetime
-from functools import partial
 import io
 import os
 import queue
 import socket
 import sys
-from time import sleep
 import traceback
+from functools import partial
+from time import sleep
 
 import pytest
 from tornado.ioloop import IOLoop
 
 import dask
+
 from distributed.metrics import time
 from distributed.utils import (
+    LRU,
     All,
     Log,
     Logs,
-    sync,
-    is_kernel,
-    is_valid_xml,
-    ensure_ip,
-    truncate_exception,
-    get_traceback,
+    LoopRunner,
+    TimeoutError,
     _maybe_complex,
-    read_block,
-    seek_delimiter,
-    funcname,
     ensure_bytes,
-    open_port,
+    ensure_ip,
+    format_dashboard_link,
+    funcname,
     get_ip_interface,
+    get_traceback,
+    is_kernel,
+    is_valid_xml,
     nbytes,
-    set_thread_state,
-    thread_state,
-    LoopRunner,
+    offload,
+    open_port,
     parse_bytes,
-    parse_timedelta,
     parse_ports,
+    parse_timedelta,
+    read_block,
+    seek_delimiter,
+    set_thread_state,
+    sync,
+    thread_state,
+    truncate_exception,
     warn_on_duration,
-    format_dashboard_link,
-    LRU,
-    offload,
-    TimeoutError,
 )
-from distributed.utils_test import loop, loop_in_thread  # noqa: F401
-from distributed.utils_test import div, has_ipv6, inc, throws, gen_test, captured_logger
+from distributed.utils_test import (  # noqa: F401
+    captured_logger,
+    div,
+    gen_test,
+    has_ipv6,
+    inc,
+    loop,
+    loop_in_thread,
+    throws,
+)
 
 
 def test_All(loop):
diff --git a/distributed/tests/test_utils_comm.py b/distributed/tests/test_utils_comm.py
index 7ab793e18e4..c0bc721f51b 100644
--- a/distributed/tests/test_utils_comm.py
+++ b/distributed/tests/test_utils_comm.py
@@ -1,12 +1,12 @@
-from distributed.core import ConnectionPool
-from distributed.comm import Comm
-from distributed.utils_test import gen_cluster, loop  # noqa: F401
-from distributed.utils_comm import pack_data, subs_multiple, gather_from_workers, retry
-
 from unittest import mock
 
 import pytest
 
+from distributed.comm import Comm
+from distributed.core import ConnectionPool
+from distributed.utils_comm import gather_from_workers, pack_data, retry, subs_multiple
+from distributed.utils_test import gen_cluster, loop  # noqa: F401
+
 
 def test_pack_data():
     data = {"x": 1}
diff --git a/distributed/tests/test_utils_test.py b/distributed/tests/test_utils_test.py
index 6093d2eaea7..4bf0230548b 100755
--- a/distributed/tests/test_utils_test.py
+++ b/distributed/tests/test_utils_test.py
@@ -1,33 +1,30 @@
 import asyncio
-from contextlib import contextmanager
 import socket
 import threading
+from contextlib import contextmanager
 from time import sleep
 
 import pytest
 from tornado import gen
 
-from distributed import Scheduler, Worker, Nanny, Client, config, default_client
+from distributed import Client, Nanny, Scheduler, Worker, config, default_client
 from distributed.core import rpc
 from distributed.metrics import time
+from distributed.utils import get_ip
 from distributed.utils_test import (  # noqa: F401
     cleanup,
     cluster,
     gen_cluster,
-    inc,
     gen_test,
-    wait_for_port,
-    new_config,
-)
-
-from distributed.utils_test import (  # noqa: F401
+    inc,
     loop,
-    tls_only_security,
+    new_config,
     security,
     tls_client,
     tls_cluster,
+    tls_only_security,
+    wait_for_port,
 )
-from distributed.utils import get_ip
 
 
 def test_bare_cluster(loop):
diff --git a/distributed/tests/test_variable.py b/distributed/tests/test_variable.py
index 37b3c756be7..9763de48335 100644
--- a/distributed/tests/test_variable.py
+++ b/distributed/tests/test_variable.py
@@ -1,17 +1,25 @@
 import asyncio
+import logging
 import random
 from datetime import timedelta
-from time import sleep, monotonic
-import logging
+from time import monotonic, sleep
 
 import pytest
 from tornado.ioloop import IOLoop
 
-from distributed import Client, Variable, worker_client, Nanny, wait, TimeoutError
-from distributed.metrics import time
+from distributed import Client, Nanny, TimeoutError, Variable, wait, worker_client
 from distributed.compatibility import WINDOWS
-from distributed.utils_test import gen_cluster, inc, div, captured_logger, popen
-from distributed.utils_test import client, cluster_fixture, loop  # noqa: F401
+from distributed.metrics import time
+from distributed.utils_test import (  # noqa: F401
+    captured_logger,
+    client,
+    cluster_fixture,
+    div,
+    gen_cluster,
+    inc,
+    loop,
+    popen,
+)
 
 
 @gen_cluster(client=True)
diff --git a/distributed/tests/test_versions.py b/distributed/tests/test_versions.py
index 7b09d5299c6..345cee3a253 100644
--- a/distributed/tests/test_versions.py
+++ b/distributed/tests/test_versions.py
@@ -3,10 +3,9 @@
 
 import pytest
 
-from distributed.versions import get_versions, error_message
 from distributed import Client, Worker
 from distributed.utils_test import gen_cluster, loop  # noqa: F401
-
+from distributed.versions import error_message, get_versions
 
 # if one of the nodes reports this version, there's a mismatch
 mismatched_version = get_versions()
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index b15030d3599..775703ef1d3 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -1,61 +1,60 @@
-from concurrent.futures import ThreadPoolExecutor
+import asyncio
 import importlib
 import logging
-from numbers import Number
-from operator import add
 import os
-import psutil
 import sys
-from time import sleep
 import threading
 import traceback
+from concurrent.futures import ThreadPoolExecutor
+from numbers import Number
+from operator import add
+from time import sleep
 from unittest import mock
-import asyncio
+
+import psutil
+import pytest
+from tlz import first, pluck, sliding_window
 
 import dask
 from dask import delayed
-from dask.utils import format_bytes
 from dask.system import CPU_COUNT
-import pytest
-from tlz import pluck, sliding_window, first
+from dask.utils import format_bytes
 
 from distributed import (
     Client,
     Nanny,
-    get_client,
+    Reschedule,
     default_client,
+    get_client,
     get_worker,
-    Reschedule,
     wait,
 )
-from distributed.diagnostics.plugin import PipInstall
 from distributed.compatibility import MACOS, WINDOWS
-from distributed.core import rpc, CommClosedError, Status
-from distributed.scheduler import Scheduler
+from distributed.core import CommClosedError, Status, rpc
+from distributed.diagnostics.plugin import PipInstall
 from distributed.metrics import time
-from distributed.worker import Worker, error_message, logger, parse_memory_limit
-from distributed.utils import tmpfile, TimeoutError
+from distributed.scheduler import Scheduler
+from distributed.utils import TimeoutError, tmpfile
 from distributed.utils_test import (  # noqa: F401
+    TaskStateMetadataPlugin,
+    a,
+    b,
+    captured_logger,
     cleanup,
-    inc,
-    mul,
-    gen_cluster,
-    div,
+    client,
+    cluster_fixture,
     dec,
-    slowinc,
+    div,
+    gen_cluster,
     gen_test,
-    captured_logger,
-)
-from distributed.utils_test import (  # noqa: F401
-    client,
+    inc,
     loop,
+    mul,
     nodebug,
-    cluster_fixture,
     s,
-    a,
-    b,
-    TaskStateMetadataPlugin,
+    slowinc,
 )
+from distributed.worker import Worker, error_message, logger, parse_memory_limit
 
 
 @pytest.mark.asyncio
diff --git a/distributed/tests/test_worker_client.py b/distributed/tests/test_worker_client.py
index 09ae20e8f20..451e9116360 100644
--- a/distributed/tests/test_worker_client.py
+++ b/distributed/tests/test_worker_client.py
@@ -1,24 +1,31 @@
 import asyncio
 import random
 import threading
-from time import sleep
 import warnings
+from time import sleep
+
+import pytest
 
 import dask
 from dask import delayed
-import pytest
 
 from distributed import (
-    worker_client,
     Client,
     as_completed,
+    get_client,
     get_worker,
     wait,
-    get_client,
+    worker_client,
 )
 from distributed.metrics import time
-from distributed.utils_test import double, gen_cluster, inc
-from distributed.utils_test import client, cluster_fixture, loop  # noqa: F401
+from distributed.utils_test import (  # noqa: F401
+    client,
+    cluster_fixture,
+    double,
+    gen_cluster,
+    inc,
+    loop,
+)
 
 
 @gen_cluster(client=True)
diff --git a/distributed/threadpoolexecutor.py b/distributed/threadpoolexecutor.py
index cc4a9894823..2bd224ff4e6 100644
--- a/distributed/threadpoolexecutor.py
+++ b/distributed/threadpoolexecutor.py
@@ -20,13 +20,13 @@
 
    Copyright 2001-2016 Python Software Foundation; All Rights Reserved
 """
-from . import _concurrent_futures_thread as thread
-import os
+import itertools
 import logging
+import os
 import queue
 import threading
-import itertools
 
+from . import _concurrent_futures_thread as thread
 from .metrics import time
 
 logger = logging.getLogger(__name__)
diff --git a/distributed/utils.py b/distributed/utils.py
index 165b13ecc21..00c4dab50dc 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -1,57 +1,58 @@
 import asyncio
-from asyncio import TimeoutError
-import click
-from collections import deque, OrderedDict, UserDict
-from concurrent.futures import ThreadPoolExecutor, CancelledError  # noqa: F401
-from contextlib import contextmanager, suppress
 import functools
-from hashlib import md5
 import html
+import importlib
+import inspect
 import json
 import logging
 import multiprocessing
 import os
+import pkgutil
 import re
 import shutil
 import socket
-from time import sleep
-import importlib
-from importlib.util import cache_from_source
-import inspect
 import sys
 import tempfile
 import threading
 import warnings
 import weakref
-import pkgutil
-import tblib.pickling_support
 import xml.etree.ElementTree
+from asyncio import TimeoutError
+from collections import OrderedDict, UserDict, deque
+from concurrent.futures import CancelledError, ThreadPoolExecutor  # noqa: F401
+from contextlib import contextmanager, suppress
+from hashlib import md5
+from importlib.util import cache_from_source
+from time import sleep
+
+import click
+import tblib.pickling_support
 
 try:
     import resource
 except ImportError:
     resource = None
 
+import tlz as toolz
+from tornado import gen
+from tornado.ioloop import IOLoop
+
 import dask
 from dask import istask
 
 # Import config serialization functions here for backward compatibility
-from dask.config import serialize as serialize_for_cli  # noqa
 from dask.config import deserialize as deserialize_for_cli  # noqa
+from dask.config import serialize as serialize_for_cli  # noqa
 
 # provide format_bytes here for backwards compatibility
-from dask.utils import (  # noqa
+from dask.utils import (  # noqa: F401
     format_bytes,
-    funcname,
     format_time,
+    funcname,
     parse_bytes,
     parse_timedelta,
 )
 
-import tlz as toolz
-from tornado import gen
-from tornado.ioloop import IOLoop
-
 try:
     from tornado.ioloop import PollIOLoop
 except ImportError:
@@ -60,7 +61,6 @@
 from .compatibility import PYPY, WINDOWS
 from .metrics import time
 
-
 try:
     from dask.context import thread_state
 except ImportError:
@@ -89,7 +89,7 @@ def _initialize_mp_context():
         if "pkg_resources" in sys.modules:
             preload.append("pkg_resources")
 
-        from .versions import required_packages, optional_packages
+        from .versions import optional_packages, required_packages
 
         for pkg, _ in required_packages + optional_packages:
             try:
diff --git a/distributed/utils_comm.py b/distributed/utils_comm.py
index 4175e0ee495..80e6e0b8ae4 100644
--- a/distributed/utils_comm.py
+++ b/distributed/utils_comm.py
@@ -1,14 +1,15 @@
 import asyncio
+import logging
+import random
 from collections import defaultdict
 from functools import partial
 from itertools import cycle
-import logging
-import random
 
-from dask.optimization import SubgraphCallable
+from tlz import concat, drop, groupby, merge
+
 import dask.config
+from dask.optimization import SubgraphCallable
 from dask.utils import parse_timedelta, stringify
-from tlz import merge, concat, groupby, drop
 
 from .core import rpc
 from .utils import All
diff --git a/distributed/utils_perf.py b/distributed/utils_perf.py
index 3b97dd46327..c81eb013c78 100644
--- a/distributed/utils_perf.py
+++ b/distributed/utils_perf.py
@@ -1,14 +1,13 @@
-from collections import deque
 import gc
 import logging
 import threading
+from collections import deque
 
 from dask.utils import format_bytes
 
 from .compatibility import PYPY
 from .metrics import thread_time
 
-
 logger = _logger = logging.getLogger(__name__)
 
 
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index e253d7760c7..59d4a8d24cd 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -1,10 +1,8 @@
 import asyncio
 import collections
-import gc
-from contextlib import contextmanager, suppress
 import copy
 import functools
-from glob import glob
+import gc
 import io
 import itertools
 import logging
@@ -19,10 +17,12 @@
 import sys
 import tempfile
 import threading
-from time import sleep
 import uuid
 import warnings
 import weakref
+from contextlib import contextmanager, suppress
+from glob import glob
+from time import sleep
 
 try:
     import ssl
@@ -30,38 +30,38 @@
     ssl = None
 
 import pytest
-
-import dask
-from tlz import merge, memoize, assoc
+from tlz import assoc, memoize, merge
 from tornado import gen
 from tornado.ioloop import IOLoop
 
+import dask
+
 from . import system
-from .client import default_client, _global_clients, Client
-from .compatibility import WINDOWS
+from .client import Client, _global_clients, default_client
 from .comm import Comm
+from .compatibility import WINDOWS
 from .config import initialize_logging
-from .core import connect, rpc, CommClosedError, Status
+from .core import CommClosedError, Status, connect, rpc
 from .deploy import SpecCluster
+from .diagnostics.plugin import WorkerPlugin
 from .metrics import time
+from .nanny import Nanny
 from .proctitle import enable_proctitle_on_children
 from .security import Security
 from .utils import (
-    log_errors,
-    mp_context,
+    DequeHandler,
+    TimeoutError,
+    _offload_executor,
     get_ip,
     get_ipv6,
-    DequeHandler,
+    iscoroutinefunction,
+    log_errors,
+    mp_context,
     reset_logger_locks,
     sync,
-    iscoroutinefunction,
     thread_state,
-    _offload_executor,
-    TimeoutError,
 )
 from .worker import Worker
-from .nanny import Nanny
-from .diagnostics.plugin import WorkerPlugin
 
 try:
     import dask.array  # register config
@@ -190,6 +190,7 @@ def pristine_loop():
 @contextmanager
 def mock_ipython():
     from unittest import mock
+
     from distributed._ipython_utils import remote_magic
 
     ip = mock.Mock()
diff --git a/distributed/variable.py b/distributed/variable.py
index 83bea566b45..5be0d097915 100644
--- a/distributed/variable.py
+++ b/distributed/variable.py
@@ -1,14 +1,15 @@
 import asyncio
-from collections import defaultdict
-from contextlib import suppress
 import logging
 import uuid
+from collections import defaultdict
+from contextlib import suppress
 
 from tlz import merge
 
 from dask.utils import stringify
-from .client import Future, Client
-from .utils import log_errors, TimeoutError, parse_timedelta
+
+from .client import Client, Future
+from .utils import TimeoutError, log_errors, parse_timedelta
 from .worker import get_client, get_worker
 
 logger = logging.getLogger(__name__)
diff --git a/distributed/versions.py b/distributed/versions.py
index a276eb418ca..d8af5eabcd0 100644
--- a/distributed/versions.py
+++ b/distributed/versions.py
@@ -1,13 +1,12 @@
 """ utilities for package version introspection """
 
-from __future__ import print_function, division, absolute_import
+from __future__ import absolute_import, division, print_function
 
+import importlib
+import os
 import platform
 import struct
-import os
 import sys
-import importlib
-
 
 required_packages = [
     ("dask", lambda p: p.__version__),
diff --git a/distributed/worker.py b/distributed/worker.py
index f448f9730e1..affdbb8d8b2 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -1,77 +1,81 @@
 import asyncio
 import bisect
-
-from collections import defaultdict, deque, namedtuple
-from collections.abc import MutableMapping
-from contextlib import suppress
-from datetime import timedelta
 import errno
-from functools import partial
 import heapq
-from inspect import isawaitable
 import logging
 import os
-from pickle import PicklingError
 import random
-import threading
 import sys
+import threading
 import uuid
 import warnings
 import weakref
+from collections import defaultdict, deque, namedtuple
+from collections.abc import MutableMapping
+from contextlib import suppress
+from datetime import timedelta
+from functools import partial
+from inspect import isawaitable
+from pickle import PicklingError
+
+from tlz import first, keymap, merge, pluck  # noqa: F401
+from tornado import gen
+from tornado.ioloop import IOLoop, PeriodicCallback
 
 import dask
-from dask.core import istask
 from dask.compatibility import apply
-from dask.utils import format_bytes, funcname
+from dask.core import istask
 from dask.system import CPU_COUNT
+from dask.utils import format_bytes, funcname
 
-from tlz import pluck, first, keymap
-from tornado import gen
-from tornado.ioloop import IOLoop, PeriodicCallback
-
-from . import profile, comm, system
+from . import comm, preloading, profile, system
 from .batched import BatchedSend
-from .comm import get_address_host, connect
+from .comm import connect, get_address_host
 from .comm.addressing import address_from_user_args
 from .comm.utils import OFFLOAD_THRESHOLD
-from .core import error_message, CommClosedError, send_recv, pingpong, coerce_to_address
+from .core import (
+    CommClosedError,
+    Status,
+    coerce_to_address,
+    error_message,
+    pingpong,
+    send_recv,
+)
 from .diskutils import WorkSpace
 from .http import get_handlers
 from .metrics import time
 from .node import ServerNode
-from . import preloading
 from .proctitle import setproctitle
-from .protocol import pickle, to_serialize, deserialize_bytes, serialize_bytelist
+from .protocol import deserialize_bytes, pickle, serialize_bytelist, to_serialize
 from .pubsub import PubSubWorkerExtension
 from .security import Security
 from .sizeof import safe_sizeof as sizeof
-from .threadpoolexecutor import ThreadPoolExecutor, secede as tpe_secede
+from .threadpoolexecutor import ThreadPoolExecutor
+from .threadpoolexecutor import secede as tpe_secede
 from .utils import (
+    LRU,
+    TimeoutError,
+    _maybe_complex,
     get_ip,
-    typename,
     has_arg,
-    _maybe_complex,
-    log_errors,
     import_file,
-    silence_logging,
-    thread_state,
+    iscoroutinefunction,
     json_load_robust,
     key_split,
+    log_errors,
     offload,
     parse_bytes,
-    parse_timedelta,
     parse_ports,
-    iscoroutinefunction,
+    parse_timedelta,
+    silence_logging,
+    thread_state,
+    typename,
     warn_on_duration,
-    LRU,
-    TimeoutError,
 )
-from .utils_comm import pack_data, gather_from_workers, retry_operation
-from .utils_perf import ThrottledGC, enable_gc_diagnosis, disable_gc_diagnosis
+from .utils_comm import gather_from_workers, pack_data, retry_operation
+from .utils_perf import ThrottledGC, disable_gc_diagnosis, enable_gc_diagnosis
 from .versions import get_versions
 
-from .core import Status
-
 logger = logging.getLogger(__name__)
 
 LOG_PDB = dask.config.get("distributed.admin.pdb-on-err")
diff --git a/distributed/worker_client.py b/distributed/worker_client.py
index 2f6eec73c62..7ad1cb7e20b 100644
--- a/distributed/worker_client.py
+++ b/distributed/worker_client.py
@@ -1,10 +1,11 @@
-from contextlib import contextmanager
 import warnings
+from contextlib import contextmanager
 
 import dask
-from .threadpoolexecutor import secede, rejoin
-from .worker import thread_state, get_client, get_worker
+
+from .threadpoolexecutor import rejoin, secede
 from .utils import parse_timedelta
+from .worker import get_client, get_worker, thread_state
 
 
 @contextmanager
diff --git a/docs/source/conf.py b/docs/source/conf.py
index d83c6b9d2c6..201cd76b00b 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -425,12 +425,12 @@ def copy_legacy_redirects(app, docname):
                 f.write(page)
 
 
+from docutils.parsers.rst import directives
+
 # -- Configuration to keep autosummary in sync with autoclass::members ----------------------------------------------
 # Fixes issues/3693
 # See https://stackoverflow.com/questions/20569011/python-sphinx-autosummary-automated-listing-of-member-functions
-from sphinx.ext.autosummary import Autosummary
-from sphinx.ext.autosummary import get_documenter
-from docutils.parsers.rst import directives
+from sphinx.ext.autosummary import Autosummary, get_documenter
 from sphinx.util.inspect import safe_getattr
 
 
diff --git a/setup.cfg b/setup.cfg
index 0689b58795d..6016c5fb133 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -27,6 +27,15 @@ ignore =
 
 max-line-length = 120
 
+[isort]
+sections = FUTURE,STDLIB,THIRDPARTY,DISTRIBUTED,FIRSTPARTY,LOCALFOLDER
+profile = black
+skip_gitignore = true
+force_to_top = true
+default_section = THIRDPARTY
+known_first_party = distributed
+known_distributed = dask
+
 [versioneer]
 VCS = git
 style = pep440
diff --git a/setup.py b/setup.py
index 8aaaeee5de8..61f2193d6c3 100755
--- a/setup.py
+++ b/setup.py
@@ -2,8 +2,10 @@
 
 import os
 import sys
-from setuptools import setup, find_packages
+
+from setuptools import find_packages, setup
 from setuptools.extension import Extension
+
 import versioneer
 
 requires = open("requirements.txt").read().strip().split("\n")
diff --git a/versioneer.py b/versioneer.py
index fa805b12b0f..bf23b9a50d4 100644
--- a/versioneer.py
+++ b/versioneer.py
@@ -277,10 +277,12 @@
 """
 
 from __future__ import print_function
+
 try:
     import configparser
 except ImportError:
     import ConfigParser as configparser
+
 import errno
 import json
 import os
@@ -1557,6 +1559,7 @@ def run(self):
 
     if "cx_Freeze" in sys.modules:  # cx_freeze enabled?
         from cx_Freeze.dist import build_exe as _build_exe
+
         # nczeczulin reports that py2exe won't like the pep440-style string
         # as FILEVERSION, but it can be used for PRODUCTVERSION, e.g.
         # setup(console=[{

From cc3074f5fb4647043241863106a8728aac038748 Mon Sep 17 00:00:00 2001
From: Gerald <awesomegtrr@gmail.com>
Date: Wed, 31 Mar 2021 16:57:15 +0800
Subject: [PATCH 1212/1550] ignore oserror exception when scaling down
 (dask#4591) (#4633)

Co-authored-by: James Bourbeau <jrbourbeau@gmail.com>
---
 distributed/deploy/adaptive_core.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/distributed/deploy/adaptive_core.py b/distributed/deploy/adaptive_core.py
index 7c95096956a..946adae0a57 100644
--- a/distributed/deploy/adaptive_core.py
+++ b/distributed/deploy/adaptive_core.py
@@ -201,7 +201,8 @@ async def adapt(self) -> None:
             if status == "down":
                 await self.scale_down(**recommendations)
         except OSError as e:
-            logger.error("Adaptive stopping due to error %s", str(e))
-            self.stop()
+            if status != "down":
+                logger.error("Adaptive stopping due to error %s", str(e))
+                self.stop()
         finally:
             self._adapting = False

From faf0a7070a1cba6eec8c8840f8cae0b71e409f46 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Wed, 31 Mar 2021 12:43:58 -0500
Subject: [PATCH 1213/1550] Avoid list mutation in pickle_loads (#4653)

Fixes https://github.com/dask/distributed/issues/4625
---
 distributed/protocol/serialize.py             | 22 ++++++++++---------
 distributed/protocol/tests/test_collection.py |  1 +
 2 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index 3179784c2a8..b9c5ad5e103 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -63,19 +63,21 @@ def pickle_dumps(x, context=None):
 
 def pickle_loads(header, frames):
     x, buffers = frames[0], frames[1:]
-    writeable = header["writeable"]
-    for i in range(len(buffers)):
-        mv = memoryview(buffers[i])
-        if writeable[i] == mv.readonly:
+
+    new = []
+    memoryviews = map(memoryview, buffers)
+    for writeable, mv in zip(header["writeable"], memoryviews):
+        if writeable == mv.readonly:
             if mv.readonly:
-                buf = memoryview(bytearray(mv))
+                mv = memoryview(bytearray(mv))
             else:
-                buf = memoryview(bytes(mv))
-            if buf.nbytes > 0:
-                buffers[i] = buf.cast(mv.format, mv.shape)
+                mv = memoryview(bytes(mv))
+            if mv.nbytes > 0:
+                mv = mv.cast(mv.format, mv.shape)
             else:
-                buffers[i] = buf.cast(mv.format)
-    return pickle.loads(x, buffers=buffers)
+                mv = mv.cast(mv.format)
+        new.append(mv)
+    return pickle.loads(x, buffers=new)
 
 
 def import_allowed_module(name):
diff --git a/distributed/protocol/tests/test_collection.py b/distributed/protocol/tests/test_collection.py
index 46b88a3a691..32d11a74755 100644
--- a/distributed/protocol/tests/test_collection.py
+++ b/distributed/protocol/tests/test_collection.py
@@ -21,6 +21,7 @@ def test_serialize_collection(collection, y, y_serializer):
         header, frames = serialize({"x": x, "y": y}, serializers=("dask", "pickle"))
     else:
         header, frames = serialize(collection((x, y)), serializers=("dask", "pickle"))
+    frames = tuple(frames)  # verify that no mutation occurs
     t = deserialize(header, frames, deserializers=("dask", "pickle", "error"))
     assert isinstance(t, collection)
 

From f19aae3fcd797f887a298c2889c96327d60670bf Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Wed, 31 Mar 2021 15:19:08 -0700
Subject: [PATCH 1214/1550] Treat empty/missing `writeable` as a no-op (#4659)

Since `writeable` filled with `None` will not equal `True` or `False`,
this will effectively be treated as a no-op along frames to be pushed
into `new` without copying.

Should fix some issues that users are running into around serialization
of data.
---
 distributed/protocol/serialize.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index b9c5ad5e103..50d87953111 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -64,10 +64,14 @@ def pickle_dumps(x, context=None):
 def pickle_loads(header, frames):
     x, buffers = frames[0], frames[1:]
 
+    writeable = header.get("writeable")
+    if not writeable:
+        writeable = len(buffers) * (None,)
+
     new = []
     memoryviews = map(memoryview, buffers)
-    for writeable, mv in zip(header["writeable"], memoryviews):
-        if writeable == mv.readonly:
+    for w, mv in zip(writeable, memoryviews):
+        if w == mv.readonly:
             if mv.readonly:
                 mv = memoryview(bytearray(mv))
             else:
@@ -77,6 +81,7 @@ def pickle_loads(header, frames):
             else:
                 mv = mv.cast(mv.format)
         new.append(mv)
+
     return pickle.loads(x, buffers=new)
 
 

From aac50f63fdacfb43be64279ab540f68cabe7351b Mon Sep 17 00:00:00 2001
From: cameron16 <cb596@cornell.edu>
Date: Wed, 31 Mar 2021 23:27:33 -0400
Subject: [PATCH 1215/1550] Remove incorrect comment regarding default
 LocalCluster creation (#4660)

Co-authored-by: cameron16 <cboroumand@gmail.com>
---
 distributed/deploy/local.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index 7e2ed49b5fc..e1f18734b0e 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -78,7 +78,7 @@ class LocalCluster(SpecCluster):
 
     Examples
     --------
-    >>> cluster = LocalCluster()  # Create a local cluster with as many workers as cores  # doctest: +SKIP
+    >>> cluster = LocalCluster()  # Create a local cluster  # doctest: +SKIP
     >>> cluster  # doctest: +SKIP
     LocalCluster("127.0.0.1:8786", workers=8, threads=8)
 

From d0bf77768bb64edc46e5f3f0a5af4c7c3ae0935b Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 1 Apr 2021 13:31:02 -0500
Subject: [PATCH 1216/1550] Add informative error message to install uvloop
 (#4664)

If you have a config value set to use uvloop but don't have uvloop
installed then things end badly, with a non-informative error message

This provides a more informative error message

```
RuntimeError: The distributed.admin.event-loop configuration value is set to 'uvloop' but the uvloop module is not installed

Please either change the config value or install one of the following
    conda install uvloop
    pip install uvloop
```
---
 distributed/__init__.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/distributed/__init__.py b/distributed/__init__.py
index ad1e4756141..92a62a300a4 100644
--- a/distributed/__init__.py
+++ b/distributed/__init__.py
@@ -1,6 +1,7 @@
 from . import config  # isort:skip; load distributed configuration first
 import dask
 from dask.config import config
+from dask.utils import import_required
 
 from ._version import get_versions
 from .actor import Actor, ActorFuture
@@ -45,6 +46,15 @@
 if dask.config.get("distributed.admin.event-loop") in ("asyncio", "tornado"):
     pass
 elif dask.config.get("distributed.admin.event-loop") == "uvloop":
+    import_required(
+        "uvloop",
+        "The distributed.admin.event-loop configuration value "
+        "is set to 'uvloop' but the uvloop module is not installed"
+        "\n\n"
+        "Please either change the config value or install one of the following\n"
+        "    conda install uvloop\n"
+        "    pip install uvloop",
+    )
     import uvloop
 
     uvloop.install()

From f2b516ab248ad98dc612edb33b207e5ffd6e75b8 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 1 Apr 2021 19:16:22 -0500
Subject: [PATCH 1217/1550] Fix un-merged frames (#4666)

* Add test for un-merged frames

* Don't double-split/compress Serialized frames

Previously we would re-serialize an object, even if it was a Serialized
object.  Instead we should just unpack its header and frames and be
done.

* specify num_sub_frames in all cases
---
 distributed/protocol/core.py                 | 11 +++++++----
 distributed/protocol/tests/test_serialize.py | 16 +++++++++++++++-
 2 files changed, 22 insertions(+), 5 deletions(-)

diff --git a/distributed/protocol/core.py b/distributed/protocol/core.py
index 05a804d3b52..fb85d32fee2 100644
--- a/distributed/protocol/core.py
+++ b/distributed/protocol/core.py
@@ -51,11 +51,14 @@ def _encode_default(obj):
                 if typ is Serialize:
                     obj = obj.data
                 offset = len(frames)
-                sub_header, sub_frames = serialize_and_split(
-                    obj, serializers=serializers, on_error=on_error, context=context
-                )
+                if typ is Serialized:
+                    sub_header, sub_frames = obj.header, obj.frames
+                else:
+                    sub_header, sub_frames = serialize_and_split(
+                        obj, serializers=serializers, on_error=on_error, context=context
+                    )
+                    _inplace_compress_frames(sub_header, sub_frames)
                 sub_header["num-sub-frames"] = len(sub_frames)
-                _inplace_compress_frames(sub_header, sub_frames)
                 frames.append(
                     msgpack.dumps(
                         sub_header, default=msgpack_encode_default, use_bin_type=True
diff --git a/distributed/protocol/tests/test_serialize.py b/distributed/protocol/tests/test_serialize.py
index cd2efa3bb28..cfb767bc0cb 100644
--- a/distributed/protocol/tests/test_serialize.py
+++ b/distributed/protocol/tests/test_serialize.py
@@ -13,7 +13,7 @@
 
 from dask.utils_test import inc
 
-from distributed import wait
+from distributed import Nanny, wait
 from distributed.comm.utils import from_frames, to_frames
 from distributed.protocol import (
     Serialize,
@@ -499,3 +499,17 @@ def test_ser_memoryview_object():
     data_in = memoryview(np.array(["hello"], dtype=object))
     with pytest.raises(TypeError):
         serialize(data_in, on_error="raise")
+
+
+@gen_cluster(client=True, Worker=Nanny)
+async def test_large_pickled_object(c, s, a, b):
+    np = pytest.importorskip("numpy")
+
+    class Data:
+        def __init__(self, n):
+            self.data = np.empty(n, dtype="u1")
+
+    x = Data(100_000_000)
+    y = await c.scatter(x, workers=[a.worker_address])
+    z = c.submit(lambda x: x, y, workers=[b.worker_address])
+    await z

From 611414c0529daba3007a13a2ee7230e1881942b9 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Fri, 2 Apr 2021 11:22:13 -0500
Subject: [PATCH 1218/1550] bump version to 2021.04.0

---
 docs/source/changelog.rst | 29 ++++++++++++++++++++++++++++-
 1 file changed, 28 insertions(+), 1 deletion(-)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index 984e3e936df..96212d1de87 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,27 @@
 Changelog
 =========
 
+2021.04.0
+---------
+
+Released on April 2, 2021
+
+- Fix un-merged frames (:pr:`4666`) `Matthew Rocklin`_
+- Add informative error message to install uvloop (:pr:`4664`) `Matthew Rocklin`_
+- Remove incorrect comment regarding default ``LocalCluster`` creation (:pr:`4660`) `cameron16`_
+- Treat empty/missing ``writeable`` as a no-op (:pr:`4659`) `jakirkham`_
+- Avoid list mutation in ``pickle_loads`` (:pr:`4653`) `Matthew Rocklin`_
+- Ignore ``OSError`` exception when scaling down (:pr:`4633`) `Gerald`_
+- Add ``isort`` to pre-commit hooks, package resorting (:pr:`4647`) `Charles Blackmon-Luca`_
+- Use powers-of-two when displaying RAM (:pr:`4649`) `crusaderky`_
+- Support Websocket communication protocols (:pr:`4396`) `Marcos Moyano`_
+- ``scheduler.py`` / ``worker.py`` code cleanup (:pr:`4626`) `crusaderky`_
+- Update out-of-date references to ``config.yaml`` (:pr:`4643`) `Hristo Georgiev`_
+- Suppress ``OSError`` on ``SpecCluster`` shutdown (:pr:`4567`) `Jacob Tomlinson`_
+- Replace conda with mamba (:pr:`4585`) `crusaderky`_
+- Expand documentation on pure functions (:pr:`4644`) `James Lamb`_
+
+
 2021.03.1
 ---------
 
@@ -2269,4 +2290,10 @@ significantly without many new features.
 .. _`Ben Greiner`: https://github.com/bnavigator
 .. _`Fabian Gebhart`: https://github.com/fgebhart
 .. _`Sultan Orazbayev`: https://github.com/SultanOrazbayev
-.. _`Doug Davis`: https://github.com/douglasdavis
\ No newline at end of file
+.. _`Doug Davis`: https://github.com/douglasdavis
+.. _`cameron16`: https://github.com/cameron16
+.. _`Gerald`: https://github.com/gerald732
+.. _`Charles Blackmon-Luca`: https://github.com/charlesbluca
+.. _`Marcos Moyano`: https://github.com/marcosmoyano
+.. _`James Lamb`: https://github.com/jameslamb
+.. _`Hristo Georgiev`: https://github.com/hristog
\ No newline at end of file

From 843c704d0bc2f3fe576ed5b7411ca7da2c4162a4 Mon Sep 17 00:00:00 2001
From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com>
Date: Mon, 5 Apr 2021 12:00:47 -0400
Subject: [PATCH 1219/1550] Expose worker SystemMonitors to scheduler via RPC
 (#4657)

---
 distributed/scheduler.py            | 12 ++++++++++++
 distributed/tests/test_scheduler.py | 14 +++++++++++++-
 distributed/worker.py               | 12 ++++++++++++
 3 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 6d0da803bac..6540dd46dfb 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -6500,6 +6500,18 @@ def get_events(self, comm=None, topic=None):
         else:
             return valmap(tuple, self.events)
 
+    async def get_worker_monitor_info(self, recent=False, starts=None):
+        parent: SchedulerState = cast(SchedulerState, self)
+        if starts is None:
+            starts = {}
+        results = await asyncio.gather(
+            *(
+                self.rpc(w).get_monitor_info(recent=recent, start=starts.get(w, 0))
+                for w in parent._workers_dv
+            )
+        )
+        return dict(zip(parent._workers_dv, results))
+
     ###########
     # Cleanup #
     ###########
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 930239e5c9e..2669dfc53bd 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -18,7 +18,7 @@
 from distributed import Client, Nanny, Worker, fire_and_forget, wait
 from distributed.client import wait
 from distributed.comm import Comm
-from distributed.compatibility import MACOS
+from distributed.compatibility import MACOS, WINDOWS
 from distributed.core import ConnectionPool, Status, connect, rpc
 from distributed.metrics import time
 from distributed.protocol.pickle import dumps
@@ -2201,3 +2201,15 @@ async def test_configurable_events_log_length(c, s, a, b):
     assert s.events["test"][0][1] == "dummy message 2"
     assert s.events["test"][1][1] == "dummy message 3"
     assert s.events["test"][2][1] == "dummy message 4"
+
+
+@gen_cluster()
+async def test_get_worker_monitor_info(s, a, b):
+    res = await s.get_worker_monitor_info()
+    ms = ["cpu", "time", "read_bytes", "write_bytes"]
+    if not WINDOWS:
+        ms += ["num_fds"]
+    for w in (a, b):
+        assert all(res[w.address]["range_query"][m] is not None for m in ms)
+        assert res[w.address]["count"] is not None
+        assert res[w.address]["last_time"] is not None
diff --git a/distributed/worker.py b/distributed/worker.py
index affdbb8d8b2..3b365af6c6c 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -681,6 +681,7 @@ def __init__(
             "actor_execute": self.actor_execute,
             "actor_attribute": self.actor_attribute,
             "plugin-add": self.plugin_add,
+            "get_monitor_info": self.get_monitor_info,
         }
 
         stream_handlers = {
@@ -1069,6 +1070,17 @@ async def gather(self, comm=None, who_has=None):
             self.update_data(data=result, report=False)
             return {"status": "OK"}
 
+    def get_monitor_info(self, comm=None, recent=False, start=0):
+        return dict(
+            range_query=(
+                self.monitor.recent()
+                if recent
+                else self.monitor.range_query(start=start)
+            ),
+            count=self.monitor.count,
+            last_time=self.monitor.last_time,
+        )
+
     #############
     # Lifecycle #
     #############

From 4d58bcd13d1f54cda3c78eb0bc735ff105faf2a7 Mon Sep 17 00:00:00 2001
From: "Mads R. B. Kristensen" <madsbk@gmail.com>
Date: Wed, 7 Apr 2021 05:51:30 +0200
Subject: [PATCH 1220/1550] Removing dumps_msgpack() and loads_msgpack()
 (#4677)

---
 distributed/protocol/core.py | 54 +-----------------------------------
 1 file changed, 1 insertion(+), 53 deletions(-)

diff --git a/distributed/protocol/core.py b/distributed/protocol/core.py
index fb85d32fee2..0d87595bf75 100644
--- a/distributed/protocol/core.py
+++ b/distributed/protocol/core.py
@@ -2,7 +2,7 @@
 
 import msgpack
 
-from .compression import compressions, decompress, maybe_compress
+from .compression import decompress, maybe_compress
 from .serialize import (
     Serialize,
     Serialized,
@@ -111,55 +111,3 @@ def _decode_default(obj):
     except Exception:
         logger.critical("Failed to deserialize", exc_info=True)
         raise
-
-
-def dumps_msgpack(msg, compression=None):
-    """Dump msg into header and payload, both bytestrings
-
-    All of the message must be msgpack encodable
-
-    See Also:
-        loads_msgpack
-    """
-    header = {}
-    payload = msgpack.dumps(msg, default=msgpack_encode_default, use_bin_type=True)
-
-    fmt, payload = maybe_compress(payload, compression=compression)
-    if fmt:
-        header["compression"] = fmt
-
-    if header:
-        header_bytes = msgpack.dumps(header, use_bin_type=True)
-    else:
-        header_bytes = b""
-
-    return [header_bytes, payload]
-
-
-def loads_msgpack(header, payload):
-    """Read msgpack header and payload back to Python object
-
-    See Also:
-        dumps_msgpack
-    """
-    header = bytes(header)
-    if header:
-        header = msgpack.loads(
-            header, object_hook=msgpack_decode_default, use_list=False, **msgpack_opts
-        )
-    else:
-        header = {}
-
-    if header.get("compression"):
-        try:
-            decompress = compressions[header["compression"]]["decompress"]
-            payload = decompress(payload)
-        except KeyError:
-            raise ValueError(
-                "Data is compressed as %s but we don't have this"
-                " installed" % str(header["compression"])
-            )
-
-    return msgpack.loads(
-        payload, object_hook=msgpack_decode_default, use_list=False, **msgpack_opts
-    )

From c3f68293df6d869355a764c81ccfa16d8064a245 Mon Sep 17 00:00:00 2001
From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com>
Date: Wed, 7 Apr 2021 13:28:40 -0400
Subject: [PATCH 1221/1550] Add GPU metrics to SystemMonitor (#4661)

* Add queries for GPU metrics if possible

* Add test for GPU monitoring

* Add NVML info to worker monitor handle

* Update tests to use scheduler RPC call

* Run pre-commit hooks
---
 distributed/diagnostics/tests/test_nvml.py | 17 +++++++++++++++++
 distributed/system_monitor.py              | 21 +++++++++++++++++++++
 distributed/worker.py                      | 11 ++++++++++-
 3 files changed, 48 insertions(+), 1 deletion(-)

diff --git a/distributed/diagnostics/tests/test_nvml.py b/distributed/diagnostics/tests/test_nvml.py
index 6938a86850f..0b19ee759b1 100644
--- a/distributed/diagnostics/tests/test_nvml.py
+++ b/distributed/diagnostics/tests/test_nvml.py
@@ -54,3 +54,20 @@ async def test_gpu_metrics(s, a, b):
         s.workers[a.address].extra["gpu"]["name"]
         == pynvml.nvmlDeviceGetName(h).decode()
     )
+
+
+@gen_cluster()
+async def test_gpu_monitoring(s, a, b):
+    h = nvml._pynvml_handles()
+    res = await s.get_worker_monitor_info(recent=True)
+
+    assert (
+        res[a.address]["range_query"]["gpu_utilization"]
+        == pynvml.nvmlDeviceGetUtilizationRates(h).gpu
+    )
+    assert (
+        res[a.address]["range_query"]["gpu_memory_used"]
+        == pynvml.nvmlDeviceGetMemoryInfo(h).used
+    )
+    assert res[a.address]["gpu_name"] == pynvml.nvmlDeviceGetName(h).decode()
+    assert res[a.address]["gpu_memory_total"] == pynvml.nvmlDeviceGetMemoryInfo(h).total
diff --git a/distributed/system_monitor.py b/distributed/system_monitor.py
index 2f8e64c6640..acaeb16c7da 100644
--- a/distributed/system_monitor.py
+++ b/distributed/system_monitor.py
@@ -5,6 +5,11 @@
 from .compatibility import WINDOWS
 from .metrics import time
 
+try:
+    from .diagnostics import nvml
+except Exception:
+    nvml = None
+
 
 class SystemMonitor:
     def __init__(self, n=10000):
@@ -34,6 +39,15 @@ def __init__(self, n=10000):
             self.num_fds = deque(maxlen=n)
             self.quantities["num_fds"] = self.num_fds
 
+        if nvml is not None:
+            gpu_extra = nvml.one_time()
+            self.gpu_name = gpu_extra["name"]
+            self.gpu_memory_total = gpu_extra["memory-total"]
+            self.gpu_utilization = deque(maxlen=n)
+            self.gpu_memory_used = deque(maxlen=n)
+            self.quantities["gpu_utilization"] = self.gpu_utilization
+            self.quantities["gpu_memory_used"] = self.gpu_memory_used
+
         self.update()
 
     def recent(self):
@@ -77,6 +91,13 @@ def update(self):
             self.num_fds.append(num_fds)
             result["num_fds"] = num_fds
 
+        if nvml is not None:
+            gpu_metrics = nvml.real_time()
+            self.gpu_utilization.append(gpu_metrics["utilization"])
+            self.gpu_memory_used.append(gpu_metrics["memory-used"])
+            result["gpu_utilization"] = gpu_metrics["utilization"]
+            result["gpu_memory_used"] = gpu_metrics["memory-used"]
+
         return result
 
     def __repr__(self):
diff --git a/distributed/worker.py b/distributed/worker.py
index 3b365af6c6c..9e79ef5ab6e 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -76,6 +76,11 @@
 from .utils_perf import ThrottledGC, disable_gc_diagnosis, enable_gc_diagnosis
 from .versions import get_versions
 
+try:
+    from .diagnostics import nvml
+except Exception:
+    nvml = None
+
 logger = logging.getLogger(__name__)
 
 LOG_PDB = dask.config.get("distributed.admin.pdb-on-err")
@@ -1071,7 +1076,7 @@ async def gather(self, comm=None, who_has=None):
             return {"status": "OK"}
 
     def get_monitor_info(self, comm=None, recent=False, start=0):
-        return dict(
+        result = dict(
             range_query=(
                 self.monitor.recent()
                 if recent
@@ -1080,6 +1085,10 @@ def get_monitor_info(self, comm=None, recent=False, start=0):
             count=self.monitor.count,
             last_time=self.monitor.last_time,
         )
+        if nvml is not None:
+            result["gpu_name"] = self.monitor.gpu_name
+            result["gpu_memory_total"] = self.monitor.gpu_memory_total
+        return result
 
     #############
     # Lifecycle #

From d70314fb8c542ad3c46de724b93271dd98844a4b Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Wed, 7 Apr 2021 20:26:27 +0200
Subject: [PATCH 1222/1550] Adjust timings in test_threadpoolworkers (#4681)

---
 distributed/tests/test_semaphore.py | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/distributed/tests/test_semaphore.py b/distributed/tests/test_semaphore.py
index caf4f106815..f9895da1049 100644
--- a/distributed/tests/test_semaphore.py
+++ b/distributed/tests/test_semaphore.py
@@ -509,10 +509,23 @@ async def test_metrics(c, s, a, b):
 def test_threadpoolworkers_pick_correct_ioloop(cleanup):
     # gh4057
 
+    # About picking appropriate values for the various timings
+    # * Sleep time in `access_limited` impacts test runtime but is arbitrary
+    # * `lease-timeout` should be smaller than the sleep time. This is what the
+    #   test builds on. assuming the leases cannot be refreshed, e.g. wrong
+    #   event loop picked / PeriodicCallback never scheduled, the semaphore
+    #   would become oversubscribed and the len(protected_ressources) becomes
+    #   non zero. This should also trigger a log message about "unknown leases"
+    #   and fails the test.
+    # * `lease-validation-interval` interval should be the smallest quantity.
+    #   How often leases are checked for staleness is hard coded atm and a fifth
+    #   of the `lease-timeout`. Accounting for this and some jitter, this should
+    #   be sufficiently small to ensure smooth operation.
+
     with dask.config.set(
         {
             "distributed.scheduler.locks.lease-validation-interval": 0.01,
-            "distributed.scheduler.locks.lease-timeout": 0.05,
+            "distributed.scheduler.locks.lease-timeout": 0.1,
         }
     ):
         with Client(processes=False, threads_per_worker=4) as client:
@@ -526,7 +539,7 @@ def access_limited(val, sem):
                     assert len(protected_ressource) == 0
                     protected_ressource.append(val)
                     # Interact with the DB
-                    time.sleep(0.1)
+                    time.sleep(0.2)
                     protected_ressource.remove(val)
 
             client.gather(client.map(access_limited, range(10), sem=sem))

From 9d8ce936edf8a54d6c41dc5bfde28a6cf23056e8 Mon Sep 17 00:00:00 2001
From: Marcos Moyano <marcos@anue.biz>
Date: Thu, 8 Apr 2021 15:23:19 -0300
Subject: [PATCH 1223/1550] Add support for extra conn args for HTTP protocols
 (#4682)

Support sending extra connection information to
websockets comms, ie: headers but can map to any
init argument of tornado.httpclient.HTTPRequest
---
 distributed/comm/tests/test_ws.py  | 24 +++++++++++++++++++
 distributed/comm/ws.py             |  4 ++--
 distributed/security.py            |  9 ++++++-
 distributed/tests/test_security.py | 38 +++++++++++++++++++++++++++++-
 4 files changed, 71 insertions(+), 4 deletions(-)

diff --git a/distributed/comm/tests/test_ws.py b/distributed/comm/tests/test_ws.py
index f63c9484958..c12de0d2212 100644
--- a/distributed/comm/tests/test_ws.py
+++ b/distributed/comm/tests/test_ws.py
@@ -155,6 +155,30 @@ async def test_http_and_comm_server(cleanup, dashboard, protocol, security, port
                 assert result == 11
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "protocol,security",
+    [
+        (
+            "ws://",
+            Security(extra_conn_args={"headers": {"Authorization": "Token abcd"}}),
+        ),
+        (
+            "wss://",
+            Security.temporary(
+                extra_conn_args={"headers": {"Authorization": "Token abcd"}}
+            ),
+        ),
+    ],
+)
+async def test_connection_made_with_extra_conn_args(cleanup, protocol, security):
+    async with Scheduler(protocol=protocol, security=security) as s:
+        connection_args = security.get_connection_args("worker")
+        comm = await connect(s.address, **connection_args)
+        assert comm.sock.request.headers.get("Authorization") == "Token abcd"
+        await comm.close()
+
+
 @pytest.mark.asyncio
 async def test_quiet_close(cleanup):
     with warnings.catch_warnings(record=True) as record:
diff --git a/distributed/comm/ws.py b/distributed/comm/ws.py
index b9e0afdd7ec..4789eb1a56a 100644
--- a/distributed/comm/ws.py
+++ b/distributed/comm/ws.py
@@ -380,7 +380,7 @@ async def connect(self, address, deserialize=True, **connection_args):
         return self.comm_class(sock, deserialize=deserialize)
 
     def _get_connect_args(self, **connection_args):
-        return {}
+        return {**connection_args.get("extra_conn_args", {})}
 
 
 class WSSConnector(WSConnector):
@@ -389,7 +389,7 @@ class WSSConnector(WSConnector):
 
     def _get_connect_args(self, **connection_args):
         ctx = _expect_tls_context(connection_args)
-        return {"ssl_options": ctx}
+        return {"ssl_options": ctx, **connection_args.get("extra_conn_args", {})}
 
 
 class WSBackend(BaseTCPBackend):
diff --git a/distributed/security.py b/distributed/security.py
index 551e29c8652..d6a211571f1 100644
--- a/distributed/security.py
+++ b/distributed/security.py
@@ -45,6 +45,8 @@ class Security:
         Path to a key file for a worker, encoded in PEM format.
         Alternatively, the key may be appended to the cert file, and this
         parameter be omitted.
+    extra_conn_args : mapping, optional
+        Mapping with keyword arguments to pass down to connections.
     """
 
     __slots__ = (
@@ -57,12 +59,14 @@ class Security:
         "tls_scheduler_cert",
         "tls_worker_key",
         "tls_worker_cert",
+        "extra_conn_args",
     )
 
     def __init__(self, require_encryption=None, **kwargs):
         extra = set(kwargs).difference(self.__slots__)
         if extra:
             raise TypeError("Unknown parameters: %r" % sorted(extra))
+        self.extra_conn_args = kwargs.pop("extra_conn_args", {})
         if require_encryption is None:
             require_encryption = dask.config.get("distributed.comm.require-encryption")
         if require_encryption is None:
@@ -82,7 +86,7 @@ def __init__(self, require_encryption=None, **kwargs):
         self._set_field(kwargs, "tls_worker_cert", "distributed.comm.tls.worker.cert")
 
     @classmethod
-    def temporary(cls):
+    def temporary(cls, **kwargs):
         """Create a new temporary Security object.
 
         This creates a new self-signed key/cert pair suitable for securing
@@ -139,6 +143,7 @@ def temporary(cls):
             tls_scheduler_cert=cert_contents,
             tls_worker_key=key_contents,
             tls_worker_cert=cert_contents,
+            **kwargs,
         )
 
     def _set_field(self, kwargs, field, config_name):
@@ -150,6 +155,7 @@ def _set_field(self, kwargs, field, config_name):
 
     def __repr__(self):
         keys = sorted(self.__slots__)
+        keys.remove("extra_conn_args")
         items = []
         for k in keys:
             val = getattr(self, k)
@@ -221,6 +227,7 @@ def get_connection_args(self, role):
         return {
             "ssl_context": self._get_tls_context(tls, ssl.Purpose.SERVER_AUTH),
             "require_encryption": self.require_encryption,
+            "extra_conn_args": self.extra_conn_args,
         }
 
     def get_listen_args(self, role):
diff --git a/distributed/tests/test_security.py b/distributed/tests/test_security.py
index aab34f0f2b1..28702650a49 100644
--- a/distributed/tests/test_security.py
+++ b/distributed/tests/test_security.py
@@ -42,6 +42,7 @@ def test_defaults():
     assert sec.tls_scheduler_cert is None
     assert sec.tls_worker_key is None
     assert sec.tls_worker_cert is None
+    assert sec.extra_conn_args == {}
 
 
 def test_constructor_errors():
@@ -92,7 +93,10 @@ def test_kwargs():
     }
     with dask.config.set(c):
         sec = Security(
-            tls_scheduler_cert="newcert.pem", require_encryption=True, tls_ca_file=None
+            tls_scheduler_cert="newcert.pem",
+            require_encryption=True,
+            tls_ca_file=None,
+            extra_conn_args={"headers": {"Auth": "Token abc"}},
         )
     assert sec.require_encryption is True
     assert sec.tls_ca_file is None
@@ -103,6 +107,7 @@ def test_kwargs():
     assert sec.tls_scheduler_cert == "newcert.pem"
     assert sec.tls_worker_key is None
     assert sec.tls_worker_cert is None
+    assert sec.extra_conn_args == {"headers": {"Auth": "Token abc"}}
 
 
 def test_repr():
@@ -200,6 +205,30 @@ def basic_checks(ctx):
     assert len(tls_13_ciphers) in (0, 3)
 
 
+def test_extra_conn_args_connection_args():
+    c = {
+        "distributed.comm.tls.ca-file": ca_file,
+        "distributed.comm.tls.scheduler.key": key1,
+        "distributed.comm.tls.scheduler.cert": cert1,
+        "distributed.comm.tls.worker.cert": keycert1,
+    }
+    with dask.config.set(c):
+        sec = Security(extra_conn_args={"headers": {"Authorization": "Token abcd"}})
+
+    d = sec.get_connection_args("scheduler")
+    assert not d["require_encryption"]
+    assert d["extra_conn_args"]["headers"] == {"Authorization": "Token abcd"}
+    ctx = d["ssl_context"]
+
+    d = sec.get_connection_args("worker")
+    assert d["extra_conn_args"]["headers"] == {"Authorization": "Token abcd"}
+
+    # No cert defined => no TLS
+    d = sec.get_connection_args("client")
+    assert d.get("ssl_context") is None
+    assert d["extra_conn_args"]["headers"] == {"Authorization": "Token abcd"}
+
+
 def test_listen_args():
     def basic_checks(ctx):
         assert ctx.verify_mode == ssl.CERT_REQUIRED
@@ -376,6 +405,13 @@ def test_temporary_credentials():
         assert val not in sec_repr
 
 
+def test_extra_conn_args_in_temporary_credentials():
+    pytest.importorskip("cryptography")
+
+    sec = Security.temporary(extra_conn_args={"headers": {"X-Request-ID": "abcd"}})
+    assert sec.extra_conn_args == {"headers": {"X-Request-ID": "abcd"}}
+
+
 @pytest.mark.asyncio
 async def test_tls_temporary_credentials_functional():
     pytest.importorskip("cryptography")

From 13657a9b8e552b1b24058c09cc57eb62d009e7ac Mon Sep 17 00:00:00 2001
From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com>
Date: Thu, 8 Apr 2021 16:03:52 -0400
Subject: [PATCH 1224/1550] Don't initialize CUDA context in monitor (#4688)

---
 distributed/system_monitor.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/distributed/system_monitor.py b/distributed/system_monitor.py
index acaeb16c7da..dba32e55e40 100644
--- a/distributed/system_monitor.py
+++ b/distributed/system_monitor.py
@@ -40,9 +40,8 @@ def __init__(self, n=10000):
             self.quantities["num_fds"] = self.num_fds
 
         if nvml is not None:
-            gpu_extra = nvml.one_time()
-            self.gpu_name = gpu_extra["name"]
-            self.gpu_memory_total = gpu_extra["memory-total"]
+            self.gpu_name = None
+            self.gpu_memory_total = None
             self.gpu_utilization = deque(maxlen=n)
             self.gpu_memory_used = deque(maxlen=n)
             self.quantities["gpu_utilization"] = self.gpu_utilization
@@ -91,7 +90,12 @@ def update(self):
             self.num_fds.append(num_fds)
             result["num_fds"] = num_fds
 
-        if nvml is not None:
+        # give external modules (like dask-cuda) a chance to initialize CUDA context
+        if nvml is not None and nvml.nvmlInit is not None:
+            if self.gpu_name is None:
+                gpu_extra = nvml.one_time()
+                self.gpu_name = gpu_extra["name"]
+                self.gpu_memory_total = gpu_extra["memory-total"]
             gpu_metrics = nvml.real_time()
             self.gpu_utilization.append(gpu_metrics["utilization"])
             self.gpu_memory_used.append(gpu_metrics["memory-used"])

From 053f99b85a56a2a412f27b1950f65eb93b5bc0be Mon Sep 17 00:00:00 2001
From: Hristo Georgiev <hristog@users.noreply.github.com>
Date: Fri, 9 Apr 2021 01:12:34 +0100
Subject: [PATCH 1225/1550] Fix typo and remove unused `time.time` import
 (#4689)

---
 distributed/tests/test_semaphore.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/distributed/tests/test_semaphore.py b/distributed/tests/test_semaphore.py
index f9895da1049..8c283f3f774 100644
--- a/distributed/tests/test_semaphore.py
+++ b/distributed/tests/test_semaphore.py
@@ -2,7 +2,7 @@
 import logging
 import pickle
 from datetime import timedelta
-from time import sleep, time
+from time import sleep
 
 import pytest
 
@@ -514,7 +514,7 @@ def test_threadpoolworkers_pick_correct_ioloop(cleanup):
     # * `lease-timeout` should be smaller than the sleep time. This is what the
     #   test builds on. assuming the leases cannot be refreshed, e.g. wrong
     #   event loop picked / PeriodicCallback never scheduled, the semaphore
-    #   would become oversubscribed and the len(protected_ressources) becomes
+    #   would become oversubscribed and the len(protected_resources) becomes
     #   non zero. This should also trigger a log message about "unknown leases"
     #   and fails the test.
     # * `lease-validation-interval` interval should be the smallest quantity.
@@ -530,17 +530,17 @@ def test_threadpoolworkers_pick_correct_ioloop(cleanup):
     ):
         with Client(processes=False, threads_per_worker=4) as client:
             sem = Semaphore(max_leases=1, name="database")
-            protected_ressource = []
+            protected_resource = []
 
             def access_limited(val, sem):
                 import time
 
                 with sem:
-                    assert len(protected_ressource) == 0
-                    protected_ressource.append(val)
+                    assert len(protected_resource) == 0
+                    protected_resource.append(val)
                     # Interact with the DB
                     time.sleep(0.2)
-                    protected_ressource.remove(val)
+                    protected_resource.remove(val)
 
             client.gather(client.map(access_limited, range(10), sem=sem))
 

From 6b69342b82e892f797cac54092eb7a7af45645e4 Mon Sep 17 00:00:00 2001
From: Marcos Moyano <marcos@anue.biz>
Date: Mon, 12 Apr 2021 23:42:39 -0300
Subject: [PATCH 1226/1550] Remove WSSConnector TLS presence check (#4695)

Let it fail at connection time if TLS is not properly configured

Co-authored-by: Matthew Rocklin <mrocklin@gmail.com>
Co-authored-by: James Bourbeau <jrbourbeau@gmail.com>
---
 distributed/comm/tests/test_ws.py | 3 ++-
 distributed/comm/ws.py            | 7 +++++--
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/distributed/comm/tests/test_ws.py b/distributed/comm/tests/test_ws.py
index c12de0d2212..a7e305f718a 100644
--- a/distributed/comm/tests/test_ws.py
+++ b/distributed/comm/tests/test_ws.py
@@ -9,6 +9,7 @@
 
 from distributed import Client, Scheduler, Worker
 from distributed.comm import connect, listen, ws
+from distributed.comm.core import FatalCommClosedError
 from distributed.comm.registry import backends, get_backend
 from distributed.security import Security
 from distributed.utils_test import (  # noqa: F401
@@ -71,7 +72,7 @@ async def test_expect_ssl_context(cleanup):
     server_ctx = get_server_ssl_context()
 
     async with listen("wss://", lambda comm: comm, ssl_context=server_ctx) as listener:
-        with pytest.raises(TypeError):
+        with pytest.raises(FatalCommClosedError, match="TLS expects a `ssl_context` *"):
             comm = await connect(listener.contact_address)
 
 
diff --git a/distributed/comm/ws.py b/distributed/comm/ws.py
index 4789eb1a56a..05e8c88b79d 100644
--- a/distributed/comm/ws.py
+++ b/distributed/comm/ws.py
@@ -376,7 +376,10 @@ async def connect(self, address, deserialize=True, **connection_args):
         except StreamClosedError as e:
             convert_stream_closed_error(self, e)
         except SSLError as err:
-            raise FatalCommClosedError() from err
+            raise FatalCommClosedError(
+                "TLS expects a `ssl_context` argument of type "
+                "ssl.SSLContext (perhaps check your TLS configuration?)"
+            ) from err
         return self.comm_class(sock, deserialize=deserialize)
 
     def _get_connect_args(self, **connection_args):
@@ -388,7 +391,7 @@ class WSSConnector(WSConnector):
     comm_class = WSS
 
     def _get_connect_args(self, **connection_args):
-        ctx = _expect_tls_context(connection_args)
+        ctx = connection_args.get("ssl_context")
         return {"ssl_options": ctx, **connection_args.get("extra_conn_args", {})}
 
 

From 5c9a7ed54d55b936fff16465cf8709a5da5dc4cd Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 13 Apr 2021 10:27:51 -0500
Subject: [PATCH 1227/1550] Add timing information to TaskGroup (#4671)

I would like for us to start thinking about visualizing TaskGroups as an alternative to the task stream.

This adds timeline information to those objects as a precursor to that work.

Example
-------

```python
from dask.distributed import Client, wait
client = Client()
client

import dask.array as da
x = da.random.random((10000, 10000))
y = x + x.T - x.mean(axis=0)
y = y.persist()
wait(y)

groups = client.cluster.scheduler.task_groups
tg = list(groups.values())[0]
tg.__dict__
```

```python
{'_name': 'sub-2005fd772a2da340e72f1de1df1ac06d',
 '_prefix': <sub: memory: 16>,
 '_states': {'waiting': 0,
  'erred': 0,
  'released': 0,
  'memory': 16,
  'no-worker': 0,
  'processing': 0,
  'forgotten': 0},
 '_dependencies': {<mean_agg-aggregate-3f80179c8a4d92eabf1c0046dbbabd74: released: 4>,
  <random_sample-87d33bde1d8da2f2c8edc106b69514a0: released: 16>},
 '_nbytes_total': 800000000,
 '_nbytes_in_memory': 800000000,
 '_duration': 4.233909368515015,
 '_types': {'numpy.ndarray'},
 '_start': 1617389525.5363212,
 '_stop': 1617389526.3259144,
 '_all_durations': defaultdict(float,
             {'compute': 4.233909368515015, 'transfer': 1.8818869590759277})}
```

This should be enough to give us information for a rich visual

Co-authored-by: James Bourbeau <jrbourbeau@gmail.com>
---
 distributed/scheduler.py            | 22 ++++++++++++++++++++++
 distributed/tests/test_scheduler.py | 23 ++++++++++++++++++++++-
 2 files changed, 44 insertions(+), 1 deletion(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 6540dd46dfb..dcb2013c194 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -796,6 +796,9 @@ class TaskGroup:
     _nbytes_in_memory: Py_ssize_t
     _duration: double
     _types: set
+    _start: double
+    _stop: double
+    _all_durations: object
 
     def __init__(self, name: str):
         self._name = name
@@ -807,6 +810,9 @@ def __init__(self, name: str):
         self._nbytes_in_memory = 0
         self._duration = 0
         self._types = set()
+        self._start = 0.0
+        self._stop = 0.0
+        self._all_durations = defaultdict(float)
 
     @property
     def name(self):
@@ -840,6 +846,18 @@ def duration(self):
     def types(self):
         return self._types
 
+    @property
+    def all_durations(self):
+        return self._all_durations
+
+    @property
+    def start(self):
+        return self._start
+
+    @property
+    def stop(self):
+        return self._stop
+
     @ccall
     def add(self, o):
         ts: TaskState = o
@@ -2305,6 +2323,7 @@ def transition_processing_memory(
                     # record timings of all actions -- a cheaper way of
                     # getting timing info compared with get_task_stream()
                     ts._prefix._all_durations[action] += stop - start
+                    ts._group._all_durations[action] += stop - start
 
             #############################
             # Update Timing Information #
@@ -2321,6 +2340,9 @@ def transition_processing_memory(
 
                 ts._prefix._duration_average = avg_duration
                 ts._group._duration += new_duration
+                ts._group._start = ts._group._start or compute_start
+                if ts._group._stop < compute_stop:
+                    ts._group._stop = compute_stop
 
                 s: set = self._unknown_durations.pop(ts._prefix._name, None)
                 tts: TaskState
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 2669dfc53bd..f2fc346d219 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -1830,12 +1830,30 @@ async def test_no_danglng_asyncio_tasks(cleanup):
     assert tasks == start
 
 
-@gen_cluster(client=True)
+class NoSchedulerDelayWorker(Worker):
+    """Custom worker class which does not update `scheduler_delay`.
+
+    This worker class is useful for some tests which make time
+    comparisons using times reported from workers.
+    """
+
+    @property
+    def scheduler_delay(self):
+        return 0
+
+    @scheduler_delay.setter
+    def scheduler_delay(self, value):
+        pass
+
+
+@gen_cluster(client=True, Worker=NoSchedulerDelayWorker)
 async def test_task_groups(c, s, a, b):
+    start = time()
     da = pytest.importorskip("dask.array")
     x = da.arange(100, chunks=(20,))
     y = (x + 1).persist(optimize_graph=False)
     y = await y
+    stop = time()
 
     tg = s.task_groups[x.name]
     tp = s.task_prefixes["arange"]
@@ -1870,6 +1888,9 @@ async def test_task_groups(c, s, a, b):
     assert tg.states["forgotten"] == 5
     # Ensure TaskGroup is removed once all tasks are in forgotten state
     assert tg.name not in s.task_groups
+    assert tg.start > start
+    assert tg.stop < stop
+    assert "compute" in tg.all_durations
     assert sys.getrefcount(tg) == 2
 
 

From de6db936e0cbfcd7d71c4258fd72b11994ec37ed Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Tue, 13 Apr 2021 19:47:36 +0100
Subject: [PATCH 1228/1550] Clean up test_dashboard (#4700)

---
 distributed/cli/tests/test_dask_scheduler.py | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/distributed/cli/tests/test_dask_scheduler.py b/distributed/cli/tests/test_dask_scheduler.py
index 2e938db558f..36399b49ca8 100644
--- a/distributed/cli/tests/test_dask_scheduler.py
+++ b/distributed/cli/tests/test_dask_scheduler.py
@@ -73,7 +73,7 @@ def test_dashboard(loop):
         else:
             raise Exception("dashboard not found")
 
-        with Client("127.0.0.1:%d" % Scheduler.default_port, loop=loop) as c:
+        with Client(f"127.0.0.1:{Scheduler.default_port}", loop=loop):
             pass
 
         names = ["localhost", "127.0.0.1", get_ip()]
@@ -85,17 +85,20 @@ def test_dashboard(loop):
             try:
                 # All addresses should respond
                 for name in names:
-                    uri = "http://%s:%d/status/" % (name, dashboard_port)
+                    uri = f"http://{name}:{dashboard_port}/status/"
                     response = requests.get(uri)
-                    assert response.ok
+                    response.raise_for_status()
                 break
-            except Exception as f:
-                print("got error on %r: %s" % (uri, f))
+            except Exception as e:
+                print(f"Got error on {uri!r}: {e.__class__.__name__}: {e}")
+                elapsed = time() - start
+                if elapsed > 10:
+                    print(f"Timed out after {elapsed:.2f} seconds")
+                    raise
                 sleep(0.1)
-                assert time() < start + 10
 
     with pytest.raises(Exception):
-        requests.get("http://127.0.0.1:%d/status/" % dashboard_port)
+        requests.get(f"http://127.0.0.1:{dashboard_port}/status/")
 
 
 def test_dashboard_non_standard_ports(loop):

From 61cc6ed2bbea768bc8015d0885c8bcbc324286aa Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Wed, 14 Apr 2021 19:01:56 +0200
Subject: [PATCH 1229/1550] Faster tests_semaphore::test_worker_dies (#4703)

---
 distributed/tests/test_semaphore.py | 10 +++-
 distributed/utils_test.py           | 90 +++++++++++++++--------------
 2 files changed, 54 insertions(+), 46 deletions(-)

diff --git a/distributed/tests/test_semaphore.py b/distributed/tests/test_semaphore.py
index 8c283f3f774..2a478d4f749 100644
--- a/distributed/tests/test_semaphore.py
+++ b/distributed/tests/test_semaphore.py
@@ -140,7 +140,11 @@ async def test_async_ctx(s, a, b):
 
 @pytest.mark.slow
 def test_worker_dies():
-    with cluster() as (scheduler, workers):
+    with cluster(
+        config={
+            "distributed.scheduler.locks.lease-timeout": "0.1s",
+        }
+    ) as (scheduler, workers):
         with Client(scheduler["address"]) as client:
             sem = Semaphore(name="x", max_leases=1)
 
@@ -156,11 +160,11 @@ def f(x, sem, kill_address):
                     return x
 
             futures = client.map(
-                f, range(100), sem=sem, kill_address=workers[0]["address"]
+                f, range(10), sem=sem, kill_address=workers[0]["address"]
             )
             results = client.gather(futures)
 
-            assert sorted(results) == list(range(100))
+            assert sorted(results) == list(range(10))
 
 
 @gen_cluster(client=True)
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index 59d4a8d24cd..2e83199c094 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -435,39 +435,21 @@ async def background_read():
         return msg
 
 
-def run_scheduler(q, nputs, port=0, **kwargs):
-    from distributed import Scheduler
+def run_scheduler(q, nputs, config, port=0, **kwargs):
+    with dask.config.set(config):
+        from distributed import Scheduler
 
-    # On Python 2.7 and Unix, fork() is used to spawn child processes,
-    # so avoid inheriting the parent's IO loop.
-    with pristine_loop() as loop:
-
-        async def _():
-            scheduler = await Scheduler(
-                validate=True, host="127.0.0.1", port=port, **kwargs
-            )
-            for i in range(nputs):
-                q.put(scheduler.address)
-            await scheduler.finished()
-
-        try:
-            loop.run_sync(_)
-        finally:
-            loop.close(all_fds=True)
-
-
-def run_worker(q, scheduler_q, **kwargs):
-    from distributed import Worker
-
-    reset_logger_locks()
-    with log_errors():
+        # On Python 2.7 and Unix, fork() is used to spawn child processes,
+        # so avoid inheriting the parent's IO loop.
         with pristine_loop() as loop:
-            scheduler_addr = scheduler_q.get()
 
             async def _():
-                worker = await Worker(scheduler_addr, validate=True, **kwargs)
-                q.put(worker.address)
-                await worker.finished()
+                scheduler = await Scheduler(
+                    validate=True, host="127.0.0.1", port=port, **kwargs
+                )
+                for i in range(nputs):
+                    q.put(scheduler.address)
+                await scheduler.finished()
 
             try:
                 loop.run_sync(_)
@@ -475,20 +457,41 @@ async def _():
                 loop.close(all_fds=True)
 
 
-def run_nanny(q, scheduler_q, **kwargs):
-    with log_errors():
-        with pristine_loop() as loop:
-            scheduler_addr = scheduler_q.get()
+def run_worker(q, scheduler_q, config, **kwargs):
+    with dask.config.set(config):
+        from distributed import Worker
 
-            async def _():
-                worker = await Nanny(scheduler_addr, validate=True, **kwargs)
-                q.put(worker.address)
-                await worker.finished()
+        reset_logger_locks()
+        with log_errors():
+            with pristine_loop() as loop:
+                scheduler_addr = scheduler_q.get()
+
+                async def _():
+                    worker = await Worker(scheduler_addr, validate=True, **kwargs)
+                    q.put(worker.address)
+                    await worker.finished()
+
+                try:
+                    loop.run_sync(_)
+                finally:
+                    loop.close(all_fds=True)
 
-            try:
-                loop.run_sync(_)
-            finally:
-                loop.close(all_fds=True)
+
+def run_nanny(q, scheduler_q, config, **kwargs):
+    with dask.config.set(config):
+        with log_errors():
+            with pristine_loop() as loop:
+                scheduler_addr = scheduler_q.get()
+
+                async def _():
+                    worker = await Nanny(scheduler_addr, validate=True, **kwargs)
+                    q.put(worker.address)
+                    await worker.finished()
+
+                try:
+                    loop.run_sync(_)
+                finally:
+                    loop.close(all_fds=True)
 
 
 @contextmanager
@@ -595,6 +598,7 @@ def cluster(
     active_rpc_timeout=10,
     disconnect_timeout=20,
     scheduler_kwargs={},
+    config={},
 ):
     ws = weakref.WeakSet()
     enable_proctitle_on_children()
@@ -612,7 +616,7 @@ def cluster(
         scheduler = mp_context.Process(
             name="Dask cluster test: Scheduler",
             target=run_scheduler,
-            args=(scheduler_q, nworkers + 1),
+            args=(scheduler_q, nworkers + 1, config),
             kwargs=scheduler_kwargs,
         )
         ws.add(scheduler)
@@ -635,7 +639,7 @@ def cluster(
             proc = mp_context.Process(
                 name="Dask cluster test: Worker",
                 target=_run_worker,
-                args=(q, scheduler_q),
+                args=(q, scheduler_q, config),
                 kwargs=kwargs,
             )
             ws.add(proc)

From 6f4950659b7097a9a98803d7f14545626faf89b3 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Wed, 14 Apr 2021 17:06:04 -0500
Subject: [PATCH 1230/1550] Remove hostname check in test_dashboard (#4706)

---
 distributed/cli/tests/test_dask_scheduler.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/distributed/cli/tests/test_dask_scheduler.py b/distributed/cli/tests/test_dask_scheduler.py
index 36399b49ca8..a407a2b8fe1 100644
--- a/distributed/cli/tests/test_dask_scheduler.py
+++ b/distributed/cli/tests/test_dask_scheduler.py
@@ -4,7 +4,6 @@
 
 import os
 import shutil
-import socket
 import sys
 import tempfile
 from time import sleep
@@ -77,9 +76,6 @@ def test_dashboard(loop):
             pass
 
         names = ["localhost", "127.0.0.1", get_ip()]
-        if "linux" in sys.platform:
-            names.append(socket.gethostname())
-
         start = time()
         while True:
             try:

From dfd8fb6858d25d228eeae44f50f145c92a8737ce Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Wed, 14 Apr 2021 17:19:32 -0700
Subject: [PATCH 1231/1550] Grab `func` for `finish` case only if used (#4702)

The transition `func` that was being grabbed depended on whether `key`
was found in the recommendations from the last transition. If `key` was
found, one function based on `v` was called. If not, `finish` was used
in place of `v`. To make this a bit more explicit, simple, and
efficient, assign `finish` to `v` if `key` is not found. This way `v`
can be used in all cases with one call to retrieve the `func`.
---
 distributed/scheduler.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index dcb2013c194..077c2359092 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1876,16 +1876,15 @@ def _transition(self, key, finish: str, *args, **kwargs):
                 a: tuple = func(key, *args, **kwargs)
                 recommendations, client_msgs, worker_msgs = a
             elif "released" not in start_finish:
-                func = self._transitions_table["released", finish]
                 assert not args and not kwargs
                 a_recs: dict
                 a_cmsgs: dict
                 a_wmsgs: dict
                 a: tuple = self._transition(key, "released")
                 a_recs, a_cmsgs, a_wmsgs = a
-                v = a_recs.get(key)
-                if v is not None:
-                    func = self._transitions_table["released", v]
+
+                v = a_recs.get(key, finish)
+                func = self._transitions_table["released", v]
                 b_recs: dict
                 b_cmsgs: dict
                 b_wmsgs: dict

From 1214efcc723b61b16c6aeaadc89a53102a8825f2 Mon Sep 17 00:00:00 2001
From: Doug Davis <ddavis@anaconda.com>
Date: Wed, 14 Apr 2021 22:36:01 -0400
Subject: [PATCH 1232/1550] Avoid repeatedly using the same worker on first
 task with quiet cluster (#4638)

---
 distributed/scheduler.py                  | 17 +++++++++++++++--
 distributed/tests/test_client_executor.py | 12 +++++-------
 distributed/tests/test_scheduler.py       |  8 ++++++++
 3 files changed, 28 insertions(+), 9 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 077c2359092..7778313db7d 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -2140,11 +2140,24 @@ def decide_worker(self, ts: TaskState) -> WorkerState:
         else:
             worker_pool = self._idle or self._workers
             worker_pool_dv = cast(dict, worker_pool)
+            wp_vals = worker_pool.values()
             n_workers: Py_ssize_t = len(worker_pool_dv)
             if n_workers < 20:  # smart but linear in small case
-                ws = min(worker_pool.values(), key=operator.attrgetter("occupancy"))
+                ws = min(wp_vals, key=operator.attrgetter("occupancy"))
+                if ws._occupancy == 0:
+                    # special case to use round-robin; linear search
+                    # for next worker with zero occupancy (or just
+                    # land back where we started).
+                    wp_i: WorkerState
+                    start: Py_ssize_t = self._n_tasks % n_workers
+                    i: Py_ssize_t
+                    for i in range(n_workers):
+                        wp_i = wp_vals[(i + start) % n_workers]
+                        if wp_i._occupancy == 0:
+                            ws = wp_i
+                            break
             else:  # dumb but fast in large case
-                ws = worker_pool.values()[self._n_tasks % n_workers]
+                ws = wp_vals[self._n_tasks % n_workers]
 
         if self._validate:
             assert ws is None or isinstance(ws, WorkerState), (
diff --git a/distributed/tests/test_client_executor.py b/distributed/tests/test_client_executor.py
index 555bcb86fbe..3942a2ca010 100644
--- a/distributed/tests/test_client_executor.py
+++ b/distributed/tests/test_client_executor.py
@@ -210,26 +210,24 @@ def test_unsupported_arguments(client, s, a, b):
 def test_retries(client):
     args = [ZeroDivisionError("one"), ZeroDivisionError("two"), 42]
 
-    with client.get_executor(retries=3, pure=False) as e:
+    with client.get_executor(retries=5, pure=False) as e:
         future = e.submit(varying(args))
         assert future.result() == 42
 
-    with client.get_executor(retries=2) as e:
+    with client.get_executor(retries=4) as e:
         future = e.submit(varying(args))
         result = future.result()
         assert result == 42
 
-    with client.get_executor(retries=1) as e:
+    with client.get_executor(retries=2) as e:
         future = e.submit(varying(args))
-        with pytest.raises(ZeroDivisionError) as exc_info:
+        with pytest.raises(ZeroDivisionError, match="two"):
             res = future.result()
-        exc_info.match("two")
 
     with client.get_executor(retries=0) as e:
         future = e.submit(varying(args))
-        with pytest.raises(ZeroDivisionError) as exc_info:
+        with pytest.raises(ZeroDivisionError, match="one"):
             res = future.result()
-        exc_info.match("one")
 
 
 def test_shutdown(loop):
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index f2fc346d219..c82a4a04839 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -2234,3 +2234,11 @@ async def test_get_worker_monitor_info(s, a, b):
         assert all(res[w.address]["range_query"][m] is not None for m in ms)
         assert res[w.address]["count"] is not None
         assert res[w.address]["last_time"] is not None
+
+
+@gen_cluster(client=True)
+async def test_quiet_cluster_round_robin(c, s, a, b):
+    await c.submit(inc, 1)
+    await c.submit(inc, 2)
+    await c.submit(inc, 3)
+    assert a.log and b.log

From fdfabf3b63f3dec83633fae144a6113d661b88e1 Mon Sep 17 00:00:00 2001
From: Matteo De Wint <matteo@mailfence.com>
Date: Fri, 16 Apr 2021 01:23:41 +0200
Subject: [PATCH 1233/1550] Fix timeout in client.restart (#4690)

---
 distributed/client.py            | 11 ++++++-----
 distributed/tests/test_client.py |  8 ++++++++
 2 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 14a5e0c8091..695c9dd39f2 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -2978,15 +2978,16 @@ def persist(
     async def _restart(self, timeout=no_default):
         if timeout == no_default:
             timeout = self._timeout * 2
+        if timeout is not None:
+            timeout = parse_timedelta(timeout, "s")
+
         self._send_to_scheduler({"op": "restart", "timeout": timeout})
         self._restart_event = asyncio.Event()
         try:
-            await asyncio.wait_for(
-                self._restart_event.wait(), self.loop.time() + timeout
-            )
+            await asyncio.wait_for(self._restart_event.wait(), timeout)
         except TimeoutError:
-            logger.error("Restart timed out after %f seconds", timeout)
-            pass
+            logger.error("Restart timed out after %.2f seconds", timeout)
+
         self.generation += 1
         with self._refcount_lock:
             self.refcount.clear()
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 1bdfb15ab6e..fb70994b567 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -3508,6 +3508,14 @@ async def test_Client_clears_references_after_restart(c, s, a, b):
     assert key not in c.refcount
 
 
+@gen_cluster(Worker=Nanny, client=True)
+async def test_restart_timeout_is_logged(c, s, a, b):
+    with captured_logger(logging.getLogger("distributed.client")) as logger:
+        await c.restart(timeout="0.5s")
+    text = logger.getvalue()
+    assert "Restart timed out after 0.50 seconds" in text
+
+
 def test_get_stops_work_after_error(c):
     with pytest.raises(RuntimeError):
         c.get({"x": (throws, 1), "y": (sleep, 1.5)}, ["x", "y"])

From deeab497b0ef96efd4a956b77aad45b4812299c9 Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Fri, 16 Apr 2021 17:48:50 +0200
Subject: [PATCH 1234/1550] When closing Server, close all listeners (#4704)

---
 distributed/core.py            |  4 +--
 distributed/tests/test_core.py | 49 ++++++++++++++++++++++++++++++++++
 2 files changed, 51 insertions(+), 2 deletions(-)

diff --git a/distributed/core.py b/distributed/core.py
index 0987564c4dc..9aaf535191c 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -599,7 +599,7 @@ def close(self):
         for pc in self.periodic_callbacks.values():
             pc.stop()
         for listener in self.listeners:
-            future = self.listener.stop()
+            future = listener.stop()
             if inspect.isawaitable(future):
                 yield future
         for i in range(20):  # let comms close naturally for a second
@@ -611,7 +611,7 @@ def close(self):
         for cb in self._ongoing_coroutines:
             cb.cancel()
         for i in range(10):
-            if all(cb.cancelled() for c in self._ongoing_coroutines):
+            if all(c.cancelled() for c in self._ongoing_coroutines):
                 break
             else:
                 yield asyncio.sleep(0.01)
diff --git a/distributed/tests/test_core.py b/distributed/tests/test_core.py
index 9502d9d4d15..398b933c02c 100644
--- a/distributed/tests/test_core.py
+++ b/distributed/tests/test_core.py
@@ -32,6 +32,7 @@
     assert_can_connect_locally_4,
     assert_can_connect_locally_6,
     assert_cannot_connect,
+    async_wait_for,
     captured_logger,
     gen_cluster,
     has_ipv6,
@@ -831,3 +832,51 @@ async def test_connection_pool_detects_remote_close():
     # while creating conn2:
     p._validate()
     await p.close()
+
+
+@pytest.mark.asyncio
+async def test_close_properly():
+    """
+    If the server is closed we should cancel all still ongoing coros and close
+    all listeners.
+    GH4704
+    """
+
+    async def sleep(comm=None):
+        # We want to ensure this is actually canceled therefore don't give it a
+        # chance to actually complete
+        await asyncio.sleep(2000000)
+
+    server = await Server({"sleep": sleep})
+    assert server.status == Status.running
+    ports = [8881, 8882, 8883]
+
+    # Previously we close *one* listener, therefore ensure we always use more
+    # than one for this test
+    assert len(ports) > 1
+    for port in ports:
+        await server.listen(port)
+    # We use TCP here for simplicity. If they are closed we should expect other
+    # backends to close properly as well
+    ip = get_ip()
+    rpc_addr = f"tcp://{ip}:{ports[-1]}"
+    async with rpc(rpc_addr) as remote:
+
+        comm = await remote.live_comm()
+        await comm.write({"op": "sleep"})
+
+        await async_wait_for(lambda: not server._ongoing_coroutines, 10)
+
+        listeners = server.listeners
+        assert len(listeners) == len(ports)
+
+        for port in ports:
+            await assert_can_connect(f"tcp://{ip}:{port}")
+
+        await server.close()
+
+        for port in ports:
+            await assert_cannot_connect(f"tcp://{ip}:{port}")
+
+        # weakref set/dict should be cleaned up
+        assert not len(server._ongoing_coroutines)

From 74a9aff7e7429c4b2f12204cfa97aba1fca5aa29 Mon Sep 17 00:00:00 2001
From: "Richard (Rick) Zamora" <rzamora217@gmail.com>
Date: Fri, 16 Apr 2021 21:28:00 -0500
Subject: [PATCH 1235/1550] Add iterate_collection argument to serialize
 (#4641)

---
 distributed/protocol/core.py                 |  2 --
 distributed/protocol/serialize.py            | 23 +++++++++++++++++---
 distributed/protocol/tests/test_serialize.py | 17 +++++++++++++++
 distributed/worker.py                        |  4 ++--
 4 files changed, 39 insertions(+), 7 deletions(-)

diff --git a/distributed/protocol/core.py b/distributed/protocol/core.py
index 0d87595bf75..1e3da14cf8b 100644
--- a/distributed/protocol/core.py
+++ b/distributed/protocol/core.py
@@ -48,8 +48,6 @@ def _inplace_compress_frames(header, frames):
         def _encode_default(obj):
             typ = type(obj)
             if typ is Serialize or typ is Serialized:
-                if typ is Serialize:
-                    obj = obj.data
                 offset = len(frames)
                 if typ is Serialized:
                     sub_header, sub_frames = obj.header, obj.frames
diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index 50d87953111..63ced42a473 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -196,7 +196,9 @@ def check_dask_serializable(x):
     return False
 
 
-def serialize(x, serializers=None, on_error="message", context=None):
+def serialize(
+    x, serializers=None, on_error="message", context=None, iterate_collection=None
+):
     r"""
     Convert object to a header and list of bytestrings
 
@@ -210,6 +212,12 @@ def serialize(x, serializers=None, on_error="message", context=None):
     per-class serialization methods. ``None`` gives the default list
     ``['dask', 'pickle']``.
 
+    Notes on the ``iterate_collection`` argument (only relevant when
+    ``x`` is a collection):
+    - ``iterate_collection=True``: Serialize collection elements separately.
+    - ``iterate_collection=False``: Serialize collection elements together.
+    - ``iterate_collection=None`` (default): Infer the best setting.
+
     Examples
     --------
     >>> serialize(1)
@@ -235,11 +243,20 @@ def serialize(x, serializers=None, on_error="message", context=None):
     if serializers is None:
         serializers = ("dask", "pickle")  # TODO: get from configuration
 
+    # Handle obects that are marked as `Serialize`, or that are
+    # already `Serialized` objects (don't want to serialize them twice)
     if isinstance(x, Serialized):
         return x.header, x.frames
+    if isinstance(x, Serialize):
+        return serialize(
+            x.data,
+            serializers=serializers,
+            on_error=on_error,
+            context=context,
+            iterate_collection=True,
+        )
 
-    if type(x) in (list, set, tuple, dict):
-        iterate_collection = False
+    if iterate_collection is None and type(x) in (list, set, tuple, dict):
         if type(x) is list and "msgpack" in serializers:
             # Note: "msgpack" will always convert lists to tuples
             #       (see GitHub #3716), so we should iterate
diff --git a/distributed/protocol/tests/test_serialize.py b/distributed/protocol/tests/test_serialize.py
index cfb767bc0cb..a72724a1136 100644
--- a/distributed/protocol/tests/test_serialize.py
+++ b/distributed/protocol/tests/test_serialize.py
@@ -145,6 +145,23 @@ def test_nested_deserialize():
     assert x == x_orig  # x wasn't mutated
 
 
+def test_serialize_iterate_collection():
+    # Use iterate_collection to ensure elements of
+    # a collection will be serialized seperately
+
+    arr = "special-data"
+    sarr = Serialized(*serialize(arr))
+    sdarr = to_serialize(arr)
+
+    task1 = (0, sarr, "('fake-key', 3)", None)
+    task2 = (0, sdarr, "('fake-key', 3)", None)
+    expect = (0, arr, "('fake-key', 3)", None)
+
+    # Check serialize/deserialize directly
+    assert deserialize(*serialize(task1, iterate_collection=True)) == expect
+    assert deserialize(*serialize(task2, iterate_collection=True)) == expect
+
+
 from dask import delayed
 
 from distributed.utils_test import gen_cluster
diff --git a/distributed/worker.py b/distributed/worker.py
index 9e79ef5ab6e..071efa8bcd9 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -3485,9 +3485,9 @@ def _deserialize(function=None, args=None, kwargs=None, task=no_value):
     """ Deserialize task inputs and regularize to func, args, kwargs """
     if function is not None:
         function = loads_function(function)
-    if args:
+    if args and isinstance(args, bytes):
         args = pickle.loads(args)
-    if kwargs:
+    if kwargs and isinstance(kwargs, bytes):
         kwargs = pickle.loads(kwargs)
 
     if task is not no_value:

From 8b827bae71aeb93cc06fe3c712909210aad55bea Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Mon, 19 Apr 2021 16:43:24 +0100
Subject: [PATCH 1236/1550] ipython causes test_profile_nested_sizeof crash on
 windows (#4713)

---
 .github/workflows/tests.yaml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
index 5b8eece7739..3b4978c0b55 100644
--- a/.github/workflows/tests.yaml
+++ b/.github/workflows/tests.yaml
@@ -41,6 +41,13 @@ jobs:
         if: ${{ matrix.os == 'ubuntu-latest' && matrix.python-version < '3.8' }}
         run: mamba install -c conda-forge -c defaults -c numba libunwind stacktrace
 
+      - name: Hack around https://github.com/ipython/ipython/issues/12197
+        # This upstream issue causes an interpreter crash when running
+        # distributed/protocol/tests/test_serialize.py::test_profile_nested_sizeof
+        shell: bash -l {0}
+        if: ${{ matrix.os == 'windows-latest' && matrix.python-version == '3.9' }}
+        run: mamba uninstall ipython
+
       - name: Cythonize
         shell: bash -l {0}
         if: ${{ matrix.python-version == '3.7' }}

From 4e59153189ad6989ebd68c6f810aefa0397cd3ce Mon Sep 17 00:00:00 2001
From: George Sakkis <george.sakkis@gmail.com>
Date: Tue, 20 Apr 2021 01:22:33 +0300
Subject: [PATCH 1237/1550] Fix DeprecationWarning on python 3.9 (#4717)

---
 distributed/deploy/spec.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index 3ba421e93f2..5190d7c0fa7 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -338,7 +338,11 @@ async def _correct_state_internal(self):
             if to_close:
                 if self.scheduler.status == Status.running:
                     await self.scheduler_comm.retire_workers(workers=list(to_close))
-                tasks = [self.workers[w].close() for w in to_close if w in self.workers]
+                tasks = [
+                    asyncio.create_task(self.workers[w].close())
+                    for w in to_close
+                    if w in self.workers
+                ]
                 await asyncio.wait(tasks)
                 for task in tasks:  # for tornado gen.coroutine support
                     with suppress(RuntimeError):

From d5fc324bdef22f19d77c3d36e63a3778ceaac0b0 Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Tue, 20 Apr 2021 12:35:14 +0100
Subject: [PATCH 1238/1550] Split RAM measure into dask keys/other old/other
 new (#4651)

* code cleanup

* code cleanup

* Early exit pattern

* refactor

* cleanup

* fix regression

* annotations

* revert

* backend prototype

* slightly faster both in CPython and Cython

* slightly faster both in CPython and Cython

* polish

* polish

* early exit

* polish

* polish

* backend review

* nonfunctional GUI prototype

* GUI prototype (unpolished)

* tooltip

* refactor

* GUI

* GUI

* GUI

* refactor

* polish

* simpler tooltip

* Reduce spilled size on delitem

* tweak cluster-wide nbytes gauge

* workers tab

* Self-review

* bokeh unit tests

* test SpillBuffer

* Code review

* cython optimizations

* test MemoryState

* test backend

* Remove unnecessary casts uint->sint

* Self-review

* Test edge cases

* fix test failure

* redesign test

* relax maximums

* fix test

* lint

* fix test

* fix test

* fix bar on small screens

* height in em

* larger

* fix flaky test
---
 distributed/dashboard/components/scheduler.py | 802 ++++++++++--------
 distributed/dashboard/scheduler.py            |  59 +-
 .../dashboard/tests/test_scheduler_bokeh.py   |  41 +-
 distributed/distributed-schema.yaml           |  72 +-
 distributed/distributed.yaml                  |  10 +-
 distributed/http/static/css/status.css        |  30 +-
 distributed/http/templates/status.html        |  10 +-
 distributed/scheduler.py                      | 164 +++-
 distributed/spill.py                          |  79 ++
 distributed/tests/test_scheduler.py           | 187 +++-
 distributed/tests/test_spill.py               |  84 ++
 distributed/tests/test_worker.py              |  10 +-
 distributed/worker.py                         |  36 +-
 docs/source/http_services.rst                 |   4 +
 14 files changed, 1103 insertions(+), 485 deletions(-)
 create mode 100644 distributed/spill.py
 create mode 100644 distributed/tests/test_spill.py

diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index eae544d3fac..fd58c000d05 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -36,7 +36,7 @@
 from bokeh.plotting import figure
 from bokeh.themes import Theme
 from bokeh.transform import cumsum, factor_cmap, linear_cmap
-from tlz import pipe
+from tlz import curry, pipe
 from tlz.curried import concat, groupby, map
 from tornado import escape
 
@@ -88,9 +88,7 @@
 
 BOKEH_THEME = Theme(os.path.join(os.path.dirname(__file__), "..", "theme.yaml"))
 TICKS_1024 = {"base": 1024, "mantissas": [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]}
-
-nan = float("nan")
-inf = float("inf")
+XLABEL_ORIENTATION = -math.pi / 12  # slanted downwards 15 degrees
 
 
 class Occupancy(DashboardComponent):
@@ -111,32 +109,30 @@ def __init__(self, scheduler, **kwargs):
                 }
             )
 
-            fig = figure(
+            self.root = figure(
                 title="Occupancy",
                 tools="",
                 id="bk-occupancy-plot",
                 x_axis_type="datetime",
                 **kwargs,
             )
-            rect = fig.rect(
+            rect = self.root.rect(
                 source=self.source, x="x", width="ms", y="y", height=1, color="color"
             )
             rect.nonselection_glyph = None
 
-            fig.xaxis.minor_tick_line_alpha = 0
-            fig.yaxis.visible = False
-            fig.ygrid.visible = False
+            self.root.xaxis.minor_tick_line_alpha = 0
+            self.root.yaxis.visible = False
+            self.root.ygrid.visible = False
             # fig.xaxis[0].formatter = NumeralTickFormatter(format='0.0s')
-            fig.x_range.start = 0
+            self.root.x_range.start = 0
 
             tap = TapTool(callback=OpenURL(url="./info/worker/@escaped_worker.html"))
 
             hover = HoverTool()
             hover.tooltips = "@worker : @occupancy s."
             hover.point_policy = "follow_mouse"
-            fig.add_tools(hover, tap)
-
-            self.root = fig
+            self.root.add_tools(hover, tap)
 
     @without_property_validation
     def update(self):
@@ -193,7 +189,7 @@ def __init__(self, scheduler, **kwargs):
             self.root = figure(
                 title="Tasks Processing (Histogram)",
                 id="bk-nprocessing-histogram-plot",
-                name="processing_hist",
+                name="processing",
                 y_axis_label="frequency",
                 tools="",
                 **kwargs,
@@ -222,8 +218,240 @@ def update(self):
         self.source.data.update({"left": x[:-1], "right": x[1:], "top": counts})
 
 
+def _nbytes_color(current: int, limit: int) -> str:
+    """Dynamic color used by NBytes and NBytesCluster"""
+    if limit and current > limit:
+        return "red"
+    elif limit and current > limit / 2:
+        return "orange"
+    else:
+        return "blue"
+
+
+class NBytesCluster(DashboardComponent):
+    """Total memory usage on the cluster"""
+
+    def __init__(self, scheduler, width=600, **kwargs):
+        with log_errors():
+            self.scheduler = scheduler
+            self.source = ColumnDataSource(
+                {
+                    "width": [0] * 4,
+                    "x": [0] * 4,
+                    "y": [0] * 4,
+                    "proc_memory": [0] * 4,
+                    "color": ["blue", "blue", "blue", "grey"],
+                    "alpha": [1, 0.7, 0.4, 1],
+                    "memtype": [
+                        "managed (in memory)",
+                        "unmanaged",
+                        "unmanaged, recently increased",
+                        "managed (spilled to disk)",
+                    ],
+                }
+            )
+
+            self.root = figure(
+                title="Bytes stored on cluster",
+                tools="",
+                id="bk-nbytes-cluster-worker-plot",
+                width=int(width / 2),
+                name="nbytes_cluster",
+                **kwargs,
+            )
+            rect = self.root.rect(
+                source=self.source,
+                x="x",
+                y="y",
+                width="width",
+                height=1,
+                color="color",
+                alpha="alpha",
+            )
+            rect.nonselection_glyph = None
+
+            self.root.axis[0].ticker = BasicTicker(**TICKS_1024)
+            self.root.xaxis[0].formatter = NumeralTickFormatter(format="0.0 b")
+            self.root.xaxis.major_label_orientation = XLABEL_ORIENTATION
+            self.root.x_range.start = 0
+            self.root.xaxis.minor_tick_line_alpha = 0
+            self.root.yaxis.visible = False
+            self.root.ygrid.visible = False
+
+            self.root.toolbar.logo = None
+            self.root.toolbar_location = None
+            self.root.yaxis.visible = False
+
+            hover = HoverTool()
+            hover.tooltips = "@width{0.00 b} @memtype"
+            hover.point_policy = "follow_mouse"
+            self.root.add_tools(hover)
+
+    @without_property_validation
+    def update(self):
+        with log_errors():
+            limit = sum(
+                getattr(ws, "memory_limit", 0) for ws in self.scheduler.workers.values()
+            )
+            meminfo = self.scheduler.memory
+            color = _nbytes_color(meminfo.process, limit)
+
+            width = [
+                meminfo.managed_in_memory,
+                meminfo.unmanaged_old,
+                meminfo.unmanaged_recent,
+                meminfo.managed_spilled,
+            ]
+            result = {
+                "width": width,
+                "x": [sum(width[:i]) + w / 2 for i, w in enumerate(width)],
+                "color": [color, color, color, "grey"],
+                "proc_memory": [meminfo.process] * 4,
+            }
+            # FIXME https://github.com/dask/distributed/issues/4675
+            #       This causes flickering after adding workers and when enough memory
+            #       is spilled out
+            self.root.x_range.end = max(
+                limit, meminfo.process + meminfo.managed_spilled
+            )
+
+            title = f"Bytes stored: {format_bytes(meminfo.process)}"
+            if meminfo.managed_spilled:
+                title += f" + {format_bytes(meminfo.managed_spilled)} spilled to disk"
+            self.root.title.text = title
+
+            update(self.source, result)
+
+
+class NBytes(DashboardComponent):
+    """Memory usage for single workers"""
+
+    def __init__(self, scheduler, width=600, **kwargs):
+        with log_errors():
+            self.scheduler = scheduler
+            self.source = ColumnDataSource(
+                {
+                    "width": [],
+                    "x": [],
+                    "y": [],
+                    "memtype": [],
+                    "color": [],
+                    "alpha": [],
+                    "proc_memory": [],
+                    "worker": [],
+                    "escaped_worker": [],
+                }
+            )
+
+            self.root = figure(
+                title="Bytes stored per worker",
+                tools="",
+                id="bk-nbytes-worker-plot",
+                width=int(width / 2),
+                name="nbytes_workers",
+                **kwargs,
+            )
+            rect = self.root.rect(
+                source=self.source,
+                x="x",
+                y="y",
+                width="width",
+                height=1,
+                color="color",
+                fill_alpha="alpha",
+                line_width=0,
+            )
+            rect.nonselection_glyph = None
+
+            self.root.axis[0].ticker = BasicTicker(**TICKS_1024)
+            self.root.xaxis[0].formatter = NumeralTickFormatter(format="0.0 b")
+            self.root.xaxis.major_label_orientation = XLABEL_ORIENTATION
+            self.root.x_range.start = 0
+            self.root.xaxis.minor_tick_line_alpha = 0
+            self.root.yaxis.visible = False
+            self.root.ygrid.visible = False
+
+            tap = TapTool(callback=OpenURL(url="./info/worker/@escaped_worker.html"))
+            self.root.add_tools(tap)
+
+            self.root.toolbar.logo = None
+            self.root.toolbar_location = None
+            self.root.yaxis.visible = False
+
+            hover = HoverTool()
+            hover.tooltips = "@worker: @proc_memory{0.00 b} (@width{0.00 b} @memtype)"
+            hover.point_policy = "follow_mouse"
+            self.root.add_tools(hover)
+
+    @without_property_validation
+    def update(self):
+        def quadlist(i) -> list:
+            out = []
+            for ii in i:
+                out += [ii, ii, ii, ii]
+            return out
+
+        with log_errors():
+            workers = self.scheduler.workers.values()
+
+            width = []
+            x = []
+            color = []
+            procmemory = []
+            max_limit = 0
+
+            for ws in workers:
+                meminfo = ws.memory
+                limit = getattr(ws, "memory_limit", 0)
+                max_limit = max(
+                    max_limit, limit, meminfo.process + meminfo.managed_spilled
+                )
+                color_i = _nbytes_color(meminfo.process, limit)
+
+                width += [
+                    meminfo.managed_in_memory,
+                    meminfo.unmanaged_old,
+                    meminfo.unmanaged_recent,
+                    meminfo.managed_spilled,
+                ]
+                x += [sum(width[-4:i]) + width[i] / 2 for i in range(-4, 0)]
+                color += [color_i, color_i, color_i, "grey"]
+                procmemory.append(meminfo.process)
+
+            result = {
+                "width": width,
+                "x": x,
+                "color": color,
+                "alpha": [1, 0.7, 0.4, 1] * len(workers),
+                "memtype": [
+                    "managed (in memory)",
+                    "unmanaged",
+                    "unmanaged, recently increased",
+                    "managed (spilled to disk)",
+                ]
+                * len(workers),
+                "proc_memory": quadlist(procmemory),
+                "worker": quadlist(ws.address for ws in workers),
+                "escaped_worker": quadlist(
+                    escape.url_escape(ws.address) for ws in workers
+                ),
+                "y": quadlist(range(len(workers))),
+            }
+            # Remove rectangles with width=0
+            result = {
+                k: [vi for vi, w in zip(v, width) if w] for k, v in result.items()
+            }
+            # FIXME https://github.com/dask/distributed/issues/4675
+            #       This causes flickering after adding workers and when enough memory
+            #       is spilled to disk
+            self.root.x_range.end = max_limit
+            update(self.source, result)
+
+
 class NBytesHistogram(DashboardComponent):
-    """ How many tasks are on each worker """
+    """Histogram of memory usage, showing how many workers there are in each bucket of
+    usage. Replaces the per-worker graph when there are >= 50 workers.
+    """
 
     def __init__(self, scheduler, **kwargs):
         with log_errors():
@@ -234,8 +462,8 @@ def __init__(self, scheduler, **kwargs):
             )
 
             self.root = figure(
-                title="Bytes Stored (Histogram)",
-                name="nbytes_hist",
+                title="Bytes stored per worker (Histogram)",
+                name="nbytes_workers",
                 id="bk-nbytes-histogram-plot",
                 y_axis_label="frequency",
                 tools="",
@@ -244,7 +472,7 @@ def __init__(self, scheduler, **kwargs):
 
             self.root.xaxis[0].formatter = NumeralTickFormatter(format="0.0 b")
             self.root.xaxis.ticker = AdaptiveTicker(**TICKS_1024)
-            self.root.xaxis.major_label_orientation = -math.pi / 12
+            self.root.xaxis.major_label_orientation = XLABEL_ORIENTATION
 
             self.root.xaxis.minor_tick_line_alpha = 0
             self.root.ygrid.visible = False
@@ -269,9 +497,7 @@ def update(self):
         )
         counts, x = np.histogram(nbytes, bins=40)
         d = {"left": x[:-1], "right": x[1:], "top": counts}
-        self.source.data.update(d)
-
-        self.root.title.text = "Bytes stored (Histogram): " + format_bytes(nbytes.sum())
+        update(self.source, d)
 
 
 class BandwidthTypes(DashboardComponent):
@@ -290,7 +516,7 @@ def __init__(self, scheduler, **kwargs):
                 }
             )
 
-            fig = figure(
+            self.root = figure(
                 title="Bandwidth by Type",
                 tools="",
                 id="bk-bandwidth-type-plot",
@@ -298,8 +524,8 @@ def __init__(self, scheduler, **kwargs):
                 y_range=["a", "b"],
                 **kwargs,
             )
-            fig.xaxis.major_label_orientation = -0.5
-            rect = fig.rect(
+            self.root.xaxis.major_label_orientation = -0.5
+            rect = self.root.rect(
                 source=self.source,
                 x="bandwidth-half",
                 y="type",
@@ -307,37 +533,36 @@ def __init__(self, scheduler, **kwargs):
                 height=1,
                 color="blue",
             )
-            fig.x_range.start = 0
-            fig.xaxis[0].formatter = NumeralTickFormatter(format="0.0 b")
-            fig.xaxis.ticker = AdaptiveTicker(**TICKS_1024)
+            self.root.x_range.start = 0
+            self.root.xaxis[0].formatter = NumeralTickFormatter(format="0.0 b")
+            self.root.xaxis.ticker = AdaptiveTicker(**TICKS_1024)
             rect.nonselection_glyph = None
 
-            fig.xaxis.minor_tick_line_alpha = 0
-            fig.ygrid.visible = False
+            self.root.xaxis.minor_tick_line_alpha = 0
+            self.root.ygrid.visible = False
 
-            fig.toolbar.logo = None
-            fig.toolbar_location = None
+            self.root.toolbar.logo = None
+            self.root.toolbar_location = None
 
             hover = HoverTool()
             hover.tooltips = "@type: @bandwidth_text / s"
             hover.point_policy = "follow_mouse"
-            fig.add_tools(hover)
-
-            self.fig = fig
+            self.root.add_tools(hover)
 
     @without_property_validation
     def update(self):
         with log_errors():
             bw = self.scheduler.bandwidth_types
-            self.fig.y_range.factors = list(sorted(bw))
+            self.root.y_range.factors = list(sorted(bw))
             result = {
                 "bandwidth": list(bw.values()),
                 "bandwidth-half": [b / 2 for b in bw.values()],
                 "type": list(bw.keys()),
                 "bandwidth_text": list(map(format_bytes, bw.values())),
             }
-            self.fig.title.text = "Bandwidth: " + format_bytes(self.scheduler.bandwidth)
-
+            self.root.title.text = "Bandwidth: " + format_bytes(
+                self.scheduler.bandwidth
+            )
             update(self.source, result)
 
 
@@ -365,7 +590,7 @@ def __init__(self, scheduler, **kwargs):
                 high=1,
             )
 
-            fig = figure(
+            self.root = figure(
                 title="Bandwidth by Worker",
                 tools="",
                 id="bk-bandwidth-worker-plot",
@@ -374,8 +599,8 @@ def __init__(self, scheduler, **kwargs):
                 y_range=["a", "b"],
                 **kwargs,
             )
-            fig.xaxis.major_label_orientation = -math.pi / 12
-            rect = fig.rect(
+            self.root.xaxis.major_label_orientation = XLABEL_ORIENTATION
+            self.root.rect(
                 source=self.source,
                 x="source",
                 y="destination",
@@ -393,10 +618,10 @@ def __init__(self, scheduler, **kwargs):
             )
             color_bar.formatter = NumeralTickFormatter(format="0.0 b")
             color_bar.ticker = AdaptiveTicker(**TICKS_1024)
-            fig.add_layout(color_bar, "right")
+            self.root.add_layout(color_bar, "right")
 
-            fig.toolbar.logo = None
-            fig.toolbar_location = None
+            self.root.toolbar.logo = None
+            self.root.toolbar_location = None
 
             hover = HoverTool()
             hover.tooltips = """
@@ -407,9 +632,7 @@ def __init__(self, scheduler, **kwargs):
             </div>
             """
             hover.point_policy = "follow_mouse"
-            fig.add_tools(hover)
-
-            self.fig = fig
+            self.root.add_tools(hover)
 
     @without_property_validation
     def update(self):
@@ -433,8 +656,8 @@ def name(address):
             self.color_map.high = max(value)
 
             factors = list(sorted(set(x + y)))
-            self.fig.x_range.factors = factors
-            self.fig.y_range.factors = factors[::-1]
+            self.root.x_range.factors = factors
+            self.root.y_range.factors = factors[::-1]
 
             result = {
                 "source": x,
@@ -442,8 +665,9 @@ def name(address):
                 "bandwidth": value,
                 "bandwidth_text": list(map(format_bytes, value)),
             }
-            self.fig.title.text = "Bandwidth: " + format_bytes(self.scheduler.bandwidth)
-
+            self.root.title.text = "Bandwidth: " + format_bytes(
+                self.scheduler.bandwidth
+            )
             update(self.source, result)
 
 
@@ -494,7 +718,7 @@ def __init__(self, scheduler, **kwargs):
             fig.yaxis.axis_label = "Time (s)"
             fig.yaxis[0].formatter = NumeralTickFormatter(format="0")
             fig.yaxis.ticker = AdaptiveTicker(**TICKS_1024)
-            fig.xaxis.major_label_orientation = -math.pi / 12
+            fig.xaxis.major_label_orientation = XLABEL_ORIENTATION
             rect.nonselection_glyph = None
 
             fig.xaxis.minor_tick_line_alpha = 0
@@ -524,14 +748,6 @@ def __init__(self, scheduler, **kwargs):
             self.fig = fig
             tab1 = Panel(child=fig, title="Bar Chart")
 
-            compute_wedge_data = {
-                "times": [0.2, 0.1],
-                "formatted_time": ["0.2 ms", "2.8 us"],
-                "angles": [1.4, 0.8],
-                "color": [ts_color_lookup["transfer"], ts_color_lookup["compute"]],
-                "names": ["sum", "sum_partial"],
-            }
-
             fig2 = figure(
                 title="Compute Time Per Task",
                 tools="",
@@ -541,7 +757,7 @@ def __init__(self, scheduler, **kwargs):
                 **kwargs,
             )
 
-            wedge = fig2.wedge(
+            fig2.wedge(
                 x=0,
                 y=1,
                 radius=0.4,
@@ -576,7 +792,7 @@ def __init__(self, scheduler, **kwargs):
             self.wedge_fig = fig2
             tab2 = Panel(child=fig2, title="Pie Chart")
 
-            self.tabs = Tabs(tabs=[tab1, tab2])
+            self.root = Tabs(tabs=[tab1, tab2])
 
     @without_property_validation
     def update(self):
@@ -646,7 +862,7 @@ def __init__(self, scheduler, **kwargs):
 
             self.action_source = ColumnDataSource(data=action_data)
 
-            fig = figure(
+            self.root = figure(
                 title="Aggregate Per Action",
                 tools="",
                 id="bk-aggregate-per-action-plot",
@@ -655,7 +871,7 @@ def __init__(self, scheduler, **kwargs):
                 **kwargs,
             )
 
-            rect = fig.vbar(
+            rect = self.root.vbar(
                 source=self.action_source,
                 x="names",
                 top="times",
@@ -663,21 +879,21 @@ def __init__(self, scheduler, **kwargs):
                 color="color",
             )
 
-            fig.y_range.start = 0
-            fig.min_border_right = 20
-            fig.min_border_bottom = 60
-            fig.yaxis[0].formatter = NumeralTickFormatter(format="0")
-            fig.yaxis.axis_label = "Time (s)"
-            fig.yaxis.ticker = AdaptiveTicker(**TICKS_1024)
-            fig.xaxis.major_label_orientation = -math.pi / 12
-            fig.xaxis.major_label_text_font_size = "16px"
+            self.root.y_range.start = 0
+            self.root.min_border_right = 20
+            self.root.min_border_bottom = 60
+            self.root.yaxis[0].formatter = NumeralTickFormatter(format="0")
+            self.root.yaxis.axis_label = "Time (s)"
+            self.root.yaxis.ticker = AdaptiveTicker(**TICKS_1024)
+            self.root.xaxis.major_label_orientation = XLABEL_ORIENTATION
+            self.root.xaxis.major_label_text_font_size = "16px"
             rect.nonselection_glyph = None
 
-            fig.xaxis.minor_tick_line_alpha = 0
-            fig.xgrid.visible = False
+            self.root.xaxis.minor_tick_line_alpha = 0
+            self.root.xgrid.visible = False
 
-            fig.toolbar.logo = None
-            fig.toolbar_location = None
+            self.root.toolbar.logo = None
+            self.root.toolbar_location = None
 
             hover = HoverTool()
             hover.tooltips = """
@@ -687,9 +903,7 @@ def __init__(self, scheduler, **kwargs):
             </div>
             """
             hover.point_policy = "follow_mouse"
-            fig.add_tools(hover)
-
-            self.fig = fig
+            self.root.add_tools(hover)
 
     @without_property_validation
     def update(self):
@@ -714,8 +928,8 @@ def update(self):
                     agg_colors.append(ts_color_lookup[action])
                 agg_time.append(t)
 
-            self.fig.x_range.factors = agg_names
-            self.fig.title.text = "Aggregate Time Per Action"
+            self.root.x_range.factors = agg_names
+            self.root.title.text = "Aggregate Time Per Action"
 
             action_result = dict(
                 times=agg_time,
@@ -743,7 +957,7 @@ def __init__(self, scheduler, **kwargs):
                 }
             )
 
-            fig = figure(
+            self.root = figure(
                 title="Memory Use",
                 tools="",
                 id="bk-memory-by-key-plot",
@@ -751,19 +965,19 @@ def __init__(self, scheduler, **kwargs):
                 x_range=["a", "b"],
                 **kwargs,
             )
-            rect = fig.vbar(
+            rect = self.root.vbar(
                 source=self.source, x="name", top="nbytes", width=0.9, color="color"
             )
-            fig.yaxis[0].formatter = NumeralTickFormatter(format="0.0 b")
-            fig.yaxis.ticker = AdaptiveTicker(**TICKS_1024)
-            fig.xaxis.major_label_orientation = -math.pi / 12
+            self.root.yaxis[0].formatter = NumeralTickFormatter(format="0.0 b")
+            self.root.yaxis.ticker = AdaptiveTicker(**TICKS_1024)
+            self.root.xaxis.major_label_orientation = XLABEL_ORIENTATION
             rect.nonselection_glyph = None
 
-            fig.xaxis.minor_tick_line_alpha = 0
-            fig.ygrid.visible = False
+            self.root.xaxis.minor_tick_line_alpha = 0
+            self.root.ygrid.visible = False
 
-            fig.toolbar.logo = None
-            fig.toolbar_location = None
+            self.root.toolbar.logo = None
+            self.root.toolbar_location = None
 
             hover = HoverTool()
             hover.tooltips = "@name: @nbytes_text"
@@ -775,9 +989,7 @@ def __init__(self, scheduler, **kwargs):
             </div>
             """
             hover.point_policy = "follow_mouse"
-            fig.add_tools(hover)
-
-            self.fig = fig
+            self.root.add_tools(hover)
 
     @without_property_validation
     def update(self):
@@ -791,7 +1003,7 @@ def update(self):
                     nbytes[ks] += ts.nbytes
 
             names = list(sorted(counts))
-            self.fig.x_range.factors = names
+            self.root.x_range.factors = names
             result = {
                 "name": names,
                 "count": [counts[name] for name in names],
@@ -799,13 +1011,13 @@ def update(self):
                 "nbytes_text": [format_bytes(nbytes[name]) for name in names],
                 "color": [color_of(name) for name in names],
             }
-            self.fig.title.text = "Total Use: " + format_bytes(sum(nbytes.values()))
+            self.root.title.text = "Total Use: " + format_bytes(sum(nbytes.values()))
 
             update(self.source, result)
 
 
 class CurrentLoad(DashboardComponent):
-    """ How many tasks are on each worker """
+    """Tasks and CPU usage on each worker"""
 
     def __init__(self, scheduler, width=600, **kwargs):
         with log_errors():
@@ -813,26 +1025,21 @@ def __init__(self, scheduler, width=600, **kwargs):
             self.scheduler = scheduler
             self.source = ColumnDataSource(
                 {
-                    "nprocessing": [1, 2],
-                    "nprocessing-half": [0.5, 1],
-                    "nprocessing-color": ["red", "blue"],
-                    "nbytes": [1, 2],
-                    "nbytes-half": [0.5, 1],
-                    "nbytes_text": ["1B", "2B"],
-                    "cpu": [1, 2],
-                    "cpu-half": [0.5, 1],
-                    "worker": ["a", "b"],
-                    "y": [1, 2],
-                    "nbytes-color": ["blue", "blue"],
-                    "escaped_worker": ["a", "b"],
+                    "nprocessing": [],
+                    "nprocessing-half": [],
+                    "nprocessing-color": [],
+                    "cpu": [],
+                    "cpu-half": [],
+                    "y": [],
+                    "worker": [],
+                    "escaped_worker": [],
                 }
             )
-
             processing = figure(
                 title="Tasks Processing",
                 tools="",
                 id="bk-nprocessing-plot",
-                name="processing_hist",
+                name="processing",
                 width=int(width / 2),
                 **kwargs,
             )
@@ -847,24 +1054,6 @@ def __init__(self, scheduler, width=600, **kwargs):
             processing.x_range.start = 0
             rect.nonselection_glyph = None
 
-            nbytes = figure(
-                title="Bytes stored",
-                tools="",
-                id="bk-nbytes-worker-plot",
-                width=int(width / 2),
-                name="nbytes_hist",
-                **kwargs,
-            )
-            rect = nbytes.rect(
-                source=self.source,
-                x="nbytes-half",
-                y="y",
-                width="nbytes",
-                height=1,
-                color="nbytes-color",
-            )
-            rect.nonselection_glyph = None
-
             cpu = figure(
                 title="CPU Utilization",
                 tools="",
@@ -884,12 +1073,7 @@ def __init__(self, scheduler, width=600, **kwargs):
             )
             rect.nonselection_glyph = None
 
-            nbytes.axis[0].ticker = BasicTicker(**TICKS_1024)
-            nbytes.xaxis[0].formatter = NumeralTickFormatter(format="0.0 b")
-            nbytes.xaxis.major_label_orientation = -math.pi / 12
-            nbytes.x_range.start = 0
-
-            for fig in [processing, nbytes, cpu]:
+            for fig in (processing, cpu):
                 fig.xaxis.minor_tick_line_alpha = 0
                 fig.yaxis.visible = False
                 fig.ygrid.visible = False
@@ -908,93 +1092,53 @@ def __init__(self, scheduler, width=600, **kwargs):
             hover.point_policy = "follow_mouse"
             processing.add_tools(hover)
 
-            hover = HoverTool()
-            hover.tooltips = "@worker : @nbytes_text"
-            hover.point_policy = "follow_mouse"
-            nbytes.add_tools(hover)
-
             hover = HoverTool()
             hover.tooltips = "@worker : @cpu %"
             hover.point_policy = "follow_mouse"
             cpu.add_tools(hover)
 
             self.processing_figure = processing
-            self.nbytes_figure = nbytes
             self.cpu_figure = cpu
 
-            processing.y_range = nbytes.y_range
-            cpu.y_range = nbytes.y_range
-
     @without_property_validation
     def update(self):
         with log_errors():
             workers = list(self.scheduler.workers.values())
-
-            y = list(range(len(workers)))
+            now = time()
+            if not any(ws.processing for ws in workers) and now < self.last + 1:
+                return
+            self.last = now
 
             cpu = [int(ws.metrics["cpu"]) for ws in workers]
-
             nprocessing = [len(ws.processing) for ws in workers]
-            processing_color = []
+
+            nprocessing_color = []
             for ws in workers:
                 if ws in self.scheduler.idle:
-                    processing_color.append("red")
+                    nprocessing_color.append("red")
                 elif ws in self.scheduler.saturated:
-                    processing_color.append("green")
+                    nprocessing_color.append("green")
                 else:
-                    processing_color.append("blue")
+                    nprocessing_color.append("blue")
 
-            nbytes = [ws.metrics["memory"] for ws in workers]
-            nbytes_text = [format_bytes(nb) for nb in nbytes]
-            nbytes_color = []
-            max_limit = 0
-            for ws, nb in zip(workers, nbytes):
-                limit = (
-                    getattr(self.scheduler.workers[ws.address], "memory_limit", inf)
-                    or inf
-                )
-
-                if limit > max_limit and limit != inf:
-                    max_limit = limit
-
-                if nb > limit:
-                    nbytes_color.append("red")
-                elif nb > limit / 2:
-                    nbytes_color.append("orange")
-                else:
-                    nbytes_color.append("blue")
-
-            now = time()
-            if any(nprocessing) or self.last + 1 < now:
-                self.last = now
-                result = {
-                    "cpu": cpu,
-                    "cpu-half": [c / 2 for c in cpu],
-                    "nprocessing": nprocessing,
-                    "nprocessing-half": [np / 2 for np in nprocessing],
-                    "nprocessing-color": processing_color,
-                    "nbytes": nbytes,
-                    "nbytes-half": [nb / 2 for nb in nbytes],
-                    "nbytes-color": nbytes_color,
-                    "nbytes_text": nbytes_text,
-                    "worker": [ws.address for ws in workers],
-                    "escaped_worker": [escape.url_escape(ws.address) for ws in workers],
-                    "y": y,
-                }
+            result = {
+                "cpu": cpu,
+                "cpu-half": [c / 2 for c in cpu],
+                "nprocessing": nprocessing,
+                "nprocessing-half": [np / 2 for np in nprocessing],
+                "nprocessing-color": nprocessing_color,
+                "worker": [ws.address for ws in workers],
+                "escaped_worker": [escape.url_escape(ws.address) for ws in workers],
+                "y": list(range(len(workers))),
+            }
 
-                self.nbytes_figure.title.text = "Bytes stored: " + format_bytes(
-                    sum(nbytes)
-                )
-                self.nbytes_figure.x_range.end = max_limit
-                if self.scheduler.workers:
-                    self.cpu_figure.x_range.end = (
-                        max(ws.nthreads or 1 for ws in self.scheduler.workers.values())
-                        * 100
-                    )
-                else:
-                    self.cpu_figure.x_range.end = 100
+            if self.scheduler.workers:
+                xrange = max(ws.nthreads or 1 for ws in workers)
+            else:
+                xrange = 1
+            self.cpu_figure.x_range.end = xrange * 100
 
-                update(self.source, result)
+            update(self.source, result)
 
 
 class StealingTimeSeries(DashboardComponent):
@@ -1006,7 +1150,7 @@ def __init__(self, scheduler, **kwargs):
 
         x_range = DataRange1d(follow="end", follow_interval=20000, range_padding=0)
 
-        fig = figure(
+        self.root = figure(
             title="Idle and Saturated Workers Over Time",
             x_axis_type="datetime",
             y_range=[-0.1, len(scheduler.workers) + 0.1],
@@ -1015,16 +1159,14 @@ def __init__(self, scheduler, **kwargs):
             x_range=x_range,
             **kwargs,
         )
-        fig.line(source=self.source, x="time", y="idle", color="red")
-        fig.line(source=self.source, x="time", y="saturated", color="green")
-        fig.yaxis.minor_tick_line_color = None
+        self.root.line(source=self.source, x="time", y="idle", color="red")
+        self.root.line(source=self.source, x="time", y="saturated", color="green")
+        self.root.yaxis.minor_tick_line_color = None
 
-        fig.add_tools(
+        self.root.add_tools(
             ResetTool(), PanTool(dimensions="width"), WheelZoomTool(dimensions="width")
         )
 
-        self.root = fig
-
     @without_property_validation
     def update(self):
         with log_errors():
@@ -1060,7 +1202,7 @@ def __init__(self, scheduler, **kwargs):
 
         x_range = DataRange1d(follow="end", follow_interval=20000, range_padding=0)
 
-        fig = figure(
+        self.root = figure(
             title="Stealing Events",
             x_axis_type="datetime",
             y_axis_type="log",
@@ -1070,7 +1212,7 @@ def __init__(self, scheduler, **kwargs):
             **kwargs,
         )
 
-        fig.circle(
+        self.root.circle(
             source=self.source,
             x="time",
             y="cost_factor",
@@ -1078,21 +1220,19 @@ def __init__(self, scheduler, **kwargs):
             size="radius",
             alpha=0.5,
         )
-        fig.yaxis.axis_label = "Cost Multiplier"
+        self.root.yaxis.axis_label = "Cost Multiplier"
 
         hover = HoverTool()
         hover.tooltips = "Level: @level, Duration: @duration, Count: @count, Cost factor: @cost_factor"
         hover.point_policy = "follow_mouse"
 
-        fig.add_tools(
+        self.root.add_tools(
             hover,
             ResetTool(),
             PanTool(dimensions="width"),
             WheelZoomTool(dimensions="width"),
         )
 
-        self.root = fig
-
     def convert(self, msgs):
         """ Convert a log message to a glyph """
         total_duration = 0
@@ -1157,7 +1297,7 @@ def __init__(self, scheduler, name, height=150, **kwargs):
 
         x_range = DataRange1d(follow="end", follow_interval=200000)
 
-        fig = figure(
+        self.root = figure(
             title=name,
             x_axis_type="datetime",
             height=height,
@@ -1166,7 +1306,7 @@ def __init__(self, scheduler, name, height=150, **kwargs):
             **kwargs,
         )
 
-        fig.circle(
+        self.root.circle(
             source=self.source,
             x="time",
             y="y",
@@ -1175,22 +1315,20 @@ def __init__(self, scheduler, name, height=150, **kwargs):
             alpha=0.5,
             **{"legend_field" if BOKEH_VERSION >= "1.4" else "legend": "action"},
         )
-        fig.yaxis.axis_label = "Action"
-        fig.legend.location = "top_left"
+        self.root.yaxis.axis_label = "Action"
+        self.root.legend.location = "top_left"
 
         hover = HoverTool()
         hover.tooltips = "@action<br>@hover"
         hover.point_policy = "follow_mouse"
 
-        fig.add_tools(
+        self.root.add_tools(
             hover,
             ResetTool(),
             PanTool(dimensions="width"),
             WheelZoomTool(dimensions="width"),
         )
 
-        self.root = fig
-
     @without_property_validation
     def update(self):
         with log_errors():
@@ -1767,7 +1905,14 @@ class WorkerTable(DashboardComponent):
     plot laying out hosts by their current memory use.
     """
 
-    excluded_names = {"executing", "in_flight", "in_memory", "ready", "time"}
+    excluded_names = {
+        "executing",
+        "in_flight",
+        "in_memory",
+        "ready",
+        "time",
+        "spilled_nbytes",
+    }
 
     def __init__(self, scheduler, width=800, **kwargs):
         self.scheduler = scheduler
@@ -1779,6 +1924,10 @@ def __init__(self, scheduler, width=800, **kwargs):
             "memory",
             "memory_limit",
             "memory_percent",
+            "memory_managed",
+            "memory_unmanaged_old",
+            "memory_unmanaged_recent",
+            "memory_spilled",
             "num_fds",
             "read_bytes",
             "write_bytes",
@@ -1803,41 +1952,55 @@ def __init__(self, scheduler, width=800, **kwargs):
             "memory",
             "memory_limit",
             "memory_percent",
+            "memory_managed",
+            "memory_unmanaged_old",
+            "memory_unmanaged_recent",
+            "memory_spilled",
             "num_fds",
             "read_bytes",
             "write_bytes",
         ]
+        column_title_renames = {
+            "memory_limit": "limit",
+            "memory_percent": "memory %",
+            "memory_managed": "managed",
+            "memory_unmanaged_old": "unmanaged",
+            "memory_unmanaged_recent": "unmanaged recent",
+            "memory_spilled": "spilled",
+            "num_fds": "# fds",
+            "read_bytes": "read",
+            "write_bytes": "write",
+        }
 
         self.source = ColumnDataSource({k: [] for k in self.names})
 
         columns = {
-            name: TableColumn(field=name, title=name.replace("_percent", " %"))
+            name: TableColumn(field=name, title=column_title_renames.get(name, name))
             for name in table_names
         }
 
         formatters = {
-            "cpu": NumberFormatter(format="0.0 %"),
+            "cpu": NumberFormatter(format="0 %"),
             "memory_percent": NumberFormatter(format="0.0 %"),
-            "memory": NumberFormatter(format="0.00 b"),
-            "memory_limit": NumberFormatter(format="0.00 b"),
+            "memory": NumberFormatter(format="0.0 b"),
+            "memory_limit": NumberFormatter(format="0.0 b"),
+            "memory_managed": NumberFormatter(format="0.0 b"),
+            "memory_unmanaged_old": NumberFormatter(format="0.0 b"),
+            "memory_unmanaged_recent": NumberFormatter(format="0.0 b"),
+            "memory_spilled": NumberFormatter(format="0.0 b"),
             "read_bytes": NumberFormatter(format="0 b"),
             "write_bytes": NumberFormatter(format="0 b"),
             "num_fds": NumberFormatter(format="0"),
             "nthreads": NumberFormatter(format="0"),
         }
 
-        if BOKEH_VERSION < "0.12.15":
-            dt_kwargs = {"row_headers": False}
-        else:
-            dt_kwargs = {"index_position": None}
-
         table = DataTable(
             source=self.source,
             columns=[columns[n] for n in table_names],
             reorderable=True,
             sortable=True,
             width=width,
-            **dt_kwargs,
+            index_position=None,
         )
 
         for name in table_names:
@@ -1846,7 +2009,7 @@ def __init__(self, scheduler, width=800, **kwargs):
 
         extra_names = ["name", "address"] + self.extra_names
         extra_columns = {
-            name: TableColumn(field=name, title=name.replace("_percent", "%"))
+            name: TableColumn(field=name, title=column_title_renames.get(name, name))
             for name in extra_names
         }
 
@@ -1856,7 +2019,7 @@ def __init__(self, scheduler, width=800, **kwargs):
             reorderable=True,
             sortable=True,
             width=width,
-            **dt_kwargs,
+            index_position=None,
         )
 
         hover = HoverTool(
@@ -1864,7 +2027,7 @@ def __init__(self, scheduler, width=800, **kwargs):
             tooltips="""
                 <div>
                   <span style="font-size: 10px; font-family: Monaco, monospace;">Worker (@name): </span>
-                  <span style="font-size: 10px; font-family: Monaco, monospace;">@memory_percent</span>
+                  <span style="font-size: 10px; font-family: Monaco, monospace;">@memory_percent{0.0 %}</span>
                 </div>
                 """,
         )
@@ -1893,7 +2056,7 @@ def __init__(self, scheduler, width=800, **kwargs):
             tooltips="""
                 <div>
                   <span style="font-size: 10px; font-family: Monaco, monospace;">Worker (@name): </span>
-                  <span style="font-size: 10px; font-family: Monaco, monospace;">@cpu_fraction</span>
+                  <span style="font-size: 10px; font-family: Monaco, monospace;">@cpu_fraction{0 %}</span>
                 </div>
                 """,
         )
@@ -1935,6 +2098,8 @@ def update(self):
         for i, (addr, ws) in enumerate(
             sorted(self.scheduler.workers.items(), key=lambda kv: str(kv[1].name))
         ):
+            minfo = ws.memory
+
             for name in self.names + self.extra_names:
                 data[name].append(ws.metrics.get(name, None))
             data["name"][-1] = ws.name if ws.name is not None else i
@@ -1944,6 +2109,11 @@ def update(self):
             else:
                 data["memory_percent"][-1] = ""
             data["memory_limit"][-1] = ws.memory_limit
+            data["memory_managed"][-1] = minfo.managed_in_memory
+            data["memory_unmanaged_old"][-1] = minfo.unmanaged_old
+            data["memory_unmanaged_recent"][-1] = minfo.unmanaged_recent
+            data["memory_unmanaged_recent"][-1] = minfo.unmanaged_recent
+            data["memory_spilled"][-1] = minfo.managed_spilled
             data["cpu"][-1] = ws.metrics["cpu"] / 100.0
             data["cpu_fraction"][-1] = ws.metrics["cpu"] / 100.0 / ws.nthreads
             data["nthreads"][-1] = ws.nthreads
@@ -2089,6 +2259,29 @@ def graph_doc(scheduler, extra, doc):
 
 def status_doc(scheduler, extra, doc):
     with log_errors():
+        nbytes_cluster = NBytesCluster(scheduler, sizing_mode="stretch_both")
+        nbytes_cluster.update()
+        add_periodic_callback(doc, nbytes_cluster, 100)
+        doc.add_root(nbytes_cluster.root)
+
+        if len(scheduler.workers) < 50:
+            nbytes_workers = NBytes(scheduler, sizing_mode="stretch_both")
+            processing = CurrentLoad(scheduler, sizing_mode="stretch_both")
+            processing_root = processing.processing_figure
+            processing_root.y_range = nbytes_workers.root.y_range
+        else:
+            nbytes_workers = NBytesHistogram(scheduler, sizing_mode="stretch_both")
+            processing = ProcessingHistogram(scheduler, sizing_mode="stretch_both")
+            processing_root = processing.root
+            row(nbytes_workers.root, processing.root, sizing_mode="stretch_both")
+
+        nbytes_workers.update()
+        processing.update()
+        add_periodic_callback(doc, nbytes_workers, 100)
+        add_periodic_callback(doc, processing, 100)
+        doc.add_root(nbytes_workers.root)
+        doc.add_root(processing_root)
+
         task_stream = TaskStream(
             scheduler,
             n_rectangles=dask.config.get(
@@ -2099,99 +2292,26 @@ def status_doc(scheduler, extra, doc):
         )
         task_stream.update()
         add_periodic_callback(doc, task_stream, 100)
+        doc.add_root(task_stream.root)
 
         task_progress = TaskProgress(scheduler, sizing_mode="stretch_both")
         task_progress.update()
         add_periodic_callback(doc, task_progress, 100)
-
-        if len(scheduler.workers) < 50:
-            current_load = CurrentLoad(scheduler, sizing_mode="stretch_both")
-            current_load.update()
-            add_periodic_callback(doc, current_load, 100)
-            doc.add_root(current_load.nbytes_figure)
-            doc.add_root(current_load.processing_figure)
-        else:
-            nbytes_hist = NBytesHistogram(scheduler, sizing_mode="stretch_both")
-            nbytes_hist.update()
-            processing_hist = ProcessingHistogram(scheduler, sizing_mode="stretch_both")
-            processing_hist.update()
-            add_periodic_callback(doc, nbytes_hist, 100)
-            add_periodic_callback(doc, processing_hist, 100)
-            current_load_fig = row(
-                nbytes_hist.root, processing_hist.root, sizing_mode="stretch_both"
-            )
-
-            doc.add_root(nbytes_hist.root)
-            doc.add_root(processing_hist.root)
+        doc.add_root(task_progress.root)
 
         doc.title = "Dask: Status"
-        doc.add_root(task_progress.root)
-        doc.add_root(task_stream.root)
         doc.theme = BOKEH_THEME
         doc.template = env.get_template("status.html")
         doc.template_variables.update(extra)
-        doc.theme = BOKEH_THEME
-
-
-def individual_task_stream_doc(scheduler, extra, doc):
-    task_stream = TaskStream(
-        scheduler, n_rectangles=1000, clear_interval="10s", sizing_mode="stretch_both"
-    )
-    task_stream.update()
-    add_periodic_callback(doc, task_stream, 100)
-    doc.add_root(task_stream.root)
-    doc.theme = BOKEH_THEME
-
-
-def individual_nbytes_doc(scheduler, extra, doc):
-    current_load = CurrentLoad(scheduler, sizing_mode="stretch_both")
-    current_load.update()
-    add_periodic_callback(doc, current_load, 100)
-    doc.add_root(current_load.nbytes_figure)
-    doc.theme = BOKEH_THEME
-
-
-def individual_cpu_doc(scheduler, extra, doc):
-    current_load = CurrentLoad(scheduler, sizing_mode="stretch_both")
-    current_load.update()
-    add_periodic_callback(doc, current_load, 100)
-    doc.add_root(current_load.cpu_figure)
-    doc.theme = BOKEH_THEME
-
-
-def individual_nprocessing_doc(scheduler, extra, doc):
-    current_load = CurrentLoad(scheduler, sizing_mode="stretch_both")
-    current_load.update()
-    add_periodic_callback(doc, current_load, 100)
-    doc.add_root(current_load.processing_figure)
-    doc.theme = BOKEH_THEME
-
-
-def individual_progress_doc(scheduler, extra, doc):
-    task_progress = TaskProgress(scheduler, height=160, sizing_mode="stretch_both")
-    task_progress.update()
-    add_periodic_callback(doc, task_progress, 100)
-    doc.add_root(task_progress.root)
-    doc.theme = BOKEH_THEME
-
-
-def individual_graph_doc(scheduler, extra, doc):
-    with log_errors():
-        graph = TaskGraph(scheduler, sizing_mode="stretch_both")
-        graph.update()
 
-        add_periodic_callback(doc, graph, 200)
-        doc.add_root(graph.root)
-        doc.theme = BOKEH_THEME
 
-
-def individual_systemmonitor_doc(scheduler, extra, doc):
+@curry
+def individual_doc(cls, interval, scheduler, extra, doc, fig_attr="root", **kwargs):
     with log_errors():
-        sysmon = SystemMonitor(scheduler, sizing_mode="stretch_both")
-        doc.title = "Dask: Scheduler System Monitor"
-        add_periodic_callback(doc, sysmon, 500)
-
-        doc.add_root(sysmon.root)
+        fig = cls(scheduler, sizing_mode="stretch_both", **kwargs)
+        fig.update()
+        add_periodic_callback(doc, fig, interval)
+        doc.add_root(getattr(fig, fig_attr))
         doc.theme = BOKEH_THEME
 
 
@@ -2211,60 +2331,6 @@ def individual_profile_server_doc(scheduler, extra, doc):
         doc.theme = BOKEH_THEME
 
 
-def individual_workers_doc(scheduler, extra, doc):
-    with log_errors():
-        table = WorkerTable(scheduler)
-        table.update()
-        add_periodic_callback(doc, table, 500)
-        doc.add_root(table.root)
-        doc.theme = BOKEH_THEME
-
-
-def individual_bandwidth_types_doc(scheduler, extra, doc):
-    with log_errors():
-        bw = BandwidthTypes(scheduler, sizing_mode="stretch_both")
-        bw.update()
-        add_periodic_callback(doc, bw, 500)
-        doc.add_root(bw.fig)
-        doc.theme = BOKEH_THEME
-
-
-def individual_bandwidth_workers_doc(scheduler, extra, doc):
-    with log_errors():
-        bw = BandwidthWorkers(scheduler, sizing_mode="stretch_both")
-        bw.update()
-        add_periodic_callback(doc, bw, 500)
-        doc.add_root(bw.fig)
-        doc.theme = BOKEH_THEME
-
-
-def individual_memory_by_key_doc(scheduler, extra, doc):
-    with log_errors():
-        component = MemoryByKey(scheduler, sizing_mode="stretch_both")
-        component.update()
-        add_periodic_callback(doc, component, 500)
-        doc.add_root(component.fig)
-        doc.theme = BOKEH_THEME
-
-
-def individual_compute_time_per_key_doc(scheduler, extra, doc):
-    with log_errors():
-        component = ComputePerKey(scheduler, sizing_mode="stretch_both")
-        component.update()
-        add_periodic_callback(doc, component, 500)
-        doc.add_root(component.tabs)
-        doc.theme = BOKEH_THEME
-
-
-def individual_aggregate_time_per_action_doc(scheduler, extra, doc):
-    with log_errors():
-        component = AggregateAction(scheduler, sizing_mode="stretch_both")
-        component.update()
-        add_periodic_callback(doc, component, 500)
-        doc.add_root(component.fig)
-        doc.theme = BOKEH_THEME
-
-
 def profile_doc(scheduler, extra, doc):
     with log_errors():
         doc.title = "Dask: Profile"
diff --git a/distributed/dashboard/scheduler.py b/distributed/dashboard/scheduler.py
index f0d85b04db9..a237d21a8a5 100644
--- a/distributed/dashboard/scheduler.py
+++ b/distributed/dashboard/scheduler.py
@@ -11,23 +11,24 @@
 from .components.nvml import gpu_doc  # noqa: 1708
 from .components.nvml import NVML_ENABLED, gpu_memory_doc, gpu_utilization_doc
 from .components.scheduler import (
+    AggregateAction,
+    BandwidthTypes,
+    BandwidthWorkers,
+    ComputePerKey,
+    CurrentLoad,
+    MemoryByKey,
+    NBytes,
+    NBytesCluster,
+    SystemMonitor,
+    TaskGraph,
+    TaskProgress,
+    TaskStream,
+    WorkerTable,
     events_doc,
     graph_doc,
-    individual_aggregate_time_per_action_doc,
-    individual_bandwidth_types_doc,
-    individual_bandwidth_workers_doc,
-    individual_compute_time_per_key_doc,
-    individual_cpu_doc,
-    individual_graph_doc,
-    individual_memory_by_key_doc,
-    individual_nbytes_doc,
-    individual_nprocessing_doc,
+    individual_doc,
     individual_profile_doc,
     individual_profile_server_doc,
-    individual_progress_doc,
-    individual_systemmonitor_doc,
-    individual_task_stream_doc,
-    individual_workers_doc,
     profile_doc,
     profile_server_doc,
     status_doc,
@@ -36,7 +37,6 @@
     tasks_doc,
     workers_doc,
 )
-from .components.worker import counters_doc
 from .core import BokehApplication
 from .worker import counters_doc
 
@@ -79,21 +79,26 @@ def connect(application, http_server, scheduler, prefix=""):
     "/profile-server": profile_server_doc,
     "/graph": graph_doc,
     "/gpu": gpu_doc,
-    "/individual-task-stream": individual_task_stream_doc,
-    "/individual-progress": individual_progress_doc,
-    "/individual-graph": individual_graph_doc,
+    "/individual-task-stream": individual_doc(
+        TaskStream, 100, n_rectangles=1000, clear_interval="10s"
+    ),
+    "/individual-progress": individual_doc(TaskProgress, 100, height=160),
+    "/individual-graph": individual_doc(TaskGraph, 200),
+    "/individual-nbytes": individual_doc(NBytes, 100),
+    "/individual-nbytes-cluster": individual_doc(NBytesCluster, 100),
+    "/individual-cpu": individual_doc(CurrentLoad, 100, fig_attr="cpu_figure"),
+    "/individual-nprocessing": individual_doc(
+        CurrentLoad, 100, fig_attr="processing_figure"
+    ),
+    "/individual-workers": individual_doc(WorkerTable, 500),
+    "/individual-bandwidth-types": individual_doc(BandwidthTypes, 500),
+    "/individual-bandwidth-workers": individual_doc(BandwidthWorkers, 500),
+    "/individual-memory-by-key": individual_doc(MemoryByKey, 500),
+    "/individual-compute-time-per-key": individual_doc(ComputePerKey, 500),
+    "/individual-aggregate-time-per-action": individual_doc(AggregateAction, 500),
+    "/individual-scheduler-system": individual_doc(SystemMonitor, 500),
     "/individual-profile": individual_profile_doc,
     "/individual-profile-server": individual_profile_server_doc,
-    "/individual-nbytes": individual_nbytes_doc,
-    "/individual-cpu": individual_cpu_doc,
-    "/individual-nprocessing": individual_nprocessing_doc,
-    "/individual-workers": individual_workers_doc,
-    "/individual-bandwidth-types": individual_bandwidth_types_doc,
-    "/individual-bandwidth-workers": individual_bandwidth_workers_doc,
-    "/individual-memory-by-key": individual_memory_by_key_doc,
-    "/individual-compute-time-per-key": individual_compute_time_per_key_doc,
-    "/individual-aggregate-time-per-action": individual_aggregate_time_per_action_doc,
     "/individual-gpu-memory": gpu_memory_doc,
     "/individual-gpu-utilization": gpu_utilization_doc,
-    "/individual-scheduler-system": individual_systemmonitor_doc,
 }
diff --git a/distributed/dashboard/tests/test_scheduler_bokeh.py b/distributed/dashboard/tests/test_scheduler_bokeh.py
index 3f8a1f2ce71..7034dc7394d 100644
--- a/distributed/dashboard/tests/test_scheduler_bokeh.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh.py
@@ -25,6 +25,8 @@
     CurrentLoad,
     Events,
     MemoryByKey,
+    NBytes,
+    NBytesCluster,
     NBytesHistogram,
     Occupancy,
     ProcessingHistogram,
@@ -255,9 +257,7 @@ async def test_CurrentLoad(c, s, a, b):
     cl.update()
     d = dict(cl.source.data)
 
-    assert all(len(L) == 2 for L in d.values())
-    assert all(d["nbytes"])
-
+    assert all(len(l) == 2 for l in d.values())
     assert cl.cpu_figure.x_range.end == 200
 
 
@@ -275,6 +275,41 @@ async def test_ProcessingHistogram(c, s, a, b):
     assert ph.source.data["right"][-1] > 2
 
 
+@gen_cluster(client=True)
+async def test_NBytes(c, s, a, b):
+    cl = NBytes(s)
+
+    futures = c.map(slowinc, range(10), delay=0.001)
+    await wait(futures)
+
+    cl.update()
+    d = dict(cl.source.data)
+    llens = {len(l) for l in d.values()}
+    # There are 2 workers. There is definitely going to be managed_in_memory and
+    # unmanaged_old; there may be unmanaged_new. There won't be managed_spilled.
+    # Empty rects are removed.
+    assert llens in ({4}, {5}, {6})
+    assert all(d["width"])
+
+
+@gen_cluster(client=True)
+async def test_NBytesCluster(c, s, a, b):
+    cl = NBytesCluster(s)
+
+    futures = c.map(slowinc, range(10), delay=0.001)
+    await wait(futures)
+
+    cl.update()
+    d = dict(cl.source.data)
+    llens = {len(l) for l in d.values()}
+    # Unlike NBytes, empty rects here aren't pruned away.
+    assert llens == {4}
+    # There is definitely going to be managed_in_memory and
+    # unmanaged_old; there may be unmanaged_new. There won't be managed_spilled.
+    assert any(d["width"])
+    assert not all(d["width"])
+
+
 @gen_cluster(client=True)
 async def test_NBytesHistogram(c, s, a, b):
     nh = NBytesHistogram(s)
diff --git a/distributed/distributed-schema.yaml b/distributed/distributed-schema.yaml
index 331fdd10ef0..a809e03ad7a 100644
--- a/distributed/distributed-schema.yaml
+++ b/distributed/distributed-schema.yaml
@@ -392,39 +392,50 @@ properties:
 
           memory:
             type: object
-            description: |
-              When Dask workers have more data than memory they spill this data to disk.
-              They do this at a few conditions.
+            description: >-
+              Settings for memory management
             properties:
+              recent_to_old_time:
+                type: string
+                description: >-
+                  When there is an increase in process memory (as observed by the
+                  operating system) that is not accounted for by the dask keys stored on
+                  the worker, ignore it for this long before considering it in
+                  non-time-sensitive heuristics. This should be set to be longer than
+                  the duration of most dask tasks.
+
               target:
-                type: number
-                minimum: 0
-                maximum: 1
-                description: |
-                  Target fraction below which to try to keep memory
+                oneOf:
+                  - {type: number, minimum: 0, maximum: 1}
+                  - {enum: [false]}
+                description: >-
+                  When the process memory (as observed by the operating system) gets
+                  above this amount we start spilling the dask keys holding the largest
+                  chunks of data to disk
 
               spill:
-                type: number
-                minimum: 0
-                maximum: 1
-                description: |
-                  When the process memory (as observed by the operating system) gets above this amount we spill data to disk.
+                oneOf:
+                  - {type: number, minimum: 0, maximum: 1}
+                  - {enum: [false]}
+                description: >-
+                  When the process memory (as observed by the operating system) gets
+                  above this amount we spill all data to disk.
 
               pause:
-                type: number
-                minimum: 0
-                maximum: 1
-                description: |
-                  When the process memory (as observed by the operating system) gets above this amount
-                  we no longer start new tasks on this worker.
+                oneOf:
+                  - {type: number, minimum: 0, maximum: 1}
+                  - {enum: [false]}
+                description: >-
+                  When the process memory (as observed by the operating system) gets
+                  above this amount we no longer start new tasks on this worker.
 
               terminate:
-                type: number
-                minimum: 0
-                maximum: 1
-                description: |
-                  When the process memory reaches this level the nanny process will kill the worker
-                  (if a nanny is present)
+                oneOf:
+                  - {type: number, minimum: 0, maximum: 1}
+                  - {enum: [false]}
+                description: >-
+                  When the process memory reaches this level the nanny process will kill
+                  the worker (if a nanny is present)
 
           http:
             type: object
@@ -435,17 +446,6 @@ properties:
                 description: |
                   A list of modules like "prometheus" and "health" that can be included or excluded as desired
 
-                  These modules will have a ``routes`` keyword that gets added to the main HTTP Server.
-                  This is also a list that can be extended with user defined modules.
-          http:
-            type: object
-            decription: Settings for Dask's embedded HTTP Server
-            properties:
-              routes:
-                type: array
-                description: |
-                  A list of modules like "prometheus" and "health" that can be included or excluded as desired
-
                   These modules will have a ``routes`` keyword that gets added to the main HTTP Server.
                   This is also a list that can be extended with user defined modules.
 
diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index fb01a168d38..08aed7f9dae 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -83,9 +83,15 @@ distributed:
       low-level: False      # Whether or not to include low-level functions
                             # Requires https://github.com/numba/stacktrace
 
-    # Fractions of worker memory at which we take action to avoid memory blowup
-    # Set any of the lower three values to False to turn off the behavior entirely
     memory:
+      # When there is an increase in process memory (as observed by the operating
+      # system) that is not accounted for by the dask keys stored on the worker, ignore
+      # it for this long before considering it in non-critical memory measures.
+      # This should be set to be longer than the duration of most dask tasks.
+      recent_to_old_time: 30s
+
+      # Fractions of worker process memory at which we take action to avoid memory
+      # blowup. Set any of the values to False to turn off the behavior entirely.
       target: 0.60  # target fraction to stay below
       spill: 0.70  # fraction at which we spill to disk
       pause: 0.80  # fraction at which we pause worker threads
diff --git a/distributed/http/static/css/status.css b/distributed/http/static/css/status.css
index 9de0d01b353..9467ba7a3c8 100644
--- a/distributed/http/static/css/status.css
+++ b/distributed/http/static/css/status.css
@@ -6,38 +6,46 @@
 @media (min-width: 0px) {
   #status-fluid {
     grid-template-columns: 1fr 1fr;
-    grid-template-rows: 1fr 3fr 1fr;
+    grid-template-rows: 70px 2fr 6fr 2fr;
   }
-  #status-history {
-    grid-column: 1;
+  #status-nbytes-cluster {
+    grid-column: 1 / span 2;
     grid-row: 1;
   }
+  #status-nbytes-workers {
+    grid-column: 1;
+    grid-row: 2;
+  }
   #status-processing {
     grid-column: 2;
-    grid-row: 1;
+    grid-row: 2;
   }
   #status-tasks {
     grid-column: 1 / span 2;
-    grid-row: 2;
+    grid-row: 3;
   }
   #status-progress {
     grid-column: 1 / span 2;
-    grid-row: 3;
+    grid-row: 4;
   }
 }
 
 @media (min-width: 992px) {
   #status-fluid {
     grid-template-columns: 1fr 3fr;
-    grid-template-rows: 1fr 1fr 1fr 1fr 1fr 1fr;
+    grid-template-rows: 80px 4fr 1fr 1fr 4fr;
+  }
+  #status-nbytes-cluster {
+    grid-column: 1;
+    grid-row: 1;
   }
-  #status-history {
+  #status-nbytes-workers {
     grid-column: 1;
-    grid-row: 1 / span 3;
+    grid-row: 2 / span 2;
   }
   #status-processing {
     grid-column: 1;
-    grid-row: 4 / span 3;
+    grid-row: 4 / span 2;
   }
   #status-tasks {
     grid-column: 2;
@@ -45,6 +53,6 @@
   }
   #status-progress {
     grid-column: 2;
-    grid-row: 5 / span 2;
+    grid-row: 5;
   }
 }
diff --git a/distributed/http/templates/status.html b/distributed/http/templates/status.html
index 26e867f62c7..57b8394c931 100644
--- a/distributed/http/templates/status.html
+++ b/distributed/http/templates/status.html
@@ -8,12 +8,16 @@
 {% from macros import embed %}
 <div id="status-fluid">
 
-  <div id="status-history">
-    {{ embed(roots.nbytes_hist) }}
+  <div id="status-nbytes-cluster">
+    {{ embed(roots.nbytes_cluster) }}
+  </div>
+
+  <div id="status-nbytes-workers">
+    {{ embed(roots.nbytes_workers) }}
   </div>
 
   <div id="status-processing">
-    {{ embed(roots.processing_hist) }}
+    {{ embed(roots.processing) }}
   </div>
 
   <div id="status-tasks">
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 7778313db7d..78062310eb6 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -164,6 +164,10 @@ def nogil(func):
     double,
     parse_timedelta(dask.config.get("distributed.scheduler.unknown-task-duration")),
 )
+MEMORY_RECENT_TO_OLD_TIME = declare(
+    double,
+    parse_timedelta(dask.config.get("distributed.worker.memory.recent_to_old_time")),
+)
 
 DEFAULT_EXTENSIONS = [
     LockExtension,
@@ -256,6 +260,120 @@ def versions(self):
         return self._versions
 
 
+@final
+@cclass
+class MemoryState:
+    """Memory readings on a worker or on the whole cluster.
+
+    managed
+        Sum of the output of sizeof() for all dask keys held by the worker, both in
+        memory and spilled to disk
+    managed_in_memory
+        Sum of the output of sizeof() for the dask keys held in RAM
+    managed_spilled
+        Sum of the output of sizeof() for the dask keys spilled to the hard drive.
+        Note that this is the size in memory; serialized size may be different.
+    process
+        Total RSS memory measured by the OS on the worker process.
+        This is always exactly equal to managed_in_memory + unmanaged.
+    unmanaged
+        process - managed_in_memory. This is the sum of
+
+        - Python interpreter and modules
+        - global variables
+        - memory temporarily allocated by the dask tasks that are currently running
+        - memory fragmentation
+        - memory leaks
+        - memory not yet garbage collected
+        - memory not yet free()'d by the Python memory manager to the OS
+
+    unmanaged_old
+        Minimum of the 'unmanaged' measures over the last
+        ``distributed.memory.recent_to_old_time`` seconds
+    unmanaged_recent
+        unmanaged - unmanaged_old; in other words process memory that has been recently
+        allocated but is not accounted for by dask; hopefully it's mostly a temporary
+        spike.
+    optimistic
+        managed_in_memory + unmanaged_old; in other words the memory held long-term by
+        the process under the hopeful assumption that all unmanaged_recent memory is a
+        temporary spike
+    """
+
+    __slots__ = ("process", "managed_in_memory", "managed_spilled", "unmanaged_old")
+
+    process: Py_ssize_t
+    managed_in_memory: Py_ssize_t
+    managed_spilled: Py_ssize_t
+    unmanaged_old: Py_ssize_t
+
+    def __init__(
+        self,
+        *,
+        process: Py_ssize_t,
+        unmanaged_old: Py_ssize_t,
+        managed: Py_ssize_t,
+        managed_spilled: Py_ssize_t,
+    ):
+        # Some data arrives with the heartbeat, some other arrives in realtime as the
+        # tasks progress. Also, sizeof() is not guaranteed to return correct results.
+        # This can cause glitches where a partial measure is larger than the whole, so
+        # we need to force all numbers to add up exactly by definition.
+        self.process = process
+        self.managed_spilled = min(managed_spilled, managed)
+        # Subtractions between unsigned ints guaranteed by construction to be >= 0
+        self.managed_in_memory = min(managed - self.managed_spilled, process)
+        self.unmanaged_old = min(unmanaged_old, process - self.managed_in_memory)
+
+    @classmethod
+    def sum(cls, *infos: "MemoryState") -> "MemoryState":
+        out = object.__new__(cls)
+        for k in cls.__slots__:
+            setattr(out, k, sum(getattr(i, k) for i in infos))
+        return out
+
+    @property
+    @ccall
+    @inline
+    @nogil
+    def managed(self) -> Py_ssize_t:
+        return self.managed_in_memory + self.managed_spilled
+
+    @property
+    @ccall
+    @inline
+    @nogil
+    def unmanaged(self) -> Py_ssize_t:
+        # This is never negative thanks to __init__
+        return self.process - self.managed_in_memory
+
+    @property
+    @ccall
+    @inline
+    @nogil
+    def unmanaged_recent(self) -> Py_ssize_t:
+        # This is never negative thanks to __init__
+        return self.process - self.managed_in_memory - self.unmanaged_old
+
+    @property
+    @ccall
+    @inline
+    @nogil
+    def optimistic(self) -> Py_ssize_t:
+        return self.managed_in_memory + self.unmanaged_old
+
+    def __repr__(self) -> str:
+        return (
+            f"Managed by Dask       : {format_bytes(self.managed)}\n"
+            f"  - in process memory : {format_bytes(self.managed_in_memory)}\n"
+            f"  - spilled to disk   : {format_bytes(self.managed_spilled)}\n"
+            f"Process memory (RSS)  : {format_bytes(self.process)}\n"
+            f"  - managed by Dask   : {format_bytes(self.managed_in_memory)}\n"
+            f"  - unmanaged (old)   : {format_bytes(self.unmanaged_old)}\n"
+            f"  - unmanaged (recent): {format_bytes(self.unmanaged_recent)}\n"
+        )
+
+
 @final
 @cclass
 class WorkerState:
@@ -355,6 +473,8 @@ class WorkerState:
     _last_seen: double
     _local_directory: str
     _memory_limit: Py_ssize_t
+    _memory_other_history: "deque[tuple[float, Py_ssize_t]]"
+    _memory_unmanaged_old: Py_ssize_t
     _metrics: dict
     _name: object
     _nanny: str
@@ -381,6 +501,8 @@ class WorkerState:
         "_last_seen",
         "_local_directory",
         "_memory_limit",
+        "_memory_other_history",
+        "_memory_unmanaged_old",
         "_metrics",
         "_name",
         "_nanny",
@@ -424,6 +546,8 @@ def __init__(
         self._status = Status.running
         self._nbytes = 0
         self._occupancy = 0
+        self._memory_unmanaged_old = 0
+        self._memory_other_history = deque()
         self._metrics = {}
         self._last_seen = 0
         self._time_delay = 0
@@ -496,6 +620,15 @@ def memory_limit(self):
     def metrics(self):
         return self._metrics
 
+    @property
+    def memory(self) -> MemoryState:
+        return MemoryState(
+            process=self._metrics["memory"],
+            managed=self._nbytes,
+            managed_spilled=self._metrics["spilled_nbytes"],
+            unmanaged_old=self._memory_unmanaged_old,
+        )
+
     @property
     def name(self):
         return self._name
@@ -1773,6 +1906,10 @@ def validate(self, v: bint):
     def workers(self):
         return self._workers
 
+    @property
+    def memory(self) -> MemoryState:
+        return MemoryState.sum(*(w.memory for w in self.workers.values()))
+
     @property
     def __pdict__(self):
         return {
@@ -3684,6 +3821,29 @@ def heartbeat_worker(
 
         ws._metrics = metrics
 
+        # Calculate RSS - dask keys, separating "old" and "new" usage
+        # See MemoryState for details
+        max_memory_unmanaged_old_hist_age = local_now - MEMORY_RECENT_TO_OLD_TIME
+        memory_unmanaged_old = ws._memory_unmanaged_old
+        while ws._memory_other_history:
+            timestamp, size = ws._memory_other_history[0]
+            if timestamp >= max_memory_unmanaged_old_hist_age:
+                break
+            ws._memory_other_history.popleft()
+            if size == memory_unmanaged_old:
+                memory_unmanaged_old = 0  # recalculate min()
+
+        size = max(0, metrics["memory"] - ws._nbytes + ws._metrics["spilled_nbytes"])
+        ws._memory_other_history.append((local_now, size))
+        if not memory_unmanaged_old:
+            # The worker has just been started or the previous minimum has been expunged
+            # because too old.
+            # Note: this algorithm is capped to 200 * MEMORY_RECENT_TO_OLD_TIME elements
+            # cluster-wide by heartbeat_interval(), regardless of the number of workers
+            ws._memory_unmanaged_old = min(map(second, ws._memory_other_history))
+        elif size < memory_unmanaged_old:
+            ws._memory_unmanaged_old = size
+
         if host_info:
             dh: dict = parent._host_info.setdefault(host, {})
             dh.update(host_info)
@@ -6478,9 +6638,9 @@ def profile_to_figure(state):
         scheduler = Panel(child=scheduler, title="Scheduler Profile (administrative)")
         task_stream = Panel(child=task_stream, title="Task Stream")
         bandwidth_workers = Panel(
-            child=bandwidth_workers.fig, title="Bandwidth (Workers)"
+            child=bandwidth_workers.root, title="Bandwidth (Workers)"
         )
-        bandwidth_types = Panel(child=bandwidth_types.fig, title="Bandwidth (Types)")
+        bandwidth_types = Panel(child=bandwidth_types.root, title="Bandwidth (Types)")
 
         tabs = Tabs(
             tabs=[
diff --git a/distributed/spill.py b/distributed/spill.py
new file mode 100644
index 00000000000..5d90361d170
--- /dev/null
+++ b/distributed/spill.py
@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+from collections.abc import Hashable, Mapping
+from functools import partial
+from typing import Any
+
+from .protocol import deserialize_bytes, serialize_bytelist
+from .sizeof import safe_sizeof
+
+try:
+    from zict import Buffer, File, Func
+except ImportError:
+    raise ImportError("Please `python -m pip install zict` for spill-to-disk workers")
+
+
+class SpillBuffer(Buffer):
+    """MutableMapping that automatically spills out dask key/value pairs to disk when
+    the total size of the stored data exceeds the target
+    """
+
+    spilled_by_key: dict[Hashable, int]
+    spilled_total: int
+
+    def __init__(self, spill_directory: str, target: int):
+        self.spilled_by_key = {}
+        self.spilled_total = 0
+        storage = Func(
+            partial(serialize_bytelist, on_error="raise"),
+            deserialize_bytes,
+            File(spill_directory),
+        )
+        super().__init__(
+            {},
+            storage,
+            target,
+            weight=self._weight,
+            fast_to_slow_callbacks=[self._on_evict],
+            slow_to_fast_callbacks=[self._on_retrieve],
+        )
+
+    @property
+    def memory(self) -> Mapping[Hashable, Any]:
+        """Key/value pairs stored in RAM. Alias of zict.Buffer.fast.
+        For inspection only - do not modify directly!
+        """
+        return self.fast
+
+    @property
+    def disk(self) -> Mapping[Hashable, Any]:
+        """Key/value pairs spilled out to disk. Alias of zict.Buffer.slow.
+        For inspection only - do not modify directly!
+        """
+        return self.slow
+
+    @staticmethod
+    def _weight(key: Hashable, value: Any) -> int:
+        return safe_sizeof(value)
+
+    def _on_evict(self, key: Hashable, value: Any) -> None:
+        b = safe_sizeof(value)
+        self.spilled_by_key[key] = b
+        self.spilled_total += b
+
+    def _on_retrieve(self, key: Hashable, value: Any) -> None:
+        self.spilled_total -= self.spilled_by_key.pop(key)
+
+    def __setitem__(self, key: Hashable, value: Any) -> None:
+        self.spilled_total -= self.spilled_by_key.pop(key, 0)
+        super().__setitem__(key, value)
+        if key in self.slow:
+            # value is individually larger than target so it went directly to slow.
+            # _on_evict was not called.
+            b = safe_sizeof(value)
+            self.spilled_by_key[key] = b
+            self.spilled_total += b
+
+    def __delitem__(self, key: Hashable) -> None:
+        self.spilled_total -= self.spilled_by_key.pop(key, 0)
+        super().__delitem__(key)
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index c82a4a04839..98b175f5946 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -1,10 +1,13 @@
 import asyncio
+import gc
 import json
 import logging
 import operator
 import re
 import sys
 from collections import defaultdict
+from itertools import product
+from textwrap import dedent
 from time import sleep
 
 import cloudpickle
@@ -22,7 +25,7 @@
 from distributed.core import ConnectionPool, Status, connect, rpc
 from distributed.metrics import time
 from distributed.protocol.pickle import dumps
-from distributed.scheduler import Scheduler
+from distributed.scheduler import MemoryState, Scheduler
 from distributed.utils import TimeoutError, tmpfile, typename
 from distributed.utils_test import (  # noqa: F401
     captured_logger,
@@ -2242,3 +2245,185 @@ async def test_quiet_cluster_round_robin(c, s, a, b):
     await c.submit(inc, 2)
     await c.submit(inc, 3)
     assert a.log and b.log
+
+
+def test_memorystate():
+    m = MemoryState(process=100, unmanaged_old=15, managed=80, managed_spilled=12)
+    assert m.process == 100
+    assert m.managed == 80
+    assert m.managed_in_memory == 68
+    assert m.managed_spilled == 12
+    assert m.unmanaged == 32
+    assert m.unmanaged_old == 15
+    assert m.unmanaged_recent == 17
+    assert m.optimistic == 83
+
+    assert (
+        repr(m)
+        == dedent(
+            """
+            Managed by Dask       : 80 B
+              - in process memory : 68 B
+              - spilled to disk   : 12 B
+            Process memory (RSS)  : 100 B
+              - managed by Dask   : 68 B
+              - unmanaged (old)   : 15 B
+              - unmanaged (recent): 17 B
+            """
+        ).lstrip()
+    )
+
+
+def test_memorystate_sum():
+    m1 = MemoryState(process=100, unmanaged_old=15, managed=80, managed_spilled=12)
+    m2 = MemoryState(process=80, unmanaged_old=10, managed=60, managed_spilled=2)
+    m3 = MemoryState.sum(m1, m2)
+    assert m3.process == 180
+    assert m3.unmanaged_old == 25
+    assert m3.managed == 140
+    assert m3.managed_spilled == 14
+
+
+@pytest.mark.parametrize(
+    "process,unmanaged_old,managed,managed_spilled", list(product(*[[0, 1, 2, 3]] * 4))
+)
+def test_memorystate_adds_up(process, unmanaged_old, managed, managed_spilled):
+    """Input data is massaged by __init__ so that everything adds up by construction"""
+    m = MemoryState(
+        process=process,
+        unmanaged_old=unmanaged_old,
+        managed=managed,
+        managed_spilled=managed_spilled,
+    )
+    assert m.managed_in_memory + m.unmanaged == m.process
+    assert m.managed_in_memory + m.managed_spilled == m.managed
+    assert m.unmanaged_old + m.unmanaged_recent == m.unmanaged
+    assert m.optimistic + m.unmanaged_recent == m.process
+
+
+def leaking(out_mib, leak_mib, sleep_time):
+    if leak_mib:
+        global __test_leak
+        __test_leak = "x" * (leak_mib * 2 ** 20)
+    out = "x" * (out_mib * 2 ** 20)
+    sleep(sleep_time)
+    return out
+
+
+def clear_leak():
+    global __test_leak
+    del __test_leak
+    gc.collect()
+
+
+def assert_memory(scheduler_or_workerstate, attr: str, min_, max_, timeout=10):
+    t0 = time()
+    while True:
+        minfo = scheduler_or_workerstate.memory
+        nbytes = getattr(minfo, attr)
+        if min_ * 2 ** 20 <= nbytes <= max_ * 2 ** 20:
+            return
+        if time() - t0 > timeout:
+            raise TimeoutError(
+                f"Expected {min_} MiB <= {attr} <= {max_} MiB; got:\n{minfo!r}"
+            )
+        sleep(0.1)
+
+
+@pytest.mark.slow
+def test_memory():
+    pytest.importorskip("zict")
+
+    with Client(n_workers=2, threads_per_worker=1, memory_limit=500 * 2 ** 20) as c:
+        c.wait_for_workers(2)
+        s = c.cluster.scheduler
+        a, b = s.workers.values()
+
+        s_m0 = s.memory
+        assert s_m0.process == a.memory.process + b.memory.process
+        assert s_m0.managed == 0
+        assert a.memory.managed == 0
+        assert b.memory.managed == 0
+        # When a worker first goes online, its RAM is immediately counted as
+        # unmanaged_old
+        assert_memory(s, "unmanaged_recent", 0, 40, timeout=0)
+        assert_memory(a, "unmanaged_recent", 0, 20, timeout=0)
+        assert_memory(b, "unmanaged_recent", 0, 20, timeout=0)
+
+        f1 = c.submit(leaking, 100, 50, 5, pure=False, workers=[a.name])
+        f2 = c.submit(leaking, 100, 50, 5, pure=False, workers=[b.name])
+        assert_memory(s, "unmanaged_recent", 300, 380)
+        assert_memory(a, "unmanaged_recent", 150, 190)
+        assert_memory(b, "unmanaged_recent", 150, 190)
+        c.gather([f1, f2])
+
+        # On each worker, we now have 100 MiB managed + 50 MiB fresh leak
+        assert_memory(s, "managed_in_memory", 200, 201)
+        assert_memory(a, "managed_in_memory", 100, 101)
+        assert_memory(b, "managed_in_memory", 100, 101)
+        assert_memory(s, "unmanaged_recent", 100, 180)
+        assert_memory(a, "unmanaged_recent", 50, 90)
+        assert_memory(b, "unmanaged_recent", 50, 90)
+
+        # Force the output of f1 and f2 to spill to disk.
+        # With target=0.6 and memory_limit=500 MiB, we'll start spilling at 300 MiB
+        # process memory per worker, or roughly after 3~7 rounds of the below depending
+        # on how much RAM the interpreter is using.
+        more_futs = []
+        for _ in range(8):
+            if s.memory.managed_spilled > 0:
+                break
+            more_futs += [
+                c.submit(leaking, 20, 0, 0, pure=False, workers=[a.name]),
+                c.submit(leaking, 20, 0, 0, pure=False, workers=[b.name]),
+            ]
+            sleep(2)
+        assert_memory(s, "managed_spilled", 1, 999)
+
+        # Delete spilled keys
+        prev = s.memory
+        del f1
+        del f2
+        assert_memory(s, "managed_spilled", 0, prev.managed_spilled / 2 ** 20 - 1)
+        assert s.memory.managed_in_memory == prev.managed_in_memory
+
+        # Empty the cluster, with the exception of leaked memory
+        del more_futs
+        assert_memory(s, "managed", 0, 0)
+
+        # Wait until 30s have passed since the spill to observe unmanaged_recent
+        # transition into unmanaged_old
+        c.run(gc.collect)
+        orig_unmanaged = s_m0.unmanaged / 2 ** 20
+        orig_old = s_m0.unmanaged_old / 2 ** 20
+        assert_memory(s, "unmanaged_old", orig_old + 90, orig_old + 190, timeout=40)
+        assert_memory(s, "unmanaged_recent", 0, 90, timeout=40)
+
+        # When the leaked memory is cleared, unmanaged and unmanaged_old drop
+        # This doesn't happen on MacOS, where the process memory of the Python
+        # interpreter does not shrink (or takes much longer to shrink)
+        if not MACOS:
+            c.run(clear_leak)
+            assert_memory(s, "unmanaged", 0, orig_unmanaged + 95)
+            assert_memory(s, "unmanaged_old", 0, orig_old + 95)
+            assert_memory(s, "unmanaged_recent", 0, 95)
+
+
+@gen_cluster(client=True, worker_kwargs={"memory_limit": 0})
+async def test_memory_no_zict(c, s, a, b):
+    """When Worker.data is not a SpillBuffer, test that querying managed_spilled
+    defaults to 0 and doesn't raise KeyError
+    """
+    await c.wait_for_workers(2)
+    assert isinstance(a.data, dict)
+    assert isinstance(b.data, dict)
+    f = c.submit(leaking, 10, 0, 0)
+    await f
+    assert 10 * 2 ** 20 < s.memory.managed_in_memory < 11 * 2 ** 20
+    assert s.memory.managed_spilled == 0
+
+
+@gen_cluster(nthreads=[])
+async def test_memory_no_workers(s):
+    assert s.memory.process == 0
+    assert s.memory.managed == 0
diff --git a/distributed/tests/test_spill.py b/distributed/tests/test_spill.py
new file mode 100644
index 00000000000..8735f1acbdf
--- /dev/null
+++ b/distributed/tests/test_spill.py
@@ -0,0 +1,84 @@
+import pytest
+
+pytest.importorskip("zict")
+
+from dask.sizeof import sizeof
+
+from distributed.spill import SpillBuffer
+
+
+def test_spillbuffer(tmpdir):
+    buf = SpillBuffer(str(tmpdir), target=300)
+    # Convenience aliases
+    assert buf.memory is buf.fast
+    assert buf.disk is buf.slow
+
+    assert not buf.spilled_by_key
+    assert buf.spilled_total == 0
+
+    a, b, c, d = "a" * 100, "b" * 100, "c" * 100, "d" * 100
+    s = sizeof(a)
+    # Test assumption made by this test, mostly for non CPython implementations
+    assert 100 < s < 200
+
+    buf["a"] = a
+    assert not buf.disk
+    assert not buf.spilled_by_key
+    assert buf.spilled_total == 0
+    assert buf["a"] == a
+
+    buf["b"] = b
+    assert not buf.disk
+    assert not buf.spilled_by_key
+    assert buf.spilled_total == 0
+
+    buf["c"] = c
+    assert set(buf.disk) == {"a"}
+    assert buf.spilled_by_key == {"a": s}
+    assert buf.spilled_total == s
+
+    assert buf["a"] == a
+    assert set(buf.disk) == {"b"}
+    assert buf.spilled_by_key == {"b": s}
+    assert buf.spilled_total == s
+
+    buf["d"] = d
+    assert set(buf.disk) == {"b", "c"}
+    assert buf.spilled_by_key == {"b": s, "c": s}
+    assert buf.spilled_total == s * 2
+
+    # Deleting an in-memory key does not automatically move spilled keys back to memory
+    del buf["a"]
+    assert set(buf.disk) == {"b", "c"}
+    assert buf.spilled_by_key == {"b": s, "c": s}
+    assert buf.spilled_total == s * 2
+    with pytest.raises(KeyError):
+        buf["a"]
+
+    # Deleting a spilled key updates the metadata
+    del buf["b"]
+    assert set(buf.disk) == {"c"}
+    assert buf.spilled_by_key == {"c": s}
+    assert buf.spilled_total == s
+    with pytest.raises(KeyError):
+        buf["b"]
+
+    # Updating a spilled key moves it to the top of the LRU and to memory
+    buf["c"] = c * 2
+    assert set(buf.disk) == {"d"}
+    assert buf.spilled_by_key == {"d": s}
+    assert buf.spilled_total == s
+
+    # Single key is larger than target and goes directly into slow
+    e = "e" * 500
+    slarge = sizeof(e)
+    buf["e"] = e
+    assert set(buf.disk) == {"d", "e"}
+    assert buf.spilled_by_key == {"d": s, "e": slarge}
+    assert buf.spilled_total == s + slarge
+
+    # Updating a spilled key with another larger than target updates slow directly
+    buf["d"] = "d" * 500
+    assert set(buf.disk) == {"d", "e"}
+    assert buf.spilled_by_key == {"d": slarge, "e": slarge}
+    assert buf.spilled_total == slarge * 2
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 775703ef1d3..7e33adb2d22 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -444,21 +444,16 @@ async def test_spill_to_disk(c, s):
 
     assert set(w.data) == {x.key, y.key}
     assert set(w.data.memory) == {x.key, y.key}
-    assert set(w.data.fast) == set(w.data.memory)
 
     z = c.submit(np.random.randint, 0, 255, size=500, dtype="u1", key="z")
     await wait(z)
     assert set(w.data) == {x.key, y.key, z.key}
     assert set(w.data.memory) == {y.key, z.key}
-    assert set(w.data.disk) == {x.key} or set(w.data.slow) == {x.key, y.key}
-    assert set(w.data.fast) == set(w.data.memory)
-    assert set(w.data.slow) == set(w.data.disk)
+    assert set(w.data.disk) == {x.key}
 
     await x
     assert set(w.data.memory) == {x.key, z.key}
-    assert set(w.data.disk) == {y.key} or set(w.data.slow) == {x.key, y.key}
-    assert set(w.data.fast) == set(w.data.memory)
-    assert set(w.data.slow) == set(w.data.disk)
+    assert set(w.data.disk) == {y.key}
     await w.close()
 
 
@@ -524,7 +519,6 @@ async def test_spill_by_default(c, s, w):
     y = c.persist(x)
     await wait(y)
     assert len(w.data.disk)  # something is on disk
-    del x, y
 
 
 @gen_cluster(nthreads=[("127.0.0.1", 1)], worker_kwargs={"reconnect": False})
diff --git a/distributed/worker.py b/distributed/worker.py
index 071efa8bcd9..821b3084a86 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -46,7 +46,7 @@
 from .metrics import time
 from .node import ServerNode
 from .proctitle import setproctitle
-from .protocol import deserialize_bytes, pickle, serialize_bytelist, to_serialize
+from .protocol import pickle, to_serialize
 from .pubsub import PubSubWorkerExtension
 from .security import Security
 from .sizeof import safe_sizeof as sizeof
@@ -609,25 +609,16 @@ def __init__(
         elif self.memory_limit and (
             self.memory_target_fraction or self.memory_spill_fraction
         ):
-            try:
-                from zict import Buffer, File, Func
-            except ImportError:
-                raise ImportError(
-                    "Please `python -m pip install zict` for spill-to-disk workers"
+            from .spill import SpillBuffer
+
+            self.data = SpillBuffer(
+                os.path.join(self.local_directory, "storage"),
+                target=int(
+                    self.memory_limit
+                    * (self.memory_target_fraction or self.memory_spill_fraction)
                 )
-            path = os.path.join(self.local_directory, "storage")
-            storage = Func(
-                partial(serialize_bytelist, on_error="raise"),
-                deserialize_bytes,
-                File(path),
+                or sys.maxsize,
             )
-            target = (
-                int(float(self.memory_limit) * self.memory_target_fraction)
-                or sys.maxsize
-            )
-            self.data = Buffer({}, storage, target, weight)
-            self.data.memory = self.data.fast
-            self.data.disk = self.data.slow
         else:
             self.data = dict()
 
@@ -826,6 +817,7 @@ async def get_metrics(self):
                 "workers": dict(self.bandwidth_workers),
                 "types": keymap(typename, self.bandwidth_types),
             },
+            spilled_nbytes=getattr(self.data, "spilled_total", 0),
         )
         out.update(self.monitor.recent())
 
@@ -2879,8 +2871,8 @@ async def check_pause(memory):
         # Dump data to disk if above 70%
         if self.memory_spill_fraction and frac > self.memory_spill_fraction:
             logger.debug(
-                "Worker is at %d%% memory usage. Start spilling data to disk.",
-                int(frac * 100),
+                "Worker is at %.0f%% memory usage. Start spilling data to disk.",
+                frac * 100,
             )
             start = time()
             target = self.memory_limit * self.memory_target_fraction
@@ -3726,10 +3718,6 @@ def convert_kwargs_to_str(kwargs, max_len=None):
         return "{{{}}}".format(", ".join(strs))
 
 
-def weight(k, v):
-    return sizeof(v)
-
-
 async def run(server, comm, function, args=(), kwargs=None, is_coro=None, wait=True):
     kwargs = kwargs or {}
     function = pickle.loads(function)
diff --git a/docs/source/http_services.rst b/docs/source/http_services.rst
index d94485319a9..5f8dbc785fe 100644
--- a/docs/source/http_services.rst
+++ b/docs/source/http_services.rst
@@ -57,12 +57,16 @@ Individual bokeh plots
 - ``/individual-profile``
 - ``/individual-profile-server``
 - ``/individual-nbytes``
+- ``/individual-nbytes-cluster``
 - ``/individual-cpu``
 - ``/individual-nprocessing``
 - ``/individual-workers``
 - ``/individual-bandwidth-types``
 - ``/individual-bandwidth-workers``
 - ``/individual-memory-by-key``
+- ``/individual-compute-time-per-key``
+- ``/individual-aggregate-time-per-action``
+- ``/individual-scheduler-system``
 - ``/individual-gpu-memory`` (GPU only)
 - ``/individual-gpu-utilization`` (GPU only)
 

From e4b534af5722a7aebb8d5b086081984e93e1784f Mon Sep 17 00:00:00 2001
From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com>
Date: Tue, 20 Apr 2021 14:24:05 -0400
Subject: [PATCH 1239/1550] Add descriptions for UCX config options (#4683)

---
 distributed/distributed-schema.yaml | 63 ++++++++++++++++-------------
 distributed/distributed.yaml        | 12 +++---
 2 files changed, 41 insertions(+), 34 deletions(-)

diff --git a/distributed/distributed-schema.yaml b/distributed/distributed-schema.yaml
index a809e03ad7a..22358fcf0e0 100644
--- a/distributed/distributed-schema.yaml
+++ b/distributed/distributed-schema.yaml
@@ -803,44 +803,51 @@ properties:
   rmm:
     type: object
     description: |
-      Configuration options for the RAPIDS Memory Manager
+      Configuration options for the RAPIDS Memory Manager.
     properties:
       pool-size:
-        type:
-        - integer
-        - "null"
-        description:
-          The size of the memory pool in bytes
+        type: [integer, 'null']
+        description: |
+          The size of the memory pool in bytes.
   ucx:
     type: object
     description: |
-      UCX provides access to other network interconnects like Infiniband and NVLINK
+      UCX provides access to other transport methods including NVLink and InfiniBand.
     properties:
+      cuda_copy:
+        type: boolean
+        description: |
+          Set environment variables to enable CUDA support over UCX. This may be used even if
+          InfiniBand and NVLink are not supported or disabled, then transferring data over TCP.
       tcp:
-        type:
-        - boolean
-        - "null"
+        type: boolean
+        description: |
+          Set environment variables to enable TCP over UCX, even if InfiniBand and NVLink
+          are not supported or disabled.
       nvlink:
-        type:
-        - boolean
-        - "null"
+        type: boolean
+        description: |
+          Set environment variables to enable UCX over NVLink, implies ``ucx.tcp=True``.
       infiniband:
-        type:
-        - boolean
-        - "null"
+        type: boolean
+        description: |
+          Set environment variables to enable UCX over InfiniBand, implies ``ucx.tcp=True``.
       rdmacm:
-        type:
-        - boolean
-        - "null"
-      cuda_copy:
-        type:
-        - boolean
-        - "null"
+        type: boolean
+        description: |
+          Set environment variables to enable UCX RDMA connection manager support,
+          requires ``ucx.infiniband=True``.
       net-devices:
-        type:
-        - string
-        - "null"
-        description: Define which Infiniband device to use
+        type: [string, 'null']
+        description: |
+          Interface(s) used by workers for UCX communication. Can be a string (like
+          ``"eth0"`` for NVLink or ``"mlx5_0:1"``/``"ib0"`` for InfiniBand), ``"auto"``
+          (requires ``ucx.infiniband=True``) to pick the optimal interface per-worker based on
+          the system's topology, or ``None`` to stay with the default value of ``"all"`` (use
+          all available interfaces). Setting to ``"auto"`` requires UCX-Py to be installed
+          and compiled with hwloc support. Unexpected errors can occur when using
+          ``"auto"`` if any interfaces are disconnected or improperly configured.
       reuse-endpoints:
         type: boolean
-        description: Whether to reuse endpoints or not, default True
+        description: | 
+          Whether to reuse endpoints or not.
diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index 08aed7f9dae..156a61abe33 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -188,10 +188,10 @@ distributed:
 rmm:
   pool-size: null
 ucx:
-  tcp: null  # enable tcp
-  nvlink: null  # enable cuda_ipc
-  infiniband: null # enable Infiniband
-  rdmacm: null # enable RDMACM
-  cuda_copy: null  # enable cuda-copy
-  net-devices: null  # define which Infiniband device to use
+  cuda_copy: False  # enable cuda-copy
+  tcp: False  # enable tcp
+  nvlink: False  # enable cuda_ipc
+  infiniband: False # enable Infiniband
+  rdmacm: False # enable RDMACM
+  net-devices: null  # define what interface to use for UCX comm
   reuse-endpoints: True  # enable endpoint reuse

From f492aa7e7a39270aaf0b58e37ed855c149b68a8f Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Wed, 21 Apr 2021 16:00:10 +0100
Subject: [PATCH 1240/1550] Fix regressions in #4651 (#4719)

---
 .github/workflows/cancel.yml        |  2 +-
 distributed/scheduler.py            | 11 ++++++++--
 distributed/tests/test_scheduler.py | 31 +++++++++++++++++++++--------
 3 files changed, 33 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/cancel.yml b/.github/workflows/cancel.yml
index 2f079bac01d..8161ba1808d 100644
--- a/.github/workflows/cancel.yml
+++ b/.github/workflows/cancel.yml
@@ -2,7 +2,7 @@ name: Cancel
 
 on:
   workflow_run:
-    workflows: ["Tests"]
+    workflows: [Tests]
     types:
       - requested
 
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 78062310eb6..58d1ccc4965 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -623,7 +623,9 @@ def metrics(self):
     @property
     def memory(self) -> MemoryState:
         return MemoryState(
-            process=self._metrics["memory"],
+            # metrics["memory"] is None if the worker sent a heartbeat before its
+            # SystemMonitor ever had a chance to run
+            process=self._metrics["memory"] or 0,
             managed=self._nbytes,
             managed_spilled=self._metrics["spilled_nbytes"],
             unmanaged_old=self._memory_unmanaged_old,
@@ -3833,7 +3835,12 @@ def heartbeat_worker(
             if size == memory_unmanaged_old:
                 memory_unmanaged_old = 0  # recalculate min()
 
-        size = max(0, metrics["memory"] - ws._nbytes + ws._metrics["spilled_nbytes"])
+        # metrics["memory"] is None if the worker sent a heartbeat before its
+        # SystemMonitor ever had a chance to run.
+        # ws._nbytes is updated at a different time and sizeof() may not be accurate,
+        # so size may be (temporarily) negative; floor it to zero.
+        size = max(0, (metrics["memory"] or 0) - ws._nbytes + metrics["spilled_nbytes"])
+
         ws._memory_other_history.append((local_now, size))
         if not memory_unmanaged_old:
             # The worker has just been started or the previous minimum has been expunged
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 98b175f5946..ff2534adeed 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -2330,6 +2330,10 @@ def assert_memory(scheduler_or_workerstate, attr: str, min_, max_, timeout=10):
         sleep(0.1)
 
 
+# This test is heavily influenced by hard-to-control factors such as memory management
+# by the Python interpreter and the OS, so it occasionally glitches
+@pytest.mark.flaky(reruns=3, reruns_delay=5)
+# ~33s runtime, or distributed.memory.recent_to_old_time + 3s
 @pytest.mark.slow
 def test_memory():
     pytest.importorskip("zict")
@@ -2379,34 +2383,45 @@ def test_memory():
             ]
             sleep(2)
         assert_memory(s, "managed_spilled", 1, 999)
+        # Wait for the spilling to finish. Note that this does not make the test take
+        # longer as we're waiting for recent_to_old_time anyway.
+        sleep(10)
 
         # Delete spilled keys
         prev = s.memory
         del f1
         del f2
-        assert_memory(s, "managed_spilled", 0, prev.managed_spilled / 2 ** 20 - 1)
-        assert s.memory.managed_in_memory == prev.managed_in_memory
+        assert_memory(s, "managed_spilled", 0, prev.managed_spilled / 2 ** 20 - 19)
 
         # Empty the cluster, with the exception of leaked memory
         del more_futs
         assert_memory(s, "managed", 0, 0)
 
+        orig_unmanaged = s_m0.unmanaged / 2 ** 20
+        orig_old = s_m0.unmanaged_old / 2 ** 20
+
         # Wait until 30s have passed since the spill to observe unmanaged_recent
         # transition into unmanaged_old
         c.run(gc.collect)
-        orig_unmanaged = s_m0.unmanaged / 2 ** 20
-        orig_old = s_m0.unmanaged_old / 2 ** 20
-        assert_memory(s, "unmanaged_old", orig_old + 90, orig_old + 190, timeout=40)
         assert_memory(s, "unmanaged_recent", 0, 90, timeout=40)
+        assert_memory(
+            s,
+            "unmanaged_old",
+            orig_old + 90,
+            # On MacOS, the process memory of the Python interpreter does not shrink as
+            # fast as on Linux/Windows
+            9999 if MACOS else orig_old + 190,
+            timeout=40,
+        )
 
         # When the leaked memory is cleared, unmanaged and unmanaged_old drop
-        # This doesn't happen on MacOS, where the process memory of the Python
-        # interpreter does not shrink (or takes much longer to shrink)
+        # On MacOS, the process memory of the Python interpreter does not shrink as fast
+        # as on Linux/Windows
         if not MACOS:
             c.run(clear_leak)
             assert_memory(s, "unmanaged", 0, orig_unmanaged + 95)
             assert_memory(s, "unmanaged_old", 0, orig_old + 95)
-            assert_memory(s, "unmanaged_recent", 0, 95)
+            assert_memory(s, "unmanaged_recent", 0, 90)
 
 
 @gen_cluster(client=True, worker_kwargs={"memory_limit": 0})

From 9fde83735a1f354107b26b03b62355c4f727d18c Mon Sep 17 00:00:00 2001
From: Gil Forsyth <gforsyth@users.noreply.github.com>
Date: Thu, 22 Apr 2021 04:18:49 -0400
Subject: [PATCH 1241/1550] Add "ready->memory" to transitions in worker
 (#4728)

* Add "ready->memory" to transitions in worker

This should remove the error message in #4721 but is more of a band-aid
than a fix.  Ready->memory transitions shouldn't happen, but since they
do occassionally crop up, we might as well dispatch them appropriately
until the worker state machine is ready.

* Default to `no_value`

* Update distributed/worker.py

Co-authored-by: James Bourbeau <jrbourbeau@users.noreply.github.com>

Co-authored-by: James Bourbeau <jrbourbeau@users.noreply.github.com>
---
 distributed/worker.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/distributed/worker.py b/distributed/worker.py
index 821b3084a86..167608eecb4 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -466,6 +466,8 @@ def __init__(
             ("executing", "memory"): self.transition_executing_done,
             ("flight", "memory"): self.transition_flight_memory,
             ("flight", "fetch"): self.transition_flight_fetch,
+            # Shouldn't be a valid transition but happens nonetheless
+            ("ready", "memory"): self.transition_ready_memory,
             # Scheduler intercession (re-assignment)
             ("fetch", "waiting"): self.transition_fetch_waiting,
             ("flight", "waiting"): self.transition_flight_waiting,
@@ -1837,8 +1839,8 @@ def transition_ready_error(self, ts):
             assert ts.traceback is not None
         self.send_task_state_to_scheduler(ts)
 
-    def transition_ready_memory(self, ts, value=None):
-        if value:
+    def transition_ready_memory(self, ts, value=no_value):
+        if value is not no_value:
             self.put_key_in_memory(ts, value=value)
         self.send_task_state_to_scheduler(ts)
 

From 4a883a7eef04130cb8c203b966837f3cf6c0d1d7 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Thu, 22 Apr 2021 03:21:08 -0500
Subject: [PATCH 1242/1550] Reintroduce `weight` function (#4723)

* Reintroduce weight function

* Remove newline

* Add deprecation cycle
---
 distributed/tests/test_utils.py  | 18 ++++++++++++++++++
 distributed/tests/test_worker.py |  7 ++++++-
 distributed/utils.py             | 28 ++++++++++++++++++++++++++++
 distributed/worker.py            |  6 ++++++
 4 files changed, 58 insertions(+), 1 deletion(-)

diff --git a/distributed/tests/test_utils.py b/distributed/tests/test_utils.py
index 669368c70fd..6d9ab29520b 100644
--- a/distributed/tests/test_utils.py
+++ b/distributed/tests/test_utils.py
@@ -24,6 +24,7 @@
     LoopRunner,
     TimeoutError,
     _maybe_complex,
+    deprecated,
     ensure_bytes,
     ensure_ip,
     format_dashboard_link,
@@ -617,3 +618,20 @@ def test_lru():
 async def test_offload():
     assert (await offload(inc, 1)) == 2
     assert (await offload(lambda x, y: x + y, 1, y=2)) == 3
+
+
+def test_deprecated():
+    @deprecated()
+    def foo():
+        return "bar"
+
+    with pytest.warns(DeprecationWarning, match="foo is deprecated"):
+        assert foo() == "bar"
+
+    # Explicit version specified
+    @deprecated(version_removed="1.2.3")
+    def foo():
+        return "bar"
+
+    with pytest.warns(DeprecationWarning, match="removed in version 1.2.3"):
+        assert foo() == "bar"
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 7e33adb2d22..4681c5c36de 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -54,7 +54,7 @@
     s,
     slowinc,
 )
-from distributed.worker import Worker, error_message, logger, parse_memory_limit
+from distributed.worker import Worker, error_message, logger, parse_memory_limit, weight
 
 
 @pytest.mark.asyncio
@@ -1787,3 +1787,8 @@ async def test_story(c, s, w):
     ts = w.tasks[future.key]
     assert ts.state in str(w.story(ts))
     assert w.story(ts) == w.story(ts.key)
+
+
+def test_weight_deprecated():
+    with pytest.warns(DeprecationWarning):
+        weight("foo", "bar")
diff --git a/distributed/utils.py b/distributed/utils.py
index 00c4dab50dc..fc4a73672fc 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -1073,6 +1073,34 @@ def time_warn(duration, text):
         print("TIME WARNING", text, end - start)
 
 
+def deprecated(*, version_removed: str = None):
+    """Decorator to mark a function as deprecated
+
+    Parameters
+    ----------
+    version_removed : str, optional
+        If specified, include the version in which the deprecated function
+        will be removed. Defaults to "a future release".
+    """
+
+    def decorator(func):
+        nonlocal version_removed
+        msg = f"{funcname(func)} is deprecated and will be removed in"
+        if version_removed is not None:
+            msg += f" version {version_removed}"
+        else:
+            msg += " a future release"
+
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            warnings.warn(msg, DeprecationWarning, stacklevel=2)
+            return func(*args, **kwargs)
+
+        return wrapper
+
+    return decorator
+
+
 def json_load_robust(fn, load=json.load):
     """ Reads a JSON file from disk that may be being written as we read """
     while not os.path.exists(fn):
diff --git a/distributed/worker.py b/distributed/worker.py
index 167608eecb4..1acd395fc12 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -56,6 +56,7 @@
     LRU,
     TimeoutError,
     _maybe_complex,
+    deprecated,
     get_ip,
     has_arg,
     import_file,
@@ -3720,6 +3721,11 @@ def convert_kwargs_to_str(kwargs, max_len=None):
         return "{{{}}}".format(", ".join(strs))
 
 
+@deprecated(version_removed="2021.06.0")
+def weight(k, v):
+    return sizeof(v)
+
+
 async def run(server, comm, function, args=(), kwargs=None, is_coro=None, wait=True):
     kwargs = kwargs or {}
     function = pickle.loads(function)

From 42e3f22ef1d9d3dcb494bd4e546f9ca6f2d54891 Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Thu, 22 Apr 2021 16:33:35 +0100
Subject: [PATCH 1243/1550] Enable configuration of prometheus metrics
 namespace (#4722)

* Enable configuration of prometheus metrics namespace

* Undo editor formatting

* Refactor into base class and remove private function import

* Rename semaphore subsystem

* Remove serving metric and rename connections metric
---
 distributed/distributed-schema.yaml           |   7 ++
 distributed/distributed.yaml                  |   2 +
 distributed/http/prometheus.py                |  17 +++
 .../http/scheduler/prometheus/__init__.py     | 104 +-----------------
 distributed/http/scheduler/prometheus/core.py | 104 ++++++++++++++++++
 .../http/scheduler/prometheus/semaphore.py    |  22 ++--
 .../scheduler/tests/test_semaphore_http.py    |  46 ++++----
 .../http/worker/prometheus/__init__.py        |   3 +
 .../{prometheus.py => prometheus/core.py}     |  52 ++++-----
 9 files changed, 199 insertions(+), 158 deletions(-)
 create mode 100644 distributed/http/prometheus.py
 create mode 100644 distributed/http/scheduler/prometheus/core.py
 create mode 100644 distributed/http/worker/prometheus/__init__.py
 rename distributed/http/worker/{prometheus.py => prometheus/core.py} (57%)

diff --git a/distributed/distributed-schema.yaml b/distributed/distributed-schema.yaml
index 22358fcf0e0..0e8faf2f19d 100644
--- a/distributed/distributed-schema.yaml
+++ b/distributed/distributed-schema.yaml
@@ -728,6 +728,13 @@ properties:
           export-tool:
             type: boolean
 
+          prometheus:
+            type: object
+            properties:
+              namespace:
+                type: string
+                description: Namespace prefix to use for all prometheus metrics.
+
       admin:
         type: object
         description: |
diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index 156a61abe33..c71845e3769 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -168,6 +168,8 @@ distributed:
     link: "{scheme}://{host}:{port}/status"
     export-tool: False
     graph-max-items: 5000  # maximum number of tasks to try to plot in graph view
+    prometheus:
+      namespace: "dask"
 
   ##################
   # Administrative #
diff --git a/distributed/http/prometheus.py b/distributed/http/prometheus.py
new file mode 100644
index 00000000000..254a8f95911
--- /dev/null
+++ b/distributed/http/prometheus.py
@@ -0,0 +1,17 @@
+import dask.config
+
+
+class PrometheusCollector:
+    def __init__(self, server):
+        self.server = server
+        self.namespace = dask.config.get("distributed.dashboard.prometheus.namespace")
+        self.subsystem = None
+
+    def build_name(self, name):
+        full_name = []
+        if self.namespace:
+            full_name.append(self.namespace)
+        if self.subsystem:
+            full_name.append(self.subsystem)
+        full_name.append(name)
+        return "_".join(full_name)
diff --git a/distributed/http/scheduler/prometheus/__init__.py b/distributed/http/scheduler/prometheus/__init__.py
index 120a01dab58..63c0310d0aa 100644
--- a/distributed/http/scheduler/prometheus/__init__.py
+++ b/distributed/http/scheduler/prometheus/__init__.py
@@ -1,105 +1,3 @@
-import toolz
-
-from distributed.http.utils import RequestHandler
-from distributed.scheduler import ALL_TASK_STATES
-
-from .semaphore import SemaphoreMetricExtension
-
-
-class _PrometheusCollector:
-    def __init__(self, dask_server):
-        self.server = dask_server
-
-    def collect(self):
-        from prometheus_client.core import CounterMetricFamily, GaugeMetricFamily
-
-        yield GaugeMetricFamily(
-            "dask_scheduler_clients",
-            "Number of clients connected.",
-            value=len([k for k in self.server.clients if k != "fire-and-forget"]),
-        )
-
-        yield GaugeMetricFamily(
-            "dask_scheduler_desired_workers",
-            "Number of workers scheduler needs for task graph.",
-            value=self.server.adaptive_target(),
-        )
-
-        worker_states = GaugeMetricFamily(
-            "dask_scheduler_workers",
-            "Number of workers known by scheduler.",
-            labels=["state"],
-        )
-        worker_states.add_metric(["connected"], len(self.server.workers))
-        worker_states.add_metric(["saturated"], len(self.server.saturated))
-        worker_states.add_metric(["idle"], len(self.server.idle))
-        yield worker_states
-
-        tasks = GaugeMetricFamily(
-            "dask_scheduler_tasks",
-            "Number of tasks known by scheduler.",
-            labels=["state"],
-        )
-
-        task_counter = toolz.merge_with(
-            sum, (tp.states for tp in self.server.task_prefixes.values())
-        )
-
-        suspicious_tasks = CounterMetricFamily(
-            "dask_scheduler_tasks_suspicious",
-            "Total number of times a task has been marked suspicious",
-            labels=["task_prefix_name"],
-        )
-
-        for tp in self.server.task_prefixes.values():
-            suspicious_tasks.add_metric([tp.name], tp.suspicious)
-        yield suspicious_tasks
-
-        yield CounterMetricFamily(
-            "dask_scheduler_tasks_forgotten",
-            (
-                "Total number of processed tasks no longer in memory and already "
-                "removed from the scheduler job queue. Note task groups on the "
-                "scheduler which have all tasks in the forgotten state are not included."
-            ),
-            value=task_counter.get("forgotten", 0.0),
-        )
-
-        for state in ALL_TASK_STATES:
-            tasks.add_metric([state], task_counter.get(state, 0.0))
-        yield tasks
-
-
-COLLECTORS = [_PrometheusCollector, SemaphoreMetricExtension]
-
-
-class PrometheusHandler(RequestHandler):
-    _collectors = None
-
-    def __init__(self, *args, dask_server=None, **kwargs):
-        import prometheus_client
-
-        super().__init__(*args, dask_server=dask_server, **kwargs)
-
-        if PrometheusHandler._collectors:
-            # Especially during testing, multiple schedulers are started
-            # sequentially in the same python process
-            for _collector in PrometheusHandler._collectors:
-                _collector.server = self.server
-            return
-
-        PrometheusHandler._collectors = tuple(
-            collector(self.server) for collector in COLLECTORS
-        )
-        # Register collectors
-        for instantiated_collector in PrometheusHandler._collectors:
-            prometheus_client.REGISTRY.register(instantiated_collector)
-
-    def get(self):
-        import prometheus_client
-
-        self.write(prometheus_client.generate_latest())
-        self.set_header("Content-Type", "text/plain; version=0.0.4")
-
+from .core import PrometheusHandler
 
 routes = [("/metrics", PrometheusHandler, {})]
diff --git a/distributed/http/scheduler/prometheus/core.py b/distributed/http/scheduler/prometheus/core.py
new file mode 100644
index 00000000000..2a274940781
--- /dev/null
+++ b/distributed/http/scheduler/prometheus/core.py
@@ -0,0 +1,104 @@
+import toolz
+
+from distributed.http.prometheus import PrometheusCollector
+from distributed.http.utils import RequestHandler
+from distributed.scheduler import ALL_TASK_STATES
+
+from .semaphore import SemaphoreMetricCollector
+
+
+class SchedulerMetricCollector(PrometheusCollector):
+    def __init__(self, server):
+        super().__init__(server)
+        self.subsystem = "scheduler"
+
+    def collect(self):
+        from prometheus_client.core import CounterMetricFamily, GaugeMetricFamily
+
+        yield GaugeMetricFamily(
+            self.build_name("clients"),
+            "Number of clients connected.",
+            value=len([k for k in self.server.clients if k != "fire-and-forget"]),
+        )
+
+        yield GaugeMetricFamily(
+            self.build_name("desired_workers"),
+            "Number of workers scheduler needs for task graph.",
+            value=self.server.adaptive_target(),
+        )
+
+        worker_states = GaugeMetricFamily(
+            self.build_name("workers"),
+            "Number of workers known by scheduler.",
+            labels=["state"],
+        )
+        worker_states.add_metric(["connected"], len(self.server.workers))
+        worker_states.add_metric(["saturated"], len(self.server.saturated))
+        worker_states.add_metric(["idle"], len(self.server.idle))
+        yield worker_states
+
+        tasks = GaugeMetricFamily(
+            self.build_name("tasks"),
+            "Number of tasks known by scheduler.",
+            labels=["state"],
+        )
+
+        task_counter = toolz.merge_with(
+            sum, (tp.states for tp in self.server.task_prefixes.values())
+        )
+
+        suspicious_tasks = CounterMetricFamily(
+            self.build_name("tasks_suspicious"),
+            "Total number of times a task has been marked suspicious",
+            labels=["task_prefix_name"],
+        )
+
+        for tp in self.server.task_prefixes.values():
+            suspicious_tasks.add_metric([tp.name], tp.suspicious)
+        yield suspicious_tasks
+
+        yield CounterMetricFamily(
+            self.build_name("tasks_forgotten"),
+            (
+                "Total number of processed tasks no longer in memory and already "
+                "removed from the scheduler job queue. Note task groups on the "
+                "scheduler which have all tasks in the forgotten state are not included."
+            ),
+            value=task_counter.get("forgotten", 0.0),
+        )
+
+        for state in ALL_TASK_STATES:
+            tasks.add_metric([state], task_counter.get(state, 0.0))
+        yield tasks
+
+
+COLLECTORS = [SchedulerMetricCollector, SemaphoreMetricCollector]
+
+
+class PrometheusHandler(RequestHandler):
+    _collectors = None
+
+    def __init__(self, *args, dask_server=None, **kwargs):
+        import prometheus_client
+
+        super().__init__(*args, dask_server=dask_server, **kwargs)
+
+        if PrometheusHandler._collectors:
+            # Especially during testing, multiple schedulers are started
+            # sequentially in the same python process
+            for _collector in PrometheusHandler._collectors:
+                _collector.server = self.server
+            return
+
+        PrometheusHandler._collectors = tuple(
+            collector(self.server) for collector in COLLECTORS
+        )
+        # Register collectors
+        for instantiated_collector in PrometheusHandler._collectors:
+            prometheus_client.REGISTRY.register(instantiated_collector)
+
+    def get(self):
+        import prometheus_client
+
+        self.write(prometheus_client.generate_latest())
+        self.set_header("Content-Type", "text/plain; version=0.0.4")
diff --git a/distributed/http/scheduler/prometheus/semaphore.py b/distributed/http/scheduler/prometheus/semaphore.py
index aac467b66cc..36defbbeeae 100644
--- a/distributed/http/scheduler/prometheus/semaphore.py
+++ b/distributed/http/scheduler/prometheus/semaphore.py
@@ -1,6 +1,10 @@
-class SemaphoreMetricExtension:
-    def __init__(self, dask_server):
-        self.server = dask_server
+from distributed.http.prometheus import PrometheusCollector
+
+
+class SemaphoreMetricCollector(PrometheusCollector):
+    def __init__(self, server):
+        super().__init__(server)
+        self.subsystem = "semaphore"
 
     def collect(self):
         from prometheus_client.core import CounterMetricFamily, GaugeMetricFamily
@@ -8,29 +12,29 @@ def collect(self):
         sem_ext = self.server.extensions["semaphores"]
 
         semaphore_max_leases_family = GaugeMetricFamily(
-            "semaphore_max_leases",
+            self.build_name("max_leases"),
             "Maximum leases allowed per semaphore, this will be constant for each semaphore during its lifetime.",
             labels=["name"],
         )
         semaphore_active_leases_family = GaugeMetricFamily(
-            "semaphore_active_leases",
+            self.build_name("active_leases"),
             "Amount of currently active leases per semaphore.",
             labels=["name"],
         )
         semaphore_pending_leases = GaugeMetricFamily(
-            "semaphore_pending_leases",
+            self.build_name("pending_leases"),
             "Amount of currently pending leases per semaphore.",
             labels=["name"],
         )
 
         semaphore_acquire_total = CounterMetricFamily(
-            "semaphore_acquire_total",
+            self.build_name("acquire_total"),
             "Total number of leases acquired per semaphore.",
             labels=["name"],
         )
 
         semaphore_release_total = CounterMetricFamily(
-            "semaphore_release_total",
+            self.build_name("release_total"),
             "Total number of leases released per semaphore.\n"
             "Note: if a semaphore is closed while there are still leases active, this count will not equal "
             "`semaphore_acquired_total` after execution.",
@@ -38,7 +42,7 @@ def collect(self):
         )
 
         semaphore_average_pending_lease_time = GaugeMetricFamily(
-            "semaphore_average_pending_lease_time",
+            self.build_name("average_pending_lease_time"),
             "Exponential moving average of the time it took to acquire a lease per semaphore.\n"
             "Note: this only includes time spent on scheduler side, "
             "it does"
diff --git a/distributed/http/scheduler/tests/test_semaphore_http.py b/distributed/http/scheduler/tests/test_semaphore_http.py
index 21996cb35f4..67842e141fb 100644
--- a/distributed/http/scheduler/tests/test_semaphore_http.py
+++ b/distributed/http/scheduler/tests/test_semaphore_http.py
@@ -19,19 +19,19 @@ async def fetch_metrics():
         families = {
             family.name: family
             for family in text_string_to_metric_families(txt)
-            if family.name.startswith("semaphore_")
+            if family.name.startswith("dask_semaphore_")
         }
         return families
 
     active_metrics = await fetch_metrics()
 
     expected_metrics = {
-        "semaphore_max_leases",
-        "semaphore_active_leases",
-        "semaphore_pending_leases",
-        "semaphore_acquire",
-        "semaphore_release",
-        "semaphore_average_pending_lease_time_s",
+        "dask_semaphore_max_leases",
+        "dask_semaphore_active_leases",
+        "dask_semaphore_pending_leases",
+        "dask_semaphore_acquire",
+        "dask_semaphore_release",
+        "dask_semaphore_average_pending_lease_time_s",
     }
 
     assert active_metrics.keys() == expected_metrics
@@ -48,28 +48,34 @@ async def fetch_metrics():
         assert len(samples) == 1
         sample = samples.pop()
         assert sample.labels["name"] == "test"
-        if name == "semaphore_max_leases":
+        if name == "dask_semaphore_max_leases":
             assert sample.value == 2
         else:
             assert sample.value == 0
 
     assert await sem.acquire()
     active_metrics = await fetch_metrics()
-    assert active_metrics["semaphore_max_leases"].samples[0].value == 2
-    assert active_metrics["semaphore_active_leases"].samples[0].value == 1
-    assert active_metrics["semaphore_average_pending_lease_time_s"].samples[0].value > 0
-    assert active_metrics["semaphore_acquire"].samples[0].value == 1
-    assert active_metrics["semaphore_release"].samples[0].value == 0
-    assert active_metrics["semaphore_pending_leases"].samples[0].value == 0
+    assert active_metrics["dask_semaphore_max_leases"].samples[0].value == 2
+    assert active_metrics["dask_semaphore_active_leases"].samples[0].value == 1
+    assert (
+        active_metrics["dask_semaphore_average_pending_lease_time_s"].samples[0].value
+        > 0
+    )
+    assert active_metrics["dask_semaphore_acquire"].samples[0].value == 1
+    assert active_metrics["dask_semaphore_release"].samples[0].value == 0
+    assert active_metrics["dask_semaphore_pending_leases"].samples[0].value == 0
 
     assert await sem.release() is True
     active_metrics = await fetch_metrics()
-    assert active_metrics["semaphore_max_leases"].samples[0].value == 2
-    assert active_metrics["semaphore_active_leases"].samples[0].value == 0
-    assert active_metrics["semaphore_average_pending_lease_time_s"].samples[0].value > 0
-    assert active_metrics["semaphore_acquire"].samples[0].value == 1
-    assert active_metrics["semaphore_release"].samples[0].value == 1
-    assert active_metrics["semaphore_pending_leases"].samples[0].value == 0
+    assert active_metrics["dask_semaphore_max_leases"].samples[0].value == 2
+    assert active_metrics["dask_semaphore_active_leases"].samples[0].value == 0
+    assert (
+        active_metrics["dask_semaphore_average_pending_lease_time_s"].samples[0].value
+        > 0
+    )
+    assert active_metrics["dask_semaphore_acquire"].samples[0].value == 1
+    assert active_metrics["dask_semaphore_release"].samples[0].value == 1
+    assert active_metrics["dask_semaphore_pending_leases"].samples[0].value == 0
 
     await sem.close()
     active_metrics = await fetch_metrics()
diff --git a/distributed/http/worker/prometheus/__init__.py b/distributed/http/worker/prometheus/__init__.py
new file mode 100644
index 00000000000..63c0310d0aa
--- /dev/null
+++ b/distributed/http/worker/prometheus/__init__.py
@@ -0,0 +1,3 @@
+from .core import PrometheusHandler
+
+routes = [("/metrics", PrometheusHandler, {})]
diff --git a/distributed/http/worker/prometheus.py b/distributed/http/worker/prometheus/core.py
similarity index 57%
rename from distributed/http/worker/prometheus.py
rename to distributed/http/worker/prometheus/core.py
index 4d0c0a55e60..cd082ea6a33 100644
--- a/distributed/http/worker/prometheus.py
+++ b/distributed/http/worker/prometheus/core.py
@@ -1,12 +1,14 @@
 import logging
 
-from ..utils import RequestHandler
+from distributed.http.prometheus import PrometheusCollector
+from distributed.http.utils import RequestHandler
 
 
-class _PrometheusCollector:
+class WorkerMetricCollector(PrometheusCollector):
     def __init__(self, server):
-        self.worker = server
+        super().__init__(server)
         self.logger = logging.getLogger("distributed.dask_worker")
+        self.subsystem = "worker"
         self.crick_available = True
         try:
             import crick  # noqa: F401
@@ -20,52 +22,53 @@ def collect(self):
         from prometheus_client.core import GaugeMetricFamily
 
         tasks = GaugeMetricFamily(
-            "dask_worker_tasks", "Number of tasks at worker.", labels=["state"]
+            self.build_name("tasks"),
+            "Number of tasks at worker.",
+            labels=["state"],
         )
-        tasks.add_metric(["stored"], len(self.worker.data))
-        tasks.add_metric(["executing"], self.worker.executing_count)
-        tasks.add_metric(["ready"], len(self.worker.ready))
-        tasks.add_metric(["waiting"], self.worker.waiting_for_data_count)
-        tasks.add_metric(["serving"], len(self.worker._comms))
+        tasks.add_metric(["stored"], len(self.server.data))
+        tasks.add_metric(["executing"], self.server.executing_count)
+        tasks.add_metric(["ready"], len(self.server.ready))
+        tasks.add_metric(["waiting"], self.server.waiting_for_data_count)
         yield tasks
 
         yield GaugeMetricFamily(
-            "dask_worker_connections",
-            "Number of task connections to other workers.",
-            value=len(self.worker.in_flight_workers),
+            self.build_name("concurrent_fetch_requests"),
+            "Number of open fetch requests to other workers.",
+            value=len(self.server.in_flight_workers),
         )
 
         yield GaugeMetricFamily(
-            "dask_worker_threads",
+            self.build_name("threads"),
             "Number of worker threads.",
-            value=self.worker.nthreads,
+            value=self.server.nthreads,
         )
 
         yield GaugeMetricFamily(
-            "dask_worker_latency_seconds",
+            self.build_name("latency_seconds"),
             "Latency of worker connection.",
-            value=self.worker.latency,
+            value=self.server.latency,
         )
 
         # all metrics using digests require crick to be installed
         # the following metrics will export NaN, if the corresponding digests are None
         if self.crick_available:
             yield GaugeMetricFamily(
-                "dask_worker_tick_duration_median_seconds",
+                self.build_name("tick_duration_median_seconds"),
                 "Median tick duration at worker.",
-                value=self.worker.digests["tick-duration"].components[1].quantile(50),
+                value=self.server.digests["tick-duration"].components[1].quantile(50),
             )
 
             yield GaugeMetricFamily(
-                "dask_worker_task_duration_median_seconds",
+                self.build_name("task_duration_median_seconds"),
                 "Median task runtime at worker.",
-                value=self.worker.digests["task-duration"].components[1].quantile(50),
+                value=self.server.digests["task-duration"].components[1].quantile(50),
             )
 
             yield GaugeMetricFamily(
-                "dask_worker_transfer_bandwidth_median_bytes",
+                self.build_name("transfer_bandwidth_median_bytes"),
                 "Bandwidth for transfer at worker in Bytes.",
-                value=self.worker.digests["transfer-bandwidth"]
+                value=self.server.digests["transfer-bandwidth"]
                 .components[1]
                 .quantile(50),
             )
@@ -82,7 +85,7 @@ def __init__(self, *args, **kwargs):
         if PrometheusHandler._initialized:
             return
 
-        prometheus_client.REGISTRY.register(_PrometheusCollector(self.server))
+        prometheus_client.REGISTRY.register(WorkerMetricCollector(self.server))
 
         PrometheusHandler._initialized = True
 
@@ -91,6 +94,3 @@ def get(self):
 
         self.write(prometheus_client.generate_latest())
         self.set_header("Content-Type", "text/plain; version=0.0.4")
-
-
-routes = [(r"metrics", PrometheusHandler, {})]

From 27b768900058c9ca9f696b48b32a58cc2492d60f Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Thu, 22 Apr 2021 21:06:40 +0100
Subject: [PATCH 1244/1550] Unit test for metrics["memory"]=None (#4727)

---
 distributed/tests/test_scheduler.py | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index ff2534adeed..5d49d841455 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -9,6 +9,7 @@
 from itertools import product
 from textwrap import dedent
 from time import sleep
+from unittest import mock
 
 import cloudpickle
 import pytest
@@ -2442,3 +2443,23 @@ async def test_memory_no_zict(c, s, a, b):
 async def test_memory_no_workers(s):
     assert s.memory.process == 0
     assert s.memory.managed == 0
+
+
+@gen_cluster(client=True, nthreads=[])
+async def test_memory_is_none(c, s):
+    """If Worker.heartbeat() runs before Worker.monitor.update(), then
+    Worker.metrics["memory"] will be None and will need special handling in
+    Worker.memory and Scheduler.heartbeat_worker().
+    """
+    with mock.patch("distributed.system_monitor.SystemMonitor.update"):
+        async with Worker(s.address, nthreads=1) as w:
+            await c.wait_for_workers(1)
+            f = await c.scatter(123)
+            await w.heartbeat()
+            assert s.memory.process == 0  # Forced from None
+            assert s.memory.managed == 0  # Capped by process even if we do have keys
+            assert s.memory.managed_in_memory == 0
+            assert s.memory.managed_spilled == 0
+            assert s.memory.unmanaged == 0
+            assert s.memory.unmanaged_old == 0
+            assert s.memory.unmanaged_recent == 0

From 9623202267230a9a30a9963047adaa9dba8036be Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Thu, 22 Apr 2021 16:01:15 -0500
Subject: [PATCH 1245/1550] Minor formatting updates for HTTP endpoints doc
 (#4736)

---
 docs/source/http_services.rst | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/docs/source/http_services.rst b/docs/source/http_services.rst
index 5f8dbc785fe..93e6549f9df 100644
--- a/docs/source/http_services.rst
+++ b/docs/source/http_services.rst
@@ -30,16 +30,15 @@ Pages and JSON endpoints served by the scheduler
 - ``/info/main/workers.html`` basic info about workers and links to their dashboards and logs
 - ``info/worker/(worker).html``: more detail about given worker, keyed by TCP address; links to tasks
 - ``info/task/(task).html``: details about a task on the cluster, by dask key; links to worker,
-            related tasks, and client
+  related tasks, and client
 - ``/info/call-stacks/(worker).html``: tasks currently handled by given worker
 - ``/info/call-stack/(task).html``: state of task (where it is running)
 - ``/info/main/logs.html``: scheduler logs
 - ``/info/logs/(worker).html``: logs of given worker
 - ``/individual-plots.json``: map of path to description for available individual
-            (i.e,. one-pane, non-dashboard) plots
+  (i.e,. one-pane, non-dashboard) plots
 - ``/eventstream``: scheduler events on a websocket
-- ``/proxy/(port)/(address)/(path): proxy to worker HTTP locations
-            (if you have jupyter-server-proxy)
+- ``/proxy/(port)/(address)/(path)``: proxy to worker HTTP locations (if you have jupyter-server-proxy)
 - ``/metrics``: prometheus endpoint
 - ``/json/counts.json``: cluster count stats
 - ``/json/identity.json``: scheduler information

From 2e446ae70f3bfc23853c2a61915f5bd0625ce7d0 Mon Sep 17 00:00:00 2001
From: Anderson Banihirwe <axbanihirwe@ualr.edu>
Date: Thu, 22 Apr 2021 15:57:19 -0600
Subject: [PATCH 1246/1550] Fix `UnboundLocalError` in `AdaptiveCore.adapt()`
 (#4731)

---
 distributed/deploy/adaptive_core.py           |  1 +
 .../deploy/tests/test_adaptive_core.py        | 22 +++++++++++++++++++
 2 files changed, 23 insertions(+)

diff --git a/distributed/deploy/adaptive_core.py b/distributed/deploy/adaptive_core.py
index 946adae0a57..4f277307b64 100644
--- a/distributed/deploy/adaptive_core.py
+++ b/distributed/deploy/adaptive_core.py
@@ -184,6 +184,7 @@ async def adapt(self) -> None:
         if self._adapting:  # Semaphore to avoid overlapping adapt calls
             return
         self._adapting = True
+        status = None
 
         try:
 
diff --git a/distributed/deploy/tests/test_adaptive_core.py b/distributed/deploy/tests/test_adaptive_core.py
index b4fc5768f82..c749d4670b7 100644
--- a/distributed/deploy/tests/test_adaptive_core.py
+++ b/distributed/deploy/tests/test_adaptive_core.py
@@ -4,6 +4,7 @@
 
 from distributed.deploy.adaptive_core import AdaptiveCore
 from distributed.metrics import time
+from distributed.utils_test import captured_logger
 
 
 class MyAdaptive(AdaptiveCore):
@@ -89,3 +90,24 @@ async def test_interval():
     adapt._target = 10
     await asyncio.sleep(0.020)
     assert len(adapt.plan) == 1  # last value from before, unchanged
+
+
+@pytest.mark.asyncio
+async def test_adapt_oserror():
+    class BadAdaptive(MyAdaptive):
+        """AdaptiveCore subclass which raises an OSError when attempting to adapt
+
+        We use this to check that error handling works properly
+        """
+
+        def safe_target(self):
+            raise OSError()
+
+    with captured_logger("distributed.deploy.adaptive_core") as log:
+        adapt = BadAdaptive(minimum=1, maximum=4)
+        await adapt.adapt()
+    text = log.getvalue()
+    assert "Adaptive stopping due to error" in text
+    assert "Adaptive stop" in text
+    assert not adapt._adapting
+    assert not adapt.periodic_callback

From fa5d9934974fa2a1a3c4b470b6ed2a6b3d4b66d5 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Fri, 23 Apr 2021 12:01:48 -0500
Subject: [PATCH 1247/1550] Avoid `active_threads` changing size during
 iteration (#4729)

---
 distributed/worker.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/distributed/worker.py b/distributed/worker.py
index 1acd395fc12..eb87d797a86 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -948,6 +948,8 @@ async def heartbeat(self):
         logger.debug("Heartbeat: %s", self.address)
         try:
             start = time()
+            with self.active_threads_lock:
+                active_keys = list(self.active_threads.values())
             response = await retry_operation(
                 self.scheduler.heartbeat_worker,
                 address=self.contact_address,
@@ -955,7 +957,7 @@ async def heartbeat(self):
                 metrics=await self.get_metrics(),
                 executing={
                     key: start - self.tasks[key].start_time
-                    for key in self.active_threads.values()
+                    for key in active_keys
                     if key in self.tasks
                 },
             )

From 4637099e2548a963197fdcc04e563401f77adef5 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Fri, 23 Apr 2021 13:49:49 -0500
Subject: [PATCH 1248/1550] bump version to 2021.04.1

---
 docs/source/changelog.rst | 39 ++++++++++++++++++++++++++++++++++++++-
 1 file changed, 38 insertions(+), 1 deletion(-)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index 96212d1de87..06a1c7cf0bc 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,42 @@
 Changelog
 =========
 
+2021.04.1
+---------
+
+Released on April 23, 2021
+
+- Avoid ``active_threads`` changing size during iteration (:pr:`4729`) `James Bourbeau`_
+- Fix ``UnboundLocalError`` in ``AdaptiveCore.adapt()`` (:pr:`4731`) `Anderson Banihirwe`_
+- Minor formatting updates for HTTP endpoints doc (:pr:`4736`) `James Bourbeau`_
+- Unit test for ``metrics["memory"]=None`` (:pr:`4727`) `crusaderky`_
+- Enable configuration of prometheus metrics namespace (:pr:`4722`) `Jacob Tomlinson`_
+- Reintroduce ``weight`` function (:pr:`4723`) `James Bourbeau`_
+- Add ``ready->memory`` to transitions in worker (:pr:`4728`) `Gil Forsyth`_
+- Fix regressions in :pr:`4651` (:pr:`4719`) `crusaderky`_
+- Add descriptions for UCX config options (:pr:`4683`) `Charles Blackmon-Luca`_
+- Split RAM measure into dask keys/other old/other new (:pr:`4651`) `crusaderky`_
+- Fix ``DeprecationWarning`` on Python 3.9 (:pr:`4717`) `George Sakkis`_
+- ipython causes ``test_profile_nested_sizeof`` crash on windows (:pr:`4713`) `crusaderky`_
+- Add ``iterate_collection`` argument to ``serialize`` (:pr:`4641`) `Richard J Zamora`_
+- When closing ``Server``, close all listeners (:pr:`4704`) `Florian Jetter`_
+- Fix timeout in ``client.restart`` (:pr:`4690`) `Matteo De Wint`_
+- Avoid repeatedly using the same worker on first task with quiet cluster (:pr:`4638`) `Doug Davis`_
+- Grab ``func`` for ``finish`` case only if used (:pr:`4702`) `jakirkham`_
+- Remove hostname check in ``test_dashboard`` (:pr:`4706`) `James Bourbeau`_
+- Faster ``tests_semaphore::test_worker_dies`` (:pr:`4703`) `Florian Jetter`_
+- Clean up ``test_dashboard`` (:pr:`4700`) `crusaderky`_
+- Add timing information to ``TaskGroup`` (:pr:`4671`) `Matthew Rocklin`_
+- Remove ``WSSConnector`` TLS presence check (:pr:`4695`) `Marcos Moyano`_
+- Fix typo and remove unused ``time.time`` import (:pr:`4689`) `Hristo Georgiev`_
+- Don't initialize CUDA context in monitor (:pr:`4688`) `Charles Blackmon-Luca`_
+- Add support for extra conn args for HTTP protocols (:pr:`4682`) `Marcos Moyano`_
+- Adjust timings in ``test_threadpoolworkers`` (:pr:`4681`) `Florian Jetter`_
+- Add GPU metrics to ``SystemMonitor`` (:pr:`4661`) `Charles Blackmon-Luca`_
+- Removing ``dumps_msgpack()`` and ``loads_msgpack()`` (:pr:`4677`) `Mads R. B. Kristensen`_
+- Expose worker ``SystemMonitor`` s to scheduler via RPC (:pr:`4657`) `Charles Blackmon-Luca`_
+
+
 2021.04.0
 ---------
 
@@ -2296,4 +2332,5 @@ significantly without many new features.
 .. _`Charles Blackmon-Luca`: https://github.com/charlesbluca
 .. _`Marcos Moyano`: https://github.com/marcosmoyano
 .. _`James Lamb`: https://github.com/jameslamb
-.. _`Hristo Georgiev`: https://github.com/hristog
\ No newline at end of file
+.. _`Hristo Georgiev`: https://github.com/hristog
+.. _`Matteo De Wint`: https://github.com/mdwint
\ No newline at end of file

From 8dfaca632403b14bf73350780201394d1dd0e5ba Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Tue, 27 Apr 2021 07:20:02 -0700
Subject: [PATCH 1249/1550] Drop temporary `set` variables `s` (#4758)

These were needed before `TaskState` was annotated. Now that `TaskState`
is annotated, Cython already knows these are `set`s and so handles them
properly.
---
 distributed/scheduler.py | 25 ++++++++++---------------
 1 file changed, 10 insertions(+), 15 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 58d1ccc4965..53b0540c98a 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -2707,10 +2707,9 @@ def transition_waiting_released(self, key):
 
             dts: TaskState
             for dts in ts._dependencies:
-                s: set = dts._waiters
-                if ts in s:
-                    s.discard(ts)
-                    if not s and not dts._who_wants:
+                if ts in dts._waiters:
+                    dts._waiters.discard(ts)
+                    if not dts._waiters and not dts._who_wants:
                         recommendations[dts._key] = "released"
             ts._waiting_on.clear()
 
@@ -2760,9 +2759,8 @@ def transition_processing_released(self, key):
             if recommendations.get(key) != "waiting":
                 for dts in ts._dependencies:
                     if dts._state != "released":
-                        s: set = dts._waiters
-                        s.discard(ts)
-                        if not s and not dts._who_wants:
+                        dts._waiters.discard(ts)
+                        if not dts._waiters and not dts._who_wants:
                             recommendations[dts._key] = "released"
                 ts._waiters.clear()
 
@@ -2817,9 +2815,8 @@ def transition_processing_erred(
                 recommendations[dts._key] = "erred"
 
             for dts in ts._dependencies:
-                s: set = dts._waiters
-                s.discard(ts)
-                if not s and not dts._who_wants:
+                dts._waiters.discard(ts)
+                if not dts._waiters and not dts._who_wants:
                     recommendations[dts._key] = "released"
 
             ts._waiters.clear()  # do anything with this?
@@ -6975,8 +6972,7 @@ def _propagate_forgotten(
 
     for dts in ts._dependencies:
         dts._dependents.remove(ts)
-        s: set = dts._waiters
-        s.discard(ts)
+        dts._waiters.discard(ts)
         if not dts._dependents and not dts._who_wants:
             # Task not needed anymore
             assert dts is not ts
@@ -7010,9 +7006,8 @@ def _client_releases_keys(
         ts = state._tasks.get(key)
         if ts is not None and ts in cs._wants_what:
             cs._wants_what.remove(ts)
-            s: set = ts._who_wants
-            s.remove(cs)
-            if not s:
+            ts._who_wants.remove(cs)
+            if not ts._who_wants:
                 if not ts._dependents:
                     # No live dependents, can forget
                     recommendations[ts._key] = "forgotten"

From b577ece3f4bf5626d5ab6040065d8ff4d3880feb Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Tue, 27 Apr 2021 17:27:33 -0700
Subject: [PATCH 1250/1550] Handle missing `key` case in `report_on_key`
 (#4755)

* Drop `ts is None` case from `_task_to_report_msg`

`ts` cannot be `None` and be properly handled here. Generally we expect
`ts is not None` and will treat it that way. Otherwise an exception will
be raised.

* Drop `ts is None` case from `_task_to_client_msgs`

This case of `ts is None` simply cannot occur as a messages cannot be
constructed to be reported. Also the handful of cases where
`_task_to_client_msgs` is called we know `ts is not None`. So there is
no point in handling this case. As such just drop the unreachable case.

* Skip gathering client messages if unneeded

If `report_msg is None`, we can skip gathering messages as there is
nothing to send. Also if there are no interested clients, there is no
need to do the work of gathering messages.

* Report `key` cancelled if missing associated task

Previously if an associated `ts` could not be found for a `key`, we
would report that `key` as cancelled. However it seems this code was
missing after a recent rewrite. This effectively readds it by
constructing a message to cancel the `key` when the task is not found.

* Test `report_on_key` with only `key`
---
 distributed/scheduler.py            | 26 ++++++++++++--------------
 distributed/tests/test_scheduler.py |  5 +++++
 2 files changed, 17 insertions(+), 14 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 53b0540c98a..25afebe3308 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -6047,7 +6047,11 @@ def report_on_key(self, key: str = None, ts: TaskState = None, client: str = Non
             assert False, (key, ts)
             return
 
-        report_msg: dict = _task_to_report_msg(parent, ts)
+        report_msg: dict
+        if ts is None:
+            report_msg = {"op": "cancelled-key", "key": key}
+        else:
+            report_msg = _task_to_report_msg(parent, ts)
         if report_msg is not None:
             self.report(report_msg, ts=ts, client=client)
 
@@ -7058,9 +7062,7 @@ def _task_to_msg(state: SchedulerState, ts: TaskState, duration: double = -1) ->
 @cfunc
 @exceptval(check=False)
 def _task_to_report_msg(state: SchedulerState, ts: TaskState) -> dict:
-    if ts is None:
-        return {"op": "cancelled-key", "key": ts._key}
-    elif ts._state == "forgotten":
+    if ts._state == "forgotten":
         return {"op": "cancelled-key", "key": ts._key}
     elif ts._state == "memory":
         return {"op": "key-in-memory", "key": ts._key}
@@ -7079,16 +7081,12 @@ def _task_to_report_msg(state: SchedulerState, ts: TaskState) -> dict:
 @cfunc
 @exceptval(check=False)
 def _task_to_client_msgs(state: SchedulerState, ts: TaskState) -> dict:
-    report_msg: dict = _task_to_report_msg(state, ts)
-    client_msgs: dict
-    if ts is None:
-        # Notify all clients
-        client_msgs = {k: [report_msg] for k in state._clients}
-    else:
-        # Notify clients interested in key
-        cs: ClientState
-        client_msgs = {cs._client_key: [report_msg] for cs in ts._who_wants}
-    return client_msgs
+    if ts._who_wants:
+        report_msg: dict = _task_to_report_msg(state, ts)
+        if report_msg is not None:
+            cs: ClientState
+            return {cs._client_key: [report_msg] for cs in ts._who_wants}
+    return {}
 
 
 @cfunc
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 5d49d841455..e2c9b7a5aa9 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -413,6 +413,7 @@ async def test_delete_data(c, s, a, b):
 async def test_delete(c, s, a):
     x = c.submit(inc, 1)
     await x
+    assert x.key in s.tasks
     assert x.key in a.data
 
     await c._cancel(x)
@@ -422,6 +423,10 @@ async def test_delete(c, s, a):
         await asyncio.sleep(0.01)
         assert time() < start + 5
 
+    assert x.key not in s.tasks
+
+    s.report_on_key(key=x.key)
+
 
 @gen_cluster()
 async def test_filtered_communication(s, a, b):

From ea5cf0ed629c3969e5eaf0af2d173040f94a1113 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Wed, 28 Apr 2021 09:20:07 -0500
Subject: [PATCH 1251/1550] Fix erros in `check_thread_leak` (#4747)

---
 distributed/utils_test.py | 29 +++++++++++++++--------------
 1 file changed, 15 insertions(+), 14 deletions(-)

diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index 2e83199c094..1bbd7c7977a 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -1417,34 +1417,35 @@ def save_sys_modules():
 
 @contextmanager
 def check_thread_leak():
-    active_threads_start = set(threading._active)
+    """ Context manager to ensure we haven't leaked any threads """
+    active_threads_start = threading.enumerate()
 
     yield
 
     start = time()
     while True:
-        bad = [
-            t
-            for t, v in threading._active.items()
-            if t not in active_threads_start
-            and "Threaded" not in v.name
-            and "watch message" not in v.name
-            and "TCP-Executor" not in v.name
+        bad_threads = [
+            thread
+            for thread in threading.enumerate()
+            if thread not in active_threads_start
+            and "Threaded" not in thread.name
+            and "watch message" not in thread.name
+            and "TCP-Executor" not in thread.name
             # TODO: Make sure profile thread is cleaned up
             # and remove the line below
-            and "Profile" not in v.name
+            and "Profile" not in thread.name
         ]
-        if not bad:
+        if not bad_threads:
             break
         else:
             sleep(0.01)
         if time() > start + 5:
+            # Raise an error with information about leaked threads
             from distributed import profile
 
-            tid = bad[0]
-            thread = threading._active[tid]
-            call_stacks = profile.call_stack(sys._current_frames()[tid])
-            assert False, (thread, call_stacks)
+            bad_thread = bad_threads[0]
+            call_stacks = profile.call_stack(sys._current_frames()[bad_thread.ident])
+            assert False, (bad_thread, call_stacks)
 
 
 @contextmanager

From 25cf4f363a28e32cb30af4cf0954dfd3cdb452db Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Wed, 28 Apr 2021 12:23:22 -0700
Subject: [PATCH 1252/1550] Cython fixes for `MemoryState` (#4761)

Cython complains if we use `object.__new__` here. Besides this class is
marked `final`. So there is no other inheritance. Thus this can just use
`MemoryState` directly for object creation.

As Cython attributes don't support access through `getattr`, this code
doesn't really work in Cython. So just do a simple `for`-loop and spell
out each attribute that is being added.

For some reason these additional decorators create properties that don't
behave particularly well in Cython. Instead of being able to access the
property as expected, we see exceptions about
`builtin_function_or_method`. Admittedly there may be a Cython bug
involved. However things like `ccall` don't make much sense when applied
to a `property` (as it shouldn't create a Python and C function for it).
Same story with `inline` (though we can just access the `_*` name for
this affect). `nogil` may make sense for the code itself. However
`@property` access does involve some Python GIL using code. So it's
probably best to drop it as well.
---
 distributed/scheduler.py | 74 ++++++++++++++++++++++------------------
 1 file changed, 41 insertions(+), 33 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 25afebe3308..283b430c2db 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -300,12 +300,12 @@ class MemoryState:
         temporary spike
     """
 
-    __slots__ = ("process", "managed_in_memory", "managed_spilled", "unmanaged_old")
+    __slots__ = ("_process", "_managed_in_memory", "_managed_spilled", "_unmanaged_old")
 
-    process: Py_ssize_t
-    managed_in_memory: Py_ssize_t
-    managed_spilled: Py_ssize_t
-    unmanaged_old: Py_ssize_t
+    _process: Py_ssize_t
+    _managed_in_memory: Py_ssize_t
+    _managed_spilled: Py_ssize_t
+    _unmanaged_old: Py_ssize_t
 
     def __init__(
         self,
@@ -319,57 +319,65 @@ def __init__(
         # tasks progress. Also, sizeof() is not guaranteed to return correct results.
         # This can cause glitches where a partial measure is larger than the whole, so
         # we need to force all numbers to add up exactly by definition.
-        self.process = process
-        self.managed_spilled = min(managed_spilled, managed)
+        self._process = process
+        self._managed_spilled = min(managed_spilled, managed)
         # Subtractions between unsigned ints guaranteed by construction to be >= 0
-        self.managed_in_memory = min(managed - self.managed_spilled, process)
-        self.unmanaged_old = min(unmanaged_old, process - self.managed_in_memory)
+        self._managed_in_memory = min(managed - self._managed_spilled, process)
+        self._unmanaged_old = min(unmanaged_old, process - self._managed_in_memory)
+
+    @property
+    def process(self) -> Py_ssize_t:
+        return self._process
+
+    @property
+    def managed_in_memory(self) -> Py_ssize_t:
+        return self._managed_in_memory
+
+    @property
+    def managed_spilled(self) -> Py_ssize_t:
+        return self._managed_spilled
+
+    @property
+    def unmanaged_old(self) -> Py_ssize_t:
+        return self._unmanaged_old
 
     @classmethod
     def sum(cls, *infos: "MemoryState") -> "MemoryState":
-        out = object.__new__(cls)
-        for k in cls.__slots__:
-            setattr(out, k, sum(getattr(i, k) for i in infos))
+        out = MemoryState(process=0, unmanaged_old=0, managed=0, managed_spilled=0)
+        ms: MemoryState
+        for ms in infos:
+            out._process += ms._process
+            out._managed_spilled += ms._managed_spilled
+            out._managed_in_memory += ms._managed_in_memory
+            out._unmanaged_old += ms._unmanaged_old
         return out
 
     @property
-    @ccall
-    @inline
-    @nogil
     def managed(self) -> Py_ssize_t:
-        return self.managed_in_memory + self.managed_spilled
+        return self._managed_in_memory + self._managed_spilled
 
     @property
-    @ccall
-    @inline
-    @nogil
     def unmanaged(self) -> Py_ssize_t:
         # This is never negative thanks to __init__
-        return self.process - self.managed_in_memory
+        return self._process - self._managed_in_memory
 
     @property
-    @ccall
-    @inline
-    @nogil
     def unmanaged_recent(self) -> Py_ssize_t:
         # This is never negative thanks to __init__
-        return self.process - self.managed_in_memory - self.unmanaged_old
+        return self._process - self._managed_in_memory - self._unmanaged_old
 
     @property
-    @ccall
-    @inline
-    @nogil
     def optimistic(self) -> Py_ssize_t:
-        return self.managed_in_memory + self.unmanaged_old
+        return self._managed_in_memory + self._unmanaged_old
 
     def __repr__(self) -> str:
         return (
             f"Managed by Dask       : {format_bytes(self.managed)}\n"
-            f"  - in process memory : {format_bytes(self.managed_in_memory)}\n"
-            f"  - spilled to disk   : {format_bytes(self.managed_spilled)}\n"
-            f"Process memory (RSS)  : {format_bytes(self.process)}\n"
-            f"  - managed by Dask   : {format_bytes(self.managed_in_memory)}\n"
-            f"  - unmanaged (old)   : {format_bytes(self.unmanaged_old)}\n"
+            f"  - in process memory : {format_bytes(self._managed_in_memory)}\n"
+            f"  - spilled to disk   : {format_bytes(self._managed_spilled)}\n"
+            f"Process memory (RSS)  : {format_bytes(self._process)}\n"
+            f"  - managed by Dask   : {format_bytes(self._managed_in_memory)}\n"
+            f"  - unmanaged (old)   : {format_bytes(self._unmanaged_old)}\n"
             f"  - unmanaged (recent): {format_bytes(self.unmanaged_recent)}\n"
         )
 

From 6f7ca0a1e9fe8ac16f83f7a74678f688e4b2ce92 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Wed, 28 Apr 2021 20:31:07 -0500
Subject: [PATCH 1253/1550] Use `contextlib.nullcontext` (#4763)

---
 distributed/utils_test.py | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index 1bbd7c7977a..8f7cc121a4c 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -20,7 +20,7 @@
 import uuid
 import warnings
 import weakref
-from contextlib import contextmanager, suppress
+from contextlib import contextmanager, nullcontext, suppress
 from glob import glob
 from time import sleep
 
@@ -1524,14 +1524,10 @@ def check_instances():
 
 @contextmanager
 def clean(threads=not WINDOWS, instances=True, timeout=1, processes=True):
-    @contextmanager
-    def null():
-        yield
-
-    with check_thread_leak() if threads else null():
+    with check_thread_leak() if threads else nullcontext():
         with pristine_loop() as loop:
             with check_process_leak(check=processes):
-                with check_instances() if instances else null():
+                with check_instances() if instances else nullcontext():
                     with check_active_rpc(loop, timeout):
                         reset_config()
 

From 56aed4489288fc95f36e9fe11f29a452c39de4ac Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Wed, 28 Apr 2021 21:07:30 -0500
Subject: [PATCH 1254/1550] Update cythonization in CI (#4764)

* Check when scheduler is cythonized in CI

* Update cythonize step

* Update test

* Run full test suite again

* Add global variable to Scheduler on compiled state (#2)

* Add `COMPILED` global variable to scheduler

This should make it easy to tell whether the scheduler was compiled with
Cython or not.

* Simplify Cythonized Scheduler check

* lint

* Add skips to distributed/cli/tests/test_dask_spec.py

* xfail tests

* Move cythonized assert into test setup

* Remove unused CYTHONIZED env var

Co-authored-by: jakirkham <jakirkham@gmail.com>
---
 .github/workflows/tests.yaml                      | 15 +++++++++------
 distributed/cli/tests/test_dask_spec.py           |  3 +++
 distributed/deploy/tests/test_local.py            |  2 ++
 distributed/diagnostics/tests/test_progress.py    |  2 ++
 .../diagnostics/tests/test_progress_stream.py     |  2 ++
 distributed/scheduler.py                          |  2 ++
 distributed/tests/test_client.py                  |  9 ++++++++-
 distributed/tests/test_failed_workers.py          | 12 ++++++++++++
 8 files changed, 40 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
index 3b4978c0b55..0c0bb0a4a38 100644
--- a/.github/workflows/tests.yaml
+++ b/.github/workflows/tests.yaml
@@ -48,14 +48,17 @@ jobs:
         if: ${{ matrix.os == 'windows-latest' && matrix.python-version == '3.9' }}
         run: mamba uninstall ipython
 
-      - name: Cythonize
-        shell: bash -l {0}
-        if: ${{ matrix.python-version == '3.7' }}
-        run: python setup.py build_ext --with-cython
-
       - name: Install
         shell: bash -l {0}
-        run: python -m pip install --no-deps -e .
+        run: |
+          # Cythonize scheduler on Python 3.7 builds
+          if [[ "${{ matrix.python-version }}" = "3.7" ]]; then
+              python -m pip install -vv --no-deps --install-option="--with-cython" -e .
+              python -c "from distributed.scheduler import COMPILED; assert COMPILED"
+          else
+              python -m pip install --no-deps -e .
+              python -c "from distributed.scheduler import COMPILED; assert not COMPILED"
+          fi
 
       - name: mamba list
         shell: bash -l {0}
diff --git a/distributed/cli/tests/test_dask_spec.py b/distributed/cli/tests/test_dask_spec.py
index 0a5f64fc484..65a0f4f7297 100644
--- a/distributed/cli/tests/test_dask_spec.py
+++ b/distributed/cli/tests/test_dask_spec.py
@@ -4,10 +4,12 @@
 import yaml
 
 from distributed import Client
+from distributed.scheduler import COMPILED
 from distributed.utils_test import cleanup  # noqa: F401
 from distributed.utils_test import popen
 
 
+@pytest.mark.skipif(COMPILED, reason="Fails with cythonized scheduler")
 @pytest.mark.asyncio
 async def test_text(cleanup):
     with popen(
@@ -37,6 +39,7 @@ async def test_text(cleanup):
                 assert w["nthreads"] == 3
 
 
+@pytest.mark.skipif(COMPILED, reason="Fails with cythonized scheduler")
 @pytest.mark.asyncio
 async def test_file(cleanup, tmp_path):
     fn = str(tmp_path / "foo.yaml")
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index bd67581385e..b5306b37818 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -21,6 +21,7 @@
 from distributed.deploy.local import LocalCluster
 from distributed.deploy.utils_test import ClusterTest
 from distributed.metrics import time
+from distributed.scheduler import COMPILED
 from distributed.system import MEMORY_LIMIT
 from distributed.utils import TimeoutError, sync
 from distributed.utils_test import (  # noqa: F401
@@ -793,6 +794,7 @@ def scale_down(self, *args, **kwargs):
     await cluster.close()
 
 
+@pytest.mark.xfail(COMPILED, reason="Fails with cythonized scheduler")
 def test_local_tls_restart(loop):
     from distributed.utils_test import tls_only_security
 
diff --git a/distributed/diagnostics/tests/test_progress.py b/distributed/diagnostics/tests/test_progress.py
index fa74bfa3b46..81821734e06 100644
--- a/distributed/diagnostics/tests/test_progress.py
+++ b/distributed/diagnostics/tests/test_progress.py
@@ -12,6 +12,7 @@
     SchedulerPlugin,
 )
 from distributed.metrics import time
+from distributed.scheduler import COMPILED
 from distributed.utils_test import dec, div, gen_cluster, inc, nodebug
 
 
@@ -94,6 +95,7 @@ def check_bar_completed(capsys, width=40):
     assert percent == "100% Completed"
 
 
+@pytest.mark.xfail(COMPILED, reason="Fails with cythonized scheduler")
 @gen_cluster(client=True, Worker=Nanny, timeout=None)
 async def test_AllProgress(c, s, a, b):
     x, y, z = c.map(inc, [1, 2, 3])
diff --git a/distributed/diagnostics/tests/test_progress_stream.py b/distributed/diagnostics/tests/test_progress_stream.py
index d65d953e6b5..d0fda785054 100644
--- a/distributed/diagnostics/tests/test_progress_stream.py
+++ b/distributed/diagnostics/tests/test_progress_stream.py
@@ -6,6 +6,7 @@
 
 from distributed.client import wait
 from distributed.diagnostics.progress_stream import progress_quads, progress_stream
+from distributed.scheduler import COMPILED
 from distributed.utils_test import div, gen_cluster, inc
 
 
@@ -56,6 +57,7 @@ def test_progress_quads_too_many():
     assert len(d["name"]) == 6 * 3
 
 
+@pytest.mark.xfail(COMPILED, reason="Fails with cythonized scheduler")
 @gen_cluster(client=True)
 async def test_progress_stream(c, s, a, b):
     futures = c.map(div, [1] * 10, range(10))
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 283b430c2db..70e88ab7d02 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -185,6 +185,8 @@ def nogil(func):
     set, {"released", "waiting", "no-worker", "processing", "erred", "memory"}
 )
 globals()["ALL_TASK_STATES"] = ALL_TASK_STATES
+COMPILED = declare(bint, compiled)
+globals()["COMPILED"] = COMPILED
 
 
 @final
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index fb70994b567..a9661f7ab4c 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -58,7 +58,12 @@
 from distributed.compatibility import MACOS, WINDOWS
 from distributed.core import Status
 from distributed.metrics import time
-from distributed.scheduler import CollectTaskMetaDataPlugin, KilledWorker, Scheduler
+from distributed.scheduler import (
+    COMPILED,
+    CollectTaskMetaDataPlugin,
+    KilledWorker,
+    Scheduler,
+)
 from distributed.sizeof import sizeof
 from distributed.utils import is_valid_xml, mp_context, sync, tmp_text, tmpfile
 from distributed.utils_test import (  # noqa: F401
@@ -5096,6 +5101,7 @@ def test_dynamic_workloads_sync_random(c):
     _test_dynamic_workloads_sync(c, delay="random")
 
 
+@pytest.mark.xfail(COMPILED, reason="Fails with cythonized scheduler")
 @gen_cluster(client=True)
 async def test_bytes_keys(c, s, a, b):
     key = b"inc-123"
@@ -5848,6 +5854,7 @@ async def test_get_mix_futures_and_SubgraphCallable_dask_dataframe(c, s, a, b):
     assert result.equals(df.astype("f8"))
 
 
+@pytest.mark.xfail(COMPILED, reason="Fails with cythonized scheduler")
 def test_direct_to_workers(s, loop):
     with Client(s["address"], loop=loop, direct_to_workers=True) as client:
         future = client.scatter(1)
diff --git a/distributed/tests/test_failed_workers.py b/distributed/tests/test_failed_workers.py
index 4bd3e77eb81..3575037f48b 100644
--- a/distributed/tests/test_failed_workers.py
+++ b/distributed/tests/test_failed_workers.py
@@ -13,6 +13,7 @@
 from distributed.comm import CommClosedError
 from distributed.compatibility import MACOS
 from distributed.metrics import time
+from distributed.scheduler import COMPILED
 from distributed.utils import CancelledError, sync
 from distributed.utils_test import loop  # noqa: F401
 from distributed.utils_test import (
@@ -99,6 +100,7 @@ async def test_gather_then_submit_after_failed_workers(c, s, w, x, y, z):
         assert result == [sum(map(inc, range(20)))]
 
 
+@pytest.mark.xfail(COMPILED, reason="Fails with cythonized scheduler")
 @gen_cluster(Worker=Nanny, timeout=60, client=True)
 async def test_failed_worker_without_warning(c, s, a, b):
     L = c.map(inc, range(10))
@@ -135,6 +137,7 @@ async def test_failed_worker_without_warning(c, s, a, b):
     assert not (set(nthreads2) & set(s.nthreads))  # no overlap
 
 
+@pytest.mark.xfail(COMPILED, reason="Fails with cythonized scheduler")
 @gen_cluster(Worker=Nanny, client=True, timeout=60)
 async def test_restart(c, s, a, b):
     assert s.nthreads == {a.worker_address: 1, b.worker_address: 2}
@@ -163,6 +166,7 @@ async def test_restart(c, s, a, b):
     assert not any(cs.wants_what for cs in s.clients.values())
 
 
+@pytest.mark.xfail(COMPILED, reason="Fails with cythonized scheduler")
 @gen_cluster(Worker=Nanny, client=True, timeout=60)
 async def test_restart_cleared(c, s, a, b):
     x = 2 * delayed(1) + 1
@@ -175,6 +179,7 @@ async def test_restart_cleared(c, s, a, b):
         assert not coll
 
 
+@pytest.mark.xfail(COMPILED, reason="Fails with cythonized scheduler")
 def test_restart_sync_no_center(loop):
     with cluster(nanny=True) as (s, [a, b]):
         with Client(s["address"], loop=loop) as c:
@@ -186,6 +191,7 @@ def test_restart_sync_no_center(loop):
             assert len(c.nthreads()) == 2
 
 
+@pytest.mark.xfail(COMPILED, reason="Fails with cythonized scheduler")
 def test_restart_sync(loop):
     with cluster(nanny=True) as (s, [a, b]):
         with Client(s["address"], loop=loop) as c:
@@ -205,6 +211,7 @@ def test_restart_sync(loop):
             assert y.result() == 1 / 3
 
 
+@pytest.mark.xfail(COMPILED, reason="Fails with cythonized scheduler")
 @gen_cluster(Worker=Nanny, client=True, timeout=60)
 async def test_restart_fast(c, s, a, b):
     L = c.map(sleep, range(10))
@@ -221,6 +228,7 @@ async def test_restart_fast(c, s, a, b):
     assert result == 2
 
 
+@pytest.mark.xfail(COMPILED, reason="Fails with cythonized scheduler")
 def test_worker_doesnt_await_task_completion(loop):
     with cluster(nanny=True, nworkers=1) as (s, [w]):
         with Client(s["address"], loop=loop) as c:
@@ -232,6 +240,7 @@ def test_worker_doesnt_await_task_completion(loop):
             assert stop - start < 5
 
 
+@pytest.mark.xfail(COMPILED, reason="Fails with cythonized scheduler")
 def test_restart_fast_sync(loop):
     with cluster(nanny=True) as (s, [a, b]):
         with Client(s["address"], loop=loop) as c:
@@ -248,6 +257,7 @@ def test_restart_fast_sync(loop):
             assert x.result() == 2
 
 
+@pytest.mark.xfail(COMPILED, reason="Fails with cythonized scheduler")
 @gen_cluster(Worker=Nanny, client=True, timeout=60)
 async def test_fast_kill(c, s, a, b):
     L = c.map(sleep, range(10))
@@ -263,6 +273,7 @@ async def test_fast_kill(c, s, a, b):
     assert result == 2
 
 
+@pytest.mark.xfail(COMPILED, reason="Fails with cythonized scheduler")
 @gen_cluster(Worker=Nanny, timeout=60)
 async def test_multiple_clients_restart(s, a, b):
     c1 = await Client(s.address, asynchronous=True)
@@ -356,6 +367,7 @@ async def test_broken_worker_during_computation(c, s, a, b):
     await n.close()
 
 
+@pytest.mark.xfail(COMPILED, reason="Fails with cythonized scheduler")
 @gen_cluster(client=True, Worker=Nanny, timeout=60)
 async def test_restart_during_computation(c, s, a, b):
     xs = [delayed(slowinc)(i, delay=0.01) for i in range(50)]

From ace6b6296eb4e7dd01604f43853af57ff0cae42e Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Thu, 29 Apr 2021 17:52:50 +0200
Subject: [PATCH 1255/1550] Improve graceful shutdown if nanny is involved
 (#4725)

---
 distributed/comm/core.py            |  4 ++-
 distributed/nanny.py                | 42 ++++++++++++++++-------------
 distributed/tests/test_scheduler.py | 20 ++++++++++++++
 distributed/utils_test.py           | 11 +++++---
 4 files changed, 55 insertions(+), 22 deletions(-)

diff --git a/distributed/comm/core.py b/distributed/comm/core.py
index 6ee79723736..d9bb4bfb6ef 100644
--- a/distributed/comm/core.py
+++ b/distributed/comm/core.py
@@ -301,7 +301,9 @@ def time_left():
             upper_cap = min(time_left(), backoff_base * (2 ** attempt))
             backoff = random.uniform(0, upper_cap)
             attempt += 1
-            logger.debug("Could not connect, waiting for %s before retrying", backoff)
+            logger.debug(
+                "Could not connect to %s, waiting for %s before retrying", loc, backoff
+            )
             await asyncio.sleep(backoff)
     else:
         raise IOError(
diff --git a/distributed/nanny.py b/distributed/nanny.py
index 5c25c3297ea..a275055029d 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -441,30 +441,35 @@ def _on_exit_sync(self, exitcode):
         self.loop.add_callback(self._on_exit, exitcode)
 
     async def _on_exit(self, exitcode):
-        if self.status not in (Status.init, Status.closing, Status.closed):
+        if self.status not in (
+            Status.init,
+            Status.closing,
+            Status.closed,
+            Status.closing_gracefully,
+        ):
             try:
-                await self.scheduler.unregister(address=self.worker_address)
+                await self._unregister()
             except (EnvironmentError, CommClosedError):
                 if not self.reconnect:
                     await self.close()
                     return
 
-            try:
-                if self.status not in (
-                    Status.closing,
-                    Status.closed,
-                    Status.closing_gracefully,
-                ):
-                    if self.auto_restart:
-                        logger.warning("Restarting worker")
-                        await self.instantiate()
-                elif self.status == Status.closing_gracefully:
-                    await self.close()
+        try:
+            if self.status not in (
+                Status.closing,
+                Status.closed,
+                Status.closing_gracefully,
+            ):
+                if self.auto_restart:
+                    logger.warning("Restarting worker")
+                    await self.instantiate()
+            elif self.status == Status.closing_gracefully:
+                await self.close()
 
-            except Exception:
-                logger.error(
-                    "Failed to restart worker after its process exited", exc_info=True
-                )
+        except Exception:
+            logger.error(
+                "Failed to restart worker after its process exited", exc_info=True
+            )
 
     @property
     def pid(self):
@@ -733,8 +738,9 @@ def _run(
         async def do_stop(timeout=5, executor_wait=True):
             try:
                 await worker.close(
-                    report=False,
+                    report=True,
                     nanny=False,
+                    safe=True,  # TODO: Graceful or not?
                     executor_wait=executor_wait,
                     timeout=timeout,
                 )
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index e2c9b7a5aa9..d83b0224d2d 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -2468,3 +2468,23 @@ async def test_memory_is_none(c, s):
             assert s.memory.unmanaged == 0
             assert s.memory.unmanaged_old == 0
             assert s.memory.unmanaged_recent == 0
+
+
+@gen_cluster()
+async def test_close_scheduler__close_workers_Worker(s, a, b):
+    with captured_logger("distributed.comm", level=logging.DEBUG) as log:
+        await s.close(close_workers=True)
+        while not a.status == Status.closed:
+            await asyncio.sleep(0.05)
+    log = log.getvalue()
+    assert "retry" not in log
+
+
+@gen_cluster(Worker=Nanny)
+async def test_close_scheduler__close_workers_Nanny(s, a, b):
+    with captured_logger("distributed.comm", level=logging.DEBUG) as log:
+        await s.close(close_workers=True)
+        while not a.status == Status.closed:
+            await asyncio.sleep(0.05)
+    log = log.getvalue()
+    assert "retry" not in log
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index 8f7cc121a4c..a95d089bd20 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -734,9 +734,14 @@ async def disconnect(addr, timeout=3, rpc_kwargs=None):
     rpc_kwargs = rpc_kwargs or {}
 
     async def do_disconnect():
-        with suppress(EnvironmentError, CommClosedError):
-            with rpc(addr, **rpc_kwargs) as w:
-                await w.terminate(close=True)
+        with rpc(addr, **rpc_kwargs) as w:
+            # If the worker was killed hard (e.g. sigterm) during test runtime,
+            # we do not know at this point and may not be able to connect
+            with suppress(EnvironmentError, CommClosedError):
+                # Do not request a reply since comms will be closed by the
+                # worker before a reply can be made and we will always trigger
+                # the timeout
+                await w.terminate(reply=False)
 
     await asyncio.wait_for(do_disconnect(), timeout=timeout)
 

From 97c66db49e6285f7c75cf1b5eddc9e917b6de2eb Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Thu, 29 Apr 2021 09:03:10 -0700
Subject: [PATCH 1256/1550] Fixes some pickling issues in the Cythonized
 Scheduler (#4768)

---
 distributed/diagnostics/progress_stream.py            | 8 +++++++-
 distributed/diagnostics/tests/test_progress_stream.py | 2 --
 distributed/scheduler.py                              | 4 ++--
 distributed/tests/test_client.py                      | 1 -
 4 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/distributed/diagnostics/progress_stream.py b/distributed/diagnostics/progress_stream.py
index 2ee2b1c5000..57b0cb38390 100644
--- a/distributed/diagnostics/progress_stream.py
+++ b/distributed/diagnostics/progress_stream.py
@@ -21,6 +21,12 @@ def counts(scheduler, allprogress):
     )
 
 
+def remove_plugin(*args, **kwargs):
+    # Wrapper function around `Scheduler.remove_plugin` to avoid raising a
+    # `PicklingError` when using a cythonized scheduler
+    return Scheduler.remove_plugin(*args, **kwargs)
+
+
 async def progress_stream(address, interval):
     """Open a TCP connection to scheduler, receive progress messages
 
@@ -47,7 +53,7 @@ async def progress_stream(address, interval):
             "setup": dumps_function(AllProgress),
             "function": dumps_function(counts),
             "interval": interval,
-            "teardown": dumps_function(Scheduler.remove_plugin),
+            "teardown": dumps_function(remove_plugin),
         }
     )
     return comm
diff --git a/distributed/diagnostics/tests/test_progress_stream.py b/distributed/diagnostics/tests/test_progress_stream.py
index d0fda785054..d65d953e6b5 100644
--- a/distributed/diagnostics/tests/test_progress_stream.py
+++ b/distributed/diagnostics/tests/test_progress_stream.py
@@ -6,7 +6,6 @@
 
 from distributed.client import wait
 from distributed.diagnostics.progress_stream import progress_quads, progress_stream
-from distributed.scheduler import COMPILED
 from distributed.utils_test import div, gen_cluster, inc
 
 
@@ -57,7 +56,6 @@ def test_progress_quads_too_many():
     assert len(d["name"]) == 6 * 3
 
 
-@pytest.mark.xfail(COMPILED, reason="Fails with cythonized scheduler")
 @gen_cluster(client=True)
 async def test_progress_stream(c, s, a, b):
     futures = c.map(div, [1] * 10, range(10))
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 70e88ab7d02..0f68d02b876 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -3453,8 +3453,8 @@ def __init__(
             maxlen=dask.config.get("distributed.scheduler.transition-log-length")
         )
         self.events = defaultdict(
-            lambda: deque(
-                maxlen=dask.config.get("distributed.scheduler.events-log-length")
+            partial(
+                deque, maxlen=dask.config.get("distributed.scheduler.events-log-length")
             )
         )
         self.event_counts = defaultdict(int)
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index a9661f7ab4c..30dbf29ef98 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -5854,7 +5854,6 @@ async def test_get_mix_futures_and_SubgraphCallable_dask_dataframe(c, s, a, b):
     assert result.equals(df.astype("f8"))
 
 
-@pytest.mark.xfail(COMPILED, reason="Fails with cythonized scheduler")
 def test_direct_to_workers(s, loop):
     with Client(s["address"], loop=loop, direct_to_workers=True) as client:
         future = client.scatter(1)

From 233ec8840671b5d705a06d0dcfe1ff42b771230b Mon Sep 17 00:00:00 2001
From: Naty Clementi <natyclementi@gmail.com>
Date: Thu, 29 Apr 2021 23:10:56 -0400
Subject: [PATCH 1257/1550] Unregister worker plugin (#4748)

---
 distributed/client.py                         | 57 ++++++++++++++++++-
 distributed/diagnostics/plugin.py             | 12 ++++
 .../diagnostics/tests/test_worker_plugin.py   | 34 +++++++++++
 distributed/scheduler.py                      | 15 ++++-
 distributed/worker.py                         | 30 +++++++---
 5 files changed, 137 insertions(+), 11 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 695c9dd39f2..b0f3f148331 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -51,7 +51,7 @@
     connect,
     rpc,
 )
-from .diagnostics.plugin import UploadFile, WorkerPlugin
+from .diagnostics.plugin import UploadFile, WorkerPlugin, _get_worker_plugin_name
 from .metrics import time
 from .protocol import to_serialize
 from .protocol.pickle import dumps, loads
@@ -4013,6 +4013,7 @@ def register_worker_plugin(self, plugin=None, name=None, **kwargs):
         name : str, optional
             A name for the plugin.
             Registering a plugin with the same name will have no effect.
+            If plugin has no name attribute a random name is used.
         **kwargs : optional
             If you pass a class as the plugin, instead of a class instance, then the
             class will be instantiated with any extra keyword arguments.
@@ -4049,12 +4050,66 @@ class will be instantiated with any extra keyword arguments.
         See Also
         --------
         distributed.WorkerPlugin
+        unregister_worker_plugin
         """
         if isinstance(plugin, type):
             plugin = plugin(**kwargs)
 
+        if name is None:
+            name = _get_worker_plugin_name(plugin)
+
+        assert name
+
         return self.sync(self._register_worker_plugin, plugin=plugin, name=name)
 
+    async def _unregister_worker_plugin(self, name):
+        responses = await self.scheduler.unregister_worker_plugin(name=name)
+
+        for response in responses.values():
+            if response["status"] == "error":
+                exc = response["exception"]
+                tb = response["traceback"]
+                raise exc.with_traceback(tb)
+        return responses
+
+    def unregister_worker_plugin(self, name):
+        """Unregisters a lifecycle worker plugin
+
+        This unregisters an existing worker plugin. As part of the unregistration process
+        the plugin's ``teardown`` method will be called.
+
+        Parameters
+        ----------
+        name : str
+            Name of the plugin to unregister. See the :meth:`Client.register_worker_plugin`
+            docstring for more information.
+
+        Examples
+        --------
+        >>> class MyPlugin(WorkerPlugin):
+        ...     def __init__(self, *args, **kwargs):
+        ...         pass  # the constructor is up to you
+        ...     def setup(self, worker: dask.distributed.Worker):
+        ...         pass
+        ...     def teardown(self, worker: dask.distributed.Worker):
+        ...         pass
+        ...     def transition(self, key: str, start: str, finish: str, **kwargs):
+        ...         pass
+        ...     def release_key(self, key: str, state: str, cause: Optional[str], reason: None, report: bool):
+        ...         pass
+        ...     def release_dep(self, dep: str, state: str, report: bool):
+        ...         pass
+
+        >>> plugin = MyPlugin(1, 2, 3)
+        >>> client.register_worker_plugin(plugin, name='foo')
+        >>> client.unregister_worker_plugin(name='foo')
+
+        See Also
+        --------
+        register_worker_plugin
+        """
+        return self.sync(self._unregister_worker_plugin, name=name)
+
 
 class _WorkerSetupPlugin(WorkerPlugin):
     """ This is used to support older setup functions as callbacks """
diff --git a/distributed/diagnostics/plugin.py b/distributed/diagnostics/plugin.py
index 58b5adbc585..80e1736ac28 100644
--- a/distributed/diagnostics/plugin.py
+++ b/distributed/diagnostics/plugin.py
@@ -3,6 +3,9 @@
 import socket
 import subprocess
 import sys
+import uuid
+
+from dask.utils import funcname
 
 logger = logging.getLogger(__name__)
 
@@ -186,6 +189,15 @@ def release_dep(self, dep, state, report):
         """
 
 
+def _get_worker_plugin_name(plugin) -> str:
+    """Returns the worker plugin name. If plugin has no name attribute
+    a random name is used."""
+    if hasattr(plugin, "name"):
+        return plugin.name
+    else:
+        return funcname(plugin) + "-" + str(uuid.uuid4())
+
+
 class PipInstall(WorkerPlugin):
     """A Worker Plugin to pip install a set of packages
 
diff --git a/distributed/diagnostics/tests/test_worker_plugin.py b/distributed/diagnostics/tests/test_worker_plugin.py
index 858f8feedc2..393311b5d7a 100644
--- a/distributed/diagnostics/tests/test_worker_plugin.py
+++ b/distributed/diagnostics/tests/test_worker_plugin.py
@@ -53,6 +53,40 @@ async def test_create_with_client(c, s):
     assert worker._my_plugin_status == "teardown"
 
 
+@gen_cluster(client=True, nthreads=[])
+async def test_remove_with_client(c, s):
+    await c.register_worker_plugin(MyPlugin(123), name="foo")
+    await c.register_worker_plugin(MyPlugin(546), name="bar")
+
+    worker = await Worker(s.address, loop=s.loop)
+    # remove the 'foo' plugin
+    await c.unregister_worker_plugin("foo")
+    assert worker._my_plugin_status == "teardown"
+
+    # check that on the scheduler registered worker plugins we only have 'bar'
+    assert len(s.worker_plugins) == 1
+    assert "bar" in s.worker_plugins
+
+    # check on the worker plugins that we only have 'bar'
+    assert len(worker.plugins) == 1
+    assert "bar" in worker.plugins
+
+    # let's remove 'bar' and we should have none worker plugins
+    await c.unregister_worker_plugin("bar")
+    assert worker._my_plugin_status == "teardown"
+    assert not s.worker_plugins
+    assert not worker.plugins
+
+
+@gen_cluster(client=True, nthreads=[])
+async def test_remove_with_client_raises(c, s):
+    await c.register_worker_plugin(MyPlugin(123), name="foo")
+
+    worker = await Worker(s.address, loop=s.loop)
+    with pytest.raises(ValueError, match="bar"):
+        await c.unregister_worker_plugin("bar")
+
+
 @gen_cluster(client=True, nthreads=[])
 async def test_create_with_client_and_plugin_from_class(c, s):
     await c.register_worker_plugin(MyPlugin, data=456)
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 0f68d02b876..fb9610c53d5 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -3458,7 +3458,7 @@ def __init__(
             )
         )
         self.event_counts = defaultdict(int)
-        self.worker_plugins = []
+        self.worker_plugins = dict()
 
         worker_handlers = {
             "task-finished": self.handle_task_finished,
@@ -3525,6 +3525,7 @@ def __init__(
             "get_task_status": self.get_task_status,
             "get_task_stream": self.get_task_stream,
             "register_worker_plugin": self.register_worker_plugin,
+            "unregister_worker_plugin": self.unregister_worker_plugin,
             "adaptive_target": self.adaptive_target,
             "workers_to_close": self.workers_to_close,
             "subscribe_worker_status": self.subscribe_worker_status,
@@ -6308,13 +6309,23 @@ def stop_task_metadata(self, comm=None, name=None):
 
     async def register_worker_plugin(self, comm, plugin, name=None):
         """ Registers a setup function, and call it on every worker """
-        self.worker_plugins.append({"plugin": plugin, "name": name})
+        self.worker_plugins[name] = plugin
 
         responses = await self.broadcast(
             msg=dict(op="plugin-add", plugin=plugin, name=name)
         )
         return responses
 
+    async def unregister_worker_plugin(self, comm, name):
+        """ Unregisters a worker plugin"""
+        try:
+            worker_plugins = self.worker_plugins.pop(name)
+        except KeyError:
+            raise ValueError(f"The worker plugin {name} does not exists")
+
+        responses = await self.broadcast(msg=dict(op="plugin-remove", name=name))
+        return responses
+
     def transition(self, key, finish: str, *args, **kwargs):
         """Transition a key from its current state to the finish state
 
diff --git a/distributed/worker.py b/distributed/worker.py
index eb87d797a86..bb641e3ff1f 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -7,7 +7,6 @@
 import random
 import sys
 import threading
-import uuid
 import warnings
 import weakref
 from collections import defaultdict, deque, namedtuple
@@ -41,6 +40,7 @@
     pingpong,
     send_recv,
 )
+from .diagnostics.plugin import _get_worker_plugin_name
 from .diskutils import WorkSpace
 from .http import get_handlers
 from .metrics import time
@@ -680,6 +680,7 @@ def __init__(
             "actor_execute": self.actor_execute,
             "actor_attribute": self.actor_attribute,
             "plugin-add": self.plugin_add,
+            "plugin-remove": self.plugin_remove,
             "get_monitor_info": self.get_monitor_info,
         }
 
@@ -921,8 +922,8 @@ async def _register_with_scheduler(self):
         else:
             await asyncio.gather(
                 *[
-                    self.plugin_add(**plugin_kwargs)
-                    for plugin_kwargs in response["worker-plugins"]
+                    self.plugin_add(name=name, plugin=plugin)
+                    for name, plugin in response["worker-plugins"].items()
                 ]
             )
 
@@ -2551,11 +2552,9 @@ async def plugin_add(self, comm=None, plugin=None, name=None):
         with log_errors(pdb=False):
             if isinstance(plugin, bytes):
                 plugin = pickle.loads(plugin)
-            if not name:
-                if hasattr(plugin, "name"):
-                    name = plugin.name
-                else:
-                    name = funcname(plugin) + "-" + str(uuid.uuid4())
+
+            if name is None:
+                name = _get_worker_plugin_name(plugin)
 
             assert name
 
@@ -2576,6 +2575,21 @@ async def plugin_add(self, comm=None, plugin=None, name=None):
 
                 return {"status": "OK"}
 
+    async def plugin_remove(self, comm=None, name=None):
+        with log_errors(pdb=False):
+            logger.info(f"Removing Worker plugin {name}")
+            try:
+                plugin = self.plugins.pop(name)
+                if hasattr(plugin, "teardown"):
+                    result = plugin.teardown(worker=self)
+                    if isawaitable(result):
+                        result = await result
+            except Exception as e:
+                msg = error_message(e)
+                return msg
+
+            return {"status": "OK"}
+
     async def actor_execute(
         self, comm=None, actor=None, function=None, args=(), kwargs={}
     ):

From 74fbd923bcf5722556b8eed60a5c3947a822fc9b Mon Sep 17 00:00:00 2001
From: Nathan <ndanielsen@users.noreply.github.com>
Date: Tue, 4 May 2021 12:10:06 -0700
Subject: [PATCH 1258/1550] 4756 - Add stacklevel keyword into
 performance_report() to allow for selecting calling code to be displayed
 (#4777)

---
 distributed/client.py            | 20 ++++++++++++++++----
 distributed/tests/test_client.py | 19 ++++++++++++++++---
 2 files changed, 32 insertions(+), 7 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index b0f3f148331..a2066c75649 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -4670,17 +4670,29 @@ class performance_report:
     browser.  Locally we recommend using ``python -m http.server`` or hosting
     the file live online.
 
+    Parameters
+    ----------
+    filename: str (optional)
+        The filename to save the performance report locally
+
+    stacklevel: int (optional)
+        The code execution frame utilized for populating the Calling Code section
+        of the report. Defaults to `1` which is the frame calling ``performance_report``
+
+
     Examples
     --------
-    >>> with performance_report(filename="myfile.html"):
+    >>> with performance_report(filename="myfile.html", stacklevel=1):
     ...     x.compute()
 
     $ python -m http.server
     $ open myfile.html
     """
 
-    def __init__(self, filename="dask-report.html"):
+    def __init__(self, filename="dask-report.html", stacklevel=1):
         self.filename = filename
+        # stacklevel 0 or less - shows dask internals which likely isn't helpful
+        self._stacklevel = stacklevel if stacklevel > 0 else 1
 
     async def __aenter__(self):
         self.start = time()
@@ -4689,7 +4701,7 @@ async def __aenter__(self):
     async def __aexit__(self, typ, value, traceback, code=None):
         if not code:
             try:
-                frame = inspect.currentframe().f_back
+                frame = sys._getframe(self._stacklevel)
                 code = inspect.getsource(frame)
             except Exception:
                 code = ""
@@ -4704,7 +4716,7 @@ def __enter__(self):
 
     def __exit__(self, typ, value, traceback):
         try:
-            frame = inspect.currentframe().f_back
+            frame = sys._getframe(self._stacklevel)
             code = inspect.getsource(frame)
         except Exception:
             code = ""
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 30dbf29ef98..57bdc67b17e 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -6178,7 +6178,7 @@ async def test_performance_report(c, s, a, b):
     pytest.importorskip("bokeh")
     da = pytest.importorskip("dask.array")
 
-    async def f():
+    async def f(stacklevel):
         """
         We wrap this in a function so that the assertions aren't in the
         performanace report itself
@@ -6187,14 +6187,15 @@ async def f():
         """
         x = da.random.random((1000, 1000), chunks=(100, 100))
         with tmpfile(extension="html") as fn:
-            async with performance_report(filename=fn):
+            async with performance_report(filename=fn, stacklevel=stacklevel):
                 await c.compute((x + x.T).sum())
 
             with open(fn) as f:
                 data = f.read()
         return data
 
-    data = await f()
+    # Ensure default kwarg maintains backward compatability
+    data = await f(stacklevel=1)
 
     assert "Also, we want this comment to appear" in data
     assert "bokeh" in data
@@ -6204,6 +6205,18 @@ async def f():
     assert "Threads: 4" in data
     assert dask.__version__ in data
 
+    # Stacklevel two captures code two frames back -- which in this case
+    # is the testing function
+    data = await f(stacklevel=2)
+    assert "async def test_performance_report(c, s, a, b):" in data
+    assert "Dask Performance Report" in data
+
+    # Stacklevel zero or lower is overridden to stacklevel=1 so we don't see
+    # distributed internals
+    data = await f(stacklevel=0)
+    assert "Also, we want this comment to appear" in data
+    assert "Dask Performance Report" in data
+
 
 @pytest.mark.asyncio
 async def test_client_gather_semaphore_loop(cleanup):

From 0a014dac4a1edb090ee17027fabcd41cdd015553 Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Wed, 5 May 2021 11:15:32 +0200
Subject: [PATCH 1259/1550] Ensure deps are actually logged in worker (#4753)

* Ensure deps are actually logged in worker

* Log only keys for gather-dependency log
---
 distributed/tests/test_worker.py | 31 +++++++++++++++++++++++++++++++
 distributed/worker.py            |  3 ++-
 2 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 4681c5c36de..44ba35df021 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -1789,6 +1789,37 @@ async def test_story(c, s, w):
     assert w.story(ts) == w.story(ts.key)
 
 
+@gen_cluster(client=True)
+async def test_story_with_deps(c, s, a, b):
+    """
+    Assert that the structure of the story does not change unintentionally and
+    expected subfields are actually filled
+    """
+    futures = c.map(inc, range(10), workers=[a.address])
+    res = c.submit(sum, futures, workers=[b.address])
+    await res
+    key = res.key
+
+    story = a.story(key)
+    assert story == []
+    story = b.story(key)
+
+    expected_story = [
+        (key, "new"),
+        (key, "new", "waiting"),
+        (
+            "gather-dependencies",
+            key,
+            {fut.key for fut in futures},
+        ),
+        (key, "waiting", "ready"),
+        (key, "ready", "executing"),
+        (key, "put-in-memory"),
+        (key, "executing", "memory"),
+    ]
+    assert story == expected_story
+
+
 def test_weight_deprecated():
     with pytest.warns(DeprecationWarning):
         weight("foo", "bar")
diff --git a/distributed/worker.py b/distributed/worker.py
index bb641e3ff1f..f4ab4947355 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -2008,7 +2008,8 @@ def ensure_communicating(self):
 
                     deps = [dep for dep in deps if dep not in missing_deps]
 
-                self.log.append(("gather-dependencies", key, deps))
+                log_keys = {d.key for d in deps}
+                self.log.append(("gather-dependencies", key, log_keys))
 
                 in_flight = False
 

From 1ee22c84a48e93df2e5bfafae1725c84d0072a6e Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Wed, 5 May 2021 11:52:44 +0100
Subject: [PATCH 1260/1550] Fix doctests in utils.py (#4785)

---
 distributed/utils.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/distributed/utils.py b/distributed/utils.py
index fc4a73672fc..a7b8befc221 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -640,7 +640,9 @@ def key_split_group(x):
     >>> key_split_group('<module.submodule.myclass object at 0xdaf372')
     'myclass'
     >>> key_split_group('x')
+    'x'
     >>> key_split_group('x-1')
+    'x'
     """
     typ = type(x)
     if typ is tuple:
@@ -1319,24 +1321,24 @@ def parse_ports(port):
     A single port can be specified using an integer:
 
     >>> parse_ports(8787)
-    >>> [8787]
+    [8787]
 
     or a string:
 
     >>> parse_ports("8787")
-    >>> [8787]
+    [8787]
 
     A sequential range of ports can be specified by a string which indicates
     the first and last ports which should be included in the sequence of ports:
 
     >>> parse_ports("8787:8790")
-    >>> [8787, 8788, 8789, 8790]
+    [8787, 8788, 8789, 8790]
 
     An input of ``None`` is also valid and can be used to indicate that no port
     has been specified:
 
     >>> parse_ports(None)
-    >>> [None]
+    [None]
 
     """
     if isinstance(port, str) and ":" not in port:
@@ -1451,7 +1453,7 @@ def import_term(name: str):
 
     Examples
     --------
-    >>> import_term("math.sin")
+    >>> import_term("math.sin") # doctest: +SKIP
     <function math.sin(x, /)>
     """
     try:

From 7bfd1ae202bb973aea36da0e0d41a7377d2a47ab Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Thu, 6 May 2021 11:50:39 +0200
Subject: [PATCH 1261/1550] Reduce complexity of
 test_gather_allow_worker_reconnect (#4739)

---
 distributed/tests/test_scheduler.py | 22 ++++++++--------------
 1 file changed, 8 insertions(+), 14 deletions(-)

diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index d83b0224d2d..18e9dc5a259 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -2043,7 +2043,6 @@ async def test_gather_no_workers(c, s, a, b):
     assert list(res["keys"]) == ["x"]
 
 
-@pytest.mark.flaky(reruns=10, reruns_delay=5, condition=MACOS)
 @gen_cluster(client=True, client_kwargs={"direct_to_workers": False})
 async def test_gather_allow_worker_reconnect(c, s, a, b):
     """
@@ -2062,9 +2061,10 @@ async def test_gather_allow_worker_reconnect(c, s, a, b):
     def inc_slow(x):
         # Once the graph below is rescheduled this computation runs again. We
         # need to sleep for at least 0.5 seconds to give the worker a chance to
-        # reconnect (Heartbeat timing)
+        # reconnect (Heartbeat timing). In slow CI situations, the actual
+        # reconnect might take a bit longer, therefore wait more
         if x in already_calculated:
-            time.sleep(1)
+            time.sleep(2)
         already_calculated.append(x)
         return x + 1
 
@@ -2076,18 +2076,16 @@ def reducer(x, y):
 
     z = c.submit(reducer, x, y)
 
-    s.rpc = await FlakyConnectionPool(failing_connections=4)
+    s.rpc = await FlakyConnectionPool(failing_connections=1)
 
-    with dask.config.set(
-        {"distributed.comm.retry.delay_min": 0.5, "distributed.comm.retry.count": 3}
-    ):
+    # This behaviour is independent of retries. Remove them to reduce complexity
+    # of this setup
+    with dask.config.set({"distributed.comm.retry.count": 0}):
         with captured_logger(
             logging.getLogger("distributed.scheduler")
         ) as sched_logger, captured_logger(
             logging.getLogger("distributed.client")
-        ) as client_logger, captured_logger(
-            logging.getLogger("distributed.utils_comm")
-        ) as utils_comm_logger:
+        ) as client_logger:
             # Gather using the client (as an ordinary user would)
             # Upon a missing key, the client will reschedule the computations
             res = await c.gather(z)
@@ -2096,14 +2094,11 @@ def reducer(x, y):
 
     sched_logger = sched_logger.getvalue()
     client_logger = client_logger.getvalue()
-    utils_comm_logger = utils_comm_logger.getvalue()
 
     # Ensure that the communication was done via the scheduler, i.e. we actually hit a
     # bad connection
     assert s.rpc.cnn_count > 0
 
-    assert "Retrying get_data_from_worker after exception" in utils_comm_logger
-
     # The reducer task was actually not found upon first collection. The client will
     # reschedule the graph
     assert "Couldn't gather 1 keys, rescheduling" in client_logger
@@ -2122,7 +2117,6 @@ def reducer(x, y):
     ]
     assert len(transitions_to_processing) == 1
 
-    starts = []
     finish_processing_transitions = 0
     for transition in s.transition_log:
         key, start, finish, recommendations, timestamp = transition

From 27e59258c79fa362df7d088ac73067e767b806a3 Mon Sep 17 00:00:00 2001
From: Peter Andreas Entschev <peter@entschev.com>
Date: Thu, 6 May 2021 18:12:30 +0200
Subject: [PATCH 1262/1550] Support for UCX 1.10+ (#4787)

* Updates for UCX 1.10

* Update RDMACM for UCX 1.10

* Ensure UCX creates CUDA context when cuda_copy is enabled

* UCX retry connection with different port if that is already taken

* Adjust UCX conditions for versions older and newer than 1.10

* Raise ImportError if Numba isn't available with UCX+CUDA

* Do not retry on ucx_create_listener

* Add comment on CUDA context initialization before UCX
---
 distributed/comm/ucx.py | 29 ++++++++++++++++++++++-------
 1 file changed, 22 insertions(+), 7 deletions(-)

diff --git a/distributed/comm/ucx.py b/distributed/comm/ucx.py
index 92b457307ac..b0bb62b5852 100644
--- a/distributed/comm/ucx.py
+++ b/distributed/comm/ucx.py
@@ -53,13 +53,26 @@ def init_once():
     if ucp is not None:
         return
 
+    # remove/process dask.ucx flags for valid ucx options
+    ucx_config = _scrub_ucx_config()
+
+    # We ensure the CUDA context is created before initializing UCX. This can't
+    # be safely handled externally because communications in Dask start before
+    # preload scripts run.
+    if "TLS" in ucx_config and "cuda_copy" in ucx_config["TLS"]:
+        try:
+            import numba.cuda
+        except ImportError:
+            raise ImportError(
+                "CUDA support with UCX requires Numba for context management"
+            )
+
+        numba.cuda.current_context()
+
     import ucp as _ucp
 
     ucp = _ucp
 
-    # remove/process dask.ucx flags for valid ucx options
-    ucx_config = _scrub_ucx_config()
-
     ucp.init(options=ucx_config, env_takes_precedence=True)
 
     # Find the function, `host_array()`, to use when allocating new host arrays
@@ -458,7 +471,9 @@ def _scrub_ucx_config():
     # 2) explicitly defined UCX configuration flags
 
     # import does not initialize ucp -- this will occur outside this function
-    from ucp import get_config
+    from ucp import get_config, get_ucx_version
+
+    ucx_110 = get_ucx_version() >= (1, 10, 0)
 
     options = {}
 
@@ -473,11 +488,11 @@ def _scrub_ucx_config():
         ]
     ):
         if dask.config.get("ucx.rdmacm"):
-            tls = "tcp,rdmacm"
+            tls = "tcp" if ucx_110 else "tcp,rdmacm"
             tls_priority = "rdmacm"
         else:
-            tls = "tcp,sockcm"
-            tls_priority = "sockcm"
+            tls = "tcp" if ucx_110 else "tcp,sockcm"
+            tls_priority = "tcp" if ucx_110 else "sockcm"
 
         # CUDA COPY can optionally be used with ucx -- we rely on the user
         # to define when messages will include CUDA objects.  Note:

From c76ed4604dcc5c3c9af23110ba0fc2a1bbb32e1b Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Fri, 7 May 2021 04:39:29 -0500
Subject: [PATCH 1263/1550] Remove release_dep from WorkerPlugin API (#4791)

---
 distributed/client.py                              |  6 +-----
 distributed/diagnostics/plugin.py                  | 14 --------------
 .../diagnostics/tests/test_worker_plugin.py        |  5 +----
 3 files changed, 2 insertions(+), 23 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index a2066c75649..cddc07e5a70 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -3994,7 +3994,7 @@ def register_worker_plugin(self, plugin=None, name=None, **kwargs):
         that connects in the future.
 
         The plugin may include methods ``setup``, ``teardown``, ``transition``,
-        ``release_key``, and ``release_dep``.  See the
+        and ``release_key``.  See the
         ``dask.distributed.WorkerPlugin`` class or the examples below for the
         interface and docstrings.  It must be serializable with the pickle or
         cloudpickle modules.
@@ -4031,8 +4031,6 @@ class will be instantiated with any extra keyword arguments.
         ...         pass
         ...     def release_key(self, key: str, state: str, cause: Optional[str], reason: None, report: bool):
         ...         pass
-        ...     def release_dep(self, dep: str, state: str, report: bool):
-        ...         pass
 
         >>> plugin = MyPlugin(1, 2, 3)
         >>> client.register_worker_plugin(plugin)
@@ -4097,8 +4095,6 @@ def unregister_worker_plugin(self, name):
         ...         pass
         ...     def release_key(self, key: str, state: str, cause: Optional[str], reason: None, report: bool):
         ...         pass
-        ...     def release_dep(self, dep: str, state: str, report: bool):
-        ...         pass
 
         >>> plugin = MyPlugin(1, 2, 3)
         >>> client.register_worker_plugin(plugin, name='foo')
diff --git a/distributed/diagnostics/plugin.py b/distributed/diagnostics/plugin.py
index 80e1736ac28..5207d54aa82 100644
--- a/distributed/diagnostics/plugin.py
+++ b/distributed/diagnostics/plugin.py
@@ -174,20 +174,6 @@ def release_key(self, key, state, cause, reason, report):
             Whether the worker should report the released task to the scheduler.
         """
 
-    def release_dep(self, dep, state, report):
-        """
-        Called when the worker releases a dependency.
-
-        Parameters
-        ----------
-        dep : string
-        state : string
-            State of the released dependency.
-            One of waiting, flight, memory.
-        report : bool
-            Whether the worker should report the released dependency to the scheduler.
-        """
-
 
 def _get_worker_plugin_name(plugin) -> str:
     """Returns the worker plugin name. If plugin has no name attribute
diff --git a/distributed/diagnostics/tests/test_worker_plugin.py b/distributed/diagnostics/tests/test_worker_plugin.py
index 393311b5d7a..cc41bd79d09 100644
--- a/distributed/diagnostics/tests/test_worker_plugin.py
+++ b/distributed/diagnostics/tests/test_worker_plugin.py
@@ -37,9 +37,6 @@ def transition(self, key, start, finish, **kwargs):
     def release_key(self, key, state, cause, reason, report):
         self.observed_notifications.append({"key": key, "state": state})
 
-    def release_dep(self, dep, state, report):
-        self.observed_notifications.append({"dep": dep, "state": state})
-
 
 @gen_cluster(client=True, nthreads=[])
 async def test_create_with_client(c, s):
@@ -164,7 +161,7 @@ async def test_superseding_task_transitions_called(c, s, w):
 
 
 @gen_cluster(nthreads=[("127.0.0.1", 1)], client=True)
-async def test_release_dep_called(c, s, w):
+async def test_dependent_tasks(c, s, w):
     dsk = {"dep": 1, "task": (inc, "dep")}
 
     expected_notifications = [

From 1afd7479d637efe47e4070d572cb3205cdd1ea97 Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Fri, 7 May 2021 16:15:38 +0100
Subject: [PATCH 1264/1550] Allow the dashboard to run on multiple ports
 (#4786)

---
 distributed/comm/__init__.py        |  2 +-
 distributed/comm/utils.py           | 26 ++++++----
 distributed/node.py                 | 77 ++++++++++++++++-------------
 distributed/tests/test_scheduler.py | 12 +++++
 distributed/utils.py                | 59 +++++++++++++---------
 5 files changed, 109 insertions(+), 67 deletions(-)

diff --git a/distributed/comm/__init__.py b/distributed/comm/__init__.py
index fe227ec1ff4..5ca2d1ede33 100644
--- a/distributed/comm/__init__.py
+++ b/distributed/comm/__init__.py
@@ -10,7 +10,7 @@
     unparse_host_port,
 )
 from .core import Comm, CommClosedError, connect, listen
-from .utils import get_tcp_server_address
+from .utils import get_tcp_server_address, get_tcp_server_addresses
 
 
 def _register_transports():
diff --git a/distributed/comm/utils.py b/distributed/comm/utils.py
index 15b9244329a..35f3c33ef3d 100644
--- a/distributed/comm/utils.py
+++ b/distributed/comm/utils.py
@@ -81,29 +81,37 @@ def _from_frames():
     return res
 
 
-def get_tcp_server_address(tcp_server):
+def get_tcp_server_addresses(tcp_server):
     """
-    Get the bound address of a started Tornado TCPServer.
+    Get all bound addresses of a started Tornado TCPServer.
     """
     sockets = list(tcp_server._sockets.values())
     if not sockets:
         raise RuntimeError("TCP Server %r not started yet?" % (tcp_server,))
 
     def _look_for_family(fam):
+        socks = []
         for sock in sockets:
             if sock.family == fam:
-                return sock
-        return None
+                socks.append(sock)
+        return socks
 
     # If listening on both IPv4 and IPv6, prefer IPv4 as defective IPv6
     # is common (e.g. Travis-CI).
-    sock = _look_for_family(socket.AF_INET)
-    if sock is None:
-        sock = _look_for_family(socket.AF_INET6)
-    if sock is None:
+    socks = _look_for_family(socket.AF_INET)
+    if not socks:
+        socks = _look_for_family(socket.AF_INET6)
+    if not socks:
         raise RuntimeError("No Internet socket found on TCPServer??")
 
-    return sock.getsockname()
+    return [sock.getsockname() for sock in socks]
+
+
+def get_tcp_server_address(tcp_server):
+    """
+    Get the first bound address of a started Tornado TCPServer.
+    """
+    return get_tcp_server_addresses(tcp_server)[0]
 
 
 def ensure_concrete_host(host):
diff --git a/distributed/node.py b/distributed/node.py
index 21000a04733..eb150ca0418 100644
--- a/distributed/node.py
+++ b/distributed/node.py
@@ -8,7 +8,7 @@
 
 import dask
 
-from .comm import get_address_host, get_tcp_server_address
+from .comm import get_address_host, get_tcp_server_addresses
 from .core import Server
 from .http.routing import RoutingApplication
 from .utils import DequeHandler, clean_dashboard_address
@@ -119,39 +119,48 @@ def start_http_server(
             ssl_options.verify_mode = ssl.CERT_NONE
 
         self.http_server = HTTPServer(self.http_application, ssl_options=ssl_options)
-        http_address = clean_dashboard_address(dashboard_address or default_port)
-
-        if http_address["address"] is None:
-            address = self._start_address
-            if isinstance(address, (list, tuple)):
-                address = address[0]
-            if address:
-                with suppress(ValueError):
-                    http_address["address"] = get_address_host(address)
-
-        change_port = False
-        retries_left = 3
-        while True:
-            try:
-                if not change_port:
-                    self.http_server.listen(**http_address)
-                else:
-                    self.http_server.listen(**tlz.merge(http_address, {"port": 0}))
-                break
-            except Exception:
-                change_port = True
-                retries_left = retries_left - 1
-                if retries_left < 1:
-                    raise
-
-        self.http_server.port = get_tcp_server_address(self.http_server)[1]
+
+        http_addresses = clean_dashboard_address(dashboard_address or default_port)
+
+        for http_address in http_addresses:
+            if http_address["address"] is None:
+                address = self._start_address
+                if isinstance(address, (list, tuple)):
+                    address = address[0]
+                if address:
+                    with suppress(ValueError):
+                        http_address["address"] = get_address_host(address)
+
+            change_port = False
+            retries_left = 3
+            while True:
+                try:
+                    if not change_port:
+                        self.http_server.listen(**http_address)
+                    else:
+                        self.http_server.listen(**tlz.merge(http_address, {"port": 0}))
+                    break
+                except Exception:
+                    change_port = True
+                    retries_left = retries_left - 1
+                    if retries_left < 1:
+                        raise
+
+        bound_addresses = get_tcp_server_addresses(self.http_server)
+
+        # If more than one address is configured we just use the first here
+        self.http_server.port = bound_addresses[0][1]
         self.services["dashboard"] = self.http_server
 
-        if change_port and dashboard_address:
-            warnings.warn(
-                "Port {} is already in use.\n"
-                "Perhaps you already have a cluster running?\n"
-                "Hosting the HTTP server on port {} instead".format(
-                    http_address["port"], self.http_server.port
+        # Warn on port changes
+        for expected, actual in zip(
+            [a["port"] for a in http_addresses], [b[1] for b in bound_addresses]
+        ):
+            if expected != actual and expected > 0:
+                warnings.warn(
+                    "Port {} is already in use.\n"
+                    "Perhaps you already have a cluster running?\n"
+                    "Hosting the HTTP server on port {} instead".format(
+                        expected, actual
+                    )
                 )
-            )
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 18e9dc5a259..367e367c797 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -1728,6 +1728,18 @@ async def test_dashboard_address():
     assert s.services["dashboard"].port
     await s.close()
 
+    s = await Scheduler(dashboard_address="127.0.0.1:8901,127.0.0.1:8902", port=0)
+    assert s.services["dashboard"].port == 8901
+    await s.close()
+
+    s = await Scheduler(dashboard_address=":8901,:8902", port=0)
+    assert s.services["dashboard"].port == 8901
+    await s.close()
+
+    s = await Scheduler(dashboard_address=[8901, 8902], port=0)
+    assert s.services["dashboard"].port == 8901
+    await s.close()
+
 
 @gen_cluster(client=True)
 async def test_adaptive_target(c, s, a, b):
diff --git a/distributed/utils.py b/distributed/utils.py
index a7b8befc221..d8eb2666136 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -24,6 +24,7 @@
 from hashlib import md5
 from importlib.util import cache_from_source
 from time import sleep
+from typing import Any, Dict, List
 
 import click
 import tblib.pickling_support
@@ -1506,42 +1507,54 @@ def __setitem__(self, key, value):
         super().__setitem__(key, value)
 
 
-def clean_dashboard_address(addr, default_listen_ip=""):
+def clean_dashboard_address(addrs: Any, default_listen_ip: str = "") -> List[Dict]:
     """
     Examples
     --------
     >>> clean_dashboard_address(8787)
-    {'address': '', 'port': 8787}
+    [{'address': '', 'port': 8787}]
     >>> clean_dashboard_address(":8787")
-    {'address': '', 'port': 8787}
+    [{'address': '', 'port': 8787}]
     >>> clean_dashboard_address("8787")
-    {'address': '', 'port': 8787}
+    [{'address': '', 'port': 8787}]
     >>> clean_dashboard_address("8787")
-    {'address': '', 'port': 8787}
+    [{'address': '', 'port': 8787}]
     >>> clean_dashboard_address("foo:8787")
-    {'address': 'foo', 'port': 8787}
+    [{'address': 'foo', 'port': 8787}]
+    >>> clean_dashboard_address([8787, 8887])
+    [{'address': '', 'port': 8787}, {'address': '', 'port': 8887}]
+    >>> clean_dashboard_address(":8787,:8887")
+    [{'address': '', 'port': 8787}, {'address': '', 'port': 8887}]
     """
 
     if default_listen_ip == "0.0.0.0":
         default_listen_ip = ""  # for IPV6
 
-    try:
-        addr = int(addr)
-    except (TypeError, ValueError):
-        pass
+    if isinstance(addrs, str):
+        addrs = addrs.split(",")
+    if not isinstance(addrs, list):
+        addrs = [addrs]
 
-    if isinstance(addr, str):
-        addr = addr.split(":")
+    addresses = []
+    for addr in addrs:
+        try:
+            addr = int(addr)
+        except (TypeError, ValueError):
+            pass
 
-    if isinstance(addr, (tuple, list)):
-        if len(addr) == 2:
-            host, port = (addr[0], int(addr[1]))
-        elif len(addr) == 1:
-            [host], port = addr, 0
-        else:
-            raise ValueError(addr)
-    elif isinstance(addr, int):
-        host = default_listen_ip
-        port = addr
+        if isinstance(addr, str):
+            addr = addr.split(":")
+
+        if isinstance(addr, (tuple, list)):
+            if len(addr) == 2:
+                host, port = (addr[0], int(addr[1]))
+            elif len(addr) == 1:
+                [host], port = addr, 0
+            else:
+                raise ValueError(addr)
+        elif isinstance(addr, int):
+            host = default_listen_ip
+            port = addr
 
-    return {"address": host, "port": port}
+        addresses.append({"address": host, "port": port})
+    return addresses

From a2a0feb33b19eea27f0ff73fe12acfa8a90d73b6 Mon Sep 17 00:00:00 2001
From: Ian Rose <ian.r.rose@gmail.com>
Date: Mon, 10 May 2021 04:11:54 -0700
Subject: [PATCH 1265/1550] Use JupyterLab theme variables (with fallbacks for
 other notebook (#4796)

viewers) for html reprs.
---
 distributed/client.py         | 18 +++++++++++++-----
 distributed/deploy/cluster.py |  5 +++--
 2 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index cddc07e5a70..1f57fc4ac21 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -399,19 +399,27 @@ def __repr__(self):
     def _repr_html_(self):
         text = "<b>Future: %s</b> " % html.escape(key_split(self.key))
         text += (
-            '<font color="gray">status: </font>'
-            '<font color="%(color)s">%(status)s</font>, '
+            '<font style="color: var(--jp-ui-font-color2, gray)">status: </font>'
+            '<font style="color: %(color)s">%(status)s</font>, '
         ) % {
             "status": self.status,
-            "color": "red" if self.status == "error" else "black",
+            "color": "var(--jp-error-color0, red)"
+            if self.status == "error"
+            else "var(--jp-ui-font-color0, black)",
         }
         if self.type:
             try:
                 typ = self.type.__module__.split(".")[0] + "." + self.type.__name__
             except AttributeError:
                 typ = str(self.type)
-            text += '<font color="gray">type: </font>%s, ' % typ
-        text += '<font color="gray">key: </font>%s' % html.escape(str(self.key))
+            text += (
+                '<font style="color: var(--jp-ui-font-color2, gray)">type: </font>%s, '
+                % typ
+            )
+        text += (
+            '<font style="color: var(--jp-ui-font-color2, gray)">key: </font>%s'
+            % html.escape(str(self.key))
+        )
         return text
 
     def __await__(self):
diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
index b872e266176..39a6aa3650c 100644
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -388,8 +388,9 @@ def _repr_html_(self):
         else:
             dashboard = "Not Available"
         return (
-            "<div style='background-color: #f2f2f2; display: inline-block; "
-            "padding: 10px; border: 1px solid #999999;'>\n"
+            "<div style='color: var(--jp-ui-font-color0, #000000); "
+            "background-color: var(--jp-layout-color2, #f2f2f2); display: inline-block; "
+            "padding: 10px; border: 1px solid var(--jp-border-color0, #999999);'>\n"
             "  <h3>{cls}</h3>\n"
             "  <ul>\n"
             "    <li><b>Dashboard: </b>{dashboard}\n"

From 5e150aa6a3bd7decd38193a3d754497f8914d83c Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Mon, 10 May 2021 11:09:12 -0500
Subject: [PATCH 1266/1550] Add HTML reprs to some scheduler classes (#4795)

---
 distributed/scheduler.py | 39 ++++++++++++++++++++++++++++++++++++---
 1 file changed, 36 insertions(+), 3 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index fb9610c53d5..33f0a0b0449 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1,4 +1,5 @@
 import asyncio
+import html
 import inspect
 import itertools
 import json
@@ -732,13 +733,22 @@ def clean(self):
         return ws
 
     def __repr__(self):
-        return "<Worker %r, name: %s, memory: %d, processing: %d>" % (
+        return "<WorkerState %r, name: %s, memory: %d, processing: %d>" % (
             self._address,
             self._name,
             len(self._has_what),
             len(self._processing),
         )
 
+    def _repr_html_(self):
+        text = (
+            f"<b>WorkerState: </b> {html.escape(self._address)} "
+            f'<font style="color: var(--jp-ui-font-color2, gray)">name: </font>{self.name} '
+            f'<font style="color: var(--jp-ui-font-color2, gray)">memory: </font>{len(self._has_what)} '
+            f'<font style="color: var(--jp-ui-font-color2, gray)">processing: </font>{len(self._processing)}'
+        )
+        return text
+
     @ccall
     @exceptval(check=False)
     def identity(self) -> dict:
@@ -1549,7 +1559,19 @@ def set_nbytes(self, nbytes: Py_ssize_t):
         self._nbytes = nbytes
 
     def __repr__(self):
-        return "<Task %r %s>" % (self._key, self._state)
+        return "<TaskState %r %s>" % (self._key, self._state)
+
+    def _repr_html_(self):
+        color = (
+            "var(--jp-error-color0, red)"
+            if self._state == "erred"
+            else "var(--jp-ui-font-color0, black)"
+        )
+        text = f'<b>TaskState: </b> <font style="color: {color}">{self._state} </font>'
+        if self._state == "memory":
+            text += f'<font style="color: var(--jp-ui-font-color2, gray)">nbytes: </font>{format_bytes(self._nbytes)} '
+        text += f'<font style="color: var(--jp-ui-font-color2, gray)">key: </font>{html.escape(self._key)}'
+        return text
 
     @ccall
     def validate(self):
@@ -3579,11 +3601,22 @@ def __init__(
 
     def __repr__(self):
         parent: SchedulerState = cast(SchedulerState, self)
-        return '<Scheduler: "%s" processes: %d cores: %d>' % (
+        return '<Scheduler: "%s" workers: %d cores: %d, tasks: %d>' % (
             self.address,
             len(parent._workers),
             parent._total_nthreads,
+            len(parent._tasks),
+        )
+
+    def _repr_html_(self):
+        parent: SchedulerState = cast(SchedulerState, self)
+        text = (
+            f"<b>Scheduler: </b>{html.escape(self.address)} "
+            f'<font color="gray">workers: </font>{len(parent._workers)} '
+            f'<font color="gray">cores: </font>{parent._total_nthreads} '
+            f'<font color="gray">tasks: </font>{len(parent._tasks)}'
         )
+        return text
 
     def identity(self, comm=None):
         """ Basic information about ourselves and our cluster """

From aa9f8d92bf22abb97a913afe16db726868571e9c Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Wed, 12 May 2021 14:48:29 -0500
Subject: [PATCH 1267/1550] Add support for `click` 8 (#4810)

---
 distributed/cli/dask_scheduler.py |  1 -
 distributed/cli/utils.py          | 16 ++++++++++++++--
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/distributed/cli/dask_scheduler.py b/distributed/cli/dask_scheduler.py
index 51ac13f2907..c6297eda5a3 100755
--- a/distributed/cli/dask_scheduler.py
+++ b/distributed/cli/dask_scheduler.py
@@ -103,7 +103,6 @@
     type=str,
     multiple=True,
     is_eager=True,
-    default="",
     help="Module that should be loaded by the scheduler process  "
     'like "foo.bar" or "/path/to/foo.py".',
 )
diff --git a/distributed/cli/utils.py b/distributed/cli/utils.py
index b2515faff11..7e812cf7f01 100644
--- a/distributed/cli/utils.py
+++ b/distributed/cli/utils.py
@@ -1,5 +1,10 @@
+from distutils.version import LooseVersion
+
+import click
 from tornado.ioloop import IOLoop
 
+CLICK_VERSION = LooseVersion(click.__version__)
+
 py3_err_msg = """
 Warning: Your terminal does not set locales.
 
@@ -24,12 +29,19 @@ def check_python_3():
     # https://github.com/pallets/click/issues/448#issuecomment-246029304
     import click.core
 
-    click.core._verify_python3_env = lambda: None
+    # TODO: Remove use of internal click functions
+    if CLICK_VERSION < "8.0.0":
+        click.core._verify_python3_env = lambda: None
+    else:
+        click.core._verify_python_env = lambda: None
 
     try:
         from click import _unicodefun
 
-        _unicodefun._verify_python3_env()
+        if CLICK_VERSION < "8.0.0":
+            _unicodefun._verify_python3_env()
+        else:
+            _unicodefun._verify_python_env()
     except (TypeError, RuntimeError) as e:
         import click
 

From d6f5609ac32d899e9ea30758e8033c3c614656eb Mon Sep 17 00:00:00 2001
From: "Mads R. B. Kristensen" <madsbk@gmail.com>
Date: Thu, 13 May 2021 23:59:40 +0200
Subject: [PATCH 1268/1550] Merge global annotations on the client (#4691)

---
 distributed/client.py            | 10 +++++-----
 distributed/scheduler.py         |  6 ++++--
 distributed/tests/test_client.py | 20 ++++++++++++++++++++
 3 files changed, 29 insertions(+), 7 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 1f57fc4ac21..293081fe330 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -2553,14 +2553,10 @@ def _graph_to_futures(
             if actors is not None and actors is not True and actors is not False:
                 actors = list(self._expand_key(actors))
 
-            keyset = set(keys)
-
             # Make sure `dsk` is a high level graph
             if not isinstance(dsk, HighLevelGraph):
                 dsk = HighLevelGraph.from_collections(id(dsk), dsk, dependencies=())
 
-            dsk = dsk.__dask_distributed_pack__(self, keyset)
-
             annotations = {}
             if user_priority:
                 annotations["priority"] = user_priority
@@ -2577,8 +2573,13 @@ def _graph_to_futures(
             if resources:
                 annotations["resources"] = resources
 
+            # Merge global and local annotations
             annotations = merge(dask.config.get("annotations", {}), annotations)
 
+            # Pack the high level graph before sending it to the scheduler
+            keyset = set(keys)
+            dsk = dsk.__dask_distributed_pack__(self, keyset, annotations)
+
             # Create futures before sending graph (helps avoid contention)
             futures = {key: Future(key, self, inform=False) for key in keyset}
             self._send_to_scheduler(
@@ -2589,7 +2590,6 @@ def _graph_to_futures(
                     "priority": priority,
                     "submitting_task": getattr(thread_state, "key", None),
                     "fifo_timeout": fifo_timeout,
-                    "annotations": annotations,
                     "actors": actors,
                 }
             )
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 33f0a0b0449..780bb2a3dcd 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -4087,9 +4087,8 @@ def update_graph_hlg(
         user_priority=0,
         actors=None,
         fifo_timeout=0,
-        annotations=None,
     ):
-        unpacked_graph = HighLevelGraph.__dask_distributed_unpack__(hlg, annotations)
+        unpacked_graph = HighLevelGraph.__dask_distributed_unpack__(hlg)
         dsk = unpacked_graph["dsk"]
         dependencies = unpacked_graph["deps"]
         annotations = unpacked_graph["annotations"]
@@ -4329,6 +4328,9 @@ def update_graph(
                     continue
                 ts._host_restrictions = set()
                 ts._worker_restrictions = set()
+                # Make sure `v` is a collection and not a single worker name / address
+                if not isinstance(v, (list, tuple, set)):
+                    v = [v]
                 for w in v:
                     try:
                         w = self.coerce_address(w)
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 57bdc67b17e..5a28c83a1dd 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -4143,6 +4143,26 @@ async def test_persist_workers_annotate(e, s, a, b, c):
     assert s.loose_restrictions == {total2.key} | {v.key for v in L2}
 
 
+@gen_cluster(nthreads=[("127.0.0.1", 1)] * 3, client=True)
+async def test_persist_workers_annotate2(e, s, a, b, c):
+    def key_to_worker(key):
+        return a.address
+
+    L1 = [delayed(inc)(i) for i in range(4)]
+    for x in L1:
+        assert all(layer.annotations is None for layer in x.dask.layers.values())
+
+    with dask.annotate(workers=key_to_worker):
+        out = e.persist(L1, optimize_graph=False)
+        await wait(out)
+
+    for x in L1:
+        assert all(layer.annotations is None for layer in x.dask.layers.values())
+
+    for v in L1:
+        assert s.worker_restrictions[v.key] == {a.address}
+
+
 @nodebug  # test timing is fragile
 @gen_cluster(nthreads=[("127.0.0.1", 1)] * 3, client=True)
 async def test_persist_workers(e, s, a, b, c):

From c24c8362be587408a93aefafe1e4fcc509ef2c91 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Fri, 14 May 2021 14:17:24 -0500
Subject: [PATCH 1269/1550] bump version to 2021.05.0

---
 docs/source/changelog.rst | 31 ++++++++++++++++++++++++++++++-
 requirements.txt          |  2 +-
 2 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index 06a1c7cf0bc..97174ef9210 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,33 @@
 Changelog
 =========
 
+2021.05.0
+---------
+
+Released on May 14, 2021
+
+- Merge global annotations on the client (:pr:`4691`) `Mads R. B. Kristensen`_
+- Add support for ``click`` 8 (:pr:`4810`) `James Bourbeau`_
+- Add HTML reprs to some scheduler classes (:pr:`4795`) `James Bourbeau`_
+- Use JupyterLab theme variables (:pr:`4796`) `Ian Rose`_
+- Allow the dashboard to run on multiple ports (:pr:`4786`) `Jacob Tomlinson`_
+- Remove ``release_dep`` from ``WorkerPlugin`` API (:pr:`4791`) `James Bourbeau`_
+- Support for UCX 1.10+ (:pr:`4787`) `Peter Andreas Entschev`_
+- Reduce complexity of ``test_gather_allow_worker_reconnect`` (:pr:`4739`) `Florian Jetter`_
+- Fix doctests in ``utils.py`` (:pr:`4785`) `Jacob Tomlinson`_
+- Ensure deps are actually logged in worker (:pr:`4753`) `Florian Jetter`_
+- Add ``stacklevel`` keyword into ``performance_report()`` to allow for selecting calling code to be displayed (:pr:`4777`) `Nathan Danielsen`_
+- Unregister worker plugin (:pr:`4748`) `Naty Clementi`_
+- Fixes some pickling issues in the Cythonized ``Scheduler`` (:pr:`4768`) `jakirkham`_
+- Improve graceful shutdown if nanny is involved (:pr:`4725`) `Florian Jetter`_
+- Update cythonization in CI (:pr:`4764`) `James Bourbeau`_
+- Use ``contextlib.nullcontext`` (:pr:`4763`) `James Bourbeau`_
+- Cython fixes for ``MemoryState`` (:pr:`4761`) `jakirkham`_
+- Fix errors in ``check_thread_leak`` (:pr:`4747`) `James Bourbeau`_
+- Handle missing ``key`` case in ``report_on_key`` (:pr:`4755`) `jakirkham`_
+- Drop temporary ``set`` variables ``s`` (:pr:`4758`) `jakirkham`_
+
+
 2021.04.1
 ---------
 
@@ -2333,4 +2360,6 @@ significantly without many new features.
 .. _`Marcos Moyano`: https://github.com/marcosmoyano
 .. _`James Lamb`: https://github.com/jameslamb
 .. _`Hristo Georgiev`: https://github.com/hristog
-.. _`Matteo De Wint`: https://github.com/mdwint
\ No newline at end of file
+.. _`Matteo De Wint`: https://github.com/mdwint
+.. _`Naty Clementi`: https://github.com/ncclementi
+.. _`Nathan Danielsen`: https://github.com/ndanielsen
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index fd8159cfefe..19032cf740b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,6 @@
 click >= 6.6
 cloudpickle >= 1.5.0
-dask >= 2021.03.0
+dask == 2021.05.0
 msgpack >= 0.6.0
 psutil >= 5.0
 sortedcontainers !=2.0.0, !=2.0.1

From 7dd4879bbde71cc8da28ebd1918195238753a891 Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Fri, 14 May 2021 22:22:48 +0200
Subject: [PATCH 1270/1550] Increase timeout of `gen_test` to 30s (#4821)

---
 distributed/deploy/tests/test_adaptive.py | 2 +-
 distributed/tests/test_nanny.py           | 2 +-
 distributed/utils_test.py                 | 6 +++---
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/distributed/deploy/tests/test_adaptive.py b/distributed/deploy/tests/test_adaptive.py
index 2e460e5e150..7adf7a5611a 100644
--- a/distributed/deploy/tests/test_adaptive.py
+++ b/distributed/deploy/tests/test_adaptive.py
@@ -312,7 +312,7 @@ async def test_adapt_down():
                 assert time() < start + 60
 
 
-@gen_test(timeout=30)
+@gen_test()
 async def test_no_more_workers_than_tasks():
     with dask.config.set(
         {"distributed.scheduler.default-task-durations": {"slowinc": 1000}}
diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
index fd07e308f7b..779cbdc4a4a 100644
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -367,7 +367,7 @@ async def test_scheduler_address_config(c, s):
 
 
 @pytest.mark.slow
-@gen_test(timeout=20)
+@gen_test()
 async def test_wait_for_scheduler():
     with captured_logger("distributed") as log:
         w = Nanny("127.0.0.1:44737")
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index a95d089bd20..c05f142b4f1 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -78,7 +78,7 @@
     if isinstance(logger, logging.Logger)
 }
 
-
+_TEST_TIMEOUT = 30
 _offload_executor.submit(lambda: None).result()  # create thread during import
 
 
@@ -750,7 +750,7 @@ async def disconnect_all(addresses, timeout=3, rpc_kwargs=None):
     await asyncio.gather(*[disconnect(addr, timeout, rpc_kwargs) for addr in addresses])
 
 
-def gen_test(timeout=10):
+def gen_test(timeout=_TEST_TIMEOUT):
     """Coroutine test
 
     @gen_test(timeout=5)
@@ -839,7 +839,7 @@ def gen_cluster(
     nthreads=[("127.0.0.1", 1), ("127.0.0.1", 2)],
     ncores=None,
     scheduler="127.0.0.1",
-    timeout=30,
+    timeout=_TEST_TIMEOUT,
     security=None,
     Worker=Worker,
     client=False,

From a4b6798ec742379777600bcbefccc09255b0ca73 Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Mon, 17 May 2021 17:29:24 +0100
Subject: [PATCH 1271/1550] `test_memory` to use `gen_cluster` (#4811)

---
 distributed/tests/test_scheduler.py | 174 ++++++++++++++--------------
 1 file changed, 88 insertions(+), 86 deletions(-)

diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 367e367c797..10d731fffe1 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -2328,7 +2328,7 @@ def clear_leak():
     gc.collect()
 
 
-def assert_memory(scheduler_or_workerstate, attr: str, min_, max_, timeout=10):
+async def assert_memory(scheduler_or_workerstate, attr: str, min_, max_, timeout=10):
     t0 = time()
     while True:
         minfo = scheduler_or_workerstate.memory
@@ -2339,7 +2339,7 @@ def assert_memory(scheduler_or_workerstate, attr: str, min_, max_, timeout=10):
             raise TimeoutError(
                 f"Expected {min_} MiB <= {attr} <= {max_} MiB; got:\n{minfo!r}"
             )
-        sleep(0.1)
+        await asyncio.sleep(0.1)
 
 
 # This test is heavily influenced by hard-to-control factors such as memory management
@@ -2347,93 +2347,95 @@ def assert_memory(scheduler_or_workerstate, attr: str, min_, max_, timeout=10):
 @pytest.mark.flaky(reruns=3, reruns_delay=5)
 # ~33s runtime, or distributed.memory.recent_to_old_time + 3s
 @pytest.mark.slow
-def test_memory():
+@gen_cluster(
+    client=True, Worker=Nanny, worker_kwargs={"memory_limit": "500 MiB"}, timeout=60
+)
+async def test_memory(c, s, *_):
     pytest.importorskip("zict")
 
-    with Client(n_workers=2, threads_per_worker=1, memory_limit=500 * 2 ** 20) as c:
-        c.wait_for_workers(2)
-        s = c.cluster.scheduler
-        a, b = s.workers.values()
-
-        s_m0 = s.memory
-        assert s_m0.process == a.memory.process + b.memory.process
-        assert s_m0.managed == 0
-        assert a.memory.managed == 0
-        assert b.memory.managed == 0
-        # When a worker first goes online, its RAM is immediately counted as
-        # unmanaged_old
-        assert_memory(s, "unmanaged_recent", 0, 40, timeout=0)
-        assert_memory(a, "unmanaged_recent", 0, 20, timeout=0)
-        assert_memory(b, "unmanaged_recent", 0, 20, timeout=0)
-
-        f1 = c.submit(leaking, 100, 50, 5, pure=False, workers=[a.name])
-        f2 = c.submit(leaking, 100, 50, 5, pure=False, workers=[b.name])
-        assert_memory(s, "unmanaged_recent", 300, 380)
-        assert_memory(a, "unmanaged_recent", 150, 190)
-        assert_memory(b, "unmanaged_recent", 150, 190)
-        c.gather([f1, f2])
-
-        # On each worker, we now have 100 MiB managed + 50 MiB fresh leak
-        assert_memory(s, "managed_in_memory", 200, 201)
-        assert_memory(a, "managed_in_memory", 100, 101)
-        assert_memory(b, "managed_in_memory", 100, 101)
-        assert_memory(s, "unmanaged_recent", 100, 180)
-        assert_memory(a, "unmanaged_recent", 50, 90)
-        assert_memory(b, "unmanaged_recent", 50, 90)
-
-        # Force the output of f1 and f2 to spill to disk.
-        # With target=0.6 and memory_limit=500 MiB, we'll start spilling at 300 MiB
-        # process memory per worker, or roughly after 3~7 rounds of the below depending
-        # on how much RAM the interpreter is using.
-        more_futs = []
-        for _ in range(8):
-            if s.memory.managed_spilled > 0:
-                break
-            more_futs += [
-                c.submit(leaking, 20, 0, 0, pure=False, workers=[a.name]),
-                c.submit(leaking, 20, 0, 0, pure=False, workers=[b.name]),
-            ]
-            sleep(2)
-        assert_memory(s, "managed_spilled", 1, 999)
-        # Wait for the spilling to finish. Note that this does not make the test take
-        # longer as we're waiting for recent_to_old_time anyway.
-        sleep(10)
-
-        # Delete spilled keys
-        prev = s.memory
-        del f1
-        del f2
-        assert_memory(s, "managed_spilled", 0, prev.managed_spilled / 2 ** 20 - 19)
-
-        # Empty the cluster, with the exception of leaked memory
-        del more_futs
-        assert_memory(s, "managed", 0, 0)
-
-        orig_unmanaged = s_m0.unmanaged / 2 ** 20
-        orig_old = s_m0.unmanaged_old / 2 ** 20
-
-        # Wait until 30s have passed since the spill to observe unmanaged_recent
-        # transition into unmanaged_old
-        c.run(gc.collect)
-        assert_memory(s, "unmanaged_recent", 0, 90, timeout=40)
-        assert_memory(
-            s,
-            "unmanaged_old",
-            orig_old + 90,
-            # On MacOS, the process memory of the Python interpreter does not shrink as
-            # fast as on Linux/Windows
-            9999 if MACOS else orig_old + 190,
-            timeout=40,
-        )
+    # WorkerState objects, as opposed to the Nanny objects passed by gen_cluster
+    a, b = s.workers.values()
+
+    s_m0 = s.memory
+    assert s_m0.process == a.memory.process + b.memory.process
+    assert s_m0.managed == 0
+    assert a.memory.managed == 0
+    assert b.memory.managed == 0
+    # When a worker first goes online, its RAM is immediately counted as
+    # unmanaged_old
+    await assert_memory(s, "unmanaged_recent", 0, 40, timeout=0)
+    await assert_memory(a, "unmanaged_recent", 0, 20, timeout=0)
+    await assert_memory(b, "unmanaged_recent", 0, 20, timeout=0)
+
+    f1 = c.submit(leaking, 100, 50, 5, pure=False, workers=[a.name])
+    f2 = c.submit(leaking, 100, 50, 5, pure=False, workers=[b.name])
+    await assert_memory(s, "unmanaged_recent", 300, 380)
+    await assert_memory(a, "unmanaged_recent", 150, 190)
+    await assert_memory(b, "unmanaged_recent", 150, 190)
+    await wait([f1, f2])
+
+    # On each worker, we now have 100 MiB managed + 50 MiB fresh leak
+    await assert_memory(s, "managed_in_memory", 200, 201)
+    await assert_memory(a, "managed_in_memory", 100, 101)
+    await assert_memory(b, "managed_in_memory", 100, 101)
+    await assert_memory(s, "unmanaged_recent", 100, 180)
+    await assert_memory(a, "unmanaged_recent", 50, 90)
+    await assert_memory(b, "unmanaged_recent", 50, 90)
+
+    # Force the output of f1 and f2 to spill to disk.
+    # With target=0.6 and memory_limit=500 MiB, we'll start spilling at 300 MiB
+    # process memory per worker, or roughly after 3~7 rounds of the below depending
+    # on how much RAM the interpreter is using.
+    more_futs = []
+    for _ in range(8):
+        if s.memory.managed_spilled > 0:
+            break
+        more_futs += [
+            c.submit(leaking, 20, 0, 0, pure=False, workers=[a.name]),
+            c.submit(leaking, 20, 0, 0, pure=False, workers=[b.name]),
+        ]
+        await asyncio.sleep(2)
+    await assert_memory(s, "managed_spilled", 1, 999)
+
+    # Wait for the spilling to finish. Note that this does not make the test take
+    # longer as we're waiting for recent_to_old_time anyway.
+    await asyncio.sleep(10)
+
+    # Delete spilled keys
+    prev = s.memory
+    del f1
+    del f2
+    await assert_memory(s, "managed_spilled", 0, prev.managed_spilled / 2 ** 20 - 19)
+
+    # Empty the cluster, with the exception of leaked memory
+    del more_futs
+    await assert_memory(s, "managed", 0, 0)
+
+    orig_unmanaged = s_m0.unmanaged / 2 ** 20
+    orig_old = s_m0.unmanaged_old / 2 ** 20
+
+    # Wait until 30s have passed since the spill to observe unmanaged_recent
+    # transition into unmanaged_old
+    await c.run(gc.collect)
+    await assert_memory(s, "unmanaged_recent", 0, 90, timeout=40)
+    await assert_memory(
+        s,
+        "unmanaged_old",
+        orig_old + 90,
+        # On MacOS, the process memory of the Python interpreter does not shrink as
+        # fast as on Linux/Windows
+        9999 if MACOS else orig_old + 190,
+        timeout=40,
+    )
 
-        # When the leaked memory is cleared, unmanaged and unmanaged_old drop
-        # On MacOS, the process memory of the Python interpreter does not shrink as fast
-        # as on Linux/Windows
-        if not MACOS:
-            c.run(clear_leak)
-            assert_memory(s, "unmanaged", 0, orig_unmanaged + 95)
-            assert_memory(s, "unmanaged_old", 0, orig_old + 95)
-            assert_memory(s, "unmanaged_recent", 0, 90)
+    # When the leaked memory is cleared, unmanaged and unmanaged_old drop
+    # On MacOS, the process memory of the Python interpreter does not shrink as fast
+    # as on Linux/Windows
+    if not MACOS:
+        await c.run(clear_leak)
+        await assert_memory(s, "unmanaged", 0, orig_unmanaged + 95)
+        await assert_memory(s, "unmanaged_old", 0, orig_old + 95)
+        await assert_memory(s, "unmanaged_recent", 0, 90)
 
 
 @gen_cluster(client=True, worker_kwargs={"memory_limit": 0})

From 7e10875528d955947340f6f95da54f6919bdd815 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Torsten=20W=C3=B6rtwein?=
 <twoertwein@users.noreply.github.com>
Date: Thu, 20 May 2021 09:29:32 -0400
Subject: [PATCH 1272/1550] adtive_target: do not mindlessly spawn workers when
 no memory limit is set (#4397)

---
 distributed/deploy/tests/test_adaptive.py | 19 +++++++++++++++++++
 distributed/scheduler.py                  | 15 +++++----------
 2 files changed, 24 insertions(+), 10 deletions(-)

diff --git a/distributed/deploy/tests/test_adaptive.py b/distributed/deploy/tests/test_adaptive.py
index 7adf7a5611a..f43f097c076 100644
--- a/distributed/deploy/tests/test_adaptive.py
+++ b/distributed/deploy/tests/test_adaptive.py
@@ -460,3 +460,22 @@ async def test_update_adaptive(cleanup):
         await asyncio.sleep(0.2)
         assert first.periodic_callback is None
         assert second.periodic_callback.is_running()
+
+
+@pytest.mark.asyncio
+async def test_adaptive_no_memory_limit(cleanup):
+    """Make sure that adapt() does not keep creating workers when no memory limit is set."""
+    async with LocalCluster(
+        n_workers=0, threads_per_worker=1, memory_limit=0, asynchronous=True
+    ) as cluster:
+        cluster.adapt(minimum=1, maximum=10, interval="1 ms")
+        async with Client(cluster, asynchronous=True) as client:
+            await client.gather(client.map(slowinc, range(5), delay=0.35))
+        assert (
+            sum(
+                state[1]["n"]
+                for state in cluster._adaptive.log
+                if state[1]["status"] == "up"
+            )
+            <= 5
+        )
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 780bb2a3dcd..3830e13fe67 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -6898,17 +6898,12 @@ def adaptive_target(self, comm=None, target_duration=None):
         if parent._unrunnable and not parent._workers_dv:
             cpu = max(1, cpu)
 
-        # Memory
-        limit_bytes = {
-            addr: ws._memory_limit for addr, ws in parent._workers_dv.items()
-        }
-        worker_bytes = [ws._nbytes for ws in parent._workers_dv.values()]
-        limit = sum(limit_bytes.values())
-        total = sum(worker_bytes)
-        if total > 0.6 * limit:
+        # add more workers if more than 60% of memory is used
+        limit = sum([ws._memory_limit for ws in parent._workers_dv.values()])
+        used = sum([ws._nbytes for ws in parent._workers_dv.values()])
+        memory = 0
+        if used > 0.6 * limit and limit > 0:
             memory = 2 * len(parent._workers_dv)
-        else:
-            memory = 0
 
         target = max(memory, cpu)
         if target >= len(parent._workers_dv):

From 0a14902ecc88938c30e48e22f0bc1478f47bba7d Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Thu, 20 May 2021 16:53:23 -0500
Subject: [PATCH 1273/1550] Bump pre-commit hook versions (#4835)

---
 .pre-commit-config.yaml                       |  4 +-
 distributed/client.py                         | 24 ++++-----
 distributed/comm/addressing.py                |  2 +-
 distributed/core.py                           |  4 +-
 distributed/dashboard/components/__init__.py  |  2 +-
 distributed/dashboard/components/nvml.py      |  2 +-
 distributed/dashboard/components/scheduler.py | 18 +++----
 distributed/dashboard/components/worker.py    |  2 +-
 distributed/deploy/adaptive_core.py           |  4 +-
 distributed/deploy/cluster.py                 |  4 +-
 distributed/deploy/spec.py                    |  6 +--
 distributed/deploy/tests/test_adaptive.py     |  6 +--
 distributed/deploy/tests/test_local.py        |  4 +-
 distributed/diagnostics/eventstream.py        |  2 +-
 distributed/diagnostics/plugin.py             | 14 ++---
 distributed/diagnostics/progress.py           |  4 +-
 distributed/diagnostics/progressbar.py        |  2 +-
 distributed/diagnostics/websocket.py          | 12 ++---
 distributed/event.py                          |  6 +--
 distributed/lock.py                           |  2 +-
 distributed/multi_lock.py                     |  2 +-
 distributed/nanny.py                          |  6 +--
 distributed/node.py                           |  2 +-
 distributed/preloading.py                     |  4 +-
 distributed/profile.py                        |  6 +--
 distributed/protocol/compression.py           |  2 +-
 distributed/protocol/core.py                  |  2 +-
 distributed/pubsub.py                         |  8 +--
 distributed/queues.py                         |  4 +-
 distributed/scheduler.py                      | 52 +++++++++----------
 distributed/tests/test_client.py              |  2 +-
 distributed/tests/test_core.py                |  2 +-
 distributed/tests/test_worker.py              |  2 +-
 distributed/utils.py                          | 16 +++---
 distributed/utils_test.py                     |  2 +-
 distributed/versions.py                       |  4 +-
 distributed/worker.py                         | 12 ++---
 37 files changed, 126 insertions(+), 126 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 222d7c1b466..5b9075a55c0 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -5,13 +5,13 @@ repos:
       - id: isort
         language_version: python3
   -   repo: https://github.com/psf/black
-      rev: 20.8b1
+      rev: 21.5b1
       hooks:
       - id: black
         language_version: python3
         exclude: versioneer.py
   -   repo: https://gitlab.com/pycqa/flake8
-      rev: 3.8.3
+      rev: 3.9.2
       hooks:
       - id: flake8
         language_version: python3
diff --git a/distributed/client.py b/distributed/client.py
index 293081fe330..f214a303b44 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -201,7 +201,7 @@ def status(self):
         return self._state.status
 
     def done(self):
-        """ Is the computation complete? """
+        """Is the computation complete?"""
         return self._state.done()
 
     def result(self, timeout=None):
@@ -309,7 +309,7 @@ def retry(self, **kwargs):
         return self.client.retry([self], **kwargs)
 
     def cancelled(self):
-        """ Returns True if the future has been cancelled """
+        """Returns True if the future has been cancelled"""
         return self._state.status == "cancelled"
 
     async def _traceback(self):
@@ -491,7 +491,7 @@ def __repr__(self):
 
 
 async def done_callback(future, callback):
-    """ Coroutine that waits on future, then calls callback """
+    """Coroutine that waits on future, then calls callback"""
     while future.status == "pending":
         await future._state.wait()
     callback(future)
@@ -953,7 +953,7 @@ def _repr_html_(self):
             return text
 
     def start(self, **kwargs):
-        """ Start scheduler running in separate thread """
+        """Start scheduler running in separate thread"""
         if self.status != "newly-created":
             return
 
@@ -1221,7 +1221,7 @@ def _dec_ref(self, key):
                 self._release_key(key)
 
     def _release_key(self, key):
-        """ Release key from distributed memory """
+        """Release key from distributed memory"""
         logger.debug("Release key %s", key)
         st = self.futures.pop(key, None)
         if st is not None:
@@ -1232,7 +1232,7 @@ def _release_key(self, key):
             )
 
     async def _handle_report(self):
-        """ Listen to scheduler """
+        """Listen to scheduler"""
         with log_errors():
             try:
                 while True:
@@ -1325,7 +1325,7 @@ def _handle_error(self, exception=None):
         logger.exception(exception)
 
     async def _close(self, fast=False):
-        """ Send close signal and wait until scheduler completes """
+        """Send close signal and wait until scheduler completes"""
         if self.status == "closed":
             return
 
@@ -1815,7 +1815,7 @@ async def _gather(self, futures, errors="raise", direct=None, local_worker=None)
                     direct = True
 
         async def wait(k):
-            """ Want to stop the All(...) early if we find an error """
+            """Want to stop the All(...) early if we find an error"""
             st = self.futures[k]
             await st.wait()
             if st.status != "finished" and errors == "raise":
@@ -4116,7 +4116,7 @@ def unregister_worker_plugin(self, name):
 
 
 class _WorkerSetupPlugin(WorkerPlugin):
-    """ This is used to support older setup functions as callbacks """
+    """This is used to support older setup functions as callbacks"""
 
     def __init__(self, setup):
         self._setup = setup
@@ -4129,7 +4129,7 @@ def setup(self, worker):
 
 
 class Executor(Client):
-    """ Deprecated: see Client """
+    """Deprecated: see Client"""
 
     def __init__(self, *args, **kwargs):
         warnings.warn("Executor has been renamed to Client")
@@ -4463,7 +4463,7 @@ def batches(self):
                 return
 
     def clear(self):
-        """ Clear out all submitted futures """
+        """Clear out all submitted futures"""
         with self.lock:
             self.futures.clear()
             while not self.queue.empty():
@@ -4475,7 +4475,7 @@ def AsCompleted(*args, **kwargs):
 
 
 def default_client(c=None):
-    """ Return a client if one has started """
+    """Return a client if one has started"""
     c = c or _get_global_client()
     if c:
         return c
diff --git a/distributed/comm/addressing.py b/distributed/comm/addressing.py
index 949fa31bcd1..8e454cfd877 100644
--- a/distributed/comm/addressing.py
+++ b/distributed/comm/addressing.py
@@ -264,7 +264,7 @@ def address_from_user_args(
     security=None,
     default_port=0,
 ) -> str:
-    """ Get an address to listen on from common user provided arguments """
+    """Get an address to listen on from common user provided arguments"""
 
     if security and security.require_encryption and not protocol:
         protocol = "tls"
diff --git a/distributed/core.py b/distributed/core.py
index 9aaf535191c..cfd84bce95a 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -261,7 +261,7 @@ def status(self, new_status):
             raise TypeError(f"expected Status or str, got {new_status}")
 
     async def finished(self):
-        """ Wait until the server has finished """
+        """Wait until the server has finished"""
         await self._event_finished.wait()
 
     def __await__(self):
@@ -969,7 +969,7 @@ def __repr__(self):
         )
 
     def __call__(self, addr=None, ip=None, port=None):
-        """ Cached rpc objects """
+        """Cached rpc objects"""
         addr = addr_from_args(addr=addr, ip=ip, port=port)
         return PooledRPCCall(
             addr, self, serializers=self.serializers, deserializers=self.deserializers
diff --git a/distributed/dashboard/components/__init__.py b/distributed/dashboard/components/__init__.py
index c80372faa08..7c891d96119 100644
--- a/distributed/dashboard/components/__init__.py
+++ b/distributed/dashboard/components/__init__.py
@@ -59,7 +59,7 @@ def __init__(self):
         self.root = None
 
     def update(self, messages):
-        """ Reads from bokeh.distributed.messages and updates self.source """
+        """Reads from bokeh.distributed.messages and updates self.source"""
 
 
 def add_periodic_callback(doc, component, interval):
diff --git a/distributed/dashboard/components/nvml.py b/distributed/dashboard/components/nvml.py
index 6a544f1b9d9..4b44902f627 100644
--- a/distributed/dashboard/components/nvml.py
+++ b/distributed/dashboard/components/nvml.py
@@ -29,7 +29,7 @@
 
 
 class GPUCurrentLoad(DashboardComponent):
-    """ How many tasks are on each worker """
+    """How many tasks are on each worker"""
 
     def __init__(self, scheduler, width=600, **kwargs):
         with log_errors():
diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index fd58c000d05..790cf655efe 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -92,7 +92,7 @@
 
 
 class Occupancy(DashboardComponent):
-    """ Occupancy (in time) per worker """
+    """Occupancy (in time) per worker"""
 
     def __init__(self, scheduler, **kwargs):
         with log_errors():
@@ -176,7 +176,7 @@ def update(self):
 
 
 class ProcessingHistogram(DashboardComponent):
-    """ How many tasks are on each worker """
+    """How many tasks are on each worker"""
 
     def __init__(self, scheduler, **kwargs):
         with log_errors():
@@ -501,7 +501,7 @@ def update(self):
 
 
 class BandwidthTypes(DashboardComponent):
-    """ Bar chart showing bandwidth per type """
+    """Bar chart showing bandwidth per type"""
 
     def __init__(self, scheduler, **kwargs):
         with log_errors():
@@ -567,7 +567,7 @@ def update(self):
 
 
 class BandwidthWorkers(DashboardComponent):
-    """ How many tasks are on each worker """
+    """How many tasks are on each worker"""
 
     def __init__(self, scheduler, **kwargs):
         with log_errors():
@@ -672,7 +672,7 @@ def name(address):
 
 
 class ComputePerKey(DashboardComponent):
-    """ Bar chart showing time spend in action by key prefix"""
+    """Bar chart showing time spend in action by key prefix"""
 
     def __init__(self, scheduler, **kwargs):
         with log_errors():
@@ -840,7 +840,7 @@ def update(self):
 
 
 class AggregateAction(DashboardComponent):
-    """ Bar chart showing time spend in action by key prefix"""
+    """Bar chart showing time spend in action by key prefix"""
 
     def __init__(self, scheduler, **kwargs):
         with log_errors():
@@ -942,7 +942,7 @@ def update(self):
 
 
 class MemoryByKey(DashboardComponent):
-    """ Bar chart showing memory use by key prefix"""
+    """Bar chart showing memory use by key prefix"""
 
     def __init__(self, scheduler, **kwargs):
         with log_errors():
@@ -1234,7 +1234,7 @@ def __init__(self, scheduler, **kwargs):
         )
 
     def convert(self, msgs):
-        """ Convert a log message to a glyph """
+        """Convert a log message to a glyph"""
         total_duration = 0
         for msg in msgs:
             time, level, key, duration, sat, occ_sat, idl, occ_idl = msg
@@ -1723,7 +1723,7 @@ def __del__(self):
 
 
 class TaskProgress(DashboardComponent):
-    """ Progress bars per task type """
+    """Progress bars per task type"""
 
     def __init__(self, scheduler, **kwargs):
         self.scheduler = scheduler
diff --git a/distributed/dashboard/components/worker.py b/distributed/dashboard/components/worker.py
index 83f8c3736f8..9266e1f2766 100644
--- a/distributed/dashboard/components/worker.py
+++ b/distributed/dashboard/components/worker.py
@@ -50,7 +50,7 @@
 
 
 class StateTable(DashboardComponent):
-    """ Currently running tasks """
+    """Currently running tasks"""
 
     def __init__(self, worker):
         self.worker = worker
diff --git a/distributed/deploy/adaptive_core.py b/distributed/deploy/adaptive_core.py
index 4f277307b64..4d4d2810c41 100644
--- a/distributed/deploy/adaptive_core.py
+++ b/distributed/deploy/adaptive_core.py
@@ -113,7 +113,7 @@ def stop(self):
             self.periodic_callback = None
 
     async def target(self) -> int:
-        """ The target number of workers that should exist """
+        """The target number of workers that should exist"""
         raise NotImplementedError()
 
     async def workers_to_close(self, target: int) -> list:
@@ -124,7 +124,7 @@ async def workers_to_close(self, target: int) -> list:
         return list(self.observed)[target:]
 
     async def safe_target(self) -> int:
-        """ Used internally, like target, but respects minimum/maximum """
+        """Used internally, like target, but respects minimum/maximum"""
         n = await self.target()
         if n > self.maximum:
             n = self.maximum
diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
index 39a6aa3650c..00360c7aa22 100644
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -114,7 +114,7 @@ def __del__(self):
                 self.loop.add_callback(self.close)
 
     async def _watch_worker_status(self, comm):
-        """ Listen to scheduler for updates on adding and removing workers """
+        """Listen to scheduler for updates on adding and removing workers"""
         while True:
             try:
                 msgs = await comm.read()
@@ -302,7 +302,7 @@ def _widget_status(self):
         return text
 
     def _widget(self):
-        """ Create IPython widget for display within a notebook """
+        """Create IPython widget for display within a notebook"""
         try:
             return self._cached_widget
         except AttributeError:
diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index 5190d7c0fa7..ea5e8b092a6 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -98,7 +98,7 @@ async def close(self):
         self._event_finished.set()
 
     async def finished(self):
-        """ Wait until the server has finished """
+        """Wait until the server has finished"""
         await self._event_finished.wait()
 
     def __repr__(self):
@@ -446,7 +446,7 @@ def __exit__(self, typ, value, traceback):
         self._loop_runner.stop()
 
     def _threads_per_worker(self) -> int:
-        """ Return the number of threads per worker for new workers """
+        """Return the number of threads per worker for new workers"""
         if not self.new_spec:
             raise ValueError("To scale by cores= you must specify cores per worker")
 
@@ -458,7 +458,7 @@ def _threads_per_worker(self) -> int:
             raise ValueError("To scale by cores= you must specify cores per worker")
 
     def _memory_per_worker(self) -> int:
-        """ Return the memory limit per worker for new workers """
+        """Return the memory limit per worker for new workers"""
         if not self.new_spec:
             raise ValueError(
                 "to scale by memory= your worker definition must include a memory_limit definition"
diff --git a/distributed/deploy/tests/test_adaptive.py b/distributed/deploy/tests/test_adaptive.py
index f43f097c076..35a9c14cc54 100644
--- a/distributed/deploy/tests/test_adaptive.py
+++ b/distributed/deploy/tests/test_adaptive.py
@@ -288,7 +288,7 @@ async def test_adapt_quickly():
 
 @gen_test(timeout=None)
 async def test_adapt_down():
-    """ Ensure that redefining adapt with a lower maximum removes workers """
+    """Ensure that redefining adapt with a lower maximum removes workers"""
     async with LocalCluster(
         0,
         asynchronous=True,
@@ -348,7 +348,7 @@ def test_basic_no_loop(loop):
 
 @pytest.mark.asyncio
 async def test_target_duration():
-    """ Ensure that redefining adapt with a lower maximum removes workers """
+    """Ensure that redefining adapt with a lower maximum removes workers"""
     with dask.config.set(
         {"distributed.scheduler.default-task-durations": {"slowinc": 1}}
     ):
@@ -376,7 +376,7 @@ async def test_target_duration():
 
 @pytest.mark.asyncio
 async def test_worker_keys(cleanup):
-    """ Ensure that redefining adapt with a lower maximum removes workers """
+    """Ensure that redefining adapt with a lower maximum removes workers"""
     async with SpecCluster(
         workers={
             "a-1": {"cls": Worker},
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index b5306b37818..a8b08513d5d 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -627,7 +627,7 @@ def test_no_ipywidgets(loop, monkeypatch):
 
 
 def test_scale(loop):
-    """ Directly calling scale both up and down works as expected """
+    """Directly calling scale both up and down works as expected"""
     with LocalCluster(
         scheduler_port=0,
         silence_logs=False,
@@ -686,7 +686,7 @@ def test_adapt(loop):
 
 
 def test_adapt_then_manual(loop):
-    """ We can revert from adaptive, back to manual """
+    """We can revert from adaptive, back to manual"""
     with LocalCluster(
         scheduler_port=0,
         silence_logs=False,
diff --git a/distributed/diagnostics/eventstream.py b/distributed/diagnostics/eventstream.py
index 1f98e47c64b..34805e9085a 100644
--- a/distributed/diagnostics/eventstream.py
+++ b/distributed/diagnostics/eventstream.py
@@ -8,7 +8,7 @@
 
 
 class EventStream(SchedulerPlugin):
-    """ Maintain a copy of worker events """
+    """Maintain a copy of worker events"""
 
     def __init__(self, scheduler=None):
         self.buffer = []
diff --git a/distributed/diagnostics/plugin.py b/distributed/diagnostics/plugin.py
index 5207d54aa82..fb8aa9029b8 100644
--- a/distributed/diagnostics/plugin.py
+++ b/distributed/diagnostics/plugin.py
@@ -60,10 +60,10 @@ async def close(self):
         pass
 
     def update_graph(self, scheduler, dsk=None, keys=None, restrictions=None, **kwargs):
-        """ Run when a new graph / tasks enter the scheduler """
+        """Run when a new graph / tasks enter the scheduler"""
 
     def restart(self, scheduler, **kwargs):
-        """ Run when the scheduler restarts itself """
+        """Run when the scheduler restarts itself"""
 
     def transition(self, key, start, finish, *args, **kwargs):
         """Run whenever a task changes state
@@ -81,16 +81,16 @@ def transition(self, key, start, finish, *args, **kwargs):
         """
 
     def add_worker(self, scheduler=None, worker=None, **kwargs):
-        """ Run when a new worker enters the cluster """
+        """Run when a new worker enters the cluster"""
 
     def remove_worker(self, scheduler=None, worker=None, **kwargs):
-        """ Run when a worker leaves the cluster """
+        """Run when a worker leaves the cluster"""
 
     def add_client(self, scheduler=None, client=None, **kwargs):
-        """ Run when a new client connects """
+        """Run when a new client connects"""
 
     def remove_client(self, scheduler=None, client=None, **kwargs):
-        """ Run when a client disconnects """
+        """Run when a client disconnects"""
 
 
 class WorkerPlugin:
@@ -134,7 +134,7 @@ def setup(self, worker):
         """
 
     def teardown(self, worker):
-        """ Run when the worker to which the plugin is attached to is closed """
+        """Run when the worker to which the plugin is attached to is closed"""
 
     def transition(self, key, start, finish, **kwargs):
         """
diff --git a/distributed/diagnostics/progress.py b/distributed/diagnostics/progress.py
index f71d56d8bf4..28500fd8077 100644
--- a/distributed/diagnostics/progress.py
+++ b/distributed/diagnostics/progress.py
@@ -237,7 +237,7 @@ def format_time(t):
 
 
 class AllProgress(SchedulerPlugin):
-    """ Keep track of all keys, grouped by key_split """
+    """Keep track of all keys, grouped by key_split"""
 
     def __init__(self, scheduler):
         self.all = defaultdict(set)
@@ -287,7 +287,7 @@ def restart(self, scheduler):
 
 
 class GroupProgress(SchedulerPlugin):
-    """ Keep track of all keys, grouped by key_split """
+    """Keep track of all keys, grouped by key_split"""
 
     def __init__(self, scheduler):
         self.scheduler = scheduler
diff --git a/distributed/diagnostics/progressbar.py b/distributed/diagnostics/progressbar.py
index 851910e45d4..1a05438cccd 100644
--- a/distributed/diagnostics/progressbar.py
+++ b/distributed/diagnostics/progressbar.py
@@ -334,7 +334,7 @@ def make_widget(self, all):
         }
 
         def keyfunc(kv):
-            """ Order keys by most numerous, then by string name """
+            """Order keys by most numerous, then by string name"""
             return kv[::-1]
 
         key_order = [k for k, v in sorted(all.items(), key=keyfunc, reverse=True)]
diff --git a/distributed/diagnostics/websocket.py b/distributed/diagnostics/websocket.py
index 51282c1e621..3796f776036 100644
--- a/distributed/diagnostics/websocket.py
+++ b/distributed/diagnostics/websocket.py
@@ -9,27 +9,27 @@ def __init__(self, socket, scheduler):
         self.scheduler = scheduler
 
     def restart(self, scheduler, **kwargs):
-        """ Run when the scheduler restarts itself """
+        """Run when the scheduler restarts itself"""
         self.socket.send("restart", {})
 
     def add_worker(self, scheduler=None, worker=None, **kwargs):
-        """ Run when a new worker enters the cluster """
+        """Run when a new worker enters the cluster"""
         self.socket.send("add_worker", {"worker": worker})
 
     def remove_worker(self, scheduler=None, worker=None, **kwargs):
-        """ Run when a worker leaves the cluster"""
+        """Run when a worker leaves the cluster"""
         self.socket.send("remove_worker", {"worker": worker})
 
     def add_client(self, scheduler=None, client=None, **kwargs):
-        """ Run when a new client connects """
+        """Run when a new client connects"""
         self.socket.send("add_client", {"client": client})
 
     def remove_client(self, scheduler=None, client=None, **kwargs):
-        """ Run when a client disconnects """
+        """Run when a client disconnects"""
         self.socket.send("remove_client", {"client": client})
 
     def update_graph(self, scheduler, client=None, **kwargs):
-        """ Run when a new graph / tasks enter the scheduler """
+        """Run when a new graph / tasks enter the scheduler"""
         self.socket.send("update_graph", {"client": client})
 
     def transition(self, key, start, finish, *args, **kwargs):
diff --git a/distributed/event.py b/distributed/event.py
index a3e2a1b7eeb..40c1c5e3daf 100644
--- a/distributed/event.py
+++ b/distributed/event.py
@@ -132,14 +132,14 @@ def event_is_set(self, comm=None, name=None):
             return self._events[name].is_set()
 
     def _normalize_name(self, name):
-        """ Helper function to normalize an event name """
+        """Helper function to normalize an event name"""
         if isinstance(name, list):
             name = tuple(name)
 
         return name
 
     def _delete_event(self, name):
-        """ Helper function to delete an event """
+        """Helper function to delete an event"""
         # suppress key errors to make calling this method
         # also possible if we do not even have such an event
         with suppress(KeyError):
@@ -242,7 +242,7 @@ def set(self):
         return result
 
     def is_set(self):
-        """ Check if the event is set """
+        """Check if the event is set"""
         result = self.client.sync(self.client.scheduler.event_is_set, name=self.name)
         return result
 
diff --git a/distributed/lock.py b/distributed/lock.py
index 7ffe8c8daf9..64e3f29c22a 100644
--- a/distributed/lock.py
+++ b/distributed/lock.py
@@ -142,7 +142,7 @@ def acquire(self, blocking=True, timeout=None):
         return result
 
     def release(self):
-        """ Release the lock if already acquired """
+        """Release the lock if already acquired"""
         if not self.locked():
             raise ValueError("Lock is not yet acquired")
         result = self.client.sync(
diff --git a/distributed/multi_lock.py b/distributed/multi_lock.py
index aaa21999a19..3eda32e0be6 100644
--- a/distributed/multi_lock.py
+++ b/distributed/multi_lock.py
@@ -209,7 +209,7 @@ def acquire(self, blocking=True, timeout=None, num_locks=None):
         return result
 
     def release(self):
-        """ Release the lock if already acquired """
+        """Release the lock if already acquired"""
         if not self.locked():
             raise ValueError("Lock is not yet acquired")
         ret = self.client.sync(self.client.scheduler.multi_lock_release, id=self.id)
diff --git a/distributed/nanny.py b/distributed/nanny.py
index a275055029d..127906b1bb1 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -253,12 +253,12 @@ def worker_dir(self):
 
     @property
     def local_dir(self):
-        """ For API compatibility with Nanny """
+        """For API compatibility with Nanny"""
         warnings.warn("The local_dir attribute has moved to local_directory")
         return self.local_directory
 
     async def start(self):
-        """ Start nanny, start local process, start watching """
+        """Start nanny, start local process, start watching"""
 
         await super().start()
 
@@ -411,7 +411,7 @@ def _psutil_process(self):
         return self._psutil_process_obj
 
     def memory_monitor(self):
-        """ Track worker's memory.  Restart if it goes above terminate fraction """
+        """Track worker's memory.  Restart if it goes above terminate fraction"""
         if self.status != Status.running:
             return
         process = self.process.process
diff --git a/distributed/node.py b/distributed/node.py
index eb150ca0418..e21713dc85d 100644
--- a/distributed/node.py
+++ b/distributed/node.py
@@ -99,7 +99,7 @@ def get_logs(self, comm=None, n=None):
     def start_http_server(
         self, routes, dashboard_address, default_port=0, ssl_options=None
     ):
-        """ This creates an HTTP Server running on this node """
+        """This creates an HTTP Server running on this node"""
 
         self.http_application = RoutingApplication(routes)
 
diff --git a/distributed/preloading.py b/distributed/preloading.py
index 9108a5c9090..40390df613b 100644
--- a/distributed/preloading.py
+++ b/distributed/preloading.py
@@ -161,7 +161,7 @@ def __init__(self, dask_server, name: str, argv: List[str], file_dir: str):
             self.module = None
 
     async def start(self):
-        """ Run when the server finishes its start method """
+        """Run when the server finishes its start method"""
         if is_webaddress(self.name):
             self.module = await _download_module(self.name)
 
@@ -185,7 +185,7 @@ async def start(self):
                 logger.info("Run preload setup function: %s", self.name)
 
     async def teardown(self):
-        """ Run when the server starts its close method """
+        """Run when the server starts its close method"""
         dask_teardown = getattr(self.module, "dask_teardown", None)
         if dask_teardown:
             future = dask_teardown(self.dask_server)
diff --git a/distributed/profile.py b/distributed/profile.py
index fbaf6ea6599..3f3230c437b 100644
--- a/distributed/profile.py
+++ b/distributed/profile.py
@@ -55,7 +55,7 @@ def identifier(frame):
 
 
 def repr_frame(frame):
-    """ Render a frame as a line for inclusion into a text traceback """
+    """Render a frame as a line for inclusion into a text traceback"""
     co = frame.f_code
     text = '  File "%s", line %s, in %s' % (co.co_filename, frame.f_lineno, co.co_name)
     line = linecache.getline(co.co_filename, frame.f_lineno, frame.f_globals).lstrip()
@@ -126,7 +126,7 @@ def process(frame, child, state, stop=None, omit=None):
 
 
 def merge(*args):
-    """ Merge multiple frame states together """
+    """Merge multiple frame states together"""
     if not args:
         return create()
     s = {arg["identifier"] for arg in args}
@@ -490,7 +490,7 @@ def llprocess(frames, child, state):
 
 
 def ll_get_stack(tid):
-    """ Collect low level stack information from thread id """
+    """Collect low level stack information from thread id"""
     from stacktrace import get_thread_stack
 
     frames = get_thread_stack(tid, show_python=False)
diff --git a/distributed/protocol/compression.py b/distributed/protocol/compression.py
index 800f79adb90..26bee14174c 100644
--- a/distributed/protocol/compression.py
+++ b/distributed/protocol/compression.py
@@ -221,7 +221,7 @@ def maybe_compress(
 
 
 def decompress(header, frames):
-    """ Decompress frames according to information in the header """
+    """Decompress frames according to information in the header"""
     return [
         compressions[c]["decompress"](frame)
         for c, frame in zip(header["compression"], frames)
diff --git a/distributed/protocol/core.py b/distributed/protocol/core.py
index 1e3da14cf8b..871e7e4df56 100644
--- a/distributed/protocol/core.py
+++ b/distributed/protocol/core.py
@@ -76,7 +76,7 @@ def _encode_default(obj):
 
 
 def loads(frames, deserialize=True, deserializers=None):
-    """ Transform bytestream back into Python value """
+    """Transform bytestream back into Python value"""
 
     try:
 
diff --git a/distributed/pubsub.py b/distributed/pubsub.py
index 5dba5b679c6..99c528eb5e0 100644
--- a/distributed/pubsub.py
+++ b/distributed/pubsub.py
@@ -13,7 +13,7 @@
 
 
 class PubSubSchedulerExtension:
-    """ Extend Dask's scheduler with routes to handle PubSub machinery """
+    """Extend Dask's scheduler with routes to handle PubSub machinery"""
 
     def __init__(self, scheduler):
         self.scheduler = scheduler
@@ -116,7 +116,7 @@ def handle_message(self, name=None, msg=None, worker=None, client=None):
 
 
 class PubSubWorkerExtension:
-    """ Extend Dask's Worker with routes to handle PubSub machinery """
+    """Extend Dask's Worker with routes to handle PubSub machinery"""
 
     def __init__(self, worker):
         self.worker = worker
@@ -169,7 +169,7 @@ def cleanup(self):
 
 
 class PubSubClientExtension:
-    """ Extend Dask's Client with handlers to handle PubSub machinery """
+    """Extend Dask's Client with handlers to handle PubSub machinery"""
 
     def __init__(self, client):
         self.client = client
@@ -344,7 +344,7 @@ def _put(self, msg):
             self.client.scheduler_comm.send(data)
 
     def put(self, msg):
-        """ Publish a message to all subscribers of this topic """
+        """Publish a message to all subscribers of this topic"""
         self.loop.add_callback(self._put, msg)
 
     def __repr__(self):
diff --git a/distributed/queues.py b/distributed/queues.py
index cadaf358f0a..3dd0361c669 100644
--- a/distributed/queues.py
+++ b/distributed/queues.py
@@ -85,7 +85,7 @@ def future_release(self, name=None, key=None, client=None):
 
     async def get(self, comm=None, name=None, client=None, timeout=None, batch=False):
         def process(record):
-            """ Add task status if known """
+            """Add task status if known"""
             if record["type"] == "Future":
                 record = record.copy()
                 key = record["value"]
@@ -244,7 +244,7 @@ def get(self, timeout=None, batch=False, **kwargs):
         return self.client.sync(self._get, timeout=timeout, batch=batch, **kwargs)
 
     def qsize(self, **kwargs):
-        """ Current number of elements in the queue """
+        """Current number of elements in the queue"""
         return self.client.sync(self._qsize, **kwargs)
 
     async def _get(self, timeout=None, batch=False):
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 3830e13fe67..8ed5d0e1cbb 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -715,7 +715,7 @@ def versions(self):
 
     @ccall
     def clean(self):
-        """ Return a version of this object that is appropriate for serialization """
+        """Return a version of this object that is appropriate for serialization"""
         ws: WorkerState = WorkerState(
             address=self._address,
             pid=self._pid,
@@ -1534,7 +1534,7 @@ def prefix_key(self):
 
     @ccall
     def add_dependency(self, other: "TaskState"):
-        """ Add another task as a dependency of this task """
+        """Add another task as a dependency of this task"""
         self._dependencies.add(other)
         self._group._dependencies.add(other._group)
         other._dependents.add(self)
@@ -1969,7 +1969,7 @@ def __pdict__(self):
     @ccall
     @exceptval(check=False)
     def new_task(self, key: str, spec: object, state: str) -> TaskState:
-        """ Create a new task, and associated states """
+        """Create a new task, and associated states"""
         ts: TaskState = TaskState(key, spec)
         ts._state = state
 
@@ -3619,7 +3619,7 @@ def _repr_html_(self):
         return text
 
     def identity(self, comm=None):
-        """ Basic information about ourselves and our cluster """
+        """Basic information about ourselves and our cluster"""
         parent: SchedulerState = cast(SchedulerState, self)
         d = {
             "type": type(self).__name__,
@@ -3662,7 +3662,7 @@ def get_worker_service_addr(self, worker, service_name, protocol=False):
             return ws.host, port
 
     async def start(self):
-        """ Clear out old state and restart all running coroutines """
+        """Clear out old state and restart all running coroutines"""
         await super().start()
         assert self.status != Status.running
 
@@ -3932,7 +3932,7 @@ async def add_worker(
         nanny=None,
         extra=None,
     ):
-        """ Add a new worker to the cluster """
+        """Add a new worker to the cluster"""
         parent: SchedulerState = cast(SchedulerState, self)
         with log_errors():
             address = self.coerce_address(address, resolve_address)
@@ -4407,7 +4407,7 @@ def update_graph(
         # TODO: balance workers
 
     def stimulus_task_finished(self, key=None, worker=None, **kwargs):
-        """ Mark that a task has finished execution on a particular worker """
+        """Mark that a task has finished execution on a particular worker"""
         parent: SchedulerState = cast(SchedulerState, self)
         logger.debug("Stimulus task finished %s, %s", key, worker)
 
@@ -4444,7 +4444,7 @@ def stimulus_task_finished(self, key=None, worker=None, **kwargs):
     def stimulus_task_erred(
         self, key=None, worker=None, exception=None, traceback=None, **kwargs
     ):
-        """ Mark that a task has erred on a particular worker """
+        """Mark that a task has erred on a particular worker"""
         parent: SchedulerState = cast(SchedulerState, self)
         logger.debug("Stimulus task erred %s, %s", key, worker)
 
@@ -4479,7 +4479,7 @@ def stimulus_task_erred(
     def stimulus_missing_data(
         self, cause=None, key=None, worker=None, ensure=True, **kwargs
     ):
-        """ Mark that certain keys have gone missing.  Recover. """
+        """Mark that certain keys have gone missing.  Recover."""
         parent: SchedulerState = cast(SchedulerState, self)
         with log_errors():
             logger.debug("Stimulus missing data %s, %s", key, worker)
@@ -4666,7 +4666,7 @@ def remove_worker_from_events():
         return "OK"
 
     def stimulus_cancel(self, comm, keys=None, client=None, force=False):
-        """ Stop execution on a list of keys """
+        """Stop execution on a list of keys"""
         logger.info("Client %s requests to cancel %d keys", client, len(keys))
         if client:
             self.log_event(
@@ -4676,7 +4676,7 @@ def stimulus_cancel(self, comm, keys=None, client=None, force=False):
             self.cancel_key(key, client, force=force)
 
     def cancel_key(self, key, client, retries=5, force=False):
-        """ Cancel a particular key and all dependents """
+        """Cancel a particular key and all dependents"""
         # TODO: this should be converted to use the transition mechanism
         parent: SchedulerState = cast(SchedulerState, self)
         ts: TaskState = parent._tasks.get(key)
@@ -4719,7 +4719,7 @@ def client_desires_keys(self, keys=None, client=None):
                 self.report_on_key(ts=ts, client=client)
 
     def client_releases_keys(self, keys=None, client=None):
-        """ Remove keys from client desired list """
+        """Remove keys from client desired list"""
 
         parent: SchedulerState = cast(SchedulerState, self)
         if not isinstance(keys, list):
@@ -4731,7 +4731,7 @@ def client_releases_keys(self, keys=None, client=None):
         self.transitions(recommendations)
 
     def client_heartbeat(self, client=None):
-        """ Handle heartbeats from Client """
+        """Handle heartbeats from Client"""
         parent: SchedulerState = cast(SchedulerState, self)
         cs: ClientState = parent._clients[client]
         cs._last_seen = time()
@@ -4976,7 +4976,7 @@ async def add_client(self, comm, client=None, versions=None):
                 pass
 
     def remove_client(self, client=None):
-        """ Remove client from network """
+        """Remove client from network"""
         parent: SchedulerState = cast(SchedulerState, self)
         if self.status == Status.running:
             logger.info("Remove client %s", client)
@@ -5010,7 +5010,7 @@ def remove_client_from_events():
         self.loop.call_later(cleanup_delay, remove_client_from_events)
 
     def send_task_to_worker(self, worker, ts: TaskState, duration: double = -1):
-        """ Send a single computational task to a worker """
+        """Send a single computational task to a worker"""
         parent: SchedulerState = cast(SchedulerState, self)
         try:
             msg: dict = _task_to_msg(parent, ts, duration)
@@ -5179,7 +5179,7 @@ def add_plugin(self, plugin=None, idempotent=False, **kwargs):
         self.plugins.append(plugin)
 
     def remove_plugin(self, plugin):
-        """ Remove external plugin from scheduler """
+        """Remove external plugin from scheduler"""
         self.plugins.remove(plugin)
 
     def worker_send(self, worker, msg):
@@ -5283,7 +5283,7 @@ async def scatter(
         return keys
 
     async def gather(self, comm=None, keys=None, serializers=None):
-        """ Collect data in from workers """
+        """Collect data in from workers"""
         parent: SchedulerState = cast(SchedulerState, self)
         ws: WorkerState
         keys = list(keys)
@@ -5360,7 +5360,7 @@ def clear_task_state(self):
             collection.clear()
 
     async def restart(self, client=None, timeout=3):
-        """ Restart all workers.  Reset local state. """
+        """Restart all workers.  Reset local state."""
         parent: SchedulerState = cast(SchedulerState, self)
         with log_errors():
 
@@ -5448,7 +5448,7 @@ async def broadcast(
         nanny=False,
         serializers=None,
     ):
-        """ Broadcast message to workers, return all results """
+        """Broadcast message to workers, return all results"""
         parent: SchedulerState = cast(SchedulerState, self)
         if workers is None or workers is True:
             if hosts is None:
@@ -5483,7 +5483,7 @@ async def send_message(addr):
         return dict(zip(workers, results))
 
     async def proxy(self, comm=None, msg=None, worker=None, serializers=None):
-        """ Proxy a communication through the scheduler to some other worker """
+        """Proxy a communication through the scheduler to some other worker"""
         d = await self.broadcast(
             comm=comm, msg=msg, workers=[worker], serializers=serializers
         )
@@ -6343,7 +6343,7 @@ def stop_task_metadata(self, comm=None, name=None):
         return {"metadata": plugin.metadata, "state": plugin.state}
 
     async def register_worker_plugin(self, comm, plugin, name=None):
-        """ Registers a setup function, and call it on every worker """
+        """Registers a setup function, and call it on every worker"""
         self.worker_plugins[name] = plugin
 
         responses = await self.broadcast(
@@ -6352,7 +6352,7 @@ async def register_worker_plugin(self, comm, plugin, name=None):
         return responses
 
     async def unregister_worker_plugin(self, comm, name):
-        """ Unregisters a worker plugin"""
+        """Unregisters a worker plugin"""
         try:
             worker_plugins = self.worker_plugins.pop(name)
         except KeyError:
@@ -6399,7 +6399,7 @@ def transitions(self, recommendations: dict):
         self.send_all(client_msgs, worker_msgs)
 
     def story(self, *keys):
-        """ Get all transitions that touch one of the input keys """
+        """Get all transitions that touch one of the input keys"""
         keys = {key.key if isinstance(key, TaskState) else key for key in keys}
         return [
             t for t in self.transition_log if t[0] in keys or keys.intersection(t[3])
@@ -7054,7 +7054,7 @@ def _propagate_forgotten(
 def _client_releases_keys(
     state: SchedulerState, keys: list, cs: ClientState, recommendations: dict
 ):
-    """ Remove keys from client desired list """
+    """Remove keys from client desired list"""
     logger.debug("Client %s releases keys: %s", cs._client_key, keys)
     ts: TaskState
     for key in keys:
@@ -7073,7 +7073,7 @@ def _client_releases_keys(
 @cfunc
 @exceptval(check=False)
 def _task_to_msg(state: SchedulerState, ts: TaskState, duration: double = -1) -> dict:
-    """ Convert a single computational task to a message """
+    """Convert a single computational task to a message"""
     ws: WorkerState
     dts: TaskState
 
@@ -7143,7 +7143,7 @@ def _task_to_client_msgs(state: SchedulerState, ts: TaskState) -> dict:
 @cfunc
 @exceptval(check=False)
 def _reevaluate_occupancy_worker(state: SchedulerState, ws: WorkerState):
-    """ See reevaluate_occupancy """
+    """See reevaluate_occupancy"""
     old: double = ws._occupancy
     new: double = 0
     diff: double
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 5a28c83a1dd..ec595dc815f 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -236,7 +236,7 @@ async def test_map_batch_size(c, s, a, b):
 
 @gen_cluster(client=True)
 async def test_custom_key_with_batches(c, s, a, b):
-    """ Test of <https://github.com/dask/distributed/issues/4588>"""
+    """Test of <https://github.com/dask/distributed/issues/4588>"""
 
     futs = c.map(
         lambda x: x ** 2,
diff --git a/distributed/tests/test_core.py b/distributed/tests/test_core.py
index 398b933c02c..6dd76f901ec 100644
--- a/distributed/tests/test_core.py
+++ b/distributed/tests/test_core.py
@@ -442,7 +442,7 @@ async def test_rpc_with_many_connections_inproc():
 
 
 async def check_large_packets(listen_arg):
-    """ tornado has a 100MB cap by default """
+    """tornado has a 100MB cap by default"""
     server = Server({"echo": echo})
     await server.listen(listen_arg)
 
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 44ba35df021..de2835a1457 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -87,7 +87,7 @@ async def test_identity(cleanup):
 @gen_cluster(client=True)
 async def test_worker_bad_args(c, s, a, b):
     class NoReprObj:
-        """ This object cannot be properly represented as a string. """
+        """This object cannot be properly represented as a string."""
 
         def __str__(self):
             raise ValueError("I have no str representation.")
diff --git a/distributed/utils.py b/distributed/utils.py
index d8eb2666136..dc63202486b 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -745,7 +745,7 @@ def get_traceback():
 
 
 def truncate_exception(e, n=10000):
-    """ Truncate exception to be about a certain length """
+    """Truncate exception to be about a certain length"""
     if len(str(e)) > n:
         try:
             return type(e)("Long error message", str(e)[:n])
@@ -763,7 +763,7 @@ def validate_key(k):
 
 
 def _maybe_complex(task):
-    """ Possibly contains a nested task """
+    """Possibly contains a nested task"""
     return (
         istask(task)
         or type(task) is list
@@ -962,7 +962,7 @@ def open_port(host=""):
 
 
 def import_file(path):
-    """ Loads modules for a file (.py, .zip, .egg) """
+    """Loads modules for a file (.py, .zip, .egg)"""
     directory, filename = os.path.split(path)
     name, ext = os.path.splitext(filename)
     names_to_import = []
@@ -1044,7 +1044,7 @@ def asciitable(columns, rows):
 
 
 def nbytes(frame, _bytes_like=(bytes, bytearray)):
-    """ Number of bytes of a frame or memoryview """
+    """Number of bytes of a frame or memoryview"""
     if isinstance(frame, _bytes_like):
         return len(frame)
     else:
@@ -1105,7 +1105,7 @@ def wrapper(*args, **kwargs):
 
 
 def json_load_robust(fn, load=json.load):
-    """ Reads a JSON file from disk that may be being written as we read """
+    """Reads a JSON file from disk that may be being written as we read"""
     while not os.path.exists(fn):
         sleep(0.01)
     for i in range(10):
@@ -1120,7 +1120,7 @@ def json_load_robust(fn, load=json.load):
 
 
 class DequeHandler(logging.Handler):
-    """ A logging.Handler that records records into a deque """
+    """A logging.Handler that records records into a deque"""
 
     _instances = weakref.WeakSet()
 
@@ -1364,7 +1364,7 @@ def parse_ports(port):
 
 
 class Log(str):
-    """ A container for logs """
+    """A container for logs"""
 
     def _repr_html_(self):
         return "<pre><code>\n{log}\n</code></pre>".format(
@@ -1373,7 +1373,7 @@ def _repr_html_(self):
 
 
 class Logs(dict):
-    """ A container for multiple logs """
+    """A container for multiple logs"""
 
     def _repr_html_(self):
         summaries = [
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index c05f142b4f1..e3f18e5de49 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -1422,7 +1422,7 @@ def save_sys_modules():
 
 @contextmanager
 def check_thread_leak():
-    """ Context manager to ensure we haven't leaked any threads """
+    """Context manager to ensure we haven't leaked any threads"""
     active_threads_start = threading.enumerate()
 
     yield
diff --git a/distributed/versions.py b/distributed/versions.py
index d8af5eabcd0..b8d0a49a80f 100644
--- a/distributed/versions.py
+++ b/distributed/versions.py
@@ -71,7 +71,7 @@ def get_system_info():
 
 
 def version_of_package(pkg):
-    """ Try a variety of common ways to get the version of a package """
+    """Try a variety of common ways to get the version of a package"""
     from contextlib import suppress
 
     with suppress(AttributeError):
@@ -84,7 +84,7 @@ def version_of_package(pkg):
 
 
 def get_package_info(pkgs):
-    """ get package versions for the passed required & optional packages """
+    """get package versions for the passed required & optional packages"""
 
     pversions = [("python", ".".join(map(str, sys.version_info)))]
     for pkg in pkgs:
diff --git a/distributed/worker.py b/distributed/worker.py
index f4ab4947355..f9829df6e63 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -799,12 +799,12 @@ def log_event(self, topic, msg):
 
     @property
     def worker_address(self):
-        """ For API compatibility with Nanny """
+        """For API compatibility with Nanny"""
         return self.address
 
     @property
     def local_dir(self):
-        """ For API compatibility with Nanny """
+        """For API compatibility with Nanny"""
         warnings.warn(
             "The local_dir attribute has moved to local_directory", stacklevel=2
         )
@@ -3482,7 +3482,7 @@ async def _get_data():
 
 
 def loads_function(bytes_object):
-    """ Load a function from bytes, cache bytes """
+    """Load a function from bytes, cache bytes"""
     if len(bytes_object) < 100000:
         try:
             result = cache_loads[bytes_object]
@@ -3494,7 +3494,7 @@ def loads_function(bytes_object):
 
 
 def _deserialize(function=None, args=None, kwargs=None, task=no_value):
-    """ Deserialize task inputs and regularize to func, args, kwargs """
+    """Deserialize task inputs and regularize to func, args, kwargs"""
     if function is not None:
         function = loads_function(function)
     if args and isinstance(args, bytes):
@@ -3534,7 +3534,7 @@ def execute_task(task):
 
 
 def dumps_function(func):
-    """ Dump a function to bytes, cache functions """
+    """Dump a function to bytes, cache functions"""
     try:
         with _cache_lock:
             result = cache_dumps[func]
@@ -3583,7 +3583,7 @@ def dumps_task(task):
 
 
 def warn_dumps(obj, dumps=pickle.dumps, limit=1e6):
-    """ Dump an object to bytes, warn if those bytes are large """
+    """Dump an object to bytes, warn if those bytes are large"""
     b = dumps(obj, protocol=4)
     if not _warn_dumps_warned[0] and len(b) > limit:
         _warn_dumps_warned[0] = True

From 09e843662b55f1bbb4ded941b8fce1398110230e Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Fri, 21 May 2021 09:31:16 -0700
Subject: [PATCH 1274/1550] Use `dict` `_workers_dv` in unordered use cases
 (#4826)

When the sorted nature of `SortedDict` is not needed with `_workers`,
make use of `_workers_dv`, which Cython can leverage more efficiently
using the associated Python C API for `dict`s.
---
 distributed/scheduler.py | 27 ++++++++++++++-------------
 1 file changed, 14 insertions(+), 13 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 8ed5d0e1cbb..a335455439d 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -3603,7 +3603,7 @@ def __repr__(self):
         parent: SchedulerState = cast(SchedulerState, self)
         return '<Scheduler: "%s" workers: %d cores: %d, tasks: %d>' % (
             self.address,
-            len(parent._workers),
+            len(parent._workers_dv),
             parent._total_nthreads,
             len(parent._tasks),
         )
@@ -3612,7 +3612,7 @@ def _repr_html_(self):
         parent: SchedulerState = cast(SchedulerState, self)
         text = (
             f"<b>Scheduler: </b>{html.escape(self.address)} "
-            f'<font color="gray">workers: </font>{len(parent._workers)} '
+            f'<font color="gray">workers: </font>{len(parent._workers_dv)} '
             f'<font color="gray">cores: </font>{parent._total_nthreads} '
             f'<font color="gray">tasks: </font>{len(parent._tasks)}'
         )
@@ -3628,7 +3628,8 @@ def identity(self, comm=None):
             "services": {key: v.port for (key, v) in self.services.items()},
             "started": self.time_started,
             "workers": {
-                worker.address: worker.identity() for worker in parent._workers.values()
+                worker.address: worker.identity()
+                for worker in parent._workers_dv.values()
             },
         }
         return d
@@ -3648,7 +3649,7 @@ def get_worker_service_addr(self, worker, service_name, protocol=False):
             or just a (host, port) pair
         """
         parent: SchedulerState = cast(SchedulerState, self)
-        ws: WorkerState = parent._workers[worker]
+        ws: WorkerState = parent._workers_dv[worker]
         port = ws._services.get(service_name)
         if port is None:
             return None
@@ -3743,10 +3744,10 @@ async def close(self, comm=None, fast=False, close_workers=False):
 
         if close_workers:
             await self.broadcast(msg={"op": "close_gracefully"}, nanny=True)
-            for worker in parent._workers:
+            for worker in parent._workers_dv:
                 self.worker_send(worker, {"op": "close"})
             for i in range(20):  # wait a second for send signals to clear
-                if parent._workers:
+                if parent._workers_dv:
                     await asyncio.sleep(0.05)
                 else:
                     break
@@ -3798,7 +3799,7 @@ async def close_worker(self, comm=None, worker=None, safe=None):
         logger.info("Closing worker %s", worker)
         with log_errors():
             self.log_event(worker, {"action": "close-worker"})
-            ws: WorkerState = parent._workers[worker]
+            ws: WorkerState = parent._workers_dv[worker]
             nanny_addr = ws._nanny
             address = nanny_addr or worker
 
@@ -3824,7 +3825,7 @@ def heartbeat_worker(
         parent: SchedulerState = cast(SchedulerState, self)
         address = self.coerce_address(address, resolve_address)
         address = normalize_address(address)
-        ws: WorkerState = parent._workers.get(address)
+        ws: WorkerState = parent._workers_dv.get(address)
         if ws is None:
             return {"status": "missing"}
 
@@ -3835,7 +3836,7 @@ def heartbeat_worker(
         dh: dict = parent._host_info.setdefault(host, {})
         dh["last-seen"] = local_now
 
-        frac = 1 / len(parent._workers)
+        frac = 1 / len(parent._workers_dv)
         parent._bandwidth = (
             parent._bandwidth * (1 - frac) + metrics["bandwidth"]["total"] * frac
         )
@@ -3907,7 +3908,7 @@ def heartbeat_worker(
         return {
             "status": "OK",
             "time": local_now,
-            "heartbeat-interval": heartbeat_interval(len(parent._workers)),
+            "heartbeat-interval": heartbeat_interval(len(parent._workers_dv)),
         }
 
     async def add_worker(
@@ -3939,7 +3940,7 @@ async def add_worker(
             address = normalize_address(address)
             host = get_address_host(address)
 
-            ws: WorkerState = parent._workers.get(address)
+            ws: WorkerState = parent._workers_dv.get(address)
             if ws is not None:
                 raise ValueError("Worker already exists %s" % ws)
 
@@ -4048,7 +4049,7 @@ async def add_worker(
             msg = {
                 "status": "OK",
                 "time": time(),
-                "heartbeat-interval": heartbeat_interval(len(parent._workers)),
+                "heartbeat-interval": heartbeat_interval(len(parent._workers_dv)),
                 "worker-plugins": self.worker_plugins,
             }
 
@@ -4056,7 +4057,7 @@ async def add_worker(
             version_warning = version_module.error_message(
                 version_module.get_versions(),
                 merge(
-                    {w: ws._versions for w, ws in parent._workers.items()},
+                    {w: ws._versions for w, ws in parent._workers_dv.items()},
                     {
                         c: cs._versions
                         for c, cs in parent._clients.items()

From 7c3ea3653b7187d685855085d9254f5d28cb8098 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Sat, 22 May 2021 15:20:02 -0500
Subject: [PATCH 1275/1550] Add performance_report to API docs (#4840)

---
 docs/source/api.rst                    | 2 ++
 docs/source/diagnosing-performance.rst | 3 ++-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/docs/source/api.rst b/docs/source/api.rst
index 472e0b9616a..0d5c858e2c6 100644
--- a/docs/source/api.rst
+++ b/docs/source/api.rst
@@ -62,6 +62,7 @@ API
    futures_of
    get_task_stream
    get_task_metadata
+   performance_report
 
 
 Asynchronous methods
@@ -158,6 +159,7 @@ Other
 .. autoclass:: distributed.Reschedule
 .. autoclass:: get_task_stream
 .. autoclass:: get_task_metadata
+.. autoclass:: performance_report
 
 .. autoclass:: Event
    :members:
diff --git a/docs/source/diagnosing-performance.rst b/docs/source/diagnosing-performance.rst
index 3b367b9b309..cf8e85aa1c0 100644
--- a/docs/source/diagnosing-performance.rst
+++ b/docs/source/diagnosing-performance.rst
@@ -121,7 +121,8 @@ Performance Reports
 Often when benchmarking and/or profiling, users may want to record a
 particular computation or even a full workflow.  Dask can save the bokeh
 dashboards as static HTML plots including the task stream, worker profiles,
-bandwidths, etc. This is done wrapping a computation with the ``performance_report`` context manager:
+bandwidths, etc. This is done wrapping a computation with the
+:class:`distributed.performance_report` context manager:
 
 .. code-block:: python
 

From d41169bf231949f7dfb69e052a9690f7381e62f7 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Mon, 24 May 2021 09:47:56 -0500
Subject: [PATCH 1276/1550] Check 'distributed.scheduler.pickle' in
 Scheduler.run_function (#4838)

---
 distributed/scheduler.py            | 7 +++++++
 distributed/tests/test_scheduler.py | 9 +++++++++
 2 files changed, 16 insertions(+)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index a335455439d..7cfe8497e24 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -6278,6 +6278,13 @@ def run_function(self, stream, function, args=(), kwargs={}, wait=True):
         """
         from .worker import run
 
+        if not dask.config.get("distributed.scheduler.pickle"):
+            raise ValueError(
+                "Cannot run function as the scheduler has been explicitly disallowed from "
+                "deserializing arbitrary bytestrings using pickle via the "
+                "'distributed.scheduler.pickle' configuration setting."
+            )
+
         self.log_event("all", {"action": "run-function", "function": function})
         return run(self, stream, function=function, args=args, kwargs=kwargs, wait=wait)
 
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 10d731fffe1..ccd7ce31dd7 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -1100,6 +1100,15 @@ def f(dask_scheduler=None):
     assert response == s.address
 
 
+@gen_cluster(client=True, config={"distributed.scheduler.pickle": False})
+async def test_run_on_scheduler_disabled(c, s, a, b):
+    def f(dask_scheduler=None):
+        return dask_scheduler.address
+
+    with pytest.raises(ValueError, match="disallowed from deserializing"):
+        await c._run_on_scheduler(f)
+
+
 @gen_cluster(client=True)
 async def test_close_worker(c, s, a, b):
     assert len(s.workers) == 2

From 833c5f6c040feaa4550fa343d8e6e4feef3f84d5 Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Tue, 25 May 2021 12:52:57 +0200
Subject: [PATCH 1277/1550] Ensure busy workloads properly look up who_has
 (#4793)

---
 distributed/tests/test_worker.py |  74 +++++++++++++++++++
 distributed/worker.py            | 117 ++++++++++++++++++-------------
 2 files changed, 143 insertions(+), 48 deletions(-)

diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index de2835a1457..b1b5eb8b48a 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -1807,11 +1807,23 @@ async def test_story_with_deps(c, s, a, b):
     expected_story = [
         (key, "new"),
         (key, "new", "waiting"),
+        # First log is what needs to be fetched in total as determined in
+        # ensure_communicating
         (
             "gather-dependencies",
             key,
             {fut.key for fut in futures},
         ),
+        # Second log may just be a subset of the above, see also
+        # Worker.select_keys_for_gather
+        # This case, it's all because Worker.target_message_size is sufficiently
+        # large
+        (
+            "request-dep",
+            key,
+            a.address,
+            {fut.key for fut in futures},
+        ),
         (key, "waiting", "ready"),
         (key, "ready", "executing"),
         (key, "put-in-memory"),
@@ -1823,3 +1835,65 @@ async def test_story_with_deps(c, s, a, b):
 def test_weight_deprecated():
     with pytest.warns(DeprecationWarning):
         weight("foo", "bar")
+
+
+@gen_cluster(client=True)
+async def test_gather_dep_one_worker_always_busy(c, s, a, b):
+    # Ensure that both dependencies for H are on another worker than H itself.
+    # The worker where the dependencies are on is then later blocked such that
+    # the data cannot be fetched
+    # In the past it was important that there is more than one key on the
+    # worker. This should be kept to avoid any edge case specific to one
+    f = c.submit(inc, 1, workers=[a.address])
+    g = c.submit(
+        inc,
+        2,
+        workers=[a.address],
+    )
+
+    await f
+    await g
+    # We will block A for any outgoing communication. This simulates an
+    # overloaded worker which will always return "busy" for get_data requests,
+    # effectively blocking H indefinitely
+    a.outgoing_current_count = 10000000
+    assert f.key in a.tasks
+    assert g.key in a.tasks
+    # Ensure there are actually two distinct tasks and not some pure=True
+    # caching
+    assert f.key != g.key
+    h = c.submit(add, f, g, workers=[b.address])
+
+    fut = asyncio.wait_for(h, 0.1)
+
+    while h.key not in b.tasks:
+        await asyncio.sleep(0.01)
+
+    ts_h = b.tasks[h.key]
+    ts_f = b.tasks[f.key]
+    ts_g = b.tasks[g.key]
+
+    with pytest.raises(asyncio.TimeoutError):
+        assert ts_h.state == "waiting"
+        assert ts_f.state in ["flight", "fetch"]
+        assert ts_g.state in ["flight", "fetch"]
+        await fut
+
+    # Ensure B wasn't lazy but tried at least once
+    assert b.repetitively_busy
+
+    x = await Worker(s.address, name="x")
+    # We "scatter" the data to another worker which is able to serve this data.
+    # In reality this could be another worker which fetched this dependency and
+    # got through to A or another worker executed the task using work stealing
+    # or any other. To avoid cross effects, we'll just put the data onto the
+    # worker ourselves
+    x.update_data(data={key: a.data[key] for key in [f.key, g.key]})
+
+    assert await h == 5
+
+    # Since we put the data onto the worker ourselves, the gather_dep might
+    # still be mid execution and we'll get a dangling task. Let it finish
+    # naturally
+    while any(["Worker.gather_dep" in str(t) for t in asyncio.all_tasks()]):
+        await asyncio.sleep(0.05)
diff --git a/distributed/worker.py b/distributed/worker.py
index f9829df6e63..494df279d75 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -16,6 +16,7 @@
 from functools import partial
 from inspect import isawaitable
 from pickle import PicklingError
+from typing import Iterable
 
 from tlz import first, keymap, merge, pluck  # noqa: F401
 from tornado import gen
@@ -1988,42 +1989,50 @@ def ensure_communicating(self):
                     changed = True
                     continue
 
-                deps = ts.dependencies
+                dependencies = ts.dependencies
                 if self.validate:
-                    assert all(dep.key in self.tasks for dep in deps)
+                    assert all(dep.key in self.tasks for dep in dependencies)
 
-                deps = {dep for dep in deps if dep.state == "fetch"}
+                dependencies_fetch = set()
+                dependencies_missing = set()
+                for dependency_ts in dependencies:
+                    if dependency_ts.state == "fetch":
+                        if not dependency_ts.who_has:
+                            dependencies_missing.add(dependency_ts)
+                        else:
+                            dependencies_fetch.add(dependency_ts)
 
-                missing_deps = {dep for dep in deps if not dep.who_has}
-                if missing_deps:
+                del dependencies
+
+                if dependencies_missing:
                     logger.info("Can't find dependencies for key %s", key)
                     missing_deps2 = {
                         dep
-                        for dep in missing_deps
+                        for dep in dependencies_missing
                         if dep.key not in self._missing_dep_flight
                     }
                     for dep in missing_deps2:
                         self._missing_dep_flight.add(dep.key)
                     self.loop.add_callback(self.handle_missing_dep, *missing_deps2)
 
-                    deps = [dep for dep in deps if dep not in missing_deps]
+                    dependencies_fetch -= dependencies_missing
 
-                log_keys = {d.key for d in deps}
-                self.log.append(("gather-dependencies", key, log_keys))
+                self.log.append(
+                    ("gather-dependencies", key, {d.key for d in dependencies_fetch})
+                )
 
                 in_flight = False
 
-                while deps and (
+                while dependencies_fetch and (
                     len(self.in_flight_workers) < self.total_out_connections
                     or self.comm_nbytes < self.total_comm_nbytes
                 ):
-                    dep = deps.pop()
-                    if dep.state != "fetch":
-                        continue
-                    if not dep.who_has:
-                        continue
+                    to_gather_ts = dependencies_fetch.pop()
+
                     workers = [
-                        w for w in dep.who_has if w not in self.in_flight_workers
+                        w
+                        for w in to_gather_ts.who_has
+                        if w not in self.in_flight_workers
                     ]
                     if not workers:
                         in_flight = True
@@ -2035,18 +2044,23 @@ def ensure_communicating(self):
                     else:
                         worker = random.choice(list(workers))
                     to_gather, total_nbytes = self.select_keys_for_gather(
-                        worker, dep.key
+                        worker, to_gather_ts.key
                     )
                     self.comm_nbytes += total_nbytes
                     self.in_flight_workers[worker] = to_gather
                     for d in to_gather:
+                        dependencies_fetch.discard(self.tasks.get(d))
                         self.transition(self.tasks[d], "flight", worker=worker)
                     self.loop.add_callback(
-                        self.gather_dep, worker, dep, to_gather, total_nbytes, cause=key
+                        self.gather_dep,
+                        worker=worker,
+                        to_gather=to_gather,
+                        total_nbytes=total_nbytes,
+                        cause=ts,
                     )
                     changed = True
 
-                if not deps and not in_flight:
+                if not dependencies_fetch and not in_flight:
                     self.data_needed.popleft()
 
         except Exception as e:
@@ -2154,60 +2168,67 @@ def select_keys_for_gather(self, worker, dep):
 
         return deps, total_bytes
 
-    async def gather_dep(self, worker, dep, deps, total_nbytes, cause=None):
+    async def gather_dep(
+        self,
+        worker: str,
+        to_gather: Iterable[str],
+        total_nbytes: int,
+        cause: TaskState,
+    ):
         """Gather dependencies for a task from a worker who has them
 
         Parameters
         ----------
         worker : str
-            address of worker to gather dependency from
-        dep : TaskState
-            task we want to gather dependencies for
-        deps : list
-            keys of dependencies to gather from worker -- this is not
+            Address of worker to gather dependencies from
+        to_gather : list
+            Keys of dependencies to gather from worker -- this is not
             necessarily equivalent to the full list of dependencies of ``dep``
             as some dependencies may already be present on this worker.
+        total_nbytes : int
+            Total number of bytes for all the dependencies in to_gather combined
+        cause : TaskState
+            Task we want to gather dependencies for
         """
         if self.status != Status.running:
             return
         with log_errors():
             response = {}
+            to_gather_keys = set()
             try:
                 if self.validate:
                     self.validate_state()
+                for dependency_key in to_gather:
+                    dependency_ts = self.tasks.get(dependency_key)
+                    if dependency_ts and dependency_ts.state == "flight":
+                        to_gather_keys.add(dependency_key)
+                del to_gather
 
-                # dep states may have changed before gather_dep runs
-                # if a dep is no longer in-flight then don't fetch it
-                deps_ts = [self.tasks.get(key, None) or TaskState(key) for key in deps]
-                deps_ts = tuple(ts for ts in deps_ts if ts.state == "flight")
-                deps = [d.key for d in deps_ts]
-
-                self.log.append(("request-dep", dep.key, worker, deps))
-                logger.debug("Request %d keys", len(deps))
+                self.log.append(("request-dep", cause.key, worker, to_gather_keys))
+                logger.debug("Request %d keys for task %s", len(to_gather_keys), cause)
 
                 start = time()
                 response = await get_data_from_worker(
-                    self.rpc, deps, worker, who=self.address
+                    self.rpc, to_gather_keys, worker, who=self.address
                 )
                 stop = time()
 
                 if response["status"] == "busy":
-                    self.log.append(("busy-gather", worker, deps))
-                    for ts in deps_ts:
-                        if ts.state == "flight":
+                    self.log.append(("busy-gather", worker, to_gather_keys))
+                    for key in to_gather_keys:
+                        ts = self.tasks.get(key)
+                        if ts and ts.state == "flight":
                             self.transition(ts, "fetch")
                     return
 
-                if cause:
-                    cause_ts = self.tasks.get(cause, TaskState(key=cause))
-                    cause_ts.startstops.append(
-                        {
-                            "action": "transfer",
-                            "start": start + self.scheduler_delay,
-                            "stop": stop + self.scheduler_delay,
-                            "source": worker,
-                        }
-                    )
+                cause.startstops.append(
+                    {
+                        "action": "transfer",
+                        "start": start + self.scheduler_delay,
+                        "stop": stop + self.scheduler_delay,
+                        "source": worker,
+                    }
+                )
 
                 total_bytes = sum(
                     self.tasks[key].get_nbytes()
@@ -2299,7 +2320,7 @@ async def gather_dep(self, worker, dep, deps, total_nbytes, cause=None):
                     self.repetitively_busy += 1
                     await asyncio.sleep(0.100 * 1.5 ** self.repetitively_busy)
 
-                    await self.query_who_has(dep.key)
+                    await self.query_who_has(*to_gather_keys)
                     self.ensure_communicating()
 
     def bad_dep(self, dep):

From 8c20aebc5992e4eceb989e1a76113e23c38b03d8 Mon Sep 17 00:00:00 2001
From: Peter Andreas Entschev <peter@entschev.com>
Date: Tue, 25 May 2021 19:52:10 +0200
Subject: [PATCH 1278/1550] Update handling of UCX exceptions on endpoint
 closing (#4836)

* Updates for UCX 1.10

* Update RDMACM for UCX 1.10

* Ensure UCX creates CUDA context when cuda_copy is enabled

* UCX retry connection with different port if that is already taken

* Adjust UCX conditions for versions older and newer than 1.10

* Raise ImportError if Numba isn't available with UCX+CUDA

* Do not retry on ucx_create_listener

* Fix UCX endpoint closing

* Fix catching UCXConnectionReset

* Catch other possible exceptions during UCX.close()

* Catch missing CommClosedError exception during send_recv

* Update UCX reuse endpoints defaults

* Add description for reuse-endpoints config

* Fix wrong formatting in distributed-schema

* Update reuse-endpoints description
---
 distributed/comm/ucx.py             | 44 +++++++++++++++++------------
 distributed/core.py                 |  2 +-
 distributed/distributed-schema.yaml | 10 +++++--
 distributed/distributed.yaml        |  2 +-
 4 files changed, 35 insertions(+), 23 deletions(-)

diff --git a/distributed/comm/ucx.py b/distributed/comm/ucx.py
index b0bb62b5852..4d10a898004 100644
--- a/distributed/comm/ucx.py
+++ b/distributed/comm/ucx.py
@@ -11,15 +11,7 @@
 
 import dask
 
-from ..utils import (
-    CancelledError,
-    ensure_ip,
-    get_ip,
-    get_ipv6,
-    log_errors,
-    nbytes,
-    parse_bytes,
-)
+from ..utils import ensure_ip, get_ip, get_ipv6, log_errors, nbytes, parse_bytes
 from .addressing import parse_host_port, unparse_host_port
 from .core import Comm, CommClosedError, Connector, Listener
 from .registry import Backend, backends
@@ -128,12 +120,15 @@ def device_array(n):
         ucx_create_endpoint = ucp.create_endpoint
         ucx_create_listener = ucp.create_listener
     else:
-        if dask.config.get("ucx.reuse-endpoints"):
-            ucx_create_endpoint = EndpointReuse.create_endpoint
-            ucx_create_listener = EndpointReuse.create_listener
-        else:
+        reuse_endpoints = dask.config.get("ucx.reuse-endpoints")
+        if (
+            reuse_endpoints is None and ucp.get_ucx_version() >= (1, 11, 0)
+        ) or reuse_endpoints is False:
             ucx_create_endpoint = ucp.create_endpoint
             ucx_create_listener = ucp.create_listener
+        else:
+            ucx_create_endpoint = EndpointReuse.create_endpoint
+            ucx_create_listener = EndpointReuse.create_listener
 
 
 class UCX(Comm):
@@ -266,7 +261,7 @@ async def read(self, deserializers=("cuda", "dask", "pickle", "error")):
                 (shutdown, nframes) = struct.unpack("?Q", msg)
 
                 if shutdown:  # The writer is closing the connection
-                    raise CancelledError("Connection closed by writer")
+                    raise CommClosedError("Connection closed by writer")
 
                 # Recv which frames are CUDA (bool) and
                 # how large each frame is (uint64)
@@ -275,9 +270,12 @@ async def read(self, deserializers=("cuda", "dask", "pickle", "error")):
                 await self.ep.recv(header)
                 header = struct.unpack(header_fmt, header)
                 cuda_frames, sizes = header[:nframes], header[nframes:]
-            except (ucp.exceptions.UCXBaseException, CancelledError):
+            except (
+                ucp.exceptions.UCXCloseError,
+                ucp.exceptions.UCXCanceled,
+            ) + (getattr(ucp.exceptions, "UCXConnectionReset", ()),):
                 self.abort()
-                raise CommClosedError("While reading, the connection was closed")
+                raise CommClosedError("Connection closed by writer")
             else:
                 # Recv frames
                 frames = [
@@ -311,7 +309,11 @@ async def close(self):
         if self._ep is not None:
             try:
                 await self.ep.send(struct.pack("?Q", True, 0))
-            except ucp.exceptions.UCXError:
+            except (
+                ucp.exceptions.UCXError,
+                ucp.exceptions.UCXCloseError,
+                ucp.exceptions.UCXCanceled,
+            ) + (getattr(ucp.exceptions, "UCXConnectionReset", ()),):
                 # If the other end is in the process of closing,
                 # UCX will sometimes raise a `Input/output` error,
                 # which we can ignore.
@@ -344,7 +346,13 @@ async def connect(self, address: str, deserialize=True, **connection_args) -> UC
         logger.debug("UCXConnector.connect: %s", address)
         ip, port = parse_host_port(address)
         init_once()
-        ep = await ucx_create_endpoint(ip, port)
+        try:
+            ep = await ucx_create_endpoint(ip, port)
+        except (
+            ucp.exceptions.UCXCloseError,
+            ucp.exceptions.UCXCanceled,
+        ) + (getattr(ucp.exceptions, "UCXConnectionReset", ()),):
+            raise CommClosedError("Connection closed before handshake completed")
         return self.comm_class(
             ep,
             local_addr=None,
diff --git a/distributed/core.py b/distributed/core.py
index cfd84bce95a..a44076becff 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -645,7 +645,7 @@ async def send_recv(comm, reply=True, serializers=None, deserializers=None, **kw
             response = await comm.read(deserializers=deserializers)
         else:
             response = None
-    except EnvironmentError:
+    except (EnvironmentError, CommClosedError):
         # On communication errors, we should simply close the communication
         force_close = True
         raise
diff --git a/distributed/distributed-schema.yaml b/distributed/distributed-schema.yaml
index 0e8faf2f19d..eac459d7aa7 100644
--- a/distributed/distributed-schema.yaml
+++ b/distributed/distributed-schema.yaml
@@ -855,6 +855,10 @@ properties:
           and compiled with hwloc support. Unexpected errors can occur when using
           ``"auto"`` if any interfaces are disconnected or improperly configured.
       reuse-endpoints:
-        type: boolean
-        description: | 
-          Whether to reuse endpoints or not.
+        type: [boolean, 'null']
+        description: |
+          Enable UCX-Py reuse endpoints mechanism if ``True`` or if it's not specified and
+          UCX < 1.11 is installed, otherwise disable reuse endpoints. This was primarily
+          introduced to resolve an issue with CUDA IPC that has been fixed in UCX 1.10, but
+          can cause establishing endpoints to be very slow, this is particularly noticeable in
+          clusters of more than a few dozen workers.
diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index c71845e3769..ff34e48bc32 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -196,4 +196,4 @@ ucx:
   infiniband: False # enable Infiniband
   rdmacm: False # enable RDMACM
   net-devices: null  # define what interface to use for UCX comm
-  reuse-endpoints: True  # enable endpoint reuse
+  reuse-endpoints: null  # enable endpoint reuse

From 1c9ebaf64ebec0d817fbfb7ea04dc012d0c9375b Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Wed, 26 May 2021 03:57:17 +0200
Subject: [PATCH 1279/1550] Ensure exceptions are raised if workers are
 incorrectly started (#4733)

---
 distributed/core.py                    |   3 -
 distributed/deploy/tests/test_local.py |  18 +++
 distributed/nanny.py                   | 209 ++++++++++++++-----------
 distributed/tests/test_core.py         |   6 +
 distributed/tests/test_nanny.py        |  12 +-
 5 files changed, 155 insertions(+), 93 deletions(-)

diff --git a/distributed/core.py b/distributed/core.py
index a44076becff..d769611a4b7 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -136,7 +136,6 @@ def __init__(
         connection_args=None,
         timeout=None,
         io_loop=None,
-        **kwargs,
     ):
         self.handlers = {
             "identity": self.identity,
@@ -238,8 +237,6 @@ def set_thread_ident():
 
         self.__stopped = False
 
-        super().__init__(**kwargs)
-
     @property
     def status(self):
         return self._status
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index a8b08513d5d..d12c8bfc131 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -1063,3 +1063,21 @@ async def test_cluster_names():
 
         async with LocalCluster(processes=False, asynchronous=True) as unnamed_cluster2:
             assert unnamed_cluster2 != unnamed_cluster
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("nanny", [True, False])
+async def test_local_cluster_redundant_kwarg(nanny):
+    with pytest.raises(TypeError, match="unexpected keyword argument"):
+        # Extra arguments are forwarded to the worker class. Depending on
+        # whether we use the nanny or not, the error treatment is quite
+        # different and we should assert that an exception is raised
+        async with await LocalCluster(
+            typo_kwarg="foo", processes=nanny, n_workers=1
+        ) as cluster:
+
+            # This will never work but is a reliable way to block without hard
+            # coding any sleep values
+            async with Client(cluster) as c:
+                f = c.submit(sleep, 0)
+                await f
diff --git a/distributed/nanny.py b/distributed/nanny.py
index 127906b1bb1..3d9fb1855e0 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -9,6 +9,7 @@
 import weakref
 from contextlib import suppress
 from multiprocessing.queues import Empty
+from time import sleep as sync_sleep
 
 import psutil
 from tornado import gen
@@ -362,7 +363,6 @@ async def instantiate(self, comm=None) -> Status:
                 config=self.config,
             )
 
-        self.auto_restart = True
         if self.death_timeout:
             try:
                 result = await asyncio.wait_for(
@@ -378,7 +378,11 @@ async def instantiate(self, comm=None) -> Status:
                 raise
 
         else:
-            result = await self.process.start()
+            try:
+                result = await self.process.start()
+            except Exception:
+                await self.close()
+                raise
         return result
 
     async def restart(self, comm=None, timeout=2, executor_wait=True):
@@ -414,9 +418,10 @@ def memory_monitor(self):
         """Track worker's memory.  Restart if it goes above terminate fraction"""
         if self.status != Status.running:
             return
+        if self.process is None or self.process.process is None:
+            return None
         process = self.process.process
-        if process is None:
-            return
+
         try:
             proc = self._psutil_process
             memory = proc.memory_info().rss
@@ -519,6 +524,9 @@ async def close(self, comm=None, timeout=5, report=None):
 
 
 class WorkerProcess:
+    # The interval how often to check the msg queue for init
+    _init_msg_interval = 0.05
+
     def __init__(
         self,
         worker_kwargs,
@@ -584,9 +592,14 @@ async def start(self) -> Status:
         except OSError:
             logger.exception("Nanny failed to start process", exc_info=True)
             self.process.terminate()
-            return
-
-        msg = await self._wait_until_connected(uid)
+            self.status = Status.failed
+            return self.status
+        try:
+            msg = await self._wait_until_connected(uid)
+        except Exception:
+            self.status = Status.failed
+            self.process.terminate()
+            raise
         if not msg:
             return self.status
         self.worker_address = msg["address"]
@@ -683,14 +696,15 @@ async def kill(self, timeout=2, executor_wait=True):
                 logger.error("Failed to kill worker process: %s", e)
 
     async def _wait_until_connected(self, uid):
-        delay = 0.05
         while True:
             if self.status != Status.starting:
                 return
+            # This is a multiprocessing queue and we'd block the event loop if
+            # we simply called get
             try:
                 msg = self.init_result_q.get_nowait()
             except Empty:
-                await asyncio.sleep(delay)
+                await asyncio.sleep(self._init_msg_interval)
                 continue
 
             if msg["uid"] != uid:  # ensure that we didn't cross queues
@@ -700,7 +714,6 @@ async def _wait_until_connected(self, uid):
                 logger.error(
                     "Failed while trying to start worker process: %s", msg["exception"]
                 )
-                await self.process.join()
                 raise msg["exception"]
             else:
                 return msg
@@ -718,88 +731,108 @@ def _run(
         config,
         Worker,
     ):  # pragma: no cover
-        os.environ.update(env)
-        dask.config.set(config)
         try:
-            from dask.multiprocessing import initialize_worker_process
-        except ImportError:  # old Dask version
-            pass
-        else:
-            initialize_worker_process()
+            os.environ.update(env)
+            dask.config.set(config)
+            try:
+                from dask.multiprocessing import initialize_worker_process
+            except ImportError:  # old Dask version
+                pass
+            else:
+                initialize_worker_process()
 
-        if silence_logs:
-            logger.setLevel(silence_logs)
+            if silence_logs:
+                logger.setLevel(silence_logs)
 
-        IOLoop.clear_instance()
-        loop = IOLoop()
-        loop.make_current()
-        worker = Worker(**worker_kwargs)
+            IOLoop.clear_instance()
+            loop = IOLoop()
+            loop.make_current()
+            worker = Worker(**worker_kwargs)
 
-        async def do_stop(timeout=5, executor_wait=True):
-            try:
-                await worker.close(
-                    report=True,
-                    nanny=False,
-                    safe=True,  # TODO: Graceful or not?
-                    executor_wait=executor_wait,
-                    timeout=timeout,
-                )
-            finally:
-                loop.stop()
-
-        def watch_stop_q():
-            """
-            Wait for an incoming stop message and then stop the
-            worker cleanly.
-            """
-            while True:
-                try:
-                    msg = child_stop_q.get(timeout=1000)
-                except Empty:
-                    pass
-                else:
-                    child_stop_q.close()
-                    assert msg.pop("op") == "stop"
-                    loop.add_callback(do_stop, **msg)
-                    break
-
-        t = threading.Thread(target=watch_stop_q, name="Nanny stop queue watch")
-        t.daemon = True
-        t.start()
-
-        async def run():
-            """
-            Try to start worker and inform parent of outcome.
-            """
-            try:
-                await worker
-            except Exception as e:
-                logger.exception("Failed to start worker")
-                init_result_q.put({"uid": uid, "exception": e})
-                init_result_q.close()
-            else:
+            async def do_stop(timeout=5, executor_wait=True):
                 try:
-                    assert worker.address
-                except ValueError:
-                    pass
-                else:
-                    init_result_q.put(
-                        {
-                            "address": worker.address,
-                            "dir": worker.local_directory,
-                            "uid": uid,
-                        }
+                    await worker.close(
+                        report=True,
+                        nanny=False,
+                        safe=True,  # TODO: Graceful or not?
+                        executor_wait=executor_wait,
+                        timeout=timeout,
                     )
+                finally:
+                    loop.stop()
+
+            def watch_stop_q():
+                """
+                Wait for an incoming stop message and then stop the
+                worker cleanly.
+                """
+                while True:
+                    try:
+                        msg = child_stop_q.get(timeout=1000)
+                    except Empty:
+                        pass
+                    else:
+                        child_stop_q.close()
+                        assert msg.pop("op") == "stop"
+                        loop.add_callback(do_stop, **msg)
+                        break
+
+            t = threading.Thread(target=watch_stop_q, name="Nanny stop queue watch")
+            t.daemon = True
+            t.start()
+
+            async def run():
+                """
+                Try to start worker and inform parent of outcome.
+                """
+                try:
+                    await worker
+                except Exception as e:
+                    logger.exception("Failed to start worker")
+                    init_result_q.put({"uid": uid, "exception": e})
                     init_result_q.close()
-                    await worker.finished()
-                    logger.info("Worker closed")
-
-        try:
-            loop.run_sync(run)
-        except (TimeoutError, gen.TimeoutError):
-            # Loop was stopped before wait_until_closed() returned, ignore
-            pass
-        except KeyboardInterrupt:
-            # At this point the loop is not running thus we have to run
-            # do_stop() explicitly.
-            loop.run_sync(do_stop)
+                    # If we hit an exception here we need to wait for a least
+                    # one interval for the outside to pick up this message.
+                    # Otherwise we arrive in a race condition where the process
+                    # cleanup wipes the queue before the exception can be
+                    # properly handled. See also
+                    # WorkerProcess._wait_until_connected (the 2 is for good
+                    # measure)
+                    sync_sleep(cls._init_msg_interval * 2)
+                else:
+                    try:
+                        assert worker.address
+                    except ValueError:
+                        pass
+                    else:
+                        init_result_q.put(
+                            {
+                                "address": worker.address,
+                                "dir": worker.local_directory,
+                                "uid": uid,
+                            }
+                        )
+                        init_result_q.close()
+                        await worker.finished()
+                        logger.info("Worker closed")
+
+        except Exception as e:
+            logger.exception("Failed to initialize Worker")
+            init_result_q.put({"uid": uid, "exception": e})
+            init_result_q.close()
+            # If we hit an exception here we need to wait for a least one
+            # interval for the outside to pick up this message. Otherwise we
+            # arrive in a race condition where the process cleanup wipes the
+            # queue before the exception can be properly handled. See also
+            # WorkerProcess._wait_until_connected (the 2 is for good measure)
+            sync_sleep(cls._init_msg_interval * 2)
+        else:
+            try:
+                loop.run_sync(run)
+            except (TimeoutError, gen.TimeoutError):
+                # Loop was stopped before wait_until_closed() returned, ignore
+                pass
+            except KeyboardInterrupt:
+                # At this point the loop is not running thus we have to run
+                # do_stop() explicitly.
+                loop.run_sync(do_stop)
diff --git a/distributed/tests/test_core.py b/distributed/tests/test_core.py
index 6dd76f901ec..de0218e7741 100644
--- a/distributed/tests/test_core.py
+++ b/distributed/tests/test_core.py
@@ -880,3 +880,9 @@ async def sleep(comm=None):
 
         # weakref set/dict should be cleaned up
         assert not len(server._ongoing_coroutines)
+
+
+@pytest.mark.asyncio
+async def test_server_redundant_kwarg():
+    with pytest.raises(TypeError, match="unexpected keyword argument"):
+        await Server({}, typo_kwarg="foo")
diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
index 779cbdc4a4a..1203fb3bbed 100644
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -14,7 +14,6 @@
 import dask
 
 from distributed import Client, Nanny, Scheduler, Worker, rpc, wait, worker
-from distributed.compatibility import MACOS
 from distributed.core import CommClosedError, Status
 from distributed.diagnostics import SchedulerPlugin
 from distributed.metrics import time
@@ -565,10 +564,19 @@ async def start(self):
         raise StartException("broken")
 
 
-@pytest.mark.flaky(reruns=10, reruns_delay=5, condition=MACOS)
 @pytest.mark.asyncio
 async def test_worker_start_exception(cleanup):
     # make sure this raises the right Exception:
     with pytest.raises(StartException):
         async with Nanny("tcp://localhost:1", worker_class=BrokenWorker) as n:
             await n.start()
+
+
+@pytest.mark.asyncio
+async def test_failure_during_worker_initialization(cleanup):
+    with captured_logger(logger="distributed.nanny", level=logging.WARNING) as logs:
+        async with Scheduler() as s:
+            with pytest.raises(Exception):
+                async with Nanny(s.address, foo="bar") as n:
+                    await n
+        assert "Restarting worker" not in logs.getvalue()

From deda73ed1de058c7e261fff3decea0f972c7d1f0 Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Wed, 26 May 2021 13:30:14 +0200
Subject: [PATCH 1280/1550] Fix warning for attribute error when deleting a
 client (#4807)

---
 distributed/client.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/client.py b/distributed/client.py
index f214a303b44..64af33758fc 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -1419,7 +1419,7 @@ def close(self, timeout=no_default):
         if timeout == no_default:
             timeout = self._timeout * 2
         # XXX handling of self.status here is not thread-safe
-        if self.status == "closed":
+        if self.status in ["closed", "newly-created"]:
             if self.asynchronous:
                 future = asyncio.Future()
                 future.set_result(None)

From e55301bc8093909b49c0e89c16d63be5111b0886 Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Wed, 26 May 2021 13:51:58 +0200
Subject: [PATCH 1281/1550] Ensure worker clients are closed (#3921)

* Ensure client set_as_default is respected

* Ensure workers close clients upon closing
---
 distributed/client.py            |  9 +--
 distributed/tests/test_client.py | 26 +++++++++
 distributed/tests/test_worker.py | 99 ++++++++++++++++++++++++++++++++
 distributed/worker.py            | 25 ++++++++
 4 files changed, 155 insertions(+), 4 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 64af33758fc..919d9181f2a 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -715,6 +715,7 @@ def __init__(
         )
 
         self._start_arg = address
+        self._set_as_default = set_as_default
         if set_as_default:
             self._set_config = dask.config.set(
                 scheduler="dask.distributed", shuffle="tasks"
@@ -958,8 +959,8 @@ def start(self, **kwargs):
             return
 
         self._loop_runner.start()
-
-        _set_global_client(self)
+        if self._set_as_default:
+            _set_global_client(self)
         self.status = "connecting"
 
         if self.asynchronous:
@@ -1150,8 +1151,8 @@ async def _ensure_connected(self, timeout=None):
         bcomm = BatchedSend(interval="10ms", loop=self.loop)
         bcomm.start(comm)
         self.scheduler_comm = bcomm
-
-        _set_global_client(self)
+        if self._set_as_default:
+            _set_global_client(self)
         self.status = "running"
 
         for msg in self._pending_msg_buffer:
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index ec595dc815f..6fbbae95700 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -3344,6 +3344,32 @@ def test_default_get():
         assert dask.base.get_scheduler() == pre_get
 
 
+@gen_cluster()
+async def test_set_as_default(s, a, b):
+    with pytest.raises(ValueError):
+        default_client()
+
+    async with Client(s.address, set_as_default=False, asynchronous=True) as c1:
+        with pytest.raises(ValueError):
+            default_client()
+        async with Client(s.address, set_as_default=True, asynchronous=True) as c2:
+            assert default_client() is c2
+            async with Client(s.address, set_as_default=True, asynchronous=True) as c3:
+                assert default_client() is c3
+                async with Client(
+                    s.address, set_as_default=False, asynchronous=True
+                ) as c4:
+                    assert default_client() is c3
+
+                    await c4.scheduler_comm.close()
+                    while c4.status != "running":
+                        await asyncio.sleep(0.01)
+                    assert default_client() is c3
+
+    with pytest.raises(ValueError):
+        default_client()
+
+
 @gen_cluster(client=True)
 async def test_get_processing(c, s, a, b):
     processing = await c.processing()
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index b1b5eb8b48a..7a9feffc1c7 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -1897,3 +1897,102 @@ async def test_gather_dep_one_worker_always_busy(c, s, a, b):
     # naturally
     while any(["Worker.gather_dep" in str(t) for t in asyncio.all_tasks()]):
         await asyncio.sleep(0.05)
+
+
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 0)])
+async def test_worker_client_uses_default_no_close(c, s, a):
+    """
+    If a default client is available in the process, the worker will pick this
+    one and will not close it if it is closed
+    """
+    assert default_client() is c
+    existing_client = c.id
+
+    def get_worker_client_id():
+        def_client = get_client()
+        return def_client.id
+
+    worker_client = await c.submit(get_worker_client_id)
+    assert worker_client == existing_client
+
+    assert not Worker._initialized_clients
+
+    await a.close()
+
+    assert len(Client._instances) == 1
+    assert c.status == "running"
+    c_def = default_client()
+    assert c is c_def
+
+
+@gen_cluster(nthreads=[("127.0.0.1", 0)])
+async def test_worker_client_closes_if_created_on_worker_one_worker(s, a):
+    async with Client(s.address, set_as_default=False, asynchronous=True) as c:
+
+        with pytest.raises(ValueError):
+            default_client()
+
+        def get_worker_client_id():
+            def_client = get_client()
+            return def_client.id
+
+        new_client_id = await c.submit(get_worker_client_id)
+        default_client_id = await c.submit(get_worker_client_id)
+        assert new_client_id != c.id
+        assert new_client_id == default_client_id
+
+        new_client = default_client()
+        assert new_client_id == new_client.id
+        assert new_client.status == "running"
+
+        # If a worker closes, all clients created on it should close as well
+        await a.close()
+        assert new_client.status == "closed"
+
+        assert len(Client._instances) == 2
+
+        assert c.status == "running"
+
+        with pytest.raises(ValueError):
+            default_client()
+
+
+@gen_cluster(timeout=3600)
+async def test_worker_client_closes_if_created_on_worker_last_worker_alive(s, a, b):
+    async with Client(s.address, set_as_default=False, asynchronous=True) as c:
+
+        with pytest.raises(ValueError):
+            default_client()
+
+        def get_worker_client_id():
+            def_client = get_client()
+            return def_client.id
+
+        new_client_id = await c.submit(get_worker_client_id, workers=[a.address])
+        default_client_id = await c.submit(get_worker_client_id, workers=[a.address])
+
+        default_client_id_b = await c.submit(get_worker_client_id, workers=[b.address])
+        assert not b._comms
+        assert new_client_id != c.id
+        assert new_client_id == default_client_id
+        assert new_client_id == default_client_id_b
+
+        new_client = default_client()
+        assert new_client_id == new_client.id
+        assert new_client.status == "running"
+
+        # We'll close A. This should *not* close the client since the client is also used by B
+        await a.close()
+        assert new_client.status == "running"
+
+        client_id_b_after = await c.submit(get_worker_client_id, workers=[b.address])
+        assert client_id_b_after == default_client_id_b
+
+        assert len(Client._instances) == 2
+        await b.close()
+        assert new_client.status == "closed"
+
+        assert c.status == "running"
+
+        with pytest.raises(ValueError):
+            default_client()
diff --git a/distributed/worker.py b/distributed/worker.py
index 494df279d75..e7746b43af8 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -363,6 +363,7 @@ class Worker(ServerNode):
     """
 
     _instances = weakref.WeakSet()
+    _initialized_clients = weakref.WeakSet()
 
     def __init__(
         self,
@@ -1236,6 +1237,29 @@ async def close(
 
             for pc in self.periodic_callbacks.values():
                 pc.stop()
+
+            if self._client:
+                # If this worker is the last one alive, clean up the worker
+                # initialized clients
+                if not any(
+                    w
+                    for w in Worker._instances
+                    if w != self and w.status == Status.running
+                ):
+                    for c in Worker._initialized_clients:
+                        # Regardless of what the client was initialized with
+                        # we'll require the result as a future. This is
+                        # necessary since the heursitics of asynchronous are not
+                        # reliable and we might deadlock here
+                        c._asynchronous = True
+                        if c.asynchronous:
+                            await c.close()
+                        else:
+                            # There is still the chance that even with us
+                            # telling the client to be async, itself will decide
+                            # otherwise
+                            c.close()
+
             with suppress(EnvironmentError, TimeoutError):
                 if report and self.contact_address is not None:
                     await asyncio.wait_for(
@@ -3256,6 +3280,7 @@ def _get_client(self, timeout=None):
                 name="worker",
                 timeout=timeout,
             )
+            Worker._initialized_clients.add(self._client)
             if not asynchronous:
                 assert self._client.status == "running"
 

From b03fd811b4b65b01650b15cbfff2a3aa79a2cdbb Mon Sep 17 00:00:00 2001
From: Peter Andreas Entschev <peter@entschev.com>
Date: Wed, 26 May 2021 13:59:07 +0200
Subject: [PATCH 1282/1550] Fix UCX scrub config logging (#4850)

---
 distributed/comm/ucx.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/distributed/comm/ucx.py b/distributed/comm/ucx.py
index 4d10a898004..de349d1f7e4 100644
--- a/distributed/comm/ucx.py
+++ b/distributed/comm/ucx.py
@@ -520,11 +520,9 @@ def _scrub_ucx_config():
             options["NET_DEVICES"] = net_devices
 
     # ANY UCX options defined in config will overwrite high level dask.ucx flags
-    valid_ucx_keys = list(get_config().keys())
-    for k, v in dask.config.get("ucx").items():
-        if k in valid_ucx_keys:
-            options[k] = v
-        else:
+    valid_ucx_vars = list(get_config().keys())
+    for k, v in options.items():
+        if k not in valid_ucx_vars:
             logger.debug(
                 "Key: %s with value: %s not a valid UCX configuration option" % (k, v)
             )

From 9dd714291b09fe05cde7b48b596b7ffdacfe84e5 Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Wed, 26 May 2021 15:55:19 +0200
Subject: [PATCH 1283/1550] Let servers close faster if there are no active
 handlers (#4805)

---
 distributed/core.py            | 13 +++++--
 distributed/tests/test_core.py | 67 ++++++++++++++++++++++++++++++++++
 2 files changed, 76 insertions(+), 4 deletions(-)

diff --git a/distributed/core.py b/distributed/core.py
index d769611a4b7..4608462de76 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -525,6 +525,8 @@ async def handle_comm(self, comm):
                             e,
                         )
                         break
+
+                self._comms[comm] = None
                 msg = result = None
                 if close_desired:
                     await comm.close()
@@ -595,15 +597,18 @@ async def handle_stream(self, comm, extra=None, every_cycle=[]):
     def close(self):
         for pc in self.periodic_callbacks.values():
             pc.stop()
+        self.__stopped = True
         for listener in self.listeners:
             future = listener.stop()
             if inspect.isawaitable(future):
                 yield future
-        for i in range(20):  # let comms close naturally for a second
-            if not self._comms:
-                break
-            else:
+        for i in range(20):
+            # If there are still handlers running at this point, give them a
+            # second to finish gracefully themselves, otherwise...
+            if any(self._comms.values()):
                 yield asyncio.sleep(0.05)
+            else:
+                break
         yield [comm.close() for comm in list(self._comms)]  # then forcefully close
         for cb in self._ongoing_coroutines:
             cb.cancel()
diff --git a/distributed/tests/test_core.py b/distributed/tests/test_core.py
index de0218e7741..ac6dff5f4e8 100644
--- a/distributed/tests/test_core.py
+++ b/distributed/tests/test_core.py
@@ -886,3 +886,70 @@ async def sleep(comm=None):
 async def test_server_redundant_kwarg():
     with pytest.raises(TypeError, match="unexpected keyword argument"):
         await Server({}, typo_kwarg="foo")
+
+
+async def test_server_comms_mark_active_handlers():
+    """Whether handlers are active can be read off of the self._comms values.
+    ensure this is properly reflected and released. The sentinel for
+    "open comm but no active handler" is `None`
+    """
+
+    async def long_handler(comm):
+        await asyncio.sleep(0.2)
+        return "done"
+
+    server = await Server({"wait": long_handler})
+    await server.listen(0)
+    assert server._comms == {}
+
+    comm = await connect(server.address)
+    await comm.write({"op": "wait"})
+    while not server._comms:
+        await asyncio.sleep(0.05)
+    assert set(server._comms.values()) == {"wait"}
+    assert await comm.read() == "done"
+    assert set(server._comms.values()) == {None}
+    await comm.close()
+    while server._comms:
+        await asyncio.sleep(0.01)
+
+
+@pytest.mark.asyncio
+async def test_close_fast_without_active_handlers():
+    async def very_fast(comm):
+        return "done"
+
+    server = await Server({"do_stuff": very_fast})
+    await server.listen(0)
+    assert server._comms == {}
+
+    comm = await connect(server.address)
+    await comm.write({"op": "do_stuff"})
+    while not server._comms:
+        await asyncio.sleep(0.05)
+    fut = server.close()
+
+    await asyncio.wait_for(fut, 0.1)
+
+
+@pytest.mark.asyncio
+async def test_close_grace_period_for_handlers():
+    async def long_handler(comm, delay=10):
+        await asyncio.sleep(delay)
+        return "done"
+
+    server = await Server({"wait": long_handler})
+    await server.listen(0)
+    assert server._comms == {}
+
+    comm = await connect(server.address)
+    await comm.write({"op": "wait"})
+    while not server._comms:
+        await asyncio.sleep(0.05)
+    fut = server.close()
+    # since the handler is running for a while, the close will not immediately
+    # go through. We'll give the comm about a second to close itself
+    with pytest.raises(asyncio.TimeoutError):
+        await asyncio.wait_for(fut, 0.5)
+    await comm.close()
+    await server.close()

From 47d3cf232f4c1f77a2f0b423ddf4981a69587118 Mon Sep 17 00:00:00 2001
From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com>
Date: Wed, 26 May 2021 12:39:12 -0400
Subject: [PATCH 1284/1550] Add System tab to performance reports (#4561)

* Add System tab to performance report

* Bump requirements for 'pip install .'

* Add last argument to system monitor + perf report

* Use run_on_scheduler to get monitor count

* More descriptive name for 'last'

* Grab latest dask from main

* Add min/max/mean labels to system plots

* Run pre-commit hooks
---
 distributed/client.py                      |  5 +-
 distributed/dashboard/components/shared.py | 55 ++++++++++++++++++++--
 distributed/scheduler.py                   | 11 ++++-
 3 files changed, 65 insertions(+), 6 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 919d9181f2a..e114d869bc8 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -4701,6 +4701,9 @@ def __init__(self, filename="dask-report.html", stacklevel=1):
 
     async def __aenter__(self):
         self.start = time()
+        self.last_count = await get_client().run_on_scheduler(
+            lambda dask_scheduler: dask_scheduler.monitor.count
+        )
         await get_client().get_task_stream(start=0, stop=0)  # ensure plugin
 
     async def __aexit__(self, typ, value, traceback, code=None):
@@ -4711,7 +4714,7 @@ async def __aexit__(self, typ, value, traceback, code=None):
             except Exception:
                 code = ""
         data = await get_client().scheduler.performance_report(
-            start=self.start, code=code
+            start=self.start, last_count=self.last_count, code=code
         )
         with open(self.filename, "w") as f:
             f.write(data)
diff --git a/distributed/dashboard/components/shared.py b/distributed/dashboard/components/shared.py
index 4aa2561c831..f50145f0dbb 100644
--- a/distributed/dashboard/components/shared.py
+++ b/distributed/dashboard/components/shared.py
@@ -1,5 +1,6 @@
 import asyncio
 import weakref
+from statistics import mean
 
 import tlz as toolz
 from bokeh.layouts import column, row
@@ -8,6 +9,7 @@
     ColumnDataSource,
     DataRange1d,
     HoverTool,
+    LabelSet,
     NumeralTickFormatter,
     Range1d,
     Select,
@@ -455,12 +457,23 @@ def trigger_update(self):
 
 
 class SystemMonitor(DashboardComponent):
-    def __init__(self, worker, height=150, **kwargs):
+    def __init__(self, worker, height=150, last_count=None, **kwargs):
         self.worker = worker
 
         names = worker.monitor.quantities
-        self.last = 0
+        self.last_count = 0
+        if last_count is not None:
+            names = worker.monitor.range_query(start=last_count)
+            self.last_count = last_count
         self.source = ColumnDataSource({name: [] for name in names})
+        self.label_source = ColumnDataSource(
+            {
+                "x": [5] * 3,
+                "y": [70, 55, 40],
+                "cpu": ["max: 45%", "min: 45%", "mean: 45%"],
+                "memory": ["max: 133.5MiB", "min: 23.6MiB", "mean: 115.4MiB"],
+            }
+        )
         update(self.source, self.get_data())
 
         x_range = DataRange1d(follow="end", follow_interval=20000, range_padding=0)
@@ -477,6 +490,18 @@ def __init__(self, worker, height=150, **kwargs):
         )
         self.cpu.line(source=self.source, x="time", y="cpu")
         self.cpu.yaxis.axis_label = "Percentage"
+        self.cpu.add_layout(
+            LabelSet(
+                x="x",
+                y="y",
+                x_units="screen",
+                y_units="screen",
+                text="cpu",
+                text_font_size="1em",
+                render_mode="css",
+                source=self.label_source,
+            )
+        )
         self.mem = figure(
             title="Memory",
             x_axis_type="datetime",
@@ -487,6 +512,18 @@ def __init__(self, worker, height=150, **kwargs):
         )
         self.mem.line(source=self.source, x="time", y="memory")
         self.mem.yaxis.axis_label = "Bytes"
+        self.mem.add_layout(
+            LabelSet(
+                x="x",
+                y="y",
+                x_units="screen",
+                y_units="screen",
+                text="memory",
+                text_font_size="1em",
+                render_mode="css",
+                source=self.label_source,
+            )
+        )
         self.bandwidth = figure(
             title="Bandwidth",
             x_axis_type="datetime",
@@ -533,12 +570,22 @@ def __init__(self, worker, height=150, **kwargs):
         self.worker.monitor.update()
 
     def get_data(self):
-        d = self.worker.monitor.range_query(start=self.last)
+        d = self.worker.monitor.range_query(start=self.last_count)
         d["time"] = [x * 1000 for x in d["time"]]
-        self.last = self.worker.monitor.count
+        self.last_count = self.worker.monitor.count
         return d
 
     @without_property_validation
     def update(self):
         with log_errors():
             self.source.stream(self.get_data(), 1000)
+            self.label_source.data["cpu"] = list(
+                "{}: {:.1f}%".format(f.__name__, f(self.source.data["cpu"]))
+                for f in [min, max, mean]
+            )
+            self.label_source.data["memory"] = list(
+                "{}: {}".format(
+                    f.__name__, dask.utils.format_bytes(f(self.source.data["memory"]))
+                )
+                for f in [min, max, mean]
+            )
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 7cfe8497e24..0603ac0b1c7 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -6606,7 +6606,7 @@ async def get_profile_metadata(
 
         return {"counts": counts, "keys": keys}
 
-    async def performance_report(self, comm=None, start=None, code=""):
+    async def performance_report(self, comm=None, start=None, last_count=None, code=""):
         parent: SchedulerState = cast(SchedulerState, self)
         stop = time()
         # Profiles
@@ -6646,6 +6646,7 @@ def profile_to_figure(state):
         source, task_stream = task_stream_figure(sizing_mode="stretch_both")
         source.data.update(rects)
 
+        # Bandwidth
         from distributed.dashboard.components.scheduler import (
             BandwidthTypes,
             BandwidthWorkers,
@@ -6656,6 +6657,12 @@ def profile_to_figure(state):
         bandwidth_types = BandwidthTypes(self, sizing_mode="stretch_both")
         bandwidth_types.update()
 
+        # System monitor
+        from distributed.dashboard.components.shared import SystemMonitor
+
+        sysmon = SystemMonitor(self, last_count=last_count, sizing_mode="stretch_both")
+        sysmon.update()
+
         from bokeh.models import Div, Panel, Tabs
 
         import distributed
@@ -6713,11 +6720,13 @@ def profile_to_figure(state):
             child=bandwidth_workers.root, title="Bandwidth (Workers)"
         )
         bandwidth_types = Panel(child=bandwidth_types.root, title="Bandwidth (Types)")
+        system = Panel(child=sysmon.root, title="System")
 
         tabs = Tabs(
             tabs=[
                 html,
                 task_stream,
+                system,
                 compute,
                 workers,
                 scheduler,

From 26a1da050636b4e37490b675ba16aa29a484e106 Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Wed, 26 May 2021 21:44:22 +0100
Subject: [PATCH 1285/1550] Prevent accidentally starting multiple Workers in
 the same process (#4852)

---
 distributed/client.py                         |  9 +++-----
 distributed/dashboard/components/scheduler.py |  4 +---
 distributed/deploy/local.py                   | 21 +++++++++++--------
 3 files changed, 16 insertions(+), 18 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index e114d869bc8..d630eb8583d 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -579,7 +579,7 @@ class Client:
 
     Extra keywords will be passed directly to LocalCluster
 
-    >>> client = Client(processes=False, threads_per_worker=1)  # doctest: +SKIP
+    >>> client = Client(n_workers=2, threads_per_worker=4)  # doctest: +SKIP
 
     See Also
     --------
@@ -927,11 +927,8 @@ def _repr_html_(self):
         if info:
             workers = list(info["workers"].values())
             cores = sum(w["nthreads"] for w in workers)
-            if all(isinstance(w["memory_limit"], Number) for w in workers):
-                memory = sum(w["memory_limit"] for w in workers)
-                memory = format_bytes(memory)
-            else:
-                memory = ""
+            memory = [w["memory_limit"] for w in workers]
+            memory = format_bytes(sum(memory)) if all(memory) else ""
 
             text2 = (
                 '<h3 style="text-align: left;">Cluster</h3>\n'
diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index 790cf655efe..28250ff9d10 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -290,9 +290,7 @@ def __init__(self, scheduler, width=600, **kwargs):
     @without_property_validation
     def update(self):
         with log_errors():
-            limit = sum(
-                getattr(ws, "memory_limit", 0) for ws in self.scheduler.workers.values()
-            )
+            limit = sum(ws.memory_limit for ws in self.scheduler.workers.values())
             meminfo = self.scheduler.memory
             color = _nbytes_color(meminfo.process, limit)
 
diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index e1f18734b0e..d68daeaa4b8 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -29,7 +29,8 @@ class LocalCluster(SpecCluster):
     n_workers: int
         Number of workers to start
     processes: bool
-        Whether to use processes (True) or threads (False).  Defaults to True
+        Whether to use processes (True) or threads (False).  Defaults to True, unless
+        worker_class=Worker, in which case it defaults to False.
     threads_per_worker: int
         Number of threads per each worker
     scheduler_port: int
@@ -71,7 +72,8 @@ class LocalCluster(SpecCluster):
     interface: str (optional)
         Network interface to use.  Defaults to lo/localhost
     worker_class: Worker
-        Worker class used to instantiate workers from.
+        Worker class used to instantiate workers from. Defaults to Worker if
+        processes=False and Nanny if processes=True or omitted.
     **worker_kwargs:
         Extra worker arguments. Any additional keyword arguments will be passed
         to the ``Worker`` class constructor.
@@ -98,7 +100,7 @@ def __init__(
         name=None,
         n_workers=None,
         threads_per_worker=None,
-        processes=True,
+        processes=None,
         loop=None,
         start=None,
         host=None,
@@ -146,6 +148,11 @@ def __init__(
                 "and `worker_dashboard_address` for the worker (less common)."
             )
 
+        if processes is None:
+            processes = worker_class is None or issubclass(worker_class, Nanny)
+        if worker_class is None:
+            worker_class = Nanny if processes else Worker
+
         self.status = None
         self.processes = processes
 
@@ -182,7 +189,7 @@ def __init__(
                 n_workers = 1
                 threads_per_worker = CPU_COUNT
         if n_workers is None and threads_per_worker is not None:
-            n_workers = max(1, CPU_COUNT // threads_per_worker)
+            n_workers = max(1, CPU_COUNT // threads_per_worker) if processes else 1
         if n_workers and threads_per_worker is None:
             # Overcommit threads per worker, rather than undercommit
             threads_per_worker = max(1, int(math.ceil(CPU_COUNT / n_workers)))
@@ -221,11 +228,7 @@ def __init__(
             ),
         }
 
-        worker = {
-            "cls": worker_class or (Worker if not processes else Nanny),
-            "options": worker_kwargs,
-        }
-
+        worker = {"cls": worker_class, "options": worker_kwargs}
         workers = {i: worker for i in range(n_workers)}
 
         super().__init__(

From 2e01873444067fc04939fe2ea50f49a40b2eeb1d Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Thu, 27 May 2021 11:14:31 +0100
Subject: [PATCH 1286/1550] Add HTML reprs for Client.who_has and
 Client.has_what (#4853)

---
 distributed/client.py  |  5 ++--
 distributed/objects.py | 67 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 70 insertions(+), 2 deletions(-)
 create mode 100644 distributed/objects.py

diff --git a/distributed/client.py b/distributed/client.py
index d630eb8583d..78dbc91a524 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -53,6 +53,7 @@
 )
 from .diagnostics.plugin import UploadFile, WorkerPlugin, _get_worker_plugin_name
 from .metrics import time
+from .objects import HasWhat, WhoHas
 from .protocol import to_serialize
 from .protocol.pickle import dumps, loads
 from .publish import Datasets
@@ -3202,7 +3203,7 @@ def who_has(self, futures=None, **kwargs):
             keys = list(map(stringify, {f.key for f in futures}))
         else:
             keys = None
-        return self.sync(self.scheduler.who_has, keys=keys, **kwargs)
+        return WhoHas(self.sync(self.scheduler.who_has, keys=keys, **kwargs))
 
     def has_what(self, workers=None, **kwargs):
         """Which keys are held by which workers
@@ -3236,7 +3237,7 @@ def has_what(self, workers=None, **kwargs):
             workers = list(workers)
         if workers is not None and not isinstance(workers, (tuple, list, set)):
             workers = [workers]
-        return self.sync(self.scheduler.has_what, workers=workers, **kwargs)
+        return HasWhat(self.sync(self.scheduler.has_what, workers=workers, **kwargs))
 
     def processing(self, workers=None):
         """The tasks currently running on each worker
diff --git a/distributed/objects.py b/distributed/objects.py
new file mode 100644
index 00000000000..d53860514ec
--- /dev/null
+++ b/distributed/objects.py
@@ -0,0 +1,67 @@
+"""This file contains custom objects.
+These are mostly regular objects with more useful _repr_ and _repr_html_ methods."""
+
+
+class HasWhat(dict):
+    """A dictionary of all workers and which keys that worker has."""
+
+    def _repr_html_(self):
+        rows = ""
+
+        for worker, keys in sorted(self.items()):
+            summary = ""
+            for key in keys:
+                summary += f"""<tr><td>{key}</td></tr>"""
+
+            rows += f"""<tr>
+            <td>{worker}</td>
+            <td>{len(keys)}</td>
+            <td>
+                <details>
+                <summary style='display:list-item'>Expand</summary>
+                <table>
+                {summary}
+                </table>
+                </details>
+            </td>
+        </tr>"""
+
+        output = f"""
+        <table>
+        <tr>
+            <th>Worker</th>
+            <th>Key count</th>
+            <th>Key list</th>
+        </tr>
+        {rows}
+        </table>
+        """
+
+        return output
+
+
+class WhoHas(dict):
+    """A dictionary of all keys and which workers have that key."""
+
+    def _repr_html_(self):
+        rows = ""
+
+        for title, keys in sorted(self.items()):
+            rows += f"""<tr>
+            <td>{title}</td>
+            <td>{len(keys)}</td>
+            <td>{", ".join(keys)}</td>
+        </tr>"""
+
+        output = f"""
+        <table>
+        <tr>
+            <th>Key</th>
+            <th>Copies</th>
+            <th>Workers</th>
+        </tr>
+        {rows}
+        </table>
+        """
+
+        return output

From 4f83686b9ffed4e8c41bbf002c1bab24b38b9dab Mon Sep 17 00:00:00 2001
From: Benjamin Zaitlen <quasiben@users.noreply.github.com>
Date: Thu, 27 May 2021 19:21:48 -0400
Subject: [PATCH 1287/1550] Fix WhoHas/HasWhat async usage (#4860)

* move object WhoHas/HasWhat to scheduler

* remove whohas from scheduler -- bail on fancy repr for async

* lint

* Update distributed/client.py

Co-authored-by: James Bourbeau <jrbourbeau@users.noreply.github.com>

* code clean up and cleaner tests

* use Whohas/HasWht in get_whohas methods

* revert objects usage in scheduler and add assertive test for objects when using sync client

Co-authored-by: James Bourbeau <jrbourbeau@users.noreply.github.com>
---
 distributed/client.py            | 14 ++++++++++++--
 distributed/tests/test_client.py | 21 +++++++++++++++++++++
 2 files changed, 33 insertions(+), 2 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 78dbc91a524..ee3e6d47935 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -3203,7 +3203,12 @@ def who_has(self, futures=None, **kwargs):
             keys = list(map(stringify, {f.key for f in futures}))
         else:
             keys = None
-        return WhoHas(self.sync(self.scheduler.who_has, keys=keys, **kwargs))
+
+        result = self.sync(self.scheduler.who_has, keys=keys, **kwargs)
+        if self.asynchronous:
+            return result
+
+        return WhoHas(result)
 
     def has_what(self, workers=None, **kwargs):
         """Which keys are held by which workers
@@ -3237,7 +3242,12 @@ def has_what(self, workers=None, **kwargs):
             workers = list(workers)
         if workers is not None and not isinstance(workers, (tuple, list, set)):
             workers = [workers]
-        return HasWhat(self.sync(self.scheduler.has_what, workers=workers, **kwargs))
+        result = self.sync(self.scheduler.has_what, workers=workers, **kwargs)
+
+        if self.asynchronous:
+            return result
+
+        return HasWhat(result)
 
     def processing(self, workers=None):
         """The tasks currently running on each worker
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 6fbbae95700..91232ac8699 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -58,6 +58,7 @@
 from distributed.compatibility import MACOS, WINDOWS
 from distributed.core import Status
 from distributed.metrics import time
+from distributed.objects import HasWhat, WhoHas
 from distributed.scheduler import (
     COMPILED,
     CollectTaskMetaDataPlugin,
@@ -3613,6 +3614,26 @@ async def test_status():
     await s.close()
 
 
+@gen_cluster(client=True)
+async def test_async_whowhat(c, s, a, b):
+    [x] = await c.scatter([1], workers=a.address)
+
+    who_has = await c.who_has()
+    has_what = await c.has_what()
+
+    assert who_has == {x.key: (a.address,)}
+    assert has_what == {a.address: (x.key,), b.address: ()}
+
+
+def test_client_repr(c):
+    x = c.submit(inc, 1)
+
+    who_has = c.who_has()
+    has_what = c.has_what()
+    assert type(who_has) is WhoHas
+    assert type(has_what) is HasWhat
+
+
 @gen_cluster(client=True)
 async def test_persist_optimize_graph(c, s, a, b):
     i = 10

From 93e286987431381d995c93bbacea1ccb07da576b Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Fri, 28 May 2021 09:50:47 +0200
Subject: [PATCH 1288/1550] Ensure adaptive scaling is properly awaited and
 closed (#4720)

* Ensure adaptive scaling is properly awaited and closed

* review comments

* Ensure no tasks are pending when closing adaptive cluster

* remvoe assert in stop

* break cyclic ref in adaptive core
---
 distributed/client.py                         |  2 +-
 distributed/deploy/adaptive.py                | 10 ++-
 distributed/deploy/adaptive_core.py           | 27 +++++-
 distributed/deploy/cluster.py                 |  8 +-
 distributed/deploy/spec.py                    | 11 ++-
 distributed/deploy/tests/test_adaptive.py     | 84 +++++++++++--------
 .../deploy/tests/test_adaptive_core.py        | 48 ++++++++++-
 7 files changed, 144 insertions(+), 46 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index ee3e6d47935..f86cc72314a 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -1196,7 +1196,7 @@ def __enter__(self):
         return self
 
     async def __aenter__(self):
-        await self._started
+        await self
         return self
 
     async def __aexit__(self, typ, value, traceback):
diff --git a/distributed/deploy/adaptive.py b/distributed/deploy/adaptive.py
index ff73015a40a..fa68ad4e933 100644
--- a/distributed/deploy/adaptive.py
+++ b/distributed/deploy/adaptive.py
@@ -184,19 +184,23 @@ async def scale_down(self, workers):
         if not workers:
             return
         with log_errors():
+            logger.info("Retiring workers %s", workers)
             # Ask scheduler to cleanly retire workers
             await self.scheduler.retire_workers(
-                names=workers, remove=True, close_workers=True
+                names=workers,
+                remove=True,
+                close_workers=True,
             )
 
             # close workers more forcefully
-            logger.info("Retiring workers %s", workers)
             f = self.cluster.scale_down(workers)
             if isawaitable(f):
                 await f
 
     async def scale_up(self, n):
-        self.cluster.scale(n)
+        f = self.cluster.scale(n)
+        if isawaitable(f):
+            await f
 
     @property
     def loop(self):
diff --git a/distributed/deploy/adaptive_core.py b/distributed/deploy/adaptive_core.py
index 4d4d2810c41..e11a07bf528 100644
--- a/distributed/deploy/adaptive_core.py
+++ b/distributed/deploy/adaptive_core.py
@@ -1,6 +1,7 @@
 import collections
 import logging
 import math
+from typing import Iterable
 
 import tlz as toolz
 from tornado.ioloop import IOLoop, PeriodicCallback
@@ -87,7 +88,16 @@ def f():
                 pass
 
         if self.interval:
-            self.periodic_callback = PeriodicCallback(self.adapt, self.interval * 1000)
+            import weakref
+
+            self_ref = weakref.ref(self)
+
+            async def _adapt():
+                core = self_ref()
+                if core:
+                    await core.adapt()
+
+            self.periodic_callback = PeriodicCallback(_adapt, self.interval * 1000)
             try:
                 self.loop.add_callback(f)
             except AttributeError:
@@ -134,6 +144,12 @@ async def safe_target(self) -> int:
 
         return n
 
+    async def scale_down(self, n: int):
+        raise NotImplementedError()
+
+    async def scale_up(self, workers: Iterable):
+        raise NotImplementedError()
+
     async def recommendations(self, target: int) -> dict:
         """
         Make scale up/down recommendations based on current state and target
@@ -203,7 +219,14 @@ async def adapt(self) -> None:
                 await self.scale_down(**recommendations)
         except OSError as e:
             if status != "down":
-                logger.error("Adaptive stopping due to error %s", str(e))
+                logger.error("Adaptive stopping due to error", exc_info=True)
                 self.stop()
+            else:
+                logger.error(
+                    "Error during adaptive downscaling. Ignoring.", exc_info=True
+                )
         finally:
             self._adapting = False
+
+    def __del__(self):
+        self.stop()
diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
index 00360c7aa22..ed5506c2f8e 100644
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -5,6 +5,7 @@
 import uuid
 import warnings
 from contextlib import suppress
+from inspect import isawaitable
 
 from tornado.ioloop import PeriodicCallback
 
@@ -82,6 +83,9 @@ async def _close(self):
         if self.status == Status.closed:
             return
 
+        with suppress(AttributeError):
+            self._adaptive.stop()
+
         if self._watch_worker_status_comm:
             await self._watch_worker_status_comm.close()
         if self._watch_worker_status_task:
@@ -419,7 +423,9 @@ async def __aenter__(self):
         return self
 
     async def __aexit__(self, typ, value, traceback):
-        await self.close()
+        f = self.close()
+        if isawaitable(f):
+            await f
 
     @property
     def scheduler_address(self):
diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index ea5e8b092a6..8ea233beab7 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -6,6 +6,7 @@
 import warnings
 import weakref
 from contextlib import suppress
+from inspect import isawaitable
 
 from tornado import gen
 
@@ -413,7 +414,15 @@ async def _close(self):
             return
         if self.status == Status.running or self.status == Status.failed:
             self.status = Status.closing
-            self.scale(0)
+
+            # Need to call stop here before we close all servers to avoid having
+            # dangling tasks in the ioloop
+            with suppress(AttributeError):
+                self._adaptive.stop()
+
+            f = self.scale(0)
+            if isawaitable(f):
+                await f
             await self._correct_state()
             for future in self._futures:
                 await future
diff --git a/distributed/deploy/tests/test_adaptive.py b/distributed/deploy/tests/test_adaptive.py
index 35a9c14cc54..434eeea8cf8 100644
--- a/distributed/deploy/tests/test_adaptive.py
+++ b/distributed/deploy/tests/test_adaptive.py
@@ -10,6 +10,7 @@
 from distributed import Adaptive, Client, LocalCluster, SpecCluster, Worker, wait
 from distributed.metrics import time
 from distributed.utils_test import (  # noqa: F401
+    async_wait_for,
     clean,
     cleanup,
     gen_test,
@@ -19,43 +20,6 @@
 )
 
 
-@pytest.mark.asyncio
-async def test_simultaneous_scale_up_and_down(cleanup):
-    class TestAdaptive(Adaptive):
-        def get_scale_up_kwargs(self):
-            assert False
-
-        def _retire_workers(self):
-            assert False
-
-    class TestCluster(LocalCluster):
-        def scale_up(self, n, **kwargs):
-            assert False
-
-        def scale_down(self, workers):
-            assert False
-
-    with dask.config.set(
-        {"distributed.scheduler.default-task-durations": {"a": 4, "b": 4, "c": 1}}
-    ):
-        async with TestCluster(
-            n_workers=4, processes=False, asynchronous=True
-        ) as cluster:
-            async with Client(cluster, asynchronous=True) as c:
-                s = cluster.scheduler
-
-                future = c.map(slowinc, [1, 1, 1], key=["a-4", "b-4", "c-1"])
-
-                while len(s.rprocessing) < 3:
-                    await asyncio.sleep(0.001)
-
-                ta = cluster.adapt(
-                    interval="100 ms", scale_factor=2, Adaptive=TestAdaptive
-                )
-
-                await asyncio.sleep(0.3)
-
-
 def test_adaptive_local_cluster(loop):
     with LocalCluster(
         n_workers=0,
@@ -479,3 +443,49 @@ async def test_adaptive_no_memory_limit(cleanup):
             )
             <= 5
         )
+
+
+@pytest.mark.asyncio
+async def test_scale_needs_to_be_awaited(cleanup):
+    """
+    This tests that the adaptive class works fine if the scale method uses the
+    `sync` method to schedule its task instead of loop.add_callback
+    """
+
+    class RequiresAwaitCluster(LocalCluster):
+        def scale(self, n):
+            # super invocation in the nested function scope is messy
+            method = super().scale
+
+            async def _():
+                return method(n)
+
+            return self.sync(_)
+
+    async with RequiresAwaitCluster(n_workers=0, asynchronous=True) as cluster:
+        async with Client(cluster, asynchronous=True) as client:
+            futures = client.map(slowinc, range(5), delay=0.05)
+            assert len(cluster.workers) == 0
+            cluster.adapt()
+
+            await client.gather(futures)
+
+            del futures
+            await async_wait_for(lambda: not cluster.workers, 10)
+
+
+@pytest.mark.asyncio
+async def test_adaptive_stopped():
+    """
+    We should ensure that the adapt PC is actually stopped once the cluster
+    stops.
+    """
+    async with LocalCluster(n_workers=0, asynchronous=True) as cluster:
+        instance = cluster.adapt(interval="10ms")
+        assert instance.periodic_callback is not None
+
+        await async_wait_for(lambda: instance.periodic_callback.is_running(), timeout=5)
+
+        pc = instance.periodic_callback
+
+    await async_wait_for(lambda: not pc.is_running(), timeout=5)
diff --git a/distributed/deploy/tests/test_adaptive_core.py b/distributed/deploy/tests/test_adaptive_core.py
index c749d4670b7..2750121e445 100644
--- a/distributed/deploy/tests/test_adaptive_core.py
+++ b/distributed/deploy/tests/test_adaptive_core.py
@@ -93,7 +93,7 @@ async def test_interval():
 
 
 @pytest.mark.asyncio
-async def test_adapt_oserror():
+async def test_adapt_oserror_safe_target():
     class BadAdaptive(MyAdaptive):
         """AdaptiveCore subclass which raises an OSError when attempting to adapt
 
@@ -111,3 +111,49 @@ def safe_target(self):
     assert "Adaptive stop" in text
     assert not adapt._adapting
     assert not adapt.periodic_callback
+
+
+@pytest.mark.asyncio
+async def test_adapt_oserror_scale():
+    """
+    FIXME:
+    If we encounter an OSError during scale down, we continue as before. It is
+    not entirely clear if this is the correct behaviour but defines the current
+    state.
+    This was probably introduced to protect against comm failures during
+    shutdown but the scale down command should be robust call to the scheduler
+    which is never scaled down.
+    """
+
+    class BadAdaptive(MyAdaptive):
+        async def scale_down(self, workers=None):
+            raise OSError()
+
+    adapt = BadAdaptive(minimum=1, maximum=4, wait_count=0, interval="10ms")
+    adapt._target = 2
+    while not adapt.periodic_callback.is_running():
+        await asyncio.sleep(0.01)
+    await adapt.adapt()
+    assert len(adapt.plan) == 2
+    assert len(adapt.requested) == 2
+    with captured_logger("distributed.deploy.adaptive_core") as log:
+        adapt._target = 0
+        await adapt.adapt()
+    text = log.getvalue()
+    assert "Error during adaptive downscaling" in text
+    assert not adapt._adapting
+    assert adapt.periodic_callback
+    assert adapt.periodic_callback.is_running()
+    adapt.stop()
+
+
+@pytest.mark.asyncio
+async def test_adapt_stop_del():
+    adapt = MyAdaptive(interval="100ms")
+    pc = adapt.periodic_callback
+    while not adapt.periodic_callback.is_running():
+        await asyncio.sleep(0.01)
+
+    del adapt
+    while pc.is_running():
+        await asyncio.sleep(0.01)

From 0eeee27444d902abd30b3e808a87ee500846e353 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Fri, 28 May 2021 13:00:45 -0700
Subject: [PATCH 1289/1550] Drop usage of `WhoHas` & `WhatHas` from `Client`
 (#4863)

* Drop usage of `WhoHas` & `WhatHas` from `Client`

* Mark `test_client_repr` as `xfail`

* Use `reason=` keyword
---
 distributed/client.py            | 15 ++-------------
 distributed/tests/test_client.py |  1 +
 2 files changed, 3 insertions(+), 13 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index f86cc72314a..b1aa94032c4 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -53,7 +53,6 @@
 )
 from .diagnostics.plugin import UploadFile, WorkerPlugin, _get_worker_plugin_name
 from .metrics import time
-from .objects import HasWhat, WhoHas
 from .protocol import to_serialize
 from .protocol.pickle import dumps, loads
 from .publish import Datasets
@@ -3203,12 +3202,7 @@ def who_has(self, futures=None, **kwargs):
             keys = list(map(stringify, {f.key for f in futures}))
         else:
             keys = None
-
-        result = self.sync(self.scheduler.who_has, keys=keys, **kwargs)
-        if self.asynchronous:
-            return result
-
-        return WhoHas(result)
+        return self.sync(self.scheduler.who_has, keys=keys, **kwargs)
 
     def has_what(self, workers=None, **kwargs):
         """Which keys are held by which workers
@@ -3242,12 +3236,7 @@ def has_what(self, workers=None, **kwargs):
             workers = list(workers)
         if workers is not None and not isinstance(workers, (tuple, list, set)):
             workers = [workers]
-        result = self.sync(self.scheduler.has_what, workers=workers, **kwargs)
-
-        if self.asynchronous:
-            return result
-
-        return HasWhat(result)
+        return self.sync(self.scheduler.has_what, workers=workers, **kwargs)
 
     def processing(self, workers=None):
         """The tasks currently running on each worker
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 91232ac8699..24359fc86fa 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -3625,6 +3625,7 @@ async def test_async_whowhat(c, s, a, b):
     assert has_what == {a.address: (x.key,), b.address: ()}
 
 
+@pytest.mark.xfail(reason="Want to fix to use `WhoHas` + `WhatHas`")
 def test_client_repr(c):
     x = c.submit(inc, 1)
 

From 6a7d8dd9beab7b37e6173f2eaeec5fcb10da469c Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Fri, 28 May 2021 15:37:42 -0500
Subject: [PATCH 1290/1550] bump version to 2021.05.1

---
 docs/source/changelog.rst | 30 +++++++++++++++++++++++++++++-
 requirements.txt          |  2 +-
 2 files changed, 30 insertions(+), 2 deletions(-)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index 97174ef9210..e1d4d91e323 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,33 @@
 Changelog
 =========
 
+2021.05.1
+---------
+
+Released on May 28, 2021
+
+- Drop usage of ``WhoHas`` & ``WhatHas`` from ``Client`` (:pr:`4863`) `jakirkham`_
+- Ensure adaptive scaling is properly awaited and closed (:pr:`4720`) `Florian Jetter`_
+- Fix ``WhoHas``/ ``HasWhat`` ``async`` usage (:pr:`4860`) `Benjamin Zaitlen`_
+- Add HTML reprs for ``Client.who_has`` and ``Client.has_what`` (:pr:`4853`) `Jacob Tomlinson`_
+- Prevent accidentally starting multiple ``Worker`` s in the same process (:pr:`4852`) `crusaderky`_
+- Add system tab to performance reports (:pr:`4561`) `Charles Blackmon-Luca`_
+- Let servers close faster if there are no active handlers (:pr:`4805`) `Florian Jetter`_
+- Fix UCX scrub config logging (:pr:`4850`) `Peter Andreas Entschev`_
+- Ensure worker clients are closed (:pr:`3921`) `Florian Jetter`_
+- Fix warning for attribute error when deleting a client (:pr:`4807`) `Florian Jetter`_
+- Ensure exceptions are raised if workers are incorrectly started (:pr:`4733`) `Florian Jetter`_
+- Update handling of UCX exceptions on endpoint closing (:pr:`4836`) `Peter Andreas Entschev`_
+- Ensure busy workloads properly look up ``who_has`` (:pr:`4793`) `Florian Jetter`_
+- Check ``distributed.scheduler.pickle`` in ``Scheduler.run_function`` (:pr:`4838`) `James Bourbeau`_
+- Add performance_report to API docs (:pr:`4840`) `James Bourbeau`_
+- Use ``dict`` ``_workers_dv`` in unordered use cases (:pr:`4826`) `jakirkham`_
+- Bump ``pre-commit`` hook versions (:pr:`4835`) `James Bourbeau`_
+- Do not mindlessly spawn workers when no memory limit is set (:pr:`4397`) `Torsten Wörtwein`_
+- ``test_memory`` to use ``gen_cluster`` (:pr:`4811`) `crusaderky`_
+- Increase timeout of ``gen_test`` to 30s (:pr:`4821`) `Florian Jetter`_
+
+
 2021.05.0
 ---------
 
@@ -2362,4 +2389,5 @@ significantly without many new features.
 .. _`Hristo Georgiev`: https://github.com/hristog
 .. _`Matteo De Wint`: https://github.com/mdwint
 .. _`Naty Clementi`: https://github.com/ncclementi
-.. _`Nathan Danielsen`: https://github.com/ndanielsen
\ No newline at end of file
+.. _`Nathan Danielsen`: https://github.com/ndanielsen
+.. _`Torsten Wörtwein`: https://github.com/twoertwein
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index 19032cf740b..986230439bb 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,6 @@
 click >= 6.6
 cloudpickle >= 1.5.0
-dask == 2021.05.0
+dask == 2021.05.1
 msgpack >= 0.6.0
 psutil >= 5.0
 sortedcontainers !=2.0.0, !=2.0.1

From 95a0689edd603c8401bdb5b242a21d57e739e1d9 Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Mon, 31 May 2021 09:52:17 +0200
Subject: [PATCH 1291/1550] Ensure repr and eq for cluster always works (#4799)

* Ensure repr and eq for cluster always works

* Allow localclusters to be spawned in function

* Add context manager to check for dangling tasks

* Improve test_get_client_functions_spawn_clusters

* verify dangling tasks

* Remove check_dangling_tasks

We do have too many dangling tasks due to tornado add_callback
scheduling we do not cleanup or await properly

* Add check about closed worker clients to check_instances
---
 distributed/deploy/cluster.py            |  6 +++--
 distributed/deploy/tests/test_cluster.py | 28 +++++++++++++++++++++++
 distributed/tests/test_client.py         | 29 ++++++++++++++++++++++++
 distributed/tests/test_worker.py         |  3 ++-
 distributed/utils_test.py                |  7 ++++++
 distributed/worker.py                    | 13 ++++++++++-
 6 files changed, 82 insertions(+), 4 deletions(-)
 create mode 100644 distributed/deploy/tests/test_cluster.py

diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
index ed5506c2f8e..d71f68f0a90 100644
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -428,7 +428,9 @@ async def __aexit__(self, typ, value, traceback):
             await f
 
     @property
-    def scheduler_address(self):
+    def scheduler_address(self) -> str:
+        if not self.scheduler_comm:
+            return "<Not Connected>"
         return self.scheduler_comm.address
 
     @property
@@ -464,7 +466,7 @@ def observed(self):
         return {d["name"] for d in self.scheduler_info["workers"].values()}
 
     def __eq__(self, other):
-        return self.name == other.name
+        return type(other) == type(self) and self.name == other.name
 
     def __hash__(self):
         return id(self)
diff --git a/distributed/deploy/tests/test_cluster.py b/distributed/deploy/tests/test_cluster.py
new file mode 100644
index 00000000000..eee8a4139ac
--- /dev/null
+++ b/distributed/deploy/tests/test_cluster.py
@@ -0,0 +1,28 @@
+import pytest
+
+from distributed.deploy.cluster import Cluster
+from distributed.utils_test import cleanup  # noqa: F401
+
+
+@pytest.mark.asyncio
+async def test_eq(cleanup):
+    clusterA = Cluster(asynchronous=True, name="A")
+    clusterA2 = Cluster(asynchronous=True, name="A2")
+    clusterB = Cluster(asynchronous=True, name="B")
+    assert clusterA != "A"
+    assert not (clusterA == "A")
+    assert clusterA == clusterA
+    assert not (clusterA != clusterA)
+    assert clusterA != clusterA2
+    assert not (clusterA == clusterA2)
+    assert clusterA != clusterB
+    assert not (clusterA == clusterB)
+
+
+@pytest.mark.asyncio
+async def test_repr(cleanup):
+    cluster = Cluster(asynchronous=True, name="A")
+    assert cluster.scheduler_address == "<Not Connected>"
+    res = repr(cluster)
+    expected = "Cluster(A, '<Not Connected>', workers=0, threads=0, memory=0 B)"
+    assert res == expected
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 24359fc86fa..f17de5949be 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -6624,3 +6624,32 @@ async def test_workers_collection_restriction(c, s, a, b):
     future = c.compute(da.arange(10), workers=a.address)
     await future
     assert a.data and not b.data
+
+
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 0)])
+async def test_get_client_functions_spawn_clusters(c, s, a):
+    # see gh4565
+
+    scheduler_addr = c.scheduler.address
+
+    def f(x):
+        ref = None
+        with LocalCluster(
+            n_workers=1,
+            processes=False,
+            dashboard_address=False,
+            worker_dashboard_address=False,
+        ) as cluster2:
+            with Client(cluster2) as c1:
+                c2 = get_client()
+
+                c1_scheduler = c1.scheduler.address
+                c2_scheduler = c2.scheduler.address
+                assert c1_scheduler != c2_scheduler
+                assert c2_scheduler == scheduler_addr
+
+    await c.gather(c.map(f, range(2)))
+    await a.close()
+
+    c_default = default_client()
+    assert c is c_default
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 7a9feffc1c7..f50c73e990a 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -1905,6 +1905,7 @@ async def test_worker_client_uses_default_no_close(c, s, a):
     If a default client is available in the process, the worker will pick this
     one and will not close it if it is closed
     """
+    assert not Worker._initialized_clients
     assert default_client() is c
     existing_client = c.id
 
@@ -1957,7 +1958,7 @@ def get_worker_client_id():
             default_client()
 
 
-@gen_cluster(timeout=3600)
+@gen_cluster()
 async def test_worker_client_closes_if_created_on_worker_last_worker_alive(s, a, b):
     async with Client(s.address, set_as_default=False, asynchronous=True) as c:
 
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index e3f18e5de49..aba7294a71d 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -1479,6 +1479,7 @@ def check_instances():
     Worker._instances.clear()
     Scheduler._instances.clear()
     SpecCluster._instances.clear()
+    Worker._initialized_clients.clear()
     # assert all(n.status == "closed" for n in Nanny._instances), {
     #     n: n.status for n in Nanny._instances
     # }
@@ -1502,6 +1503,12 @@ def check_instances():
                 w.loop.add_callback(w.close)
     Worker._instances.clear()
 
+    start = time()
+    while any(c.status != "closed" for c in Worker._initialized_clients):
+        sleep(0.1)
+        assert time() < start + 10
+    Worker._initialized_clients.clear()
+
     for i in range(5):
         if all(c.closed() for c in Comm._instances):
             break
diff --git a/distributed/worker.py b/distributed/worker.py
index e7746b43af8..9864ec87743 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -3259,10 +3259,21 @@ def _get_client(self, timeout=None):
         except ValueError:  # no clients found, need to make a new one
             pass
         else:
+            # must be lazy import otherwise cyclic import
+            from distributed.deploy.cluster import Cluster
+
             if (
                 client.scheduler
                 and client.scheduler.address == self.scheduler.address
-                or client._start_arg == self.scheduler.address
+                # The below conditions should only happen in case a second
+                # cluster is alive, e.g. if a submitted task spawned its onwn
+                # LocalCluster, see gh4565
+                or (
+                    isinstance(client._start_arg, str)
+                    and client._start_arg == self.scheduler.address
+                    or isinstance(client._start_arg, Cluster)
+                    and client._start_arg.scheduler_address == self.scheduler.address
+                )
             ):
                 self._client = client
 

From 9d4f0bf2fc804f955a869febd3b51423c4382908 Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Tue, 1 Jun 2021 15:18:04 +0100
Subject: [PATCH 1292/1550] O(1) rebalance (#4774)

* partial prototype

* incomplete poc

* poc (incomplete)

* complete POC

* polish

* polish

* bugfix

* fixes

* fix

* Use arbitrary measure in rebalance

* Code review

* renames

* suggest tweaking malloc_trim

* self-review

* test_tls_functional

* test_memory to use gen_cluster

* test_memory to use gen_cluster

* half memory

* tests

* tests

* tests

* tests

* make Cython happy

* test_rebalance_managed_memory

* tests

* robustness

* improve test stability

* tests stability

* trivial

* reload dask.config on Scheduler.__init__

* code review
---
 distributed/client.py                    |   9 +-
 distributed/distributed-schema.yaml      |  42 +-
 distributed/distributed.yaml             |  38 +-
 distributed/scheduler.py                 | 546 ++++++++++++++++-------
 distributed/tests/test_client.py         | 162 ++++---
 distributed/tests/test_scheduler.py      | 287 +++++++++++-
 distributed/tests/test_tls_functional.py |  40 +-
 docs/source/memory.rst                   |   3 +-
 8 files changed, 893 insertions(+), 234 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index b1aa94032c4..b5771266937 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -3057,11 +3057,14 @@ def upload_file(self, filename, **kwargs):
         )
 
     async def _rebalance(self, futures=None, workers=None):
-        await _wait(futures)
-        keys = list({stringify(f.key) for f in self.futures_of(futures)})
+        if futures is not None:
+            await _wait(futures)
+            keys = list({stringify(f.key) for f in self.futures_of(futures)})
+        else:
+            keys = None
         result = await self.scheduler.rebalance(keys=keys, workers=workers)
         if result["status"] == "missing-data":
-            raise ValueError(
+            raise KeyError(
                 f"During rebalance {len(result['keys'])} keys were found to be missing"
             )
         assert result["status"] == "OK"
diff --git a/distributed/distributed-schema.yaml b/distributed/distributed-schema.yaml
index eac459d7aa7..86c00a8f318 100644
--- a/distributed/distributed-schema.yaml
+++ b/distributed/distributed-schema.yaml
@@ -395,7 +395,7 @@ properties:
             description: >-
               Settings for memory management
             properties:
-              recent_to_old_time:
+              recent-to-old-time:
                 type: string
                 description: >-
                   When there is an increase in process memory (as observed by the
@@ -403,6 +403,46 @@ properties:
                   the worker, ignore it for this long before considering it in
                   non-time-sensitive heuristics. This should be set to be longer than
                   the duration of most dask tasks.
+              rebalance:
+                type: object
+                description: >-
+                  Settings for memory rebalance operations
+                properties:
+                  measure:
+                    enum:
+                      - process
+                      - optimistic
+                      - managed
+                      - managed_in_memory
+                    description: >-
+                      Which of the properties of distributed.scheduler.MemoryState
+                      should be used for measuring worker memory usage
+                  sender-min:
+                    type: number
+                    minimum: 0
+                    maximum: 1
+                    description: >-
+                      Fraction of worker process memory at which we start potentially
+                      transferring data to other workers.
+                  recipient-max:
+                    type: number
+                    minimum: 0
+                    maximum: 1
+                    description: >-
+                      Fraction of worker process memory at which we stop potentially
+                      receiving data from other workers. Ignored when max_memory is not
+                      set.
+                  sender-recipient-gap:
+                    type: number
+                    minimum: 0
+                    maximum: 1
+                    description: >-
+                      Fraction of worker process memory, around the cluster mean, where
+                      a worker is neither a sender nor a recipient of data during a
+                      rebalance operation. E.g. if the mean cluster occupation is 50%,
+                      sender-recipient-gap=0.1 means that only nodes above 55% will
+                      donate data and only nodes below 45% will receive them. This helps
+                      avoid data from bouncing around the cluster repeatedly.
 
               target:
                 oneOf:
diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index ff34e48bc32..a55700cd085 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -88,7 +88,43 @@ distributed:
       # system) that is not accounted for by the dask keys stored on the worker, ignore
       # it for this long before considering it in non-critical memory measures.
       # This should be set to be longer than the duration of most dask tasks.
-      recent_to_old_time: 30s
+      recent-to-old-time: 30s
+
+      rebalance:
+        # Memory measure to rebalance upon. Possible choices are:
+        # process
+        #     Total process memory, as measured by the OS.
+        # optimistic
+        #     Managed by dask (instantaneous) + unmanaged (without any increases
+        #     happened in the last <distributed.worker.memory.recent-to-old-time>).
+        #     Recommended for use on CPython with large (2MiB+) numpy-based data chunks.
+        # managed_in_memory
+        #     Only consider the data allocated by dask in RAM. Recommended if RAM is not
+        #     released in a timely fashion back to the OS after the Python objects are
+        #     dereferenced, but remains available for reuse by PyMalloc.
+        #
+        #     If this is your problem on Linux, you should alternatively consider
+        #     setting the MALLOC_TRIM_THRESHOLD_ environment variable (note the final
+        #     underscore) to a low value; refer to the mallopt man page and to the
+        #     comments about M_TRIM_THRESHOLD on
+        #     https://sourceware.org/git/?p=glibc.git;a=blob;f=malloc/malloc.c
+        # managed
+        #     Only consider data allocated by dask, including that spilled to disk.
+        #     Recommended if disk occupation of the spill file is an issue.
+        measure: optimistic
+        # Fraction of worker process memory at which we start potentially sending
+        # data to other workers. Ignored when max_memory is not set.
+        sender-min: 0.30
+        # Fraction of worker process memory at which we stop potentially accepting
+        # data from other workers. Ignored when max_memory is not set.
+        recipient-max: 0.60
+        # Fraction of worker process memory, around the cluster mean, where a worker is
+        # neither a sender nor a recipient of data during a rebalance operation. E.g. if
+        # the mean cluster occupation is 50%, sender-recipient-gap=0.10 means that only
+        # nodes above 55% will donate data and only nodes below 45% will receive them.
+        # This helps avoid data from bouncing around the cluster repeatedly.
+        # Ignored when max_memory is not set.
+        sender-recipient-gap: 0.10
 
       # Fractions of worker process memory at which we take action to avoid memory
       # blowup. Set any of the values to False to turn off the behavior entirely.
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 0603ac0b1c7..de3843c17f4 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1,4 +1,5 @@
 import asyncio
+import heapq
 import html
 import inspect
 import itertools
@@ -12,11 +13,12 @@
 import warnings
 import weakref
 from collections import defaultdict, deque
-from collections.abc import Mapping, Set
+from collections.abc import Hashable, Iterable, Iterator, Mapping, Set
 from contextlib import suppress
 from datetime import timedelta
 from functools import partial
 from numbers import Number
+from typing import Optional
 
 import psutil
 import sortedcontainers
@@ -161,14 +163,6 @@ def nogil(func):
 DEFAULT_DATA_SIZE = declare(
     Py_ssize_t, parse_bytes(dask.config.get("distributed.scheduler.default-data-size"))
 )
-UNKNOWN_TASK_DURATION = declare(
-    double,
-    parse_timedelta(dask.config.get("distributed.scheduler.unknown-task-duration")),
-)
-MEMORY_RECENT_TO_OLD_TIME = declare(
-    double,
-    parse_timedelta(dask.config.get("distributed.worker.memory.recent_to_old_time")),
-)
 
 DEFAULT_EXTENSIONS = [
     LockExtension,
@@ -292,7 +286,7 @@ class MemoryState:
 
     unmanaged_old
         Minimum of the 'unmanaged' measures over the last
-        ``distributed.memory.recent_to_old_time`` seconds
+        ``distributed.memory.recent-to-old-time`` seconds
     unmanaged_recent
         unmanaged - unmanaged_old; in other words process memory that has been recently
         allocated but is not accounted for by dask; hopefully it's mostly a temporary
@@ -419,7 +413,7 @@ class WorkerState:
 
     .. attribute:: has_what: {TaskState}
 
-       The set of tasks which currently reside on this worker.
+       An insertion-sorted set-like of tasks which currently reside on this worker.
        All the tasks here are in the "memory" state.
 
        This is the reverse mapping of :class:`TaskState.who_has`.
@@ -479,7 +473,9 @@ class WorkerState:
     _bandwidth: double
     _executing: dict
     _extra: dict
-    _has_what: set
+    # _has_what is a dict with all values set to None as rebalance() relies on the
+    # property of Python >=3.7 dicts to be insertion-sorted.
+    _has_what: dict
     _hash: Py_hash_t
     _last_seen: double
     _local_directory: str
@@ -567,7 +563,7 @@ def __init__(
         )
 
         self._actors = set()
-        self._has_what = set()
+        self._has_what = {}
         self._processing = {}
         self._executing = {}
         self._resources = {}
@@ -608,8 +604,8 @@ def extra(self):
         return self._extra
 
     @property
-    def has_what(self):
-        return self._has_what
+    def has_what(self) -> "Set[TaskState]":
+        return self._has_what.keys()
 
     @property
     def host(self):
@@ -1780,6 +1776,14 @@ class SchedulerState:
     _workers: object
     _workers_dv: dict
 
+    # Variables from dask.config, cached by __init__ for performance
+    UNKNOWN_TASK_DURATION: double
+    MEMORY_RECENT_TO_OLD_TIME: double
+    MEMORY_REBALANCE_MEASURE: str
+    MEMORY_REBALANCE_SENDER_MIN: double
+    MEMORY_REBALANCE_RECIPIENT_MAX: double
+    MEMORY_REBALANCE_HALF_GAP: double
+
     def __init__(
         self,
         aliases: dict = None,
@@ -1854,6 +1858,28 @@ def __init__(
         else:
             self._workers = sortedcontainers.SortedDict()
         self._workers_dv: dict = cast(dict, self._workers)
+
+        # Variables from dask.config, cached by __init__ for performance
+        self.UNKNOWN_TASK_DURATION = parse_timedelta(
+            dask.config.get("distributed.scheduler.unknown-task-duration")
+        )
+        self.MEMORY_RECENT_TO_OLD_TIME = parse_timedelta(
+            dask.config.get("distributed.worker.memory.recent-to-old-time")
+        )
+        self.MEMORY_REBALANCE_MEASURE = dask.config.get(
+            "distributed.worker.memory.rebalance.measure"
+        )
+        self.MEMORY_REBALANCE_SENDER_MIN = dask.config.get(
+            "distributed.worker.memory.rebalance.sender-min"
+        )
+        self.MEMORY_REBALANCE_RECIPIENT_MAX = dask.config.get(
+            "distributed.worker.memory.rebalance.recipient-max"
+        )
+        self.MEMORY_REBALANCE_HALF_GAP = (
+            dask.config.get("distributed.worker.memory.rebalance.sender-recipient-gap")
+            / 2.0
+        )
+
         super().__init__(**kwargs)
 
     @property
@@ -2608,7 +2634,7 @@ def transition_memory_released(self, key, safe: bint = False):
                 "report": False,
             }
             for ws in ts._who_has:
-                ws._has_what.remove(ts)
+                del ws._has_what[ts]
                 ws._nbytes -= ts_nbytes
                 ts._group._nbytes_in_memory -= ts_nbytes
                 worker_msgs[ws._address] = [worker_msg]
@@ -3061,7 +3087,7 @@ def get_comm_cost(self, ts: TaskState, ws: WorkerState) -> double:
         on the given worker.
         """
         dts: TaskState
-        deps: set = ts._dependencies - ws._has_what
+        deps: set = ts._dependencies.difference(ws._has_what)
         nbytes: Py_ssize_t = 0
         for dts in deps:
             nbytes += dts._nbytes
@@ -3074,18 +3100,14 @@ def get_task_duration(self, ts: TaskState, default: double = -1) -> double:
         (not including any communication cost).
         """
         duration: double = ts._prefix._duration_average
-        if duration < 0:
-            s: set = self._unknown_durations.get(ts._prefix._name)
-            if s is None:
-                self._unknown_durations[ts._prefix._name] = s = set()
-            s.add(ts)
+        if duration >= 0:
+            return duration
 
-            if default < 0:
-                duration = UNKNOWN_TASK_DURATION
-            else:
-                duration = default
-
-        return duration
+        s: set = self._unknown_durations.get(ts._prefix._name)
+        if s is None:
+            self._unknown_durations[ts._prefix._name] = s = set()
+        s.add(ts)
+        return default if default >= 0 else self.UNKNOWN_TASK_DURATION
 
     @ccall
     @exceptval(check=False)
@@ -3867,7 +3889,7 @@ def heartbeat_worker(
 
         # Calculate RSS - dask keys, separating "old" and "new" usage
         # See MemoryState for details
-        max_memory_unmanaged_old_hist_age = local_now - MEMORY_RECENT_TO_OLD_TIME
+        max_memory_unmanaged_old_hist_age = local_now - parent.MEMORY_RECENT_TO_OLD_TIME
         memory_unmanaged_old = ws._memory_unmanaged_old
         while ws._memory_other_history:
             timestamp, size = ws._memory_other_history[0]
@@ -4498,7 +4520,7 @@ def stimulus_missing_data(
                 ws: WorkerState
                 cts_nbytes: Py_ssize_t = cts.get_nbytes()
                 for ws in cts._who_has:  # TODO: this behavior is extreme
-                    ws._has_what.remove(cts)
+                    del ws._has_what[ts]
                     ws._nbytes -= cts_nbytes
                 cts._who_has.clear()
                 recommendations[cause] = "released"
@@ -5084,7 +5106,7 @@ def handle_missing_data(self, key=None, errant_worker=None, **kwargs):
         ws: WorkerState = parent._workers_dv.get(errant_worker)
         if ws is not None and ws in ts._who_has:
             ts._who_has.remove(ws)
-            ws._has_what.remove(ts)
+            del ws._has_what[ts]
             ws._nbytes -= ts.get_nbytes()
         if not ts._who_has:
             if ts._run_spec:
@@ -5096,12 +5118,12 @@ def release_worker_data(self, comm=None, keys=None, worker=None):
         parent: SchedulerState = cast(SchedulerState, self)
         ws: WorkerState = parent._workers_dv[worker]
         tasks: set = {parent._tasks[k] for k in keys}
-        removed_tasks: set = tasks & ws._has_what
-        ws._has_what -= removed_tasks
+        removed_tasks: set = tasks.intersection(ws._has_what)
 
         ts: TaskState
         recommendations: dict = {}
         for ts in removed_tasks:
+            del ws._has_what[ts]
             ws._nbytes -= ts.get_nbytes()
             wh: set = ts._who_has
             wh.remove(ws)
@@ -5342,7 +5364,7 @@ async def gather(self, comm=None, keys=None, serializers=None):
                     for worker in workers:
                         ws = parent._workers_dv.get(worker)
                         if ws is not None and ts in ws._has_what:
-                            ws._has_what.remove(ts)
+                            del ws._has_what[ts]
                             ts._who_has.remove(ws)
                             ws._nbytes -= ts_nbytes
                             parent._transitions(
@@ -5508,144 +5530,364 @@ async def _delete_worker_data(self, worker_address, keys):
         ws: WorkerState = parent._workers_dv[worker_address]
         ts: TaskState
         tasks: set = {parent._tasks[key] for key in keys}
-        ws._has_what -= tasks
         for ts in tasks:
+            del ws._has_what[ts]
             ts._who_has.remove(ws)
             ws._nbytes -= ts.get_nbytes()
         self.log_event(ws._address, {"action": "remove-worker-data", "keys": keys})
 
-    async def rebalance(self, comm=None, keys=None, workers=None):
-        """Rebalance keys so that each worker stores roughly equal bytes
-
-        **Policy**
+    async def rebalance(
+        self,
+        comm=None,
+        keys: "Iterable[Hashable]" = None,
+        workers: "Iterable[str]" = None,
+    ) -> dict:
+        """Rebalance keys so that each worker ends up with roughly the same process
+        memory (managed+unmanaged).
+
+        FIXME this method is not robust when the cluster is not idle.
+
+        **Algorithm**
+
+        #. Find the mean occupancy of the cluster, defined as data managed by dask +
+           unmanaged process memory that has been there for at least 30 seconds
+           (``distributed.worker.memory.recent-to-old-time``).
+           This lets us ignore temporary spikes caused by task heap usage.
+        #. Discard workers whose occupancy is within 5% of the mean cluster occupancy
+           (``distributed.worker.memory.rebalance.sender-recipient-gap`` / 2).
+           This helps avoid data from bouncing around the cluster repeatedly.
+        #. Workers above the mean are senders; those below are recipients.
+        #. Discard senders whose absolute occupancy is below 40%
+           (``distributed.worker.memory.rebalance.sender-min``). In other words, no data
+           is moved regardless of imbalancing as long as all workers are below 40%.
+        #. Discard recipients whose absolute occupancy is above 60%
+           (``distributed.worker.memory.rebalance.recipient-max``).
+           Note that this threshold by default is the same as
+           ``distributed.worker.memory.target`` to prevent workers from accepting data
+           and immediately spilling it out to disk.
+        #. Iteratively pick the sender and recipient that are farthest from the mean and
+           move the *least recently inserted* key between the two, until either all
+           senders or all recipients fall within 5% of the mean.
+
+           A recipient will be skipped if it already has a copy of the data. In other
+           words, this method does not degrade replication.
+           A key will be skipped if there are no recipients that have both enough memory
+           to accept and don't already hold a copy.
+
+        The least recently insertd (LRI) policy is a greedy choice with the advantage of
+        being O(1), trivial to implement (it relies on python dict insertion-sorting)
+        and hopefully good enough in most cases. Discarded alternative policies were:
+
+        - Largest first. O(n*log(n)) save for non-trivial additional data structures and
+          risks causing the largest chunks of data to repeatedly move around the
+          cluster like pinballs.
+        - Least recently utilized. This information is currently available on the
+          workers only and not trivial to replicate on the scheduler; transmitting it
+          over the network would be very expensive. Also, note that dask will go out of
+          its way to minimise the amount of time intermediate keys are held in memory,
+          so in such a case LRI is a close approximation of LRU.
 
-        This orders the workers by what fraction of bytes of the existing keys
-        they have.  It walks down this list from most-to-least.  At each worker
-        it sends the largest results it can find and sends them to the least
-        occupied worker until either the sender or the recipient are at the
-        average expected load.
+        Parameters
+        ----------
+        keys: optional
+            whitelist of dask keys that should be considered for moving. All other keys
+            will be ignored. Note that this offers no guarantee that a key will actually
+            be moved (e.g. because it is unnecessary or because there are no viable
+            recipient workers for it).
+        workers: optional
+            whitelist of workers addresses to be considered as senders or recipients.
+            All other workers will be ignored. The mean cluster occupancy will be
+            calculated only using the whitelisted workers.
         """
-        parent: SchedulerState = cast(SchedulerState, self)
-        ts: TaskState
-        with log_errors():
-            async with self._lock:
-                if keys:
-                    tasks = {parent._tasks[k] for k in keys}
-                    missing_data = [ts._key for ts in tasks if not ts._who_has]
-                    if missing_data:
-                        return {"status": "missing-data", "keys": missing_data}
-                else:
-                    tasks = set(parent._tasks.values())
+        parent: SchedulerState = self
 
-                if workers:
-                    workers = {parent._workers_dv[w] for w in workers}
-                    workers_by_task = {ts: ts._who_has & workers for ts in tasks}
-                else:
-                    workers = set(parent._workers_dv.values())
-                    workers_by_task = {ts: ts._who_has for ts in tasks}
+        with log_errors():
+            if workers is not None:
+                workers = [parent._workers_dv[w] for w in workers]
+            else:
+                workers = parent._workers_dv.values()
+            if not workers:
+                return {"status": "OK"}
 
-                ws: WorkerState
-                tasks_by_worker = {ws: set() for ws in workers}
+            if keys is not None:
+                if not isinstance(keys, Set):
+                    keys = set(keys)  # unless already a set-like
+                if not keys:
+                    return {"status": "OK"}
+                missing_data = [
+                    k
+                    for k in keys
+                    if k not in parent._tasks or not parent._tasks[k].who_has
+                ]
+                if missing_data:
+                    return {"status": "missing-data", "keys": missing_data}
 
-                for k, v in workers_by_task.items():
-                    for vv in v:
-                        tasks_by_worker[vv].add(k)
+            msgs = self._rebalance_find_msgs(keys, workers)
+            if not msgs:
+                return {"status": "OK"}
 
-                worker_bytes = {
-                    ws: sum(ts.get_nbytes() for ts in v)
-                    for ws, v in tasks_by_worker.items()
-                }
+            async with self._lock:
+                return await self._rebalance_move_data(msgs)
+
+    def _rebalance_find_msgs(
+        self: SchedulerState,
+        keys: "Optional[Set[Hashable]]",
+        workers: "Iterable[WorkerState]",
+    ) -> "list[tuple[WorkerState, WorkerState, TaskState]]":
+        """Identify workers that need to lose keys and those that can receive them,
+        together with how many bytes each needs to lose/receive. Then, pair a sender
+        worker with a recipient worker for each key, until the cluster is rebalanced.
+
+        This method only defines the work to be performed; it does not start any network
+        transfers itself.
+
+        The big-O complexity is O(wt + ke*log(we)), where
+
+        - wt is the total number of workers on the cluster (or the number of whitelisted
+          workers, if explicitly stated by the user)
+        - we is the number of workers that are eligible to be senders or recipients
+        - kt is the total number of keys on the cluster (or on the whitelisted workers)
+        - ke is the number of keys that need to be moved in order to achieve a balanced
+          cluster
+
+        There is a degenerate edge case O(wt + kt*log(we)) when kt is much greater than
+        the number of whitelisted keys, or when most keys are replicated or cannot be
+        moved for some other reason.
+
+        Returns list of tuples to feed into _rebalance_move_data:
+
+        - sender worker
+        - recipient worker
+        - task to be transferred
+        """
+        parent: SchedulerState = self
+        ts: TaskState
+        ws: WorkerState
 
-                avg = sum(worker_bytes.values()) / len(worker_bytes)
+        # Heaps of workers, managed by the heapq module, that need to send/receive data,
+        # with how many bytes each needs to send/receive.
+        #
+        # Each element of the heap is a tuple constructed as follows:
+        # - snd_bytes_max/rec_bytes_max: maximum number of bytes to send or receive.
+        #   This number is negative, so that the workers farthest from the cluster mean
+        #   are at the top of the smallest-first heaps.
+        # - snd_bytes_min/rec_bytes_min: minimum number of bytes after sending/receiving
+        #   which the worker should not be considered anymore. This is also negative.
+        # - arbitrary unique number, there just to to make sure that WorkerState objects
+        #   are never used for sorting in the unlikely event that two processes have
+        #   exactly the same number of bytes allocated.
+        # - WorkerState
+        # - iterator of all tasks in memory on the worker (senders only), insertion
+        #   sorted (least recently inserted first).
+        #   Note that this iterator will typically *not* be exhausted. It will only be
+        #   exhausted if, after moving away from the worker all keys that can be moved,
+        #   is insufficient to drop snd_bytes_min above 0.
+        senders: "list[tuple[int, int, int, WorkerState, Iterator[TaskState]]]" = []
+        recipients: "list[tuple[int, int, int, WorkerState]]" = []
+
+        # Output: [(sender, recipient, task), ...]
+        msgs: "list[tuple[WorkerState, WorkerState, TaskState]]" = []
+
+        # By default, this is the optimistic memory, meaning total process memory minus
+        # unmanaged memory that appeared over the last 30 seconds
+        # (distributed.worker.memory.recent-to-old-time).
+        # This lets us ignore temporary spikes caused by task heap usage.
+        memory_by_worker = [
+            (ws, getattr(ws.memory, parent.MEMORY_REBALANCE_MEASURE)) for ws in workers
+        ]
+        mean_memory = sum(m for _, m in memory_by_worker) // len(memory_by_worker)
 
-                sorted_workers = list(
-                    map(first, sorted(worker_bytes.items(), key=second, reverse=True))
+        for ws, ws_memory in memory_by_worker:
+            if ws.memory_limit:
+                half_gap = int(parent.MEMORY_REBALANCE_HALF_GAP * ws.memory_limit)
+                sender_min = parent.MEMORY_REBALANCE_SENDER_MIN * ws.memory_limit
+                recipient_max = parent.MEMORY_REBALANCE_RECIPIENT_MAX * ws.memory_limit
+            else:
+                half_gap = 0
+                sender_min = 0.0
+                recipient_max = math.inf
+
+            if (
+                ws._has_what
+                and ws_memory >= mean_memory + half_gap
+                and ws_memory >= sender_min
+            ):
+                # This may send the worker below sender_min (by design)
+                snd_bytes_max = mean_memory - ws_memory  # negative
+                snd_bytes_min = snd_bytes_max + half_gap  # negative
+                # See definition of senders above
+                senders.append(
+                    (snd_bytes_max, snd_bytes_min, id(ws), ws, iter(ws._has_what))
                 )
+            elif ws_memory < mean_memory - half_gap and ws_memory < recipient_max:
+                # This may send the worker above recipient_max (by design)
+                rec_bytes_max = ws_memory - mean_memory  # negative
+                rec_bytes_min = rec_bytes_max + half_gap  # negative
+                # See definition of recipients above
+                recipients.append((rec_bytes_max, rec_bytes_min, id(ws), ws))
+
+        # Fast exit in case no transfers are necessary or possible
+        if not senders or not recipients:
+            self.log_event(
+                "all",
+                {
+                    "action": "rebalance",
+                    "senders": len(senders),
+                    "recipients": len(recipients),
+                    "moved_keys": 0,
+                },
+            )
+            return []
 
-                recipients = reversed(sorted_workers)
-                recipient = next(recipients)
-                msgs = []  # (sender, recipient, key)
-                for sender in sorted_workers[: len(workers) // 2]:
-                    sender_keys = {
-                        ts: ts.get_nbytes() for ts in tasks_by_worker[sender]
-                    }
-                    sender_keys = iter(
-                        sorted(sender_keys.items(), key=second, reverse=True)
-                    )
+        heapq.heapify(senders)
+        heapq.heapify(recipients)
 
-                    try:
-                        while avg < worker_bytes[sender]:
-                            while worker_bytes[recipient] < avg < worker_bytes[sender]:
-                                ts, nb = next(sender_keys)
-                                if ts not in tasks_by_worker[recipient]:
-                                    tasks_by_worker[recipient].add(ts)
-                                    # tasks_by_worker[sender].remove(ts)
-                                    msgs.append((sender, recipient, ts))
-                                    worker_bytes[sender] -= nb
-                                    worker_bytes[recipient] += nb
-                            if avg < worker_bytes[sender]:
-                                recipient = next(recipients)
-                    except StopIteration:
+        snd_ws: WorkerState
+        rec_ws: WorkerState
+
+        while senders and recipients:
+            snd_bytes_max, snd_bytes_min, _, snd_ws, ts_iter = senders[0]
+
+            # Iterate through tasks in memory, least recently inserted first
+            for ts in ts_iter:
+                if keys is not None and ts.key not in keys:
+                    continue
+                nbytes = ts.nbytes
+                if nbytes + snd_bytes_max > 0:
+                    # Moving this task would cause the sender to go below mean and
+                    # potentially risk becoming a recipient, which would cause tasks to
+                    # bounce around. Move on to the next task of the same sender.
+                    continue
+
+                # Find the recipient, farthest from the mean, which
+                # 1. has enough available RAM for this task, and
+                # 2. doesn't hold a copy of this task already
+                # There may not be any that satisfies these conditions; in this case
+                # this task won't be moved.
+                skipped_recipients = []
+                use_recipient = False
+                while recipients and not use_recipient:
+                    rec_bytes_max, rec_bytes_min, _, rec_ws = recipients[0]
+                    if nbytes + rec_bytes_max > 0:
+                        # recipients are sorted by rec_bytes_max.
+                        # The next ones will be worse; no reason to continue iterating
                         break
+                    use_recipient = ts not in rec_ws._has_what
+                    if not use_recipient:
+                        skipped_recipients.append(heapq.heappop(recipients))
 
-                to_recipients = defaultdict(lambda: defaultdict(list))
-                to_senders = defaultdict(list)
-                for sender, recipient, ts in msgs:
-                    to_recipients[recipient.address][ts._key].append(sender.address)
-                    to_senders[sender.address].append(ts._key)
+                for recipient in skipped_recipients:
+                    heapq.heappush(recipients, recipient)
 
-                result = await asyncio.gather(
-                    *(
-                        retry_operation(self.rpc(addr=r).gather, who_has=v)
-                        for r, v in to_recipients.items()
+                if not use_recipient:
+                    # This task has no recipients available. Leave it on the sender and
+                    # move on to the next task of the same sender.
+                    continue
+
+                # Schedule task for transfer from sender to receiver
+                msgs.append((snd_ws, rec_ws, ts))
+
+                # *_bytes_max/min are all negative for heap sorting
+                snd_bytes_max += nbytes
+                snd_bytes_min += nbytes
+                rec_bytes_max += nbytes
+                rec_bytes_min += nbytes
+
+                # Stop iterating on the tasks of this sender for now and, if it still
+                # has bytes to lose, push it back into the senders heap; it may or may
+                # not come back on top again.
+                if snd_bytes_min < 0:
+                    # See definition of senders above
+                    heapq.heapreplace(
+                        senders,
+                        (snd_bytes_max, snd_bytes_min, id(snd_ws), snd_ws, ts_iter),
                     )
-                )
-                for r, v in to_recipients.items():
-                    self.log_event(r, {"action": "rebalance", "who_has": v})
+                else:
+                    heapq.heappop(senders)
+
+                # If receiver still has bytes to gain, push it back into the receivers
+                # heap; it may or may not come back on top again.
+                if rec_bytes_min < 0:
+                    # See definition of recipients above
+                    heapq.heapreplace(
+                        recipients,
+                        (rec_bytes_max, rec_bytes_min, id(rec_ws), rec_ws),
+                    )
+                else:
+                    heapq.heappop(recipients)
 
-                self.log_event(
-                    "all",
-                    {
-                        "action": "rebalance",
-                        "total-keys": len(tasks),
-                        "senders": valmap(len, to_senders),
-                        "recipients": valmap(len, to_recipients),
-                        "moved_keys": len(msgs),
-                    },
-                )
+                # Move to next sender with the most data to lose.
+                # It may or may not be the same sender again.
+                break
 
-                if any(r["status"] != "OK" for r in result):
-                    return {
-                        "status": "missing-data",
-                        "keys": tuple(
-                            concat(
-                                r["keys"].keys()
-                                for r in result
-                                if r["status"] == "missing-data"
-                            )
-                        ),
-                    }
+            else:  # for ts in ts_iter
+                # Exhausted tasks on this sender
+                heapq.heappop(senders)
 
-                for sender, recipient, ts in msgs:
-                    assert ts._state == "memory"
-                    ts._who_has.add(recipient)
-                    recipient.has_what.add(ts)
-                    recipient.nbytes += ts.get_nbytes()
-                    self.log.append(
-                        (
-                            "rebalance",
-                            ts._key,
-                            time(),
-                            sender.address,
-                            recipient.address,
-                        )
+        return msgs
+
+    async def _rebalance_move_data(
+        self, msgs: "list[tuple[WorkerState, WorkerState, TaskState]]"
+    ) -> dict:
+        """Perform the actual transfer of data across the network in rebalance().
+        Takes in input the output of _rebalance_find_msgs().
+
+        FIXME this method is not robust when the cluster is not idle.
+        """
+        ts: TaskState
+        snd_ws: WorkerState
+        rec_ws: WorkerState
+
+        to_recipients = defaultdict(lambda: defaultdict(list))
+        to_senders = defaultdict(list)
+        for sender, recipient, ts in msgs:
+            to_recipients[recipient.address][ts._key].append(sender.address)
+            to_senders[sender.address].append(ts._key)
+
+        result = await asyncio.gather(
+            *(
+                retry_operation(self.rpc(addr=r).gather, who_has=v)
+                for r, v in to_recipients.items()
+            )
+        )
+        for r, v in to_recipients.items():
+            self.log_event(r, {"action": "rebalance", "who_has": v})
+
+        self.log_event(
+            "all",
+            {
+                "action": "rebalance",
+                "senders": valmap(len, to_senders),
+                "recipients": valmap(len, to_recipients),
+                "moved_keys": len(msgs),
+            },
+        )
+
+        if any(r["status"] != "OK" for r in result):
+            return {
+                "status": "missing-data",
+                "keys": list(
+                    concat(
+                        r["keys"].keys()
+                        for r in result
+                        if r["status"] == "missing-data"
                     )
+                ),
+            }
 
-                await asyncio.gather(
-                    *(self._delete_worker_data(r, v) for r, v in to_senders.items())
-                )
+        for snd_ws, rec_ws, ts in msgs:
+            assert ts._state == "memory"
+            ts._who_has.add(rec_ws)
+            rec_ws._has_what[ts] = None
+            rec_ws.nbytes += ts.get_nbytes()
+            self.log.append(
+                ("rebalance", ts._key, time(), snd_ws.address, rec_ws.address)
+            )
 
-                return {"status": "OK"}
+        await asyncio.gather(
+            *(self._delete_worker_data(r, v) for r, v in to_senders.items())
+        )
+        return {"status": "OK"}
 
     async def replicate(
         self,
@@ -5976,7 +6218,7 @@ async def retire_workers(
                 logger.info("Retire workers %s", workers)
 
                 # Keys orphaned by retiring those workers
-                keys = set.union(*[w.has_what for w in workers])
+                keys = {k for w in workers for k in w.has_what}
                 keys = {ts._key for ts in keys if ts._who_has.issubset(workers)}
 
                 if keys:
@@ -6030,7 +6272,7 @@ def add_keys(self, comm=None, worker=None, keys=()):
             if ts is not None and ts._state == "memory":
                 if ts not in ws._has_what:
                     ws._nbytes += ts.get_nbytes()
-                    ws._has_what.add(ts)
+                    ws._has_what[ts] = None
                     ts._who_has.add(ws)
             else:
                 self.worker_send(
@@ -6075,7 +6317,7 @@ def update_data(
                     ws: WorkerState = parent._workers_dv[w]
                     if ts not in ws._has_what:
                         ws._nbytes += ts_nbytes
-                        ws._has_what.add(ts)
+                        ws._has_what[ts] = None
                         ts._who_has.add(ws)
                 self.report(
                     {"op": "key-in-memory", "key": key, "workers": list(workers)}
@@ -6202,7 +6444,7 @@ def get_has_what(self, comm=None, workers=None):
             }
         else:
             return {
-                w: [ts._key for ts in ws._has_what]
+                w: [ts._key for ts in ws.has_what]
                 for w, ws in parent._workers_dv.items()
             }
 
@@ -6975,7 +7217,7 @@ def _add_to_memory(
         assert ts not in ws._has_what
 
     ts._who_has.add(ws)
-    ws._has_what.add(ts)
+    ws._has_what[ts] = None
     ws._nbytes += ts.get_nbytes()
 
     deps: list = list(ts._dependents)
@@ -7058,7 +7300,7 @@ def _propagate_forgotten(
 
     ws: WorkerState
     for ws in ts._who_has:
-        ws._has_what.remove(ts)
+        del ws._has_what[ts]
         ws._nbytes -= ts_nbytes
         w: str = ws._address
         if w in state._workers_dv:  # in case worker has died
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index f17de5949be..2e3d1ef2aa9 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -2892,92 +2892,128 @@ def __reduce__(self):
         raise BadlySerializedException("hello world")
 
     x = c.submit(f)
+    with pytest.raises(Exception, match="hello world"):
+        await x
 
-    try:
-        result = await x
-    except Exception as e:
-        assert "hello world" in str(e)
-    else:
-        assert False
 
+@gen_cluster(
+    client=True,
+    Worker=Nanny,
+    worker_kwargs={"memory_limit": "1 GiB"},
+    config={"distributed.worker.memory.rebalance.sender-min": 0.3},
+)
+async def test_rebalance(c, s, *_):
+    """Test Client.rebalance(). These are just to test the Client wrapper around
+    Scheduler.rebalance(); for more thorough tests on the latter see test_scheduler.py.
+    """
+    # We used nannies to have separate processes for each worker
+    a, b = s.workers
 
-@gen_cluster(client=True)
-async def test_rebalance(c, s, a, b):
-    aws = s.workers[a.address]
-    bws = s.workers[b.address]
+    # Generate 10 buffers worth 512 MiB total on worker a. This sends its memory
+    # utilisation slightly above 50% (after counting unmanaged) which is above the
+    # distributed.worker.memory.rebalance.sender-min threshold.
+    futures = c.map(lambda _: "x" * (2 ** 29 // 10), range(10), workers=[a])
+    await wait(futures)
+    # Wait for heartbeats
+    while s.memory.process < 2 ** 29:
+        await asyncio.sleep(0.1)
 
-    x, y = await c.scatter([1, 2], workers=[a.address])
-    assert len(a.data) == 2
-    assert len(b.data) == 0
+    assert await c.run(lambda dask_worker: len(dask_worker.data)) == {a: 10, b: 0}
 
-    s.validate_state()
     await c.rebalance()
-    s.validate_state()
 
-    assert len(b.data) == 1
-    assert {ts.key for ts in bws.has_what} == set(b.data)
-    assert bws in s.tasks[x.key].who_has or bws in s.tasks[y.key].who_has
+    ndata = await c.run(lambda dask_worker: len(dask_worker.data))
+    # Allow for some uncertainty as the unmanaged memory is not stable
+    assert sum(ndata.values()) == 10
+    assert 3 <= ndata[a] <= 7
+    assert 3 <= ndata[b] <= 7
 
-    assert len(a.data) == 1
-    assert {ts.key for ts in aws.has_what} == set(a.data)
-    assert aws not in s.tasks[x.key].who_has or aws not in s.tasks[y.key].who_has
-
-
-@gen_cluster(nthreads=[("127.0.0.1", 1)] * 4, client=True)
-async def test_rebalance_workers(e, s, a, b, c, d):
-    w, x, y, z = await e.scatter([1, 2, 3, 4], workers=[a.address])
-    assert len(a.data) == 4
-    assert len(b.data) == 0
-    assert len(c.data) == 0
-    assert len(d.data) == 0
-
-    await e.rebalance([x, y], workers=[a.address, c.address])
-    assert len(a.data) == 3
-    assert len(b.data) == 0
-    assert len(c.data) == 1
-    assert len(d.data) == 0
-    assert c.data == {x.key: 2} or c.data == {y.key: 3}
-
-    await e.rebalance()
-    assert len(a.data) == 1
-    assert len(b.data) == 1
-    assert len(c.data) == 1
-    assert len(d.data) == 1
-    s.validate_state()
 
+@gen_cluster(
+    nthreads=[("127.0.0.1", 1)] * 3,
+    client=True,
+    Worker=Nanny,
+    worker_kwargs={"memory_limit": "1 GiB"},
+)
+async def test_rebalance_workers_and_keys(client, s, *_):
+    """Test Client.rebalance(). These are just to test the Client wrapper around
+    Scheduler.rebalance(); for more thorough tests on the latter see test_scheduler.py.
+    """
+    a, b, c = s.workers
+    futures = client.map(lambda _: "x" * (2 ** 29 // 10), range(10), workers=[a])
+    await wait(futures)
+    # Wait for heartbeats
+    while s.memory.process < 2 ** 29:
+        await asyncio.sleep(0.1)
 
-@gen_cluster(client=True)
-async def test_rebalance_execution(c, s, a, b):
-    futures = c.map(inc, range(10), workers=a.address)
-    await c.rebalance(futures)
-    assert len(a.data) == len(b.data) == 5
-    s.validate_state()
+    # Passing empty iterables is not the same as omitting the arguments
+    await client.rebalance([])
+    await client.rebalance(workers=[])
+    assert await client.run(lambda dask_worker: len(dask_worker.data)) == {
+        a: 10,
+        b: 0,
+        c: 0,
+    }
 
+    # Limit rebalancing to two arbitrary keys and two arbitrary workers.
+    await client.rebalance([futures[3], futures[7]], [a, b])
+    assert await client.run(lambda dask_worker: len(dask_worker.data)) == {
+        a: 8,
+        b: 2,
+        c: 0,
+    }
 
-def test_rebalance_sync(c, s, a, b):
-    futures = c.map(inc, range(10), workers=[a["address"]])
-    c.rebalance(futures)
+    with pytest.raises(KeyError):
+        await client.rebalance(workers=["notexist"])
+
+
+def test_rebalance_sync():
+    # can't use the 'c' fixture because we need workers to run in a separate process
+    with Client(n_workers=2, memory_limit="1 GiB") as c:
+        s = c.cluster.scheduler
+        a, b = [ws.address for ws in s.workers.values()]
+        futures = c.map(lambda _: "x" * (2 ** 29 // 10), range(10), workers=[a])
+        wait(futures)
+        # Wait for heartbeat
+        while s.memory.process < 2 ** 29:
+            sleep(0.1)
 
-    has_what = c.has_what()
-    assert len(has_what) == 2
-    assert list(valmap(len, has_what).values()) == [5, 5]
+        assert c.run(lambda dask_worker: len(dask_worker.data)) == {a: 10, b: 0}
+        c.rebalance()
+        ndata = c.run(lambda dask_worker: len(dask_worker.data))
+        # Allow for some uncertainty as the unmanaged memory is not stable
+        assert sum(ndata.values()) == 10
+        assert 3 <= ndata[a] <= 7
+        assert 3 <= ndata[b] <= 7
 
 
 @gen_cluster(client=True)
 async def test_rebalance_unprepared(c, s, a, b):
+    """Client.rebalance() internally waits for unfinished futures"""
     futures = c.map(slowinc, range(10), delay=0.05, workers=a.address)
+    # Let the futures reach the scheduler
     await asyncio.sleep(0.1)
+    # We didn't wait enough for futures to complete. However, Client.rebalance() will
+    # block until all futures are completed before invoking Scheduler.rebalance().
     await c.rebalance(futures)
     s.validate_state()
 
 
-@gen_cluster(client=True)
-async def test_rebalance_raises_missing_data(c, s, a, b):
-    with pytest.raises(ValueError, match="keys were found to be missing"):
-        futures = await c.scatter(range(100))
-        keys = [f.key for f in futures]
-        del futures
-        await c.rebalance(keys)
+@gen_cluster(client=True, Worker=Nanny, worker_kwargs={"memory_limit": "1 GiB"})
+async def test_rebalance_raises_missing_data(c, s, *_):
+    a, b = s.workers
+    futures = c.map(lambda _: "x" * (2 ** 29 // 10), range(10), workers=[a])
+    await wait(futures)
+    # Wait for heartbeats
+    while s.memory.process < 2 ** 29:
+        await asyncio.sleep(0.1)
+
+    # Descoping the futures enqueues a coroutine to release the data on the server
+    del futures
+    with pytest.raises(KeyError, match="keys were found to be missing"):
+        # During the synchronous part of rebalance, the futures still exist, but they
+        # will be (partially) gone by the time the actual transferring happens.
+        await c.rebalance()
 
 
 @gen_cluster(client=True)
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index ccd7ce31dd7..f1ddae3615e 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -2354,7 +2354,7 @@ async def assert_memory(scheduler_or_workerstate, attr: str, min_, max_, timeout
 # This test is heavily influenced by hard-to-control factors such as memory management
 # by the Python interpreter and the OS, so it occasionally glitches
 @pytest.mark.flaky(reruns=3, reruns_delay=5)
-# ~33s runtime, or distributed.memory.recent_to_old_time + 3s
+# ~33s runtime, or distributed.memory.recent-to-old-time + 3s
 @pytest.mark.slow
 @gen_cluster(
     client=True, Worker=Nanny, worker_kwargs={"memory_limit": "500 MiB"}, timeout=60
@@ -2407,7 +2407,7 @@ async def test_memory(c, s, *_):
     await assert_memory(s, "managed_spilled", 1, 999)
 
     # Wait for the spilling to finish. Note that this does not make the test take
-    # longer as we're waiting for recent_to_old_time anyway.
+    # longer as we're waiting for recent-to-old-time anyway.
     await asyncio.sleep(10)
 
     # Delete spilled keys
@@ -2505,3 +2505,286 @@ async def test_close_scheduler__close_workers_Nanny(s, a, b):
             await asyncio.sleep(0.05)
     log = log.getvalue()
     assert "retry" not in log
+
+
+async def assert_ndata(client, by_addr, total=None):
+    """Test that the number of elements in Worker.data is as expected.
+    To be used when the worker is wrapped by a nanny.
+
+    by_addr: dict of either exact numbers or (min, max) tuples
+    total: optional exact match on the total number of keys (with duplicates) across all
+    workers
+    """
+    out = await client.run(lambda dask_worker: len(dask_worker.data))
+    try:
+        for k, v in by_addr.items():
+            if isinstance(v, tuple):
+                assert v[0] <= out[k] <= v[1]
+            else:
+                assert out[k] == v
+        if total is not None:
+            assert sum(out.values()) == total
+    except AssertionError:
+        raise AssertionError(f"Expected {by_addr}, total={total}; got {out}")
+
+
+@gen_cluster(
+    client=True,
+    Worker=Nanny,
+    worker_kwargs={"memory_limit": "1 GiB"},
+    config={"distributed.worker.memory.rebalance.sender-min": 0.3},
+)
+async def test_rebalance(c, s, *_):
+    # We used nannies to have separate processes for each worker
+    a, b = s.workers
+
+    # Generate 10 buffers worth 512 MiB total on worker a. This sends its memory
+    # utilisation slightly above 50% (after counting unmanaged) which is above the
+    # distributed.worker.memory.rebalance.sender-min threshold.
+    futures = c.map(lambda _: "x" * (2 ** 29 // 10), range(10), workers=[a])
+    await wait(futures)
+    # Wait for heartbeats
+    await assert_memory(s, "process", 512, 1024)
+    await assert_ndata(c, {a: 10, b: 0})
+    await s.rebalance()
+    # Allow for some uncertainty as the unmanaged memory is not stable
+    await assert_ndata(c, {a: (3, 7), b: (3, 7)}, total=10)
+
+    # rebalance() when there is nothing to do
+    await s.rebalance()
+    await assert_ndata(c, {a: (3, 7), b: (3, 7)}, total=10)
+    s.validate_state()
+
+
+@gen_cluster(
+    nthreads=[("127.0.0.1", 1)] * 3,
+    client=True,
+    Worker=Nanny,
+    worker_kwargs={"memory_limit": "1 GiB"},
+)
+async def test_rebalance_workers_and_keys(client, s, *_):
+    a, b, c = s.workers
+    futures = client.map(lambda _: "x" * (2 ** 29 // 10), range(10), workers=[a])
+    await wait(futures)
+    # Wait for heartbeats
+    await assert_memory(s, "process", 512, 1024)
+
+    # Passing empty iterables is not the same as omitting the arguments
+    await s.rebalance(keys=[])
+    await assert_ndata(client, {a: 10, b: 0, c: 0})
+    await s.rebalance(workers=[])
+    await assert_ndata(client, {a: 10, b: 0, c: 0})
+    # Limit operation to workers that have nothing to do
+    await s.rebalance(workers=[b, c])
+    await assert_ndata(client, {a: 10, b: 0, c: 0})
+
+    # Limit rebalancing to two arbitrary keys and two arbitrary workers
+    await s.rebalance(keys=[futures[3].key, futures[7].key], workers=[a, b])
+    await assert_ndata(client, {a: 8, b: 2, c: 0}, total=10)
+
+    with pytest.raises(KeyError):
+        await s.rebalance(workers=["notexist"])
+
+    s.validate_state()
+
+
+@gen_cluster()
+async def test_rebalance_missing_data1(s, a, b):
+    """key never existed"""
+    out = await s.rebalance(keys=["notexist"])
+    assert out == {"status": "missing-data", "keys": ["notexist"]}
+    s.validate_state()
+
+
+@gen_cluster(client=True)
+async def test_rebalance_missing_data2(c, s, a, b):
+    """keys exist but belong to unfinished futures. Unlike Client.rebalance(),
+    Scheduler.rebalance() does not wait for unfinished futures.
+    """
+    futures = c.map(slowinc, range(10), delay=0.05, workers=a.address)
+    await asyncio.sleep(0.1)
+    out = await s.rebalance(keys=[f.key for f in futures])
+    assert out["status"] == "missing-data"
+    assert 8 <= len(out["keys"]) <= 10
+    s.validate_state()
+
+
+@gen_cluster(client=True, Worker=Nanny, worker_kwargs={"memory_limit": "1 GiB"})
+async def test_rebalance_raises_missing_data3(c, s, *_):
+    """keys exist when the sync part of rebalance runs, but are gone by the time the
+    actual data movement runs
+    """
+    a, _ = s.workers
+    futures = c.map(lambda _: "x" * (2 ** 29 // 10), range(10), workers=[a])
+    await wait(futures)
+    # Wait for heartbeats
+    await assert_memory(s, "process", 512, 1024)
+    del futures
+    out = await s.rebalance()
+    assert out["status"] == "missing-data"
+    assert 1 <= len(out["keys"]) <= 10
+    s.validate_state()
+
+
+@gen_cluster(nthreads=[])
+async def test_rebalance_no_workers(s):
+    await s.rebalance()
+    s.validate_state()
+
+
+@gen_cluster(
+    client=True,
+    Worker=Nanny,
+    worker_kwargs={"memory_limit": "1000 MiB"},
+    config={
+        "distributed.worker.memory.rebalance.measure": "managed",
+        "distributed.worker.memory.rebalance.sender-min": 0.3,
+    },
+)
+async def test_rebalance_managed_memory(c, s, *_):
+    a, b = s.workers
+    # Generate 100 buffers worth 400 MiB total on worker a. This sends its memory
+    # utilisation to exactly 40%, ignoring unmanaged, which is above the
+    # distributed.worker.memory.rebalance.sender-min threshold.
+    futures = c.map(lambda _: "x" * (2 ** 22), range(100), workers=[a])
+    await wait(futures)
+    # Even if we're just using managed memory, which is instantaneously accounted for as
+    # soon as the tasks finish, MemoryState.managed is still capped by the process
+    # memory, so we need to wait for the heartbeat.
+    await assert_memory(s, "managed", 400, 401)
+    await assert_ndata(c, {a: 100, b: 0})
+    await s.rebalance()
+    # We can expect an exact, stable result because we are completely bypassing the
+    # unpredictability of unmanaged memory.
+    await assert_ndata(c, {a: 62, b: 38})
+    s.validate_state()
+
+
+@gen_cluster(
+    client=True,
+    worker_kwargs={"memory_limit": 0},
+    config={"distributed.worker.memory.rebalance.measure": "managed"},
+)
+async def test_rebalance_no_limit(c, s, a, b):
+    # See notes in test_rebalance_managed_memory
+    futures = c.map(lambda _: "x", range(100), workers=[a.address])
+    await wait(futures)
+    # No reason to wait for memory here as we're allocating hundreds of bytes, so
+    # there's plenty of unmanaged process memory to pad it out
+    await assert_ndata(c, {a.address: 100, b.address: 0})
+    await s.rebalance()
+    # Disabling memory_limit made us ignore all % thresholds set in the config
+    await assert_ndata(c, {a.address: 50, b.address: 50})
+    s.validate_state()
+
+
+@gen_cluster(
+    client=True,
+    Worker=Nanny,
+    worker_kwargs={"memory_limit": "1000 MiB"},
+    config={
+        "distributed.worker.memory.rebalance.measure": "managed",
+        "distributed.worker.memory.rebalance.recipient-max": 0.4,
+    },
+)
+async def test_rebalance_no_recipients(c, s, *_):
+    """There are sender workers, but no recipient workers"""
+    a, b = s.workers
+    futures = [
+        c.submit(lambda: "x" * (400 * 2 ** 20), pure=False, workers=[a]),  # 40%
+        c.submit(lambda: "x" * (400 * 2 ** 20), pure=False, workers=[b]),  # 40%
+    ] + c.map(
+        lambda _: "x" * (2 ** 21), range(100), workers=[a]
+    )  # 20%
+    await wait(futures)
+    await assert_memory(s, "managed", 1000, 1001)
+    await assert_ndata(c, {a: 101, b: 1})
+    await s.rebalance()
+    await assert_ndata(c, {a: 101, b: 1})
+    s.validate_state()
+
+
+@gen_cluster(
+    nthreads=[("127.0.0.1", 1)] * 3,
+    client=True,
+    worker_kwargs={"memory_limit": 0},
+    config={"distributed.worker.memory.rebalance.measure": "managed"},
+)
+async def test_rebalance_skip_recipient(client, s, a, b, c):
+    """A recipient is skipped because it already holds a copy of the key to be sent"""
+    futures = client.map(lambda _: "x", range(10), workers=[a.address])
+    await wait(futures)
+    await client.replicate(futures[0:2], workers=[a.address, b.address])
+    await client.replicate(futures[2:4], workers=[a.address, c.address])
+    await assert_ndata(client, {a.address: 10, b.address: 2, c.address: 2})
+    await client.rebalance(futures[:2])
+    await assert_ndata(client, {a.address: 8, b.address: 2, c.address: 4})
+    s.validate_state()
+
+
+@gen_cluster(
+    client=True,
+    worker_kwargs={"memory_limit": 0},
+    config={"distributed.worker.memory.rebalance.measure": "managed"},
+)
+async def test_rebalance_skip_all_recipients(c, s, a, b):
+    """All recipients are skipped because they already hold copies"""
+    futures = c.map(lambda _: "x", range(10), workers=[a.address])
+    await wait(futures)
+    await c.replicate([futures[0]])
+    await assert_ndata(c, {a.address: 10, b.address: 1})
+    await c.rebalance(futures[:2])
+    await assert_ndata(c, {a.address: 9, b.address: 2})
+    s.validate_state()
+
+
+@gen_cluster(
+    client=True,
+    Worker=Nanny,
+    worker_kwargs={"memory_limit": "1000 MiB"},
+    config={"distributed.worker.memory.rebalance.measure": "managed"},
+)
+async def test_rebalance_sender_below_mean(c, s, *_):
+    """A task remains on the sender because moving it would send it below the mean"""
+    a, b = s.workers
+    f1 = c.submit(lambda: "x" * (400 * 2 ** 20), workers=[a])
+    await wait([f1])
+    f2 = c.submit(lambda: "x" * (10 * 2 ** 20), workers=[a])
+    await wait([f2])
+    await assert_memory(s, "managed", 410, 411)
+    await assert_ndata(c, {a: 2, b: 0})
+    await s.rebalance()
+    assert await c.has_what() == {a: (f1.key,), b: (f2.key,)}
+
+
+@gen_cluster(
+    client=True,
+    Worker=Nanny,
+    worker_kwargs={"memory_limit": "1000 MiB"},
+    config={
+        "distributed.worker.memory.rebalance.measure": "managed",
+        "distributed.worker.memory.rebalance.sender-min": 0.3,
+    },
+)
+async def test_rebalance_least_recently_inserted_sender_min(c, s, *_):
+    """
+    1. keys are picked using a least recently inserted policy
+    2. workers below sender-min are never senders
+    """
+    a, b = s.workers
+    small_futures = c.map(lambda _: "x", range(10), workers=[a])
+    await wait(small_futures)
+    await assert_ndata(c, {a: 10, b: 0})
+    await s.rebalance()
+    await assert_ndata(c, {a: 10, b: 0})
+
+    large_future = c.submit(lambda: "x" * (300 * 2 ** 20), workers=[a])
+    await wait([large_future])
+    await assert_memory(s, "managed", 300, 301)
+    await assert_ndata(c, {a: 11, b: 0})
+    await s.rebalance()
+    await assert_ndata(c, {a: 1, b: 10})
+    assert await c.has_what() == {
+        a: (large_future.key,),
+        b: tuple(f.key for f in small_futures),
+    }
diff --git a/distributed/tests/test_tls_functional.py b/distributed/tests/test_tls_functional.py
index 59e0bbb429d..84a0f5922d9 100644
--- a/distributed/tests/test_tls_functional.py
+++ b/distributed/tests/test_tls_functional.py
@@ -6,11 +6,9 @@
 
 import pytest
 
-from distributed import Client, Nanny, Queue, Scheduler, Worker, worker_client
-from distributed.client import wait
+from distributed import Client, Nanny, Queue, Scheduler, Worker, wait, worker_client
 from distributed.core import Status
 from distributed.metrics import time
-from distributed.nanny import Nanny
 from distributed.utils_test import (  # noqa: F401
     cleanup,
     double,
@@ -101,16 +99,36 @@ async def test_nanny(c, s, a, b):
     assert result == 11
 
 
-@gen_tls_cluster(client=True)
-async def test_rebalance(c, s, a, b):
-    x, y = await c._scatter([1, 2], workers=[a.address])
-    assert len(a.data) == 2
-    assert len(b.data) == 0
+@gen_tls_cluster(
+    client=True,
+    Worker=Nanny,
+    worker_kwargs={"memory_limit": "1 GiB"},
+    config={"distributed.worker.memory.rebalance.sender-min": 0.3},
+)
+async def test_rebalance(c, s, *_):
+    # We used nannies to have separate processes for each worker
+    a, b = s.workers
+    assert a.startswith("tls://")
+
+    # Generate 10 buffers worth 512 MiB total on worker a. This sends its memory
+    # utilisation slightly above 50% (after counting unmanaged) which is above the
+    # distributed.worker.memory.rebalance.sender-min threshold.
+    futures = c.map(lambda _: "x" * (2 ** 29 // 10), range(10), workers=[a])
+    await wait(futures)
+
+    # Wait for heartbeats
+    while s.memory.process < 2 ** 29:
+        await asyncio.sleep(0.1)
+
+    assert await c.run(lambda dask_worker: len(dask_worker.data)) == {a: 10, b: 0}
 
-    await c._rebalance()
+    await c.rebalance()
 
-    assert len(a.data) == 1
-    assert len(b.data) == 1
+    ndata = await c.run(lambda dask_worker: len(dask_worker.data))
+    # Allow for some uncertainty as the unmanaged memory is not stable
+    assert sum(ndata.values()) == 10
+    assert 3 <= ndata[a] <= 7
+    assert 3 <= ndata[b] <= 7
 
 
 @gen_tls_cluster(client=True, nthreads=[("tls://127.0.0.1", 2)] * 2)
diff --git a/docs/source/memory.rst b/docs/source/memory.rst
index 2ae16698ea8..849b71ec616 100644
--- a/docs/source/memory.rst
+++ b/docs/source/memory.rst
@@ -166,7 +166,8 @@ copied to another worker node in the course of normal computation if that
 result is required by another task that is intended to by run by a different
 worker.  This occurs if a task requires two pieces of data on different
 machines (at least one must move) or through work stealing.  In these cases it
-is the policy for the second machine to maintain its redundant copy of the data.  This helps to organically spread around data that is in high demand.
+is the policy for the second machine to maintain its redundant copy of the data.
+This helps to organically spread around data that is in high demand.
 
 However, advanced users may want to control the location, replication, and
 balancing of data more directly throughout the cluster.  They may know ahead of

From abed5f4b5be667c88661a1ce950bf85cfafa5e29 Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Wed, 2 Jun 2021 16:38:02 +0100
Subject: [PATCH 1293/1550] Who has has what html reprs v2 (#4865)

---
 distributed/client.py            | 13 +++++++++++--
 distributed/tests/test_client.py |  5 +++--
 2 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index b5771266937..ca8f87ffaea 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -53,6 +53,7 @@
 )
 from .diagnostics.plugin import UploadFile, WorkerPlugin, _get_worker_plugin_name
 from .metrics import time
+from .objects import HasWhat, WhoHas
 from .protocol import to_serialize
 from .protocol.pickle import dumps, loads
 from .publish import Datasets
@@ -3205,7 +3206,11 @@ def who_has(self, futures=None, **kwargs):
             keys = list(map(stringify, {f.key for f in futures}))
         else:
             keys = None
-        return self.sync(self.scheduler.who_has, keys=keys, **kwargs)
+
+        async def _():
+            return WhoHas(await self.scheduler.who_has(keys=keys, **kwargs))
+
+        return self.sync(_)
 
     def has_what(self, workers=None, **kwargs):
         """Which keys are held by which workers
@@ -3239,7 +3244,11 @@ def has_what(self, workers=None, **kwargs):
             workers = list(workers)
         if workers is not None and not isinstance(workers, (tuple, list, set)):
             workers = [workers]
-        return self.sync(self.scheduler.has_what, workers=workers, **kwargs)
+
+        async def _():
+            return HasWhat(await self.scheduler.has_what(workers=workers, **kwargs))
+
+        return self.sync(_)
 
     def processing(self, workers=None):
         """The tasks currently running on each worker
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 2e3d1ef2aa9..4b86ae05c81 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -3656,13 +3656,14 @@ async def test_async_whowhat(c, s, a, b):
 
     who_has = await c.who_has()
     has_what = await c.has_what()
+    assert type(who_has) is WhoHas
+    assert type(has_what) is HasWhat
 
     assert who_has == {x.key: (a.address,)}
     assert has_what == {a.address: (x.key,), b.address: ()}
 
 
-@pytest.mark.xfail(reason="Want to fix to use `WhoHas` + `WhatHas`")
-def test_client_repr(c):
+def test_client_repr_html(c):
     x = c.submit(inc, 1)
 
     who_has = c.who_has()

From 726f80dbe65dd095ba9446819f2ab4dfe6e0374f Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Thu, 3 Jun 2021 13:31:03 -0500
Subject: [PATCH 1294/1550] Mark `test_server_comms_mark_active_handlers` with
 `pytest.mark.asyncio` (#4876)

---
 distributed/tests/test_core.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/distributed/tests/test_core.py b/distributed/tests/test_core.py
index ac6dff5f4e8..44757b512d2 100644
--- a/distributed/tests/test_core.py
+++ b/distributed/tests/test_core.py
@@ -888,6 +888,7 @@ async def test_server_redundant_kwarg():
         await Server({}, typo_kwarg="foo")
 
 
+@pytest.mark.asyncio
 async def test_server_comms_mark_active_handlers():
     """Whether handlers are active can be read off of the self._comms values.
     ensure this is properly reflected and released. The sentinel for

From 1754b48864563a30b576636611db7266c6b4c9b6 Mon Sep 17 00:00:00 2001
From: Peter Andreas Entschev <peter@entschev.com>
Date: Thu, 3 Jun 2021 21:37:02 +0200
Subject: [PATCH 1295/1550] Move SystemMonitor's GPU initialization back to
 constructor (#4866)

* Always use index 0 to get NVML GPU handle

* Move SystemMonitor's GPU initialization back to constructor

* Use nvmlDeviceGetHandleByIndex directly

* Remove redundant nvmlInit check

* Revert "Use nvmlDeviceGetHandleByIndex directly"

This reverts commit ddf9a434154fad4802bc75915b61957fbbf13600.

* Revert "Always use index 0 to get NVML GPU handle"

This reverts commit d860e585c8455f285f5e5ca0d1470cb2c255e281.
---
 distributed/system_monitor.py | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/distributed/system_monitor.py b/distributed/system_monitor.py
index dba32e55e40..acaeb16c7da 100644
--- a/distributed/system_monitor.py
+++ b/distributed/system_monitor.py
@@ -40,8 +40,9 @@ def __init__(self, n=10000):
             self.quantities["num_fds"] = self.num_fds
 
         if nvml is not None:
-            self.gpu_name = None
-            self.gpu_memory_total = None
+            gpu_extra = nvml.one_time()
+            self.gpu_name = gpu_extra["name"]
+            self.gpu_memory_total = gpu_extra["memory-total"]
             self.gpu_utilization = deque(maxlen=n)
             self.gpu_memory_used = deque(maxlen=n)
             self.quantities["gpu_utilization"] = self.gpu_utilization
@@ -90,12 +91,7 @@ def update(self):
             self.num_fds.append(num_fds)
             result["num_fds"] = num_fds
 
-        # give external modules (like dask-cuda) a chance to initialize CUDA context
-        if nvml is not None and nvml.nvmlInit is not None:
-            if self.gpu_name is None:
-                gpu_extra = nvml.one_time()
-                self.gpu_name = gpu_extra["name"]
-                self.gpu_memory_total = gpu_extra["memory-total"]
+        if nvml is not None:
             gpu_metrics = nvml.real_time()
             self.gpu_utilization.append(gpu_metrics["utilization"])
             self.gpu_memory_used.append(gpu_metrics["memory-used"])

From 6b089268387121e8220803fdef6d074bc7d349df Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Thu, 3 Jun 2021 16:28:53 -0500
Subject: [PATCH 1296/1550] Show more in test summary (#4875)

---
 setup.cfg | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.cfg b/setup.cfg
index 6016c5fb133..1db115623f2 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -45,7 +45,7 @@ tag_prefix =
 parentdir_prefix = distributed-
 
 [tool:pytest]
-addopts = -v -r s --durations=20
+addopts = -v -rsxfE --durations=20
 filterwarnings =
     error:Since distributed.*:PendingDeprecationWarning
 minversion = 4

From 2bdec05aefa1df5332dc9b52f93a543a4d382434 Mon Sep 17 00:00:00 2001
From: Peter Andreas Entschev <peter@entschev.com>
Date: Fri, 4 Jun 2021 16:23:21 +0200
Subject: [PATCH 1297/1550] Ensure PyNVML works correctly when installed with
 no GPUs (#4873)

---
 continuous_integration/environment-3.9.yaml |  1 +
 distributed/dashboard/components/nvml.py    | 10 ++----
 distributed/diagnostics/nvml.py             | 39 +++++++++++++++------
 distributed/diagnostics/tests/test_nvml.py  | 16 +++++++--
 distributed/system_monitor.py               |  4 +--
 distributed/worker.py                       | 13 ++++---
 6 files changed, 54 insertions(+), 29 deletions(-)

diff --git a/continuous_integration/environment-3.9.yaml b/continuous_integration/environment-3.9.yaml
index fa0a661a96b..0dcd972db56 100644
--- a/continuous_integration/environment-3.9.yaml
+++ b/continuous_integration/environment-3.9.yaml
@@ -23,6 +23,7 @@ dependencies:
   - paramiko
   - prometheus_client
   - psutil
+  - pynvml  # Only tested here
   - pytest
   - pytest-asyncio<0.14.0
   - pytest-faulthandler
diff --git a/distributed/dashboard/components/nvml.py b/distributed/dashboard/components/nvml.py
index 4b44902f627..b5c5547d9bd 100644
--- a/distributed/dashboard/components/nvml.py
+++ b/distributed/dashboard/components/nvml.py
@@ -16,16 +16,10 @@
 from distributed.dashboard.components import DashboardComponent, add_periodic_callback
 from distributed.dashboard.components.scheduler import BOKEH_THEME, TICKS_1024, env
 from distributed.dashboard.utils import update, without_property_validation
+from distributed.diagnostics import nvml
 from distributed.utils import log_errors
 
-try:
-    import pynvml
-
-    pynvml.nvmlInit()
-
-    NVML_ENABLED = True
-except Exception:
-    NVML_ENABLED = False
+NVML_ENABLED = nvml.device_get_count() > 0
 
 
 class GPUCurrentLoad(DashboardComponent):
diff --git a/distributed/diagnostics/nvml.py b/distributed/diagnostics/nvml.py
index cd39f9e04c5..2acbe590ee5 100644
--- a/distributed/diagnostics/nvml.py
+++ b/distributed/diagnostics/nvml.py
@@ -1,23 +1,44 @@
 import os
 
-import pynvml
+try:
+    import pynvml
+except ImportError:
+    pynvml = None
 
-nvmlInit = None
+nvmlInitialized = False
+nvmlLibraryNotFound = False
+nvmlOwnerPID = None
 
 
 def init_once():
-    global nvmlInit
-    if nvmlInit is not None:
+    global nvmlInitialized, nvmlLibraryNotFound, nvmlOwnerPID
+    if pynvml is None or (nvmlInitialized is True and nvmlOwnerPID == os.getpid()):
         return
 
-    from pynvml import nvmlInit as _nvmlInit
+    nvmlInitialized = True
+    nvmlOwnerPID = os.getpid()
+    try:
+        pynvml.nvmlInit()
+    except pynvml.NVMLError_LibraryNotFound:
+        nvmlLibraryNotFound = True
+
 
-    nvmlInit = _nvmlInit
-    nvmlInit()
+def device_get_count():
+    init_once()
+    if nvmlLibraryNotFound or not nvmlInitialized:
+        return 0
+    else:
+        return pynvml.nvmlDeviceGetCount()
 
 
 def _pynvml_handles():
-    count = pynvml.nvmlDeviceGetCount()
+    count = device_get_count()
+    if count == 0:
+        if nvmlLibraryNotFound:
+            raise RuntimeError("PyNVML is installed, but NVML is not")
+        else:
+            raise RuntimeError("No GPUs available")
+
     try:
         cuda_visible_devices = [
             int(idx) for idx in os.environ.get("CUDA_VISIBLE_DEVICES", "").split(",")
@@ -32,7 +53,6 @@ def _pynvml_handles():
 
 
 def real_time():
-    init_once()
     h = _pynvml_handles()
     return {
         "utilization": pynvml.nvmlDeviceGetUtilizationRates(h).gpu,
@@ -41,7 +61,6 @@ def real_time():
 
 
 def one_time():
-    init_once()
     h = _pynvml_handles()
     return {
         "memory-total": pynvml.nvmlDeviceGetMemoryInfo(h).total,
diff --git a/distributed/diagnostics/tests/test_nvml.py b/distributed/diagnostics/tests/test_nvml.py
index 0b19ee759b1..587fde19263 100644
--- a/distributed/diagnostics/tests/test_nvml.py
+++ b/distributed/diagnostics/tests/test_nvml.py
@@ -9,6 +9,9 @@
 
 
 def test_one_time():
+    if nvml.device_get_count() < 1:
+        pytest.skip("No GPUs available")
+
     output = nvml.one_time()
     assert "memory-total" in output
     assert "name" in output
@@ -17,6 +20,9 @@ def test_one_time():
 
 
 def test_1_visible_devices():
+    if nvml.device_get_count() < 1:
+        pytest.skip("No GPUs available")
+
     os.environ["CUDA_VISIBLE_DEVICES"] = "0"
     output = nvml.one_time()
     h = nvml._pynvml_handles()
@@ -25,8 +31,8 @@ def test_1_visible_devices():
 
 @pytest.mark.parametrize("CVD", ["1,0", "0,1"])
 def test_2_visible_devices(CVD):
-    if pynvml.nvmlDeviceGetCount() <= 1:
-        pytest.skip("Machine only has a single GPU")
+    if nvml.device_get_count() < 2:
+        pytest.skip("Less than two GPUs available")
 
     os.environ["CUDA_VISIBLE_DEVICES"] = CVD
     idx = int(CVD.split(",")[0])
@@ -42,6 +48,9 @@ def test_2_visible_devices(CVD):
 
 @gen_cluster()
 async def test_gpu_metrics(s, a, b):
+    if nvml.device_get_count() < 1:
+        pytest.skip("No GPUs available")
+
     h = nvml._pynvml_handles()
 
     assert "gpu" in a.metrics
@@ -58,6 +67,9 @@ async def test_gpu_metrics(s, a, b):
 
 @gen_cluster()
 async def test_gpu_monitoring(s, a, b):
+    if nvml.device_get_count() < 1:
+        pytest.skip("No GPUs available")
+
     h = nvml._pynvml_handles()
     res = await s.get_worker_monitor_info(recent=True)
 
diff --git a/distributed/system_monitor.py b/distributed/system_monitor.py
index acaeb16c7da..8824ef2db12 100644
--- a/distributed/system_monitor.py
+++ b/distributed/system_monitor.py
@@ -39,7 +39,7 @@ def __init__(self, n=10000):
             self.num_fds = deque(maxlen=n)
             self.quantities["num_fds"] = self.num_fds
 
-        if nvml is not None:
+        if nvml.device_get_count() > 0:
             gpu_extra = nvml.one_time()
             self.gpu_name = gpu_extra["name"]
             self.gpu_memory_total = gpu_extra["memory-total"]
@@ -91,7 +91,7 @@ def update(self):
             self.num_fds.append(num_fds)
             result["num_fds"] = num_fds
 
-        if nvml is not None:
+        if nvml.device_get_count() > 0:
             gpu_metrics = nvml.real_time()
             self.gpu_utilization.append(gpu_metrics["utilization"])
             self.gpu_memory_used.append(gpu_metrics["memory-used"])
diff --git a/distributed/worker.py b/distributed/worker.py
index 9864ec87743..15e76544a8a 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -41,6 +41,7 @@
     pingpong,
     send_recv,
 )
+from .diagnostics import nvml
 from .diagnostics.plugin import _get_worker_plugin_name
 from .diskutils import WorkSpace
 from .http import get_handlers
@@ -78,11 +79,6 @@
 from .utils_perf import ThrottledGC, disable_gc_diagnosis, enable_gc_diagnosis
 from .versions import get_versions
 
-try:
-    from .diagnostics import nvml
-except Exception:
-    nvml = None
-
 logger = logging.getLogger(__name__)
 
 LOG_PDB = dask.config.get("distributed.admin.pdb-on-err")
@@ -1085,7 +1081,7 @@ def get_monitor_info(self, comm=None, recent=False, start=0):
             count=self.monitor.count,
             last_time=self.monitor.last_time,
         )
-        if nvml is not None:
+        if nvml.device_get_count() > 0:
             result["gpu_name"] = self.monitor.gpu_name
             result["gpu_memory_total"] = self.monitor.gpu_memory_total
         return result
@@ -3849,7 +3845,10 @@ async def run(server, comm, function, args=(), kwargs=None, is_coro=None, wait=T
 
 try:
     from .diagnostics import nvml
-except Exception:
+
+    if nvml.device_get_count() < 1:
+        raise RuntimeError
+except (Exception, RuntimeError):
     pass
 else:
 

From 0a54d95cb26f70427b80b180b5ba7a67b2553a26 Mon Sep 17 00:00:00 2001
From: "Mads R. B. Kristensen" <madsbk@gmail.com>
Date: Fri, 4 Jun 2021 18:33:04 +0200
Subject: [PATCH 1298/1550] Multiple worker executors (#4869)

---
 distributed/scheduler.py         |  2 +
 distributed/tests/test_worker.py | 24 +++++++++
 distributed/worker.py            | 88 +++++++++++++++++++++++---------
 3 files changed, 89 insertions(+), 25 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index de3843c17f4..863e05bc1c1 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -7366,6 +7366,8 @@ def _task_to_msg(state: SchedulerState, ts: TaskState, duration: double = -1) ->
     else:
         msg["task"] = task
 
+    if ts._annotations:
+        msg["annotations"] = ts._annotations
     return msg
 
 
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index f50c73e990a..05672364ea8 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -1997,3 +1997,27 @@ def get_worker_client_id():
 
         with pytest.raises(ValueError):
             default_client()
+
+
+@pytest.mark.asyncio
+async def test_multiple_executors(cleanup):
+    def get_thread_name():
+        return threading.current_thread().name
+
+    async with Scheduler() as s:
+        async with Worker(
+            s.address,
+            nthreads=2,
+            executor={
+                "GPU": ThreadPoolExecutor(1, thread_name_prefix="Dask-GPU-Threads")
+            },
+        ) as w:
+            async with Client(s.address, asynchronous=True) as c:
+                futures = []
+                with dask.annotate(executor="default"):
+                    futures.append(c.submit(get_thread_name, pure=False))
+                with dask.annotate(executor="GPU"):
+                    futures.append(c.submit(get_thread_name, pure=False))
+                default_result, gpu_result = await c.gather(futures)
+                assert "Dask-Default-Threads" in default_result
+                assert "Dask-GPU-Threads" in gpu_result
diff --git a/distributed/worker.py b/distributed/worker.py
index 15e76544a8a..8e803c594a6 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -1,5 +1,6 @@
 import asyncio
 import bisect
+import concurrent.futures
 import errno
 import heapq
 import logging
@@ -16,7 +17,7 @@
 from functools import partial
 from inspect import isawaitable
 from pickle import PicklingError
-from typing import Iterable
+from typing import Dict, Iterable
 
 from tlz import first, keymap, merge, pluck  # noqa: F401
 from tornado import gen
@@ -28,7 +29,7 @@
 from dask.system import CPU_COUNT
 from dask.utils import format_bytes, funcname
 
-from . import comm, preloading, profile, system
+from . import comm, preloading, profile, system, utils
 from .batched import BatchedSend
 from .comm import connect, get_address_host
 from .comm.addressing import address_from_user_args
@@ -150,6 +151,8 @@ class TaskState:
         serializable (e.g. int, string, list, dict).
     * **nbytes**: ``int``
         The size of a particular piece of data
+    * **annotations**: ``dict``
+        Task annotations
 
     Parameters
     ----------
@@ -184,6 +187,7 @@ def __init__(self, key, runspec=None):
         self.stop_time = None
         self.metadata = {}
         self.nbytes = None
+        self.annotations = None
 
     def __repr__(self):
         return "<Task %r %s>" % (self.key, self.state)
@@ -223,11 +227,9 @@ class Worker(ServerNode):
 
     * **nthreads:** ``int``:
         Number of nthreads used by this worker process
-    * **executor:** ``concurrent.futures.ThreadPoolExecutor``:
-        Executor used to perform computation
-        This can also be the string "offload" in which case this uses the same
-        thread pool used for offloading communications.  This results in the
-        same thread being used for deserialization and computation.
+    * **executors:** ``Dict[str, concurrent.futures.Executor]``:
+        Executors used to perform computation. Always contains the default
+        executor.
     * **local_directory:** ``path``:
         Path on local machine to store temporary files
     * **scheduler:** ``rpc``:
@@ -324,7 +326,15 @@ class Worker(ServerNode):
         Fraction of memory at which we start spilling to disk
     memory_pause_fraction: float
         Fraction of memory at which we stop running new tasks
-    executor: concurrent.futures.Executor
+    executor: concurrent.futures.Executor, dict[str, concurrent.futures.Executor], str
+        The executor(s) to use. Depending on the type, it has the following meanings:
+            - Executor instance: The default executor.
+            - Dict[str, Executor]: mapping names to Executor instances. If the
+              "default" key isn't in the dict, a "default" executor will be created
+              using ``ThreadPoolExecutor(nthreads)``.
+            - Str: The string "offload", which refer to the same thread pool used for
+              offloading communications. This results in the same thread being used
+              for deserialization and computation.
     resources: dict
         Resources that this worker has like ``{'GPU': 2}``
     nanny: str
@@ -626,14 +636,25 @@ def __init__(
         self.actors = {}
         self.loop = loop or IOLoop.current()
         self.reconnect = reconnect
+
+        # Common executors always available
+        self.executors: Dict[str, concurrent.futures.Executor] = {
+            "offload": utils._offload_executor,
+            "actor": ThreadPoolExecutor(1, thread_name_prefix="Dask-Actor-Threads"),
+        }
+
+        # Find the default executor
         if executor == "offload":
-            from distributed.utils import _offload_executor as executor
-        self.executor = executor or ThreadPoolExecutor(
-            self.nthreads, thread_name_prefix="Dask-Worker-Threads'"
-        )
-        self.actor_executor = ThreadPoolExecutor(
-            1, thread_name_prefix="Dask-Actor-Threads"
-        )
+            self.executors["default"] = self.executors["offload"]
+        elif isinstance(executor, dict):
+            self.executors.update(executor)
+        elif executor is not None:
+            self.executors["default"] = executor
+        if "default" not in self.executors:
+            self.executors["default"] = ThreadPoolExecutor(
+                self.nthreads, thread_name_prefix="Dask-Default-Threads"
+            )
+
         self.batched_stream = BatchedSend(interval="2ms", loop=self.loop)
         self.name = name
         self.scheduler_delay = 0
@@ -808,6 +829,10 @@ def local_dir(self):
         )
         return self.local_directory
 
+    @property
+    def executor(self):
+        return self.executors["default"]
+
     async def get_metrics(self):
         out = dict(
             executing=self.executing_count,
@@ -1280,13 +1305,14 @@ async def close(
                 with suppress(TimeoutError):
                     await self.batched_stream.close(timedelta(seconds=timeout))
 
-            self.actor_executor._work_queue.queue.clear()
-            if isinstance(self.executor, ThreadPoolExecutor):
-                self.executor._work_queue.queue.clear()
-                self.executor.shutdown(wait=executor_wait, timeout=timeout)
-            else:
-                self.executor.shutdown(wait=False)
-            self.actor_executor.shutdown(wait=executor_wait, timeout=timeout)
+            for executor in self.executors.values():
+                if executor is utils._offload_executor:
+                    continue  # Never shutdown the offload executor
+                if isinstance(executor, ThreadPoolExecutor):
+                    executor._work_queue.queue.clear()
+                    executor.shutdown(wait=executor_wait, timeout=timeout)
+                else:
+                    executor.shutdown(wait=executor_wait)
 
             self.stop()
             await self.rpc.close()
@@ -1499,6 +1525,7 @@ def add_task(
         duration=None,
         resource_restrictions=None,
         actor=False,
+        annotations=None,
         **kwargs2,
     ):
         try:
@@ -1545,6 +1572,7 @@ def add_task(
             ts.duration = duration
             if resource_restrictions:
                 ts.resource_restrictions = resource_restrictions
+            ts.annotations = annotations
 
             who_has = who_has or {}
 
@@ -2560,7 +2588,7 @@ def executor_submit(self, key, function, args=(), kwargs=None, executor=None):
         callbacks to ensure things run smoothly.  This can get tricky, so we
         pull it off into an separate method.
         """
-        executor = executor or self.executor
+        executor = executor or self.executors["default"]
         job_counter[0] += 1
         # logger.info("%s:%d Starts job %d, %s", self.ip, self.port, i, key)
         kwargs = kwargs or {}
@@ -2656,7 +2684,7 @@ async def actor_execute(
                     self.active_threads,
                     self.active_threads_lock,
                 ),
-                executor=self.actor_executor,
+                executor=self.executors["actor"],
             )
         else:
             result = func(*args, **kwargs)
@@ -2790,8 +2818,17 @@ async def execute(self, key, report=False):
                 if self.digests is not None:
                     self.digests["disk-load-duration"].add(stop - start)
 
+            if ts.annotations is not None and "executor" in ts.annotations:
+                executor = ts.annotations["executor"]
+            else:
+                executor = "default"
+            assert executor in self.executors
+
             logger.debug(
-                "Execute key: %s worker: %s", ts.key, self.address
+                "Execute key: %s worker: %s, executor: %s",
+                ts.key,
+                self.address,
+                executor,
             )  # TODO: comment out?
             assert key == ts.key
             try:
@@ -2808,6 +2845,7 @@ async def execute(self, key, report=False):
                         self.active_threads_lock,
                         self.scheduler_delay,
                     ),
+                    executor=self.executors[executor],
                 )
             except RuntimeError as e:
                 executor_error = e

From 611923e2d718350b8e8443d0d94c853c18b91574 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Fri, 4 Jun 2021 14:47:26 -0500
Subject: [PATCH 1299/1550] bump version to 2021.06.0

---
 docs/source/changelog.rst | 15 +++++++++++++++
 requirements.txt          |  2 +-
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index e1d4d91e323..8759f49665d 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,21 @@
 Changelog
 =========
 
+2021.06.0
+---------
+
+Released on June 4, 2021
+
+- Multiple worker executors (:pr:`4869`) `Mads R. B. Kristensen`_
+- Ensure PyNVML works correctly when installed with no GPUs (:pr:`4873`) `Peter Andreas Entschev`_
+- Show more in test summary (:pr:`4875`) `James Bourbeau`_
+- Move ``SystemMonitor`` s GPU initialization back to constructor (:pr:`4866`) `Peter Andreas Entschev`_
+- Mark ``test_server_comms_mark_active_handlers`` with ``pytest.mark.asyncio`` (:pr:`4876`) `James Bourbeau`_
+- Who has has what html reprs v2 (:pr:`4865`) `Jacob Tomlinson`_
+- O(1) rebalance (:pr:`4774`) `crusaderky`_
+- Ensure repr and eq for cluster always works (:pr:`4799`) `Florian Jetter`_
+
+
 2021.05.1
 ---------
 
diff --git a/requirements.txt b/requirements.txt
index 986230439bb..d91f3c4e4b2 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,6 @@
 click >= 6.6
 cloudpickle >= 1.5.0
-dask == 2021.05.1
+dask == 2021.06.0
 msgpack >= 0.6.0
 psutil >= 5.0
 sortedcontainers !=2.0.0, !=2.0.1

From 2c8684701eb3e19a48ef1ef7125ed4f9ccd9d682 Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Mon, 7 Jun 2021 11:26:07 +0200
Subject: [PATCH 1300/1550] No longer cancel result future in async process
 when using timeouts (#4882)

---
 distributed/process.py                 | 10 +++++-----
 distributed/tests/test_asyncprocess.py |  3 +--
 2 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/distributed/process.py b/distributed/process.py
index 862ea4edeb8..1540bf3752c 100644
--- a/distributed/process.py
+++ b/distributed/process.py
@@ -6,7 +6,6 @@
 import weakref
 from queue import Queue as PyQueue
 
-from tornado import gen
 from tornado.concurrent import Future
 from tornado.ioloop import IOLoop
 
@@ -268,8 +267,7 @@ def terminate(self):
         self._watch_q.put_nowait({"op": "terminate", "future": fut})
         return fut
 
-    @gen.coroutine
-    def join(self, timeout=None):
+    async def join(self, timeout=None):
         """
         Wait for the child process to exit.
 
@@ -280,10 +278,12 @@ def join(self, timeout=None):
         if self._state.exitcode is not None:
             return
         if timeout is None:
-            yield self._exit_future
+            await self._exit_future
         else:
             try:
-                yield asyncio.wait_for(self._exit_future, timeout)
+                # Shield otherwise the timeout cancels the future and our
+                # on_exit callback will try to set a result on a canceled future
+                await asyncio.wait_for(asyncio.shield(self._exit_future), timeout)
             except TimeoutError:
                 pass
 
diff --git a/distributed/tests/test_asyncprocess.py b/distributed/tests/test_asyncprocess.py
index 32c4e5c5589..26c695933bb 100644
--- a/distributed/tests/test_asyncprocess.py
+++ b/distributed/tests/test_asyncprocess.py
@@ -49,7 +49,6 @@ def threads_info(q):
     q.put(threading.current_thread().name)
 
 
-@pytest.mark.xfail()
 @nodebug
 @gen_test()
 async def test_simple():
@@ -362,7 +361,7 @@ async def parent_process_coroutine():
 
     with pristine_loop() as loop:
         try:
-            loop.run_sync(parent_process_coroutine(), timeout=10)
+            loop.run_sync(parent_process_coroutine, timeout=10)
         finally:
             loop.stop()
 

From 8d890164611b53b0eb24776219979ed5e45f1533 Mon Sep 17 00:00:00 2001
From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com>
Date: Tue, 8 Jun 2021 15:27:44 -0400
Subject: [PATCH 1301/1550] Add `range_query` tests to NVML test suite (#4879)

---
 distributed/diagnostics/tests/test_nvml.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/distributed/diagnostics/tests/test_nvml.py b/distributed/diagnostics/tests/test_nvml.py
index 587fde19263..4a57ffa8317 100644
--- a/distributed/diagnostics/tests/test_nvml.py
+++ b/distributed/diagnostics/tests/test_nvml.py
@@ -66,7 +66,7 @@ async def test_gpu_metrics(s, a, b):
 
 
 @gen_cluster()
-async def test_gpu_monitoring(s, a, b):
+async def test_gpu_monitoring_recent(s, a, b):
     if nvml.device_get_count() < 1:
         pytest.skip("No GPUs available")
 
@@ -83,3 +83,16 @@ async def test_gpu_monitoring(s, a, b):
     )
     assert res[a.address]["gpu_name"] == pynvml.nvmlDeviceGetName(h).decode()
     assert res[a.address]["gpu_memory_total"] == pynvml.nvmlDeviceGetMemoryInfo(h).total
+
+
+@gen_cluster()
+async def test_gpu_monitoring_range_query(s, a, b):
+    if nvml.device_get_count() < 1:
+        pytest.skip("No GPUs available")
+
+    res = await s.get_worker_monitor_info()
+    ms = ["gpu_utilization", "gpu_memory_used"]
+    for w in (a, b):
+        assert all(res[w.address]["range_query"][m] is not None for m in ms)
+        assert res[w.address]["count"] is not None
+        assert res[w.address]["last_time"] is not None

From 0fd26ca91f38a0d7593a9aee5c2c1cad042e0558 Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Tue, 8 Jun 2021 20:33:45 +0100
Subject: [PATCH 1302/1550] Add more documention on memory management (#4874)

---
 distributed/client.py    |  11 +-
 distributed/scheduler.py |  20 ++--
 distributed/worker.py    |  13 ++-
 docs/source/memory.rst   | 125 +++++++++++++---------
 docs/source/worker.rst   | 223 ++++++++++++++++++++++++++++++++++-----
 5 files changed, 303 insertions(+), 89 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index ca8f87ffaea..e5a5c5ea9b0 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -530,7 +530,7 @@ class Client:
     timeout: int
         Timeout duration for initial connection to the scheduler
     set_as_default: bool (True)
-        Claim this scheduler as the global dask scheduler
+        Use this Client as the global dask scheduler
     scheduler_file: string (optional)
         Path to a file with scheduler information if available
     security: Security or bool, optional
@@ -3077,9 +3077,12 @@ def rebalance(self, futures=None, workers=None, **kwargs):
         either affects a subset of the keys/workers or the entire network,
         depending on keyword arguments.
 
-        This operation is generally not well tested against normal operation of
-        the scheduler.  It is not recommended to use it while waiting on
-        computations.
+        For details on the algorithm and configuration options, refer to the matching
+        scheduler-side method :meth:`~distributed.scheduler.Scheduler.rebalance`.
+
+        .. warning::
+           This operation is generally not well tested against normal operation of the
+           scheduler. It is not recommended to use it while waiting on computations.
 
         Parameters
         ----------
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 863e05bc1c1..9913d0b069f 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -5545,7 +5545,9 @@ async def rebalance(
         """Rebalance keys so that each worker ends up with roughly the same process
         memory (managed+unmanaged).
 
-        FIXME this method is not robust when the cluster is not idle.
+        .. warning::
+           This operation is generally not well tested against normal operation of the
+           scheduler. It is not recommended to use it while waiting on computations.
 
         **Algorithm**
 
@@ -5553,13 +5555,19 @@ async def rebalance(
            unmanaged process memory that has been there for at least 30 seconds
            (``distributed.worker.memory.recent-to-old-time``).
            This lets us ignore temporary spikes caused by task heap usage.
+
+           Alternatively, you may change how memory is measured both for the individual
+           workers as well as to calculate the mean through
+           ``distributed.worker.memory.rebalance.measure``. Namely, this can be useful
+           to disregard inaccurate OS memory measurements.
+
         #. Discard workers whose occupancy is within 5% of the mean cluster occupancy
            (``distributed.worker.memory.rebalance.sender-recipient-gap`` / 2).
            This helps avoid data from bouncing around the cluster repeatedly.
         #. Workers above the mean are senders; those below are recipients.
-        #. Discard senders whose absolute occupancy is below 40%
+        #. Discard senders whose absolute occupancy is below 30%
            (``distributed.worker.memory.rebalance.sender-min``). In other words, no data
-           is moved regardless of imbalancing as long as all workers are below 40%.
+           is moved regardless of imbalancing as long as all workers are below 30%.
         #. Discard recipients whose absolute occupancy is above 60%
            (``distributed.worker.memory.rebalance.recipient-max``).
            Note that this threshold by default is the same as
@@ -5571,8 +5579,8 @@ async def rebalance(
 
            A recipient will be skipped if it already has a copy of the data. In other
            words, this method does not degrade replication.
-           A key will be skipped if there are no recipients that have both enough memory
-           to accept and don't already hold a copy.
+           A key will be skipped if there are no recipients available with enough memory
+           to accept the key and that don't already hold a copy.
 
         The least recently insertd (LRI) policy is a greedy choice with the advantage of
         being O(1), trivial to implement (it relies on python dict insertion-sorting)
@@ -5581,7 +5589,7 @@ async def rebalance(
         - Largest first. O(n*log(n)) save for non-trivial additional data structures and
           risks causing the largest chunks of data to repeatedly move around the
           cluster like pinballs.
-        - Least recently utilized. This information is currently available on the
+        - Least recently used (LRU). This information is currently available on the
           workers only and not trivial to replicate on the scheduler; transmitting it
           over the network would be very expensive. Also, note that dask will go out of
           its way to minimise the amount of time intermediate keys are held in memory,
diff --git a/distributed/worker.py b/distributed/worker.py
index 8e803c594a6..6f8a22b566a 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -2980,14 +2980,13 @@ async def check_pause(memory):
             while memory > target:
                 if not self.data.fast:
                     logger.warning(
-                        "Memory use is high but worker has no data "
-                        "to store to disk.  Perhaps some other process "
-                        "is leaking memory?  Process memory: %s -- "
-                        "Worker memory limit: %s",
+                        "Unmanaged memory use is high. This may indicate a memory leak "
+                        "or the memory may not be released to the OS; see "
+                        "https://distributed.dask.org/en/latest/worker.html#memtrim "
+                        "for more information. "
+                        "-- Unmanaged memory: %s -- Worker memory limit: %s",
                         format_bytes(memory),
-                        format_bytes(self.memory_limit)
-                        if self.memory_limit is not None
-                        else "None",
+                        format_bytes(self.memory_limit),
                     )
                     break
                 k, v, weight = self.data.fast.evict()
diff --git a/docs/source/memory.rst b/docs/source/memory.rst
index 849b71ec616..6b8e76d7d71 100644
--- a/docs/source/memory.rst
+++ b/docs/source/memory.rst
@@ -2,30 +2,31 @@ Managing Memory
 ===============
 
 Dask.distributed stores the results of tasks in the distributed memory of the
-worker nodes.  The central scheduler tracks all data on the cluster and
-determines when data should be freed.  Completed results are usually cleared
+worker nodes. The central scheduler tracks all data on the cluster and
+determines when data should be freed. Completed results are usually cleared
 from memory as quickly as possible in order to make room for more computation.
 The result of a task is kept in memory if either of the following conditions
 hold:
 
-1.  A client holds a future pointing to this task.  The data should stay in RAM
-    so that the client can gather the data on demand.
-2.  The task is necessary for ongoing computations that are working to produce
-    the final results pointed to by futures.  These tasks will be removed once
-    no ongoing tasks require them.
+1. A client holds a future pointing to this task. The data should stay in RAM so that
+   the client can gather the data on demand.
+2. The task is necessary for ongoing computations that are working to produce
+   the final results pointed to by futures. These tasks will be removed once
+   no ongoing tasks require them.
+
+When users hold Future objects or persisted collections (which contain many such Futures
+inside their dask graph, typically accessible through their ``.dask`` attribute) they
+pin those results to active memory. When the user deletes futures or collections from
+their local Python process, the scheduler removes the associated data from distributed
+RAM. Because of this relationship, distributed memory reflects the state of local
+memory. A user may free distributed memory on the cluster by deleting persisted
+collections in the local session.
 
-When users hold Future objects or persisted collections (which contain many
-such Futures inside their ``.dask`` attribute) they pin those results to active
-memory.  When the user deletes futures or collections from their local Python
-process the scheduler removes the associated data from distributed RAM.
-Because of this relationship, distributed memory reflects the state of local
-memory.  A user may free distributed memory on the cluster by deleting
-persisted collections in the local session.
 
 Creating Futures
 ----------------
 
-The following functions produce Futures
+The following functions produce Futures:
 
 .. currentmodule:: distributed.client
 
@@ -36,10 +37,10 @@ The following functions produce Futures
    Client.persist
    Client.scatter
 
-The ``submit`` and ``map`` methods handle raw Python functions.  The
-``compute`` and ``persist`` methods handle Dask collections like arrays, bags,
-delayed values, and dataframes.  The ``scatter`` method sends data directly
-from the local process.
+The ``submit`` and ``map`` methods handle raw Python functions. The ``compute`` and
+``persist`` methods handle Dask collections like arrays, bags, delayed values, and
+dataframes. The ``scatter`` method sends data directly from the local process.
+
 
 Persisting Collections
 ----------------------
@@ -47,8 +48,8 @@ Persisting Collections
 Calls to ``Client.compute`` or ``Client.persist`` submit task graphs to the
 cluster and return ``Future`` objects that point to particular output tasks.
 
-Compute returns a single future per input, persist returns a copy of the
-collection with each block or partition replaced by a single future.  In short,
+Compute returns a single future per input; persist returns a copy of the
+collection with each block or partition replaced by a single future. In short,
 use ``persist`` to keep full collection on the cluster and use ``compute`` when
 you want a small result as a single future.
 
@@ -56,25 +57,25 @@ Persist is more common and is often used as follows with collections:
 
 .. code-block:: python
 
-   >>> # Construct dataframe, no work happens
+   >>> # Construct dataframe; no work happens
    >>> df = dd.read_csv(...)
    >>> df = df[df.x > 0]
    >>> df = df.assign(z = df.x + df.y)
 
-   >>> # Pin data in distributed ram, this triggers computation
+   >>> # Pin data in distributed RAM; this triggers computation
    >>> df = client.persist(df)
 
    >>> # continue operating on df
 
-*Note for Spark users: this differs from what you're accustomed to.  Persist is
-an immediate action.  However, you'll get control back immediately as
+*Note for Spark users: this differs from what you're accustomed to. Persist is
+an immediate action. However, you'll get control back immediately as
 computation occurs in the background.*
 
 In this example we build a computation by parsing CSV data, filtering rows, and
-then adding a new column.  Up until this point all work is lazy; we've just
+then adding a new column. Up until this point all work is lazy; we've just
 built up a recipe to perform the work as a graph in the ``df`` object.
 
-When we call ``df = client.persist(df)`` we cut this graph off of the ``df`` object,
+When we call ``df = client.persist(df)``, we cut the graph off the ``df`` object,
 send it up to the scheduler, receive ``Future`` objects in return and create a
 new dataframe with a very shallow graph that points directly to these futures.
 This happens more or less immediately (as long as it takes to serialize and
@@ -84,14 +85,20 @@ cluster works to evaluate the graph in the background.
 
 Difference with dask.compute
 ----------------------------
+If a Client is set as the default scheduler, then ``dask.compute``, ``dask.persist``,
+and the ``.compute`` and ``.persist`` methods of all dask collections will invoke
+``Client.compute`` and ``Client.persist`` under the hood, unless a different scheduler
+is explicitly specified. This happens by default whenever a new Client is created,
+unless the user explicitly passes the ``set_as_default=False`` parameter to it.
+
+There is however a difference: the operation ``client.compute(df)`` is asynchronous and
+so differs from the traditional ``df.compute()`` method or ``dask.compute`` function,
+which block until a result is available, do not persist any data on the cluster, and
+bring the entire result back to the local machine, so it is unwise to use them on large
+datasets, but can be very convenient for smaller results, particularly because they
+return concrete results in a way that most other tools expect.
 
-The operations ``client.persist(df)`` and ``client.compute(df)`` are asynchronous and so differ
-from the traditional ``df.compute()`` method or ``dask.compute`` function, which
-blocks until a result is available.  The ``.compute()`` method does not persist
-any data on the cluster.  The ``.compute()`` method also brings the entire
-result back to the local machine, so it is unwise to use it on large datasets.
-However, ``.compute()`` is very convenient for smaller results particularly
-because it does return concrete results in a way that most other tools expect.
+In other words, ``df.compute()`` is equivalent to ``client.compute(df).result()``.
 
 Typically we use asynchronous methods like ``client.persist`` to set up large
 collections and then use ``df.compute()`` for fast analyses.
@@ -107,8 +114,8 @@ collections and then use ``df.compute()`` for fast analyses.
 Clearing data
 -------------
 
-We remove data from distributed ram by removing the collection from our local
-process.  Remote data is removed once all Futures pointing to that data are
+We remove data from distributed RAM by removing the collection from our local
+process. Remote data is removed once all Futures pointing to that data are
 removed from all client machines.
 
 .. code-block:: python
@@ -118,7 +125,7 @@ removed from all client machines.
 If this is the only copy then this will likely trigger the cluster to delete
 the data as well.
 
-However if we have multiple copies or other collections based on this one then
+However, if we have multiple copies or other collections based on this one, then
 we'll have to delete them all.
 
 .. code-block:: python
@@ -126,6 +133,7 @@ we'll have to delete them all.
    >>> df2 = df[df.x < 10]
    >>> del df  # would not delete data, because df2 still tracks the futures
 
+
 Aggressively Clearing Data
 --------------------------
 
@@ -136,22 +144,38 @@ can always ``cancel`` the futures/collection.
 
    >>> client.cancel(df)  # kills df, df2, and every other dependent computation
 
-Alternatively, if you want a clean slate, you can restart the cluster.  This
-clears all state and does a hard restart of all worker processes.  It generally
-completes in around a second.
+Alternatively, if you want a clean slate, you can restart the cluster. This
+clears all state and does a hard restart of all worker processes. It generally
+completes in a few seconds.
 
 .. code-block:: python
 
    >>> client.restart()
 
 
+Client references
+-----------------
+Futures live on the cluster for as long as at least one Client holds a reference to
+them. When the last Client holding a reference is shut down or crashes, then everything
+that was referenced exclusively by it is pruned from the cluster. This is generally
+desirable to prevent unclean client shutdowns from polluting the memory of long-running
+clusters.
+
+It is possible to prevent this behaviour in order to improve resilience or simply to be
+able to shut down one's laptop while a computation runs overnight. See:
+
+.. autosummary::
+   distributed.Client.publish_dataset
+   distributed.fire_and_forget
+
+
 Resilience
 ----------
 
 Results are not intentionally copied unless necessary for computations on other
-worker nodes.  Resilience is achieved through recomputation by maintaining the
-provenance of any result.  If a worker node goes down the scheduler is able to
-recompute all of its results.  The complete graph for any desired Future is
+worker nodes. Resilience is achieved through recomputation by maintaining the
+provenance of any result. If a worker node goes down, the scheduler is able to
+recompute all of its results. The complete graph for any desired Future is
 maintained until no references to that future exist.
 
 For more information see :doc:`Resilience <resilience>`.
@@ -161,19 +185,19 @@ Advanced techniques
 -------------------
 
 At first the result of a task is not intentionally copied, but only persists on
-the node where it was originally computed or scattered.  However result may be
+the node where it was originally computed or scattered. However, a result may be
 copied to another worker node in the course of normal computation if that
 result is required by another task that is intended to by run by a different
-worker.  This occurs if a task requires two pieces of data on different
-machines (at least one must move) or through work stealing.  In these cases it
+worker. This occurs if a task requires two pieces of data on different
+machines (at least one must move) or through work stealing. In these cases it
 is the policy for the second machine to maintain its redundant copy of the data.
 This helps to organically spread around data that is in high demand.
 
 However, advanced users may want to control the location, replication, and
-balancing of data more directly throughout the cluster.  They may know ahead of
+balancing of data more directly throughout the cluster. They may know ahead of
 time that certain data should be broadcast throughout the network or that their
 data has become particularly imbalanced, or that they want certain pieces of
-data to live on certain parts of their network.  These considerations are not
+data to live on certain parts of their network. These considerations are not
 usually necessary.
 
 .. currentmodule:: distributed.client
@@ -182,3 +206,8 @@ usually necessary.
    Client.rebalance
    Client.replicate
    Client.scatter
+
+
+Worker memory management
+------------------------
+Memory usage can be optimized by configuring worker-side :ref:`memman`.
diff --git a/docs/source/worker.rst b/docs/source/worker.rst
index eaf547a8321..20b4ab0067e 100644
--- a/docs/source/worker.rst
+++ b/docs/source/worker.rst
@@ -27,6 +27,7 @@ look like the following::
    Alice -> Bob:        Sure.  x is 3!
    Bob -> Scheduler:    I've computed y and am holding on to it!
 
+
 Storing Data
 ------------
 
@@ -52,7 +53,7 @@ Thread Pool
 -----------
 
 Each worker sends computations to a thread in a
-`concurrent.futures.ThreadPoolExecutor <https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.ThreadPoolExecutor>`_
+:class:`concurrent.futures.ThreadPoolExecutor`
 for computation.  These computations occur in the same process as the Worker
 communication server so that they can access and share data efficiently between
 each other.  For the purposes of data locality all threads within a worker are
@@ -78,12 +79,15 @@ will not be able to communicate to other workers or to the scheduler.  This
 situation should be avoided.  If you don't link in your own custom C/Fortran
 code then this topic probably doesn't apply.
 
+
 Command Line tool
 -----------------
 
 Use the ``dask-worker`` command line tool to start an individual worker. For
 more details on the command line options, please have a look at the
-`command line tools documentation <https://docs.dask.org/en/latest/setup/cli.html#dask-worker>`_.
+`command line tools documentation
+<https://docs.dask.org/en/latest/setup/cli.html#dask-worker>`_.
+
 
 Internal Scheduling
 -------------------
@@ -140,6 +144,7 @@ exceptions to this are when:
   previously assigned to a separate worker to a new worker.  This most commonly
   occurs when a `worker dies <killed>`_ during computation.
 
+
 .. _memman:
 
 Memory Management
@@ -151,17 +156,21 @@ keyword argument, which sets the memory limit per worker processes launched
 by dask-worker ::
 
     $ dask-worker tcp://scheduler:port --memory-limit=auto  # TOTAL_MEMORY * min(1, nthreads / total_nthreads)
-    $ dask-worker tcp://scheduler:port --memory-limit=4e9  # four gigabytes per worker process.
+    $ dask-worker tcp://scheduler:port --memory-limit="4 GiB"  # four gigabytes per worker process.
 
 Workers use a few different heuristics to keep memory use beneath this limit:
 
-1.  At 60% of memory load (as estimated by ``sizeof``), spill least recently used data to disk
-2.  At 70% of memory load, spill least recently used data to disk regardless of
-    what is reported by ``sizeof``
-3.  At 80% of memory load, stop accepting new work on local thread pool
-4.  At 95% of memory load, terminate and restart the worker
+1.  At 60% of memory load (as estimated by ``sizeof``), spill least recently used data
+    to disk
+2.  At 70% of memory load (as reported by the OS), spill least recently used data to
+    disk regardless of what is reported by ``sizeof``; this accounts for memory used by
+    the python interpreter, modules, global variables, memory leaks, etc.
+3.  At 80% of memory load (as reported by the OS), stop accepting new work on local
+    thread pool
+4.  At 95% of memory load (as reported by the OS), terminate and restart the worker
 
-These values can be configured by modifying the ``~/.config/dask/distributed.yaml`` file
+These values can be configured by modifying the ``~/.config/dask/distributed.yaml``
+file:
 
 .. code-block:: yaml
 
@@ -176,34 +185,32 @@ These values can be configured by modifying the ``~/.config/dask/distributed.yam
          terminate: 0.95  # fraction at which we terminate the worker
 
 
-Spill data to Disk
+Spill data to disk
 ~~~~~~~~~~~~~~~~~~
 
 Every time the worker finishes a task it estimates the size in bytes that the
 result costs to keep in memory using the ``sizeof`` function.  This function
-defaults to ``sys.getsizeof`` for arbitrary objects which uses the standard
-Python `__sizeof__ protocol
-<https://docs.python.org/3/library/sys.html#sys.getsizeof>`_, but also has
-special-cased implementations for common data types like NumPy arrays and
-Pandas dataframes.
+defaults to :func:`sys.getsizeof` for arbitrary objects, which uses the standard
+Python ``__sizeof__`` protocol, but also has special-cased implementations for
+common data types like NumPy arrays and Pandas dataframes.
 
 When the sum of the number of bytes of the data in memory exceeds 60% of the
-available threshold the worker will begin to dump the least recently used data
+memory limit, the worker will begin to dump the least recently used data
 to disk.  You can control this location with the ``--local-directory``
 keyword.::
 
-   $ dask-worker tcp://scheduler:port --memory-limit 4e9 --local-directory /scratch
+   $ dask-worker tcp://scheduler:port --memory-limit="4 GiB" --local-directory /scratch
 
 That data is still available and will be read back from disk when necessary.
 On the diagnostic dashboard status page disk I/O will show up in the task
-stream plot as orange blocks.  Additionally the memory plot in the upper left
-will become orange and then red.
+stream plot as orange blocks. Additionally, the memory plot in the upper left
+will become yellow and then red.
 
 
 Monitor process memory load
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-The approach above can fail for a few reasons
+The approach above can fail for a few reasons:
 
 1.  Custom objects may not report their memory size accurately
 2.  User functions may take up more RAM than expected
@@ -218,7 +225,7 @@ usage then the worker will start dumping unused data to disk, even if internal
 Halt worker threads
 ~~~~~~~~~~~~~~~~~~~
 
-At 80% load the worker's thread pool will stop accepting new tasks.  This
+At 80% load, the worker's thread pool will stop accepting new tasks.  This
 gives time for the write-to-disk functionality to take effect even in the face
 of rapidly accumulating data.
 
@@ -226,14 +233,174 @@ of rapidly accumulating data.
 Kill Worker
 ~~~~~~~~~~~
 
-At 95% memory load a worker's nanny process will terminate it.  This is to
+At 95% memory load, a worker's nanny process will terminate it. This is to
 avoid having our worker job being terminated by an external job scheduler (like
-YARN, Mesos, SGE, etc..).  After termination the nanny will restart the worker
+YARN, Mesos, SGE, etc..).  After termination, the nanny will restart the worker
 in a fresh state.
 
 
+Using the dashboard to monitor memory usage
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+The dashboard (typically available on port 8787) shows a summary of the overall memory
+usage on the cluster, as well as the individual usage on each worker. It provides
+different memory readings:
+
+process
+    Overall memory used by the worker process (RSS), as measured by the OS
+
+managed
+    This is the sum of the ``sizeof`` of all dask data stored on the worker, excluding
+    spilled data.
+
+unmanaged
+    This is the memory usage that dask is not directly aware of. It is estimated by
+    subtracting managed memory from the total process memory and typically includes:
+
+    - The Python interpreter code, loaded modules, and global variables
+    - Memory temporarily used by running tasks
+    - Dereferenced Python objects that have not been garbage-collected yet
+    - Unused memory that the Python memory allocator did not return to libc through
+      `free`_ yet
+    - Unused memory that the user-space libc `free`_ function did not release to the OS
+      yet (see memory allocators below)
+    - Memory fragmentation
+    - Memory leaks
+
+unmanaged recent
+    Unmanaged memory that has appeared within the last 30 seconds. This is not included
+    in the 'unmanaged' memory measure above. Ideally, this memory should be for the most
+    part a temporary spike caused by tasks' heap use plus soon-to-be garbage collected
+    objects.
+
+    The time it takes for unmanaged memory to transition away from its "recent" state
+    can be tweaked through the ``distributed.worker.memory.recent-to-old-time`` key in
+    the ``~/.config/dask/distributed.yaml`` file. If your tasks typically run for longer
+    than 30 seconds, it's recommended that you increase this setting accordingly.
+
+    By default, :meth:`distributed.Client.rebalance` and
+    :meth:`distributed.scheduler.Scheduler.rebalance` ignore unmanaged recent memory.
+    This behaviour can also be tweaked using the dask config - see the methods'
+    documentation.
+
+spilled
+    managed memory that has been spilled to disk. This is not included in the 'managed'
+    measure above.
+
+The sum of managed + unmanaged + unmanaged recent is equal by definition to the process
+memory.
+
+
+.. _memtrim:
+
+Memory not released back to the OS
+----------------------------------
+In many cases, high unmanaged memory usage or "memory leak" warnings on workers can be
+misleading: a worker may not actually be using its memory for anything, but simply
+hasn't returned that unused memory back to the operating system, and is hoarding it just
+in case it needs the memory capacity again. This is not a bug in your code, nor in
+Dask — it's actually normal behavior for all processes on Linux and MacOS, and is a
+consequence of how the low-level memory allocator works (see below for details).
+
+Because Dask makes decisions (spill-to-disk, pause, terminate,
+:meth:`~distributed.Client.rebalance`) based on the worker's memory usage as reported by
+the OS, and is unaware of how much of this memory is actually in use versus empty and
+"hoarded", it can overestimate — sometimes significantly — how much memory the process
+is using and think the worker is running out of memory when in fact it isn't.
+
+More in detail: both the Linux and MacOS memory allocators try to avoid performing a
+`brk`_ kernel call every time the application calls `free`_ by implementing a user-space
+memory management system. Upon `free`_, memory can remain allocated in user space and
+potentially reusable by the next `malloc`_ - which in turn won't require a kernel call
+either. This is generally very desirable for C/C++ applications which have no memory
+allocator of their own, as it can drastically boost performance at the cost of a larger
+memory footprint. CPython however adds its own memory allocator on top, which reduces
+the need for this additional abstraction (with caveats).
+
+There are steps you can take to alleviate situations where worker memory is not released
+back to the OS. These steps are discussed in the following sections.
+
+Manually trim memory
+~~~~~~~~~~~~~~~~~~~~
+*Linux workers only*
+
+It is possible to forcefully release allocated but unutilized memory as follows:
+
+.. code-block:: python
+
+    import ctypes
+
+    def trim_memory() -> int:
+        libc = ctypes.CDLL("libc.so.6")
+        return libc.malloc_trim(0)
+
+    client.run(trim_memory)
+
+This should be only used as a one-off debugging experiment. Watch the dashboard while
+running the above code. If unmanaged worker memory (on the "Bytes stored" plot)
+decreases significantly after calling ``client.run(trim_memory)``, then move on to the
+next section. Otherwise, you likely do have a memory leak.
+
+Note that you should only run this `malloc_trim`_ if you are using the default glibc
+memory allocator. When using a custom allocator such as `jemalloc`_ (see below), this
+could cause unexpected behavior including segfaults. (If you don't know what this means,
+you're probably using the default glibc allocator and are safe to run this).
+
+Automatically trim memory
+~~~~~~~~~~~~~~~~~~~~~~~~~
+*Linux workers only*
+
+To aggressively and automatically trim the memory in a production environment, you
+should instead set the environment variable ``MALLOC_TRIM_THRESHOLD_`` (note the final
+underscore) to 0 or a low number; see the `mallopt`_ man page for details. Reducing
+this value will increase the number of syscalls, and as a consequence may degrade
+performance. **The variable must be set before starting the ``dask-worker`` process.**
+
+jemalloc
+~~~~~~~~
+*Linux and MacOS workers*
+
+Alternatively to the above, you may experiment with the `jemalloc`_ memory allocator, as
+follows:
+
+On Linux:
+
+.. code-block:: bash
+
+    conda install jemalloc
+    LD_PRELOAD=$CONDA_PREFIX/lib/libjemalloc.so dask-worker <...>
+
+On MacOS:
+
+.. code-block:: bash
+
+    conda install jemalloc
+    DYLD_INSERT_LIBRARIES=$CONDA_PREFIX/lib/libjemalloc.dylib dask-worker <...>
+
+`jemalloc`_ offers a wealth of configuration settings; please refer to its
+documentation.
+
+Ignore process memory
+~~~~~~~~~~~~~~~~~~~~~
+If all else fails, you may want to stop dask from using memory metrics from the OS (RSS)
+in its decision-making:
+
+.. code-block:: yaml
+
+   distributed:
+     worker:
+       memory:
+         rebalance:
+           measure: managed_in_memory
+         spill: false
+         pause: false
+         terminate: false
+
+This of course will be problematic if you have a genuine issue with unmanaged memory,
+e.g. memory leaks and/or suffer from heavy fragmentation.
+
+
 Nanny
-~~~~~
+-----
 
 Dask workers are by default launched, monitored, and managed by a small Nanny
 process.
@@ -246,3 +413,11 @@ API Documentation
 
 .. autoclass:: distributed.worker.TaskState
 .. autoclass:: distributed.worker.Worker
+
+
+.. _malloc: https://www.man7.org/linux/man-pages/man3/malloc.3.html
+.. _free: https://www.man7.org/linux/man-pages/man3/free.3.html
+.. _mallopt: https://man7.org/linux/man-pages/man3/mallopt.3.html
+.. _malloc_trim: https://man7.org/linux/man-pages/man3/malloc_trim.3.html
+.. _brk: https://www.man7.org/linux/man-pages/man2/brk.2.html
+.. _jemalloc: http://jemalloc.net

From 64ac3616ee886e53467f2ae74d522a8f3a0cb6a3 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Tue, 8 Jun 2021 15:59:21 -0500
Subject: [PATCH 1303/1550] Don't strip scheduler protocol when determining
 host (#4883)

---
 distributed/comm/addressing.py   |  4 +---
 distributed/tests/test_worker.py | 21 +++++++++++++++++++++
 distributed/worker.py            |  2 +-
 3 files changed, 23 insertions(+), 4 deletions(-)

diff --git a/distributed/comm/addressing.py b/distributed/comm/addressing.py
index 8e454cfd877..3612b284d0d 100644
--- a/distributed/comm/addressing.py
+++ b/distributed/comm/addressing.py
@@ -5,8 +5,6 @@
 from ..utils import get_ip_interface
 from . import registry
 
-DEFAULT_SCHEME = dask.config.get("distributed.comm.default-scheme")
-
 
 def parse_address(addr, strict=False):
     """
@@ -28,7 +26,7 @@ def parse_address(addr, strict=False):
         )
         raise ValueError(msg)
     if not sep:
-        scheme = DEFAULT_SCHEME
+        scheme = dask.config.get("distributed.comm.default-scheme")
     return scheme, loc
 
 
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 05672364ea8..633325ef6d4 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -29,6 +29,8 @@
     get_worker,
     wait,
 )
+from distributed.comm.registry import backends
+from distributed.comm.tcp import TCPBackend
 from distributed.compatibility import MACOS, WINDOWS
 from distributed.core import CommClosedError, Status, rpc
 from distributed.diagnostics.plugin import PipInstall
@@ -1547,6 +1549,25 @@ async def test_protocol_from_scheduler_address(Worker):
                 assert info["address"].startswith("ucx://")
 
 
+@pytest.mark.asyncio
+async def test_host_uses_scheduler_protocol(cleanup, monkeypatch):
+    # Ensure worker uses scheduler's protocol to determine host address, not the default scheme
+    # See https://github.com/dask/distributed/pull/4883
+
+    class BadBackend(TCPBackend):
+        def get_address_host(self, loc):
+            raise ValueError("asdf")
+
+    monkeypatch.setitem(backends, "foo", BadBackend())
+
+    with dask.config.set({"distributed.comm.default-scheme": "foo"}):
+        async with Scheduler(protocol="tcp") as s:
+            async with Worker(s.address) as w:
+                # Ensure that worker is able to properly start up
+                # without BadBackend.get_address_host raising a ValueError
+                pass
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize("Worker", [Worker, Nanny])
 async def test_worker_listens_on_same_interface_by_default(Worker):
diff --git a/distributed/worker.py b/distributed/worker.py
index 6f8a22b566a..11bdc69c648 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -530,7 +530,7 @@ def __init__(
         # Target interface on which we contact the scheduler by default
         # TODO: it is unfortunate that we special-case inproc here
         if not host and not interface and not scheduler_addr.startswith("inproc://"):
-            host = get_ip(get_address_host(scheduler_addr.split("://")[-1]))
+            host = get_ip(get_address_host(scheduler_addr))
 
         self._start_port = port
         self._start_host = host

From ee06a44cc4b43270b5ae6ee102481f3bbc9daf27 Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Wed, 9 Jun 2021 00:40:40 +0200
Subject: [PATCH 1304/1550] Flake8 config cleanup (#4888)

---
 conftest.py                                   |  3 +
 distributed/cli/tests/test_dask_scheduler.py  | 25 ++++----
 distributed/cli/tests/test_dask_spec.py       |  1 -
 distributed/cli/tests/test_dask_worker.py     |  8 +--
 distributed/cli/tests/test_tls_cli.py         |  1 -
 distributed/cli/utils.py                      |  2 +-
 distributed/client.py                         |  3 +-
 distributed/comm/core.py                      |  2 +-
 distributed/comm/tests/test_comms.py          |  1 -
 distributed/comm/tests/test_ucx.py            |  2 +-
 distributed/comm/tests/test_ucx_config.py     |  2 +-
 distributed/comm/tests/test_ws.py             |  3 +-
 distributed/core.py                           |  6 +-
 distributed/deploy/adaptive_core.py           |  2 +-
 distributed/deploy/spec.py                    |  1 -
 distributed/deploy/tests/test_adaptive.py     | 10 +---
 distributed/deploy/tests/test_cluster.py      |  1 -
 distributed/deploy/tests/test_local.py        |  4 +-
 distributed/deploy/tests/test_old_ssh.py      |  1 -
 .../deploy/tests/test_slow_adaptive.py        |  2 +-
 distributed/deploy/tests/test_spec_cluster.py |  1 -
 distributed/deploy/tests/test_ssh.py          |  1 -
 distributed/deploy/utils_test.py              |  2 +-
 distributed/diagnostics/graph_layout.py       |  4 +-
 .../diagnostics/tests/test_progressbar.py     | 13 +---
 .../tests/test_scheduler_plugin.py            |  2 +-
 .../diagnostics/tests/test_task_stream.py     | 10 +---
 distributed/diagnostics/tests/test_widgets.py | 11 +---
 distributed/diskutils.py                      |  2 +-
 distributed/nanny.py                          |  5 +-
 distributed/profile.py                        |  5 +-
 distributed/protocol/pickle.py                |  2 +-
 distributed/protocol/scipy.py                 |  1 -
 distributed/protocol/serialize.py             |  4 +-
 distributed/protocol/tests/test_serialize.py  |  2 -
 distributed/pytest_resourceleaks.py           |  4 +-
 distributed/scheduler.py                      | 14 ++---
 distributed/tests/test_actor.py               |  8 +--
 distributed/tests/test_as_completed.py        |  9 +--
 distributed/tests/test_client.py              | 59 ++++++++-----------
 distributed/tests/test_client_executor.py     |  8 +--
 distributed/tests/test_collections.py         |  7 +--
 distributed/tests/test_core.py                |  1 -
 distributed/tests/test_counter.py             |  1 -
 distributed/tests/test_events.py              |  7 +--
 distributed/tests/test_failed_workers.py      |  1 -
 distributed/tests/test_ipython.py             |  2 +-
 distributed/tests/test_locks.py               |  7 +--
 distributed/tests/test_multi_locks.py         |  7 +--
 distributed/tests/test_nanny.py               |  3 +-
 distributed/tests/test_preload.py             |  2 +-
 distributed/tests/test_publish.py             |  8 +--
 distributed/tests/test_queues.py              | 10 +---
 distributed/tests/test_resources.py           | 53 +++++++----------
 distributed/tests/test_scheduler.py           |  7 +--
 distributed/tests/test_semaphore.py           | 12 +---
 distributed/tests/test_steal.py               | 32 +++++-----
 distributed/tests/test_stress.py              |  4 +-
 distributed/tests/test_tls_functional.py      |  3 +-
 distributed/tests/test_utils.py               | 11 +---
 distributed/tests/test_utils_comm.py          |  2 +-
 distributed/tests/test_utils_test.py          |  9 +--
 distributed/tests/test_variable.py            | 11 +---
 distributed/tests/test_versions.py            |  2 +-
 distributed/tests/test_worker.py              |  9 +--
 distributed/tests/test_worker_client.py       |  9 +--
 distributed/utils.py                          |  7 ++-
 distributed/utils_test.py                     | 10 ++--
 distributed/worker.py                         |  6 +-
 setup.cfg                                     | 25 ++++----
 70 files changed, 159 insertions(+), 356 deletions(-)

diff --git a/conftest.py b/conftest.py
index e8a159fa4a3..ebb9dae1457 100644
--- a/conftest.py
+++ b/conftest.py
@@ -16,6 +16,9 @@
     except Exception:
         pass
 
+# Make all fixtures available
+from distributed.utils_test import *  # noqa
+
 
 def pytest_addoption(parser):
     parser.addoption("--runslow", action="store_true", help="run slow tests")
diff --git a/distributed/cli/tests/test_dask_scheduler.py b/distributed/cli/tests/test_dask_scheduler.py
index a407a2b8fe1..da3990a949a 100644
--- a/distributed/cli/tests/test_dask_scheduler.py
+++ b/distributed/cli/tests/test_dask_scheduler.py
@@ -16,7 +16,6 @@
 from distributed import Client, Scheduler
 from distributed.metrics import time
 from distributed.utils import get_ip, get_ip_interface, tmpfile
-from distributed.utils_test import loop  # noqa: F401
 from distributed.utils_test import (
     assert_can_connect_from_everywhere_4_6,
     assert_can_connect_locally_4,
@@ -144,17 +143,6 @@ def test_dashboard_whitelist(loop):
                 assert time() < start + 20
 
 
-def test_multiple_workers(loop):
-    with popen(["dask-scheduler", "--no-dashboard"]) as s:
-        with popen(["dask-worker", "localhost:8786", "--no-dashboard"]) as a:
-            with popen(["dask-worker", "localhost:8786", "--no-dashboard"]) as b:
-                with Client("127.0.0.1:%d" % Scheduler.default_port, loop=loop) as c:
-                    start = time()
-                    while len(c.nthreads()) < 2:
-                        sleep(0.1)
-                        assert time() < start + 10
-
-
 def test_interface(loop):
     psutil = pytest.importorskip("psutil")
     if_names = sorted(psutil.net_if_addrs())
@@ -418,7 +406,7 @@ def test_idle_timeout(loop):
     assert 1 < stop - start < 10
 
 
-def test_multiple_workers(loop):
+def test_multiple_workers_2(loop):
     text = """
 def dask_setup(worker):
     worker.foo = 'setup'
@@ -441,3 +429,14 @@ def dask_setup(worker):
                 assert foo == "setup"
                 [foo] = c.run(lambda dask_worker: dask_worker.foo, nanny=True).values()
                 assert foo == "setup"
+
+
+def test_multiple_workers(loop):
+    with popen(["dask-scheduler", "--no-dashboard"]) as s:
+        with popen(["dask-worker", "localhost:8786", "--no-dashboard"]) as a:
+            with popen(["dask-worker", "localhost:8786", "--no-dashboard"]) as b:
+                with Client("127.0.0.1:%d" % Scheduler.default_port, loop=loop) as c:
+                    start = time()
+                    while len(c.nthreads()) < 2:
+                        sleep(0.1)
+                        assert time() < start + 10
diff --git a/distributed/cli/tests/test_dask_spec.py b/distributed/cli/tests/test_dask_spec.py
index 65a0f4f7297..d90fccf954b 100644
--- a/distributed/cli/tests/test_dask_spec.py
+++ b/distributed/cli/tests/test_dask_spec.py
@@ -5,7 +5,6 @@
 
 from distributed import Client
 from distributed.scheduler import COMPILED
-from distributed.utils_test import cleanup  # noqa: F401
 from distributed.utils_test import popen
 
 
diff --git a/distributed/cli/tests/test_dask_worker.py b/distributed/cli/tests/test_dask_worker.py
index 98724c6754b..5615c7ff558 100644
--- a/distributed/cli/tests/test_dask_worker.py
+++ b/distributed/cli/tests/test_dask_worker.py
@@ -17,13 +17,7 @@
 from distributed.deploy.utils import nprocesses_nthreads
 from distributed.metrics import time
 from distributed.utils import parse_ports, sync, tmpfile
-from distributed.utils_test import (  # noqa: F401
-    cleanup,
-    loop,
-    popen,
-    terminate_process,
-    wait_for_port,
-)
+from distributed.utils_test import popen, terminate_process, wait_for_port
 
 
 def test_nanny_worker_ports(loop):
diff --git a/distributed/cli/tests/test_tls_cli.py b/distributed/cli/tests/test_tls_cli.py
index 9301b47ab3e..f88d961bfbe 100644
--- a/distributed/cli/tests/test_tls_cli.py
+++ b/distributed/cli/tests/test_tls_cli.py
@@ -2,7 +2,6 @@
 
 from distributed import Client
 from distributed.metrics import time
-from distributed.utils_test import loop  # noqa: F401
 from distributed.utils_test import (
     get_cert,
     new_config_file,
diff --git a/distributed/cli/utils.py b/distributed/cli/utils.py
index 7e812cf7f01..e1f8a19feb1 100644
--- a/distributed/cli/utils.py
+++ b/distributed/cli/utils.py
@@ -42,7 +42,7 @@ def check_python_3():
             _unicodefun._verify_python3_env()
         else:
             _unicodefun._verify_python_env()
-    except (TypeError, RuntimeError) as e:
+    except (TypeError, RuntimeError):
         import click
 
         click.echo(py3_err_msg, err=True)
diff --git a/distributed/client.py b/distributed/client.py
index e5a5c5ea9b0..582aa242c49 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -1127,7 +1127,7 @@ async def _ensure_connected(self, timeout=None):
                     "versions": version_module.get_versions(),
                 }
             )
-        except Exception as e:
+        except Exception:
             if self.status == "closed":
                 return
             else:
@@ -4000,7 +4000,6 @@ async def _register_worker_plugin(self, plugin=None, name=None):
         for response in responses.values():
             if response["status"] == "error":
                 exc = response["exception"]
-                typ = type(exc)
                 tb = response["traceback"]
                 raise exc.with_traceback(tb)
         return responses
diff --git a/distributed/comm/core.py b/distributed/comm/core.py
index d9bb4bfb6ef..b9032a7369e 100644
--- a/distributed/comm/core.py
+++ b/distributed/comm/core.py
@@ -219,7 +219,7 @@ async def on_connection(self, comm: Comm, handshake_overrides=None):
             # Timeout is to ensure that we'll terminate connections eventually.
             # Connector side will employ smaller timeouts and we should only
             # reach this if the comm is dead anyhow.
-            write = await asyncio.wait_for(comm.write(local_info), timeout=timeout)
+            await asyncio.wait_for(comm.write(local_info), timeout=timeout)
             handshake = await asyncio.wait_for(comm.read(), timeout=timeout)
             # This would be better, but connections leak if worker is closed quickly
             # write, handshake = await asyncio.gather(comm.write(local_info), comm.read())
diff --git a/distributed/comm/tests/test_comms.py b/distributed/comm/tests/test_comms.py
index 8b78f7f37d4..eef6ce478de 100644
--- a/distributed/comm/tests/test_comms.py
+++ b/distributed/comm/tests/test_comms.py
@@ -33,7 +33,6 @@
 from distributed.metrics import time
 from distributed.protocol import Serialized, deserialize, serialize, to_serialize
 from distributed.utils import get_ip, get_ipv6
-from distributed.utils_test import loop  # noqa: F401
 from distributed.utils_test import (
     get_cert,
     get_client_ssl_context,
diff --git a/distributed/comm/tests/test_ucx.py b/distributed/comm/tests/test_ucx.py
index 2a388904475..5daaf7e8693 100644
--- a/distributed/comm/tests/test_ucx.py
+++ b/distributed/comm/tests/test_ucx.py
@@ -9,7 +9,7 @@
 from distributed.comm.registry import backends, get_backend
 from distributed.deploy.local import LocalCluster
 from distributed.protocol import to_serialize
-from distributed.utils_test import cleanup, gen_test, inc, loop, popen  # noqa: 401
+from distributed.utils_test import gen_test, inc
 
 try:
     HOST = ucp.get_address()
diff --git a/distributed/comm/tests/test_ucx_config.py b/distributed/comm/tests/test_ucx_config.py
index 891793ae972..d9eeabbe3ed 100644
--- a/distributed/comm/tests/test_ucx_config.py
+++ b/distributed/comm/tests/test_ucx_config.py
@@ -8,7 +8,7 @@
 from distributed import Client
 from distributed.comm.ucx import _scrub_ucx_config
 from distributed.utils import get_ip
-from distributed.utils_test import cleanup, gen_test, inc, loop, popen  # noqa: 401
+from distributed.utils_test import popen
 
 try:
     HOST = get_ip()
diff --git a/distributed/comm/tests/test_ws.py b/distributed/comm/tests/test_ws.py
index a7e305f718a..c5e98abb8f2 100644
--- a/distributed/comm/tests/test_ws.py
+++ b/distributed/comm/tests/test_ws.py
@@ -12,8 +12,7 @@
 from distributed.comm.core import FatalCommClosedError
 from distributed.comm.registry import backends, get_backend
 from distributed.security import Security
-from distributed.utils_test import (  # noqa: F401
-    cleanup,
+from distributed.utils_test import (
     gen_cluster,
     get_client_ssl_context,
     get_server_ssl_context,
diff --git a/distributed/core.py b/distributed/core.py
index 4608462de76..1c95f6bd019 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -547,7 +547,6 @@ async def handle_stream(self, comm, extra=None, every_cycle=[]):
         extra = extra or {}
         logger.info("Starting established connection")
 
-        io_error = None
         closed = False
         try:
             while not closed:
@@ -580,8 +579,9 @@ async def handle_stream(self, comm, extra=None, every_cycle=[]):
                     else:
                         func()
 
-        except (CommClosedError, EnvironmentError) as e:
-            io_error = e
+        except (CommClosedError, EnvironmentError):
+            # FIXME: This is silently ignored, is this intentional?
+            pass
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
diff --git a/distributed/deploy/adaptive_core.py b/distributed/deploy/adaptive_core.py
index e11a07bf528..db8ef77e9ad 100644
--- a/distributed/deploy/adaptive_core.py
+++ b/distributed/deploy/adaptive_core.py
@@ -217,7 +217,7 @@ async def adapt(self) -> None:
                 await self.scale_up(**recommendations)
             if status == "down":
                 await self.scale_down(**recommendations)
-        except OSError as e:
+        except OSError:
             if status != "down":
                 logger.error("Adaptive stopping due to error", exc_info=True)
                 self.stop()
diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index 8ea233beab7..070d6d0624d 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -334,7 +334,6 @@ async def _correct_state_internal(self):
         async with self._lock:
             self._correct_state_waiting = None
 
-            pre = list(set(self.workers))
             to_close = set(self.workers) - set(self.worker_spec)
             if to_close:
                 if self.scheduler.status == Status.running:
diff --git a/distributed/deploy/tests/test_adaptive.py b/distributed/deploy/tests/test_adaptive.py
index 434eeea8cf8..30496a7d233 100644
--- a/distributed/deploy/tests/test_adaptive.py
+++ b/distributed/deploy/tests/test_adaptive.py
@@ -9,15 +9,7 @@
 
 from distributed import Adaptive, Client, LocalCluster, SpecCluster, Worker, wait
 from distributed.metrics import time
-from distributed.utils_test import (  # noqa: F401
-    async_wait_for,
-    clean,
-    cleanup,
-    gen_test,
-    loop,
-    nodebug,
-    slowinc,
-)
+from distributed.utils_test import async_wait_for, clean, gen_test, slowinc
 
 
 def test_adaptive_local_cluster(loop):
diff --git a/distributed/deploy/tests/test_cluster.py b/distributed/deploy/tests/test_cluster.py
index eee8a4139ac..db64f67e4f7 100644
--- a/distributed/deploy/tests/test_cluster.py
+++ b/distributed/deploy/tests/test_cluster.py
@@ -1,7 +1,6 @@
 import pytest
 
 from distributed.deploy.cluster import Cluster
-from distributed.utils_test import cleanup  # noqa: F401
 
 
 @pytest.mark.asyncio
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index d12c8bfc131..ee1b12913dc 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -24,17 +24,15 @@
 from distributed.scheduler import COMPILED
 from distributed.system import MEMORY_LIMIT
 from distributed.utils import TimeoutError, sync
-from distributed.utils_test import (  # noqa: F401
+from distributed.utils_test import (
     assert_can_connect_from_everywhere_4,
     assert_can_connect_from_everywhere_4_6,
     assert_can_connect_locally_4,
     assert_cannot_connect,
     captured_logger,
     clean,
-    cleanup,
     gen_test,
     inc,
-    loop,
     slowinc,
     tls_only_security,
 )
diff --git a/distributed/deploy/tests/test_old_ssh.py b/distributed/deploy/tests/test_old_ssh.py
index 7f3aaeadbbf..61791b11af3 100644
--- a/distributed/deploy/tests/test_old_ssh.py
+++ b/distributed/deploy/tests/test_old_ssh.py
@@ -7,7 +7,6 @@
 from distributed import Client
 from distributed.deploy.old_ssh import SSHCluster
 from distributed.metrics import time
-from distributed.utils_test import loop  # noqa: F401
 
 
 @pytest.mark.avoid_ci
diff --git a/distributed/deploy/tests/test_slow_adaptive.py b/distributed/deploy/tests/test_slow_adaptive.py
index 8a8919c7eea..ac3721d9a2b 100644
--- a/distributed/deploy/tests/test_slow_adaptive.py
+++ b/distributed/deploy/tests/test_slow_adaptive.py
@@ -5,7 +5,7 @@
 from dask.distributed import Client, Scheduler, SpecCluster, Worker
 
 from distributed.metrics import time
-from distributed.utils_test import cleanup, slowinc  # noqa: F401
+from distributed.utils_test import slowinc
 
 
 class SlowWorker:
diff --git a/distributed/deploy/tests/test_spec_cluster.py b/distributed/deploy/tests/test_spec_cluster.py
index c1798fd580b..40928ee96e2 100644
--- a/distributed/deploy/tests/test_spec_cluster.py
+++ b/distributed/deploy/tests/test_spec_cluster.py
@@ -14,7 +14,6 @@
 from distributed.deploy.spec import ProcessInterface, close_clusters, run_spec
 from distributed.metrics import time
 from distributed.utils import is_valid_xml
-from distributed.utils_test import cleanup, loop  # noqa: F401
 
 
 class MyWorker(Worker):
diff --git a/distributed/deploy/tests/test_ssh.py b/distributed/deploy/tests/test_ssh.py
index 6c330383d7d..16cb100b8ff 100644
--- a/distributed/deploy/tests/test_ssh.py
+++ b/distributed/deploy/tests/test_ssh.py
@@ -9,7 +9,6 @@
 from distributed import Client
 from distributed.compatibility import MACOS, WINDOWS
 from distributed.deploy.ssh import SSHCluster
-from distributed.utils_test import loop  # noqa: F401
 
 pytestmark = [
     pytest.mark.xfail(MACOS, reason="very high flakiness; see distributed/issues/4543"),
diff --git a/distributed/deploy/utils_test.py b/distributed/deploy/utils_test.py
index 543020ec21b..09b2a3d1799 100644
--- a/distributed/deploy/utils_test.py
+++ b/distributed/deploy/utils_test.py
@@ -17,7 +17,7 @@ def tearDown(self):
 
     @pytest.mark.xfail()
     def test_cores(self):
-        info = self.client.scheduler_info()
+        self.client.scheduler_info()
         assert len(self.client.nthreads()) == 2
 
     def test_submit(self):
diff --git a/distributed/diagnostics/graph_layout.py b/distributed/diagnostics/graph_layout.py
index 4404aa6e111..c971b2a870d 100644
--- a/distributed/diagnostics/graph_layout.py
+++ b/distributed/diagnostics/graph_layout.py
@@ -99,9 +99,7 @@ def transition(self, key, start, finish, *args, **kwargs):
             task = self.scheduler.tasks[key]
             for dep in task.dependents:
                 edge = (key, dep.key)
-                self.visible_edge_updates.append(
-                    (self.index_edge.pop((key, dep.key)), "False")
-                )
+                self.visible_edge_updates.append((self.index_edge.pop(edge), "False"))
             for dep in task.dependencies:
                 self.visible_edge_updates.append(
                     (self.index_edge.pop((dep.key, key)), "False")
diff --git a/distributed/diagnostics/tests/test_progressbar.py b/distributed/diagnostics/tests/test_progressbar.py
index 36db0d7b45b..d358c9a494a 100644
--- a/distributed/diagnostics/tests/test_progressbar.py
+++ b/distributed/diagnostics/tests/test_progressbar.py
@@ -5,14 +5,7 @@
 from distributed import Scheduler, Worker
 from distributed.diagnostics.progressbar import TextProgressBar, progress
 from distributed.metrics import time
-from distributed.utils_test import (  # noqa: F401
-    client,
-    cluster_fixture,
-    div,
-    gen_cluster,
-    inc,
-    loop,
-)
+from distributed.utils_test import div, gen_cluster, inc
 
 
 def test_text_progressbar(capsys, client):
@@ -49,8 +42,8 @@ async def test_TextProgressBar_error(c, s, a, b):
 @pytest.mark.asyncio
 async def test_TextProgressBar_empty(capsys):
     async with Scheduler(port=0) as s:
-        async with Worker(s.address, nthreads=1) as a:
-            async with Worker(s.address, nthreads=1) as b:
+        async with Worker(s.address, nthreads=1):
+            async with Worker(s.address, nthreads=1):
                 progress = TextProgressBar(
                     [], scheduler=s.address, start=False, interval=0.01
                 )
diff --git a/distributed/diagnostics/tests/test_scheduler_plugin.py b/distributed/diagnostics/tests/test_scheduler_plugin.py
index 465b674971f..b48f0b2ae7d 100644
--- a/distributed/diagnostics/tests/test_scheduler_plugin.py
+++ b/distributed/diagnostics/tests/test_scheduler_plugin.py
@@ -1,7 +1,7 @@
 import pytest
 
 from distributed import Scheduler, SchedulerPlugin, Worker
-from distributed.utils_test import cleanup, gen_cluster, inc  # noqa: F401
+from distributed.utils_test import gen_cluster, inc
 
 
 @gen_cluster(client=True)
diff --git a/distributed/diagnostics/tests/test_task_stream.py b/distributed/diagnostics/tests/test_task_stream.py
index 642277e81dc..cdee6b0fc96 100644
--- a/distributed/diagnostics/tests/test_task_stream.py
+++ b/distributed/diagnostics/tests/test_task_stream.py
@@ -8,15 +8,7 @@
 from distributed.client import wait
 from distributed.diagnostics.task_stream import TaskStreamPlugin
 from distributed.metrics import time
-from distributed.utils_test import (  # noqa: F401
-    client,
-    cluster_fixture,
-    div,
-    gen_cluster,
-    inc,
-    loop,
-    slowinc,
-)
+from distributed.utils_test import div, gen_cluster, inc, slowinc
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 3)
diff --git a/distributed/diagnostics/tests/test_widgets.py b/distributed/diagnostics/tests/test_widgets.py
index bc86e436b24..f78dfc8af5d 100644
--- a/distributed/diagnostics/tests/test_widgets.py
+++ b/distributed/diagnostics/tests/test_widgets.py
@@ -84,16 +84,7 @@ def record_display(*args):
     ProgressWidget,
     progress,
 )
-from distributed.utils_test import (  # noqa: F401
-    client,
-    cluster_fixture,
-    dec,
-    gen_cluster,
-    gen_tls_cluster,
-    inc,
-    loop,
-    throws,
-)
+from distributed.utils_test import dec, gen_cluster, gen_tls_cluster, inc, throws
 from distributed.worker import dumps_task
 
 
diff --git a/distributed/diskutils.py b/distributed/diskutils.py
index 49d0a26222c..fe393872531 100644
--- a/distributed/diskutils.py
+++ b/distributed/diskutils.py
@@ -54,7 +54,7 @@ def __init__(self, workspace, name=None, prefix=None):
                     with workspace._global_lock():
                         self._lock_file = locket.lock_file(self._lock_path)
                         self._lock_file.acquire()
-                except OSError as e:
+                except OSError:
                     logger.exception(
                         "Could not acquire workspace lock on "
                         "path: %s ."
diff --git a/distributed/nanny.py b/distributed/nanny.py
index 3d9fb1855e0..bf51a5c9699 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -22,7 +22,6 @@
 from .comm import get_address_host, unparse_host_port
 from .comm.addressing import address_from_user_args
 from .core import CommClosedError, RPCClosed, Status, coerce_to_address
-from .metrics import time
 from .node import ServerNode
 from .process import AsyncProcess
 from .proctitle import enable_proctitle_on_children
@@ -386,8 +385,6 @@ async def instantiate(self, comm=None) -> Status:
         return result
 
     async def restart(self, comm=None, timeout=2, executor_wait=True):
-        start = time()
-
         async def _():
             if self.process is not None:
                 await self.kill()
@@ -405,7 +402,7 @@ async def _():
     def _psutil_process(self):
         pid = self.process.process.pid
         try:
-            proc = self._psutil_process_obj
+            self._psutil_process_obj
         except AttributeError:
             self._psutil_process_obj = psutil.Process(pid)
 
diff --git a/distributed/profile.py b/distributed/profile.py
index 3f3230c437b..9c375291d3e 100644
--- a/distributed/profile.py
+++ b/distributed/profile.py
@@ -208,8 +208,6 @@ def traverse(state, start, stop, height):
         line_numbers.append(desc["line_number"])
         names.append(desc["name"])
 
-        ident = state["identifier"]
-
         try:
             fn = desc["filename"]
         except IndexError:
@@ -224,7 +222,7 @@ def traverse(state, start, stop, height):
 
         x = start
 
-        for name, child in state["children"].items():
+        for _, child in state["children"].items():
             width = child["count"] * delta
             traverse(child, x, x + width, height + 1)
             x += width
@@ -335,7 +333,6 @@ def get_profile(history, recent=None, start=None, stop=None, key=None):
     start : time
     stop : time
     """
-    now = time()
     if start is None:
         istart = 0
     else:
diff --git a/distributed/protocol/pickle.py b/distributed/protocol/pickle.py
index 843f6a28d9a..f856696d251 100644
--- a/distributed/protocol/pickle.py
+++ b/distributed/protocol/pickle.py
@@ -73,6 +73,6 @@ def loads(x, *, buffers=()):
             return pickle.loads(x, buffers=buffers)
         else:
             return pickle.loads(x)
-    except Exception as e:
+    except Exception:
         logger.info("Failed to deserialize %s", x[:10000], exc_info=True)
         raise
diff --git a/distributed/protocol/scipy.py b/distributed/protocol/scipy.py
index 9ed533bc850..0ad0adc0a28 100644
--- a/distributed/protocol/scipy.py
+++ b/distributed/protocol/scipy.py
@@ -10,7 +10,6 @@
 
 @dask_serialize.register(scipy.sparse.dok.dok_matrix)
 def serialize_scipy_sparse_dok(x):
-    x_coo = x.tocoo()
     coo_header, coo_frames = dask_serialize(x.tocoo())
 
     header = {"coo_header": coo_header}
diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index 63ced42a473..c88d36a8995 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -324,14 +324,14 @@ def serialize(
     tb = ""
 
     for name in serializers:
-        dumps, loads, wants_context = families[name]
+        dumps, _, wants_context = families[name]
         try:
             header, frames = dumps(x, context=context) if wants_context else dumps(x)
             header["serializer"] = name
             return header, frames
         except NotImplementedError:
             continue
-        except Exception as e:
+        except Exception:
             tb = traceback.format_exc()
             break
 
diff --git a/distributed/protocol/tests/test_serialize.py b/distributed/protocol/tests/test_serialize.py
index a72724a1136..d946b01496b 100644
--- a/distributed/protocol/tests/test_serialize.py
+++ b/distributed/protocol/tests/test_serialize.py
@@ -11,8 +11,6 @@
 except ImportError:
     np = None
 
-from dask.utils_test import inc
-
 from distributed import Nanny, wait
 from distributed.comm.utils import from_frames, to_frames
 from distributed.protocol import (
diff --git a/distributed/pytest_resourceleaks.py b/distributed/pytest_resourceleaks.py
index 6d9959ac46e..185f649761c 100644
--- a/distributed/pytest_resourceleaks.py
+++ b/distributed/pytest_resourceleaks.py
@@ -345,7 +345,7 @@ def run_test_again():
             from _pytest.runner import runtestprotocol
 
             item._initrequest()  # Re-init fixtures
-            reports = runtestprotocol(item, nextitem=nextitem, log=False)
+            runtestprotocol(item, nextitem=nextitem, log=False)
 
         nodeid = item.nodeid
         leaks = self.leaks.get(nodeid)
@@ -354,7 +354,7 @@ def run_test_again():
             try:
                 for i in range(self.max_retries):
                     run_test_again()
-            except Exception as e:
+            except Exception:
                 print("--- Exception when re-running test ---")
                 import traceback
 
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 9913d0b069f..f5ad740b523 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -2158,7 +2158,7 @@ def _transition(self, key, finish: str, *args, **kwargs):
                     del parent._task_groups[tg._name]
 
             return recommendations, client_msgs, worker_msgs
-        except Exception as e:
+        except Exception:
             logger.exception("Error transitioning %r from %r to %r", key, start, finish)
             if LOG_PDB:
                 import pdb
@@ -3817,14 +3817,10 @@ async def close_worker(self, comm=None, worker=None, safe=None):
         signal to the worker to shut down.  This works regardless of whether or
         not the worker has a nanny process restarting it
         """
-        parent: SchedulerState = cast(SchedulerState, self)
         logger.info("Closing worker %s", worker)
         with log_errors():
             self.log_event(worker, {"action": "close-worker"})
-            ws: WorkerState = parent._workers_dv[worker]
-            nanny_addr = ws._nanny
-            address = nanny_addr or worker
-
+            # FIXME: This does not handly nannys
             self.worker_send(worker, {"op": "close", "report": False})
             await self.remove_worker(address=worker, safe=safe)
 
@@ -5405,7 +5401,7 @@ async def restart(self, client=None, timeout=3):
                     # Ask the worker to close if it doesn't have a nanny,
                     # otherwise the nanny will kill it anyway
                     await self.remove_worker(address=addr, close=addr not in nannies)
-                except Exception as e:
+                except Exception:
                     logger.info(
                         "Exception while restarting.  This is normal", exc_info=True
                     )
@@ -6547,7 +6543,7 @@ def set_metadata(self, comm=None, keys=None, value=None):
                     metadata[key] = dict()
                 metadata = metadata[key]
             metadata[keys[-1]] = value
-        except Exception as e:
+        except Exception:
             import pdb
 
             pdb.set_trace()
@@ -6612,7 +6608,7 @@ async def register_worker_plugin(self, comm, plugin, name=None):
     async def unregister_worker_plugin(self, comm, name):
         """Unregisters a worker plugin"""
         try:
-            worker_plugins = self.worker_plugins.pop(name)
+            self.worker_plugins.pop(name)
         except KeyError:
             raise ValueError(f"The worker plugin {name} does not exists")
 
diff --git a/distributed/tests/test_actor.py b/distributed/tests/test_actor.py
index e639769ae17..d2298c1c2c2 100644
--- a/distributed/tests/test_actor.py
+++ b/distributed/tests/test_actor.py
@@ -8,13 +8,7 @@
 
 from distributed import Actor, ActorFuture, Client, Future, Nanny, wait
 from distributed.metrics import time
-from distributed.utils_test import (  # noqa: F401
-    client,
-    cluster,
-    cluster_fixture,
-    gen_cluster,
-    loop,
-)
+from distributed.utils_test import cluster, gen_cluster
 
 
 class Counter:
diff --git a/distributed/tests/test_as_completed.py b/distributed/tests/test_as_completed.py
index 1002259391b..370563e8359 100644
--- a/distributed/tests/test_as_completed.py
+++ b/distributed/tests/test_as_completed.py
@@ -10,14 +10,7 @@
 from distributed.client import _as_completed, _first_completed, as_completed, wait
 from distributed.metrics import time
 from distributed.utils import CancelledError
-from distributed.utils_test import (  # noqa: F401
-    client,
-    cluster_fixture,
-    gen_cluster,
-    inc,
-    loop,
-    throws,
-)
+from distributed.utils_test import gen_cluster, inc, throws
 
 
 @gen_cluster(client=True)
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 4b86ae05c81..afb303171ed 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -67,20 +67,12 @@
 )
 from distributed.sizeof import sizeof
 from distributed.utils import is_valid_xml, mp_context, sync, tmp_text, tmpfile
-from distributed.utils_test import (  # noqa: F401
+from distributed.utils_test import (
     TaskStateMetadataPlugin,
-    a,
     async_wait_for,
     asyncinc,
-    b,
     captured_logger,
-    cleanup,
-)
-from distributed.utils_test import client as c  # noqa: F401
-from distributed.utils_test import client_secondary as c2  # noqa: F401
-from distributed.utils_test import (  # noqa: F401
     cluster,
-    cluster_fixture,
     dec,
     div,
     double,
@@ -88,14 +80,11 @@
     gen_test,
     geninc,
     inc,
-    loop,
-    loop_in_thread,
     map_varying,
     nodebug,
     popen,
     pristine_loop,
     randominc,
-    s,
     save_sys_modules,
     slowadd,
     slowdec,
@@ -1172,15 +1161,6 @@ async def test_scatter_non_list(c, s, a, b):
     assert result == 1
 
 
-@gen_cluster(client=True)
-async def test_scatter_hash(c, s, a, b):
-    [a] = await c.scatter([1])
-    [b] = await c.scatter([1])
-
-    assert a.key == b.key
-    s.validate_state()
-
-
 @gen_cluster(client=True)
 async def test_scatter_tokenize_local(c, s, a, b):
     from dask.base import normalize_token
@@ -1227,6 +1207,15 @@ async def test_scatter_hash(c, s, a, b):
     assert z.key != y.key
 
 
+@gen_cluster(client=True)
+async def test_scatter_hash_2(c, s, a, b):
+    [a] = await c.scatter([1])
+    [b] = await c.scatter([1])
+
+    assert a.key == b.key
+    s.validate_state()
+
+
 @gen_cluster(client=True)
 async def test_get_releases_data(c, s, a, b):
     await c.gather(c.get({"x": (inc, 1)}, ["x"], sync=False))
@@ -1400,6 +1389,20 @@ async def test_scatter_direct(c, s, a, b):
     assert not s.counters["op"].components[0]["gather"]
 
 
+@gen_cluster()
+async def test_scatter_direct_2(s, a, b):
+    c = await Client(s.address, asynchronous=True, heartbeat_interval=10)
+
+    last = s.clients[c.id].last_seen
+
+    start = time()
+    while s.clients[c.id].last_seen == last:
+        await asyncio.sleep(0.10)
+        assert time() < start + 5
+
+    await c.close()
+
+
 @gen_cluster(client=True)
 async def test_scatter_direct_numpy(c, s, a, b):
     np = pytest.importorskip("numpy")
@@ -5551,20 +5554,6 @@ async def test_warn_when_submitting_large_values(c, s, a, b):
     assert len(record) < 2
 
 
-@gen_cluster()
-async def test_scatter_direct(s, a, b):
-    c = await Client(s.address, asynchronous=True, heartbeat_interval=10)
-
-    last = s.clients[c.id].last_seen
-
-    start = time()
-    while s.clients[c.id].last_seen == last:
-        await asyncio.sleep(0.10)
-        assert time() < start + 5
-
-    await c.close()
-
-
 @gen_cluster(client=True)
 async def test_unhashable_function(c, s, a, b):
     d = {"a": 1}
diff --git a/distributed/tests/test_client_executor.py b/distributed/tests/test_client_executor.py
index 3942a2ca010..d53a0375f68 100644
--- a/distributed/tests/test_client_executor.py
+++ b/distributed/tests/test_client_executor.py
@@ -14,15 +14,9 @@
 
 from distributed import Client
 from distributed.utils import CancelledError
-from distributed.utils_test import (  # noqa: F401
-    a,
-    b,
-    client,
+from distributed.utils_test import (
     cluster,
-    cluster_fixture,
     inc,
-    loop,
-    s,
     slowadd,
     slowdec,
     slowinc,
diff --git a/distributed/tests/test_collections.py b/distributed/tests/test_collections.py
index ea045bb49a5..00c9baf928d 100644
--- a/distributed/tests/test_collections.py
+++ b/distributed/tests/test_collections.py
@@ -10,12 +10,7 @@
 import dask.dataframe as dd
 
 from distributed.client import wait
-from distributed.utils_test import (  # noqa F401
-    client,
-    cluster_fixture,
-    gen_cluster,
-    loop,
-)
+from distributed.utils_test import gen_cluster
 
 PANDAS_VERSION = LooseVersion(pd.__version__)
 PANDAS_GT_100 = PANDAS_VERSION >= LooseVersion("1.0.0")
diff --git a/distributed/tests/test_core.py b/distributed/tests/test_core.py
index 44757b512d2..9fc98f276f2 100644
--- a/distributed/tests/test_core.py
+++ b/distributed/tests/test_core.py
@@ -23,7 +23,6 @@
 from distributed.protocol import to_serialize
 from distributed.protocol.compression import compressions
 from distributed.utils import get_ip, get_ipv6
-from distributed.utils_test import loop  # noqa F401
 from distributed.utils_test import (
     assert_can_connect,
     assert_can_connect_from_everywhere_4,
diff --git a/distributed/tests/test_counter.py b/distributed/tests/test_counter.py
index bb38a2812e5..0a293633dd2 100644
--- a/distributed/tests/test_counter.py
+++ b/distributed/tests/test_counter.py
@@ -1,7 +1,6 @@
 import pytest
 
 from distributed.counter import Counter
-from distributed.utils_test import loop  # noqa F401
 
 try:
     from distributed.counter import Digest
diff --git a/distributed/tests/test_events.py b/distributed/tests/test_events.py
index 9f27a19a65d..54f7560189e 100644
--- a/distributed/tests/test_events.py
+++ b/distributed/tests/test_events.py
@@ -2,12 +2,7 @@
 from datetime import timedelta
 
 from distributed import Event
-from distributed.utils_test import (  # noqa F401
-    client,
-    cluster_fixture,
-    gen_cluster,
-    loop,
-)
+from distributed.utils_test import gen_cluster
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 8)] * 2)
diff --git a/distributed/tests/test_failed_workers.py b/distributed/tests/test_failed_workers.py
index 3575037f48b..fa571d0f371 100644
--- a/distributed/tests/test_failed_workers.py
+++ b/distributed/tests/test_failed_workers.py
@@ -15,7 +15,6 @@
 from distributed.metrics import time
 from distributed.scheduler import COMPILED
 from distributed.utils import CancelledError, sync
-from distributed.utils_test import loop  # noqa: F401
 from distributed.utils_test import (
     captured_logger,
     cluster,
diff --git a/distributed/tests/test_ipython.py b/distributed/tests/test_ipython.py
index fbe5cfc8f9d..6e6b317fd0c 100644
--- a/distributed/tests/test_ipython.py
+++ b/distributed/tests/test_ipython.py
@@ -4,7 +4,7 @@
 from tlz import first
 
 from distributed import Client
-from distributed.utils_test import cluster, loop, mock_ipython, zmq_ctx  # noqa F401
+from distributed.utils_test import cluster, mock_ipython
 
 
 def need_functional_ipython(func):
diff --git a/distributed/tests/test_locks.py b/distributed/tests/test_locks.py
index 54c25c49099..99bd5ebdca5 100644
--- a/distributed/tests/test_locks.py
+++ b/distributed/tests/test_locks.py
@@ -6,12 +6,7 @@
 
 from distributed import Client, Lock, get_client
 from distributed.metrics import time
-from distributed.utils_test import (  # noqa F401
-    client,
-    cluster_fixture,
-    gen_cluster,
-    loop,
-)
+from distributed.utils_test import gen_cluster
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 8)] * 2)
diff --git a/distributed/tests/test_multi_locks.py b/distributed/tests/test_multi_locks.py
index 26f4a6dbdb7..9ece4e9c2c2 100644
--- a/distributed/tests/test_multi_locks.py
+++ b/distributed/tests/test_multi_locks.py
@@ -4,12 +4,7 @@
 from distributed import MultiLock, get_client
 from distributed.metrics import time
 from distributed.multi_lock import MultiLockExtension
-from distributed.utils_test import (  # noqa F401
-    client,
-    cluster_fixture,
-    gen_cluster,
-    loop,
-)
+from distributed.utils_test import gen_cluster
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 8)] * 2)
diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
index 1203fb3bbed..4ac628a599b 100644
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -19,7 +19,6 @@
 from distributed.metrics import time
 from distributed.protocol.pickle import dumps
 from distributed.utils import TimeoutError, parse_ports, tmpfile
-from distributed.utils_test import cleanup  # noqa: F401
 from distributed.utils_test import captured_logger, gen_cluster, gen_test, inc
 
 
@@ -478,7 +477,7 @@ def remove_worker(self, **kwargs):
 
 
 @pytest.mark.asyncio
-async def test_nanny_closes_cleanly(cleanup):
+async def test_nanny_closes_cleanly_2(cleanup):
     async with Scheduler() as s:
         async with Nanny(s.address) as n:
             async with Client(s.address, asynchronous=True) as client:
diff --git a/distributed/tests/test_preload.py b/distributed/tests/test_preload.py
index 7915aafab15..20650ff49cf 100644
--- a/distributed/tests/test_preload.py
+++ b/distributed/tests/test_preload.py
@@ -9,7 +9,7 @@
 import dask
 
 from distributed import Client, Nanny, Scheduler, Worker
-from distributed.utils_test import captured_logger, cleanup, cluster, loop  # noqa F401
+from distributed.utils_test import captured_logger, cluster
 
 PRELOAD_TEXT = """
 _worker_info = {}
diff --git a/distributed/tests/test_publish.py b/distributed/tests/test_publish.py
index 7f4c03fe71a..898a2f27acb 100644
--- a/distributed/tests/test_publish.py
+++ b/distributed/tests/test_publish.py
@@ -8,13 +8,7 @@
 from distributed.client import futures_of
 from distributed.metrics import time
 from distributed.protocol import Serialized
-from distributed.utils_test import (  # noqa F401
-    client,
-    cluster_fixture,
-    gen_cluster,
-    inc,
-    loop,
-)
+from distributed.utils_test import gen_cluster, inc
 
 
 @gen_cluster(client=False)
diff --git a/distributed/tests/test_queues.py b/distributed/tests/test_queues.py
index c7efa9a0d23..9db84f941ab 100644
--- a/distributed/tests/test_queues.py
+++ b/distributed/tests/test_queues.py
@@ -6,15 +6,7 @@
 
 from distributed import Client, Nanny, Queue, TimeoutError, wait, worker_client
 from distributed.metrics import time
-from distributed.utils_test import (  # noqa: F401
-    client,
-    cluster_fixture,
-    div,
-    gen_cluster,
-    inc,
-    loop,
-    popen,
-)
+from distributed.utils_test import div, gen_cluster, inc, popen
 
 
 @gen_cluster(client=True)
diff --git a/distributed/tests/test_resources.py b/distributed/tests/test_resources.py
index 15e9f344a83..f3422270bdd 100644
--- a/distributed/tests/test_resources.py
+++ b/distributed/tests/test_resources.py
@@ -10,18 +10,7 @@
 from distributed import Worker
 from distributed.client import wait
 from distributed.compatibility import WINDOWS
-from distributed.utils_test import (  # noqa: F401
-    a,
-    b,
-    client,
-    cluster_fixture,
-    gen_cluster,
-    inc,
-    loop,
-    s,
-    slowadd,
-    slowinc,
-)
+from distributed.utils_test import gen_cluster, inc, slowadd, slowinc
 
 
 @gen_cluster(client=True, nthreads=[])
@@ -87,6 +76,26 @@ async def test_submit_many_non_overlapping(c, s, a, b):
     assert len(b.data) == 0
 
 
+@gen_cluster(
+    client=True,
+    nthreads=[
+        ("127.0.0.1", 4, {"resources": {"A": 2}}),
+        ("127.0.0.1", 4, {"resources": {"A": 1}}),
+    ],
+)
+async def test_submit_many_non_overlapping_2(c, s, a, b):
+    futures = c.map(slowinc, range(100), resources={"A": 1}, delay=0.02)
+
+    while len(a.data) + len(b.data) < 100:
+        await asyncio.sleep(0.01)
+        assert a.executing_count <= 2
+        assert b.executing_count <= 1
+
+    await wait(futures)
+    assert a.total_resources == a.available_resources
+    assert b.total_resources == b.available_resources
+
+
 @gen_cluster(
     client=True,
     nthreads=[
@@ -236,26 +245,6 @@ async def test_resources_str(c, s, a, b):
     assert ts_last.resource_restrictions == {"MyRes": 1}
 
 
-@gen_cluster(
-    client=True,
-    nthreads=[
-        ("127.0.0.1", 4, {"resources": {"A": 2}}),
-        ("127.0.0.1", 4, {"resources": {"A": 1}}),
-    ],
-)
-async def test_submit_many_non_overlapping(c, s, a, b):
-    futures = c.map(slowinc, range(100), resources={"A": 1}, delay=0.02)
-
-    while len(a.data) + len(b.data) < 100:
-        await asyncio.sleep(0.01)
-        assert a.executing_count <= 2
-        assert b.executing_count <= 1
-
-    await wait(futures)
-    assert a.total_resources == a.available_resources
-    assert b.total_resources == b.available_resources
-
-
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 4, {"resources": {"A": 2, "B": 1}})])
 async def test_minimum_resource(c, s, a):
     futures = c.map(slowinc, range(30), resources={"A": 1, "B": 1}, delay=0.02)
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index f1ddae3615e..18c05b849f5 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -20,7 +20,6 @@
 from dask.compatibility import apply
 
 from distributed import Client, Nanny, Worker, fire_and_forget, wait
-from distributed.client import wait
 from distributed.comm import Comm
 from distributed.compatibility import MACOS, WINDOWS
 from distributed.core import ConnectionPool, Status, connect, rpc
@@ -28,16 +27,14 @@
 from distributed.protocol.pickle import dumps
 from distributed.scheduler import MemoryState, Scheduler
 from distributed.utils import TimeoutError, tmpfile, typename
-from distributed.utils_test import (  # noqa: F401
+from distributed.utils_test import (
     captured_logger,
-    cleanup,
     cluster,
     dec,
     div,
     gen_cluster,
     gen_test,
     inc,
-    loop,
     nodebug,
     slowadd,
     slowdec,
@@ -2214,7 +2211,7 @@ async def test_unknown_task_duration_config(client, s, a, b):
 
 
 @gen_cluster()
-async def test_unknown_task_duration_config(s, a, b):
+async def test_unknown_task_duration_config_2(s, a, b):
     assert s.idle_since == s.time_started
 
 
diff --git a/distributed/tests/test_semaphore.py b/distributed/tests/test_semaphore.py
index 2a478d4f749..5a5b9d02597 100644
--- a/distributed/tests/test_semaphore.py
+++ b/distributed/tests/test_semaphore.py
@@ -14,17 +14,7 @@
 from distributed.compatibility import WINDOWS
 from distributed.core import ConnectionPool
 from distributed.metrics import time
-from distributed.utils_test import (  # noqa: F401
-    async_wait_for,
-    captured_logger,
-    cleanup,
-    client,
-    cluster,
-    cluster_fixture,
-    gen_cluster,
-    loop,
-    slowidentity,
-)
+from distributed.utils_test import captured_logger, cluster, gen_cluster, slowidentity
 
 
 @gen_cluster(client=True)
diff --git a/distributed/tests/test_steal.py b/distributed/tests/test_steal.py
index b5abb67d35a..ee2695cea87 100644
--- a/distributed/tests/test_steal.py
+++ b/distributed/tests/test_steal.py
@@ -409,6 +409,22 @@ async def test_dont_steal_executing_tasks(c, s, a, b):
     assert len(b.data) == 0
 
 
+@gen_cluster(client=True)
+async def test_dont_steal_executing_tasks_2(c, s, a, b):
+    steal = s.extensions["stealing"]
+
+    future = c.submit(slowinc, 1, delay=0.5, workers=a.address)
+    while not a.executing_count:
+        await asyncio.sleep(0.01)
+
+    steal.move_task_request(
+        s.tasks[future.key], s.workers[a.address], s.workers[b.address]
+    )
+    await asyncio.sleep(0.1)
+    assert a.tasks[future.key].state == "executing"
+    assert not b.executing_count
+
+
 @gen_cluster(
     client=True,
     nthreads=[("127.0.0.1", 1)] * 10,
@@ -669,22 +685,6 @@ async def test_steal_twice(c, s, a, b):
     await asyncio.gather(*[w.close() for w in workers])
 
 
-@gen_cluster(client=True)
-async def test_dont_steal_executing_tasks(c, s, a, b):
-    steal = s.extensions["stealing"]
-
-    future = c.submit(slowinc, 1, delay=0.5, workers=a.address)
-    while not a.executing_count:
-        await asyncio.sleep(0.01)
-
-    steal.move_task_request(
-        s.tasks[future.key], s.workers[a.address], s.workers[b.address]
-    )
-    await asyncio.sleep(0.1)
-    assert a.tasks[future.key].state == "executing"
-    assert not b.executing_count
-
-
 @gen_cluster(client=True)
 async def test_dont_steal_already_released(c, s, a, b):
     future = c.submit(slowinc, 1, delay=0.05, workers=a.address)
diff --git a/distributed/tests/test_stress.py b/distributed/tests/test_stress.py
index b3424ed4c62..cf09c9dbcd5 100644
--- a/distributed/tests/test_stress.py
+++ b/distributed/tests/test_stress.py
@@ -11,16 +11,14 @@
 from dask import delayed
 
 from distributed import Client, Nanny, wait
-from distributed.client import wait
 from distributed.config import config
 from distributed.metrics import time
 from distributed.utils import All, CancelledError
-from distributed.utils_test import (  # noqa: F401
+from distributed.utils_test import (
     bump_rlimit,
     cluster,
     gen_cluster,
     inc,
-    loop,
     nodebug_setup_module,
     nodebug_teardown_module,
     slowadd,
diff --git a/distributed/tests/test_tls_functional.py b/distributed/tests/test_tls_functional.py
index 84a0f5922d9..8950f804cb6 100644
--- a/distributed/tests/test_tls_functional.py
+++ b/distributed/tests/test_tls_functional.py
@@ -9,8 +9,7 @@
 from distributed import Client, Nanny, Queue, Scheduler, Worker, wait, worker_client
 from distributed.core import Status
 from distributed.metrics import time
-from distributed.utils_test import (  # noqa: F401
-    cleanup,
+from distributed.utils_test import (
     double,
     gen_tls_cluster,
     inc,
diff --git a/distributed/tests/test_utils.py b/distributed/tests/test_utils.py
index 6d9ab29520b..0a073484c93 100644
--- a/distributed/tests/test_utils.py
+++ b/distributed/tests/test_utils.py
@@ -47,16 +47,7 @@
     truncate_exception,
     warn_on_duration,
 )
-from distributed.utils_test import (  # noqa: F401
-    captured_logger,
-    div,
-    gen_test,
-    has_ipv6,
-    inc,
-    loop,
-    loop_in_thread,
-    throws,
-)
+from distributed.utils_test import captured_logger, div, gen_test, has_ipv6, inc, throws
 
 
 def test_All(loop):
diff --git a/distributed/tests/test_utils_comm.py b/distributed/tests/test_utils_comm.py
index c0bc721f51b..d17e892ebf0 100644
--- a/distributed/tests/test_utils_comm.py
+++ b/distributed/tests/test_utils_comm.py
@@ -5,7 +5,7 @@
 from distributed.comm import Comm
 from distributed.core import ConnectionPool
 from distributed.utils_comm import gather_from_workers, pack_data, retry, subs_multiple
-from distributed.utils_test import gen_cluster, loop  # noqa: F401
+from distributed.utils_test import gen_cluster
 
 
 def test_pack_data():
diff --git a/distributed/tests/test_utils_test.py b/distributed/tests/test_utils_test.py
index 4bf0230548b..1458cf2af11 100755
--- a/distributed/tests/test_utils_test.py
+++ b/distributed/tests/test_utils_test.py
@@ -11,17 +11,12 @@
 from distributed.core import rpc
 from distributed.metrics import time
 from distributed.utils import get_ip
-from distributed.utils_test import (  # noqa: F401
-    cleanup,
+from distributed.utils_test import (
     cluster,
     gen_cluster,
     gen_test,
     inc,
-    loop,
     new_config,
-    security,
-    tls_client,
-    tls_cluster,
     tls_only_security,
     wait_for_port,
 )
@@ -216,7 +211,7 @@ async def f(s, a, b):
         default_client()
 
 
-def test_lingering_client(loop):
+def test_lingering_client_2(loop):
     with cluster() as (s, [a, b]):
         client = Client(s["address"], loop=loop)
 
diff --git a/distributed/tests/test_variable.py b/distributed/tests/test_variable.py
index 9763de48335..1819da29049 100644
--- a/distributed/tests/test_variable.py
+++ b/distributed/tests/test_variable.py
@@ -10,16 +10,7 @@
 from distributed import Client, Nanny, TimeoutError, Variable, wait, worker_client
 from distributed.compatibility import WINDOWS
 from distributed.metrics import time
-from distributed.utils_test import (  # noqa: F401
-    captured_logger,
-    client,
-    cluster_fixture,
-    div,
-    gen_cluster,
-    inc,
-    loop,
-    popen,
-)
+from distributed.utils_test import captured_logger, div, gen_cluster, inc, popen
 
 
 @gen_cluster(client=True)
diff --git a/distributed/tests/test_versions.py b/distributed/tests/test_versions.py
index 345cee3a253..5504daea31c 100644
--- a/distributed/tests/test_versions.py
+++ b/distributed/tests/test_versions.py
@@ -4,7 +4,7 @@
 import pytest
 
 from distributed import Client, Worker
-from distributed.utils_test import gen_cluster, loop  # noqa: F401
+from distributed.utils_test import gen_cluster
 from distributed.versions import error_message, get_versions
 
 # if one of the nodes reports this version, there's a mismatch
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 633325ef6d4..41b0e171c7e 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -37,23 +37,16 @@
 from distributed.metrics import time
 from distributed.scheduler import Scheduler
 from distributed.utils import TimeoutError, tmpfile
-from distributed.utils_test import (  # noqa: F401
+from distributed.utils_test import (
     TaskStateMetadataPlugin,
-    a,
-    b,
     captured_logger,
-    cleanup,
-    client,
-    cluster_fixture,
     dec,
     div,
     gen_cluster,
     gen_test,
     inc,
-    loop,
     mul,
     nodebug,
-    s,
     slowinc,
 )
 from distributed.worker import Worker, error_message, logger, parse_memory_limit, weight
diff --git a/distributed/tests/test_worker_client.py b/distributed/tests/test_worker_client.py
index 451e9116360..5fcc01877ef 100644
--- a/distributed/tests/test_worker_client.py
+++ b/distributed/tests/test_worker_client.py
@@ -18,14 +18,7 @@
     worker_client,
 )
 from distributed.metrics import time
-from distributed.utils_test import (  # noqa: F401
-    client,
-    cluster_fixture,
-    double,
-    gen_cluster,
-    inc,
-    loop,
-)
+from distributed.utils_test import double, gen_cluster, inc
 
 
 @gen_cluster(client=True)
diff --git a/distributed/utils.py b/distributed/utils.py
index dc63202486b..2bac5c05b3c 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -24,7 +24,8 @@
 from hashlib import md5
 from importlib.util import cache_from_source
 from time import sleep
-from typing import Any, Dict, List
+from typing import Any as AnyType
+from typing import Dict, List
 
 import click
 import tblib.pickling_support
@@ -335,7 +336,7 @@ def f():
             if callback_timeout is not None:
                 future = asyncio.wait_for(future, callback_timeout)
             result[0] = yield future
-        except Exception as exc:
+        except Exception:
             error[0] = sys.exc_info()
         finally:
             assert thread_state.asynchronous > 0
@@ -1507,7 +1508,7 @@ def __setitem__(self, key, value):
         super().__setitem__(key, value)
 
 
-def clean_dashboard_address(addrs: Any, default_listen_ip: str = "") -> List[Dict]:
+def clean_dashboard_address(addrs: AnyType, default_listen_ip: str = "") -> List[Dict]:
     """
     Examples
     --------
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index aba7294a71d..a40cf14151d 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -24,6 +24,8 @@
 from glob import glob
 from time import sleep
 
+from distributed.scheduler import Scheduler
+
 try:
     import ssl
 except ImportError:
@@ -551,6 +553,10 @@ def client(loop, cluster_fixture):
         yield client
 
 
+# Compatibility. A lot of tests simply use `c` as fixture name
+c = client
+
+
 @pytest.fixture
 def client_secondary(loop, cluster_fixture):
     scheduler, workers = cluster_fixture
@@ -772,10 +778,6 @@ def test_func():
     return _
 
 
-from .scheduler import Scheduler
-from .worker import Worker
-
-
 async def start_cluster(
     nthreads,
     scheduler_addr,
diff --git a/distributed/worker.py b/distributed/worker.py
index 11bdc69c648..0735043355d 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -2127,7 +2127,7 @@ def send_task_state_to_scheduler(self, ts):
                     value = self.data[ts.key]
                 except KeyError:
                     value = self.actors[ts.key]
-                nbytes = ts.nbytes = sizeof(value)
+                ts.nbytes = sizeof(value)
                 typ = ts.type = type(value)
                 del value
             try:
@@ -2319,7 +2319,7 @@ async def gather_dep(
                 self.incoming_count += 1
 
                 self.log.append(("receive-dep", worker, list(response["data"])))
-            except EnvironmentError as e:
+            except EnvironmentError:
                 logger.exception("Worker stream died during communication: %s", worker)
                 self.log.append(("receive-dep-failed", worker))
                 for d in self.has_what.pop(worker):
@@ -3881,8 +3881,6 @@ async def run(server, comm, function, args=(), kwargs=None, is_coro=None, wait=T
 _global_workers = Worker._instances
 
 try:
-    from .diagnostics import nvml
-
     if nvml.device_get_count() < 1:
         raise RuntimeError
 except (Exception, RuntimeError):
diff --git a/setup.cfg b/setup.cfg
index 1db115623f2..136ae4c363f 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -3,27 +3,24 @@
 # https://flake8.readthedocs.io/en/latest/user/configuration.html
 # https://flake8.readthedocs.io/en/latest/user/error-codes.html
 
+# Aligned with black https://github.com/psf/black/blob/main/.flake8
+extend-ignore = E203, E266, E501
 # Note: there cannot be spaces after comma's here
 exclude = __init__.py,versioneer.py,distributed/_concurrent_futures_thread.py
 ignore =
-    E20,        # Extra space in brackets
-    E231,E241,  # Multiple spaces around ","
     E26,        # Comments
     E4,         # Import formatting
-    E721,       # Comparing types instead of isinstance
     E731,       # Assigning lambda expression
-    E121,       # continuation line under-indented for hanging indent
-    E126,       # continuation line over-indented for hanging indent
-    E127,       # continuation line over-indented for visual indent
-    E128,       # E128 continuation line under-indented for visual indent
-    E702,       # multiple statements on one line (semicolon)
     W503,       # line break before binary operator
-    E129,       # visually indented line with same indent as next logical line
-    E116,       # unexpected indentation
-    F811,       # redefinition of unused 'loop' from line 10
-    F841,       # local variable is assigned to but never used
-    E741        # Ambiguous variable names
-    W504,       # line break after binary operator
+
+
+per-file-ignores =
+    **/tests/*:
+        # local variable is assigned to but never used
+        F841,
+        # Ambiguous variable name
+        E741,
+
 
 max-line-length = 120
 

From 668f3f1d38c27277448af6f5aa88741cd1d33f3b Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Wed, 9 Jun 2021 08:57:53 -0500
Subject: [PATCH 1305/1550] Remove tests for `process_time` and `thread_time`
 (#4895)

---
 distributed/tests/test_metrics.py | 46 -------------------------------
 1 file changed, 46 deletions(-)

diff --git a/distributed/tests/test_metrics.py b/distributed/tests/test_metrics.py
index 3a27e638ef3..58c3326689d 100644
--- a/distributed/tests/test_metrics.py
+++ b/distributed/tests/test_metrics.py
@@ -1,9 +1,6 @@
-import sys
-import threading
 import time
 
 from distributed import metrics
-from distributed.utils_test import run_for
 
 
 def test_wall_clock():
@@ -18,46 +15,3 @@ def test_wall_clock():
         assert any(lambda d: 0.0 < d < 0.0001 for d in deltas), deltas
         # Close to time.time()
         assert t - 0.5 < samples[0] < t + 0.5
-
-
-def test_process_time():
-    start = metrics.process_time()
-    run_for(0.05)
-    dt = metrics.process_time() - start
-    assert 0.03 <= dt <= 0.2
-
-    # All threads counted
-    t = threading.Thread(target=run_for, args=(0.1,))
-    start = metrics.process_time()
-    t.start()
-    t.join()
-    dt = metrics.process_time() - start
-    assert dt >= 0.05
-
-    # Sleep time not counted
-    start = metrics.process_time()
-    time.sleep(0.1)
-    dt = metrics.process_time() - start
-    assert dt <= 0.05
-
-
-def test_thread_time():
-    start = metrics.thread_time()
-    run_for(0.05)
-    dt = metrics.thread_time() - start
-    assert 0.03 <= dt <= 0.2
-
-    # Sleep time not counted
-    start = metrics.thread_time()
-    time.sleep(0.1)
-    dt = metrics.thread_time() - start
-    assert dt <= 0.05
-
-    if sys.platform == "linux":
-        # Always per-thread on Linux
-        t = threading.Thread(target=run_for, args=(0.1,))
-        start = metrics.thread_time()
-        t.start()
-        t.join()
-        dt = metrics.thread_time() - start
-        assert dt <= 0.05

From ddec3c9c7b5a5d3cb3b806ff25ee36ef640d8dda Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Wed, 9 Jun 2021 11:16:12 -0500
Subject: [PATCH 1306/1550] Update `dask_cuda` usage in `SSHCluster` docstring
 (#4894)

---
 distributed/deploy/ssh.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/deploy/ssh.py b/distributed/deploy/ssh.py
index a70a7ca50c5..64452c31721 100644
--- a/distributed/deploy/ssh.py
+++ b/distributed/deploy/ssh.py
@@ -302,7 +302,7 @@ def SSHCluster(
     ...     ["localhost", "hostwithgpus", "anothergpuhost"],
     ...     connect_options={"known_hosts": None},
     ...     scheduler_options={"port": 0, "dashboard_address": ":8797"},
-    ...     worker_module='dask_cuda.dask_cuda_worker')
+    ...     worker_module="dask_cuda.cli.dask_cuda_worker")
     >>> client = Client(cluster)
 
     See Also

From 13f3d06abf0e031bbe25a8b20b52db090574a356 Mon Sep 17 00:00:00 2001
From: Marcos Moyano <marcos@anue.biz>
Date: Fri, 11 Jun 2021 10:51:00 -0300
Subject: [PATCH 1307/1550] Handle HTTPClientError in websocket connector
 (#4900)

Catch websocket client connection errors and re-raise them as
CommClosedError so that they are handled correctly by the Client.

This happens when a Client tries to reconnect to a closed Cluster behind a proxy.
The proxy closes the connection back to the client since it can't connect to the Cluster
which leads to a HTTPClientError that gets bubbled up.
---
 distributed/comm/ws.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/distributed/comm/ws.py b/distributed/comm/ws.py
index 05e8c88b79d..be7679513aa 100644
--- a/distributed/comm/ws.py
+++ b/distributed/comm/ws.py
@@ -7,7 +7,7 @@
 from typing import Callable
 
 from tornado import web
-from tornado.httpclient import HTTPRequest
+from tornado.httpclient import HTTPClientError, HTTPRequest
 from tornado.httpserver import HTTPServer
 from tornado.iostream import StreamClosedError
 from tornado.websocket import WebSocketClosedError, WebSocketHandler, websocket_connect
@@ -380,6 +380,8 @@ async def connect(self, address, deserialize=True, **connection_args):
                 "TLS expects a `ssl_context` argument of type "
                 "ssl.SSLContext (perhaps check your TLS configuration?)"
             ) from err
+        except HTTPClientError as e:
+            raise CommClosedError(f"in {self}: {e}") from e
         return self.comm_class(sock, deserialize=deserialize)
 
     def _get_connect_args(self, **connection_args):

From 5e5b9835a5e6929bb97acf94f0a2eecccc0b6a2f Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Fri, 11 Jun 2021 19:12:02 +0200
Subject: [PATCH 1308/1550] Forget erred tasks // Fix deadlocks on worker
 (#4784)

* Add a test about expected task states in an exception case

* Minor refactoring about who_has state transitions

* Remvoe worker kwarg from gather_dep fetch transition

* Add special case for inflight tasks without dependents

* Fix compute deadlocks

* Code review comments

* Improve request X keys for task debug log

* Before calling missing-data ensure key is removed from who_has

* review comments

* Add test case for same-host missing deps

* Add test about superfluous data race condition
---
 .../diagnostics/tests/test_worker_plugin.py   |   1 +
 distributed/scheduler.py                      |  64 +++-
 distributed/tests/test_failed_workers.py      | 192 ++++++++++-
 distributed/tests/test_scheduler.py           |  17 +-
 distributed/tests/test_worker.py              | 320 +++++++++++++++++-
 distributed/worker.py                         | 304 +++++++++++++----
 6 files changed, 805 insertions(+), 93 deletions(-)

diff --git a/distributed/diagnostics/tests/test_worker_plugin.py b/distributed/diagnostics/tests/test_worker_plugin.py
index cc41bd79d09..86eb11e49ed 100644
--- a/distributed/diagnostics/tests/test_worker_plugin.py
+++ b/distributed/diagnostics/tests/test_worker_plugin.py
@@ -131,6 +131,7 @@ def failing(x):
         {"key": "task", "start": "waiting", "finish": "ready"},
         {"key": "task", "start": "ready", "finish": "executing"},
         {"key": "task", "start": "executing", "finish": "error"},
+        {"key": "task", "state": "error"},
     ]
 
     plugin = MyPlugin(1, expected_notifications=expected_notifications)
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index f5ad740b523..45e38b24a51 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1205,6 +1205,10 @@ class TaskState:
        failed task is stored here (possibly itself).  Otherwise this
        is ``None``.
 
+    .. attribute:: erred_on: set(str)
+
+        Worker addresses on which errors appeared causing this task to be in an error state.
+
     .. attribute:: suspicious: int
 
        The number of times this task has been involved in a worker death.
@@ -1293,6 +1297,7 @@ class TaskState:
     _exception: object
     _traceback: object
     _exception_blame: object
+    _erred_on: set
     _suspicious: Py_ssize_t
     _host_restrictions: set
     _worker_restrictions: set
@@ -1343,6 +1348,7 @@ class TaskState:
         "_who_wants",
         "_exception",
         "_traceback",
+        "_erred_on",
         "_exception_blame",
         "_suspicious",
         "_retries",
@@ -1381,6 +1387,7 @@ def __init__(self, key: str, run_spec: object):
         self._group = None
         self._metadata = {}
         self._annotations = {}
+        self._erred_on = set()
 
     def __hash__(self):
         return self._hash
@@ -1528,6 +1535,10 @@ def group_key(self):
     def prefix_key(self):
         return self._prefix._name
 
+    @property
+    def erred_on(self):
+        return self._erred_on
+
     @ccall
     def add_dependency(self, other: "TaskState"):
         """Add another task as a dependency of this task"""
@@ -1842,7 +1853,6 @@ def __init__(
             ("no-worker", "waiting"): self.transition_no_worker_waiting,
             ("released", "forgotten"): self.transition_released_forgotten,
             ("memory", "forgotten"): self.transition_memory_forgotten,
-            ("erred", "forgotten"): self.transition_released_forgotten,
             ("erred", "released"): self.transition_erred_released,
             ("memory", "released"): self.transition_memory_released,
             ("released", "erred"): self.transition_released_erred,
@@ -2629,9 +2639,9 @@ def transition_memory_released(self, key, safe: bint = False):
             # XXX factor this out?
             ts_nbytes: Py_ssize_t = ts.get_nbytes()
             worker_msg = {
-                "op": "delete-data",
+                "op": "free-keys",
                 "keys": [key],
-                "report": False,
+                "reason": f"Memory->Released {key}",
             }
             for ws in ts._who_has:
                 del ws._has_what[ts]
@@ -2722,7 +2732,6 @@ def transition_erred_released(self, key):
 
             if self._validate:
                 with log_errors(pdb=LOG_PDB):
-                    assert all([dts._state != "erred" for dts in ts._dependencies])
                     assert ts._exception_blame
                     assert not ts._who_has
                     assert not ts._waiting_on
@@ -2736,6 +2745,11 @@ def transition_erred_released(self, key):
                 if dts._state == "erred":
                     recommendations[dts._key] = "waiting"
 
+            w_msg = {"op": "free-keys", "keys": [key], "reason": "Erred->Released"}
+            for ws_addr in ts._erred_on:
+                worker_msgs[ws_addr] = [w_msg]
+            ts._erred_on.clear()
+
             report_msg = {"op": "task-retried", "key": key}
             cs: ClientState
             for cs in ts._who_wants:
@@ -2805,7 +2819,9 @@ def transition_processing_released(self, key):
 
             w: str = _remove_from_processing(self, ts)
             if w:
-                worker_msgs[w] = [{"op": "release-task", "key": key}]
+                worker_msgs[w] = [
+                    {"op": "free-keys", "keys": [key], "reason": "Processing->Released"}
+                ]
 
             ts.state = "released"
 
@@ -2835,7 +2851,7 @@ def transition_processing_released(self, key):
             raise
 
     def transition_processing_erred(
-        self, key, cause=None, exception=None, traceback=None, **kwargs
+        self, key, cause=None, exception=None, traceback=None, worker=None, **kwargs
     ):
         ws: WorkerState
         try:
@@ -2856,8 +2872,9 @@ def transition_processing_erred(
                 ws = ts._processing_on
                 ws._actors.remove(ts)
 
-            _remove_from_processing(self, ts)
+            w = _remove_from_processing(self, ts)
 
+            ts._erred_on.add(w or worker)
             if exception is not None:
                 ts._exception = exception
             if traceback is not None:
@@ -4456,7 +4473,9 @@ def stimulus_task_finished(self, key=None, worker=None, **kwargs):
                 ts._who_has,
             )
             if ws not in ts._who_has:
-                worker_msgs[worker] = [{"op": "release-task", "key": key}]
+                worker_msgs[worker] = [
+                    {"op": "free-keys", "keys": [key], "reason": "Stimulus Finished"}
+                ]
 
         return recommendations, client_msgs, worker_msgs
 
@@ -5113,7 +5132,7 @@ def handle_missing_data(self, key=None, errant_worker=None, **kwargs):
     def release_worker_data(self, comm=None, keys=None, worker=None):
         parent: SchedulerState = cast(SchedulerState, self)
         ws: WorkerState = parent._workers_dv[worker]
-        tasks: set = {parent._tasks[k] for k in keys}
+        tasks: set = {parent._tasks[k] for k in keys if k in parent._tasks}
         removed_tasks: set = tasks.intersection(ws._has_what)
 
         ts: TaskState
@@ -5519,8 +5538,11 @@ async def _delete_worker_data(self, worker_address, keys):
             List of keys to delete on the specified worker
         """
         parent: SchedulerState = cast(SchedulerState, self)
+
         await retry_operation(
-            self.rpc(addr=worker_address).delete_data, keys=list(keys), report=False
+            self.rpc(addr=worker_address).free_keys,
+            keys=list(keys),
+            reason="rebalance/replicate",
         )
 
         ws: WorkerState = parent._workers_dv[worker_address]
@@ -6271,6 +6293,7 @@ def add_keys(self, comm=None, worker=None, keys=()):
         if worker not in parent._workers_dv:
             return "not found"
         ws: WorkerState = parent._workers_dv[worker]
+        superfluous_data = []
         for key in keys:
             ts: TaskState = parent._tasks.get(key)
             if ts is not None and ts._state == "memory":
@@ -6279,9 +6302,16 @@ def add_keys(self, comm=None, worker=None, keys=()):
                     ws._has_what[ts] = None
                     ts._who_has.add(ws)
             else:
-                self.worker_send(
-                    worker, {"op": "delete-data", "keys": [key], "report": False}
-                )
+                superfluous_data.append(key)
+        if superfluous_data:
+            self.worker_send(
+                worker,
+                {
+                    "op": "superfluous-data",
+                    "keys": superfluous_data,
+                    "reason": f"Add keys which are not in-memory {superfluous_data}",
+                },
+            )
 
         return "OK"
 
@@ -7308,7 +7338,13 @@ def _propagate_forgotten(
         ws._nbytes -= ts_nbytes
         w: str = ws._address
         if w in state._workers_dv:  # in case worker has died
-            worker_msgs[w] = [{"op": "delete-data", "keys": [key], "report": False}]
+            worker_msgs[w] = [
+                {
+                    "op": "free-keys",
+                    "keys": [key],
+                    "reason": f"propagate-forgotten {ts.key}",
+                }
+            ]
     ts._who_has.clear()
 
 
diff --git a/distributed/tests/test_failed_workers.py b/distributed/tests/test_failed_workers.py
index fa571d0f371..32c872dec61 100644
--- a/distributed/tests/test_failed_workers.py
+++ b/distributed/tests/test_failed_workers.py
@@ -3,6 +3,7 @@
 import random
 from contextlib import suppress
 from time import sleep
+from unittest import mock
 
 import pytest
 from tlz import first, partition_all
@@ -384,7 +385,26 @@ async def test_restart_during_computation(c, s, a, b):
     assert not s.tasks
 
 
-@gen_cluster(client=True, timeout=60)
+class SlowTransmitData:
+    def __init__(self, data, delay=0.1):
+        self.delay = delay
+        self.data = data
+
+    def __reduce__(self):
+        import time
+
+        time.sleep(self.delay)
+        return (SlowTransmitData, (self.delay,))
+
+    def __sizeof__(self) -> int:
+        # Ensure this is offloaded to avoid blocking loop
+        import dask
+        from dask.utils import parse_bytes
+
+        return parse_bytes(dask.config.get("distributed.comm.offload")) + 1
+
+
+@gen_cluster(client=True)
 async def test_worker_who_has_clears_after_failed_connection(c, s, a, b):
     n = await Nanny(s.address, nthreads=2, loop=s.loop)
 
@@ -393,23 +413,32 @@ async def test_worker_who_has_clears_after_failed_connection(c, s, a, b):
         await asyncio.sleep(0.01)
         assert time() < start + 5
 
-    futures = c.map(slowinc, range(20), delay=0.01, key=["f%d" % i for i in range(20)])
-    await wait(futures)
-
-    result = await c.submit(sum, futures, workers=a.address)
-    deps = [dep for dep in a.tasks.values() if dep.key not in a.data_needed]
-    for dep in deps:
-        a.release_key(dep.key, report=True)
+    def slow_ser(x, delay):
+        return SlowTransmitData(x, delay=delay)
 
     n_worker_address = n.worker_address
+    futures = c.map(
+        slow_ser,
+        range(20),
+        delay=0.1,
+        key=["f%d" % i for i in range(20)],
+        workers=[n_worker_address],
+        allow_other_workers=True,
+    )
+
+    def sink(*args):
+        pass
+
+    await wait(futures)
+    result_fut = c.submit(sink, futures, workers=a.address)
+
     with suppress(CommClosedError):
         await c._run(os._exit, 1, workers=[n_worker_address])
 
     while len(s.workers) > 2:
         await asyncio.sleep(0.01)
 
-    total = c.submit(sum, futures, workers=a.address)
-    await total
+    await result_fut
 
     assert not a.has_what.get(n_worker_address)
     assert not any(n_worker_address in s for ts in a.tasks.values() for s in ts.who_has)
@@ -417,6 +446,51 @@ async def test_worker_who_has_clears_after_failed_connection(c, s, a, b):
     await n.close()
 
 
+@gen_cluster(
+    client=True,
+    nthreads=[("127.0.0.1", 1), ("127.0.0.1", 2), ("127.0.0.1", 3)],
+)
+async def test_worker_same_host_replicas_missing(c, s, a, b, x):
+    # See GH4784
+    def mock_address_host(addr):
+        # act as if A and X are on the same host
+        nonlocal a, b, x
+        if addr in [a.address, x.address]:
+            return "A"
+        else:
+            return "B"
+
+    with mock.patch("distributed.worker.get_address_host", mock_address_host):
+        futures = c.map(
+            slowinc,
+            range(20),
+            delay=0.1,
+            key=["f%d" % i for i in range(20)],
+            workers=[a.address],
+            allow_other_workers=True,
+        )
+        await wait(futures)
+
+        # replicate data to avoid the scheduler retriggering the computation
+        # retriggering cleans up the state nicely but doesn't reflect real world
+        # scenarios where there may be replicas on the cluster, e.g. they are
+        # replicated as a dependency somewhere else
+        await c.replicate(futures, n=2, workers=[a.address, b.address])
+
+        def sink(*args):
+            pass
+
+        # Since A and X are mocked to be co-located, X will consistently pick A
+        # to fetch data from. It will never succeed since we're removing data
+        # artificially, without notifying the scheduler.
+        # This can only succeed if B handles the missing data properly by
+        # removing A from the known sources of keys
+        a.handle_free_keys(keys=["f1"], reason="Am I evil?")  # Yes, I am!
+        result_fut = c.submit(sink, futures, workers=x.address)
+
+        await result_fut
+
+
 @pytest.mark.slow
 @gen_cluster(client=True, timeout=60, Worker=Nanny, nthreads=[("127.0.0.1", 1)])
 async def test_restart_timeout_on_long_running_task(c, s, a):
@@ -448,3 +522,101 @@ async def test_worker_time_to_live(c, s, a, b):
         assert time() < start + interval + 0.1
 
     set(s.workers) == {b.address}
+
+
+class SlowDeserialize:
+    def __init__(self, data, delay=0.1):
+        self.delay = delay
+        self.data = data
+
+    def __getstate__(self):
+        return self.delay
+
+    def __setstate__(self, state):
+        delay = state
+        import time
+
+        time.sleep(delay)
+        return SlowDeserialize(delay)
+
+    def __sizeof__(self) -> int:
+        # Ensure this is offloaded to avoid blocking loop
+        import dask
+        from dask.utils import parse_bytes
+
+        return parse_bytes(dask.config.get("distributed.comm.offload")) + 1
+
+
+@gen_cluster(client=True, timeout=None)
+async def test_handle_superfluous_data(c, s, a, b):
+    """
+    See https://github.com/dask/distributed/pull/4784#discussion_r649210094
+    """
+
+    def slow_deser(x, delay):
+        return SlowDeserialize(x, delay=delay)
+
+    futA = c.submit(
+        slow_deser, 1, delay=1, workers=[a.address], key="A", allow_other_workers=True
+    )
+    futB = c.submit(inc, 1, workers=[b.address], key="B")
+    await wait([futA, futB])
+
+    def reducer(*args):
+        return
+
+    assert len(a.tasks) == 1
+    assert futA.key in a.tasks
+
+    assert len(b.tasks) == 1
+    assert futB.key in b.tasks
+
+    red = c.submit(reducer, [futA, futB], workers=[b.address], key="reducer")
+
+    dep_key = futA.key
+
+    # Wait for the connection to be established
+    while dep_key not in b.tasks or not b.tasks[dep_key].state == "flight":
+        await asyncio.sleep(0.001)
+
+    # Wait for the connection to be returned to the pool. this signals that
+    # worker B is done with the communication and is about to deserialize the
+    # result
+    while a.address not in b.rpc.available and not b.rpc.available[a.address]:
+        await asyncio.sleep(0.001)
+
+    assert b.tasks[dep_key].state == "flight"
+    # After the comm is finished and the deserialization starts, Worker B
+    # wouldn't notice that A dies.
+    await a.close()
+    # However, while B is busy deserializing a third worker might notice that A
+    # is dead and issues a handle-missing signal to the scheduler. Since at this
+    # point in time, A was the only worker with a verified replica, the
+    # scheduler reschedules the computation by transitioning it to released. The
+    # released transition has the side effect that it purges all data which is
+    # in memory which exposes us to a race condition on B if B also receives the
+    # signal to compute that task in the meantime.
+    s.handle_missing_data(key=dep_key, errant_worker=a.address)
+    await red
+
+
+@gen_cluster()
+async def test_forget_data_not_supposed_to_have(s, a, b):
+    """
+    If a depednecy fetch finishes on a worker after the scheduler already
+    released everything, the worker might be stuck with a redundant replica
+    which is never cleaned up.
+    """
+    # FIXME: Replace with "blackbox test" which shows an actual example where
+    # this situation is provoked if this is even possible.
+    # If this cannot be constructed, the entire superfuous_data handler and its
+    # corresponding pieces on the scheduler side may be removed
+    from distributed.worker import TaskState
+
+    ts = TaskState("key")
+    ts.state = "flight"
+    a.tasks["key"] = ts
+    a.transition_flight_memory(ts, value=123)
+    assert a.data
+    while a.data:
+        await asyncio.sleep(0.001)
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 18c05b849f5..1746be6d4fe 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -1890,11 +1890,25 @@ async def test_task_groups(c, s, a, b):
     assert tg.states["released"] == 5
     assert tp.states["memory"] == 0
     assert tp.states["released"] == 5
+    assert tp.groups == [tg]
     assert tg.prefix is tp
-    assert tg in tp.groups
+    # these must be true since in this simple case there is a 1to1 mapping
+    # between prefix and group
     assert tg.duration == tp.duration
     assert tg.nbytes_in_memory == tp.nbytes_in_memory
     assert tg.nbytes_total == tp.nbytes_total
+    # It should map down to individual tasks
+    assert tg.nbytes_total == sum(
+        [ts.get_nbytes() for ts in s.tasks.values() if ts.group is tg]
+    )
+    in_memory_ts = sum(
+        [
+            ts.get_nbytes()
+            for ts in s.tasks.values()
+            if ts.group is tg and ts.state == "memory"
+        ]
+    )
+    assert tg.nbytes_in_memory == in_memory_ts
 
     tg = s.task_groups[y.name]
     assert tg.states["memory"] == 5
@@ -1902,6 +1916,7 @@ async def test_task_groups(c, s, a, b):
     assert s.task_groups[y.name].dependencies == {s.task_groups[x.name]}
 
     await c.replicate(y)
+    # TODO: Are we supposed to track replicated memory here? See also Scheduler.add_keys
     assert tg.nbytes_in_memory == y.nbytes
     assert "array" in str(tg.types)
     assert "array" in str(tp.types)
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 41b0e171c7e..ba4de5199d2 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -748,7 +748,6 @@ async def test_log_exception_on_failed_task(c, s, a, b):
             logger.removeHandler(fh)
 
 
-@pytest.mark.flaky(reruns=10, reruns_delay=5)
 @gen_cluster(client=True)
 async def test_clean_up_dependencies(c, s, a, b):
     x = delayed(inc)(1)
@@ -760,15 +759,12 @@ async def test_clean_up_dependencies(c, s, a, b):
     zz = c.persist(z)
     await wait(zz)
 
-    start = time()
     while len(a.data) + len(b.data) > 1:
         await asyncio.sleep(0.01)
-        assert time() < start + 2
 
     assert set(a.data) | set(b.data) == {zz.key}
 
 
-@pytest.mark.flaky(reruns=10, reruns_delay=5)
 @gen_cluster(client=True)
 async def test_hold_onto_dependents(c, s, a, b):
     x = c.submit(inc, 1, workers=a.address)
@@ -778,9 +774,8 @@ async def test_hold_onto_dependents(c, s, a, b):
     assert x.key in b.data
 
     await c._cancel(y)
-    await asyncio.sleep(0.1)
-
-    assert x.key in b.data
+    while x.key not in b.data:
+        await asyncio.sleep(0.1)
 
 
 @pytest.mark.slow
@@ -1840,8 +1835,8 @@ async def test_story_with_deps(c, s, a, b):
         ),
         (key, "waiting", "ready"),
         (key, "ready", "executing"),
-        (key, "put-in-memory"),
         (key, "executing", "memory"),
+        (key, "put-in-memory"),
     ]
     assert story == expected_story
 
@@ -2035,3 +2030,312 @@ def get_thread_name():
                 default_result, gpu_result = await c.gather(futures)
                 assert "Dask-Default-Threads" in default_result
                 assert "Dask-GPU-Threads" in gpu_result
+
+
+def assert_task_states_on_worker(expected, worker):
+    for dep_key, expected_state in expected.items():
+        assert dep_key in worker.tasks, (worker.name, dep_key, worker.tasks)
+        dep_ts = worker.tasks[dep_key]
+        assert dep_ts.state == expected_state, (worker.name, dep_ts, expected_state)
+    assert set(expected) == set(worker.tasks)
+
+
+@gen_cluster(client=True)
+async def test_worker_state_error_release_error_last(c, s, a, b):
+    """
+    Create a chain of tasks and err one of them. Then release tasks in a certain
+    order and ensure the tasks are released and/or kept in memory as appropriate
+
+    F -- RES (error)
+        /
+       /
+    G
+
+    Free error last
+    """
+
+    def raise_exc(*args):
+        raise RuntimeError()
+
+    f = c.submit(inc, 1, workers=[a.address], key="f")
+    g = c.submit(inc, 1, workers=[b.address], key="g")
+    res = c.submit(raise_exc, f, g, workers=[a.address])
+
+    with pytest.raises(RuntimeError):
+        await res.result()
+
+    # Nothing bad happened on B, therefore B should hold on to G
+    assert len(b.tasks) == 1
+    assert g.key in b.tasks
+
+    # A raised the exception therefore we should hold on to the erroneous task
+    assert res.key in a.tasks
+    ts = a.tasks[res.key]
+    assert ts.state == "error"
+
+    expected_states = {
+        # A was instructed to compute this result and we're still holding a ref via `f`
+        f.key: "memory",
+        # This was fetched from another worker. While we hold a ref via `g`, the
+        # scheduler only instructed to compute this on B
+        g.key: "memory",
+        res.key: "error",
+    }
+    assert_task_states_on_worker(expected_states, a)
+    # Expected states after we release references to the futures
+    f.release()
+    g.release()
+
+    # We no longer hold any refs to f or g and B didn't have any erros. It
+    # releases everything as expected
+    while b.tasks:
+        await asyncio.sleep(0.01)
+
+    expected_states = {
+        # We currently don't have a good way to actually release this memory as
+        # long as the tasks still have a dependent. We'll need to live with this
+        # memory for now
+        f.key: "memory",
+        g.key: "memory",
+        res.key: "error",
+    }
+
+    assert_task_states_on_worker(expected_states, a)
+
+    res.release()
+
+    # We no longer hold any refs. Cluster should reset completely
+    # This is not happening
+    for server in [s, a, b]:
+        while server.tasks:
+            await asyncio.sleep(0.01)
+
+
+@gen_cluster(client=True)
+async def test_worker_state_error_release_error_first(c, s, a, b):
+    """
+    Create a chain of tasks and err one of them. Then release tasks in a certain
+    order and ensure the tasks are released and/or kept in memory as appropriate
+
+    F -- RES (error)
+        /
+       /
+    G
+
+    Free error first
+    """
+
+    def raise_exc(*args):
+        raise RuntimeError()
+
+    f = c.submit(inc, 1, workers=[a.address], key="f")
+    g = c.submit(inc, 1, workers=[b.address], key="g")
+    res = c.submit(raise_exc, f, g, workers=[a.address])
+
+    with pytest.raises(RuntimeError):
+        await res.result()
+
+    # Nothing bad happened on B, therefore B should hold on to G
+    assert len(b.tasks) == 1
+    assert g.key in b.tasks
+
+    # A raised the exception therefore we should hold on to the erroneous task
+    assert res.key in a.tasks
+    ts = a.tasks[res.key]
+    assert ts.state == "error"
+
+    expected_states = {
+        # A was instructed to compute this result and we're still holding a ref
+        # via `f`
+        f.key: "memory",
+        # This was fetched from another worker. While we hold a ref via `g`, the
+        # scheduler only instructed to compute this on B
+        g.key: "memory",
+        res.key: "error",
+    }
+    assert_task_states_on_worker(expected_states, a)
+    # Expected states after we release references to the futures
+
+    res.release()
+    # We no longer hold any refs to f or g and B didn't have any erros. It
+    # releases everything as expected
+    while res.key in a.tasks:
+        await asyncio.sleep(0.01)
+
+    expected_states = {
+        f.key: "memory",
+    }
+
+    assert_task_states_on_worker(expected_states, a)
+
+    f.release()
+    g.release()
+
+    # This is not happening
+    for server in [s, a, b]:
+        while server.tasks:
+            await asyncio.sleep(0.01)
+
+
+@gen_cluster(client=True)
+async def test_worker_state_error_release_error_int(c, s, a, b):
+    """
+    Create a chain of tasks and err one of them. Then release tasks in a certain
+    order and ensure the tasks are released and/or kept in memory as appropriate
+
+    F -- RES (error)
+        /
+       /
+    G
+
+    Free one successful task, then error, then last task
+    """
+
+    def raise_exc(*args):
+        raise RuntimeError()
+
+    f = c.submit(inc, 1, workers=[a.address], key="f")
+    g = c.submit(inc, 1, workers=[b.address], key="g")
+    res = c.submit(raise_exc, f, g, workers=[a.address])
+
+    with pytest.raises(RuntimeError):
+        await res.result()
+
+    # Nothing bad happened on B, therefore B should hold on to G
+    assert len(b.tasks) == 1
+    assert g.key in b.tasks
+
+    # A raised the exception therefore we should hold on to the erroneous task
+    assert res.key in a.tasks
+    ts = a.tasks[res.key]
+    assert ts.state == "error"
+
+    expected_states = {
+        # A was instructed to compute this result and we're still holding a ref via `f`
+        f.key: "memory",
+        # This was fetched from another worker. While we hold a ref via `g`, the
+        # scheduler only instructed to compute this on B
+        g.key: "memory",
+        res.key: "error",
+    }
+    assert_task_states_on_worker(expected_states, a)
+    # Expected states after we release references to the futures
+
+    f.release()
+    res.release()
+    # We no longer hold any refs to f or g and B didn't have any erros. It
+    # releases everything as expected
+    while a.tasks:
+        await asyncio.sleep(0.01)
+
+    expected_states = {
+        g.key: "memory",
+    }
+
+    assert_task_states_on_worker(expected_states, b)
+
+    g.release()
+
+    # We no longer hold any refs. Cluster should reset completely
+    for server in [s, a, b]:
+        while server.tasks:
+            await asyncio.sleep(0.01)
+
+
+@gen_cluster(client=True)
+async def test_worker_state_error_long_chain(c, s, a, b):
+    def raise_exc(*args):
+        raise RuntimeError()
+
+    # f (A) --------> res (B)
+    #                /
+    # g (B) -> h (A)
+
+    f = c.submit(inc, 1, workers=[a.address], key="f", allow_other_workers=False)
+    g = c.submit(inc, 1, workers=[b.address], key="g", allow_other_workers=False)
+    h = c.submit(inc, g, workers=[a.address], key="h", allow_other_workers=False)
+    res = c.submit(
+        raise_exc, f, h, workers=[b.address], allow_other_workers=False, key="res"
+    )
+
+    with pytest.raises(RuntimeError):
+        await res.result()
+
+    expected_states_A = {
+        f.key: "memory",
+        g.key: "memory",
+        h.key: "memory",
+    }
+    await asyncio.sleep(0.05)
+    assert_task_states_on_worker(expected_states_A, a)
+
+    expected_states_B = {
+        f.key: "memory",
+        g.key: "memory",
+        h.key: "memory",
+        res.key: "error",
+    }
+    await asyncio.sleep(0.05)
+    assert_task_states_on_worker(expected_states_B, b)
+
+    f.release()
+
+    expected_states_A = {
+        g.key: "memory",
+        h.key: "memory",
+    }
+    await asyncio.sleep(0.05)
+    assert_task_states_on_worker(expected_states_A, a)
+
+    expected_states_B = {
+        f.key: "memory",
+        g.key: "memory",
+        h.key: "memory",
+        res.key: "error",
+    }
+    await asyncio.sleep(0.05)
+    assert_task_states_on_worker(expected_states_B, b)
+
+    g.release()
+
+    expected_states_A = {
+        h.key: "memory",
+    }
+    await asyncio.sleep(0.05)
+    assert_task_states_on_worker(expected_states_A, a)
+
+    # B must not forget a task since all have a still valid dependent
+    expected_states_B = {
+        f.key: "memory",
+        # We actually cannot hold on to G even though the graph would suggest
+        # otherwise. This is because H was only introduced as a dependency and
+        # the scheduler never told the worker how H fits into the big picture.
+        # Therefore, it thinks that G does not have any dependents anymore and
+        # releases it. Too bad. Once we have speculative task assignments this
+        # should be more exact since we should always tell the worker what's
+        # going on
+        # g.key: released,
+        h.key: "memory",
+        res.key: "error",
+    }
+    assert_task_states_on_worker(expected_states_B, b)
+    h.release()
+    await asyncio.sleep(0.05)
+
+    expected_states_A = {}
+    assert_task_states_on_worker(expected_states_A, a)
+    expected_states_B = {
+        f.key: "memory",
+        # See above
+        # g.key: released,
+        h.key: "memory",
+        res.key: "error",
+    }
+
+    assert_task_states_on_worker(expected_states_B, b)
+    res.release()
+
+    # We no longer hold any refs. Cluster should reset completely
+    for server in [s, a, b]:
+        while server.tasks:
+            await asyncio.sleep(0.01)
diff --git a/distributed/worker.py b/distributed/worker.py
index 0735043355d..e802af68c71 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -14,10 +14,9 @@
 from collections.abc import MutableMapping
 from contextlib import suppress
 from datetime import timedelta
-from functools import partial
 from inspect import isawaitable
 from pickle import PicklingError
-from typing import Dict, Iterable
+from typing import Dict, Iterable, Optional
 
 from tlz import first, keymap, merge, pluck  # noqa: F401
 from tornado import gen
@@ -188,6 +187,7 @@ def __init__(self, key, runspec=None):
         self.metadata = {}
         self.nbytes = None
         self.annotations = None
+        self.scheduler_holds_ref = False
 
     def __repr__(self):
         return "<Task %r %s>" % (self.key, self.state)
@@ -685,7 +685,7 @@ def __init__(
             "run_coroutine": self.run_coroutine,
             "get_data": self.get_data,
             "update_data": self.update_data,
-            "delete_data": self.delete_data,
+            "free_keys": self.handle_free_keys,
             "terminate": self.close,
             "ping": pingpong,
             "upload_file": self.upload_file,
@@ -706,8 +706,8 @@ def __init__(
         stream_handlers = {
             "close": self.close,
             "compute-task": self.add_task,
-            "release-task": partial(self.release_key, report=False),
-            "delete-data": self.delete_data,
+            "free-keys": self.handle_free_keys,
+            "superfluous-data": self.handle_superfluous_data,
             "steal-request": self.steal_request,
         }
 
@@ -1477,21 +1477,62 @@ def update_data(self, comm=None, data=None, report=True, serializers=None):
                 self.put_key_in_memory(ts, value)
                 ts.priority = None
                 ts.duration = None
+            ts.scheduler_holds_ref = True
 
             self.log.append((key, "receive-from-scatter"))
 
         if report:
+
+            self.log.append(
+                ("Notifying scheduler about in-memory in update-data", list(data))
+            )
             self.batched_stream.send({"op": "add-keys", "keys": list(data)})
         info = {"nbytes": {k: sizeof(v) for k, v in data.items()}, "status": "OK"}
         return info
 
-    def delete_data(self, comm=None, keys=None, report=True):
-        if keys:
-            for key in list(keys):
-                self.log.append((key, "delete"))
-                self.release_key(key, cause="delete data")
+    def handle_free_keys(self, comm=None, keys=None, reason=None):
+        """
+        Handler to be called by the scheduler.
+
+        The given keys are no longer referred to and required by the scheduler.
+        The worker is now allowed to release the key, if applicable.
+
+        This does not guarantee that the memory is released since the worker may
+        still decide to hold on to the data and task since it is required by an
+        upstream dependency.
+        """
+        self.log.append(("free-keys", keys, reason))
+        for key in keys:
+            ts = self.tasks.get(key)
+            if ts:
+                ts.scheduler_holds_ref = False
+            self.release_key(key, report=False, reason=reason)
+
+    def handle_superfluous_data(self, keys=(), reason=None):
+        """Stream handler notifying the worker that it might be holding unreferenced, superfluous data.
+
+        This should not actually happen during ordinary operations and is only
+        intended to correct any erroneous state. An example where this is
+        necessary is if a worker fetches data for a downstream task but that
+        task is released before the data arrives.
+        In this case, the scheduler will notify the worker that it may be
+        holding this unnecessary data, if the worker hasn't released the data itself, already.
+
+        This handler does not guarantee the task nor the data to be actually
+        released but only asks the worker to release the data on a best effort
+        guarantee. This protects from race conditions where the given keys may
+        already have been rescheduled for compute in which case the compute
+        would win and this handler is ignored.
+
+        For stronger guarantees, see handler free_keys
+        """
+        self.log.append(("Handle superfluous data", keys, reason))
+        for key in list(keys):
+            ts = self.tasks.get(key)
+            if ts and not ts.scheduler_holds_ref:
+                self.release_key(key, reason=f"delete data: {reason}", report=False)
 
-            logger.debug("Worker %s -- Deleted %d keys", self.name, len(keys))
+        logger.debug("Worker %s -- Deleted %d keys", self.name, len(keys))
         return "OK"
 
     async def set_resources(self, **resources):
@@ -1532,6 +1573,7 @@ def add_task(
             runspec = SerializedTask(function, args, kwargs, task)
             if key in self.tasks:
                 ts = self.tasks[key]
+                ts.scheduler_holds_ref = True
                 if ts.state == "memory":
                     assert key in self.data or key in self.actors
                     logger.debug(
@@ -1555,7 +1597,6 @@ def add_task(
                     key=key, runspec=SerializedTask(function, args, kwargs, task)
                 )
                 self.transition(ts, "waiting")
-
             # TODO: move transition of `ts` to end of `add_task`
             # This will require a chained recommendation transition system like
             # the scheduler
@@ -1567,6 +1608,7 @@ def add_task(
             if actor:
                 self.actors[ts.key] = None
 
+            ts.scheduler_holds_ref = True
             ts.runspec = runspec
             ts.priority = priority
             ts.duration = duration
@@ -1593,10 +1635,23 @@ def add_task(
 
                     # transition from new -> fetch handles adding dependency
                     # to waiting_for_data
-                    self.transition(dep_ts, state)
+                    discarded_self = False
+                    if self.address in workers and state == "fetch":
+                        discarded_self = True
+                        workers = set(workers)
+                        workers.discard(self.address)
+                        who_has[dependency] = tuple(workers)
+
+                    self.transition(dep_ts, state, who_has=workers)
 
                     self.log.append(
-                        (dependency, "new-dep", dep_ts.state, f"requested by {ts.key}")
+                        (
+                            dependency,
+                            "new-dep",
+                            dep_ts.state,
+                            f"requested by {ts.key}",
+                            discarded_self,
+                        )
                     )
 
                 else:
@@ -1610,13 +1665,8 @@ def add_task(
                 if dep_ts.state in ("fetch", "flight"):
                     # if we _need_ to grab data or are in the process
                     ts.waiting_for_data.add(dep_ts.key)
-                    # Ensure we know which workers to grab data from
-                    dep_ts.who_has.update(workers)
-
-                    for worker in workers:
-                        self.has_what[worker].add(dep_ts.key)
-                        self.pending_data_per_worker[worker].append(dep_ts.key)
 
+            self.update_who_has(who_has=who_has)
             if nbytes is not None:
                 for key, value in nbytes.items():
                     self.tasks[key].nbytes = value
@@ -1650,8 +1700,10 @@ def transition(self, ts, finish, **kwargs):
         if start == finish:
             return
         func = self._transitions[start, finish]
+        self.log.append((ts.key, start, finish))
         state = func(ts, **kwargs)
-        self.log.append((ts.key, start, state or finish))
+        if state and finish != state:
+            self.log.append((ts.key, start, finish, state))
         ts.state = state or finish
         if self.validate:
             self.validate_task(ts)
@@ -1671,15 +1723,21 @@ def transition_new_waiting(self, ts):
                 pdb.set_trace()
             raise
 
-    def transition_new_fetch(self, ts):
+    def transition_new_fetch(self, ts, who_has):
         try:
             if self.validate:
                 assert ts.state == "new"
                 assert ts.runspec is None
+                assert who_has
 
             for dependent in ts.dependents:
                 dependent.waiting_for_data.add(ts.key)
 
+            ts.who_has.update(who_has)
+            for w in who_has:
+                self.has_what[w].add(ts.key)
+                self.pending_data_per_worker[w].append(ts.key)
+
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
@@ -1767,19 +1825,22 @@ def transition_fetch_flight(self, ts, worker=None):
                 pdb.set_trace()
             raise
 
-    def transition_flight_fetch(self, ts, worker=None, runspec=None):
+    def transition_flight_fetch(self, ts):
         try:
             if self.validate:
                 assert ts.state == "flight"
 
             self.in_flight_tasks -= 1
             ts.coming_from = None
-            ts.runspec = runspec or ts.runspec
+            ts.runspec = None
 
             if not ts.who_has:
                 if ts.key not in self._missing_dep_flight:
                     self._missing_dep_flight.add(ts.key)
+                    logger.info("Task %s does not know who has", ts)
                     self.loop.add_callback(self.handle_missing_dep, ts)
+            for w in ts.who_has:
+                self.pending_data_per_worker[w].append(ts.key)
             for dependent in ts.dependents:
                 dependent.waiting_for_data.add(ts.key)
                 if dependent.state == "waiting":
@@ -1808,6 +1869,7 @@ def transition_flight_memory(self, ts, value=None):
                 except KeyError:
                     pass
 
+            self.log.append(("Notifying scheduler about in-memory", ts.key))
             self.batched_stream.send({"op": "add-keys", "keys": [ts.key]})
 
         except Exception as e:
@@ -2050,10 +2112,9 @@ def ensure_communicating(self):
                         else:
                             dependencies_fetch.add(dependency_ts)
 
-                del dependencies
+                del dependencies, dependency_ts
 
                 if dependencies_missing:
-                    logger.info("Can't find dependencies for key %s", key)
                     missing_deps2 = {
                         dep
                         for dep in dependencies_missing
@@ -2061,8 +2122,13 @@ def ensure_communicating(self):
                     }
                     for dep in missing_deps2:
                         self._missing_dep_flight.add(dep.key)
-                    self.loop.add_callback(self.handle_missing_dep, *missing_deps2)
-
+                    if missing_deps2:
+                        logger.info(
+                            "Can't find dependencies %s for key %s",
+                            missing_deps2.copy(),
+                            key,
+                        )
+                        self.loop.add_callback(self.handle_missing_dep, *missing_deps2)
                     dependencies_fetch -= dependencies_missing
 
                 self.log.append(
@@ -2099,6 +2165,7 @@ def ensure_communicating(self):
                     for d in to_gather:
                         dependencies_fetch.discard(self.tasks.get(d))
                         self.transition(self.tasks[d], "flight", worker=worker)
+                    assert not worker == self.address
                     self.loop.add_callback(
                         self.gather_dep,
                         worker=worker,
@@ -2238,6 +2305,9 @@ async def gather_dep(
         cause : TaskState
             Task we want to gather dependencies for
         """
+
+        if self.validate:
+            self.validate_state()
         if self.status != Status.running:
             return
         with log_errors():
@@ -2250,10 +2320,17 @@ async def gather_dep(
                     dependency_ts = self.tasks.get(dependency_key)
                     if dependency_ts and dependency_ts.state == "flight":
                         to_gather_keys.add(dependency_key)
-                del to_gather
+                # Keep namespace clean since this func is long and has many
+                # dep*, *ts* variables
+                del to_gather, dependency_key, dependency_ts
 
                 self.log.append(("request-dep", cause.key, worker, to_gather_keys))
-                logger.debug("Request %d keys for task %s", len(to_gather_keys), cause)
+                logger.debug(
+                    "Request %d keys for task %s from %s",
+                    len(to_gather_keys),
+                    cause,
+                    worker,
+                )
 
                 start = time()
                 response = await get_data_from_worker(
@@ -2321,9 +2398,12 @@ async def gather_dep(
                 self.log.append(("receive-dep", worker, list(response["data"])))
             except EnvironmentError:
                 logger.exception("Worker stream died during communication: %s", worker)
-                self.log.append(("receive-dep-failed", worker))
-                for d in self.has_what.pop(worker):
-                    self.tasks[d].who_has.remove(worker)
+                has_what = self.has_what.pop(worker)
+                self.pending_data_per_worker.pop(worker)
+                self.log.append(("receive-dep-failed", worker, has_what))
+                for d in has_what:
+                    ts = self.tasks[d]
+                    ts.who_has.remove(worker)
 
             except Exception as e:
                 logger.exception(e)
@@ -2337,6 +2417,10 @@ async def gather_dep(
                 busy = response.get("status", "") == "busy"
                 data = response.get("data", {})
 
+                # FIXME: We should not handle keys which were skipped by this coro. to_gather_keys is only a subset
+                assert set(to_gather_keys).issubset(
+                    set(self.in_flight_workers.get(worker))
+                )
                 for d in self.in_flight_workers.pop(worker):
 
                     ts = self.tasks.get(d)
@@ -2344,16 +2428,32 @@ async def gather_dep(
                     if not busy and d in data:
                         self.transition(ts, "memory", value=data[d])
                     elif ts is None or ts.state == "executing":
-                        self.release_key(d, cause="already executing at gather")
-                        continue
-                    elif ts.state not in ("ready", "memory"):
-                        self.transition(ts, "fetch", worker=worker)
-
-                    if not busy and d not in data and ts.dependents:
+                        self.log.append(("already-executing", d))
+                        self.release_key(d, reason="already executing at gather")
+                    elif ts.state == "flight" and not ts.dependents:
+                        self.log.append(("flight no-dependents", d))
+                        self.release_key(
+                            d, reason="In-flight task no longer has dependents."
+                        )
+                    elif (
+                        not busy
+                        and d not in data
+                        and ts.dependents
+                        and ts.state != "memory"
+                    ):
+                        ts.who_has.discard(worker)
+                        self.has_what[worker].discard(ts.key)
                         self.log.append(("missing-dep", d))
                         self.batched_stream.send(
                             {"op": "missing-data", "errant_worker": worker, "key": d}
                         )
+                        self.transition(ts, "fetch")
+                    elif ts.state not in ("ready", "memory"):
+                        self.transition(ts, "fetch")
+                    else:
+                        logger.debug(
+                            "Unexpected task state encountered for %s after gather_dep"
+                        )
 
                 if self.validate:
                     self.validate_state()
@@ -2380,7 +2480,7 @@ def bad_dep(self, dep):
             ts.exception = msg["exception"]
             ts.traceback = msg["traceback"]
             self.transition(ts, "error")
-        self.release_key(dep.key, cause="bad dep")
+        self.release_key(dep.key, reason="bad dep")
 
     async def handle_missing_dep(self, *deps, **kwargs):
         self.log.append(("handle-missing", deps))
@@ -2408,18 +2508,40 @@ async def handle_missing_dep(self, *deps, **kwargs):
             )
             who_has = {k: v for k, v in who_has.items() if v}
             self.update_who_has(who_has)
+            still_missing = set()
             for dep in deps:
                 dep.suspicious_count += 1
 
                 if not who_has.get(dep.key):
+                    logger.info(
+                        "No workers found for %s",
+                        dep.key,
+                    )
                     self.log.append((dep.key, "no workers found", dep.dependents))
-                    self.release_key(dep.key)
+                    self.release_key(dep.key, reason="Handle missing no workers")
+                elif self.address in who_has and dep.state != "memory":
+
+                    still_missing.add(dep)
+                    self.batched_stream.send(
+                        {
+                            "op": "release-worker-data",
+                            "keys": [dep.key],
+                            "worker": self.address,
+                        }
+                    )
                 else:
+                    logger.debug("New workers found for %s", dep.key)
                     self.log.append((dep.key, "new workers found"))
                     for dependent in dep.dependents:
                         if dependent.key in dep.waiting_for_data:
                             self.data_needed.append(dependent.key)
-
+            if still_missing:
+                logger.debug(
+                    "Found self referencing who has response from scheduler for keys %s.\n"
+                    "Trying again handle_missing",
+                    deps,
+                )
+                await self.handle_missing_dep(*deps)
         except Exception:
             logger.error("Handle missing dep failed, retrying", exc_info=True)
             retries = kwargs.get("retries", 5)
@@ -2450,6 +2572,14 @@ def update_who_has(self, who_has):
                     continue
 
                 if dep in self.tasks:
+                    if self.address in workers and self.tasks[dep].state != "memory":
+                        logger.debug(
+                            "Scheduler claims worker %s holds data for task %s which is not true.",
+                            self.name,
+                            dep,
+                        )
+                        # Do not mutate the input dict. That's rude
+                        workers = set(workers) - {self.address}
                     self.tasks[dep].who_has.update(workers)
 
                     for worker in workers:
@@ -2479,20 +2609,41 @@ def steal_request(self, key):
             # If task is marked as "constrained" we haven't yet assigned it an
             # `available_resources` to run on, that happens in
             # `transition_constrained_executing`
-            self.release_key(ts.key, cause="stolen")
+            ts.scheduler_holds_ref = False
+            self.release_key(ts.key, reason="stolen")
             if self.validate:
                 assert ts.key not in self.tasks
 
-    def release_key(self, key, cause=None, reason=None, report=True):
+    def release_key(
+        self,
+        key: str,
+        cause: Optional[TaskState] = None,
+        reason: Optional[str] = None,
+        report: bool = True,
+    ):
         try:
+
             if self.validate:
                 assert isinstance(key, str)
             ts = self.tasks.get(key, TaskState(key=key))
-
+            # If the scheduler holds a reference which is usually the
+            # case when it instructed the task to be computed here or if
+            # data was scattered we must not release it unless the
+            # scheduler allow us to. See also handle_delete_data and
+            if ts and ts.scheduler_holds_ref:
+                return
+            logger.debug(
+                "Release key %s",
+                {
+                    "key": key,
+                    "cause": cause,
+                    "reason": reason,
+                },
+            )
             if cause:
-                self.log.append((key, "release-key", {"cause": cause}))
+                self.log.append((key, "release-key", {"cause": cause}, reason))
             else:
-                self.log.append((key, "release-key"))
+                self.log.append((key, "release-key", reason))
             if key in self.data and not ts.dependents:
                 try:
                     del self.data[key]
@@ -2504,14 +2655,16 @@ def release_key(self, key, cause=None, reason=None, report=True):
             # for any dependencies of key we are releasing remove task as dependent
             for dependency in ts.dependencies:
                 dependency.dependents.discard(ts)
-                # don't boot keys that are in flight
-                # we don't know if they're already queued up for transit
-                # in a gather_dep callback
-                if not dependency.dependents and dependency.state in (
-                    "waiting",
-                    "fetch",
+
+                if not dependency.dependents and dependency.state not in (
+                    # don't boot keys that are in flight
+                    # we don't know if they're already queued up for transit
+                    # in a gather_dep callback
+                    "flight",
+                    # The same is true for already executing keys.
+                    "executing",
                 ):
-                    self.release_key(dependency.key, cause=f"Dependent {ts} released")
+                    self.release_key(dependency.key, reason=f"Dependent {ts} released")
 
             for worker in ts.who_has:
                 self.has_what[worker].discard(ts.key)
@@ -2528,10 +2681,18 @@ def release_key(self, key, cause=None, reason=None, report=True):
                     for resource, quantity in ts.resource_restrictions.items():
                         self.available_resources[resource] += quantity
 
-            # Inform the scheduler of keys which will have gone missing
-            # We are releasing them before they have completed
-            if report and ts.state in PROCESSING:
-                self.batched_stream.send({"op": "release", "key": key, "cause": cause})
+            if report:
+                # Inform the scheduler of keys which will have gone missing
+                # We are releasing them before they have completed
+                if ts.state in PROCESSING:
+                    msg = {"op": "release", "key": key, "cause": cause}
+                else:
+                    msg = {
+                        "op": "release-worker-data",
+                        "keys": [key],
+                        "worker": self.address,
+                    }
+                self.batched_stream.send(msg)
 
             self._notify_plugins("release_key", key, ts.state, cause, reason, report)
             if key in self.tasks and not ts.dependents:
@@ -2880,7 +3041,7 @@ async def execute(self, key, report=False):
             elif isinstance(result.pop("actual-exception"), Reschedule):
                 self.batched_stream.send({"op": "reschedule", "key": ts.key})
                 self.transition(ts, "rescheduled", report=False)
-                self.release_key(ts.key, report=False)
+                self.release_key(ts.key, report=False, reason="Reschedule")
             else:
                 ts.exception = result["exception"]
                 ts.traceback = result["traceback"]
@@ -3184,11 +3345,33 @@ def validate_task_waiting(self, ts):
     def validate_task_flight(self, ts):
         assert ts.key not in self.data
         assert not any(dep.key in self.ready for dep in ts.dependents)
+        assert ts.coming_from
+        assert ts.coming_from in self.in_flight_workers
         assert ts.key in self.in_flight_workers[ts.coming_from]
 
     def validate_task_fetch(self, ts):
         assert ts.runspec is None
         assert ts.key not in self.data
+        assert self.address not in ts.who_has  #!!!!!!!!
+        # FIXME This is currently not an invariant since upon comm failure we
+        # remove the erroneous worker from all who_has and correct the state
+        # upon the next ensure_communicate
+
+        # if not ts.who_has:
+        #     # If we do not know who_has for a fetch task, it must be logged in
+        #     # the missing dep. There should be a handle_missing_dep running for
+        #     # all of these keys
+
+        #     assert ts.key in self._missing_dep_flight, (
+        #         ts.key,
+        #         self.story(ts),
+        #         self._missing_dep_flight.copy(),
+        #         self.in_flight_workers.copy(),
+        #     )
+        assert ts.dependents
+
+        for w in ts.who_has:
+            assert ts.key in self.has_what[w]
 
     def validate_task(self, ts):
         try:
@@ -3229,7 +3412,7 @@ def validate_state(self):
                     # dependency can still be in `memory` before GC grabs it...?
                     # Might need better bookkeeping
                     assert dep.state is not None
-                    assert ts in dep.dependents
+                    assert ts in dep.dependents, ts
                 for key in ts.waiting_for_data:
                     ts_wait = self.tasks[key]
                     assert (
@@ -3251,6 +3434,7 @@ def validate_state(self):
                 self.validate_task(ts)
 
         except Exception as e:
+            self.loop.add_callback(self.close)
             logger.exception(e)
             if LOG_PDB:
                 import pdb

From e690e82cd26b39fe87a4dd1886b71e6dc31c5a26 Mon Sep 17 00:00:00 2001
From: ArtinSarraf <sarraf.artin@gmail.com>
Date: Mon, 14 Jun 2021 05:02:30 -0400
Subject: [PATCH 1309/1550] Locally rerun successfully completed futures
 (#4813)

* initial commit, add recreate_task_locally, rename some classes/files

* make code more organized/simple

* simplify api, update docstrings.

* add recreate_task_locally tests, update recreate_error_locally tests

* use future key instead of future, formatting

* remove unneccesary import

Co-authored-by: Artin Sarraf <asarraf@Artins-MacBook-Pro.local>
Co-authored-by: James Bourbeau <jrbourbeau@gmail.com>
Co-authored-by: darle <darle@mit.edu>
---
 distributed/client.py              |   4 +-
 distributed/recreate_exceptions.py | 179 -------------------------
 distributed/recreate_tasks.py      | 203 +++++++++++++++++++++++++++++
 distributed/scheduler.py           |   4 +-
 distributed/tests/test_client.py   | 149 ++++++++++++++++-----
 5 files changed, 320 insertions(+), 219 deletions(-)
 delete mode 100644 distributed/recreate_exceptions.py
 create mode 100644 distributed/recreate_tasks.py

diff --git a/distributed/client.py b/distributed/client.py
index 582aa242c49..21f949088a9 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -754,9 +754,9 @@ def __init__(
         self.start(timeout=timeout)
         Client._instances.add(self)
 
-        from distributed.recreate_exceptions import ReplayExceptionClient
+        from distributed.recreate_tasks import ReplayTaskClient
 
-        ReplayExceptionClient(self)
+        ReplayTaskClient(self)
 
     @contextmanager
     def as_current(self):
diff --git a/distributed/recreate_exceptions.py b/distributed/recreate_exceptions.py
deleted file mode 100644
index 6b498113b5e..00000000000
--- a/distributed/recreate_exceptions.py
+++ /dev/null
@@ -1,179 +0,0 @@
-import logging
-
-from dask.utils import stringify
-
-from .client import futures_of, wait
-from .utils import sync
-from .utils_comm import pack_data
-from .worker import _deserialize
-
-logger = logging.getLogger(__name__)
-
-
-class ReplayExceptionScheduler:
-    """A plugin for the scheduler to recreate exceptions locally
-
-    This adds the following routes to the scheduler
-
-    *  cause_of_failure
-    """
-
-    def __init__(self, scheduler):
-        self.scheduler = scheduler
-        self.scheduler.handlers["cause_of_failure"] = self.cause_of_failure
-        self.scheduler.extensions["exceptions"] = self
-
-    def cause_of_failure(self, *args, keys=(), **kwargs):
-        """
-        Return details of first failed task required by set of keys
-
-        Parameters
-        ----------
-        keys : list of keys known to the scheduler
-
-        Returns
-        -------
-        Dictionary with:
-        cause: the key that failed
-        task: the definition of that key
-        deps: keys that the task depends on
-        """
-        for key in keys:
-            if isinstance(key, list):
-                key = tuple(key)  # ensure not a list from msgpack
-            key = stringify(key)
-            ts = self.scheduler.tasks.get(key)
-            if ts is not None and ts.exception_blame is not None:
-                cause = ts.exception_blame
-                # NOTE: cannot serialize sets
-                return {
-                    "deps": [dts.key for dts in cause.dependencies],
-                    "cause": cause.key,
-                    "task": cause.run_spec,
-                }
-
-
-class ReplayExceptionClient:
-    """
-    A plugin for the client allowing replay of remote exceptions locally
-
-    Adds the following methods (and their async variants)to the given client:
-
-    - ``recreate_error_locally``: main user method
-    - ``get_futures_error``: gets the task, its details and dependencies,
-        responsible for failure of the given future.
-    """
-
-    def __init__(self, client):
-        self.client = client
-        self.client.extensions["exceptions"] = self
-        # monkey patch
-        self.client.recreate_error_locally = self.recreate_error_locally
-        self.client._recreate_error_locally = self._recreate_error_locally
-        self.client._get_futures_error = self._get_futures_error
-        self.client.get_futures_error = self.get_futures_error
-
-    @property
-    def scheduler(self):
-        return self.client.scheduler
-
-    async def _get_futures_error(self, future):
-        # only get errors for futures that errored.
-        futures = [f for f in futures_of(future) if f.status == "error"]
-        if not futures:
-            raise ValueError("No errored futures passed")
-        out = await self.scheduler.cause_of_failure(keys=[f.key for f in futures])
-        deps, task = out["deps"], out["task"]
-        if isinstance(task, dict):
-            function, args, kwargs = _deserialize(**task)
-            return (function, args, kwargs, deps)
-        else:
-            function, args, kwargs = _deserialize(task=task)
-            return (function, args, kwargs, deps)
-
-    def get_futures_error(self, future):
-        """
-        Ask the scheduler details of the sub-task of the given failed future
-
-        When a future evaluates to a status of "error", i.e., an exception
-        was raised in a task within its graph, we an get information from
-        the scheduler. This function gets the details of the specific task
-        that raised the exception and led to the error, but does not fetch
-        data from the cluster or execute the function.
-
-        Parameters
-        ----------
-        future : future that failed, having ``status=="error"``, typically
-            after an attempt to ``gather()`` shows a stack-stace.
-
-        Returns
-        -------
-        Tuple:
-        - the function that raised an exception
-        - argument list (a tuple), may include values and keys
-        - keyword arguments (a dictionary), may include values and keys
-        - list of keys that the function requires to be fetched to run
-
-        See Also
-        --------
-        ReplayExceptionClient.recreate_error_locally
-        """
-        return self.client.sync(self._get_futures_error, future)
-
-    async def _recreate_error_locally(self, future):
-        await wait(future)
-        out = await self._get_futures_error(future)
-        function, args, kwargs, deps = out
-        futures = self.client._graph_to_futures({}, deps)
-        data = await self.client._gather(futures)
-        args = pack_data(args, data)
-        kwargs = pack_data(kwargs, data)
-        return (function, args, kwargs)
-
-    def recreate_error_locally(self, future):
-        """
-        For a failed calculation, perform the blamed task locally for debugging.
-
-        This operation should be performed after a future (result of ``gather``,
-        ``compute``, etc) comes back with a status of "error", if the stack-
-        trace is not informative enough to diagnose the problem. The specific
-        task (part of the graph pointing to the future) responsible for the
-        error will be fetched from the scheduler, together with the values of
-        its inputs. The function will then be executed, so that ``pdb`` can
-        be used for debugging.
-
-        Examples
-        --------
-        >>> future = c.submit(div, 1, 0)         # doctest: +SKIP
-        >>> future.status                        # doctest: +SKIP
-        'error'
-        >>> c.recreate_error_locally(future)     # doctest: +SKIP
-        ZeroDivisionError: division by zero
-
-        If you're in IPython you might take this opportunity to use pdb
-
-        >>> %pdb                                 # doctest: +SKIP
-        Automatic pdb calling has been turned ON
-
-        >>> c.recreate_error_locally(future)     # doctest: +SKIP
-        ZeroDivisionError: division by zero
-              1 def div(x, y):
-        ----> 2     return x / y
-        ipdb>
-
-        Parameters
-        ----------
-        future : future or collection that failed
-            The same thing as was given to ``gather``, but came back with
-            an exception/stack-trace. Can also be a (persisted) dask collection
-            containing any errored futures.
-
-        Returns
-        -------
-        Nothing; the function runs and should raise an exception, allowing
-        the debugger to run.
-        """
-        func, args, kwargs = sync(
-            self.client.loop, self._recreate_error_locally, future
-        )
-        func(*args, **kwargs)
diff --git a/distributed/recreate_tasks.py b/distributed/recreate_tasks.py
new file mode 100644
index 00000000000..ec596bc4614
--- /dev/null
+++ b/distributed/recreate_tasks.py
@@ -0,0 +1,203 @@
+import logging
+
+from dask.utils import stringify
+
+from .client import futures_of, wait
+from .utils import sync
+from .utils_comm import pack_data
+from .worker import _deserialize
+
+logger = logging.getLogger(__name__)
+
+
+class ReplayTaskScheduler:
+    """A plugin for the scheduler to recreate tasks locally
+
+    This adds the following routes to the scheduler
+
+    *  get_runspec
+    *  get_error_cause
+    """
+
+    def __init__(self, scheduler):
+        self.scheduler = scheduler
+        self.scheduler.handlers["get_runspec"] = self.get_runspec
+        self.scheduler.handlers["get_error_cause"] = self.get_error_cause
+        self.scheduler.extensions["replay-tasks"] = self
+
+    def _process_key(self, key):
+        if isinstance(key, list):
+            key = tuple(key)  # ensure not a list from msgpack
+        key = stringify(key)
+        return key
+
+    def get_error_cause(self, *args, keys=(), **kwargs):
+        for key in keys:
+            key = self._process_key(key)
+            ts = self.scheduler.tasks.get(key)
+            if ts is not None and ts.exception_blame is not None:
+                return ts.exception_blame.key
+
+    def get_runspec(self, *args, key=None, **kwargs):
+        key = self._process_key(key)
+        ts = self.scheduler.tasks.get(key)
+        return {"task": ts.run_spec, "deps": [dts.key for dts in ts.dependencies]}
+
+
+class ReplayTaskClient:
+    """
+    A plugin for the client allowing replay of remote tasks locally
+
+    Adds the following methods to the given client:
+
+    - ``recreate_error_locally``: main user method for replaying failed tasks
+    - ``recreate_task_locally``: main user method for replaying any task
+    """
+
+    def __init__(self, client):
+        self.client = client
+        self.client.extensions["replay-tasks"] = self
+        # monkey patch
+        self.client._get_raw_components_from_future = (
+            self._get_raw_components_from_future
+        )
+        self.client._prepare_raw_components = self._prepare_raw_components
+        self.client._get_components_from_future = self._get_components_from_future
+        self.client._get_errored_future = self._get_errored_future
+        self.client.recreate_task_locally = self.recreate_task_locally
+        self.client.recreate_error_locally = self.recreate_error_locally
+
+    @property
+    def scheduler(self):
+        return self.client.scheduler
+
+    async def _get_raw_components_from_future(self, future):
+        """
+        For a given future return the func, args and kwargs and future
+        deps that would be executed remotely.
+        """
+        if isinstance(future, str):
+            key = future
+        else:
+            await wait(future)
+            key = future.key
+        spec = await self.scheduler.get_runspec(key=key)
+        deps, task = spec["deps"], spec["task"]
+        if isinstance(task, dict):
+            function, args, kwargs = _deserialize(**task)
+            return (function, args, kwargs, deps)
+        else:
+            function, args, kwargs = _deserialize(task=task)
+            return (function, args, kwargs, deps)
+
+    async def _prepare_raw_components(self, raw_components):
+        """
+        Take raw components and resolve future dependencies.
+        """
+        function, args, kwargs, deps = raw_components
+        futures = self.client._graph_to_futures({}, deps)
+        data = await self.client._gather(futures)
+        args = pack_data(args, data)
+        kwargs = pack_data(kwargs, data)
+        return (function, args, kwargs)
+
+    async def _get_components_from_future(self, future):
+        """
+        For a given future return the func, args and kwargs that would be
+        executed remotely. Any args/kwargs that are themselves futures will
+        be resolved to the return value of those futures.
+        """
+        raw_components = await self._get_raw_components_from_future(future)
+        return await self._prepare_raw_components(raw_components)
+
+    def recreate_task_locally(self, future):
+        """
+        For any calculation, whether it succeeded or failed, perform the task
+        locally for debugging.
+
+        This operation should be performed after a future (result of ``gather``,
+        ``compute``, etc) comes back with a status other than "pending". Cases
+        where you might want to debug a successfully completed future could
+        include a calculation that returns an unexpected results. A common
+        debugging process might include running the task locally in debug mode,
+        with `pdb.runcall`.
+
+        Examples
+        --------
+        >>> import pdb                                    # doctest: +SKIP
+        >>> future = c.submit(div, 1, 1)                  # doctest: +SKIP
+        >>> future.status                                 # doctest: +SKIP
+        'finished'
+        >>> pdb.runcall(c.recreate_task_locally, future)  # doctest: +SKIP
+
+        Parameters
+        ----------
+        future : future
+            The same thing as was given to ``gather``.
+
+        Returns
+        -------
+        Any; will return the result of the task future.
+        """
+        func, args, kwargs = sync(
+            self.client.loop, self._get_components_from_future, future
+        )
+        return func(*args, **kwargs)
+
+    async def _get_errored_future(self, future):
+        """
+        For a given future collection, return the first future that raised
+        an error.
+        """
+        await wait(future)
+        futures = [f.key for f in futures_of(future) if f.status == "error"]
+        if not futures:
+            raise ValueError("No errored futures passed")
+        cause_key = await self.scheduler.get_error_cause(keys=futures)
+        return cause_key
+
+    def recreate_error_locally(self, future):
+        """
+        For a failed calculation, perform the blamed task locally for debugging.
+
+        This operation should be performed after a future (result of ``gather``,
+        ``compute``, etc) comes back with a status of "error", if the stack-
+        trace is not informative enough to diagnose the problem. The specific
+        task (part of the graph pointing to the future) responsible for the
+        error will be fetched from the scheduler, together with the values of
+        its inputs. The function will then be executed, so that ``pdb`` can
+        be used for debugging.
+
+        Examples
+        --------
+        >>> future = c.submit(div, 1, 0)         # doctest: +SKIP
+        >>> future.status                        # doctest: +SKIP
+        'error'
+        >>> c.recreate_error_locally(future)     # doctest: +SKIP
+        ZeroDivisionError: division by zero
+
+        If you're in IPython you might take this opportunity to use pdb
+
+        >>> %pdb                                 # doctest: +SKIP
+        Automatic pdb calling has been turned ON
+
+        >>> c.recreate_error_locally(future)     # doctest: +SKIP
+        ZeroDivisionError: division by zero
+              1 def div(x, y):
+        ----> 2     return x / y
+        ipdb>
+
+        Parameters
+        ----------
+        future : future or collection that failed
+            The same thing as was given to ``gather``, but came back with
+            an exception/stack-trace. Can also be a (persisted) dask collection
+            containing any errored futures.
+
+        Returns
+        -------
+        Nothing; the function runs and should raise an exception, allowing
+        the debugger to run.
+        """
+        errored_future_key = sync(self.client.loop, self._get_errored_future, future)
+        return self.recreate_task_locally(errored_future_key)
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 45e38b24a51..a8571e8b627 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -61,7 +61,7 @@
 from .publish import PublishExtension
 from .pubsub import PubSubSchedulerExtension
 from .queues import QueueExtension
-from .recreate_exceptions import ReplayExceptionScheduler
+from .recreate_tasks import ReplayTaskScheduler
 from .security import Security
 from .semaphore import SemaphoreExtension
 from .stealing import WorkStealing
@@ -168,7 +168,7 @@ def nogil(func):
     LockExtension,
     MultiLockExtension,
     PublishExtension,
-    ReplayExceptionScheduler,
+    ReplayTaskScheduler,
     QueueExtension,
     VariableExtension,
     PubSubSchedulerExtension,
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index afb303171ed..b7db9544b45 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -4678,36 +4678,6 @@ async def test_dont_clear_waiting_data(c, s, a, b):
         await asyncio.sleep(0)
 
 
-@gen_cluster(client=True)
-async def test_get_future_error_simple(c, s, a, b):
-    f = c.submit(div, 1, 0)
-    await wait(f)
-    assert f.status == "error"
-
-    function, args, kwargs, deps = await c._get_futures_error(f)
-    # args contains only solid values, not keys
-    assert function.__name__ == "div"
-    with pytest.raises(ZeroDivisionError):
-        function(*args, **kwargs)
-
-
-@gen_cluster(client=True)
-async def test_get_futures_error(c, s, a, b):
-    x0 = delayed(dec)(2, dask_key_name="x0")
-    y0 = delayed(dec)(1, dask_key_name="y0")
-    x = delayed(div)(1, x0, dask_key_name="x")
-    y = delayed(div)(1, y0, dask_key_name="y")
-    tot = delayed(sum)(x, y, dask_key_name="tot")
-
-    f = c.compute(tot)
-    await wait(f)
-    assert f.status == "error"
-
-    function, args, kwargs, deps = await c._get_futures_error(f)
-    assert function.__name__ == "div"
-    assert args == (1, y0.key)
-
-
 @gen_cluster(client=True)
 async def test_recreate_error_delayed(c, s, a, b):
     x0 = delayed(dec)(2)
@@ -4720,7 +4690,8 @@ async def test_recreate_error_delayed(c, s, a, b):
 
     assert f.status == "pending"
 
-    function, args, kwargs = await c._recreate_error_locally(f)
+    error_f = await c._get_errored_future(f)
+    function, args, kwargs = await c._get_components_from_future(error_f)
     assert f.status == "error"
     assert function.__name__ == "div"
     assert args == (1, 0)
@@ -4739,7 +4710,8 @@ async def test_recreate_error_futures(c, s, a, b):
 
     assert f.status == "pending"
 
-    function, args, kwargs = await c._recreate_error_locally(f)
+    error_f = await c._get_errored_future(f)
+    function, args, kwargs = await c._get_components_from_future(error_f)
     assert f.status == "error"
     assert function.__name__ == "div"
     assert args == (1, 0)
@@ -4754,7 +4726,8 @@ async def test_recreate_error_collection(c, s, a, b):
     b = b.persist()
     f = c.compute(b)
 
-    function, args, kwargs = await c._recreate_error_locally(f)
+    error_f = await c._get_errored_future(f)
+    function, args, kwargs = await c._get_components_from_future(error_f)
     with pytest.raises(ZeroDivisionError):
         function(*args, **kwargs)
 
@@ -4771,13 +4744,15 @@ def make_err(x):
 
     df2 = df.a.map(make_err)
     f = c.compute(df2)
-    function, args, kwargs = await c._recreate_error_locally(f)
+    error_f = await c._get_errored_future(f)
+    function, args, kwargs = await c._get_components_from_future(error_f)
     with pytest.raises(ValueError):
         function(*args, **kwargs)
 
     # with persist
     df3 = c.persist(df2)
-    function, args, kwargs = await c._recreate_error_locally(df3)
+    error_f = await c._get_errored_future(df3)
+    function, args, kwargs = await c._get_components_from_future(error_f)
     with pytest.raises(ValueError):
         function(*args, **kwargs)
 
@@ -4788,7 +4763,8 @@ async def test_recreate_error_array(c, s, a, b):
     pytest.importorskip("scipy")
     z = (da.linalg.inv(da.zeros((10, 10), chunks=10)) + 1).sum()
     zz = z.persist()
-    func, args, kwargs = await c._recreate_error_locally(zz)
+    error_f = await c._get_errored_future(zz)
+    function, args, kwargs = await c._get_components_from_future(error_f)
     assert "0.,0.,0." in str(args).replace(" ", "")  # args contain actual arrays
 
 
@@ -4811,6 +4787,107 @@ def test_recreate_error_not_error(c):
         c.recreate_error_locally(f)
 
 
+@gen_cluster(client=True)
+async def test_recreate_task_delayed(c, s, a, b):
+    x0 = delayed(dec)(2)
+    y0 = delayed(dec)(2)
+    x = delayed(div)(1, x0)
+    y = delayed(div)(1, y0)
+    tot = delayed(sum)([x, y])
+
+    f = c.compute(tot)
+
+    assert f.status == "pending"
+
+    function, args, kwargs = await c._get_components_from_future(f)
+    assert f.status == "finished"
+    assert function.__name__ == "sum"
+    assert args == ([1, 1],)
+    assert function(*args, **kwargs) == 2
+
+
+@gen_cluster(client=True)
+async def test_recreate_task_futures(c, s, a, b):
+    x0 = c.submit(dec, 2)
+    y0 = c.submit(dec, 2)
+    x = c.submit(div, 1, x0)
+    y = c.submit(div, 1, y0)
+    tot = c.submit(sum, [x, y])
+    f = c.compute(tot)
+
+    assert f.status == "pending"
+
+    function, args, kwargs = await c._get_components_from_future(f)
+    assert f.status == "finished"
+    assert function.__name__ == "sum"
+    assert args == ([1, 1],)
+    assert function(*args, **kwargs) == 2
+
+
+@gen_cluster(client=True)
+async def test_recreate_task_collection(c, s, a, b):
+    b = db.range(10, npartitions=4)
+    b = b.map(lambda x: int(3628800 / (x + 1)))
+    b = b.persist()
+    f = c.compute(b)
+
+    function, args, kwargs = await c._get_components_from_future(f)
+    assert function(*args, **kwargs) == [
+        3628800,
+        1814400,
+        1209600,
+        907200,
+        725760,
+        604800,
+        518400,
+        453600,
+        403200,
+        362880,
+    ]
+
+    dd = pytest.importorskip("dask.dataframe")
+    import pandas as pd
+
+    df = dd.from_pandas(pd.DataFrame({"a": [0, 1, 2, 3, 4]}), chunksize=2)
+
+    df2 = df.a.map(lambda x: x + 1)
+    f = c.compute(df2)
+
+    function, args, kwargs = await c._get_components_from_future(f)
+    expected = pd.DataFrame({"a": [1, 2, 3, 4, 5]})["a"]
+    assert function(*args, **kwargs).equals(expected)
+
+    # with persist
+    df3 = c.persist(df2)
+    # recreate_task_locally only works with futures
+    with pytest.raises(AttributeError):
+        function, args, kwargs = await c._get_components_from_future(df3)
+
+    f = c.compute(df3)
+    function, args, kwargs = await c._get_components_from_future(f)
+    assert function(*args, **kwargs).equals(expected)
+
+
+@gen_cluster(client=True)
+async def test_recreate_task_array(c, s, a, b):
+    da = pytest.importorskip("dask.array")
+    z = (da.zeros((10, 10), chunks=10) + 1).sum()
+    f = c.compute(z)
+    function, args, kwargs = await c._get_components_from_future(f)
+    assert function(*args, **kwargs) == 100
+
+
+def test_recreate_task_sync(c):
+    x0 = c.submit(dec, 2)
+    y0 = c.submit(dec, 2)
+    x = c.submit(div, 1, x0)
+    y = c.submit(div, 1, y0)
+    tot = c.submit(sum, [x, y])
+    f = c.compute(tot)
+
+    assert c.recreate_task_locally(f) == 2
+
+
 @gen_cluster(client=True)
 async def test_retire_workers(c, s, a, b):
     assert set(s.workers) == {a.address, b.address}

From 0628bc814a9d36006cbae6cd8e565e413b912e47 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Mon, 14 Jun 2021 04:03:16 -0500
Subject: [PATCH 1310/1550] Removed unused utility functions (#4911)

---
 distributed/utils.py | 95 --------------------------------------------
 1 file changed, 95 deletions(-)

diff --git a/distributed/utils.py b/distributed/utils.py
index 2bac5c05b3c..5780cfb54e9 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -202,23 +202,6 @@ def get_ip_interface(ifname):
     raise ValueError("interface %r doesn't have an IPv4 address" % (ifname,))
 
 
-# FIXME: this breaks if changed to async def...
-@gen.coroutine
-def ignore_exceptions(coroutines, *exceptions):
-    """Process list of coroutines, ignoring certain exceptions
-
-    >>> coroutines = [cor(...) for ...]  # doctest: +SKIP
-    >>> x = yield ignore_exceptions(coroutines, TypeError)  # doctest: +SKIP
-    """
-    wait_iterator = gen.WaitIterator(*coroutines)
-    results = []
-    while not wait_iterator.done():
-        with suppress(*exceptions):
-            result = yield wait_iterator.next()
-            results.append(result)
-    raise gen.Return(results)
-
-
 async def All(args, quiet_exceptions=()):
     """Wait on many tasks at the same time
 
@@ -552,11 +535,6 @@ def tmp_text(filename, text):
             os.remove(fn)
 
 
-def clear_queue(q):
-    while not q.empty():
-        q.get_nowait()
-
-
 def is_kernel():
     """Determine if we're running within an IPython kernel
 
@@ -919,34 +897,6 @@ def ensure_bytes(s):
             ) from e
 
 
-def divide_n_among_bins(n, bins):
-    """
-    >>> divide_n_among_bins(12, [1, 1])
-    [6, 6]
-    >>> divide_n_among_bins(12, [1, 2])
-    [4, 8]
-    >>> divide_n_among_bins(12, [1, 2, 1])
-    [3, 6, 3]
-    >>> divide_n_among_bins(11, [1, 2, 1])
-    [2, 6, 3]
-    >>> divide_n_among_bins(11, [.1, .2, .1])
-    [2, 6, 3]
-    """
-    total = sum(bins)
-    acc = 0.0
-    out = []
-    for b in bins:
-        now = n / total * b + acc
-        now, acc = divmod(now, 1)
-        out.append(int(now))
-    return out
-
-
-def mean(seq):
-    seq = list(seq)
-    return sum(seq) / len(seq)
-
-
 def open_port(host=""):
     """Return a probably-open port
 
@@ -1000,29 +950,6 @@ def import_file(path):
     return loaded
 
 
-class itemgetter:
-    """A picklable itemgetter.
-
-    Examples
-    --------
-    >>> data = [0, 1, 2]
-    >>> get_1 = itemgetter(1)
-    >>> get_1(data)
-    1
-    """
-
-    __slots__ = ("index",)
-
-    def __init__(self, index):
-        self.index = index
-
-    def __call__(self, x):
-        return x[self.index]
-
-    def __reduce__(self):
-        return (itemgetter, (self.index,))
-
-
 def asciitable(columns, rows):
     """Formats an ascii table for given columns and rows.
 
@@ -1055,28 +982,6 @@ def nbytes(frame, _bytes_like=(bytes, bytearray)):
             return len(frame)
 
 
-def is_writeable(frame):
-    """
-    Check whether frame is writeable
-
-    Will return ``True`` if writeable, ``False`` if readonly, and
-    ``None`` if undetermined.
-    """
-    try:
-        return not memoryview(frame).readonly
-    except TypeError:
-        return None
-
-
-@contextmanager
-def time_warn(duration, text):
-    start = time()
-    yield
-    end = time()
-    if end - start > duration:
-        print("TIME WARNING", text, end - start)
-
-
 def deprecated(*, version_removed: str = None):
     """Decorator to mark a function as deprecated
 

From b707c296dd529b7b669531b7bc7dad96387e4a80 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Mon, 14 Jun 2021 04:05:17 -0500
Subject: [PATCH 1311/1550] Use more informative default name for WorkerPlugins
 (#4908)

---
 distributed/diagnostics/plugin.py                   |  2 +-
 distributed/diagnostics/tests/test_worker_plugin.py | 10 ++++++++++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/distributed/diagnostics/plugin.py b/distributed/diagnostics/plugin.py
index fb8aa9029b8..85be264205b 100644
--- a/distributed/diagnostics/plugin.py
+++ b/distributed/diagnostics/plugin.py
@@ -181,7 +181,7 @@ def _get_worker_plugin_name(plugin) -> str:
     if hasattr(plugin, "name"):
         return plugin.name
     else:
-        return funcname(plugin) + "-" + str(uuid.uuid4())
+        return funcname(type(plugin)) + "-" + str(uuid.uuid4())
 
 
 class PipInstall(WorkerPlugin):
diff --git a/distributed/diagnostics/tests/test_worker_plugin.py b/distributed/diagnostics/tests/test_worker_plugin.py
index 86eb11e49ed..f3aa8cc6daf 100644
--- a/distributed/diagnostics/tests/test_worker_plugin.py
+++ b/distributed/diagnostics/tests/test_worker_plugin.py
@@ -208,3 +208,13 @@ class EmptyPlugin:
         pass
 
     await c.register_worker_plugin(EmptyPlugin())
+
+
+@gen_cluster(nthreads=[("127.0.0.1", 1)], client=True)
+async def test_default_name(c, s, w):
+    class MyCustomPlugin(WorkerPlugin):
+        pass
+
+    await c.register_worker_plugin(MyCustomPlugin())
+    assert len(w.plugins) == 1
+    assert next(iter(w.plugins)).startswith("MyCustomPlugin-")

From eafd9d427806e8fdc587dc3bc0ca217fe4f4dfe4 Mon Sep 17 00:00:00 2001
From: Elliott Sales de Andrade <quantum.analyst@gmail.com>
Date: Mon, 14 Jun 2021 12:01:42 -0400
Subject: [PATCH 1312/1550] Remove importing `apply` from `dask.compatibility`
 (#4913)

---
 distributed/client.py               | 3 +--
 distributed/tests/test_scheduler.py | 2 +-
 distributed/worker.py               | 3 +--
 3 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 21f949088a9..cb2285320e8 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -27,11 +27,10 @@
 
 import dask
 from dask.base import collections_to_dsk, normalize_token, tokenize
-from dask.compatibility import apply
 from dask.core import flatten
 from dask.highlevelgraph import HighLevelGraph
 from dask.optimization import SubgraphCallable
-from dask.utils import ensure_dict, format_bytes, funcname, stringify
+from dask.utils import apply, ensure_dict, format_bytes, funcname, stringify
 
 try:
     from dask.delayed import single_key
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 1746be6d4fe..f1aeef606d2 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -17,7 +17,7 @@
 
 import dask
 from dask import delayed
-from dask.compatibility import apply
+from dask.utils import apply
 
 from distributed import Client, Nanny, Worker, fire_and_forget, wait
 from distributed.comm import Comm
diff --git a/distributed/worker.py b/distributed/worker.py
index e802af68c71..8cfa73c911f 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -23,10 +23,9 @@
 from tornado.ioloop import IOLoop, PeriodicCallback
 
 import dask
-from dask.compatibility import apply
 from dask.core import istask
 from dask.system import CPU_COUNT
-from dask.utils import format_bytes, funcname
+from dask.utils import apply, format_bytes, funcname
 
 from . import comm, preloading, profile, system, utils
 from .batched import BatchedSend

From 05c56214cca6bccb0de0cbf85d7f01990f6923d8 Mon Sep 17 00:00:00 2001
From: Martin Durant <martindurant@users.noreply.github.com>
Date: Mon, 14 Jun 2021 13:37:29 -0400
Subject: [PATCH 1313/1550] Allow actor exceptions to propagate (#4232)

---
 distributed/actor.py            | 26 ++++++++++++++----
 distributed/tests/test_actor.py | 45 +++++++++++++++++++++++++++++++
 distributed/worker.py           | 48 ++++++++++++++++++---------------
 3 files changed, 93 insertions(+), 26 deletions(-)

diff --git a/distributed/actor.py b/distributed/actor.py
index 0facdda4cb8..77b2cda67de 100644
--- a/distributed/actor.py
+++ b/distributed/actor.py
@@ -162,10 +162,17 @@ async def run_actor_function_on_worker():
                             await self._future
                         else:
                             raise OSError("Unable to contact Actor's worker")
-                    return result["result"]
+                    return result
 
                 if self._asynchronous:
-                    return asyncio.ensure_future(run_actor_function_on_worker())
+
+                    async def unwrap():
+                        result = await run_actor_function_on_worker()
+                        if result["status"] == "OK":
+                            return result["result"]
+                        raise result["exception"]
+
+                    return asyncio.ensure_future(unwrap())
                 else:
                     # TODO: this mechanism is error prone
                     # we should endeavor to make dask's standard code work here
@@ -187,7 +194,10 @@ async def get_actor_attribute_from_worker():
                 x = await self._worker_rpc.actor_attribute(
                     attribute=key, actor=self.key
                 )
-                return x["result"]
+                if x["status"] == "OK":
+                    return x["result"]
+                else:
+                    raise x["exception"]
 
             return self._sync(get_actor_attribute_from_worker)
 
@@ -237,10 +247,16 @@ def __await__(self):
 
     def result(self, timeout=None):
         try:
+            if isinstance(self._cached_result, Exception):
+                raise self._cached_result
             return self._cached_result
         except AttributeError:
-            self._cached_result = self.q.get(timeout=timeout)
-            return self._cached_result
+            out = self.q.get(timeout=timeout)
+            if out["status"] == "OK":
+                self._cached_result = out["result"]
+            else:
+                self._cached_result = out["exception"]
+        return self.result()
 
     def __repr__(self):
         return "<ActorFuture>"
diff --git a/distributed/tests/test_actor.py b/distributed/tests/test_actor.py
index d2298c1c2c2..851ee7e8b2a 100644
--- a/distributed/tests/test_actor.py
+++ b/distributed/tests/test_actor.py
@@ -581,3 +581,48 @@ async def test_async_deadlock(client, s, a, b):
     ac2 = await client.submit(UsesCounter, actor=True, workers=[ac._address])
 
     assert (await ac2.ado_inc(ac)) == 1
+
+
+def test_exception():
+    class MyException(Exception):
+        pass
+
+    class Broken:
+        def method(self):
+            raise MyException
+
+        @property
+        def prop(self):
+            raise MyException
+
+    with cluster(nworkers=2) as (cl, w):
+        client = Client(cl["address"])
+        ac = client.submit(Broken, actor=True).result()
+        acfut = ac.method()
+        with pytest.raises(MyException):
+            acfut.result()
+
+        with pytest.raises(MyException):
+            ac.prop
+
+
+@gen_cluster(client=True)
+async def test_exception_async(client, s, a, b):
+    class MyException(Exception):
+        pass
+
+    class Broken:
+        def method(self):
+            raise MyException
+
+        @property
+        def prop(self):
+            raise MyException
+
+    ac = await client.submit(Broken, actor=True)
+    acfut = ac.method()
+    with pytest.raises(MyException):
+        await acfut
+
+    with pytest.raises(MyException):
+        await ac.prop
diff --git a/distributed/worker.py b/distributed/worker.py
index 8cfa73c911f..7282c13a4a5 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -2829,30 +2829,36 @@ async def actor_execute(
         func = getattr(actor, function)
         name = key_split(key) + "." + function
 
-        if iscoroutinefunction(func):
-            result = await func(*args, **kwargs)
-        elif separate_thread:
-            result = await self.executor_submit(
-                name,
-                apply_function_actor,
-                args=(
-                    func,
-                    args,
-                    kwargs,
-                    self.execution_state,
+        try:
+            if iscoroutinefunction(func):
+                result = await func(*args, **kwargs)
+            elif separate_thread:
+                result = await self.executor_submit(
                     name,
-                    self.active_threads,
-                    self.active_threads_lock,
-                ),
-                executor=self.executors["actor"],
-            )
-        else:
-            result = func(*args, **kwargs)
-        return {"status": "OK", "result": to_serialize(result)}
+                    apply_function_actor,
+                    args=(
+                        func,
+                        args,
+                        kwargs,
+                        self.execution_state,
+                        name,
+                        self.active_threads,
+                        self.active_threads_lock,
+                    ),
+                    executor=self.executors["actor"],
+                )
+            else:
+                result = func(*args, **kwargs)
+            return {"status": "OK", "result": to_serialize(result)}
+        except Exception as ex:
+            return {"status": "error", "exception": to_serialize(ex)}
 
     def actor_attribute(self, comm=None, actor=None, attribute=None):
-        value = getattr(self.actors[actor], attribute)
-        return {"status": "OK", "result": to_serialize(value)}
+        try:
+            value = getattr(self.actors[actor], attribute)
+            return {"status": "OK", "result": to_serialize(value)}
+        except Exception as ex:
+            return {"status": "error", "exception": to_serialize(ex)}
 
     def meets_resource_constraints(self, key):
         ts = self.tasks[key]

From ec9b569bbc8b70fbc342e5c2e310fbcd3ac149f1 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Mon, 14 Jun 2021 15:12:42 -0500
Subject: [PATCH 1314/1550] Fix error state typo (#4898)

---
 distributed/worker.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/worker.py b/distributed/worker.py
index 7282c13a4a5..9cbb7d42979 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -1582,7 +1582,7 @@ def add_task(
                     return
                 if ts.state in IN_PLAY:
                     return
-                if ts.state == "erred":
+                if ts.state == "error":
                     ts.exception = None
                     ts.traceback = None
                 else:

From 42d631dea9f0e64f4553cf53230167457525eaae Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Tue, 15 Jun 2021 15:59:31 +0100
Subject: [PATCH 1315/1550] Add HTML repr to `scheduler_info` and incorporate
 into client and cluster reprs (#4857)

---
 distributed/client.py                         | 123 +++++++-----
 distributed/deploy/cluster.py                 | 156 +++++++-------
 distributed/deploy/local.py                   |  13 +-
 distributed/deploy/tests/test_spec_cluster.py |  10 +-
 distributed/objects.py                        | 190 ++++++++++++++++++
 distributed/tests/test_client.py              |   4 +-
 6 files changed, 373 insertions(+), 123 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index cb2285320e8..5475f480d97 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -52,7 +52,7 @@
 )
 from .diagnostics.plugin import UploadFile, WorkerPlugin, _get_worker_plugin_name
 from .metrics import time
-from .objects import HasWhat, WhoHas
+from .objects import HasWhat, SchedulerInfo, WhoHas
 from .protocol import to_serialize
 from .protocol.pickle import dumps, loads
 from .publish import Datasets
@@ -827,7 +827,10 @@ def dashboard_link(self):
             return self.cluster.dashboard_link
         except AttributeError:
             scheduler, info = self._get_scheduler_info()
-            protocol, rest = scheduler.address.split("://")
+            if scheduler is None:
+                return None
+            else:
+                protocol, rest = scheduler.address.split("://")
 
             port = info["services"]["dashboard"]
             if protocol == "inproc":
@@ -874,7 +877,7 @@ def _get_scheduler_info(self):
             info = self._scheduler_identity
             scheduler = self.scheduler
 
-        return scheduler, info
+        return scheduler, SchedulerInfo(info)
 
     def __repr__(self):
         # Note: avoid doing I/O here...
@@ -902,53 +905,83 @@ def __repr__(self):
                 self.scheduler.address,
             )
         else:
-            return "<%s: not connected>" % (self.__class__.__name__,)
+            return "<%s: No scheduler connected>" % (self.__class__.__name__,)
 
     def _repr_html_(self):
         scheduler, info = self._get_scheduler_info()
 
-        text = (
-            '<h3 style="text-align: left;">Client</h3>\n'
-            '<ul style="text-align: left; list-style: none; margin: 0; padding: 0;">\n'
-        )
-        if scheduler is not None:
-            text += "  <li><b>Scheduler: </b>%s</li>\n" % scheduler.address
+        if scheduler is None:
+            child_repr = """<p>No scheduler connected.</p>"""
+        elif self.cluster:
+            child_repr = f"""
+                <details>
+                <summary style="margin-bottom: 20px;"><h3 style="display: inline;">Cluster Info</h3></summary>
+                {self.cluster._repr_html_()}
+                </details>
+                """
         else:
-            text += "  <li><b>Scheduler: not connected</b></li>\n"
-
-        if info and "dashboard" in info["services"]:
-            text += (
-                "  <li><b>Dashboard: </b><a href='%(web)s' target='_blank'>%(web)s</a></li>\n"
-                % {"web": self.dashboard_link}
-            )
+            child_repr = f"""
+                <details>
+                <summary style="margin-bottom: 20px;"><h3 style="display: inline;">Scheduler Info</h3></summary>
+                {info._repr_html_()}
+                </details>
+                """
+
+        client_status = ""
+
+        if not self.cluster and not self.scheduler_file:
+            client_status += """
+                <tr>
+                    <td style="text-align: left;"><strong>Connection method:</strong> Direct</td>
+                    <td style="text-align: left;"></td>
+                </tr>
+                """
 
-        text += "</ul>\n"
-
-        if info:
-            workers = list(info["workers"].values())
-            cores = sum(w["nthreads"] for w in workers)
-            memory = [w["memory_limit"] for w in workers]
-            memory = format_bytes(sum(memory)) if all(memory) else ""
-
-            text2 = (
-                '<h3 style="text-align: left;">Cluster</h3>\n'
-                '<ul style="text-align: left; list-style:none; margin: 0; padding: 0;">\n'
-                "  <li><b>Workers: </b>%d</li>\n"
-                "  <li><b>Cores: </b>%d</li>\n"
-                "  <li><b>Memory: </b>%s</li>\n"
-                "</ul>\n"
-            ) % (len(workers), cores, memory)
-
-            return (
-                '<table style="border: 2px solid white;">\n'
-                "<tr>\n"
-                '<td style="vertical-align: top; border: 0px solid white">\n%s</td>\n'
-                '<td style="vertical-align: top; border: 0px solid white">\n%s</td>\n'
-                "</tr>\n</table>"
-            ) % (text, text2)
-
-        else:
-            return text
+        if self.cluster:
+            client_status += f"""
+                <tr>
+                    <td style="text-align: left;"><strong>Connection method:</strong> Cluster object</td>
+                    <td style="text-align: left;"><strong>Cluster type:</strong> {type(self.cluster).__name__}</td>
+                </tr>
+                """
+        elif self.scheduler_file:
+            client_status += f"""
+                <tr>
+                    <td style="text-align: left;"><strong>Connection method:</strong> Scheduler file</td>
+                    <td style="text-align: left;"><strong>Scheduler file:</strong> {self.scheduler_file}</td>
+                </tr>
+                """
+
+        if self.dashboard_link:
+            client_status += f"""
+                <tr>
+                    <td style="text-align: left;">
+                        <strong>Dashboard: </strong>
+                        <a href="{self.dashboard_link}">{self.dashboard_link}</a>
+                    </td>
+                    <td style="text-align: left;"></td>
+                </tr>
+                """
+
+        return f"""
+            <div>
+                <div style="
+                    width: 24px;
+                    height: 24px;
+                    background-color: #e1e1e1;
+                    border: 3px solid #9D9D9D;
+                    border-radius: 5px;
+                    position: absolute;"> </div>
+                <div style="margin-left: 48px;">
+                    <h3 style="margin-bottom: 0px;">Client</h3>
+                    <p style="color: #9D9D9D; margin-bottom: 0px;">{self.id}</p>
+                    <table style="width: 100%; text-align: left;">
+                    {client_status}
+                    </table>
+                    {child_repr}
+                </div>
+            </div>
+        """
 
     def start(self, **kwargs):
         """Start scheduler running in separate thread"""
@@ -1162,7 +1195,7 @@ async def _update_scheduler_info(self):
         if self.status not in ("running", "connecting"):
             return
         try:
-            self._scheduler_identity = await self.scheduler.identity()
+            self._scheduler_identity = SchedulerInfo(await self.scheduler.identity())
         except EnvironmentError:
             logger.debug("Not able to query scheduler for identity")
 
diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
index d71f68f0a90..8807cdec990 100644
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -13,6 +13,7 @@
 from dask.utils import format_bytes
 
 from ..core import Status
+from ..objects import SchedulerInfo
 from ..utils import (
     Log,
     Logs,
@@ -62,6 +63,7 @@ def __init__(self, asynchronous, quiet=False, name=None):
         self._cluster_manager_logs = []
         self.quiet = quiet
         self.scheduler_comm = None
+        self._adaptive = None
 
         if name is not None:
             self.name = name
@@ -72,7 +74,7 @@ def __init__(self, asynchronous, quiet=False, name=None):
     async def _start(self):
         comm = await self.scheduler_comm.live_comm()
         await comm.write({"op": "subscribe_worker_status"})
-        self.scheduler_info = await comm.read()
+        self.scheduler_info = SchedulerInfo(await comm.read())
         self._watch_worker_status_comm = comm
         self._watch_worker_status_task = asyncio.ensure_future(
             self._watch_worker_status(comm)
@@ -263,7 +265,11 @@ def dashboard_link(self):
             host = self.scheduler_address.split("://")[1].split("/")[0].split(":")[0]
             return format_dashboard_link(host, port)
 
-    def _widget_status(self):
+    def _scaling_status(self):
+        if self._adaptive and self._adaptive.periodic_callback:
+            mode = "Adaptive"
+        else:
+            mode = "Manual"
         workers = len(self.scheduler_info["workers"])
         if hasattr(self, "worker_spec"):
             requested = sum(
@@ -274,36 +280,14 @@ def _widget_status(self):
             requested = len(self.workers)
         else:
             requested = workers
-        cores = sum(v["nthreads"] for v in self.scheduler_info["workers"].values())
-        memory = sum(v["memory_limit"] for v in self.scheduler_info["workers"].values())
-        memory = format_bytes(memory)
-        text = """
-<div>
-  <style scoped>
-    .dataframe tbody tr th:only-of-type {
-        vertical-align: middle;
-    }
-
-    .dataframe tbody tr th {
-        vertical-align: top;
-    }
-
-    .dataframe thead th {
-        text-align: right;
-    }
-  </style>
-  <table style="text-align: right;">
-    <tr> <th>Workers</th> <td>%s</td></tr>
-    <tr> <th>Cores</th> <td>%d</td></tr>
-    <tr> <th>Memory</th> <td>%s</td></tr>
-  </table>
-</div>
-""" % (
-            workers if workers == requested else "%d / %d" % (workers, requested),
-            cores,
-            memory,
-        )
-        return text
+
+        worker_count = workers if workers == requested else f"{workers} / {requested}"
+        return f"""
+        <table>
+            <tr><td style="text-align: left;">Scaling mode: {mode}</td></tr>
+            <tr><td style="text-align: left;">Workers: {worker_count}</td></tr>
+        </table>
+        """
 
     def _widget(self):
         """Create IPython widget for display within a notebook"""
@@ -313,26 +297,23 @@ def _widget(self):
             pass
 
         try:
-            from ipywidgets import HTML, Accordion, Button, HBox, IntText, Layout, VBox
+            from ipywidgets import (
+                HTML,
+                Accordion,
+                Button,
+                HBox,
+                IntText,
+                Layout,
+                Tab,
+                VBox,
+            )
         except ImportError:
             self._cached_widget = None
             return None
 
         layout = Layout(width="150px")
 
-        if self.dashboard_link:
-            link = '<p><b>Dashboard: </b><a href="%s" target="_blank">%s</a></p>\n' % (
-                self.dashboard_link,
-                self.dashboard_link,
-            )
-        else:
-            link = ""
-
-        title = "<h2>%s</h2>" % self._cluster_class_name
-        title = HTML(title)
-        dashboard = HTML(link)
-
-        status = HTML(self._widget_status(), layout=Layout(min_width="150px"))
+        status = HTML(self._repr_html_())
 
         if self._supports_scaling:
             request = IntText(0, description="Workers", layout=layout)
@@ -368,12 +349,18 @@ def scale_cb(b):
         else:
             accordion = HTML("")
 
-        box = VBox([title, HBox([status, accordion]), dashboard])
+        scale_status = HTML(self._scaling_status())
 
-        self._cached_widget = box
+        tab = Tab()
+        tab.children = [status, VBox([scale_status, accordion])]
+        tab.set_title(0, "Status")
+        tab.set_title(1, "Scaling")
+
+        self._cached_widget = tab
 
         def update():
-            status.value = self._widget_status()
+            status.value = self._repr_html_()
+            scale_status.value = self._scaling_status()
 
         cluster_repr_interval = parse_timedelta(
             dask.config.get("distributed.deploy.cluster-repr-interval", default="ms")
@@ -382,25 +369,58 @@ def update():
         self.periodic_callbacks["cluster-repr"] = pc
         pc.start()
 
-        return box
-
-    def _repr_html_(self):
-        if self.dashboard_link:
-            dashboard = "<a href='{0}' target='_blank'>{0}</a>".format(
-                self.dashboard_link
-            )
-        else:
-            dashboard = "Not Available"
-        return (
-            "<div style='color: var(--jp-ui-font-color0, #000000); "
-            "background-color: var(--jp-layout-color2, #f2f2f2); display: inline-block; "
-            "padding: 10px; border: 1px solid var(--jp-border-color0, #999999);'>\n"
-            "  <h3>{cls}</h3>\n"
-            "  <ul>\n"
-            "    <li><b>Dashboard: </b>{dashboard}\n"
-            "  </ul>\n"
-            "</div>\n"
-        ).format(cls=self._cluster_class_name, dashboard=dashboard)
+        return tab
+
+    def _repr_html_(self, cluster_status=None):
+
+        if not cluster_status:
+            cluster_status = ""
+
+        cluster_status += f"""
+            <tr>
+                <td style="text-align: left;">
+                    <strong>Dashboard:</strong> <a href="{self.dashboard_link}">{self.dashboard_link}</a>
+                </td>
+                <td style="text-align: left;"><strong>Workers:</strong> {len(self.scheduler_info["workers"])}</td>
+            </tr>
+            <tr>
+                <td style="text-align: left;">
+                    <strong>Total threads:</strong>
+                    {sum([w["nthreads"] for w in self.scheduler_info["workers"].values()])}
+                </td>
+                <td style="text-align: left;">
+                    <strong>Total memory:</strong>
+                    {format_bytes(sum([w["memory_limit"] for w in self.scheduler_info["workers"].values()]))}
+                </td>
+            </tr>
+        """
+        try:
+            scheduler_info_repr = self.scheduler_info._repr_html_()
+        except AttributeError:
+            scheduler_info_repr = "Scheduler not started yet."
+
+        return f"""
+            <div class="jp-RenderedHTMLCommon jp-RenderedHTML jp-mod-trusted jp-OutputArea-output">
+                <div style="
+                    width: 24px;
+                    height: 24px;
+                    background-color: #e1e1e1;
+                    border: 3px solid #9D9D9D;
+                    border-radius: 5px;
+                    position: absolute;"> </div>
+                <div style="margin-left: 48px;">
+                    <h3 style="margin-bottom: 0px; margin-top: 0px;">{type(self).__name__}</h3>
+                    <p style="color: #9D9D9D; margin-bottom: 0px;">{self.name}</p>
+                    <table style="width: 100%; text-align: left;">
+                    {cluster_status}
+                    </table>
+                    <details>
+                    <summary style="margin-bottom: 20px;"><h3 style="display: inline;">Scheduler Info</h3></summary>
+                    {scheduler_info_repr}
+                    </details>
+                </div>
+            </div>
+        """
 
     def _ipython_display_(self, **kwargs):
         widget = self._widget()
diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index d68daeaa4b8..e759d8b1216 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -120,7 +120,7 @@ def __init__(
         interface=None,
         worker_class=None,
         scheduler_kwargs=None,
-        **worker_kwargs
+        **worker_kwargs,
     ):
         if ip is not None:
             # In the future we should warn users about this move
@@ -248,6 +248,17 @@ def start_worker(self, *args, **kwargs):
             "Please see the `cluster.scale` method instead."
         )
 
+    def _repr_html_(self, cluster_status=None):
+        if cluster_status is None:
+            cluster_status = ""
+        cluster_status += f"""
+            <tr>
+                <td style="text-align: left;"><strong>Status:</strong> {self.status.name}</td>
+                <td style="text-align: left;"><strong>Using processes:</strong> {self.processes}</td>
+            </tr>
+        """
+        return super()._repr_html_(cluster_status=cluster_status)
+
 
 clusters_to_close = weakref.WeakSet()
 
diff --git a/distributed/deploy/tests/test_spec_cluster.py b/distributed/deploy/tests/test_spec_cluster.py
index 40928ee96e2..ca96104de3b 100644
--- a/distributed/deploy/tests/test_spec_cluster.py
+++ b/distributed/deploy/tests/test_spec_cluster.py
@@ -362,12 +362,8 @@ async def test_widget(cleanup):
             await asyncio.sleep(0.01)
             assert time() < start + 1
 
-        text = cluster._widget_status()
-        assert "3" in text
-        assert "GB" in text or "GiB" in text
-
         cluster.scale(5)
-        assert "3 / 5" in cluster._widget_status()
+        assert "3 / 5" in cluster._scaling_status()
 
 
 @pytest.mark.asyncio
@@ -448,8 +444,8 @@ async def test_MultiWorker(cleanup):
             while "workers=4" not in repr(cluster):
                 await asyncio.sleep(0.1)
 
-            workers_line = re.search("(Workers.+)", cluster._widget_status()).group(1)
-            assert re.match("Workers.*<td>4</td>", workers_line)
+            workers_line = re.search("(Workers.+)", cluster._repr_html_()).group(1)
+            assert re.match("Workers.*4", workers_line)
 
             cluster.scale(1)
             await cluster
diff --git a/distributed/objects.py b/distributed/objects.py
index d53860514ec..521e19f8316 100644
--- a/distributed/objects.py
+++ b/distributed/objects.py
@@ -1,5 +1,11 @@
 """This file contains custom objects.
 These are mostly regular objects with more useful _repr_ and _repr_html_ methods."""
+import datetime
+from urllib.parse import urlparse
+
+from dask.utils import format_bytes, format_time_ago
+
+from distributed.utils import format_dashboard_link
 
 
 class HasWhat(dict):
@@ -65,3 +71,187 @@ def _repr_html_(self):
         """
 
         return output
+
+
+class SchedulerInfo(dict):
+    """A dictionary of information about the scheduler and workers."""
+
+    def _repr_html_(self):
+        dashboard_address = None
+        if "dashboard" in self["services"]:
+            host = urlparse(self["address"]).hostname
+            dashboard_address = format_dashboard_link(
+                host, self["services"]["dashboard"]
+            )
+
+        scheduler = f"""
+            <div>
+                <div style="
+                    width: 24px;
+                    height: 24px;
+                    background-color: #FFF7E5;
+                    border: 3px solid #FF6132;
+                    border-radius: 5px;
+                    position: absolute;"> </div>
+                <div style="margin-left: 48px;">
+                    <h3 style="margin-bottom: 0px;">{self["type"]}</h3>
+                    <p style="color: #9D9D9D; margin-bottom: 0px;">{self["id"]}</p>
+                    <table style="width: 100%; text-align: left;">
+                        <tr>
+                            <td style="text-align: left;"><strong>Comm:</strong> {self["address"]}</td>
+                            <td style="text-align: left;"><strong>Workers:</strong> {len(self["workers"])}</td>
+                        </tr>
+                        <tr>
+                            <td style="text-align: left;">
+                                <strong>Dashboard:</strong> <a href="{dashboard_address}">{dashboard_address}</a>
+                            </td>
+                            <td style="text-align: left;">
+                                <strong>Total threads:</strong>
+                                {sum([w["nthreads"] for w in self["workers"].values()])}
+                            </td>
+                        </tr>
+                        <tr>
+                            <td style="text-align: left;">
+                                <strong>Started:</strong>
+                                {format_time_ago(datetime.datetime.fromtimestamp(self["started"]))}
+                            </td>
+                            <td style="text-align: left;">
+                                <strong>Total memory:</strong>
+                                {format_bytes(sum([w["memory_limit"] for w in self["workers"].values()]))}
+                            </td>
+                        </tr>
+                    </table>
+                </div>
+            </div>
+        """
+
+        workers = ""
+        for worker_name in self["workers"]:
+            self["workers"][worker_name]["comm"] = worker_name
+        for worker in sorted(self["workers"].values(), key=lambda k: k["name"]):
+            dashboard_address = None
+            if "dashboard" in worker["services"]:
+                host = urlparse(worker["comm"]).hostname
+                dashboard_address = format_dashboard_link(
+                    host, worker["services"]["dashboard"]
+                )
+
+            metrics = ""
+
+            if "metrics" in worker:
+                metrics = f"""
+                <tr>
+                    <td style="text-align: left;">
+                        <strong>Tasks executing: </strong> {worker["metrics"]["executing"]}
+                    </td>
+                    <td style="text-align: left;">
+                        <strong>Tasks in memory: </strong> {worker["metrics"]["in_memory"]}
+                    </td>
+                </tr>
+                <tr>
+                    <td style="text-align: left;">
+                        <strong>Tasks ready: </strong> {worker["metrics"]["ready"]}
+                    </td>
+                    <td style="text-align: left;">
+                        <strong>Tasks in flight: </strong>{worker["metrics"]["in_flight"]}
+                    </td>
+                </tr>
+                <tr>
+                    <td style="text-align: left;"><strong>CPU usage:</strong> {worker["metrics"]["cpu"]}%</td>
+                    <td style="text-align: left;">
+                        <strong>Last seen: </strong>
+                        {format_time_ago(datetime.datetime.fromtimestamp(worker["last_seen"]))}
+                    </td>
+                </tr>
+                <tr>
+                    <td style="text-align: left;">
+                        <strong>Memory usage: </strong>
+                        {format_bytes(worker["metrics"]["memory"])}
+                    </td>
+                    <td style="text-align: left;">
+                        <strong>Spilled bytes: </strong>
+                        {format_bytes(worker["metrics"]["spilled_nbytes"])}
+                    </td>
+                </tr>
+                <tr>
+                    <td style="text-align: left;">
+                        <strong>Read bytes: </strong>
+                        {format_bytes(worker["metrics"]["read_bytes"])}
+                    </td>
+                    <td style="text-align: left;">
+                        <strong>Write bytes: </strong>
+                        {format_bytes(worker["metrics"]["write_bytes"])}
+                    </td>
+                </tr>
+                """
+
+            gpu = ""
+
+            if "gpu" in worker:
+                gpu = f"""
+                <tr>
+                    <td style="text-align: left;">
+                        <strong>GPU: </strong>{worker["gpu"]["name"]}
+                    </td>
+                    <td style="text-align: left;">
+                        <strong>GPU memory: </strong>
+                        {format_bytes(worker["gpu"]["memory-total"])}
+                    </td>
+                </tr>
+                """
+
+            workers += f"""
+            <div style="margin-bottom: 20px;">
+                <div style="width: 24px;
+                            height: 24px;
+                            background-color: #DBF5FF;
+                            border: 3px solid #4CC9FF;
+                            border-radius: 5px;
+                            position: absolute;"> </div>
+                <div style="margin-left: 48px;">
+                <details>
+                    <summary>
+                        <h4 style="margin-bottom: 0px; display: inline;">{worker["type"]}: {worker["name"]}</h4>
+                    </summary>
+                    <table style="width: 100%; text-align: left;">
+                        <tr>
+                            <td style="text-align: left;"><strong>Comm: </strong> {worker["comm"]}</td>
+                            <td style="text-align: left;"><strong>Total threads: </strong> {worker["nthreads"]}</td>
+                        </tr>
+                        <tr>
+                            <td style="text-align: left;">
+                                <strong>Dashboard: </strong>
+                                <a href="{dashboard_address}">{dashboard_address}</a>
+                            </td>
+                            <td style="text-align: left;">
+                                <strong>Memory: </strong>
+                                {format_bytes(worker["memory_limit"])}
+                            </td>
+                        </tr>
+                        <tr>
+                            <td style="text-align: left;"><strong>Nanny: </strong> {worker["nanny"]}</td>
+                            <td style="text-align: left;"></td>
+                        </tr>
+                        <tr>
+                            <td colspan="2" style="text-align: left;">
+                                <strong>Local directory: </strong>
+                                {worker["local_directory"]}
+                            </td>
+                        </tr>
+                        {gpu}
+                        {metrics}
+                    </table>
+                </details>
+                </div>
+            </div>
+            """
+
+        return f"""
+        <div style="">
+            {scheduler}
+            <details style="margin-left: 48px;">
+            <summary style="margin-bottom: 20px;"><h3 style="display: inline;">Workers</h3></summary>
+            {workers}
+            </details>
+        </div>
+        """
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index b7db9544b45..ec47479e157 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -1999,14 +1999,14 @@ def test_repr(loop):
             for func in funcs:
                 text = func(c)
                 assert c.scheduler.address in text
-                assert "threads=3" in text or "Cores: </b>3" in text
+                assert "threads=3" in text or "Total threads: </strong>" in text
                 assert "6.00 GB" in text or "5.59 GiB" in text
                 if "<table" not in text:
                     assert len(text) < 80
 
         for func in funcs:
             text = func(c)
-            assert "not connected" in text
+            assert "No scheduler connected" in text
 
 
 @gen_cluster(client=True)

From 6340b5b34a582ff1ffe3eaf9dd7f3aac33d0a983 Mon Sep 17 00:00:00 2001
From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com>
Date: Wed, 16 Jun 2021 12:49:38 -0400
Subject: [PATCH 1316/1550] Add scheduler log tab to performance reports
 (#4909)

Adds a tab to the performance report with the logs of the scheduler. Note that these are all the logs currently contained in the scheduler's log deque. and not just the logs generated in the performance_report context:
---
 distributed/dashboard/components/scheduler.py | 10 ++++-
 distributed/deploy/cluster.py                 | 13 +++----
 distributed/scheduler.py                      |  7 ++++
 distributed/tests/test_client.py              |  1 +
 distributed/tests/test_utils.py               |  5 +--
 distributed/utils.py                          | 37 ++++++++++++++-----
 6 files changed, 52 insertions(+), 21 deletions(-)

diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index 28250ff9d10..b00bcfb94b0 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -32,6 +32,7 @@
     value,
 )
 from bokeh.models.widgets import DataTable, TableColumn
+from bokeh.models.widgets.markups import Div
 from bokeh.palettes import Viridis11
 from bokeh.plotting import figure
 from bokeh.themes import Theme
@@ -69,7 +70,7 @@
 from distributed.diagnostics.task_stream import color_of as ts_color_of
 from distributed.diagnostics.task_stream import colors as ts_color_lookup
 from distributed.metrics import time
-from distributed.utils import format_time, log_errors, parse_timedelta
+from distributed.utils import Logs, format_time, log_errors, parse_timedelta
 
 if dask.config.get("distributed.dashboard.export-tool"):
     from distributed.dashboard.export_tool import ExportTool
@@ -2162,6 +2163,13 @@ def update(self):
         self.source.data.update(data)
 
 
+class SchedulerLogs:
+    def __init__(self, scheduler):
+        logs = Logs(scheduler.get_logs())._repr_html_()
+
+        self.root = Div(text=logs)
+
+
 def systemmonitor_doc(scheduler, extra, doc):
     with log_errors():
         sysmon = SystemMonitor(scheduler, sizing_mode="stretch_both")
diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
index 8807cdec990..2439ab6bba3 100644
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -15,8 +15,7 @@
 from ..core import Status
 from ..objects import SchedulerInfo
 from ..utils import (
-    Log,
-    Logs,
+    MultiLogs,
     format_dashboard_link,
     log_errors,
     parse_timedelta,
@@ -210,21 +209,19 @@ def _log(self, log):
             print(log)
 
     async def _get_logs(self, cluster=True, scheduler=True, workers=True):
-        logs = Logs()
+        logs = MultiLogs()
 
         if cluster:
-            logs["Cluster"] = Log(
-                "\n".join(line[1] for line in self._cluster_manager_logs)
-            )
+            logs["Cluster"] = self._cluster_manager_logs
 
         if scheduler:
             L = await self.scheduler_comm.get_logs()
-            logs["Scheduler"] = Log("\n".join(line for level, line in L))
+            logs["Scheduler"] = L
 
         if workers:
             d = await self.scheduler_comm.worker_logs(workers=workers)
             for k, v in d.items():
-                logs[k] = Log("\n".join(line for level, line in v))
+                logs[k] = v
 
         return logs
 
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index a8571e8b627..b8a756b07be 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -6939,6 +6939,11 @@ def profile_to_figure(state):
         sysmon = SystemMonitor(self, last_count=last_count, sizing_mode="stretch_both")
         sysmon.update()
 
+        # Scheduler logs
+        from distributed.dashboard.components.scheduler import SchedulerLogs
+
+        logs = SchedulerLogs(self)
+
         from bokeh.models import Div, Panel, Tabs
 
         import distributed
@@ -6997,12 +7002,14 @@ def profile_to_figure(state):
         )
         bandwidth_types = Panel(child=bandwidth_types.root, title="Bandwidth (Types)")
         system = Panel(child=sysmon.root, title="System")
+        logs = Panel(child=logs.root, title="Scheduler Logs")
 
         tabs = Tabs(
             tabs=[
                 html,
                 task_stream,
                 system,
+                logs,
                 compute,
                 workers,
                 scheduler,
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index ec47479e157..0c6b8c088b3 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -6374,6 +6374,7 @@ async def f(stacklevel):
     assert "Dask Performance Report" in data
     assert "x = da.random" in data
     assert "Threads: 4" in data
+    assert "distributed.scheduler - INFO - Clear task state" in data
     assert dask.__version__ in data
 
     # Stacklevel two captures code two frames back -- which in this case
diff --git a/distributed/tests/test_utils.py b/distributed/tests/test_utils.py
index 0a073484c93..bb274098e3f 100644
--- a/distributed/tests/test_utils.py
+++ b/distributed/tests/test_utils.py
@@ -19,9 +19,8 @@
 from distributed.utils import (
     LRU,
     All,
-    Log,
-    Logs,
     LoopRunner,
+    MultiLogs,
     TimeoutError,
     _maybe_complex,
     deprecated,
@@ -550,7 +549,7 @@ def test_format_bytes_compat():
 
 
 def test_logs():
-    d = Logs({"123": Log("Hello"), "456": Log("World!")})
+    d = MultiLogs({"123": [("INFO", "Hello")], "456": [("INFO", "World!")]})
     text = d._repr_html_()
     assert is_valid_xml("<div>" + text + "</div>")
     assert "Hello" in text
diff --git a/distributed/utils.py b/distributed/utils.py
index 5780cfb54e9..d8a22ac05d2 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -1269,25 +1269,44 @@ def parse_ports(port):
 is_coroutine_function = iscoroutinefunction
 
 
-class Log(str):
-    """A container for logs"""
+class Log(tuple):
+    """A container for a single log entry"""
+
+    level_styles = {
+        "WARNING": "font-weight: bold; color: orange;",
+        "CRITICAL": "font-weight: bold; color: orangered;",
+        "ERROR": "font-weight: bold; color: crimson;",
+    }
 
     def _repr_html_(self):
-        return "<pre><code>\n{log}\n</code></pre>".format(
-            log=html.escape(self.rstrip())
+        level, message = self
+
+        style = "font-family: monospace; margin: 0;"
+        style += self.level_styles.get(level, "")
+
+        return '<p style="{style}">{message}</p>'.format(
+            style=html.escape(style),
+            message=html.escape(message),
         )
 
 
-class Logs(dict):
-    """A container for multiple logs"""
+class Logs(list):
+    """A container for a list of log entries"""
+
+    def _repr_html_(self):
+        return "\n".join(Log(entry)._repr_html_() for entry in self)
+
+
+class MultiLogs(dict):
+    """A container for a dict mapping strings to lists of log entries"""
 
     def _repr_html_(self):
         summaries = [
             "<details>\n"
             "<summary style='display:list-item'>{title}</summary>\n"
-            "{log}\n"
-            "</details>".format(title=title, log=log._repr_html_())
-            for title, log in sorted(self.items())
+            "{logs}\n"
+            "</details>".format(title=title, logs=Logs(entries)._repr_html_())
+            for title, entries in sorted(self.items())
         ]
         return "\n".join(summaries)
 

From fced981adf6b8d95900cc3e089237201251f803f Mon Sep 17 00:00:00 2001
From: Peter Andreas Entschev <peter@entschev.com>
Date: Fri, 18 Jun 2021 00:43:43 +0200
Subject: [PATCH 1317/1550] Add configuration to enable/disable NVML
 diagnostics (#4893)

---
 distributed/diagnostics/nvml.py            |  7 +++++++
 distributed/diagnostics/tests/test_nvml.py | 19 +++++++++++++++++++
 distributed/distributed-schema.yaml        | 11 +++++++++++
 distributed/distributed.yaml               |  2 ++
 4 files changed, 39 insertions(+)

diff --git a/distributed/diagnostics/nvml.py b/distributed/diagnostics/nvml.py
index 2acbe590ee5..19cc843eee9 100644
--- a/distributed/diagnostics/nvml.py
+++ b/distributed/diagnostics/nvml.py
@@ -1,5 +1,7 @@
 import os
 
+import dask
+
 try:
     import pynvml
 except ImportError:
@@ -12,6 +14,11 @@
 
 def init_once():
     global nvmlInitialized, nvmlLibraryNotFound, nvmlOwnerPID
+
+    if dask.config.get("distributed.diagnostics.nvml") is False:
+        nvmlInitialized = False
+        return
+
     if pynvml is None or (nvmlInitialized is True and nvmlOwnerPID == os.getpid()):
         return
 
diff --git a/distributed/diagnostics/tests/test_nvml.py b/distributed/diagnostics/tests/test_nvml.py
index 4a57ffa8317..1dcdf353dbd 100644
--- a/distributed/diagnostics/tests/test_nvml.py
+++ b/distributed/diagnostics/tests/test_nvml.py
@@ -4,6 +4,8 @@
 
 pynvml = pytest.importorskip("pynvml")
 
+import dask
+
 from distributed.diagnostics import nvml
 from distributed.utils_test import gen_cluster
 
@@ -19,6 +21,23 @@ def test_one_time():
     assert len(output["name"]) > 0
 
 
+def test_enable_disable_nvml():
+    try:
+        pynvml.nvmlShutdown()
+    except pynvml.NVMLError_Uninitialized:
+        pass
+    else:
+        nvml.nvmlInitialized = False
+
+    with dask.config.set({"distributed.diagnostics.nvml": False}):
+        nvml.init_once()
+        assert nvml.nvmlInitialized is False
+
+    with dask.config.set({"distributed.diagnostics.nvml": True}):
+        nvml.init_once()
+        assert nvml.nvmlInitialized is True
+
+
 def test_1_visible_devices():
     if nvml.device_get_count() < 1:
         pytest.skip("No GPUs available")
diff --git a/distributed/distributed-schema.yaml b/distributed/distributed-schema.yaml
index 86c00a8f318..80f7adce25f 100644
--- a/distributed/distributed-schema.yaml
+++ b/distributed/distributed-schema.yaml
@@ -750,6 +750,17 @@ properties:
                       Alternatively, the key can be appended to the cert file
                       above, and this field left blank
 
+      diagnostics:
+        type: object
+        properties:
+          nvml:
+            type: boolean
+            description: |
+              If ``True``, enables GPU diagnostics with NVML. Generally leaving it enabled is
+              not a problem and will be automatically disabled if no GPUs are found in the
+              system, but in certain cases it may be desirable to completely disable NVML
+              diagnostics.
+
       dashboard:
         type: object
         properties:
diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index a55700cd085..f948c45fd99 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -195,6 +195,8 @@ distributed:
         key: null
         cert: null
 
+  diagnostics:
+    nvml: True
 
   ###################
   # Bokeh dashboard #

From 2d99691d8bd03c69d828fa3e1b09d9bb47a8e610 Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Fri, 18 Jun 2021 18:08:29 +0200
Subject: [PATCH 1318/1550] Fix deadlock in `handle_missing_dep` if additional
 replicas are available (#4929)

---
 distributed/tests/test_failed_workers.py | 48 ++++++++++++++++++++++++
 distributed/worker.py                    |  2 +-
 2 files changed, 49 insertions(+), 1 deletion(-)

diff --git a/distributed/tests/test_failed_workers.py b/distributed/tests/test_failed_workers.py
index 32c872dec61..fb3922a8b99 100644
--- a/distributed/tests/test_failed_workers.py
+++ b/distributed/tests/test_failed_workers.py
@@ -620,3 +620,51 @@ async def test_forget_data_not_supposed_to_have(s, a, b):
     assert a.data
     while a.data:
         await asyncio.sleep(0.001)
+
+
+@gen_cluster(
+    client=True,
+    nthreads=[("127.0.0.1", 1) for _ in range(3)],
+    config={"distributed.comm.timeouts.connect": "1s"},
+    Worker=Nanny,
+)
+async def test_failing_worker_with_additional_replicas_on_cluster(c, s, *workers):
+    """
+    If a worker detects a missing dependency, the scheduler is notified. If no
+    other replica is available, the dependency is rescheduled. A reschedule
+    typically causes a lot of state to be reset. However, if another replica is
+    available, we'll need to ensure that the worker can detect outdated state
+    and correct its state.
+    """
+
+    def slow_transfer(x, delay=0.1):
+        return SlowTransmitData(x, delay=delay)
+
+    def dummy(*args, **kwargs):
+        return
+
+    import psutil
+
+    proc = psutil.Process(workers[1].pid)
+    f1 = c.submit(
+        slow_transfer,
+        1,
+        key="f1",
+        workers=[workers[0].worker_address],
+    )
+    # We'll schedule tasks on two workers, s.t. f1 is replicated. We will
+    # suspend one of the workers and kill the origin worker of f1 such that a
+    # comm failure causes the worker to handle a missing dependency. It will ask
+    # the schedule such that it knows that a replica is available on f2 and
+    # reschedules the fetch
+    f2 = c.submit(dummy, f1, pure=False, key="f2", workers=[workers[1].worker_address])
+    f3 = c.submit(dummy, f1, pure=False, key="f3", workers=[workers[2].worker_address])
+
+    await wait(f1)
+    proc.suspend()
+
+    await wait(f3)
+    await workers[0].close()
+
+    proc.resume()
+    await c.gather([f1, f2, f3])
diff --git a/distributed/worker.py b/distributed/worker.py
index 9cbb7d42979..bd1bf6f0d75 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -2532,7 +2532,7 @@ async def handle_missing_dep(self, *deps, **kwargs):
                     logger.debug("New workers found for %s", dep.key)
                     self.log.append((dep.key, "new workers found"))
                     for dependent in dep.dependents:
-                        if dependent.key in dep.waiting_for_data:
+                        if dep.key in dependent.waiting_for_data:
                             self.data_needed.append(dependent.key)
             if still_missing:
                 logger.debug(

From edc1238203255bd5f7d5a69e8aa08dab8b9d95ec Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Fri, 18 Jun 2021 11:58:37 -0500
Subject: [PATCH 1319/1550] bump version to 2021.06.1

---
 docs/source/changelog.rst | 29 ++++++++++++++++++++++++++++-
 requirements.txt          |  2 +-
 2 files changed, 29 insertions(+), 2 deletions(-)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index 8759f49665d..d346a6c0a1c 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,32 @@
 Changelog
 =========
 
+2021.06.1
+---------
+
+Released on June 18, 2021
+
+- Fix deadlock in ``handle_missing_dep`` if additional replicas are available (:pr:`4929`) `Florian Jetter`_
+- Add configuration to enable/disable NVML diagnostics (:pr:`4893`) `Peter Andreas Entschev`_
+- Add scheduler log tab to performance reports (:pr:`4909`) `Charles Blackmon-Luca`_
+- Add HTML repr to ``scheduler_info`` and incorporate into client and cluster reprs (:pr:`4857`) `Jacob Tomlinson`_
+- Fix error state typo (:pr:`4898`) `James Bourbeau`_
+- Allow actor exceptions to propagate (:pr:`4232`) `Martin Durant`_
+- Remove importing ``apply`` from ``dask.compatibility`` (:pr:`4913`) `Elliott Sales de Andrade`_
+- Use more informative default name for ``WorkerPlugin`` s (:pr:`4908`) `James Bourbeau`_
+- Removed unused utility functions (:pr:`4911`) `James Bourbeau`_
+- Locally rerun successfully completed futures (:pr:`4813`) `ArtinSarraf`_
+- Forget erred tasks and fix deadlocks on worker (:pr:`4784`) `Florian Jetter`_
+- Handle ``HTTPClientError`` in websocket connector (:pr:`4900`) `Marcos Moyano`_
+- Update ``dask_cuda`` usage in ``SSHCluster`` docstring (:pr:`4894`) `James Bourbeau`_
+- Remove tests for ``process_time`` and ``thread_time`` (:pr:`4895`) `James Bourbeau`_
+- Flake8 config cleanup (:pr:`4888`) `Florian Jetter`_
+- Don't strip scheduler protocol when determining host (:pr:`4883`) `James Bourbeau`_
+- Add more documentation on memory management (:pr:`4874`) `crusaderky`_
+- Add ``range_query`` tests to NVML test suite (:pr:`4879`) `Charles Blackmon-Luca`_
+- No longer cancel result future in async process when using timeouts (:pr:`4882`) `Florian Jetter`_
+
+
 2021.06.0
 ---------
 
@@ -2405,4 +2431,5 @@ significantly without many new features.
 .. _`Matteo De Wint`: https://github.com/mdwint
 .. _`Naty Clementi`: https://github.com/ncclementi
 .. _`Nathan Danielsen`: https://github.com/ndanielsen
-.. _`Torsten Wörtwein`: https://github.com/twoertwein
\ No newline at end of file
+.. _`Torsten Wörtwein`: https://github.com/twoertwein
+.. _`ArtinSarraf`: https://github.com/ArtinSarraf
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index d91f3c4e4b2..ad0339775d9 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,6 @@
 click >= 6.6
 cloudpickle >= 1.5.0
-dask == 2021.06.0
+dask == 2021.06.1
 msgpack >= 0.6.0
 psutil >= 5.0
 sortedcontainers !=2.0.0, !=2.0.1

From d9bc3c634a4a3983cdc75e5c39cbdac2afbf6184 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 18 Jun 2021 12:29:38 -0500
Subject: [PATCH 1320/1550] Remove nbytes_in_memory (#4930)

This doesn't appear to be necessary
---
 distributed/scheduler.py            | 19 -------------------
 distributed/tests/test_scheduler.py |  5 -----
 2 files changed, 24 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index b8a756b07be..9ebc33d3b4c 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -872,11 +872,6 @@ def __repr__(self):
             + ">"
         )
 
-    @property
-    def nbytes_in_memory(self):
-        tg: TaskGroup
-        return sum([tg._nbytes_in_memory for tg in self._groups])
-
     @property
     def nbytes_total(self):
         tg: TaskGroup
@@ -922,10 +917,6 @@ class TaskGroup:
 
        The total number of bytes that this task group has produced
 
-    .. attribute:: nbytes_in_memory: int
-
-       The number of bytes currently stored by this TaskGroup
-
     .. attribute:: duration: float
 
        The total amount of time spent on all tasks in this TaskGroup
@@ -944,7 +935,6 @@ class TaskGroup:
     _states: dict
     _dependencies: set
     _nbytes_total: Py_ssize_t
-    _nbytes_in_memory: Py_ssize_t
     _duration: double
     _types: set
     _start: double
@@ -958,7 +948,6 @@ def __init__(self, name: str):
         self._states["forgotten"] = 0
         self._dependencies = set()
         self._nbytes_total = 0
-        self._nbytes_in_memory = 0
         self._duration = 0
         self._types = set()
         self._start = 0.0
@@ -985,10 +974,6 @@ def dependencies(self):
     def nbytes_total(self):
         return self._nbytes_total
 
-    @property
-    def nbytes_in_memory(self):
-        return self._nbytes_in_memory
-
     @property
     def duration(self):
         return self._duration
@@ -1559,7 +1544,6 @@ def set_nbytes(self, nbytes: Py_ssize_t):
         if old_nbytes >= 0:
             diff -= old_nbytes
         self._group._nbytes_total += diff
-        self._group._nbytes_in_memory += diff
         ws: WorkerState
         for ws in self._who_has:
             ws._nbytes += diff
@@ -2646,7 +2630,6 @@ def transition_memory_released(self, key, safe: bint = False):
             for ws in ts._who_has:
                 del ws._has_what[ts]
                 ws._nbytes -= ts_nbytes
-                ts._group._nbytes_in_memory -= ts_nbytes
                 worker_msgs[ws._address] = [worker_msg]
 
             ts._who_has.clear()
@@ -7336,8 +7319,6 @@ def _propagate_forgotten(
     ts._waiting_on.clear()
 
     ts_nbytes: Py_ssize_t = ts.get_nbytes()
-    if ts._who_has:
-        ts._group._nbytes_in_memory -= ts_nbytes
 
     ws: WorkerState
     for ws in ts._who_has:
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index f1aeef606d2..3c613601841 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -1895,7 +1895,6 @@ async def test_task_groups(c, s, a, b):
     # these must be true since in this simple case there is a 1to1 mapping
     # between prefix and group
     assert tg.duration == tp.duration
-    assert tg.nbytes_in_memory == tp.nbytes_in_memory
     assert tg.nbytes_total == tp.nbytes_total
     # It should map down to individual tasks
     assert tg.nbytes_total == sum(
@@ -1908,8 +1907,6 @@ async def test_task_groups(c, s, a, b):
             if ts.group is tg and ts.state == "memory"
         ]
     )
-    assert tg.nbytes_in_memory == in_memory_ts
-
     tg = s.task_groups[y.name]
     assert tg.states["memory"] == 5
 
@@ -1917,7 +1914,6 @@ async def test_task_groups(c, s, a, b):
 
     await c.replicate(y)
     # TODO: Are we supposed to track replicated memory here? See also Scheduler.add_keys
-    assert tg.nbytes_in_memory == y.nbytes
     assert "array" in str(tg.types)
     assert "array" in str(tp.types)
 
@@ -1926,7 +1922,6 @@ async def test_task_groups(c, s, a, b):
     while s.tasks:
         await asyncio.sleep(0.01)
 
-    assert tg.nbytes_in_memory == 0
     assert tg.states["forgotten"] == 5
     # Ensure TaskGroup is removed once all tasks are in forgotten state
     assert tg.name not in s.task_groups

From 6d15b013ebe59f07e6f563cf73a4e8a7268d260c Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 21 Jun 2021 13:57:52 -0500
Subject: [PATCH 1321/1550] Add transition counter to Scheduler (#4934)

---
 distributed/scheduler.py            | 7 +++++++
 distributed/tests/test_scheduler.py | 7 +++++++
 2 files changed, 14 insertions(+)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 9ebc33d3b4c..a2996482ac9 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1770,6 +1770,7 @@ class SchedulerState:
     _validate: bint
     _workers: object
     _workers_dv: dict
+    _transition_counter: Py_ssize_t
 
     # Variables from dask.config, cached by __init__ for performance
     UNKNOWN_TASK_DURATION: double
@@ -1873,6 +1874,7 @@ def __init__(
             dask.config.get("distributed.worker.memory.rebalance.sender-recipient-gap")
             / 2.0
         )
+        self._transition_counter = 0
 
         super().__init__(**kwargs)
 
@@ -1940,6 +1942,10 @@ def total_occupancy(self):
     def total_occupancy(self, v: double):
         self._total_occupancy = v
 
+    @property
+    def transition_counter(self):
+        return self._transition_counter
+
     @property
     def unknown_durations(self):
         return self._unknown_durations
@@ -2065,6 +2071,7 @@ def _transition(self, key, finish: str, *args, **kwargs):
             func = self._transitions_table.get(start_finish)
             if func is not None:
                 a: tuple = func(key, *args, **kwargs)
+                self._transition_counter += 1
                 recommendations, client_msgs, worker_msgs = a
             elif "released" not in start_finish:
                 assert not args and not kwargs
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 3c613601841..04f266f8e61 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -2795,3 +2795,10 @@ async def test_rebalance_least_recently_inserted_sender_min(c, s, *_):
         a: (large_future.key,),
         b: tuple(f.key for f in small_futures),
     }
+
+
+@gen_cluster(client=True)
+async def test_transition_counter(c, s, a, b):
+    assert s.transition_counter == 0
+    await c.submit(inc, 1)
+    assert s.transition_counter > 1

From 9af19d46131384e92a6db989ed3c034060058fa1 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Mon, 21 Jun 2021 14:58:24 -0500
Subject: [PATCH 1322/1550] Use deprecation utility from Dask (#4924)

---
 distributed/client.py                    | 16 ++++++++------
 distributed/deploy/cluster.py            |  5 ++---
 distributed/deploy/tests/test_cluster.py |  7 ++++++
 distributed/tests/test_client.py         | 10 +++++++++
 distributed/tests/test_utils.py          | 18 ---------------
 distributed/tests/test_worker.py         |  7 +-----
 distributed/utils.py                     | 28 ------------------------
 distributed/worker.py                    |  6 -----
 docs/requirements.txt                    |  2 +-
 9 files changed, 30 insertions(+), 69 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 5475f480d97..a0352e9c70e 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -30,7 +30,14 @@
 from dask.core import flatten
 from dask.highlevelgraph import HighLevelGraph
 from dask.optimization import SubgraphCallable
-from dask.utils import apply, ensure_dict, format_bytes, funcname, stringify
+from dask.utils import (
+    _deprecated,
+    apply,
+    ensure_dict,
+    format_bytes,
+    funcname,
+    stringify,
+)
 
 try:
     from dask.delayed import single_key
@@ -2538,6 +2545,7 @@ def run(self, function, *args, **kwargs):
         """
         return self.sync(self._run, function, *args, **kwargs)
 
+    @_deprecated(use_instead="Client.run which detects async functions automatically")
     def run_coroutine(self, function, *args, **kwargs):
         """
         Spawn a coroutine on all workers.
@@ -2559,12 +2567,6 @@ def run_coroutine(self, function, *args, **kwargs):
             Workers on which to run the function. Defaults to all known workers.
 
         """
-        warnings.warn(
-            "This method has been deprecated. "
-            "Instead use Client.run which detects async functions "
-            "automatically",
-            stacklevel=2,
-        )
         return self.run(function, *args, **kwargs)
 
     def _graph_to_futures(
diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
index 2439ab6bba3..aaa05cf5617 100644
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -3,14 +3,13 @@
 import logging
 import threading
 import uuid
-import warnings
 from contextlib import suppress
 from inspect import isawaitable
 
 from tornado.ioloop import PeriodicCallback
 
 import dask.config
-from dask.utils import format_bytes
+from dask.utils import _deprecated, format_bytes
 
 from ..core import Status
 from ..objects import SchedulerInfo
@@ -248,8 +247,8 @@ def get_logs(self, cluster=True, scheduler=True, workers=True):
             self._get_logs, cluster=cluster, scheduler=scheduler, workers=workers
         )
 
+    @_deprecated(use_instead="get_logs")
     def logs(self, *args, **kwargs):
-        warnings.warn("logs is deprecated, use get_logs instead", DeprecationWarning)
         return self.get_logs(*args, **kwargs)
 
     @property
diff --git a/distributed/deploy/tests/test_cluster.py b/distributed/deploy/tests/test_cluster.py
index db64f67e4f7..159f64596f3 100644
--- a/distributed/deploy/tests/test_cluster.py
+++ b/distributed/deploy/tests/test_cluster.py
@@ -25,3 +25,10 @@ async def test_repr(cleanup):
     res = repr(cluster)
     expected = "Cluster(A, '<Not Connected>', workers=0, threads=0, memory=0 B)"
     assert res == expected
+
+
+@pytest.mark.asyncio
+async def test_logs_deprecated(cleanup):
+    cluster = Cluster(asynchronous=True)
+    with pytest.warns(FutureWarning, match="get_logs"):
+        cluster.logs()
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 0c6b8c088b3..93314062a70 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -2715,6 +2715,16 @@ def test_run_coroutine_sync(c, s, a, b):
     assert t2 - t1 <= 1.0
 
 
+@gen_cluster(client=True)
+async def test_run_coroutine_deprecated(c, s, a, b):
+    async def foo():
+        return "bar"
+
+    with pytest.warns(FutureWarning, match="Client.run "):
+        results = await c.run_coroutine(foo)
+    assert results == {a.address: "bar", b.address: "bar"}
+
+
 def test_run_exception(c):
     def raise_exception(exc_type, exc_msg):
         raise exc_type(exc_msg)
diff --git a/distributed/tests/test_utils.py b/distributed/tests/test_utils.py
index bb274098e3f..16bc7074f7d 100644
--- a/distributed/tests/test_utils.py
+++ b/distributed/tests/test_utils.py
@@ -23,7 +23,6 @@
     MultiLogs,
     TimeoutError,
     _maybe_complex,
-    deprecated,
     ensure_bytes,
     ensure_ip,
     format_dashboard_link,
@@ -608,20 +607,3 @@ def test_lru():
 async def test_offload():
     assert (await offload(inc, 1)) == 2
     assert (await offload(lambda x, y: x + y, 1, y=2)) == 3
-
-
-def test_deprecated():
-    @deprecated()
-    def foo():
-        return "bar"
-
-    with pytest.warns(DeprecationWarning, match="foo is deprecated"):
-        assert foo() == "bar"
-
-    # Explicit version specified
-    @deprecated(version_removed="1.2.3")
-    def foo():
-        return "bar"
-
-    with pytest.warns(DeprecationWarning, match="removed in version 1.2.3"):
-        assert foo() == "bar"
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index ba4de5199d2..2f3a7f58ede 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -49,7 +49,7 @@
     nodebug,
     slowinc,
 )
-from distributed.worker import Worker, error_message, logger, parse_memory_limit, weight
+from distributed.worker import Worker, error_message, logger, parse_memory_limit
 
 
 @pytest.mark.asyncio
@@ -1841,11 +1841,6 @@ async def test_story_with_deps(c, s, a, b):
     assert story == expected_story
 
 
-def test_weight_deprecated():
-    with pytest.warns(DeprecationWarning):
-        weight("foo", "bar")
-
-
 @gen_cluster(client=True)
 async def test_gather_dep_one_worker_always_busy(c, s, a, b):
     # Ensure that both dependencies for H are on another worker than H itself.
diff --git a/distributed/utils.py b/distributed/utils.py
index d8a22ac05d2..fae79ff4552 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -982,34 +982,6 @@ def nbytes(frame, _bytes_like=(bytes, bytearray)):
             return len(frame)
 
 
-def deprecated(*, version_removed: str = None):
-    """Decorator to mark a function as deprecated
-
-    Parameters
-    ----------
-    version_removed : str, optional
-        If specified, include the version in which the deprecated function
-        will be removed. Defaults to "a future release".
-    """
-
-    def decorator(func):
-        nonlocal version_removed
-        msg = f"{funcname(func)} is deprecated and will be removed in"
-        if version_removed is not None:
-            msg += f" version {version_removed}"
-        else:
-            msg += " a future release"
-
-        @functools.wraps(func)
-        def wrapper(*args, **kwargs):
-            warnings.warn(msg, DeprecationWarning, stacklevel=2)
-            return func(*args, **kwargs)
-
-        return wrapper
-
-    return decorator
-
-
 def json_load_robust(fn, load=json.load):
     """Reads a JSON file from disk that may be being written as we read"""
     while not os.path.exists(fn):
diff --git a/distributed/worker.py b/distributed/worker.py
index bd1bf6f0d75..685f6df6240 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -57,7 +57,6 @@
     LRU,
     TimeoutError,
     _maybe_complex,
-    deprecated,
     get_ip,
     has_arg,
     import_file,
@@ -4017,11 +4016,6 @@ def convert_kwargs_to_str(kwargs, max_len=None):
         return "{{{}}}".format(", ".join(strs))
 
 
-@deprecated(version_removed="2021.06.0")
-def weight(k, v):
-    return sizeof(v)
-
-
 async def run(server, comm, function, args=(), kwargs=None, is_coro=None, wait=True):
     kwargs = kwargs or {}
     function = pickle.loads(function)
diff --git a/docs/requirements.txt b/docs/requirements.txt
index e5cbc5caaca..5a33e930b2a 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -2,7 +2,7 @@ numpydoc
 tornado
 toolz
 cloudpickle
-dask>=2020.12.0
+git+https://github.com/dask/dask
 sphinx
 dask-sphinx-theme>=1.3.5
 sphinx-click

From 7d0f0102ddfec1195257c82889ff695d32d61fbc Mon Sep 17 00:00:00 2001
From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com>
Date: Mon, 21 Jun 2021 18:18:22 -0400
Subject: [PATCH 1323/1550] Revert refactor to `utils.Log[s]` and
 `Cluster.get_logs` (#4941)

---
 distributed/dashboard/components/scheduler.py |  6 ++-
 distributed/deploy/cluster.py                 | 13 +++---
 distributed/deploy/tests/test_spec_cluster.py |  2 +
 distributed/tests/test_utils.py               |  8 +++-
 distributed/utils.py                          | 44 +++++++++----------
 5 files changed, 42 insertions(+), 31 deletions(-)

diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index b00bcfb94b0..9311f187e6a 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -70,7 +70,7 @@
 from distributed.diagnostics.task_stream import color_of as ts_color_of
 from distributed.diagnostics.task_stream import colors as ts_color_lookup
 from distributed.metrics import time
-from distributed.utils import Logs, format_time, log_errors, parse_timedelta
+from distributed.utils import Log, format_time, log_errors, parse_timedelta
 
 if dask.config.get("distributed.dashboard.export-tool"):
     from distributed.dashboard.export_tool import ExportTool
@@ -2165,7 +2165,9 @@ def update(self):
 
 class SchedulerLogs:
     def __init__(self, scheduler):
-        logs = Logs(scheduler.get_logs())._repr_html_()
+        logs = Log(
+            "\n".join(line for level, line in scheduler.get_logs())
+        )._repr_html_()
 
         self.root = Div(text=logs)
 
diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
index aaa05cf5617..b00886ac963 100644
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -14,7 +14,8 @@
 from ..core import Status
 from ..objects import SchedulerInfo
 from ..utils import (
-    MultiLogs,
+    Log,
+    Logs,
     format_dashboard_link,
     log_errors,
     parse_timedelta,
@@ -208,19 +209,21 @@ def _log(self, log):
             print(log)
 
     async def _get_logs(self, cluster=True, scheduler=True, workers=True):
-        logs = MultiLogs()
+        logs = Logs()
 
         if cluster:
-            logs["Cluster"] = self._cluster_manager_logs
+            logs["Cluster"] = Log(
+                "\n".join(line[1] for line in self._cluster_manager_logs)
+            )
 
         if scheduler:
             L = await self.scheduler_comm.get_logs()
-            logs["Scheduler"] = L
+            logs["Scheduler"] = Log("\n".join(line for level, line in L))
 
         if workers:
             d = await self.scheduler_comm.worker_logs(workers=workers)
             for k, v in d.items():
-                logs[k] = v
+                logs[k] = Log("\n".join(line for level, line in v))
 
         return logs
 
diff --git a/distributed/deploy/tests/test_spec_cluster.py b/distributed/deploy/tests/test_spec_cluster.py
index ca96104de3b..dff5d06831b 100644
--- a/distributed/deploy/tests/test_spec_cluster.py
+++ b/distributed/deploy/tests/test_spec_cluster.py
@@ -287,6 +287,8 @@ async def test_logs(cleanup):
         await cluster
 
         logs = await cluster.get_logs()
+        assert isinstance(logs, dict)
+        assert all(isinstance(log, str) for log in logs)
         assert is_valid_xml("<div>" + logs._repr_html_() + "</div>")
         assert "Scheduler" in logs
         for worker in cluster.scheduler.workers:
diff --git a/distributed/tests/test_utils.py b/distributed/tests/test_utils.py
index 16bc7074f7d..fa384cd8451 100644
--- a/distributed/tests/test_utils.py
+++ b/distributed/tests/test_utils.py
@@ -19,8 +19,9 @@
 from distributed.utils import (
     LRU,
     All,
+    Log,
+    Logs,
     LoopRunner,
-    MultiLogs,
     TimeoutError,
     _maybe_complex,
     ensure_bytes,
@@ -548,7 +549,10 @@ def test_format_bytes_compat():
 
 
 def test_logs():
-    d = MultiLogs({"123": [("INFO", "Hello")], "456": [("INFO", "World!")]})
+    log = Log("Hello")
+    assert isinstance(log, str)
+    d = Logs({"123": log, "456": Log("World!")})
+    assert isinstance(d, dict)
     text = d._repr_html_()
     assert is_valid_xml("<div>" + text + "</div>")
     assert "Hello" in text
diff --git a/distributed/utils.py b/distributed/utils.py
index fae79ff4552..33793b2f504 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -1241,8 +1241,8 @@ def parse_ports(port):
 is_coroutine_function = iscoroutinefunction
 
 
-class Log(tuple):
-    """A container for a single log entry"""
+class Log(str):
+    """A container for newline-delimited string of log entries"""
 
     level_styles = {
         "WARNING": "font-weight: bold; color: orange;",
@@ -1251,34 +1251,34 @@ class Log(tuple):
     }
 
     def _repr_html_(self):
-        level, message = self
-
-        style = "font-family: monospace; margin: 0;"
-        style += self.level_styles.get(level, "")
-
-        return '<p style="{style}">{message}</p>'.format(
-            style=html.escape(style),
-            message=html.escape(message),
-        )
-
-
-class Logs(list):
-    """A container for a list of log entries"""
+        logs_html = []
+        for message in self.split("\n"):
+            style = "font-family: monospace; margin: 0;"
+            for level in self.level_styles:
+                if level in message:
+                    style += self.level_styles[level]
+                    break
+
+            logs_html.append(
+                '<p style="{style}">{message}</p>'.format(
+                    style=html.escape(style),
+                    message=html.escape(message),
+                )
+            )
 
-    def _repr_html_(self):
-        return "\n".join(Log(entry)._repr_html_() for entry in self)
+        return "\n".join(logs_html)
 
 
-class MultiLogs(dict):
-    """A container for a dict mapping strings to lists of log entries"""
+class Logs(dict):
+    """A container for a dict mapping names to strings of log entries"""
 
     def _repr_html_(self):
         summaries = [
             "<details>\n"
             "<summary style='display:list-item'>{title}</summary>\n"
-            "{logs}\n"
-            "</details>".format(title=title, logs=Logs(entries)._repr_html_())
-            for title, entries in sorted(self.items())
+            "{log}\n"
+            "</details>".format(title=title, log=log._repr_html_())
+            for title, log in sorted(self.items())
         ]
         return "\n".join(summaries)
 

From ac35e0f8104baaccfb44f98b67a48ee359069316 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Tue, 22 Jun 2021 11:56:21 -0500
Subject: [PATCH 1324/1550] bump version to 2021.06.2

---
 docs/source/changelog.rst | 11 +++++++++++
 requirements.txt          |  2 +-
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index d346a6c0a1c..3fa30848dd7 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,17 @@
 Changelog
 =========
 
+2021.06.2
+---------
+
+Released on June 22, 2021
+
+- Revert refactor to ``utils.Log[s]`` and ``Cluster.get_logs`` (:pr:`4941`) `Charles Blackmon-Luca`_
+- Use deprecation utility from Dask (:pr:`4924`) `James Bourbeau`_
+- Add transition counter to ``Scheduler`` (:pr:`4934`) `Matthew Rocklin`_
+- Remove ``nbytes_in_memory`` (:pr:`4930`) `Matthew Rocklin`_
+
+
 2021.06.1
 ---------
 
diff --git a/requirements.txt b/requirements.txt
index ad0339775d9..70f7b61dd13 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,6 @@
 click >= 6.6
 cloudpickle >= 1.5.0
-dask == 2021.06.1
+dask == 2021.06.2
 msgpack >= 0.6.0
 psutil >= 5.0
 sortedcontainers !=2.0.0, !=2.0.1

From 9f4165a4278bc67f628089b72001ae5d97f5146b Mon Sep 17 00:00:00 2001
From: Gabe Joseph <gjoseph92@gmail.com>
Date: Wed, 23 Jun 2021 03:17:17 -0600
Subject: [PATCH 1325/1550] Support fixtures and `pytest.mark.parametrize` with
 `gen_cluster` (#4958)

Support fixtures and `pytest.mark.parametrize` with `gen_cluster` (#4958)
---
 distributed/tests/test_utils_test.py | 42 ++++++++++++++++++++++++++++
 distributed/utils_test.py            | 29 +++++++++++++++++--
 2 files changed, 69 insertions(+), 2 deletions(-)

diff --git a/distributed/tests/test_utils_test.py b/distributed/tests/test_utils_test.py
index 1458cf2af11..1e546adb4bc 100755
--- a/distributed/tests/test_utils_test.py
+++ b/distributed/tests/test_utils_test.py
@@ -1,4 +1,5 @@
 import asyncio
+import pathlib
 import socket
 import threading
 from contextlib import contextmanager
@@ -45,6 +46,47 @@ async def test_gen_cluster(c, s, a, b):
     assert await c.submit(lambda: 123) == 123
 
 
+@gen_cluster(client=True)
+async def test_gen_cluster_pytest_fixture(c, s, a, b, tmp_path):
+    assert isinstance(tmp_path, pathlib.Path)
+    assert isinstance(c, Client)
+    assert isinstance(s, Scheduler)
+    for w in [a, b]:
+        assert isinstance(w, Worker)
+
+
+@pytest.mark.parametrize("foo", [True])
+@gen_cluster(client=True)
+async def test_gen_cluster_parametrized(c, s, a, b, foo):
+    assert foo is True
+    assert isinstance(c, Client)
+    assert isinstance(s, Scheduler)
+    for w in [a, b]:
+        assert isinstance(w, Worker)
+
+
+@pytest.mark.parametrize("foo", [True])
+@pytest.mark.parametrize("bar", ["a", "b"])
+@gen_cluster(client=True)
+async def test_gen_cluster_multi_parametrized(c, s, a, b, foo, bar):
+    assert foo is True
+    assert bar in ("a", "b")
+    assert isinstance(c, Client)
+    assert isinstance(s, Scheduler)
+    for w in [a, b]:
+        assert isinstance(w, Worker)
+
+
+@pytest.mark.parametrize("foo", [True])
+@gen_cluster(client=True)
+async def test_gen_cluster_parametrized_variadic_workers(c, s, *workers, foo):
+    assert foo is True
+    assert isinstance(c, Client)
+    assert isinstance(s, Scheduler)
+    for w in workers:
+        assert isinstance(w, Worker)
+
+
 @gen_cluster(
     client=True,
     Worker=Nanny,
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index a40cf14151d..0779d7de285 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -3,6 +3,7 @@
 import copy
 import functools
 import gc
+import inspect
 import io
 import itertools
 import logging
@@ -861,6 +862,15 @@ def gen_cluster(
     async def test_foo(scheduler, worker1, worker2):
         await ...  # use tornado coroutines
 
+    @pytest.mark.parametrize("param", [1, 2, 3])
+    @gen_cluster()
+    async def test_foo(scheduler, worker1, worker2, param):
+        await ...  # use tornado coroutines
+
+    @gen_cluster()
+    async def test_foo(scheduler, worker1, worker2, pytest_fixture_a, pytest_fixture_b):
+        await ...  # use tornado coroutines
+
     See also:
         start
         end
@@ -877,7 +887,7 @@ def _(func):
         if not iscoroutinefunction(func):
             func = gen.coroutine(func)
 
-        def test_func():
+        def test_func(*outer_args, **kwargs):
             result = None
             workers = []
             with clean(timeout=active_rpc_timeout, **clean_kwargs) as loop:
@@ -919,7 +929,7 @@ async def coro():
                             )
                             args = [c] + args
                         try:
-                            future = func(*args)
+                            future = func(*args, *outer_args, **kwargs)
                             if timeout:
                                 future = asyncio.wait_for(future, timeout)
                             result = await future
@@ -979,6 +989,21 @@ def get_unclosed():
 
             return result
 
+        # Patch the signature so pytest can inject fixtures
+        orig_sig = inspect.signature(func)
+        args = [None] * (1 + len(nthreads))  # scheduler, *workers
+        if client:
+            args.insert(0, None)
+
+        bound = orig_sig.bind_partial(*args)
+        test_func.__signature__ = orig_sig.replace(
+            parameters=[
+                p
+                for name, p in orig_sig.parameters.items()
+                if name not in bound.arguments
+            ]
+        )
+
         return test_func
 
     return _

From 6db04476cf43c7e999e7cd06df463b45e64b7970 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Wed, 23 Jun 2021 12:55:10 -0500
Subject: [PATCH 1326/1550] Add `Log` and `Logs` to API docs (#4946)

---
 docs/source/api.rst | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/docs/source/api.rst b/docs/source/api.rst
index 0d5c858e2c6..dc49a2d477f 100644
--- a/docs/source/api.rst
+++ b/docs/source/api.rst
@@ -65,6 +65,13 @@ API
    performance_report
 
 
+**Utilities**
+
+.. autosummary::
+   distributed.utils.Log
+   distributed.utils.Logs
+
+
 Asynchronous methods
 --------------------
 
@@ -175,6 +182,13 @@ Other
    :members:
 
 
+Utilities
+---------
+
+.. autoclass:: distributed.utils.Log
+.. autoclass:: distributed.utils.Logs
+
+
 Adaptive
 --------
 

From f617bbf4beceb85a386815e9d9830de3919e6a85 Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Wed, 23 Jun 2021 19:56:29 +0200
Subject: [PATCH 1327/1550] Allow requirements verification to be ignored when
 loading backends from entrypoints (#4961)

---
 distributed/comm/registry.py         | 29 ++++++++++++++++++----------
 distributed/comm/tests/test_comms.py |  6 +++++-
 2 files changed, 24 insertions(+), 11 deletions(-)

diff --git a/distributed/comm/registry.py b/distributed/comm/registry.py
index 8fb7a6026f8..9df481cd466 100644
--- a/distributed/comm/registry.py
+++ b/distributed/comm/registry.py
@@ -57,26 +57,35 @@ def get_local_address_for(self, loc):
 backends = {}
 
 
-def get_backend(scheme):
+def get_backend(scheme: str, require: bool = True) -> Backend:
     """
     Get the Backend instance for the given *scheme*.
     It looks for matching scheme in dask's internal cache, and falls-back to
     package metadata for the group name ``distributed.comm.backends``
+
+    Parameters
+    ----------
+
+    require : bool
+        Verify that the backends requirements are properly installed. See
+        https://setuptools.readthedocs.io/en/latest/pkg_resources.html for more
+        information.
     """
 
     backend = backends.get(scheme)
     if backend is None:
         import pkg_resources
 
-        backend = next(
-            iter(
-                backend_class_ep.load()()
-                for backend_class_ep in pkg_resources.iter_entry_points(
-                    "distributed.comm.backends", scheme
-                )
-            ),
-            None,
-        )
+        backend = None
+        for backend_class_ep in pkg_resources.iter_entry_points(
+            "distributed.comm.backends", scheme
+        ):
+            # resolve and require are equivalent to load
+            backend_factory = backend_class_ep.resolve()
+            if require:
+                backend_class_ep.require()
+            backend = backend_factory()
+
         if backend is None:
             raise ValueError(
                 "unknown address scheme %r (known schemes: %s)"
diff --git a/distributed/comm/tests/test_comms.py b/distributed/comm/tests/test_comms.py
index eef6ce478de..277bb05916c 100644
--- a/distributed/comm/tests/test_comms.py
+++ b/distributed/comm/tests/test_comms.py
@@ -1275,5 +1275,9 @@ def test_register_backend_entrypoint():
         "udp", mod.__name__, attrs=["UDPBackend"], dist=dist
     )
 
-    result = get_backend("udp")
+    # The require is disabled here since particularly unit tests may install
+    # dirty or dev versions which are conflicting with backend entrypoints if
+    # they are demanding for exact, stable versions. This should not fail the
+    # test
+    result = get_backend("udp", require=False)
     assert result == 1

From 82d591f8793465623302abf7f378393b2606c963 Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Wed, 23 Jun 2021 22:05:43 +0200
Subject: [PATCH 1328/1550] Stealing dashboard fixes (#4948)

---
 distributed/dashboard/components/scheduler.py | 38 ++++++++++---------
 .../dashboard/tests/test_scheduler_bokeh.py   |  9 ++---
 2 files changed, 25 insertions(+), 22 deletions(-)

diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index 9311f187e6a..7f197aa54d2 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -1144,7 +1144,11 @@ class StealingTimeSeries(DashboardComponent):
     def __init__(self, scheduler, **kwargs):
         self.scheduler = scheduler
         self.source = ColumnDataSource(
-            {"time": [time(), time() + 1], "idle": [0, 0.1], "saturated": [0, 0.1]}
+            {
+                "time": [time() * 1000, time() * 1000 + 1],
+                "idle": [0, 0],
+                "saturated": [0, 0],
+            }
         )
 
         x_range = DataRange1d(follow="end", follow_interval=20000, range_padding=0)
@@ -1152,8 +1156,6 @@ def __init__(self, scheduler, **kwargs):
         self.root = figure(
             title="Idle and Saturated Workers Over Time",
             x_axis_type="datetime",
-            y_range=[-0.1, len(scheduler.workers) + 0.1],
-            height=150,
             tools="",
             x_range=x_range,
             **kwargs,
@@ -1204,8 +1206,6 @@ def __init__(self, scheduler, **kwargs):
         self.root = figure(
             title="Stealing Events",
             x_axis_type="datetime",
-            y_axis_type="log",
-            height=250,
             tools="",
             x_range=x_range,
             **kwargs,
@@ -1214,12 +1214,12 @@ def __init__(self, scheduler, **kwargs):
         self.root.circle(
             source=self.source,
             x="time",
-            y="cost_factor",
+            y="level",
             color="color",
             size="radius",
             alpha=0.5,
         )
-        self.root.yaxis.axis_label = "Cost Multiplier"
+        self.root.yaxis.axis_label = "Level"
 
         hover = HoverTool()
         hover.tooltips = "Level: @level, Duration: @duration, Count: @count, Cost factor: @cost_factor"
@@ -1253,7 +1253,7 @@ def convert(self, msgs):
             "color": color,
             "duration": total_duration,
             "radius": radius,
-            "cost_factor": min(10, self.steal.cost_multipliers[level]),
+            "cost_factor": self.steal.cost_multipliers[level],
         }
 
         return d
@@ -1262,9 +1262,11 @@ def convert(self, msgs):
     def update(self):
         with log_errors():
             log = self.scheduler.get_events(topic="stealing")
-            n = self.steal.count - self.last
+            current = len(self.scheduler.events["stealing"])
+            n = current - self.last
+
             log = [log[-i][1] for i in range(1, n + 1) if isinstance(log[-i][1], list)]
-            self.last = self.steal.count
+            self.last = current
 
             if log:
                 new = pipe(
@@ -2186,9 +2188,9 @@ def systemmonitor_doc(scheduler, extra, doc):
 
 def stealing_doc(scheduler, extra, doc):
     with log_errors():
-        occupancy = Occupancy(scheduler, height=200, sizing_mode="scale_width")
-        stealing_ts = StealingTimeSeries(scheduler, sizing_mode="scale_width")
-        stealing_events = StealingEvents(scheduler, sizing_mode="scale_width")
+        occupancy = Occupancy(scheduler)
+        stealing_ts = StealingTimeSeries(scheduler)
+        stealing_events = StealingEvents(scheduler)
         stealing_events.root.x_range = stealing_ts.root.x_range
         doc.title = "Dask: Work Stealing"
         add_periodic_callback(doc, occupancy, 500)
@@ -2196,11 +2198,13 @@ def stealing_doc(scheduler, extra, doc):
         add_periodic_callback(doc, stealing_events, 500)
 
         doc.add_root(
-            column(
+            row(
                 occupancy.root,
-                stealing_ts.root,
-                stealing_events.root,
-                sizing_mode="scale_width",
+                column(
+                    stealing_ts.root,
+                    stealing_events.root,
+                    sizing_mode="stretch_both",
+                ),
             )
         )
 
diff --git a/distributed/dashboard/tests/test_scheduler_bokeh.py b/distributed/dashboard/tests/test_scheduler_bokeh.py
index 7034dc7394d..68578565bf5 100644
--- a/distributed/dashboard/tests/test_scheduler_bokeh.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh.py
@@ -103,15 +103,14 @@ async def test_stealing_events(c, s, a, b):
     se = StealingEvents(s)
 
     futures = c.map(
-        slowinc, range(100), delay=0.1, workers=a.address, allow_other_workers=True
+        slowinc, range(10), delay=0.1, workers=a.address, allow_other_workers=True
     )
 
-    while not b.tasks:  # will steal soon
-        await asyncio.sleep(0.01)
-
+    await wait(futures)
     se.update()
-
     assert len(first(se.source.data.values()))
+    assert b.tasks
+    assert sum(se.source.data["count"]) >= len(b.tasks)
 
 
 @gen_cluster(client=True)

From 1bcd8a9c2cac231efa26b7c9dac391c84fd1cd1f Mon Sep 17 00:00:00 2001
From: Doug Davis <ddavis@anaconda.com>
Date: Wed, 23 Jun 2021 17:38:42 -0400
Subject: [PATCH 1329/1550] Add support for registering scheduler plugins from
 `Client` (#4808)

---
 distributed/client.py                         | 28 ++++++++++++++
 .../tests/test_scheduler_plugin.py            | 38 +++++++++++++++++++
 distributed/scheduler.py                      | 20 ++++++++++
 3 files changed, 86 insertions(+)

diff --git a/distributed/client.py b/distributed/client.py
index a0352e9c70e..4a1090516a0 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -4007,6 +4007,34 @@ async def _get_task_stream(
         else:
             return msgs
 
+    async def _register_scheduler_plugin(self, plugin, **kwargs):
+        if isinstance(plugin, type):
+            plugin = plugin(**kwargs)
+
+        return await self.scheduler.register_scheduler_plugin(
+            plugin=dumps(plugin, protocol=4)
+        )
+
+    def register_scheduler_plugin(self, plugin, **kwargs):
+        """Register a scheduler plugin.
+
+        See https://distributed.readthedocs.io/en/latest/plugins.html#scheduler-plugins
+
+        Parameters
+        ----------
+        plugin : SchedulerPlugin
+            Plugin class or object to pass to the scheduler.
+        **kwargs : Any
+            Arguments passed to the Plugin class (if Plugin is an
+            instance kwargs are unused).
+
+        """
+        return self.sync(
+            self._register_scheduler_plugin,
+            plugin=plugin,
+            **kwargs,
+        )
+
     def register_worker_callbacks(self, setup=None):
         """
         Registers a setup callback function for all current and future workers.
diff --git a/distributed/diagnostics/tests/test_scheduler_plugin.py b/distributed/diagnostics/tests/test_scheduler_plugin.py
index b48f0b2ae7d..d1298307cba 100644
--- a/distributed/diagnostics/tests/test_scheduler_plugin.py
+++ b/distributed/diagnostics/tests/test_scheduler_plugin.py
@@ -125,3 +125,41 @@ async def close(self):
 
     assert plugin.history == ["started", "closed"]
     assert plugin.scheduler is s
+
+
+@gen_cluster(client=True)
+async def test_register_scheduler_plugin(c, s, a, b):
+    class Dummy1(SchedulerPlugin):
+        def start(self, scheduler):
+            scheduler.foo = "bar"
+
+    assert not hasattr(s, "foo")
+    await c.register_scheduler_plugin(Dummy1)
+    assert s.foo == "bar"
+
+    class Dummy2(SchedulerPlugin):
+        def start(self, scheduler):
+            raise RuntimeError("raising in start method")
+
+    n_plugins = len(s.plugins)
+    with pytest.raises(RuntimeError, match="raising in start method"):
+        await c.register_scheduler_plugin(Dummy2)
+    # total number of plugins should be unchanged
+    assert n_plugins == len(s.plugins)
+
+
+@gen_cluster(client=True, config={"distributed.scheduler.pickle": False})
+async def test_register_scheduler_plugin_pickle_disabled(c, s, a, b):
+    class Dummy1(SchedulerPlugin):
+        def start(self, scheduler):
+            scheduler.foo = "bar"
+
+    n_plugins = len(s.plugins)
+    with pytest.raises(ValueError) as excinfo:
+        await c.register_scheduler_plugin(Dummy1)
+
+    msg = str(excinfo.value)
+    assert "disallowed from deserializing" in msg
+    assert "distributed.scheduler.pickle" in msg
+
+    assert n_plugins == len(s.plugins)
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index a2996482ac9..7dc2f8d0c33 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -58,6 +58,7 @@
 from .multi_lock import MultiLockExtension
 from .node import ServerNode
 from .proctitle import setproctitle
+from .protocol.pickle import loads
 from .publish import PublishExtension
 from .pubsub import PubSubSchedulerExtension
 from .queues import QueueExtension
@@ -3575,6 +3576,7 @@ def __init__(
             "heartbeat_worker": self.heartbeat_worker,
             "get_task_status": self.get_task_status,
             "get_task_stream": self.get_task_stream,
+            "register_scheduler_plugin": self.register_scheduler_plugin,
             "register_worker_plugin": self.register_worker_plugin,
             "unregister_worker_plugin": self.unregister_worker_plugin,
             "adaptive_target": self.adaptive_target,
@@ -5210,6 +5212,24 @@ def remove_plugin(self, plugin):
         """Remove external plugin from scheduler"""
         self.plugins.remove(plugin)
 
+    async def register_scheduler_plugin(self, comm=None, plugin=None):
+        """Register a plugin on the scheduler."""
+        if not dask.config.get("distributed.scheduler.pickle"):
+            raise ValueError(
+                "Cannot register a scheduler plugin as the scheduler "
+                "has been explicitly disallowed from deserializing "
+                "arbitrary bytestrings using pickle via the "
+                "'distributed.scheduler.pickle' configuration setting."
+            )
+        plugin = loads(plugin)
+
+        if hasattr(plugin, "start"):
+            result = plugin.start(self)
+            if inspect.isawaitable(result):
+                result = await result
+
+        self.add_plugin(plugin=plugin)
+
     def worker_send(self, worker, msg):
         """Send message to worker
 

From f1b017203fc12976855d251447a81f7582df7516 Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Thu, 24 Jun 2021 02:07:29 +0200
Subject: [PATCH 1330/1550] Ensure connectionpool does not leave comms if
 closed mid connect (#4951)

---
 distributed/core.py            | 62 ++++++++++++++++++++++++++--------
 distributed/tests/test_core.py | 50 +++++++++++++++++++++++++++
 2 files changed, 97 insertions(+), 15 deletions(-)

diff --git a/distributed/core.py b/distributed/core.py
index 1c95f6bd019..227c42610f2 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -946,6 +946,12 @@ def __init__(
         self.server = weakref.ref(server) if server else None
         self._created = weakref.WeakSet()
         self._instances.add(self)
+        # _n_connecting and _connecting have subtle different semantics. The set
+        # _connecting contains futures actively trying to establish a connection
+        # while the _n_connecting also accounts for connection attempts which
+        # are waiting due to the connection limit
+        self._connecting = set()
+        self.status = Status.init
 
     def _validate(self):
         """
@@ -987,6 +993,7 @@ async def _():
     async def start(self):
         # Invariant: semaphore._value == limit - open - _n_connecting
         self.semaphore = asyncio.Semaphore(self.limit)
+        self.status = Status.running
 
     async def connect(self, addr, timeout=None):
         """
@@ -1007,28 +1014,43 @@ async def connect(self, addr, timeout=None):
 
         self._n_connecting += 1
         await self.semaphore.acquire()
-
+        fut = None
         try:
-            comm = await connect(
-                addr,
-                timeout=timeout or self.timeout,
-                deserialize=self.deserialize,
-                **self.connection_args,
+            if self.status != Status.running:
+                raise CommClosedError(
+                    f"ConnectionPool not running.  Status: {self.status}"
+                )
+
+            fut = asyncio.ensure_future(
+                connect(
+                    addr,
+                    timeout=timeout or self.timeout,
+                    deserialize=self.deserialize,
+                    **self.connection_args,
+                )
             )
+            self._connecting.add(fut)
+            comm = await fut
             comm.name = "ConnectionPool"
             comm._pool = weakref.ref(self)
             comm.allow_offload = self.allow_offload
             self._created.add(comm)
-        except Exception:
+
+            occupied.add(comm)
+
+            return comm
+        except asyncio.CancelledError as exc:
             self.semaphore.release()
-            raise
+            raise CommClosedError(
+                f"ConnectionPool not running.  Status: {self.status}"
+            ) from exc
+        except Exception as exc:
+            self.semaphore.release()
+            raise exc
         finally:
+            self._connecting.discard(fut)
             self._n_connecting -= 1
 
-        occupied.add(comm)
-
-        return comm
-
     def reuse(self, addr, comm):
         """
         Reuse an open communication to the given address.  For internal use.
@@ -1082,16 +1104,26 @@ async def close(self):
         """
         Close all communications
         """
+        self.status = Status.closed
         for d in [self.available, self.occupied]:
-            comms = [comm for comms in d.values() for comm in comms]
+            comms = set()
+            while d:
+                comms.update(d.popitem()[1])
+
             await asyncio.gather(
                 *[comm.close() for comm in comms], return_exceptions=True
             )
+
             for _ in comms:
                 self.semaphore.release()
 
-        for comm in self._created:
-            IOLoop.current().add_callback(comm.abort)
+        for conn_fut in self._connecting:
+            conn_fut.cancel()
+
+        # We might still have tasks haning in the semaphore. This will let them
+        # run into an exception and raise a commclosed
+        while self._n_connecting:
+            await asyncio.sleep(0.005)
 
 
 def coerce_to_address(o):
diff --git a/distributed/tests/test_core.py b/distributed/tests/test_core.py
index 9fc98f276f2..d3df2e61867 100644
--- a/distributed/tests/test_core.py
+++ b/distributed/tests/test_core.py
@@ -9,6 +9,7 @@
 
 import dask
 
+from distributed.comm.core import CommClosedError
 from distributed.core import (
     ConnectionPool,
     Server,
@@ -614,6 +615,55 @@ async def ping(comm, delay=0.1):
     await rpc.close()
 
 
+@pytest.mark.asyncio
+async def test_connection_pool_close_while_connecting(monkeypatch):
+    """
+    Ensure a closed connection pool guarantees to have no connections left open
+    even if it is closed mid-connecting
+    """
+    from distributed.comm.registry import backends
+    from distributed.comm.tcp import TCPBackend, TCPConnector
+
+    class SlowConnector(TCPConnector):
+        async def connect(self, address, deserialize, **connection_args):
+            await asyncio.sleep(0.1)
+            return await super().connect(
+                address, deserialize=deserialize, **connection_args
+            )
+
+    class SlowBackend(TCPBackend):
+        _connector_class = SlowConnector
+
+    monkeypatch.setitem(backends, "tcp", SlowBackend())
+
+    server = Server({})
+    await server.listen("tcp://")
+
+    pool = await ConnectionPool(limit=2)
+
+    async def connect_to_server():
+        comm = await pool.connect(server.address)
+        pool.reuse(server.address, comm)
+
+    tasks = [asyncio.create_task(connect_to_server()) for _ in range(30)]
+
+    await asyncio.sleep(0)
+    assert pool._connecting
+    close_fut = asyncio.create_task(pool.close())
+
+    with pytest.raises(
+        CommClosedError, match="ConnectionPool not running.  Status: Status.closed"
+    ):
+        await asyncio.gather(*tasks)
+
+    await close_fut
+    assert not pool.open
+    assert not pool._n_connecting
+
+    for t in tasks:
+        t.cancel()
+
+
 @pytest.mark.asyncio
 async def test_connection_pool_respects_limit():
 

From 06a70aaf23d3c2858496e6a21a37a7b46f3a8498 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Thu, 24 Jun 2021 02:39:24 -0500
Subject: [PATCH 1331/1550] Deprecate utilities which have moved to dask
 (#4966)

---
 distributed/batched.py                 |   2 +-
 distributed/cfexecutor.py              |   4 +-
 distributed/client.py                  |   2 +-
 distributed/comm/core.py               |   3 +-
 distributed/comm/tcp.py                |   3 +-
 distributed/comm/ucx.py                |   3 +-
 distributed/core.py                    |   2 +-
 distributed/deploy/adaptive.py         |   3 +-
 distributed/deploy/adaptive_core.py    |   3 +-
 distributed/deploy/cluster.py          |  12 +--
 distributed/deploy/spec.py             |  10 +--
 distributed/diagnostics/progressbar.py |   6 +-
 distributed/event.py                   |   4 +-
 distributed/lock.py                    |   4 +-
 distributed/multi_lock.py              |   4 +-
 distributed/nanny.py                   |   2 +-
 distributed/profile.py                 |   4 +-
 distributed/pubsub.py                  |   4 +-
 distributed/queues.py                  |   4 +-
 distributed/scheduler.py               |   5 +-
 distributed/semaphore.py               |   3 +-
 distributed/stealing.py                |   3 +-
 distributed/tests/test_utils.py        | 100 +++++++++++--------------
 distributed/utils.py                   |  44 +++++++----
 distributed/variable.py                |   4 +-
 distributed/worker.py                  |   4 +-
 distributed/worker_client.py           |   3 +-
 27 files changed, 122 insertions(+), 123 deletions(-)

diff --git a/distributed/batched.py b/distributed/batched.py
index b932f1b24c7..4e59a09e80b 100644
--- a/distributed/batched.py
+++ b/distributed/batched.py
@@ -5,9 +5,9 @@
 from tornado.ioloop import IOLoop
 
 import dask
+from dask.utils import parse_timedelta
 
 from .core import CommClosedError
-from .utils import parse_timedelta
 
 logger = logging.getLogger(__name__)
 
diff --git a/distributed/cfexecutor.py b/distributed/cfexecutor.py
index f54db42b9a7..8028a4bc7f2 100644
--- a/distributed/cfexecutor.py
+++ b/distributed/cfexecutor.py
@@ -4,8 +4,10 @@
 from tlz import merge
 from tornado import gen
 
+from dask.utils import parse_timedelta
+
 from .metrics import time
-from .utils import TimeoutError, parse_timedelta, sync
+from .utils import TimeoutError, sync
 
 
 @gen.coroutine
diff --git a/distributed/client.py b/distributed/client.py
index 4a1090516a0..51047dc34c3 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -36,6 +36,7 @@
     ensure_dict,
     format_bytes,
     funcname,
+    parse_timedelta,
     stringify,
 )
 
@@ -78,7 +79,6 @@
     key_split,
     log_errors,
     no_default,
-    parse_timedelta,
     sync,
     thread_state,
 )
diff --git a/distributed/comm/core.py b/distributed/comm/core.py
index b9032a7369e..ccdcbb99c20 100644
--- a/distributed/comm/core.py
+++ b/distributed/comm/core.py
@@ -8,11 +8,12 @@
 from contextlib import suppress
 
 import dask
+from dask.utils import parse_timedelta
 
 from ..metrics import time
 from ..protocol import pickle
 from ..protocol.compression import get_default_compression
-from ..utils import TimeoutError, parse_timedelta
+from ..utils import TimeoutError
 from . import registry
 from .addressing import parse_address
 
diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py
index 258f4e88a5a..91e6af308e4 100644
--- a/distributed/comm/tcp.py
+++ b/distributed/comm/tcp.py
@@ -20,11 +20,12 @@
 from tornado.tcpserver import TCPServer
 
 import dask
+from dask.utils import parse_timedelta
 
 from ..protocol.utils import pack_frames_prelude, unpack_frames
 from ..system import MEMORY_LIMIT
 from ..threadpoolexecutor import ThreadPoolExecutor
-from ..utils import ensure_ip, get_ip, get_ipv6, nbytes, parse_timedelta
+from ..utils import ensure_ip, get_ip, get_ipv6, nbytes
 from .addressing import parse_host_port, unparse_host_port
 from .core import Comm, CommClosedError, Connector, FatalCommClosedError, Listener
 from .registry import Backend, backends
diff --git a/distributed/comm/ucx.py b/distributed/comm/ucx.py
index de349d1f7e4..647ed8313a3 100644
--- a/distributed/comm/ucx.py
+++ b/distributed/comm/ucx.py
@@ -10,8 +10,9 @@
 import weakref
 
 import dask
+from dask.utils import parse_bytes
 
-from ..utils import ensure_ip, get_ip, get_ipv6, log_errors, nbytes, parse_bytes
+from ..utils import ensure_ip, get_ip, get_ipv6, log_errors, nbytes
 from .addressing import parse_host_port, unparse_host_port
 from .core import Comm, CommClosedError, Connector, Listener
 from .registry import Backend, backends
diff --git a/distributed/core.py b/distributed/core.py
index 227c42610f2..8369bab1f5f 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -18,6 +18,7 @@
 from tornado.ioloop import IOLoop, PeriodicCallback
 
 import dask
+from dask.utils import parse_timedelta
 
 from . import profile, protocol
 from .comm import (
@@ -36,7 +37,6 @@
     get_traceback,
     has_keyword,
     is_coroutine_function,
-    parse_timedelta,
     truncate_exception,
 )
 
diff --git a/distributed/deploy/adaptive.py b/distributed/deploy/adaptive.py
index fa68ad4e933..8ffb8555d83 100644
--- a/distributed/deploy/adaptive.py
+++ b/distributed/deploy/adaptive.py
@@ -2,9 +2,10 @@
 from inspect import isawaitable
 
 import dask.config
+from dask.utils import parse_timedelta
 
 from ..protocol import pickle
-from ..utils import log_errors, parse_timedelta
+from ..utils import log_errors
 from .adaptive_core import AdaptiveCore
 
 logger = logging.getLogger(__name__)
diff --git a/distributed/deploy/adaptive_core.py b/distributed/deploy/adaptive_core.py
index db8ef77e9ad..b077261dc1d 100644
--- a/distributed/deploy/adaptive_core.py
+++ b/distributed/deploy/adaptive_core.py
@@ -6,8 +6,9 @@
 import tlz as toolz
 from tornado.ioloop import IOLoop, PeriodicCallback
 
+from dask.utils import parse_timedelta
+
 from ..metrics import time
-from ..utils import parse_timedelta
 
 logger = logging.getLogger(__name__)
 
diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
index b00886ac963..37ddc31f39b 100644
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -9,19 +9,11 @@
 from tornado.ioloop import PeriodicCallback
 
 import dask.config
-from dask.utils import _deprecated, format_bytes
+from dask.utils import _deprecated, format_bytes, parse_timedelta
 
 from ..core import Status
 from ..objects import SchedulerInfo
-from ..utils import (
-    Log,
-    Logs,
-    format_dashboard_link,
-    log_errors,
-    parse_timedelta,
-    sync,
-    thread_state,
-)
+from ..utils import Log, Logs, format_dashboard_link, log_errors, sync, thread_state
 from .adaptive import Adaptive
 
 logger = logging.getLogger(__name__)
diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index 070d6d0624d..9c2f1292649 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -11,18 +11,12 @@
 from tornado import gen
 
 import dask
+from dask.utils import parse_bytes, parse_timedelta
 
 from ..core import CommClosedError, Status, rpc
 from ..scheduler import Scheduler
 from ..security import Security
-from ..utils import (
-    LoopRunner,
-    TimeoutError,
-    import_term,
-    parse_bytes,
-    parse_timedelta,
-    silence_logging,
-)
+from ..utils import LoopRunner, TimeoutError, import_term, silence_logging
 from .adaptive import Adaptive
 from .cluster import Cluster
 
diff --git a/distributed/diagnostics/progressbar.py b/distributed/diagnostics/progressbar.py
index 1a05438cccd..a85b72d4b02 100644
--- a/distributed/diagnostics/progressbar.py
+++ b/distributed/diagnostics/progressbar.py
@@ -8,10 +8,12 @@
 from tlz import valmap
 from tornado.ioloop import IOLoop
 
+import dask
+
 from ..client import default_client, futures_of
 from ..core import CommClosedError, coerce_to_address, connect
 from ..protocol.pickle import dumps
-from ..utils import LoopRunner, is_kernel, key_split, parse_timedelta
+from ..utils import LoopRunner, is_kernel, key_split
 from .progress import MultiProgress, Progress, format_time
 
 logger = logging.getLogger(__name__)
@@ -34,7 +36,7 @@ def __init__(self, keys, scheduler=None, interval="100ms", complete=True):
                 break
 
         self.keys = {k.key if hasattr(k, "key") else k for k in keys}
-        self.interval = parse_timedelta(interval, default="s")
+        self.interval = dask.utils.parse_timedelta(interval, default="s")
         self.complete = complete
         self._start_time = default_timer()
 
diff --git a/distributed/event.py b/distributed/event.py
index 40c1c5e3daf..882281692c4 100644
--- a/distributed/event.py
+++ b/distributed/event.py
@@ -4,8 +4,10 @@
 from collections import defaultdict
 from contextlib import suppress
 
+from dask.utils import parse_timedelta
+
 from .client import Client
-from .utils import TimeoutError, log_errors, parse_timedelta
+from .utils import TimeoutError, log_errors
 from .worker import get_worker
 
 logger = logging.getLogger(__name__)
diff --git a/distributed/lock.py b/distributed/lock.py
index 64e3f29c22a..d8b50ac8d03 100644
--- a/distributed/lock.py
+++ b/distributed/lock.py
@@ -3,8 +3,10 @@
 import uuid
 from collections import defaultdict, deque
 
+from dask.utils import parse_timedelta
+
 from .client import Client
-from .utils import TimeoutError, log_errors, parse_timedelta
+from .utils import TimeoutError, log_errors
 from .worker import get_worker
 
 logger = logging.getLogger(__name__)
diff --git a/distributed/multi_lock.py b/distributed/multi_lock.py
index 3eda32e0be6..68752296188 100644
--- a/distributed/multi_lock.py
+++ b/distributed/multi_lock.py
@@ -4,8 +4,10 @@
 from collections import defaultdict
 from typing import Hashable, List
 
+from dask.utils import parse_timedelta
+
 from .client import Client
-from .utils import TimeoutError, log_errors, parse_timedelta
+from .utils import TimeoutError, log_errors
 from .worker import get_worker
 
 logger = logging.getLogger(__name__)
diff --git a/distributed/nanny.py b/distributed/nanny.py
index bf51a5c9699..bcff9eee69c 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -17,6 +17,7 @@
 
 import dask
 from dask.system import CPU_COUNT
+from dask.utils import parse_timedelta
 
 from . import preloading
 from .comm import get_address_host, unparse_host_port
@@ -32,7 +33,6 @@
     json_load_robust,
     mp_context,
     parse_ports,
-    parse_timedelta,
     silence_logging,
 )
 from .worker import Worker, parse_memory_limit, run
diff --git a/distributed/profile.py b/distributed/profile.py
index 9c375291d3e..160fe5a7b62 100644
--- a/distributed/profile.py
+++ b/distributed/profile.py
@@ -33,8 +33,10 @@
 
 import tlz as toolz
 
+from dask.utils import format_time, parse_timedelta
+
 from .metrics import time
-from .utils import color_of, format_time, parse_timedelta
+from .utils import color_of
 
 
 def identifier(frame):
diff --git a/distributed/pubsub.py b/distributed/pubsub.py
index 99c528eb5e0..91b006423ba 100644
--- a/distributed/pubsub.py
+++ b/distributed/pubsub.py
@@ -4,10 +4,12 @@
 import weakref
 from collections import defaultdict, deque
 
+from dask.utils import parse_timedelta
+
 from .core import CommClosedError
 from .metrics import time
 from .protocol.serialize import to_serialize
-from .utils import TimeoutError, parse_timedelta, sync
+from .utils import TimeoutError, sync
 
 logger = logging.getLogger(__name__)
 
diff --git a/distributed/queues.py b/distributed/queues.py
index 3dd0361c669..481f497373c 100644
--- a/distributed/queues.py
+++ b/distributed/queues.py
@@ -3,10 +3,10 @@
 import uuid
 from collections import defaultdict
 
-from dask.utils import stringify
+from dask.utils import parse_timedelta, stringify
 
 from .client import Client, Future
-from .utils import parse_timedelta, sync, thread_state
+from .utils import sync, thread_state
 from .worker import get_client, get_worker
 
 logger = logging.getLogger(__name__)
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 7dc2f8d0c33..b9c2e480c85 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -38,6 +38,7 @@
 
 import dask
 from dask.highlevelgraph import HighLevelGraph
+from dask.utils import format_bytes, format_time, parse_bytes, parse_timedelta
 
 from . import preloading, profile
 from . import versions as version_module
@@ -70,15 +71,11 @@
     All,
     TimeoutError,
     empty_context,
-    format_bytes,
-    format_time,
     get_fileno_limit,
     key_split,
     key_split_group,
     log_errors,
     no_default,
-    parse_bytes,
-    parse_timedelta,
     tmpfile,
     validate_key,
 )
diff --git a/distributed/semaphore.py b/distributed/semaphore.py
index 2de45c42f34..48aab78b38b 100644
--- a/distributed/semaphore.py
+++ b/distributed/semaphore.py
@@ -8,11 +8,12 @@
 from tornado.ioloop import IOLoop, PeriodicCallback
 
 import dask
+from dask.utils import parse_timedelta
 
 from distributed.utils_comm import retry_operation
 
 from .metrics import time
-from .utils import log_errors, parse_timedelta, sync, thread_state
+from .utils import log_errors, sync, thread_state
 from .worker import get_client, get_worker
 
 logger = logging.getLogger(__name__)
diff --git a/distributed/stealing.py b/distributed/stealing.py
index b352040bff7..1929661abc5 100644
--- a/distributed/stealing.py
+++ b/distributed/stealing.py
@@ -7,11 +7,12 @@
 from tornado.ioloop import PeriodicCallback
 
 import dask
+from dask.utils import parse_timedelta
 
 from .comm.addressing import get_address_host
 from .core import CommClosedError
 from .diagnostics.plugin import SchedulerPlugin
-from .utils import log_errors, parse_timedelta
+from .utils import log_errors
 
 LATENCY = 10e-3
 
diff --git a/distributed/tests/test_utils.py b/distributed/tests/test_utils.py
index fa384cd8451..6f94ca3d506 100644
--- a/distributed/tests/test_utils.py
+++ b/distributed/tests/test_utils.py
@@ -1,13 +1,11 @@
 import array
 import asyncio
-import datetime
 import io
 import os
 import queue
 import socket
 import sys
 import traceback
-from functools import partial
 from time import sleep
 
 import pytest
@@ -27,7 +25,6 @@
     ensure_bytes,
     ensure_ip,
     format_dashboard_link,
-    funcname,
     get_ip_interface,
     get_traceback,
     is_kernel,
@@ -35,9 +32,7 @@
     nbytes,
     offload,
     open_port,
-    parse_bytes,
     parse_ports,
-    parse_timedelta,
     read_block,
     seek_delimiter,
     set_thread_state,
@@ -250,15 +245,6 @@ def test_seek_delimiter_endline():
     assert f.tell() == 7
 
 
-def test_funcname():
-    def f():
-        pass
-
-    assert funcname(f) == "f"
-    assert funcname(partial(f)) == "f"
-    assert funcname(partial(partial(f))) == "f"
-
-
 def test_ensure_bytes():
     data = [b"1", "1", memoryview(b"1"), bytearray(b"1"), array.array("b", [49])]
     for d in data:
@@ -471,45 +457,6 @@ async def test_loop_runner_gen():
     await asyncio.sleep(0.01)
 
 
-def test_parse_bytes():
-    assert parse_bytes("100") == 100
-    assert parse_bytes("100 MB") == 100000000
-    assert parse_bytes("100M") == 100000000
-    assert parse_bytes("5kB") == 5000
-    assert parse_bytes("5.4 kB") == 5400
-    assert parse_bytes("1kiB") == 1024
-    assert parse_bytes("1Mi") == 2 ** 20
-    assert parse_bytes("1e6") == 1000000
-    assert parse_bytes("1e6 kB") == 1000000000
-    assert parse_bytes("MB") == 1000000
-
-
-def test_parse_timedelta():
-    for text, value in [
-        ("1s", 1),
-        ("100ms", 0.1),
-        ("5S", 5),
-        ("5.5s", 5.5),
-        ("5.5 s", 5.5),
-        ("1 second", 1),
-        ("3.3 seconds", 3.3),
-        ("3.3 milliseconds", 0.0033),
-        ("3500 us", 0.0035),
-        ("1 ns", 1e-9),
-        ("2m", 120),
-        ("2 minutes", 120),
-        (datetime.timedelta(seconds=2), 2),
-        (datetime.timedelta(milliseconds=100), 0.1),
-    ]:
-        result = parse_timedelta(text)
-        assert abs(result - value) < 1e-14
-
-    assert parse_timedelta("1ms", default="seconds") == 0.001
-    assert parse_timedelta("1", default="seconds") == 1
-    assert parse_timedelta("1", default="ms") == 0.001
-    assert parse_timedelta(1, default="ms") == 0.001
-
-
 @gen_test()
 async def test_all_exceptions_logging():
     async def throws():
@@ -543,11 +490,6 @@ def test_warn_on_duration():
     assert any("foo" in str(rec.message) for rec in record)
 
 
-def test_format_bytes_compat():
-    # moved to dask, but exported here for compatibility
-    from distributed.utils import format_bytes  # noqa
-
-
 def test_logs():
     log = Log("Hello")
     assert isinstance(log, str)
@@ -611,3 +553,45 @@ def test_lru():
 async def test_offload():
     assert (await offload(inc, 1)) == 2
     assert (await offload(lambda x, y: x + y, 1, y=2)) == 3
+
+
+def test_serialize_for_cli_deprecated():
+    with pytest.warns(FutureWarning, match="serialize_for_cli is deprecated"):
+        from distributed.utils import serialize_for_cli
+    assert serialize_for_cli is dask.config.serialize
+
+
+def test_deserialize_for_cli_deprecated():
+    with pytest.warns(FutureWarning, match="deserialize_for_cli is deprecated"):
+        from distributed.utils import deserialize_for_cli
+    assert deserialize_for_cli is dask.config.deserialize
+
+
+def test_parse_bytes_deprecated():
+    with pytest.warns(FutureWarning, match="parse_bytes is deprecated"):
+        from distributed.utils import parse_bytes
+    assert parse_bytes is dask.utils.parse_bytes
+
+
+def test_format_bytes_deprecated():
+    with pytest.warns(FutureWarning, match="format_bytes is deprecated"):
+        from distributed.utils import format_bytes
+    assert format_bytes is dask.utils.format_bytes
+
+
+def test_format_time_deprecated():
+    with pytest.warns(FutureWarning, match="format_time is deprecated"):
+        from distributed.utils import format_time
+    assert format_time is dask.utils.format_time
+
+
+def test_funcname_deprecated():
+    with pytest.warns(FutureWarning, match="funcname is deprecated"):
+        from distributed.utils import funcname
+    assert funcname is dask.utils.funcname
+
+
+def test_parse_timedelta_deprecated():
+    with pytest.warns(FutureWarning, match="parse_timedelta is deprecated"):
+        from distributed.utils import parse_timedelta
+    assert parse_timedelta is dask.utils.parse_timedelta
diff --git a/distributed/utils.py b/distributed/utils.py
index 33793b2f504..dcea26e628a 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -41,19 +41,7 @@
 
 import dask
 from dask import istask
-
-# Import config serialization functions here for backward compatibility
-from dask.config import deserialize as deserialize_for_cli  # noqa
-from dask.config import serialize as serialize_for_cli  # noqa
-
-# provide format_bytes here for backwards compatibility
-from dask.utils import (  # noqa: F401
-    format_bytes,
-    format_time,
-    funcname,
-    parse_bytes,
-    parse_timedelta,
-)
+from dask.utils import parse_timedelta as _parse_timedelta
 
 try:
     from tornado.ioloop import PollIOLoop
@@ -286,7 +274,7 @@ def sync(loop, func, *args, callback_timeout=None, **kwargs):
     """
     Run coroutine in loop running in separate thread.
     """
-    callback_timeout = parse_timedelta(callback_timeout, "s")
+    callback_timeout = _parse_timedelta(callback_timeout, "s")
     # Tornado's PollIOLoop doesn't raise when using closed, do it ourselves
     if PollIOLoop and (
         (isinstance(loop, PollIOLoop) and getattr(loop, "_closing", False))
@@ -1150,7 +1138,7 @@ def warn_on_duration(duration, msg):
     start = time()
     yield
     stop = time()
-    if stop - start > parse_timedelta(duration):
+    if stop - start > _parse_timedelta(duration):
         warnings.warn(msg, stacklevel=2)
 
 
@@ -1455,3 +1443,29 @@ def clean_dashboard_address(addrs: AnyType, default_listen_ip: str = "") -> List
 
         addresses.append({"address": host, "port": port})
     return addresses
+
+
+_deprecations = {
+    "deserialize_for_cli": "dask.config.deserialize",
+    "serialize_for_cli": "dask.config.serialize",
+    "format_bytes": "dask.utils.format_bytes",
+    "format_time": "dask.utils.format_time",
+    "funcname": "dask.utils.funcname",
+    "parse_bytes": "dask.utils.parse_bytes",
+    "parse_timedelta": "dask.utils.parse_timedelta",
+}
+
+
+def __getattr__(name):
+    if name in _deprecations:
+        use_instead = _deprecations[name]
+
+        warnings.warn(
+            f"{name} is deprecated and will be removed in a future release. "
+            f"Please use {use_instead} instead.",
+            category=FutureWarning,
+            stacklevel=2,
+        )
+        return import_term(use_instead)
+    else:
+        raise AttributeError(f"module {__name__} has no attribute {name}")
diff --git a/distributed/variable.py b/distributed/variable.py
index 5be0d097915..db74cabddc1 100644
--- a/distributed/variable.py
+++ b/distributed/variable.py
@@ -6,10 +6,10 @@
 
 from tlz import merge
 
-from dask.utils import stringify
+from dask.utils import parse_timedelta, stringify
 
 from .client import Client, Future
-from .utils import TimeoutError, log_errors, parse_timedelta
+from .utils import TimeoutError, log_errors
 from .worker import get_client, get_worker
 
 logger = logging.getLogger(__name__)
diff --git a/distributed/worker.py b/distributed/worker.py
index 685f6df6240..3d5146f09e1 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -25,7 +25,7 @@
 import dask
 from dask.core import istask
 from dask.system import CPU_COUNT
-from dask.utils import apply, format_bytes, funcname
+from dask.utils import apply, format_bytes, funcname, parse_bytes, parse_timedelta
 
 from . import comm, preloading, profile, system, utils
 from .batched import BatchedSend
@@ -65,9 +65,7 @@
     key_split,
     log_errors,
     offload,
-    parse_bytes,
     parse_ports,
-    parse_timedelta,
     silence_logging,
     thread_state,
     typename,
diff --git a/distributed/worker_client.py b/distributed/worker_client.py
index 7ad1cb7e20b..989a3f8f0d9 100644
--- a/distributed/worker_client.py
+++ b/distributed/worker_client.py
@@ -4,7 +4,6 @@
 import dask
 
 from .threadpoolexecutor import rejoin, secede
-from .utils import parse_timedelta
 from .worker import get_client, get_worker, thread_state
 
 
@@ -46,7 +45,7 @@ def worker_client(timeout=None, separate_thread=True):
     if timeout is None:
         timeout = dask.config.get("distributed.comm.timeouts.connect")
 
-    timeout = parse_timedelta(timeout, "s")
+    timeout = dask.utils.parse_timedelta(timeout, "s")
 
     worker = get_worker()
     client = get_client(timeout=timeout)

From 73dd5709a2e25c834c5507234f363a5d62a90a2d Mon Sep 17 00:00:00 2001
From: Naty Clementi <natyclementi@gmail.com>
Date: Thu, 24 Jun 2021 15:11:34 -0400
Subject: [PATCH 1332/1550] Add occupancy as individual plot (#4963)

---
 distributed/dashboard/scheduler.py | 2 ++
 docs/source/http_services.rst      | 1 +
 2 files changed, 3 insertions(+)

diff --git a/distributed/dashboard/scheduler.py b/distributed/dashboard/scheduler.py
index a237d21a8a5..9605c593ace 100644
--- a/distributed/dashboard/scheduler.py
+++ b/distributed/dashboard/scheduler.py
@@ -19,6 +19,7 @@
     MemoryByKey,
     NBytes,
     NBytesCluster,
+    Occupancy,
     SystemMonitor,
     TaskGraph,
     TaskProgress,
@@ -90,6 +91,7 @@ def connect(application, http_server, scheduler, prefix=""):
     "/individual-nprocessing": individual_doc(
         CurrentLoad, 100, fig_attr="processing_figure"
     ),
+    "/individual-occupancy": individual_doc(Occupancy, 100),
     "/individual-workers": individual_doc(WorkerTable, 500),
     "/individual-bandwidth-types": individual_doc(BandwidthTypes, 500),
     "/individual-bandwidth-workers": individual_doc(BandwidthWorkers, 500),
diff --git a/docs/source/http_services.rst b/docs/source/http_services.rst
index 93e6549f9df..9596f5551ef 100644
--- a/docs/source/http_services.rst
+++ b/docs/source/http_services.rst
@@ -59,6 +59,7 @@ Individual bokeh plots
 - ``/individual-nbytes-cluster``
 - ``/individual-cpu``
 - ``/individual-nprocessing``
+- ``/individual-occupancy``
 - ``/individual-workers``
 - ``/individual-bandwidth-types``
 - ``/individual-bandwidth-workers``

From d1cd7fa0e69e2f60a1938b83786c5b3745af4811 Mon Sep 17 00:00:00 2001
From: gerrymanoim <gerrymanoim@gmail.com>
Date: Thu, 24 Jun 2021 17:07:28 -0400
Subject: [PATCH 1333/1550] Except more errors in pynvml.nvmlInit() (#4970)

Closes https://github.com/dask/distributed/issues/4965

Longer term it is probably worth handling any errors emanating from `pynvml.nvmlInit()`.
---
 distributed/diagnostics/nvml.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/diagnostics/nvml.py b/distributed/diagnostics/nvml.py
index 19cc843eee9..3c66ed22760 100644
--- a/distributed/diagnostics/nvml.py
+++ b/distributed/diagnostics/nvml.py
@@ -26,7 +26,7 @@ def init_once():
     nvmlOwnerPID = os.getpid()
     try:
         pynvml.nvmlInit()
-    except pynvml.NVMLError_LibraryNotFound:
+    except (pynvml.NVMLError_LibraryNotFound, pynvml.NVMLError_DriverNotLoaded):
         nvmlLibraryNotFound = True
 
 

From d7c5e02e488833d25008ddb087ba441315a9234a Mon Sep 17 00:00:00 2001
From: Naty Clementi <natyclementi@gmail.com>
Date: Thu, 24 Jun 2021 18:40:08 -0400
Subject: [PATCH 1334/1550] Restructure nbytes hover (#4952)

---
 distributed/dashboard/components/scheduler.py | 111 ++++++++++++++----
 distributed/scheduler.py                      |   4 +-
 distributed/tests/test_scheduler.py           |   4 +-
 3 files changed, 88 insertions(+), 31 deletions(-)

diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index 7f197aa54d2..f26064ace2e 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -240,15 +240,13 @@ def __init__(self, scheduler, width=600, **kwargs):
                     "width": [0] * 4,
                     "x": [0] * 4,
                     "y": [0] * 4,
-                    "proc_memory": [0] * 4,
                     "color": ["blue", "blue", "blue", "grey"],
                     "alpha": [1, 0.7, 0.4, 1],
-                    "memtype": [
-                        "managed (in memory)",
-                        "unmanaged",
-                        "unmanaged, recently increased",
-                        "managed (spilled to disk)",
-                    ],
+                    "proc_memory": [0] * 4,
+                    "managed": [0] * 4,
+                    "unmanaged_old": [0] * 4,
+                    "unmanaged_recent": [0] * 4,
+                    "spilled": [0] * 4,
                 }
             )
 
@@ -283,9 +281,31 @@ def __init__(self, scheduler, width=600, **kwargs):
             self.root.toolbar_location = None
             self.root.yaxis.visible = False
 
-            hover = HoverTool()
-            hover.tooltips = "@width{0.00 b} @memtype"
-            hover.point_policy = "follow_mouse"
+            hover = HoverTool(
+                point_policy="follow_mouse",
+                tooltips="""
+                            <div>
+                                <span style="font-size: 12px; font-weight: bold;">Process memory (RSS):</span>&nbsp;
+                                <span style="font-size: 10px; font-family: Monaco, monospace;">@proc_memory{0.00 b}</span>
+                            </div>
+                            <div style="margin-left: 1em;">
+                                <span style="font-size: 12px; font-weight: bold;">Managed:</span>&nbsp;
+                                <span style="font-size: 10px; font-family: Monaco, monospace;">@managed{0.00 b}</span>
+                            </div>
+                            <div style="margin-left: 1em;">
+                                <span style="font-size: 12px; font-weight: bold;">Unmanaged (old):</span>&nbsp;
+                                <span style="font-size: 10px; font-family: Monaco, monospace;">@unmanaged_old{0.00 b}</span>
+                            </div>
+                            <div style="margin-left: 1em;">
+                                <span style="font-size: 12px; font-weight: bold;">Unmanaged (recent):</span>&nbsp;
+                                <span style="font-size: 10px; font-family: Monaco, monospace;">@unmanaged_recent{0.00 b}</span>
+                            </div>
+                            <div>
+                                <span style="font-size: 12px; font-weight: bold;">Spilled to disk:</span>&nbsp;
+                                <span style="font-size: 10px; font-family: Monaco, monospace;">@spilled{0.00 b}</span>
+                            </div>
+                            """,
+            )
             self.root.add_tools(hover)
 
     @without_property_validation
@@ -301,11 +321,16 @@ def update(self):
                 meminfo.unmanaged_recent,
                 meminfo.managed_spilled,
             ]
+
             result = {
                 "width": width,
                 "x": [sum(width[:i]) + w / 2 for i, w in enumerate(width)],
                 "color": [color, color, color, "grey"],
                 "proc_memory": [meminfo.process] * 4,
+                "managed": [meminfo.managed_in_memory] * 4,
+                "unmanaged_old": [meminfo.unmanaged_old] * 4,
+                "unmanaged_recent": [meminfo.unmanaged_recent] * 4,
+                "spilled": [meminfo.managed_spilled] * 4,
             }
             # FIXME https://github.com/dask/distributed/issues/4675
             #       This causes flickering after adding workers and when enough memory
@@ -333,12 +358,15 @@ def __init__(self, scheduler, width=600, **kwargs):
                     "width": [],
                     "x": [],
                     "y": [],
-                    "memtype": [],
                     "color": [],
                     "alpha": [],
-                    "proc_memory": [],
                     "worker": [],
                     "escaped_worker": [],
+                    "proc_memory": [],
+                    "managed": [],
+                    "unmanaged_old": [],
+                    "unmanaged_recent": [],
+                    "spilled": [],
                 }
             )
 
@@ -377,9 +405,35 @@ def __init__(self, scheduler, width=600, **kwargs):
             self.root.toolbar_location = None
             self.root.yaxis.visible = False
 
-            hover = HoverTool()
-            hover.tooltips = "@worker: @proc_memory{0.00 b} (@width{0.00 b} @memtype)"
-            hover.point_policy = "follow_mouse"
+            hover = HoverTool(
+                point_policy="follow_mouse",
+                tooltips="""
+                            <div>
+                                <span style="font-size: 12px; font-weight: bold;">Worker:</span>&nbsp;
+                                <span style="font-size: 10px; font-family: Monaco, monospace;">@worker</span>
+                            </div>
+                            <div>
+                                <span style="font-size: 12px; font-weight: bold;">Process memory (RSS):</span>&nbsp;
+                                <span style="font-size: 10px; font-family: Monaco, monospace;">@proc_memory{0.00 b}</span>
+                            </div>
+                            <div style="margin-left: 1em;">
+                                <span style="font-size: 12px; font-weight: bold;">Managed:</span>&nbsp;
+                                <span style="font-size: 10px; font-family: Monaco, monospace;">@managed{0.00 b}</span>
+                            </div>
+                            <div style="margin-left: 1em;">
+                                <span style="font-size: 12px; font-weight: bold;">Unmanaged (old):</span>&nbsp;
+                                <span style="font-size: 10px; font-family: Monaco, monospace;">@unmanaged_old{0.00 b}</span>
+                            </div>
+                            <div style="margin-left: 1em;">
+                                <span style="font-size: 12px; font-weight: bold;">Unmanaged (recent):</span>&nbsp;
+                                <span style="font-size: 10px; font-family: Monaco, monospace;">@unmanaged_recent{0.00 b}</span>
+                            </div>
+                            <div>
+                                <span style="font-size: 12px; font-weight: bold;">Spilled to disk:</span>&nbsp;
+                                <span style="font-size: 10px; font-family: Monaco, monospace;">@spilled{0.00 b}</span>
+                            </div>
+                            """,
+            )
             self.root.add_tools(hover)
 
     @without_property_validation
@@ -396,8 +450,12 @@ def quadlist(i) -> list:
             width = []
             x = []
             color = []
-            procmemory = []
             max_limit = 0
+            procmemory = []
+            managed = []
+            spilled = []
+            unmanaged_old = []
+            unmanaged_recent = []
 
             for ws in workers:
                 meminfo = ws.memory
@@ -415,26 +473,29 @@ def quadlist(i) -> list:
                 ]
                 x += [sum(width[-4:i]) + width[i] / 2 for i in range(-4, 0)]
                 color += [color_i, color_i, color_i, "grey"]
+
+                # memory info
                 procmemory.append(meminfo.process)
+                managed.append(meminfo.managed_in_memory)
+                unmanaged_old.append(meminfo.unmanaged_old)
+                unmanaged_recent.append(meminfo.unmanaged_recent)
+                spilled.append(meminfo.managed_spilled)
 
             result = {
                 "width": width,
                 "x": x,
                 "color": color,
                 "alpha": [1, 0.7, 0.4, 1] * len(workers),
-                "memtype": [
-                    "managed (in memory)",
-                    "unmanaged",
-                    "unmanaged, recently increased",
-                    "managed (spilled to disk)",
-                ]
-                * len(workers),
-                "proc_memory": quadlist(procmemory),
                 "worker": quadlist(ws.address for ws in workers),
                 "escaped_worker": quadlist(
                     escape.url_escape(ws.address) for ws in workers
                 ),
                 "y": quadlist(range(len(workers))),
+                "proc_memory": quadlist(procmemory),
+                "managed": quadlist(managed),
+                "unmanaged_old": quadlist(unmanaged_old),
+                "unmanaged_recent": quadlist(unmanaged_recent),
+                "spilled": quadlist(spilled),
             }
             # Remove rectangles with width=0
             result = {
@@ -1965,7 +2026,7 @@ def __init__(self, scheduler, width=800, **kwargs):
             "memory_limit": "limit",
             "memory_percent": "memory %",
             "memory_managed": "managed",
-            "memory_unmanaged_old": "unmanaged",
+            "memory_unmanaged_old": "unmanaged old",
             "memory_unmanaged_recent": "unmanaged recent",
             "memory_spilled": "spilled",
             "num_fds": "# fds",
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index b9c2e480c85..df7a43510b8 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -367,13 +367,11 @@ def optimistic(self) -> Py_ssize_t:
 
     def __repr__(self) -> str:
         return (
-            f"Managed by Dask       : {format_bytes(self.managed)}\n"
-            f"  - in process memory : {format_bytes(self._managed_in_memory)}\n"
-            f"  - spilled to disk   : {format_bytes(self._managed_spilled)}\n"
             f"Process memory (RSS)  : {format_bytes(self._process)}\n"
             f"  - managed by Dask   : {format_bytes(self._managed_in_memory)}\n"
             f"  - unmanaged (old)   : {format_bytes(self._unmanaged_old)}\n"
             f"  - unmanaged (recent): {format_bytes(self.unmanaged_recent)}\n"
+            f"Spilled to disk       : {format_bytes(self._managed_spilled)}\n"
         )
 
 
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 04f266f8e61..418ab7e7f5e 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -2290,13 +2290,11 @@ def test_memorystate():
         repr(m)
         == dedent(
             """
-            Managed by Dask       : 80 B
-              - in process memory : 68 B
-              - spilled to disk   : 12 B
             Process memory (RSS)  : 100 B
               - managed by Dask   : 68 B
               - unmanaged (old)   : 15 B
               - unmanaged (recent): 17 B
+            Spilled to disk       : 12 B
             """
         ).lstrip()
     )

From 9d60b20297ebaf23c4e2422f0a07c3339ef44048 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Thu, 24 Jun 2021 18:49:27 -0500
Subject: [PATCH 1335/1550] Remove more internal references to deprecated
 utilities (#4971)

---
 distributed/dashboard/components/__init__.py  | 4 ++--
 distributed/dashboard/components/scheduler.py | 4 ++--
 distributed/dashboard/components/shared.py    | 4 ++--
 distributed/dashboard/components/worker.py    | 4 ++--
 distributed/diagnostics/task_stream.py        | 3 ++-
 distributed/http/scheduler/info.py            | 4 ++--
 6 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/distributed/dashboard/components/__init__.py b/distributed/dashboard/components/__init__.py
index 7c891d96119..f0798417cd9 100644
--- a/distributed/dashboard/components/__init__.py
+++ b/distributed/dashboard/components/__init__.py
@@ -30,7 +30,7 @@
 
 from distributed import profile
 from distributed.dashboard.utils import BOKEH_VERSION, without_property_validation
-from distributed.utils import log_errors, parse_timedelta
+from distributed.utils import log_errors
 
 if dask.config.get("distributed.dashboard.export-tool"):
     from distributed.dashboard.export_tool import ExportTool
@@ -39,7 +39,7 @@
 
 
 profile_interval = dask.config.get("distributed.worker.profile.interval")
-profile_interval = parse_timedelta(profile_interval, default="ms")
+profile_interval = dask.utils.parse_timedelta(profile_interval, default="ms")
 
 
 class DashboardComponent:
diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index f26064ace2e..5413efca81d 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -43,7 +43,7 @@
 
 import dask
 from dask import config
-from dask.utils import format_bytes, key_split
+from dask.utils import format_bytes, format_time, key_split, parse_timedelta
 
 try:
     import numpy as np
@@ -70,7 +70,7 @@
 from distributed.diagnostics.task_stream import color_of as ts_color_of
 from distributed.diagnostics.task_stream import colors as ts_color_lookup
 from distributed.metrics import time
-from distributed.utils import Log, format_time, log_errors, parse_timedelta
+from distributed.utils import Log, log_errors
 
 if dask.config.get("distributed.dashboard.export-tool"):
     from distributed.dashboard.export_tool import ExportTool
diff --git a/distributed/dashboard/components/shared.py b/distributed/dashboard/components/shared.py
index f50145f0dbb..360f4433aa7 100644
--- a/distributed/dashboard/components/shared.py
+++ b/distributed/dashboard/components/shared.py
@@ -28,7 +28,7 @@
     update,
     without_property_validation,
 )
-from distributed.utils import log_errors, parse_timedelta
+from distributed.utils import log_errors
 
 if dask.config.get("distributed.dashboard.export-tool"):
     from distributed.dashboard.export_tool import ExportTool
@@ -37,7 +37,7 @@
 
 
 profile_interval = dask.config.get("distributed.worker.profile.interval")
-profile_interval = parse_timedelta(profile_interval, default="ms")
+profile_interval = dask.utils.parse_timedelta(profile_interval, default="ms")
 
 
 class Processing(DashboardComponent):
diff --git a/distributed/dashboard/components/worker.py b/distributed/dashboard/components/worker.py
index 9266e1f2766..e664cd4cae5 100644
--- a/distributed/dashboard/components/worker.py
+++ b/distributed/dashboard/components/worker.py
@@ -20,7 +20,7 @@
 from bokeh.themes import Theme
 from tlz import merge, partition_all
 
-from dask.utils import format_bytes
+from dask.utils import format_bytes, format_time
 
 from distributed.dashboard.components import add_periodic_callback
 from distributed.dashboard.components.shared import (
@@ -32,7 +32,7 @@
 from distributed.dashboard.utils import transpose, update, without_property_validation
 from distributed.diagnostics.progress_stream import color_of
 from distributed.metrics import time
-from distributed.utils import format_time, key_split, log_errors
+from distributed.utils import key_split, log_errors
 
 logger = logging.getLogger(__name__)
 
diff --git a/distributed/diagnostics/task_stream.py b/distributed/diagnostics/task_stream.py
index e276c477776..fab787ae07c 100644
--- a/distributed/diagnostics/task_stream.py
+++ b/distributed/diagnostics/task_stream.py
@@ -2,9 +2,10 @@
 from collections import deque
 
 import dask
+from dask.utils import format_time, parse_timedelta
 
 from ..metrics import time
-from ..utils import format_time, key_split, parse_timedelta
+from ..utils import key_split
 from .plugin import SchedulerPlugin
 from .progress_stream import color_of
 
diff --git a/distributed/http/scheduler/info.py b/distributed/http/scheduler/info.py
index 588fbc1505f..44197141e9e 100644
--- a/distributed/http/scheduler/info.py
+++ b/distributed/http/scheduler/info.py
@@ -8,11 +8,11 @@
 from tornado import escape
 from tornado.websocket import WebSocketHandler
 
-from dask.utils import format_bytes
+from dask.utils import format_bytes, format_time
 
 from ...diagnostics.websocket import WebsocketPlugin
 from ...metrics import time
-from ...utils import format_time, log_errors
+from ...utils import log_errors
 from ..utils import RequestHandler, redirect
 
 ns = {

From c2557938e6c4175534031cba5ca5ac9d2cdc95f7 Mon Sep 17 00:00:00 2001
From: Naty Clementi <natyclementi@gmail.com>
Date: Thu, 24 Jun 2021 20:17:46 -0400
Subject: [PATCH 1336/1550] Task Group Graph Visualization (#4886)

---
 distributed/dashboard/components/scheduler.py | 470 ++++++++++++++++++
 distributed/dashboard/scheduler.py            |  15 +-
 .../dashboard/tests/test_scheduler_bokeh.py   |  63 +++
 distributed/http/static/images/numpy.png      | Bin 0 -> 18663 bytes
 distributed/http/static/images/pandas.png     | Bin 0 -> 1213 bytes
 distributed/http/static/images/python.png     | Bin 0 -> 830916 bytes
 docs/source/http_services.rst                 |   4 +-
 7 files changed, 550 insertions(+), 2 deletions(-)
 create mode 100644 distributed/http/static/images/numpy.png
 create mode 100644 distributed/http/static/images/pandas.png
 create mode 100644 distributed/http/static/images/python.png

diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index 5413efca81d..f5631f78989 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -9,6 +9,7 @@
 from bokeh.layouts import column, row
 from bokeh.models import (
     AdaptiveTicker,
+    Arrow,
     BasicTicker,
     BoxSelectTool,
     BoxZoomTool,
@@ -28,6 +29,7 @@
     Tabs,
     TapTool,
     Title,
+    VeeHead,
     WheelZoomTool,
     value,
 )
@@ -92,6 +94,13 @@
 XLABEL_ORIENTATION = -math.pi / 12  # slanted downwards 15 degrees
 
 
+logos_dict = {
+    "numpy": "statics/images/numpy.png",
+    "pandas": "statics/images/pandas.png",
+    "builtins": "statics/images/python.png",
+}
+
+
 class Occupancy(DashboardComponent):
     """Occupancy (in time) per worker"""
 
@@ -1784,6 +1793,455 @@ def __del__(self):
         self.scheduler.remove_plugin(self.layout)
 
 
+class TaskGroupGraph(DashboardComponent):
+    """
+    Task Group Graph
+
+    Creates a graph layout for TaskGroups on the scheduler.  It assigns
+    (x, y) locations to all the TaskGroups and lays them out by according
+    to their dependencies. The layout gets updated every time that new
+    TaskGroups are added.
+
+    Each task group node incodes information about task progress, memory,
+    and output type into glyphs, as well as a hover tooltip with more detailed
+    information on name, computation time, memory, and tasks status.
+    """
+
+    def __init__(self, scheduler, **kwargs):
+        self.scheduler = scheduler
+
+        self.nodes_layout = {}
+        self.arrows_layout = {}
+
+        self.old_counter = -1
+
+        self.nodes_source = ColumnDataSource(
+            {
+                "x": [],
+                "y": [],
+                "w_box": [],
+                "h_box": [],
+                "name": [],
+                "tot_tasks": [],
+                "color": [],
+                "x_start": [],
+                "x_end": [],
+                "y_start": [],
+                "y_end": [],
+                "x_end_progress": [],
+                "mem_alpha": [],
+                "node_line_width": [],
+                "comp_tasks": [],
+                "url_logo": [],
+                "x_logo": [],
+                "y_logo": [],
+                "w_logo": [],
+                "h_logo": [],
+                "in_processing": [],
+                "in_memory": [],
+                "in_released": [],
+                "in_erred": [],
+                "compute_time": [],
+                "memory": [],
+            }
+        )
+
+        self.arrows_source = ColumnDataSource({"xs": [], "ys": [], "xe": [], "ye": []})
+
+        self.root = figure(title="Task Groups Graph", match_aspect=True, **kwargs)
+        self.root.axis.visible = False
+        self.subtitle = Title(text=" ", text_font_style="italic")
+        self.root.add_layout(self.subtitle, "above")
+
+        rect = self.root.rect(
+            x="x",
+            y="y",
+            width="w_box",
+            height="h_box",
+            color="color",
+            fill_alpha="mem_alpha",
+            line_color="black",
+            line_width="node_line_width",
+            source=self.nodes_source,
+        )
+
+        ####plot tg log
+        self.root.image_url(
+            url="url_logo",
+            x="x_logo",
+            y="y_logo",
+            w="w_logo",
+            h="h_logo",
+            anchor="center",
+            source=self.nodes_source,
+        )
+
+        # progress bar plain box
+        self.root.quad(
+            left="x_start",
+            right="x_end",
+            bottom="y_start",
+            top="y_end",
+            color=None,
+            line_color="black",
+            source=self.nodes_source,
+        )
+
+        # progress bar
+        self.root.quad(
+            left="x_start",
+            right="x_end_progress",
+            bottom="y_start",
+            top="y_end",
+            color="color",
+            line_color=None,
+            fill_alpha=0.6,
+            source=self.nodes_source,
+        )
+
+        self.arrows = Arrow(
+            end=VeeHead(size=8),
+            line_color="black",
+            line_alpha=0.5,
+            line_width=1,
+            x_start="xs",
+            y_start="ys",
+            x_end="xe",
+            y_end="ye",
+            source=self.arrows_source,
+        )
+        self.root.add_layout(self.arrows)
+
+        self.root.xgrid.grid_line_color = None
+        self.root.ygrid.grid_line_color = None
+        self.root.x_range.range_padding = 0.5
+        self.root.y_range.range_padding = 0.5
+
+        hover = HoverTool(
+            point_policy="follow_mouse",
+            tooltips="""
+                <div>
+                    <span style="font-size: 12px; font-weight: bold;">Name:</span>&nbsp;
+                    <span style="font-size: 10px; font-family: Monaco, monospace;">@name</span>
+                </div>
+                <div>
+                    <span style="font-size: 12px; font-weight: bold;">Compute time:</span>&nbsp;
+                    <span style="font-size: 10px; font-family: Monaco, monospace;">@compute_time</span>
+                </div>
+                <div>
+                    <span style="font-size: 12px; font-weight: bold;">Memory:</span>&nbsp;
+                    <span style="font-size: 10px; font-family: Monaco, monospace;">@memory</span>
+                </div>
+                <div>
+                    <span style="font-size: 12px; font-weight: bold;">Tasks:</span>&nbsp;
+                    <span style="font-size: 10px; font-family: Monaco, monospace;">@tot_tasks</span>
+                </div>
+                <div style="margin-left: 2em;">
+                    <span style="font-size: 12px; font-weight: bold;">Completed:</span>&nbsp;
+                    <span style="font-size: 10px; font-family: Monaco, monospace;">@comp_tasks</span>
+                </div>
+                <div style="margin-left: 2em;">
+                    <span style="font-size: 12px; font-weight: bold;">Processing:</span>&nbsp;
+                    <span style="font-size: 10px; font-family: Monaco, monospace;">@in_processing</span>
+                </div>
+                <div style="margin-left: 2em;">
+                    <span style="font-size: 12px; font-weight: bold;">In memory:</span>&nbsp;
+                    <span style="font-size: 10px; font-family: Monaco, monospace;">@in_memory</span>
+                </div>
+                <div style="margin-left: 2em;">
+                    <span style="font-size: 12px; font-weight: bold;">Erred:</span>&nbsp;
+                    <span style="font-size: 10px; font-family: Monaco, monospace;">@in_erred</span>
+                </div>
+                <div style="margin-left: 2em;">
+                    <span style="font-size: 12px; font-weight: bold;">Released:</span>&nbsp;
+                    <span style="font-size: 10px; font-family: Monaco, monospace;">@in_released</span>
+                </div>
+                """,
+            renderers=[rect],
+        )
+
+        self.root.add_tools(hover)
+
+    @without_property_validation
+    def update_layout(self):
+
+        with log_errors():
+            # get dependecies per task group
+            # in some cases there are tg that have themeselves as dependencies, we remove those.
+            dependencies = {
+                k: {ds.name for ds in ts.dependencies if ds.name != k}
+                for k, ts in self.scheduler.task_groups.items()
+            }
+
+            import dask
+
+            order = dask.order.order(
+                dsk={group.name: 1 for k, group in self.scheduler.task_groups.items()},
+                dependencies=dependencies,
+            )
+
+            ordered = sorted(self.scheduler.task_groups, key=order.get)
+
+            xs = {}
+            ys = {}
+            locations = set()
+            nodes_layout = {}
+            arrows_layout = {}
+            for tg in ordered:
+                if dependencies[tg]:
+                    x = max(xs[dep] for dep in dependencies[tg]) + 1
+                    y = max(ys[dep] for dep in dependencies[tg])
+                    if (
+                        len(dependencies[tg]) > 1
+                        and len({ys[dep] for dep in dependencies[tg]}) == 1
+                    ):
+                        y += 1
+                else:
+                    x = 0
+                    y = max(ys.values()) + 1 if ys else 0
+
+                while (x, y) in locations:  # avoid collisions by moving up
+                    y += 1
+
+                locations.add((x, y))
+
+                xs[tg], ys[tg] = x, y
+
+                # info neded for node layout to coulmn data source
+                nodes_layout[tg] = {"x": xs[tg], "y": ys[tg]}
+
+                # info needed for arrow layout
+                arrows_layout[tg] = {
+                    "nstart": dependencies[tg],
+                    "nend": [tg] * len(dependencies[tg]),
+                }
+
+            return nodes_layout, arrows_layout
+
+    def compute_size(self, x, min_box, max_box):
+        start = 0.4
+        end = 0.8
+
+        y = (end - start) / (max_box - min_box) * (x - min_box) + start
+
+        return y
+
+    @without_property_validation
+    def update(self):
+
+        if self.scheduler.transition_counter == self.old_counter:
+            return
+        else:
+            self.old_counter = self.scheduler.transition_counter
+
+        if not self.scheduler.task_groups:
+            self.subtitle.text = "Scheduler is empty."
+        else:
+            self.subtitle.text = " "
+
+        if self.nodes_layout.keys() != self.scheduler.task_groups.keys():
+            self.nodes_layout, self.arrows_layout = self.update_layout()
+
+        nodes_data = {
+            "x": [],
+            "y": [],
+            "w_box": [],
+            "h_box": [],
+            "name": [],
+            "color": [],
+            "tot_tasks": [],
+            "x_start": [],
+            "x_end": [],
+            "y_start": [],
+            "y_end": [],
+            "x_end_progress": [],
+            "mem_alpha": [],
+            "node_line_width": [],
+            "comp_tasks": [],
+            "url_logo": [],
+            "x_logo": [],
+            "y_logo": [],
+            "w_logo": [],
+            "h_logo": [],
+            "in_processing": [],
+            "in_memory": [],
+            "in_released": [],
+            "in_erred": [],
+            "compute_time": [],
+            "memory": [],
+        }
+
+        arrows_data = {
+            "xs": [],
+            "ys": [],
+            "xe": [],
+            "ye": [],
+        }
+
+        durations = set()
+        nbytes = set()
+        for key, tg in self.scheduler.task_groups.items():
+
+            if tg.duration and tg.nbytes_total:
+                durations.add(tg.duration)
+                nbytes.add(tg.nbytes_total)
+
+        durations_min = min(durations, default=0)
+        durations_max = max(durations, default=0)
+        nbytes_min = min(nbytes, default=0)
+        nbytes_max = max(nbytes, default=0)
+
+        box_dim = {}
+        for key, tg in self.scheduler.task_groups.items():
+
+            comp_tasks = (
+                tg.states["released"] + tg.states["memory"] + tg.states["erred"]
+            )
+            tot_tasks = sum(tg.states.values())
+
+            # compute width and height of boxes
+            if (
+                tg.duration
+                and tg.nbytes_total
+                and comp_tasks
+                and len(durations) > 1
+                and len(nbytes) > 1
+            ):
+
+                # scale duration (width)
+                width_box = self.compute_size(
+                    tg.duration / comp_tasks * tot_tasks,
+                    min_box=durations_min / comp_tasks * tot_tasks,
+                    max_box=durations_max / comp_tasks * tot_tasks,
+                )
+
+                # need to scale memory (height)
+                height_box = self.compute_size(
+                    tg.nbytes_total / comp_tasks * tot_tasks,
+                    min_box=nbytes_min / comp_tasks * tot_tasks,
+                    max_box=nbytes_max / comp_tasks * tot_tasks,
+                )
+
+            else:
+                width_box = 0.6
+                height_box = width_box / 2
+
+            box_dim[key] = {"width": width_box, "height": height_box}
+
+        for key, tg in self.scheduler.task_groups.items():
+            x = self.nodes_layout[key]["x"]
+            y = self.nodes_layout[key]["y"]
+            width = box_dim[key]["width"]
+            height = box_dim[key]["height"]
+
+            # main boxes layout
+            nodes_data["x"].append(x)
+            nodes_data["y"].append(y)
+            nodes_data["w_box"].append(width)
+            nodes_data["h_box"].append(height)
+
+            comp_tasks = (
+                tg.states["released"] + tg.states["memory"] + tg.states["erred"]
+            )
+            tot_tasks = sum(tg.states.values())
+
+            nodes_data["name"].append(tg.prefix.name)
+
+            nodes_data["color"].append(color_of(tg.prefix.name))
+            nodes_data["tot_tasks"].append(tot_tasks)
+
+            # memory alpha factor by 0.4 if not get's too dark
+            nodes_data["mem_alpha"].append(
+                (tg.states["memory"] / sum(tg.states.values())) * 0.4
+            )
+
+            # main box line width
+            if tg.states["processing"]:
+                nodes_data["node_line_width"].append(5)
+            else:
+                nodes_data["node_line_width"].append(1)
+
+            # progress bar data update
+            nodes_data["x_start"].append(x - width / 2)
+            nodes_data["x_end"].append(x + width / 2)
+
+            nodes_data["y_start"].append(y - height / 2)
+            nodes_data["y_end"].append(y - height / 2 + height * 0.4)
+
+            nodes_data["x_end_progress"].append(
+                x - width / 2 + width * comp_tasks / tot_tasks
+            )
+
+            # arrows
+            arrows_data["xs"] += [
+                self.nodes_layout[k]["x"] + box_dim[k]["width"] / 2
+                for k in self.arrows_layout[key]["nstart"]
+            ]
+            arrows_data["ys"] += [
+                self.nodes_layout[k]["y"] for k in self.arrows_layout[key]["nstart"]
+            ]
+            arrows_data["xe"] += [
+                self.nodes_layout[k]["x"] - box_dim[k]["width"] / 2
+                for k in self.arrows_layout[key]["nend"]
+            ]
+            arrows_data["ye"] += [
+                self.nodes_layout[k]["y"] for k in self.arrows_layout[key]["nend"]
+            ]
+
+            # LOGOS
+            if len(tg.types) == 1:
+                logo_type = next(iter(tg.types)).split(".")[0]
+                try:
+                    url_logo = logos_dict[logo_type]
+                except KeyError:
+                    url_logo = ""
+            else:
+                url_logo = ""
+
+            nodes_data["url_logo"].append(url_logo)
+
+            nodes_data["x_logo"].append(x + width / 3)
+            nodes_data["y_logo"].append(y + height / 3)
+
+            ratio = width / height
+
+            if ratio > 1:
+                nodes_data["h_logo"].append(height * 0.3)
+                nodes_data["w_logo"].append(width * 0.3 / ratio)
+            else:
+                nodes_data["h_logo"].append(height * 0.3 * ratio)
+                nodes_data["w_logo"].append(width * 0.3)
+
+            # compute_time and memory
+            nodes_data["compute_time"].append(format_time(tg.duration))
+            nodes_data["memory"].append(format_bytes(tg.nbytes_total))
+
+            # Add some status to hover
+            tasks_processing = tg.states["processing"]
+            tasks_memory = tg.states["memory"]
+            tasks_relased = tg.states["released"]
+            tasks_erred = tg.states["erred"]
+
+            nodes_data["comp_tasks"].append(
+                f"{comp_tasks} ({comp_tasks / tot_tasks * 100:.0f} %)"
+            )
+            nodes_data["in_processing"].append(
+                f"{tasks_processing} ({tasks_processing/ tot_tasks * 100:.0f} %)"
+            )
+            nodes_data["in_memory"].append(
+                f"{tasks_memory} ({tasks_memory/ tot_tasks * 100:.0f} %)"
+            )
+            nodes_data["in_released"].append(
+                f"{tasks_relased} ({tasks_relased/ tot_tasks * 100:.0f} %)"
+            )
+            nodes_data["in_erred"].append(
+                f"{ tasks_erred} ({tasks_erred/ tot_tasks * 100:.0f} %)"
+            )
+
+        self.nodes_source.data.update(nodes_data)
+        self.arrows_source.data.update(arrows_data)
+
+
 class TaskProgress(DashboardComponent):
     """Progress bars per task type"""
 
@@ -2330,6 +2788,18 @@ def graph_doc(scheduler, extra, doc):
         doc.theme = BOKEH_THEME
 
 
+def tg_graph_doc(scheduler, extra, doc):
+    with log_errors():
+        tg_graph = TaskGroupGraph(scheduler, sizing_mode="stretch_both")
+        doc.title = "Dask: Task Groups Graph"
+        tg_graph.update()
+        add_periodic_callback(doc, tg_graph, 200)
+        doc.add_root(tg_graph.root)
+        doc.template = env.get_template("simple.html")
+        doc.template_variables.update(extra)
+        doc.theme = BOKEH_THEME
+
+
 def status_doc(scheduler, extra, doc):
     with log_errors():
         nbytes_cluster = NBytesCluster(scheduler, sizing_mode="stretch_both")
diff --git a/distributed/dashboard/scheduler.py b/distributed/dashboard/scheduler.py
index 9605c593ace..a259b481ef6 100644
--- a/distributed/dashboard/scheduler.py
+++ b/distributed/dashboard/scheduler.py
@@ -22,6 +22,7 @@
     Occupancy,
     SystemMonitor,
     TaskGraph,
+    TaskGroupGraph,
     TaskProgress,
     TaskStream,
     WorkerTable,
@@ -36,13 +37,23 @@
     stealing_doc,
     systemmonitor_doc,
     tasks_doc,
+    tg_graph_doc,
     workers_doc,
 )
 from .core import BokehApplication
 from .worker import counters_doc
 
 template_variables = {
-    "pages": ["status", "workers", "tasks", "system", "profile", "graph", "info"]
+    "pages": [
+        "status",
+        "workers",
+        "tasks",
+        "system",
+        "profile",
+        "graph",
+        "groups",
+        "info",
+    ]
 }
 
 if NVML_ENABLED:
@@ -79,12 +90,14 @@ def connect(application, http_server, scheduler, prefix=""):
     "/profile": profile_doc,
     "/profile-server": profile_server_doc,
     "/graph": graph_doc,
+    "/groups": tg_graph_doc,
     "/gpu": gpu_doc,
     "/individual-task-stream": individual_doc(
         TaskStream, 100, n_rectangles=1000, clear_interval="10s"
     ),
     "/individual-progress": individual_doc(TaskProgress, 100, height=160),
     "/individual-graph": individual_doc(TaskGraph, 200),
+    "/individual-groups": individual_doc(TaskGroupGraph, 200),
     "/individual-nbytes": individual_doc(NBytes, 100),
     "/individual-nbytes-cluster": individual_doc(NBytesCluster, 100),
     "/individual-cpu": individual_doc(CurrentLoad, 100, fig_attr="cpu_figure"),
diff --git a/distributed/dashboard/tests/test_scheduler_bokeh.py b/distributed/dashboard/tests/test_scheduler_bokeh.py
index 68578565bf5..38b8d08c1c6 100644
--- a/distributed/dashboard/tests/test_scheduler_bokeh.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh.py
@@ -35,6 +35,7 @@
     StealingTimeSeries,
     SystemMonitor,
     TaskGraph,
+    TaskGroupGraph,
     TaskProgress,
     TaskStream,
     WorkerTable,
@@ -609,6 +610,68 @@ async def test_TaskGraph_order(c, s, a, b):
     assert gp.node_source.data["state"][gp.layout.index[y.key]] == "erred"
 
 
+@gen_cluster(client=True)
+async def test_TaskGroupGraph(c, s, a, b):
+    tgg = TaskGroupGraph(s)
+    futures = c.map(inc, range(10))
+    await wait(futures)
+
+    tgg.update()
+    assert all(len(L) == 1 for L in tgg.nodes_source.data.values())
+    assert tgg.nodes_source.data["name"] == ["inc"]
+    assert tgg.nodes_source.data["tot_tasks"] == [10]
+
+    assert all(len(L) == 0 for L in tgg.arrows_source.data.values())
+
+    futures2 = c.map(dec, range(5))
+    await wait(futures2)
+
+    tgg.update()
+    assert all(len(L) == 2 for L in tgg.nodes_source.data.values())
+    assert tgg.nodes_source.data["name"] == ["inc", "dec"]
+    assert tgg.nodes_source.data["tot_tasks"] == [10, 5]
+
+    del futures, futures2
+    while s.task_groups:
+        await asyncio.sleep(0.01)
+
+    tgg.update()
+    assert not any(tgg.nodes_source.data.values())
+
+
+@gen_cluster(client=True)
+async def test_TaskGroupGraph_arrows(c, s, a, b):
+    tgg = TaskGroupGraph(s)
+
+    futures = c.map(inc, range(10))
+    await wait(futures)
+
+    tgg.update()
+    assert all(len(L) == 1 for L in tgg.nodes_source.data.values())
+    assert tgg.nodes_source.data["name"] == ["inc"]
+    assert tgg.nodes_source.data["tot_tasks"] == [10]
+
+    assert all(len(L) == 0 for L in tgg.arrows_source.data.values())
+
+    futures2 = c.map(dec, futures)
+    await wait(futures2)
+
+    tgg.update()
+    assert all(len(L) == 2 for L in tgg.nodes_source.data.values())
+    assert tgg.nodes_source.data["name"] == ["inc", "dec"]
+    assert tgg.nodes_source.data["tot_tasks"] == [10, 10]
+
+    assert all(len(L) == 1 for L in tgg.arrows_source.data.values())
+
+    del futures, futures2
+    while s.task_groups:
+        await asyncio.sleep(0.01)
+
+    tgg.update()  ###for some reason after deleting the futures the tgg.node_source.data.values are not clear.
+    assert not any(tgg.nodes_source.data.values())
+    assert not any(tgg.arrows_source.data.values())
+
+
 @gen_cluster(
     client=True,
     config={
diff --git a/distributed/http/static/images/numpy.png b/distributed/http/static/images/numpy.png
new file mode 100644
index 0000000000000000000000000000000000000000..4d663fe0a4791b0f7f7bb44c2c923a91cedcabd6
GIT binary patch
literal 18663
zcmeHv`9IX{_y21QjmkY}gzjeSTP2iYW(Z}^k~@`AWUExlHufx|&4@xuma$Y)cS)oT
zVagOmqYP8_rDYmZB4i2kJ)`^m+xJiS{LrJ<>w2B*T<1FHT<1FHd0ivf#o6|+Wq&V2
z5acg=JFA@tg650<NiKnwBU_Foz<*1F?cAvdBCjd>hqB+PF^V86h`rU;T~Um0y$+%H
z_f=(sJ=n=-t7McChf`@2;yZg}zvBK*42kK$wOhi&vv}+5vZZwVGw1WnPm)-YSYcC}
zn6V4%hF-LLz<ew2s-*E7M<gdGZue#3WT9~GCJKiAGNr(S|B#<5ZSbmsQzO93M%Dj&
z`9B@}fBc|m#rNAMzI%vSbS?<Yg)?ccl88irTW>UH(-kMq(_^_`sKWa9{wb?@a>#4-
zHdK+zzK}YnQnW>)6&aEHl0)p*;-ZRFz{i$V_%niV0>lN$z#=&&8`+PO!;z+JI}FY7
zFFdcCMMe(T6{9T<*<2Vu?n^*6s<ok1aJa@9qY5mtN`{XZROWn)&7)vwtCGWzZ2Lot
z!!V^_7)Ral#bLa+!7y4oDT~7xpkNsFb*78MC}zMg8N1{che@?4Ml+df(-wzWqS1!(
zR(UqG$QeZsLx?*^7nu!60yC@zyHJCR8bILrmj-)Ejf?V@hUW+kcADrJ1J4DjY>Xz*
zEQMz!Rkp3@83oUe=;QV<!zY$7DK)k~G)vx-<s&L=Lv?s`Dn`p>5nFR97!{2StOarC
zsZPQsCu)V+zF2y^3PRrs-_H{DyBZ&qri3H6wc1d(3W)d4Q7}bE)d;ks$>X<y_iYgV
z)&-)pbt&3qC2-lyiazX1psj^zoYQ7kSwizBXs%$9cE|`2Is0O?3qFLc1gf?MN#raU
zRPhR|g)FGB6UURXD@CVo2J%NNlKKh(asbHF2_fv$K<*CY2P_$e>Od|J<drzy0jpB9
z!MY4A^B9Y?3e4KFtr%@U3US^F<e4k@h@%a|)t~~))By79INmN1`C1@9%p$D>@>CHy
zF~nH~<lPDeU82rj>Ifu~K=O`7iUl1J&)H!HRKWY*ZF*xp`-An*=@Y;k1EPGhW;E=F
z=M$o5D@MaV@N5FlruK}6B6!{b<D1$s8g9X}G(7jHvwe2LKqsuo$m${dA+W^<@Th0W
z)F#lVq;oM^T$8OD;7d@^$-v5_5|ggc!)e}DWDA)g{5kO7lR$hK$8%X(fpsBeU|rUP
zV0XhDE(7b!vOF2!aVymhbL%uQ=`D+Fp$Q!aS)|}33Py)vhtau2#1yD-QOhP3w6RDL
z`Ce|xOs!|Qa@X#uF200L5>T|y9veFt_vi_%rUyrIz4A(ChCRgg&DKln<K<|fQ8gwO
znos8mPd*03)ck&UO3XzbWLQ;!y*e4vmoyVD49xqAK^Ix65sG$3`pzrx6t_*UzsnoK
z7kNWd==a+K^USV<i21wJzjnu58Jf%FEO#-3d7h>u4Ns)Go4{@m4$6B*57#O1FGBBf
z6sd!Il~~9L)nlX9g6o~c3DRp@kC5s^ZX?6mZJ<0sI-!+}k%R>soJL9NjOLOoU@#gc
zIXv51NPM_F1S@j|tY_~_&>D2-A;-WYBh<J)a>|_0KGIWU6VS#bhbx};&qt4{fXTsY
z@SmuA+5IN5h^ZBDm8-y-jFnQMR~t$ZB*nxzrR}T<i$Xq}vBP+~e80V~*x$51^uW@V
zN;Zl}hc-Fzl>Dsl3g=A);aKJ~F!l8n0`Vaj9S@7mU}=T(3sV7}_6$VP(&YY2hoKFK
zFtjd<EHMrv*+BHlzD&8PBI^DXy1UB=4v4sCgKbsT`R1oaaY;xf4pxGFsi9$-E81Hg
zrcs(4uB8P;m+de`lD_#F(Tu;LC#){(QbU6fti>KHvI`@kqxNgzKRuh?R5hx;(BzUX
zHvWrB*UZ4meRO6_3G;hqgx?RDl*<X`s}fetQ_67(`eIe%zJ1#Ln~_VfP&!Yju^(5>
zU`sv2dHh2jrQ<SbuFF3qM|Ms7TBw{*BQ$Z9M9~nV#qCsMqmQ`t9+{m__}x;!w-nvE
zf^QLVfex{5i$TQ&nH0Y?V*eL^@umxQnEM;NMUgTwc{2U9n+(7DoiJWF-;_{MxJqTU
z8X>!v{vS`vQQ|`to2P;FaPJ6ce{=f3_B$HvQ`w`{s@%j*dYE^p;Y$eBO4*cqC;JC2
z4$bDT6fi$okqvTEA-1lbXR=~4=GaRvzWQ;Cg3(fMOMG__f_~35$ec0|ax}5nKYxn%
zyD@d4!<V3#Z-=pQb@@*?-Q;nLos6maQ};G5xCyh$aPdls&yvHugC#_J0}R-D_VY%1
zM92EJlntcD8PENuVm7L6iO(L~8QFAZCVDWW?rz#7wI5H&q!hlXpRLy(dK%B~dMCV4
zyM97>9XAo8&%oi8J>K76`xS`c1es$e<gL!U$5vpaK=YX8a=dOUEAKVjd+Sa-w!<#i
zHCOfDdEGl>*RFqg!QlltSS|zmZGEsv)~&lIYL}Bnd!@IE&(+Ebm|LyL<%S^(Iu%Nn
zDCV!<KVYPcBVD^AGs=`@6Lzr3<q8>CQwPQm$5Plnrqn26W1iTl=}ruIe4=gg_^IT{
zBGznW{7Rh-^l|Ov@Bw=@!t@2Aaw<i7duc*rCHvH2w_ZWmyv}EL9(9F)dCH1BdY-6!
zUiA3{H(G9JPREz<#I_hc6ifEqa*I|m;=hF+HvdGHcS=UUJZqQ4WX&8CcXUDd_5!cq
zdBw|=qya0k_@^&xR|EYrKT)%tdCmJW<xClY%q6=d3s&?aR*cI^R<(aVX7Kk)2&1Zs
z8CVp3+&9+o8OPdO=AE3P#WhvQkb;%<kus<4lFlWM`$l{0b~z)S0bMc*!sFMoE7iGI
zL)8dtHQ0Alxu;%l;?A2c$ryTCfA(o=yzWMA?$w>}QH^~^^TNs38r<J$K8)Wpcf4wb
z`-;Efcn4(#E~_%ID`j{-^vR+_vo^=Q4l7#K4DY`g0PFdRQ>i5cUpYrck|y_5?$^i#
zh$t_jKeM7AvdD7I#g>UIQZhyQXmLHNAYAEmfSsZ*VVP#zRh7WfsMFA1RKL`V)7lz7
z@zLGneR?_cu_?Ah^-wC*E0o$36V!X2U^41BMZ%fs6BB;2jQ(gHm}6=(6|2EL)%7vx
zWf0$=bw1|Bvgi>r4n$1YDORoENm-R*CRzP5=D4lkkvkdVQWe;o75sGB5VqZEcbCY8
zl4J6_&C!#hoQljV2xphE$OmBDB`gvSX1gYYjX=Y;;)YZf$q*WnCrZc7O5?+vSJ5d+
zImuz!i9~I5;PWEyHSFJlrgglGj3>@aM=P?U9V1iK*vBPZv@(8_68+bf;S4rpU~w`$
zIi{21ftK*oQb1{CMNVyUS}75VTe2#IplS(38#A!#!LHU4J*S_g!X~A*h1XNbesx(O
zo@!OxK!fcYXmKlea!ui4OSIa2kRb*Fq)m&itvSS4QRU3Eup;BPGH~g628xuLVK(RC
zryG6_5+2$Ec4`Ey3j3LpBHn@FN_xMPm~NM}g-)HmkZ170b<vK={`t5>f%rH3;sz{>
zWZLYcXllv8F`&h+xB<r^eS71ic+CF&#f7OM?Zzd*lB7mZQeh7s4ty@z`cS;!6B(5z
z!w*hcy;jj90T%%YRS)rjmB{zL?-er%$)~MXBd|!0gH)l(7p>IX8^n4$A#7jY8pFj_
zWH=ul!fWxgQithO#Y+P5!Q`-QD|iMb#y-1Lw+QC;G8zUvv?U#iap#Ch-<~YJZYR?D
zD+hlWo6qE}M74Yw7U@8%lcKd<E4f~lX8`kiF2mpD#IOwMIh}DsjyFH~>9V1CWoUj;
zj9UKMWU5X<p22+;w(7nz+%AV=*OejIju6~SU5_0_iqK+P>?&dG>axYy$MN~3_&FG2
zvlZEzYA7-IWT~GV-(AG+#{~nqH{mCTlEcnvuvJZseWcW0%vjI1R<f@umEq)osay#5
z7%+VX=2^qO8Uu~MS3Cr3dcA(BiC5uzGop*>P0_xR3tOd366L0%T7GFV)q96B$0?h_
zGa16m2rgL^XKIFE@%E+TPBGk>aLy)k@>&Zu?yD?2?R7(V#cYannP<3h*%~H}Meda4
zAK1!3tB<*dKYc^^aE7A2S}orlIBEnwn!Z56L=4P6`({P1l;JV$dOXH-E3n^~lc!+|
z(b8lmX2gE4S3Cx(P#CTAGOrkIA|r5gE;d{mf=&GtFv}e=ds|Oz`gX@NTz}`Sl<<Ky
zzWJ}{VK$<Ubm|gKZXzK$+-B;OrIdwF-=wN0d)3a_=;~u`y$>mvg3&eenClh5D}`)8
zi#$>rOl5u56)>~X_<w)OC`g4-)>L5e77WAH71+UKs+M+J&KZz)bVKdkT55TA9wjN5
zMXHqLw`?y(O9%T|Jz|k}wQi<&CWqaJ?9SNcl2xf?xW}ocv!D}S!nRpPA9taTe{+*B
z0dnzI7wohhpmns={kAIPRzr+q&`*ndb^8$GXmWT{L4ZsrjxV>h)N*B=krTDnY@Y`?
zzumdiGJ-{VA;X7c!ZO1sq(E#w#nn#3mvBg%z2<`5O*`NX?qgrk=k<7DB%7%L#4Gr6
z5C$Lh-3U6Szza?i$fr6pTun2aSN2R;81>!AyS~GDsVO^r`4VmRzh~`kt__UwHX6cX
z2dX7(Axj6}&@|p+kdwwg{ypCkn?&tBS({9~t=aZ(Zl03jo5nEe#52{igHHmVRkBDO
zIR5E@5)7p7K4NA2MhPL#;5#lo6U=8l_-9fckC~H6QC4KtDt=;q73rkZlR(*YfwT{-
zDJln@7D^Rg722zYyjOmX<B#&3S_NU8@z-N2PwqK!Jf?(qTjjm3DN8j$Lbh!Pzaj%0
zNe>r;6kMgkS%O76Cd(g1Dx6h@@M~4LYto5Hw)C*5la!mGKid9k)7Jib7+Pt>B!7C?
zheXQFjX-wAw>IHQ%MSMD92s7@dIt7{j6lwrfoUtN9-o*RK9jbI^|{&_Vy-Cepn4`;
zveWoW_@{7g^%GP7ru|fYyiie>pknZT{UHc77HWhG10_i<keK|MKB2)yQ+s{IhVU0O
zAQ%S5G^G-yKhf*WA>n$z-bRJ(qoPJw@~I>#r)ul61I(bYh1~eSvg^tmy2oQhT7a~a
zV{wCI&xFASW4pxEY5IC|+r+BC5G29STSJXt1N}0fU$JNS<Nh|YU*q@YJtV$VzrfC&
zr)5Cwr{yUv`BKSBEEYwve8*}1NOh%bfxPsnF-l9V&1YL+4DljS`oK%b`CgiWICFo|
z_@XT%PYFAEhnRNVGhBfm#A1Zhy*VG_UA9kkCO;{M9+sVvrxY=&*Q*g1T0Z<<q#@(M
zgD-^?;}H<;%z)=U14G!_f;kOnR<z0))?yYW*mD_z-?m~)o1Tfcn;_3>RY@6FZ2J+L
z;5z5`Oy~`XnqbdLAc3vzY|q5$i4AAiNnbH)C$}dwz7UOVy@Hp%2G~+LxRXR_jiw(D
zDAJJc2JM`_Jn`;4RE-Cmi`|Mm1I?PFs%QL8d`>%lo)vR>!7eC3frk?1dcAT&xvz!m
zx#)EhC*+T6v#-ji)rx(;FVH_(A6uhZ9csC2UB!Nh!4-Dr#up-g9Uo{I@HaE4D3A!j
zo{{CRe1hXYX2pDlEcNtum7%$<g`67OKL2Uq`HbXAcRE$)lHENfIwRhp)Xlk6VK87B
zu0BaLs&zsgCqesNTBCZbw57Li+=?OmGVQi?qgKWJD+HHu0{75NIXcmtPLXyjO?w`W
zS*zN%uH5o}AA>XH9BaDf{*vb<E#~1{pNv=>w2G1VMW5K`TN~;dXH;Kpf6mfF%DEKt
zWs0vd#5ghX^p0SD_SicT`SS|?N}@_zQ1ayK8<ZdV1Lij7^6Dq93=O$<2)7%tSYKH&
z-tqMjYV0arqW-%1y!xvLanM6;u}2ku)p^iz?bq;+Q-?Q-T=gy{nK}|ZWUr=H8w`p=
zTg~20Tf?2->Gs;G)qCM`U4<tDGw@Np5(KRZJWi*MbPjGP{EyaVc74TulJOP{Zh0bo
z{M+>Wa^0`t4h(85HYDDmSOE;@ecG;Gle<ov8>xjAvp3#qJY>2rUKzbSw08w$7q*Q4
z75p<Go#KW?>5?aH=QH}aBmU+zbl}PCE8q!jf-EAhK)uZ2&yYB)#UU#e-tfZ4e>BUX
zWq2<@vWIs@)aDlo<YMJmq$#Oj&p%wV4HmgdTnG<QYRc!ma-NyW9E1FEWKPdZPF?_7
zT@w3_@<-0Ii*hRHaO+F*ehZ`)>J9Fn4Eov$Y4z=*{^KF+E$UCN|2L(VcJ&?oGWqxJ
znO?ZPXJupc>Kl+D6MyV1tud0w9Q#onfSpR64q2D6pO~>fQA%x|Tl1i_M)meC>xvPZ
zHH!U@WpXLfYp+ux-wYNkty^xrIKImD#qr&uM(45uKlJKIsEA-3%x1sR6zWLMXv%i$
zm_cxmjj4+caz-M3^@IiFXj;Aa6zZdyeKPH*EA-e^{%3E}=@ZYBCzc(+FS_-{v~c53
ziQuo{v+l7g=P9GQ39C>3y@C6JG}SUq-|<I8F}3+D$~})m6$?CtzpGKwTSf4YQaf#t
z<4a<_X4`UbZ5NkC@697|x5TeW4^t0QXRo4Bq=)Gg1zu0)*8f>j2XsDPT~+Q`Ro-!Q
z2)-pA2vfI;gZkmb%j!a0-K)`DjX(T?i;buDpOF`I{7vGT-+cVAMiC~RlP<Y@IEOMt
zeP+DME7NLm(#z6d(pzYWe)H7of2xz?J=E=t-^{nN`>!i6*V2A{vg_RBR6O>{Zszir
zf1Kp9$Vrr#8888K&e+V$a@ZJyC2I5j*stimNY&^MhG6Q|p9EhU*NcTXzhsfcT?cIM
z{uy!Y`Na|aeS07CmSn~&mAxS4-~=vJ6H%=TFNO-2{l0Q;QOmoO_>X5@YA9aODbzQQ
zd|{<zc2a-&xYf^^NQB#m#Ji7v-15h8Ct+E&@>dCbYkdK(;B0z>F|OS}r>-5aw!UQ{
z@xRrgLJ{fM<;l6gvjbU_B(dF5TmE>m<lkUUC`%YP8Z_^U{a-VT4@CmxU@3BXLkt~w
z)?uc+_@f22c_pYg){MT}Y_O>UyGnAxS#QzasEmao(F#vInIlTTDM_ou=K#gf?~#5d
zjdTnPC53%@CKXs56ecNhYD4%D6MOyJUJvd#mo_xTVwT-l!H4{$!D=VNt61oakBquR
zNs>*bZr5&0-d5UR$0Fe!OGVYf!|4=5dxOUn3s*e@>yG?Q+*UI@YaqO|&9bP$C(v&m
zl72nTaxdnESz?fzr#W*@@9n>dn)$b8%?E@Wy_(1)?!r&ns0h(6^kC1^un*n_S>Yc}
zeP-;PiI`e89Wk|)9{wSbm}Ea=6ct4yCS(Og>N6hKG!)JTU0sN*`4-sq`Afw=OalMz
zX;023q0Co<oSL9h^}!#K+1*{q&oAyrIn3(l3cT;qdSd%L^4RjmY^jVV^55TF6s=lR
zhS_Z|;~>MQKaDJXLoCJJae~tWd&3O=)1O(jaNe;9vX1XZ$3xPZCuC(*f_x#bAHSZw
z=oa*eSM>1Pn(V8Gp_M-V43^}~5b;B=Fg~%hP(RgTY?im-@tv~z3gKgmB;QG|FVkre
z9dX-tc+;*;ZzDcwU6k&a*-iZ`l^!k(Gxl%)4L60(jIFoM9n4pwux<V9^%-{eXm*9Y
z%5h&y@6Bz{NEpIzlNB6TQ~U9w2P4g<Sb^Q$I~ZmDE2wpQD<xqt>f)V<htXSqxW@dm
zqsVgp>0bq<6Xr?tt$sr4nzw@?YJ{`nMsfS+&--n69_f)wrqTiEx^tJ9z(2<1Sdpdf
zNEanMY98qdIT#ptj<()6-@Xd_%m)g{k3Z(S`w9ELq}OS_caj}dtYF^)``Z>bMD;BT
zFL^L2x!Ty8NoA2wK`|wjf&CaG$Qe$3Gk!2pz4v^0|DET3ERvK%al^D;tiZ$%V2vYC
z?x#?_RrHPf@7y0YE9SX1WU@%9A2E$9Z6Yrg*dPDLUnY)mv+(BQ3pxfT-STWx%Bs`&
zBtgOBmM#2`vR9ttU;5Z59iebYd-t+`{~%32w!mn;ZJa$A<0GTXGmv#4J2vF9pLKP7
zktj%+DC9BFuI1Op&ws|3Xtw!~=X2w0>-NTmv~->!Vj$m}v0*frLP6b_%XVgw4A3EL
z<01SPpxnz-%O^aQ{mQgrk^1S>rc?SCP93Q|b|r7Qis{j~M_=plWx{JE{di4pjW~wz
zc-*6?^>4?{1g9&&`1cad%)Siatr1kdiB#}_$lUdjOGCM{XI_S4qwg0tOqBA!K^-*x
zz--@yxqia&+%Ng+7<^MG+c=Bbl&4Af;^XA6A6Ginf8=dhwL>$x9Meqdr?9U|Wrbzi
z)|b?WeCxb&knkbnGb2cs=OXVwZn-J^H8OR<JgK}ComS2e&Gz*H`=pttN3Zk-G#2_%
zxQVPq?m8mw9re}g+ex_x3D55{D&Lh=E5KP#1;cVimv5jSKt6T9)KW2vYBWpSq5pO-
z`|1i9)Ju6+V7cO@gM`}$N&f2Vc?$e`oUOz#*kPvj_?6y&$C-+{0++v<NvroY&*+O0
zhW+f5Og&1h2{u0NNxtsv&T@t&J}a#0)aHD93im3oe^mRJVJW3X_!+|YFaYSia|I6|
z`>>|Qyojpkb*mk1N-d38Bm}U^pCK2`{Fh|n1O|EL&=Ltm&1W{}lK_ow;guYT-(R?1
zeh5#iu_AwZk?SnaBBy>aV($>0S|zQb9f$tvZLiK!lAh5Q@n%(X?eF);Tf*+EaR<Gj
zTP;1TDYLO6LlYor9H7Zo#g+y8_i|q4P?BtusmKMjHdQZf*nMs8;OY=8Qw0XO<HhMG
znqRUg#bpx#tcC%;<n^sNN70r60v#auzPskzeiPeEHl?l`py?Jgwfb|`WKnKHprzTv
zfQ6g$hmymWiI8Q0-Zn<zSMUCCZZeI%b`^*KXDybi`d}%xS6!1|!~GpepK~hBTm_s;
zt;m-e{5k)!;Y49;X(rUoqVt+d4Y|$$ff<4V4_Fslx|<s3!|{=p1{>9M(Jtve<8kxI
z8LG{97Fk_H;RF=UrF<(j?y7VmfZA$=*Hj-T?Y}G7vY=czIdV}7Ap0bT|59P2bfEvn
ze3Kw6EpGHlAU+Jl6ZbtMw&jnq$dRDW+bV5WJ+80`bM~c4FqvED?2<y=IX)>AY3oJD
z;zV@JtU_*&opN1jEvUC(pI>^6?^k;y_q_<s?+gF=dc$oSW!{zN5dJtnrn<uno(BK5
z>)A!@!QU00fokGWw1c&4!4pSxdBQr}SEQr7*m7oe<<?sbI9-AD@7%0$0Tdb4m@F!7
zt$W+m*n6e?^3}~T|I8AF50B)!CMxB9#?xP3#D<rbG$`o`+yx(>ru`I5b`CjAoRZCf
zZ`+G43A3B$!{Q!Je~NuR!iw3$BEb<79ni!rdU2KO@iwKc+ZmOrY}CRhL%vJe<N4f?
z*yn?>hvIfdnp5M~y#4qyhf4dc%Xd)-AWJdto2`Wv0MaBShuw0hFYzk9*E9WGhBaFR
zZ243CYe#Ym@v^i;Cf)9TcL9C?WvlhMrQRyO^EyL}zv|7Anz0;F8~70Mo`Gqhdw)tR
zD6=sdwpNAec!jS>Yh|%z9N5r6uX#38mv?M>9wDaMC6PrXBc<M!Q8_q;8X&3>hQr$Q
zYQ`r!hF;E`8Iwp04`H7rViW-!j+r#xu13&$uu?GpqrdrFT0PO<l7WUGZCMMh@1wMa
zyQ6z|WWmsl5=iZi(W$f$df2VJ&4fb_%(|c2MY!5Sz6y(z`NZ_(U&Gvc{2+#3wwtm*
zQ2F8rfJeqo<5lIH!GDe(i@NkTa}y}1ej%@~-=x)`{&`mT9|au&bH(3-?R&}cCaW*a
z)t?y<JD8@rd>PAnnq+vB<DvhxJzZ=o(PsoSaa#Usv`~UZTVdw}mACrm=CS*MB>#p`
ztSIub#vXY_++;^qbsd~io83V7{dBN5lzsLyUS&sC82xiw?*0cmzTf{%oq4)qkF3<w
z(tV=Hg#PsABDR9UX^v>y^hSRTSAV@A1Yh;vw@X`#Elsl~ega_sEA@{GbyDn-qV%jh
zH$%SL%Bo&e-lbRmZK?&C6HY&_u;VsE>hs*6>(@Ek5+gh@-{;RB12?&!97aEaqi!n`
zA@<#rw0QhA2YoIh1-?Yrn6TO#dsg%6_Vz82PMiqoE34k&#F?`!LSNcZZ0XTAvfsP{
zyR!S-xSvuh#=^#m%$kICSjXS81e#^xh!4k?$=kd|z5*)&5o7~BjQ%E9DU*_P5{yT&
zBCE80Tp>t@U?h`ihuQzI`Hf+CvO}q*4U1%POpS0j)+pbqR0KgI=j|{n0?3gNS|K<L
z=w)Dges+dEg0s8~cvV#+Jed^f_rels8G(TcL?l-$a`2Vbl5sfDvz6f?-{4Rs(w)=!
z;&^FfL8Wvn$np(N$h-CMvJvf#FU`R!_N5I|Fi?^VpZSOy<acO?FJT0F;zeZO(|Eg5
zTrPxFh)YpT5areB;oD$-QgB=?O&aS7TcX1KSM(irj;PefPYvZNoOc5|zZHC@-~MH5
zmuPX<h(0efC%$Il^zuroLTHK~j+aiEdtDLSIa9dden**6f9sgm{1&+{NGkgx*7QX#
zJ6l(9;7-~&WKXKELb1$ZIBda>u|5FW`QhBSSDV5SG3EhiW4L|4f8x8&22=$bj>VBG
zn1Y@jK~qbEo0Ce!s2HmO9t!&sJ={heVlNySMtrW~x4=oEA_VFYI4`<W*mW>+`0xhx
z3idN#a03{af$s72@Cb;WI-vYtW87S48x}bYLiMHX#pvJnOEOnih*A<-8=Q)H1%U>i
z|D$dRH9hcL4?OST`X`*fuwno$qjk~424obtVarsw>qOxgM8Ntu-PDNmn?G;pW&}OH
z#|NKHMCborL0ZOXoHg%0I8`C+^z;IpIPvq>kEXT6DAg9!tsG4PvE3m^wZ!Gmadw0<
z4BKVj0^yW)SeMV-FuR(!+BSV#@5}0gTP(Av$hG^$=*IFAS9EiXjV=$d^0&ua2#1kZ
zWKFJeXQKvS@eNSS_bUIJdN6Jq&_ni%mAYF0Oy02l<$W`?W)foB0xg4=Uu2wf=VIE+
zs$u7tRL+=50OIKaO@iX~`m^qT|C@7gUJ_En$lj;xe<`C7TzN6ta^_#pz`*BMBeEv9
z^tiKy>))aVqSXkBZ8xslLrh&dgh#M&AgNmWumqP*sfRen41t5oO#$S8&H!Y9MH;{g
zZ~GoJzqx){<#tU)-K}}WVjiqYL=n!N&qiFV;}fn4{N|l#e&yh&2KWqY>frNSEQ0*T
z@tKOHaPN!r8_s^U8*7i5TbU!w!|b?ZJi^qNCDwZst`EeI{B&<itay6tN^jNm<%mJ9
z89M<o7H5f=EuhKlS<PMvG~!Lk8b1eQ8{9c4<g6b_nt2hM`du_iuv~MrN>-TnF>^==
z9$uG1${z!SI(L+mW?hVS9bNweL29~La9<~mQIodJ2HK(yI$@9~ZY>eu|2pD<v1v!L
z!R3WYzB2weLi;fh65lJ-Zm9r5P#i2}BZLV_XPuutrldWg<h6PyLi?)AGktv#>&U;m
znSe;Pgt99^(sr1sg;o7tcIO#QLUvbPL}zVqJBp^#x7lgjt(jB<9^h(X-#;;uQ;1l+
znI-;cn9HneF@v6XJyF*ifmg<<iELF-gXsOZE>Cm>OC>453xK|-env!k>*;qN<n8N2
z(RN1fGr11dr4C=2SnZD>I`q$se?UPbP!Mfr>;s0Ogag^&@$vvNl3tN^gd`l$*s$25
z3@+W}4mX`RjWhjR|CIM)`D>sbexlGxVHmN%>hfA(YM%x!!_=~KJ~MPFZHXT6F><Ro
zQgVo-y)G+EMZx?bBww8fqUB4{c7O)$d4JB9i+>@CnV%V@z>V?rVF?;hp8>PCW>G`I
z+e7_8>1QWM>v_f()@p*_Rnbu2DzKL!!NuL5=XMpju5ciy=l$6qq>x~wnPgpAj3!l=
z1EUl78Ll9Fx;o6Jd|fkXw=CbHjBQOuM#V#%FF)-iG<7sbbk1Z&BHrC@K>;@p!JPnD
zqR5ku#u;!1Iu$MuB{D>ssR`+;ubp?8R^QeUt3_z?6iS5dUyq?hfiI2-abK)_LDDHd
zQFiKpnBMRVzln9(tjjxn$R4x(ZGw^KE%fLmvo>?mHtR|TZa=uhva#3TgB|x74TP+)
zA~{flIdLB%%JWUZLoW^8jNp%-MO>02v#3$ongkttVP{y{0W>XCba$odhcwL=)O%5+
zEsG3D$^^5=LiFAa9sypPxM7G+(r5gy12Fb@iD!g9V>8l}`x%cFHBe#I&9~*CY2PU9
zK|3I~f~bZxo%@Vm0#n$2zzIV$rLcYAQi@JT6<q%6jCBlIc(#V-0~@K!?JJPr!@@AQ
zYsMBnGTZl5{;L7jf$VZW-o8F$3CcSxi<(UbVROI*ULGVRC4)uVJSiNMcQAZOhA+Ko
zqCl@J##DiN(h$0yg7vcvE{yq;k{n@1e?Prnf@W?t+s9Vb2jS#FFTEZJGtlhj=bou*
zi$l76i}mqPdHq|t6xlKSagGNm=dL2l0%&)dZ7n&Wu^NeO4aJ@TKiM1|0FyZZRh%M>
zvl={r!nQo>mZ#q~55-~Zxp>{}VBD^!N3QgyKqi-BxZJ{G$g~Jf8(qe33m+|*wO?1x
zyqFwTWTef{>(kG7N5o`#sN*k4mu2_{jC*Tja{3{YEW67`#NYNOc-(81-_O8WG2S~B
zHw2f#RiqbVEhje<XkBqmQ%Z3~C;lNzhA5q~OB(dJY4D(gH|PPE%2Y#?(urv=)IWHu
z@b7k9Gf@6-k>0XSC>#smY?{B`t|t*A6~}mdXT*}Tsebtc5ems7kha=0Vm^1FE#%EM
z;k9J_x2U2hND){2Z@@N3AIl0;EZDq78O&M))!^S5*oJ*x$CcuKc3yq+(s}}oAfKEV
z)GKyLAr#K|m|soL(}Plo%DV5jO^&<uo+t7kHBm1w@o!AgLM)6%STpr`CDDSGVCXOw
zl^1TdU$h#klBp4@Z9W=;J1>5h95Ex;b&g(JiWn>pVVeQ0(cE1+KIU5U&H1{rIO0tn
zVp!6{6yMDr9eb8ruu?x45#4KptfXQ3w2xWeUSL;2r=C@5^Etqs_!jj}98tVP^oIlQ
z5>=`3F_HgX12zLuLkvj1vMu&<`bk`O;+WG9gt>u5g7n0I5D;S~iiFHM7ReQilK~3K
z+`MHqXu#Wlh_ME;_bgBd5AZo;bW;c$51>R-Zoy^nX528sj07O~nrHa?AqHt~;8w#Y
zC|jIjnm3ORx4t+J*Niu<VRLMU^m^`W5aAEHclqxxoRAld_C2((KB~X}=Dgb5oJ^?w
zL%y%<I_3f1uA<dUA^{d5#dS;3=T01(`=D3z{++ZKVg*spA8P(IkY?U`?=7^&XI<VU
zSnq}vCH|j9fphpE>81uyToF-3Ps_}!AQnzQ(*!iz17^=q)6a-9oxerxe?dDOg0;XX
z3wbZhO{CqQq{bsi(Saex-%tskBDOT_1Fyei%W%cP0TI~r#d0m=^Gw&KHwwJ<N+HhY
zp?tQ)e;IVD>4|+Mh0F*(&b===;tcgYd*+-a!xc`ea-i2o)|DC5fggbkud-J8Jrfpy
zO^!<%H8sQ5SZ|_TS#$VGZ}{x5IOONp<}Gy-d$SukqN6W%!<Bbw5|BNoi_BqJ^Uci2
zY5a9`PRIA$5p_u<T85XHOuZ({PcL)`h>?ew%>2fR_8wx~g7CVLbC2)Di4EI-XzD(s
zAXp^3pp}8)E*5I7454KaE<=;>^MF&UWr9*5cn58%I~!feXeiX>r>h3U7=d*-<qXUr
zQW6x9>Gu{=nim>pa(d5fCU|ek8fWVAMhzUu2ad)){F+-5&6{f(X@or&8!v^NJw%dj
z3B~S!GO%yf?Mgq{wo=e~$^}C#_-ty>O!@{D@2Y)})46jhM<ITTQV8!X>W4It#vi1C
zx(PsFF=l}-FVUSnXrM;e`2HRrBA#};|CDG!aB5Z~QC&PUHrIc-*HeHAl0MSIbSx9D
ziXiRe@y&M@OrtbbGe3$nfj>J4g)!|x-+mLf$`jr9_|MW%OiPIM(Q%K6E5DA3P9<uF
z$%UjOA4=O*{Cht29d{zyvsj3fGm=18d)Z#r=JEy_4rDpsxQ8>(2?4WnY}muqX15~G
zD;b!qtnr2BT?q<b4NM)#mtN$W)n|rb^{y*Rs%R|ED*LP^VVSQ}>q}6Z<jYW{Y=HR=
z+9&<=xEZ84^+I=?WY1$R*RuG&I9DH~tS}wNy`0*PnD{zV?5&(deYHT{o5*#`TV2x1
z?k(N_sF`iZN$jGJGg+ih&$@d7;NV9Wx7;+x1o|fa9kuU-cOMrUT~Ta#wUqA;+xHES
z5^QJ+740$Z#7zD^PNLYNp)LEM^eul-A{@lChj-WX^n61uF&8C#?F|xo7N@Kq=f9V{
ziv85`r`6X!>G|Dz^7HG4l=B<i-8qRKzy|>Oe*w$`JdNzJm*f9F!D!xc)DQX85UNB`
z=JERmj!*cZH18K%N`N>~-Fn8x)(G|fAH_Wt?klpa0x6_Ht9qPkScA=rO1qV!Nm%RS
z)C!S5J9kkB?XLhOA?;8&`h}XYfL7!*K4ARTOQpfKWb8oRbpY^@_Dye0cwu{Z;3u-v
z{nDvWFV({_{ym8p(<~}nXts>g%jdY~!|o_cA0Jq}kD|IH1A7@_hA3T@vSLsL57ki{
zARTRg#cTf(ez8t@fc$eSa$(4jhinDyZR1YpMm3uROE!}%f&sngupM&aMH-2An;br~
z3<@Ry1J^zT@OCUh&^fche1HTs58V1~|K62rwl07?3U!v~rrfTU+yV(GT{YRLC_u1g
z-8sC;+ny0hdFVe7;MGtVY5EwM1q2_UmRFp$6Mpk(eVL0L0|G1{ezXTA`>4Fnm3)~~
zQ1|Mn5l&LP)(4jID`UF#YB)?`{H=T_GG)w(sWRK^ON0$;3g6tYCftUc^M(P&TPG__
zl9g*_5<qSTSc0hoU`5*Ixcv%z^CCy6XKcVfz1wmZlE%&Q#$icI&ITo2geocfv?jrK
z_)1TI{%?0i@y8$CzX0#0B3eLB1NY2(lg6Gu20KW4Z6s6x9l*IAIKRKKInD@n*^(o<
zc}2qbrp}rNjD*lEB>R*m;pBaA>F!=t{m0ESRTda;XElN^?%a5Er=CfxW6})3oA+U|
zA{T9LWgdHKCpHeHw&-SfLFbmjc&7y1q_VW3s0l#BV8NX?PQx`{8+~l{Ya|{4MV(V=
zOMKZ~<}cFiJ+$z*&oMB^HF_8`*UVi1B6AcBW({Tedbn#G@8o4Q-`D^v3bBT&$!0Gn
zfx79?$4Y)MK14}$hphXuf=MD7%Ga`ZC*d%?W#~A>A@oAcAZ5t39Fm=Ap)F?<Y_ZZs
zShy&82>cPCnuqEeS1Xffy~*RuM?-T5CqAOup(yHxWT&#Cq+xt9JkjCj08<o>f27h}
zzjT_cD4eUneG^l#Phkoq)gGF2`XNdH<f?(Fkzipm?Ahh2Y&4E%aR4xex3fpzB_wzH
z+xp9M!_;u1{<M;rXB&`$;GDoq*?RztKa%_9Wr%jV8C(BMv?Elf0T&D~&mp-_+xy37
z#)wAO7UMidRLMqN*c_(>J{?)GUnj9E)~WwQNUd3SEN00w3LB-Q-Il05nejo-#CYfU
zlZmXzwd(9s0HU;l(!lcLU*As{i?SKIFzUA2;+GouQrVW<*Y8~$`atWw`30w9*R>%^
z3?k-3Apj5^{Riep4m*vA5b1mO<dsj(*R07SN;@;qc(@mD!dI{Gj8F!BeGLlp!pudw
zDlK*;7620A<5UPo4TUyyNv=|ht%`7~IO32~;bmS)=AAOYS3{IC0Z$as9&{>f?4}D3
z|I}vt0dnziCdF_0H}$y0&U*YNQS}kz)_Ze;;+%*z5OW|0(-$hd%}Jri2AFXPR2jb>
z)$N^F_u9l3TNDW3DIfB2OV3%xCX0N)62_ra#Z;7_+Z#i%2IkOPDu8Su0Uu@{d&W2f
z%m!UhvB(j<17YLvl`HXyq!10%Yi|;!=?dT&uOXwmaU?guR#pYT1lk-hSM|8*e{&xP
zwebK6qmu+Qxtj@q4^nAAA$nbf)M%p*e1Oi1R(m=61Si@I2>KdK^Q;u2bM8W?tVDJe
z5pyf)aShtC#?d$#Np0!wLcy5_W(kC+9^e3V1?HBB!ztL!#ewxKjQb3d(zEb^-T>A8
zjRbtRIl+_G04sk8M86IZrVpUgckd)V6arqU2lZ`9{QOL>?2->~nuEY=E~F^HZy*|?
zLLl&~0Dej!syCeJC_-K~Z~{Iki7I(-Ck()v<!I<s{h*|tTAX+bH1thW)Zg&B&`0jF
z&;Zh32kkLh6%bFMn)wH&NW27<{l(3MTTq74{9u4r6i(t^kWcX(d;lfg0w9%k6+YPC
z*gWrM3ndi1SAfz;9>z{o_#sE~b?Hki2uvTK%Ks$FE7!E6&VePo*Jg!z>v6+WL=u)j
zxp~OP2@`q<KIp**XZWxVK70{<$mC8q_#@dOsL2B^p6?Odv92#kh>)j|AmoI8{61oV
z)#LoNk*V?YE`jC&vUI|hY0M4tUIrW%OL1Ur<43|2$wB4596pGG=yuRbXq)+k?}cp1
zU|qnQZUA1<`9GDfLSexyMPbmvc>d>PO{Lg32;3Iyp<deqvycWRZouTIhzZo51>iSD
zdw|%48#b^(^g)CG0PIp%!9dFZJkqpNl>eK^{D+rWuxC5I-8B>+10B=B1_uHlejRX1
zqJT{RD!bQk^DBmS+nu8dr~oqe>f-)HFf_#T`Wd5RM}e+d1fiK_>`z{Bcm(3S_s!q#
zNv?2F0Psp0$G=>K6NoT(((G6f_K1_PgMa!bME0m$yL-EY^3%X-SE*};pyWs$`KpT+
z^r?~MUv?;T^D17sbHJq(zB?=nF9P@w`K8{5{`wJu>sNEmiQ)APa_n5}8Nj@P6*J&w
z4Mlh-sJX<73#V5^4DBO*f;Ou^2qBK`S}I%zV-w*EfW|Z*`UG=lY5mv-qwN>DfCij$
zgdA`ez;3}-<nC6Aut8Qe5@OiHoAMS$&M!!alx^|2)49nrjgS73ghBx6kmbkv3swot
zemi8feUzxp7Dp_0TanYY%Z3fFg5PWaAR&6+(G~L&*1kLmVgu5{=~svgwR^4A)@grk
za(OF7+fu=K86xAM%rl3Q_okroj2)b&`|Iy(rA&P=I`p`1@6)MDNklx{m>o1H(>@57
z)o2Y96@8oD2w+EcE?ge}*=6JP1^>}VZ3L0LmNxF-vtQeRY}*<0vbuGNVafvUEyyA6
zEkf$pDmyju0i83wo2yT7*E$49(9we=Y1Tz{(TJ&EuU~ieC8(KhrGGY6$zbb@m_BkU
zHPmbsQK`T-!%sZ~lU3?|`L!)~#Q!fu=Q7c@*sxAcz$>U=TZxdeBck)?FfA%?l|b0&
zpR~#$#<WP`6hd1ur^@csYRl<?Evh&ASRTb+3Tnxt9MbjO{2z%T<U^N5>V~_xRpY6X
zLO8rQ3%*;l&5O~#w&&#x9D{wS2^%m`%NH%69~{5#czJ<T6U^$<Hb$jkhO^PEf2pxk
z<>&{av+3LkFM=cDZ8OBEg^R8CCqEcfuq(>urBH~W3=i*EoO#2fqHeU)7uzAk-*CYo
z*(Zs-!*J7BB$j_ERR`>9U=+fJA6;2L!HFijUsRgX0ri7xv7gy~@cY2uA|%Xm1uxa6
z*tOWCVx;1@>AJgorB_Q3xJ+(H&TxJ=+xVzA`epTvOST~J`~7cGq95?(q}ttF23Kx?
zCsoALj75SU@$CAT=wI5fh5rH%Bagzb;*MF7J7Apvm8FQv5AON`RRCyS7+P{f)YQNt
zbxuF~w<sbGW-S7w?s3k|zP6YC0f%-8Hs+EDQwuhD5~-pX_mXSK6$XF#2+rSkf^CMJ
z2HbF8R@n2!qfJ0t*qr+dZ9BHw%~uJxKp8{x*5w^&z35DdET~vW*z#z7!G!O5@Ce^>
zhO1PDbChT?_?ISq&g(-J1r9NW=wZ_fs|58Vtqrh|6zv^i_$G%vQGYRe28etCEQ+C1
z&uO;}qQI9QKiC7M{{D0n94=9BtF{d;UBeAh?$+W8AR5s&LYsP<4@NX@KUAs_EyHc~
zihxu?mL|IO>mx17B_=*RDtr+W4mwCT4{@G^yZr-fL++MGcUOoYw_Kt&-084=GXCL_
z7ol;j2sh9lVg$zrzaAvcgWWvg{InKep5=P=R_5~XGfq(&M3PMoqx}YWiC=g*Vdx3M
z{KO&+0yGCVb@FB3Us#GbzV7Z)g-tKE75oi~;0CzDz?27DrLse@X|S=I!ZLTsvgT~^
z)rVa>3W~e|RNX`6BBDOyeyKa#$5xFnAp76e5Q_{z9s29j_Yq`=;Qd4b(FQ432nt9-
zySJeV%Kz-g-u_us;zi&R4U>w3{Y``;Mx6yKC;<YP2$T^by$InYg90z3!tR>9ZsPrQ
z%b4$(-?YWw$=<?0GdO94dD|DZ<h$pOp2|C#Ij9_z`%=p5qI{G#dX3mBP19xT-PL~I
zj1Sx$r_2=Je>^JKx!$d7LYlpdc80^SY~931k^;IXghYKr-j0ezzbi)LZ~a?8=XcEs
z<88l*XJFt&W`;cABf5!S@HW@%F~pNb1X>P<(a;248#yIt1LIt(-L_0@aB*9SIoTLF
zWI@HgI^2eGuDn36H!(r9wo|e8N8tT!dxxu4kEq24C`2vq7mVhW+0wM7Zy60%qAtZS
zs2B{oAvcURewc%50+DbxU{K2=oS=MF#QrebX9&jSfye^-bcef`!HPGemeX2<0Y@j#
zxK#RTo+MKe7#ISBz@AHwIS=`W)K)6i5SV<Wf#iw6;R|r+eOH2hd&MZdUZKTj0p1CJ
z-WkikmP>6TTRiScU$|)~fegsOOVakdy6_SNVQjO*e5JU$MRtWDh_h-jdcrafi!k4v
z4x=f7un6%?I`*ydao8_xW!q3Oph0AS0{!bBz<~_j5jtlW5fGZ6Bp;dIXX1w-SP_rX
zfA1vGrw14)B<g%R)<CLW)NS`RG84(Ij=3j#*$gkiW%-*CL?8GrX<O7IIS4`GZR{|L
zdwzFM>|o;&nzHC)dwYlNl_nH2Y6zW}xz*LlbRjtsLF%-N(W!pldr)w61*4q&snK0q
zi9x&z;icfA!zTp!AfhPtQ1ky4YPmE7c9=xb-vxmAxdA6Sa7v*|*TK*CLFVG%<G*0X
zgL!-m7#@Dr4bwZFj!n0WHtGFFK)eS*A*L;TDCen9-)JP6Hq=F(%E-sHXgtqMZK3Pw
z?@(E7=BK56B>7TzpK=_I=C&niT=2#7hBfBUM}@)_D)|w=>Qd;&W`8k-M;d1K7+}6^
z&%~xXy1FT=550~=O`rXOr--W(RIZkkb!p9e)%`^a#_Fbk0K1~IzV}U`<hK(s@~?v0
zP@w@qC+?{Y-Tfi1d%B;2PARd+*d6;Z#qZmJ`Ql8)>h6ab%v_%0=u3oqmz<{3w!loq
zFQt6q)~z&9e@<db!*|@f_6|?0InI=rmNO9(l?eJ$JxMn3fzK$IXR>U2VzMk7bwQ;V
z?U-JKbp%<0#Z(F2*S5O|d++|R*a(EHfN&{j>Z)n$ubV$4Qyq!mjKC=5bM&LNYD#eq
z-r~<m7D@3T*$C%-dq*Sv$=>4;ZnE##0|}Gvi3JY_W7lP)!yGcP2I>bF;;-(;#1xaj
znY0MrhaL=D?AXLZ&~J;;E$#d7#0yrV{olg92|FP6yRqBro>5Iw+wJ<X+C=mbHp&L1
z)U|MR`_a88?qTO~^`ZVc(x})V-rMTS2(rAbefR1%p}eeeY~+B+q3_62Sd#K3Pw&K6
zb$#O=-}l9rL@|9svQUGyb>`^s@Js6P^251Q+G$um%O6d7bl*t%?7abmNrtr@RYxg1
zM>l6IY03J6hc(SdV*lAY`Tb~R&APj@LC#AK!J;%!-9|?G4-e}Ln;t7Pd)l(j1B3fu
zhou`MF#}29a_bbDdrefc<tig+OF6~pQP+cF3AM(Fkrnt)H%ORllq!Mdb~)v<-kv<q
z-(|C1;z~Qg6o*(n1lp<1;Sbk8W+nt5H6TOpRc-JK-%GVApUYmq2-O-j(0xN<rh=G6
zdsjE<8@ubnPZ~%<lNijH=w3w$^wi6$8GA&Y*ha;ctALdL0mDb#v_`+*2;Ha9yj2@y
zlmHov44piCqxKQlhq%~s@7yqiJlWeleEe{N;PjwME)_W)hRpucx98r|yIB2MIa%rF
zB&4;a7xlHHuX{Mn`R}@;4KvE$TP<6<P#<pEV~Qd_AD_I_aVT<?Y-Haa|7T{G3ln=b
z{ivJUB#zWwwa0iHwBn}QRGwe_X3X1%BG3vURvX+3t>}5=p&Q9N9)1X1n)wfmY4Z<A
z+CA*1Sz1H6Kmv^rldDXS>tM30Pw&PH91PH4T!wWf{OI_#czo^Dqw1xmuGgV{z`;z9
zYzlw44mbVFbMo#|Q=AF|{dNsC5@|a!JY3>9@FR|Jabp0cNCD=$b1iOKP`h+)pVv8Y
zF#5!H_|5V|-s29ZfmJ!da4QWantgg>BCNl63Vs$aq7<xR=7CF(JMQht>pd8~Z~pu}
zTaoXkLRj?<eJ#G=`>^xj_|(Itri2wx-(3coGdtQlxZ(vDvLB+;Pihfp_T70*uggo?
z#S8onvr%@UDd=trpLE$fX&(CzwuppL(R6~p6#zQ#&wAX^UEd$;SqBqS*$j@-xdu1w
zfQ$Mi$wM4Z0`JO$V(?RR#PvF8mitR<RQiVf=0vY0Z{Qc#8pY^xNhi<mAXD{e<DfgO
znJ}T^oEmhwj+5s@h1Z@|th@T=>tIeTBB@>g=kO|tKJoZ2m`=;NbS%?1hq)scI_Zi!
zMQM#@`f}l_95~!S_tLrX^`k-g4r{?d<hQ~>;T9*)+MxbDUdJN@zxYPD>32(ci0(O%
za1K_gF?jrsSQ{Vct|scP4Kl_2kdMMmFTi@HY*Hm4`+daBB?s}e7t6s=M8?useQ+r6
zG%W&h!CS0<6-;Ojf(@c=kZ2BuLG%qGF*L((P6NvMA5(yG00CnXU2~XTca3VA>4*JC
zgBW}_5RP~fdJ$l5DIJI2v-{sKH+LJvBhC;)EE;pGyRu;2$wxf`pJs?=p#*l2lijhL
zybp-TK!ot7DSD@IY*Z0CVP<Zok}8fMG;z~sqCZDt4}ldy(qYs1znA~h!T-+>R#~xx
Z$kyWTKb*Bb9RVjn?5&-x%C>o*{C}%`fm{Fp

literal 0
HcmV?d00001

diff --git a/distributed/http/static/images/pandas.png b/distributed/http/static/images/pandas.png
new file mode 100644
index 0000000000000000000000000000000000000000..5fb1f91fc91e2ca036ed18bcb5f04be5bb378046
GIT binary patch
literal 1213
zcmeAS@N?(olHy`uVBq!ia0vp^CqS5k4M?tyST~P>fu+UM#WAE}&YQdYyYH0B9sl^f
zy!?I9>_<CV9`DM2Ct|oLV{t1N*DwAbi-mGT)&_AXuHC(rh1-#Z`Pw9x-3m@!CY>I)
za*`FgBX*`Ka{WGhb7p_S`);0^H1*BDEB-L*+o^n3nSZjpEmG@&J|_!Pql1C~2PU<_
z=soj?0_)iPzr``RJC9%cEA=<3H1kgK>kS+&85SG=|Mj|>Y`e`*|LW)4yGs`zzm`~d
zv<Ztzaw2{YOKdmoiC!K1?@U<s_k`W=bgl$Hee9tSEa7MK_v6V`n`5`NMXld!oo%~q
zhHSVF7N=l!AMt`43hWmYXMH^@K27ay)$*OUzkk!WznYny78ZVPPhHW|Gtc*aD|3_9
zf7fT8_x+ajDKWhXQ?Ipo?yvtj&HC+`u+{s+R|nty)^_xF^?Y$8=iOW5o4$Tk-Rn(P
zvi6_mGgh6TdFS^2ntiLzo&PN(BZs6AJ%IaX`8+&a{`%soU-`>-?tU%)I&Zmdnu|hk
zMD_l2VXHUC9*bJLK7P$sz34MN+G3dTg2&$n!t<7I_~u$05c=HeY_IiO181h0hkh-u
z-?w7#hJL|Dorb#~&rRR`_@USSqyKwWnf*E|o5&Q&F?;f=v-h9Zc0E2Zy96Wf&pR&r
z@V<V}$4g7Lrtj*}-8%dB?)cj4sogULPl!aVzTPIa>(8Hq7uvqA5<21Rr@;N-&L!>t
z*4zKQI=%Osp3lC&zZQSo`ZliGeWrt$!|HD_*>989?>(8aBQ%t$lHuMGXFPsEOG*!<
zTe}xr<#)cCzW15h+P{~wU#}28!L;MTy1Tmy3f^AZbG}e!XObf>^J=c7Fs{+F|M%wA
z{dpDZuiv{@k(`^p<zw*ry^DPmxD$W;zwl~qT=v?l>Y=I=*dKJUB4vpm@oO}7LVu?j
z@qGWi#&_NKn=`j2hFs<QmE)^cTgKP#ux!KD`lnf;v2(v$@?GBh*W3TxZ=TOjP>jF`
zR3zhxm3#2rQS5-@?Oi*IH+(Exzc=vr&*O2??sFZ)4rFZFYPo9d*~<BkEsC(_K&Hlo
zhx3-?zvVUe3;nzL_%*MqJC5@m*oeoCKa?35{{Mfwc%8zNE%(m>izNn6S3j3^P6<r_
DS8Lz!

literal 0
HcmV?d00001

diff --git a/distributed/http/static/images/python.png b/distributed/http/static/images/python.png
new file mode 100644
index 0000000000000000000000000000000000000000..89cffdd9365400b00b1b1ea7dd4a46cb787d713b
GIT binary patch
literal 830916
zcmcFs2UwF=+qS>`q^$~V9VoN?Y(=Gti#<|nm8uLELlB~%A_NG?2oOkY`6?9zS`ksE
z6$Kd~A_QcFND(MdKtKWnlCY7H1X%&H{__UVB;cqbS8cV|)t+;n^UV9XpYvWmY+|_b
zGr7-}ELpPhz)wFOU9x1=Z{U~LC(FRU8M_v`0{r76kE4b^EXk-*=m&rJ*kO<Ho+V2%
z!<TbU{|Ef}Q@5W^cq~~WT>}03s2q9k4EUGpJoj089z&k<M4i5Hc8S|*M_12nu4g^J
z+P!W2Hf`r!7?UMS1V116anErd+ad1CYj>vhGzFB{U%Rzu=`!XgU;OfM_0OMMpppku
zTP`LpTlslm;KonZeQo!@&o48LNJbXpOIAK#w!<mxN86>}RZJYW@NzM>@ap?d{4LdG
zzkTxMH%<>ve5?8K3cVvGG0Ry08F_jq^(K`GuP3JDIpGHfYWreYd4A{MoU`Zh`2r7&
zGIY*Y{<LjGwPgt&+j$65!gD5iX$`^E9d}2c4h7KSiUi7qhg#VN7=8yN28kcnpyh*T
zQv-wGoCod#n{vYvEBVAe*_cFG{lflQYSshoHWPx5rfnTXfhs@P*6iX!^Z82CZ!{a_
z|2r-Bu*D#a=QU*0Ky$x@@uw_RzJ)$k+RMXRwcNl|mol*jGV?TfRsm-ku*_e}NkvHS
zCQ7J-p8pc$^f>R@l-eqZy?{SP57kdB4M$IYWsUY=Im#%f=HK`2(w0%Svu-IAe5>r~
zkJ}}8=}&@GvZAsD`n;8riLOj(F+pcdc~kb&{j7KcWdkR6d$zs3cLO{;d|jqIaXK)x
ziI5>pde%YWJCKzB*+0O1=peW}O4pdbcuj(?g-n9yZi@Hm072O9=9~xGe$TTRiB>Tj
zwvw4M&8MsG`_?f=u5kj($C9OMy&>|5!pXE_j_F2y=|M^Q{;|xh4K&z63uhWuD}Ybi
zM0Nh3>vkk7ExhcRIXS3S-{42o=>S&6DE<FFnzy*YZo>_(j2fOPOitIy<x0HZ{uj9|
zKGuC@s!AF(@8;<6Tx0a4TkGBinnmdTEs0tW&67JVZ>*NvMnNYD;-{8ShpQW0rJoQT
zDgOShla<yb9)VG93ULS<#lqod?2gD#4l7?<2bDFW3D)yZe^?!rp=8kCd#lxkDqnkJ
z<)@+X0lRz6Pb%6JX!mOH2Jt#_9x)S1WaqybTN}bIwPEZjoaok{&aRef8aOWZQN&UA
zVccichJBZ0OwgyuJhm~1Se!?0Bv#u6u=cT#F2h51E<XJk1pZiU2sPQ(l8zqXhJQIv
z4(+|$YpYM^E+6Ee+7!GCpxZYran&@Wo&$d`;_(t)E!b>p6+^00^n^hiS#<pyI0Khh
z0*Af>G1-M1?^s$|HtaUdHx+R61hkmF9-0N}yEO~P_!_7LzpN=`#Jr`4*R|020zJg&
zv?07rF5qYyTCmGPr=<2|p3ZckBmC$|q984}THeAsCJ`Asfx}hHvvJO0(n(0c;WUpf
z94;*}A!dS}3`4fLYz{rX-`x#oa`%hvo7K}%?w@4b$&kk42<`&4)$(kT%QUyc5ud>4
zx*`H56;%gIknv;}99@7)R&J6t5u|gu%PkBO2p7i75{k61O{Z5k1&l4Xz0c<KVk`eo
za^G&YHk2avbIl;fZRsrf5oc*NLI!Q}ww=e=?a1mnZ?LsBxN06cxt2=q4V6kFo~7s}
z5j`~Yk_I%7u7`{#a{r?{PdSFQ1J-ixX&9EXOPe5!JU#rgo&984e_sc!zm|C6to?m`
z#<!)@w2)vz$6ZL9wm@=pM6*5}h1Jhf=X+NtdWhk99sz=5uF_@M8m|7{?p}iB7CMP|
zXD?UhwQ)S8ZHodipP-#jBKUT}jp+P}{?Xnit$3Q;m&ZosQm%BiTg4sRaO$X&C2M5T
zYHxv{DI~O`HdkW;Gthg#t#X<cYS+bQ#TW5{BgaqYTGBbGb^&8e$uRyQSq(#3N(|c6
zy1%eD9MjC9u{%bQ8ziN+Wb)KD7dmGW@|ddf5e323by>+s(mg^ZwzjhOd_l7UDZg<&
zD@TK^e1p$zO{$IAi<F5|kZPE6y|Xho5aG2Y^x}b()5MSsPM_&Nr92cUHuPB|rK^l#
zZ1!4k#ZEwC#0Lic?g)F<R4HZnD0-Zq3>RvB>uSl(T=XljM++{?YQWhl+nmF;rF)iR
zeJN4t&P?ohHXh+sCUCyHi{|LXrau`N5;TvM%a|kx5(#;NqAV}})kgU(B4$wH7Zvm*
z+YPPBfj7`vDC@Y#Gk0){9RnlN&~$RXj?dH>E6-uNiGL0eFq~IP;o+cY5l`^tCHB#!
z=UwErKFA-dYOoXXAE@W0H7n?&rzb+J;^Yyyn#mX|Di0k}GZ~6wPHx53@O6`TW!Y@~
z)XvXOZdn<4V9q^$Y;<ttppAk?Lnjfdz}(BwOd{G-V!(Bt9vq+%RpkR2k)cGqs|^Et
zBr^xy-+QA?kv#-+=i}B&%E^spL!qJns%y7KkCu6W)n#*ZIBXy-!L7!;=ZX!=bW(kb
zzyesN;Pmt;sg2SmU>L5*r`Ei3N2PVSSLKQvD79<lOO$V7Ov+2kXsBE=yC~&hKI-gr
zlc4w4abjP#Ih$EHIzUs&QDX&kbFH-{H+E|q^Q69iJ>)qzX&BaS(_q7&4zx8pdriPw
z2CkrzYo(I(n+KRA4!f*HY4DZ1x%)-#!SNbLq>Bv|-H?q=4mgGgxKFl{FT8KrN_<Ld
zLsP21ZlB7%B)R<e>zTA0{>RZCBiR$=$<E5&$>rucU685yyQCEea1DL;8(RcT4Y}BJ
z)Bd#^%9>Jb6@EE%_w;s0aa}GsXH}^6w>gQ-#-OIT_y(Gl^AOSfto>v_1HYqkx_zW_
zng=g@kjIwYc9Ib=eXy@gISyO!spNLrJRLbdm#>N8n7Y^ypQoEUjBzJ=`4+NqyeQc@
zjj7>VTgN>wFj<kN{-@T*Q3p#-)r%>yY*2N!&*}3F|1qYWJ<A1`hDJtvv&IlZJ*x6(
zxIS;H&lJl|xAS19Xp$V_upB5a6#E(XWS4a(S`D(XP6&hnmrQ()YNb|au8w<-GLBgt
z$5W1(5EyZAjx4l~yAjT@^y`Pm<(5UvhSvH|Zd{Wi7ic*rm4=i!<~IuBCxA<&h38~2
zsc;|@r)l@W)q`j?t2~{_Dn=SQ!M)Fkbzl_vtE2&)$^-dSa=NoI(+TCOm)opB^r3jy
zDU$M^4Jqg7a$p7lMWY>o$nYesBy#pE^`dmo?tt?|G{qViO-D9TjV#sV8us@ZL_rkf
z=nAp}SY3Veqf5okzhudymbOfy;PG-QPxo9GQq%UR=LFo=lg)}TP`)t{<|so|xS*Q&
z`gQ-<71_FZ!vz^VS3Yi@(i@}P8D-@*Lri+zz!g~TvoQEc;IM4a9s=Ev8h%GM*21%l
z<u`e%b!UjZld-CF@!k6EH+GA}ocpxW)4R6XDyTKs(*@BJ?PCbR&yk_UH9*s1LdH9)
zf<yUm7vvBw8(7_6+*Y*2`Ie++5O(-gX5VfkKHrC{39Y`JB}jD)0*=T!gvl>ZHu&58
zuE~lN^~_i5m-`K)@qT?s%~!~Tgc2<XWGO_0GK@s1Qv<P7-<Llkx#YzMkXpxiJ5oYG
z#F)9sEA-=@6Uwm@Pf84%297EmLs+?gvMusb@tr>^H%VtSrtJQ|jdw4NZ<CI3eBRl9
zBf1fttlylU&rSmIbpRaKJF|#=+s*|wZDq$!8I6*aT+)AgLQ4y$IRsoIB~HF}s|K?0
z3<b6+pTh=WRC7(7JTk!m%j+m{GTs!HqK<p@VU3LX@N6SaKa_d+al-5Z++5Du51&;<
zps{}vFuY@GP&}Nfl8jCld-Jn-kGd!^5y=Eiq+Va|l9L6y_(_f>w{<ax>eiD~SKd?4
zd%YU{xqb6)A0+Vhts$Ys^2q^c$gNSD1NfC?OdC|jMEq{-Wl8Nl93V?($G7GCb)CSa
zan!!k;WPy^`NW34r42OS-B*%$datt>VdABAh*W2#T<A63mM~FCwi3hxX^%3k24P+G
zhuHDkvA!Qm$og;*x$N0ySzS%aK6d+Z;GeAO)(f)C9kAGG?qx^P&aKk7`^{A@J&m?}
zrTc^~yK`Mz1sUe1Iv7ZgHUKV{9e_m;1#t}y4rg81hbR^(H5zfp)a2A5Rf+X4W)<8o
zVt11p<!iTSl&fviiHMSod+rLv{Tv9J?Yx8;7-?9yTlKZ~><Go%8uN9(qOTDV(=1<)
zOaq_sYy^(Lm&G{`(Y(s_Ii|6sZzSbi6Qhe~^8PSKjr`L&Fge;9!+}!=uE39YmZSGM
zaP1o~Q-45yN-sF+#B2TKAt-5c%Nc2JWent65Ap;%1Nb38_-N}vG`!$}0k?i)nwC)T
zP-ptiRF$Mpxs@SI%0fUKwd@~);Tu~>$<K^z%{<FM#@(JB<Qke!%^2AfeJ*a@QH0$q
z%}nUNV{v)|Isee~8C!AVM6f!qnmoW9WkJ{9iy%!^eJ>&W_DpXmY!P(XFd{YYd7HUI
zp}A){-PW^A#t7ZFp}e(nTlj59b70?g?kku+j_ZuX)6~xUH`l5`NnvqIWuQx=qy6xy
z%p#OlK+%P>I6-46rM(4GXgzHIClU*{nIseu*;33Rjj3ad>Ix65n69WCoi2;j)aSf?
z4kN7Knn`S|0>awcceQa==kHdlLhRkj4=orlp}QwyEg56=Us(rCmx7Q*a>cRXSr0TO
z^ZPkG&5-e8DhfG-;c~xKk=VuX&%$UCiL0=|F+<h1i0EGbsHT!OaHf#pOTQ*;TwC5A
z7PX*%n-hFK`S%H$c0f@|b2Hiufy359S(Fi|_`cmDv23hzX)pg+Ym8=m!@v;zm{8y<
z$Za}Zs5k|9njdszlPmYs30ligg}Ya5l&GtkjY6WJ>ZRSe^a#PD4sA7vh8Zwu{jCVq
zL0;e5m;%!{(j5uZ<D#FF3#mg!mRgtKf;6#xLTh8Kj;lbKax8$Jho}UR2P~ELHKc3Z
zAwLF&{Cwe<1r@x3S&=~vHE7aPE0$o8Ole13kfo@d1MSO0#`y_EkS;3UB(0V}Z7!D6
z;#z}DL4_Y|=P{xgT+PEHP-S=*uQHgGXBp|jh2yx*Jx6v|2k&(<{w}OoQEkCIdd9n1
z`zjbK--;YB79gj&b-=_3K+*9oct*g1%$!y$sQjsc>qyx7BL>QmLo0_rgQ7Ml6TKa{
z#<&_E(z7dc%mqDM|64D9dWo^}1(`L2fw{&oen}*O%9t}Rd;U&&iCeBoYK-G5<I*(a
z%{<xg<DO*#i%e}=i~;rn1=%KdKo>uMSu+Kjv_nE)Hd8yyow?g6mju6wv8A<H$@?P_
z0Z5POE)uyYpS@l)sHIP>S|S6}3OC5gVrR<g1~I%*+`Ic61U>s4Ks;F0fMJloigp$H
zQ)wtkm{TxwcTe25xx)RWg7tj8Yip)m!0>28UyM1KLnl+7(X$@puzhjk9GDk>r;_-Q
zFD6;`w@oI3wVb?E)SSJ_;PinZc5qEM-Gq**t@vVh&x{*8f^d4_P0~A`S=r1My&~S^
zGyAithvfqW1(tN(@5oYu?B;ca-G7xa;e~Ez+2OVD35hE{d=?j4cMQrETtKazquX^F
zL}?ew-f#uohD24N4}+ZNoDW-YRkln>oUPn9$87w{vw4nc@RoIY_yAv)<7k(Je3x8j
zFheWKbMqr$aAy<S`hg+RgXT7D$=DQp<k3xoG3b=~cHiSFN_RZ$5>^}!f)f#5n`f(M
zsN#S^<}wG>@UWCJpiT)$9E<m;(obM*CC`h>NN4bVEztCaVKA}c0$%dywn3jYZTJfu
zv->6kSFs$Rbns)IWxAj+El;k6W4nL9>53I~nYsp_ZKOKidDZ$?LQ*n$ewj2uE(nzf
z3-YvegR+3I+yb4z|Ak1TYr-%R8~7?Oo5=njx$NF1kdoaIhA9=Kt5+GTqU-FITw$R6
z33vHb=g!x^`}OvIAK*j`A3uFIoq{q&A44PxD(wQ;`((*@x;qq%s>3%+ELsx<VU$mq
zZ9cAPyKI5)D7n20(^~PzPJN9x*pPYyWAn<N50uNt;<S{K&dHW{7#x-{Mk6w5*zrm$
zL7IhzH!ELQN|djcSaci()lw8|jw*vWdB_@+Q6N6?z`~r0BRkC!Uw_oZb1#m%ro80o
zQ0FIYiO$3Ec8P)wEagyZ<&(Bh5ha^3#D6|jCndh}_g++@{?w#5N}!gL-$k<a+=$<j
z-vx5*kOo>*`QOKJ^Fz{>u#CpLvnh92bpOP)OONh!fqY0o=2+Gtdt`1}G2S#mkVL2x
z&>{)^=}HQ-50_4yi0Cu?MttohRjBd_{;H7b=UGrDSfH!4mioMv2KzWF*#1naxu|B%
z1miUz{XZ`Ee-UAt0P%Zjl$*T!<Bf_)x13I&8__m$Ue-Uz^Q4mD9wp>yrd6hf&x9ho
zjq`Zwh(zW3h~WXDY9K1!4GJyR_p$q!xpd7djw%gR7tkV>tj>EkT}>T7G18%dSonOq
zdcpR#3CnZ4rNz%=`8X>>31e4B6~vmO9#}btl0eypb*3N<?8X@Br8{<-r7duo5ZWcw
z98eKECAA(Co>9x%UC*d`+lp*6Jlz?`;x}irPMn|<^?1?zA>D#Rm<B9b!d28{LD@A9
z5rj>77N#a^9Rcuq`c6g@7TNCdN9Vemsb=%y?g=42DZD`t1?FjR^FTco=`x1BM<5Hx
zc0_@hthrP|V^JY{$LIzH@sr=Wyv+qR{18fS7$E3w%kM%b6B%Cwy5?qS?N4%AC=5XO
zEo7o4ezJzup6G*nPt`vMekv>|rKPlvkk6*|H;u+v%kwO9$bM?h_=$J}Sg*v92ksHI
zeXRfl^B0-i9k~xRavkLHtjk~L?tGOF3DNXd=#YX1!ijZrA_-)2!I7b~QDKdWSuIh;
z96wOtF*#=Lz~z$3lY=45GrJiMT|U+=|KfI!1~!Crop$ne|B=y_{h)}WTe@GM0!lab
z8U#;On8`NTa@<EZY!H|6sr3}=poAy1SiNMT*M0JUiS02@8NN+j9GBEUL!~_*33-L#
z%sVs`5t3ygZ!vN{dYTnIu@pB>wa6wfQ**{q1^-NCN;D4KbhCAXOx5x^YFqLnkVr2T
zWSoJrUpGkK%Bs}D{z$P}$Tvc*8^~Y)3~^)^HGU>{wT1jdNXhx>iz{i`Ji<<A+&I(G
zWh_M4H^Gca$l8CqES`Zqh(>V#11{IAVSj^Cp*q+5`a7+9Xz6)B&y|?7)BX*W`vnQ<
z&MS)21J1H*3)}yLCLu9DO60M9O}{eviu+L9&fDnW0Y_w2BGSR7koyKhdV0S|^0bHS
z9rBGqsW|A=Y&A&rIL(_@jLK2=Y2$vfT>O5c9q)Q{nF80}ms=2lyrHozKO#X_dt~BI
z4O@)+6{E&23*;6rV0`+Fhk}Yz6E&d?9oZGxq?^Pr<xuSG+4z$WA8^Y1%1??fdDv98
zKr{Y`voS!gRM^I7&IT!}t3y|h?CU(D5O3KkbpFEPU`*(N6}C5AK%b{TX@y0G^H@Ae
z2-(?h7hmvRd`PRNf$Lpeeigs~oa)xILAa~!-jUl%^_hB|2?-OLfDmD{gStLi#F;2t
zIkqnM0ZoAGAKh9y&CJc!n1<IC`3LmW;3sS7FxWQn<n3ZlKGX?-%0P;>6abc_l{V!^
zu;Uewss*pGpK3A8GIz#9L*>bg4g1C#t#M*}K};iPtOJgUebSPIDZk5*QqqK(NUYtO
z))=Rk^~|U!o!~WGd0ThN0wl?Q*t=A@E~HkQZg^&iAzYFYr}|$sYGnM;<)&iI5=4<H
zOt<N_*oi3SNrt{7rcI(S3O9kgmK|c`>TTUTBt=(AyJo^tb0NKw>~~DB6?$hO?BEB*
zBqp!RF7<-r=gcuoqNRLcCV>J1)I_`&RQJ?|cZuJ3`*o;O<?Nzpkk+ijMT2Bg9aqy0
z>pq>cW%oO+o`Fw|DyJ(2r19Xfm>LyTL?B+3?3aLLmodTWL*lDzneyu?GTc8ytRrec
zVpFvbb%8!Mm0%*BX_mRrbtj&x7r$;HL_Gt!2+t0!eu?G^XF1_k+G;Z7&CR813xs7t
z_?Y<8=c6HD*eZ^PwxR~|0PGX5_sG0w#&P-eoZijoo2&Hq7vG6_Roy*fT5vP9@chV@
zy$u4Uby7gXuj4e2ZA4Gd8tgI<Uv!{5jKNc}E%Y9irBEyU>f(k>I4y=@xJDc}?cOt~
zw!Z6(IlgmQ^O5cS=jJz&W(q-}bv0QuRxNe?HWyt8cf^g`!qXY9_;hvx`ZSNGQXBrU
zcrd<q)ggoTNZAP`%|Dr@dt&RkKsOnK3#R~Z90gbS@3Ha9lJk_>xw1R-mjjyeVqM6W
z1zEo^zbXPf6kwFAy4+IWo5<eC^mL}TwLGyFL+%|9k5^tV-lx59gu;}f`23K#67M>F
zpA%`izQ%Qmf@Hl6b2gB<^dBkjAoH-?NE?MfT>>b+1$w+blyw@haiRx+YaX0onVRs`
z;^%)zwTi(PWKp#zD{t?bVmiVir5dB30;l%%PBXk71gOp-9`h_)h+T`s$HBCL(&sr3
zs=AQMRz!goLtPcY0}Z@>*{GacWlfl?c$93f%fHW`Rskg<h|W?V^@>2nDjY^!0A4)k
z@8iUg?a>pXbEwwaD7Y*H#f279$+Q+LDlr$tjoT21C0==-xXgb)2{j83ItYA0Y-Tjj
z_<t8?kIp~vD|(gUY8KMva#mJ~dJt#X8qhjM;#DpmB=(*3>!+>iHH;M3@xBHJ?+F9%
zySh__!UPOgAs@ml#?}Tp_N7Q_`Ky{v;ubADu*}1)c_@N8N=0?BfNIB>&SHU<-&5`;
zaT3j~n@nzS!X&^AFQDr5AD;A7DPVxbwY7iuj+a*>Y5kZq5E)dLoe=aU40|^Hnojj|
zFT@RXT3WA}HdRFw&@Iy0q*Q_*uV!k8ID+^+(PoI|vp-FDYQ!AHRV*cUtBh`s?t~Ig
zF$H^7Z+HQD2eqU&4KR_ihyvJ79v0J6>BI_AbG8+%8r^VMOy(Dz3ZNE6YfMC6cCK_$
zN@HB1dibCHXS#aUifPzhx0Kglh%bB@WLN9#(o8{hJX%Q^XOJT?J77xU>DRqWkSS-X
z=hZV!E`dxzc63R!f!lj52;eRxC=^bgB?<;Wg&37<jxun;c#2nt_Z1kp64UcuWwmNK
z8wGc$`jd5feo0yCwyy5K88xcPBnt8ASNEUh+wk(Gi0T0g3$i!8*@}U6K_s$UP)Pq;
z!|HHzarfbxFhS-Y1lP;i*LMr<sRF}ZcxDt{^!&n%--e*+8&(a$uM#oJe~Hob_p_<r
zxY@W+G4>Gl=RU~c<n)ggB(Vf^Ct(eBL0r&2o=};SM^xwk0)kX{FtfWFfa0wxQZ-|L
zYR;Z}7MK4w=dN1n)Rz+3NJA&pY8r8(?EpW?A%i#}QyqRr{M;rG8|$cNxtgn&%tQ#Y
zThjzeEO)=h%7KoQ01Jc{n?|OLtL1maqZ71Xsp5yONz=`hNjUDTh`|)-PL<gWw*TBe
znmcmqrYn3-#ULzx2O()qQ2Wu>v?9b}mDZJ{2x0*tU-!fbmdonFz=FQ=hhld0eFM(F
zx+7s*&PFvyFBSY|kl*WTpGq2$Q_B^4j<@SQ&7}8pcl`D4xD>ujF4<3Tewqto36sCa
z;)TUiY8GI=)B)hmBLr7b6~dg90)ocID{MP96&htN49<z?qt}!_jW&O6<4xRY)jzvw
zkUb4+u%h$gL5L(#Z5jFj7_P+HrKQYJ2haeZZ`P5bTYIV?VcO<NJsR(RzlSz}a*CG=
zZ1@gfZBVQj`hz6~LjlSLFgfv@H?{i*lpySO0m%les;zxEsXXItt!bv|A?C0Y)Q>ri
zE5{nZkEAnfg&o^o!!ohkePsV9!3%U#ae)?VsTF}LK^0@o?_oiQmY%b+<*|(t*aUJ0
zw(f2eQ~xyP8brf=`E#vj7l4AyJ!OjSg9gATC@I(~QWeMJ%Br>sQy!?(@mg+xu+K76
z$OM=XwP@zvj`xkiPlKKdm61zE30x64_CHfQcD-B@{O<m}kfsUP-DgUL_nut=2$zYH
zo;C=4b6)ZtyhX^;v#BN|t8}JPMmk%=ob1P#dM>`En>ho&9t*-BzW`wVYWWdx9qsRv
z2Hr!DLITgEy2==n%=tapcs25}T7GO4v13E8VT0IVAC;_$b5EnjrOQ&E?*VD0sgm+B
zg8X|J>_<BrQ^u*vF&G}VtG{nl;0&j6ZcP25F1Bi{scb$61<VFyBqL-!t)%Jq@QBOp
z?NiY#R~(a8@PLA*-DRMaVX#UuW4YKXY})1OYQ_T)nBF!JhTSa_nu7Uv?!iG-`nQz4
zInU(2L;aqe3mXO^Ey)3d-Mg|Dcx*lQhIsR8fc4kJV8TgHHt9YD@jDx1w#ZybDSD5E
zd-m*5f7nPxtp@2-lhq&(w>%0jxIFb-Q%u57%S#cQgrlHc_H@9cDEXNQW}^Gvjp#a`
zyqSFJE&K4D7jHycgZfOE)5tXV4$3A`Ez&1!+pI|Zlu>(ku{H0-A8JK|^a5&F34<L`
zrTp>J^nYiqS!<*oKW%%FINqp1vEnfLOb`Lo3dib+D*uF6iAC7<%&ZHH_6R`E0_?Dg
zfCkL-KbG1#x&FI2*Afi<H?uCnUBSrD2UNhU3DAkPAegg3B6UR^3GWdmj21eaTGjjr
zsv}_ax!s7SzlUrUmPY(QOa!FlB#=~u%!Ii@@ilX6<EVAY2vj-zsIzfVn$RuDC5^t%
zjY95`q$!Om1eF1r2$Y$1@+1pe$!%5?yH_P5lukUQMHk+BW>VxrX;s!<oiy6?PRI4y
zB>gb*gplyPj`<^iv}$D$M3(glMnFrP1)Otvw97|q?a4>mTv)J7D1W=H%fAi~o(A7d
zfQj_)LY;HzM?2JWiv|uWxT$7A-JOwjZH~C~asKF7f*>PuoF+TNt-T@MhII}5i=qE#
z0HZw*)FTN-oC>|qlSW%bfxOpI|7NPNoEZ~(d2U}WWA8JVF^5bZs2OdIVjG;p_{)f%
z`H}C4wi>#`Fk4flc5}`~I;g?~6Wvf%EsE*9E?b@>HAoz~^GU1Ad3JJAt;JdkZ$Twh
zo<4eH)ILCT?IqCzP*!0#INSrOl43_z48%lx&vonXTW=<25onyG0`he24dqcW;PA=E
zjGE#EU`Gsdb~~Wx{Ix!)K?)%X8NQjRW$pgGQ8f;)vs$XH@;eAh)#^6)CN)?wcvuTE
zTE28T0PBt6ib?tVxNMF6rkiyl5IAOswQ1O2ii>d4PM_8X03?n{A`S~VVkB`6@~vR=
z3`PG71s=nZK)QTK(!}4hZSD6xE_tFkhr&_AH4f7A9$<L?sihS1>qZB~5e=n7wI?}0
z!=BRQuzTwCxj6v%Z?Bla5JWo;Z_cRZeDq%TN4S5WqVTgW=f*)5bF#PKbf!12WmHuh
z!nvOE%!mnq8^&$)<OW~Rkz5Q>ud}{ueS*jj#5K}-AtK-?Lc9p7e3I`ynF7+)sGNK$
z&OY|A=ID|(DOJFT5yk)CTm#L<hQ>d&+Re{FjFO&@a^%M20fu6mNkKo)%3ua`a`VMv
z(#3skzFw^GY@MK5Bub_z>zNX_t)ebB>-ziK5;WMrxoJmaZ!NWAjl%lQA}4>FQWqDu
zgO@vE<U}dC1JpttqzZ_42jLa~r`&LiQ2BPH&|Iz))OpeRCIqgmks`}I9ar8$_OoVG
zHCgmFiTBXHVbpXq0HCTWNpW>tAl<L~@75qIz=(cN#X1=16w1&qrHm8p2<%P=8!Bjd
zdx_oiqbV{CWSGEX;blR~0~AA~-wkO7Pa)!!C<6aFUhNF7fz$dbsh4b_7`)W=>~+xj
zIj(R-SB+<sN#k6%!?C9lMjvSUKjzAbIj?puNO!PBjr!cO+aANKeA`^=7Gq&!-2|Mi
z;7>bTZX;;+oW<b8)a!|nY*BA}L#wqEdo?vs(+K_S52z8n2zR`{z6OyN{!G8Y57$T~
zO*z<35S4XFOtA=Yj|7HZj@Ie7ZOxZr>1+3T7eM@l_Y8`or*|tV2kFuZk^+vo*D{aw
zH7ScNI{sipJqVlMsNDuqOUx#jyvQT(uUrHD_w+uC@-sz@Pcthc2jSB<<A~vL@DQ<;
zk2e6u9riwGX*C7o0$JPgrAVF^Ah-Aa`eka^^Yz09KrNjdv!7ZCC@i+IaB(#F_+C(w
zodZa$4g;=#A=~8yM1gwGSiugD=|AT<$*5zeHF#TOoD|g84h;niai_(c$f-O^6n1Pw
zNX;PM33MFBv*G^TfQr8eSFNuYSx#HaP_asDg42o~pxrTd#lqC#jESZ~{;ptLNvNYt
zcX7_yMev$_I$OdEs5KZb3KO8fRoehm*VWtfo!CX^8gfQxhHK+0;6SnQ@e?883XbS>
zXtLxzbsHR5ids1&a^P{X0}ub{2v@}Jue<3CJg3yiPp$T5i=*4^Id{CV1cD!t^nk*7
z5Q4z-9!`0QAuNxigmli_+%ouKVNqd3yD3Fcs#+A~F0IQcPfs|HV-^)Ybm(JFtyw85
z??)SKhKiW$8O#pQ$KzJ%{bT;?qO98UN6wZp1W1H<AYz=AZF2Q*<5I-H04;F2z?J_9
zI>v?G>~>ucwbWu2&ZG%}P|)O<<MX@?O<v*iV;+NP(xo5e?u63))a1mG?bhS@7kG5s
z$$nm-T>v<@nOGg?W_yx?X@3s#ky4NYx~K#-r&_+(br02le#5gcUddL@KFtRt1)-`I
z-toF5ide{qP9=IhJnF1iR0whrPQ(Uhv#sxu2tha+puX6e%BWqzH>N8)+2j>@H_=Hf
z(`vlfn9aqFn{uRKSPE(lH4tu~4LZd0@AxH^RCH?dWxw=g{w1=kOhzL?fTqBT?(0mZ
zPmPOZ_J=iPbCLchIh0~L$T<&yOqUAvrX^PvC9(dCmBZ83Z&BfKMOeUQBC0{?*56Zv
z&qX!+@L5J$SM4SpDJlRgHGO?>(vn_#u2Z_;Jkty}d=>@S3$@SYE?B`FaR#{sB-yk|
z3IgVfF&6XK#}8y`W4!CMCmle8BTO&fJZh0nER&+14^W9#D+@pR!v|zUEk0EAxEISo
zB^eR!gpwiMd8RN1tVy7!onxN+{;MAx_dLGNRM!DC9sGD`i)}3b;^h+)7&=!+Y-Z5%
z(nN2!BQT~5b=p4d-G5uzNJgnSM!<p-i?jUU8E7vEYAiwNZezm{-|dW8?||tYdHiWR
zFZQ>LDs;kjlC7BQt~~6y5hY6v#A-2M1{Y4^nD2Yjpq2tziPlwb+XDXI<P1RpC^II;
z7`lB;#-cKRbX!PAoFI3|f(#Iy%*!>D29uBy8KAIB8>njw+{^wgX<Cp*yc|rFKqnx8
z57In+#WjW?2`n$`P-;;lkiCoS&oYW?W8izl+)21OdT@9Z2+>hc0vfAehBydykwl#5
z+i>DvfHHsx0o?+K(qVVvnmRX~96(uFWQ1F6g9}<^g<L^|7?HU>Fb|?R?L$4VaaJr;
z4Nq0S`;JGqnZ-6mj_&>m^E-f6<Y)N`8D5AjqJnSlu(FogS9jCjqfURaB?2myCH&|d
z?L|1?BFZL)2KeR*4M3m<<P(S$06{g!O$|%`$R4#)z@V><(~hK16rG*s4owl(h^`!q
zoUO2k4N899zgZo7osz$bqz)11X7TG;C3pXZQFy(Kww7sJN(qSSZL$<y)2%4oQ#zq(
zhKY&cE$*so%c-vR!~^a<(;-%^x>z3!L9DK;9{Y#T#QziY2mZRPyB@?3y0-Q$a`)@J
zio_^_IH7r#a3i8;^|!3G*Q7OJ-KOdO{c-Fzc6o1;jp*^8tdPycws%qD0`bnkyTQVf
z)roJdXA(?jz^s}fo&azvJ}h{HV;@1U+uRS)g!Md&%f+}81)E0))kIgnm+atD#5B#$
z$6JGh6y|?nH>2>H1b{HddjU!6PiG+l`B6xb7hoLEzx1`2TCdYd@VpDW!K8#3hZ=l|
z=;F7Otf&>c0DRE2###y%Y1mM6=&cQ0lDocuiKR0#hl&CI33IY(+1%S?FS_($V+O9t
z#u=$tc>9?WzfbRmcf4d0F6z&Ps{Sf|`2tecdSV{QAI!#LIx1`6vG6ESzxKU)c7`JV
zE<mXmAe=;*O5cquf4#y%{M5aG*bN4BC8q^uSx&QQVdqa2->qQ}%<j|hCJF4d2x4sl
zJVA8rNA~RjEY2|CR;7WpiyTQ)fB&34dQu`aoFkYC=eo><&&P;zkfBVlxFr}OU(fYB
z0bm8fr~(+0y8CrbCHrtDMLrv9xF#c6yq~9Va-yQYhP(@MVgL*FMs%I;dxNfnCuD?)
z33$Xpaz<medjom9*=QEhh4320siqP8Y(VzwKB_AAg3LoA1>oQhsQTP`VDI^QVcxXY
zd$-q&6fAV*Ip%%*p2~rmDrlM07=~Cm5aLUznwbOo6$rk)8!#hY7sX#YQ?QUzw?rI-
znWx*3@-OtLV=rJ<ip{8m*Tlh}$B{_j=vODm#ad8v`QKjoHMv+J?R+7HOI6PRWl#<|
za**x@F|$VzVl%T^HtsHhqnTdA-z)vtfctl<F2aqTZ{fa1xDFy}8t{OZ53&t>G5f?$
zU0%ovP_}he<c|Yt3g3}c6?i^hvgUn*rbCI$tB;)=7h`UY7xa-MX_~eV`@p%abww3@
z^1ciLX>9vsz~mB3`O+?F)TPBd`)<a`9B-aOQb36f0v}Mu{Y<nxxj1@Hh$S(LZ#C;o
z+H|=&NLFIv8N}?64tN4Uk|4c6?sEDYV@*!DsznBPM}fc|n~D_cXZ7s?S=;hp=9yfu
zGJ`JvmIsHZ(U$Z0_X|Xnpt`ZY`2p^Ca~p--VO25qZuEJ`w{%j9p*#Q6a7D@bAl<#M
zk0~!+Ktj&UL%kg9Sir&9pPycWOSctNfQ-2@l0Y<#hfj(6*C!u?vdAJBA;ks}O~84)
zJ=Xo<uGhy?;amw6Pv>#O7jQCqza%xN0-b9GwuV6Qc}<v3?q{NP;YA%MxUP@uAf*EV
zfJAoux?gp-Ko0^eo(!r=9Q>}~Yy@@vO=SI<IqieuJ9yX}K;324Bt6mfRa;5O>#a_l
z!2CA%f2k(>yv$mQ4AL{`)t?yn7oxS&;eDz?#-pzpa5r?5D_}VJb*C53T8g}sG;cIw
zAy=!};F8Av9%PgjI8WPA!#;~kqDFswPul1uQ`HPbZXi&>K9d#2cRQgG-dE&QiIQab
zd9x7<gxEd1B9CPHfkC}m{6pagH~ec+Aq-odhv<OL#u{MM2^k95j#|GZS-&dWye*Mh
zG{um)nKD*H@;cjBr6#1%tL_sO!SxO>ZEHxtEBc>B47LY>SxPB!9TcJWLWSdhB3w1*
zzqleqD8yM+FSCr&j-6YxRGZ)=(1tuT^Crle4WfHEyE%gacXClIQkQ-da9`vG-ig33
zSwpQ;ZLn{#Ry$e5(jHCQFS@R4GMP|PyESK{E(r6eE~pnMqJUz|=65~%&pXX-$6+$b
z9PU)3*qKAl&rFC+l0lG1o2($k-4(g`&u^w$DD7{+D9u4qk~1oP!sm>H)9hmGNYNT@
zf5q`k43UsCswiuY$C9~QMEi3W4HJ3lUA5<tivGty;X0)ShRhaQ@}Rlsh3*4cmZm8w
zEJ2$TEs9Qoiv6Mb^c^y}<6NfKWP>|2F1lwo6?*6E{doK@%PFnzljM%4PteF#t^FrY
z68Q&4(+-N>zY?<^%;Q>HOQ}KSl!6pp$=LmK<J{TW;%l)fhPNj5i<a+7O2?`J@*8BG
zrmHsdegoV_)c9jPSZI2XtaT99o-)|g+?ptS4`$Z=i|OC19@i2=RWpN)^CRH$z4ocT
z#3o!i!@q*NR+QJ@?w|jJN=j_j@o9hjck6#k0c~=jQ@=g6Ui8#*vd04bz|77d>L2`W
z(STbM1Yo(^)lJfTN1?}GQMmP}0(co?wt^NNnR(^gPOtYgdv>UU7l~kWx>Y)>O`p^$
zs&MC%KqY|%bbc~GsS?7LB12X~^y3Be?NyX>=#Wp_#YnQEDA7+~QcU1z(;v_^ljK#3
zO8k~K1@1BVEe=<~E_ct}7Xo_jP?(9Cqy5dz3ccgOpKp(g@q?MAPiR=l>zdw#V@1{M
zsf9aZ7|Q91z|ZBHt|*{}bwV`_G!8Y#ikq96dAGAWZa3=7R<_E!K)0X~pX)2DlcYtV
zn|tArHG_gP?u>Tj)z%U4-%4C_6SV-}&=GQLbEL5=Z}kq)+)$b%bfWcTk_Q}l^kmD_
z;Aq+r5i5sAIiOb+$i;_3aDln8>pTnMg!&Fe^M6M~d{ux0(Yl{K&2YpM-KL8A{=pv<
zRr*wivxo~8M<%wnRnHUpJ5cUL(5lX_Zd~&_Pa(>ZSpg$lis&<2?U_w93Vd^pG~CQt
zkxRJ@rcG^xmIftJg>9X}_yWC2^JXH|++3VwSXKBa;s@Q_{0E~6!30i*T<}s6nJXO=
zJ{UqRF#$?9v1+gDS)r*3XW-tQNGulHfF+hdh%_6YPGgsjs+x%`fAL#iI>(@}IokgO
z$Z6n@q3L+Y-u8>w+uLzoUgcS#{A!DIiGHDg3M61ts;`TxMOunpdu79bi9x|Lr8Zf^
zw#qGGP!SKMPVX}XAZ#WIQ6nLr?GyITe&89$7AKb>y|WnVXCeY{<3N*YKcENO*OZcr
z_o^%d9Jmj$h5ticS1O#*d0WxLuWc&w9?k`$7vYaV8&v4)7F+ft{b-ECG;4}B;m4}2
z#-A2lUWqtC9^94#h?1+I$-WP9^2v#>y?KfNAa7L#P4w<5!gnI#f7DNDr3-d2aDfC+
zRxnfmu6ksVtq+Pjj@^u$vAsA|T{!<2*&$xhl=@bYO?*!VchF|eJ_VD`-8$dfz8%~9
zRo)izLY?<p8H~!B06@iZsKp~hFTm9iX%bHcWV=wnlC|h`YlZA@7SDP2IgOVU1>_0V
zl(kh3R0fbNf&F7~Cw;3EV&QbrsMlvxfS$ZtN(jWRF>JkVrZ5L*%}lnv!_0<gQ3YAx
z_D*weizFM`R9&PhJD{hpaA>9S4pOCP;H!xd&bXn6!T2I%hppF7yxTrznP~GGM35eo
z1O2M2c7GU@>lct4kjIXlB5V^;1UIEN@HpT-P-f@@0-$L*;Log;s`E?}W?lbg7HW<F
zC>*tubjH>0Ip3oE6`1CvlOP87_R~ilM-jFn+qjiNi7QD4ggm6dj;9lrs~3L@H6v9e
zZsq8vY}pz-T0|W$Zp?c)K5gn`qszSn>`~Tq1wnu9%UWY*qVw%SvDj1qRDI0>=|CX~
zSzN#&mtu|t6fjhEvF;e2=wR4W8{myYKn9|$x1)Z-a?OW0{-LFA4g=G=!Ht7W%)-7R
zm<H)HQCxc<iKiX`196G(3&JwV<Vv=_`3Dh0=WuHxV|p;8aS-c;*}6hR37+l)nJ8;D
zXdt=U-C|^Z_oCeXH{*cd=_Mh@MmRDqX_Rr`B@dcGH{WouG5~)bn*SvIS5ye819t3j
z6v#r6YJj&VZ%9!}dr|hj<>cv188B~vV76BcwKymzFuvUk@39hG6`O~f^4cLHg2U<z
zW_J~&2)aAKixjf&r3f#$xPY7>?At;W3S<X>{7Jt#VN*&66Sx>LhL;|h0w0~O1te8L
zA$Xty+x2~`{Y$4L^6bhx&|_!2){s-utx}T#91&AfdMybneAPYejEEH4Ga+_adLx)O
z))JG#dohwFq_DxY>K=?Y{v5E+&x)*l>V#F?^JEC~O>frmXI%y_bG-CMi@AP@%b~P~
z4@O6ERl4xjB5t(l4fqbxH2JLXB|o?>x5@nm?z5Ms`0_29ci^Ind!)Zkwc0mUM#gF8
zFk5{2+*Hwr*d9`aFNA`cFk!8=6v@}_8g9#n*~CxQ{{aZRDw`M4FQ6w^<KsoWbL*c(
zui;V<nwGkMs11y0S$pLdjwUwrMb+YOi7qZ<2EB0(ibGX^z{7QSfJaGRxC4>brQTt*
zM%*1Z{*Jq9BcOIxj$4@0xAj`oi|FH{#3Rr=E9k%!ud;VG6U89kk9hc^Rsc>hsCf$=
zYk7i*$s2zV2h$_Pi#kPv8;4qD^oqbe1VgQu54MY_@}_}?o;aysG&5j;n=0CA{>eON
zBMR6g61L;9dFh9_12<eeudK2sF6tkAx~LLth{|CnSbECjfL;)3b$N5D`HLiNQT-vV
zk%$f<H0AVKG>crnTM;4G7M1x->16m`XS0_f@qKBhBDWt^-Y8cG`0Oxk-a||&cZ=vi
zYE3dAH*|t7oN3wDIa^+4hKuM3mnS-{9736Aw6R?p=rQryB(jL|T^#M`Bxw2{1R!)W
zsF>)%A5I8Q<ywI&1Yc!H7xZ$2MTHQJ1ECN+Z`aM~`o!97p*EQP>mHnk0$$*}5vW<B
z1C*gSIZXZf5y7J#X+jrZ+osWM%m6oEYy&-@L1wxsU&;bh&1#Ro$VvSmH&IpM*q0iq
zstaq*cj%iMO+$+4;rUB|Ja}-B(yi`(5m)(P0)Zt(n@Md5uc&H`f{%VWif|KI{~jLD
zOm2fRCA|EjoGmZ&qD551DnPO-ifkN|!l^dFP#8r~8us`F_rdXmr}-LFhM-mx=<O&n
zAO9e3==3_{@Mp5-+vrU$a6U8qCOljuC-{CG@Y-q}k%~pz!tmgIf(3ekZ-(sV%|6c3
z%b=>=`#N0d$7AQudO17+or&67e1(W7ycI2E-FUfxmpm1@ZI?R;V19D0R9wV+Sia%B
z&fUhigqsE|M>s!OS(~IG!UnlGaNdPUZ=vaM<FyLSgBJf1`~<_z7Y=nf=y<3`p^A{x
z-}N^2i74W5Gh(JU=hJi<Jj=PL63~mf1YVD2d`H^ulO&UZ$`xj1a-zf4O7f=Y7I}l)
z&C$-;;7J~JX*`%STr@RU>e!bjeBjk#s^`yL0nd0f*;w*4x(Vim!HHT!l?R(pk+3C0
z5n~NKn=R57d-0^}I}ADlDzQ|3jfe1&Oi-mPb;w`2#=@KS@PRkIuNvPZs)_y#<{<w<
zrn;~F2U~wRAgM8UC*6CBRE6*5_gXMSwBT^q1R)UrU5+#jV+Uv_@9z&^3`jtXW+6{5
zt|zMepX26HEKW@bg6HrLzD+~|?Gt(TdDAKomE00gn%R#5u`qZ6e#XBEOrnzmLi!(g
zj20AXD{rK5x3TnqL!iTB;3N@u^r+KZN0U-Y-XnP2Y(-uC<pF5YR_}K6dJEPj&mYc)
zs0h{gPEjfB&3XV;2ZR#rDu{fuaANjO10%v~5EkpMzTgR%J*5LRY}`o<!)8>~S40oA
z3cynbf>Q;${Q$Ng`P(_;To=*>F^ZPtS6b#s-m}xB5Xl>I1LI@EB1ziX{gq=k*w^jG
zi%2_2@uL2)fuPf6iU6H@2L<ai>xFDll>RD0ks*eNJclki&H%katgl}PO%=?fCwV>Y
z;N5M(4xT~)&!NbfW#0!rHu>`BcLMMPt^mQ3>U@PB?0t*R3(Sq=L!dnL{PSRchmhqy
zgBNTKznbDVZk>h?Aag54S!zltfF4&53KIv@6}ZOCg`DBPAD@|{q0Sv2bo(<n=WL2t
zxCUkH6n8*$0|rAS!r=-qQEcXZ9v8M4_YZ)YIg<bE@uByNs>?f+>S|PxG<q$5M8piV
z@WITEgsmBhw&8mbUJKO-3r<(`3u<Ws!$U=MuQ)Z2mA3dxAFYYKxSjj?*Gq%lrHN@1
z+Iu)U{Qo<#;YxS#-<xXAKmGnER6_R{tswfQ_mi}3tAE^f>h~uOs5Se{no@{q2$lct
z*Wa?G3t#o@H}`J4)rN<U1n#fas_XF1N^1_H;P11w2_!n-pF}VBz?JLVpZsHQ{S;X_
zf$jIlKhayIV={@m|8H5v*oDrE>lo^PehS8F8tXq<_l-Td^Cv3{04M*ne>`~anQJPc
zB{(X445`GgY>X)S=D=04zfut6GS?FdbS?QewX(@Sx_=(J%W>U`wd3F+=xg7G-*(?3
zokBRG5ZDT6e5XvzI-gk`y955u=c?BRRX5wpeH3`cPuu<bD-+RAJ_ch{PL3b1xVi)3
z{tYe-+DW1xcnGI5Vt+@?lrAwQQ4<$G@vX%22W%YKH6x>(z#dL*Xu3LRDs&miEeqVo
z>s+_~Y(?VvSn!z*m(}I%8lWY7<5;9tK6y2?#)&NOU8)>_8-8`u)<og+MC!(C+ro0q
zwuWBZv-8~6usdF6(AJLQwSL_13qBEIdfJmU9ivp}H~x>B<OZA&rbHedJIO3A;qgyR
z)xP^SWL~?C`Xz0p-mcP7yj*EKKAt;I5H5q1Q_nx;6^{(ou1r@40OC6c;`KWinQXmF
zRlu+kVQ>LspdT)-2nVh{y&g}qwjGqY9^tVRVGPP8tq5@GM?l%U|HQ~9xn%HR8+q`p
z1Y@8dy~n=|%TV7337PEd-h71tsasOis`8Taxu4zzoozGV27bSCB4mT34fuHFSL+*9
zrT;h*zkPDzV4DJeG|rkWWlDU`fO&W=`)b~WTqr<YPoR^sxE}6ja4DT%$vM<-U-#Vr
z0P!{*Qrg#bT;V7pQEOT5=9PiPv8j*i2uVMIwS*Sou3rE7I_GGh60euUUr$;m_{|Qa
zvy=U49!n%QaCFt6ax8a(c4`XM^|#eA;GAmioZfEv6g*RY<hovPSqk{@%zpP3uE&6)
zYMWh7L>T`8DWbZyf_j(Jov*hy?<U4-DeW&X-ejYIOtXT7^pBOofx%SBT<_Vs|8`~M
zv3qXXNCv;U1fHj}{C6>f1iMkcJBWgF*Yk=?bPIrCbbhE!9LY_a$o10<BDr||ciuH@
zA@XU@RmP=M=$}iGqk)HX-v|VL8{jur9&{hsZ2~?sq!DcI76b0)pOk|NHi%&8c<C*j
z_tsl%Sow2tQh(3m=i4bRKlH!SF@@!s^#l-O*n%L3cyzg8dd+_%C2`<evix_P0+dos
z=Oe3#x44O`mmZyJ_;;IlXy)2S9u9|1dyK44%lc@ZHXImVRt2Xke@Ei9J+x|g(;fm5
z#{Yf~;kk_tV=Ab4q5xsuw^Y(D?|eyVfld7aoD<)p%gGoN@~<^a=?wJ)a{m`-oOKQi
zjhqbK{qeGxH(TcJ=7}E*v3=x@K77q5^KQZP0l&RF4{SiKHZ)D$=1sooA^97}PgZDD
zM*}+j+nQwO;ns*TyfSZUO1}x&#<|yc;AhEgq}0UwGCK^%q^jubzp>l{B~C7%e+XHC
z@uf=+LG&E7$5@C(-+$&^hfcnRrT-7u``j{*54DBumFCe7hAbv^dZN}kZv9V^dq^GX
z4@B{A^V>T%g}uv#eA5sDp2&W!abC$>GsqX@cpxTqE&7&8Zs5+Ad~dc5zF+1#OItIY
zyzEVRL%UUk&@&=(w4BNsA%|_Gw)PFtV)tL<CADEi)6-O0I6+5$o}08WBkHOo<f8k!
z)gHSOk^smUd!Xdh{d)jcN&L=tb?7nopKzfd`h%BwmU*OpDb*DH4dj<V?r>yR#(rV$
zlKBQd{l5;vvVOW8X3znq$sKF_^cP8;bZqsOF)6wOv*cI5i!wm@o1MGf74MkC`QYhH
z0^~w(5VWq6&7dsj$ZQsbSUGl|J;ZCmn|vX;{sDY8KQ4$=0JJg4{UomcP0_NLu<^j&
zQ}-P~)P8y;O&GUqzY7bhb6s^15CBubn>=5w&su9(pASBsyk}>sP?R7;&vOmqkJMDx
z$!3^*Id2PUxwQ5(c$n5yomK|f^X)VNWRPnNs~^fi$Lufd^jXKy_|tQN400NL(4q3n
zWn@YRhm*UXWbYL{FHq$hnNl%qL#Fg~5==Ucu;*>O6O=-Fyp<XV<K5%a1Pxz$O;$-@
zGHr4~1|GL%o<$lQH<js4Xjaza_G-PQJqYvYe<1~_MymmSlm2L%u7Ru9gL!hGq@#w1
zv-|>kv_6m&?tDS;Zp~9y(4XEqmEPDt-xjEZ0+ydK_P&2@-Vs91Z?Ch*@amKw6iy<p
z3Uh`$Va@<49MSUCncGYrdeKVr3M~umF`1IKYMErLu4nQ`qS#VZumIQNxUH|Yh0tG$
z?LAUIhdctbDk9^poRTpA+Z|2T#wMah`LcME^^&HLQZO#9#7_m8V_(;OEBVK-&K&;}
zSl)a5YhkE{a_E!)!WzHBB{Bu=HCaJO+qm75dU%vQK-Hcq1fg2LZ+|1@zpqZ)Hm%ds
zNdC;-4R-)4)v|hng~`I(IHUt0sn?uFh4Z=m_(7!K@Ia$)prj<c5`ho~Ie`%*rJzR2
zSEAb(={~(X417*trj}T}LMi6huUb&bAW_%7$t^i5gc9PcOOO=bL7u5?oc26X6tpVp
z`R~OJ4<k!~{femLe-rh5*M)Y0uPMx=@e-HuvY?R#-&o3S{a-!!ltL<0;gqEVcqQW?
zRNBmyBlacL9nS3Dj+%kCJ(59^(!#We^FOeFK`h=5*8R}67wtW1$0GAUrl3Dx2m>H<
zO-J45{w(g6AYecy(LFZkDP>bwvCF~4Q^U=%TOLF4VXm5*#TVWbpFUlB{^VcJkgPTZ
z9=UO9o+c0y`Ee2SI_2-M!)^DhtY6v$R1oT2DRT!SIl^*I(kOGZoXaL4-F$7XBE%_H
zCOqHCjo{-rpqvVn%oFaMYbx*<=>e5>Lae+QB1G%AvYQ|y2$>dw6G4D3vv(F^II`Wx
z*<=lP{x3oj<9JP;FcTn(ci9$UdCzLOv;l=5B=sP=g?Y+15d=tCcT3#Lmu@NO?@gvk
z`@6JcW+9gS#qF8eX1_rt#B4#SVUK>h2pLr4)5EuC^*&3((~0a|=E+}3ZsKX|bYM`e
z{s{kddho5VwtwlD{)H(hXmx!y)F#L8h*N`c^U_v*D(RfJkod(Ho+y7O2=_>Ia{1zA
z<_<8Lk?nHj;G;63;QQ)VA)x7<E}{GMu8gP31>S?Y2B*EYd@Z?t4=5$+a)ynRZ^8hq
zHB-xiQv8KC`ugEF5Tfu-HgMVg-0_ay;nEp*Sn)qecNl3QIHmne=DMJx?Uq%{V^BT=
zJPRNSVm9z`6o46SyqhyT{rE~M@d;@MSIr`MmE_eQH{t14wE_d6C&(X1L}&pB0~kQ~
z)`2x*8u-g0VJ-hMF1K{vU*8G=Ij_ytg<(<4cK=Fm*OjEvIuv(2ZPjOz!uJ~iRdnuz
zR23HOdKhQi3E-r95P@>;StyA5!cVq=41T*vSULv)cW3a;=C?@=+}3-^@uuP&Sc2%r
z&A&=sfqtI1RFyms0zB@85Iz$X!g283=os*A_IJMo$Mh>I-FOyFA)sU6&O9$LfCTxO
z>U3ZP_6wkd|Amwga(N*$AgN)U7WjGsWutWQ``F;-u8L{ha6VzGzR_`>1R5={ZX*TL
zfjux?&3ojH?w~?)1mV~EsSukJ0$WLB4e-hCcfW&9t@ZzD&eP@w5gme%C*v&Vg`@Ft
zwlY7A*4`llJ_nui8vuBPk>|D$6%)crI|w<K_qrVrVr%c>3a<uLY^n?&YAeKgcsb2;
z0*#Fz7PJBx$iy$C1rWOUzFGhPvYotEKCZ4Ed{m;GAJNz{?<Q<qh{Yo$hRk)W;DtYc
zg3l2lU@gR2Z&<ZHelL%-;mm3lU%<(t`3W3wHENRP(C;*`(!>_PT6F9}1_@QqMmU%E
z^2c{Nc0}6);9Xe0m&^;x42ojm=t^La`hCl0o$_MagPcO5?|wK5Jz89QBF<h?8S7Qk
zNP=M!z%T@Zd3y*Nmp_CyUMf4E%ExtR>WvRgCR#YGm-NkJBNQsk*9Cds9}b@Q3K9Bd
z5x2A$3h8_1$|bb-?g9+E)B;ztYn~=t(AW2wteTdpBCY#Sj%nT6M0F@Uq_D3)$L8oX
z%u|Mhe135Swnbi#8{rO>ZW8ffG1Rg9$PUU~!O*i9Jdsfha8^0VtGI^lttl~O$y`^9
zo%IdEJaG~Iv(l=>yROo7{y!e{=k94)D<z}kbOl4&1cPC<cbpa~!)&Iyh&xyZdS$K#
z8q?QlF^0+{9s0bBXo%0}F~f&suG4*leqpZrGLtI2jXFa8IO4+nIAIB|ajE1yNg2FE
z+(^>qk`cK@U8~F;6eKSrpEh2gR+I{o=0KrBEdD-hcgpS8EIgyYRB4kVcJ(`uc&jY>
z=I^5oGzRg(q8aPkJL%ch>??t-5ubHeOb&FpAvE{1r!|YZBFt43=Ly0RRIug%-+Lqm
zs?6j~Z(|oT{n=OJi|gyv9ap>V74!~IAv8&hW9G^8_i!BeCws7mKg3e^#S_HNJBYv5
zwfVA}m@r;WAGUbl@m>iz9F?W$YfW7Rzy|HZ?EwOd6V?S;ua`!{eOUnP)R%OYggky6
zir)mnH?mw5R9MvO0KLVAUahtiel>?)<S{thV6LJpA&#q*W+uM^0s7`L38F>q+q;UW
zRZ@@7<3Bb2T&gYg+Cs4J-u#UK7^v)WgLRO3tRjAr6y7!tBN;@Uj_Ja^Mh=0DDnUmx
zmAv)^{41m$Km>xY6oN7MCfKWM3l-_#>TFj<e<l6UY%`SUi!|_s4&#!iF(4qUUe@tT
zoIKl57fBRUq^<f~(s#N%O9`ARpJ06#BvxeqTfen?kAhnoc;Zj2XDh7nSM*0iUFY!P
zfS)v(q^_HIpYn2?$sS1!QO_$tVTS}{fCQI_?c`<u`b`<;j0b(=PF6g!`Ku<BL(d-V
z34)=#zWlF*NK#S&wIlZxK;)Y;{aCH9_M?r@{{p_qaCf1u1n?SA3tS0XkS)(9F@@|3
zueI~G;0U$=Cj;=OyKP}WbW!%P?bAf0T0>2BhqBK=C{i%o==+<*wcJU13KD-`kZp2m
zRxa2iuy=A<71VxJ*eiO848H`SsxB|J%WnGL#^08@nh6m=5psjD1pe9v`tS87CqCoF
zH(s8%fsNU_d?s6X#T5ym3pa7!l<~jd`CPP)l;rt9DA?JbnxKjytk9(k#%xniN;laj
zA%t%t+X%&-tAMS3$lLg2+vTN5i&K*<Kh1dT;J}$5C0y6h%K^+7!D9gt#FNBV^sE-c
zLZC`MS9Ls5S5ul^sH6CYgfNa<0Z?R*f(0V~B_IfTO;_Kh5Puur2?}}lC%Df9V|6f^
zyhYyz2`OAXn1b!=MvY0Sz~mO@SG=hZ?s@#t4hrIjRp3Q{EenCuRljFIg|xzjv?m=h
z%r-qPE^XJB5-#F^Y5ojO-$U2cT(Eh!DBc@*|BKdJiv)kD*~UdkMi9TFHU*rFrZHA5
zH&N?8`OJ$5@`8f4NPFmXvyP+D3%u|o9o2c-fJ*QfR)OWfv~E8BnXqwzUh<o&;TJTx
zN4SF!W-CI26E-nHiUX?YFQr=J|3|7T@Dhxg_kAm=69ehOeD@Tt;w4D@g7vgGHu3n+
z9F=}G7EO+4k+y1;q&j##)%BSyb^o9n_&MjxLN=w9Nj&G}G%sFM`f);P2W3q7xmpj%
z$3L7ZY`5Mqtr=vfn=1`-@wzZ?8^O>vnianzsfH~`N?fd9QE47)&89!sMq-9{q&3XD
z4vUs!C{xTiPykrJf&{i0;q#MipYC-&%h}t+Dw<4f{4`j?QQk>Qg;^@a%3C5HEWW4I
z+nfIl<S8%We0YEqn6_%Qq)J*G0ms(_O<k<F>%w%v_c9F70a($06~MmT&k6e<x(j&W
z{!;_1_eof0y#~N74H^I^F%7UGE-$uqzSzY7>Ih}p+~T+xHtCGlRY|q%kIdH9nwla8
z24yYG)qOul3HmUXJP$yrdb8DD>O+AhM^hT)mC^II(UYTVi7{5NLfl>~h4>C4<^a0O
z=j(gc*MAEj=ubBwR_1z)@)HKB`AzV+WXYhB?_7W~DJq0nVustESgl%smG=&o@jLk}
z#9Ip;0AIb|RQvljD10heB7T);hr>8qOHAQ$Df9Mm4<S{K2b@_JOmji{^gCRM^c}Ag
z5P>go8tPE)x^oDM8~YorlCY!v3J4W(^2(TZCws@I3s~H1pgvZqS&36WGM@$W4nAwC
zqs?`QP<amnkydBz1R$l9Jom|<1i!9Un7>z!<W@E}f?IJ>utr3`pLPgW24tuOZ{;W=
z3slD_a|cxkOTDTwNY|cH1c^RqcD>&oAU+gy`WsQ;EqR;Bd4l+eAnk)?mGRWzad=6+
zu*%IMO~!)Gx8Pfd<nbQ20dc7z#@0WFgxmw@x5r5kuYJZ=JKe0Jxa~F79QXqrd&#WE
zlgA9f4*)DLz||S&xS-d1!K7ZW@A;zUfMyE-wOVfUsF2+M`5d`_Ry(B{@}C71)Y8z5
zfmli3IpEFHm0^_uX2IpxF@i+-I15x=5@HZiFreCg)=U)+4nBZ+Fv`H=j3!o+0nC*Q
z_}pc{G&b}t1C$?8GRk}^^MdWqIfw$9F@Rjv4mI!zk6DhK5c-)7V_${Fuvsu3nFB+5
zQ6vq+D=iMMGxaTST>KyYpSEw0Yr1Uz$LuTBlVGKJ;FOOQsU?;NkfUWucz`@Xf<R`9
zfH08b7%+QC$pcW5k|Lm?BI3YQ7%;{}6cU5UVIYh%!Z@D>Y;0`%UH4}LJ{t!S{P{00
zU!QCDeVwoO`??WcdaQX2gE1zYfsRz=2rCG{-kc0rJC6sJOKtSw{<W6CHIUqI%@}hV
z<}pNG7Ie8a1Gl~!A@G1j`lts7^2X4ITJ1IIl5+`b7Ai#bc(s0IRl*~{?&DxN<DoVu
zt;8py87#GZ7b<bR6Ps^C1Tkr^(i{4V6@OYmD@r<H;=%0D_>}yCO47O)pa#bUD2=pf
zCII=%{N>WMfh&S*?vkAOeXY+Gufxf~0LTs*26{vXbvx((O|B9v+&wAFF|-zLEH0-K
z^6*jIZJ0GU?8^%ga>XH}n0xvGijJZJjfUsC`%>N<$4N{`x4;vS#)ofW?a1K>#`Gn+
zRxzmKe0>jmT}>DiY_cS`Y|D%A?b9KiNHt=$ch1VS6*f`#6}@%G4Ij?)g@7O>e;k^0
zZq?)t0?&0A!AEFMjG@K(2gm1UJ{$U{fKo@;8}&d@i2Nj7lvUZUd?CJmToTmej2Bt3
zxWAY!j1)x%Gmd(xEK{(_b<}DrI`=fRlo|TEnU>?UMey!PUKgiIGmu&4k(>qPX+(*E
zR)Dt20k8MRNRkvKinV~GN+(+FrH%u`bwlP<{N&`sz&%{l&|KGWsw!O)+NRqXP||9+
zSs|BjvJ@F;lRr_peeVAU{l($i2-L58`@)WE&{)qMqYjR|+UL+$5AF(EcH|l`g(n5K
zo0eVtF9v0m;_FeC0K~irf#o>QaB}vc-EB1kgty7O)@_O>u!7O%qm_|v@_i<+Qpsk#
zuEYe+Vd}&WnP&Pf?)Q}>tMTlp`VorWDK8OPg%x@aEN{NpKbm0-u$SRt$wFq~7f$Pj
z6000?lWe2@RlJOYD1&H%41!P)a{C<YbDx|Hg@DZvoq;8ZVcemn*se204*?LO{hU$}
z!B5)AZaJe6wswO~b`9xxt7?jm>pip_rvj6=QiShCoVb@ik>^*wh`!-q#aW|Z6+8E0
zq_E&;TVRpHt7js~T!eW}IPYg?-A=lz*H?=E;ap*7ZJiplu^Gm?|6h;b;r=i3dK`QR
z)xt*!yOt>U%a+Fj|9YsJBto^rNn4uo?l?8*Fu)k&2?i-L!zRbHVZ&9KaeT12aqxUq
zq^ejeA$+VDP;aT!FR~dgzn8Plm*jrn;uchYU-c8_5|YfF6x8vm_AW_5Pk&WbuX6nd
zHfAr%9;Pn>@s~6IgG4Sj%)hYpo}!z&Nd>~B|DLXb$NSZPS~*U&wMmXdtiVs33dH!J
z)LEYiE1XZ>&nvCdv5k7B=p_1C05DyoT)ySqm$ZwifbOMjk(03p);<mq@-QOYt%4(p
zx6#_xp<^5i8p%xiGUe<j)_Q#As#GI^xoU+mR{`LD{Y8lPC(^FSBDL7x1=LPe%eeK5
z-l4xAy0_n{A6)s(*rIprHM}GdN0w4w`Fu%RjB)Ry1P?U@ql{1Y?z-gvCB%ye&gSwh
zIF#|YB<AT5Fl43P4`-T$Ps*ylNKetHDF&>>OlCQ``*Ts&QKIV~l=)-Z*eY_Q`VqkS
z)X6|!NV`J8j}C`q*LCI-L>m&gif7T{)He6LZvAST$&>o&GGU~SdhI8h+3l5j88m&y
zm{Uj;i{ZJPpvIbzh9ef%6RU$F?*3(u;cV{1!!ghfk{scD#;dulpFKvB>%f$xK6Fpf
zhW8*Q3f<Eu#0<~E-;K>zk+gT-<=N5nA1TU^k^nWvX%w^!bPEQqP7*B?=qNA24U$4O
zwlwVbd{~xL;I8N$tg3fWwfujQ)IQ&@ewcJt&RM@AA_AIn0f<I#$!4FS7?`}0g{_~O
zD!2&0gB>cGf#1lBAcsVUbraL)LJW(rXftmpL~PT^vy9uIAVJX+X9tOG4B!$Zd1*hV
zXfGet3OpkK?K+q~KS_ESr#9{PEGfv(=Ad>dxvBMQ#nVXl2cCaY=rR|JEu$6!eelQG
zht)YtWgL42-NcpuFCxrI!83oDK4{Kxu+r97Jcvd=zwYjcewePfxnAus!*%HJuza-Z
zKtU^~&J00wXxI}_zk1QoGk~i|7}fb4a)0aYF$a+f`E@Qc<et&;QHS~JXiCFIIwvT`
ztwG$gOZS8pO}|XhN{u~fLBWJ(RdeHcxfQLr9T<3$3ZfJPMi)<Ed<m)q;6z}b$??xu
zf9t9twnzxqPzYhmdFixVEtb}&5u6hh(Z7tAJZP6n@CaRLVx-Ds)b3%R2&ne{*Wdmt
zJ2&trs5W$OnfoX@3oamc2%!gDyl}H{yiUx7CLmKvK%EWtxThwY(1h*uvc!ThGTe<$
z<8~J#79q^dTi*w020ZbCHj?0PV<wa#8Kw&==icy~I?HH@fc0jW!bx~qN^0w>^hzzM
zGUAK^b8Iru&=ju%;RV*J_&9NL=sMp=^(ZEmS_AuOBvrB_;r0;C{WirMV{R-OWo`_U
z=w`xQk%bQEt3ThLx`rp&EcuK_O7PHBP);v;^$&wyBSPTlBd=D7t6!#a1-hyhmK;L`
zm(_Yyl;mfXgta7ou!&MciZ6Nv#D!6<IGbB*{0B^9$z`p#6#bPg1^RWi>MjXe)d3cb
z{Nwx5eM?gUJx;RxU>Hq?wjM!F-up)7|A_^ilx+KS{SF0xwX>|E%3j=CmGxNlQC*5Y
zMl5^9L=u_b1VrSAp$ywn-NXWtf`ODI3t&sk{9tzLJY=CLa({z0m|z5O>C}N2;Obt&
zMDzsSL~5r*2&~_lJNi9}XJLggDIoQ)O`GT}Dp;hNWnIwp&y1roe0{oV`PCH#)kB?I
zjRak^!WskdotP~INi#)Fvc_IiB#8<Wa~En2I-7{Y(#S(%!&hTqzhqIZn&@RUeYLEF
zU0R9;cBMM1(H-p#8mm<|splRkYZ95U<Y*w;F|;!Wdk(W3dQzZU$|U!dwSki`>91gG
zW*q7d7d&h;(v|S5L<{YN{4N(xT7wS1=kGLju8)9ZO3tAA40W1jGN?X%S=@#nJlWHu
zJ>qmjg--tw8p}1e{oemWUO#rSY2I!_*m{d`*e#j?1{|T5$Rkbj7`shFJ;1#yjmWrK
zM@{JL`E6g<Kw%q+8pgOsCW+6^N{iJpHi5lLGm5*IMg7Arb**ZHEnM=Fo-JvDerqg(
zypEN$q-kDdfw;%Szpu=sznL*G*MO93Uud8w*ke&;Jjk>%IceSB#||ZbNst_8vdaz9
z%V`09xqc*59pm0jCdA!myr)U+9QDk<?58GhZIj4P0tlBByrN4YnZx%((B<*ehCEbV
zE&g$!*(CGsUGmWXiRcX$)}o=71t~Pi17{L5pV3@sLbb1=1^8F84Fq-EpaW}jXRHyS
z>9&6h?dg6G!sSnSr<%00Kc1{!W%aYe#6y7=1PQ;*Psr6tA4pM7+AAuFuh&%V>UY&5
zx5P36H-TR8nh-W(dQ8#L^uz3c>p6Zc)^_&0qbO%&1^9{Eu-YZqW0@=L`I6%*Uj6xA
z!C+>UsG%p;gi7_{X^JbD8~!A>w{o^yq{b<8E!w-7lD6vVp#(otu8v`^h<N#ow!`#@
z&7<+?-Qdcb_xP7|$?V))(Jk`ldbZ@epU?SR_kSpFDNsB0&X>+Qt!mgCMFW-n8{N5;
z$w8-Y%c{MH(yKdS>DASRr^EcawZqQsNSYr1Tvl@dP}~f3usVcOlkRs`@jTW11hr8N
z)Gw{A{LN;8LgaSTqg)#Zm28sK@@YQ2u*~b=qST_(gxw96Ci|Uy%LbZa?s$;IABBm%
z#IYiJEaRf-zc+Gbtfg;LSxsWq_R3VrN=plO1*>xM69qNW^_sj{Agpu?9mXeEw8J@u
zA!(MB61ue18G|kK!DN=jdFx7yvg9Ld?x%}lsRIL{_RSp{q6RM7tTE$rUE-2ZPRZ_Z
zp!bWJ)<#(hVIw}?)<6%k5f7TvNb!U*FaUq_NGMd!&5jZ^==d7GpCX9gX$n-ptOVp(
z`+jeL%U3Q4yVR5SFK#2tOP4dqtv>jk{oWmqhHh$7bfuzdmH_#hn)@X+VPs~2^|$M9
zSZ8a9f?(4ey^3whXhbUz6JoSmv{@wfe;Bu^y5s|3>pp!0q2kAbg=<z#U_eu#IAQzx
zAcs*+Hu|!LgaJjOD-@-?=clYHi2>|TOpKZ*ZHxR#!satpJ-Mq(L)Aw$R7{I9cw%8E
z`qV7qCRI|q#*Z47nL#D5ZG#Kva+e+cX;L`LgbPJvwKAIVyI-cfPX4Js<74@ozIq)h
z!>*`t#Jjbw=ro<3YwsuO46;fJJav`g*IFJPR$Xo&u%(L{80R55TQ|PLlwt_RT~~;w
zzCfAuZg6>?dz&4(|JE3}N@~dbZIFG{dBnO}eKg!rk*+(ulKk<Ynn)B|gTbO$7QgK_
zqthGgt7;4c6`+7M8%zM@HRB`O{~N{dv~-rcVY`AUZ@+=f5Zk>MjI_NlQ&!)%vL>sC
z;oHZPbkL|_flrDFMgn3&jAs1#Xw833+&Ke7J_EeaPzdCb&Y>TyBPK8tDMD!CC0e6V
z{Nd**UsR=&5A>f=Ji;QHY^R!*p6(by7ul_>Aqh!b8lBUm#SlFHJ^C<BsFRUGCHJkP
z(@|;^UFtpaBm9MOq>I#A-eZ~+%~UXt-A8Fb;Wb*hRVnb;mUQqZHDbQ?V{>m_XkbUH
z!{!x<(osy>;KKfa&`iz{Ec%%sVS9-qo0U@EPtXweu<l+^5Nb<DMfK1w0br^u)qUBU
zhg)b^{c&>kk&gtqqM!ysdxwUkm1S|J<X7)#Qr&+vMQQXhNY642f%7ownN3Rf87|QH
z)@v#yg;9~g_NjiW6r<kB4i#v^-)FB^(7PX3=JSjMBvAk@s3D4Fak}Im0qJfVGT<_{
zfnV3)@5$FMj=u|J1a~w!Q4Os1?B=p14m|=oK~kO&_l3M8`S4uH8m7_EudvyXne#i<
zN`9yfN_tdXU2@Ho9N_98X)d-jS-Ep%I?Ch4)kCgOB9XYT3V2dmcoCrwkq%EDa1OcW
z!5xfD&>kyZXH+96cZ_7ee3|l9-#{3YE^d8~q>y#y$&$Q8wD!h(z>a^ay1IwQnU!*O
zQmvbcaE~`R&mG8&i`F3bwGy`}_>*fTT894ff~)OLjyC8JeJHp*IwQ8Ly+bFE4WXJA
zSla6Fb?es_jQ(a$N4p038$wsJ&LyoMR~C++xr_iVCn0&Ewlp-1fGhJFj}h4asiGTs
zx5CJmgH|(2a?`F*t>kxrUEQCe?d&7r(=-RWNCZt$Ttdve_;6bin`T#BV<N}{Z)|U9
zl4p5p{DfsHq@x3{y#qivj^9((iCLZb**g1-PAceLi&+`0rRu4C%3i53qT9$0zw7Hm
z=;DVY(MUr(5@J-$!Yb&4wH)3+7y;!0y@SxOcN{;Fo*D-S?PuKu{Q3yBU{xPNU>kF<
zQScb~ygEAJM6^CNadU0&Xq1QFI9~2JIVgstCBl{=tWU5sQ9JupSy#1iF}UWi;QB!R
z(Dyb~`1Nk1&GIQS#}Ewt_~*&zE7FKvK9uMz1*g|{wH3Bb>Z@eZ)kks^j4c&<&*)9b
zKdUeG^$Jg3NoAXMJpqj51pC#d;zpq10sD;q2rjX1H?{L=s5YfU-&c@jw~uu3;ir-a
zk064M>97aJc2}O<<l-e1LN;8(?Tc!#zb%OWt?#Xhw1B>?P!*AHK>NXh8J4=|;LNY{
zJ7;>74O3FhDvnSPur(3Th3tlA_CfiX8+3H<KJw|~g_=0_zqhpIEkS!wtbGaFAkaq>
za=+<E1=J4)@K3L=wW<GQUQ=OiQZ!vbq}>jXV?M7OBhuPZ%eY6`)#CcomL_x0CZ_6U
z)MZ(;b>%Z|!AUt3sM2hYPv0!jQXMc$_q7z~O!xlh3Jb`urnlDY9m<0dOqNGBAwu3N
zi+-@N4HY{{28QynU7Lx4OFrE5tg|+Y+0~)d0dbz8qe`d0JL5B}|K3{JNCJ$_6Aah>
zN<|6vk<u@PCZs%q`)Xl|wZEGWxkqU4!{f(Vn!I_|sU3B}t}}f4a=EqaC!bH2^-fN4
zhA9VN>6XvE=2b3QXvN&85Cv|gI|p?MegXIkA$L}43~qu4>+4TuyRe`rtNWd4D^mZ~
z?|<O`=~v()Z8^a;s}9Uq<(GUL>(kb~)yiUwP<Nn5pQES5+l3qF0l0d9D&^>DUf;k%
zLV&26Drsj~{CxE_=`Q0gzAJ$sW^k6K{xn0mK6ta?@X+^?CQ(=AU~hH|&97eJ$ad}l
z=yzEC8<oNGFqtA>n@0nosK-MO?0@fr`1>fMbRkF%pI5H4bDEJ<kZ3$^IDDzK#!tM`
z%ECrLpqqbHi5hia?Te(da*R|Dd~K4&t)1c_pUk?fpqLcFo4@ULOQN|C?ST5lfds8o
z&d};h?x}g>ZBCp@Fb4!EO4LjXedC1^ZfAa7Tw-*Jg1hi!K@JJaZGk5z_dEkEOlYhz
z?lmPTtC~3Xw?vd&76`cxs1(iD+c$T-J%&>ZkZ@pT2M&J-^Lt4yEBw5IIEi<#6d&rT
z-#rk)82?hiN$8XfnnHuIErFSNbxs+{Rs0Pd;t;mRZ3JyI^>`@b2cmq(@Ryw6KCv@_
z#=FS!7$eio>ZnJA(`e2-yeVd3F1#0@?w*j7RUhfYaOyd272|LA*$svX`H(}`;rYt2
zG4Z93XoQc27BrrO#h2}2x1#SToR24H_>(R}U04<|Tm@rQ^e^vFoiZ3cLjG2(7(cVm
zZuyV~N})j*#5qymJ2*p8vK{snDEq|Fc@?ZRif5BT;Z+Vb0Ipk^Irjk_Ui`tmEOBy^
zs})Q^XhKqM%soNLRwwte>VaA>5q(gy#FUwOX}ra(lO_%$JTbIFv%Yz;4lI_bd))fw
zRt2kL1ESV&p=n*I{CzDtO7oNa<RfV&`?hvIHcc|-p4ESvOdpJclpk7xx)LfUhslqB
zgn)z!drHT(wuo!!JqFlg4Wnx*a0^o<-<&4{WW<Soz<O`Yam7Q+H{K)G%~{^sLep%M
zO{$>DQFF_<>#Y9wWP<o0WHyX<gP-c@Oii_wi_*t19nsHtcx8><-yFt)eow`iXH}6L
z)KK<le`>D$IYFb(S~-`OSQ7y2cR98@gP~>*59@?fVGYu`nTd1t=QSxThI^NwwPlOK
zwIyAJSjSTfCVe#eCdVQV+E+ATu1Q~W64R(Zhf3zR7OyhXO1Vq!^YDR+j83NyGB)U*
z$q8$m+O=)8p5WSCbI!Y}iy^wy1*r;es|qd5?GqJ@ukVgu>p)Ogd(B?)o(4A^E8Jr;
zu2#)SeSkdbX}!D3jMLhYU&H^06j-EZT{cB^nn<qIu&CDcQVXN8TQ@QKmZCsMTj!o<
zWa(8tkzp4Ah**D_&Dx+Bcm?^QezS<Hs(U2^?H(j5hrA;6n%DM`#n`wu>@Wijd_JCm
zrpTK`+LZCr(=!R&v+k+q2K#9~_Q`&C6fUvoU>nfC?z*Q&EE{>!<Cia~D@)iW;`BII
z6F$=_#=|WAHq%&7@C}{5u=*X-;`IMcrc*DVN+MsN68~V(Jyna!Jr7hxzr-;j0T3zC
z0``uPnR=B6rQ1<4)M~5uH-X)SF+q_D7HJ>SZ~6I19+K1V>PoaoB(9c`E;PMHc39(J
zQF5#(98vD|;TKq>NK&a_UoAR8I{C<<I-tAQin&KYvh*9(S=l=4RDAZ1zKaB-t#jlO
zl54enK+Z=k3#fNZgfL+n5^*+{RIGT3bAliudl^RtrUWGL(mnsgx|uvK2qD?u_Peei
zOwl(GTHb4Li}-!yWuE#Q>~qZ>TI8-|s*!Jg>wBxrY^c~+kHqCgt08XuIJ>n+TsWex
zK~ZqPZiW1#NmAfB>O%+bUV0`g;EfuZ-(QNtSRll;PP>NT&=tdxrOH?nq$)9|kZR<?
zsf$Z8t~=`qfWJ^5KdSwo(w$jj_ZyG270{w><guHs$v<KxBg;UlboAQ=z4I!AOBh7H
zjg`f21(^ijbyV*a^N4bJQb@XONR{YOnSX&eC~R*)=0`1g>?%@sJR~MH?<%9mZP1e`
zif<Xo)?-OkTkcf^95Nn7lTA10&8c1G$?dk4&h)icD0CC9!SF+f9;6u6AXS=60+l?J
zSw(LUVO5?@K>A1VckCFvf*3kz_iQ?wD_I#zSvRROnCQu?(c3dI-jcItYXc#IS3`-8
zQ*cf9%8P@10)B+@SQQp96Gxx@!i4LaP0#94vx;#syGe-xCr}Eb;os2%Trae@iCKu?
zoz!<8HxvvU9eNz+$`CEHve>0?gMN+%LT&rsf(!r29(l{7(uH(dtJ6rgE6KS0>~4~Q
z@Gcad`LMavm$H01Pm)O)i71ELaP=)>cgI0?ntRuOESWpU$O7}f?jiiy%!L~K5y_jU
z8Q9wD@tK*$ttgHiza5M}m>B}p!O-zu#R5&5>{Tqh?Z2^!V8gFwg)3kv&X&Tn57}XD
z09dUabibFlb>wUupLMKIH}dx*1iJL6oxiGW2WiPlaE&Hp*Iy{R$0v+V{x+4IXVfIJ
z^9n*Z<l<H{#Z(}y4Gh2D5Yh8TGzU)S=b%SHrZpU<b->M!;{Qynik94|77tlGLVN`L
zTU{gM(Fx!Fz>!B~SGB*gV+A~9XZ*d9se*~MxQl|q33(V#fB4O^2e;YhzCYv5v-j<2
zUGtgQqyA<CRvV~|6q7$LmG&i0W|24TE3L{jFaBTx=M#BpuDbF{svsMK7H8KGdW&!5
z^{}-t)(OpY;S}_STg5n<tw^b&FAQ|-8u$-a@9DsYhz~vTmf3X9hgi~alYXFozCspR
z(g7g+B<3bZTl9*b^g*|x+w28;-}`e>6URQ_uaDySXyr|FRW#XD-HG(BoK?XyEiXf1
z!(8HTxs{34HY$>Xk{o(lLY(3Wbs|Dtmf`u*b)(AjHqG{AY!X?T_cPCy{0Ua(??mou
z9@Kkg8HaoN?`3J8;M$-6w&Lnis*~t<)*1!V+UWyURpSx(DaCm-rCs99wnGpU%b0^^
zPCsE&hHA0@`xwM++DuE7n<4L4Ye}K12i8_Z!V0ARw-xV-?iX~B0kw4)2bL<Jjr6i7
z*W9Tt9i2LvO|VXxoYPwTsaZ$|w(yX^WUND*6LFpgz3hXwRlw7bFJWAc8I+F=bfYEo
z1nn`m#nQS|nHTG5JDPU%rm`3CW1$<^{d)rQ&WGzVG0aOxepif|djg_JPn!WLji{U1
zPW|UO3<5k!7DtJ`tPr`<_eg#id@xveFY((EPw4*4#Tn``lYtFFa{U3zn4ip?wi^gd
zV^TbX#~sztIDh(FNLik^<Ou0C0bxT1-7iofQnZ?5Jbsc}?<S%lrDIf+G=10=fNrSv
z;D*@;oc$=?%P7FD@3bG4lj{zi8|xZDgK_vhrkB69{Wm(^6vV^~p7QapY#o&MLm#3w
z3)+kgVR6Z}BX!W5*{)U%Y<oyFEcOP3eiWbFKcH#IqKO{F{5qS!#x|}aDPHCcJ^AG9
zTXwh?w|jd+oVL6)S6xd0`qMDjb1!8ye8HuW03X#CV<6D6jQP>bDFCjFCSseC^#M`%
z?hLOi(0}cjg!4J&dEK4T7Y3uF6%?t$9@sEijqfC<Y#C9c{m(kKd|J&#DEwr>+go=!
zw03EbcS3(Oo0C&*mx$%l;16HwdAre!4DxB!y9*9*J`0Xv_<1X+k&{oEN~nE=DU=Z-
zmJeGD6K?x5rRn{ePtBrWe0RKTAZ78%(^3K-?y{y#DV1gRZ{In0U2<Mnlje7Hj3o3R
zALB?wCdNy$vPaUPp)R_~(*FLezKx{zxWoz$rrM){u<)dUG?1%L`koVx;5uAq{Jo&H
zMh3+)SCFTJ0%@uS5w`G++LpRE?Jdb=29+O@z)bqkD^uLDFwh5P)Ao;`tBU3HP}&SX
zgZKiHC5{n4?57`E-U9j%xuZ2rUXeeHk23NSR%w-kI!gv&O(VRQAbWKV*>nbjO7$&n
z<*qcdX4M!hc?jcLMKy+Uc&bcGY3%xCR>~YciDvKKm7Ks)oc9hSgWIuhguP61g50d!
z8{qemnNCZ*Zl)+(NYe3lQ9o`Y)0ydzKBc0-p0pFbV#+vTK#(wuKpW&o6z1U$>dHdY
zRA6QZYoQdRpS>D_AXaz9)rH)gy5^LZd4x?aFNFXT8l@IY?!3UM?s;ddFy1T^U;YQG
zu<3Qg@jV8|cZEvT^Kbn@{C5H~@xzu)C2Vr{ViA3z`I+p(?Jx5LmXwVvQ<T-IrJph-
zsZnZ7$#vE)Mf?Y;f-bD(v}?kinJ{>&Q9&G^z5-{eJ^x>pgVkBjFQFh!swt0@eAM9P
z`Q+A1Jc}mhUO@97x9C+KKyd6#ze5U!<YYn5x)YPjy0~iOnx2D)`I}!SjZsHGGK;{*
zXNiEmBWn)7>pBx*%JnKM5J$CywMOxD3;?T;8iQ)^5oxZ1(r_(c`U6FW$4WE53osH2
zAa)H5)x>DQikz2yF>cJ!Ko{y<(q5y_6^t6*srrF<*0l8EsOLrMTg*1e{9916<*eO)
zD9MuyOkgw0%Zq>(SR=d(l$*#G)UU3r5GFEO)Bqm+qvHK|TT~ehUVy^0#=r!2V2<X;
zVbFD|78go5Yr2f#;bJD`!<Gf!9WJVN#iY*T?wUWY+=2BlUY;*1%}Y^SbD!^;Z6vW!
zfN$<JfCp$R@;5neAE_URy|>w{?+yJ01wA_72RMCXX5;Nb2&DLs#8sT!XIsMIx1tA;
zr}}=?D1I3FUsw*3Q(Mqh<1e-_frhf=U)FnyisOO3(QKGfx>kBXh$sksp5KESv|1os
z7qzZgX_o#MLVMkXzN(fUg0x`whU=+Lor9wy#l{-W;a%A$%p)<p`!f_0iIPf0g84%k
z_KQh183rN0zyr%%CZI(FpR~RQ;?(;w^9hE%<VHj}Md1-%L!IgyXK1^#4EO{6CiI!G
z`=URpdxxl$XvGP}MF`B#bsAAMDUD1p{BSSnXl>V+YZ+r@w)2~gN@GDb5%`|SgW=zj
zf6-fSql6DAs8I6yCbX8gVLpUV=&Cy^SX4Zb-r3n>Dlf<WryXzg4}^tTAwiB~_!&rs
z$sI%R{_k%y)tYyMYwS*tbS*6ur%m|@y5EaW7-p2dKB_HXPEn8SD-jq4JC#Q;pBJ5~
z?ieH$sHU)Hem`?F>4u_6rlh6Krz_K`|DK}4-11Y_mh94m1xM1D0Gwa5AXlqaQz6Ne
zLF{)t_aSSyByC~l7phd&(iv-uY9{?%RNZ+cI4D${R}Vqp&M;7gEk|>MM3VAB$q$ct
z^u(@^Q%zt0y7T=dv9%xNoZ5Z6dmpbnfuwTrNW%Q3;s^VdhcIeC{)csH_1(Jmy(a7Z
zOw+7BJ9)TPFuT7!S?J03v`D(Zat=NA)*;g!ssGFQ<iB=HHJ4e1EdBM=v3ER}M#5VJ
z(m+dp#a+McNnJk1F<rc#8-ArjZ0GIyBr)HUe$|TyyD!gKLD|OYR8L}LY)DY@JmSee
z?B$gK)zuq+{$b6sRbO8$2Hg4b<cv%`&Fg{G?^@uU#uRw3Y^PJ=cJ~imZlFJ|Ys=Kz
zeErTvr^E1cJ@!8@-UxwV>-NSgR0kJ-UcX6^;qqBZ!~aSec4_(NhN;Eu^Y_u*zcP94
z+aFVC@Jw5LM(<@d#oJT-lc%K3`0ouACC~QS?jELwhLPYj@gGXatC8t<s*%t@Cv&KR
zc_jMWuuMuyt=*mBT%V?QQg%Y+RFp2PpLZc|<(agqqduWbNt}Ucn8{r@AVcj2VHi9@
z*j;Es*KhJ>V`p=drg!AJ2s;#FfV<%~k?WPB(+w^z5u5K{E!h`Y|82#0*dI#++|O(~
zxQ_J{-gL+Q=eiwgFc!Obs=<|7#IJ5+BolYOHX_lD>?o_4p@qa>DG!TQ|9P)7Q}65R
zy?H^d)T(BF59wm~TlYQI>-+W5xHs4?=g^mKcGlm%FaQs^nZ-Bu!_(9KwBuWJLa9)f
z&I;e@;Q|XbH+_cwxR(bH{+?s)-n0_lp1@@M+GW0SnbQhYTGfvW^F7`j5n-c7PLiGf
zUd;n5DR4>j@^m(KDq(3=e%hQk9YbG97{Ppn60`iY{m{A?Pd9$E8Pwv`wXc1A&C+XO
zaQ;2A<LleDe3qmW8XRBq*=|30d|0X;&b!pj)SS>C^V5Nf8Gq~g5^G=&(9r+<*eO;o
z)pD2n(+*36R3Cv(F3;TFDf*8Qp_KA;1N$HE)%<B?{Mxrhg?c+J`Ds)Gmq7E73|-z3
zr!>F5ps<0dUS%Sr5ubPL#V%lvkNdN2LhL8^3hI6T6jXh64W4AKScQ%yZ9OdiyaKN?
zUoE*MtY35?&*|os1-Z3SW&dmasUYWd=aj3gTRNZf)-ugT#c5QFF&wN;2s-i+&@!*;
z+W6S0|92O$@pJ0>B^o9IR{rw;ySzF!n?>q<%DxNv$t$?0GpWhtRb`UGt*V;-+qjor
zACcv|#KIndNkQz9Yp~m4RFtLE^Mdz~92<4~9}7+Pd-|pn1QB$>{hGz^sd}`3O@}>F
zktc2N6l<%`f6m#d6P&TDXsVcRT6Dr1{qb%mfFD;R#GkN)Gz&YglNFz+B(ioda%;OB
zL`#xfZ`$zYh#;5u`Gm#{-7);v@+-&r+x+f&QKhcN-T7K!&mb?X(?syF>5tdGyRp!o
z$S)|Yi`7cg7o<2Ng`l*qQi*hwV76{&udcd_VRNIe-q%KIFlE>E&aN*(f4t}g4`=V%
zyM5g~Scc(^{^$0R&5-zA@Ab7=7-AnN66Np0X>wXl?n3*Ex#iJ5c!4yJI#XhE8P_(%
zD$RHG<ny0!`E0LOV$*rqE*b;N;JmHKUvdAyUD0)eT#M=m<%(zrydaC*oT-h?S?$_u
z9p)c<ppn6OOkS<;-^WQ*xo<>yHTHCy!p-5CGg66LMt#gD#m&zj>vDci;p|!Rf8o_f
zs$ATE-PX83xw4|YD=f6YBzD6UrQP+l-GxbN8peXGgiUVouS6K)t&R=-aTgsv9MPC#
zy&2rB!Q3#5X#XzOSb3~F3VH-SYqjSe%Trf37UUVe!7F_w;ouQ`tE{Sw%r97IQB!5O
zY&1;$(OAnm?bCQAg|h>9Hs|1MI(o`#i2WiHIy#*uEJSG;3rY>&<{f$^8vJ0R%{x)z
zfu_YcgZebKp+BkMfQxMe&a$udq?8JOFBF+JH2J^wU3_6(YuS?E;=mc3P|<3erBwkU
zgGg=M1>HGgzABKUd8F+Q-tC4`y$Y??$Gvy#6uqxDBy6#UQM6ic@!jN^#>BV4iFxt*
z7V0v;9nle`#zj#X<g=!1*b4kVGeSyd3JQs*z|#8o1~mbF&5xQqU;8etr>^zx=HRf*
zGQCT#r85qSXZ}9->`8b{vEfpjY8>$FeIChmvsrj;$Lf~+(yI4^|JiNlm`aK4ymU!R
z*WtZm(TlXA1HRD&$C;20xV#DmGMn>d3v;tey53(w*Q+kYTvmSa^3bbulhGd!RRNf8
zx8<(66nFs%3)$}_?@-^D5Wo7N<>3ZW61lDUKMp4B({EKPDM|XwzW#Fe72;PoOU~v$
ze;l??!?o=%{Pn0(y-IxGX8JVp%U{2&ZO1=5sC`C-nJH<V_vgK;86S@7-{&g%)h%1|
z-y3zc_qjeCz9{BwE;}9Cnr3aTr#JnyB1x~JbT6o|BzZ%mQs2K>h2<-W^R`von}M3x
zz$ni;Z}yREx`BiJNlwvWqk0m}C@d73EDJ-q&zx}C^$_n-5;o^@xDlEeJ)fh~aIo`l
zth3Mn$99*vXxmJZT?&lyK5Ohr-XgHksf-j0<&VRQvo&VOig=%rkL7Wg(&ojwM0t6N
z=?&Rjrjz-{N=EC_CC$1`?yKk0@DaJ{)a)Ny2~&^AD?$@lJJoFy5@<=H6HS9}j2ubr
z;1Y&Ftn*}<s|!G+uFB%^q9bkT;r&{U1s^FH=^Fd>^%FnbmVE@5B^{;KANy0lo^qCy
zaw^t2d45i*Q+s1sTz>I8Bj3NzU1v-@>gV;4LK!`Tw+^^e@pP*Lj*1JK_PzGhBUY$N
z?rC=G59|H@&l4=Xa=Jd6^?5&a>gntP(=w;!Z&O$mkS29TMQEGQiN>S0;v1NwZIRj2
z`kRd!jEx;-K`*x=dQ)3?)9%++&x3SL>z|<gOU&`cOIx7)u%u?{#=8gZ;o9mumL#1R
zaYa)L{w@|n#%N%bTL}p-s(n5~wNKG^Y>$?t>J91qz4`I4|L}oIs8$VlQ-P4Ew@pNt
z`JJ!?PoIRFlo)D^x-mm79V6d{15kCjoF7#R7)`GV31xPzi+lIj&x^r2W)o3KGh+!(
zumr`t^~}v9I-$hOjgt&$$6Pe6a!nXLh~Dqm+~$IwK`Pz1ui0TeCWTf^$YxUqvQF5q
z@iF-5)@i9eZx?y$38E!{HO>PeI%x^RU+^R{ZrO-1$2vn8eHNm>4BNeNN^vf@BHGHw
ze3hMJ!G}t-*XipE1Bl;l6FHA_80+RkX7K5Y3kl3b{!nK5l~onjlo_gxUWX^abLctv
z0BQhMfy!#Fdq%n9UYfnALbA)_<5`PtEDY16>**T`X;!%pOOo(qmd{F+U*j2{?cU^$
zTn$Q*Coi)S^wKSN?cYX;3E*y3t>*uU*U`&14GpJ-#9EgaS?vk+{i)e5%;dtygAr@B
zK{E{W{Q5}T8-EH4$1zl=PgGKq_fDbiT!89MmH2+DPV~@&rt(oE0Rh6b?cKfPB*;S&
z#OBuY)+e-CIGMwh=<56NA7s^)`*7SpXUWAgnag-NN#4HEPlsA2R8bNhdAp_`>HO$f
zywz~zBhNm5W0T=)_k%-1Eg1%?XX2}8b{kOi?l*K<6tIc=(`dc$l+yD!oRFs9H}EOu
ztFRWrf)nL=%_KWNH$f;S0G*$g9itCQQJ1jnz|FhkL=e_@>Tb?=QJyhDO}7;wfsVA3
z2wKV7t!}PnZqLs(`J1?Y)N3{uK!)LCr$M4QIzqVg!OE{k`%}Kq->bDg?j0rLJic51
z0^*TvUijoM&BlG>Hj&GfC8uV)x?siixD63hSiJtRi|!5^)DU?q$Z@u_KcV+ge)wnz
z*=TCa&(eZ0(#x>b@TZxL3UPc#%8!0SQP8L{zLplM)V;&b(7!JRr8!-#wSTrczPz;t
z*Y0$DV_SvmUn$bEzj)3wqTH>7L}(fgr5d7|b>u9r?MOMA?vWwxZCd-<#<lu}1`K`u
z>-zY-qLQNb!F6%UcXJ=jqOPB~iMBL#_mbDbQzCCrP!sLZm3vnPmUU(L2zum>LufbD
zVPU*ZmC>;iOLDOEk@yGYQmN#yr=&1-Lx|G6kS>#<1+m$8B@M0oKl8@(;V*Do@&^GA
za?GjXH$LX9x4^ma4mGVP#vi#gi7QdMC+!og{m1fq?KYF5OW!E@-Yl#uk4>2U(+cV|
z6!U*_imCZ+@$<0t%x^~Awo(U)*CQl*X&xh4kKV-b&r|BHCEZc6xiRIK1VPy>LLN#(
zXDq>z&AW>2CHHt4T?ZnyapA)|#G+a<NPNAs13(U_eFzPt6(LYN2-+gU-%v7GtPveh
z#zK9!mv@&cS=WkCy8g8b<}O43YWYkfW2o9^o!mjBd-RGMV0{+u`*Z(3SKj=*C&|V9
zze?>t`AgbQ_kMCA@9@m+p|Gzo1fHwCmhXYHPg-Y&;5qL(Y5`08l@K%X;MKDfu^>Y8
zQfBOilS;S47piE9&wxVM)AhGq=hV#jFv6}Z_vH0CM{@o<q7%p9E~1C7B%mfMO7Ae2
zd2aKwypDp$0)}hBQl-j&nwtx|@@60OldF}7vxyV+$7~|8#`d`Frg{5T%>CIcez^bj
zWpS|})2Aw+i|HDDdPE4NiZ#Z>FX;NHPceEL%Z~gX&$dfk#FgCM5Uk|3%tytlOH(6S
z#7$Us(9d|h%;v?ipp-)<i~7dmbs=3*fp*+|xCjz$plwk~T+}HoKd@H)DJYwGd-9jR
ze$3$uWB<O@Vdvu6(+FN~til@4vi5OUjAO*d<a@*wqcP(!lmA>KiE`KLsFs;TXwrh3
ztjDqT6_>tJ^8GkA_O9d$)${SR8Gk$^x@C^2--5&x8mACf;Uvg~s6~aAge%M#P^Y<<
zZ~cD(B#$|>YA;}Qacs6}O=R$slE86t{hL?xYD=-6&X_{oZr9v+c+n1Csom)$Z4NI`
zW_o6;*addcF3Kt68E9z*yX)E3$WA~dX;0Ckg}M&&lm@;?Cl3gFeJ@nFyo5=@T*CS0
z(gnyf<C>n&F1S|nJKO9l3>S=Ml=lw!_dcJUQ`gs5D(x;fLi$^3GE|_<(W<%hrP8w}
zQ4M!39?w4H^fHbS`O)|jyWtT#%toX$jJ!<_eofVfqfg;1;B(_lzA}7$Kz8&+&DPr~
zlJ7lUn^kLhCe(5v@t<wfm+=5Ftt#8SX$5kJD7%%;m>&JSrlZUxu;;YB@M=^1XaZR~
z0A9e~UK7>-SZH)~qoim`IQehBe$3``V?!Z@`0q9<`z0LE_@8%QQ}vZH&)kHD*Zb4}
z)KQS8NZ0;OuCOLMc!-s1OEXX>2I8?+s{Y~Y>?xEHOCwI)zuE!4A=F?RqjKkh^F0kT
zDh&dj;%~h&(@^F5eku%%zyWVL;JjzxTK?edV>auc!q4Txh-kcOQZ5hp`Q8h@Yntzw
zKrb|WZ&cMP6BlL*l7^lc&bwZhBK`EKP0sz$<47_Q`HKE?@x3>csI9flJ)Gz6aA7Yx
z_Sue`R)Ldc!*~c4$Jdwaga-@p&L;Fi%YZOfC$PskrbO<?N2C5usNuWU_shh13&a?h
zi=6<EY#hiBSM7`mAZc!pZ&Hq%r*5qHZR@5Vio9X?F|N{|+H}Y%K8O?-Qzb6~xS^QA
zXNL2y*Cor2Vwv*-s<C5%meRfYTDSgl#O7_f6W)UKhxeCabHm9#pSSFg)8Dit@?P$?
zQFre~+N_${sr~%#)pHYI!V4M4o+aNWq~)6UyAAQpK?^T0P&N$<+@onwoIieUiUz#s
zvmW$hY}kTQ515FJ>vtabUPy0lyXb(6zHDw$kgoB^swZIEoYdtO-M4Oxx<uaPAyB%!
z2eiK{P2cx=l?`sXJenAf{Wwf|u!-9Cbywk$#ufN*w`$iyXj2E<3o?p*)-t~ybv@LE
z<JrL`pNE}rTqKj>L8`SXWJvsTO?|UACa9<|1Ie1k#r@-lTr_P$<xpC+_2`@u>(Y!>
z#53pPi&Z+K0=?xf!N3l7emC^G{V5oB`(X+Fm&t+Mt&eCIi<Olq`g(SvJBESR<EXe6
z=%^B+=`phWCO_xVDLd9Vyy)~TKreTr+FI5iHvIm2C`~AcSSNo1*OcSB{y_shSxT;V
zt2DfHVSf4SW2FHVB=Ntwp(<LUM#KMIuN>DHvB5E_LfR;!b5X1(F*f=<`0hfqm3^6<
zKZrI&rH;%A#X5-`J^cfdHV~Dbo`i9q$#20=J(I^z#?kEYb@@6~+Cm-^b523hWH*5V
zEW=twvXFnnI%Y`~x$k8D<Jrf|&p-w`WO(9w*!bnpc;n!Se)h%-(PW<*AvaP^|35e0
zOSad1Zx=%r001=_-Cv{2QBl(O;4}uCIC}aHNYeCwy&i^df6L!JniJy-Wo$0Dj`iCq
z*#u5DB;Wt%wr@P$y84CrYep0C`yGp7_6;$y{*Wy8xo@M8HH^gc{4_X#5EZTMy{j)`
zwdKg{SJr8UKEXJ%-wDn?FZ-tzu|c#r@*U!TM>DQejfU|4n!MN_TTNw}xB6;nu}Ub@
zgtoy)>1ljVT{$7Y;qa!h{%@jb`~=63YMGYW>b4qjZI&|G6K{XF83x)5Yt9Eh+D(?p
zU<fh<)*_oYT%e)k=>!$Bb`!)o1@oH(yfMKQ5P#d&xGvL2@L28&#5vk+gnt`;yh)S!
zVD%ksF0l<<qDQMzZQ3G@Xu7)>pl1}^&m_a{30$D&e|KF7v5)q5<$gUH0&X)39}5go
zLrQl2#IGr3;DGnbK5itaB7W*}c;+8px+@KpfeZ5A!)YJ<bPIFtKYz9!kyc&{a*9tp
zEgaO=jZ;yIKHXE;dd=|9^;hK+{>ktm6Zl=#-{Si|t|>KqXEX<CC-@INXXI}>a5G7E
z>Wj?Pz7CW8=+Oj+PUMj^C4cnP#dZ~BJ8qH|!PH%-(P9II-^(BU63ty2+cS9uPr_rW
z@t&{U!81b~h4sN?-Y?rR8yecsZ(gIU#AIa^#_rX$Xd^+r5ql^-3ajj4&N2xaWdi*#
zYW~y73A$u^<&HlPW?Z~p)b^4UzQM&Oilm*84NAU=DxEQ+^M;p8IdB1x#Tj66uF-We
zxDbZj7@K%EW_`{ob?$PE{&h6&4y@5n<0lh6w|ODF5~}}W9XGW`sTP=IZtzg*-iI2M
zfU!!>Oo6E9be+8%<#?a@tenRE&kcz$Du~p^B_5(PVFmVi*#}lIWVf%mP3`-d>Pvb`
zpFC*pZdXo(KJ(uYb9St$GRL}m;BsHy-raxN)<0mB&z(2&tM3Qqm~vt+<>v189uBqV
z*JqfQk~22AD}DD1G&^+nY3ypVnem0!-~V$(*KwV}R*S><LyxX9=!zoyYksgRZ8+Ka
zu41E_Milu=k9wucZ#2Z3s`lQnwny3Cy5f#MP+oDKwqUYRG&av3TW;inKW%{i_^GCo
zxI(L(?#A!dZhGg7bLFxUSM}OlYX1%F*8^}4a!t#&O*8)Wy|-c3!0&v18y{?Zy<?T_
zdGGb2<W#2bbBa!|o1YI67=T9r#E6R0sqXs1*q9B_9{)L^!6_Me{F$j<*r|A{?HBQ?
z=b-Didd)nmZ-#@-+4bG$^NF*M{RPQJz%I?b2`JZ@q26uVX5{SO)lYDV4jxfGrRb`-
z8V28o`@w}gnY}N_E%eNQQbSE?{y~&zr0d_YbRIqYKddJQRZSFU-rJ=kT&PnSP{Wkh
zy?it}N;TYa{fpspcmG;?dDi1cIxd(9{=Nf|+-KG<5Ntv(ws>RxlF31m6U15fueTX>
z)^-X5&G4bFyS;ZcBw@qne&)I=Zy5|V`R@6SaQc-SjD>#~!dzwe_ajj2^nU-mT{Q;u
zGJLc#jAZ&^WuEzU4tuz}^_OGuV}Z`PxVcKFP#IMHIgAWRL|@8@VRt)aEUjXezgGSF
zmwmm@!`4~g>Kta%apf*a5ApIg=19o<x|reEcgU0zhmPRCAu3kn^<hb@r#GwXZ(MMV
zn>r3fvAwl9d+pWCc%tTLrswNTyt<7UH|%Y{DL$yB87y=Qt0}&;PU({(esg;QblX}e
zt)ETt>Sg#?#Vpr-^|tI*Q>dtF56D?vB&jY-2##;XE}qQs^?q`3mGB?E%0Xq-&{D?D
z4K7OGT@4aCFg^q1b>)mtAos!AhnalOO1u_SRz^i=i4BOiwpGcBoxIq?>FpHQA3~8d
zG~JuE@%PO<ElIuM152j_KWW5&u@=gpjfw^DqBo1(g!PXJawh27c}<y+u?%%lhts|J
z+aqi>f(;~z%)v{+CfBiJ7;e0)+G_l`^6x!#sXKxW;?&@|iv5?je{p;o(zRW4EnXdU
zd3)^#y(He6pY77c-N=6$+_cm+D3aYCrAn6%Lj!=}?RAE7&$1t$ZiNhF+9m<Lrv18^
zB!PA$XX%JbJQgY3Yk&BI&Our01zpFdq1gd9f*QCrHApclnwyXOZd$&@aSy@0WE-S_
z)5gAPDu)}F*t_B#vTNrqEyE#qV#c-(oIZRyH(L~GXC(Pw)2&$>3pWQlFu1Ls*eri(
z%NT<kkNu(OdKZn`8^bq~%L4N=x>@AL+@@J;lFMYiDeSM7J9=;N&UdPth%F-T&Qbct
zK+nIgtyhoD#aF`90!A-T&bk`b3aN0ATxQVJ`|E`YX<TYN7hL#<a$O3kC9B2YVmN%A
z=HWlEoOro|h;qRh;pg)X{7}TQ6S5Rf;Q#DJ2e`joa7FnbG;ggmH}BF6q7@Ym*0`E4
zQJUXX@I;kd1eu}zJ`G2b9ljizJVs0aJVs4veNk<5XB#6HpQEH+fMGHJPs2mk*)EWy
z0KgW`V*ZKR>P*66kYo3LPtSoU;w?%Wz9yO$W*!X$qFnv#xrU>~pqmHr={71-0z_y~
zja?pkpB|viDQ>znYvTe0lU%Nv;#H1s0ZofLfZB58H7dNU9(oyrj*k+l7cejMaTtVu
zlp`Q;{QkAaXQj63Z;;sLo6SD90<t9fd4P2~;l?1-(nG96Y?@PdQQWgt)x4%q&QsqH
zQZ~_71#CIENz_g!F513-$R!wn;f%Dj!%E*BFBGx50Jm+6Z_7-ct4u)XKv_;}o?c}z
zsjP}8ZkMx(IS9pYE;s+4JtQrgV3d-898?ISqln)<bob)vIZAr~J1S~21c)f4Ei=Q8
zqG;cX<QVLi&)ZxdZmguB5di>b)AdhXe^S)VPNcUz?_oN2E>_xIhqx2}0%Bvvwuz2n
z#hC~6YdrvWl8>DK!emaOqZ-xH1hE2j`JpM*+svLT;OM9u51q{40Gg5E>pKun1k~Uw
zv5m$ZsZ?*bu4IWRK8JK$)6npr+3wU~(ht~(N@mr)uEXYV5)Jw8#f63CCQl9k@#d9S
ziQyYQ*COm=*2@LXHO6Utlf=-dmP4awQU8p+=AO7XfBEcVH_xb8)%>np<tm5mnHFCI
zMZ5iPban{;BUi)bKhIiIbxVc|1xLBWkCbCxU=g_s^bxQN(5`ygu^!*DpB5nu#rg%N
zwb7+KUFN2dkgcS<1}24N#J+8`WGSXQFZ=LY%LqbGPjSuKxVM#Veqn5A)3l$;#zPmT
z?I3omAETx8(2Ffx@l~|9E?mLy39$|pl#x>?KIq=uvDiS*E=DSa{ytHmVP^{A)z~GU
zy{d{hm@K)64;IgCv9W{{6N=K=t#A%&jrAoNO@GYVxJRWkN%Q&7)BybCX4*G#F1W_K
z^RzkH1#b<vX+1px`^H&o=>an6c0$l$!MBdJE;Tu(ZT7{NzEHZ6hvBz@RrP8VVor@~
z#gjNwx$xbYqrFffWQ`G{vcpneUvfrz7@?LiF~QZ**YJac=^I6R$KoY50UZf)HAYp{
zhBSTz@r~_P!^Ex~Yi-{Tu5yhJveKz$VW+_tx0)~y5OncV{ON0_CW(}zDOcsu)U?!o
zWsj&O)LD)t9qUQFh-gW$)H?v5>NxCJ`}mdQFsGIgnAK0>YKrjDv)$GIXiv6P?$f?P
z8DJSxPDDpU1<+OLg{WvrDiJ8|l^Do_`)sjDgb$uS_%wu{)=I}lCo)HY1y7*<nK%|K
zQrgWyW4~npi7R>~u3E3tTTRqfST)Fyn{Blpg0l#+GA;BRVMIc1z503SmWFU{mn$Yn
zaTkMEqXE#KgoFk{86FWi?TliD3a!c-m>{BzrVFF(w;yyNk3ooEl;5?resIxv=aQq&
ze=|xieXeu^Ww+4x1P<9NF%j5(wqA`fRH5S}=Q&aE%1S^Z9VwLl9GuKvV6@Q)*jb=?
zmueEEowlz;Ef7hV*V|_QR8Bq7mFOg@T%DUX?7hKE_GeH|S06_Afn@j@(brzdIqd(3
zLg~yL0Dcs%{akg%VpU*mE0BHhp5g>s-$@h~xdy`M63~VYgi4!<m0}2XRB61m%(Y{{
zbc_VzLDrRfOJ&q3to8*85KHjah6769U4ul%o*C;FIg?}xp~knOLIW_Y-ZYY*`~>FE
zkq<gI;T*)1SQpy?Ulyi|{O)!e?&``&{LdeT+)b6?Ir{S(T&nsu%LT5;p2v?&X!<Z*
z8fN@JAEYrW#T6b1Fm#q`0lzu&OtvEr9ia8JzYUDDD44n?69{9>AkNAZ#x27<Y48eb
zO9%Gp>yrr{IZBteG-Tf;tW}At!n&Zt1K+7q#WB7UmQKTsmbFRb`ryy<`9B#w{W%q<
z{O&h8Hh8)Di-Tp#7ZpGO6q=OjJ1t%j${ZbmMJi_fC)i%}p4`y84TVWDnk(VX{u*#A
zE48Hd=b!3#MBUs%%1)maod;EzdT^S4J&h)VYvjL<0H5K8f0dTuzJA<2g;SkX*igfk
zdBVc7MZv{vjQu?=UK1j3@78o8*;NsVN1x(He)|aEeSS_@<`m$9^MFenR$d8$*~>+l
z@kiw#uK!3uH!VI9R?vOR<{t6y>br&y1GRn)S#LkYS>D#{BiDZVK;>A)9+P>aRQ&01
zL@;ZIeJY$Pp!#v-P$p$+F}L7dlAHbUIK^}PgK{IZKoZ$>X4b}Yn}ZWr=YXg;=66qm
zt&}H0A|4e+4v5Gt-*VWWmkiP8<Y?`Xi1PNDL=_~16z-%6p5Og9%5CGm4bTEcelI9H
z*rFaf(W6xNQMkk8w1{D<`wB#hK(pc}*+(9E^7^5*fCIn5{om>A*;py<w6#iu@PU<g
zbIG@)ez&fbxo@%kO8Q8jYuNZ-K#LgJ6M@kpwjfj%JivE5M~4AJcRb{yAJXL-lGpbX
zRb1co34pH$t}umunDYTro0`>}XIn<;^}`PE1sLNIdc7s0*Q0s0Ei7NSU96yf;o1^s
zO+cDSZ-K@7a&5rXwGS-4+mf)<%+d3nph@qc<&PsP7>12d4thA=3I;hY2AW>h&$Ee-
z(#6A_KoR%#9r?gjHUmTnh~^I~X{sVNsd6|GO3L{>rRb&g5HP;z*xyUugUGmuq_WWT
z*_O@_g<o%_BX-xcPWW5y*JoOq*~`3-Tfc4+P>C%^7OIRw-d=(Wf=%o7?pykiDa4OQ
z2QjJD{O9%LH2r&)2ZkqM(9(I(Y`rd^nM*z}eEmMQi(zTT%j%#!H3vHkcYn$W<MBM;
z0q0~{+Z{{N|JYo%qAf(`2OPA1%|7zc{E@97Q*=cD`$Sj%u~P!t3WDbLs{ld03y=13
zqr3OdcE|tnXVxK$f1Q2V33Q&fZ}Fj(S6;!7CVEmXb}spK_K53o{lVYAC#<*IdM?i9
z{<F#1%e#+HSnmLB3k(T)2o&GJ9s1nw&-kZrIS!sQom5V%;bz7Z87>@!z8F?Tg`Xz>
znOnAnfHMJgYgmB(6zP9~jvgRi8#c1JpN?$Tn4<Yh=??7eB~tT;%&B4tSi&Lo&U}?X
zww6Ry4(=WYhBy?I(BOPc20+Gg(6?`~s#T%cQOA(>4``hxL*?#QJBSNrcqWjC(hnY?
zq*Q;Q!aD>@eFSlG>>vwW<3txZudv9f?VFfDNqyWptgn<VIjjyGPt%umN(g<ABZ#5Q
zU=3KGO@kZ3o74FCOFRLi+Y_o*Nf!%N3)x5^;f>HkI1hZ-BAU6T<Pugo=L9D(Pr-VJ
zmk!lZ`zVodFi?_I?1u(!ou@x$+{QB@eA6L6=Iuk%KU|=6+)c62P)`D=NS@N=6%E1!
zdo5*l`gD{82h2s8VqE|E6Ll9=piFwApdeaFD;NEWKZ!XkK`EbKZ-q74j=Fgcu<H-o
zSiN+xmg6H}E%<xY4?d~1QG=#V*;+hT3e+Nvf1d^G(q*6pTaB-aO}89yqPI0~GHELu
z<Y3xy)Qvp&feO`Y$8?-F9Dxj<i*h#ew$I*v;G&!9(@IhqaUl0yQIm_y+L5voFUP3^
zP62JrcY<R3pA7eY$+dZ~Y~od%NauqOD!rkJ5D=)SqS`>OzH8Kln;}2?JUNqfCOaUt
zfve+;n&AgMQ!nmY3;@xWaHox9QLEUPs?>H-&{hrq!WM!fo;DTj>VIdI8{TBg&n-<h
z9w>EBeIou2Q&Qz%2aEb&ZEU7|9B9fi20ScPkj@A5D%oCcKD}%OC57c=wXNVTWgteZ
zrJ)WWF00%+V?ASQlOxje;CI{9!y18CBgz|<#rmnjiYIth-TmSai&f%Iv=WDfPcM-y
z`_((MHH^jQkptNtSOS|9jI2&n!dj-`{YfwK@Gy!Si>k+r*K6UN+Qg3Pwl2r&(<Az-
z6z#h2-{CBG1vXE<UP>)mU1=%+2u_gJ&<^R$hTkd(um6+n@<LyB6!W3577b>i<EsgM
z!+A@0#~~ML=tVr+GLGia`<$E?<YJ2-vWLd^Qw-nY?Lc@mhqW9Cz-}D3{!YDS@5sq7
zNV(|5K8$e-JEbN$lL+XSJHa|T9hc}c_(Q>pzj0()JtxAi=N#8(Jthh`lXBP)nKF3>
ziaJ;r6B>5v!EA0U+F$BTR-MVLx*DY8*A>+Cc(fN-+k(n~058~dqF}}gPShR^GtmPE
zkn=?kG-A}CWPS+!#Lbfjz84aKAn=tqRvpixMjrZL)UiD)hkTTl2(Ggb4?X~80R~5F
z&%s~>5n7{k1Hm#?b70}Y=$KPh6=T(*H>Mtm)c3I^WQdC}`TZ=+sgC|=w)@D{+%L#s
z9>lwny~t{SNCZyJc0f5&dJu`4wu@nD-8e2yk%WOLA+S7eTpk#jY<XGx>~HSayJnv9
zhzRXRKskI8VH?6QHd1CbxoB&d);buN7uGeHVs-;FKJsAReYWv<$|_|=xagMIoJM>6
z5CL`X1i;*3QJ%ZRy!~)AMKn<J(VvG;$`BkDb`4dP!hl#`&u@r=q^J)lfEM~8&s0IM
z)#mEx<S85r{PJ~e6L3>x+WK}zN950XBd79%17t+1`!tR^$Kp~E_irg-Xn|y6X!bhM
zMvnre%f7HbZRovvSe!Rr*?^V_B5u+?!e^sPlqNu~%JkHo%Km4g`iLC2@49Qt6%O^k
zS!=-qhirHZE(|dQ)@uWhZ@s=hpWE<^y?s0_MT?wJA~>w25ccm*K$jSfIJem@2jz^V
z`FTBovxCZZ!l&*#+YX?Oh^t{*%$02QC<5eEgV4c#zH*K0&T;Y)Mcx#o=@as%WFCGq
zjFW-Ap`Cvhhu%lwCX?F=J0<b>DVeI+E(xMlWYsORzdYyiTd}b$ty8fC+IVG+^lF9|
zkg#X}P{e+rilkS0EBjd3J4T0$+z$ktq1JH5x!kG&gq(rpnbJ5cI~!OPynZ`|B{I<4
z7jQs5kbP|ebw$@d1v<@eV&dU^jTFbb=z!z*LtS&rMlNlAKe$BjJz9ns3gZ=mBSI9l
z6<2dJ!!$IaL?S)0x9vE-^(ELvs7Ed6?%qqkwv?KZ&J&QU<aYN|fr~XrRLSTl#=F?7
zrI=%JSL(c2AbAXRB0p}e1L*MDz=J*9dRT5ak#AAh3^JJt=NH~RwG|dg4PZkU{5~Nh
z(`+kKa2w;8brGNK3i$s)0}OknpUN0&hxc>ujqWVPy22CfES=ofIo_u<!g52CB%=Ok
zSHw5nAa@~oO&GoE-Z<j)0^lViOfw)5ffB!Mh-C%oy~gv#AI~JqFef{nd!6v0?Lv)8
zBaxTf1}w0wXn&F>Y%aldDux|SV<A<|Yl!~@JDVjMQTYu}cH{H7Y4B4}5$3fJ22gFM
z?S$tPWho`8K)YB<i3ga$=}k@npTa&^6honF&DGA|i=l^^Vke~IuR@fF&20T6L%q)z
zuCp*5HE~%jR!FtQm4qX<6-GBY01bRVZ5F_`)4+XFSpY|ldU8*yPEG%gey}NnRSB6T
zhFYT`(hw<dlD!5vl5sSq{vUR96yxco^-7=Q!7M8x_sdjBO=iTQ84lIAI*6X#$Gb#n
z`CJ8GZ0zPW)RB(Q=AasSAFZWraHT%28w3a;Li+os3X1gJqAtgP)-ygE*udok1BUQO
zj+_T%`1P1Hy^~5EW?G;wtNL)9YWLo%p>86`Ly+ekn}b!Euz){uhMb3Cf9yz#rVEX_
zgv;fR2hE(EiEpeLFH&iG(;kWaJt~wgFNzt6<^{Avb!2OGqV7y~GM4I3lk+9DsW}zV
zge1vRrr<%+5LIqTW}I5oEgyA!cVk4ARUp7gJpDbzj}bHEzm*&I(b;<Y%BD}ik|__g
z7SniP7x7os780f-4n{jt$5EWq2V6T~)veWf4#b7k#{PVpkZiBLGvL8tXM%<^%>-R0
z&a%tYv`mQ0!XCKDnj5I=HF6rT!}u96;gqqnfK^TAAjF;-3lUuYR!cv@8NUk>n-3*e
znbxfz$J<4iR$6y-_rx}3%-Z-4z_M$8R8KrT14bT%Y<&t%FVAwfJsa;>_7PAX^P{>s
zEke#{s!P9A&JAM-ufgl$iNKfY-WSj;xd!{2QUyyJ+@uun=~1<$32)<<DR<I+1er~5
zzB%G?VN%%OHGU=&22cS$zc5Z0SX0pnL9r~(qF{`T?Lvid#U%URC&~Ve4ecxmj%O$j
zJEO!|igIWZU?kfwrx;;?Mn+THBH-=$CL*Z3HbM@A!{6yNIdo<M;t+Bo0lulXs*C2?
zT!DrYNo%dew<D*ageFJ+3GRAEbVHb#jKt%tlupvU!{+KrhE}kaQsS>UwMF=o9E5q>
zsx!bs$*L+76XX_z!tBmMe8*jn9zX=Z!lPmTzR872VelS2jwLfN?&KN!y+auQQI-UH
zz0?wO#Z@sFOr1ho4O1j#`@U*>7Jlh5OfrX_hoJSeW3p|%qJe#CT1J9z@-NIj#;LD8
z$Kr5cm4_9oCZ@JJTw!I*a6QDgo-12tC4ViqBsEq!u$&#++Jj3`V0rdi&AfFEh)!I%
zGQm+(ulVlNce+n>92u5dWsd^<KB^p;^Ej_pUurq_IR3TVLcRhw^Az5XcrJ=roz4&>
zCd_v58^FLNQ(_P(@_U90v$7SfG4yVrFEm8PeK;b`v9=J4(yF-+v7H{+kXu$T*MDMT
zCQ1OFQ<sxDaNAqqYGBi=gI~%_C5eDFs;z{z_!Z2w#ac-<zXBNA7Hwabvs{%s3(CNy
zGYLotCvahR3_Y($&iH14zh}j>lbpb>!~Q`xHoRXq_e(1K(=plsPvf-1o|#)4ykOY?
z=!Z!}Zh`0BnH>coG<?s#B{m6nx$;uLrUU!*FHnCG?ODO!NiU43L&CJkJRLo8dQ9ad
z1EuDNS#|ZC(9R@S0s`BbET?WaiYxb}Y7!A#>XeUbW*``QzZ$%bw*J%s*wioQ&E82V
zlA3NB?0-<txFZ-cgbgg%2ExgG>SP3md!4=iSm#Hrw=f$~Q6<{qm|K$ia{$%;WbDP1
zt!QDmpzRCwwPL3Wrp$Xb$Tv@of2R06ge)}#zyY1xw0tzW{h$l2CBmuM1mS#Gr;Rus
z9OAH<!1^gs2xhELFBvKwX3?te@;4~+tN^zFt6)$%HN9s}WQiDYA4L_W&LfB63Yf$|
z+H7|j<Hs|4xM84@xCacSf$3<jsdNun>%ykGH8Nz@wBgHfUJ=18!C73Ft@cporCXA(
z)M0U{jQi7Ks4DIG*=izPHYvf5A#0_o7N=*85gj$_rnjS9*5(Vl*}oj$f6YGwEO_Hk
zD8NxK8$kW~Oj(&3u^BeA6X^U}XR?#W0PG~v$?WZpQck*+X`*_$rDNurdD*(=3tS(z
z@?Xl^4`n2<Vxp$3XwkcE8K7*+k$i)RpqzAxeo6Uh-@wc@VH+h(Q4_>9xlrR`j32`@
z*SWBd1nbm*(k#?45g$N%c`Vh{qm3RfJDfLj0luXq$qO*lDhSaVO4&NBjDcQXQ<6}(
z^#N={_Up6N$xH_jci*i06u+M4!qWhpP<_61tu6Lo%=BK9!A8;PPXH8Gpx~t()PGet
z2UzQLvp*pyJC_l^AwbCu_A^=WxbrS1VhdxZ4W)<4MB}XFOb1-EHT6QzT}4HJ2|8NG
z+|!WcXs)GHc@uIhs;$OX#jL5v48=D{(Eo%&z%Y9KFXoXBAC0(#V^x*J{(9ykD56)&
z7*7RBp*Q-IVU^%po18=)T4&4)j42z)N^UEGHY~0OSs0o6u&fAsa%Dyr0ZUI>GKkir
z{jq2H&3u2}&#7{@3Cs2Bf3;2mYN#gz6GH`{8Y1Gn?OND#F*A|?c7f!{s{#aZyT$+8
zDsPzRK1%#*>S85QE-l*FN6$b*#m(R!mZ1U4j6Id=^|_zPdn$Wd@+S3PNY29sBK6k)
z+j_k2-mG=Tfqg5{20PMXjGBk|a&W^hSS2tSO%!zF{Q8IZ@eJZX9r|FEOe<EM1`xnX
zxQyPz6~F208B4OUQ#b?kA6pGX0GV;%+@j_*r+!p1BNxY9l9cPg6DHUJU{$JgSawdi
zcurZ*U`;@oC>T~a1u1oJlkQi>G8v%k&G;nFu756f5*{2@f6V79RvxGxvR#av^&6`e
zoG3a`SvOeK^!97_*DS8G7j-6NAI<}I;R~w57i2ISE1u%#b(R0Z8&V0fn2kGx1#^n#
zATV(fq2L3h%4f|#iMuo^?M14@4_yA4xwlr|E_W46qHam<xu-69f>GCDs14pBZ?>b~
zP*zZQzmB6hT40+cYUH5(RsWB-D*>lEZTmINOYe-(OihxgX<sQqiBr==ikg<8$W{%>
z-a^(>V<t^mrc9|MZ$p|AD*K)hDj|e&EJgN^or80}`}v>6Ip=?zC4ArYUGLR3&7A-9
z+|T{o>+g3zf``|*$E@fj0y8Be*4`~x$PV(fTKiZg<N-}y^a8F+f8$X2jlEMRsjN*)
zCzChQ-(ectCAbjgcd~q=;z$V=xHTyhms<OEY@gRE?B4DEE(;!LQ*Mx)<ha)5m!8>Z
zjONQNmRqoy`(v_8z;A2J54^j0<TfBKN7`x$#7RYoC;cwgNZt4Wg$sT4GG9E|DQ@iC
z@748Fx#ZY`6{#DM97pgx@od6!+UNrM`ENPu5(xCF*bX#GYsvIUx0>^Zy+lI>!-@8%
zmfD;jyAn>-UC(Y2{mM*SPceILG-EJR<kd;{9KW`8Xa&bk5_^)P#P3XdG-iA32SFdl
zJ@oGheh~r#=CDNOnn5)heW&0SX5?BM)Q9FtU7RF2S4N%^0<Rvab()s{iOLCMvjvlG
z+`7dUKWl1<)Vx^50J@;*lfE!!7<G+|Pkc9d-iFh^oFafH!QR9gTi_hrL}oLA;0#U^
zWtS9pv3*_QR@1q;!Gmj+C)u@BwmT2)&F<4DF7peUcIUGW0DGZ?9=b~eTgJOB8M%jG
z2A-3(WtSFrtw8%<f#P;>NB_RMek&0w4Z)Q+H7E5ih0<91ym9%y-!02N4hgK4)Yro*
zVgN&%34lxI4?2S<J<8FCRWy1sGUZfYd~&R_s?F5TQhw~w9`s)VY{U7Jj!G_jXj0pg
zRdQ*qdN8X2C`!2A+xDrWkJ#8ZI6Kez%QF&ls!fK;8*eKAhQ$L6yp_;wCKM|=h6S=m
zs*lbl{J?4gzREODip}=&qx^V6cJjMwm!__r)oYTdpT76I+e6FUKATbkwMpRNzh%#u
zf<n4Z$*$b%gcW&mUX%4;;&UAgu`+L<enj&Dtg#0;U;!%i9WiNR<JF*0{gR@(E1Mro
zJdvVN5b{fN7^?|TG7r<+;5KA%;`_-`4`<1yP;>m+qpE^MUy~1F6_wAXHbDX{njd4r
zwu0HS&%64v?*I4cFi@qVR<8e*%}~J&#s5Cp<kKGPZY%p!%LyzbV}RNPed?0QrVA=H
zl2s|DuqSP9k3DYY<i56Ly#;1ZbRwMWyZd|Ub-qy8gw47N0bEF+)@w5MP$M)&rwIRM
z6r-dB-As-pxOFU*xOOj<^%}s$@(oR1@IVpuF?udZAw>Tw^%vRQE!g{wFwKwl49#bW
zU|n%UcX=R@4Q6ld=4LxDvY2%RHAk;+XdQA#4<&8VlrV=UEvfH=^O220StpD$oQ$4}
z68!BKsB@y3{v<dBG$OCNUiBucBbe>ItJDHbD>uPPd^!<!*a+Sgd&}m^s8a$$Aa8xV
zK46mA(|%MGhXOv9w+hT}p+7N7du!wzkR9MUqDF4HcP8s$+{)-b;(jCN<;i4LjjVeF
zMk-s&FFg&d;0gAU4|0iJ;4(3n`1w>LKo6kycIqjaw#+FGck@onU`9w7bXlV7f7flJ
zSTE<sJunL`qocOK2EA#f0wwXw;h%ph=xrdDQ;AU*?|XkXp{suVSIl*wa$oG(1U<G;
zx%P+)#YaMpSc_)kS)!-j?rJWv2v!DnL8#vxtAS045%_f2fc`9}vmLPhY*E|oj*hIg
zXub@qrowo-7nW|_{SVQ*cdr5OnA`O~gJ59}h713OelT~;64(-lInr{$bIx1cVLOp+
zVKwAHIr5;(?e#(E<-PZv0gy#fI!c5T1UO(;#sjM$LhS9>3ej7j5M9#@z7h)^RB5o!
z<&fD{P)tv3nB3?uD&f(cF<i0|>V>q(g%$P5{k;W@6Nu+xIT<+7W_bNZVyiNtC6pbU
zi#kKBS)AyzeaF50(%a!hQ+b>GC6m<Gq!D`)o%cn^V<E)nbFzx*!(62fvQ;>?9EQ0v
z`&o^Kl-%A$LGoDmQg-vFNz411`s6*zVS>b*u7?J4<D?h!6XMyTnJpo;7oE|-ch+!X
zd&uGY_h2X*^*(yW)VBLYX2`||rCw}-3?F772Q^3%6}{rI;hH{A)Y<AQ)1Y;A7oseB
z|C>=+$k}JuP~B7H3FF)0-P`T7cdWo_P{K2hxi1yMmZLWz5EhjFL=IMaq)*$MRB<Zk
zmh!HBOq&5s3b`oihms7%R%6zqk(S;Y0WBLUAyDzFO?QB}O1P0za*U0!%*UhfSOpAw
z3A&}iu?mxyou95p%CjaIe%|-$^Qi$aBwOBY=dwvxX3ME#U0<jIM(Vk;#Tb0vd7}}7
zn8Z0K7zLFY(gwg*J96Gvu%Q-AL_Z`t$6WX8s2>~t*RzlK!-<`f|9KiDzkcwq!eKAs
z8i@xbY>^hfr^VP`aKZu&`ryC9_&~%^rV`YMatjC2-t@T7c((bCsrTe5nwRqYS6Z+9
zvf)-!m8-nHg80-qW?z1jvG%N$mqqB-0M|GR^Y*O|<c|C><BROczt!p9Ezu2kb#bwn
zwX96y$F$#<=_tf0IDD1-w`A6v)rm(IT-o7x(AUVJ*UVr;UwrV1q`{$j(%>M)Bg&~H
z+}w)tCR$CgXZY6r_EN3O3$>a5zu|F5R<yJ1rrKffx5j@QKIZ<d*SpKA%Qb^<5)u>C
zcP`StqU7>$pnl^+J$PA7<T+WvVUst-V7I60JH0*9lRb|--wCe=+(&;^(_JGqyK=3?
zQ4sCrb5G`7-~9uyNwv|xWDuo#EW2`>r(Mok$t5EnN2~DtSXEbSm%HHT5$72``ZeIt
zpUzKM=>13G7E*hn)4+#-O4)mtLw!|T?1$&%%UXLV_&q3r_YpQf*3glW3`y5}Jx@I0
z3b5Aa9eDgLK<Kf*QcYFW)Qo>Q`v%mMLZ>TiGyh8x-0Wrem#&wt@V()nQuK!8Wj1do
zQwrE0=DvNlsbYHPQt7Ctkc|DY#a>(bn~<FPbaD$}@UWfV&)R#mlKY;A2d}?jtuaV`
zVgSz%N-b>3d^&jz*=p-Fcuq;%@=LMHi`39t2I#LY7Q<`eCv$J!#(oN<p!r|m7B5Es
zl4WYmKeL?{OiOj&FH*yD8nBL{OFpY_<#$R+2g8-=*bZR+K-dNPNZZi!_EdGfMcQ+1
zy5EuKbUibO7{dxGYaY&Sy>lsES?nqMfrKuYW1=xgsN1aM@uTe4+AHANRG>Qc-%K;H
zf8S>HCg(`Rr}y_By|VDsgF?@|D4AW#RBa#y??&hBeob|rCNbil%`)~Ypp?G(@sTBf
zTK^gSm4l0ew|Coh#f9)&+6zqoe7ST!(9IXZzqB^OP)0c<Rq0dd)w8PXYdLwO)@#>Y
zq!_HnA}fA))9PSrTR%PRJzg#ig$-$@N{N&m&PO9Ok#Kv=YR*Uv$Nnt>17kvwVqOk{
zCqDL}zshQYDtwYf{+~$4(5FLw1H!F8`j=vC9o~5N7A263uI;o7j$}WS%i_74Dny@$
z98LKKa>T{r8H>zTdY^S6yW0BcgIY-IDih!JT#7XS2fMA*>yIgiy2~y|Kb;(Z86D6L
zNA$=E`m68m!c4x#TkE%c19uF4)cYIsKmq!fijx49TfQUH*6A0Q6N4ugamZnR?Zr7$
z9CovuhxP(9u{YG=sycrYN7tv@FIU<3t&THUcf&fZz2|nU^q5x?V(avL(@l0UfU&46
zrq;H=6dZ^+PA>v3k}l-SHJhZSI5|>|xH}GfSW+z;eK}O+_1n7U9hwbg#MOQgC5uPD
zt}CaV-CPWYLvnVdqQy33IM5|!F&scy(R8(Z)fzebyg&P5v>e;tN9Q=hWn<F~(Zh{#
zYQ%TF*RCu~y{#z`N!br=VBJxa6NpFv3iJ%)uNT)0|I6*B+KN3Lez@^M_6m4>!Va^0
z8SlJu-W&I)xy~X*C8-}<q`ky;$h3>3>7PfWl3_;lXjc|$>_w7NN~?Cll)ysu6XzC!
zk|C9yz33@S)Ie4&>=bY_Vgk-pY+r*jqSkH$A5HQOTW;}ZIhP^@DYq)siu_=;WVg;m
z@i`F-n|}0JOzL=hq^r`IZi%>EmOqt`SWXqY!CzY!*e=2!i?T<5w_z95DnGC|6eTec
z2bs3g9=T+7TM$X(VSfvP8C=>p0KQ<?kJYajpd$f0DzHdIDz+=)MKbiMNeRk!TUjqn
zrs<!W?=`C5ejt}s$|hb8{nux4pxtSA>j-6b%aIZhqx7YS?vAPznEet+MNX5J>D-Mk
zIKb{zw7u4)xsjt%G;S!*VD)g*JU7e!_BExP<ewLnpzgRx`*&&GI1}|7)|BEhy0bbU
zw^;mqp~;G4k~yU9JGPj<iusU=nUeVmM-6To%X}Z=S}&#z7ZB-`;`Z~NR_l;u)oB5?
z%2D}g@j)-T>wC3#u&<}-VVk;J@7=D`2GtCYKO^Ve4bT;^iG5CK!}*04k;JOj!WG&{
zwb|{)e%bzcCNJT|>uvDU4<x=7ecBi`bKUt`n>zsNJ!;TtjJZ2W<nAtPIdZb#=6o*h
zj^=C%<M!g+zxA~#4wzhkl?1K&FwhjUNo|uX0!V~`{E((uRkG=qL)DrO``b}nwftLt
zmNg3B%6^;eRsd6MbMfWA3kxi_K-xR+z)d}VvWW}2f%i2bFc0oBl$an!1joClB}7%E
z?5}ef^Ze51zVGbqzp}7$cZg`9T|PfyV7{N9FdoIJWN@*snHI$5fM20m`p{A>2eHUy
zpeizMYA?X~4r(=&r4RH@y}B^g+AVE>3=Wuj&8zI#pJ>;kX$$6fX7X`7TWstu;3Wdy
zS9Gl;X6u0+wOhv13Lf74Q?y#+uddc~YVSaAK$UFf<xq*Y=t99&qL#IcBAjhkKR$)A
zCy@L7xpF7K0(hD@&GM>+Qye(-_E~m70p*D7z7~JX-U8mKM(W^RX%SC_&eGC>-ujP<
z`h=F=rB@fuecM>m98vh8xuD^FtTeKmBQCiku87km^OMXJ8#^w#0+ONrDEo8EMb5>v
zIcnvvjwcQUX$24dN~+y|`J`$3htB4^w{(>+ZcNehL~eh?*j4F4J(Denj^XDEZ$^C0
zC1fMGZODUk@=s*`S&qu9vhDRieI;!jwa<IfZ*tWZbSiKE5d8W?pUOJvGWZ%7rSz>1
zrpR?B7_?$88ES+V4%_QmN{Rgo?z4d;=SJyps*rttIaJ+eNnjV7G&~FoPdOl~%O0;;
z@{8A&8V2at*#;-G#|D?htzPGzP2fD64|=cqJD*ceD82RQX??U(+a)*~?vuMxBKhpb
zw9->_7jRl?IPUM3n$w1s=pV4g^Zf4g=D0|1uGzUuz5#+m%-lkukZx{2Oph<ho<$-$
zb=wc-+WDDl?+NKR($%8npVyb7F`_6Ny+2npQiij*cxP}%IVN%k!U4z$vHggJk+oC1
z(vj6`qNYad{MydXLc7SO_F|q|$n$#n^l@2^q@g~S<+SB(I>@;a8@1Suy50is#U%M8
z^}iQce)6eGr93V?()G3swD;1vC;zjAL_=0HJ)$v5^)qhJZxOj%Zjgz_aDcHK#EP8D
zv2xv;4c>@S8q(fRvh%almbUKGaPLtpSRA-j);N4KidG<u`h2yS92bcXUp`i4MH8-S
z!G1Zev{rdl3>;Kp>zY;Co*%20M0r#keLMh5M~-o<OX-M}mf{j<*M!yJZqvs6(&^O7
z#?I%-@mjG9r-j2CINkjj^?_(s5Q=0YB)-kbY>c02Gz&#9!djT9l7I2uv9K;0%2j1a
zt2rFB@8eT1dx+|3DQQI4gFA{$rPUKC*_e5O7!ES@4h6E!*0QY6F#aJ{>$u(Mj&5SG
z=$oXK9P99TbgCG6>4Meq>0Q0gu7*}Y+2Yh1i0RU!B%(^@p)dt(|L0%ncLp|GHFt1<
z^KTL197Y$=+3`kHi;&zU)b7?ttF%*FKW6kS0sBJiB9F%z`9zdF!=F({o$gj#L}%$1
zAx*6P=r$M|uNA+rC$iA9r+qFY%`H91L!&rSZn3Xdq1}q4)xPmpps~bO&52O`QM2#m
z?8{uLIjh723yq}>!mn_PT?!~p&fXDL9Ac?yG_V0c#gW01_#9YESchkx-5g!z>b~(2
z%7c=ZT!fOP`yZT89IPYss2Tl3<P~;)_S*N;qD2F%*MhE8kB_cAF=*wcD-|1GgjgGK
z7!^sa(qnz@eXFC=yL+VqLXXV}GI4iPdOP&0CDUr%Hk5flwM`<VTJNNh420Vpl^p|z
z^SmoNk)@)=O|9QvXLr&?W<INfJx<7)xBSZCs8u6NB?3t4Yc~)(cU8#NUEXJ8;ooBi
z3RBI{+=VT}V=%a8v1!+;Dm47ri!2N0mNwQjQI-9oQ!P0&l6?_QkyJaoU=wOKsrKOI
z`<8AV!z3-JE||-jv?QUR#U*_V9tq}CaXTxG!Ay8NL>C$wDc^X#CNK5PSO<^2Z-XLj
z&j*5RkO{S(w7}@{21sX{TTp?a15;s0KgJ}T&9Q0RY@Ty5U^O@nO6l)X*J@Ygx0EP5
zzf;=H*0o(N)r1??9`CSwjUw+scyw<x;|niVH(upY41;|h!53-!-xpJLVM6J*ocH>4
z`KFBn1vwDW4fY-l2+fCD*y`XxI|A`!#CKR|=P|nFZ31pf(l0#ff`1yA;G-AE_Rc1`
z7l&=`Ughx7n#Cdw>C!>fLusU1IH1BZx4aU+^?}xucSny86FzjrMk0s!pzmU>SvToI
z<VBQCgS1$4Bp|I9uHsP!(tJ(kYH0F8HokG(&Y`~!D{T@Mlk5`IYZf*6sDO=WWaepO
zS%b)Hvh9XccMXop+=d5*nHJPTvNmd0IP^u~&unUcMS6efK*_EY_2xxQAKZtAnlFM+
z3O?;V9>;XLToR=Nh3u9F(t(o>T_U1294cdceu`y^ed6%dc=<TOU~fV|sQ;<<8uBbu
z6e5nxdhE-8!pBp}5QFPRNyG>Jz#3}${F?E#*$kfor?kqO?&P;xwtjBf%d1kK*Xw&P
z4kUK<z*IymmmSNE3bEXBi0h3a&X<5L3ab`}n11>}YD?{~TiM%z9;jsYF1oiLU2j#;
zp42iFmm)9SK5O(C&sb(bh+(~$E~*-Uw&M%_*OZmG)Kukns<T%d&`M-B*Ln_%TFg?R
zerPR}*LJFcf`gOKV*TnsMm{^Hm0VT+k=sh_C<7Zm=xvwPtX+(1GH4^%VhFVg#IBDu
z<Q7{$qIQyhm2-EZylUV<cb9I{E)!<aL4bq$7FFK37Vqv%;}+6f4AgRd7V+Cg?`&P5
z_h9uY&5U=ytGZ-g%5ESPR2Wab{QCZ^TKn4P^@d%I2CJ}QGcQlc=un|Fom;V4v*?^P
z-0(wOZ}>sk6UkQ2Qa*ve^T7&LNQIq0dP_8XJTw>)k{eF^)Vuu8(2zO0akbIU&4b28
zY5r6`SsX~RP-getp_$S~8p9S{|05Xz1FPZ#qx3LC%=p;?i<pyX!(|2eQN)tgQ}eV%
z`+5h+D!y>38hYrZbwS+JsAuI9(lKOp7<aA8`rk)u!D9lEAIy|&I~mo@uSDd3gGP@U
z)r{zlVhFpaFSW_<WYx)qkZJFQ@*?%)xGZW~NC($?p2~l~3MxpeAif}0Kl_j8tSZH$
z*_8vA91?=$Tqpy?w%S?M1OB#ABHAtyejdjL`+V}FsQYE<t=v?T(E~x$;<LT3sDz4^
zT)S&ar3qZpI_N+XuH{sgbDrFrJ^!}U+PANptDZu6OlsMD*(0HFz~9erabRl;4B<t!
zZb4k=ViY!kcV+DY)AEKNYrD8Rb(Q+F@yaY>x_mCfbjN?4XGAt7qaJi#>&*GugyxS!
z%~s&LOq`kh1O5@ltEZ^O&8_u!_-IN#K66NxJD<x?Xw;Dw@m#g&!<vM?E_DkCbf`gE
z#ques%NI_2XmZ@SXD^f_2&x;Zhh{O`J7ZbVmZL|wSjE-0lufcOg9=RXpl&}Vo-f-b
z?UD`FjU*g*HQFZqQ^<32fM0=%zB8k{GM2S0IGRG>S`yaq&#CeS>PEtIR9ZuQRWT96
z?dLP?v<7l(pZiz4xc;)O+1aFPoj#%7`uOmvkmn^P1>>SUn`%Ar+M}G7lIRI>GQ&ax
z0KrV*&7L1*-3ApXfrSG-D#59)J<+yN;@axg-EBPqKC;$nizFgS00^}Cah3e&SuhjR
z08I&Sz4#{SkoPdRBx2+6Nm+NbeM~0??IHy!>t*vLQe(S6M1xC@N=emN*z-@cQ&gHG
z<b1!EK{AN7QO4v_-LlQNtwzJWpWH~g4azfuvOqfh)FEDN?ORG7NGPkZ(1MyzvifmP
zme1mg^3uU}X_m0AcP$I7Vo@M5wmYw0YtKcg<!xo_I^x~2_>ckqK+6i<9sm6;x3;Rf
zo`5RjMVFUbD;@AbSKi&-Gy*Vu05%wlQhQp+Sb%p8!+BR%iD0Ap$q5pdQA9@XveQmY
zJn>5R2iF{M_9;E~gNn9MGqh(a0XC*9TSA$58pNa@1=uA(2#-;jK6=;cLLlA38A!KO
zW#wI>mon&BooSvKSHGz_LwfGk0L8Cx+o+k^bK5><)W>@-)+cI=%W4|EC35h#v9PZA
z;>IrnIQq<n+n<Hl++FvU645cx3v$#LxJ+u<y2ne}y}3CeTC|LK9Hi72<riK8bki|P
z>4UdK0e)Q_nRgqU!3DD5cW6dH+RD{`aT&Gu?SOyvao4PEE}5xae>BbVl|4wUihf?7
z+y$MJF^6m-`A50Psv`>$!>N77>@8?GxS~yl78?h8<K+$xcay?3i{>;|Wt9}nh2laY
zASFe|tmC!jF)Be-8nx{48q`|)JIya^y?4Z2xu?71@&{T%p7(Yp$|t2BUl{a1*I)yW
zCRAERa1L{LXB}&z@hZ45k!Ywx$d0_CwW7@w+p?iKzJ3=+lM!o<&y&bZ#TsdBH_z|U
zsj?%3c0TfP!TmR)63J`g_5DJ>NIm7z^KHT6G82?lceZ#Qb4Li4vE@YomBSZcpdaAh
zH~Tzbz?m2%XspHk^{2?2<!dymDc2jE^7Si>376X*LhCaej8t?P!YYkf$6t9T|AGr{
z-!^tOC1=0){gw`g!j>YbS=J2Yji4YCBJ~9EL#wF1iHGYn?PG=g8NG)?y7Isv1X2rH
zpseFRW*M1hSK@mn*4>}p-u2F$yPECb0A}`q++}o-8e0gh&<3YhyOL9V7tSrzl6&Bh
zfBLbr5}aIi3q%hXif~L$?}+AXF3x4OZIvP|rR7{#=o<090Sy(6bV{W~z%Gx@*4Twj
zQ(Ri0Bx2&bVy7~3pl5n)+Sq;Gnh+VTv6Dx-L&7iva!E7e07t7?T44o!QEEVRS6=&2
ze^&5-c6^zCXrI!Jm1&K%7D7w<UkE25v~F#OXKpa`h5-%N^~C!zmmq=QMt-hzKHKj+
zXbM83(jq!Qw>&>=FK0by=PMtV-Wt&r9(>wWSKG5K!iyBV#>Kv8Fn@7i`ZdwlWZ}IZ
z=ekn7JMrsMl<dpE2@3{(?xyq<X7v!d@<YPqh^|K3p2xcF2Z|w2Nb|tLDX-#l^N^(I
zfK{#~bX?~27ra1r5#<SVT%!H#yIcUP{WbNwrJ*|EM+4%~Lj-_Til@2nC-y+R+hQBF
z@?KwRSK|p21rPh4KcI`M9)(a*EEpQDT~wS2UFJY9&{@*aJZpEN;N_dY*h7R34_pIO
zs{hO`Jcql53w={6&$c@GEGg@&-jGhNA+_vi{<%-%)bO1UQT<sV&wET>K!FOYE77qm
zT-!moK$!FN9(otwfb#yd?d--K%KPv0+ILK`-0~fp)e;40@VUyiigap)iZALkldN>^
zClwx9=Rr{%_R?$!q%M^Rn}*?->G;;q5g4F-uR0iTu@3UVRl5LrF_Uvm-{e$mC(nWS
zoY08@ZHz7U{G2-cu6YFjvVs9|3JojZWwYWudDW3elHhOXWti)wt1u|^{@Hge@b<mG
z^N+I$h9js(NPiJ{+2Kj+K1%5cza0N?6Ia&=?VwhafyxBgc@#qvy0?LnTp+h?4vel~
zf?T-5I_Ty0kz{xebmg#W72sfZE4FD$FR#s0z(e}n9J>Bcx~i;%Usp`4?!b=Pi{ZOc
zx*D5vs*N4?EOBYsn;U#wIgy;y@qtWu3FnSA7C&3H5Q2TdFDdJxmDP2dUFnO=Dw7ii
zl9a9~FT^h`G~#QDoJB(*K?CA+zu|vSqrCOhlAsuQ*FF-(w}If9{@(iWKq*|!;4}?y
zO<4he)NEB2WGnm;T#+HXrL}qJ0xZH^fLEe5I15&zU^r`dj>W99U>SH|Zir;HYk4_1
z>{-D<cCx5QHrRuZ;4J21RCG7Tq7br^Wl>9N^SFBK=k2<gmRlro>hJYtL+i&OGGVrE
zB()nQ7H#gG%{i6EXN!Wmhb)@gHPq6{<q+zS!0fjdcpe=(&!Tm_`3a1ua2bZj2dFDR
zYeUQ{e3W!FRk)PvlTMReivTZ)o(jRhjab48rNa-C<L>LkQS<xV26byMwi3we^r6bo
zyWub2=Ah~mJ+(RQk?EaagtgET^S2yL!7>hkhzpD)e>+B@09XfDT|^o1xm-AxixF8J
z=Q-y*8sWg^FaLYks%NkZRu<BcRUbsM+U-*9(N;$6%}GOc1UwYw(_7DQqKd@-Iw)A)
zAqbg)Q7XQJ;gX?ue}isF=FP6;)?DbUcTz!4nKSlT=o{XX^MOoN*--6P*6-GJy~DFL
z$2MwPlNRlYAYd#M)rY^fKmgQCi;qCQW&t^hR&2+>ge<VNk%q$#1VG7Kx(G205a<G&
z=xgufOWDsFWw``Gvs8k&LHIWVr-JZiPkWSxlY??1<yq^gdy{+eLkiq88i$%|A(|yX
zqcg3c=aHf>7R@rzm%J7RwHK}nJNK+Htf{+q{T_CY)$}lX)-DfyX~Y4cu`skVO&{LD
zI18$`lvnz<r1uV4b*zFDUeab{Ol)fHxp(W7V;cE)FgMi$L#`U?&~vQacrv>~;NwL_
zMo_oexZ|BVyK-G>qWcXl{(cwHLK<+HxHT$VRX$CUJ5A|`E^x1H%&6D*u<rc0q{8^_
z{Ri=~?m4?=2S=qJ2OMh{1m?-Nh`XqyjrQ4If}b-(oOQ8}V#^VXgrSO)W>=bX@o6>n
z-@;Q+@;DP%q$(Xnq68GU4?<h1r3&Jtx_JHY4R6%^H2VG%^8DCFIO+ETeQo9^(*|xg
zQ0|s#<t5C)GGm@p6Rf5D3Q<jv4tnpwxE^r+x-BqK8-BpeyDbp7CaXDMlvuM;tq7Qs
zOdb-$;DpXs5|19PYLE-o=qvDD5@<p#6jdQK^d^=3F$WoMkfVw^)p+Q3gB)S7f=D?o
z*_Mj&eAqy++;}OpRy<}sz-B9M{{w-^0r;T=<eYW2wxRFWG%LXAzrdY76c&~gLEQo`
z)W+F1-tv*u%m&Kq{+i}*<(&JT%b!MUop-%omCl8`)HDm(q1=MSE!90kRvL4I?9v)y
zp*)UJK2UA#bELbl*o78B%ngx6o;r-vQ~SsV5A`Ymoeih1pnR1Z)Y2&w5Tm?zK@_zl
z`gxCM$A{)Jk0Zx>Br1$^$!Ds_bBScT!Dq?gCJIUWsplX^%4{&nd3%Cd*5L834oaJ2
zkT(=Jby{X(x?%5~gc{lEINiWPpvQXQ_|=<dFSDWMA`GW1rNz#pO~ntm9wdoK4|3}%
z^@Lm0;q;>?J$gE#pF`&~UxnD6SfJkLN!ZxgYlcyeP-T?~@1J73XcRyKB0nK^i1}aN
zpuibbq8Xezsh;bg&OhLn{Ak$563AJ&(8%(EccfG>R12WR>_ng*a8-s>{ApK#k+=M<
zp}l>w2ZtYHo%smDaQm=LyuQo2%E1pMerm}F-JieB@+W=(-)flC5Ln=yuL97igTujE
zvjV{+mJvQV1Al1|1`ZQ6ngUuH()W<Rn2a!E%xRp<{-+p9MTz_Hwi)~ebP^b($&oZQ
z<jB4{uUmsv4U`urf?aJj(h?|p!HWUttZNz;qaydK*`O~cXW}+koql34eR2fGV|$mH
z-q3%jEB`GJ>zq5PWrCur)dL^MsZ%TBePN4~iY@}FJ}QkIZA<YA@U*!LFUA}N?k+R8
z<V%K}&)x>3ks+T+_1u7iJHUBx;Sw+-jLB<aWB}IQ@5vbr#$tKyXYwRcl>w<N?0tm=
zvl8!UViw8%2VT!%uM(a-0+bBuNogn=(0P!wIx32ZWTtwW?xj@`gfJCcKb@WRPc@uy
z@G<tUBRp8Ab}SbeAk(i)%Yub5tn6Jsx|#YeL=InSm_tYz_)!{=Ww=|8V%#tqRT1I5
z_yugd=s3IAkTbayuj~A)Z-@Z46&J4|J-bJvYL29LqXgP~_<TQkd*Caa58_}<BWIW+
z%zpXMkJhXqb195NtA%n^>1pX!oTdwN%KX?hhr0do$}N|gGGKCNO<W|-f}qy6bu&ie
z8vS+)$esMro5F>4wJ&2+g%t{VS?g>3KMRk@3HaGf-X2`jy?T|qyW=8prM>a|^hR$A
zmrtLps{Abd5}1W}E>lW`TXwOFa!)D0Hw>3OA~csWjng?wW`jx=i0uH}X+8*B>1=y^
z;tB-W;wW9<FYUouPn6Dcx1PfD9!;e(@p<2YG7)^K0VN@UG@>(Gw1FcN3P6L+{#S}i
zF4tyMG30OgPpd9cFgno&dfjxUIal+A2qcj3ENY*L#s@&8{#cbS;QXt@*SHvM;8g||
z?I+#Kn$ILY#W5!jynRi>oUnx>pFv&p`EAWXziCg|T~%nUw7OeoHdJ>QI3o-MfnxYk
zED3_yBh`BDnk6tTPvl=IWv&WN4X})9-8Z4%&dzwk#aCqpR$n_u+BsjGNlv6^$D_Fo
zMCzI<9I4Ap;q=h2IOBcL?ikFm73Q-LGYG>agU_xP@8j~}zSXzfbfsaw!o+^P|Gh)P
zES!?pk{j!aqjoN^z4Me+$r&}}(UcrHF_uvfd<g?$BEc$LHy`q?I7~Qaxn84v`E$8p
zQewt)yj=0j+{nVLJiBd&ZJk!Jik;b|VGGGdN4)dnnr50x!telm;oIuuui(3wWW9{5
z%z8PSIUOB~%a;%h6ytLwd`xELHt?bB?(X<x7XHDTJ<Ns+n+y4T0-Tw``4Ibde3!W_
zbr)gyzS2j>S!}lDj}RIgojpSOe0mDZCXkH854(tj5f0x&^v^&_LJ{W9Q#uM8^V?7`
z9xJ~L9r(L%a7Hzu&8TW92_~Xwj?mCecQ89}kb7X`W5!A+!#MwCmBfKUoNbLPyc<09
zt>uT-%nMj)6q^uWy=;+hWXbpRL;#hA#!TH3@ywoDG3dNT9M^+#zoyyXq#`-Xi%EQ8
zS3J~?88b6UOr(AGnZbF<4yw2_$W%&m>$NI=Lwdxid{|*D22!DMk^l*iXH{z4B#<5K
zLB3-7sa1+y2@z@;BwzlT*1;Py+0rv3_jdg74KBzyFH5&7TXsiBtB-)<gXzIqbz|k6
zB2U*DXClqb?8dnQEJ9b>L+wZQi-4w=KoJVAA=bAU`!)P@{GCm@l2|+*$4%0b(-_Pt
z?k8zX_#rsN4;L-1aK)iYa~yl^UAs3OoNyVu0U1m!!;`K^0QVtW)qGdAi_a3qMuP+A
z>zmXXkj=#+zR?UQ6P4ib{aEK7f>>}vSaX&SHk9AkiE|n6>i$Rau$<a9j7z3+ls!&&
zg%?BTkf#KoFv1I$#&@^_l0af>faW`m^SF@H8X`aUt<t}_MzKc%Xo3`&0Gno7R{>SS
z_=8eMpMa$xl_GPTBtPv*E0?>FdU1)IDyL0G%z!T-pQH_WLz&%JNM-00&ci7TwMya|
zosTMA`@8jFBq&Efz)W;##q;qHfjx*IeeMgKV~m49FiJ^0eiwq`)0$)`kMFsF86rOC
zPY2bEKIjUae>|LjwJ#p+Lt2U+bzzNt=>_<=uW5B$e#B22*=O|mJjf7NfFX8M!w+J!
z4)NMWnU(#%3hjFqYg#*g-!6x5yACSgl8(c3Og0W{`D8A#1Ue63=^0Z6$5G)-LE#!(
zGx3Eo8<q%Ux58;N8Vm+24bo=#p<rlCuzhsov2f&7<@mV2wJI~vWI7`Pny~MS!}mq1
z%JI8z{isOnmBY%Dh^e+zDusJ~Jr;@?y6@XS{1vUM@mAN-u_KfUF*qU=0l5r_9n7oF
ztd5RV`r<LI0f7I))1^S?3p|`<V8{;|1ciupz*C?z##jn>V?vCIiOA=??cKq3+Fo2?
zrVSBlItZ=;G%eOk$E+odiH?O#y*b#qq{?-;@b^vjES2`B4zcCfP%faYaV?|qa&|Zb
z;mG()ItUx=%?r5mCsyF|Z7lFRAhSRG0G)ODFK`U#vl3ZyUI84<2=Z-Y2q;Z_BAx-B
z)`Rf!{2si?X{m&8i^lhym@glOFHeuZjQ_Ih;LuxnpDkAFV0IFX?==BB=k=y{#}wN;
zTp~@!f$o5(0H?rU3mCIw{H4AC_TCY<7jN$(X2D&DMl2KM!coT?tDTtyb|#^9A71El
zf(Qer5ja_fL65?hjK{3&X`t&iYKypBwSWSR@4U*6N%CmMjM>UXfsH5a%)<vX9d&W}
zP+ej$)tKIOq`AsJoE$-NGH)qkqN(Yp<0m1lDD7R$#k{u{?;=Q}NjeBsfana2JIYc?
z9J7f#2qrEzbs0XI3^LM4oTc?9Y57Ec-H;^j6O*cq|N77CAdqH3w{w*0$9&)saNy-t
zN_chC45mcas%Dq2o5-v7tlYzr+o-z3zy`0BgSQbU&bWz|_U_2EHsTUN_jfieg<(+>
zddnl>mjBX%uj##Mb0^oY6puc&c_MAh1Ni!p7HRyUx9M)!FroV9dml3QH%a)BP;r3a
zvCBCdr?PAUui2;L`&vEk%#VxY7Q_}8XnX@>)Bc($8wW!n{>|MSD{)oxO%i@6OI-9W
z6DJEOqD_#AgQ)z|aUIw_*Ew=FF!@AJ#!3ynzBz)P;pqR4#lT~mpAVbQE<0w516`8(
z-#ZP}sKQ8^i83}QE)Q66Om5{gZZqTJIx<@W=Z>GbVHCg_L(3sBPLcU((A4r|#A3XO
zuS`DOd$Gozv~1j>DMr5w*c%wS4e|nG-@EP)l@jz|KsX0P?M)RFt?JNNO<spjM_F>6
zc*!{*gxLV052MC}pJ^$#lIt+IIF+?LP0pAYQJC5*f!g1WHQ*M<Qa;Seo*uu4Hk6wV
zUgHkw5t=u{l)QKtNdB>&B3|-4c^lg?>^%7r%9!_;i|&|W6^qBY*mQp)=hj>!!7I$8
zd)P5?U@{@1TMBY_h}FPH@U2w?#Y9!{q{=%Jp>kM_4N_*UB8~mhdM#h$<$eLxhyidt
zO6EcbxN@CIIYJ+x&nsfR>!{U(>SKTQW)%Hn5R4CHdoOwI^4KinnEW*_L&p!VhhB>q
zAz09xjSO}QsudK%#`m>wyXurFP^!65YilFG{@0!70q|vxOF_2gG6IPJTNBmS(ZNf>
z!LNzc;NoRlyINpai2z!6Op+Hxf-x+Lle>YGekmwDbR+SNVyr(QrB$WzKme`d{w@R^
ztk0Ovf&=SAnNIv~xb1(JTHx)x525QrL(c7Zi3L2IJio^=c+;n#SaJ9$IJtj%g7BSz
z+UIkp38MFW&Ycz5&2jEi;oLi7CadBNPHR=2lba<BDA(i>*N#lQ=J#yO|133ev@9_Z
zzrL@5B!-XZD|0luKFXaBvGanLQ%9|qjE$R<@vg(eOvz?xJ{Mp{>jHlZIS-z(oj}LV
z4nBu=CWFc+;+ZqJPNewUhdR}Yuknfp8SKDiRZ85h36;HT!T$1BL!Y>=!2KWIwS&*$
za4;D%Qev|0PHKNC|HvH<WF1m)keZ_UMW2TNUY`4VETiyVm^=(@1;hmIprAO{Sp^=p
zAo}tBJao_)G!XjdYq*LYIHwz|$6NQHgB3$3J|M;K^I9RE*~-P;6&6$s-Bg8%B4c%T
z=-!SH&Y+hTM(?GeTc1og=rE%#6<@G(9js9!I?DV`Vk2bohtw`$JQ2b4_85tb?fDE^
z6OkeMc?a>(ZEHId>uTb-h#n4`buywp1}cF_Bc*!yJyd}Iz#Dsu;DKmDuBwca9ZNH6
zoY)A++*Ha%s(#t$ftUFllZv6??Dq+VT-z`L(OO4)k4r&nV^urFK5Ke{1RhIFvf^D8
z<`-1)B0(@DJ;i>Io09LcqiWRJSNL#k{P1=wd|qeq%%~l1lhBtTs{MvPVmSiUS6*3M
z%k?uP&1Y6N<AjGCm*S)6Dm0b>Mk<Y3DBx+$`8=(~IOD?j!(2q&dpDR;(1luEOtK?i
zd_2o#7Esz;TibS2$`+BDk>MG1d>eX`Y*eDlV+8$!7vA`AP8jNS*iP;o{+MCN(Z$ZC
z6+4C4vfz{ehWX^$)^4r3n_g>C4?qfGwv8q`2)b>zYlE3Hw7565??zQYJ&aF|#u}@h
zjhT<Ve*_Qv@mFuJM1qdPy9juild~M6q3qn-C+$_M!1r3QV&f6WQ1B0PZDaSAX0{e6
zi063l{xAn9U$_syX^rG^tp=tzjMi5f2~62(2<L!i&;JD_uwR)Ka$%~?jsbX2;X0U6
zvP(rgXOP5#))?t0F|7#4Gh4uoa1a0+?k91Ac&!FTz2cJ*sD8ZNl{;Gdkm)FBG=Ljd
zQ}mgSR|){ll9j8cC}gdP{|3KdMM<qV8ZV!tV>vFL<l3PgY)sxoxwTO;+ulYHf(z?I
zodWNbYlC`lQ9$tUJ?DkqX65`(7Yr!N^NW0+^8p{B@p9Pg0@vPj@5R+oVox2cc!xco
z@yZ7jHu@F%yq5XK2jK}9)^_gAY98BVzdpP_?k`69m8Ct&c8!E5I?$Bao^+4P<t@E?
zRisA5jt%&YUhM?OIHo79nEQ<jx{UWqXcF0R*=dKRM&It|vnyzKC?jq!{_eI`g@mW8
zNq7|!4Y!X!#GfYT{iHxHJ&M#_(&y)daiIHS=yNDWpUBFWCI+z<=8S-khNo7vVq=y&
z(Nm>&#-B$*S7@${XP5?7-~M93q3&W19tq4{8V4{p=J`drcAnFQ=7>F=mc#bh@xqW}
zpUrujgy)IH#pg$P9FqMu#Dq<wvFTUifSsrL!Iz~o@#PsGS@+)dm!5~ZCveMJZ<C(l
zK^=u<@?uYW$}vf^!Gi%c54L0UcP9h@Nq3mFN$i#3+GR!&*0-GxrFuv}5~AUP!^V%T
zLh}uWt&O-K)X}V#+O7ntQxo{*E!IX{YR_tZks#`P5H$x9d)!2A*0L5K%%**;B-XY+
zTX6WbB{+!jCq!T#QT*X4F6CyE-nRJJ=UaT}_~-GZqR_5Wn%)%rmaAO4=Mr6I=Y*^V
z4}pN7C_-lPaCkp9!=HLbGnwLCyU4D=DP#v{dv6z@Krx0&0p|}5l|tNYUJy|J)H`}4
zouv0Ie#>G{OKkDS%LisIaD0~$Th1lyK$Cgu->*qRfd_|P@%gX-_;y+V@F-Rq-?Qxv
zPp3o^l>Ph)ZJ6f^V-Us*(@(P=TdZQaG_M+@&xgOF-(W;irw`|VghT-J+2~K2Lk^DR
zD$fk^@HxYvm#oonK_|@Bih%%t{tj{&t@g@Ksp!?q(I5Y1q-PKH)R0m&Q48{eKW-zf
zYIeE$4_W4`%Xh9sKiBS`(YSuoeuWdg6%A9Z*UtFz(oT!Eg9EFYrFD#MB(0uyEh*CK
zO@Ck2y>(7*eFoD_(gw1rzEWk2Dy4@DWu#pu(f`k*gwX9rRir+a{l2)D{K}y2UHP_>
zsn_<U`WgD(XmY(jUE!Wf@<Cs(rZm@*(%g=?*l_eu3N6-OR<>IA#p0g};^lLm#YF79
zaZl#g@MB-<-1jvIkvTZ$p7dn@`<ZX*9$vqypJ`e|Npd`Kwf0!wpJ!DWAD;A1e|QV^
zKCxm`g{1q@h~xG)q3A!a%2}nA*R75%!Dw|(=E>^RV=Z;3N#)o-X^fW94va;=-9QRB
ze571&N!~v3jGxg$v?6wkNBZpZIip=-_VM@|o!oFU1xun)^7>q}Sl8#9WSHOm6i&H9
zeM1<QZ{57sCR}3VKO0P_`{#PvUG(emxpk+nwI%D@q{`SCCZm5c%l+yC%V@m?rh9ta
z29JB|keWlUznGtJU-L<5^FQleWW-)VqT1kXBpzt#yiRO!Ym}9W#rpe>yJnxC!u&J{
zd&r5BpB^guJSIpZMst0F@*cGI_S&l<=Hy%0{#OUonkBUx$Mzggv1vvB#JKx_Z)lzY
z+E`a>?iyILzxj%sclqcbHXQR|2u}QPK&H8Y@ib5E!Itv81LF594Ml30Up`~fBSFm+
zsfnxfDqSjCrircFz2f+5o$9^lmq~TUH|q3NhQ|SeCBg{(lZzU;mtqcJr=8}$UNg+*
zRFQR^RE0BdS+9w8eaR~8T*@QjN0W-oBjn-1SJETOMH3Tj-7X-vL!r_ss@!FLRh%RG
zae4Aets?UXB-nx8I;rb%CEC$f4%;^%1(+Mll$qC~Tr<U0QRgy3hflTEuk?D|1~7Q$
zexJxIO2!wmikK#z@!ae5Sl4{ky|z%=(*F^C(LLC6o2d{ka>+8v#hv4Nvj0@NvYu>I
z<E6F38tzA#noyOq+AA+6MPNOC`(jN(7g`~Yt81xvWS%%vy8Rcta^$G#!Co&;4o<9$
z-^UzSD=EtSuv=DUtsk|$;P5X=HgY3|rjMozNM%}&ua^xQZ2Q?zK8I8qiT=sf;JR()
z50Lh#SEtD=D304;8FyT)4U<pE$YoG)TyGs#SE6t&COxw9X5G<p9oCzhRs$n<+ACNu
z56#F3Y#$?JJk_FjTZ?PTY*#kw^y#4v>VR{MPJGiAtG_(07W|CNj|Od_$VLlfAoYAR
zg7o1(JoTD+|K?`XG(^O|w=6a=T4qP!8rHj`zj3G#c3V8LyZuYqicO8N-BO%WbXs~R
z^#1sU>XM49hc6O)pf$+UtaoMV8!7$7;f<zYJ>-eBR1d(owKl=L1h(r$@Q^y6d|*OV
zx(9B-@G#=aGdKN2AKNr#<B_oT_6b!nS#aBgIbMpacezAPmN-Okd8JcbHldoP4xx$#
zqQ^7$eI|+w=(81l6RKx^;IzFnhk=dB{O-FIvasc9qU^fjzU~SCxLBsUIlGpD(T)#B
zzJWFBp#^EkvN$HnNEEyG-H!;UwXs4bqv!8};|py{k#D$VXX1Ow(lAKLaJ`6=F><53
znI`F!6wS#=>6Dw4E!BEIXVN5UxAu>l_u1yVz<+*dMq2^f!Dqo6OEV|@lC2VXpU?hT
z<U3jlrdW_&Cj6phNgd?4;W_E#^zK*JJyIu7_H=nqZL@p98ko$K@RJj%NxL)gU;eVq
zE|=rW1WEzHFQ}<(!p^(8+{9MJ0$uVolbn{ZS+|WF!K@m-o*3)$)#Bk&%JJm&Ef6MO
z?1;C*$0@}HAr?uI6C?E*I^>AqI>!^Gkd=IwX;;Q^(+R^R)JHi5-SHD+<9vSAPLEBm
zpzB+At3K!EVfbkJ1Vu}1a?gl2$-^m}1VQ3|O?GYL;?NB08q1ersRun~`j5Yd6)~~$
zCUU1)Z%CXw5B3;pcb%$B7$3Pu0Q>GQWPa({9jn~y4}EN1!5*W^Q;yDSOBy92KUH_;
zT3bBtl$IqG2$wOMa|h)R^ejHaw8*YiK0Yv7c3WVgl*`~THh+qOxAZ}0F<q+ZD7H1}
z_$Hk`lE4Y(&%Tj9dyt!<C-b`%n&Nu{NO=b&Lh}xKr>tVj=x0h0$sOsfyOoAh|35Z=
z+VBRuQ%7PSRp0c*&o_DSDV~fZ2eB^-V|TNb5}4|KHs<<8hxqnqICWPcNQ5jyUY${Q
zVNHj|B=+jPsGN}7TK_<_$b9v=%Cr?6=9N-HO@hA~?W%qL*0I_XD&U}-wyFGO);}aC
zP7_B_knEEfC5(z(A^P-WI-k<fNM^)(%dzqkN&_owtFZ9pRckVP9D(B2D3_%+*}Rp_
z=O^)mhy(m<DXglF>M?YgM?j*mD!ZZ-f8YTGi9303M)7KqGLpr^z4xcTm>j=uv|1gz
zx?Q?IU!f@lt~plyf}9YpnIVc^#S$an9b?w$RqW|-*C>^SmKT%PRN6&J-<GjAj2@+8
z#%5knk?;%ud^K+)mcvfL%<6g44R*u!z9WZP(~uE!eUvDo?8=lwN!7;depvr5&YXV{
zNe?T$eYrTu2_jdB#rL5S<?&=^%#}}$2VxFe$>H@B2i`_abE#-W=>z`7hUKX8cJ4`N
z7npOb;D;h+TNOJjc&T!{DE4$RQ`#mSpZaTwlrH;nu`x;<`=YNwIA$J3k+?l6g42cj
zQQsESTJ}|qNzb4rJXL2y`#-T*g>)1>^Ny>878!HrPxLqHs7Q|DKp7YNx8qN2@{nib
zS|0U!cOTQOW3;+5S_0H}j)k?~L-oQs$1W^v^0bL(E8sz#OrYE)`2AC9HKy~P8j0Pg
zkW!xKUE1r4uc`S5Lm~6gC?TWK`gvi4tv@gLYl^KER*B}Co_(obB6;m()o!aRsA9Cm
zw8$AsrMnf~ccH2@IPiFsi%T6FOyxZAoG-_y=Ys9KsMnRll0#ZrQQ&#3Z?r&NFq6iq
zYEX=)wJ>OIgl1Vbk=mgp%2YK-$SK{m#EQ&geXqt*&#_Jb{^P2o?>J?iD*kj#w;MVT
z$eu86HwxG2T?|$`9&H12E0~*pqCfP$BZ{J>{!sb&IO;hve+ss^OBP~n$dQaftVZ~F
zDR%cL4FSPy`4;=1r=4XlLd4@t`R2F?&Me$j`uteG%US%ue|JsXD!ht+EZAbTV3g9P
z;8qPzn6+q)m-iNHLynBdY0cP;Dc{d7ur}ndrNp6rxo$|F#;V^)uef&Dn6}K6<bq2^
zs;SP9I(=?{)_ST)r%v)9s<(45n=?Wr?<gO(p5_YKb8|ma!6dK8m1!$p0mbH2u#xH-
zBUVUJ-p;n;{)>d+{`Q_pq^qB&+`@c2NqG<3w;}Zlr`q~Gw2A#Jw<Uhb(ltnF?|N`s
zqc-@M4%;e(^FI5X6{!*Gl&^=QC<NNaTaJ_)^J^a`R?^(YoAUFxfRxzKFh6dU5Eoh;
zGFPx!4X6zT&GrqJVaLH^j@reAv<piC&XsC%Txf_=NCdw}HW5Mye4b4`7KuSiRgl9j
zqnCD$*nD6QN*@lKO_=!4n%Z=;e->il^q#JzBs~EV0LOI@+4ucT0HV2h1oDnhyB0*t
zppA^0Gpn*o+UB#l%Z<<(vZHK#8TUIsO+(Fdbi}ER1z7i72vq~50+xPoq7`OJ1j_KY
zD5!VNzU$_JIH68@U+XQ-;KYxbT`<Ka-GX0`fI6*6&c@vhBdyjrL3A7;6|hW`1FaBs
zy1y3C)53}yHDl9k*8dpB*M6h6Cl&|qP0qn<8l~$eQhOo4fDnOU&;idHb{;{)tnzvm
zlPUlj21`J;9P8t#Btd1N%tO{H`P9VA-20!(6P$*;KS)$ip|b%*gp>JC$$jiHM`(f)
zS=;2Tg6tZ_1FSuLp%l&gZXSofmZdhQ7eT*^Z@$R|j$@NMf2~cFzxi^MZ*qi4Y;Xds
zs$pNg66ZwPp@wyvS@UB>i;RSXWBxD>%K+sR;y?owhz0@hMXztC`i)?6un>GS43co0
zHZ8ZtZJOBBFg-pNfz7ynn+N_$xNq+-_vZMvIm&REm2LJqDG&UUaO%B1kPV3hXbKSb
zBhkvVXo(1Bt#6Q}?P|mevLc*%hvwa6OK&Nk`t_;5%bpsH)QHhrpQOAUgPS2N6u`|;
zS)?_I7w}8Cjc*H7XY-pwS+zMAI3<o%B9N<c9LH^ku(bo$p>{xUyF>gP1T3U?PyiK$
zRd!Krgmj7*ypMGJybh^F0C*o(w(|~44`36yEF~d(6ZF28Yn5r<L&nPeBL=g>;sPUm
zUQkM*oJYt}atpf|g&-+-!1<kxHciK|@~SHU&`^aF)q(8Fu8iB=ns<Ye!SVIG>Ewc*
zS0sG~iWEs2)EuiSL)p<aIg%H=z)+|V?Nv92@9MA3QJDs{l#znp1c`mi(Ps92Bj@#2
z)@|?#x?u>=iIf{aT3EFaqN1wE6}(`qR|UyK7{TFmrQtjdleUXMQB)^80vwK6;GL*5
zpLB!L%f9zdIZ#;ofB16$_0<*}P!>PxNxyWX<}D*-&USiHeJY@cd7@~A^CF+FN8&LJ
zdkLd_94gY|`SHf<kp=7xzx0h*#DD<DDEI^)hFMg;P+H-#floT=c;M0!qU_nhKyKF?
zVxMt>Ok+0HG0cR0*F;of_|Y#T2n9<*Z=H@J=2Kf?q`ZwfChX@;?C5vM!Yn$yebX!~
zaGG~5iJ6P6FD~QgJON;-zrfR;2u&EQmw>qwAyL}`eq6QqF~{BR?}!&;T2!Rh;||To
z5?%+zRqTh2x-f?c5}C8T3}cxsZ3h9>T)4A-qf^ZW0tI{?wV7~iUfa15sQP&GAh662
zx_@b$;$Fa>@Qyzx0L!$2)PYyB^9~7fMgYdNqB7F8aY9pL>Cd(6X#~#j(2x4iA>m3c
zojt*t{_r&5g3}<dEnl|`VwZK<sEw}iL66g#*w;(h?FiEACd39{P&URNsE_H^6mtU=
zHbHM3YN{ptRcmgb$6&~?W*_4L`lMG20lUrNrU}q)>7C&HX8w|-iz{SEP?YS$q5>uk
zG>=t}Ru#9h7tQ3Lok;j|iv_u7f;a663<$PO_h-sci#Am_ja=h#=Luk2UQyi2X+Lt(
zsdpbK4jVA(OykbA!5qbHpt-OIzre|OJi&5a@C$jE{fIN=2o(4AZeh72E@Gr|GCpi3
ztPCvzOd@<2u}X$eTx~qT_xPqxI+0DepvSWV5S<1S{M~ChBo4F_k=)<+Vd{+}W*ji?
zOdrfvFTEaDOgO}vVf;$i=rNRh0+5^&Z9^6o$FVDziPU*WmFXNx)*96~Iwzpjaa1j2
zsh!nVJfWsA!50dpyVPgtY*kp`-gpeD5v6}DHkn{)M^9gC6P>OC&ozoO@euttM5j0q
zPfV=!8jPO2)+Rn(mYo8Vmsy*WUVGO}X}D5$LJxd+xMPh?GSOa!tx3NQ@~RHI0zj5d
zYzS8Fui4n(kWkI4=R-+^jg&iQ6A1E5I_h{b>PE2nu0>OOR4eRQJ%>l72`C)tI6ycj
zK-))<(g3tt;`5UP2M^~IlqdV+>-U#iPk;x}%m^JJMC_8jE&Kam7i;jF-uAw5(?JP3
zQ+gcZKvqIRZ|L;q%?X^}?Z`=Aq*ZLbYy!@k36cgb20CvJ_RT2?KjQlKO$W!peO(E$
z3fYHXA#~0Y=eFl3q_c5QqFkXeX;OE(Jarr_8?#};AaJsh5!5YzbK&IQv_Sp8V}a=F
zw#0GpC(MF2ohrNZTDKtv4u__Bo3>$;=)2Wy=!{duH6-pbrvn)U<>-$3VNu$LbE@G1
zXcmHhi!7x-KyH1gqaU{mM#q;4O0Z`(5O5R>fK=Ry>2c|AE$6sk9kudA={?Cg+-wUS
zCM_u4o!PUY)7?j|_n*oqc9pxRlY7}XARd*OwI)^62@pF2I(wpYilt&-XhfN(T{>Iy
zx=ekjqgGw3r0otjT^j=rfdvzfcnC<<a;nl`_8HTIzbhMtmS**Hz)p+|H(>E5bO5QK
zyhheL&VdehEh=rf9UsLOm!ASJnow{yVdA!pa6qlFZLO-SY>?qo#E!dicIyGh$t`%H
z6MlrDA?!m<Dljgb90;W6?`$kta{`k-FfHWbC*{w0T(a>1<Rg)D@?KhG+brj;=H^HQ
z!wNH~_#?$+JpG~EKh9Vp*<GJw(|7%d%X3OKS4{lJ{gT|_wF0J+p8<*Is7amY7OIVa
z1$d$2a|A)f_oAz9Tg`@&M%?(*&w-f4m~+BfsT1tm0wMCzw-4J|ap-t|aSq{nrqu$j
zl?^(@XlmUV(hUS4Oyx2nMh}gMfv%j>Yub1l)R+WX8H-(ETrjVSQZ}$M_|KfpqBT4W
zIUl|Y2R@rS`$FBarH6y^f<>qs8vxMx`Nu7=$bnA4mJdO=MA*6h-?)<hhkc(Wy%?TD
zRGY@ae1Q<PLq2Er7!)11^ZVlTYB~$y2D~R4Ig{>_$(kCZHNNkttsW|tBz-q-l1Kc_
zQNJI|KAAw?<@As$MJYQjcHD#y%{e?N09iLcD^>>kactkLWs1Y+dlSG+Bcy>=#LtXa
z+l<z(Vj;DGCqu(o+T7Ta(XCiA`ob)3)wo@B1Srm?&68pw^no+OW-H0GL4S*daP{fa
zT{-GGuN*(3qyKP+F?mN*+F)7BAX%+QzPs0IxJ7AKl=MAf_toLOMORX{5w}o7yT4YC
zBBl2hkhOxTiRoy97Md-C(O}p{GmU!q>%{u{T>7<ci4&=T-!Uyc_J^16O**3159PXE
zgJRs0Jj{kclb3bO)o7B9^m<Q80yM^8!G8ANRDeQ<VQc7HZiB0EbQQ2l*xapo70xVP
z2>Zcn-e(h5Il??B0U+{pMKkir_<59{;oYF5BkdVKGc99Sm-W7)^_Z@O!B(XLV(01C
ze%%*+@vPYj`kgb+rOuP~2#5)1h-Xs89ab$uzoRf4#N5OOK*V4#ETPXBBPK6qY1*Kl
znWO_aqVEKRiPJAt<hkldR<t<wJN`fbadi@m(ZG<iPuegm#Q^&qbnMm90>T{`@_9#M
zi<Z$3Avq7G^4l048_Zl#syi!*30n{nP<tW2gQqw48P~qCHJqCQ*;WeqJ0EP|Sy*ft
z%>O|GGIZo4XhyrAaq#SHa_qMjAD-8y$Bz_2hpRIVKD#n)0XGF&C8V6Ev4_0adeFXN
z)%frgIM0hcdS(93Iw5~&T9UgHH#x@&xi6o@h)XOfBSX%%J_3TZAvVRv;%u?C;$}An
z+l2g`1-4eKStMEyb}#aU{2h+qjrlu~LjF#zO+0Jw1(Odl9CL-#!*e8CR?;~3JD|_D
zmi+809peV+9yVsOFwT&hd<=_(yrvIZOSxV5v~nST=Rs>9SNU8KQqEg!(pghVOg`v-
znY8k87-*m*LPs$i7zdA*dGNw62pAq*FPN2(7u6@)hLtvhZ44Rcb;2sp+c?(R4nu+3
z_&R0|1==0Ait9xs328W{J+^_@NU&uv{|D`>`n8Z6<ZW}2n<m~D(xFO{^jI74XiZO!
zh&>JB=ioX0jv47VH@iw`7gP`FsV_VG5)%>9=K_K61#MJCN9=Ug_7G!R!+hg9hI?0{
z4S!&#MlzQpu#jnk1y0(8;-b3iiJ~<^ZYm?T5WlJVx#tDGTC~X$*`=DSQ&q7|`CmUV
zuT99KGMK~4Ou&9d$2{$9f{?=DAO)h|QMjHcBCK%E#unlgxqi0L!eOWD(RbyGcEx{8
zF5ttUIf;$$I9Jk!4>0Q~=*&mvr7#D(a!mTg3^V;g{G5CjuRDXXurK~|M-dtU8adc-
zi=SGn@XFKX1gFk2M`e^@%LoVtT{vtA4984fOhEpLP_PzOK<p!SX@ah&pmP^aK)aJ0
zxClsEXaQ|%t->o%n}yH<>WG=mMbHp~3n^%hVYNs=PmW~E3B4~Rn=rh59tp3AI)Qk{
zWnLllxn~!d<E_tZWPOCw!`m?j@W+)sM`-aZw5jJJ9^uuHovuVvXY-rFi>I)=pxTEs
zwWwbUA%XydtFdvg+O&}_nMcX4_y~a{yaLV8C9>A<v8^qRZWVTGk(!0wWn317{FmUS
z4AB=QVJ;jl*m$WdoJylf>|2TpxGYFG9YPD5s>?+{5qt$SGW75kgd2!-{cBkIdR%m9
z4rBYFWH<WYWsbf*R*sycOJtqvjUC#xm&$zfngjWORqQy}GU%7iJRhQb#+v$d%J4Dt
zP}*nwQnyYSYmb4xCXdhfB@Su|Ca1++gAf6;&ZcGU;i3WT95{^^h`bG$Ro1~F*e_js
z*~+e!OmZ&6<FiQoIwM)Vt>}(7we@lAyBi(70R!$B<+5kj({6FvlEo*I*)@|NCIQo@
zcRIOvuV$+#>BKt6pUFq%GG?)Y)*>+Sffo(%mzT_IbMSxzt>B*TLPJSlE8<{V&`vJ8
zjonVNA|6<#R$*f&86`7l%H!4Yfy(Gx!?*gd@$e#f-KltU0<6kWRS6FVz={=X6x~??
zvvf`BlfeDZr~|neTU%Sa0l%VZHIA*J=E~@;dQ8|)bz1N;a;xSPHl47@We^@iw<jw`
z1mZNZ3z)vF7LHG26_a*$Zj1+Bk2*P38$ATVePA1-1jhmRr%U@W{f7jY{Koj9kYAFF
z@GB~#834|CV%$a~GK$mFt~SNtkIjVUmjoJ%#;EG(g^{?}Niue9w3$~8b7Pr2Qy$00
ziIzuFGF_XljA;^kn~skrh3cf6;HA`qDS)7zb=q~~K(wPJoq2wcc5iV@DjBtN7@7UN
zdk`<BgR~TjUEGMpgr7>rP=L%M<(l}dRcmk&5;lu=)Q+1bJm`WCO4FUOAejRMz^fl?
zH0`pbAzqWzM;63BkZRQCCcIF?%05D(p5#BkTn`cp>rkyxyMHh&=sN!PONe8Z4L`Gq
zPSazVG{`V)^NStv(Lcq7As~SlH0gRAK6Hcna*-gzl$gzuoS0!Q#|4_$Jbg{2g&@PI
zm@0Hwnrq+Xh)~T@U(1`g^*|T#TY<)bF>Hrw*dx5h7kjF^hvySX^ch!-0OZZJY`o&E
zkyP0%j8L{5mR7_?#gGsM1qd?UjIp1!VGeke5qnzcz;h`aeKC=2B}d#9hx(-2Dufc@
zH?)Nzm4s{AtRz+1by_3o<i+!pIZLkJ6}z)yo6xVIo=<W#Pu=BgtX*jG;C9TFyM+=<
z)=3k*oe`;N3+4BY7p=5JYla;@l$6d*Z=HVPsd5~Z`uT9>xE)bOF24u1So<O0hYfP;
ze5=?CC6=crtGS6qL?~^6x5HSeu9&u<mcbdJv_-sFhU-#R2&1@1uhp|+qZx{;@{>?f
z$yDOnv)arJNJ1mS(O*DgJ)rNx2Ds_%G@*3mwn84>AA$L=6-u=KElwH!7N>F-&n8Uc
zZJ-#v20DUhp3Vt;7y=lQqrwO`J7Xa?Qz;^p@{27V{x%&K?<c}q_6xHXn2t;|qv4{4
zsBZ`4mTDLO7TLA+KXdg@c6^om&x4D7zWV#OCi*JQUjM{E&1g1JI_Ez#vsNT|5?@uC
z9hz>|=zT0kZ|i}?q~)?H{+d#A%xy2PH`CEHpSD2ZP*d~wQ(K;$)ap4|d+NIsi^Pw+
zwX&|vjO<Ij?tkg0;p}+DkOei)wa!%ygK7;YsvfL&zaPHbVtKH|&NP#Tn<w(u^ZFm>
zyiEsxRQOVB=@*MHUHak4JA>`$PiEx`)?q!nrF5iM-VI#h8(Mci>Wx{-Dy6SHwFxu5
zk`K+}{iC$Mt%H`yr6-ayx2InFQS+<Go>t4ypUzokKD=J41*3%@-n6}?diiR2+99uw
zxK~TX7N;&3cgottZ^h`xbKwE3@6XIv{b9PoAKo**5GmV({v=j>&P*}F?_bj*)Af}b
z9Tu+Ach3^9wYd%ZFgTxSv__JbAQ(I9ya5QUOd(bcr+(qAssKM7P$u@&X8O5zQ@1`@
zxOUG}se?QIi~giD=UiTl21wSk?qt`~OECv~W~zC||E_7K@ZU^(6|tv0kB_$M#1!*+
zwmXD5MXBFifr23Dq9+TpXixiJL~5`@G|kGjqrR@W(`)fvbm<)V;6CGGty{xA0R;ta
zSL8mrWe?qTU`U<&p|8o`bTCV(ZF#~dh>BAEl+NG%E(Qdz3RT!}QR91?J;^Te%Qnlo
zzo<>LOI2E_QABRPaA5F#T~2|Ki&jVIz}A2Q*R0&Ug0)J#B{J)%NX_!!1W>y>lcw0t
z2etcGvcd|~E>2^L!&i5sr%o{`x?f&W+kfrYy`C7k<nF?nJN4DcZW>8SE6ww~0t%v5
z-cm~M?hH=Xe`#R1(!5W@nNnB6`}`}Fr#Q^a5I|k`oS7W|CtNbKFWaCpieyx@M)K~Y
z4Hp$YEsD5rr0#wG-Yw&qUasySoi}$Rj6B?CULLH61)h-edzM!oY%*lppjdC>5yaD=
zbly6f@RjLQ-=DtJ`F_FgXEI{{LVwDPF?*xa5~6TTY5z`@T{7CaKbtIja>j7m3=w~Y
zNlvq-IM}wVF!t!RUemTHB7`BP%=6cp|NFOHU{Aq?B@q%c<t%mOsqVzvr|%7{%X-j5
zx}bkUEr2>8e~aks$Wx2L`^299=bSKy)=ye1_B1gf>hztiYYR+@mb1x7p;0eZZkh2h
zN@{Myr_O<cNPUp*ESIewd^!+d%Tt3AyDjFeo+p+u{lXdTBcYNC30J>Cf4ZhD7HH`X
z(h$jQs{dnwWn%M+|47+?{l~wi-<+$y`pdk08+VbKm;*ujnNI>@?)N&HZqw=cEhBFK
z(1$w#7yM%KS5sXFbOX|>oM!CW;8U}uPvaNLqDr26vpQ(@`LDm+V*lIsrr*q7_N7?l
z9qdnG(=yEs<bKZx%{ZRqT(iv}lDvJgNW~9-D_r}nFup8v(@hfjLO`&(p|_iMcw>sR
zNLuL6vGG$D9J;^6`|+K3)y6KjPZ!;o5#1ZS+&n^F-ne2grAF?gh7ILT74Oxg!D=Q6
zvzm<fvRrI6`WyV#_by764caO-a9OXgN*w92Ldw;vOTPB#Ec!YiJ>6lAm4^OCOOL|3
zoZ$7;&UWsmhOR>Km9x*26ofgT>?%DobU<0T1-DjZr-hb*0wbj<C!L+XreyC5_lph*
zq?F^91}|?AANOzTa{4YhpBmAhkza6K?rWZFX}EtK!m59Tx!a9402PM2Ep@)w0JOGq
zQN-V=bET&5RgW#onVwMAA}UXe>^f9R*&9He)BiYEj+#^x(pRNGSkBuO*g*ABdFo;N
zUuI19f3|JDsr{-&=uZkZQp&|-uqo577UzAdRdl>W|0{)e-+l9Y#zOCu2%5H_^BG!H
zd51$NGOnb*{ER4S6c;$1l%al)HOYO?yFxs98kY((JWV@a%5y~szlefq?B3ZxnGzYh
zp^tA5_3ud9VKH43lr{!a+PivW)#mStaxN@sP5(LN^-x#Vg8|po!FIH(?C>v7oo(yM
z+f*T9Di-N2@D*~1*Hr*#kXCag|FHNO?I@g3>`CY35>Gldg<n__bS{JUnXH03vemy*
z2WRt(Ca!!Tmhk(JD4MVhgusoisUlB8%Zd*Cc1hlHW^3ycFtzutXo$yV)p*GV&D?dP
zV3wR^>_za=<wFVYc`9jN^X&76Q?lQ^;2S{BFnzLWA%r>l3LoQ^F|@RZrloZMW6)H)
z)bORpKc|b_s_%#f1bgarbdm_CJcjIfS@?r(pcdbIXI$AWnA7=>f28Sa8-L8iD@fkG
zbMia0cRLJzPIjSRl*w_ghvWkmXFVucVce{AG2qs~)~?#~JT>!pAiT>cxGpTccJe=-
zwt?p8G=4v;T6vGv=fOevMwDa_Rzob%(>d^SKyY~F->3T(8*;wo>03&`M;I>=P7E(s
z!OJD!e8rwN>4639zpPghDXaN(->!&mhy4+^%p=l?O@S8=4CY=~;+lTQV;|3W?ciJh
zkeqrVuu8(iWM7@D_yNU7Y5#7zX}s5BI_6Nqy=aDW#_rr4LUQ+6h(Y95nMXKwyq6#P
zy({D*&y#3YnBuTQ@N0qApjq+5TE?~X^;~AhDF~%i7IMi$6~6Mp=~4Y>^DSQbt>LAP
zH+P&8skwS)O~P8ioc!OWMKj@sV#+{sA{l?h=YKB-F7}wd1o+mwdg}HIb>+&2R04>a
z)d#>(g#-}NOx*g~3WDUUc;3!{7f|$@EU@9;ni0A}$}arZv#RkuGev5;m$Qq)<;l_;
z#>G1#+T|hq88}o*w#g#6k>rEq9C$8b7K}5f{6e@z*oH$+MO(yPw(v_s_NyIo|9KUY
z5@0}0I9;@HrFjHlzgs`0;iaD%&r`VwL!XQVH&hpX1w&P?2e>vAn>KB%-+!&M7@ns`
zr(!314z7rtpXVK)Uo8yEs+ZkT3YIsIP|5ex;JpMhD4YmvDzO#3(SszK5gpG68#-w`
z4ND>z!G(-UvAOzcNFK7jZguTC{3p-55eL7uNO1T0)ncQJ{Ul|n^Ct{V%;xQ^Q-uM^
zHv3%F6uu90Id;eIu=!WH`*|LGJWO15{YtnEzm8i)_YJuRIeoQp*s<BXZ~uKrKh_Ip
z0z+*A>7F3qQU3QRusv!gN;iQIbV$Ff<awmwheT>tn+X~OKnC>N**F_TrnT|9SH&5o
zIjr4!+qrg2z%80a>GJ$M>GflYr-FSxoAd+vyhEJS|1o#vaWUs_{I~3Cmy}&mOT^k;
ztnEUI)a)+Gu4u`zx~5p7bdu6FO9(}~>qtn5ND?}xYZ^&HDoLfLCQ8yd(lOOEGr#Ba
zJ*MfH3Ss}7*Q-C8XFi|LbG)DT^M0@cZFn7J<=dc^XVUFQR{Jeft7D~N{br$l!=D^}
zKsIG%(l+7N7y8rs3_cN2SB5$c>}ugykL7XYvE)G;^zt1n6zlUgW-2M4uAM;Dr5QFn
zw-q*YXjSxe-{K1rx4hw(IHLSL`6U+j@+E$fuD6SnGi(v8P`44}L#nL1!7JKH>_p;g
zO5C+P7L><FWhcGPXY;(8D;n8HOZZ&=os3HD)3H$${)vpr)Eg!Is$lD;&{Strfnw=G
z3O9z!PsylEr;SX^W<+|nof3`{7Fn6;QW`U<Jhox5aC>(<S*_!!8XX4Gm-4<I*n7gk
zI)y)d?`I{->}?c|&xntRP#YDnAHU;dWNC-<cD49leRHdz>>gXwT$g*mC-@cQ8b691
zx9NS>4({_=FV050+ZR%J=W6y96|OPYjl5-BI@>?;;CNncdVrD;>S29V{@n}byt1x#
z9M15kCV1!yS~-a>b0EL>*F1gFH0=f3u`P+#Z1tVl_$B|Izvy}JR_p3t|2?lf_sIEZ
zkE3#tL6oL7Ot~r6&U++B&#Th)`J!TjJ$4)v8!|W2?)-Gh*D`HJP`aX=k!QDr&gbWu
zX+yp3Yh2GB*9e~Bb*Ay_#`0rgj#6c>#%XQdgCG~fahliZcLarviM(8j^B6aq+F*)x
zQ<iYqdRlu<5Wgzje1d*PNqV`1+eq!Xu6AoVhi@*N>|s_dI1zo`v4|a^#-5>E)nrEv
z?>IGWJ2h%#=?}_Pe`pG>c@+sKDOX)`qeiq&_(4<BD}hwx{!=>wx4iH_9Jee}F!2NT
zq_7$*V174f&RM&Ar|RZab=8&&4VoYSrZnuHoUC+`?R|=}x%I>`Q4Cc8Qg|1${LRI0
z>C$?3YI<;M^(z)n@U+5T$y~oQl;&wgYd@P&>7O*uHMGJ}ZfSv9=xR-cT~h#t_+Et8
z-oepY09Q}3N0wV&rjN!ERmil{6LtbviE?iJyi|q2GN(($k}_bFS9m8n<LKtmAvn6Z
zf58@Lx>m0H!*bHqJ6&SDaaHQ1z_Ojo>1yWIUQ~E```2w3HdgAZiQUO!Gv2?dJ2=_x
z9g*%$8SERv<dPsMdFF-dl)5=B*+N!E<?{0Qs`8A=i~vEi^Ie0J>C0!Er5-mf0dVRF
z=u92mF9N$Aaj+W<x#*JWdc3-B;q_EklTE0?y>4d#C^GuAyje{zG*0nNkMDDn=UFdG
zi|UK>S1arb59^xNwW}=-b^Ymf#5l_82PfdGa+J-zP&$<^_WCnif_blMjo7cBb`)@$
z*xU;T`B9bQ8C6ufT0>#`g#Z=)6T|fTE0qtOs~UIbarH0&Q2_s5D5}x5@ERR3{oF*$
z`3B9~&tX75q^8gR&{X^P*bMaPlAFR5BU729<0XYoC|EW)&<VJH#b1OA_fU4z=FC&b
zV%C*l&SlMluK8k<<=+MkHyv1WJX#+Gt88@7e&D}B>{{m8+qWO!2+x;PW|hXV+?yWM
z3ko?URYCmgL6xB)@D59@YYi!GRYGpHT41by$fkt7>M;oj>jWTixd=aBhT-SPj3zq;
zStCZ!wCxJ^BMw4UJ~w})-}#CI=+otickapsET|YprlDsowkeMh12G`k`MNlQ=yk|h
z;O<kAy5x3Y2dgZIRWy94qegC*w|7~}@89V%3f8RH;AtB(eKRU)tCF0A!ugbQVz@y-
zzApD!;l-qil(tCwrY!b#o<;l8L`IeCMNR(iA)5T_OFr|tE2lKbr?)F7W~CVJDt3BN
zqGpTbHs!DJ(=%hk=^?Ch!TR3}yVPLe6Hz)~aY}g(WqX?qe@WUF&elqO4Y6lfR(O)#
z$PbQA6uyjN7ikM`-SOT`zfR#df{i@5rb`06%E~8e?V3C@&8PnZ+%{KXl#%DV9iYtl
ziYuaKm7>{_2ZbFbRd9e9ZYh0mx=z<S3N=k&*<b3h$N8)bD);>HnB*kD<}POX+lt-P
zRdjSmTaWRhREr8}rCSfOx5cqedX+A-Z@<2$m1U;3FtpX7e-El6HFVwO$borKIlrqH
zb|GX5hh#+_bZ7Lb&i>GPn|p;j3Q`;db<J6}wtIHQjeGgyEd&v7Z^n}JyrFvc<fNpp
zU3l)UqC-t8ZjWg!KPeXZPO`F6Wj*x3QSwXJO+nyV?CqMelB04?3LgIQ(sqo(H%1;>
z2SRNWjKaqQCJNxqXc4Agj$`_O#4QW({zN^mU_ai*15dNQ<R(DJ2oMUXw)8V_oV(Ia
zZP*$;tXDT&uVBT`oxM6JevhV<0M`7IaaQ|*33!rv`+Cg6piE{Rx|eFB)(q`}dqd2_
zjJkTE+DFxM2_&h7ZQ}KKe1!|=*{hNEZSDHZ*oNXJI;Y-HxEP+A(l|zxu<dnQBz254
zOHTTq>AH%Gay9(I@<u{;3n1@a142g1U&pI!1{M%L3$Y11)7jw^#q&wZR-!)St-Hoh
zl0$C+Qn9#h<%Z=E1IL?pjVE*@uFIdNYQBXpC=1E<YVTMq_TGxaxi8p-m2=#3OAA~b
z1epifZKZ`(ODVCV97lY%;LiiH`Z?Q1uD5pfhJFXMSkH-C45#oG14ObaC$&kT?emt`
zszLSy;Gxr%@e03Yj6t7P>iGRLUB&1ANKuZN^3wPRQA0TM3^?966X@V9&R9#ugCult
zt{saGa+x4qS$dj-g}Ehv^pZ+j{1M>=(Y*8mIP|5BA!1#m3|o7T0x7R$A>5Y*buAP5
z^+D{mV7P`9hbkY{F$JM{w?+k)rFKT|-xUl_OcQl1E@B-E^yL#2rT2v;FO$QXr*Oj*
zYli7Pa$1Bwt&&~;=Yfo~PTpi%`MX-evEc}wIX=hl#O7HD!bVx8)FY?-HQ4WhvdSxY
zH^KS0Am4#dY~9c}=dqU0ml!W1OJLcMKMxGu{kWPlRn^m`;knm(uiIje%ef>=&_vnX
zq~gW0<8=6IxK(lzo9-K?KfV6}O_Ik@dD0~5@H~JU0aj6^xW_6e85MUcK7E$YO#7sK
zbNt-Ij;Pj>G<~n!>|iBsXEtJbIg$H{5nSwQ=SG9<QA3;I8>PqT2>wy;sd+}U@te5|
z9HSq&yI4NUb{ITU@D^=YXNvR|yO+llJ;A(S1RwW;x1j*&aZYz0z1FbdAA~AL+t<IA
z|8Lv%?9MoymP~J93&UO9DyjKTD8gBZyvou%SBHjdEy4DH%`tE0-|^;eKN&Le@4J7o
zSWyT~6ph@CaH+>Z`k~r|YN5C)JTR(_-mXoL0W|Kb?r=RyKSnLiD5<^Z!MRYWBFy3+
zZ7dJ&yn()!_HL~&OIe<3o;6wCzQW55PdhB=E&@+*8ip!&*yfR&v7~n3k!xU<p=M$~
zmO(%P>L&&~e|N51Ye?ZcBgNAP&Q%rpMQ?<JWBdX67B7B>!pqKUc)55Pr-M@LS)as+
zqIh1tPIJ7$d)hHY(bXh98p~t{9<v5^2@$_!vRc@dF5aTs(+bN2@dgaB{OV!1z6v_1
zrAIUCn^FS=9u4biC?}PqIvt?tz7qFVwqf${TXni|twsmgX{DQ+vS=RaFJphr#p36)
zX(z5)8bsJ3CWQ3&OGSJaKb-G^@>?^RVYgROr{djOh(U*(he8|q9Qt(GM}1~m#qaSc
zBrrb>F;Di~JbwdNKGvb6Q>S8Wd682;x7q~D4ym#Zb~ex-SacOTCZI2%zC83Oz^I8!
zb!=M_JgywC;~xc(o>iItOJ}H}XR$k8v2+;?7VRxTQn|uV$O>cc2-^Ie?kevd+>3YU
z_s<P}7(ZQZ02fbjd3bzOTg?#MQS1YUF5db<NI^*0#mv<$LdI;BjS)ik2E*j%m0AA#
zI+MZrtBegoPv$1NrmcpXyzF*2c`JBNl(o|IRhGtB{2X{%fs-I4)R|C;4>^2aZ}?}3
zI5q0{$~gw(^)jQsMl1oBniswpO8~2MZdrVFC)ddWZzjbp?$xFng?b()>NcJ`@ILFg
zPA`#I2svQKF8X|013I3>@r*{Ud3$=KOpI;npuQ%!yq|u5^q}1EfxM=R70?l4wL{hQ
zWp$U7_JXYUp9l1fHbfod)dWyD9A&R-jU!&@>xnL2{CqkpMz*(PCtF{1ag5E?7Aoi$
z#cmLtP0XIZjVD+FMWydN;t19@m5@h3c7^=G`QH(PWBATrHvq%svBAloA?1QHIk^j~
z&2qN#c>F0T6Hh6$Tf0vhJ*A_WL);#994br8=cwwoFAAPXnKw^lRGxdV(AnQ>ZFx#a
zX);S&2o*>g*Xg+Out9YYP+6mUwqhU-LhHmah)v<ch=<e_a;yFXGNaUV?+tE6S7s{x
zxY3<<=6H<2uyM+H(Y-VEY{}>ACmxilq%v6AN}6t!LQa~3$(B$IOS(1SbZ_vE=|^RK
z0t@zC*h>{Wd(|A)0Zu}Pi!dOK@)8>3)2c)fDS*9QX@2_!ZC-9^5oG07+uh|$Grx@e
zV`(_@y?X{;_+qRd<j;FSE&$m<QFZZIjD9f$=oe-ZGY5s=GRSE34MZa`#1Ka<;I_rq
z-eZ@Sr7>K_s?Kjz%JM(b>EayixhWy^T0lN)?s)qih78YVY^u%l<Ev2`J!0S$21W*v
zP+_2&3qsI8zod|q(*()E=-*>76M<z5wrhJi8q>79O+Q3k>Zn@+=VQt*Ys+~gO6^n*
zL3C{$JwAIWhXdg?6pXo$_Qcrw9q-vH#NVh8;by2f1}`vW&n}YjwoS4>n51lj`-A1k
zAE4f%L*V6{F1^KwCR7Y=y1Zp<1bRN4X&AZe%UqFJSKArQDbFeFsvuh71@3IO%BZxv
z#+TpGsIIkA7axb!N-AQtQrGN9=6u!DErNtyTTM)`#!}AARo!V&D6Kp3!^!WS{#xUx
z&f*PC<8Jdx>A2MSh~&4!`w{->LTi^A$E(y2@Y*-jX8OrNzjIxvxAGDr>fW4fnM4FK
z=`HbIb5@;k|3?jX4%0)Bh0!zgmx<hUd}dulwpmg<B-~I_<YG0Fie6szI#CcBAnwp#
zf&i(57#*t*f3PUm5bHYL0$s#js-p!EP|pX+Mdj0UDcd{4PDCs~?C!MH<!$=h?e3M-
z(w9NB$_q*~=fx3+2K%l-q!nj!?~z#U{XDy*^}au0jcHmhCG6R-8*7z2I05ZYJ_NQk
zbiSJUdPY?Zo5$hQGdY@b+~iB2ybKZ=`hF5KZ?Nwfcs+57x5Rt4V*1|DHHn8nNp|b<
zD}Z&K?FAD8H?Ufp;?@=~*cy->)TaJe<OS;UcpB8k2^4#7n=jb1R9Fj$<m5va^GXIg
zbAvLyx9NZ$C9C0{htV%DB4J)rIj9;|o$sZ%122$QM1g)lsY4sDBd@Xik|-;1Vn@2Q
zvV$3{ld<C%zV=B?48^4xX@g}<!`zk2`e2jyFsAYI6HmfBZ8_7sEgQ;H16ovtr`sO1
zUGF@auj{OmD&l?V8WT?m8zAU#RQ5h{njL}K5b^^+`ztJdGH7-_hjHdmYoklA22fd<
zqoeyEsdldq9__S?QozPG5Z^5Bri3?hESdKx@jyVSAgr-GOmrsI2|-f+x-<X|ni1M)
zDt*3U%i#DiK+P3@n2K7CUHzDdRtlJ0mpF5NXvyAkhcy_0iW(L`R;J=uV+=_egD^Ay
zgj>;%?y6&=R=ayGbSBdpQ?vXh!S%JZhQR=c@DIU}OdVUkDSmJsK0=dy?tjt8m;`h`
z)5K>xs(zwBQ!5_XN$!Dpn_eroi<wPEr=tbc6~Xt|9ha)foHJgo;N?=U5&%MEscy6o
z7XedMFK3(WyB&-bsyG2KfNo9Y?{oW@D(F$<On5m*5gRGN5Rf5V+YngImcHs!1ImW)
zUrh-RY-@PN3!X)}Aqq4qEhtGFsm*9qwVAIaoRt2s$@*O?o1vTMhCn5=eWlLZ9n7TS
z@Y<wC3oY7)TUZN%ZrQbIgf%w7Er%B5w@U#jk9NtmtuA;TC85)zP@EV7d9-FrHZwlw
zWb%W_vz1T0OFFYS<7<>MLr-O&K|ijn$JCZo!pPfC%~K0In!l)wjfE^=Sg)7uM&L2v
zTc$Be)hWp$TkKfwH3ifputE1pRglnD=1f)0yHqydK>Sc811<G_J%ET<PC(em11xZe
zgefr$92gI&nOJdy(INap?{9Pq@+y1u^fl)SaN@3rJlW;qlS&4{kP^OcDeGlWa6R-G
zVik0AOWrD#(E|XW_7B5U)ae3TW5O!g1%DpMKI^n=0Fx%<RNB0Uwpv46#*?+pPrcMO
zP<G~y^NKrO=eeNa*$b;o|0vR-of;9N$*AN}sv_XoY^kMov|+W))4*ezZysh)zZ(je
zbhmNPGg);A=LGZ{XGGH~1P%ppCiRoJM1nDBx3uf4!CI{kR~8zKf6Yj%6|5O6jN}Og
z9e>n_eA-gm%9>JPXbHQ`3n(|o*IT)Hb~&2~i0USZ-XIE@U2Iu28TEfgyxrLdVx@_*
zi8$K@vsooelrx*Rw%DDb3R>w|{;{HiF>&Tj)b&aQ-@IhJY?jasyz^n-e`ZB&?DWpk
zVR>JoCR=}SE^3G(Pj=;^B}qiVx)FWH&E8&3s{N9}9hJ~4zP|@mX}?vfr)O`YXN6+`
zD0H;GMWjyD(SllFmTkV_XJuD)3VHcuQuBh<DP^6`ps;SfYf~GzYDytM9kO#BLi5C&
z!Q_~Eh<x2oJ9YX^#IHfIrg^rAp)q#vUAf=|6$2ScB3nP(&ai%Js&k%qlIivgT95dU
zhk;9t_+%0cc_0!4dPf_NmN{se7rR$IO7z9DoRF&`->}IZ8vCk6I~*Dk$1xi18BMCx
z3DB1@=tOWnxM>cr>Y6cPI=JiMy{Bw}b4J{9@KphJK6O5&UUW$H8|`;E2x?ros)EIE
z`E-1Lo@JXQmPM}!pMsb^{Q;ner-h!6SR?f<A5u#!Ah{TT!&nARp>t9Ts9wOoKu+N#
zaQ+HjUoX4Ir<mY;ZU*_FOiz<#PD)$AY`0v?@)T$zJTy2ttJBf#eO!$VC`ll%5!4Hu
zR@bdHhx*G~FP$35)FRm)k=Zt0h7oa}h(eJIdTDrL7Y>Fk?NsYMODihNnLp~faYi<l
zN0L`AAe~+0E{wFBuSIi#L!ytJvT^wOxAPF)8a{ja&EI;AUZebNhlvl-!8-Q1_YPL{
z+EHHa3`EKGvTJ#vZ$SbcS5M<)JZo?bEf9rN&DzhdNzahW@BPk+xGg=J_eU^pU^meH
zE$`f`SQX;VDTDLwGLlC+@2E*Fu?e&lNpalt14^3&uiKVCA#3aK^zFggwDsjF(TUJO
z*nTKr`W3OJ)%n%>h{lO{XwwwTmVAcF{Z1YlNn`q~7arQ%c{zt-+tCDV#Kos~h#)lc
z)?L=H*}{#$h;hmEPZd3$ghZLtAlg>AV(WbYmy>YCNM26&o}&F!+}RG!72D3*7vwTg
zx-!yU3+rjy0iCtW2pZ8Ujb1CaopmWVsUvBEoT^TgHf5|fdDk}AA};F_=+b9G5Xu2+
z#d<|$BMq@U0kwCCJVC_r6Qe;^SV*KnVho19OY2@zo}tU>BehR5@1qc;u+M!Y?%+g&
zRDk3*Y^vTo%XaLrl3T8hoC`}hDmX_sU35mL+#3s$xw$SAO+C`UBEI4bqnM9qQR>h1
zrJ^APM5ybM{2bNA8df^Cp_#k(nT$DI>q*7PFAGI;*ky&4p8T{(>so3P1XIphzayKq
zKc+eA^ieow&{i0g223*urkz9YINLbqaqh;VA3vwBs?k@aO`x4SULxGk7;=Mj)^tW)
zJ+))6o_)I&G)QUD3e$31B;*DpWeTGeXP!aXTY{n=OnUTAO?pf3_IxZkAaz<o4OFBk
z9c!EJ>t1RO100&!{u!d%=IoxkCaLo9_KBt)$AD~g3Fv+!WR7fh@BfGpQw4hM+Ga9$
zQdbVHYj7NKj|it*yOs^2Vmou^Vpa=3cL|J9?|h*etaOs(P{cd9H>TS2nqbO8Y190;
z);ThVH}gKKF%Bch=0QMjcMYXG-j1=yJ**o3d0`)<6eO}Fvp#~s6;?yr%^WC;=7)ee
zA_jy2?2d?(h}b7$u${sgY*2y*sYpp?=)YZ(bBMkw1)a))+M$kJPeoMBo%H@WR)OD$
zrbouACe;aNQhpKn>Ka>ae52-^iKnpeQ|=4R(9S-*hIkEx!VlI`f3N1q&{wUi43bK1
zvjMV%?V|jhK*8)XXK8_V-9j@{Mw!$K^crr>VtRV{iTs+maJ#$zX<+A|IRk!98YGSq
zQcw|TV;N4`fLaG=vM_JO>XjyI{dSPfLe>x4=g=kqpETm`oKXc`un<wBH+rrDbxG=B
z2F<a}y)`tWGV-|6MbUjrq&IeiSOM?NyF{u|sr+LbB*nUn&A~1@*7ay2#frF}U6nOd
z>vfWwv|s33lUczbIuPsA4)PmAApI=;aoAlsadKOibd4ZCfqO>er}yNZ^J7O6WP%x5
zDAbTD42WR~?J20+*4cys-(w97*llfdZe5e*pHKSd;;A0Da8^BJo{yH4Ba4*pq|4DI
zC7>U~B62YEPx*Z{Ki;|<5Pbl2NbngX;A8s$=vMUOzAJgDrO6$9epXvU5s^uoUIor*
z!&E>)g}!?BZYj^y)9ib%azw<S_v!-<McW?6%ACk;6Ssmxl~`D(T01vprq!~iX8RYC
zN8?mz(o)Lxg~;iVRxfDM)|wYOTT&W;cnyf~l)D#eWdcLlO>@xGfQPX#)=cVK^kvO@
zYmK}P3#bizEA?&3SL4jC)R`~*4#-3Z^kBv)j5jit6gQFP2Y4|U2)h(G#~X1}7~=;e
z2_UsPPzh?~yAwN`wV#$rIv3w>testi`Uvu%dI%YYK}zwYH_O{>ze7+Kz^OH4Zlt?3
zlqh~|b2j?obn6(|+W@zBrzB%Q+t{?+8&1Ya(cJMb(nZ=qVH~J&?Bj-~qVD16K6j3s
z9@x`|B5fIGuf!-r3}f`w0~Iw6NVN`V9OxIGyWW`P{jB5YSI|2+hegdUbJ7G?1u9Sr
z<3k(MeBC)e9x!Z$DN8z$KF2?fZLGg%g&0K+P=|8mhpzvvKms;Ek~pDaL)qrQrt<n~
zB8tFJLYoF!2*Y-5`4T`wV|Y2MA7g>D_aYQ%R3+3O%oBuj8>hsPw#c=7##3sIhImOW
z1o}WpTHNtRb9x7&h%8{?ovK>{c{Dgtx39jN2)7Y`H0ym~Js<SlVx%SCvO76Vj+w&i
zP=2~R@P9_SvvnmH_Y(#usn?@u0y9N<qoTgvKeTDJaFWS;IhMWnL}+aCx$P+YqdV4R
zPI-27bl@D{bDMr>s|-j!>)K|^+~%d|LKq{TQLm$zeD*p`Km|IK(jR^Y!X|!9Tc&JL
z$4cmPtGlK$2#<!bPt2AP>o>VA&D`8j#?S9qTu-z?>=I{q-9fzr7k7Cp#W6-YeFYfc
zWjNx*x%cz-rJk&_Ld4=lwnMrV0G(yo+5erW&+pJrTjEYN!$nClNsqjWQ6lA^9G9a_
zG4tCWX5Q<0h)?3aPgBwhR$v_IMaq4c@+#2$B6CGj>w}Y(>08M#xvEwfT5QE7!Gb)L
zi>chXy;8MjRfguh#O7|_<ef;I2^Uw%6o2yOOYp#QPYT7!9#{reUtXUT@q$BjOYhi6
z-Yk{J<FXAonr`y6_#AEF8sEw4dD%U^u$)1{OtI-&oVo-rzp6xSVyK(;z~;vK5E2!S
zECnktMXE)Qy1x;f&PwcVV1A=Lg!|GzJpSYaJn*k3z1N%=dM#5p0*Gm}>tN4CUyu#T
z?bn{Snidv~Q%jaSL!D`Fn=nkx=$pcyK)ggu%m^Q(MJ<6B7R3f9Uzf4tVA9mW%art9
z^ek*ncs5ZgANEBPm6$CaT7IFkWQcYlOeqKUHv86BX4!rAO*9fHS#=Qgaj(Zb$2kx>
zX;5A+o+-J!95p+4Bcs((`1Ex-1eaR!c+5H5**a~0dYrPFag3@FiZIy=jY2dCoVsaV
zIRVn5`OjVlFW+GjzqdzYHus^;&WFagvR*`nA?W$uXWbr7^Iv3Tc2RC-?hCW@XLcPK
zWobMbF%Dc$npI!K@bNEw_;M+Yx?WhOo(?z|_yS;LJX1T7rA1VlXyUrp*~`IvLr5S<
zrpZ|A#<7`Lqhkd~!JuiDp3U6(>q6DcC-}G*IbuMq2YKtb#;g)r)1rgyrbtIEgJ$3j
zKo3?)CZx)5fwl8)G%N}KVz9*^&<tN!bxHWODt?OGGt?#wi<sYpq1V$zrccztt(*R>
zOsY<uyDCMNxz%g8$dD%X4GMo(5Iap61+JjLH%F}krg5+#U%lL~eTe2DD8Cu>Ha*o>
zCDVO+N~Al`Q@0ma+Cy<*r}>EVyy}}6boM6}ZhCEb#$5bc>vMfURRskK8rc%wk+$We
zQ08m%@-{AQJJ29TLU|^<_3;hz$*aa@at?gkw?Ut}_`J9TDy^Yyjv`}NCd?O}!k<H2
z8hoJ86z)(T8=3@ZO&4b%aUf?<of~pELm6WaUC!H&CCxcl1Qi#`6hSmLhLbHjY&X;B
z&<n6_oWdXqfDJc%WN9z#?AsGvluSLO(r+CJfa(k>04OZAQiRreACvO^7iXQ5@_ndU
z<8zo0sGZIf-1<2t0S`LWeLkB*$&6*0O7R`0b@G&l9XlFBNPF^JfOl`@(vs~z7X-$-
zor>f!srPa!VR?8dlQT?W{f|zi)^=nR!>i#JUcm!@F5#HhL~68L)CO0k3hBSlL;b7J
z`Q5_JhqVPlHa>#0T(T`S<tgfWe%VRSbTyq{$SZ66vXlVL9+l!jCWw;Mv_2Av5_XF*
z??t9ZqGz6B9TS5g02=au03i8NK+(EOsfzjN*@qHs?;DO)N){e%T%Sz5n+u5`p%lOY
z3}BAbts`_|s`{h6nIWIVNT152Iy<8I8>Bn^C{|fg*Eaw3FXfk5FBsRjl?UWc2y;5(
zNCn}lEiS?`&Dn193uL^1&BY*mQ4a87&!>FzTufN@n-ksg&KaHx-(q}dgOf2AA5MEd
zHs9TMDtCP26ee*fX^d=Pf<6;j;*Z{qt2#<CSEHt+tWW<^67M6OIdyCVmd>2}9Kjhq
z$8!NZw=z8IcvCq^zC~~mQZ(iUuUaAzh}WV6W_|;5PJ0Jb$3!5y#bB%J^s8zPX@=s<
z7>C))Q!#9Ii%z%PSwvGUI;FEm?^)~*nJv07-r-fy@shk-4Oalj*a4BBn%s0)5#=r6
zh}AOy4V^a;LF%`xSp>peQf|tM1{tuX6(A;)krL0Ovb;fU&%%`0y4L+mO8)?{_}mw3
z{zp{MEOzo4ZXGl3IxWYhNyR*V<wrZ5O)x%<$XlTmpcn^z%?QbL0J>zCY!hL4wfRwH
zR^%sSoZ)W=m578~_oli6ORaK;gv<d8UkqCG$F<=Qd-&KfTt^}HE7X<gF8zabWls6~
z+>-k$t+1@YENujr>EJbnP*aY#MiUY6>S<v)iLH)c3hQH<a^zST7h?`D-!Tcq1~8dZ
zTx>v`Z7VlUzi?gS3V_T?DNGqcBHJ|1Gesb!YfI<8v>DIvEj09b=9gMEhw#b$t0xRv
z{*No$!E8qg(8AeUF`D{;-z4|Wqb@$wx-&M6C%o)>w(VN$=`7XbR8kb)E{au1Ycg?h
z_h$>M)lNBTm25Ot#wwcrWsBaadhSJr5Uv_%#fAH)Kf8(@(RJ7tg~1#2p$m{eaVNzV
z3N*iK7tZYX-6O7TZDaUOBF;oG^*A|^AgG8F(e6BpP8zVR%3uo*6363(Z<_%pa32a9
z(Q1XREzK)Bdv-sI*`0KaYnvupt22u^k+^cj)!@n4*K-qFpR7~J(mG`~>qEpEF(tR~
zWKxF@8fPHjlowv+kd~z_ySH)8##10!^jn`*+O)W1p|WFCYx!5(#-tEQr+MjiuBz99
z^}>#QsSazVbtVD>nD~B&L;%HUOjHVA3xb&8n2!cxcy*p*9LRNq&q8dCSACW@G7%SQ
z_ArDI$4h8ek9*$BK204%;OX@(o`S6$;hsc>K9%*-Ea`^Ap2VL~k^Ui^ok-sd-=%@;
zn*nVd$x4zFd5a$QY1!~Ad3ue^8L$n;)60w$!e`sr7Zug%jN2+#zZrh60V_(qN!2Ez
z7!g1rWFbAYS5Y$!lt^xoqinmdcq*}+UKz1nz$@Fs3)W%IBp;F?CsUSLmy5KQN?)3F
zJY0u!H{Z+YkSteXQ<T{0bgIvhr$dgj{oLn6(h6N|7{zGJsI1J497&`I+%JN!H9jq)
z*w2r1y$vWD=;jSL6cIu`y^?ObLs25?F%Cr`untMX?1%BUni-uVIo>dLXgTWY11#Y=
zC>iglDPOL-=a(;I+IiY(BQi_1nDdAuaxy~#$1q~#k$OR?0w3=fu=gHp(L=&}s}0k~
zD4kC>p=6<Vc6W>1IH||g_OaY`C|%HD>XP=D01(9jkpU%BznhG)|ApiEygmjGtk+3j
z(0!uS;AC^dzJOoOIT)J@3d+FMxq0m&yg|ja625+L-b<Z!8dRe~C-os*V>DSb(2{}V
ztG|nGY8Tl}APvA6*`6=sSh!LD^jVf_9Y6LDuV5pWmgL$HC=6{C(1q@e;k)t7;p$aZ
z-BKFYj*}z23ma^ZsNsOKptlxpoa24V;t{6BEUw*%CqXl$Z<gSBT}=194sVGfsyBcJ
zsTRh|UO2{GWH<;_<y4t<0a=lwasLoMuTGgNd|3eVhr%*DPMPrO=fyTi({{ia1TpG1
zASGWhE!a+lqSIP`AM2EY#S@4zcS?`pn~aRRcjNGmHR@i~PM<9uTJb5&QEhbhN%P^*
zvO0bv7Q`%}rDboLFMP>G4Mv)}V7vSWcvyN86tSR#g}J>}+qGB?enqiceMIU=!C(}A
zR+EOWX}QI>PwXgdq0M8iCS1jel9Lm4+aH&(L)6nt>)+<~^(o;&t}+sn4n;%}{PuZt
z*<eWAKEGsTTa3b5qX|h-+Hd_H)jEgB4cmo@zOt0YMUCNm@K>c0QwJxN0ksm7so-UT
zbQRy!p$|+8Z~!WYxU)!u{bE&H1j7%Z;lzaQFRPPn3>A~w@nfU+-7Klt!&92Rk<0Y(
zVi11t!nL4ECH?_Gf3oMe`FnY&cf?A+#T^_p5j_AJAc-Bm-aZ^(`{djw*kGxqJarV!
zgzY~T((-3*V^Z<7T)q(k6%(Xna-`-G_IxZ+K*F8{_TkuA;<~DCZe6U)P56lr`^fAO
zT-pujpp3@lM5!7?TCQFfL2XPHp2=}<(uaE}%wE@IyB14jk(m4-34nF#z4ouuMtDIz
zc0TR}(K3PEGdgEUdltIL6RxwjK~u5~48v1YP13Y207L}BHH=gqR%$MbuHc{^*&E2T
z5l1%Ez6M`$RAH4p79`O7DtA+&PyPPXc619Q>Potev>@tS!Yo$N_|oQ`!BL5{hE<>h
zlA-8nG8Q?G!4wgN9?gKY9EmJLv{en3h4#@lhQr!@%WnMaL?JtaQkfk@)h5hZWseP=
z!ywFG-u97F{{f3E=}WPWURi#B7Ja(QZt{bACe!tqt$h$xhqNdnC(%_O^pxG@*V{+R
z{|78uPG5@OKiEKoH(H2Ijbih^&<sbO{qy)e&6>WlrA!LbiwMHM*x_R&^V??uBF<dv
z$6%Nbwx{T#cGi!gBjG8AmCV7Xd?fQOArAyj>@rx2$gUM|O$BYj(6%weMql}LiLw(t
z9ibEn@;ysmh2$vTMIEHOgm+hgJs_fwB#7v}NGOvf@7596w9nZ74~f2N^LAlsns2s8
zRmV7d+wUpX<75lr(BN3`5BlSH@|0WG=>@)!%7{_22Bah3rzUSV6;asvEKw<pw907&
zmam&nkT|%2P(E;bKV7rc7A$YL#V?)3JV4yN{1TJXiLR_=9FGU@{g8mnX(?F+k#OZx
z>?VU)m`SN26!LeT$^9-aMvhB2Kt8+4SViq|tWFe9SaTur6jh&a4B=NH+VVhIOUVQl
zbRKxGX9Gjr12`Lp5!U!X)_llj3wFjGa@6|lF}}nVcMaHFCY;aS95|ozwycd@;i87J
zS)9|<UkRfktqxX}3JVXiL+qd3WnIP!-}ffF2|UO?WH*Ua_7nPW=q^`c-R1fphn1l{
ztfCE<Xtj;VD0JygDNm#<PjV1is`FM++`Tl29g8I~cf9rO?Ex|Wdy0KHY~URv@i2VR
z3*xBk?6QJbqPQao@i~~RLWqm~<WW97C3jAPx<(yd8-{FY2rLoiruo!+-tTZDd_xGM
zY6iRUR~XQWv};n)O;B((c^{D$Xz5~GrzLA}lDQu1@}G#^9620k&XFrXaUE#9(Tt_#
z>wJr5(XO{jxikOtT0|U>4&zQEM_3De4Z6cq+X?R>APrO!k#uxh!eOU7y_vsql8jjQ
zj2PEGni)OUefUW4&2!KG&t!;soN&#7LWX8m<a7d*2-lc6-_0Gse{axH{|}taouXJQ
z80k*jhi$zkR+{vz+mnv{TgX3Lw#UjVv*R*;-At4531_IBs3kAWJhrLV$Epl=8ZLP+
zb8qTsXvN0fqF5X`^4~U5wRd(fQ8(?f+3Xf}yoA@(=A4F=f$<&rt^$5=31EC<4<$wn
zW?1b<O00(}B9V6g6KhM96j@*Uz}o9@e1*JmCy^E!=;-k%7%wTL%Ny2fzF0FB=L5V(
zI@U-RjdItK?u=w;1|{)%E3G%ws8u7T=&U9!JDEf5zwLoINMFfIq01|IZFKRn#XZ1P
zcK2IW8~eqexq*Has{o(u9U+QJ!-=Q<umwDU(ChDPp+~jIeKjoCX_sY8yX!N7i=67$
z`*BQ2gNiXup=>bG`cD`l9hF9kz#A{=NovMZ>q(bZ8<s9ldKF90id4m2S)R>7DStK_
z6hbEXl+4-r&RI^@K=s`up2&@~+o<^h8c4=#d&}3b8sYqED=_5Y?62@gQXS<kt3+D|
z4mlj)5q}lEk~w?c(G*a=cvn-Tu~mz>DZffI<;-eY6oZr0!Kbl1#+74vnrVW~e4!xQ
zu4$s@2>lOgip5-Fr=pnZ9$I|ocRlScaVl;lMdU(Vku(tgOolv*KBYiBkg!Mq;aKCL
zZS?#SEA(CKwRco=ME#Y?v45aa04?W@L=Obp%iAg8Mm&(?$3o=mY1aUP>u7VngGZBX
z`OpY2f8@`c_aYYWAHN0YHb@GH*jZd#WP{fMKSLb=vrWnoJtpQ6@fSg)zhx9YSd8cX
zOEyuj33nO-6C?RL@a0DPf^N;*Wy;+(1H)=T+x>0y4^h{_aksLQRbzoS&@j-G|4ri2
z{2e-j7Sg)Y3MPRf@R*{K#fWifiBFVO5&R%kGH3r=y?C^q5lK?kY(`nf9<<mCBM$Yg
zP+PGL-_JP)TcBmBMmAcLj}0Mow5pcX)!h_re?cp4qSv|){{m9V6txp}59);n*KPb3
z_rZ%20E*X1jG{G|DQ;tZ+dcOZ$(V?Sk$UbY8;<sqA;Hu2IwM89Y~D`V3`@QWR$#a;
zT&|Lyatt<*tISrGAGL6_>x)D`9HEJSiFQHp0diX6t4?2v)V_Kj^Zf*EXo2}lv1O@U
z^yZLaPJXUv+2-4+OkwS;ZOY#fvJ$xw-8b=sW)>N3QdpYtMcTvLr`xV(Fe?SQZDZ2$
z2VYm$$o3R8f_9M)g3yfp;%#Abt~hoprja2UagiWthQwR1pWZ9l{`z*>GN?5|4$MKE
zH#h29eO<&Z7gn!)exm-Brm8S`77-d_=~%zp_0`M2*4v##ayY#UN{|y2-8Ie7o}@t&
z*2L&9PmB)`@@!Kd6A5JJd~6{ci$TU0ce0~TG1q#yax@=A&ck2HDmq;kTf37+cYI)*
z+ZDQaGrdXiRkd%Y5Q-V{(D*W8$i>gZcMKy@UBp)23y6h9lR;$q2(FU6D0LiTXC*9i
zf#rRk``+n521R5JJ$AkP6_}1pn*Kb0`JtBg*L1yGVwJsi?q%~IFm6=^;ej8pn@+=4
zS{PP9TXux83Qi-s<&bY6pE)Q}hRAbBU%8Nl5qSK!WsRL;d7peO*s2{2_apk)YEGfe
z9_|2<*`KDm<6)9jJ1#rOb0Z+l@?)pio$Z>r5R-o3p-1l_xdJ_Rd*WQfuFLw~QnUzV
zE{-wtu$Cyid)=!N3rEe|ji=;cJB`hYHA;9=JSVC|ibJyraZz9E7q<s%$BR~Ob(fPu
zTSmLT{6Up0!=ef?6bEts955FZzBJN_>3Jj|G~{OAFjQCwSS$@c6BGMH*(~OHAYQxs
zeo6mb^3(4kC)bSDZC}htk+`M$9iILr;~<Q!V>wA>EAjNjG^B9oDCKzIFRsr5Q_B``
zb+>wb=7hO1yBN^M^H>GfEA>BO${FJl(v<a(viwahtP@XJcJ2$j<@PMlOWnOCx3XXo
ze!q^Do@BPLbUiP+2rjDfwPoV>{3J2DkX*tpcWh7;aqvaD(%+N3oX6E<p%Q3qFQs>T
zrJ-m)5DNN>QFOVTvGN}V^AC^K5wvF=KRm}SS#b%k*vBgP(G#ZYgMr_%!Pms%3k%y=
z7HbOTM{>rW@Nr!C`Db&~_osZnL2<m{kk2xk%ujAUrE67X88dYEkUPt^s2%?Odo`VD
z2A|Ag6r7%}H1Xn1*l)L=^~;Ul79YDXwro>m$*f_cXWbdKZ|d=?#z~vjH~#+VaSMam
zPxnm--RG45??R3zhjVOOizk(SwI(^4dD*ZkuPD)4C#jVgmkj7k`#dbr{cSu&r`?nm
zO<VW8I4xFVRKVGj8Iaqa2s?>38_G-=W4kgU7Hi&3GY&PjEn?d63cLg~7n%%C%iy-_
zB=8@Z_}c~Ze?Ek}A~T8~Y|n-5e=>U?p8zr9MeyHN4=1TT$B=N+<)1+{ev{)w(19v^
zA>w#$oJa0U-poLQqy|UG<sMf=`SB6h-y1d`ymV?~)9c%idX2;v-CC-JMXpByLMBY(
zNg~fmFUxF;$xz3@_;_i*ePMwKxW4l89KAg28Q)@rg$)UQOL=r#dTDDacj`Yl_7hp3
zbGTVGK7qj#tQl%I86Y|10%F-$wD;tg??Kc1OGp=w{z7`A4M;uB8ujXpPl4*7b!@z7
z`*WZSk?H<vtcoDH(IQFj8KSMd7hr4e5B^ZxyqeFLk7!Oq$>O}aMgb*;L#>yB?ueNO
znDVtmQ@+`daAj!?wIp~Uo=!26^bI*)6?t`y!c%yh`^Z+{b9pMs5j*1yicN><-F|FP
zf^GJCd1+IKb<7w9#0_zrnfBBt_GEq~*NQj|rMe&&-G-O<8nY>IB9U5UV~4|!@61=z
z3yp(it=bAJY`*bJ8G}w~I=M$SXC~++8EHV>H%2)Nga$QOrnp!&bgT@dP^dy0#S!u_
z4;Q>&w4`8+C`Ep>*K*cTeQ<WJr_E2J{e?S8FVyTTjY9UbXjAa1@}<B@7-pw3bjmvG
zP&;hwW82DT*uYDafq`WWbjs@pUhZA&nN%bh3KA-XuR>UrOz*xg7A}`IMj3UfS9wE;
zkyAEbLDncL6R$@hm_OL|7g2KLx*@@Zz-grH(;uQiwrQ|C#%KpTyuFoG75kxV95;<1
zLa(5vW2F0~88G<m(qKFCwnlO022ilee7qEG6!$rFV9w>$CK$#5Lz5nY7cHqZ_UvGN
z>~5)~dsg4$cK9om=#F=~sf-HRns&6?9CovAS394-5agf%d59M6PCWPKFC@ohoW}qD
zCv5t4;m<2!@%tOR7VpjRKp4U9*|5Z39{y~<(25}dC>Ky!UJm>jEmQx+o_)q<|I=(U
zodkw^z<=XxmSn*AH-0uxmhu75+E^Wm@GO0O)uKzrwpbI`Vz<HaM1e0$_A0GPV1Q>J
zb8C(0*@8-cJC0zQRPit9Ag(Ii!8R>zmRexF6r?P^ffJj7ujfNfD|BSY#>>M1T{xGv
z17vMBp2Yd`AZr&`^*;h_2yI4%?X2abUfT6vNL8EVMQU3@Uqj9|GCMb8G_2Ux_?_HO
zl6(m3C>%L9W2j?&8Ef1=n@cC#E%946?4F$eA#?1tV0CTl@qb_aTr-2kWtGNJj$$F(
z^f343y%N<#c=u29@KKojW&)e5vsfIlAM{@r!YjQ%ZXR8ffi{okAsx$TE!jJ%kGE{a
zT`44rvnHS$kCrP%n<Fsmd(noX)uIyrH_*dbIT75;vmI_b&|d9f<Z^|_1?pHS^~ecI
zDa;d(+-5`}f(mM4=lG%rgY|euO)xUn-Nc}KM)F6rRN}YGrfgz~F0a(Agw4-egHJ3<
za>pO)?Jv#<!tNLIbs@)HLoIY$$zxf<k%1%^a%9d$Sc`P$ljDzGr<8KNSzK*lli*MM
znk_9f6gJadDtR<D)oYrAUGzwNpl6`B?b{n#BL^c`;@n#W10q<KZ-P$TH&0q-1bcGx
z?jC>C^_WoX-#i1+wx92TKQ`rQ6=(4Zgel<OB1?mp;_j{4lb_pcnC#ZxkvJE@ZW6-{
zA<svK8>k4=HV=xM##W!+vLM~GUhW~%72FoN0&nCB7L8wZF#?f*4<B2x#xK<~&zfgd
zBz!K|h96Q`y-{gfht+&HC(ZIVc|d?YjJ;?%-yn0&8tg5jaGCoFL_V|?)1U@1N;aG$
zmqe`JLk@w5L|Z;FsDMyB?)1fgsj$A?`V1*><v9Vv?BteGH;@m=fp|cfnA%Xtxr3K;
zS0ixf*Ck=)YQ?^LSrmR@@HI*>?hCBlxA88}(JS&+v>$O0(xn711>kSMdh-d=VH&VN
z<^cGh2+>C6O@W{V9X1X<__Em&Q%o$47+8vlFV_{Nn9}E}HURvXMu&7cj__)>!^m_w
zC>EP}F#c_V)U%t-oMJ(kq2Mn6iwew^6@?*FYp71n?p%K4URE`#swAY#2umGxeBm)9
znRpr1@INj~bv=H<_9{8;@Mq{y$Pf=8SDlY(TP-9N6YmTskyyRWsn-JUBgA&Tp{gNl
zHqZMB=i!U;e}xm~<f(j-7^Z}!W*cv!T_m0F<@|-mmY>(}lQk*5PC1Jo$#7QsGeLg%
zv)FlnqV(m?;9+b)TrO**VCkDHImRI8K679(h7_<QV$AOEU%lMQ&k=<EyH%bAY1uhQ
z%NF#OmhITLXZi-YzqIS@?KLCp3xk*AZ~ldjmlM|}5`yNP|AawWyF&}yT!g0g;(cvx
zCaL_a8L;z9bqym;Pt=o}_XCyzAevtEk$H!hL6!V`YN6Ld!3CbteJs~Zi_@oPpEd=z
zT(>6G*7;1|NdTNm2I2?hoq2B$|9?x<p*@FX%<Cj&%$uh>%X#wR*i3-~FQqw~6>Nl`
z%`wjsZEDAI$j|t9z%0SUbXWu>!6NQz(Y=4)uckF{8Ax<tiRNxe1<j)s)zey9UI<#Z
z=e9Qq&f{J=fIXGTmlZw%TfC%vBsqZ34%bJB^tpaS%+6IvX?X1^(6p6M(4aIgV84?$
zWO!&tabE><cW)M4HJ|im`y>_r;LY^e_uY~`BVpfF@G9(Z>R0gEOJC_w8mjL#a$Y!?
zOh8T;3lOmByC~&v<I%8wWZyZ9M~tJQjPXr;0g`}Ph``IfsxwjzlBmvdW(8&cbSkkm
zc39b04qLwl9CDU(E~aFKf=u3Ogb|@26KArh=y08TaNDoET>RPOF$_J^ON1}0t4jWo
z^lV_V^er!0d<q~3D3tb<apD6vYR8F;b69p$16gW5@8jVpIEO#+iQus9bQ?psl}E9X
zCfvsV^c7)XI4hA>nYYQrOms9LI_MTalmgw%!DGdM+(5AfoGjePK8v$%+1Yxl_EzYr
z_}YU{TPMk<&EL&DGH-aoo>!JNnw;zz7#g`I{i<EC@Jcw#iq?5DP`@LAJ~M!Ou=qdx
z`x2AsB(AbH6vI{C-;)Wd$|O(bd~YW+GGueS>*rqJ)SO0C@bhJDDVu}|{QY<i@JoI5
zy(s(EyUcV^_=K7iWQLkld<ej-n*+cwi2<bmZ*<SLMMf52Rhm<%CW$irlRzW^2%l+q
z`4Y0saL6)Gpe$4WOZqFpofhvJ>UByKo(JqR^GpvgUSk+cQxbVIOq)Y5H&Ky*XF%A1
zzJe^YZlD?Ml@sA9^i>032;%69VF)~gEUr&n1522Zmq|94f{OLl&A|I42=IB=x~2(p
z0c4)(YNDgNg&*(VQN|C$59V@lUT!5tSl!YlJSuVlALF8dDuX#S63d#44Jyds`KMjA
z50MCfnAJQt49mponaQs%99bS6;Z@4?jPzts==j-;LDh35<uyAaC3YSTWqyo%23j5D
zJ=+p$O+K52U5$aPW**=gL{<}HwwPC1nq1oUT<{!^n;W(_KQB~ef<ap&^;p%Y<7Gzq
z$QKRr^7TYP^G}@U)%*>w22joF=rQ8})r2HK?a_8irC4D}9j(2El0Zqs4yX8JZFN-?
z)!CU>E6kGM26jWC^P33K16&g!dIPFy#v@b|GQ0lEGZX^a-6mihgpHSv5vAs$r=ea%
z==^zRT1aPocF*(fl3zo#On!uyQ^Ts$4t^4Bj^_g9wGFwd{2~ZA!q=Sv2N$-)0}f0U
zA;3n2<OgC%K1x;`$4SI%usZMi`y)|_2c<-x&=Xb`EJ``=?#YlrNUi#;Rb(!pQcD<g
z@)Lm37Kqi(K7R$Q&|4n6$&?f0l`3e$Z(C)3LtW$4HFOv8;Q3K)nLKL`uy!!?NuGo?
zdMTb$r<wdH;}|C;^!M+0#Bamh+M&8A<_mXb3)ob;4JuEr`N7yc#G!;L8qhG%Rd$(+
zkXuZvPb`OiKluPegMxw@41!iS(jpR1m&uyVsi^D3k|v9sPIusUBdzQLfHgh^lvg>W
z>}J8Ic(W!tJ84$C)I<t{r*a54h~bR&jX;2u<Ag9m7?MJ`0158@nEPG=shESa2)#z6
zIQ2tt2e+oTvJEWybciEYH%Y**d(j?T7;K3@qBPGIt_P=JoR3jbJ_fp`TFz!<puQ83
z?4j!UU{Q2#>R*Ak08GOwoD(GyS~keJ-5opn^Hj!`I8R|N#g29WXIb$)TMEDq-DFPz
zORH$LJn}LkgX$~{A~t{6qXELNntA9e;%KhwNy<K<q6T8`r``@{noU|R9>=a3FTo|b
zH3Yz?Suz(FN7NANgGDLFjdRhaRMdNV&uJ%-4eX3QWf^DC415v%RQjkp(WwkiVchX4
z5N7J@(kba!%z0hnsaOx?jeP|w2vEXYfazQ6=R|v}$AqFa_aE+#fi_R-rod`4fNI3`
z(yl>$T7*6Qt^EC{o!WJ(9!)XKO|*aUj+$n<!vR(ThUX(3&@W75BpUTZ%~`TN0j($0
zp6DB!#kyC4)6nXHb5Q&Ai=81=E~0mC5m-%PXFL0OBYw=F8sQC3VS9t%N9<Z<SOs?+
zrB&;j>!{0I{Nsb)D_o$DXqo#R4f7%#vHU|xy+Gv)GTvL%Q(m&tn|XA~tH9tJluYbe
zonRUY?$c`YQ6u2k8gE<YbKnr|&9I&~A_3fad}7!FhxeY$Am+07uLRb7$sm9XmpdiD
zlhGDOl*cN#yU$XJO?Kx6d$!ncD+AH#m~Q0;x+HV-O~3XriY6S_^4TBCapAobvh8$#
z+g$Q&Zc02GjZgFn*M0I2{k0uw!7T15ANTeS&8^W9Dsrft{2-o-yZ2%<&dR_y7<WSE
zN2+d+?zh+CAI-Ib`-R9jq}*;*>q!>3R6ykz&PE78{KmTR%+@p|ujZyeN3BKHAK-35
zmZCX7Le$9Ua=A_!1d!G87!oq?S}kKe8K32`e^@+;n8%^g;ygL+8)>1K<I-4P+Io=M
z@yy>oSol54TTtmr<SmHPU}Q567fzoc>a9R)6ZKXG1fWX<%^-@fUqus~22wJSK+P;?
zrooIMhMN6AW|vjifUa0?6C6Oumg|3=F;Sqo@}8Mc*3&Ly23=sdunWCzX`L#Ixr%5N
zATzvKYQ6PR@As1ML)aRBI-h7Z<J9<+^F$-yw@_EIyj;{rd5P}W0=rKbd|oWaS-|6M
z%|v#n<S-hurhd3Mk8&c3ujSwAD#W8-v$Jd49LO2V@Ln^19%ueC840L)9ce{MN%}JY
z%k|)9p`nTQ5g<t>7tvfo-rm(%)Lp`RjYNcmo(>k_S&<$sJoAyKt2~H2|0K3M8L`Vc
z+hxdwK`G1-wrJKF&cPo3?${E68U*@+k8B6<A7DV*%mdj`vL?i8UN+H-oimiA%yc=G
zzbs7nf?N8u8m0v;7JDV%U02%f%<GsS^D)k)@F$!DCCXer4=f`y7yFlzO5<ZIFc)^E
zZYC5evauYx;36*<e=GH7Z<-HF4P9q_wO4XncdSpetDN94LGT)BSW|)29LIMjF-<Zm
zChM({#RP|87FwCq4T8BpcErADGqy!y+XH`gusv37IYKVnSo(m(#s-=S%MQE!*tAz}
zNe`@oBnc8U8Pau0J#w1UohPFkdKaf*>Q-PhOs+Z!T}m*=FKSU)-9=s_ejdqYI-UYs
z04yj?*4HAIL5e!ytR>6)6LpP(UP*G@drBf9RGfq<@?cPzbZ226@}K?%r#|@ixh~g%
zVdg0f71v}5Jxp-NyKimz$5mJ-T*vdOp%w?@GG-06)h+b}9?yl%PFEuAK8Q)<5Md5w
zFuhphcWC+q60HdBqfIufaNUsQv8^5HEN-L{CmWMbiO=9P98RP<HaE4&I-!K@y`K-p
ze~@g9^r7j>q%nK@HfEQ4td^%MpX1fk=3i@w`Lg(ox`y0DVP0Nc3Ds5O$?B<y1&C1*
zF&1EG{_H-mq{@`Pj-RmUgT>I<!Vu}Y{szmVSB`}LlA7z*)*CtN&YdF@37(nuvllZv
z8FLua7aoBWm7sy)^K<O+pLQ#IF>Psh)6Z;Uu-oend0ij?Ox7o0<$)G1^A$!JZS9qn
z*`uTsi<pSa0Wc^8Oe#jl=;oLo9)E-d8sPitOD<e6*u^jM!c&e%6Q94P1;t9JS{Uw4
zZIwX)!E$JCc`6uX2SKtuxFuar<qO0Z1h<BG9f<OipCR8H`)_9GHlle^Y}1c5(vcU+
z`8Be14&9h{81v6pPwM~;D;F`#mlOEjfU|%dYLh}=oS1>Nh7dtUjLV4gOghtKa@+Ez
zBRFXYe>7>mFw3R=rlTdpi1s7ZThk8>fNl=^LJfk==T|ROwY7NDh5PwyvBQzy?yoDX
zpBf3NnY*HG_=v8=vc7ZvMvI>eGBwuV@OE!(tj=z%97#)q*oj!sz|4fu=?iVK9{gW<
zqjs_&dn)n8ocJH4{5N4=99y8na!GzMnwN{6jmm?o$>p*g(Qw!D!pR6KBFgszCgBkj
z?+!yvBgW3`89@m~<iST68yXT?C}EO&qh(vFYj_jg3r27Dpcba@G0dJbH;spSJZX<1
zY`NObKXtLR(VwcbV-ALlbzw2|1M}f&jYz3WjPu`_*6c<k?my02;G3flK-%1o6S#Ez
zL@H5^nUhR8Y<@vRHHDQbL^*cN*vP|`FHb+2FI-2%_l&Kg2lJl0({APET6vNWN5q%w
z2PYfA!@-ot#vB~;1`p>z*jnPQ*>SpdmgKSV5~L9f3=>oUERT87HbtqeiZ?nN#hR-f
zx{X=#w5D=arO^F9VGQhV-9hG@Q@U0&3~ZQzZ2N&5fFC>0fp^rtRh>5rH|I8cN9AZ#
z`p@}K7$dzuED$s1r01mit|ueT$wX}>wR8DfIP~Q8v=>+oz-U8!%w_)xD}aj~k6Yo~
zpB0l%AM;^e2OO4K6zqn%G9?;{FbUys!A6QG41UA_Q$N@xo&w<O4uPH_Hkctw0idZi
z<nVzz!-*BxD`)qc%Rqgx1q(Bob4IyH=E-7ba_ZP&PN|@<CY`qxyI=JkW;$(;B9oy{
zPD>s)_;Ci8B^@{84dRTfQ6rWE%*ptvxbLF*CTvZA!OFDqDib_M`OGP82E-Tp_Pf*_
zbBQFgbJS!&NoHT^40JnSR$%t(X$q!GAf5Sk=5Mg!x6dSVZ|V^#EXnk$h$`h;{}geL
z0Ga4tqn&SJw0U4NDs1rmZ11Dj&6YKZEmMP$ImHcV1fWr=ZL6vjs~Qb2bTWSj`JO=^
zLl5T-_mOIDI{q5N$Z<?#iuF?GzAxLPzmFwyBZW24v4qFti9MdA^ioG|<5fEj+g$_t
ze5ek1Uq^(<bNwFCu+$^+*b!|Uy$=R}kbCPf`G-16N1&s$IF~`W?#ZbOj-l|e{Mj6o
z6HWN%!!No}?usZG>;F73VgL=Y{&N~!hxFX_Us6y`13jtbI4!!5^m%OPgZbEO<^co>
zWPUTsMKYr?b&`pJnqO$-xGlmmE5S~F8<uv`Sp~Gpk^&uQ#C|73&qMS3oz(e3N(ID8
z8U2~0H8%7|Mk4JHvd7kAQoY2)6D!p$MlFqOcjh(XuFWR-#bJ85FePNF3)5dj&$>fQ
zH}wNJPn3oj@j|3%E;eaOi~)=X(7hXR7L*`ih(K~Z;J14^6?4?K+xoTS^QE=f_N6pl
z6mRrH6hG&3YS?XM{%%dr>s_|SMaW=?fM~v{?*=)6*oXx6#l{**#0H@EZp2xPL)=IH
zyi=_LM0}wIaB$@9{(6~bs^U~_g%4Q5R)O?3@B^k-6QNSm)+$~tHTWJtQcpG_Koy%Z
z!Nwur-x8faMFWhsU<>Qu`swnQ1B%Oe61)>RvJ6LHn0q$n0aGRosRj*R3DD2jucT^5
zZ3AZnHyFQNrv2WoCWA^hVLNAwradiJAcq~vGIS)=xCBo@fL4z|rnj#*fdpmM_w-Ni
zPyqa*(3qHrwP#|N?~Id(Iav=u?U`LxZli_tI(K`WqF`_AfGi<Pi-|m4%-?*7z6VL&
z3`+3qcX}4Uz}}z$xNkIyj0;XLJNQMfFJ_!xu2aqmyvIB*_vCnc=IfD^jkK@kp|a*)
zBAiUr)ziAd)E*;E@8FmFozs+n27vo9Kag5N&NOI}Jk#$wd8RNri$~6}3R*^MIZS>V
z|JRogH{(O=s~b6jBao`4^&Fph3n;(}CCs7CsqbRTe!>>pDMK9lsnI?^8TJEhN8*ke
z`P%GXP|(z{Eq!I1?ezeS&yVggZt}))1{WI~v$O_n`F`8CW2snw8bdb@%ibm%1K2W}
zZCyCcSWRFZ`df_P;r3j97%vx#HA{<t*31m-F?#yOX9G)s26{FU=7GTtSSr+mEGGn-
zU!}_WZqn{rw{dtGKcB-6-ius-&3(u-lbfqs3QIjT-r1!Md*p<4`!*S4EaJD(W&qjn
zuk@ao<9)$DA>7~$MoFE`FK6v(a-p;bwE|pw7S#hQN)f2>Ixja->G@wKi{CN>*o005
z(wR58X5@fo*U$+J@mrHj@xk-M(%r6E=SVP7+YEak#}1%V?L6IR-yn~sl82_~pcDwU
z)p|PitGB{8ah<z!f#$oBz5$G<pU9U5#(MU^1{=E~=`j1gTU~1%*0g;tasm?Qr9)CT
zT+Z-`D1=ATz?+qW611?o%K1A4t-zTT?&~26Fu<3CUL3NbY7rVBN09ku{#)ysK*&+p
z<vszcZoVJ0k0hS-HmeD&rcpl{@qreQ#Sg@M^pTd4r46;gZXWLR7ropy(KLN;<ZLo^
zCY0i4=lUXD6Qr8{_PQq6+j)8jo71qJ3F!bX4ba;VEdmA4&NRC(>(#`2?aRG*4tqOK
zrk3sy$WoF$b$h-YF?E1}+k<ldce7{M(`C3lUt#IZ-@R$*;2`HNuqR8Y?PcZb+?Gnr
zo^xz%jz?L};Y8ZO9+ypAcBU6*qX@Bu$!G*L8>rp_^#W92zJrJkx@<IUA1vgAK*;Hv
z04#l^>JeNvO~ROU?2xJ1l<ZIFe|OEJiGe^ULVBHQAi*?5O+=nKRNnI4RU7uCd@Ici
z%D@87atJsQ1Y3|(`_uP}-5^ZCHDNl@W@0Ywj*i<6!FO0!-3e!aiJ==}S|`2DHj#h;
z?gPY|9B(f{0mk?x2i4_NCaV=uDxK=rvGm3xn1j%wAcwYbfg_SQ`>oNR#x!f5z$Cah
z_#ld-zMElO3)GZZi4?ANUlK0;nboy-^G;?RO7=7Wy1+I>7YKpa)AxNl@Xz$$ILwk#
zJLU_YIzxSww545352m%uiiAPIFZ-0CA}8ZPh6+u_qiIK?BQ0kEXWXs#iAW)D9e%FU
zHF1FLCBz@eGu+a^IREC)11!*@VDYOkcfEvDsoTa5c5rsr+@aAUns!q?+J(j(#ReY9
zmK3UE!3&7Bq~2PF3c2gOZ1xe{9}MsBeD?WI|4ewV#k(b&9qf;;T;g+lN@IZfU|kNT
zcfo^qI+%axGqHBT_@#YR)_Cjl)$tV#mf=j#N*mqll*vI5dtCM-G%X&cQH4}O7q7Dv
zl^}T5X^=z(hj%h)Cmko(j$hQPL1Mknzn&ids;MQiDEM!5F5&Jt{Q=qqjf%LuXD2);
zEv~GgSFi%ppy-^Sc^~3hi!9Z?0QN67;!u~?6~Pe=Ja!hMa^sBtg}9T;l8t0~(e$vw
z`l=_))dllkw$%GO-Btb+0$J}qo8*>=&!fH%*&q~v5?8^x3#fR2$W1U)yv4Mf@R>)1
zdVI6~w-Qc?U1{5G!4V9^dFjd5E^Qo*q$z^d3{=SthAiKk9vzmqm7rxYz=iZ3)~$f+
zoJb{+0(qm+N!*r5cJROGx{YAqvIoV#^dU--6SERYg}iS$Cv+|et8mCuenfxO&cd|E
z-%_rXB=N)ebE^>O@jJk4NiHx<w;*!wf5aTW_&pZ*xqRB~T)GU`qKoFe&iZu9cHTjX
z2W<uRYO2k2cC`3HeJywtKAApBV%{d=X1-^YJ>&=$*qz1U!WL2EFjN+MIO57j7)5ol
z!NI)tJl^3)aLKl=g7NzH9N?xY(KMcq9n!TD?LBtBR}D0TdE8e17AxHH$MsjZ$&E5f
z_1wW@oA|d_T|lMI<tODAvxPa~Q(d+mieJ@-_z}41Ur^pmh#vtg0GZAEi8r6nA7TO2
z#6;9@?=e_Qu!rXx>`<^{a%NH1(TuUHR(tjIe4d%&*6@LgYrQWmM=adfT6)$$%f<Y2
zjOgJ=>gejF9ytubW(Q%*0}PRxB4h?Eu$$bE5E3%tJ!?7wgftu)m&NJN;B*$pj-1%f
zW&3`Vw2@tMl4qoI3`G^?Hr(&sJDLhHNw<nM>+M!aQrh`&0)p^(7^7^%GbYQx$jzb~
zwhw<oVwH<~6RRW!)c!c`)FfaDTUrb~U+dQ!8ldXH^A!Ko)NCax^xyCOXY8tjQ8_nt
z!mewWUA>BM;&$y^eEfB@z&pDRw<svlkj87I)i%^ceJ53NxhtEXOBRajh^>q^z&W%#
ze_iC;XdzMF4UM<<15!&w58d`f<%KKwKojAc!a0k3%|yj`xN>C{_gN}3E*wUkdqg-t
z5DTk^-7A9(=exOa6X(8M{aqi)8^+l<-zhHIr?l7`$X9li7K7WTv%9c*kz<_uurkw6
zS*zhi^<8eO?gsq<Dzs3?+iFs|AuSg}kq<vq2ha{R`)B$OxD3a7f>a<5?O?QkPA7qi
zx<2HdI`wN;NdNU~<g7Q46@q~1GtGoCP1*(0g+<qFFZ9uk`)vZy*>A}1`c+Cb{aFiJ
zaEu>@ol`+wPF<7`6oyNvo*Hj*Qjpp8B?9+OC-o_u)bcoaU>kk8#bj-N_s?{RYN}ST
zH<$9-m1c)Ura1l5*!Ex7fOPWTA$F_KEcAfHQwJtx^C^tL!kNB-U=rloj~4hvn9J>+
z;o20h>erHe>r~>C*rO1nB%xRs`PkQ{YJ=il_Z$b@tok&(hycZ26Q{=!KXmOgh+(C_
ztKqxB{|v;54+Byf8Hhl4H@z`<)znK$x1T<wdps-Q=c-kz#M)N%;r_7MR!3!qpSMF~
z`<Z5DVelzbOlX6E3XHN}GFjC}wu&gXV8=`|j14`YLGS@-DR&j=gr9WcH(Q#|jVi8z
zj1?~554|#V&{CR3OP2lxbD_oh$>-&N|GeIr?auSJFLkd8E(#7pRw2orQ2!1lRVHB-
zHoqq#?sh51he57~^q}5E8g5rlQb-T2>#k~QkRMe|zPVcUz|9FT#Rsm!dZ$(=*4UGb
z9CyrI@5?x&WDh=zYjqJEUjP>+dwT(D6h6=UubfnwP4l_`6;H<;da|P^fz8F1orZKL
zB%_O0W^c8bFVEt#+dt#w=E5muY^8ATaRs|kQZAFYsw=Ik^PsvYl8rz`#(VV@uHY*$
z&|W?z)?QAO`(X=sdd$NiFQMSY`-5E~NG}jR6m%fK;WZ0dlFa?7cB^_V(9yt`m_E$=
zRVIZ7UehSEaQQu@Sum=&=vrO6Og~k6T1?YSnPg87r6hh0w-UR51rv4Rc^i-(_KZV`
zN8vt;(MHHDshif?`xRRR=f|CR&ZhY+=qLa<jLZT!Rf765-fB(ryuHNjlRa#9ad6w+
za{d?9kkD3K59f58)`!+asnGgK#PD97I-2+v>$JRq4;-)<2I^YlqrOP=fipR7wbJlp
zPgCq|ot_}Nf)bjP1&=bcUa!8zQsO7t3)#438Z`gXHgAdmW;3U!m)!4F!o1r(>_qbW
z)h+Ivlp!@+zV5k#p{s7;fpajQ@AJgi6U02fhq~(XSe+UD@v^U82(Ww_;YXJhu=SHS
z)ApkDB9je-657#nyA7fo@FPh65vmf4AWM}!8`J*#%etjlzT`f0fHpCee6(BrV`-s$
z!+OKzVAs_^cMo0%H3<TjD)k`w$UcTN8{|ub1|WVJZ>&~2kaQG(WjP`YNy5Z7#IC(N
z=`+pur%v!F5Xd=$H#|W%)Zw2F_5_q~>c_PVT|r#k5V!0aJhP;uxLdTrVDdhOYo8mp
z{~(=obf;yUH&c)!Y-?<(yachi;FRZU7PECd<b_IGpGHXI#TF3ao~fN+SM)C4`abk@
z79(|%=H_|WY7TTQ9pPFQM-0b&$h!Wsc<LTFHnrhvnxtYmwD$Bm!xiXKew_$&o!VNj
zRu9ARKiDQCQUwE5*>0}!JxlB1F60`xklNK_hB~V3VgL--cO6;qXbpG}X}^@~s#H)8
z{S`U=3(WBBNa3FFso8>g`fRq`f5x&)U|IF*?=Z_0`?oAZ?R87&q#-WF8+mN2t1VVx
zC{wJTNUMwr-T`kd-F#%yQ8XjlUEuLcxS}%#R}=+}N0577@Z1-M^lNIIvCTb@?M&Qd
z-F)_pNgne1EqQ$uDVR7?Jou1*=$d;ziOXfXCo(BZ08@*fjiiKHL(t?WOL2$05gw|6
z-1xDy-QsZ{5(?fC>liG$W;UsQFYpwI?Hr__1+fDLho1=3K6^%Q<Ued-WOuygZ^rH<
zJsrfx9iG=M?I|$&0YsmGzJQVeT)`55W`DzFd_MNTPA&Nqm9hIUQNbRotbm3CxQ3H5
zLwbt6{qPWbr2<+U|7jg<9qlv<)m61Qxrx+{mo0_V@`tRGcrGli1N%XQ#(Fr$FTpWl
zjRcilX5WtLr090hgn#@gOdg?3*e{7O-H*Wtg#9ozg83+(J$e$tvZC-oRe|E60Z$O>
zW>Nx<)E@pgNA?#S+0(Hn(2-fo-tKV?v5mCKBn*HZ**EqT9Zy^MbU{3SKe~o#jP%m>
zG+`D1QXEgrwoZAwk7b)5!f)nB=*roL!(=klWYnsBd9e#QaRM<f3OsS7q<DWUR+H_R
zF?5Q3g@*&5N2et>ujg7_2KOk>L51`j83yAEbl*R+mLdj==2kCw7Mp<Zjw`W0DPnsg
z@Jf@Ej24MtA*Ai5$=9P?!XKEFCQyu!SJdLR-fN_~g1Zl(j_hjHhZ9;G1}l=FY(eV8
zh&~Ho??SLDCw3l6iLV?RkX3Td;g6+QnJfTsF2fr$HRU~kg#|fCJGLPYIS(X{^Z^9R
zdMsWc{G#;v3hW}P!A0B>UKo<?X&Fs3e*yDX{QH?)##h_IzY;sUuENE{jdYzc)Y~qP
z_q6VwU?uV+WkvAJm;!649ezjJ-XB2qOMJr=g6L825EuP|Q}aA)KFTI>pMBe{L|)|v
z*!#zd+*X=)&18%?*`j;Tn7ouuBP#9fOA`g(@wVmxvg5mcjp1qjYlK*3hhL}r`=ZqQ
z1E{lgO+@e!BwU;g1L2N9!XIOX1+`tDbs>>RuuH*lL|tJE-%uFGzlR*8*>-=5u+|D`
zG(rc=UP&HRfCJWS8LYY*A3?ykHDU1fw{R;rCqi#rW!Gn4;GSOs2N-ZT>7n);ERrH7
zDw%RyriO!8bV<P^+e+E||1)>x;ZUw`__Wa?IwwUXI`wN&CzJ{m)9zGiDoIC{v?;Ph
z$vUT0DwXz9m{Y=$P{*Dvq#c!n?1U&}9gLazJ@5B@!(e7iC!GFsU0tp`-}}DLazFQT
zKii}n=*s_%q$#$Lh}hPIfZ&1q8gDJh@xfe39pOTF;~xr8p{&TMZ1f8^aQghY$p|yc
zT@^ZDDL;-$#d{lXDQ^=&N${3^lSHviVccb@un30yg{j2X_vQuEK;#k`_6-d!e!*U9
z6sos*Ir6yu;}Cf&I6|swH`F`BC2{`)pqfp-endIez1t2)9DI8o)}SSMc$V`rQ;E8S
zMg#iq_LPSxMQp5^5LcD$7G+nr|HYdKtBw8S2iX#0nAlEd^+POty{TNjG5k6fl>ccD
zUxXYQv1bz(ZN6ku5rEyj4|3-J<i*vWVzeMPkr4vvJY|r2LF!6WA`7gy!+Z5vyAT2o
zd0liHc&G<r2=IykiIv$qV2)$u{WSwEdF^vS)6YRZ!HJM&NE*2&D^;aw!y(DFNf$Wz
z+e4Bep?@~cv%h>fAEiyqnV;AqKm4EMCrdnBxBCq0!K7=3P+Gybx^zt}Lp@xkEreuC
zNkkX0sHw5WA&u-{ZBLVPDM(%VgB;hg0Ey(R9l#G<d4CgUvk2QJ)`^%7;4g*)JR9pR
z7GzGFeh;O&W@ypVD$#gPS4f`w;`VVUaUoJ}Pq%L=^dGWzU_hIaX!ptlTawzjK>juS
zu;Xr=zJ)B(W}w7<r8c>mxey%_f;9vc80&F$UB{XS_QHB%3m=Di?f$>IO?lZ3{VT(D
z^8Pu9sF5|HJ)Q6-79(_0OG9bWvz(iWL$~FsMKVd%X<Om$1$|IuwuMmb;9=Fok9S-o
zfsiO%dB0}`TSCo>_in)sj2`XuMeKG0qerP&(LUId40DKG_9SD;MB6=m@$Ig#*-|1w
z?GO>Bx^Cvh^||mu%U*Mo!P=yP4^AzC2BD<gs4S^O8l5%=nyaKRm<G`il6tO3`bI(#
z&@Qo}5W9^Ivxguol)jbAdqN9sZ~IJp1o}Lv=YV|0;n`Db%vd35Oj=Vpb_1QzoB9Vb
z+IWWX4`bLHFzoQ`G03n*i#V_hE-Y`%QEU+cdSm@;y48C~(q~1G9qIMdS|E9oE!wKY
z5p><R@`JFlh>#z&ErwFD>uy-M|1O1p#Ljql^%s=NdX8b2`63r(+sk}n8%#%0wbMkl
zmFdRZ<Me@f2h~;i4%)EcZXN9O!`9mf>L$Kdhmb7w`^&BZu~_I`6d*Aqa}(D2ztn#B
zx-7f}=LZULMDs);^y)t~AE=>VOYY4o=-?S7*G6bUVj{Ts_`j2h<u7%Ivhrm{jkmLO
zPNNj|(7g8hhSn%`B}W(Q*o_m)fAQ#%P9^h2Q*1;K%_tBYIjy#-xuUfk)+)x*jQi!z
z^`J(C?%XrDbD>vfV!-<Xfvk$uQFO3S&8LQ;Giv$V)=Fv+Q**MT(UpI)5&0KxTp&@x
z4mXYwtzpOVzE}q?+wL(hUI<?NkG1Z|iz^6X(w~3pq{q<S$H$ONkq57Dg1ynf<v-=l
z`4=9XpQOs1$lE)9(y;Rc0iRDG-~&v(9z_YUY|jj`y?Z{XDBBAjjF;~GTac4{-)peS
ztF%9`k{Gt1At|3&0Jv=x5-3wqFVuHInNd3{=sUC`uzjolMh6oGCPVPwx`D&6zQ`tl
zO2XyF>zZ0{gJPYK<k}R#$O5znSv>7boEC7l8jZ<z9zgu};*r=n8N)dR%s-Cq9n(o5
ziv}65qZx(E=u;TX$H;#l2KSv`G&J|~k73YWAa32A{|ra`ya4fQ8ij|Yjv;thB~L7p
zg{csBpSU(u*MWvxq@M>!pAo4}TVM3i#N}8^uJ@GjoWEHa5B!t-JF%n2XX|Gh9JXPt
zNPz{p^eZ=ON_fp$V@)2%G02_o)zMdLegy-GQONUY(<;&^6;Jh4W9bYjZ0{AvcKO~=
zUqXWz_t=cem5`RB4ZCO!Gk4h{43TipaNS|N4y&E2v>(C-D~*)&buv8KadE=|4O{Zt
zY_e+=&9O}V3t0~mb2HS9qYQGv_TU`El&vfT&dB+>@^9DuLqT`)p2pkfbu<wa_G*c)
z99Jba*zH49f(n<f2Fu4YUN-p!p?Evrpx(+$jcx=Zy$nAoKo2_calcuHg*_9`A@>Cm
z1}k?SKbjDB8O_U$_ZcZbbb18QX`{(p6miF$e7Tg{axa+Y{Rlb4D(KJ;e%lb6LnARt
zn##eZC=Z$@C57fxg(;Y!)s{#GyjMlP{>8OBc`d_rpz%61UZF6^a$P3R8;AF{E$8Lg
z*94m9Q={P>9mx0gvnhB5JBLvU-dD<h0%=Vn`0Loq%9XXDQV9Nn`*w-j;$Ivn!YZYu
zu#mnnVH;(C^ghN;`lU=jD$jv{w!C8<5R0j0lX6|L%oQ!X$MzY2w-PWkS34;IKbwJ5
z@_36hqepamcjIjf9XS+f_e=`%=kka+%fF?g)_u&OO%I0KCS?Q5B?~_GwH-jO7RZjH
zV8>pa^@JU#da%QA=(>;25%DpMx@`s#Bzws0N<#zu3|dPe#<Jwpi$weMYy`Dw2iUY-
z$BeLPUniSTX-<s65*6&R*9~^SP2JXHgpNnI2|A#ZR_MmN#g0E_|1(Oy`^sZu7z#!}
zhW~|M^8Gisx3w=p14tr4<njE~n0sq4$wfLD1dWTqudQED3*s^#exSx7Z;%BH$Aae@
zBM0?`H{jgN)|0j=R=&ytcK7xG!tRh?d$UeP2-H2!>4$C%`!IYYl4{39S`k$OvEynL
zJxiQFvtR`%q;i=J2;ef0dFa?=U?o{C-TP+)sVm2FR%=1hgUbNg?NdRZ-Ss1p(?0Q-
zg=P0ORnuEww!qw>!D?SkngQen>+&*5daRG?@O~z_6jisAP<u_?6bZC)j3M=;rVHQH
zMH%YX^H&$)(R8gITsJ-y`JaXE`H%Nfn1FtpR7ykUc2?>{w0bWmrH~84<LB5nEr^Rw
z+|_tn`9^iT64Gllnk2V7j}PY_x?{_Y>=ap5upYqa4Ri1zxNf0HXr?>})D5*tEEX2@
zR(YqmaXs3z31ctZ_U7`I0+hZD1}WS)g!?r)tvvqy%T=kHpV*r-m{nyHDDRje$e_!r
zPUT|EW&j;C0t}T)EbKgc#kci5oXAssQg!fim=2rgcm?-uK7V&qL7g|v;_%O(k?wcv
z*3lKSAS=*ByQd?@xum!BO11_>GU}-V{HAfZ?*_q~+7!U#-ZT*T=h}5V=9Jk^ZKI-A
zxtJ;IOCu?ibR1%+Pux>zL>M7*N~*F)YDmX?+jr#CK&{od%x?~i$Icey07iiWIN_^J
zR0wMx@>pq%^XWxIZulD;IMFk|kPX41$<~C#m!q}D>Cpv7QtQkA$o!rK6ap(7@_9Cs
z;n|G30qtZIJ$>$QqaCh7v_q^S{qwy0vX;>~Bs$up>4FaDArx<<@3rI`G#dF>G7u~o
zdP5)dO<lJO6>oyz$*&CE{#fJJSR2P^+>A)Fs$)|K(8}opz?qqLh~s>-$T@6b5?BS0
zcU^tb4GNsS3o!%^Y<bP!mL)M72b&AMspaGX>SIW1HsyHdM9WLMr0@HV`Ua7(Q0gCw
zE!>{=Vi*ueyM)UL&|*M&Mv<+ZEw>Fv=(T*^8au<uH=S_Hhak%EXzlNo1fsE|oTg)(
z3FVhX<sNJIp-G{J3o>&>{Yt*aiT(a^-1o~EioJPYx3O8H9MtkqDs#PuI?eFD#=+LY
zs`yaGk%%}$%2;HNp>{#75A_gYi=R!Zi2OIfWjr#Zx1=T;Mo9rrA<bG1Pi0)SdH@C<
zzp(z*oH2nOg5t|?m{s^<1M$W3+pidsr2`m4F?J~I__>W18U)Jp9AMa#_Z=?Xm}UzA
z2KA~aA<jmU%h_N9tVCb<_V4a;Ww7R`tSR>MO~$@2z_}>Hwl+N}vd-<4#{B%G@HDE~
zxGiIE7c@t^Baxh`-OsDu9$5LS!YK`GYwf=lBp7-YAT^`bENRE;5#Ef^O677323bC+
z@D=OY>&C>sP!|j6`$LY*)yo&oEWyorugZKck7Y88AdQLTxM>_xGVEb8UNUUZEJ$gX
zeCAH+X{f~9-+zzRSHtnvrJpp+KW&TqTFrF7X(e$JUXXRV%f`9Mx4dCzOU4$CF=OJC
zjw-kpf6dP!HM}WTs!rPpv2#u}L`?$|srA7T9XS9X^v7^0hwGF@`x>gwGObX_q+`uj
z_Zb3>P9Ku&#+cjLc>9@qz6Xk%;gqH_SQH8isWI<X`-#mazJcv;gVAZsCSVg!(_9h6
z`p8@i>83=ET4Plj<w*(9+aGl*2-q{8LYFw^Ry*beIhFr7p3%@O7S)q7^6LJS`c}2`
zV_pp%a$xJ&1<MSFC=X5?C{kzf@KB>dp4N1E`&9#%4bV9G)neR{pKnxq4c;ZE7p)VS
zB>CGo_oajLuH2dHd1vnG-Rm!EMg<5@{wdp3<+n4hud9{Mdwyoz&Rs);c8n5w{(h=>
zPAqv(<vabxz*mdy1C6T6|45@r)ShD8)!+OsNP;z{)?w=008H+|pPe*nv_GOVxE>qW
zY)!oBTkJPI(pm&PA{gOLiiEgf;d=nZjYIWQuF0!x^=;L~>qqV2oVz*`kDovyWLMDk
z<=a#9&_LCU-+h}GGpIG>UFfz{6<21|lmY<jj}{C_pas7jm&K%gBqyMKPw`9avV?8G
zDQZv{W)47q4BC-XiAjvy_1F%WGTfcf?Y~`u|27brJqd}CzuDB4K@**kBCw=JtHN*4
z)wt=u827XGd#~YZiZeck-`%2q%TQtVbW^M&y?3<@nx7heL3TY^NhycQ7~N#_51=lR
zP0iT>2G*7b84(2{&Z|Lp_Kb!lwzaTB&$GRV{bvmpmC?}lVoc9-gwIci!U})3)x_<D
zIO#o1_|msMrf+1+X3kyDj_GfRn115)M0~gRyw`IuMc{-xV%|F%Z{K&ni>4F-rVy)`
zi`BpnXMVW5klV2=^Nmz(31YZ|r>(w4n{Ckc0w`byV~drq*M<M9!$E0CuN}?|IGh{)
z3lXbBaTINX96nPYa69mB`o_JI8ZTFG^G6cpI&bw5r?PnGX#*3h5JoEXKIsZ1z^WhR
zy!0qyPZKUjz@ng4zL8k(qQlM3V)KFiz-M;Bm;T{}5eAOq`>dhRjS|I1w?NEy3&x+&
zEFFvsZ@AB3JLz$LBa`;bbb~hPr^P@ff@55sEQ&Le$A6MUP8$BnWe(&huQf7YnFPvF
zTrAwNg~fVfOtn1EEkpatK4AOG-dwCkq(gCO&4oGzVYnIZq)GHCq|e2bt>pM|!kYOD
zIxzkYxsAZ~WvW*eh`@zL!lSlGYB_r>eoH`^v~1xcqHXa9#;(%1!Y;FJw+kD4V80E3
zn|FN!zIlDjBBY=TF5$18m{7CxYMB#{D_71@)FRUP%0t;Al2bLMcwRDP-_XznN;x%c
z!{}Do&@Eg3<xNIoP1%y`bz$7^7@L09$pL`IP&mHkwpZ`iWJpllWkqlY5RkqbCnDe7
z8&GL2j;&tXf{oH#wU&Uk-(OkRFE;3Z)~-(54Swuk&6|va<!~;+apQ(+#(l%NoN?)s
za}k1b0bO`Rv4O{bFlxWhnTxWbye`?OahflJ%$*t*!ASXB$bth*3R*l1SOrm$550a6
zgPtT+t{k9voB&-9MrmMmW6){joiVZ`=AGy4TP>4-sJfDxm2vs4SLm1Emg(x*TOVYY
zsQlu)*tD{}>C&MYt<$bOIJ13Rw{Eedm}baqaqPyU;BxR!6eqwxhu|s6Om<4L;#X|_
zB4B&U_-_kcT2)?AdJ1jEoJpi4Z<jp9oVw5@H4&E#*W%s7*$)O@Kl@+;&n-*DUq3XL
z-(FNZ3c+&HZ@#{faV>9Zntd7BC<ktkQ)AWn>FLI{lB{NEv}C_)M<zNHpTZE2WF`XV
zw9r^JuG#4&Tw-V(ejJbCz3I;0+_&2psnLkj-6LOvXGRGl^2`|<e<wUMaHw_Osv}nT
zDv-N0FlW4ATN4z-(l%d}c-gN<zl^m|p|C2oRgm`Gvkl+ZjgsEaKcNBzoZ6m7Oyo(6
z4Naa}#uhcV#K+4;rkzbqe>oaa+&5z^Q7C<noJ&BS5N3TOMWE1nxmgB(|07VmBwQJ!
zF&LTQ%0lSC-hnjcLInNj_3Jmv88!xa4vo*1YRE?51a38*G9opbx&`YqR-QX9E6+Uz
zXqSg2*7ZS+1vyP0xwNt1(ysdJ<6%?c#crImsuijgws1JMRCX3pCttnx;fN{bv>`C+
zpJ`viyvwLSSM8mgmw}kdSl^iaJw6)5l0yPxY0k9@^K+Kr`x4+q=n7-A_mBvDXP&tM
z^2{y1tpu7k6ml~pNv`!~OGf34|Do<t;5$R;_s_17I}?*9F&Oq78c=8%DF$|RDfD{e
z0y3hT+Zt{3boE%7=1&qYU*mn<uudvCf<CwDcd%)*uM3f%X+?AMXwLh=quBuaHg1x%
z?_vi=o<SIi3JHx?lzEg6tHh0J&Qumoy6nco9FZWOeluqYn6t^(g@`y>54)R#&Do-~
z=2?#8(s>#!&8^YFX3V#jpymG>@UXidyp3+ASt)3=WbyE7#LC`4d}S|!z52{epK%6g
zZ@1l6{N-q0d5I%pkPH|f6vy^2ZNc_0v2R?`Wh#xztYi$U0uuXaRVU^ZvMxC3da03`
z(!8<(Toi33q&A4c0|-m#VoWD`1fDeZW-XAk9r{S_(y-|V0mxcKm~UJB&@K->AgrGZ
z4+xffVCGyI@Iavyd|Uj1dOvB_a=PLNctF{Dx~6fD-zbzC!Y)T*hiOL*{Jwn)X=s1A
zwc+5_F3E}#hw~oX+V8%Hts5VrnA5Eq7_`jrMJAn0X=I*(ie7t<n^%16ZVBy73M&N`
zRo>^3cM{wk9fG^lr-5fM1fJ*OVtC-0RYDYM#NiO$4L=;{Cl7#qdR3omoJz}r7c*U&
z{6eAjq<^V3XEG@b#m4oglMIdH1a$`&O0n^(K1g%U9LaD!y6dzne&O;mi6er@b8sQv
zBZtVd+jS*G9>g}2(s<Zd?gj5h7T!;_ntfYahIz5Y^&92_T{f)38UG~d-Tv==I`|q#
zVAFM!(aYiK0HKxeYcxy#<=Utrmzdkgl&@tjai!C%)Am7HAY(*rt|EP{m)0|ldxGL4
zY(t)+sjGb2cVdU2TVTi1>nr%ScnF_$*qqMyR&Dqa;Z9ZCsz;S8q2u`+Z^Ebrfp<Hv
zQTvHksWa*N=C4klGrth#NPt!hO$yRuUfDFUj(iKgjvT28l9P~J^eqf^L8lcohz(h0
zpak~iW|aiknhQDHwBhVE8c%RA4bEaAS)MGl`enujoJ1{jNob)EO8G#>&G09nmHet1
zgEua!OCo!ya|T}8@LAa0>5~NOF=I8lY96sxjdkS)cwPV(h*!>rZ}R;IUlZHIeZ!7d
zFxZC6eJe2l5kdFcO`k#-wail<R$>4wmp|lh7cdLX9gc4_X__7e-XJ}?rB*^an5iKA
z_?<2vgAn@E_XasJCaL)jb_ik}VEvvNB@Cdll`c+GN5$KEF3{RZ%IfK1PVj`BYRS3O
z3`h;mRTZ~1+E^~pNZ|z&Aa@M`GXUqiz(u->O_p<q2~Z8VTQHmu+w%+<2nSBsswF?%
zO?H618H)kePMC@e!f}7?mD{ueyfG!>@J%2ee?9b$+jfK%QdS$$s~LZw3uj6{PGu#q
z@{{YeDVNc_Gap}#U?`t<b;rdcc~)>(L>3H?VoT7q15%sfog}3-+w{L6a^sgWmv}~i
zxzL-23!O5aU(kpVz#q@XUb@=wGh#I*$X7TRm^+$Mwg6sWp99WK?d8vN=BIOWZh1Ph
z*brI2+0$gH4Ds}0UfZ+MwW8j`cAEGn@F*Fk0Ae_3hrwINwy5{PS=9G%-%isXOXxEi
zTmwF{3R6yAzp<}-7#ecPEO5*~LM`>)+jeb7uM<2=AZ&%MU{E_-EwvqL_ctJkxXC&N
zCu0KadnQ$z@-DRXByFSRPdcooMdaevL`QyDTn<cjquw;b85%ms5%L2JyN8~L*2cuI
z*Ne4JCP6KC;nfl;u0T#l*`x4v>Xp~6^5${O25JE{6V67I{xL&_(E<rvj?!}2Hqrh4
zz>I1b2hxtw#RPNNBT*|6z!AR40b9tMKn`GVzA|$}($3Q|kBiKR0`(35LVKm9y#K%&
z(!}aC6UYx-RPDlBRHa61hIB8qVK6l?tR2R{;G8=22lC0=@Ui%rY@y9>38*l$9AF@i
z?%AjliwHm`Q;(Z(TZ(*Jv_jn>gHeqa@76J+Nlj;|AXJkD2P7Bw^~!XKv>?_@;pBHx
zSo}fCfBFXqh#Ka}-|Mu4W*m?k?`#LB)?Z;yCG+j<bn)1588U(;PCOMSolJ}+m>Qy7
z-VWA#X4}+o@oobw|M@+aRz=<qM+6y<u#~u_6&w(xO)g1$?Vb~G+scGH+qkx=(s`p)
zh?hTCi#;F3x8=t619{^=nG(uz#0wH$8aWv+7@`k$#}Khy$Ee|kAywBl`-qiT6nxE3
zXEm2GY*<Eg#T=02%_}i7XpxWu2(DfU_O_gZmxPNM%Bv79E4UCQf$a?ej&=g?WSdBS
z1b<2B3l54MiJEwDGq%L*qi8fqNS#ROdZ<*F%KOiBQO->eCbKeet8&wr#WdD2@ZB{O
z#TmY3046Nobs7nxUL-n;pkEa6zCF)`ER|TI)pN?g9w6u2^1&l4+`IX&b1>8wLPg`}
z<3)3l#g<u^V?(bogDiV!FAlX^ttI=U=un(feHfW>hT#vnLknt_+E{JYNP(i25RIGa
z^6th^H1N?S9%1_10%hJ&36i=$qdp41_9C9AqU-Qi1csM(c)b<avdzg`<BmiUWklsc
zlm@+40GTc<4cdyH6(V)V-nnZbRb3cSO*RFkYZGeQY|>X!WaLBUXmh}YMZi_yBBA~2
zD$s2ZvYUYF5#(*1VigwqHZT3DS{%Ku(s3sNmqYY!UdbvtsG=FUCSGm+H%av${l26j
z+l<_(v90T*#%i_#_(Lc%e#ANs=g<frM>>Rc46%M=kBNie5f{1GCFWRXo5TObdnTcR
zuVqLv|Muyiqxze)6ov}t+tUoPsW~8_`Bd7mXcnmi`deu+<GLFK{r3OHsLMhp38QQU
zGHS+P+5(xyj1r|98&(nHIlQTqg5-=wi2@N<S;sEd59rN6D&k1-`sYb`?_4wJ7$HpV
zFwpWE&RgZt=*CWJ#z31F!erp%9avPZ8$x@Y7H@|-k@7L)Bs4#CxqkoANv(PcCl%)?
zg#f#kQ%2eF?-Eb?G*#<(v3YsR6#6BIQ#WFtrWrWO2tT<Tz7CfHkz9vID15xK-#z)8
zIX(l9n#){6*wzi1lhOQZ^HGpYBo;+jF_a3vu5B`kfHm+|Zb3nd5CZu*CGz^)aDT+X
zkrDN-z??ZQl-bt_Qis<A4ZlqgbSG+#pVV|n2Az<}p-I*Mj8j<St8CpBuUoygbtp8y
z|NQIP<|{OF!{#c7n%G78)I1>Ozk;UZc&qIi6(8Oq78(THQ0I<`YmR)pgZa{9K`VqJ
z8vtQ^d#u>T4CK0x<O>bjrvjU$-UUmb8#gpI4?81~$16n<Ew6KVj-{}uJU_GS)!Mt`
zhZLv<(Y%;>H)#cqHWWMbl$x3WbkNr|1%>}Jvg>Yg(E;hNECNxlK@Pt$@zeFUKjAw@
zhrZ8UlFPHqz)$J=warsc$L3`UlPN4_!=couvM-d@ZA?!}b8SYzHK}o$Hx^(m(=I|Z
z{p9m-VNA%3(0dwaz8lp}_uR280+EmLR!cYFVZa=D|25*av&w^W9jel6zNDGM(9RQ}
z_pt1Ks8h0xn2Cv{+43>ST4Ggx^@tt%o34PI_;Dcl#CuxFF&sZj1XC)LL$nP_UtfP4
zf_n!<jO&=B=;NFmX<)YZ#fbw-G}A)N-K`-IllXUEAIIF8X_*x`napfU5hgpmXAZVW
zjS8hTSGGnR*sD>Ijk^c+{&No)E3|`xK|G_45i_wHry|e+DO7HGAePT)!;G<}eabGQ
z+5)L${<*WVv-{mMbfJ{7-S?+0^HI(^9|CO872#Bn=&qH?kD*nQ9*~mY-lga*&S{hT
zk?biOdlZG(w_=aEU=SFu8gVgEZ@g@()~V%YMGjv-hR_vrL616TQGBBt^D+YP3T{4_
z<Fi|ni{??U$JmGS7Y1INe(p}G@!#0WZ%*78Fd{6q%N=OjkS$Td7`>=mdjCFQl!|X`
z=gVLO>usH&i293nR}6^w%8b29VoYQ`V~V541d%DAASq#v&(40Z3stf`+U1gav_+Ry
z$~@(1B^GXtmA3%5zG1Ux=W()UKL(0mRny^ZdQOd)LDq+A_t$@d>Xz~ff`Q?WV$|T=
zWoRgN>kYw5+t5I@npk>WS=m!YGef1?!)o6b$ioX96qS9g_haN_7w^3v5;~^+d$$Zk
zDcvf+Mx3g7WnfpTCZpBxutu%(;cXLfwsukhGec?6{y75Fx3{+EEI(ZHTt1gt0+e``
zTJ0BMY1A?+Re~#><craiQ9H9kGN_j^O9VVv>$G<P(J%)`7w}!M48r$@Mr6P;Y}f5d
zqysG%m+zkg43wS<_gfosmbVn<)iR~vRW;jKhPA{>0s7PzSrMD|MR2zf>gAn|4`iKK
zGoUMZ+lhgp5TSwVQ9{0UI`Zg9@r@^e6N-#w7s6c{BjRiAS%)bXA%BgrZ*!=B4lXdh
zZV#M%P9MbGcH$&KeFPBd?E24$`=~OZJNL1*F$ti5;e_yqDDeJv7V=lV6kAkfj1dh@
zF0`p39{>Su8#gZlip<le;t@n0*hzqfD+)^t3$#l@a`4gdCSV7m$EJW2ByLntydfkZ
zyR*27rrD(TO{36j8cX3>F|ipAuh-rk?vlNuw((}LYRf))2t=vkoaB(t=8O~ZF9ys_
zHWAqAVXAywDI49Of3_wm(MxZCMj*?#0VnBkc}8KX81$74OSS~4Y&V8SI()4zZ!(I4
zT?rAcMmH(0gLa+H6xb=)DopY%)8WeI?23~Q2cgELfN2t=DG22H6sqr~eEw=k_t_Mp
zVK!=?{qN`rxhe^SI_%g#r?s#`CO+G#oSa2118<k62B8T!ok6Ve>d(aeewds0x(O}H
z2H_UDN!<y=Fbm8k5CfKSYRCTC*4^}Ubmg*>Gs<XI$22U7DqVmy9G_3&#!2gj^|N`$
z7~j?yK*8~z?_Z7<etTd+cfM`oXBPyC7w?~gI!#+W_@v=j#~Ca9qzqlNWj<gB^Hi;+
ztyL4DC?zRF-HSs7u9MIxL=;z2O}91^d-?zdhzOM}ln9C5Dc#?FfT%K43^f6dW$(cD
zf;IE2lkk=^s{0;|jS%x<R++c44#$wvJgusWXrGMhg_WTZBY4VyPpWXEb`!*ab{vJ%
zVIzVoFrvX<yK?8M6aJcz{r)Y858i<Dhq2P!VC~bId}=x9Y~2UW?Mti^IPK{(@oV4u
zblO$J1asPi#1{B$OaPL%&we&`0xFY|M<Zw-4WD-&BIxKnV`YW6iEK-2N~Tv^DB~2Q
zbYc;%0V&s%VT9;gj|Xq%b!OgkJD#a%P8A&X-|lyjrZc2H1{nE?m%l#wAo&?edn^Zc
zt|dBg?{P|i?P{&!LLEE6^<nSaLl}#CC9S7NijA~uj8FywB}e<Yha8nX8Z{u|JnBYs
zj7D|0h#-S~gGEChB)>owi2=4MzzHog@yC2NJ|*D}htyaDdG||m7%1fAd25=C7fC>y
zzW8Md0$n;bts1f(K_Ai2;nz{p^Tm)|8g(MCV>(-Cb`23X;FNAZ(`j|eW@<oIYDar6
zA7e)YgUsW{frbqSnYcqukQ-u`<~#uC{L9*fM>)9o1QDgZpVY1<*vZeyV})$DK}o{T
z4bIwugx&7DjObjQzgD<$)iu0ZbzWk;?yF0a5nQ^AXm#zIYU=RrIG7vXe>Ufk(%y|3
z;{W1oYQw}a?vo+gW%Dmv@9Su`Wu;{1Gd?mmqXRlr+p>;c+!7#DqTnu_-@$+Cp&B9*
z9}K<jGRDGAe4qsdQx)sPha;L165#W|si81o1a)3A3?z;0K9Wb&d0%yAqWoYBWTrjv
ziBSt(O3UO%K`bf5(<;bIHSQtPS!!B(QcedAzXu?nJ;F*zr*e$2vQq@K*9<t+r<Spo
z(VtB}u~AXWIT2<hL-pg7FJ2Acox|y^J%xav%8}@*Iqy2H&TF<9>l^`0+R%7Y#;Gp7
zsT}Ux@jUN<&#6jnep1uZlRo??Ch4v{i<yMI9?T@*4*KzGn*Sacg*k299!TK|%7D%P
z{ztiMg#7gua{FE5F=ULL4@g4N@-LAr>%9wV*U>NKsgx&vsm&;>0~c2lH||F|B%QUG
zrc)w@(k-2v?9I0g>vi?)^mqr#KTvvK*IOJfec&$*F%HIEhei!o+!3$p@P+oB=MCGd
zI%s!7YA}sGYO&%;yI5V%Ny0s!S16>S@Y8cg8C>`k&~?fNZ_ujBOl*umU{9>lpxt)L
z9CTve_wyuqcVKY=gj*OJi!0+h3NgNstjNY0lEAJ;43zb3Wj8=ZT98x?m$7yN8QDJj
zAS&xdNC;G<CWPi9mt6oZdq22rV;iXHD22bfOF!weCxq3h7%NmD7pHA95GAF@hdtKg
zPmacr)2%vQ`Sx}3dDJQxXvvdS&1tj32U>n4leU-)M<$IsX+NUJ3U%4Ome$r3I<>We
zo(>M-E|jSOb6j3*j_=JJ0h1c$l!a&)M}(@5f3~wC<Hlq3elofM_MMV!Y`_)ke*JH@
zIm{p0)BK{XQm+(v%RBhkgDwO$cHB%&MsFr?zbY?-d&bEi6#T|*HW`nOM{5F~BpaKh
z9)(&EAkLc`L;Fb1#l~IJY|yF<fEFdpPxSsF0gs=9;VJkbJ$`ostAsKzE>K5TrPCpG
z<k&z-t^atf2H6tLk|q^6pa^6hGj5h<?2mFrn@q3`3|az}s>7W3&b<DGFh@_1Rig}I
zEWK6JsqvpK=rM&Ol&!CD?d5H9;ouX)ZF@XtUTnovv$&>)ZM5kzR;i9{s?5v<Bvb^c
z7l>T-SL>B1w0zg;H2!l(J$Z$}?`~j>VkJ^rm4TyT>mf8q#htsr(ds|u8(~MIeIXw%
z_n<f1x9zWAq^#p>*DD7lazXZc4~qQq2ue)NoIch~eOj=d3q8Nn7%xy~ei3?Yok&-r
zgx71*jNr69<H-(18DxreSj%{qm*Txs5dnu%k?&xCk1$qIs4iCJIduUVyAShXF1$^n
z{BTW0a2~Z5wm~xGg7g`CM6b!l^h!Je4!IHk@gqWPlqYI>d!GtoV`M#Lzo+(+=nbdb
z5>SP(l>4?(t8yHL@?=qYt}Xqlyi=)+y~9BUrRcDG;`dcVn6~8}jN<y8#{(%Ec045=
zM9xf8Ezrd1rig<P>Jk(k7gK7=(o`Am8L~AVR(>|MUs^M*Vl)gnh7!LS#X5gg5s7t$
zU_@khz7iij7T{ZT&S8yKr}m<V*FY~;fG<jS1p0zaHQBb@V^Q3611U;9FjHmh74?1Z
ze@{3fbQS-=xc&cD=7+!(Hs}OlCh51E#IMeqZ1gZ5ux)iEN>h$g9n%96?+B2|X$&s>
z(>;+xxa6C$1wx2akZZ?V?t^GQ5W3e{6&SdUrpUWe9yuF$LsB>AV*(i}aOKPqb+|6&
zUm9}@(;xkO5L4>{WTaOPXo*fWc0h{->K7X-YL$xBq``$NQFUsnlz<6;<}6Wnf4grF
zuY_2nRjeo)%tcjQ045eG*tA<WVL5j5PCdP<CiI9ht<c_Hpa!dY%4ScQXxo}R@)xol
zj3pwXt-91)&BjhI=#+wx_6mU3xw7c+?hxLQM|Na>#-y%}e5dReyJ39BcOuHVEd?@1
zIAwDfa8l1PR>O6(p@F^}*<dlHai^4`W*CP>9MWIhW_7q*glh8d5;9|3blTSqbKEHc
zwevw>>%0x?U5Z#3E1DjXuTrk|p(NmQ!P7a4S|;766Jd)9H+FYBw$S<7eYXS<;S-MT
zefSa;dp4cGk`6Va8SQFDXrd*C;<{|l@UqWjW-{XsIHzdU*0z`Cpq#~|#E<)t4(f!j
z5=MBmgpGTI78p0lso~JYGZt6S+)LR@c@GJLnS}OD)=0RDo*cOkhjd=YWN*hP3%fsr
zG++mJ>JNR7bRPT09h<fXbspT<aDAs%3ZVnk(P7p1{mOijVoz-~qWQ&I$<Z8(vt=0W
zZd3!7%DEf<<2iiYf1}e#09u3BnPdGFy-C#<olRu<4=bb2v953H;v8yIYE-U`Wn*Q8
z6*Q=M=;p9husFs?+nW!-jqTEW_#O={;sLQWvCj^D+;cg!)z^+eS0XjcCRtMQ7nL8#
zi(rO-rrH5s;#Y8n=K-<lRQ({NNgPm{h95>-YJu(7JvEJY?zS6Rqss81*XOHT`;5)T
z8!BFj;2|~hCTugNUBF%Av==1~*%XB|YiSgOrcOM`S4IofK^yOAi0!*is`0c^soqA(
zX(T^9?8KO^V&;2*djRT3?MWgj{p6mCJ)B*z@Vbj$TgG5QVg`wuA##G$jP&KuD>jI;
zt*J=Mr#8dmX=*mE5BNrV`bM*nu&2;n4L_Z6-A|`mpIF&_a_z^6FJy;uU42mfa0^->
zuVUd#=jhz=lm*1+$ooi4k8lK_kll<D(yO|YI3{qqwuo^$FnGMlbv1dpi(7<gMsZeh
z*Yp-gGkv6#(-|BU-G3w6NDyIeDdxE$Y~<o_<ko6GvFBVtK)U;!>v@egwls;FG&mVK
zBxe+#bU`sMt)t!0N&1H%w%vORGlp8ueJN~4m;!$b6*FbnV@&ynZ3?%M-R&neF0{U7
zzOqRzy|1pF>4_MntUVt!XdylNv%c0fBV2Q#Mkz``e>=5Aswc*85m3b%CCu|p7_S)q
zH6A9dF#aCj_Cr~ds{vETU`?w4*8JpNoS8#yO^upXGdj(wuoeiV@u?308{wYTa70aP
z2XG&1PqJeTthYw7W+lWLwBQ*@Hdf4-*G<$6!$71phpQc?xmA%J%XKf&KD7kYlw`T9
zaZ&g{)5eIe+fU+#>~eIz=5A@nrsTt{dud3>NI8#;Mhba#?KfC!?1MN{%FMM%op0$D
z_f_5L2~z#>ayD96z{3%+i5`Ro7_@t@mM8F(&cf9G`9s)j=`3FCF-U=zQ9sA$lB%q#
z6ULf$xM`kN?bjFv@O4!hoY80mx*3!fx~K9=ZTe3|j~#LL+{Hu=Z;Q02p4`-%F>1NU
zS=o)G1NBTlr&i`iIHP?9tvNxIx7g$zWEH&JTff@`<s~+_|7J)r*P&bvgz5!7LSdfo
zu2{M%{ZfHS<{do;W^1&voo_uyd!bVd?JU)XI7YV8eVX2!bWg<Q2sd-9LOXcNSQQE?
z&c#8r>G4)UtD_Hx|6$Rr;m0c%=yy8+_CpHpSt()$PK{?_j^q(vc_~z}$@#82TxZo}
z1ojLA5hJlSDN$-gDEF$moyXR$M9WX}=es1SiIkvr?hx9i5cwzc=}xJtSy=x^D5vrm
zk2Y$1uiJ35#>l#88fFw6QcHxglWz&H+Mu)IX%8xG->}W0i{_7{1`&OpzTE5zGig+q
zNfQ=pcL<>1$~jid(VCm#JFTVgIW-DVN!6SD;;rOLVP~r_B^8if*5{OF_j9~5>(0`&
zJ?MFS!_E!108m#UFv;1Pj?Ax2cg(brJ)1UTn(rRFnhN<mDl;|eG8)DmC;YnM5U(@=
zlRLa~8xR9y&<}@({L3Eoj095M?xLL~GL3NC&b#lX8^tZoCe=e2YTC}tH(;XZxOOmV
zI<9>DZkItmvSu^QG>w@>jswIkK6c!a^c?;^;@M1m@uEN~S^|l@V-!wc`Lv6?6EEt)
z+>9D4MiJ8=J0!@oyS9co6@;;-BM2Neu_9Y;HIbr7EKe)h>RM=*ln%X-#V1_=0!Y8f
z&yAXyzPzG_+Nwg^9-umkV{jFxSacm+eO#+hTIg)U@?o^pc&xPqVf5IKqi*E$*!6;~
z%kCEqN5VU-deN{Xzhys5><ezud`^pajN*a-@<-^IP|9_U>5nrSNi3MwD6kM&XT$H(
zfr3SUUAm_i=Bj$wsMQzKs;G2yOtUegyaR}4D(p$_STFJnmK`B=k_3g)QuBO=um03%
z*5E?e>E;%s->7oJH=2(hNBx5<atNS90v&vmYKfs5LH3JumkY!(k&-1qcJ)q+YgVGA
zF+A+=XZ(T}6YS~YuDf|~j@<PF4f*K>F_=ZWsQ_DzkS==i*TSEEVl}K@v2?SZDRaCq
z=v;D(zyDT0kiC7cXDKaCiDA5owe--K(_9xHN2x2%b)J2Al4y@a=gFXJ4bmPKtYME-
zuDt-Ye%5-?+Z9u~`}8g{Zr^Zf|3-07vR!fS^hMe6)RwmrG%^!TZ3u)ZJ}SdlN<RFU
ztPLwUr3q$@DMA*h$#FJu&FYjFP&SbsZ=6@vJ&Q$SnjFK%4+W1`-=x1sgkKJUy5D^x
z9jcF`-sFg}0+hDGm*R@_;+MiXNt5gwKf|;%mv#zF+H(q5_K8qVN1W=ArQ5OVbm}51
zqfVkJ4Lc|T6yhNVjUCjPqpR%!;f~L5^O+8BOuW#;)fOnIpOs+39q;Y;yW$>98{$ir
z(4D|#+$_&%%&BBGF+xZnwjfk|Ae|rlAU^ogX<*g5Hc<z9SUspN9&xh?T9d4!C3~YF
z>jSpfe_&B}4MDfbyG~~;qT1F%&CY;iX^X7@HTzJFWi?I(OwB*7<ld{=9b!02slUvF
zoMN)3gYp^QYf?lzuRrb|IvfycLNozzON8906i9eWI|i{P(8w(C6k6jupl@Dak>r3&
z`IVoQC}q#d*%VQ572)K?^dnST=o?AopYbj=(<C7_oYboHwW9v>e@4Qtpjr6eDDR1$
zmo;-!qnD;tQ&}h`|HheUu-U1dy2>*x2W!|xi+MR@GOggy!eOCArIIBOU%<KlC>n{D
z4jRLnU%3|Cr!XO61zzOY7wcdV*UDF1rg@#AQ3s*be~aR1ZjR~6^`A-4VU!6HDB#8Q
zyS6G^itgH~AVN}N#W0AV4EzL^S$Dj!4t4K#`7jY|VDIZSYp?jWk*o&hB{;9Ec<%x<
z>$^TOp7V!2)>q#UuejQjFlTDn%lJL4!LOr)xB2rttPVjPZ-ci_Yuy>s2bAmn1`D)Q
zi#O<;gM^S%&GJ8z6FdRpZo|*XPOWh$OSY+^{<M&`J=%unA>lH`h*tbSzM=%`d$BL1
zVanw@vb-r(!ttdrxd)JWXfEUyoc9lgQ(!A)Kgrg0c*8U2WJgL#Y3k&+yt)R262&DD
zxM*0viJKx?2Xrx`VQOAlL3DN;{S?D!04&z4sSQb~Dcq`K_G26w(&GbdDb*SWbJn$L
z>Hs)0dX@lXzIxo9xj{|TAuxFzWp3ZZAhp1b6PW{$@ptir|Djx`%ahB#u4#@f*S=5(
zBD=PBx2vlj^VD<pt9LVG%+2DO;Wmy`E|)fz(3X$C>AWwB&_`qI6~l-JFREIfiZFv7
zr1D%}Yt`2MENTvEh;?<sp+lTctZNgOP~l3_-!wnpJ|dbmuh7H(Zy1API*Iz0?jpoZ
zh+vF{K@;w9r0w{6Hh~bCZ_qV|1b1Y^0w7q9gZ~W@!iB!`EVgUxvu|)GeOj(dD_k_I
z=h}D~G^W--qN~(I6@okX02-iVd0a7jpGx~t{ntvD=Un{<sL_h^7*@z%Oa~?eOp3F&
zJ8|u`iPvA8+4ymS8iNwwqUpwZ!zc$gpJ(6DmO-<eH(R3LCGF7UFcKOA$?Q&ImmBBI
ze<I(U>pGoY{wKiF=Y)#+u6n|hi$M*pVu5G`Edc;rmhB@I?ecuvxE2|-G;yIz`hoib
zD~0s_0~30C0QRT)8^~jRevGD7Ouf6j;{n$HASmZ-BhHNvsd~sxXe8*nCgs`Hq(kA4
z8HZ_Kiu#pq7g#~4_k{N6agn$K)&Jk(3+)XYc3@^rAdZY5ulflvjm8vPEMleIq>&oa
zY1E;xQnji$J0SSL;=qc&kmKUBN|}oq7w$VWG)6W=G3B#J!@wIhts)Q_J{{og2&jHi
zB+8ValLx0pEd-^2Yh<LiA8-C7{f1e;pBJLba;&l;BnekMez(;vE`pWzg*pVnm*S{v
z7x{^uCm>bXGl{Wg4_GS-Jpy(DJ(S`($aozgW89BIrD$7xH=)uaQ+62tofZ?>a<!mY
znDv|yf;^(lz;<pJx82uhMh1IX3PZIUsJZ*8ZEXyrf#tf-2r1Tj?HbD!f&Tq-5BbW%
zVHa@;v<VvBW7C*NSWM==zo5BwQzowMu4QD_!JZzOp~yMMs^{h(DAO!bkLQ*c$x|k(
zzq?`F4>u7x_nuTFlT3;f<`+;VxO`?^W4)s>ndTozBT?p2UoLMDhtyBdV(E@;tgux6
z&Q!<2Xn`EatNkJbZ0se2t0RNIkVY$)FGJ0f)60$539bzF@BKZhxRBHpwptcnP2cTi
z7KOn=yP8ohIW3FmMn_UrKbe6r^*vsTa#26X5Q)YK6|QfwR|3`gs|-K5lP7_{T$Y2J
z2=gXz`vjNq`uAen9jbn?(RhetH#uFT>KEa5LRQSC#3F)Ohh52sps{qJlAngNBy=8*
z)<)FF&>9T#C^O+cW&z9_?+WrA{h@i>(}^K*vYO)u>h?%gzo!<UYq->A=OnphNP)2n
zZN8%^iQT}4s-3oWv8>JD9PMEKgyn*E;ru3?af6)&uS^&1K@8#>0yKw2S(ZjAmP7G-
zW=Nx~U_|LLbW*RFw%|Olld9-`QXRFW0@C9L!TWT3Dw73M;Qo|DtSCsUN?|I}N$CeO
zEXl&2!Aftp?B^aW8?wWk)|l@{sb^Xpu?l*jx7a?c@NXT>Z<Y7@R`s<7nzm?XOOJAy
zv}Gy@uwEptJXyObZIGw^>paPmt}D1m*|)<S#}Jopl<h2|5SA|7L1S1FtcGAwn}RcU
z=O_Ot!i@5lM-NXN6H~8hz_g)}2NQ-Z;WVtPh}1A=w3p=TAxe#7b3G}x<-;XvJZl%2
zxC34oWx7ko4Z&03z7^4omv}Mkap7y4LrXzE>1W(^R3AannZ0*AV3@*-v94D;KQf<D
zHnTo25ViTryi^d0ESnXLV;&I6D32|}-kA!lBK+?p0XZ)vtWw~q;R=`X)+njgx)jtk
zf==_Wg|xq-=ZvELNyO(9+@_F+!0}yaGK`oRBkhu78-?3PUTohYZ=pN;puiO;G5p1Q
zm1xI8BR1U?*W4Wmxi2QU3yTMM3d_W9Ks=VX7@CiKHHTUJMt3CvT5Pzp(ELe}8N>4?
ziQZ6NMVg>o{^ASFx*i*3A}hG??!QyoL<}7Plr|xl9sO1Lz{w=XB%}DY99kOj=Ikfw
z#hPJTt%)6v_`M{ye2h5*a&&f^CZ0>GieZIDu=W>vMAxX*=1>aD(K7Y3O8sh}1eRZ6
zYQCK_e<E6gfpt1k)yoZ9bD91`Y!my7pK6l=1{UOQ`T2BWUafU3ZlAP)9&5|Fdp?5a
z#R8}|InTO{?YHI7<P}ij5qXmng}=`crIp?UYKKovK4}c@Q-b#J6NEmBMqbEwotH-5
zCbP0qgsH7jbh2=2lv}L4Q>{$o;X^RU6MFS_U+HpU<ZQtu5>cQVC1tB9YV-MxQC!1I
zTJR7&C<JWe$FfE>_S;L9+eNBP)fln3ebG;;@ytV_fj)aXMoN3gxW458olRsh;i~l$
zFcLlHRrt$Q;a)`<O!G{0sDQV}Hf?p!rWqR!tqx>9W2PUmsxGMpjr^L@EFEJGX3l>p
z0|}|*U|3j`WFhPZ7t<h;A_on#>L*KdN1{l;x7POB>^jBbY1{OIW-F2rtL2f3O|>+M
z^@3CjTX(1>z^DANO!T%0wl7ySXl<cScgAbCA@BLI3Gk?(BlYK^D(C6cJc^n{s%Nyi
zLGz3F$3P(GAhLY3NX5mgv#{aq4yje_rLDNN{hOs$anCYTxahPrrED$h9rHQiGzWA;
z5;|{w+6a~Di%Z5Eo2zsU?Em?ITL<FbL)0@jJ?Iz<f?Cd*PXmKJnJ$#O4Ee^gg^xh<
z4GGGE7y>BlYFwFs1Y2Q9W%O~390&@Z<NZboD)g&A1F<)yoTn7j$x{kQag@2l-JHaw
zhEqFwBw{X3I)}Qn?v9SQBmFS*eTqGq(!?Bn)T;WOT1$&|jV{DIPbF^?e~wYxZ`QB)
z-)wEfnY-5;+L(|=!#5I5h)s!LexQ_0ucD5`b1{idsWhRmgl(fn{pv)tz-xL&abeC=
zT9_X-o3ae9=LZm$RB_4N1luJ+I#)sfB(Gl%@Q8gtno|pjZb_<!#Tt15OUtDXDG3$N
z#GYT}jQ<h+jq|go&8}!4`$jcu>FTf<4TsFh*XZP-sZl%a>ym?=>Ol;YLM<_Qz(M5x
z0S<8JBhMxvHwt;$e}(!kU;}3v)Kwh|tMABnVYTaAHHH#wM)qetXDY%KHI0UlzE(MI
zY%~X_@w=6EG1a_(r>u*I^n~Ud6=pFr8(d2qph9DMtjOWqU|r}D61}1(o3hFxH5QZK
z|4BG&M|dCJ%*iDJE6w3iZ*Y=%xgCYIA6hpBXOBb<N6Inm@^02a$z86i*LKLFBeV;7
zl|uokJ6PiqRMhRu>p!z<(LC6ZDf=IMp9&K{cLLtj5@$u?HG%*s^wBpFZY88F(nNq>
z0rLvF!zqy7UHuDC*Tw#0)${JwGcUzE)hAq{hOsU~?FzX{sg2dK*;_aWssE*BD4v8V
zG&eoKP=`up1#u&%a-JO5!n`!8Q0L%`Ph+2$XHnKzq>h2Pw+wBS$78+w?4iK~$vD3e
z-x%r~=#kQnWiX#;P25R)0)geOafaY!NS!U7&Av>rVaX<_&|2lZOr=p>7pnvdbb9?#
zcG@v4u+}S)JU1O8$vf3BTZluWy<@Y#x$TxhO)8)18AT_z{Z6TxKyjhiu~ZUNY_d><
zdz!U_W3;a~QWTVq-ENoO@xb}j0*G|m-}2ZF?)78f8F|$Lk_`Ubs-^ih?g_L($|4{k
zonnhw6+PXct&(&YG;Os+|4H7}n>7#U`*vQxiO<dmur`9W48viD<iD`1ozO%cibpp3
zbwP}iGz)l6C$qjXtstr$awAa~RZ_cLpLYzw^oq1Pc*ax9*xD%H<RdZR{OBvRPf!n(
z(x)FwRc(e|j-yDA#dw5a*y#<Sfm^7|03`$ZyR<CICW}<W?W9>;o;|jg{UnAq3_!&y
z{6uU3g0X|1v~D4|p>-irHUWvr<@zv^^(%2NHF`7;u!e+i;jdw2Rx4xrkn+0x2#4BP
ztSn$r2U}jBE!11~eV=M4+rC2qFXWrG_1iocip$ES9U!Aj#e{@-5{@yLPe<IuUNQu?
z4pK>j1jBI2UMc3<r8_^e((+l9)n27Gj?>G&>SY}s>6#L;vTn`&Rg=!E&%fexVd2h`
zC-42dKxo4KS~&xWUwxxj9r@|kgBe2?51epjj=G6|^knfBzZ{!<dBw~znt#mR9wzyR
ziGs=NRBN*0Y2#Dob(3vPN7R~@I&Yn_U`zJ6it4tHA#=SyTh;hgGjhv6dpwo-+S2G%
z@8pE)F!vqsKGvh3_T?+=_icrJF%$FmZ;(58=Zf*|cZM;u7XYTZQ%`uxKA+ogwTz{K
znHhI$4ZM$;=N=w!h?A#3p@yhVv969Xum4zWqd)R?#YEw|I<q|Dc7UwoYP%N`S7mQ{
zb44!c(YA>&iL?ydw)m#$zlnya%9H;2!^Ww#>5qh{0}-@0Pe-2%Q8o8Vfz1eS6F1KD
zOuDX<gIp&F0mur81eG3xc&YLn4NS;@Ue1A-39#cQ_UelPLb*FoWqHne0QwW*r?DwO
zV128oy&6#7@3c_#;gBY`0^06^=Ft&G-~(u;Ub&9e^H2L)31H&6P*nqSYo{D!;9MKJ
zNnE2<le7oZMryCU*ZP~Vo6P69_uhprbHc)rOgb=8H;G8<zC<eN2QDg!-1Jw2nDg<l
zlH@2&``K3pi9{x^fH^>^ih--%u2~5`{dNP$tUg9T7tu$8nL5SfKNG?{&`;?xhFKbo
z8znUs#$^AcE46x7#tQ$W!`t@5LcJ#sMbWPV-SFLF5jR=K<yXKFXh~YR7MYu;4fR~R
z1G0Cr<zIKh5-Vmc5b?co$1m6L^6W8s6>~*mG6pP_mlX%F_}7Dpx0SbzpO#}Zr2Iv0
zzIuFBQZLxF3MSKy-z<hlqH1*oAx=oNs{Q2M%RqYUt{aTD6r~82f^fY5v^5i-<m~W<
zJNf>g!o@8!5%gaM`wuH7+?^-s9FTZ^+iwM7a53dhnTk9<GDs_M%oc+=h6;iIQ!N$s
z-Y;9RrsffGC81IlUjtV%Wa<u<y^^_f+n1{JqgJJ+H3=pKFD`_amxkI(PSMnXfgUSK
zO%n{Ta9%yD{c}&ZO}g)1>5cB74J>2{%iN^$=+A8$D5_qN+de697fAZw?Zr!ZCUT_}
z-N%LEWWVZ@?k_n|ODb>3&@pTrGg>Hp&0EcCg=beU1~rJH1Dhi+`#%x8)wiq@6K8DO
zq0cG|&QY7Kto=UjYiXugmFnR(x1d!Yyso8}8J^MDS2W{Bs-Z@pDS9>{(*nf@31uEY
z+T$J{u}_blU;ap;$M5vlmi{!zY|ciwY3dLEJQ8{lWG?UO%!BcnHEFt2(Org5vF9{|
zB^C6OFaCKHu3JamT<WfTam(FPn_Ozcv(RFygA>o(xpEo~^>15AY0cIXGe<rtysKP1
zHY2u3$&@?{zmGKs(0#0&HI2BB=VwpVtC%QxIA9R0RsJ2Su4NU+R?Z0*kIim*hz@Rd
zRjTt-n##MT#U2a9cJ5j;Lj8@k=uTr(0;ckjREWGf8ojOmnM*FvbVTP=YX&RonsjCg
zOIo;|LWaH_B68=-8BpoTxNQp~G<Ib(*>iuI>_<8`1@lGFq)5sWdGYu{=%#g-7oLeo
z2J+v&d!M)eFK{!ww^b>?oOr9Wc^~Le4yg?n%C)P#^)y7zAFib7#J`OL@3)>F=Dcre
zg{+O_uC=SRABs#dPaDoQ%XZP_v7*Pb-Zq{}p%t|F=adgAAD;bGt*OMkfYLTm%EaWu
zzIp)BLM_DCl`HtZUx2S_*wX&x*Vlp8P=-448Z|<Lb{Gfy9QboUL>9>PX-%xi!`{r1
z)mpA)Fm6)e9`HZ(>+mf!<^aez9sM*9|2{&OL~Rsj0PkhZ{Ed|g`C})5$q#~@!C-JP
z<L3k2U`@dWp9D$WF;8=D)K;oTWFAYM`T)vh87nRaH}_18>MX#geudIxxX<XZ>RG?g
zW{HKTjYz%nAV;3GuyE_P^l$f$=(~gesQkU=1p2#r_d?G2@cu6e8K=Yc>(P4?mpQqp
z`+Kt<w<_Oj3^01m11xp??XVFgCMwN^G9T3Bm9?KVskJt{S>6v^{q4~Wxj4`$J$T0^
z349(>T3i{Tu)??DmDVo+3ZkmSmwlgi-N2#-_(7NGahG91r_YTU2PMs2&%*oKnwfxp
z9iH3`=%e|V|NVBmNYL=3rX!;nZv*0;2+Jl4WLc6)c5`Y~{ZXr`0#7$bdi+1M!rJM<
zLOD0AKP*0TL3Xy(%5;@8{$VhWb+<=9Qeyf59oE9P(T3>3z@@wWM{m8i<b-hFD#u+}
z*<RucuRUInEL1GIIc&>T$K+!Pp%)w82}$l=yRyrRt2mbYCd{KPB*ma0KE!c{MNnRj
zeM{)mH{re<u4K8f=`qND;5+k=0Vv{v=RenACb@dn7zpeC>>YdJ*=4V&HPFLEj`dZ$
zi`A&dUaFx&g@uyyTub;AqkTltw}A)tc_I3Bcyk-;VPR~;NUiZ_60|JuRoQ0IzkXGW
zn0_1{+qw%!BJl|Kz;IrmxN6t{BI$aT6LJ(1F2tMLu+Y%bVm8U#;cGxRStK>g-d;@K
z{z{VRI%QMoC(kaAJ$Ji8>cqPbLm?jk$Hm`^AotzjMbIsL^IOk>+e~P;0D-%BLr#5N
zTyQf=D`oNMy$|`xk>0qUWS`nWAOymm90&oP7BfPx24`Yc*+i%{l{3R)h5g9N`D1f9
z9ORw%s~|0a9F~8c)y0!QkaOMBYYtNVgO@j@Eh@ffCuyJk7R;MSoLuxe8s#Pr+H({C
zPiw9j76yvKjQcI<eGc~@Evo+hK$95dOSDr(lY2tcq3Tq11G<>XcNo|ykpvl*9umpi
zl)p3tNhJ5$r|L5n<y=};elgqk?<KiM@=dX`BVrtWc7x3Evjd_#oSpLhtMQpMgIqO5
z<;>J7$A;SLkX(3nMDA~SZs0#4<?3}$veGjAe-4>VcV0*B3YVQW%BJW1_||=$Z81@-
zlpG_W@(&T)WK<bL>}4FQ%q<?zzRHnp{T_H05(AWu`)pNscy$D?tOT4mo|T|<34Z!x
z9_WJgPPV1+>gur^dDXW1oDl5LPQszxeLvhUI7{LBmWy5oTsDSpw)S}mXuU8<A#VUO
z#c}JgY9o%_Hm<6(Q6f8)%IB)hSI!J_EP^YUB4z2ynJsTJnF7uVyqJ#c*fT+;N0AX+
zN;nkcQ2NYK!hu9>4<%gno(<CkC{jg3e2U+%6)PvqYs4Mpdj8K%IOD@7B~$8n50gxY
z-5;pM=eLP)#P0Jt6AVRyP~N)c!FKD}5-4u1#;a$jqf7F(FVUz9-;Cu-sG7Soq7MSV
z<=Imss#V(_R{e3y>iK@G^geqxEV-w%dp8W&x2E&*(3yF~vP|^p_hN<J;%khb%|7uh
z7gO5z(6*UacF|WCQ~BA|8{6d7+HO?sBP{d|FbrFOYMr8nF%yVd=f&02LDu7h@+(2I
zKzuk1nVg3(_j_QVT5QXT8l(Nx2_7a-%}9saZYr0J<xH13qQP&HdM}4`tGdNBqi0{i
z@_Cd$b5z~E<}9Ulu6+}DO`l<!la9T-D}r3jvFh(WRHdZ4gr|*64cloiCY=`iBzM^&
z?jam_NKt>m!|GHBlo%RE_}~>7T-GUi(2(~>FiG#_m8X~Qe{NS>6rATZcHy;G3j#+x
z?o#UPhEm5)m^d^nH_1PD^Z4@G(8&&8#Z2MMUQ%k}S+xeCkXecW&*V5qW+}pvS&mqh
zc>rw-$12p}NK`cgD!IPDnv)(~joc<5YiDAw|4DalPUuWLB=9d$j~9h5i^*<r%_$cu
zuPgAR`^CF@o897(R|4f*7eP6IE&T1C_!p`GVkUrSU*1(5_2@HidEcix)y0KmPFO|+
zJARhgE;1e~x-$Jm9eDxOuBbkOESg#x8%y_FD;j;=qN5_<ac<sw`tjV9<E|c&$wIgC
z-GacOU2kRg0vyVLgwVQ}H!W|gE;(DZR5vlw4N$eiIz0c&*hKERC5JaTb3;}Hs4ZGg
zqe0*EgQ-7;rlJu4y96q}dpB)P{Hin7tiZlGGrSxsg0PfJVks&Quex@|y_^y5b<OZ&
z)gMbCuS8_)8J_oNT;6h5sVUx;{;^OuF1o(3LHcQ4<wBkw1!%!=8!syLt98X+byi86
zrj77iYkV`Jdc8`_RcApgySOF1CTDR=F*U^TkcC`Ym8vphi)c)SC68-~0560h05CLP
zQxYrQY@BNq!g4m@_-P-dm&X&r?g21VG%Keb0)b8(yOv|4m0V9v`MhuDqIf6zmqJ}9
zdzLn(?OKiM;i?)dI9ndcGw`cCV1x4Ah0lT1C@uuMdhbH2vbgV46^q$Q5e5#2DfT-)
z>27+d*x6Ywcr;_%%!hAxOWN9Rfi;B72rzcIcSWMoe(FSzm*)0j>*Fs2E>_6_!*V8;
zs5gLIKV)FWz6w}fS|ijCQEsP+%EQLuq=Hb@Io5LirBKM>u+W56>1*C-PS}p)=3U{g
zH`YZdsg&q!hc8dfsnQ-VO(JTjYQ?W}LU@$nLlBJmU%a^2IzeW!&G#cP>&BgYnTsH1
zWE+B*@w96dk#+&Pa6bRs)#FQ-&buK0K}}WpT3U765fbesN}dbKvvTYY{#Y+A`9hdS
zRcu3L`~y`*barC|D#Be}ugL%8P})?H5an>F+}n+m_#lGzqut6uCbkop6f;UoCTneu
z;&SjOYT!>|bFkCnoirD17`%>5+f!~$Zz$ANgD6ltOt>K5l~+js2s(;v?di#ibLJ;{
z#Zh8?E?x-wO3Sdeyf`HX6XZz_yqwZ=B_2Cbc7*f5y$_#@ndKiYvI<WVOZAD*p^+)i
z^PMxD&U>3j^9ppGpB;+Y&3hz!qbR_eL%dpXQrafR0c(ER^y+og$E&bdZi$x3@pSF|
zk{YvClcSOdUkSYrkU^LMM3H5c$C7FltBO-n&GXbq%Ga1p)g-42Kb~Eqv`j96UFo=8
zhkCZlZjr`Vp_OS3(qAYgO)Uvg4G}i1u&FKuhXf4X5UXobc-I1`G4xWfZWU^pT<6)X
zn)LVkxe*3-jc#UXBA#o_DsYrp*)jP9LYXfwI`fau?OA#!jzlbk5%l9QuOWM9{Z>iU
zbo;L~>^8=VIi!>r#MrQES&wQXZt|)J@Yp?hdP^dz2T=OY=fPp^N&#C|d>>r$(y*p+
zPP^=l;km6P0yqE0aB-n~egOGUQ#g54s!wzdU|uhknTIa%@go@KWq>7I{d8tSli%rl
z%KuuB=QV!ewcG{dYH7WZp{is7TKdC1E5=o^j#Q04W>sNUlTe{#^;7jiUceC1MOz(*
zr6mX$x@pKW+d>fhRlzy~c=wfSX8>A4L;b_v<;QB9nUhDtb-X#<#w{4Ncspmyo%`Rm
z^&&&C{(#{}jkGUk8KeluqZV>0dA2RJMvzpt4bmQDk*UI?AV7W+gBV~f^>I{;OBIOl
zOun#orGTv~tQX6@bv5*y1wcGljBBfk0(8hUB#s`48lnjP3JAVVK$lCyFI)TAY%dHv
zw+70ckwm#;;cJ)wrR{auSqU{Gx=4|(U9=VJO-<p@+;pyw&xo@X(!WNo03+iYzQ(gG
zNoMu(`_qUMmC-vJHopo5h74^DA}~zaw&vm6MF^#3_aN&IIs7Jd`ljp|tW3b{WE9q(
zS#I&Mdmb&Wu`q;#eIk!)yPiTGBCzZZLVZMpNONUsup2%&Nr#iEVYDhGk!Xw$R#(3L
z-Yl3&itsy;IyEZgQZ$8W482nH9Ph>LNW@7*LfM<%Udca#+$bQ>z>k-`ynd_re9hm3
z_fvoMSZiW0PilP?D;&(Qk#Y{;64QrVO1;<KZu9sEwA<m%2)T?K_ay0^*;>5YhBF#a
z0~xh_Fj9nj03wqhS`lAHLzftFd;44WjUhz44)tmMt`ugZqpP~w46dqyWn|zmF80YK
zb?Mi=)-ZJ3QSa$Vz3+J1Le3L)N6bqlbu+p{4<NL;hlAPh4gM7C+yneMmoKxSOPGMq
zdFIA(qLU2#BjrGm2!ru)4^@?{<(>mciZM$YbryT>le_NRbGpPeocwdDPe`m=T$7*j
zP?{O;QIibksTLDZJNlqk+L>yXaDo#Y(o>}*p3n-`jJCoEPCfD(L=_6hj_%+!z<W?M
z9!oIYepOB^3f+-w-@M)4?33=roW)V}=K;_G<?JS3UL^95&mD;K7b1*c4H=ttQ+(3B
zxWzHYZ)o}3tUxy}=FvrB(k#<+iF#1%K`U5V)Vtzm7oz9d<%YI%Zu;kj4)wiKmY-Dc
zP*sIE!0G_VdN@tw)i58RnA}h%9KV|`UZgwTp2bj)zv^uv0Td2sDk7=R$G?UxNCa}o
z%7i<LTpQ*+Co+f?#EZ$--UgD|{%H0zXv!Nsu^W%tWL@9jNuM#FMyR!o^sIW0o*{Hh
zXnJQOlRJ@%xPcKb{?Pqx(&=md!$|+Mq^6Ry)zp<~Z4rP1wi%08BM=bS7#5Q2+?0uy
zLYJ4=!7=JF8~uit2rAXDwvsUpPS?BoAoQ0!W?e9=CFc@Uf4OCWVvL`jLW_VqIDwP;
zfFlN^z9I|v@%0H5bGlSfZhaoThSFwKt*Y#}Lu!4^NYE-gNpxaQA6?xkCbpA|dJg=E
zx`2EuH?CrPiy1GFkcVlUhc(XN;&|_2Gy7oNZ_TXq09Hx4W2p@5OjR~<m<93UfyKC_
zCLA$5K%9VA?yW5On`Vew^V9IOpHo!}@|d4Y;-4sc<WK1Lr#O6?=eW=NFd!H{C}%vJ
zadAs%{6F|hTtKQe$ckyaMM+#5abC2+48?qY#h~+px$~>fGFW!>|Di+7cYex>vQ(}*
z*WV8jT@oG-QRL2;swUKt#qa0n_*Os5*R$`S47G>0?C}>34$wmJyGf4y+LB{5tlTRt
zpLUkod<yztEUglY4;|MOlhou0i2eQykA%L>dr13g5@F<mvYKsq)aM%`HXw(}w>FGe
zzQb_C9gVacmx#1qQdOz3tkO&;A2`7i>=We873K#}d~>#j+}J%oj5UUC>iiK&J6|Mi
zK=jTc<r_|~VqM}7+8;vUFs5ZHiJD~gs}#@~Er6!OkLt1Bx4i0XDm7nituhTDG^PpV
zwX%oV6Lz5I&GvRs<lX#2gz>rBh+&Cn(R@i&c<F>Z=KFMvCW7ZNZ|0O5{*8FlOT^dJ
zG}pNp#HcPpVKBe-wb6mxz@VP5=Gf%tmyc2+9j{tcx9qQBR#5gyt<Nk`xFHR_=(BtX
zwY?hLyLpsP*XtBEBO3gqlfE`|KO42hZMHcfSX79vVIDx8ke;b5#R~2hYZIb#{Dzg^
z$fJ{5n@2QBFJ@jShXozL(P&T7{`0fjK5E@H)oibc3$J}yVD9)yR6XA-qh~h62xq{7
zgjt16`x2w}KvZv_NxGz`Te9OhP-1y-;rb@nAsJgJ92WX67QqOfXKq#8B}Jp#m~HxA
zrt-k2sy}*uXZL{=Jh;WI%qtB}O5YIDBE-!DnS4ac9EmIQ_IWam(!HIkBZc0sL4@dc
zzkwR6*mA_{AH00;WSX=Sqv;FKsUljcoefeDjNZANozY&-0_ljL)VeO;wI&^KT!*5N
z@U+pX^Fwgtj8^zYY{%o-V=)_X>gj)P<K(?|cXNQea=NMl;0d#ORU4M0Xk4rWm!hw0
zQ45O_J}{lD>W^8ad`V?wItj1J-I0J^TF)~vec>h1`~u>!C$>8i<)glS^rI3o%tblr
z;7jsQhPmKeTveAN?2bsh9p+lHb4@;znx#fo&U^((b!)oh0oMU=u3`dk^m{RX?!wXM
zu<cLZ&yEQ$F7ej(cAs?rf!^`!wNg$w7_F)cuJlr&uIU~J7y$zeWAi_?Ddm(8D=)5b
zmAU7Xn(v$*e{kx@v4c>phxoNm-hB{)$reDh*0i(iIf!FD<AZ4I3h@T`lD>tzlfI9)
zhd7Hi4W(9@>NYQPm?)^n&hL3%aNKyoj1-@!xt@6~#ZSMvt!ngfD@*X0Z{pqNM7+kT
zpq}Rtdd+Kg8)xEhFiA!q5~}QzSbNYJzRig_NFx>flzWh~nB`Tk7hbQcj}9k`rzS({
zoE{wii`9sc{d_&UkC1R&PY2(LvCUs^hWkNM8Jn`rCcelR5NIx5r5lvASL2Tlq+GR)
z$`a<l+J8xoXB0M!JoE7QQB<b)Jcssuxutv3&5^{2%piXDO*-sL`;_5knkMGC)()`i
z_zL5})%!RPjt2^H{KA~wU(#=q+bXHyj=mOhsb-Xh5`$xqargbjgmlvCa0oHlbl}f!
zqfLKEPSlh>fejEPr#9&4F~5FQGGkjv4n9%%3`ecA6mD!e?e*a&@BJ77yfaBhS;05g
z@nL$lnx?XZkv&a@VJH%P{Lel=I|QN#cy`K%d}-B9YlEHMG_}2sx)Wi0fVL?32^dFE
z!1>YlfAbA*uB=i7hY}7cn7sb5AHp_#bPMIr%GSO{byH-TpDDHFR<!`)fM#8%gLQWW
zD9MvH&Ix<MG^T45={8rXZe`pALJRBpmB?*+XwpT1slx97$e<)pfgw<wyRY)7o7W_U
zKv7yY#mH`pyqc~#x|vQl8}s51&Ic$~*e(Li-tn6SP5A4&P8nS(n8wIH>Kd{LfcT+6
z?R?bdDGS2DU>FpiDS0t~*DxrWYa6AksS4RxI6AxM=<LJ{L=$6kdM*!KcJ46t&Ri5*
zrFq=S5S0fvg9rDCZEPwlF$wf7wkc_vq6Tz)mY2-JYYhv`r5&D>J<ns+WGHx8Du?oi
z8`dj}|Ey^r+mb-~$1%u4F1OmDVVhGvgua^SUx1L+2NyKqla3t6v^mEG&E_+ujOKw$
z_S~&`zc!dzT{Y}BdV#U5rZh#-2ygzg%APCZ2HyJfIwDQOjET0ggqe+6Yi+@ww<3&=
z2<AT7BiP)?;9X%A_LKuLuH5cEVM><IbSUOx_WLUQpSd#+ggX7>c(*O0En6yOx3)um
zqHKl2wxu>HMQh5LY{^ZMoU<iVRNH|rY$=jZiE)j|kq(PWh+!tggqU&-2J`zo-x-W+
zW@^>$|10}=p6};;e_kIOkR-)Wg1<-CSeG1PDQP3bTm0d&`onjk<1=3!QcH2}^2I-G
zeVU#qpX|<hQDpMIo`_~BP}Yz<736Zzh2k@@Hk8JZqa^4nzz=tQzsm5tzR-P>XoW<a
zs*~$c6qVgkYgPxG>RmqXbEl-3&O`{S<Ll4E7l^zAGdhz(kr6Q4OHPBUw;WP=JFq>}
zJITQrdxg@FB!CbWir(<I;sHW{S-$kO3gcTcuwa@BUtX|%g5@1PrOfH-89Pp|T8%%z
z{Jq?u%e@?@<}upSmOfs<L;;&Hi5qglgd-&AO%!_Oim>TW&?Cci=&IQlPK-5QaAo-E
zP*e-s{kP^dh&9QCMaOd&kf1F~DpoAjxx;gc{uy~~jNM|1u=Mw=ICNnxPR#C_YzcSr
zcVF~im=syygZ=zOb32mT7Flae!CYP1sLK5}_4zfMC4_bC;UT^|)BYXVOqf;+cz_y9
z+;!XJp3h7$V8xL7{edM+E^mNT6)+e2VV~y#K69aHAXxO0^l4OgxQ}ckc)G$ewc}uW
zxY-AonwZT#;F8;VMHtu~0@+z{dZHP1hFbwhlHgnOi9SX>^kvN?m<3PBhXtTIAV)5_
zOlLhDucCe?cD%O@MQCn#kPxDun)n%cG(5^FVK283@}}piFm0%6UG)<9<cXj5Cb1Ur
zfN`#&J~m7k2?&l?nkK+55d^`3qOXd*Cu@;&d&5k9KO0}zD)J)nEy20&jh5-+sP(+h
z2k45^o)P<*S8`@Esd4V{b`8ZSQqeLxK`I+3VDdVcf6zN=$X3er^(Oy^ASFmV4!jto
zw-$^SgUlp!id$>(M3uD%m-<=<OS+N$x9Oc2q}Lf#R}s=FZY3DC(bTG8xMW5yk<oSX
zj7mo!WA4@Hm~n4xj*b9GWq_1rdXJ<WTmc#c00M&T%Ef@y<3&Kg$QO2(fPk>y)%%^n
zbn+UdHW}O=^P9T{T9$#ixrwCa-sIyM)<iUDlJDg}cLz;)_Y#qZhC-&Fi?0~@>-*)!
z@)cF=_zOP1T~@KGhlN97ZngJx7GpVD$Nn0Kbj6F$bWdau>%GcbYtj=LsW6+(`sfb@
znNMd|Ttty0*T@=2PuAB~?wN78Tn!A_PckgBIu4+<s!c;*-v8p^%fqMJ=}knhlV@Ho
zFB*1#yJUkke0WKA4MbaT3V6o_V;DbuJ~5_ee7P_7`UZV&m-eI~P!o*ZZ)x)WM6VDK
zv0crr?j{b89q%mxn~e;X?2MHue=rrfbc79HgK^lk2hQ+0OxVNlc~;tvl_D92Z8Ub@
zb&&||5Vbx=ugeV_j$a?}fmhX?6Nh0M4J0h%nt<ww!yv*(QyiN2F#HIB69628w5|Y-
zf$B;kA8-u9vt&Jcm7-;vHzg>0&lS2-+yDH+<I4ES-gm>dm$A5XukE4KgUn;)r48g{
zR+~D=6h5DbGe|!MO(ZzZzLPzp94m6z12{mAfwI^m`0x4T2J$<?Xr}Qh>yX`~C?A%Z
z@T$a$-&j3*m}^B&r?jWrVdqNjdUMcwZGzKC;I%jnSKtJ}Ys?WB=FCl5TxJi2u_G~G
z-T)FT#Dqro)2KjiInJdm66%r}9T%_92OcKr=NVx&<)~{Cy+qYSkw9*6drxNhgIS@v
zE`XB7$bEUJRrb%mywewI+7sPvn=RwRkLT{amvi2edU(NKh5LA?b+yMl?dAO^&$lab
z4Q;waHWQ)QQ0ilpx_*=JO8#Ty+#R%RVI1tT^k=)f@}DmSM@qS~h&U+vi6gtYOG-o@
zbWj2vufjIDt&T}@VEUDp=AelasdV|$nP1MJJo$;OiWJzJrzu_vCb135zAKA_@NF8<
z%;$|pcr{UYtjUrY5H%Ohv<_KzHQ5<SQ^r@?C02YEU5mFZ0aa=kdmun?Bk2u71y?cB
zUKFFOB;OI+AhE;pJns^UV|x0}7oVxWATZueLys|<IzzRsO|H!K!#kf-8CA#mcMp0>
zC__^0IKfxH*N7Cw)tE?E@Z4a^0-fbHWv*>;WS#Z2`^D9p@Lt>F;&<{fCjS#T7GWUa
z=1Ny>Ph(hzEWhg70hB(E2>S#{d;a|_F0R&au?Z@?z^IhMBP=!n@ms#zr{xC6LVwqI
z4E-!>f{OaVrB@(w?z}03ovcqD4Y;2kc{EI!aOBYv-{=&7-`Z-GYQ?-zUYZXBG%qq0
z4QvFcI1j<G8yJ;fwy~dR7-+vRRVuJkQ~Wh7)2;nopsm)oLD~DRw4~`v;Ic7)&68sX
z(WE!j?q0#?wft(O`<uwQEMEwhJqY#*S3eDe#^b^Uv{*4cJg0KP$E{&ptzUr6LM*3t
zl%hskN$`B-z(*^uoat%<^DmV&RlrkjI7`q=wPao`m!!9Po?*a;c5K8xwEhyzPkE0B
z7dd5~&gF*5Bx~1`$b`5hF=O`ZmNny>q#~~k)V{h9Yt{BgAq4lAb`7<F=4cl3%cwf~
zUpd+jI!QoP$-~@rEZmo|%YwTBv4ik~11WYWMD|Pe`nZoqunGC;YH|_)5bzBa0M`3F
zsv@`D=fmdXpP6F0`?x0;9+0J++sFhIiNxlFyUr1bs@l(+Kv;9k3k4$AZPpUyOI6Jk
zot%5P951M}?=QVf<F85iI-J%&rJ)DnmOxP1h%6)}iHXmAt!E=pZ2_3?rN$-L;hX@^
zI$iXf^c)Vt?sGF5RGc0OhG|9H#d8dIQiV)vgE0k}ksN3Xki=l*s;{QNBkT5FM)*J}
zWtHcgN0Ux>%1c*6a`WzojZZI35I~H=sWA^(_JYb8Z;YJ)<w}&agQ(m2@B`%5BCQL%
zSdzf25vZOp;RnJkqc)&M&dLp<$C5hpSa0Eji!nZRPI~g^-raKKr3Hau3i!coczoeh
zIsO59k&%$K?c9A>7P>h|y}gBdl-HRv$_p4%cG!PvjLli%Jr)Sm(cyBt&d?n$YUW7^
zAdnhd#1r`hh$;V66qDGdEvtIl3ITsktK^HsJE%~yT0S)(iEon_{Pu*8@%uWx>SMhf
zXwajolykVMnK$E$pGz<}>!Y+~d#gemja*v7ZSx=?v4DVNn#kRo0_E8z2~jg`-?N@q
zPcv>NkyWh~8I>NTl&+03qLBW5o&$iQ{RmS`<yVH4!VuFOEIKZ!W-6=gZF>7g%3$K%
zn>PG}+gLH-L6kYZ%agQ^nNqGnjiRz-StXRNe3I*mZDBM46hz!P0Gri1L;g=)fxkKd
z5wyp$E`+;^u`=1Ovpv`|FMX8$Gk!^%Q049=volMA&%~6e+9teo0%baD<wlq`_>_pU
zKNM3?Ya-ye)@NOaA~U>6&aS858R(MTI^GB`OAp}7=}yDKWk6WR)i-&|pt`b$tk1Jo
zYd*#PBjPVqLFN-@BXqc+l<uC{Pu`U7iYFKEw}jG}AgsAxuLNaf>!w1VZWWPxH}~^-
zlXvB(gdH%sQSaPwB<G58TP#6KGWu;jv9m4D;f`hwC)@6&EA}vQhh#njT@h6t&r&8s
z$?EzPM36^4J>CTQ-nld0jC>J4Y1@$3!Sj>@%U0HP`z*@}xGYGr?R6xcMn4}!te@s1
z<49*cvu(da4Af%H70P}k`CkFGrYBr%Tvd$1lCqVyEv#uU0r|Tim$vqz?02>eRH~|d
zg0>T_>M`kv=Uxj@<OetdW_)xCY#Gy+BI}y$)ReVjQZii04p2oP?{jxL2?WgH`}rx(
z-U!7Jgx=$Q>}c;ptU?qGB9glrYw%gnH1R~a4QB`v(&HhO_r3LTv~L08kc}>@dOJ8H
zhWb?eU)Z77%qJhlvX53Bl}tTt5-&b<OWz-`!L!)dVAP-T1o8;wFqvA?nIqR87@sE@
zompR|)|DIic?aZf0E`?t_|`?eN?SJy&LiK<3ORAW9|+;QG%m`HAC#Q-8<j`6?%hlD
z7XUcFNwHKnJl4J1?xr=}vim*h9_oTC8HFWaRTD1k6t9&OUy^4kax!8IWT*kiS{7S3
zts=c$WD6Cc=l+W1CT#W*qg!+I&BU>YwcCe!Lw?+Pot1QwxX<eR_LKEGJ~Gx=(sMdc
zdg-UymHw@`k)x9D*<%snDkjCbLDa|*0XoN^8|>HPhaNa8FEDG8ruw*&4>DhsYmgxT
zqdtV5Gd4IO2(mS9hbY3^59sla7_)orubNQ@nG_k41@hsC$Zk%ou@dI4MgnE2t@)4a
z6BJ@>LzE57$;tQ*T+D>O#j&XgZllSCjqJCQ;tK=p9{sQ^?j@_aU8>zXBpD?<$&k%~
z#4es@B1nC~TFp?>o9t2^V%yyzOtNB@mTMGKJZ+h)7!4ua?+Rds{tYUg^MxgUdGM$F
zys&r}>Le3rP(uM!MK?VQ))n+!;4tEC^x4(JsyDmuLTWq1>K%M!_Bz4QmV#Obi^RCG
z>;Dq7b<&tU76b^K^RXxlO~*@R?AAXo(9wd%7fw}Qrr4Y=MOAf5(2j>X^D)zXlY9sI
zy&ml#7rx<ZkTH)QzNklUC6?QkqFAMBWu2$e2J+?moH=prp*I1rb8&c?5=UKclkU1V
z*e!VPpw=p@W=`*})UtoBS&pR!{{@V&m`+{Slb|zhl`Iq2bMFiS5TGJ(S!v-hEGPHl
zUgG1|Z;1~|4&p<Wo_=UZ3$ad(ZPJO?nj2k@8sK9kf6`Gzg_L2atr?)L_-eFWdC9Ql
zodhgqDXV^(MopM%W0@{UU!SZsL+Fi7Ye#pWRmd6xGGuk3F>gRpz{{F^Rqp2td4JOc
zWG}O=Rm-|?tF%PBYy3f@&n^=UPkck&VQV#OaXk4^T~Ht)54$ydM-DC&?uEx!ha_nT
z07{uWEc9mNb{$N}*m0qpj&(n(R#KVQ^kKJ-HJ|*zj%_JVrQ0UExU=%CLR1W{Fpk8y
zbjmeAut|x8bvBPt$$1!W{#EgxdrX%6%|e27N=fjK$gbFybveQr%>_K;bzAzQI;Pgj
zy1cGU<Z!W~LP%b3W!Nhe3>B{|LJAVVcq82pBGf`LWh7XHeAa>q>o|AUqDqPtVHX20
z2ZN})1nH(S%|qXiV?TzAT2m=qk4b@J&0G!*GA92PYxkP`ckfy^XR!95AS|A|riNuK
z|8{x?Kc>9N_nK8DTC;s9!P{w@30hLoQ78(_!Tp!c_UM|te@!8rEuVd7&Idkxl=*=m
z!LRy-Bnw3zCGw5B8Fg}Q8Lc%LarV74g03+8#5g_3sBolcL2ogg-B<Gkic`b(ba|3V
zmd#4!^N^n-hp;F*v-B5!njx&s4-fO9zF`cVb+bO-$Fj1al+~u{liN(e8vXrTIY;6O
zk;>P#ndv+310|EmzQ6YF)oPAVqZNu7SN01x3~ktMe`~(FJwd+vV|C69RZHf$ijsID
zQ;J2DT%GSa*s35t;lE&2rU>bLWXm*Afu*e4Rkhi7S?%;^$fNV?{6n-NnmqnhdmMSu
zl%jZ4fT)hVr$AiA-@5RGfsR~-MRXyN^~i@+*z*BCCrxz}-a9Y}V(&ia+slLsED)?r
zDEq~=T&)Cny({bA$gpff?g154Us5HXE!p5IKJ+TKILvE4aLG?dVR^tMp<lOFT+@Xd
ziKjJ@I_p>j&KJ0ewRu^CZqdN|8@v&1_Y$&4l6>{_wD49|47$NESQ;ZmdiX-2-kXns
z;e+b?l&EC@>TSRZJA|!>u|e<CEFL9!Fk~%+7l$eeca>~NAUg_Oh;ue>p5|3r(e8sC
z5Ao(zBvtcs1{RM#@%}~z4vN@YFss~D>qU)4L1&$Hp^z}($s+|(H^g@xw9k}{E@IHF
zLe?7SLymXQeuZ1u6)w|xdjm#z<4CZ-HmX(~jZCs;`H<bxd`RW&5+?<wu}FTP7P{BF
z_HCHKe<V@|<8qR)qSDC*4mi8j*|>cR`T(1wB0nk^o*D9e!e@7$EuIk!MTn*tk+N;5
zt)@NwlEJJ@uC1nsMneHDJ?jz97*7p!uhc3<v1k3tE4f;)Qyf2ZNw~Vk2$Zwehx=w&
z=d``*F0u{z1&-@w+|@Q-^skzlA)hMx{GMX^E>s-jj~U^$dbhW6(gW-pV!1Hv$Ag(C
zL;D1W46Ghu3((Bsp0n}Q;}hrnvU!?1`EpATIebcycUdB<1ZOvHi{F3Z4l$xobY}TV
z(5rwLhizldC##469Yzp&04kPQS(Z@AZWX@x+*YN`s#Id2RG!kYu*m*b_qjw@B+MRe
zpYQ-i-;fY<Hs7Q`v6N}-QiV@-r2FTUp~6ecO05p#lP?H{e3#vUm<pMjJN4~tuY`rj
zv?+m26vjPUra27GhuDd13FI<5*9YJhKLLCy(5l)2U{_G1_O7Z$3dpn{PiAyo1U;=O
zN+()DPup!OLaN^*h&n&WD{rbgtQW4)VUnDfk`)IiOlZ$8d;I+1Os;0wK-%Ib5QzK6
zVg_%99{wh4{=T&fST<-GFZ+bwWLTGZ)D);CA*aCv`RU&=pl{~GvD>OB0drfF)4?}=
z=UL|!9mLKT#5|LF)N{X!(7$&+<Ru22Pc)+|rh(W<w?Qo<51@{)mS<6I?(-Q;#?IH8
zp9PKDx|S|2tB`dDxv(ts#nn(a-&dz)_I`;&obMYT9T0IO7JIJ~Jkvt5m9m;iQ(+pz
z6yPy}X!TLC$!s%<+iI$H%8VblPq$A*+mP^2pc>PQxdQ+n`t@HHZ3GLN;Unbiv+W@h
zVhe~dFygqi2`9;N#13N{Br8T7i@YotH4`WWCSQv^xc#E+f*AaVBI^)!gVV08t}|4m
z-|6MS0SbD6!TB;^N$`Hd@Lj<%p76B8fs&^HD~VF~l`-e}JygjNu!1qZSi9-z4ApqJ
zLM0ohWzSIZTX>Rd_3Kx`5Dy|1_OifXoOJCs4bo^7PY834FnR777`^^Y@IF3uHB1mv
zAAV|&m9&dHK8|y~nl=lGE#h32Z1~l?dgCvyWt9Abntwu=>?37MXjV)q)p28R@U^Im
zi=vxFfUoRZPK>RMxFN%C-@*_cXHyrF7(n3mi&aerC5PI1E+oZ$mVKf%@FoFe9P0mH
zg1mx;L*j@lFz|@VR8YSZodmye)#yYTKwfo?`z^K_IEv2h<5qm5Ut&aeOKUaHta?Be
zOKoXt#hQUX>pfNU3)?8kNroYx;Re@Z3$RBVTt5T~wa?&Wew$EN+WkR>t;~{%hoV~<
z?x-D`L(H!i50_bCxeL)P@{ebmvpeyCuI@p-N=RNCW?k_=!*P}OPd0d;Z*;=!&N8By
zn3@AS#6i4QJUEXtyCoX_7BuNk2{=*209Bqdx=n@@evZ#ejm&==`&!<EeT6yw^fq!1
z$SSJqrLQ|gy{?YxR`o8=6)5M2JG^e`+%8O}eH)mMn1&>W|3&=-w9WaVu&NRZp@!D{
zj3m0AL)%0GJx&DB%N&Ck^&3+CyFwphXL~fUArlX(NLH+U!ouc$^|oQ^HHrxVS6GZ{
z*VSs4%=HxqUPlGuSRJ7I#2;J{pgXIFn#~USzqseW4mh%V!4r^74?S8gswglOr^0fe
zX0g;a85zbXRE_=)MdQcq6JB?moXCL^54GYEO*o*WN<TY4yj3m1eV)D2`_}9;9&?29
zHPZj>C)$f`L{Ib`ITvksJdAG95*eB&iUyjz*V&`P)z{nYy1e*)K&;jw)%)mp04T0h
z`}@TM?OtsCzm2&CZz1XKs%7L>#X#d=mN_eJ=eVUS8vGAKxewE#XAh^q)rs%SE~^WA
z(Rw1Y>MmI>q<fcWxc?s+Ai7%?7Z1GZuW3WUJ)uoem-~2|aB)Xr0Um=?>hZChf~I73
zt(u6W4l^{zH^0(u8SFA1cB!6nIhs$0((t2#l5#oQkWB`)%=8%0xKf}=FQgb;i1pR}
z3shh9{bf-o$yh4PexEPN3ZgUC@m{peN%eO|$#$nzw}=5<Pv^vDXK;j+s>6C@0R}Kf
z3@G}CzNR>5=fJNr12N3-KBu_2ZXmz|wM6q%acEvKhBRxHjtb93Uz{ZI(e}7Q?Z2DZ
zGIvyHBe+^j^&fid%4t~PN}$?&1;(@${0`vpqk#}q!{B04f`1Stszk!7?f2H~Z?CSq
z=)haeyf7Xs;9pHdl6kU6$>Lju%0V%&Rs9n#z<lXJ=M^yAF2DrNi1|JwPEWgWFB)5g
zfol1}p*@^>`0g_SMdAozUQ%nZad(TFbFbwzLtV{iK@KP%R;U4(F1WcerfRFTT753S
z@ybDf@HHRjAZ}wH?G}A_8z~mfAN|$t!5~%iZ{YP*omU8YJlI;I%Q}O;I!B<2;{O0O
zVJ82fzQM^-xdsA7!veSKeb^E~cD5kBv*#wR67Y1GoASA)0lXl{bSGBW{E*W2p$H}O
z;1sC&tVH<m#Eee`)ew2_p+ghvaUm;D--;_%qquc=8Q*nZZZF-Cn|($#9o&KdLpdXc
z=H~^3MZp=Y$)=ZgbTN$DFQM5SJ#02pG}zpUr)k*<)7^<se;OUv>Kfa0t^WLp&GvAE
zT8g&{7k=pvkPXH=eZ$}4y5w#?H<)g71Ym3aAin!Sx#%_;&0MYI5>^IYZ}?@TCLFbe
zSH8x_h#4N#)~A^_6JWrP>a^mEpRNb{C(!9mWZ0x(?-QUimIa_JRP$H?hwE!(KMOb)
z-7G4?mfc-gmA4I&`J2<GPe|VpA3J5&^~INU&uP`OBGBzw6H+_k^2$I}5{e78uohsn
zcJNm?8b!uHg_xc*kSI46>1|*=hJbw0eI^Groh5D~<Qh1R@83;-N%UdG$fYoz$(5pX
z{<d|Ylaxc7=BhfGg(7T);h(Y#3~T1VfOqIr;L2tE6)4UTW!r9i@^U@wQSSsKARLp9
zZ-0S*ViLC!li_JbS$~=J7?2-S!xZXr44PLps?z%2_MuiLB4{4+?gxeIAK_A+5f!5@
z9+nai6g|S~`tA`X1335Cx7>@<Gdz9M{~ewJOXB~ok3F#&lK3^<^CfZFa|9S{{CuK4
z%J(piBV739#X|t!eXxoX*2{qa^rF59Q1d<@wv{g0l~@c`RE;!CEx)CIZZwQA!RO_q
z)-=2&^I(N`Nsv%DK47e+QYC+$$){M*9@#TMpOA9>6L0T;Z>3ae65|=orsqSHOB{T<
zBoWvmK537U_OB8!ZQGkZNc<{8!C8lFHCV#D9>{vn+6VPCDsk4%Y^)7qYGLh(`6Baw
zx=rYR5{B91(W1Sa+1@!jq`0n-ABqzte}C$W6QE0RnEgTluI=P7Zp)qit1gE`=fU8>
zPsDp_WYwUA9f#`DLXFeT#u*3@yl~Ja+CIpC^`G?^j9q&XXMlQ!fIWPz*@9FCY$E!f
zy7y18{{cvf9Os~sCaw+|Vs@eEGx5;@CVYH*dOtoK>Sp$TnZw*2lvyrm)#LY|r&$~n
zD1|?^&QeKm)Q&)FABTII5b#W6{{uV>i=@5pJ0Fbe`x-gGB(R&TdC=n9#VLZujRrB-
z3;&^zfUc{53P7d#IUyyivpM1TDwYS+twQ@X6kk3TJl8vowR?3eXR|HXkgN4L9G5Rl
z_Js&^Lg_r$s>6uv3)YiN47mr@397@CE%bLB|1i{My<Y#2^>Y2%pA1{7g;}|+6Cd5Y
zi{Ke87FM#Du8U80``Luw9<1Xup_dvfx7VV{pZ=MIScWN-d9g0B{KfSj<J-_R(n|r)
z4A&Uk+^PsmmDtJq3!T}!FUWppSkn&+!Y1cICwu!hTi)k%G0BU>87WRo2ELB=9+Yc~
zaKLd8AP%KWo{;&<NU#m1Tu-$Rop(u5pwI9$VZ|Fd;^LB0fixHGNWQ})odC&1(Qw29
zZ234_if$;rHs5h;yw>6dXF66p+gNl!fB=TW2Kll`T=4D^HkMhwF`Z8l6gR3AL_un>
za?Ip!O%oG&ZtCG14)qY`<$CYVe$<vCS9j*IieLuf#_Ea2RQ-W;B)m&YJ+tnghzv*a
zeXx%LvT#vgR#NyES<0hol?|YbEzxL8QMC7qW6Ko0ywZ4VKtx1U$Q*t$q=#4;vHVGP
zOuvd?8p2*(t+b2~z_WASBwH*9c7J2iAKb(mbMCFV@Ftb6O}V(^lc|SLNV9(swG4Je
z)w+}D2#P!IMPi|K(+NgbW=_W&TO(+B36#P^C}-$)zlmi=qlW&N)0CX5WAOXLzmGl~
zGvD;*yAr0+M|B-_&%ByB`rBReCkp@WIqJx`<$>FAgoS_p{Hwq0-I6ztB#&K;3yYn%
z@ad16mVdK+;fdUHnbuxE8$CRHc3;v)h4TvzrtOXu-th2bw#y@pyX8+Keobn6>a=%5
zQB!AdM^`YN>5yBu2LFiUytO;GJyp%D<w8Ypp+^m_i`v~qmQ!o<5WQOk;v|=GrWS^r
zSmT<we1h&&)XBPGC9zV+9c9*TDiTw4oaRK1ld#)-VjjrIYQ}7Mc^!rOVf7I1Hy*gL
zpR)r`a>;ih(f$}tb8Rs|diOy9Q9>$Sy!7-9^|<g-lN8w%@Kq!yiK3r}wan&H-j6Sk
z?#*KtOUI?f$AhI?b!Y38pt`pKQtelbpB;JdB>uvz3HkG8-aaU?H~Z1gNg`;n^!a9s
zNm(g37PedgJMgG5d$xLB^U7w@9_)madtD_ClDknD)%;D?t0d0mqhF+V+eQ6|4z#IQ
z-YTpE#TM-@^52IwDC<s?e-khjg-H41{cfnEb$*&;K<rml{rZBlXIg2;5(xttj6dvo
zAH>-?zz2PRkKJw{xyjc;D>37+z6E?2hfu7ct`3XGUaE?mk;a><bLt6uthLgzU`Gk1
z7T(fUoAwX6wr_vc+o7)kYYWec{HE-8ziQ9-hTdq+^t%Ns#+w>xpYO<lT0%u_)nrdI
z7k$-NA$NkcOY=({peN}!-wI*<=B8cLD?*|v2~EAl<Ix`;E=Nh49pu4TjK#o!8$Mn#
zdP1o-wn!R%EVr;72uwiWrbtTn2z7ik01>(XeJqaKR{C9zm<dN|wOx4<m}A@PrJW>f
zH=hFY5RHmmx8Up8QUXzamF=Bh*%=fFrli|utI^A-ZJQ#et*0p`Ruu=~qT!B9v@Ikw
zeSg38(&U2djIlM#+4*RMv_#Xn%dc!98x=2dUh_Xw@zwcch6|mQ>Y`z<Wl1phxSR1q
zJfh&8*{vC6$^Ip>7sgB|(m<Ky)eAi<iZ8_ojonilZR$U2tc^6b-qr9S{87;rKX3;c
zr_pKgo&^HyZh}1Y95#fs7$B&x5Raa@YW5?5px$bQt~cQ(TFr$kg1wVF;P4CUVFyd&
zDqT(<D*vkWfm>NEte45PagJX>+o|`HAUAjF7c&=a)?w$?QmtJo;R)gQ$c(M536EbN
z08PBg(PeSboF>7xN$6=Ig`@ULkt-3Zhys^7SLAp^^6B5kh$P%xkJd{bgdBT2I%Iw;
zAYz@Bv|&Fu#&;F<pM?H0XgcRFE5Jr8zzx+?t;7KveZ54dL7oyY3*rkyLm~9@5}KDE
zKz#9xy7Mgi<)r^g&TouaWmMY!z5te#1eG<u&-IDPdU?H=@7?Lk8A~eZ%%Qe-nneZc
z7?52b7j=7uxVtjbNC~q{vnP+t=f}n~D*v<3Z@#rwzVXf=DdnPK**r>nQ-GP1$6E_|
z*~_vwp;i8?_mVjqiVrORlX8sn0V3Ul4t#)$7x&av`BuOJkd_?kJr`|zYkF&uYMYzr
zeFViD3B$=|+0OY#hV$5w`0%=hE^4f@+QH~p{b1{^S)yU;_itX1zc~HrwwoE>>u;U#
z@{0TiU7<{{nI>R7HAuTaCTB%-+x}W}a&c!z!TFPxt^|S>UTtGB3ZO>fbwULPrab=H
z>pZGR&S%WeOQ~}YN#Y!?EYWEN94kOOeYf6^>-<GMt(0c|YMcI0f6f%JDnOu&knY+C
zgAMmu8bPwS!mKGTCU}PdIhok?rp*G(5Y#B=OX$?iVrMVHH$kF!4yBNb_F@G9@M_g{
zu)HIwZs&rBf9Pu<W1=}j{@nEMkbAI}%6f7kaK1Y7mgs}q7J>N`bk1imBew!YQQbKk
z?CpP)y{%FuT4@~tUtCFnBl>yx2sbh1*MxPR`i_%;`o*d^5{!~qW;j@@=ID|gyLuxo
z)O@>@^a7DE_2oNi<evnbKZ3Q##RpxS$#cz9E`X%Z7F*MFBGT-;qG}skwYDb(^7^8X
zpKXNh!3uT_ZMw#2WZU6fTc7o8%>KK`bM!LDqI(*Dj3!(_4&l<$-9O4=3;s$}0fZ~J
z@g#DZXu9f}HqgH}A>Q(w3ovqJfk2yGS58$_fT47KHYba$9SV*xvJU+`*wpi_fL3dt
zEyF2qNt$J+!*YS#0@y|wL}`2<6yvZasxJP*RE|ggJ^Lwm^!c`0s5)|99$eo|DgYIv
z<Bd`m?+g1$HKl#CZr+mUC}JgSBVP8nICv$%BEI#DaO4p3ytE&p|2w3IzmHV_;3At^
zcK@td=6Gi+>}6ko0X2msHu;`JflEB>6nK_>_Byh=$l2Z6fubGYp1l3b;FB%q(Z%|z
z-3_tE-INf{ho@TFp#_$u8;`3rHzf4`{17!ld(aD2|5o8<)3qGxT^Buj>-tur5b5k#
zYprQtz;Yt*vTf-n`*v{LvuDdSypP^07F5?9AVzl^E%YkMYO}P7yzIGa8a`{|uZrA2
z<)0g{k#Yr>*<5|*=-!iK_G#3$Q1C?d=~`=~D!|*Me*yCX4pQ<0V0%CHdG5^y40smF
z#1R%Yh7&s91yS?9jf*bP*1*D+r4JYDFP-pmn5+A^JkBJZeX?vZqsndM{D=Rcll`9k
z_w2mX=34w+WtD@``T8Esxgk}5>dmm0Uf~y}zHG+~C>)@W4CZj8k2`PLlKxlVs|y~n
zFiiB?rM<%LQpv}xTJuM7ai{8q)Sldnj~|(63>DtsA>QiEGCt$GOPA0;WJ5g2Qi+6(
z(#j-=HN}i4M>xI+(9YP8d4y1_M2TMmP2OTz^_Da^*bNT{&cXIyDrvk%5tF69!*|p=
zKe|b&4<uFL?CwNakC<OTZ*fU)IKCLBR|JhVA)<?fv?y~~yhzuFsAPsuB<#6WCRT0q
zW)(wNMk3DAse_2cyEkH4_UF<J<oKG;T|T-@E@D$xHOMyZxvv=$uxL_1gnJA_)u|=3
zLYtl#DAlPe8XpH`l@W?rz~Zj!b)7<0>9;OjhHXLStbL&gisz;m1RsFv?JE?B)+!_8
z3vTg-JBa(cvrZzAFGSe#nE^NTos7G}3W~usP~mtWYY^yoEsg5emaEK>s&%ibJpnhE
zMHIqc2(`9`lGX@hL=YCB>~~vlvbZ);@I>}{nuG!P`5#5^?R}rEYX(l;1Dv`s>K-oO
zoiG3rV-NFin~bselKlLS*C<R~t^X};K`vbwHzH^2l$1t3dBEA&iLgNV0+tA)itFnG
ztuM?x7)+1CTG*tWYYVw7!ER@iF#kNg_2WBS>SFD%@S0Nv(|k84=5^HCcBqw6JU);N
zCKKY0{KMBd<x-==4J;?^|2o-$onYxYT3?0=ce}GoqGmXydN>|>XFwwuc%+}v3nn)C
zRa)F6^URB2XNkDJoUP>I{QMRYvNF$VBOcu>SsicedK#j^e`-bN>3EzLN6{m4=Ui5l
zU8@l&X~$|cpyeNHb%}ZH)C?z&s@`jqt4`3X;eCx>N8}$BWY2#5^n$0D3hO;~xcd4c
zj~POx+W)j9u|90X9%)vQ#)p#Hx6c!&=&PnQjhEahQw;8boiK3R1H0(DCG8?|53!(d
zPsp8bJTr9hrN)mgtfH7;J%igSPV{Ehvu_DI{$WH7SZ1}2CpklyN;`V9{cTvbFIXjl
z;~S2Z$IUBsv8OpjZ50g?zE_nIrIvm+P8+;BCuMjR;0q}O>_Pnl{GPZUc=p;=z*#sM
z8MtcFIcbG1)}fYH725T!t)8~Yv+Or}edaeeFBDZ)t)&3S95Np(at&~MSQSdJTmh>>
z!Fa+4y&qC7E1f(l@hQqm2bX4|Q2J%T6TPQ>`2xgiKf6rhobE>vmrYx0u+v2a%0b$)
zj&7_Hmmuz1&G5($ry#RpI63j==S5{>&CenZ?&Wx~b-zE$JByVlA%_z6?<qq`vdd6{
zC45*JN@StQ!6-?&GS>Xa^Xq88Shb%`!zZjsZaNdap->~Pf4VSqwr+o1)43zMz`=Jj
zDkU)`9V};kR;nO!LmiQDElO1RKh3(Ck1}}|$v97QTw2`yPCB81Lg&0qC)={}=Xxjl
zZVzR9?AwEnuIKhDcb|mYJN>7$G0UaBBc`J`a?y;U=3v+Mw9S$SEvNo>&UY*=?&Z}x
zxs!v0W2-h2QZJiY-Yn)M3NKCmlwCAzFg@zh0mikQ)!4agNS(X?-cASuh}S0+HzTm8
zSQCYn5zq!aCTthyV=QCL#HaDD0QPwFtiqGAA74h2+#PM+8C;ajYv{Zbc`xpz0A&c;
zvKY<nhc)*eCNdfd&ih)r5a>bOlSPwPe3C1N>N2V{Xxe(A@-o^pM)vt(c@^zftV9VJ
zy-#lm1F9^5V%i9iCSE1<qF|AxV!ZBOtAd@2n5x$3fX8LkKK#9CX&!7Gwt#+sz6NOY
zQ3YP)e%0?#fj>)FG#-W0pwzhh(snxL&W*+A=qMRc-RQN&k5(jh%g|X92?VLmCk1!)
zr*T|^Iy-cKz_XN#s?6xK3nQeXGV>THQ=?Vfd|Y~w!pRf7>+l?xK5?C;ddXpX8c9EB
zw(^X9*=MZW7j7)lxYysZ43o{KE%UF|$K@@JeyYM~toftT#u1BEevR@)y&d?2U+c};
zhUyIUze9+>%#%m*iTV^yYLR0XsS~fkI$pQKG$*^mQR6tbS1_K_8yd2bq_{Hdx>Ycq
z)!j&DhNQT-*l(#%bfq>|Ngj>_Xqn5nbry!l*kK><Y9MOyK&5Y=i%f5i$ZG`Osr-H2
z`|Y)_o+ob8_c&Nblr+vR!%|xDb&jCbIDm4!C*<=-2boU4QxX58*(dm)S;N|l(~^sh
zDD!Af6WvG&7Du+z`&H66?2L<dArPf&XsKGWZf~&+=rsI1pUZ}2d`g`?o^N;5egZTZ
zUKT`LbGQ}{Uau%7tLOmX<}3NjSdF@;(t>Rz&_R{J2rBGbLzO@P$Ty}en8MgV7iOOi
zJbO{}hR-@6ccpXb2s*#aBXQ~OZ|2DP;Y-jJ%IkX^ZgU8P0OnD_DE(aCF*Un|Im0_^
zM=&9)T(ufn4I~#2`z?1^&uYTg5W3z-9e7_~k44u|z%dbtM(H%~ZYK16P`R3`n>nH{
zHCi|2RB}xBHTGq?CA1jZ9h1fJ&v5Iigx8ra(OMwt)UYm(Ra4O9Yq^L1x;ucT*xGvN
zQ|--wPINlo^p6z0U6W6@mXrQT;$$eBSF0Vadlck!XDfL8ICe+#^?^8eu<WJ^=fNK4
zJXolH_B>b$JlOj`OE#P(5z#eTRa$N(UG%iI1^Ph$?-3LVH~$)@Fps`RFr!?B`H_6C
z<4ocj6b*xBD$lTi0tLRx;<)TaL9TDIsp%47?{Z3WeV3MzN?Jmas|_=)$f4QV5G$W)
z*Udw)AJmWj&M(^k=n(yA>&NV-0UUc@Uq5<8oyCR%Wz)(>LB=Fyg@a391yGyXLz1Ah
z%kqa-5$~=hCwrHet+VU<Z*VOg)z!5#vkJ-_7;`(FMM+VEyaiy7wFmW%4qr^F-^msP
z^zR+b;GZnD`0e4-Sm{ZYZt3AllaiZegrLC2-f`bMgIlsB*E&(5;sf`2zEJ?uNk<#8
z->8VF3TrhkPd++Vl~IQOcs`h{+LbH`8>$6*sUPJX$@kcWzAyJuGq=Z`Zogk}>6$}1
zcnt@-1w`7sUn)m6fnKWDo4Iv$*!d_Q8*9w$I_uGh!h8vS?-VGdU*VmCG^Ltl4>l$w
z6}{9_cD@UCg}1xllwh*5c1#ES0>#G#Z{8wOS^K`&HQYFJbV41LT5!c}JLHpeC!xAu
zzqn0IhcHJ5Q+9koI|_Z)tEZ9~1l%I;TrYAXT@?Qk3cU>D$+1;?dRs`_Q449638x;@
z(}$RX`Vi@_FFqSHI^l|nfy$!<W%A+nX0Zd~8tk!{`$l1J%)LayPzDK{+46~*C`f}I
z?$`MaFX{=oQ?rs6k?b7zyfa9UW#z8MFYqZoaJPTdJ#;>AVx_q^t&Obgakza0D<7A)
ztYH)3Ue}vr(0d&im~xpKRBK0RMCx2Wp9u14e@wm7Um|kI<hW+Lk~St^0V8y+w@nJ#
zB_x!FrnGV1;)T3rSv}QkkqF0^S-z|BR*pvR44xqUgsW0+>jUe4ZR+NQNZd0hb3&$6
zzB4|xh?!B~;OF`h6?@M{!cKP7D*GXBX*yOgzu~0~jBaU89~X(MP<6>z?CX4oKChH(
zZZcW#i<224#z8ZbXB<F2vD)Hly*XKs1Rr~)X#_%D3ymHfY+8E$eKAx-yX)(PSk)E~
zt^J3t^7t*s*IAnxthco(&lCUD@6O0$eT2hO-B&zs{i*+j4yQYSCwt^I`Fz{+Cg@t9
zv)8(RDyFO2ID2gZVPJ$BvHKwC?cies;Q`V{pd=r(gkQz-NN%rtkjx<M<~&(dTCCE`
z)&~NVG>C5KqdfAQeBQ;q=?YG3dBdi7_$23UEMGd<u(HJizkzh*D}R$W9HwlmyL&3o
zs4GmX?O4?ry`E<q8hJK8d~dI99kx9WkvMXY>nXL)3G7>R6%$nwaRT6O|Ae?#HYd%N
z&B2Zs5Lm3yEIV`u({m0GSd3iG2)vxi<9dPCDL-}BN9O-ROGl!~kcyr9<eq|}uspli
zjWWT(`WR$d0lKS};ugxPi8PzZIOOEYY+mh`L3Hou|B-GIy-W~=BEc%E`NjxUn$*Ud
zvh_Pqe^Tg^6^>lJ25=dYAf<R;ScYR&E6`o2;n7=#gM<^Sgo4{Mt1?zgRqxinBKT1g
z^c7S~rJ{rPm6D62V%$F{*h23RmsL)g3+DUVtT(rz87q6CS1Aaw08$E~jK9cJfI#lX
ztWoFZ(zv>!y`4h=>x|cTzf=;v<wyZFP-=lW)NlxEl4I>x6c)xqL;J;p#oo1JI2zcq
z91Sc;@IV8L8l&vTDt4^wa-$2qq1HT$7-a31V;N#C*-)Rxl@j-gvA(v!0ahOE`>WG4
z#5}TaDKs73aIajivPP@A@u%bC_HK?a`3cj?Y>ziUS{abxjU(O&+x_=u3vH4TTZU({
zU$g4SE&W(ZtjlG4TQh>OsyplzZh_S!z2&zasZyV{Iq|oxw*Xg%se*@UItf_WhmknZ
z4D)5B?k#8T<W33_CRP;}hP<`YpUpc=?W=0c6-ioAe??bIp`qT&l$jQpRbJ@X&-HTz
zW<hL$87f?mW{!yK$7J0@iC?kOH+quC!ybJ+6XIS|K+`qDE{zV;`p$=F^Klv0)~^L=
z^<oiDbzw^SinB{bjbb%#TEBKj(JJ^7&EQL{ug|w#A?l>S>l=jJsT~bu+N{>a#JZnf
zxM9orh811FgfYBCKj`@$!YzWsB0XN7LyBx1e@RJ8qxZdK{^NhKsp<UCc&a-*CU}3T
zvTK73l~su}3dW0%hppoESW$G!<FRryhrBwZJ&$z<CWszNtBtjx;IW<#H9yjM#f(m2
z6-37VLi@$mIljeJ@lC)>o*iJeK*%`*GLEC)5*`@Q#c*0b*#woPI(5{Dug||C-!I?5
zCIKMdU<>LX!tUHRd|bnxk>Djv=&ek#C8tBp5unNxGBZ$pdIr)UXx3=IGiZ~QN_EJB
z1~u<bkt<_7PKYaCmq)tma^1cTXhY$oBiDy=_(%`S7WI79oAWd;cWdvWrYK7vT-qEU
z)m?yni_GW^rP{m;H)xaqCufTlZb}OSib7c41q!G1&1Qq(ew5dXgU<%p8~EzZAHe1~
zpb}M$upq-!Z8(ZwG{-Sqi@T?4!=AJlN)8KE8*JKJ!?z|)t|J}SO)S_1b#Q{)VKd)#
zHqustwC_=!e~Ptb(W<Cm^HOEdGnZVb*#BbCSBGW#Y&9z%%fcRT5va+=!?IumuBqNH
ze@SYo$hF-SG);L1h@PSXm8uIL3}aXt4QY$nQeUV=u?zh7ci)XNjImWUP{~hlJcI(Z
z+x8FM^?h^)ZMgbRU;T7GFORx>@HKK~kqF9*4leBk9c)V0T0&S~#`Fx7+E@y_{RBuZ
z07B3wro__2d-36wK7qE)YVIvNDx=8?a&oGm!KfkKn(%TSY?Z;P`rwZexdhE$@pZDb
z@DiJZ>k7*l9v^G3K2Q9huj)n%N6lc*$YtCoMVfS8$^NU><`fFJKg&`_C=Oq772o7L
ziPK?2_KUj<h4lHJsQiRrO2Y))gXVa8^!h@*vaaE+XHDr1Lyqc@{Y;@Dp5(|Vu<dZB
zcV}tou^;Oh5hf@Nz&!{sQGmSQ7;7zZ9kQX(_C?l}l$zwl7p8yP(`N%G%RQ=uX_EOw
zG=qk*dIbAJbh`(9i0H{|N@fogaHz>p(>VmgAaX`QQ<Z6|@l_d4-D^)Ltq?pX%q(vF
z&=N_T<y2;48{)BU#qfvgox}Pv_Xio$!21(=Igfm}rhAopLo(^(dEM}1xrt}2AzYI)
zJoPC@^oApLRPSjqfKM@tSk%2{eEz(iL}I{K*f&;i#bns5w`Z&Xidm=(Ig~K3r;8tf
zO;M)zPSZh}d*y*JHj(v$J&{%Mv);kkaw#1PA{Bn2l|GK|j9#r~24@U0nw-4CMDWzU
zxUn&-_Gy;Np_r7$P+O8)2T^>B@0eje#ooyf{!tnx;N@3T#D*_G*+x6mqKF#GLp{_`
zbJ`VPwSRaTxfwg?8dn<4t~4r|Gn2)Y5Dp>9z{2h)0U@?@*6V`UQ=w(uwME?uC&Sc_
za9q85+6{*x`}zERJr-;A7Xc4WciAJO@Jo_iB#|Cqh}Y`<qIubDU_!HR(jq(7$z!Y#
zrS`QT;{{$=ylqunTc)Q%RxHUX@@R^=X{R@?r*;9T3&K!h6Ay)9i^6?#gOO+)bVw;%
zPeY$QQi*;^<YR0m671TR5-)7*b3bS7Q(}u0Bc}@|s_N{x;oVhz^c~t7PEIMfhc)#d
zZ8Q9XlkpCo4&h|<Uayz8q~%Z(T;lB?kjdf3hLgk9|M>6R--Z{>Vbmn$9}eb5LGLN{
z<1!X28wLa+*xtZBkUXC=kPJ(Udk2zHH-GN&yH)SkU18{ItvVfVPd^wk#=_A1zjL|?
zFAkPBblEyL$gtYZ=m$+vrs*P|Pp^jBry!ZHvBWGL=kFf|5NS?0I3WaWaimQ!J(G@w
zGs8|L^GNMK-4TNIBo|a-vuhArEUZt*rgneg(4&rr-G=LvI+#n`3q|h7xOKK#d!QP_
zo?IHf8k2%5z}MRJMyoz(w~h>-jVp7ANnwSTQdGN+SgenV#G+D+PsJU~d=U$ZBBp^B
zbh%&e{NS~k`)aJelPQc!+#+y=1p#8zc5?{6NqNwYFctKuM?UB&b<+x(PDM6A<03c|
z-5)AE!DuR~8TJTXx#q4%)Zc?;sjs!iCM?04o<9Txu)PvJ1B4RmbPCFFcyg`Vb0tQA
ztdvdH3`sZF)x7gv4if<DkVgAgWrTD&;ItGQqM(5oM_rHRuF(;PADsWPPxRfzXYxIR
z<=-D$2u}Lob0r)a@R1_U;(IpkTt3SGT}!uJ%dtvI5}9Gdw3UW26XYM*;rzeg{O0J)
z%))f;>AR<TKVJ`{tz_MB&)mdksI}cu@Q$3s`{K9`_f3?4?K+}URIpvCKH;9bi^lf2
z=!B`;>y~`=!yz!h%cXv&BSd;>4)gZATwSGWhMkP+PP+u^AgR<@9{P`YmYA?10~f5s
zH0EsFIrsfdBz=|21VRHOf<m|9d=J0=v;1CmIhxuX@X@W!H!V$KQXq$ztV~;iyuxNs
zg}}@;zYz+~>5jxT=HT)a8}{Vcl31X`SM8z|+||SJp*egBtMcb`gw%x;Nja>~Zl3<D
z=H9-Ly9V923Tpmxh`tZkS5p9O_?~VHB8~JY)ghDbQL4if1)4K}GK>VS?kL>=?#`EJ
z%8a8ezEe1!&q(<f2s)97fAG^ByWb@R-~GOfNpf*q`R;0@Rl9o#QwNlYsVobSzx0<o
z6DJ~=3N#|7LhKUOd<Cf808^QP45vt><0Ccw?y_<%Jr!C)6P&Yo^Wtb-Wk24t*CUxE
zp`5``MHDI|p@<bTW5eSe-aV|CV*nANHAFISi#&6cUN<Jw-Bg9%M8^s&y11oBaNFM$
z7!&MJ+P*Sgs&ke2PMMi}STQ_`^aEKjLyY+JD`2CG<ARN(_wxxJVOKi4$i}MOcM6{V
z<sgd0vOX~r2AytbAvcyc?6ELx?r0*}v|O#s1>>n4`a~}x`WPsIu`0;W+R>3Z-C&eT
zzthQkgv~B>fvrv{$}<k;s_cWnOe%5K5!b7|B#sG)#KA-BJVm=MItF``ASS6_z4(Fd
zysz~r2XSx!x9)G!%fac?T$RAA#i)C7dO^a<wB3;MsV2ky#l(2OdD6eY1Ov+To7N8b
z?fKnu+)3}Q-Sk)`xvK<k2h~uP;p8a)Z+K&x__rYXFim~q$-Q%0eMRg+Oo#s7VTx+L
z3wU;>F70S;;$ti5%Hs}*L|X`4m>B;`xuGS8l;ZNz9;PU)wpm5}^d*gF5E%eeY^Bpw
zU<MBWQ1vq^<6bC4)!`2}czw*<iLP&MEO-XeffcYJhhtdyrnJJD`ST9DH%UbWA1tkR
zjp>>`DcYY8Uh5y+8>(?2iV-%TwPOswPtPT6MHzc`aD0%%nc~J|AvHWC?NY{*(}@KF
z&l*hG>RZv0Bh~(zTE;ru@?e@P4E=b_Q7_Usf+B17+Jh0UOdwAH&=IyacFAZy4-^*H
zmdx#ZDc9bC$2%^0ch&W82tPfT1>cwIv*`y2t<obMVb4jTNy6GyfR^WY_%+xeo9Im;
zaV{k%h%jt-W^sz{m@j8%P!a^pCicYEM1IVsH<t<-8F>^U%PUf0Ev<9~LjG(L%?J`?
zHW^FXm!v(|uR8ZQxl=YWJm__DSEDEEM1^Hy25+F)Kgf$vbqv5b>}e5ypd#}G<A?;;
zSigozrrnP7-d#=JmrW-%<-;AOGAtx=6%AzgD)3X~e+p~dvB&(y)yL(z?siR<xV+W&
zMfj>_9hkp+6SD0OS+kKdIY{Ovjxa%hyhB(yRFmLka{4jvGSd{I=!L#9-4B5dXA_k>
zF36$Cs{Msiw{oSO)<pi?I|$VR2*+N|OFEw6Za~+%6YNycFo!Xnu=rts!pObWUI{S(
z#t;;+@3qb&^CgT`ytzaP)|V^8=hd!%uH=*vAVqAN3#S_`*y(-)l+tGVgfq;-f~Kvd
zDz5Zq-^J+xGJFolVE_#!Q5#sSAMS9d*TDa9uP;&J>50(#j4C+smqWv4SrXkr&D^;)
zmo6__w4BFKHrv>&Lg~bndxobnf|!(oX<DV$rK`Kfitow(mQOIs0T&LX7#SFh_R6E#
z^X90ggy4QBV8od}+|T5lrdD;GF6~He;_Ub}<XI9vmNR)=Twewskn>s*=6rktWAKIe
z2QJko5>L6t%+dCfUB&0q<IuN;nxN>RZy_xNs?QJi>51M@kLQb<Ja$$=<;bbP239={
z2Gg7)n|`L1PSy`DYllxymeUFxmKDHGbnnlc?E}LwS=Fm`^VTeLVp3{Yzk=kFPoB%6
z77isZ9MtXXk$rQzogi<q0mjNj#XqMP^I)tXAq|vZd~%zAw;?7pCF8>3`c|Ms0_T?z
zH;zAY611V&kH_w}-##U}_EZiv1+I^93fEnsl|ppyI1~8=&)9>J5%7o`bDNJfP*?$G
zl$OV;j*fG|U6`Q3*J@DT`G{o3jp{w?@87etU7+9fWV^t!U}w7lXCCKOT$hk+-%YP7
z(9ud!A#~9^MynSe;1kktAdN#Q@CP+pCT%FvSc?=S+!(bz(Cqu2KSpheqK@orCpy(A
zXo51W@O5J-y>k*g(ue#Bc+f@KtFUrXno92MD|CI8mQ3h(<{NJ`iI@B`k5q6D5a*2w
z<?hIK#hIb^35w7roc#W<>fKyyF>Eko5TFWaDjGk&+hrhrBsq6>9_ti#Yfkhy>PGwc
z@X5RrrBCA7`f<kCK8G4=6E+Kde&sL%6S6C4Ja%<fAA|$N=$)ew6-ofgA9Ny=#Dd3f
zfFT(7yxPP?6zIuQVzK~)Czu1|1;Oj-RwD<9Ab7ulk@=ka%ZFl<nd4Mw0Xk&+WF#YD
zF*kT}_&No=%+sW@_os63M7Ori@Tg$7J=RcXlGC|SZ=N`ncOhE<Mq{6sK-}NYWHf$r
zB_bZ21U}!vIFX^FyNLZzXU{c-F&cghJa(5e`|&z6`|WM@)-@^wm&Oxus8nw>DNbdw
zV1AGng>pnrcp6TKM`LKL1CFBDCDP}1c2*UCkTNONY~lVEhvNKk&bZG?!p$|=#rT0s
z>xo!>W9Az(RP{Y;ajV|^A@8K(Yf*|}?rAd0DM-daW0h%?tN^Jw-AnzhM9js$LY_E0
zWNpCJXHkM53I$V*RG68?4&<1Y#f60<@f8k0QGV7}=#%s64|t^S6pSTUjRi^&h+11Y
zue`eF4P4+)vL~-4jp~8o4O4T1Xl-$G>)VBqW+94@4%P%}-;LVx&mZWBU{a4?jsO`&
z8S7+$G<L9dVn#@E=et6+hYv3W!y*nYHu#XOc*v20a}pFAbsrlWsdT~J!K%i!C*$MB
z|8qAn67=3oP{CEF$tAk77R78G9pqqZnjKq?_d24WqQp0SkP|r-vF6G>`?vK^-9NOG
zVqH2;#KI=Zx!V(>M!>v`$nA+-j!;r-gNB6hUFEURiqAn!Txq|nHBULFW%x$C8ybr!
z_B%Dkwx#QN<LV9&Pj*f-MU0s<;<lSdz{`)s^&MNQu;N`|8I!&%XcX9$>Hsl(L7ov5
zWQa15V9exug`5n~zBFp4ROcOX<f6YC=!r3ItTUjVY?jgnhV4K1^aF9XpJii&a$en9
z73|J<8yJ;43Z5xAdwt`h*A!r$YIw#w*F^nF)6iS`=T#HhXX|SHTt>}m^)&t!@WUfd
zDn9GUmd*-B{Nn)nP4`*62I8Lwqjnr94aPx5t=@2AP)$9Ljwjd10C<P-WtUrz<UnI$
zv?D&xn$#MilPUY%2wxpadOw4hT5<8xEJg{MF$2jf`;kg@{HA6KgAwkyJ!UShJPK-D
zxB#oH4a*2Wbu$YTwlSQTRryXjPh`jcH%^%T=cUVs7~N{g?u_o3T|q1*S}^op%5Aft
zQtC52E_EO5`?F`ywm>W@nt1bL8l|Ro<}D`qwW_<=qteTV7ND_@;k7=W%+Ur({^z{T
zU-oHKor&4Xy6T{nP_@yAp7HKdp{JndI+<{@IW0#lI`~KuiA)yjo(fO(81GuWAlnBp
zzB~75`Dc7}^Y>|#H{`0d`^6m23%`S%?`{>f4&MmryRqSkU~`pT+X8&GEwJQo>`@@|
zr`Ac1XE&_-Qg{ep-ULv@qJod2Q-(`LPZ+6F_Sd?8#wq8)?NL=-83oitW!TB0QXU{x
z+_6U_q-G!IbQnDjoer%qyzJeNVb9LsPZQ!Q*M)R7;qum&m90*#jSE3T62`xU+5o}w
zpKbvGNexy``$x=Mzl+jd$x^JXx@P#?#962#@l>W<p-hB}t~R{f0EM3=g||V>B>Acg
z+PZhwnh+KWR-xdv{9`aE^%F4f96X-(g56;Ub1UtfoK)}HGrAQbW7TM-(`?$$8n@t)
zo9~kOs9-22=qcr27ba0tU8dGH1SdOnsEa#A3Xez>3U-By$J{#_A;5wf4)VzVk4rC|
z#eMWund!NKQCqvOI;@EAeD}f0WYm^391F^?F%NO*ohWx(E040;O}CQK!iBw)QVIks
zp$)g^gb&1m|85};#Nq}aGy6iJx@1E`m)ggPND+@)guPQht>x_*w}tze4g&#1igVSu
z3w?cmvDV6i?mRl8dlC)=&;R6zd^sJTctnzK2C?H>o|{SM``XexC7fxtkCVqoFsD++
zC;mDiV8vpem87|=3*~iZVL|LRC-C(`tZ819;BkWWB!*n3#21t&I69m%RZd5bwN#W4
zo6-W0HF|3oJXR<kDx^=j9E}z3`2+K$#LhiZhqWp7rzq52D#DIVFmm1Hags<h{0B)d
zf4&p{#TgvFlgs85O>^P`Q39Bp5VvYWIy18%BE%uO8rq<9cZ+NZ#BP#*THhqGYqz*r
zcAC&C7AYIXA|oQhuQk*s#&iXQ*n<Pvc>5lK@HrOf3-Et?j?7pl<=lD`7os23p+vK=
zfmWIss+evTxenV3hFe|GOZZ2%n%J2$yowJr8s<UCplKVUwX{g}u+VraMfB5y)n9%6
z5F7&5N%7@$^52)m;dGjRw^YX^u+lDDUoJ&iFQ{FKHtThx)nPHy{YfIB+}pr<NC@n0
zU>p*{I@2Y2`>e!4p(qs0I?6sDF<<UFh-*JRl|u4QDEz`oDZX|{U|!IT-x4%yeygxU
z`h;-ye2;<}&0s8q1nzbI5|@|qO{v<`pQB5hJ-+yUzAK~F`IamBwMYG2FKXeZTiUUo
zz;W&wei?po4~J%1e@hluVYgZ=gvdOV7qhkVs>5Wf_A-4$Mj$0lMbqK}F#@f!ykXS?
z=o9L2TI<Wm^?_1;(%pxq3I0k~pvzMreuviS3CiQ{(RmHOyN6?fO1A4qXO#!!rQU7w
zk8Dz<8LUG=-74JC{lIYMF1ii5kPitHAgId>h!nHQWgpKq9DjC<U6G*iDpL8NL*?t6
zpxr_v{!Hh@c~#gs(OH*_n=5j)b_Ox9?~tj=I3Hl9kFV4~gl2(8MW8}8&&rvpA8B^I
z0S)!;S~1FL4qRd{b!CVNir&q2NC|_OGd*x;*fBMdoItolc~L?Wkzjn-7VlU>yK1=~
z6qg~s0D@2&{*Cra`m;vImv*TI(pl+Ju`Ww$bA#K7)W(R}<ZzbMq_nd#-@?RC55$Yt
zGA1Ow>wB#*e->6hlE}{07Q&Cq17KB=TS4uEJf)&HXhvJ85(ealzr$S;-{=?+v2h=`
z!%EA&Z9lVga9ZkR#n2=*TRd)K-~9O}ft?{?98s{d`9|v_@}-Z7A8I=<IT7iLqP8-@
z;#jvEI(d{pWOqsijmisnn;Txl-M5Mk_DzZ<lViF9#XTe0^IC#IWxW7Ceu**(Z4v}T
zME*!(=5E0Em~MAwO)2zfaM8IPa6&|Unz0*v6ptb*+HJs5<wO2q7bTy^3M+8f?$lb^
zT7%m*r6KCSbBE>frSs+9yS3NefcN<U8JP|Z?!)!W)kFH<A3u&VkCmnwul3G0M4_8I
zRiDq;Cf<<le_RhIC=S@`%i(|zy_$Y9ooL?4Nvqs*6DJTlpS%eK#(L*`k(=tA4U1U%
z$lkEne^pp-3sF)}roa`awV|xIKG4|pRJtAQQ!WpYO8S;$j`Y(EwVTo1;Xrqx%k2tk
z0#89{v}*fJLc{kPK{J2F`?}F$ty|g0tIrqH&FFu;UX|7mJ3x`J_sbuzCQCS`JFGxD
z#9?U~yiTgi0vP@0czPa-`Vj3HO2aVo<4&u6dTVwasj)i*iPl4A`6P?N^i4mwBce}&
z>F8V12YvyHPrrxtE$JIMh8&=fZplh|F`ZU6q;~k|hP&k^j??$}y-pPNo2GGzXNK3}
zBNbb8;dqu^d$(NYvdHkzMyEQ<)b<%O`5o;`<MFVgO*!Toc`A9eFQ}h_we*7;m1t9~
zps4>iI`dP`cc*ZURNH${Ix8>1&nAnh<|&p|VmpP0WH^rqm}}Urw49>b%hM%e)+U@J
zIn>?62}LSE@Oe}=Pn8}nmlv<<w&3FQ9%U^5XSp^m+S#;eP9zO(4&G$6z2Yx0dvmQ<
z^!fFnxDcrbl`l<fNskOIm<Zq63#3HQh0OK{D(Lg49;nV0^Yuj&J}^0d*F|*+o<JwI
zZuHCG!QK99ceX6yHY{~>H)i?UvhGDrgL}I!o9C?~60Uw^0OM~Mqysbw|J1{|-(vWq
z8v___NysB?o}3+@^a?()72aQw)(CF~f-nh#f+*nL=a`YK8nbn2uDOZpQC6Gvxbxvn
zFeegQxOQFp0S*@x^3fuZSGMdvs4;kQXY7b(Dy+$oi*CRP-8%@_XjadX1}-T4r$JM1
zkCv~o=LvNXB(I&~nOs{S<<<)8gj1Bu9C8<wHiqgOpDKUw4UY)3XHf&F?jfiBHcpiJ
zrrNFL*pAkY7&qn_B4|cg>3``3qE+fZKKX_F7QZTU6T8Lq?+psNSlP7)^OOSZva4j&
zT)<>n<I>H)<(uyH3KA>~8Clg-1C{yyM12yyOKpy)h2c?9W5Fz~@Xa30?!SPAof9|F
z)DD;O0$=4XmCU(5s}D*NT6M}iVrVWYy3`r3N9rFROmV^ETl{vk29rIae9&ZaPlgAW
zW^5TSht-Jkfp(V>ffw@K7<uT`!qaKLm9i+O0vGB>#;VW^DrD7^J1Zr&)$vLYxDI#a
zo{6IDaWR|jD^*U-u>Vs!q7*oyH8V_MdtuL(o57gPuLjz_;Q+27>rHe?f-6f#cM({Z
zO#N<R_abn59v?J*j>_iy1!7cxh{aw!*q<89^2OzCs<MFrWmgHD%$$FWfE@vjs14ZA
zDW#@YqjOuXI>@3>WiJjvQ3UQ}-&AkTY9bwKxw@larIE*7&m(c?dJ>?275)k-kZ9Uj
zQtSIb_eweapgJX*-fI9lj1_?@5RY0J*R({F{LbCt8osrsD87-j8wVKgv%{%=u-Wgw
zbG#!s7L5$`vdaY#!4A=0vQnZxfEtlX#e}--edRos52862+V;B-7oVH1M;D6;zVNIp
ztF>k}Zl2}M-ttk;rtz<CT+lqzEm?`9>sYOY!ZD|H&0KO517O;v!&?UNdV2Oo_AZ(S
zm)^5z{(EfEJiAE`6ZwqJ5F-_7|3ovR_5djoICrg>V}%;M>!k+Mw14LTY#O;777dDm
zaq)TcT6i_y%`pt>faclsq$Y;|sj?0mk=u^k0rfB|`q-07CX!pEnsB&0jlB&Dc-E?}
za0z`GPE~v`UYZYp@)LFv4dC8E6wz;=F#7_<kGT4A(xKFlIOTkY+_(oo(w5t^Eow8z
zc7CtFb;8TxmoY#YFk`>VTUT9lK3K=TP6UnbVhf9>YmE1_T!5{O*ItS3QHL@gui4#C
zsBZGdzJh@{{!k@1YQ`be!*$t4Wd=d-z{Zp|mIv9U!73PmRobxrgw)i@O6~k$#(G<j
zuk8$-l@G?Smr}fEf!z`b3~qbl_WAo?6Oa1ab_5#>jsJOE!#$(JEg{w^^4;eOoWasO
z0x$1IgE}YG=)`Tu@q!j4yx0C@`0+@N@qHJgCAPDn)>$NBBr|$|7d&y5y)|AiLL;vQ
z%DotLzRc-FD0I%VD@uTwL8+ZV)o_haOR{uZaJ%q3ZaL7<o7p*1bg$=`sZFuWNZQTD
z*7pD~90$A9e0t;0w77HFR-&&@@I?-ORB4=1X)*OMV^w!%fjM@8urj16K#Y(o1pAXY
z-9OmHG!rdp=b=YG8dleNx?q|PyiwJzY#f?T?hQi`G90YS*GKqDo%u6jviKj#xxTBZ
zt)+n^M&O;?MhIqXomaZ8$|}&nv@-mY_?4!Wqb^4?m^dxXDwugm+qewPwD#=I0;_{X
zThiF_cvK$a;UzeeW&-Vc>A^B4CyAn-A;PM#$C0nI2ltylesihh=g|q4Wvp&}=%sYm
z{+Or45J#@i7gu3n_=libd}Z4uo950gxV>FWzZk`?3#VWOy|Bo=s@`RUm^Fnq(AH<4
zfPv+;`g%CRrpUrAB|hBXLgecING>F`zow_UIi90kb)5w3`s$kG5#EM7klDc5qCAKW
zaK+xFP1Djj)G5hj-x5nYV8x96eu$Uk#A^R4#1~bU-bBZJ1mmI~R0%rRmr%;36Q@E3
zz#|atgY^eDJ;&`9SItlAZmqo(aR+uuHajfK3Yd-F<^bm8C&14IB#%re)rL3uAhHe-
z{SZBL0MQS5K~mXUv@)X<iM6Y1C9&t(*^riJ43oPaes}yQu?zB?WfFjE{2ox#<A;z?
z04AcNyQn}%XBRoasbeYZf5kjKgc97$yRQ!Lg+AE$3fon{=~&HPcq$$2&>(H?yt4Cv
zK&L}1`m#a5wI-kJSjd_fNeep=4{@kZ^tj>!s6;Uk!u&<Jg>ZKq6N4s}UoI6nBaJsz
z_k@8G6Zx9T(aYnGz!8~?U|BS)qF^R=!1g29E}|&g<f*%4zT^8=XLAGw2g4RMZD1+r
zYHh80554!0ba{PIl*=RF#E?5OM~ZVKl5lf<AH@k?%>X<q>OA0+UqHmx5ViFK2)FIC
z&xm-;_l9g>_;j~=w<O9Z(6WW9HMQSKsOL7nO?7B$LdS#M8O<?96Jx7Jp@4M2GyPI1
z<^gEN!<y~0WoEGvd|+vgQ48&XHO*)?cCLCH%{gUkRH)3gxTym7-N%^Su5-E*eDxW0
z-HTFZjJ4L9hN3jUXy`BQ<rGO{oFc)(G)%t&Jtm%OquB`sPkty^n{P-_RbD|bG3JNQ
zs_o1LSBQ}P4()bzA*L`?+AB_<0n5M9{xhuL3li>Ewp<$tz}F{SvT*GdyAvGr?6n-#
zgQZmldL|!v*hqy+qKIz8#eM%W#A)<P0H%h#{@_m%YTMU7koAnZA_>MWEx0?mba=1M
zFwCoV5@JanWzl{L3V4GSSxpRF-qyYE6YJF&!vCoQrojQb3t$L_F1Is?0*<i}BGT^j
zAlY&<o(Nf9uiM}P1w3;K3gw{ud-oB-s1M9;U!_Ui2@ZbtIu;TOnuLYxtgmKskYS6S
zD}*E%;Ez|C&+vYHfpm`(zdZ<zw`;SW!+KljyV^KOIoTn1?_D?{J#i`|bzGSfmg3)n
zrCDJnz(dd=?c%H<f^qX=zsa8aEDD>b_gq|-n8E@Ls@J{vcoUKT0MHjz*+@|}-9>k{
zHe7X>Y2^(54W@jIGd}1|6g0n|41->UO|<T`+V%)P6y1C$5+675D{%3UzZl}~`jCcj
zce|Udx5&Nk{3zF%&R|_EcpBue)KgY1sYM;7t^=>{k0CdKLyh}y&76+Ik`1P+OsdCw
z{0Wv_%Q$~lFp*Ib@^HVt{(l1N3%*00TG(0}(q3iUO=0bKw4q(i{vDKQ_z9Je<_X%G
zm){q*&oHxG7hEKu9RF&Bp~4#DgsIk+!Z7)PDG!EFq<sNIWuH~$#>9eEO?yYaZ1lQ4
z9`yJIxy$}?SR3E@<SuE=?C#AXcdG=PIRNF)#Y>leJCPCOY6_7+-nMqzN{XdaI;^AT
zF_|`!9RI--YA%{8(rsP0tz$lcfLztgaUWbHNDQOY4lPIxSj}lVp`9T-_DO=tfv<9U
zW_d30V<&J`*HsZg(Qrh|?GUqBNd8U0)c?qBmV?IH<4rMifa41;!M7PIORB?lhr?;X
z!0ygwbCLLNCFGp;7=VL>bhWlS(_4x8T95zG*ZAtU>4(awh7w%VX3$-vzaX>#w4@+%
z+G?7?K2Xbm%*iC19gqupwhiSJixiiW&s+(q)oH8cD*`5c-5VSd8Qc8v-Nllh0gPT%
zVB2;KXVm@zJK_d?4=Sx2HZDf7Mz6C<Oy933zOHY6M1Rf{DE1#klDsZXaE=J8W(rIF
zE=BB;I2PFfYQr5AR7I^@_BLS5f8d<Rk9?@NK5TE+fza^gkxk0zA|XANMHjhYc<FW%
z{%#Xv`?+vuLt4H4w^F*>aUUqZ?f8AW{o1+jBc9oWEuU9-{LV~q9C38j2KDJ>*KS|l
zG}r1~?b_l)A178ly>#;A)gPPw&)k)OLxI2lEgKzDZI>dtRT5S?61r_6>9THWnaWMM
z22%;Cuq7l{DHNgH_k?nWBFT|k$T99~nEAionQ^o^TkY@P^XyYk)3c9vzTfYAeBRId
zd0+dv`MGVBdGm`;VA&1n^xYzSZ!f9mu3h^zI5e)_VTXbFUc2Ut>zi}ZjWLI739*)!
zmD((DIi{XZ#R97EyY+K=JR0qh3NaVjpx}K>i+ShC-ON@Sa5y7}ZCF{|a`JDtV|;BW
zh2#YvR-WiHOVQ}!QytKyGejbMm2aRMU2F6(Pc5EH03l~8929B!R~thMVa?pL?Qu6@
zfk_7f57*SfE};hqDXoqLb&zbd6<PVMnI!9xTNfpEy}F`s45}=);t!2UAkwsq-if~$
z<X#KZo=`TqF>X%yZOuuRExp}y&*&@^>wT?I*7rWl|8h>uYypu(QX$XlGwYG}JKoHb
zdFE!~rPcIh53W3Py*EEK{>eSF22|%=&a!^QTj0Nxg+BWplTC$kt}UkSX=qQ4VGo|k
zN@puop#H%Y2FBn53K|#yHP8oEv|z@B_qOeYm^Du?G3#d8!vdwE!g*)~2~Y%^B0%15
zTUIcPy_N1k=8Frd>IH2^-%K?1JEC`rTp7kj5t=-)5S%Lu#7QYnN<InppGShIws+Q3
z;lQ1z6xQ~3|Dm%GPsm_-T-GG#X&C%U%CS(iS@L?&DOCMDELOW0w9KMmCu96BsD8FC
zzioPCYgZm+R>cLNhyV!=e#Qq|*W7;PNr>)QfWbU7h-!}{#xY-(m_tpv4i5i-GM_;|
z@e4Yclv}WCC#!j{Z;FY64*J;>k5P6!fIT5dB2CU{IrN;qAR?h~CTvUE^$38()XOO^
z+#Y~bba=x3uKR*_R}{EhywVm~3QG*MzwSXxXu#baMoQqjZ!beYA{Z{~bM7MDoN~5|
z&nrVpSChJnoZziIau@xRxZ?IpXL?i<oD#kr>g>bqX>&DdGeAkc*Lb<N`YP{63G?Sr
z*c5TO;kC^^K3}@Fs6+9Xu9F%R3riCSNiP)UQ$70T;UCfAE1)gA#E}Y^GOWCk*J0SU
zCbh)iG5qx1muJ&h)!1fO7?>Fv2$-3GBRX6eCT7LJd8qacrrZGdxtrfSPeY=wk*DZx
zN7W)V>63MoH7(!W(uD;p7!$d4YQMCyD<zorbYOBDJPfUM%=Sd{TU=;P<4r&F+oabN
z;D`cW>NUMyzM?C!rMPM5>6SBPM&tv`#+vj0ADrW(;@L@2;)Xpc@dF|%d$33AJUC7A
zt0Qy;m!9KXh!W?gM$GD*hg5biq|=q!`%dC00tAmHg!3%+i;h)?rH^0p?4ay9%HX5x
zSW`^!_Vmjv&Fm;2l;O<%Oq>UI`|8i~IA-UGXB#Q9s9XggiyDKR&<9iGvf!BQk@E6o
z=B=pl1d!T+h&fj9^OeI@73HHD$)MZ+p`oo<Y|qlas1@E6Ew#O->sW7ntf;5j-;0<d
z{rZ){(w}o%4<$XZjpU9T_Bdsec^pCTIKidy#59fEdbp~(W_x<sc{9(<AOD(OKr#-O
zvqoLh*JV4ZMFMd~Mqb+tWy_54j(4K6k}1-}JBMGB!6^Yo_h&*Ve6*frUX*Vje7pgK
zcUj<}g@!Ce{sKKdP0*Y?<)U;5#-vS=Cvq@=l0phODSfKDn!i~j+~Va_9S&U7mr>$I
zn!WZ3+^ppIB0lgpN$!c{{t!>Lkil1OTNJjUN_KBV3jC=*?*g-l=4?<r*v8y(_rzCU
z#210c2+*f1B{Iq{#>RPwi3!U%$2ixukV$<1J2Q!}gs$Cw*sop#zxt+(t?09^oJC`a
zm`3;Y93k0WKW-(m0b`gmMU|1_Sf6kw%%?rcd9d(#8$3xSzfZBX(C0)?@|?!e1qXCp
zJG=_W>P9}>45bsBOT6EFDtY`rI?Qjad{lf54l~Ta&_u8H%V$S#5pTns<%tapM7$PU
zS_Nhdwv=|5YZZ?4=LPb#e*6_MmwP-AM)D*UAhl_Q9YCwt;rR(F+3>aPAKxx|n{!IN
z<*Q*&xdS+~nesZoNR%Iy=PZ+)*z0twnMl^MjPg2Xbsd{NwsI?3G^YCJc;yJyZTr;e
z+lX6St>wr<^R9n!9#Jt{j*^`FuJ@3@<|(raYSbi$8a-0q-dh15_rEN13wRq8-%N^9
z>(rK+A)m8@St)SNNQ9i!=Rni*RN_#U(|J@+Qv2bjk_~ahD9GhPvc0k?7Fu&y=5(4g
z$K|{(ViTZwpY$_#A(uN`wqwCr2q3`iJ<;Gx*_tu@D)~;zLr`q72ow{sAY%sFi(&%e
zgZ3Dci8%+$S6I(b7#aUSL%E!pz6&GY9q2hYPz&j@c%oYvhha|4QGN^k);QjG-GT_U
zzk1PjU*z3R-+Z2SBe})kc#<J+J<82>+iGj1P7I8D#9lfJkVM|K7YauSa)bAfg(7{W
zgSgvmQMaf&49Cn`qnR0RTrizl)*1t%+ABt7ttb&s`~<oAAGfq5sA(ajI+xi{+6&pw
zV+&aG*>{+>dnoU?b|oBVEHOpaE3qRElUwUyxLmtGuUPbOG26=NoaEw563w4=72Q1p
zGvx_=x`WotxVX90)X{D@5Im+<52)jCE9uI8m%^(@ZBgC{F~Y+0)gSVwkR8fsxmjq_
zHfT;{qkPZk-h#u07m*<B?M6yc2-x7G=37B&udjxuN1odKeOd=SY-in};sesU+X*6}
z-mzvKEd&n{SHqms;SKuaY)E==hMV)cqY-F7<%7Jhzyt?r5$1GS4*Rt`PIMNXULCCh
z7%N=r@iH&YZ{Iz&a0!fIuCc-;C>Xm?zo&2V^M88S<d;zr(H*uL(r|{(+Gg>Q`oq?_
z%sO0j^u6n`l=KpHSfp`uya2Lar$~{Qf{as$u^{h0T%o+^XB?sLKDdaKWNt0j)L!8M
zUB|CE_(X$qe|~u^Qdes2ZrPg6E+C>ze8%@EGHuS|3o?x1jI{D`JeT8rOpzH}&NBV1
zdJB~BL~hfR^-k!S1CDh9-@1dI?yS7>)T)nBzV&eSJ$h7qO!u1N47*ZW12v>;;2rAp
z=T)%qe&`kiVM=sEWg<^dT+PQ{e}$ldLgNPs3TInk2Z$SrLTEhicWG^fixLJyH7<^P
zn%T<Tb^=d&mS?SB?d7RK0HZwV2gwUC_JrJwA_W4NvYR0Um#!}J5^;n_7~Rm;<&5%i
zp`{nGtP`Yi6voA9kYkyy<{K3^&*02rr`*qEddB306_el6i&thvc}&C6$nXu_Qhj~U
z)t_>#5Q`sBQW3$fuJZ`IlwaM5TX9ZX8=>@c1x$~%eCs9r)hdGK%{4pnOcmbYGgr#A
zOOzrkh_u-KQth2&gKBipCeNPiW3uYa8IY*Xxt;9!YW#G3$iE<XV(Db*IQSQ-Ex&Hg
zW+`0wSmUS_)cD<RK2}$O+f|?z*8m7S>!1-zSejE>?`H~}ppd2Loorxe9Fo+`>^K!Z
z><Wr;D+|kb;03<tLrpKL52f8uNZu$n^vlQ%<)<ZYa6^%N6^>t6#W@0H;^f?emB*$K
z3h9~kvg{@SaAGI`&TOYI_Kh|_a(?@DUG;0fc#833e*+6m@)<O7tF5%P5vzH^<I{p+
zB_-PNy$I4CN@wN;dZ2WsR}xlnc&v6G3!WMZ^#74{2IqRy#Ci<zU1IJccJKyc8`>)J
zRk~A{p${~j+W(xkW7-1G@}0uxoL@?tkrc!XYGa7xWI>O-3DcSXz)EX4!xvztzi5M@
zH#xn?Nw7}wjXM;Kc6>>Z{1aSLG#wwPFzTAF&FlVCiREq1pXyC^7<`8;i|tAIrNcO=
zI2A&J^n_`p)hu4F5W0e2=yZe-&PLpQ74RnHY7!9PP`&8%PJm@WRr^cxmX!#h+^?D!
z4<+$MznuthQ_&LH0h`Dpd?`3AU>EMY7f~69;qOHm9iN$RI7AiRyzn$~Z+){t%Yd7I
z;s6V5t`qVw46c$A6YQ0OT!wwm(&-(d?QCtwBI0hmu~Fu>bJ7-FgMq1)_j1O>7~iPw
zmCS0~O9gtz#u$+Ue)YsuI1|W<yMX$=Q&&;o_Jgep!Jd+p3XZn`vD1D=0b#-I_~lrk
zZwQPJUzBogTlpV%s19It02i|v_7Z25jS6{d?m!>eyn7iG!+m~fnz$y(@F1=UaMa|z
z_p82+#!+vCIagR~;!G&!i3;=)Zl`RupC}cbC_*4q>FSXVNJZ%jFYaH*bD<0{;$0_K
z1J;cYo0Z~uKE)>}m<R;@Q5f+z)f2*qJB?P!>XZ%oIY%W9KGw*sfK*MlKe3G4jV;8Y
zR}yIv4?)EnEgDYKFpZ(=={rmV!<if*wo4e}{L;Ckr~EAIv23Y(bm%8UxM=}mnK*#U
z6jEfQe;pprbc&8Ky7nM~z8il<Hw<B^#jpa!SzVU0P<^iCgQ&-fs8N7y1(5iuTi!#6
z34qx|Q$XS=B%!k(MP<cwbZZI_NO?s|1?x(G(=v<i-i1ll)<!!Rz>QjIZEBDRNd^>c
zf`ZA1M`2xA*lsV+u*^@H%|u+M)~_vHkaGO6d!gw1?$j0kw;=)&QJvaUJz5i^PB$lJ
z)2Ge@x}=5h1Q&VOMb+8>WsN>y4SmtkJXd9FgoHyBgb|pn&dTx>B;0_8g!A(G&XI|{
z3rKjJ21n5E-8Id+<P{E3@&MI%qEcd_P!E|Nx|<FLFdy`9LjCo!9km}@pY{xNu=8U3
zzj!Fzl~P%4zKg!pfEEiWhv*NS^w5dS+ZUM^uiHh$fO(ParGxVrDwQ=w0R@~f`|ju`
zOxuk%IDdlX<|C+;AWURS$JaKBnRPd13q@^oXTvuv1z$rDUaFThkJ`~-40V129SuMW
zuT6l~y#92wGqHf(%`+nNlT#FdtmKgF2`^`c>_iCa^z9r87Xp^=ySX2Tm>TAAz4uGA
zDq+^onll~iu75h(1+XHf|A2R&A;v!_4G-Q1SEIG8<yY`_GviVz)jiPLr=N6FUgVe0
zA%oYW5)(WajB{fC8mbPlm6ETyriWsvaenUU5j8R@GmBR+Z7dTHDorFiVN&!`D?81`
z>`u|@!+=@vI4B-<jP4XgJUzQaX2)=M3Z@l?>%%vc?$i`ujJXLSv2o;<O#4c9Pm#-T
znw5Tw=v8diMNzA#EZAH?DKI8^m2DBL<KMwz2Ghni1fN<a=nHb`jM*TjRfnrpUvY7a
z(;$Y97D{K<)WW<TwLxqKvryBn4wC?cBHFp)8v`|ROFaxFq7gJ~36Z0WZ(Wok&w~|j
z_D<Ej$_V-RkT`x}eUUOHQ=>dyCjU2*|0*bsZk%={|3FLxU`D(v7s7sDXwg!Ey3)Hk
zl#sYyCuZ`(9?B+B&u9wf9ec*9=dviysLCupuDxlnT$}aS${#K&ARbbnwV`a`cH=Z2
z9|f5M_jXD_u)F&ysmNI&MIo$C8)D(-&Lnry%V1uf+usUI!@10OZSMsK&IL$Q+un17
z3%*!us_Ygv6e$zwizMpqV)F0!H`&a$9q;KJ9Hn8-ED6~tD;9f-TwhD*w+iz|=9K}+
zE$uZg2Fd_Rl&7I;D3<|zI|(Ah3GOvC@OP(QKY+gv-SVz}l6MZaXl{DjWr7$Br0e>E
z35oy0YmUcNm^=@qP#q}UQEPy;zcR?`l=cmVVW^h%_GfnbM&NGjgGB@5_tmIi3>qn*
z_SJSTlMgJ2urGSn*rrq4J*drOb>v5iq}rm>LsMDbwGF3MBT>4JLpVjjEm@s~xZI{A
z?uI%U0v39M?mYD_zl~qvUZS+pMs>Gs*K01v=0;>RedscYG9Z#vss!(!o&IAx@YlB7
zU<yOzACHZKV^cxU$>61L9vifKnG@^2X2;<TyFDOhsd2Z|3latt27|c<JeX;qFl%3&
z!Q+L5jYY~3MuDq^@@e|X^~~D0w!1qW2sFC2w+I6ti%#;_DP92v&EZKb`>t^3365k`
ze&89$MmVEWPAt!%PhM6y{jzrbNmHZcv?03C$3yg~zZ5B*@Tpa`pwOhl1{snpt;Jgq
zR7vgz_EVc(7)C26<`;KS=J*T!sz72fl9)9<fkqx_FODYASgA#*w^fHZ6;_v=R3Qb4
z;cB}Ryaie2os^vV(@RG{V2LKXP5LP>B?`cR>cU}&YE>bnc$^wU2=IoxI7}-!S3qJ7
zv94zgygspW-M6!aR{jNbQJdRSg>4hWg6~TA+W}nej&7a_Y`MWAL{I#K18O%hQ%z9*
zUa<-R5kL)E?3Ni8NMWEs0zDRkZ3^%;g#gF68G1+#2_g?zQW6A8nJwCz=D8d0NQYFq
zoEM&+047;yE3JtH1zo4FEDhq_3KIiIeO=LfivmErLm1%&wd2|`@%|kw82^y8Fi54n
zEB?HH)@}H^7aC!&jR6mWa>8L$<3X3~pkI5_Hrk~1#O$?kk%>_e0Xo;`p|UlB!cR8a
zD@3SnHtGYaypC2lNjtm0gN87P2|BuqPbwVs%xUWEGU@)%jMe?3dcW%a|JbxSVA_`!
zX_Xk$9_irtu4eXK*w`-~3&I<$8oh1xqx`1BJlOIiGF%jEL^9i>VA?_=)SbV7c2e`~
zQHBitO~j2vC}6~*N&P04m&SA&zePO~7kGdMM5=XWNht~A(9DTZbtj!C)JZuq^zZ@@
zKZ<m)d*KFM*EvAS+XQ2}&pPvxA*MCDETB>kZ@X>fy8V<ZguULpt?AcplWs^=60fDG
z$y-kBR8R*U0Y@Q)$*6z(GwcJJ)7%zPHK#yJsfUQHVa}XVNW&K7zW)4RT4=<q2;aiK
z4Pi?^RTpJK8n!GgCA@Uvk`56L5tOMCVvX?1sd~(_AJ0wFRH1oHDA9p`d?((dHnp`U
z$14EdoKC-h89R6tY`GianNh(&%EQ}vJ6>tU1e};HUMrG?H83yI!Z^I+@NW<%J#mtW
z8t=%YLQR@#^j{%Lpxr)6*t;d?Xg4vKI~4XF6(=V}SqA|>%{aI$x<f7ub1^68p*x$3
zN*c7GX}QXj=o86$^2>-}eCXhsv4q_$&{CfyR{^pw<<TXq7zR2)Ld6*)OF^2yEe*aW
z!;X)0j+6$A&9NJ!Y;X;biv@;pP%N{o&&I1k?bDNc5LhoLpikP9MoBcH!ARYz3CTXD
z=7`~pB3P6c-MH_PKXJ-}*uLnkHAW0`ws$bzdA28xLhP873p*~_K%vq?*ynuD8*vqg
zj}@8O!nn4tEAfLTF}e93hF1kF8u|;*);Ir7@tRXC%B_>aRf8?`jOJtaAYy^7;AX$t
z-zNq+5&03~IMIx2n6C|Lr<0`9({h_szP5l@Ja2z=QU1<4#R@4$Vqm`ON6TgBbX`G(
z5#hK)ya`%`2lSqi3XdNuysBiN_xSuN{|EUmBPLCpcjrKqWewa?a=VtKWS08GSqt#)
z0cju2yZ<R?SoN{BG5B*lQ34|kCzvE4!WO?2sDemDbHu`lYuKdY@~6av2H%(Ex_iDZ
z{=5{|D=hZF>a0B5<iX+Fju;YZnxxp{))3h|NkfcQ>?zwhfWnA5_E*R^z1>8P06)un
z*I-2j8C66Xe9(%Z^&F<wdEdwt$==I_H!xbEX)K%At{`Ur_^dqB(%K>7!&p5(6#jmq
zh0u2YwVW+!k#fwg0d|et_a25V7%4|AUJqF5j*x1@!<zUEi)bV?ghL2E@XPOcZkgG%
z^JB}I#II$mZQq>2+Zhbej=m>nQHKJ)C1=IWaN7A;_f`z=fIF(!iA`G{80I8|0x86P
zN{2V9@4lB=O03#ifD{#$|7ToWe^X6|&<?*?sbT8aKf7V2OjP37Ke}Nv93+-mD0~R2
ztOR_FGYoSchT=2**kehW{?N~?lWc01nO4Dm(@!=5+hwRx_tGc|@IUU_SNFf}0!*38
z=N!e+22#kl9BtT`Fj>Jj<@BY&xfKe&_fd0#U?2n*ZnxK2b~Vj|BOLUHx;^pBBo=;G
zxUaq8mv327tbi;u?&?ku%eli_aKT3rmToUV-1AYROJ~O)U7ArBdR+TzeaYDt(yc%N
zLVUB6Xb(^wx(D70)sgc3(h6mvSwyaoY;BeqF8?SDt#9}*-<xUU7giS4U)ppKo}rF(
zSq6+YVgAQf!pQ%AW%(HWx?38#IpAd8qBfY|QgiIG>D29WA2nQ7B=#5CJ^!qfU_oF@
z1C;O|{^i1PC>D&Pna7{Xn&BufI1qV*AZOa2qw6Um1ObB2T{N0h0xytVbg6~HvSCbc
zh?5NwK9k7qL*?-@ULVEd1sCeG_`y5unY1uIxj4vU2)f?(KuP~5K%n*jlnjLb>J35(
zq1W4dR=|eHDL@eA#p-<VFvReSkV+jr*h1EL*(GvORDMngQyQ6+34*t8;-7pNk63@r
zuOnX5Xq%=nN<{0m^d!6PZ&`UexZoP%5d?sq1GKFmEd%<l$$giSG3mQA93kwgbXP+3
zOZC^eZCTs&%*`;T5}qD@Zi?=$=`aeCe|eD8W31sHY6N(0nY5?9!?SNg`+t3M4bHvy
zi}$xhbqs!h99xASdlrr^B(8yoWfz9AND;)J9#y<gIx&!%d+CuTyN731amiQ1@<Co!
zpm}f31dZ&7hX6jqJRNGTBIKy@zjK~dzm%FP$D4KJu-_Nqg~w+Ixo2r=Z^@z2kjn*T
z)41(7U>K4Hm!&3+g3I1YUXbaPT+nY<6!Sbm#FR)%(>Iq?cc<J*O6L)>7SwP2FTG0V
zJ7CR&WtR1JQSsGFF}YdbF-xVyKH)Ax^=E~F3yW4J3~^m1qe)o5=i_P-lPoVJZVC&N
z6IL+oF5AWQ_cKNCE2yW6EIUQzQUNOm*k)}!Tv2pXT<E7|o-jm!+Fw_&7Ev}O(h0R|
z@E5wnA?)oa?M$lhWWFN2Xw=o56jx&OYLKz0C{D}G(P-AHEyCihHvF|M|KKneXH5Zx
zkcukylsBs5&a=)OI&lal?Ud~(@`P9mau0fcVeKEIa37lFz`rO(H%JtU>B3p+=Kfld
zgSH3L%hS`QA}4{+=6W_5loclYF6<eBpxwlJt?QEN4@P)@|M_*@a-VWt=ax2_60*o&
z7HH&Vco^zDf+B)k+u7T%k_w+tZYviZZF+YSuFJukOQwXF?k`1s&plC;=M;iz4apH=
z9Ly#oW+Na=!itQkfU*7TOdHK;K@<dnSSjA1L&v!L9$eHFWzfvT@Xj`7yA6vF-~IlI
zHY7O8)r^p@S%7@44`MJE_Eku__ITMMQlzLR24?7q`K%jH3V|D?u^I`GgCg4B?1{(c
zXD$czQO!GWAi{csUOmu01jJ0wb?S#<{>l-GYNh3$+@AAZc``g6J$`-X`|)c7weyl)
z)bVS+l>#UELS9!Y_jM>>IKqvLAk)^Z+~StWL*=KS@D7zO3WpFkn7D0b#NSXjBx90W
zDYZP+-z>h0T|+((1A^S{BJ2<@#LM7@D&2mSz&8Fw>v#G9Ihx5D7h5(m_1w(q2wN&I
z9A`!>6Jc^{XoH|J#J;7yLU5n9D^q49k)g2i#^Qcb<pr)CSp+>KrX<hH=XwP~01b`P
zkWPcBaR&n8KnoPS(q48`F>_>aH)lJ%vSV*5mD?PORt0$iPpL;B*CwO07i!)ijw<ab
zLIFPonShCUox`9ng~1K*;CBpO3h0a6Dj*it73o2hIZnDU9qyRiREm{@k8%g@gPAJ3
zmtMtEHkCh|njC`XY2QoyOah*XFJi@neJMVb&BGo}^Lw8<$!XH2%BIvWoBT8re%k<&
z7QY&QB!?Mj_KJyg)hdwRG-cd}=y5UMH5j)E2+_l}#dp*6-|-DFr?<+mwqORFi?&kF
z@*IXv-OPO0@j(7Y;pYHcF$Fz@iJG4OknI?2G?3EtL^Ms{_s|Ov-nf!m*gZs+!-JI2
z?SB*@An91h<}oj6$@q`*yD{^efD=_brIK}@!wls~L3_xb*&__F?F6Aye%-YDiYzMt
z3DKigsM3gM9zSwW8Ou`w@~M<=AdrlxTF|EE0LVwG7SN-BY4!5U;B%6ip26wuD(*DV
zHs1^j4aeuepxN>Qu2CGAz@zgX>-i7VH+i_=q{|Y?`mEbd5@}~g$-V!>v!hB(K!VC#
zx)q4@)n!4I+#VuJ;qVRvEmF!`1fWqnJ}`P3g*`mN*ZZPP*}XhJv=hF{;u{yTa-NL)
z9~*d;xzxk;j82)US@#|Y_^WmMlSte^sW9iDuT-XVU&huW`HoHz5pYV7KCKL;-}&-o
zN){zicsncgi=#OsRQBzE<45Nr`9PF>UnfeR(P83{)s3%?dh2B)RHt|s!D>~Kz$zJb
zEu&zIp;2<!vjBQI!=vOM15P}eTWU%w%n9-I{Mg>|EXsmdXZS+0c{JmI!bgSqu^H9N
zSvoo%nw@{FM4ddQAG)Gf*OmshnD>4KJ}a(@tGrVs+sm3_CGJJfd&H;ro(H3SqY915
zYG7pQFQtc+>n*C9;$4iTMX}@{R<4u6qr1G*c#2b$l|e%lK;@#z%AoP<^Pn^dL-RDH
zNpPb)^92mg^;*d4FyaVJLf~eAC~2a7Y4hA3`Oz(2%Dl$}+={@X)>XQ<OdbKLDM!<^
zr>2>hYZxurH<~3m4G&5U*&yLD7Q1P@h>tWp-h)Un8cU*z6axYU8#Z4DRdzc`m-l6d
z`K#s#xn9a=C?vmvHhsCb9Skr|qZ)?SxMd*ng(hCp=xkOThi*BJAua3P=*RdIw2#ne
z=x~%kDK|@VGG+R4Cv`Gdd7cwAnyiVd#9|@ZzXjHny%?^W5!}vQo&HAxWE)6r%0-u2
z3)6aPl<AMjrmZzkKU3{&UJ~C^yfQmS%M>vE1bq4|zFNMq8l(S4cLgjP?y3DF4@cD@
z2t82*NfuIvXdxu3MK@$-R~oB;3Q|<I7Y^Q}i<DysS&lJ91Sv8Pjkio@V58bS$s@-B
z9S1>8*|$R`W-j5Y<^3;AT%`TAi=u@E$+vd+0?MEnWzLD>UKiV7i|`2mO$jIhFQ^(L
zjrTCzk>$s3o{y0R*)H?Vj2PAy^{HYwT|9sI{RIkVPnDy)Bf?H90I3p75MllOq}iAa
z(g}*Hk&!`nq<%FfC2y!hLmfImXXvL)nHe1WIQ64@K+vjgQMQ*k9A%_bIoXW@F}m8P
z+_^eMX#h<~P@9?!%6NsNYCog?cD+`*WW#9BGdyiJdG0n1&x0PLyO{{m7#ibULb36t
zQirjpT(5eKu7^mEK~BtbAei9Ts`VDZ-tL>zuui|eIn)SK7a3m-4gE3MB{L}{w-l91
z{9R@g@&@{2u{A}r;yhJzc42`_n0~Z(tid^xIO-7AR}giO6A-b?cZb#mndru&*eh6i
zU}P$}hBj0VjKgnn4a!jY^KF|P;+4u41eWg-j;SqS!GBf4U$fFUIyS5FXXpYVZrGTi
zF(<V$vGJ?@drY>MDcsI(DO`Au1OoW81zE?@BBNGMwaM@9f6b?8k)i6TBIsF9Fv+v1
zh^yQy-hx~$2<7qUxIgo9awzU2DH5p!ina>+BdCy3^nhr(?*}4kFEO|HgHjIp6rrz9
z>}>bzELdVw)M?UsTVHUhf`>VrZM?qw^`n-R#r3Ni8D&Lrcw-A$gg^lBe`DW>%37G7
zTRT>YwIwsCQbj4I%^}xIq#2&zM}Co(kYg|~m@ZfvKcZXlO;?nvTQO?fT6v!1*5&i#
zd^@Bu{!!*}{iO-HRncw_MklF1MbvSOm-v3PbmeAczj$T)l)*sg0F&IH$=yvh>wb=o
zv80nCd*mBIE}Dj&XPkc%|9ql9qrn*WCD+rlq8$KnBHob>6B|D!U2g(>>N}mBkzt;p
z`GR3W6m>csL8T`{Xo~2RjC{?>HN#p!npNtVjT@PaH4#K*o>rIZWe6{_(LHfz^aWO(
zvT>BwzMBXMJZfj9(~$6qOn!Sz&lKRrLGH?Qw;lfrUo%>vv-5$8TYJBoU@2Y++KGmo
zDg)hQjqZEsB+KDJHgIDd^`i?{Pf-*Qxtw$^*OgxVuZvdg$n`SsCb3t(b7T8FCT(xV
zL^@HTic;`rg)l*%PMM%z@c3Ln%FqP;g9Z1udVAZYSdg9F3B3f2bTq(kFM-IddY-i<
zVjQ*?7qq#9bw)9T<W3(2m7K5k(m6SLrjaoIQfSyD>8LR#neAr;i!nuHv**0MmzyG3
z8dRAId>UzYk#cY~X^{=L26KOB>X9D_h2Q9_P?k$XjUoy`C}fOQ?{3jdDB6n7t%bCL
zAvZkwKYM$07w#b8%73WGuH-o>Dqu^%_I2FC<VK=11M#0OX)h42(5}|wpdQ@NKsgBi
z3=Nde1!>ixfpU;mMa5IuN1L8t%d#d`R}5!v89o=fi~r-3+xJ@16j;EXoVpgov>Y!j
zp}Nv@<<OoHPOUr!qOl{)guUZSQ#CqA-#_tCPeDQX&jcaiwAvz&{vkH9isrxh=|ONJ
z%~&u^D4qx$U;!_aEMZ(uQ;dsYjz~9<?{z`ZVNe|M(1la95X!;h;XzW|&qW1<Girh9
zOyt%ZkgOYX=xpXv(TO!iudn0t?FtDgi|^<T1b->*DUuJ5Zau$9V&d<>Q4i6Nww-D*
zh|GHtWT7G;oP~_TiW%@*`i(u{IdcUB(7?ZKR(#_ufx7#fJj4P`PJzST0$49dvEOI`
z|6WYDUsYR+wdK)NRpI1MZ)4TOvhrj6V_+<#y9{43_ra&18we&=Yu?E8V5>c#zvNDJ
zaz18{MP1=rFAa$0GDxraKx!o$3Uf$@a<bNa&}rXf5Ll~D2ow*?SL~-D_rx;X9+rJe
zf5O$INf%OO#SzE-oBjk6;2%y`eiP5W_x3%60)bpFb+AnzMj_w1vn>*RJspU``EPJ5
zf0R^r;os&MHuSs>EAI;wy6zV?(_RJDdsEl;y$-%yuOp|ez}*;M1tgP>mAPJjQEVht
zmv>9}b1R8eAWUtFJ$t}KtH8$2!LX%D{U9i+<ekjB)02OuK^T;nOj+M|<BbPfuf~ZD
zw%2_u@F(oqg}sTMpN^UNU`zw=9MGXYze<&=mp;kd3nbJG#cWZo8XU2maqZ4Mv8)I4
z1OM~m`$p*Jpc>z%R%4{7c_@tIPf#%G^y3a1{0JbS*q8I8wZz7)+9_;=AgJ0w#K=1*
z=TZYg2g{RWb`IX2=}cu<O_L%+F@{L`H)@)o{%>E}*9sM1p_r*=FAF9&1XQT%rEslL
z1DRl;vEF1e$fL9h;pl9OGF!_Eyzg5!){!t_hDyaErr36C*27DYfdBVOM$-umL%d<y
z%r2~ES%k#4sIvvDRnhRp_+ndjRT>xT-#EJ)dwYn)LCo96kLF$sreU12oz2$eH$$Dj
z&=p!XG%30oK{iIt3sc*P7AUCEHVr_B{zruThgf|yA%6t>fs`M0>MURu8e5w{%8y6K
zFAJ;J)gY|Ww@TnJ-n_deQ>ZTB6HqPcptEb~fy6Y&Zk>H6<sTMsZ%gVKpPnvJ9ivc*
zxImD;aYVYa|GH8b8o8*Z3dW9-VV;6sD6?RG?ZE4-vbJ_Y9gBu^5M0m)QY!Mq%g|H8
z=fGapaq1wda>GCBptGEX{%>=PE+ERiE*k3_6VRn;z>USq`wq|@qSMl5tP!k<B+~4l
zGMHQ<Eg355I^i=q@zr4%>E~csEb0kfo!*%VCZBoH^0i3|?qn3fQRv=^_hTk2G}*T)
zt(F+?MnWk!xws^cfuDE}2>=%JHz!pRyHwtK$wIKZ?4AgI^FCxO8DZle+E~RavnuE3
zh};LBEqOYiOV}givLgF`Y~Uq7@wU#^6iCU+2W(g#OM8n1BBvr?@eU7(G|`gD{}9jE
z`%lKAQY81v&rz;7CpDS78X=OJG83#%87rC($%9Twt$%0^2&4(MhNTHNGd(#JIjJ<^
zyt5dyQ$0{85QQ<h&)`-TOX2GI#4h$%^fTG!uV<bHsA3Ydg8O-yhi5CCU3UVzAPbDZ
zj_FSRwRG*&6taIy%?W?%p3<WXBLw|RbxrHIy+nNB>;eQp2)uyx*ez8GlWAx&9&I2X
z;BKJbC_6Jo1o{DPM>Wu;pv{ElJ;K73zO*(9v?<06=|3otJ#7iC#e4F7NPUolnWuod
ztM~8N{g8+i=t3f92=Y>5scqrv0&q2voO|as*zdu+5_HZ}sPhs3)Hyd}o^$fIWr~5>
zSXpvS(`|}>8HBxq;d94K$o{>y^3=4m)H)>>L>qMEL?sO1TEjG?{ec0ym4EP6g#dD%
zuGJT^bl?!DBySM%Ytp^hZ$1WGYR+l8=c=PS5MVs$<BTB;?qvAjQZ$j5nMgfzcta1k
z(H~aNtZH@J(HW)O{l>fftv5LrLo<^y3j#_bPSY_n@aUWangzw4o*jA3_9|V$DoL{<
z)W<9shFr`<vV_A6LH>;_A&<FV2UN1X>blq}38>OA(tdD^&-lT8&Uf(ViknWcE*Dsc
zm4#kdoo~s>T#FJ8gM->~WA-wqqMQkI9HhMbuaq-4-tcZ<H6}Tt7?{GMb6eD+`xInl
zE;{)uiQc;`UUWakj6S}}v;dUtu56t%|9Jrci{VJPg#8-WqEh>J@$X}{HOJ5x8J@WQ
z=Z36d#{Q28EHB%e-`M*M3idWgs~N&LP5_Jfo_k_=ccb5;kEF~Zlj2a8X}}Wt5&a2z
z%@$!{%CS&0{0`bPp<_0S{xkcS_bnwlK!nF*x_6?pGlO(fEDgPzG3n<Z4D-)P7by5{
z26EYgj$~I}^Rtq2Qk>w^NXdy@=Y+C@^x_nKhY?ygWYg2t=KJmNyGtL-6AUy>8fiGh
zA%JgB=t5)<0s(h&`jQCsUDL3S3G7>WhTPW_?y4g#YD`$9cS>9C-|6%HOVQemjE@?^
z7onceW2DU6TV%MU&O@PJN{okDQbm!tR{VoY!6@;;@%rpT75mKqE?1nGMWvkzk8qnW
zz|>_pPTK;Pq#q3pGa{^<%=|#Ps~Tpk@qM-BuLa(E?SuUKhLlP?DbR0GFise3ESj*r
zH)_QoYVTdls<SR2Uo`q65f&K5k!z?JWHeD(G@&&1wHc7co@aI5tU$#w#r{1<u9q0t
z_Ed`fxy9!pmommavoSdYe`w1k5bg15$Sw|VPxCblA20rJ(3=r0UY;9%u1YbcbtPzX
zBBr}3L#QrU8&b1d;4&@Swf1VeL3@EVw=?n9JyfduM7}LcFuQE@6&)MFUNH1b;6$nC
zPZz{~t+sO^UK6<2<vnOAq@jki7m%<@{Rrf@j-F8WwS^(?Id)w{oBJ3w5D{>~=Sp!X
zwkM*jsnk^i>Y|eBck=c>>^;8`ZM8-#1*b3Cs>aL}i?{4*h{M&}t8`w*pYQ$vC%At|
z<cEl5l~Fh|PUn)k8RFZ{<xT%Xt;t^r9-aevO<$Xd<w-9iif6`oB2h_2?rAHwCkx8|
ziG)psxp9^m{&>*V;=@+K?vh7P9W^%!`9wY+2WuItY+rWlR#k#0_kF5$*zznGfBYBL
zy%itM%6`x~fP0Q}KZg~9FdV4c_o*m<bDhTmJcPZdV~&L+I@=E}@5k|)-;D??daQiZ
z#^=JLOI}`{gOtM~)WW)-nYo>ie*Vzy+{op_fKY02uh?A!*NdFMJVS2X{;Np@gu2`b
zL$Q7A*Ec3Ap(@WqNa${A>V((l9oLh>svP+5!rnL3<R7@4$K|&R5C&Saym12yp~bfc
zn+aD_ClX+4e~1r*_LCSmLZgSZvmc@4z5DP_`Geew#Nwhp?(E>Q#2%c!dC;YQ6ZyVf
zlSmyi0J`oJc0$l3aMrr4$RZ~KDY=WA{3SuH`#SiSQcb9Bb&0;Pzl7XTbfY+wN@^Kn
zRNx<5=MaDXxV$3l^_az9&LKh$VmF21KwQ?W$h$vW9l<yQ3}%hmJ^KSnydd+oj9jB$
zj<gB+G;&=%40DoD&G2c+xT&nv1LG?HkXIxq4Z0M~d(r3Bn^NBQo>i6ehl_T|%qVr=
zh&Jdnt|(0;Z6i#lwnE;yklY09c{w)`1p`ot*$o_4vJYHSd=*qRhy^(%138y)`|iEE
zbIseFE7``i-8I!h$!7w7B*0H1D>-$cVdi5BI@0BXm9B#guDH4~9&9i?z!nc$ZjHTj
z?qZh45b`ovO?$?9e%kwMaoz8XvyWW``#1@>)wZ9GPOPz?^LmV(!80ffx^Itm%%Bw~
z3_dtXi(1q{D|)L?okyH1QW1j@kLEZ*zeufVP$X1Yxnear5{-?Rv=TXUkL_kX$2lQ%
zy-RSpjwT(thNJhePOkAY`Q9g*MBzNTixL;$*;#Q45G<PuU~38Jz&XJK<a*K2R%cn$
znWUP|8==leLkK5b_z1vK_4)To*r1gf97$iOG*4LhIbj@3!rhFFutu78qnCG2Dh3A&
z=BB8v{CZu)rp^lWTzq9}y%Q$)E|@2h3=SsotJWXdtA+?XlWl<j2a^o#j>0b%=Z${T
zN<_^tvLI|UpYR2?9(@rcHN3a`ceHo~cCt(=RXBx}EHtUY<iCqg#27T@Fu<r>-veiE
zOCpXB>wuo&5+sb>LBV+UMX3KijUpa2XpQU#@{8oajlw?5{Y+ubsuN-KtM}ydqZAAl
z%>NX5Uj5_Rb_1UZ6bL4y{KO`A%Z#Q~chR(}R{vGfNr-~wIDYgDC)IF}=GXx;sK9x-
zel@&4*t07+0=G+eA%A%R2szw^4Xz~<7V##2+O&gQ%k?u3@?mjkZ<#aT^hHhri4D~n
zxsGs8l??J11r6SWSh0?wz$JSm9W|wGLk3;?ufE@{(1VFp$}<!p>-Q%w#j6^v*NUW6
z@(-K!Cq@Z#!|kar@#hE;GoMzVCm^*ukFt(j-0P;IJ8+OTu)xcn=1Hd{xeo+ZX7O7s
z#eFae*x7XOlNL<n_Smytz$#AcX!x{g3+53^6RFx3iq4>@=5ICy7rUKgb<6q|*i=HH
zI|vHW7*xpjYe{MsP1tiErQZOBJdeYQ&k?s0T33;~nj}AIK@%76bU@oNW#a4kMIR|s
z{WLM#d32f){V!eZ(O%$OFJ=gXu*Jiw5vK|Y=j1&sCl&aoQRv=&e8^av7pjS8VJFqX
zU}bI%x8Cz0?Fm!u3b(YY`;pQzRrD3v8e%bng0>rPKQE|fQ0cxb>z&x~q3Hs;nXla~
zRhy%Zu|F3P{1FTF2?`d~O^TEwv>+D!sZcV>{dA-oC5pNZ#=|0JQZDWKqln`8*3gi|
z_D)af2O?V`7j;zw|B|2vaBq*^ac#E=FKH-kvlWonhAydic)wp!#*3n0Q7j?E5H`3?
z=LBJNzPorP6&aI1$xBhTC+Gs8-3wb*?ZAVo7kB!Ww`+pnoh|Drh0Wt!(y@5t=#n<S
zr)V-_QZ$et9Kq8*GGb!U*F7DANYZIYFB|PUo>9RxSTU3&4O>>(6v2rmAj-9Pw%1~K
zjaktR3-E(0CE1^U7MkF$j%(=MGk86t-6%{D64HTwI-Q$}1{r7erdCGe4W<JNO+dx8
z{}`@YD;+>z2?!Gw3n2_<jD$fGS{Q_$^;h5TP#Ibn)U2>q`CWJoe+lS<lh-kn8cY%b
z_b?#=pu}U8=~BpRW>hzO<LwaVCKLkWEovKty?0`FK(ZG6MoNXX)B{i%yD*fjQBWCL
zLw*)4mm5-a-#bdto$&!mWv=N%DzmGjOfw2om!g5haL8|#L`@b49sgNV%%pDUJUXpS
z{XNhrbAta=On`}o*Xou;Czyo_Uyp`cn&iJ9U(7l?R2oh`7f&fFBr5E2XN~r2y{Pjh
z7jeRTz!9dHTAqn`F$=v+S^u#UcxE5tS<Y-RxtLw7aGd5HO_%?MkdYP+>SDss8zj^{
zEe=K0m?;(a0Vk3K<u4WHnN${gyL&F-#tT>$#91z}Ej9Q?pb^m~p&T?A?t13JW&L(#
z=gi5dSto70{nELH5CZzI@8K-H`LQcdWx=b{UsWM+jGZyjY4kBpp~6pn?-S>e*WGl_
zq%X+4pyr&=7nfqtiKA_}AH|lWaMPOi`eGrgL2+W#*0tnG>-%91obmTo(-waKDP$Q=
zw>H%?w^4*VApX{i4m>ILtvkuSOk9#(QGm%ml*${Ks%@WGj4^euvk1>h2>rh@#rp1?
zP{8+RUr%*xsihuN_wFH<8TC?O9D8$w>gE@BCKVTYE)3vung*o;rQ7mMK&jwRwv;V1
zBIgT!b;%s(<fsCFKl|i7nqR`p8)yMaMPk=8m>9b+GN=z+>`xHsaVC(Iih0=waNU}&
zjXTQugtsbN0+}~1RV`{6h~SJUY^w3?LNoiUJLs#`<aHRfedhbAe$;M*c2JyYoo$Gy
zMNhLdknh~W>O|2Ae(t2RN`g}al~ZKl)JOc))^dev3Jo?F$+fk|^+)RBmZp7v07_dU
z0<a_gg{I*9ll{xwh=^urpVdOBcU89N&k6As=|HF0>c`GJ-~eSW9WV;4rfXu1g{qN!
z{J<F<TnWaoBFJuHk>a0h;*E%iI(7&Utjb7lWAHhOUSV}2=pf61l&$LqQJ)khGZgi2
zCL)mS!4+WO&8JUza?fl?0S^>^{FkmweS41zL|rO0AXXNcqJwNo;I4p`8{ep*Bc&}=
zEe#89{_OV{&J_nC0@qcW+x&X_nO}Y(cOE&o?-Vz#N&+p!;z0IpE9%@w&g^{Ch_k1~
z4eA64untj@v-~!Q#CGK+QP<WCuMPxts_Q7piKz6>K6cx++$oAfB^|W?zln?A2^aUg
zt=@~$(ow6BXR_3&GEq3pZdruH3^_2lCQxUbj%-+@OnwblR>XnocVr^JR|}`k#eX{^
zfr8>khaP2OYtvhkKC0#>>|OlE^?WDpD*|=KPm_nGcjBpVNg)r0rnn9!!KvaecAKX`
z{$}l-i0^p`+CRpE0g>CsY5k5xv!{d)Y#Fw>TuQ~Hs_t!$y(>z_Nz4r8Y;P!GWHVZK
zGL}&6&%U?mToD9|B3Ttk$bZL%RzogcC8feVP}N|SG?b){e@hoqmn%UkK730Fpc>HD
zRAUmOWYJwFpx3D_GqKQb3&0xxz`kd)=I>lg|M^7kyR0&O$OEEqUlhtoh8XF~=)xv8
zboP0R0Qt~Amwx=BYK-}RDz>=C0VHOdnoMFq3rE)q<htQO{zHKC8`b8Y>1NUR9Z%E8
z*ZBDU&d-$UU(7r;-`Fy{$<zUI@=bsNB*_{A6>S`VLmpG5dNW92lwJkQE}?hX-D^$}
zHg<sf+Ki3E{68?W)kqH4n3-=(PN%jMGaz0@U3X86?xu8v`?V%8^Pj5*hj5&<#zF74
zjIjLb+Q{~%8c~Ld9KI1+Ax$mz8`=AxnYp4#E?7M-lR$=+a2K$LK4L8gIy9v9;nH$7
z_;E8Oo=2FYAj@fm71M=t+K5*R0>e6xouvu*%q7u<|7$~=v}MFX4K$-o;eGa)_rv?-
z4XYD?Xz5}OMPx4GNj*Z@0q-<10q07KLbI@?HMJn{P}bvV|0=-?(jeO2c;O+b|51v7
zKmC378{Hvys1vz~R<ih-HF%@$npB7>e@i)*cYbNGis<<!U#BctoIo``Xxnmx&jqm(
zEUR@rJ=bwZIhQ!~7&7XVeWyD<T5vQL!^S{sRoz{&95q%8rz7S}_pKNpOfMHx$0sRT
zf@su>8HW`S>kB3P39P;>uim@H1Aaw(EI_<T;um341pUeZE=opw(U!6rZK`H&Cnyhk
z6m4xcKrkdjE({b*tONY-q9%2g;hWiP3!1p&IlT^~P;z~VV7mxnhHaps<@VraAG(X0
z<oh23R;>i<83lTHn^szz8YCtY$OlymO2?x;o>;u{ZO$Evs}(cj2ld47LMNPE&9GM9
z_fBYLMS#`jwnSK-uWg7^!NTLr*T}s99%xQ^Cp)94rP&_AiWkMd58*pg7__G#b<Pde
z0%i9-H=^s>V>zG&%8JAHHC^U=9nvw*F@@PJGjzhmr^EQi6wYy9v@NLxx{d=GfdXZ1
ze!1q2CAgNRRdAex_@+{e`yQrBby(RYOP0L1^5L8x+N8WCZpZFRvX~RnX#rn(t}PN@
z^}g%Ej+OeG-Hycay?MfL{BiG#mi#RT^4ui%`^Vn&+2-8(;_+G^hbp?Go9QHDch3*5
zmdjz&myx@?+&Zkxf41?RXJ2!lY_lML4K62;63Yv_Vgpu1U+GjF=opk?_sPAvg9~^1
zLrB8DMS<%3mOpZ`(h`04%j25#5*<5U?pK?H>gPnCFp_YvN{YOTuldBBd?H-R!7V9h
zw&I=Y^B-#myN;9e%w0;AVj?=oxAdd$Wh6!X{d5V>-Ue$4hX+Xucg?2X@;Rf%ggG>b
zuDqabpZ%3tM_CRSY|P$<J_+i~;QhVVcb-JeVh^9guYU2pD)+aQ<~o@Ts#ijn@C#|#
z)O_G!)-&79pq;nj!Wqx~b8D`YY=5T4Q?W7lki!!>Uz?;WOB<Y0&b5~Betzei-p$qY
zbce+&SLk2|br*@^wi#D+Mbwp>ciOedVs6~HA$l@Afxzh#RO@8+C)a|2xt^?lbsU+)
zyh*&wNaD8XtQ{HqxL-MkDj2XbMmi5{N=ser;-hWwmQiPU#Z6Z6upNXb+x8GUX{W;S
z4o5GGs;c5Xa<-u2&6*1NKR(Q5e%P73Nj$s4X7wz(do6z%>|reWc$M(A;~}!jex1-=
z4QKSpnf@B)RRej~HyO-5wu=21IyPy#u(b?@`;qbL8?+2QF=in1IxjI5KM~J+zT0|n
z`m89q6Dg{%jws$@JrZ_wq17yoJPzBm3jF0iH`rDz+r>z4uEE^$Ii^IJTx?Aj5^~Rx
z&0_z6^Q!ghebyXW1S&D_eT!q+&SYKN7P{9Vw9jYu?YkRv<2I$n|KeKnaAn2OA}%2&
z`jVrhyXVf$m3YrDq(L7wU+)Oh6FHYfugbg~a>3k%;yIQ}s`a+sr5#P)0$0{uRxB5P
zWjr41l+niXmjXj#V6GLikKd-|j|ZpGC-MzW=l*JySn86#PVf*q^?z?DD>2)eE)93K
z?IRZ7Q2+k1_Zpm(&5axN`nT9j+g6GzL|}~_0@M#Jms=}dWG*x7aK@de&%d2hz9M1p
zZh@X}lM5?@mG&lF#!KGozUSRp`K%P?L~mrs+n&YXqbF_d6HjvaSk|sZUv*+V?Ld52
z?hv|2FMtmB5{FZ@T6!dtxvVz>y13em8FzLDqEEu}IMUK9TsLpOa{%rid<FCx`<Q~Q
z4}!^=hd<T4SatU#oL)l%YxohzCvunPult)$C+rDaT&L(2lv_bhckGEYlVcLD@x@`L
zq^%d38Eq1D^~o<EFCaQ5tR=}af@e5&2-g~<!mQ0z{@Au4)aT~A3>kivQ&(gRYPja{
z!UJ8;knoFZW=pz0&!;o(uB<EvOl}%w@bX2U2HV$W$C__xxs`jbu>9;`Qy~wN>ujOu
z6~7@X67(`;_@W=7D=O^9y68aU<%+oeJ?QmbIQIv7U+5EOQK7^1_?>EYtk+H=gLm>Q
zFwo=Lh|ZWld9!Wy-yZPlj!D-&lrr0x9dL5)RaQOsO>iweUPq*(?+707dJ6*+Mv{W<
zF1Yn~=+s`+KBm)^`-E%C5ix$wovLEJZvFb$U?1DW+6)<7pIE(2#Pg@?FB#m+Ub$TL
z9;=DOenzud!Rg?Pd_#q4kJC1Fu@^kfcZ@B|D(<4k$^LgjcJC?EL9WkTUqaMbby9!m
zS?)4m)baSSLkt$YvqNYPJvk_4wa~Br=`VTi>oqwtc-vbKrdgU?AioT;$CLISJQyD6
zL&}!E9qEEQ<B0C+Bd5n&iBG}bzEb$@EL_t=4&aEKKi{cVeGyc{Ewt07b(yx4m4==_
zJXgU7m;2s%IOaMRmgfcL;5Jn2@ooH(tBPD@^^Y9o7G}LSc9$|Ld3$x&e5AZ<n|1j*
z!X%P)i38qJjr`KJIp*>pXLU&Y;w4KqZM*4UX%rqYATJW&vi$V*V>ItWr+efEINIex
zU*<%gG7_f*mU3EPVS13{Ia~2A3M>aY9eV#l=V@UX!peWZVCc@7sX3)+{BaC<6yx88
z*le^KUm16q9*2mLm-;vIkmwUm3xnH?mKTy&3(*Tu{^>NGkNve-vL1i9va(r`KdRZo
z#eWR}?Ca+R3=U+;mRF5<Q-YE|o`>{c!IC9+!h9sM#Jir#pE<Q;w@~JnTl0C2GR1FN
zN1yPEY|Z_Z6(@?;P#mnK6V1V%r#A)%>+o34b<u~i^4robaGLTV+tafDzRjw*zz*F$
zWvbsmHhD@q*sd$t-Ku24M)5Yh8zC&5K%X4-X5C<=^5Df`ha~%x=r3MvPp8G+qPgzq
zD+5;|DB@|w-vH0>*t%a0UTM3a*Z5oEdi4Kztomv>N8YB9P$IP6rm6QY9des7SCiP8
z@-Xrz!|IR-ILYiM=L2JX#J_Po6*R|_261G_@wkKI`SWIE#y-C2li+w*!SST;c#X+w
z)NN}Kz#nAtQ0p0al~m6G#!enAr~N89k5#o%e9w0mmg(j?^F?*tA?ym#&AmMf_i&K5
z!5O}nywzV2aXzqnk?;go>SSEy`E<H{3@ht*yzvJs^@EJVScOtrhK`}ZT8hbiozT=$
zAk7-)QbF`j7rwnVbPw!bWT^3N^oL-PpD%RYiS0<(*Jb<HGL&i@p<K|{)psB(dM3kD
zadT<IU0N0dQHXWLf%_x_2q7w~2P`bQs)8EJ!!ccAILm>q<&YMfy1C<Y<NGtsy3}Yn
zjt9C6emby5K#1Sw#kU@a@e}b;ues@9ZHp**(4d~NP?{F*j&f7;7nJ-R+nkp2xAvOe
zV9`Y=P!u!YML#JDEWXkXcapfJ{@jpOkxKas)fGdRLhCy9`-0fvm9x*i6EX?^#j}Mf
z_$4kbV~noa$mU3S-~W7@HR;UBp%Eq}ZA3!`W%2>64xV7zR(Ki`z7yQV0l|y|5tQ_2
zKL;%<xEi(bHDo?FSvzJcZj0N*kgO`3p=+SUsJ69uGedU8F-QdUcQ&>cwH_y>w9V`2
zSCALR-C`9^>AEwV|4^UTl=7dC>AcbU=sQ?_f6h?cWuf{o$mRU~4c~1RxvNpI_4;0>
z=<K_z=g<WU+NEt6`+@S)lPwG?L~aos*JOV-GT%xvn-7;)5iW1Z<Dg=d7S_{QH@E%M
z&P;~j<5tR7IMt(S_A+a6_DML{uiJj8q~_tO(rw^{9W<0gB}n4tsqrKO@tiextuXh!
zhVI`_^h4-n9@)c2$$*f}-vrM}UXzATWO;a^R*(b@+FH{!mPeQn2UV8hHd)`ek$%gE
z)Gh%QbBx?zs99t);AB%Fu`L`wu|0(RZmvh~J-ELP9tuv6G4~z*$JRbw^1De5nZ+cx
zigd1rX<%=QxN^_sHreBFf8#m7%IZ2sUSuy4qJ~hW(7q?L6*n=I-CaC!Ub{Z+{qro$
zdk&xadQ2KecBAHn<^dTYbDNBzx7JBZ-F|`tKAd#rdk1`7Z4IRoV7jYThBAh$M8ha{
z+?%|sg}8GA&P&&?UmtRd?b#ZK$cwlb!&6+GL)BR{C74cJtxyj&*jC5H!2Dx;qKktE
zSxCszM$_qk@?OQ$7Ta9@b3N2%yplGieO9HU>TE~?nHH9sq;R$nOM5jpA3S()-!hRC
zO1B~}k~QmSsYCh~CrBL};+tKV!7<&XJz2rgPxk)ElP$3{&UJn54=3B!(h=fBcizPO
zKgf_MIYXNXrN%=p`Cx2KJ#+F4CRD$#7tj3LOH2+PZOoez2X?u5i$Qk-<M@_mZG}hS
zE??VrQ?$|hpyLP2?Zp8WXMfCUBHK3eL*`ZSi7N_o%DKhgJsd$B{>xDN_}tu--IRil
zb$g1|V9_gXCQGtdB7~ro<tB~&u5VN}KuI${vi^X8h4QEjnDTt5Qh!r!bW|nXn-IHP
zk3w0U=4>gU>o%VM>Eq3!RBlQJml)Olv_*KuooaPGLI^#cprRDt+>6ip982;PV2vdN
zR)^R_mZ<Jizo^?%E&Mbw#?$Je+=)Xg(uayv?Wos~3EySyoUOPuZc|#4szQbeE#RiT
ztvCY#_v+59cAI5*e$mUyDlu`nJvGd$5C4Z}OLxHq!7kriXWO?GspU1@>^Rw$S8atm
z%3?`(S3WQ82=CQg5AWBFGgh%r46r0@Cwjkk<_wjqrX^gwP}yvRid;5RCu6uGM~U%p
zMIIW@MTl#g(-zawpBdyy_FZ%CJoBdEQjQ9IO$l|FW+^wHUQz!yM{z)R{3y`b-BCAr
zS4zvkXNW=gD7}haix_yqf5tbPtTRq)^jeMMx_;xvmi$8{#KiT*0c`n?FUrY`eZJe`
zHVmI{<<{a&!{^)Bp4y~zQ`(DioAH~)-?v`)4;|Lt2kNLAf-GaV4bStG-%yEuhfk8=
z$be6a+R-G|?G#f|i0A9sz(Fz{SiTs-(aA$4<(<#Z2O5!c_C-8@HiAc(P6?0D&9ycc
z62-!8j_AP0om;lhd-G2}-(3{crTWfTx0xZgwU=a6A+jK94W2uS{F+osI@edaCIowN
zz<Cjv8+M2gIM{JsGv*OPLFkhafN88c&y{sG08FFnym%RFi?#;pWq#+Nw)3DF4I%nT
z)p}c`975GbW1^cz;;zsY8yTEa32s0shAvGd4m`4i@(Y99{HhRR@KZ91og|A~$f|UU
zh5TH0O&geKjy>KR**jorQTs7Rm%uC@cI+qDKV|%y=<eESQmVawx~a5x&xs{hdl)Un
zI97DCmw1=pw&Qz(^HrE}o>mZc556Y`mM_*W$RV(v%zE%$aX>4AsBd<RA}F=lLe6}4
z0?bF(v47_MTf^-qC(H5LA4fqHaKr#Nz2_ZD9vvR93MW-2)hVoi6!O6vg+i6~YtC4&
z6}ZdDRKG1>2@1C(0~Xcqi`YKu!HxwWZl<C8BLJeKZHL%m?|mU5rNw=V2+RN^{V$Kz
z$0qN00z1PF*lYa}O1*(JgQyT(T^{M23JXpO8EYi-A><2`K*Gij&(x*HgAvCo;!Vwq
zIjc_AcR4EvlQF_JdD};z{TVB5Wj(x_MlfMrcdB<9!GyobEH21}N0?Of7A1N2r5O^X
z<9GIK2qDJya;}0<Tl28~PHo*Gg$VH<vz1XKyr;HJsIx!wvjyp#0Z;aCLv?rp6<?Ve
z{%y}{#o~|t>GwS<RLnMHjI?YfaZgWHMA9|lUPhBY5q7W&l59}AoUul*O<N$os;usf
z3Vto1G^>_%X$puB*4{EZ&Lx4WA@z<!(5phFA1VzD-Y(F41}%-bR!W=jLoJP+R>F{Z
zpwjQ$!Aj@0BVaOQ7jbK>F1XAzCPgXoq>YBb3x}xe2dEEFB~K|1{t;_$=tsQTUTQSp
zNbilmUxFvGuk2<gR=yE-Sg8|#c`&OgL>@Wj`EbUr;tE-~a6?Hh%_-XqZ5&Fud#>~V
z7PnLMma>Y$&g}XBgEOYJ<xuG-68+_+<Dz)o%2#ayfTl)9q9BB-A-6W!ZSZ{0A3HQy
z><J*;W8t(9p?;Kj>wW4<;7}Xj2otnC?tMn@7qtPXAlx|(y+8n!p$%ZV1y+%tGGFbP
z7!smud6o58*xs4;Rwz|83mpNEzxT|Y)J6cctbfs0^%!ue!Ip}!mykbdLM*}-<y|Ev
z@=_B>>uX^3o-Jne3SO+*1(pUCOCKM&D0k{mea%{T>20r08=ypE!n~Oi0m^&Wa#Q-q
z%-(<ahnQcgK4P$ktMT6!MTZ=Mt@n@W)}b2LpF?;UZcQuEkV0KoeJ)DZ8o<dvys3==
z)Df%MQnF_t`xS8o6DuwsvS)Ea=-PE!M+K9!+TTBxzJcGi4_uhn3T*=M=2I2o91&{D
zF+Cusx>%Xr`-nexPe7E&0j8(w|27IgPk$eA9JTp|8U?79@BE9>Wq0wsMyN3<c|LZR
zom%u>boZ5J97w7VBf75WVUIIbz1+}UU2dj6x5Y-bw5O(*a|zhI?D+c)S-AjlV2qfa
zsmn~8S-_i-Voj_*+0e-qlu%n_aF8o-tdqV}bA>rHZ~|(<C|vy}tI`5%)L)_D>LWhv
zZ;hg@y~oDzVWQNQ9;IHGB_J!hyVVO?(B=1p^#&p!l_n>?DIr)?E;!R-DnAgx0C{xg
zUKyOwN~M6rLypwAJUu!EuQRkU6?a=;KbCbX3O#W-G**qY+h8waQNaZ4f>8~V@o)DZ
zVNJ`-IVQQ?PfI$Po|985I;>&_(*T35=pBS67=E*{>L76?K~Oom@|}gs8eGIxc-_A%
zcBZ#?_yiT({@{T}B(St)T<Txp;DJqqY<2ZmE_{v9TAQY?Q)~IH7cbn8n%hIf3e+gc
zcUlvEWYH5jqxp$W)!JdG3?8Eqe-Ik+XIrJxo5e`w4Bq%Cl$u;6--~PR%}gdOEr0of
z{3-dBYnqA*MC$cc;L5Y#`+hw%`fX1c<w@%@iEfST)!J>bAY>15w$M|TO*4DurN<$S
zrer6;br7CYo-sytG7`D&-B}Xt$V4i(hAf-jy9Y|+NSC<FgK;Z$@jcd%pCtOHsYoNM
zS8pv2NZ3;xKzg6EE@FGY`UN#x+_|^C+9K2*sc;sOlc5MkYh;g#V6HV#jhiasjbHoc
zHz>$AI@bb%GFE=lkUop|FWCvWC8K&P%Y|ZB%t!4r=&d}~&^v&7D-<U_g=?-rZhd8)
zBu0K!MOxYjuNs$~Oga}8-}<qu-D@z-OCeFc`bb;hBDmr={C32iisLH|uslV1-qU58
zoHMrECF*@l#gCzCCY>1~all1JcUyu&GMV6Rr$>r2h>8ptsSc9f45m5kU=t*@qe|-c
z-ty)JhQ_7jx4oKkz<nRgxyjb=!U3!r@7p^HL|+>I!4)`?l8Fzcp+H%9*5ok4tnokZ
zI?SwrG}O3kjkk<HcNm@iOkV1ztA;PNFJeiJf2qZUw$O)o_I|49c%M|KyD)x(v8u%e
zV`6{qjwTfqmgrjRs1R2b$j$KamCJiXL`bo}01;0n;LxBhsU5<#sfdru>4^~ne9b%N
z<dfl2qo@lnu>?EmXZfxfR|Us+VQYWw3a9ixzh~Nvsx<CA)_jDx4Tuegs7Qzn7iwih
z>F6gJ8;p~bN!vTFJA~WaTK~!#T4#%x;u|tQm7wHgC1fet&M%`%NlUrcFeP#Ugxl0h
zCP~9_*J#V&2kxEM>|0|7p?2h?zb!b1PWln=b${ScjT9Uk+#1<$WPQF&@-_I2>(uy%
zy>IT<3lz4(gk%lmxqWP2UD5ycHM>$e;Nu6a8r4;z&DC_WHYN8e@0YyuKm1v=`L!my
zaEsBKD(y7<;7K}NIxHlnEzWnM+PPSyZ<2Kf&{cQa7JEtoT2F*hp@%N&6QB1ORFylW
z8`_Rs?sxf^Q@-Bz<Ngs&&Ipj}zk%WP-%4w)KlI{9PU?I5*eD>Elk)A&-TsH8^2>08
zFL@OmNzy`>zDQZ++H<au<#%w%3etMMJxHsx-Nb-FAQ9Ut=T%Wun7mTK4tc1dUs@})
zxyY&?LPUJTKZFKM4CsANY1mU5&FX#HABY=<Vq4Na?%yy5csXsG49;u2kaRB8z9lo8
zE3YyvSv_Ffn{ZWyWndukYIjwp8<D}ghfx%FvDLIE?`X9{X=dsot{|_z;3O4nTVCd2
zQ~k>w))JiVD%g}q*&$_NW?v<|uxX8=+?Hh<ARvwWdVrN7f8jXn>Oyenk8u3DB;t9%
z2z)y=G-#15TPC@%H-5K;gfYI)wiY;^<2!7t#UJwh5Hj#R&vSGphAK3sXeCGKFK75g
z?+Rb?6YK5!3`Drym%e-e>%Cc`HQU~~;o_o0+nR3$vRId{(d~^beMIz@5jE?a*LIQ>
zT+_%enJ+gZ@eOj2cQkfhk#{*g4*7^nXfb`<(J{zJwhCk}(U-@{tBPYvKuB}TkD7N(
z=78#}XnKu9$h?J#yGR<X7@|{B?n(yBO~&<a-RZqUa^Vo|d=Frlb_99!m5BthaLWsZ
zzkKnc-=?lquO^ZErgy>6t)>3{cBQtgt51-;Zk>tz$LU+9xu|*Y4|5IAMBfc{DjRGI
zfigp7*!r%V){8`h&e?RSyLjM5W^1yqF;UT}eXHPdhNWjmuokP5d{MM|YhSZfZu_e+
z-RKU;jHLfU-elz0CC@{y0p&}e-o^fqC28CDc#v=Pe%NgRT?AtSv+{zgW8tq3tcwpA
zU0!_6tav<%7~+-AWn?aV^}K#oWS^Y&zy+}6jc-9JomDQ89!JJ$UGQydFRjehRsOTc
zB*{KX$5L*f^U#PT=udoGiUZc%*ik?-88OCbzMeCUp*c*SUOSpT{ce~(?O4>;*ju{W
z;+AF|PLb(FpCr04h^#l7HYCn;W8MD|_uX+(Ud#K}5YZb}&=@JwL_`v$C@3gZML<P^
z9qA${sC1CJ?8cyg(gYL~l#L*wQ32_FMWl(i(nRS<m)=>*?r+{@L0D-^^8H=@xS##U
z+;h%5XXcriXP%QHwQ{el$y~G{g!B9(qsM585&MO`t6~d^zBcJH%OzNDHN>2^2(!4*
zO%zYMQ^4SOnv+~s^su1vb)6)-v_KL!et-C)`%E)CF67XVrOP!CfZYNuY4qf-kVgE(
zt`<QG_x?OGI^2%zDlOFVT7lXUBj1nVPx!W-t-Nz=`@3uF%~d)xlN3C`7dpaL94)tU
zZgs}#2Bzri|7NS)cCW{8-8VOY{IWky>1d4PhYwKi3w>KqcictBhpxz1bt6O;-$@xA
zyWF@s$a{$nVda!V@J>0>rjJ{HRnufg0P_R;f?8g@wch#q<&EQpxS!<@&fLSXJ&N8Z
zpB0`S;C6tT4gx>u;^uqDj&%kTyj7!$?`S&qiRNyXsF^3D;Cm8&1hKsD2N5nKg9r%l
zSoQ8Ej}vDM(I*2xDC>-?P<iHw)6uw?iYC;ZKCd_6NgzJ|bwseIc<sz>EZ4DqIyeq>
zy>V&RVj+aZe5*hZVKFb*7M>_?74<)?JW(&+ZVk;P-iqznO`J<akigYm&Au(-syYY9
zRq4JmX95c<^-6c2tcvS<E-jf#yS&$5?={z$_g@vvA*jr}=eILL5L<}9WV>qYbBj?R
z!ri5hxu9ImvqfH3F6Vb(E{r)zGk!*Yqtq|O17|5*FmLorx+Bi}?!}Jk-R9?7kJjhH
z2D*?qVxIY6NV`UX=0gA*p{Q34GIK&(-3})D?aAKmN%y**gEq;~`I5Bphz5<uf8wtE
z7!2=L=P=@PR#>@8cBvzw13@#Q;B7U=4$nOQ{r&2N{Bg%)Pf6pgb`;#Cdu}5qRR&!j
z^ChO;TDWSwN!;?jN%cRu%YSTCCPWp>qtIt91`gY{n%B3~+74AQZRHgGwI;gf(eS~U
zPSwpw888V8l=6qVxQ@hx+!VS+Ww<XqQREAq>&ESBjm_ENpIbGwq-%T`O!}4xrzfT{
zegWFdoork)vn1wnk9j>QqpS6O)*W$Coik4LR$sz-zkx>NSI7-TZ0y*17(xh#GFDAH
ztbbZ7d~vfDGEJ>RKk2gJjKa4j*rdnk|78-q^wWpTZ=*}0kW}SxeMWOdz-dWxUr}8F
zua1@p>rb|+bt?6SJTyCK6G_(kyw;?&I@U_4V1a(GuB8zQI2zMQuXht%`^Zvq#mLRA
z+`YR*f@SFe4eHBl6fS-qVqxGtqa|CcEmgToYFHLM^!jZ!)7u1Nncpx)z>=E&cSx%w
z(y8W^<G|TKf%&trS+edrXV;<hZE~f*mlc&0ZF677$IeC|qSln=iU8qLqiBj=;pYoO
zo#nfNkoc>zm9vrUxv`8eCYff<mOA~p?)Y6_)vz*O%SiiJ9s~2KWwI95?SG9ONKZZp
zS@XPY1BJ|Y<QxC?HyE!2{w+_soo-L|uz|F;%4IXm9_IM4gLFC@W}giCBNWadl3iLJ
z5N3FfHs~h4zlz04R}QBG<R)*yJf>NBpgoL+F@<7GtTq%uhmh{^1y@xbk=tKlI;@2w
z9$;O$OjO+I1*aRGv4K%YtrY=#B~co8S=H7Du(FuNv#N1MYRSYVnbqCl)a8-Shwk9?
zzc7v1w6SR5>q~oNY9FKjxRat1SaGJv1tn=qD^(QH8=YKpYjINpU%$x4vbxFh4i8km
zz31r9Y*roJY+lc4noewQenZ$s<Jzr$`x=1zdEj#7`r(!J=4x}D`P^BXw@CYV)9^KE
zZD7fflewU+zF2_p7=bk~aLFe&Jl)D8i0zi0=Aki(<#wfzg|anw8p(=dENN_|H>Aa>
zF<h}Cp9Ou1-OD2J>Re2cIqCG>rGU$B5)vr@Eu|6WI2U(4Hx!5=c>i*kDvgo<Q6PxD
zqbb|h32984%2v;SE_D+jxgoDIC!Mi3FljaYuR$APlI{NSa<gsBl1S5$TC1LMKM{U>
z`xfGC_tU|sc3+)z{fSP%Wz}PUaajbdThy^rh<39bJ9BhFMa~Lx6_1vNY+qnlsZv}j
zx&6eK=3l0}L--x(E@xfb4Cfx{0W|OnT&7RHYZW&#s6C+$uFjEpvFK`ZxTbvhQr&e^
zHMux0)`pa3nQJmNb}^j_KG9B_UeEx}50V7O!O-aptKa?JKRK&3JBN~ZU5x7~cHdfH
zRufY`{C1cKWWniwzupbOG`59cI^Gz%)UmsAPQv2aExWt@-+TgUXyBaG(1=`Q7Nfh=
zoU8LD6S3<CA(@h)DDG}*3S5-Cfu_d^q(T9cx2rkEiYb$~$b4}~PI*yOqC-=c)mG0M
z4UMWXg1c=nF<AClL31`iV7l?Z@4c)@K?2jE;E^eNda{#AnKzi9*gCm-{hYc8yDra!
zH{n&S2qoi{*LKOWd0|2G-c?tZ)tB1|Q6?m2`v9_)68cDeOr5}OI(D@4Xnk9K^O~9F
z6Y`h4$f@8p1J9I0F&+F%Tl?qXJ(XFn2bkZaC5b9mY0WkRQ|!ZVE;oXC-%*&2MXqZ7
zO9}@BDLBB8+6*V(U>D@*Dme<-l(_pd&x!zMlP0;gmO_FTZvS+Pf!|`s`VYjd9|$0L
zqB6HLOCFtM3M`_*G(nLVHbx^h<486^H$8{|R+oI`_ODGh=QA3ZeT6Jsi_=>GnZHp$
zhU@}nuK&%gpslW)*wonaT`Ax(8=)*oY-<bBT`hl=CPtnAyCtL7FXYA7OR?Ic3fXnp
zNF9qVjz5pw2?!Bd@9-PthA5d7wrYEO)6{U-^x>Sz3ee0AfR%s0fhU?94dtR^%k9o~
zca7_V-wZNMj2Z7A&O=yb)Bb*qQPjM1-N=#H@1?gZyN;3W6p97+R4?)2P}5M3U=3pQ
zSnHfb&ExaJ$-cjk7-b^kKQ+gxZRv(0U3`u7-x0pTF$S}gUZc+LCRrw^B5vuDAH6+%
zgJ$8m`ziUyc)6o3)pF=1qIch0f1UGKBEQ@y<=DwF`an*Z*WcS8MO=&okIK~0w8-xL
zjO1@a(;%nT`uskzBBNOv{0(mbTv;5_O(l5Zc1}BV<mbCz((t{a2(vt-qdbSh^~B$~
zKTzjN7ak4~*nR7QI***6v$?)Vr}hm39>T3O*@qh$`Jb-{3S)XJXO8Kq%nY~Z`b~T@
zJ;~{b<t-vFc65HSO6*zHBhEM-mBB1*9@x9BQbm53Z>G5du&`z%uhg+$7+0Sh`=#hf
zoQPGK(361d?_*=#)}4&uRUEiKDCmq}i%RQoS>VbdQ-#x-Xf!A9){~HvAZu>f(|wQk
z%o(Rl$oiOOdP(3u2W%m@jpS3jq%>n4e8vS)yb7}?pUlJ*-4u#1QZO&idad2_n;lX$
zAUAuxOlVu^)M<tzX-?hpN7KqK=CRSROvCgmgs-1PZ{W7lnhTrk@fS*(aebDnJ6|>|
zDRJ7m^9NgDwx1$z5;7>kwI;VPp^Z;@f;=&6&HVeRPTGZ|I&b26>_)#_sz2p;T2e)_
z20du?Merw@lVAUf>=_JHI<!4U;#fM_ojC<p6w^=LWzl`QEb+!5$q3CFj5vDlE_QSZ
zkEONnFGyZ7oq|5WoPi$Xce)(OGtq&V#7_xD!{(*~?;!mPTEWU9Nxm|+388XL|I<fY
z<P+#ed4a|S^MW=r8?yARH#?DdSAIRtg<|l2(n>Xt1?D>X*#~NAjvmKzT~n$uF*erW
zg44w?Vwz+%G21YUuzLE!;dlQouzJxO&>-t%2O8UghBHgw6YlDm#0#b}`PJb$HJc|}
z)kRF-D8$7+bE;+t%z%@RyhXT8EAg@APqGQjG%MZlu%)+mRpDT74|L?;EO6BGfcWE~
zW@TaB;yw55ES0^LaKis+&0NbE<n(*Gn|N2rsftA0Ck9tO92EZ|h)}?${{<fHHzFP^
zwCUIb3Oi;wD=X`L=1H~ZPWvJoYAdcJM-A2FG=IsfLJzh#=PpaUy!0dI`H_K9P9`eJ
z+OE%^vrA^+`S3l2!ZrQxu~4#>u=4mN6|YyaJiG5+I7m0ieE5)OTE~0rP1S6_t-Ma%
z|F!(FvuBcdNSTcXc_6@xZNDqesc|q3ep&v4ig&WMh%40As!F{A#XQ|Ns!TMdV$H3{
zo7}1rO6sj*z;05mg^BUQ(@S_WrdcTS{1b91RMmo0z^g_`rsy!&YMZv^`r@7j)5NZa
zggeMkJ(W8MunVD|Ao0G_dQM#<>u+iHMwo{x)@4cEnNYqf)-Y}$@=NoN(@g|^mn$B_
z558b9<K(~uxlFBiHa_t8Y$__)J>;F*QJwWVt%qo5NsxCKyn!g0EPc(&ug~Vy6<~8;
z7?XO7-Zp77xk|8p5&Ul5=L^HkMhs!`;tW^`u`!8NR6C)N9v7b<-C1v^(rJfQ3a&h2
zn>yatu%S{vkIGO5JE6!(E#6Mf38DODc6I9~NlW&@`=>Jz${GQoucFwlSCRQW2?wc8
zk3KGLrkX=#!F9i$-|Q_LLs{ewdc9r+YrRED9H5g~f-<H2w3V7u0%}^Q@p0D6P0e<M
zma=?Wv-z`Lb!Bgi&6w668ngbd`$!xcs#3bUhk#1gW|@A-ix%pt6PNAuue?6iI{ZjR
z+mgL=f*_9^E8Qo2X%~uP@MTzOQ(BTj5OMh8AJg3i{N`FGYn|DwG+zFKfyMJ5EO=*o
z)8V?qXpP{?Rm!|}V)f=x&!B;i(4dgkmNfUA>Mzr>282RLs1bP~Q7tp<NV>_{g|tQt
zm(g56%QGc4YHsm=ui8B#A-u%n-Tk%cLl&@iQ)QBxAu;(SyYB1>`VB(iT&};n%CiET
z&Mxc6?DETLvPz%&5K_A64d~55Ym2v&f^)VN_(C;q#b@;K#Y<jg8eaNP5XVH6%rb1s
z>qu)e(aqh()lUCqvpkIY+}-T{EXeHL3u6q~bD%)f9rqs^doW@I8iXitaU{Fg9aD5*
z0>_u`rj{$A<ZgxWeNWW_zx=8~hb>?X2r{M97>;fKF|r4Gxj~WY&^cU<8=pa$g0T!G
z#jdUL!sR;Yf?vzzlJwh6BpZ+hsKHh0_|3vnalTv!VnkSUTlo?`CnnnR%k5U}bm2F|
zyIJh|R0zlj(M^L)Yd+mHUj>71)d}6SRqlLewg&O)$W*4vCluLTv?qTKf76VivEIOA
zg|4O!mr5e5*O-)vK&n}FD|C82-)mBi2vkod<%lSHZ>pg8_Dzn+HL}F5;&6x4Wwb+Z
zCDZBZ-W{@tPS`XT0lkHJwEU*WJjU$Rk28btV#BFcdIRtJ>etHC_~C*?C@oZkVg`gQ
z8%z=whM%)Fe71r#j6T~>X6^)nUw-8H7q&JeXdo3F>b--^a6ToPX~22An9GN;x-WGF
zWsS^cn7#?VNG!sOoc<>c>|aN{5?zdTN#uB`7s!n)ap60w{au)d<eza51rwTEK8qG+
zUgo1f3M#rlDmM!&KOvnH7^7cy2CA2%m51t0R9u4+grBGP?C#l(qq&2}e@#AVuM~SK
zW$V|ncpHxt0f8lET6OXLGOKZ+f(fyM{)TNmJ8PcuEk*1RWT(=+;jF7;n%{diX9=Qd
zZ2Hj47RwGt=)#KTTO>SM_?Tx+Ph`veAM1Lk7Wt*ry<ZU=vEmb&LeryL)(^&ga#+<Q
zvHGp%ede9D+4A11L(}a7e&0TK41e4${_>FWc)6erDkW*d($2RvBmVX}%x`i>R3prX
zD)1-0qlgqmGf^!Hv{=BRRVTESC=+OV7)8gczFZ1Tu{P{;w6cv2UXN?1ad*FFh2)Dm
z*px()?7*PQxI;}Nax-?Q%tij4bJ{Rd&GNYkvq(b?5AKOzfa9rA4hOdaWlHj0=ZmoM
z5wgv~YHgG$n>Av?&4po^L6P~~mp*<Gm!<I=VOsrXiT{NTy(^UCC2dj3Nt@<5Xiduv
zD4RQD-%tBO$>3qUebOzgGWze}%GX;7m?R$kK2M7HS|SDD({TmeP&yaKNYI#9Yi8b=
zE+<Ez;i43B-4rSH55>d@tf&HEm+7u~rc&HI`CrL_3iP-oIs&jzq!f9b1JH_=biPkO
zZ}l8dPw+x7cIXC6mtd68{(>{_UdZlWoT5hJAZe`ukU>t-e^;<`D*v0*Swa-FCIvk}
zLF=2K2PkNr2DL^?*X9n7QBcfl>H3&~hQaEAc<kSxz(rA;q~``wn9=>eax}dl`ebE#
zjslmA_;L6bTHkE7#UX0!+7sOrfv=+ERxiE;P?DWEX}ZX8EV_9X5~4EfxJdBKrH1(9
zEHT7A6qX`A7my-urYH99Grzm5B=%D4hYKQd8MIb$|K<n*dk*AboPjSE(0Ho?^!p|*
zA3}XwwD6H@WdF7xUXcduPC0#55!*7N;a7`zdkV*XS|w&domR>$jpcn*UtdJnoosF(
zJP^E8yJ)=Jj+~JE(k=kX%?)l9&!;bU(JHz~TR_6BeLQaXwrtXMHT=M{{yh~k$Rq^)
za(TOty7Z5iLe&0P^e{Vr@Okj~*l@$|G;6kkI))~IPtU7Zb`GRpK2ge`Z!WJ+97^za
zbiGx}{WLWk^J4}MRHRO*9Mh{iwELuswUd~wos~Y}QeRTfy$3Xbx=G90o|p}GELk2g
zPV|CdEx&1Yyl8<a9=T#?<8=FNawe*s*=jhfKhLEgigGR8Gw*^qpzB2T*ZiDncrh$a
zQZDj4J9a14D(dnAjaC8}Fa|m1n(L=$){CBmb0ze%2VudG`Q<EBO)qu`6F%yjObm<A
zZ^%)o)?qtJqbRa+xOe#ZcUB=Xa&};Mq0K<)F?@WSeyp(YQqz*vL6N}8C>So0P2$vb
zBB?t>2_K=YJ1<VC)XQIQkhCo!nkcRUy>t&kD~%=ZXbxc!!m7t=OM^B5K0!Z9*o3yn
zvM6C^$fZ^W)xcy{s$hV8?<=*o!uZ%F8nc$m=dUbd`zhULaVeTuU69oB(9GTvxH@Lg
ze^#?a#T7U&iBg9S1L!uS+0*K1$rOocQ){h@-?796>$rTj$C`%qgd&3^oHzmvBFc3R
z>1Yg49B)i`3;19%g3PVzO7d0EUrC!r^$%;T#7N6wsoofww?xnc{YLx>C0=M})Scun
zo!CRDkJ(LgzV=6BwVf<VBI5xHVY}Eb2C+?rX*wi@?S~JsL|(7kN3#7QKHM;vI&w9#
z1d*sl23n@gODrgWJu$)L#!-1WT`-~Ox#$aH8ELry8fW3NLUG)Q0NtJ@F9=eTo^9^)
zoo#R>vZO`h437t)&ac9;tb+8E2iQ0W?Tv2F7Vh${w&ZP-*(yw#t0e$r4qV=B=Acm$
z`G9@6&qq&!cQWG5ro}Na8aIm;2UHBrg||7@ti@#s+Gc`q^K~A}pyf+iv0(PQ+M>wj
z(-JHV%yGCixSUzmTaKuP6hlo`P1@x%dWmUB-pjZT?P*_zzVr<@P-M|%fp4=UX<|gY
zx*n+^3nS%HJ~{VUEaGc|Vntj*MsrQx0%ts524XC{1GnBarMIsu$FMiAQD-3Duw}1g
z05TFYo#C{gG(Tlzi)*KXB_1-THZQZp`c=B?ahWtHZS-ET&~V)E34W}|C}r?5a1jpz
zLSUNj0?sMMGknPF#q|rjm?h0Ex*Y5Mhz+LC@x=W8CqY}67DqsI*r$mf_%&+6)#=2Y
zJskJH!x$xa;tfEZjv%DEdp9y=u(bXTth;G@y*ZA@EoC+Cy`Wm+XnVj{JxR8u9r}97
z%+8%W1<-SN7hLLzU5CSef9SktNPiloj4*(G$8+-=r+-VQCWm$qv3PXtPdg{xtzf=3
z?cEBP(Vcv^qHgQ%c-D}YR})=LQ`vDJ;9@>tk<s=}5uj+^)9?3QaoDtm0@&A0NG$ra
z7MJ$@$|nkoqy$*`AUA_Ca5QMrlXJ!}Hh+XF{4#f=hxx^IBVab2;PFyz_0vopEg-w{
zz)&OYB-Nq>nDczZwea4}r#a1Xk!;&ak|x4H1kdPQ3*p88Dpc|#ukWEYx#`nvK?@*L
zu?Pijo7_`XQ-sG*2!xcSopi$oj)-Cy`uYsyL-IOY()S0=QbxL1qMObS_}PU^x9<V9
zp|)278fx%j!a`z{)ZFzRmnM@ZmkX!EF1Z(~BKW~<(T@sdDHGZ2w;7hjMaBM491+TW
z9Wcz*C5kjcdu{~L-pDht<3|WLN8&Oe?Y6b?07;tZN4<E`+hMqpN?xyJvpw1B*N!&m
zMD_ZIw*n7kX@^|`3(rgmV3ss<%y^q!{H0ie)<9l9(Ixd7JsSxGPZ*9+E+{aXj&=k^
zx+&Ap65RKt)uyg1>Q9U|nC`M1bnmaN7@)}d(~q0FQ76u5RiW)H;(Fdza=$Rl7Brbz
z;@B9de+=InVwB>5h|!)w*!KVAxHCGe^}4PPH^?Qy|6BOi!i0-U%V%ZLE(h21a-m6Y
zf~6&c*>KUc#{IjZcj~Zv=!-^PkCFUlxR<;(s2A4{P}WXA%MGz`_CFnE*ap)j<?yNj
zJkWh!Qj9;5X?VBR54u*4Y&uCjjXlIbTJt_hk%sALRD-UdSqJ+WrhAlIN$+eBq~Y<b
z>$m@4dqk3iLA&58%gSnq%N@o{CZ02)SI!M)-^_(wIXkYGHo1Q(-briQPg^WsK&C~u
zsCDMgyrj;jTnz7$eA79h&USzr+th3aqOC5Yn9@KIYDTztFx%m)fBE4t2J^M!I%qE(
zrIQ)W^G6En67e4_xJyUtV(J*8w3O8Inf9cQi#WuNJ$j-$9`Qu?vQkAHIY}SXw4sT$
zgn=3)M6TaWlK7mJhUdgJ2__g0v^Te{Nz4gdQf|=jpY^fwIxq#4e5PhBa9HaIO?vwq
zn+di!1`15j7V1#5@pyXPbBYT=dF*Bed!uvmu^U<~QW>-WB3R&3^Z3ekvUJZ4%tW7_
zWn#)?IvZ?}S@~aj;i^L1rq%ewa-lbdmuNi6p`<5xv+d<N7?apj<9|?Oi{`;n6Ysiw
z|EdBcVTj}QNY}>Ic&QHfrL=k_V1s$6ii6YO2vu}pfrl0H_;z@xYgD0gz8vav{L;6B
z#<Ih-pz0_LjH*1qt!pSUQxi?(Szkz)QyjGW4kbpSQ*`j_aP5LaW1)t)Co6zHNsC1$
zrn(n+!=zZ>DDhHjcT*Sy_N`W~omeGw;O3c!9di37kB~fHxdy2dSa$@kp4jJ;D2mP$
zvf-14QNyPSQz-Oy0rmr3b}iYTBx!Tr#T|C*-W}-d97YVEZTGuXJqN$Lkz10rwn9Y#
z2U)MHEh=1nzLad9h6R$6A-&zCs3FOEaZ<xuWnT23JT58UA-_ePwx6>RyT)f3ymcb+
zMbB^Ft@+pFu@FOOWxaq3nXog#+Z***edE#^Ey%jvd=fT^R1S0Y9K>2usG#hw3S!gq
z^~K40p+fXphAf|Xj#Ea+aSv(3qYE`BL!SW7JrSHB1LQR~=5Zo(KIuYrB8=6(j@#tA
zR14v;v4UM(Hj(t-#A7?2o08-K`r$rXHD~vIX%nZpj0I6obcxAEj`ll2Q0oM9nELDu
ztwp9Z8)2lTBr?a4)OwDKIi+|ScA`#c%>9BHrGV40EzlnL5nZ2R%F(81OWF&_vCio%
zk!hj{;I9h7oSdc)6IxU0<79hiG0$+gMmBuVaBD~mV#2fH#=_k#c4^^f8)}$a)&gEW
zDy}!mcCxbwBX;yWy+?yOE}Vo?x=YJ(sPSTnRlw|%qOOx(dK#AolMk1TRsEN0+pKVM
z>s#*KLp=`|&c3Rj-m+L=vNY_!6eU8K_x$+fYg3M;D<@fx@PL!xPIyNX2OAPwSn^f9
zMM~2}zm^3a!;chlhZ+s<_y=oezhc#<_QmjzRt*hlvZMntnDf11>^rI&grL9gpYWr!
zhLFeIvPuhVQ5f4A4qO1u)yuUg<UQR7#~cnr3(I%Axjb893xML7{|dU?<$+^(yw$d$
zl3claf`Kyil+xl38Xnat!D?z<Bn=0bW~*g~pPv~@Hd>yfFmpfl+A3GXt!P}m-e$6t
zK<AK-AK*J5ebbokvtv89X(#bFlg>4>vv;5Q-n}g#`6Z}abXY?w)!wG(LsR<jBU)7e
z=9@g}KyAegqyX-J^J;~1Ho@HFA7ooF0w^KU?xZ|+c$>Dn^80ZXS&)21C`!o7VZfVD
zo*L_#$1Zj$dWUhu$eK4JZ$utL|4!>KepJNU7N3AcSHPm#z&7}$T?pI$5%~Wip~5zl
zE{hSn8&T<Fg-x~s@zRj*{qlTLTL_(3Mf@?*CG3v4W4*W-jYsZ?euC{=8;WUEgk-##
zQ6#re{IK#r$+66X(sXa)SSa@FYRl}rj85ACF^@BW(2@=6;vVDCvFIx>loyoaqbZ~o
zirf@Y%ZC*Uf-JaM{!4K!`Q3pR7V{eBOwB$Zb3iU`nJT=TsOy1S^BAV2J5~E(H1_MX
z=gbyy2SG?hx!7a4A`9Vilyd6w;=E{_MvIiYh}z)(ic&Q|8j;Ln^|^^0zCNMT`4O{I
z*JkOtAbz97TvNjoy#aHaVj1Ks%}ZJIMp(4b*a-M&{0-G?8rKZ`a(Dt```5BguHmq&
z-q;JEAHQ%<0Gs!nT;9YXUy2o0WE&y{4BP7^!{b{{=FJD#jp;vTrQ*ZGF_mwd2tpL@
zSJGXYI0(Y{HT_qPVQV^|_SkgQ+tcG-#w5#ip4TVm%u-PCVI5zp6S7oS72lDz#Ate7
z&+w-&a3jREUBaWxk~Q(xE$<St4bRbd=fI=Zt&g#+CLN=7Mvl7DK*o&L&qa0S&BIQb
zr~iT&^ttCIa@FhQr4Orh<|cuX{8H9@4%RFethxIHwTef3&^+Ok$UPF*<vx;c#xXp+
zr~l&4cY*T&P-+%CivZt57VBExRDX_iq-X6LYk02=i|1M8yozq;(O{>My%TEzAnD8S
z4%)B+MX=;w1_Ig`%(#d9o`srwWQHkGYhiQ_$s>i1HT))hS0pfAlpDxq>*bQ+LbqjH
zk`XQ@HNPTMn;Z2#tB=`6<DIh<$9qoz3z`kdsn!pBURBlMcIH&ed_JFXEYLWz{LAFM
zsc{ZrZE>7+L#5*Y@?16jA?0X(w5@3M=ds{nF4|Le4q>V6e+QJ;^T+TuX22k=dC#{i
z%w>w(F<bgN4;zf)?REvN5NVRE55GwPq`?o6Q7gI++j5U^&}j*>MYo~c;A`g~vY6a*
znz(XL<ZG8B2Ka#oN_0(`Wu&A?5(ihsY+iCQxcY{LguL+^0xRM?ND2LJ?9or@K5O=4
z`4K~+Jf@@r=pSe>hnQyUsR~Y$4(Xo<0?7HVbiKUNwY}G0I5=h0Q*!grA)0)q310S`
zeG-V$x^VrBw!n^E+pdjh*D0{v;B%Z_Mw%KboLkrwUK?5y=W%HV8&uLg2zZ*P^7tl*
zJe&yapmc6=9Ckym<aN01D_(qC^iujQP|Jztk-rH3?m$;r|6TUz>D~NAXm=t4O2_ga
z$%EKXYBahZ)G3NZXI@308zGp1XQ8BY?An0<Zx$IbsZRfst70beS48=YBJpr-5f9oy
zomAm6pRcnqCMj{Kde~kc0-^&xWJT63?#25%>cDlwEseqFJF4TO%F)Nry#g~s%UkP0
zeo@kw7E6=eP_(-OSF!CCmY4gA72L*ukO8dGH`Yn+X|8YfqTI&82hQhmy%u{Lrz8^2
z9T0XTp=;wDyBvz;Jqc+vhH`H!rMQcaTs?5v6}kYD-~w4XxU8&R{G+uWQ!q|X?vc+X
z@FL<Moy%l}T)GFQy$|`lD9E?h?58v}eDp7Uj*ceQBC>2Z38}Oz2ZdcnJ?o3a_$PR@
z6db%kGbq56Zd3UO3CyKy1%{HOl;PhD5b%_Cr8$cI^<wkdZzC2^X3iI6#6T|I0W@@-
znZ^=Q>pmjsqzz?+6`>tYk=`Tuc=>Mc&`^_$Pxbu&L55rMpP*)uChQi>`g=$c(`@7A
z&S$)koqs3bvk{)^syFX@m~|&RzrI*x=H)yVrHG9h+X<)-Mui&5+qyXmKlavmF~=1s
zOkUA%t&GJ<whvR9iyo2JYtNC4#!iYM$*zPAvj?Bnyi1==1ZN{-X>_AD(*!q<Mt|^^
zlgG$Gcp0ej`Ws!YQDv3`1=X(6Cp;H}0@hzR;`Mm0r|0H*TCkha%?16`9PeyZK|~Xw
z>?4vy%qu<00FMG9!zA#S_>>4mF1!Ud{OoF3wYlMFYtO-enMgZ#bV51tnaVM&y-6)M
z120}GyD4a5)zf6bdg*8#!HjQRqIrGc@Nef4Gv8^82oKPI-pZw7dKa(s)d8~}RA{tj
zn)yaEcOG?&4qQ*rzN=A%Y)n*1GoSaPzH$0fy^jkIU(7z7gLLW=Sp(f2bvfi;P++&`
z1AYCKDzhCryl#%n^m*n`-if9?^%lK2kHlLkp#$%RJ!p)j;mE}7Q;iyXqHOu!RR|3g
zTid1VTDI@#{E%=?Df5Zk?3}Uiezu{AY=SU?Q#u>oXKcR~IvY-*NN2E@W$Wj`GwYOH
z+>!ex{?YJ6qZ`!Buy~lXZXSFtkuD}z_w&q~S`Y{CgpS9w^tz|>e=Wmo{3?pSQd?N~
zFm83}w#!uixRk~Wy9U!o&lUB}0j+PVN~Iqo553P8kY;MeD9~ueIA{qojJUgU%KwZr
zrAgn9i&9?mk?jqUg}jm<ihp)M^^|rdkatQ&uqK1mPeCrq!w@sXw&oGTqclwUFY~1U
z=Gr8VI2jD4xto1#R^L8TatO*+8m-gj=a`=@Hv?^|S9^BKRnVMz*zwr$0)%m4<{5uU
z9)yM*8F;z4;f=h1wr+;YS5xj{7033&!c*RlJId~1#LgN^X9_$)-Szaex!}V3OL~_{
z^{)t`l#D*lf2N-?x<F-7c5x<3g1MMAg97*h#xl1s8>NfKu9pAjP_>=Pn1Q&qit>AM
zEDxVEZdQ~#zetZoT$FHxF{u!yVVrxO(yTid<jUWQhvdf{N79MdG_)$b7TcIXz=pVj
zSnm9uMg^E$5x%pIc~Lk4&;8f2G-Eel-rkGIN_kIC<N;7M=lW+zwNF2HhRb@F)2%3(
z@?jcJ6Waz!XR}g3fbP%Q8kqAXx^mnmqU86reWFUBG4|}At^RL+0k$EG*;uML)M6bX
zi}Ve;=kk-lXN=vx2@$A5NX0V*s<ecSp?A|ZL2Wjoxvr15jC2>^cn8czT%O>&YW7!*
zVl2DuCBBO4-i-H`PqO-DmD)Ev2L9&wvIt)WGm2=B!0ttm={9bsY$9C<Gl+m1$+CeB
z9AmTAli|+Nyn!%H420(l)5o_>t}C#lkv{KWr*r6F8UXK0QR@*YJA^nL=wZtu(=#jN
z$v1fRWY#~M3CkA!R66*4BjNq$jtXDqN$-)a5C026@wP_IOWFd^TRuVk&<x&az$l~r
zY1Th-7-FndXnsD7aR=yf3<RjM$YBk;iEN<?hKZP+?|iA{A_`w^;z781pVOHXfB;_L
z0RXc_!Dik#*T?YSd%Lj1T~yy(UUQx=pw3hZ1UqS+sOX;9;Y~ExKq*D)#{@HhWGxs_
z(ZQ6x!ZGi6s70lyWrCjLx9KH1#(|Lj@Dsi`kK|jTIRqt&JYk~)%psN1wj$!0XX{-~
zHyw4R5`gl9F47gM-h`JkQ;^$rA+Dywl`WQyy88p3jlx~l1V}H>rd@3-k(`R=E7u|Q
zsjkZS%_;HJKiZ1hnm8ed0LR9j@6dL)eK5LL_NWVSW}mQUn}O!6uNqB@Twc$Y|4kHB
zag}7rHas&Et-U>pK%NT}#e$*nJ&>H_gmyG4K_)x&%GK5bf27D&lg5nbq&>19%};ji
zD;-X{m<$iCJ9+c`mWiIup<a~kNr=+NeWhqsn6)Sx{*lIrma&5F(>^LE%nZ`10VBtn
zkymD`NIf_&RH{JXKi+|AOzpmOE_uZxYMl{%qNNJw59rRoor_VNxnv-G7+BPjqA%`$
z=EUjF^ryM?6D2)L^;B2tsCRro7l+;1S_K@3MbI6&%D<_eh)(GMAr>5kFj2Q-YlPzl
zDBV=_pW!H?+y#nq0DIX3FD@2^_$#$30Nm!5KI{LWad&+V>*<>A`@{W*W?qHKQOWVI
zs-AdlOrh=mfT1cLWPYn$#}t*iwY<8F7maBbhv}=f%&98W68VKkVj9vUMcxq9p|twQ
z){jR>!2!ZDG8BvmTtbs^Q3_G&OdskB@LE!-#IL=Do`PA1YBTRdpdJNTU<NotI-Eqk
zM7l=npwO_RTt|+>qzq%xUB<&RHs<34KDEE@G*mB@XUL3AKIsiIs$tR6mRyq$SOkY?
zTfRdcu%L=MP`*sYj>{sf>2jFPz=*K8zEiRqF4k~2lnZKI5S`R3;ruz7w-yxY+8)dL
z7z0;Bp(p9iOl<cvtE>j*xHVCc;3aY@A7Y#n<dVWvcYC(hsc<me+Y1$O{SiyTrMr{E
zjPrDHM0f9S>Hft88N&$wPXOidP%76fOuaui%rHmwTydOv1Ns<CfZibeUk0cVg(?20
z*Gp!FpPu<i)^DZY4X2jJhDzAPE?t;PF$1_A1ocAjyp3usW6f8UfTxp{(d@`5x2v0t
z;n?N@%Ey_fv^@E7T(egPhnQG?d%hXwb`hS@!yV5s_~D&B;^o%D)lN3vJffM#0Ke({
zI=f?8LkhAo3$WZc^_jQ+@Gr8xUqH%aG?;TBz^ZW2orE1E$!hfuM6{*$h;2A<@+i-4
znI}(LUj5<b(yx)C_8|6siN%#qULi)em#x@QRQ1%l>sLZzDaV?<%Z_d1`P6#q&Y@E;
zeqXJ(pYcMxxZB+{+dbDd?kRdEwSTq2FO6sSaA66Dj&F5TD;3nXRjV2pt#Tb|PSz<i
z8>q$(@Q&ilx<;#TL1X6QOd?gOW5h@;48E?YzsOcVUTc)3B9CchlV?BRUd1(d(!xmW
zF00>9Xu(k9h+i3=8PuBaZfsfpX(I@e9}#dg;6hn!O+J#HT^q2k(%=JXutCqyBQ#57
zg<9UHK(qaugfG<C_8!caw-K+1lOB>+Uh&R+S2M=%nE0oM`_8^fe#*hYqVQ?6dDRA{
zEdVR@wS0VH&pZX3>QN-1U|>#Iz1;`&@mxNl`*gj{a88W_8K2E`g3oa1du-2UpY}iR
z+y&&ziHQx|o~rhEd;Wm2v!S=&wRz^_bjO#)WrT&-bh)-+<?WTn@2S)TkC}Hkiel3e
z`}+cXJ1(7(bd^2D6^MGA_2`G1ykCR1zJ0ZY!>+=0rNgrL+`DXNPvr4%tWc=DY3lcK
z`*7H5Nsb-Pvi6@g)f>Ic&p0I(-^97*fV)m>y+794Fg{csOOCOAgL1X#l;qgy%)RAD
z#^*tSK0Vv6nfy?;D*CUDXa5k(UdM6J{m#ulmc9K|TWmk$5Ngd@jz3Qr7#B#&W2{Hp
zn|fp4PTv`2qo)Jj$q%iSV>eIfjl7H-B;BeQ)i;eJI39f0!N8YbyCeM^ho$3YV%<oB
zj)P6>gMezbE29YkzWpUprp%6i@7lp}*!|%0biZQ~YPYYR66=m<S=0Z6vCnqVSC4&V
z#h6_Z*HvxS*Y8&0GZiypbnJ~fm|b`-!1)=nrY&ABjl4{OUBmAiim&9N1|N(bQ7Sm=
z`qR~AZI3S)$e{FsUR`k(dV+j03q_@(b79C94+WrnGGrCr?02dCg+sHipu)>+75PK&
zbIQh83w#w@`TZ2hC#t)0MsN>OV{{$x?$yl#U0?AdifMa$-**;sii}_d>_0d%ZtE41
z31+_K%et>ue~*wH+W~_25Ai#qhnU~!bF9s*SNDuGaZs)&=68+$X<RP-wJT|Cv)(hk
z1<f|aU<0aW%baFw&9zGU$1f51&LC_!2#iKp;q#twv>%1qQdNnEhby1)+C<Y^%hy5X
zy*(oLfX4HaP&`m-_2m;wG<3V0yZZh{{mbb@<ucP%effS{5G%basS+%l);_M6VEX{m
zbgAUU(AB#YisZHcy$G}PF<XMp2q_=Orh>QpEkiDOTaQTlV+OTU@slRkT5_}se~2F~
zbV7@m?<qJU|HG$}12xjCy&c#MZi8IdKi?_)D&D9u-SL8<n<+6he>-1@YOXXByTmAc
z;g73}yPMe`Wvse_BC67dQUaP!BA4>z;UUte3g^z|RQbWd^bZw^Lj)btNNYpL(2(DX
zG)Yl@v#!Lkd#^N`lDF|?8h(mkP|J<dSXsW(2Niez;c{caP_xG;KQKjwB^_i;|3iCc
zks$xMl_z9TVuwJzYhioV7F+FNH@7`Y-E?6S#xdUQx)T}X^e5=Uc};xWyUSc2SLE5T
zzm~Y##;AT2&=E)dO^-)ehZNxv*xMo;8=W0p+-{&Ue%lh?bT7zI*Q)<KmRuhYKCxqG
zA@zRw&Fa>29a$}!X6CKWd=<Oyo$u~gAv*pp%wQ|W7KM?2VY30Zud?O6s*u|cr_;Ei
zT6uZee>4x-<n3Yqa@sn;!!twldSc#R9PMv2Svi#5wV5M)U+&~>{dK)ui(vp^<vZJs
z0kiQVQpPUh6L&ZVwQWb$@7bBF<NvEAnfoW=kl57>HVL0b9C4_1-fE6Y@%@JlR{p-`
z<xX#{lq+2UUS>6p$^Bmq<t*)$&Le)wv0cEuNO^PDSIpNHnPw~wssiqvWt-fqgZx;o
z)vyLsk7guRq#HazwYI&!hQehWULkTzFK%`0nvEQn@((fD?ci_{5Bn3sNCEhy<?Vm_
ztm61t;qF(G-!U<Gl=O<#r&C#cd1o~aZ|Q!3icWfjN<F|>J@}Hf?+0T)fLs2)+7qrz
zfa<y~FsAfIDN<dRQGv3Uo69NpMk|KC2bDyB-SGMgp9o1c1|=w)cmGi+?pTcP;@0lV
zRn<+oVsqJEj<R7t<QgO2x^cWG!M&4LrJ4<!(t>r0ZgH}*EOaCl5uU%PU_RpV_Ex&c
z;oKIEch<E(aGYk^%gN{K6aPH+4`shX$rD?emVYX#+{CZ@N@V@#TWhvod)g__Apj5^
zB9ZnfE?;US+7P7j_LVy&+`>QD1hXQ-0x1*WR)rdF-=S<8_1vl9_Cuhz0cuqckNj#i
zhmm{UC0E7UJ16h*!&VzH9+#*iwaI%~5oeX1xRL+j%j1ZZhpA@g*820E+XzitRD^v)
zwKsQF>QK0U_zC1LE2cI)DA4sw4Ih1)mm-DTrfyB%Z&^t0MJ}_5lVfzt_|e|KhOD+;
zjTg#0%28By{l6Tt&i9zLHe_DnGvePbR^)$&wcAK+<ieq>)c*tp1{pLkeNyD-6x-;s
zt#n*4)BvD-TT|4tn+m=ev3F0_N^=-k9ptd{L8W#myn%mUfcOD}01szAQ$^dnPwZpI
z-GYFD6d_~mV-fqQv33iGMxPJr;a+=EgK~f>E039!rAyU)N{E`@y!-RmIV!~{=6x(<
zXXIE#Xa#{wRP`}p)$V8WV!{Fh+~0=_ut;?@>?L1-9gnEg1ZwqTx{DmuHX29xJ`{Tt
z$+T9?fl-`0agR&sz7I-n29Jcd3fC4I^cUQ|X}?Tjr(|9(Ta>$!gu#B4-ru9DN_l|t
zKNjezt#jSvd~y5JSGHogO}=7Lhm@*q1kS-F5Cc$kJX$3?V|7AEih(f-^V^#v+3(J(
z@x<ru(rC9dzvuCSsI>n>_pe~nE;$v2RipSQ4yjDj_u~Sg7ozkT*#k`l>-APv{20M$
zuov-K9$UqHS+^ol>ez^Z2g;OV<7P2n6VwJudYM&?MyT9rOuCJrsAa-QEf{ok3%3B<
zxV$y7ug!Nw8s1N<KfR-9bfdZaPzGXI6PMT{c={#2ICY7U^%oow>s{U+0+)Ere3$sS
zuTO&Z4Zu}?Fl`%uyWQ7f+nX0HA{?B~x<6U?Z5RJ^$5mL&i#3HyYcoVqpJh8_%9--R
znY|N2Q-;X6f)5-#HA(n|Z!ylpCqsAwfgrb7Q-@eyxHTaBIAYZHiJR<u`F(k%$NL*H
zJWpE|VvqFxMUD%o!QK~gkiRW7LRQZ%MGGQ{Kig)5e$)ji^Q)+5*Z=Zdop(U<w8kqN
zv9SyL=&<KA0x+`rGoY=_900q1`z%m@)=jfeTTFrx|I3%n`WK?MvS=D`3ES*WJe>Uz
z&0lZ$sfk!QB!t^5Q|%R;Og6LbMm5p@m6bc4isLGVFhjkEDYz>qKt|I+y*Hwma9G3v
z7a-Fjdq>E@?01n7g6;@WCGWA$-8nz<r{7&_4nIM#u$7+&dRdB*fr8ST)sK_OlOV+A
zl*Iu`mauOFHA`qPQQJ7L+wwy_$`V=;K*<t@QICVQ<-F}uxvSF1XT|Lpt(r-`kRohn
zKpjc&n+rWVi77EAOrYnJ<^}wcg!HDC{A^yljE@+6<#!JSx=fvnEer7=={g>o=S(UN
zsJ3oRGAJao1g%MA31iz#4a=d1+lsF;8JrRR(d60zrSj;aAL9M$B-bv+P5-$%mzprx
zSKM)fTc7tA+gX)75lX_)=}S!*#GSPvVPLqpJwM|-k}!aad|4^4Ef+-DuzU^Ns&{nS
z^ct<cH$5+6bfwarx>}2h3FwAi8RRR!^zcuOA-4J14(R%%J{xuQb0-A|x)(d<g^)!>
zu%vsXpDOCVUO)JKfO_crFkUG3j}C+3f@=%7ug#$sUq0FJquAS~n>6n0WVG^>`y&1J
zMup<OB=Ytk?&}WXzLpcZ&s1Z2h4Y(#p<L*ibPl;p8#+EVCx^3Uc;@Bc;nvFu?@fgX
zA&A3PvB>kb#yCdrGTFp3BI&2t#R=Pv3BXwHY`>r&dRXkFi?Q`f&ne$^bOM0vW1^V#
zQ{;<HnDwPcs#9j&`J^t45}5o7Cj!%XD@6|EB9E>H?7_+bAY{vDe(FWu!AORS4B%Oi
zC1~OzA3`qjIm1tTJJG~9UwUk;o5yrp{<B^#)(VTH67!=?ISzhjO&2#$kBk?7g*#PI
z*!GseH2FB08;D)E?@_`0HCULJ$%&V@|1dgcU@;Ql>rUz~!YB8uh$>YwBZmE(f<*Es
zB_+by#|&(_Pc(jA09;pL#SSpqmUJISAlE}G<noA43PC;hy_(?U21n!$51LBcY4@Re
z>-El9@YZ|9Kl!;Hnef&E;H{566mM?;5_w~>J0(Don=kxl6>mA9u1~1Zpm`DYv^}3F
z+-W9*C61y?d$}l{OJx#uNhh_8d*?ykB#N(bB#skHjWjhJ^n8893_{&asE*p>@9QDp
zsE=y0B90n>2ZE9Tc+@g|I>qlq1s=`@=E?~02*0V|M+F`e-<M&Q?`uMJ)ZcyIjfS79
zDpZ5j<#HxGm#PH6(TDfrn=>zbs+kvDQ_+X90m}4&M+C91>9Z|w)%sHZ@S{k0>PT#;
zdf|P9=YEr~BH7)~y*N^Zryt+e+l(uLm+1yxZ{k0ntHf3@0WW`|!b|Gy1@Q3`gw-9A
zd9T=_o+*t`!3Sertrr45RM)v~rh<=&+Y5l24-mMp82IYR+gm{eAB+WORe2=xDBy!~
zdw+Z%6=|j=R|vbUamv@cf?D;A-rc%I5bMdicu*?&;=dw^F)Bc4gx4?inWL}PeatZ=
z3d84>7A1dHHFo)|^?}LH<2E0s_T5``;>KlHw^DtyOL_DM@X-jA2Fz4cxrssdRmA#M
zKPr^o<ShNqi#i;VnbnrM3F!D9JLzOc(gp7QneOy%)+55cuKY!enS44i+pqDz6tLxe
zrdK~47cAsJR)}sPmjb%Ps)&qqeMY@ZdP2Rp|AVY}i!q~0`X!=AJ$R}il6cVG9jPsl
zJ3y{jFmJ9zMOjMOwI5r%1KCWO@r`$M2j6WCXjVlmIn5x?SrDsWStz`TZM<v~TR4{7
zJ6aj0i`kys<PLb_5$babk{RlyFl+HqGlmlX>#R`^JkswSM{>NT%^br&*pwc?dARW%
z9@v)qE9mEEChu;4`sIVcHsQ_dhJRQ3taXg>^kiDBm<?IOL{5ClI_O(YoR_(IDkrYV
zeZpaE@7Re|m+JsimcK&gyu!XpEU{@VheRgn!XJxBmSWcYd1AkH0cU34<pfu}tv%rD
z?b#u^(Jk1+y|u9qwED?!!@5%Yp}+qTT}r<Z^Ds{4g9}t9ML1R}q|Q^B+~KTE(8W3s
zOC~;SnW;<R2ItFiK6>KqW{XJ}B*YjKepf6TGe@_zAIXs<$T`^$QnsoohV5<4%d8#%
z16J`}5ssBq?<Xrlo|IMaDv;n%H<=)zT{@)9uT#PEd_FmI!(=kJfKL`+&~sBOoaU3K
zz%vqaH~o1b$|MR+>aj~1EYTzdGC#^{Uy32)WczNGrS{m0s0PEAJ@bnSj}6ALHv$78
zm+TH;%0Is=#rmZ1FKOnJX|LS$w6|m`?S<$Q6bMCgB>HfQZJI0XJwKK9{s{kon)afu
z&6oB<9vt7SN^0-jom<~A{4_!41*G&+LqVt@J*@bv57yFiR@Mde4b1abzx3Ijcx7X9
zlec7Q@1W$ke`As#U@0ccypVo)jLG&k;*c#1M=QV~r-+V+m;RyA0<m@xiLcs3>{%f3
z?cRtK+2_a18>vP1*e3opVwrj}*xu{;f*S!jN}JiIF^6T*>!b2o0@<JxjVDdy30Dw2
zwO1I*InwqY@b|R(@dHJKi0q<s#Y}#vJA6(McM8Stw!T^}_MG)kiG7>+P1G+g^WKcq
zC*LteJxw~O1og=s(R@gK5+D?{KAH0Ou1(Z>`(BBuk{$emgNzgPx?R)jb$6%N>mKgp
zz2U~e+?>=FsUyc`pQ`Qnl_boq;$A=}f-0`t$;(y4L-MQ)NNv>*F#H|%`L0bMj}2?r
z8&N%AP&jG@UsWinB~4%4m-X2^ncqZ>?*A;kU$ab{>!X#Rq`&F`9y4Xz_xO`5iqBa>
z?_{x#m|x$#&DySDMVV>U=w;l_U860HHxM)Kv&>5wi9EoXCXYDdj4*86dd18Y{jWF8
znbe2owFLdP4q*KyQ(qs^vuvKmxo+&QPY&TceBrH`=RuhfTbK_r7N0wQwaegEfx!8*
zcRW%%4lSlgB?c`JPq%=&?qK<gA~?T^q(0nwhcGKN!sznEu4OXaHD0FGqxq4;UnJ4P
zy{UOfIU}ZFZ`Y94h#!2@E-i<OF!JEJm8*j0xvXcUIWXc6|L?wPo!Fy#qSi%XYD14r
zN2jKou50l*b2`7Ofn9-pPQrUkYf*qGArPGK-98nb`=rhWy{<mL72}ZyMMi2KXx&e@
z75|g*#lMluL-UVPa^1o~PcFYkSmCCna(?q>@JivG_(M6x`)udmsRqj*%Xr`ZO08Z{
z659I9Ji|$sB?+BEs+67jsfd`4#Y2tNKF_%r?+=cttq7~(3B@-`HrsziZyz<cn`oSP
z#GVg>#!05x=0!OszFX8rU~l}^HDK|+b5_$2G8Uclch09Y0?)XdOp_(6=Qj&_C)9_(
zSCRUQH5z&+Q5=Jkiz(9}8x_pFj7hY&v9f=UfQb9*z1`V@o7vKS2i%Yk1seIJF;Yq?
zRQPmf-R2!?e`w)t?%+7$uDG1OZBqN48&A9RrwKzZi*Ss`EHc$f=i{2VqtGyMDTu;z
zoHa^*FlYPEZ@XCHdACow?3Ge$wEHp&9PxHUr;gs+$Q3=GZcw=Ucb@*>Lr`vm9S@l@
zzK=@z-tYb^r3t7pek)B#%wC$<cRn|JhzI+&_u^Xs4+Cbq3;E(!)4ffdzE_h)sLoyw
z%0{Cb>RQ7E*~Udv`&oPyvFYofTb{NJKvHyJmISq4u2j0O5bEVmgqeiF6HWAhgrR&)
z=>f@6%g1}g5<^`A0ui@+gz9#07pw!fi}VHnBxxETG1$PH+8g-5K6b(_2o;zSEMx3T
z?F}Gq_cotGn75|fDF~)1R%9#d!@@i0^}0FWb#)`FWJa0t*KUTS)xX@AO$skz|3MVl
zCw`3Nm)}K*aCvtYG27$QYfZ=ErhQPQn#0su6ADVG-)c>_P~U1z3)UjE#b&QHh5m3p
zw*<NtqPPf|YPn_ten9-nQQ=MWZq=nM)x4uA4WB5Zi(sP_y1n(|H3%@;pxzJskRdZh
zpl`tEJTUEZ?xVIp7V$ZurFUX6^(6^vH-NUjw8lcrpd-NVvmf4p=+W3?GgcBLuc%zQ
z4`Ex-i5Qp?2<aE>lMz3}9NqA_DBsszN6!93N)W(>!Z4hUr#~Mzb)(SJf+8h%;{V5u
z`XC!6`jKPNMZLq)$$kE+AE9vyjV*fJP*y<<e&nVoCeu|^H2?{sacS~D4gIBdkd+ca
z-3urT!T(v6WzA-1e#<U(IWzk)*`EG($*+llUG!F2+#>Aths$t@1CsKW?BkISI;hhd
z)mP!?Ud@Z|AA%a1>lLJXb^V~y+gqN{mhI+<Z+o?lis=<h)Qy%^P_Q5P&5NeKS4GdN
zdG_Za*pJ0|68i<u?}P#%Q`1pQpf)JKb3!+LGdRrT)rD}$GNePQ@}oY=ww+{2ZuP!W
zI&nQzM&-F`q(+0nL?5J~?W2*++UQtXj5vyp-)jhwKL`LRs9+aI^#8Kvu*q59Qa%+u
z7+vAj8BG3@`2~^QG!05gED=?5qnbFUzI)#4!CQdt^)g4tvM^teWh7FSb)4MMJ8C6$
z*rk2owEz{+52Grs)GnqN>biHb6T3h$RE6>N>d25ei=kP$wZUy>x^h=Po0N4{lWnEq
zn}X<sL(s^OOSI{0=hW#b@5vD++(s^F4^&g2Ni}-2A%y(C07w|cOaatnbe<w;iIxDR
z2>SUJ-&}lxU%&AQplUkFC)j|{_a^xSAExsOJ_UN8^guB?*buocByd+Hk&mt@LoXpW
zTWe<5*cME@)_B7g5;MmJUt0;RlVxEWlmsII66t<I-O4bbKm-bXV2(NprQJGJNBLLj
z13npw1YL4OW8d8>r25HX#z_A@4Xg`T5(uQa!plmyR2fwycv@0wpS|ykb)t8T2SLrq
zhMUKEbe^n?ZteiRB@{mA%m{2-@6&gtNJ;xo=15}gRBh~pxY!04^Z5uONU9ArPc}$X
zw6PsQn2O#V2;zGoWN+K1@27~z$+Tci0LhaIfTvOPZ&pxFi7l9@OwfKN5|urS3EJ|y
zazUR>5I<WZmEM&6?P3<>{=xow=WHCWX&tDw&~*bp>sUNcmIgIIf-hXmPon{;&3QF<
zWp<_cKxs@HN@L5op$+zLeo#8eAXz!Jsm+l%VcduIcwk51syrf-5X#R)*?w_f6pAMZ
z6iVq2iwO`_MK-!ox7WD!e4ew8K+>-9QK%^xO4Oh0)|@pZ^Pyy(GJ#0B9$}M0xt{og
zO<}GJQm(&3&YLXP|NZ3_wOr5mS?f5I>vJn2_AjC&CgS&UXN_Ao`&jp&pP@hDqD@6l
zRbRr_l|j=QPdoV;&aEt)hxI}!V(%|QTa32jc*c_?)6dy^LF8m@MLf}w-l(A@*rb`(
zh}yHg$eixKKntLwlrT}^q|gG^a68LaW5zEBs9v_IhzqY4E8>%2`<9bkQeu3wI$}1i
z-t=}NprJ3&bLz7H@=@Kw{wA`|I$w8FKJ|XmaWL8DybxR}4&|PK{pNAMBWy)IK;iEl
zUCj5qwujn&S@3&q<~ZQ4&OGrwYuRP2#v_GspD1QauD;+_p!h9Q3G`XE{o3W(=+e_m
zCI|<htjx6LvMAveVze5)?>TYzg`oZKI^HbLex0A0E%E#OdNtDeM`%lnVYC>D1+u#T
zv?NB$Nn=#CJ!zykzgZ%sqos2^vdZISQWKq5b0VgQL{y>cdDVkA00>fyYLJZ_(a!tQ
zk3GZ_94rVU1fN!Bu5o2LGwbbKxM1?F=Wj%zSyFrcKGdE+Ftn!i{5hfL-{VVRXdMx|
zg)p>`p8uMM+wHatl{Xd#RoV1I%jIvc=%g=n?i9ibue=p{wB6mmSs6m-GEsZG`n7(x
zL*z5jE%HD`!=m|D4jmU0VkATRTVY!J8(~)}0=sgGyfWGT2JRhA`&;J{GnD2RL4_Ak
z(C=2AjS4TuXqGnKpel;1Ymcu_F(X=XU$Dyf2H`Il>_OE9zFOfNOBrC8Mu(r)CPi8h
z5mm`UDme+R2?62AZm)`{TYFJhE3V@>q$xw)=kzxJ4;-ha?{mI3|3BX6#%+vvS=;f_
z{N`OL9mAdD6;kj0<GzcaEQ}XV(6FSZMNsHK=xU|x9&^u+;Ro>k%3mh2=PeX8*&rRm
znfzGGqv(<angSCo;ByEg3P_`iXaPe^*}zeGY~dF0U@oQ2GFJ=OEs=d}D890__u_j~
zY0Os=`4NMO`nTp;i3iG!PiyV)*;^8{EdOa`K7?4PhRrUH@jesG4cBvdRD+-cUcmkn
z`86bCM<7p$T7#`n`XBEUq;0y#4qhQ1HcUWDV5%*^Vf>b3Q=_LG`=cYXb4DrAld){f
z!g1I(a{Qhn{-V7lM3tr<U!otck4$ReA=b3WlzwLDDDY*T>z=HE>FskNL}{;x_(u3f
z_*V1Pz!simSZLVjB&u#Wy-igJ*WMm~IZ_&X+_gOVTiXG~8CEMKZtl+5U_v`;G}RE@
zrMSOL#sX<}m+6i^u8EhC1SH>Qod?4Vo7_olqOvJJJ5*e!hEJpjMo(=>@75%*eSspF
z@6~N@p0&(b1Ot@P*g+&{$W<Y!=X(vQkN)xWTu&HckV<WG1_B+ary2EL&dUB0@<BiQ
z+Ex!Y0G&(_d{F`9D_xTN%u;UYZ6A*O-w{(bZg%OG!Hsg~8_GZ+eXmLSgE>B|^}F!m
zytjM~T01|`+L<HmfnpU?W?Jod&i(?j*Fs;I!@1BKMIL~C#&811EjJ2L30Xv@J(F(q
zuQ4@#R`}}>Zvy&#Oof*&mV7|DqUc+yy$H*NA-Vl!V?ASqmt9L6Z}>nRVTDR{UjtbZ
zFnkD5UYk_#u|M0y?4|gk#Ho=C`iVE+kUSv2<BU3+sEj#(UOV1(6YORnXnbQg18Hss
zLF3WBb1REd`+FOUgR2JLRDJF$ga`S)6*j3L7rpI35Y0f!6v$oqyHDXx-Lc%Bf*8*H
zzKu|j&j}c_6U7NR==uOde~(L*F&OQ&hf_d4b2`5lxMM{>@EoB!q7znkD8+aK1((_R
z%xFPpW0Bt&L=hU~bErX7ENUXb&5iFQJ_=nOGwOwN=nHSLt9bYwj$gzU-}bz4ywi9Q
z`I@0Bqz*qa4!NDsRe?zaEQY`3LeK(0{{MRN0<ygQ1bmIEFm9whU2Tk*fDU0!<SwEI
z5+M_U+I^V?F=QL-K2PUHb!4~BDy(UtRAOtNScA>VDdaZ?BcI;peXSy82y))u=d_Z-
zWyna{7%UQp?4!`VF{3&bwu7A3Z#s_B6-IvycWVh$_ClsudmrUe2<q;n9IX-lyi8Je
z?`W4iyQzxvzx_e!8$Iarg#oz1b6lNHc+%CzKHT_Y1yfhLA}IJFWyrK4f>x+Nr8~ND
zH}X9hR9JDN|IG^-IvL#}D)HLn;$&2rn2Y<KfIo3T7c+1`y`aww792R3n3JLm9QfC5
zyVezEk;r0aLqr+jk*R|&Nqy6JIxlHq;ZARKMd9W2R=jTT&j0|=r#9%}?+K06n78Jr
z{b_R1ml{0h+UBC?43<<-2wMB*oH+B49C@^N_Xmu0x+A$t>g$DkdT*yzUtUFU;+4}{
z!E2B$#x{iij)h@2@@q<1Azu;r=r6&+>P()e5Vm-}$;zu!Dd+-2LP!c4GchEzd#{i+
zS7*_HnU375a-b1)5hO!R<LM1589T9+aw)r#caNkt{uiwF45JF`IN^7mLHt<nX#5>O
z%?4*sBxoajKRiL5?pw_81ZDPX*5Qd+XTOl)iFXsTUrj&0yrq}OvR(wYvIB;p%<O}_
zQaak`ghDGLh>dJsK$a?MB;JBOzE$BFL~7Hk0x)QS5B>ske#j@2?fM0YLl5(eG=#QH
z`;Zg!1B<C8ARqFVPDRoC==$ELV6sr9#<sw~E1iB8M%>gQs<=p5jjhNY52;|j_DdqY
z`opY}>AThMlgd=2O8l2k?RDDp05DajO%IGsO%K4>;<xDmV5%yw65WXmYX*8OBpcA_
z`*8bk`1i5_K|{(Js~Of^(;TWC0Ja||Vz}?|ipteCSEjnzivxZu%2F*y+s*J_TFu7e
z@{9Q(-)0b{kQ(jk^Zl|~E96#RPbspUj4ji9_j~^TQ85bsd+y;m^W&YZjeQTtZ91MN
zcv!cXuzgqd<3$xcrcp^i%6?Rpg#E?^;y?<!kRypB-fsL1U-#3pfYi2ralb52L*J}c
zN3~4zH+AMlG!z9ow)syf3ShzpD?1hJM1~sk4fl^i@lTT!;gu$fXb4cLQq&g2tOb;*
zF>|TJrhA6E#PgwQBC0M>;AD<%sPimt2*LUb{f;#iM=5hLyMClH%Z>Rd5%mA#cm#x&
zeTua_jn~_BqtU;|)_8TMHa@%3-C?mfrwd)piP3T&DDRJbDC~$KXa_SvbWzvo<*(8W
zNFB{)+vHxHV9Hph^LG~`^`atUJ}`5;XiE9Gp!2^m=F^wHl0D?FAIvoyNyEJt3o%eG
z8oS8tB)+(7sxT#DlKnB&61X*3TXbK0Y5obgrrX_Bc&dlrzznMB5nwYubv73*opDY7
z2h|0{e#Z)4Q8>9CgD_>>ca<4`Z`Ty9e74HiyOS$XXCx2dzKMQtYxNbxX%6Wm$gG6|
zN=z)ZY@Pi8Kz&*e$k#9Vqb_|;V8(2o)t_gcbO$|UizxF~^&G6|IT54QHe4YwpF29>
zeh*PSQEK&@()kF1`uFbions@7H^3F$y)M9FHb#QJy_&CUxcn0Z^FuYOrq^;e8~p!f
zDKt6NiSFO4Si_@2G9Wep(v=_hDE~Kux%Yg<g{!f5pRQ()GQBPLP4xoU4G$_ZdkO<Q
z<uUdfp@b5j<7<0GekauF+aSa^O2U70GQ@lviC8RaimuAow_$OhhfNKOLn}f~YA?#5
zyJLIW_zhKUIaws25*(#|7<xFLQ`#VW0TJop3{!OhLdS#*!pax+EyEu_56M`xRmU~j
zBT>d{0t*HQ>yE7B{=@NhIwycZy_P93*gak9;o&!_2Vhz(nVLaSR+^+{&`Lf0zM5$r
z+EJ^<aqPEz!2_cmcWmG1OZ^x=%T{y$M9dZxA7B9a1R`Mr)Z;1HA#39T1#(k)M~q-V
z_+IEmNDHaoAM|w{kIC=C>UgH#`<k*>DQmUMJe@SUCj=-e9()M-d?CeyJ1F1|{rK|e
z%LMJzqa%rpCmxO4jE#&cOED&V-{y7(Weda&Wvk>NiafL)34|hw3nZES4Ut`XC&Ad#
zsk|@N_qFbm0t#A!o~BDL1n&Qjr8}zGM)&=4vc*KQ>qrx>XHCA~LFAiuVO^MCr<Kf-
zvI#vwuBb<VFQS)4ds{a~H$APdFvLFV?=KeB3|<bAl|$%Y@K|Zymu&Ja8JPzH?#1MI
z4zweelkG&|^Ix2Ip^vR0=yshst)uC>A5KT^NI5p?|1o#v@lfY){I?ytL~R!#9UG}!
z$yJW+v_(`Zg<KWMO+t>TgtpL@oJ~|hDoL)K6B8Oj2+5TO<0!|VanJAh&f#b=qgA{A
zdF^ZMYah?&`<(Ch`$?+q)v#XNlFiuU(Gp>9CCO*4x&<SXyF#m@WadEDDSu9pE}@YB
zKXpdjz88PIvy#4|IOIAui2esd8It8YT{s@TE_bSmb+{ix4zxZ06}X2jj$<@9C%3u0
zykLDI`PNt>MBv~^l=a!ZD&C`k)4{v4dN>uwGzDP#%iq{&+K<nLeSB{VqbbV|A3toL
zRrR&YF(+(qU#nPPpyMk<vqXuTO&>Ux!>!FcO8mu#j3qsE<A%8K@g(c7(ZD^_N|Bif
zZtYv3xI(6f3n=f)_^uV1RRbJ6xRq_Fmm6D_h4}#lO=^e<m6C4;5EZ$Xn#smJCvkb6
zEfr+atD0w(zyTjxuTALAEiv^`WIMZQkXQM9`fnFi-vTFsU>5n#mbSR-jAPSJ7(7TC
z4Ym^-Z{FdNwI;8!kdpcEt#PCDW)e?cVQr!5&Abf2m<A1CgG=3=28c6Y@cDTyjnA)r
zM<%&lna8GFt(s@t3GaS7m7lTZ3P?2^J+xpNCta#xY|RyrYFJCrT)BO6C8D{4&_<yg
zY<PHERv0Z4?T~)b07gjU1r|~cr-4T~T<}gL_%_XG#nx2o)}+7};=Qst9lX$k*WDz^
z^c(;JkGwy4Q<djj^u2N4=?FT66GajWp+kV0&Z8T2Is~i^hv*QysVp=q9RgTrR5}Dm
zB{8By<n{$?UAZ_nv5e%A6Q*<yK|5k@rHKc<{<brnNC0ii^ZF<ott7|D6qx}4bS7!u
zba)@Rcj<uq&5g?B0{yqSag^eSHYxKEin~E0Q9fFF6-bm50j^0wtT)qm#FO7~gP;r(
zvXhFgnID73ke!6$h{tN{g$gyHpv`4v02yn&2`i&XS-I(1ePp=-#^G$o=lqTNPSAl1
z`N||)1Z*@_I{RjXi4;uv4>WN+Wlb=V({k&rW|VHv6C~sWl7l@O0)**88weJKp)}*d
zqa79PU&NA7(KCQWNSR%0qN@~_hGAmhzkPYgjU#Pj1~*T0BXA}e;O431o$+w<Ffm%S
z#uP}KU8Ng?B+JSmd*r|F4rm^>kqP-po}(R5-L?wJ4t&n>;g<(>uX7-sf?Ao-fLR5T
zz^vRW<@Y)(SN^-mg{o!kQ2f38@>WZ#XHkxU^11p`E0t+fk2AY7nULtS5GN0mtEIO1
zs~ipN<2bP~do8<4&>up|7$Gq7o&L)&fAyNu%K=0jW)d0`z-&guVg9l5Vk^7tDwg$C
zKCT31(~Ut}5r3Acy#^@fr2NUL)=`O)NwoVqGe;Go?5_CQn#WGsAROWisJI&a<S%c(
z+t8_!n1qscZhXk6-Vl5?vCQiZvR%3b@*aP<=IVeborw6F@l@JIX3ZA?q^CeR)Alit
zgExBy*N^iB@`DOgctMq{@gevA0|S)T<{HbZmboQ$#(Cu*lS4g^4&F-8UVp;)=?$uM
zms0L)CiFD7JgrZ{YZyca3v;3RiEpwvP8`^1m+yrUg@>C^`>8V_7es#OJ{6iGCC$If
z;7%+Xs=$BcBK*~LIo<4omIZdl)QPRzip&A8X9H!JDn>UQ@Ydr=EGiDq7Cax%{5T85
z=p>i=@FSkp!#P(Gz^ka|oW_c(^m)-Ztd=1K$$v6d7@zS0I)6x7G20sB`>Ljj(5EbC
z*t0l0&$Cr|D~Nr~MFfq-UcO{b0T+9wD(PHvl`v1WT;KgVyX&Hvm$WSlBBAlL_Z_S*
zKC1D7;?wb98Wb6}5nEt7Ok+r&i=uryBs(_}rg4QCZNarWka5w*lMpXTKA8(^l2KkL
zCt|aY%Kguuy=|%w{-rAc!dZw=7YlPT$2@BDUa1MI*aP0W^QK#5uOlv_vHFjtVe%=Q
zMv58})VDNE!5@cUczhDHP$4F{$Mbwp=Wdxkh$kMqm2y0(ucUed-uM_I#SJ=p%zdnz
z9xc60!Hsp5rZ&b4Qp~Nr8PqV~bFu(bP^IpeA=*qfZvQTf0jl}e0J*_QRrK|D6YaoQ
zt~X9etTCtd?4CpjZg{4?(y6nd<YAmlGi(vf5j>T`+GSzdx1i1~(+9QB|HAMDyc)oY
zJ>GM#PE5ew{!?R_*)33-oz&9>XmMtHu6^XOz)q+fUaW0Rv<QGwS6cy~QVd<mJXfGw
z`8&8D=#@)1Rh>J%X)fs?S+D0%b``>(d>))cz`R&^Cg+kn=r?of=rC0y6&#4wsgUU_
z=l!b7`}MJ$c>(lj=03@Nn|;BbrXa?Zu76ihk+Dz9ac#Hf{4k*qr0B()fJ&QrbLxqn
zli4juX<xhhiJ-}(RJmsqrxWj}B!Bf+(Fi1hSFfz?;MWe^sEi5rXY2ak|Hz$tc5XZL
z3B+u%#zGBAJ7^H)@kCykVN3i2{ZDGr+05yzn&+z$_POOlWe4w#g1DH;z2ZagiX)<J
zc&k^O^EHqGB_lnk2-*4BRjQ*o^{uu|5Q3+C#4sf5ii{Zasj{w=pro-JB>#%$l?EM?
z3woXe&)BT|N?gb;YFJ`UjT@fvUx*sFb<Uy{(6wQxh@(Y7MVP#a%s?9KXqON`1Co9T
zfg)@gwm_JZE^JD>gn(>?UCRqsZa3%6wkkz9`qby!$^{rtsxz4sS>N(-=}jmK$shmJ
zh`z2K*am31^0lo_#Rk8ztB}tFpRzF*=&bar5MgYM(7Sv#QtuKwmY6)PrTNEtm&mg8
zh2Z?v(@5)}gdo;gtxwvTxHU<5x~bc5!IRKrZn&$y-!#yXw82cSqb#R`2mG3V46lPU
zJGH%FMwZM5^5>%}{_l{xqybhK-nkeTutM`LY{KvKm(@ihw%U_9IW4v>rdEcGCC4bo
zJYKy^@7SQAea=TCum${@10`WqNc_{??=?p+zv*%zKT&pp=Fjzm`X3XTQN}$nl=q9u
z-3?gJNl8v~s?N)6kEkIh%9O)i*haC+4o~8T(g9Kp>YUiC`uqQd*$URga6yfvs<5V+
zV9?RB%63(PN{-)cTApD-1QAh;_Y8M2=r)N=9;mN=PO()|(dl`I1&mlC_bm-0pBGE7
ze>B?GN$4+3;T#FEwIDvr(&4$onNv2vnlR0|dwU->r7~&i-0+!=vJ}Rl$~mAA<iZAm
z7g%^sw>_UFd_PEK>hKImJAMe(-?BDD&K$y#b2eY{KF>S#E_dmqzE0P7m(BtaPk3X#
zH5i#e;e>#s&;D8OHB0^r*c2q(evP|E(CV3!W1x1<<}5)lh#zYz!8pCp*lEZOk1Y?D
z-bZE*Mv)lOZ6CsxOb)B_IE+0gTHZ&A0b^ntkB-$b2E8~%EPG5I-V`ZJAb3y6n^)eJ
zn+_@xd4jqMatL5>=R3Q7nU!Ce*8HdNx7^-M>k2M44ID_T3-<)&EiFq6b>OG{=^2^u
zl$KW}E7@s2_WcINLY5$pGU9%NMn?gy;@x4E)N2DKezie%{0Kgu*@sI}vfOlk=@Y6Z
z_*r<kpKtQEIvoUkF|sgBdNs02HA|7#yNY#~_B@Wz#Aw0JDHNKZ+X)il-34`G8~e7V
ziuIgD#OA1743pYoJO}0#Lu%m)dLiETu+AUxrY4L(9|5vTMIzWJm2oxy;}IKwwa4x7
z0>(^=NzdY(LFJa@9<P%PCTf$@6TYJU!nGRJQ;lsW?u4mep5U?MrXm1VX?sJzhSdip
z3dX(Kq{TWX4PHR{Z}+6KHl7^*V{ZPeDPOivN}_4W{&^L-_CfJ7xlkuqDK3@N0!_m~
zD`1xHnxJ9%&Y98VBrEBiDAx3x4C&R`o{h~S2q2qm{Hkh&@8m>g1DC!q0+9(_J0Ac&
zhJl_T%mKN~U^+&&7JE_e#dyIzY48mbB96n#3rOFR31@qDR(?K@CEAtf!eX@l4p&-y
z)~ZQKxwAcV`HbMkxaYSxmJm0>qgCG66aIk$%kYIA@JoF-sK)rw{-gneX&exd{HezF
zc!2on<RsL77c|G1!*+C@55F|zPd!{xUme7t`dpYIPa*W=yDn_d9hzD3*}mn^QsJO8
zJo*YY2Gs(1vvT?tnBiR!ieM$)P_(<LP8y=@CaNsTr_ON2*5e*edLE6t%%|T1Vjgrd
zf-Es9)(bMe-ue2={=lTcraG?(?C0*59wiopTt6ALN>1f#X*O~eM&02#R@}hwNvzH4
zi-np`<17r0Cwi`HX}f&J^RRtPKo&qXNcD;@^}5r|i`45*a66Dv9T+ZH6_4$~<``(W
zQP;q;E8JF3>f2-u%lN3~X)14(ERzA#Tz$ZEUf;7c%*M-2Qb=Nrkz8{Ul^cs3!6|E?
zD-ysNt9!^9E2=0xsG`tC=|Sa&CQ46LxjBGUq?~qQd|e^w<Y`v8jk1sBo9ha+a_{w(
z@){Y^&J#TjNVpa2Yer5sF{`n9+XHS7VhwL_40hFVB5zg$Dj?oJmP+z%c~&1ix)1w-
zn1u;o8Jxgzoc)6514c$#ORi?i8_&Qks1)SR$)t3UcNMJ$IRe!2B*#`8;e2zD<m|l}
z)O+c=@`}JJWg%sh(yS7P(cQ|wH%wR!8*j?WQ1vPDeSTIvxu7M4zVaUb;@}7^jYS19
zw*ACdVS7BUcHl=C=60=ZmCJ<%zZ<eU9n-5n8Kps)B*ewYkg`>d$`Qb;Fte`lSXrS+
zr@Sy1Y=aFKat_|9>+ulHJb3)3OwU4Gl36m}F?60J+=gfv`amanHDanHLy%QOg%8U@
z6UBU#a-g7NE%8{X?$=kU(XOo6l<Si!h}Qcp^^TXf=&Neg2&@K&#;(1l#<TX?px71=
znjh`ZNsLRTNUjozxZu#g3gR{oAU4*VJ8NL6#4WUs>7xR03CtUR8!LP)unLUxUW;=Z
zctp7qk<p0xfZXR9HOia}rDtlVB`2Q=nt)PemGuk9ygOwm1#Tfhrfu8;H<9rgSjp5+
z*Q(oUa*DW4xbryRuAh?=va>z^H2BQD?|!dSb>)QiUSa)QJ+BmF)AOp?d~0t9{yIOK
zuPf6^);vnr-oWus@i)TxrtYHW+SV?Ulml_5Ua>8J=A*w<MH(Y^Duq>@J~~p4z@9|3
zRfms1jxjt~iA0}!ac|HLeMJA#ny=xJJsOz$hL#@TU){DoVA@hn#cTPwH9B^duoiR{
zvQr_c0;dpE=9N*1!f?TNO%Hx!c<$qbARJ!eW9GIeZArGoFz215V30k}IRzVi`FoS9
zfo6wfG&;2?R&_?@SRwD#7?-3{4&+}IL@L@*^7TWGbhx-g6C0(0Hh<pbi#xPc?^~aU
zWy)}jsJ8hT(r@nAV(L0D@x|7W@>=`V-K#oYMd+TeJA*h9ffDh0!<hiU!O2y)hr5I_
zEZcvu3}-N{*?d)|*_OUYMKH*T;!3jSEKr1J{<ZGCR@nZ<UPU<H7(UCt*0O7N^#T7?
zzJfK}<>AzuI5}ZTq?CmT&>~$4H5r?MYd-lHwj{uYRmX8kVOyBGuS7ZPu}*Mzsp%cZ
zKB}f_Gz>_(*0P&AI6Z&SF1sakX~2JGY*402tiXBHEqaD+0j<A#Y<FtfnLnEa5B6#0
zmcFv!+a!>|)`J1bV@07v-XNLJ<T_}=Fr}igZyEDem_PoI>P5Lq^Ie0k+0PT|6Zo&@
zgz30dw%_T~jyz#tE5m%sytOo<oRoUTPK#pX(y$D$&sgmK(npLv`QsT5$b%J?kaVmk
zO25~I-ASh*kBoBy(;wu*KdzPEr--QL8@Y?>n*h@)2x&$3yS>}~ZVFK}-=(-oN71~h
z4_!tKdg1Wvx%WKPDJTtvd>wtJ&w#NM-aqHiQAK_s69J#1_uAG*r!(*=iUZIsvf(fF
zD-#WU_R>!FF+Qt`2rAqkbLhhi`XCcQ9~cXrZ|Tin+mYXWrLGA1Cap>2TWoM-j%jD|
z{Bf-;w%j0C&7qx3&cRL{GD!-M(qrzxY2ZT&@3s&3mZR=kz#)XGAaXEq*Me4~FIXjE
za7S}vkSmW<mQ9{nLpSFN6Q=z+3;fwprNW(Z?&$HrVvZBWt<Ztw!U(S_*H*n@7+di#
z-4dnm;?wJ6cs1qNxAr3%1MHL=s!aQFp~4PJ2cvF`kjfcgESJLaASh6?v-&RH@ZRve
z2WcmV<~>lj56^qXJVHosCz+l1j3*#ZpZEOlHUjig`rKy!DV8L^PvPxd$Q?{-H*?Q7
zXccg5{9KpgwH)q-(ij|0Zk}N-^tImT>!{3J;+rDb1O)E{8|jk{xwG(fs!>>B8=J2h
z)^L|<nEYp6xpLmPQj8#j>h4P7a%v}{5neI~nqYo|)$9Qtm3hbMt6AzL)Jf3l`&3w_
z;u0#|cWnXuf!)1zR#DhUB=qxEN~2o!;VLTACa%Bg8X_sF-4XAUOEL3$3aEUjATl+Z
z0RCR6FD1wVbvZ?vaMCwArbwEQ2I{(d-$={ue2?));4}CST=u+QGo5KGH(hmQ?~X6k
zvOl67SOEOLC|?;F$JNKs%93zN`ajbrYj)(6KgH<xoJCBOCYRnmu-W0|Fw5~N*U4=Y
zD5UN#cy{(nNeU)T=ALNg`xEe7$$vbTaxB+}GjIP97p~MZk~?3%C<x;q=ac8DWi$B1
zGdRN(@&$8>+rg*1@1lBM8D#}gY=lD<$Q7hx6}FCWfTU;(da3h|oCi>^J^k&_%C|@Y
z>M%fDJH<RD&S5v0*;0jx1mOeZ_%uzQ+1h#I!`U2n-N(^qUb$TBQ+Iuvq7OkVlS534
zm{)}g+Gv`*O`%ASG6VqS^kIREpM3L^4k1`=Bxi5mvLq+$WaH<Cyq@ZE)(08J!;TzB
zLn)qhA+pm7b3Scb0OBm6Dir~a?}!c%U3bHQX*tPbXs0ZFII2r|uFOsX_OieYO!29`
zfk~SLtbSIX)G9xCtIq2A*>jzVWoix3D-8#o359%LSZ}%Fx<uabY~N=q-!R0J>co$k
zzQ7pgtwa!zQ{N&_?ik0=6ri}Dl!LpD?oX`q0xQpw9PXYljPf6Ttqjz}IhISXl6#59
z;6x2jK_z+JgrSuZFgTNu6QutY2m$`gs%~6e^z+VhX9@i(rf^|Z3{D14q=U7_S;O^;
z2P}qES8aZ>HC5K3#XG*Ln=>aG5j(KNeZs-P&_Jhnwjbik@0xaa>BfpAn>$81sQXlT
zvJhl>_Fc;u&K?Z(wX@!?tdQ#y%kiR_7h}^%)|wxZ!FZIz{F0e@Wdp%Yo`||XAEItV
z<w)&3B)Ad84U#ve@KoNZI}2sWC}#ZlBI)SW+fOfWtxl5hg{-*7SF@r!8Jhn5W*v2<
ziTcGqDEzM<uTO!XK3IS%k&b9biYl6A+3*EDh3;TPx|rYt1nt@-+p(^<NlxwE^<^G)
z3BVGsy~+E~@hl+JgM<ev&cFz3q6IqjqA=5Vk8QJ04sre&T%Lsmx?kN#<nk;X#-n#;
z+m@cBoG|^yFTEclFvn)S+b+S>M+kVy)t6n_>*nXE!t2B#F#J|nDW`7Bf<ws@Og0V;
zZ_OFp5_x+Pililzn`|6sdwLVZ<0z92AC@Jm^WIvOlCy~k$Ijamab=NmiB^NWl55V~
z3t%dgA}Yk&oa0`cGxF08%!Xe01ajLPdf}2>$T<_e)Zxj7>Uv;hf;#vb?UzrNH+n6k
z-{IPN#x`8~^CLJ|zj@@r`q$-hJ3+Ex7aWDYy_a^az!whI%e%b5-R1l|>R>&6^>}bz
zF<4na#2riA3I^(k0ge@#Vdg}$7jAYq-AMMB?=EWQBO{Xn()}<-^;sHBui6k@(-CS7
zPkyFa0|E=U@^I#tKjoi0obze%Y^_2m&nn)&iP;Cb>4Gc!7n7ilQ&_U89Y}&ldBaB4
zaSs&W&p7v7J3P8yEPvfVRqOhBCpCb63H0G@Gv95u(L5$6t7rq`8q9InD*^M8i986O
zAXum%<@BXNJf{PWZaJFcw%*3bL5~=sh<ULbPV0<mA~W~!rnV)1XsbRWRhVS&tMaz6
zGPX3g>H)0o{h0j*d49*PanxuDmvsD01P0fMR&8RS+Q8G37WnyF;7<9_OQi@>Oky)W
z#YBRdn4=EWIDWyPdm%}1XM(JO;4ql}4%@|KK<BE{%{L4=het;4jzZ;~qYq56?`{-Z
z<T-zH-g@S!q?;^g(lVs@n0iHtc883iNlT;n+&66RE~|A>gA3yV>WN9`2>rrMF9Y(K
ztGz)2JTw1UaocSl1Db(F{IE(zg|9J;0y#Mv)o_v^j;UUX@4J>YuM`+Qiv<UeqbT`J
zuLB~PHqQmj%e9xc_)=fSs~MO96Adh!wE=Op&Xn7bce}n<PUJkt$y=-xV*A%a6UyHt
zom|P<zCLjxZ5RY+6`Dclsk4gdv@sFsbKp_$C}6H01)=&jPIt{D$FJTwUI5@AuH6Io
z;IM<fAXA`S0j=hf@1zWf6i+dJW24}1^68$9h!3|$G`|lpW7>so`8s;t1EsSJf)Zr@
zg!5rqL{CuNl93r$53~QFU^<Qr!wBk!gt!{+C30#?;^ZH!x{vx)cH=&}E-#^A0Vz`7
zFKs|N;pQf4cQ_mFY2koKsfZKeoV}ysDuqZ%{SJ@Yy{7__50g<}2U-3gIz~3j{4!_O
z{NLvvcRt*vd(wNMCZ}-x<HYAH3L^}fS4H=pcYJ?pcgX>VPc@whc1<rgJyqawv(PM<
zb7$9L*YZ1iZX6@5-tuD8lXaf%FJ-xP-$rCq=6q?ICw2K#L-*#E*QZ;g(HoOCbU1D$
z^EdmF$*na5f7<R+IIEywpm27lTL0?Q*7W!4SJl_g%q;hrzr*m<`4eCA_bC@ORi|{Y
zJN`1O^q{4{;mwa&rt4*w?iXk%_E@d+Yr@<4?hm}L!cUy<*iW3is;C{3wt)LygMXC^
zyHX$)Z8g)ZzjFZN&|mvG-*RQd$x11mqe^xSwR3iydQ=_$=w+M#Bk2RIB2RprPt?uc
zVX{y9rWR|k+2I3k6z4X!Nw7NXxwhpr%kEdV!Uj5ZZz);4FN)dV68rd6x{msubGNpz
zKF>I*ah_$mnawYv!UDgsp5JpX2AeMBmmRl6*xf@w@b)RJpMUy?{p&asHdH3+x!j9g
z9D8W4d%Pgm&ceN4>j%hZEd!5`kK|$M#H15_t}2G#KOoMrKsD8M@3u(O#lLvHb}`lY
zE#eUJ(_JTyHL4E1>*u(BoXwJdsFFQiyYHz=pp2U|VQyu2VqcFLC&fNnu&=Rs$i9Y=
z_N^IhpZU_=sSkE>Z=SwJFT2R8S7dLf)MvvF-svBTH|dH*{I+#34@;Teb3s|>i}6Vp
z{CpSx%98S6{|8-NZlzP@SC?gG-tx(PuKPww*JRyd_h4_+<5B5vhrBn<FvCa|jeb~+
zDe2MfkMZuvLE+kC-yh$*#N$!g1wW$rp}FU+gJ=8g6goS7jbX@r_=%+pyTJcbk=h=S
z844n{p*HxCEOJ_{z#D0`hRjj}4YQuF<+_`7A4CXnELH8@Vwb+(zscs&dXe*?XOtID
zXMJ4v@u+nA9~!Ej50+sYG!8HBxVe1`+U16@;lf$DFR%C!V}j(|Zwj#42b>Feb}m+l
zLumuf^Pt<dI6q^>c->l)0}J36((n2%-n(_9yK}g)Mtk~uzryFoB;5T4p3HZ-WD!#s
zu|c0hLvf#C$V=Pu=Lfj=;U!B_%`HPq-W<u(CgaMw)jtb`T-{x)^Se^d^7S)Q5)aH!
zTD>gs;*FSt$WP9_x>uCi8z2A1acRwhtwl-Cd)wOMRRUk4tz1m|qeyq$BlA-YBs`BK
zn#zc54DGZgUcR@fxb@Isold1<^g0%fBROeNmX~kDg>vr^b}fj1w61=w%UP*;YqSej
zvjtpSC|YMI@5r%MmHX@FFMD;robea&J93jJgw5Glw9Lm8`?~!8O8AA#+k^#fojb}g
zOSLTSlS`Cn$f;;!4l&iETytmQPlsrzy%CA?3Gmr_u|4jTvq(Tydre=Tmujs3xz_$J
z-jB02o-k`5w@gtIF<*^WPR?KEBD~Ow?h}>5-14%0C=28xocc#v>3So6*EaZS=QwK}
zT=(Uf2wUiLlZ<&MPVCLmFB8!ZeI~JZZf3#W_@{ol#)9I4-qY|VIV=Y@%XZW^Mczg~
z7LWE;(<ZaAMP)P`e02KIe3n<YBJUl1`<QFj%ik>TadGU~!0Gbxf#FCWa)1wGz=x$O
zffxugrVwFB9zlTv9J31dmUzr^&A64S94K-qwD-=Vx$c4C0gE#Jhdm=Lb;MJou;kVn
zLpfH=;cXB!9$=4g^xRx(@s`#7ihxRABX*ttr)Ob?YIBX$axSOE-2A-Gn&ad(T-;f%
z-Bn9xG81K7qhjC1P@~L@g0tmZw}Zr`j~VHc^0ax)-329l5AF&+BeF8oVZQezt;%)N
z_-@RreEGXNZru#NeO&L@cbvScqP-SkXw7^7su155WNfk9^>cjr+Ib0=Bvzo(`8^|f
z9qPJqW%70KONs3-pZOJ*;ug7PAHt{}zE~i3*y=uq&QZZxS=kGBe+aPrE!V9kemzrf
zweEkx#!>H?d8>1#8?GJwRyhJf2a4KlV&zX+Cc^iqmC*0S4S%(eLj@0)vtAL<-Vt(F
zP4eo6J+<2<MgBUnY3{{6MI*z_%Cn&v$^G3bDuJosN)w05fdqVWG}&0fJ=Fhq4KGID
zV~1{c#N4ZPdsgw}rN8!#x#Fy^vs20Lg9`6%++sJ`eMtw8CaJ@J%3DuC`sF%}jd3C6
zBE-0LSKt?P-zv@x;aQrQb|@~wR@Yd`uxuk&=9|SOuG`b$uUoQ2%snC`G~}R8vPqeU
zYv?iFJLsq4@x_o0mnf9!S&WqpSs!@2Z`Ub|*)|}uGqk(&@UN_C3$LIJ56GtyFJ?7+
zg$WZhS8>VPwJBzm8fb6j+Pw(8bV*k<B+`qY)jQlN?9sGfgGco{MPEKsXY=&lz#1!F
z^G5(X*6i?pBqjbmW!Qdhw4*#$*8=KeJ#{UE@>pA@U4k!>qx0tWGj7fnBHkt&v^cm_
z&r9UEh>E$y-unyukZ7GT_k>ZnVfbPWv&Yjj-)_balJ@1529#&RStn7qGClK=7K&d)
z>>B&AroHxGN<!V1v4>bw)}6Hlqu<||CTYRFkK>Hdw4qN?AAM%U(5IMye2VW%E>l0n
zT}WW0eu~!n<!PVdO8-w$VT9uXPmT+6u@von9{f)Cmf+IzLK6<#q<OO;+3UdHB%Fth
z=DyQPjo?0(+{Jm|J}LFak|i5x?t2lzws`xZ^HyKV`S^`<O}hHKW^o%BeHi&XOJ^XD
zQm9h~DQCWQAS>F?f;WNG*1k+BFa=(yvBSV%K8JSDK%4R6eXR|dzhL$DYV)gb2&jrF
zi?i>}Ka~5Y-h~6Pn%o-{;17^~V>9pW>PPn>xn<XEq|`JId1*DxUrO!oi#nbzE_v!R
z-`Q|&*?}gJ?4{RC8YNhhD`(Dtd@ll7;~m>9yw3XaBgrD0LpzP(U(}y#uw#`jcf{G|
zt4lIV4i+605UDsa>&o3myuH;IE`9^!+%HudBF)V2M(i0CZWbXyCh2X5$ju|`zx@V9
z%tN<w_SfLNe65~d@Q6(M(q0tZS}PKuzuIJ98a2r5&VJ{vn(IsnGDm4a#`QyhIRqIb
zAvUaav6PxOUAyo;Taz0tA^xT+gCxY7&u7YL36U0LUX~yK9X`I9C$tX!@x`2)F6PtH
z#pzIUiN(KC(#26)y1<%PU-(}@eNV8LYFvHXdAWX)6om0n1>2m()4W2DH`qb4SmN^X
zw@nY1xxQW;l)CrQDyi)jVwD+9IoHW~%N)w^`L|{4qzRj<yZ6|bLDoo;cISi7Y%CvM
z)Q#n4(uYu&Mn67<rPihH&olHi66YgzDZJ~&?ht8Wv1ydLRFYPg-t|u}+y4ixE~RD+
z*7)LI@CE^d+`bf?w1$;^{wyJ<wCtW{vc3d-C*#@*DRQK(=@-Q+bHP^38Dd2pI>!Ou
zOmfWY?)}WKAAI_}f+QQQdi%pazyhi_htMA7#q(&@+kTBjo)3CNdXFrb2``V$K=-4f
z`a1|@t3zeJgt0rHyqXmldRzd_%-8sW7EH(a8W+f^ikm%gEU0}DQC7JXHNb2BEJAtQ
zrwWPV<bGk+8v=5U-ZOTXU&iYvMz?C)*EwVKi0{BgTYOk0$s;n~$z0-UsLfgY1!_EX
zzZK32@f2$i(7H2<8c)~I;wjQ%!0W3sU&;a{IL|wFN(sJVs01%7eoiaFcbDl!30Ary
zB{+P)b2F14`~@ZWN~O9FSGi~<_*<nK<8{B$O7QpUXP|*{EpPfCd?uvK0cYGXjc5(K
zuC5K9am95biyNb9NJ+Eo{EFR$dv`4q`IEKDW!0vPu*jQlc7Bvw!FpT39XhB4v=yfr
z<iiLgACmqaZ4S?L&EH)8ryeseRYr&2K{0r#>A(AB$1Y*;QZE^tx2ZbX*N-#Qh)%K@
z?A0ciYw(P!Px<!G9#w#!A1=T-#8qbt%MP~mvGWdm!io7>2^B%e(cYXY*LS0AT|iZV
z$8<{8NTn5UtU-pUi!!Fs3OH)ka8Er<&l=q~m(sI_-Aj4_XP}njN-N-~IgzD`mJ^}V
zIOFDWQN-J5gDK=hb%`9;ZDKCbkQ1T&AI^!UkEg%N>ugDBugdpB$BQR<s|m@!g33JU
z*81g#oqrLnGkUnVGvIe{on`~ew@%cV?fLsVrJCf<ZqmNM7icSgQ}E)a_O7z)dAkcb
zJx#f(MU4xs$>>(<xU+64t;x6-$H%>=L$@<x{ai?L+<ECQePbR%?TZvO`P1n|jT96$
ztfAf~6~eLWG?d~(3`E+_En5KjzUN*%^0=X{VM!lR*TAO;bxqn(Vth!gYoPX;Swa32
zXQ$DjA&VL?lKvQ{^v#z@tx^9_4@auiZ7IWr&FlaHv@b7NzI7l~K^{eX1>d4Me2c=I
zkQxVu{pdDSh_eQbEX3bIA>Noli#LC<nPDk*>g)XRZSlrOvvcK;ovKiYZ=OWI?~)gD
zmlutGA&&9xC?&r&ere+W&08JaERJ}q3pT4B6=rwptF_CK_N~cON#sQ`Kb~qHqdtRb
zUIs)B2&c7*>aVtm(ZXr)$*Wi-oI-K4h*sSECCFD^D25a_6W69RAF@V-(0nLj8R`<7
z>S$SC(%*7N+i8wlg-;DV)WqM@D>a%n)JHA(CS6Pu6`NUdosc9fMYx0nnruk>s5ITp
zxidZlTpqVBUE>}jUiZg&QREw(L=)A?ihP55kn)kb)X>dks4hjmK}ub^pZX25((6(U
ziMi-T<toj-OkB7|M-nO9&1pkc#dhj9*e+6M^6-7v1mEEPH%e(Koej|F$3Y}Dk#QnJ
z-6n5q;^6odL;Vo{Bi;TB|CW~A5;Ya&s(|Y3o;smfmD>|3afY#tzrR*eY-`whB=u}t
zqgR+9L6eUQV@Hh6DrFy3^}OH6^IvKAb;V{{l0$H18Xfkn8+?QE55(`MeuFYfc3kVv
zrSR<^_ZwWisc6%a)c$T8<-k~orN;fozo@LxmhWD~y{l;JR5ro?U|<vc(0^L5hgVU2
zeY~Yyw-Q^D3ye!e9<ktcv&;8KHH%;TW?VXNxYhs5KV8G|Jvpzdx?`Wb8Sx3kQ5`(S
zzSj`4ex(2W`;GpLhf>Tm!PGeVPnzJ<p?8mzdv(Ou+jNJ3P|GnWY*6VI@k#JjISxBB
zeyirMm2xavYw*>wc7>LCr=GQ6Pazv1V;jO$d-5NQGlKp6FWGkt5|orbetMtX<Mvb<
zGOO>wnJ~9m5t=-INs`tg+T3Q0oapIiXed|c^d(8oj)<;9D>~pGlgMu%@vG&t6m%*{
zt{8WGnvhN_!26HPWRNkqb{Fl93g*_KwZnF_b{Mk$a64>_w8P7WrWqsJ;p7Om1!AhF
z$(H@jUxKzuJ6}5i!zuD@Zaa;4A&1f)E_ZhmFx!#wv~&h-VgudcHzoLt>H6yT?b&%9
zakT=F6}vO>rE-SQOUc))3kYl+>W9~l?1yzn_QS42{jfVS;~(mWZ;j}Oga2Va43py+
z$<n9%2IlRbp&cgMv84!pI`~NE(ELp7Ee2^Z{TDI80>$f?z7!0JOa27$7R8gzvmz}P
z?8_hBVFb*gO>7>j3we4^++p;J{1G}lv6-McrB7^@BNLnPs?*DFVz|woI=0rb3I#?4
zVg_Tm?d{`*LgvB@oN-89Agnrng=0@GC&hm%EBXiFjbf<5hE|+jzOhLhf2e~#0Y99R
z0Tb|vic=L{7Tn_N$i!__aY~uE(TdYFSY#aB%1s&ZTVSapepV<W=et~{jQEexM*PrB
ztkeswv$Oi5!5?PcZVWS<QmO8*rkZ<~xIap>^&?6g`h0%h-ETG+=|dZgb(*2e9r6d<
zt~<3;Ol-?6WltpvlCS=~3$~-~F#d%uSft1@#g1&mPe2Ux&Z)-27=>B?aR{l-Kcel_
zNvxd%9t86$I}it=DcNF_0}tyk-Cazv1GPf8^k2m7(UeIdFR8Z7gh@2B)^Q7|3d;2d
z3ZwcJ`BaYPCG3=RI{%fZG03SfqIsge&q<g!PV@Bl_qmbva=4}jgZC+-8LA%RR9d@H
zD6kUM)g|tU$+coK<ENeWKV#Wi6RgIUuZ9?x+Tzp$Ga6LDKl0wxnzv3pbh$NOm_@ts
z7jLtHCEOS6ZzzUef(0qNYL)BN<B@Z|S&(XRtfntW6><)F`SzJrHOmgxc@Z`cG%s<<
zH#*@E3;)^soLUP-j)rb=)HbrVT(K<wL*#L%CMdk+2@ALrpzzLhV_eMq{!Gcb?<%gy
z+e#=689$LU1_!@z1ei9{5}-E6des5tA2y;HUUI`Qad^ocg)F(JA;UyH*s+tI4nD{I
zsi~<UjT*?(-(g-0ec@(a#tHCJ)TZKQ{_;pie3(*T1lsCP6G-nio{{;3Ze=TJIX-&_
z%)IF)QfA&m^U~}aoq8OOkFUZG;t#iY>L5`}<TuJ7ag5IQ)==jQWBm!XGJorQR~`!G
z>&L_937>3o+^(*xwPs~S2hvq^=rT@$92-<~kr4W!7MCsU`yRI3WI}v@1I>lOdL!(k
z&Gc#Z85+^;Lwea^`>2D`G3@*P&i+uSet%o7l2B>{TH9%ZLJg7tJ?gRJB|t2P|EEIA
z3Sa6-=m*&Zk|nuRIq(VEYK}>NL3Gz}hM4-cH!uyE=`*ncKB{|Ka2It2><SeIA&EcC
zvK^K(JG66jU&JF21!nAn$so%UIfs66FT#3)=5{XB=g+qAM9r)y8=2`?>PCk8Iw$mA
zv<Pvn;HSHs91B$gzZSV2*p#8GQbgIv%%N>$UN7~6jf{J;;M_@$E?94c?=h<&hvE`6
z9gyLgINm(*c7elq5fp)#Xi_02ZrSfHAj@BS_C79(fvdT+ZOZXs?1Oxi3^?`f`!0-v
zl>cgTvV0KwO-6jNnlS3Rg_NM__|b(o^Yvn1%0gxu_+7uNZZPP!%wcK5uVnyFARz#Z
z6B1WK4gM?ryomoNb7WpY_1#6VO`jRt(!sWR?MnM_)MmR!mU(NRwR;t5i#zpx+%eTH
zF7v~RzisS;*ZJRXuOA?(C=))FH*E;^v5-o!5AGSSza0?S9r}X0%!W6;?J;{_Fis58
z-bD{oXXi4=eR5J<JB!2^PnznnZ5m6M!UkDhI1qP~DQwW?i!nDxVS~2)rF7$%$-X==
zZ6NYRzcJqYyv_Itp;y;k$%2Xt;$Ej3-ICIoR=ve8)G}{ZYP$lPydwk^6-Gftjke~6
zw)QSGIE6JEk+xQ9Ea)2cw7S5#ThnPP!Zl{sE=>sS`V9aD2(*j3SIR7J`hDA`U3FNJ
zQ%~(>q14zsOij^|JYjN=G0e^w`fs^=+CIFLe}FYiV9YS&HY2B!TK{b6W$l@GmF`!J
zXB3|R;x#m*pf3qwsy)0Ugxr@E7-)k)yo@>aQ6OG)H+~UX2^-=|un>K4PHGqdWdI1M
z&8kvdbLUVHP)}$GsG7Jfzq>0%_2AmI2S1|>EO##>ATY0WSuY<D80H5<5f(`g{K+#_
zDZab|PjMZ$tb~XjL1;j2AQKq+3UoiR0;M&7)7>A{jj(Q_6)4?0Du9h<-A9^r$S{1!
zx_hD_<6EcWu2hgf-r=<<(Kg=FsYf<iRHV_4IS-2X2iSlIF$pF1{qy+qRr3<ANW$jq
zdAl_!h&p7Vn@HL6{%a=gzoKN3NcQD7INNSo*H6?iaMmgGV3_{qXMPN|@KdsQL24`5
z;1wo7(0s*(>iiskJ(9P7#^2jN+IWJQ-PzQ3x9hft+E$W#aRSGaHSA7x8c4D7G50W$
z`Y;8O`<*`az93n<0Eje+?OAzM@{l9eP;!I?4b?>fTpYewBbIA(|K|fP?*7mJ{@2~j
z?ECT>KUHQw_C@`LpioZBjW-^YcfTPZ<mk<E!n{aoZ|IfeitFUGHwo#IP9J2sVPp2u
z_-~`L$t>roqca2_JMnV9Mu&orWw4xY7hQX|D~XTay1k`p1GYL1Sq3vq?#`kw3>U0X
zH6E<9*Y3&8_mfeSCaeZ%r9m^y=`o^|3@Amz|E@W8CX<41uHSlK<1oHi>SjF!9SiF(
zX|&5lVZ#Hny08><>?}Gu)*K0);|G_c9J5su;y$_3z@uY_&i%GZdu&e(oU}Z$thEJ|
zQOBu}ekhcwFAtJoK_eN5^MaU}?m?F4$LPSW3>&>|vuqJ<DIX|uFjOD?XzstYl;4?k
z%Tuxyl~)@hRFy*V&dDe2_sbRwU>|SMLdT_ZRBWDmLtr<F_p7`;@wYnkHDjcnj>q#G
zOsA~LXs(;LgAPsqxvsnKv-bj$^~{wTZLPj&U#~um%B)QW6A8sR+UZdu-lv#)pxiKp
zIl?P97>ct37e4pMigYXlxF|1y30TD>*YTra2Q=_DOwNYE+c4Q30&gGTP**%a1#fRN
zR|@Zm!r2|KFv5I29+Yf3*jr6F?UyaWloQ!*2q=@ZdOw=okx_sTu)I6bKJUQhlwht|
ztk(qC@5tyrRO;G#cnJq!jQFqq;#zEJ{)8LBtt%8>LogK-plFpV<=bO~OwZRr5<3&X
z?I$8JKHxwExQz_8J8HnVehq0%f2%>T?s_b9L34ZP>4Dg<tttKQaH`<D`^ZeGTw^fp
z3uWLk^(`$Y0y2CB*aQ(M%nI4&(Ljb0p)mk5JfT0=XvaMrD#*~a<ls918CJy^b1YZA
zT4g-YgZq<eUruz5NSzr|FOy@X>Z>E#=V4;a2{mT{j9X^Gw}&#Ig0?PBHQ3<n_aL@+
zX5M;M5)_Co%fNW6?XG@_pnRAzj(er*J15(^oYY2?q@63I^ylqGq=7y#up0MYqD|;1
z?Xi#U^7c|q>W(KiHLzO&sAz&=r<>@ejF8Ae9ahujnOTdId{+!a;TFN$yh&^PlB!=Z
zhZUI!!#pPK<+~UK7NQ}oxg}<|(d8K!WFW!r_Ucv;Q+aQjt&%`@P*q<YZv|Rr8-$hj
zw*7-aqEJ=`Fo*Q{IgEnK8Mpo{X{oIh8m!#PYc0(en(WSKScWu1^RXxPzJ!{SoiAS{
zFX0b~R5^QV<~}>-{g(X;4J*lz3GypT;oi8BOpx6NL{H&oNjP;;43ozsXKfR$JHgy{
zw?cI}GQ+*ik61Qa0yZfHknU70A#hDJj+hH%Go%<}JdQy6*bL8qGLG1<B;B(N&udJs
zz?B{Ew{Oj<?mx!8M0EwjnZ#xe=i8YDq9M94Dzix?#76d&c@g*tnpiGvOsiF<O8y{c
zR>9uV3D_=AhavVO*e-OiB!%rV1hEH9!3c=`<S_*tOH}2Yj4jXkW>%30bAqCvt0fs=
z_4ZaXm;JVcUIq?G2(HM>>S>KfkX*MlQdMsOsKeGodPC$o0)fr}2=rw@bD<&7YbHRT
zcP3L<gN)JH!mJ(wvcMV~2c7M#myP2xLRnj$_sxT;3(8zVC^Uwtrf<F;Frjh=9>=Q5
zb#LIySI$ddksOR7kr6~_#@}JR|00RtPXT!tS%0kU7rFy0he3J$=_mG<GUdUQ5}~yf
z_C4*qcVwic3Ck;ND+X&}B%xx?u}=(mb0Zt0`ZNkLlBvc>j`gO%3WPSupfRG&lqPPB
zeg|wR&|2A5l-ml+ntJL=t$;PcIsUS5ME=XlyyXXA*I;jU8Lc3yg|N`vFb1st-vbmZ
zRLx7ZwY-G9PHplIZooD(6=PUnQq|O6n}Cxp{YWXx!VKiaftv)M*7suRr?g$a@Qdgv
z6KexYSKqvrlui9S+}x^Lm@>@uhUnTeT{Xd-D>BhJ#&~{qiLBB}-j9HR9ttUwp0Tjg
zS)L3?Ezd+2gj$+pOiXIIYwI_+&CW|T`D`4qshP*qQ(!)GGr-q=H@l9Lc(n$f;H)5E
zUi6Uf9n{_g<w{31l$#GD=6om5gYXM7o??RMfq=`OMlj%caImSi07)PWFInAT#YeDh
zH2$(7wJwEwg=#-@{<|o;_H&iu^?ZJneR&D4P`5xiCK3QiZ3gsuj&~e}dc7hC{;M--
zy&hPvQ0PEgKd*C(7HL~6*aQS_WWm>Q;Q(#Hmp3uxYTk6+^`%EDM5x`+IFzf|ZsDwE
z6_^0@r;u3h4znI5&+7uF%sKF--_5VVMk!fAl6Ge-^TeOEXRUB(gn*)*55%X^Dd0bh
zAh4bS0xK1JjqKxT*lQShQ8|G${zL3N8ul7EfdXd`PN0Uw71!{U5oSSX{Mxuut-c39
zBM<VM|CfLkQzl|x_!W|@z0~*!n$uxD>&OrJ{CeJybw7X=zzm0c#2Dg$h|(?qnB^-g
z6AQ~zI&yeF0(^|QK@z;a=_IWt8-5j~yo3PB!PA6N*k2Ad88aw5Xa7VE06pFH`L`pg
z<)w2J%{CBxQHB!0NCog5h5kPVc#eu4pzKbNs;vn-Xyepk<STDQd{v8nY(WPkoB4SJ
zdq3r3r+k1z!xq^IhArc>az_z6m|*2n@gE~txhofkZW@S)6%Ihxhs1lU31e<J!v=~*
zIvxYUI&wz6F;u3)zPmw@FJBSyAxz)p>zU$Ir*I&2P6m6vkrEY&^BL~>s1gpMSo)sN
zXZ~v4_3xYVYBvj~K&v<KizkRD(1_z>0FY@5TFQd&q2F_h6k-UuI5JR1SekH#O8$ol
zJhtq=W1hz}-SryHYWwm!D4!nr3KrINO!-Y@U8sA7ePcV>kr+dHm)Gw^T#wAJLeGET
z{e<zebl`A$e2D_v`GLTXtIpx(Wb^)dd#A6pJHdhW)-m}Oo5YQ1$_8UFa%k8HWW~U}
zSXB!qxFFpi2;Te9y@p-Rs=qePP3q>6NB<Qb02|-HN3bl{ahqAU9jwi*=(-z!r*Y8s
z>y%zrRJ1OX%<~0uNiDAErq$C>1o|)P`M~?pR6350#-V12*g;6}&#f~?kd0WK_n0M+
zwN;cm3o8hi&Ioy%y5byECb0C(nDX7?UsZjyocarEkKv@%il9Sxs@8D2y2Eg7Q|9>d
z+sUB^5aF94^Z=D$3`Dpgf-xd|BYV3pog#em2ge^2;TyUnMe)USok9&@Qwj4MoNq*y
zvaNZDbi~B6H-Je(ma^M+DkbzbEOW3UtL+QCg|-qm29iGvTb!?(uZuWDQTCxK-@xXJ
z*5Dz^KG&$qKGb2$;l<R=7wEzaZN3m)m=*Tts}SXziD<X9&E7KC%v*t=d@~O1_JBxZ
z=4}#2&+?IP%aMFxjF<DNNBMtOp`ddbMkE9v<|#$UeI|$Ftl(+7ddL$~^LgYO2bWOZ
zC$d~{gyq5mrCh0E84hZ`+g)sF0n3Gn3eF<8ZJRPqsN_>sD251|f5#kCG4bu`Z~Y3R
zk4cP&iJ#3U+iDg&sN!XnayI__VDBtx_0aS5q&}9W4+RiNN$S9HexGw4g{_r_w~0mY
z@ZKQ((=^1^7^z;s44=4Dm!e)UMa*|S5SM5*icrq$Dy(b*w#Gs>U&9`HR2fUnhl1Kc
zq6$m9$f++?*sSWoIYDi6g1YmJZd#d^acKw1Qd7cMo@1s}pm`3kfDBp%syYLsR^j5F
zS~m$)Pwh$JN+hA1diokBFlApREFlp4;9C8yTC@fcD5&<Zxs*%47xSz2caz<xv_b#h
zLXkPKPI@w?C}OR#aAicGcA(`;r5rKFVqqNIH5dGze+?$do3~4Qh5bgT_*HW7wR1kV
zyma3+?!6lh48e6Kt5`6#oh*Re<(^9N8Q2?|b$QoB;;;DooES@|h{~^}XY*B3m*8=l
zcGch`gJjl76MT|^UMrvZj~q9pDg_sQ5uu2Nlv6B=AQ4bC0D{K^%=Z^aW}Fm23oQ$K
z3sdYiP`%l1w`HE71)jfB*da&0=BT>ie`LaPq1$(JHE$5QB?PG7IsDP0VlLv0pC#xo
znfnprdkgWBAA}w8`FCVif%jslJH6)UR{!}S-VZ4Y=N~kLaB46?QYCPt8T9EhI6$Ys
z74x76<}A-~JzBlU*14q&CfJ-qA_YymzY!^*k2Pm|noB92dO;%M$tQ^p5L^l~d8PYp
zAi`+pN5vFUh;DWFhNOwGBczECq8pt8j$rz*<<v0?C&#bj;i$6_95u3Z&Nu`x@%apt
z!bZTvSqg;16Z2{>3lBt~uc!KB^%IkrdYlT_L$1J593WTqT{QcEmf@xNu&WGPwqN~E
zc%Kjbh;Jv*&YE<e_gzwiESLvP6d{c{+P#QSh<qlf)#Ng+SU?CRSa*F^iraN^Gp^`{
zmFXVX3Ynn*t+OVY;=3u|s54LBpSxptgyguVHBX)sMe5DJjc!Li!cy%|Fa4KFGW8Ma
z_+?wR>XLy3rjVDg*0>)Jy%1Qq%;1D!NaF-0wF1XC9E~5??74%5CDnoG_PQvDFx66A
zQbnJbNbHx^aL-e<2&}U2DTqYN@W9i2MnT|a7Va&YV53LXubaHCgb9b8#$sl`9-K5q
zDFp1%d$TG9R2EGvGkvpKx`bM-8kAb;UHS+~w{MH-9p*E3=&mmcx+B9yF)J%kTM9<E
z{L#R8J&R}bE}36B(-bnl3n~B|h;D?utaM$@iB`v++w#{=fS?UiZsVAv8m@J|lZ!up
zcV5CvE_uhcW>}h3Z{<?juxA2#H%%S0M}#rGn<@x0Oz&pkxK+9!>ezgBgyR;2B2H&!
zxqO_KKf2@8^Y-pVY5)?iGHz2Uo=E?b)+@|VSP83(l+Nxxs-}e68iM9akI3?%O)w*C
zmJT>_^*=I(2>qm0cs!ZMt<Nx`db4z6jsc80c>_H_?UAYUFazXq03d!3XLZ9$s12HR
zt$G0GPeeaiGPG=Fq^Ss7iX&Hf7>W=WL~By@KY^hE`k!?4dwbj#nPsG#EuX40YpRI#
zz1g)7go6-@w+Xj=(X}yM7{GR64{tt2rWWbfp)`Ayb|{$&F&47Sn_1DVP#rH1EZQ~P
z${V(gc&8&k@Sp=W0tDYbqyzR(ApR4A;FZ$BfwH`A_V_{aCa2&5POs=x2iSzw1=ap9
zKrVG+FVfU_v%pM@h~0XMjY%JZWFXR{m7P}KGr9n!>DF=R92$ulm`Jy7ickZ(zka>h
z(sYEo#e{^?vn=r~J(A}pY%yb>@dWL|kaFE6T@k}%s6vGUWR^%1rXeCy)4BI;*fqK<
z`hf`liPz~*_UOKORa1429I2GoOZ<?c=GsbF5THNi1N~NG81V8bU9I@6mX!VmHRZso
zXse47PQ9tsBH+Jv@Snh#{`>hb_@iFwAEhcZFz3k4{9%rECS4okW70)8S4vD!!0e!_
zxZYdmx4FF#ui#~{nxJ{t16BKZ86x1i!L4ZT+wX=`H)1Kum>^QiK;w+6A~CVX88%>X
zX4W-q3ZA5r*GE)u?!S#HfEG|C7t~7Lj8%pHUA+}%QU<VOua>amgOBh-cVn5Fg;d0h
zp^4z6vGB}7KA6J7!y8SOVv}5f>K;T%Nw?-8yD;869w3awoDT0wzs@(o<W_s_!ALnH
z;?7$1yl3s!21uQKRswzG5?#sdUlt=Mc_31sQBxHqF5RAEMuKF&1(0EXud&CHr@eY{
zk1EtN27LKD@)G{XMQ$t0=Y^tXm1kJpQ(B8l9eW~}>VIX=?j|mP?y-0gpE`Ja;FOmb
zraum_WwwY|_un%FltPFQ==>(PRTMQvM2K&;(zI5`?lc%utbhmlo&wv!8m^@axBg(B
z{wOKMF?jmA`{o+(v615Zf3L{OjC4%r-<Q`<L`a$P>7ZTy_gFgJaBc7czs)B5<%cCP
zcjy$l1333>N~A+)2S4J-t)Ob&@)b-Cj&6^spw9_~X(Gc@)yX3owRHISbX|%@?S$}g
zqk~e#pUlmG)GUUPv1u(XMJ;nH+&g?!fF?35%Z`ZBe`Dkf5YRfmS<A7z+lWtvYGDD1
zjE1J^v_zm!66HgbIm?u8yDb+*_wz?c@0;wK8+M+%MbKyf7p@(s;n>x+!!yKNe<<0H
zlKosIV)y|{wjtpR5a%N{gjGUI*j{V+TG}QH_G4XAcYUH67u=H;rT-qjejJ}m$5ZfM
zPumAxe{^qMf^frp`RhQFAD%S@{Eisq66l8L?t=iR^9U(LL@{J5e8&#0_Nm>?up32e
zbOYs246Kqq2U+YCRGSF1`<>v4?`aPaLrwAUP}`bX{m^}N3EriH#IQ&-X7IW+1pVQ#
zGg5W<8)?PuFm&rLm#VOZ+?b5vx@0A8;?Omy!lNB@{0+-HOU5GgKc~ruPx2bna8a_-
zJvlZAPQ)HbYfM`x1GU4K6k(JIIZKFe28pdT#=i?1F!m!e*z13MCL&@_&BhgO_RXY7
za*~%9#sQ()`s(8i6S#(kqoW=yawy`P2m1wzzrhc+(v+tfbXEd-eR&zK138CB^ibnm
z!eGAFf2T{42>G5N4fxOIPi%?>8E4x+CrC=VFURIFArd91@Xm60(2c~^QO8<_)d%l^
zhL-*(b9}KcFRB<ZGyY?y{MW%CM6y8C?VaU2hQp{yP#<P-(DmsV98*9x*JX)%!^afL
z0~Y61!vAKwTPxgMD$}k)6m2$0Kkx}2`BX>WY#AX*gnXM^@DK=Og1xY8qP!g&tSeqX
z?(ogs{m!y@0vpGM(2sdA0yOv!eqQ(8HWouAfPZT7A}H=FfzJSCXI58SV^-s*ZaO@c
znn)w_7q8h+0TPM9t${K#r3p)+<KJS04>Lfyoj8*)(Xey`{(vS@&!i}yJ6)t6_+!H&
z^%96k{X7bP44F)g5~+tu&V4140c^X-o1_PF>{}hJV{qGI(Zeh3iA3F|@hzN<t+Z`)
zPQj`EWo4S?5p)(4c;bxo@Dh7`$-$o>!H-zCAFOjEe=ze$57wm=M|PKJXTl`_rp>#O
zF54lGAx(ECDF<FcTg@|uDh^u4{K3+)ztla)Y>EGck;cBcYfi#IqdF;ZlQbEeh$7eJ
zAC=}Eko9~VVoE7>80ae^xlw0vOLZct&AD36js-qO&;mUxX9RV?R_Rcm+oRj6e~si1
zZPoUy8aW4YP)b*97FpkNnjhSY62{m_JoQ1E55`((5O~rLyeMCR%BY<KvGrvpNG&>f
zhS!CTYOUyqlAn%CK%VSd%99=74M3B<ql!C96}XQn7BeFQdKoJ6dmI(+<T}7;x2ObO
zKwGt1Qc?B>&|-gU<YfBCDFt63MxKHi=W9(LO4oAQ(M*IMv4~q;5zf<$r)DtX-T+vc
z#=+f_hU>U)WqJ`xa>wbjiY*Jjdww&e8uiHh<~h<*4fbh^N&m7Dv|NQTmq9)oN^-^$
zc8KIM$E6*Q+2y-=RRU2|{~3^LC_r?U|B{h;6eHW3e3kS5m-9YFO-sK1qxxRRjEr^O
zpzWCL_EF9g<=9#yX{tmqQxBf06N0yo%pGCozkF{1e3_>Nadp9Yjd!<=6ueYlZi*}6
za$2EKdQg-ZA!($P6PZR@jWQv2;?efLpczK~pF*+a_nuSG9{V2F_tFXCV2pDr3qC`~
z4Vd$mbB|^Yi7uk&M}{z^%S*`{|7&ppm8J$bkKqIlRu5<e5$#>h{LzqSzYno6vGUvZ
z(Bz>gw}yc8f-LWjc|0KCW%be$nOW5vI|!a*N=6z4moHB@w}RO?95{2NnmMcbH3$$g
zUZDO!UH;+4KH|*t5O)TSk;q9n0i9!jT_En@*1g7b=D66Vv6i==E2!;*QP5_)uNHZS
zE8vjgF76{aBiB>ov{pam$RKo&@0P%J@$mpoTIX1jjZ;sS^sA*v!@*SE-wvGA6=XQ6
zblvLdClKh>mT<ECS}F~d!w(qee2;#ghF5yHI2);G@MG<7&z?l25_^3u;Y1zN5A?oS
zafF;hjF=sf;=JB8BW^Xt<Ba>_pXNN+Eao&V+owD&=R?zD-Yqq7K!oXxp-^4r(%Mr|
zDuk-9#brsW=>q1`X#g@tqp_x-^?zr+vH4dw`|=O<wHv{MokD{oZs@R?E^9uPCv_Ej
z5mpd1T|Baa+WrK+itAF(e4aCYpg-l_K*<C1<=3#4WHmgmde0l2Aw-#;!Zet}X%AkH
z){Q$lf%w4)DRAf--BVJG9t+EELCcV&2<wd<D!*9j2%ef6C;nrB00=Ximw^+3OExX1
z0+w&p0rR6XbjH^|7;)JQ`Y7|yUn=eUtK$(ZKt2`OyaZ{4ZAWf_r;52V|4GP1q!>*)
zc71$O%n)`RE+|ru4N<V`MC!4jG`NaI*K?<wY>hr9L_OIGvW#%Dm9FP*tHA@kRDcII
zrHiu#R5cBC{E-2UhSb}V2jdHLj2_@@zZX9<g>oj+4sEqWV(>Ic+!Ha64)A_B5}FTO
z&=JslCyEp?!UYAfS2`DTYN7dvd7dx}?P~=F0b-t`b*DHtz|a_y=aHLc&|_SDU4BPq
zP|(X@1)%~Bla1Dtx;4;=9sX&%ewPWc8n8!wwE$izshWHfX9xDl@;Y+5eF$JE+Jsy)
zMJm&EZRhw^)beHpN!!(1<mHxuOb06MM(iQ_Zi=6Bu#I+3A&QxUZBvS2f5zvsP?5WZ
zXO>sT%T4j;o$Rus;u0B(JM7V4ZO+RMP5z~e@mmu}g|lXP@l*VVib;}$bngXS<~JDj
ze@!01=pcs$ObB~X9|sLolxO5rq^P-Je4I_J843X1ge@x4DstN<J+^1ZwA-iz+M=zj
z5FMG#Tz`9_|JqqWFfb|unbxcUA6k+EK>;Wa!l`_KM_e&u^)%#ABoa<51TUcaXKH`#
z?%-3A$xB!U*`g;G5~F+u(5Ob={+Nt_;gsU`k;ke5kF{D(df`U<$C0Un)49zw;>M4p
zBebQF$Eq8@5-5hfakwH+tCpEuWM>kp3G#(-9-{R1pBZc@J6ODE@5YkI?`oBveMYd~
z-qTHUHZxL*qX!(R&r7r65~kK2nJ-cVEkM^gSC4ez@GkscIS>ghTyz3ls4XYGXruiG
zLa=<=!uV6nX^~Kdit!f)bDI=-jrRqfLtF70;CUMzN+>zs1CAC?H8+8V+^5{6Wty9S
zXb2#EqQ6u{xk$+oOsm;|D=u88OvA2@m?K>q$I6tB0CJ{Yq+#iMJVw;6#3C<62O54g
z1DJ484h@YE>35|b-u>Pq1VbYI*TR9@y_^MZpu(w7ikj#$+NsYsO1dWNs8sE!^tb9)
z;nZhlxeu)T`Rl<%x#YnO{9%_)wWdUBt}uTi7{l;$L}-=Bar(tp8m?K|vWR((Xb<@h
zJ>p;^Z@~PgArBZ6V~14@nIzg%FGa!?OqygP<roj8czb?Kf?B*?q8;NIl5B(;^%X2q
z@q-FkpcVZ9m!MOYL~3d)wR|f$LH$OHIzwh!YTN_~9amg5L>;^_K&XnN(_(M^s}qf3
z@-OmIyJ6?=nAcY=D!t@vL<gqZ@G-%cd;x2^YJyH<gl+;-#a$^cbZ9ff5Uycb2ugw0
zXe$ta&&%xYP0Nx>_PNBkv6?iH<2yV5J*%MeKSIvICET^Q2QyJ_cvBm3MzJ*{4P;c2
z;nZr=oakO+u)L&Q9_7)3djC0zL?0%g;Xlnl09E<0N6Mkm^Ah9XCSR{U8&HoJR^y0%
zF`Rc>==nmhIOlq*IT4LVJ3~3H42ysANeQNW$b54gj-?zm{T}TLXpRqF#SX+2d*N`1
z3!w;5kRrxCWIF0BFw;m4N+v%egQ^@iq2gVpZ!4J-0*_fBjAQRI4m`|DDSSU8ANeS2
z5yVkdG0MzRqM@n^gj&c~c9>n`U`n+MDR$c9LNpw^8_M}qr1PZT17V$f0dEx5%#c2`
znU@JYf2z*Mr1boakB`v#K$1s>R0S6)G7JBeUnP&FvRp%&>;wF}cpK)VN0IeLR(X?o
zYY4kjajig>v}A1i^`L=%05`<vXO`+sULSCB;07k82LRl#CKs~gV0NwX+e+iE4XHi8
zh{9ZR84RIBnT|Vf2_D)kJQ+)~)N7V@_Ahbohnh$LIC6#!-Q;^V@+Kd4=<4_G5m&!^
z;p+GBO}>Jdo1fQN51~{>UW{huCLe5Yb|=C*RgOFmc?Uf|k$?b>Zj?1>9A>)Ik@|2X
znD>1CGE`-LpbTD$5Ln*bdrj^XB+Et7?Xkm#j_j62U;galgNUKo$lkbmz?Uz~xuqu}
zxwqHGyx$~{>F~p0{J3lu#*}y&kG}l@h5;M6b*>%zr^AjZI#3UtWH_>^8Zf$pYrJdk
zv?0=ckagBat&`&q<>;JmP;f!M$czt7@x4TZxKT6Ohrh)64L#P8KKxfE9yfMyw}hpT
z#P|r{!b<zDE-TNMjxx-@)sm76C<pTN>TYQ-en=r!T=KHb7MBH&{CMm?Q*w_mv#R~o
zt*>pMxxJV$_y*U5xRT(Dv5=TXYG=BnOWUuprbV{@BV3U-LtEv;CA2hQKyf<SN~AA+
zJQ}F-uN@<V>7yl}0Pz@qG<#wrta1b~o>0gzrLu>x!_6N?NM9#MdJ(uNUX*UOxFWdw
z`@J}&*9;IZE<rXqC{bzNvqc)@t7g4+;70OkiP@xNrsK`x(?%a=k@)f@^Ad2GypCiz
zQ!n2w=NXkjZ<hZ9EQcph>S*+(crDtl06cyND~Y)_%;_LgLQYx0*R;pp!aNSWwT+q5
zl4aeE&g|x}(4+hD$DTGQi~HmfB^q*ya?s}}*W8&y2YqO#4W8_&LD*uu3t}G_t0>Rr
zG3hEU752b}CysTZ{@ZX`77*Xt)(*>X6SS3=hx~b>B5YJ!@lZWeKyf38{54{St?nc0
zz_Y08z?Q#JiTtpsD6BEz0BWrDTUORT1ym-1ZW#Aa{pf|b4Bniq*Pm%GZ)CZ<;WSB$
zbKn5p{E_}xH|psU44|xzOmi&aIG&_-HrOap?F$M_d9y9f52lR=L`MiP&@K%>Q)h!q
z!|pLd0u0F6p<w}r9|C8v119R%>F_W~?dMv(Yg$Fv!g|I!1Nu$N&#iwg!(Pt3eI8%F
zL|#I?#Gw71{N)fkTa97d!g&9t#TpDn8>mGznlD9)Htrup+2F4Rl=l%gFEKC=fEo!#
z+{(gZUKk$T?ANT**w`ZL&{@i_0@mRu6k_;C6H$m4?*0%!e~~{}LJ(;!nx!gK`kbkD
zea(&^oQpT_?PkVrr><q^TpxU2i_4TIm1N2n_B?@pf|Lgu?*$WprGKV60bp!;vT8E%
z?A2s>W#u`QS(Qd`>m<9U;XEHG-ZOCx2F4gO^o$p(4-geFc0@fuOc_D<>lO_+y%>+A
zeu_!~&`?yj10W1#R441Dx}9_%nStBz%_hl=2U{GWXGR%fMD1z_`+y-4Rr&M9^Oj)R
zugiBM$LL9KT+8&{GIZo5kp0#iCqZTQEaRgaV<G#M1uR}jwvbcWQKh{xIO%)#1wfM4
zc7t=E<DGFGp(==!FFIEQm4#({&G=NrkthRuDJd@fYfRvXn5Ss`133nGsdbR}di$OZ
z#^MC--3Fyn1zEUCn6*EuS^Odl4HMqZcYol06@KD;H{x6o@<3M`AF%It;|IIIdkQNY
z>S4rhua7t0H_`lJ1fu^(-y?)I#({$f7952fGQVtEbwtZC<pj4fa?AAZ2iix6&wKyV
z=-lEJww!Ra8s?*l+^3BWjtd@|a$y1}RmQ*02PNmNrLgRFQ{z?6n^T#bLMpzNFV3dM
zOsN7R7ZN_A!>oH3#k8vKtSa)gtWxB3>xY2h<r-Y}MW0#W4q_FHGX58QCPC_mC+%2+
zd+Om`AgOc<dyG=7?3Z`HLMc`#tZ%|`fhWfWxmZRWYp_k42ZxjI=|G|k=V8lCI}{gT
zs|(8ZWgEFzau?^hZchiv^O7YS++i~ma!}_*1RFHCYmtmT5K%26EpWEuYeMAl!8gSn
z1zD`Gyw5N$ST424F79{1iFsIMm^oBB{yE)b-!KVdq>|tTKxJ}oP@o;i{7EE?gD}Rg
zAcwc#HUc5YgAI?2yZVV+J+R)VnJ}y-_O~ag{oRlGR5s@=NrBVS@^(N<$dW27xiz}i
z`!`7m*#M;Z!q3O=^0es)RGI!|6CZ@8=s!YqscBQ+BKJbl?!Lv-1~`a0pzIkV9x{ZV
zL8L8z2u7bk&d<Zl!UYg)90?aZZivplm?8LL(WXqDqiG^xO(&0U+E*E#{bq|G%TkdU
zEE}DV9JI+l87p+9i^D=Z^Yc@e44q?@#mBOr7Hxk$W7(fe;*LiBT2!>=sBWPjPp-7A
z=CYH2{hzrjkB2(_<J)#ibSPTNw$*l6R|i8eTW#5*6qQ|PI>>#eT(c##C2iY*Tw9_L
zw%m6Jm4@88LyRMoaSdbU_j$fE<0#D3_S^Qq{?W_h`97cH{ds@h?<u*{o_ues{;W&p
zI;G=k{?h@ud~#AUetzUVe4ftg??eLY`QJGng;8ExHm1jKS}t)S*!(2l0#GB53;#ve
zSag%dy6LW!H(y-IShJa@F9I)#Uu%6Z?%lIY%poc9(3_nmf59B``@}P7uN6jiYD5&v
z)*pgDZQObI)82ZeF58h}O-bp~p(O_32=FI<Nz@ajs_AuZ=~oppG|D_JB=JLV_gpe4
z1;iZv3y~yB1{j+4!AL@M?%hH|mS+9xTo6+WZ;L;TH0wPIk%VZ&zKg+Jq#tdNiAnEu
zOaWFoaqd(zh61en>0QfvGSYEg&q^<0)`!)YK<b+{E8PRHn`&VGv9SlZxZ^h6TH&HA
z)S;DT=5(kz)byye-0MHQx~#%qUaI&J@~D#aRJ3^wCyls?I+Wx0k@wSl)S(o_L?_d-
zsi(6OZc7gqJM-g;OI^u0<mw^VG#exya(+bMM_A9IIUpnRN(ni|3hoUmkzjhM?!AFr
z;L9w?AhWt<X(MJcaBK0NMR{jhT|pM=jR*$1`dbUms2KrWTV0U+U6`OWh}w!KF??C}
z|16*&QXPB51=twIAX9H@v`v946IZ24!g~cQN(`;+f9sR>$I&Cta$}B28%~8^eSHBd
zyCxVeLmt!*c7x9H4y<J^Rp}mZBjs;^r(WJLGg8#8qQZO`Mkvcfa6r4jJ>+v3lX?Ux
zqWZ2yaV#?=`Wsdpd%B+6VON{1>9U=VuiXvYa`>8(1+MsZNQTV5Y6-7He>w*71R6hh
zdDS!%d85Bj<eGjI1o%@hYY3YLOj{)mv%lSp+22+k!UTZ88$HuxPxH@p0&YnK-f$*T
zd3IK2rw-4BwZdf=s_)H(-~7o>(h>~Wt=FNaQLfEK8oi6LVY0enkvP4uj9yJCa<|Gd
z?M-pc_emkYnkn<L^bYIB017ay`I0xLzm}&@%>8a-$4vd&nn5+vU6+p2v8OGDNHXw9
zv_z%{VuSI8)sKEw4}nVdN>~%}X<iZrimw_!A;9W{=zhWwE!T|JM!vg*bfNl<-%P@J
zXOlg$m#4aBT}#zPyBtk(4@`H>k5<>gz}38KAX*qxcR+nEYAWG2T3$;ru5YF*y1s5*
z*7eEm^BcZC$?~iIkC@jt8)+6|*Z0t-0Rua*>-%Lh26iy6?>1&_Z_|dZ&%Qn!X32OF
z>-;3*MK_&X3(25%^|b<<E<wQXZl1BTaQ6;4ismL%^X2}6W~<+!ns0^C)5N~6M(O7P
z$9+XkMky20#c#?vy-mqYkNw^+XG-RgXPFYp0_#-CGY|nOG&mANJM@tkQibI@IYIPw
zqTcc;LOV)rs(4TVp@6wPncv%?>SFVktgb}(%0Y&H+G@EoXX286gCAIcI1hFh=b<fo
z&QY)|f<UPk{P#&dE@q%SA-V$sCD>4xUdfQ;iE9od7j5fHt~}LP=O!Z&Y|Y8N9TPq7
zBb-QmLH#v5;fS<7$tCjj2@TaeOcu_}Yc<SoPES2}reMRtA~hr*?YyB8_Wn&|uKQ!R
zhM<6BO8^S65sI4%ipE9z(fkG{WLR)Ocj>*o$T&4s#SNg7H@lR_m^W(wf;^`a88inC
zjOMV^1<@Q#fTIeVV3>JhqRkz1_e@dyL7F1tU=EGMDGJ%wf{lU>7o)NFvD<<IyF`%4
zt7XX+cksJCiu`UbqE?INevqlv#ttI7Gs~zPVQNTJ4sVURN9?A?Ez6J_i|gqOu$-3O
zw+M+xJ7#^AX()qJMGf-})s}J;CbPhb5yB*>ZRJ>O#=>L}Zy>m_pfFi_ru7^%OhTUp
zVRC0p>2!d7R)Y5Tt?rBrEb?PC6^Q&mQ)y?=R5Y+jBSe1pxRUS{E=PqtQf%laoV1($
zZ3Ts$$o!U2Bqki<+PMdjY$i@_b@A%0SaR#6XI!TBQ=;6hc9{0|yMSyz*r;8ei9I-f
z=7Zare_)S}B;s$vgL99LX*^&jg-TikQe8bRK>4%Og54L1BpYnD!d<f-98d752e(0W
z_`y9#cHyHRoH#vKKQ*6FN=<$t*I7e%5gytv$Y0iJ4z8;4lb%yy&9v`D>*LjVq;ygp
ztY|A6G_Uv&Q{O5Om*M-Pa_eHh_su~H?zO`Y53snAhqv>+hqoR!sM~O+GqJQVUR-fD
zhPZ;58;QgAqLX}~dIB1X#8F?UTy7<qye+OnN^F@^r6s$wO=FzxZ*_U?p)QIQc6ZG#
zI$7^x6aCm8D%`a$jv;pUkb8wrp{z&cnfa29Eo&T+4H8exiO|1!4y;D9<~kUaq4y4=
zGKigrdPEl@Y!Lrq)Qy-XBOjeN3|CPF%vH1!(<_XEZX`L}bm>~52J`&h(xEaE$;Gs~
zI_Xq@3^ECdxPEp&B4b?n-SqIv@B=9C9VAQa432j{_zidZe6BOQF0Q$E!>5dglrNdL
zD4{%X=24#bk`}h_&0rIE+Cj_^kKG5!a5Y@b!3PNneyb`Ez5hXm)r&NtBbj*AQ0L|3
z@Wo9QNq`5w?Qr3D7eX2vYjQmUoak<=12z*pEF$8GHK*%peIN&1_MB>JBg^*1sn3OB
z_rA%4r71EtOvQ{1-|ULheQlyO?4Kk41LA~LpVBo}GAI`n%zn;S>ai^Y+(Tj)A@dbI
zWaFz*Evtgdt_!6+9wJkIzolX86PYae>MQShn{Yn&n+TK{ub2cL@hm0v$6I6GBzIRx
z=La0Yf~7cB*CW1UvN(xU7+h5+y*G<{cd<ALDh#8_$(^in64aDiUC<0aToCg}NMd*-
zKsgB>2_xmC8Ka!k++<cuuf>y-`poi`xvLW^9U2E}>t=b^TeCmiwB<3H&<SiNDZ@t@
zR~{(dzbYz6d`tZw8d`anUOOYrEu9*Eb~w#lgfzF^!~}Q!5Li+&ToQhD1&vaz93t`u
zx4wpHqPf{spfqR7OmhHKqMJh=#h}1&76n*sY%U6E4%rB3U^W6vUCuGn93zN8EVI5N
z;huT0uXLz|`uIh)jUYd78bJo!fix$pyUwlvVXehme-b_FBc4*>gU`gNRq1v55*8^`
zwOWp;nNKW2utw?;)cdomnY)pT0Is+wn5&JG<_2?rVAI^ln9$zA7n#)7p|yp&m<QY?
z%66yULsR+js%B8iba@1mh5flNYaQu|4FOUFSBr>yL|S&aazKg|P0FpRb!<z(Ivyu2
z0rPllTLN<lEXA>F33LVAG%}mi^7FZ?wc;$^HPJ7j0wG34!Vpc7#%pwd)DD77^Jf&V
z?;8m4$ETx9yPQwh{9ij&zD6IRn))1ljl!w4?|80;IC5}ne99t^+{6fF)Y;IMO8NLv
zo68%>W{qANCrB2L{`M-Ae!dV4{yffvK6>kYs7++Cz@gl9Z_9JHVx@d{haop`E-0UT
zf5?2~86DBkqzh)qvb%+6JmtH*UG(NU->D^YLEEZatrZ!{$hAlzzibdZ+b02z;Kg&0
z?SqCirmlE0*LCX#m#Dx$BtVnSBnuqWLdwA}yQ=h@2cN4&d{2ra-7`;lZnak58MT_1
z4LI)#_CfNyd2qHAPjTVr0f&7>45M;LavljTC&xUrFW8*ImzZ?1Gaz6|;yft-eVnqh
zMa*blDZn3_O@>1-)G75Tw09{5ZFpPSv<A)nK60{jS)N1}>;KB(Q0ILSzZt*5`c|yP
zySI@)aCWYAbM@uggC3Nki&i^KdYjYf2ECv<`0J>XIe=jyQSb|18#<X!+-AZFC&T7#
zCJpsOBaiDLzp{C05L&hsL|Ym=HFwb^33eztE(;(`SE$}5!+Epacx(xX?%6Dr=Ed4J
zY#rvkwr&4N+XnfpORf{i5$zWLf&1L3VgvwfC`OPITr37?pBOGi6fNRyw@%yB*&a@;
zmBnv@y94~LHY<JoCemb0H)A^uck|Gn_Tp(YjV5LByhTyv`2JUy{h;7ln!7)4AC|To
zb%(oCpcssFr;xT;-6?070#^HDn8FVr$o@hC&RRHU4i-(c1;Dcvq6xTE!~|k~j1Wyg
zOuqu6NzI`+4?Yizc!`>ts+$3ViThAZQf3vcM4Yg**>Gr=1XTa1I}N_xRFfM???NAE
zmT_-0Rn6+9=sEH!CV<C;gSwSUx))7l0(d`bXzUHr&jj)!A1n*VV)e+<oc2eIdIVgX
zc-+)C9{goyD5h5)S&THy><bqPz%+uB`hOAZ8byz)|8G(0dbKw6)p_}IweC5-$}AIp
zW$xW-J8r!N+MfFNhquxcMPBIG)A^mc6^H@az9M>2w#yt=<WzUGvTX$Zy~bPYP8{j3
ztJrI))Yq+1(qhYKDUGMRTgOiWV@@ZqTcofl*RxiFaMB{eizx3%qd}iT2mt8UVj1Sb
z2uLL(sxYfACQ@l&lpE4*j38xXcL$Ji*Q)S__@zupdH?#XIs_?0)rBa`04Z}n%sK{;
z@(Q!Ivm9VSUQE!Og$40fls$)wYeu~rKki7d-T1ZSd8Kzt!d41EEr~=~3U}Ld29;@h
z;}dXd&-FT!-yVbw!;Alij}i&%@ejKnO!C;!ItWVLtjG4Yrh;lhw%X2bCm85PaNn-S
z)j2kn?1bv70Gd+dt$A<(G^KzNom*jW+`iqM_PSnkzK@(d-4~$c+8EA`R>JZJ_<tio
z5z7KzzVr_eN_@e9?*Qi;-EK2<bQf!F;@y7iLjEtTow_w1hmzCc-qpXp`#eC?SL8^P
z@?64Fg_=?uwx443H!L>&Q#R-bLp7fhbOd(H|1=SK9~~)a2p=Ta_o~?iY{1J{--?s5
zdr1O6oY6QJ{2Yjrs#}+A;l8p~{^)h)3N~bwcTze|LmEx_q9_R<JRv2#fo&L>v@ab4
zx&C83p(qZ;$E8Vqi^}i>$lrvgx*MviM|A*9iRaq@n2&jEIR(}-Nq{PM2H@jxYLE3U
zy9q*^q$&&9+iR2FF^gr_XT`}-EQ33Hi^vQ{vFySumixoGE&!_I_e16r&uEW^lZP3_
z^3JDxZm3wk55;nzLC{MzY}W&d<uT%b%wid(H@lO$E?9a)?u(;}<r-$OeA9%cPrkZc
ztqi~|+T~o<>87G4ZK2#;hA?n+GfX`|HaC0SZ0O&hKFQGCIP|x@cuKDkNg1y0L3sne
ze8z(~8(##MGf}(EGt{RKqjvlZxZHdl0=PogKn}LV0Ioz2AGcwEYYR_LbV!CQ40E75
zm|&R0?A);B`ALR3P%j((NRlQh^Jr;RCZFmT6~Yw_pa&<2fXAE71_)P1XN18bvLPPM
z+#a`Ee(xik>PjHrT}*md`^KyrZws|$j!bWF@-UnLMDkH^Lg<ZAZ~{u-$aL{n*E9cD
zIAPNr@)=6`tuQ0O$Ye@@<!hxj^x#bXn&%aL;`Uu#1mArP@Lkl{Na=jS?g+UtH*E_+
z&m!VHQGO`H?A6OkGHZzAFQF2Ak~^9U!;~K|G3BvCocHQVOcMarm32ZK;3O8AFTDyM
zPL7joYP8pfx<n@+U}<6^8c}J;SO=3(T~dLxjthQL&ScG~^u|89nFZ0um<L;iKXr7%
z8~cGd=U@Urlie5;T)3H`82tkZu4|%;?F&s%!8J_(;K%46rkEQc0Itq@T>1*jb-|Xp
z^1>o;_CbJaeE968Yq=ZDiKho?1Qa4|I%=|{JIvoG<)cD52DD-eZ~yM^Iy~t{S}G$S
zB=@b^DYQniKK3>G>6Cawm5djkS?izkADVDS{Ay<7luv#0q9mb4L3`-_rHgL(c%dz-
zHmKE^i-4>>-5Z&ls|3Tv$Z+GqZl(gjKIkEi!$&zp*>KUS+zcLDXA|7XCnE~arm})0
z8$<zl&Z}{RC^W$srz|g@N2|Ir{#3tKF5rk(9DI-pFS`9{WxUDMxp*_$<9f|fA3549
zWj`1WC&^F`WqtDfb^XRFC)qF*GJf=WR>BEGA!2DjJpCf+u3JQhiF-tve}leVECLp0
z0Omh>2=~L-{QcEU|MqqY`2xDO{&d3MLXKgrTX>k$x1YT$Bw_j{Uu5W*^)++)#z2)>
zj4PN1U<1=F*uWIiX~eR()nAFsfhq7`AHXyK4NRdo<;sSU9SlsrRcdTy&n%JTDRXTx
z?oAbz#LHRRbk>NNX`6oq?JD->-kS&`eddv>OTsC5QAdgcrGk1oE8z_cO+i<j7W<5K
zUoN-SZ!-HBMhW}-y4HD`e>!6t{4Ju*u%^K$Mo)u{(KL7pgEsT)h0K&W5N%r82Wz+B
zH(T#9w(9;nfBwb?*sZ-cIvsm!b^YFlP-T*ZPIn*3C3A$muu(^YSs0eTIT&{%a{utS
z8<G3R9e2a+J!Fi#e}NA57P|@6`yHyufFHSNd)Z?tvDCWSiuhCHnKG`ym5d}lc1ZPJ
zmV+UE^Y%#EGxTNtioQ%w@toR20G{m3y5^?)83(7IJjyS7NaSyV*A{~Mw_2-td*ZZ2
zP6kRL(U(yijQ8@}wEXTZw5PonBkA`kMcB#m=G11-kMFZ&+YNklV`}#`;yQ6K@y3K1
zMoqjUOy~~1r6~b;cyxPAj&{EllsO-~I$}{0R9=%^+B9c<J9&Zu?LG>Vj;#)tSD1HB
zHlg~rC;1VMFrn(8wA?bX6pY+L)e_oK#G5o8Pwi=!m(5pS(Yts3wDrK)Oz!U2B4d(D
z9VjCP9Q4h#1Bf}zbG!L#w(ZWph??-2RTCVY?3!;@r4;z2d1Ri*K&-nDXkB9yII3hi
z0qsiJLO5h0R!OAgq+dqE(^AV>9_+{X|K7y=-6?zWbqWP;(1*V)%^mio#=XF?m&W5~
zpcT0^58c>FY$%W}`g!KT4D!rBod?%<c}m%n$=l1i5jlZw2{^)doV8!abZ^_vY^dJR
zxgyf3l)5V`!5FcE=x3eHl{bHuLHFl6Snm*F<6!?xrzR@(#io=?1({}z;3LaElAN2W
zpLjKGBVjN0EoRRo47_>gOxKl})fYVp)Rn*@P%`}@C&7a--vJTKcVH4`kbku$gBi3E
zAc7H2s5_GrDz=jziN3|7D4JvnDT)xs<4=x#XostmZ2Nl1BtFu^>Nf?>wNLiP3y#JT
zJkA_SAI1{0Mh0*F!-F@#5=I7ZP$pmAE@y%$CDKT35GW<EO@&cPMyU;o3o*$Fh~N%P
zcp$g~IbnDnfXE5v8VhAr1hVVuyGw`YN1U9RD_*7)^bj5cgG!TqT5}~@tBGsnfyQu!
z2=i?3oO2u3R2BV~`t2R}$-;U0yjp^TkFxIMg7X9iCp$;9wiO@xo9L<2o*$s<n+rTO
zowsxJE(=rJ;dqx=<WL5kH|#KD8sHw<SkH#e8^r5;>_W~he<?H#VBef3OKc48X49Nn
z3_fnkrcNt}d63tvOnnXg8{HgP?-*s77SEWpoc#6t&9&~Qg*rCSvW7g<I>=4=Yx9l+
zH;63WBvF+#?iGHa2He8Fjka1p;1vC`cfhxau_w)}hx~;0<(9L5cv}{2RK+RRn|BWQ
zQ#L@i17tj^53JIj{sh)?H-mxk87T^i>C!~_GJRk`MT~4XuO7iZMjl%RtiHGg^Ug$n
zd?3!VQSWkICvllv|LW|z;`>k`BN%8>F7GG8K$wMKA}26VP+dEbe#+@=b43-fu?sB0
ztcrbNvkZ~_A*W8SPdl-De(rIgn3E%6aaQjWmH^ndY<!F*0FGO1vE@GE&z0wC`6Ss|
zIA%-6RIv{rPl8&%f+4HptHv^?9_kY4MWuXFIu)MRoPJA!uTSO^zP``TnODr>eV#f<
zBN6lAY=czKREibu-i@p514qJvrQdS*5&HiO3y>u2v=B=o%J)nkgb&b{6EgXGAK>tO
zn+*;*`3DHJ<liv9miY~M>={5D0g~ipBHS7`dR%GGVb;gdwf(q1ezko0;1dN}31uM!
z3=?QVJ2$|m$W5LAJrKQH7{z4BEcdHofXD5RU|9sXXyeR&uMN+B0g7aCKWBi$@-X)k
zzyTu`_w)VE`uB3`QS!o{_%oqs^za01CVsq5hOu$O9BE%sEIAWst~_{^Oj<~KSo!8^
z8ZAoyvamxH&iAI|^M%Yw#@kVojB{6FV|{n$yCTzHVDMO5A{RiLrT;e`c@U|6P-hE%
zyS1b-3~IzlkJO)y7YLbzR_5&RwTjKFZ>vQ6wC&n?p9d)W=9)&?YM+{S5O#JxH;KG^
z8rro%!(jm!5VW|s43i_#u8o&>M#cU$%w3yXF?=SlYhxCsp}>`c)4VO4MBadIOM$mf
z=HZRd@!l4b9?zVtp|>@|Ew#IZPNq^GL)@?@rI#JP6E6C>5B15+c(MI{X)=QsEAhHc
zuFXeY&P=8HIMqO;r`yOx-?+`u3D->PTh>8niQhnww-5^>(oE@A@I_K>U|=yag+fak
z!&9i4FopV;x#K}??Slk6bjFJ8EFUu~bJ11f<uT?`5p%o-dF;n&9ZP6+gZM3!DoWAs
zR==9`=FQ;+R_+Y$i)$Sxp&)QR_YLCtXRP?3_Jw_fPAYdnRy467vkV7ngdANP?(4o9
z?Nq8FSsx1|(3=SzHQAHNpbw5S5oN@YFkSW&(b-J;;FHBy<<LYld=4kC4Vm7d%{JC+
z8>mU78<F(Siv?HPHxlRYL<YT5e;^ox7RNtaB4Q2doz^G?zgY8CTw$9hcCLyy7M)SR
z{KTj785w=UW?_R9`uOMRsx9B95#>{RE;ikT<C$n~p<d6ei{TqDrM<WRr5Va2K+rHD
zXO*QI(v0gh5BWIS#b1U6DP6`!;&#S<+bBZQXK$!)uvTqQLv1{2)rPTTDaKDYjh>kX
zDYLdaN?QALzp@MnMoscbGFOQpvrWk`I|#^JMWjF82j^LWXx5u{R^B|TD;YHYjuhAA
z<Z(#Y%ZAD0d$1iTCVBidlROR!=Og5C7>Y(|U^@Ol9tRzNv?Dc^Jl@#QPLytfZ!+6N
zGnpdE^B}8v;%}0**ao@nx8_Q6|0BgnCmQwX<AnQC_S9Etnb58?S7UFX)!2g)cEN@N
z7p5Qyf6i88m3=V&0b8=Jw!k-te=sRoH#s?m(uIOZJnA`nEI|irlK9A4y7Ua4I<3~f
zsMe0IKr6oNjrYcR67??Yyz~XwQ5@Jr(#$A{R!Uj?@h?M<7sH#G^`hsNjqRk&EI(q&
ze)|{VzxmsCnG{-WUVx%+u3?k{uh;~q`ZBEe+=9UJ)iv}65;|D*;6pbB<byI^BRzHl
z7&@BY9>m2^3n*4tPU}w(c>J4&-{?(k=I4l6^Kte$e#T90F}2!_P(eO0ZMbeWuml!0
z=%hvo4em+m$Zy=!MbQGc*uj~=T{=z<%H<`ZJJVzCk;>B*6sN?=GQ;GGk6helB0cOE
zw;7u@94u~YrpgykLMdx+Up(y-D4lk7s)-2)>-MOg1PhWr?c^;!`0LnX6Q)YJ$CpfJ
z(tXydZpOA_)x#ryL0VuVbRSgyGw423bqa^+KD?}bHgf<Unfq)^@bN7Zd<0gc3PwiO
ztd3`HMok7jcJ-Dzb{eX_%ogFL3n;*XAV<&a$5-tU9gfXir2K&0=+bgXwSV(jgw0`Q
z?2pAkK2IBpb`KeCsC#DP8{y}vZ8YV8fbSB$&<@H)ByPabOZ&<qE-A6Eu(2;MU>7=@
zf;jaJKqAWGE>*3+VW|yi@}u?Nx((r_Y8;_*n9?BPA+w>yEQVe*f;}w;_7G14Di`1b
zr+W#^!T9_KvBBewLO(1==Go49twG9PnRh&@d^4|;Tz1tqMh(e7+>3Iwf<l266c%9w
zSaqx#@)@cjXERq&{$P^EfiR*W%_5D%3d#s+{OBmscpeD!Xw6%qnpjgq9U`m##6MU=
zFO(jNW`L@o3cyiKawgMMH~ADrp^0(ChM5FDXVc{nM@|bg&1RNc{iFHg`$qWV&|k_X
z%*EatyZzDC66n3L)al3FL}Zfxxx<4-$p6`OTn~DTP+oIE70FT|Q=<%yKFS~W$Nh$O
zn{QDTLIyDIjg#+RsDC;CuHx3X7sFKvMz0vIO3Wb*K<ml`NZ7C<S@(za0NDL;!b5(a
z^shpFq@RkXwNqqnE9&|{8W8b_D{1Gf<oZkFgm8bM+GUb5e4D_GoVlGl(dFEPrQ|2B
z+jFo%%DA`pH%NDP`N22o`AkZ@hT1M<9#+KBwS6TSC+mQ4$O7u`j}v}Cc!Kcbdm3y0
zGk8M8Mx71#9o8Yn4Y@T{P3JYVILq0y5=vg1`8hld7%CvIUO@V@)_5SD0B9Zcs3I()
zeD)9Wlbh1hHK}xsCenGa>y9)i=kow5NlOs24C5>lJX#*7US;3XCH_1>!8i8^A|i8f
zC%G_^z{DC5XgPM#$1J0KGVg!AqjAF!+3F|HLPSIMMjM5kbpbzBwMJ9A+H|A|-&lxU
zggq&Gj-qbdJa!jnD8O`JW4ot;NzHSzr?Fs0;qKV*_xCi^?&Z<-d$05jJwc(wpby&8
zL7CRY&p3N@Q{o0vdKZ!J&L=&5`KEvpkORUf>^6uv-)LYr!Je?;W@uYP9E&6>&(|_!
z>_x$L6Gv%nN`pyHQeS<WurK}+&XbCkWAjw1+9M{sG31$ti8ZmmvCI>r#hO+y#hO5}
zV_2*SNp`Rq%tsiTRnzOyZJ7uL;%SPwj+5g~p5BuIf{yfD<;{nb!hTO;z)oL9Cy*m?
zLb4$)!;ya!W&HkV%J>8u=wm<xhS17C+dyw==u1LxtV%CR_<@DRP#M@)L~wR#w@vnd
z4L~FQ6k7Yr$=AUS2y0(*t~I4+7zg|haN-dea`WKJZ2kQF`C1ncRhO6x$|fAvlO{%m
zO^TlMA?hwhf};*f0Oz0iU1B0)U$iydi+`hm>a7w)8N80xNH~j-(<*PqQVJV0DOb;1
zJvHmibJjWYAdbZRDo0-$NA*8L7lD2K&p9pL4oP;vu}qxGGvyJjowLyRAXdsawR=F=
z3;#P7MuZ(?X4Lw#q;WpBZ~Odf3@ru$%|dzb9fl@J+mT78?#pPZJZT3lIO|I9g^T)#
zUm^^mZ}GLXy#tM?m>x$O{nEZ*(+0Z?b`=_Zwcd_{gLqpfk7gA#xam~q{CIc559$oG
zM*8C2aGu?I0S<Ng&mnXB1p)2<KSLhBTbXKr&Q_b9)uSNuRo@SsJ-6u<6}_u;=yOQ<
z%kx&HKxeA`2#Tox&Z%~onK?C5g$jaF?O|~!OX!TfI7bJY0|)B?Gg6Rvk|dMnk^ZcC
zqe>!t1)On4LGrm2?w<NH=A>sq`mL5Mh6E!(ZwvMqN-weWwqA_rZ6T$_3$X{jW+*L2
z@wwUaPs==TBd$Z32v4uIQYfXMVV+RJ#M5n-o@BxSE%3&TjcZbdYfD3_7CINw!eIdL
zA6Y}kU5V>V+c5Z3rxgi;Wl#0sg<OJFLw#NCuO8{5907P*xV+LtqNkW}UcfeAMZ2gp
z%BxyG*p@vL^i|cm&)vp-@~qpprV_|Z^R>h(1>~rQJTj^$VJ3{ayKkQ_zQiw1FQRNX
zZ?y|0xmx!r@%rg(W{M-+1P;ZvJ{c_O{(<@I8Xp1*L@<}(vTfi9O}Mp2LrFWCdYgH$
zerllN|KnhvtRkk~ehPx5WqGz+ZU4dSx&@m);EFQ`08xC{5;_b3wr!1lVw~!bfG5nb
z5GyAJ3scLW3!pz!nbwiQuDSVWd4x~xNO9r4E#<avsCOv4Zs#8N(V=DMOY(UB^}!MY
zOCD`_3c!E<!HyQi2&Zi7lE(?hqzAj5O<+s?KQpusxd~dk3f8~2uMnnOLGz7Ji}-V}
zSvi4z7(<NKG9+;yD8ASBDejD<3zF3!`UC%%;wCvGZWa^&XQ*R-NVMRb-7oWqMLuSu
zXSZAzNbVo%*@d>Y=XH7otVpZ^4H<J|A+h-%?9+SzB8v$fxmaveD@UN2i?slFfY-c`
zE}Fq6YH-d(7}?Ec!$iPD`eTp0$3*b;&wmG*x-k|)#tNi<IpKj6L3Ro$n}&)B=w2*0
zs@;MPR8CCoOIP-nlP4teowJ{qgCN3O@ckzeb}GXxabv_1=g($I*fCXO7!r02mU#CT
zz_MEMsV0=GO{;$wi!({?YML_RDWB(r+JhsHi~H{#9gStQPBMuX`v(8M3>5g`<W(X0
zV%@OXX980D99FTz6y6!?gEm!+eM3y3m5quW;Nrt7cHEcN%1thc?CW2q(U*1_1w>8#
z?g8Z*0T48o1ijRW1^O5dtjlzsQp{id>Rj3m?U}5zQPhqK471SCGUf2$kbM2{;l^L4
zY{>-q`r*SdrA%Q&`+?#!Q=*T9!<81o+)(Cru1%j<P5)wUPz+-y8VuhfpSI%T6n>qd
zSJakF2#3(gkZxp{XV<NlVoHS#Z!8?l9ksDAnN%2aV_}k^A!B2Klc8aBM4YrTYCp@W
z1ykM)O70Cld1*p1#o=K1qqXR&KAqHLX~KZH*i#~{*Es+WCmYJH+YMC6ok#(Vd8{v?
zfR?QSpUu3luimhxHNDxq8GqX8&t}W&l#5Rzvwrlh*f61Vog^k*R}j?tb$ZNWDw$U^
zc|et<ES$Frk@;&63q|mK0^j)av)zldk{#(js0t>TIP|yjn`$P%6GvLO6JH!d1=Bht
z(cD~gd1T5U`6^i-iy!aa8i7s0VXxH~4)SGN>ck}R0F=F8#+zL!Afg&Iu}UyQVK(uA
zmPGuGKW!V#@qFh8%}C12@>V6mRi&K}60`k;tNP5;a8Te-705w&<Dis+nZwc<xy8N{
zMfjURv#RWGfpkT)icR_}_1&iR=J?kol&fx5yG(oIW&}vvTho*;Wi}dr`I~}?bz+h-
zu_m$4EKhlXR!36bvmXAVpb9pz%s42XjXGLZojjRf*N*qbpT>E<)(cohTi0MNBiQu0
z8t4y0{0E}`BShl|h-f^GA?lAw#B=|)EB{4;KYWM_PIlH`TFUJi;xuuXXK0jaLD`|H
z_!OLzHc06tQp3rs;c-=gKzy1HRvqfF(N6!^WpX41z|u}1vn@Y~cKTiOo$x8DWk|_e
z_LPG}_2nD&$3x55sg!z#&Y4lB@=(alzAVL4d$S+o%Ai@{&Ow^@9Bq<UGQ1_&2!Va_
zzV%*R8;FGw*Gx1u31pQ=N<B*^l)E4E9pb=tLx;9Xr%#cTzcg=ETzQ{-|I;W15<wq1
z0>H)$Q{><aPT_GbJ!N`~b*+_&DzTJjg2L&4cxMQ5ctCFHzBH5Mex0Ff6e~BYr=So?
zHUI2q`_zH1?>3;l7@<$Hu;WtAsoy}NY14$6nIS3*nfYvb%(cUO9R>s|T5+A`pFWO;
z{l|p_R1^ObTK9WxP7TuN$D#V&T>*4{K}6^8!T-v2bM1t2pQS1;c95StB50bk`5f(`
zm(-BVgq0vm@pi6<``SJK395UDm|1O_H1y3uplFb-^Z0M?4$}z@3NTsm1;Cht?7RT_
zNtkvT5suKF*J)aU{>z?#P4*k9Z9>3|o#fUyS2$yuN_?_nx~gN1Ke4871+-!j(I(nk
zsOs)MEvA#!9w6fYlxb(9wRHkQ<qq#+BK`1IIG8E||Mv9@ufh$(+d#v+$Vdbj+Xnjg
zdgs>eQb)5Bitr5zrY)JOuU<P@tVaeDA+m>IH$&*Gy?7sNCp@EUz+y%Sxi;?sxU3Fq
zkq%3UgHy%`3!EKGLU3o%?gx>WryWv|VyBKczE&Hqq`iF*YcF?K%@nJlQZHR>BP8(2
z6xnfjU9+m9(9F#2CiA}X@QFeofcoKgII6t_<^E6Xoj*{3pVUyux3w={4-+ARY*vi2
zS)WR?Qoh-^&um0OkonAb9)(+WW<}y};w`W*F|QUpTh}hdg2$I;fbZRNDvZnzOzDW%
zaICaMzRgACagy}*(F#PsCWe4`uEFeJ6@k(w59(2Mnkqhy59t?SQ<qM$v{o_z79hqZ
z{<3@HhR-zm=wyHeL{1AvY7*vyavX7-mo{2EkoD4>!kTrG>6yYqRg{HL=$rJ0q(_q8
zdeyT;#J^w*?P`JB1Z0A=lPH<hq!hfC2V0{G&dvo2u*T$|GI=>djk#C?TB>#y5>~B(
zNA|ITSWFBD9E2tp#Hu^YnbjIvg?6OBN$Moj0IYaD`KmZ23UAj?Z<ol2FNzw~KCcQy
z{mOfikF0BRw3qi3_Q9Jm-dUQ7jC^qXCoEWhewVeqmnx}*JmvX*_O*i@TADh#)Akq-
zM~|OHmW-qJ$joE6URKI#51BP73+AngDevPUGy%HZCpz;9U*sq3!hdECI9Nx7%r!Hi
z`NJc#>a?$IY981wu&wCfa4$W9bSyK3Y%^<P=4ca~XR%(umQIUvN*3RIc@Mc9obNYr
z6z}|+CK`N=34JMM4cFc1%d%fsR}Me4KMt)&rZZrUw|uk$HE^w<Ez7tS7n9OoLARkr
zqu>al_I|nZ*Jl}ceQ{R1ZxF632`xQzZHM)Jy?~b#1E};#HGN=})KZdP9v@FDR++a)
zd90!(=#nNHE*m3_h_>!D6=zSY={P~&9!Ym2-<?bPv*OJ=inIlCXX5}f_+u}a;UUqN
zlZSuC+!j7B)!hREr71bPhssxwu(}e?)u6Z^lwsMatg`AypfkeSogsmCzfN<#k7L(s
zSSioPXKe*1#NmQ#BJp?Rwli!fH@Wc>!%&U`f2;cvuzNhp-%7QCb{SWtFYnx^b2DIt
zY~QiIon>iLK{yK5K}V@}jw5Z%tV=wSTtLRGj6k9O3h=ko;A>~G9B@SB3_0M;yz7%S
z=<a6AEdpR8M@k9+ggm+OCQq$^!w^hh$^-}6SWtu<58?T}4jro=#YzMx=QPrCJ@Jl=
z%Y)}v--yJ*-mCr<nWi-Vt~~nPY*BE4uA#n0A7*8n<Li?A3ivwp6+vVbtT!jskwrUd
z&}W+$i~dEBB@~*Ag%Fqd(XTYZKxIVe;KSDDqsauHJaad!0;tW+m|6#0TvoYAPCOv{
z7j=WN;=jndYpg6Htb;xRY*nIpacOSk9#Z}il<XR2(wDkDE#?r*;_Qp$pckDEZ|Fn!
zF4cxLJbl}pi0IN9VXfp{336Uyryt7<l8>tl%{znC=owTQ?5TCPD0|7-z3s!nOlLAU
zi*qXYKa}jho2@GV2q~=t4Q1@1vaITo(n)Zd0y5s12=JKg^09)X%v^rdbQ2|;V&!V}
z)VMeA?@WvJF-F5-a&%wHSSjJ0z5h{t=|hvO_f-3Fn>;eit)N8t9et3=_JL)D4g~$E
zbKSiT60(!3M=|3hY!WurC`Jx>l1bd8Kg*%~GW*rzYKYc9M1PQw_yxy*DsUfSG0(-m
zLc=HWIVvtYQ4^QKr=0ttGwk?r(ju$~z~iAO)V1ilgh@t#=*(xu{^RBV_Otj?LaheK
zQZ!L_BW5YeRLH%;RLBKO(Ld~%4FM97%{NV8Sc;<cgc0xMY|MKZ8iGSV@^#$E>p(BA
zucj%hNB9xwM}26%JVq0dmA!w{sP0HPMad%H#qR7KC7=#DZej^wcmj%K?MDrECy=6p
z@&B+C9oxY!!)twVDY{}Vt_-=xcB1lg`FAwoO|Ti%k<xdj;I-P(pG6fu;8BbfT4g1G
z<nY-(;!Z=6-uUA<Pf+l;2k6e4c0vXj-7;;ZS-bN)zr^l5F6QgA<`;oXu?ZHsNd76j
z4ixF3WCRLD^Qq<%KQJyH|MN68dY0muN>h6tu-;b@R)c1wT|0W*uoR3p(M~T@!k?p^
zUPpnIs#}mpC7YdI&d2k`T<6noT_O;iE;LuQB21`KWuNG34zDjdd{>{2QxAWBfFwE?
za?Y3uGe~Af_hm)yLl*I!-Cip_*fAI&4d13|4-5hNhmZpVhqq>I1Q>gBNq{UN+g$8-
zC{PGJmVYY|mwbF*_xV@Mebvt_kMHZc3wjlK2P<caeV-$srpRfuG%`x(%?pt5RfK(I
z62U=6Js32zKH9_m^UJsG4SxhPYI}RK?eqM3eqpipfzpgDr^$6c$e6P%2SlqzRq|>q
zTB?#%Jv>9Szx<pYEXq_Hz6Ir)p%luJ+g2rUuo*<U@tsk~Hy=pej8~LQTVY~iXHRpe
z(_G*q*FirE1O!5Ykhp4!D~091`~?-@?`H3(ngpfE4*KhKnW6L@mU>C8rauMAt8n(x
z1Wp*0H^$`!h+{EC4%hITck71dAGu?GT6V;gV4@xKeW@fZ<QxLRU}spasYn<MtaRR+
z*E6Mxuz5Y=#~-GKUr+<VqCB};Bmz7Iun~5W&CHrPtNgZvH(m$JZmv2{=Wi&{A1){-
z>bw69(>2^Co=me6Y7h}q<LIEk0+S$KzoC9$5TwCEpjv)Ui)q{sGt>A%|4j<n#cCJw
zU+ZbaE{-Ks_Zj-PPbPyG<IblZ>?v*PBhsE_r)>hgkq%0nWFqa5dn8-6RaU+HaL4PC
z+D(K+M(|Hd5t3Au`*4mU^U03qoD=xpC#`-n6BIvc2HY>3(4JCe-?nNqA=h83b)FLI
z1&#N6G-H=>U_C?+Pj;ZxQ(H0;nxzL1(Dl()uecBhmQK<Qe6+y*Cp)hJv;fjWIY}1r
z!oANxxG16LB%!-$s{7wrhMZ*qL=f)*e>u3j0pc>=%Q&E|e9o);WJ$igm+aTU#-smZ
zU??q3ejo=QrfEt~YBNzd337t7;__Gl4zF-|EY2jg+t7?=U8mU%YW~z!Jir1Nx8q@0
z)^o0arsB30*;anC{6TU2>1LCq$~krNv6KhpZ&u(fK{TvtAj$S!cHCibROGO@JTH0(
zRA!U$gjnnr{Rq}FCxXo<xkk<>`v26kOtqouR-Db2rEY_|;5&M@xvCnOs)k8|sqP2s
z3*#b0xQxY7L{DwHZBzc@yd7zf(K~<f0Fmd<a)19+@V+>n*dq$JN0zDSKa1jvZDoqu
zGXJAqqBA(Wy5B@kb`n|w@g_9mI?ZB)I<F?m+t3_^q>HMd)clb5^l=;LYkRMcXa?9$
z0iid(kK91t8~#$0H><9u!eQg!V-MShV$bE@-ja4|0xAPR81$MrE6mO{7!}yRiy@Q_
zJ_!rhguc^O;<CB6IMd)H=n_CcCg(@i=ok!0M@f!T4=qAgGz<a!G7GZAcfnhh2Jr@t
zk{ky%!^27ausD$%H_U3X0~T6GnidG``pt<VUTnj=083<|Z4{ROC}aBbbzO`#wVGn(
zVpU+)n?g$a9dSIyHmw4O4~UNPof}_>iqW*PeCNDAgYTTGh|`+pUg{d|zQ<;M0pH5<
z>w%YL+Coqa{IGE0J$mdy?y%b<1Lm6+Xtb6h+?2lvdwVm}+*atT=YR6aK5ILEf!Jft
z*OATZ&_o`jfPgS5<4ujH+?I@AG>SgNjNX?IMvNgK_Zp@e(KNBNVAwyyEw7u(oQ$)!
z8#u9w`aev@4;e?efxfYWF}bpd<d)}B+3QKjRDKCV1G?Ds%9|S}+G}H)m@y0L?9xJV
zS7U%TUJb`T1XK;6q4>2>SZL8Y35Vd4feT+o+73k(;T?7Uz3(p3kAX*n=1W+-c`GOh
zuIV47g%326(cs|Q=o=AgBxU$4Q_5E>%iBXyJ$U0Nxb}Sc9u{%UMQSd^hyQ-oToHDE
zQ9MZ^76|HGco&=aVa?(u2?NGqKVwEbIeR6>8Z5sQnivOcAm@`_d4~#qJf6}x1tMkU
zE%uUqAJDSP_y5e)e`YGuni_3W;L61Lsy|4OhaIT?mrXi;LuqkNRbbSR@ONeZi=OXi
zRJC&WX-+Fu>8o$-^uZs*@%QL;ZaH-Zw)OP>6NGxhs-}T8Gu)(T&0zJC@Qswu(0ND(
zmybRP=lLL~f3U-cO~@aYbiG|f_T(bnt^s3|`EJ10kr<(!|35?tjjK<Mh!Vc%kiLrH
zh=M~7>HDEyK5peGfOv(326R3<RE`-*G%c?90D<^WQwDnZv$`=`pds2p*2j(qKP%%D
zTryH>%Bya2%=&8BVDE-1!!HVQ_$LqlIW<Rz;XfShjJ+}*H`rTq*)Kp?RxKa6H}st#
zc{bs9=`0z)1N^9&F#C@OL|2Pz(*~Kx1C=8Uo&bZGNs3K7DYWwj-eh;K{=dofmnt=d
z6<*a;=19@|sogIU22Rp5uj>2_&3cZ&$vL_oG<-}I_EE~hko3;{rLHHX<6t5|2u4L1
z)SA0^{%T2?Y$%CY0sI?Eci0W`{ie~_ASj-v%<C6s3(FC;v3wjKuI?{&(~dN%oh|LB
zI?ZTgzbym6Eq%}utQJj%!K4qTlm2r9%?|EIB#vWCc7iW}5IbLhf6(#SI-3~X?yoe&
zOMp1gAPLfYNfJ>lf*=io{6{#u!#Xc@4=0MJL$$acem{=C9Rg_oBB8AdAwe*;7)ME8
z%VhjKqQwX+q_V%dUJ*A~n$!O1{6u~Z(0Ot~4BjW<flQn~7{J0a6e5JZqQzl%NC0J5
zJ0KaCaTZ`Qmd9v9yWX1b%{ba+*R<=~PpHF&=9gDv2aZ^RJaCY^-w`8k1}u|RqsyDW
zYf`RGj;C$RuJPyeg!5zWr{E}S$*UoRRs(;kvr*ut>121uxr&;Nz_E)@C-DJeqoTI=
z(%`Lqm3C_edbh|srELb+#0YN__hn8iK+PzJv-|t{UG&$+DP21BcnbNPRe>oKW_HLa
zb0kH`jZP7Tr8iXnO*EqBQZ&?MX8EMe`esgVb-dvg%bs_nSUR48j%=c~VMN;lTchx}
zyXaK_;Vn_xaTe<LnyC%BTQ|y$PsjOm2ler{^yQKoGcb1!w54Aid&Gt17cf$tE#pa*
zi^NNS4{Jn1b()v8B)I!>c6Vp%qE#H|RD+C!6KFYFCg2_3LSPRJ0;j;+`^X_WQ2U05
z=%2;Utr+!tZ%pR)<%3sg-ln)3h18)aDY1w&{k33+%i#eFqW`4&QHrn+^C`d661ReL
z;G%Q*%PX^EFbn9{y?sZ70a0&0-51<Ft%#<esv`jAqsKW*4C#l}4VL8xtn$5hGD;y+
zC(9zbkk#&eVRAsd+t()fmx2E~C{EIc>r1l_q~kZ!>sPj=*7i7N2yj)d{82Oe`nqk8
zJdSqE7s|U?BNykIU>|H>YO(x?>4Rq{U%yN*C@7C|nRWPFeEGhe9wA<bXDK$cY%mH~
zxOs-p*KfXlyr;16hQzEY$(<EO&S!AAS1YFrShb%J?kTP*8j=qS7<%4!+P}7OD8z1P
zs#b}dYFcRLU}M_(<v7a?(|{H9TALRrh9Za`RD%b73sKup(DyIh4<@*AtzXY5vpoAC
z`Wwgm)?YIcj>S<^NjmvwqHk(>!rH?(*-DjQmq%2(&cz*e{nhSO8T%^di3PPw43zRg
z-Mfb02bL~lY~X)AAEKmB0WE|=i%^?6Su*>}B)kq?J=(|<XdHt3b6Mu2O4T1<C|)=V
z#bn`1q*)i284YFqX_<S-=ulth3gp`)xgf5iMhWEyMJ1|A-5&5kLpDRw&~1g8^4rpU
z4Zo$bWpt+|ViNv1sjYU%NZ{OHyw7rE<lATW+m&c_bQ&^E*ImgVLDfWJMw|zXt`ZLN
z!*19@?x2IMSz#qn=d7NZQfT@#aiLvdB{tq*zNY)Ucp7XRR38X=^tbx#hy&ko$IPu$
zso)7T+XMZ?1<MDsr|PI^kcKkPdp)zhf#?2n?b5sIar=B1+RSw}*d%kPL*m(u3oBYA
zxs&ClPlLaQ0Qt}F_SD0gfX!wlL6E@obWYTLp=8m!Xm-Pim`c@9iP#%HS|wj+nm7t5
zO1ftn%nR(5=qjz4ncF8nghxk|e(s+_?&!=}Q{~Lz;V#@c*sh_bya0C68{0_qa=n$|
zhsut_4uq?u>OwFDKdBl7NYAQxmER?brOqiDY<5rGgcgmSWqr@>u*<}+k9Wn9bw$6@
zBz_LI_BE{DX0f+;6L)aQ>t7PUq;j3dog9g`y?pb!|Ga(px#ES5zk`fLNxOuJfS{^2
zPb(NsWR<I}bPp^sRSp4#Fw>I<=WxgM=>CuejHj*c6%9`+7bf>~M(*YD6EqEsB=nUX
z7U<Y6z5OrH@$cyQYK1`zQuJ>c0tBnj@@~apZqub`Lo7@THpEs7{2W%@`K#P6oadVx
zop`#uM^H&qqLRScA_+aiuQT;4OhXSG{v~{x+Pe9!6|UxMP239Nf`JRZBjM$_GvXoP
z@GbnLbX-;hOzN+>s)lQ<$@@LgH9h*Kkch3oamB1F-2yXe{J#eEGvqIK<@*N`^MdV`
z2G2cu^jYrKYKarUxWNnOETSvg*B%tm$R$P?f>6A?dk~m!Zalx=Uf`7dZMDrhQL-}D
zrtz0dQzbU>+@)5vP1#v!xC2IS&j$hd2x{CLlP*vtiyq)1EdX}*N_Yh4r}pz=mzB^M
z;6v+&8zT@l+&JXZwqEy+Ql?kPZ0-Pqq!1M_n^>S3@{@p}U`9~%(w((8Z^J5f<B^46
z>=m0g=r&*Sddq!oQ1<_3Z;a_J-0IeEW)vA(-mlWVCb~$W@F@ItT;Nu=U!rN%j+>7`
z)2-FUOUnJyTJf1SFy*d)BeEd7z%cIa0bOHJ>7>x<u9X84H|hb|>D<}AzpA~3_N?pj
zWvBYqH!l+EV(E%${g>O5$z6^O=Ff6cmK}!A3-y%A1c4Q*>Kz9J-P6MZ;Sx<trgKL$
zp368f@FFVTDc7NACG5>=q|&M9;h`jmp*en<_svIdy<>gWEfP2+l74;nxm&TzHFsBn
z8QWTzuJ0F<(tV|9*arXcd{ytfXTR-VA5?dsZSe)G+Yw+=|9#H0r8~=2ELO^aN&Q{4
znLhofw#uDdKX9K5(RMS)eCgF?xlfw==Gwf}?{tO2PulNZlwE2W9rn%P_;B8%Yx8yr
zb!>;J|84jv<28Qsqo1&`Y_57M2#*aM{?_ucjUhaNEi<<azKNs8mROwwwFHV|e%j4I
zgP8b<j&@6Pw;5QnVV`8;DQ+x|WP;BeiX)&nI~T=~`wMMAMx7Z){#?<rf_ZaSKn8z@
zH+FL;y~KuZ?nt``7xU&$h$_I%WmhIjv2ITHrRb9|zx)XF)|~QdoG58vgV$Zn?wiD2
z2)^u|+N<G|+bWZhurkhpPLa06g)fE>vd?=!8if$m4XQx^dtf1?Q0kgu3lW9?tmHdC
z^IyiU%s~LVvMx8)mD$`DV_w;eauv&!j?<=^DI<?_Bqp%t>MsH_RjYZ5Tpma^HvVNQ
zutD{{;|`EeZ|}Xfjyt^Zgj&p_QyOL2Dj)#ST76IMLgJ1AkAlk@enPU+)S7_-arw&A
zr0TSsDRE*b<g>rH+Xlq4)c(0)q*)-A(B({`tMS||c?FSVohlt7Nua)=UML?klI*N-
zwOpIB1U)!##b`OQ5ISb5JhwCG!TmsBJ-A<1w=Q5lxIaW!DHJMTAreBrF64~$zk}Z#
zgh(xBh?LxrRiVbm43RI?Js?E7$Jv<tkR{1;x8_ns;9nZ|;%G%B3Ae0Tj7d$6aaNDR
zMbHCd_td6sJ29}{G|ixg4|`}%v_Yv%gH7>bsFy@9t=apg_qyrKmv$v~_;ZAqkofZ4
z74eWuWZlCkLgs=i10zC~{+@V_86l4;K7|Omv!>!K{2gXHYpYz_^#j-wo4U1RFkYIw
z_uA6b+lR52wgg^URG9kV<Zz>-YhhER$Fz9ajOvUbVKtI#UQm>uk8RrN4fUmqI4MQ8
z&EM!Klb_mA>L%&gjpBZh&_0oJNY)2}X~g1bJM)X5A6Uyw*6z`B8&&K!poE6%imdP5
zA))#5PGu&v-N74CLVK{ccoj3DeXFq>CA6;6#o$$d+yQSZ9nr;-HIlqU$@;m{gWg3z
znulcljG3$(Otb_QEnqp!Hn11hw79+Vhcu(;Bw1-|F~3VSrzvSoUK$%YIavmr>~tW!
zOjC4amKrpd$N{$-!dvKJLi<HfIWo}+D}bZ3XCGu=>j$$}=%@&iG6CM0AdEb@0bxYa
zJvnAN2hVN2dw8SXd(X`y!R9cD_kP}7KSiTJvs09YlcqI8W>U_kSJuEfy1m?^pVh<B
zzfRL7b_PdRg-8iFYbZd<s|uYXpqV>+=8NbOx)bAoPPK0c>GNY$ppx8vytP(o52KR&
znpsKyT)7WblGM^^SSaz4=vozHA<xaMB-eNsnxjfmcE2aHl5{p+D+86}6AcY;(O~z~
z)@CxQk<m{r^A8OL?UbI#NOU-}&C;DtZF<F`T&UuRoGGv?Y!UoxeW!5tlWR9NmjJTZ
zhR&&B1LHKA^tayM2srj;sYUuP_f<5_<(0R3vOH$NXF1vRub4G5yOV((TZysqcqa=y
zui;r^-)3%O+N^1Z!lT)%_)1lF;ZaQ@@x~mUfDmteok3B$Pss+Tk4-4Fo_vy*x<N!P
z4Cs5zg1-mTGQHs4U|P1@JV-_Np2;lF!jl3{MrXnC@6N?N0+c^1aKbZoK0Xi}Mb*8g
zWPmG4bUEU8&q!cTSk{BxzwM7bAhheU?(1;T^H60m(;CjJbtJ8Mc|%$Q(!62p-D}K(
zgq7BczLn>WjQMd&+}?I-U89;q7O%RWpsZ)jN%=mPfGj8j*wv8{hpk2Vh2&JN0-+=k
z|3z*)?9$$d#9L)<jrmh^yst8d)!&TyDo6So_$n<zFKeuUND1cjqiV^W%vur{2v@$h
zkl+bUTpA2(0nhTcrodXjFHlRK8C^@h8>uBX3z)aP42w%OurX}bylLOo1{Gp^RWfxM
zPfE;I^spw&;OWikVz^Vm%FS)sRI4z3$Z(aA;cB1nk5R$SH`2eMS64sMzfmXY-<ao{
zxK?JJZ#z04yIOfOpIKX`z1h<6z3}R6H+LovYRL5V?&9d<oaGmC9X`%ZLP_iFjD***
z4nx$7K8&-C?oGdT4;_8B$$HbzdNy8XBhh_OUUpuhz3kFhO@Gx|{35|R&dQ2DN2b4b
z7rWk+@&5cm{0j7S9D0~1q*LN7G(~Nh9bAzV$Sgwn^@Tb(hqdv&Pxr@%i$|Vxs&0M$
zA|bl(dVu4*6}FHQHv6FBmQ79&5zxw=u8>z;)7<8i^KSSO^G&i#EN!*EcfV#Ju))N+
zV8f;j2iHm5KXg{)`mF6H+vBgrW;Sc6qH<l+IOCCLaVcb*?}FF+HS#(i+np0qD#}Ot
zNh|ZP`W}z#o82D<)&FeAjVes&YPqr@v0ue8Doj^qg}G7H*KtP%qrzmSJaY}<Y!%Q~
z|K{;MIj*>#rtqi|Z>YdJ%ZW*J$(-8QTWmU1Mr6Si6+usMkrYOn3G2dLgYx7+&qzEu
z4!ncIKE9)a0&}IREC(H3Iu=0Lb#zAr+={;4*{&NME**J8Sjp)n`Z~Mg$azV%kEX;+
zb9O@MqM{||twKyGBt5*}3ZsNXF_2x{=>G3_a;Uz<IysPxn0usnnL_0N*^nfCKabll
z_h<kuad5lGBS#r_h4Ui!h{m&-dKYxR%?nCw91KXPhl(6h-Ieuok3y<*2yPGF4XhDJ
zbqiRj&e7-sfCSed)%E{bA=@d8rMmTpnW?Vc7gC*4*wY7=Zp>8os(tOZ(7mNY-E6R6
z&Q#sdGTjOZso-Rx;<?Ux7osA8dJtCauL9t9)tk`80;X-5zg##fv^PaCMj<`6MlMOy
zZ;7mD{>d7rzNsFJ)_O9egz+72QuT!xqI5`^7gkLh`?Tn}LV#J=Brr=#juEayGgzJJ
z<OtUh-EWn8WZ5EI%%WMMv^1$xTkLzhgN<IZCbVyZ)dU}Bb78D(WRu<Y;tYdv`SOGH
zl-8E?ggdd+R~ynUKtcA85QUNU6c7bgdkVpW)t=58-JWh7)t*A|7;aB}FX5`(5Z<&T
zCg(vc&9SVxR^3lP*0W6B(TZ**!`1g20wBw#y5A|e0RQst0}9&iTt7n?%G;pNG&~VR
z#(d3RC_UhQoNyoxQd^uEK(SU@--A~mVZ$cy3WTW0H4+sQuGMfeaY^fU0vHnHc4qni
zuQA<Mxlg-YGjcCHFr%wx4(~EiC*b3wi)%0+SULGO3FsZr$dgBXlWlFaxdkzr;}%`3
zGO$)a8jJZ*+G`nSJn<~frbsYje&Dg)nJDJB{iKz_i22+PgI@nUFbe^AP=>CAqH8|@
zc*8{(0`Rb+>nsz%Q_n2`0PkIF#HAoyc3pAOetSLx!8}4Lb>UUv1X)RIu{5Z-1`Bb%
z6-jI_?1@tKsorw_e6@TY^w<EvGu)7P<06FndrCt-A+z26L%c&&eq#c727=Fmy7dGg
zh)iBDD>8$~*4S&D)XNMQ=>2*b0eBX7On#=9wkE$w*dlF5swNQOQ0%|L$4b?b^GGTH
zk3`aRb9w#gi##HOIzA^xa#Y~75poG<cUi}oXzS?95;`IHQbR3l;dWA2RbE*>!QBGq
zXgMzMqp)V$_u)wg)e!2>;}v3*)<C-NzZ7=dayuPq%9akIHY2zeR#;7dYyAt&D6aBR
zji4i`%igI1{}6qHwU^+16$y<G{eZMRC`7cGmMp?h*5j?2%~9g1c0I3OX|C5u^<QfK
zo9OXLj%Ki#J8jd}e+4Ab;R=n}ZV|hY>JvuZM(V1`+bmAl<o4)}&3G4v3G5DEm}iLa
zjC#!r8cuJ$2_7qqL!j805X`CsoBG*6+_?g<@oPoHh6ypqbyJwI@s+sPZvp(u0O<Jm
zf;kO`Lo0O?nTh$R-h~)Q%s+>f`>Vj34VIYA+A3IyIY&2kqr;#EiE*GPA5TxAsaNGY
z#+j<QzM2K<M>cV-ToB4L3fA}J{OGG_kz1BuTjTzBQ0pWQ%EYZM?)x<yfOBEfnBxm}
z;M{j0->+xCdpvM%M|t5pliVz2+AYSBYNQ=#3J?O5Uaj%!zBL}mg9*@<R)>mUAWv?w
z_*4tAEKz*_bF+E@4a3g>7L2N(UNb8wxF~}KGlwsVDpWjY_@YKtP|#lfazR&Uud_f*
z=Kk1g)SD#+x2%pETa&xutOS%2#|h{Rna^rw2EMN}w1E-(Nc8zLp&&pRc%&~u8Ti6T
zU-EXiFM%92(w9Jv$_bG-y#>(lWV0AX<t_#y@c2n)G0e){{0~vlPR9WewYx5P*B|)h
zr%?HEkmJUK^cjk?9f!1l_q`W)AXr!6ywzH4*7>ah3{A#%T3iyLqfazhr}bG+B=3GN
z=10lc;$-Q8?YWI=)Tv=b+ept|syX#$U9L(vO|g|i)$%yF_t@YEoxjH!oM$}%Ib=B@
zJ;ik9BsY341`dCzNv_U?dybZTM-_`AHT@RJs+FEJ)u!bUfA(s%9tT{+q)Q7EHEgCs
zEnD}ax9f3eNa9g3VX)!(jkC<&(Lj(<OmHv_{QvZh7Ve^wn+ctRX^Jz$qTsG7Aefus
z76&Je2TG0o2KNTnp~lgKqOV5}Y#hv|FP>T9YR&u*QE_V`{RLZo_Xn$5`XckV{6x$G
zeLOdBmKLY?>PaZ|#7`70SopTv(`z#z^u>tcPth81FQo2#e4GE72e-^%VR2scV4AVb
zp=M3*N_a~%r2=(BGsk~RBLivN?qF+(6mUzlH~N-$Ox7@Ar;i~HERkT&S=bN<fQ4Vu
z#-2F9n2I&b(g>xwHO8fF(ngP|>10g;{h$!9YWaA52e=LlGg&gWt`EwBoNJQH8<CF#
z!owH8ALd});+*L5G+P^$X3Ye_YKJ<nsYljL5b8!<&(7d(=Jn`1P06VkIH@Hz+#GWF
zey+oo1fzYL2r_%t=Wre&9M~v*WWWcb^x*;DgroGYG+I-gutWJcG+Y#pwIX*VNulxE
zdtaVi$AN_q#YY|Or{Il@_&_5^HXhcf9VsCkM&=C&gjwTR=)<H4ToD5z5oBHH3|C@k
zAnRx&ogtus%+BzLC=^qf6BT=Fm?+Gx7@>ARVMZ5w?_!FFVmj&Cj&|r^<v<bWlD1G%
zIzFb0+J0<zzMmm;Cy6=xQh0Jr42uo51llDb<F=wnW_lkzwY+>70ZWtBuah20h+0r0
zbIYpHth+af{=>|y10G|lyq|)*t3tjPU^s_`CxR@Sula9E2B<Eo>>47`#Q<)@>LLu8
z!0=iYGO@yBI}4fkh!G{mbeMD`G%?e`;*{t@v!Qy%dFY-_?}Q=Z_2rxYqC^?fzK?fD
zIXTzlu{+Yi(=+s<!^J0Qnnv*){)K+9SzU^j(Unv?Fgwg@8>#hWb5o-kDc-B=Bvw8k
zQB@nDG@Vy!Nl@J*0e#c(Y_jgHSJz=EA7W9K{VJz2@#HXs^cgPdAI*{@8XHQQvs$mg
zE{>GlAT5CO$g9f+&hvBFWanx2>O%NzEGmaAH`J8IOoFXMFLkZG`PWBqHU0?>!Kax^
zv=7wzTXvOogui2}pDq2VUtA|W05s;EATdof*o5?aO>6MnSJukehR%j=aD>h#B&4bh
zNMCWR1ARc;5=b7Ce0iL=>#M-b4I}BX<21~M-)=lkgAUi7nC;N=LQ$@otG1cmk8Ufm
z_f1nHKMqi9DD+@7yAyOAtAUvHo$7~yTq>A~nhg_5ty2lFPActr?dv}@_(#I|sPXXv
zT_`a)8Xq&@uzxr{R_#pX``O6th|%eRJz=WwrYfW0wl;xYxiDZ6Kylp<@|)6~!mdxQ
z`EM?n1~oZCY#EeMo=a<$0gKoTSOhi4;z{_Nac?VxNvUKHu55A-E~)=zoZA|viy{U}
zfU_NgfdaQRiZ0qPLKi)Wi@tFeuIV|trh(oTTY36*RJ~T{Su356FU!$I^t`XHmJ$a+
z$v23!cT+l)<s<aG6-S?WLhn>~gFvI)MPwF)NC2U-V#Y8Hif@EaSvO3m1iBE5P>Ion
zJX9U45U~awYZOzo>qDlfmH<2s_Ds=RPt;>i(%xFWyGm$AN0W_%Qlih;HRoEYy`vDK
zRZj8=N(3iEEdifkde4%X)!)6xT>eti;r*Khe$9Ou=BM*Oc7=6B%_X4Z<;wgRY+c>f
zfY{go=N+pNA2(9)tI{A+T#KbzmEi*dP4}|~R3=O=#?F%67HkGoRNX(6T5u&f7;65x
zCZ$Wc?l-nail>pwHNrN)d#cW>?6XH-U!LAMC4)rf*N4cVI^%|lAA;92LPdHH{Y<9m
z{P)hq$>5E7u0$6$gEt0j#CGt;gpJO`B&XQ5*+%T0<!6PclY=sd+r+phs97hvr$wqV
zjC(>usRmPJoBv9C%XR;FXShzX>~iZoL{ZC?0d6<@<M65eiX8_=QS{^`w)UYP-#c*(
zLvTh)W{t2lMXi8i(bh>fq2E<oeJ(FSzcab27C}bR&;pERC6E8%eIMM#{P4)OGF_#m
zOvLCTxc{GdGKz8kb+~Q=QXaDaQt#3*@>Wt6OxUz(XNZXUHVQN1A2&b!9-@Thr&J(*
zO0O_8VFkTChx?T(_i(hXar_aBWH<S`#^M<n2O`<8Yk<zYU0;xZ`ewdEBH0>vf{lE$
zul>tBVy@1ECEpjl9o@Nu%4wn1-J6HUbnd{h&4te00F9q!d!#t3lr;`hkU8CA_+&6*
zgrfTrQFNy=wiwFKXF6iNF);oQie&fwkfu)!R3ka(CAc{bwC4-I0iKq<&zbdeNuy=~
z<Hz)Qs^!CvD`koO;GKwE7bd~(Gc~+umw`21o5$jxaWD!MIw<ngOxuq~mQ)%??i~_-
z4ZLq#cHf_eTZjXEf9o`8X-)<BhYpTI(lP_RyEvQgZgIsHG}zIP=cBC}4v4y;ASNFO
zTQy^dy7vy<=&O;{@#AG2D~T?f^Q<^>RMVR#!DxB~=f{h?jVk=yLU^!)Id$R(kK%yF
z#o3wu?=x%OsV7ADD}`?%*xAT7`v#`;<PFv35t9f3fyZ0fN5F4_?GTb=Ore$Fvw*lY
zS$&SdEu4(jo-M`b{c^7CMr%xHV!Y9sWXuA`p;qe`GJTB*J<^co=r{twiHzf<lOtch
z)mS)gb^g1=d)o^C|1Q4a5B1a9muOvGDawRK#;H&%75Vy<cOq(hStR3eW4eb+h&<y>
zQS#qdP)*QM&_tYqf4A>d5h)2PsBSfeTME8ps=XI?IJjf<2ms^-KkPNO>xN#1UgOyf
zVTUH)spe*r;YYEVaEW}Jb3g33A?0layy+a;WLvoyTPU<<E)-@z=QGmT&sZpg1%7Oy
z5Kx2%d?-Un8(#j-aFu-w`Qjmch~p;qC&^vB7EhGkvCU6cBuY?zI-XlrhId<Gu9G(j
zyvI3Ze>QW2kS%3D<41L3@3rwL`&{>~w=0Y3v{M_xqM(Y1g&T5Ok@K9djcrl_q7x*g
z1F>l5leio={tbco8pg<Ee?yrk)I~iGe$8ZLvb&)x5`P?uk_#HSp^=r(sfI3<l(}J7
zw~{okwmWo?@fV%-E?BXBKhC?b=L03tMd|J4ug9IXFJdkWesXUMD?Y3(NVMWJ(%S+%
zfBY@T^i%3}HG)GajkcFO)K<esw^Kum@FI^mB;543xBqAw!T6=ajH~j;01tZzd_~F(
zo(#t-?8DQ2&K!86ZJ4#3t*#ElGr0K^6Bb$H_Km?XZl9R2i29~}mF_oJ15c?hI_Xr~
zstKWCS1fQ#p%K!vyE#st9Sbh=DcK>+@*!Y+X?7cC^;tj!M=Pr=!t+;whC!UIthP2R
zzF;%yiXf6fM{|uAUR_ont_Z$IEah=63VoMQlJJ*R$JICG#G(AAN8V#tVIYC~A@n~p
z{mx|smX~Afd);_mRyqE~x2FRQY))+&P|P(Gq+i2MwXj{#ll}<Ga{R+3GOC+Lv2vWo
zMVAG<qVpJGFMwhK+pizR{q1std0YegEl$Q2O|-@N-W4r#DKdO#*nPxiU9+zdY?pb~
zMlX>qTFtg*<vFv<M;pq9r%(m;D8hCjT1n!#Tl&8&w_m<(aPV{+DSYpAKP!{omi&}0
z^{>2dj-FzS`?&$C+*PlLg>Gop@lrf@onSZaOVNC>W)>FNy$)UP)OI385v&<E6FE@N
zPk4*Iw8SMqwqy6DA(+zXTONG~3(<`;11QeG)C~B7M04gM<IJK~AhAVi_|29j`9Jit
z)utJG1hf#@l?&Tob;p6v4F!dYt{!sV1&>U}S=$+Fa2$VgpSeTVU%}J29IaZ*bNj~p
z^dPzdkOz(2PJum#^v~uc+y%-M<tPYd<#P1>;g~noMiM@JAe@i6O*_e6{^wtUHhymj
zI+YIxaKC}gfXDxsyYjdg*EY;Swv^qZob2mKD@D|CA_oyge1)`-v`dRtL|Kw!DMC7Q
zid4!{)1Dd?QX%cjlq8iF+Lvj*`+44()~Ok5{_oeXyXU?4>%Ok^QxLy>VJA&)cR&Nm
ztbIc_e(X{XF<3CVZ)gxIdmo&M6GcRHd!Xd41Ky8WT|vM3=~vJ^-sxpguVNl7A9pBa
zjecE2l6sP;#7bf}*zIny`x*m=m&c2Z1;ub`s;{>vk5gGRxKB(aaj(t+1`#L}@PY8x
zcj)S)t_8~hYe$r1i|}P?^T))%O+!6Q$H8qt`*DZ;EiA5|yW<C7{W*?@vQh*L%R2aJ
zo9rP7>C}YBswsV5(SGs9{p;_98OB|WBG)GSO@JcyjZFO#DosvaQaI);Wq%x;(DvkG
z>GdU!SX(56@~%cwOwBg>1aAKlZbtOLbS&MDXulLW|MlHdqsANmIivpDW>8u!q+WUb
z9(uI%xnW^+GXc12=>hrxb_f=a78ihGV%BF`*}3?qs`q|{_k!H@k4gmDuCaW@%Vkrd
zJS+-UmO!`FHN+DPCEX{MD8w6cyP(Okesff&>_|DiWz30XFK{{ME8Q^4H=n5k(MX9!
zbp4s_rAJf3KG5D7wY}xdhjDr3HE_A!_|YCqLKub>jm^8P1&KHa>hn~%gXd?!j(P(M
zIPToPzAx<P+<7VcTkP!3oAZn951Re{;d+Ipv=NnVg@|E#$M)_FNBFIVR&9iuX(F_{
zWvK~>kL6n~?#BT6hm^{9Xx|aDNSm}@+MlI6w_7(fOi7Uf=A<qS6zfBKC;{UJhZ+#}
z6l8OI>4h;<RhMs%{et??b!G!4(UoK6JcDBem>>RI$84a0`N4FvFlK&0;Bp4X3c>sU
z94o|S9?#lWb%hb~Q?Nmo5i-Ym*>x2Mjtu%lo=X^1*qlnKQRRUL>Z{EKN(2T&6Hp@b
zU}*ZTZ7wNTZk2>>UiB`IVarRy3qHKhSEqb>^t!hf^&tDf87gK80``IzWz#1Y>-kcC
zjTU$Kt2&cF^JsIk$>2j)(w?ntsX2I=HsrvZuC-sUi#3D`Ls&}z+t%6mp#aNLA@bNw
z?3Xy?op$=BZgNU%QngJ(%ue)LEol2hKO}xj<t|5`Uo2ViIZnBaI@FWnLbhT`rU3_G
zYmD7^E>hC!{_>HBQtE?sB&(w-qQS4bj$c`lO`-44B`O#psnGW)sc}+h&6kpAB?)pp
zyX3;2Jc~gHz!7<t@URy-H%V6?!$H?Y2zlz(6{f=WE$~n=;e1;cJ?mXV^kA4m+3BU4
z8WicH)S~Ta6`~1lQ}O5b1)y^FpjhILd<SWSauiP*hj--MH+3m~uk1s10uXAq$58nG
z@&Lp)MGEfPIr{t{=HhzxbA7#0-pTvkT$tnF3q};*xm{$8%YkR;V6>qb55x*<X``6d
zB?8-%Sd@A!o(V|*5H3f*Irif<(dTaCzdv^Qi=K%=DCP3+DOz%gXVY^y{TvQ11*@9;
zQu-;FBLzwh(Obq@0JJ!R#+Rr(is1wL7G~<@m~M~tq8iwV2%ImR6*u4DMF|e*qyRYI
zNhbx2du(^r0;|$4H&**^8th>zX8@d1F=W2~C8cxZSwU(O0UmooNH(-xh_r75vsbL?
zN+xx)lS~T!P7d07OiF|hzo&x%sX2xXo75yn#M{=LO=|edx{}DLwESxNi`4D6kEDd%
zTpHdSms$cOKfSB;9R1^6ftS$Mk64>OllT%_TF3^+vz3nY=C)yg6Tfu%`Aytrqy8;R
zhRwjebiRL04WF)T^(NClz<D8k+1kv5CSBGYEIEnLyiDrD<Yb(&c<-t2`Pbm@pU-L5
zn~unK!Hx``cp%lZsI*Bg=YF|NvJT-|5|V+dhQ(ubCw)YZm6#XKbS?{$F8`?SH?Q~>
zLbDU8Z=QJWjLvBm^%MWIH18!*sz<wpa}?WWkna(FVoOW@SvYs^v-S9iV+x=%1L%oe
zW%<yT%Pz-7m!DB@i-loXjG6djw+9?qHIBTPaI5xu`y=;5qJNEbH4r|D)AJkc+l$jj
z{%g9}72Q?u70}%9ilYd4L@kpjZCAE>kn}5lkqcyKyz2`-7@D=00zyjK)7s(@fF%MA
z60(%Ku0{Kf69pN=aa>ndr&pcbqvqM?_xBSw*OzDRt=wFr;Q8{QCwV#D6cGQAgJVp^
zAzg6v^%aFgGkvPjV0CWGvED&liOJ&f4TNP^pD?a=frnFecs)>*GYHE%8B`MxgDL~8
zxaXsS=L<D^?dAie_K}2HP_wsd@;vk|<Y-FqC;%bPp}381r||>n?4CP749o07fLB7P
zlNc6m9~i{2z*EU2hQ&OU%Qw}VAo9FS5N7W}!~C>24f7$5{R++OX_$v==^QA!?7xYA
z-E=`!0J+*e4wdYod^-F&9Jv*ocoA+zYva{;9IyemA|0@ESRi!5&RO#KGFlM^?3^t@
zF^918qL=P6VY1}52C`jwd0pzV%jIZBdjj1wvA6{7U~cK?gZ2220mLQFEijOtCI9tS
zv1KRzk_Kyww7_y@X?j)pN3iEJ5D7J(L5GkWxfGOkOqY_DoN2f#w(G!RfJn%kbBdn2
ztMaSgOKFF1uUgqNs@1a!2#I3~A&zlWP^`^yL;#AQlGcZ^E~4c9zd{=fh*H?T8_}v5
zyoR)8rAkyZu1MP#6rk8;SIU~w94&T+7|0Q$^!obY{9j?#52D6irmRO5{ao-S>^iE^
zF9Fv%tE=kXzNE9^DE6glbzzb6$-@S?`6bBb&;6!jb&TBpq>*0zS!??|`;;(&2H(Ku
z0D29S>C^<^Am(fi_|Q8Jm+6_A^mKXP935Z|H6}-IO5N=0sQQh5rn03wH=(ofjn~+j
zgcv9BXz79X6T+m2jiObP2mi>`HkrbN_JLMF2ec2g0+!`$8$)OXbXjd*2wrU>zB1ju
zmPTz=3{0V2H(gelW@KNq2RdPD4tJPYQGaQ8lziqvM7s&(8%vmJxRzgoJfdU4-Gwe1
zuw+cSXgD-3dn<|=^?8TL5sdo135e$-0^*JqzfUTfeolFjX@0L_$jj?ZQ%XWn!K5P&
zcU&Bq<Sev?Y0?IM6Y}jomsMRZUw;C__o!<N;;8C&iTlF|qh2LPcoyGLyVPIC!o-qj
z2QnjME3>l~HW~_qVBc*^`c3>|0!x*TZ}*}GFRz7X3fJ+pZnUolAh3NB{M%Vk(sYEw
zW5AMzN9-CK+;KQW1kofby&>PtuCL$z0(SP+zSDKW0V!?ZR7Ry9mm39OsKe1a4Gww3
zd?z}2+%IE~!@kqC2bxzF3r69WiqrwH$QqZ(;I3fpUR~8NShgYRVX0`hA<V9-k^eqf
zaa5m=^fP&`i~n`kq&-MK<!QRO@hr8SmtKk>Fdz4Q2sq2pQxyg8B#J-;sQ`AMK8H#i
zb5=3&B5da`{cu(xK(j-MpND72@=vBIu0bAT_2|OF+<E=Z$vR#t-0f`OlCh<)NkHx2
zXV~L>N6+Grmo=4qUuV}ad)en>8MD`{p!E8py&#RUtQh{&{gr)~Ef~tap~v)=HRlMT
z37H7o9UuKMO)rQ~ux9CI-Sip*fWEyIUgu)^VNcV0mpUN_!<WLQw%=tHR1}*KZJ*j1
zV56Ue5p#SA)R0&P@QS(3?g9q}vl$9Di15Et!3NlDzI2=2OEIJe)|AKT#hbIj{+n8_
zn|fCQIdHY{psJ1iH68bmbPc+|w*3RHlFr2!LD#$05%_k&o?ijpy!OQ#*G6-@qVh+n
z%eLnfw9HX&^F`s|?Y^N^N&Pmxh^}-qT+EYC`<EOL!stISXW-oZ6*_qv((iglf7@4{
zV(lgSsI5wob~0TTzQZSQ3ej#3G`jsRb(n@V&NLe9)cOy~x<Igj)u9HOQ*6{Z;1m;4
zCKu!$wl>RvP+<lexVB{ipnb(Ml+Qw69)qP1F@^|#=F$iqj*31CC=q?>n)FA|Z9BcX
z?=~OKhX8t-hWIAll>5VnSU#6tKE=KX-9(g#<T<f&Ft`<kkI%xd!@;7S@~>uK<Uk8d
z>Q^xC@-3~(&5srGt*QfzR6XjJm9D2gyw<9|Prz_=FB;%(B!tK`^!XiEL-1!2=Qz;W
z_(B0M)*D644bn}(i$fkRB;aL363{#Kc5iux-z!fwyTlJV{+n7L4jR>^M96AEH>JLm
zr*UsDL8$I*eShtF9X{o-14#h!%aR<6WPTB@?N4H6)#)jcfho<`O@l#rCtt1DxVIn~
ztB>w@VYC3M`ZNG4+|d*cj2f8~zey;uB|dO24me@eo7<_54(ZDv^tr9HE^RO6b5e8Y
zrH$8(bQQ76?tV9Brm>MG`tpj67XXKGKO$sb+vE<>=)$)ORd;Y8v&p^Rn}VxItiG5W
zunC@Ki9t2L4BZPBh|zJIRnPW>&5r3txcb)&kyDXRrwlC&K_A-{lhCdBDx!;G5Xk#=
zQw-9=r0y*AOS<tf3&tY(LJqDCM*#vf2jmO8UK@JBc`iN3Cb%}$9~l&E04RH-Cp%OP
z*w_Ly2(aY|^(w-IfGrchkNVL!@rVJTZAsLtq;x*`z!EBL3}vj$Yn$o3@@WT-_WzYZ
z-TH1#^z^ojODMZKQ@r<e7nb?T<@?uP9=9rK`%fF?=Br+P)~>4D8B$P%xuFbwJV<Eu
zRygWLev58zW~z^6AmH{1ES>{g?Y(ia1XuePFrs}u(bnbPAnW)m*tROVQE9${50)33
zrlg1K05rCiq_}8WFHJ&B)X5a=131;;^TH#dZIw`!P~4;FW~KS|6RxOtxVJRyGNO(4
zn;=uVGnt>-Mzybl*|@9=s7YalBg8So6d>%zD+r^>^HZ9SWvTIFbKn;(0WwQw57A)`
z=bhenKx5Abm@(WrWR!U3v~IyBRU%TsOwGRoR(D*4GlvY_Sg06oQ<vd}+(v4LPdn(9
z_Le~VnLFFJnw6VjA!!+08|+uRvh;fE=|Ibdlq8zRwbq#EamtIvu=?CO38;wA?PTQ2
z6WAhEAG;nFEo3OLLf6Pf^6a|+utEQ2+UZNW^Qlp7TAnTU36qXi=H=0c^MgoKx8kN3
z_Zj}!4m-e770pYWDSOKcx{I6*gq)PXf;3~h?D9jY%SvT(K2)~aj|Az)E-{S-)gziW
zLZ<+NTIf)Sl$i7;$~`2uLqIUTQ>wFxl~gC=$WsvOS+du~o#Kdy&e-YVnoDi=iT&8-
zjwU{jmgk$Mgq;hit*;fV{rJ9JQEQKoPN(Ov;>W?BQ!u@1ax7X>2nZ{aJ2}n<YS|q7
zr#n}Dd4PJ%@?0<{xiZhDT~bq9Xgf!ie4LDEZ8Zc$lr5N+a3X*EBM`h>q$kcxd)fZ>
zb3}nv2w}_0h%kD#v}B{b-mFJ%vpRbnR=3hPff|i9JA-zzA}&{a`3GJCiszE<j+nz{
z4@$3_s@q9d=g}8~%)-!F{SrE&bHOu_t6mCS#i2~w#}bRSaLCL1BYNvQD_t(??x0gk
zEL`jx%C#W^M7POT+qA2rr5&6&iKAj4)RtMWFQNOG$fx{I2<V-N3#fa3$#L{|MjkVk
zuf^I^bF96l_?beUv97(5Z}Ta!8X~j%S#%o0i8;sf!f{u_1G2yaXb!=Qb!CV2#JS3&
z+N(ZC$UjUEyJ4DQKm`Il`ckeeCCC2RC>hc0ZN?nbc$&OX5FJ1XY=(l4z+C0=1uZcF
zo9eHd-piJurO=kigCVF7=$lrPVMt-d5^PA8f+AKJ{$hHsE}E|0SE?MNrWqo<T^YXS
zD2qjF)s0rUZI!Yt(<arh%3jzpOX)aA7BTR3CV2{q;ypE!JFkU@MRnCo9uG3t{c9%Q
zpIm$4*4Ky%Pj?^xptcpY1NN3?cOj}J1GQIa%w#Qn-&iEzIurz4nFLtfh&dRg00Ua*
zJVfU}4++q(?9{#qCe3|efshd`$xj^Bo7&gf{M6fuP_i+d_0(;AN3!XYDFNnc1{H{k
zC{F62kNNkJeEt?1VGvGN0+Oh#&$$V*R%Sl%I-WLQ4mEPz-VMV#%4e3GO1ONM91&8F
zQQrc<dV&0>S9{ZCfB)mVH7}C;>)L{pT)zqMJbGkB`>K*xpc6so;>vNv&I8qDElX|@
zgkXo+lcs?jnsJa4Q|g}e=muu~Y>8*V1d8gdvF=KB^Da~C%j`AHZo%_ZHbT)k8bQ%H
zdY#}gM5(qy*_7pIfa8-{S7|JC_Du_WsKAgSN$IlEytELcOe#6kKn}+M=#=}70p~+z
zppCt=7YL-YoYrt41N@zMoI%#?xWWP}qdHqy8OR!#-GE0?%w-72gwbd5BkVb1KJJry
zmeH#Hnfnr<P!o{+2rI|od@k_K8({TOPQh~H#Jnq_Jb-=i{j$=XFK$Gc((6eE6{9)%
z%uYV^evCT;*|cjtMuT6}LoZpTQq{phwsP3esQ>1JF#4C#QVTj#RR101uGG-dYOgF4
zwLBU6n=5Cd+w(n}z;jw*9=e31FuiXBF%~%DBYExX5zlZwtyRLq5Y<MLTR5XShP2OS
z^DubgI6^!-4bfho8xyuX35>~tiGY2rJP6!(G;1s={228O+`8fVX3$3U0tE&I)WFeD
z3kWK|F0it1_DHpRU;YPl1)7azsmbTiyi-!-UB)nuo<@7aS)4(S&0Wgm=o&CQaBR~R
zu^J=)O`oCiyjBTN;9bsPYbZ=%`u^$~(U6HarpfKkkr;3@ikq$|H57bkbqh3?310pT
zJ~KliLTehn>R1l2q=aL%o*?;yr<|m#zxThiDbbvO^+H>nfMmCfW0Gd#?tvqn;pmCn
zdQNY()>Hk+cC>N<_{hR@Z`(baV*do~bhA%}ui+@?j0?Gh-v?2^uxr?I7x_&eG>Bfe
zZuP3IA~kb@nJVo+rWcLVdLG-jI7X{ZT&2_!`h&_F`FSx82t>)rcgH}#IeYDI8T=(v
zg718&%(#H%Ik#YVb!A(4`T}S|&0=0<#<Whb0MbZdkQC&|oKL|5#Kd!d+}Nv2H3o{$
z3is6-^Zn<Xc3Ch5ctI+SXm!}8dSHcXH7Yn68GM4dm+G$ojnfMw3eZ;vzOEhZQ~Js#
z6-xvIn(EIh<>prw!wHcv7XE`1BBsC%X0aOVh)#&!>OU!xS&b-7VUz<h6*1;!j!^2*
z0C_}9EeLCFqdo@2TgR@gKbvmpEyEl4;ja%Y{nm!GQziLp8`9#!E1y593&m8XsfR76
zK%}!n{U|jap$q%v#APJZP&a;@P@%?SSyF%Tw(Xnrhl~{tH!ty<u%h&Og*`NGF~+l6
zGA43Vvjvnq#WmY{zgOKi1p}dhEWod<V&4AZ^SSD_N9iLW6x7Ek(k=+Mp9JTTjT~p6
z%QZ_k_!jhN?`d&;j|`7z97G%^BEZ7Kx=r}fi|Vgm>hcFW-51?kn*L#XU`q9MQ^Qag
zOWO|zs$L$B9>>4r1%4c0NBu63^G$u*Us}&oflMN-tqPr{wWTa8b<9<RmF{tmYpM3a
z356LP^*fJMt8l;b>X9nT&YJVhVU-BJ;{W<B^_`WYKvM^3XLQZ1Af{0OMU+H6zAE$_
zM}-^6BoG-#cgYo28rz)nVUOQ)NodmA*X##4$6+R0xqy`(^+Z19d+mZ$I)9zA%5963
zWs+@B4A4IkE6@|xM>xtk2fT@#leT`KJ5j(Qi}~z~%4y!__fQ({i*=-p?Av!08|R{7
zV84@_TL<P4N#@OZlw`Whq1Y2oNnAgW&y)~isf^SOP|bld(tHu0>q2hTmF@4+vpwLN
z_~ENuPx`Qgq5>rA<le|2GwLqc$Jj+}T|_iDw^SamU3`MrbOm4!#xaMt#akJRPcL0|
ztu%*=PmnXREiQitF8?~GvxZ*EgEij{Gw8s&i}f+awm_zj%ja1Kysvij&s(})3<La~
zaWUMxAz)q%f3N<zSo3*~8a@7VdWxc|M;;6~ta1;RmYg*SHN$fn27aR#Mc{dW3;v_(
zi{sXe;PJ^FG}m{&H_bJN@<0wIqLT`@-%F{yZn`)*s=k4ym<5c74+Ylyjf3w;teOGc
z=*7JGZcV}TTPk9>a8U@581OCUG@dgH?Cq_Zr+ImXg1%Nt*da-)*4mg;TDrk#5(|q#
znB*SpA<M#83=Vy>9wSqE?z(EjFwn-Q(r5jE=`|&qh?Z1Ek?rEk-ykO`NGXl3d1zh`
zO|Q5X`{*i;T8yjN?n;S_dGlM{B{=l1O<ymHN@Gs9PwgTgqlBVXpXL4@z<Wtx!X6Hk
zZV80#5WQ1zn$M|#uvN8dRFl;(I{?YC%;=CQ9nt0uf%;dwpij11SU*bzOvW>SAk~or
zxDzA+G5E<X))OZy^8ugD=X7@nz$N$5*_)r#4VC3xt?13m2z%$%X$SD4*2S;H@Jl`~
zu|1a*bfwbS%7Kjr_-UpFTYuLQ*c8>OG{S${>C?X4f%4Q^ntgG58N(Vy+hil!3@@^J
z`#gayi@K5Sa%mY?%>T`YE-P&>_(x@QshRDm=Ux+c4E4G+d-6#|=hL5F?9Pwa-l)2<
zUe#~=?ac6Df^+8&A>DG=Dxi}R)xdq?_*>Gc6NPuq&(2uAbfafgs|;nG#M40|=SUqn
z_`blJ=k4n4e3P~;I@R<K>AdKZ;X9UD`|!5z-X4>h9~IcXJF~uZS4`vW@@;-Cbak_U
z_Ttvb|MMG3+L<&GqSJ%t{6o{$BPvf39{-Y$CtN^zdzyJ5f{yT3y;(~-P!XrJoZHJN
z5oLwF_j!k?EA1jzQchlJ(JkO5%~76exV8R{ROSH?wO@bGJ5K{@C-Jgn+>stgZD^wN
z;dSSI?u(4|Ik?LQlfSLW7TNE!W5SwPDb=!s^+-1O(7o|{LIcLkT9C3^plZUzx(m0s
zE4=PpX?d4fwdo@S)W9*j?aH<r{~d4+jpz+pML>B?f9q7|*lk;&`p{c4*g<a@@>`X|
z4m(s&;iParcxTj0QHVj5Dx`B&AM4RFLe5~<|JGMKz9MTO(yT|8JlK%@$NzvKH)NeJ
zEEpqs*J`W3-jb0Hx^u#m(8T+Kd))$tAy_sE8}eY6&uA^J=Sg75zAlCUCDoyON_!#$
zbZ0Gq<LO|-9fHH--w4uk($?zOCs#dzF!VX6)sr3|OZ(`MY5$MI0b3L}^>Yz5-(IiR
zd${kNhAO1nyBvSL9yd-r0VN-ZcPZVCv>XKUe9wuz&T#dvGbGtcPmYiL@9#6EFyKlz
z%N^r<(%18SGX?2(dEH$bAFgr{Sl2HsD&cX^U*NvZGwyM2r$+G9(Zzcf=I#v_0U6Jo
zm&R!AJH2GfL}xEoL;tNB`-Rq)E*zGSdv)mrq)%PpSZ}gf!S?gMn%JsHWJ_LQQM!fk
z)&OC)kMg(bGyEt>LKGgw%BMJQ(J+!qg@bhcIQg9L2bT)Qpsx#fzjW{O0#)ZUa{goV
zb+LRDXfYU%qRG&*f=A);;rvnJpS+Qze@+wRqq#=ZE-$^FWa{O#-M*yIL727$Vv7sB
zL3RkP1m(SBNjs4fKTw$<66!Vqg(z3a5&yYj(q)02NzX=48{(z2JS9pAI9Fssl?65a
z3s|(FEMKI>{0}6KG`Use^PF@Oc5jlZ@h)I-htP2XT-7}{cgJ&}DSIhEC9`qju;TsR
z{EKj(5w`y5rx2gB{~?G)xQ4C31ZAqtineVLX~-07ijy!+SSrX(9R01iGS2502=Ti@
z=-GzKDb<aSfzcK{*S3P`E_0#iIBRl9Xo~P5Zs!|iQjO~eJEpf3Zn8aoKIGmX-WHS(
zx%cM-(YNYH%G1kR6AdIqX8PD&E#>tBt?;vVf1gyLx-o8#>biTE&<0X_?n<Zg<nRvh
z>ApX#OB(t@lHEQmJBjvfyeFQ#SxcqYrKi#!<?Eu%5M|U}c*dSmSX3VqCHTK2{0s)T
zj5@pCK{RwBGB98+zN7=w78;F}PdkSj4X4b5HGd^O{^0In8aR;|!*na2Lky!BB_bVQ
zPQNk%`GWFe@be=P8=dYSAQ7T}xhxJCZVzr*iWqKJBLSiC`$|Z&>bl=Vl967{-=`rz
z1n_4h&Je9nPNc}T*1c-A&#dwXhiSQAM5(ZM2C*R&t3oPKCG}v7t_aumH4p~;(}x0_
z47DNuD;aZZe+ve|!|`zVl;vFVsfVsxdk>-<ur<Cd6Zf~;cDL6WBs*y??Dn>#<_qPa
zm0a>q-YC|#X+W+E-IDI5xyx@INnz15*Mm;0LN2}duT<o-%9IYBG%XDQg}o69NKW?h
znFmNtcJAq8CXZg9bw%1i?{|7Xwp2@#SAtRSf&6m`G37TQM3xm7?Xyrqlg8}q;ek}a
z6_M>1TNzxhxeXf53cA^MBuEZ^Xc&}n;5V+!gDATBf96^xv63hYgS=60c_lurM+`ee
za-_PfLd!QzLEtD_ZWwWNoM?U83VUX5*^zl*JkBjLNXdBLYg5u;ztR1AKL{Z`OIXhb
zxy7_=NUmB-QLo>;T7g!#ugP;I?WOU3X<syGto#VhUo92shHI&Q-Nw<!7S~cE)b8&7
zKq9S7GpTX~qIHyDKm=*mj`dC&O_d}hQzg`(@ovN2;t+o517(wIowkU17s+osciJq#
zM?(U*jLL_A2$CiTelwf<KUaPKPQS?!xd1#;%JUqNDYgx2=B0;V*$^{ECMJpxUgbQk
zNwu79Odfmn3f|Q{al6&y3xgvXwJF3GHW|Q3^UvTfEZjh_;|nVbeU*#-AH<xH|999y
ztWjY${aOO8QA3gtCr5MG{3!__w{`1p(%*~nclgEuA!Ew6QQYKUjoI-~;BMlsC^~Yb
zr9yFqt%Q}m#U}KL<rO+)^1B3iqlDORFh2Qj89ySpAth!h)&1qPr%oX;Y&*!#XH2mu
zpT!+)rm<ieGvPnTaso-gd+@Eo2OyD(X>0kTD!1IfmoCBW<N9uW7=`h5SxOTtZg|9v
z#d`=>L47&6%1Z^4xu=g~S6Rq#6=OAic_nU;ig^<BJI_g9svV{35_0#5*rZH*X{Riu
z)$FqT(3f?F#F9?!!0r!Y8NpaSz2z+g<C`Ii|L~@ujDl1-Afw<cFT=?4)7@X2c#&5w
z%m}{f82jkL&IJ~vy&K{ctayeP-97^4+Au93=hj8!+$eM*b)zjyxdG|8`aBW3PB223
zp&sS$)1cVoLCT<f5~<IPX{lT5WKYAESVaok6bX|e+g?|{1KN4EpH`DQ4}Oy2*-D?8
zPqGl6WH@I}vTfzU6b`H*gn&4*CjVOO@Cb3pj}>%BVRhO{0l-k9tln=rkx4VbMa!0u
zt&uNh5RTrZl0=SPSGfZftA8|T6mGa%5n{`5^i?2dgdE)z99<+}_V(w~U^yas@dgD|
zUciPLi&D^3P|<?Q3%46kdBL@t_?Fp>%8N*lj=7zVzGa1iZKLo?TeHvN7COk=>9$qd
z?VE3p`F%J2C{09d6x&CEbVx*DpqTrT3jc;IAnT@~!uY!05&-Tq3ggr6bpl)kn+Aig
zV}9K<DbjG|?8O8P@-2Omg!1)MGc^PP4rzG<2T$0t?F)c|$MO6`K<zFU^*>jvQ63Xx
zVeuFEddbI+MfGIUe*-q45(WN=vy@erE`$#7F0v{3Dw9XeFLD*|01`8}l#E~qT8^oQ
zvZ9oZGAge-0i!pRi8x13I|E(ErFa>j*)hux;hm*fe*3U2Y5`GNe7n<6mys6XT~S)B
zF&{Mo{7M80ovR73fqQP|j+fa`dHw7DR$<~uTzPFHsU;;t<y922)cHSXWZ#^E%8N&t
zPODX<(!)F>rEfwSIk>c`sxoz|j7^yqPq5ihwxP?T>>=SNIpK9su++HoL~Q58PpXGA
zdzJp51Tj;Gv40ZSkg7`hpQFH&(96q`3#+p{U^Tz<FuzSrOQx}ZfC1Zg`Jhmb2k)~C
zb{)Q{1#($V`HLno>U+nW@SsaPxR!)Z|9>ATk1ub{gcc6fl8r;;pIvg~XY25Pa3A;G
zd6})lGb*nl5!b@vBBNXn2*Nwv`Rz&@Y4r|Hne_K|K>-`t+VW1Jo<M&X?6TzCmUfnP
zG2}(<xwfMip_oxcf-Rj@B!uFQDiT7mfCsaRd^p9abtuZdvQt~`I^EJ(NV!&I-&Q9_
z@yjo}4%8+QvBLxDTX@L9Y(KLR*UKbua@nDliz*g~YaJC!pDpo^>%P`dwqCH;v{(wB
zzqgu}8nc`qN*4Q%iMdzh*`>)R$%n#=EbR7APcf6>2?;J^Sxk0mh&ii_6*USgCys?w
zk1^C|Pnd|fR*E6j#@&mbJBo_OU!4z2`1O)1SV7p-6%>Rw@~<;-rudpsTBJxm2iw#X
z1wCwQtMuV>%d085Y&Uvs-c|OEee%R2GJI(3oV^STkkfP&E+|VnKQi09Ac|j}>avED
zvN%uUAE7xga&eAK9|k`^o(sO4v>il1K5<{Pqyz%8xU&DLTYGRoMrxWUAQxmjI*;^D
zk@q`jEKCZddc0VM(9_jXc`01Hf%G?aPXk7>^B9dRh1;zLvRD*B0LqA8pd}uyysm&-
z)~dy;nx2>c4BRCw(5R`Vg>3p#Xgs^TPg<H*X7>ldT;f7rQu%#8vafHyf@Dsg@LgfV
zMZGV5*)(NYV?zq0{5Vuvx=;+qu}k}uAoop;x&m07QNUR5HS5O#j-Y7{&`<prF9T#b
zW_jjAaMo-W0dl&$JLRZCp3SI`q5k8f{~~64z4|X0iJges`fsB>HEp|WqquqLL2#P8
z{3~qAJc~nfeL1pb1><c(mz1!4o1fQBf;EnDTkohxOS{6$td+L5q>Ww%^#tqD3xgC9
zMlT}zK53PSMKVZsLxd-6NzuUDu{Lh25A{>skhGZn77HOXyEI7J*2KqH)_kt71|^MY
ztb+iPht4k&1%YCzym`1{2Aa{HBc&6JE~d?#yN|h8M;G&|YZo(%(ZwK%p^h#F1~8^u
z7UKa7(g%hC46}<d>e9uW7n!76GfUBk%f6yY0mbHsr8btdq-6~a|3g#b-XjYptE2yC
zM-6cg7mBeuQ80(EvA!wSIkY?jLrX?cQv(AsqNsrZ8MCM<<#o_rpLn*Q;SP)~`WcBM
zx?VK2lmie49*Ij$SQFPV5=UbTJQ4@NBs0w&yz+5d{m)V91^K5?IFoD5n-Wc*#|xpL
zidLAbgUBd*&yg39b|v9H=Q+P)@}cD}aZ$uY$e}Y`I_0|q(kai=2%I{9tj`6S`|ZF;
zx~auH6bM|hvndtoDMe&0w4P)8Kn}(PvK3e$rz{^zjA1!x2Jngwg*6jS8h?JqjEr65
z4^>@^hQ;sTJ5^b>Rk_iPjMr((oxzdwFvxG5@@Ecww`nq6A%3b&NtxHfn0<sv4^fjQ
zTsk68l67U%?NgiRpk<7CUj;0r(;?$fbe4bI?aSO+yvaD>eF!M_TrfSGrXXMp!VPl)
zbc`o{vz3>#-zG6!r*oUspX&&6#cd5~v@fZWKtVt!A2u+b6;YGcx{R2-_UkzE$rByq
zjHxH_tgOcT^uV{BP}aRF;Et&-8}1`(VlfpyxO*unyQ5q73LLlQXIb6&?13Ra`Un>q
zk(MZPyF89MgU$J@lKCaC6Di272l{s$Esg04CE*I}90w1Vp}KMKJw*SM#ye(8#6u}C
z#;mcuG^XvMBd>ak9S_C!7|SkNStQ(TG_j3wzPf;Gcp(ol`7bE{f}ZV1^`Fo{R3#Tm
zR&RO>q1q$-;pIR(cM~a(eZ9xZ3|7N!0iPYWh{qQ;<%$w(GIDA;lmy?pgoTbknZy`n
z%C?O})5D(YOHTH%B3J5O)%dBm%Cz#E&ocoZ+Qf9B$QHX|<GVU6>_S_np>vsbL7K#N
z#gbFs5_WZ20KDl~Vxq&>jwL4dDYi?@h0y9uWz*^)vZducS{=%s*Yl!nEqLH#TYSI)
z_GL=N#w{Ov?_(ySeav5bl+qb)6KzYNC}1ttbMij|CLhezGQc|?WjUP}_I1fM+(S&7
znNtoJ(jI3FX+auSp=BbPpmET!v9iJotY~k(E$QGHF6{*6=ak^GDJrs`AWT=kWw~DJ
zVGd8UtqL9s)Z~i(D(2MQK9^!wR5cMkvYmC%fZCPWbuR=Eb>ZEN1EbJ#X@-dJ0ar=n
z{)q+U1Wc03^9yv+@G$2Ni?^v)rxb$!FS*Znmh|ek)34N-byhW)FsipPP6g$dU}bn=
z0qg$eiw7Lv(tD0NwobkAQw_bP&VdrM*MbXW8($bY(_Yg!vSw|;Cfwz?Jhq+HvnL?d
z%oLHndjeMYg|FY03!b%1ms=%LW+2G>yZJn0_{gs2z0I|+RwTg$YKjz8;|3j249w|w
zSJiI;g2(_#=3VI84)~A+ETvs>>KwY3wa;*{{F{q+0$;uSW1=x+Mp~w?76ogJxME7n
z{5)IvA|{~xxzVU5w!)|c2HC+K>kUjJ>3bl+hrv60SLHC}j=QFDo)A_hk>c83|BVZC
zK?IoQU;>4>o3*vPSxulc_q;ZxQv;)1jj}yrU<{?Fb;gJuC96?W)1qFu`O5Q$Y|Y6M
zI*lJ^L}N(bD@`^_b6wKLMe(jJgWxjs%DQgf+SLR&bz|T%WT%uMiJy%Vfw~!R8O-g#
zr_SxbN`WC4@3y<=mtBK)ZK<SX7){0Hgzq1Hx<4$+PoNs$j@oRD1)W%BAbe;<=i=W4
z7XRP4mPM1qj<qaYw}#E<=SrTa18do}eXV83UfC+zM7~kL?HG|J2_Kh1RjFO8ENuc8
zZ?64OFBnR9tZN94#k;{*F;Ol=Z9f|gpdevt&k3Ap1t0^XNjtJ7I2vJP`rFtLP1*+o
zVqMN|ixw^-;nT)`i?(Nk!riGh$=R}xq^%)kX2v^0jv47hd1K$lyFgpbc$X_`6L1Zq
z5vthHpTV_meLR6H`?=ddY{!a~QGD%Z6x9#Y1G7A$z+oyhsdXoW=}oao+hZ;c(JT+_
z0Ll?;tK0OJ3}XW2n&XiFKMh!bN&s=5?GuPE4X$-D#Dwvs0kVt%0BE{U-U$HsN1=>^
zT^ivnobA$9k|3{hdA8AvJY{~jsi=CF3>Cm-ewoOYIRXHl=n<E@BK`<No&>C88p8Qf
z@JGx6WSoz#Gisg9)959W=MjM(#j4ArXla&T?)oS_AWZ70JL3smf~PbIMF$3KBS<W1
zdA0Vu&TY35sG?f8G)0}s4@04BA`H>$H%QI*Y0v7w$N^Vm56lk^WtM9E$Yu5kREft8
z)NZqfB^OiGvpmA#SvKtTYz+x)4~?Z%&?h*>Uy5P-EaYHJ*pW~qGlbi-EXdlM6JN1=
z%tT>^siZMbn2gMVmQH{jz-aKZz_;vLA!fi>I)Qwv`~2~YrBjI8HCQ@P3=BxjrIQHP
z+^EV*sk}nom9~c(9$MTsgi=N+2hGl&0_feKqaPz^BD>*9@f<KD_!+ukBmI>o^J7fu
z+@FCdo%^#8H!oAyL4+v{Z;d_wMR4LBf(cMwk$*FRI!4pt{d;K2g7N$;am?=_^x-bf
z3lwp1F3Y|%o54cV6~>5W=_Ww|2FU#>q^mb5HQiu;2LSeGCCLB<{1<NuRKDfmm0XT}
zVdb_UtQ@fs1b&OJ*a(V1+Z#2>sr7??Yh9g1`-uo~ls6YteRQZ4p1LBkMZEQ<XbH5;
z+~*gp<acG9gY<r)U^C1~NLrnCyL0l)0xtVW0FTf51k5ougLn?W0s3ztH{guBq5pzp
zxR0*eW|UaOMNdxKUO=xfpNBuU0>EWD2?!|5K;H~^pAWaJ13LVkfDODA*sQi4g#pJc
z5;W3xfSD5ObPhn7CvJ9GrxVZsC+Hc=|0hsY1}F#nFli&G0WP;V3Py%6w{9+%EvmWl
znrtBG`1I0^v+EfrVBJuelzq#mt<4F?l!V+~Wv2!h`HOPqLX*&QTrt&pbWa+uMX3P<
zf`eHA%a#om{9U<)?bG~8$e20;oK0h3_s_g?FlPbqzoKm=%_gbp5z7CI`R!`rV{3})
z1xrV`zkbB-WfHTM2)&d4T)A1i%+K6$IzmK?lFG}xZkt-=k4!NHkVDJbhFQR~1Qqj4
zWvitNM?%Hi6IsrEt?ZBhgWm?oJr=TXm+#*$Lkly9HMAUkU?Mz?YoY$PE9fvp-yVEF
zT`>0?K+7RApH%2CGG7v!gCp~1)l)3u7!L)8Pn<x^-wbD<8fO`qt%3pDaYTP0F}VX`
zdKh|{1SZ5Zdk5B=28gLMd5OX7cu}6)8DlCa(94`K2?ap*r$YO1X<g*|6)C@mP?G_d
ztHCZH^E~iQz&UdRkQaABV|T&IesyE%&I<6~cVmfOWj;^i5<%W*3yTz#FH6TnTem*s
z5{zu82+UI^vvqrt+G!k-VHDX3JL!U#g#{cskO90bOfft0vM|MTVd7<-yWwRIGw`yA
z5{ikJg|!(6cv+NO#yYk>xfaVSYg<Bln7<4anR${n&E?xo{#D4&3Bing@^fN)7nyiV
zE2r;^$7S+q>zT;%JF4pFI)?q8sV)Rv$Mjd1IWWLLQ<d!i4AfU$CJJDn2^1@o`d5*v
zQu*y#t1cn07i<Vqr<HgvZ7NK!F8bR&{JsI^gX$-_WGU@Km;mnEu7emMY#>|ZTl4cW
z$EdG_ckz5Ay%#mim&XY1rj6%u^|j44XXE)tgv_w#TdGot+`<X6kpaqP3kW3e%xnRH
z1fJOevi|8?#?pek^rp0HvBFs%_fcb(9$j-HW=~BJO@_zKcO_eUY=j7joFiZl#{fqL
z766WAN>w*fh+`QBSpF!vlXHwQm3ahUc@)XmOl2sfD|Jgx#zj>o7iNCklm$cPdK1cJ
zbt_Rz%lcWZuf&42au@esUh)K%V5w2k<_dAnJAgKg@89^_l7?I?;93=0YFC<RH;d*S
zF#!PsZSw*CPq%it?E62q?{Y|J9{$cws;YDTF5ek=N_=NdXc|j8Zk=mi7fe1BBdv}+
zrgRLfW`{XAReHWV>3)@M1Q?!q5Z_^P`vS;hfZ)xH!!q<Tj;lgX7a3)F+yjSMq@g1i
z+44J&`+|ayU2JL?wWRZC5-Z}9AqjX>jYf0^jxWplJ7%CAVP44e51fZP-G!NKf%9)a
z#uv$~LUlqA8p>>&2oF=!C`)wAdQd2@&&n>1sY`Zn+D>n<gA1sTN<?7OPb2eE^&U=}
ztAb@>`<Z8am%Gwli`njXrBGll92K{+c9x>7L{ogk!zJJ}a}1hd{e+ES=*rfTr>ZGL
zOBNgrGp4R*a-ID$h66JDW%<7PWqe!qEeKkaG&ROng`FEte^aH6vZik9`|7N=V0ahx
zd|oJs*+zgpLKZjzz*TH#IrH@>+HdWZ$2#~~iCfkNS3d?k&;zK??mdA?2CsmWGW(>t
zGk_OAj7f0bFVK)pdQ$kZraIrl4|%_tXKPAwssOCY2J^t-LN2?)3?|8V^T+L9!m1tn
znJ2QiVjjBIF~-c|E_#nTSYX=b+I>*)%x4?PwpC0(`xp%*!O->A@9Q@HRy^z)7!bUI
zWlUcU%$gZeHJ@tK?XAQ~Rh!Zw^SefuO^I$(223z*vCEAzDYnkCIlu^iVK5wF@r6xu
zMBqEl5e(mI&#)PxM(>gfr7d(0Kt#?vXHXOCr@jK}n9>FSjp?(iV_baei98eH#{Vt7
za{XY&f!0&CUHx7HjOTj2eyPfH5n?FY^%4LV|3j<zn=Na+Zwp>&zm)HO74m;$8Ne9Y
z<7rLq1qSS_;qk4<%Vl(F{&m)*iCk4Vz35x5VYV`0O>5%~J7ddW=`CPN>5b~BR3pRT
zU{5h^#MLaeuCt{=9J>yH&$_t|`!m^Mc_VAY!vN|%UCzzq8RupI`T3i9ZicQKN^kAN
z*9~a5+~wS?p!3{pI82u&$rnc7mvrDYFVzKG1Yk?<M|~kGv69TLDI5N#1)uwqb|($)
zZ2j5qV^OipJd<Tt`5)5YXdfF3GmK$_LoE)AH_6&nN~WoYODm$6v8=YYYn8mQGO}g+
zveQ0-580WNn6@5|%A5ned47RD^Vp!g2wJYX_2q~8y4Y!|#>iOGJOtz?aOGV+K10k~
z$Lqo#KxDvHxhvNPx@|}-dZdI-&$x%^FQDLye@<9Se#lKD{>Zorc}UTD9^Q~KO`R+n
zWYy`7uq&q~AANMeAQ!Rq_hSkG-?1sP^nK$7))c(?o8NB!vIPN4o<%&A0CTEB$H-dJ
zgrF|7-zR%P<*q@52*^a~Zyir)t99dIKu0uES5&5A{|5$Q&YOT{3chFt-je<|X2fKM
z-NuZV%(S45*_dJ@r?&fmmn*884Q1x_R(&*+Eug_<l5M@rAeC{P5i;?Y=rQf3-=(76
z-y)0B{U~qhP-`MWN*yi-w(oNBan)p-XuMtVwH8Th!<b*8$^1m7$o$@BWN{lAvy<i#
zWm8`PyjI89eqk2~;xu&q#`zcao!d-;8ApR#!HWMl)#cN^n!pvd^|Tlhi_=hQrleD|
zhpV?`!_&}3Ad%=Zce8yh{&*PNGBEBQ_7zXlh4rJ`Ci7NX%{+9e0;6UChIMkw{M*Sb
zgD_05I=E%5O+pnn!wgu=A7X*hq$gCKR$*8JM4MSSBC|ZAfyyN2@~i{49U!y57G{aB
z(7sHR`yud`bYsf7)4`Mj$mH{rQ;8dTe$x0JWElir?)fyXY$dUaGACeewiS9)32p+;
z{3XOcsB3nPnN}=Wx#=Ys5q8Bk4T|UD;V{-1pZKPw1EAgK<9L-N;s{-=zl%sV-RNbA
zJ@Xfw(cMPJ=N!=fY$K_5c^ZQD2k&Ou%$=!?b#m3T>-i&5;+#ST><_&-mf}Z0Bzq@g
z9uw8n=i+?gMo&CF=gJ8`-kX=@i;<?B4g&frpb>M5{pYepI&g(%hrYA;>W>cFsT8>I
zXQEB|Tm(4mw<5|k-HK72Wm*ScP6Q@v=$DnH+lO|8!mbw}u7dk9<!_xBI}0mh#*&8*
zFJJSNwX~!Knt&=H+h^h4*h(~MA&bAqMB^Log3(@f(i*A@EX_Qvt;Cy>%x~+UAZ-e}
z*_JtLGY+!#f2WsEur2pPLVksY^A*|4gZAmR;MASfDJduj>}w2j%9PcrsVQ%)a>_Ue
zYvzQfm)uxWOyRQTn2!;rfF|kdF;H_~N6f?uW7^j$u%89({FVm3NIsh&>1C)B$jL5r
z*bZl#HRUlmy!T4_WdOs_nL!(?@yf|LBzPF373j)j#_`tN_6UPdgB{HjFeYDMMKeVJ
zO~%@^>PI)h9jfO8hC#X66hC`6irzA21YpU@hBxQ3X&QJC2!uS{md%qo&NbOBn}N@0
z!M!faW`aWJ%a2dd%4w8?k!L)weWizsdFejnGZE6Q_k=g1dB77p8=)?j0KGZ0hyzre
z<t})NUXUTtIS=SU3Dnj>2?Qi_W>8vm5dX10ru}+KAf`RQXr>-4y##$};n{^6t-Pkk
zg|;!n87uNHYzTZ;H?Sdv8M7IN3>3E!Hbm3NGrPv~uvJTSn}x+~{5F!Uc^0&Wm;*JR
z(_`X9L*Deh3G70ai9mEbH_9I3n=|>mtH*Ha9IyjNUR2kdj<VgSez@hs$I|MQ7bvb7
zO2Qb;pp_F_T$n^Oj(Jy)`~tQ(N7<^#ey)3!Eu8Cd1$EN-MlC1O6$NA>+ou6MCbk1~
z&VnNdL(X2hm*cld0DCyTqWd930o+N8G&E#K@wF94)QW*Ii#<GSYrIPULqoTg1tOyk
z<S7d}Q#|50FP@GM(UE>pKYOR{^k(0gBd80Gc6yyk;<ebAraGW&3}cuSFh)JPMMwZh
z7TLbdWA{zOhBJK4uYd{$Yf|lN;ah`y%_QVEbxSR?+9RQa7R=)X`Z#98#>Ij{6DQC-
zh1;0KNk0G99*%JV-+BMcKsVz3vz4E2lb?moUX*dd`J?mNZ3Xo85bCT?S}p7{LGGHk
zwE<3%UZlBw9yhcO_F`~>_62YFOd(LdQ_<44lV=v#&N_K!LEENuA!yqG+gY!rt(Biv
zlbUCA3!>OsgwoFId}_0WU9<2C>#TNjpg&^&KyMT<kDH?jSs8{m40b>U6&;gg2Qe1D
zU4|LM%!kM!#xPdgC!iv`?=@=LPOpa*iDO?FG-8G08~0Mxg#`v}%}ugu`%9n%mbJG~
zLd}79OylpncX7i8N4gb=T{~|bSkM9E%yV^t8_uTCJa(1^UrpQkt$5l|az<utPKahB
zvFbXa4@%N;;0=j!oMpJNU*bkwv570Mz#~GSzx_2#sgLAqyPChi9X_?onOxLQEy9SN
zDqq0#ObFY?K-*^@b|LtO{0d#$>Vk4;*bEe9MQ95mlyT;XLGUdDgn00z{j2sL+srdg
zd`TikJzBT=cdMwslC}98_;e9;d(=#$dRh{?c(Q*aJqtyH&-p+_8|?%^tM~bK7dl-8
z+#deo%^&z%tEqfAz}&#t7FYz+)~Z`uQy)mrz@`*f$F$T-!fHCE2zVmd+LM^{hVUIA
zq3pI^BR11N?-13oUIWhR;-2P?OtGu1vOyuBA~9vRq1~Ld1?4#%M?x%!v~dVntSpV^
z1F(2z;WZpDq<wjjJ}Hn=vZHK9j6n}rQZ}P!IsKc_ZK>8u2xqcp7BVP2*K5J0P&Lg2
z;ZgE<qtfrQjRUGa4aAhts0$q}B%DPII$HKCcE?wbly<VUh&=}qMm}PT5D$`Pu7?Y%
zrfE4f4#|gLL_p2w1x^FD?VH2i1DA2a8PvGA^GoY;_}J7zg9?1doiwO`=K6v~E}X@b
zvgHlMJGNm3n@k!!S3pJmHb46c_)cc3M^s+BDo@NbEw<&Fzd|KZt`$a)`vrPY_}=D0
zls1dFC<-rgF9zCBL%a9T8&v0?i<#ChK`kCeZS^rEa*pw@a9evUG`9>*Z|rN;A4v0|
zsMZn{OKGTv1N#=`)^VXm?B`McDdZEaTj5<I<8Z<w`a@v0XxV-Oibh&tT4Hq+S$Ybx
z=Xn-Frp`_*yvsH~4}bnf2q&De_eR0yV;~uj5+46Y*2t^eN)H5rjrOX1U*t90Qwr#9
z4Mxyaw_NcuR)@<cHiOSiK2?YrmD|DCi1R*_Q$#n16t7*8wics}8!9YxG~rox9+rlF
zfF=N!1Z+DU5%<>w_#w`A+E=XDz9)k)g4oJykVoj`A{MkXrY)^L!Zyu`yfT(iaJo|5
zRp?}xmtNSvaBB<v4G8=I5tf4MGso7U17%7Vv@n=5$cRgDC9TjnNO~N0m;AyAb=ooL
zre)#fszbV{9ZYn-SAxi$K<xmRXU{H8M&82bbmq{HRi5d_6~;<=2JrP;hk@fmg3(d0
z4Tz`}fzX^X1}Yj46DeD*u5udPIUP-$70OWXx?&PKzboXqLreoUu&OHv5_-cue`6AQ
zII6(}$X%#`_I6MML5IoYavg!lwV7P55}jPGEU1C-&gKd5$nmvNBGT|5Z~LH8Hl}?M
zKvC#PyFs9)=P;lLkt5)s@sVS(@s>`i;2so59qKthjw%?<HAh6mTY_M3D$O}x7e%eo
z=;%h7ESsdlk_JUBV5enc2^<b${hVUsaeZ3y<H?%vi#X=bY<qV59JMvcLPrA(x$bA!
z^f}!q_KKQ>a}Sl49-q|_1zl9Lz-OhL|BbpiY|G=m6-{v00-aWSwYs$vbS)#%YX}%L
z<x?9od1bOdObztAdq|pebMn)%gT=fF2Fig)(w@*DPp4dtFvac>rRQ)t*JxT~UfX4o
zh4iEzd9Wva-nj*Y0tlYPBB_ZEtU1Ts3aB(70<dp|&#u`*o-Qi^!lDbXYq(^CS9m^w
z?y;vpR+qeict0!^4ikh~sda`uRDZtzeQDp)MF_(%y?pZfj~1wy_wTTx6{x4fp>E`@
zTvJdU*Y_voweNK7E72010tNm|lQjQYxu0)mFCzBj`M0jq&SQufH)?}TJ@-<aw#R70
zjoJCAtX!QFqTO}A5NT#taw!4j-KZ5#7$<R`srK_D?@L)Nl}~A4Dwg@>lS@9zTDIJb
zrAA8+MvlJIV~pTZ`cR<l>}^GNiY$I*J8Z=aiOLA9#L`#m`%Ran2MN(4*d~K$5mSi;
z7)LskSkPtZM6=PgRGBC~o5ZTNB=jIaE=eg51nfdNuV!22*j{nBJ8$93E1z*1bh%j;
zMmz{h)Ne0toF9PtJ+Xx>1q2y~Em}&@l0w;{A-19V_Z9l|y4{1ScJ*?1MDqjfOYn1k
zdEhUdrS%*)@KPsz6j1`%R?&sS7>RuK5@z@@9L6}GhsRsteEuT84Dp$l&NOd`^{a;S
z-j3X{7E?#2_v9;dUSNc_`^EFG+P&aR_2Wkn{QQo-D-tj$J9*03RY&D#V<T=gBvbD<
z>!t2CY@g0mRN%;P^uAc+rYN^#7txT=fPQzth;eaS_)_S#r)A6YY~dUa8{{I3t#ly#
z-h=QPlXV7KBt|mqi$ZRP0u6UlE?%xx1IY2qG{v3<t325t=|;pSy;GSVJmgc9$aRxY
z1!$wX(_nS_hnHqKFX}x^fSays-M+X_-KFn~A!UepVGJo({xJ_)LcTBltghx|C)KNq
z!$R_iNoZa_7C2~@Pr(+<4%j*n7BKW7u?6vYw>%6y?JVo4R1YikMcpg&SgGHRz9>aL
zr=)lgRfJo9^xs!eMyD>Lz@dOZ9+Ab(^=Ze$AE(#re`XWd!L`m%?SzkM;i&LoEtWMy
z6lJB_)Xa5u!`F0OPtSpOH*KE_QLF>G65@tc-55(Gjq}^0F+(K4SFwJKmacagCu%-R
zQU$-c2h{39a;qcE(}=l#iMjl-f7Sf$OUs5k37gL&tT1(`wyNLGyW5cNO#`g`edx)7
zxd*ZJ5+N7&<fBn-1(e)G-ngQ2m06G=7DA98B;b_5_aFqSR*3tajX2~tZy6m7O$xZ%
zUt|^(2j+NK;LqyJ7#VszT#JR#lv_2#%#1)mwOhbN>?+TvpaeKXvs@Cp2k;2?sCU0q
z<?@>iIc8YoNcYRv#SR{x2Oz3zQk#9MdO0_m@(|>9D->uL{{hhcw|$Dga;A0jEmW|$
zpfVjzwre?xyK#mAt)A4jJT%~Wd{ZpApEsarEObCy1StUacYaJ*%krmVb6*JM2wD<Y
zsc<N#mR*i>6eiOz#wJzTM++`ejsW>F%xBq6_f;5C4X>qa<-uRlih$~G94LHi^LGMd
zHt1v+=(<H*Tnu40#rsh#<-MfIBZbN=5aI-S;qHpBM0ny>{vR$q1_#S@ROf`;(9dpO
z;qT|VPK(Q5hCd{w+z%1a)qzt1uwsXJA7IMXo4yV^vzErfbNEBLm9f2tGjrU$e(Q-O
zNI#*2&n+;3(xJbsx`dkxU&ET3_)pM+$7J(f@m<E!0xZPAbHY~vm$Lt;3icT2{A@;~
zH8aR_fu$;qSAl*je*y6l1vMq_M+=B4i}yoPHBvbYe@3Fg(S+P)UrioM5_H&ac2TPQ
zdW{#n-~7vPBeAiy&#rPN9>VBzrF#1YCpf%|1p%*|zKH{;E9@l{_|jY6;n3QJIS4dq
z844AkN!vxCVh$)&`~c=4^es`U;j+8waKr@c&m}OlrN<`~B~`|x05xz{-)0CXc(=ZU
zMS!LNw000&y$;xR=~Vk+as2_EDtuA$ARXf8@TtlMzeborQxL6nxG>q?To%Y04f;G7
zoX<Viw#kcVj6h7Alj4b~;`;pwgGR|2ZUp9IyZtrfs!e2^Bq~snkQkn_mD)EQRUDAN
z#}|POTN8l8fio4vAF{o~BthI^@vW9Mo-(wTh1)--gkURbqnjfn?JKjBDyvIhy30iL
z%|>-1_CLj9V+d3Wiyy0|RJjhA_$hH*mv^=1Qz1Tk+!b+A2qnCF4INKv`z>_tG;et7
zp~!wsDK(LT#O-PVL=FSdA$9HXj^|A^ZQ!>kRYhGzMH*mb?1k0>sa(H*KaHub1<Vbc
zeOEsYWIEAA?|C*b>A5!Aw^Z%KUT&L1ZS(MpdH1m>-K|*1irk+qvt1~*n14zDL{&K(
z<$PZO*RDIuabzEEI?~aIv9h>}hifubQJ1g1%7#n2g==3v={pHd<hm%u-H2JvilGPO
z%t88q%yy9pEpd{?v_%q=k#G@K&}NW=>D^*NOS2HP^fy95JNpc%<QxOZTMa~}b$9-|
zjFYl;b^Ao}JaloJlq%iY=10%YPD-Yq)7o!k*yk<%S`Cp?UdBfJb`eLVGHFubZWlZ2
zbfW3297dHH{WGsMxj`I{3YEHknDi?=(HSWXOs&>vklAk@F&TT0(4*Xy6d$E@r0Ye~
zmlDr-#v#>aPzR1fY{gSYW7Xzxk)=M7z&Eee700|E;h2Xd<T4c+7{IuH)F!mQ1H5mH
zzQqmveJM2*l&aINKfIhv7k7Gfe6VQ!ls>pX(HF)=E)|p*Tl$*hlzGs&`-0==ADq6+
zI8JW?S+LaVO%G8v*`5QztC|b)V!Itki}p=uJj!9+*kiNkh3D{mCoxmW;u}pdB^~w~
z4Y#(l+1vrMlAuI$X!x?;eU$u|@&KDrj)_3eKT_)VtJ*fxnEG@vAwS6z<o$7=n{kN3
z=!{9c$v5KRA1EQZhR2jS2)D<>xw(qKhDFK!)+uU;NMsR1Of$&4yt_4=t7v|^_$;wJ
z%eyMp_8H;gtt~EKKY86)Kxv4ArhIK{?4t{P6HP$fSozk-Y3o~}RvD{yx<_aJv$3$Z
zPO53OKL%5%k<y7MY+Y}?k4O!Yz)|e2WMKf(Uv~VD_uLS;zDKY=XDxl-j`r#cyL7hj
z(Suu-I=h4VbpL?ZQL_w_n=ES*Z^$gaR=w#y@)jTh`dIQ_^5CBVsr@ELnH4<8i=y#S
zwg7CqnQ>U1q5HvJb5;UaX5X%7aar1fWtO8|;8+vA6i+)K&Z<rc!d5gi79<JNcvECz
zns*5(Rd}&`lP#f0uX%wp@b`z}z2^5|%!PyW4#r$Zju~@FCVB~p4)hwJG2i}i6JbvD
zCm;b!hBk|yVy6&b#O`H=7TqN}t&oX%1H2ME8M>rY)*i-jf>|EwsANf*1|x_m^xs`T
zUZvj<=9GApA)A?)9gepDTLXlv-mE1ZsEAWq&h2HCcnjt1`@BQcm3EOUDFjl9`@({0
zl2*B1z?Gp(QdsoN^`H~ubQun6MgXgX2qb%^*v}Wpmx#R0xP+5qb%Doj!i*SbP3%t>
z9|+6svv=P@QfR4>8hfl&9zVy?kj0!8_dld*#E{o<WG)X{OMW)eGzMy!Tx(DJdQ*a6
z&cEb9f7N3drTEXGRlBeu=R@x0EgCTFF7AFug)Y%;I=9lc-c;7m!W^ZPWsTX5?MapT
zH3wUbKB8m)K2;06K=rW5WcnZrLiiK7qz6LqqYHh}+20H|PHlH1^!~zdQt^v{(GV3$
zO9?l^!oP2Q48}KRFYSourWqL_zl1Rv1Q@PV1!YLn+eYvz=c?9b9=y-)Rp9O&&)|as
zHV#B9-*wqJEN^OkOPNza>#_*tMJRNIU5RSk<riZE#@M!g3VzdXL1}9#aFVGixnE7~
zn>t9%xhtKzaZ021@m^9FPHB$XM-;`V&2^uwsgI-|;?L|1y0N0lT8GiD$muWWdzi}j
z%4}NwaR4X^2|<ZPr2?sz00F~;O9f*X5m~oGwPXe%`a4H?DiZw#z3lN^Y3mPq=V^d$
zK&N`?-y{&qQ9!b?dm|L40I~e#GY<&(&oNN00QgTIswI#npI0O8qDJ$1#7CB_-xSn5
z2f%*7kCWhbm0ilePhQTjWof4^@}0JT3Q4Ce3<2Zbwg5X7!<KJQU_rLTw86wh*rIeB
zc9>+tnuE0!QgHRe&WxMnaBK!#m-bjU5Kn^4xX(LFU1=IQ|M8E7^E!tvDNZgtXIHt4
zTD8d!*I`$wg}G!|Gq^`0n_Lgj<9F2-{;lyWE3!WrP@oj^u5u#Yo*YrT*rU*|c6<fW
z1wWC#6oyBDvc9Hnea$|%%KtfvWA~!q7^$r-u-O5f!?P=29_RYvr-5J8=NHD|(|jwm
zccx)-bE7Bb_)@~jGP50OW7--Dc*_wj0|T}-jKvv@1mc02%>eh3@_ADa;tMgG+^Xxu
zt3&`pf!(v%kfrPb@`#8v#Ec^ERu5<4csF-Rn)`vbHpJue<ZuoqN0&78g(O)_j!+wE
z_pYos*5czPU+gO3`E{e!o6ewl%QYG#40c8S>in6wWpOY(-@{XWc1s+0QcWB<{<5Kd
zQGY1yg%isLEd~-Lx@Dz+vEFNTzVfe{1Wx;Jgr0xLUJ@l1fnJHFe_Sc_lCQ7D%07_3
zNf@FBkh%AeRqRb=A$_t7$D-u9xVZ-`eaEYGa^mio2+8h`290+cTIdgJ8(&?hJiM#a
z*W66fdHJkkVk2IkemTa#@8xCdnVVK@STVumcICbxgJ7MBJDvZX<XW2d*A`M#`N^en
zXU5#3$ZgG&nm78<p2xbse;n@3^(uT`ZN`zOqdu$}r!(}CcVeRMsz0?hsjS(!?_*P*
zvxl{PV?hkHJz7>TDx-a#(G6E=O^=wWMvruV8?l*=R#y`g<)4jp8F?evAwcx>Trb@l
z&SRzd_KuDpD>ZkP(czQ-PT8}DD{9b&A<|+zT-27L3-?8z+%CPweKmH|6gO>w<OrU>
z=4k2N*QJc+Gh8G$I%t&ukL&(@v*h;td0|B8q%xD@MLea-&Y}q<EscB4<rHdus9lq=
zqLyKXVbLk>*aZuJ<C?EFaTQml=B=4~jyyPZjr-aR*-2bOGRjW=qqb$lllx<RiVY7(
z9UJ>JNOjjATh@xbQ8_Wp_=a<|xA(E#M_ij1@HPGR;Hs;l{FBX!X-&_j^TmnAJ~R(n
zBL7NTB>Cbv&3Rt>H%Y%GY8MLej;s64|LK(}dvy1#O^7X2lievM;n-AXb8(e;R$1Hy
zXRpJVtL|v)3oSLUoIx?zyMOS~la!m|ll8SnaQ{7iFnQ)mL!0>aR+;+C0x7piT3}J!
zuHtkjbssN76}G0FR6CDN8Dexe5=^->z^06ILVy1bg@{P2)WoH`ZJRbOuX4*2T5*1c
z)S5Mq2kkvG^J$@~!q%bOo0png)DVAnJ8sY8<<rI)FPeH(YI5A6cS3&K$ByDt@pG6{
z{<mY!-Jn4!7hjPdZGAa9NKAIl5cNM*9)5myl2oocRhDnIoYdUEb7E6SYc9r0D9WGo
zaDH$?bA-UfqvRE<;pb;+?LOjtbQ<4q2P0b0vn<lv*R>nDlK0#my+O?7{jT9RFWe8#
zdQ^0Ro9udXSk`nNr}YvK&(9E<zHVsV>D2-*=&KUF_o=~2FcZG2i?`vIvV96mH9q_l
zaz_7)JhmCWs%e~kRjXd}m?zA?%I$Ml?5~g7mxLzD6<*<vESUjc;UvD=-~Jf$@u?Ka
zrow4sLc-X~H;lygb4l;oalu5Qdie-f=YV*hF~ZuR0o+25xlE+C-g5VFJ~}uReN~L7
zcmYq*q{g4gYyVcF&MltpQy4B9ji2IU<(jBp$BJngp;k?Id7`6Es#(sIGd_L#$l&Lb
z*ZdP3eC5sv(!cs|=Ti1PRXh=U#BBM@9F@7hnf!6QtT6YC<*=LoXaq0camz*fPQa+q
zg)8`MH*6RdHE2<-vRKV9?InvmoloAp;Zr}|OM3)m=sM{vKCk^^SKdDxasT3$kj>mj
zwZv-NoJLp;R?&VjI!GdD&Jc|nKf$GZ{Bk?3qM!ZC_wkU%x`bDe71Othg-pJ-H1eOK
z5V1#t_@5MQKDEzWjEif<hR%=4OtrYY=yFAk+Ej9ld_v&`<N_h{k8$6NUyb}?f+XKk
zIlG12T>fd}VnZH`JE=88VJ-Ew)j${^4F-hwHXwHcuc7Sww=Ma+W`n8@8y1~)j-BI+
z{`p&m^6flx3cbi&Lot2Fwyh-Y9k*v`8f=?BHu0}3JTcy%M*TH|a=<KT^|WW3)h11T
zd|0f-&21F?Ti@T-E*$;zfQk&?IJuP@D^@|ea2$ETJV?CX7`uThu5i1ozSvU7(0*d9
z`KGtVigzH#cw?7XH}kE^#J9#MUHDZj4vhJfHg$V-`=aGTZ|BAF^Eqr#8Yl8+_N@_O
zp_3;}8a(9E#yssT?q**RQqZQg$8+xfHQnWe$TWND(n-hva!fDZb87j8n;{GTI8v)!
zCnPkf?zcaZ&pmrQ<)3u_sbaI4i4R+{ME-n7wwS^9o8q&_YQN9?Fl@(#Rl<*-KX&6j
z9!s_h)z;@WDQlkVurDb6#3a78wf7u0Kg&{e&Z+q;A@_8~Zy(1x$(3Erem0i$&+M6o
zYi95yLXx;29%y1)>wdN(SMbfMP@Q~sMy$@cyBZsD1R3#XhY@%GI8us3Ec(J_=i(51
z;3*@-&XFUb5Su|-bAbr4+z(D@k61AWlFdm-ck3W~FB+oOnZ3PBuQ|KNs!UN~WN$e=
z(F2gZjTWhn4w4h#@z&cn4`uI@p_5KX37|v~>2E@-F`XJH)3)~ZmQdcX6Ov+Fm;XcQ
zdd)a(W1+byhhGbQ7R2{iSwm!xhs%gZ+s^(eFj`~zpvDz}(}#-rs|?@p=DhQxZF>i=
z@I;wg6M{Zu?&jYfT-hea$lUY!;zS_Eh9u0tz=*N`2+fu&T)=0UB=;n8&;~Jh55BAm
zCL&E4yqBHN&RD#Xs{MjHBs^$l@%{p{u?zWB&urn}{nS$JmFkgcZ;j{g5|`%tZODby
zPXgvYDMI=By+Lw?le&TckKlAI<8}Si9a3r|aE6#=Y*f<u2<Jn(Kf;_@>VHmt$mS6Q
zE}kd*uZBmP?EuBfQ6Ze?j2U?@;l&=m(Ti}Nn{koIbI#;R0mHIp@Ho9*^6(<cbF1@C
z|3&1v-z6;#YBkPQIN3W&T{mU(01r<b!w)d@HYzt5IqtQJ>#!Od<khDS4|W5uemDB5
zrpg%X)y4Ce!F8(F-;Ch8D&!v&TtB<Ho`j#*M%g`RPVt_a0KvtK;Ch8m-%Mi-4zAja
z;QA@#R`4KZY#Pcn`_xIbEhi}`QNO5NTyVJ7k3X}zecxN-P0vU>FZ7cio=+Z+^8;z;
z>qR4ua{HynjyIb<%V6982BpsPGSc-$b)4OL`RF5m>`VV=oamc02|Ka%jv@;GJVIsc
zYzLx@)gJu}m$Cml#epNtWt6dfJ3`jQzw;z_?%qjlH{%;CXUoQfr<12*6Q<`q+`Vdq
z_KpA^w>z`=bp4+HCZ*^|eZ5&!`0?wuOU^cf|M%kVj9Hp@ZtJ#bU*gudGF^ALn%2a1
zTxl!#W|)gToqS=2v4VWTgYoe57q}rFbA@ONa7Wiu=4ff1+>k2CC+-li*#D{2Rf}oI
z6%sNu(;TOW6{tL#yvIpOV14F`O^%O`%pPa5Xq1T*Jc9H6+j`MfwVo39jOZ!rR&&?S
z?%x+kYA3NTco_?R9`)CBMt=BPZ5zrDZElX7d`V{088!Gjp+)1MQhEmAe(#<Qj2fKC
z52E5(<#89d|2v$q>M5fJFE&`CH`p9%@I%QTaSc9>d}}N&TuIWn@_rc_O`9@LRP@c6
z{&U=4`Zs6d97kCRIqpks*e`bVef)^;Z-aHk5-7YAwYPHDvx!F|)SjX@kQZ3IhX|#v
z+`-#4249~?G19%m+7FBsYW%0^pWPzCx3O0Hc-FJ|YV#c*i}D>#k{b%i0EN=2S?4=K
zsm#h4B9zV^qKiYR+L?_wl=7(_8S~co&T4V~?x8f{MvS5@%v+Nyasw}$h(6iaf2lM$
z<RvPVoTYaD3tq4odBFj{IKm5*P>LnIAo=+`h8O(9U38`Jf6`e~`z`clF=EkFIUBZx
z-cm6}Ec#tC`mOQjCnzB#+#tVqxTdHZY1LXOCgv!`#+{=l4?*rc&S~^N&Ue$}AsopW
zY}+xY^w|eB)rIgMb@#k*RCv1I|C+VItt8HRa$lGsFkyur^5u;TUw%IMH25-C!RN7H
z^=W5_s$Av_UzXE4sR+I-;o!0Ok7wAI*E4)MPsMTa9tSCbg|A-7GkjU0mj1||TH++Q
z=>|3=D{RR<Dvh5%lT1H%@g?c)tp1(4M)eQI2lXc-H#0sc6j+gS+O6k`KH|Tx<RT-m
z{_SXt0;>ni2N;2sm=B-|1I-5#AU`wa0~!}$J^&R4C-Z^Rp_2;l{hzpN0gEwh|MqWJ
zK2q#b4iRguvy^BYXKBeHVj~oVa_FRHMLN#fD3r276t%XLBsq1^X^=`tl+K6gq*Q8j
zoNAhx|NXr0ywfzz)C|_{^<Ce0UEg(W?>vY5^t*re?`hO!k{RyuAsbNAXZ&I15z66a
zD1@TPl<$=^mzc=TS<^>Fy3_rCs0=!p^xZ0ct|XA~-FgYB97^~Iq+<|5I{Hmrad@D=
z)D?#ZKA6bL9=$mD;i$6Pjb052_M}65u!U^TiBC*8xP#o4tG3MbPCr*GMqwPfT4}4t
z+>|5*ubAp$GB~yWN(eU)X)2-PMW?NE+Z!E>tVtAn9d@t|_KYzTHDmfY3Iq2M=qOBX
z$iDObPpi4(MMEf!(M;PjbUTt2ZxObQ4pGYafDuHh{p$MhS>LJ01BIE`X>1$we9G4^
zh<E`H-KrrEM_ua}$cjs`T6kpvvJMZ`u!NQq5nSJ@)+FXLz`7>&ZRpSRUm`>Q*=6rE
zFJ21Om(L298N;h3`i5&}j~VP@)IVIqOJh~%mTsR>sdv1pg1xhXyy(h&d?hISv9G7J
z)>Cc2hp+Z?ySBNH$8OgH*>|6g6+k)*r7R8vI&yzNP8%wUV}cdMYr%@bpu3{D>P;%b
z{sq&0Q;nfGI{ol6tAV}K_;->ie#KVSzpHags%ot_9y3kza6hZr8is|EH0v>^r}A$Y
zq*TpBEb&$L#$w5Qlhs%(@i;e0_J^P9cbR+qn6}Gz>g1;z2Ia3%`_}&)AYLdh^zA1v
zT<e;%=dFmmaLl3mZS)VyJ8k5vwK2DK>;3%OR^+h8+9c&=r_Ns++ee*IdaBj#$6I=e
z-<EG1?A@p~avjeDe^R0J+10AY!~9tDu4J|mwc}mV^dC@oA3f7tajQl^AviEMwQa6V
zV$FW#<3Fb6RG+$`e|0!QbOycLsKf&|R~6j6H^5(K6+QDe$+Pv3=MgRlHSUx#fH=!q
zq+z(=Ivy^#Z;-*xZGU07;AI{z7$1IM_9Hn8>HO`>S7ncVv*yRDef75+&#*MeY%=MJ
zDrPTt@8trvW>(SJwtvGvd#AaHO?gMk9^K>QKlo<$Fw9_{$|e&A6BIsUXlVzAg+Lab
zu9Nmm@oX|+A^awQ5EkNM0a!??Nq<@UX$(0Ro3*QInA@JQt*6sF=ezyzGv>tfvJsYG
ztY+xCM{Ho^oyjHr@fgbH`f``f=iIN)F2tV}-GT-{E61UbR&Je$=Z`mwN{1$PD$b9v
zqOD!^a-*zRfNI+4k54<7jQA}K4=hkNrwP>L*8Z}z>(`{fL@s&6@163Q_lr{Tf=RvK
zJi;w&3U($6PsUnN+bMul6?&f?^P&4({(2L+O#r>+oy$r6tJt{!R<9*^F3l$Gvxkk6
z;Cj{mvMPUt>bIV0&kXc_tl~p$9xeh<n=yjgri@wLe9Pr5Ky4PHptd~+FHX?lv?h?F
zu!Q*4MJyrSwwl#qThb*1h!6eez<qbR<K)U!xqGe&beg^!=)Wys>i?M?)<U@3nE5Wl
z1Wo^ycUiRhW(;)e8}GR>kl!-v8*8{|GBN$W+zN46!{FHbPDPfMqLCwMD;`*`-EOR1
zA3||rEu9LTn5q1LFM1*6rW4||CwhK27Xj6hR*M8`0?j`|kBu^Fc$u<_SEfK{=`K?M
za?r_8!61iOX}=)Ifhbc-gvyjLHuqcZ!%Ao@>zlZ_te_{iChcwRY?Cpml+Ew<#-eYS
zwXcjK6U$yI_B}&)Y)?0w+m6AxD7yz2)}7sp?PH3lQ_9X=lemv58@s)e+Q1kaaEPRf
zAIe>Xh(>D|Ca@jUj<_mz*IOb_#ph(OUE@!3=!U6{A!!~_yQ!xLfT#pT7g5D)3s&&}
zh+<VdSPj35mpwCS833ZMWsk((`H2UJ;#Iu0LC-4Q<aK%`Jpa10E7k1>p6M6PxqXmo
z)!bP|_IWcesGr3!wV|&npIin<v+?K2`=6~^)#GSzr2Q)tURYoD+;WX|bP9KAgYqkH
z-0*Li8x}jBk_{+Z!uQ70)M6Ee^4kRqP~#)O_T+WDW(y@xKuPzi*|=H|=p2qqpm>kt
z60(5K^~5Crom(tPMgKDoos*!VA7eAmrqP<qd0}KrT902aRmSxUmM`@_Bw{K)->ApW
z7rC0ZviNGEo2HL(n+zD&W!bA`$wX}Fzf=A%^fZ5i6HzpJ$Q|dS>_A$FFXnJwX(Tf-
z+>ZUR&}8ti5Pov}&$IXXvv^(RP4n_LLi50><hcqIUvJa$#u$2h{gMUmFnePq%lmzm
z7&uW8$z~aK8Yyi!hu^TPZB5y6<ZG~+Vzra5{jO$|n*)Z$s~vT1k>r*wea7pA^xbW?
zNc&}USFKdt=1m$iZO<svxD#&6g{;Q2{oCftVmi)45vCFF>zj|u`Pu#^6B}sj0Ayxa
zj>6|P=>K=dUoe^>%bbLz`^Zi33zx6fD-<Y=PsVPyE9yFiDoXN|NqB}F%Xp{a$P8H!
zq3rAjNb=9!p%Sr6R_yvE6FQ$}gT|r$ZH7U|^?$50FkBdO7X6bM^yP^LL*F=<f!c6_
zCxhc)-JLdL)M*=t^JETv@8+-qb-4lEmIa2%ZjEu&!`%0KD5T}M)fnbDR<93)z)!#W
zYi9g8)tH}<51W1ZM;CL|P0$-Mg>E#YD~Kyo9p3fZ@|d46)S`WiKanY4>uy;=^QOCR
zcowH-^y(V|HG}9I0;>0^eM8x}?d|Rm_<h`(Cu6B)m8A-;v%#>Uv&{Ad6t4o84D}9s
zU$Q|Vcp3KkV-8_3sA+O^8R7*b1Ju|I9e>~jgt=n8fJX04YR7l~8HDiepT^*ka-zX%
z*GIq}xacN9>uR&w1BF1~1qipU((V{IdyQih&=^hL@*!H7#k{N=8&TrPC{Ba)q~C>n
z%hbaXn8^oTi&==STGMwXs{i&vPTlc4k$Ogj&TY+6AzvAfS&J$oF2*^=BEIt@%jB2v
zS0=I>ush}c561if%2UD9^{di%X#!iL)ovzj^S2Fl*jh9XkO)u=>jL4B1*9Ur0q6cw
zndrWKJ`esD16lN+@Z-I8u?hc^gB|QFcA&dT3C;FnR8raqtkgGF-T|fY{RVdFCbXKt
z{j6p{gA!&eqsMv@(egj(j;6&JEVDgBbKs8e>PiabEhPk;GTRa=G1{T;b;HJu`BQf@
zI*)&*VYCP6X#*)sAY6{3u5&@nUMv{>r%guN7xm>144MnKrtIZO+$TpHR@k!^!djHc
zr*3ASK!iVJ@S8>=GRhFZI)FEDV~oWOpl~d+dgn8Qc6}vQhpkMK5j>cGp+r7+Og&@K
zU~v%4jE0L_n^hQ$vEE6ic*jzm3qVp=N_IW>r-A0G-awT*@#v2rvHti?wBmAXDXWqU
z-EcArhKzmf3OT6!pxw?TMYK0v^77RyF(>A6-Ke6oYB?B8K)|m#d!_(W3mt{Q$d#f4
z<)2n6r?DI6S#=f9$)@mNWFSh-=+pN9Nju=HIQi`{9_vpW8zQTD9ma-Or*>9=_2(hX
z`b#eUy(1rNpSy>e+x|Yuly@>JiGaH@te`Fbat-h|)xHh60W6#cD>q@}ly8BYa#Zk=
zole+`pUmoJ?a!+#CyV08N>NwF@3qZsk7H^0h9#}xt)(L;CbmNpRV+ewkLXVECfynT
zx)UY$u=K?v$ZkrY+2OnW@V0nOqYR8F)Grs)It|*0SrT$FOPY=rwVN~PG;XE!2E5k4
z^w`j$UQs>bAUDP%;=~5m(vhRUaYao#&8O0-2qrIoMK8gli|K^4|CAhiEP978bvKu@
zuflpQZzaWZB1)Re&TO?x+_5Teg^;meNRp3-?_<{?F5lEO>p6!$GBCta<h$x#M9}|F
zipo((Uj%l9wAr5VT6|>$9`g5fckR!qM!kME{0d~nJxE+V?IdnH#trniZYN2y|L=7r
zzbjF*FM_UQL<6*ntCcsL#0s&|GY`7tmaw8y!sKad08E@Df^a%Tuz2~f?*!(>E>)e9
zpt9sO@S*9l-{T)%(}f3F^4NiYp_afSPYklm>t+XL0c1(A0}*8DTKN3+|5*S#a0;*k
zN00tEYlPj4TOCKigUxo!nuKM@DbMp?7{zoLX+#|4T=BTSWj(SNEZLukl6^#ae7a6B
z8+)7nJ%;Y9rw^5YhAmCyLhF>UgAZ-6NB1{Z&c1I|Opbj&;gu`zPyp<AOwuv}XS^M_
zPNzjs{Ck+o-+uG4Obq4II8U%ISnR<Bf+5v61`}m6QP#ONsXq*c$B;^S^v8a{p%-|K
zqweEYy$G--jloN>TfXy48V{HKT8<J*anw_kx14@B>KJyn{Pk?V+A`j{%BW%Fkm{f0
zn8y)Q7rS6*e_COs(l8Yo;N&~pS2|tM^3^Nwu)ifPXR(#DT}5_b#KkvJ_mKUIBXJ@e
zX4LO9hhVo1Wz@XH(u`k-LFw9Eh{>J;g0`qaY#P511A;ceAq0YUH;3@=ZVsW?LM*S)
zy>lFtl-#(a%fs<}zwB1#>MJ5H7k_>P)x;x!UynL^tKQbGg-DMG!NT3iufwJFNJzL8
zmty-tnYLufmBF|-Rw#+JqGqU=14m?@8ql6mycb6|M{vde9bAbZzDu&9*?;l;*C+8I
zkFG@oBnilb;+n=I92_>;o{1>N>f-9<_}w}Yr{lnid+z=;t-o7kHFF?#tF*%p61PgI
zyqByhMs&8d5-@|+#t?KiA!hK0`s6Nec14PE0XMQuW!aTMymR3g+P98dcGJ#@wMDIV
z-%1g_c)e@-*#IKoTN`Z0au*k~!$0ycvHzwrcCk_^S7s>hoFzY%cPPn`D<)G=IxKDg
zARxMF-O9q}80UP<mM|iiU~O-SdtWE7VubV0Zm|xcXU3U3Hgg(ZBqS~5nK`AQT;76w
z55U%$d!Z5CmyKj??+E(Saxb!xyfvG{=LlY|uTQvjT*si6kYn4uSqw|=>7?nzdKl$Q
z3vyF=XUfmtPZg1{bWh044A^`td<d3GQX=y{tr#Bvj8FXR+fsn|>3?{*I7<P_KsbZO
zUF@7&_ARfj@BGkIyOj0-x?r?(F#6S3eB7}Ao%fK)EqLyAM`0eGu&i~(=f?W)uX?{x
zS6TJB{Z(Ql5vC7p8fUXK{9KYc<nc(-TKFzIcfVJkj_c~H=Tk?ShT%z<$!xtoI;#S-
z$+bY6BszFskcYduW%h-DSms_JhNX_*K*jT}FvcRImIki_KxPfu`)xybU%?$)&4Dm%
zn9t<E>{YNDfqo>gip7#ZiPe0`eq?)nd}_LK9)12YhILaCwwg;`hnI1SmEXjy9dil+
zEn*p<S1;(yc%|@vSOy6M0TVF$Zgz2BUA%vTT?}9dV0kfAo5nG2hm`Te-qG@7fdf4U
zpw_=&a;DGuO0<@j3PZ59l)_s085d$o_qCRPp(8oMc`?HXI^IT!RPH)uDyelhj(%4=
zvmt}r4N$(%_imaCMUTdTAO6Q;(j+Wbfx$Um;?-jk>nrq!abm6q?dttPaKpuJKk!Z@
zFT<fe(&)8gYq3!#3=m9*QK46&XB5E$pPj9a_q!su5mPk(Nmun;V|2AcQ9G@kZs93c
zy%bNm?(WZxiM5M_4w08lAN2|Hzo+k>IEmFK@d^X~(}MBWuKEt=HV%!Y3#b>Z*17;U
z_|dxNhID7=irTi+4g9v9sG;y%j=xQc+K46fDY+}U$&>#@4^s9~LS*V0<pTQr#8T@_
z-a-~fSGA@y4(urAglA1Faub_*h1_pivodeRDhwjtHT1Fsh{V4|&xI+o(vshr3jr%i
zE4$)`XTcYbq=yyKi(+Eg&gIt64vL%zM;_~u!XnIL4K}zTiO1r1w*}_>&hzK|fYw6H
z`AN;nMI(!+A0=F<3Mhoe&~7)~3AhK&bH8nOxOr{G<NEOwQhdzp)7_8?KaIE1TNc``
zFN;Hv7L(5L`Rjbbq(sA`7-1409_@?KMhuU>7CAf`^rLmAA4_8#kd4Z4uj}z{d$ZYM
zvmM2bxuyeTkIob=mjISg7P$@UoTlWhAOO2hszfkau>@0#*Unvw935G9!b5m!(Y18L
z(u48l-1c`q^3{uRZ>8$K(~urDcB=-RG8Th^MNL(s&|$8MhmLzEh|nQiv-eHK`%>k|
z@zM?KkDNEj*_;Fy`NdZn=HmBh7PBU|P{({nv9q`5pWo<+o~s7q{w{ZqCT84who1PH
zaviB~^i3(`&GCw(6nZ=!A7|cW#I#zV<>oyeU&OXez>h5AIGhRd^UH$Xo5(p{l=Bi(
z9hYW*-Wcy^Q&tDrexSy44@J~7#wt6Ja(R|9D#53O(%g7PCYq}_{(mkyVol>(<TgBL
z<lp>Rut=A9i1DuX7a505n^-i@mud&#@2#ZEa(Dr9>t#Yqt%l>~ipS|&-)|f3D0;vh
zMe$q79#|BAtvBUAWvLE%;`A(8hoKC!bes;xgbg-$4Pw0ILoThkN^t>d`*^xOb`$Gt
zAL)O<oGk!YfAO3x#!g={#(AA4M)Bwe16p*JC;iahbn==(nM>*>iknUjomRx9`#~X9
zt+e3~ZxNGjHI}y8)*c;QYbiGB-PV)A@<K>z@w{*=PnV{DCbzGLpn`}8p&$YQhENbe
zlL7($Mrzok0JNzE6hz{?6-3UUNddJX%;kt(uT1M|Dx1X%-3pn$xf)?9F7;k@!1UQK
zb`iwv#@(oU051rL>%bxiaUB@Z?-thqV%onVuCpm-YQtD^Jk^F0NoABK$xU6zOK^wh
z#*}c{++{S#f4l6t;U)Gu@h3M5pa1lqwnC4W`9mwTiajp<P1#7|I6BAXY6RrR$9B-_
z?u;&v7VB#_hv8Fz2MjJtl1g=%pfTBhqxf@mc-t-ma69!VmmO~8OiJQ;+vIyGtx5Zu
zYugNLcF8sth}7XgZYnez9|e36K1+gT2n_Cj`T0_;JN2tl1Es<Z(%P!BE>|A7TQTQ$
zH1}Y0ZB^aj1G0@`UG0$2>`@p>4Vpdw=^8*Eh5V3O)Sk)qQ=vX7Ya!OrjDysa!>qlF
zNUlf4rXefe*5WavL{I%ct^8mi^t*tKC@XI`$n&(3`Q{PqKU1>P6?g6^?yAW8IQofr
z>)05!7%MdXF!)On>);m}UxY@>8_q(aO^b#kvGX7;R4#%3hp~M10-mp0bQ{jX4iYNm
zg~QHH+95U*X;hcTL*K!bMCki^sz-z{2dEy^<&$<a8?;$1sCJt-j+Ye{=N`SorZ2dC
zuw==wza}jayM{7#6}(nDxRMwy`@A(Ia{mYVJZM(#7o?#@%*q|;o|QuyS~rHt;~HA8
z)+jY#|AP8`;A&5-0a<<JeDXoZHypN#)7!GAJR_;vt0;ePSs*2nZFLxgHS)wdtYAA#
zItZpAiR5r2Qn-*}921xUpFfTnl)`3Shq*s3#Q+K^oac1yieAW<utVWuSuGjnea&9%
zp*&W%!Vbmd;6=hfKB=$vxwP4$`Oend_vL2X+yYv=wH^-_X^zJ;^|1P_bui|BS8Nnq
z*Nh`osPiVeWh+tn_c;TlAuD!03eAOrsnG4vSll#5vq}dUee^ea3y@(ToCVcx>`cvv
zV)4%vriNI#;aWV4zzx4TzjV>JgO&s~I<bKHn@6k8<rui>5{m)-6vY3ltzrbMda=<{
zY+kk6*T_f*rpZ*t*uMCz@!rqs4Z<`=DW$>)DnFz95I}0&ejwK2z%fjWy*mH)K~AG-
zylk@AjKnj@uC-P97pxqI=_$c`B{3ay&(zPHht>Jfa7^gL(H|$-rpK`ZUt}!3@`?v3
zZ0p=<NLE^KJBl8vx$lnH5IBBKbFB*{Mt)Fv;>BN-!W@w@lOVECoz^cM6Q3UMmyQV{
zjC?xgb{SFPRJzOOmb!3~6C#Re+@{>|ulDgm`iJCPN9*S4in-B7Aujvw+!tr&Xf39^
z@vD}LJpdBPmXX@L+-kr8Zp)fqmv4rF_%E=etW9%E)0mJiTB&Qf9U8;4`xntEg~}Y2
zaa%R0l_g}dr|;twaoqx&pVGrBNJ@}4J~shfM!aP!U16xYqQ91He_B0HW%d}VY=9|U
zt#su?rVNu_$Qk9PORVO;0YL1{0K##TOiN|+Ek%j~0(hmB8NcXE(o&oz>*z2-T!#?!
zw?4JsDcQv0_(3jK<U@=)IFv%SABbhl+y{YO5!^6i(J&Zz>uVXAl$fblSOjgG|BPB{
zqa^EJJfF*AF;c7+Ur8a%zA-Mo#2FMP+w`<D&Wde9!L^z>-jy{R%YMPazx?#5A5TuN
zpXt$2GA{e6ERI9t6s+_7Y7fumZ^cJ~u-glj+g2QRNai#AQ!+j}ic3>6|5A^z8QHKM
zgHIZx)rnN)t8-*!o|I-~6tg<ntZ`S97RusKoW~Z_zPSk!M7BCGwIepZPc}8InYaU|
z19a6BY7sn2!`G9-&AK)>CaZlW90%2z<iblB@6OGEB;^*I1Z#uw0a8>SE_iKf=}7Ea
z4Jc|R@{}*o-;{1PQTD`RpBKsxxf9sROfMtO1-BnY^L+16b4B?mpEF^FR++oipMkuj
zw+JQKnE{xA{OK?gyHX=!q6rq(*HbYiq@ZTWdUn;ZcQ)^2(!%WBvJ{DR*j}|Qa&!bk
z*Qxu(cYmLz!Jo6TKK);{1hXE`ZExHO{aG$6DQzgvFp}>+HjhZCFMpiQfZj)!OV&nT
ztv(mRAF<kT?%!Mpp`eMBqM;Xc3|ER?cq(b>wLd{}bKj9gqX#KFx0EQCP_63d3$Mfx
z&ezf0&{7LRNWhX|!FHmBOrZe+dCJ;UdM3t$mwmF7NP@r2WToJoXg`XqA%VNCuQi6_
zP4lWJJkLvsK-v({-kdOACoE1}8eg;8ck80AiwYQ?HL^jIaJfR&;a3={9P8Q{*VP6?
zyU|Lo42bBl%%SFSeX2Sn@lO-D9i+eY`Zkc3`juFd)3wZ>z#cwH*ygkPwAQ1|8X8Ip
zne?oVlS!t8jbtB6mno!Y!vK$8h-zf4Sa1=_k?TO##Ad3VmJMHuX@UxjxAxQf$V<b<
zTR(^YI$mDWawq2*hf?A70dy^FNTx*a=!~Ewu+N;S^Rvo^#*A3V1TDeXYCAd!50cV9
zKYtPv`LkM^1NYpx9#=(<qjK`utT|BJD)78*b0iOG@Kd2N=cNL9vd{eRTJ$ZHAFfi8
zdxMk1zg|U{{$ca4vWHv!*?#HV^s?h8aELm^F`}citr;o~vdvh1_cwi%8#Bk-y})Ka
z;TBE#`pU9ePd94VrZXFqcRDw47Lyz4%dSN5-0NtEdnws<-jIYu2f$EapZ-;*qzMhO
zji!)()sl{R{JE@9kh%_8vFC;l@DF3C91fcv)0LN$KZF<Z?p7Mwv+8KUkPRALb<N!)
zwo8r$iQ-w9Tf#;?V+q~ye#Ma9n5d__kN9LUBx15?+3;^1C^LR`UDP4NimURWb0>7Z
zO?t>*$RS`?-I_C@sLOKvo}qnTs3Z4r74@y|#or(99`hs=PGQ8mTj3N&y!i^Jt8lgA
zPm&H2R5+DM-Y8GDWwe4#yeZQrOd`+yL;2%6s4m)Q+0cg<+wFk5HFJ~3I+O`!9gEDH
z{a?HnVgri4AIGz_e7N~#ONa$*%h{P2+ffOvOsbw+{t8i73F+zal3APk7%y!6HuwDh
zL}peic{q(0Ne%2++{PFmUvKS8BpkCChMn;d9U9=AA`Br7q-_{zxurMkhUuNfU(i`}
zuIhp(Z!~JrJ%zCU;<=DFyH%flpftZ>H`{X)UwAFO5=6MD`z6WQ=^+}o4?4eBhpMMn
z`g9uTJxC}d2vvK;V>FuPs`~vrRMF_YT<y3q2vU6T00HLysM3-Y-%n-!*_D6e(da#`
z8f~^d)Lj6tZ&lu~iI+injC%MYtWd|ou(&HUr8Qlr1|*+)SFS?_Ex8M$YThVvPd>O3
z8y)rxPu|z9s`b^Rn?Bf>$liTu&UwvlmS@nZ9cgYqjN}Dx<Gr^VcY$C)SUv0Tqwv+e
zD-2ApEgD0pGQWxb2SeFqAi(r9{rqKLt>D#rn-9oR>>N8LBZwV_AlrwzW&J>`V@rKg
zZLy-_*64`3vj!tz@Y&;b@(O`BL<Enc!g4QGV0hk677#fTc^o+|gsw*4Q>p09iU{Yt
zex_2fH!J+VeWLaoh&ETIHn4p_7|c*<!v^AJow#GfedTAAchs8o3Sv2Z;jq~4NQgpE
zP^%+Uz}zo;Mqj`tAs59f^!GZ@7amI$5Y(};mZh|(L7kEDflhy~v|&AeJuzS@`C#_c
z@QQ<WNQPpk`w4+JQ84k$5-9sX8o;gaIUWj)hC(P5Blay`ptZXrh~7CO-hUVj?;H{D
zKY(_kQ1#k_NtyLV^?1YWj*JE}Eh2&2<>vMS|8iYpysNWiKL=y?fo3fy%@kf_0wIb(
z^Sc|dSz(OW%6DHd+fxIa5|rfz544zMfVnTx%+@}oKtHEq3WeT&TYFOo(`#%YoXb4<
zkSRP`Xv!Xg=EwC`04i|ij|Zq3R47)7kZ1&xm&sg`#ea1tZ$%`S{8y&8F8!bD-k@^h
z!G$}UGhon;6P&c>e?+o)$rgzC;{aMWIyBa4ojC~Ubc>tq=8KzQH!HQ^Cf18W$r#+M
zuU<Qaz!I`5s&wdNdIHVdF{kYOabD@MZD(740NbBhQU18C9>5XhCBq&q%}`h=Ako-8
zQLnS;Y}I8s-elLJ6a`-h?^ymWtzhn_KX1W2X1#5}j1{iK*;%ZNx}A`8y-4~W|43qQ
zW3_Bpy+#c@BO+!MMp%;aV9P1QtXu@IZG8Pzz#|=Q&xW5%O$WkN*C9Z2+pQN78Q06C
zK@kZ|%k;2ZS`ZW>_t679`8_<afFuY?e~(4o#Dbz${t6@sf?3GdW4X$I`}E!>Rkh=l
zjACjEwUfhK&9GMEuccz_)<W(LSjPCLWov)xc2@$@1{j#Tf9N*|tg->tG4XIvU$0#d
z`lZ;bF9Q0deLtCk^h@`PelkTSZTO$5=`>^<`J6ML1u}!b1lF90DCI(ccqTKQ|0L~r
zELtcJ-6=WZ5uR6ik;I*nWTy6u%sXCZ{jz%2xKUFy2OoigP&tmvZVYw%VFZz#d^nmO
z=C>7OKRM=@P%L^+tCz48Dr?~u;X&{~Sf*e-C{Fh0i`0XHQnGGo8?UFPGm1gK=$u2q
z@1$Woph`8R#9n1-4EvJ1Od+`_rht>T1{#`a=-I+v;$_&KG8u{qV%r3TukF@e?Vfe)
z*~vtN#4kDP2nqZil?}<T7L7KqIW%irW#WJQwPZ4Q<*kbU@mC+qdAdbH>Wi48;tg^w
z0y>Fg)zr}(^rlw-?-s6eMXclg-Yq;M2cq!Qx)M;u!-i~z6E|vB;aEm{tk*%q1kM!5
z-np!w>|JfduS4$c2@L*R1cjGm>-e&dNWS???R=dPLwSDUVc_Clg|wHGne@NGykM@n
z?>?R<wi%cV7-qE|5G)b;k*F->z0DJBY`GWDtA>buzwpbKg}=^QJbsquLQWZb|Hs2j
z!-ZFzh|<<L`NQ?Cw6efCx1*?cfdBT!d-&K5eHh2Cq1f{ZHh03cyN%=Py#^B!xOZBK
zd<n~&r4{}n>7fj940^A|ZjI1saJujDPnnDO%j6l6{0A1j3-m7bbD6$y&4E958<Mnl
z+S_+V#q{|l54njcQ^&trLKGZ#n{0u41cUO2n^!WOe}!QLtcQu@$xitR$VOp3ns|kw
zs2hvw_x`4s{~da(&$WqMGwzMXa3+Io`+;uTu};2vJdvkwYmIX}+?)o+J#H#A@%uQA
z+sVhV<A7JlzV310uNQ_)-gk#dE@8z2%j#WOkP9!j9X-^lo*vCfPOr<|3X5P~{WfdK
zU#Oux96e1cs1$`o@7u|)*dj{xf<A2#X~S`)6(l6hBxBu3cmPnM0#LAhX)<V?ZMt$1
zeJAG>DBrzEn$EWl1<ss^=ztvHknF86V{w<B3dz1IQeF+?Tv*Irg~6imiW>glE%6v2
z2nSuPf(Me(6Q9Gzd=F22ihT?ay`JKQ1_OWY`#D2VgMqrL^G3&Zg#lKw6{LxD7G1IC
zJFCX}cVW5I=H%=;>KfO#wX2c4?Io;+SqSfG9zf$p5O3*8-8cIze(Qljlj%`Zn}<c1
z_-M@_!c!?Q4ehO2s)}0+$$5~BgnR$3f0h4+a^pDxxlSyr%GrYhyoc^b5jncFLaLGu
zX^LX+Wo?>Qnc1Gms#d1c@-38K&F8zT&ieA&sKTQkJ6EM-1LIMY_!yQ=@!sRH0Wtai
zqJo_YZvs?OnnM@?^xqk_^|WaAOu7AI`JSqt**s1@D&P>q0h6~NUW@G^pNMKDO5`FO
zc#GY>9Z-=^>LY*p11ZXbz4r$Gxc?}{%a3fs?NVWoy~@he`R=MYI>oGc4Rm5vEO%IO
z6A^wW-Y|HxTv@v{hDu#A!E^7Q_f973mn%?fEJ|_*N$ywRbm?)_=65=zD<*skzeP#t
zcnRGbzYR~!RlDDV7IwOxH?y(#+-!<wM}X6nvKi>&En#0=jM`1H%>V7pzmYd4v-1th
zwmlnikz{9Xvooh`B>zl|Gj6k4IkXr%cIQn<s}jA@*n6)RaFKW#GIN>iT0Dk)yqHQ@
ziDc~7FT`klGw%v>CzHXs3YY?oN}}x|EF|wCZ{)Od-(;XQV_~qz3*AWb88dX<u*q0X
z0?)Lpd)xoRlSC;l{FDg46qoiX5x0zwInK451g8qlnW|Fj-yic`)vUOqt!)}RcUaO^
zrodD$-&gUc`az8o3mQRsTd~Loo&N5$%9a1b1A2h9`-zA10#8r+3cfMkB>2WS{<7Yj
zF%msJx#$m9T{1A&JL=&W76X%&{Lb2}dmZuP93OAx{!`_WOQumGo#<=84fTHD8mSCo
ztsAq$c<8Wy(O<71MF)HC&_5Erh}-YOjS@bH5(om|Z5(#-y8uw^ajqzf|C30sj8}pt
zRND`TDIJ8B9J^qSQM_%utwoP->|WH?_D}bzd_|d?)Yl)ElaFvcOCn*WAfTA_ENT09
zL^Mx%o|l{*To_Z}Yzgw!x>G?J9xI+eo)5IQ;TfI~-h&8ONgiT{qqzWH{};tmVBmog
zb`d+S;}xqq8^qISZPshMZ`aKu@Bn#!dRTpQ8YE5P&%-S%9uI|y*AcMZ7kBbs5qbqI
z86at91Kom?#<E?s<cbbIc+5JFt3ky(J7vcdB-(v%bN{E_)(#pTn6>*>;UT1b_L)!1
z+mqqN=pGLSp$8o#ltpd-B@ZIunnTr1QyT&z81w_2D7H~O=U!6&xBR=cVrTQ?^bmM}
zIq?CQ$zAUl1H<0tjzoJfnwV(`Mk`6Bzd{+x=S|s}pogsPOab}LFl9+!1-)4HoCyyE
za!Sx6C^wQqyVtR&Albz)<G_-mPFaxg7Oi<ddngih7wD$JLpw<324z~HhK%>0tTXQS
zbG^Q7tM&-~Xo$_+@CN735~ElqtIew*z?tO5kA>Sl!~vWeenzH{y9)fzIq@9hwaqwY
z^?yvg{#=$6^|^HOj#-TlVmr4oIZX*FiBzlbq&3|KQXAEvd=vylf$IY}yT*oct7?7;
z!uMK>K(q(*T0wSr9QjihUkmZT#%JidihCPgl&<TdfAloZFcPVWN4SZdZu15cIHHGL
zUNT6GwI*k`$uEeACZo5p#hVAbSB*70QcKUFReXI$YmnYdD_~F>*;~bfbb+N?vrP9w
zir!^c<<6f_<Y)@zdEiu}|0e!!X(3V66r14<+ll;rNdSlyUYFuo!0BUPwoL*-)EQ&+
zyB{_k5Bg_20Y}^%<^lrle%xlx?Z(RhhdQ^HRUYGq_75&`NAD!W$KX(d_tUb{EO<-D
z!nF%MrbXEJZgt1uSlb3T#v)GRE3jJi-1F&$FbAB>x&3D06%s#!cr2x!v9(A!Q&%4g
zK%9zAudD<qB4UO4n~x*0C5&!3MHC=kubT;dK$)C9&4-xWi_8+5J2{qmUlqM_vfC|d
z5dT`;RV++)DWt!3yu%&}B#VLkhs`=dLCA+6Hx7_6^DejceD;-FfIzvAQgElnQs1_>
zv8ov~s(g6T^nv{Fw~gb}wx3o_KM-%sDjc`9knws%)W*;r#mC`AJRY28(sWdOjBq=&
zLb_T|Vht$t;DqQ2$E-JzkLEjrzKtSuY^V&F__rcmQR%)?qwf1Sb91{Qpo!*eTk*Ip
zPRH2d8ZOn&uRw^L<oKX7DA#>jrgZk}zX8&&f5-VK_x<nC&4lWlHHX}2eeV%bMXBnQ
zVEyGYGJY~(BHTnzP@fl|DZx`&NLLqYb&M^+6hluYEu5%{zkJ{Cys3oGe+bT-?vz2J
z9C;6rtk{KX-{s*?vb`A*^|~v;!6=#<xDanW&HJv~B+PQ{TkE)2fuR(J;Av<oeJ9#1
z1|B%N47*V$!xoOgN6tRE5JXRE-60CWahi_yiD`e2`CzkqP0E}2x`~_Bq7?k#QK_;h
zwFd*nJuTaKYTp9jNc_po;JkSkO()Zia&NSb!5yu7x1*gEpWw*8ZDdDj?c4$)B@uN8
z47T%L$m5kA`~VJ59OA|irpjANU)9qW7Cc?_Qtv8s$nCg{4qq_x7;bSFWBi5@{A-m`
zp8qbp%^2+4R(*m6ty&%AM?@b<(*D(M$T-wyxwcDEc6`YPOdKcPYAfITceo?Bvc#wW
zAogqUq<1`hnDd*mu0Qy}Z&pJ>OKF?2uY8o1)sj4Crs=cr!8863UX_VSgPAm4c(srp
zJ~GZuK;gq=hb+bWYYU!Q*xU-A73W|?1Ek?04)|#Blibl^)p&n)NleF5W=UAmK?Tv*
zO3uG58LK!3r3_b*;9h-M{h<wRg{PbDGfVPlZyYVDxd$e^8rprO9P6`6IJb>*96xqe
zYfP}<J~xH8+(ln1V?b50alpS+pRRGmm8QSqSmp?gb=(gnAj>sB@dKxtUB&jh3UKJi
z?%;9PoqnJF%pDxAJgPf5>VSfXIghK8a8)RHk7qK^?Y=i4==jIF>G)q*9Ta!FS<9p2
zo$aZ)UFNK~7i1Jc+E4`PuwKy}K~hT0@ON0unhnOvN6~G^f`*&u5b|uyJ?sesG;hok
zOhY*?8r18UKBm1}Ys~&E?@hmcDAeabCNx{VKMFOG<Jvj^)XuFf33M<bf+4lI%%t1W
zJrdqK*PAmsqJ0^8oDet}GL>O8L94XnwCJk#*t|+y<55X-$-iHl2QFBGhm*xqVW@pG
z#Gj^PB!dMrTH5wFw{RHw9{_IfbO2G^9|aC)R&GpU$4el|M3^th<LDszn3yExplcUC
zC~thyrRN(grq0@f(4c?z*-5Hzvp}zFtt)}^Vw5XrAnY9~(C$13Pe$tvLT|7nwkA#C
zzk?e-=hib4_GGkc&YrififYgwR@lXRgGGk!UmZYa`8yZ^9tZx52@(Ztu$@1Xwh!Rv
zj-zMVtI#T14Iwh>%%5OEJx@})m;ba^x@9qF9A^Aws>gM}`0ou6NZ~JGFZ$k`G<naB
z5YsN=H5mirk!uFw6D+_0ob>U$Ea>t{@WT29S)@v&ViF&IVD=-PNR-^8De5S19M9ym
zL#*EqkI2D*-8bE?*mz4rQ$0wC!g;(&aq**tPKHOAPVpZusk!BdHEiF?9>LC|!g#Qs
z$Ac|<zxk>K;TRW(e;Uaj7`{|zAjtd!U7Z?sbD%u;1-bh)hEjaCMh6^i^h{S=Fd?3k
zm<6vJ-CAE{k|*1<7TI?Pf7EZHW=!8bHU>BK6+KlymdSM5x_QfwX6r(a4*&k<#FHl%
zO&hs2D#L%?e)3-lxj!r(F(|V-X#Q&j?@s^A#vyjoOeuqPOgHK3>;K?bRdT5O_>e8n
zzMpvg!la9H-zG9Dj2Bb34xE3E`(BfC=kFDqz`uVSHjkNL6_9W4)R3RMgJm1u%GycG
zU1wjXUobZPWr_Kk$!8KfpKemie)C(L<(8-u!L#IoTi-_NtH|BUetW@&Mrv9RHMeTY
zg;{d;Yp%!6GEkDyrSE)epfpa`;3{*c(mdV#SDSB=JaZgEZaySM<>&|8@F_o9_;%@q
z=X;el$S~5{8@!&dYyBdM6YV%3>!|e=jE9REG6Mz-_<R1;3~nwr&1+c#yUbfLf|JeO
zJ+~&RzWmPR2}WZFc^k^rZ4Y?&*8{^B#`|Uj93z<x*Lt>jKP9MsTk&d!if7`m*=Ow5
zOey@JsFfI`8t$d5G$Lz>L8Ot=6y5Q&BMbHR8XS0DzDr+cXIN|bM1%D^*D&2DTNw{E
zHvBEE{BNJlH(%UKrp#?EUYU@5S2Z%*j-ISK??S!y_0tsx5^Qh!x*MB^x1O?}?iszl
zST}S>N}0{a2G^2hw?IJC`CY4qf7gjznsrmE5x2{m&g>kg%-+v<)$t9i>XNsYVG*a)
zyE=liXTlI&YH`#~r3G0P>ymb5zSzCA?*5W<Ip%iHw6jbmdX9ZMNiBP;lIdbSjZBt%
zpq1u2^y3z=lK)aM)vl`;sr6JnL$6*@D{ZsZy85Us`3k=GNSAVSbIlS+&-8S+X0pql
z73Mv;<g@u6>HU-owf8J2O4?4go2)1n+Lnh)Y$-3b_b%2AEhp(SuHLLF&(k+|`p58e
zg{3ZY^9&yfX?#41bFq^~tD&+w{iafzS~Zq&78_AJDs79eHVlN7ymt38bYwG^*l}tD
z7TPt|(u<Ub=_-p{&r|KJy=XnEYS~-x^&};iMNhh}suF7A2S3wCJ-C%}dh#CM{b6nH
zfhIQtuOBJAX)$#Db|vk)Q1eJbC6lnwnKjOoN=u8~FW1f3uGDgF?cwcFB)Q<kxEs-=
z1;J5^7;&V1!S}9)Zl)-OGdFBGM6n2uT((IixiKo#VaA0Qq$_fUN`E>u+c`G*S#>_t
zpj8DpvxiQlaa(C^jg0ukS?I?7L2oNe<aX$=EvE(~cHV_M=X`|vi*|Fps7p6+pYMW!
zl(!S*&V~AyuP(eB``3gya<pO7CA)Oya><*b^kdI8C~B3YD5S&(ol9P>61v%EuY1L=
z+BkQ6cVlC-u;xT}xY~u6*DA<l+Ba;Tub7&nU-v`nZ%S5vu_3=b^hl~HO&IUOYNu11
zyDn(3tqiHnl{$%>lvFCc&WO~Jm`m-j;*MjyYC;#UG>%p272hRKZ7I*Dvno64jYx24
z+ywHp;0~L@Ba}RCFO^{~YgF7?Z)G&C){l7CsJ0%rs<h2IlpFP<i&x7ly3t+FqaSP8
z=04dd^HNUgbkCEf{^12Gp65b6FBVva-J7j`n4%u;b9=s9`80#s5?a+1_kpHMMOxJ*
z&Q%R|04ywx)*ie+g4Wnr+nPb{7;jt6gf$KLNFsmi%I-)D3E=8ZRawT>XnJ(1K;&^H
z?OmijsQkuqWd6z5qKs`OFWsT!Jnpx#&F9ue<*Rr)t}!-HP9*&pY!_ulBdra#*RL&(
zno+nyS5Yg=zE(e6Q7hl9W^JsIlAVM0?$_&PY(aPM$oAXacd%^{BO1ShCX|Wj4w`r0
zLCu!UGrTFJ^;t$v?BrB3yVFmH^R~|DXeY~vTb@ckWV=ruEho#q(<`Af<7#}2Q)6Y;
zR&MGNxZ1InaU_f2R3F#JK|V?LIs5H?3eRmia9Pd=xjeTto38SWg{r*UH+vX3;ero}
zGKoQV!{v<#m-mzI(%G#C4K7$-3w3ufU3xe?$34&@(q>Arid_8C;JM%(W_W_HjGMKU
z@ZMu}9gz1{*M0a3+y~~pBMI+qT{y{p#)UM}l?hpj%G@}sWm;rrRV#4vKQ@v((8@T-
z_f@uXJ3f#pw!dL3OKq*piZBX@KiZmQ)hXZL<r#YBVBs7&znaGD;MCvU@o(KeVNG4+
zl5N3B`;e!gsEv;%y*=V_Fvtz_d@hE+g@WfhrebPQm!!z^d@^t|*Ck-Cyq0j~yh}IG
z%4@nT?8Ry1GUvP)mHPOttzxT!TgqF*Y9~dR^v$>DU6|z=*jf^5Hrq2iGG}V-byCrh
zlm2V>s>y|vvB`$i#ztCa)x0K^#F|t?I3sd<Ft@zUs4J0H)?hOM9ae2yV@Lqirm4J{
zT8=KDQS<Q2wRNIgh4J1+j@aFsmeqJ#(=zq_D*G5x=Yj|7u<>U1jg;EYr9>@rr2MMg
zY?19uIji0J;4K`MxrB$clL)naA65gC>y$<72oA7gpYb9)u-$dIPyI5|lpUKNlBn!;
zI^21-)s1LfuM@o^*c~)5stC?)h*OEqP376NIP%)yMIx`_#><}GSoD2vbA`ByDEf-O
ziV%I@=^v2j$PO!}ro7eb_sDV&(Ku*jU|Wq<9d1U{iV9d5hsvl6Na*rw*iec*;NQy1
zPWA7oHEHf!sg+<})ahR5Ye*skR*-h;i(i!HpVWktBYdA0-Sy(2grMKR_YE}4%*%08
z;rTuk0g&(e9r?bw;UTxP?J4KOFWwG<`x>1(68$^LDAM`7JB>N_Si-rx?3<Ab&OP;w
zJ!5mE0^!^bF8)>Ly4n~wze6x>nAp+j?nUiPZ>BZ|lR2H^bU2T5jg~xgW;g3N=OQQX
z$9PqQ7Wap(Wnx<f+ik-#b{DO)vLh@2x%XVA%nO?sk(RbGxB|1ZyK3X3`mwa;X@q<4
zcZqx52kx$oc9(F8<}o2OIIl3P&dVARRz~3E1MJynrjolLW8#)*FzOl3>=!=N7sdZu
zjBe(~(aPLs^~@`dogK|6Lc7~N(#|?8Dt3YUAY;M#VvRN5+dGZZ9}7-9GG9ebz}x$3
zQO;{0mdxS|lm_e4cD|Lw;$Dz;Sh87Govj*9jls0`@Um#DB^`}%Wzouxkay@9qQtPV
zbH56=Gp^YNEw1$Jvd)%wOfsZXQ#TmKcb;LRgj19|Niw-P87<E9V$EFX7srmGwRIZN
zVlM=_v}|45=I~SK<2f(dW@fTtZ$uki3}WP5UfXs((IH@UrW?~ewIOu#?dS?)FGdK&
z;EDhgy=ZM!tyb}NS65}&*VFB@B50iQnv4WH&WC7cJtN1iALtVtS?yXVWXmUTI;Q$q
zC2|s0(a{qBRt@NS_JaF1&ApJ^UYT4Sfz}z#OigbQeMKP(s}0v0R&IScSVb;!eBij6
zvhw$ZuQO44p6*O3T{pI{^JDS|)%5$>OHK=?jaFxp66*IeI75e`PY)Qd$^pAd(StIa
z%PL!1HNeH253V1l<J?}Dbim4f@wqmGfrpg24BArvHUlU~oSj0{6Wg98&fjOt&FBc9
z6djj%v`K}XLvN0DEp`r5=HxJ2ul^Nn#avu+glUX@!oIQ5tJpb1D}&7a3nh=8>uGk>
z_u9%OmW9c?=PgZCm;E_GDd+yO7n4mBE<*6yiUkFgtOw{64^K{@5kbM?5RLS4${9<i
zE9GY|RpsAgmruJ~dmt1~NEBosA&E{YE7><u4*M+rUn%xYU3F8HojN<n&P`p}M$aqu
z+f$=072VQX+B4@~YWaOzM&~TApjQ0}lXdqxrT?OI_#~vTw@H~tZXFC_DkAUJUuL%#
zdAA1GSNOLDzPzNsi`#Bw+hk0RJPM&AU|B~?wn~+%dND)9?H#IVax)^rX)$(zFPZw{
zsdI^u60*+jTufg%q&pW|q;D1-{yggS`Q*nB{0qjpIrtqT34DKAO;akhD}vfolN+I9
z7$57DWmTfQGAlYxMB;R071W5f71prxE0xrAJEZ>n=?ZJBdBah)K>VY-u%keS6FxPA
z#*y)d1jl=-iXppzvr*qmRA^zyo((N28RIr@xYq3QOP8N3Xcc&7pK0M>7aP@TaIq`b
zjm_#LVHbM~D_N-Y-mK)x+Aye`_&WS=>2Qza1W}$MZk)1ialJ><n3I7Dl-JttPEWQ>
ztwp&<*McB*DA_F__vkp6@?hBpBKIIXq2;ydC{Y}xRaVmTtd;Zwj42{+Rl$C<Yjfq8
zSViRN8>)?$;gS^$*tII3Sah)GsxEbob{BPo%XEjH*d4>b3S?we_ziY*PqNZnD^Y<=
zk)5-*hv}G6Iy9qc>_Ek6^3kT6uz*DnIW^kca`ZolxWFv!*&|nB+cN)T#Xfs}dLVIW
zR6e}O7Lp5r1AP0wLWj)q0=sErIzf3BunbjZ_sH=%E5vQ};ApS`v*E~f5;+$XpG3|D
zh=*YMKSdz@4}?>7D@`=$vDbfp2X@n0nburL?tJ6El^ciPmH?KqMWm?c>J+Rn#MpW|
z7GJNJ@;9LG{Ky`&;J3(vPfxaH^xS|4RcY>IW}N>L8rzL*Phr(UaG+h3De%k};W^tb
z>raz?>=Z5gbdw9|_h3ptArPv$)lkhL4-)wr|NcO0`JNX*{|I&g(n5@A&Ts*!oXX^|
zAt~je@}`WpD|Ua|-~=W$hyt$NP|0lZDvivf<TugRuA>Sb;EYE;>QSv_?mRfd#pGrP
zf-`muUB45-86?hvp9WF{PJixHWol=KmX4t#C(;X!p&1HU^2c38;+Fns>3QtvU$yL(
ze(7K&J-yeV);HwN4(f+WEp8F@9m<NEVDq`pAxY+zaBdicira)P%hg!;T9WmVxPfA_
z{`5eor*R)noIs$J^0^*9{CzT$%nha02Ldj3%v*=dRot-5sny-?y`{D|6X;r?js+2a
z({25v=vA8t(2poPk{?IQ_jGJ6wH3+_+5X;&(JCk_snK9NFyTfY5s_V@`;RXG>hFve
zYw5p!H(GVfNra&h|2P*~u@b438JuAdz;*X4FQvBDXh9h1Ksce<gyRm;g;DNF;B8Dt
zYuhipwcqKYR;GRZyvU8C9a*n2LvRjeD!LVTzBvQGr^WJsDL@q2`*>MId-+MoBFrZ6
zL&@zE!E*q`-K=Cku4gFSMW=JzxJC5#ff+_u$#xW0ppHMCbHZCOfJ+CssZ21Dz+8UY
zf}b%z@OMod`aD09L5g-Soo5P^)0(1@JyKKKCE2H_dLjNv>U!i}$XE&=E(a3?un}OY
z!f`vzuM-GJbb@1y!0NHz55Vv1^X_<TqhgB(j4&S--!$cnDWp(n@x4>18327ixFWs`
z|NCe#nNvY6P6d0QH&!Os%L5L>-cN72KQ+UOYs7TTGn<707MteQSgI&D;NUYB4oW6q
z6|mdu2S!rEwUnapUwb&?KhzQ#9RVyIG71MvJ6h_fyuC=yrOx(zFg^_1i&c+<g!ba7
zg9T=gnpte=vznxX*VTGtHQLpz04Lz__*2>PS70CJp{%CJ&-sv5O#N^?n;r6VLjAiW
z9%da7dg4ip!71!6o(zuQr5aKkhiB}Cmp!7QHHpsD;(nl31}X+nyWY4*a5Z)OX`TBw
z0DeAms!Afqs(x53v>gzJ)ayuGi`Qa^^CoqtIO!9F*WorjkEDXd*&o1e@UAxE4xFlT
zkoywcBe^-4o;S{ZoGymn2ky!&G+b7MC-*2sd0^|Ae=_NSeFFLQk(UU`k~~FuX?eP*
zWgPB&uSMo&P+Pj{XiO-el#!9p5csYFwO1i?P-)55UO!|h^Rk(WZ0v<s2+M%Rg(9A8
zT2tCBSDQ*^BoEhL=;5gUo(<6|pV9Rm>;uJXW2;Ud<S@(E(@d2bh6|j;u?PMv?MTIx
zqTpxf^8-y2_zp^v&m-J*x@XM#%RN8u<U_V$73w6mwQ`@JLMvfv#=(GqCDc|j+0L0o
zCE0OWsTGjO0JZ4#<z?~e-Iw%i&Y^(NwkKx`H)FwjVt6$Xw<OP;w2FdzUedW%KGgR{
z?S9Us$@NTs$ND-Nw=zGM`_a2PnoNOWDB}wJG;_Pb>sh^bW)=3w=AVRWP@!+v4c=?w
z<yzBFTdXUBrxB$K4)?sHiB4TL`)SQ-xrxr~H}?ybm%-i8NvH|g03Q5<pEH{SpnmTk
zF{mgQyB(UQOF!X9lpHzP^P0wIq~$s{SuM5WG__Rb*V7L*b+vgV)MUiZ`;obo-jZ9=
zVxNd1zL?+O9m&zc7Ix4v(#_$inf5dLF7%-uTCSFzg<6sPAa-!^<{mpSVC~VxKQRZY
z9K;Oeg$DbzR>nbUd*FT*YFDL!%kR<A$d|oLYu<`*>j-MxR_^0ynLM*OsQrQbdc79_
z!~1V0kV+ODCU=BmJ`Eac{lxhK6=5K-{6OKPp4pbQHl1lqZ6ia1S<h(4d`cYU;S1%X
zN8+3pt1%m17q!DLT};dUXt=hbQ=i8EBXJ#f=Jd8k<<1YbC2ZZnT0bDOkbaLW7WedM
z5boaVC~c^_CN=a|zO7KTXQ&u3;1_E7(SjoPzM<{;UVIcI7uqU-V$6Vb%<K_@;b*l>
zmAjGtQ2L*>x=^*}USmJ&WWf&fv!Lq5LlLA!`BgmVh?G1Dpd(Qg_JPGmdfq5(w|?_8
z?$uo`;$ey6(2CO)7V}Md6q^9ma-OP2J{*PnVrdHhePL5X?AFcnP^m#2=*_L~nSla2
zN2YAHoRj>Q02Cz=Mj(@K*0Y*}Lu8*yGCMR4B`XNC`&f83Q!;47I83$o3O&Q11B`XS
zrRW-yp6Z#4IKyq`52a5Bj)zzE^m5Q72_W6e`JM0xlGpwN4C+PfyGeMq^sxv$Rz{D6
z3EOu~^Vy%^#}ji_h}bLx^YDJ`uwGHP2bWwHsVBlcl<KdOzE}-3pWU-g$86_Kq@L7G
zv{5p-`l2qW`RtLBMP7hj38w|^!b?4J64>|7A5wG8B*&=vSiplEGZFi~E9?{e-!uIe
zB5v04@Eqc1NnV`~G!Nh3+h<LR8r!#rw}j@UFo2kvU3FH(X62coI1kQ&O#mk*e!T9k
zSY8Wdo8=r4D?A$>EPc5XXm;p%Z`*;M_(<u!@@BuDU>aNp2(A0Vu1W8eFBK?>h+(9{
zyPr_qhNjQ&?MmeyjK43v|NGVQm57Z8q7@!2dJ!qagZHNgnjI45v@BCUAz0|tP3pZu
zPv{d7PD_$<E0*L7ncb3bHxV8_=g51)cqA(k5M=I_WuMmjYPndt&5|r@V_;0KSj6N4
zB29>hpH*<=2Jy4R*fg-*5amOl&kne2i3qn0R1WEVDP-wAiy$oU0C9|8--!mGh$=?F
zm&#pl_3iB*WQu@fC~3cs?@Q0+91`J6DSkJJ>ktSGIuSgQ-reo;U-TIwF8njg)BWtb
z2&A*(K(B50D!*h<BmG}WI?yR9E47)q9WZD<MGzL4wt$H<6jG<lof2ke;mww5zKNpz
z-@)tB<2G{t)HsXS_aiemNe_j|u4jqZ_oDi{(r2Y^ug{BcrH=}YrC*^ZfSV%X;a}0b
zpCy(5J77;0v0Dg5k~+h64u&qGU?&o*c=8I}cs)gFV4eWeZ6baafXl=^m4w<@TYf-a
zCHc*~VIo|-i<R<kM6aZ*yH^6#Pk~+u^zcKjm+)DHA?5EcyNcL(=(R|eOcE46tk(3e
zN$O>aR8*S~OO)K1-ks-kRK#{s9d7jFYk2uGuyr3Q_~JhPy>^9&pEaO}CB0+TdZoTh
zL_~b#6y47fMfk0O`|Wp#SmK}krSZM#G^3zN*)AfC4T?4(E`*roncYZ&+sxmsJng+A
zc57l$Tt62=#Lv=7-#kaeZar}dfkWhTB7nVh&awd~R04<mdyyRS<mK}&^lT?#Au-9?
zs#oIH4L%^=iYV}5o^D64-BPZwHcQ0VArh73ZZFl>_c}Z;3TggMiD?$6C-3v^y*;U<
zE55Ulok+xTPX1=z`kwtIR4N%OZAe#;q+-TH)zQnrz1@kbT*%K-py@H%ZoPA|H0jw8
zz@_VTGyzL^G*wBn&2^MJ+HSpXOPaBzTLVtc;1ffEoFPCAJ&%Z?_zcDzPt?7em9@Kf
z0BG7(I4KUe=zV)L0XFcc)aN!L>~jgEnNqQDldq|bpW54?<fF&`Bc8Y=3eTOM)!5s{
zn_y-HG0^*F83?d2%YV7-+57T&LvHy&8v5<1RA5+(IP~%bTl#4Gi||~Rv=8?N0xKs)
zPe;^kF^46>FGGkFvB_Q!FJN~^G^tn=@_H`xwun`|*SjZhRuXh@R|R*9II9~5X+1b)
z-TqVb3{El(y7cVxAV=Y?RlJtZij&CiX+7I))ywbcOgJ(V+TzjxtrK;Nd*`8ckcUb&
zS(MzUnhg^auoQuRI=y8|CS*zXb0+AH6Y+Ck7zW)yiRWlYi-M1cIIGu%m!xY=T5H1#
zErQHLgvwRkjh3&MkXjb%9%z~{5_sMEG&A8pB=-)O88!X<p`LDVG)$VYYokRh4Y;Db
zr4e->OiuMNKTt1wCNv>&sa&s=bJ;uC<^H{o0LRIGzlpYGTQ=!Ruk$~jN}qs{@I>3H
zhyehNxxq)`$fADLH!;k^XpTiRv$;3(DORtp-b<Li373<dHrbjUE@1gW^xEED8Vvx7
z{!5zHq2m9L08f{w+hM$Gf-s{Ie$soL2`+`$jS{_2y?q)C>G<^GD`Jatv1S|*zfHYU
zLzs_w;bI*p!sbRcEbikPbn}b^h6|fU@914>rzKg_;g(C@lS2C~kCEO4=`yGfu|$>`
zMc}BC&eKge=3vyzPeOipW`@MM9gIyj4Jgv=y+<i!Esrr!SK=Q-9oLf!24{wnMCHvA
zC#!I;!9uTz{4vWV_mJo+I<e07ii?naj};yk6OVDm<|sYy#cxAv`oZl{P_e|#u(y#_
z8-Ym=>@MLGCk^^J!9ZwsP2|scoBZa1A^JJSR>r;EWBJ5OEmz^^@cHyWJ0|$_cwLh9
z8v#ZFMhau4OIjB6WK6-TTBdsbxyDMW8Lg&gg`$4W+)E=RPVQjD#;&yfZtnv}Bd(I?
zi0ID<)30Dn0N=x$2ho}6pVEo$&5@Zsc{136Ydv|pUt%^M=2C@8Q1EBir!;rhMi8R`
zLi1vx-JneI+Fmh?^YrfA)-7T}g=imWd{DP{%KG&2+*Vzw{*2~9-QEeq(>HTlHxYj(
z>LG6kZUbdCJ_IM!M?xNX@<H966AxH7uxxHRJ~kmqf{6aiz=OIX_NH-edYr`dHTp9Z
zmTo|6g5weRhka_`9$zbo8HgxgxfRNZx;HS{E%87Ac5kA_tMx9u5c(^f_Y+MM`97=b
zv?SFnK7Q`2X?LgowFuMH)^46535<x3<(tdByBlRJ;{GjHO%IV|f&oi2ZNe9xhYqi(
zJx*5llFs{O9(gIk8P5uak#*6Xh2Gt96g7V-yxY5KFORuYAg&7Jb#nwtcI*O*B$gag
zjWi&vN@&ncWIFh6ca*+}(D3Y1!7MQ~Y46?=TGTXyDLaM6g1|7qbs3w9b6p}bU3@ON
zC7coyE#m%wY{4io5u!WuA{gFC)&XXL**T=5LJ_A%3O;gL%&GP62k{0vdrv-LOAItN
z3zWt?qe*g8E2Rd@UKyB<(ga;m4RJT|LJ!aNbcO#+sX=-BrCRJVVJKaH6eUQWl9xyM
z_8x4)+6e<L*Gdlfp-l%Zjbv;*N4P|MIzlabJ9alE8S^53RiHIPgxAM}F(jQBBmR`g
zl%C?D-8xEQG!oqeL^v3|qc$RelN)g`dpY&G4NdFZ*bn!1(0)vaaF1o}U9<LTsEJW6
z+v&#(4Qa)f75EY2ku`knu=HOWe#O2a;`|5Z>Z~>&?u*RhJ1^F64Jw{fTkn(YT<yeM
z=bJe1LhZVG5Kv$xk7;>VL#=Pn=1o1qw|B*$0T`(qI5EBqEv^$Nji;b#g1!0Wq)VL*
z|CE&HT`H)JXQ1h|Bf;mA5zja0sE^s~GvV`dr&`ez1p?6{nBE6D4{6->NWRd*ID^q}
zQxs%ietPG{*`6+;j5%-@&xD4*V$Jcq9BQ#2gg{o;J=D$Kp#)4Noy20FfUaD|s|Wo3
z9^mSpWU|Ovi0y3YgwI3ZBLUr{+V!~9$lkE)nAAwU>IpQ@gO;AZ_o2Yj>*DF%OD|cx
zB|Q0VWKJwhB^D~`aL3v1*YEy*QZlAv(ADRtKM&F6sp@#%%g#iqIv?9EJwbD>B*<&v
zZV}Qg2ej*vbPEjgEz(WHrCV~4bW6fKIcAiRy&#^_ZcX`HJ9TE?ZoFqu`BYVi#V{1f
zDz9PC7H9n*fAyp1c*1zSR9UnVZ}Uru%|{${V)Gk_d-wO{=ZE3mQ63PRKLmaL`n;a^
z4jbK<l!zcG@Ntw7_<+mD7x;)HL}m^I{|?rTi_BmGA5Z2MSsm`QExBsiU8l(^0;ZlP
zW>#4=n9}FD+Y*vJz0UpmrWo7KCU07uki~<JhSaeViM0snW+=!!d26L8UW7q!^zlN$
zR>s^_O0%<eOC4EzOC|KHBQVqB@%6;6N=9POD<|h;+9_t=MPpB*uUvllFPA#&|LNn(
zX#!xT;?G$sWbYuFBRc=;z-486j54FZv?<Cr_FN(wM#6Mh_$pe1d#Grs3#e!z9hN=2
z2RFpEt`i9Te~c*q<8m-z?NbURjQad<gPY-hTfBl-B<TTs8|ELKD4k59E3XAjM`>yw
zMIa8>uI;>uxjM;Is&dL->${)$CPrp4o7K?M(QEFTA^8kBFC&mterXh@&@ms$FU>qd
z8z$IFoVx-koJm!yeWGrfX}jevBj+Y2y{Dr=QZ~{QuI?@t^ACgYk7xXu*&d+z<Bk|n
zb6zO$1af$4E2%FxN6iCKAWR@<4lrKzMguUG*wG?i)bsPuM87RbbjQZYp=|l&&jS*c
zs4l>E@fZ%D4wmB63qLinnA^4>7%cs9+`QTR*!D*mra~tP2#1Ll00?)Qr?@1^#?O)q
zYAp&i!#Cc3T%pR<u>#|w%XTPfWK~F%&|&OGBnbChZyhcOXFLQQI8wN~_u}_t3SZ$a
z*N%oO+)V*3E>O7BM+$eNRza^yF@*a_D2pxCu0Y*)(OnhbY<wYACcV)|9UQr(LLaZx
zt`HYsL>AnH4w+DB>rWC&Psw)XUjp5m(xl{TK>HB`=k?|`env`Be#hW)qGP9~P;3F#
z(YCMg`6tl61yiTl*#>{dzwvW-D4|Jj(ef)_G^IJQJUY|KPAHRY(M}R~=wj>jPs%DD
zu73j;K3vGyKVFuiM~{bfOxzE8y}F}8b7)j5h~_~NBX>NMJ_l-ZyoizW4c&o?7_GVl
z-E4hS#LTT+u*Jx+i7u4Ycm{M6<%Qws-c$ntdk4NFZtp0*f}-WIxvhmOiwQ+bU@g^Y
zPjSZ-EqV6dbG{MMxnHtuF)d9IYF@qB2_jbB>O;p=v+%_WRB*6(sNlAAFaGY_@&%UM
z*Fx6*?fQoVxYEBW4ldJ2Bo#+M&nB}kSw%dqM=F5Q$1J&+^@SP%oFpNeo1_i`o03Z7
z@X&$VK@q2svym{o&Xx<0+&0>3@vlaLPJY!%;{%{JC?&@6N$Ffn{YFUVyo4gV%$t@i
z(x-DsAhG|top<V3-9ESA$TQ){{-&k=Yot`W0jG9qOrur{6;B?)RuU;)Vgxn*p@S}K
z|8U7~c-fkcFoPk+mZ>#v!hppA`>T?}QJ<``ISN4WEEPeJbrm)s{xv?ms=%`#i`yV%
z=x82Wl0iX4eanR2C{efxYQpl%b)*R^T8}dm;CF{kH=*hQ8^P|@_XBNH0ewFZzw^IH
za68^S@E7oMT0Q;yOdd!j2Rsr0se1c&^;deDA%ayKC?sXt_XD+m8H~pCQuw*VC2TFB
z`(|vKAPy+t4Kx!g2jw1tl?z2S-pc!kY=BflWScDHU*V66i|f6)(qaP}FOlsduno#L
zt6L`s5Di8uwO=KjQH`46&>1O|P+05Be*t8yUzlR8rbT31OTLh)EwOA6_wg^8O<Y)&
zX9n5=-6`Y>rHy`v#AL9&)oD;)Q#2}OL7Aw{KC-C~RP39mg2hhGLiy_^Kl6X(<J~IS
z2#R&T4j&h6@Ocb(dt0@QXv#^_>IN7VYILV9N<1i%uXjCMtlqUb<v@6HX1o4Cr;apt
zp}Tgepa`=u;~t|yy@i{5hO<w|VGPZ$tAL1^P&B=<0OmM=Wg^K0I689-AA#Hhfx!JV
zA9&F;Jo#Fr!cdqNZoMv0u<bfdy14*1aj8Bny`=qa>+pyEocBUjw#`xwvoc9}P@gDr
zo(Wg(Dko3q<2&Hv5;H@Azw)Ucf2|)Tlth??e~sK}?PT-yOyS&Vp!sYmzWwRBlT1!Q
zgS!sru8^HQ(+`tq&vz=Vue)xZeytmn=~tyc4Mq|(cq;uTD3J?Wgf2Ruve4z?tA2s-
z)#rD)80EZ1%dzf83#dBOxQ9?Hg-jftbr9oO30y=#xdz>~zWO{R<*zRl)Ioo-cb_NF
z?0#*Ha)J=#e>)Oq3ij6a<xw64#=!bTWui&yD07S>^=kBF{#{VCXubdh=@>(b%~{*e
z!n=oE`~9@|9#bX^hA(Ct3*}B_kqTcUce>#Ewn}hF7yN3Xp2JsJ_Q`@<tO}u3e-q^^
zj&Jitkswf^iGygCDwHozwun?ffPe@kaIqwmPim~`)Ll~<Ccyoi{1gc=NXuFI6bwu@
zq%{*-*$zS*uNqZAT(|xwjB(0LFad>t6hFVy04+r%#ow*#-rZ?HQvBOSZ>S5-f`<co
zf{*Y52*1tIt&S{O+7DUUCpMR^LwzU6hk`akzS(qXSkd@nGDf4Y0;wPV0w4X-H@~ZX
zDQ745sZbcd6z(d0?j(HzLzxv$OYBG^3nRY!7azgwToS1`{-%Ed1av+}aBSxfc}1B5
zPpPRwpVH0q3~EKJ4VKVV2Mr|2Clo&e0D?{nQz(*y9JfR;2>uF_Xc$Dy(>Ul^l4{=5
zI8R8vC5be$aESzBwnA(f(E^%CoUcaz&@`exli&@gdn8bM<Glg3d-QsGjgC<NIJH1-
zXI~AqFZCizIfx$!LtmIYhgkjOFZX=P?&0g>*z!W)Q=p|AHg_QcVi!sM{8W|>MRvWz
zQns$pT`!B$l!|vxaKog0+G6IgYtsdJ*P|!F(^>d9ve>*$K(qNXkSFy&gBb-rA>1UP
zbp4OQ^WspDHg{pw=Khk_KA0i{LGl5RFLG?pv=yW+rDEfYIrV;@q}bi1Eide3Di^F2
zlfQjY80JLTykYfJN1;f5Xr{Ph7deISsq3ad%a9cy+|(#uD{b}$FT?SIIowkzx>+Rq
z;4;C|7`pa{XYx;6A{YpxYTz)?VgPSy`hS;B-yP|{$z-%FU&;;;@~>{iYcWHY=#>Y4
zP?H`88oM~l%~q&=izHl-$b)oPNLV@QUn1D5|CRe9zQP69SL!!abhs@-X%s+7pYVC9
z-<x~#>~ab-c`Em<(C48=@m10&>)=#j1gvF2loTA9Pp9W-=$_)#2sv1oEdCm)3@~v{
zx3o}?2)P*t(5{iBpY+LzR4aFXwb3psBPRQH*nv?`*Qz{d-8EeP!GRGE)GzoaRL9iD
zaMoD<v~cX7>MQoFSgCz$?2o<&(#yWz{A=a!uTO_x-w|Z?_-6P6>ql)i0rzvRm>*Wu
zGBBtd{N1&SnP%4>{3uslwExQOg(>YH5>k~aI?iyLyZmz(+wHw|!D+jB2KQ23v^QNN
zz!&uqgEq~3t~z|`nsYx1{HI}>T%yxyQr8i8^{VftB<z6q`mDX!btfn9@yNREL^<!V
z$l{^R-?xToqG9Z9^9Dg!SC~nH*9a$!7?kk$dD8purp$hEs$y4*<)#SIf>jPx?9;~k
z&d(1Ff;28}dgN)awlyyBf3OQT?APz{<)79>L(BXi(A*DRK!3`|oKiWoA(9G@>!pzD
z3*f!V8>tt)#!oX(dJTF(Iyn#KKo+^@yJ|4@$xr@PJpX-a{NF))7gCfw5@*+ra!Wao
z5pP9#?eY7*^CN+Wr*T~mp4FRg`qHC(SwNSUsE@;kN5BstK3*#LaT$S+!{eWy_VL}%
zJa)&=>%HTiG$##8OTC`y>!EAy9xBKrmKw6E;T;HU$Ip7$qaD8jI}S2&+p}T;{&OF7
z1u<?1yBKa<tJM7Gxr^_fQs#C>-GJ9q17|lUf~?H*%bR`!ibU~weRR^tN6d3=?x?gP
zUo{mtYIN+yhDv|1HbAWyo(Wsn^!sc%$Hxh68Sqlx=#ZVP@`+<w^M0@SW#Kqi^m@>}
z;KRJ*h9VU|Znz`FaYOHPl|hx0JDl!3IBrkkxc@hBpX`TpZoAF@({|-?G3Rf5tff7u
z-F0-0;%6O;B%&cTtle5?uB}xnZAj8Zht9FvuB<M12*XAp)!KAuOd-`0rbDHsY1Bw}
z>6~hsncwr7d+OUk_!qCwd_Ldj^E~h8{k)&&6E#DAaW@~=Pn+m}MLH+1fsH)LNhG`-
z!!9Ml>w~rzJQ@K?y|B%>Ho9t`iaET0@%VfZdinC!tL;$T_(7%p_sx}Ol~3K7H!CzC
zFZoAC<x3UEb8Ice^P8RrpFsNds26j96qp$juCsi!tGu1nhE{&H{Tu8roi#1=#HK*4
zaK%#zHhbXNqFD~+sPE?X?+2i+@U;zmpucRbHs=3(i}00E`eb)_k?qc|@?^puqos@1
zpeJ4<HphU>_Pp-)tOM)#p#|s8$lk4QSLZio>ZGj2{?<<T%Gs*!!#M7vqixz9HVt&#
zt0Q+U5BrVK=ctWcziP7mxa$V&T3*6`jrEl$HqCmJmuvy=oXoDOGRJrQ?$bzyNAPqu
zP+X5h?tEc0^jDs%_y>Gts`M)t1#gyq<=Kj7TGo+Gc+auU+a^UxY~Jnr=L^g`e!Ajc
zuy`lDYf<-4RsL`PE*bY7v{CF_UytS+o4!2_(&8fvz>|RpX!fpcrg()&=OAxTKb0<Q
zw!klnyrp{W5p)r(zwtBP++mZ-Tvm6v>XDQ3U)pmgKD6^5TvKb8cr5pMRCJmDGd(%U
zcLvC~{}=4Fa<wZM*tRnuF|48gZRNTzIO!>8a`w~N4o2TW=YZ#yy^}z4H1Y70gHvF)
zaTvQ*4UE*%F0TAfxy+Wotn!Eo>y86FTpX!rF(3KOW9YF<a9vO-+d0PiFXTbMm^b6$
zGM(84nojT_W+D%wv*^kBO>0CR1gd0z9$#pc7m?1v_`F#ad$Qr~wvJ)HOb?a%>WnjM
z6`qtXxC#R)PcDu{`rU00?h;x(;unC=80uiRfv9c0+H98l`tgtCE%~YmjevJNcz3dK
z+A9rC@z)|pCAvA=)DKS-IjUo^GX-&}X;K!%1&B-6=H<XUilVrL9F;AKOQ!^mYB55A
zt64kmrPVSIEGv-r7pqgL*$S2U;BMOYYlZqgFzAe*F0fuK1A;a-Q<wRn_ZV}B^F7hy
zjVBL3zW%WKq~`Me&iRrnp|10Tt+^U_9J?)h(V7#C9$){fuaju6t*S`2xT>Q)yB2O~
z_wPOp*2W@VEV`w;991?8e6h|EP$mB|jPE%EwRx?-A^4v7ZzE@d-Iin6ZB<Kh^UM&q
z00SE@3i{eb!7HH#l=;PT&`kM2m3Q`ciry=~%lWsGpyFroVs2<i-Wf0GvQpjk4F2Y7
zhtrx^)a3L~MYW^5_Zhz(!+O1NO_bH5@B=0+4L`7D+nj90Q;84<bY<hfYf&74|CIml
z1%jdxb8^Q~hy!*QabUp@Mi()B!zqPVX(x5TB=2n(I*2xSv@OfNIF0a=kwKDg!wRtD
zp#yF_K^O29M!=oEbL+kyqJDpSveNjC<1t{MZ@oPlKJd`NmRgys(ufso#~<62=sC7h
z-Vx?cshXDZ?RfZ1@NMrampSm^#oA!k(nNUnU{l!8kyd!Z14h~fhDKldSJkdtD}QHe
zJ6l!Ct_^s^+m~&8e)tM{RP2_$rO2XwOE&%k{QZnjFvFbvLW_d7L4ieGR;_wE`#WfB
z5Lr~^TNTGHGWq!(`MsWS(pGxtH}9iumaxo7e_TI@s2%o@`$zw+>M4rR`rz1c7pQxC
z5(mVNj~?ilw>CXvdSbXmm?5)L4Wfrt9K5icQq1cXgdFui!WyF~Nkg9Tzm+RgxaKbg
zI^%~gRT|N)t;D#sSrrG<AU}|S=pP2S{k;HEmU8T%Lu0=$eE3WjKIry^4{DJ=r`KWc
zF)kcnCrU%6h^~wnoF>}*Bm$>Uulo1*_(#Zi2X%)tM8*q^P?i6wH1-rmFVgB0S?P-u
zv)jNUO&=iR8v$fd$l3!l_2k!UuT>3XEd|p#uAP$2bAKfAfFjc=Iy_B$J$J@D9*)Bo
z?(i9M_?0OtjU4Ms7}qmw&=?~Jkg6;Hqf;M3)SsvT+32oqqQo*gYfla&mQREK-3=)x
zN-TGy#M1SbVAlnGQ#(WIa-owS&XHaSn(oX7)@n#VCP>~NZPLlAWV9Dwz}lqu%FhQ5
zq$Aj*&BkEYZr@fmrsMN@5INMxR1FX%uS4#1GjwQ|oD%Hx4;`JX>=hT}J#A|A@}@Ci
zgTTPaP-(iodmL)w84ezk#YTr8sl-_OV7WWUPrU`mQzkz@1^IdROO!)sfx=jrLkO2N
z_}@5)dI-{+pP3+m^qo&RlLU|+Er}pKf@6SUBWX;|zE&m7?+KVw-o*l{|29&?!q7v&
z#0<#`nBmnT4KR3t?c<K<SZQWe7#?((x<NjGGB#=S{BF{LMF@-(C%lPlS&-E@E2|={
z>KXAM);V{MAy>*Lik<L$Q{8RI)%L?VOL8@I&M0*fobyxZIXj6`CxvIT?Tl87QYSd)
z18R!4z0YC4XL+kI{9jIhwM&eASSf{vN`Q8PRHkIX9Z@PHg1*8W@!BJByY|~Q2y(<!
zagL}Pc|mmslI)`#(Zb=`A6DB#GW>R695VNJ3rr9=Wb_;Yq0DUmUucG34$&9ue0#?*
zpUKlLtgnDx6Qpc|@6#0jS)_H)Ap3l6lcLEkc};T1T9-G5`S>HPdv#Mv7<F-92i!DU
z>Cl7ZHaRP15FUy0{vm8>lETB9;=c#7rMB2j3%rkwt%&1J$1<hqJC4bJS1B9=@>Cp3
zpGdNQFkx}_56eoje~-nxjl34E`Ab4A{rx%hzW|r&a<zVh!>PdgQ@xhL#A#p;i@y#O
z5_s5oan~9Fi3ZGJC9r<~&I1*`480P1Zj+Q=fhMCGi+~;NQ$rs7SYV9_0zVxaK9u7J
zkd2~v+Bb+e97aB#cogbU=W~MypRygf@h1J5;_&SIuWUa`jGN$n-1t@6gnp!9K~BJq
zw-biB(P8ALGH?a%s?vQwJ$?jMX1r-Z*Q=q@@gg*c2T`M56f>1!eARhCjqd(lyS0n%
ziS8<9tQ$A>V;%2drc*~zzknXO95U&lj`zI)gs($akjB-#WRuo6#hA$DkGyzcvElZB
zKs#%ysMs(hCY-2{@4ZBb2`6}et-v%aA$nu^!f%Yo1Ml_<Okk8rO^=IGQy@DesVO2m
zE_tQ*BRjyD!l;alzUSN1-nvnR3)KbS&9KqSkWtmjbx$y$m&tvj>Y^*kUNwzY{Fhi+
z_%AAcm?aGnWE#<;uN(cz>v^77?l-`uo%Jyg1nkxRRikS^k|R3YIHM**KDK?B5(c`b
zDRh>hPn$YM(JDMJLnZwqL6)mF$(zeJ^e14o!vb}j<jI_nRbrOe4ytnl7s$4`P7U_g
zssj<HAc^W*ozpx_t@&QfeeT|#bgWX9##grY;Ldz_ssZkdV(KI|iGW3!K{e`QEgE)r
z`~!@OdQ64&&X9%KI~CDRK!vW5r9v+wDpZ{NL&wL=_NhvD$5-(&VfW0eE@{PLRo{Yd
zKg?43D?o0T1G4c6l*1%6C@P2THrgh|EO&|!W?AKw(Y3Et9W#_aQVAc*|C%FV?z<8l
z@L5Ar#^+$1@MRTN%kX%0Q88#3H49V>%KO9W^{8YUzs@&$7qJraqdJNoE2?JE$Fko9
z@C&e<@INYzJ|ClEat;t$`I0-%Sh<a#x7YUIq(KA(VtYaruy^A=*46xocUCVY;{!Rb
zG%+28N~Gmb=lpJ!gnerS#1dFkX!7{RefXw>y$$v0-59i;*UsQ2L+x5dHZeqb3dtrC
zz7v$EM28*Zn&Ny;r7!v?TWLUwKk%N1Sv?s5P=qEMtWr6`X?W(m6w5<`=s>u#l-mcQ
zGE7TTe`67_>_bBtMFeG-m3>)u<d)IOfgG%WtB@_IJNDNMh>A7?7w^SIn^o9Fo6o67
zG8}1N1y=Z%(e?O`ur5@M3MxVN7itU_)eMwW0=#fdX;ghUyX-Eh70b8|2KRt*jYdUf
zxth@jpbOAZ<NwfSV?lF6BZhno6cJERRvtXyi{v;11$;orf%Hq$f*<^7&A)<`{ft^z
zhsi+=ElL?&C4nweS?4(W&zk5x(O9_3_`mA~@^J2hQ6!HllI)Ja!wFag*l7#v-Z}KU
z3Yf3K*$R<iRpQD#+f1;Mps?ecP!RqQ6&0D-p9HbrKyaI~nQ0VSc`q3Y00707lXF*~
z{s9HSw_>$d9|Zw4H*b4jCRTfiN%^S)wO4;q-V_>}e+2W&rXQX>P{IktR;%YiI5|Z5
z;oXS~2#X7PI_>@IvD^o+GJ{apvI>TGV`}z%uU&bNo?H2f9_@w2=+0zy3>BI{U-rHM
z*<8hklJhzl6Zo8LNgX)b9YLi>7?8EbQyGncn5OmQ;2=?P1f>v>X@Tz1rZ1}t$meQs
z@i6h_yJ<zxHNNM#2efFL(z#gi;d*fOP)P5+iX!5L%NkwD)b1=Sqy9ye2CjM_eGDp&
z{Q6Agkx0Wg6j5~oH7-ZJ5D`^i)QkRbP5uUp^;F0?cuAN&3_~jVpGYbS6+Nj`6r2a`
zp0%)|hu9mDR8$n3#!)7<b!cMZ9xXxnQYIb!KxG+(Z=HM&>(-gR2*WC3DuwIZllRr4
zGKEMTJ8=FzggREVWyRFfyD^OVUa-wa#U2OLAKGDYIiMe{{zzO70$Y{G=nH^d>Bh3@
zBdQsgIB%qaT+tVX$z}5}h~`zhPq9Bn$wCYQ4$*VVvFs`gi2sj?d037aKKlI>=TIaE
z`79(5GUA{Q#3zZbaIyi7f`u+Y05A7>i{%vV;skZbA`N0hr4mi3GwO=m^qt0YX_%bS
zlE~nXBJB``L&YQm47o*>%_0r3tARYUSOSX_jPJ~PJZee0l6gC+_${UsD@8R-$UJ{w
z#il4FCcNle1~*tA3&$%wPho;he{gjwlyuRa|2&>l+>Vto?a%W2n1E;89ct=hJuJi%
z_`J()U>;z^i04dJ$<4R=SPN%A@6sJ=;SrE7T8asU4>72`SE|W_AfES^frt^$so0tP
zr&zI}Ml~73s5@SY<q_*;dltMzJqu87756M4H9N>C=L*WL2=9n`7J%?acn6S@w{Pbg
z{(h|Dk#Cl#@Fw@^)ex^eRKLVP5>B+X3t+94Nj+E;6OpA4XZ`-P#~>9XK_phO@H6D}
zRA2#HT!?e?M+5;6zp{ue-+-m9ynOs)f*uu{7<%(EEK==zR;hd}{$S-K^o1Y6urY2E
zR!r2<eJ;OYsc>OLI#LnjA*2<3f&qB~_V|vVH6(t(1|X3^KdMMH?Y)Gx-A%gwW%B3z
z!FZnSAxQ>n@s<wRcFw$bEMS|lM<vkqps3YrfcBr6b43&Y$ZXTP1!`O9)d0n=v{z%L
zY_G=c#L4SxR-*J_K}wMZ#!0i#qQ4AFA@V2_)Q|9v%Ao2;S)wJosI;x~S+fC_Jwr3v
zFypY0;~9qVQg-RXMWr1aOaPGYJ<Jx0;e#I3&L9E+na2}7uyjXZS3BxHl57J}JP+@i
zb9g0K27Tp@#>eD%@Mh`(d^%P<{Kee)V<H~Kn2<mFONgsL!VfyD>4Qm@M^bHpQGoxm
z=2v4G76?p-;&?{DcfjkKr7zcE1SF;CjzRQ5<}f2J^E_62j##7UjWmjjn#}}#Y^Xg)
zs!;?$Yxh&Y-sy0M!!TLA=n&Bzeup+0TiG_5!7DCVFS2U+{#bbD@{-;138JWbL{woK
zvX5i#;1wqPX7%XgewvOcRbMCG2EA{8yz`;b|G$?w@q4lE(y>w%`5-K64f;@Jo{j1h
z5+=-A=eX}<7J5RXJMLp(_{*hd2NFwu5X^C*D2PeIva6d*Cd=E^hb10>^JFFH@hh=D
z@js<1A9K$O&K5;rnY3$Sk-TNS6TBN5bet)QCe>mA_#LtC+R&?y8o-uSg2Dx}GZjR3
z;?7jil?<yhb-h3gV2)o?U&v=+FT6{-@CR&(6)(KH{v1YJI-Y1c%rN|J-M?aLX=>~N
zYkoLZ9v)a^{!#i=MG3JzK7^gVq@S~l4hP-~N|(c7m)@)9VXqQ$l2nEDfn8!}48=Q$
zdY<2{rAJ)i$?N(YY96Dp5PYOV>@W*?Xhg<4Ydsbm$-0YmiD820#{@kK4jZ-dG1`%>
zG^}F=0r7VT=Zk_C+oH*bXkNnWMk^brOdo}dj4cp53S;a<^AbRE!2-!ec5X_i0+vlH
zyxc1U@iKhJY?w_U<~xSTwe*`0Ax;mc>HdvXjsH5yT;5rJpblNJ`ry-$=2zC3+0w5u
zdGcAQDEt3F+7^Y}t_{GL24dmb?3a7xB?upA4v1*eHod_XSlkly2Vm}agcCr-U$|nC
zQ(HVHsQtI_qx2eyi!DGCdSlh%cgH-C-|q+G9gx1{vj98ao7l}i&pS)S5bxX7U3?sl
z-@f{Xm0Le(@2*I>h1X{laF1hUslNej{xM5jxvFfeLQ{U{d8y)Kny}@*f5Yi6NvjIX
zArg;{7;PgnIwC+iIs%O}611v-;)Ae1i_5dd3KY{F<EyH+3r3#|I8;H@#W7bfX=YH*
zDQ*W6NNd4!t_m7$x4aUNW;51VUxBjB@B;Jj84!`!RZs$PU9<+P^R=jR-^UEgfzM{M
zu&d)q21vQ}u_`{9n-(4I?n%c?Q%gJNeZ(!F84G2`Uoi^vP{jV2%uC2QokJu{Xbn=7
zq4qnXiqAQnKR`2f8_TFiB~Kd!UkCLWPWJA|>({Jf801S>%6LRm!)z4s1S7!=nA=sh
zXx0q^Kxo#Dcs7b)h|sl1FjQig<jaxiyT?_*HGYGYFdn0SCHC7wU;Sa#CJT0C<znsl
z?rA_?iv6a7iILX`D@~7D79Q^Ge1@&!cAJX^(a4O8`Y>I<qR9NvMU~b%3kv}mGd|83
zHgO5WJS*WIR;w!IH2CTMM?JLGRXA^Npow5&;Pp?i9fyzEUrsSVT(!(Wzsp>%p^GoK
z7hY;+Vivg$`aJrGAn3@dUH23*N+2+OK%Ukcp!R|W8_G|*8<wcp71V!id`wvHO+llw
zWF+fS&?g<0B`<VseWrva_CS;Fpfk|K($5vD@pbk5PAmuVk}e~69T5CT`gC1&-i*06
z)SxaI9B|Ebp%sI!hs6w(8x{^TBEMhl=kfJSelr#|@V0TwP>Hfg@H%`J2F=fshy#Mp
z_g*EpM_<^GC_kU#gwx!xG{#4*0KrhJrTx0V$-ij$Q!<Bn*zYusO)Pi;g?js3|NL~8
z+s(f794)+h8z~r?2*FUp5shXO34IoPZ$u4E@2>*6(+awF7+yIj`81MZ%tM^2JI9!p
zz<*wFqwh#Z$EvSWoRL8BgHKEoA9;&T9eY&{*>_o?{EUv(sj<sH4Ijk7`@*2%&uA4K
zc;`49-{78pqYcUC8UYj`#W)-~24h_3Q1X)sozl0R$~?-3=zb$&Pb*XyeoR2}N1jfp
zYuA^2M)a0`^O2rJx)3Ym1LcQBB(GE)`)93*FlDO`Rq#QjAb&tTu#IEIdx2rvFrmr4
zK{TuL!)`TK#j)o~71^V|O>41SmNP%>C^2y(nQEjZvA?qUSC}RiCu*m9r)tQ+L(|0i
z-UUW<e@-#q0K=FT!musz(SA_4^@poYDI0E2E%u@ZhQH`L>P_Zl5&9E*=J+Gx8K@t&
zqZrX1Uq=2KoUO-a@yju6*D@qS^+@FP4?BESRDKZ*!g|wB_t|qNI78>c$60Y*aLdwp
zq$jmo`%XGcf;EGOTJwDxY|T}^vAy6=e+{Gm`UM`iq^FVxoR%T#)*eO;K*(ezpqa^n
z8o;VAr~evU+}z&_>cE0S&cS=H1xNoHOb_#Al3n}nGIsf(EYdrbIt(H<%Wjip6U?Mr
zcWp7j2yuaFueLM}NE2d-J1~b7%gkOCBZ-OyJ^hAI`R(;;_|h;FV*(#)-L<+k<jfU=
zHvde)ykIo6O*C;FIY4L7py7=))3)>OU|i3t2ijc|rDM+DZH(H0gGdD3vwKvyU9kmZ
zf2<GMH6~dqUI~zy@jB9W;jtMtInBC#2T&Q_Ufwe%`KA0&^I;Zo2Ap{+YEaI4SVME>
z_s|Fz_sUOJ{FY&=!TY^=_lKRdDq8jM!<vHrLn|J&b}9SLl77zjomwCAOw+G9yPWUL
z$>Pv7m-#Q{{M<|=z3p=9nVVJ>)J<yZa_Tg@7||3)HVJ3uDJP`&HW1q~8^ZlMm`k}9
zX&uF0dYlesk3(7nuZc}g=X1%L-8+O+fW;u-^GIuES1J1vM#_eUg(XNH9DugeVN8DO
zKM`%)THaeudbM_#^>cr&4~f&lXh`?=)#uQB9qu(YwpLR~M*Na!v*@4QmxDuZ$*E#%
zFxRJb#-<l5vG$WmDrY!FE(|3qzxTGL<|g$R;iCy+5(oNSu>kp~fSsbRJpu*G;vR0c
zwq1G!x0>nYtIv17$M5B_luz;>wKTTcU9t)a{g<VrMQUwfv{q{n$#G%McKl41a)Blx
zErQF<_H3guNxX>a00*se!YTM-JAKe!^YfzhE0yiyizp>Q-gWp~i32CRoPUA;m`x@g
z^wr~3Wz(ujwJnVi<OALvH^P>sS@K`7n$L3ohM#He?W@OEhaa8Z#Ema$_H8!tUHr&2
zQvP^V#u;|~+L^E+Y>)hvx=MdyyW{Qck)uQdx~3sm_4sV<DB@dQSUSJUqq}WGJfUMn
zwsttNvCCtbfxjWAzMP-Rd%`Btwc9q*9nK1lQ`|{y^x)MjUzP*Y$9rNQC;->~(*W{6
z5SD)u3|Q_zEfz}i;vp))N{feQ_nR1VxZU|4o6h7sVEdP}1Z5V7Ig={L{@&Ma{HE`3
z$ZZR4jv|$`FnZ1K%xC`Gmc|A$MFZw>z*`eADPAOtQTa3=%&+Yk&CtN34^05f$nOsn
zh8o>IbmQ}_Tb>D|3M$j3_t&1$&HP>tlaJ^37Syl(Ik&|)t(ithvv^w059Xz^iDc_;
zURwg2_sB=n-(xg7PD$nLOWPeTweZiGB=4td<kM?7&anYQ0jr^TBh!bP`rwXEHWT@8
z*kpb!m}-4{>gEu)-YIR?F24F)hp^_|>2Ih^A43=^3FasoeTO>8KONg&mT;rW71ZA&
z#$E{=zoi@laIPIb=?z_(?bT!}lf9o#A+hUd%i-r%vMAk5bfvVKx%JiXM#p(M_%eOR
z+eY8ca9>)j&K&uh;%4CmGoA*9FQO{vkhAI<6x(QF*6T>G(3Rljw$373t={{33em!!
z1`ajen?I#1u$;uJXan}ePIfwr*0@6tJX=HJ{hDiUf7kD~eH(1}0e+h)sh27^oq8ro
zkAI?I?oaMzoaPMLvb2ca>QHAp<c#=?;tUtfb%NRVV*J&j-Q`sNoOi1ra<jax)h40w
zY$YRpK%vA_hnoy;_za6uOX6^8ub<T>M!FRS>RezmkEd61*;Qs5%xAo9ESK>|;K>6q
zuhu&zx$D<fO@e_&|I7MkUig5QV7St1_oZV!-;g1kbDPun>}MwNYS$w4(=PIsW+%7t
zA99-W;tj#(+RQTG1qPX1Akx}pZa@BhOAZWo!+PFgm@{rfF9c+6N+BNX-QP(h_;<h2
zHl$je(r7yP3-1(*Qey1|E@hP~^wOheA;eTz`<G4~0;|%Q-Os9YgZ;&6_4^kD6W|xM
zmJ@w_^|&z*rbtD#iGS<<l<j%4#D`SCX{OT?9xS{xOE|hw(4~bCG_lo>)LEO?=M^Wo
zgu20o(x0>v4bK(TD24BF3_QfvJM#(>h+E)ch%IsHICf++mmO+7FU{Zk^lXDzHj_$r
zhvQkx<S2wXN0VkNp4()F7q&wn-HEOB+Z%=vd<B!R<?^Ql5i|pG9ucwg-nGl4-TONg
zLXpGl^5`+clQ_kstlCL`x8IyXBtw{|myoK#nFX@0tL}OPFCU4CnD#O3{u+hE<r-Nj
z(n7A_kltm&u&>J7x?j|EBTosg`=o5z8Rh|R$JswaY;+^R8AZfrJv+Eg=5VG%n*yzz
zlU1C?Zr{`A!5H9jr)C`-!LNDBIb;0nU>VU}fP<W$^UDhAzrXQUDA~j+)OsDioXtB;
zIGELl=A#Rta7HGmaqBY(WeV^epVuCrDgVL1hR{0=@~O)^s<D(JTF#Q%0lk}PlB(OI
z$yO8AtoP`xd{)dPh4YKp{xmXvIm-jhPZzdoS46sMAADpIoZ<I2!x?5Y8=sRetK>UY
zLE*}7L{QhOESS@KobBJy!l=~N*C6GS_j?=64*44b9ty|Z>Hu$u`xpOiBrf!TykbL!
z)!XY~*%gLef^YX(|8F-lD@A<W-jB@-yU<e0gX<M-?e3f0dpFwb_xwHjDDG)8sogQi
z;<ZEnZehuMKK0rC#-e-QSw1`K;K+W6(!~~gUKN`c%A(h{x6j#F7sYD6z-wnw9$Oy(
z*Q8|zvxvoh5k@FEU%rsznnFwSU(T!1?m1u*Hc#G+;NAmEmm$3OiE!`xJx7Es`j%~R
ztGd0&siUe_3GgOAj!jxlq<(TLwXut0MtTG2Gu;h9)D^;+q6i2bFl!JogFd6H(8TSx
z9c+GSJ5Ar<ro0_Nc}^)PFKxiMHgr+Y*Wp!+wf);*`n?E#Ot$uJ5)Bf$*7iD+{2MH#
z^zM>u&jLPVO0UUjYU}HN6pjW&YANb(`_mY3<%qp+sxN44l1BlnlTT-7?xAB)F>)hv
z2AxK<Hqx%F(%PrF*}(q*kGzc9da~VdnSWC6C6;mm|49p@J=tpf^^k4yzo-B?f&P;|
z=P3o}#SL{xgHnrC6^qg$w3X8<x}3TXn8f#d%Xrxmc!t>B<<Xjh(ulWv*Et4@Y#L40
z8-J>e0rS~~*X*Z0rvol>p;5#LUSIN6=>dmr2seE_q^r<+Gl_gvU*Es&nYEj*UU#!5
za|6GMaBIm}kwb#nQJrI-+}(w53h=bndplR&unl9E!}~iU_%-0LvuRyPlhaw;Y7)2W
znI4hT?rpO9H}Kd?z<jI^ERHKmm7*aq>9S>t><5UeBzb=-|6>s417rAx%Bluep^x6<
zt4?}F<yW)>)zXO6qLaESnn_{Yd^WEI{S3LE;`;;{aZeC0BicAj{&InnexOr2vRTSr
zpN727>5ex@i)h>s-}deJvX*~(FR|0Bc|G8jh!X;f&A*bC;lTx&Oo3UD#?k%72d9m*
z_hbxsp@!r(wU}z$Iy1GE)4Oj2q6V0lb+Y|+sB<{6tt+!ZAwH{kud>|l@R1mA%5nKq
z0pne6#Kkg~VZR%3Ixx5EWq2DAGTs(FO=ep6(rB}TC#-J6&2A<FevXQnmZXN>dla#+
zdK<H-EEi2*!|t#P8jJc|CU7;`*Ja??jCXxcf~kh9H&gq8Wy^o*lm5EBZo}poR6ak1
z)CoQYn1Kx>ejz-*As4S$$6ZsD%Kf?37d3RSehz&^=&)*~ZrZLEnnwBs?)Yp?|F#5^
z_{ML4e%Zp<=LrGA-Su+gV)5ZXdo&zQBsv_H1L;9+-v8vU6JVc1!QLYRW0Q^Zy3TmE
zS^M}>x=b!v6|cT|k45h=G4R*n#!q3cClwb0vm?&I1aUO@8~8o_-v<Q>`Fo>_BYKl_
zZR8%`w^b-@*5|!!HPNTuJo#c{Grzm@4PfQwph!pBHzF$rtW&;1^%}C`<IS9~OB%EK
zeW$?xuG8ed(>qRq^im0dh`<BlYmrHO&9{roTG~q3n#}e5w}ohUofMy{&yIWzUPi#y
zYE19<L$d$7o+J7p@AKr8(yAR()@JPKEr*Oxo5)KGCw8=ZZ^-{_S=z-7kle69Zz=y<
z+6iUJ>v#_frvD)SQy_Oo@1%-=^81mQ8pic>N=ZD4&v<#kDrdFYHUS&RZEoWi!F95}
zr{*^IJ89;3@3;L&!!oGs?uhi7_O4_791Td?MfUQ3k6oO}%IoP2Ea&{jYCg~17Hziw
zgi%>b&|~eT0w-gW{Eghf*d2je-CsJYrF62TXYijP88ig1B>A=lpf}9mkw#giea>im
z15R|9^;2GJK@4H_MK@p0n`&?_{3LM4niMsdBM13a-X9vM>EWOd^nI08-qg}r&%a$T
zeO13NMNo|&tN~H_G-e7sq>zvGJciO$_kEr0d!)(SAES$1PAqFc=}kJjyCj!xPzld#
zL2@NTGK-M$_K!$s6ksOdP=R`I`cv_$dwm$<ah?>^>#e@%*}JdX00`yurHbi{|G|qL
zA~9h3ilwW<00awJi)sB_9Ox7Wj?%-KxZV1bIXP{6Z!U*sz2CcI=4xi00L6e)aYm-n
z5;H2L`CGZ)LxW+y5T}X!_9Okxc57DK2)McK1z!~1);&=aTN1yN7uu~&pn9CV8iP2B
zM!?~0czE|)(G3DG14lzs3*?;9Bem4#P3?Cww}PunN5;jiuFs`(BjT64K07&w?B!hr
z$AgSk+n-MhCROeo*R)y03<$G=OMvZu>(@y7R~2wRc0;}8I%r>`{p?s{>-oT%dOqc!
zFNj&PI`sxrL&!i-bjGk2lqtx8U9TQOnGc!7RlcC{x%}OM+{8HNVCaC^z|d5n;kW?z
zdE>9S+jmAV`QE+PqKo&PXkR@=K;g7BwgamZ#ZPr5oS!7b-CiLBxD}XwID%gR4`!~i
zH!$zbVuk0BJ-jn#uJ-io_3ecERRM7?KJ<+!NwabQ)bH%;=;b}GPZ|P9TU#(drQg%D
z0}q?(`v6#A{<8L}24@3a4Y2K0tCLp)p|H|DBRjc(>0uz&LeSoheRA*_9D@IAj(<h%
z*Zs}(ez1)cqrjs^5{?hE4?*UkDgnpWW`n<FYd<emn^ePPzltYPO;29^dkUZ5(Cc;V
zb-jPF=NG^KxNp(Ctow%)vd&KXY>bi06^o07r_UX;m@;jr>K_kwPxx)YH#t+E{(2^-
z%lqXR6_tPo^U@C8Z}@NO<l7o6F8id$^t4a<pNZzQCF%jsew{G&pB=yWA4qL(&cC>g
zOpl(^_V`wHIIVtCOK49dzxHO0|G)oB`<fiBT+JxVAf5^=KAd~k{5!?uDUrvwMsYL(
zeaa)-OzO!bocPZn`s2~Nmu(NvhNq-!buGrH6Ml;FHm@rznnV9SaK2(x;*0I8w1Hd5
zUspO$@m}K=mG%>BCS~8ddSZPJJh1%2<D_@cDqA%xR#HxYy;g@=YP2%BnEV2#__xnO
zo(7|%JFpFB@$6#EX`otl)K>4hX@hU_8e1t|9e0TGWcu!9>)wC)RCqjG%~Y^_WBbnD
zA|LK$-i5I)8yRMyyvC-UiujbJ4^17a_g*8Ec?276KDSpj&?m2k(@8Uw{D%9Ducs|K
z-Q3y*&&OWe9a901m2EXy>zHhMck(hfcp&oH8#QluvHi^M{N1jv9LWzahi|<}cr)Yl
zKJz$&VpLMZ3roCu!9QxrmTK!9Vu<-vKb=|=dz^6|bt!i&<75!Iz>?T&tOqYI)NPNQ
z7)TGz`)SUWpII)=9g~y{sif9Xk{{wUE9~5jeSOYiFD3CwV(t}qwq9#-aW3kL(LK2Y
z-mT|M980~rIa5db#?FHM@a%LV<7!jo%d>|IQyN=$nClx{^mX+#;9cUXj(=5?THi)%
zj??6IL}ct+>=tKwG{Z4_&$p*<B)O(kTdE!zE%^oT(%1PL5@Wb%<a%nI`<)ja4yPTi
ztWs7eFy9+@1wG_$KhtrtLV4$R<Gfd2J8EmS`Ek^)pwTvalUFlfEXw$p>d`ji!^akw
z*VVqxU_=x+=aO@AP3t|mZ!3Jb@e!<jY=+b()v_xtv5zO%^Pft8McD?8#aqtJ4}&-C
z3+$7Yg;dTySGVUzW6_-Ap-ghJ!owF%CxtdaQ+TL0e>@m)L6CJR)rY6m)24c#S|6wF
zzXE<rY{V)@cAoTin4{lOr0~3FmWJdsUU{_sbFNcj7zMLKPuvJSQ|c7EU(2lq-e+0+
zY;ik!8u8iHXE0FP@XwXnH@3PyT|1}b&OX<v>l&Bbpb(Zs*>w0$y;YIF!`y<~+Z2}8
z62d7^U9mVdrtL@e>t0$_UXrc5<b%W%!3Qx|s`r(G<TsS9*U&5Qx70`{{4c6tpSv2o
zwqBUGnI8Ld+@Fe;_;n69?jE*+r+2TNyR!{{kRH9DX2$6`p<b(JM4Zu5{Nn~=rn_=>
zmhP=A$8d#$(wf>XGL_vCz<oUK@|tGq7FJVT1^Fq<F}r5FmdG%RAzW1dc_oQcmshir
ze@E_{5DJ<s%S~K=S{Gfiw@dOnlKxcOCjA{Z&b9wbc$;Kmle>1#Haz%>#ab5l&$iIl
z*TDm|_P<b7K6gmgJzuoOB#WQov#|Rd_wKmMdl@I&65y(8rY$Y^sCizv#`RufZphXB
zg@%gkd&%P;NiT@!DdXpNEW+7a1cssJFELNYE34q8XMuUbJ=;5zD)d!tC)U8=t*e{Y
z=S&v)DxtGj8)+X4&O)i2quP6lw{`4-&ltwpObW%nOTj5LWy;M1iJS4x`fk&2Y)8)~
z{1VAqE&U)V7ni$5Q9QkfVsBymc&jn;7A2FH?G$^9W3f@Y^K%47fi9>koJJuS1%92Q
z-VJClin3YkM1f=2K%)5gw2tAP8@FR)YIZd_-@vO&zwNTcW9cD`x9jULAKTic4(ALU
zCmWZ0Xa$2U!0R%bZ@?90?bu6MO%+^GS9(?69n+rFrp6#zt;tSYz>C;u<~=xoDn01i
zLKew#40QVTw8Jkf7fZqYJSArf%L^JQ_sEq;&TdF3d>a(G^Je6=B#W4RUrzZKo_E}Q
zU_2G(pKn=YpFn6$a<G9bA+ZGoi7kZC=Z8fl<}<zGHR<GR+}K!(cUQAVu@Wi!nQOA2
zzS-Q&mZVjBC!Q?6113rB_$`pTNKtYvT=c!#*Cp4t`X8PC>#}F28QS8j`LCASf8whN
zGg#5p{Er?7kDL$i6_$fB@-I^^_5Q)r9qaN<YFKc(F}MeIIo^w&C}RhBLFf$PeJ*DS
zzESMv5Az$W=?2uUhrD@XT{fpCYx>8-)zo|v;OtooF(aF<$oOea@kZpL9<9Jj55d^0
zcU}BoKfwKqOc)h@esu)A^ltej^3;_mdfAs$63&)7ZFY9uw=QFEnTMBd>}g)N4_BGH
zdfcmT-OZHA<q)&}jR3(yqjdda*%em0YHSld$q#_YIZD^Ba(;8Pz&kDOm3V3d9zx;k
zZjat5_D*n~3m|IFQ-q*#{)|>=z>UV49q^=uz&jZisMS?gcT_Jpbh5~j=-XxG>;zd@
zl<)%>O~0`&KojEjRZ^1F89=nWTsy(jt5?Pfqpw*x#)1jdz%%!<-uXYVcZQ2ta%a2r
zB38gfgnrYXOtR81*BjQ>3w{B+oWkZ(W-0N1)%bm_6uT@#zELh(&J4T!oy-pi*m+a>
z0|pBZk{_TOg?@m*A<nXX0f6B(Z{pb8;HJ9Xkfz5OG0Q(=@0CKA4d4o>&DHx#iN%sH
zfJ;6ZnCq61!%{YW<aR#xFuZ8H#XzfLufu6@iGxP$s-q%za7AZ)eD!pQ*hU5BwY9H<
zI1^M>oJ`G*G--^}^ly9yFM@hX0#xm(EU>|^N4rXX!2KyzI=YS?(yt&m1n>phznqdP
z_61K&*CJmaiVHvE!7ne-GQ$ge!F&`K3^YG?NI?!UokgWxYJNe!k2_xo&d@fsK9EEs
zY|qH7NV3&E@x*N}%Z27s-^=Ns=}Lb><Li~W)P|AcH;wfUk+WFZ_2vFu)Sfow6>YFO
zv60HV=9)4#NC=r~C!$NZr=820r6&0SfD>k)H!P}Z2@*LY#S=F!4GSlT3_*YsL?I!J
z$szS}dkU0s)TW+ZWz|53TO#PX!M$%Nc#^RP`RVxr=>EQFO%Mu}crWBRFzCwU0x}hM
z@JF^=j}}}2k{|w?g4#(!%v$grupqa#@t4a50%HfN*!VU?64pd5Gan9M{TF6$CekH<
zHGhw}J~xBS)8yN6AAfo1vN#er<0lr>ijj#u;_3`8J>x{H%AWiR^PCTWj*JURYil!U
zO*Bi~JtKB!ceHzPHK`}-`dQeNAaS_s6r5q89eiv2Qd#4-tU7+j6a~U#0|&w<5D05`
zi8F_P34qY{nkNyyLYadF${f56zbbOEhdkp_C}~@<Urjx`dOoh)nEm>}O^PAECd5?-
zvNAwMMw{=Nt3VhkRG7>?IZkp05DI6<W@7u`d7=ak2)P&QyoCr6+Xq{X`y<f;5}2Eh
z$qnTf`~!F|#@yF(H&d#H94<XYXF*iOWMLRGn+sQv^IypwQmC9SzLr>2#hobjLNm5Z
z(Mn{P@3N>l1`p7mIqy(|9#=iTT3e90WI_H2J+U{o7W^yBT*^h63yR#9_eu;d`tK#Z
z`Q%k7x_~fs(EgwZ3l!!P%G%1+v^dc!lKsQ&L;nkq=??=V)15Il5JDa-1|e@*R~eti
z?nvmFpp4V6NqZCGTxJ>O?}*12xQOsjOd>+)-v88*f0g_INTC5wx(x#n?sk3cSp4v8
z_*RJ(2=YgP6+r$7F#0B`+(4E8h<9(yeT~#vN<3=Q<8IBP8MZei763rx(r*OG;;-Ub
zUCh56`R*Y<Fs~19kJiF{QKJ!(M)LII<*Bo5Z&JjT2c^MZ!nqUV>~4AEiw}VUuPrXh
z<mVb(^5^jGj9D=aEH2DHp6_RSXdU1K0roMYL%z%%_q@D@Y%SS3qAvDEmknZd^%IYZ
z^5yByl*t2I-7n_026ww!**q;@Chyjm6*i2H45cicg@r$h(ty~o3<H`h@-p-Kp>Zqr
z-ctcjtT}M;dog7&=qFcO>kt=|UIO3zGVkh``=i{f@AmFyoDAe!;^7L3tR70|ww4z`
z@8152Wb=?CZ`SW~>+Nrt2{1VN1p*8%io31CeC%hGk4fS#%Eu0d(6vIZEq>XN3D9dE
zPVu^hNxBz&`NevNAG`5kbrf@V;+uA9E{*7<&6g|=xKAf>OW%k~Ek*Y|SA5?^!1K3c
zzb{b!QHjjn{;n8!?pc@NQ04)=+1;7xw=GCd^y+BaaQI(5{aQ-k`!nzAY>eW}74lXy
zSuPO<-fz!wPkt%+1>p47HNHq)?k~RdD;E|PY>rPtS=g{R8w<Qz!CQ3a);ipp#R_xN
zoI+~mof>ohxLdIjQ6=z+Td#<*TBzu`NZ#eIOx#j8a<BnkJ~o;~Dx(^7ALpumd1!iF
zky4twr#p{Qx#L`@8Qw`B!CojKuCycmLvoVywKp%W87WH?<ob`tT3dUq&exUUowMfQ
znpUz8>NdsE((AJJoD1C|UEWo5Dd!gN#u&-bLjruu;|p=j1<XJ7T%G-}*cl;VC}!-|
zu7cIwdIKrR#>~4!n^7(%sr7lvFv?^~k_hkwXHe5Xxhh#1u%>I`6$#+BpD0efha~Dy
zxb+Ybb>2GTikyJXe>|V}#)lY2tD%|U$`YEq+Z|X(&M8WFUY?$8#JAx;lDl)isP7k_
z{QKjr%qw?{1ms8U7l*!Z(=_0p0>Dp$?yu8YLW`dd-i~=%*2{;s{#Uw@A~N?n-J{}+
zdq|Xx*cZzBIswXtj63A&VtS@?eNlmPIXMn@{RiNn0zHbW`LD7rpL-$#df~1q^p2Vb
zc5K>Q>Crd7zLGdi0`sLuFUxwIlCmBsxLbGM)9nX#(3M;5kmOm;ITzfMFbQ|M7I?`M
zgndy9Mb6IM7ErDYemza0k{8Ldm$SR1C3gVlr~}Go43_eTXM)oyRf8-;8H@eNnQpJe
zF6cQq2{*~RIia57Zlb{}Avi^GMKL~yP{vp&OTqoDv0Z)h$XO?-!j9MLrZPHu`K9Dr
zxa(Wlof(=XmSldL`e?gSswmI~XEf%PDAkdFmYhAp7Y?gxhru%lHvYwcX};^!*osS`
zd{cH_(9fi}PcH9=FlVhE_}q;R!_cp&udZ6Cm~7$f_ez0nGDosH`1*>(X(L59-a4Pb
zrJEYtaVePif_xLFa;T}Nl2{W*W2WhNL{Xqvg(wcNJO)LrPl_F1SUyOI7X!;77!G^<
zdh5>e>O9Xnlb><pegKAN)}y>dj5z6`7h6QqYROw9`$p(u-{Rr$d_$l*4Pyt&EM@+6
z;Fyl_){VJ;$j!PfORp*Jc{(AKT-zau@lO1>aq?W}%e{?*eDq;*JFxR+_m;Um6DHwC
zx0mZ1)`E|JlZ88P=qjRFRRiZ1L@F4akVKe6l*g;~E%q1$&o28@OnVQ?tr3Ew*gY1E
zg4*N;ldZUMTiIQ`(b>~<8S_Gur=%X=3SMrEKBtVQDtE`TOa~x0m5mF`?c=u078RpH
z#l<MN;xff3Ab71<9yu9%yz?@v+120S#@^!GRPv=jE*-{HSyiwbcC8X}bs_A!tVF4i
zyh-+h2kc99mogdPW2JPCJf9@b_V=N4VB1ZPyV#xx%5lBEIP_cnrC|Rb09M622mf7b
zE21(0R^j9<eYO%LX?-{Z@5IC>Ny97tlvn1SJzSj9n7`osC3aY9eGsk5WDD-X5!iTg
zxr))2aA6jI(lsT+ZE=qcx9xMu&H+q8c}rYx0b2dox{hI0eMB6Ru<8Rmf{m_0b8S~~
zyWMxV4L^ByxRir)Yg8X?uca(qeAiSdi<=U&{Hf_GaX=N-9*4sp$Q8VTz6K~0xu@r#
zH3L}76{Ittyn1wxoy9*vI92p3v@#1nxBfmeCEiJ1?WdA$1CV*{!5le|$%)+KU82A_
zIT1JOI;W;P+O0Tp7Agct0c4kkL6-?9dV*wW$XQp={V>AmJ0Tf}r5ZmJ<nXq$J8SgW
zv*6ITLOlPHwa0v})W`^y>T(6XusaMh@rl1bgaMVHW2Y_W0POjePkSvh3Q7lotb?UD
zq+fcvM`s4dPHxW|>3(kF`z3`C@4V+#1xCuwozb&#YPpsM7qUhrZkaE^O~wrk2RO6j
zR{e#3`+bp<xNZf=DfEhcLJ(x<)Gj5?mWj9!h+QO3$k&NmM3^m39e9zo8N25c)EKwZ
zW=K9Y#Lovww#b`K6IPeSpmkv6gk3Kp9F9^)RbA5IPcBbm(5uOXmShqm1QIc+Kh18A
z%?oQFua&Hhu!CcE-5_BEN|=H=$za%_uKTT7#<gj>*Lbw1$FCJS?cnGOEJOS)@LTn-
zNw9+_v2QIIWCuB%@5I6aF*%r&J55~84%oMN@kuK|EqUNV6|4))*jm!>pIlxEsWBNr
zrULcIz0u+#T~<X?ys?u1R{mBwtPT~tVSu(l%Vf%t){SS3a@+svnZ@YHRH`xAh|@)+
z+}jYFM2JM<Fvg}wCTKE<Cnd)Yw7f!@V#&yHi`S>?$=9Qm`F<q-PcGXq3cKyximR=C
zY<D>RCjkykyhZY8)m4A%NH<;`U=1bw-1b!7CiYJ65;7ULXFGdmdju=D^s69YX)uSd
zea1yUC;ks|hrVC`hmo!gfn<~}HI4a#d<pmJB(Pwnzz7gibpZWKY|U&_q-Fu}xa@&v
zm#rBLtql{iEccHpW-C#`Lc_VWpX031EoX&1NSr1`s88f_gH0N4bFsg3F3kNfJwF@@
zsyKzw+I6lxKKi6~Eo}qtt+QL0F<A|?I~#$160!hukXx(%TC!gVAlEG(%Y^JxC}hXP
z{tV@KOZ<8{S%8;F7664ekOj;Y72X1~bnjgX^IN=KjaX0*Vx4ft$;V0!u>3z)z`&>I
zfhb>F=v9G#aqp~o5Y|XGlB|#Xiqo|Zz`wxijy-<d$DiYzY9JFlMgX;rBxc7X_euXj
zG}|{`izzQr*?F*1lNMr<!IdkKYfabt?<DR2<dAx5eRJD*N=zH^8<^NGC7j&ZH4SY&
z(Q*f#LQm_v_;55S@~lTtq+RMR(oN2n)y%(%OGmAEVW7%QLB(?ErqdndIS)u?`yTXg
z^dG_xEzy4%)7<Lk`|l??eR9a8hDr-^rh^;(QyutGQGo@J;cvP<uH1yrBwrbTA+VP+
zR`#C2F!RA8M%rU{_QBrf_Uf6qgR#JO*gbMh;)volSXf4FY#ynI_?KjDRM{-|s1*}!
z?<u}P<=ul(6!(_4o-Gq__u|=xq;v~_;^EHY?l9~mXRdB)CM!zQe`4$G+4upLhSYX(
zcMPMW%cnDXCNAYP>{~et)HSmucqOAQFF=z?43yos0<$S!oCrj3LPRjCrBs;@!L625
zt~gujxY<)w)OT%I=h;--W!7m2D<?L!w9`Eyba>j{YJ4d|heE35)VfLTop-qh<UV_Q
zW75dAKAn4$b={&YL1j|wR!TmHd$j?CXiPWTf8ZAgwH>?gPG>F2e*XN4azVtOvVIsr
zWC)Q@&MLdoYsjUVti!q3F$!xARPLeQmU29Y^f|xtzLQG_c%A;ksHBJ#PS~mz1I>=T
zDZ2Zk3{ty#d5=EDc@_zTHHQ3Fb*LaqA;W0r_6a@FUr9bYICNZDed@?HHHsJP!!Eg-
zL^H1ubw=Ia0O6D5gm=;3DarQrf9pdtyfWN0jME+zCqc;mZBb)EK}&oCZ7uGfgr+>G
zpqf1ATj2o@NLLPt)98J~Q<XdOPs2yLX{t&a7@AX*AE%gMtZmBm`W)Ap5A_mP{DHZa
zq5@7<#fE5rh@1JJ${AV!y=fTGcqj=GP=}mIu01SUWU!Q9;hGyObK0?eSRI85s}I2H
zc)eUVBUOWcig)KzoKL-mUr%!Fr3=K+45t}V1F!9<X_&<}nJTd{NJWsE1C!Wsa|I31
zn_+?^-MvwpxgFrlDD3DNxY>ULVBnXjc+A}-J?(@(LCgj%JW{Dn@^V-ha-bCA+)*oZ
z7{<@j)*{g(ka$Si8=P^`#g0rTO2y46hE!aVS@*pZIvu``Vk+{lpZmh2C4iIg4Q^tg
z<wgD#LVMC$N&L|7@L$7kmoR9!vQSsTlyp$Wz1M;$XzhvGZ|Qr-g4k8W-S#=|XkFZM
z(q6m|MT(6arpclNhDu<8t607L&J0RQV<pi4?s@Je^Zj>`Hh*%cQmFr-@x#<XgC^3C
zLzH%O>v;7gCt68jQm;IMGinctbw|UAQlbi=BueG3t4gg=Cz+yU18QKKBs{o($;mp*
zlHI1IDs0pIeTgFBHancND&kT-Y3V--m%Ma=f0Y29UqYG0#l}7N-^*|4^EuNB-=DI;
z>u$Mg2y#{{6YaF?#95>6KM<CE9(OZCrn``@OA20z#gPg8d!$au0ppxzp)_-Z>R)Ar
zXItaUvOvG*6^WK#3C@{hhW~x7Xw$p!anv|tSqMb=YqV3~yb%hEGFL7-?cjrNs+^k|
zfAH=4o&(?D@LK_Wb?Y5|wOROy_k+OcP9C%*ESGHg5dE*|PJwe^Z8v1m7_>p8lr{hf
zx<ua!5_GA)6|LQvwW3VB&SW*t#RXDl&t~}4+O89u3PovTstyQ@b!SFRXozN;NM+ym
z8y`(eUNvc?QkJvlo_Ihli%RC!Y~gHyo&N|`hqaZn%d({jG~HksXCuE%#?m$jBIXcE
zmO&8+;;NvHzy=T6-g+nQcg*X=8KdrR0#;J{NQMhdviJoE#=*S1QZ-k~MZz0Xp{2<y
zIptJDrSpJ!H&+=*S{-D&dyWM@upZ%#^(3QD4n@|+Y4|n+3tbPM;`Kts=xW&WIqm_e
zj@CtA5HW|?^HA!Y?5A6k7K~&6ORJ^*gmc*nlz|yIi`BdE8#J5qN&<VLrL^-at~<?;
z<&(83dGnAW?jedh_(fYv&?n{iFgj`|ozdUo2sdG`%Ibjyk9XoXL~x;WE<NFcBt}Ax
zJst&0)Y&kIfPDzGC8!mLKBSgLa8*CYg~n0535yk@PyR=^kmTdmd8N;qlo-tk*^ExD
z0g1TpCW@Lm233p;3)BxCJKHPm*mkhf)TX({d=HQ;6gx?%uqXX?ObOS#r_+?=ELLiW
z8yF;w0~@XoBtn%z#Nq;@9yh<kt^Eetbd%sZwH~2gQA35xXp<L_2}&I_sXF>T%>VOa
znIYw~SeGdy5`Pzka57<c)W3gDODQv^2;A7BZ*XNnfPQ8%@UHqh1uBj9RE>+=3%ql3
z-<lG`oCG<n3<27=v#L2uq){Kdpaftbb6SeYi}-pIux;m5<7#pdV344QRZfA>$McwQ
zp*&umDRFDI^6w70l58o~#|tE>?@>-dz+jT`E30B9!P@cAxMHUVR2vBM!M>Ag(bbwI
zceR?Wcp>VLKTFIFHI%(4B$=e}%8DtdAHu%T+SXnN09^}?+>I`m$K+9cx;>r7r7^CF
zwdn7uO9=|tJ;zLi<dY7i>U?q-?p3*a*mIJbhR0lJ+&bw>VBE-4;K**dBbD#i{Vdf}
zn_m)=mrS254jf2{{ho;E3zl<o?c)q<Ci$3PePi-@xWJBrPu)~bt0zkZfQO3fi`w`;
z0?>q9=m^9iiqQLI)79AN=TG|nlS4kme)+TfkO(;vu|NWVZJ*3zawkY#_gim90<G(k
zYi}c0XlNs#D*y-{jdb?h(|VaWZPbcM@UfZR2Lub>4bHGnmi^et1+iwzuDB+5q5B>I
zHv~89+Jj_$Xj}vya6M<jH@K}=C>q>quZG4rXhDJq5@&gl8_sPTD}g6y&sBC=U#FOC
z9e#gW$7$MHQ5Q}>l%f?1M$0S)Q@gANU7KDSzLd4ldZY(Zd*#FTufB-P3+$;LFIoRp
zN2({YtHSlOrU{b*MWTNUx$P#2-1g35ko?ZaYP(QIL6gwsk)5U4Kr_Z|3~8cS`_jR)
zIWzHo6i4_GSrvDj<)|@>y7Fy)RtPzYnkBKyS;J|6LO%7{?PN=DLNi(#QDZsIsOLi<
z449J_DW|$?bZbs)v3{fj5PpcBBazg4V393E?EGB1A{6q7sRHSl=BO1vL-$oWLFqm<
zPPlquM*#mOsZ?_2IIDOckM1i3=*Ji^j6omXa8$+uoYJ0jfECTH{S0@r0F;d-UY|Bi
zSbX9MV3iqy@NF*>MyO3O%&saxt*jwWd>B|IA39+SdOL(m`(*|+p9_4oS@4~=rY1l4
z_vhb{V$pk6>?ZZ}`HG->9}qd@1_?@dOM(U$*a8+lGq$|5AyS!`NSrcig}ab;&#opJ
zC;^H=${)g%Z)Gz*Eq90`N$MFcSruvySa5WBjYR?rU(jUwcQlJuV*++|4eTte9RG(;
zqTov}+Zy)Ve<lBx{I2WP24^IeVKq6v3s!j<yS2E06odAA8oU|*%^LuU0+m(ts?M5*
zRonzQr(%}g0uy7*9pyJWNEek%F((^}Ia!U91-4aZ8s|=0^2wpS?qSB((6e<(7^(xC
z_O^e?v6J^B+UZXT+k;U0K0~H$2!u8=jlRZmjlObiL#}j`Wz7vuJzf8}OR3#%I_|nD
zYK{rXsJ)Cb^`0O5?2!^qDkW@p5c37^+@~a3a9rx7<_hg5nh|beY|~r~-)#7(4zTD{
zShPj1Jhy?}R7IxAw&lD71`_vEBMfvpt4uNXWDu<pEt&}Hl@cx-wZm`7enwa%aT6~3
zfbYY~#NhsYt0m}6$K9QB>u~^Y->0~r>ov%HMV8B)^KeHf#&+pfN%@i$kz-=F=CqwX
zyCFtQH@$1~6(o!(^4mXZ+~H@*e$%cl_>ku8gY6Fptuq==0OT<adv5rGXfIthzSw}$
ztFD**$jw~*F^2<n`j#kA%%AS0u&?(HQFYXcj#OhMLw!ZR)t=7?7n-ig2LSixxJz#~
z>0C658DY2|mAp!iXTW_7&bQb=&ESZwb-pF!4?eXGwgx@-l{-wNja~QaRgdUbnT`z9
ztYJAC*P6A@g3DnA9p-6U@VzIR;83Sf{Xyv}A6yrH<#{vq{rjkpp`wlnW<ij4gNYFi
z`kD)CNNGA<GZ;w%c?yCcR%il3fOQekPUdskO>Npb60!wpm&YYfiHBxl(X!g*D%=xJ
zfS0FG8(mYBxvjjVGEvQ7v}^QBEC(x!qQhZ0T<<g<^sCBHLPx9eM^j4xb4qsl)nbUE
z#@eBM`Ze^&R&IzLq*Hk~sNvxb`X-;Q_|!1Nu25t?W3Tuu@2PUdt?T&#{7zmF%&D%g
zYcOl4N&D08*FzZd_tJn~1H@WlP+7wdpY+gjW;5+a+yWIwVE}oe@%^i((BUrlg1uL8
zJ7qs-v4IaOb~*H!Ct`98p0>fUNobtKpuH&qZsQlaU%PV8fv<7nj64_jJkT=|FZ)?C
zqp*a;kh7v<$ib9bU!6T%FvzDJ4v|B6;bF~v&fESeE+Eg+psSrQb|w|MKk9<9%AEW0
z4P>d$h5YA|C9fnRz*cPo%<dQvjt*zDKhe1z?Ug~I?fSJ$0UDg7YW(18HTuA7SGkZc
z$G|Kq$-Oy5uf1fM=5@erUB({yuq~v_-UXxNdN82}X*KsQ7}hn48kegcxg);~IJ25R
zzfLJ;SMa`*xjxjUk}A#grOTsw<S*?Nc&jc}xA-0SoZy6?^FW184#L3*f|xw(G@WrX
zVYAeUDCduFA}^F~wz$6hZtrw2=*ki@SMMk|k-;dsPfb^wz#I<J{7(+8tc^1!8KPMg
zuve?2qg|TAWg8znQtiT>%kfjUP{ncPeL+Jo$ZKKHgutkvJjlOI_$F$}B5?fQg+kV$
z#5R$;wzT=A>StdsEJUL&-n+L5SpSDpEgTJ0NnxU8!wm6(4_{biRk&$qRwT2V^MZI0
zeZDLshj}XdzpZ8?7(|204G&c;Li`FF=pEJn!>K@n$pg}q?1wK*(aMe?X^f;W^stxI
zq?}NK!>2Aaib@?}ePz<&mwOv(GmAayO}dRl=B(kjK=JU6i|7k`?|R!9=jS`MsYStm
zJ^z#(@NQ^TW}*gm@uwaRq`;^reX(j^M<<aT4I94z8#f7#wAnkI<&qk1r3w`^O_(I0
z^A-EPY&)Mm16azZ89P(?BMkOOr|(<LKFz%WmpGc@{g(WwV!=g!bmkqaoGI;fsC#u8
zCy~2F9{F4IUM$}b#c_Q9@h*Y_58=M459neA+inTSA!tMu&<Gbo%S)OZQR_B8iS$Hr
zn*lglSo6A;Lz|t$NS94BC7EQAE)yq>y8p_}Ov%t7G_dv#MNnLU{-3&i>{i$}QBI_|
zf!)x0z*+HBt-rhjlPE)nTjJ3Qjvs#|T3NHn1R^5<Ra6rGs-Si^fPm)Avvj9xP<Uo{
zJ#DU#?EF4(3G;~hVX*dKF?k48s(=h17e9UmB(3Al{+5yj5Hgbe>wk{P`LoMn<Zf1s
zg?2hgV@^TK09zG6fZ;3`95?<(G|Re>d=$6CCAGdb9q~WjC^Y%#R3=&2f8|6;*68)@
zV_n7fTBvQmoC~FoFeWcZS#sx+Rc}6TD@5f2G*GD$vJU75U*>H`3D0`9IZSp`Px1>Y
zrI+;j!NMu=gTSt#2E+(B|9y@}K-b?byi}-F^k~4Z%aAXrG0=3T-`4$<{g-!rgI>G5
z11Hrg_{J<Zk7C`ygh0xIIYGlIvy6)!pVXqj*#Zz~Ge96aJ)=xFHFW6ubgJ$r>>htx
zez2?;1KJcKr5{W44Qgc^;pXjkI^G9Zpeoq>4S)ra>KVVPA&=7j+(R1(D{DD8f}l`H
z1Z&r_eOORvnCcEewd1$M2kf`}Lh@g@9mfC{Fey6oqii7GHExFM+(mcnS<hW0<HUHW
zo~9;k&_CG+`5`UpwY%@Yn%Hk|{1j$Uv~bUX3q>1_RUd7u<|N*5I%A0}OVsF2o6o$r
z^w+2i;a7ftNV4FT#~h#p%XsU+x?fm^@ajU2>x^H7qQ-L%^a}MArQSC@RcXLPwLHBv
zT{N*x9I4-<0ooNZ;4eVR{$c|1BNm<pJ?col_XOp?35=s4zG!GE3FO#G+wM{Eaz4@b
zz|(cRkUx3PX??;X$5#DBpw$<+Kyoxy5Fdl@FsQnnNo!?|ic$kcRT%+Qkzz3xdUyG$
zAU-XK@;K_}*1L+cNOYHOsw32n!Xs5>55>p6#<93uj=ovRZU`Zt34C^kBtGN!7|SO>
zL&r)VFX3$xS(OI&Gy$+P0)Msp*uVxianJPqwmCHF(tS}}@fp8o0qeBUiw!B!%EUP0
zgi-gWx|_j_#;mtz>mV$n@0&B-nX^;29|~}N>!P;%=RBN87?T%V_NWu26XI*+te2I&
z)e{}{#YQWQgY*QY-l9HR)_1;8b$n4{(MT&u%VYn_BF2h0Ogad29Na~oqcX+fFWEwl
zXD&6N%gL)|o##k5ta|~5IL-IpK$;0DMO~Z`zdB^p!omoYGbS*CGMuh?zoqxcUn4<;
zfJ7=ibt&j{69=3#D4Cpsz|357IGVc%6XjskAew$@D;PhC?vsVe28U?=O|<b>ZpF-M
zbq}*hlDq96>)eqJ#LFHHfh%ds+?4z~)x9%QHcAV4!*rhmZ`XNZa^0)6Wd^v!!Yng4
z%!zWyaYNI=VehWBJrm%HZ2&SaB754xLd34aF{}-6y5_S_*4`pv`^9B=nC3Q8Y%)G-
zaS;q!EFvF4*d+0{l$B5(1_*`&2&R*~s=ccrqu-9TDNDVh=-3p6Fg_0<@eM*^6P2|N
zw*eK=g)7$wHGic5^E*+%unLZEG*#QLzdb!t7Wq0wL#k5|kzwb}AFPXf2-T+`N?y5R
z>f_x~!TSM0BEQ!Slt-X$I4(Tx91t<px13{UNWj4%N^Lo{x7!Q1o}4m5TwwEUY}F;h
zUIzCsL8f8IQL0>Uu8D}in6^=|$K3HLQ|>ACD)l*XeP`d58Y{OrVR02A8yk+iN9896
zYBY<OCp__(1TXxo0<HjY??avBUjrKR?36`gfVi#vO*OtRz?{@K%tmHch<MQ3=w0qk
z`rjvq9>Cmy-eUwvUvh*g&NY}2*6~s{sWWC!$Sw9><JL!@e6nwebU2MUFiW)(N>`Hu
z^V;u4E0bc0UkfuFe_#A}1ze$l=axEMgCU-sT@9WcefB(`=L~rze_0rG{mTD@!Fq!v
z<6H$pk)St7@Dq&k+V}GR;pEQp;|d^3z9q41o9p#;*@94_yvo5-dwkNAbS8tiOOol}
zk`YU9?G!3mUpgA@U28mwf7?@x6PydsHwU6f!Fpo$6Sp#y&@ZFME-&)v)sSs7^~g1#
z`1&4DS?{{fanH#3ldsP2H_2uSj{Y)e*qAB`Id1m^M1syg9_;*$K-u~9M9bi#PNc;6
zMdK%jyKu(@`rG#`h}b~yT}m{}!87>*w_rBtyVh31XcVLXS!+i*ZCu(EpKYfp$B@wl
ziX$`=Y=p33613j+T-tt1;KtWL*0D&>D97!BaND<F+bif)_A9phC3Cm->)z6~(Ogi^
z8<F_LE_FTe7BcqBZf23udWUn|x`bbvO33w~85nZ><Q$6+5=6YYeW%F${Tqq}g+xVi
z?Gj6CUn_LMffqC))n^?%rwBouu~vI*p~u{igu+}_f;?5%=-%v~6<q~G#zwk&@TFQs
z1IROxOO50lRbZw<`ca^c#$yX1>K1w|FqdncFLHUZ#f%hclS6q|5<@?nud5WDm^(uJ
zWDh8yA-R1P&^W<Oc^S=<jGwxt!131ra8uhA6GP0E<=j0Bs0rS)!lEyFziVAgW)v-`
z&HqI`TCv1Jav*NU0vN=jbOg=iMj8|W-a(*oXN;C??&So`EUDqmLGv%=eVb(9n^b{<
zMpX|m@C~`wY?wfjmP%9_b$<tRw~{pjpKdi2(F)lw=QNDA&X?aYIJFMF?1Bx{L4xRT
z3Obd!=RE4k9<<%eC}iwY!Pu=2+Wxn$3KD}z&m<q$Be6vqy<yw~(nQxq4=TK5zXK~8
zPmL5Yg<h}^ztnCp#c!{`+)<~g14>mypBiryBR6U=G}SH!?CZwGU$Ihqm{2Yd-3+73
zVWkQKrTu*IO+PA;c8HcXBE3NafKcfS^7~4hMHN6)1x~e$TXjnHB(e%x&z9a846qG@
zP>I!CQowVB*-phUc8rb8{0}InlR`3fmkRI;B9mZjJu5-4@A{_9D}dZgoD^?f+&scV
zn7{QVR3)pael0=B$8L@1v?()_C~AI;5SvAfWQ5?9ZS5(i4;LRU(5xd%hm9Vh0LS9r
z{^ok=jS&({jhTnetvNlm^3rzb>DTz=P#sKLO!LIU#}ScB%Tt$JO}iso>~D(m>C6g$
z14FQqu}CfGNEZ(3A5x)B|8vcm?22-S-s8OGpW+JZHHay}FoVcIbn6s~4;P~IpCzLf
zX#$2P{adA5<55_J(J119&+jzu4wV$=!}P;(Z0DW`7!7aI1hEz56g>}2!lv7mY6(e*
zt3W~|E%{HQw}Q7?O1Pv@nl2VrR~=8h^8V!~M*$D<<>`50AY<;#Xf6ae1M|mOdCtMB
z>{8*`F_a#g;?33ffRjJoXZ4b*_BSwvOB~wY<BbHAB^I~cUN~uP$b-!1d=|<ZK;99?
z^Tq2IFa#omeaZR%mHs2kl{dt3>S(>8;CJFFOQ8oIa+Z;ZdEKv^Kw9G5j!arTI{ers
zZG&D@I09x!fL*Ge=m<1^QTKo&$m4C3et?(jfwb3V;9(fn2-z4g0ReOBgY14U^rFxz
zU!8M;TPHwPS%n)SiA6*QDUPK;nU(ij1WduQhty>Ry53^2dxcl!(Sc@MFVKI$1Py)-
z0di1qeC+&^CKxYQ!IsS-{)uj4<Y+%(IV7SBHv&CR{r5A1K+{{(Q>mF=>{){j{u)@l
z_ED6D3Sr%0VDNuw+}U$Kyn`kqK>@#ON)A_exz}lgln_|`_;kj#WEyZYUn5CceO`E0
z%OWa-TEuZ`hu*xn5#H5pATO32{2_)LZGN=g2Uxs1VCNiDHDAgBmL?P3xiuh6(PpX;
z3pfI1DDyMOqE;rJtbd@{EWf_>VvYxxKFQ_4k=U#e7UxnoKCXO8-@tu>?p-0&otcS%
z@PgD?cHVxnNC&?~|A7XX`xj3}hxyWy-Bf1}7-%3B$&h1y(d0W_<o<3hZ%=ZtE(fG_
zc^POZmMg*-bKxm;l{j6Smbj&{r6ht|AO#)&AkXLgAJmBk3%$eOO2F-(h8B<Jo=Ru;
zdce2H4t@r9pGMU=6lHQj5cM)3w*k`fZnhoiUs(mLM=_O?rThk_dXBJa*7*IkHKUzp
zq+aq(7x=$g==n3j12=$x4^_ngK=x&i9`21kr{A#O{Qyj<8UbUYzH0XzCY-=w#*>ae
z5z4=r2MSo{e9Mruo~*CVJBb*K)`O+Ba!z353bt7|A5|<JoI?{Bo+F1I%R!@AAH>W@
zH$jbIQElf6B!#e!_kY^1JfO)ld8c(N-CCgCR<VfC*5B4!YE?iH&|0PS!V?q(bg3dj
z2q<?9p=*^^3)!v*RV%0ns2Jh_2@pb1s<EP+LX?n%qsV1~AW7s%@|*X|^`U^Eu79iF
zoA;P`=9y<^!oUjH(T_VmN$bUFzPCET8{pX*RZCxI+ycs`nN8aWQ}CeiUm9isk-DZM
z_=8o}I><Yho?<y3rwh1)*#BcEClM8uubDp_b97o<&csxz(n}qUVCOXGa-t?$EUn<7
z)G317U($0_E7f^=zKrlV&4b%f@e~?34G6Y}FLRN6{m&wwz}S}7`aKxjI|E3sGH&f$
z$_!ncQSSb@=(!#5>AcptRWQ5%r$?-?flsdhCh)N&mYav73{G3(!e<I#P}KhypPZGb
z^?9-7=8{qEau;0rQgm)w1?jl6AYQxihW55_M!1ngf9XaN0X?lBNrYNTLwj2f1h^7S
zf4Q*9uRu~82lXsJVf7?dTW$qsR%!Adtom==VWcMR%PNTMy!Y&Z&S^R!#w%)WFs}#d
zNAD2QYNs6R*SBUkJ2{!0%K~GAoc-8aWnEr0Zab?zX}-bjD1k3jr7o-uyZ!oLr?HMX
zN&dvC6<jV<5Ve(A#ZU61!&6t3KV5un;_||st-W3Cl}>x~UnlgoByu_qKZVM&*HpqF
zsCAv0tXR<tAx6zf2<FwkpsMBa&SZ|$?fsM!s#R{j75Iw12QDnHy}K--bHeMxEMVqL
z9rhwtzE%X`tyP%M|IsTE-dF(1>w~FosIBsPLp{&sBq?dHob@=7u1Z_j?0WSz88kDP
z`?8D8Qc<C8mpBb2MmWt-b(#xP#@#hQ)63$UiXEtR*L~cpM^dENK$F~~uQBgP5A-vI
zdm54+pbcowwM|Q^@r+9pLCOFNH4y4K)3VIXv@gRD=^Oz@rRSZ<2lQ`7%BbijKI!WB
z&wn5MJN;lZ8CkpJHqiogG?=(7ZG3P3^?E7Y$casr8g8B(2`HCzLZhMA6amA{Yuv_k
z?@4c0PUI3%PAs|A(&^bl)Rs{9{lyh+J^S%>xR<#AcH2e3uH>GC`C^&4gcyu-(G>wh
zh&kMZ7zQ^Uwzlr-+l&&g@fQJYU7xVeZzk4wMw<^qy}mM_^duBt27ROq6R-kK8p_#X
z1D1$!k>8~ZIN=CMbP(9^3799B--J4FumE#WnQlH`99QT!6XWk8{rl{AU<IiaUx5A<
za4q8hK<Oq5H*~|taC?Bu#tk~s2CF<V^Vv3St@*#wnXy!iVk!%vl~5xrR{MXZ^*ga8
zAtyl*=BYevc!X<~vAihJmLOFd>c<a*=>ez-?X)6EtGMg2E6|+b%$*$%R3W@QBp-=O
zY>_)SlCM!a4Oj3^bJZc2d!RcQ>ZcDQNrcWGXq+-Lx$ox$yKL#b7TabZbk(<@N2CVq
zehXk1I7d9ExX4c^Gwn+i+NXziJ`oUy0-D39I0`$nuCXp_TbKM9O}mJWZqqmz8K42W
zqqeh;-YU*qLA``;w{WKgM6UG<p&!H`tR^<b;w9aNQL2jU_<ri=DP>vPWT8UAVI%Ya
zOfIq)sAa)0IY5iHAo0zH%&1FOiJ|+zqWSRit?!NsVhorH9)YA7(Ht4Z>1`lh0>OpF
zO2`HuW~Z0&MdR1nBOd@@`wIZuyHkbaz49IS_9aJ9eI+VN8!+A&8O?O7>cGRK$0hLg
zz66qi_S2GGX}@8CM)LRUqoa#6Nz{wf9^CzF$)MKSN<^DEhk3Qk7qJ(92qLH_G(!-f
z+xUQ4voJ7&XR@}1Q97)}y|!N=jbP9R5P8~)$HVXhjT@vd3{2&apYZTqO-^5uo+`2~
z-WSApfod9R;vT-W-Yq@V6nDuuKUTdRy}<fh+V7MEkfeIk_U%do`921LV()D@$0}Cs
zRQDNt;E_sjLN^w^r|5Wn1avbv>u_^Pm{rd?h8U!Sr_W0}nt6Y~6a%fwXnFFEX%7?R
zC06=VShQIf75Xc*&Ut;-qo{CKT7|&u8d41|1^tUS%x8_{h?eT=JbnD2yq5Yn$w~j|
zn9kgAH+3oGbj8D(a@fDi9p?4T_IX2)1KK8-<8%mJ{f{ePpiB|ezlGW^t6cq0E#Cn(
zPdjNPjx|m{=s!+q3~k>MeFd&33yrgTX>)!22rF$BRS7SD!=<^YVJAGf3T3na``IBb
z#oRAV(ocPjFb`+^r|{@@q;_H=kdF*q*VcF`id)CH>;kEXey-t&#%^kg;{bCP!5gcg
zHN$Pc%*yCJ$KZR?@@K@S8w0(IF+Ik|TfT%Op0RQc{_6lbFTyWR8`dv>Zma;R9|N#w
zxyD|WpNVw9BGjW`9ItXU=IE>s@F02jTT2jpHAwxwp#|3wJ{Ey<*iq)ZsspaNB>jk6
z-JvEzYyRwva32+?zD{nhR7jvu=5hjDbT=Nb<r;_6{2(yptvM5!EXOBBJ-)csYhOaV
zV->&8f5|jmgNWUO5BT&RY^s5V+p+t-Y7M`^6sfQuY&KY2w+T^rhRNKzq7G&#L06J~
z%LsRMniij7oH9_MkU<db=MSytJaxl0`wdjoMffZI8QcdjGf}+)l~|m@dcTbFUs9}-
zN_;gj4R+z$^Pe9lrjwaHMz)akSfe|O5sz>}eQ&_?Uddc$ucr0x`H=AV9ESUF-!B7I
zFnqs%1H2#rUDll0`L1OB^&jC_Pb3Ed+%eX<++A8nMYgWCIM+G3T?}&0?l?#sXc?Yz
z7_cJnuzL1kzov(FLi?hm3C<68&D7lj7$i8NS&~Q+<avGrj9HLbs!l2LRlTVEL3$1f
z0hR{uB>tom4ZEf`gonyEKlgR^A(ul-0_Kf-oeCy|_~NUgOr5G^-7ctDEXW_rvS7#y
zTK8&khu`^?iVElAROMlV0SK}ax-ES6g^-<|hEj=x!rtLwzp=bBcZL;=j3R~d&BSM*
zy0d^D;@0FeFP?4;p%*3@Uv?*i3U{=<0^jvBw$j0a@(i^_d2zz*2xP=4@&pA4Ol-_A
zYmzs8PJrg{Xc94SeUENEs4mq~PJX9XDg>xzKt01?lq_7q5If)8#x&QDS<w$R9p+dk
zU7qVdRlEHl(du1=nYpB22(?*IQfBoLB-pUxJ<#Mee4i@@vN=6aS?i%F`CR9V)Ks}@
zbhTV|x{WL$PTAx)hI9YyJ4F+3eSOs9!e``r4&rV3k3ah1ZPyvh%w^lB&`A+D-QSsU
z)IINXoJ=Y4%48ngd9&wjqMt)n!9(BnESIluA16;vU7qvP+7B-*Gv+rd#aV~{?pu-*
z{ax|#w&Qyo)v~>gK6`l!CVevhp|6CKczkxEZ{OPl3pYiCtRi=^LM6`%X70hN>-;6G
zwGUkvgrSCk$8wF)fKm5v+0bTZiDAHKq-MaVD;hAW^sf#?fcPYY!|*cd?P`qJ7V199
z(GLlGzx7l!s4hr%Kl9Bn&X7%oi?PCL{8+LiCHSk#Qt<=JL%QuC5YoyVauiJ&#+j$C
zfq9H_Z+mDnO|YDY>f`UPqhKS-zpkDcpC>$r%|OPQ9VcMuV25m6?5+FUR1G5gdHFcF
zo7TVFkj(8)m4u>22=}qQ$2;?TpeXlL%JprH+u-Ac*X@}+{b>*i{6c7a?@wZ@i`QZW
zO0s|nM~Nvu*o4gs`PfYC&m_>tJaC<UK>9t+fb?r<Kzgfp1rtmDeFJHro%Dp!bs4fr
zLJwzBwHMSq-*?FEjcs`1ZUwjUTGtP=ukgLhp&p-O7ZFXy-OJyXDLOeN2YrZXvrq!C
zDvB9*hgp6F^-|B;AV!nHGyf9?^Yv^EbGaXB%Jny+2@SGzKdfBe|LlKn0{M5yCn$#%
zPZi7Wyi3Svv5a#}XIrR}_y_ns|Iuti%Hppot87WMW%@ISsc242;%1O{x9|7gM@*Z6
zt)iNgyqg~K7)|b*S*bvCL1vy4!sO#shN)kXni?%v)bmfdt2zvc-cP`7u%OFsyjv67
zMsMJjaY7@)$;7`A>~oz9#g;t&F3`0Vm)BnI9DlsEO%c#<6)mFf8rwhWnCm}AN>6kj
zbI^SRZsS-hI1l=<api%*wURFk1G)1wBRbO2pv~xcM6A8yD#$~Ft=vYCZk!icgtj4i
zrQ%6k(gRNuaX4`=T@LZV3}X{(+OHRwQq%k$<b?(k@%w7U_sSe-Z2ABbx4HKcH)I@*
z$L4XT^kDP2_h))oM6ka@bFek@POUM>bPr8py5PM^Q*!^&(HX!2dznX*E0vAXPANBs
z1C1L@n`gyxyQ`nMOTb}lHn;FH|9m~2@=zXO=$mSHN=kf6YfG#l_h!At^3lJ`m+pgR
zF!@0H5Zoonl2!dly@Hlt1(orAm*8DW{YM4N1Nh|Y7Zefs<I2jgrZG(C|I^=y0>4li
zpKA{@v(qum;?Iq{ll77-sTmbN(3KqHLyDDgy=4a^SsxPO+F%zNs|g-x7yg`J;RQ9I
zl|rj60km)RpO2|%wPbMZOmZe_MvuPU=MOC**|w<B5kRN?ePzC335DF<#2EUcjJQIF
znZ_`-7@dXHdZmN8`iNvUZhnfpknO1OZb@9IaI1&u-I(vBD>3_o(PkR^bpPw96uT#3
zhi||LX3YRdFU=UxN?_~Pb5Ag_{NNXGUtp*z{Kadc1*&wnriKYDH)k|<Ag@;z{3E>z
zuH|2FEsbdxRF)>um&siRXw{a7^?O1{<2yu3O?jxf9cs0P?k4kSJT{{p-A(9Gv%hOO
z_B`}MvGX*e$<xtja_o5kkXfaH65DF%n+}s6cujb#^fcViQ_`|0<c1GC;?SY=7HoR3
zEv0U@EG0aE_MLtOF;AP}fkFvBG^0gAku163e8BXH*U&WJ6*6ox_?VWXn2FGjs_N%3
zkcq$?#s*{}%Flb0<uk~Bb02t3e5>>~SODwRXH0W|GKX#L@)j;g&F>0&;zaDf0Gp>Y
z3A_$<Rxs-%hN!e&YlqHC0K`F7;x9?5VIC(O2b#xeI1Wgj=#HZ~?8h%i577$#Bdwd@
zbcJn^-(IcXhi_pYu7ib*-RG3+FwGFg7pH--bn%=3mH-2_RneH+z-HBh+dwsF;5M8^
zCWblfk2G`IH={Z2n8^kflX!L=GTAGZ1Uk3U8``-GY!3+1ClSocJVi?DhVdRm&1o!x
zRlq-D#G0ue%FpAhMN;6N_M-Yy!DoZs2(VU>_L&AUGXa~=4W5&C>plZKC+aNo^Kyjh
ztbfU#$Y!lp)yiAnjTs$cEu>Jw=kFC~&1P1E`#P)$v3OiiUiw?lL*@08EFNwSN_R+m
zbj9SGS6F;^6kE`<#LKpOwETv~MY>;XN8@3Kh7tX5#f=DoUldgl^z0Ks+ay9@4%n*6
zjJPY6$G~~zU9U^|b+(LUW_?T0@DKgHtj)`Xv2(}ldgQw>0aL$wDLkNaUAF~i4CCSl
zei>Ko0N6qQ(j&^Br9x6qvqbhGfwdugeqVjMs;pRh9$>P}$FJv->!fK!+DiQ@VxG1p
zUZO6wh$hd&p9WSamlC){=e~w<F)@T|_aLlCWYb-aEn$tK**K{bE=3jUs0eJsvC*+d
zZ{57Fh<?&~sn3_i`$E5rf=JNKR=BjoBPq%1V9_?fnK;mOum-8a4Y0;p2W!x%Ed#9a
zL9iwa3%UT-jPj$K-Ks9GP*(}9_TG`d_xBi&RJMh(6}J-J&HHLmuESS`_t9Dkwu<}n
zq#&z{SS6bd)S<D(22f|B19kC{Fu`I7P-l{Lis78h5P4a>6?7;hq0oifO1Iw^Fc0KC
z;J`LTZd%DY%ckxi2N*p0{WF4vihl;bJ8LY4B^(6kQa;dKN-ehifpt<tG$M=nR5N_)
z5*oPG!_UN8C(m2^XT6<!+x{PzPZc6*UEfD6w{HnG{}3jZ+5Q1>1j2Q5?Jn6(OZBKI
zJj4p1t<s-E%z_XaRnDz;$X9TAi?e+(#X7o~Tc%HB>ntW5<`_Sg&Uak2gMMxSUKtu=
z<!SI8IbT53gM3G^>k2NJy#H?nGfO?=U4@N81q@r*gWPya%LfnMN?ANxnBkT8Y`Y-&
zBmF{7TM7?1MN=*I_m*J_E9AfUfUk#f6llf-W+hOUVJ*H!X(BPZbqv~wt}XC17!2cU
zl^SSShZb;EDiT<p(tn;HYqn7&LJt*}H?jk_U40bnrY45KYtmb#>jB^=P2fE&K7}FM
zx|_3S3v)&CPpnuDj<~y6#%X)%c!u1>X}9@+&#D>@u+{W48VK0df8fD!FsIy<oBut=
zvUQeoQniDHiWGcW^1BYy{i9;$<N8yA2HX)i3J=Sueij)n$T&8-zvLcjhNxdcU8;I2
z``^R2nAPOD(M=m0B=i13aHs%_&;!bSsTMkz{M-^2*3rb`P1bM!B1$e#;mEVw;_d!=
zAZkp5KgX{#PWbdGpUC(xq4{TNU2_vWMXOw!V*LW3om`c?UMS9Y_!_tW6`UXiFWf~+
zfiaNP9%vcN^4W&yW?t5Ryr$|*!3?aOVZu!48zM1yDY$<+FBO*|qnB9=z5e-D=_p}%
zeDrR(f@Ao%9u^B>f_GX9noqi~aK)&9<qA0w9Ysvu6vI}W$_;l^WJMlX8JanezvisN
zfW3bF0|0yA;O@1;KxGWrBM$DPqth^8@6op1=>dnR3KpdQA7L8b!6JI2#ZG%)u*{vK
zV-s!_hV+s<{&m@58EzNuJ9tr}EWfDeetrkQj_G$mKf`?pKW##nWtZ6o!XcdBq-d6=
zg(eg#JSP~wexUIFGE(pR`)(V4xia_O9;%J$k(Rx$>7C09+)K!FRUoruW54{qgOmD!
zwC>5t`fmj}2qpk>n`@rx8kOm=9QWI+zzP;+yn*lE$gZaQb#g;UNh&du{~cjNDmg>}
zt$>+(z!gj;K`b+OCB3V@mKI1`r~mmexuGY!d1176j-qGH@zvsikn;#|Qrtx$^9%81
z&g!>Me=a;^mUQ3Kg!-%+_xs1EGNU@`r!#p|^-EZ!1g<68&2mOX3-%6i4*)y<FzB-w
zYZzqy^y<yvFSCT;inJyGxL2FC7mzDTnVseFP|WFRQvO#G`VYd`=6v724q<Yg)uAQC
zCg>BrT2%X_eA92{kBiL$XdCnki0KJ+sY*(4E($q;4B%CI$XzV551oJ}sUv=27z_Bb
zevy08&CjwtYMS>;iG?K(yr#Ytv_shVWI7`*^AzeB5xKAM(2{MtGR%zB+2o)mZ-M0n
z{SpYWHXH9Q$cvMyvuOz}+5N6!s}0twdU&Auzc+u#Jho!gu2o$|xz40#=OtPHAo!Mh
znmDO1T0Qv0&)!Y@4iDV8zr)%j-<N#OfD5{u70*P}y_CF_Jfb#~8y?jJu8^CmX~hVx
z(5T$qIzBC{%!9|>O-}h+!6zZsH3%^GSuu>WPC-+e1uOOyE}yh*7b|ujsoFB#unS79
zJKEc><U<8e-bm;k&doCr2K1wQ#w7a@Wg=+`n<4g^{#MYxgvo5KZG6_L1scvx9wkZn
z0j`qrx*L(UQNN0q=$6Eq-dhX5PwmuWL!*A!aJZBgFlFWeooTsd)~ZHy4wK8lDJ_LN
zLE+!3UafJ=+fstRY^kel3ZS{-S77O+xETrYAZ8F>#x4v_c)`4JV`xs9L}VN#*!_UT
zvb!T9iI*hzkw@++IL-IBHs1H0>F9Qer7^Wt9$^?u+ngZn?uM53ihL*9i3=-VmvQ=Z
zReQUaxiQmi1;Ju-`1~FwfcQH)wrSsY?GG&J`rl7b-Up+2GUE+=H`rp83%Tu?J-g`7
zQ<BPm5S+__cdQJ)EIFcuB$lHOC(6l8-gN!f7deGmGnNS($kg_p?Bjt|uSs-`Us@4O
z%_Gn4{!Y;LPr`d`mL}pF2XoaayXvruy5Ilr?*!7ma><8d^ouC<79J*xk2x2IY#P?i
z55aN0zv=>Ob##+_*WY9Qotq%d&j-%)N!E4ddcelWEd+nCGAh+f8eJ{_4R`tsX`{^g
z_7dMH$?M{>59d;{VWMJra{`HxK$voxEV${?W@)V6Vi91!U55$^&YhYiszhLjM03~g
zLX4`_MxnYX{gcyvXP<AB?8T)({LJ`R_a0?(4afl(f!?*jML?OC)<xWWYcAz4)$;K9
z-F=d=9}#Rz!AT=}Mezr0NEp#V>W4|RYM<1P|1E@05?N;H7ei1Yl8N6`Hiwhj&rbjY
z45^!$=iCkg5A8?NCn|>kNfwS3>FrdB<*Mi^`N6-(j0!bvSM_LL1R}ajfB25<8KSXF
z`0Z#D$s&_V;c?`*75etv?&l_Se1djAbi*p{6^-LvzaIvaJjAYUK*^(o;WsG<fs!{f
zD(P~E*CEhE4RhXlVjh~XZ#z&MYIHv!bBNuKcUM-o*Y6Wr-DUA+>UX0+Qc9F6ix<YE
znhFO34I`o3npfKH>^jMETS>?pEiA0dB+uRZv9n<HQ4Ote^RqAgxDvaXeTTSJ4pD_J
z)s{K>RS;N96rFn%x%W8KBeA6+oPP9%mUe?6vNLSwwqapcy9(ppSYdJpqs9w%V_Jx{
zS6`QOcb9L&PQe8&gtPnG7ZQa}0}NN0-zz53w&>RqBS-<_i(BNgvs&jpbwnuA&S8m~
ziH0E}L!Arn-iB0e6=!abg2`V=*gGDelzO+~iPS1SnV1F&76fGKKlm+E483|>$Ppct
z0v|K|I%>mOM3L=gj~376ln-<=BY<=9CoTk#+GnAZ&Ahih2%(9dtOf7I@|>W5xn{8i
z=ah1&n3dPxED>39^s6ASRw$&l$`xQot-Y+4Rv7hz_V<Zc+1~K>1o130fsG6K#WF6b
zJ)gYWmn{5SK_akUNBN@#tB>QZ$(JUI+In>bhw|(rmb0~fF$Izeh%h$7RQmomWaI?c
zwm(+#B=o~hm7EZTOwjfPfeFcWYoP6^it7G%#Ee&$VcypwDKi-LAp6jhmA(M-#7U5@
z*Ig2<+x0pYHV_0c9{9^@V~xpeG=Ua%m0+eO(3$~aa<B1UR!r>~B~<+46isgYTWKB&
zIztnZx#qEg)zFRuW%aP9GxvQ(vatJ(qyx7Qawx4`nb<bTZziki8pf4l15|N&oXCE8
z9Nj8R;x51{u-3`iJdVYziVIP5RWo5PVp2m%95y6!Xc=+LQ`0m!@8FWH<<Wj0OCL&W
z1eVwJCqiJkDdH=AAAu^|<24De_m8!7dB#U~%kTXC^moGK?#U2&Pv$Sv*bn_CyvwQ}
zV`k2>5Vr^qMFhpk6o|KBIeG0a43NzlKdmAsiV9vOM=*VIOlk;<sS*m?vu*TFe!9sI
z-}T;HoXr_c4J5yBC@`#v7Vc0U>V)y-FFH3Y$2=epHvs=}^VhFVu@oVslY3j_)qg+z
zcZhpjBycYuYwsmS#)0iZUF-IFUvW5Sh(QCn(q6}=2KHn{Vo8G^3wtB4L&dF5HHRya
zFWuc|6~IU&1ZGL=K!xdCNcyFam<Dke<T#Ok?F#UFcNXL^CV_b)J+R~%&V|@)Dem3K
z=T!ufUlaXo14g53r(i_`w}vxU8P0TbA+Xjd<aI8OA=9QdY)u|?+ja05|L@*X<xNSj
zp#;1%KJZ%KR4`(+4BB~N)oT)DiJweb@2PCItD%L>kM3^uR^O1wJ}RJ>Ben$2tNo|v
z_9ysn(!#mVN?6W@j<kzBa>FyYU(qfInXkE+TNksG!P#z67XX|7j;*5<Bcn`Xx${g|
z*!x8xtIyW*YbkGK(@GnWfiJ=xQj3iudKpBF><rV3!qit>4M-%gj2EnC;rogJ1-qzY
zH~F1+O8tel)G{X_@CQ#37I_YuWqOZ~>3u@XW!-ZMB+oUxrpz?*-u9V%=gPnp&nLf*
zpyb!|o7(53>CEr~P&9A}aU1<3KbR#`Ld=-N<ox^XW;)=2V_MLTAR6|b-^w->7aoHA
z*D=Fus=iOGGJ_lA6hzrVZ+}gQX6s6C{4Cp4*!YZ{q1sbIWSk*Hr9k?V$|TS2v4-KU
zcvxa~OYF4@Aj=I&2kr)E0DKl7b*}2E9taJL0H^s>x|_$4=Xu6kLp0SyLQ6T*8{feT
zPbd_J5j7zdmb8BRuzXA*iIKq-dPYA=3#9GTUz(V(Q1LZY=EBS>-hdEVSU-$R6ejob
zsoY{tCCfO9iyiW5??F8~kgve@IYmCVkV9x8v1TPC+$spb`>Jlsu4LtB4McVhKRG!e
z{ZVJLz+ZAoSomi`0OKs-Vo0c~@(9Eerhk+%*K2vVU;xn8zvOdLrG2fE*@mK<MT!XJ
zJCPxD>jCe`Q0k)oX~A0;a~8_OwkEerAbGx2&^9IJB9@{A0`z&n=xx52yh^E*-aKR*
zK-;B17D6Iu3e+K%-s%1E(TET<0wpz@=9xj}tj08i_Jz7mzTmyn@3|$FIdLH5>_GCF
zy9Vs)ZLyb31Rk0$NH$b{$5>SYP+M%dH9e;vlpi@zgZp6_KE1M+9b0HiVw@usUnD=f
zg{;0f__-T`6c}CgH{Akg{<saPgdmIAHYp&Bsl6g*kRSBBAXsCuJv{{pU51f5Mobv#
zl6z5dAg|e<<+hICTL$cdA3$iK4!G*@$(LPWJAoxr%5NJ=pldgS74eRuKt~om5tiIf
z_cTeTdx9AUf9iBkSQ5QI5jG*cs`EdBjwruQdv<Jvi^yx$TfeVFsav3@i7zY#0H~>T
zK85_f!LDA0i5n+X-C#*bMTnHugGML~z=6j8;-5i5qo`u-mm4>m2iY%8XmG0Hg!av0
z@z)ca`M{2Cht28DFlL_m3I`vz{U!MVZVE&>6Z9t(a}JVL-oK$4eso_<YcZ_Mp@)U}
zdbLJ%qRSj!*)`Lcb&9DzL-u?2P|!9tCb>CZw#rI~h$3*xWNqxH!%t?TMS<00!hHn|
z{T9>PGdTn>b$44~z&tF4;y*m4T&^xG-v@0nG)g&S^RF{2!sq8do;BGioV@g%H5(K=
zI}hg;m3h=}6-)&ch<-7*gn=UOYD=+xUkH2cgbgyYzkXou!EWwrEhFU&^QqixQZUpg
z?ND72HZ}_y^xxMYCW~-(BwUs-@{fkg^5j?8367)(X#Zg))&{39(Y;;dF_Tg$Klonl
zGA()Km}65F)~cn>;(z$AWZVJa^3d+%N%;=93|{!zw}6w>9H<3teMMxVlj|X5a>vyz
zp+U^|jCTwrZ+z#;iBQ@y?rE}JpA^SpVoO;Db@onyv|z(|c*SLq3!Ou0Nxc=3nUW2|
zLivMdIRTdV{X(z)f_jE+Va?)iBrujM`ucnTTr*l{#X5xmwQGTdF-SW$>Tu%B_}sCN
zi>Y}I#fF`U$jVJf>g|rqmVEsBive*~PkvJ?Uo{3T#>!PuUhCa%X`)_avCH0>54^xJ
z)HxL366{WI%39XHlxxDNPp&bY7Z`Sk;i|aj-Q^Mv$&YEK*%Pnzv;5$Kpv#0#JB93A
zJxIwef_7@rZ`FYsv8_g4iAR$8RSD7Aptwy#%t@cT`n}o{(y}0-XMFxl&`Y2c1eT3~
z&@}iyed|7_D-Pwj4Wa{|l*9+=t-5GQAaCb$sy2+w(W=_?6W4~2QGtD-Wymg<S+!cm
zN8dtg;f0tk^RD(Z@y@p?56jRcsL=*3X59cow}Q#TZBHx&Lu6IYe+07!Opj^v!^6mq
zf-5?-#Gew-mF&=BDO6@kvc{HXV<f*(A>-#s#3mfG<UM}RbTOGx+(i>wwi}Mh;w-sa
z#1(>Xq%lmPR(YVUL4xGr@m#j9_ih3RlL4D1I6%!I`*k%6S@|u=EWX>KeYr({o+sP&
zkQxlxLqxrq@oMfUp42Nyv~a_+0?z{Mr+He1POlsY;||YTBcT`R(uqSp{uF<zMPQoK
zeTNf$mn5>WpnmI=T1$WiqdmV#yY1b>^LLW}Fc`m}kA#nMf-S?@e$NLWBUImA*)=Pk
zE~DGM#}bosV;h>7Zkq~b@`;bE1fC}9W~5Bl@%IQ`#$`{Wx3_~*IgorGx0v;yZ&7`3
z30UR%M|B$>)pKOiyuk@;5#TyrRgbI1zTQ&K+^7}E<}mrNp!*y<^AOqr@dTslVmuLN
zS09#V>9w~|e%?^yd-_LdBRL5Ht5N(|k%?cw%QG<lHOP{5P!la_d<;2MxgQ|+9;Vyo
zf|+HA`(qc?@wdoWVFw=W(x!b8Tk1VN$bMIN^j{Ytj_o#Fo+ITB(bBf3wE}<LiiTq3
zhOr`$xy_`AE5%LxQ1SepCMJLLB3h`mu@f6m@CARs2T+8Kq*+*808MK-q=)_1bD!v%
zR0g}$DJx^~%iT+uy3zx@)Egj?M(Pb>>xX$-sW;NHs^$@~*}X{W{hz8R2-++>+B|}v
zNG{KE7Kc>sBB#95wNg=Atj;2T*%u0QyLWs;Koe9+A>}!?O%^2k=mazF_v9|aO<Low
z_$$XdDHZ}u|JT!tfd>l?pp*9h$~vi_Zjn_&MSxQznf^}KBE`;U^8x;OWW-&q)RrY+
z1m>4DVJEz#`A?0Vwv+q%E$NxBIvp^wIGEBWne@7#bW_$i?23m^f9T!pWv={PQaNtT
z1u1c#OUtZSb!7PbsFEzu#_4oz$QA+9T5-rsd|COf;X2iBLW;|)3*T0pd42R#|G-^*
za36#Z?IG52D04??l}C*^|801rq6xxob$2yno-~<ER|^z&Cle>P71OrJ9tvm%hen7`
z(ZY97y3npmSSX}<EntYu_So2(cwzV!*%+qVmV)D<z&lKU+eEGOj8{cNkOU>CD9^9q
zkII;b<ue5|gVcZ!Pbc&CJYD#%B4Ahy-{7?jx(l8s3E|9DH3$p4CkHZeV&(u3?Oo#;
zFWN(dxnFp+Xe3xIIArFXr!>qHCe(hcp#;`uTVb^POGSW4geBZDfCvydS@0m!-)Ejm
zE-GVZL_2tgR=Pa&3VmyGse=VoR9Pan<!N-o0OGZ@qrW}HI^0zJ>`wvBkb)(wK11$I
z4ORRl5`Sf)m{$FAbtF>Ohn=j{sw0J2n2H)Jd;uVhuF`v;gOR4xsjGo64`y|LqP{D?
zGsf=U5Cz7;!-KOdw4a2PK45vj_g6+S?W)3hKs9+l;%s<$gfx_%7sM4(ejJ8e@})cV
z%eBh;jm;;a2BzepSL9p!A?dcayt3p0qkUGaONQ~~N@!hx<jpr{f7$Zz#u~w9^S*Dv
ze)*D|J~zBNyA-Ks3FivH0MA=>j^FfCoI{Mb>ocu_4fUfJBw?Tw5^$)Uy&*9Q5tueH
z3K1CGaK~}W9`0(}-L(XwKvFxFt8E=)w+4J4GMm=GB~Uw1#fs;IShBl$*MFDK!_E^J
z_UhvDq?%}h{+SSD0xbVLr0nlkAc2FD@>_Z5%5Nfx-Vg#OWIK1#Y5Qk0b9?I`-!nbY
zWDAkOjp3hWTSP~JszUBx=ug4BJb>X8Ye!`il(1NhHEh@KOB9@^G^BG;+5NX_SHJDT
z1p31l{E5JB##Z%gve??TqFoB=hJYz1UZ;@grrpw*i)sec)tWLdB!EK@j%5p|H!;2t
zkZz#*<a_#6oqk)mL$*D53jO9_n>4Jl1s!n8t>S_ePXXGth}~|k`a`mD{F7hOlMa`(
z&T3bIVEIXG{Q&E7v@=m;G9E++=$AzZS}jx*JEfBIR1N%F<?j9lg_th$KO))^x#Mj+
z>1qct$Uq?3+&wvf@yBUDVIlh(td*TxR42~0$*xMn#RU#?O2dlL3`<|Q_*|y)nPG9Y
zHMx^Zgl#r$ia+Wy$5p>jbn*%q&yBrGX$`@CLYR1t>X{^K{M^RYroA;l#q%~&eSF~p
z>LN_=IOH85@8y7j%4CS6e>ALX%^~TZJ#F<7)kK`4?v?i|c#VW9m%^Fd(}-+u)w;HM
z<@WXf?3(~!ua{v=m(9jSbpn9LEPE4$O1dA!U(rYUt?N_?Wf$cA)YUQ7#6ZJgj$G~+
zw{Z`GGB-cW8rM?{XXnp0OLHQUmyTvFR|GwYbfXrl*8s~V;4iD{fk}>-5z~kn1UZ6<
zvE*v2hta>|mUc4mT2D&ww-Or+@oCC-hEr^+r-M<kX(Q0VcFZbPeUQ69XRoZcJ1N{W
zmQ>?)crgisqj0~t)XJpXRx9;`%hg1ju@Env=m~S;s<|BHP6K#2Z6johyUIY0M4e#*
zc^-WQYI(n$HCEwu(GK-v)2Z}Pg%@ybSU)zT(uWBO5DQ~L&9SQ8R64|aaNEmvkv&Iq
z${c==-mTdBY&tAYy_dL2VuJ(8?Puf5?P`?vVx<3YCOZAguxIDjDmpv8fJo;vKzn*j
zzqnVZq|kY8lXETml*)9~$@KOLvrnAQgh2{Ot&r9U8B$AQtDD=h1lnRz44fW{T0Sb+
z)CkCK9|Ln6a{?5l-e$bY9?|QRR%oH}&9!qIMP<NUfQn50qonea4M6_fMfp^4LG5gt
z^I<1(pfr4~3zT2qJfsLIDRJ7-8*K2~i|*yJoD>&;9vs6TT?I>fsoGv!^)W!US+_J0
zcl~ioOqIz@!$=cV0-=EKw2d4$dU7RVbzzC#K<ZK);~`tAfnZn~-1Gi=%UE`gVl+3f
z_X=)(Rv|7`-2aQeB+GhvMdgsBA;LP#3w;-yZbv9EQr1*&EJ&Yg2OQ7@)qe1s;ucF2
za7}r=iT;yzvF3r{0Hk^lDHU9J{Ap8-3n`b|rC!;+JrZX_uL=UC4002_S!c2wnXglP
zs{^c{dTLX;i3Oybo;jU48oPG?^iObfF0GdDb5qH4`$Tr9!%k}GUJNM;mO3<U%Qg*D
z7xM$04j5iR^I_?qyMzv2>VmB6S&msRU)|txLEY+VRxX`DE!n8v2w5aEr2MfCVDM|_
zwLpsLIQWJlu)0)grC9LEDt|oEhod#$Pfp>vDZ3=jhB7n)r2^=*lc?T%#5fjJh~l_<
zWIv#lOoPqdmW$%3$|kh7l0e48PZ7Pg3<9Hsa1hgl1D<W)GGV1ZR$s3TL(sk3PiBfU
zl{*cQ$Z1;cJs?mgx3$wo1<V^pS@lcz&M$9e8nI6)T#-y_wls0_Aqz)a<?W%^!3gZ#
zT@qNBl7FDw9maiQGA^WpFq0iIs!NnmgvqvhZ^(TNkSHb}3g?_`<1bfM5A|9(N{wt0
zD)&BJNwhW8w8D%7ZB47@Tz&j5uk?K^v>Ja1=wpz>SY6{pC4UGy!!$%mRskSEMTPnP
zAPx-gFNVwn@6fn7TPeD#%*JhAd+y{mu^I2duKuWK%E<JPdJH*by#s|5K3~<~#&zRf
zW#^H7$?x!HmUBWuKN6dqv`5z4D}xQR&xleNe|o~}PeVEpEpK+VP%7@um$F6AZb^6s
zMI3>`=j@Rk2g4`GFa{~!i#3?T*G!_&04%^bsGK7fQg_Q__KbX06>Qobwwbk~ANilo
zB_Y=FqtO##L|buNF1Ghx0nKSM4$XPqUu(W^g~Rwhri3@%@Q7JUS(`cYO<>%|8(815
z(YNnk6Y)Vp$b_}c6E;RX1nQ8@Nlg9<DH^IAEb?=}!0dD5dbf){afHSwxUH<Xp7q}H
zB|$lgaD`FQRplmwiLPZfi<GG}sU<_z2E|pyF9P)uv%h1Zc%b(dGzl$lw)RxYz+yXd
zv&n7E;5|_J!qJJ7%Q0+FD{TTv6(g^Q^6XJMqrLhQ`wD*;gQS;2!ufxeU8B6-UK4%S
zfL?+GAufr%N2DD+9R9NE;>s7d_9c0C{(-w2@_)vvfVJfCbmK!Uu3_CjKz*q~>5&_o
z%xIp)E0UJ=eZ=I4e++_RswaTHE&!`5GV=Lg-P>56GL_W?+lqTQ32FFRXTQHmm*@~`
z7jDTD5P0~ISG`&|;??3x$h9y~KKhRka&N6&jm$Dm^=D<$ex4hpXd}0=wM1^6*CYO5
zNnx<n!%9W_dPWZ`qbCdMCY_*Yhud(Gn>A-=>TVp;XP-|7Qmd{<BA|V3xV3@lrhQJ$
zPF`ZJ4G8*i_u*9u-72G7*YAJrH(okPp-LBpRa!|b<MYn~by_36-aN-+LVRxl<<W|8
zZ?UJzF*kLeca%)ocObp}PnIXeDLTj(YFBiY6$xbFR)9u_-<umt>ia{&Gjwqvur@0k
z+CYfTfW{Y)3d1!F7<oamHst)(eJYJB%v5;E;?td0h`~6_bfZKnYQce;#wL|vHF5q0
zqGqY7V+tTRMzoS7)q46{@Jc7TQPjoX9xsMzOZrgZ(1JUof~+pE0-SJFn>!b?Hd9ww
z1ljMv0{3An6Z+TaXz{N9IoCT4fs=PW>uMus0L2KXXfH6aO^#pY%6E2!T8-aIUhqd4
zXa!7vx#}RfSynr060t;%_<y&bt=6IDA=yupB104v(U04^pGfyu^|j~pHd<Lw*u7+i
z%3j@U6zz~uSd{KvqVS?n#2W?PjEf)x(Xq|wgrF4rJ~=5`9_Vzy@C<e?RuosdNI?kr
zH2Ah;{Lms*AYTkp*%w5r$h`oyTlFpdV_<B{)=Wb?tW#bbio$kSoJo0Puyc!6<nr3A
zkqYs;zfWMw>*@;-W`WBAs=BjhHhh7Do8b6)V#mFT_Pk_}rzBmW)Z^CRpL-7?=<^;I
zG*Qi~6(t)^`CuV2ZReDu;4=+-?~u~La;w;P-7PRlZg|tBXQdNKJ-}}&%^F~M<uFJ-
zP0yjh=}@?21AM+BIBWTG&}XF@nI#CvgnM%-)wrLZj#T|D!6o%#RD1dc2<v(4rr)e%
zuAh74jwZH@2u2L2=`ZP(YoQyB4C<D@Gc{}iy$w#EgPLUw<M+Re<b3}3YviwR`p#l6
zqF9WIP$YW{{ZI++d<QROCU`9Y!<aL%d?@U96RXc3QZax?in#mBp?Sl{#wlNE(gm12
zKyo$_^lRzu*K6GZk~$5)y}&56eG_E|gw-nleI!Q8%fqqlhekm?-#`s+wA8ctg`@RB
zf~n@q8+A?)1h;~UwQvLA3-rlY{FBsyTf=lz7U~1Zw0W=7Xh{3^AACDg`}X;`T4<D<
zh&Bs7VkGvWn9c(C$hUszC^p!j7*E7~Sc5;Neq~ZX$;P=itozo8y4wKvVM3Q(0Xq=S
zn#ex0VHVf{oeIHfysJIc?};Q!zV%5g%LSL({<KY$qk0Tn+9~MQkcBdV{oA={6s<rW
z>lLkoRmW<{V{{Rqhm&y%-nWw%Ty$OR_gIpp;OCl&vL84KaGPLC2U^|j=Im@-cOQ!j
zi=keTe*U{}7H@#26*wY$m>likdjaRc6PdKMF-t-Xx1qLBQy@#@Q=6+lt<lsh^y}|4
z@+3dlqy{1{aLUo=VB{IrlH9!3G*VKV;rj*7RSuiEXUKWcLB#RJb~PWmzj7BBEihZQ
zE5%L}_V~a$H2h2_D2C9$GRpo^?>_GwN~*yY5*S{J?`4nR7Fe0b+V6O-uh57oUgQXQ
zf_UMYX)q)g2Q`*MowK~QI7yQ4)7K%GjMJbzsfMZ*?mfGjEB~9{?J#kD`sukKJXBbV
z_zu}mmkQDhqY(&CszXnbA)UZbr!5RhEz~v88F^ha=yee_gltiS;K~GpErP5B0_1;5
zb|_VnYlg891iLNJ3~)bqLMWE;U2qBYJA>iwuh6r`01*wZUq?D23nU6G_vY-%qx=_l
z>b4b7Vs~p{SSr72xK00FsKmzk|Aqnlf8(#LLh_f-PDs|{esp#YrV6^(qmo^p!|54j
zpE;J@TqHig_nFg@Nt3eHSp?cIU2<WirajMCR5X?te}3aS^9c54u>f;__ytT!CU@4P
zg`AX;8Ky}slUOh5uBPPz2dDsl-uZ0rPn1U1N_KF9AUEa`WO(YQF98jIHSn5B?Pay7
zA~x`Kbet$p{ygl!bRX?!6U(y&_j;+MB$46M1;Ox}ZMRs#xZ*ecRSnnyu-g!5E3pn2
zNBUl+gyHTQuZ-iP{1YgWq~d)$qf%bh*l*BMs6nu{CVM-pbt;X0w3$yzs=@#0P?yA$
z;U{H^Em)o4D|dj=a?*n$oy&rIY-$HxSZEtvFV8XP@Cg>)iq6&*>;%60Q&Z5ezpMe!
z@WGzWv1mCy7{b)0$mq!t_}UnLe$WE2E}?voiof*CQNB9x+Q5q8{x?lmdGJ1z?w*bB
zyzFs7l`eT|(Ag90=+X{O(tV<ovd0o;iQ{_ZZ8!AFLoMd9q3bW_710wX*@rE;umvyr
zL^U$VrG-wfP}-9u`E79!;&|b1T^gU5I3b+Iw{Aoo(VlauBPQi+0N{5peQb}Qp`QBE
zWk?>K_xRJYrn^z#I}f+-j*xAB1kaq!cLrsDich}=btBGyKN!%brWN{)Ju+Ij92ong
zW|4~2_oL)6{uEG^J^xWhra-iu_lm{BS`j6(G6Vkh(;g31QIIy~NY>-O%*9MRFTLsM
zPt`=|>e}zukEjA>D32OhN)Pd7QI_G3T-I!9Z3I`ow&y?9#DM?Vhwsuu)IK0Jo+_AZ
z7-ULVlOUa#Zrx(jW|iv*X|v~V+)KE?7umHzLEN6{LensXk&|z4h#@UVXob=+b&G6m
zmY`gB-mgrP4S3&xF7gR%WIH{(Czw3PaM#x`q-0d!d9RgQJj$N}hDLMTBd%r%)aqjg
zfzBzPv3$B}i&>$NlUdJn*<o10=|E`nH30|>D#!6^6<^az`8D*68hi6Rq%cX{xW;_9
z{c7qpa$T`mVA37K0v0)ioUG(l604})Jpch;)@N++es_a-8BBi1KA*PmQP?l}VZGnr
zPQ52ZP}7S5g%RJG-vcK<oc?kH^l!#Nxq+yhMzr~6x!kT>E+R*q`Y%S+;^jppqT;SM
z?bS5}<^U>|fJBkBMb<6woo(3iQWv3OZ!r%%ySlB1;BoKOJ0?R329h+O*H#mD@W8s`
znq)F6DkuqC>k@fevPtPHn2i56N%%-qlHE$Z?Gupo>iu&D!H@c6?1wBLSrgk#s+uln
z`V#lcsX}|#w!mxAJQ-^7`9DK6KMAh%w7EG=N@7t)<4i2e88S&Zf=krqo-aL&U$*Uo
z_w##x+jWtuZrlHmL_pumd-85WX8}Tuqp+xrac2~<32gh&t-@ag&O|Yh;3AMGxk`>!
z<w`1WNxp1KE*!)&cxMiLXGrz*duMp2<|4}p>uh{t@K$=-rxtK-oR+q1tG27DC`Xki
zm=MvAi-`F^-s3+(sQ9nm$2iqyAOp2WMdYb(1}*DUG7ef9+7L_h1H6W4eTyz%sk_yL
zL88^j0!cOmFA1b1w&CU#NPszX0Vtb*6&q<gndx*_ScjUazd%P(Y<SqKf7>rl@=Tc_
znS<N@oz4KFo8YmVZn1)3?>UZQ1ig>Zt%LgDL0v1!KG1ZudJp(^1}Q1q@YA6({vOBs
zuu)C?L!EQg5>q42Ne&FATE2CECj~H8xVqjb*C;$cs5cXL?$s5cPST`SFE4ch*rV~f
zCUUy1AF)KxfT%S+lCVxIvs$ObK@y=iJi1W68TaZk)6sK)j>q<$(2M(dj&S%bp9f+o
zbQ~qH6bC@>&LSmUGi*9RxTBjCc9N`KVi`8QOt<fE9MlvNc|cNc3#SHMq1-XZ4+sk@
zIeXi$0j#(#gQnNI&VeKEE;A&EZdnB94vB*hbv;L@wC8T^{SkNW5v~|R^hm_+gl5yo
z*g`LlnFEUnaeZ{5c_BYjSdTRvK)nOoaJV85$J1+ZMe^L5u-yE>-VofEzlGmz<_Tpk
zsF((fErc-xH8AD@H*9EN{K8xYjn;WkmZB3#`<^*&lydn)gDi#c<P4yxgvK(K8yZJ3
zjLg(4z!})_E)k~RbaXSY(FDi;YGNkH84mB$Bh5;I#;T6l5Uz$Y3W{}pK<`T2mPgFX
z<z7+!9U@2jr!d7~*UMboSgiHda3c18a>vB-Ai;&Gpe9y8(s|rlV=Pt3Z7J(mo7J}b
z*xGfk-Yi)w!+bs^iPdU2Ok9?JZZ7n#k$Uf4kEM4hiMSV+=L#HATKLJzq>|(yr|Op-
z!SJ~fO<q!;7=!*jd}aD*fv=%aD52mCSorTC_uVAhX4eojdFqfC@0CC8ELdGh&giL^
z?D^QR`L(IJ3H&5J(RBiRc<38H{0~3J55b4|%Ige);}YC`U*6)CudsqiEKXZ3q~8qL
zq<erQptkLu#<<Yp?tiUG0`7jkW&yF^(7{7!)hE()cmD|Y+fSbZP<0e8y(uFO8vW|V
z35>j>3MyV0F#iPFlDQAUs`InheRo}ijt;orAAg2?wI?q4{>tqK_i36R>>F8AV1(tC
zK-udcF)NhXua{>-yvUIpHngQCpfWCQOEMrS($5V$ayRRKG*-p=W|D45Py?D7RozDJ
z_q{cS(FFuMriOoSb@7^DS5c=s^AS!9cv+q)z&=!bnX&`-wtru)s47vpv3YGy|KJsO
zRrlNQVRga9+VE;+@q-;Sl;l@izONEU*$140d;h@y?v%he>IcBS$NQshkN<<;?J#jE
z<e8wRDW8klfz&w8#ND)FNbjk-px@ImyXaI?Zwqa23xwL=TypUWbG;%X-I3aR4*$c`
zAhl-<4~)(qUD|a9>;L_RK@70Kn*=|zpBB1_4_gZz|B3tEyGk84n*ujP&9V70b<3;e
zw{*&WEW7=J$|dw;UUhNlK6~!A-XC#4`}}ukxKpc5dV8MDfm9&?C;T|2+q!W|XmpHj
zoRW697Ji)4^HHjn9Ke+34imotX9njjD#uC(l4u5tPRMGpIn<P!;Fb_Pew9#mC9?{R
z<QU=f0~~6@t1I+u5ZraCDctq_Z1o-a!*>l|Uy}|Otq<;cA`;sU9Vly1!~iq3UV`1q
zHwi94#zTTK9=GH$mQVj@4I9252^TKIA11@=xL?#vWw41}uYh4T30iuk=~V*X&-LFW
zEb<O%-=yq?sDb(iQ2sg#qwTXj+&Nw^qh-(|<1?beR2<1uoPhr{2~OL)n&?VZRs%@a
z*_WY?eDKnTLDNjI>0~Mp?hKLR7~CuCxd_$;wC1g`8F6Vs`H-{2hEK^n&xAqy4~kGe
zA9vKH;Q-heQk!%xmH3T>IrLe-DkXDg2{8=<G}BP)8l@S_WwGJFV+5x`!gV<CT9i6#
zc9y_ZXH~Ubl=^oJ8gyLP3t7b%R6H(5Wz(iWd+}tWa8*D!bc4yIyp3~hHDAD$8ymbq
z!u;4sA12|eLN{ZxMX`ZpxVZ)}<5fG?K(8YjNIH?WZ|B$}2Vf8kz{Ycu3Y?0p)&|U*
zIHJ<&;U}(aW8ICzJoy_-Z*{H_%<qw=N6Y1Z8k#K-f=)sk>-=27B@vbQJ)QAC{Pc9F
zAK+f2wBjy%=4o<X-yN`S&;tgwjrgCq(lg9GUAAna@aPWt&$u66`d`TLh)apCPP>WF
zT&>>^e{%^QXm6vgy=X0Vlo;+h!Tmn*m>L2UPCKd#{-1r>MPcNfMZtBfA!J@6=k?IQ
zhg)r#uq{1HH$&%@ntTkd4k9+SUOy!+r#v-NRtTDKf{{gSAeZ$PfMHcv!z3u|hrd~M
zjw0lJ`oGJ+6?FFqp-850jKCN7d4d(R%;2Vioe@EYh7$*q{pgu*9zH=Inx09B2<gV<
zb_z1!l0zGd@NlRVpBmq9gwr5%IY%hbnb`O%^Jxg=k5NIIpTO79b&NpRtcd7t1TS95
zA6;FnSHlfxS_f~&Hncc;IkXIDp|(Q${H|mVPBRf>%781kQCO7D{Rm1Gh8_d?svifp
z($uWln3P3H#Tl0;t)P-&G=oQ@pjLjmXZB{?xE{IP59^VNBYBl2rA@>nU)&?lvjJ{@
z54<%f;}Wc1tFvuG?fQ!ygI{oH8cDidCIg-<P2n|9<XD9JvfWBh<XVAQSM9I}ws+Nq
zq$3>uIfy9Bj8|~Z7bP&^hg)vxJDSe$zDkM5ZF|rhxM|zrw<r2Tx^K$=fP1kTza%|*
z?)|WM%lGpcY^R8)@=tJ2ZP6!i*{mgS>`}y0`u9Wr`j^(|=_P4fwCp<i_2dNyT)*=>
zFIlKu1oU4*V3Id(^N&<=@_M?34VCuw*@9Ju5lsUOpnXL#G!TzxpzWW!o)l(^#z9=e
z5&SU?%{E0P?KbH~eCR*8&T#4UgDp|(2ohD(hhhXU<^>>yXnsI4;{4VGVv+}L1JGRL
zoX0k8-0?h-m%ncN4SYzZE%YFprJYd~zYjm7$}masWn>KU=~ORSI<sU3?$fK2$-Jls
z;9gYK(56X8`@Lb&G0ZtG6LVYn0ZDGSPyg-;F>2C7kmLVsr;`7w3s^?XL7l+*iVem=
zVdq{<3sD$BN%SQgZmnII%S%tHBD#trhy-cjSS@(PdUU)ol@45a3nBLzPl!e6zvDj-
z<0CVmFu$`ioCo@EUC@G!yLe+6EwFVAaW(}v5R9-U7Mt~JQ^cSF%j~YZ?r)Zgfyr31
zlgyIIxYKSe4~3#bU^`DB?0TzC?(t?}4A88R=OhcE_VW><_?RyJJ`Q8h)Yb6eZ4JPz
z_v^0k((x6zj%Wkpw=^FJQBVb>()5=?%3XO5j)z$!gt(Y{NpLB;U{OuK?~bUe!yw!b
z`f5clyEj}_k<lBhPLRy`yJ5#;qQQ^bhFdd46w2I}&$1q&#u({MvrgqwgDM$f$2h}R
z3C&3u|523Qp6~d3)z@f3?@;q52ksg>WK~!7YQerkR{1?W{!AAf|3T<UC-ds}u){BT
z--gx$SubmzJ8a~jk>Z%Xe-?_Fpq&);8k^b#MLl2Resq^ND@V~IZLB=pwqzIU$}m4I
zRhJZV3pSq+hH|KX)m#1Z+n7%mul@WR<F`Lsa6H4aw`YT-b<6*KTRhuh)W4T4USqQR
z?cIO>>9cqL@weI2K7H@K+l?_XMsueuFRdH@m9SEHT$&s5`8V(WRNy`TA9JT{eRApP
z-OItL7o-30xF@$dE77Z6(I$(e3!g6kYEEV498OZ!Q)8ohNBAp`?+1>|b|ZbikN*<?
zv##&O3T(<Y-GPkJtdz2-{m{3eh1ojQ(q9lll920KC*dd`f+Bu*=H6xVb9-iXoUi7T
zCm!%ZC%vNO`?>-_<a^MZ^wN#l)>E&|Vgy;NB(l!4g926-?h=L7oj%@qV(PUZ=-8Bc
zF)Jv5xO3J0<#j=VFU(iWhXSgUx$fK`s2&UOkbO+cBIa7PQ6+g1vN9vXcE^xiB9(hY
zT#I(4EMg0>S>o5cAJJS#R*y5uJC(Mzbj6)%7Ek~CRom_{NY;BOcMR{=+)Iy$W+o@5
zGLC;W26dyn=st#5b)f1S`o|_L;@K-l03>=YPnL}Vu)<?*bm`b6*uiGX4qW9bgVSjX
z4c{izu$Q-pQfesjrlSq%PSmjPiRb?Z-!I!Fcx2xUJDzNU`!;b3k4#Oq@K%;xL&xQ^
z#2U5CP6@dhR4ZBd4s>vJt@|zE$mRqO)UC|pz9;l7+%}KI4k}$4dGE+(G@gEarMn%|
zCi9D3DM8u9kK$7Dekt^8h^w-Oj<%KpSBINE#FNq;IW|3$(rLE14`-%WIQG8N_^j2x
zBtFA9#le^p1km_c`JXp_U_Jf#{iyFY?}ypjJ3IsHQ0v&Wq`C*c9jMHc+*mxP(Pq7C
z9Sjg$wb8%sLG=eG&+orc$6Ph{Sm%N{CW@f4(+Ni^cPei{|FIB{P;}l*e>X3II+rYM
z^}&Ti1mZNF+vLV)55?rLy>z*JZr2TnM%|aw^anlzNT#c;yT_oc@S!N(fyW=8$~u1a
zJ?(+VRvmZ%2OiJz4Dp$2-4IkA0S68R6BWeDwR5YO-Ih~&-NM|gd~rW~r$>&!;fjS*
z@5T<8&W#Ql!ILAFGc6+QU9O|ehQ2)8D;vilR5ngFE<Vn0(>=i~i#h89=WHJfLpwzq
zvc{Rvjj#OdKGlN6Zoak7>sDQ6$Fepv5OAjnE+$7@B35wD)MTrtrqggDJHnI8orkTu
zy&5Ch$2K;%GH#*!op#0y0kd^cnb>)w9j}88+z96lgT_7Xd+5&l?#i5dM>gutJJaS5
zbl!QAvlCWC!V9}Kf$j~>!cGOI$JbwBu2OhX6m-XY+>b8sYm`=&%;1%%9!C5wxhkkA
z9fMHO)KfDoB0X4J@2<G>^4&h6dtL+X_QERW!;CnUS7cDHUpftsN1sfX>g?upriK$W
zo|ft9N;Z$GLy#!t@cv1pwsD}ZI=&y>bGT`C{chT9+)dj9H?8!<`J*>Fn1=#qG=gG%
z0x2`P?t|Y#a_I{W-m10{s*W;dsH5ch>9lFM-~B{>gk2IDIA7H32Jb)hP5*9(g$G?1
z!^7UuOQKj$4|nRaWjLP2eZIi{qJ<~LRqXVOv%sbjnNuK{kU70A$bIJ}hrJn%5MN)p
zMfbP|JnZYtds$rB93|tVi#KHkF8+)OJ;HB~-a0?Or_vWWfJEtjcwg@B%O{U<7FEq0
z54Lm+4C}s>TXO@IK%N!dxH8)~JyNjWXU834>fIYK7xU!BM>noyotd|G`u+x@nQ+q`
zNWj~O;AwvbPn-6BvS)PJR>2PP61D8v;dI(0+|LfOZ1i<(WgTOay*I^YsQ3}LC7)MU
zaMtLY$uNIC-n8W*)Al<kS=X7<-E>#|I&)PR=dpL>^;RDohfnw(-M4%TZ%Wa_*{nl5
zaW?fYolS)|wmHGg@W$$fwyFI;oLFDRTqQfuSu3w_wQ|7y?oT~(*orcEX;ufFg;%TL
zk)b3JdSuA7pmz;}xwls}j@6nLlQ_Q!WLoB~_nNbPoYu6&(~arKv;+#i@tHcgAt?9A
zvbM1r(~=gENn8?_e{IVI<(W!nyc@}$(kLtGUIL#c{mc^<3tr~ASwTBj7Wxmzb2Eb8
zUAaK$>aahVIH9fW&b?lzowBt!`7B|Udt}2?5zE3;S-yL(ETK3)kN(on4<9(_K7TWr
zc(Qf+9o13AX38J9Est3>&6!Ni&^S;MjWwtr^U_$CaGW2Tw|<Agk73pY^JBZA>}`b~
zSQjWndVru3QSc3SQ$}3=>4l}7qE;WA!iliu!0soHO{eqR#1Ua#l8HpzUF+|^wRde8
z{#$$3ywQv$COzsPdc7XKj?uEUxKNWY%VU0YvGb9Jba6;npG1-cjwJqSW%;@Vn_&oi
z;=dPLPfZU3tA43yrp3GK*)v;bSxl(7e1hd^w=*?w2N-)8?sj3<Obd?*?jL2Xv#VkL
zd~4lmAa|zXfI}hiWV5UIkgL^axcxpou2H&2#I|Us1cPlk<QrRiR^z%c(_$nlUjHR{
z&$01f)kAakqcZoU!u>uz=B`hfV}?MdI@1EE;R>;G{X$rf%1>O}nvB0`_s;`Ho1BoG
zPWs8YE}ApwQDcyWxOUK^ehFkT1@uU2?j4m*!%<3vHP?1Od)#z1IUOmB04zY|S4!7T
z9WK6m1MT=MRuFWQVx4ChaPjVwa1dHtyp#A7!o?ASrMAa9;gZGK0xrg0*TkEJU|~Ia
zz1GSZc_pnr_>+Er@%#vR>`}IxIx#q#d?^Sxq+{jD`$v(QroP0AU%C;TS#|Pa)r~V*
zpV<s(#SuVbc?R&ADhf6Kds!Lopa}<3^Ex8A^crFBjope?>mSh}Y8<!o80?(El#K%d
zYcrg4!`5Bwr5xQk^X?uItlX@NLYDY134kbsCzzFc<p`8e6Z6V?v{A?ua}xVm;87#S
z-p8$zaL$YH{$?<wX+^?x=ep+K4y+x4*VIC!p}i)62x*dBa>)@#m=X#&r}LsCv%r%0
z);;?T9t`j&2#5bg6#g3pJwXtIT!iNYqugh<&eT1p9jP@tAT+_A(=6R{5`~rFaQNGY
zSF<)ccfheLj49Nup-NoFvw(9BV0a5)!uG^ZK3&hzVA0j%z=(Q$Ijjx7rwj82++a%*
z`?jcBaqud&k>bi%;m)@v{Z5@|MNx;yqvV%gFJ#<6S=Ql!1MRgM0vy1#GR^k<o-~uc
z%-YNIDeeuI{(E<3pAY=pdA`g3+~86-#bf3$f#8lh7#Y55<Iq6RsAkqx4~n(BXFL9z
z|AqTIr&r}=w>lNhPa9qh<5*`v6V99qyIwffei-B})}hDfE@jAY7kKB!`6BXnVh@-1
zkF<2Yrvm|zIuHOB(f|S=z5oy~Qxji|gRxL3zR-aH(u*J<Iif&PUv0&?xUkeAIP@s)
zGA(MmBjnJe<WRd!srSYX><*^u?yq^plVQ1o>Ch>Cwc^Pd_kS*(6lhjKZS7q_;Sxjl
z<GxGaruN;wmgE($$bWQq=W(0$I-EBkX)to=adSzaSs6XwWs|1>cdP}QQx?{px+82%
zOK9L&In3FbdT6@ta9`8+Y5`_q{b#I0&Fmu8(TA<MmbkYgV(!jU{JuL=LGe9s_lWn9
zKX>?0N6(R%=4@9~<#VePZ>3L<!hJS^cVwThIuTITba=On-#?kRezooeVeSCp<@Z3q
zem6Hy1L~zZP`_`-9e)7zT6drY_2A!k&S(TFbwar&UdKG|qhrDeMf=<wiJ>#`N8C0w
zT;Z5wA(^;;lrMO#!Ks2F{O%ACYfq7+x+)!ZnzqIBCjOASk0f(40BtCa|2hYN|A1^1
z^12e{aM&n)tgOXWw`w|jb9XnWOg%lX<G#8mzO$z(4`!^$7)RJ-^?VD53VYW#{cT=N
zXKq-d(=p}rByW?%4rzLNIPR=(yTvC^uUfPV7tYz6%0gER;>I!^H=~c3U&_tsBIeK?
z7Sg)22&91bg+@NW0ZuUwDI#(odv|A7ZnVdJddsEG9?shN)xA|ge_Iv&rd_^eNRhw#
zj?)ZNk&~AvuAeQTV8@Z9zT1jvu|)|-lR4_~CTHRIVqQs02@Q*+a33=$CFHR1cL>%`
zd${%{d(Yx8qh|Fz`78WVMitI(Z+!|D)Nj65{#1))>wkWe42i?SMELl54=+*+xvX{C
zMBIMQa>EEIYwm31Z#4Ms&l|t8LdQU(B0aFg2v<F@{Oz~_Smx*m*8zcLa^Pm+EC!`2
zmYz|H`=zHX8+W$5faRRv3++{BylJp=UeGNN_uN$Vb#_*F7-eMK!++YbF{mjYENH0j
z=nD5~Z}QWM+cDLRX2$I8nNGQDGvd7Ra;M=*=G>#5v)i}CGgR&H<EL1xVe|C?`pa1`
z;7g$E#Kj?4Fo%Vw;fybPEbC1sO*0Mj-t74+&Zrawfi0;3r$Mj#shZ=iGbh93LWX35
zq;+p)y?5{c#Qmz~$wgUZO~;IKCud~b#BKW4h&!e(qGUkxmtf~fCBIw{`s*0>cs;}3
z1q^%Vdj`Ta?~!C2Mz~@>9>=i9;~Dn%5aH^u@GSiJY(b>JiUQi`p7WI(aU(+r?|aX8
z_nOHgtDEmaW?`M~*S`^jP&jGV(T|ORcF3B}-6Ch~-5x1<w$F`^8}mm9YM)|MqLKhV
zl5n(vlQe(MWt+crd|3aWHBuMMEb%e-8ailwE~R1)cSt$iR7R<9y@S^sNj>4SPV3;>
zT|+Nbx9k1$8rogY)zRE9&ueIR{haG}WpIZi-rZH5waSLdd3ch)ruR-;xYK2ecFtJ~
z38(vRbM+cM8Xbs|3N_QQ-7lr2%%HBBCaL=@Yx%)tqN<kaDwhQ0$t)ut9d_jlaOSP_
z*zUZ&YXJ|}-UXT8oBH*-tWH^}Ac~WVaexc(n_j&!`mcvB*5L`%WXLIX9{-GKu7B(O
zfB85xoMzlee>pKdvHlE;$IqgBOG>z#qn~Z&QbXhL`Hh8_EM_Ko$19@ccY*F6_;>Ff
zel|-NIfL)^5nN2$`B>7Sc$9CJkwM43@|cP{*QHj|cx3PJ9ly~J41B|1gO-uYpH*8?
z)<-^3b+qQ<8l(^+JOGG~0aRLY>JvZqrRP5!`UJ}hp|r*a{n8qx{n8ptl-4+-O=A`R
zQs}T{E`&<+u<YRiVPW#Fu^-2A9P&k5Fw>rhkKRRV8zY(~-6<1^;n%BWcee_@K<B+{
zs)IWs#8yKI`NNl|8T{)?Q9?@xnx0ED7>O-i#OvgYyRQ6l+yyUKaHpe^2DZ&9Vh3d4
z-gr9ciDaTbwM~4)Ke=RvQAvcNawD!s3jw%xV;Io^8IJn=r1`UJ!+w3E3m|{LMbdd_
znOM}3&)G5|G`$U<O8azuq_Wa!OT0o=c6>JL@tfulX!)E;oGSoOAFKL^>C^FFuKipC
z7M;mSp7F=jUs#0OliqYzDON1{VVQVyU=4SR=Nz0PC!Aa(m98r4=$uf9K-!x`iIBCl
z>yw9(*Naya)pc=P-SC3Of-jD?_x%<=3k$@&g{E*FNxJZF7@4$T5hy&d$V+{`{b73B
z_qZEg`_<7xj;Yp|avJY83+&%0enDx=4i>AD+otYO|0^d{eAvzHHtze2X5MK_pkAus
z^o*(_)Ci{O!u&xw9H2Hl>@aE$aX5IUTZCh}F=_)*bMSw*l8f$@J#}@vh5u|EL<3?^
zsh9FBV5SP+1GQ&$hRs<=bOQl;JoMEugXItQf-HKHv?SrEk(7F;^&+mtJ>k6P{Jrvu
zA|c1WZb}Vf5PtvF2p!f&m$DDG)ZFW=?oG<cHOE`h1Lta_i5nnow;z6VR6~$?h>qlb
zRq=;DhqELojlAR`xAP$d+7SGs-$yT6NaBmYdfS?$E!HalG?I^3ghnWhFgiS!D^}-@
zXJo|Sy(D?*{7w0nA!BtR0=(plnupiO740JjIBm4d>-=}+QES{)3bLNY;P#N=7be`d
z{h0M8Wi6l9Fcj)8N+cxSThi0La38Jpa+7*_9!uts9PRIiee(b9eQ8*e=eBnC@oed-
z3cI^iR1n(jX;nrowupeyZI!la0Y#!rQK_N=2}U4746&t^wg{<PhB_cB3L-*OhCo6R
zl?s(8h)e<rkz!;BNfZdLBqaIP8?=3c6L|Xb<A)cQKU}W$KF=EObw6mM)woS0y8g|=
z;<hty(k6Nvk!d^m%%DIyc2;4(Kq^)Gz)Mm-&^|a_2s1ka#NbU6u4jObX9%A8q6+Qj
zP+N%lS^E2$f`UlwNA*p0c0+QZu$!WZ8+ib=@VKmu!IA)Y=-y}cPr*aYNB5K_B%+8X
z0CqsCYXEl4RCMr9lR2F0*b(59Po$4}&sZwxD?w-LkSdYGnb!}7u=RU!#486w5oT$j
zS)MkIjO(uX1>qQtaWyt5uEJ8G-Qw1p*7z4<NWL<w2y9ai>qbJQQCVi21M9KhxIA2X
zN^=0}SaVI>-~*_3pY^52hvax=inY|;l#P$w2rfc!v0w*QSF21)k*fn>rP!8oZGX#Y
zcGO=stuQmYyz$P>J*Cb=^dN~QzaSVZw~1hi3_)61*K&<rRgaEW`%6gULkWFi{zn&o
zEK0~fsW32_IVYhF+V}w$?om-%3|$EB=PXNm2@3{M&|;<{eNWl|#r+onb@D4)9Jn3{
z9dVZ+t$G(4=+5~CfmnU}Q#++vK!p*~vv}+ibG}Xd%goH2wUc`*4&KR8(K88FB3(va
zD3D_R@pB-NV4CJ==RspFb$tuDhdSkPw7?1@;0?Fz3$JK_<sL2(%FL^a>j<e9h(y5g
z#opa}7q1BRXHrT>Zh@t@Sbjxjk7wNX++1jEPGN3Z9L|^dLn4($Amsj`BK(q`C+a;J
zkep5=CJ(jmw)|=<Ac39&E1@_02e8u6=L>Bobr}2vs^rUnm4?_FxUivQIPOA#ME#_(
z5kmojTOL?_t-lO>;&EX|KcV9A!s4;;fHzAY%#(}5FWKTZ7m(?DB+9_RbgYAD$g{N9
zZ-RoPD&*kxhM*T`4-=dM1uni<av27hh(<YZqc;{qw}IO-vs$e9Q5rG4fUc)5vCO6k
z#y}@0481Une;h2s+q|-rjvB%;Qq+go>$z~P!2y<!!1XNOyJPuUg!Ez^Cp!A6+u62y
z>HKE)vb^%>kv2l}DeTFqASe|3ce?n+?UHz&nEB6o9$^g7<LQ^2Akf1fI07$=@2Sgl
zwc2K35}FMTVZiRVQY57_;|mbGPIBDy&`&+>^T%MB-mHCMKNFCXaC$r>)<J=Lv8M-G
zR^q%`I}aftbJooIh*7A){}od5cL1=p9j{j9o97pNf^~PT8%&X91(x;#<t@OsZX9FM
z@ndJ6K|s+=VTy;SpUaQhL<-{-i8x9stKefJksIT}l<trTX4-mi@s`QVT*Y5B)<L1&
z-+sUzT#a5#-WgG=?op%aL+l|E6u1>>zLD-6j*;#}x^KO8w09Jm_J2jaCRU2pYrY4N
z0XbdfU*8yIEcCDeR1VL4gQLxLivRjT&R+s{b+C+F6E+yWU0>}{P%sZ`;(Z>e72+5d
zKV1f2Pucd%+Nk&b5<JD373?BD%P=sD`d_SBWj0k@>y63j6078t9?IL}09d{rL(y^G
ze>JN|V5Sd%dYD#F@IF?--@HuIYVAFQZ6TG_MJ?kAlutoOFAtP~OZuYQp|eJ&`^KE2
zu^Ut8Vl`|r_}HDpKSC@ao|xrOf5ZGQG3oU^?7fTCHz{=4bm=v@88Ofa>kA_4s^y9=
zq>A===!k!60y=9SNBV_=C`3N5NWT&%w>K%w0Zb?eEFiBVH2;KEHO#N9(Y?&l5npU6
z*Lf}3oqb|F_I+(45(xkE9a&^sh$ob&BrII@>ULWXAz}+KM!uyZaqkY%+AdqmzXB_g
z2#xt(4XDZV546DA)$4fwa>PNp1^EzML1D<jn!zVy{OlWcG_FHS9Q<r?T_U0RJFHO~
z3SjaihQm`>UDyBcXq^);cuXz&6td8M{q&+~EWjbZ7P7!I;E+4+wRQGfqwJM^2<Jxv
ztqg4N+VVMIoJ>@_B1=c>w>|E>Xa1MWT)vMdD|a@l^+~mM5UaODO*i_OKc_YmwLEDA
zG6gh!bZh3Zu^TKCZ)X^09W#>oJZ%A4o^IjKI6d-+E}%uJFtbg7ubs7&(?54<)n2ER
zms<dsIA}~Xd&2}_Av7c{#Pgj?3~9+Xd3npQA6Btl8lkKZb|}o#JEhF;R_?MN)4iF%
zhy%N@Gb2gByG=sD5gQ7UPI5Syx2|xwQy;=aEhH^Vnm(kx_akr-y=^?hd&3FG_md*_
zG_!-6_0(l-QMo?emk@(hwdf*><%d)#2F<{}3Ozk0&wR5L9sh;@Wgaa(O$nBrlQzNF
zl7K~l;2xc)AT9pVeZ#4c;3_K?71Q$nkZNLPK5Z0!2{2m>{1RZc82BY@6@{Id@pK7H
zc58{KVime=F}fF3zbZIl9w!iHrg1T1K(iAtj|ky#k1TAd+?<q79sx}4m@#W3mw+0&
zJ{RDSuz~bCIHbL$pT#y5UHlvz(%kgAHkRzrTu@#vii3F}VK3IskCJVxT&=y6hk}BV
z(-J}^HysQxoaFI=Fkc}x`MOH7(mz3}ZX((^XR0&=!XB*LOMD9A;%QqYtX<$C3=OGl
zI_NtAxzC{lN0+bwJZC<wA}JYi&-Bz#w3))g2J+=64W2jxxdFa0spCoOKmMBRA*8Yc
zVtHUpOqur9y!?Vzte*FOPqA*J)ly0|In|H*GryY~26)520&n`&)QUh`XV(7c(X$ka
z6?(}<+C*3SC8seiDqW$U!a6ZCI4?iqr5$_&R(+h{+Sv!Wd{r;s(_XLUBgDXSSOZ^`
z+IR@N(ksN|?po)}nLF}Mk4NTn75-R2FY?<G5t5>{Wt;&Jk-Km64y>+T>6A-ce?UWC
zuykN7wS-N}zhd79mI3swnG(MOvbL_je_nZg@3E;^slC1mMObQuln8ktz#&^pa*gtq
zarPGWVU{*ou(YYp)K*;)sHl)dbX_d?6)O-RB^~Vv=@n8TM7o-z0EC5QvS$$2H{5eE
z`}PYnq?1mB0%je7umvk_;eLoT@zr&Tkkl+`Xzii0?Jo}Ju~QN%7nhG~HS{NsYYnl^
zs}rP^P39{7NbJMjYAoa*r*M~&kQ8Wjc2#%uH+BHH<;7{zcs3jiU=feWJS#@s!e9K9
zev+;fnZJM<bpzXv!Xf;A@<&Q<(=@>TISQODWBo^KCDdh)E`m<Ho8e}f?aN&h8&!5$
zl`C!P(-H`NSi9cIrW7@UNs&RG>eu^ioeRbk_@m28<Hko=%^LD!?fl=h_R;wUR7Nkh
zb%J{m1--?yfal!o;@9r(ghNW+Jamu3Z`3zdZrz=uZ(>;Q4=pQGd8~;+`{a384G*AM
z(S4X^{fAOMlHT#YD0s|goIt+tOlbVp-KS+)nccN48)tSwK|!{WK&p=c($^Ok<|5MI
zg?0V*l^u(_M|Y<uj#gbf8=6`cSz5m1d{$d*Wv`8qNtZAw<)Q@%Tm}yyK6Oh@OeTyj
znW2pt?EdOv?EWfrreS||{^*h!%c(PBG?#Gyc&Iq2F5(x3oODR{g(P!<Gdh%P#ZVC7
zaKHH^4=p<j^-+$cJ@H*LUcO=D=gl@jJSGd8Y-gL*^I93*8k+QBH#4RGJ4q1-%d?8Z
zIjna<fuYd+KV#`2$Ws!&GUdCU9OqKTi}L}icj|dfNp)Kpx>QPNQ88ydwmiZ)tSl=8
ze0S+cFF(`GMK?6`_V|#PFPP*YzddP=UA1x>hs}X2D9HhklprLEj;&H~4W}%X`#*z|
z@-5|(h-uPc^?z9X@R)oREc$Le?=GW9Q=;n{W~TZh+NQ>+cd!T4D+^jO0?H-L7vNK^
z!G-HXVWZaZwF}oT20tcx+*mx)eh+CD-u7m5W*eFE;8_#}Wtz@l;~AB(Kf3<4%hfNb
z+szVp{U)BNjbOx)%J?c1-{$35g|7ab+9uRMliM*pUMi1hSW`X!<;uJQ$;q7hZ9ZYK
zL-y&RCcZvTA4A^*v#4cxq1ISo_k-3aKNRFy_U+&8A{y33++X9zenmR`%4EW7R{bM2
zLMQr@l25WZzhhGuoS&B|$J<4ut>hDzE*kvMz5Use*$c41V`l_LaeN;vd-mo)S#F+U
zf#A1CWnj@PxT5oh!Vx&jASIj>a!_!VO+dfW*cFxW?5Q%7cFoiVv`BOh`AmB@7gA3$
z)^fO$kG@;ap)N4tS~ntWP5XN&h|u0Z?C=NdbOO=4%7@Wht&*p!^!v*gQ5%i4g5o7X
zTrcdX2}!9CTQ6~X`y=;&H{HH9W9GK&GAZ;y`|S)(C#+H^i#cgVpK4!Ks>!Hc8<W$B
z8%)#jR;J(Q^4^dwgSGF#+Nfxra*8m%jo?ebyp;pS>>`+QbJ8m^YUd|#@ew-~u9d%M
z*tUGyvKe<SQU{R}TCw^DBwt(GXzgf55+J=Cqip1T;sWb|8t4&?@L3BIT=7=PKlHTQ
zVL#Ud@T!R>H+@z%duX)7{giNUxqe0lymx|f&z)zf<U<!jml)x-7WrzT#23#NXBU5&
zEtIRfyEFrBP<1h3g3}%nm1N|VB4WNAMv+u$wT>pP#CoK&<LWO5{r`-c<HLhKg(oCd
z-i(Y9&y+*!o7mv&7kJd@#5d$v_KoPk)&2)=mt)OT*!IE@Ym94K21BedH*1@0`m~gu
zkJZC`vMuyv_-+w^mTER?*5XvQl3@V4E9}7YQu!Q9h3?+bjz!b5o;j(vj8+(TVpbT@
zjo{~Nb}O6u9b9ix7~MNY+_W=lx{)mx21uikv!L8Ay2rfADj({b5?HwTX5SuM12>GT
zK1sCdZr>gyH##B_<OL>T@Ktf)E+YEC(G?#zZg_1lc?nYoGp*-(8@~nF1x2Z6P@u7o
zf(caJ2sJfc@viV%ziV}eqBe8Z{Ces1*Y3K$WEHbwJLf7XRo2EbaFGyZ866L~jfc1p
zhO`5aBz_ZdY3*^_m&dIuqpT+{3Bo3~QX~RfO&dcHAc^%J4O>Rl3@$FPOmkWQOI-A?
zT{h`%Lkd7&V%VB{9nZ>3Btk#FF-cBT)Dl!gnWtm(YU?YUGe+9omoM-bkZd$f%iLrd
z&Cjw_jO<4j&|eY9UOuNl9=+9aY!$SHdn$Jd@Pe~$rMyjntBk~5DxfsDqj5RTMHKVC
zrE~r8+i%2iF?Z$Hvgt|3ddhjZ0oL?D2xq@gWPBtb5W6OY!~L9V8JPtKe<KXjx@x9o
z^gK3y`58I-{iI#?&KsXUBPd7FPgS>z;#fVJv(OFtwNYhszi{UHBKgVn$!XT;WzL1m
ztZ}LciPfNJ<{g1gp=t(tp5}7#bpvS%3iW(S&-0Wt1?YL6(i#LkzIxP(>Ti|?-pYrR
z{83(>Q7(#eSQ8^|Pk@-|X`3Ka%4>;ki@wb+9pw^u1F*o-`p$025Ir1hNUWYDewWaU
zh5g}h4Jg{s&y*@SrA(SPxehk!4Rl9;P;)HK9w#-2+rDh-S&J+}dT5rp#rH<3z|5cF
z>rw3!xV4}XtFJcxUHBB{Zxl2=DyyM@?>^e@E4YVT;MdwYv-S6TZH!LgfzvjIU6Y8e
zPH-O1DOuu7H>YDwUm9sP0U`*uZ?mhSgEi?IRwIJ2@NBnI`gYZCMj7b=+PbF&=XyB#
zQQ#kCN8c%?6&roF@gf7<-L{nMsGE7rs=i512_O7B3|4aTt>i`~z4H^L6zPD>pa&)$
zG<96ZucumhJ`)X~PuWe(P0!@&TSvSYaa;AF`}ncnVHZzvu+`jALx*8iAy^YH*O2EK
zI>Lqq)A+#$hFHx!bQH4tLru{vx#mAQ8OKigKgy!m03&g@kkoi|`*FCOQaaEJaXYW$
z5p-n@Yqb-=OFms4c)Z+?7HBO;y3$j%@~#xFwfZ=LDK!df_va8ysa@bgrf3EdzB<He
zY`-?zq8|_S*0W6o?0V-a7{Wva4|4bd-yt?+0O)RMmD1>lMl3s5K9W(rl}#{>)_n=b
z<q_<tgul^#g!1s=Le)>O$k(mzzN93lY31fA{s^zD*W1`x1{?z4Mv~Ul?1g9t=ynCP
z@Bjq&rpdzM6VYTuty5}3)x<nUJ2MgTpmDCz^a6M9;^?)f(cp|GYILd}PnQT^Q5L=?
zyf!F!1x%=NqaZ^GMv*dCVk5B>JUCp`d=&kfobC!AQwn!u`4Yo6#+z|l)&1PcYk&?#
z*)sJp7>}<qj5lLHGnS~bHIv&_XFvz`nnKX0X^d#F&wG~SO^Bz${o`*-g6iN7-%-`6
zK%A{wP{p)C3plHIql<qvBGpw~WZ9>?ffkE&0cD#=UjF)n^J3bDx=XDQ8YS41IMgN7
zu!L7c+B=f+SP^!O3Wq~ZQrI*<sDpt!V*l-<m)TYRugLpeIs9i{oOZbx`mT-(%t-if
z)#uoaF&wUSy+WbA1_Tv@pj#kauYb3bOlG~pp0BR#{VY~Huf~g!Ia2kIKB$1~9<1ud
z?o8ou`!<yoj?m$x0DP_MIJh@v#;osXVd$M0%TC3PFfhQOq+l<m#ftsfPX5?Dsi09}
zAym~lgOZF3Zl&6y#<{zVuKxY-1=MIp2ms*E1MDdqxNK%8)bIFn>vR0&cU5z5pSt@L
zUT{G#(@CoHTQn@IRYr#eCz}sPC^cTpw1G0KohpCub9zePpS)Y+4(^Mu`v$g|4Ryh;
zU01rnyBu~h((312Kw1R~uPI0_-GJ=x8`&aZov=jyQ?ucjwDAXA!jekgRf4-nJe|0S
z6fRGJ!hW;6pK;pq={YoQQAB3?nnnkSku~$G_-Cu$M3&pF=%c)1<tH$dTmuc74Fo2Z
zMI47aFgpcSZ)~X1K%mz$><XT2q1E4xoOuI=hu;sEtfTrAF!=?56WxFYb(YmT#-HmG
zTk`&PvMrh@nQ)2g6@6}ZG6nxMi~nLY6pgD`ea4;zbbB|b^g&{^t_iOsGPXj~uCKr1
z2(rojZDUg&ykP05bTr{Q_Gk(yAzdNApI%r@cmux6*(C07q(kj@kZID^?%I^*WPF(F
zyt&aSDK$?u5NH&QJb}PoEIs9VD4`-!*({xK9#H^!#Qii0!&c+3zoH+Y?}5=nf?;Kg
ziCs#7@7I)q`b$OJ9v4yBEVCbb+Vc#QTv!gF7Zl)Lqm*1hFvqoWm<{#W?0llz!?Z^K
zYTXc3)ip0w=5Ku|_G{z%4!{B^du!jab5H_pkT6eJPOs;AzX6q`TW|D99j!&87H9^9
zXb(lfP>%5PJPM7hi6awRh<wkodUt_kU)&ci_lDEydijGpzZ^tA-$dyv-5!;EMYhqj
z=|dR0VL%dY&Z}Y>i|ZngsIURa1-7@ok$BxmU14YN1uk%Iy`UZXO3tl)-SRndy_^;f
zu{9Jfp0z#RZFo+e$3<g#<~b-<+cX+~0nE9ptZ-mCbbHSgqWHyE3VkugZr`9h9PWcs
zGvKOOmK7kGoo|(8T;~Oxq}-t|m!SSq3H}n6cD%BM=xLcyBnJYw*$hn*OdD3hz^__Y
zJ1jyIuRM=)#@eYCP%{VGrKe~ls2DcG$IGGPsT62HOjx4($0Eob-Vz0K7cK3p6%YVI
zNYV2~ZZ&p#8Hqg5v49<oFNL!bCb1Q+K(>HM^Bx1j9Y}qvp67UdRhRxyDcrxe*myNn
zs_y1$2KK7;+EuUs#lC12V{BBB|7i8lwLyc{&V1d2^G7K!CY61Kz$_Il@*fI3f{%+r
zCq{HxmM~w_&QN`hc>@u2J!LOFt=KI63b_ll!*a0cxx(O(*!@NXX+Z<gQ>svcG_)ar
zrD?AElcu?9D$KWrJjlJZEJpw&`H=7fU*vhD8CerI!hy7Cn@jgee^V-PvuSjDMD)m7
zvfvpv^LPRF?@bkGE45(~Ho_}KS5Z`({%IuV0M*B(d)Y<HXh(+xS7M-|uP;45L%%2b
zAM2QZ4hY6cWO@CaQV&>p;rEdld-dvijeewbG~*k@Lg;X~<p;q%Csb!BHmC<=!fQ41
zLMYgqpVDTIdlG$p1+4URu?>JZeXU*fHK%sEYNyd040j;Wmn23Cq!DF-$<g}#@W~#6
zXmuexiqkMrdAhFmMV^CQR1oNB@9aTC+oFw~D~+s)+wBETtd9oXlD^U9UW~BigJRm{
zXWwj`3j^8QGHse}MCUQNqkx<QT@~&{t7*o^-iw={uRSe2WdYgx8n7UOh~wk;OfZbE
z8^oJ*luWyBRIcJgq17u35qlHyUif#r5IPmEarO~_nT-av@pDb?(QYa$(`IOg?*e>t
zcNtgPo>xUPww6d%E$%9(y7UdfVb}}99DE-w*zH%};0b7gGA)+J73r<i$&|gCLQS)f
z!lm8qeT=2x0jX{s{{tWF7C#+YU&;jr_j}8C<!j}P<&x}pl>VOTAqb+OgQ-D@Yi)pg
znsJtOfX%XW(}<x2F1PICpB44+&xu;fEX+#6uS}3YvZ|RiB<>S<^7+aQLft+t<oo`<
zOf2sSD0Y9OdMh$W$<Y)-52X#h+3^}vAm;<kUQW0C{F0t>6I*y&PDqn=n!_Fye!FUu
zagl~|2wLFzWSz98-P(>Ct&4<E=`Y>V^$DHq;?kQoK|bpZ4Y%c)3Jw1XUi}}P<^eP9
zc)gjQ`_eIHT$<7*J_6@r7?R@Y(&ovXaXk<2v@JFAwkYkW!P{;grK5bl{p#L&928Ub
z)TTp6G|LbwtF{@R!i7^&*bIEHfoeGMFnUvhitYQNaf<L)(u6-h$!?WDd~zGDcM+C=
z!_eMjPQ5PXDA@&8G~+vV75KyDCpVd<6;dt@K#Q%*qB<j^Ed*sfomlG8?CZ=XO7xZt
zk1`ka2vFaT=p>s3G}Y(EnIn|GB&sLe=|m_(ZQ-8X2Dwd6ousY_WglW0lrlTuQdd*x
zBWDc!$jl4IwgiR}S%{JdrA3aV-7>Cav}C<0r@|?o%*=UF9``a>JyjXCS93}<lfH%#
zhr$i_J}p0u&4^<X42SY`26H}jMkb8YS@RPT8Q!McOn0aMHh)IL{EYL)!0`TD|KKSD
zV+EO^nWrAuNGJM{NQ^K@HYx~A<53?(ll29T5|$_^z#bOQy-a85PgzE?;mYY3n&z;d
zl5DeLxPOvt8|Ycb@bQs;(B|;10ahfq0D<yLeQbd4Hed{yKaa7b&_@yNxq!_zPl;F9
zs)IHtBFQeUhF1domyD;fo|bjIjob%C7hUrqWWhziZrs*e_YY$!UR>~&>&tjf3#c5=
z9?b<^mDykyV}B%!or{3yc04T7d4*az1Y&(7vPvssE?}2r@?-mq?=ntg3m8HzT(2u4
zg^tN*Ihu&$A&(-~n8M20u9s8!8c_-o*5KQb3zWTs+qJdAdC&nt(4oA)JNE5AR6D=*
zR;}8r6k69@IO$y}uC9^*MWmTZiyWC7JbTAD-<Bv9_O6pY973S4>*%P-!fzg$R|}+H
z`n1L@<0cR8Kp}MKl(yg%oB8RfZLX4*Ols%fK5iWE;yuYpEYkg`8wsHky-A0HM2rk5
zmr)*=xmdCY(CdH-hL)3HW#;c-x*K#>ucpPL>Ng+4cr{Awc{t>+W+_9}1E~Ostb$=f
zm}-*|A^?Zm;soIz?`72+w8^S9W^-9R?C@=s_*>7hsed@$_bD>idha2&a)`)PDg-Bq
z{lCo=$x0Nqqe>a4)VKq&DaTy}0?0q-o495Y6_Juw#LJ~SZur3uM;)@qNiHqwl*78u
zkV+S-O4`mCDU+(7WC?YH2y{1%k2`Ma1_M^`(v$NLF-5u&6X_}LW<}pfbK#_rrNlp<
zp#XlavD5+2o}n0p(<PrNZuM00IC?*AzAkhicRQoppL82)hJ9m>{{41e@V&h<br5($
zmLrwZMi)d<9<5dG`%~JhAHB$-?VGCDNN3G&u5z?9T(7Uk_`$0z+k|?=#>~D%m~kW1
zhRSiYH4!62T>=-E@3ISDLT5Y%IoE2HplRV-_iDI+({oZ2iHuUwY{sKrZ{ztP4w`-_
z@MiQB-sO|~xJ}4Iq6IJ{znCF<Oh_iUc1GxV%JU+Tt^nHmYt+k5GwA1_yXQ_|8h3MW
zxYEy|cN!->ov{jz;g@A%-WAJ8{Y8ehb6!=wLiOUeJH`fOq<4u3%{KHp-9|=SxMWQ=
z@W52x?lH-?GM?1qa9&5i7w(o!sa^>=b%vr%Lqd3}I`3lIxp8UM%v)p%yslZ09CE95
z-BLJ&-%Fc@>Iuv-XAn2@0>n-l)LHqIkjMW6GZUf%m80yYombj3O$(;wj!)bjc#(_E
z;NuY#2%~!<l~#5$(~q|itm5#3*ihr47j8LdbwuKAN}VAc`n@(KKEn}QFp}0LmVu`E
zAt;6MpKrkYF=T$8ka;7rMC3RqwsQu`s7ywgUZ(x7pL6p(e7~D{?A|eoM);PA-E&#C
z5L#rK3;gCm-Y~q)=}<UEzZZ|r34xsv<oOUKbMQp(0w%FRU+h+Bh++9y3OrQg2~XIe
z<fb*>#zVG!l4#0FAM7F^+5rkz!)Bt%@aRo@4clS;_CpnNvOYn#Kb#S_m%+a%nyGyZ
z=+Q<&LR^2Z981cD3JnW_k!-$YWG&<Zp0;eRpe|F=tba01JKW@fZqdM#8?Rq=i|pu(
z=wvi!LeOnSy4Gm$T~i=i#czt+*Kd`_u0-Wve!K+e0gXA|1{TG)A<b%jj`F-RU0v<k
zG%jf=E~eDInk4OTtuU#sIpug(?R?-#Jq!KfNzzPrxt<-HHzsc!9nNfvIZuO6$t$&t
zBzh|%S-rH1n_Ek(tDtxJj3@mS&ko3vDzxh*-jFs1YKMQK1P}gBp@;fFS^l804a*RL
zDtdn_`uu?Aa7U@@fIeT;1B+UlN5mA`W0=%FVQXc2<kux_+sDCTn8S8AMG^&{{+F|^
zN;~YV)-hq!?V+}2Y;xIG$z`=4G_I$?tJ#<gNb1@CrW{WjwmH#HaZ8XT+T4d|ML)u;
z5$`?AAz#j$nB!s65m%09TvEz_YN{(;D+h$+YABLm)t_v%z6^RESZB+Uo%B^i);hAe
z=_#%QEPLt<vq5d&kKXKyuVZ!YUep<Mw^P<to&UI4H<ES*9~V=$NcV(7|Ja7jna66M
zF;=&gUXV%JnMKw1(!9Zwt|%*flEx3EL%$29ynEe?8Sf>>0TX17TA2!c-!?$9h6#%F
zkEaNe9y?jb5!tbc*fmA;BZ`F8E0DiQhicA>PU;`?1)EG~5D6s5xKr>gKQD9APw$LC
zw!MOS(|gI}>>;%$9jG=v%RbRrXpzd06pJJJVtq&tUFr+x#{Uyk4(11PFq9+$3jBUA
z<T^h+tu2wIq@7acSk3w4U5r18{GX!W3{Cj!rx%I~*Tt2HW)o#Z28|K7rzLa$`ZAYR
z+ve4tfjse8TmH5YplfjLts$yAb_3gLh^|;#^SegV?nQEJkB$CiZMp=W(ZW7%gsrw6
zX8)u9Ds7EKP6-`x*VaC^h3lT40@wkB(N|$*)f&VLA1m?vO7L!Edup11W!Yy3f%gW;
zb4&#dX<si_4E<iJ#N(IxW#3*5&xXD18L25%H+&d)B8g<3;@?R~wdfQClEN54G2#bI
zgw+Kr_R0m);Px2R(AaOe<w<ou53w^5C!{9KsI`SEE5Pa)V^10HNhKsu*K{L?`TC*m
zOGj&7skEilhB?{*iQHrIeJHNN;k16sH%~vF#iNA6;|L_cM752L{W!GmmFf?fhK$pd
zmcYULD{g`FvE%b#x2Cpo?#`c%!)qrm+aUWs>+9A4a*wOJ_-~~*LEpB_M*WjjQuWt9
zB#}O)(SJ;I#Nj?h$AAmaJ89KB1S$5BYS}|aN4B}-9<p3=T-~k?MIYT8V`1zo6jMxu
zDQ&eXu^qcoQqQ<e+)N?~&i0W7P{iU3VbD+>E5n>u!>`MyRbMNV@I;$+{C>_b32x<g
z;OS!E6gNtTMrfL5XHDG0Iuz&g?A@jp&<db7sA{eq6x%qL=)Wi{))vzlfVPR-CK;&K
z4bD|F7<^?f8~ZYZK{ga*`HGu{j+pr%4=$?UifHqGS{ZL25P^J!5*s~}`GU8i0+&Q*
zAAwdt5)??2V5`+a!!Pag)P<E@){1mY#wI;&U|xQl&=|`AVqx?F^^1(-4e_L7_h$;*
zUhBhue8m?G+rJOqnXh`tcXnH<`!r@1bd({-pHi)RuvuEE_1vuWH6HJSfc{jj_+6oV
zSZR;qCRQM4`tSViq_JkKg7aF!XKYm1wcp9q57#}0_w0ZR=w&1K@8fqNtDP(Lr}eE?
zgDl1=pw~&dNr0OpxTAl}bac+4L0*@D=t(q5Bl(DNE`DDh+7<Wj2j#LQ$>z6Ce`&fR
z_MLw{@H)5U+JV1QLr#V~y!^wU*W!ftKC`j==cNS8f6e%@`-7iXUVdk9_Xi38-f-)K
zEoLiLt?%Ay`b|m8@IcsGi?*0e@%_PTr`g}!|9eDzJmjMzYc+X8wTm-umIpn0=<M3E
z-PyI!skUbl?vm*DFN^U98&@B?P|@Qy+co2eH?=l9XieZcp7SiKrYNe3+jr{d3_|Xz
zv^gIB$lLazEtiWo=H*8%a1}*6Rv)^MzgnNXBGg%ZoENFtuNIt0mf!BYEN=Ztw|aW~
zZI3erL+czvja7wWk^bwc*!j`?fuz#AF>bvz%x%zyJkJ&U$cc4Xok6IVFI_FR=ozWa
zSh1utqVP)m4!?kNGd4$ChKp^BE^e=C^35-4I@cG`#+&b2J~Uh`=1h-o!rzs9tA`!3
zCAWojlKsoX$UvloSwGAhbTJ9`iu+LasKGotJ<D*2F@Id0_fGQ#-lB_E$(##!ha(#@
zLN?@V%b_{+UP&hjQaLS$vh&xptZ=d~Sf6O;np~oo?$>lBI5f}KU$#EFR34NvYo{iR
z=Gy2f(QKRQ7t%?QeHK8%A5plTI61JY{PLdnV;fKQoSTagX!!M#-5G-W%&4gMqc8mH
z;Ef|Ni(S-r;)jfyDVOWr8O^_lXnRo^(RbD+<~96Fn4FbQ`#!C>?_TSL#wJe691_=j
zyNPHS$vOB=>$<LUk-C^q)lCw&8EyIb-DIC0x0&QPntt%(=X_PEWL2O%kuy;2W82$r
zk~=q-m>p*xg#;X0%E~^NE6AxGZVo#kA8>HpnSEp9?!()w>Y8}}7Wzepg*WaF4Xie0
zixaN|lRMj!<V}*h!?A<T`jndVyTkjJG7i^do@Hs{!pJ4RQy#1K(FcU}!bM}1KH%w4
z;G@YZFkKz!gKBKYth}`MU%iUNT*6&C?_N)Ur}#HhuKk&p9=DQ5?O#YjoilhdiyY3j
zh1AD<T;j~EiVn4PATK+5l4`Jb!A4q6`kJw#al!>bl96pr^PO$@vj|I>9%8ypbaluX
z#fVMR($!LB{cw82@m%(LWp;m71Y2l&Y}cMtzpq>N6h*P_TuGbb?_b>Nm);*t(&zc-
zGFA(ogsRnRdWNj)<>}kFs<y$$j`daeq_~(R4s0uOfq#iBG3gqcznFJ+?77EnBJRm=
zwSW%~7jA0eyfY5$ZJVqTw|0Hj7g_U}*xfwqih0&o_~PT7m=Mw0%;yi?-F<!GciPu=
zHTpQJZ`k&97B1SEaY+7qGkW3TWDy*}mt9$pD=WY0*=`J`Kn;R?8IU_vbt$%VR{X4#
zgl(HhH_q=ZpOu!!4GC^=w?4A9Z0RRfY1~g{t*X!X(*8o<l2#63=VoDYe*U(jA1uoM
zH#b+78x(8l_-lmQU{jofI{)3W)9mhzv~05WIdhB1m2;iE2_|QV?n|BRob8zF4oa8i
z9N$yMp?&bMO4?rLbRk;cv)lWRBepYqy3mx_Q>fgr+wD>GCyj|)HNQvRm#=kt^y&O$
z=egvz{G55p!c}#L%Fkt!Z#*aO(h@MMtMfu<yOW1*UKQ2#TovW+*g|TcfB4a)z1oI<
zn%+wF^ZDa*`uX3uQn6jF=N*~N<OX`Gj#YM&j(o%JI`$1aPO}q4-g9dxuMKCLemaR9
zRZLd<8c!h&woHeAUUaGT7mK47DIUiTm+i@_Il5zKfg<RK$9Dn(gT6$P5*}8Hj~yzf
z-^Ccy6pY!@QK;cr<UYRSpD~-s2`qQ&Yg+2!+}A{&Lu&V*ysB<0H%W$Cb;Wa9UJd!M
zcQ?^#vGaMWU6lupT`ueIY4Dx8w6>=5A>5+d_eAI2GJDtcHuQ)xqHR;Hm)9qxkGkx=
zsLQr>Jy{Z#lpv}MT&FsA@Wyb<DSxMjX{{uD+f>dc(>?cZX$bdnUra9>o5|sDP2PhS
z+;86x*dV(f7D@8aEj(6N*8lm`#CO+4#n?oZ?D7d(yY66DZ%t!lyVXhV;c8}He(b2R
zKTb#Q&XwRWYxeQnwn?Ps4M+aQ-tlYr()*v1?ksh&pvc(FnuDUr#}jt2lG59lw<|^H
z1$G>RNkA=;_W?MMfk}}!sKIW_sa)@_eGd+He-s>f<O7aba$OS5HPt%O|LK)Ah&{Mt
zWqAEh?H9K`ZXMa;uUQ+*`^B{6+~gaNTXwdU#>pP_eX?D&W#FC8@aOD|Tl3N6#+B67
zJ?kraJIdb4-ILWHA#(R}p3nSM{tc%ssoMP5H=K@uuDe?5WZ6Tow}gs6>z`Z7uqjF@
zuHP#!c$jd{;davHb(33aoL7eJP0hJ0`R&KyR*$ZOEtS)%SKgM~6$g^=pIYqX2C`b6
zwDn^<Wl-lyXYhKjl`Z0OElQl1Exej4rw3E5+b<6eD=l=@i|ssfDpzi`K8-LF^S5O6
zuN^`jdCxA&{Bf(TmW1EYS!Ji$lwHZB@eZGybhQ2AkZ0!7=au~be=`9KTr4uNdg6F4
X|NRsn<{U#{{bJQupB4Oj%eVgz56LLA

literal 0
HcmV?d00001

diff --git a/docs/source/http_services.rst b/docs/source/http_services.rst
index 9596f5551ef..8e8ed1cdd19 100644
--- a/docs/source/http_services.rst
+++ b/docs/source/http_services.rst
@@ -18,7 +18,8 @@ others via a header navbar.
 - ``/tasks``: task block view with longer look-back than on /status
 - ``/system``: global stats for the cluster, CPU, memory, bandwidth, file descriptors
 - ``/profile``: flamegraph statistical profiling across the cluster
-- ``/graph``: currently processing graphs in a dependnecy tree view
+- ``/graph``: currently processing graphs in a dependency tree view
+- ``/groups``: graph layout for task groups (dependencies, memory, output type, progress, tasks status)
 - ``/info``: redirect to ``/info/main/workers.html``
 
 Scheduler HTTP
@@ -53,6 +54,7 @@ Individual bokeh plots
 - ``/individual-task-stream``
 - ``/individual-progress``
 - ``/individual-graph``
+- ``/individual-groups``
 - ``/individual-profile``
 - ``/individual-profile-server``
 - ``/individual-nbytes``

From 94254184bb4d81eca914baf374ce0763488d327e Mon Sep 17 00:00:00 2001
From: Freyam Mehta <62539811+freyam@users.noreply.github.com>
Date: Mon, 28 Jun 2021 13:14:36 +0400
Subject: [PATCH 1337/1550] Edited the path to `scheduler.py` (#4983)

---
 distributed/diagnostics/graph_layout.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/diagnostics/graph_layout.py b/distributed/diagnostics/graph_layout.py
index c971b2a870d..4a4eb483bfd 100644
--- a/distributed/diagnostics/graph_layout.py
+++ b/distributed/diagnostics/graph_layout.py
@@ -7,7 +7,7 @@ class GraphLayout(SchedulerPlugin):
     This assigns (x, y) locations to all tasks quickly and dynamically as new
     tasks are added.  This scales to a few thousand nodes.
 
-    It is commonly used with distributed/bokeh/scheduler.py::TaskGraph, which
+    It is commonly used with distributed/dashboard/components/scheduler.py::TaskGraph, which
     is rendered at /graph on the diagnostic dashboard.
     """
 

From 06af2bae42ba754589b98b4fafe81d6a83591c67 Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Tue, 29 Jun 2021 12:06:52 +0100
Subject: [PATCH 1338/1550] Add dropdown menu to access individual plots
 (#4984)

* Add dropdown menu to access individual plots

* Fix capitalization
---
 distributed/dashboard/scheduler.py   | 73 ++++++++++-----------
 distributed/http/static/css/base.css | 23 +++++++
 distributed/http/templates/base.html | 97 ++++++++++++++++------------
 3 files changed, 117 insertions(+), 76 deletions(-)

diff --git a/distributed/dashboard/scheduler.py b/distributed/dashboard/scheduler.py
index a259b481ef6..e4c609b840d 100644
--- a/distributed/dashboard/scheduler.py
+++ b/distributed/dashboard/scheduler.py
@@ -43,42 +43,6 @@
 from .core import BokehApplication
 from .worker import counters_doc
 
-template_variables = {
-    "pages": [
-        "status",
-        "workers",
-        "tasks",
-        "system",
-        "profile",
-        "graph",
-        "groups",
-        "info",
-    ]
-}
-
-if NVML_ENABLED:
-    template_variables["pages"].insert(4, "gpu")
-
-
-def connect(application, http_server, scheduler, prefix=""):
-    bokeh_app = BokehApplication(
-        applications, scheduler, prefix=prefix, template_variables=template_variables
-    )
-    application.add_application(bokeh_app)
-    bokeh_app.initialize(IOLoop.current())
-
-    bokeh_app.add_handlers(
-        r".*",
-        [
-            (
-                r"/",
-                web.RedirectHandler,
-                {"url": urljoin((prefix or "").strip("/") + "/", r"status")},
-            )
-        ],
-    )
-
-
 applications = {
     "/system": systemmonitor_doc,
     "/stealing": stealing_doc,
@@ -117,3 +81,40 @@ def connect(application, http_server, scheduler, prefix=""):
     "/individual-gpu-memory": gpu_memory_doc,
     "/individual-gpu-utilization": gpu_utilization_doc,
 }
+
+
+template_variables = {
+    "pages": [
+        "status",
+        "workers",
+        "tasks",
+        "system",
+        "profile",
+        "graph",
+        "groups",
+        "info",
+    ],
+    "plots": [x.replace("/", "") for x in applications if "individual" in x],
+}
+
+if NVML_ENABLED:
+    template_variables["pages"].insert(4, "gpu")
+
+
+def connect(application, http_server, scheduler, prefix=""):
+    bokeh_app = BokehApplication(
+        applications, scheduler, prefix=prefix, template_variables=template_variables
+    )
+    application.add_application(bokeh_app)
+    bokeh_app.initialize(IOLoop.current())
+
+    bokeh_app.add_handlers(
+        r".*",
+        [
+            (
+                r"/",
+                web.RedirectHandler,
+                {"url": urljoin((prefix or "").strip("/") + "/", r"status")},
+            )
+        ],
+    )
diff --git a/distributed/http/static/css/base.css b/distributed/http/static/css/base.css
index 4731f93973b..211cc067d67 100644
--- a/distributed/http/static/css/base.css
+++ b/distributed/http/static/css/base.css
@@ -106,3 +106,26 @@ body {
   width: 100%;
   height: 100%;
 }
+
+.dropdown {
+  position: relative;
+  display: inline-block;
+}
+
+.dropdown-content {
+  display: none;
+  position: fixed;
+  top: 40px;
+  background-color: #EEE;
+  min-width: 160px;
+  box-shadow: 0px 8px 16px 0px rgba(0,0,0,0.2);
+  z-index: 2;
+}
+
+.dropdown-content ul li {
+  float: none;
+}
+
+.dropdown:hover .dropdown-content {
+  display: block;
+}
\ No newline at end of file
diff --git a/distributed/http/templates/base.html b/distributed/http/templates/base.html
index a9f00ca17c8..b0d428fac37 100644
--- a/distributed/http/templates/base.html
+++ b/distributed/http/templates/base.html
@@ -1,11 +1,12 @@
 <!DOCTYPE html>
 <html>
-  <head>
-    <meta charset="utf-8"/>
+
+<head>
+    <meta charset="utf-8" />
     <title>Dask Diagnostic UI</title>
     <meta name='viewport' content='width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no' />
-    <link rel="stylesheet" href="statics/css/base.css"/>
-    <link rel="shortcut icon" href="statics/images/favicon.ico"/>
+    <link rel="stylesheet" href="statics/css/base.css" />
+    <link rel="shortcut icon" href="statics/images/favicon.ico" />
     {% block resources %}
     {% block css_resources %}
     {{ bokeh_css | indent(8) if bokeh_css }}
@@ -16,47 +17,63 @@
     {% block extra_resources %}
     {% endblock %}
     {% endblock %}
-          <style>
          .commit-tease,
          .user-profile-mini-avatar,
          .avatar,
          .vcard-details,
          .signup-prompt-bg {
            display: none !IMPORTANT;
          }
        </style>
         <script>
          document.addEventListener('DOMContentLoaded', function() {
            this.querySelectorAll('a').forEach(anchor => {
              anchor.addEventListener('click', e => {
                e.preventDefault();

                const redact = new URLSearchParams(window.location.search).get('redact');
                const hasExistingParams = anchor.href.includes('?');
                window.location.href = anchor.href + (hasExistingParams ? `&redact=${redact}` : `?redact=${redact}`);
              });
            });
          });
        </script>
 </head>
-  <body>
+        <style>
          .commit-tease,
          .user-profile-mini-avatar,
          .avatar,
          .vcard-details,
          .signup-prompt-bg {
            display: none !IMPORTANT;
          }
        </style>
         <script>
          document.addEventListener('DOMContentLoaded', function() {
            this.querySelectorAll('a').forEach(anchor => {
              anchor.addEventListener('click', e => {
                e.preventDefault();

                const redact = new URLSearchParams(window.location.search).get('redact');
                const hasExistingParams = anchor.href.includes('?');
                window.location.href = anchor.href + (hasExistingParams ? `&redact=${redact}` : `?redact=${redact}`);
              });
            });
          });
        </script>
 </head>
+
+<body>
     <div class="navbar" id="myTopnav">
-      <ul>
-        <li id="dask-logo">
-          <a href="https://dask.org/">
-            <img src="statics/images/dask-logo.svg"></img>
-          </a>
-        </li>
-        {% for page in pages %}
-        <li>
-          <a href="{{ page }}">{{ page|title }}</a>
-        </li>
-        {% endfor %}
-        <li id="navbar-toggle-icon">
-          <a href="javascript:void(0);" onclick="myFunction()">
-            <img src="statics/images/fa-bars.svg"></img>
-          </a>
-        </li>
-      </ul>
+        <ul>
+            <li id="dask-logo">
+                <a href="https://dask.org/">
+                    <img src="statics/images/dask-logo.svg"></img>
+                </a>
+            </li>
+            {% for page in pages %}
+            <li>
+                <a href="{{ page }}">{{ page|title }}</a>
+            </li>
+            {% endfor %}
+            <div class="dropdown">
+                <li>
+                    <a href="javascript:void(0);">Plots</a>
+                </li>
+                <div class="dropdown-content">
+                    <ul>
+                        {% for plot in plots %}
+                        <li><a href="{{ plot }}">
+                                {{ " ".join(plot.split("-")[1:]).title().replace("Cpu", "CPU").replace("Gpu", "GPU") }}
+                            </a></li>
+                        {% endfor %}
+                    </ul>
+                </div>
+            </div>
+            <li id="navbar-toggle-icon">
+                <a href="javascript:void(0);" onclick="myFunction()">
+                    <img src="statics/images/fa-bars.svg"></img>
+                </a>
+            </li>
+        </ul>
     </div>
     <div class="content">
-      {% block content %}
-      {% endblock %}
+        {% block content %}
+        {% endblock %}
     </div>
     <script>
-      /* Add the "active" class to the current navbar li element */
-      var active = document.querySelectorAll("a[href='" + location.pathname + "']");
-      if (active.length > 0) {
-        active[0].closest('li').classList.add('active')
-      }
+        /* Add the "active" class to the current navbar li element */
+        var active = document.querySelectorAll("a[href='" + location.pathname + "']");
+        if (active.length > 0) {
+            active[0].closest('li').classList.add('active')
+        }
 
-      /* Toggle between adding and removing the "responsive" class to navbar when the user clicks on the icon */
-      function myFunction() {
-        var x = document.getElementById("myTopnav");
-        if (x.className === "navbar") {
-          x.className += " responsive";
-        } else {
-          x.className = "navbar";
+        /* Toggle between adding and removing the "responsive" class to navbar when the user clicks on the icon */
+        function myFunction() {
+            var x = document.getElementById("myTopnav");
+            if (x.className === "navbar") {
+                x.className += " responsive";
+            } else {
+                x.className = "navbar";
+            }
         }
-      }
     </script>
-  </body>
-</html>
+</body>
+
+</html>
\ No newline at end of file

From d419e41952c4da376e584c0874dbf91014c14ec8 Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Tue, 29 Jun 2021 15:57:47 +0200
Subject: [PATCH 1339/1550] Ensure shuffle split operations are blacklisted
 from work stealing (#4964)

If shuffle split tasks are not blacklisted from work stealing, this can have
catastrophic effects on performance.
See also https://github.com/dask/distributed/issues/4962
---
 distributed/stealing.py         |  2 +-
 distributed/tests/test_steal.py | 35 +++++++++++++++++++++++++++++++++
 2 files changed, 36 insertions(+), 1 deletion(-)

diff --git a/distributed/stealing.py b/distributed/stealing.py
index 1929661abc5..e3398b4c9a1 100644
--- a/distributed/stealing.py
+++ b/distributed/stealing.py
@@ -453,4 +453,4 @@ def _can_steal(thief, ts, victim):
     return True
 
 
-fast_tasks = {"shuffle-split"}
+fast_tasks = {"split-shuffle"}
diff --git a/distributed/tests/test_steal.py b/distributed/tests/test_steal.py
index ee2695cea87..fbabd2a6086 100644
--- a/distributed/tests/test_steal.py
+++ b/distributed/tests/test_steal.py
@@ -827,3 +827,38 @@ async def test_balance_with_longer_task(c, s, a, b):
     )  # a task after y, suggesting a, but open to b
     await z
     assert z.key in b.data
+
+
+@gen_cluster(client=True)
+async def test_blacklist_shuffle_split(c, s, a, b):
+
+    pd = pytest.importorskip("pandas")
+    dd = pytest.importorskip("dask.dataframe")
+    npart = 10
+    df = dd.from_pandas(pd.DataFrame({"A": range(100), "B": 1}), npartitions=npart)
+    graph = df.shuffle(
+        "A",
+        shuffle="tasks",
+        # If we don't have enough partitions, we'll fall back to a simple shuffle
+        max_branch=npart - 1,
+    ).sum()
+    res = c.compute(graph)
+
+    while not s.tasks:
+        await asyncio.sleep(0.005)
+    prefixes = set(s.task_prefixes.keys())
+    from distributed.stealing import fast_tasks
+
+    blacklisted = fast_tasks & prefixes
+    assert blacklisted
+    assert any(["split" in prefix for prefix in blacklisted])
+
+    stealable = s.extensions["stealing"].stealable
+    while not res.done():
+        for tasks_per_level in stealable.values():
+            for tasks in tasks_per_level:
+                for ts in tasks:
+                    assert ts.prefix.name not in fast_tasks
+                    assert "split" not in ts.prefix.name
+        await asyncio.sleep(0.001)
+    await res

From b9d2e3bc151c21ed96975f5c71b8c1d5ece6fc8a Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 29 Jun 2021 06:58:54 -0700
Subject: [PATCH 1340/1550] Add maximum shard size to config (#4986)

---
 distributed/distributed-schema.yaml | 9 +++++++++
 distributed/distributed.yaml        | 1 +
 distributed/protocol/utils.py       | 4 +++-
 3 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/distributed/distributed-schema.yaml b/distributed/distributed-schema.yaml
index 80f7adce25f..440a39fb2f0 100644
--- a/distributed/distributed-schema.yaml
+++ b/distributed/distributed-schema.yaml
@@ -629,6 +629,15 @@ properties:
               This is useful if you want to include serialization in profiling data,
               or if you have data types that are particularly sensitive to deserialization
 
+          shard:
+            type: string
+            description: |
+              The maximum size of a frame to send through a comm
+
+              Some network infrastructure doesn't like sending through very large messages.
+              Dask comms will cut up these large messages into many small ones.
+              This attribute determines the maximum size of such a shard.
+
           socket-backlog:
             type: integer
             description: |
diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index f948c45fd99..9ea6360b49f 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -165,6 +165,7 @@ distributed:
          min: 1s  # the first non-zero delay between re-tries
          max: 20s  # the maximum delay between re-tries
     compression: auto
+    shard: 64MiB
     offload: 10MiB # Size after which we choose to offload serialization to another thread
     default-scheme: tcp
     socket-backlog: 2048
diff --git a/distributed/protocol/utils.py b/distributed/protocol/utils.py
index 25ccce7c9f6..3f5a2f8f500 100644
--- a/distributed/protocol/utils.py
+++ b/distributed/protocol/utils.py
@@ -1,8 +1,10 @@
 import struct
 
+import dask
+
 from ..utils import nbytes
 
-BIG_BYTES_SHARD_SIZE = 2 ** 26
+BIG_BYTES_SHARD_SIZE = dask.utils.parse_bytes(dask.config.get("distributed.comm.shard"))
 
 
 msgpack_opts = {

From dbb13ecf3c78f6ad301c8c40b18cebdef71789bf Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Tue, 29 Jun 2021 16:02:56 +0200
Subject: [PATCH 1341/1550] No longer hold dependencies of erred tasks in
 memory #4918

This is a follow up to #4784 and reduces complexity of Worker.release_key significantly.

There is one non-trivial behavioural change regarding erred tasks. Current main branch holds on to dependencies of an erred task on a worker and implements a release mechanism once that erred task is released. I implemented this recently trying to capture status quo but I'm not convinced any longer that this is the correct behaviour. It treats the erred case specially which introduces a lot of complexity. The only place where this might be of interest is if an erred task wants to be recomputed locally. Not forgetting the data keys until the erred task was released would speed up this process. However, we'd still need to potentially compute some keys and I'm inclined to strike this feature in favour of reduced complexity.
---
 distributed/tests/test_worker.py | 51 ++++++++++++++----------
 distributed/worker.py            | 66 ++++++--------------------------
 2 files changed, 41 insertions(+), 76 deletions(-)

diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 2f3a7f58ede..903241f7225 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -48,6 +48,7 @@
     mul,
     nodebug,
     slowinc,
+    slowsum,
 )
 from distributed.worker import Worker, error_message, logger, parse_memory_limit
 
@@ -2087,11 +2088,6 @@ def raise_exc(*args):
         await asyncio.sleep(0.01)
 
     expected_states = {
-        # We currently don't have a good way to actually release this memory as
-        # long as the tasks still have a dependent. We'll need to live with this
-        # memory for now
-        f.key: "memory",
-        g.key: "memory",
         res.key: "error",
     }
 
@@ -2159,6 +2155,7 @@ def raise_exc(*args):
 
     expected_states = {
         f.key: "memory",
+        g.key: "memory",
     }
 
     assert_task_states_on_worker(expected_states, a)
@@ -2166,7 +2163,6 @@ def raise_exc(*args):
     f.release()
     g.release()
 
-    # This is not happening
     for server in [s, a, b]:
         while server.tasks:
             await asyncio.sleep(0.01)
@@ -2220,13 +2216,14 @@ def raise_exc(*args):
     res.release()
     # We no longer hold any refs to f or g and B didn't have any erros. It
     # releases everything as expected
-    while a.tasks:
+    while len(a.tasks) > 1:
         await asyncio.sleep(0.01)
 
     expected_states = {
         g.key: "memory",
     }
 
+    assert_task_states_on_worker(expected_states, a)
     assert_task_states_on_worker(expected_states, b)
 
     g.release()
@@ -2283,7 +2280,6 @@ def raise_exc(*args):
     assert_task_states_on_worker(expected_states_A, a)
 
     expected_states_B = {
-        f.key: "memory",
         g.key: "memory",
         h.key: "memory",
         res.key: "error",
@@ -2301,15 +2297,6 @@ def raise_exc(*args):
 
     # B must not forget a task since all have a still valid dependent
     expected_states_B = {
-        f.key: "memory",
-        # We actually cannot hold on to G even though the graph would suggest
-        # otherwise. This is because H was only introduced as a dependency and
-        # the scheduler never told the worker how H fits into the big picture.
-        # Therefore, it thinks that G does not have any dependents anymore and
-        # releases it. Too bad. Once we have speculative task assignments this
-        # should be more exact since we should always tell the worker what's
-        # going on
-        # g.key: released,
         h.key: "memory",
         res.key: "error",
     }
@@ -2320,10 +2307,6 @@ def raise_exc(*args):
     expected_states_A = {}
     assert_task_states_on_worker(expected_states_A, a)
     expected_states_B = {
-        f.key: "memory",
-        # See above
-        # g.key: released,
-        h.key: "memory",
         res.key: "error",
     }
 
@@ -2334,3 +2317,29 @@ def raise_exc(*args):
     for server in [s, a, b]:
         while server.tasks:
             await asyncio.sleep(0.01)
+
+
+@gen_cluster(client=True, nthreads=[("127.0.0.1", x) for x in range(4)], timeout=None)
+async def test_hold_on_to_replicas(c, s, *workers):
+    f1 = c.submit(inc, 1, workers=[workers[0].address], key="f1")
+    f2 = c.submit(inc, 2, workers=[workers[1].address], key="f2")
+
+    sum_1 = c.submit(
+        slowsum, [f1, f2], delay=0.1, workers=[workers[2].address], key="sum"
+    )
+    sum_2 = c.submit(
+        slowsum, [f1, sum_1], delay=0.2, workers=[workers[3].address], key="sum_2"
+    )
+    f1.release()
+    f2.release()
+
+    while sum_2.key not in workers[3].tasks:
+        await asyncio.sleep(0.01)
+
+    while not workers[3].tasks[sum_2.key].state == "memory":
+        assert len(s.tasks[f1.key].who_has) >= 2
+        assert s.tasks[f2.key].state == "released"
+        await asyncio.sleep(0.01)
+
+    while len(workers[2].tasks) > 1:
+        await asyncio.sleep(0.01)
diff --git a/distributed/worker.py b/distributed/worker.py
index 3d5146f09e1..1f04b4cbde5 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -1658,8 +1658,7 @@ def add_task(
                     ts.dependencies.add(dep_ts)
                     dep_ts.dependents.add(ts)
 
-                if dep_ts.state in ("fetch", "flight"):
-                    # if we _need_ to grab data or are in the process
+                if dep_ts.state not in ("memory",):
                     ts.waiting_for_data.add(dep_ts.key)
 
             self.update_who_has(who_has=who_has)
@@ -1762,9 +1761,6 @@ def transition_fetch_waiting(self, ts, runspec):
             # clear `who_has` of stale info
             ts.who_has.clear()
 
-            # remove entry from dependents to avoid a spurious `gather_dep` call``
-            for dependent in ts.dependents:
-                dependent.waiting_for_data.discard(ts.key)
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
@@ -1794,9 +1790,6 @@ def transition_flight_waiting(self, ts, runspec):
             # clear `who_has` of stale info
             ts.who_has.clear()
 
-            # remove entry from dependents to avoid a spurious `gather_dep` call``
-            for dependent in ts.dependents:
-                dependent.waiting_for_data.discard(ts.key)
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
@@ -1991,6 +1984,8 @@ def transition_executing_done(self, ts, value=no_value, report=True):
                     ts.traceback = msg["traceback"]
                     ts.state = "error"
                     out = "error"
+                    for d in ts.dependents:
+                        d.waiting_for_data.add(ts.key)
 
                 # Don't release the dependency keys, but do remove them from `dependents`
                 for dependency in ts.dependencies:
@@ -2621,12 +2616,12 @@ def release_key(
 
             if self.validate:
                 assert isinstance(key, str)
-            ts = self.tasks.get(key, TaskState(key=key))
+            ts = self.tasks.get(key, None)
             # If the scheduler holds a reference which is usually the
             # case when it instructed the task to be computed here or if
             # data was scattered we must not release it unless the
             # scheduler allow us to. See also handle_delete_data and
-            if ts and ts.scheduler_holds_ref:
+            if ts is None or ts.scheduler_holds_ref:
                 return
             logger.debug(
                 "Release key %s",
@@ -2640,28 +2635,14 @@ def release_key(
                 self.log.append((key, "release-key", {"cause": cause}, reason))
             else:
                 self.log.append((key, "release-key", reason))
-            if key in self.data and not ts.dependents:
+            if key in self.data:
                 try:
                     del self.data[key]
                 except FileNotFoundError:
                     logger.error("Tried to delete %s but no file found", exc_info=True)
-            if key in self.actors and not ts.dependents:
+            if key in self.actors:
                 del self.actors[key]
 
-            # for any dependencies of key we are releasing remove task as dependent
-            for dependency in ts.dependencies:
-                dependency.dependents.discard(ts)
-
-                if not dependency.dependents and dependency.state not in (
-                    # don't boot keys that are in flight
-                    # we don't know if they're already queued up for transit
-                    # in a gather_dep callback
-                    "flight",
-                    # The same is true for already executing keys.
-                    "executing",
-                ):
-                    self.release_key(dependency.key, reason=f"Dependent {ts} released")
-
             for worker in ts.who_has:
                 self.has_what[worker].discard(ts.key)
             ts.who_has.clear()
@@ -2681,8 +2662,10 @@ def release_key(
                 # Inform the scheduler of keys which will have gone missing
                 # We are releasing them before they have completed
                 if ts.state in PROCESSING:
+                    # This path is only hit with work stealing
                     msg = {"op": "release", "key": key, "cause": cause}
                 else:
+                    # This path is only hit when calling release_key manually
                     msg = {
                         "op": "release-worker-data",
                         "keys": [key],
@@ -2691,9 +2674,8 @@ def release_key(
                 self.batched_stream.send(msg)
 
             self._notify_plugins("release_key", key, ts.state, cause, reason, report)
-            if key in self.tasks and not ts.dependents:
-                self.tasks.pop(key)
-            del ts
+            del self.tasks[key]
+
         except CommClosedError:
             pass
         except Exception as e:
@@ -2704,32 +2686,6 @@ def release_key(
                 pdb.set_trace()
             raise
 
-    def rescind_key(self, key):
-        try:
-            if self.tasks[key].state not in PENDING:
-                return
-
-            ts = self.tasks.pop(key)
-
-            # Task has been rescinded
-            # For every task that it required
-            for dependency in ts.dependencies:
-                # Remove it as a dependent
-                dependency.dependents.remove(key)
-                # If the dependent is now without purpose (no dependencies), remove it
-                if not dependency.dependents:
-                    self.release_key(
-                        dependency.key, reason="All dependent keys rescinded"
-                    )
-
-        except Exception as e:
-            logger.exception(e)
-            if LOG_PDB:
-                import pdb
-
-                pdb.set_trace()
-            raise
-
     ################
     # Execute Task #
     ################

From 661728267bed7d8ecdc039a134b247ad404e5291 Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Tue, 29 Jun 2021 15:04:46 +0100
Subject: [PATCH 1342/1550] Misc Sphinx tweaks (#4988)

---
 distributed/diagnostics/__init__.py |  9 ---------
 docs/source/api.rst                 | 12 ++++++------
 docs/source/develop.rst             | 20 ++++++++------------
 docs/source/examples/word-count.rst |  3 ++-
 docs/source/killed.rst              |  2 +-
 docs/source/worker.rst              |  5 ++++-
 6 files changed, 21 insertions(+), 30 deletions(-)

diff --git a/distributed/diagnostics/__init__.py b/distributed/diagnostics/__init__.py
index 390a7b94f39..b286654974c 100644
--- a/distributed/diagnostics/__init__.py
+++ b/distributed/diagnostics/__init__.py
@@ -1,11 +1,2 @@
-from contextlib import suppress
-
 from .graph_layout import GraphLayout
 from .plugin import SchedulerPlugin
-
-with suppress(ImportError):
-    from .progressbar import progress
-with suppress(ImportError):
-    from .resource_monitor import Occupancy
-with suppress(ImportError):
-    from .scheduler_widgets import scheduler_status
diff --git a/docs/source/api.rst b/docs/source/api.rst
index dc49a2d477f..b5d66b759c4 100644
--- a/docs/source/api.rst
+++ b/docs/source/api.rst
@@ -23,11 +23,11 @@ API
    rejoin
    Reschedule
 
-.. currentmodule:: distributed.recreate_exceptions
+.. currentmodule:: distributed.recreate_tasks
 
 .. autosummary::
-   ReplayExceptionClient.get_futures_error
-   ReplayExceptionClient.recreate_error_locally
+   ReplayTaskClient.recreate_task_locally
+   ReplayTaskClient.recreate_error_locally
 
 .. currentmodule:: distributed
 
@@ -56,7 +56,7 @@ API
 
 .. autosummary::
    as_completed
-   distributed.diagnostics.progress
+   distributed.diagnostics.progressbar.progress
    wait
    fire_and_forget
    futures_of
@@ -112,7 +112,7 @@ Client
 .. autoclass:: Client
    :members:
 
-.. autoclass:: distributed.recreate_exceptions.ReplayExceptionClient
+.. autoclass:: distributed.recreate_tasks.ReplayTaskClient
    :members:
 
 
@@ -151,7 +151,7 @@ Other
 .. autoclass:: as_completed
    :members:
 
-.. autofunction:: distributed.diagnostics.progress
+.. autofunction:: distributed.diagnostics.progressbar.progress
 .. autofunction:: wait
 .. autofunction:: fire_and_forget
 .. autofunction:: futures_of
diff --git a/docs/source/develop.rst b/docs/source/develop.rst
index 4d30f038b17..857008f1cbf 100644
--- a/docs/source/develop.rst
+++ b/docs/source/develop.rst
@@ -12,21 +12,17 @@ guidelines`_ in the main documentation.
 Install
 -------
 
-Clone this repository with git::
+1. Clone this repository with git::
 
-   git clone git@github.com:dask/distributed.git
-   cd distributed
-
-Install all dependencies:
-
-All OS::
+     git clone git@github.com:dask/distributed.git
+     cd distributed
 
-1. Install anaconda or miniconda
-2. ::
+2. Install anaconda or miniconda (OS-dependent)
+3. ::
 
-    conda env create --file continuous_integration/environment-3.8.yaml
-    conda activate dask-distributed
-    python -m pip install -e .
+     conda env create --file continuous_integration/environment-3.8.yaml
+     conda activate dask-distributed
+     python -m pip install -e .
 
 
 To keep a fork in sync with the upstream source::
diff --git a/docs/source/examples/word-count.rst b/docs/source/examples/word-count.rst
index ad81a45028a..50535dbf2ac 100644
--- a/docs/source/examples/word-count.rst
+++ b/docs/source/examples/word-count.rst
@@ -237,7 +237,8 @@ The complete Python script for this example is shown below:
 
    import hdfs3
    from collections import defaultdict, Counter
-   from distributed import Client, progress
+   from distributed import Client
+   from distributed.diagnostics.progressbar import progress
 
    hdfs = hdfs3.HDFileSystem('NAMENODE_HOSTNAME', port=NAMENODE_PORT)
    client = Client('SCHEDULER_IP:SCHEDULER:PORT')
diff --git a/docs/source/killed.rst b/docs/source/killed.rst
index 837ccd944b4..707adc1bec8 100644
--- a/docs/source/killed.rst
+++ b/docs/source/killed.rst
@@ -80,7 +80,7 @@ of distributed may do this automatically)
 
 For other errors, you might want to run the computation in your local client, if
 possible, or try grabbing just the task that errored and using
-:func:`recreate_error_locally <distributed.recreate_exceptions.ReplayExceptionClient.recreate_error_locally>`,
+:meth:`~distributed.recreate_tasks.ReplayTaskClient.recreate_error_locally`,
 as you would for ordinary exceptions happening during task execution.
 
 Specifically for connectivity problems (e.g., timeout exceptions in the worker
diff --git a/docs/source/worker.rst b/docs/source/worker.rst
index 20b4ab0067e..faa392d132a 100644
--- a/docs/source/worker.rst
+++ b/docs/source/worker.rst
@@ -353,7 +353,10 @@ To aggressively and automatically trim the memory in a production environment, y
 should instead set the environment variable ``MALLOC_TRIM_THRESHOLD_`` (note the final
 underscore) to 0 or a low number; see the `mallopt`_ man page for details. Reducing
 this value will increase the number of syscalls, and as a consequence may degrade
-performance. **The variable must be set before starting the ``dask-worker`` process.**
+performance.
+
+.. note::
+   The variable must be set before starting the ``dask-worker`` process.
 
 jemalloc
 ~~~~~~~~

From 5dc591bbdd4427fe49fe90338a34fc85ee35f2c9 Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Tue, 29 Jun 2021 17:51:22 +0200
Subject: [PATCH 1343/1550] Pyupgrade (#4741)

* Add pyupgrade to pre-commit

* Add minimal py37 to black formatter

* Apply pyupgrade

* Remove pyupgrade from pre-commit config
---
 .pre-commit-config.yaml                       |  2 ++
 distributed/actor.py                          |  2 +-
 distributed/cli/dask_scheduler.py             |  4 ++--
 distributed/cli/dask_ssh.py                   |  6 ++---
 distributed/cli/dask_worker.py                |  4 ++--
 distributed/client.py                         | 21 +++++++---------
 distributed/comm/addressing.py                | 10 ++++----
 distributed/comm/core.py                      |  8 +++----
 distributed/comm/inproc.py                    |  6 ++---
 distributed/comm/tcp.py                       | 20 +++++++---------
 distributed/comm/tests/test_comms.py          |  6 ++---
 distributed/comm/tests/test_ucx.py            |  2 +-
 distributed/comm/tests/test_ucx_config.py     |  2 +-
 distributed/comm/ucx.py                       |  4 ++--
 distributed/comm/utils.py                     |  2 +-
 distributed/core.py                           | 16 ++++++-------
 distributed/dashboard/components/nvml.py      |  6 ++---
 distributed/dashboard/components/scheduler.py | 12 +++++-----
 distributed/dashboard/components/shared.py    | 14 +++++------
 distributed/dashboard/components/worker.py    | 10 ++++----
 distributed/deploy/adaptive.py                |  4 ++--
 distributed/deploy/adaptive_core.py           |  2 +-
 distributed/deploy/old_ssh.py                 | 16 +++++--------
 distributed/deploy/spec.py                    |  2 +-
 distributed/deploy/ssh.py                     |  2 +-
 distributed/deploy/tests/test_local.py        |  2 +-
 distributed/diagnostics/progress.py           |  6 ++---
 distributed/diskutils.py                      |  6 ++---
 distributed/http/proxy.py                     | 10 ++++----
 distributed/http/tests/test_core.py           |  4 +---
 distributed/locket.py                         |  4 ++--
 distributed/nanny.py                          |  2 +-
 distributed/node.py                           |  2 +-
 distributed/process.py                        | 10 ++++----
 distributed/profile.py                        |  4 ++--
 distributed/protocol/core.py                  |  2 +-
 distributed/publish.py                        |  3 +--
 distributed/pubsub.py                         |  4 ++--
 distributed/pytest_resourceleaks.py           |  9 ++++---
 distributed/queues.py                         |  2 +-
 distributed/scheduler.py                      | 12 +++++-----
 distributed/security.py                       |  4 ++--
 distributed/tests/make_tls_certs.py           |  6 ++---
 distributed/tests/test_asyncprocess.py        |  6 ++---
 distributed/tests/test_client.py              | 18 +++++++-------
 distributed/tests/test_diskutils.py           |  2 +-
 distributed/tests/test_failed_workers.py      |  2 +-
 distributed/tests/test_scheduler.py           |  6 ++---
 distributed/tests/test_security.py            |  2 +-
 distributed/tests/test_semaphore.py           |  4 ++--
 distributed/tests/test_steal.py               |  2 +-
 distributed/tests/test_utils.py               |  4 ++--
 distributed/tests/test_utils_comm.py          |  4 ++--
 distributed/utils.py                          | 24 +++++++++----------
 distributed/utils_comm.py                     |  4 ++--
 distributed/utils_test.py                     | 14 +++++------
 distributed/versions.py                       |  9 +++----
 distributed/worker.py                         | 18 +++++++-------
 docs/source/conf.py                           |  7 ++----
 59 files changed, 189 insertions(+), 212 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 5b9075a55c0..7b497037a1a 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -10,6 +10,8 @@ repos:
       - id: black
         language_version: python3
         exclude: versioneer.py
+        args:
+          - --target-version=py37
   -   repo: https://gitlab.com/pycqa/flake8
       rev: 3.9.2
       hooks:
diff --git a/distributed/actor.py b/distributed/actor.py
index 77b2cda67de..2ebbba53a1c 100644
--- a/distributed/actor.py
+++ b/distributed/actor.py
@@ -69,7 +69,7 @@ def __init__(self, cls, address, key, worker=None):
                 self._client = None
 
     def __repr__(self):
-        return "<Actor: %s, key=%s>" % (self._cls.__name__, self.key)
+        return f"<Actor: {self._cls.__name__}, key={self.key}>"
 
     def __reduce__(self):
         return (Actor, (self._cls, self._address, self.key))
diff --git a/distributed/cli/dask_scheduler.py b/distributed/cli/dask_scheduler.py
index c6297eda5a3..acb4d04198f 100755
--- a/distributed/cli/dask_scheduler.py
+++ b/distributed/cli/dask_scheduler.py
@@ -130,7 +130,7 @@ def main(
     tls_cert,
     tls_key,
     dashboard_address,
-    **kwargs
+    **kwargs,
 ):
     g0, g1, g2 = gc.get_threshold()  # https://github.com/dask/distributed/issues/1653
     gc.set_threshold(g0 * 3, g1 * 3, g2 * 3)
@@ -194,7 +194,7 @@ def del_pid_file():
         dashboard=dashboard,
         dashboard_address=dashboard_address,
         http_prefix=dashboard_prefix,
-        **kwargs
+        **kwargs,
     )
     logger.info("-" * 47)
 
diff --git a/distributed/cli/dask_ssh.py b/distributed/cli/dask_ssh.py
index f81cd73d495..8619949b588 100755
--- a/distributed/cli/dask_ssh.py
+++ b/distributed/cli/dask_ssh.py
@@ -175,10 +175,10 @@ def main(
             version=distributed.__version__
         )
     )
-    print("Worker nodes: {n}".format(n=len(hostnames)))
+    print(f"Worker nodes: {len(hostnames)}")
     for i, host in enumerate(hostnames):
-        print("  {num}: {host}".format(num=i, host=host))
-    print("\nscheduler node: {addr}:{port}".format(addr=scheduler, port=scheduler_port))
+        print(f"  {i}: {host}")
+    print(f"\nscheduler node: {scheduler}:{scheduler_port}")
     print("---------------------------------------------------------------\n\n")
 
     # Monitor the output of remote processes.  This blocks until the user issues a KeyboardInterrupt.
diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
index cc004baf631..d297ef57923 100755
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -265,7 +265,7 @@ def main(
     dashboard_address,
     worker_class,
     preload_nanny,
-    **kwargs
+    **kwargs,
 ):
     g0, g1, g2 = gc.get_threshold()  # https://github.com/dask/distributed/issues/1653
     gc.set_threshold(g0 * 3, g1 * 3, g2 * 3)
@@ -419,7 +419,7 @@ def del_pid_file():
             name=name
             if nprocs == 1 or name is None or name == ""
             else str(name) + "-" + str(i),
-            **kwargs
+            **kwargs,
         )
         for i in range(nprocs)
     ]
diff --git a/distributed/client.py b/distributed/client.py
index 51047dc34c3..6a81fd6da6d 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -7,7 +7,6 @@
 import json
 import logging
 import os
-import socket
 import sys
 import threading
 import uuid
@@ -399,9 +398,9 @@ def __repr__(self):
                 typ = self.type.__module__.split(".")[0] + "." + self.type.__name__
             except AttributeError:
                 typ = str(self.type)
-            return "<Future: %s, type: %s, key: %s>" % (self.status, typ, self.key)
+            return f"<Future: {self.status}, type: {typ}, key: {self.key}>"
         else:
-            return "<Future: %s, key: %s>" % (self.status, self.key)
+            return f"<Future: {self.status}, key: {self.key}>"
 
     def _repr_html_(self):
         text = "<b>Future: %s</b> " % html.escape(key_split(self.key))
@@ -494,7 +493,7 @@ async def wait(self, timeout=None):
         await asyncio.wait_for(self._get_event().wait(), timeout)
 
     def __repr__(self):
-        return "<%s: %s>" % (self.__class__.__name__, self.status)
+        return f"<{self.__class__.__name__}: {self.status}>"
 
 
 async def done_callback(future, callback):
@@ -660,9 +659,7 @@ def __init__(
                 logger.info("Config value `scheduler-address` found: %s", address)
 
         if address is not None and kwargs:
-            raise ValueError(
-                "Unexpected keyword arguments: {}".format(str(sorted(kwargs)))
-            )
+            raise ValueError(f"Unexpected keyword arguments: {str(sorted(kwargs))}")
 
         if isinstance(address, (rpc, PooledRPCCall)):
             self.scheduler = address
@@ -907,12 +904,12 @@ def __repr__(self):
             return text
 
         elif self.scheduler is not None:
-            return "<%s: scheduler=%r>" % (
+            return "<{}: scheduler={!r}>".format(
                 self.__class__.__name__,
                 self.scheduler.address,
             )
         else:
-            return "<%s: No scheduler connected>" % (self.__class__.__name__,)
+            return f"<{self.__class__.__name__}: No scheduler connected>"
 
     def _repr_html_(self):
         scheduler, info = self._get_scheduler_info()
@@ -1073,7 +1070,7 @@ async def _start(self, timeout=no_default, **kwargs):
                     asynchronous=self._asynchronous,
                     **self._startup_kwargs,
                 )
-            except (OSError, socket.error) as e:
+            except OSError as e:
                 if e.errno != errno.EADDRINUSE:
                     raise
                 # The default port was taken, use a random one
@@ -1123,7 +1120,7 @@ async def _reconnect(self):
                 try:
                     await self._ensure_connected(timeout=timeout)
                     break
-                except EnvironmentError:
+                except OSError:
                     # Wait a bit before retrying
                     await asyncio.sleep(0.1)
                     timeout = deadline - self.loop.time()
@@ -1203,7 +1200,7 @@ async def _update_scheduler_info(self):
             return
         try:
             self._scheduler_identity = SchedulerInfo(await self.scheduler.identity())
-        except EnvironmentError:
+        except OSError:
             logger.debug("Not able to query scheduler for identity")
 
     async def _wait_for_workers(self, n_workers=0, timeout=None):
diff --git a/distributed/comm/addressing.py b/distributed/comm/addressing.py
index 3612b284d0d..e51f2dfd71e 100644
--- a/distributed/comm/addressing.py
+++ b/distributed/comm/addressing.py
@@ -37,7 +37,7 @@ def unparse_address(scheme, loc):
     >>> unparse_address('tcp', '127.0.0.1')
     'tcp://127.0.0.1'
     """
-    return "%s://%s" % (scheme, loc)
+    return f"{scheme}://{loc}"
 
 
 def normalize_address(addr):
@@ -60,11 +60,11 @@ def parse_host_port(address, default_port=None):
         return address
 
     def _fail():
-        raise ValueError("invalid address %r" % (address,))
+        raise ValueError(f"invalid address {address!r}")
 
     def _default():
         if default_port is None:
-            raise ValueError("missing port number in address %r" % (address,))
+            raise ValueError(f"missing port number in address {address!r}")
         return default_port
 
     if "://" in address:
@@ -99,7 +99,7 @@ def unparse_host_port(host, port=None):
     if ":" in host and not host.startswith("["):
         host = "[%s]" % host
     if port is not None:
-        return "%s:%s" % (host, port)
+        return f"{host}:{port}"
     else:
         return host
 
@@ -120,7 +120,7 @@ def get_address_host_port(addr, strict=False):
         return backend.get_address_host_port(loc)
     except NotImplementedError:
         raise ValueError(
-            "don't know how to extract host and port for address %r" % (addr,)
+            f"don't know how to extract host and port for address {addr!r}"
         )
 
 
diff --git a/distributed/comm/core.py b/distributed/comm/core.py
index ccdcbb99c20..a80863155e4 100644
--- a/distributed/comm/core.py
+++ b/distributed/comm/core.py
@@ -157,9 +157,9 @@ def handshake_configuration(local, remote):
     def __repr__(self):
         clsname = self.__class__.__name__
         if self.closed():
-            return "<closed %s>" % (clsname,)
+            return f"<closed {clsname}>"
         else:
-            return "<%s %s local=%s remote=%s>" % (
+            return "<{} {} local={} remote={}>".format(
                 clsname,
                 self.name or "",
                 self.local_address,
@@ -307,7 +307,7 @@ def time_left():
             )
             await asyncio.sleep(backoff)
     else:
-        raise IOError(
+        raise OSError(
             f"Timed out trying to connect to {addr} after {timeout} s"
         ) from active_exception
 
@@ -323,7 +323,7 @@ def time_left():
     except Exception as exc:
         with suppress(Exception):
             await comm.close()
-        raise IOError(
+        raise OSError(
             f"Timed out during handshake while connecting to {addr} after {timeout} s"
         ) from exc
 
diff --git a/distributed/comm/inproc.py b/distributed/comm/inproc.py
index 7374fba188d..bc812540a5e 100644
--- a/distributed/comm/inproc.py
+++ b/distributed/comm/inproc.py
@@ -43,7 +43,7 @@ def __init__(self):
     def add_listener(self, addr, listener):
         with self.lock:
             if addr in self.listeners:
-                raise RuntimeError("already listening on %r" % (addr,))
+                raise RuntimeError(f"already listening on {addr!r}")
             self.listeners[addr] = listener
 
     def remove_listener(self, addr):
@@ -170,7 +170,7 @@ def __init__(
 
     def _get_finalizer(self):
         def finalize(write_q=self._write_q, write_loop=self._write_loop, r=repr(self)):
-            logger.warning("Closing dangling queue in %s" % (r,))
+            logger.warning(f"Closing dangling queue in {r}")
             write_loop.add_callback(write_q.put_nowait, _EOF)
 
         return finalize
@@ -296,7 +296,7 @@ def __init__(self, manager):
     async def connect(self, address, deserialize=True, **connection_args):
         listener = self.manager.get_listener_for(address)
         if listener is None:
-            raise IOError("no endpoint for inproc address %r" % (address,))
+            raise OSError(f"no endpoint for inproc address {address!r}")
 
         conn_req = ConnectionRequest(
             c2s_q=Queue(),
diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py
index 91e6af308e4..b938bd3752a 100644
--- a/distributed/comm/tcp.py
+++ b/distributed/comm/tcp.py
@@ -92,7 +92,7 @@ def set_tcp_timeout(comm):
             logger.debug("Setting TCP user timeout: %d ms", timeout * 1000)
             TCP_USER_TIMEOUT = 18  # since Linux 2.6.37
             sock.setsockopt(socket.SOL_TCP, TCP_USER_TIMEOUT, timeout * 1000)
-    except EnvironmentError as e:
+    except OSError as e:
         logger.warning("Could not set timeout on TCP stream: %s", e)
 
 
@@ -105,7 +105,7 @@ def get_stream_address(comm):
 
     try:
         return unparse_host_port(*comm.socket.getsockname()[:2])
-    except EnvironmentError:
+    except OSError:
         # Probably EBADF
         return "<closed>"
 
@@ -119,14 +119,10 @@ def convert_stream_closed_error(obj, exc):
         exc = exc.real_error
         if ssl and isinstance(exc, ssl.SSLError):
             if "UNKNOWN_CA" in exc.reason:
-                raise FatalCommClosedError(
-                    "in %s: %s: %s" % (obj, exc.__class__.__name__, exc)
-                )
-        raise CommClosedError(
-            "in %s: %s: %s" % (obj, exc.__class__.__name__, exc)
-        ) from exc
+                raise FatalCommClosedError(f"in {obj}: {exc.__class__.__name__}: {exc}")
+        raise CommClosedError(f"in {obj}: {exc.__class__.__name__}: {exc}") from exc
     else:
-        raise CommClosedError("in %s: %s" % (obj, exc)) from exc
+        raise CommClosedError(f"in {obj}: {exc}") from exc
 
 
 def _close_comm(ref):
@@ -304,7 +300,7 @@ def close(self):
                 if stream.writing():
                     yield stream.write(b"")
                 stream.socket.shutdown(socket.SHUT_RDWR)
-            except EnvironmentError:
+            except OSError:
                 pass
             finally:
                 self._finalizer.detach()
@@ -452,7 +448,7 @@ async def start(self):
                 sockets = netutil.bind_sockets(
                     self.port, address=self.ip, backlog=backlog
                 )
-            except EnvironmentError as e:
+            except OSError as e:
                 # EADDRINUSE can happen sporadically when trying to bind
                 # to an ephemeral port
                 if self.port != 0 or e.errno != errno.EADDRINUSE:
@@ -545,7 +541,7 @@ def _get_server_args(self, **connection_args):
     async def _prepare_stream(self, stream, address):
         try:
             await stream.wait_for_handshake()
-        except EnvironmentError as e:
+        except OSError as e:
             # The handshake went wrong, log and ignore
             logger.warning(
                 "Listener on %r: TLS handshake failed with remote %r: %s",
diff --git a/distributed/comm/tests/test_comms.py b/distributed/comm/tests/test_comms.py
index 277bb05916c..a9539e83621 100644
--- a/distributed/comm/tests/test_comms.py
+++ b/distributed/comm/tests/test_comms.py
@@ -862,7 +862,7 @@ async def connect(self, address, deserialize=True, **connection_args):
                 return await super().connect(address, deserialize, **connection_args)
             else:
                 self.failures += 1
-                raise IOError()
+                raise OSError()
 
     class UnreliableBackend(TCPBackend):
         _connector_class = UnreliableConnector
@@ -950,8 +950,8 @@ async def handle_comm(comm):
         listener = await listen(addr, handle_comm)
         listeners.append(listener)
 
-    assert len(set(l.listen_address for l in listeners)) == N
-    assert len(set(l.contact_address for l in listeners)) == N
+    assert len({l.listen_address for l in listeners}) == N
+    assert len({l.contact_address for l in listeners}) == N
 
     for listener in listeners:
         listener.stop()
diff --git a/distributed/comm/tests/test_ucx.py b/distributed/comm/tests/test_ucx.py
index 5daaf7e8693..ecb6b471114 100644
--- a/distributed/comm/tests/test_ucx.py
+++ b/distributed/comm/tests/test_ucx.py
@@ -79,7 +79,7 @@ def test_ucx_specific():
     # 3. Test peer_address
     # 4. Test cleanup
     async def f():
-        address = "ucx://{}:{}".format(HOST, 0)
+        address = f"ucx://{HOST}:{0}"
 
         async def handle_comm(comm):
             msg = await comm.read()
diff --git a/distributed/comm/tests/test_ucx_config.py b/distributed/comm/tests/test_ucx_config.py
index d9eeabbe3ed..09e71acbb06 100644
--- a/distributed/comm/tests/test_ucx_config.py
+++ b/distributed/comm/tests/test_ucx_config.py
@@ -76,7 +76,7 @@ def test_ucx_config_w_env_var(cleanup, loop, monkeypatch):
     dask.config.refresh()
 
     port = "13339"
-    sched_addr = "ucx://%s:%s" % (HOST, port)
+    sched_addr = f"ucx://{HOST}:{port}"
 
     with popen(
         ["dask-scheduler", "--no-dashboard", "--protocol", "ucx", "--port", port]
diff --git a/distributed/comm/ucx.py b/distributed/comm/ucx.py
index 647ed8313a3..457a5c96f5c 100644
--- a/distributed/comm/ucx.py
+++ b/distributed/comm/ucx.py
@@ -373,7 +373,7 @@ def __init__(
         comm_handler: None,
         deserialize=False,
         allow_offload=True,
-        **connection_args
+        **connection_args,
     ):
         if not address.startswith("ucx"):
             address = "ucx://" + address
@@ -525,7 +525,7 @@ def _scrub_ucx_config():
     for k, v in options.items():
         if k not in valid_ucx_vars:
             logger.debug(
-                "Key: %s with value: %s not a valid UCX configuration option" % (k, v)
+                f"Key: {k} with value: {v} not a valid UCX configuration option"
             )
 
     return options
diff --git a/distributed/comm/utils.py b/distributed/comm/utils.py
index 35f3c33ef3d..5301265caf5 100644
--- a/distributed/comm/utils.py
+++ b/distributed/comm/utils.py
@@ -87,7 +87,7 @@ def get_tcp_server_addresses(tcp_server):
     """
     sockets = list(tcp_server._sockets.values())
     if not sockets:
-        raise RuntimeError("TCP Server %r not started yet?" % (tcp_server,))
+        raise RuntimeError(f"TCP Server {tcp_server!r} not started yet?")
 
     def _look_for_family(fam):
         socks = []
diff --git a/distributed/core.py b/distributed/core.py
index 8369bab1f5f..b8c2ba46e20 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -428,7 +428,7 @@ async def handle_comm(self, comm):
                 try:
                     msg = await comm.read()
                     logger.debug("Message from %r: %s", address, msg)
-                except EnvironmentError as e:
+                except OSError as e:
                     if not sys.is_finalizing():
                         logger.debug(
                             "Lost connection to %r while reading message: %s."
@@ -517,7 +517,7 @@ async def handle_comm(self, comm):
                 if reply and not is_dont_reply:
                     try:
                         await comm.write(result, serializers=serializers)
-                    except (EnvironmentError, TypeError) as e:
+                    except (OSError, TypeError) as e:
                         logger.debug(
                             "Lost connection to %r while sending result for op %r: %s",
                             address,
@@ -579,7 +579,7 @@ async def handle_stream(self, comm, extra=None, every_cycle=[]):
                     else:
                         func()
 
-        except (CommClosedError, EnvironmentError):
+        except (CommClosedError, OSError):
             # FIXME: This is silently ignored, is this intentional?
             pass
         except Exception as e:
@@ -647,7 +647,7 @@ async def send_recv(comm, reply=True, serializers=None, deserializers=None, **kw
             response = await comm.read(deserializers=deserializers)
         else:
             response = None
-    except (EnvironmentError, CommClosedError):
+    except (OSError, CommClosedError):
         # On communication errors, we should simply close the communication
         force_close = True
         raise
@@ -763,7 +763,7 @@ async def _close_comm(comm):
                 if not comm.closed():
                     await comm.write({"op": "close", "reply": False})
                     await comm.close()
-            except EnvironmentError:
+            except OSError:
                 comm.abort()
 
         tasks = []
@@ -792,9 +792,7 @@ async def send_recv_from_rpc(**kwargs):
                 comm.name = "rpc." + key
                 result = await send_recv(comm=comm, op=key, **kwargs)
             except (RPCClosed, CommClosedError) as e:
-                raise e.__class__(
-                    "%s: while trying to call remote method %r" % (e, key)
-                )
+                raise e.__class__(f"{e}: while trying to call remote method {key!r}")
 
             self.comms[comm] = True  # mark as open
             return result
@@ -881,7 +879,7 @@ def __exit__(self, *args):
         pass
 
     def __repr__(self):
-        return "<pooled rpc to %r>" % (self.addr,)
+        return f"<pooled rpc to {self.addr!r}>"
 
 
 class ConnectionPool:
diff --git a/distributed/dashboard/components/nvml.py b/distributed/dashboard/components/nvml.py
index b5c5547d9bd..cdb331016c6 100644
--- a/distributed/dashboard/components/nvml.py
+++ b/distributed/dashboard/components/nvml.py
@@ -49,7 +49,7 @@ def __init__(self, scheduler, width=600, **kwargs):
                 id="bk-gpu-memory-worker-plot",
                 width=int(width / 2),
                 name="gpu_memory_histogram",
-                **kwargs
+                **kwargs,
             )
             rect = memory.rect(
                 source=self.source,
@@ -67,7 +67,7 @@ def __init__(self, scheduler, width=600, **kwargs):
                 id="bk-gpu-utilization-worker-plot",
                 width=int(width / 2),
                 name="gpu_utilization_histogram",
-                **kwargs
+                **kwargs,
             )
             rect = utilization.rect(
                 source=self.source,
@@ -159,7 +159,7 @@ def update(self):
                 "escaped_worker": [escape.url_escape(w) for w in worker],
             }
 
-            self.memory_figure.title.text = "GPU Memory: %s / %s" % (
+            self.memory_figure.title.text = "GPU Memory: {} / {}".format(
                 format_bytes(sum(memory)),
                 format_bytes(memory_total),
             )
diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index f5631f78989..fe72c4e98e0 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -164,9 +164,11 @@ def update(self):
                     color.append("blue")
 
             if total:
-                self.root.title.text = "Occupancy -- total time: %s  wall time: %s" % (
-                    format_time(total),
-                    format_time(total / self.scheduler.total_nthreads),
+                self.root.title.text = (
+                    "Occupancy -- total time: {}  wall time: {}".format(
+                        format_time(total),
+                        format_time(total / self.scheduler.total_nthreads),
+                    )
                 )
             else:
                 self.root.title.text = "Occupancy"
@@ -2640,9 +2642,7 @@ def update(self):
 
         for name in self.names + self.extra_names:
             if name == "name":
-                data[name].insert(
-                    0, "Total ({nworkers})".format(nworkers=len(data[name]))
-                )
+                data[name].insert(0, f"Total ({len(data[name])})")
                 continue
             try:
                 if len(self.scheduler.workers) == 0:
diff --git a/distributed/dashboard/components/shared.py b/distributed/dashboard/components/shared.py
index 360f4433aa7..6ea83576c2e 100644
--- a/distributed/dashboard/components/shared.py
+++ b/distributed/dashboard/components/shared.py
@@ -57,7 +57,7 @@ def __init__(self, **kwargs):
             tools="",
             x_range=x_range,
             id="bk-processing-stacks-plot",
-            **kwargs
+            **kwargs,
         )
         fig.quad(
             source=self.source,
@@ -297,7 +297,7 @@ def select_cb(attr, old, new):
             ),
             self.profile_plot,
             self.ts_plot,
-            **kwargs
+            **kwargs,
         )
 
     @without_property_validation
@@ -434,7 +434,7 @@ def ts_change(attr, old, new):
             row(self.reset_button, self.update_button, sizing_mode="scale_width"),
             self.profile_plot,
             self.ts_plot,
-            **kwargs
+            **kwargs,
         )
 
     @without_property_validation
@@ -486,7 +486,7 @@ def __init__(self, worker, height=150, last_count=None, **kwargs):
             height=height,
             tools=tools,
             x_range=x_range,
-            **kwargs
+            **kwargs,
         )
         self.cpu.line(source=self.source, x="time", y="cpu")
         self.cpu.yaxis.axis_label = "Percentage"
@@ -508,7 +508,7 @@ def __init__(self, worker, height=150, last_count=None, **kwargs):
             height=height,
             tools=tools,
             x_range=x_range,
-            **kwargs
+            **kwargs,
         )
         self.mem.line(source=self.source, x="time", y="memory")
         self.mem.yaxis.axis_label = "Bytes"
@@ -530,7 +530,7 @@ def __init__(self, worker, height=150, last_count=None, **kwargs):
             height=height,
             x_range=x_range,
             tools=tools,
-            **kwargs
+            **kwargs,
         )
         self.bandwidth.line(source=self.source, x="time", y="read_bytes", color="red")
         self.bandwidth.line(source=self.source, x="time", y="write_bytes", color="blue")
@@ -549,7 +549,7 @@ def __init__(self, worker, height=150, last_count=None, **kwargs):
                 height=height,
                 x_range=x_range,
                 tools=tools,
-                **kwargs
+                **kwargs,
             )
 
             self.num_fds.line(source=self.source, x="time", y="num_fds")
diff --git a/distributed/dashboard/components/worker.py b/distributed/dashboard/components/worker.py
index e664cd4cae5..5ede5529afc 100644
--- a/distributed/dashboard/components/worker.py
+++ b/distributed/dashboard/components/worker.py
@@ -110,7 +110,7 @@ def __init__(self, worker, height=300, **kwargs):
                 y_range=y_range,
                 height=height,
                 tools="",
-                **kwargs
+                **kwargs,
             )
 
             fig.rect(
@@ -178,7 +178,7 @@ def update(self):
                         self.who[msg["who"]] = len(self.who)
                         msg["y"] = self.who[msg["who"]]
 
-                    msg["hover"] = "%s / %s = %s/s" % (
+                    msg["hover"] = "{} / {} = {}/s".format(
                         format_bytes(msg["total"]),
                         format_time(msg["duration"]),
                         format_bytes(msg["total"] / msg["duration"]),
@@ -212,7 +212,7 @@ def __init__(self, worker, **kwargs):
             height=150,
             tools="",
             x_range=x_range,
-            **kwargs
+            **kwargs,
         )
         fig.line(source=self.source, x="x", y="in", color="red")
         fig.line(source=self.source, x="x", y="out", color="blue")
@@ -250,7 +250,7 @@ def __init__(self, worker, **kwargs):
             height=150,
             tools="",
             x_range=x_range,
-            **kwargs
+            **kwargs,
         )
         fig.line(source=self.source, x="x", y="y")
 
@@ -440,7 +440,7 @@ def __init__(self, server, sizing_mode="stretch_both", **kwargs):
                     row(*pair, sizing_mode=sizing_mode)
                     for pair in partition_all(2, figures)
                 ],
-                sizing_mode=sizing_mode
+                sizing_mode=sizing_mode,
             )
 
     def add_digest_figure(self, name):
diff --git a/distributed/deploy/adaptive.py b/distributed/deploy/adaptive.py
index 8ffb8555d83..2f966a3f578 100644
--- a/distributed/deploy/adaptive.py
+++ b/distributed/deploy/adaptive.py
@@ -86,7 +86,7 @@ def __init__(
         wait_count=None,
         target_duration=None,
         worker_key=None,
-        **kwargs
+        **kwargs,
     ):
         self.cluster = cluster
         self.worker_key = worker_key
@@ -178,7 +178,7 @@ async def workers_to_close(self, target: int):
             target=target,
             key=pickle.dumps(self.worker_key) if self.worker_key else None,
             attribute="name",
-            **self._workers_to_close_kwargs
+            **self._workers_to_close_kwargs,
         )
 
     async def scale_down(self, workers):
diff --git a/distributed/deploy/adaptive_core.py b/distributed/deploy/adaptive_core.py
index b077261dc1d..64c9dd03a61 100644
--- a/distributed/deploy/adaptive_core.py
+++ b/distributed/deploy/adaptive_core.py
@@ -171,7 +171,7 @@ async def recommendations(self, target: int) -> dict:
             not_yet_arrived = requested - observed
             to_close = set()
             if not_yet_arrived:
-                to_close.update((toolz.take(len(plan) - target, not_yet_arrived)))
+                to_close.update(toolz.take(len(plan) - target, not_yet_arrived))
 
             if target < len(plan) - len(to_close):
                 L = await self.workers_to_close(target=target)
diff --git a/distributed/deploy/old_ssh.py b/distributed/deploy/old_ssh.py
index 77b01e2388f..6d158e27f05 100644
--- a/distributed/deploy/old_ssh.py
+++ b/distributed/deploy/old_ssh.py
@@ -100,7 +100,7 @@ def async_ssh(cmd_dict):
             print(
                 "               "
                 + bcolors.FAIL
-                + "Retrying... (attempt {n}/{total})".format(n=retries, total=3)
+                + f"Retrying... (attempt {retries}/{3})"
                 + bcolors.ENDC
             )
 
@@ -152,7 +152,7 @@ def read_from_stderr():
                 cmd_dict["output_queue"].put(
                     "[ {label} ] : ".format(label=cmd_dict["label"])
                     + bcolors.FAIL
-                    + "{output}".format(output=line)
+                    + f"{line}"
                     + bcolors.ENDC
                 )
                 line = stderr.readline()
@@ -215,18 +215,14 @@ def start_scheduler(
 
     # Optionally re-direct stdout and stderr to a logfile
     if logdir is not None:
-        cmd = "mkdir -p {logdir} && ".format(logdir=logdir) + cmd
+        cmd = f"mkdir -p {logdir} && " + cmd
         cmd += "&> {logdir}/dask_scheduler_{addr}:{port}.log".format(
             addr=addr, port=port, logdir=logdir
         )
 
     # Format output labels we can prepend to each line of output, and create
     # a 'status' key to keep track of jobs that terminate prematurely.
-    label = (
-        bcolors.BOLD
-        + "scheduler {addr}:{port}".format(addr=addr, port=port)
-        + bcolors.ENDC
-    )
+    label = bcolors.BOLD + f"scheduler {addr}:{port}" + bcolors.ENDC
 
     # Create a command dictionary, which contains everything we need to run and
     # interact with this command.
@@ -309,12 +305,12 @@ def start_worker(
 
     # Optionally redirect stdout and stderr to a logfile
     if logdir is not None:
-        cmd = "mkdir -p {logdir} && ".format(logdir=logdir) + cmd
+        cmd = f"mkdir -p {logdir} && " + cmd
         cmd += "&> {logdir}/dask_scheduler_{addr}.log".format(
             addr=worker_addr, logdir=logdir
         )
 
-    label = "worker {addr}".format(addr=worker_addr)
+    label = f"worker {worker_addr}"
 
     # Create a command dictionary, which contains everything we need to run and
     # interact with this command.
diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index 9c2f1292649..aa3dc3f7b84 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -97,7 +97,7 @@ async def finished(self):
         await self._event_finished.wait()
 
     def __repr__(self):
-        return "<%s: status=%s>" % (type(self).__name__, self.status)
+        return f"<{type(self).__name__}: status={self.status}>"
 
     async def __aenter__(self):
         await self
diff --git a/distributed/deploy/ssh.py b/distributed/deploy/ssh.py
index 64452c31721..492c4ce05ef 100644
--- a/distributed/deploy/ssh.py
+++ b/distributed/deploy/ssh.py
@@ -43,7 +43,7 @@ async def close(self):
         await super().close()
 
     def __repr__(self):
-        return "<SSH %s: status=%s>" % (type(self).__name__, self.status)
+        return f"<SSH {type(self).__name__}: status={self.status}>"
 
 
 class Worker(Process):
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index ee1b12913dc..13a3a16d928 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -558,7 +558,7 @@ async def test_bokeh_kwargs(cleanup):
     ) as c:
         client = AsyncHTTPClient()
         response = await client.fetch(
-            "http://localhost:{}/foo/status".format(c.scheduler.http_server.port)
+            f"http://localhost:{c.scheduler.http_server.port}/foo/status"
         )
         assert "bokeh" in response.body.decode()
 
diff --git a/distributed/diagnostics/progress.py b/distributed/diagnostics/progress.py
index 28500fd8077..40e67471259 100644
--- a/distributed/diagnostics/progress.py
+++ b/distributed/diagnostics/progress.py
@@ -229,11 +229,11 @@ def format_time(t):
     m, s = divmod(t, 60)
     h, m = divmod(m, 60)
     if h:
-        return "{0:2.0f}hr {1:2.0f}min {2:4.1f}s".format(h, m, s)
+        return f"{h:2.0f}hr {m:2.0f}min {s:4.1f}s"
     elif m:
-        return "{0:2.0f}min {1:4.1f}s".format(m, s)
+        return f"{m:2.0f}min {s:4.1f}s"
     else:
-        return "{0:4.1f}s".format(s)
+        return f"{s:4.1f}s"
 
 
 class AllProgress(SchedulerPlugin):
diff --git a/distributed/diskutils.py b/distributed/diskutils.py
index fe393872531..8d35d6a1e73 100644
--- a/distributed/diskutils.py
+++ b/distributed/diskutils.py
@@ -23,7 +23,7 @@ def is_locking_enabled():
 def safe_unlink(path):
     try:
         os.unlink(path)
-    except EnvironmentError as e:
+    except OSError as e:
         # Perhaps it was removed by someone else?
         if e.errno != errno.ENOENT:
             logger.error("Failed to remove %r", str(e))
@@ -121,7 +121,7 @@ def __init__(self, base_dir):
     def _init_workspace(self):
         try:
             os.mkdir(self.base_dir)
-        except EnvironmentError as e:
+        except OSError as e:
             if e.errno != errno.EEXIST:
                 raise
 
@@ -174,7 +174,7 @@ def _list_unknown_locks(self):
         for p in glob.glob(os.path.join(self.base_dir, "*" + DIR_LOCK_EXT)):
             try:
                 st = os.stat(p)
-            except EnvironmentError:
+            except OSError:
                 # May have been removed in the meantime
                 pass
             else:
diff --git a/distributed/http/proxy.py b/distributed/http/proxy.py
index 6e39a999990..73e1f3d42a8 100644
--- a/distributed/http/proxy.py
+++ b/distributed/http/proxy.py
@@ -24,13 +24,13 @@ async def http_get(self, port, host, proxied_path):
             self.host = host
 
             # rewrite uri for jupyter-server-proxy handling
-            uri = "/proxy/%s/%s" % (str(port), proxied_path)
+            uri = f"/proxy/{str(port)}/{proxied_path}"
             self.request.uri = uri
 
             # slash is removed during regex in handler
             proxied_path = "/%s" % proxied_path
 
-            worker = "%s:%s" % (self.host, str(port))
+            worker = f"{self.host}:{str(port)}"
             if not check_worker_dashboard_exits(self.scheduler, worker):
                 msg = "Worker <%s> does not exist" % worker
                 self.set_status(400)
@@ -81,9 +81,9 @@ def initialize(self, dask_server=None, extra=None):
             self.extra = extra or {}
 
         def get(self, port, host, proxied_path):
-            worker_url = "%s:%s/%s" % (host, str(port), proxied_path)
+            worker_url = f"{host}:{str(port)}/{proxied_path}"
             msg = """
-                <p> Try navigating to <a href=http://%s>%s</a> for your worker dashboard </p>
+                <p> Try navigating to <a href=http://{}>{}</a> for your worker dashboard </p>
 
                 <p>
                 Dask tried to proxy you to that page through your
@@ -101,7 +101,7 @@ def get(self, port, host, proxied_path):
                 but less common in production clusters.  Your IT administrators
                 will know more
                 </p>
-            """ % (
+            """.format(
                 worker_url,
                 worker_url,
             )
diff --git a/distributed/http/tests/test_core.py b/distributed/http/tests/test_core.py
index 61cb713fcf2..ea3a313525b 100644
--- a/distributed/http/tests/test_core.py
+++ b/distributed/http/tests/test_core.py
@@ -6,7 +6,5 @@
 @gen_cluster(client=True)
 async def test_scheduler(c, s, a, b):
     client = AsyncHTTPClient()
-    response = await client.fetch(
-        "http://localhost:{}/health".format(s.http_server.port)
-    )
+    response = await client.fetch(f"http://localhost:{s.http_server.port}/health")
     assert response.code == 200
diff --git a/distributed/locket.py b/distributed/locket.py
index 906938e6085..bb383345b9a 100644
--- a/distributed/locket.py
+++ b/distributed/locket.py
@@ -63,7 +63,7 @@ def _lock_file_non_blocking(file_):
         try:
             fcntl.flock(file_.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
             return True
-        except IOError as error:
+        except OSError as error:
             if error.errno in [errno.EACCES, errno.EAGAIN]:
                 return False
             else:
@@ -109,7 +109,7 @@ def _acquire_non_blocking(acquire, timeout, retry_period, path):
         if success:
             return
         elif timeout is not None and time.time() - start_time > timeout:
-            raise LockError("Couldn't lock {0}".format(path))
+            raise LockError(f"Couldn't lock {path}")
         else:
             time.sleep(retry_period)
 
diff --git a/distributed/nanny.py b/distributed/nanny.py
index bcff9eee69c..9cc4c30c6e2 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -451,7 +451,7 @@ async def _on_exit(self, exitcode):
         ):
             try:
                 await self._unregister()
-            except (EnvironmentError, CommClosedError):
+            except (OSError, CommClosedError):
                 if not self.reconnect:
                     await self.close()
                     return
diff --git a/distributed/node.py b/distributed/node.py
index e21713dc85d..a7f9b8d31ac 100644
--- a/distributed/node.py
+++ b/distributed/node.py
@@ -63,7 +63,7 @@ def start_services(self, default_listen_ip):
                 self.services[k] = service
             except Exception as e:
                 warnings.warn(
-                    "\nCould not launch service '%s' on port %s. " % (k, port)
+                    f"\nCould not launch service '{k}' on port {port}. "
                     + "Got the following message:\n\n"
                     + str(e),
                     stacklevel=3,
diff --git a/distributed/process.py b/distributed/process.py
index 1540bf3752c..9be72b07566 100644
--- a/distributed/process.py
+++ b/distributed/process.py
@@ -52,7 +52,7 @@ class AsyncProcess:
 
     def __init__(self, loop=None, target=None, name=None, args=(), kwargs={}):
         if not callable(target):
-            raise TypeError("`target` needs to be callable, not %r" % (type(target),))
+            raise TypeError(f"`target` needs to be callable, not {type(target)!r}")
         self._state = _ProcessState()
         self._loop = loop or IOLoop.current(instance=False)
 
@@ -91,7 +91,7 @@ def __init__(self, loop=None, target=None, name=None, args=(), kwargs={}):
         self._start_threads()
 
     def __repr__(self):
-        return "<%s %s>" % (self.__class__.__name__, self._name)
+        return f"<{self.__class__.__name__} {self._name}>"
 
     def _check_closed(self):
         if self._closed:
@@ -211,11 +211,11 @@ def _start():
 
             state.is_alive = True
             state.pid = process.pid
-            logger.debug("[%s] created process with pid %r" % (r, state.pid))
+            logger.debug(f"[{r}] created process with pid {state.pid!r}")
 
         while True:
             msg = q.get()
-            logger.debug("[%s] got message %r" % (r, msg))
+            logger.debug(f"[{r}] got message {msg!r}")
             op = msg["op"]
             if op == "start":
                 _call_and_set_future(loop, msg["future"], _start)
@@ -338,7 +338,7 @@ def daemon(self, value):
 def _asyncprocess_finalizer(proc):
     if proc.is_alive():
         try:
-            logger.info("reaping stray process %s" % (proc,))
+            logger.info(f"reaping stray process {proc}")
             proc.terminate()
         except OSError:
             pass
diff --git a/distributed/profile.py b/distributed/profile.py
index 160fe5a7b62..958e342754a 100644
--- a/distributed/profile.py
+++ b/distributed/profile.py
@@ -59,7 +59,7 @@ def identifier(frame):
 def repr_frame(frame):
     """Render a frame as a line for inclusion into a text traceback"""
     co = frame.f_code
-    text = '  File "%s", line %s, in %s' % (co.co_filename, frame.f_lineno, co.co_name)
+    text = f'  File "{co.co_filename}", line {frame.f_lineno}, in {co.co_name}'
     line = linecache.getline(co.co_filename, frame.f_lineno, frame.f_globals).lstrip()
     return text + "\n\t" + line
 
@@ -230,7 +230,7 @@ def traverse(state, start, stop, height):
             x += width
 
     traverse(state, 0, 1, 0)
-    percentages = ["{:.1f}%".format(100 * w) for w in widths]
+    percentages = [f"{100 * w:.1f}%" for w in widths]
     return {
         "left": starts,
         "right": stops,
diff --git a/distributed/protocol/core.py b/distributed/protocol/core.py
index 871e7e4df56..c4bd909ecb3 100644
--- a/distributed/protocol/core.py
+++ b/distributed/protocol/core.py
@@ -87,7 +87,7 @@ def _decode_default(obj):
                     frames[offset],
                     object_hook=msgpack_decode_default,
                     use_list=False,
-                    **msgpack_opts
+                    **msgpack_opts,
                 )
                 offset += 1
                 sub_frames = frames[offset : offset + sub_header["num-sub-frames"]]
diff --git a/distributed/publish.py b/distributed/publish.py
index 485b874d5f3..85150eecdb7 100644
--- a/distributed/publish.py
+++ b/distributed/publish.py
@@ -96,8 +96,7 @@ def __iter__(self):
                 "Can't invoke iter() or 'for' on client.datasets when client is "
                 "asynchronous; use 'async for' instead"
             )
-        for key in self._client.list_datasets():
-            yield key
+        yield from self._client.list_datasets()
 
     def __aiter__(self):
         if not self._client.asynchronous:
diff --git a/distributed/pubsub.py b/distributed/pubsub.py
index 91b006423ba..20822e145ca 100644
--- a/distributed/pubsub.py
+++ b/distributed/pubsub.py
@@ -350,7 +350,7 @@ def put(self, msg):
         self.loop.add_callback(self._put, msg)
 
     def __repr__(self):
-        return "<Pub: {}>".format(self.name)
+        return f"<Pub: {self.name}>"
 
     __str__ = __repr__
 
@@ -462,6 +462,6 @@ async def _put(self, msg):
             self.condition.notify()
 
     def __repr__(self):
-        return "<Sub: {}>".format(self.name)
+        return f"<Sub: {self.name}>"
 
     __str__ = __repr__
diff --git a/distributed/pytest_resourceleaks.py b/distributed/pytest_resourceleaks.py
index 185f649761c..55d94762f4c 100644
--- a/distributed/pytest_resourceleaks.py
+++ b/distributed/pytest_resourceleaks.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 """
 A pytest plugin to trace resource leaks.
 """
@@ -67,7 +66,7 @@ def pytest_configure(config):
             leaks = leaks.split(",")
         unknown = sorted(set(leaks) - set(all_checkers))
         if unknown:
-            raise ValueError("unknown resources: %r" % (unknown,))
+            raise ValueError(f"unknown resources: {unknown!r}")
 
         checkers = [all_checkers[leak]() for leak in leaks]
         checker = LeakChecker(
@@ -389,7 +388,7 @@ def pytest_runtest_protocol(self, item, nextitem):
                 unknown = sorted(set(leaking.args) - set(all_checkers))
                 if unknown:
                     raise ValueError(
-                        "pytest.mark.leaking: unknown resources %r" % (unknown,)
+                        f"pytest.mark.leaking: unknown resources {unknown!r}"
                     )
                 classes = tuple(all_checkers[a] for a in leaking.args)
                 self.skip_checkers[nodeid] = {
@@ -428,7 +427,7 @@ def pytest_report_teststatus(self, report):
                         report.outcome = "failed"
                         report.longrepr = "\n".join(
                             [
-                                "%s %s" % (nodeid, checker.format(before, after))
+                                f"{nodeid} {checker.format(before, after)}"
                                 for checker, before, after in leaks
                             ]
                         )
@@ -447,4 +446,4 @@ def pytest_terminal_summary(self, terminalreporter, exitstatus):
             for rep in leaked:
                 nodeid = rep.nodeid
                 for checker, before, after in self.leaks[nodeid]:
-                    tr.line("%s %s" % (rep.nodeid, checker.format(before, after)))
+                    tr.line(f"{rep.nodeid} {checker.format(before, after)}")
diff --git a/distributed/queues.py b/distributed/queues.py
index 481f497373c..5c81d25b848 100644
--- a/distributed/queues.py
+++ b/distributed/queues.py
@@ -46,7 +46,7 @@ def __init__(self, scheduler):
         self.scheduler.extensions["queues"] = self
 
     def create(self, comm=None, name=None, client=None, maxsize=0):
-        logger.debug("Queue name: {}".format(name))
+        logger.debug(f"Queue name: {name}")
         if name not in self.queues:
             self.queues[name] = asyncio.Queue(maxsize=maxsize)
             self.client_refcount[name] = 1
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index df7a43510b8..243f86d577d 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1546,7 +1546,7 @@ def set_nbytes(self, nbytes: Py_ssize_t):
         self._nbytes = nbytes
 
     def __repr__(self):
-        return "<TaskState %r %s>" % (self._key, self._state)
+        return f"<TaskState {self._key!r} {self._state}>"
 
     def _repr_html_(self):
         color = (
@@ -1606,7 +1606,7 @@ def __getitem__(self, key):
         return self._accessor(self._states[key])
 
     def __repr__(self):
-        return "%s(%s)" % (self.__class__, dict(self))
+        return f"{self.__class__}({dict(self)})"
 
 
 class _OptionalStateLegacyMapping(_StateLegacyMapping):
@@ -1658,7 +1658,7 @@ def __contains__(self, k):
         return st is not None and bool(self._accessor(st))
 
     def __repr__(self):
-        return "%s(%s)" % (self.__class__, set(self))
+        return f"{self.__class__}({set(self)})"
 
 
 def _legacy_task_key_set(tasks):
@@ -3746,7 +3746,7 @@ def del_scheduler_file():
 
         self.start_periodic_callbacks()
 
-        setproctitle("dask-scheduler [%s]" % (self.address,))
+        setproctitle(f"dask-scheduler [{self.address}]")
         return self
 
     async def close(self, comm=None, fast=False, close_workers=False):
@@ -6420,7 +6420,7 @@ async def feed(
                         response = function(self, state)
                     await comm.write(response)
                     await asyncio.sleep(interval)
-            except (EnvironmentError, CommClosedError):
+            except (OSError, CommClosedError):
                 pass
             finally:
                 if teardown:
@@ -6759,7 +6759,7 @@ def coerce_address(self, addr, resolve=True):
         if isinstance(addr, tuple):
             addr = unparse_host_port(*addr)
         if not isinstance(addr, str):
-            raise TypeError("addresses should be strings or tuples, got %r" % (addr,))
+            raise TypeError(f"addresses should be strings or tuples, got {addr!r}")
 
         if resolve:
             addr = resolve_address(addr)
diff --git a/distributed/security.py b/distributed/security.py
index d6a211571f1..4078590cf6b 100644
--- a/distributed/security.py
+++ b/distributed/security.py
@@ -164,14 +164,14 @@ def __repr__(self):
                     items.append((k, "..."))
                 else:
                     items.append((k, repr(val)))
-        return "Security(" + ", ".join("%s=%s" % (k, v) for k, v in items) + ")"
+        return "Security(" + ", ".join(f"{k}={v}" for k, v in items) + ")"
 
     def get_tls_config_for_role(self, role):
         """
         Return the TLS configuration for the given role, as a flat dict.
         """
         if role not in {"client", "scheduler", "worker"}:
-            raise ValueError("unknown role %r" % (role,))
+            raise ValueError(f"unknown role {role!r}")
         return {
             "ca_file": self.tls_ca_file,
             "ciphers": self.tls_ciphers,
diff --git a/distributed/tests/make_tls_certs.py b/distributed/tests/make_tls_certs.py
index 7286b780449..ac4616d7c8f 100644
--- a/distributed/tests/make_tls_certs.py
+++ b/distributed/tests/make_tls_certs.py
@@ -120,9 +120,9 @@ def make_cert_key(hostname, sign=False):
             ]
             subprocess.check_call(["openssl"] + args)
 
-        with open(cert_file, "r") as f:
+        with open(cert_file) as f:
             cert = f.read()
-        with open(key_file, "r") as f:
+        with open(key_file) as f:
             key = f.read()
         return cert, key
     finally:
@@ -203,7 +203,7 @@ def make_ca():
 
     # For certificate matching tests
     make_ca()
-    with open("tls-ca-cert.pem", "r") as f:
+    with open("tls-ca-cert.pem") as f:
         ca_cert = f.read()
 
     cert, key = make_cert_key("localhost", sign=True)
diff --git a/distributed/tests/test_asyncprocess.py b/distributed/tests/test_asyncprocess.py
index 26c695933bb..695f23cc4aa 100644
--- a/distributed/tests/test_asyncprocess.py
+++ b/distributed/tests/test_asyncprocess.py
@@ -406,7 +406,7 @@ def test_asyncprocess_child_teardown_on_parent_exit():
         # test failure.
         try:
             readable = children_alive.poll(short_timeout)
-        except EnvironmentError:
+        except OSError:
             # Windows can raise BrokenPipeError. EnvironmentError is caught for
             # Python2/3 portability.
             assert sys.platform.startswith("win"), "should only raise on windows"
@@ -423,7 +423,7 @@ def test_asyncprocess_child_teardown_on_parent_exit():
             result = children_alive.recv()
         except EOFError:
             pass  # Test passes.
-        except EnvironmentError:
+        except OSError:
             # Windows can raise BrokenPipeError. EnvironmentError is caught for
             # Python2/3 portability.
             assert sys.platform.startswith("win"), "should only raise on windows"
@@ -432,7 +432,7 @@ def test_asyncprocess_child_teardown_on_parent_exit():
             # Oops, children_alive read something. It should be closed. If
             # something was read, it's a message from the child telling us they
             # are still alive!
-            raise RuntimeError("unreachable: {}".format(result))
+            raise RuntimeError(f"unreachable: {result}")
 
     finally:
         # Cleanup.
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 93314062a70..d40b2548ff1 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -1549,7 +1549,7 @@ def g():
 
     with save_sys_modules():
         for value in [123, 456]:
-            with tmp_text("myfile.py", "def f():\n    return {}".format(value)) as fn:
+            with tmp_text("myfile.py", f"def f():\n    return {value}") as fn:
                 await c.upload_file(fn)
 
             x = c.submit(g, pure=False)
@@ -1561,7 +1561,7 @@ def g():
 async def test_upload_file_refresh_delayed(c, s, a, b):
     with save_sys_modules():
         for value in [123, 456]:
-            with tmp_text("myfile.py", "def f():\n    return {}".format(value)) as fn:
+            with tmp_text("myfile.py", f"def f():\n    return {value}") as fn:
                 await c.upload_file(fn)
 
             sys.path.append(os.path.dirname(fn))
@@ -1590,7 +1590,7 @@ def g():
         try:
             for value in [123, 456]:
                 with tmp_text(
-                    "myfile.py", "def f():\n    return {}".format(value)
+                    "myfile.py", f"def f():\n    return {value}"
                 ) as fn_my_file:
                     with zipfile.ZipFile("myfile.zip", "w") as z:
                         z.write(fn_my_file, arcname=os.path.basename(fn_my_file))
@@ -1635,13 +1635,13 @@ def g():
                 package_1 = os.path.join(dirname, "package_1")
                 os.mkdir(package_1)
                 with open(os.path.join(package_1, "__init__.py"), "w") as f:
-                    f.write("a = {}\n".format(value))
+                    f.write(f"a = {value}\n")
 
                 # test multiple top-level packages
                 package_2 = os.path.join(dirname, "package_2")
                 os.mkdir(package_2)
                 with open(os.path.join(package_2, "__init__.py"), "w") as f:
-                    f.write("b = {}\n".format(value))
+                    f.write(f"b = {value}\n")
 
                 # compile these into an egg
                 subprocess.check_call(
@@ -1887,12 +1887,12 @@ async def test_allow_restrictions(c, s, a, b):
 def test_bad_address():
     try:
         Client("123.123.123.123:1234", timeout=0.1)
-    except (IOError, TimeoutError) as e:
+    except (OSError, TimeoutError) as e:
         assert "connect" in str(e).lower()
 
     try:
         Client("127.0.0.1:1234", timeout=0.1)
-    except (IOError, TimeoutError) as e:
+    except (OSError, TimeoutError) as e:
         assert "connect" in str(e).lower()
 
 
@@ -4637,7 +4637,7 @@ async def test_client_timeout():
     await asyncio.sleep(4)
     try:
         await s
-    except EnvironmentError:  # port in use
+    except OSError:  # port in use
         await c.close()
         return
 
@@ -5155,7 +5155,7 @@ def test_get_client_no_cluster():
     Worker._instances.clear()
 
     msg = "No global client found and no address provided"
-    with pytest.raises(ValueError, match=r"^{}$".format(msg)):
+    with pytest.raises(ValueError, match=fr"^{msg}$"):
         get_client()
 
 
diff --git a/distributed/tests/test_diskutils.py b/distributed/tests/test_diskutils.py
index 8a9d99e5844..a62fe1ca4e4 100644
--- a/distributed/tests/test_diskutils.py
+++ b/distributed/tests/test_diskutils.py
@@ -162,7 +162,7 @@ def test_workspace_rmtree_failure(tmpdir):
     # shutil.rmtree() may call its onerror callback several times
     assert lines
     for line in lines:
-        assert line.startswith("Failed to remove %r" % (a.dir_path,))
+        assert line.startswith(f"Failed to remove {a.dir_path!r}")
 
 
 def test_locking_disabled(tmpdir):
diff --git a/distributed/tests/test_failed_workers.py b/distributed/tests/test_failed_workers.py
index fb3922a8b99..da23d0c48ed 100644
--- a/distributed/tests/test_failed_workers.py
+++ b/distributed/tests/test_failed_workers.py
@@ -344,7 +344,7 @@ async def test_broken_worker_during_computation(c, s, a, b):
         L = c.map(
             slowadd,
             *zip(*partition_all(2, L)),
-            key=["add-%d-%d" % (i, j) for j in range(len(L) // 2)]
+            key=["add-%d-%d" % (i, j) for j in range(len(L) // 2)],
         )
 
     await asyncio.sleep(random.random() / 20)
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 418ab7e7f5e..88b3f2de4d8 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -786,7 +786,7 @@ async def test_workers_to_close_grouped(c, s, *workers):
     def key(ws):
         return groups[ws.address]
 
-    assert set(s.workers_to_close(key=key)) == set(w.address for w in workers)
+    assert set(s.workers_to_close(key=key)) == {w.address for w in workers}
 
     # Assert that job in one worker blocks closure of group
     future = c.submit(slowinc, 1, delay=0.2, workers=workers[0].address)
@@ -2016,10 +2016,10 @@ def abort(self):
         pass
 
     def read(self, deserializers=None):
-        raise EnvironmentError
+        raise OSError
 
     def write(self, msg, serializers=None, on_error=None):
-        raise EnvironmentError
+        raise OSError
 
 
 class FlakyConnectionPool(ConnectionPool):
diff --git a/distributed/tests/test_security.py b/distributed/tests/test_security.py
index 28702650a49..305b8983752 100644
--- a/distributed/tests/test_security.py
+++ b/distributed/tests/test_security.py
@@ -395,7 +395,7 @@ def test_temporary_credentials():
     sec_repr = repr(sec)
     fields = ["tls_ca_file"]
     fields.extend(
-        "tls_%s_%s" % (role, kind)
+        f"tls_{role}_{kind}"
         for role in ["client", "scheduler", "worker"]
         for kind in ["key", "cert"]
     )
diff --git a/distributed/tests/test_semaphore.py b/distributed/tests/test_semaphore.py
index 5a5b9d02597..47678696651 100644
--- a/distributed/tests/test_semaphore.py
+++ b/distributed/tests/test_semaphore.py
@@ -283,10 +283,10 @@ def abort(self):
         pass
 
     def read(self, deserializers=None):
-        raise EnvironmentError
+        raise OSError
 
     def write(self, msg, serializers=None, on_error=None):
-        raise EnvironmentError
+        raise OSError
 
 
 class FlakyConnectionPool(ConnectionPool):
diff --git a/distributed/tests/test_steal.py b/distributed/tests/test_steal.py
index fbabd2a6086..03ee701fc50 100644
--- a/distributed/tests/test_steal.py
+++ b/distributed/tests/test_steal.py
@@ -553,7 +553,7 @@ async def assert_balanced(inp, expected, c, s, *workers):
 
         if result2 == expected2:
             return
-    raise Exception("Expected: {}; got: {}".format(str(expected2), str(result2)))
+    raise Exception(f"Expected: {str(expected2)}; got: {str(result2)}")
 
 
 @pytest.mark.parametrize(
diff --git a/distributed/tests/test_utils.py b/distributed/tests/test_utils.py
index 6f94ca3d506..be0e9847f78 100644
--- a/distributed/tests/test_utils.py
+++ b/distributed/tests/test_utils.py
@@ -146,10 +146,10 @@ def test_get_ip_interface():
     elif sys.platform.startswith("linux"):
         assert get_ip_interface("lo") == "127.0.0.1"
     else:
-        pytest.skip("test needs to be enhanced for platform %r" % (sys.platform,))
+        pytest.skip(f"test needs to be enhanced for platform {sys.platform!r}")
 
     non_existent_interface = "__non-existent-interface"
-    expected_error_message = "{!r}.+network interface.+".format(non_existent_interface)
+    expected_error_message = f"{non_existent_interface!r}.+network interface.+"
 
     if sys.platform == "darwin":
         expected_error_message += "'lo0'"
diff --git a/distributed/tests/test_utils_comm.py b/distributed/tests/test_utils_comm.py
index d17e892ebf0..ff0d6b09da6 100644
--- a/distributed/tests/test_utils_comm.py
+++ b/distributed/tests/test_utils_comm.py
@@ -56,10 +56,10 @@ def abort(self):
         pass
 
     def read(self, deserializers=None):
-        raise EnvironmentError
+        raise OSError
 
     def write(self, msg, serializers=None, on_error=None):
-        raise EnvironmentError
+        raise OSError
 
 
 class BrokenConnectionPool(ConnectionPool):
diff --git a/distributed/utils.py b/distributed/utils.py
index dcea26e628a..c691f3f8804 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -134,7 +134,7 @@ def _get_ip(host, port, family):
         sock.connect((host, port))
         ip = sock.getsockname()[0]
         return ip
-    except EnvironmentError as e:
+    except OSError as e:
         warnings.warn(
             "Couldn't detect a suitable IP address for "
             "reaching %r, defaulting to hostname: %s" % (host, e),
@@ -187,7 +187,7 @@ def get_ip_interface(ifname):
     for info in net_if_addrs[ifname]:
         if info.family == socket.AF_INET:
             return info.address
-    raise ValueError("interface %r doesn't have an IPv4 address" % (ifname,))
+    raise ValueError(f"interface {ifname!r} doesn't have an IPv4 address")
 
 
 async def All(args, quiet_exceptions=()):
@@ -317,7 +317,7 @@ def f():
     loop.add_callback(f)
     if callback_timeout is not None:
         if not e.wait(callback_timeout):
-            raise TimeoutError("timed out after %s s." % (callback_timeout,))
+            raise TimeoutError(f"timed out after {callback_timeout} s.")
     else:
         while not e.is_set():
             e.wait(10)
@@ -726,7 +726,7 @@ def validate_key(k):
     """Validate a key as received on a stream."""
     typ = type(k)
     if typ is not str and typ is not bytes:
-        raise TypeError("Unexpected key type %s (value: %r)" % (typ, k))
+        raise TypeError(f"Unexpected key type {typ} (value: {k!r})")
 
 
 def _maybe_complex(task):
@@ -1085,13 +1085,11 @@ def command_has_keyword(cmd, k):
         if isinstance(getattr(cmd, "main"), click.core.Command):
             cmd = cmd.main
         if isinstance(cmd, click.core.Command):
-            cmd_params = set(
-                [
-                    p.human_readable_name
-                    for p in cmd.params
-                    if isinstance(p, click.core.Option)
-                ]
-            )
+            cmd_params = {
+                p.human_readable_name
+                for p in cmd.params
+                if isinstance(p, click.core.Option)
+            }
             return k in cmd_params
 
     return False
@@ -1308,11 +1306,11 @@ def cli_keywords(d: dict, cls=None, cmd=None):
                     )
                 elif cls:
                     raise ValueError(
-                        "Class %s does not support keyword %s" % (typename(cls), k)
+                        f"Class {typename(cls)} does not support keyword {k}"
                     )
                 else:
                     raise ValueError(
-                        "Module %s does not support keyword %s" % (typename(cmd), k)
+                        f"Module {typename(cmd)} does not support keyword {k}"
                     )
 
     def convert_value(v):
diff --git a/distributed/utils_comm.py b/distributed/utils_comm.py
index 80e6e0b8ae4..728b4b4c144 100644
--- a/distributed/utils_comm.py
+++ b/distributed/utils_comm.py
@@ -77,7 +77,7 @@ async def gather_from_workers(who_has, rpc, close=True, serializers=None, who=No
             for worker, c in coroutines.items():
                 try:
                     r = await c
-                except EnvironmentError:
+                except OSError:
                     missing_workers.add(worker)
                 except ValueError as e:
                     logger.info(
@@ -112,7 +112,7 @@ def __init__(self, key):
         self.key = key
 
     def __repr__(self):
-        return "%s('%s')" % (type(self).__name__, self.key)
+        return f"{type(self).__name__}('{self.key}')"
 
 
 _round_robin_counter = [0]
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index 0779d7de285..6f810354681 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -981,7 +981,7 @@ def get_unclosed():
                 if getattr(w, "data", None):
                     try:
                         w.data.clear()
-                    except EnvironmentError:
+                    except OSError:
                         # zict backends can fail if their storage directory
                         # was already removed
                         pass
@@ -1075,10 +1075,10 @@ def wait_for_port(address, timeout=5):
     while True:
         timeout = deadline - time()
         if timeout < 0:
-            raise RuntimeError("Failed to connect to %s" % (address,))
+            raise RuntimeError(f"Failed to connect to {address}")
         try:
             sock = socket.create_connection(address, timeout=timeout)
-        except EnvironmentError:
+        except OSError:
             pass
         else:
             sock.close()
@@ -1092,7 +1092,7 @@ def wait_for(predicate, timeout, fail_func=None, period=0.001):
         if time() > deadline:
             if fail_func is not None:
                 fail_func()
-            pytest.fail("condition not reached until %s seconds" % (timeout,))
+            pytest.fail(f"condition not reached until {timeout} seconds")
 
 
 async def async_wait_for(predicate, timeout, fail_func=None, period=0.001):
@@ -1102,7 +1102,7 @@ async def async_wait_for(predicate, timeout, fail_func=None, period=0.001):
         if time() > deadline:
             if fail_func is not None:
                 fail_func()
-            pytest.fail("condition not reached until %s seconds" % (timeout,))
+            pytest.fail(f"condition not reached until {timeout} seconds")
 
 
 @memoize
@@ -1120,7 +1120,7 @@ def has_ipv6():
         serv.bind(("::", 0))
         serv.listen(5)
         cli = socket.create_connection(serv.getsockname()[:2])
-    except EnvironmentError:
+    except OSError:
         return False
     else:
         return True
@@ -1422,7 +1422,7 @@ def bump_rlimit(limit, desired):
         if soft < desired:
             resource.setrlimit(limit, (desired, max(hard, desired)))
     except Exception as e:
-        pytest.skip("rlimit too low (%s) and can't be increased: %s" % (soft, e))
+        pytest.skip(f"rlimit too low ({soft}) and can't be increased: {e}")
 
 
 def gen_tls_cluster(**kwargs):
diff --git a/distributed/versions.py b/distributed/versions.py
index b8d0a49a80f..13a282977bc 100644
--- a/distributed/versions.py
+++ b/distributed/versions.py
@@ -1,6 +1,5 @@
 """ utilities for package version introspection """
 
-from __future__ import absolute_import, division, print_function
 
 import importlib
 import os
@@ -25,9 +24,7 @@
 
 
 # only these scheduler packages will be checked for version mismatch
-scheduler_relevant_packages = set(pkg for pkg, _ in required_packages) | set(
-    ["lz4", "blosc"]
-)
+scheduler_relevant_packages = {pkg for pkg, _ in required_packages} | {"lz4", "blosc"}
 
 
 # notes to be displayed for mismatch packages
@@ -135,12 +132,12 @@ def error_message(scheduler, workers, client, client_name="client"):
         )
         versions.add(client_version)
 
-        worker_versions = set(
+        worker_versions = {
             workers[w].get(pkg, "MISSING")
             if isinstance(workers[w], dict)
             else workers[w]
             for w in workers
-        )
+        }
         versions |= worker_versions
 
         if len(versions) <= 1:
diff --git a/distributed/worker.py b/distributed/worker.py
index 1f04b4cbde5..fd0d32c8885 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -186,7 +186,7 @@ def __init__(self, key, runspec=None):
         self.scheduler_holds_ref = False
 
     def __repr__(self):
-        return "<Task %r %s>" % (self.key, self.state)
+        return f"<Task {self.key!r} {self.state}>"
 
     def get_nbytes(self) -> int:
         nbytes = self.nbytes
@@ -931,13 +931,13 @@ async def _register_with_scheduler(self):
                 self.scheduler_delay = response["time"] - middle
                 self.status = Status.running
                 break
-            except EnvironmentError:
+            except OSError:
                 logger.info("Waiting to connect to: %26s", self.scheduler.address)
                 await asyncio.sleep(0.1)
             except TimeoutError:
                 logger.info("Timed out when connecting to scheduler")
         if response["status"] != "OK":
-            raise ValueError("Unexpected response from register: %r" % (response,))
+            raise ValueError(f"Unexpected response from register: {response!r}")
         else:
             await asyncio.gather(
                 *[
@@ -1005,7 +1005,7 @@ async def heartbeat(self):
             logger.warning("Heartbeat to scheduler failed")
             if not self.reconnect:
                 await self.close(report=False)
-        except IOError as e:
+        except OSError as e:
             # Scheduler is gone. Respect distributed.comm.timeouts.connect
             if "Timed out trying to connect" in str(e):
                 await self.close(report=False)
@@ -1187,12 +1187,12 @@ async def start(self):
         try:
             listening_address = "%s%s:%d" % (self.listener.prefix, self.ip, self.port)
         except Exception:
-            listening_address = "%s%s" % (self.listener.prefix, self.ip)
+            listening_address = f"{self.listener.prefix}{self.ip}"
 
         logger.info("      Start worker at: %26s", self.address)
         logger.info("         Listening to: %26s", listening_address)
         for k, v in self.service_ports.items():
-            logger.info("  %16s at: %26s" % (k, self.ip + ":" + str(v)))
+            logger.info("  {:>16} at: {:>26}".format(k, self.ip + ":" + str(v)))
         logger.info("Waiting to connect to: %26s", self.scheduler.address)
         logger.info("-" * 49)
         logger.info("              Threads: %26d", self.nthreads)
@@ -1427,7 +1427,7 @@ async def get_data(
             compressed = await comm.write(msg, serializers=serializers)
             response = await comm.read(deserializers=serializers)
             assert response == "OK", response
-        except EnvironmentError:
+        except OSError:
             logger.exception(
                 "failed during get data with %s -> %s", self.address, who, exc_info=True
             )
@@ -1999,7 +1999,7 @@ def transition_executing_done(self, ts, value=no_value, report=True):
 
             return out
 
-        except EnvironmentError:
+        except OSError:
             logger.info("Comm closed")
         except Exception as e:
             logger.exception(e)
@@ -2387,7 +2387,7 @@ async def gather_dep(
                 self.incoming_count += 1
 
                 self.log.append(("receive-dep", worker, list(response["data"])))
-            except EnvironmentError:
+            except OSError:
                 logger.exception("Worker stream died during communication: %s", worker)
                 has_what = self.has_what.pop(worker)
                 self.pending_data_per_worker.pop(worker)
diff --git a/docs/source/conf.py b/docs/source/conf.py
index 201cd76b00b..ac7e07fdff9 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 #
 # Dask.distributed documentation build configuration file, created by
 # sphinx-quickstart on Tue Oct  6 14:42:44 2015.
@@ -471,16 +470,14 @@ def run(self):
         if "methods" in self.options:
             _, methods = self.get_members(app, c, ["method"], ["__init__"])
             self.content = [
-                "%s.%s" % (class_name, method)
+                f"{class_name}.{method}"
                 for method in methods
                 if not method.startswith("_")
             ]
         if "attributes" in self.options:
             _, attribs = self.get_members(app, c, ["attribute", "property"])
             self.content = [
-                "~%s.%s" % (clazz, attrib)
-                for attrib in attribs
-                if not attrib.startswith("_")
+                f"~{clazz}.{attrib}" for attrib in attribs if not attrib.startswith("_")
             ]
         return super().run()
 

From 5688e503d9411e69d0a8278d2b9692d453814c73 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Tue, 29 Jun 2021 16:08:27 -0500
Subject: [PATCH 1344/1550] Rename plot dropdown (#4992)

---
 distributed/http/templates/base.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/http/templates/base.html b/distributed/http/templates/base.html
index b0d428fac37..73e252f08e1 100644
--- a/distributed/http/templates/base.html
+++ b/distributed/http/templates/base.html
@@ -34,7 +34,7 @@
             {% endfor %}
             <div class="dropdown">
                 <li>
-                    <a href="javascript:void(0);">Plots</a>
+                    <a href="javascript:void(0);">More...</a>
                 </li>
                 <div class="dropdown-content">
                     <ul>

From 048bc1bd695497e4822528b3ff95777a078bf619 Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Wed, 30 Jun 2021 01:50:48 +0200
Subject: [PATCH 1345/1550] Follow up `pyupgrade` formatting (#4993)

---
 distributed/client.py                         |  2 +-
 distributed/dashboard/components/scheduler.py |  6 ++----
 distributed/deploy/old_ssh.py                 | 10 +++++-----
 distributed/tests/test_scheduler.py           |  4 ++--
 distributed/tests/test_semaphore.py           |  4 ++--
 distributed/tests/test_steal.py               |  2 +-
 distributed/tests/test_utils_comm.py          |  4 ++--
 7 files changed, 15 insertions(+), 17 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 6a81fd6da6d..58620aae831 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -659,7 +659,7 @@ def __init__(
                 logger.info("Config value `scheduler-address` found: %s", address)
 
         if address is not None and kwargs:
-            raise ValueError(f"Unexpected keyword arguments: {str(sorted(kwargs))}")
+            raise ValueError(f"Unexpected keyword arguments: {sorted(kwargs)}")
 
         if isinstance(address, (rpc, PooledRPCCall)):
             self.scheduler = address
diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index fe72c4e98e0..379eb22ce70 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -165,10 +165,8 @@ def update(self):
 
             if total:
                 self.root.title.text = (
-                    "Occupancy -- total time: {}  wall time: {}".format(
-                        format_time(total),
-                        format_time(total / self.scheduler.total_nthreads),
-                    )
+                    f"Occupancy -- total time: {format_time(total)} "
+                    f"wall time: {format_time(total / self.scheduler.total_nthreads)}"
                 )
             else:
                 self.root.title.text = "Occupancy"
diff --git a/distributed/deploy/old_ssh.py b/distributed/deploy/old_ssh.py
index 6d158e27f05..dcf9fd83eb2 100644
--- a/distributed/deploy/old_ssh.py
+++ b/distributed/deploy/old_ssh.py
@@ -100,7 +100,7 @@ def async_ssh(cmd_dict):
             print(
                 "               "
                 + bcolors.FAIL
-                + f"Retrying... (attempt {retries}/{3})"
+                + f"Retrying... (attempt {retries}/3)"
                 + bcolors.ENDC
             )
 
@@ -152,7 +152,7 @@ def read_from_stderr():
                 cmd_dict["output_queue"].put(
                     "[ {label} ] : ".format(label=cmd_dict["label"])
                     + bcolors.FAIL
-                    + f"{line}"
+                    + line
                     + bcolors.ENDC
                 )
                 line = stderr.readline()
@@ -215,14 +215,14 @@ def start_scheduler(
 
     # Optionally re-direct stdout and stderr to a logfile
     if logdir is not None:
-        cmd = f"mkdir -p {logdir} && " + cmd
+        cmd = f"mkdir -p {logdir} && {cmd}"
         cmd += "&> {logdir}/dask_scheduler_{addr}:{port}.log".format(
             addr=addr, port=port, logdir=logdir
         )
 
     # Format output labels we can prepend to each line of output, and create
     # a 'status' key to keep track of jobs that terminate prematurely.
-    label = bcolors.BOLD + f"scheduler {addr}:{port}" + bcolors.ENDC
+    label = f"{bcolors.BOLD}scheduler {addr}:{port}{bcolors.ENDC}"
 
     # Create a command dictionary, which contains everything we need to run and
     # interact with this command.
@@ -305,7 +305,7 @@ def start_worker(
 
     # Optionally redirect stdout and stderr to a logfile
     if logdir is not None:
-        cmd = f"mkdir -p {logdir} && " + cmd
+        cmd = f"mkdir -p {logdir} && {cmd}"
         cmd += "&> {logdir}/dask_scheduler_{addr}.log".format(
             addr=worker_addr, logdir=logdir
         )
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 88b3f2de4d8..b18a199b496 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -2016,10 +2016,10 @@ def abort(self):
         pass
 
     def read(self, deserializers=None):
-        raise OSError
+        raise OSError()
 
     def write(self, msg, serializers=None, on_error=None):
-        raise OSError
+        raise OSError()
 
 
 class FlakyConnectionPool(ConnectionPool):
diff --git a/distributed/tests/test_semaphore.py b/distributed/tests/test_semaphore.py
index 47678696651..bf671df9fff 100644
--- a/distributed/tests/test_semaphore.py
+++ b/distributed/tests/test_semaphore.py
@@ -283,10 +283,10 @@ def abort(self):
         pass
 
     def read(self, deserializers=None):
-        raise OSError
+        raise OSError()
 
     def write(self, msg, serializers=None, on_error=None):
-        raise OSError
+        raise OSError()
 
 
 class FlakyConnectionPool(ConnectionPool):
diff --git a/distributed/tests/test_steal.py b/distributed/tests/test_steal.py
index 03ee701fc50..c47b3127b0f 100644
--- a/distributed/tests/test_steal.py
+++ b/distributed/tests/test_steal.py
@@ -553,7 +553,7 @@ async def assert_balanced(inp, expected, c, s, *workers):
 
         if result2 == expected2:
             return
-    raise Exception(f"Expected: {str(expected2)}; got: {str(result2)}")
+    raise Exception(f"Expected: {expected2}; got: {result2}")
 
 
 @pytest.mark.parametrize(
diff --git a/distributed/tests/test_utils_comm.py b/distributed/tests/test_utils_comm.py
index ff0d6b09da6..ca058ab06ec 100644
--- a/distributed/tests/test_utils_comm.py
+++ b/distributed/tests/test_utils_comm.py
@@ -56,10 +56,10 @@ def abort(self):
         pass
 
     def read(self, deserializers=None):
-        raise OSError
+        raise OSError()
 
     def write(self, msg, serializers=None, on_error=None):
-        raise OSError
+        raise OSError()
 
 
 class BrokenConnectionPool(ConnectionPool):

From 84641e7ab2d702760070edfe1a8f60e19c7323b4 Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Wed, 30 Jun 2021 16:56:12 +0200
Subject: [PATCH 1346/1550] Ensure shuffle split default durations uses proper
 prefix (#4991)

---
 distributed/distributed.yaml        |  2 +-
 distributed/tests/test_scheduler.py | 30 ++++++++++++++++++++++++++++-
 2 files changed, 30 insertions(+), 2 deletions(-)

diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index 9ea6360b49f..870198bc306 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -28,7 +28,7 @@ distributed:
     unknown-task-duration: 500ms  # Default duration for all tasks with unknown durations ("15m", "2h")
     default-task-durations:  # How long we expect function names to run ("1h", "1s") (helps for long tasks)
       rechunk-split: 1us
-      shuffle-split: 1us
+      split-shuffle: 1us
     validate: False         # Check scheduler state at every step for debugging
     dashboard:
       status:
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index b18a199b496..8d38c69e10b 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -17,7 +17,7 @@
 
 import dask
 from dask import delayed
-from dask.utils import apply
+from dask.utils import apply, parse_timedelta
 
 from distributed import Client, Nanny, Worker, fire_and_forget, wait
 from distributed.comm import Comm
@@ -1845,6 +1845,34 @@ async def test_get_task_duration(c, s, a, b):
         assert len(s.unknown_durations["slowinc"]) == 1
 
 
+@gen_cluster(client=True)
+async def test_default_task_duration_splits(c, s, a, b):
+    """This test ensures that the default task durations for shuffle split tasks are, by default, aligned with the task names of dask.dask"""
+
+    pd = pytest.importorskip("pandas")
+    dd = pytest.importorskip("dask.dataframe")
+
+    # We don't care about the actual computation here but we'll schedule one anyhow to verify that we're looking for the correct key
+    npart = 10
+    df = dd.from_pandas(pd.DataFrame({"A": range(100), "B": 1}), npartitions=npart)
+    graph = df.shuffle(
+        "A",
+        shuffle="tasks",
+        # If we don't have enough partitions, we'll fall back to a simple shuffle
+        max_branch=npart - 1,
+    ).sum()
+    fut = c.compute(graph)
+    await wait(fut)
+
+    split_prefix = [pre for pre in s.task_prefixes.keys() if "split" in pre]
+    assert len(split_prefix) == 1
+    split_prefix = split_prefix[0]
+    default_time = parse_timedelta(
+        dask.config.get("distributed.scheduler.default-task-durations")[split_prefix]
+    )
+    assert default_time <= 1e-6
+
+
 @pytest.mark.asyncio
 async def test_no_danglng_asyncio_tasks(cleanup):
     start = asyncio.all_tasks()

From e3f06ba7f21b681fe00787809e899c0654dadb24 Mon Sep 17 00:00:00 2001
From: Peter Andreas Entschev <peter@entschev.com>
Date: Thu, 1 Jul 2021 00:54:50 +0200
Subject: [PATCH 1347/1550] Update imports to cudf.testing._utils (#5005)

---
 distributed/comm/tests/test_ucx.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/comm/tests/test_ucx.py b/distributed/comm/tests/test_ucx.py
index ecb6b471114..74e1e654975 100644
--- a/distributed/comm/tests/test_ucx.py
+++ b/distributed/comm/tests/test_ucx.py
@@ -189,7 +189,7 @@ async def test_ping_pong_cudf(g):
     # *** ImportError: /usr/lib/x86_64-linux-gnu/libstdc++.so.6: version `CXXABI_1.3.11'
     # not found (required by python3.7/site-packages/pyarrow/../../../libarrow.so.12)
     cudf = pytest.importorskip("cudf")
-    from cudf.tests.utils import assert_eq
+    from cudf.testing._utils import assert_eq
 
     cudf_obj = g(cudf)
 

From 7ed517c47de90a68abd537d29df9740a2c20b638 Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Wed, 30 Jun 2021 23:57:57 +0100
Subject: [PATCH 1348/1550] OSError tweaks (#5003)

Follow-up to #4741
Related to #4989
---
 distributed/core.py                    |  4 ++--
 distributed/diskutils.py               | 13 ++++++-------
 distributed/locket.py                  |  7 ++-----
 distributed/nanny.py                   |  4 ++--
 distributed/scheduler.py               |  2 +-
 distributed/tests/test_asyncprocess.py |  8 ++------
 distributed/tests/test_client.py       | 10 ++--------
 distributed/utils_test.py              |  3 +--
 distributed/worker.py                  |  3 ++-
 9 files changed, 20 insertions(+), 34 deletions(-)

diff --git a/distributed/core.py b/distributed/core.py
index b8c2ba46e20..ab6ac05d000 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -579,7 +579,7 @@ async def handle_stream(self, comm, extra=None, every_cycle=[]):
                     else:
                         func()
 
-        except (CommClosedError, OSError):
+        except OSError:
             # FIXME: This is silently ignored, is this intentional?
             pass
         except Exception as e:
@@ -647,7 +647,7 @@ async def send_recv(comm, reply=True, serializers=None, deserializers=None, **kw
             response = await comm.read(deserializers=deserializers)
         else:
             response = None
-    except (OSError, CommClosedError):
+    except OSError:
         # On communication errors, we should simply close the communication
         force_close = True
         raise
diff --git a/distributed/diskutils.py b/distributed/diskutils.py
index 8d35d6a1e73..e76633b3586 100644
--- a/distributed/diskutils.py
+++ b/distributed/diskutils.py
@@ -1,4 +1,3 @@
-import errno
 import glob
 import logging
 import os
@@ -23,10 +22,11 @@ def is_locking_enabled():
 def safe_unlink(path):
     try:
         os.unlink(path)
-    except OSError as e:
+    except FileNotFoundError:
         # Perhaps it was removed by someone else?
-        if e.errno != errno.ENOENT:
-            logger.error("Failed to remove %r", str(e))
+        pass
+    except OSError as e:
+        logger.error(f"Failed to remove {path}: {e}")
 
 
 class WorkDir:
@@ -121,9 +121,8 @@ def __init__(self, base_dir):
     def _init_workspace(self):
         try:
             os.mkdir(self.base_dir)
-        except OSError as e:
-            if e.errno != errno.EEXIST:
-                raise
+        except FileExistsError:
+            pass
 
     def _global_lock(self, **kwargs):
         return locket.lock_file(self._global_lock_path, **kwargs)
diff --git a/distributed/locket.py b/distributed/locket.py
index bb383345b9a..f35f6384ef4 100644
--- a/distributed/locket.py
+++ b/distributed/locket.py
@@ -63,11 +63,8 @@ def _lock_file_non_blocking(file_):
         try:
             fcntl.flock(file_.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
             return True
-        except OSError as error:
-            if error.errno in [errno.EACCES, errno.EAGAIN]:
-                return False
-            else:
-                raise
+        except (PermissionError, BlockingIOError):
+            return False
 
     def _unlock_file(file_):
         fcntl.flock(file_.fileno(), fcntl.LOCK_UN)
diff --git a/distributed/nanny.py b/distributed/nanny.py
index 9cc4c30c6e2..882bc783cb8 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -279,7 +279,7 @@ async def start(self):
                 if len(ports) > 1 and e.errno == errno.EADDRINUSE:
                     continue
                 else:
-                    raise e
+                    raise
             else:
                 self._start_address = start_address
                 break
@@ -451,7 +451,7 @@ async def _on_exit(self, exitcode):
         ):
             try:
                 await self._unregister()
-            except (OSError, CommClosedError):
+            except OSError:
                 if not self.reconnect:
                     await self.close()
                     return
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 243f86d577d..81a35fb5968 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -6420,7 +6420,7 @@ async def feed(
                         response = function(self, state)
                     await comm.write(response)
                     await asyncio.sleep(interval)
-            except (OSError, CommClosedError):
+            except OSError:
                 pass
             finally:
                 if teardown:
diff --git a/distributed/tests/test_asyncprocess.py b/distributed/tests/test_asyncprocess.py
index 695f23cc4aa..b98b4d867bc 100644
--- a/distributed/tests/test_asyncprocess.py
+++ b/distributed/tests/test_asyncprocess.py
@@ -406,9 +406,7 @@ def test_asyncprocess_child_teardown_on_parent_exit():
         # test failure.
         try:
             readable = children_alive.poll(short_timeout)
-        except OSError:
-            # Windows can raise BrokenPipeError. EnvironmentError is caught for
-            # Python2/3 portability.
+        except BrokenPipeError:
             assert sys.platform.startswith("win"), "should only raise on windows"
             # Broken pipe implies closed, which is readable.
             readable = True
@@ -423,9 +421,7 @@ def test_asyncprocess_child_teardown_on_parent_exit():
             result = children_alive.recv()
         except EOFError:
             pass  # Test passes.
-        except OSError:
-            # Windows can raise BrokenPipeError. EnvironmentError is caught for
-            # Python2/3 portability.
+        except BrokenPipeError:
             assert sys.platform.startswith("win"), "should only raise on windows"
             # Test passes.
         else:
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index d40b2548ff1..f78351e8a92 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -1883,17 +1883,11 @@ async def test_allow_restrictions(c, s, a, b):
         c.map(inc, [20], workers="127.0.0.1", allow_other_workers="Hello!")
 
 
-@pytest.mark.skipif("True", reason="because")
 def test_bad_address():
-    try:
+    with pytest.raises(OSError, match="connect"):
         Client("123.123.123.123:1234", timeout=0.1)
-    except (OSError, TimeoutError) as e:
-        assert "connect" in str(e).lower()
-
-    try:
+    with pytest.raises(OSError, match="connect"):
         Client("127.0.0.1:1234", timeout=0.1)
-    except (OSError, TimeoutError) as e:
-        assert "connect" in str(e).lower()
 
 
 def test_informative_error_on_cluster_type():
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index 6f810354681..0b43caa323c 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -1120,10 +1120,9 @@ def has_ipv6():
         serv.bind(("::", 0))
         serv.listen(5)
         cli = socket.create_connection(serv.getsockname()[:2])
+        return True
     except OSError:
         return False
-    else:
-        return True
     finally:
         if cli is not None:
             cli.close()
diff --git a/distributed/worker.py b/distributed/worker.py
index fd0d32c8885..c44bff14c93 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -1142,7 +1142,7 @@ async def start(self):
                 if len(ports) > 1 and e.errno == errno.EADDRINUSE:
                     continue
                 else:
-                    raise e
+                    raise
             else:
                 self._start_address = start_address
                 break
@@ -2387,6 +2387,7 @@ async def gather_dep(
                 self.incoming_count += 1
 
                 self.log.append(("receive-dep", worker, list(response["data"])))
+
             except OSError:
                 logger.exception("Worker stream died during communication: %s", worker)
                 has_what = self.has_what.pop(worker)

From fc47318f6fdd80d755f494fe1870e465bad4cbf7 Mon Sep 17 00:00:00 2001
From: Gabe Joseph <gjoseph92@gmail.com>
Date: Wed, 30 Jun 2021 17:30:02 -0600
Subject: [PATCH 1349/1550] Co-assign root-ish tasks (#4967)

In `decide_worker`, rather than spreading out root tasks as much as possible, schedule consecutive (by priority order) root(ish) tasks on the same worker. This ensures the dependencies of a reduction start out on the same worker, reducing future data transfer.

Closes #4892

Closes #2602
---
 distributed/scheduler.py            |  66 ++++++++++++++++-
 distributed/tests/test_scheduler.py | 110 +++++++++++++++++++++++++++-
 distributed/tests/test_steal.py     |  13 ++--
 distributed/tests/test_worker.py    |   4 +-
 4 files changed, 180 insertions(+), 13 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 81a35fb5968..fe8d0af4b50 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -921,6 +921,16 @@ class TaskGroup:
 
        The result types of this TaskGroup
 
+    .. attribute:: last_worker: WorkerState
+
+       The worker most recently assigned a task from this group, or None when the group
+       is not identified to be root-like by `SchedulerState.decide_worker`.
+
+    .. attribute:: last_worker_tasks_left: int
+
+       If `last_worker` is not None, the number of times that worker should be assigned
+       subsequent tasks until a new worker is chosen.
+
     See also
     --------
     TaskPrefix
@@ -936,6 +946,8 @@ class TaskGroup:
     _start: double
     _stop: double
     _all_durations: object
+    _last_worker: WorkerState
+    _last_worker_tasks_left: Py_ssize_t
 
     def __init__(self, name: str):
         self._name = name
@@ -949,6 +961,8 @@ def __init__(self, name: str):
         self._start = 0.0
         self._stop = 0.0
         self._all_durations = defaultdict(float)
+        self._last_worker = None
+        self._last_worker_tasks_left = 0
 
     @property
     def name(self):
@@ -990,6 +1004,14 @@ def start(self):
     def stop(self):
         return self._stop
 
+    @property
+    def last_worker(self):
+        return self._last_worker
+
+    @property
+    def last_worker_tasks_left(self):
+        return self._last_worker_tasks_left
+
     @ccall
     def add(self, o):
         ts: TaskState = o
@@ -2309,21 +2331,60 @@ def transition_no_worker_waiting(self, key):
     @exceptval(check=False)
     def decide_worker(self, ts: TaskState) -> WorkerState:
         """
-        Decide on a worker for task *ts*.  Return a WorkerState.
+        Decide on a worker for task *ts*. Return a WorkerState.
+
+        If it's a root or root-like task, we place it with its relatives to
+        reduce future data tansfer.
+
+        If it has dependencies or restrictions, we use
+        `decide_worker_from_deps_and_restrictions`.
+
+        Otherwise, we pick the least occupied worker, or pick from all workers
+        in a round-robin fashion.
         """
+        if not self._workers_dv:
+            return None
+
         ws: WorkerState = None
+        group: TaskGroup = ts._group
         valid_workers: set = self.valid_workers(ts)
 
         if (
             valid_workers is not None
             and not valid_workers
             and not ts._loose_restrictions
-            and self._workers_dv
         ):
             self._unrunnable.add(ts)
             ts.state = "no-worker"
             return ws
 
+        # Group is larger than cluster with few dependencies? Minimize future data transfers.
+        if (
+            valid_workers is None
+            and len(group) > self._total_nthreads * 2
+            and sum(map(len, group._dependencies)) < 5
+        ):
+            ws: WorkerState = group._last_worker
+
+            if not (
+                ws and group._last_worker_tasks_left and ws._address in self._workers_dv
+            ):
+                # Last-used worker is full or unknown; pick a new worker for the next few tasks
+                ws = min(
+                    (self._idle_dv or self._workers_dv).values(),
+                    key=partial(self.worker_objective, ts),
+                )
+                group._last_worker_tasks_left = math.floor(
+                    (len(group) / self._total_nthreads) * ws._nthreads
+                )
+
+            # Record `last_worker`, or clear it on the final task
+            group._last_worker = (
+                ws if group.states["released"] + group.states["waiting"] > 1 else None
+            )
+            group._last_worker_tasks_left -= 1
+            return ws
+
         if ts._dependencies or valid_workers is not None:
             ws = decide_worker(
                 ts,
@@ -2332,6 +2393,7 @@ def decide_worker(self, ts: TaskState) -> WorkerState:
                 partial(self.worker_objective, ts),
             )
         else:
+            # Fastpath when there are no related tasks or restrictions
             worker_pool = self._idle or self._workers
             worker_pool_dv = cast(dict, worker_pool)
             wp_vals = worker_pool.values()
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 8d38c69e10b..1608a7d67b0 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -17,7 +17,7 @@
 
 import dask
 from dask import delayed
-from dask.utils import apply, parse_timedelta
+from dask.utils import apply, parse_timedelta, stringify
 
 from distributed import Client, Nanny, Worker, fire_and_forget, wait
 from distributed.comm import Comm
@@ -126,6 +126,114 @@ async def test_decide_worker_with_restrictions(client, s, a, b, c):
     assert x.key in a.data or x.key in b.data
 
 
+@pytest.mark.parametrize("ndeps", [0, 1, 4])
+@pytest.mark.parametrize(
+    "nthreads",
+    [
+        [("127.0.0.1", 1)] * 5,
+        [("127.0.0.1", 3), ("127.0.0.1", 2), ("127.0.0.1", 1)],
+    ],
+)
+def test_decide_worker_coschedule_order_neighbors(ndeps, nthreads):
+    @gen_cluster(
+        client=True,
+        nthreads=nthreads,
+        config={"distributed.scheduler.work-stealing": False},
+    )
+    async def test(c, s, *workers):
+        r"""
+        Ensure that sibling root tasks are scheduled to the same node, reducing future data transfer.
+
+        We generate a wide layer of "root" tasks (random NumPy arrays). All of those tasks share 0-5
+        trivial dependencies. The ``ndeps=0`` and ``ndeps=1`` cases are most common in real-world use
+        (``ndeps=1`` is basically ``da.from_array(..., inline_array=False)`` or ``da.from_zarr``).
+        The graph is structured like this (though the number of tasks and workers is different):
+
+            |-W1-|  |-W2-| |-W3-|  |-W4-|   < ---- ideal task scheduling
+
+              q       r       s       t      < --- `sum-aggregate-`
+             / \     / \     / \     / \
+            i   j   k   l   m   n   o   p    < --- `sum-`
+            |   |   |   |   |   |   |   |
+            a   b   c   d   e   f   g   h    < --- `random-`
+            \   \   \   |   |   /   /   /
+                   TRIVIAL * 0..5
+
+        Neighboring `random-` tasks should be scheduled on the same worker. We test that generally,
+        only one worker holds each row of the array, that the `random-` tasks are never transferred,
+        and that there are few transfers overall.
+        """
+        da = pytest.importorskip("dask.array")
+        np = pytest.importorskip("numpy")
+
+        if ndeps == 0:
+            x = da.random.random((100, 100), chunks=(10, 10))
+        else:
+
+            def random(**kwargs):
+                assert len(kwargs) == ndeps
+                return np.random.random((10, 10))
+
+            trivial_deps = {f"k{i}": delayed(object()) for i in range(ndeps)}
+
+            # TODO is there a simpler (non-blockwise) way to make this sort of graph?
+            x = da.blockwise(
+                random,
+                "yx",
+                new_axes={"y": (10,) * 10, "x": (10,) * 10},
+                dtype=float,
+                **trivial_deps,
+            )
+
+        xx, xsum = dask.persist(x, x.sum(axis=1, split_every=20))
+        await xsum
+
+        # Check that each chunk-row of the array is (mostly) stored on the same worker
+        primary_worker_key_fractions = []
+        secondary_worker_key_fractions = []
+        for i, keys in enumerate(x.__dask_keys__()):
+            # Iterate along rows of the array.
+            keys = set(stringify(k) for k in keys)
+
+            # No more than 2 workers should have any keys
+            assert sum(any(k in w.data for k in keys) for w in workers) <= 2
+
+            # What fraction of the keys for this row does each worker hold?
+            key_fractions = [
+                len(set(w.data).intersection(keys)) / len(keys) for w in workers
+            ]
+            key_fractions.sort()
+            # Primary worker: holds the highest percentage of keys
+            # Secondary worker: holds the second highest percentage of keys
+            primary_worker_key_fractions.append(key_fractions[-1])
+            secondary_worker_key_fractions.append(key_fractions[-2])
+
+        # There may be one or two rows that were poorly split across workers,
+        # but the vast majority of rows should only be on one worker.
+        assert np.mean(primary_worker_key_fractions) >= 0.9
+        assert np.median(primary_worker_key_fractions) == 1.0
+        assert np.mean(secondary_worker_key_fractions) <= 0.1
+        assert np.median(secondary_worker_key_fractions) == 0.0
+
+        # Check that there were few transfers
+        unexpected_transfers = []
+        for worker in workers:
+            for log in worker.incoming_transfer_log:
+                keys = log["keys"]
+                # The root-ish tasks should never be transferred
+                assert not any(k.startswith("random") for k in keys), keys
+                # `object-` keys (the trivial deps of the root random tasks) should be transferred
+                if any(not k.startswith("object") for k in keys):
+                    # But not many other things should be
+                    unexpected_transfers.append(list(keys))
+
+        # A transfer at the very end to move aggregated results is fine (necessary with unbalanced workers in fact),
+        # but generally there should be very very few transfers.
+        assert len(unexpected_transfers) <= 3, unexpected_transfers
+
+    test()
+
+
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 3)
 async def test_move_data_over_break_restrictions(client, s, a, b, c):
     [x] = await client.scatter([1], workers=b.address)
diff --git a/distributed/tests/test_steal.py b/distributed/tests/test_steal.py
index c47b3127b0f..7ca8f29bcc4 100644
--- a/distributed/tests/test_steal.py
+++ b/distributed/tests/test_steal.py
@@ -146,21 +146,18 @@ async def test_steal_related_tasks(e, s, a, b, c):
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 10, timeout=1000)
 async def test_dont_steal_fast_tasks_compute_time(c, s, *workers):
-    np = pytest.importorskip("numpy")
-    x = c.submit(np.random.random, 10000000, workers=workers[0].address)
-
     def do_nothing(x, y=None):
         pass
 
-    # execute and measure runtime once
-    await wait(c.submit(do_nothing, 1))
+    xs = c.map(do_nothing, range(10), workers=workers[0].address)
+    await wait(xs)
 
-    futures = c.map(do_nothing, range(1000), y=x)
+    futures = c.map(do_nothing, range(1000), y=xs)
 
     await wait(futures)
 
-    assert len(s.who_has[x.key]) == 1
-    assert len(s.has_what[workers[0].address]) == 1001
+    assert len(set.union(*(s.who_has[x.key] for x in xs))) == 1
+    assert len(s.has_what[workers[0].address]) == len(xs) + len(futures)
 
 
 @gen_cluster(client=True)
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 903241f7225..c38b8b76363 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -1590,12 +1590,12 @@ async def test_lifetime(cleanup):
     async with Scheduler() as s:
         async with Worker(s.address) as a, Worker(s.address, lifetime="1 seconds") as b:
             async with Client(s.address, asynchronous=True) as c:
-                futures = c.map(slowinc, range(200), delay=0.1)
+                futures = c.map(slowinc, range(200), delay=0.1, worker=[b.address])
                 await asyncio.sleep(1.5)
                 assert b.status != Status.running
                 await b.finished()
 
-                assert set(b.data).issubset(a.data)  # successfully moved data over
+                assert set(b.data) == set(a.data)  # successfully moved data over
 
 
 @gen_cluster(client=True, worker_kwargs={"lifetime": "10s", "lifetime_stagger": "2s"})

From b8b4ff81f4071b46453ecdee4bdc26b648139603 Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Thu, 1 Jul 2021 16:27:47 +0100
Subject: [PATCH 1350/1550] Rename nbytes widgets (#4878)

Rename widgets:
- NBytes -> WorkersMemory
- NBytesHistogram -> WorkersMemoryHistogram
- NBytesCluster -> ClusterMemory

Rename URIs:
- /individual-nbytes -> /individual-workers-memory
- /individual-nbytes-cluster -> /individual-cluster-memory

Closes #4871
---
 distributed/dashboard/components/scheduler.py | 50 ++++++++++---------
 distributed/dashboard/scheduler.py            | 13 +++--
 .../dashboard/tests/test_scheduler_bokeh.py   | 20 ++++----
 distributed/http/static/css/status.css        |  8 +--
 distributed/http/templates/status.html        |  8 +--
 5 files changed, 53 insertions(+), 46 deletions(-)

diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index 379eb22ce70..b0543b77edd 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -228,8 +228,8 @@ def update(self):
         self.source.data.update({"left": x[:-1], "right": x[1:], "top": counts})
 
 
-def _nbytes_color(current: int, limit: int) -> str:
-    """Dynamic color used by NBytes and NBytesCluster"""
+def _memory_color(current: int, limit: int) -> str:
+    """Dynamic color used by WorkersMemory and ClusterMemory"""
     if limit and current > limit:
         return "red"
     elif limit and current > limit / 2:
@@ -238,7 +238,7 @@ def _nbytes_color(current: int, limit: int) -> str:
         return "blue"
 
 
-class NBytesCluster(DashboardComponent):
+class ClusterMemory(DashboardComponent):
     """Total memory usage on the cluster"""
 
     def __init__(self, scheduler, width=600, **kwargs):
@@ -262,9 +262,9 @@ def __init__(self, scheduler, width=600, **kwargs):
             self.root = figure(
                 title="Bytes stored on cluster",
                 tools="",
-                id="bk-nbytes-cluster-worker-plot",
+                id="bk-cluster-memory-plot",
                 width=int(width / 2),
-                name="nbytes_cluster",
+                name="cluster_memory",
                 **kwargs,
             )
             rect = self.root.rect(
@@ -322,7 +322,7 @@ def update(self):
         with log_errors():
             limit = sum(ws.memory_limit for ws in self.scheduler.workers.values())
             meminfo = self.scheduler.memory
-            color = _nbytes_color(meminfo.process, limit)
+            color = _memory_color(meminfo.process, limit)
 
             width = [
                 meminfo.managed_in_memory,
@@ -356,7 +356,7 @@ def update(self):
             update(self.source, result)
 
 
-class NBytes(DashboardComponent):
+class WorkersMemory(DashboardComponent):
     """Memory usage for single workers"""
 
     def __init__(self, scheduler, width=600, **kwargs):
@@ -382,9 +382,9 @@ def __init__(self, scheduler, width=600, **kwargs):
             self.root = figure(
                 title="Bytes stored per worker",
                 tools="",
-                id="bk-nbytes-worker-plot",
+                id="bk-workers-memory-plot",
                 width=int(width / 2),
-                name="nbytes_workers",
+                name="workers_memory",
                 **kwargs,
             )
             rect = self.root.rect(
@@ -472,7 +472,7 @@ def quadlist(i) -> list:
                 max_limit = max(
                     max_limit, limit, meminfo.process + meminfo.managed_spilled
                 )
-                color_i = _nbytes_color(meminfo.process, limit)
+                color_i = _memory_color(meminfo.process, limit)
 
                 width += [
                     meminfo.managed_in_memory,
@@ -517,7 +517,7 @@ def quadlist(i) -> list:
             update(self.source, result)
 
 
-class NBytesHistogram(DashboardComponent):
+class WorkersMemoryHistogram(DashboardComponent):
     """Histogram of memory usage, showing how many workers there are in each bucket of
     usage. Replaces the per-worker graph when there are >= 50 workers.
     """
@@ -532,8 +532,8 @@ def __init__(self, scheduler, **kwargs):
 
             self.root = figure(
                 title="Bytes stored per worker (Histogram)",
-                name="nbytes_workers",
-                id="bk-nbytes-histogram-plot",
+                name="workers_memory",
+                id="bk-workers-memory-histogram-plot",
                 y_axis_label="frequency",
                 tools="",
                 **kwargs,
@@ -2800,27 +2800,29 @@ def tg_graph_doc(scheduler, extra, doc):
 
 def status_doc(scheduler, extra, doc):
     with log_errors():
-        nbytes_cluster = NBytesCluster(scheduler, sizing_mode="stretch_both")
-        nbytes_cluster.update()
-        add_periodic_callback(doc, nbytes_cluster, 100)
-        doc.add_root(nbytes_cluster.root)
+        cluster_memory = ClusterMemory(scheduler, sizing_mode="stretch_both")
+        cluster_memory.update()
+        add_periodic_callback(doc, cluster_memory, 100)
+        doc.add_root(cluster_memory.root)
 
         if len(scheduler.workers) < 50:
-            nbytes_workers = NBytes(scheduler, sizing_mode="stretch_both")
+            workers_memory = WorkersMemory(scheduler, sizing_mode="stretch_both")
             processing = CurrentLoad(scheduler, sizing_mode="stretch_both")
             processing_root = processing.processing_figure
-            processing_root.y_range = nbytes_workers.root.y_range
+            processing_root.y_range = workers_memory.root.y_range
         else:
-            nbytes_workers = NBytesHistogram(scheduler, sizing_mode="stretch_both")
+            workers_memory = WorkersMemoryHistogram(
+                scheduler, sizing_mode="stretch_both"
+            )
             processing = ProcessingHistogram(scheduler, sizing_mode="stretch_both")
             processing_root = processing.root
-            row(nbytes_workers.root, processing.root, sizing_mode="stretch_both")
+            row(workers_memory.root, processing.root, sizing_mode="stretch_both")
 
-        nbytes_workers.update()
+        workers_memory.update()
         processing.update()
-        add_periodic_callback(doc, nbytes_workers, 100)
+        add_periodic_callback(doc, workers_memory, 100)
         add_periodic_callback(doc, processing, 100)
-        doc.add_root(nbytes_workers.root)
+        doc.add_root(workers_memory.root)
         doc.add_root(processing_root)
 
         task_stream = TaskStream(
diff --git a/distributed/dashboard/scheduler.py b/distributed/dashboard/scheduler.py
index e4c609b840d..b6a213e71ea 100644
--- a/distributed/dashboard/scheduler.py
+++ b/distributed/dashboard/scheduler.py
@@ -14,17 +14,17 @@
     AggregateAction,
     BandwidthTypes,
     BandwidthWorkers,
+    ClusterMemory,
     ComputePerKey,
     CurrentLoad,
     MemoryByKey,
-    NBytes,
-    NBytesCluster,
     Occupancy,
     SystemMonitor,
     TaskGraph,
     TaskGroupGraph,
     TaskProgress,
     TaskStream,
+    WorkersMemory,
     WorkerTable,
     events_doc,
     graph_doc,
@@ -62,8 +62,13 @@
     "/individual-progress": individual_doc(TaskProgress, 100, height=160),
     "/individual-graph": individual_doc(TaskGraph, 200),
     "/individual-groups": individual_doc(TaskGroupGraph, 200),
-    "/individual-nbytes": individual_doc(NBytes, 100),
-    "/individual-nbytes-cluster": individual_doc(NBytesCluster, 100),
+    "/individual-workers-memory": individual_doc(WorkersMemory, 100),
+    "/individual-cluster-memory": individual_doc(ClusterMemory, 100),
+    # Temporary backwards compatibility with dask-labextension
+    # See https://github.com/dask/dask-labextension/issues/198
+    "/individual-nbytes": individual_doc(WorkersMemory, 100),
+    "/individual-nbytes-cluster": individual_doc(ClusterMemory, 100),
+    # End backwards compatibility
     "/individual-cpu": individual_doc(CurrentLoad, 100, fig_attr="cpu_figure"),
     "/individual-nprocessing": individual_doc(
         CurrentLoad, 100, fig_attr="processing_figure"
diff --git a/distributed/dashboard/tests/test_scheduler_bokeh.py b/distributed/dashboard/tests/test_scheduler_bokeh.py
index 38b8d08c1c6..1cf3f1621ab 100644
--- a/distributed/dashboard/tests/test_scheduler_bokeh.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh.py
@@ -21,13 +21,11 @@
 from distributed.dashboard import scheduler
 from distributed.dashboard.components.scheduler import (
     AggregateAction,
+    ClusterMemory,
     ComputePerKey,
     CurrentLoad,
     Events,
     MemoryByKey,
-    NBytes,
-    NBytesCluster,
-    NBytesHistogram,
     Occupancy,
     ProcessingHistogram,
     ProfileServer,
@@ -38,6 +36,8 @@
     TaskGroupGraph,
     TaskProgress,
     TaskStream,
+    WorkersMemory,
+    WorkersMemoryHistogram,
     WorkerTable,
 )
 from distributed.dashboard.components.worker import Counters
@@ -276,8 +276,8 @@ async def test_ProcessingHistogram(c, s, a, b):
 
 
 @gen_cluster(client=True)
-async def test_NBytes(c, s, a, b):
-    cl = NBytes(s)
+async def test_WorkersMemory(c, s, a, b):
+    cl = WorkersMemory(s)
 
     futures = c.map(slowinc, range(10), delay=0.001)
     await wait(futures)
@@ -293,8 +293,8 @@ async def test_NBytes(c, s, a, b):
 
 
 @gen_cluster(client=True)
-async def test_NBytesCluster(c, s, a, b):
-    cl = NBytesCluster(s)
+async def test_ClusterMemory(c, s, a, b):
+    cl = ClusterMemory(s)
 
     futures = c.map(slowinc, range(10), delay=0.001)
     await wait(futures)
@@ -302,7 +302,7 @@ async def test_NBytesCluster(c, s, a, b):
     cl.update()
     d = dict(cl.source.data)
     llens = {len(l) for l in d.values()}
-    # Unlike NBytes, empty rects here aren't pruned away.
+    # Unlike WorkersMemory, empty rects here aren't pruned away.
     assert llens == {4}
     # There is definitely going to be managed_in_memory and
     # unmanaged_old; there may be unmanaged_new. There won't be managed_spilled.
@@ -311,8 +311,8 @@ async def test_NBytesCluster(c, s, a, b):
 
 
 @gen_cluster(client=True)
-async def test_NBytesHistogram(c, s, a, b):
-    nh = NBytesHistogram(s)
+async def test_WorkersMemoryHistogram(c, s, a, b):
+    nh = WorkersMemoryHistogram(s)
     nh.update()
     assert any(nh.source.data["top"] != 0)
 
diff --git a/distributed/http/static/css/status.css b/distributed/http/static/css/status.css
index 9467ba7a3c8..2fd9803702d 100644
--- a/distributed/http/static/css/status.css
+++ b/distributed/http/static/css/status.css
@@ -8,11 +8,11 @@
     grid-template-columns: 1fr 1fr;
     grid-template-rows: 70px 2fr 6fr 2fr;
   }
-  #status-nbytes-cluster {
+  #status-cluster-memory {
     grid-column: 1 / span 2;
     grid-row: 1;
   }
-  #status-nbytes-workers {
+  #status-workers-memory {
     grid-column: 1;
     grid-row: 2;
   }
@@ -35,11 +35,11 @@
     grid-template-columns: 1fr 3fr;
     grid-template-rows: 80px 4fr 1fr 1fr 4fr;
   }
-  #status-nbytes-cluster {
+  #status-cluster-memory {
     grid-column: 1;
     grid-row: 1;
   }
-  #status-nbytes-workers {
+  #status-workers-memory {
     grid-column: 1;
     grid-row: 2 / span 2;
   }
diff --git a/distributed/http/templates/status.html b/distributed/http/templates/status.html
index 57b8394c931..0071cf2a98b 100644
--- a/distributed/http/templates/status.html
+++ b/distributed/http/templates/status.html
@@ -8,12 +8,12 @@
 {% from macros import embed %}
 <div id="status-fluid">
 
-  <div id="status-nbytes-cluster">
-    {{ embed(roots.nbytes_cluster) }}
+  <div id="status-cluster-memory">
+    {{ embed(roots.cluster_memory) }}
   </div>
 
-  <div id="status-nbytes-workers">
-    {{ embed(roots.nbytes_workers) }}
+  <div id="status-workers-memory">
+    {{ embed(roots.workers_memory) }}
   </div>
 
   <div id="status-processing">

From c6a12d843f91efcefd3bd020ed0eb5165a1ab808 Mon Sep 17 00:00:00 2001
From: Naty Clementi <natyclementi@gmail.com>
Date: Thu, 1 Jul 2021 15:54:08 -0400
Subject: [PATCH 1351/1550] Tabs on bottom left corner on dashboard (#5006)

---
 distributed/dashboard/components/scheduler.py | 24 ++++++++++++++++---
 distributed/http/templates/status.html        |  2 +-
 2 files changed, 22 insertions(+), 4 deletions(-)

diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index b0543b77edd..be408da1119 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -2808,22 +2808,40 @@ def status_doc(scheduler, extra, doc):
         if len(scheduler.workers) < 50:
             workers_memory = WorkersMemory(scheduler, sizing_mode="stretch_both")
             processing = CurrentLoad(scheduler, sizing_mode="stretch_both")
+
             processing_root = processing.processing_figure
-            processing_root.y_range = workers_memory.root.y_range
         else:
             workers_memory = WorkersMemoryHistogram(
                 scheduler, sizing_mode="stretch_both"
             )
             processing = ProcessingHistogram(scheduler, sizing_mode="stretch_both")
+
             processing_root = processing.root
-            row(workers_memory.root, processing.root, sizing_mode="stretch_both")
+
+        current_load = CurrentLoad(scheduler, sizing_mode="stretch_both")
+        occupancy = Occupancy(scheduler, sizing_mode="stretch_both")
+
+        cpu_root = current_load.cpu_figure
+        occupancy_root = occupancy.root
 
         workers_memory.update()
         processing.update()
+        current_load.update()
+        occupancy.update()
+
         add_periodic_callback(doc, workers_memory, 100)
         add_periodic_callback(doc, processing, 100)
+        add_periodic_callback(doc, current_load, 100)
+        add_periodic_callback(doc, occupancy, 100)
+
         doc.add_root(workers_memory.root)
-        doc.add_root(processing_root)
+
+        tab1 = Panel(child=processing_root, title="Processing")
+        tab2 = Panel(child=cpu_root, title="CPU")
+        tab3 = Panel(child=occupancy_root, title="Occupancy")
+
+        proc_tabs = Tabs(tabs=[tab1, tab2, tab3], name="processing_tabs")
+        doc.add_root(proc_tabs)
 
         task_stream = TaskStream(
             scheduler,
diff --git a/distributed/http/templates/status.html b/distributed/http/templates/status.html
index 0071cf2a98b..0bbeef4769b 100644
--- a/distributed/http/templates/status.html
+++ b/distributed/http/templates/status.html
@@ -17,7 +17,7 @@
   </div>
 
   <div id="status-processing">
-    {{ embed(roots.processing) }}
+    {{ embed(roots.processing_tabs) }}
   </div>
 
   <div id="status-tasks">

From 8b79bd95d0c5a681f6d0543eb20d645dc1182ec1 Mon Sep 17 00:00:00 2001
From: Naty Clementi <natyclementi@gmail.com>
Date: Thu, 1 Jul 2021 15:55:49 -0400
Subject: [PATCH 1352/1550] Fix worker memory dashboard flickering (#4997)

---
 distributed/dashboard/components/scheduler.py | 17 +++++------------
 1 file changed, 5 insertions(+), 12 deletions(-)

diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index be408da1119..d5e8365a808 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -281,7 +281,6 @@ def __init__(self, scheduler, width=600, **kwargs):
             self.root.axis[0].ticker = BasicTicker(**TICKS_1024)
             self.root.xaxis[0].formatter = NumeralTickFormatter(format="0.0 b")
             self.root.xaxis.major_label_orientation = XLABEL_ORIENTATION
-            self.root.x_range.start = 0
             self.root.xaxis.minor_tick_line_alpha = 0
             self.root.yaxis.visible = False
             self.root.ygrid.visible = False
@@ -341,12 +340,9 @@ def update(self):
                 "unmanaged_recent": [meminfo.unmanaged_recent] * 4,
                 "spilled": [meminfo.managed_spilled] * 4,
             }
-            # FIXME https://github.com/dask/distributed/issues/4675
-            #       This causes flickering after adding workers and when enough memory
-            #       is spilled out
-            self.root.x_range.end = max(
-                limit, meminfo.process + meminfo.managed_spilled
-            )
+
+            x_end = max(limit, meminfo.process + meminfo.managed_spilled)
+            self.root.x_range = DataRange1d(start=0, end=x_end, range_padding=0)
 
             title = f"Bytes stored: {format_bytes(meminfo.process)}"
             if meminfo.managed_spilled:
@@ -402,7 +398,6 @@ def __init__(self, scheduler, width=600, **kwargs):
             self.root.axis[0].ticker = BasicTicker(**TICKS_1024)
             self.root.xaxis[0].formatter = NumeralTickFormatter(format="0.0 b")
             self.root.xaxis.major_label_orientation = XLABEL_ORIENTATION
-            self.root.x_range.start = 0
             self.root.xaxis.minor_tick_line_alpha = 0
             self.root.yaxis.visible = False
             self.root.ygrid.visible = False
@@ -510,10 +505,8 @@ def quadlist(i) -> list:
             result = {
                 k: [vi for vi, w in zip(v, width) if w] for k, v in result.items()
             }
-            # FIXME https://github.com/dask/distributed/issues/4675
-            #       This causes flickering after adding workers and when enough memory
-            #       is spilled to disk
-            self.root.x_range.end = max_limit
+
+            self.root.x_range = DataRange1d(start=0, end=max_limit, range_padding=0)
             update(self.source, result)
 
 

From a5549a1ef380e8b8672b39881ae5890fb6626ab6 Mon Sep 17 00:00:00 2001
From: Ian Rose <ian.r.rose@gmail.com>
Date: Thu, 1 Jul 2021 13:30:57 -0700
Subject: [PATCH 1353/1550] Remove duplicated dashboard panes (#5017)

---
 distributed/dashboard/scheduler.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/distributed/dashboard/scheduler.py b/distributed/dashboard/scheduler.py
index b6a213e71ea..e7752e33cae 100644
--- a/distributed/dashboard/scheduler.py
+++ b/distributed/dashboard/scheduler.py
@@ -64,11 +64,6 @@
     "/individual-groups": individual_doc(TaskGroupGraph, 200),
     "/individual-workers-memory": individual_doc(WorkersMemory, 100),
     "/individual-cluster-memory": individual_doc(ClusterMemory, 100),
-    # Temporary backwards compatibility with dask-labextension
-    # See https://github.com/dask/dask-labextension/issues/198
-    "/individual-nbytes": individual_doc(WorkersMemory, 100),
-    "/individual-nbytes-cluster": individual_doc(ClusterMemory, 100),
-    # End backwards compatibility
     "/individual-cpu": individual_doc(CurrentLoad, 100, fig_attr="cpu_figure"),
     "/individual-nprocessing": individual_doc(
         CurrentLoad, 100, fig_attr="processing_figure"

From 28366d6bc28fd8f01a94d4804b4314ef468bd337 Mon Sep 17 00:00:00 2001
From: Ian Rose <ian.r.rose@gmail.com>
Date: Thu, 1 Jul 2021 14:52:41 -0700
Subject: [PATCH 1354/1550] Also include pngs when bundling package (#5016)

---
 MANIFEST.in | 1 +
 1 file changed, 1 insertion(+)

diff --git a/MANIFEST.in b/MANIFEST.in
index 2a8f054e213..66893de2b73 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -3,6 +3,7 @@ recursive-include distributed *.js
 recursive-include distributed *.coffee
 recursive-include distributed *.html
 recursive-include distributed *.css
+recursive-include distributed *.png
 recursive-include distributed *.svg
 recursive-include distributed *.ico
 recursive-include distributed *.yaml

From bf8253f18c6b5524e527482aa6a2c850289fa33e Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 1 Jul 2021 17:41:20 -0700
Subject: [PATCH 1355/1550] Add echo handler to Server class (#5020)

This adds an echo route to all Dask servers, including schedulers and
workers.

This makes it a bit easier to test connection and bandwidth issues

```python
result = await client.scheduler.echo(b"...")
```

This returns the same data back without modification
---
 distributed/core.py            |  4 ++++
 distributed/tests/test_core.py | 18 +++++++-----------
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/distributed/core.py b/distributed/core.py
index ab6ac05d000..aad0d706058 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -139,6 +139,7 @@ def __init__(
     ):
         self.handlers = {
             "identity": self.identity,
+            "echo": self.echo,
             "connection_stream": self.handle_stream,
         }
         self.handlers.update(handlers)
@@ -384,6 +385,9 @@ def port(self):
     def identity(self, comm=None):
         return {"type": type(self).__name__, "id": self.id}
 
+    def echo(self, comm=None, data=None):
+        return data
+
     async def listen(self, port_or_addr=None, allow_offload=True, **kwargs):
         if port_or_addr is None:
             port_or_addr = self.default_port
diff --git a/distributed/tests/test_core.py b/distributed/tests/test_core.py
index d3df2e61867..85c4550e467 100644
--- a/distributed/tests/test_core.py
+++ b/distributed/tests/test_core.py
@@ -42,10 +42,6 @@
 )
 
 
-def echo(comm, x):
-    return x
-
-
 class CountedObject:
     """
     A class which counts the number of live instances.
@@ -443,16 +439,16 @@ async def test_rpc_with_many_connections_inproc():
 
 async def check_large_packets(listen_arg):
     """tornado has a 100MB cap by default"""
-    server = Server({"echo": echo})
+    server = Server({})
     await server.listen(listen_arg)
 
     data = b"0" * int(200e6)  # slightly more than 100MB
     async with rpc(server.address) as conn:
-        result = await conn.echo(x=data)
+        result = await conn.echo(data=data)
         assert result == data
 
         d = {"x": data}
-        result = await conn.echo(x=d)
+        result = await conn.echo(data=d)
         assert result == d
 
     server.stop()
@@ -544,17 +540,17 @@ async def test_connect_raises():
 
 @pytest.mark.asyncio
 async def test_send_recv_args():
-    server = Server({"echo": echo})
+    server = Server({})
     await server.listen(0)
 
     comm = await connect(server.address)
-    result = await send_recv(comm, op="echo", x=b"1")
+    result = await send_recv(comm, op="echo", data=b"1")
     assert result == b"1"
     assert not comm.closed()
-    result = await send_recv(comm, op="echo", x=b"2", reply=False)
+    result = await send_recv(comm, op="echo", data=b"2", reply=False)
     assert result is None
     assert not comm.closed()
-    result = await send_recv(comm, op="echo", x=b"3", close=True)
+    result = await send_recv(comm, op="echo", data=b"3", close=True)
     assert result == b"3"
     assert comm.closed()
 

From cbcec9cdc020a36c024d55c216d5391c7b09ccae Mon Sep 17 00:00:00 2001
From: Gabe Joseph <gjoseph92@gmail.com>
Date: Fri, 2 Jul 2021 10:23:42 -0600
Subject: [PATCH 1356/1550] Update scheduling policy docs for #4967 (#5018)

---
 docs/source/conf.py                 |  1 +
 docs/source/scheduling-policies.rst | 99 +++++++++++++++++++++++------
 docs/source/scheduling-state.rst    |  1 +
 3 files changed, 80 insertions(+), 21 deletions(-)

diff --git a/docs/source/conf.py b/docs/source/conf.py
index ac7e07fdff9..7d5021c6115 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -383,6 +383,7 @@
 intersphinx_mapping = {
     "python": ("https://docs.python.org/3", None),
     "numpy": ("https://docs.scipy.org/doc/numpy", None),
+    "dask": ("https://docs.dask.org/en/latest", None),
 }
 
 # Redirects
diff --git a/docs/source/scheduling-policies.rst b/docs/source/scheduling-policies.rst
index cf93fb612ff..63fa9fa41f4 100644
--- a/docs/source/scheduling-policies.rst
+++ b/docs/source/scheduling-policies.rst
@@ -12,28 +12,95 @@ more information on how this these policies are enacted efficiently see
 Choosing Workers
 ----------------
 
-When a task transitions from waiting to a processing state we decide a suitable
-worker for that task.  If the task has significant data dependencies or if the
-workers are under heavy load then this choice of worker can strongly impact
-global performance.  Currently workers for tasks are determined as follows:
+When a task transitions from waiting to a processing state, we decide a suitable
+worker for that task. If the task has significant data dependencies or if the
+workers are under heavy load, then this choice of worker can strongly impact
+global performance. Similarly, the placement of root tasks affects performance
+of downstream computations, since it can determine how much data will need to be
+transferred between workers in the future. Different heuristics are used for these
+different scenarios:
 
-1.  If the task has no major dependencies and no restrictions then we find the
-    least occupied worker.
+Initial Task Placement
+~~~~~~~~~~~~~~~~~~~~~~
+
+We want neighboring root tasks to run on the same worker, since there's a
+good chance those neighbors will be combined in a downstream operation::
+
+      i       j
+     / \     / \
+    e   f   g   h
+    |   |   |   |
+    a   b   c   d
+    \   \  /   /
+         X
+
+In the above case, we want ``a`` and ``b`` to run on the same worker,
+and ``c`` and ``d`` to run on the same worker, reducing future
+data transfer. We can also ignore the location of ``X``, because assuming
+we split the ``a b c d`` group across all workers to maximize parallelism,
+then ``X`` will eventually get transferred everywhere.
+(Note that wanting to co-locate ``a b`` and ``c d`` would still apply even if
+``X`` didn't exist.)
+
+Calculating these cousin tasks directly by traversing the graph would be expensive.
+Instead, we use the task's TaskGroup, which is the collection of all tasks with the
+same key prefix. (``(random-a1b2c3, 0)``, ``(random-a1b2c3, 1)``, ``(random-a1b2c3, 2)``
+would all belong to the TaskGroup ``random-a1b2c3``.)
+
+To identify the root(ish) tasks, we use this heuristic:
+
+1.  The TaskGroup has 2x more tasks than there are threads in the cluster
+2.  The TaskGroup has fewer than 5 dependencies across *all* tasks in the group.
+
+    We don't just say "The task has no dependencies", because real-world cases like
+    :obj:`dask.array.from_zarr` and :obj:`dask.array.from_array` produce graphs like the one
+    above, where the data-creation tasks (``a b c d``) all share one dependency
+    (``X``)---the Zarr dataset, for example. Though ``a b c d`` are not technically
+    root tasks, we want to treat them as such, hence allowing a small number of trivial
+    dependencies shard by all tasks.
+
+Then, we use the same priority described in :ref:`priority-break-ties` to
+determine which tasks are related. This depth-first-with-child-weights metric
+can usually be used to properly segment the leaves of a graph into decently
+well-separated sub-graphs with relatively low inter-sub-graph connectedness.
+
+Iterating through tasks in this priority order, we assign a batch of subsequent tasks
+to a worker, then select a new worker (the least-busy one) and repeat.
+
+Though this does not provide perfect initial task assignment (a handful of sibling
+tasks may be split across workers), it does well in most cases, while adding
+minimal scheduling overhead.
+
+Initial task placement is a forward-looking decision. By colocating related root tasks,
+we ensure that their downstream tasks are set up for success.
+
+Downstream Task Placement
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+When initial tasks are well-placed, placing subsequent tasks is backwards-looking:
+where can the task run the soonest, considering both data transfer and worker busyness?
+
+Tasks that don't meet the root-ish criteria described above are selected as follows:
+
+1.  If the task has no dependencies and no restrictions, then we find the
+    least-occupied worker.
 2.  Otherwise, if a task has user-provided restrictions (for example it must
     run on a machine with a GPU) then we restrict the available pool of workers
-    to just that set, otherwise we consider all workers
-3.  From among this pool of workers we determine the workers to whom the least
+    to just that set, otherwise we consider all workers.
+3.  From among this pool of workers, we determine the workers to whom the least
     amount of data would need to be transferred.
 4.  We break ties by choosing the worker that currently has the fewest tasks,
     counting both those tasks in memory and those tasks processing currently.
 
 This process is easy to change (and indeed this document may be outdated).  We
-encourage readers to inspect the ``decide_worker`` function in scheduler.py
+encourage readers to inspect the ``decide_worker`` functions in ``scheduler.py``.
 
 .. currentmodule:: distributed.scheduler
 
 .. autosummary:: decide_worker
 
+.. autosummary:: Scheduler.decide_worker
+
 
 Choosing Tasks
 --------------
@@ -64,6 +131,8 @@ before starting new work.  This often conflicts with the
 first-come-first-served objective but often results in shorter total runtimes
 and significantly reduced memory footprints.
 
+.. _priority-break-ties:
+
 Break ties with children and depth
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -80,18 +149,6 @@ more complex and are described in detail in `dask/order.py`_
 
 .. _`dask/order.py`: https://github.com/dask/dask/blob/main/dask/order.py
 
-Initial Task Placement
-~~~~~~~~~~~~~~~~~~~~~~
-
-When a new large batch of tasks come in and there are many idle workers then we
-want to give each worker a set of tasks that are close together/related and
-unrelated from the tasks given to other workers.  This usually avoids
-inter-worker communication down the line.  The same
-depth-first-with-child-weights priority given to workers described above can
-usually be used to properly segment the leaves of a graph into decently well
-separated sub-graphs with relatively low inter-sub-graph connectedness.
-
-
 First-Come-First-Served, Coarsely
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
diff --git a/docs/source/scheduling-state.rst b/docs/source/scheduling-state.rst
index 74a44573d59..665d7c7fe2d 100644
--- a/docs/source/scheduling-state.rst
+++ b/docs/source/scheduling-state.rst
@@ -310,5 +310,6 @@ API
 
 .. autoclass:: Scheduler
    :members:
+   :inherited-members:
 
 .. autofunction:: decide_worker

From c6a51ce213f6ce11ba13667f18a4983489ce8435 Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Tue, 6 Jul 2021 18:18:40 +0200
Subject: [PATCH 1357/1550] Fix flaky `test_oversubscribing_leases` (#5030)

---
 distributed/tests/test_semaphore.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/distributed/tests/test_semaphore.py b/distributed/tests/test_semaphore.py
index bf671df9fff..228ff5e7d4b 100644
--- a/distributed/tests/test_semaphore.py
+++ b/distributed/tests/test_semaphore.py
@@ -338,7 +338,6 @@ async def test_retry_acquire(c, s, a, b):
         assert result is False
 
 
-@pytest.mark.flaky(reruns=10, reruns_delay=5, condition=WINDOWS)
 @gen_cluster(
     client=True,
     config={
@@ -437,12 +436,13 @@ def observe_state(sem):
     logs = caplog.getvalue().split("\n")
     timeouts = [log for log in logs if "timed out" in log]
     refresh_unknown = [log for log in logs if "Refreshing an unknown lease ID" in log]
-    assert len(timeouts) == 2
-    assert len(refresh_unknown) == 2
+    assert len(timeouts) >= 2
+    assert len(refresh_unknown) >= 2
 
     assert sorted(payload) == [0, 1]
     # Back to normal
-    assert await sem.get_value() == 0
+    while await sem.get_value():
+        await asyncio.sleep(0.01)
 
 
 @gen_cluster(client=True)

From e93825906e169fb3ae45d9ebc3ec37a08ec5c03b Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 6 Jul 2021 09:56:52 -0700
Subject: [PATCH 1358/1550] Move worker preload before scheduler address is set
 (#5024)

---
 distributed/nanny.py              | 62 +++++++++++++++----------------
 distributed/tests/test_preload.py | 22 +++++++++++
 distributed/worker.py             | 58 ++++++++++++++---------------
 3 files changed, 82 insertions(+), 60 deletions(-)

diff --git a/distributed/nanny.py b/distributed/nanny.py
index 882bc783cb8..d891a65fd7e 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -104,6 +104,37 @@ def __init__(
         assert isinstance(self.security, Security)
         self.connection_args = self.security.get_connection_args("worker")
 
+        if local_dir is not None:
+            warnings.warn("The local_dir keyword has moved to local_directory")
+            local_directory = local_dir
+
+        if local_directory is None:
+            local_directory = dask.config.get("temporary-directory") or os.getcwd()
+            if not os.path.exists(local_directory):
+                os.makedirs(local_directory)
+            self._original_local_dir = local_directory
+            local_directory = os.path.join(local_directory, "dask-worker-space")
+        else:
+            self._original_local_dir = local_directory
+
+        self.local_directory = local_directory
+
+        self.preload = preload
+        if self.preload is None:
+            self.preload = dask.config.get("distributed.worker.preload")
+        self.preload_argv = preload_argv
+        if self.preload_argv is None:
+            self.preload_argv = dask.config.get("distributed.worker.preload-argv")
+
+        if preload_nanny is None:
+            preload_nanny = dask.config.get("distributed.nanny.preload")
+        if preload_nanny_argv is None:
+            preload_nanny_argv = dask.config.get("distributed.nanny.preload-argv")
+
+        self.preloads = preloading.process_preloads(
+            self, preload_nanny, preload_nanny_argv, file_dir=self.local_directory
+        )
+
         if scheduler_file:
             cfg = json_load_robust(scheduler_file)
             self.scheduler_addr = cfg["address"]
@@ -130,18 +161,6 @@ def __init__(
         self.resources = resources
         self.death_timeout = parse_timedelta(death_timeout)
 
-        self.preload = preload
-        if self.preload is None:
-            self.preload = dask.config.get("distributed.worker.preload")
-        self.preload_argv = preload_argv
-        if self.preload_argv is None:
-            self.preload_argv = dask.config.get("distributed.worker.preload-argv")
-
-        if preload_nanny is None:
-            preload_nanny = dask.config.get("distributed.nanny.preload")
-        if preload_nanny_argv is None:
-            preload_nanny_argv = dask.config.get("distributed.nanny.preload-argv")
-
         self.Worker = Worker if worker_class is None else worker_class
         self.env = env or {}
         self.config = config or dask.config.config
@@ -160,25 +179,6 @@ def __init__(
             "distributed.worker.memory.terminate"
         )
 
-        if local_dir is not None:
-            warnings.warn("The local_dir keyword has moved to local_directory")
-            local_directory = local_dir
-
-        if local_directory is None:
-            local_directory = dask.config.get("temporary-directory") or os.getcwd()
-            if not os.path.exists(local_directory):
-                os.makedirs(local_directory)
-            self._original_local_dir = local_directory
-            local_directory = os.path.join(local_directory, "dask-worker-space")
-        else:
-            self._original_local_dir = local_directory
-
-        self.local_directory = local_directory
-
-        self.preloads = preloading.process_preloads(
-            self, preload_nanny, preload_nanny_argv, file_dir=self.local_directory
-        )
-
         self.services = services
         self.name = name
         self.quiet = quiet
diff --git a/distributed/tests/test_preload.py b/distributed/tests/test_preload.py
index 20650ff49cf..75946676d5c 100644
--- a/distributed/tests/test_preload.py
+++ b/distributed/tests/test_preload.py
@@ -173,3 +173,25 @@ def dask_setup(dask_server):
         assert "12345/preload" in log.getvalue()
     finally:
         server.stop()
+
+
+@pytest.mark.asyncio
+async def test_scheduler_startup(cleanup):
+    async with Scheduler(port=0) as s:
+        text = f"""
+import dask
+dask.config.set(scheduler_address="{s.address}")
+"""
+        async with Worker(preload=text) as w:
+            assert w.scheduler.address == s.address
+
+
+@pytest.mark.asyncio
+async def test_scheduler_startup_nanny(cleanup):
+    async with Scheduler(port=0) as s:
+        text = f"""
+import dask
+dask.config.set(scheduler_address="{s.address}")
+"""
+        async with Nanny(preload_nanny=text) as w:
+            assert w.scheduler.address == s.address
diff --git a/distributed/worker.py b/distributed/worker.py
index c44bff14c93..94ae0418a2d 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -507,6 +507,35 @@ def __init__(
 
         self._setup_logging(logger)
 
+        if local_dir is not None:
+            warnings.warn("The local_dir keyword has moved to local_directory")
+            local_directory = local_dir
+
+        if not local_directory:
+            local_directory = dask.config.get("temporary-directory") or os.getcwd()
+
+        os.makedirs(local_directory, exist_ok=True)
+        local_directory = os.path.join(local_directory, "dask-worker-space")
+
+        with warn_on_duration(
+            "1s",
+            "Creating scratch directories is taking a surprisingly long time. "
+            "This is often due to running workers on a network file system. "
+            "Consider specifying a local-directory to point workers to write "
+            "scratch data to a local disk.",
+        ):
+            self._workspace = WorkSpace(os.path.abspath(local_directory))
+            self._workdir = self._workspace.new_work_dir(prefix="worker-")
+            self.local_directory = self._workdir.dir_path
+
+        if preload is None:
+            preload = dask.config.get("distributed.worker.preload")
+        if preload_argv is None:
+            preload_argv = dask.config.get("distributed.worker.preload-argv")
+        self.preloads = preloading.process_preloads(
+            self, preload, preload_argv, file_dir=self.local_directory
+        )
+
         if scheduler_file:
             cfg = json_load_robust(scheduler_file)
             scheduler_addr = cfg["address"]
@@ -549,35 +578,6 @@ def __init__(
         if silence_logs:
             silence_logging(level=silence_logs)
 
-        if local_dir is not None:
-            warnings.warn("The local_dir keyword has moved to local_directory")
-            local_directory = local_dir
-
-        if not local_directory:
-            local_directory = dask.config.get("temporary-directory") or os.getcwd()
-
-        os.makedirs(local_directory, exist_ok=True)
-        local_directory = os.path.join(local_directory, "dask-worker-space")
-
-        with warn_on_duration(
-            "1s",
-            "Creating scratch directories is taking a surprisingly long time. "
-            "This is often due to running workers on a network file system. "
-            "Consider specifying a local-directory to point workers to write "
-            "scratch data to a local disk.",
-        ):
-            self._workspace = WorkSpace(os.path.abspath(local_directory))
-            self._workdir = self._workspace.new_work_dir(prefix="worker-")
-            self.local_directory = self._workdir.dir_path
-
-        if preload is None:
-            preload = dask.config.get("distributed.worker.preload")
-        if preload_argv is None:
-            preload_argv = dask.config.get("distributed.worker.preload-argv")
-        self.preloads = preloading.process_preloads(
-            self, preload, preload_argv, file_dir=self.local_directory
-        )
-
         if isinstance(security, dict):
             security = Security(**security)
         self.security = security or Security()

From 88b99ae2ab2fca96395fe1975b0e0ebb8f7b25a0 Mon Sep 17 00:00:00 2001
From: Ian Rose <ian.r.rose@gmail.com>
Date: Wed, 7 Jul 2021 12:45:47 +0200
Subject: [PATCH 1359/1550] Add pandas to the list of checked packages for
 `client.get_versions()`. (#5029)

---
 distributed/versions.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/distributed/versions.py b/distributed/versions.py
index 13a282977bc..5932a92c2d4 100644
--- a/distributed/versions.py
+++ b/distributed/versions.py
@@ -18,6 +18,7 @@
 
 optional_packages = [
     ("numpy", lambda p: p.__version__),
+    ("pandas", lambda p: p.__version__),
     ("lz4", lambda p: p.__version__),
     ("blosc", lambda p: p.__version__),
 ]

From 9520c49d89fd21dbb9924b0458db5d4ecf161c05 Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Thu, 8 Jul 2021 10:04:04 +0200
Subject: [PATCH 1360/1550] Dont require workers to report to scheduler if
 scheduler shuts down (#5032)

If the scheduler is shutting down it is not necessary for the workers to
report back since there is no failrure handling happening anymore. The
report would otherwise delay worker shutdown and may even result in
connection failure logs
---
 distributed/scheduler.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index fe8d0af4b50..e0dd312d3a2 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -3833,7 +3833,10 @@ async def close(self, comm=None, fast=False, close_workers=False):
         if close_workers:
             await self.broadcast(msg={"op": "close_gracefully"}, nanny=True)
             for worker in parent._workers_dv:
-                self.worker_send(worker, {"op": "close"})
+                # Report would require the worker to unregister with the
+                # currently closing scheduler. This is not necessary and might
+                # delay shutdown of the worker unnecessarily
+                self.worker_send(worker, {"op": "close", "report": False})
             for i in range(20):  # wait a second for send signals to clear
                 if parent._workers_dv:
                     await asyncio.sleep(0.05)

From 04b6be4ffd5051a3bc43d6d0b2450b9a05370f6c Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Thu, 8 Jul 2021 15:13:38 +0200
Subject: [PATCH 1361/1550] Use native GH actions cancel feature (#5037)

* Use native GH actions cancel feature

Co-authored-by: crusaderky <crusaderky@gmail.com>
---
 .github/workflows/cancel.yml | 15 ---------------
 .github/workflows/tests.yaml |  6 ++++++
 2 files changed, 6 insertions(+), 15 deletions(-)
 delete mode 100644 .github/workflows/cancel.yml

diff --git a/.github/workflows/cancel.yml b/.github/workflows/cancel.yml
deleted file mode 100644
index 8161ba1808d..00000000000
--- a/.github/workflows/cancel.yml
+++ /dev/null
@@ -1,15 +0,0 @@
-name: Cancel
-
-on:
-  workflow_run:
-    workflows: [Tests]
-    types:
-      - requested
-
-jobs:
-  cancel:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: styfle/cancel-workflow-action@0.8.0
-      with:
-        workflow_id: ${{ github.event.workflow.id }}
\ No newline at end of file
diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
index 0c0bb0a4a38..a020cb92630 100644
--- a/.github/workflows/tests.yaml
+++ b/.github/workflows/tests.yaml
@@ -2,6 +2,12 @@ name: Tests
 
 on: [push, pull_request]
 
+# When this workflow is queued, automatically cancel any previous running
+# or pending jobs from the same branch
+concurrency:
+  group: ${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   test:
     runs-on: ${{ matrix.os }}

From 50aa29678d4c9442b7011e386dec7b244ad36bd2 Mon Sep 17 00:00:00 2001
From: Naty Clementi <natyclementi@gmail.com>
Date: Fri, 9 Jul 2021 15:17:19 -0400
Subject: [PATCH 1362/1550] Fix Nbytes jitter - less expensive  (#5043)

---
 distributed/dashboard/components/scheduler.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index d5e8365a808..62239f4fbed 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -282,6 +282,7 @@ def __init__(self, scheduler, width=600, **kwargs):
             self.root.xaxis[0].formatter = NumeralTickFormatter(format="0.0 b")
             self.root.xaxis.major_label_orientation = XLABEL_ORIENTATION
             self.root.xaxis.minor_tick_line_alpha = 0
+            self.root.x_range = Range1d(start=0)
             self.root.yaxis.visible = False
             self.root.ygrid.visible = False
 
@@ -342,7 +343,7 @@ def update(self):
             }
 
             x_end = max(limit, meminfo.process + meminfo.managed_spilled)
-            self.root.x_range = DataRange1d(start=0, end=x_end, range_padding=0)
+            self.root.x_range.end = x_end
 
             title = f"Bytes stored: {format_bytes(meminfo.process)}"
             if meminfo.managed_spilled:
@@ -399,6 +400,7 @@ def __init__(self, scheduler, width=600, **kwargs):
             self.root.xaxis[0].formatter = NumeralTickFormatter(format="0.0 b")
             self.root.xaxis.major_label_orientation = XLABEL_ORIENTATION
             self.root.xaxis.minor_tick_line_alpha = 0
+            self.root.x_range = Range1d(start=0)
             self.root.yaxis.visible = False
             self.root.ygrid.visible = False
 
@@ -506,7 +508,7 @@ def quadlist(i) -> list:
                 k: [vi for vi, w in zip(v, width) if w] for k, v in result.items()
             }
 
-            self.root.x_range = DataRange1d(start=0, end=max_limit, range_padding=0)
+            self.root.x_range.end = max_limit
             update(self.source, result)
 
 

From d7b64167dd3c643381425139f6f5ad4fb5363928 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Fri, 9 Jul 2021 14:41:43 -0500
Subject: [PATCH 1363/1550] bump version to 2021.07.0

---
 docs/source/changelog.rst | 50 ++++++++++++++++++++++++++++++++++++++-
 requirements.txt          |  2 +-
 2 files changed, 50 insertions(+), 2 deletions(-)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index 3fa30848dd7..3978e6f848d 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,51 @@
 Changelog
 =========
 
+2021.07.0
+---------
+
+Released on July 9, 2021
+
+- Fix Nbytes jitter - less expensive (#5043) `Naty Clementi`_
+- Use native GH actions cancel feature (:pr:`5037`) `Florian Jetter`_
+- Don't require workers to report to scheduler if scheduler shuts down (:pr:`5032`) `Florian Jetter`_
+- Add pandas to the list of checked packages for ``client.get_versions()`` (:pr:`5029`) `Ian Rose`_
+- Move worker preload before scheduler address is set (:pr:`5024`) `Matthew Rocklin`_
+- Fix flaky ``test_oversubscribing_leases`` (:pr:`5030`) `Florian Jetter`_
+- Update scheduling policy docs for #4967 (:pr:`5018`) `Gabe Joseph`_
+- Add echo handler to ``Server`` class (:pr:`5020`) `Matthew Rocklin`_
+- Also include pngs when bundling package (:pr:`5016`) `Ian Rose`_
+- Remove duplicated dashboard panes (:pr:`5017`) `Ian Rose`_
+- Fix worker memory dashboard flickering (:pr:`4997`) `Naty Clementi`_
+- Tabs on bottom left corner on dashboard (:pr:`5006`) `Naty Clementi`_
+- Rename nbytes widgets (:pr:`4878`) `crusaderky`_
+- Co-assign root-ish tasks (:pr:`4967`) `Gabe Joseph`_
+- ``OSError`` tweaks (:pr:`5003`) `crusaderky`_
+- Update imports to ``cudf.testing._utils`` (:pr:`5005`) `Peter Andreas Entschev`_
+- Ensure shuffle split default durations uses proper prefix (:pr:`4991`) `Florian Jetter`_
+- Follow up ``pyupgrade`` formatting (:pr:`4993`) `Florian Jetter`_
+- Rename plot dropdown (:pr:`4992`) `James Bourbeau`_
+- Pyupgrade (:pr:`4741`) `Florian Jetter`_
+- Misc Sphinx tweaks (:pr:`4988`) `crusaderky`_
+- No longer hold dependencies of erred tasks in memory #4918 `Florian Jetter`_
+- Add maximum shard size to config (:pr:`4986`) `Matthew Rocklin`_
+- Ensure shuffle split operations are blacklisted from work stealing (:pr:`4964`) `Florian Jetter`_
+- Add dropdown menu to access individual plots (:pr:`4984`) `Jacob Tomlinson`_
+- Edited the path to ``scheduler.py`` (:pr:`4983`) `Freyam Mehta`_
+- Task Group Graph Visualization (:pr:`4886`) `Naty Clementi`_
+- Remove more internal references to deprecated utilities (:pr:`4971`) `James Bourbeau`_
+- Restructure nbytes hover (:pr:`4952`) `Naty Clementi`_
+- Except more errors in ``pynvml.nvmlInit()`` (:pr:`4970`) `gerrymanoim`_
+- Add occupancy as individual plot (:pr:`4963`) `Naty Clementi`_
+- Deprecate utilities which have moved to dask (:pr:`4966`) `James Bourbeau`_
+- Ensure connectionpool does not leave comms if closed mid connect (:pr:`4951`) `Florian Jetter`_
+- Add support for registering scheduler plugins from `Client` (:pr:`4808`) `Doug Davis`_
+- Stealing dashboard fixes (:pr:`4948`) `Florian Jetter`_
+- Allow requirements verification to be ignored when loading backends from entrypoints (:pr:`4961`) `Florian Jetter`_
+- Add ``Log`` and ``Logs`` to API docs (:pr:`4946`) `James Bourbeau`_
+- Support fixtures and ``pytest.mark.parametrize`` with ``gen_cluster`` (:pr:`4958`) `Gabe Joseph`_
+
+
 2021.06.2
 ---------
 
@@ -2443,4 +2488,7 @@ significantly without many new features.
 .. _`Naty Clementi`: https://github.com/ncclementi
 .. _`Nathan Danielsen`: https://github.com/ndanielsen
 .. _`Torsten Wörtwein`: https://github.com/twoertwein
-.. _`ArtinSarraf`: https://github.com/ArtinSarraf
\ No newline at end of file
+.. _`ArtinSarraf`: https://github.com/ArtinSarraf
+.. _`Gabe Joseph`: https://github.com/gjoseph92
+.. _`Freyam Mehta`: https://github.com/freyam
+.. _`gerrymanoim`: https://github.com/gerrymanoim
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index 70f7b61dd13..f08d41d60fa 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,6 @@
 click >= 6.6
 cloudpickle >= 1.5.0
-dask == 2021.06.2
+dask == 2021.07.0
 msgpack >= 0.6.0
 psutil >= 5.0
 sortedcontainers !=2.0.0, !=2.0.1

From 8dbb8df9dc585fd77f57763eac9159098182ee15 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Mon, 12 Jul 2021 16:35:23 -0500
Subject: [PATCH 1364/1550] Add bokeh `mode` parameter to performance reports
 (#5025)

---
 distributed/client.py            | 11 +++++++----
 distributed/scheduler.py         |  6 ++++--
 distributed/tests/test_client.py | 11 +++++++++--
 docs/source/conf.py              |  1 +
 4 files changed, 21 insertions(+), 8 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 58620aae831..fa18dfd3d17 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -4747,13 +4747,15 @@ class performance_report:
 
     Parameters
     ----------
-    filename: str (optional)
+    filename: str, optional
         The filename to save the performance report locally
 
-    stacklevel: int (optional)
+    stacklevel: int, optional
         The code execution frame utilized for populating the Calling Code section
         of the report. Defaults to `1` which is the frame calling ``performance_report``
 
+    mode: str, optional
+        Mode parameter to pass to :func:`bokeh.io.output.output_file`. Defaults to ``None``.
 
     Examples
     --------
@@ -4764,10 +4766,11 @@ class performance_report:
     $ open myfile.html
     """
 
-    def __init__(self, filename="dask-report.html", stacklevel=1):
+    def __init__(self, filename="dask-report.html", stacklevel=1, mode=None):
         self.filename = filename
         # stacklevel 0 or less - shows dask internals which likely isn't helpful
         self._stacklevel = stacklevel if stacklevel > 0 else 1
+        self.mode = mode
 
     async def __aenter__(self):
         self.start = time()
@@ -4784,7 +4787,7 @@ async def __aexit__(self, typ, value, traceback, code=None):
             except Exception:
                 code = ""
         data = await get_client().scheduler.performance_report(
-            start=self.start, last_count=self.last_count, code=code
+            start=self.start, last_count=self.last_count, code=code, mode=self.mode
         )
         with open(self.filename, "w") as f:
             f.write(data)
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index e0dd312d3a2..f7f4faaf374 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -6952,7 +6952,9 @@ async def get_profile_metadata(
 
         return {"counts": counts, "keys": keys}
 
-    async def performance_report(self, comm=None, start=None, last_count=None, code=""):
+    async def performance_report(
+        self, comm=None, start=None, last_count=None, code="", mode=None
+    ):
         parent: SchedulerState = cast(SchedulerState, self)
         stop = time()
         # Profiles
@@ -7092,7 +7094,7 @@ def profile_to_figure(state):
         from bokeh.plotting import output_file, save
 
         with tmpfile(extension=".html") as fn:
-            output_file(filename=fn, title="Dask Performance Report")
+            output_file(filename=fn, title="Dask Performance Report", mode=mode)
             template_directory = os.path.join(
                 os.path.dirname(os.path.abspath(__file__)), "dashboard", "templates"
             )
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index f78351e8a92..6a45e28bc7a 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -6353,7 +6353,7 @@ async def test_performance_report(c, s, a, b):
     pytest.importorskip("bokeh")
     da = pytest.importorskip("dask.array")
 
-    async def f(stacklevel):
+    async def f(stacklevel, mode=None):
         """
         We wrap this in a function so that the assertions aren't in the
         performanace report itself
@@ -6362,7 +6362,9 @@ async def f(stacklevel):
         """
         x = da.random.random((1000, 1000), chunks=(100, 100))
         with tmpfile(extension="html") as fn:
-            async with performance_report(filename=fn, stacklevel=stacklevel):
+            async with performance_report(
+                filename=fn, stacklevel=stacklevel, mode=mode
+            ):
                 await c.compute((x + x.T).sum())
 
             with open(fn) as f:
@@ -6393,6 +6395,11 @@ async def f(stacklevel):
     assert "Also, we want this comment to appear" in data
     assert "Dask Performance Report" in data
 
+    data = await f(stacklevel=1, mode="inline")
+    assert "cdn.bokeh.org" not in data
+    data = await f(stacklevel=1, mode="cdn")
+    assert "cdn.bokeh.org" in data
+
 
 @pytest.mark.asyncio
 async def test_client_gather_semaphore_loop(cleanup):
diff --git a/docs/source/conf.py b/docs/source/conf.py
index 7d5021c6115..69d7e78de0e 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -384,6 +384,7 @@
     "python": ("https://docs.python.org/3", None),
     "numpy": ("https://docs.scipy.org/doc/numpy", None),
     "dask": ("https://docs.dask.org/en/latest", None),
+    "bokeh": ("https://docs.bokeh.org/en/latest", None),
 }
 
 # Redirects

From 07fe11d688f896a887aad6caf258de5c999af73b Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Tue, 13 Jul 2021 00:48:55 +0100
Subject: [PATCH 1365/1550] Add more useful exception message on TLS cert
 mismatch (#5040)

---
 distributed/comm/tcp.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py
index b938bd3752a..de6cee016d8 100644
--- a/distributed/comm/tcp.py
+++ b/distributed/comm/tcp.py
@@ -5,7 +5,7 @@
 import struct
 import sys
 import weakref
-from ssl import SSLError
+from ssl import SSLCertVerificationError, SSLError
 
 from tornado import gen
 
@@ -386,6 +386,11 @@ async def connect(self, address, deserialize=True, **connection_args):
         except StreamClosedError as e:
             # The socket connect() call failed
             convert_stream_closed_error(self, e)
+        except SSLCertVerificationError as err:
+            raise FatalCommClosedError(
+                "TLS certificate does not match. Check your security settings. "
+                "More info at https://distributed.dask.org/en/latest/tls.html"
+            ) from err
         except SSLError as err:
             raise FatalCommClosedError() from err
 

From 2c4efaee9dac945c0649acc9ad475a6822fc0b2f Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Tue, 13 Jul 2021 18:28:35 +0200
Subject: [PATCH 1366/1550] Ensure heartbeats after cancelation do not raise
 KeyErrors (#5053)

---
 distributed/scheduler.py            |  4 ++-
 distributed/tests/test_scheduler.py | 44 +++++++++++++++++++++++++++++
 2 files changed, 47 insertions(+), 1 deletion(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index f7f4faaf374..d9e2bec5e2c 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -3947,7 +3947,9 @@ def heartbeat_worker(
         ws._last_seen = local_now
         if executing is not None:
             ws._executing = {
-                parent._tasks[key]: duration for key, duration in executing.items()
+                parent._tasks[key]: duration
+                for key, duration in executing.items()
+                if key in parent._tasks
             }
 
         ws._metrics = metrics
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 1608a7d67b0..8640baf1745 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -2936,3 +2936,47 @@ async def test_transition_counter(c, s, a, b):
     assert s.transition_counter == 0
     await c.submit(inc, 1)
     assert s.transition_counter > 1
+
+
+@pytest.mark.slow
+@gen_cluster(
+    client=True,
+    nthreads=[("127.0.0.1", 1) for _ in range(10)],
+    # typical runtime just 2-3s but on CI this may increase significantly
+    timeout=60,
+)
+async def test_worker_heartbeat_after_cancel(c, s, *workers):
+    """This test is intended to ensure that after cancelation of a graph, the
+    worker heartbeat is always successful. The hearbeat may not be successful if
+    the worker and scheduler state drift and the scheduler doesn't handle
+    unknown information gracefully. One example would be a released/cancelled
+    computation where the worker returns metrics about duration, type, etc. and
+    the scheduler doesn't handle the forgotten task gracefully.
+
+    Failures are not triggered reliably since the race conditions for this error
+    case are very hard to produce. Likelihood of failure increases with the
+    number of workers.
+
+    See also https://github.com/dask/distributed/issues/4587
+    """
+    da = pytest.importorskip("dask.array")
+    for w in workers:
+        w.periodic_callbacks["heartbeat"].stop()
+    x = da.random.random((2000000, 100), chunks=(10000, None))
+    svd = da.linalg.svd(x)
+
+    futs = c.compute(svd)
+
+    while not s.tasks:
+        await asyncio.sleep(0.001)
+
+    while sum(w.executing_count for w in workers) < len(workers) / 2:
+        await asyncio.sleep(0.001)
+
+    await c.cancel(futs)
+
+    while s.tasks:
+        await asyncio.sleep(0.001)
+
+    while any(w.tasks for w in workers):
+        await asyncio.gather(*[w.heartbeat() for w in workers])

From 67f57bc0c880101bc3336557c4dce43d4fa5ec43 Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Wed, 14 Jul 2021 09:16:06 +0100
Subject: [PATCH 1367/1550] Improve CI stability (#5022)

* Drastically reduced CI failure rates
* Extensive review of all timeouts
* Statistical review of flaky tests
* Generate and publish junit reports for later ingestion by your choice of aggregator
* On Linux and MacOS, pytest-timeout will now kill the individual offending test instead of the whole suite
* pytest-timeout should kick in a lot less frequently now
* gen_test and gen_cluster no longer accept timeout=None (which meant letting pytest-timeout kill everything off)
* Hopefully increased mamba resiliency to transitory network issues
---
 .github/workflows/tests.yaml                  |  41 +++-
 continuous_integration/condarc                |   9 +
 distributed/cli/tests/test_dask_scheduler.py  |  14 +-
 distributed/cli/tests/test_dask_worker.py     |  10 +-
 distributed/comm/tests/test_comms.py          |   4 +-
 distributed/comm/tests/test_ucx.py            |   2 +-
 distributed/comm/tests/test_ws.py             |   4 +-
 distributed/compatibility.py                  |   1 +
 .../dashboard/tests/test_scheduler_bokeh.py   |  10 +-
 distributed/deploy/local.py                   |   4 +-
 distributed/deploy/old_ssh.py                 |   4 +-
 distributed/deploy/tests/test_adaptive.py     |  31 ++-
 .../deploy/tests/test_adaptive_core.py        |   4 +-
 distributed/deploy/tests/test_local.py        |  16 +-
 distributed/deploy/tests/test_spec_cluster.py |  49 ++---
 .../diagnostics/tests/test_progress.py        |  11 +-
 .../tests/test_scheduler_plugin.py            |   4 +-
 distributed/diagnostics/tests/test_widgets.py |   3 -
 distributed/distributed.yaml                  |   2 +-
 distributed/nanny.py                          |   6 +-
 distributed/protocol/tests/test_pickle.py     |   1 -
 distributed/scheduler.py                      |   9 +-
 distributed/tests/test_actor.py               |   2 -
 distributed/tests/test_as_completed.py        |   7 +-
 distributed/tests/test_asyncprocess.py        |  16 +-
 distributed/tests/test_client.py              | 118 +++++------
 distributed/tests/test_client_executor.py     |  11 +-
 distributed/tests/test_collections.py         |   2 +-
 distributed/tests/test_diskutils.py           |   8 +-
 distributed/tests/test_failed_workers.py      |  36 ++--
 distributed/tests/test_nanny.py               |  52 ++---
 distributed/tests/test_publish.py             |   8 +-
 distributed/tests/test_pubsub.py              |   2 +-
 distributed/tests/test_queues.py              |   2 +-
 distributed/tests/test_resources.py           |   5 +-
 distributed/tests/test_scheduler.py           | 196 ++++++++----------
 distributed/tests/test_semaphore.py           |  10 +-
 distributed/tests/test_steal.py               |  25 +--
 distributed/tests/test_stress.py              |  39 ++--
 distributed/tests/test_tls_functional.py      |   2 +-
 distributed/tests/test_utils.py               |  23 +-
 distributed/tests/test_utils_test.py          |   2 +-
 distributed/tests/test_variable.py            |   4 +-
 distributed/tests/test_worker.py              |  62 ++----
 distributed/utils_test.py                     |  15 +-
 distributed/worker.py                         |   2 +-
 setup.cfg                                     |   7 +
 47 files changed, 400 insertions(+), 495 deletions(-)
 create mode 100644 continuous_integration/condarc

diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
index a020cb92630..8301b6713eb 100644
--- a/.github/workflows/tests.yaml
+++ b/.github/workflows/tests.yaml
@@ -11,6 +11,7 @@ concurrency:
 jobs:
   test:
     runs-on: ${{ matrix.os }}
+    timeout-minutes: 180
 
     strategy:
       fail-fast: false
@@ -18,10 +19,18 @@ jobs:
         os: [ubuntu-latest, windows-latest, macos-latest]
         python-version: ["3.7", "3.8", "3.9"]
 
-        # Uncomment to stress-test the test suite for random failures
+        # Uncomment to stress-test the test suite for random failures.
+        # Must also change env.TEST_ID below.
         # This will take a LONG time and delay all PRs across the whole github.com/dask!
+        # To avoid hamstringing other people, change 'on: [push, pull_request]' above
+        # to just 'on: [push]'; this way the stress test will run exclusively in your
+        # branch (https://github.com/<your name>/distributed/actions).
         # run: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]
 
+    env:
+      TEST_ID: ${{ matrix.os }}-${{ matrix.python-version }}
+      # TEST_ID: ${{ matrix.os }}-${{ matrix.python-version }}-${{ matrix.run }}
+
     steps:
       - name: Checkout source
         uses: actions/checkout@v2
@@ -33,13 +42,15 @@ jobs:
         with:
           miniforge-variant: Mambaforge
           miniforge-version: latest
+          condarc-file: continuous_integration/condarc
           use-mamba: true
-          channels: conda-forge,defaults
-          channel-priority: true
           python-version: ${{ matrix.python-version }}
           environment-file: continuous_integration/environment-${{ matrix.python-version }}.yaml
           activate-environment: dask-distributed
-          auto-activate-base: false
+
+      - name: Show conda options
+        shell: bash -l {0}
+        run: conda config --show
 
       - name: Install stacktrace
         shell: bash -l {0}
@@ -83,7 +94,14 @@ jobs:
         if: ${{ matrix.os != 'windows-latest' }}
         run: bash continuous_integration/scripts/setup_ssh.sh
 
+      - name: Reconfigure pytest-timeout
+        shell: bash -l {0}
+        # No SIGALRM available on Windows
+        if: ${{ matrix.os != 'windows-latest' }}
+        run: sed -i.bak 's/timeout_method = thread/timeout_method = signal/' setup.cfg
+
       - name: Test
+        id: run_tests
         shell: bash -l {0}
         env:
           PYTHONFAULTHANDLER: 1
@@ -93,10 +111,21 @@ jobs:
               # https://github.com/dask/distributed/issues/4514
               export DISABLE_IPV6=1
           fi
-
           source continuous_integration/scripts/set_ulimit.sh
-          pytest distributed -m "not avoid_ci" --runslow
+
+          pytest distributed -m "not avoid_ci" --runslow \
+            --junitxml reports/pytest.xml -o junit_suite_name=$TEST_ID
 
       # - name: Debug with tmate on failure
       #   if: ${{ failure() }}
       #   uses: mxschmitt/action-tmate@v3
+
+      - name: Upload test artifacts
+        # ensure this runs even if pytest fails
+        if: >
+          always() &&
+          (steps.run_tests.outcome == 'success' || steps.run_tests.outcome == 'failure')
+        uses: actions/upload-artifact@v2
+        with:
+          name: ${{ env.TEST_ID }}
+          path: reports
diff --git a/continuous_integration/condarc b/continuous_integration/condarc
new file mode 100644
index 00000000000..258ac5fa5db
--- /dev/null
+++ b/continuous_integration/condarc
@@ -0,0 +1,9 @@
+channels:
+  - conda-forge
+  - defaults
+channel_priority: true
+auto_activate_base: false
+remote_backoff_factor: 20
+remote_connect_timeout_secs: 20.0
+remote_max_retries: 10
+remote_read_timeout_secs: 60.0
diff --git a/distributed/cli/tests/test_dask_scheduler.py b/distributed/cli/tests/test_dask_scheduler.py
index da3990a949a..6b310d8e544 100644
--- a/distributed/cli/tests/test_dask_scheduler.py
+++ b/distributed/cli/tests/test_dask_scheduler.py
@@ -1,3 +1,4 @@
+import psutil
 import pytest
 
 pytest.importorskip("requests")
@@ -14,6 +15,7 @@
 import distributed
 import distributed.cli.dask_scheduler
 from distributed import Client, Scheduler
+from distributed.compatibility import LINUX
 from distributed.metrics import time
 from distributed.utils import get_ip, get_ip_interface, tmpfile
 from distributed.utils_test import (
@@ -118,9 +120,7 @@ def test_dashboard_non_standard_ports(loop):
         requests.get("http://localhost:4832/status/")
 
 
-@pytest.mark.skipif(
-    not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
-)
+@pytest.mark.skipif(not LINUX, reason="Need 127.0.0.2 to mean localhost")
 def test_dashboard_whitelist(loop):
     pytest.importorskip("bokeh")
     with pytest.raises(Exception):
@@ -144,7 +144,6 @@ def test_dashboard_whitelist(loop):
 
 
 def test_interface(loop):
-    psutil = pytest.importorskip("psutil")
     if_names = sorted(psutil.net_if_addrs())
     for if_name in if_names:
         try:
@@ -168,25 +167,24 @@ def test_interface(loop):
                 start = time()
                 while not len(c.nthreads()):
                     sleep(0.1)
-                    assert time() - start < 5
+                    assert time() - start < 30
                 info = c.scheduler_info()
                 assert "tcp://127.0.0.1" in info["address"]
                 assert all("127.0.0.1" == d["host"] for d in info["workers"].values())
 
 
-@pytest.mark.flaky(reruns=10, reruns_delay=5)
 def test_pid_file(loop):
     def check_pidfile(proc, pidfile):
         start = time()
         while not os.path.exists(pidfile):
             sleep(0.01)
-            assert time() < start + 5
+            assert time() < start + 30
 
         text = False
         start = time()
         while not text:
             sleep(0.01)
-            assert time() < start + 5
+            assert time() < start + 30
             with open(pidfile) as f:
                 text = f.read()
         pid = int(text)
diff --git a/distributed/cli/tests/test_dask_worker.py b/distributed/cli/tests/test_dask_worker.py
index 5615c7ff558..2d49b3b0977 100644
--- a/distributed/cli/tests/test_dask_worker.py
+++ b/distributed/cli/tests/test_dask_worker.py
@@ -6,7 +6,6 @@
 pytest.importorskip("requests")
 
 import os
-import sys
 from multiprocessing import cpu_count
 from time import sleep
 
@@ -14,6 +13,7 @@
 
 import distributed.cli.dask_worker
 from distributed import Client, Scheduler
+from distributed.compatibility import LINUX
 from distributed.deploy.utils import nprocesses_nthreads
 from distributed.metrics import time
 from distributed.utils import parse_ports, sync, tmpfile
@@ -275,9 +275,7 @@ def test_nprocs_expands_name(loop):
                     assert len(set(names)) == 4
 
 
-@pytest.mark.skipif(
-    not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
-)
+@pytest.mark.skipif(not LINUX, reason="Need 127.0.0.2 to mean localhost")
 @pytest.mark.parametrize("nanny", ["--nanny", "--no-nanny"])
 @pytest.mark.parametrize(
     "listen_address", ["tcp://0.0.0.0:39837", "tcp://127.0.0.2:39837"]
@@ -311,9 +309,7 @@ def func(dask_worker):
                 assert client.run(func) == {"tcp://127.0.0.2:39837": listen_address}
 
 
-@pytest.mark.skipif(
-    not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
-)
+@pytest.mark.skipif(not LINUX, reason="Need 127.0.0.2 to mean localhost")
 @pytest.mark.parametrize("nanny", ["--nanny", "--no-nanny"])
 @pytest.mark.parametrize("host", ["127.0.0.2", "0.0.0.0"])
 def test_respect_host_listen_address(loop, nanny, host):
diff --git a/distributed/comm/tests/test_comms.py b/distributed/comm/tests/test_comms.py
index a9539e83621..402149ac692 100644
--- a/distributed/comm/tests/test_comms.py
+++ b/distributed/comm/tests/test_comms.py
@@ -29,7 +29,6 @@
 )
 from distributed.comm.registry import backends, get_backend
 from distributed.comm.tcp import TCP, TCPBackend, TCPConnector
-from distributed.compatibility import WINDOWS
 from distributed.metrics import time
 from distributed.protocol import Serialized, deserialize, serialize, to_serialize
 from distributed.utils import get_ip, get_ipv6
@@ -110,7 +109,7 @@ async def debug_loop():
     while True:
         loop = ioloop.IOLoop.current()
         print(".", loop, loop._handlers)
-        await asyncio.sleep(0.50)
+        await asyncio.sleep(0.5)
 
 
 #
@@ -1107,7 +1106,6 @@ def check_out(deserialize_flag, out_value):
     await check_connector_deserialize(addr, True, msg, partial(check_out, True))
 
 
-@pytest.mark.flaky(reruns=10, reruns_delay=5, condition=WINDOWS)
 @pytest.mark.asyncio
 async def test_tcp_deserialize():
     await check_deserialize("tcp://")
diff --git a/distributed/comm/tests/test_ucx.py b/distributed/comm/tests/test_ucx.py
index 74e1e654975..de634b571d1 100644
--- a/distributed/comm/tests/test_ucx.py
+++ b/distributed/comm/tests/test_ucx.py
@@ -276,7 +276,7 @@ async def test_ucx_localcluster(processes, cleanup):
     ) as cluster:
         async with Client(cluster, asynchronous=True) as client:
             x = client.submit(inc, 1)
-            await x.result()
+            await x
             assert x.key in cluster.scheduler.tasks
             if not processes:
                 assert any(w.data == {x.key: 2} for w in cluster.workers.values())
diff --git a/distributed/comm/tests/test_ws.py b/distributed/comm/tests/test_ws.py
index c5e98abb8f2..54dd1940fd1 100644
--- a/distributed/comm/tests/test_ws.py
+++ b/distributed/comm/tests/test_ws.py
@@ -122,9 +122,9 @@ async def test_collections(cleanup):
 async def test_large_transfer(cleanup):
     np = pytest.importorskip("numpy")
     async with Scheduler(protocol="ws://") as s:
-        async with Worker(s.address, protocol="ws://") as w:
+        async with Worker(s.address, protocol="ws://"):
             async with Client(s.address, asynchronous=True) as c:
-                future = await c.scatter(np.random.random(1000000))
+                await c.scatter(np.random.random(1_000_000))
 
 
 @pytest.mark.asyncio
diff --git a/distributed/compatibility.py b/distributed/compatibility.py
index 982b7017951..5687c2b6157 100644
--- a/distributed/compatibility.py
+++ b/distributed/compatibility.py
@@ -8,6 +8,7 @@
 logging_names.update(logging._nameToLevel)
 
 PYPY = platform.python_implementation().lower() == "pypy"
+LINUX = sys.platform == "linux"
 MACOS = sys.platform == "darwin"
 WINDOWS = sys.platform.startswith("win")
 TORNADO6 = tornado.version_info[0] >= 6
diff --git a/distributed/dashboard/tests/test_scheduler_bokeh.py b/distributed/dashboard/tests/test_scheduler_bokeh.py
index 1cf3f1621ab..c271e5ac7c7 100644
--- a/distributed/dashboard/tests/test_scheduler_bokeh.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh.py
@@ -17,7 +17,6 @@
 from dask.utils import stringify
 
 from distributed.client import wait
-from distributed.compatibility import MACOS
 from distributed.dashboard import scheduler
 from distributed.dashboard.components.scheduler import (
     AggregateAction,
@@ -93,10 +92,8 @@ async def test_counters(c, s, a, b):
     await asyncio.sleep(0.1)
     ss.update()
 
-    start = time()
     while not len(ss.digest_sources["tick-duration"][0].data["x"]):
-        await asyncio.sleep(1)
-        assert time() < start + 5
+        await asyncio.sleep(0.01)
 
 
 @gen_cluster(client=True)
@@ -184,7 +181,7 @@ async def test_task_stream_clear_interval(c, s, a, b):
 
     await wait(c.map(inc, range(10)))
     ts.update()
-    await asyncio.sleep(0.010)
+    await asyncio.sleep(0.01)
     await wait(c.map(dec, range(10)))
     ts.update()
 
@@ -192,7 +189,7 @@ async def test_task_stream_clear_interval(c, s, a, b):
     assert ts.source.data["name"].count("inc") == 10
     assert ts.source.data["name"].count("dec") == 10
 
-    await asyncio.sleep(0.300)
+    await asyncio.sleep(0.3)
     await wait(c.map(inc, range(10, 20)))
     ts.update()
 
@@ -848,7 +845,6 @@ async def test_aggregate_action(c, s, a, b):
     assert ("compute") in mbk.action_source.data["names"]
 
 
-@pytest.mark.flaky(reruns=10, reruns_delay=5, condition=MACOS)
 @gen_cluster(client=True, scheduler_kwargs={"dashboard": True})
 async def test_compute_per_key(c, s, a, b):
     mbk = ComputePerKey(s)
diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index e759d8b1216..0cf98ea3cd8 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -136,8 +136,8 @@ def __init__(
 
         if threads_per_worker == 0:
             warnings.warn(
-                "Setting `threads_per_worker` to 0 is discouraged. "
-                "Please set to None or to a specific int to get best behavior."
+                "Setting `threads_per_worker` to 0 has been deprecated. "
+                "Please set to None or to a specific int."
             )
             threads_per_worker = None
 
diff --git a/distributed/deploy/old_ssh.py b/distributed/deploy/old_ssh.py
index dcf9fd83eb2..9d07f1f9bba 100644
--- a/distributed/deploy/old_ssh.py
+++ b/distributed/deploy/old_ssh.py
@@ -51,8 +51,8 @@ def async_ssh(cmd_dict):
                 port=cmd_dict["ssh_port"],
                 key_filename=cmd_dict["ssh_private_key"],
                 compress=True,
-                timeout=20,
-                banner_timeout=20,
+                timeout=30,
+                banner_timeout=30,
             )  # Helps prevent timeouts when many concurrent ssh connections are opened.
             # Connection successful, break out of while loop
             break
diff --git a/distributed/deploy/tests/test_adaptive.py b/distributed/deploy/tests/test_adaptive.py
index 30496a7d233..4ad27d85df1 100644
--- a/distributed/deploy/tests/test_adaptive.py
+++ b/distributed/deploy/tests/test_adaptive.py
@@ -8,6 +8,7 @@
 import dask
 
 from distributed import Adaptive, Client, LocalCluster, SpecCluster, Worker, wait
+from distributed.compatibility import WINDOWS
 from distributed.metrics import time
 from distributed.utils_test import async_wait_for, clean, gen_test, slowinc
 
@@ -40,8 +41,8 @@ def test_adaptive_local_cluster(loop):
             assert not c.nthreads()
 
 
-@pytest.mark.asyncio
-async def test_adaptive_local_cluster_multi_workers(cleanup):
+@gen_test()
+async def test_adaptive_local_cluster_multi_workers():
     async with LocalCluster(
         n_workers=0,
         scheduler_port=0,
@@ -56,19 +57,14 @@ async def test_adaptive_local_cluster_multi_workers(cleanup):
         async with Client(cluster, asynchronous=True) as c:
             futures = c.map(slowinc, range(100), delay=0.01)
 
-            start = time()
             while not cluster.scheduler.workers:
                 await asyncio.sleep(0.01)
-                assert time() < start + 15, adapt.log
 
             await c.gather(futures)
             del futures
 
-            start = time()
-            # while cluster.workers:
             while cluster.scheduler.workers:
                 await asyncio.sleep(0.01)
-                assert time() < start + 15, adapt.log
 
             # no workers for a while
             for i in range(10):
@@ -242,7 +238,7 @@ async def test_adapt_quickly():
         await cluster.close()
 
 
-@gen_test(timeout=None)
+@gen_test()
 async def test_adapt_down():
     """Ensure that redefining adapt with a lower maximum removes workers"""
     async with LocalCluster(
@@ -302,14 +298,15 @@ def test_basic_no_loop(loop):
             loop.add_callback(loop.stop)
 
 
-@pytest.mark.asyncio
+@pytest.mark.flaky(condition=not WINDOWS, reruns=10, reruns_delay=5)
+@pytest.mark.xfail(condition=WINDOWS, reason="extremely flaky")
+@gen_test()
 async def test_target_duration():
-    """Ensure that redefining adapt with a lower maximum removes workers"""
     with dask.config.set(
         {"distributed.scheduler.default-task-durations": {"slowinc": 1}}
     ):
         async with LocalCluster(
-            0,
+            n_workers=0,
             asynchronous=True,
             processes=False,
             scheduler_port=0,
@@ -318,16 +315,12 @@ async def test_target_duration():
         ) as cluster:
             adapt = cluster.adapt(interval="20ms", minimum=2, target_duration="5s")
             async with Client(cluster, asynchronous=True) as client:
-                while len(cluster.scheduler.workers) < 2:
-                    await asyncio.sleep(0.01)
-
+                await client.wait_for_workers(2)
                 futures = client.map(slowinc, range(100), delay=0.3)
+                await wait(futures)
 
-                while len(adapt.log) < 2:
-                    await asyncio.sleep(0.01)
-
-                assert adapt.log[0][1] == {"status": "up", "n": 2}
-                assert adapt.log[1][1] == {"status": "up", "n": 20}
+            assert adapt.log[0][1] == {"status": "up", "n": 2}
+            assert adapt.log[1][1] == {"status": "up", "n": 20}
 
 
 @pytest.mark.asyncio
diff --git a/distributed/deploy/tests/test_adaptive_core.py b/distributed/deploy/tests/test_adaptive_core.py
index 2750121e445..7de0ffe87cc 100644
--- a/distributed/deploy/tests/test_adaptive_core.py
+++ b/distributed/deploy/tests/test_adaptive_core.py
@@ -85,10 +85,10 @@ async def test_interval():
             assert time() < start + 2
 
     adapt.stop()
-    await asyncio.sleep(0.050)
+    await asyncio.sleep(0.05)
 
     adapt._target = 10
-    await asyncio.sleep(0.020)
+    await asyncio.sleep(0.02)
     assert len(adapt.plan) == 1  # last value from before, unchanged
 
 
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index 13a3a16d928..bddd6b6af0b 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -17,6 +17,7 @@
 from dask.system import CPU_COUNT
 
 from distributed import Client, Nanny, Worker, get_client
+from distributed.compatibility import LINUX
 from distributed.core import Status
 from distributed.deploy.local import LocalCluster
 from distributed.deploy.utils_test import ClusterTest
@@ -693,13 +694,12 @@ def test_adapt_then_manual(loop):
         processes=False,
         n_workers=8,
     ) as cluster:
-        sleep(0.1)
         cluster.adapt(minimum=0, maximum=4, interval="10ms")
 
         start = time()
         while cluster.scheduler.workers or cluster.workers:
-            sleep(0.1)
-            assert time() < start + 5
+            sleep(0.01)
+            assert time() < start + 30
 
         assert not cluster.workers
 
@@ -715,8 +715,8 @@ def test_adapt_then_manual(loop):
 
             start = time()
             while len(cluster.scheduler.workers) != 2:
-                sleep(0.1)
-                assert time() < start + 5
+                sleep(0.01)
+                assert time() < start + 30
 
 
 @pytest.mark.parametrize("temporary", [True, False])
@@ -843,9 +843,7 @@ def test_protocol_tcp(loop):
         assert cluster.scheduler.address.startswith("tcp://")
 
 
-@pytest.mark.skipif(
-    not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
-)
+@pytest.mark.skipif(not LINUX, reason="Need 127.0.0.2 to mean localhost")
 def test_protocol_ip(loop):
     with LocalCluster(
         host="tcp://127.0.0.2", loop=loop, n_workers=0, processes=False
@@ -990,7 +988,7 @@ async def test_repr(cleanup):
 @pytest.mark.asyncio
 async def test_threads_per_worker_set_to_0(cleanup):
     with pytest.warns(
-        Warning, match="Setting `threads_per_worker` to 0 is discouraged."
+        Warning, match="Setting `threads_per_worker` to 0 has been deprecated."
     ):
         async with LocalCluster(
             n_workers=2, processes=False, threads_per_worker=0, asynchronous=True
diff --git a/distributed/deploy/tests/test_spec_cluster.py b/distributed/deploy/tests/test_spec_cluster.py
index dff5d06831b..6b8ea8ee0e6 100644
--- a/distributed/deploy/tests/test_spec_cluster.py
+++ b/distributed/deploy/tests/test_spec_cluster.py
@@ -14,6 +14,7 @@
 from distributed.deploy.spec import ProcessInterface, close_clusters, run_spec
 from distributed.metrics import time
 from distributed.utils import is_valid_xml
+from distributed.utils_test import gen_test
 
 
 class MyWorker(Worker):
@@ -138,8 +139,8 @@ async def test_scale(cleanup):
 
 
 @pytest.mark.slow
-@pytest.mark.asyncio
-async def test_adaptive_killed_worker(cleanup):
+@gen_test()
+async def test_adaptive_killed_worker():
     with dask.config.set({"distributed.deploy.lost-worker-timeout": 0.1}):
 
         async with SpecCluster(
@@ -147,13 +148,10 @@ async def test_adaptive_killed_worker(cleanup):
             worker={"cls": Nanny, "options": {"nthreads": 1}},
             scheduler={"cls": Scheduler, "options": {"port": 0}},
         ) as cluster:
-
             async with Client(cluster, asynchronous=True) as client:
-
-                cluster.adapt(minimum=1, maximum=1)
-
                 # Scale up a cluster with 1 worker.
-                while len(cluster.workers) != 1:
+                cluster.adapt(minimum=1, maximum=1)
+                while not cluster.workers:
                     await asyncio.sleep(0.01)
 
                 future = client.submit(sleep, 0.1)
@@ -163,11 +161,11 @@ async def test_adaptive_killed_worker(cleanup):
                 await cluster.workers[worker_id].kill()
 
                 # Wait for the worker to re-spawn and finish sleeping.
-                await future.result(timeout=5)
+                await future
 
 
-@pytest.mark.asyncio
-async def test_unexpected_closed_worker(cleanup):
+@gen_test()
+async def test_unexpected_closed_worker():
     worker = {"cls": Worker, "options": {"nthreads": 1}}
     with dask.config.set({"distributed.deploy.lost-worker-timeout": "10ms"}):
         async with SpecCluster(
@@ -197,25 +195,20 @@ async def test_unexpected_closed_worker(cleanup):
             assert len(cluster.workers) == 2
 
 
-@pytest.mark.flaky(reruns=10, reruns_delay=5)
-@pytest.mark.slow
-@pytest.mark.asyncio
-async def test_restart(cleanup):
-    # Regression test for https://github.com/dask/distributed/issues/3062
+@gen_test(timeout=60)
+async def test_restart():
+    """Regression test for https://github.com/dask/distributed/issues/3062"""
     worker = {"cls": Nanny, "options": {"nthreads": 1}}
-    with dask.config.set({"distributed.deploy.lost-worker-timeout": "2s"}):
-        async with SpecCluster(
-            asynchronous=True, scheduler=scheduler, worker=worker
-        ) as cluster:
-            async with Client(cluster, asynchronous=True) as client:
-                cluster.scale(2)
-                await cluster
-                assert len(cluster.workers) == 2
-                await client.restart()
-                start = time()
-                while len(cluster.workers) < 2:
-                    await asyncio.sleep(0.5)
-                    assert time() < start + 60
+    async with SpecCluster(
+        asynchronous=True, scheduler=scheduler, worker=worker
+    ) as cluster:
+        async with Client(cluster, asynchronous=True) as client:
+            cluster.scale(2)
+            await cluster
+            assert len(cluster.workers) == 2
+            await client.restart()
+            while len(cluster.workers) < 2:
+                await asyncio.sleep(0.01)
 
 
 @pytest.mark.skipif(WINDOWS, reason="HTTP Server doesn't close out")
diff --git a/distributed/diagnostics/tests/test_progress.py b/distributed/diagnostics/tests/test_progress.py
index 81821734e06..f671c2e0681 100644
--- a/distributed/diagnostics/tests/test_progress.py
+++ b/distributed/diagnostics/tests/test_progress.py
@@ -4,6 +4,7 @@
 
 from distributed import Nanny
 from distributed.client import wait
+from distributed.compatibility import LINUX
 from distributed.diagnostics.progress import (
     AllProgress,
     GroupProgress,
@@ -95,8 +96,9 @@ def check_bar_completed(capsys, width=40):
     assert percent == "100% Completed"
 
 
+@pytest.mark.flaky(condition=not COMPILED and LINUX, reruns=10, reruns_delay=5)
 @pytest.mark.xfail(COMPILED, reason="Fails with cythonized scheduler")
-@gen_cluster(client=True, Worker=Nanny, timeout=None)
+@gen_cluster(client=True, Worker=Nanny)
 async def test_AllProgress(c, s, a, b):
     x, y, z = c.map(inc, [1, 2, 3])
     xx, yy, zz = c.map(dec, [x, y, z])
@@ -178,8 +180,9 @@ def f(x):
     assert all(set(d) == {"div"} for d in p.state.values())
 
 
+@pytest.mark.flaky(condition=LINUX, reruns=10, reruns_delay=5)
 @gen_cluster(client=True, Worker=Nanny)
-async def test_AllProgress_lost_key(c, s, a, b, timeout=None):
+async def test_AllProgress_lost_key(c, s, a, b):
     p = AllProgress(s)
     futures = c.map(inc, range(5))
     await wait(futures)
@@ -188,10 +191,8 @@ async def test_AllProgress_lost_key(c, s, a, b, timeout=None):
     await a.close()
     await b.close()
 
-    start = time()
     while len(p.state["memory"]["inc"]) > 0:
-        await asyncio.sleep(0.1)
-        assert time() < start + 5
+        await asyncio.sleep(0.01)
 
 
 @gen_cluster(client=True)
diff --git a/distributed/diagnostics/tests/test_scheduler_plugin.py b/distributed/diagnostics/tests/test_scheduler_plugin.py
index d1298307cba..111ad5e24d9 100644
--- a/distributed/diagnostics/tests/test_scheduler_plugin.py
+++ b/distributed/diagnostics/tests/test_scheduler_plugin.py
@@ -33,7 +33,7 @@ def transition(self, key, start, finish, *args, **kwargs):
     assert counter not in s.plugins
 
 
-@gen_cluster(nthreads=[], client=False)
+@gen_cluster(nthreads=[])
 async def test_add_remove_worker(s):
     events = []
 
@@ -71,7 +71,7 @@ def remove_worker(self, worker, scheduler):
     assert events == []
 
 
-@gen_cluster(nthreads=[], client=False)
+@gen_cluster(nthreads=[])
 async def test_async_add_remove_worker(s):
     events = []
 
diff --git a/distributed/diagnostics/tests/test_widgets.py b/distributed/diagnostics/tests/test_widgets.py
index f78dfc8af5d..e47c1bd5bc9 100644
--- a/distributed/diagnostics/tests/test_widgets.py
+++ b/distributed/diagnostics/tests/test_widgets.py
@@ -5,8 +5,6 @@
 from ipykernel.comm import Comm
 from ipywidgets import Widget
 
-from distributed.compatibility import WINDOWS
-
 #################
 # Utility stuff #
 #################
@@ -145,7 +143,6 @@ async def test_multi_progressbar_widget(c, s, a, b):
     assert sorted(capacities, reverse=True) == capacities
 
 
-@pytest.mark.flaky(reruns=10, reruns_delay=5, condition=WINDOWS)
 @gen_cluster()
 async def test_multi_progressbar_widget_after_close(s, a, b):
     s.update_graph(
diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index 870198bc306..28ca8cf7bf5 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -176,7 +176,7 @@ distributed:
       threads: 0    # Threads to use. 0 for single-threaded, -1 to infer from cpu count.
 
     timeouts:
-      connect: 10s          # time before connecting fails
+      connect: 30s          # time before connecting fails
       tcp: 30s              # time before calling an unresponsive connection dead
 
     require-encryption: null # Whether to require encryption on non-local comms
diff --git a/distributed/nanny.py b/distributed/nanny.py
index d891a65fd7e..a714489da0d 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -384,7 +384,7 @@ async def instantiate(self, comm=None) -> Status:
                 raise
         return result
 
-    async def restart(self, comm=None, timeout=2, executor_wait=True):
+    async def restart(self, comm=None, timeout=30, executor_wait=True):
         async def _():
             if self.process is not None:
                 await self.kill()
@@ -393,7 +393,9 @@ async def _():
         try:
             await asyncio.wait_for(_(), timeout)
         except TimeoutError:
-            logger.error("Restart timed out, returning before finished")
+            logger.error(
+                f"Restart timed out after {timeout}s; returning before finished"
+            )
             return "timed out"
         else:
             return "OK"
diff --git a/distributed/protocol/tests/test_pickle.py b/distributed/protocol/tests/test_pickle.py
index 25989bfcde4..a25d499ea0f 100644
--- a/distributed/protocol/tests/test_pickle.py
+++ b/distributed/protocol/tests/test_pickle.py
@@ -128,7 +128,6 @@ def test_pickle_numpy():
         assert (deserialize(h, f) == x).all()
 
 
-@pytest.mark.flaky(reruns=10, reruns_delay=5, condition=sys.version_info[:2] == (3, 8))
 def test_pickle_functions():
     def make_closure():
         value = 1
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index d9e2bec5e2c..29ca0fb7759 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -3440,19 +3440,18 @@ def __init__(
 
         http_server_modules = dask.config.get("distributed.scheduler.http.routes")
         show_dashboard = dashboard or (dashboard is None and dashboard_address)
-        missing_bokeh = False
         # install vanilla route if show_dashboard but bokeh is not installed
         if show_dashboard:
             try:
                 import distributed.dashboard.scheduler
             except ImportError:
-                missing_bokeh = True
+                show_dashboard = False
                 http_server_modules.append("distributed.http.scheduler.missing_bokeh")
         routes = get_handlers(
             server=self, modules=http_server_modules, prefix=http_prefix
         )
         self.start_http_server(routes, dashboard_address, default_port=8787)
-        if show_dashboard and not missing_bokeh:
+        if show_dashboard:
             distributed.dashboard.scheduler.connect(
                 self.http_application, self.http_server, self, prefix=http_prefix
             )
@@ -5469,8 +5468,8 @@ def clear_task_state(self):
         for collection in self._task_state_collections:
             collection.clear()
 
-    async def restart(self, client=None, timeout=3):
-        """Restart all workers.  Reset local state."""
+    async def restart(self, client=None, timeout=30):
+        """Restart all workers. Reset local state."""
         parent: SchedulerState = cast(SchedulerState, self)
         with log_errors():
 
diff --git a/distributed/tests/test_actor.py b/distributed/tests/test_actor.py
index 851ee7e8b2a..92d55425f55 100644
--- a/distributed/tests/test_actor.py
+++ b/distributed/tests/test_actor.py
@@ -485,10 +485,8 @@ def check(counter, blanks):
     result = await c.compute(final, actors=counter)
     assert result == 0 + 1 + 2 + 3 + 4
 
-    start = time()
     while a.data or b.data:
         await asyncio.sleep(0.01)
-        assert time() < start + 30
 
 
 def test_compute_sync(client):
diff --git a/distributed/tests/test_as_completed.py b/distributed/tests/test_as_completed.py
index 370563e8359..0e0f4d254fc 100644
--- a/distributed/tests/test_as_completed.py
+++ b/distributed/tests/test_as_completed.py
@@ -14,7 +14,7 @@
 
 
 @gen_cluster(client=True)
-async def test__as_completed(c, s, a, b):
+async def test_as_completed_async(c, s, a, b):
     x = c.submit(inc, 1)
     y = c.submit(inc, 1)
     z = c.submit(inc, 2)
@@ -29,7 +29,7 @@ async def test__as_completed(c, s, a, b):
     assert result in [x, y, z]
 
 
-def test_as_completed(client):
+def test_as_completed_sync(client):
     x = client.submit(inc, 1)
     y = client.submit(inc, 2)
     z = client.submit(inc, 1)
@@ -201,7 +201,6 @@ async def test_as_completed_with_results_async(c, s, a, b):
     assert str(exc.value) == "hello!"
 
 
-@pytest.mark.flaky(reruns=10, reruns_delay=5)
 def test_as_completed_with_results_no_raise(client):
     x = client.submit(throws, 1)
     y = client.submit(inc, 5)
@@ -260,7 +259,7 @@ async def test_as_completed_with_results_no_raise_async(c, s, a, b):
     assert dd[z][0] == 2
 
 
-@gen_cluster(client=True, timeout=None)
+@gen_cluster(client=True)
 async def test_clear(c, s, a, b):
     futures = c.map(inc, range(3))
     ac = as_completed(futures)
diff --git a/distributed/tests/test_asyncprocess.py b/distributed/tests/test_asyncprocess.py
index b98b4d867bc..211de45f23c 100644
--- a/distributed/tests/test_asyncprocess.py
+++ b/distributed/tests/test_asyncprocess.py
@@ -8,6 +8,7 @@
 from datetime import timedelta
 from time import sleep
 
+import psutil
 import pytest
 from tornado import gen
 from tornado.locks import Event
@@ -280,13 +281,9 @@ async def test_child_main_thread():
     q._writer.close()
 
 
-@pytest.mark.skipif(
-    sys.platform.startswith("win"), reason="num_fds not supported on windows"
-)
+@pytest.mark.skipif(WINDOWS, reason="num_fds not supported on windows")
 @gen_test()
 async def test_num_fds():
-    psutil = pytest.importorskip("psutil")
-
     # Warm up
     proc = AsyncProcess(target=exit_now)
     proc.daemon = True
@@ -303,11 +300,8 @@ async def test_num_fds():
     assert not proc.is_alive()
     assert proc.exitcode == 0
 
-    start = time()
     while p.num_fds() > before:
-        await asyncio.sleep(0.1)
-        print("fds:", before, p.num_fds())
-        assert time() < start + 10
+        await asyncio.sleep(0.01)
 
 
 @gen_test()
@@ -407,7 +401,7 @@ def test_asyncprocess_child_teardown_on_parent_exit():
         try:
             readable = children_alive.poll(short_timeout)
         except BrokenPipeError:
-            assert sys.platform.startswith("win"), "should only raise on windows"
+            assert WINDOWS, "should only raise on windows"
             # Broken pipe implies closed, which is readable.
             readable = True
 
@@ -422,7 +416,7 @@ def test_asyncprocess_child_teardown_on_parent_exit():
         except EOFError:
             pass  # Test passes.
         except BrokenPipeError:
-            assert sys.platform.startswith("win"), "should only raise on windows"
+            assert WINDOWS, "should only raise on windows"
             # Test passes.
         else:
             # Oops, children_alive read something. It should be closed. If
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 6a45e28bc7a..e6ef29b4673 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -55,7 +55,7 @@
     wait,
 )
 from distributed.comm import CommClosedError
-from distributed.compatibility import MACOS, WINDOWS
+from distributed.compatibility import LINUX, WINDOWS
 from distributed.core import Status
 from distributed.metrics import time
 from distributed.objects import HasWhat, WhoHas
@@ -95,7 +95,7 @@
 )
 
 
-@gen_cluster(client=True, timeout=None)
+@gen_cluster(client=True)
 async def test_submit(c, s, a, b):
     x = c.submit(inc, 10)
     assert not x.done()
@@ -624,7 +624,7 @@ async def test_limit_concurrent_gathering(c, s, a, b):
     assert len(a.outgoing_transfer_log) + len(b.outgoing_transfer_log) < 100
 
 
-@gen_cluster(client=True, timeout=None)
+@gen_cluster(client=True)
 async def test_get(c, s, a, b):
     future = c.get({"x": (inc, 1)}, "x", sync=False)
     assert isinstance(future, Future)
@@ -717,7 +717,7 @@ async def test_wait_first_completed(c, s, a, b):
     assert y.status == "pending"
 
 
-@gen_cluster(client=True, timeout=2)
+@gen_cluster(client=True)
 async def test_wait_timeout(c, s, a, b):
     future = c.submit(sleep, 0.3)
     with pytest.raises(TimeoutError):
@@ -794,7 +794,7 @@ async def test_garbage_collection_with_scatter(c, s, a, b):
             await asyncio.sleep(0.1)
 
 
-@gen_cluster(timeout=1000, client=True)
+@gen_cluster(client=True)
 async def test_recompute_released_key(c, s, a, b):
     x = c.submit(inc, 100)
     result1 = await x
@@ -907,9 +907,7 @@ async def test_tokenize_on_futures(c, s, a, b):
     assert tok == tokenize(y)
 
 
-@pytest.mark.skipif(
-    not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
-)
+@pytest.mark.skipif(not LINUX, reason="Need 127.0.0.2 to mean localhost")
 @gen_cluster([("127.0.0.1", 1), ("127.0.0.2", 2)], client=True)
 async def test_restrictions_submit(c, s, a, b):
     x = c.submit(inc, 1, workers={a.ip})
@@ -936,9 +934,7 @@ async def test_restrictions_ip_port(c, s, a, b):
     assert y.key in b.data
 
 
-@pytest.mark.skipif(
-    not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
-)
+@pytest.mark.skipif(not LINUX, reason="Need 127.0.0.2 to mean localhost")
 @gen_cluster([("127.0.0.1", 1), ("127.0.0.2", 2)], client=True)
 async def test_restrictions_map(c, s, a, b):
     L = c.map(inc, range(5), workers={a.ip})
@@ -950,9 +946,7 @@ async def test_restrictions_map(c, s, a, b):
         assert s.host_restrictions[x.key] == {a.ip}
 
 
-@pytest.mark.skipif(
-    not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
-)
+@pytest.mark.skipif(not LINUX, reason="Need 127.0.0.2 to mean localhost")
 @gen_cluster([("127.0.0.1", 1), ("127.0.0.2", 2)], client=True)
 async def test_restrictions_get(c, s, a, b):
     dsk = {"x": 1, "y": (inc, "x"), "z": (inc, "y")}
@@ -991,7 +985,7 @@ async def dont_test_bad_restrictions_raise_exception(c, s, a, b):
         assert z.key in str(e)
 
 
-@gen_cluster(client=True, timeout=None)
+@gen_cluster(client=True)
 async def test_remove_worker(c, s, a, b):
     L = c.map(inc, range(20))
     await wait(L)
@@ -1280,9 +1274,7 @@ async def test_get_nbytes(c, s, a, b):
     assert s.get_nbytes(summary=False) == {x.key: sizeof(1), y.key: sizeof(2)}
 
 
-@pytest.mark.skipif(
-    not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
-)
+@pytest.mark.skipif(not LINUX, reason="Need 127.0.0.2 to mean localhost")
 @gen_cluster([("127.0.0.1", 1), ("127.0.0.2", 2)], client=True)
 async def test_nbytes_determines_worker(c, s, a, b):
     x = c.submit(identity, 1, workers=[a.ip])
@@ -1455,7 +1447,7 @@ async def test_scatter_direct_empty(c, s):
         await c.scatter(123, direct=True, timeout=0.1)
 
 
-@gen_cluster(client=True, timeout=None, nthreads=[("127.0.0.1", 1)] * 5)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 5)
 async def test_scatter_direct_spread_evenly(c, s, *workers):
     futures = []
     for i in range(10):
@@ -1799,7 +1791,7 @@ async def test_remote_scatter_gather(c, s, a, b):
     assert (xx, yy, zz) == (1, 2, 3)
 
 
-@gen_cluster(timeout=1000, client=True)
+@gen_cluster(client=True)
 async def test_remote_submit_on_Future(c, s, a, b):
     x = c.submit(lambda x: x + 1, 1)
     y = c.submit(lambda x: x + 1, x)
@@ -1836,9 +1828,7 @@ async def test_client_with_scheduler(c, s, a, b):
     assert result == 12
 
 
-@pytest.mark.skipif(
-    not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
-)
+@pytest.mark.skipif(not LINUX, reason="Need 127.0.0.2 to mean localhost")
 @gen_cluster([("127.0.0.1", 1), ("127.0.0.2", 2)], client=True)
 async def test_allow_restrictions(c, s, a, b):
     aws = s.workers[a.address]
@@ -1971,7 +1961,7 @@ async def test_badly_serialized_input(c, s, a, b):
     assert "hello!" in str(info.value)
 
 
-@pytest.mark.skipif("True", reason="")
+@pytest.mark.skip
 async def test_badly_serialized_input_stderr(capsys, c):
     o = BadlySerializedObject()
     future = c.submit(inc, o)
@@ -2853,7 +2843,6 @@ async def test_persist_get(c, s, a, b):
 
 @pytest.mark.skipif(WINDOWS, reason="num_fds not supported on windows")
 def test_client_num_fds(loop):
-    psutil = pytest.importorskip("psutil")
     with cluster() as (s, [a, b]):
         proc = psutil.Process()
         with Client(s["address"], loop=loop) as c:  # first client to start loop
@@ -2864,7 +2853,7 @@ def test_client_num_fds(loop):
             start = time()
             while proc.num_fds() > before:
                 sleep(0.01)
-                assert time() < start + 4
+                assert time() < start + 10, (before, proc.num_fds())
 
 
 @gen_cluster()
@@ -3035,9 +3024,7 @@ async def test_receive_lost_key(c, s, a, b):
         await asyncio.sleep(0.01)
 
 
-@pytest.mark.skipif(
-    not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
-)
+@pytest.mark.skipif(not LINUX, reason="Need 127.0.0.2 to mean localhost")
 @gen_cluster([("127.0.0.1", 1), ("127.0.0.2", 2)], client=True)
 async def test_unrunnable_task_runs(c, s, a, b):
     x = c.submit(inc, 1, workers=[a.ip])
@@ -3073,9 +3060,7 @@ async def test_add_worker_after_tasks(c, s):
     await n.close()
 
 
-@pytest.mark.skipif(
-    not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
-)
+@pytest.mark.skipif(not LINUX, reason="Need 127.0.0.2 to mean localhost")
 @gen_cluster([("127.0.0.1", 1), ("127.0.0.2", 2)], client=True)
 async def test_workers_register_indirect_data(c, s, a, b):
     [x] = await c.scatter([1], workers=a.address)
@@ -3205,13 +3190,10 @@ async def test_client_replicate(c, s, *workers):
     assert len(s.tasks[y.key].who_has) == 10
 
 
-@pytest.mark.skipif(
-    not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
-)
+@pytest.mark.skipif(not LINUX, reason="Need 127.0.0.2 to mean localhost")
 @gen_cluster(
     client=True,
     nthreads=[("127.0.0.1", 1), ("127.0.0.2", 1), ("127.0.0.2", 1)],
-    timeout=None,
 )
 async def test_client_replicate_host(client, s, a, b, c):
     aws = s.workers[a.address]
@@ -3565,7 +3547,7 @@ def test_get_returns_early(c):
 
 
 @pytest.mark.slow
-@gen_cluster(Worker=Nanny, client=True)
+@gen_cluster(Worker=Nanny, client=True, timeout=60)
 async def test_Client_clears_references_after_restart(c, s, a, b):
     x = c.submit(inc, 1)
     assert x.key in c.refcount
@@ -3789,7 +3771,6 @@ async def test_reconnect_timeout(c, s):
 @pytest.mark.skipif(WINDOWS, reason="num_fds not supported on windows")
 @pytest.mark.parametrize("worker,count,repeat", [(Worker, 100, 5), (Nanny, 10, 20)])
 def test_open_close_many_workers(loop, worker, count, repeat):
-    psutil = pytest.importorskip("psutil")
     proc = psutil.Process()
 
     with cluster(nworkers=0, active_rpc_timeout=2) as (s, _):
@@ -3857,7 +3838,7 @@ async def start_worker(sleep, duration, repeat=1):
                 raise ValueError("File descriptors did not clean up")
 
 
-@gen_cluster(client=False, timeout=None)
+@gen_cluster()
 async def test_idempotence(s, a, b):
     c = await Client(s.address, asynchronous=True)
     f = await Client(s.address, asynchronous=True)
@@ -4070,7 +4051,7 @@ async def test_scatter_compute_store_lose_processing(c, s, a, b):
     assert z.status == "cancelled"
 
 
-@gen_cluster(client=False)
+@gen_cluster()
 async def test_serialize_future(s, a, b):
     c1 = await Client(s.address, asynchronous=True)
     c2 = await Client(s.address, asynchronous=True)
@@ -4091,7 +4072,7 @@ async def test_serialize_future(s, a, b):
     await c2.close()
 
 
-@gen_cluster(client=False)
+@gen_cluster()
 async def test_temp_default_client(s, a, b):
     c1 = await Client(s.address, asynchronous=True)
     c2 = await Client(s.address, asynchronous=True)
@@ -4172,7 +4153,7 @@ def run2():
     t2.join()
 
 
-@gen_cluster(client=False)
+@gen_cluster()
 async def test_as_current_is_task_local(s, a, b):
     l1 = asyncio.Lock()
     l2 = asyncio.Lock()
@@ -4557,7 +4538,7 @@ def assert_no_data_loss(scheduler):
                 assert not (k == key and v == "waiting")
 
 
-@gen_cluster(client=True, timeout=None)
+@gen_cluster(client=True)
 async def test_interleave_computations(c, s, a, b):
     import distributed
 
@@ -4592,7 +4573,7 @@ async def test_interleave_computations(c, s, a, b):
 
 
 @pytest.mark.skip(reason="Now prefer first-in-first-out")
-@gen_cluster(client=True, timeout=None)
+@gen_cluster(client=True)
 async def test_interleave_computations_map(c, s, a, b):
     xs = c.map(slowinc, range(30), delay=0.02)
     ys = c.map(slowdec, xs, delay=0.02)
@@ -4621,7 +4602,6 @@ async def test_scatter_dict_workers(c, s, a, b):
     assert "a" in a.data or "a" in b.data
 
 
-@pytest.mark.flaky(reruns=10, reruns_delay=5, condition=MACOS)
 @pytest.mark.slow
 @gen_test()
 async def test_client_timeout():
@@ -4629,18 +4609,17 @@ async def test_client_timeout():
 
     s = Scheduler(loop=c.loop, port=57484)
     await asyncio.sleep(4)
+
     try:
         await s
     except OSError:  # port in use
         await c.close()
         return
 
-    start = time()
-    await c
     try:
-        assert time() < start + 2
-    finally:
+        await c
         await c.close()
+    finally:
         await s.close()
 
 
@@ -5167,7 +5146,7 @@ def f(x):
     assert result == sum(range(10))
 
 
-@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 1, timeout=100)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 1)
 async def test_secede_simple(c, s, a):
     def f():
         client = get_client()
@@ -5178,7 +5157,6 @@ def f():
     assert result == 2
 
 
-@pytest.mark.flaky(reruns=10, reruns_delay=5)
 @pytest.mark.slow
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 2, timeout=60)
 async def test_secede_balances(c, s, a, b):
@@ -5276,18 +5254,15 @@ def _dynamic_workload(x, delay=0.01):
     return total.result()
 
 
-def _test_dynamic_workloads_sync(c, delay):
-    future = c.submit(_dynamic_workload, 0, delay=delay)
-    assert future.result(timeout=40) == 52
-
-
 def test_dynamic_workloads_sync(c):
-    _test_dynamic_workloads_sync(c, delay=0.02)
+    future = c.submit(_dynamic_workload, 0, delay=0.02)
+    assert future.result(timeout=20) == 52
 
 
 @pytest.mark.slow
 def test_dynamic_workloads_sync_random(c):
-    _test_dynamic_workloads_sync(c, delay="random")
+    future = c.submit(_dynamic_workload, 0, delay="random")
+    assert future.result(timeout=20) == 52
 
 
 @pytest.mark.xfail(COMPILED, reason="Fails with cythonized scheduler")
@@ -5424,6 +5399,7 @@ async def test_call_stack_collections_all(c, s, a, b):
     assert result
 
 
+@pytest.mark.flaky(condition=WINDOWS, reruns=10, reruns_delay=5)
 @gen_cluster(client=True, worker_kwargs={"profile_cycle_interval": "100ms"})
 async def test_profile(c, s, a, b):
     futures = c.map(slowinc, range(10), delay=0.05, workers=a.address)
@@ -5843,9 +5819,15 @@ def bad_fn(x):
     assert y.status == "error"  # not cancelled
 
 
+@pytest.mark.xfail(reason="GH#5409 Dask-Default-Threads are frequently detected")
 def test_no_threads_lingering():
+    if threading.active_count() < 40:
+        return
     active = dict(threading._active)
-    assert threading.active_count() < 40, list(active.values())
+    print(f"==== Found {len(active)} active threads: ====")
+    for t in active.values():
+        print(t)
+    assert False
 
 
 @gen_cluster()
@@ -6071,18 +6053,18 @@ async def test_file_descriptors_dont_leak(Worker):
     df = dask.datasets.timeseries(freq="10s", dtypes={"x": int, "y": float})
 
     proc = psutil.Process()
-    start = proc.num_fds()
+    before = proc.num_fds()
     async with Scheduler(port=0, dashboard_address=":0") as s:
-        async with Worker(s.address, nthreads=2) as a, Worker(
-            s.address, nthreads=2
-        ) as b:
-            async with Client(s.address, asynchronous=True) as c:
-                await df.sum().persist()
+        async with Worker(s.address), Worker(s.address), Client(
+            s.address, asynchronous=True
+        ):
+            assert proc.num_fds() > before
+            await df.sum().persist()
 
-    begin = time()
-    while proc.num_fds() > begin:
+    start = time()
+    while proc.num_fds() > before:
         await asyncio.sleep(0.01)
-        assert time() < begin + 5, (start, proc.num_fds())
+        assert time() < start + 10, (before, proc.num_fds())
 
 
 @pytest.mark.asyncio
diff --git a/distributed/tests/test_client_executor.py b/distributed/tests/test_client_executor.py
index d53a0375f68..7b20069788c 100644
--- a/distributed/tests/test_client_executor.py
+++ b/distributed/tests/test_client_executor.py
@@ -13,6 +13,7 @@
 from tlz import take
 
 from distributed import Client
+from distributed.compatibility import MACOS
 from distributed.utils import CancelledError
 from distributed.utils_test import (
     cluster,
@@ -93,13 +94,12 @@ def test_wait(client):
         assert "hello" in str(errors[0])
 
 
-@pytest.mark.flaky(reruns=10, reruns_delay=5)
 def test_cancellation(client):
     with client.get_executor(pure=False) as e:
         fut = e.submit(time.sleep, 2.0)
         start = time.time()
         while number_of_processing_tasks(client) == 0:
-            assert time.time() < start + 1
+            assert time.time() < start + 10
             time.sleep(0.01)
         assert not fut.done()
 
@@ -107,7 +107,7 @@ def test_cancellation(client):
         assert fut.cancelled()
         start = time.time()
         while number_of_processing_tasks(client) != 0:
-            assert time.time() < start + 1
+            assert time.time() < start + 10
             time.sleep(0.01)
 
         with pytest.raises(CancelledError):
@@ -118,7 +118,7 @@ def test_cancellation(client):
         N = 10
         fs = [e.submit(slowinc, i, delay=0.02) for i in range(N)]
         fs[3].cancel()
-        res = wait(fs, return_when=FIRST_COMPLETED)
+        res = wait(fs, return_when=FIRST_COMPLETED, timeout=30)
         assert len(res.not_done) > 0
         assert len(res.done) >= 1
 
@@ -132,10 +132,11 @@ def test_cancellation(client):
         fs[3].cancel()
         fs[8].cancel()
 
-        n_cancelled = sum(f.cancelled() for f in as_completed(fs))
+        n_cancelled = sum(f.cancelled() for f in as_completed(fs, timeout=30))
         assert n_cancelled == 2
 
 
+@pytest.mark.flaky(condition=MACOS, reruns=10, reruns_delay=5)
 def test_map(client):
     with client.get_executor() as e:
         N = 10
diff --git a/distributed/tests/test_collections.py b/distributed/tests/test_collections.py
index 00c9baf928d..373160d5f7c 100644
--- a/distributed/tests/test_collections.py
+++ b/distributed/tests/test_collections.py
@@ -40,7 +40,7 @@ def assert_equal(a, b):
         assert a == b
 
 
-@gen_cluster(timeout=240, client=True)
+@gen_cluster(client=True)
 async def test_dataframes(c, s, a, b):
     df = pd.DataFrame(
         {"x": np.random.random(1000), "y": np.random.random(1000)},
diff --git a/distributed/tests/test_diskutils.py b/distributed/tests/test_diskutils.py
index a62fe1ca4e4..077a37d4fba 100644
--- a/distributed/tests/test_diskutils.py
+++ b/distributed/tests/test_diskutils.py
@@ -12,7 +12,7 @@
 
 import dask
 
-from distributed.compatibility import MACOS, WINDOWS
+from distributed.compatibility import MACOS
 from distributed.diskutils import WorkSpace
 from distributed.metrics import time
 from distributed.utils import mp_context
@@ -273,16 +273,14 @@ def _test_workspace_concurrency(tmpdir, timeout, max_procs):
     return n_created, n_purged
 
 
-@pytest.mark.flaky(reruns=10, reruns_delay=5, condition=MACOS)
 @pytest.mark.slow
+@pytest.mark.xfail(condition=MACOS, reason="extremely flaky")
 def test_workspace_concurrency(tmpdir):
-    if WINDOWS:
-        raise pytest.xfail.Exception("TODO: unknown failure on windows")
     _test_workspace_concurrency(tmpdir, 5.0, 6)
 
 
-@pytest.mark.flaky(reruns=10, reruns_delay=5, condition=MACOS)
 @pytest.mark.slow
+@pytest.mark.xfail(condition=MACOS, reason="extremely flaky")
 def test_workspace_concurrency_intense(tmpdir):
     n_created, n_purged = _test_workspace_concurrency(tmpdir, 8.0, 16)
     assert n_created >= 100
diff --git a/distributed/tests/test_failed_workers.py b/distributed/tests/test_failed_workers.py
index da23d0c48ed..2b3e229c38d 100644
--- a/distributed/tests/test_failed_workers.py
+++ b/distributed/tests/test_failed_workers.py
@@ -75,12 +75,7 @@ def test_gather_after_failed_worker(loop):
             assert result == list(map(inc, range(10)))
 
 
-@gen_cluster(
-    client=True,
-    Worker=Nanny,
-    nthreads=[("127.0.0.1", 1)] * 4,
-    config={"distributed.comm.timeouts.connect": "1s"},
-)
+@gen_cluster(client=True, Worker=Nanny, nthreads=[("127.0.0.1", 1)] * 4)
 async def test_gather_then_submit_after_failed_workers(c, s, w, x, y, z):
     L = c.map(inc, range(20))
     await wait(L)
@@ -88,7 +83,7 @@ async def test_gather_then_submit_after_failed_workers(c, s, w, x, y, z):
     w.process.process._process.terminate()
     total = c.submit(sum, L)
 
-    for i in range(3):
+    for _ in range(3):
         await wait(total)
         addr = first(s.tasks[total.key].who_has).address
         for worker in [x, y, z]:
@@ -101,7 +96,7 @@ async def test_gather_then_submit_after_failed_workers(c, s, w, x, y, z):
 
 
 @pytest.mark.xfail(COMPILED, reason="Fails with cythonized scheduler")
-@gen_cluster(Worker=Nanny, timeout=60, client=True)
+@gen_cluster(Worker=Nanny, client=True, timeout=60)
 async def test_failed_worker_without_warning(c, s, a, b):
     L = c.map(inc, range(10))
     await wait(L)
@@ -298,18 +293,20 @@ async def test_multiple_clients_restart(s, a, b):
     await c2.close()
 
 
-@pytest.mark.flaky(reruns=10, reruns_timeout=5, condition=MACOS)
 @gen_cluster(Worker=Nanny, timeout=60)
 async def test_restart_scheduler(s, a, b):
-    import gc
+    assert len(s.nthreads) == 2
+    pids = (a.pid, b.pid)
+    assert pids[0]
+    assert pids[1]
 
-    gc.collect()
-    addrs = (a.worker_address, b.worker_address)
     await s.restart()
-    assert len(s.nthreads) == 2
-    addrs2 = (a.worker_address, b.worker_address)
 
-    assert addrs != addrs2
+    assert len(s.nthreads) == 2
+    pids2 = (a.pid, b.pid)
+    assert pids2[0]
+    assert pids2[1]
+    assert pids != pids2
 
 
 @gen_cluster(Worker=Nanny, client=True, timeout=60)
@@ -325,6 +322,8 @@ async def test_forgotten_futures_dont_clean_up_new_futures(c, s, a, b):
     await y
 
 
+@pytest.mark.slow
+@pytest.mark.flaky(condition=MACOS, reruns=10, reruns_delay=5)
 @gen_cluster(client=True, timeout=60, active_rpc_timeout=10)
 async def test_broken_worker_during_computation(c, s, a, b):
     s.allowed_failures = 100
@@ -404,14 +403,13 @@ def __sizeof__(self) -> int:
         return parse_bytes(dask.config.get("distributed.comm.offload")) + 1
 
 
+@pytest.mark.flaky(reruns=10, reruns_delay=5)
 @gen_cluster(client=True)
 async def test_worker_who_has_clears_after_failed_connection(c, s, a, b):
     n = await Nanny(s.address, nthreads=2, loop=s.loop)
 
-    start = time()
     while len(s.nthreads) < 3:
         await asyncio.sleep(0.01)
-        assert time() < start + 5
 
     def slow_ser(x, delay):
         return SlowTransmitData(x, delay=delay)
@@ -497,7 +495,7 @@ async def test_restart_timeout_on_long_running_task(c, s, a):
     with captured_logger("distributed.scheduler") as sio:
         future = c.submit(sleep, 3600)
         await asyncio.sleep(0.1)
-        await c.restart(timeout=20)
+        await c.restart()
 
     text = sio.getvalue()
     assert "timeout" not in text.lower()
@@ -547,7 +545,7 @@ def __sizeof__(self) -> int:
         return parse_bytes(dask.config.get("distributed.comm.offload")) + 1
 
 
-@gen_cluster(client=True, timeout=None)
+@gen_cluster(client=True)
 async def test_handle_superfluous_data(c, s, a, b):
     """
     See https://github.com/dask/distributed/pull/4784#discussion_r649210094
diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
index 4ac628a599b..ca854754192 100644
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -4,9 +4,10 @@
 import multiprocessing as mp
 import os
 import random
-import sys
 from contextlib import suppress
+from time import sleep
 
+import psutil
 import pytest
 from tlz import first, valmap
 from tornado.ioloop import IOLoop
@@ -14,6 +15,7 @@
 import dask
 
 from distributed import Client, Nanny, Scheduler, Worker, rpc, wait, worker
+from distributed.compatibility import LINUX, WINDOWS
 from distributed.core import CommClosedError, Status
 from distributed.diagnostics import SchedulerPlugin
 from distributed.metrics import time
@@ -72,12 +74,11 @@ async def test_str(s, a, b):
 
 @gen_cluster(nthreads=[], client=True)
 async def test_nanny_process_failure(c, s):
-    n = await Nanny(s.address, nthreads=2, loop=s.loop)
+    n = await Nanny(s.address, nthreads=2)
     first_dir = n.worker_dir
 
     assert os.path.exists(first_dir)
 
-    original_address = n.worker_address
     ww = rpc(n.worker_address)
     await ww.update_data(data=valmap(dumps, {"x": 1, "y": 2}))
     pid = n.pid
@@ -85,23 +86,17 @@ async def test_nanny_process_failure(c, s):
     with suppress(CommClosedError):
         await c.run(os._exit, 0, workers=[n.worker_address])
 
-    start = time()
     while n.pid == pid:  # wait while process dies and comes back
         await asyncio.sleep(0.01)
-        assert time() - start < 5
 
-    start = time()
     await asyncio.sleep(1)
     while not n.is_alive():  # wait while process comes back
         await asyncio.sleep(0.01)
-        assert time() - start < 5
 
     # assert n.worker_address != original_address  # most likely
 
-    start = time()
     while n.worker_address not in s.nthreads or n.worker_dir is None:
         await asyncio.sleep(0.01)
-        assert time() - start < 5
 
     second_dir = n.worker_dir
 
@@ -115,7 +110,6 @@ async def test_nanny_process_failure(c, s):
 
 @gen_cluster(nthreads=[])
 async def test_run(s):
-    pytest.importorskip("psutil")
     n = await Nanny(s.address, nthreads=2, loop=s.loop)
 
     with rpc(n.address) as nn:
@@ -173,7 +167,7 @@ async def test_nanny_alt_worker_class(c, s, w1, w2):
 
 
 @pytest.mark.slow
-@gen_cluster(client=False, nthreads=[])
+@gen_cluster(nthreads=[])
 async def test_nanny_death_timeout(s):
     await s.close()
     w = Nanny(s.address, death_timeout=1)
@@ -198,12 +192,9 @@ async def check_func(func):
     await check_func(lambda a, b: np.random.randint(a, b))
 
 
-@pytest.mark.skipif(
-    sys.platform.startswith("win"), reason="num_fds not supported on windows"
-)
-@gen_cluster(client=False, nthreads=[])
+@pytest.mark.skipif(WINDOWS, reason="num_fds not supported on windows")
+@gen_cluster(nthreads=[])
 async def test_num_fds(s):
-    psutil = pytest.importorskip("psutil")
     proc = psutil.Process()
 
     # Warm up
@@ -219,16 +210,12 @@ async def test_num_fds(s):
         await asyncio.sleep(0.1)
         await w.close()
 
-    start = time()
     while proc.num_fds() > before:
         print("fds:", before, proc.num_fds())
         await asyncio.sleep(0.1)
-        assert time() < start + 10
 
 
-@pytest.mark.skipif(
-    not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
-)
+@pytest.mark.skipif(not LINUX, reason="Need 127.0.0.2 to mean localhost")
 @gen_cluster(client=True, nthreads=[])
 async def test_worker_uses_same_host_as_nanny(c, s):
     for host in ["tcp://0.0.0.0", "tcp://127.0.0.2"]:
@@ -273,29 +260,24 @@ async def test_nanny_timeout(c, s, a):
     nthreads=[("127.0.0.1", 1)],
     client=True,
     Worker=Nanny,
-    worker_kwargs={"memory_limit": 1e8},
-    timeout=20,
-    clean_kwargs={"threads": False},
+    worker_kwargs={"memory_limit": "400 MiB"},
 )
 async def test_nanny_terminate(c, s, a):
-    from time import sleep
-
     def leak():
         L = []
         while True:
-            L.append(b"0" * 5000000)
+            L.append(b"0" * 5_000_000)
             sleep(0.01)
 
-    proc = a.process.pid
+    before = a.process.pid
     with captured_logger(logging.getLogger("distributed.nanny")) as logger:
         future = c.submit(leak)
-        start = time()
-        while a.process.pid == proc:
-            await asyncio.sleep(0.1)
-            assert time() < start + 10
-        out = logger.getvalue()
-        assert "restart" in out.lower()
-        assert "memory" in out.lower()
+        while a.process.pid == before:
+            await asyncio.sleep(0.01)
+
+    out = logger.getvalue()
+    assert "restart" in out.lower()
+    assert "memory" in out.lower()
 
 
 @gen_cluster(
diff --git a/distributed/tests/test_publish.py b/distributed/tests/test_publish.py
index 898a2f27acb..6abed31daba 100644
--- a/distributed/tests/test_publish.py
+++ b/distributed/tests/test_publish.py
@@ -11,7 +11,7 @@
 from distributed.utils_test import gen_cluster, inc
 
 
-@gen_cluster(client=False)
+@gen_cluster()
 async def test_publish_simple(s, a, b):
     c = Client(s.address, asynchronous=True)
     f = Client(s.address, asynchronous=True)
@@ -37,7 +37,7 @@ async def test_publish_simple(s, a, b):
     await asyncio.gather(c.close(), f.close())
 
 
-@gen_cluster(client=False)
+@gen_cluster()
 async def test_publish_non_string_key(s, a, b):
     async with Client(s.address, asynchronous=True) as c:
         for name in [("a", "b"), 9.0, 8]:
@@ -52,7 +52,7 @@ async def test_publish_non_string_key(s, a, b):
             assert name in datasets
 
 
-@gen_cluster(client=False)
+@gen_cluster()
 async def test_publish_roundtrip(s, a, b):
     c = await Client(s.address, asynchronous=True)
     f = await Client(s.address, asynchronous=True)
@@ -147,7 +147,7 @@ def test_unpublish_multiple_datasets_sync(client):
     assert "y" in str(exc_info.value)
 
 
-@gen_cluster(client=False)
+@gen_cluster()
 async def test_publish_bag(s, a, b):
     db = pytest.importorskip("dask.bag")
     c = await Client(s.address, asynchronous=True)
diff --git a/distributed/tests/test_pubsub.py b/distributed/tests/test_pubsub.py
index 6f555c09fc6..4aefa6d7ee1 100644
--- a/distributed/tests/test_pubsub.py
+++ b/distributed/tests/test_pubsub.py
@@ -10,7 +10,7 @@
 from distributed.utils_test import gen_cluster
 
 
-@gen_cluster(client=True, timeout=None)
+@gen_cluster(client=True)
 async def test_speed(c, s, a, b):
     """
     This tests how quickly we can move messages back and forth
diff --git a/distributed/tests/test_queues.py b/distributed/tests/test_queues.py
index 9db84f941ab..a948a8229b3 100644
--- a/distributed/tests/test_queues.py
+++ b/distributed/tests/test_queues.py
@@ -110,7 +110,7 @@ def f(x):
 
 
 @pytest.mark.slow
-@gen_cluster(client=True, nthreads=[("127.0.0.1", 2)] * 5, Worker=Nanny, timeout=None)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 2)] * 5, Worker=Nanny)
 async def test_race(c, s, *workers):
     def f(i):
         with worker_client() as c:
diff --git a/distributed/tests/test_resources.py b/distributed/tests/test_resources.py
index f3422270bdd..6adb9d2dd08 100644
--- a/distributed/tests/test_resources.py
+++ b/distributed/tests/test_resources.py
@@ -9,7 +9,6 @@
 
 from distributed import Worker
 from distributed.client import wait
-from distributed.compatibility import WINDOWS
 from distributed.utils_test import gen_cluster, inc, slowadd, slowinc
 
 
@@ -378,9 +377,7 @@ async def test_full_collections(c, s, a, b):
                 reason="don't track resources through optimization"
             ),
         ),
-        pytest.param(
-            False, marks=pytest.mark.flaky(reruns=10, reruns_delay=5, condition=WINDOWS)
-        ),
+        False,
     ],
 )
 def test_collections_get(client, optimize_graph, s, a, b):
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 8640baf1745..4bab24aa003 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -12,6 +12,7 @@
 from unittest import mock
 
 import cloudpickle
+import psutil
 import pytest
 from tlz import concat, first, frequencies, merge, valmap
 
@@ -21,7 +22,7 @@
 
 from distributed import Client, Nanny, Worker, fire_and_forget, wait
 from distributed.comm import Comm
-from distributed.compatibility import MACOS, WINDOWS
+from distributed.compatibility import LINUX, WINDOWS
 from distributed.core import ConnectionPool, Status, connect, rpc
 from distributed.metrics import time
 from distributed.protocol.pickle import dumps
@@ -618,7 +619,7 @@ async def test_ready_remove_worker(s, a, b):
     assert all(len(w.processing) > w.nthreads for w in s.workers.values())
 
 
-@gen_cluster(client=True, Worker=Nanny)
+@gen_cluster(client=True, Worker=Nanny, timeout=60)
 async def test_restart(c, s, a, b):
     futures = c.map(inc, range(20))
     await wait(futures)
@@ -742,24 +743,17 @@ async def test_config_stealing(cleanup):
             assert "stealing" not in s.extensions
 
 
-@pytest.mark.skipif(
-    sys.platform.startswith("win"), reason="file descriptors not really a thing"
-)
+@pytest.mark.skipif(WINDOWS, reason="num_fds not supported on windows")
 @gen_cluster(nthreads=[])
 async def test_file_descriptors_dont_leak(s):
-    psutil = pytest.importorskip("psutil")
     proc = psutil.Process()
     before = proc.num_fds()
 
-    w = await Worker(s.address)
-    await w.close()
+    async with Worker(s.address):
+        assert proc.num_fds() > before
 
-    during = proc.num_fds()
-
-    start = time()
     while proc.num_fds() > before:
         await asyncio.sleep(0.01)
-        assert time() < start + 5
 
 
 @gen_cluster()
@@ -827,7 +821,7 @@ async def test_scheduler_sees_memory_limits(s):
     await w.close()
 
 
-@gen_cluster(client=True, timeout=1000)
+@gen_cluster(client=True)
 async def test_retire_workers(c, s, a, b):
     [x] = await c.scatter([1], workers=a.address)
     [y] = await c.scatter([list(range(1000))], workers=b.address)
@@ -929,13 +923,10 @@ async def test_retire_workers_no_suspicious_tasks(c, s, a, b):
 
 
 @pytest.mark.slow
-@pytest.mark.skipif(
-    sys.platform.startswith("win"), reason="file descriptors not really a thing"
-)
-@gen_cluster(client=True, nthreads=[], timeout=240)
+@pytest.mark.skipif(WINDOWS, reason="num_fds not supported on windows")
+@gen_cluster(client=True, nthreads=[], timeout=60)
 async def test_file_descriptors(c, s):
     await asyncio.sleep(0.1)
-    psutil = pytest.importorskip("psutil")
     da = pytest.importorskip("dask.array")
     proc = psutil.Process()
     num_fds_1 = proc.num_fds()
@@ -980,10 +971,8 @@ async def test_file_descriptors(c, s):
             assert comm.closed() or comm.peer_address != s.address, comm
     assert not s.stream_comms
 
-    start = time()
     while proc.num_fds() > num_fds_1 + N:
         await asyncio.sleep(0.01)
-        assert time() < start + 3
 
 
 @pytest.mark.slow
@@ -1325,36 +1314,30 @@ async def test_non_existent_worker(c, s):
 async def test_correct_bad_time_estimate(c, s, *workers):
     future = c.submit(slowinc, 1, delay=0)
     await wait(future)
-
     futures = [c.submit(slowinc, future, delay=0.1, pure=False) for i in range(20)]
-
     await asyncio.sleep(0.5)
-
     await wait(futures)
-
     assert all(w.data for w in workers), [sorted(w.data) for w in workers]
 
 
-@gen_test()
-async def test_service_hosts():
-    port = 0
-    for url, expected in [
-        ("tcp://0.0.0.0", ("::", "0.0.0.0")),
-        ("tcp://127.0.0.1", ("::", "0.0.0.0")),
-        ("tcp://127.0.0.1:38275", ("::", "0.0.0.0")),
-    ]:
-        async with Scheduler(host=url) as s:
-            sock = first(s.http_server._sockets.values())
-            if isinstance(expected, tuple):
-                assert sock.getsockname()[0] in expected
-            else:
-                assert sock.getsockname()[0] == expected
-
-    port = ("127.0.0.1", 0)
-    for url in ["tcp://0.0.0.0", "tcp://127.0.0.1", "tcp://127.0.0.1:38275"]:
-        async with Scheduler(dashboard_address="127.0.0.1:0", host=url) as s:
-            sock = first(s.http_server._sockets.values())
-            assert sock.getsockname()[0] == "127.0.0.1"
+@pytest.mark.parametrize(
+    "host", ["tcp://0.0.0.0", "tcp://127.0.0.1", "tcp://127.0.0.1:38275"]
+)
+@pytest.mark.parametrize(
+    "dashboard_address,expect",
+    [
+        (None, ("::", "0.0.0.0")),
+        ("127.0.0.1:0", ("127.0.0.1",)),
+    ],
+)
+@pytest.mark.asyncio
+async def test_dashboard_host(host, dashboard_address, expect):
+    """Dashboard is accessible from any host by default, but it can be also bound to
+    localhost.
+    """
+    async with Scheduler(host=host, dashboard_address=dashboard_address) as s:
+        sock = first(s.http_server._sockets.values())
+        assert sock.getsockname()[0] in expect
 
 
 @gen_cluster(client=True, worker_kwargs={"profile_cycle_interval": "100ms"})
@@ -1543,9 +1526,6 @@ async def test_retries(c, s, a, b):
     exc_info.match("one")
 
 
-@pytest.mark.flaky(
-    reruns=10, reruns_delay=5, reason="second worker also errant for some reason"
-)
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 3)
 async def test_missing_data_errant_worker(c, s, w1, w2, w3):
     with dask.config.set({"distributed.comm.timeouts.connect": "1s"}):
@@ -1779,7 +1759,7 @@ async def test_bandwidth(c, s, a, b):
     assert not s.bandwidth_workers
 
 
-@gen_cluster(client=True, Worker=Nanny)
+@gen_cluster(client=True, Worker=Nanny, timeout=60)
 async def test_bandwidth_clear(c, s, a, b):
     np = pytest.importorskip("numpy")
     x = c.submit(np.arange, 1000000, workers=[a.worker_address], pure=False)
@@ -1821,9 +1801,7 @@ async def test_close_workers(s, a, b):
     assert b.status == Status.closed
 
 
-@pytest.mark.skipif(
-    not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
-)
+@pytest.mark.skipif(not LINUX, reason="Need 127.0.0.2 to mean localhost")
 @gen_test()
 async def test_host_address():
     s = await Scheduler(host="127.0.0.2", port=0)
@@ -2361,7 +2339,7 @@ async def test_unknown_task_duration_config_2(s, a, b):
     assert s.idle_since == s.time_started
 
 
-@gen_cluster(client=True, timeout=None)
+@gen_cluster(client=True)
 async def test_retire_state_change(c, s, a, b):
     np = pytest.importorskip("numpy")
     y = c.map(lambda x: x ** 2, range(10))
@@ -2482,8 +2460,8 @@ async def assert_memory(scheduler_or_workerstate, attr: str, min_, max_, timeout
     t0 = time()
     while True:
         minfo = scheduler_or_workerstate.memory
-        nbytes = getattr(minfo, attr)
-        if min_ * 2 ** 20 <= nbytes <= max_ * 2 ** 20:
+        nmib = getattr(minfo, attr) / 2 ** 20
+        if min_ <= nmib <= max_:
             return
         if time() - t0 > timeout:
             raise TimeoutError(
@@ -2492,13 +2470,12 @@ async def assert_memory(scheduler_or_workerstate, attr: str, min_, max_, timeout
         await asyncio.sleep(0.1)
 
 
-# This test is heavily influenced by hard-to-control factors such as memory management
-# by the Python interpreter and the OS, so it occasionally glitches
-@pytest.mark.flaky(reruns=3, reruns_delay=5)
-# ~33s runtime, or distributed.memory.recent-to-old-time + 3s
+# ~31s runtime, or distributed.worker.memory.recent-to-old-time + 1s.
+# On Windows, it can take ~65s due to worker memory needing to stabilize first.
 @pytest.mark.slow
+@pytest.mark.flaky(condition=LINUX, reason="see comments", reruns=10, reruns_delay=5)
 @gen_cluster(
-    client=True, Worker=Nanny, worker_kwargs={"memory_limit": "500 MiB"}, timeout=60
+    client=True, Worker=Nanny, worker_kwargs={"memory_limit": "500 MiB"}, timeout=120
 )
 async def test_memory(c, s, *_):
     pytest.importorskip("zict")
@@ -2511,14 +2488,25 @@ async def test_memory(c, s, *_):
     assert s_m0.managed == 0
     assert a.memory.managed == 0
     assert b.memory.managed == 0
-    # When a worker first goes online, its RAM is immediately counted as
-    # unmanaged_old
-    await assert_memory(s, "unmanaged_recent", 0, 40, timeout=0)
-    await assert_memory(a, "unmanaged_recent", 0, 20, timeout=0)
-    await assert_memory(b, "unmanaged_recent", 0, 20, timeout=0)
-
-    f1 = c.submit(leaking, 100, 50, 5, pure=False, workers=[a.name])
-    f2 = c.submit(leaking, 100, 50, 5, pure=False, workers=[b.name])
+
+    # When a worker first goes online, its RAM is immediately counted as unmanaged_old.
+    # On Windows, however, there is somehow enough time between the worker start and
+    # this line for 2 heartbeats and the memory keeps growing substantially for a while.
+    # Sometimes there is a single heartbeat but on the consecutive test we observe
+    # a large unexplained increase in unmanaged_recent memory.
+    # Wait for the situation to stabilize.
+    if WINDOWS:
+        await asyncio.sleep(10)
+        initial_timeout = 40
+    else:
+        initial_timeout = 0
+
+    await assert_memory(s, "unmanaged_recent", 0, 40, timeout=initial_timeout)
+    await assert_memory(a, "unmanaged_recent", 0, 20, timeout=initial_timeout)
+    await assert_memory(b, "unmanaged_recent", 0, 20, timeout=initial_timeout)
+
+    f1 = c.submit(leaking, 100, 50, 10, pure=False, workers=[a.name])
+    f2 = c.submit(leaking, 100, 50, 10, pure=False, workers=[b.name])
     await assert_memory(s, "unmanaged_recent", 300, 380)
     await assert_memory(a, "unmanaged_recent", 150, 190)
     await assert_memory(b, "unmanaged_recent", 150, 190)
@@ -2533,19 +2521,17 @@ async def test_memory(c, s, *_):
     await assert_memory(b, "unmanaged_recent", 50, 90)
 
     # Force the output of f1 and f2 to spill to disk.
-    # With target=0.6 and memory_limit=500 MiB, we'll start spilling at 300 MiB
-    # process memory per worker, or roughly after 3~7 rounds of the below depending
-    # on how much RAM the interpreter is using.
+    # With spill=0.7 and memory_limit=500 MiB, we'll start spilling at 350 MiB process
+    # memory per worker, or up to 20 iterations of the below depending on how much RAM
+    # the interpreter is using.
     more_futs = []
-    for _ in range(8):
-        if s.memory.managed_spilled > 0:
-            break
-        more_futs += [
-            c.submit(leaking, 20, 0, 0, pure=False, workers=[a.name]),
-            c.submit(leaking, 20, 0, 0, pure=False, workers=[b.name]),
-        ]
-        await asyncio.sleep(2)
-    await assert_memory(s, "managed_spilled", 1, 999)
+    while not s.memory.managed_spilled:
+        if a.memory.process < 0.7 * 500 * 2 ** 20:
+            more_futs.append(c.submit(leaking, 10, 0, 0, pure=False, workers=[a.name]))
+        if b.memory.process < 0.7 * 500 * 2 ** 20:
+            more_futs.append(c.submit(leaking, 10, 0, 0, pure=False, workers=[b.name]))
+        await wait(more_futs)
+        await asyncio.sleep(1)
 
     # Wait for the spilling to finish. Note that this does not make the test take
     # longer as we're waiting for recent-to-old-time anyway.
@@ -2568,24 +2554,23 @@ async def test_memory(c, s, *_):
     # transition into unmanaged_old
     await c.run(gc.collect)
     await assert_memory(s, "unmanaged_recent", 0, 90, timeout=40)
-    await assert_memory(
-        s,
-        "unmanaged_old",
-        orig_old + 90,
-        # On MacOS, the process memory of the Python interpreter does not shrink as
-        # fast as on Linux/Windows
-        9999 if MACOS else orig_old + 190,
-        timeout=40,
-    )
+    await assert_memory(s, "unmanaged_old", orig_old + 90, 9999, timeout=40)
 
-    # When the leaked memory is cleared, unmanaged and unmanaged_old drop
-    # On MacOS, the process memory of the Python interpreter does not shrink as fast
-    # as on Linux/Windows
-    if not MACOS:
-        await c.run(clear_leak)
-        await assert_memory(s, "unmanaged", 0, orig_unmanaged + 95)
-        await assert_memory(s, "unmanaged_old", 0, orig_old + 95)
-        await assert_memory(s, "unmanaged_recent", 0, 90)
+    # When the leaked memory is cleared, unmanaged and unmanaged_old drop.
+    # On MacOS and Windows, the process memory of the Python interpreter does not shrink
+    # as fast as on Linux. Note that this behaviour is heavily impacted by OS tweaks,
+    # meaning that what you observe on your local host may behave differently on CI.
+    # Even on Linux, this occasionally glitches - hence why there is a flaky marker on
+    # this test.
+    if not LINUX:
+        return
+
+    orig_unmanaged = s.memory.unmanaged / 2 ** 20
+    orig_old = s.memory.unmanaged_old / 2 ** 20
+    await c.run(clear_leak)
+    await assert_memory(s, "unmanaged", 0, orig_unmanaged - 60)
+    await assert_memory(s, "unmanaged_old", 0, orig_old - 60)
+    await assert_memory(s, "unmanaged_recent", 0, 90)
 
 
 @gen_cluster(client=True, worker_kwargs={"memory_limit": 0})
@@ -2825,23 +2810,20 @@ async def test_rebalance_no_limit(c, s, a, b):
     worker_kwargs={"memory_limit": "1000 MiB"},
     config={
         "distributed.worker.memory.rebalance.measure": "managed",
-        "distributed.worker.memory.rebalance.recipient-max": 0.4,
+        "distributed.worker.memory.rebalance.sender-min": 0.2,
+        "distributed.worker.memory.rebalance.recipient-max": 0.1,
     },
 )
 async def test_rebalance_no_recipients(c, s, *_):
     """There are sender workers, but no recipient workers"""
     a, b = s.workers
-    futures = [
-        c.submit(lambda: "x" * (400 * 2 ** 20), pure=False, workers=[a]),  # 40%
-        c.submit(lambda: "x" * (400 * 2 ** 20), pure=False, workers=[b]),  # 40%
-    ] + c.map(
-        lambda _: "x" * (2 ** 21), range(100), workers=[a]
-    )  # 20%
-    await wait(futures)
-    await assert_memory(s, "managed", 1000, 1001)
-    await assert_ndata(c, {a: 101, b: 1})
+    fut_a = c.map(lambda _: "x" * (2 ** 20), range(250), workers=[a])  # 25%
+    fut_b = c.map(lambda _: "x" * (2 ** 20), range(100), workers=[b])  # 10%
+    await wait(fut_a + fut_b)
+    await assert_memory(s, "managed", 350, 351)
+    await assert_ndata(c, {a: 250, b: 100})
     await s.rebalance()
-    await assert_ndata(c, {a: 101, b: 1})
+    await assert_ndata(c, {a: 250, b: 100})
     s.validate_state()
 
 
diff --git a/distributed/tests/test_semaphore.py b/distributed/tests/test_semaphore.py
index 228ff5e7d4b..0cc4bc9b155 100644
--- a/distributed/tests/test_semaphore.py
+++ b/distributed/tests/test_semaphore.py
@@ -11,7 +11,6 @@
 
 from distributed import Semaphore, fire_and_forget
 from distributed.comm import Comm
-from distributed.compatibility import WINDOWS
 from distributed.core import ConnectionPool
 from distributed.metrics import time
 from distributed.utils_test import captured_logger, cluster, gen_cluster, slowidentity
@@ -91,21 +90,17 @@ def test_timeout_sync(client):
 
 
 @gen_cluster(
-    client=True,
-    timeout=20,
     config={
         "distributed.scheduler.locks.lease-validation-interval": "200ms",
         "distributed.scheduler.locks.lease-timeout": "200ms",
     },
 )
-async def test_release_semaphore_after_timeout(c, s, a, b):
+async def test_release_semaphore_after_timeout(s, a, b):
     sem = await Semaphore(name="x", max_leases=2)
     await sem.acquire()  # leases: 2 - 1 = 1
 
     semB = await Semaphore(name="x", max_leases=2)
-
     assert await semB.acquire()  # leases: 1 - 1 = 0
-
     assert not (await sem.acquire(timeout=0.01))
     assert not (await semB.acquire(timeout=0.01))
 
@@ -114,9 +109,7 @@ async def test_release_semaphore_after_timeout(c, s, a, b):
 
     semB.refresh_callback.stop()
     del semB
-
     assert await sem.acquire(timeout=1)
-
     assert not (await sem.acquire(timeout=0.1))
 
 
@@ -564,7 +557,6 @@ async def test_release_retry(c, s, a, b):
         assert await semaphore.release() is True
 
 
-@pytest.mark.flaky(reruns=10, reruns_delay=5, condition=WINDOWS)
 @gen_cluster(
     client=True,
     config={
diff --git a/distributed/tests/test_steal.py b/distributed/tests/test_steal.py
index 7ca8f29bcc4..c0cecd8c5aa 100644
--- a/distributed/tests/test_steal.py
+++ b/distributed/tests/test_steal.py
@@ -2,7 +2,6 @@
 import itertools
 import logging
 import random
-import sys
 import weakref
 from operator import mul
 from time import sleep
@@ -13,6 +12,7 @@
 import dask
 
 from distributed import Nanny, Worker, wait, worker_client
+from distributed.compatibility import LINUX
 from distributed.config import config
 from distributed.metrics import time
 from distributed.scheduler import key_split
@@ -33,9 +33,7 @@
 teardown_module = nodebug_teardown_module
 
 
-@pytest.mark.skipif(
-    not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
-)
+@pytest.mark.skipif(not LINUX, reason="Need 127.0.0.2 to mean localhost")
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 2), ("127.0.0.2", 2)])
 async def test_work_stealing(c, s, a, b):
     [x] = await c._scatter([1], workers=a.address)
@@ -144,7 +142,7 @@ async def test_steal_related_tasks(e, s, a, b, c):
     assert nearby > 10
 
 
-@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 10, timeout=1000)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 10)
 async def test_dont_steal_fast_tasks_compute_time(c, s, *workers):
     def do_nothing(x, y=None):
         pass
@@ -285,9 +283,7 @@ async def test_steal_worker_restrictions(c, s, wa, wb, wc):
     assert len(wc.tasks) == 0
 
 
-@pytest.mark.skipif(
-    not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
-)
+@pytest.mark.skipif(not LINUX, reason="Need 127.0.0.2 to mean localhost")
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1), ("127.0.0.2", 1)])
 async def test_dont_steal_host_restrictions(c, s, a, b):
     future = c.submit(slowinc, 1, delay=0.10, workers=a.address)
@@ -306,9 +302,7 @@ async def test_dont_steal_host_restrictions(c, s, a, b):
     assert len(b.tasks) == 0
 
 
-@pytest.mark.skipif(
-    not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
-)
+@pytest.mark.skipif(not LINUX, reason="Need 127.0.0.2 to mean localhost")
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1), ("127.0.0.2", 2)])
 async def test_steal_host_restrictions(c, s, wa, wb):
     future = c.submit(slowinc, 1, delay=0.10, workers=wa.address)
@@ -354,9 +348,7 @@ async def test_dont_steal_resource_restrictions(c, s, a, b):
     assert len(b.tasks) == 0
 
 
-@gen_cluster(
-    client=True, nthreads=[("127.0.0.1", 1, {"resources": {"A": 2}})], timeout=3
-)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1, {"resources": {"A": 2}})])
 async def test_steal_resource_restrictions(c, s, a):
     future = c.submit(slowinc, 1, delay=0.10, workers=a.address)
     await future
@@ -601,7 +593,7 @@ async def test(*args, **kwargs):
     test()
 
 
-@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 2, Worker=Nanny)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 2, Worker=Nanny, timeout=60)
 async def test_restart(c, s, a, b):
     futures = c.map(
         slowinc, range(100), delay=0.1, workers=a.address, allow_other_workers=True
@@ -613,7 +605,7 @@ async def test_restart(c, s, a, b):
     assert any(st for st in steal.stealable_all)
     assert any(x for L in steal.stealable.values() for x in L)
 
-    await c.restart(timeout=10)
+    await c.restart()
 
     assert not any(x for x in steal.stealable_all)
     assert not any(x for L in steal.stealable.values() for x in L)
@@ -740,7 +732,6 @@ def long(delay):
         ) <= 1
 
 
-@pytest.mark.flaky(reruns=10, reruns_delay=5, condition=sys.version_info[:2] == (3, 8))
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 5)] * 2)
 async def test_cleanup_repeated_tasks(c, s, a, b):
     class Foo:
diff --git a/distributed/tests/test_stress.py b/distributed/tests/test_stress.py
index cf09c9dbcd5..dba1daa721f 100644
--- a/distributed/tests/test_stress.py
+++ b/distributed/tests/test_stress.py
@@ -1,6 +1,5 @@
 import asyncio
 import random
-import sys
 from contextlib import suppress
 from operator import add
 from time import sleep
@@ -11,9 +10,10 @@
 from dask import delayed
 
 from distributed import Client, Nanny, wait
+from distributed.compatibility import WINDOWS
 from distributed.config import config
 from distributed.metrics import time
-from distributed.utils import All, CancelledError
+from distributed.utils import CancelledError
 from distributed.utils_test import (
     bump_rlimit,
     cluster,
@@ -43,6 +43,7 @@ async def test_stress_1(c, s, a, b):
     assert result == sum(map(inc, range(n)))
 
 
+@pytest.mark.slow
 @pytest.mark.parametrize(("func", "n"), [(slowinc, 100), (inc, 1000)])
 def test_stress_gc(loop, func, n):
     with cluster() as (s, [a, b]):
@@ -54,10 +55,8 @@ def test_stress_gc(loop, func, n):
             assert x.result() == n + 2
 
 
-@pytest.mark.skipif(
-    sys.platform.startswith("win"), reason="test can leave dangling RPC objects"
-)
-@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 8, timeout=None)
+@pytest.mark.skipif(WINDOWS, reason="test can leave dangling RPC objects")
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 8)
 async def test_cancel_stress(c, s, *workers):
     da = pytest.importorskip("dask.array")
     x = da.random.random((50, 50), chunks=(2, 2))
@@ -86,28 +85,33 @@ def test_cancel_stress_sync(loop):
                 c.cancel(f)
 
 
-@gen_cluster(nthreads=[], client=True, timeout=None)
+@pytest.mark.slow
+@gen_cluster(
+    nthreads=[],
+    client=True,
+    timeout=120,
+    scheduler_kwargs={"allowed_failures": 100_000},
+)
 async def test_stress_creation_and_deletion(c, s):
     # Assertions are handled by the validate mechanism in the scheduler
-    s.allowed_failures = 100000
     da = pytest.importorskip("dask.array")
 
-    x = da.random.random(size=(2000, 2000), chunks=(100, 100))
-    y = (x + 1).T + (x * 2) - x.mean(axis=1)
-
+    rng = da.random.RandomState(0)
+    x = rng.random(size=(2000, 2000), chunks=(100, 100))
+    y = ((x + 1).T + (x * 2) - x.mean(axis=1)).sum().round(2)
     z = c.persist(y)
 
     async def create_and_destroy_worker(delay):
         start = time()
         while time() < start + 5:
-            n = await Nanny(s.address, nthreads=2, loop=s.loop)
-            await asyncio.sleep(delay)
-            await n.close()
+            async with Nanny(s.address, nthreads=2):
+                await asyncio.sleep(delay)
             print("Killed nanny")
 
-    await asyncio.wait_for(
-        All([create_and_destroy_worker(0.1 * i) for i in range(20)]), 60
-    )
+    await asyncio.gather(*(create_and_destroy_worker(0.1 * i) for i in range(20)))
+
+    async with Nanny(s.address, nthreads=2):
+        assert await c.compute(z) == 8000884.93
 
 
 @gen_cluster(nthreads=[("127.0.0.1", 1)] * 10, client=True, timeout=60)
@@ -169,6 +173,7 @@ def vsum(*args):
 
 @pytest.mark.avoid_ci
 @pytest.mark.slow
+@pytest.mark.timeout(1100)  # Override timeout from setup.cfg
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 80, timeout=1000)
 async def test_stress_communication(c, s, *workers):
     s.validate = False  # very slow otherwise
diff --git a/distributed/tests/test_tls_functional.py b/distributed/tests/test_tls_functional.py
index 8950f804cb6..54a1202400e 100644
--- a/distributed/tests/test_tls_functional.py
+++ b/distributed/tests/test_tls_functional.py
@@ -42,7 +42,7 @@ async def test_Queue(c, s, a, b):
     assert future.key == future2.key
 
 
-@gen_tls_cluster(client=True, timeout=None)
+@gen_tls_cluster(client=True)
 async def test_client_submit(c, s, a, b):
     assert s.address.startswith("tls://")
 
diff --git a/distributed/tests/test_utils.py b/distributed/tests/test_utils.py
index be0e9847f78..ab84be59d03 100644
--- a/distributed/tests/test_utils.py
+++ b/distributed/tests/test_utils.py
@@ -4,7 +4,6 @@
 import os
 import queue
 import socket
-import sys
 import traceback
 from time import sleep
 
@@ -13,6 +12,7 @@
 
 import dask
 
+from distributed.compatibility import MACOS, WINDOWS
 from distributed.metrics import time
 from distributed.utils import (
     LRU,
@@ -140,23 +140,12 @@ def test_ensure_ip():
         assert ensure_ip("::1") == "::1"
 
 
+@pytest.mark.skipif(WINDOWS, reason="TODO")
 def test_get_ip_interface():
-    if sys.platform == "darwin":
-        assert get_ip_interface("lo0") == "127.0.0.1"
-    elif sys.platform.startswith("linux"):
-        assert get_ip_interface("lo") == "127.0.0.1"
-    else:
-        pytest.skip(f"test needs to be enhanced for platform {sys.platform!r}")
-
-    non_existent_interface = "__non-existent-interface"
-    expected_error_message = f"{non_existent_interface!r}.+network interface.+"
-
-    if sys.platform == "darwin":
-        expected_error_message += "'lo0'"
-    elif sys.platform.startswith("linux"):
-        expected_error_message += "'lo'"
-    with pytest.raises(ValueError, match=expected_error_message):
-        get_ip_interface(non_existent_interface)
+    iface = "lo0" if MACOS else "lo"
+    assert get_ip_interface(iface) == "127.0.0.1"
+    with pytest.raises(ValueError, match=f"'__notexist'.+network interface.+'{iface}'"):
+        get_ip_interface("__notexist")
 
 
 def test_truncate_exception():
diff --git a/distributed/tests/test_utils_test.py b/distributed/tests/test_utils_test.py
index 1e546adb4bc..3dca5766f85 100755
--- a/distributed/tests/test_utils_test.py
+++ b/distributed/tests/test_utils_test.py
@@ -141,7 +141,7 @@ async def f(c, s, a, b):
     assert not dask.config.get("get", None)
 
 
-@gen_cluster(client=False)
+@gen_cluster()
 async def test_gen_cluster_without_client(s, a, b):
     assert isinstance(s, Scheduler)
     for w in [a, b]:
diff --git a/distributed/tests/test_variable.py b/distributed/tests/test_variable.py
index 1819da29049..46eb8604058 100644
--- a/distributed/tests/test_variable.py
+++ b/distributed/tests/test_variable.py
@@ -191,7 +191,7 @@ async def test_timeout_get(c, s, a, b):
 
 
 @pytest.mark.slow
-@gen_cluster(client=True, nthreads=[("127.0.0.1", 2)] * 5, Worker=Nanny, timeout=None)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 2)] * 5, Worker=Nanny)
 async def test_race(c, s, *workers):
     NITERS = 50
 
@@ -216,10 +216,8 @@ def f(i):
     results = await c.gather(futures)
     assert all(r > NITERS * 0.8 for r in results)
 
-    start = time()
     while len(s.wants_what["variable-x"]) != 1:
         await asyncio.sleep(0.01)
-        assert time() - start < 2
 
 
 @gen_cluster(client=True)
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index c38b8b76363..3231e6ce752 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -18,7 +18,6 @@
 import dask
 from dask import delayed
 from dask.system import CPU_COUNT
-from dask.utils import format_bytes
 
 from distributed import (
     Client,
@@ -31,7 +30,7 @@
 )
 from distributed.comm.registry import backends
 from distributed.comm.tcp import TCPBackend
-from distributed.compatibility import MACOS, WINDOWS
+from distributed.compatibility import LINUX, MACOS, WINDOWS
 from distributed.core import CommClosedError, Status, rpc
 from distributed.diagnostics.plugin import PipInstall
 from distributed.metrics import time
@@ -780,7 +779,7 @@ async def test_hold_onto_dependents(c, s, a, b):
 
 
 @pytest.mark.slow
-@gen_cluster(client=False, nthreads=[])
+@gen_cluster(nthreads=[])
 async def test_worker_death_timeout(s):
     with dask.config.set({"distributed.comm.timeouts.connect": "1s"}):
         await s.close()
@@ -1005,27 +1004,17 @@ async def test_global_workers(s, a, b):
     assert w is a or w is b
 
 
-@pytest.mark.skipif(WINDOWS, reason="file descriptors")
+@pytest.mark.skipif(WINDOWS, reason="num_fds not supported on windows")
 @gen_cluster(nthreads=[])
 async def test_worker_fds(s):
-    psutil = pytest.importorskip("psutil")
-    await asyncio.sleep(0.05)
-    start = psutil.Process().num_fds()
-
-    worker = await Worker(s.address, loop=s.loop)
-    await asyncio.sleep(0.1)
-    middle = psutil.Process().num_fds()
-    start = time()
-    while middle > start:
-        await asyncio.sleep(0.01)
-        assert time() < start + 1
+    proc = psutil.Process()
+    before = psutil.Process().num_fds()
 
-    await worker.close()
+    async with Worker(s.address, loop=s.loop):
+        assert proc.num_fds() > before
 
-    start = time()
-    while psutil.Process().num_fds() > start:
+    while proc.num_fds() > before:
         await asyncio.sleep(0.01)
-        assert time() < start + 0.5
 
 
 @gen_cluster(nthreads=[])
@@ -1064,13 +1053,13 @@ async def test_scheduler_file():
 @gen_cluster(client=True)
 async def test_scheduler_delay(c, s, a, b):
     old = a.scheduler_delay
-    assert abs(a.scheduler_delay) < 0.3
-    assert abs(b.scheduler_delay) < 0.3
-    await asyncio.sleep(a.periodic_callbacks["heartbeat"].callback_time / 1000 + 0.3)
+    assert abs(a.scheduler_delay) < 0.6
+    assert abs(b.scheduler_delay) < 0.6
+    await asyncio.sleep(a.periodic_callbacks["heartbeat"].callback_time / 1000 + 0.6)
     assert a.scheduler_delay != old
 
 
-@pytest.mark.flaky(reruns=10, reruns_delay=5, condition=MACOS)
+@pytest.mark.flaky(reruns=10, reruns_delay=5)
 @gen_cluster(client=True)
 async def test_statistical_profiling(c, s, a, b):
     futures = c.map(slowinc, range(10), delay=0.1)
@@ -1134,7 +1123,6 @@ def f(n):
 
 
 @pytest.mark.slow
-@pytest.mark.flaky(reruns=10, reruns_delay=5, condition=sys.version_info[:2] == (3, 8))
 @gen_cluster(
     nthreads=[("127.0.0.1", 2)],
     client=True,
@@ -1144,7 +1132,6 @@ def f(n):
         "memory_target_fraction": False,
         "memory_pause_fraction": 0.5,
     },
-    timeout=20,
 )
 async def test_pause_executor(c, s, a):
     memory = psutil.Process().memory_info().rss
@@ -1159,14 +1146,9 @@ def f():
         future = c.submit(f)
         futures = c.map(slowinc, range(30), delay=0.1)
 
-        start = time()
         while not a.paused:
             await asyncio.sleep(0.01)
-            assert time() < start + 4, (
-                format_bytes(psutil.Process().memory_info().rss),
-                format_bytes(a.memory_limit),
-                len(a.data),
-            )
+
         out = logger.getvalue()
         assert "memory" in out.lower()
         assert "pausing" in out.lower()
@@ -1312,9 +1294,7 @@ async def test_wait_for_outgoing(c, s, a, b):
     assert 1 / 3 < ratio < 3
 
 
-@pytest.mark.skipif(
-    not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
-)
+@pytest.mark.skipif(not LINUX, reason="Need 127.0.0.2 to mean localhost")
 @gen_cluster(
     nthreads=[("127.0.0.1", 1), ("127.0.0.1", 1), ("127.0.0.2", 1)], client=True
 )
@@ -1467,9 +1447,7 @@ async def test_local_directory_make_new_directory(s):
         assert "dask-worker-space" in w.local_directory
 
 
-@pytest.mark.skipif(
-    not sys.platform.startswith("linux"), reason="Need 127.0.0.2 to mean localhost"
-)
+@pytest.mark.skipif(not LINUX, reason="Need 127.0.0.2 to mean localhost")
 @gen_cluster(nthreads=[], client=True)
 async def test_host_address(c, s):
     w = await Worker(s.address, host="127.0.0.2")
@@ -2058,7 +2036,7 @@ def raise_exc(*args):
     res = c.submit(raise_exc, f, g, workers=[a.address])
 
     with pytest.raises(RuntimeError):
-        await res.result()
+        await res
 
     # Nothing bad happened on B, therefore B should hold on to G
     assert len(b.tasks) == 1
@@ -2124,7 +2102,7 @@ def raise_exc(*args):
     res = c.submit(raise_exc, f, g, workers=[a.address])
 
     with pytest.raises(RuntimeError):
-        await res.result()
+        await res
 
     # Nothing bad happened on B, therefore B should hold on to G
     assert len(b.tasks) == 1
@@ -2190,7 +2168,7 @@ def raise_exc(*args):
     res = c.submit(raise_exc, f, g, workers=[a.address])
 
     with pytest.raises(RuntimeError):
-        await res.result()
+        await res
 
     # Nothing bad happened on B, therefore B should hold on to G
     assert len(b.tasks) == 1
@@ -2251,7 +2229,7 @@ def raise_exc(*args):
     )
 
     with pytest.raises(RuntimeError):
-        await res.result()
+        await res
 
     expected_states_A = {
         f.key: "memory",
@@ -2319,7 +2297,7 @@ def raise_exc(*args):
             await asyncio.sleep(0.01)
 
 
-@gen_cluster(client=True, nthreads=[("127.0.0.1", x) for x in range(4)], timeout=None)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", x) for x in range(4)])
 async def test_hold_on_to_replicas(c, s, *workers):
     f1 = c.submit(inc, 1, workers=[workers[0].address], key="f1")
     f2 = c.submit(inc, 2, workers=[workers[1].address], key="f2")
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index 0b43caa323c..70531d8b0ef 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -764,6 +764,10 @@ def gen_test(timeout=_TEST_TIMEOUT):
     async def test_foo():
         await ...  # use tornado coroutines
     """
+    assert timeout, (
+        "timeout should always be set and it should be smaller than the global one from"
+        "pytest-timeout"
+    )
 
     def _(func):
         def test_func():
@@ -809,8 +813,6 @@ async def start_cluster(
         )
         for i, ncore in enumerate(nthreads)
     ]
-    # for w in workers:
-    #     w.rpc = workers[0].rpc
 
     await asyncio.gather(*workers)
 
@@ -875,12 +877,16 @@ async def test_foo(scheduler, worker1, worker2, pytest_fixture_a, pytest_fixture
         start
         end
     """
+    assert timeout, (
+        "timeout should always be set and it should be smaller than the global one from"
+        "pytest-timeout"
+    )
     if ncores is not None:
         warnings.warn("ncores= has moved to nthreads=", stacklevel=2)
         nthreads = ncores
 
     worker_kwargs = merge(
-        {"memory_limit": system.MEMORY_LIMIT, "death_timeout": 10}, worker_kwargs
+        {"memory_limit": system.MEMORY_LIMIT, "death_timeout": 15}, worker_kwargs
     )
 
     def _(func):
@@ -930,8 +936,7 @@ async def coro():
                             args = [c] + args
                         try:
                             future = func(*args, *outer_args, **kwargs)
-                            if timeout:
-                                future = asyncio.wait_for(future, timeout)
+                            future = asyncio.wait_for(future, timeout)
                             result = await future
                             if s.validate:
                                 s.validate_state()
diff --git a/distributed/worker.py b/distributed/worker.py
index 94ae0418a2d..03c02a36c32 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -1217,7 +1217,7 @@ def _close(self, *args, **kwargs):
         return self.close(*args, **kwargs)
 
     async def close(
-        self, report=True, timeout=10, nanny=True, executor_wait=True, safe=False
+        self, report=True, timeout=30, nanny=True, executor_wait=True, safe=False
     ):
         with log_errors():
             if self.status in (Status.closed, Status.closing):
diff --git a/setup.cfg b/setup.cfg
index 136ae4c363f..2ef82cb04f7 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -50,5 +50,12 @@ markers =
     slow: marks tests as slow (deselect with '-m "not slow"')
     avoid_ci: marks tests as flaky on CI on all OSs
     ipython: marks tests as exercising IPython
+
+# pytest-timeout settings
+# 'thread' kills off the whole test suite. 'signal' only kills the offending test.
+# However, 'signal' does not work on Windows. The CI script modifies this config file on
+# the fly on Linux and MacOS.
 timeout_method = thread
+# This should not be reduced; Windows CI has been observed to be occasionally
+# exceptionally slow.
 timeout = 300

From 5f01fe6010a0a3621956920d0260ac5f0111f17d Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Wed, 14 Jul 2021 14:18:10 +0100
Subject: [PATCH 1368/1550] rebalance() resilience to computations (#4968)

- Let rebalance() gracefully handle all possible race conditions that could be caused by a computation running at the same time on the cluster
- Thorough unit test coverage for all the above race condition cases
- Code deduplication with replicate(), which also grants it increased resiliency
---
 distributed/client.py               |   8 +-
 distributed/scheduler.py            | 208 ++++++++++++++++--------
 distributed/tests/test_client.py    |  21 +--
 distributed/tests/test_scheduler.py | 238 ++++++++++++++++++++++++----
 distributed/tests/test_worker.py    |  80 ++++++----
 distributed/worker.py               |  15 +-
 6 files changed, 416 insertions(+), 154 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index fa18dfd3d17..09730508270 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -3095,11 +3095,9 @@ async def _rebalance(self, futures=None, workers=None):
         else:
             keys = None
         result = await self.scheduler.rebalance(keys=keys, workers=workers)
-        if result["status"] == "missing-data":
-            raise KeyError(
-                f"During rebalance {len(result['keys'])} keys were found to be missing"
-            )
-        assert result["status"] == "OK"
+        if result["status"] == "partial-fail":
+            raise KeyError(f"Could not rebalance keys: {result['keys']}")
+        assert result["status"] == "OK", result
 
     def rebalance(self, futures=None, workers=None, **kwargs):
         """Rebalance data within network
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 29ca0fb7759..9c25d2c85b3 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -24,7 +24,6 @@
 import sortedcontainers
 from tlz import (
     compose,
-    concat,
     first,
     groupby,
     merge,
@@ -5392,7 +5391,7 @@ async def scatter(
         return keys
 
     async def gather(self, comm=None, keys=None, serializers=None):
-        """Collect data in from workers"""
+        """Collect data from workers to the scheduler"""
         parent: SchedulerState = cast(SchedulerState, self)
         ws: WorkerState
         keys = list(keys)
@@ -5598,31 +5597,108 @@ async def proxy(self, comm=None, msg=None, worker=None, serializers=None):
         )
         return d[worker]
 
-    async def _delete_worker_data(self, worker_address, keys):
+    async def _gather_on_worker(
+        self, worker_address: str, who_has: "dict[Hashable, list[str]]"
+    ) -> set:
+        """Peer-to-peer copy of keys from multiple workers to a single worker
+
+        Parameters
+        ----------
+        worker_address: str
+            Recipient worker address to copy keys to
+        who_has: dict[Hashable, list[str]]
+            {key: [sender address, sender address, ...], key: ...}
+
+        Returns
+        -------
+        returns:
+            set of keys that failed to be copied
+        """
+        try:
+            result = await retry_operation(
+                self.rpc(addr=worker_address).gather, who_has=who_has
+            )
+        except OSError as e:
+            # This can happen e.g. if the worker is going through controlled shutdown;
+            # it doesn't necessarily mean that it went unexpectedly missing
+            logger.warning(
+                f"Communication with worker {worker_address} failed during "
+                f"replication: {e.__class__.__name__}: {e}"
+            )
+            return set(who_has)
+
+        parent: SchedulerState = cast(SchedulerState, self)
+        ws: WorkerState = parent._workers_dv.get(worker_address)
+
+        if ws is None:
+            logger.warning(f"Worker {worker_address} lost during replication")
+            return set(who_has)
+        elif result["status"] == "OK":
+            keys_failed = set()
+            keys_ok = who_has.keys()
+        elif result["status"] == "partial-fail":
+            keys_failed = set(result["keys"])
+            keys_ok = who_has.keys() - keys_failed
+            logger.warning(
+                f"Worker {worker_address} failed to acquire keys: {result['keys']}"
+            )
+        else:  # pragma: nocover
+            raise ValueError(f"Unexpected message from {worker_address}: {result}")
+
+        for key in keys_ok:
+            ts: TaskState = parent._tasks.get(key)
+            if ts is None or ts._state != "memory":
+                logger.warning(f"Key lost during replication: {key}")
+                continue
+            if ts not in ws._has_what:
+                ws._nbytes += ts.get_nbytes()
+                ws._has_what[ts] = None
+                ts._who_has.add(ws)
+
+        return keys_failed
+
+    async def _delete_worker_data(self, worker_address: str, keys: "list[str]") -> None:
         """Delete data from a worker and update the corresponding worker/task states
 
         Parameters
         ----------
         worker_address: str
             Worker address to delete keys from
-        keys: List[str]
+        keys: list[str]
             List of keys to delete on the specified worker
         """
         parent: SchedulerState = cast(SchedulerState, self)
 
-        await retry_operation(
-            self.rpc(addr=worker_address).free_keys,
-            keys=list(keys),
-            reason="rebalance/replicate",
-        )
+        try:
+            await retry_operation(
+                self.rpc(addr=worker_address).free_keys,
+                keys=list(keys),
+                reason="rebalance/replicate",
+            )
+        except OSError as e:
+            # This can happen e.g. if the worker is going through controlled shutdown;
+            # it doesn't necessarily mean that it went unexpectedly missing
+            logger.warning(
+                f"Communication with worker {worker_address} failed during "
+                f"replication: {e.__class__.__name__}: {e}"
+            )
+            return
+
+        ws: WorkerState = parent._workers_dv.get(worker_address)
+        if ws is None:
+            return
+
+        for key in keys:
+            ts: TaskState = parent._tasks.get(key)
+            if ts is not None and ts in ws._has_what:
+                assert ts._state == "memory"
+                del ws._has_what[ts]
+                ts._who_has.remove(ws)
+                ws._nbytes -= ts.get_nbytes()
+                if not ts._who_has:
+                    # Last copy deleted
+                    self.transitions({key: "released"})
 
-        ws: WorkerState = parent._workers_dv[worker_address]
-        ts: TaskState
-        tasks: set = {parent._tasks[key] for key in keys}
-        for ts in tasks:
-            del ws._has_what[ts]
-            ts._who_has.remove(ws)
-            ws._nbytes -= ts.get_nbytes()
         self.log_event(ws._address, {"action": "remove-worker-data", "keys": keys})
 
     async def rebalance(
@@ -5717,14 +5793,18 @@ async def rebalance(
                     if k not in parent._tasks or not parent._tasks[k].who_has
                 ]
                 if missing_data:
-                    return {"status": "missing-data", "keys": missing_data}
+                    return {"status": "partial-fail", "keys": missing_data}
 
             msgs = self._rebalance_find_msgs(keys, workers)
             if not msgs:
                 return {"status": "OK"}
 
             async with self._lock:
-                return await self._rebalance_move_data(msgs)
+                result = await self._rebalance_move_data(msgs)
+                if result["status"] == "partial-fail" and keys is None:
+                    # Only return failed keys if the client explicitly asked for them
+                    result = {"status": "OK"}
+                return result
 
     def _rebalance_find_msgs(
         self: SchedulerState,
@@ -5881,7 +5961,7 @@ def _rebalance_find_msgs(
                     # move on to the next task of the same sender.
                     continue
 
-                # Schedule task for transfer from sender to receiver
+                # Schedule task for transfer from sender to recipient
                 msgs.append((snd_ws, rec_ws, ts))
 
                 # *_bytes_max/min are all negative for heap sorting
@@ -5902,7 +5982,7 @@ def _rebalance_find_msgs(
                 else:
                     heapq.heappop(senders)
 
-                # If receiver still has bytes to gain, push it back into the receivers
+                # If recipient still has bytes to gain, push it back into the recipients
                 # heap; it may or may not come back on top again.
                 if rec_bytes_min < 0:
                     # See definition of recipients above
@@ -5927,29 +6007,46 @@ async def _rebalance_move_data(
         self, msgs: "list[tuple[WorkerState, WorkerState, TaskState]]"
     ) -> dict:
         """Perform the actual transfer of data across the network in rebalance().
-        Takes in input the output of _rebalance_find_msgs().
+        Takes in input the output of _rebalance_find_msgs(), that is a list of tuples:
+
+        - sender worker
+        - recipient worker
+        - task to be transferred
 
         FIXME this method is not robust when the cluster is not idle.
         """
-        ts: TaskState
         snd_ws: WorkerState
         rec_ws: WorkerState
+        ts: TaskState
 
         to_recipients = defaultdict(lambda: defaultdict(list))
+        for snd_ws, rec_ws, ts in msgs:
+            to_recipients[rec_ws.address][ts._key].append(snd_ws.address)
+        failed_keys_by_recipient = dict(
+            zip(
+                to_recipients,
+                await asyncio.gather(
+                    *(
+                        # Note: this never raises exceptions
+                        self._gather_on_worker(w, who_has)
+                        for w, who_has in to_recipients.items()
+                    )
+                ),
+            )
+        )
+
         to_senders = defaultdict(list)
-        for sender, recipient, ts in msgs:
-            to_recipients[recipient.address][ts._key].append(sender.address)
-            to_senders[sender.address].append(ts._key)
+        for snd_ws, rec_ws, ts in msgs:
+            if ts._key not in failed_keys_by_recipient[rec_ws.address]:
+                to_senders[snd_ws.address].append(ts._key)
 
-        result = await asyncio.gather(
-            *(
-                retry_operation(self.rpc(addr=r).gather, who_has=v)
-                for r, v in to_recipients.items()
-            )
+        # Note: this never raises exceptions
+        await asyncio.gather(
+            *(self._delete_worker_data(r, v) for r, v in to_senders.items())
         )
+
         for r, v in to_recipients.items():
             self.log_event(r, {"action": "rebalance", "who_has": v})
-
         self.log_event(
             "all",
             {
@@ -5960,31 +6057,11 @@ async def _rebalance_move_data(
             },
         )
 
-        if any(r["status"] != "OK" for r in result):
-            return {
-                "status": "missing-data",
-                "keys": list(
-                    concat(
-                        r["keys"].keys()
-                        for r in result
-                        if r["status"] == "missing-data"
-                    )
-                ),
-            }
-
-        for snd_ws, rec_ws, ts in msgs:
-            assert ts._state == "memory"
-            ts._who_has.add(rec_ws)
-            rec_ws._has_what[ts] = None
-            rec_ws.nbytes += ts.get_nbytes()
-            self.log.append(
-                ("rebalance", ts._key, time(), snd_ws.address, rec_ws.address)
-            )
-
-        await asyncio.gather(
-            *(self._delete_worker_data(r, v) for r, v in to_senders.items())
-        )
-        return {"status": "OK"}
+        missing_keys = {k for r in failed_keys_by_recipient.values() for k in r}
+        if missing_keys:
+            return {"status": "partial-fail", "keys": list(missing_keys)}
+        else:
+            return {"status": "OK"}
 
     async def replicate(
         self,
@@ -6035,7 +6112,7 @@ async def replicate(
             tasks = {parent._tasks[k] for k in keys}
             missing_data = [ts._key for ts in tasks if not ts._who_has]
             if missing_data:
-                return {"status": "missing-data", "keys": missing_data}
+                return {"status": "partial-fail", "keys": missing_data}
 
             # Delete extraneous data
             if delete:
@@ -6048,6 +6125,7 @@ async def replicate(
                         ):
                             del_worker_tasks[ws].add(ts)
 
+                # Note: this never raises exceptions
                 await asyncio.gather(
                     *[
                         self._delete_worker_data(ws._address, [t.key for t in tasks])
@@ -6077,19 +6155,15 @@ async def replicate(
                             wws._address for wws in ts._who_has
                         ]
 
-                results = await asyncio.gather(
+                await asyncio.gather(
                     *(
-                        retry_operation(self.rpc(addr=w).gather, who_has=who_has)
+                        # Note: this never raises exceptions
+                        self._gather_on_worker(w, who_has)
                         for w, who_has in gathers.items()
                     )
                 )
-                for w, v in zip(gathers, results):
-                    if v["status"] == "OK":
-                        self.add_keys(worker=w, keys=list(gathers[w]))
-                    else:
-                        logger.warning("Communication failed during replication: %s", v)
-
-                    self.log_event(w, {"action": "replicate-add", "keys": gathers[w]})
+                for r, v in gathers.items():
+                    self.log_event(r, {"action": "replicate-add", "who_has": v})
 
             self.log_event(
                 "all",
@@ -7655,7 +7729,7 @@ def validate_task_state(ts: TaskState):
         assert dts._state != "forgotten"
 
     assert (ts._processing_on is not None) == (ts._state == "processing")
-    assert (not not ts._who_has) == (ts._state == "memory"), (ts, ts._who_has)
+    assert bool(ts._who_has) == (ts._state == "memory"), (ts, ts._who_has, ts._state)
 
     if ts._state == "processing":
         assert all([dts._who_has for dts in ts._dependencies]), (
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index e6ef29b4673..7bdeaff003b 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -2995,21 +2995,12 @@ async def test_rebalance_unprepared(c, s, a, b):
     s.validate_state()
 
 
-@gen_cluster(client=True, Worker=Nanny, worker_kwargs={"memory_limit": "1 GiB"})
-async def test_rebalance_raises_missing_data(c, s, *_):
-    a, b = s.workers
-    futures = c.map(lambda _: "x" * (2 ** 29 // 10), range(10), workers=[a])
-    await wait(futures)
-    # Wait for heartbeats
-    while s.memory.process < 2 ** 29:
-        await asyncio.sleep(0.1)
-
-    # Descoping the futures enqueues a coroutine to release the data on the server
-    del futures
-    with pytest.raises(KeyError, match="keys were found to be missing"):
-        # During the synchronous part of rebalance, the futures still exist, but they
-        # will be (partially) gone by the time the actual transferring happens.
-        await c.rebalance()
+@gen_cluster(client=True)
+async def test_rebalance_raises_on_explicit_missing_data(c, s, a, b):
+    """rebalance() raises KeyError if explicitly listed futures disappear"""
+    f = Future("x", client=c, state="memory")
+    with pytest.raises(KeyError, match="Could not rebalance keys:"):
+        await c.rebalance(futures=[f])
 
 
 @gen_cluster(client=True)
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 4bab24aa003..353781e5e11 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -330,7 +330,6 @@ async def test_remove_worker_from_scheduler(s, a, b):
     await s.remove_worker(address=a.address)
     assert a.address not in s.nthreads
     assert len(s.workers[b.address].processing) == len(dsk)  # b owns everything
-    s.validate_state()
 
 
 @gen_cluster()
@@ -339,7 +338,6 @@ async def test_remove_worker_by_name_from_scheduler(s, a, b):
     assert await s.remove_worker(address=a.name) == "OK"
     assert a.address not in s.nthreads
     assert await s.remove_worker(address=a.address) == "already-removed"
-    s.validate_state()
 
 
 @gen_cluster(config={"distributed.scheduler.events-cleanup-delay": "10 ms"})
@@ -1040,7 +1038,6 @@ async def test_learn_occupancy_multiple_workers(c, s, a, b):
     await wait(x)
 
     assert not any(v == 0.5 for w in s.workers.values() for v in w.processing.values())
-    s.validate_state()
 
 
 @gen_cluster(client=True)
@@ -2307,7 +2304,7 @@ async def test_multiple_listeners(cleanup):
 
 @gen_cluster(nthreads=[("127.0.0.1", 1)])
 async def test_worker_name_collision(s, a):
-    # test that a name collision for workers produces the expected respsone
+    # test that a name collision for workers produces the expected response
     # and leaves the data structures of Scheduler in a good state
     # is not updated by the second worker
     with captured_logger(logging.getLogger("distributed.scheduler")) as log:
@@ -2679,7 +2676,6 @@ async def test_rebalance(c, s, *_):
     # rebalance() when there is nothing to do
     await s.rebalance()
     await assert_ndata(c, {a: (3, 7), b: (3, 7)}, total=10)
-    s.validate_state()
 
 
 @gen_cluster(
@@ -2711,15 +2707,12 @@ async def test_rebalance_workers_and_keys(client, s, *_):
     with pytest.raises(KeyError):
         await s.rebalance(workers=["notexist"])
 
-    s.validate_state()
-
 
 @gen_cluster()
 async def test_rebalance_missing_data1(s, a, b):
     """key never existed"""
     out = await s.rebalance(keys=["notexist"])
-    assert out == {"status": "missing-data", "keys": ["notexist"]}
-    s.validate_state()
+    assert out == {"status": "partial-fail", "keys": ["notexist"]}
 
 
 @gen_cluster(client=True)
@@ -2730,32 +2723,38 @@ async def test_rebalance_missing_data2(c, s, a, b):
     futures = c.map(slowinc, range(10), delay=0.05, workers=a.address)
     await asyncio.sleep(0.1)
     out = await s.rebalance(keys=[f.key for f in futures])
-    assert out["status"] == "missing-data"
+    assert out["status"] == "partial-fail"
     assert 8 <= len(out["keys"]) <= 10
-    s.validate_state()
 
 
+@pytest.mark.parametrize("explicit", [False, True])
 @gen_cluster(client=True, Worker=Nanny, worker_kwargs={"memory_limit": "1 GiB"})
-async def test_rebalance_raises_missing_data3(c, s, *_):
+async def test_rebalance_raises_missing_data3(c, s, *_, explicit):
     """keys exist when the sync part of rebalance runs, but are gone by the time the
-    actual data movement runs
+    actual data movement runs.
+    There is an error message only if the keys are explicitly listed in the API call.
     """
     a, _ = s.workers
     futures = c.map(lambda _: "x" * (2 ** 29 // 10), range(10), workers=[a])
     await wait(futures)
     # Wait for heartbeats
     await assert_memory(s, "process", 512, 1024)
-    del futures
-    out = await s.rebalance()
-    assert out["status"] == "missing-data"
-    assert 1 <= len(out["keys"]) <= 10
-    s.validate_state()
+
+    if explicit:
+        keys = [f.key for f in futures]
+        del futures
+        out = await s.rebalance(keys=keys)
+        assert out["status"] == "partial-fail"
+        assert 1 <= len(out["keys"]) <= 10
+    else:
+        del futures
+        out = await s.rebalance()
+        assert out == {"status": "OK"}
 
 
 @gen_cluster(nthreads=[])
 async def test_rebalance_no_workers(s):
     await s.rebalance()
-    s.validate_state()
 
 
 @gen_cluster(
@@ -2783,7 +2782,6 @@ async def test_rebalance_managed_memory(c, s, *_):
     # We can expect an exact, stable result because we are completely bypassing the
     # unpredictability of unmanaged memory.
     await assert_ndata(c, {a: 62, b: 38})
-    s.validate_state()
 
 
 @gen_cluster(
@@ -2801,7 +2799,6 @@ async def test_rebalance_no_limit(c, s, a, b):
     await s.rebalance()
     # Disabling memory_limit made us ignore all % thresholds set in the config
     await assert_ndata(c, {a.address: 50, b.address: 50})
-    s.validate_state()
 
 
 @gen_cluster(
@@ -2824,7 +2821,6 @@ async def test_rebalance_no_recipients(c, s, *_):
     await assert_ndata(c, {a: 250, b: 100})
     await s.rebalance()
     await assert_ndata(c, {a: 250, b: 100})
-    s.validate_state()
 
 
 @gen_cluster(
@@ -2842,7 +2838,6 @@ async def test_rebalance_skip_recipient(client, s, a, b, c):
     await assert_ndata(client, {a.address: 10, b.address: 2, c.address: 2})
     await client.rebalance(futures[:2])
     await assert_ndata(client, {a.address: 8, b.address: 2, c.address: 4})
-    s.validate_state()
 
 
 @gen_cluster(
@@ -2858,7 +2853,6 @@ async def test_rebalance_skip_all_recipients(c, s, a, b):
     await assert_ndata(c, {a.address: 10, b.address: 1})
     await c.rebalance(futures[:2])
     await assert_ndata(c, {a.address: 9, b.address: 2})
-    s.validate_state()
 
 
 @gen_cluster(
@@ -2907,10 +2901,198 @@ async def test_rebalance_least_recently_inserted_sender_min(c, s, *_):
     await assert_ndata(c, {a: 11, b: 0})
     await s.rebalance()
     await assert_ndata(c, {a: 1, b: 10})
-    assert await c.has_what() == {
-        a: (large_future.key,),
-        b: tuple(f.key for f in small_futures),
-    }
+    has_what = await c.has_what()
+    assert has_what[a] == (large_future.key,)
+    assert sorted(has_what[b]) == sorted(f.key for f in small_futures)
+
+
+@gen_cluster(client=True)
+async def test_gather_on_worker(c, s, a, b):
+    x = await c.scatter("x", workers=[a.address])
+    x_ts = s.tasks[x.key]
+    a_ws = s.workers[a.address]
+    b_ws = s.workers[b.address]
+
+    assert a_ws.nbytes > 0
+    assert b_ws.nbytes == 0
+    assert x_ts in a_ws.has_what
+    assert x_ts not in b_ws.has_what
+    assert x_ts.who_has == {a_ws}
+
+    out = await s._gather_on_worker(b.address, {x.key: [a.address]})
+    assert out == set()
+    assert a.data[x.key] == "x"
+    assert b.data[x.key] == "x"
+
+    assert b_ws.nbytes == a_ws.nbytes
+    assert x_ts in b_ws.has_what
+    assert x_ts.who_has == {a_ws, b_ws}
+
+
+@gen_cluster(client=True, scheduler_kwargs={"timeout": "100ms"})
+async def test_gather_on_worker_bad_recipient(c, s, a, b):
+    """The recipient is missing"""
+    x = await c.scatter("x")
+    await b.close()
+    assert s.workers.keys() == {a.address}
+    out = await s._gather_on_worker(b.address, {x.key: [a.address]})
+    assert out == {x.key}
+
+
+@gen_cluster(client=True, worker_kwargs={"timeout": "100ms"})
+async def test_gather_on_worker_bad_sender(c, s, a, b):
+    """The only sender for a key is missing"""
+    out = await s._gather_on_worker(a.address, {"x": ["tcp://127.0.0.1:12345"]})
+    assert out == {"x"}
+
+
+@pytest.mark.parametrize("missing_first", [False, True])
+@gen_cluster(client=True, worker_kwargs={"timeout": "100ms"})
+async def test_gather_on_worker_bad_sender_replicated(c, s, a, b, missing_first):
+    """One of the senders for a key is missing, but the key is available somewhere else"""
+    x = await c.scatter("x", workers=[a.address])
+    bad_addr = "tcp://127.0.0.1:12345"
+    # Order matters; test both
+    addrs = [bad_addr, a.address] if missing_first else [a.address, bad_addr]
+    out = await s._gather_on_worker(b.address, {x.key: addrs})
+    assert out == set()
+    assert a.data[x.key] == "x"
+    assert b.data[x.key] == "x"
+
+
+@gen_cluster(client=True)
+async def test_gather_on_worker_key_not_on_sender(c, s, a, b):
+    """The only sender for a key does not actually hold it"""
+    out = await s._gather_on_worker(a.address, {"x": [b.address]})
+    assert out == {"x"}
+
+
+@pytest.mark.parametrize("missing_first", [False, True])
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 3)
+async def test_gather_on_worker_key_not_on_sender_replicated(
+    client, s, a, b, c, missing_first
+):
+    """One of the senders for a key does not actually hold it, but the key is available
+    somewhere else
+    """
+    x = await client.scatter("x", workers=[a.address])
+    # Order matters; test both
+    addrs = [b.address, a.address] if missing_first else [a.address, b.address]
+    out = await s._gather_on_worker(c.address, {x.key: addrs})
+    assert out == set()
+    assert a.data[x.key] == "x"
+    assert c.data[x.key] == "x"
+
+
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 3)
+async def test_gather_on_worker_duplicate_task(client, s, a, b, c):
+    """Race condition where the recipient worker receives the same task twice.
+    Test that the task nbytes are not double-counted on the recipient.
+    """
+    x = await client.scatter("x", workers=[a.address, b.address], broadcast=True)
+    assert a.data[x.key] == "x"
+    assert b.data[x.key] == "x"
+    assert x.key not in c.data
+
+    out = await asyncio.gather(
+        s._gather_on_worker(c.address, {x.key: [a.address]}),
+        s._gather_on_worker(c.address, {x.key: [b.address]}),
+    )
+    assert out == [set(), set()]
+    assert c.data[x.key] == "x"
+
+    a_ws = s.workers[a.address]
+    b_ws = s.workers[b.address]
+    c_ws = s.workers[c.address]
+    assert a_ws.nbytes > 0
+    assert c_ws.nbytes == b_ws.nbytes == a_ws.nbytes
+
+
+@gen_cluster(
+    client=True, nthreads=[("127.0.0.1", 1)] * 3, scheduler_kwargs={"timeout": "100ms"}
+)
+async def test_rebalance_dead_recipient(client, s, a, b, c):
+    """A key fails to be rebalanced due to recipient failure.
+    The key is not deleted from the sender.
+    Unrelated, successful keys are deleted from the senders.
+    """
+    x, y = await client.scatter(["x", "y"], workers=[a.address])
+    a_ws = s.workers[a.address]
+    b_ws = s.workers[b.address]
+    c_ws = s.workers[c.address]
+    x_ts = s.tasks[x.key]
+    y_ts = s.tasks[y.key]
+    await c.close()
+    assert s.workers.keys() == {a.address, b.address}
+
+    out = await s._rebalance_move_data([(a_ws, b_ws, x_ts), (a_ws, c_ws, y_ts)])
+    assert out == {"status": "partial-fail", "keys": [y.key]}
+    assert a.data == {y.key: "y"}
+    assert b.data == {x.key: "x"}
+    assert await client.has_what() == {a.address: (y.key,), b.address: (x.key,)}
+
+
+@gen_cluster(client=True)
+async def test_delete_worker_data(c, s, a, b):
+    # delete only copy of x
+    # delete one of the copies of y
+    # don't touch z
+    x, y, z = await c.scatter(["x", "y", "z"], workers=[a.address])
+    await c.replicate(y)
+
+    assert a.data == {x.key: "x", y.key: "y", z.key: "z"}
+    assert b.data == {y.key: "y"}
+    assert s.tasks.keys() == {x.key, y.key, z.key}
+
+    await s._delete_worker_data(a.address, [x.key, y.key])
+    assert a.data == {z.key: "z"}
+    assert b.data == {y.key: "y"}
+    assert s.tasks.keys() == {y.key, z.key}
+    assert s.workers[a.address].nbytes == s.tasks[z.key].nbytes
+
+
+@gen_cluster(nthreads=[("127.0.0.1", 1)], client=True)
+async def test_delete_worker_data_double_delete(c, s, a):
+    """_delete_worker_data race condition where the same key is deleted twice.
+    WorkerState.nbytes is not double-decreased.
+    """
+    x, y = await c.scatter(["x", "y"])
+    await asyncio.gather(
+        s._delete_worker_data(a.address, [x.key]),
+        s._delete_worker_data(a.address, [x.key]),
+    )
+    assert a.data == {y.key: "y"}
+    a_ws = s.workers[a.address]
+    y_ts = s.tasks[y.key]
+    assert a_ws.nbytes == y_ts.nbytes
+
+
+@gen_cluster(scheduler_kwargs={"timeout": "100ms"})
+async def test_delete_worker_data_bad_worker(s, a, b):
+    """_delete_worker_data gracefully handles a non-existing worker;
+    e.g. a sender died in the middle of rebalance()
+    """
+    await a.close()
+    assert s.workers.keys() == {b.address}
+    await s._delete_worker_data(a.address, ["x"])
+
+
+@pytest.mark.parametrize("bad_first", [False, True])
+@gen_cluster(nthreads=[("127.0.0.1", 1)], client=True)
+async def test_delete_worker_data_bad_task(c, s, a, bad_first):
+    """_delete_worker_data gracefully handles a non-existing key;
+    e.g. a task was stolen by work stealing in the middle of a rebalance().
+    Other tasks on the same worker are deleted.
+    """
+    x, y = await c.scatter(["x", "y"])
+    assert a.data == {x.key: "x", y.key: "y"}
+    assert s.tasks.keys() == {x.key, y.key}
+
+    keys = ["notexist", x.key] if bad_first else [x.key, "notexist"]
+    await s._delete_worker_data(a.address, keys)
+    assert a.data == {y.key: "y"}
+    assert s.tasks.keys() == {y.key}
+    assert s.workers[a.address].nbytes == s.tasks[y.key].nbytes
 
 
 @gen_cluster(client=True)
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 3231e6ce752..e6db76fdb26 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -146,27 +146,6 @@ def reset(self):
     assert tuple(results) == (3, 7)
 
 
-@pytest.mark.slow
-@gen_cluster()
-async def dont_test_delete_data_with_missing_worker(c, a, b):
-    bad = "127.0.0.1:9001"  # this worker doesn't exist
-    c.who_has["z"].add(bad)
-    c.who_has["z"].add(a.address)
-    c.has_what[bad].add("z")
-    c.has_what[a.address].add("z")
-    a.data["z"] = 5
-
-    cc = rpc(ip=c.ip, port=c.port)
-
-    await cc.delete_data(keys=["z"])  # TODO: this hangs for a while
-    assert "z" not in a.data
-    assert not c.who_has["z"]
-    assert not c.has_what[bad]
-    assert not c.has_what[a.address]
-
-    await cc.close_rpc()
-
-
 @gen_cluster(client=True)
 async def test_upload_file(c, s, a, b):
     assert not os.path.exists(os.path.join(a.local_directory, "foobar.py"))
@@ -402,16 +381,59 @@ def __str__(self):
         assert "Bar" in str(e.__cause__)
 
 
-@gen_cluster()
-async def test_gather(s, a, b):
-    b.data["x"] = 1
-    b.data["y"] = 2
+@gen_cluster(client=True)
+async def test_gather(c, s, a, b):
+    x, y = await c.scatter(["x", "y"], workers=[b.address])
+    with rpc(a.address) as aa:
+        resp = await aa.gather(who_has={x.key: [b.address], y.key: [b.address]})
+
+    assert resp == {"status": "OK"}
+    assert a.data[x.key] == b.data[x.key] == "x"
+    assert a.data[y.key] == b.data[y.key] == "y"
+
+
+@gen_cluster(client=True)
+async def test_gather_missing_keys(c, s, a, b):
+    """A key is missing. Other keys are gathered successfully."""
+    x = await c.scatter("x", workers=[b.address])
+    with rpc(a.address) as aa:
+        resp = await aa.gather(who_has={x.key: [b.address], "y": [b.address]})
+
+    assert resp == {"status": "partial-fail", "keys": {"y": (b.address,)}}
+    assert a.data[x.key] == b.data[x.key] == "x"
+
+
+@gen_cluster(client=True, worker_kwargs={"timeout": "100ms"})
+async def test_gather_missing_workers(c, s, a, b):
+    """A worker owning the only copy of a key is missing.
+    Keys from other workers are gathered successfully.
+    """
+    assert b.address.startswith("tcp://127.0.0.1:")
+    bad_addr = "tcp://127.0.0.1:12345"
+    x = await c.scatter("x", workers=[b.address])
+
     with rpc(a.address) as aa:
-        resp = await aa.gather(who_has={"x": [b.address], "y": [b.address]})
-        assert resp["status"] == "OK"
+        resp = await aa.gather(who_has={x.key: [b.address], "y": [bad_addr]})
+
+    assert resp == {"status": "partial-fail", "keys": {"y": (bad_addr,)}}
+    assert a.data[x.key] == b.data[x.key] == "x"
 
-        assert a.data["x"] == b.data["x"]
-        assert a.data["y"] == b.data["y"]
+
+@pytest.mark.parametrize("missing_first", [False, True])
+@gen_cluster(client=True, worker_kwargs={"timeout": "100ms"})
+async def test_gather_missing_workers_replicated(c, s, a, b, missing_first):
+    """A worker owning a redundant copy of a key is missing.
+    The key is successfully gathered from other workers.
+    """
+    assert b.address.startswith("tcp://127.0.0.1:")
+    x = await c.scatter("x", workers=[b.address])
+    bad_addr = "tcp://127.0.0.1:12345"
+    # Order matters! Test both
+    addrs = [bad_addr, b.address] if missing_first else [b.address, bad_addr]
+    with rpc(a.address) as aa:
+        resp = await aa.gather(who_has={x.key: addrs})
+    assert resp == {"status": "OK"}
+    assert a.data[x.key] == b.data[x.key] == "x"
 
 
 @pytest.mark.asyncio
diff --git a/distributed/worker.py b/distributed/worker.py
index 03c02a36c32..31a828ebddf 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -1080,6 +1080,7 @@ async def gather(self, comm=None, who_has=None):
         result, missing_keys, missing_workers = await gather_from_workers(
             who_has, rpc=self.rpc, who=self.address
         )
+        self.update_data(data=result, report=False)
         if missing_keys:
             logger.warning(
                 "Could not find data: %s on workers: %s (who_has: %s)",
@@ -1087,9 +1088,8 @@ async def gather(self, comm=None, who_has=None):
                 missing_workers,
                 who_has,
             )
-            return {"status": "missing-data", "keys": missing_keys}
+            return {"status": "partial-fail", "keys": missing_keys}
         else:
-            self.update_data(data=result, report=False)
             return {"status": "OK"}
 
     def get_monitor_info(self, comm=None, recent=False, start=0):
@@ -1500,9 +1500,9 @@ def handle_free_keys(self, comm=None, keys=None, reason=None):
         self.log.append(("free-keys", keys, reason))
         for key in keys:
             ts = self.tasks.get(key)
-            if ts:
+            if ts is not None:
                 ts.scheduler_holds_ref = False
-            self.release_key(key, report=False, reason=reason)
+                self.release_key(key, report=False, reason=reason)
 
     def handle_superfluous_data(self, keys=(), reason=None):
         """Stream handler notifying the worker that it might be holding unreferenced, superfluous data.
@@ -2625,12 +2625,7 @@ def release_key(
             if ts is None or ts.scheduler_holds_ref:
                 return
             logger.debug(
-                "Release key %s",
-                {
-                    "key": key,
-                    "cause": cause,
-                    "reason": reason,
-                },
+                "Release key %s", {"key": key, "cause": cause, "reason": reason}
             )
             if cause:
                 self.log.append((key, "release-key", {"cause": cause}, reason))

From 942f2355892117ed95723b909cbcc7a672c6b269 Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Thu, 15 Jul 2021 10:06:32 +0100
Subject: [PATCH 1369/1550] Disable pytest-timeout SIGALARM on MacOS (#5057)

* Disable pytest-timeout SIGALARM on MacOS

* Disable hanging test
---
 .github/workflows/tests.yaml        | 4 ++--
 distributed/tests/test_scheduler.py | 7 ++++---
 setup.cfg                           | 5 +++--
 3 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
index 8301b6713eb..84e6d8b75de 100644
--- a/.github/workflows/tests.yaml
+++ b/.github/workflows/tests.yaml
@@ -96,8 +96,8 @@ jobs:
 
       - name: Reconfigure pytest-timeout
         shell: bash -l {0}
-        # No SIGALRM available on Windows
-        if: ${{ matrix.os != 'windows-latest' }}
+        # No SIGALRM available on Windows. On MacOS, it kills the whole test suite.
+        if: ${{ matrix.os == 'ubuntu-latest' }}
         run: sed -i.bak 's/timeout_method = thread/timeout_method = signal/' setup.cfg
 
       - name: Test
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 353781e5e11..9527709a355 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -22,7 +22,7 @@
 
 from distributed import Client, Nanny, Worker, fire_and_forget, wait
 from distributed.comm import Comm
-from distributed.compatibility import LINUX, WINDOWS
+from distributed.compatibility import LINUX, MACOS, WINDOWS
 from distributed.core import ConnectionPool, Status, connect, rpc
 from distributed.metrics import time
 from distributed.protocol.pickle import dumps
@@ -3102,6 +3102,7 @@ async def test_transition_counter(c, s, a, b):
     assert s.transition_counter > 1
 
 
+@pytest.mark.skipif(MACOS and sys.version_info < (3, 9), reason="GH#5056")
 @pytest.mark.slow
 @gen_cluster(
     client=True,
@@ -3110,8 +3111,8 @@ async def test_transition_counter(c, s, a, b):
     timeout=60,
 )
 async def test_worker_heartbeat_after_cancel(c, s, *workers):
-    """This test is intended to ensure that after cancelation of a graph, the
-    worker heartbeat is always successful. The hearbeat may not be successful if
+    """This test is intended to ensure that after cancellation of a graph, the
+    worker heartbeat is always successful. The heartbeat may not be successful if
     the worker and scheduler state drift and the scheduler doesn't handle
     unknown information gracefully. One example would be a released/cancelled
     computation where the worker returns metrics about duration, type, etc. and
diff --git a/setup.cfg b/setup.cfg
index 2ef82cb04f7..c8646ecca9d 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -53,8 +53,9 @@ markers =
 
 # pytest-timeout settings
 # 'thread' kills off the whole test suite. 'signal' only kills the offending test.
-# However, 'signal' does not work on Windows. The CI script modifies this config file on
-# the fly on Linux and MacOS.
+# However, 'signal' doesn't work on Windows (due to lack of SIGALRM) and doesn't work on
+# the MacOS GitHub CI (although it's been reported to work on MacBooks).
+# The CI script modifies this config file on the fly on Linux.
 timeout_method = thread
 # This should not be reduced; Windows CI has been observed to be occasionally
 # exceptionally slow.

From 48ec3c5795c3f1bac74bb1ce908a310d0cdfa66b Mon Sep 17 00:00:00 2001
From: Ian Rose <ian.r.rose@gmail.com>
Date: Thu, 15 Jul 2021 16:44:10 +0200
Subject: [PATCH 1370/1550] Reduce default websocket frame size and make
 configurable (#5070)

Supersedes #5052 . In addition to making the default websocket maximum-frame-size smaller, this makes the specific value configurable. It's somewhat redundant with distributed.comm.shard, but the constraints on websockets are sufficiently different that a separate config seems okay.

This does not implement the fix in #5061, as that would read a config value for every frame, which is costly. So the config value will in general not be changed after import time.
---
 distributed/comm/utils.py                    | 11 +++++------
 distributed/comm/ws.py                       |  8 ++++++++
 distributed/distributed-schema.yaml          | 14 ++++++++++++++
 distributed/distributed.yaml                 |  3 +++
 distributed/protocol/core.py                 | 10 ++++++++--
 distributed/protocol/serialize.py            |  6 ++++--
 distributed/protocol/tests/test_serialize.py | 17 +++++++++++++++++
 distributed/protocol/utils.py                |  1 +
 8 files changed, 60 insertions(+), 10 deletions(-)

diff --git a/distributed/comm/utils.py b/distributed/comm/utils.py
index 5301265caf5..0ce4f8f891b 100644
--- a/distributed/comm/utils.py
+++ b/distributed/comm/utils.py
@@ -19,19 +19,18 @@
 
 
 async def to_frames(
-    msg, serializers=None, on_error="message", context=None, allow_offload=True
+    msg,
+    allow_offload=True,
+    **kwargs,
 ):
     """
     Serialize a message into a list of Distributed protocol frames.
+    Any kwargs are forwarded to protocol.dumps().
     """
 
     def _to_frames():
         try:
-            return list(
-                protocol.dumps(
-                    msg, serializers=serializers, on_error=on_error, context=context
-                )
-            )
+            return list(protocol.dumps(msg, **kwargs))
         except Exception as e:
             logger.info("Unserializable Message: %s", msg)
             logger.exception(e)
diff --git a/distributed/comm/ws.py b/distributed/comm/ws.py
index be7679513aa..8a64f209dbd 100644
--- a/distributed/comm/ws.py
+++ b/distributed/comm/ws.py
@@ -12,6 +12,8 @@
 from tornado.iostream import StreamClosedError
 from tornado.websocket import WebSocketClosedError, WebSocketHandler, websocket_connect
 
+import dask
+
 from ..utils import ensure_bytes, nbytes
 from .addressing import parse_host_port, unparse_host_port
 from .core import Comm, CommClosedError, Connector, FatalCommClosedError, Listener
@@ -22,6 +24,11 @@
 logger = logging.getLogger(__name__)
 
 
+BIG_BYTES_SHARD_SIZE = dask.utils.parse_bytes(
+    dask.config.get("distributed.comm.websockets.shard")
+)
+
+
 class WSHandler(WebSocketHandler):
     def __init__(
         self,
@@ -106,6 +113,7 @@ async def write(self, msg, serializers=None, on_error=None):
                 "recipient": self.remote_info,
                 **self.handshake_options,
             },
+            frame_split_size=BIG_BYTES_SHARD_SIZE,
         )
         n = struct.pack("Q", len(frames))
         try:
diff --git a/distributed/distributed-schema.yaml b/distributed/distributed-schema.yaml
index 440a39fb2f0..f0beee814e3 100644
--- a/distributed/distributed-schema.yaml
+++ b/distributed/distributed-schema.yaml
@@ -759,6 +759,20 @@ properties:
                       Alternatively, the key can be appended to the cert file
                       above, and this field left blank
 
+          websockets:
+            type: object
+            properties:
+              shard:
+                type:
+                - string
+                description: |
+                  The maximum size of a websocket frame to send through a comm.
+
+                  This is somewhat duplicative of distributed.comm.shard, but websockets
+                  often have much smaller maximum message sizes than othe protocols, so
+                  this attribute is used to set a smaller default shard size and to
+                  allow separate control of websocket message sharding.
+
       diagnostics:
         type: object
         properties:
diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index 28ca8cf7bf5..86e2e5489d6 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -196,6 +196,9 @@ distributed:
         key: null
         cert: null
 
+    websockets:
+      shard: 8MiB
+
   diagnostics:
     nvml: True
 
diff --git a/distributed/protocol/core.py b/distributed/protocol/core.py
index c4bd909ecb3..1be2d761e35 100644
--- a/distributed/protocol/core.py
+++ b/distributed/protocol/core.py
@@ -16,7 +16,9 @@
 logger = logging.getLogger(__name__)
 
 
-def dumps(msg, serializers=None, on_error="message", context=None) -> list:
+def dumps(
+    msg, serializers=None, on_error="message", context=None, frame_split_size=None
+) -> list:
     """Transform Python message to bytestream suitable for communication
 
     Developer Notes
@@ -53,7 +55,11 @@ def _encode_default(obj):
                     sub_header, sub_frames = obj.header, obj.frames
                 else:
                     sub_header, sub_frames = serialize_and_split(
-                        obj, serializers=serializers, on_error=on_error, context=context
+                        obj,
+                        serializers=serializers,
+                        on_error=on_error,
+                        context=context,
+                        size=frame_split_size,
                     )
                     _inplace_compress_frames(sub_header, sub_frames)
                 sub_header["num-sub-frames"] = len(sub_frames)
diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index c88d36a8995..51815677f5a 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -406,7 +406,9 @@ def deserialize(header, frames, deserializers=None):
     return loads(header, frames)
 
 
-def serialize_and_split(x, serializers=None, on_error="message", context=None):
+def serialize_and_split(
+    x, serializers=None, on_error="message", context=None, size=None
+):
     """Serialize and split compressable frames
 
     This function is a drop-in replacement of `serialize()` that calls `serialize()`
@@ -428,7 +430,7 @@ def serialize_and_split(x, serializers=None, on_error="message", context=None):
         frames, header.get("compression") or [None] * len(frames)
     ):
         if compression is None:  # default behavior
-            sub_frames = frame_split_size(frame)
+            sub_frames = frame_split_size(frame, n=size)
             num_sub_frames.append(len(sub_frames))
             offsets.append(len(out_frames))
             out_frames.extend(sub_frames)
diff --git a/distributed/protocol/tests/test_serialize.py b/distributed/protocol/tests/test_serialize.py
index d946b01496b..36359304830 100644
--- a/distributed/protocol/tests/test_serialize.py
+++ b/distributed/protocol/tests/test_serialize.py
@@ -11,6 +11,8 @@
 except ImportError:
     np = None
 
+import dask
+
 from distributed import Nanny, wait
 from distributed.comm.utils import from_frames, to_frames
 from distributed.protocol import (
@@ -442,6 +444,21 @@ def _(x):
     assert header["compression"] == [False, False]
 
 
+@gen_test()
+async def test_frame_split():
+    data = b"1234abcd" * (2 ** 20)  # 8 MiB
+    assert dask.sizeof.sizeof(data) == dask.utils.parse_bytes("8MiB")
+
+    size = dask.utils.parse_bytes("3MiB")
+    split_frames = await to_frames({"x": to_serialize(data)}, frame_split_size=size)
+    print(split_frames)
+    assert len(split_frames) == 3 + 2  # Three splits and two headers
+
+    size = dask.utils.parse_bytes("5MiB")
+    split_frames = await to_frames({"x": to_serialize(data)}, frame_split_size=size)
+    assert len(split_frames) == 2 + 2  # Two splits and two headers
+
+
 @pytest.mark.parametrize(
     "data,is_serializable",
     [
diff --git a/distributed/protocol/utils.py b/distributed/protocol/utils.py
index 3f5a2f8f500..cf4f1815ea5 100644
--- a/distributed/protocol/utils.py
+++ b/distributed/protocol/utils.py
@@ -25,6 +25,7 @@ def frame_split_size(frame, n=BIG_BYTES_SHARD_SIZE) -> list:
     >>> frame_split_size([b'12345', b'678'], n=3)  # doctest: +SKIP
     [b'123', b'45', b'678']
     """
+    n = n or BIG_BYTES_SHARD_SIZE
     frame = memoryview(frame)
 
     if frame.nbytes <= n:

From a92d92ba0083091bd8f302749463ee9017fb3de9 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 15 Jul 2021 07:54:25 -0700
Subject: [PATCH 1371/1550] Avoid property validation in Bokeh (#5065)

I'm not sure why this additional check is necessary, but it appears to help

Fixes #5064
---
 distributed/dashboard/components/__init__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/distributed/dashboard/components/__init__.py b/distributed/dashboard/components/__init__.py
index f0798417cd9..9cba35f3010 100644
--- a/distributed/dashboard/components/__init__.py
+++ b/distributed/dashboard/components/__init__.py
@@ -79,6 +79,7 @@ def add_periodic_callback(doc, component, interval):
     _attach(doc, component)
 
 
+@without_property_validation
 def update(ref):
     comp = ref()
     if comp is not None:

From 4734833cc08f6c22b2202723e3a6be3bc5b7c2bc Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Thu, 15 Jul 2021 17:48:19 +0200
Subject: [PATCH 1372/1550] Simplify test_worker_heartbeat_after_cancel (#5067)

---
 distributed/tests/test_scheduler.py | 23 +++--------------------
 1 file changed, 3 insertions(+), 20 deletions(-)

diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 9527709a355..6a151091787 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -22,7 +22,7 @@
 
 from distributed import Client, Nanny, Worker, fire_and_forget, wait
 from distributed.comm import Comm
-from distributed.compatibility import LINUX, MACOS, WINDOWS
+from distributed.compatibility import LINUX, WINDOWS
 from distributed.core import ConnectionPool, Status, connect, rpc
 from distributed.metrics import time
 from distributed.protocol.pickle import dumps
@@ -3102,13 +3102,9 @@ async def test_transition_counter(c, s, a, b):
     assert s.transition_counter > 1
 
 
-@pytest.mark.skipif(MACOS and sys.version_info < (3, 9), reason="GH#5056")
-@pytest.mark.slow
 @gen_cluster(
     client=True,
     nthreads=[("127.0.0.1", 1) for _ in range(10)],
-    # typical runtime just 2-3s but on CI this may increase significantly
-    timeout=60,
 )
 async def test_worker_heartbeat_after_cancel(c, s, *workers):
     """This test is intended to ensure that after cancellation of a graph, the
@@ -3118,30 +3114,17 @@ async def test_worker_heartbeat_after_cancel(c, s, *workers):
     computation where the worker returns metrics about duration, type, etc. and
     the scheduler doesn't handle the forgotten task gracefully.
 
-    Failures are not triggered reliably since the race conditions for this error
-    case are very hard to produce. Likelihood of failure increases with the
-    number of workers.
-
     See also https://github.com/dask/distributed/issues/4587
     """
-    da = pytest.importorskip("dask.array")
     for w in workers:
         w.periodic_callbacks["heartbeat"].stop()
-    x = da.random.random((2000000, 100), chunks=(10000, None))
-    svd = da.linalg.svd(x)
 
-    futs = c.compute(svd)
+    futs = c.map(slowinc, range(100), delay=0.1)
 
-    while not s.tasks:
-        await asyncio.sleep(0.001)
-
-    while sum(w.executing_count for w in workers) < len(workers) / 2:
+    while sum(w.executing_count for w in workers) < len(workers):
         await asyncio.sleep(0.001)
 
     await c.cancel(futs)
 
-    while s.tasks:
-        await asyncio.sleep(0.001)
-
     while any(w.tasks for w in workers):
         await asyncio.gather(*[w.heartbeat() for w in workers])

From ea2b643edcedc171acea0484a39e3a29512b3fb7 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 15 Jul 2021 13:09:21 -0700
Subject: [PATCH 1373/1550] Better support ProcessPoolExecutors (#5063)

Previously we sent non-serializable state, like locks and the worker
itself into the function that was sent to the executor

Now we only do this if we think tha the executor is a
ThreadPoolExecutor

* Remove Worker.executor_submit

Fixes https://github.com/dask/distributed/issues/3938

This is old and was a workaround for a Tornado issue

* remove gen.coroutine from worker.py

We're generally trying to clean this up from the codebase

* add Worker.active_keys
---
 distributed/tests/test_worker.py |  45 ++++++++---
 distributed/worker.py            | 127 ++++++++++++++++---------------
 2 files changed, 99 insertions(+), 73 deletions(-)

diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index e6db76fdb26..e5e877a2a98 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -5,7 +5,7 @@
 import sys
 import threading
 import traceback
-from concurrent.futures import ThreadPoolExecutor
+from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
 from numbers import Number
 from operator import add
 from time import sleep
@@ -508,17 +508,14 @@ async def f(dask_worker=None):
 @gen_cluster(client=True, nthreads=[])
 async def test_Executor(c, s):
     with ThreadPoolExecutor(2) as e:
-        w = Worker(s.address, executor=e)
-        assert w.executor is e
-        w = await w
+        async with Worker(s.address, executor=e) as w:
+            assert w.executor is e
 
-        future = c.submit(inc, 1)
-        result = await future
-        assert result == 2
-
-        assert e._threads  # had to do some work
+            future = c.submit(inc, 1)
+            result = await future
+            assert result == 2
 
-        await w.close()
+            assert e._threads  # had to do some work
 
 
 @pytest.mark.skip(
@@ -2028,6 +2025,34 @@ def get_thread_name():
                 assert "Dask-GPU-Threads" in gpu_result
 
 
+@gen_cluster(client=True)
+async def test_process_executor(c, s, a, b):
+    with ProcessPoolExecutor() as e:
+        a.executors["processes"] = e
+        b.executors["processes"] = e
+
+        future = c.submit(os.getpid, pure=False)
+        assert (await future) == os.getpid()
+
+        with dask.annotate(executor="processes"):
+            future = c.submit(os.getpid, pure=False)
+
+        assert (await future) != os.getpid()
+
+
+@gen_cluster(client=True)
+async def test_process_executor_kills_process(c, s, a, b):
+    with ProcessPoolExecutor() as e:
+        a.executors["processes"] = e
+        b.executors["processes"] = e
+
+        with dask.annotate(executor="processes", retries=1):
+            future = c.submit(sys.exit, 1)
+
+        exc = await future.exception()
+        assert "SystemExit(1)" in repr(exc)
+
+
 def assert_task_states_on_worker(expected, worker):
     for dep_key, expected_state in expected.items():
         assert dep_key in worker.tasks, (worker.name, dep_key, worker.tasks)
diff --git a/distributed/worker.py b/distributed/worker.py
index 31a828ebddf..95d6a116e63 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -19,7 +19,6 @@
 from typing import Dict, Iterable, Optional
 
 from tlz import first, keymap, merge, pluck  # noqa: F401
-from tornado import gen
 from tornado.ioloop import IOLoop, PeriodicCallback
 
 import dask
@@ -438,6 +437,7 @@ def __init__(
 
         self.active_threads_lock = threading.Lock()
         self.active_threads = dict()
+        self.active_keys = set()
         self.profile_keys = defaultdict(profile.create)
         self.profile_keys_history = deque(maxlen=3600)
         self.profile_recent = profile.create()
@@ -968,8 +968,6 @@ async def heartbeat(self):
         logger.debug("Heartbeat: %s", self.address)
         try:
             start = time()
-            with self.active_threads_lock:
-                active_keys = list(self.active_threads.values())
             response = await retry_operation(
                 self.scheduler.heartbeat_worker,
                 address=self.contact_address,
@@ -977,7 +975,7 @@ async def heartbeat(self):
                 metrics=await self.get_metrics(),
                 executing={
                     key: start - self.tasks[key].start_time
-                    for key in active_keys
+                    for key in self.active_keys
                     if key in self.tasks
                 },
             )
@@ -2686,41 +2684,6 @@ def release_key(
     # Execute Task #
     ################
 
-    # FIXME: this breaks if changed to async def...
-    # xref: https://github.com/dask/distributed/issues/3938
-    @gen.coroutine
-    def executor_submit(self, key, function, args=(), kwargs=None, executor=None):
-        """Safely run function in thread pool executor
-
-        We've run into issues running concurrent.future futures within
-        tornado.  Apparently it's advantageous to use timeouts and periodic
-        callbacks to ensure things run smoothly.  This can get tricky, so we
-        pull it off into an separate method.
-        """
-        executor = executor or self.executors["default"]
-        job_counter[0] += 1
-        # logger.info("%s:%d Starts job %d, %s", self.ip, self.port, i, key)
-        kwargs = kwargs or {}
-        future = executor.submit(function, *args, **kwargs)
-        pc = PeriodicCallback(
-            lambda: logger.debug("future state: %s - %s", key, future._state), 1000
-        )
-        ts = self.tasks.get(key)
-        if ts is not None:
-            ts.start_time = time()
-        pc.start()
-        try:
-            yield future
-        finally:
-            pc.stop()
-            if ts is not None:
-                ts.stop_time = time()
-
-        result = future.result()
-
-        # logger.info("Finish job %d, %s", i, key)
-        raise gen.Return(result)
-
     def run(self, comm, function, args=(), wait=True, kwargs=None):
         return run(self, comm, function=function, args=args, kwargs=kwargs, wait=wait)
 
@@ -2782,19 +2745,16 @@ async def actor_execute(
             if iscoroutinefunction(func):
                 result = await func(*args, **kwargs)
             elif separate_thread:
-                result = await self.executor_submit(
-                    name,
+                result = await self.loop.run_in_executor(
+                    self.executors["actor"],
                     apply_function_actor,
-                    args=(
-                        func,
-                        args,
-                        kwargs,
-                        self.execution_state,
-                        name,
-                        self.active_threads,
-                        self.active_threads_lock,
-                    ),
-                    executor=self.executors["actor"],
+                    func,
+                    args,
+                    kwargs,
+                    self.execution_state,
+                    name,
+                    self.active_threads,
+                    self.active_threads_lock,
                 )
             else:
                 result = func(*args, **kwargs)
@@ -2946,11 +2906,14 @@ async def execute(self, key, report=False):
                 executor,
             )  # TODO: comment out?
             assert key == ts.key
+            self.active_keys.add(ts.key)
             try:
-                result = await self.executor_submit(
-                    ts.key,
-                    apply_function,
-                    args=(
+                e = self.executors[executor]
+                ts.start_time = time()
+                if "ThreadPoolExecutor" in str(type(e)):
+                    result = await self.loop.run_in_executor(
+                        e,
+                        apply_function,
                         function,
                         args2,
                         kwargs2,
@@ -2959,12 +2922,32 @@ async def execute(self, key, report=False):
                         self.active_threads,
                         self.active_threads_lock,
                         self.scheduler_delay,
-                    ),
-                    executor=self.executors[executor],
-                )
+                    )
+                else:
+                    try:
+                        start = time() + self.scheduler_delay
+                        result = await self.loop.run_in_executor(
+                            e,
+                            apply_function_simple,
+                            function,
+                            args2,
+                            kwargs2,
+                            self.scheduler_delay,
+                        )
+                    except BaseException as e:
+                        msg = error_message(e)
+                        msg["op"] = "task-erred"
+                        msg["actual-exception"] = e
+                        msg["start"] = start
+                        msg["stop"] = time() + self.scheduler_delay
+                        msg["thread"] = None
+                        result = msg
+
             except RuntimeError as e:
                 executor_error = e
                 raise
+            finally:
+                self.active_keys.discard(ts.key)
 
             # We'll need to check again for the task state since it may have
             # changed since the execution was kicked off. In particular, it may
@@ -3854,6 +3837,27 @@ def apply_function(
     thread_state.start_time = time()
     thread_state.execution_state = execution_state
     thread_state.key = key
+
+    msg = apply_function_simple(function, args, kwargs, time_delay)
+
+    with active_threads_lock:
+        del active_threads[ident]
+    return msg
+
+
+def apply_function_simple(
+    function,
+    args,
+    kwargs,
+    time_delay,
+):
+    """Run a function, collect information
+
+    Returns
+    -------
+    msg: dictionary with status, result/error, timings, etc..
+    """
+    ident = threading.get_ident()
     start = time()
     try:
         result = function(*args, **kwargs)
@@ -3874,8 +3878,6 @@ def apply_function(
     msg["start"] = start + time_delay
     msg["stop"] = end + time_delay
     msg["thread"] = ident
-    with active_threads_lock:
-        del active_threads[ident]
     return msg
 
 
@@ -4020,9 +4022,8 @@ async def run(server, comm, function, args=(), kwargs=None, is_coro=None, wait=T
     pass
 else:
 
-    @gen.coroutine
-    def gpu_metric(worker):
-        result = yield offload(nvml.real_time)
+    async def gpu_metric(worker):
+        result = await offload(nvml.real_time)
         return result
 
     DEFAULT_METRICS["gpu"] = gpu_metric

From bd5367b925d25dc9c6ef0700294036b447a1839d Mon Sep 17 00:00:00 2001
From: Peter Andreas Entschev <peter@entschev.com>
Date: Fri, 16 Jul 2021 03:22:49 +0200
Subject: [PATCH 1374/1550] Reenable UCX-Py tests that used to segfault (#5076)

---
 distributed/comm/tests/test_ucx.py | 14 ++------------
 1 file changed, 2 insertions(+), 12 deletions(-)

diff --git a/distributed/comm/tests/test_ucx.py b/distributed/comm/tests/test_ucx.py
index de634b571d1..64948b68028 100644
--- a/distributed/comm/tests/test_ucx.py
+++ b/distributed/comm/tests/test_ucx.py
@@ -167,21 +167,11 @@ async def test_ucx_deserialize():
         lambda cudf: cudf.DataFrame([1]).head(0),
         lambda cudf: cudf.DataFrame([1.0]).head(0),
         lambda cudf: cudf.DataFrame({"a": []}),
-        pytest.param(
-            lambda cudf: cudf.DataFrame({"a": ["a"]}).head(0),
-            marks=pytest.mark.skip(
-                reason="This test segfaults for some reason. So skip running it entirely."
-            ),
-        ),
+        lambda cudf: cudf.DataFrame({"a": ["a"]}).head(0),
         lambda cudf: cudf.DataFrame({"a": [1.0]}).head(0),
         lambda cudf: cudf.DataFrame({"a": [1]}).head(0),
         lambda cudf: cudf.DataFrame({"a": [1, 2, None], "b": [1.0, 2.0, None]}),
-        pytest.param(
-            lambda cudf: cudf.DataFrame({"a": ["Check", "str"], "b": ["Sup", "port"]}),
-            marks=pytest.mark.skip(
-                reason="This test segfaults for some reason. So skip running it entirely."
-            ),
-        ),
+        lambda cudf: cudf.DataFrame({"a": ["Check", "str"], "b": ["Sup", "port"]}),
     ],
 )
 async def test_ping_pong_cudf(g):

From 30ad02b85d0cd5ab58d8bfa2649515c668298c3c Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 19 Jul 2021 01:33:34 -0700
Subject: [PATCH 1375/1550] Verify that actors survive pickling (#5086)

---
 distributed/tests/test_actor.py | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/distributed/tests/test_actor.py b/distributed/tests/test_actor.py
index 92d55425f55..379546c1869 100644
--- a/distributed/tests/test_actor.py
+++ b/distributed/tests/test_actor.py
@@ -6,7 +6,7 @@
 
 import dask
 
-from distributed import Actor, ActorFuture, Client, Future, Nanny, wait
+from distributed import Actor, ActorFuture, Client, Future, Nanny, get_client, wait
 from distributed.metrics import time
 from distributed.utils_test import cluster, gen_cluster
 
@@ -624,3 +624,20 @@ def prop(self):
 
     with pytest.raises(MyException):
         await ac.prop
+
+
+@gen_cluster(client=True)
+async def test_serialize_with_pickle(c, s, a, b):
+    class Foo:
+        def __init__(self):
+            self.actor = get_client().submit(Counter, actor=True).result()
+
+        def __getstate__(self):
+            return self.actor
+
+        def __setstate__(self, state):
+            self.actor = state
+
+    future = c.submit(Foo, workers=a.address)
+    foo = await future
+    assert isinstance(foo.actor, Actor)

From cb18bd1d9d2f14678cdd66a0b816d9b6c8d16f19 Mon Sep 17 00:00:00 2001
From: Julia Signell <jsignell@gmail.com>
Date: Tue, 20 Jul 2021 04:30:39 -0400
Subject: [PATCH 1376/1550] Collapse nav to hamburger sooner (#5094)

---
 distributed/http/static/css/base.css | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/distributed/http/static/css/base.css b/distributed/http/static/css/base.css
index 211cc067d67..0cc3583a15a 100644
--- a/distributed/http/static/css/base.css
+++ b/distributed/http/static/css/base.css
@@ -73,7 +73,7 @@ body {
   height: 22px;
 }
 
-@media screen and (max-width: 650px) {
+@media screen and (max-width: 800px) {
   .navbar li:not(#dask-logo):not(#navbar-toggle-icon) a {
     display: none;
   }
@@ -82,7 +82,7 @@ body {
   }
 }
 
-@media screen and (max-width: 650px) {
+@media screen and (max-width: 800px) {
   .navbar.responsive li:not(#navbar-toggle-icon) {
     float: none;
   }
@@ -128,4 +128,4 @@ body {
 
 .dropdown:hover .dropdown-content {
   display: block;
-}
\ No newline at end of file
+}

From f28c719962f36976dc514a50b9cbb1ce62d5b704 Mon Sep 17 00:00:00 2001
From: Gabe Joseph <gjoseph92@gmail.com>
Date: Tue, 20 Jul 2021 06:21:30 -0800
Subject: [PATCH 1377/1550] Actor: don't hold key references on workers (#4937)

Fixes #4936

When constructing an Actor handle, if there is a current worker, make our Future a weakref.
---
 distributed/actor.py            | 14 ++++++---
 distributed/tests/test_actor.py | 55 +++++++++++++++++++++++++++++++++
 distributed/worker.py           |  2 +-
 3 files changed, 65 insertions(+), 6 deletions(-)

diff --git a/distributed/actor.py b/distributed/actor.py
index 2ebbba53a1c..19828281dc3 100644
--- a/distributed/actor.py
+++ b/distributed/actor.py
@@ -3,11 +3,11 @@
 import threading
 from queue import Queue
 
-from .client import Future, default_client
+from .client import Future
 from .protocol import to_serialize
 from .utils import iscoroutinefunction, sync, thread_state
 from .utils_comm import WrappedKey
-from .worker import get_worker
+from .worker import get_client, get_worker
 
 
 class Actor(WrappedKey):
@@ -59,12 +59,15 @@ def __init__(self, cls, address, key, worker=None):
             self._client = None
         else:
             try:
+                # TODO: `get_worker` may return the wrong worker instance for async local clusters (most tests)
+                # when run outside of a task (when deserializing a key pointing to an Actor, etc.)
                 self._worker = get_worker()
             except ValueError:
                 self._worker = None
             try:
-                self._client = default_client()
-                self._future = Future(key)
+                self._client = get_client()
+                self._future = Future(key, inform=self._worker is None)
+                # ^ When running on a worker, only hold a weak reference to the key, otherwise the key could become unreleasable.
             except ValueError:
                 self._client = None
 
@@ -109,7 +112,8 @@ def _sync(self, func, *args, **kwargs):
         if self._client:
             return self._client.sync(func, *args, **kwargs)
         else:
-            # TODO support sync operation by checking against thread ident of loop
+            if self._asynchronous:
+                return func(*args, **kwargs)
             return sync(self._worker.loop, func, *args, **kwargs)
 
     def __dir__(self):
diff --git a/distributed/tests/test_actor.py b/distributed/tests/test_actor.py
index 379546c1869..d529edcc985 100644
--- a/distributed/tests/test_actor.py
+++ b/distributed/tests/test_actor.py
@@ -564,6 +564,61 @@ async def wait(self):
     await c.gather(futures)
 
 
+@gen_cluster(client=True, client_kwargs=dict(set_as_default=False))
+# ^ NOTE: without `set_as_default=False`, `get_client()` within worker would return
+# the same client instance the test is using (because it's all one process).
+# Even with this, both workers will share the same client instance.
+async def test_worker_actor_handle_is_weakref(c, s, a, b):
+    counter = c.submit(Counter, actor=True, workers=[a.address])
+
+    await c.submit(lambda _: None, counter, workers=[b.address])
+
+    del counter
+
+    start = time()
+    while a.actors or b.data:
+        await asyncio.sleep(0.1)
+        assert time() < start + 30
+
+
+def test_worker_actor_handle_is_weakref_sync(client):
+    workers = list(client.run(lambda: None))
+    counter = client.submit(Counter, actor=True, workers=[workers[0]])
+
+    client.submit(lambda _: None, counter, workers=[workers[1]]).result()
+
+    del counter
+
+    def check(dask_worker):
+        return len(dask_worker.data) + len(dask_worker.actors)
+
+    start = time()
+    while any(client.run(check).values()):
+        sleep(0.01)
+        assert time() < start + 30
+
+
+def test_worker_actor_handle_is_weakref_from_compute_sync(client):
+    workers = list(client.run(lambda: None))
+
+    with dask.annotate(workers=workers[0]):
+        counter = dask.delayed(Counter)()
+    with dask.annotate(workers=workers[1]):
+        intermediate = dask.delayed(lambda c: None)(counter)
+    with dask.annotate(workers=workers[0]):
+        final = dask.delayed(lambda x, c: x)(intermediate, counter)
+
+    final.compute(actors=counter, optimize_graph=False)
+
+    def worker_tasks_running(dask_worker):
+        return len(dask_worker.data) + len(dask_worker.actors)
+
+    start = time()
+    while any(client.run(worker_tasks_running).values()):
+        sleep(0.01)
+        assert time() < start + 30
+
+
 def test_one_thread_deadlock():
     with cluster(nworkers=2) as (cl, w):
         client = Client(cl["address"])
diff --git a/distributed/worker.py b/distributed/worker.py
index 95d6a116e63..04bc32e2e95 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -1412,7 +1412,7 @@ async def get_data(
                 if k in self.actors:
                     from .actor import Actor
 
-                    data[k] = Actor(type(self.actors[k]), self.address, k)
+                    data[k] = Actor(type(self.actors[k]), self.address, k, worker=self)
 
         msg = {"status": "OK", "data": {k: to_serialize(v) for k, v in data.items()}}
         nbytes = {k: self.tasks[k].nbytes for k in data if k in self.tasks}

From 9975baa54b5d2079891e93f57ed8e0f402ef89d8 Mon Sep 17 00:00:00 2001
From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com>
Date: Tue, 20 Jul 2021 19:27:44 -0400
Subject: [PATCH 1378/1550] Add pytest marker for GPU tests (#5023)

---
 distributed/diagnostics/tests/test_nvml.py         | 2 ++
 distributed/protocol/tests/test_collection_cuda.py | 2 ++
 distributed/protocol/tests/test_cupy.py            | 2 ++
 distributed/protocol/tests/test_numba.py           | 2 ++
 distributed/protocol/tests/test_rmm.py             | 2 ++
 distributed/tests/test_nanny.py                    | 3 +++
 setup.cfg                                          | 1 +
 7 files changed, 14 insertions(+)

diff --git a/distributed/diagnostics/tests/test_nvml.py b/distributed/diagnostics/tests/test_nvml.py
index 1dcdf353dbd..b669eef4035 100644
--- a/distributed/diagnostics/tests/test_nvml.py
+++ b/distributed/diagnostics/tests/test_nvml.py
@@ -2,6 +2,8 @@
 
 import pytest
 
+pytestmark = pytest.mark.gpu
+
 pynvml = pytest.importorskip("pynvml")
 
 import dask
diff --git a/distributed/protocol/tests/test_collection_cuda.py b/distributed/protocol/tests/test_collection_cuda.py
index a50fb7e2bb8..22be1c66a08 100644
--- a/distributed/protocol/tests/test_collection_cuda.py
+++ b/distributed/protocol/tests/test_collection_cuda.py
@@ -1,5 +1,7 @@
 import pytest
 
+pytestmark = pytest.mark.gpu
+
 from dask.dataframe.utils import assert_eq
 
 from distributed.protocol import deserialize, serialize
diff --git a/distributed/protocol/tests/test_cupy.py b/distributed/protocol/tests/test_cupy.py
index 5c684e46d62..ec813cba616 100644
--- a/distributed/protocol/tests/test_cupy.py
+++ b/distributed/protocol/tests/test_cupy.py
@@ -2,6 +2,8 @@
 
 import pytest
 
+pytestmark = pytest.mark.gpu
+
 from distributed.protocol import deserialize, serialize
 
 cupy = pytest.importorskip("cupy")
diff --git a/distributed/protocol/tests/test_numba.py b/distributed/protocol/tests/test_numba.py
index b34d4be25ab..ab91465e164 100644
--- a/distributed/protocol/tests/test_numba.py
+++ b/distributed/protocol/tests/test_numba.py
@@ -2,6 +2,8 @@
 
 import pytest
 
+pytestmark = pytest.mark.gpu
+
 from distributed.protocol import deserialize, serialize
 
 cuda = pytest.importorskip("numba.cuda")
diff --git a/distributed/protocol/tests/test_rmm.py b/distributed/protocol/tests/test_rmm.py
index bd9d7f4cab9..528b9460ef1 100644
--- a/distributed/protocol/tests/test_rmm.py
+++ b/distributed/protocol/tests/test_rmm.py
@@ -1,5 +1,7 @@
 import pytest
 
+pytestmark = pytest.mark.gpu
+
 from distributed.protocol import deserialize, serialize
 
 numpy = pytest.importorskip("numpy")
diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
index ca854754192..b23d6b1c994 100644
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -9,6 +9,9 @@
 
 import psutil
 import pytest
+
+pytestmark = pytest.mark.gpu
+
 from tlz import first, valmap
 from tornado.ioloop import IOLoop
 
diff --git a/setup.cfg b/setup.cfg
index c8646ecca9d..62f852ce307 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -50,6 +50,7 @@ markers =
     slow: marks tests as slow (deselect with '-m "not slow"')
     avoid_ci: marks tests as flaky on CI on all OSs
     ipython: marks tests as exercising IPython
+    gpu: marks tests we want to run on GPUs
 
 # pytest-timeout settings
 # 'thread' kills off the whole test suite. 'signal' only kills the offending test.

From cbf8ef0e0dadcec4b82022dd36a0aa6ce13a2633 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Wed, 21 Jul 2021 15:08:37 -0500
Subject: [PATCH 1379/1550] Don't package tests (#5054)

---
 MANIFEST.in | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/MANIFEST.in b/MANIFEST.in
index 66893de2b73..8a959c45673 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -8,15 +8,14 @@ recursive-include distributed *.svg
 recursive-include distributed *.ico
 recursive-include distributed *.yaml
 recursive-include docs *.rst
+recursive-exclude distributed **/tests/*
 
 include setup.py
 include README.rst
 include LICENSE.txt
 include MANIFEST.in
 include requirements.txt
-include distributed/tests/testegg-1.0.0-py3.4.egg
-include distributed/tests/mytest.pyz
-include distributed/tests/*.pem
+exclude distributed/pytest_resourceleaks.py
 
 prune docs/_build
 include versioneer.py

From 2f7392cd9e041eaf66ce6b3e45391524feb9a93e Mon Sep 17 00:00:00 2001
From: Bryan Van de Ven <bryan@bokeh.org>
Date: Wed, 21 Jul 2021 16:43:24 -0700
Subject: [PATCH 1380/1550] Some light dashboard cleanup (#5102)

I thought it was time I familiarized myself with Dask's use of Bokeh. Cleaning up code is one way I approach learning new codebase, so here are some light suggested changes. The commits are pretty cleanly split up, so I am happy to :

* drop any commits you don't want
* split this up into smaller PRs instead

All tests pass and a light manual testing looks OK but certainly it would be good for others to put things though paces.

Summary:

#### Bokeh specific
  * Make Bokeh 1.0 the minimum supported version. This is truly the lowest reasonable Bokeh version anyone should even *consider* supporting. I will plan to make the case that Dask should set even higher, but I will do that in a separate issue when I can collect some downloads and/or CDN data to present
  * Removes the Bokeh logo once via the theme, instead of individually in many places with code
  * Deletes obsolete coffeescript cruft
  * Streamline doc creation with a decorator to avoid repeated boilerplate
#### General
  * removes some unused imports, functions
  * drop unnecessary calls to `list`
  * fixes cases of "if-return-else"

I'd like to see about offloading more explicit styling code into the theme, add additional tests, and make more minor improvements but will save those for another PR.
---
 distributed/dashboard/components/__init__.py  |  43 +----
 distributed/dashboard/components/nvml.py      |   4 +-
 distributed/dashboard/components/scheduler.py |  34 +---
 distributed/dashboard/components/shared.py    |  90 +++-------
 distributed/dashboard/components/worker.py    | 161 ++++++++----------
 distributed/dashboard/core.py                 |  27 ++-
 distributed/dashboard/export_tool.coffee      |  44 -----
 distributed/dashboard/scheduler.py            |   5 -
 distributed/dashboard/theme.yaml              |   2 +
 distributed/dashboard/utils.py                |  24 +--
 10 files changed, 116 insertions(+), 318 deletions(-)
 delete mode 100644 distributed/dashboard/export_tool.coffee

diff --git a/distributed/dashboard/components/__init__.py b/distributed/dashboard/components/__init__.py
index 9cba35f3010..f4ddb418049 100644
--- a/distributed/dashboard/components/__init__.py
+++ b/distributed/dashboard/components/__init__.py
@@ -1,45 +1,6 @@
-import asyncio
 import weakref
-from bisect import bisect
-from operator import add
-from time import time
-
-from bokeh.layouts import column, row
-from bokeh.models import (
-    BoxZoomTool,
-    Button,
-    ColumnDataSource,
-    DataRange1d,
-    HoverTool,
-    LinearAxis,
-    OpenURL,
-    PanTool,
-    Plot,
-    Quad,
-    Range1d,
-    ResetTool,
-    Select,
-    TapTool,
-    WheelZoomTool,
-)
-from bokeh.palettes import Spectral9
-from bokeh.plotting import figure
-from tornado import gen
-
-import dask
-
-from distributed import profile
-from distributed.dashboard.utils import BOKEH_VERSION, without_property_validation
-from distributed.utils import log_errors
-
-if dask.config.get("distributed.dashboard.export-tool"):
-    from distributed.dashboard.export_tool import ExportTool
-else:
-    ExportTool = None
-
-
-profile_interval = dask.config.get("distributed.worker.profile.interval")
-profile_interval = dask.utils.parse_timedelta(profile_interval, default="ms")
+
+from bokeh.core.properties import without_property_validation
 
 
 class DashboardComponent:
diff --git a/distributed/dashboard/components/nvml.py b/distributed/dashboard/components/nvml.py
index cdb331016c6..bed8f2974bd 100644
--- a/distributed/dashboard/components/nvml.py
+++ b/distributed/dashboard/components/nvml.py
@@ -1,5 +1,6 @@
 import math
 
+from bokeh.core.properties import without_property_validation
 from bokeh.models import (
     BasicTicker,
     ColumnDataSource,
@@ -15,7 +16,7 @@
 
 from distributed.dashboard.components import DashboardComponent, add_periodic_callback
 from distributed.dashboard.components.scheduler import BOKEH_THEME, TICKS_1024, env
-from distributed.dashboard.utils import update, without_property_validation
+from distributed.dashboard.utils import update
 from distributed.diagnostics import nvml
 from distributed.utils import log_errors
 
@@ -94,7 +95,6 @@ def __init__(self, scheduler, width=600, **kwargs):
                 )
                 fig.add_tools(tap)
 
-                fig.toolbar.logo = None
                 fig.toolbar_location = None
                 fig.yaxis.visible = False
 
diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index 62239f4fbed..8cbf914ea26 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -5,6 +5,7 @@
 from collections import defaultdict
 from numbers import Number
 
+from bokeh.core.properties import without_property_validation
 from bokeh.io import curdoc
 from bokeh.layouts import column, row
 from bokeh.models import (
@@ -59,13 +60,7 @@
     ProfileTimePlot,
     SystemMonitor,
 )
-from distributed.dashboard.utils import (
-    BOKEH_VERSION,
-    PROFILING,
-    transpose,
-    update,
-    without_property_validation,
-)
+from distributed.dashboard.utils import BOKEH_VERSION, PROFILING, transpose, update
 from distributed.diagnostics.graph_layout import GraphLayout
 from distributed.diagnostics.progress_stream import color_of, progress_quads
 from distributed.diagnostics.task_stream import TaskStreamPlugin
@@ -208,7 +203,6 @@ def __init__(self, scheduler, **kwargs):
             self.root.xaxis.minor_tick_line_alpha = 0
             self.root.ygrid.visible = False
 
-            self.root.toolbar.logo = None
             self.root.toolbar_location = None
 
             self.root.quad(
@@ -232,10 +226,9 @@ def _memory_color(current: int, limit: int) -> str:
     """Dynamic color used by WorkersMemory and ClusterMemory"""
     if limit and current > limit:
         return "red"
-    elif limit and current > limit / 2:
+    if limit and current > limit / 2:
         return "orange"
-    else:
-        return "blue"
+    return "blue"
 
 
 class ClusterMemory(DashboardComponent):
@@ -286,7 +279,6 @@ def __init__(self, scheduler, width=600, **kwargs):
             self.root.yaxis.visible = False
             self.root.ygrid.visible = False
 
-            self.root.toolbar.logo = None
             self.root.toolbar_location = None
             self.root.yaxis.visible = False
 
@@ -407,7 +399,6 @@ def __init__(self, scheduler, width=600, **kwargs):
             tap = TapTool(callback=OpenURL(url="./info/worker/@escaped_worker.html"))
             self.root.add_tools(tap)
 
-            self.root.toolbar.logo = None
             self.root.toolbar_location = None
             self.root.yaxis.visible = False
 
@@ -541,7 +532,6 @@ def __init__(self, scheduler, **kwargs):
             self.root.xaxis.minor_tick_line_alpha = 0
             self.root.ygrid.visible = False
 
-            self.root.toolbar.logo = None
             self.root.toolbar_location = None
 
             self.root.quad(
@@ -605,7 +595,6 @@ def __init__(self, scheduler, **kwargs):
             self.root.xaxis.minor_tick_line_alpha = 0
             self.root.ygrid.visible = False
 
-            self.root.toolbar.logo = None
             self.root.toolbar_location = None
 
             hover = HoverTool()
@@ -622,7 +611,7 @@ def update(self):
                 "bandwidth": list(bw.values()),
                 "bandwidth-half": [b / 2 for b in bw.values()],
                 "type": list(bw.keys()),
-                "bandwidth_text": list(map(format_bytes, bw.values())),
+                "bandwidth_text": [format_bytes(x) for x in bw.values()],
             }
             self.root.title.text = "Bandwidth: " + format_bytes(
                 self.scheduler.bandwidth
@@ -684,7 +673,6 @@ def __init__(self, scheduler, **kwargs):
             color_bar.ticker = AdaptiveTicker(**TICKS_1024)
             self.root.add_layout(color_bar, "right")
 
-            self.root.toolbar.logo = None
             self.root.toolbar_location = None
 
             hover = HoverTool()
@@ -712,8 +700,7 @@ def name(address):
                     return address
                 if ws.name is not None:
                     return str(ws.name)
-                else:
-                    return address
+                return address
 
             x, y, value = zip(*[(name(a), name(b), c) for (a, b), c in bw.items()])
 
@@ -788,7 +775,6 @@ def __init__(self, scheduler, **kwargs):
             fig.xaxis.minor_tick_line_alpha = 0
             fig.xgrid.visible = False
 
-            fig.toolbar.logo = None
             fig.toolbar_location = None
 
             hover = HoverTool()
@@ -956,7 +942,6 @@ def __init__(self, scheduler, **kwargs):
             self.root.xaxis.minor_tick_line_alpha = 0
             self.root.xgrid.visible = False
 
-            self.root.toolbar.logo = None
             self.root.toolbar_location = None
 
             hover = HoverTool()
@@ -1040,7 +1025,6 @@ def __init__(self, scheduler, **kwargs):
             self.root.xaxis.minor_tick_line_alpha = 0
             self.root.ygrid.visible = False
 
-            self.root.toolbar.logo = None
             self.root.toolbar_location = None
 
             hover = HoverTool()
@@ -1147,7 +1131,6 @@ def __init__(self, scheduler, width=600, **kwargs):
                 )
                 fig.add_tools(tap)
 
-                fig.toolbar.logo = None
                 fig.toolbar_location = None
                 fig.yaxis.visible = False
 
@@ -1167,7 +1150,7 @@ def __init__(self, scheduler, width=600, **kwargs):
     @without_property_validation
     def update(self):
         with log_errors():
-            workers = list(self.scheduler.workers.values())
+            workers = self.scheduler.workers.values()
             now = time()
             if not any(ws.processing for ws in workers) and now < self.last + 1:
                 return
@@ -2026,8 +2009,7 @@ def update(self):
 
         if self.scheduler.transition_counter == self.old_counter:
             return
-        else:
-            self.old_counter = self.scheduler.transition_counter
+        self.old_counter = self.scheduler.transition_counter
 
         if not self.scheduler.task_groups:
             self.subtitle.text = "Scheduler is empty."
diff --git a/distributed/dashboard/components/shared.py b/distributed/dashboard/components/shared.py
index 6ea83576c2e..dbe6a8fa9d6 100644
--- a/distributed/dashboard/components/shared.py
+++ b/distributed/dashboard/components/shared.py
@@ -3,6 +3,7 @@
 from statistics import mean
 
 import tlz as toolz
+from bokeh.core.properties import without_property_validation
 from bokeh.layouts import column, row
 from bokeh.models import (
     Button,
@@ -23,11 +24,7 @@
 from distributed import profile
 from distributed.compatibility import WINDOWS
 from distributed.dashboard.components import DashboardComponent
-from distributed.dashboard.utils import (
-    BOKEH_VERSION,
-    update,
-    without_property_validation,
-)
+from distributed.dashboard.utils import update
 from distributed.utils import log_errors
 
 if dask.config.get("distributed.dashboard.export-tool"):
@@ -146,11 +143,7 @@ def __init__(self, **kwargs):
         def cb(attr, old, new):
             with log_errors():
                 try:
-                    selected = new.indices
-                except AttributeError:
-                    selected = new["1d"]["indices"]
-                try:
-                    ind = selected[0]
+                    ind = new.indices[0]
                 except IndexError:
                     return
                 data = profile.plot_data(self.states[ind], profile_interval)
@@ -159,10 +152,7 @@ def cb(attr, old, new):
                 update(self.source, data)
                 self.source.selected = old
 
-        if BOKEH_VERSION >= "1.0.0":
-            self.source.selected.on_change("indices", cb)
-        else:
-            self.source.on_change("selected", cb)
+        self.source.selected.on_change("indices", cb)
 
     @without_property_validation
     def update(self, state):
@@ -208,32 +198,18 @@ def __init__(self, server, doc=None, **kwargs):
 
         @without_property_validation
         def cb(attr, old, new):
-            if changing[0]:
+            if changing[0] or len(new) == 0:
                 return
             with log_errors():
-                if isinstance(new, list):  # bokeh >= 1.0
-                    selected = new
-                else:
-                    selected = new["1d"]["indices"]
-                try:
-                    ind = selected[0]
-                except IndexError:
-                    return
-                data = profile.plot_data(self.states[ind], profile_interval)
+                data = profile.plot_data(self.states[new[0]], profile_interval)
                 del self.states[:]
                 self.states.extend(data.pop("states"))
                 changing[0] = True  # don't recursively trigger callback
                 update(self.source, data)
-                if isinstance(new, list):  # bokeh >= 1.0
-                    self.source.selected.indices = old
-                else:
-                    self.source.selected = old
+                self.source.selected.indices = old
                 changing[0] = False
 
-        if BOKEH_VERSION >= "1.0.0":
-            self.source.selected.on_change("indices", cb)
-        else:
-            self.source.on_change("selected", cb)
+        self.source.selected.on_change("indices", cb)
 
         self.ts_source = ColumnDataSource({"time": [], "count": []})
         self.ts_plot = figure(
@@ -254,10 +230,7 @@ def cb(attr, old, new):
 
         def ts_change(attr, old, new):
             with log_errors():
-                try:
-                    selected = self.ts_source.selected.indices
-                except AttributeError:
-                    selected = self.ts_source.selected["1d"]["indices"]
+                selected = self.ts_source.selected.indices
                 if selected:
                     start = self.ts_source.data["time"][min(selected)] / 1000
                     stop = self.ts_source.data["time"][max(selected)] / 1000
@@ -266,10 +239,7 @@ def ts_change(attr, old, new):
                     self.start = self.stop = None
                 self.trigger_update(update_metadata=False)
 
-        if BOKEH_VERSION >= "1.0.0":
-            self.ts_source.selected.on_change("indices", ts_change)
-        else:
-            self.ts_source.on_change("selected", ts_change)
+        self.ts_source.selected.on_change("indices", ts_change)
 
         self.reset_button = Button(label="Reset", button_type="success")
         self.reset_button.on_click(lambda: self.update(self.state))
@@ -361,32 +331,18 @@ def __init__(self, server, doc=None, **kwargs):
 
         @without_property_validation
         def cb(attr, old, new):
-            if changing[0]:
+            if changing[0] or len(new) == 0:
                 return
             with log_errors():
-                if isinstance(new, list):  # bokeh >= 1.0
-                    selected = new
-                else:
-                    selected = new["1d"]["indices"]
-                try:
-                    ind = selected[0]
-                except IndexError:
-                    return
-                data = profile.plot_data(self.states[ind], profile_interval)
+                data = profile.plot_data(self.states[new[0]], profile_interval)
                 del self.states[:]
                 self.states.extend(data.pop("states"))
                 changing[0] = True  # don't recursively trigger callback
                 update(self.source, data)
-                if isinstance(new, list):  # bokeh >= 1.0
-                    self.source.selected.indices = old
-                else:
-                    self.source.selected = old
+                self.source.selected.indices = old
                 changing[0] = False
 
-        if BOKEH_VERSION >= "1.0.0":
-            self.source.selected.on_change("indices", cb)
-        else:
-            self.source.on_change("selected", cb)
+        self.source.selected.on_change("indices", cb)
 
         self.ts_source = ColumnDataSource({"time": [], "count": []})
         self.ts_plot = figure(
@@ -407,10 +363,7 @@ def cb(attr, old, new):
 
         def ts_change(attr, old, new):
             with log_errors():
-                try:
-                    selected = self.ts_source.selected.indices
-                except AttributeError:
-                    selected = self.ts_source.selected["1d"]["indices"]
+                selected = self.ts_source.selected.indices
                 if selected:
                     start = self.ts_source.data["time"][min(selected)] / 1000
                     stop = self.ts_source.data["time"][max(selected)] / 1000
@@ -419,10 +372,7 @@ def ts_change(attr, old, new):
                     self.start = self.stop = None
                 self.trigger_update()
 
-        if BOKEH_VERSION >= "1.0.0":
-            self.ts_source.selected.on_change("indices", ts_change)
-        else:
-            self.ts_source.on_change("selected", ts_change)
+        self.ts_source.selected.on_change("indices", ts_change)
 
         self.reset_button = Button(label="Reset", button_type="success")
         self.reset_button.on_click(lambda: self.update(self.state))
@@ -579,13 +529,13 @@ def get_data(self):
     def update(self):
         with log_errors():
             self.source.stream(self.get_data(), 1000)
-            self.label_source.data["cpu"] = list(
+            self.label_source.data["cpu"] = [
                 "{}: {:.1f}%".format(f.__name__, f(self.source.data["cpu"]))
                 for f in [min, max, mean]
-            )
-            self.label_source.data["memory"] = list(
+            ]
+            self.label_source.data["memory"] = [
                 "{}: {}".format(
                     f.__name__, dask.utils.format_bytes(f(self.source.data["memory"]))
                 )
                 for f in [min, max, mean]
-            )
+            ]
diff --git a/distributed/dashboard/components/worker.py b/distributed/dashboard/components/worker.py
index 5ede5529afc..d47e0a7e0d6 100644
--- a/distributed/dashboard/components/worker.py
+++ b/distributed/dashboard/components/worker.py
@@ -2,6 +2,7 @@
 import math
 import os
 
+from bokeh.core.properties import without_property_validation
 from bokeh.layouts import column, row
 from bokeh.models import (
     BoxZoomTool,
@@ -29,7 +30,7 @@
     ProfileTimePlot,
     SystemMonitor,
 )
-from distributed.dashboard.utils import transpose, update, without_property_validation
+from distributed.dashboard.utils import transpose, update
 from distributed.diagnostics.progress_stream import color_of
 from distributed.metrics import time
 from distributed.utils import key_split, log_errors
@@ -49,6 +50,23 @@
 template_variables = {"pages": ["status", "system", "profile", "crossfilter"]}
 
 
+def standard_doc(title, active_page, *, template="simple.html"):
+    def decorator(f):
+        def wrapper(arg, extra, doc):
+            with log_errors():
+                doc.title = title
+                doc.template = env.get_template(template)
+                if active_page is not None:
+                    doc.template_variables["active_page"] = active_page
+                doc.template_variables.update(extra)
+                doc.theme = BOKEH_THEME
+                return f(arg, extra, doc)
+
+        return wrapper
+
+    return decorator
+
+
 class StateTable(DashboardComponent):
     """Currently running tasks"""
 
@@ -389,11 +407,7 @@ def update_figure(self, attr, old, new):
 
     def process_msg(self, msg):
         try:
-
-            def func(k):
-                return msg["keys"].get(k, 0)
-
-            status_key = max(msg["keys"], key=func)
+            status_key = max(msg["keys"], key=lambda x: msg["keys"].get(x, 0))
             typ = self.worker.types.get(status_key, object).__name__
             keyname = key_split(status_key)
             d = {
@@ -471,7 +485,6 @@ def add_digest_figure(self, name):
                 )
 
             fig.xaxis.major_label_orientation = math.pi / 12
-            fig.toolbar.logo = None
             self.digest_sources[name] = sources
             self.digest_figures[name] = fig
             return fig
@@ -489,7 +502,9 @@ def add_counter_figure(self, name):
                 tools="",
                 height=150,
                 sizing_mode=self.sizing_mode,
-                x_range=sorted(map(str, self.server.counters[name].components[0])),
+                x_range=sorted(
+                    str(x) for x in self.server.counters[name].components[0]
+                ),
             )
             fig.ygrid.visible = False
 
@@ -510,8 +525,6 @@ def add_counter_figure(self, name):
                 fig.add_tools(hover)
                 fig.xaxis.major_label_orientation = math.pi / 12
 
-            fig.toolbar.logo = None
-
             self.counter_sources[name] = sources
             self.counter_figures[name] = fig
             return fig
@@ -541,107 +554,71 @@ def update(self):
                         counts = [d[x] for x in xs]
                         ys = [factor * c for c in counts]
                         y_centers = [y / 2 for y in ys]
-                        xs = list(map(str, xs))
+                        xs = [str(x) for x in xs]
                         d = {"x": xs, "y": ys, "y-center": y_centers, "counts": counts}
                         self.counter_sources[name][i].data.update(d)
                     fig.title.text = "%s: %d" % (name, counter.size())
-                    fig.x_range.factors = list(map(str, xs))
+                    fig.x_range.factors = [str(x) for x in xs]
 
 
+@standard_doc("Dask Worker Internal Monitor", active_page="status")
 def status_doc(worker, extra, doc):
-    with log_errors():
-        statetable = StateTable(worker)
-        executing_ts = ExecutingTimeSeries(worker, sizing_mode="scale_width")
-        communicating_ts = CommunicatingTimeSeries(worker, sizing_mode="scale_width")
-        communicating_stream = CommunicatingStream(worker, sizing_mode="scale_width")
-
-        xr = executing_ts.root.x_range
-        communicating_ts.root.x_range = xr
-        communicating_stream.root.x_range = xr
-
-        doc.title = "Dask Worker Internal Monitor"
-        add_periodic_callback(doc, statetable, 200)
-        add_periodic_callback(doc, executing_ts, 200)
-        add_periodic_callback(doc, communicating_ts, 200)
-        add_periodic_callback(doc, communicating_stream, 200)
-        doc.add_root(
-            column(
-                statetable.root,
-                executing_ts.root,
-                communicating_ts.root,
-                communicating_stream.root,
-                sizing_mode="scale_width",
-            )
+    statetable = StateTable(worker)
+    executing_ts = ExecutingTimeSeries(worker, sizing_mode="scale_width")
+    communicating_ts = CommunicatingTimeSeries(worker, sizing_mode="scale_width")
+    communicating_stream = CommunicatingStream(worker, sizing_mode="scale_width")
+
+    xr = executing_ts.root.x_range
+    communicating_ts.root.x_range = xr
+    communicating_stream.root.x_range = xr
+
+    add_periodic_callback(doc, statetable, 200)
+    add_periodic_callback(doc, executing_ts, 200)
+    add_periodic_callback(doc, communicating_ts, 200)
+    add_periodic_callback(doc, communicating_stream, 200)
+    doc.add_root(
+        column(
+            statetable.root,
+            executing_ts.root,
+            communicating_ts.root,
+            communicating_stream.root,
+            sizing_mode="scale_width",
         )
-        doc.template = env.get_template("simple.html")
-        doc.template_variables["active_page"] = "status"
-        doc.template_variables.update(extra)
-        doc.theme = BOKEH_THEME
+    )
 
 
+@standard_doc("Dask Worker Cross-filter", active_page="crossfilter")
 def crossfilter_doc(worker, extra, doc):
-    with log_errors():
-        statetable = StateTable(worker)
-        crossfilter = CrossFilter(worker)
-
-        doc.title = "Dask Worker Cross-filter"
-        add_periodic_callback(doc, statetable, 500)
-        add_periodic_callback(doc, crossfilter, 500)
-
-        doc.add_root(column(statetable.root, crossfilter.root))
-        doc.template = env.get_template("simple.html")
-        doc.template_variables["active_page"] = "crossfilter"
-        doc.template_variables.update(extra)
-        doc.theme = BOKEH_THEME
+    statetable = StateTable(worker)
+    crossfilter = CrossFilter(worker)
+    add_periodic_callback(doc, statetable, 500)
+    add_periodic_callback(doc, crossfilter, 500)
+    doc.add_root(column(statetable.root, crossfilter.root))
 
 
+@standard_doc("Dask Worker Monitor", active_page="system")
 def systemmonitor_doc(worker, extra, doc):
-    with log_errors():
-        sysmon = SystemMonitor(worker, sizing_mode="scale_width")
-        doc.title = "Dask Worker Monitor"
-        add_periodic_callback(doc, sysmon, 500)
-
-        doc.add_root(sysmon.root)
-        doc.template = env.get_template("simple.html")
-        doc.template_variables["active_page"] = "system"
-        doc.template_variables.update(extra)
-        doc.theme = BOKEH_THEME
+    sysmon = SystemMonitor(worker, sizing_mode="scale_width")
+    add_periodic_callback(doc, sysmon, 500)
+    doc.add_root(sysmon.root)
 
 
+@standard_doc("Dask Work Counters", active_page="counters")
 def counters_doc(server, extra, doc):
-    with log_errors():
-        doc.title = "Dask Worker Counters"
-        counter = Counters(server, sizing_mode="stretch_both")
-        add_periodic_callback(doc, counter, 500)
-
-        doc.add_root(counter.root)
-        doc.template = env.get_template("simple.html")
-        doc.template_variables["active_page"] = "counters"
-        doc.template_variables.update(extra)
-        doc.theme = BOKEH_THEME
+    counter = Counters(server, sizing_mode="stretch_both")
+    add_periodic_callback(doc, counter, 500)
+    doc.add_root(counter.root)
 
 
+@standard_doc("Dask Worker Profile", active_page="profile")
 def profile_doc(server, extra, doc):
-    with log_errors():
-        doc.title = "Dask Worker Profile"
-        profile = ProfileTimePlot(server, sizing_mode="stretch_both", doc=doc)
-        profile.trigger_update()
-
-        doc.add_root(profile.root)
-        doc.template = env.get_template("simple.html")
-        doc.template_variables["active_page"] = "profile"
-        doc.template_variables.update(extra)
-        doc.theme = BOKEH_THEME
+    profile = ProfileTimePlot(server, sizing_mode="stretch_both", doc=doc)
+    doc.add_root(profile.root)
+    profile.trigger_update()
 
 
+@standard_doc("Dask: Profile of Event Loop", active_page=None)
 def profile_server_doc(server, extra, doc):
-    with log_errors():
-        doc.title = "Dask: Profile of Event Loop"
-        prof = ProfileServer(server, sizing_mode="stretch_both", doc=doc)
-        doc.add_root(prof.root)
-        doc.template = env.get_template("simple.html")
-        # doc.template_variables['active_page'] = ''
-        doc.template_variables.update(extra)
-        doc.theme = BOKEH_THEME
-
-        prof.trigger_update()
+    profile = ProfileServer(server, sizing_mode="stretch_both", doc=doc)
+    doc.add_root(profile.root)
+    profile.trigger_update()
diff --git a/distributed/dashboard/core.py b/distributed/dashboard/core.py
index 3b6ea0b23a4..63754d3d09c 100644
--- a/distributed/dashboard/core.py
+++ b/distributed/dashboard/core.py
@@ -3,6 +3,8 @@
 from distutils.version import LooseVersion
 
 import bokeh
+from bokeh.application import Application
+from bokeh.application.handlers.function import FunctionHandler
 from bokeh.server.server import BokehTornado
 
 try:
@@ -10,40 +12,33 @@
 except ImportError:
     from bokeh.server.util import create_hosts_whitelist as create_hosts_allowlist
 
-import toolz
-from bokeh.application import Application
-from bokeh.application.handlers.function import FunctionHandler
-
 import dask
 
-if LooseVersion(bokeh.__version__) < LooseVersion("0.13.0"):
+if LooseVersion(bokeh.__version__) < LooseVersion("1.0.0"):
     warnings.warn(
-        "\nDask needs bokeh >= 0.13.0 for the dashboard."
+        "\nDask needs bokeh >= 1.0 for the dashboard."
         "\nContinuing without the dashboard."
     )
-    raise ImportError("Dask needs bokeh >= 0.13.0")
+    raise ImportError("Dask needs bokeh >= 1.0")
 
 
 def BokehApplication(applications, server, prefix="/", template_variables={}):
-    prefix = prefix or ""
-    prefix = "/" + prefix.strip("/")
-    if not prefix.endswith("/"):
-        prefix = prefix + "/"
+    prefix = "/" + prefix.strip("/") + "/" if prefix else "/"
+
+    extra = {"prefix": prefix, **template_variables}
 
-    extra = toolz.merge({"prefix": prefix}, template_variables)
+    funcs = {k: functools.partial(v, server, extra) for k, v in applications.items()}
+    apps = {k: Application(FunctionHandler(v)) for k, v in funcs.items()}
 
-    apps = {k: functools.partial(v, server, extra) for k, v in applications.items()}
-    apps = {k: Application(FunctionHandler(v)) for k, v in apps.items()}
     kwargs = dask.config.get("distributed.scheduler.dashboard.bokeh-application").copy()
     extra_websocket_origins = create_hosts_allowlist(
         kwargs.pop("allow_websocket_origin"), server.http_server.port
     )
 
-    application = BokehTornado(
+    return BokehTornado(
         apps,
         prefix=prefix,
         use_index=False,
         extra_websocket_origins=extra_websocket_origins,
         **kwargs,
     )
-    return application
diff --git a/distributed/dashboard/export_tool.coffee b/distributed/dashboard/export_tool.coffee
deleted file mode 100644
index 14130529c74..00000000000
--- a/distributed/dashboard/export_tool.coffee
+++ /dev/null
@@ -1,44 +0,0 @@
-p = require "core/properties"
-ActionTool = require "models/tools/actions/action_tool"
-
-
-class ExportToolView extends ActionTool.View
-
-  initialize: (options) ->
-    super(options)
-    @listenTo(@model, 'change:content', @export)
-
-  do: () ->
-    # This is just to trigger an event a python callback can respond to
-    @model.event = @model.event + 1
-
-  export: () ->
-    if @model.content?
-      blob = new Blob([@model.content], {type: "text/plain"})
-      url = window.URL.createObjectURL(blob);
-
-      a = document.createElement("a")
-      a.id = "bk-export-tool-link"
-      a.style = "display: none"
-      a.href = url
-      a.download = 'task-stream.html'
-      document.body.appendChild(a)
-      a.click()
-
-      document.getElementById('bk-export-tool-link').remove()
-      window.URL.revokeObjectURL(url);
-
-class ExportTool extends ActionTool.Model
-  default_view: ExportToolView
-  type: "ExportTool"
-  tool_name: "Export"
-  icon: "bk-tool-icon-save"
-
-  @define {
-    event:   [ p.Int,   0 ]
-    content: [ p.String   ]
-  }
-
-module.exports =
-  Model: ExportTool
-  View: ExportToolView
diff --git a/distributed/dashboard/scheduler.py b/distributed/dashboard/scheduler.py
index e7752e33cae..fb0efff1d01 100644
--- a/distributed/dashboard/scheduler.py
+++ b/distributed/dashboard/scheduler.py
@@ -3,11 +3,6 @@
 from tornado import web
 from tornado.ioloop import IOLoop
 
-try:
-    import numpy as np
-except ImportError:
-    np = False
-
 from .components.nvml import gpu_doc  # noqa: 1708
 from .components.nvml import NVML_ENABLED, gpu_memory_doc, gpu_utilization_doc
 from .components.scheduler import (
diff --git a/distributed/dashboard/theme.yaml b/distributed/dashboard/theme.yaml
index 5483a49d325..feb7a266f37 100644
--- a/distributed/dashboard/theme.yaml
+++ b/distributed/dashboard/theme.yaml
@@ -3,3 +3,5 @@ attrs:
     Plot:
         background_fill_color: null
         border_fill_color: null
+    Toolbar:
+      logo: null
diff --git a/distributed/dashboard/utils.py b/distributed/dashboard/utils.py
index 1c54e8b478e..f8ff45b3ce8 100644
--- a/distributed/dashboard/utils.py
+++ b/distributed/dashboard/utils.py
@@ -2,8 +2,8 @@
 from numbers import Number
 
 import bokeh
+from bokeh.core.properties import without_property_validation
 from bokeh.io import curdoc
-from tlz import partition
 from tlz.curried import first
 
 try:
@@ -18,28 +18,8 @@
 PROFILING = False
 
 
-if BOKEH_VERSION >= "1.0.0":
-    # This decorator is only available in bokeh >= 1.0.0, and doesn't work for
-    # callbacks in Python 2, since the signature introspection won't line up.
-    from bokeh.core.properties import without_property_validation
-else:
-
-    def without_property_validation(f):
-        return f
-
-
-def parse_args(args):
-    options = dict(partition(2, args))
-    for k, v in options.items():
-        if v.isdigit():
-            options[k] = int(v)
-
-    return options
-
-
 def transpose(lod):
-    keys = list(lod[0].keys())
-    return {k: [d[k] for d in lod] for k in keys}
+    return {k: [d[k] for d in lod] for k in lod[0]}
 
 
 @without_property_validation

From 1275af3f9986badddbed660cc53762044880e09f Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Thu, 22 Jul 2021 11:46:25 +0200
Subject: [PATCH 1381/1550] Computation class (#5001)

Add a high level Computation class as a collection of TaskGroups
---
 distributed/client.py               |  39 +++++++
 distributed/distributed-schema.yaml |  14 +++
 distributed/distributed.yaml        |  10 ++
 distributed/scheduler.py            | 143 ++++++++++++++++++++++-
 distributed/tests/test_client.py    | 169 ++++++++++++++++++++++++++++
 distributed/tests/test_scheduler.py |  37 +++++-
 distributed/utils_test.py           |   1 +
 7 files changed, 409 insertions(+), 4 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 09730508270..752d94b31c6 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -7,8 +7,10 @@
 import json
 import logging
 import os
+import re
 import sys
 import threading
+import traceback
 import uuid
 import warnings
 import weakref
@@ -2566,6 +2568,41 @@ def run_coroutine(self, function, *args, **kwargs):
         """
         return self.run(function, *args, **kwargs)
 
+    @staticmethod
+    def _get_computation_code() -> str:
+        """Walk up the stack to the user code and extract the code surrounding
+        the compute/submit/persist call. All modules encountered which are
+        blacklisted by the option
+        `distributed.diagnostics.computations.ignore-modules` will be ignored.
+        This can be used to blacklist commonly used libraries which wrap
+        dask/distributed compute calls.
+        """
+
+        ignore_modules = dask.config.get(
+            "distributed.diagnostics.computations.ignore-modules"
+        )
+        if not isinstance(ignore_modules, list):
+            raise TypeError(
+                f"Ignored modules must be a list. Instead got ({type(ignore_modules)}, {ignore_modules})"
+            )
+
+        if ignore_modules:
+            pattern = "|".join([f"(?:{mod})" for mod in ignore_modules])
+            pattern = re.compile(pattern)
+        else:
+            pattern = None
+
+        for fr, _ in traceback.walk_stack(None):
+            if pattern is None or (
+                not pattern.match(fr.f_globals["__name__"])
+                and fr.f_code.co_name not in ("<listcomp>", "<dictcomp>")
+            ):
+                try:
+                    return inspect.getsource(fr)
+                except OSError:
+                    break
+        return "<Code not available>"
+
     def _graph_to_futures(
         self,
         dsk,
@@ -2612,6 +2649,7 @@ def _graph_to_futures(
 
             # Create futures before sending graph (helps avoid contention)
             futures = {key: Future(key, self, inform=False) for key in keyset}
+
             self._send_to_scheduler(
                 {
                     "op": "update-graph-hlg",
@@ -2621,6 +2659,7 @@ def _graph_to_futures(
                     "submitting_task": getattr(thread_state, "key", None),
                     "fifo_timeout": fifo_timeout,
                     "actors": actors,
+                    "code": self._get_computation_code(),
                 }
             )
             return futures
diff --git a/distributed/distributed-schema.yaml b/distributed/distributed-schema.yaml
index f0beee814e3..a76e2f70f1d 100644
--- a/distributed/distributed-schema.yaml
+++ b/distributed/distributed-schema.yaml
@@ -783,6 +783,20 @@ properties:
               not a problem and will be automatically disabled if no GPUs are found in the
               system, but in certain cases it may be desirable to completely disable NVML
               diagnostics.
+          computations:
+            type: object
+            properties:
+              max-history:
+                type: integer
+                minimum: 0
+                description: |
+                  The maximum number of Computations to remember.
+              ignore-modules:
+                type: array
+                description: |
+                  A list of modules which are ignored when trying to collect the
+                  code context when submitting a computation. Accepts regular
+                  expressions.
 
       dashboard:
         type: object
diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index 86e2e5489d6..bdee2b271da 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -201,6 +201,16 @@ distributed:
 
   diagnostics:
     nvml: True
+    computations:
+      max-history: 100
+      ignore-modules:
+        - distributed
+        - dask
+        - xarray
+        - cudf
+        - cuml
+        - prefect
+        - xgboost
 
   ###################
   # Bokeh dashboard #
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 9c25d2c85b3..d9b22933fb2 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -10,6 +10,7 @@
 import os
 import random
 import sys
+import uuid
 import warnings
 import weakref
 from collections import defaultdict, deque
@@ -765,6 +766,114 @@ def ncores(self):
         return self._nthreads
 
 
+@final
+@cclass
+class Computation:
+    """
+    Collection tracking a single compute or persist call
+
+    See also
+    --------
+    TaskPrefix
+    TaskGroup
+    TaskState
+    """
+
+    _start: double
+    _groups: set
+    _code: object
+    _id: object
+
+    def __init__(self):
+        self._start = time()
+        self._groups = set()
+        self._code = sortedcontainers.SortedSet()
+        self._id = uuid.uuid4()
+
+    @property
+    def code(self):
+        return self._code
+
+    @property
+    def start(self):
+        return self._start
+
+    @property
+    def stop(self):
+        if self.groups:
+            return max(tg.stop for tg in self.groups)
+        else:
+            return -1
+
+    @property
+    def states(self):
+        tg: TaskGroup
+        return merge_with(sum, [tg._states for tg in self._groups])
+
+    @property
+    def groups(self):
+        return self._groups
+
+    def __repr__(self):
+        return (
+            f"<Computation {self._id}: "
+            + "Tasks: "
+            + ", ".join(
+                "%s: %d" % (k, v) for (k, v) in sorted(self.states.items()) if v
+            )
+            + ">"
+        )
+
+    def _repr_html_(self):
+        text = f"""<b>Computation</b> {self._id}
+
+        <table>
+            <tr>
+                <td style="text-align: left;"><strong>Duration: </strong>{self.stop - self.start:.3f}</td>
+                <td style="text-align: left;"></td>
+            </tr>
+            <tr>
+                <td style="text-align: left;"><strong>Start: </strong>{self.start}</td>
+                <td style="text-align: left;"></td>
+            </tr>
+            <tr>
+                <td style="text-align: left;"><strong>Groups: </strong>{len(self.groups)}</td>
+                <td style="text-align: left;"></td>
+            </tr>
+            <tr>
+                <td style="text-align: left;"><strong>Tasks: </strong>
+                {", ".join(
+                    "%s: %d" % (k, v) for (k, v) in sorted(self.states.items()) if v
+                )}</td>
+                <td style="text-align: left;"></td>
+            </tr>
+        </table>
+
+        <details>
+        <summary style="margin-bottom": 20px><h4 style="display:inline">Code</h4></summary>
+        """
+
+        for ix, code in enumerate(self.code):
+            text += f"<h5>Code segment {ix + 1} / {len(self.code)}</h5>"
+            text += f"<code>{code}</code>"
+
+        text += """
+        </details>
+        <details>
+        <summary style="margin-bottom": 20px><h4 style="display:inline">Task Groups</h4></summary>
+        <ul>
+        """
+        for gr in self.groups:
+            text += f"""
+            <li> {gr._repr_html_()} </li>
+            """
+        text += """
+        </ul>
+        </details>
+        """
+        return text
+
+
 @final
 @cclass
 class TaskPrefix:
@@ -1028,6 +1137,9 @@ def __repr__(self):
             + ">"
         )
 
+    def _repr_html_(self):
+        return repr(self)[1:-1]
+
     def __len__(self):
         return sum(self._states.values())
 
@@ -1768,6 +1880,7 @@ class SchedulerState:
     _aliases: dict
     _bandwidth: double
     _clients: dict
+    _computations: object
     _extensions: dict
     _host_info: dict
     _idle: object
@@ -1838,6 +1951,9 @@ def __init__(
             self._tasks = tasks
         else:
             self._tasks = dict()
+        self._computations = deque(
+            maxlen=dask.config.get("distributed.diagnostics.computations.max-history")
+        )
         self._task_groups = dict()
         self._task_prefixes = dict()
         self._task_metadata = dict()
@@ -1907,6 +2023,10 @@ def bandwidth(self):
     def clients(self):
         return self._clients
 
+    @property
+    def computations(self):
+        return self._computations
+
     @property
     def extensions(self):
         return self._extensions
@@ -2011,7 +2131,9 @@ def __pdict__(self):
 
     @ccall
     @exceptval(check=False)
-    def new_task(self, key: str, spec: object, state: str) -> TaskState:
+    def new_task(
+        self, key: str, spec: object, state: str, computation: Computation = None
+    ) -> TaskState:
         """Create a new task, and associated states"""
         ts: TaskState = TaskState(key, spec)
         ts._state = state
@@ -2028,6 +2150,8 @@ def new_task(self, key: str, spec: object, state: str) -> TaskState:
         tg = self._task_groups.get(group_key)
         if tg is None:
             self._task_groups[group_key] = tg = TaskGroup(group_key)
+            if computation:
+                computation.groups.add(tg)
             tg._prefix = tp
             tp._groups.append(tg)
         tg.add(ts)
@@ -4175,6 +4299,7 @@ def update_graph_hlg(
         user_priority=0,
         actors=None,
         fifo_timeout=0,
+        code=None,
     ):
         unpacked_graph = HighLevelGraph.__dask_distributed_unpack__(hlg)
         dsk = unpacked_graph["dsk"]
@@ -4213,6 +4338,7 @@ def update_graph_hlg(
             actors,
             fifo_timeout,
             annotations,
+            code=code,
         )
 
     def update_graph(
@@ -4231,6 +4357,7 @@ def update_graph(
         actors=None,
         fifo_timeout=0,
         annotations=None,
+        code=None,
     ):
         """
         Add new computations to the internal dask graph
@@ -4253,6 +4380,16 @@ def update_graph(
 
         dependencies = dependencies or {}
 
+        if parent._total_occupancy > 1e-9 and parent._computations:
+            # Still working on something. Assign new tasks to same computation
+            computation = cast(Computation, parent._computations[-1])
+        else:
+            computation = Computation()
+            parent._computations.append(computation)
+
+        if code and code not in computation._code:  # add new code blocks
+            computation._code.add(code)
+
         n = 0
         while len(tasks) != n:  # walk through new tasks, cancel any bad deps
             n = len(tasks)
@@ -4314,7 +4451,9 @@ def update_graph(
             # XXX Have a method get_task_state(self, k) ?
             ts = parent._tasks.get(k)
             if ts is None:
-                ts = parent.new_task(k, tasks.get(k), "released")
+                ts = parent.new_task(
+                    k, tasks.get(k), "released", computation=computation
+                )
             elif not ts._run_spec:
                 ts._run_spec = tasks.get(k)
 
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 7bdeaff003b..e690930603d 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -1,5 +1,6 @@
 import asyncio
 import gc
+import inspect
 import logging
 import os
 import pickle
@@ -6741,3 +6742,171 @@ def f(x):
 
     c_default = default_client()
     assert c is c_default
+
+
+def test_computation_code_walk_frames():
+
+    test_function_code = inspect.getsource(test_computation_code_walk_frames)
+    code = Client._get_computation_code()
+
+    assert test_function_code == code
+
+    def nested_call():
+        return Client._get_computation_code()
+
+    assert nested_call() == inspect.getsource(nested_call)
+
+    with pytest.raises(TypeError, match="Ignored modules must be a list"):
+        with dask.config.set(
+            {"distributed.diagnostics.computations.ignore-modules": "test_client"}
+        ):
+            code = Client._get_computation_code()
+
+    with dask.config.set(
+        {"distributed.diagnostics.computations.ignore-modules": ["test_client"]}
+    ):
+        import sys
+
+        upper_frame_code = inspect.getsource(sys._getframe(1))
+        code = Client._get_computation_code()
+        assert code == upper_frame_code
+        assert nested_call() == upper_frame_code
+
+
+def test_computation_object_code_dask_compute(client):
+    da = pytest.importorskip("dask.array")
+    x = da.ones((10, 10), chunks=(3, 3))
+    future = x.sum().compute()
+    y = future
+
+    test_function_code = inspect.getsource(test_computation_object_code_dask_compute)
+
+    def fetch_comp_code(dask_scheduler):
+        computations = list(dask_scheduler.computations)
+        assert len(computations) == 1
+        comp = computations[0]
+        assert len(comp.code) == 1
+        return comp.code[0]
+
+    code = client.run_on_scheduler(fetch_comp_code)
+
+    assert code == test_function_code
+
+
+@gen_cluster(client=True)
+async def test_computation_object_code_dask_persist(c, s, a, b):
+    da = pytest.importorskip("dask.array")
+    x = da.ones((10, 10), chunks=(3, 3))
+    future = x.sum().persist()
+    await future
+
+    test_function_code = inspect.getsource(
+        test_computation_object_code_dask_persist.__wrapped__
+    )
+    computations = list(s.computations)
+    assert len(computations) == 1
+    comp = computations[0]
+    assert len(comp.code) == 1
+
+    assert comp.code[0] == test_function_code
+
+
+@gen_cluster(client=True)
+async def test_computation_object_code_client_submit_simple(c, s, a, b):
+    def func(x):
+        return x
+
+    fut = c.submit(func, 1)
+
+    await fut
+
+    test_function_code = inspect.getsource(
+        test_computation_object_code_client_submit_simple.__wrapped__
+    )
+    computations = list(s.computations)
+    assert len(computations) == 1
+    comp = computations[0]
+
+    assert len(comp.code) == 1
+
+    assert comp.code[0] == test_function_code
+
+
+@gen_cluster(client=True)
+async def test_computation_object_code_client_submit_list_comp(c, s, a, b):
+    def func(x):
+        return x
+
+    futs = [c.submit(func, x) for x in range(10)]
+
+    await c.gather(futs)
+
+    test_function_code = inspect.getsource(
+        test_computation_object_code_client_submit_list_comp.__wrapped__
+    )
+    computations = list(s.computations)
+    assert len(computations) == 1
+    comp = computations[0]
+
+    # Code is deduplicated
+    assert len(comp.code) == 1
+
+    assert comp.code[0] == test_function_code
+
+
+@gen_cluster(client=True)
+async def test_computation_object_code_client_submit_dict_comp(c, s, a, b):
+    def func(x):
+        return x
+
+    futs = {x: c.submit(func, x) for x in range(10)}
+
+    await c.gather(futs)
+
+    test_function_code = inspect.getsource(
+        test_computation_object_code_client_submit_dict_comp.__wrapped__
+    )
+    computations = list(s.computations)
+    assert len(computations) == 1
+    comp = computations[0]
+
+    # Code is deduplicated
+    assert len(comp.code) == 1
+
+    assert comp.code[0] == test_function_code
+
+
+@gen_cluster(client=True)
+async def test_computation_object_code_client_map(c, s, a, b):
+    da = pytest.importorskip("dask.array")
+    x = da.ones((10, 10), chunks=(3, 3))
+    future = c.compute(x.sum(), retries=2)
+    y = await future
+
+    test_function_code = inspect.getsource(
+        test_computation_object_code_client_map.__wrapped__
+    )
+    computations = list(s.computations)
+    assert len(computations) == 1
+    comp = computations[0]
+    assert len(comp.code) == 1
+
+    assert comp.code[0] == test_function_code
+
+
+@gen_cluster(client=True)
+async def test_computation_object_code_client_compute(c, s, a, b):
+    da = pytest.importorskip("dask.array")
+    x = da.ones((10, 10), chunks=(3, 3))
+    future = c.compute(x.sum(), retries=2)
+    y = await future
+
+    test_function_code = inspect.getsource(
+        test_computation_object_code_client_compute.__wrapped__
+    )
+    computations = list(s.computations)
+    assert len(computations) == 1
+    comp = computations[0]
+    assert len(comp.code) == 1
+
+    assert comp.code[0] == test_function_code
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 6a151091787..cb848075d59 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -2034,12 +2034,10 @@ async def test_task_groups(c, s, a, b):
         await asyncio.sleep(0.01)
 
     assert tg.states["forgotten"] == 5
-    # Ensure TaskGroup is removed once all tasks are in forgotten state
     assert tg.name not in s.task_groups
     assert tg.start > start
     assert tg.stop < stop
     assert "compute" in tg.all_durations
-    assert sys.getrefcount(tg) == 2
 
 
 @gen_cluster(client=True)
@@ -3095,6 +3093,41 @@ async def test_delete_worker_data_bad_task(c, s, a, bad_first):
     assert s.workers[a.address].nbytes == s.tasks[y.key].nbytes
 
 
+@gen_cluster(client=True)
+async def test_computations(c, s, a, b):
+    da = pytest.importorskip("dask.array")
+
+    x = da.ones(100, chunks=(10,))
+    y = (x + 1).persist()
+    await y
+
+    z = (x - 2).persist()
+    await z
+
+    assert len(s.computations) == 2
+    assert "add" in str(s.computations[0].groups)
+    assert "sub" in str(s.computations[1].groups)
+    assert "sub" not in str(s.computations[0].groups)
+
+    assert isinstance(repr(s.computations[1]), str)
+    assert "x + 1" in s.computations[1]._repr_html_()
+
+    assert s.computations[1].stop == max(tg.stop for tg in s.task_groups.values())
+
+    assert s.computations[0].states["memory"] == y.npartitions
+
+
+@gen_cluster(client=True)
+async def test_computations_futures(c, s, a, b):
+    futures = [c.submit(inc, i) for i in range(10)]
+    total = c.submit(sum, futures)
+    await total
+
+    [computation] = s.computations
+    assert "sum" in str(computation.groups)
+    assert "inc" in str(computation.groups)
+
+
 @gen_cluster(client=True)
 async def test_transition_counter(c, s, a, b):
     assert s.transition_counter == 0
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index 70531d8b0ef..cc21c7db753 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -893,6 +893,7 @@ def _(func):
         if not iscoroutinefunction(func):
             func = gen.coroutine(func)
 
+        @functools.wraps(func)
         def test_func(*outer_args, **kwargs):
             result = None
             workers = []

From 6ecb4a088e474dbf2d49ba87d25ed237de4207ed Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Thu, 22 Jul 2021 11:49:33 +0200
Subject: [PATCH 1382/1550] Simplify test_secede_balances (#5071)

---
 distributed/scheduler.py         |  3 +++
 distributed/tests/test_client.py | 19 +++++++------------
 2 files changed, 10 insertions(+), 12 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index d9b22933fb2..d683384a09d 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -5345,6 +5345,9 @@ def handle_long_running(self, key=None, worker=None, compute_duration=None):
         duration accounting as if the task has stopped.
         """
         parent: SchedulerState = cast(SchedulerState, self)
+        if key not in parent._tasks:
+            logger.debug("Skipping long_running since key %s was already released", key)
+            return
         ts: TaskState = parent._tasks[key]
         steal = parent._extensions.get("stealing")
         if steal is not None:
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index e690930603d..82d0c3b86c6 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -5149,30 +5149,25 @@ def f():
     assert result == 2
 
 
-@pytest.mark.slow
-@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 2, timeout=60)
+@gen_cluster(client=True)
 async def test_secede_balances(c, s, a, b):
     count = threading.active_count()
 
     def f(x):
         client = get_client()
-        sleep(0.01)  # do some work
         secede()
-        futures = client.map(slowinc, range(10), pure=False, delay=0.01)
+        futures = client.map(inc, range(10), pure=False)
         total = client.submit(sum, futures).result()
         return total
 
     futures = c.map(f, range(100))
-    start = time()
-    while not all(f.status == "finished" for f in futures):
-        await asyncio.sleep(0.01)
-        assert threading.active_count() < count + 50
-        assert time() < start + 60
-
-    assert len(a.log) < 2 * len(b.log)
-    assert len(b.log) < 2 * len(a.log)
 
     results = await c.gather(futures)
+
+    assert a.executed_count + b.executed_count == 1100
+    assert a.executed_count > 200
+    assert b.executed_count > 200
+
     assert results == [sum(map(inc, range(10)))] * 100
 
 

From 19cd1f4a6241f2998050bbee9501393797f9ba0a Mon Sep 17 00:00:00 2001
From: Martin Durant <martindurant@users.noreply.github.com>
Date: Thu, 22 Jul 2021 11:16:11 -0400
Subject: [PATCH 1383/1550] Make Actor futures awaitable and work with
 as_completed (#5092)

---
 distributed/actor.py            | 58 +++++++++++++++------------------
 distributed/client.py           |  4 ++-
 distributed/tests/test_actor.py | 37 ++++++++++++++++++++-
 3 files changed, 65 insertions(+), 34 deletions(-)

diff --git a/distributed/actor.py b/distributed/actor.py
index 19828281dc3..e99cac66b11 100644
--- a/distributed/actor.py
+++ b/distributed/actor.py
@@ -1,7 +1,6 @@
 import asyncio
 import functools
 import threading
-from queue import Queue
 
 from .client import Future
 from .protocol import to_serialize
@@ -50,9 +49,9 @@ class Actor(WrappedKey):
     """
 
     def __init__(self, cls, address, key, worker=None):
+        super().__init__(key)
         self._cls = cls
         self._address = address
-        self.key = key
         self._future = None
         if worker:
             self._worker = worker
@@ -142,7 +141,7 @@ def __getattr__(self, key):
 
             elif callable(attr):
                 return lambda *args, **kwargs: ActorFuture(
-                    None, None, result=attr(*args, **kwargs)
+                    None, self._io_loop, result=attr(*args, **kwargs)
                 )
             else:
                 return attr
@@ -162,33 +161,21 @@ async def run_actor_function_on_worker():
                             kwargs={k: to_serialize(v) for k, v in kwargs.items()},
                         )
                     except OSError:
-                        if self._future:
+                        if self._future and not self._future.done():
                             await self._future
+                            return await run_actor_function_on_worker()
                         else:
                             raise OSError("Unable to contact Actor's worker")
                     return result
 
-                if self._asynchronous:
+                q = asyncio.Queue(loop=self._io_loop.asyncio_loop)
 
-                    async def unwrap():
-                        result = await run_actor_function_on_worker()
-                        if result["status"] == "OK":
-                            return result["result"]
-                        raise result["exception"]
+                async def wait_then_add_to_queue():
+                    x = await run_actor_function_on_worker()
+                    await q.put(x)
 
-                    return asyncio.ensure_future(unwrap())
-                else:
-                    # TODO: this mechanism is error prone
-                    # we should endeavor to make dask's standard code work here
-                    q = Queue()
-
-                    async def wait_then_add_to_queue():
-                        x = await run_actor_function_on_worker()
-                        q.put(x)
-
-                    self._io_loop.add_callback(wait_then_add_to_queue)
-
-                    return ActorFuture(q, self._io_loop)
+                self._io_loop.add_callback(wait_then_add_to_queue)
+                return ActorFuture(q, self._io_loop)
 
             return func
 
@@ -245,22 +232,29 @@ def __init__(self, q, io_loop, result=None):
         self.io_loop = io_loop
         if result:
             self._cached_result = result
+        self.status = "pending"
 
     def __await__(self):
-        return self.result()
+        return self._result().__await__()
 
-    def result(self, timeout=None):
-        try:
-            if isinstance(self._cached_result, Exception):
-                raise self._cached_result
-            return self._cached_result
-        except AttributeError:
-            out = self.q.get(timeout=timeout)
+    def done(self):
+        return self.status != "pending"
+
+    async def _result(self, raiseit=True):
+        if not hasattr(self, "_cached_result"):
+            out = await self.q.get()
             if out["status"] == "OK":
+                self.status = "finished"
                 self._cached_result = out["result"]
             else:
+                self.status = "error"
                 self._cached_result = out["exception"]
-        return self.result()
+        if self.status == "error":
+            raise self._cached_result
+        return self._cached_result
+
+    def result(self, timeout=None):
+        return sync(self.io_loop, self._result, callback_timeout=timeout)
 
     def __repr__(self):
         return "<ActorFuture>"
diff --git a/distributed/client.py b/distributed/client.py
index 752d94b31c6..b93b67f3335 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -4442,9 +4442,11 @@ def update(self, futures):
         """Add multiple futures to the collection.
 
         The added futures will emit from the iterator once they finish"""
+        from .actor import ActorFuture
+
         with self.lock:
             for f in futures:
-                if not isinstance(f, Future):
+                if not isinstance(f, (Future, ActorFuture)):
                     raise TypeError("Input must be a future, got %s" % f)
                 self.futures[f] += 1
                 self.loop.add_callback(self._track_future, f)
diff --git a/distributed/tests/test_actor.py b/distributed/tests/test_actor.py
index d529edcc985..52808ca03af 100644
--- a/distributed/tests/test_actor.py
+++ b/distributed/tests/test_actor.py
@@ -6,7 +6,16 @@
 
 import dask
 
-from distributed import Actor, ActorFuture, Client, Future, Nanny, get_client, wait
+from distributed import (
+    Actor,
+    ActorFuture,
+    Client,
+    Future,
+    Nanny,
+    as_completed,
+    get_client,
+    wait,
+)
 from distributed.metrics import time
 from distributed.utils_test import cluster, gen_cluster
 
@@ -681,6 +690,32 @@ def prop(self):
         await ac.prop
 
 
+def test_as_completed(client):
+    ac = client.submit(Counter, actor=True).result()
+    futures = [ac.increment() for _ in range(10)]
+    max = 0
+
+    for future in as_completed(futures):
+        value = future.result()
+        if value > max:
+            max = value
+
+    assert all(future.done() for future in futures)
+    assert max == 10
+
+
+@gen_cluster(client=True, timeout=3)
+async def test_actor_future_awaitable(client, s, a, b):
+    ac = await client.submit(Counter, actor=True)
+    futures = [ac.increment() for _ in range(10)]
+
+    assert all([isinstance(future, ActorFuture) for future in futures])
+
+    out = await asyncio.gather(*futures)
+    assert all([future.done() for future in futures])
+    assert max(out) == 10
+
+
 @gen_cluster(client=True)
 async def test_serialize_with_pickle(c, s, a, b):
     class Foo:

From 64bb53a511e2813a10989a5d036268cd7e61272c Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 22 Jul 2021 16:13:51 -0700
Subject: [PATCH 1384/1550] Add Scheduler.set_restrictions (#5101)

This creates a route for other players, like tasks, to set worker
restrictions for others.  This is an advanced technique that I'm finding
useful currently.
---
 distributed/scheduler.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index d683384a09d..75c6fe259be 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -3752,6 +3752,7 @@ def __init__(
             "retire_workers": self.retire_workers,
             "get_metadata": self.get_metadata,
             "set_metadata": self.set_metadata,
+            "set_restrictions": self.set_restrictions,
             "heartbeat_worker": self.heartbeat_worker,
             "get_task_status": self.get_task_status,
             "get_task_stream": self.get_task_stream,
@@ -6878,6 +6879,10 @@ def get_metadata(self, comm=None, keys=None, default=no_default):
             else:
                 raise
 
+    def set_restrictions(self, comm=None, worker=None):
+        for key, restrictions in worker.items():
+            self.tasks[key]._worker_restrictions = set(restrictions)
+
     def get_task_status(self, comm=None, keys=None):
         parent: SchedulerState = cast(SchedulerState, self)
         return {

From 1c3a51061603739c52a76886e81befb1ad4919eb Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Fri, 23 Jul 2021 12:41:08 +0200
Subject: [PATCH 1385/1550] keep dependents in worker dependency if TS is still
 known (#5034)

Unless the task is entirely forgotten by
the worker it is confusing behaviour if
the dependent or a task is removed
once it finished executing. if this
information is required a dedicated
dynamic attribute should be used like
waiters on the scheduler side
---
 distributed/tests/test_worker.py | 19 +++++++++++++++++++
 distributed/worker.py            | 23 ++++++++++-------------
 2 files changed, 29 insertions(+), 13 deletions(-)

diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index e5e877a2a98..ca2b5a70d13 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -2368,3 +2368,22 @@ async def test_hold_on_to_replicas(c, s, *workers):
 
     while len(workers[2].tasks) > 1:
         await asyncio.sleep(0.01)
+
+
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)])
+async def test_forget_dependents_after_release(c, s, a):
+
+    fut = c.submit(inc, 1, key="f-1")
+    fut2 = c.submit(inc, fut, key="f-2")
+
+    await asyncio.wait([fut, fut2])
+
+    assert fut.key in a.tasks
+    assert fut2.key in a.tasks
+    assert fut2.key in {d.key for d in a.tasks[fut.key].dependents}
+
+    fut2.release()
+
+    while fut2.key in a.tasks:
+        await asyncio.sleep(0.001)
+    assert fut2.key not in {d.key for d in a.tasks[fut.key].dependents}
diff --git a/distributed/worker.py b/distributed/worker.py
index 04bc32e2e95..806699845f2 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -16,7 +16,7 @@
 from datetime import timedelta
 from inspect import isawaitable
 from pickle import PicklingError
-from typing import Dict, Iterable, Optional
+from typing import Dict, Hashable, Iterable, Optional
 
 from tlz import first, keymap, merge, pluck  # noqa: F401
 from tornado.ioloop import IOLoop, PeriodicCallback
@@ -106,10 +106,7 @@ class TaskState:
     * **dependencies**: ``set(TaskState instances)``
         The data needed by this key to run
     * **dependents**: ``set(TaskState instances)``
-        The keys that use this dependency. Only keys which are not available
-        already are tracked in this structure and dependents made available are
-        actively removed. Only after all dependents have been removed, this task
-        is allowed to be forgotten
+        The keys that use this dependency.
     * **duration**: ``float``
         Expected duration the a task
     * **priority**: ``tuple``
@@ -1985,11 +1982,6 @@ def transition_executing_done(self, ts, value=no_value, report=True):
                     for d in ts.dependents:
                         d.waiting_for_data.add(ts.key)
 
-                # Don't release the dependency keys, but do remove them from `dependents`
-                for dependency in ts.dependencies:
-                    dependency.dependents.discard(ts)
-                ts.dependencies.clear()
-
             if report and self.batched_stream and self.status == Status.running:
                 self.send_task_state_to_scheduler(ts)
             else:
@@ -2606,15 +2598,14 @@ def steal_request(self, key):
 
     def release_key(
         self,
-        key: str,
+        key: Hashable,
         cause: Optional[TaskState] = None,
         reason: Optional[str] = None,
         report: bool = True,
     ):
         try:
-
             if self.validate:
-                assert isinstance(key, str)
+                assert not isinstance(key, TaskState)
             ts = self.tasks.get(key, None)
             # If the scheduler holds a reference which is usually the
             # case when it instructed the task to be computed here or if
@@ -2652,6 +2643,12 @@ def release_key(
                     for resource, quantity in ts.resource_restrictions.items():
                         self.available_resources[resource] += quantity
 
+            for d in ts.dependencies:
+                d.dependents.discard(ts)
+
+                if not d.dependents and d.state in ("flight", "fetch"):
+                    self.release_key(d.key, reason="Dependent released")
+
             if report:
                 # Inform the scheduler of keys which will have gone missing
                 # We are releasing them before they have completed

From b5e9373a334260845e0ce9a97377de890e38c5e5 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Fri, 23 Jul 2021 06:14:23 -0500
Subject: [PATCH 1386/1550] Remove experimental feature warning from actors
 docs (#5108)

---
 docs/source/actors.rst | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/docs/source/actors.rst b/docs/source/actors.rst
index 370837629f3..d1f94ca6945 100644
--- a/docs/source/actors.rst
+++ b/docs/source/actors.rst
@@ -1,10 +1,6 @@
 Actors
 ======
 
-.. note:: This is an experimental feature and is subject to change without notice
-.. note:: This is an advanced feature and may not be suitable for beginning users.
-   It is rarely necessary for common workloads.
-
 Actors enable stateful computations within a Dask workflow.  They are useful
 for some rare algorithms that require additional performance and are willing to
 sacrifice resilience.

From a1893b15d781b22f1ef07c2582d4f70c08e38cf5 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Fri, 23 Jul 2021 14:02:59 -0500
Subject: [PATCH 1387/1550] bump version to 2021.07.1

---
 docs/source/changelog.rst | 33 ++++++++++++++++++++++++++++++++-
 requirements.txt          |  2 +-
 2 files changed, 33 insertions(+), 2 deletions(-)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index 3978e6f848d..e4e80765af8 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,36 @@
 Changelog
 =========
 
+2021.07.1
+---------
+
+Released on July 23, 2021
+
+- Remove experimental feature warning from actors docs (:pr:`5108`) `James Bourbeau`_
+- Keep dependents in worker dependency if TS is still known (:pr:`5034`) `Florian Jetter`_
+- Add ``Scheduler.set_restrictions`` (:pr:`5101`) `Matthew Rocklin`_
+- Make ``Actor`` futures awaitable and work with ``as_completed`` (:pr:`5092`) `Martin Durant`_
+- Simplify ``test_secede_balances`` (:pr:`5071`) `Florian Jetter`_
+- ``Computation`` class (:pr:`5001`) `Florian Jetter`_
+- Some light dashboard cleanup (:pr:`5102`) `Bryan Van de Ven`_
+- Don't package tests (:pr:`5054`) `James Bourbeau`_
+- Add pytest marker for GPU tests (:pr:`5023`) `Charles Blackmon-Luca`_
+- Actor: don't hold key references on workers (:pr:`4937`) `Gabe Joseph`_
+- Collapse nav to hamburger sooner (:pr:`5094`) `Julia Signell`_
+- Verify that actors survive pickling (:pr:`5086`) `Matthew Rocklin`_
+- Reenable UCX-Py tests that used to segfault (:pr:`5076`) `Peter Andreas Entschev`_
+- Better support ``ProcessPoolExecutors`` (:pr:`5063`) `Matthew Rocklin`_
+- Simplify ``test_worker_heartbeat_after_cancel`` (:pr:`5067`) `Florian Jetter`_
+- Avoid property validation in Bokeh (:pr:`5065`) `Matthew Rocklin`_
+- Reduce default websocket frame size and make configurable (:pr:`5070`) `Ian Rose`_
+- Disable pytest-timeout ``SIGALARM`` on MacOS (:pr:`5057`) `crusaderky`_
+- ``rebalance()`` resilience to computations (:pr:`4968`) `crusaderky`_
+- Improve CI stability (:pr:`5022`) `crusaderky`_
+- Ensure heartbeats after cancelation do not raise ``KeyError`` s (:pr:`5053`) `Florian Jetter`_
+- Add more useful exception message on TLS cert mismatch (:pr:`5040`) `Jacob Tomlinson`_
+- Add bokeh ``mode`` parameter to performance reports (:pr:`5025`) `James Bourbeau`_
+
+
 2021.07.0
 ---------
 
@@ -2491,4 +2521,5 @@ significantly without many new features.
 .. _`ArtinSarraf`: https://github.com/ArtinSarraf
 .. _`Gabe Joseph`: https://github.com/gjoseph92
 .. _`Freyam Mehta`: https://github.com/freyam
-.. _`gerrymanoim`: https://github.com/gerrymanoim
\ No newline at end of file
+.. _`gerrymanoim`: https://github.com/gerrymanoim
+.. _`Bryan Van de Ven`: https://github.com/bryevdv
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index f08d41d60fa..4905ad87815 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,6 @@
 click >= 6.6
 cloudpickle >= 1.5.0
-dask == 2021.07.0
+dask == 2021.07.1
 msgpack >= 0.6.0
 psutil >= 5.0
 sortedcontainers !=2.0.0, !=2.0.1

From 9c30f38b3ea065e852331b1b6237c5e6239a3a1f Mon Sep 17 00:00:00 2001
From: Gabe Joseph <gjoseph92@gmail.com>
Date: Fri, 23 Jul 2021 18:10:26 -0800
Subject: [PATCH 1388/1550] Short-circuit root-ish check for many deps (#5113)

While looking into https://github.com/dask/distributed/issues/5083 I happened to notice that the dashboard felt very sluggish. I profiled with py-spy and discovered that the scheduler was spending 20% of runtime calculaing `sum(map(len, group._dependencies)) < 5`! A quick print statement showed some task groups depended on 25,728 other groups (each of size 1). We can easily skip those.

I originally had this conditional in https://github.com/dask/distributed/pull/4967 but we removed it for simplicity: https://github.com/dask/distributed/pull/4967#discussion_r661479904; turns out it was relevant after all!
---
 distributed/scheduler.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 75c6fe259be..2ba9f124161 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -2485,6 +2485,7 @@ def decide_worker(self, ts: TaskState) -> WorkerState:
         if (
             valid_workers is None
             and len(group) > self._total_nthreads * 2
+            and len(group._dependencies) < 5
             and sum(map(len, group._dependencies)) < 5
         ):
             ws: WorkerState = group._last_worker

From 117d27781e05149c0d745376641ed9ca33269245 Mon Sep 17 00:00:00 2001
From: Marcos Moyano <marcos@anue.biz>
Date: Fri, 23 Jul 2021 23:13:30 -0300
Subject: [PATCH 1389/1550] Allow ws(s) messages greater than 10Mb (#5110)

* Allow messages greater than 10Mb

* Write large trnsfer test case with no compression
---
 distributed/comm/tests/test_ws.py | 10 ++++++++++
 distributed/comm/ws.py            |  7 ++++++-
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/distributed/comm/tests/test_ws.py b/distributed/comm/tests/test_ws.py
index 54dd1940fd1..dee4c8f6a8e 100644
--- a/distributed/comm/tests/test_ws.py
+++ b/distributed/comm/tests/test_ws.py
@@ -127,6 +127,16 @@ async def test_large_transfer(cleanup):
                 await c.scatter(np.random.random(1_000_000))
 
 
+@pytest.mark.asyncio
+async def test_large_transfer_with_no_compression(cleanup):
+    np = pytest.importorskip("numpy")
+    with dask.config.set({"distributed.comm.compression": None}):
+        async with Scheduler(protocol="ws://") as s:
+            async with Worker(s.address, protocol="ws://"):
+                async with Client(s.address, asynchronous=True) as c:
+                    await c.scatter(np.random.random(1_500_000))
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize(
     "dashboard,protocol,security,port",
diff --git a/distributed/comm/ws.py b/distributed/comm/ws.py
index 8a64f209dbd..2d254666555 100644
--- a/distributed/comm/ws.py
+++ b/distributed/comm/ws.py
@@ -23,6 +23,7 @@
 
 logger = logging.getLogger(__name__)
 
+MAX_MESSAGE_SIZE = 10_000_000_000
 
 BIG_BYTES_SHARD_SIZE = dask.utils.parse_bytes(
     dask.config.get("distributed.comm.websockets.shard")
@@ -76,6 +77,10 @@ def close(self):
         super().close()
         self.closed = True
 
+    @property
+    def max_message_size(self) -> int:
+        return self.settings.get("websocket_max_message_size", MAX_MESSAGE_SIZE)
+
 
 class WSHandlerComm(Comm):
     def __init__(self, handler, deserialize=True, allow_offload=True):
@@ -378,7 +383,7 @@ async def connect(self, address, deserialize=True, **connection_args):
         kwargs = self._get_connect_args(**connection_args)
         try:
             request = HTTPRequest(f"{self.prefix}{address}", **kwargs)
-            sock = await websocket_connect(request, max_message_size=10_000_000_000)
+            sock = await websocket_connect(request, max_message_size=MAX_MESSAGE_SIZE)
             if sock.stream.closed() and sock.stream.error:
                 raise StreamClosedError(sock.stream.error)
         except StreamClosedError as e:

From f5139fd80b95c0e8aa489b1e23794c6f64a9925f Mon Sep 17 00:00:00 2001
From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com>
Date: Mon, 26 Jul 2021 15:00:09 -0400
Subject: [PATCH 1390/1550] Move UCX/RMM config variables to Distributed
 namespace  (#4916)

---
 distributed/comm/ucx.py             |  25 ++++---
 distributed/config.py               |   2 +
 distributed/distributed-schema.yaml | 109 ++++++++++++++--------------
 distributed/distributed.yaml        |  20 ++---
 4 files changed, 82 insertions(+), 74 deletions(-)

diff --git a/distributed/comm/ucx.py b/distributed/comm/ucx.py
index 457a5c96f5c..0e048c9f6d9 100644
--- a/distributed/comm/ucx.py
+++ b/distributed/comm/ucx.py
@@ -108,7 +108,7 @@ def device_array(n):
                     "In order to send/recv CUDA arrays, Numba or RMM is required"
                 )
 
-    pool_size_str = dask.config.get("rmm.pool-size")
+    pool_size_str = dask.config.get("distributed.rmm.pool-size")
     if pool_size_str is not None:
         pool_size = parse_bytes(pool_size_str)
         rmm.reinitialize(
@@ -121,7 +121,7 @@ def device_array(n):
         ucx_create_endpoint = ucp.create_endpoint
         ucx_create_listener = ucp.create_listener
     else:
-        reuse_endpoints = dask.config.get("ucx.reuse-endpoints")
+        reuse_endpoints = dask.config.get("distributed.comm.ucx.reuse-endpoints")
         if (
             reuse_endpoints is None and ucp.get_ucx_version() >= (1, 11, 0)
         ) or reuse_endpoints is False:
@@ -491,12 +491,12 @@ def _scrub_ucx_config():
     # leave UCX to its default configuration
     if any(
         [
-            dask.config.get("ucx.tcp"),
-            dask.config.get("ucx.nvlink"),
-            dask.config.get("ucx.infiniband"),
+            dask.config.get("distributed.comm.ucx.tcp"),
+            dask.config.get("distributed.comm.ucx.nvlink"),
+            dask.config.get("distributed.comm.ucx.infiniband"),
         ]
     ):
-        if dask.config.get("ucx.rdmacm"):
+        if dask.config.get("distributed.comm.ucx.rdmacm"):
             tls = "tcp" if ucx_110 else "tcp,rdmacm"
             tls_priority = "rdmacm"
         else:
@@ -506,17 +506,22 @@ def _scrub_ucx_config():
         # CUDA COPY can optionally be used with ucx -- we rely on the user
         # to define when messages will include CUDA objects.  Note:
         # defining only the Infiniband flag will not enable cuda_copy
-        if any([dask.config.get("ucx.nvlink"), dask.config.get("ucx.cuda_copy")]):
+        if any(
+            [
+                dask.config.get("distributed.comm.ucx.nvlink"),
+                dask.config.get("distributed.comm.ucx.cuda_copy"),
+            ]
+        ):
             tls = tls + ",cuda_copy"
 
-        if dask.config.get("ucx.infiniband"):
+        if dask.config.get("distributed.comm.ucx.infiniband"):
             tls = "rc," + tls
-        if dask.config.get("ucx.nvlink"):
+        if dask.config.get("distributed.comm.ucx.nvlink"):
             tls = tls + ",cuda_ipc"
 
         options = {"TLS": tls, "SOCKADDR_TLS_PRIORITY": tls_priority}
 
-        net_devices = dask.config.get("ucx.net-devices")
+        net_devices = dask.config.get("distributed.comm.ucx.net-devices")
         if net_devices is not None and net_devices != "":
             options["NET_DEVICES"] = net_devices
 
diff --git a/distributed/config.py b/distributed/config.py
index 4b1cfb35042..3163c9e8360 100644
--- a/distributed/config.py
+++ b/distributed/config.py
@@ -48,6 +48,8 @@
     "log-length": "distributed.admin.log-length",
     "log-format": "distributed.admin.log-format",
     "pdb-on-err": "distributed.admin.pdb-on-err",
+    "ucx": "distributed.comm.ucx",
+    "rmm": "distributed.rmm",
 }
 
 dask.config.rename(aliases)
diff --git a/distributed/distributed-schema.yaml b/distributed/distributed-schema.yaml
index a76e2f70f1d..a63f042a6b9 100644
--- a/distributed/distributed-schema.yaml
+++ b/distributed/distributed-schema.yaml
@@ -759,6 +759,53 @@ properties:
                       Alternatively, the key can be appended to the cert file
                       above, and this field left blank
 
+          ucx:
+            type: object
+            description: |
+              UCX provides access to other transport methods including NVLink and InfiniBand.
+            properties:
+              cuda_copy:
+                type: boolean
+                description: |
+                  Set environment variables to enable CUDA support over UCX. This may be used even if
+                  InfiniBand and NVLink are not supported or disabled, then transferring data over TCP.
+              tcp:
+                type: boolean
+                description: |
+                  Set environment variables to enable TCP over UCX, even if InfiniBand and NVLink
+                  are not supported or disabled.
+              nvlink:
+                type: boolean
+                description: |
+                  Set environment variables to enable UCX over NVLink, implies ``distributed.comm.ucx.tcp=True``.
+              infiniband:
+                type: boolean
+                description: |
+                  Set environment variables to enable UCX over InfiniBand, implies ``distributed.comm.ucx.tcp=True``.
+              rdmacm:
+                type: boolean
+                description: |
+                  Set environment variables to enable UCX RDMA connection manager support,
+                  requires ``distributed.comm.ucx.infiniband=True``.
+              net-devices:
+                type: [string, 'null']
+                description: |
+                  Interface(s) used by workers for UCX communication. Can be a string (like
+                  ``"eth0"`` for NVLink or ``"mlx5_0:1"``/``"ib0"`` for InfiniBand), ``"auto"``
+                  (requires ``distributed.comm.ucx.infiniband=True``) to pick the optimal interface per-worker based on
+                  the system's topology, or ``None`` to stay with the default value of ``"all"`` (use
+                  all available interfaces). Setting to ``"auto"`` requires UCX-Py to be installed
+                  and compiled with hwloc support. Unexpected errors can occur when using
+                  ``"auto"`` if any interfaces are disconnected or improperly configured.
+              reuse-endpoints:
+                type: [boolean, 'null']
+                description: |
+                  Enable UCX-Py reuse endpoints mechanism if ``True`` or if it's not specified and
+                  UCX < 1.11 is installed, otherwise disable reuse endpoints. This was primarily
+                  introduced to resolve an issue with CUDA IPC that has been fixed in UCX 1.10, but
+                  can cause establishing endpoints to be very slow, this is particularly noticeable in
+                  clusters of more than a few dozen workers.
+
           websockets:
             type: object
             properties:
@@ -895,58 +942,12 @@ properties:
                 type: string
                 description: Polling time to query cpu/memory statistics default 500ms
 
-  rmm:
-    type: object
-    description: |
-      Configuration options for the RAPIDS Memory Manager.
-    properties:
-      pool-size:
-        type: [integer, 'null']
-        description: |
-          The size of the memory pool in bytes.
-  ucx:
-    type: object
-    description: |
-      UCX provides access to other transport methods including NVLink and InfiniBand.
-    properties:
-      cuda_copy:
-        type: boolean
-        description: |
-          Set environment variables to enable CUDA support over UCX. This may be used even if
-          InfiniBand and NVLink are not supported or disabled, then transferring data over TCP.
-      tcp:
-        type: boolean
-        description: |
-          Set environment variables to enable TCP over UCX, even if InfiniBand and NVLink
-          are not supported or disabled.
-      nvlink:
-        type: boolean
-        description: |
-          Set environment variables to enable UCX over NVLink, implies ``ucx.tcp=True``.
-      infiniband:
-        type: boolean
-        description: |
-          Set environment variables to enable UCX over InfiniBand, implies ``ucx.tcp=True``.
-      rdmacm:
-        type: boolean
-        description: |
-          Set environment variables to enable UCX RDMA connection manager support,
-          requires ``ucx.infiniband=True``.
-      net-devices:
-        type: [string, 'null']
-        description: |
-          Interface(s) used by workers for UCX communication. Can be a string (like
-          ``"eth0"`` for NVLink or ``"mlx5_0:1"``/``"ib0"`` for InfiniBand), ``"auto"``
-          (requires ``ucx.infiniband=True``) to pick the optimal interface per-worker based on
-          the system's topology, or ``None`` to stay with the default value of ``"all"`` (use
-          all available interfaces). Setting to ``"auto"`` requires UCX-Py to be installed
-          and compiled with hwloc support. Unexpected errors can occur when using
-          ``"auto"`` if any interfaces are disconnected or improperly configured.
-      reuse-endpoints:
-        type: [boolean, 'null']
+      rmm:
+        type: object
         description: |
-          Enable UCX-Py reuse endpoints mechanism if ``True`` or if it's not specified and
-          UCX < 1.11 is installed, otherwise disable reuse endpoints. This was primarily
-          introduced to resolve an issue with CUDA IPC that has been fixed in UCX 1.10, but
-          can cause establishing endpoints to be very slow, this is particularly noticeable in
-          clusters of more than a few dozen workers.
+          Configuration options for the RAPIDS Memory Manager.
+        properties:
+          pool-size:
+            type: [integer, 'null']
+            description: |
+              The size of the memory pool in bytes.
diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index bdee2b271da..9429a46409e 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -170,6 +170,14 @@ distributed:
     default-scheme: tcp
     socket-backlog: 2048
     recent-messages-log-length: 0  # number of messages to keep for debugging
+    ucx:
+      cuda_copy: False  # enable cuda-copy
+      tcp: False  # enable tcp
+      nvlink: False  # enable cuda_ipc
+      infiniband: False # enable Infiniband
+      rdmacm: False # enable RDMACM
+      net-devices: null  # define what interface to use for UCX comm
+      reuse-endpoints: null  # enable endpoint reuse
 
     zstd:
       level: 3      # Compression level, between 1 and 22.
@@ -239,13 +247,5 @@ distributed:
     system-monitor:
       interval: 500ms
     event-loop: tornado
-rmm:
-  pool-size: null
-ucx:
-  cuda_copy: False  # enable cuda-copy
-  tcp: False  # enable tcp
-  nvlink: False  # enable cuda_ipc
-  infiniband: False # enable Infiniband
-  rdmacm: False # enable RDMACM
-  net-devices: null  # define what interface to use for UCX comm
-  reuse-endpoints: null  # enable endpoint reuse
+  rmm:
+    pool-size: null

From 145e8aa38b45d603aaecae553181d719b683daeb Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Mon, 26 Jul 2021 13:25:16 -0700
Subject: [PATCH 1391/1550] Read smaller frames to workaround OpenSSL bug
 (#5115)

As older versions of OpenSSL (in particular 1.0.2) have limitations on
the size of buffers they can work with, take small views into our larger
buffer and read those in instead. This should keep the buffer sizes more
manageable for OpenSSL.
---
 distributed/comm/tcp.py | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py
index de6cee016d8..0088ed6500e 100644
--- a/distributed/comm/tcp.py
+++ b/distributed/comm/tcp.py
@@ -1,3 +1,4 @@
+import ctypes
 import errno
 import functools
 import logging
@@ -14,6 +15,7 @@
 except ImportError:
     ssl = None
 
+from tlz import sliding_window
 from tornado import netutil
 from tornado.iostream import StreamClosedError
 from tornado.tcpclient import TCPClient
@@ -34,6 +36,7 @@
 logger = logging.getLogger(__name__)
 
 
+C_INT_MAX = 256 ** ctypes.sizeof(ctypes.c_int) // 2 - 1
 MAX_BUFFER_SIZE = MEMORY_LIMIT / 2
 
 
@@ -193,9 +196,15 @@ async def read(self, deserializers=None):
             frames_nbytes = await stream.read_bytes(fmt_size)
             (frames_nbytes,) = struct.unpack(fmt, frames_nbytes)
 
-            frames = bytearray(frames_nbytes)
-            n = await stream.read_into(frames)
-            assert n == frames_nbytes, (n, frames_nbytes)
+            frames = memoryview(bytearray(frames_nbytes))
+            # Workaround for OpenSSL 1.0.2 (can drop with OpenSSL 1.1.1)
+            for i, j in sliding_window(
+                2, range(0, frames_nbytes + C_INT_MAX, C_INT_MAX)
+            ):
+                chunk = frames[i:j]
+                chunk_nbytes = len(chunk)
+                n = await stream.read_into(chunk)
+                assert n == chunk_nbytes, (n, chunk_nbytes)
         except StreamClosedError as e:
             self.stream = None
             self._closed = True

From b37ac9d2032413d745887f1d593a9cdac917e332 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 26 Jul 2021 14:20:18 -0700
Subject: [PATCH 1392/1550] Set nanny environment variables in config (#5098)

With the nanny we have the ability to set environment variables in dask workers.
This makes this configurable with the dask.config system.

Somewhat more controversially, this also sets a few common defaults like
MKL_NUM_THREADS, OMP_NUM_THREADS, and MALLOC_TRIM_THRESHOLD

These *may* have surprising effects, however I suspect that setting
these defaults will help far more people than it would harm.
---
 distributed/distributed-schema.yaml |  5 +++++
 distributed/distributed.yaml        |  4 ++++
 distributed/nanny.py                |  8 +++++++-
 distributed/tests/test_config.py    |  2 +-
 distributed/tests/test_nanny.py     | 15 +++++++++++++++
 5 files changed, 32 insertions(+), 2 deletions(-)

diff --git a/distributed/distributed-schema.yaml b/distributed/distributed-schema.yaml
index a63f042a6b9..d10022f7574 100644
--- a/distributed/distributed-schema.yaml
+++ b/distributed/distributed-schema.yaml
@@ -510,6 +510,11 @@ properties:
 
               See https://docs.dask.org/en/latest/setup/custom-startup.html for more information
 
+          environ:
+            type: object
+            description: |
+              Environment variables to set on all worker processes started by nannies
+
       client:
         type: object
         description: |
diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index 9429a46409e..929b58676a7 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -142,6 +142,10 @@ distributed:
   nanny:
     preload: []             # Run custom modules with Nanny
     preload-argv: []        # See https://docs.dask.org/en/latest/setup/custom-startup.html
+    environ:
+      MALLOC_TRIM_THRESHOLD_: 65536
+      OMP_NUM_THREADS: 1
+      MKL_NUM_THREADS: 1
 
   client:
     heartbeat: 5s  # Interval between client heartbeats
diff --git a/distributed/nanny.py b/distributed/nanny.py
index a714489da0d..aab940f917b 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -162,7 +162,13 @@ def __init__(
         self.death_timeout = parse_timedelta(death_timeout)
 
         self.Worker = Worker if worker_class is None else worker_class
-        self.env = env or {}
+        self.env = dask.config.get("distributed.nanny.environ")
+        for k in self.env:
+            if k in os.environ:
+                self.env[k] = os.environ[k]
+        if env:
+            self.env.update(env)
+        self.env = {k: str(v) for k, v in self.env.items()}
         self.config = config or dask.config.config
         worker_kwargs.update(
             {
diff --git a/distributed/tests/test_config.py b/distributed/tests/test_config.py
index 200493a822c..cd2c5418d7c 100644
--- a/distributed/tests/test_config.py
+++ b/distributed/tests/test_config.py
@@ -331,7 +331,7 @@ def test_schema_is_complete():
     with open(schema_fn) as f:
         schema = yaml.safe_load(f)
 
-    skip = {"default-task-durations", "bokeh-application"}
+    skip = {"default-task-durations", "bokeh-application", "environ"}
 
     def test_matches(c, s):
         if set(c) != set(s["properties"]):
diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
index b23d6b1c994..92f709568e9 100644
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -373,6 +373,21 @@ async def test_environment_variable(c, s):
     await asyncio.gather(a.close(), b.close())
 
 
+@gen_cluster(
+    nthreads=[],
+    client=True,
+    config={"distributed.nanny.environ": {"A": 1, "B": 2, "D": 4}},
+)
+async def test_environment_variable_config(c, s, monkeypatch):
+    monkeypatch.setenv("D", "123")
+    async with Nanny(s.address, env={"B": 3, "C": 4}) as n:
+        results = await c.run(lambda: os.environ)
+        assert results[n.worker_address]["A"] == "1"
+        assert results[n.worker_address]["B"] == "3"
+        assert results[n.worker_address]["C"] == "4"
+        assert results[n.worker_address]["D"] == "123"
+
+
 @gen_cluster(nthreads=[], client=True)
 async def test_data_types(c, s):
     w = await Nanny(s.address, data=dict)

From cf1e412fafedb53a919b92b2c3ed9b914a5740c8 Mon Sep 17 00:00:00 2001
From: Naty Clementi <natyclementi@gmail.com>
Date: Mon, 26 Jul 2021 21:43:51 -0400
Subject: [PATCH 1393/1550] Add `WorkerNetworkBandwidth` chart to dashboard
 (#5104)

---
 distributed/dashboard/components/scheduler.py | 94 ++++++++++++++++++-
 distributed/dashboard/components/shared.py    | 16 +++-
 distributed/dashboard/scheduler.py            |  4 +
 .../dashboard/tests/test_scheduler_bokeh.py   | 32 +++++++
 docs/source/http_services.rst                 |  5 +-
 5 files changed, 146 insertions(+), 5 deletions(-)

diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index 8cbf914ea26..83e54cf94bc 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -142,7 +142,7 @@ def __init__(self, scheduler, **kwargs):
     @without_property_validation
     def update(self):
         with log_errors():
-            workers = list(self.scheduler.workers.values())
+            workers = self.scheduler.workers.values()
 
             y = list(range(len(workers)))
             occupancy = [ws.occupancy for ws in workers]
@@ -722,6 +722,98 @@ def name(address):
             update(self.source, result)
 
 
+class WorkerNetworkBandwidth(DashboardComponent):
+    """Worker network bandwidth chart
+
+    Plots horizontal bars with the read_bytes and write_bytes worker state
+    """
+
+    def __init__(self, scheduler, **kwargs):
+        with log_errors():
+            self.scheduler = scheduler
+            self.source = ColumnDataSource(
+                {
+                    "y_read": [],
+                    "y_write": [],
+                    "x_read": [],
+                    "x_write": [],
+                }
+            )
+            self.root = figure(
+                title="Worker Network Bandwidth",
+                tools="",
+                id="bk-worker-net-bandwidth",
+                name="worker_network_bandwidth",
+                **kwargs,
+            )
+
+            # read_bytes
+            self.root.hbar(
+                y="y_read",
+                right="x_read",
+                line_color=None,
+                left=0,
+                height=0.5,
+                fill_color="red",
+                legend_label="read",
+                source=self.source,
+            )
+
+            # write_bytes
+            self.root.hbar(
+                y="y_write",
+                right="x_write",
+                line_color=None,
+                left=0,
+                height=0.5,
+                fill_color="blue",
+                legend_label="write",
+                source=self.source,
+            )
+
+            self.root.axis[0].ticker = BasicTicker(**TICKS_1024)
+            self.root.xaxis[0].formatter = NumeralTickFormatter(format="0.0 b")
+            self.root.xaxis.major_label_orientation = XLABEL_ORIENTATION
+            self.root.xaxis.minor_tick_line_alpha = 0
+            self.root.x_range = Range1d(start=0)
+            self.root.yaxis.visible = False
+            self.root.ygrid.visible = False
+            self.root.toolbar_location = None
+            self.root.yaxis.visible = False
+
+    @without_property_validation
+    def update(self):
+        with log_errors():
+            workers = self.scheduler.workers.values()
+
+            h = 0.1
+            y_read = [i + 0.75 + i * h for i in range(len(workers))]
+            y_write = [i + 0.25 + i * h for i in range(len(workers))]
+
+            x_read = []
+            x_write = []
+
+            for ws in workers:
+                x_read.append(ws.metrics["read_bytes"])
+                x_write.append(ws.metrics["write_bytes"])
+
+            self.root.x_range.end = max(
+                max(x_read),
+                max(x_write),
+                100_000_000,
+                0.95 * self.root.x_range.end,
+            )
+
+            result = {
+                "y_read": y_read,
+                "y_write": y_write,
+                "x_read": x_read,
+                "x_write": x_write,
+            }
+
+            update(self.source, result)
+
+
 class ComputePerKey(DashboardComponent):
     """Bar chart showing time spend in action by key prefix"""
 
diff --git a/distributed/dashboard/components/shared.py b/distributed/dashboard/components/shared.py
index dbe6a8fa9d6..ef857be6ba0 100644
--- a/distributed/dashboard/components/shared.py
+++ b/distributed/dashboard/components/shared.py
@@ -482,8 +482,20 @@ def __init__(self, worker, height=150, last_count=None, **kwargs):
             tools=tools,
             **kwargs,
         )
-        self.bandwidth.line(source=self.source, x="time", y="read_bytes", color="red")
-        self.bandwidth.line(source=self.source, x="time", y="write_bytes", color="blue")
+        self.bandwidth.line(
+            source=self.source,
+            x="time",
+            y="read_bytes",
+            color="red",
+            legend_label="read",
+        )
+        self.bandwidth.line(
+            source=self.source,
+            x="time",
+            y="write_bytes",
+            color="blue",
+            legend_label="write",
+        )
         self.bandwidth.yaxis.axis_label = "Bytes / second"
 
         # self.cpu.yaxis[0].formatter = NumeralTickFormatter(format='0%')
diff --git a/distributed/dashboard/scheduler.py b/distributed/dashboard/scheduler.py
index fb0efff1d01..cd321130043 100644
--- a/distributed/dashboard/scheduler.py
+++ b/distributed/dashboard/scheduler.py
@@ -19,6 +19,7 @@
     TaskGroupGraph,
     TaskProgress,
     TaskStream,
+    WorkerNetworkBandwidth,
     WorkersMemory,
     WorkerTable,
     events_doc,
@@ -67,6 +68,9 @@
     "/individual-workers": individual_doc(WorkerTable, 500),
     "/individual-bandwidth-types": individual_doc(BandwidthTypes, 500),
     "/individual-bandwidth-workers": individual_doc(BandwidthWorkers, 500),
+    "/individual-workers-network-bandwidth": individual_doc(
+        WorkerNetworkBandwidth, 500
+    ),
     "/individual-memory-by-key": individual_doc(MemoryByKey, 500),
     "/individual-compute-time-per-key": individual_doc(ComputePerKey, 500),
     "/individual-aggregate-time-per-action": individual_doc(AggregateAction, 500),
diff --git a/distributed/dashboard/tests/test_scheduler_bokeh.py b/distributed/dashboard/tests/test_scheduler_bokeh.py
index c271e5ac7c7..f003c46d2a8 100644
--- a/distributed/dashboard/tests/test_scheduler_bokeh.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh.py
@@ -35,6 +35,7 @@
     TaskGroupGraph,
     TaskProgress,
     TaskStream,
+    WorkerNetworkBandwidth,
     WorkersMemory,
     WorkersMemoryHistogram,
     WorkerTable,
@@ -476,6 +477,37 @@ async def test_WorkerTable_with_memory_limit_as_0(c, s, a, b):
     assert wt.source.data["memory_percent"][0] == ""
 
 
+@gen_cluster(client=True)
+async def test_WorkerNetworkBandwidth(c, s, a, b):
+    nb = WorkerNetworkBandwidth(s)
+    nb.update()
+
+    assert all(len(v) == 2 for v in nb.source.data.values())
+
+    assert nb.source.data["y_read"] == [0.75, 1.85]
+    assert nb.source.data["y_write"] == [0.25, 1.35]
+
+
+@gen_cluster(client=True)
+async def test_WorkerNetworkBandwidth_metrics(c, s, a, b):
+    # Disable system monitor periodic callback to allow us to manually control
+    # when it is called below
+    a.periodic_callbacks["monitor"].stop()
+    b.periodic_callbacks["monitor"].stop()
+
+    # Update worker system monitors and send updated metrics to the scheduler
+    a.monitor.update()
+    b.monitor.update()
+    await asyncio.gather(a.heartbeat(), b.heartbeat())
+
+    nb = WorkerNetworkBandwidth(s)
+    nb.update()
+
+    for idx, ws in enumerate(s.workers.values()):
+        assert ws.metrics["read_bytes"] == nb.source.data["x_read"][idx]
+        assert ws.metrics["write_bytes"] == nb.source.data["x_write"][idx]
+
+
 @gen_cluster(client=True)
 async def test_TaskGraph(c, s, a, b):
     gp = TaskGraph(s)
diff --git a/docs/source/http_services.rst b/docs/source/http_services.rst
index 8e8ed1cdd19..75027a74a38 100644
--- a/docs/source/http_services.rst
+++ b/docs/source/http_services.rst
@@ -57,14 +57,15 @@ Individual bokeh plots
 - ``/individual-groups``
 - ``/individual-profile``
 - ``/individual-profile-server``
-- ``/individual-nbytes``
-- ``/individual-nbytes-cluster``
+- ``/individual-workers-memory``
+- ``/individual-cluster-memory``
 - ``/individual-cpu``
 - ``/individual-nprocessing``
 - ``/individual-occupancy``
 - ``/individual-workers``
 - ``/individual-bandwidth-types``
 - ``/individual-bandwidth-workers``
+- ``/individual-workers-network-bandwidth``
 - ``/individual-memory-by-key``
 - ``/individual-compute-time-per-key``
 - ``/individual-aggregate-time-per-action``

From 50fd3ff34e1a66e2fe0b27bce1457e8fd4b00d7d Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 27 Jul 2021 14:46:17 -0700
Subject: [PATCH 1394/1550] Add NannyPlugins (#5118)

This is like WorkerPlugin, but allows for code to run before the Worker
starts up.

Unfortunately this requires the Nanny to check in with the scheduler
before starting the Worker.  In principle this should be fast, but it
does delay the common case for the uncommon case.

This PR includes an Environ nanny-plugin.
If we go with this I think that we should move over PipInstall.
We might also move over UploadFile and make a new UploadDirectory
---
 distributed/__init__.py           |  10 ++-
 distributed/client.py             |  41 +++++++----
 distributed/diagnostics/plugin.py | 117 +++++++++++++++++++++++++++++-
 distributed/nanny.py              |  67 ++++++++++++++++-
 distributed/scheduler.py          |  34 +++++++++
 distributed/tests/test_client.py  |  33 +++++++++
 distributed/tests/test_nanny.py   |  21 +++++-
 docs/source/plugins.rst           |  16 +++-
 8 files changed, 316 insertions(+), 23 deletions(-)

diff --git a/distributed/__init__.py b/distributed/__init__.py
index 92a62a300a4..9f7a8d6f4bc 100644
--- a/distributed/__init__.py
+++ b/distributed/__init__.py
@@ -21,7 +21,15 @@
 )
 from .core import Status, connect, rpc
 from .deploy import Adaptive, LocalCluster, SpecCluster, SSHCluster
-from .diagnostics.plugin import PipInstall, SchedulerPlugin, WorkerPlugin
+from .diagnostics.plugin import (
+    Environ,
+    NannyPlugin,
+    PipInstall,
+    SchedulerPlugin,
+    UploadDirectory,
+    UploadFile,
+    WorkerPlugin,
+)
 from .diagnostics.progressbar import progress
 from .event import Event
 from .lock import Lock
diff --git a/distributed/client.py b/distributed/client.py
index b93b67f3335..75937435f07 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -59,7 +59,12 @@
     connect,
     rpc,
 )
-from .diagnostics.plugin import UploadFile, WorkerPlugin, _get_worker_plugin_name
+from .diagnostics.plugin import (
+    NannyPlugin,
+    UploadFile,
+    WorkerPlugin,
+    _get_worker_plugin_name,
+)
 from .metrics import time
 from .objects import HasWhat, SchedulerInfo, WhoHas
 from .protocol import to_serialize
@@ -4089,10 +4094,13 @@ def register_worker_callbacks(self, setup=None):
         """
         return self.register_worker_plugin(_WorkerSetupPlugin(setup))
 
-    async def _register_worker_plugin(self, plugin=None, name=None):
-        responses = await self.scheduler.register_worker_plugin(
-            plugin=dumps(plugin, protocol=4), name=name
-        )
+    async def _register_worker_plugin(self, plugin=None, name=None, nanny=None):
+        if nanny or nanny is None and isinstance(plugin, NannyPlugin):
+            method = self.scheduler.register_nanny_plugin
+        else:
+            method = self.scheduler.register_worker_plugin
+
+        responses = await method(plugin=dumps(plugin, protocol=4), name=name)
         for response in responses.values():
             if response["status"] == "error":
                 exc = response["exception"]
@@ -4100,7 +4108,7 @@ async def _register_worker_plugin(self, plugin=None, name=None):
                 raise exc.with_traceback(tb)
         return responses
 
-    def register_worker_plugin(self, plugin=None, name=None, **kwargs):
+    def register_worker_plugin(self, plugin=None, name=None, nanny=None, **kwargs):
         """
         Registers a lifecycle worker plugin for all current and future workers.
 
@@ -4124,12 +4132,14 @@ def register_worker_plugin(self, plugin=None, name=None, **kwargs):
 
         Parameters
         ----------
-        plugin : WorkerPlugin
-            The plugin object to pass to the workers
+        plugin : WorkerPlugin or NannyPlugin
+            The plugin object to register.
         name : str, optional
             A name for the plugin.
             Registering a plugin with the same name will have no effect.
             If plugin has no name attribute a random name is used.
+        nanny : bool, optional
+            Whether to register the plugin with workers or nannies.
         **kwargs : optional
             If you pass a class as the plugin, instead of a class instance, then the
             class will be instantiated with any extra keyword arguments.
@@ -4174,10 +4184,15 @@ class will be instantiated with any extra keyword arguments.
 
         assert name
 
-        return self.sync(self._register_worker_plugin, plugin=plugin, name=name)
+        return self.sync(
+            self._register_worker_plugin, plugin=plugin, name=name, nanny=nanny
+        )
 
-    async def _unregister_worker_plugin(self, name):
-        responses = await self.scheduler.unregister_worker_plugin(name=name)
+    async def _unregister_worker_plugin(self, name, nanny=None):
+        if nanny:
+            responses = await self.scheduler.unregister_nanny_plugin(name=name)
+        else:
+            responses = await self.scheduler.unregister_worker_plugin(name=name)
 
         for response in responses.values():
             if response["status"] == "error":
@@ -4186,7 +4201,7 @@ async def _unregister_worker_plugin(self, name):
                 raise exc.with_traceback(tb)
         return responses
 
-    def unregister_worker_plugin(self, name):
+    def unregister_worker_plugin(self, name, nanny=None):
         """Unregisters a lifecycle worker plugin
 
         This unregisters an existing worker plugin. As part of the unregistration process
@@ -4220,7 +4235,7 @@ def unregister_worker_plugin(self, name):
         --------
         register_worker_plugin
         """
-        return self.sync(self._unregister_worker_plugin, name=name)
+        return self.sync(self._unregister_worker_plugin, name=name, nanny=nanny)
 
 
 class _WorkerSetupPlugin(WorkerPlugin):
diff --git a/distributed/diagnostics/plugin.py b/distributed/diagnostics/plugin.py
index 85be264205b..f9077afddd1 100644
--- a/distributed/diagnostics/plugin.py
+++ b/distributed/diagnostics/plugin.py
@@ -4,8 +4,9 @@
 import subprocess
 import sys
 import uuid
+import zipfile
 
-from dask.utils import funcname
+from dask.utils import funcname, tmpfile
 
 logger = logging.getLogger(__name__)
 
@@ -175,6 +176,41 @@ def release_key(self, key, state, cause, reason, report):
         """
 
 
+class NannyPlugin:
+    """Interface to extend the Nanny
+
+    A worker plugin enables custom code to run at different stages of the Workers'
+    lifecycle. A nanny plugin does the same thing, but benefits from being able
+    to run code before the worker is started, or to restart the worker if
+    necessary.
+
+    To implement a plugin implement some of the methods of this class and register
+    the plugin to your client in order to have it attached to every existing and
+    future nanny by passing ``nanny=True`` to
+    :meth:`Client.register_worker_plugin<distributed.Client.register_worker_plugin>`.
+
+    The ``restart`` attribute is used to control whether or not a running ``Worker``
+    needs to be restarted when registering the plugin.
+
+    See Also
+    --------
+    WorkerPlugin
+    SchedulerPlugin
+    """
+
+    restart = False
+
+    def setup(self, nanny):
+        """
+        Run when the plugin is attached to a nanny. This happens when the plugin is registered
+        and attached to existing nannies, or when a nanny is created after the plugin has been
+        registered.
+        """
+
+    def teardown(self, nanny):
+        """Run when the nanny to which the plugin is attached to is closed"""
+
+
 def _get_worker_plugin_name(plugin) -> str:
     """Returns the worker plugin name. If plugin has no name attribute
     a random name is used."""
@@ -289,3 +325,82 @@ async def setup(self, worker):
             comm=None, filename=self.filename, data=self.data, load=True
         )
         assert len(self.data) == response["nbytes"]
+
+
+class Environ(NannyPlugin):
+    restart = True
+
+    def __init__(self, environ={}):
+        self.environ = {k: str(v) for k, v in environ.items()}
+
+    async def setup(self, nanny):
+        nanny.env.update(self.environ)
+
+
+class UploadDirectory(NannyPlugin):
+    """A NannyPlugin to upload a local file to workers.
+
+    Parameters
+    ----------
+    path: str
+        A path to the directory to upload
+
+    Examples
+    --------
+    >>> from distributed.diagnostics.plugin import UploadDirectory
+    >>> client.register_worker_plugin(UploadDirectory("/path/to/directory"), nanny=True)  # doctest: +SKIP
+    """
+
+    def __init__(
+        self,
+        path,
+        restart=False,
+        update_path=False,
+        skip_words=(".git", ".github", ".pytest_cache", "tests", "docs"),
+        skip=(lambda fn: os.path.splitext(fn)[1] == ".pyc",),
+    ):
+        """
+        Initialize the plugin by reading in the data from the given file.
+        """
+        path = os.path.expanduser(path)
+        self.path = os.path.split(path)[-1]
+        self.restart = restart
+        self.update_path = update_path
+
+        self.name = "upload-directory-" + os.path.split(path)[-1]
+
+        with tmpfile(extension="zip") as fn:
+            with zipfile.ZipFile(fn, "w", zipfile.ZIP_DEFLATED) as z:
+                for root, dirs, files in os.walk(path):
+                    for file in files:
+                        filename = os.path.join(root, file)
+                        if any(predicate(filename) for predicate in skip):
+                            continue
+                        dirs = filename.split(os.sep)
+                        if any(word in dirs for word in skip_words):
+                            continue
+
+                        archive_name = os.path.relpath(
+                            os.path.join(root, file), os.path.join(path, "..")
+                        )
+                        z.write(filename, archive_name)
+
+            with open(fn, "rb") as f:
+                self.data = f.read()
+
+    async def setup(self, nanny):
+        fn = os.path.join(nanny.local_directory, f"tmp-{str(uuid.uuid4())}.zip")
+        with open(fn, "wb") as f:
+            f.write(self.data)
+
+        import zipfile
+
+        with zipfile.ZipFile(fn) as z:
+            z.extractall(path=nanny.local_directory)
+
+        if self.update_path:
+            path = os.path.join(nanny.local_directory, self.path)
+            if path not in sys.path:
+                sys.path.insert(0, path)
+
+        os.remove(fn)
diff --git a/distributed/nanny.py b/distributed/nanny.py
index aab940f917b..eea2f3113f2 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -8,6 +8,7 @@
 import warnings
 import weakref
 from contextlib import suppress
+from inspect import isawaitable
 from multiprocessing.queues import Empty
 from time import sleep as sync_sleep
 
@@ -22,15 +23,18 @@
 from . import preloading
 from .comm import get_address_host, unparse_host_port
 from .comm.addressing import address_from_user_args
-from .core import CommClosedError, RPCClosed, Status, coerce_to_address
+from .core import CommClosedError, RPCClosed, Status, coerce_to_address, error_message
+from .diagnostics.plugin import _get_worker_plugin_name
 from .node import ServerNode
 from .process import AsyncProcess
 from .proctitle import enable_proctitle_on_children
+from .protocol import pickle
 from .security import Security
 from .utils import (
     TimeoutError,
     get_ip,
     json_load_robust,
+    log_errors,
     mp_context,
     parse_ports,
     silence_logging,
@@ -110,14 +114,14 @@ def __init__(
 
         if local_directory is None:
             local_directory = dask.config.get("temporary-directory") or os.getcwd()
-            if not os.path.exists(local_directory):
-                os.makedirs(local_directory)
             self._original_local_dir = local_directory
             local_directory = os.path.join(local_directory, "dask-worker-space")
         else:
             self._original_local_dir = local_directory
 
         self.local_directory = local_directory
+        if not os.path.exists(self.local_directory):
+            os.makedirs(self.local_directory, exist_ok=True)
 
         self.preload = preload
         if self.preload is None:
@@ -205,8 +209,12 @@ def __init__(
             "terminate": self.close,
             "close_gracefully": self.close_gracefully,
             "run": self.run,
+            "plugin_add": self.plugin_add,
+            "plugin_remove": self.plugin_remove,
         }
 
+        self.plugins = {}
+
         super().__init__(
             handlers=handlers, io_loop=self.loop, connection_args=self.connection_args
         )
@@ -300,6 +308,10 @@ async def start(self):
         for preload in self.preloads:
             await preload.start()
 
+        msg = await self.scheduler.register_nanny()
+        for name, plugin in msg["nanny-plugins"].items():
+            await self.plugin_add(plugin=plugin, name=name)
+
         logger.info("        Start Nanny at: %r", self.address)
         response = await self.instantiate()
         if response == Status.running:
@@ -390,6 +402,47 @@ async def instantiate(self, comm=None) -> Status:
                 raise
         return result
 
+    async def plugin_add(self, comm=None, plugin=None, name=None):
+        with log_errors(pdb=False):
+            if isinstance(plugin, bytes):
+                plugin = pickle.loads(plugin)
+
+            if name is None:
+                name = _get_worker_plugin_name(plugin)
+
+            assert name
+
+            self.plugins[name] = plugin
+
+            logger.info("Starting Nanny plugin %s" % name)
+            if hasattr(plugin, "setup"):
+                try:
+                    result = plugin.setup(nanny=self)
+                    if isawaitable(result):
+                        result = await result
+                except Exception as e:
+                    msg = error_message(e)
+                    return msg
+            if getattr(plugin, "restart", False):
+                await self.restart()
+
+            return {"status": "OK"}
+
+    async def plugin_remove(self, comm=None, name=None):
+        with log_errors(pdb=False):
+            logger.info(f"Removing Nanny plugin {name}")
+            try:
+                plugin = self.plugins.pop(name)
+                if hasattr(plugin, "teardown"):
+                    result = plugin.teardown(nanny=self)
+                    if isawaitable(result):
+                        result = await result
+            except Exception as e:
+                msg = error_message(e)
+                return msg
+
+            return {"status": "OK"}
+
     async def restart(self, comm=None, timeout=30, executor_wait=True):
         async def _():
             if self.process is not None:
@@ -514,6 +567,14 @@ async def close(self, comm=None, timeout=5, report=None):
         for preload in self.preloads:
             await preload.teardown()
 
+        teardowns = [
+            plugin.teardown(self)
+            for plugin in self.plugins.values()
+            if hasattr(plugin, "teardown")
+        ]
+
+        await asyncio.gather(*[td for td in teardowns if isawaitable(td)])
+
         self.stop()
         try:
             if self.process is not None:
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 2ba9f124161..881ed7ababe 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -3691,6 +3691,7 @@ def __init__(
         )
         self.event_counts = defaultdict(int)
         self.worker_plugins = dict()
+        self.nanny_plugins = dict()
 
         worker_handlers = {
             "task-finished": self.handle_task_finished,
@@ -3721,6 +3722,7 @@ def __init__(
             "register-client": self.add_client,
             "scatter": self.scatter,
             "register-worker": self.add_worker,
+            "register_nanny": self.add_nanny,
             "unregister": self.remove_worker,
             "gather": self.gather,
             "cancel": self.stimulus_cancel,
@@ -3760,6 +3762,8 @@ def __init__(
             "register_scheduler_plugin": self.register_scheduler_plugin,
             "register_worker_plugin": self.register_worker_plugin,
             "unregister_worker_plugin": self.unregister_worker_plugin,
+            "register_nanny_plugin": self.register_nanny_plugin,
+            "unregister_nanny_plugin": self.unregister_nanny_plugin,
             "adaptive_target": self.adaptive_target,
             "workers_to_close": self.workers_to_close,
             "subscribe_worker_status": self.subscribe_worker_status,
@@ -4284,8 +4288,16 @@ async def add_worker(
 
             if comm:
                 await comm.write(msg)
+
             await self.handle_worker(comm=comm, worker=address)
 
+    async def add_nanny(self, comm):
+        msg = {
+            "status": "OK",
+            "nanny-plugins": self.nanny_plugins,
+        }
+        return msg
+
     def update_graph_hlg(
         self,
         client=None,
@@ -6938,6 +6950,28 @@ async def unregister_worker_plugin(self, comm, name):
         responses = await self.broadcast(msg=dict(op="plugin-remove", name=name))
         return responses
 
+    async def register_nanny_plugin(self, comm, plugin, name=None):
+        """Registers a setup function, and call it on every worker"""
+        self.nanny_plugins[name] = plugin
+
+        responses = await self.broadcast(
+            msg=dict(op="plugin_add", plugin=plugin, name=name),
+            nanny=True,
+        )
+        return responses
+
+    async def unregister_nanny_plugin(self, comm, name):
+        """Unregisters a worker plugin"""
+        try:
+            self.nanny_plugins.pop(name)
+        except KeyError:
+            raise ValueError(f"The nanny plugin {name} does not exists")
+
+        responses = await self.broadcast(
+            msg=dict(op="plugin_remove", name=name), nanny=True
+        )
+        return responses
+
     def transition(self, key, finish: str, *args, **kwargs):
         """Transition a key from its current state to the finish state
 
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 82d0c3b86c6..7305ab1fdd8 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -6905,3 +6905,36 @@ async def test_computation_object_code_client_compute(c, s, a, b):
     assert len(comp.code) == 1
 
     assert comp.code[0] == test_function_code
+
+
+@gen_cluster(client=True, Worker=Nanny)
+async def test_upload_directory(c, s, a, b, tmp_path):
+    from dask.distributed import UploadDirectory
+
+    files = set(os.listdir())
+
+    with open(tmp_path / "foo.py", "w") as f:
+        f.write("x = 123")
+    with open(tmp_path / "bar.py", "w") as f:
+        f.write("from foo import x")
+
+    plugin = UploadDirectory(tmp_path, restart=True, update_path=True)
+    await c.register_worker_plugin(plugin)
+
+    [name] = a.plugins
+    assert os.path.split(tmp_path)[-1] in name
+
+    def f():
+        import bar
+
+        return bar.x
+
+    results = await c.run(f)
+    assert results[a.worker_address] == 123
+    assert results[b.worker_address] == 123
+
+    async with Nanny(s.address, local_directory=tmp_path / "foo", name="foo") as n:
+        results = await c.run(f)
+        assert results[n.worker_address] == 123
+
+    assert files == set(os.listdir())  # no change
diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
index 92f709568e9..87ea7b9bc53 100644
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -563,12 +563,12 @@ async def start(self):
         raise StartException("broken")
 
 
-@pytest.mark.asyncio
-async def test_worker_start_exception(cleanup):
+@gen_cluster(nthreads=[])
+async def test_worker_start_exception(s):
     # make sure this raises the right Exception:
     with pytest.raises(StartException):
-        async with Nanny("tcp://localhost:1", worker_class=BrokenWorker) as n:
-            await n.start()
+        async with Nanny(s.address, worker_class=BrokenWorker) as n:
+            pass
 
 
 @pytest.mark.asyncio
@@ -579,3 +579,16 @@ async def test_failure_during_worker_initialization(cleanup):
                 async with Nanny(s.address, foo="bar") as n:
                     await n
         assert "Restarting worker" not in logs.getvalue()
+
+
+@gen_cluster(client=True, Worker=Nanny, timeout=10000000)
+async def test_environ_plugin(c, s, a, b):
+    from dask.distributed import Environ
+
+    await c.register_worker_plugin(Environ({"ABC": 123}))
+
+    async with Nanny(s.address, name="new") as n:
+        results = await c.run(os.getenv, "ABC")
+        assert results[a.worker_address] == "123"
+        assert results[b.worker_address] == "123"
+        assert results[n.worker_address] == "123"
diff --git a/docs/source/plugins.rst b/docs/source/plugins.rst
index add010f862b..bf91f6b5f9b 100644
--- a/docs/source/plugins.rst
+++ b/docs/source/plugins.rst
@@ -75,7 +75,7 @@ the scheduler as so:
        scheduler.add_plugin(plugin)
 
 Worker Plugins
-=================
+==============
 
 :class:`distributed.diagnostics.plugin.WorkerPlugin` provides a base class
 for creating your own worker plugins. In addition, Dask provides some
@@ -91,3 +91,17 @@ Built-In Worker Plugins
 
 .. autoclass:: distributed.diagnostics.plugin.PipInstall
 .. autoclass:: distributed.diagnostics.plugin.UploadFile
+
+
+Nanny Plugins
+=============
+
+.. autoclass:: distributed.diagnostics.plugin.NannyPlugin
+   :members:
+
+
+Built-In Nanny Plugins
+----------------------
+
+.. autoclass:: distributed.diagnostics.plugin.Environ
+.. autoclass:: distributed.diagnostics.plugin.UploadDirectory
\ No newline at end of file

From ca4e020aec096d83c0716bf71f1f7e0ef595283f Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Wed, 28 Jul 2021 11:55:15 +0100
Subject: [PATCH 1395/1550] Halve CI runtime! (#5074)

---
 .github/workflows/tests.yaml             | 4 +++-
 distributed/tests/test_client.py         | 2 ++
 distributed/tests/test_failed_workers.py | 2 ++
 distributed/tests/test_nanny.py          | 2 ++
 distributed/tests/test_scheduler.py      | 2 ++
 distributed/tests/test_steal.py          | 2 ++
 distributed/tests/test_worker.py         | 2 ++
 setup.cfg                                | 5 +++--
 8 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
index 84e6d8b75de..703e5ba947a 100644
--- a/.github/workflows/tests.yaml
+++ b/.github/workflows/tests.yaml
@@ -18,6 +18,8 @@ jobs:
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
         python-version: ["3.7", "3.8", "3.9"]
+        # Cherry-pick test modules to split the overall runtime roughly in half
+        partition: [ci1, not ci1]
 
         # Uncomment to stress-test the test suite for random failures.
         # Must also change env.TEST_ID below.
@@ -113,7 +115,7 @@ jobs:
           fi
           source continuous_integration/scripts/set_ulimit.sh
 
-          pytest distributed -m "not avoid_ci" --runslow \
+          pytest distributed -m "not avoid_ci and ${{ matrix.partition }}" --runslow \
             --junitxml reports/pytest.xml -o junit_suite_name=$TEST_ID
 
       # - name: Debug with tmate on failure
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 7305ab1fdd8..e3e8dcd3fa0 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -95,6 +95,8 @@
     wait_for,
 )
 
+pytestmark = pytest.mark.ci1
+
 
 @gen_cluster(client=True)
 async def test_submit(c, s, a, b):
diff --git a/distributed/tests/test_failed_workers.py b/distributed/tests/test_failed_workers.py
index 2b3e229c38d..88b5e2db7a3 100644
--- a/distributed/tests/test_failed_workers.py
+++ b/distributed/tests/test_failed_workers.py
@@ -26,6 +26,8 @@
     slowinc,
 )
 
+pytestmark = pytest.mark.ci1
+
 
 def test_submit_after_failed_worker_sync(loop):
     with cluster() as (s, [a, b]):
diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
index 87ea7b9bc53..013412469be 100644
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -26,6 +26,8 @@
 from distributed.utils import TimeoutError, parse_ports, tmpfile
 from distributed.utils_test import captured_logger, gen_cluster, gen_test, inc
 
+pytestmark = pytest.mark.ci1
+
 
 # FIXME why does this leave behind unclosed Comm objects?
 @gen_cluster(nthreads=[], allow_unclosed=True)
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index cb848075d59..d612e3e746b 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -53,6 +53,8 @@
 else:
     import pickle
 
+pytestmark = pytest.mark.ci1
+
 
 alice = "alice:1234"
 bob = "bob:1234"
diff --git a/distributed/tests/test_steal.py b/distributed/tests/test_steal.py
index c0cecd8c5aa..00c6efb74ff 100644
--- a/distributed/tests/test_steal.py
+++ b/distributed/tests/test_steal.py
@@ -28,6 +28,8 @@
     slowinc,
 )
 
+pytestmark = pytest.mark.ci1
+
 # Most tests here are timing-dependent
 setup_module = nodebug_setup_module
 teardown_module = nodebug_teardown_module
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index ca2b5a70d13..2f2aa82fd47 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -51,6 +51,8 @@
 )
 from distributed.worker import Worker, error_message, logger, parse_memory_limit
 
+pytestmark = pytest.mark.ci1
+
 
 @pytest.mark.asyncio
 async def test_worker_nthreads(cleanup):
diff --git a/setup.cfg b/setup.cfg
index 62f852ce307..f77b02739c2 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -47,8 +47,9 @@ filterwarnings =
     error:Since distributed.*:PendingDeprecationWarning
 minversion = 4
 markers =
-    slow: marks tests as slow (deselect with '-m "not slow"')
-    avoid_ci: marks tests as flaky on CI on all OSs
+    ci1: marks tests as belonging to 1 out of 2 partitions to run on CI ('-m "not ci1"' for second partition)
+    slow: marks tests as slow (deselected by default; select with '--runslow')
+    avoid_ci: marks tests as flaky or broken on CI on all OSs
     ipython: marks tests as exercising IPython
     gpu: marks tests we want to run on GPUs
 

From 85c95bed58454bcf07dc7e4c4fb049e41527601c Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Wed, 28 Jul 2021 12:59:26 +0200
Subject: [PATCH 1396/1550] Ensure worker reconnect registers existing tasks
 properly (#5103)

Resolve a deadlock triggered by a worker reconnect raising an exception
---
 distributed/scheduler.py            |  48 ++++++++----
 distributed/tests/test_scheduler.py | 111 ++++++++++++++++++----------
 distributed/tests/test_worker.py    |  65 ++++++++++++++++
 distributed/worker.py               |  26 ++++++-
 4 files changed, 194 insertions(+), 56 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 881ed7ababe..566f149fc3c 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -2696,7 +2696,13 @@ def transition_processing_memory(
                     ws,
                     key,
                 )
-                return recommendations, client_msgs, worker_msgs
+                worker_msgs[ts._processing_on.address] = [
+                    {
+                        "op": "cancel-compute",
+                        "key": key,
+                        "reason": "Finished on different worker",
+                    }
+                ]
 
             has_compute_startstop: bool = False
             compute_start: double
@@ -4234,19 +4240,25 @@ async def add_worker(
             client_msgs: dict = {}
             worker_msgs: dict = {}
             if nbytes:
+                assert isinstance(nbytes, dict)
                 for key in nbytes:
                     ts: TaskState = parent._tasks.get(key)
-                    if ts is not None and ts._state in ("processing", "waiting"):
-                        t: tuple = parent._transition(
-                            key,
-                            "memory",
-                            worker=address,
-                            nbytes=nbytes[key],
-                            typename=types[key],
-                        )
-                        recommendations, client_msgs, worker_msgs = t
-                        parent._transitions(recommendations, client_msgs, worker_msgs)
-                        recommendations = {}
+                    if ts is not None:
+                        if ts.state == "memory":
+                            self.add_keys(worker=address, keys=[key])
+                        else:
+                            t: tuple = parent._transition(
+                                key,
+                                "memory",
+                                worker=address,
+                                nbytes=nbytes[key],
+                                typename=types[key],
+                            )
+                            recommendations, client_msgs, worker_msgs = t
+                            parent._transitions(
+                                recommendations, client_msgs, worker_msgs
+                            )
+                            recommendations = {}
 
             for ts in list(parent._unrunnable):
                 valid: set = self.valid_workers(ts)
@@ -4659,10 +4671,15 @@ def stimulus_task_finished(self, key=None, worker=None, **kwargs):
         ts: TaskState = parent._tasks.get(key)
         if ts is None:
             return recommendations, client_msgs, worker_msgs
+
+        if ts.state == "memory":
+            self.add_keys(worker=worker, keys=[key])
+            return recommendations, client_msgs, worker_msgs
+
         ws: WorkerState = parent._workers_dv[worker]
         ts._metadata.update(kwargs["metadata"])
 
-        if ts._state == "processing":
+        if ts._state != "released":
             r: tuple = parent._transition(key, "memory", worker=worker, **kwargs)
             recommendations, client_msgs, worker_msgs = r
 
@@ -5580,12 +5597,11 @@ async def gather(self, comm=None, keys=None, serializers=None):
                 # Remove suspicious workers from the scheduler but allow them to
                 # reconnect.
                 await asyncio.gather(
-                    *[
+                    *(
                         self.remove_worker(address=worker, close=False)
                         for worker in missing_workers
-                    ]
+                    )
                 )
-
                 recommendations: dict
                 client_msgs: dict = {}
                 worker_msgs: dict = {}
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index d612e3e746b..45702266fea 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -43,7 +43,7 @@
     tls_only_security,
     varying,
 )
-from distributed.worker import dumps_function, dumps_task
+from distributed.worker import dumps_function, dumps_task, get_worker
 
 if sys.version_info < (3, 8):
     try:
@@ -2182,8 +2182,13 @@ async def test_gather_no_workers(c, s, a, b):
     assert list(res["keys"]) == ["x"]
 
 
+@pytest.mark.slow
+@pytest.mark.parametrize("reschedule_different_worker", [True, False])
+@pytest.mark.parametrize("swap_data_insert_order", [True, False])
 @gen_cluster(client=True, client_kwargs={"direct_to_workers": False})
-async def test_gather_allow_worker_reconnect(c, s, a, b):
+async def test_gather_allow_worker_reconnect(
+    c, s, a, b, reschedule_different_worker, swap_data_insert_order
+):
     """
     Test that client resubmissions allow failed workers to reconnect and re-use
     their results. Failure scenario would be a connection issue during result
@@ -2191,29 +2196,53 @@ async def test_gather_allow_worker_reconnect(c, s, a, b):
     Upon connection failure, the worker is flagged as suspicious and removed
     from the scheduler. If the worker is healthy and reconnencts we want to use
     its results instead of recomputing them.
+
+    See also distributed.tests.test_worker.py::test_worker_reconnects_mid_compute
     """
     # GH3246
-    already_calculated = []
-
-    import time
-
-    def inc_slow(x):
-        # Once the graph below is rescheduled this computation runs again. We
-        # need to sleep for at least 0.5 seconds to give the worker a chance to
-        # reconnect (Heartbeat timing). In slow CI situations, the actual
-        # reconnect might take a bit longer, therefore wait more
-        if x in already_calculated:
-            time.sleep(2)
-        already_calculated.append(x)
+    if reschedule_different_worker:
+        from distributed.diagnostics.plugin import SchedulerPlugin
+
+        class SwitchRestrictions(SchedulerPlugin):
+            def __init__(self, scheduler):
+                self.scheduler = scheduler
+
+            def transition(self, key, start, finish, **kwargs):
+                if key in ("reducer", "final") and finish == "memory":
+                    self.scheduler.tasks[key]._worker_restrictions = {b.address}
+
+        plugin = SwitchRestrictions(s)
+        s.add_plugin(plugin)
+
+    from distributed import Lock
+
+    b_address = b.address
+
+    def inc_slow(x, lock):
+        w = get_worker()
+        if w.address == b_address:
+            with lock:
+                return x + 1
         return x + 1
 
-    x = c.submit(inc_slow, 1)
-    y = c.submit(inc_slow, 2)
+    lock = Lock()
+
+    await lock.acquire()
+
+    x = c.submit(inc_slow, 1, lock, workers=[a.address], allow_other_workers=True)
+
+    def reducer(*args):
+        return get_worker().address
 
-    def reducer(x, y):
-        return x + y
+    def finalizer(addr):
+        if swap_data_insert_order:
+            w = get_worker()
+            new_data = {k: w.data[k] for k in list(w.data.keys())[::-1]}
+            w.data = new_data
+        return addr
 
-    z = c.submit(reducer, x, y)
+    z = c.submit(reducer, x, key="reducer", workers=[a.address])
+    fin = c.submit(finalizer, z, key="final", workers=[a.address])
 
     s.rpc = await FlakyConnectionPool(failing_connections=1)
 
@@ -2227,9 +2256,31 @@ def reducer(x, y):
         ) as client_logger:
             # Gather using the client (as an ordinary user would)
             # Upon a missing key, the client will reschedule the computations
-            res = await c.gather(z)
+            res = None
+            while not res:
+                try:
+                    # This reduces test runtime by about a second since we're
+                    # depending on a worker heartbeat for a reconnect.
+                    res = await asyncio.wait_for(fin, 0.1)
+                except asyncio.TimeoutError:
+                    await a.heartbeat()
+
+    # Ensure that we're actually reusing the result
+    assert res == a.address
+    await lock.release()
+
+    while not all(all(ts.state == "memory" for ts in w.tasks.values()) for w in [a, b]):
+        await asyncio.sleep(0.01)
 
-    assert res == 5
+    assert z.key in a.tasks
+    assert z.key not in b.tasks
+    assert b.executed_count == 1
+    for w in [a, b]:
+        assert x.key in w.tasks
+        assert w.tasks[x.key].state == "memory"
+    while not len(s.tasks[x.key].who_has) == 2:
+        await asyncio.sleep(0.01)
+    assert len(s.tasks[z.key].who_has) == 1
 
     sched_logger = sched_logger.getvalue()
     client_logger = client_logger.getvalue()
@@ -2245,24 +2296,6 @@ def reducer(x, y):
     # is rather an artifact and not the intention
     assert "Workers don't have promised key" in sched_logger
 
-    # Once the worker reconnects, it will also submit the keys it holds such
-    # that the scheduler again knows about the result.
-    # The final reduce step should then be used from the re-connected worker
-    # instead of recomputing it.
-    transitions_to_processing = [
-        (key, start, timestamp)
-        for key, start, finish, recommendations, timestamp in s.transition_log
-        if finish == "processing" and "reducer" in key
-    ]
-    assert len(transitions_to_processing) == 1
-
-    finish_processing_transitions = 0
-    for transition in s.transition_log:
-        key, start, finish, recommendations, timestamp = transition
-        if "reducer" in key and finish == "processing":
-            finish_processing_transitions += 1
-    assert finish_processing_transitions == 1
-
 
 @gen_cluster(client=True)
 async def test_too_many_groups(c, s, a, b):
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 2f2aa82fd47..7ba5ef166cd 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -2372,6 +2372,71 @@ async def test_hold_on_to_replicas(c, s, *workers):
         await asyncio.sleep(0.01)
 
 
+@gen_cluster(client=True)
+async def test_worker_reconnects_mid_compute(c, s, a, b):
+    """
+    This test ensure that if a worker disconnects while computing a result, the scheduler will still accept the result.
+
+    There is also an edge case tested which ensures that the reconnect is
+    successful if a task is currently executing, see
+    https://github.com/dask/distributed/issues/5078
+
+    See also distributed.tests.test_scheduler.py::test_gather_allow_worker_reconnect
+    """
+    with captured_logger("distributed.scheduler") as s_logs:
+        # Let's put one task in memory to ensure the reconnect has tasks in
+        # different states
+        f1 = c.submit(inc, 1, workers=[a.address], allow_other_workers=True)
+        await f1
+        a_address = a.address
+        a.periodic_callbacks["heartbeat"].stop()
+        await a.heartbeat()
+        a.heartbeat_active = True
+
+        from distributed import Lock
+
+        def fast_on_a(lock):
+            w = get_worker()
+            import time
+
+            if w.address != a_address:
+                lock.acquire()
+            else:
+                time.sleep(1)
+
+        lock = Lock()
+        # We want to be sure that A is the only one computing this result
+        async with lock:
+
+            f2 = c.submit(
+                fast_on_a, lock, workers=[a.address], allow_other_workers=True
+            )
+
+            while f2.key not in a.tasks:
+                await asyncio.sleep(0.01)
+
+            await s.stream_comms[a.address].close()
+
+            assert len(s.workers) == 1
+            a.heartbeat_active = False
+            await a.heartbeat()
+            assert len(s.workers) == 2
+            # Since B is locked, this is ensured to originate from A
+            await f2
+
+    assert "Unexpected worker completed task" in s_logs.getvalue()
+
+    while not len(s.tasks[f2.key].who_has) == 2:
+        await asyncio.sleep(0.001)
+
+    # Ensure that all keys have been properly registered and will also be
+    # cleaned up nicely.
+    del f1, f2
+
+    while any(w.tasks for w in [a, b]):
+        await asyncio.sleep(0.001)
+
+
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)])
 async def test_forget_dependents_after_release(c, s, a):
 
diff --git a/distributed/worker.py b/distributed/worker.py
index 806699845f2..c182a84249d 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -699,6 +699,7 @@ def __init__(
         stream_handlers = {
             "close": self.close,
             "compute-task": self.add_task,
+            "cancel-compute": self.cancel_compute,
             "free-keys": self.handle_free_keys,
             "superfluous-data": self.handle_superfluous_data,
             "steal-request": self.steal_request,
@@ -901,7 +902,14 @@ async def _register_with_scheduler(self):
                         keys=list(self.data),
                         nthreads=self.nthreads,
                         name=self.name,
-                        nbytes={ts.key: ts.get_nbytes() for ts in self.tasks.values()},
+                        nbytes={
+                            ts.key: ts.get_nbytes()
+                            for ts in self.tasks.values()
+                            # Only if the task is in memory this is a sensible
+                            # result since otherwise it simply submits the
+                            # default value
+                            if ts.state == "memory"
+                        },
                         types={k: typename(v) for k, v in self.data.items()},
                         now=time(),
                         resources=self.total_resources,
@@ -1544,6 +1552,22 @@ async def set_resources(self, **resources):
     # Task Management #
     ###################
 
+    def cancel_compute(self, key, reason):
+        """
+        Cancel a task on a best effort basis. This is only possible while a task
+        is in state `waiting` or `ready`.
+        Nothing will happen otherwise.
+        """
+        ts = self.tasks.get(key)
+        if ts and ts.state in ("waiting", "ready"):
+            self.log.append((key, "cancel-compute", reason))
+            ts.scheduler_holds_ref = False
+            # All possible dependents of TS should not be in state Processing on
+            # scheduler side and therefore should not be assigned to a worker,
+            # yet.
+            assert not ts.dependents
+            self.release_key(key, reason=reason, report=False)
+
     def add_task(
         self,
         key,

From 33b795f8510eec7e5a1f4fa29c5ba55c6c87a94c Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Wed, 28 Jul 2021 13:35:02 +0100
Subject: [PATCH 1397/1550] Unit tests to use a random port for the dashboard
 (#5060)

* tests to bind on port 0 1/2

* tests to bind on port 0 2/2

* Self-review

* simplify test

* Stability enhancements

* Revisit timeouts and slow markers

* test_cancellation

* relax timings

* Remove rendundant port=0
---
 distributed/cli/tests/test_dask_scheduler.py  |  14 +-
 distributed/cli/tests/test_dask_spec.py       |  14 +-
 distributed/cli/tests/test_dask_worker.py     |  84 ++--
 distributed/client.py                         |   6 +-
 distributed/comm/tests/test_ucx.py            |  34 +-
 distributed/comm/tests/test_ws.py             |  81 ++--
 .../dashboard/tests/test_scheduler_bokeh.py   |   2 +-
 distributed/deploy/tests/test_adaptive.py     |  73 ++--
 distributed/deploy/tests/test_local.py        | 302 +++++++-------
 .../deploy/tests/test_slow_adaptive.py        |   2 +-
 distributed/deploy/tests/test_spec_cluster.py | 105 +++--
 distributed/deploy/tests/test_ssh.py          |  18 +-
 .../diagnostics/tests/test_progressbar.py     |  20 +-
 .../tests/test_scheduler_plugin.py            |   8 +-
 distributed/node.py                           |   6 +-
 distributed/tests/test_actor.py               |   3 +-
 distributed/tests/test_client.py              | 163 ++++----
 distributed/tests/test_client_executor.py     |  16 +-
 distributed/tests/test_client_loop.py         |   2 +-
 distributed/tests/test_collections.py         |   3 +-
 distributed/tests/test_failed_workers.py      |   4 +-
 distributed/tests/test_locks.py               |  17 +-
 distributed/tests/test_multi_locks.py         |  25 +-
 distributed/tests/test_nanny.py               | 132 +++---
 distributed/tests/test_preload.py             |  77 ++--
 distributed/tests/test_queues.py              |   2 +-
 distributed/tests/test_resources.py           |   2 +-
 distributed/tests/test_scheduler.py           | 235 +++++------
 distributed/tests/test_semaphore.py           |   7 +-
 distributed/tests/test_stress.py              |   4 +-
 distributed/tests/test_tls_functional.py      |  16 +-
 distributed/tests/test_utils_perf.py          |   1 +
 distributed/tests/test_utils_test.py          |   4 +-
 distributed/tests/test_variable.py            |   2 +-
 distributed/tests/test_worker.py              | 381 ++++++++----------
 distributed/tests/test_worker_client.py       |   2 +-
 distributed/utils_test.py                     |   9 +-
 37 files changed, 876 insertions(+), 1000 deletions(-)

diff --git a/distributed/cli/tests/test_dask_scheduler.py b/distributed/cli/tests/test_dask_scheduler.py
index 6b310d8e544..a04e84d4370 100644
--- a/distributed/cli/tests/test_dask_scheduler.py
+++ b/distributed/cli/tests/test_dask_scheduler.py
@@ -26,20 +26,17 @@
 
 
 def test_defaults(loop):
-    with popen(["dask-scheduler", "--no-dashboard"]) as proc:
+    with popen(["dask-scheduler"]):
 
         async def f():
             # Default behaviour is to listen on all addresses
             await assert_can_connect_from_everywhere_4_6(8786, timeout=5.0)
 
-        with Client("127.0.0.1:%d" % Scheduler.default_port, loop=loop) as c:
+        with Client(f"127.0.0.1:{Scheduler.default_port}", loop=loop) as c:
             c.sync(f)
 
         response = requests.get("http://127.0.0.1:8787/status/")
-        assert response.status_code == 404
-
-    with pytest.raises(Exception):
-        response = requests.get("http://127.0.0.1:9786/info.json")
+        response.raise_for_status()
 
 
 def test_hostport(loop):
@@ -55,9 +52,8 @@ async def f():
 
 
 def test_no_dashboard(loop):
-    pytest.importorskip("bokeh")
-    with popen(["dask-scheduler", "--no-dashboard"]) as proc:
-        with Client("127.0.0.1:%d" % Scheduler.default_port, loop=loop) as c:
+    with popen(["dask-scheduler", "--no-dashboard"]):
+        with Client(f"127.0.0.1:{Scheduler.default_port}", loop=loop):
             response = requests.get("http://127.0.0.1:8787/status/")
             assert response.status_code == 404
 
diff --git a/distributed/cli/tests/test_dask_spec.py b/distributed/cli/tests/test_dask_spec.py
index d90fccf954b..45f88c894b3 100644
--- a/distributed/cli/tests/test_dask_spec.py
+++ b/distributed/cli/tests/test_dask_spec.py
@@ -5,12 +5,12 @@
 
 from distributed import Client
 from distributed.scheduler import COMPILED
-from distributed.utils_test import popen
+from distributed.utils_test import gen_test, popen
 
 
 @pytest.mark.skipif(COMPILED, reason="Fails with cythonized scheduler")
-@pytest.mark.asyncio
-async def test_text(cleanup):
+@gen_test(timeout=120)
+async def test_text():
     with popen(
         [
             sys.executable,
@@ -19,7 +19,7 @@ async def test_text(cleanup):
             "--spec",
             '{"cls": "dask.distributed.Scheduler", "opts": {"port": 9373}}',
         ]
-    ) as sched:
+    ):
         with popen(
             [
                 sys.executable,
@@ -29,7 +29,7 @@ async def test_text(cleanup):
                 "--spec",
                 '{"cls": "dask.distributed.Worker", "opts": {"nanny": false, "nthreads": 3, "name": "foo"}}',
             ]
-        ) as w:
+        ):
             async with Client("tcp://localhost:9373", asynchronous=True) as client:
                 await client.wait_for_workers(1)
                 info = await client.scheduler.identity()
@@ -51,7 +51,7 @@ async def test_file(cleanup, tmp_path):
             f,
         )
 
-    with popen(["dask-scheduler", "--port", "9373", "--no-dashboard"]) as sched:
+    with popen(["dask-scheduler", "--port", "9373", "--no-dashboard"]):
         with popen(
             [
                 sys.executable,
@@ -61,7 +61,7 @@ async def test_file(cleanup, tmp_path):
                 "--spec-file",
                 fn,
             ]
-        ) as w:
+        ):
             async with Client("tcp://localhost:9373", asynchronous=True) as client:
                 await client.wait_for_workers(1)
                 info = await client.scheduler.identity()
diff --git a/distributed/cli/tests/test_dask_worker.py b/distributed/cli/tests/test_dask_worker.py
index 2d49b3b0977..d807791ea7d 100644
--- a/distributed/cli/tests/test_dask_worker.py
+++ b/distributed/cli/tests/test_dask_worker.py
@@ -17,11 +17,11 @@
 from distributed.deploy.utils import nprocesses_nthreads
 from distributed.metrics import time
 from distributed.utils import parse_ports, sync, tmpfile
-from distributed.utils_test import popen, terminate_process, wait_for_port
+from distributed.utils_test import gen_cluster, popen, terminate_process, wait_for_port
 
 
 def test_nanny_worker_ports(loop):
-    with popen(["dask-scheduler", "--port", "9359", "--no-dashboard"]) as sched:
+    with popen(["dask-scheduler", "--port", "9359", "--no-dashboard"]):
         with popen(
             [
                 "dask-worker",
@@ -34,7 +34,7 @@ def test_nanny_worker_ports(loop):
                 "5273",
                 "--no-dashboard",
             ]
-        ) as worker:
+        ):
             with Client("127.0.0.1:9359", loop=loop) as c:
                 start = time()
                 while True:
@@ -50,6 +50,7 @@ def test_nanny_worker_ports(loop):
                 )
 
 
+@pytest.mark.slow
 def test_nanny_worker_port_range(loop):
     with popen(["dask-scheduler", "--port", "9359", "--no-dashboard"]) as sched:
         nprocs = 3
@@ -69,7 +70,7 @@ def test_nanny_worker_port_range(loop):
                 nanny_port,
                 "--no-dashboard",
             ]
-        ) as worker:
+        ):
             with Client("127.0.0.1:9359", loop=loop) as c:
                 start = time()
                 while len(c.scheduler_info()["workers"]) < nprocs:
@@ -89,7 +90,7 @@ def get_port(dask_worker):
 
 
 def test_nanny_worker_port_range_too_many_workers_raises(loop):
-    with popen(["dask-scheduler", "--port", "9359", "--no-dashboard"]) as sched:
+    with popen(["dask-scheduler", "--port", "9359", "--no-dashboard"]):
         with popen(
             [
                 "dask-worker",
@@ -111,7 +112,7 @@ def test_nanny_worker_port_range_too_many_workers_raises(loop):
 
 
 def test_memory_limit(loop):
-    with popen(["dask-scheduler", "--no-dashboard"]) as sched:
+    with popen(["dask-scheduler", "--no-dashboard"]):
         with popen(
             [
                 "dask-worker",
@@ -120,7 +121,7 @@ def test_memory_limit(loop):
                 "2e3MB",
                 "--no-dashboard",
             ]
-        ) as worker:
+        ):
             with Client("127.0.0.1:8786", loop=loop) as c:
                 while not c.nthreads():
                     sleep(0.1)
@@ -131,7 +132,7 @@ def test_memory_limit(loop):
 
 
 def test_no_nanny(loop):
-    with popen(["dask-scheduler", "--no-dashboard"]) as sched:
+    with popen(["dask-scheduler", "--no-dashboard"]):
         with popen(
             ["dask-worker", "127.0.0.1:8786", "--no-nanny", "--no-dashboard"]
         ) as worker:
@@ -157,11 +158,11 @@ def test_no_reconnect(nanny, loop):
         start = time()
         while worker.poll() is None:
             sleep(0.1)
-            assert time() < start + 10
+            assert time() < start + 30
 
 
 def test_resources(loop):
-    with popen(["dask-scheduler", "--no-dashboard"]) as sched:
+    with popen(["dask-scheduler", "--no-dashboard"]):
         with popen(
             [
                 "dask-worker",
@@ -170,7 +171,7 @@ def test_resources(loop):
                 "--resources",
                 "A=1 B=2,C=3",
             ]
-        ) as worker:
+        ):
             with Client("127.0.0.1:8786", loop=loop) as c:
                 while not c.scheduler_info()["workers"]:
                     sleep(0.1)
@@ -182,7 +183,7 @@ def test_resources(loop):
 @pytest.mark.parametrize("nanny", ["--nanny", "--no-nanny"])
 def test_local_directory(loop, nanny):
     with tmpfile() as fn:
-        with popen(["dask-scheduler", "--no-dashboard"]) as sched:
+        with popen(["dask-scheduler", "--no-dashboard"]):
             with popen(
                 [
                     "dask-worker",
@@ -192,7 +193,7 @@ def test_local_directory(loop, nanny):
                     "--local-directory",
                     fn,
                 ]
-            ) as worker:
+            ):
                 with Client("127.0.0.1:8786", loop=loop, timeout=10) as c:
                     start = time()
                     while not c.scheduler_info()["workers"]:
@@ -206,9 +207,7 @@ def test_local_directory(loop, nanny):
 @pytest.mark.parametrize("nanny", ["--nanny", "--no-nanny"])
 def test_scheduler_file(loop, nanny):
     with tmpfile() as fn:
-        with popen(
-            ["dask-scheduler", "--no-dashboard", "--scheduler-file", fn]
-        ) as sched:
+        with popen(["dask-scheduler", "--no-dashboard", "--scheduler-file", fn]):
             with popen(
                 ["dask-worker", "--scheduler-file", fn, nanny, "--no-dashboard"]
             ):
@@ -221,7 +220,7 @@ def test_scheduler_file(loop, nanny):
 
 def test_scheduler_address_env(loop, monkeypatch):
     monkeypatch.setenv("DASK_SCHEDULER_ADDRESS", "tcp://127.0.0.1:8786")
-    with popen(["dask-scheduler", "--no-dashboard"]) as sched:
+    with popen(["dask-scheduler", "--no-dashboard"]):
         with popen(["dask-worker", "--no-dashboard"]):
             with Client(os.environ["DASK_SCHEDULER_ADDRESS"], loop=loop) as c:
                 start = time()
@@ -231,7 +230,7 @@ def test_scheduler_address_env(loop, monkeypatch):
 
 
 def test_nprocs_requires_nanny(loop):
-    with popen(["dask-scheduler", "--no-dashboard"]) as sched:
+    with popen(["dask-scheduler", "--no-dashboard"]):
         with popen(
             ["dask-worker", "127.0.0.1:8786", "--nprocs=2", "--no-nanny"]
         ) as worker:
@@ -242,31 +241,29 @@ def test_nprocs_requires_nanny(loop):
 
 
 def test_nprocs_negative(loop):
-    with popen(["dask-scheduler", "--no-dashboard"]) as sched:
-        with popen(["dask-worker", "127.0.0.1:8786", "--nprocs=-1"]) as worker:
+    with popen(["dask-scheduler", "--no-dashboard"]):
+        with popen(["dask-worker", "127.0.0.1:8786", "--nprocs=-1"]):
             with Client("tcp://127.0.0.1:8786", loop=loop) as c:
                 c.wait_for_workers(cpu_count(), timeout="10 seconds")
 
 
 def test_nprocs_auto(loop):
-    with popen(["dask-scheduler", "--no-dashboard"]) as sched:
-        with popen(["dask-worker", "127.0.0.1:8786", "--nprocs=auto"]) as worker:
+    with popen(["dask-scheduler", "--no-dashboard"]):
+        with popen(["dask-worker", "127.0.0.1:8786", "--nprocs=auto"]):
             with Client("tcp://127.0.0.1:8786", loop=loop) as c:
                 procs, _ = nprocesses_nthreads()
                 c.wait_for_workers(procs, timeout="10 seconds")
 
 
 def test_nprocs_expands_name(loop):
-    with popen(["dask-scheduler", "--no-dashboard"]) as sched:
-        with popen(
-            ["dask-worker", "127.0.0.1:8786", "--nprocs", "2", "--name", "0"]
-        ) as worker:
-            with popen(["dask-worker", "127.0.0.1:8786", "--nprocs", "2"]) as worker:
+    with popen(["dask-scheduler", "--no-dashboard"]):
+        with popen(["dask-worker", "127.0.0.1:8786", "--nprocs", "2", "--name", "0"]):
+            with popen(["dask-worker", "127.0.0.1:8786", "--nprocs", "2"]):
                 with Client("tcp://127.0.0.1:8786", loop=loop) as c:
                     start = time()
                     while len(c.scheduler_info()["workers"]) < 4:
                         sleep(0.2)
-                        assert time() < start + 10
+                        assert time() < start + 30
 
                     info = c.scheduler_info()
                     names = [d["name"] for d in info["workers"].values()]
@@ -281,7 +278,7 @@ def test_nprocs_expands_name(loop):
     "listen_address", ["tcp://0.0.0.0:39837", "tcp://127.0.0.2:39837"]
 )
 def test_contact_listen_address(loop, nanny, listen_address):
-    with popen(["dask-scheduler", "--no-dashboard"]) as sched:
+    with popen(["dask-scheduler", "--no-dashboard"]):
         with popen(
             [
                 "dask-worker",
@@ -293,7 +290,7 @@ def test_contact_listen_address(loop, nanny, listen_address):
                 "--listen-address",
                 listen_address,
             ]
-        ) as worker:
+        ):
             with Client("127.0.0.1:8786") as client:
                 while not client.nthreads():
                     sleep(0.1)
@@ -313,14 +310,14 @@ def func(dask_worker):
 @pytest.mark.parametrize("nanny", ["--nanny", "--no-nanny"])
 @pytest.mark.parametrize("host", ["127.0.0.2", "0.0.0.0"])
 def test_respect_host_listen_address(loop, nanny, host):
-    with popen(["dask-scheduler", "--no-dashboard"]) as sched:
+    with popen(["dask-scheduler", "--no-dashboard"]):
         with popen(
             ["dask-worker", "127.0.0.1:8786", nanny, "--no-dashboard", "--host", host]
         ) as worker:
             with Client("127.0.0.1:8786") as client:
                 while not client.nthreads():
                     sleep(0.1)
-                info = client.scheduler_info()
+                client.scheduler_info()
 
                 # roundtrip works
                 assert client.submit(lambda x: x + 1, 10).result() == 11
@@ -341,7 +338,7 @@ def test_dashboard_non_standard_ports(loop):
     except ImportError:
         proxy_exists = False
 
-    with popen(["dask-scheduler", "--port", "3449"]) as s:
+    with popen(["dask-scheduler", "--port", "3449"]):
         with popen(
             [
                 "dask-worker",
@@ -351,7 +348,7 @@ def test_dashboard_non_standard_ports(loop):
                 "--host",
                 "127.0.0.1",
             ]
-        ) as proc:
+        ):
             with Client("127.0.0.1:3449", loop=loop) as c:
                 c.wait_for_workers(1)
                 pass
@@ -406,14 +403,13 @@ def test_bokeh_deprecation():
             pass
 
 
-@pytest.mark.asyncio
-async def test_integer_names(cleanup):
-    async with Scheduler(port=0) as s:
-        with popen(["dask-worker", s.address, "--name", "123"]) as worker:
-            while not s.workers:
-                await asyncio.sleep(0.01)
-            [ws] = s.workers.values()
-            assert ws.name == 123
+@gen_cluster(nthreads=[])
+async def test_integer_names(s):
+    with popen(["dask-worker", s.address, "--name", "123"]):
+        while not s.workers:
+            await asyncio.sleep(0.01)
+        [ws] = s.workers.values()
+        assert ws.name == 123
 
 
 @pytest.mark.asyncio
@@ -438,7 +434,7 @@ class MyWorker(Worker):
     else:
         env["PYTHONPATH"] = tmpdir
 
-    async with Scheduler(port=0) as s:
+    async with Scheduler(dashboard_address=":0") as s:
         async with Client(s.address, asynchronous=True) as c:
             with popen(
                 [
@@ -449,7 +445,7 @@ class MyWorker(Worker):
                     "myworker.MyWorker",
                 ],
                 env=env,
-            ) as worker:
+            ):
                 await c.wait_for_workers(1)
 
                 def worker_type(dask_worker):
diff --git a/distributed/client.py b/distributed/client.py
index 75937435f07..39153bd6228 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -1474,10 +1474,10 @@ def close(self, timeout=no_default):
                 pc.stop()
 
         if self.asynchronous:
-            future = self._close()
+            coro = self._close()
             if timeout:
-                future = asyncio.wait_for(future, timeout)
-            return future
+                coro = asyncio.wait_for(coro, timeout)
+            return coro
 
         if self._start_arg is None:
             with suppress(AttributeError):
diff --git a/distributed/comm/tests/test_ucx.py b/distributed/comm/tests/test_ucx.py
index 64948b68028..d463104dc79 100644
--- a/distributed/comm/tests/test_ucx.py
+++ b/distributed/comm/tests/test_ucx.py
@@ -4,12 +4,12 @@
 
 ucp = pytest.importorskip("ucp")
 
-from distributed import Client, Scheduler, Worker, wait
+from distributed import Client, Scheduler, wait
 from distributed.comm import connect, listen, parse_address, ucx
 from distributed.comm.registry import backends, get_backend
 from distributed.deploy.local import LocalCluster
 from distributed.protocol import to_serialize
-from distributed.utils_test import gen_test, inc
+from distributed.utils_test import gen_cluster, gen_test, inc
 
 try:
     HOST = ucp.get_address()
@@ -300,31 +300,25 @@ async def test_stress(cleanup):
                 await wait(x)
 
 
-@pytest.mark.asyncio
-async def test_simple(cleanup):
-    async with Scheduler(protocol="ucx") as s:
-        async with Worker(s.address) as a:
-            async with Client(s.address, asynchronous=True) as c:
-                result = await c.submit(lambda x: x + 1, 10)
-                assert result == 11
+@gen_cluster(client=True, scheduler_kwargs={"protocol": "ucx"})
+async def test_simple(c, s, a, b):
+    assert s.address.startswith("ucx://")
+    assert await c.submit(lambda x: x + 1, 10) == 11
 
 
-@pytest.mark.asyncio
-async def test_transpose(cleanup):
+@gen_cluster(client=True, scheduler_kwargs={"protocol": "ucx"})
+async def test_transpose(c, s, a, b):
     da = pytest.importorskip("dask.array")
 
-    async with Scheduler(protocol="ucx") as s:
-        async with Worker(s.address) as a, Worker(s.address) as b:
-            async with Client(s.address, asynchronous=True) as c:
-                x = da.ones((10000, 10000), chunks=(1000, 1000)).persist()
-                await x
-
-                y = (x + x.T).sum()
-                await y
+    assert s.address.startswith("ucx://")
+    x = da.ones((10000, 10000), chunks=(1000, 1000)).persist()
+    await x
+    y = (x + x.T).sum()
+    await y
 
 
 @pytest.mark.asyncio
 @pytest.mark.parametrize("port", [0, 1234])
 async def test_ucx_protocol(cleanup, port):
-    async with Scheduler(protocol="ucx", port=port) as s:
+    async with Scheduler(protocol="ucx", port=port, dashboard_address=":0") as s:
         assert s.address.startswith("ucx://")
diff --git a/distributed/comm/tests/test_ws.py b/distributed/comm/tests/test_ws.py
index dee4c8f6a8e..08d1cbce0ee 100644
--- a/distributed/comm/tests/test_ws.py
+++ b/distributed/comm/tests/test_ws.py
@@ -14,6 +14,7 @@
 from distributed.security import Security
 from distributed.utils_test import (
     gen_cluster,
+    gen_test,
     get_client_ssl_context,
     get_server_ssl_context,
     inc,
@@ -30,8 +31,8 @@ def test_registered():
     assert isinstance(backend, ws.WSBackend)
 
 
-@pytest.mark.asyncio
-async def test_listen_connect(cleanup):
+@gen_test()
+async def test_listen_connect():
     async def handle_comm(comm):
         while True:
             msg = await comm.read()
@@ -46,8 +47,8 @@ async def handle_comm(comm):
         await comm.close()
 
 
-@pytest.mark.asyncio
-async def test_listen_connect_wss(cleanup):
+@gen_test()
+async def test_listen_connect_wss():
     async def handle_comm(comm):
         while True:
             msg = await comm.read()
@@ -66,8 +67,8 @@ async def handle_comm(comm):
         await comm.close()
 
 
-@pytest.mark.asyncio
-async def test_expect_ssl_context(cleanup):
+@gen_test()
+async def test_expect_ssl_context():
     server_ctx = get_server_ssl_context()
 
     async with listen("wss://", lambda comm: comm, ssl_context=server_ctx) as listener:
@@ -75,8 +76,8 @@ async def test_expect_ssl_context(cleanup):
             comm = await connect(listener.contact_address)
 
 
-@pytest.mark.asyncio
-async def test_expect_scheduler_ssl_when_sharing_server(cleanup):
+@gen_test()
+async def test_expect_scheduler_ssl_when_sharing_server():
     with tempfile.TemporaryDirectory() as tempdir:
         key_path = os.path.join(tempdir, "dask.pem")
         cert_path = os.path.join(tempdir, "dask.crt")
@@ -90,41 +91,30 @@ async def test_expect_scheduler_ssl_when_sharing_server(cleanup):
         }
         with dask.config.set(c):
             with pytest.raises(RuntimeError):
-                async with Scheduler(protocol="ws://", dashboard=True, port=8787) as s:
+                async with Scheduler(protocol="ws://", dashboard=True, port=8787):
                     pass
 
 
-@pytest.mark.asyncio
-async def test_roundtrip(cleanup):
-    async with Scheduler(protocol="ws://") as s:
-        async with Worker(s.address) as w:
-            async with Client(s.address, asynchronous=True) as c:
-                assert c.scheduler.address.startswith("ws://")
-                assert w.address.startswith("ws://")
-                future = c.submit(inc, 1)
-                result = await future
-                assert result == 2
+@gen_cluster(client=True, scheduler_kwargs={"protocol": "ws://"})
+async def test_roundtrip(c, s, a, b):
+    assert a.address.startswith("ws://")
+    assert b.address.startswith("ws://")
+    assert c.scheduler.address.startswith("ws://")
+    assert await c.submit(inc, 1) == 2
 
 
-@pytest.mark.asyncio
-async def test_collections(cleanup):
+@gen_cluster(client=True, scheduler_kwargs={"protocol": "ws://"})
+async def test_collections(c, s, a, b):
     da = pytest.importorskip("dask.array")
-    async with Scheduler(protocol="ws://") as s:
-        async with Worker(s.address) as a:
-            async with Worker(s.address) as b:
-                async with Client(s.address, asynchronous=True) as c:
-                    x = da.random.random((1000, 1000), chunks=(100, 100))
-                    x = x + x.T
-                    await x.persist()
+    x = da.random.random((1000, 1000), chunks=(100, 100))
+    x = x + x.T
+    await x.persist()
 
 
-@pytest.mark.asyncio
-async def test_large_transfer(cleanup):
+@gen_cluster(client=True, scheduler_kwargs={"protocol": "ws://"})
+async def test_large_transfer(c, s, a, b):
     np = pytest.importorskip("numpy")
-    async with Scheduler(protocol="ws://") as s:
-        async with Worker(s.address, protocol="ws://"):
-            async with Client(s.address, asynchronous=True) as c:
-                await c.scatter(np.random.random(1_000_000))
+    await c.scatter(np.random.random(1_000_000))
 
 
 @pytest.mark.asyncio
@@ -182,17 +172,21 @@ async def test_http_and_comm_server(cleanup, dashboard, protocol, security, port
     ],
 )
 async def test_connection_made_with_extra_conn_args(cleanup, protocol, security):
-    async with Scheduler(protocol=protocol, security=security) as s:
+    async with Scheduler(
+        protocol=protocol, security=security, dashboard_address=":0"
+    ) as s:
         connection_args = security.get_connection_args("worker")
         comm = await connect(s.address, **connection_args)
         assert comm.sock.request.headers.get("Authorization") == "Token abcd"
         await comm.close()
 
 
-@pytest.mark.asyncio
-async def test_quiet_close(cleanup):
+@gen_test()
+async def test_quiet_close():
     with warnings.catch_warnings(record=True) as record:
-        async with Client(protocol="ws", processes=False, asynchronous=True) as c:
+        async with Client(
+            protocol="ws", processes=False, asynchronous=True, dashboard_address=":0"
+        ):
             pass
 
     # For some reason unrelated @coroutine warnings are showing up
@@ -201,10 +195,7 @@ async def test_quiet_close(cleanup):
     assert not record, record[0].message
 
 
-@gen_cluster(
-    client=True,
-    scheduler_kwargs={"protocol": "ws://"},
-)
+@gen_cluster(client=True, scheduler_kwargs={"protocol": "ws://"})
 async def test_ws_roundtrip(c, s, a, b):
     x = np.arange(100)
     future = await c.scatter(x)
@@ -212,11 +203,7 @@ async def test_ws_roundtrip(c, s, a, b):
     assert (x == y).all()
 
 
-@gen_cluster(
-    client=True,
-    security=security,
-    scheduler_kwargs={"protocol": "wss://"},
-)
+@gen_cluster(client=True, security=security, scheduler_kwargs={"protocol": "wss://"})
 async def test_wss_roundtrip(c, s, a, b):
     x = np.arange(100)
     future = await c.scatter(x)
diff --git a/distributed/dashboard/tests/test_scheduler_bokeh.py b/distributed/dashboard/tests/test_scheduler_bokeh.py
index f003c46d2a8..473595e0d73 100644
--- a/distributed/dashboard/tests/test_scheduler_bokeh.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh.py
@@ -58,7 +58,7 @@ async def test_simple(c, s, a, b):
 
     http_client = AsyncHTTPClient()
     for suffix in applications:
-        response = await http_client.fetch("http://localhost:%d%s" % (port, suffix))
+        response = await http_client.fetch(f"http://localhost:{port}{suffix}")
         body = response.body.decode()
         assert "bokeh" in body.lower()
         assert not re.search("href=./", body)  # no absolute links
diff --git a/distributed/deploy/tests/test_adaptive.py b/distributed/deploy/tests/test_adaptive.py
index 4ad27d85df1..d463796dc5e 100644
--- a/distributed/deploy/tests/test_adaptive.py
+++ b/distributed/deploy/tests/test_adaptive.py
@@ -7,7 +7,15 @@
 
 import dask
 
-from distributed import Adaptive, Client, LocalCluster, SpecCluster, Worker, wait
+from distributed import (
+    Adaptive,
+    Client,
+    LocalCluster,
+    Scheduler,
+    SpecCluster,
+    Worker,
+    wait,
+)
 from distributed.compatibility import WINDOWS
 from distributed.metrics import time
 from distributed.utils_test import async_wait_for, clean, gen_test, slowinc
@@ -16,9 +24,8 @@
 def test_adaptive_local_cluster(loop):
     with LocalCluster(
         n_workers=0,
-        scheduler_port=0,
         silence_logs=False,
-        dashboard_address=None,
+        dashboard_address=":0",
         loop=loop,
     ) as cluster:
         alc = cluster.adapt(interval="100 ms")
@@ -45,10 +52,9 @@ def test_adaptive_local_cluster(loop):
 async def test_adaptive_local_cluster_multi_workers():
     async with LocalCluster(
         n_workers=0,
-        scheduler_port=0,
         silence_logs=False,
         processes=False,
-        dashboard_address=None,
+        dashboard_address=":0",
         asynchronous=True,
     ) as cluster:
 
@@ -76,8 +82,8 @@ async def test_adaptive_local_cluster_multi_workers():
 
 
 @pytest.mark.xfail(reason="changed API")
-@pytest.mark.asyncio
-async def test_adaptive_scale_down_override(cleanup):
+@gen_test()
+async def test_adaptive_scale_down_override():
     class TestAdaptive(Adaptive):
         def __init__(self, *args, **kwargs):
             self.min_size = kwargs.pop("min_size", 0)
@@ -95,7 +101,9 @@ class TestCluster(LocalCluster):
         def scale_up(self, n, **kwargs):
             assert False
 
-    async with TestCluster(n_workers=10, processes=False, asynchronous=True) as cluster:
+    async with TestCluster(
+        n_workers=10, processes=False, asynchronous=True, dashboard_address=":0"
+    ) as cluster:
         ta = cluster.adapt(
             min_size=2, interval=0.1, scale_factor=2, Adaptive=TestAdaptive
         )
@@ -110,10 +118,9 @@ def scale_up(self, n, **kwargs):
 async def test_min_max():
     cluster = await LocalCluster(
         n_workers=0,
-        scheduler_port=0,
         silence_logs=False,
         processes=False,
-        dashboard_address=None,
+        dashboard_address=":0",
         asynchronous=True,
         threads_per_worker=1,
     )
@@ -167,9 +174,8 @@ async def test_avoid_churn(cleanup):
         n_workers=0,
         asynchronous=True,
         processes=False,
-        scheduler_port=0,
         silence_logs=False,
-        dashboard_address=None,
+        dashboard_address=":0",
     ) as cluster:
         async with Client(cluster, asynchronous=True) as client:
             adapt = cluster.adapt(interval="20 ms", wait_count=5)
@@ -192,9 +198,8 @@ async def test_adapt_quickly():
         n_workers=0,
         asynchronous=True,
         processes=False,
-        scheduler_port=0,
         silence_logs=False,
-        dashboard_address=None,
+        dashboard_address=":0",
     )
     client = await Client(cluster, asynchronous=True)
     adapt = cluster.adapt(interval="20 ms", wait_count=5, maximum=10)
@@ -242,12 +247,11 @@ async def test_adapt_quickly():
 async def test_adapt_down():
     """Ensure that redefining adapt with a lower maximum removes workers"""
     async with LocalCluster(
-        0,
+        n_workers=0,
         asynchronous=True,
         processes=False,
-        scheduler_port=0,
         silence_logs=False,
-        dashboard_address=None,
+        dashboard_address=":0",
     ) as cluster:
         async with Client(cluster, asynchronous=True) as client:
             cluster.adapt(interval="20ms", maximum=5)
@@ -270,11 +274,10 @@ async def test_no_more_workers_than_tasks():
         {"distributed.scheduler.default-task-durations": {"slowinc": 1000}}
     ):
         async with LocalCluster(
-            0,
-            scheduler_port=0,
+            n_workers=0,
             silence_logs=False,
             processes=False,
-            dashboard_address=None,
+            dashboard_address=":0",
             asynchronous=True,
         ) as cluster:
             adapt = cluster.adapt(minimum=0, maximum=4, interval="10 ms")
@@ -287,7 +290,7 @@ def test_basic_no_loop(loop):
     with clean(threads=False):
         try:
             with LocalCluster(
-                0, scheduler_port=0, silence_logs=False, dashboard_address=None
+                n_workers=0, silence_logs=False, dashboard_address=":0"
             ) as cluster:
                 with Client(cluster) as client:
                     cluster.adapt()
@@ -309,9 +312,8 @@ async def test_target_duration():
             n_workers=0,
             asynchronous=True,
             processes=False,
-            scheduler_port=0,
             silence_logs=False,
-            dashboard_address=None,
+            dashboard_address=":0",
         ) as cluster:
             adapt = cluster.adapt(interval="20ms", minimum=2, target_duration="5s")
             async with Client(cluster, asynchronous=True) as client:
@@ -327,6 +329,7 @@ async def test_target_duration():
 async def test_worker_keys(cleanup):
     """Ensure that redefining adapt with a lower maximum removes workers"""
     async with SpecCluster(
+        scheduler={"cls": Scheduler, "options": {"dashboard_address": ":0"}},
         workers={
             "a-1": {"cls": Worker},
             "a-2": {"cls": Worker},
@@ -354,13 +357,12 @@ def key(ws):
 @pytest.mark.asyncio
 async def test_adapt_cores_memory(cleanup):
     async with LocalCluster(
-        0,
+        n_workers=0,
         threads_per_worker=2,
         memory_limit="3 GB",
-        scheduler_port=0,
         silence_logs=False,
         processes=False,
-        dashboard_address=None,
+        dashboard_address=":0",
         asynchronous=True,
     ) as cluster:
         adapt = cluster.adapt(minimum_cores=3, maximum_cores=9)
@@ -395,13 +397,12 @@ def test_adaptive_config():
 @pytest.mark.asyncio
 async def test_update_adaptive(cleanup):
     async with LocalCluster(
-        0,
+        n_workers=0,
         threads_per_worker=2,
         memory_limit="3 GB",
-        scheduler_port=0,
         silence_logs=False,
         processes=False,
-        dashboard_address=None,
+        dashboard_address=":0",
         asynchronous=True,
     ) as cluster:
         first = cluster.adapt(maxmimum=1)
@@ -415,7 +416,11 @@ async def test_update_adaptive(cleanup):
 async def test_adaptive_no_memory_limit(cleanup):
     """Make sure that adapt() does not keep creating workers when no memory limit is set."""
     async with LocalCluster(
-        n_workers=0, threads_per_worker=1, memory_limit=0, asynchronous=True
+        n_workers=0,
+        threads_per_worker=1,
+        memory_limit=0,
+        asynchronous=True,
+        dashboard_address=":0",
     ) as cluster:
         cluster.adapt(minimum=1, maximum=10, interval="1 ms")
         async with Client(cluster, asynchronous=True) as client:
@@ -447,7 +452,9 @@ async def _():
 
             return self.sync(_)
 
-    async with RequiresAwaitCluster(n_workers=0, asynchronous=True) as cluster:
+    async with RequiresAwaitCluster(
+        n_workers=0, asynchronous=True, dashboard_address=":0"
+    ) as cluster:
         async with Client(cluster, asynchronous=True) as client:
             futures = client.map(slowinc, range(5), delay=0.05)
             assert len(cluster.workers) == 0
@@ -465,7 +472,9 @@ async def test_adaptive_stopped():
     We should ensure that the adapt PC is actually stopped once the cluster
     stops.
     """
-    async with LocalCluster(n_workers=0, asynchronous=True) as cluster:
+    async with LocalCluster(
+        n_workers=0, asynchronous=True, dashboard_address=":0"
+    ) as cluster:
         instance = cluster.adapt(interval="10ms")
         assert instance.periodic_callback is not None
 
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index bddd6b6af0b..80bb39e45ca 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -5,7 +5,6 @@
 import unittest
 import weakref
 from distutils.version import LooseVersion
-from functools import partial
 from threading import Lock
 from time import sleep
 
@@ -41,11 +40,10 @@
 
 def test_simple(loop):
     with LocalCluster(
-        4,
-        scheduler_port=0,
+        n_workers=4,
         processes=False,
         silence_logs=False,
-        dashboard_address=None,
+        dashboard_address=":0",
         loop=loop,
     ) as c:
         with Client(c) as e:
@@ -58,7 +56,12 @@ def test_simple(loop):
 
 
 def test_local_cluster_supports_blocked_handlers(loop):
-    with LocalCluster(blocked_handlers=["run_function"], n_workers=0, loop=loop) as c:
+    with LocalCluster(
+        blocked_handlers=["run_function"],
+        n_workers=0,
+        loop=loop,
+        dashboard_address=":0",
+    ) as c:
         with Client(c) as client:
             with pytest.raises(ValueError) as exc:
                 client.run_on_scheduler(lambda x: x, 42)
@@ -69,7 +72,7 @@ def test_local_cluster_supports_blocked_handlers(loop):
 
 
 def test_close_twice():
-    with LocalCluster() as cluster:
+    with LocalCluster(dashboard_address=":0") as cluster:
         with Client(cluster.scheduler_address) as client:
             f = client.map(inc, range(100))
             client.gather(f)
@@ -84,10 +87,9 @@ def test_close_twice():
 def test_procs():
     with LocalCluster(
         n_workers=2,
-        scheduler_port=0,
         processes=False,
         threads_per_worker=3,
-        dashboard_address=None,
+        dashboard_address=":0",
         silence_logs=False,
     ) as c:
         assert len(c.workers) == 2
@@ -99,10 +101,9 @@ def test_procs():
 
     with LocalCluster(
         n_workers=2,
-        scheduler_port=0,
         processes=True,
         threads_per_worker=3,
-        dashboard_address=None,
+        dashboard_address=":0",
         silence_logs=False,
     ) as c:
         assert len(c.workers) == 2
@@ -121,7 +122,7 @@ def test_move_unserializable_data():
     transports.
     """
     with LocalCluster(
-        processes=False, silence_logs=False, dashboard_address=None
+        processes=False, silence_logs=False, dashboard_address=":0"
     ) as cluster:
         assert cluster.scheduler_address.startswith("inproc://")
         assert cluster.workers[0].address.startswith("inproc://")
@@ -137,7 +138,7 @@ def test_transports_inproc():
     Test the transport chosen by LocalCluster depending on arguments.
     """
     with LocalCluster(
-        1, processes=False, silence_logs=False, dashboard_address=None
+        n_workers=1, processes=False, silence_logs=False, dashboard_address=":0"
     ) as c:
         assert c.scheduler_address.startswith("inproc://")
         assert c.workers[0].address.startswith("inproc://")
@@ -148,7 +149,7 @@ def test_transports_inproc():
 def test_transports_tcp():
     # Have nannies => need TCP
     with LocalCluster(
-        1, processes=True, silence_logs=False, dashboard_address=None
+        n_workers=1, processes=True, silence_logs=False, dashboard_address=":0"
     ) as c:
         assert c.scheduler_address.startswith("tcp://")
         assert c.workers[0].address.startswith("tcp://")
@@ -159,11 +160,11 @@ def test_transports_tcp():
 def test_transports_tcp_port():
     # Scheduler port specified => need TCP
     with LocalCluster(
-        1,
+        n_workers=1,
         processes=False,
         scheduler_port=8786,
         silence_logs=False,
-        dashboard_address=None,
+        dashboard_address=":0",
     ) as c:
 
         assert c.scheduler_address == "tcp://127.0.0.1:8786"
@@ -173,13 +174,13 @@ def test_transports_tcp_port():
 
 
 class LocalTest(ClusterTest, unittest.TestCase):
-    Cluster = partial(LocalCluster, silence_logs=False, dashboard_address=None)
-    kwargs = {"dashboard_address": None, "processes": False}
+    Cluster = LocalCluster
+    kwargs = {"silence_logs": False, "dashboard_address": ":0", "processes": False}
 
 
 def test_Client_with_local(loop):
     with LocalCluster(
-        1, scheduler_port=0, silence_logs=False, dashboard_address=None, loop=loop
+        n_workers=1, silence_logs=False, dashboard_address=":0", loop=loop
     ) as c:
         with Client(c) as e:
             assert len(e.nthreads()) == len(c.workers)
@@ -187,7 +188,7 @@ def test_Client_with_local(loop):
 
 
 def test_Client_solo(loop):
-    with Client(loop=loop, silence_logs=False) as c:
+    with Client(loop=loop, silence_logs=False, dashboard_address=":0") as c:
         pass
     assert c.cluster.status == Status.closed
 
@@ -221,14 +222,20 @@ async def test_duplicate_clients():
 
 
 def test_Client_kwargs(loop):
-    with Client(loop=loop, processes=False, n_workers=2, silence_logs=False) as c:
+    with Client(
+        loop=loop,
+        processes=False,
+        n_workers=2,
+        silence_logs=False,
+        dashboard_address=":0",
+    ) as c:
         assert len(c.cluster.workers) == 2
         assert all(isinstance(w, Worker) for w in c.cluster.workers.values())
     assert c.cluster.status == Status.closed
 
 
 def test_Client_unused_kwargs_with_cluster(loop):
-    with LocalCluster() as cluster:
+    with LocalCluster(dashboard_address=":0") as cluster:
         with pytest.raises(Exception) as argexcept:
             c = Client(cluster, n_workers=2, dashboard_port=8000, silence_logs=None)
         assert (
@@ -249,37 +256,36 @@ def test_Client_unused_kwargs_with_address(loop):
 
 
 def test_Client_twice(loop):
-    with Client(loop=loop, silence_logs=False, dashboard_address=None) as c:
-        with Client(loop=loop, silence_logs=False, dashboard_address=None) as f:
+    with Client(loop=loop, silence_logs=False, dashboard_address=":0") as c:
+        with Client(loop=loop, silence_logs=False, dashboard_address=":0") as f:
             assert c.cluster.scheduler.port != f.cluster.scheduler.port
 
 
-@pytest.mark.asyncio
-async def test_client_constructor_with_temporary_security(cleanup):
+@gen_test()
+async def test_client_constructor_with_temporary_security():
     pytest.importorskip("cryptography")
     async with Client(
-        security=True, silence_logs=False, dashboard_address=None, asynchronous=True
+        security=True, silence_logs=False, dashboard_address=":0", asynchronous=True
     ) as c:
         assert c.cluster.scheduler_address.startswith("tls")
         assert c.security == c.cluster.security
 
 
-@pytest.mark.asyncio
-async def test_defaults(cleanup):
+@gen_test()
+async def test_defaults():
     async with LocalCluster(
-        scheduler_port=0, silence_logs=False, dashboard_address=None, asynchronous=True
+        silence_logs=False, dashboard_address=":0", asynchronous=True
     ) as c:
         assert sum(w.nthreads for w in c.workers.values()) == CPU_COUNT
         assert all(isinstance(w, Nanny) for w in c.workers.values())
 
 
-@pytest.mark.asyncio
-async def test_defaults_2(cleanup):
+@gen_test()
+async def test_defaults_2():
     async with LocalCluster(
         processes=False,
-        scheduler_port=0,
         silence_logs=False,
-        dashboard_address=None,
+        dashboard_address=":0",
         asynchronous=True,
     ) as c:
         assert sum(w.nthreads for w in c.workers.values()) == CPU_COUNT
@@ -287,13 +293,12 @@ async def test_defaults_2(cleanup):
         assert len(c.workers) == 1
 
 
-@pytest.mark.asyncio
-async def test_defaults_3(cleanup):
+@gen_test()
+async def test_defaults_3():
     async with LocalCluster(
         n_workers=2,
-        scheduler_port=0,
         silence_logs=False,
-        dashboard_address=None,
+        dashboard_address=":0",
         asynchronous=True,
     ) as c:
         if CPU_COUNT % 2 == 0:
@@ -304,66 +309,61 @@ async def test_defaults_3(cleanup):
         assert sum(w.nthreads for w in c.workers.values()) == expected_total_threads
 
 
-@pytest.mark.asyncio
-async def test_defaults_4(cleanup):
+@gen_test()
+async def test_defaults_4():
     async with LocalCluster(
         threads_per_worker=CPU_COUNT * 2,
-        scheduler_port=0,
         silence_logs=False,
-        dashboard_address=None,
+        dashboard_address=":0",
         asynchronous=True,
     ) as c:
         assert len(c.workers) == 1
 
 
-@pytest.mark.asyncio
-async def test_defaults_5(cleanup):
+@gen_test()
+async def test_defaults_5():
     async with LocalCluster(
         n_workers=CPU_COUNT * 2,
-        scheduler_port=0,
         silence_logs=False,
-        dashboard_address=None,
+        dashboard_address=":0",
         asynchronous=True,
     ) as c:
         assert all(w.nthreads == 1 for w in c.workers.values())
 
 
-@pytest.mark.asyncio
-async def test_defaults_6(cleanup):
+@gen_test()
+async def test_defaults_6():
     async with LocalCluster(
         threads_per_worker=2,
         n_workers=3,
-        scheduler_port=0,
         silence_logs=False,
-        dashboard_address=None,
+        dashboard_address=":0",
         asynchronous=True,
     ) as c:
         assert len(c.workers) == 3
         assert all(w.nthreads == 2 for w in c.workers.values())
 
 
-@pytest.mark.asyncio
-async def test_worker_params(cleanup):
+@gen_test()
+async def test_worker_params():
     async with LocalCluster(
         processes=False,
         n_workers=2,
-        scheduler_port=0,
         silence_logs=False,
-        dashboard_address=None,
+        dashboard_address=":0",
         memory_limit=500,
         asynchronous=True,
     ) as c:
         assert [w.memory_limit for w in c.workers.values()] == [500] * 2
 
 
-@pytest.mark.asyncio
-async def test_memory_limit_none(cleanup):
+@gen_test()
+async def test_memory_limit_none():
     async with LocalCluster(
         n_workers=2,
-        scheduler_port=0,
         silence_logs=False,
         processes=False,
-        dashboard_address=None,
+        dashboard_address=":0",
         memory_limit=None,
         asynchronous=True,
     ) as c:
@@ -374,13 +374,11 @@ async def test_memory_limit_none(cleanup):
 
 def test_cleanup():
     with clean(threads=False):
-        c = LocalCluster(
-            2, scheduler_port=0, silence_logs=False, dashboard_address=None
-        )
+        c = LocalCluster(n_workers=2, silence_logs=False, dashboard_address=":0")
         port = c.scheduler.port
         c.close()
         c2 = LocalCluster(
-            2, scheduler_port=port, silence_logs=False, dashboard_address=None
+            n_workers=2, scheduler_port=port, silence_logs=False, dashboard_address=":0"
         )
         c2.close()
 
@@ -388,12 +386,12 @@ def test_cleanup():
 def test_repeated():
     with clean(threads=False):
         with LocalCluster(
-            0, scheduler_port=8448, silence_logs=False, dashboard_address=None
-        ) as c:
+            n_workers=0, scheduler_port=8448, silence_logs=False, dashboard_address=":0"
+        ):
             pass
         with LocalCluster(
-            0, scheduler_port=8448, silence_logs=False, dashboard_address=None
-        ) as c:
+            n_workers=0, scheduler_port=8448, silence_logs=False, dashboard_address=":0"
+        ):
             pass
 
 
@@ -403,11 +401,10 @@ def test_bokeh(loop, processes):
     requests = pytest.importorskip("requests")
     with LocalCluster(
         n_workers=0,
-        scheduler_port=0,
         silence_logs=False,
         loop=loop,
         processes=processes,
-        dashboard_address=0,
+        dashboard_address=":0",
     ) as c:
         bokeh_port = c.scheduler.http_server.port
         url = "http://127.0.0.1:%d/status/" % bokeh_port
@@ -427,18 +424,17 @@ def test_bokeh(loop, processes):
 
 
 def test_blocks_until_full(loop):
-    with Client(loop=loop) as c:
+    with Client(loop=loop, dashboard_address=":0") as c:
         assert len(c.nthreads()) > 0
 
 
-@pytest.mark.asyncio
+@gen_test()
 async def test_scale_up_and_down():
     async with LocalCluster(
         n_workers=0,
-        scheduler_port=0,
         processes=False,
         silence_logs=False,
-        dashboard_address=None,
+        dashboard_address=":0",
         asynchronous=True,
     ) as cluster:
         async with Client(cluster, asynchronous=True) as c:
@@ -468,7 +464,7 @@ def test_silent_startup():
         from distributed import LocalCluster
 
         if __name__ == "__main__":
-            with LocalCluster(1, dashboard_address=None, scheduler_port=0):
+            with LocalCluster(n_workers=1, dashboard_address=":0"):
                 sleep(.1)
         """
 
@@ -486,17 +482,16 @@ def test_silent_startup():
 
 def test_only_local_access(loop):
     with LocalCluster(
-        0, scheduler_port=0, silence_logs=False, dashboard_address=None, loop=loop
+        n_workers=0, silence_logs=False, dashboard_address=":0", loop=loop
     ) as c:
         sync(loop, assert_can_connect_locally_4, c.scheduler.port)
 
 
 def test_remote_access(loop):
     with LocalCluster(
-        0,
-        scheduler_port=0,
+        n_workers=0,
         silence_logs=False,
-        dashboard_address=None,
+        dashboard_address=":0",
         host="",
         loop=loop,
     ) as c:
@@ -507,10 +502,9 @@ def test_remote_access(loop):
 def test_memory(loop, n_workers):
     with LocalCluster(
         n_workers=n_workers,
-        scheduler_port=0,
         processes=False,
         silence_logs=False,
-        dashboard_address=None,
+        dashboard_address=":0",
         loop=loop,
     ) as cluster:
         assert sum(w.memory_limit for w in cluster.workers.values()) <= MEMORY_LIMIT
@@ -520,10 +514,9 @@ def test_memory(loop, n_workers):
 def test_memory_nanny(loop, n_workers):
     with LocalCluster(
         n_workers=n_workers,
-        scheduler_port=0,
         processes=True,
         silence_logs=False,
-        dashboard_address=None,
+        dashboard_address=":0",
         loop=loop,
     ) as cluster:
         with Client(cluster.scheduler_address, loop=loop) as c:
@@ -536,24 +529,22 @@ def test_memory_nanny(loop, n_workers):
 def test_death_timeout_raises(loop):
     with pytest.raises(TimeoutError):
         with LocalCluster(
-            scheduler_port=0,
             silence_logs=False,
             death_timeout=1e-10,
-            dashboard_address=None,
+            dashboard_address=":0",
             loop=loop,
         ) as cluster:
             pass
     LocalCluster._instances.clear()  # ignore test hygiene checks
 
 
-@pytest.mark.asyncio
-async def test_bokeh_kwargs(cleanup):
+@gen_test()
+async def test_bokeh_kwargs():
     pytest.importorskip("bokeh")
     async with LocalCluster(
         n_workers=0,
-        scheduler_port=0,
         silence_logs=False,
-        dashboard_address=0,
+        dashboard_address=":0",
         asynchronous=True,
         scheduler_kwargs={"http_prefix": "/foo"},
     ) as c:
@@ -565,9 +556,7 @@ async def test_bokeh_kwargs(cleanup):
 
 
 def test_io_loop_periodic_callbacks(loop):
-    with LocalCluster(
-        loop=loop, port=0, dashboard_address=None, silence_logs=False
-    ) as cluster:
+    with LocalCluster(loop=loop, dashboard_address=":0", silence_logs=False) as cluster:
         assert cluster.scheduler.loop is loop
         for pc in cluster.scheduler.periodic_callbacks.values():
             assert pc.io_loop is loop
@@ -580,7 +569,7 @@ def test_logging():
     """
     Workers and scheduler have logs even when silenced
     """
-    with LocalCluster(1, processes=False, dashboard_address=None) as c:
+    with LocalCluster(n_workers=1, processes=False, dashboard_address=":0") as c:
         assert c.scheduler._deque_handler.deque
         assert c.workers[0]._deque_handler.deque
 
@@ -589,10 +578,9 @@ def test_ipywidgets(loop):
     ipywidgets = pytest.importorskip("ipywidgets")
     with LocalCluster(
         n_workers=0,
-        scheduler_port=0,
         silence_logs=False,
         loop=loop,
-        dashboard_address=False,
+        dashboard_address=":0",
         processes=False,
     ) as cluster:
         cluster._ipython_display_()
@@ -610,10 +598,9 @@ def test_no_ipywidgets(loop, monkeypatch):
 
     with LocalCluster(
         n_workers=0,
-        scheduler_port=0,
         silence_logs=False,
         loop=loop,
-        dashboard_address=False,
+        dashboard_address=":0",
         processes=False,
     ) as cluster:
         cluster._ipython_display_()
@@ -628,10 +615,9 @@ def test_no_ipywidgets(loop, monkeypatch):
 def test_scale(loop):
     """Directly calling scale both up and down works as expected"""
     with LocalCluster(
-        scheduler_port=0,
         silence_logs=False,
         loop=loop,
-        dashboard_address=False,
+        dashboard_address=":0",
         processes=False,
         n_workers=0,
     ) as cluster:
@@ -655,10 +641,9 @@ def test_scale(loop):
 
 def test_adapt(loop):
     with LocalCluster(
-        scheduler_port=0,
         silence_logs=False,
         loop=loop,
-        dashboard_address=False,
+        dashboard_address=":0",
         processes=False,
         n_workers=0,
     ) as cluster:
@@ -687,10 +672,9 @@ def test_adapt(loop):
 def test_adapt_then_manual(loop):
     """We can revert from adaptive, back to manual"""
     with LocalCluster(
-        scheduler_port=0,
         silence_logs=False,
         loop=loop,
-        dashboard_address=False,
+        dashboard_address=":0",
         processes=False,
         n_workers=8,
     ) as cluster:
@@ -731,7 +715,7 @@ def test_local_tls(loop, temporary):
         scheduler_port=8786,
         silence_logs=False,
         security=security,
-        dashboard_address=False,
+        dashboard_address=":0",
         host="tls://0.0.0.0",
         loop=loop,
     ) as c:
@@ -763,10 +747,9 @@ def scale_down(self, *args, **kwargs):
     loop = IOLoop.current()
     cluster = await MyCluster(
         n_workers=0,
-        scheduler_port=0,
         processes=False,
         silence_logs=False,
-        dashboard_address=None,
+        dashboard_address=":0",
         loop=loop,
         asynchronous=True,
     )
@@ -802,7 +785,7 @@ def test_local_tls_restart(loop):
         scheduler_port=8786,
         silence_logs=False,
         security=security,
-        dashboard_address=False,
+        dashboard_address=":0",
         host="tls://0.0.0.0",
         loop=loop,
     ) as c:
@@ -817,11 +800,10 @@ def test_local_tls_restart(loop):
 
 def test_asynchronous_property(loop):
     with LocalCluster(
-        4,
-        scheduler_port=0,
+        n_workers=4,
         processes=False,
         silence_logs=False,
-        dashboard_address=None,
+        dashboard_address=":0",
         loop=loop,
     ) as cluster:
 
@@ -832,13 +814,15 @@ async def _():
 
 
 def test_protocol_inproc(loop):
-    with LocalCluster(protocol="inproc://", loop=loop, processes=False) as cluster:
+    with LocalCluster(
+        protocol="inproc://", loop=loop, processes=False, dashboard_address=":0"
+    ) as cluster:
         assert cluster.scheduler.address.startswith("inproc://")
 
 
 def test_protocol_tcp(loop):
     with LocalCluster(
-        protocol="tcp", loop=loop, n_workers=0, processes=False
+        protocol="tcp", loop=loop, n_workers=0, processes=False, dashboard_address=":0"
     ) as cluster:
         assert cluster.scheduler.address.startswith("tcp://")
 
@@ -846,7 +830,11 @@ def test_protocol_tcp(loop):
 @pytest.mark.skipif(not LINUX, reason="Need 127.0.0.2 to mean localhost")
 def test_protocol_ip(loop):
     with LocalCluster(
-        host="tcp://127.0.0.2", loop=loop, n_workers=0, processes=False
+        host="tcp://127.0.0.2",
+        loop=loop,
+        n_workers=0,
+        processes=False,
+        dashboard_address=":0",
     ) as cluster:
         assert cluster.scheduler.address.startswith("tcp://127.0.0.2")
 
@@ -861,8 +849,7 @@ def test_worker_class_worker(loop):
         loop=loop,
         worker_class=MyWorker,
         processes=False,
-        scheduler_port=0,
-        dashboard_address=None,
+        dashboard_address=":0",
     ) as cluster:
         assert all(isinstance(w, MyWorker) for w in cluster.workers.values())
 
@@ -875,22 +862,20 @@ class MyNanny(Nanny):
         n_workers=2,
         loop=loop,
         worker_class=MyNanny,
-        scheduler_port=0,
-        dashboard_address=None,
+        dashboard_address=":0",
     ) as cluster:
         assert all(isinstance(w, MyNanny) for w in cluster.workers.values())
 
 
-@pytest.mark.asyncio
-async def test_worker_class_nanny_async(cleanup):
+@gen_test()
+async def test_worker_class_nanny_async():
     class MyNanny(Nanny):
         pass
 
     async with LocalCluster(
         n_workers=2,
         worker_class=MyNanny,
-        scheduler_port=0,
-        dashboard_address=None,
+        dashboard_address=":0",
         asynchronous=True,
     ) as cluster:
         assert all(isinstance(w, MyNanny) for w in cluster.workers.values())
@@ -901,8 +886,7 @@ def test_starts_up_sync(loop):
         n_workers=2,
         loop=loop,
         processes=False,
-        scheduler_port=0,
-        dashboard_address=None,
+        dashboard_address=":0",
     )
     try:
         assert len(cluster.scheduler.workers) == 2
@@ -914,7 +898,7 @@ def test_dont_select_closed_worker():
     # Make sure distributed does not try to reuse a client from a
     # closed cluster (https://github.com/dask/distributed/issues/2840).
     with clean(threads=False):
-        cluster = LocalCluster(n_workers=0)
+        cluster = LocalCluster(n_workers=0, dashboard_address=":0")
         c = Client(cluster)
         cluster.scale(2)
         assert c == get_client()
@@ -922,7 +906,7 @@ def test_dont_select_closed_worker():
         c.close()
         cluster.close()
 
-        cluster2 = LocalCluster(n_workers=0)
+        cluster2 = LocalCluster(n_workers=0, dashboard_address=":0")
         c2 = Client(cluster2)
         cluster2.scale(2)
 
@@ -935,19 +919,20 @@ def test_dont_select_closed_worker():
 
 def test_client_cluster_synchronous(loop):
     with clean(threads=False):
-        with Client(loop=loop, processes=False) as c:
+        with Client(loop=loop, processes=False, dashboard_address=":0") as c:
             assert not c.asynchronous
             assert not c.cluster.asynchronous
 
 
-@pytest.mark.asyncio
-async def test_scale_memory_cores(cleanup):
+@gen_test()
+async def test_scale_memory_cores():
     async with LocalCluster(
         n_workers=0,
         processes=False,
         threads_per_worker=2,
         memory_limit="2GB",
         asynchronous=True,
+        dashboard_address=":0",
     ) as cluster:
         cluster.scale(cores=4)
         assert len(cluster.worker_spec) == 2
@@ -963,30 +948,32 @@ async def test_scale_memory_cores(cleanup):
 
 
 @pytest.mark.asyncio
-async def test_repr(cleanup):
+@pytest.mark.parametrize("memory_limit", ["2 GiB", None])
+async def test_repr(memory_limit, cleanup):
     async with LocalCluster(
         n_workers=2,
         processes=False,
         threads_per_worker=2,
-        memory_limit="2GB",
+        memory_limit=memory_limit,
         asynchronous=True,
+        dashboard_address=":0",
     ) as cluster:
-        async with Client(cluster, asynchronous=True) as client:
-            await client.wait_for_workers(2)
+        # __repr__ uses cluster.scheduler_info, which slightly lags behind
+        # cluster.scheduler.workers and client.wait_for_workers.
+        while len(cluster.scheduler_info["workers"]) < 2:
+            await asyncio.sleep(0.01)
+
         text = repr(cluster)
-        assert "workers=2" in text
         assert cluster.scheduler_address in text
-        assert "cores=4" in text or "threads=4" in text
-        assert "4.00 GB" in text or "3.73 GiB" in text
-
-    async with LocalCluster(
-        n_workers=2, processes=False, memory_limit=None, asynchronous=True
-    ) as cluster:
-        assert "memory" not in repr(cluster)
+        assert "workers=2, threads=4" in text
+        if memory_limit:
+            assert "memory=4.00 GiB" in text
+        else:
+            assert "memory" not in text
 
 
-@pytest.mark.asyncio
-async def test_threads_per_worker_set_to_0(cleanup):
+@gen_test()
+async def test_threads_per_worker_set_to_0():
     with pytest.warns(
         Warning, match="Setting `threads_per_worker` to 0 has been deprecated."
     ):
@@ -1010,45 +997,49 @@ async def test_capture_security(cleanup, temporary):
         silence_logs=False,
         security=security,
         asynchronous=True,
-        dashboard_address=False,
+        dashboard_address=":0",
         host="tls://0.0.0.0",
     ) as cluster:
         async with Client(cluster, asynchronous=True) as client:
             assert client.security == cluster.security
 
 
-@pytest.mark.asyncio
-async def test_no_danglng_asyncio_tasks(cleanup):
+@gen_test()
+async def test_no_dangling_asyncio_tasks():
     start = asyncio.all_tasks()
-    async with LocalCluster(asynchronous=True, processes=False):
+    async with LocalCluster(asynchronous=True, processes=False, dashboard_address=":0"):
         await asyncio.sleep(0.01)
 
     tasks = asyncio.all_tasks()
     assert tasks == start
 
 
-@pytest.mark.asyncio
+@gen_test()
 async def test_async_with():
-    async with LocalCluster(processes=False, asynchronous=True) as cluster:
+    async with LocalCluster(
+        processes=False, asynchronous=True, dashboard_address=":0"
+    ) as cluster:
         w = cluster.workers
         assert w
 
     assert not w
 
 
-@pytest.mark.asyncio
-async def test_no_workers(cleanup):
+@gen_test()
+async def test_no_workers():
     async with Client(
-        n_workers=0, silence_logs=False, dashboard_address=None, asynchronous=True
+        n_workers=0, silence_logs=False, dashboard_address=":0", asynchronous=True
     ) as c:
         pass
 
 
-@pytest.mark.asyncio
+@gen_test()
 async def test_cluster_names():
-    async with LocalCluster(processes=False, asynchronous=True) as unnamed_cluster:
+    async with LocalCluster(
+        processes=False, asynchronous=True, dashboard_address=":0"
+    ) as unnamed_cluster:
         async with LocalCluster(
-            processes=False, asynchronous=True, name="mycluster"
+            processes=False, asynchronous=True, name="mycluster", dashboard_address=":0"
         ) as named_cluster:
             assert isinstance(unnamed_cluster.name, str)
             assert isinstance(named_cluster.name, str)
@@ -1057,7 +1048,9 @@ async def test_cluster_names():
             assert named_cluster == named_cluster
             assert unnamed_cluster != named_cluster
 
-        async with LocalCluster(processes=False, asynchronous=True) as unnamed_cluster2:
+        async with LocalCluster(
+            processes=False, asynchronous=True, dashboard_address=":0"
+        ) as unnamed_cluster2:
             assert unnamed_cluster2 != unnamed_cluster
 
 
@@ -1069,7 +1062,10 @@ async def test_local_cluster_redundant_kwarg(nanny):
         # whether we use the nanny or not, the error treatment is quite
         # different and we should assert that an exception is raised
         async with await LocalCluster(
-            typo_kwarg="foo", processes=nanny, n_workers=1
+            typo_kwarg="foo",
+            processes=nanny,
+            n_workers=1,
+            dashboard_address=":0",
         ) as cluster:
 
             # This will never work but is a reliable way to block without hard
diff --git a/distributed/deploy/tests/test_slow_adaptive.py b/distributed/deploy/tests/test_slow_adaptive.py
index ac3721d9a2b..853a5c4b4dc 100644
--- a/distributed/deploy/tests/test_slow_adaptive.py
+++ b/distributed/deploy/tests/test_slow_adaptive.py
@@ -32,7 +32,7 @@ async def close(self):
         self.status = "closed"
 
 
-scheduler = {"cls": Scheduler, "options": {"port": 0}}
+scheduler = {"cls": Scheduler, "options": {"dashboard_address": ":0"}}
 
 
 @pytest.mark.asyncio
diff --git a/distributed/deploy/tests/test_spec_cluster.py b/distributed/deploy/tests/test_spec_cluster.py
index 6b8ea8ee0e6..c51748c2549 100644
--- a/distributed/deploy/tests/test_spec_cluster.py
+++ b/distributed/deploy/tests/test_spec_cluster.py
@@ -14,7 +14,7 @@
 from distributed.deploy.spec import ProcessInterface, close_clusters, run_spec
 from distributed.metrics import time
 from distributed.utils import is_valid_xml
-from distributed.utils_test import gen_test
+from distributed.utils_test import gen_cluster, gen_test
 
 
 class MyWorker(Worker):
@@ -34,11 +34,11 @@ async def _():
     1: {"cls": Worker, "options": {"nthreads": 2}},
     "my-worker": {"cls": MyWorker, "options": {"nthreads": 3}},
 }
-scheduler = {"cls": Scheduler, "options": {"port": 0}}
+scheduler = {"cls": Scheduler, "options": {"dashboard_address": ":0"}}
 
 
-@pytest.mark.asyncio
-async def test_specification(cleanup):
+@gen_test()
+async def test_specification():
     async with SpecCluster(
         workers=worker_spec, scheduler=scheduler, asynchronous=True
     ) as cluster:
@@ -89,14 +89,12 @@ def test_spec_sync(loop):
 
 
 def test_loop_started():
-    with SpecCluster(
-        worker_spec, scheduler={"cls": Scheduler, "options": {"port": 0}}
-    ) as cluster:
+    with SpecCluster(worker_spec, scheduler=scheduler):
         pass
 
 
-@pytest.mark.asyncio
-async def test_repr(cleanup):
+@gen_test()
+async def test_repr():
     worker = {"cls": Worker, "options": {"nthreads": 1}}
 
     class MyCluster(SpecCluster):
@@ -108,8 +106,8 @@ class MyCluster(SpecCluster):
         assert "MyCluster" in str(cluster)
 
 
-@pytest.mark.asyncio
-async def test_scale(cleanup):
+@gen_test()
+async def test_scale():
     worker = {"cls": Worker, "options": {"nthreads": 1}}
     async with SpecCluster(
         asynchronous=True, scheduler=scheduler, worker=worker
@@ -146,7 +144,7 @@ async def test_adaptive_killed_worker():
         async with SpecCluster(
             asynchronous=True,
             worker={"cls": Nanny, "options": {"nthreads": 1}},
-            scheduler={"cls": Scheduler, "options": {"port": 0}},
+            scheduler=scheduler,
         ) as cluster:
             async with Client(cluster, asynchronous=True) as client:
                 # Scale up a cluster with 1 worker.
@@ -212,32 +210,33 @@ async def test_restart():
 
 
 @pytest.mark.skipif(WINDOWS, reason="HTTP Server doesn't close out")
+# FIXME cleanup fails:
+#       some RPCs left active by test: {<rpc to 'tcp://10.19.0.6:35045', 1 comms>}
+# @gen_test()
 @pytest.mark.asyncio
 async def test_broken_worker():
     with pytest.raises(Exception) as info:
         async with SpecCluster(
             asynchronous=True,
             workers={"good": {"cls": Worker}, "bad": {"cls": BrokenWorker}},
-            scheduler={"cls": Scheduler, "options": {"port": 0}},
-        ) as cluster:
+            scheduler=scheduler,
+        ):
             pass
 
     assert "Broken" in str(info.value)
 
 
 @pytest.mark.skipif(WINDOWS, reason="HTTP Server doesn't close out")
-@pytest.mark.slow
 def test_spec_close_clusters(loop):
     workers = {0: {"cls": Worker}}
-    scheduler = {"cls": Scheduler, "options": {"port": 0}}
     cluster = SpecCluster(workers=workers, scheduler=scheduler, loop=loop)
     assert cluster in SpecCluster._instances
     close_clusters()
     assert cluster.status == Status.closed
 
 
-@pytest.mark.asyncio
-async def test_new_worker_spec(cleanup):
+@gen_test()
+async def test_new_worker_spec():
     class MyCluster(SpecCluster):
         def new_worker_spec(self):
             i = len(self.worker_spec)
@@ -249,18 +248,14 @@ def new_worker_spec(self):
             assert cluster.worker_spec[i]["options"]["nthreads"] == i + 1
 
 
-@pytest.mark.asyncio
+@gen_test()
 async def test_nanny_port():
-    scheduler = {"cls": Scheduler}
     workers = {0: {"cls": Nanny, "options": {"port": 9200}}}
-
-    async with SpecCluster(
-        scheduler=scheduler, workers=workers, asynchronous=True
-    ) as cluster:
+    async with SpecCluster(scheduler=scheduler, workers=workers, asynchronous=True):
         pass
 
 
-@pytest.mark.asyncio
+@gen_test()
 async def test_spec_process():
     proc = ProcessInterface()
     assert proc.status == Status.created
@@ -270,8 +265,8 @@ async def test_spec_process():
     assert proc.status == Status.closed
 
 
-@pytest.mark.asyncio
-async def test_logs(cleanup):
+@gen_test()
+async def test_logs():
     worker = {"cls": Worker, "options": {"nthreads": 1}}
     async with SpecCluster(
         asynchronous=True, scheduler=scheduler, worker=worker
@@ -306,8 +301,8 @@ async def test_logs(cleanup):
         assert set(logs) == {w}
 
 
-@pytest.mark.asyncio
-async def test_scheduler_info(cleanup):
+@gen_test()
+async def test_scheduler_info():
     async with SpecCluster(
         workers=worker_spec, scheduler=scheduler, asynchronous=True
     ) as cluster:
@@ -330,21 +325,18 @@ async def test_scheduler_info(cleanup):
         assert len(cluster.scheduler_info["workers"]) == len(cluster.workers)
 
 
-@pytest.mark.asyncio
-async def test_dashboard_link(cleanup):
+@gen_test()
+async def test_dashboard_link():
     async with SpecCluster(
         workers=worker_spec,
-        scheduler={
-            "cls": Scheduler,
-            "options": {"port": 0, "dashboard_address": ":12345"},
-        },
+        scheduler={"cls": Scheduler, "options": {"dashboard_address": ":12345"}},
         asynchronous=True,
     ) as cluster:
         assert "12345" in cluster.dashboard_link
 
 
-@pytest.mark.asyncio
-async def test_widget(cleanup):
+@gen_test()
+async def test_widget():
     async with SpecCluster(
         workers=worker_spec,
         scheduler=scheduler,
@@ -361,8 +353,8 @@ async def test_widget(cleanup):
         assert "3 / 5" in cluster._scaling_status()
 
 
-@pytest.mark.asyncio
-async def test_scale_cores_memory(cleanup):
+@gen_test()
+async def test_scale_cores_memory():
     async with SpecCluster(
         scheduler=scheduler,
         worker={"cls": Worker, "options": {"nthreads": 1}},
@@ -376,8 +368,8 @@ async def test_scale_cores_memory(cleanup):
         assert "memory" in str(info.value)
 
 
-@pytest.mark.asyncio
-async def test_ProcessInterfaceValid(cleanup):
+@gen_test()
+async def test_ProcessInterfaceValid():
     async with SpecCluster(
         scheduler=scheduler, worker={"cls": ProcessInterface}, asynchronous=True
     ) as cluster:
@@ -416,8 +408,8 @@ async def close(self):
         await asyncio.gather(*[w.close() for w in self.workers])
 
 
-@pytest.mark.asyncio
-async def test_MultiWorker(cleanup):
+@gen_test()
+async def test_MultiWorker():
     async with SpecCluster(
         scheduler=scheduler,
         worker={
@@ -468,22 +460,17 @@ async def test_MultiWorker(cleanup):
             assert len(cluster.workers) == 1
 
 
-@pytest.mark.asyncio
-async def test_run_spec(cleanup):
-    async with Scheduler(port=0) as s:
-        workers = await run_spec(worker_spec, s.address)
-        async with Client(s.address, asynchronous=True) as c:
-            await c.wait_for_workers(len(worker_spec))
-
-            await asyncio.gather(*[w.close() for w in workers.values()])
-
-            assert not s.workers
-
-            await asyncio.gather(*[w.finished() for w in workers.values()])
+@gen_cluster(client=True, nthreads=[])
+async def test_run_spec(c, s):
+    workers = await run_spec(worker_spec, s.address)
+    await c.wait_for_workers(len(worker_spec))
+    await asyncio.gather(*[w.close() for w in workers.values()])
+    assert not s.workers
+    await asyncio.gather(*[w.finished() for w in workers.values()])
 
 
-@pytest.mark.asyncio
-async def test_run_spec_cluster_worker_names(cleanup):
+@gen_test()
+async def test_run_spec_cluster_worker_names():
     worker = {"cls": Worker, "options": {"nthreads": 1}}
 
     class MyCluster(SpecCluster):
@@ -509,8 +496,8 @@ def _new_worker_name(self, worker_number):
         assert sorted(list(cluster.workers)) == worker_names
 
 
-@pytest.mark.asyncio
-async def test_bad_close(cleanup):
+@gen_test()
+async def test_bad_close():
     with warnings.catch_warnings(record=True) as record:
         cluster = SpecCluster(
             workers=worker_spec, scheduler=scheduler, asynchronous=True
diff --git a/distributed/deploy/tests/test_ssh.py b/distributed/deploy/tests/test_ssh.py
index 16cb100b8ff..33baf565d6c 100644
--- a/distributed/deploy/tests/test_ssh.py
+++ b/distributed/deploy/tests/test_ssh.py
@@ -32,7 +32,7 @@ async def test_basic():
         ["127.0.0.1"] * 3,
         connect_options=dict(known_hosts=None),
         asynchronous=True,
-        scheduler_options={"port": 0, "idle_timeout": "5s"},
+        scheduler_options={"idle_timeout": "5s"},
         worker_options={"death_timeout": "5s"},
     ) as cluster:
         assert len(cluster.workers) == 2
@@ -56,7 +56,7 @@ async def test_keywords():
             "memory_limit": "2 GiB",
             "death_timeout": "5s",
         },
-        scheduler_options={"idle_timeout": "10s", "port": 0},
+        scheduler_options={"idle_timeout": "10s"},
     ) as cluster:
         async with Client(cluster, asynchronous=True) as client:
             assert (
@@ -109,7 +109,7 @@ def f(x):
             ["127.0.0.1"] * 2,
             connect_options=dict(known_hosts=None),
             asynchronous=True,
-            scheduler_options={"port": 0, "idle_timeout": "5s"},
+            scheduler_options={"idle_timeout": "5s"},
             worker_options={"death_timeout": "5s"},
         ) as cluster:
             async with Client(cluster, asynchronous=True) as client:
@@ -130,7 +130,7 @@ async def test_unimplemented_options():
                 "death_timeout": "5s",
                 "unimplemented_option": 2,
             },
-            scheduler_kwargs={"idle_timeout": "5s", "port": 0},
+            scheduler_kwargs={"idle_timeout": "5s"},
         ) as cluster:
             assert cluster
 
@@ -141,7 +141,7 @@ async def test_list_of_connect_options():
         ["127.0.0.1"] * 3,
         connect_options=[dict(known_hosts=None)] * 3,
         asynchronous=True,
-        scheduler_options={"port": 0, "idle_timeout": "5s"},
+        scheduler_options={"idle_timeout": "5s"},
         worker_options={"death_timeout": "5s"},
     ) as cluster:
         assert len(cluster.workers) == 2
@@ -160,7 +160,7 @@ async def test_list_of_connect_options_raises():
             ["127.0.0.1"] * 3,
             connect_options=[dict(known_hosts=None)] * 4,  # Mismatch in length 4 != 3
             asynchronous=True,
-            scheduler_options={"port": 0, "idle_timeout": "5s"},
+            scheduler_options={"idle_timeout": "5s"},
             worker_options={"death_timeout": "5s"},
         ) as _:
             pass
@@ -172,7 +172,7 @@ async def test_remote_python():
         ["127.0.0.1"] * 3,
         connect_options=[dict(known_hosts=None)] * 3,
         asynchronous=True,
-        scheduler_options={"port": 0, "idle_timeout": "5s"},
+        scheduler_options={"idle_timeout": "5s"},
         worker_options={"death_timeout": "5s"},
         remote_python=sys.executable,
     ) as cluster:
@@ -185,7 +185,7 @@ async def test_remote_python_as_dict():
         ["127.0.0.1"] * 3,
         connect_options=[dict(known_hosts=None)] * 3,
         asynchronous=True,
-        scheduler_options={"port": 0, "idle_timeout": "5s"},
+        scheduler_options={"idle_timeout": "5s"},
         worker_options={"death_timeout": "5s"},
         remote_python=[sys.executable] * 3,
     ) as cluster:
@@ -199,7 +199,7 @@ async def test_list_of_remote_python_raises():
             ["127.0.0.1"] * 3,
             connect_options=[dict(known_hosts=None)] * 3,
             asynchronous=True,
-            scheduler_options={"port": 0, "idle_timeout": "5s"},
+            scheduler_options={"idle_timeout": "5s"},
             worker_options={"death_timeout": "5s"},
             remote_python=[sys.executable] * 4,  # Mismatch in length 4 != 3
         ) as _:
diff --git a/distributed/diagnostics/tests/test_progressbar.py b/distributed/diagnostics/tests/test_progressbar.py
index d358c9a494a..ef14e20a1f3 100644
--- a/distributed/diagnostics/tests/test_progressbar.py
+++ b/distributed/diagnostics/tests/test_progressbar.py
@@ -1,8 +1,5 @@
 from time import sleep
 
-import pytest
-
-from distributed import Scheduler, Worker
 from distributed.diagnostics.progressbar import TextProgressBar, progress
 from distributed.metrics import time
 from distributed.utils_test import div, gen_cluster, inc
@@ -39,18 +36,13 @@ async def test_TextProgressBar_error(c, s, a, b):
     assert progress.comm.closed()
 
 
-@pytest.mark.asyncio
-async def test_TextProgressBar_empty(capsys):
-    async with Scheduler(port=0) as s:
-        async with Worker(s.address, nthreads=1):
-            async with Worker(s.address, nthreads=1):
-                progress = TextProgressBar(
-                    [], scheduler=s.address, start=False, interval=0.01
-                )
-                await progress.listen()
+@gen_cluster()
+async def test_TextProgressBar_empty(s, a, b, capsys):
+    progress = TextProgressBar([], scheduler=s.address, start=False, interval=0.01)
+    await progress.listen()
 
-                assert progress.status == "finished"
-                check_bar_completed(capsys)
+    assert progress.status == "finished"
+    check_bar_completed(capsys)
 
 
 def check_bar_completed(capsys, width=40):
diff --git a/distributed/diagnostics/tests/test_scheduler_plugin.py b/distributed/diagnostics/tests/test_scheduler_plugin.py
index 111ad5e24d9..75cd31f22f6 100644
--- a/distributed/diagnostics/tests/test_scheduler_plugin.py
+++ b/distributed/diagnostics/tests/test_scheduler_plugin.py
@@ -1,7 +1,7 @@
 import pytest
 
 from distributed import Scheduler, SchedulerPlugin, Worker
-from distributed.utils_test import gen_cluster, inc
+from distributed.utils_test import gen_cluster, gen_test, inc
 
 
 @gen_cluster(client=True)
@@ -106,8 +106,8 @@ async def remove_worker(self, worker, scheduler):
     assert events == []
 
 
-@pytest.mark.asyncio
-async def test_lifecycle(cleanup):
+@gen_test()
+async def test_lifecycle():
     class LifeCycle(SchedulerPlugin):
         def __init__(self):
             self.history = []
@@ -120,7 +120,7 @@ async def close(self):
             self.history.append("closed")
 
     plugin = LifeCycle()
-    async with Scheduler(plugins=[plugin]) as s:
+    async with Scheduler(plugins=[plugin], dashboard_address=":0") as s:
         pass
 
     assert plugin.history == ["started", "closed"]
diff --git a/distributed/node.py b/distributed/node.py
index a7f9b8d31ac..527218c187b 100644
--- a/distributed/node.py
+++ b/distributed/node.py
@@ -158,9 +158,7 @@ def start_http_server(
         ):
             if expected != actual and expected > 0:
                 warnings.warn(
-                    "Port {} is already in use.\n"
+                    f"Port {expected} is already in use.\n"
                     "Perhaps you already have a cluster running?\n"
-                    "Hosting the HTTP server on port {} instead".format(
-                        expected, actual
-                    )
+                    f"Hosting the HTTP server on port {actual} instead"
                 )
diff --git a/distributed/tests/test_actor.py b/distributed/tests/test_actor.py
index 52808ca03af..b89f4aa3013 100644
--- a/distributed/tests/test_actor.py
+++ b/distributed/tests/test_actor.py
@@ -474,8 +474,9 @@ def f(block, ps=None):
     print(format_time(end - start))
 
 
+@pytest.mark.slow
 @pytest.mark.flaky(reruns=10, reruns_delay=5)
-@gen_cluster(client=True)
+@gen_cluster(client=True, timeout=120)
 async def test_compute(c, s, a, b):
     @dask.delayed
     def f(n, counter):
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index e3e8dcd3fa0..e642e6f6ae7 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -1981,13 +1981,13 @@ async def test_badly_serialized_input_stderr(capsys, c):
 
 def test_repr(loop):
     funcs = [str, repr, lambda x: x._repr_html_()]
-    with cluster(nworkers=3, worker_kwargs={"memory_limit": "2 GB"}) as (s, [a, b, c]):
+    with cluster(nworkers=3, worker_kwargs={"memory_limit": "2 GiB"}) as (s, [a, b, c]):
         with Client(s["address"], loop=loop) as c:
             for func in funcs:
                 text = func(c)
                 assert c.scheduler.address in text
                 assert "threads=3" in text or "Total threads: </strong>" in text
-                assert "6.00 GB" in text or "5.59 GiB" in text
+                assert "6.00 GiB" in text
                 if "<table" not in text:
                     assert len(text) < 80
 
@@ -2009,7 +2009,7 @@ async def test_repr_no_memory_limit(c, s, a, b):
 @gen_test()
 async def test_repr_localcluster():
     cluster = await LocalCluster(
-        processes=False, dashboard_address=None, asynchronous=True
+        processes=False, dashboard_address=":0", asynchronous=True
     )
     client = await Client(cluster, asynchronous=True)
     try:
@@ -2781,14 +2781,12 @@ async def test_diagnostic_nbytes(c, s, a, b):
     assert s.get_nbytes(summary=True) == {"inc": sizeof(1) * 3, "double": sizeof(1) * 3}
 
 
-@gen_test()
-async def test_worker_aliases():
-    s = await Scheduler(validate=True, port=0)
+@gen_cluster(client=True, nthreads=[])
+async def test_worker_aliases(c, s):
     a = Worker(s.address, name="alice")
     b = Worker(s.address, name="bob")
     w = Worker(s.address, name=3)
     await asyncio.gather(a, b, w)
-    c = await Client(s.address, asynchronous=True)
 
     L = c.map(inc, range(10), workers="alice")
     future = await c.scatter(123, workers=3)
@@ -2801,13 +2799,10 @@ async def test_worker_aliases():
         result = await c.submit(lambda x: x + 1, i, workers=alias)
         assert result == i + 1
 
-    await c.close()
     await asyncio.gather(a.close(), b.close(), w.close())
-    await s.close()
 
 
 def test_persist_get_sync(c):
-    dadd = delayed(add)
     x, y = delayed(1), delayed(2)
     xx = delayed(add)(x, x)
     yy = delayed(add)(y, y)
@@ -2821,7 +2816,6 @@ def test_persist_get_sync(c):
 
 @gen_cluster(client=True)
 async def test_persist_get(c, s, a, b):
-    dadd = delayed(add)
     x, y = delayed(1), delayed(2)
     xx = delayed(add)(x, x)
     yy = delayed(add)(y, y)
@@ -2968,7 +2962,7 @@ async def test_rebalance_workers_and_keys(client, s, *_):
 
 def test_rebalance_sync():
     # can't use the 'c' fixture because we need workers to run in a separate process
-    with Client(n_workers=2, memory_limit="1 GiB") as c:
+    with Client(n_workers=2, memory_limit="1 GiB", dashboard_address=":0") as c:
         s = c.cluster.scheduler
         a, b = [ws.address for ws in s.workers.values()]
         futures = c.map(lambda _: "x" * (2 ** 29 // 10), range(10), workers=[a])
@@ -3049,7 +3043,7 @@ async def test_unrunnable_task_runs(c, s, a, b):
 @gen_cluster(client=True, nthreads=[])
 async def test_add_worker_after_tasks(c, s):
     futures = c.map(inc, range(10))
-    n = await Nanny(s.address, nthreads=2, loop=s.loop, port=0)
+    n = await Nanny(s.address, nthreads=2, loop=s.loop)
     await c.gather(futures)
     await n.close()
 
@@ -3619,10 +3613,8 @@ def test_as_completed_next_batch(c):
     assert not ac.has_ready()
 
 
-@gen_test()
-async def test_status():
-    s = await Scheduler(port=0)
-
+@gen_cluster(nthreads=[])
+async def test_status(s):
     c = await Client(s.address, asynchronous=True)
     assert c.status == "running"
     x = c.submit(inc, 1)
@@ -3630,8 +3622,6 @@ async def test_status():
     await c.close()
     assert c.status == "closed"
 
-    await s.close()
-
 
 @gen_cluster(client=True)
 async def test_async_whowhat(c, s, a, b):
@@ -4597,11 +4587,11 @@ async def test_scatter_dict_workers(c, s, a, b):
 
 
 @pytest.mark.slow
-@gen_test()
+@gen_test(timeout=180)
 async def test_client_timeout():
     c = Client("127.0.0.1:57484", asynchronous=True)
 
-    s = Scheduler(loop=c.loop, port=57484)
+    s = Scheduler(loop=c.loop, port=57484, dashboard_address=":0")
     await asyncio.sleep(4)
 
     try:
@@ -4982,7 +4972,12 @@ async def test_fire_and_forget_err(c, s, a, b):
 
 def test_quiet_client_close(loop):
     with captured_logger(logging.getLogger("distributed")) as logger:
-        with Client(loop=loop, processes=False, threads_per_worker=4) as c:
+        with Client(
+            loop=loop,
+            processes=False,
+            dashboard_address=":0",
+            threads_per_worker=4,
+        ) as c:
             futures = c.map(slowinc, range(1000), delay=0.01)
             sleep(0.200)  # stop part-way
         sleep(0.1)  # let things settle
@@ -5311,9 +5306,7 @@ async def f():
 
 def test_quiet_quit_when_cluster_leaves(loop_in_thread):
     loop = loop_in_thread
-    with LocalCluster(
-        loop=loop, scheduler_port=0, dashboard_address=None, silence_logs=False
-    ) as cluster:
+    with LocalCluster(loop=loop, dashboard_address=":0", silence_logs=False) as cluster:
         with captured_logger("distributed.comm") as sio:
             with Client(cluster, loop=loop) as client:
                 futures = client.map(lambda x: x + 1, range(10))
@@ -5647,9 +5640,9 @@ def test_dashboard_link(loop, monkeypatch):
                 assert link in text
 
 
-@pytest.mark.asyncio
-async def test_dashboard_link_inproc(cleanup):
-    async with Client(processes=False, asynchronous=True) as c:
+@gen_test()
+async def test_dashboard_link_inproc():
+    async with Client(processes=False, asynchronous=True, dashboard_address=":0") as c:
         with dask.config.set({"distributed.dashboard.link": "{host}"}):
             assert "/" not in c.dashboard_link
 
@@ -5770,9 +5763,9 @@ async def test_client_repr_closed(s, a, b):
 
 
 def test_client_repr_closed_sync(loop):
-    with Client(loop=loop, processes=False, dashboard_address=None) as c:
-        c.close()
-        c._repr_html_()
+    with Client(loop=loop, processes=False, dashboard_address=":0") as c:
+        pass
+    c._repr_html_()
 
 
 @pytest.mark.xfail(reason="https://github.com/dask/dask/pull/6807")
@@ -6043,7 +6036,7 @@ async def test_file_descriptors_dont_leak(Worker):
 
     proc = psutil.Process()
     before = proc.num_fds()
-    async with Scheduler(port=0, dashboard_address=":0") as s:
+    async with Scheduler(dashboard_address=":0") as s:
         async with Worker(s.address), Worker(s.address), Client(
             s.address, asynchronous=True
         ):
@@ -6056,48 +6049,53 @@ async def test_file_descriptors_dont_leak(Worker):
         assert time() < start + 10, (before, proc.num_fds())
 
 
-@pytest.mark.asyncio
-async def test_dashboard_link_cluster(cleanup):
+@gen_test()
+async def test_dashboard_link_cluster():
     class MyCluster(LocalCluster):
         @property
         def dashboard_link(self):
             return "http://foo.com"
 
-    async with MyCluster(processes=False, asynchronous=True) as cluster:
+    async with MyCluster(
+        processes=False, asynchronous=True, dashboard_address=":0"
+    ) as cluster:
         async with Client(cluster, asynchronous=True) as client:
             assert "http://foo.com" in client._repr_html_()
 
 
-@pytest.mark.asyncio
-async def test_shutdown(cleanup):
-    async with Scheduler(port=0) as s:
+@gen_test()
+async def test_shutdown():
+    async with Scheduler(dashboard_address=":0") as s:
         async with Worker(s.address) as w:
             async with Client(s.address, asynchronous=True) as c:
                 await c.shutdown()
 
-            assert s.status == Status.closed
-            assert w.status == Status.closed
+                assert s.status == Status.closed
+                assert w.status == Status.closed
 
 
 @pytest.mark.asyncio
 async def test_shutdown_localcluster(cleanup):
-    async with LocalCluster(n_workers=1, asynchronous=True, processes=False) as lc:
+    async with LocalCluster(
+        n_workers=1, asynchronous=True, processes=False, dashboard_address=":0"
+    ) as lc:
         async with Client(lc, asynchronous=True) as c:
             await c.shutdown()
 
         assert lc.scheduler.status == Status.closed
 
 
-@pytest.mark.asyncio
-async def test_config_inherited_by_subprocess(cleanup):
-    def f(x):
-        return dask.config.get("foo") + 1
-
+@gen_test()
+async def test_config_inherited_by_subprocess():
     with dask.config.set(foo=100):
-        async with LocalCluster(n_workers=1, asynchronous=True, processes=True) as lc:
+        async with LocalCluster(
+            n_workers=1,
+            asynchronous=True,
+            processes=True,
+            dashboard_address=":0",
+        ) as lc:
             async with Client(lc, asynchronous=True) as c:
-                result = await c.submit(f, 1)
-                assert result == 101
+                assert await c.submit(dask.config.get, "foo") == 100
 
 
 @gen_cluster(client=True)
@@ -6207,24 +6205,12 @@ async def f():
     assert L == [x, y]
 
 
-def test_async_with(loop):
-    result = None
-    client = None
-    cluster = None
-
-    async def f():
-        async with Client(processes=False, asynchronous=True) as c:
-            nonlocal result, client, cluster
-            result = await c.submit(lambda x: x + 1, 10)
-
-            client = c
-            cluster = c.cluster
-
-    loop.run_sync(f)
-
-    assert result == 11
-    assert client.status == "closed"
-    assert cluster.status == Status.closed
+@gen_test()
+async def test_async_with():
+    async with Client(processes=False, dashboard_address=":0", asynchronous=True) as c:
+        assert await c.submit(lambda x: x + 1, 10) == 11
+    assert c.status == "closed"
+    assert c.cluster.status == Status.closed
 
 
 def test_client_sync_with_async_def(loop):
@@ -6372,11 +6358,10 @@ async def f(stacklevel, mode=None):
     assert "cdn.bokeh.org" in data
 
 
-@pytest.mark.asyncio
-async def test_client_gather_semaphore_loop(cleanup):
-    async with Scheduler(port=0) as s:
-        async with Client(s.address, asynchronous=True) as c:
-            assert c._gather_semaphore._loop is c.loop.asyncio_loop
+@gen_cluster(nthreads=[])
+async def test_client_gather_semaphore_loop(s):
+    async with Client(s.address, asynchronous=True) as c:
+        assert c._gather_semaphore._loop is c.loop.asyncio_loop
 
 
 @gen_cluster(client=True)
@@ -6392,22 +6377,21 @@ def test_client_connectionpool_semaphore_loop(s, a, b):
 
 
 @pytest.mark.slow
-@pytest.mark.asyncio
-async def test_mixed_compression(cleanup):
+@gen_cluster(nthreads=[], timeout=60)
+async def test_mixed_compression(s):
     pytest.importorskip("lz4")
     da = pytest.importorskip("dask.array")
-    async with Scheduler(port=0, dashboard_address=":0") as s:
+    async with Nanny(
+        s.address, nthreads=1, config={"distributed.comm.compression": None}
+    ):
         async with Nanny(
-            s.address, nthreads=1, config={"distributed.comm.compression": None}
-        ) as a:
-            async with Nanny(
-                s.address, nthreads=1, config={"distributed.comm.compression": "lz4"}
-            ) as b:
-                async with Client(s.address, asynchronous=True) as c:
-                    await c.get_versions()
-                    x = da.ones((10000, 10000))
-                    y = x + x.T
-                    await c.compute(y.sum())
+            s.address, nthreads=1, config={"distributed.comm.compression": "lz4"}
+        ):
+            async with Client(s.address, asynchronous=True) as c:
+                await c.get_versions()
+                x = da.ones((10000, 10000))
+                y = x + x.T
+                await c.compute(y.sum())
 
 
 @gen_cluster(client=True)
@@ -6631,9 +6615,9 @@ def reliable_double(x):
 
     # The later annotations should not override the earlier annotations
     with dask.annotate(retries=2):
-        y = x.map_blocks(flaky_double, meta=np.array((), dtype=np.float))
+        y = x.map_blocks(flaky_double, meta=np.array((), dtype=float))
     with dask.annotate(retries=0):
-        z = y.map_blocks(reliable_double, meta=np.array((), dtype=np.float))
+        z = y.map_blocks(reliable_double, meta=np.array((), dtype=float))
 
     with dask.config.set(optimization__fuse__active=False):
         z = await c.compute(z)
@@ -6719,12 +6703,11 @@ async def test_get_client_functions_spawn_clusters(c, s, a):
     scheduler_addr = c.scheduler.address
 
     def f(x):
-        ref = None
         with LocalCluster(
             n_workers=1,
             processes=False,
-            dashboard_address=False,
-            worker_dashboard_address=False,
+            dashboard_address=":0",
+            worker_dashboard_address=":0",
         ) as cluster2:
             with Client(cluster2) as c1:
                 c2 = get_client()
diff --git a/distributed/tests/test_client_executor.py b/distributed/tests/test_client_executor.py
index 7b20069788c..3bcf0b784fa 100644
--- a/distributed/tests/test_client_executor.py
+++ b/distributed/tests/test_client_executor.py
@@ -99,7 +99,7 @@ def test_cancellation(client):
         fut = e.submit(time.sleep, 2.0)
         start = time.time()
         while number_of_processing_tasks(client) == 0:
-            assert time.time() < start + 10
+            assert time.time() < start + 30
             time.sleep(0.01)
         assert not fut.done()
 
@@ -107,16 +107,16 @@ def test_cancellation(client):
         assert fut.cancelled()
         start = time.time()
         while number_of_processing_tasks(client) != 0:
-            assert time.time() < start + 10
+            assert time.time() < start + 30
             time.sleep(0.01)
 
         with pytest.raises(CancelledError):
             fut.result()
 
-    # With wait()
+
+def test_cancellation_wait(client):
     with client.get_executor(pure=False) as e:
-        N = 10
-        fs = [e.submit(slowinc, i, delay=0.02) for i in range(N)]
+        fs = [e.submit(slowinc, i, delay=0.1) for i in range(10)]
         fs[3].cancel()
         res = wait(fs, return_when=FIRST_COMPLETED, timeout=30)
         assert len(res.not_done) > 0
@@ -125,10 +125,10 @@ def test_cancellation(client):
         assert fs[3] in res.done
         assert fs[3].cancelled()
 
-    # With as_completed()
+
+def test_cancellation_as_completed(client):
     with client.get_executor(pure=False) as e:
-        N = 10
-        fs = [e.submit(slowinc, i, delay=0.02) for i in range(N)]
+        fs = [e.submit(slowinc, i, delay=0.1) for i in range(10)]
         fs[3].cancel()
         fs[8].cancel()
 
diff --git a/distributed/tests/test_client_loop.py b/distributed/tests/test_client_loop.py
index 63a08cb1639..46cd2ec6b11 100644
--- a/distributed/tests/test_client_loop.py
+++ b/distributed/tests/test_client_loop.py
@@ -11,7 +11,7 @@ def test_close_loop_sync(with_own_loop):
 
     # Setup simple cluster with one threaded worker.
     # Complex setup is not required here since we test only IO loop teardown.
-    cluster_params = dict(n_workers=1, dashboard_address=None, processes=False)
+    cluster_params = dict(n_workers=1, dashboard_address=":0", processes=False)
 
     loops_before = LoopRunner._all_loops.copy()
 
diff --git a/distributed/tests/test_collections.py b/distributed/tests/test_collections.py
index 373160d5f7c..4ab2730736b 100644
--- a/distributed/tests/test_collections.py
+++ b/distributed/tests/test_collections.py
@@ -48,8 +48,7 @@ async def test_dataframes(c, s, a, b):
     )
     ldf = dd.from_pandas(df, npartitions=10)
 
-    rdf = c.persist(ldf)
-
+    rdf = await c.persist(ldf)
     assert rdf.divisions == ldf.divisions
 
     remote = c.compute(rdf)
diff --git a/distributed/tests/test_failed_workers.py b/distributed/tests/test_failed_workers.py
index 88b5e2db7a3..83eb8ae8e50 100644
--- a/distributed/tests/test_failed_workers.py
+++ b/distributed/tests/test_failed_workers.py
@@ -67,6 +67,7 @@ async def test_submit_after_failed_worker(c, s, a, b):
     assert result == sum(map(inc, range(10)))
 
 
+@pytest.mark.slow
 def test_gather_after_failed_worker(loop):
     with cluster() as (s, [a, b]):
         with Client(s["address"], loop=loop) as c:
@@ -77,7 +78,8 @@ def test_gather_after_failed_worker(loop):
             assert result == list(map(inc, range(10)))
 
 
-@gen_cluster(client=True, Worker=Nanny, nthreads=[("127.0.0.1", 1)] * 4)
+@pytest.mark.slow
+@gen_cluster(client=True, Worker=Nanny, nthreads=[("127.0.0.1", 1)] * 4, timeout=60)
 async def test_gather_then_submit_after_failed_workers(c, s, w, x, y, z):
     L = c.map(inc, range(20))
     await wait(L)
diff --git a/distributed/tests/test_locks.py b/distributed/tests/test_locks.py
index 99bd5ebdca5..a20a298b808 100644
--- a/distributed/tests/test_locks.py
+++ b/distributed/tests/test_locks.py
@@ -4,7 +4,7 @@
 
 import pytest
 
-from distributed import Client, Lock, get_client
+from distributed import Lock, get_client
 from distributed.metrics import time
 from distributed.utils_test import gen_cluster
 
@@ -130,11 +130,10 @@ def f(x, lock=None):
     assert lock2.client is lock.client
 
 
-@pytest.mark.asyncio
-async def test_locks():
-    async with Client(processes=False, asynchronous=True) as c:
-        assert c.asynchronous
-        async with Lock("x"):
-            lock2 = Lock("x")
-            result = await lock2.acquire(timeout=0.1)
-            assert result is False
+@gen_cluster(client=True, nthreads=[])
+async def test_locks(c, s):
+    async with Lock("x") as l1:
+        l2 = Lock("x")
+        assert l1.client is c
+        assert l2.client is c
+        assert await l2.acquire(timeout=0.01) is False
diff --git a/distributed/tests/test_multi_locks.py b/distributed/tests/test_multi_locks.py
index 9ece4e9c2c2..ff46d40b0ed 100644
--- a/distributed/tests/test_multi_locks.py
+++ b/distributed/tests/test_multi_locks.py
@@ -1,6 +1,8 @@
 import asyncio
 from time import sleep
 
+import pytest
+
 from distributed import MultiLock, get_client
 from distributed.metrics import time
 from distributed.multi_lock import MultiLockExtension
@@ -53,34 +55,25 @@ async def test_timeout(c, s, a, b):
     await lock1.release()
 
 
-@gen_cluster(client=True)
-async def test_timeout_wake_waiter(c, s, a, b):
-    ext: MultiLockExtension = s.extensions["multi_locks"]
+@gen_cluster()
+async def test_timeout_wake_waiter(s, a, b):
     l1 = MultiLock(names=["x"])
     l2 = MultiLock(names=["x", "y"])
     l3 = MultiLock(names=["y"])
     await l1.acquire()
 
-    l2_acquire = asyncio.ensure_future(l2.acquire(timeout=1))
-    try:
+    l2_acquire = asyncio.ensure_future(l2.acquire(timeout=0.5))
+    with pytest.raises(asyncio.TimeoutError):
         await asyncio.wait_for(asyncio.shield(l2_acquire), 0.1)
-    except asyncio.TimeoutError:
-        pass
-    else:
-        assert False
 
     l3_acquire = asyncio.ensure_future(l3.acquire())
-    try:
+    with pytest.raises(asyncio.TimeoutError):
         await asyncio.wait_for(asyncio.shield(l3_acquire), 0.1)
-    except asyncio.TimeoutError:
-        pass
-    else:
-        assert False
 
     assert await l2_acquire is False
     assert await l3_acquire
-    l1.release()
-    l3.release()
+    await l1.release()
+    await l3.release()
 
 
 @gen_cluster(client=True)
diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
index 013412469be..a5206e4f2e4 100644
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -17,7 +17,7 @@
 
 import dask
 
-from distributed import Client, Nanny, Scheduler, Worker, rpc, wait, worker
+from distributed import Nanny, Scheduler, Worker, rpc, wait, worker
 from distributed.compatibility import LINUX, WINDOWS
 from distributed.core import CommClosedError, Status
 from distributed.diagnostics import SchedulerPlugin
@@ -29,8 +29,8 @@
 pytestmark = pytest.mark.ci1
 
 
-# FIXME why does this leave behind unclosed Comm objects?
-@gen_cluster(nthreads=[], allow_unclosed=True)
+@pytest.mark.slow
+@gen_cluster(nthreads=[], timeout=120)
 async def test_nanny(s):
     async with Nanny(s.address, nthreads=2, loop=s.loop) as n:
         async with rpc(n.address) as nn:
@@ -126,7 +126,7 @@ async def test_run(s):
 
 
 @pytest.mark.slow
-@gen_cluster(config={"distributed.comm.timeouts.connect": "1s"})
+@gen_cluster(config={"distributed.comm.timeouts.connect": "1s"}, timeout=120)
 async def test_no_hang_when_scheduler_closes(s, a, b):
     # https://github.com/dask/distributed/issues/2880
     with captured_logger("tornado.application", logging.ERROR) as logger:
@@ -237,7 +237,7 @@ def func(dask_worker):
 @gen_test()
 async def test_scheduler_file():
     with tmpfile() as fn:
-        s = await Scheduler(scheduler_file=fn, port=8008)
+        s = await Scheduler(scheduler_file=fn, dashboard_address=":0")
         w = await Nanny(scheduler_file=fn)
         assert set(s.workers) == {w.worker_address}
         await w.close()
@@ -443,21 +443,19 @@ def pool_worker(world_size):
     await c.submit(pool_worker, 4)
 
 
-@pytest.mark.asyncio
-async def test_nanny_closes_cleanly(cleanup):
-    async with Scheduler() as s:
-        n = await Nanny(s.address)
+@gen_cluster(nthreads=[])
+async def test_nanny_closes_cleanly(s):
+    async with Nanny(s.address) as n:
         assert n.process.pid
         proc = n.process.process
-        await n.close()
-        assert not n.process
-        assert not proc.is_alive()
-        assert proc.exitcode == 0
+    assert not n.process
+    assert not proc.is_alive()
+    assert proc.exitcode == 0
 
 
 @pytest.mark.slow
-@pytest.mark.asyncio
-async def test_lifetime(cleanup):
+@gen_cluster(nthreads=[], timeout=60)
+async def test_lifetime(s):
     counter = 0
     event = asyncio.Event()
 
@@ -471,63 +469,52 @@ def remove_worker(self, **kwargs):
             if counter == 2:  # wait twice, then trigger closing event
                 event.set()
 
-    async with Scheduler() as s:
-        s.add_plugin(Plugin())
-        async with Nanny(s.address) as a:
-            async with Nanny(s.address, lifetime="500 ms", lifetime_restart=True) as b:
-                await event.wait()
+    s.add_plugin(Plugin())
+    async with Nanny(s.address):
+        async with Nanny(s.address, lifetime="500 ms", lifetime_restart=True):
+            await event.wait()
 
 
-@pytest.mark.asyncio
-async def test_nanny_closes_cleanly_2(cleanup):
-    async with Scheduler() as s:
-        async with Nanny(s.address) as n:
-            async with Client(s.address, asynchronous=True) as client:
-                with client.rpc(n.worker_address) as w:
-                    IOLoop.current().add_callback(w.terminate)
-                    start = time()
-                    while n.status != Status.closed:
-                        await asyncio.sleep(0.01)
-                        assert time() < start + 5
+@gen_cluster(client=True, nthreads=[])
+async def test_nanny_closes_cleanly_2(c, s):
+    async with Nanny(s.address) as n:
+        with c.rpc(n.worker_address) as w:
+            IOLoop.current().add_callback(w.terminate)
+            start = time()
+            while n.status != Status.closed:
+                await asyncio.sleep(0.01)
+                assert time() < start + 5
 
-                    assert n.status == Status.closed
+            assert n.status == Status.closed
 
 
-@pytest.mark.asyncio
-async def test_config(cleanup):
-    async with Scheduler() as s:
-        async with Nanny(s.address, config={"foo": "bar"}) as n:
-            async with Client(s.address, asynchronous=True) as client:
-                config = await client.run(dask.config.get, "foo")
-                assert config[n.worker_address] == "bar"
+@gen_cluster(client=True, nthreads=[])
+async def test_config(c, s):
+    async with Nanny(s.address, config={"foo": "bar"}) as n:
+        config = await c.run(dask.config.get, "foo")
+        assert config[n.worker_address] == "bar"
 
 
-@pytest.mark.asyncio
-async def test_nanny_port_range(cleanup):
-    async with Scheduler() as s:
-        async with Client(s.address, asynchronous=True) as client:
-            nanny_port = "9867:9868"
-            worker_port = "9869:9870"
-            async with Nanny(s.address, port=nanny_port, worker_port=worker_port) as n1:
-                assert n1.port == 9867  # Selects first port in range
-                async with Nanny(
-                    s.address, port=nanny_port, worker_port=worker_port
-                ) as n2:
-                    assert n2.port == 9868  # Selects next port in range
-                    with pytest.raises(
-                        ValueError, match="Could not start Nanny"
-                    ):  # No more ports left
-                        async with Nanny(
-                            s.address, port=nanny_port, worker_port=worker_port
-                        ):
-                            pass
-
-                    # Ensure Worker ports are in worker_port range
-                    def get_worker_port(dask_worker):
-                        return dask_worker.port
-
-                    worker_ports = await client.run(get_worker_port)
-                    assert list(worker_ports.values()) == parse_ports(worker_port)
+@gen_cluster(client=True, nthreads=[])
+async def test_nanny_port_range(c, s):
+    nanny_port = "9867:9868"
+    worker_port = "9869:9870"
+    async with Nanny(s.address, port=nanny_port, worker_port=worker_port) as n1:
+        assert n1.port == 9867  # Selects first port in range
+        async with Nanny(s.address, port=nanny_port, worker_port=worker_port) as n2:
+            assert n2.port == 9868  # Selects next port in range
+            with pytest.raises(
+                ValueError, match="Could not start Nanny"
+            ):  # No more ports left
+                async with Nanny(s.address, port=nanny_port, worker_port=worker_port):
+                    pass
+
+            # Ensure Worker ports are in worker_port range
+            def get_worker_port(dask_worker):
+                return dask_worker.port
+
+            worker_ports = await c.run(get_worker_port)
+            assert list(worker_ports.values()) == parse_ports(worker_port)
 
 
 class KeyboardInterruptWorker(worker.Worker):
@@ -546,7 +533,7 @@ async def test_nanny_closed_by_keyboard_interrupt(cleanup, protocol):
     if protocol == "ucx":  # Skip if UCX isn't available
         pytest.importorskip("ucp")
 
-    async with Scheduler(protocol=protocol) as s:
+    async with Scheduler(protocol=protocol, dashboard_address=":0") as s:
         async with Nanny(
             s.address, nthreads=1, worker_class=KeyboardInterruptWorker
         ) as n:
@@ -573,14 +560,13 @@ async def test_worker_start_exception(s):
             pass
 
 
-@pytest.mark.asyncio
-async def test_failure_during_worker_initialization(cleanup):
+@gen_cluster(nthreads=[])
+async def test_failure_during_worker_initialization(s):
     with captured_logger(logger="distributed.nanny", level=logging.WARNING) as logs:
-        async with Scheduler() as s:
-            with pytest.raises(Exception):
-                async with Nanny(s.address, foo="bar") as n:
-                    await n
-        assert "Restarting worker" not in logs.getvalue()
+        with pytest.raises(Exception):
+            async with Nanny(s.address, foo="bar") as n:
+                await n
+    assert "Restarting worker" not in logs.getvalue()
 
 
 @gen_cluster(client=True, Worker=Nanny, timeout=10000000)
diff --git a/distributed/tests/test_preload.py b/distributed/tests/test_preload.py
index 75946676d5c..1fc74ee346c 100644
--- a/distributed/tests/test_preload.py
+++ b/distributed/tests/test_preload.py
@@ -9,7 +9,7 @@
 import dask
 
 from distributed import Client, Nanny, Scheduler, Worker
-from distributed.utils_test import captured_logger, cluster
+from distributed.utils_test import captured_logger, cluster, gen_cluster, gen_test
 
 PRELOAD_TEXT = """
 _worker_info = {}
@@ -45,20 +45,20 @@ def check_worker():
         shutil.rmtree(tmpdir)
 
 
-@pytest.mark.asyncio
-async def test_worker_preload_text(cleanup):
+@gen_test()
+async def test_worker_preload_text():
     text = """
 def dask_setup(worker):
     worker.foo = 'setup'
 """
-    async with Scheduler(port=0, preload=text) as s:
+    async with Scheduler(dashboard_address=":0", preload=text) as s:
         assert s.foo == "setup"
         async with Worker(s.address, preload=[text]) as w:
             assert w.foo == "setup"
 
 
-@pytest.mark.asyncio
-async def test_worker_preload_config(cleanup):
+@gen_cluster(nthreads=[])
+async def test_worker_preload_config(s):
     text = """
 def dask_setup(worker):
     worker.foo = 'setup'
@@ -69,13 +69,12 @@ def dask_teardown(worker):
     with dask.config.set(
         {"distributed.worker.preload": text, "distributed.nanny.preload": text}
     ):
-        async with Scheduler(port=0) as s:
-            async with Nanny(s.address) as w:
-                assert w.foo == "setup"
-                async with Client(s.address, asynchronous=True) as c:
-                    d = await c.run(lambda dask_worker: dask_worker.foo)
-                    assert d == {w.worker_address: "setup"}
-            assert w.foo == "teardown"
+        async with Nanny(s.address) as w:
+            assert w.foo == "setup"
+            async with Client(s.address, asynchronous=True) as c:
+                d = await c.run(lambda dask_worker: dask_worker.foo)
+                assert d == {w.worker_address: "setup"}
+        assert w.foo == "teardown"
 
 
 def test_worker_preload_module(loop):
@@ -104,34 +103,33 @@ def check_worker():
         shutil.rmtree(tmpdir)
 
 
-@pytest.mark.asyncio
-async def test_worker_preload_click(cleanup, tmpdir):
-    CLICK_PRELOAD_TEXT = """
+@gen_cluster(nthreads=[])
+async def test_worker_preload_click(s):
+    text = """
 import click
 
 @click.command()
 def dask_setup(worker):
     worker.foo = 'setup'
 """
-    async with Scheduler(port=0) as s:
-        async with Worker(s.address, preload=CLICK_PRELOAD_TEXT) as w:
-            assert w.foo == "setup"
 
+    async with Worker(s.address, preload=text) as w:
+        assert w.foo == "setup"
 
-@pytest.mark.asyncio
-async def test_worker_preload_click_async(cleanup, tmpdir):
+
+@gen_cluster(nthreads=[])
+async def test_worker_preload_click_async(s, tmpdir):
     # Ensure we allow for click commands wrapping coroutines
     # https://github.com/dask/distributed/issues/4169
-    CLICK_PRELOAD_TEXT = """
+    text = """
 import click
 
 @click.command()
 async def dask_setup(worker):
     worker.foo = 'setup'
 """
-    async with Scheduler(port=0) as s:
-        async with Worker(s.address, preload=CLICK_PRELOAD_TEXT) as w:
-            assert w.foo == "setup"
+    async with Worker(s.address, preload=text) as w:
+        assert w.foo == "setup"
 
 
 @pytest.mark.asyncio
@@ -143,7 +141,7 @@ async def test_preload_import_time(cleanup):
 backends["foo"] = TCPBackend()
 """.strip()
     try:
-        async with Scheduler(port=0, preload=text, protocol="foo") as s:
+        async with Scheduler(dashboard_address=":0", preload=text, protocol="foo") as s:
             async with Nanny(s.address, preload=text, protocol="foo") as n:
                 async with Client(s.address, asynchronous=True) as c:
                     await c.wait_for_workers(1)
@@ -168,30 +166,31 @@ def dask_setup(dask_server):
     server = app.listen(12345)
     try:
         with captured_logger("distributed.preloading") as log:
-            async with Scheduler(preload=["http://localhost:12345/preload"]) as s:
+            async with Scheduler(
+                dashboard_address=":0",
+                preload=["http://localhost:12345/preload"],
+            ) as s:
                 assert s.foo == 1
         assert "12345/preload" in log.getvalue()
     finally:
         server.stop()
 
 
-@pytest.mark.asyncio
-async def test_scheduler_startup(cleanup):
-    async with Scheduler(port=0) as s:
-        text = f"""
+@gen_cluster(nthreads=[])
+async def test_scheduler_startup(s):
+    text = f"""
 import dask
 dask.config.set(scheduler_address="{s.address}")
 """
-        async with Worker(preload=text) as w:
-            assert w.scheduler.address == s.address
+    async with Worker(preload=text) as w:
+        assert w.scheduler.address == s.address
 
 
-@pytest.mark.asyncio
-async def test_scheduler_startup_nanny(cleanup):
-    async with Scheduler(port=0) as s:
-        text = f"""
+@gen_cluster(nthreads=[])
+async def test_scheduler_startup_nanny(s):
+    text = f"""
 import dask
 dask.config.set(scheduler_address="{s.address}")
 """
-        async with Nanny(preload_nanny=text) as w:
-            assert w.scheduler.address == s.address
+    async with Nanny(preload_nanny=text) as w:
+        assert w.scheduler.address == s.address
diff --git a/distributed/tests/test_queues.py b/distributed/tests/test_queues.py
index a948a8229b3..202aa928d84 100644
--- a/distributed/tests/test_queues.py
+++ b/distributed/tests/test_queues.py
@@ -110,7 +110,7 @@ def f(x):
 
 
 @pytest.mark.slow
-@gen_cluster(client=True, nthreads=[("127.0.0.1", 2)] * 5, Worker=Nanny)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 2)] * 5, Worker=Nanny, timeout=60)
 async def test_race(c, s, *workers):
     def f(i):
         with worker_client() as c:
diff --git a/distributed/tests/test_resources.py b/distributed/tests/test_resources.py
index 6adb9d2dd08..6f74279297f 100644
--- a/distributed/tests/test_resources.py
+++ b/distributed/tests/test_resources.py
@@ -348,7 +348,7 @@ async def test_dont_optimize_out(c, s, a, b):
         assert "executing" in str(a.story(key))
 
 
-@pytest.mark.xfail(reason="atop fusion seemed to break this")
+@pytest.mark.skip(reason="atop fusion seemed to break this")
 @gen_cluster(
     client=True,
     nthreads=[
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 45702266fea..536fd3fd45f 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -422,9 +422,10 @@ def func(scheduler):
     await comm.close()
 
 
-def test_scheduler_init_pulls_blocked_handlers_from_config():
-    with dask.config.set({"distributed.scheduler.blocked-handlers": ["test-handler"]}):
-        s = Scheduler()
+@gen_cluster(
+    nthreads=[], config={"distributed.scheduler.blocked-handlers": ["test-handler"]}
+)
+def test_scheduler_init_pulls_blocked_handlers_from_config(s):
     assert s.blocked_handlers == ["test-handler"]
 
 
@@ -675,9 +676,8 @@ async def test_broadcast_nanny(s, a, b):
     assert result1 == result3
 
 
-@gen_test()
-async def test_worker_name():
-    s = await Scheduler(validate=True, port=0)
+@gen_cluster(nthreads=[])
+async def test_worker_name(s):
     w = await Worker(s.address, name="alice")
     assert s.workers[w.address].name == "alice"
     assert s.aliases["alice"] == w.address
@@ -687,60 +687,53 @@ async def test_worker_name():
         await w2.close()
 
     await w.close()
-    await s.close()
 
 
-@gen_test()
-async def test_coerce_address():
-    with dask.config.set({"distributed.comm.timeouts.connect": "100ms"}):
-        s = await Scheduler(validate=True, port=0)
-        print("scheduler:", s.address, s.listen_address)
-        a = Worker(s.address, name="alice")
-        b = Worker(s.address, name=123)
-        c = Worker("127.0.0.1", s.port, name="charlie")
-        await asyncio.gather(a, b, c)
-
-        assert s.coerce_address("127.0.0.1:8000") == "tcp://127.0.0.1:8000"
-        assert s.coerce_address("[::1]:8000") == "tcp://[::1]:8000"
-        assert s.coerce_address("tcp://127.0.0.1:8000") == "tcp://127.0.0.1:8000"
-        assert s.coerce_address("tcp://[::1]:8000") == "tcp://[::1]:8000"
-        assert s.coerce_address("localhost:8000") in (
-            "tcp://127.0.0.1:8000",
-            "tcp://[::1]:8000",
-        )
-        assert s.coerce_address("localhost:8000") in (
-            "tcp://127.0.0.1:8000",
-            "tcp://[::1]:8000",
-        )
-        assert s.coerce_address(a.address) == a.address
-        # Aliases
-        assert s.coerce_address("alice") == a.address
-        assert s.coerce_address(123) == b.address
-        assert s.coerce_address("charlie") == c.address
+@gen_cluster(nthreads=[])
+async def test_coerce_address(s):
+    print("scheduler:", s.address, s.listen_address)
+    a = Worker(s.address, name="alice")
+    b = Worker(s.address, name=123)
+    c = Worker("127.0.0.1", s.port, name="charlie")
+    await asyncio.gather(a, b, c)
+
+    assert s.coerce_address("127.0.0.1:8000") == "tcp://127.0.0.1:8000"
+    assert s.coerce_address("[::1]:8000") == "tcp://[::1]:8000"
+    assert s.coerce_address("tcp://127.0.0.1:8000") == "tcp://127.0.0.1:8000"
+    assert s.coerce_address("tcp://[::1]:8000") == "tcp://[::1]:8000"
+    assert s.coerce_address("localhost:8000") in (
+        "tcp://127.0.0.1:8000",
+        "tcp://[::1]:8000",
+    )
+    assert s.coerce_address("localhost:8000") in (
+        "tcp://127.0.0.1:8000",
+        "tcp://[::1]:8000",
+    )
+    assert s.coerce_address(a.address) == a.address
+    # Aliases
+    assert s.coerce_address("alice") == a.address
+    assert s.coerce_address(123) == b.address
+    assert s.coerce_address("charlie") == c.address
 
-        assert s.coerce_hostname("127.0.0.1") == "127.0.0.1"
-        assert s.coerce_hostname("alice") == a.ip
-        assert s.coerce_hostname(123) == b.ip
-        assert s.coerce_hostname("charlie") == c.ip
-        assert s.coerce_hostname("jimmy") == "jimmy"
+    assert s.coerce_hostname("127.0.0.1") == "127.0.0.1"
+    assert s.coerce_hostname("alice") == a.ip
+    assert s.coerce_hostname(123) == b.ip
+    assert s.coerce_hostname("charlie") == c.ip
+    assert s.coerce_hostname("jimmy") == "jimmy"
 
-        assert s.coerce_address("zzzt:8000", resolve=False) == "tcp://zzzt:8000"
+    assert s.coerce_address("zzzt:8000", resolve=False) == "tcp://zzzt:8000"
+    await asyncio.gather(a.close(), b.close(), c.close())
 
-        await s.close()
-        await asyncio.gather(a.close(), b.close(), c.close())
 
+@gen_cluster(nthreads=[], config={"distributed.scheduler.work-stealing": True})
+async def test_config_stealing(s):
+    """Regression test for https://github.com/dask/distributed/issues/3409"""
+    assert "stealing" in s.extensions
 
-@pytest.mark.asyncio
-async def test_config_stealing(cleanup):
-    # Regression test for https://github.com/dask/distributed/issues/3409
-
-    with dask.config.set({"distributed.scheduler.work-stealing": True}):
-        async with Scheduler(port=0) as s:
-            assert "stealing" in s.extensions
 
-    with dask.config.set({"distributed.scheduler.work-stealing": False}):
-        async with Scheduler(port=0) as s:
-            assert "stealing" not in s.extensions
+@gen_cluster(nthreads=[], config={"distributed.scheduler.work-stealing": False})
+async def test_config_no_stealing(s):
+    assert "stealing" not in s.extensions
 
 
 @pytest.mark.skipif(WINDOWS, reason="num_fds not supported on windows")
@@ -773,7 +766,7 @@ async def test_update_graph_culls(s, a, b):
 
 
 def test_io_loop(loop):
-    s = Scheduler(loop=loop, validate=True)
+    s = Scheduler(loop=loop, dashboard_address=":0", validate=True)
     assert s.io_loop is loop
 
 
@@ -924,7 +917,7 @@ async def test_retire_workers_no_suspicious_tasks(c, s, a, b):
 
 @pytest.mark.slow
 @pytest.mark.skipif(WINDOWS, reason="num_fds not supported on windows")
-@gen_cluster(client=True, nthreads=[], timeout=60)
+@gen_cluster(client=True, nthreads=[], timeout=120)
 async def test_file_descriptors(c, s):
     await asyncio.sleep(0.1)
     da = pytest.importorskip("dask.array")
@@ -1286,7 +1279,7 @@ async def test_fifo_submission(c, s, w):
 @gen_test()
 async def test_scheduler_file():
     with tmpfile() as fn:
-        s = await Scheduler(scheduler_file=fn, port=0)
+        s = await Scheduler(scheduler_file=fn, dashboard_address=":0")
         with open(fn) as f:
             data = json.load(f)
         assert data["address"] == s.address
@@ -1486,10 +1479,10 @@ async def test_get_task_status(c, s, a, b):
     assert result == {future.key: "memory"}
 
 
-def test_deque_handler():
+@gen_cluster(nthreads=[])
+async def test_deque_handler(s):
     from distributed.scheduler import logger
 
-    s = Scheduler()
     deque_handler = s._deque_handler
     logger.info("foo123")
     assert len(deque_handler.deque) >= 1
@@ -1803,7 +1796,7 @@ async def test_close_workers(s, a, b):
 @pytest.mark.skipif(not LINUX, reason="Need 127.0.0.2 to mean localhost")
 @gen_test()
 async def test_host_address():
-    s = await Scheduler(host="127.0.0.2", port=0)
+    s = await Scheduler(host="127.0.0.2", dashboard_address=":0")
     assert "127.0.0.2" in s.address
     await s.close()
 
@@ -1811,25 +1804,20 @@ async def test_host_address():
 @gen_test()
 async def test_dashboard_address():
     pytest.importorskip("bokeh")
-    s = await Scheduler(dashboard_address="127.0.0.1:8901", port=0)
-    assert s.services["dashboard"].port == 8901
-    await s.close()
+    async with Scheduler(dashboard_address="127.0.0.1:8901") as s:
+        assert s.services["dashboard"].port == 8901
 
-    s = await Scheduler(dashboard_address="127.0.0.1", port=0)
-    assert s.services["dashboard"].port
-    await s.close()
+    async with Scheduler(dashboard_address="127.0.0.1") as s:
+        assert s.services["dashboard"].port
 
-    s = await Scheduler(dashboard_address="127.0.0.1:8901,127.0.0.1:8902", port=0)
-    assert s.services["dashboard"].port == 8901
-    await s.close()
+    async with Scheduler(dashboard_address="127.0.0.1:8901,127.0.0.1:8902") as s:
+        assert s.services["dashboard"].port == 8901
 
-    s = await Scheduler(dashboard_address=":8901,:8902", port=0)
-    assert s.services["dashboard"].port == 8901
-    await s.close()
+    async with Scheduler(dashboard_address=":8901,:8902") as s:
+        assert s.services["dashboard"].port == 8901
 
-    s = await Scheduler(dashboard_address=[8901, 8902], port=0)
-    assert s.services["dashboard"].port == 8901
-    await s.close()
+    async with Scheduler(dashboard_address=[8901, 8902]) as s:
+        assert s.services["dashboard"].port == 8901
 
 
 @gen_cluster(client=True)
@@ -1858,9 +1846,9 @@ async def test_adaptive_target(c, s, a, b):
         assert s.adaptive_target(target_duration=".1s") == 0
 
 
-@pytest.mark.asyncio
-async def test_async_context_manager(cleanup):
-    async with Scheduler(port=0) as s:
+@gen_test()
+async def test_async_context_manager():
+    async with Scheduler(dashboard_address=":0") as s:
         assert s.status == Status.running
         async with Worker(s.address) as w:
             assert w.status == Status.running
@@ -1868,23 +1856,23 @@ async def test_async_context_manager(cleanup):
         assert not s.workers
 
 
-@pytest.mark.asyncio
-async def test_allowed_failures_config(cleanup):
-    async with Scheduler(port=0, allowed_failures=10) as s:
+@gen_test()
+async def test_allowed_failures_config():
+    async with Scheduler(dashboard_address=":0", allowed_failures=10) as s:
         assert s.allowed_failures == 10
 
     with dask.config.set({"distributed.scheduler.allowed_failures": 100}):
-        async with Scheduler(port=0) as s:
+        async with Scheduler(dashboard_address=":0") as s:
             assert s.allowed_failures == 100
 
     with dask.config.set({"distributed.scheduler.allowed_failures": 0}):
-        async with Scheduler(port=0) as s:
+        async with Scheduler(dashboard_address=":0") as s:
             assert s.allowed_failures == 0
 
 
-@pytest.mark.asyncio
+@gen_test()
 async def test_finished():
-    async with Scheduler(port=0) as s:
+    async with Scheduler(dashboard_address=":0") as s:
         async with Worker(s.address) as w:
             pass
 
@@ -1892,48 +1880,45 @@ async def test_finished():
     await w.finished()
 
 
-@pytest.mark.asyncio
-async def test_retire_names_str(cleanup):
-    async with Scheduler(port=0) as s:
-        async with Worker(s.address, name="0") as a:
-            async with Worker(s.address, name="1") as b:
-                async with Client(s.address, asynchronous=True) as c:
-                    futures = c.map(inc, range(10))
-                    await wait(futures)
-                    assert a.data and b.data
-                    await s.retire_workers(names=[0])
-                    assert all(f.done() for f in futures)
-                    assert len(b.data) == 10
+@gen_cluster(nthreads=[], client=True)
+async def test_retire_names_str(c, s):
+    async with Worker(s.address, name="0") as a, Worker(s.address, name="1") as b:
+        futures = c.map(inc, range(10))
+        await wait(futures)
+        assert a.data and b.data
+        await s.retire_workers(names=[0])
+        assert all(f.done() for f in futures)
+        assert len(b.data) == 10
 
 
-@gen_cluster(client=True)
+@gen_cluster(
+    client=True, config={"distributed.scheduler.default-task-durations": {"inc": 100}}
+)
 async def test_get_task_duration(c, s, a, b):
-    with dask.config.set(
-        {"distributed.scheduler.default-task-durations": {"inc": 100}}
-    ):
-        future = c.submit(inc, 1)
-        await future
-        assert 10 < s.task_prefixes["inc"].duration_average < 100
+    future = c.submit(inc, 1)
+    await future
+    assert 10 < s.task_prefixes["inc"].duration_average < 100
 
-        ts_pref1 = s.new_task("inc-abcdefab", None, "released")
-        assert 10 < s.get_task_duration(ts_pref1) < 100
+    ts_pref1 = s.new_task("inc-abcdefab", None, "released")
+    assert 10 < s.get_task_duration(ts_pref1) < 100
 
-        # make sure get_task_duration adds TaskStates to unknown dict
-        assert len(s.unknown_durations) == 0
-        x = c.submit(slowinc, 1, delay=0.5)
-        while len(s.tasks) < 3:
-            await asyncio.sleep(0.01)
+    # make sure get_task_duration adds TaskStates to unknown dict
+    assert len(s.unknown_durations) == 0
+    x = c.submit(slowinc, 1, delay=0.5)
+    while len(s.tasks) < 3:
+        await asyncio.sleep(0.01)
 
-        ts = s.tasks[x.key]
-        assert s.get_task_duration(ts) == 0.5  # default
-        assert len(s.unknown_durations) == 1
-        assert len(s.unknown_durations["slowinc"]) == 1
+    ts = s.tasks[x.key]
+    assert s.get_task_duration(ts) == 0.5  # default
+    assert len(s.unknown_durations) == 1
+    assert len(s.unknown_durations["slowinc"]) == 1
 
 
 @gen_cluster(client=True)
 async def test_default_task_duration_splits(c, s, a, b):
-    """This test ensures that the default task durations for shuffle split tasks are, by default, aligned with the task names of dask.dask"""
-
+    """Ensure that the default task durations for shuffle split tasks are, by default,
+    aligned with the task names of dask.dask
+    """
     pd = pytest.importorskip("pandas")
     dd = pytest.importorskip("dask.dataframe")
 
@@ -1958,13 +1943,13 @@ async def test_default_task_duration_splits(c, s, a, b):
     assert default_time <= 1e-6
 
 
-@pytest.mark.asyncio
-async def test_no_danglng_asyncio_tasks(cleanup):
+@gen_test()
+async def test_no_danglng_asyncio_tasks():
     start = asyncio.all_tasks()
-    async with Scheduler(port=0) as s:
-        async with Worker(s.address, name="0") as a:
+    async with Scheduler(dashboard_address=":0") as s:
+        async with Worker(s.address, name="0"):
             async with Client(s.address, asynchronous=True) as c:
-                await asyncio.sleep(0.01)
+                await c.submit(lambda: 1)
 
     tasks = asyncio.all_tasks()
     assert tasks == start
@@ -2311,10 +2296,10 @@ async def test_too_many_groups(c, s, a, b):
     assert len(s.task_groups) < 3
 
 
-@pytest.mark.asyncio
-async def test_multiple_listeners(cleanup):
+@gen_test()
+async def test_multiple_listeners():
     with captured_logger(logging.getLogger("distributed.scheduler")) as log:
-        async with Scheduler(port=0, protocol=["inproc", "tcp"]) as s:
+        async with Scheduler(dashboard_address=":0", protocol=["inproc", "tcp"]) as s:
             async with Worker(s.listeners[0].contact_address) as a:
                 async with Worker(s.listeners[1].contact_address) as b:
                     assert a.address.startswith("inproc")
@@ -2374,13 +2359,15 @@ async def test_retire_state_change(c, s, a, b):
     np = pytest.importorskip("numpy")
     y = c.map(lambda x: x ** 2, range(10))
     await c.scatter(y)
+    coros = []
     for x in range(2):
         v = c.map(lambda i: i * np.random.randint(1000), y)
         k = c.map(lambda i: i * np.random.randint(1000), v)
         foo = c.map(lambda j: j * 6, k)
         step = c.compute(foo)
-        c.gather(step)
+        coros.append(c.gather(step))
     await c.retire_workers(workers=[a.address])
+    await asyncio.gather(*coros)
 
 
 @gen_cluster(client=True, config={"distributed.scheduler.events-log-length": 3})
diff --git a/distributed/tests/test_semaphore.py b/distributed/tests/test_semaphore.py
index 0cc4bc9b155..bf0f8181e17 100644
--- a/distributed/tests/test_semaphore.py
+++ b/distributed/tests/test_semaphore.py
@@ -183,7 +183,8 @@ def f(x, release=True):
     assert result.count(False) == 9
 
 
-@gen_cluster(client=True)
+@pytest.mark.slow
+@gen_cluster(client=True, timeout=120)
 async def test_close_async(c, s, a, b):
     sem = await Semaphore(name="test")
 
@@ -515,7 +516,9 @@ def test_threadpoolworkers_pick_correct_ioloop(cleanup):
             "distributed.scheduler.locks.lease-timeout": 0.1,
         }
     ):
-        with Client(processes=False, threads_per_worker=4) as client:
+        with Client(
+            processes=False, dashboard_address=":0", threads_per_worker=4
+        ) as client:
             sem = Semaphore(max_leases=1, name="database")
             protected_resource = []
 
diff --git a/distributed/tests/test_stress.py b/distributed/tests/test_stress.py
index dba1daa721f..3781e7c38e2 100644
--- a/distributed/tests/test_stress.py
+++ b/distributed/tests/test_stress.py
@@ -89,7 +89,7 @@ def test_cancel_stress_sync(loop):
 @gen_cluster(
     nthreads=[],
     client=True,
-    timeout=120,
+    timeout=180,
     scheduler_kwargs={"allowed_failures": 100_000},
 )
 async def test_stress_creation_and_deletion(c, s):
@@ -220,7 +220,7 @@ async def test_stress_steal(c, s, *workers):
 
 
 @pytest.mark.slow
-@gen_cluster(nthreads=[("127.0.0.1", 1)] * 10, client=True, timeout=120)
+@gen_cluster(nthreads=[("127.0.0.1", 1)] * 10, client=True, timeout=180)
 async def test_close_connections(c, s, *workers):
     da = pytest.importorskip("dask.array")
     x = da.random.random(size=(1000, 1000), chunks=(1000, 1))
diff --git a/distributed/tests/test_tls_functional.py b/distributed/tests/test_tls_functional.py
index 54a1202400e..afd7ad96e41 100644
--- a/distributed/tests/test_tls_functional.py
+++ b/distributed/tests/test_tls_functional.py
@@ -4,13 +4,12 @@
 """
 import asyncio
 
-import pytest
-
 from distributed import Client, Nanny, Queue, Scheduler, Worker, wait, worker_client
 from distributed.core import Status
 from distributed.metrics import time
 from distributed.utils_test import (
     double,
+    gen_test,
     gen_tls_cluster,
     inc,
     slowadd,
@@ -202,17 +201,17 @@ async def test_retire_workers(c, s, a, b):
         assert time() < start + 5
 
 
-@pytest.mark.asyncio
-async def test_security_dict_input_no_security(cleanup):
-    async with Scheduler(security={}) as s:
-        async with Worker(s.address, security={}) as w:
+@gen_test()
+async def test_security_dict_input_no_security():
+    async with Scheduler(dashboard_address=":0", security={}) as s:
+        async with Worker(s.address, security={}):
             async with Client(s.address, security={}, asynchronous=True) as c:
                 result = await c.submit(inc, 1)
                 assert result == 2
 
 
-@pytest.mark.asyncio
-async def test_security_dict_input(cleanup):
+@gen_test()
+async def test_security_dict_input():
     conf = tls_config()
     ca_file = conf["distributed"]["comm"]["tls"]["ca-file"]
     client = conf["distributed"]["comm"]["tls"]["client"]["cert"]
@@ -221,6 +220,7 @@ async def test_security_dict_input(cleanup):
 
     async with Scheduler(
         host="localhost",
+        dashboard_address=":0",
         security={"tls_ca_file": ca_file, "tls_scheduler_cert": scheduler},
     ) as s:
         assert s.address.startswith("tls://")
diff --git a/distributed/tests/test_utils_perf.py b/distributed/tests/test_utils_perf.py
index 82dfac5653e..5bf54daf178 100644
--- a/distributed/tests/test_utils_perf.py
+++ b/distributed/tests/test_utils_perf.py
@@ -81,6 +81,7 @@ def enable_gc_diagnosis_and_log(diag, level="INFO"):
             gc.enable()
 
 
+@pytest.mark.slow
 def test_gc_diagnosis_cpu_time():
     diag = GCDiagnosis(warn_over_frac=0.75)
     diag.N_SAMPLES = 3  # shorten tests
diff --git a/distributed/tests/test_utils_test.py b/distributed/tests/test_utils_test.py
index 3dca5766f85..1b02e6a4251 100755
--- a/distributed/tests/test_utils_test.py
+++ b/distributed/tests/test_utils_test.py
@@ -265,5 +265,7 @@ def test_tls_cluster(tls_client):
 
 @pytest.mark.asyncio
 async def test_tls_scheduler(security, cleanup):
-    async with Scheduler(security=security, host="localhost") as s:
+    async with Scheduler(
+        security=security, host="localhost", dashboard_address=":0"
+    ) as s:
         assert s.address.startswith("tls")
diff --git a/distributed/tests/test_variable.py b/distributed/tests/test_variable.py
index 46eb8604058..3319d92f9d5 100644
--- a/distributed/tests/test_variable.py
+++ b/distributed/tests/test_variable.py
@@ -191,7 +191,7 @@ async def test_timeout_get(c, s, a, b):
 
 
 @pytest.mark.slow
-@gen_cluster(client=True, nthreads=[("127.0.0.1", 2)] * 5, Worker=Nanny)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 2)] * 5, Worker=Nanny, timeout=60)
 async def test_race(c, s, *workers):
     NITERS = 50
 
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 7ba5ef166cd..c558c6ae09d 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -30,7 +30,7 @@
 )
 from distributed.comm.registry import backends
 from distributed.comm.tcp import TCPBackend
-from distributed.compatibility import LINUX, MACOS, WINDOWS
+from distributed.compatibility import LINUX, WINDOWS
 from distributed.core import CommClosedError, Status, rpc
 from distributed.diagnostics.plugin import PipInstall
 from distributed.metrics import time
@@ -54,11 +54,10 @@
 pytestmark = pytest.mark.ci1
 
 
-@pytest.mark.asyncio
-async def test_worker_nthreads(cleanup):
-    async with Scheduler() as s:
-        async with Worker(s.address) as w:
-            assert w.executor._max_workers == CPU_COUNT
+@gen_cluster(nthreads=[])
+async def test_worker_nthreads(s):
+    async with Worker(s.address) as w:
+        assert w.executor._max_workers == CPU_COUNT
 
 
 @gen_cluster()
@@ -70,15 +69,14 @@ async def test_str(s, a, b):
     assert str(a.executing_count) in repr(a)
 
 
-@pytest.mark.asyncio
-async def test_identity(cleanup):
-    async with Scheduler() as s:
-        async with Worker(s.address) as w:
-            ident = w.identity(None)
-            assert "Worker" in ident["type"]
-            assert ident["scheduler"] == s.address
-            assert isinstance(ident["nthreads"], int)
-            assert isinstance(ident["memory_limit"], Number)
+@gen_cluster(nthreads=[])
+async def test_identity(s):
+    async with Worker(s.address) as w:
+        ident = w.identity(None)
+        assert "Worker" in ident["type"]
+        assert ident["scheduler"] == s.address
+        assert isinstance(ident["nthreads"], int)
+        assert isinstance(ident["memory_limit"], Number)
 
 
 @gen_cluster(client=True)
@@ -282,34 +280,30 @@ async def test_broadcast(s, a, b):
         assert results == {a.address: b"pong", b.address: b"pong"}
 
 
-@gen_test()
-async def test_worker_with_port_zero():
-    s = await Scheduler(port=8007)
-    w = await Worker(s.address)
-    assert isinstance(w.port, int)
-    assert w.port > 1024
-
-    await w.close()
+@gen_cluster(nthreads=[])
+async def test_worker_with_port_zero(s):
+    async with Worker(s.address) as w:
+        assert isinstance(w.port, int)
+        assert w.port > 1024
 
 
-@pytest.mark.asyncio
-async def test_worker_port_range(cleanup):
-    async with Scheduler() as s:
-        port = "9867:9868"
-        async with Worker(s.address, port=port) as w1:
-            assert w1.port == 9867  # Selects first port in range
-            async with Worker(s.address, port=port) as w2:
-                assert w2.port == 9868  # Selects next port in range
-                with pytest.raises(
-                    ValueError, match="Could not start Worker"
-                ):  # No more ports left
-                    async with Worker(s.address, port=port):
-                        pass
+@gen_cluster(nthreads=[])
+async def test_worker_port_range(s):
+    port = "9867:9868"
+    async with Worker(s.address, port=port) as w1:
+        assert w1.port == 9867  # Selects first port in range
+        async with Worker(s.address, port=port) as w2:
+            assert w2.port == 9868  # Selects next port in range
+            with pytest.raises(
+                ValueError, match="Could not start Worker"
+            ):  # No more ports left
+                async with Worker(s.address, port=port):
+                    pass
 
 
 @pytest.mark.slow
-@pytest.mark.asyncio
-async def test_worker_waits_for_scheduler(cleanup):
+@gen_test(timeout=60)
+async def test_worker_waits_for_scheduler():
     w = Worker("127.0.0.1:8724")
     try:
         await asyncio.wait_for(w, 3)
@@ -438,11 +432,10 @@ async def test_gather_missing_workers_replicated(c, s, a, b, missing_first):
     assert a.data[x.key] == b.data[x.key] == "x"
 
 
-@pytest.mark.asyncio
-async def test_io_loop(cleanup):
-    async with Scheduler(port=0) as s:
-        async with Worker(s.address, loop=s.loop) as w:
-            assert w.io_loop is s.loop
+@gen_cluster(nthreads=[])
+async def test_io_loop(s):
+    async with Worker(s.address, loop=s.loop) as w:
+        assert w.io_loop is s.loop
 
 
 @gen_cluster(client=True, nthreads=[])
@@ -547,21 +540,18 @@ async def test_close_on_disconnect(s, w):
         assert time() < start + 5
 
 
-@pytest.mark.asyncio
-async def test_memory_limit_auto():
-    async with Scheduler() as s:
-        async with Worker(s.address, nthreads=1) as a, Worker(
-            s.address, nthreads=2
-        ) as b, Worker(s.address, nthreads=100) as c, Worker(
-            s.address, nthreads=200
-        ) as d:
-            assert isinstance(a.memory_limit, Number)
-            assert isinstance(b.memory_limit, Number)
+@gen_cluster(nthreads=[])
+async def test_memory_limit_auto(s):
+    async with Worker(s.address, nthreads=1) as a, Worker(
+        s.address, nthreads=2
+    ) as b, Worker(s.address, nthreads=100) as c, Worker(s.address, nthreads=200) as d:
+        assert isinstance(a.memory_limit, Number)
+        assert isinstance(b.memory_limit, Number)
 
-            if CPU_COUNT > 1:
-                assert a.memory_limit < b.memory_limit
+        if CPU_COUNT > 1:
+            assert a.memory_limit < b.memory_limit
 
-            assert c.memory_limit == d.memory_limit
+        assert c.memory_limit == d.memory_limit
 
 
 @gen_cluster(client=True)
@@ -799,19 +789,16 @@ async def test_hold_onto_dependents(c, s, a, b):
         await asyncio.sleep(0.1)
 
 
+# Normally takes >2s but it has been observed to take >30s occasionally
 @pytest.mark.slow
-@gen_cluster(nthreads=[])
-async def test_worker_death_timeout(s):
-    with dask.config.set({"distributed.comm.timeouts.connect": "1s"}):
-        await s.close()
-        w = Worker(s.address, death_timeout=1)
-
+@gen_test(timeout=120)
+async def test_worker_death_timeout():
+    w = Worker("tcp://127.0.0.1:12345", death_timeout=0.1)
     with pytest.raises(TimeoutError) as info:
         await w
 
     assert "Worker" in str(info.value)
     assert "timed out" in str(info.value) or "failed to start" in str(info.value)
-
     assert w.status == Status.closed
 
 
@@ -1064,11 +1051,9 @@ async def test_start_services(s):
 @gen_test()
 async def test_scheduler_file():
     with tmpfile() as fn:
-        s = await Scheduler(scheduler_file=fn, port=8009)
-        w = await Worker(scheduler_file=fn)
-        assert set(s.workers) == {w.address}
-        await w.close()
-        s.stop()
+        async with Scheduler(scheduler_file=fn, dashboard_address=":0") as s:
+            async with Worker(scheduler_file=fn) as w:
+                assert set(s.workers) == {w.address}
 
 
 @gen_cluster(client=True)
@@ -1226,18 +1211,17 @@ def f(x):
     assert all(f.key in b.data for f in futures)
 
 
-@pytest.mark.asyncio
-async def test_deque_handler(cleanup):
+@gen_cluster(nthreads=[])
+async def test_deque_handler(s):
     from distributed.worker import logger
 
-    async with Scheduler() as s:
-        async with Worker(s.address) as w:
-            deque_handler = w._deque_handler
-            logger.info("foo456")
-            assert deque_handler.deque
-            msg = deque_handler.deque[-1]
-            assert "distributed.worker" in deque_handler.format(msg)
-            assert any(msg.msg == "foo456" for msg in deque_handler.deque)
+    async with Worker(s.address) as w:
+        deque_handler = w._deque_handler
+        logger.info("foo456")
+        assert deque_handler.deque
+        msg = deque_handler.deque[-1]
+        assert "distributed.worker" in deque_handler.format(msg)
+        assert any(msg.msg == "foo456" for msg in deque_handler.deque)
 
 
 @gen_cluster(nthreads=[], client=True)
@@ -1493,7 +1477,7 @@ def test_resource_limit(monkeypatch):
 
 @pytest.mark.asyncio
 @pytest.mark.parametrize("Worker", [Worker, Nanny])
-async def test_interface_async(loop, Worker):
+async def test_interface_async(cleanup, loop, Worker):
     from distributed.utils import get_ip_interface
 
     psutil = pytest.importorskip("psutil")
@@ -1512,7 +1496,7 @@ async def test_interface_async(loop, Worker):
             "Available interfaces are: %s." % (if_names,)
         )
 
-    async with Scheduler(interface=if_name) as s:
+    async with Scheduler(dashboard_address=":0", interface=if_name) as s:
         assert s.address.startswith("tcp://127.0.0.1")
         async with Worker(s.address, interface=if_name) as w:
             assert w.address.startswith("tcp://127.0.0.1")
@@ -1525,10 +1509,10 @@ async def test_interface_async(loop, Worker):
 
 @pytest.mark.asyncio
 @pytest.mark.parametrize("Worker", [Worker, Nanny])
-async def test_protocol_from_scheduler_address(Worker):
-    ucp = pytest.importorskip("ucp")
+async def test_protocol_from_scheduler_address(cleanup, Worker):
+    pytest.importorskip("ucp")
 
-    async with Scheduler(protocol="ucx") as s:
+    async with Scheduler(protocol="ucx", dashboard_address=":0") as s:
         assert s.address.startswith("ucx://")
         async with Worker(s.address) as w:
             assert w.address.startswith("ucx://")
@@ -1549,8 +1533,8 @@ def get_address_host(self, loc):
     monkeypatch.setitem(backends, "foo", BadBackend())
 
     with dask.config.set({"distributed.comm.default-scheme": "foo"}):
-        async with Scheduler(protocol="tcp") as s:
-            async with Worker(s.address) as w:
+        async with Scheduler(protocol="tcp", dashboard_address=":0") as s:
+            async with Worker(s.address):
                 # Ensure that worker is able to properly start up
                 # without BadBackend.get_address_host raising a ValueError
                 pass
@@ -1558,8 +1542,8 @@ def get_address_host(self, loc):
 
 @pytest.mark.asyncio
 @pytest.mark.parametrize("Worker", [Worker, Nanny])
-async def test_worker_listens_on_same_interface_by_default(Worker):
-    async with Scheduler(host="localhost") as s:
+async def test_worker_listens_on_same_interface_by_default(cleanup, Worker):
+    async with Scheduler(host="localhost", dashboard_address=":0") as s:
         assert s.ip in {"127.0.0.1", "localhost"}
         async with Worker(s.address) as w:
             assert s.ip == w.ip
@@ -1584,46 +1568,41 @@ async def test_close_gracefully(c, s, a, b):
 
 
 @pytest.mark.slow
-@pytest.mark.asyncio
-async def test_lifetime(cleanup):
-    async with Scheduler() as s:
-        async with Worker(s.address) as a, Worker(s.address, lifetime="1 seconds") as b:
-            async with Client(s.address, asynchronous=True) as c:
-                futures = c.map(slowinc, range(200), delay=0.1, worker=[b.address])
-                await asyncio.sleep(1.5)
-                assert b.status != Status.running
-                await b.finished()
-
-                assert set(b.data) == set(a.data)  # successfully moved data over
+@gen_cluster(client=True, nthreads=[])
+async def test_lifetime(c, s):
+    async with Worker(s.address) as a, Worker(s.address, lifetime="1 seconds") as b:
+        futures = c.map(slowinc, range(200), delay=0.1, worker=[b.address])
+        await asyncio.sleep(1.5)
+        assert b.status != Status.running
+        await b.finished()
+        assert set(b.data) == set(a.data)  # successfully moved data over
 
 
-@gen_cluster(client=True, worker_kwargs={"lifetime": "10s", "lifetime_stagger": "2s"})
-async def test_lifetime_stagger(c, s, a, b):
+@gen_cluster(worker_kwargs={"lifetime": "10s", "lifetime_stagger": "2s"})
+async def test_lifetime_stagger(s, a, b):
     assert a.lifetime != b.lifetime
     assert 8 <= a.lifetime <= 12
     assert 8 <= b.lifetime <= 12
 
 
-@pytest.mark.asyncio
-async def test_bad_metrics(cleanup):
+@gen_cluster(nthreads=[])
+async def test_bad_metrics(s):
     def bad_metric(w):
         raise Exception("Hello")
 
-    async with Scheduler() as s:
-        async with Worker(s.address, metrics={"bad": bad_metric}) as w:
-            assert "bad" not in s.workers[w.address].metrics
+    async with Worker(s.address, metrics={"bad": bad_metric}) as w:
+        assert "bad" not in s.workers[w.address].metrics
 
 
-@pytest.mark.asyncio
-async def test_bad_startup(cleanup):
+@gen_cluster(nthreads=[])
+async def test_bad_startup(s):
     def bad_startup(w):
         raise Exception("Hello")
 
-    async with Scheduler() as s:
-        try:
-            w = await Worker(s.address, startup_information={"bad": bad_startup})
-        except Exception:
-            pytest.fail("Startup exception was raised")
+    try:
+        await Worker(s.address, startup_information={"bad": bad_startup})
+    except Exception:
+        pytest.fail("Startup exception was raised")
 
 
 @gen_cluster(client=True)
@@ -1673,97 +1652,87 @@ async def test_pip_install_fails(c, s, a, b):
 #             assert args[1:] == ["-m", "pip", "--upgrade", "install", "requests"]
 
 
-@pytest.mark.asyncio
-async def test_update_latency(cleanup):
-    async with await Scheduler() as s:
-        async with await Worker(s.address) as w:
-            original = w.latency
-            await w.heartbeat()
-            assert original != w.latency
+@gen_cluster(nthreads=[])
+async def test_update_latency(s):
+    async with await Worker(s.address) as w:
+        original = w.latency
+        await w.heartbeat()
+        assert original != w.latency
 
-            if w.digests is not None:
-                assert w.digests["latency"].size() > 0
+        if w.digests is not None:
+            assert w.digests["latency"].size() > 0
 
 
-@pytest.mark.skipif(MACOS, reason="frequently hangs")
-@pytest.mark.asyncio
-async def test_workerstate_executing(cleanup):
-    async with await Scheduler() as s:
-        async with await Worker(s.address) as w:
-            async with Client(s.address, asynchronous=True) as c:
-                ws = s.workers[w.address]
-                # Initially there are no active tasks
-                assert not ws.executing
-                # Submit a task and ensure the WorkerState is updated with the task
-                # it's executing
-                f = c.submit(slowinc, 1, delay=1)
-                while not ws.executing:
-                    await asyncio.sleep(0.01)
-                assert s.tasks[f.key] in ws.executing
-                await f
+@pytest.mark.slow
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)])
+async def test_workerstate_executing(c, s, a):
+    ws = s.workers[a.address]
+    # Initially there are no active tasks
+    assert not ws.executing
+    # Submit a task and ensure the WorkerState is updated with the task
+    # it's executing
+    f = c.submit(slowinc, 1, delay=3)
+    while not ws.executing:
+        assert f.status == "pending"
+        await asyncio.sleep(0.01)
+    assert s.tasks[f.key] in ws.executing
+    await f
 
 
-@pytest.mark.asyncio
 @pytest.mark.parametrize("reconnect", [True, False])
-async def test_heartbeat_comm_closed(cleanup, monkeypatch, reconnect):
+@gen_cluster(nthreads=[])
+async def test_heartbeat_comm_closed(s, monkeypatch, reconnect):
     with captured_logger("distributed.worker", level=logging.WARNING) as logger:
-        async with await Scheduler() as s:
 
-            def bad_heartbeat_worker(*args, **kwargs):
-                raise CommClosedError()
+        def bad_heartbeat_worker(*args, **kwargs):
+            raise CommClosedError()
 
-            async with await Worker(s.address, reconnect=reconnect) as w:
-                # Trigger CommClosedError during worker heartbeat
-                monkeypatch.setattr(
-                    w.scheduler, "heartbeat_worker", bad_heartbeat_worker
-                )
+        async with await Worker(s.address, reconnect=reconnect) as w:
+            # Trigger CommClosedError during worker heartbeat
+            monkeypatch.setattr(w.scheduler, "heartbeat_worker", bad_heartbeat_worker)
 
-                await w.heartbeat()
-                if reconnect:
-                    assert w.status == Status.running
-                else:
-                    assert w.status == Status.closed
+            await w.heartbeat()
+            if reconnect:
+                assert w.status == Status.running
+            else:
+                assert w.status == Status.closed
     assert "Heartbeat to scheduler failed" in logger.getvalue()
 
 
-@pytest.mark.asyncio
-async def test_bad_local_directory(cleanup):
-    async with await Scheduler() as s:
-        try:
-            async with Worker(s.address, local_directory="/not/a/valid-directory"):
-                pass
-        except OSError:
-            # On Linux: [Errno 13] Permission denied: '/not'
-            # On MacOSX: [Errno 30] Read-only file system: '/not'
+@gen_cluster(nthreads=[])
+async def test_bad_local_directory(s):
+    try:
+        async with Worker(s.address, local_directory="/not/a/valid-directory"):
             pass
-        else:
-            assert WINDOWS
+    except OSError:
+        # On Linux: [Errno 13] Permission denied: '/not'
+        # On MacOSX: [Errno 30] Read-only file system: '/not'
+        pass
+    else:
+        assert WINDOWS
 
-        assert not any("error" in log for log in s.get_logs())
+    assert not any("error" in log for log in s.get_logs())
 
 
-@pytest.mark.asyncio
-async def test_taskstate_metadata(cleanup):
-
-    async with await Scheduler() as s:
-        async with await Worker(s.address) as w:
-            async with Client(s.address, asynchronous=True) as c:
-                await c.register_worker_plugin(TaskStateMetadataPlugin())
+@gen_cluster(client=True, nthreads=[])
+async def test_taskstate_metadata(c, s):
+    async with await Worker(s.address) as w:
+        await c.register_worker_plugin(TaskStateMetadataPlugin())
 
-                f = c.submit(inc, 1)
-                await f
+        f = c.submit(inc, 1)
+        await f
 
-                ts = w.tasks[f.key]
-                assert "start_time" in ts.metadata
-                assert "stop_time" in ts.metadata
-                assert ts.metadata["stop_time"] > ts.metadata["start_time"]
+        ts = w.tasks[f.key]
+        assert "start_time" in ts.metadata
+        assert "stop_time" in ts.metadata
+        assert ts.metadata["stop_time"] > ts.metadata["start_time"]
 
-                # Check that Scheduler TaskState.metadata was also updated
-                assert s.tasks[f.key].metadata == ts.metadata
+        # Check that Scheduler TaskState.metadata was also updated
+        assert s.tasks[f.key].metadata == ts.metadata
 
 
-@pytest.mark.asyncio
-async def test_executor_offload(cleanup, monkeypatch):
+@gen_cluster(client=True, nthreads=[])
+async def test_executor_offload(c, s, monkeypatch):
     class SameThreadClass:
         def __getstate__(self):
             return ()
@@ -1774,19 +1743,17 @@ def __setstate__(self, state):
 
     monkeypatch.setattr("distributed.worker.OFFLOAD_THRESHOLD", 1)
 
-    async with Scheduler() as s:
-        async with Worker(s.address, executor="offload") as w:
-            from distributed.utils import _offload_executor
+    async with Worker(s.address, executor="offload") as w:
+        from distributed.utils import _offload_executor
 
-            assert w.executor is _offload_executor
+        assert w.executor is _offload_executor
 
-            async with Client(s.address, asynchronous=True) as c:
-                x = SameThreadClass()
+        x = SameThreadClass()
 
-                def f(x):
-                    return threading.get_ident() == x._thread_ident
+        def f(x):
+            return threading.get_ident() == x._thread_ident
 
-                assert await c.submit(f, x)
+        assert await c.submit(f, x)
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)])
@@ -2003,28 +1970,24 @@ def get_worker_client_id():
             default_client()
 
 
-@pytest.mark.asyncio
-async def test_multiple_executors(cleanup):
+@gen_cluster(client=True, nthreads=[])
+async def test_multiple_executors(c, s):
     def get_thread_name():
         return threading.current_thread().name
 
-    async with Scheduler() as s:
-        async with Worker(
-            s.address,
-            nthreads=2,
-            executor={
-                "GPU": ThreadPoolExecutor(1, thread_name_prefix="Dask-GPU-Threads")
-            },
-        ) as w:
-            async with Client(s.address, asynchronous=True) as c:
-                futures = []
-                with dask.annotate(executor="default"):
-                    futures.append(c.submit(get_thread_name, pure=False))
-                with dask.annotate(executor="GPU"):
-                    futures.append(c.submit(get_thread_name, pure=False))
-                default_result, gpu_result = await c.gather(futures)
-                assert "Dask-Default-Threads" in default_result
-                assert "Dask-GPU-Threads" in gpu_result
+    async with Worker(
+        s.address,
+        nthreads=2,
+        executor={"GPU": ThreadPoolExecutor(1, thread_name_prefix="Dask-GPU-Threads")},
+    ):
+        futures = []
+        with dask.annotate(executor="default"):
+            futures.append(c.submit(get_thread_name, pure=False))
+        with dask.annotate(executor="GPU"):
+            futures.append(c.submit(get_thread_name, pure=False))
+        default_result, gpu_result = await c.gather(futures)
+        assert "Dask-Default-Threads" in default_result
+        assert "Dask-GPU-Threads" in gpu_result
 
 
 @gen_cluster(client=True)
diff --git a/distributed/tests/test_worker_client.py b/distributed/tests/test_worker_client.py
index 5fcc01877ef..64b5da6d037 100644
--- a/distributed/tests/test_worker_client.py
+++ b/distributed/tests/test_worker_client.py
@@ -201,7 +201,7 @@ def f(x):
     b2 = b.map(f)
 
     with Client(
-        loop=loop, processes=False, set_as_default=True, dashboard_address=None
+        loop=loop, processes=False, set_as_default=True, dashboard_address=":0"
     ) as c:
         assert dask.base.get_scheduler() == c.get
         for i in range(2):
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index cc21c7db753..bc02239188b 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -821,10 +821,10 @@ async def start_cluster(
         comm.comm is None for comm in s.stream_comms.values()
     ):
         await asyncio.sleep(0.01)
-        if time() - start > 5:
+        if time() > start + 30:
             await asyncio.gather(*[w.close(timeout=1) for w in workers])
             await s.close(fast=True)
-            raise Exception("Cluster creation timeout")
+            raise TimeoutError("Cluster creation timeout")
     return s, workers
 
 
@@ -885,6 +885,9 @@ async def test_foo(scheduler, worker1, worker2, pytest_fixture_a, pytest_fixture
         warnings.warn("ncores= has moved to nthreads=", stacklevel=2)
         nthreads = ncores
 
+    scheduler_kwargs = merge(
+        {"dashboard": False, "dashboard_address": ":0"}, scheduler_kwargs
+    )
     worker_kwargs = merge(
         {"memory_limit": system.MEMORY_LIMIT, "death_timeout": 15}, worker_kwargs
     )
@@ -1030,7 +1033,7 @@ def terminate_process(proc):
         else:
             proc.send_signal(signal.SIGINT)
         try:
-            proc.wait(10)
+            proc.wait(30)
         finally:
             # Make sure we don't leave the process lingering around
             with suppress(OSError):

From 465f14a7382aea6101ea2bfe350585cd0bdb235f Mon Sep 17 00:00:00 2001
From: Peter Andreas Entschev <peter@entschev.com>
Date: Wed, 28 Jul 2021 18:55:26 +0200
Subject: [PATCH 1398/1550] Handle `NVMLError_Unknown` in NVML diagnostics
 (#5121)

---
 distributed/diagnostics/nvml.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/distributed/diagnostics/nvml.py b/distributed/diagnostics/nvml.py
index 3c66ed22760..3aba966d992 100644
--- a/distributed/diagnostics/nvml.py
+++ b/distributed/diagnostics/nvml.py
@@ -26,7 +26,11 @@ def init_once():
     nvmlOwnerPID = os.getpid()
     try:
         pynvml.nvmlInit()
-    except (pynvml.NVMLError_LibraryNotFound, pynvml.NVMLError_DriverNotLoaded):
+    except (
+        pynvml.NVMLError_LibraryNotFound,
+        pynvml.NVMLError_DriverNotLoaded,
+        pynvml.NVMLError_Unknown,
+    ):
         nvmlLibraryNotFound = True
 
 

From a5ccc8fdb5779889441a646bdf45ae3fac22f0ff Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Wed, 28 Jul 2021 14:27:57 -0500
Subject: [PATCH 1399/1550] Update individual profile plot sizing (#5131)

---
 distributed/dashboard/components/scheduler.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index 83e54cf94bc..eebe1a58dea 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -2947,7 +2947,7 @@ def individual_doc(cls, interval, scheduler, extra, doc, fig_attr="root", **kwar
 
 def individual_profile_doc(scheduler, extra, doc):
     with log_errors():
-        prof = ProfileTimePlot(scheduler, sizing_mode="scale_width", doc=doc)
+        prof = ProfileTimePlot(scheduler, sizing_mode="stretch_both", doc=doc)
         doc.add_root(prof.root)
         prof.trigger_update()
         doc.theme = BOKEH_THEME
@@ -2955,7 +2955,7 @@ def individual_profile_doc(scheduler, extra, doc):
 
 def individual_profile_server_doc(scheduler, extra, doc):
     with log_errors():
-        prof = ProfileServer(scheduler, sizing_mode="scale_width", doc=doc)
+        prof = ProfileServer(scheduler, sizing_mode="stretch_both", doc=doc)
         doc.add_root(prof.root)
         prof.trigger_update()
         doc.theme = BOKEH_THEME

From 1be9265ac11876df766bb8bd6d6eb519d04d3bac Mon Sep 17 00:00:00 2001
From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com>
Date: Wed, 28 Jul 2021 17:22:12 -0700
Subject: [PATCH 1400/1550] Mark `ucx-py` tests for GPU (#5133)

---
 distributed/comm/tests/test_ucx.py        | 2 ++
 distributed/comm/tests/test_ucx_config.py | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/distributed/comm/tests/test_ucx.py b/distributed/comm/tests/test_ucx.py
index d463104dc79..502afdfc4f0 100644
--- a/distributed/comm/tests/test_ucx.py
+++ b/distributed/comm/tests/test_ucx.py
@@ -2,6 +2,8 @@
 
 import pytest
 
+pytestmark = pytest.mark.gpu
+
 ucp = pytest.importorskip("ucp")
 
 from distributed import Client, Scheduler, wait
diff --git a/distributed/comm/tests/test_ucx_config.py b/distributed/comm/tests/test_ucx_config.py
index 09e71acbb06..f33b1398063 100644
--- a/distributed/comm/tests/test_ucx_config.py
+++ b/distributed/comm/tests/test_ucx_config.py
@@ -2,6 +2,8 @@
 
 import pytest
 
+pytestmark = pytest.mark.gpu
+
 import dask
 from dask.utils import format_bytes
 

From 83e209b933430de63539a939a6499e79370d7b91 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Thu, 29 Jul 2021 10:15:37 -0500
Subject: [PATCH 1401/1550] Document automatic `MALLOC_TRIM_THRESHOLD_`
 environment variable (#5139)

---
 docs/source/worker.rst | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/docs/source/worker.rst b/docs/source/worker.rst
index faa392d132a..3b409bcfdcd 100644
--- a/docs/source/worker.rst
+++ b/docs/source/worker.rst
@@ -358,6 +358,12 @@ performance.
 .. note::
    The variable must be set before starting the ``dask-worker`` process.
 
+.. note::
+   If using a :ref:`nanny`, the ``MALLOC_TRIM_THRESHOLD_`` environment variable
+   will automatically be set to ``65536`` for the worker process which the nanny is
+   monitoring. You can modify this behavior using the ``distributed.nanny.environ``
+   configuration value.
+
 jemalloc
 ~~~~~~~~
 *Linux and MacOS workers*
@@ -402,6 +408,8 @@ This of course will be problematic if you have a genuine issue with unmanaged me
 e.g. memory leaks and/or suffer from heavy fragmentation.
 
 
+.. _nanny:
+
 Nanny
 -----
 

From 89f8789f916f60331a5fae1a8697eca05bbadf9b Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Thu, 29 Jul 2021 16:59:21 -0500
Subject: [PATCH 1402/1550] Cap maximum shard size at the size of an integer
 (#5141)

Supercedes https://github.com/dask/distributed/pull/5134

Copying over the summary of that PR

Works around the OpenSSL 1.0.2 bug demonstrated in issue ( #4538 ), except unlike PR ( #5115 ) which did this for reading, this does the same thing for writing. The error may be less likely to show up in the write path (as frames may simply be smaller than this limit). Still it seems like a good idea to protect against OverflowErrors from OpenSSL
---
 distributed/comm/tcp.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py
index 0088ed6500e..22d7c72461a 100644
--- a/distributed/comm/tcp.py
+++ b/distributed/comm/tcp.py
@@ -145,6 +145,8 @@ class TCP(Comm):
     An established communication based on an underlying Tornado IOStream.
     """
 
+    max_shard_size = dask.utils.parse_bytes(dask.config.get("distributed.comm.shard"))
+
     def __init__(self, stream, local_addr, peer_addr, deserialize=True):
         self._closed = False
         Comm.__init__(self)
@@ -248,6 +250,7 @@ async def write(self, msg, serializers=None, on_error="message"):
                 "recipient": self.remote_info,
                 **self.handshake_options,
             },
+            frame_split_size=self.max_shard_size,
         )
         frames_nbytes = [nbytes(f) for f in frames]
         frames_nbytes_total = sum(frames_nbytes)
@@ -335,6 +338,9 @@ class TLS(TCP):
     A TLS-specific version of TCP.
     """
 
+    # Workaround for OpenSSL 1.0.2 (can drop with OpenSSL 1.1.1)
+    max_shard_size = min(C_INT_MAX, TCP.max_shard_size)
+
     def _read_extra(self):
         TCP._read_extra(self)
         sock = self.stream.socket

From 595abbf0a5d52bafa1847b408f0dd930710d1f02 Mon Sep 17 00:00:00 2001
From: Bryan Van de Ven <bryan@bokeh.org>
Date: Thu, 29 Jul 2021 17:11:17 -0700
Subject: [PATCH 1403/1550] Minor dashboard style updates (#5143)

- line up plot borders better, especially vertical axes in stacked columns
- place toolbars above plots where there is already space to accommodate titles
- give hbars a slight separation (I find this much easier to interpret)
- angle tick labels a little less aggressively
---
 distributed/dashboard/components/scheduler.py | 30 +++++++++++--------
 distributed/dashboard/components/shared.py    |  4 +++
 distributed/dashboard/theme.yaml              |  2 ++
 distributed/http/static/css/status.css        |  4 +--
 4 files changed, 26 insertions(+), 14 deletions(-)

diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index eebe1a58dea..f7e8d53edea 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -86,7 +86,7 @@
 
 BOKEH_THEME = Theme(os.path.join(os.path.dirname(__file__), "..", "theme.yaml"))
 TICKS_1024 = {"base": 1024, "mantissas": [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]}
-XLABEL_ORIENTATION = -math.pi / 12  # slanted downwards 15 degrees
+XLABEL_ORIENTATION = -math.pi / 9  # slanted downwards 20 degrees
 
 
 logos_dict = {
@@ -117,12 +117,14 @@ def __init__(self, scheduler, **kwargs):
             self.root = figure(
                 title="Occupancy",
                 tools="",
+                toolbar_location="above",
                 id="bk-occupancy-plot",
                 x_axis_type="datetime",
+                min_border_bottom=50,
                 **kwargs,
             )
             rect = self.root.rect(
-                source=self.source, x="x", width="ms", y="y", height=1, color="color"
+                source=self.source, x="x", width="ms", y="y", height=0.9, color="color"
             )
             rect.nonselection_glyph = None
 
@@ -258,6 +260,7 @@ def __init__(self, scheduler, width=600, **kwargs):
                 id="bk-cluster-memory-plot",
                 width=int(width / 2),
                 name="cluster_memory",
+                min_border_bottom=50,
                 **kwargs,
             )
             rect = self.root.rect(
@@ -265,7 +268,7 @@ def __init__(self, scheduler, width=600, **kwargs):
                 x="x",
                 y="y",
                 width="width",
-                height=1,
+                height=0.9,
                 color="color",
                 alpha="alpha",
             )
@@ -374,6 +377,7 @@ def __init__(self, scheduler, width=600, **kwargs):
                 id="bk-workers-memory-plot",
                 width=int(width / 2),
                 name="workers_memory",
+                min_border_bottom=50,
                 **kwargs,
             )
             rect = self.root.rect(
@@ -381,7 +385,7 @@ def __init__(self, scheduler, width=600, **kwargs):
                 x="x",
                 y="y",
                 width="width",
-                height=1,
+                height=0.9,
                 color="color",
                 fill_alpha="alpha",
                 line_width=0,
@@ -584,7 +588,7 @@ def __init__(self, scheduler, **kwargs):
                 x="bandwidth-half",
                 y="type",
                 width="bandwidth",
-                height=1,
+                height=0.9,
                 color="blue",
             )
             self.root.x_range.start = 0
@@ -856,8 +860,6 @@ def __init__(self, scheduler, **kwargs):
             )
 
             fig.y_range.start = 0
-            fig.min_border_right = 20
-            fig.min_border_bottom = 60
             fig.yaxis.axis_label = "Time (s)"
             fig.yaxis[0].formatter = NumeralTickFormatter(format="0")
             fig.yaxis.ticker = AdaptiveTicker(**TICKS_1024)
@@ -1022,8 +1024,6 @@ def __init__(self, scheduler, **kwargs):
             )
 
             self.root.y_range.start = 0
-            self.root.min_border_right = 20
-            self.root.min_border_bottom = 60
             self.root.yaxis[0].formatter = NumeralTickFormatter(format="0")
             self.root.yaxis.axis_label = "Time (s)"
             self.root.yaxis.ticker = AdaptiveTicker(**TICKS_1024)
@@ -1181,6 +1181,7 @@ def __init__(self, scheduler, width=600, **kwargs):
                 id="bk-nprocessing-plot",
                 name="processing",
                 width=int(width / 2),
+                min_border_bottom=50,
                 **kwargs,
             )
             rect = processing.rect(
@@ -1188,7 +1189,7 @@ def __init__(self, scheduler, width=600, **kwargs):
                 x="nprocessing-half",
                 y="y",
                 width="nprocessing",
-                height=1,
+                height=0.9,
                 color="nprocessing-color",
             )
             processing.x_range.start = 0
@@ -1201,6 +1202,7 @@ def __init__(self, scheduler, width=600, **kwargs):
                 width=int(width / 2),
                 name="cpu_hist",
                 x_range=(0, 100),
+                min_border_bottom=50,
                 **kwargs,
             )
             rect = cpu.rect(
@@ -1208,7 +1210,7 @@ def __init__(self, scheduler, width=600, **kwargs):
                 x="cpu-half",
                 y="y",
                 width="cpu",
-                height=1,
+                height=0.9,
                 color="blue",
             )
             rect.nonselection_glyph = None
@@ -1627,8 +1629,9 @@ def task_stream_figure(clear_interval="20s", **kwargs):
         y_range=y_range,
         toolbar_location="above",
         x_axis_type="datetime",
-        min_border_right=35,
+        y_axis_location=None,
         tools="",
+        min_border_bottom=50,
         **kwargs,
     )
 
@@ -2333,6 +2336,7 @@ def __init__(self, scheduler, **kwargs):
             y_range=y_range,
             toolbar_location=None,
             tools="",
+            min_border_bottom=50,
             **kwargs,
         )
         self.root.line(  # just to define early ranges
@@ -2629,6 +2633,7 @@ def __init__(self, scheduler, width=800, **kwargs):
             height=60,
             width=width,
             tools="",
+            min_border_right=0,
             **kwargs,
         )
         mem_plot.circle(
@@ -2658,6 +2663,7 @@ def __init__(self, scheduler, width=800, **kwargs):
             height=60,
             width=width,
             tools="",
+            min_border_right=0,
             **kwargs,
         )
         cpu_plot.circle(
diff --git a/distributed/dashboard/components/shared.py b/distributed/dashboard/components/shared.py
index ef857be6ba0..0b585a2e4ee 100644
--- a/distributed/dashboard/components/shared.py
+++ b/distributed/dashboard/components/shared.py
@@ -435,6 +435,7 @@ def __init__(self, worker, height=150, last_count=None, **kwargs):
             x_axis_type="datetime",
             height=height,
             tools=tools,
+            toolbar_location="above",
             x_range=x_range,
             **kwargs,
         )
@@ -457,6 +458,7 @@ def __init__(self, worker, height=150, last_count=None, **kwargs):
             x_axis_type="datetime",
             height=height,
             tools=tools,
+            toolbar_location="above",
             x_range=x_range,
             **kwargs,
         )
@@ -480,6 +482,7 @@ def __init__(self, worker, height=150, last_count=None, **kwargs):
             height=height,
             x_range=x_range,
             tools=tools,
+            toolbar_location="above",
             **kwargs,
         )
         self.bandwidth.line(
@@ -511,6 +514,7 @@ def __init__(self, worker, height=150, last_count=None, **kwargs):
                 height=height,
                 x_range=x_range,
                 tools=tools,
+                toolbar_location="above",
                 **kwargs,
             )
 
diff --git a/distributed/dashboard/theme.yaml b/distributed/dashboard/theme.yaml
index feb7a266f37..787416d2789 100644
--- a/distributed/dashboard/theme.yaml
+++ b/distributed/dashboard/theme.yaml
@@ -3,5 +3,7 @@ attrs:
     Plot:
         background_fill_color: null
         border_fill_color: null
+        min_border_left: 10
+        min_border_right: 10
     Toolbar:
       logo: null
diff --git a/distributed/http/static/css/status.css b/distributed/http/static/css/status.css
index 2fd9803702d..f03541e2e53 100644
--- a/distributed/http/static/css/status.css
+++ b/distributed/http/static/css/status.css
@@ -6,7 +6,7 @@
 @media (min-width: 0px) {
   #status-fluid {
     grid-template-columns: 1fr 1fr;
-    grid-template-rows: 70px 2fr 6fr 2fr;
+    grid-template-rows: 90px 2fr 6fr 2fr;
   }
   #status-cluster-memory {
     grid-column: 1 / span 2;
@@ -33,7 +33,7 @@
 @media (min-width: 992px) {
   #status-fluid {
     grid-template-columns: 1fr 3fr;
-    grid-template-rows: 80px 4fr 1fr 1fr 4fr;
+    grid-template-rows: 100px 4fr 1fr 1fr 4fr;
   }
   #status-cluster-memory {
     grid-column: 1;

From 3f1b25097feb683c228e63b7165ac1a568e01330 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Fri, 30 Jul 2021 10:10:24 -0500
Subject: [PATCH 1404/1550] Include maximum shard size in second `to_frames`
 method (#5145)

Websockets are weird in that they have two different Comm objects.
Previously we added the maximum shard size setting to one,
but not the other
---
 distributed/comm/ws.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/distributed/comm/ws.py b/distributed/comm/ws.py
index 2d254666555..46ce55b86e5 100644
--- a/distributed/comm/ws.py
+++ b/distributed/comm/ws.py
@@ -216,6 +216,7 @@ async def write(self, msg, serializers=None, on_error=None):
                 "recipient": self.remote_info,
                 **self.handshake_options,
             },
+            frame_split_size=BIG_BYTES_SHARD_SIZE,
         )
         n = struct.pack("Q", len(frames))
         try:

From 1999c158521db8a1ec356f4389c8e9c1666b6a94 Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Fri, 30 Jul 2021 18:03:55 +0200
Subject: [PATCH 1405/1550] Fix a deadlock connected to task stealing and task
 deserialization (#5128)

* Fix a deadlock connected to task stealing and deserialization

If a task is stolen while the task runspec is being deserialized this allows
for an edge case where the executing_count is never decreased again
such that the ready queue is never worked off

* Simplify exception handling for Worker.execute

* remove test about unintended behaviour

* function naming fix
---
 distributed/tests/test_worker.py | 103 ++++++++++++++++++++-
 distributed/worker.py            | 151 ++++++++++++-------------------
 2 files changed, 159 insertions(+), 95 deletions(-)

diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index c558c6ae09d..3749a8787c3 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -6,6 +6,7 @@
 import threading
 import traceback
 from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
+from concurrent.futures.process import BrokenProcessPool
 from numbers import Number
 from operator import add
 from time import sleep
@@ -2005,17 +2006,53 @@ async def test_process_executor(c, s, a, b):
         assert (await future) != os.getpid()
 
 
+def kill_process():
+    import os
+    import signal
+
+    os.kill(os.getpid(), signal.SIGTERM)
+
+
 @gen_cluster(client=True)
 async def test_process_executor_kills_process(c, s, a, b):
     with ProcessPoolExecutor() as e:
         a.executors["processes"] = e
         b.executors["processes"] = e
+        with dask.annotate(executor="processes", retries=1):
+            future = c.submit(kill_process)
+
+        with pytest.raises(
+            BrokenProcessPool,
+            match="A child process terminated abruptly, the process pool is not usable anymore",
+        ):
+            await future
+
+        with dask.annotate(executor="processes", retries=1):
+            future = c.submit(inc, 1)
+
+        # FIXME: The processpool is now unusable and the worker is effectively
+        # dead
+        with pytest.raises(
+            BrokenProcessPool,
+            match="A child process terminated abruptly, the process pool is not usable anymore",
+        ):
+            assert await future == 2
 
+
+def raise_exc():
+    raise RuntimeError("foo")
+
+
+@gen_cluster(client=True)
+async def test_process_executor_raise_exception(c, s, a, b):
+    with ProcessPoolExecutor() as e:
+        a.executors["processes"] = e
+        b.executors["processes"] = e
         with dask.annotate(executor="processes", retries=1):
-            future = c.submit(sys.exit, 1)
+            future = c.submit(raise_exc)
 
-        exc = await future.exception()
-        assert "SystemExit(1)" in repr(exc)
+        with pytest.raises(RuntimeError, match="foo"):
+            await future
 
 
 def assert_task_states_on_worker(expected, worker):
@@ -2417,3 +2454,63 @@ async def test_forget_dependents_after_release(c, s, a):
     while fut2.key in a.tasks:
         await asyncio.sleep(0.001)
     assert fut2.key not in {d.key for d in a.tasks[fut.key].dependents}
+
+
+@gen_cluster(client=True, nthreads=[("", 1)] * 2, timeout=5000000)
+async def test_steak_during_task_deserialization(c, s, a, b, monkeypatch):
+    stealing_ext = s.extensions["stealing"]
+    stealing_ext._pc.stop()
+    from distributed.utils import ThreadPoolExecutor
+
+    class CountingThreadPool(ThreadPoolExecutor):
+        counter = 0
+
+        def submit(self, *args, **kwargs):
+            CountingThreadPool.counter += 1
+            return super().submit(*args, **kwargs)
+
+    # Ensure we're always offloading
+    monkeypatch.setattr("distributed.worker.OFFLOAD_THRESHOLD", 1)
+    threadpool = CountingThreadPool(
+        max_workers=1, thread_name_prefix="Counting-Offload-Threadpool"
+    )
+    try:
+        monkeypatch.setattr("distributed.utils._offload_executor", threadpool)
+
+        class SlowDeserializeCallable:
+            def __init__(self, delay=0.1):
+                self.delay = delay
+
+            def __getstate__(self):
+                return self.delay
+
+            def __setstate__(self, state):
+                delay = state
+                import time
+
+                time.sleep(delay)
+                return SlowDeserializeCallable(delay)
+
+            def __call__(self, *args, **kwargs):
+                return 41
+
+        slow_deserialized_func = SlowDeserializeCallable()
+        fut = c.submit(
+            slow_deserialized_func, 1, workers=[a.address], allow_other_workers=True
+        )
+
+        while CountingThreadPool.counter == 0:
+            await asyncio.sleep(0)
+
+        ts = s.tasks[fut.key]
+        a.steal_request(fut.key)
+        stealing_ext.scheduler.send_task_to_worker(b.address, ts)
+
+        fut2 = c.submit(inc, fut, workers=[a.address])
+        fut3 = c.submit(inc, fut2, workers=[a.address])
+
+        assert await fut2 == 42
+        await fut3
+
+    finally:
+        threadpool.shutdown()
diff --git a/distributed/worker.py b/distributed/worker.py
index c182a84249d..21188d21412 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -2817,12 +2817,8 @@ async def _maybe_deserialize_task(self, ts):
                     {"action": "deserialize", "start": start, "stop": stop}
                 )
             return function, args, kwargs
-        except Exception as e:
-            logger.warning("Could not deserialize task", exc_info=True)
-            emsg = error_message(e)
-            emsg["key"] = ts.key
-            emsg["op"] = "task-erred"
-            self.batched_stream.send(emsg)
+        except Exception:
+            logger.error("Could not deserialize task", exc_info=True)
             self.log.append((ts.key, "deserialize-error"))
             raise
 
@@ -2838,11 +2834,6 @@ async def ensure_computing(self):
                     continue
                 if self.meets_resource_constraints(key):
                     self.constrained.popleft()
-                    try:
-                        # Ensure task is deserialized prior to execution
-                        ts.runspec = await self._maybe_deserialize_task(ts)
-                    except Exception:
-                        continue
                     self.transition(ts, "executing")
                 else:
                     break
@@ -2857,11 +2848,6 @@ async def ensure_computing(self):
                 elif ts.key in self.data:
                     self.transition(ts, "memory")
                 elif ts.state in READY:
-                    try:
-                        # Ensure task is deserialized prior to execution
-                        ts.runspec = await self._maybe_deserialize_task(ts)
-                    except Exception:
-                        continue
                     self.transition(ts, "executing")
         except Exception as e:
             logger.exception(e)
@@ -2871,61 +2857,38 @@ async def ensure_computing(self):
                 pdb.set_trace()
             raise
 
-    async def execute(self, key, report=False):
-        executor_error = None
+    async def execute(self, key):
         if self.status in (Status.closing, Status.closed, Status.closing_gracefully):
             return
+
+        if key not in self.tasks:
+            return
+
+        ts = self.tasks[key]
+
+        if ts.state != "executing":
+            # This might happen if keys are canceled
+            logger.debug(
+                "Trying to execute a task %s which is not in executing state anymore"
+                % ts
+            )
+            return
+
         try:
-            if key not in self.tasks:
-                return
-            ts = self.tasks[key]
-            if ts.state != "executing":
-                # This might happen if keys are canceled
-                logger.debug(
-                    "Trying to execute a task %s which is not in executing state anymore"
-                    % ts
-                )
-                return
-            if ts.runspec is None:
-                logger.critical("No runspec available for task %s." % ts)
             if self.validate:
                 assert not ts.waiting_for_data
                 assert ts.state == "executing"
+                assert ts.runspec is not None
 
-            function, args, kwargs = ts.runspec
-
-            start = time()
-            data = {}
-            for dep in ts.dependencies:
-                k = dep.key
-                try:
-                    data[k] = self.data[k]
-                except KeyError:
-                    from .actor import Actor  # TODO: create local actor
+            function, args, kwargs = await self._maybe_deserialize_task(ts)
 
-                    data[k] = Actor(type(self.actors[k]), self.address, k, self)
-            args2 = pack_data(args, data, key_types=(bytes, str))
-            kwargs2 = pack_data(kwargs, data, key_types=(bytes, str))
-            stop = time()
-            if stop - start > 0.005:
-                ts.startstops.append(
-                    {"action": "disk-read", "start": start, "stop": stop}
-                )
-                if self.digests is not None:
-                    self.digests["disk-load-duration"].add(stop - start)
+            args2, kwargs2 = self._prepare_args_for_execution(ts, args, kwargs)
 
             if ts.annotations is not None and "executor" in ts.annotations:
                 executor = ts.annotations["executor"]
             else:
                 executor = "default"
             assert executor in self.executors
-
-            logger.debug(
-                "Execute key: %s worker: %s, executor: %s",
-                ts.key,
-                self.address,
-                executor,
-            )  # TODO: comment out?
             assert key == ts.key
             self.active_keys.add(ts.key)
             try:
@@ -2945,28 +2908,14 @@ async def execute(self, key, report=False):
                         self.scheduler_delay,
                     )
                 else:
-                    try:
-                        start = time() + self.scheduler_delay
-                        result = await self.loop.run_in_executor(
-                            e,
-                            apply_function_simple,
-                            function,
-                            args2,
-                            kwargs2,
-                            self.scheduler_delay,
-                        )
-                    except BaseException as e:
-                        msg = error_message(e)
-                        msg["op"] = "task-erred"
-                        msg["actual-exception"] = e
-                        msg["start"] = start
-                        msg["stop"] = time() + self.scheduler_delay
-                        msg["thread"] = None
-                        result = msg
-
-            except RuntimeError as e:
-                executor_error = e
-                raise
+                    result = await self.loop.run_in_executor(
+                        e,
+                        apply_function_simple,
+                        function,
+                        args2,
+                        kwargs2,
+                        self.scheduler_delay,
+                    )
             finally:
                 self.active_keys.discard(ts.key)
 
@@ -2974,12 +2923,11 @@ async def execute(self, key, report=False):
             # changed since the execution was kicked off. In particular, it may
             # have been canceled and released already in which case we'll have
             # to drop the result immediately
-            key = ts.key
-            ts = self.tasks.get(key)
 
-            if ts is None:
+            if ts.key not in self.tasks:
                 logger.debug(
-                    "Dropping result for %s since task has already been released." % key
+                    "Dropping result for %s since task has already been released."
+                    % ts.key
                 )
                 return
 
@@ -3022,18 +2970,37 @@ async def execute(self, key, report=False):
                 assert ts.state != "executing"
                 assert not ts.waiting_for_data
 
+        except Exception as exc:
+            logger.error(
+                "Exception during execution of task %s.", ts.key, exc_info=True
+            )
+            emsg = error_message(exc)
+            ts.exception = emsg["exception"]
+            ts.traceback = emsg["traceback"]
+            self.transition(ts, "error")
+        finally:
             await self.ensure_computing()
             self.ensure_communicating()
-        except Exception as e:
-            if executor_error is e:
-                logger.error("Thread Pool Executor error: %s", e)
-            else:
-                logger.exception(e)
-                if LOG_PDB:
-                    import pdb
 
-                    pdb.set_trace()
-                raise
+    def _prepare_args_for_execution(self, ts, args, kwargs):
+        start = time()
+        data = {}
+        for dep in ts.dependencies:
+            k = dep.key
+            try:
+                data[k] = self.data[k]
+            except KeyError:
+                from .actor import Actor  # TODO: create local actor
+
+                data[k] = Actor(type(self.actors[k]), self.address, k, self)
+        args2 = pack_data(args, data, key_types=(bytes, str))
+        kwargs2 = pack_data(kwargs, data, key_types=(bytes, str))
+        stop = time()
+        if stop - start > 0.005:
+            ts.startstops.append({"action": "disk-read", "start": start, "stop": stop})
+            if self.digests is not None:
+                self.digests["disk-load-duration"].add(stop - start)
+        return args2, kwargs2
 
     ##################
     # Administrative #

From 7392de636c3dc6e52f08d9d3fbd171ff3177b4d1 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Fri, 30 Jul 2021 14:45:34 -0500
Subject: [PATCH 1406/1550] bump version to 2021.07.2

---
 docs/source/changelog.rst | 27 ++++++++++++++++++++++++++-
 requirements.txt          |  2 +-
 2 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index e4e80765af8..04ae7f2add4 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,31 @@
 Changelog
 =========
 
+2021.07.2
+---------
+
+Released on July 30, 2021
+
+- Fix a deadlock connected to task stealing and task deserialization (:pr:`5128`) `Florian Jetter`_
+- Include maximum shard size in second ``to_frames`` method (:pr:`5145`) `Matthew Rocklin`_
+- Minor dashboard style updates (:pr:`5143`) `Bryan Van de Ven`_
+- Cap maximum shard size at the size of an integer (:pr:`5141`) `Matthew Rocklin`_
+- Document automatic ``MALLOC_TRIM_THRESHOLD_`` environment variable (:pr:`5139`) `James Bourbeau`_
+- Mark ``ucx-py`` tests for GPU (:pr:`5133`) `Charles Blackmon-Luca`_
+- Update individual profile plot sizing (:pr:`5131`) `James Bourbeau`_
+- Handle ``NVMLError_Unknown`` in NVML diagnostics (:pr:`5121`) `Peter Andreas Entschev`_
+- Unit tests to use a random port for the dashboard (:pr:`5060`) `crusaderky`_
+- Ensure worker reconnect registers existing tasks properly (:pr:`5103`) `Florian Jetter`_
+- Halve CI runtime! (:pr:`5074`) `crusaderky`_
+- Add ``NannyPlugins`` (:pr:`5118`) `Matthew Rocklin`_
+- Add ``WorkerNetworkBandwidth`` chart to dashboard (:pr:`5104`) `Naty Clementi`_
+- Set nanny environment variables in config (:pr:`5098`) `Matthew Rocklin`_
+- Read smaller frames to workaround OpenSSL bug (:pr:`5115`) `jakirkham`_
+- Move UCX/RMM config variables to Distributed namespace  (:pr:`4916`) `Charles Blackmon-Luca`_
+- Allow ws(s) messages greater than 10Mb (:pr:`5110`) `Marcos Moyano`_
+- Short-circuit root-ish check for many deps (:pr:`5113`) `Gabe Joseph`_
+
+
 2021.07.1
 ---------
 
@@ -36,7 +61,7 @@ Released on July 23, 2021
 
 Released on July 9, 2021
 
-- Fix Nbytes jitter - less expensive (#5043) `Naty Clementi`_
+- Fix Nbytes jitter - less expensive (:pr:`5043`) `Naty Clementi`_
 - Use native GH actions cancel feature (:pr:`5037`) `Florian Jetter`_
 - Don't require workers to report to scheduler if scheduler shuts down (:pr:`5032`) `Florian Jetter`_
 - Add pandas to the list of checked packages for ``client.get_versions()`` (:pr:`5029`) `Ian Rose`_
diff --git a/requirements.txt b/requirements.txt
index 4905ad87815..4eafe78d78f 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,6 @@
 click >= 6.6
 cloudpickle >= 1.5.0
-dask == 2021.07.1
+dask == 2021.07.2
 msgpack >= 0.6.0
 psutil >= 5.0
 sortedcontainers !=2.0.0, !=2.0.1

From eaf05ac37cae06e0ced057d18c44fe71d9f48488 Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Mon, 2 Aug 2021 16:10:39 +0200
Subject: [PATCH 1407/1550] Demote Worker.ensure_computing to function (#5153)

Follow up to https://github.com/dask/distributed/pull/5128
---
 distributed/worker.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/distributed/worker.py b/distributed/worker.py
index 21188d21412..1e4987cccb8 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -2038,7 +2038,7 @@ def transition_executing_long_running(self, ts, compute_duration=None):
                 }
             )
 
-            self.io_loop.add_callback(self.ensure_computing)
+            self.ensure_computing()
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
@@ -2464,7 +2464,7 @@ async def gather_dep(
                 if self.validate:
                     self.validate_state()
 
-                await self.ensure_computing()
+                self.ensure_computing()
 
                 if not busy:
                     self.repetitively_busy = 0
@@ -2822,7 +2822,7 @@ async def _maybe_deserialize_task(self, ts):
             self.log.append((ts.key, "deserialize-error"))
             raise
 
-    async def ensure_computing(self):
+    def ensure_computing(self):
         if self.paused:
             return
         try:
@@ -2979,7 +2979,7 @@ async def execute(self, key):
             ts.traceback = emsg["traceback"]
             self.transition(ts, "error")
         finally:
-            await self.ensure_computing()
+            self.ensure_computing()
             self.ensure_communicating()
 
     def _prepare_args_for_execution(self, ts, args, kwargs):
@@ -3050,7 +3050,7 @@ async def check_pause(memory):
                     else "None",
                 )
                 self.paused = False
-                await self.ensure_computing()
+                self.ensure_computing()
 
         await check_pause(memory)
         # Dump data to disk if above 70%

From 72a55ee7498e6fba88a9cecea28230d35047615c Mon Sep 17 00:00:00 2001
From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com>
Date: Mon, 2 Aug 2021 10:52:18 -0400
Subject: [PATCH 1408/1550] Add gpuCI build script (#5147)

---
 continuous_integration/gpuci/axis.yaml | 13 +++++++
 continuous_integration/gpuci/build.sh  | 52 ++++++++++++++++++++++++++
 2 files changed, 65 insertions(+)
 create mode 100644 continuous_integration/gpuci/axis.yaml
 create mode 100644 continuous_integration/gpuci/build.sh

diff --git a/continuous_integration/gpuci/axis.yaml b/continuous_integration/gpuci/axis.yaml
new file mode 100644
index 00000000000..76f19ff492c
--- /dev/null
+++ b/continuous_integration/gpuci/axis.yaml
@@ -0,0 +1,13 @@
+PYTHON_VER:
+- 3.8
+
+CUDA_VER:
+- 11.2
+
+LINUX_VER:
+- ubuntu18.04
+
+RAPIDS_VER:
+- 21.08
+
+excludes:
\ No newline at end of file
diff --git a/continuous_integration/gpuci/build.sh b/continuous_integration/gpuci/build.sh
new file mode 100644
index 00000000000..b9524b7c596
--- /dev/null
+++ b/continuous_integration/gpuci/build.sh
@@ -0,0 +1,52 @@
+##############################################
+# Dask GPU build and test script for CI      #
+##############################################
+set -e
+NUMARGS=$#
+ARGS=$*
+
+# Arg parsing function
+function hasArg {
+    (( ${NUMARGS} != 0 )) && (echo " ${ARGS} " | grep -q " $1 ")
+}
+
+# Set path and build parallel level
+export PATH=/opt/conda/bin:/usr/local/cuda/bin:$PATH
+export PARALLEL_LEVEL=${PARALLEL_LEVEL:-4}
+
+# Set home to the job's workspace
+export HOME="$WORKSPACE"
+
+# Switch to project root; also root of repo checkout
+cd "$WORKSPACE"
+
+# Determine CUDA release version
+export CUDA_REL=${CUDA_VERSION%.*}
+
+################################################################################
+# SETUP - Check environment
+################################################################################
+
+gpuci_logger "Check environment variables"
+env
+
+gpuci_logger "Check GPU usage"
+nvidia-smi
+
+gpuci_logger "Activate conda env"
+. /opt/conda/etc/profile.d/conda.sh
+conda activate dask
+
+gpuci_logger "Install distributed"
+python setup.py install
+
+gpuci_logger "Check compiler versions"
+python --version
+
+gpuci_logger "Check conda environment"
+conda info
+conda config --show-sources
+conda list --show-channel-urls
+
+gpuci_logger "Python py.test for dask"
+py.test $WORKSPACE -n 4 -v -m gpu --junitxml="$WORKSPACE/junit-distributed.xml" --cov-config="$WORKSPACE/.coveragerc" --cov=distributed --cov-report=xml:"$WORKSPACE/distributed-coverage.xml" --cov-report term

From 7bfd82cfebcb9b13579c8782bb5567d3c08c6bae Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Mon, 2 Aug 2021 21:38:26 +0200
Subject: [PATCH 1409/1550] Remove excessive timeout of
 `test_steal_during_task_deserialization` (#5156)

---
 distributed/tests/test_worker.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 3749a8787c3..a9274623ebc 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -2456,8 +2456,8 @@ async def test_forget_dependents_after_release(c, s, a):
     assert fut2.key not in {d.key for d in a.tasks[fut.key].dependents}
 
 
-@gen_cluster(client=True, nthreads=[("", 1)] * 2, timeout=5000000)
-async def test_steak_during_task_deserialization(c, s, a, b, monkeypatch):
+@gen_cluster(client=True)
+async def test_steal_during_task_deserialization(c, s, a, b, monkeypatch):
     stealing_ext = s.extensions["stealing"]
     stealing_ext._pc.stop()
     from distributed.utils import ThreadPoolExecutor

From eea0248231bde053f06ef0b3ac08ed186dd211d0 Mon Sep 17 00:00:00 2001
From: Peter Andreas Entschev <peter@entschev.com>
Date: Mon, 2 Aug 2021 22:53:48 +0200
Subject: [PATCH 1410/1550] Fix RMM and UCX tests (#5158)

The `rmm.get_info()` function was remove a while back, and it's not
possible anymore to query for a pool size. We can still verify the
memory resource type to verify that an RMM pool is indeed used.
---
 continuous_integration/gpuci/build.sh     |  2 +-
 distributed/comm/tests/test_ucx_config.py | 46 ++++++++++++++---------
 2 files changed, 29 insertions(+), 19 deletions(-)

diff --git a/continuous_integration/gpuci/build.sh b/continuous_integration/gpuci/build.sh
index b9524b7c596..8e98736c8d2 100644
--- a/continuous_integration/gpuci/build.sh
+++ b/continuous_integration/gpuci/build.sh
@@ -49,4 +49,4 @@ conda config --show-sources
 conda list --show-channel-urls
 
 gpuci_logger "Python py.test for dask"
-py.test $WORKSPACE -n 4 -v -m gpu --junitxml="$WORKSPACE/junit-distributed.xml" --cov-config="$WORKSPACE/.coveragerc" --cov=distributed --cov-report=xml:"$WORKSPACE/distributed-coverage.xml" --cov-report term
+py.test $WORKSPACE -v -m gpu --runslow --junitxml="$WORKSPACE/junit-distributed.xml"
diff --git a/distributed/comm/tests/test_ucx_config.py b/distributed/comm/tests/test_ucx_config.py
index f33b1398063..1efd51c0bae 100644
--- a/distributed/comm/tests/test_ucx_config.py
+++ b/distributed/comm/tests/test_ucx_config.py
@@ -5,7 +5,6 @@
 pytestmark = pytest.mark.gpu
 
 import dask
-from dask.utils import format_bytes
 
 from distributed import Client
 from distributed.comm.ucx import _scrub_ucx_config
@@ -23,6 +22,7 @@
 
 @pytest.mark.asyncio
 async def test_ucx_config(cleanup):
+    ucx_110 = ucp.get_ucx_version() >= (1, 10, 0)
 
     ucx = {
         "nvlink": True,
@@ -33,10 +33,14 @@ async def test_ucx_config(cleanup):
         "cuda_copy": True,
     }
 
-    with dask.config.set(ucx=ucx):
+    with dask.config.set({"distributed.comm.ucx": ucx}):
         ucx_config = _scrub_ucx_config()
-        assert ucx_config.get("TLS") == "rc,tcp,sockcm,cuda_copy,cuda_ipc"
-        assert ucx_config.get("SOCKADDR_TLS_PRIORITY") == "sockcm"
+        if ucx_110:
+            assert ucx_config.get("TLS") == "rc,tcp,cuda_copy,cuda_ipc"
+            assert ucx_config.get("SOCKADDR_TLS_PRIORITY") == "tcp"
+        else:
+            assert ucx_config.get("TLS") == "rc,tcp,sockcm,cuda_copy,cuda_ipc"
+            assert ucx_config.get("SOCKADDR_TLS_PRIORITY") == "sockcm"
         assert ucx_config.get("NET_DEVICES") is None
 
     ucx = {
@@ -48,10 +52,14 @@ async def test_ucx_config(cleanup):
         "cuda_copy": False,
     }
 
-    with dask.config.set(ucx=ucx):
+    with dask.config.set({"distributed.comm.ucx": ucx}):
         ucx_config = _scrub_ucx_config()
-        assert ucx_config.get("TLS") == "rc,tcp,sockcm"
-        assert ucx_config.get("SOCKADDR_TLS_PRIORITY") == "sockcm"
+        if ucx_110:
+            assert ucx_config.get("TLS") == "rc,tcp"
+            assert ucx_config.get("SOCKADDR_TLS_PRIORITY") == "tcp"
+        else:
+            assert ucx_config.get("TLS") == "rc,tcp,sockcm"
+            assert ucx_config.get("SOCKADDR_TLS_PRIORITY") == "sockcm"
         assert ucx_config.get("NET_DEVICES") == "mlx5_0:1"
 
     ucx = {
@@ -59,16 +67,17 @@ async def test_ucx_config(cleanup):
         "infiniband": True,
         "rdmacm": True,
         "net-devices": "all",
-        "MEMTYPE_CACHE": "y",
         "tcp": True,
         "cuda_copy": True,
     }
 
-    with dask.config.set(ucx=ucx):
+    with dask.config.set({"distributed.comm.ucx": ucx}):
         ucx_config = _scrub_ucx_config()
-        assert ucx_config.get("TLS") == "rc,tcp,rdmacm,cuda_copy"
+        if ucx_110:
+            assert ucx_config.get("TLS") == "rc,tcp,cuda_copy"
+        else:
+            assert ucx_config.get("TLS") == "rc,tcp,rdmacm,cuda_copy"
         assert ucx_config.get("SOCKADDR_TLS_PRIORITY") == "rdmacm"
-        assert ucx_config.get("MEMTYPE_CACHE") == "y"
 
 
 def test_ucx_config_w_env_var(cleanup, loop, monkeypatch):
@@ -97,12 +106,13 @@ def test_ucx_config_w_env_var(cleanup, loop, monkeypatch):
                 while not c.scheduler_info()["workers"]:
                     sleep(0.1)
 
-                # configured with 1G pool
-                rmm_usage = c.run_on_scheduler(rmm.get_info)
-                assert size == format_bytes(rmm_usage.free)
+                # Check for RMM pool resource type
+                rmm_resource = c.run_on_scheduler(
+                    rmm.mr.get_current_device_resource_type
+                )
+                assert rmm_resource == rmm.mr.PoolMemoryResource
 
-                # configured with 1G pool
                 worker_addr = list(c.scheduler_info()["workers"])[0]
-                worker_rmm_usage = c.run(rmm.get_info)
-                rmm_usage = worker_rmm_usage[worker_addr]
-                assert size == format_bytes(rmm_usage.free)
+                worker_rmm_usage = c.run(rmm.mr.get_current_device_resource_type)
+                rmm_resource = worker_rmm_usage[worker_addr]
+                assert rmm_resource == rmm.mr.PoolMemoryResource

From 6afe2334ffa192ba11beeaa98841268f333ec187 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 2 Aug 2021 22:56:25 -0500
Subject: [PATCH 1411/1550] Add GPU executor if GPU is present (#5123)

Co-authored-by: James Bourbeau <jrbourbeau@gmail.com>
---
 distributed/tests/test_worker.py | 19 ++++++++++++++++---
 distributed/worker.py            |  4 ++++
 2 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index a9274623ebc..10bcd5456d4 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -20,6 +20,7 @@
 from dask import delayed
 from dask.system import CPU_COUNT
 
+import distributed
 from distributed import (
     Client,
     Nanny,
@@ -33,6 +34,7 @@
 from distributed.comm.tcp import TCPBackend
 from distributed.compatibility import LINUX, WINDOWS
 from distributed.core import CommClosedError, Status, rpc
+from distributed.diagnostics import nvml
 from distributed.diagnostics.plugin import PipInstall
 from distributed.metrics import time
 from distributed.scheduler import Scheduler
@@ -1979,16 +1981,16 @@ def get_thread_name():
     async with Worker(
         s.address,
         nthreads=2,
-        executor={"GPU": ThreadPoolExecutor(1, thread_name_prefix="Dask-GPU-Threads")},
+        executor={"foo": ThreadPoolExecutor(1, thread_name_prefix="Dask-Foo-Threads")},
     ):
         futures = []
         with dask.annotate(executor="default"):
             futures.append(c.submit(get_thread_name, pure=False))
-        with dask.annotate(executor="GPU"):
+        with dask.annotate(executor="foo"):
             futures.append(c.submit(get_thread_name, pure=False))
         default_result, gpu_result = await c.gather(futures)
         assert "Dask-Default-Threads" in default_result
-        assert "Dask-GPU-Threads" in gpu_result
+        assert "Dask-Foo-Threads" in gpu_result
 
 
 @gen_cluster(client=True)
@@ -2055,6 +2057,17 @@ async def test_process_executor_raise_exception(c, s, a, b):
             await future
 
 
+@pytest.mark.gpu
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)])
+async def test_gpu_executor(c, s, w):
+    if nvml.device_get_count() > 0:
+        e = w.executors["gpu"]
+        assert isinstance(e, distributed.threadpoolexecutor.ThreadPoolExecutor)
+        assert e._max_workers == 1
+    else:
+        assert "gpu" not in w.executors
+
+
 def assert_task_states_on_worker(expected, worker):
     for dep_key, expected_state in expected.items():
         assert dep_key in worker.tasks, (worker.name, dep_key, worker.tasks)
diff --git a/distributed/worker.py b/distributed/worker.py
index 1e4987cccb8..5952a813d81 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -635,6 +635,10 @@ def __init__(
             "offload": utils._offload_executor,
             "actor": ThreadPoolExecutor(1, thread_name_prefix="Dask-Actor-Threads"),
         }
+        if nvml.device_get_count() > 0:
+            self.executors["gpu"] = ThreadPoolExecutor(
+                1, thread_name_prefix="Dask-GPU-Threads"
+            )
 
         # Find the default executor
         if executor == "offload":

From 5db0290c1daf11bfa818050176e4b0e6cfa838e8 Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Tue, 3 Aug 2021 17:26:36 +0200
Subject: [PATCH 1412/1550] Rename total comm bytes and provide doc string
 (#5155)

---
 distributed/tests/test_worker.py |  3 +++
 distributed/worker.py            | 18 +++++++++++++++---
 2 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 10bcd5456d4..e5aa9c8250f 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -678,6 +678,9 @@ async def test_clean_nbytes(c, s, a, b):
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 20)
 async def test_gather_many_small(c, s, a, *workers):
+    """If the dependencies of a given task are very small, do not limit the
+    number of concurrent outgoing connections
+    """
     a.total_out_connections = 2
     futures = await c._scatter(list(range(100)))
 
diff --git a/distributed/worker.py b/distributed/worker.py
index 5952a813d81..b1e8542ef59 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -235,7 +235,10 @@ class Worker(ServerNode):
         The maximum number of concurrent outgoing requests for data
     * **total_in_connections**: ``int``
         The maximum number of concurrent incoming requests for data
-    * **total_comm_nbytes**: ``int``
+    * **comm_threshold_bytes**: ``int``
+        As long as the total number of bytes in flight is below this threshold
+        we will not limit the number of outgoing connections for a single tasks
+        dependency fetch.
     * **batched_stream**: ``BatchedSend``
         A batched stream along which we communicate to the scheduler
     * **log**: ``[(message)]``
@@ -426,7 +429,7 @@ def __init__(
         self.total_in_connections = dask.config.get(
             "distributed.worker.connections.incoming"
         )
-        self.total_comm_nbytes = 10e6
+        self.comm_threshold_bytes = 10e6
         self.comm_nbytes = 0
         self._missing_dep_flight = set()
 
@@ -2148,7 +2151,7 @@ def ensure_communicating(self):
 
                 while dependencies_fetch and (
                     len(self.in_flight_workers) < self.total_out_connections
-                    or self.comm_nbytes < self.total_comm_nbytes
+                    or self.comm_nbytes < self.comm_threshold_bytes
                 ):
                     to_gather_ts = dependencies_fetch.pop()
 
@@ -2292,6 +2295,15 @@ def select_keys_for_gather(self, worker, dep):
 
         return deps, total_bytes
 
+    @property
+    def total_comm_bytes(self):
+        warnings.warn(
+            "The attribute `Worker.total_comm_bytes` has been renamed to `comm_threshold_bytes`. "
+            "Future versions will only support the new name.",
+            DeprecationWarning,
+        )
+        return self.comm_threshold_bytes
+
     async def gather_dep(
         self,
         worker: str,

From 51ca4d905722dbea8eeb4b2966eda1a776730019 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 3 Aug 2021 17:09:48 -0500
Subject: [PATCH 1413/1550] Support asynchronous tasks (#5151)

This allows for user-defined async def functions to be submitted as
normal tasks and be run within the event loop.
---
 distributed/client.py                |  6 ++++
 distributed/tests/test_client.py     | 17 +++++++++--
 distributed/tests/test_utils.py      | 16 +++++++++-
 distributed/tests/test_utils_test.py |  8 +++++
 distributed/utils.py                 | 14 ++++++++-
 distributed/utils_test.py            |  7 +++++
 distributed/worker.py                | 45 +++++++++++++++++++++++++++-
 7 files changed, 107 insertions(+), 6 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 39153bd6228..c7b408a235b 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -1556,6 +1556,9 @@ def submit(
         Parameters
         ----------
         func : callable
+            Callable to be scheduled as ``func(*args **kwargs)``. If ``func`` is a coroutine,
+            it will be run on the main event loop of a worker. Otherwise ``func`` will be run
+            in a worker's task executor pool (see ``Worker.executors`` for more information.)
         *args
         **kwargs
         pure : bool (defaults to True)
@@ -1674,6 +1677,9 @@ def map(
         Parameters
         ----------
         func : callable
+            Callable to be scheduled for execution. If ``func`` is a coroutine,
+            it will be run on the main event loop of a worker. Otherwise ``func`` will be run
+            in a worker's task executor pool (see ``Worker.executors`` for more information.)
         iterables : Iterables
             List-like objects to map over.  They should have the same length.
         key : str, list
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index e642e6f6ae7..8c1e0645ee8 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -70,6 +70,7 @@
 from distributed.utils import is_valid_xml, mp_context, sync, tmp_text, tmpfile
 from distributed.utils_test import (
     TaskStateMetadataPlugin,
+    _UnhashableCallable,
     async_wait_for,
     asyncinc,
     captured_logger,
@@ -5595,9 +5596,9 @@ async def test_warn_when_submitting_large_values(c, s, a, b):
 
 @gen_cluster(client=True)
 async def test_unhashable_function(c, s, a, b):
-    d = {"a": 1}
-    result = await c.submit(d.get, "a")
-    assert result == 1
+    func = _UnhashableCallable()
+    result = await c.submit(func, 1)
+    assert result == 2
 
 
 @gen_cluster()
@@ -6923,3 +6924,13 @@ def f():
         assert results[n.worker_address] == 123
 
     assert files == set(os.listdir())  # no change
+
+
+@gen_cluster(client=True)
+async def test_async_task(c, s, a, b):
+    async def f(x):
+        return x + 1
+
+    future = c.submit(f, 10)
+    result = await future
+    assert result == 11
diff --git a/distributed/tests/test_utils.py b/distributed/tests/test_utils.py
index ab84be59d03..64802b5eb9d 100644
--- a/distributed/tests/test_utils.py
+++ b/distributed/tests/test_utils.py
@@ -29,6 +29,7 @@
     get_traceback,
     is_kernel,
     is_valid_xml,
+    iscoroutinefunction,
     nbytes,
     offload,
     open_port,
@@ -41,7 +42,15 @@
     truncate_exception,
     warn_on_duration,
 )
-from distributed.utils_test import captured_logger, div, gen_test, has_ipv6, inc, throws
+from distributed.utils_test import (
+    _UnhashableCallable,
+    captured_logger,
+    div,
+    gen_test,
+    has_ipv6,
+    inc,
+    throws,
+)
 
 
 def test_All(loop):
@@ -584,3 +593,8 @@ def test_parse_timedelta_deprecated():
     with pytest.warns(FutureWarning, match="parse_timedelta is deprecated"):
         from distributed.utils import parse_timedelta
     assert parse_timedelta is dask.utils.parse_timedelta
+
+
+def test_iscoroutinefunction_unhashable_input():
+    # Ensure iscoroutinefunction can handle unhashable callables
+    assert not iscoroutinefunction(_UnhashableCallable())
diff --git a/distributed/tests/test_utils_test.py b/distributed/tests/test_utils_test.py
index 1b02e6a4251..581a793c50c 100755
--- a/distributed/tests/test_utils_test.py
+++ b/distributed/tests/test_utils_test.py
@@ -13,6 +13,7 @@
 from distributed.metrics import time
 from distributed.utils import get_ip
 from distributed.utils_test import (
+    _UnhashableCallable,
     cluster,
     gen_cluster,
     gen_test,
@@ -269,3 +270,10 @@ async def test_tls_scheduler(security, cleanup):
         security=security, host="localhost", dashboard_address=":0"
     ) as s:
         assert s.address.startswith("tls")
+
+
+def test__UnhashableCallable():
+    func = _UnhashableCallable()
+    assert func(1) == 2
+    with pytest.raises(TypeError, match="unhashable"):
+        hash(func)
diff --git a/distributed/utils.py b/distributed/utils.py
index c691f3f8804..918d41c371f 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -1126,9 +1126,21 @@ def color_of(x, palette=palette):
     return palette[n % len(palette)]
 
 
+def _iscoroutinefunction(f):
+    return inspect.iscoroutinefunction(f) or gen.is_coroutine_function(f)
+
+
 @functools.lru_cache(None)
+def _iscoroutinefunction_cached(f):
+    return _iscoroutinefunction(f)
+
+
 def iscoroutinefunction(f):
-    return inspect.iscoroutinefunction(f) or gen.is_coroutine_function(f)
+    # Attempt to use lru_cache version and fall back to non-cached version if needed
+    try:
+        return _iscoroutinefunction_cached(f)
+    except TypeError:  # unhashable type
+        return _iscoroutinefunction(f)
 
 
 @contextmanager
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index bc02239188b..3072ff53d07 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -332,6 +332,13 @@ def slowidentity(*args, **kwargs):
         return args
 
 
+class _UnhashableCallable:
+    __hash__ = None
+
+    def __call__(self, x):
+        return x + 1
+
+
 def run_for(duration, timer=time):
     """
     Burn CPU for *duration* seconds.
diff --git a/distributed/worker.py b/distributed/worker.py
index b1e8542ef59..58ce124790b 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -2910,7 +2910,14 @@ async def execute(self, key):
             try:
                 e = self.executors[executor]
                 ts.start_time = time()
-                if "ThreadPoolExecutor" in str(type(e)):
+                if iscoroutinefunction(function):
+                    result = await apply_function_async(
+                        function,
+                        args2,
+                        kwargs2,
+                        self.scheduler_delay,
+                    )
+                elif "ThreadPoolExecutor" in str(type(e)):
                     result = await self.loop.run_in_executor(
                         e,
                         apply_function,
@@ -3885,6 +3892,42 @@ def apply_function_simple(
     return msg
 
 
+async def apply_function_async(
+    function,
+    args,
+    kwargs,
+    time_delay,
+):
+    """Run a function, collect information
+
+    Returns
+    -------
+    msg: dictionary with status, result/error, timings, etc..
+    """
+    ident = threading.get_ident()
+    start = time()
+    try:
+        result = await function(*args, **kwargs)
+    except Exception as e:
+        msg = error_message(e)
+        msg["op"] = "task-erred"
+        msg["actual-exception"] = e
+    else:
+        msg = {
+            "op": "task-finished",
+            "status": "OK",
+            "result": result,
+            "nbytes": sizeof(result),
+            "type": type(result) if result is not None else None,
+        }
+    finally:
+        end = time()
+    msg["start"] = start + time_delay
+    msg["stop"] = end + time_delay
+    msg["thread"] = ident
+    return msg
+
+
 def apply_function_actor(
     function, args, kwargs, execution_state, key, active_threads, active_threads_lock
 ):

From e34b3e868f95bd375a02640a66abcc4f2074429a Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Wed, 4 Aug 2021 15:38:31 +0100
Subject: [PATCH 1414/1550] Use Python 3 super() calls (#5167)

---
 distributed/client.py                     | 2 +-
 distributed/comm/inproc.py                | 2 +-
 distributed/comm/tcp.py                   | 2 +-
 distributed/comm/ucx.py                   | 2 +-
 distributed/comm/ws.py                    | 2 +-
 distributed/deploy/tests/test_adaptive.py | 2 +-
 distributed/diagnostics/progress.py       | 4 +---
 distributed/nanny.py                      | 2 +-
 distributed/tests/test_worker.py          | 2 +-
 distributed/worker.py                     | 2 +-
 10 files changed, 10 insertions(+), 12 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index c7b408a235b..9da60faa9c6 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -378,7 +378,7 @@ def __setstate__(self, state):
             c = Client.current(allow_global=False)
         except ValueError:
             c = get_client(address)
-        Future.__init__(self, key, c)
+        self.__init__(key, c)
         c._send_to_scheduler(
             {
                 "op": "update-graph",
diff --git a/distributed/comm/inproc.py b/distributed/comm/inproc.py
index bc812540a5e..804f8ec3916 100644
--- a/distributed/comm/inproc.py
+++ b/distributed/comm/inproc.py
@@ -155,7 +155,7 @@ class InProc(Comm):
     def __init__(
         self, local_addr, peer_addr, read_q, write_q, write_loop, deserialize=True
     ):
-        Comm.__init__(self)
+        super().__init__()
         self._local_addr = local_addr
         self._peer_addr = peer_addr
         self.deserialize = deserialize
diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py
index 22d7c72461a..0e66378862e 100644
--- a/distributed/comm/tcp.py
+++ b/distributed/comm/tcp.py
@@ -149,7 +149,7 @@ class TCP(Comm):
 
     def __init__(self, stream, local_addr, peer_addr, deserialize=True):
         self._closed = False
-        Comm.__init__(self)
+        super().__init__()
         self._local_addr = local_addr
         self._peer_addr = peer_addr
         self.stream = stream
diff --git a/distributed/comm/ucx.py b/distributed/comm/ucx.py
index 0e048c9f6d9..b98fb9e9475 100644
--- a/distributed/comm/ucx.py
+++ b/distributed/comm/ucx.py
@@ -167,7 +167,7 @@ class UCX(Comm):
     """
 
     def __init__(self, ep, local_addr: str, peer_addr: str, deserialize=True):
-        Comm.__init__(self)
+        super().__init__()
         self._ep = ep
         if local_addr:
             assert local_addr.startswith("ucx")
diff --git a/distributed/comm/ws.py b/distributed/comm/ws.py
index 46ce55b86e5..1b6c5996833 100644
--- a/distributed/comm/ws.py
+++ b/distributed/comm/ws.py
@@ -166,7 +166,7 @@ class WS(Comm):
 
     def __init__(self, sock, deserialize=True, allow_offload=True):
         self._closed = False
-        Comm.__init__(self)
+        super().__init__()
         self.sock = sock
         self._peer_addr = f"{self.prefix}{self.sock.parsed.netloc}"
         self._local_addr = f"{self.prefix}{self.sock.parsed.netloc}"
diff --git a/distributed/deploy/tests/test_adaptive.py b/distributed/deploy/tests/test_adaptive.py
index d463796dc5e..739fa708656 100644
--- a/distributed/deploy/tests/test_adaptive.py
+++ b/distributed/deploy/tests/test_adaptive.py
@@ -87,7 +87,7 @@ async def test_adaptive_scale_down_override():
     class TestAdaptive(Adaptive):
         def __init__(self, *args, **kwargs):
             self.min_size = kwargs.pop("min_size", 0)
-            Adaptive.__init__(self, *args, **kwargs)
+            super().__init__(*args, **kwargs)
 
         async def workers_to_close(self, **kwargs):
             num_workers = len(self.cluster.workers)
diff --git a/distributed/diagnostics/progress.py b/distributed/diagnostics/progress.py
index 40e67471259..1f7940c0302 100644
--- a/distributed/diagnostics/progress.py
+++ b/distributed/diagnostics/progress.py
@@ -157,9 +157,7 @@ def __init__(
         self, keys, scheduler=None, func=key_split, minimum=0, dt=0.1, complete=False
     ):
         self.func = func
-        Progress.__init__(
-            self, keys, scheduler, minimum=minimum, dt=dt, complete=complete
-        )
+        super().__init__(keys, scheduler, minimum=minimum, dt=dt, complete=complete)
 
     async def setup(self):
         keys = self.keys
diff --git a/distributed/nanny.py b/distributed/nanny.py
index eea2f3113f2..3afbcfd3819 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -586,7 +586,7 @@ async def close(self, comm=None, timeout=5, report=None):
         self.status = Status.closed
         if comm:
             await comm.write("OK")
-        await ServerNode.close(self)
+        await super().close()
 
 
 class WorkerProcess:
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index e5aa9c8250f..010e6d56cb4 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -106,7 +106,7 @@ class MockLoggingHandler(logging.Handler):
 
         def __init__(self, *args, **kwargs):
             self.reset()
-            logging.Handler.__init__(self, *args, **kwargs)
+            super().__init__(*args, **kwargs)
 
         def emit(self, record):
             self.messages[record.levelname.lower()].append(record.getMessage())
diff --git a/distributed/worker.py b/distributed/worker.py
index 58ce124790b..42a08ff2727 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -1324,7 +1324,7 @@ async def close(
             await self.rpc.close()
 
             self.status = Status.closed
-            await ServerNode.close(self)
+            await super().close()
 
             setproctitle("dask-worker [closed]")
         return "OK"

From f15739c816f7c8c991b21772167105772ef0dfc9 Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Wed, 4 Aug 2021 16:20:00 +0100
Subject: [PATCH 1415/1550] Tweak verbiage around async functions (#5166)

---
 distributed/client.py    | 14 ++++++++------
 distributed/comm/core.py | 16 ++++++++--------
 distributed/process.py   |  6 +++---
 3 files changed, 19 insertions(+), 17 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 9da60faa9c6..a5484f809bd 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -1556,9 +1556,10 @@ def submit(
         Parameters
         ----------
         func : callable
-            Callable to be scheduled as ``func(*args **kwargs)``. If ``func`` is a coroutine,
-            it will be run on the main event loop of a worker. Otherwise ``func`` will be run
-            in a worker's task executor pool (see ``Worker.executors`` for more information.)
+            Callable to be scheduled as ``func(*args **kwargs)``. If ``func`` returns a
+            coroutine, it will be run on the main event loop of a worker. Otherwise
+            ``func`` will be run in a worker's task executor pool (see
+            ``Worker.executors`` for more information.)
         *args
         **kwargs
         pure : bool (defaults to True)
@@ -1677,9 +1678,10 @@ def map(
         Parameters
         ----------
         func : callable
-            Callable to be scheduled for execution. If ``func`` is a coroutine,
-            it will be run on the main event loop of a worker. Otherwise ``func`` will be run
-            in a worker's task executor pool (see ``Worker.executors`` for more information.)
+            Callable to be scheduled for execution. If ``func`` returns a coroutine, it
+            will be run on the main event loop of a worker. Otherwise ``func`` will be
+            run in a worker's task executor pool (see ``Worker.executors`` for more
+            information.)
         iterables : Iterables
             List-like objects to map over.  They should have the same length.
         key : str, list
diff --git a/distributed/comm/core.py b/distributed/comm/core.py
index a80863155e4..8c3e25cc159 100644
--- a/distributed/comm/core.py
+++ b/distributed/comm/core.py
@@ -53,11 +53,11 @@ def __init__(self):
     # XXX add set_close_callback()?
 
     @abstractmethod
-    def read(self, deserializers=None):
+    async def read(self, deserializers=None):
         """
         Read and return a message (a Python object).
 
-        This method is a coroutine.
+        This method returns a coroutine.
 
         Parameters
         ----------
@@ -67,11 +67,11 @@ def read(self, deserializers=None):
         """
 
     @abstractmethod
-    def write(self, msg, serializers=None, on_error=None):
+    async def write(self, msg, serializers=None, on_error=None):
         """
         Write a message (a Python object).
 
-        This method is a coroutine.
+        This method returns a coroutine.
 
         Parameters
         ----------
@@ -82,12 +82,12 @@ def write(self, msg, serializers=None, on_error=None):
         """
 
     @abstractmethod
-    def close(self):
+    async def close(self):
         """
         Close the communication cleanly.  This will attempt to flush
         outgoing buffers before actually closing the underlying transport.
 
-        This method is a coroutine.
+        This method returns a coroutine.
         """
 
     @abstractmethod
@@ -241,10 +241,10 @@ async def on_connection(self, comm: Comm, handshake_overrides=None):
 
 class Connector(ABC):
     @abstractmethod
-    def connect(self, address, deserialize=True):
+    async def connect(self, address, deserialize=True):
         """
         Connect to the given address and return a Comm object.
-        This function is a coroutine.   It may raise EnvironmentError
+        This function returns a coroutine. It may raise EnvironmentError
         if the other endpoint is unreachable or unavailable.  It
         may raise ValueError if the address is malformed.
         """
diff --git a/distributed/process.py b/distributed/process.py
index 9be72b07566..4794b3aebab 100644
--- a/distributed/process.py
+++ b/distributed/process.py
@@ -249,7 +249,7 @@ def start(self):
         """
         Start the child process.
 
-        This method is a coroutine.
+        This method returns a future.
         """
         self._check_closed()
         fut = Future()
@@ -260,7 +260,7 @@ def terminate(self):
         """
         Terminate the child process.
 
-        This method is a coroutine.
+        This method returns a future.
         """
         self._check_closed()
         fut = Future()
@@ -271,7 +271,7 @@ async def join(self, timeout=None):
         """
         Wait for the child process to exit.
 
-        This method is a coroutine.
+        This method returns a coroutine.
         """
         self._check_closed()
         assert self._state.pid is not None, "can only join a started process"

From fa8475bb95d92f12ea24db69384a65ee3d1140ce Mon Sep 17 00:00:00 2001
From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com>
Date: Wed, 4 Aug 2021 12:06:33 -0400
Subject: [PATCH 1416/1550] Bump `RAPIDS_VER` to 21.10 (#5165)

---
 continuous_integration/gpuci/axis.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/continuous_integration/gpuci/axis.yaml b/continuous_integration/gpuci/axis.yaml
index 76f19ff492c..587b193b003 100644
--- a/continuous_integration/gpuci/axis.yaml
+++ b/continuous_integration/gpuci/axis.yaml
@@ -1,13 +1,13 @@
 PYTHON_VER:
-- 3.8
+- "3.8"
 
 CUDA_VER:
-- 11.2
+- "11.2"
 
 LINUX_VER:
 - ubuntu18.04
 
 RAPIDS_VER:
-- 21.08
+- "21.10"
 
 excludes:
\ No newline at end of file

From b795021bacec7ba4b3a34b4ff492cf1f8b6c6dd6 Mon Sep 17 00:00:00 2001
From: Gabe Joseph <gjoseph92@gmail.com>
Date: Wed, 4 Aug 2021 11:48:45 -0600
Subject: [PATCH 1417/1550] Instructions for jemalloc with brew on macOS
 (#4996)

For those of us who prefer not to use conda, having a global install of jemalloc with homebrew is handy.
---
 docs/source/worker.rst | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/docs/source/worker.rst b/docs/source/worker.rst
index 3b409bcfdcd..53ed2abb9ee 100644
--- a/docs/source/worker.rst
+++ b/docs/source/worker.rst
@@ -378,13 +378,20 @@ On Linux:
     conda install jemalloc
     LD_PRELOAD=$CONDA_PREFIX/lib/libjemalloc.so dask-worker <...>
 
-On MacOS:
+On macOS:
 
 .. code-block:: bash
 
     conda install jemalloc
     DYLD_INSERT_LIBRARIES=$CONDA_PREFIX/lib/libjemalloc.dylib dask-worker <...>
 
+Alternatively on macOS, install globally with `homebrew`_:
+
+.. code-block:: bash
+
+    brew install jemalloc
+    DYLD_INSERT_LIBRARIES=$(brew --prefix jemalloc)/lib/libjemalloc.dylib dask-worker <...>
+
 `jemalloc`_ offers a wealth of configuration settings; please refer to its
 documentation.
 
@@ -432,3 +439,4 @@ API Documentation
 .. _malloc_trim: https://man7.org/linux/man-pages/man3/malloc_trim.3.html
 .. _brk: https://www.man7.org/linux/man-pages/man2/brk.2.html
 .. _jemalloc: http://jemalloc.net
+.. _homebrew: https://brew.sh/
\ No newline at end of file

From 3d73623da9c9575d87aa4284da7ab652619845e7 Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Thu, 5 Aug 2021 10:21:21 +0100
Subject: [PATCH 1418/1550] Fix flaky tests in CI (#5168)

---
 distributed/comm/tests/test_ucx.py            | 16 ++---
 .../diagnostics/tests/test_graph_layout.py    | 15 ++--
 distributed/tests/test_client_executor.py     | 71 ++++++++-----------
 distributed/tests/test_diskutils.py           | 39 ++++------
 4 files changed, 58 insertions(+), 83 deletions(-)

diff --git a/distributed/comm/tests/test_ucx.py b/distributed/comm/tests/test_ucx.py
index 502afdfc4f0..2f2a2b04768 100644
--- a/distributed/comm/tests/test_ucx.py
+++ b/distributed/comm/tests/test_ucx.py
@@ -236,8 +236,8 @@ async def test_large_cupy(n, cleanup):
     await serv_com.close()
 
 
-@pytest.mark.asyncio
-async def test_ping_pong_numba(cleanup):
+@gen_test()
+async def test_ping_pong_numba():
     np = pytest.importorskip("numpy")
     numba = pytest.importorskip("numba")
     import numba.cuda
@@ -258,9 +258,9 @@ async def test_ping_pong_numba(cleanup):
 @pytest.mark.asyncio
 async def test_ucx_localcluster(processes, cleanup):
     async with LocalCluster(
-        protocol="ucx:://",
+        protocol="ucx",
         host=HOST,
-        dashboard_address=None,
+        dashboard_address=":0",
         n_workers=2,
         threads_per_worker=1,
         processes=processes,
@@ -276,20 +276,20 @@ async def test_ucx_localcluster(processes, cleanup):
 
 
 @pytest.mark.slow
-@pytest.mark.asyncio
-async def test_stress(cleanup):
+@gen_test(timeout=240)
+async def test_stress():
     da = pytest.importorskip("dask.array")
 
     chunksize = "10 MB"
 
     async with LocalCluster(
         protocol="ucx",
-        dashboard_address=None,
+        dashboard_address=":0",
         asynchronous=True,
         processes=False,
         host=HOST,
     ) as cluster:
-        async with Client(cluster, asynchronous=True) as client:
+        async with Client(cluster, asynchronous=True):
             rs = da.random.RandomState()
             x = rs.random((10000, 10000), chunks=(-1, chunksize))
             x = x.persist()
diff --git a/distributed/diagnostics/tests/test_graph_layout.py b/distributed/diagnostics/tests/test_graph_layout.py
index b714b261a4c..494cd26e3ca 100644
--- a/distributed/diagnostics/tests/test_graph_layout.py
+++ b/distributed/diagnostics/tests/test_graph_layout.py
@@ -38,16 +38,13 @@ async def test_construct_after_call(c, s, a, b):
 @gen_cluster(client=True)
 async def test_states(c, s, a, b):
     gl = GraphLayout(s)
-    futures = c.map(inc, range(5))
-    total = c.submit(sum, futures)
-    del futures
+    await c.submit(sum, c.map(inc, range(5)))
 
-    await total
-
-    updates = {state for idx, state in gl.state_updates}
-    assert "memory" in updates
-    assert "processing" in updates
-    assert "released" in updates
+    while True:
+        updates = {state for _, state in gl.state_updates}
+        if updates == {"waiting", "processing", "memory", "released"}:
+            break
+        await asyncio.sleep(0.01)
 
 
 @gen_cluster(client=True)
diff --git a/distributed/tests/test_client_executor.py b/distributed/tests/test_client_executor.py
index 3bcf0b784fa..8a50f5f27dc 100644
--- a/distributed/tests/test_client_executor.py
+++ b/distributed/tests/test_client_executor.py
@@ -12,18 +12,9 @@
 import pytest
 from tlz import take
 
-from distributed import Client
 from distributed.compatibility import MACOS
 from distributed.utils import CancelledError
-from distributed.utils_test import (
-    cluster,
-    inc,
-    slowadd,
-    slowdec,
-    slowinc,
-    throws,
-    varying,
-)
+from distributed.utils_test import inc, slowadd, slowdec, slowinc, throws, varying
 
 
 def number_of_processing_tasks(client):
@@ -116,7 +107,7 @@ def test_cancellation(client):
 
 def test_cancellation_wait(client):
     with client.get_executor(pure=False) as e:
-        fs = [e.submit(slowinc, i, delay=0.1) for i in range(10)]
+        fs = [e.submit(slowinc, i, delay=0.2) for i in range(10)]
         fs[3].cancel()
         res = wait(fs, return_when=FIRST_COMPLETED, timeout=30)
         assert len(res.not_done) > 0
@@ -128,7 +119,7 @@ def test_cancellation_wait(client):
 
 def test_cancellation_as_completed(client):
     with client.get_executor(pure=False) as e:
-        fs = [e.submit(slowinc, i, delay=0.1) for i in range(10)]
+        fs = [e.submit(slowinc, i, delay=0.2) for i in range(10)]
         fs[3].cancel()
         fs[8].cancel()
 
@@ -225,32 +216,30 @@ def test_retries(client):
             res = future.result()
 
 
-def test_shutdown(loop):
-    with cluster() as (s, [a, b]):
-        with Client(s["address"], loop=loop) as client:
-            # shutdown(wait=True) waits for pending tasks to finish
-            e = client.get_executor()
-            fut = e.submit(time.sleep, 1.0)
-            t1 = time.time()
-            e.shutdown()
-            dt = time.time() - t1
-            assert 0.5 <= dt <= 2.0
-            time.sleep(0.1)  # wait for future outcome to propagate
-            assert fut.done()
-            fut.result()  # doesn't raise
-
-            with pytest.raises(RuntimeError):
-                e.submit(time.sleep, 1.0)
-
-            # shutdown(wait=False) cancels pending tasks
-            e = client.get_executor()
-            fut = e.submit(time.sleep, 2.0)
-            t1 = time.time()
-            e.shutdown(wait=False)
-            dt = time.time() - t1
-            assert dt < 0.5
-            time.sleep(0.1)  # wait for future outcome to propagate
-            assert fut.cancelled()
-
-            with pytest.raises(RuntimeError):
-                e.submit(time.sleep, 1.0)
+def test_shutdown_wait(client):
+    # shutdown(wait=True) waits for pending tasks to finish
+    e = client.get_executor()
+    start = time.time()
+    fut = e.submit(time.sleep, 1.0)
+    e.shutdown()
+    assert time.time() >= start + 1.0
+    time.sleep(0.1)  # wait for future outcome to propagate
+    assert fut.done()
+    fut.result()  # doesn't raise
+
+    with pytest.raises(RuntimeError):
+        e.submit(time.sleep, 1.0)
+
+
+def test_shutdown_nowait(client):
+    # shutdown(wait=False) cancels pending tasks
+    e = client.get_executor()
+    start = time.time()
+    fut = e.submit(time.sleep, 5.0)
+    e.shutdown(wait=False)
+    assert time.time() < start + 2.0
+    time.sleep(0.1)  # wait for future outcome to propagate
+    assert fut.cancelled()
+
+    with pytest.raises(RuntimeError):
+        e.submit(time.sleep, 1.0)
diff --git a/distributed/tests/test_diskutils.py b/distributed/tests/test_diskutils.py
index 077a37d4fba..de37fe76ca2 100644
--- a/distributed/tests/test_diskutils.py
+++ b/distributed/tests/test_diskutils.py
@@ -12,7 +12,6 @@
 
 import dask
 
-from distributed.compatibility import MACOS
 from distributed.diskutils import WorkSpace
 from distributed.metrics import time
 from distributed.utils import mp_context
@@ -190,7 +189,8 @@ def test_locking_disabled(tmpdir):
         lock_file.assert_not_called()
 
 
-def _workspace_concurrency(base_dir, purged_q, err_q, stop_evt):
+def _workspace_concurrency(base_dir, purged_q, err_q, stop_evt, barrier):
+    barrier.wait()
     ws = WorkSpace(base_dir)
     n_purged = 0
     with captured_logger("distributed.diskutils", "ERROR") as sio:
@@ -213,9 +213,10 @@ def _workspace_concurrency(base_dir, purged_q, err_q, stop_evt):
     purged_q.put(n_purged)
 
 
-def _test_workspace_concurrency(tmpdir, timeout, max_procs):
-    """
-    WorkSpace concurrency test.  We merely check that no exception or
+@pytest.mark.slow
+@pytest.mark.parametrize("timeout,max_procs", [(5.0, 6), (10.0, 16)])
+def test_workspace_concurrency(tmpdir, timeout, max_procs):
+    """WorkSpace concurrency test. We merely check that no exception or
     deadlock happens.
     """
     base_dir = str(tmpdir)
@@ -228,20 +229,21 @@ def _test_workspace_concurrency(tmpdir, timeout, max_procs):
     ws._purge_leftovers = lambda: None
 
     # Run a bunch of child processes that will try to purge concurrently
-    NPROCS = 2 if sys.platform == "win32" else max_procs
+    barrier = mp_context.Barrier(parties=max_procs + 1)
     processes = [
         mp_context.Process(
-            target=_workspace_concurrency, args=(base_dir, purged_q, err_q, stop_evt)
+            target=_workspace_concurrency,
+            args=(base_dir, purged_q, err_q, stop_evt, barrier),
         )
-        for i in range(NPROCS)
+        for _ in range(max_procs)
     ]
     for p in processes:
         p.start()
-
+    barrier.wait()
     n_created = 0
     n_purged = 0
+    t1 = time()
     try:
-        t1 = time()
         while time() - t1 < timeout:
             # Add a bunch of locks, and simulate forgetting them.
             # The concurrent processes should try to purge them.
@@ -249,7 +251,7 @@ def _test_workspace_concurrency(tmpdir, timeout, max_procs):
                 d = ws.new_work_dir(prefix="workspace-concurrency-")
                 d._finalizer.detach()
                 n_created += 1
-            sleep(1e-2)
+            sleep(0.01)
     finally:
         stop_evt.set()
         for p in processes:
@@ -268,19 +270,6 @@ def _test_workspace_concurrency(tmpdir, timeout, max_procs):
             n_purged += purged_q.get_nowait()
     except queue.Empty:
         pass
+    assert n_created >= 100
     # We attempted to purge most directories at some point
     assert n_purged >= 0.5 * n_created > 0
-    return n_created, n_purged
-
-
-@pytest.mark.slow
-@pytest.mark.xfail(condition=MACOS, reason="extremely flaky")
-def test_workspace_concurrency(tmpdir):
-    _test_workspace_concurrency(tmpdir, 5.0, 6)
-
-
-@pytest.mark.slow
-@pytest.mark.xfail(condition=MACOS, reason="extremely flaky")
-def test_workspace_concurrency_intense(tmpdir):
-    n_created, n_purged = _test_workspace_concurrency(tmpdir, 8.0, 16)
-    assert n_created >= 100

From 9fc7e312decb13025f835453ababd5ba4bd8141e Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Thu, 5 Aug 2021 13:12:42 -0500
Subject: [PATCH 1419/1550] Deprecate worker plugin overwrite policy (#5146)

---
 distributed/diagnostics/tests/test_worker_plugin.py | 3 ++-
 distributed/worker.py                               | 7 +++++++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/distributed/diagnostics/tests/test_worker_plugin.py b/distributed/diagnostics/tests/test_worker_plugin.py
index f3aa8cc6daf..7ae01f09227 100644
--- a/distributed/diagnostics/tests/test_worker_plugin.py
+++ b/distributed/diagnostics/tests/test_worker_plugin.py
@@ -198,7 +198,8 @@ def setup(self, worker):
     assert list(responses.values()) == [{"status": "OK"}]
 
     async with Worker(s.address, loop=s.loop):
-        responses = await c.register_worker_plugin(FooWorkerPlugin(), name="foo")
+        with pytest.warns(FutureWarning, match="worker plugin will be overwritten"):
+            responses = await c.register_worker_plugin(FooWorkerPlugin(), name="foo")
         assert list(responses.values()) == [{"status": "repeat"}] * 2
 
 
diff --git a/distributed/worker.py b/distributed/worker.py
index 42a08ff2727..4ac7e7ec24f 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -2738,6 +2738,13 @@ async def plugin_add(self, comm=None, plugin=None, name=None):
             assert name
 
             if name in self.plugins:
+                warnings.warn(
+                    "Attempting to add a worker plugin with the same name as an already registered "
+                    f"plugin ({name}). Currently this results in no change and the previously registered "
+                    "plugin is not overwritten. This behavior is deprecated and in a future release "
+                    f"the previously registered {name} worker plugin will be overwritten.",
+                    category=FutureWarning,
+                )
                 return {"status": "repeat"}
             else:
                 self.plugins[name] = plugin

From f59e4343e45a5428d4cca8ef5b35123dbfe1148f Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Thu, 5 Aug 2021 21:55:42 +0200
Subject: [PATCH 1420/1550] Add no-worker memory transition to scheduler
 (#5174)

---
 distributed/scheduler.py            | 37 +++++++++++++++++++++++++++++
 distributed/tests/test_scheduler.py | 21 ++++++++++++++++
 2 files changed, 58 insertions(+)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 566f149fc3c..3025695b367 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1969,6 +1969,7 @@ def __init__(
             ("processing", "erred"): self.transition_processing_erred,
             ("no-worker", "released"): self.transition_no_worker_released,
             ("no-worker", "waiting"): self.transition_no_worker_waiting,
+            ("no-worker", "memory"): self.transition_no_worker_memory,
             ("released", "forgotten"): self.transition_released_forgotten,
             ("memory", "forgotten"): self.transition_memory_forgotten,
             ("erred", "released"): self.transition_erred_released,
@@ -2450,6 +2451,42 @@ def transition_no_worker_waiting(self, key):
                 pdb.set_trace()
             raise
 
+    def transition_no_worker_memory(
+        self, key, nbytes=None, type=None, typename: str = None, worker=None
+    ):
+        try:
+            ws: WorkerState = self._workers_dv[worker]
+            ts: TaskState = self._tasks[key]
+            recommendations: dict = {}
+            client_msgs: dict = {}
+            worker_msgs: dict = {}
+
+            if self._validate:
+                assert not ts._processing_on
+                assert not ts._waiting_on
+                assert ts._state == "no-worker"
+
+            self._unrunnable.remove(ts)
+
+            if nbytes is not None:
+                ts.set_nbytes(nbytes)
+
+            self.check_idle_saturated(ws)
+
+            _add_to_memory(
+                self, ts, ws, recommendations, client_msgs, type=type, typename=typename
+            )
+            ts.state = "memory"
+
+            return recommendations, client_msgs, worker_msgs
+        except Exception as e:
+            logger.exception(e)
+            if LOG_PDB:
+                import pdb
+
+                pdb.set_trace()
+            raise
+
     @ccall
     @exceptval(check=False)
     def decide_worker(self, ts: TaskState) -> WorkerState:
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 536fd3fd45f..b8b51a27905 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -3183,3 +3183,24 @@ async def test_worker_heartbeat_after_cancel(c, s, *workers):
 
     while any(w.tasks for w in workers):
         await asyncio.gather(*[w.heartbeat() for w in workers])
+
+
+@gen_cluster(client=True, nthreads=[("", 1)])
+async def test_worker_reconnect_task_memory_with_resources(c, s, a):
+    async with Worker(s.address, resources={"A": 1}) as b:
+        b.periodic_callbacks["heartbeat"].stop()
+
+        futs = c.map(inc, range(10), resources={"A": 1})
+        res = c.submit(sum, futs)
+
+        while not b.executing_count and not b.data:
+            await asyncio.sleep(0.001)
+
+        await s.remove_worker(address=b.address, close=False)
+        while not res.done():
+            await b.heartbeat()
+
+        await res
+        assert ("no-worker", "memory") in {
+            (start, finish) for (_, start, finish, _, _) in s.transition_log
+        }

From 1e1ff52c6141aef48a1ca72b475854eb510c7c12 Mon Sep 17 00:00:00 2001
From: Julia Signell <jsignell@gmail.com>
Date: Fri, 6 Aug 2021 06:31:11 -0400
Subject: [PATCH 1421/1550] Set z-index of data-table lower (#5175)

---
 distributed/http/static/css/base.css | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/distributed/http/static/css/base.css b/distributed/http/static/css/base.css
index 0cc3583a15a..78607c7aefe 100644
--- a/distributed/http/static/css/base.css
+++ b/distributed/http/static/css/base.css
@@ -102,6 +102,10 @@ body {
   right: 4px;
 }
 
+.bk-root .bk-data-table {
+  z-index: 0;
+}
+
 .content {
   width: 100%;
   height: 100%;

From 17c066a8cbfbb9e61b84096296394b0263d915fb Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Fri, 6 Aug 2021 13:41:07 +0200
Subject: [PATCH 1422/1550] Resolve deadlock cause by transition error after
 fetching dependency (#5157)

---
 distributed/tests/test_utils_test.py | 98 +++++++++++++++++++++++++++-
 distributed/tests/test_worker.py     | 95 +++++++++++++++++++++++++++
 distributed/utils_test.py            | 78 +++++++++++++++++++++-
 distributed/worker.py                | 74 +++++++++++++--------
 4 files changed, 315 insertions(+), 30 deletions(-)

diff --git a/distributed/tests/test_utils_test.py b/distributed/tests/test_utils_test.py
index 581a793c50c..0de9c7793ca 100755
--- a/distributed/tests/test_utils_test.py
+++ b/distributed/tests/test_utils_test.py
@@ -9,10 +9,11 @@
 from tornado import gen
 
 from distributed import Client, Nanny, Scheduler, Worker, config, default_client
-from distributed.core import rpc
+from distributed.core import Server, rpc
 from distributed.metrics import time
 from distributed.utils import get_ip
 from distributed.utils_test import (
+    _LockedCommPool,
     _UnhashableCallable,
     cluster,
     gen_cluster,
@@ -277,3 +278,98 @@ def test__UnhashableCallable():
     assert func(1) == 2
     with pytest.raises(TypeError, match="unhashable"):
         hash(func)
+
+
+class MyServer(Server):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.handlers["ping"] = self.pong
+        self.counter = 0
+
+    def pong(self, comm):
+        self.counter += 1
+        return "pong"
+
+
+@pytest.mark.asyncio
+async def test_locked_comm_drop_in_replacement(loop):
+
+    a = await MyServer({})
+    await a.listen(0)
+
+    read_event = asyncio.Event()
+    read_event.set()
+    read_queue = asyncio.Queue()
+    original_pool = a.rpc
+    a.rpc = _LockedCommPool(original_pool, read_event=read_event, read_queue=read_queue)
+
+    b = await MyServer({})
+    await b.listen(0)
+    # Event is set, the pool works like an ordinary pool
+    res = await a.rpc(b.address).ping()
+    assert await read_queue.get() == (b.address, "pong")
+    assert res == "pong"
+    assert b.counter == 1
+
+    read_event.clear()
+    # Can also be used without a lock to intercept network traffic
+    a.rpc = _LockedCommPool(original_pool, read_queue=read_queue)
+    a.rpc.remove(b.address)
+    res = await a.rpc(b.address).ping()
+    assert await read_queue.get() == (b.address, "pong")
+
+
+@pytest.mark.asyncio
+async def test_locked_comm_intercept_read(loop):
+
+    a = await MyServer({})
+    await a.listen(0)
+    b = await MyServer({})
+    await b.listen(0)
+
+    read_event = asyncio.Event()
+    read_queue = asyncio.Queue()
+    a.rpc = _LockedCommPool(a.rpc, read_event=read_event, read_queue=read_queue)
+
+    async def ping_pong():
+        return await a.rpc(b.address).ping()
+
+    fut = asyncio.create_task(ping_pong())
+
+    # We didn't block the write but merely the read. The remove should have
+    # received the message and responded already
+    while not b.counter:
+        await asyncio.sleep(0.001)
+
+    with pytest.raises(asyncio.TimeoutError):
+        await asyncio.wait_for(asyncio.shield(fut), 0.01)
+
+    assert await read_queue.get() == (b.address, "pong")
+    read_event.set()
+    assert await fut == "pong"
+
+
+@pytest.mark.asyncio
+async def test_locked_comm_intercept_write(loop):
+
+    a = await MyServer({})
+    await a.listen(0)
+    b = await MyServer({})
+    await b.listen(0)
+
+    write_event = asyncio.Event()
+    write_queue = asyncio.Queue()
+    a.rpc = _LockedCommPool(a.rpc, write_event=write_event, write_queue=write_queue)
+
+    async def ping_pong():
+        return await a.rpc(b.address).ping()
+
+    fut = asyncio.create_task(ping_pong())
+
+    with pytest.raises(asyncio.TimeoutError):
+        await asyncio.wait_for(asyncio.shield(fut), 0.01)
+    # Write was blocked. The remote hasn't received the message, yet
+    assert b.counter == 0
+    assert await write_queue.get() == (b.address, {"op": "ping", "reply": True})
+    write_event.set()
+    assert await fut == "pong"
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 010e6d56cb4..a22903db371 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -7,6 +7,7 @@
 import traceback
 from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
 from concurrent.futures.process import BrokenProcessPool
+from contextlib import suppress
 from numbers import Number
 from operator import add
 from time import sleep
@@ -41,6 +42,7 @@
 from distributed.utils import TimeoutError, tmpfile
 from distributed.utils_test import (
     TaskStateMetadataPlugin,
+    _LockedCommPool,
     captured_logger,
     dec,
     div,
@@ -2530,3 +2532,96 @@ def __call__(self, *args, **kwargs):
 
     finally:
         threadpool.shutdown()
+
+
+@gen_cluster(client=True)
+async def test_gather_dep_exception_one_task(c, s, a, b):
+    """Ensure an exception in a single task does not tear down an entire batch of gather_dep
+
+
+    See also https://github.com/dask/distributed/issues/5152
+    See also test_gather_dep_exception_one_task_2
+    """
+    fut = c.submit(inc, 1, workers=[a.address], key="f1")
+    fut2 = c.submit(inc, 2, workers=[a.address], key="f2")
+    fut3 = c.submit(inc, 3, workers=[a.address], key="f3")
+
+    import asyncio
+
+    event = asyncio.Event()
+    write_queue = asyncio.Queue()
+    event.clear()
+    b.rpc = _LockedCommPool(b.rpc, write_event=event, write_queue=write_queue)
+    b.rpc.remove(a.address)
+
+    def sink(a, b, *args):
+        return a + b
+
+    res1 = c.submit(sink, fut, fut2, fut3, workers=[b.address])
+    res2 = c.submit(sink, fut, fut2, workers=[b.address])
+
+    # Wait until we're sure the worker is attempting to fetch the data
+    while True:
+        peer_addr, msg = await write_queue.get()
+        if peer_addr == a.address and msg["op"] == "get_data":
+            break
+
+    # Provoke an "impossible transision exception"
+    # By choosing a state which doesn't exist we're not running into validation
+    # errors and the state machine should raise if we want to transition from
+    # fetch to memory
+
+    b.validate = False
+    b.tasks[fut3.key].state = "fetch"
+    event.set()
+
+    with captured_logger("distributed.worker", level=logging.DEBUG) as worker_logs:
+
+        # FIXME: We currently have no reliable, safe way to release the task and
+        # its dependent without race conditions
+
+        # Unfortunately res1 is deadlocking. IRL this is not always a problem
+        # since a commonly reported transition is Fetch->Memory, i.e. the task
+        # exists already in memory for whatever reason but a gather_dep was
+        # still runnign, e.g. the task was rescheduled on that worker and it was
+        # computed faster than fetched.
+
+        with suppress(TimeoutError):
+            await asyncio.wait_for(res1, 0.1)
+
+        assert await res2 == 5
+
+        del res1, res2, fut, fut2
+        fut3.release()
+
+        while a.tasks and b.tasks:
+            await asyncio.sleep(0.1)
+
+    expected_msg = (
+        "Exception occured while handling `gather_dep` response for <Task 'f3' fetch>"
+    )
+    assert expected_msg in worker_logs.getvalue()
+    assert any("except-gather-dep-result" in msg for msg in b.story(fut3.key))
+
+
+@gen_cluster(client=True)
+async def test_gather_dep_exception_one_task_2(c, s, a, b):
+    """Ensure an exception in a single task does not tear down an entire batch of gather_dep
+
+    The below triggers an fetch->memory transition
+
+    See also https://github.com/dask/distributed/issues/5152
+    See also test_gather_dep_exception_one_task
+    """
+    # This test does not trigger the condition reliably but is a very easy case
+    # which should function correctly regardles
+
+    fut1 = c.submit(inc, 1, workers=[a.address], key="f1")
+    fut2 = c.submit(inc, fut1, workers=[b.address], key="f2")
+
+    while fut1.key not in b.tasks or b.tasks[fut1.key].state == "flight":
+        await asyncio.sleep(0)
+
+    s.handle_missing_data(key="f1", errant_worker=a.address)
+
+    await fut2
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index 3072ff53d07..a9c5496efce 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -39,12 +39,14 @@
 
 import dask
 
+from distributed.comm.tcp import TCP
+
 from . import system
 from .client import Client, _global_clients, default_client
 from .comm import Comm
 from .compatibility import WINDOWS
 from .config import initialize_logging
-from .core import CommClosedError, Status, connect, rpc
+from .core import CommClosedError, ConnectionPool, Status, connect, rpc
 from .deploy import SpecCluster
 from .diagnostics.plugin import WorkerPlugin
 from .metrics import time
@@ -1616,3 +1618,77 @@ def transition(self, key, start, finish, **kwargs):
             ts.metadata["start_time"] = time()
         elif start == "executing" and finish == "memory":
             ts.metadata["stop_time"] = time()
+
+
+class LockedComm(TCP):
+    def __init__(self, comm, read_event, read_queue, write_event, write_queue):
+        self.write_event = write_event
+        self.write_queue = write_queue
+        self.read_event = read_event
+        self.read_queue = read_queue
+        self.comm = comm
+        assert isinstance(comm, TCP)
+
+    def __getattr__(self, name):
+        return getattr(self.comm, name)
+
+    async def write(self, msg, serializers=None, on_error="message"):
+        if self.write_queue:
+            await self.write_queue.put((self.comm.peer_address, msg))
+        if self.write_event:
+            await self.write_event.wait()
+        return await self.comm.write(msg, serializers=serializers, on_error=on_error)
+
+    async def read(self, deserializers=None):
+        msg = await self.comm.read(deserializers=deserializers)
+        if self.read_queue:
+            await self.read_queue.put((self.comm.peer_address, msg))
+        if self.read_event:
+            await self.read_event.wait()
+        return msg
+
+
+class _LockedCommPool(ConnectionPool):
+    """A ConnectionPool wrapper to intercept network traffic between servers
+
+    This wrapper can be attached to a running server to intercept outgoing read or write requests in test environments.
+
+    Examples
+    --------
+    >>> w = await Worker(...)
+    >>> read_event = asyncio.Event()
+    >>> read_queue = asyncio.Queue()
+    >>> w.rpc = _LockedCommPool(
+            w.rpc,
+            read_event=read_event,
+            read_queue=read_queue,
+        )
+    # It might be necessary to remove all existing comms
+    # if the wrapped pool has been used before
+    >>> w.remove(remote_address)
+
+    >>> async def ping_pong():
+            return await w.rpc(remote_address).ping()
+    >>> with pytest.raises(asyncio.TimeoutError):
+    >>>     await asyncio.wait_for(ping_pong(), 0.01)
+    >>> read_event.set()
+    >>> await ping_pong()
+    """
+
+    def __init__(
+        self, pool, read_event=None, read_queue=None, write_event=None, write_queue=None
+    ):
+        self.write_event = write_event
+        self.write_queue = write_queue
+        self.read_event = read_event
+        self.read_queue = read_queue
+        self.pool = pool
+
+    def __getattr__(self, name):
+        return getattr(self.pool, name)
+
+    async def connect(self, *args, **kwargs):
+        comm = await self.pool.connect(*args, **kwargs)
+        return LockedComm(
+            comm, self.read_event, self.read_queue, self.write_event, self.write_queue
+        )
diff --git a/distributed/worker.py b/distributed/worker.py
index 4ac7e7ec24f..8aa08c71675 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -2443,38 +2443,56 @@ async def gather_dep(
                 assert set(to_gather_keys).issubset(
                     set(self.in_flight_workers.get(worker))
                 )
-                for d in self.in_flight_workers.pop(worker):
 
+                for d in self.in_flight_workers.pop(worker):
                     ts = self.tasks.get(d)
-
-                    if not busy and d in data:
-                        self.transition(ts, "memory", value=data[d])
-                    elif ts is None or ts.state == "executing":
-                        self.log.append(("already-executing", d))
-                        self.release_key(d, reason="already executing at gather")
-                    elif ts.state == "flight" and not ts.dependents:
-                        self.log.append(("flight no-dependents", d))
-                        self.release_key(
-                            d, reason="In-flight task no longer has dependents."
-                        )
-                    elif (
-                        not busy
-                        and d not in data
-                        and ts.dependents
-                        and ts.state != "memory"
-                    ):
-                        ts.who_has.discard(worker)
-                        self.has_what[worker].discard(ts.key)
-                        self.log.append(("missing-dep", d))
-                        self.batched_stream.send(
-                            {"op": "missing-data", "errant_worker": worker, "key": d}
+                    try:
+                        if not busy and d in data:
+                            self.transition(ts, "memory", value=data[d])
+                        elif ts is None or ts.state == "executing":
+                            self.log.append(("already-executing", d))
+                            self.release_key(d, reason="already executing at gather")
+                        elif ts.state == "flight" and not ts.dependents:
+                            self.log.append(("flight no-dependents", d))
+                            self.release_key(
+                                d, reason="In-flight task no longer has dependents."
+                            )
+                        elif (
+                            not busy
+                            and d not in data
+                            and ts.dependents
+                            and ts.state != "memory"
+                        ):
+                            ts.who_has.discard(worker)
+                            self.has_what[worker].discard(ts.key)
+                            self.log.append(("missing-dep", d))
+                            self.batched_stream.send(
+                                {
+                                    "op": "missing-data",
+                                    "errant_worker": worker,
+                                    "key": d,
+                                }
+                            )
+                            self.transition(ts, "fetch")
+                        elif ts.state not in ("ready", "memory"):
+                            self.transition(ts, "fetch")
+                        else:
+                            logger.debug(
+                                "Unexpected task state encountered for %r after gather_dep",
+                                ts,
+                            )
+                    except Exception as exc:
+                        emsg = error_message(exc)
+                        assert ts is not None, ts
+                        self.log.append(
+                            (ts.key, "except-gather-dep-result", emsg, time())
                         )
-                        self.transition(ts, "fetch")
-                    elif ts.state not in ("ready", "memory"):
-                        self.transition(ts, "fetch")
-                    else:
+                        # FIXME: We currently cannot release this task and its
+                        # dependent safely
                         logger.debug(
-                            "Unexpected task state encountered for %s after gather_dep"
+                            "Exception occured while handling `gather_dep` response for %r",
+                            ts,
+                            exc_info=True,
                         )
 
                 if self.validate:

From 9569e08f28df3b870048ce7ae8b44961973af6c1 Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Fri, 6 Aug 2021 15:25:47 +0100
Subject: [PATCH 1423/1550] Replace not not with bool() (#5182)

---
 distributed/deploy/spec.py | 2 +-
 distributed/scheduler.py   | 2 +-
 distributed/security.py    | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index aa3dc3f7b84..1052b5a1f1d 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -528,7 +528,7 @@ def new_worker_spec(self):
 
     @property
     def _supports_scaling(self):
-        return not not self.new_spec
+        return bool(self.new_spec)
 
     async def scale_down(self, workers):
         # We may have groups, if so, map worker addresses to job names
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 3025695b367..6a9702e43e4 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -5059,7 +5059,7 @@ def validate_waiting(self, key):
         assert ts not in parent._unrunnable
         for dts in ts._dependencies:
             # We are waiting on a dependency iff it's not stored
-            assert (not not dts._who_has) != (dts in ts._waiting_on)
+            assert bool(dts._who_has) != (dts in ts._waiting_on)
             assert ts in dts._waiters  # XXX even if dts._who_has?
 
     def validate_processing(self, key):
diff --git a/distributed/security.py b/distributed/security.py
index 4078590cf6b..c24bb2e2d7f 100644
--- a/distributed/security.py
+++ b/distributed/security.py
@@ -70,7 +70,7 @@ def __init__(self, require_encryption=None, **kwargs):
         if require_encryption is None:
             require_encryption = dask.config.get("distributed.comm.require-encryption")
         if require_encryption is None:
-            require_encryption = not not kwargs
+            require_encryption = bool(kwargs)
         self.require_encryption = require_encryption
         self._set_field(kwargs, "tls_ciphers", "distributed.comm.tls.ciphers")
         self._set_field(kwargs, "tls_ca_file", "distributed.comm.tls.ca-file")

From 0ff6aafe1c4631c649307ef18c983c5639f4bc39 Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Mon, 9 Aug 2021 17:37:37 +0200
Subject: [PATCH 1424/1550] Add a .git-ignore-revs file (#5187)

This file includes revisions which applied some large scale auto
fomatting and usually do not give insight into the overall history of
the code. This can be used to ignore commits when doing git blame
see also https://git-scm.com/docs/git-blame#Documentation/git-blame.txt---ignore-revs-fileltfilegt
---
 .git-blame-ignore-revs | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)
 create mode 100644 .git-blame-ignore-revs

diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs
new file mode 100644
index 00000000000..b41430633a4
--- /dev/null
+++ b/.git-blame-ignore-revs
@@ -0,0 +1,33 @@
+# Auto flake and pep8 (#1353)
+7d252089a27ede17e6c78cecde485f651c1a7bdb
+
+# Apply Black to standardize code styling (#2614)
+04ae4212cf6d1e513a4ea83666719c79b7e5867a
+
+# Update black (#2901)
+741ffb60b94b15d2f243fc4ad4a849df76c46092
+
+# Fixup black string normalization (#2929)
+cf10db7b6a4fd091c2e1385162e3d36ab59c8f6e
+
+726f65438815317bd6c430b983463cfdbe34712b
+# Use latest release of black (#3388)
+726f65438815317bd6c430b983463cfdbe34712b
+
+# Rerun `black` on the code base (#3444)
+9af811d8f9858c63b9586bcfb78ce2dec8f5d6b3
+
+# Update for black (#4081)
+44bf0b981039ef5e474fbc5ccc6cd5a98b42e5e4
+
+# Pin black pre-commit (#4533)
+fdeca218134dbfe6c2c46f947413c7d0e1d2acab
+
+# Add isort to pre-commit hooks, package resorting (#4647)
+20a55e91b52d8e51a62ba1b47ccc1ece07adb72e
+
+# Pyupgrade (#4741)
+5dc591bbdd4427fe49fe90338a34fc85ee35f2c9
+
+# Flake8 config cleanup (#4888)
+ee06a44cc4b43270b5ae6ee102481f3bbc9daf27

From b0eefbdc7cca948dc53a53f91fd06b587d18fdb8 Mon Sep 17 00:00:00 2001
From: Bryan Van de Ven <bryan@bokeh.org>
Date: Mon, 9 Aug 2021 12:32:18 -0700
Subject: [PATCH 1425/1550] Use kwarg for Theme filename (#5190)

There was a recent change on Bokeh branch-2.4 to make the json and filename args to Theme be kwarg-only, but this is incompatible with Dask current usage. I will revert the change in Bokeh to maintain current compatibility, but I am also submitting the change here to set the stage for eventually re-introducing kwarg-only at some point in the future.

Secondarily, I have questions about what to do about testing? This was not caught by the "downstream" tests that Bokeh runs. A simple no-op test that just performs the import would have caught this since the use of Theme is at module scope. It's probably a good idea to make sure all of the bokeh-related modules cleanly import. But all of the bokeh related tests still seem to be in the dask repo, not this one. Should those tests be moved here? Or should I open a separate PR against dask to add tests?

Edit: reverted on Bokeh side at bokeh/bokeh#11484
---
 distributed/dashboard/components/scheduler.py | 4 +++-
 distributed/dashboard/components/worker.py    | 4 +++-
 distributed/http/scheduler/missing_bokeh.py   | 6 +++---
 3 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index f7e8d53edea..4bcab54d9df 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -84,7 +84,9 @@
     )
 )
 
-BOKEH_THEME = Theme(os.path.join(os.path.dirname(__file__), "..", "theme.yaml"))
+BOKEH_THEME = Theme(
+    filename=os.path.join(os.path.dirname(__file__), "..", "theme.yaml")
+)
 TICKS_1024 = {"base": 1024, "mantissas": [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]}
 XLABEL_ORIENTATION = -math.pi / 9  # slanted downwards 20 degrees
 
diff --git a/distributed/dashboard/components/worker.py b/distributed/dashboard/components/worker.py
index d47e0a7e0d6..7263d2b6fa2 100644
--- a/distributed/dashboard/components/worker.py
+++ b/distributed/dashboard/components/worker.py
@@ -45,7 +45,9 @@
     )
 )
 
-BOKEH_THEME = Theme(os.path.join(os.path.dirname(__file__), "..", "theme.yaml"))
+BOKEH_THEME = Theme(
+    filename=os.path.join(os.path.dirname(__file__), "..", "theme.yaml")
+)
 
 template_variables = {"pages": ["status", "system", "profile", "crossfilter"]}
 
diff --git a/distributed/http/scheduler/missing_bokeh.py b/distributed/http/scheduler/missing_bokeh.py
index 917e79f610b..63fd58a2241 100644
--- a/distributed/http/scheduler/missing_bokeh.py
+++ b/distributed/http/scheduler/missing_bokeh.py
@@ -6,9 +6,9 @@ class MissingBokeh(RequestHandler):
     def get(self):
         with log_errors():
             self.write(
-                "<p>Dask needs bokeh >= 0.13.0 for the dashboard.</p>"
-                "<p>Install with conda: conda install bokeh>=0.13.0</p>"
-                "<p>Install with pip: pip install bokeh>=0.13.0</p>"
+                "<p>Dask needs bokeh >= 1.0 for the dashboard.</p>"
+                "<p>Install with conda: conda install bokeh>=1.0</p>"
+                "<p>Install with pip: pip install bokeh>=1.0</p>"
             )
 
 

From a742de4ad2479fff2260444f61dad81a137294f2 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Mon, 9 Aug 2021 20:18:43 -0400
Subject: [PATCH 1426/1550] Add text exceptions to the Scheduler (#5148)

Fixes https://github.com/dask/distributed/issues/5126
---
 distributed/core.py              | 10 ++++++++--
 distributed/scheduler.py         | 25 ++++++++++++++++++++++++-
 distributed/tests/test_client.py | 16 ++++++++++++++++
 distributed/tests/test_worker.py | 10 +++-------
 distributed/worker.py            | 16 ++++++++++++++++
 5 files changed, 67 insertions(+), 10 deletions(-)

diff --git a/distributed/core.py b/distributed/core.py
index aad0d706058..47fe40a13a5 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -666,7 +666,7 @@ async def send_recv(comm, reply=True, serializers=None, deserializers=None, **kw
             typ, exc, tb = clean_exception(**response)
             raise exc.with_traceback(tb)
         else:
-            raise Exception(response["text"])
+            raise Exception(response["exception_text"])
     return response
 
 
@@ -1179,7 +1179,13 @@ def error_message(e, status="error"):
     else:
         tb_result = protocol.to_serialize(tb)
 
-    return {"status": status, "exception": e4, "traceback": tb_result, "text": str(e2)}
+    return {
+        "status": status,
+        "exception": e4,
+        "traceback": tb_result,
+        "exception_text": repr(e2),
+        "traceback_text": "".join(traceback.format_tb(tb)),
+    }
 
 
 def clean_exception(exception, traceback, **kwargs):
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 6a9702e43e4..8beafdbf88a 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1409,7 +1409,9 @@ class TaskState:
     _nbytes: Py_ssize_t
     _type: str
     _exception: object
+    _exception_text: str
     _traceback: object
+    _traceback_text: str
     _exception_blame: object
     _erred_on: set
     _suspicious: Py_ssize_t
@@ -1461,7 +1463,9 @@ class TaskState:
         # Which clients want us
         "_who_wants",
         "_exception",
+        "_exception_text",
         "_traceback",
+        "_traceback_text",
         "_erred_on",
         "_exception_blame",
         "_suspicious",
@@ -1480,6 +1484,7 @@ def __init__(self, key: str, run_spec: object):
         self._run_spec = run_spec
         self._state = None
         self._exception = self._traceback = self._exception_blame = None
+        self._exception_text = self._traceback_text = ""
         self._suspicious = self._retries = 0
         self._nbytes = -1
         self._priority = None
@@ -1597,10 +1602,18 @@ def type(self):
     def exception(self):
         return self._exception
 
+    @property
+    def exception_text(self):
+        return self._exception_text
+
     @property
     def traceback(self):
         return self._traceback
 
+    @property
+    def traceback_text(self):
+        return self._traceback_text
+
     @property
     def exception_blame(self):
         return self._exception_blame
@@ -3066,7 +3079,15 @@ def transition_processing_released(self, key):
             raise
 
     def transition_processing_erred(
-        self, key, cause=None, exception=None, traceback=None, worker=None, **kwargs
+        self,
+        key: str,
+        cause: str = None,
+        exception=None,
+        traceback=None,
+        exception_text: str = None,
+        traceback_text: str = None,
+        worker: str = None,
+        **kwargs,
     ):
         ws: WorkerState
         try:
@@ -3092,8 +3113,10 @@ def transition_processing_erred(
             ts._erred_on.add(w or worker)
             if exception is not None:
                 ts._exception = exception
+                ts._exception_text = exception_text
             if traceback is not None:
                 ts._traceback = traceback
+                ts._traceback_text = traceback_text
             if cause is not None:
                 failing_ts = self._tasks[cause]
                 ts._exception_blame = failing_ts
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 8c1e0645ee8..a65ab4494d8 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -6926,6 +6926,22 @@ def f():
     assert files == set(os.listdir())  # no change
 
 
+@gen_cluster(client=True)
+async def test_exception_text(c, s, a, b):
+    def bad(x):
+        raise Exception(x)
+
+    future = c.submit(bad, 123)
+    await wait(future)
+
+    ts = s.tasks[future.key]
+
+    assert isinstance(ts.exception_text, str)
+    assert "123" in ts.exception_text
+    assert "Exception(x)" in ts.traceback_text
+    assert "bad" in ts.traceback_text
+
+
 @gen_cluster(client=True)
 async def test_async_task(c, s, a, b):
     async def f(x):
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index a22903db371..6cee3388f21 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -342,18 +342,14 @@ def __str__(self):
     max_error_len = 100
     with dask.config.set({"distributed.admin.max-error-length": max_error_len}):
         msg = error_message(RuntimeError("-" * max_error_len))
-        assert len(msg["text"]) <= max_error_len
-        assert len(msg["text"]) < max_error_len * 2
+        assert len(msg["exception_text"]) <= max_error_len + 30
+        assert len(msg["exception_text"]) < max_error_len * 2
         msg = error_message(RuntimeError("-" * max_error_len * 20))
-        cut_text = msg["text"].replace("('Long error message', '", "")[:-2]
-        assert len(cut_text) == max_error_len
 
     max_error_len = 1000000
     with dask.config.set({"distributed.admin.max-error-length": max_error_len}):
         msg = error_message(RuntimeError("-" * max_error_len * 2))
-        cut_text = msg["text"].replace("('Long error message', '", "")[:-2]
-        assert len(cut_text) == max_error_len
-        assert len(msg["text"]) > 10100  # default + 100
+        assert len(msg["exception_text"]) > 10100  # default + 100
 
 
 @gen_cluster(client=True)
diff --git a/distributed/worker.py b/distributed/worker.py
index 8aa08c71675..39e77a8bb92 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -170,7 +170,9 @@ def __init__(self, key, runspec=None):
         self.waiting_for_data = set()
         self.resource_restrictions = None
         self.exception = None
+        self.exception_text = ""
         self.traceback = None
+        self.traceback_text = ""
         self.type = None
         self.suspicious_count = 0
         self.startstops = list()
@@ -1607,7 +1609,9 @@ def add_task(
                     return
                 if ts.state == "error":
                     ts.exception = None
+                    ts.exception_text = ""
                     ts.traceback = None
+                    ts.traceback_text = ""
                 else:
                     # This is a scheduler re-assignment
                     # Either `fetch` -> `waiting` or `flight` -> `waiting`
@@ -1967,6 +1971,8 @@ def transition_ready_error(self, ts):
         if self.validate:
             assert ts.exception is not None
             assert ts.traceback is not None
+            assert ts.exception_text
+            assert ts.traceback_text
         self.send_task_state_to_scheduler(ts)
 
     def transition_ready_memory(self, ts, value=no_value):
@@ -2007,7 +2013,9 @@ def transition_executing_done(self, ts, value=no_value, report=True):
                     logger.info("Failed to put key in memory", exc_info=True)
                     msg = error_message(e)
                     ts.exception = msg["exception"]
+                    ts.exception_text = msg["exception_text"]
                     ts.traceback = msg["traceback"]
+                    ts.traceback_text = msg["traceback_text"]
                     ts.state = "error"
                     out = "error"
                     for d in ts.dependents:
@@ -2233,6 +2241,8 @@ def send_task_state_to_scheduler(self, ts):
                 "thread": self.threads.get(ts.key),
                 "exception": ts.exception,
                 "traceback": ts.traceback,
+                "exception_text": ts.exception_text,
+                "traceback_text": ts.traceback_text,
             }
         else:
             logger.error("Key not ready to send to worker, %s: %s", ts.key, ts.state)
@@ -2519,6 +2529,8 @@ def bad_dep(self, dep):
             msg = error_message(exc)
             ts.exception = msg["exception"]
             ts.traceback = msg["traceback"]
+            ts.exception_text = msg["exception_text"]
+            ts.traceback_text = msg["traceback_text"]
             self.transition(ts, "error")
         self.release_key(dep.key, reason="bad dep")
 
@@ -2999,6 +3011,8 @@ async def execute(self, key):
             else:
                 ts.exception = result["exception"]
                 ts.traceback = result["traceback"]
+                ts.exception_text = result["exception_text"]
+                ts.traceback_text = result["traceback_text"]
                 logger.warning(
                     "Compute Failed\n"
                     "Function:  %s\n"
@@ -3025,6 +3039,8 @@ async def execute(self, key):
             emsg = error_message(exc)
             ts.exception = emsg["exception"]
             ts.traceback = emsg["traceback"]
+            ts.exception_text = emsg["exception_text"]
+            ts.traceback_text = emsg["traceback_text"]
             self.transition(ts, "error")
         finally:
             self.ensure_computing()

From bd9a68ca4af0e96e9226dab232cd3e0598cc3b6b Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 10 Aug 2021 10:19:48 -0400
Subject: [PATCH 1427/1550] avoid recursion error in profile.merge (#5195)

---
 distributed/profile.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/distributed/profile.py b/distributed/profile.py
index 958e342754a..f0535ef0da8 100644
--- a/distributed/profile.py
+++ b/distributed/profile.py
@@ -139,7 +139,10 @@ def merge(*args):
         for child in arg["children"]:
             children[child].append(arg["children"][child])
 
-    children = {k: merge(*v) for k, v in children.items()}
+    try:
+        children = {k: merge(*v) for k, v in children.items()}
+    except RecursionError:
+        children = {}
     count = sum(arg["count"] for arg in args)
     return {
         "description": args[0]["description"],

From ac55f25d230c7144ed618d1d4374f254303a4e0a Mon Sep 17 00:00:00 2001
From: Marcos Moyano <marcos@anue.biz>
Date: Tue, 10 Aug 2021 15:39:15 -0300
Subject: [PATCH 1428/1550] Download preload urls in the Preload constructor 
 (#5194)

Turn _download_module into a sync function.

This allows for modules to be initialized while creating the plugin.
---
 distributed/preloading.py         |  20 +++--
 distributed/tests/test_preload.py | 128 +++++++++++++++++++++++++-----
 2 files changed, 119 insertions(+), 29 deletions(-)

diff --git a/distributed/preloading.py b/distributed/preloading.py
index 40390df613b..0b1a4fbb8cf 100644
--- a/distributed/preloading.py
+++ b/distributed/preloading.py
@@ -9,7 +9,7 @@
 from typing import List
 
 import click
-from tornado.httpclient import AsyncHTTPClient
+import urllib3
 
 from dask.utils import tmpfile
 
@@ -119,13 +119,14 @@ def _import_module(name, file_dir=None) -> ModuleType:
     return module
 
 
-async def _download_module(url: str) -> ModuleType:
+def _download_module(url: str) -> ModuleType:
     logger.info("Downloading preload at %s", url)
     assert is_webaddress(url)
 
-    client = AsyncHTTPClient()
-    response = await client.fetch(url)
-    source = response.body.decode()
+    client = urllib3.PoolManager()
+
+    response = client.request("GET", url)
+    source = response.data.decode()
 
     compiled = compile(source, url, "exec")
     module = ModuleType(url)
@@ -155,16 +156,13 @@ def __init__(self, dask_server, name: str, argv: List[str], file_dir: str):
         self.argv = argv
         self.file_dir = file_dir
 
-        if not is_webaddress(name):
-            self.module = _import_module(name, file_dir)
+        if is_webaddress(name):
+            self.module = _download_module(name)
         else:
-            self.module = None
+            self.module = _import_module(name, file_dir)
 
     async def start(self):
         """Run when the server finishes its start method"""
-        if is_webaddress(self.name):
-            self.module = await _download_module(self.name)
-
         dask_setup = getattr(self.module, "dask_setup", None)
 
         if dask_setup:
diff --git a/distributed/tests/test_preload.py b/distributed/tests/test_preload.py
index 1fc74ee346c..7281bb17a2c 100644
--- a/distributed/tests/test_preload.py
+++ b/distributed/tests/test_preload.py
@@ -1,16 +1,23 @@
+import multiprocessing
 import os
 import shutil
 import sys
 import tempfile
+import time
 
 import pytest
+import tornado
+import urllib3
 from tornado import web
 
 import dask
 
 from distributed import Client, Nanny, Scheduler, Worker
+from distributed.compatibility import MACOS
 from distributed.utils_test import captured_logger, cluster, gen_cluster, gen_test
 
+PY_VERSION = sys.version_info[:2]
+
 PRELOAD_TEXT = """
 _worker_info = {}
 
@@ -151,29 +158,61 @@ async def test_preload_import_time(cleanup):
         del backends["foo"]
 
 
-@pytest.mark.asyncio
-async def test_web_preload(cleanup):
-    class MyHandler(web.RequestHandler):
-        def get(self):
-            self.write(
-                """
+class MyHandler(web.RequestHandler):
+    def get(self):
+        self.write(
+            """
 def dask_setup(dask_server):
     dask_server.foo = 1
 """.strip()
-            )
+        )
+
 
+def create_preload_application():
     app = web.Application([(r"/preload", MyHandler)])
-    server = app.listen(12345)
-    try:
-        with captured_logger("distributed.preloading") as log:
-            async with Scheduler(
-                dashboard_address=":0",
-                preload=["http://localhost:12345/preload"],
-            ) as s:
-                assert s.foo == 1
-        assert "12345/preload" in log.getvalue()
-    finally:
-        server.stop()
+    server = app.listen(12345, address="127.0.0.1")
+    tornado.ioloop.IOLoop.instance().start()
+
+
+@pytest.fixture
+def scheduler_preload():
+    p = multiprocessing.Process(target=create_preload_application)
+    p.start()
+    start = time.time()
+    while not p.is_alive():
+        if time.time() > start + 5:
+            raise AssertionError("Process didn't come up")
+        time.sleep(0.5)
+    # Make sure we can query the server
+    client = urllib3.PoolManager()
+    start = time.time()
+    while True:
+        try:
+            response = client.request("GET", "http://127.0.0.1:12345/preload")
+            if response.status == 200:
+                break
+        except urllib3.exceptions.HTTPError as e:
+            if time.time() > start + 10:
+                raise AssertionError("Webserver didn't come up", e)
+            time.sleep(0.5)
+
+    yield
+    p.kill()
+    p.join(timeout=5)
+
+
+@pytest.mark.skipif(
+    MACOS and PY_VERSION == (3, 7), reason="HTTP Server doesn't come up"
+)
+@pytest.mark.asyncio
+async def test_web_preload(cleanup, scheduler_preload):
+    with captured_logger("distributed.preloading") as log:
+        async with Scheduler(
+            host="localhost",
+            preload=["http://127.0.0.1:12345/preload"],
+        ) as s:
+            assert s.foo == 1
+    assert "12345/preload" in log.getvalue()
 
 
 @gen_cluster(nthreads=[])
@@ -194,3 +233,56 @@ async def test_scheduler_startup_nanny(s):
 """
     async with Nanny(preload_nanny=text) as w:
         assert w.scheduler.address == s.address
+
+
+class WorkerPreloadHandler(web.RequestHandler):
+    def get(self):
+        self.write(
+            """
+import dask
+dask.config.set(scheduler_address="tcp://127.0.0.1:8786")
+""".strip()
+        )
+
+
+def create_worker_preload_application():
+    application = web.Application([(r"/preload", WorkerPreloadHandler)])
+    server = application.listen(12346, address="127.0.0.1")
+    tornado.ioloop.IOLoop.instance().start()
+
+
+@pytest.fixture
+def worker_preload():
+    p = multiprocessing.Process(target=create_worker_preload_application)
+    p.start()
+    start = time.time()
+    while not p.is_alive():
+        if time.time() > start + 5:
+            raise AssertionError("Process didn't come up")
+        time.sleep(0.5)
+    # Make sure we can query the server
+    client = urllib3.PoolManager()
+    start = time.time()
+    while True:
+        try:
+            response = client.request("GET", "http://127.0.0.1:12346/preload")
+            if response.status == 200:
+                break
+        except urllib3.exceptions.HTTPError as e:
+            if time.time() > start + 10:
+                raise AssertionError("Webserver didn't come up", e)
+            time.sleep(0.5)
+
+    yield
+    p.kill()
+    p.join(timeout=5)
+
+
+@pytest.mark.skipif(
+    MACOS and PY_VERSION == (3, 7), reason="HTTP Server doesn't come up"
+)
+@pytest.mark.asyncio
+async def test_web_preload_worker(cleanup, worker_preload):
+    async with Scheduler(port=8786, host="localhost") as s:
+        async with Nanny(preload_nanny=["http://127.0.0.1:12346/preload"]) as nanny:
+            assert nanny.scheduler_addr == s.address

From cbe97d10289c889e95e678dd5265dd160e8f29c0 Mon Sep 17 00:00:00 2001
From: Marcos Moyano <marcos@anue.biz>
Date: Tue, 10 Aug 2021 21:33:57 -0300
Subject: [PATCH 1429/1550] Remove urllib3 as a dependency downloading preloads
 (#5199)

We accidentally introduced urllib3 as a dependency in #5194. This PR fixes that mistake.
---
 distributed/preloading.py         |  9 ++++-----
 distributed/tests/test_preload.py | 15 ++++++++-------
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/distributed/preloading.py b/distributed/preloading.py
index 0b1a4fbb8cf..1241ad1b630 100644
--- a/distributed/preloading.py
+++ b/distributed/preloading.py
@@ -4,12 +4,12 @@
 import os
 import shutil
 import sys
+import urllib.request
 from importlib import import_module
 from types import ModuleType
 from typing import List
 
 import click
-import urllib3
 
 from dask.utils import tmpfile
 
@@ -123,10 +123,9 @@ def _download_module(url: str) -> ModuleType:
     logger.info("Downloading preload at %s", url)
     assert is_webaddress(url)
 
-    client = urllib3.PoolManager()
-
-    response = client.request("GET", url)
-    source = response.data.decode()
+    request = urllib.request.Request(url, method="GET")
+    response = urllib.request.urlopen(request)
+    source = response.read().decode()
 
     compiled = compile(source, url, "exec")
     module = ModuleType(url)
diff --git a/distributed/tests/test_preload.py b/distributed/tests/test_preload.py
index 7281bb17a2c..d893a1251f4 100644
--- a/distributed/tests/test_preload.py
+++ b/distributed/tests/test_preload.py
@@ -4,10 +4,11 @@
 import sys
 import tempfile
 import time
+import urllib.error
+import urllib.request
 
 import pytest
 import tornado
-import urllib3
 from tornado import web
 
 import dask
@@ -184,14 +185,14 @@ def scheduler_preload():
             raise AssertionError("Process didn't come up")
         time.sleep(0.5)
     # Make sure we can query the server
-    client = urllib3.PoolManager()
     start = time.time()
+    request = urllib.request.Request("http://127.0.0.1:12345/preload", method="GET")
     while True:
         try:
-            response = client.request("GET", "http://127.0.0.1:12345/preload")
+            response = urllib.request.urlopen(request)
             if response.status == 200:
                 break
-        except urllib3.exceptions.HTTPError as e:
+        except urllib.error.URLError as e:
             if time.time() > start + 10:
                 raise AssertionError("Webserver didn't come up", e)
             time.sleep(0.5)
@@ -261,14 +262,14 @@ def worker_preload():
             raise AssertionError("Process didn't come up")
         time.sleep(0.5)
     # Make sure we can query the server
-    client = urllib3.PoolManager()
+    request = urllib.request.Request("http://127.0.0.1:12346/preload", method="GET")
     start = time.time()
     while True:
         try:
-            response = client.request("GET", "http://127.0.0.1:12346/preload")
+            response = urllib.request.urlopen(request)
             if response.status == 200:
                 break
-        except urllib3.exceptions.HTTPError as e:
+        except urllib.error.URLError as e:
             if time.time() > start + 10:
                 raise AssertionError("Webserver didn't come up", e)
             time.sleep(0.5)

From 2801f1175fcbb3f364a25e4059573ac2621af5ef Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Wed, 11 Aug 2021 09:25:43 +0100
Subject: [PATCH 1430/1550] Fix test_process_executor_kills_process flakyness
 (#5183)

* Fix test_process_executor_kills_process flakiness

* Increase test grace time / add flaky

* fix regression on Windows

* tweak

* More informative failure
---
 distributed/cli/tests/test_dask_scheduler.py  |  1 +
 distributed/deploy/tests/test_adaptive.py     |  6 +--
 .../deploy/tests/test_slow_adaptive.py        |  5 ++-
 distributed/scheduler.py                      |  2 +-
 distributed/tests/test_worker.py              | 39 ++++++++++---------
 5 files changed, 29 insertions(+), 24 deletions(-)

diff --git a/distributed/cli/tests/test_dask_scheduler.py b/distributed/cli/tests/test_dask_scheduler.py
index a04e84d4370..a3efa41b97a 100644
--- a/distributed/cli/tests/test_dask_scheduler.py
+++ b/distributed/cli/tests/test_dask_scheduler.py
@@ -169,6 +169,7 @@ def test_interface(loop):
                 assert all("127.0.0.1" == d["host"] for d in info["workers"].values())
 
 
+@pytest.mark.flaky(reruns=10, reruns_delay=5)
 def test_pid_file(loop):
     def check_pidfile(proc, pidfile):
         start = time()
diff --git a/distributed/deploy/tests/test_adaptive.py b/distributed/deploy/tests/test_adaptive.py
index 739fa708656..6f8067c4d29 100644
--- a/distributed/deploy/tests/test_adaptive.py
+++ b/distributed/deploy/tests/test_adaptive.py
@@ -16,7 +16,7 @@
     Worker,
     wait,
 )
-from distributed.compatibility import WINDOWS
+from distributed.compatibility import LINUX, MACOS, WINDOWS
 from distributed.metrics import time
 from distributed.utils_test import async_wait_for, clean, gen_test, slowinc
 
@@ -301,8 +301,8 @@ def test_basic_no_loop(loop):
             loop.add_callback(loop.stop)
 
 
-@pytest.mark.flaky(condition=not WINDOWS, reruns=10, reruns_delay=5)
-@pytest.mark.xfail(condition=WINDOWS, reason="extremely flaky")
+@pytest.mark.flaky(condition=LINUX, reruns=10, reruns_delay=5)
+@pytest.mark.xfail(condition=MACOS or WINDOWS, reason="extremely flaky")
 @gen_test()
 async def test_target_duration():
     with dask.config.set(
diff --git a/distributed/deploy/tests/test_slow_adaptive.py b/distributed/deploy/tests/test_slow_adaptive.py
index 853a5c4b4dc..c6697d1fa75 100644
--- a/distributed/deploy/tests/test_slow_adaptive.py
+++ b/distributed/deploy/tests/test_slow_adaptive.py
@@ -42,13 +42,13 @@ async def test_startup(cleanup):
         scheduler=scheduler,
         workers={
             0: {"cls": Worker, "options": {}},
-            1: {"cls": SlowWorker, "options": {"delay": 5}},
+            1: {"cls": SlowWorker, "options": {"delay": 120}},
             2: {"cls": SlowWorker, "options": {"delay": 0}},
         },
         asynchronous=True,
     ) as cluster:
         assert len(cluster.workers) == len(cluster.worker_spec) == 3
-        assert time() < start + 5
+        assert time() < start + 60
         assert 0 <= len(cluster.scheduler_info["workers"]) <= 2
 
         async with Client(cluster, asynchronous=True) as client:
@@ -56,6 +56,7 @@ async def test_startup(cleanup):
 
 
 @pytest.mark.asyncio
+@pytest.mark.flaky(reruns=10, reruns_delay=5)
 async def test_scale_up_down(cleanup):
     start = time()
     async with SpecCluster(
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 8beafdbf88a..33dbe31374c 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -2229,7 +2229,7 @@ def _transition(self, key, finish: str, *args, **kwargs):
                 self._transition_counter += 1
                 recommendations, client_msgs, worker_msgs = a
             elif "released" not in start_finish:
-                assert not args and not kwargs
+                assert not args and not kwargs, (args, kwargs)
                 a_recs: dict
                 a_cmsgs: dict
                 a_wmsgs: dict
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 6cee3388f21..3c7eb70f028 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -2013,33 +2013,36 @@ def kill_process():
     import os
     import signal
 
-    os.kill(os.getpid(), signal.SIGTERM)
+    if WINDOWS:
+        # There's no SIGKILL on Windows
+        sig = signal.SIGTERM
+    else:
+        # With SIGTERM there may be several seconds worth of delay before the worker
+        # actually shuts down - particularly on slow CI. Use SIGKILL for instant
+        # termination.
+        sig = signal.SIGKILL
 
+    os.kill(os.getpid(), sig)
+    sleep(60)  # Cope with non-instantaneous termination
 
-@gen_cluster(client=True)
-async def test_process_executor_kills_process(c, s, a, b):
+
+@gen_cluster(nthreads=[("127.0.0.1", 1)], client=True)
+async def test_process_executor_kills_process(c, s, a):
     with ProcessPoolExecutor() as e:
         a.executors["processes"] = e
-        b.executors["processes"] = e
         with dask.annotate(executor="processes", retries=1):
             future = c.submit(kill_process)
 
-        with pytest.raises(
-            BrokenProcessPool,
-            match="A child process terminated abruptly, the process pool is not usable anymore",
-        ):
+        msg = "A child process terminated abruptly, the process pool is not usable anymore"
+        with pytest.raises(BrokenProcessPool, match=msg):
             await future
 
         with dask.annotate(executor="processes", retries=1):
             future = c.submit(inc, 1)
 
-        # FIXME: The processpool is now unusable and the worker is effectively
-        # dead
-        with pytest.raises(
-            BrokenProcessPool,
-            match="A child process terminated abruptly, the process pool is not usable anymore",
-        ):
-            assert await future == 2
+        # The process pool is now unusable and the worker is effectively dead
+        with pytest.raises(BrokenProcessPool, match=msg):
+            await future
 
 
 def raise_exc():
@@ -2388,11 +2391,11 @@ async def test_hold_on_to_replicas(c, s, *workers):
 
 @gen_cluster(client=True)
 async def test_worker_reconnects_mid_compute(c, s, a, b):
-    """
-    This test ensure that if a worker disconnects while computing a result, the scheduler will still accept the result.
+    """Ensure that, if a worker disconnects while computing a result, the scheduler will
+    still accept the result.
 
     There is also an edge case tested which ensures that the reconnect is
-    successful if a task is currently executing, see
+    successful if a task is currently executing; see
     https://github.com/dask/distributed/issues/5078
 
     See also distributed.tests.test_scheduler.py::test_gather_allow_worker_reconnect

From 0528d8dc56dd42ef0049ee8293d240acdf909bd7 Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Wed, 11 Aug 2021 16:37:05 +0100
Subject: [PATCH 1431/1550] Migrating HTML reprs to jinja2 (#5188)

---
 MANIFEST.in                                   |   1 +
 continuous_integration/environment-3.7.yaml   |   1 +
 continuous_integration/environment-3.8.yaml   |   1 +
 continuous_integration/environment-3.9.yaml   |   1 +
 distributed/client.py                         | 119 ++-------
 distributed/deploy/cluster.py                 |  52 +---
 distributed/deploy/local.py                   |  14 +-
 distributed/objects.py                        | 252 ++----------------
 distributed/scheduler.py                      |  95 ++-----
 distributed/tests/test_scheduler.py           |   1 -
 distributed/utils.py                          |  35 +--
 distributed/widgets/__init__.py               |   1 +
 distributed/widgets/templates/client.html.j2  |  47 ++++
 distributed/widgets/templates/cluster.html.j2 |  38 +++
 .../widgets/templates/computation.html.j2     |  42 +++
 distributed/widgets/templates/future.html.j2  |  14 +
 .../widgets/templates/has_what.html.j2        |  24 ++
 .../widgets/templates/local_cluster.html.j2   |   7 +
 distributed/widgets/templates/log.html.j2     |  11 +
 distributed/widgets/templates/logs.html.j2    |   6 +
 .../widgets/templates/scheduler.html.j2       |   4 +
 .../widgets/templates/scheduler_info.html.j2  | 139 ++++++++++
 .../widgets/templates/task_state.html.j2      |  13 +
 distributed/widgets/templates/who_has.html.j2 |  15 ++
 .../widgets/templates/worker_state.html.j2    |   4 +
 distributed/widgets/utils.py                  |  28 ++
 requirements.txt                              |   1 +
 27 files changed, 477 insertions(+), 489 deletions(-)
 create mode 100644 distributed/widgets/__init__.py
 create mode 100644 distributed/widgets/templates/client.html.j2
 create mode 100644 distributed/widgets/templates/cluster.html.j2
 create mode 100644 distributed/widgets/templates/computation.html.j2
 create mode 100644 distributed/widgets/templates/future.html.j2
 create mode 100644 distributed/widgets/templates/has_what.html.j2
 create mode 100644 distributed/widgets/templates/local_cluster.html.j2
 create mode 100644 distributed/widgets/templates/log.html.j2
 create mode 100644 distributed/widgets/templates/logs.html.j2
 create mode 100644 distributed/widgets/templates/scheduler.html.j2
 create mode 100644 distributed/widgets/templates/scheduler_info.html.j2
 create mode 100644 distributed/widgets/templates/task_state.html.j2
 create mode 100644 distributed/widgets/templates/who_has.html.j2
 create mode 100644 distributed/widgets/templates/worker_state.html.j2
 create mode 100644 distributed/widgets/utils.py

diff --git a/MANIFEST.in b/MANIFEST.in
index 8a959c45673..aa82328eb70 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,5 +1,6 @@
 recursive-include distributed *.py
 recursive-include distributed *.js
+recursive-include distributed *.j2
 recursive-include distributed *.coffee
 recursive-include distributed *.html
 recursive-include distributed *.css
diff --git a/continuous_integration/environment-3.7.yaml b/continuous_integration/environment-3.7.yaml
index bb7838fe205..25f1a30d518 100644
--- a/continuous_integration/environment-3.7.yaml
+++ b/continuous_integration/environment-3.7.yaml
@@ -15,6 +15,7 @@ dependencies:
   - h5py
   - ipykernel
   - ipywidgets
+  - jinja2
   - joblib
   - jupyter_client
   - msgpack-python
diff --git a/continuous_integration/environment-3.8.yaml b/continuous_integration/environment-3.8.yaml
index 118d4231e4b..1e593d49d12 100644
--- a/continuous_integration/environment-3.8.yaml
+++ b/continuous_integration/environment-3.8.yaml
@@ -14,6 +14,7 @@ dependencies:
   - h5py
   - ipykernel
   - ipywidgets
+  - jinja2
   - joblib
   - jupyter_client
   - msgpack-python
diff --git a/continuous_integration/environment-3.9.yaml b/continuous_integration/environment-3.9.yaml
index 0dcd972db56..accd7b6446c 100644
--- a/continuous_integration/environment-3.9.yaml
+++ b/continuous_integration/environment-3.9.yaml
@@ -15,6 +15,7 @@ dependencies:
   - h5py
   - ipykernel
   - ipywidgets
+  - jinja2
   - joblib  # overridden by git tip below
   - jupyter_client
   - lz4  # Only tested here
diff --git a/distributed/client.py b/distributed/client.py
index a5484f809bd..e7389220d42 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -2,7 +2,6 @@
 import atexit
 import copy
 import errno
-import html
 import inspect
 import json
 import logging
@@ -82,11 +81,11 @@
     TimeoutError,
     format_dashboard_link,
     has_keyword,
-    key_split,
     log_errors,
     no_default,
     sync,
     thread_state,
+    typename,
 )
 from .utils_comm import (
     WrappedKey,
@@ -96,6 +95,7 @@
     scatter_to_workers,
     unpack_remotedata,
 )
+from .widgets import get_template
 from .worker import get_client, get_worker, secede
 
 logger = logging.getLogger(__name__)
@@ -401,39 +401,18 @@ def __del__(self):
 
     def __repr__(self):
         if self.type:
-            try:
-                typ = self.type.__module__.split(".")[0] + "." + self.type.__name__
-            except AttributeError:
-                typ = str(self.type)
-            return f"<Future: {self.status}, type: {typ}, key: {self.key}>"
+            return (
+                f"<Future: {self.status}, type: {typename(self.type)}, key: {self.key}>"
+            )
         else:
             return f"<Future: {self.status}, key: {self.key}>"
 
     def _repr_html_(self):
-        text = "<b>Future: %s</b> " % html.escape(key_split(self.key))
-        text += (
-            '<font style="color: var(--jp-ui-font-color2, gray)">status: </font>'
-            '<font style="color: %(color)s">%(status)s</font>, '
-        ) % {
-            "status": self.status,
-            "color": "var(--jp-error-color0, red)"
-            if self.status == "error"
-            else "var(--jp-ui-font-color0, black)",
-        }
-        if self.type:
-            try:
-                typ = self.type.__module__.split(".")[0] + "." + self.type.__name__
-            except AttributeError:
-                typ = str(self.type)
-            text += (
-                '<font style="color: var(--jp-ui-font-color2, gray)">type: </font>%s, '
-                % typ
-            )
-        text += (
-            '<font style="color: var(--jp-ui-font-color2, gray)">key: </font>%s'
-            % html.escape(str(self.key))
+        return get_template("future.html.j2").render(
+            key=str(self.key),
+            type=typename(self.type),
+            status=self.status,
         )
-        return text
 
     def __await__(self):
         return self.result().__await__()
@@ -921,78 +900,14 @@ def __repr__(self):
     def _repr_html_(self):
         scheduler, info = self._get_scheduler_info()
 
-        if scheduler is None:
-            child_repr = """<p>No scheduler connected.</p>"""
-        elif self.cluster:
-            child_repr = f"""
-                <details>
-                <summary style="margin-bottom: 20px;"><h3 style="display: inline;">Cluster Info</h3></summary>
-                {self.cluster._repr_html_()}
-                </details>
-                """
-        else:
-            child_repr = f"""
-                <details>
-                <summary style="margin-bottom: 20px;"><h3 style="display: inline;">Scheduler Info</h3></summary>
-                {info._repr_html_()}
-                </details>
-                """
-
-        client_status = ""
-
-        if not self.cluster and not self.scheduler_file:
-            client_status += """
-                <tr>
-                    <td style="text-align: left;"><strong>Connection method:</strong> Direct</td>
-                    <td style="text-align: left;"></td>
-                </tr>
-                """
-
-        if self.cluster:
-            client_status += f"""
-                <tr>
-                    <td style="text-align: left;"><strong>Connection method:</strong> Cluster object</td>
-                    <td style="text-align: left;"><strong>Cluster type:</strong> {type(self.cluster).__name__}</td>
-                </tr>
-                """
-        elif self.scheduler_file:
-            client_status += f"""
-                <tr>
-                    <td style="text-align: left;"><strong>Connection method:</strong> Scheduler file</td>
-                    <td style="text-align: left;"><strong>Scheduler file:</strong> {self.scheduler_file}</td>
-                </tr>
-                """
-
-        if self.dashboard_link:
-            client_status += f"""
-                <tr>
-                    <td style="text-align: left;">
-                        <strong>Dashboard: </strong>
-                        <a href="{self.dashboard_link}">{self.dashboard_link}</a>
-                    </td>
-                    <td style="text-align: left;"></td>
-                </tr>
-                """
-
-        return f"""
-            <div>
-                <div style="
-                    width: 24px;
-                    height: 24px;
-                    background-color: #e1e1e1;
-                    border: 3px solid #9D9D9D;
-                    border-radius: 5px;
-                    position: absolute;"> </div>
-                <div style="margin-left: 48px;">
-                    <h3 style="margin-bottom: 0px;">Client</h3>
-                    <p style="color: #9D9D9D; margin-bottom: 0px;">{self.id}</p>
-                    <table style="width: 100%; text-align: left;">
-                    {client_status}
-                    </table>
-                    {child_repr}
-                </div>
-            </div>
-        """
+        return get_template("client.html.j2").render(
+            id=self.id,
+            scheduler=scheduler,
+            info=info,
+            cluster=self.cluster,
+            scheduler_file=self.scheduler_file,
+            dashboard_link=self.dashboard_link,
+        )
 
     def start(self, **kwargs):
         """Start scheduler running in separate thread"""
diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
index 37ddc31f39b..fcb54e1dcd5 100644
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -14,6 +14,7 @@
 from ..core import Status
 from ..objects import SchedulerInfo
 from ..utils import Log, Logs, format_dashboard_link, log_errors, sync, thread_state
+from ..widgets import get_template
 from .adaptive import Adaptive
 
 logger = logging.getLogger(__name__)
@@ -364,54 +365,19 @@ def update():
 
     def _repr_html_(self, cluster_status=None):
 
-        if not cluster_status:
-            cluster_status = ""
-
-        cluster_status += f"""
-            <tr>
-                <td style="text-align: left;">
-                    <strong>Dashboard:</strong> <a href="{self.dashboard_link}">{self.dashboard_link}</a>
-                </td>
-                <td style="text-align: left;"><strong>Workers:</strong> {len(self.scheduler_info["workers"])}</td>
-            </tr>
-            <tr>
-                <td style="text-align: left;">
-                    <strong>Total threads:</strong>
-                    {sum([w["nthreads"] for w in self.scheduler_info["workers"].values()])}
-                </td>
-                <td style="text-align: left;">
-                    <strong>Total memory:</strong>
-                    {format_bytes(sum([w["memory_limit"] for w in self.scheduler_info["workers"].values()]))}
-                </td>
-            </tr>
-        """
         try:
             scheduler_info_repr = self.scheduler_info._repr_html_()
         except AttributeError:
             scheduler_info_repr = "Scheduler not started yet."
 
-        return f"""
-            <div class="jp-RenderedHTMLCommon jp-RenderedHTML jp-mod-trusted jp-OutputArea-output">
-                <div style="
-                    width: 24px;
-                    height: 24px;
-                    background-color: #e1e1e1;
-                    border: 3px solid #9D9D9D;
-                    border-radius: 5px;
-                    position: absolute;"> </div>
-                <div style="margin-left: 48px;">
-                    <h3 style="margin-bottom: 0px; margin-top: 0px;">{type(self).__name__}</h3>
-                    <p style="color: #9D9D9D; margin-bottom: 0px;">{self.name}</p>
-                    <table style="width: 100%; text-align: left;">
-                    {cluster_status}
-                    </table>
-                    <details>
-                    <summary style="margin-bottom: 20px;"><h3 style="display: inline;">Scheduler Info</h3></summary>
-                    {scheduler_info_repr}
-                    </details>
-                </div>
-            </div>
-        """
+        return get_template("cluster.html.j2").render(
+            type=type(self).__name__,
+            name=self.name,
+            workers=self.scheduler_info["workers"],
+            dashboard_link=self.dashboard_link,
+            scheduler_info_repr=scheduler_info_repr,
+            cluster_status=cluster_status,
+        )
 
     def _ipython_display_(self, **kwargs):
         widget = self._widget()
diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index 0cf98ea3cd8..c736f932a7d 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -11,6 +11,7 @@
 from ..nanny import Nanny
 from ..scheduler import Scheduler
 from ..security import Security
+from ..widgets import get_template
 from ..worker import Worker, parse_memory_limit
 from .spec import SpecCluster
 from .utils import nprocesses_nthreads
@@ -249,14 +250,11 @@ def start_worker(self, *args, **kwargs):
         )
 
     def _repr_html_(self, cluster_status=None):
-        if cluster_status is None:
-            cluster_status = ""
-        cluster_status += f"""
-            <tr>
-                <td style="text-align: left;"><strong>Status:</strong> {self.status.name}</td>
-                <td style="text-align: left;"><strong>Using processes:</strong> {self.processes}</td>
-            </tr>
-        """
+        cluster_status = get_template("local_cluster.html.j2").render(
+            status=self.status.name,
+            processes=self.processes,
+            cluster_status=cluster_status,
+        )
         return super()._repr_html_(cluster_status=cluster_status)
 
 
diff --git a/distributed/objects.py b/distributed/objects.py
index 521e19f8316..837b5c92fe3 100644
--- a/distributed/objects.py
+++ b/distributed/objects.py
@@ -1,257 +1,43 @@
 """This file contains custom objects.
 These are mostly regular objects with more useful _repr_ and _repr_html_ methods."""
-import datetime
 from urllib.parse import urlparse
 
-from dask.utils import format_bytes, format_time_ago
-
 from distributed.utils import format_dashboard_link
+from distributed.widgets import get_environment, get_template
 
 
 class HasWhat(dict):
     """A dictionary of all workers and which keys that worker has."""
 
     def _repr_html_(self):
-        rows = ""
-
-        for worker, keys in sorted(self.items()):
-            summary = ""
-            for key in keys:
-                summary += f"""<tr><td>{key}</td></tr>"""
-
-            rows += f"""<tr>
-            <td>{worker}</td>
-            <td>{len(keys)}</td>
-            <td>
-                <details>
-                <summary style='display:list-item'>Expand</summary>
-                <table>
-                {summary}
-                </table>
-                </details>
-            </td>
-        </tr>"""
-
-        output = f"""
-        <table>
-        <tr>
-            <th>Worker</th>
-            <th>Key count</th>
-            <th>Key list</th>
-        </tr>
-        {rows}
-        </table>
-        """
-
-        return output
+        return get_template("has_what.html.j2").render(has_what=self)
 
 
 class WhoHas(dict):
     """A dictionary of all keys and which workers have that key."""
 
     def _repr_html_(self):
-        rows = ""
-
-        for title, keys in sorted(self.items()):
-            rows += f"""<tr>
-            <td>{title}</td>
-            <td>{len(keys)}</td>
-            <td>{", ".join(keys)}</td>
-        </tr>"""
-
-        output = f"""
-        <table>
-        <tr>
-            <th>Key</th>
-            <th>Copies</th>
-            <th>Workers</th>
-        </tr>
-        {rows}
-        </table>
-        """
-
-        return output
+        return get_template("who_has.html.j2").render(who_has=self)
 
 
 class SchedulerInfo(dict):
     """A dictionary of information about the scheduler and workers."""
 
     def _repr_html_(self):
-        dashboard_address = None
-        if "dashboard" in self["services"]:
-            host = urlparse(self["address"]).hostname
-            dashboard_address = format_dashboard_link(
-                host, self["services"]["dashboard"]
-            )
-
-        scheduler = f"""
-            <div>
-                <div style="
-                    width: 24px;
-                    height: 24px;
-                    background-color: #FFF7E5;
-                    border: 3px solid #FF6132;
-                    border-radius: 5px;
-                    position: absolute;"> </div>
-                <div style="margin-left: 48px;">
-                    <h3 style="margin-bottom: 0px;">{self["type"]}</h3>
-                    <p style="color: #9D9D9D; margin-bottom: 0px;">{self["id"]}</p>
-                    <table style="width: 100%; text-align: left;">
-                        <tr>
-                            <td style="text-align: left;"><strong>Comm:</strong> {self["address"]}</td>
-                            <td style="text-align: left;"><strong>Workers:</strong> {len(self["workers"])}</td>
-                        </tr>
-                        <tr>
-                            <td style="text-align: left;">
-                                <strong>Dashboard:</strong> <a href="{dashboard_address}">{dashboard_address}</a>
-                            </td>
-                            <td style="text-align: left;">
-                                <strong>Total threads:</strong>
-                                {sum([w["nthreads"] for w in self["workers"].values()])}
-                            </td>
-                        </tr>
-                        <tr>
-                            <td style="text-align: left;">
-                                <strong>Started:</strong>
-                                {format_time_ago(datetime.datetime.fromtimestamp(self["started"]))}
-                            </td>
-                            <td style="text-align: left;">
-                                <strong>Total memory:</strong>
-                                {format_bytes(sum([w["memory_limit"] for w in self["workers"].values()]))}
-                            </td>
-                        </tr>
-                    </table>
-                </div>
-            </div>
-        """
-
-        workers = ""
-        for worker_name in self["workers"]:
-            self["workers"][worker_name]["comm"] = worker_name
-        for worker in sorted(self["workers"].values(), key=lambda k: k["name"]):
-            dashboard_address = None
-            if "dashboard" in worker["services"]:
-                host = urlparse(worker["comm"]).hostname
-                dashboard_address = format_dashboard_link(
-                    host, worker["services"]["dashboard"]
+        def _format_dashboard_address(server):
+            try:
+                host = (
+                    server["host"]
+                    if "host" in server
+                    else urlparse(server["address"]).hostname
                 )
-
-            metrics = ""
-
-            if "metrics" in worker:
-                metrics = f"""
-                <tr>
-                    <td style="text-align: left;">
-                        <strong>Tasks executing: </strong> {worker["metrics"]["executing"]}
-                    </td>
-                    <td style="text-align: left;">
-                        <strong>Tasks in memory: </strong> {worker["metrics"]["in_memory"]}
-                    </td>
-                </tr>
-                <tr>
-                    <td style="text-align: left;">
-                        <strong>Tasks ready: </strong> {worker["metrics"]["ready"]}
-                    </td>
-                    <td style="text-align: left;">
-                        <strong>Tasks in flight: </strong>{worker["metrics"]["in_flight"]}
-                    </td>
-                </tr>
-                <tr>
-                    <td style="text-align: left;"><strong>CPU usage:</strong> {worker["metrics"]["cpu"]}%</td>
-                    <td style="text-align: left;">
-                        <strong>Last seen: </strong>
-                        {format_time_ago(datetime.datetime.fromtimestamp(worker["last_seen"]))}
-                    </td>
-                </tr>
-                <tr>
-                    <td style="text-align: left;">
-                        <strong>Memory usage: </strong>
-                        {format_bytes(worker["metrics"]["memory"])}
-                    </td>
-                    <td style="text-align: left;">
-                        <strong>Spilled bytes: </strong>
-                        {format_bytes(worker["metrics"]["spilled_nbytes"])}
-                    </td>
-                </tr>
-                <tr>
-                    <td style="text-align: left;">
-                        <strong>Read bytes: </strong>
-                        {format_bytes(worker["metrics"]["read_bytes"])}
-                    </td>
-                    <td style="text-align: left;">
-                        <strong>Write bytes: </strong>
-                        {format_bytes(worker["metrics"]["write_bytes"])}
-                    </td>
-                </tr>
-                """
-
-            gpu = ""
-
-            if "gpu" in worker:
-                gpu = f"""
-                <tr>
-                    <td style="text-align: left;">
-                        <strong>GPU: </strong>{worker["gpu"]["name"]}
-                    </td>
-                    <td style="text-align: left;">
-                        <strong>GPU memory: </strong>
-                        {format_bytes(worker["gpu"]["memory-total"])}
-                    </td>
-                </tr>
-                """
-
-            workers += f"""
-            <div style="margin-bottom: 20px;">
-                <div style="width: 24px;
-                            height: 24px;
-                            background-color: #DBF5FF;
-                            border: 3px solid #4CC9FF;
-                            border-radius: 5px;
-                            position: absolute;"> </div>
-                <div style="margin-left: 48px;">
-                <details>
-                    <summary>
-                        <h4 style="margin-bottom: 0px; display: inline;">{worker["type"]}: {worker["name"]}</h4>
-                    </summary>
-                    <table style="width: 100%; text-align: left;">
-                        <tr>
-                            <td style="text-align: left;"><strong>Comm: </strong> {worker["comm"]}</td>
-                            <td style="text-align: left;"><strong>Total threads: </strong> {worker["nthreads"]}</td>
-                        </tr>
-                        <tr>
-                            <td style="text-align: left;">
-                                <strong>Dashboard: </strong>
-                                <a href="{dashboard_address}">{dashboard_address}</a>
-                            </td>
-                            <td style="text-align: left;">
-                                <strong>Memory: </strong>
-                                {format_bytes(worker["memory_limit"])}
-                            </td>
-                        </tr>
-                        <tr>
-                            <td style="text-align: left;"><strong>Nanny: </strong> {worker["nanny"]}</td>
-                            <td style="text-align: left;"></td>
-                        </tr>
-                        <tr>
-                            <td colspan="2" style="text-align: left;">
-                                <strong>Local directory: </strong>
-                                {worker["local_directory"]}
-                            </td>
-                        </tr>
-                        {gpu}
-                        {metrics}
-                    </table>
-                </details>
-                </div>
-            </div>
-            """
-
-        return f"""
-        <div style="">
-            {scheduler}
-            <details style="margin-left: 48px;">
-            <summary style="margin-bottom: 20px;"><h3 style="display: inline;">Workers</h3></summary>
-            {workers}
-            </details>
-        </div>
-        """
+                return format_dashboard_link(host, server["services"]["dashboard"])
+            except KeyError:
+                return None
+
+        environment = get_environment()
+        environment.filters["format_dashboard_address"] = _format_dashboard_address
+        return environment.get_template("scheduler_info.html.j2").render(
+            scheduler=self,
+            **self,
+        )
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 33dbe31374c..ec17b96f5f6 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1,6 +1,5 @@
 import asyncio
 import heapq
-import html
 import inspect
 import itertools
 import json
@@ -82,6 +81,7 @@
 from .utils_comm import gather_from_workers, retry_operation, scatter_to_workers
 from .utils_perf import disable_gc_diagnosis, enable_gc_diagnosis
 from .variable import VariableExtension
+from .widgets import get_template
 
 try:
     from cython import compiled
@@ -733,13 +733,12 @@ def __repr__(self):
         )
 
     def _repr_html_(self):
-        text = (
-            f"<b>WorkerState: </b> {html.escape(self._address)} "
-            f'<font style="color: var(--jp-ui-font-color2, gray)">name: </font>{self.name} '
-            f'<font style="color: var(--jp-ui-font-color2, gray)">memory: </font>{len(self._has_what)} '
-            f'<font style="color: var(--jp-ui-font-color2, gray)">processing: </font>{len(self._processing)}'
+        return get_template("worker_state.html.j2").render(
+            address=self.address,
+            name=self.name,
+            has_what=self._has_what,
+            processing=self.processing,
         )
-        return text
 
     @ccall
     @exceptval(check=False)
@@ -825,53 +824,14 @@ def __repr__(self):
         )
 
     def _repr_html_(self):
-        text = f"""<b>Computation</b> {self._id}
-
-        <table>
-            <tr>
-                <td style="text-align: left;"><strong>Duration: </strong>{self.stop - self.start:.3f}</td>
-                <td style="text-align: left;"></td>
-            </tr>
-            <tr>
-                <td style="text-align: left;"><strong>Start: </strong>{self.start}</td>
-                <td style="text-align: left;"></td>
-            </tr>
-            <tr>
-                <td style="text-align: left;"><strong>Groups: </strong>{len(self.groups)}</td>
-                <td style="text-align: left;"></td>
-            </tr>
-            <tr>
-                <td style="text-align: left;"><strong>Tasks: </strong>
-                {", ".join(
-                    "%s: %d" % (k, v) for (k, v) in sorted(self.states.items()) if v
-                )}</td>
-                <td style="text-align: left;"></td>
-            </tr>
-        </table>
-
-        <details>
-        <summary style="margin-bottom": 20px><h4 style="display:inline">Code</h4></summary>
-        """
-
-        for ix, code in enumerate(self.code):
-            text += f"<h5>Code segment {ix + 1} / {len(self.code)}</h5>"
-            text += f"<code>{code}</code>"
-
-        text += """
-        </details>
-        <details>
-        <summary style="margin-bottom": 20px><h4 style="display:inline">Task Groups</h4></summary>
-        <ul>
-        """
-        for gr in self.groups:
-            text += f"""
-            <li> {gr._repr_html_()} </li>
-            """
-        text += """
-        </ul>
-        </details>
-        """
-        return text
+        return get_template("computation.html.j2").render(
+            id=self._id,
+            start=self.start,
+            stop=self.stop,
+            groups=self.groups,
+            states=self.states,
+            code=self.code,
+        )
 
 
 @final
@@ -1137,9 +1097,6 @@ def __repr__(self):
             + ">"
         )
 
-    def _repr_html_(self):
-        return repr(self)[1:-1]
-
     def __len__(self):
         return sum(self._states.values())
 
@@ -1695,16 +1652,11 @@ def __repr__(self):
         return f"<TaskState {self._key!r} {self._state}>"
 
     def _repr_html_(self):
-        color = (
-            "var(--jp-error-color0, red)"
-            if self._state == "erred"
-            else "var(--jp-ui-font-color0, black)"
+        return get_template("task_state.html.j2").render(
+            state=self._state,
+            nbytes=self._nbytes,
+            key=self._key,
         )
-        text = f'<b>TaskState: </b> <font style="color: {color}">{self._state} </font>'
-        if self._state == "memory":
-            text += f'<font style="color: var(--jp-ui-font-color2, gray)">nbytes: </font>{format_bytes(self._nbytes)} '
-        text += f'<font style="color: var(--jp-ui-font-color2, gray)">key: </font>{html.escape(self._key)}'
-        return text
 
     @ccall
     def validate(self):
@@ -3892,13 +3844,12 @@ def __repr__(self):
 
     def _repr_html_(self):
         parent: SchedulerState = cast(SchedulerState, self)
-        text = (
-            f"<b>Scheduler: </b>{html.escape(self.address)} "
-            f'<font color="gray">workers: </font>{len(parent._workers_dv)} '
-            f'<font color="gray">cores: </font>{parent._total_nthreads} '
-            f'<font color="gray">tasks: </font>{len(parent._tasks)}'
+        return get_template("scheduler.html.j2").render(
+            address=self.address,
+            workers=parent._workers_dv,
+            threads=parent._total_nthreads,
+            tasks=parent._tasks,
         )
-        return text
 
     def identity(self, comm=None):
         """Basic information about ourselves and our cluster"""
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index b8b51a27905..143c3f8a50b 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -3132,7 +3132,6 @@ async def test_computations(c, s, a, b):
     assert "sub" not in str(s.computations[0].groups)
 
     assert isinstance(repr(s.computations[1]), str)
-    assert "x + 1" in s.computations[1]._repr_html_()
 
     assert s.computations[1].stop == max(tg.stop for tg in s.task_groups.values())
 
diff --git a/distributed/utils.py b/distributed/utils.py
index 918d41c371f..6c3c8ca8a9f 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -1,6 +1,5 @@
 import asyncio
 import functools
-import html
 import importlib
 import inspect
 import json
@@ -1242,43 +1241,19 @@ def parse_ports(port):
 class Log(str):
     """A container for newline-delimited string of log entries"""
 
-    level_styles = {
-        "WARNING": "font-weight: bold; color: orange;",
-        "CRITICAL": "font-weight: bold; color: orangered;",
-        "ERROR": "font-weight: bold; color: crimson;",
-    }
-
     def _repr_html_(self):
-        logs_html = []
-        for message in self.split("\n"):
-            style = "font-family: monospace; margin: 0;"
-            for level in self.level_styles:
-                if level in message:
-                    style += self.level_styles[level]
-                    break
-
-            logs_html.append(
-                '<p style="{style}">{message}</p>'.format(
-                    style=html.escape(style),
-                    message=html.escape(message),
-                )
-            )
+        from .widgets import get_template  # Avoiding circular import
 
-        return "\n".join(logs_html)
+        return get_template("log.html.j2").render(log=self)
 
 
 class Logs(dict):
     """A container for a dict mapping names to strings of log entries"""
 
     def _repr_html_(self):
-        summaries = [
-            "<details>\n"
-            "<summary style='display:list-item'>{title}</summary>\n"
-            "{log}\n"
-            "</details>".format(title=title, log=log._repr_html_())
-            for title, log in sorted(self.items())
-        ]
-        return "\n".join(summaries)
+        from .widgets import get_template  # Avoiding circular import
+
+        return get_template("logs.html.j2").render(logs=self)
 
 
 def cli_keywords(d: dict, cls=None, cmd=None):
diff --git a/distributed/widgets/__init__.py b/distributed/widgets/__init__.py
new file mode 100644
index 00000000000..511cb30ea25
--- /dev/null
+++ b/distributed/widgets/__init__.py
@@ -0,0 +1 @@
+from .utils import get_environment, get_template
diff --git a/distributed/widgets/templates/client.html.j2 b/distributed/widgets/templates/client.html.j2
new file mode 100644
index 00000000000..19206feded9
--- /dev/null
+++ b/distributed/widgets/templates/client.html.j2
@@ -0,0 +1,47 @@
+<div>
+    <div style="width: 24px; height: 24px; background-color: #e1e1e1; border: 3px solid #9D9D9D; border-radius: 5px; position: absolute;"> </div>
+    <div style="margin-left: 48px;">
+        <h3 style="margin-bottom: 0px;">Client</h3>
+        <p style="color: #9D9D9D; margin-bottom: 0px;">{{ id }}</p>
+        <table style="width: 100%; text-align: left;">
+
+        <tr>
+        {% if not cluster and not scheduler_file %}
+            <td style="text-align: left;"><strong>Connection method:</strong> Direct</td>
+            <td style="text-align: left;"></td>
+        {% elif cluster %}
+            <td style="text-align: left;"><strong>Connection method:</strong> Cluster object</td>
+            <td style="text-align: left;"><strong>Cluster type:</strong> {{ cluster | type }}</td>
+        {% elif scheduler_file %}
+            <td style="text-align: left;"><strong>Connection method:</strong> Scheduler file</td>
+            <td style="text-align: left;"><strong>Scheduler file:</strong> {{ scheduler_file }}</td>
+        {% endif %}
+        </tr>
+
+        {% if dashboard_link %}
+            <tr>
+                <td style="text-align: left;">
+                    <strong>Dashboard: </strong> <a href="{{ dashboard_link }}">{{ dashboard_link }}</a>
+                </td>
+                <td style="text-align: left;"></td>
+            </tr>
+        {% endif %}
+
+        </table>
+
+        {% if scheduler is none %}
+            <p>No scheduler connected.</p>
+        {% elif cluster %}
+            <details>
+            <summary style="margin-bottom: 20px;"><h3 style="display: inline;">Cluster Info</h3></summary>
+            {{ cluster._repr_html_() }}
+            </details>
+        {% else %}
+            <details>
+            <summary style="margin-bottom: 20px;"><h3 style="display: inline;">Scheduler Info</h3></summary>
+            {{ info._repr_html_() }}
+            </details>
+        {% endif %}
+
+    </div>
+</div>
diff --git a/distributed/widgets/templates/cluster.html.j2 b/distributed/widgets/templates/cluster.html.j2
new file mode 100644
index 00000000000..c1c57c32996
--- /dev/null
+++ b/distributed/widgets/templates/cluster.html.j2
@@ -0,0 +1,38 @@
+<div class="jp-RenderedHTMLCommon jp-RenderedHTML jp-mod-trusted jp-OutputArea-output">
+    <div style="width: 24px; height: 24px; background-color: #e1e1e1; border: 3px solid #9D9D9D; border-radius: 5px; position: absolute;">
+    </div>
+    <div style="margin-left: 48px;">
+        <h3 style="margin-bottom: 0px; margin-top: 0px;">{{ type }}</h3>
+        <p style="color: #9D9D9D; margin-bottom: 0px;">{{ name }}</p>
+        <table style="width: 100%; text-align: left;">
+            <tr>
+                <td style="text-align: left;">
+                    <strong>Dashboard:</strong> <a href="{{ dashboard_link }}">{{ dashboard_link }}</a>
+                </td>
+                <td style="text-align: left;">
+                    <strong>Workers:</strong> {{ workers | length }}
+                </td>
+            </tr>
+            <tr>
+                <td style="text-align: left;">
+                    <strong>Total threads:</strong> {{ workers.values() | map(attribute='nthreads') | sum }}
+                </td>
+                <td style="text-align: left;">
+                    <strong>Total memory:</strong> {{ workers.values() | map(attribute='memory_limit') | sum | format_bytes }}
+                </td>
+            </tr>
+            {% if cluster_status %}
+            {{ cluster_status }}
+            {% endif %}
+        </table>
+
+        <details>
+            <summary style="margin-bottom: 20px;">
+                <h3 style="display: inline;">Scheduler Info</h3>
+            </summary>
+
+            {{ scheduler_info_repr }}
+
+        </details>
+    </div>
+</div>
diff --git a/distributed/widgets/templates/computation.html.j2 b/distributed/widgets/templates/computation.html.j2
new file mode 100644
index 00000000000..97a44b5c244
--- /dev/null
+++ b/distributed/widgets/templates/computation.html.j2
@@ -0,0 +1,42 @@
+<strong>Computation</strong> {{ id }}
+
+<table>
+    <tr>
+        <td style="text-align: left;"><strong>Duration: </strong>{{ '%0.3f' | format(stop - start) }}</td>
+        <td style="text-align: left;"></td>
+    </tr>
+    <tr>
+        <td style="text-align: left;"><strong>Start: </strong>{{ start }}</td>
+        <td style="text-align: left;"></td>
+    </tr>
+    <tr>
+        <td style="text-align: left;"><strong>Groups: </strong>{{ groups | length }}</td>
+        <td style="text-align: left;"></td>
+    </tr>
+    <tr>
+        <td style="text-align: left;"><strong>Tasks: </strong>
+        {% for k, v in states.items() if v %}
+            {{ k }}: {{ v }}
+            {{ ", " if not loop.last else "" }}
+        {% endfor %}
+        </td>
+        <td style="text-align: left;"></td>
+    </tr>
+</table>
+
+<details>
+<summary style="margin-bottom": 20px><h4 style="display:inline">Code</h4></summary>
+{% for segment in code %}
+<h5>Code segment {{ loop.index }} / {{ code | length }}</h5>
+<pre><code>{{ segment }}</code></pre>
+{% endfor %}
+</details>
+
+<details>
+<summary style="margin-bottom": 20px><h4 style="display:inline">Task Groups</h4></summary>
+<ul>
+{% for gr in groups %}
+    <li> {{ gr._repr_html_() }} </li>
+{% endfor %}
+</ul>
+</details>
diff --git a/distributed/widgets/templates/future.html.j2 b/distributed/widgets/templates/future.html.j2
new file mode 100644
index 00000000000..46e96b07097
--- /dev/null
+++ b/distributed/widgets/templates/future.html.j2
@@ -0,0 +1,14 @@
+<strong>Future: {{ key | key_split | html_escape }}</strong>
+<span style="color: var(--jp-ui-font-color2, gray)"> status: </span>
+
+{% if status == "error" %}
+<span style="color: var(--jp-error-color0, red)">{{ status }}</span>,
+{% else %}
+<span style="color: var(--jp-error-color0, black)">{{ status }}</span>,
+{% endif %}
+
+{% if type %}
+<span style="color: var(--jp-ui-font-color2, gray)"> type:</span> {{ type }},
+{% endif %}
+
+<span style="color: var(--jp-ui-font-color2, gray)"> key:</span> {{ key | html_escape }}
diff --git a/distributed/widgets/templates/has_what.html.j2 b/distributed/widgets/templates/has_what.html.j2
new file mode 100644
index 00000000000..0c3fdee6e01
--- /dev/null
+++ b/distributed/widgets/templates/has_what.html.j2
@@ -0,0 +1,24 @@
+<table>
+    <tr>
+        <th>Worker</th>
+        <th>Key count</th>
+        <th>Key list</th>
+    </tr>
+
+    {% for worker, keys in has_what.items() %}
+    <tr>
+        <td>{{ worker }}</td>
+        <td>{{ keys | length }}</td>
+        <td>
+            <details>
+            <summary style='display:list-item'>Expand</summary>
+            <table>
+            {% for key in keys %}
+                <tr><td>{{ key }}</td></tr>
+            {% endfor %}
+            </table>
+            </details>
+        </td>
+    </tr>
+    {% endfor %}
+</table>
diff --git a/distributed/widgets/templates/local_cluster.html.j2 b/distributed/widgets/templates/local_cluster.html.j2
new file mode 100644
index 00000000000..a0aaa9a94c3
--- /dev/null
+++ b/distributed/widgets/templates/local_cluster.html.j2
@@ -0,0 +1,7 @@
+<tr>
+    <td style="text-align: left;"><strong>Status:</strong> {{ status }}</td>
+    <td style="text-align: left;"><strong>Using processes:</strong> {{ processes }}</td>
+</tr>
+{% if cluster_status %}
+{{ cluster_status }}
+{% endif %}
diff --git a/distributed/widgets/templates/log.html.j2 b/distributed/widgets/templates/log.html.j2
new file mode 100644
index 00000000000..34286424eab
--- /dev/null
+++ b/distributed/widgets/templates/log.html.j2
@@ -0,0 +1,11 @@
+{% for message in log.split("\n") %}
+    {% if "WARNING" in message %}
+        <p style="font-family: monospace; margin: 0; font-weight: bold; color: orange;">{{ message | html_escape }}</p>
+    {% elif "CRITICAL" in message %}
+        <p style="font-family: monospace; margin: 0; font-weight: bold; color: orangered;">{{ message | html_escape }}</p>
+    {% elif "ERROR" in message %}
+        <p style="font-family: monospace; margin: 0; font-weight: bold; color: crimson;">{{ message | html_escape }}</p>
+    {% else %}
+        <p style="font-family: monospace; margin: 0;">{{ message | html_escape }}</p>
+    {% endif %}
+{% endfor %}
diff --git a/distributed/widgets/templates/logs.html.j2 b/distributed/widgets/templates/logs.html.j2
new file mode 100644
index 00000000000..d69f3818f71
--- /dev/null
+++ b/distributed/widgets/templates/logs.html.j2
@@ -0,0 +1,6 @@
+{% for title, log in logs.items() %}
+    <details>
+    <summary style='display:list-item'>{{ title }}</summary>
+    {{ log._repr_html_() }}
+    </details>
+{% endfor %}
diff --git a/distributed/widgets/templates/scheduler.html.j2 b/distributed/widgets/templates/scheduler.html.j2
new file mode 100644
index 00000000000..c9e01981f2b
--- /dev/null
+++ b/distributed/widgets/templates/scheduler.html.j2
@@ -0,0 +1,4 @@
+<strong>Scheduler: </strong>{{ address | html_escape }}
+<span style="color: var(--jp-ui-font-color2, gray)"> workers: </span>{{ workers | length }}
+<span style="color: var(--jp-ui-font-color2, gray)"> cores: </span>{{ threads }}
+<span style="color: var(--jp-ui-font-color2, gray)"> tasks: </span>{{ tasks | length }}
diff --git a/distributed/widgets/templates/scheduler_info.html.j2 b/distributed/widgets/templates/scheduler_info.html.j2
new file mode 100644
index 00000000000..a4766353c93
--- /dev/null
+++ b/distributed/widgets/templates/scheduler_info.html.j2
@@ -0,0 +1,139 @@
+<div style="">
+    <div>
+        <div style="width: 24px; height: 24px; background-color: #FFF7E5; border: 3px solid #FF6132; border-radius: 5px; position: absolute;"> </div>
+        <div style="margin-left: 48px;">
+            <h3 style="margin-bottom: 0px;">{{ type }}</h3>
+            <p style="color: #9D9D9D; margin-bottom: 0px;">{{ id }}</p>
+            <table style="width: 100%; text-align: left;">
+                <tr>
+                    <td style="text-align: left;">
+                        <strong>Comm:</strong> {{ address }}
+                    </td>
+                    <td style="text-align: left;">
+                        <strong>Workers:</strong> {{ workers | length }}
+                    </td>
+                </tr>
+                <tr>
+                    <td style="text-align: left;">
+                        <strong>Dashboard:</strong> <a href="{{ scheduler | format_dashboard_address }}">{{ scheduler | format_dashboard_address }}</a>
+                    </td>
+                    <td style="text-align: left;">
+                        <strong>Total threads:</strong> {{ workers.values() | map(attribute='nthreads') | sum }}
+                    </td>
+                </tr>
+                <tr>
+                    <td style="text-align: left;">
+                        <strong>Started:</strong> {{ started | datetime_from_timestamp | format_time_ago }}
+                    </td>
+                    <td style="text-align: left;">
+                        <strong>Total memory:</strong> {{ workers.values() | map(attribute='memory_limit') | sum | format_bytes }}
+                    </td>
+                </tr>
+            </table>
+        </div>
+    </div>
+
+    <details style="margin-left: 48px;">
+        <summary style="margin-bottom: 20px;">
+            <h3 style="display: inline;">Workers</h3>
+        </summary>
+
+        {% for worker_name, worker in workers.items()|sort(attribute='1.name') %}
+        <div style="margin-bottom: 20px;">
+            <div style="width: 24px; height: 24px; background-color: #DBF5FF; border: 3px solid #4CC9FF; border-radius: 5px; position: absolute;"> </div>
+            <div style="margin-left: 48px;">
+            <details>
+                <summary>
+                    <h4 style="margin-bottom: 0px; display: inline;">{{ worker["type"] }}: {{ worker["name"] }}</h4>
+                </summary>
+                <table style="width: 100%; text-align: left;">
+                    <tr>
+                        <td style="text-align: left;">
+                            <strong>Comm: </strong> {{ worker_name }}
+                        </td>
+                        <td style="text-align: left;">
+                            <strong>Total threads: </strong> {{ worker["nthreads"] }}
+                        </td>
+                    </tr>
+                    <tr>
+                        <td style="text-align: left;">
+                            <strong>Dashboard: </strong> <a href="{{ worker | format_dashboard_address }}">{{ worker | format_dashboard_address }}</a>
+                        </td>
+                        <td style="text-align: left;">
+                            <strong>Memory: </strong> {{ worker["memory_limit"] | format_bytes }}
+                        </td>
+                    </tr>
+                    <tr>
+                        <td style="text-align: left;">
+                            <strong>Nanny: </strong> {{ worker["nanny"] }}
+                        </td>
+                        <td style="text-align: left;"></td>
+                    </tr>
+                    <tr>
+                        <td colspan="2" style="text-align: left;">
+                            <strong>Local directory: </strong> {{ worker["local_directory"] }}
+                        </td>
+                    </tr>
+
+                    {% if "gpu" in worker %}
+                    <tr>
+                        <td style="text-align: left;">
+                            <strong>GPU: </strong>{{ worker["gpu"]["name"]}}
+                        </td>
+                        <td style="text-align: left;">
+                            <strong>GPU memory: </strong> {{ worker["gpu"]["memory-total"] | format_bytes }}
+                        </td>
+                    </tr>
+                    {% endif %}
+
+                    {% if "metrics" in worker %}
+                    <tr>
+                        <td style="text-align: left;">
+                            <strong>Tasks executing: </strong> {{ worker["metrics"]["executing"] }}
+                        </td>
+                        <td style="text-align: left;">
+                            <strong>Tasks in memory: </strong> {{ worker["metrics"]["in_memory"] }}
+                        </td>
+                    </tr>
+                    <tr>
+                        <td style="text-align: left;">
+                            <strong>Tasks ready: </strong> {{ worker["metrics"]["ready"] }}
+                        </td>
+                        <td style="text-align: left;">
+                            <strong>Tasks in flight: </strong>{{ worker["metrics"]["in_flight"] }}
+                        </td>
+                    </tr>
+                    <tr>
+                        <td style="text-align: left;">
+                            <strong>CPU usage:</strong> {{ worker["metrics"]["cpu"]}}%
+                        </td>
+                        <td style="text-align: left;">
+                            <strong>Last seen: </strong> {{ worker["last_seen"] | datetime_from_timestamp | format_time_ago }}
+                        </td>
+                    </tr>
+                    <tr>
+                        <td style="text-align: left;">
+                            <strong>Memory usage: </strong> {{ worker["metrics"]["memory"] | format_bytes}}
+                        </td>
+                        <td style="text-align: left;">
+                            <strong>Spilled bytes: </strong> {{ worker["metrics"]["spilled_nbytes"] | format_bytes }}
+                        </td>
+                    </tr>
+                    <tr>
+                        <td style="text-align: left;">
+                            <strong>Read bytes: </strong> {{ worker["metrics"]["read_bytes"] | format_bytes }}
+                        </td>
+                        <td style="text-align: left;">
+                            <strong>Write bytes: </strong> {{ worker["metrics"]["write_bytes"] | format_bytes }}
+                        </td>
+                    </tr>
+                    {% endif %}
+
+                </table>
+            </details>
+            </div>
+        </div>
+        {% endfor %}
+
+    </details>
+</div>
diff --git a/distributed/widgets/templates/task_state.html.j2 b/distributed/widgets/templates/task_state.html.j2
new file mode 100644
index 00000000000..2a7e0e51e50
--- /dev/null
+++ b/distributed/widgets/templates/task_state.html.j2
@@ -0,0 +1,13 @@
+<strong>TaskState: </strong>
+
+{% if state == "erred" %}
+<span style="color: var(--jp-error-color0, red)"> {{ state }} </span>
+{% else %}
+<span style="color: var(--jp-ui-font-color0, black)"> {{ state }} </span>
+{% endif %}
+
+{% if state == "memory" %}
+<span style="color: var(--jp-ui-font-color2, gray)">nbytes: </span>{{  nbytes | format_bytes }}
+{% endif %}
+
+<span style="color: var(--jp-ui-font-color2, gray)">key: </span>{{ key | html_escape }}
diff --git a/distributed/widgets/templates/who_has.html.j2 b/distributed/widgets/templates/who_has.html.j2
new file mode 100644
index 00000000000..7d13b7b5c72
--- /dev/null
+++ b/distributed/widgets/templates/who_has.html.j2
@@ -0,0 +1,15 @@
+<table>
+    <tr>
+        <th>Key</th>
+        <th>Copies</th>
+        <th>Workers</th>
+    </tr>
+
+    {% for title, keys in who_has.items() %}
+    <tr>
+        <td>{{ title }}</td>
+        <td>{{ keys | length }}</td>
+        <td>{{ keys | join(", ") }}</td>
+    </tr>
+    {% endfor %}
+</table>
diff --git a/distributed/widgets/templates/worker_state.html.j2 b/distributed/widgets/templates/worker_state.html.j2
new file mode 100644
index 00000000000..2646d0fa262
--- /dev/null
+++ b/distributed/widgets/templates/worker_state.html.j2
@@ -0,0 +1,4 @@
+<strong>WorkerState: </strong> {{ address | html_escape }}
+<span style="color: var(--jp-ui-font-color2, gray)"> name: </span>{{ name }}
+<span style="color: var(--jp-ui-font-color2, gray)"> memory: </span>{{ has_what | length }}
+<span style="color: var(--jp-ui-font-color2, gray)"> processing: </span>{{ processing | length }}
diff --git a/distributed/widgets/utils.py b/distributed/widgets/utils.py
new file mode 100644
index 00000000000..3a5d2c15002
--- /dev/null
+++ b/distributed/widgets/utils.py
@@ -0,0 +1,28 @@
+import datetime
+import html
+import os.path
+
+from jinja2 import Environment, FileSystemLoader, Template
+
+from dask.utils import format_bytes, format_time_ago
+
+from ..utils import key_split
+
+
+def get_environment() -> Environment:
+    loader = FileSystemLoader(
+        [os.path.join(os.path.dirname(os.path.abspath(__file__)), "templates")]
+    )
+    environment = Environment(loader=loader)
+    environment.filters["format_bytes"] = format_bytes
+    environment.filters["format_time_ago"] = format_time_ago
+    environment.filters["datetime_from_timestamp"] = datetime.datetime.fromtimestamp
+    environment.filters["type"] = lambda cls: type(cls).__name__
+    environment.filters["key_split"] = key_split
+    environment.filters["html_escape"] = html.escape
+
+    return environment
+
+
+def get_template(name: str) -> Template:
+    return get_environment().get_template(name)
diff --git a/requirements.txt b/requirements.txt
index 4eafe78d78f..5812215bdb6 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,7 @@
 click >= 6.6
 cloudpickle >= 1.5.0
 dask == 2021.07.2
+jinja2
 msgpack >= 0.6.0
 psutil >= 5.0
 sortedcontainers !=2.0.0, !=2.0.1

From 22988613d261eb145e90a53f047785ede5558e5b Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Thu, 12 Aug 2021 15:28:28 +0200
Subject: [PATCH 1432/1550] Test nanny.environ precedence (#5204)

---
 distributed/nanny.py            | 23 +++++++++++++++++++++--
 distributed/tests/test_nanny.py | 29 +++++++++++++++++++++++++++++
 2 files changed, 50 insertions(+), 2 deletions(-)

diff --git a/distributed/nanny.py b/distributed/nanny.py
index 3afbcfd3819..65cb857174f 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -52,7 +52,21 @@ class Nanny(ServerNode):
     ``Client.restart`` method, or to restart the worker automatically if
     it gets to the terminate fractiom of its memory limit.
 
-    The parameters for the Nanny are mostly the same as those for the Worker.
+    The parameters for the Nanny are mostly the same as those for the Worker
+    with exceptions listed below.
+
+    Parameters
+    ----------
+    env: dict, optional
+        Environment variables set at time of Nanny initialization will be
+        ensured to be set in the Worker process as well. This argument allows to
+        overwrite or otherwise set environment variables for the Worker. It is
+        also possible to set environment variables using the option
+        `distributed.nanny.environ`. Precedence as follows
+
+            1. Nanny arguments
+            2. Existing environment variables
+            3. Dask configuration
 
     See Also
     --------
@@ -166,7 +180,12 @@ def __init__(
         self.death_timeout = parse_timedelta(death_timeout)
 
         self.Worker = Worker if worker_class is None else worker_class
-        self.env = dask.config.get("distributed.nanny.environ")
+        config_environ = dask.config.get("distributed.nanny.environ", {})
+        if not isinstance(config_environ, dict):
+            raise TypeError(
+                f"distributed.nanny.environ configuration must be of type dict. Instead got {type(config_environ)}"
+            )
+        self.env = config_environ.copy()
         for k in self.env:
             if k in os.environ:
                 self.env[k] = os.environ[k]
diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
index a5206e4f2e4..a7a4b6d1e2f 100644
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -6,6 +6,7 @@
 import random
 from contextlib import suppress
 from time import sleep
+from unittest import mock
 
 import psutil
 import pytest
@@ -375,6 +376,34 @@ async def test_environment_variable(c, s):
     await asyncio.gather(a.close(), b.close())
 
 
+@gen_cluster(nthreads=[], client=True)
+async def test_environment_variable_by_config(c, s, monkeypatch):
+
+    with dask.config.set({"distributed.nanny.environ": "456"}):
+        with pytest.raises(TypeError, match="configuration must be of type dict"):
+            Nanny(s.address, loop=s.loop, memory_limit=0)
+
+    with dask.config.set({"distributed.nanny.environ": {"FOO": "456"}}):
+
+        # precedence
+        # kwargs > env var > config
+
+        with mock.patch.dict(os.environ, {"FOO": "BAR"}, clear=True):
+            a = Nanny(s.address, loop=s.loop, memory_limit=0, env={"FOO": "123"})
+            x = Nanny(s.address, loop=s.loop, memory_limit=0)
+
+        b = Nanny(s.address, loop=s.loop, memory_limit=0)
+
+        await asyncio.gather(a, b, x)
+        results = await c.run(lambda: os.environ["FOO"])
+        assert results == {
+            a.worker_address: "123",
+            b.worker_address: "456",
+            x.worker_address: "BAR",
+        }
+        await asyncio.gather(a.close(), b.close(), x.close())
+
+
 @gen_cluster(
     nthreads=[],
     client=True,

From 17cd4db5a86d8d78b6a4ac15a6fa41ad4e609ff6 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Fri, 13 Aug 2021 05:18:31 -0500
Subject: [PATCH 1433/1550] Include addresses in closed comm repr  (#5203)

---
 distributed/comm/core.py             | 17 +++++++----------
 distributed/comm/tests/test_comms.py |  4 ++++
 2 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/distributed/comm/core.py b/distributed/comm/core.py
index 8c3e25cc159..1e8476a2539 100644
--- a/distributed/comm/core.py
+++ b/distributed/comm/core.py
@@ -155,16 +155,13 @@ def handshake_configuration(local, remote):
         return out
 
     def __repr__(self):
-        clsname = self.__class__.__name__
-        if self.closed():
-            return f"<closed {clsname}>"
-        else:
-            return "<{} {} local={} remote={}>".format(
-                clsname,
-                self.name or "",
-                self.local_address,
-                self.peer_address,
-            )
+        return "<{}{} {} local={} remote={}>".format(
+            self.__class__.__name__,
+            " (closed)" if self.closed() else "",
+            self.name or "",
+            self.local_address,
+            self.peer_address,
+        )
 
 
 class Listener(ABC):
diff --git a/distributed/comm/tests/test_comms.py b/distributed/comm/tests/test_comms.py
index 402149ac692..b83d44ccbba 100644
--- a/distributed/comm/tests/test_comms.py
+++ b/distributed/comm/tests/test_comms.py
@@ -1199,8 +1199,12 @@ async def check_repr(a, b):
     assert "closed" not in repr(b)
     await a.close()
     assert "closed" in repr(a)
+    assert a.local_address in repr(a)
+    assert b.peer_address in repr(a)
     await b.close()
     assert "closed" in repr(b)
+    assert a.local_address in repr(b)
+    assert b.peer_address in repr(b)
 
 
 @pytest.mark.asyncio

From bc91bcc12436b8a70753b297f51a9ec9d5dd40c8 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Fri, 13 Aug 2021 18:02:58 -0500
Subject: [PATCH 1434/1550] bump version to 2021.08.0

---
 docs/source/changelog.rst | 34 ++++++++++++++++++++++++++++++++++
 requirements.txt          |  2 +-
 2 files changed, 35 insertions(+), 1 deletion(-)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index 04ae7f2add4..61ee2999039 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,40 @@
 Changelog
 =========
 
+2021.08.0
+---------
+
+Released on August 13, 2021
+
+- Include addresses in closed comm repr  (:pr:`5203`) `James Bourbeau`_
+- Test ``nanny.environ`` precedence (:pr:`5204`) `Florian Jetter`_
+- Migrating HTML reprs to jinja2 (:pr:`5188`) `Jacob Tomlinson`_
+- Fix ``test_process_executor_kills_process`` flakyness (:pr:`5183`) `crusaderky`_
+- Remove ``urllib3`` as a dependency downloading preloads (:pr:`5199`) `Marcos Moyano`_
+- Download preload urls in the ``Preload`` constructor  (:pr:`5194`) `Marcos Moyano`_
+- Avoid recursion error in ``profile.merge`` (:pr:`5195`) `Matthew Rocklin`_
+- Add text exceptions to the ``Scheduler`` (:pr:`5148`) `Matthew Rocklin`_
+- Use ``kwarg`` for ``Theme`` filename (:pr:`5190`) `Bryan Van de Ven`_
+- Add a ``.git-ignore-revs`` file (:pr:`5187`) `Florian Jetter`_
+- Replace ``not not`` with ``bool()`` (:pr:`5182`) `Jacob Tomlinson`_
+- Resolve deadlock cause by transition error after fetching dependency (:pr:`5157`) `Florian Jetter`_
+- Set z-index of data-table lower (:pr:`5175`) `Julia Signell`_
+- Add ``no-worker`` - ``memory`` transition to scheduler (:pr:`5174`) `Florian Jetter`_
+- Deprecate worker plugin overwrite policy (:pr:`5146`) `James Bourbeau`_
+- Fix flaky tests in CI (:pr:`5168`) `crusaderky`_
+- Instructions for jemalloc with brew on macOS (:pr:`4996`) `Gabe Joseph`_
+- Bump ``RAPIDS_VER`` to 21.10 (:pr:`5165`) `Charles Blackmon-Luca`_
+- Tweak verbiage around ``async`` functions (:pr:`5166`) `crusaderky`_
+- Use Python 3 ``super()`` calls (:pr:`5167`) `crusaderky`_
+- Support asynchronous tasks (:pr:`5151`) `Matthew Rocklin`_
+- Rename total comm bytes and provide doc string (:pr:`5155`) `Florian Jetter`_
+- Add GPU executor if GPU is present (:pr:`5123`) `Matthew Rocklin`_
+- Fix RMM and UCX tests (:pr:`5158`) `Peter Andreas Entschev`_
+- Remove excessive timeout of ``test_steal_during_task_deserialization`` (:pr:`5156`) `Florian Jetter`_
+- Add gpuCI build script (:pr:`5147`) `Charles Blackmon-Luca`_
+- Demote ``Worker.ensure_computing`` to function (:pr:`5153`) `Florian Jetter`_
+
+
 2021.07.2
 ---------
 
diff --git a/requirements.txt b/requirements.txt
index 5812215bdb6..db69c79c6a6 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,6 @@
 click >= 6.6
 cloudpickle >= 1.5.0
-dask == 2021.07.2
+dask == 2021.08.0
 jinja2
 msgpack >= 0.6.0
 psutil >= 5.0

From db0b2352bed85306459d17643ba5bb79f7109754 Mon Sep 17 00:00:00 2001
From: Peter Andreas Entschev <peter@entschev.com>
Date: Mon, 16 Aug 2021 15:39:09 +0200
Subject: [PATCH 1435/1550] Drop RMM compatibility code from RAPIDS < 0.11
 (#5214)

Cleaning up old compatibility code from RAPIDS < 0.11, which hasn't been supported for over a year.
---
 distributed/comm/ucx.py | 12 +-----------
 1 file changed, 1 insertion(+), 11 deletions(-)

diff --git a/distributed/comm/ucx.py b/distributed/comm/ucx.py
index b98fb9e9475..33285711b7b 100644
--- a/distributed/comm/ucx.py
+++ b/distributed/comm/ucx.py
@@ -80,17 +80,7 @@ def init_once():
     try:
         import rmm
 
-        if hasattr(rmm, "DeviceBuffer"):
-            device_array = lambda n: rmm.DeviceBuffer(size=n)
-        else:  # pre-0.11.0
-            import numba.cuda
-
-            def rmm_device_array(n):
-                a = rmm.device_array(n, dtype="u1")
-                weakref.finalize(a, numba.cuda.current_context)
-                return a
-
-            device_array = rmm_device_array
+        device_array = lambda n: rmm.DeviceBuffer(size=n)
     except ImportError:
         try:
             import numba.cuda

From e50e35df06e8cd191edc3c3c30c60680dbe84c4a Mon Sep 17 00:00:00 2001
From: Ian Rose <ian.r.rose@gmail.com>
Date: Mon, 16 Aug 2021 19:54:52 +0200
Subject: [PATCH 1436/1550] Fix performance report sizing issue (#5213)

---
 distributed/dashboard/components/scheduler.py | 14 +++++++++++++-
 distributed/scheduler.py                      | 14 +++++++++++++-
 2 files changed, 26 insertions(+), 2 deletions(-)

diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index 4bcab54d9df..4cf03e85a9a 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -2765,7 +2765,19 @@ def __init__(self, scheduler):
             "\n".join(line for level, line in scheduler.get_logs())
         )._repr_html_()
 
-        self.root = Div(text=logs)
+        self.root = Div(
+            text=logs,
+            style={
+                "width": "100%",
+                "height": "100%",
+                "max-width": "1920px",
+                "max-height": "1080px",
+                "padding": "12px",
+                "border": "1px solid lightgray",
+                "box-shadow": "inset 1px 0 8px 0 lightgray",
+                "overflow": "auto",
+            },
+        )
 
 
 def systemmonitor_doc(scheduler, extra, doc):
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index ec17b96f5f6..1aedbd11d87 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -7346,7 +7346,19 @@ def profile_to_figure(state):
             dask_version=dask.__version__,
             distributed_version=distributed.__version__,
         )
-        html = Div(text=html)
+        html = Div(
+            text=html,
+            style={
+                "width": "100%",
+                "height": "100%",
+                "max-width": "1920px",
+                "max-height": "1080px",
+                "padding": "12px",
+                "border": "1px solid lightgray",
+                "box-shadow": "inset 1px 0 8px 0 lightgray",
+                "overflow": "auto",
+            },
+        )
 
         html = Panel(child=html, title="Summary")
         compute = Panel(child=compute, title="Worker Profile (compute)")

From 281438033f1e00d11ac043fea80758d607f480a8 Mon Sep 17 00:00:00 2001
From: Freyam Mehta <62539811+freyam@users.noreply.github.com>
Date: Tue, 17 Aug 2021 06:33:19 +0400
Subject: [PATCH 1437/1550] Add HTML Repr for `Security` Class (#5178)

HTML Representation of the Security class and also tweaked the __repr__ to specify where the keys/certs are stored.

Co-authored-by: Genevieve Buckley <30920819+GenevieveBuckley@users.noreply.github.com>
Co-authored-by: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
---
 distributed/security.py            | 49 +++++++++++++++++++++++++++---
 distributed/tests/test_security.py | 15 ++++++---
 2 files changed, 54 insertions(+), 10 deletions(-)

diff --git a/distributed/security.py b/distributed/security.py
index c24bb2e2d7f..02fd0a52276 100644
--- a/distributed/security.py
+++ b/distributed/security.py
@@ -153,18 +153,57 @@ def _set_field(self, kwargs, field, config_name):
             out = dask.config.get(config_name)
         setattr(self, field, out)
 
-    def __repr__(self):
+    def _attr_to_dict(self):
         keys = sorted(self.__slots__)
         keys.remove("extra_conn_args")
-        items = []
+
+        attr = {}
+
         for k in keys:
             val = getattr(self, k)
             if val is not None:
                 if isinstance(val, str) and "\n" in val:
-                    items.append((k, "..."))
+                    attr[k] = "Temporary (In-memory)"
+                elif isinstance(val, str):
+                    attr[k] = f"Local ({os.path.abspath(val)})"
                 else:
-                    items.append((k, repr(val)))
-        return "Security(" + ", ".join(f"{k}={v}" for k, v in items) + ")"
+                    attr[k] = val
+
+        return attr
+
+    def __repr__(self):
+        attr = self._attr_to_dict()
+        return (
+            "Security("
+            + ", ".join(f"{key}={value}" for key, value in attr.items())
+            + ")"
+        )
+
+    def _repr_html_(self):
+        attr = self._attr_to_dict()
+
+        rows = ""
+
+        for key, val in attr.items():
+            rows += f"""
+            <tr>
+                <th style="text-align: left; width: 150px;">{key}</th>
+                <td style="text-align: left;">{val}</td>
+            </tr>
+            """
+
+        html = f"""
+        <div style="margin-left: auto;">
+            <h3 style="margin-bottom: 0px;"><b>Security</b></h3>
+            <p>
+                <table style="width: 100%;">
+                {rows}
+                </table>
+            </p>
+        </div>
+        """
+
+        return html
 
     def get_tls_config_for_role(self, role):
         """
diff --git a/distributed/tests/test_security.py b/distributed/tests/test_security.py
index 305b8983752..9bb128ebd04 100644
--- a/distributed/tests/test_security.py
+++ b/distributed/tests/test_security.py
@@ -110,12 +110,17 @@ def test_kwargs():
     assert sec.extra_conn_args == {"headers": {"Auth": "Token abc"}}
 
 
-def test_repr():
+def test_repr_temp_keys():
+    sec = Security.temporary()
+    representation = repr(sec)
+    assert "Temporary (In-memory)" in representation
+
+
+def test_repr_local_keys():
     sec = Security(tls_ca_file="ca.pem", tls_scheduler_cert="scert.pem")
-    assert (
-        repr(sec)
-        == "Security(require_encryption=True, tls_ca_file='ca.pem', tls_scheduler_cert='scert.pem')"
-    )
+    representation = repr(sec)
+    assert "ca.pem" in representation
+    assert "scert.pem" in representation
 
 
 def test_tls_config_for_role():

From 91d50700df6dea7601fc5001011a9edfb2d7b416 Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Tue, 17 Aug 2021 14:16:19 +0200
Subject: [PATCH 1438/1550] Minor fixes for cfexecutor (#5177)

---
 distributed/cfexecutor.py                 | 5 +++++
 distributed/tests/test_client_executor.py | 3 +--
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/distributed/cfexecutor.py b/distributed/cfexecutor.py
index 8028a4bc7f2..fda2fe63fb3 100644
--- a/distributed/cfexecutor.py
+++ b/distributed/cfexecutor.py
@@ -128,6 +128,11 @@ def map(self, fn, *iterables, **kwargs):
 
         fs = self._client.map(fn, *iterables, **self._kwargs)
 
+        if isinstance(fs, list):
+            # Below iterator relies on this being a generator to cancel
+            # remaining futures
+            fs = (val for val in fs)
+
         # Yield must be hidden in closure so that the tasks are submitted
         # before the first iterator value is required.
         def result_iterator():
diff --git a/distributed/tests/test_client_executor.py b/distributed/tests/test_client_executor.py
index 8a50f5f27dc..0e088ee762d 100644
--- a/distributed/tests/test_client_executor.py
+++ b/distributed/tests/test_client_executor.py
@@ -12,7 +12,6 @@
 import pytest
 from tlz import take
 
-from distributed.compatibility import MACOS
 from distributed.utils import CancelledError
 from distributed.utils_test import inc, slowadd, slowdec, slowinc, throws, varying
 
@@ -127,7 +126,6 @@ def test_cancellation_as_completed(client):
         assert n_cancelled == 2
 
 
-@pytest.mark.flaky(condition=MACOS, reruns=10, reruns_delay=5)
 def test_map(client):
     with client.get_executor() as e:
         N = 10
@@ -156,6 +154,7 @@ def test_map(client):
         assert number_of_processing_tasks(client) > 0
         # Garbage collect the iterator => remaining tasks are cancelled
         del it
+        time.sleep(0.05)
         assert number_of_processing_tasks(client) == 0
 
 

From 6ac1d1330150a26904f01827f1896a0b38e322d1 Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Tue, 17 Aug 2021 17:15:35 +0200
Subject: [PATCH 1439/1550] Fix an issue where a reconnecting worker could
 cause an invalid transition (#5210)

If a worker reconnects with a key in memory which was already released by the scheduler, this would raise an invalid transition keyerror and would break the scheduler causing yet another deadlock.

There was also a weird issue about the `typename` function called on an instance instead of a type here
https://github.com/dask/distributed/blob/17cd4db5a86d8d78b6a4ac15a6fa41ad4e609ff6/distributed/worker.py#L922

would cause the type being submitted to be a `str` of the data object, unless it's a native object like an integer in which case it works as expected. I figured it is safer to let typename handle this situation. Very difficult to test for this particular position in the code which is why I only left another validation assert
---
 distributed/scheduler.py         | 58 ++++++++++++++++-------------
 distributed/tests/test_utils.py  | 10 +++++
 distributed/tests/test_worker.py | 64 ++++++++++++++++++++++++++++++++
 distributed/utils.py             |  2 +
 4 files changed, 109 insertions(+), 25 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 1aedbd11d87..ca743cdb443 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -4252,9 +4252,10 @@ async def add_worker(
             worker_msgs: dict = {}
             if nbytes:
                 assert isinstance(nbytes, dict)
+                already_released_keys = list()
                 for key in nbytes:
                     ts: TaskState = parent._tasks.get(key)
-                    if ts is not None:
+                    if ts is not None and ts.state != "released":
                         if ts.state == "memory":
                             self.add_keys(worker=address, keys=[key])
                         else:
@@ -4270,7 +4271,18 @@ async def add_worker(
                                 recommendations, client_msgs, worker_msgs
                             )
                             recommendations = {}
-
+                    else:
+                        already_released_keys.append(key)
+                if already_released_keys:
+                    if address not in worker_msgs:
+                        worker_msgs[address] = list()
+                    worker_msgs[address].append(
+                        {
+                            "op": "free-keys",
+                            "keys": already_released_keys,
+                            "reason": f"reconnect-already-released-{time()}",
+                        }
+                    )
             for ts in list(parent._unrunnable):
                 valid: set = self.valid_workers(ts)
                 if valid is None or ws in valid:
@@ -4679,37 +4691,33 @@ def stimulus_task_finished(self, key=None, worker=None, **kwargs):
         client_msgs: dict = {}
         worker_msgs: dict = {}
 
-        ts: TaskState = parent._tasks.get(key)
-        if ts is None:
-            return recommendations, client_msgs, worker_msgs
-
-        if ts.state == "memory":
-            self.add_keys(worker=worker, keys=[key])
-            return recommendations, client_msgs, worker_msgs
-
         ws: WorkerState = parent._workers_dv[worker]
-        ts._metadata.update(kwargs["metadata"])
-
-        if ts._state != "released":
-            r: tuple = parent._transition(key, "memory", worker=worker, **kwargs)
-            recommendations, client_msgs, worker_msgs = r
-
-            if ts._state == "memory":
-                assert ws in ts._who_has
-        else:
+        ts: TaskState = parent._tasks.get(key)
+        if ts is None or ts._state == "released":
             logger.debug(
                 "Received already computed task, worker: %s, state: %s"
                 ", key: %s, who_has: %s",
                 worker,
-                ts._state,
+                ts._state if ts else "forgotten",
                 key,
-                ts._who_has,
+                ts._who_has if ts else {},
             )
-            if ws not in ts._who_has:
-                worker_msgs[worker] = [
-                    {"op": "free-keys", "keys": [key], "reason": "Stimulus Finished"}
-                ]
+            worker_msgs[worker] = [
+                {
+                    "op": "free-keys",
+                    "keys": [key],
+                    "reason": f"already-released-or-forgotten-{time()}",
+                }
+            ]
+        elif ts.state == "memory":
+            self.add_keys(worker=worker, keys=[key])
+        else:
+            ts._metadata.update(kwargs["metadata"])
+            r: tuple = parent._transition(key, "memory", worker=worker, **kwargs)
+            recommendations, client_msgs, worker_msgs = r
 
+            if ts._state == "memory":
+                assert ws in ts._who_has
         return recommendations, client_msgs, worker_msgs
 
     def stimulus_task_erred(
diff --git a/distributed/tests/test_utils.py b/distributed/tests/test_utils.py
index 64802b5eb9d..285b8407000 100644
--- a/distributed/tests/test_utils.py
+++ b/distributed/tests/test_utils.py
@@ -40,6 +40,7 @@
     sync,
     thread_state,
     truncate_exception,
+    typename,
     warn_on_duration,
 )
 from distributed.utils_test import (
@@ -598,3 +599,12 @@ def test_parse_timedelta_deprecated():
 def test_iscoroutinefunction_unhashable_input():
     # Ensure iscoroutinefunction can handle unhashable callables
     assert not iscoroutinefunction(_UnhashableCallable())
+
+
+class MyType:
+    pass
+
+
+def test_typename_on_instances():
+    instance = MyType()
+    assert typename(instance) == typename(MyType)
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 3c7eb70f028..f1895a64895 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -2454,6 +2454,70 @@ def fast_on_a(lock):
         await asyncio.sleep(0.001)
 
 
+@gen_cluster(client=True)
+async def test_worker_reconnects_mid_compute_multiple_states_on_scheduler(c, s, a, b):
+    """
+    Ensure that a reconnecting worker does not break the scheduler regardless of
+    what state the keys of the worker are in when it connects back
+
+    See also test_worker_reconnects_mid_compute which uses a smaller chain of
+    tasks and does not release f1 in between
+    """
+
+    with captured_logger("distributed.scheduler") as s_logs:
+        # Let's put one task in memory to ensure the reconnect has tasks in
+        # different states
+        f1 = c.submit(inc, 1, workers=[a.address], allow_other_workers=True)
+        f2 = c.submit(inc, f1, workers=[a.address], allow_other_workers=True)
+        a_address = a.address
+
+        a.periodic_callbacks["heartbeat"].stop()
+        await a.heartbeat()
+        a.heartbeat_active = True
+
+        from distributed import Lock
+
+        def fast_on_a(lock):
+            w = get_worker()
+            import time
+
+            if w.address != a_address:
+                lock.acquire()
+            else:
+                time.sleep(1)
+
+        lock = Lock()
+        # We want to be sure that A is the only one computing this result
+        async with lock:
+
+            f3 = c.submit(
+                fast_on_a, lock, workers=[a.address], allow_other_workers=True
+            )
+
+            while f3.key not in a.tasks:
+                await asyncio.sleep(0.01)
+
+            await s.stream_comms[a.address].close()
+            f1.release()
+            assert len(s.workers) == 1
+            while s.tasks[f1.key].state != "released":
+                await asyncio.sleep(0)
+            a.heartbeat_active = False
+            await a.heartbeat()
+            assert len(s.workers) == 2
+            # Since B is locked, this is ensured to originate from A
+            await f3
+
+    assert "Unexpected worker completed task" in s_logs.getvalue()
+
+    while not len(s.tasks[f2.key].who_has) == 2:
+        await asyncio.sleep(0.001)
+
+    del f1, f2, f3
+    while any(w.tasks for w in [a, b]):
+        await asyncio.sleep(0.001)
+
+
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)])
 async def test_forget_dependents_after_release(c, s, a):
 
diff --git a/distributed/utils.py b/distributed/utils.py
index 6c3c8ca8a9f..22b1e9f718e 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -1160,6 +1160,8 @@ def typename(typ):
     >>> typename(Scheduler)
     'distributed.scheduler.Scheduler'
     """
+    if not isinstance(typ, type):
+        return typename(type(typ))
     try:
         return typ.__module__ + "." + typ.__name__
     except AttributeError:

From 3d8ed5fe8c095eead4301fd67be0d68f35df5ca8 Mon Sep 17 00:00:00 2001
From: Freyam Mehta <62539811+freyam@users.noreply.github.com>
Date: Tue, 17 Aug 2021 19:52:57 +0400
Subject: [PATCH 1440/1550] Add HTML Repr for `ProcessInterface` Class and all
 its subclasses (#5181)

* Basic HTML Repr

* colored box depending on status

* removed testing files

* swapped created and running colors

* added address and external_address

* added super class html repr for Process

* removed testing files

* added html repr for Scheduler and Worker

* accurate heading

* cleaner code and removed explicit headings

* removed redundant function

* removed redundant __repr__

* cleaner __repr__ status

* Update distributed/deploy/spec.py

Co-authored-by: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>

* used enum name propoerty to display status

* removed another redundant _repr_html_ :)

* remove redundant __repr__ as well :)

* removed testing files

* removed testing files (2)

Co-authored-by: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
---
 distributed/deploy/spec.py | 48 +++++++++++++++++++++++++++++++++++++-
 distributed/deploy/ssh.py  |  3 ---
 2 files changed, 47 insertions(+), 4 deletions(-)

diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index 1052b5a1f1d..99ad49cfc22 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -97,7 +97,53 @@ async def finished(self):
         await self._event_finished.wait()
 
     def __repr__(self):
-        return f"<{type(self).__name__}: status={self.status}>"
+        return f"<{dask.utils.typename(type(self))}: status={self.status.name}>"
+
+    def _repr_html_(self):
+        if self.status == Status.created:
+            status = "Created"
+            bg_color = "#caf0f8"
+            border_color = "#48cae4"
+        elif self.status == Status.running:
+            status = "Running"
+            bg_color = "#c7f9cc"
+            border_color = "#78c6a3"
+        elif self.status == Status.closed:
+            status = "Closed"
+            bg_color = "#ffbfad"
+            border_color = "#ff6132"
+
+        html = f"""
+          <div>
+            <div
+                style="
+                width: 24px;
+                height: 24px;
+                background-color: {bg_color};
+                border: 3px solid {border_color};
+                border-radius: 5px;
+                position: absolute;"
+            ></div>
+            <div style="margin-left: 48px">
+                <h3 style="margin-bottom: 0px">{dask.utils.typename(type(self))}</h3>
+                <p style="color: #9d9d9d; margin-bottom: 0px">Status: {status}</p>
+            </div>
+            <p>
+            <table style="width: 100%">
+                <tr>
+                <th style="text-align: left; width: 150px">Address</th>
+                <td style="text-align: left">{self.address}</td>
+                </tr>
+                <tr>
+                <th style="text-align: left; width: 150px">External Address</th>
+                <td style="text-align: left">{self.external_address}</td>
+                </tr>
+            </table> </p>
+            </div>
+          </div>
+        """
+
+        return html
 
     async def __aenter__(self):
         await self
diff --git a/distributed/deploy/ssh.py b/distributed/deploy/ssh.py
index 492c4ce05ef..81e08e72399 100644
--- a/distributed/deploy/ssh.py
+++ b/distributed/deploy/ssh.py
@@ -42,9 +42,6 @@ async def close(self):
         self.connection.close()
         await super().close()
 
-    def __repr__(self):
-        return f"<SSH {type(self).__name__}: status={self.status}>"
-
 
 class Worker(Process):
     """A Remote Dask Worker controled by SSH

From 5467f9c1428ada75724700da737919317a885f56 Mon Sep 17 00:00:00 2001
From: Naty Clementi <natyclementi@gmail.com>
Date: Tue, 17 Aug 2021 13:07:52 -0400
Subject: [PATCH 1441/1550] Worker Network Timeseries (#5129)

---
 distributed/dashboard/components/scheduler.py | 286 ++++++++++++++++--
 distributed/dashboard/scheduler.py            |  18 +-
 .../dashboard/tests/test_scheduler_bokeh.py   |  73 +++++
 distributed/system_monitor.py                 |  34 +++
 docs/source/http_services.rst                 |   5 +
 5 files changed, 397 insertions(+), 19 deletions(-)

diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index 4cf03e85a9a..d67885405c8 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -743,9 +743,12 @@ def __init__(self, scheduler, **kwargs):
                     "y_write": [],
                     "x_read": [],
                     "x_write": [],
+                    "x_read_disk": [],
+                    "x_write_disk": [],
                 }
             )
-            self.root = figure(
+
+            self.bandwidth = figure(
                 title="Worker Network Bandwidth",
                 tools="",
                 id="bk-worker-net-bandwidth",
@@ -754,7 +757,7 @@ def __init__(self, scheduler, **kwargs):
             )
 
             # read_bytes
-            self.root.hbar(
+            self.bandwidth.hbar(
                 y="y_read",
                 right="x_read",
                 line_color=None,
@@ -766,7 +769,7 @@ def __init__(self, scheduler, **kwargs):
             )
 
             # write_bytes
-            self.root.hbar(
+            self.bandwidth.hbar(
                 y="y_write",
                 right="x_write",
                 line_color=None,
@@ -777,15 +780,55 @@ def __init__(self, scheduler, **kwargs):
                 source=self.source,
             )
 
-            self.root.axis[0].ticker = BasicTicker(**TICKS_1024)
-            self.root.xaxis[0].formatter = NumeralTickFormatter(format="0.0 b")
-            self.root.xaxis.major_label_orientation = XLABEL_ORIENTATION
-            self.root.xaxis.minor_tick_line_alpha = 0
-            self.root.x_range = Range1d(start=0)
-            self.root.yaxis.visible = False
-            self.root.ygrid.visible = False
-            self.root.toolbar_location = None
-            self.root.yaxis.visible = False
+            self.bandwidth.axis[0].ticker = BasicTicker(**TICKS_1024)
+            self.bandwidth.xaxis[0].formatter = NumeralTickFormatter(format="0.0 b")
+            self.bandwidth.xaxis.major_label_orientation = XLABEL_ORIENTATION
+            self.bandwidth.xaxis.minor_tick_line_alpha = 0
+            self.bandwidth.x_range = Range1d(start=0)
+            self.bandwidth.yaxis.visible = False
+            self.bandwidth.ygrid.visible = False
+            self.bandwidth.toolbar_location = None
+
+            self.disk = figure(
+                title="Workers Disk",
+                tools="",
+                id="bk-workers-disk",
+                name="worker_disk",
+                **kwargs,
+            )
+
+            # read_bytes_disk
+            self.disk.hbar(
+                y="y_read",
+                right="x_read_disk",
+                line_color=None,
+                left=0,
+                height=0.5,
+                fill_color="red",
+                legend_label="read",
+                source=self.source,
+            )
+
+            # write_bytes_disk
+            self.disk.hbar(
+                y="y_write",
+                right="x_write_disk",
+                line_color=None,
+                left=0,
+                height=0.5,
+                fill_color="blue",
+                legend_label="write",
+                source=self.source,
+            )
+
+            self.disk.axis[0].ticker = BasicTicker(**TICKS_1024)
+            self.disk.xaxis[0].formatter = NumeralTickFormatter(format="0.0 b")
+            self.disk.xaxis.major_label_orientation = XLABEL_ORIENTATION
+            self.disk.xaxis.minor_tick_line_alpha = 0
+            self.disk.x_range = Range1d(start=0)
+            self.disk.yaxis.visible = False
+            self.disk.ygrid.visible = False
+            self.disk.toolbar_location = None
 
     @without_property_validation
     def update(self):
@@ -798,28 +841,235 @@ def update(self):
 
             x_read = []
             x_write = []
+            x_read_disk = []
+            x_write_disk = []
 
             for ws in workers:
                 x_read.append(ws.metrics["read_bytes"])
                 x_write.append(ws.metrics["write_bytes"])
+                x_read_disk.append(ws.metrics["read_bytes_disk"])
+                x_write_disk.append(ws.metrics["write_bytes_disk"])
 
-            self.root.x_range.end = max(
-                max(x_read),
-                max(x_write),
-                100_000_000,
-                0.95 * self.root.x_range.end,
-            )
+            if self.scheduler.workers:
+                self.bandwidth.x_range.end = max(
+                    max(x_read),
+                    max(x_write),
+                    100_000_000,
+                    0.95 * self.bandwidth.x_range.end,
+                )
+
+                self.disk.x_range.end = max(
+                    max(x_read_disk),
+                    max(x_write_disk),
+                    100_000_000,
+                    0.95 * self.disk.x_range.end,
+                )
+            else:
+                self.bandwidth.x_range.end = 100_000_000
+                self.disk.x_range.end = 100_000_000
 
             result = {
                 "y_read": y_read,
                 "y_write": y_write,
                 "x_read": x_read,
                 "x_write": x_write,
+                "x_read_disk": x_read_disk,
+                "x_write_disk": x_write_disk,
             }
 
             update(self.source, result)
 
 
+class SystemTimeseries(DashboardComponent):
+    """Timeseries for worker network bandwidth, cpu, memory and disk.
+
+    bandwidth: plots the average of read_bytes and write_bytes for the workers
+    as a function of time.
+    cpu: plots the average of cpu for the workers as a function of time.
+    memory: plots the average of memory for the workers as a function of time.
+    disk: plots the average of read_bytes_disk and write_bytes_disk for the workers
+    as a function of time.
+
+    The metrics plotted come from the aggregation of
+    from ws.metrics["val"] for ws in scheduler.workers.values() divided by nuber of workers.
+    """
+
+    def __init__(self, scheduler, **kwargs):
+        with log_errors():
+            self.scheduler = scheduler
+            self.source = ColumnDataSource(
+                {
+                    "time": [],
+                    "read_bytes": [],
+                    "write_bytes": [],
+                    "cpu": [],
+                    "memory": [],
+                    "read_bytes_disk": [],
+                    "write_bytes_disk": [],
+                }
+            )
+
+            update(self.source, self.get_data())
+
+            x_range = DataRange1d(follow="end", follow_interval=20000, range_padding=0)
+            tools = "reset, xpan, xwheel_zoom"
+
+            self.bandwidth = figure(
+                title="Workers Network Bandwidth",
+                x_axis_type="datetime",
+                tools=tools,
+                x_range=x_range,
+                id="bk-worker-network-bandwidth-ts",
+                name="worker_network_bandwidth-timeseries",
+                **kwargs,
+            )
+
+            self.bandwidth.line(
+                source=self.source,
+                x="time",
+                y="read_bytes",
+                color="red",
+                legend_label="read (mean)",
+            )
+            self.bandwidth.line(
+                source=self.source,
+                x="time",
+                y="write_bytes",
+                color="blue",
+                legend_label="write (mean)",
+            )
+
+            self.bandwidth.legend.location = "top_left"
+            self.bandwidth.yaxis.axis_label = "bytes / second"
+            self.bandwidth.yaxis[0].formatter = NumeralTickFormatter(format="0.0b")
+            self.bandwidth.y_range.start = 0
+            self.bandwidth.yaxis.minor_tick_line_alpha = 0
+            self.bandwidth.xgrid.visible = False
+
+            self.cpu = figure(
+                title="Workers CPU",
+                x_axis_type="datetime",
+                tools=tools,
+                x_range=x_range,
+                id="bk-worker-cpu-ts",
+                name="worker_cpu-timeseries",
+                **kwargs,
+            )
+
+            self.cpu.line(
+                source=self.source,
+                x="time",
+                y="cpu",
+            )
+            self.cpu.yaxis.axis_label = "Utilization"
+            self.cpu.y_range.start = 0
+            self.cpu.yaxis.minor_tick_line_alpha = 0
+            self.cpu.xgrid.visible = False
+
+            self.memory = figure(
+                title="Workers Memory",
+                x_axis_type="datetime",
+                tools=tools,
+                x_range=x_range,
+                id="bk-worker-memory-ts",
+                name="worker_memory-timeseries",
+                **kwargs,
+            )
+
+            self.memory.line(
+                source=self.source,
+                x="time",
+                y="memory",
+            )
+            self.memory.yaxis.axis_label = "Bytes"
+            self.memory.yaxis[0].formatter = NumeralTickFormatter(format="0.0b")
+            self.memory.y_range.start = 0
+            self.memory.yaxis.minor_tick_line_alpha = 0
+            self.memory.xgrid.visible = False
+
+            self.disk = figure(
+                title="Workers Disk",
+                x_axis_type="datetime",
+                tools=tools,
+                x_range=x_range,
+                id="bk-worker-disk-ts",
+                name="worker_disk-timeseries",
+                **kwargs,
+            )
+
+            self.disk.line(
+                source=self.source,
+                x="time",
+                y="read_bytes_disk",
+                color="red",
+                legend_label="read (mean)",
+            )
+            self.disk.line(
+                source=self.source,
+                x="time",
+                y="write_bytes_disk",
+                color="blue",
+                legend_label="write (mean)",
+            )
+
+            self.disk.legend.location = "top_left"
+            self.disk.yaxis.axis_label = "bytes / second"
+            self.disk.yaxis[0].formatter = NumeralTickFormatter(format="0.0b")
+            self.disk.y_range.start = 0
+            self.disk.yaxis.minor_tick_line_alpha = 0
+            self.disk.xgrid.visible = False
+
+    def get_data(self):
+        workers = self.scheduler.workers.values()
+
+        read_bytes = 0
+        write_bytes = 0
+        cpu = 0
+        memory = 0
+        read_bytes_disk = 0
+        write_bytes_disk = 0
+        time = 0
+        for ws in workers:
+            read_bytes += ws.metrics["read_bytes"]
+            write_bytes += ws.metrics["write_bytes"]
+            cpu += ws.metrics["cpu"]
+            memory += ws.metrics["memory"]
+            read_bytes_disk += ws.metrics["read_bytes_disk"]
+            write_bytes_disk += ws.metrics["write_bytes_disk"]
+            time += ws.metrics["time"]
+
+        result = {
+            # use `or` to avoid ZeroDivision when no workers
+            "time": [time / (len(workers) or 1) * 1000],
+            "read_bytes": [read_bytes / (len(workers) or 1)],
+            "write_bytes": [write_bytes / (len(workers) or 1)],
+            "cpu": [cpu / (len(workers) or 1)],
+            "memory": [memory / (len(workers) or 1)],
+            "read_bytes_disk": [read_bytes_disk / (len(workers) or 1)],
+            "write_bytes_disk": [write_bytes_disk / (len(workers) or 1)],
+        }
+        return result
+
+    @without_property_validation
+    def update(self):
+        with log_errors():
+            self.source.stream(self.get_data(), 1000)
+
+            if self.scheduler.workers:
+                y_end_cpu = sum(
+                    ws.nthreads or 1 for ws in self.scheduler.workers.values()
+                ) / len(self.scheduler.workers.values())
+                y_end_mem = sum(
+                    ws.memory_limit for ws in self.scheduler.workers.values()
+                ) / len(self.scheduler.workers.values())
+            else:
+                y_end_cpu = 1
+                y_end_mem = 100_000_000
+
+            self.cpu.y_range.end = y_end_cpu * 100
+            self.memory.y_range.end = y_end_mem
+
+
 class ComputePerKey(DashboardComponent):
     """Bar chart showing time spend in action by key prefix"""
 
diff --git a/distributed/dashboard/scheduler.py b/distributed/dashboard/scheduler.py
index cd321130043..78286e22de6 100644
--- a/distributed/dashboard/scheduler.py
+++ b/distributed/dashboard/scheduler.py
@@ -15,6 +15,7 @@
     MemoryByKey,
     Occupancy,
     SystemMonitor,
+    SystemTimeseries,
     TaskGraph,
     TaskGroupGraph,
     TaskProgress,
@@ -69,7 +70,22 @@
     "/individual-bandwidth-types": individual_doc(BandwidthTypes, 500),
     "/individual-bandwidth-workers": individual_doc(BandwidthWorkers, 500),
     "/individual-workers-network-bandwidth": individual_doc(
-        WorkerNetworkBandwidth, 500
+        WorkerNetworkBandwidth, 500, fig_attr="bandwidth"
+    ),
+    "/individual-workers-disk": individual_doc(
+        WorkerNetworkBandwidth, 500, fig_attr="disk"
+    ),
+    "/individual-workers-network-bandwidth-timeseries": individual_doc(
+        SystemTimeseries, 500, fig_attr="bandwidth"
+    ),
+    "/individual-workers-cpu-timeseries": individual_doc(
+        SystemTimeseries, 500, fig_attr="cpu"
+    ),
+    "/individual-workers-memory-timeseries": individual_doc(
+        SystemTimeseries, 500, fig_attr="memory"
+    ),
+    "/individual-workers-disk-timeseries": individual_doc(
+        SystemTimeseries, 500, fig_attr="disk"
     ),
     "/individual-memory-by-key": individual_doc(MemoryByKey, 500),
     "/individual-compute-time-per-key": individual_doc(ComputePerKey, 500),
diff --git a/distributed/dashboard/tests/test_scheduler_bokeh.py b/distributed/dashboard/tests/test_scheduler_bokeh.py
index 473595e0d73..3304aa0c6a1 100644
--- a/distributed/dashboard/tests/test_scheduler_bokeh.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh.py
@@ -31,6 +31,7 @@
     StealingEvents,
     StealingTimeSeries,
     SystemMonitor,
+    SystemTimeseries,
     TaskGraph,
     TaskGroupGraph,
     TaskProgress,
@@ -506,6 +507,78 @@ async def test_WorkerNetworkBandwidth_metrics(c, s, a, b):
     for idx, ws in enumerate(s.workers.values()):
         assert ws.metrics["read_bytes"] == nb.source.data["x_read"][idx]
         assert ws.metrics["write_bytes"] == nb.source.data["x_write"][idx]
+        assert ws.metrics["read_bytes_disk"] == nb.source.data["x_read_disk"][idx]
+        assert ws.metrics["write_bytes_disk"] == nb.source.data["x_write_disk"][idx]
+
+
+@gen_cluster(client=True)
+async def test_SystemTimeseries(c, s, a, b):
+    # Disable system monitor periodic callback to allow us to manually control
+    # when it is called below
+    a.periodic_callbacks["monitor"].stop()
+    b.periodic_callbacks["monitor"].stop()
+
+    # Update worker system monitors and send updated metrics to the scheduler
+    a.monitor.update()
+    b.monitor.update()
+    await asyncio.gather(a.heartbeat(), b.heartbeat())
+
+    systs = SystemTimeseries(s)
+    workers = s.workers.values()
+
+    assert all(len(v) == 1 for v in systs.source.data.values())
+    assert systs.source.data["read_bytes"][0] == sum(
+        [ws.metrics["read_bytes"] for ws in workers]
+    ) / len(workers)
+    assert systs.source.data["write_bytes"][0] == sum(
+        [ws.metrics["write_bytes"] for ws in workers]
+    ) / len(workers)
+    assert systs.source.data["cpu"][0] == sum(
+        [ws.metrics["cpu"] for ws in workers]
+    ) / len(workers)
+    assert systs.source.data["memory"][0] == sum(
+        [ws.metrics["memory"] for ws in workers]
+    ) / len(workers)
+    assert systs.source.data["read_bytes_disk"][0] == sum(
+        [ws.metrics["read_bytes_disk"] for ws in workers]
+    ) / len(workers)
+    assert systs.source.data["write_bytes_disk"][0] == sum(
+        [ws.metrics["write_bytes_disk"] for ws in workers]
+    ) / len(workers)
+    assert (
+        systs.source.data["time"][0]
+        == sum([ws.metrics["time"] for ws in workers]) / len(workers) * 1000
+    )
+
+    # Update worker system monitors and send updated metrics to the scheduler
+    a.monitor.update()
+    b.monitor.update()
+    await asyncio.gather(a.heartbeat(), b.heartbeat())
+    systs.update()
+
+    assert all(len(v) == 2 for v in systs.source.data.values())
+    assert systs.source.data["read_bytes"][1] == sum(
+        [ws.metrics["read_bytes"] for ws in workers]
+    ) / len(workers)
+    assert systs.source.data["write_bytes"][1] == sum(
+        [ws.metrics["write_bytes"] for ws in workers]
+    ) / len(workers)
+    assert systs.source.data["cpu"][1] == sum(
+        [ws.metrics["cpu"] for ws in workers]
+    ) / len(workers)
+    assert systs.source.data["memory"][1] == sum(
+        [ws.metrics["memory"] for ws in workers]
+    ) / len(workers)
+    assert systs.source.data["read_bytes_disk"][1] == sum(
+        [ws.metrics["read_bytes_disk"] for ws in workers]
+    ) / len(workers)
+    assert systs.source.data["write_bytes_disk"][1] == sum(
+        [ws.metrics["write_bytes_disk"] for ws in workers]
+    ) / len(workers)
+    assert (
+        systs.source.data["time"][1]
+        == sum([ws.metrics["time"] for ws in workers]) / len(workers) * 1000
+    )
 
 
 @gen_cluster(client=True)
diff --git a/distributed/system_monitor.py b/distributed/system_monitor.py
index 8824ef2db12..dc749f58536 100644
--- a/distributed/system_monitor.py
+++ b/distributed/system_monitor.py
@@ -35,6 +35,19 @@ def __init__(self, n=10000):
             self._last_io_counters = ioc
             self._collect_net_io_counters = True
 
+        try:
+            disk_ioc = psutil.disk_io_counters()
+        except Exception:
+            self._collect_disk_io_counters = False
+        else:
+            self.last_time_disk = time()
+            self.read_bytes_disk = deque(maxlen=n)
+            self.write_bytes_disk = deque(maxlen=n)
+            self.quantities["read_bytes_disk"] = self.read_bytes_disk
+            self.quantities["write_bytes_disk"] = self.write_bytes_disk
+            self._last_disk_io_counters = disk_ioc
+            self._collect_disk_io_counters = True
+
         if not WINDOWS:
             self.num_fds = deque(maxlen=n)
             self.quantities["num_fds"] = self.num_fds
@@ -86,6 +99,27 @@ def update(self):
                 result["read_bytes"] = read_bytes
                 result["write_bytes"] = write_bytes
 
+        if self._collect_disk_io_counters:
+            try:
+                disk_ioc = psutil.disk_io_counters()
+            except Exception:
+                pass
+            else:
+                last_disk = self._last_disk_io_counters
+                duration_disk = now - self.last_time_disk
+                read_bytes_disk = (disk_ioc.read_bytes - last_disk.read_bytes) / (
+                    duration_disk or 0.5
+                )
+                write_bytes_disk = (disk_ioc.write_bytes - last_disk.write_bytes) / (
+                    duration_disk or 0.5
+                )
+                self.last_time_disk = now
+                self._last_disk_io_counters = disk_ioc
+                self.read_bytes_disk.append(read_bytes_disk)
+                self.write_bytes_disk.append(write_bytes_disk)
+                result["read_bytes_disk"] = read_bytes_disk
+                result["write_bytes_disk"] = write_bytes_disk
+
         if not WINDOWS:
             num_fds = self.proc.num_fds()
             self.num_fds.append(num_fds)
diff --git a/docs/source/http_services.rst b/docs/source/http_services.rst
index 75027a74a38..dee90e448ad 100644
--- a/docs/source/http_services.rst
+++ b/docs/source/http_services.rst
@@ -66,6 +66,11 @@ Individual bokeh plots
 - ``/individual-bandwidth-types``
 - ``/individual-bandwidth-workers``
 - ``/individual-workers-network-bandwidth``
+- ``/individual-workers-disk``
+- ``/individual-workers-network-bandwidth-timeseries``
+- ``/individual-workers-cpu-timeseries``
+- ``/individual-workers-memory-timeseries``
+- ``/individual-workers-disk-timeseries``
 - ``/individual-memory-by-key``
 - ``/individual-compute-time-per-key``
 - ``/individual-aggregate-time-per-action``

From 8c73a18ba5c2ffe61977ea936da8ffeacb815c61 Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Wed, 18 Aug 2021 14:44:55 +0200
Subject: [PATCH 1442/1550] Update WorkerPlugin docstring about usage of
 TaskState objects (#5226)

* Update WorkerPlugin docstring for usage of task exceptions

* Avoid usage of StateLegacyMapping in progress plugin
---
 distributed/diagnostics/plugin.py   | 8 ++++++--
 distributed/diagnostics/progress.py | 4 +++-
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/distributed/diagnostics/plugin.py b/distributed/diagnostics/plugin.py
index f9077afddd1..ba2b031e5ce 100644
--- a/distributed/diagnostics/plugin.py
+++ b/distributed/diagnostics/plugin.py
@@ -120,8 +120,12 @@ class WorkerPlugin:
     ...
     ...     def transition(self, key, start, finish, *args, **kwargs):
     ...         if finish == 'error':
-    ...             exc = self.worker.exceptions[key]
-    ...             self.logger.error("Task '%s' has failed with exception: %s" % (key, str(exc)))
+    ...             ts = self.worker.tasks[key]
+    ...             exc_info = (type(ts.exception), ts.exception, ts.traceback)
+    ...             self.logger.error(
+    ...                 "Error during computation of '%s'.", key,
+    ...                 exc_info=exc_info
+    ...             )
 
     >>> plugin = ErrorLogger()
     >>> client.register_worker_plugin(plugin)  # doctest: +SKIP
diff --git a/distributed/diagnostics/progress.py b/distributed/diagnostics/progress.py
index 1f7940c0302..f126dce0a95 100644
--- a/distributed/diagnostics/progress.py
+++ b/distributed/diagnostics/progress.py
@@ -124,7 +124,9 @@ def stop(self, exception=None, key=None):
             self.scheduler.plugins.remove(self)
         if exception:
             self.status = "error"
-            self.extra.update({"exception": self.scheduler.exceptions[key], "key": key})
+            self.extra.update(
+                {"exception": self.scheduler.tasks[key].exception, "key": key}
+            )
         else:
             self.status = "finished"
         logger.debug("Remove Progress plugin")

From a0124ac56079320eac81f2210ec29a027fccbf56 Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Wed, 18 Aug 2021 18:51:26 +0200
Subject: [PATCH 1443/1550] Fix flaky `test_worker_reconnects_mid_compute`
 (#5227)

---
 distributed/tests/test_worker.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index f1895a64895..21920c44da7 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -2443,7 +2443,7 @@ def fast_on_a(lock):
 
     assert "Unexpected worker completed task" in s_logs.getvalue()
 
-    while not len(s.tasks[f2.key].who_has) == 2:
+    while a.address not in {w.address for w in s.tasks[f2.key].who_has}:
         await asyncio.sleep(0.001)
 
     # Ensure that all keys have been properly registered and will also be
@@ -2510,7 +2510,7 @@ def fast_on_a(lock):
 
     assert "Unexpected worker completed task" in s_logs.getvalue()
 
-    while not len(s.tasks[f2.key].who_has) == 2:
+    while a.address not in {w.address for w in s.tasks[f2.key].who_has}:
         await asyncio.sleep(0.001)
 
     del f1, f2, f3

From e5f25e7fa10b97bc78776a44b25798550ba4bf37 Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Thu, 19 Aug 2021 02:35:27 +0100
Subject: [PATCH 1444/1550] Use upstream `dask.widgets` (#5205)

Co-authored-by: James Bourbeau <jrbourbeau@gmail.com>
---
 distributed/__init__.py                       |  1 +
 distributed/client.py                         |  4 +--
 distributed/deploy/cluster.py                 |  2 +-
 distributed/deploy/local.py                   |  2 +-
 distributed/objects.py                        |  3 +-
 distributed/protocol/serialize.py             |  3 +-
 distributed/protocol/tests/test_serialize.py  |  2 +-
 distributed/scheduler.py                      |  2 +-
 distributed/tests/test_scheduler.py           |  4 +--
 distributed/tests/test_utils.py               | 16 ++++-------
 distributed/utils.py                          | 25 +++--------------
 distributed/widgets/__init__.py               | 11 +++++++-
 distributed/widgets/templates/client.html.j2  |  2 +-
 .../widgets/templates/computation.html.j2     |  2 +-
 distributed/widgets/utils.py                  | 28 -------------------
 distributed/worker.py                         | 10 +++++--
 16 files changed, 43 insertions(+), 74 deletions(-)
 delete mode 100644 distributed/widgets/utils.py

diff --git a/distributed/__init__.py b/distributed/__init__.py
index 9f7a8d6f4bc..57edc10d6bd 100644
--- a/distributed/__init__.py
+++ b/distributed/__init__.py
@@ -1,4 +1,5 @@
 from . import config  # isort:skip; load distributed configuration first
+from . import widgets  # isort:skip; load distributed widgets second
 import dask
 from dask.config import config
 from dask.utils import import_required
diff --git a/distributed/client.py b/distributed/client.py
index e7389220d42..e3c9f074f2b 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -38,7 +38,9 @@
     funcname,
     parse_timedelta,
     stringify,
+    typename,
 )
+from dask.widgets import get_template
 
 try:
     from dask.delayed import single_key
@@ -85,7 +87,6 @@
     no_default,
     sync,
     thread_state,
-    typename,
 )
 from .utils_comm import (
     WrappedKey,
@@ -95,7 +96,6 @@
     scatter_to_workers,
     unpack_remotedata,
 )
-from .widgets import get_template
 from .worker import get_client, get_worker, secede
 
 logger = logging.getLogger(__name__)
diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
index fcb54e1dcd5..bc96ccfe23b 100644
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -10,11 +10,11 @@
 
 import dask.config
 from dask.utils import _deprecated, format_bytes, parse_timedelta
+from dask.widgets import get_template
 
 from ..core import Status
 from ..objects import SchedulerInfo
 from ..utils import Log, Logs, format_dashboard_link, log_errors, sync, thread_state
-from ..widgets import get_template
 from .adaptive import Adaptive
 
 logger = logging.getLogger(__name__)
diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index c736f932a7d..19bc7f73e8a 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -7,11 +7,11 @@
 import toolz
 
 from dask.system import CPU_COUNT
+from dask.widgets import get_template
 
 from ..nanny import Nanny
 from ..scheduler import Scheduler
 from ..security import Security
-from ..widgets import get_template
 from ..worker import Worker, parse_memory_limit
 from .spec import SpecCluster
 from .utils import nprocesses_nthreads
diff --git a/distributed/objects.py b/distributed/objects.py
index 837b5c92fe3..22717622a0e 100644
--- a/distributed/objects.py
+++ b/distributed/objects.py
@@ -2,8 +2,9 @@
 These are mostly regular objects with more useful _repr_ and _repr_html_ methods."""
 from urllib.parse import urlparse
 
+from dask.widgets import get_environment, get_template
+
 from distributed.utils import format_dashboard_link
-from distributed.widgets import get_environment, get_template
 
 
 class HasWhat(dict):
diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index 51815677f5a..877cd22c824 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -8,8 +8,9 @@
 
 import dask
 from dask.base import normalize_token
+from dask.utils import typename
 
-from ..utils import ensure_bytes, has_keyword, typename
+from ..utils import ensure_bytes, has_keyword
 from . import pickle
 from .compression import decompress, maybe_compress
 from .utils import frame_split_size, msgpack_opts, pack_frames_prelude, unpack_frames
diff --git a/distributed/protocol/tests/test_serialize.py b/distributed/protocol/tests/test_serialize.py
index 36359304830..c552742930e 100644
--- a/distributed/protocol/tests/test_serialize.py
+++ b/distributed/protocol/tests/test_serialize.py
@@ -520,7 +520,7 @@ def test_serialize_lists(serializers):
 )
 def test_deser_memoryview(data_in):
     header, frames = serialize(data_in)
-    assert header["type"] == "builtins.memoryview"
+    assert header["type"] == "memoryview"
     assert frames[0] is data_in
     data_out = deserialize(header, frames)
     assert data_in == data_out
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index ca743cdb443..a2d358da04d 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -38,6 +38,7 @@
 import dask
 from dask.highlevelgraph import HighLevelGraph
 from dask.utils import format_bytes, format_time, parse_bytes, parse_timedelta
+from dask.widgets import get_template
 
 from . import preloading, profile
 from . import versions as version_module
@@ -81,7 +82,6 @@
 from .utils_comm import gather_from_workers, retry_operation, scatter_to_workers
 from .utils_perf import disable_gc_diagnosis, enable_gc_diagnosis
 from .variable import VariableExtension
-from .widgets import get_template
 
 try:
     from cython import compiled
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 143c3f8a50b..260513e954e 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -18,7 +18,7 @@
 
 import dask
 from dask import delayed
-from dask.utils import apply, parse_timedelta, stringify
+from dask.utils import apply, parse_timedelta, stringify, typename
 
 from distributed import Client, Nanny, Worker, fire_and_forget, wait
 from distributed.comm import Comm
@@ -27,7 +27,7 @@
 from distributed.metrics import time
 from distributed.protocol.pickle import dumps
 from distributed.scheduler import MemoryState, Scheduler
-from distributed.utils import TimeoutError, tmpfile, typename
+from distributed.utils import TimeoutError, tmpfile
 from distributed.utils_test import (
     captured_logger,
     cluster,
diff --git a/distributed/tests/test_utils.py b/distributed/tests/test_utils.py
index 285b8407000..bbc5892eb71 100644
--- a/distributed/tests/test_utils.py
+++ b/distributed/tests/test_utils.py
@@ -40,7 +40,6 @@
     sync,
     thread_state,
     truncate_exception,
-    typename,
     warn_on_duration,
 )
 from distributed.utils_test import (
@@ -596,15 +595,12 @@ def test_parse_timedelta_deprecated():
     assert parse_timedelta is dask.utils.parse_timedelta
 
 
+def test_typename_deprecated():
+    with pytest.warns(FutureWarning, match="typename is deprecated"):
+        from distributed.utils import typename
+    assert typename is dask.utils.typename
+
+
 def test_iscoroutinefunction_unhashable_input():
     # Ensure iscoroutinefunction can handle unhashable callables
     assert not iscoroutinefunction(_UnhashableCallable())
-
-
-class MyType:
-    pass
-
-
-def test_typename_on_instances():
-    instance = MyType()
-    assert typename(instance) == typename(MyType)
diff --git a/distributed/utils.py b/distributed/utils.py
index 22b1e9f718e..5edcc51738d 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -41,6 +41,7 @@
 import dask
 from dask import istask
 from dask.utils import parse_timedelta as _parse_timedelta
+from dask.widgets import get_template
 
 try:
     from tornado.ioloop import PollIOLoop
@@ -1151,23 +1152,6 @@ def warn_on_duration(duration, msg):
         warnings.warn(msg, stacklevel=2)
 
 
-def typename(typ):
-    """Return name of type
-
-    Examples
-    --------
-    >>> from distributed import Scheduler
-    >>> typename(Scheduler)
-    'distributed.scheduler.Scheduler'
-    """
-    if not isinstance(typ, type):
-        return typename(type(typ))
-    try:
-        return typ.__module__ + "." + typ.__name__
-    except AttributeError:
-        return str(typ)
-
-
 def format_dashboard_link(host, port):
     template = dask.config.get("distributed.dashboard.link")
     if dask.config.get("distributed.scheduler.dashboard.tls.cert"):
@@ -1244,8 +1228,6 @@ class Log(str):
     """A container for newline-delimited string of log entries"""
 
     def _repr_html_(self):
-        from .widgets import get_template  # Avoiding circular import
-
         return get_template("log.html.j2").render(log=self)
 
 
@@ -1253,8 +1235,6 @@ class Logs(dict):
     """A container for a dict mapping names to strings of log entries"""
 
     def _repr_html_(self):
-        from .widgets import get_template  # Avoiding circular import
-
         return get_template("logs.html.j2").render(logs=self)
 
 
@@ -1285,6 +1265,8 @@ def cli_keywords(d: dict, cls=None, cmd=None):
     ...
     ValueError: Class distributed.worker.Worker does not support keyword x
     """
+    from dask.utils import typename
+
     if cls or cmd:
         for k in d:
             if not has_keyword(cls, k) and not command_has_keyword(cmd, k):
@@ -1440,6 +1422,7 @@ def clean_dashboard_address(addrs: AnyType, default_listen_ip: str = "") -> List
     "funcname": "dask.utils.funcname",
     "parse_bytes": "dask.utils.parse_bytes",
     "parse_timedelta": "dask.utils.parse_timedelta",
+    "typename": "dask.utils.typename",
 }
 
 
diff --git a/distributed/widgets/__init__.py b/distributed/widgets/__init__.py
index 511cb30ea25..efdb7eccacd 100644
--- a/distributed/widgets/__init__.py
+++ b/distributed/widgets/__init__.py
@@ -1 +1,10 @@
-from .utils import get_environment, get_template
+import os.path
+
+from dask.widgets import FILTERS, TEMPLATE_PATHS
+
+from ..utils import key_split
+
+TEMPLATE_PATHS.append(
+    os.path.join(os.path.dirname(os.path.abspath(__file__)), "templates")
+)
+FILTERS["key_split"] = key_split
diff --git a/distributed/widgets/templates/client.html.j2 b/distributed/widgets/templates/client.html.j2
index 19206feded9..6c792e737ab 100644
--- a/distributed/widgets/templates/client.html.j2
+++ b/distributed/widgets/templates/client.html.j2
@@ -11,7 +11,7 @@
             <td style="text-align: left;"></td>
         {% elif cluster %}
             <td style="text-align: left;"><strong>Connection method:</strong> Cluster object</td>
-            <td style="text-align: left;"><strong>Cluster type:</strong> {{ cluster | type }}</td>
+            <td style="text-align: left;"><strong>Cluster type:</strong> {{ cluster | type | typename(short=True) }}</td>
         {% elif scheduler_file %}
             <td style="text-align: left;"><strong>Connection method:</strong> Scheduler file</td>
             <td style="text-align: left;"><strong>Scheduler file:</strong> {{ scheduler_file }}</td>
diff --git a/distributed/widgets/templates/computation.html.j2 b/distributed/widgets/templates/computation.html.j2
index 97a44b5c244..0da18a08b18 100644
--- a/distributed/widgets/templates/computation.html.j2
+++ b/distributed/widgets/templates/computation.html.j2
@@ -36,7 +36,7 @@
 <summary style="margin-bottom": 20px><h4 style="display:inline">Task Groups</h4></summary>
 <ul>
 {% for gr in groups %}
-    <li> {{ gr._repr_html_() }} </li>
+    <li> {{ gr.__repr__() | html_escape }} </li>
 {% endfor %}
 </ul>
 </details>
diff --git a/distributed/widgets/utils.py b/distributed/widgets/utils.py
deleted file mode 100644
index 3a5d2c15002..00000000000
--- a/distributed/widgets/utils.py
+++ /dev/null
@@ -1,28 +0,0 @@
-import datetime
-import html
-import os.path
-
-from jinja2 import Environment, FileSystemLoader, Template
-
-from dask.utils import format_bytes, format_time_ago
-
-from ..utils import key_split
-
-
-def get_environment() -> Environment:
-    loader = FileSystemLoader(
-        [os.path.join(os.path.dirname(os.path.abspath(__file__)), "templates")]
-    )
-    environment = Environment(loader=loader)
-    environment.filters["format_bytes"] = format_bytes
-    environment.filters["format_time_ago"] = format_time_ago
-    environment.filters["datetime_from_timestamp"] = datetime.datetime.fromtimestamp
-    environment.filters["type"] = lambda cls: type(cls).__name__
-    environment.filters["key_split"] = key_split
-    environment.filters["html_escape"] = html.escape
-
-    return environment
-
-
-def get_template(name: str) -> Template:
-    return get_environment().get_template(name)
diff --git a/distributed/worker.py b/distributed/worker.py
index 39e77a8bb92..ef32a15e605 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -24,7 +24,14 @@
 import dask
 from dask.core import istask
 from dask.system import CPU_COUNT
-from dask.utils import apply, format_bytes, funcname, parse_bytes, parse_timedelta
+from dask.utils import (
+    apply,
+    format_bytes,
+    funcname,
+    parse_bytes,
+    parse_timedelta,
+    typename,
+)
 
 from . import comm, preloading, profile, system, utils
 from .batched import BatchedSend
@@ -67,7 +74,6 @@
     parse_ports,
     silence_logging,
     thread_state,
-    typename,
     warn_on_duration,
 )
 from .utils_comm import gather_from_workers, pack_data, retry_operation

From ddce05f6d11da974df46540c278588156a6b47f8 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Wed, 18 Aug 2021 22:18:46 -0500
Subject: [PATCH 1445/1550] Use development version of `dask` in gpuCI build
 (#5232)

---
 continuous_integration/gpuci/build.sh | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/continuous_integration/gpuci/build.sh b/continuous_integration/gpuci/build.sh
index 8e98736c8d2..a2346238feb 100644
--- a/continuous_integration/gpuci/build.sh
+++ b/continuous_integration/gpuci/build.sh
@@ -37,10 +37,13 @@ gpuci_logger "Activate conda env"
 . /opt/conda/etc/profile.d/conda.sh
 conda activate dask
 
+gpuci_logger "Install dask"
+python -m pip install git+https://github.com/dask/dask
+
 gpuci_logger "Install distributed"
 python setup.py install
 
-gpuci_logger "Check compiler versions"
+gpuci_logger "Check Python versions"
 python --version
 
 gpuci_logger "Check conda environment"
@@ -48,5 +51,5 @@ conda info
 conda config --show-sources
 conda list --show-channel-urls
 
-gpuci_logger "Python py.test for dask"
+gpuci_logger "Python py.test for distributed"
 py.test $WORKSPACE -v -m gpu --runslow --junitxml="$WORKSPACE/junit-distributed.xml"

From d904773d73a46856709a19fc08bed1cb88aa3279 Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Thu, 19 Aug 2021 19:47:34 +0100
Subject: [PATCH 1446/1550] Minor polish on cfexecutor (#5233)

---
 distributed/cfexecutor.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/distributed/cfexecutor.py b/distributed/cfexecutor.py
index fda2fe63fb3..92031247c26 100644
--- a/distributed/cfexecutor.py
+++ b/distributed/cfexecutor.py
@@ -128,10 +128,9 @@ def map(self, fn, *iterables, **kwargs):
 
         fs = self._client.map(fn, *iterables, **self._kwargs)
 
-        if isinstance(fs, list):
-            # Below iterator relies on this being a generator to cancel
-            # remaining futures
-            fs = (val for val in fs)
+        # Below iterator relies on fs being an iterator itself, and not just an iterable
+        # (such as a list), in order to cancel remaining futures
+        fs = iter(fs)
 
         # Yield must be hidden in closure so that the tasks are submitted
         # before the first iterator value is required.
@@ -148,8 +147,7 @@ def result_iterator():
                         yield future.result()
             finally:
                 remaining = list(fs)
-                for future in remaining:
-                    self._futures.add(future)
+                self._futures.update(remaining)
                 self._client.cancel(remaining)
 
         return result_iterator()

From 8cff8b714cee6396c62707788cdb9899bb2ff330 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Thu, 19 Aug 2021 14:36:35 -0500
Subject: [PATCH 1447/1550] Update computation code retrieval logic (#5236)

---
 distributed/client.py            |  2 +-
 distributed/tests/test_client.py | 19 +++++++++++++++++++
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/distributed/client.py b/distributed/client.py
index e3c9f074f2b..b228b494eae 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -2522,7 +2522,7 @@ def _get_computation_code() -> str:
 
         for fr, _ in traceback.walk_stack(None):
             if pattern is None or (
-                not pattern.match(fr.f_globals["__name__"])
+                not pattern.match(fr.f_globals.get("__name__", ""))
                 and fr.f_code.co_name not in ("<listcomp>", "<dictcomp>")
             ):
                 try:
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index a65ab4494d8..5d61e5571c4 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -6774,6 +6774,25 @@ def fetch_comp_code(dask_scheduler):
     assert code == test_function_code
 
 
+def test_computation_object_code_not_available(client):
+    np = pytest.importorskip("numpy")
+    pd = pytest.importorskip("pandas")
+    dd = pytest.importorskip("dask.dataframe")
+    df = pd.DataFrame({"a": range(10)})
+    ddf = dd.from_pandas(df, npartitions=3)
+    result = np.where(ddf.a > 4)
+
+    def fetch_comp_code(dask_scheduler):
+        computations = list(dask_scheduler.computations)
+        assert len(computations) == 1
+        comp = computations[0]
+        assert len(comp.code) == 1
+        return comp.code[0]
+
+    code = client.run_on_scheduler(fetch_comp_code)
+    assert code == "<Code not available>"
+
+
 @gen_cluster(client=True)
 async def test_computation_object_code_dask_persist(c, s, a, b):
     da = pytest.importorskip("dask.array")

From 6a8b49db67896caeea1b6e6075c1dd50b45ae73b Mon Sep 17 00:00:00 2001
From: Naty Clementi <natyclementi@gmail.com>
Date: Fri, 20 Aug 2021 05:27:32 -0400
Subject: [PATCH 1448/1550] Add target='_blank' for redirects of dashboard link
 (#5237)

* add target='_blank' for redirects of dashboard link

* Update distributed/widgets/templates/cluster.html.j2

Co-authored-by: James Bourbeau <jrbourbeau@users.noreply.github.com>

Co-authored-by: James Bourbeau <jrbourbeau@users.noreply.github.com>
---
 distributed/widgets/templates/client.html.j2         | 2 +-
 distributed/widgets/templates/cluster.html.j2        | 2 +-
 distributed/widgets/templates/scheduler_info.html.j2 | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/distributed/widgets/templates/client.html.j2 b/distributed/widgets/templates/client.html.j2
index 6c792e737ab..ad1c2e509ed 100644
--- a/distributed/widgets/templates/client.html.j2
+++ b/distributed/widgets/templates/client.html.j2
@@ -21,7 +21,7 @@
         {% if dashboard_link %}
             <tr>
                 <td style="text-align: left;">
-                    <strong>Dashboard: </strong> <a href="{{ dashboard_link }}">{{ dashboard_link }}</a>
+                    <strong>Dashboard: </strong> <a href="{{ dashboard_link }}" target="_blank">{{ dashboard_link }}</a>
                 </td>
                 <td style="text-align: left;"></td>
             </tr>
diff --git a/distributed/widgets/templates/cluster.html.j2 b/distributed/widgets/templates/cluster.html.j2
index c1c57c32996..f3d6d418ce5 100644
--- a/distributed/widgets/templates/cluster.html.j2
+++ b/distributed/widgets/templates/cluster.html.j2
@@ -7,7 +7,7 @@
         <table style="width: 100%; text-align: left;">
             <tr>
                 <td style="text-align: left;">
-                    <strong>Dashboard:</strong> <a href="{{ dashboard_link }}">{{ dashboard_link }}</a>
+                    <strong>Dashboard:</strong> <a href="{{ dashboard_link }}" target="_blank">{{ dashboard_link }}</a>
                 </td>
                 <td style="text-align: left;">
                     <strong>Workers:</strong> {{ workers | length }}
diff --git a/distributed/widgets/templates/scheduler_info.html.j2 b/distributed/widgets/templates/scheduler_info.html.j2
index a4766353c93..68036bca7f3 100644
--- a/distributed/widgets/templates/scheduler_info.html.j2
+++ b/distributed/widgets/templates/scheduler_info.html.j2
@@ -15,7 +15,7 @@
                 </tr>
                 <tr>
                     <td style="text-align: left;">
-                        <strong>Dashboard:</strong> <a href="{{ scheduler | format_dashboard_address }}">{{ scheduler | format_dashboard_address }}</a>
+                        <strong>Dashboard:</strong> <a href="{{ scheduler | format_dashboard_address }}" target="_blank">{{ scheduler | format_dashboard_address }}</a>
                     </td>
                     <td style="text-align: left;">
                         <strong>Total threads:</strong> {{ workers.values() | map(attribute='nthreads') | sum }}
@@ -57,7 +57,7 @@
                     </tr>
                     <tr>
                         <td style="text-align: left;">
-                            <strong>Dashboard: </strong> <a href="{{ worker | format_dashboard_address }}">{{ worker | format_dashboard_address }}</a>
+                            <strong>Dashboard: </strong> <a href="{{ worker | format_dashboard_address }}" target="_blank">{{ worker | format_dashboard_address }}</a>
                         </td>
                         <td style="text-align: left;">
                             <strong>Memory: </strong> {{ worker["memory_limit"] | format_bytes }}

From 2c63b7ea50223939ec3b49510938e3fb310af35a Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Fri, 20 Aug 2021 20:12:33 +0200
Subject: [PATCH 1449/1550] Log messages for `CommClosedError` now includes
 information about remote address (#5209)

Co-authored-by: James Bourbeau <jrbourbeau@users.noreply.github.com>
Co-authored-by: crusaderky <crusaderky@gmail.com>
---
 distributed/batched.py         |  6 +++---
 distributed/comm/core.py       |  2 +-
 distributed/comm/tcp.py        |  2 +-
 distributed/core.py            | 18 +++++++++++++-----
 distributed/scheduler.py       | 15 ++++++++++++---
 distributed/stealing.py        |  2 +-
 distributed/tests/test_core.py |  2 +-
 distributed/worker.py          |  2 +-
 8 files changed, 33 insertions(+), 16 deletions(-)

diff --git a/distributed/batched.py b/distributed/batched.py
index 4e59a09e80b..960f4fa828e 100644
--- a/distributed/batched.py
+++ b/distributed/batched.py
@@ -98,8 +98,8 @@ def _background_send(self):
                 else:
                     self.recent_message_log.append("large-message")
                 self.byte_count += nbytes
-            except CommClosedError as e:
-                logger.info("Batched Comm Closed: %s", e)
+            except CommClosedError:
+                logger.info("Batched Comm Closed %r", self.comm, exc_info=True)
                 break
             except Exception:
                 # We cannot safely retry self.comm.write, as we have no idea
@@ -133,7 +133,7 @@ def send(self, *msgs):
         This completes quickly and synchronously
         """
         if self.comm is not None and self.comm.closed():
-            raise CommClosedError()
+            raise CommClosedError(f"Comm {self.comm!r} already closed.")
 
         self.message_count += len(msgs)
         self.buffer.extend(msgs)
diff --git a/distributed/comm/core.py b/distributed/comm/core.py
index 1e8476a2539..fb4a3eb1ff5 100644
--- a/distributed/comm/core.py
+++ b/distributed/comm/core.py
@@ -224,7 +224,7 @@ async def on_connection(self, comm: Comm, handshake_overrides=None):
         except Exception as e:
             with suppress(Exception):
                 await comm.close()
-            raise CommClosedError() from e
+            raise CommClosedError(f"Comm {comm!r} closed.") from e
 
         comm.remote_info = handshake
         comm.remote_info["address"] = comm._peer_addr
diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py
index 0e66378862e..b4a2d39b2d8 100644
--- a/distributed/comm/tcp.py
+++ b/distributed/comm/tcp.py
@@ -504,7 +504,7 @@ async def _handle_stream(self, stream, address):
         try:
             await self.on_connection(comm)
         except CommClosedError:
-            logger.info("Connection closed before handshake completed")
+            logger.info("Connection from %s closed before handshake completed", address)
             return
 
         await self.comm_handler(comm)
diff --git a/distributed/core.py b/distributed/core.py
index 47fe40a13a5..9c5c49dd3f7 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -500,9 +500,9 @@ async def handle_comm(self, comm):
                             result = asyncio.ensure_future(result)
                             self._ongoing_coroutines.add(result)
                             result = await result
-                    except (CommClosedError, CancelledError) as e:
+                    except (CommClosedError, CancelledError):
                         if self.status == Status.running:
-                            logger.info("Lost connection to %r: %s", address, e)
+                            logger.info("Lost connection to %r", address, exc_info=True)
                         break
                     except Exception as e:
                         logger.exception("Exception while handling op %s", op)
@@ -791,12 +791,20 @@ async def send_recv_from_rpc(**kwargs):
                 kwargs["serializers"] = self.serializers
             if self.deserializers is not None and kwargs.get("deserializers") is None:
                 kwargs["deserializers"] = self.deserializers
+            comm = None
             try:
                 comm = await self.live_comm()
                 comm.name = "rpc." + key
                 result = await send_recv(comm=comm, op=key, **kwargs)
             except (RPCClosed, CommClosedError) as e:
-                raise e.__class__(f"{e}: while trying to call remote method {key!r}")
+                if comm:
+                    raise type(e)(
+                        f"Exception while trying to call remote method {key!r} before comm was established."
+                    ) from e
+                else:
+                    raise type(e)(
+                        f"Exception while trying to call remote method {key!r} using comm {comm!r}."
+                    ) from e
 
             self.comms[comm] = True  # mark as open
             return result
@@ -1020,7 +1028,7 @@ async def connect(self, addr, timeout=None):
         try:
             if self.status != Status.running:
                 raise CommClosedError(
-                    f"ConnectionPool not running.  Status: {self.status}"
+                    f"ConnectionPool not running. Status: {self.status}"
                 )
 
             fut = asyncio.ensure_future(
@@ -1044,7 +1052,7 @@ async def connect(self, addr, timeout=None):
         except asyncio.CancelledError as exc:
             self.semaphore.release()
             raise CommClosedError(
-                f"ConnectionPool not running.  Status: {self.status}"
+                f"ConnectionPool not running. Status: {self.status}"
             ) from exc
         except Exception as exc:
             self.semaphore.release()
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index a2d358da04d..4be9d9be602 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -5203,7 +5203,9 @@ def report(self, msg: dict, ts: TaskState = None, client: str = None):
                 # logger.debug("Scheduler sends message to client %s", msg)
             except CommClosedError:
                 if self.status == Status.running:
-                    logger.critical("Tried writing to closed comm: %s", msg)
+                    logger.critical(
+                        "Closed comm %r while trying to write %s", c, msg, exc_info=True
+                    )
 
     async def add_client(self, comm, client=None, versions=None):
         """Add client to network
@@ -5504,7 +5506,9 @@ def client_send(self, client, msg):
             c.send(msg)
         except CommClosedError:
             if self.status == Status.running:
-                logger.critical("Tried writing to closed comm: %s", msg)
+                logger.critical(
+                    "Closed comm %r while trying to write %s", c, msg, exc_info=True
+                )
 
     def send_all(self, client_msgs: dict, worker_msgs: dict):
         """Send messages to client and workers"""
@@ -5520,7 +5524,12 @@ def send_all(self, client_msgs: dict, worker_msgs: dict):
                 c.send(*msgs)
             except CommClosedError:
                 if self.status == Status.running:
-                    logger.critical("Tried writing to closed comm: %s", msgs)
+                    logger.critical(
+                        "Closed comm %r while trying to write %s",
+                        c,
+                        msgs,
+                        exc_info=True,
+                    )
 
         for worker, msgs in worker_msgs.items():
             try:
diff --git a/distributed/stealing.py b/distributed/stealing.py
index e3398b4c9a1..1c45e1cfb9f 100644
--- a/distributed/stealing.py
+++ b/distributed/stealing.py
@@ -177,7 +177,7 @@ def move_task_request(self, ts, victim, thief):
             self.in_flight_occupancy[victim] -= victim_duration
             self.in_flight_occupancy[thief] += thief_duration
         except CommClosedError:
-            logger.info("Worker comm closed while stealing: %s", victim)
+            logger.info("Worker comm %r closed while stealing: %r", victim, ts)
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
diff --git a/distributed/tests/test_core.py b/distributed/tests/test_core.py
index 85c4550e467..78e5b7cc5a6 100644
--- a/distributed/tests/test_core.py
+++ b/distributed/tests/test_core.py
@@ -648,7 +648,7 @@ async def connect_to_server():
     close_fut = asyncio.create_task(pool.close())
 
     with pytest.raises(
-        CommClosedError, match="ConnectionPool not running.  Status: Status.closed"
+        CommClosedError, match="ConnectionPool not running. Status: Status.closed"
     ):
         await asyncio.gather(*tasks)
 
diff --git a/distributed/worker.py b/distributed/worker.py
index ef32a15e605..c1f3a0c47d2 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -1020,7 +1020,7 @@ async def heartbeat(self):
             self.bandwidth_workers.clear()
             self.bandwidth_types.clear()
         except CommClosedError:
-            logger.warning("Heartbeat to scheduler failed")
+            logger.warning("Heartbeat to scheduler failed", exc_info=True)
             if not self.reconnect:
                 await self.close(report=False)
         except OSError as e:

From ce2417bd0dcb44bcb582f0c473fdd9518b02cf8c Mon Sep 17 00:00:00 2001
From: Naty Clementi <natyclementi@gmail.com>
Date: Fri, 20 Aug 2021 16:23:22 -0400
Subject: [PATCH 1450/1550] Rename plots to fit in the labextension (#5239)

---
 distributed/dashboard/scheduler.py | 4 ++--
 docs/source/http_services.rst      | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/distributed/dashboard/scheduler.py b/distributed/dashboard/scheduler.py
index 78286e22de6..c70d1d44ab2 100644
--- a/distributed/dashboard/scheduler.py
+++ b/distributed/dashboard/scheduler.py
@@ -69,13 +69,13 @@
     "/individual-workers": individual_doc(WorkerTable, 500),
     "/individual-bandwidth-types": individual_doc(BandwidthTypes, 500),
     "/individual-bandwidth-workers": individual_doc(BandwidthWorkers, 500),
-    "/individual-workers-network-bandwidth": individual_doc(
+    "/individual-workers-network": individual_doc(
         WorkerNetworkBandwidth, 500, fig_attr="bandwidth"
     ),
     "/individual-workers-disk": individual_doc(
         WorkerNetworkBandwidth, 500, fig_attr="disk"
     ),
-    "/individual-workers-network-bandwidth-timeseries": individual_doc(
+    "/individual-workers-network-timeseries": individual_doc(
         SystemTimeseries, 500, fig_attr="bandwidth"
     ),
     "/individual-workers-cpu-timeseries": individual_doc(
diff --git a/docs/source/http_services.rst b/docs/source/http_services.rst
index dee90e448ad..119733197f5 100644
--- a/docs/source/http_services.rst
+++ b/docs/source/http_services.rst
@@ -65,9 +65,9 @@ Individual bokeh plots
 - ``/individual-workers``
 - ``/individual-bandwidth-types``
 - ``/individual-bandwidth-workers``
-- ``/individual-workers-network-bandwidth``
+- ``/individual-workers-network``
 - ``/individual-workers-disk``
-- ``/individual-workers-network-bandwidth-timeseries``
+- ``/individual-workers-network-timeseries``
 - ``/individual-workers-cpu-timeseries``
 - ``/individual-workers-memory-timeseries``
 - ``/individual-workers-disk-timeseries``

From 591bca00af4fc07d0c5cac5189fc3b08ef8a93cd Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Fri, 20 Aug 2021 15:52:44 -0500
Subject: [PATCH 1451/1550] bump version to 2021.08.1

---
 docs/source/changelog.rst | 23 +++++++++++++++++++++++
 requirements.txt          |  2 +-
 2 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index 61ee2999039..b183276a87c 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,29 @@
 Changelog
 =========
 
+2021.08.1
+---------
+
+Released on August 20, 2021
+
+- Rename plots to fit in the labextension (:pr:`5239`) `Naty Clementi`_
+- Log messages for ``CommClosedError`` now includes information about remote address (:pr:`5209`) `Florian Jetter`_
+- Add ``target='_blank'`` for redirects of dashboard link (:pr:`5237`) `Naty Clementi`_
+- Update computation code retrieval logic (:pr:`5236`) `James Bourbeau`_
+- Minor polish on cfexecutor (:pr:`5233`) `crusaderky`_
+- Use development version of ``dask`` in gpuCI build (:pr:`5232`) `James Bourbeau`_
+- Use upstream ``dask.widgets`` (:pr:`5205`) `Jacob Tomlinson`_
+- Fix flaky ``test_worker_reconnects_mid_compute`` (:pr:`5227`) `Florian Jetter`_
+- Update ``WorkerPlugin`` docstring about usage of ``TaskState`` objects (:pr:`5226`) `Florian Jetter`_
+- Worker Network Timeseries (:pr:`5129`) `Naty Clementi`_
+- Add HTML Repr for ``ProcessInterface`` class and all its subclasses (:pr:`5181`) `Freyam Mehta`_
+- Fix an issue where a reconnecting worker could cause an invalid transition (:pr:`5210`) `Florian Jetter`_
+- Minor fixes for cfexecutor (:pr:`5177`) `Florian Jetter`_
+- Add HTML Repr for ``Security`` class (:pr:`5178`) `Freyam Mehta`_
+- Fix performance report sizing issue (:pr:`5213`) `Ian Rose`_
+- Drop RMM compatibility code from RAPIDS < 0.11 (:pr:`5214`) `Peter Andreas Entschev`_
+
+
 2021.08.0
 ---------
 
diff --git a/requirements.txt b/requirements.txt
index db69c79c6a6..b1542068ace 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,6 @@
 click >= 6.6
 cloudpickle >= 1.5.0
-dask == 2021.08.0
+dask == 2021.08.1
 jinja2
 msgpack >= 0.6.0
 psutil >= 5.0

From ba4701d18a2ed349a3fcab828b92edbab319e638 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sun, 22 Aug 2021 14:18:43 -0500
Subject: [PATCH 1452/1550] Use non-histogram plots up to 100 workers (#5249)

---
 distributed/dashboard/components/scheduler.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index d67885405c8..27cdf9b2167 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -196,7 +196,7 @@ def __init__(self, scheduler, **kwargs):
             )
 
             self.root = figure(
-                title="Tasks Processing (Histogram)",
+                title="Tasks Processing (count)",
                 id="bk-nprocessing-histogram-plot",
                 name="processing",
                 y_axis_label="frequency",
@@ -523,7 +523,7 @@ def __init__(self, scheduler, **kwargs):
             )
 
             self.root = figure(
-                title="Bytes stored per worker (Histogram)",
+                title="Bytes stored per worker",
                 name="workers_memory",
                 id="bk-workers-memory-histogram-plot",
                 y_axis_label="frequency",
@@ -3144,7 +3144,7 @@ def status_doc(scheduler, extra, doc):
         add_periodic_callback(doc, cluster_memory, 100)
         doc.add_root(cluster_memory.root)
 
-        if len(scheduler.workers) < 50:
+        if len(scheduler.workers) <= 100:
             workers_memory = WorkersMemory(scheduler, sizing_mode="stretch_both")
             processing = CurrentLoad(scheduler, sizing_mode="stretch_both")
 

From c168c2036165dd289920c34354fa66a255f9fb7f Mon Sep 17 00:00:00 2001
From: David Chudzicki <dchudz@gmail.com>
Date: Sun, 22 Aug 2021 15:19:02 -0400
Subject: [PATCH 1453/1550] Fix "schedulers" -> "scheduler" typo in docs
 (#5246)

---
 docs/source/scheduling-state.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/source/scheduling-state.rst b/docs/source/scheduling-state.rst
index 665d7c7fe2d..9ccc1e95094 100644
--- a/docs/source/scheduling-state.rst
+++ b/docs/source/scheduling-state.rst
@@ -11,7 +11,7 @@ The life of a computation with Dask can be described in the following stages:
 1.  The user authors a graph using some library, perhaps dask.delayed or
     dask.dataframe or the ``submit/map`` functions on the client.  They submit
     these tasks to the scheduler.
-2.  The schedulers assimilates these tasks into its graph of all tasks to
+2.  The scheduler assimilates these tasks into its graph of all tasks to
     track, and as their dependencies become available it asks workers to run
     each of these tasks in turn.
 3.  The worker receives information about how to run the task, communicates

From 730ae4653fceff441a14462f5725b6021730f691 Mon Sep 17 00:00:00 2001
From: David Chudzicki <dchudz@gmail.com>
Date: Sun, 22 Aug 2021 15:19:13 -0400
Subject: [PATCH 1454/1550] Fix "fractiom" -> "fraction" typo in docstring
 (#5245)

---
 distributed/nanny.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/nanny.py b/distributed/nanny.py
index 65cb857174f..624f02b8aba 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -50,7 +50,7 @@ class Nanny(ServerNode):
     The nanny spins up Worker processes, watches then, and kills or restarts
     them as necessary. It is necessary if you want to use the
     ``Client.restart`` method, or to restart the worker automatically if
-    it gets to the terminate fractiom of its memory limit.
+    it gets to the terminate fraction of its memory limit.
 
     The parameters for the Nanny are mostly the same as those for the Worker
     with exceptions listed below.

From ee49807b7a5094856ac7b82d1b3e832e10b5160d Mon Sep 17 00:00:00 2001
From: David Chudzicki <dchudz@gmail.com>
Date: Sun, 22 Aug 2021 15:19:23 -0400
Subject: [PATCH 1455/1550] Fix typo (remove extra verb "creates") in docs
 (#5244)

---
 docs/source/task-launch.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/source/task-launch.rst b/docs/source/task-launch.rst
index 291c87ba6c1..80b0c1f8c88 100644
--- a/docs/source/task-launch.rst
+++ b/docs/source/task-launch.rst
@@ -76,7 +76,7 @@ and so this only added a few milliseconds to our total processing time.
 Extended Example
 ~~~~~~~~~~~~~~~~
 
-Computing the Fibonacci numbers creates involves a recursive function. When the
+Computing the Fibonacci numbers involves a recursive function. When the
 function is run, it calls itself using values it computed. We will use this as
 an example throughout this documentation to illustrate different techniques of
 submitting tasks from tasks.

From c0b6dde267ff6dedfb79b73b10a36be28511c6a4 Mon Sep 17 00:00:00 2001
From: David Chudzicki <dchudz@gmail.com>
Date: Sun, 22 Aug 2021 15:19:36 -0400
Subject: [PATCH 1456/1550] Fix "then" -> "than" typo in docs (#5247)

---
 docs/source/efficiency.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/source/efficiency.rst b/docs/source/efficiency.rst
index ed3ad2428d5..c8e43a67f33 100644
--- a/docs/source/efficiency.rst
+++ b/docs/source/efficiency.rst
@@ -11,7 +11,7 @@ Leave data on the cluster
 
 Wait as long as possible to gather data locally.  If you want to ask a question
 of a large piece of data on the cluster it is often faster to submit a function
-onto that data then to bring the data down to your local computer.
+onto that data than to bring the data down to your local computer.
 
 
 For example if we have a numpy array on the cluster and we want to know its

From 9c92a61ef69df212e56f8bccb380cb872286ab19 Mon Sep 17 00:00:00 2001
From: Doug Davis <ddavis@ddavis.io>
Date: Sun, 22 Aug 2021 12:25:24 -0700
Subject: [PATCH 1457/1550] Refactor scheduler plugins; store in a dictionary
 (#5120)

Following up on some discussion in #4808, this converts scheduler plugin storage from a list to a dictionary (key: name, value: plugin instance)
---
 distributed/client.py                         |  23 ++--
 distributed/dashboard/components/scheduler.py |  28 ++---
 .../dashboard/tests/test_scheduler_bokeh.py   |   3 +-
 distributed/diagnostics/eventstream.py        |   1 +
 distributed/diagnostics/graph_layout.py       |   2 +
 distributed/diagnostics/plugin.py             |   9 +-
 distributed/diagnostics/progress.py           |   5 +-
 distributed/diagnostics/task_stream.py        |   2 +
 .../tests/test_scheduler_plugin.py            |  31 ++++-
 .../diagnostics/tests/test_task_stream.py     |   2 +-
 distributed/nanny.py                          |   4 +-
 .../protocol/tests/test_highlevelgraph.py     |   4 +-
 distributed/scheduler.py                      | 111 ++++++++++++++----
 distributed/stealing.py                       |   2 +-
 distributed/worker.py                         |   4 +-
 15 files changed, 164 insertions(+), 67 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index b228b494eae..921e4fa0db2 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -60,12 +60,7 @@
     connect,
     rpc,
 )
-from .diagnostics.plugin import (
-    NannyPlugin,
-    UploadFile,
-    WorkerPlugin,
-    _get_worker_plugin_name,
-)
+from .diagnostics.plugin import NannyPlugin, UploadFile, WorkerPlugin, _get_plugin_name
 from .metrics import time
 from .objects import HasWhat, SchedulerInfo, WhoHas
 from .protocol import to_serialize
@@ -3969,15 +3964,16 @@ async def _get_task_stream(
         else:
             return msgs
 
-    async def _register_scheduler_plugin(self, plugin, **kwargs):
+    async def _register_scheduler_plugin(self, plugin, name, **kwargs):
         if isinstance(plugin, type):
             plugin = plugin(**kwargs)
 
         return await self.scheduler.register_scheduler_plugin(
-            plugin=dumps(plugin, protocol=4)
+            plugin=dumps(plugin, protocol=4),
+            name=name,
         )
 
-    def register_scheduler_plugin(self, plugin, **kwargs):
+    def register_scheduler_plugin(self, plugin, name=None, **kwargs):
         """Register a scheduler plugin.
 
         See https://distributed.readthedocs.io/en/latest/plugins.html#scheduler-plugins
@@ -3986,14 +3982,21 @@ def register_scheduler_plugin(self, plugin, **kwargs):
         ----------
         plugin : SchedulerPlugin
             Plugin class or object to pass to the scheduler.
+        name : str
+            Name for the plugin; if None, a name is taken from the
+            plugin instance or automatically generated if not present.
         **kwargs : Any
             Arguments passed to the Plugin class (if Plugin is an
             instance kwargs are unused).
 
         """
+        if name is None:
+            name = _get_plugin_name(plugin)
+
         return self.sync(
             self._register_scheduler_plugin,
             plugin=plugin,
+            name=name,
             **kwargs,
         )
 
@@ -4103,7 +4106,7 @@ class will be instantiated with any extra keyword arguments.
             plugin = plugin(**kwargs)
 
         if name is None:
-            name = _get_worker_plugin_name(plugin)
+            name = _get_plugin_name(plugin)
 
         assert name
 
diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index 27cdf9b2167..c9dddf1ec56 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -1078,11 +1078,10 @@ def __init__(self, scheduler, **kwargs):
             self.last = 0
             self.scheduler = scheduler
 
-            es = [p for p in self.scheduler.plugins if isinstance(p, TaskStreamPlugin)]
-            if not es:
-                self.plugin = TaskStreamPlugin(self.scheduler)
-            else:
-                self.plugin = es[0]
+            if TaskStreamPlugin.name not in self.scheduler.plugins:
+                self.scheduler.add_plugin(
+                    plugin=TaskStreamPlugin,
+                )
 
             compute_data = {
                 "times": [0.2, 0.1],
@@ -1243,11 +1242,8 @@ def __init__(self, scheduler, **kwargs):
             self.last = 0
             self.scheduler = scheduler
 
-            es = [p for p in self.scheduler.plugins if isinstance(p, TaskStreamPlugin)]
-            if not es:
-                self.plugin = TaskStreamPlugin(self.scheduler)
-            else:
-                self.plugin = es[0]
+            if TaskStreamPlugin.name not in self.scheduler.plugins:
+                self.scheduler.add_plugin(plugin=TaskStreamPlugin)
 
             action_data = {
                 "times": [0.2, 0.1],
@@ -1771,11 +1767,13 @@ class TaskStream(DashboardComponent):
     def __init__(self, scheduler, n_rectangles=1000, clear_interval="20s", **kwargs):
         self.scheduler = scheduler
         self.offset = 0
-        es = [p for p in self.scheduler.plugins if isinstance(p, TaskStreamPlugin)]
-        if not es:
-            self.plugin = TaskStreamPlugin(self.scheduler)
-        else:
-            self.plugin = es[0]
+
+        if TaskStreamPlugin.name not in self.scheduler.plugins:
+            self.scheduler.add_plugin(
+                plugin=TaskStreamPlugin,
+            )
+        self.plugin = self.scheduler.plugins[TaskStreamPlugin.name]
+
         self.index = max(0, self.plugin.index - n_rectangles)
         self.workers = dict()
         self.n_rectangles = n_rectangles
diff --git a/distributed/dashboard/tests/test_scheduler_bokeh.py b/distributed/dashboard/tests/test_scheduler_bokeh.py
index 3304aa0c6a1..7127170df30 100644
--- a/distributed/dashboard/tests/test_scheduler_bokeh.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh.py
@@ -43,6 +43,7 @@
 )
 from distributed.dashboard.components.worker import Counters
 from distributed.dashboard.scheduler import applications
+from distributed.diagnostics.task_stream import TaskStreamPlugin
 from distributed.metrics import time
 from distributed.utils import format_dashboard_link
 from distributed.utils_test import dec, div, gen_cluster, get_cert, inc, slowinc
@@ -857,7 +858,7 @@ async def test_lots_of_tasks(c, s, a, b):
     futures = c.map(toolz.identity, range(100))
     await wait(futures)
 
-    tsp = [p for p in s.plugins if "taskstream" in type(p).__name__.lower()][0]
+    tsp = s.plugins[TaskStreamPlugin.name]
     assert len(tsp.buffer) == 10
     ts.update()
     assert len(ts.source.data["start"]) == 10
diff --git a/distributed/diagnostics/eventstream.py b/distributed/diagnostics/eventstream.py
index 34805e9085a..1f4c096f7e0 100644
--- a/distributed/diagnostics/eventstream.py
+++ b/distributed/diagnostics/eventstream.py
@@ -11,6 +11,7 @@ class EventStream(SchedulerPlugin):
     """Maintain a copy of worker events"""
 
     def __init__(self, scheduler=None):
+        self.name = "EventStream"
         self.buffer = []
         if scheduler:
             scheduler.add_plugin(self)
diff --git a/distributed/diagnostics/graph_layout.py b/distributed/diagnostics/graph_layout.py
index 4a4eb483bfd..10f5dd8058a 100644
--- a/distributed/diagnostics/graph_layout.py
+++ b/distributed/diagnostics/graph_layout.py
@@ -11,6 +11,8 @@ class GraphLayout(SchedulerPlugin):
     is rendered at /graph on the diagnostic dashboard.
     """
 
+    name = "graph-layout"
+
     def __init__(self, scheduler):
         self.x = {}
         self.y = {}
diff --git a/distributed/diagnostics/plugin.py b/distributed/diagnostics/plugin.py
index ba2b031e5ce..233ff626374 100644
--- a/distributed/diagnostics/plugin.py
+++ b/distributed/diagnostics/plugin.py
@@ -215,9 +215,12 @@ def teardown(self, nanny):
         """Run when the nanny to which the plugin is attached to is closed"""
 
 
-def _get_worker_plugin_name(plugin) -> str:
-    """Returns the worker plugin name. If plugin has no name attribute
-    a random name is used."""
+def _get_plugin_name(plugin) -> str:
+    """Return plugin name.
+
+    If plugin has no name attribute a random name is used.
+
+    """
     if hasattr(plugin, "name"):
         return plugin.name
     else:
diff --git a/distributed/diagnostics/progress.py b/distributed/diagnostics/progress.py
index f126dce0a95..c386c865dce 100644
--- a/distributed/diagnostics/progress.py
+++ b/distributed/diagnostics/progress.py
@@ -61,6 +61,7 @@ class Progress(SchedulerPlugin):
     """
 
     def __init__(self, keys, scheduler, minimum=0, dt=0.1, complete=False):
+        self.name = "Progress"
         self.keys = {k.key if hasattr(k, "key") else k for k in keys}
         self.keys = {stringify(k) for k in self.keys}
         self.scheduler = scheduler
@@ -120,8 +121,8 @@ def restart(self, scheduler):
         self.stop()
 
     def stop(self, exception=None, key=None):
-        if self in self.scheduler.plugins:
-            self.scheduler.plugins.remove(self)
+        if self.name in self.scheduler.plugins:
+            self.scheduler.remove_plugin(name=self.name)
         if exception:
             self.status = "error"
             self.extra.update(
diff --git a/distributed/diagnostics/task_stream.py b/distributed/diagnostics/task_stream.py
index fab787ae07c..120be52a38e 100644
--- a/distributed/diagnostics/task_stream.py
+++ b/distributed/diagnostics/task_stream.py
@@ -13,6 +13,8 @@
 
 
 class TaskStreamPlugin(SchedulerPlugin):
+    name = "task-stream"
+
     def __init__(self, scheduler, maxlen=None):
         if maxlen is None:
             maxlen = max(
diff --git a/distributed/diagnostics/tests/test_scheduler_plugin.py b/distributed/diagnostics/tests/test_scheduler_plugin.py
index 75cd31f22f6..95b74f252eb 100644
--- a/distributed/diagnostics/tests/test_scheduler_plugin.py
+++ b/distributed/diagnostics/tests/test_scheduler_plugin.py
@@ -9,7 +9,7 @@ async def test_simple(c, s, a, b):
     class Counter(SchedulerPlugin):
         def start(self, scheduler):
             self.scheduler = scheduler
-            scheduler.add_plugin(self)
+            scheduler.add_plugin(self, name="counter")
             self.count = 0
 
         def transition(self, key, start, finish, *args, **kwargs):
@@ -18,7 +18,7 @@ def transition(self, key, start, finish, *args, **kwargs):
 
     counter = Counter()
     counter.start(s)
-    assert counter in s.plugins
+    assert counter in s.plugins.values()
 
     assert counter.count == 0
 
@@ -29,7 +29,7 @@ def transition(self, key, start, finish, *args, **kwargs):
     await z
 
     assert counter.count == 3
-    s.remove_plugin(counter)
+    s.remove_plugin(name="counter")
     assert counter not in s.plugins
 
 
@@ -38,6 +38,8 @@ async def test_add_remove_worker(s):
     events = []
 
     class MyPlugin(SchedulerPlugin):
+        name = "MyPlugin"
+
         def add_worker(self, worker, scheduler):
             assert scheduler is s
             events.append(("add_worker", worker))
@@ -76,6 +78,8 @@ async def test_async_add_remove_worker(s):
     events = []
 
     class MyPlugin(SchedulerPlugin):
+        name = "MyPlugin"
+
         async def add_worker(self, worker, scheduler):
             assert scheduler is s
             events.append(("add_worker", worker))
@@ -105,6 +109,19 @@ async def remove_worker(self, worker, scheduler):
         pass
     assert events == []
 
+    class UnnamedPlugin(SchedulerPlugin):
+        async def start(self, scheduler):
+            self.scheduler = scheduler
+
+    plugin = UnnamedPlugin()
+    s.add_plugin(plugin)
+    s.add_plugin(plugin, name="another")
+    with pytest.raises(ValueError) as excinfo:
+        s.remove_plugin(plugin)
+
+    msg = str(excinfo.value)
+    assert "Multiple instances of" in msg
+
 
 @gen_test()
 async def test_lifecycle():
@@ -130,6 +147,8 @@ async def close(self):
 @gen_cluster(client=True)
 async def test_register_scheduler_plugin(c, s, a, b):
     class Dummy1(SchedulerPlugin):
+        name = "Dummy1"
+
         def start(self, scheduler):
             scheduler.foo = "bar"
 
@@ -137,7 +156,13 @@ def start(self, scheduler):
     await c.register_scheduler_plugin(Dummy1)
     assert s.foo == "bar"
 
+    with pytest.warns(UserWarning) as w:
+        await c.register_scheduler_plugin(Dummy1)
+    assert "Scheduler already contains" in w[0].message.args[0]
+
     class Dummy2(SchedulerPlugin):
+        name = "Dummy2"
+
         def start(self, scheduler):
             raise RuntimeError("raising in start method")
 
diff --git a/distributed/diagnostics/tests/test_task_stream.py b/distributed/diagnostics/tests/test_task_stream.py
index cdee6b0fc96..c65fdb5b573 100644
--- a/distributed/diagnostics/tests/test_task_stream.py
+++ b/distributed/diagnostics/tests/test_task_stream.py
@@ -88,7 +88,7 @@ async def test_client(c, s, a, b):
     futures = c.map(slowinc, range(10), delay=0.1)
     await wait(futures)
 
-    tasks = [p for p in s.plugins if isinstance(p, TaskStreamPlugin)][0]
+    tasks = s.plugins[TaskStreamPlugin.name]
     L = await c.get_task_stream()
     assert L == tuple(tasks.buffer)
 
diff --git a/distributed/nanny.py b/distributed/nanny.py
index 624f02b8aba..a981b8d479b 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -24,7 +24,7 @@
 from .comm import get_address_host, unparse_host_port
 from .comm.addressing import address_from_user_args
 from .core import CommClosedError, RPCClosed, Status, coerce_to_address, error_message
-from .diagnostics.plugin import _get_worker_plugin_name
+from .diagnostics.plugin import _get_plugin_name
 from .node import ServerNode
 from .process import AsyncProcess
 from .proctitle import enable_proctitle_on_children
@@ -427,7 +427,7 @@ async def plugin_add(self, comm=None, plugin=None, name=None):
                 plugin = pickle.loads(plugin)
 
             if name is None:
-                name = _get_worker_plugin_name(plugin)
+                name = _get_plugin_name(plugin)
 
             assert name
 
diff --git a/distributed/protocol/tests/test_highlevelgraph.py b/distributed/protocol/tests/test_highlevelgraph.py
index df9f2942179..518dceb66f6 100644
--- a/distributed/protocol/tests/test_highlevelgraph.py
+++ b/distributed/protocol/tests/test_highlevelgraph.py
@@ -136,7 +136,7 @@ def fn(k):
     plugin = ExampleAnnotationPlugin(priority_fn=fn, qux=qux, resource=resource)
     s.add_plugin(plugin)
 
-    assert plugin in s.plugins
+    assert plugin in s.plugins.values()
 
     with dask.annotate(priority=fn, qux=qux):
         A = da.ones((10, 10), chunks=(2, 2))
@@ -161,7 +161,7 @@ async def test_dataframe_annotations(c, s, a, b):
     plugin = ExampleAnnotationPlugin(retries=retries)
     s.add_plugin(plugin)
 
-    assert plugin in s.plugins
+    assert plugin in s.plugins.values()
 
     df = dd.from_pandas(
         pd.DataFrame(
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 4be9d9be602..63a7191cb3f 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -51,7 +51,7 @@
 )
 from .comm.addressing import addresses_from_user_args
 from .core import CommClosedError, Status, clean_exception, rpc, send_recv
-from .diagnostics.plugin import SchedulerPlugin
+from .diagnostics.plugin import SchedulerPlugin, _get_plugin_name
 from .event import EventExtension
 from .http import get_handlers
 from .lock import LockExtension
@@ -2245,7 +2245,7 @@ def _transition(self, key, finish: str, *args, **kwargs):
                     ts._dependents = dependents
                     ts._dependencies = dependencies
                     parent._tasks[ts._key] = ts
-                for plugin in list(self.plugins):
+                for plugin in self.plugins.values():
                     try:
                         plugin.transition(key, start, finish2, *args, **kwargs)
                     except Exception:
@@ -3695,7 +3695,7 @@ def __init__(
             aliases,
         ]
 
-        self.plugins = list(plugins)
+        self.plugins = {} if not plugins else {_get_plugin_name(p): p for p in plugins}
         self.transition_log = deque(
             maxlen=dask.config.get("distributed.scheduler.transition-log-length")
         )
@@ -3949,7 +3949,7 @@ def del_scheduler_file():
         for preload in self.preloads:
             await preload.start()
 
-        await asyncio.gather(*[plugin.start(self) for plugin in self.plugins])
+        await asyncio.gather(*[plugin.start(self) for plugin in self.plugins.values()])
 
         self.start_periodic_callbacks()
 
@@ -3988,7 +3988,7 @@ async def close(self, comm=None, fast=False, close_workers=False):
                 else:
                     break
 
-        await asyncio.gather(*[plugin.close() for plugin in self.plugins])
+        await asyncio.gather(*[plugin.close() for plugin in self.plugins.values()])
 
         for pc in self.periodic_callbacks.values():
             pc.stop()
@@ -4239,7 +4239,7 @@ async def add_worker(
             if ws._nthreads > len(ws._processing):
                 parent._idle[ws._address] = ws
 
-            for plugin in self.plugins[:]:
+            for plugin in self.plugins.values():
                 try:
                     result = plugin.add_worker(scheduler=self, worker=address)
                     if inspect.isawaitable(result):
@@ -4653,7 +4653,7 @@ def update_graph(
                     recommendations[ts._key] = "erred"
                     break
 
-        for plugin in self.plugins[:]:
+        for plugin in self.plugins.values():
             try:
                 plugin.update_graph(
                     self,
@@ -4916,7 +4916,7 @@ async def remove_worker(self, comm=None, address=None, safe=False, close=True):
 
             self.transitions(recommendations)
 
-            for plugin in self.plugins[:]:
+            for plugin in self.plugins.values():
                 try:
                     result = plugin.remove_worker(scheduler=self, worker=address)
                     if inspect.isawaitable(result):
@@ -5219,7 +5219,7 @@ async def add_client(self, comm, client=None, versions=None):
         self.log_event(["all", client], {"action": "add-client", "client": client})
         parent._clients[client] = ClientState(client, versions=versions)
 
-        for plugin in self.plugins[:]:
+        for plugin in self.plugins.values():
             try:
                 plugin.add_client(scheduler=self, client=client)
             except Exception as e:
@@ -5274,7 +5274,7 @@ def remove_client(self, client=None):
             )
             del parent._clients[client]
 
-            for plugin in self.plugins[:]:
+            for plugin in self.plugins.values():
                 try:
                     plugin.remove_client(scheduler=self, client=client)
                 except Exception as e:
@@ -5448,25 +5448,83 @@ async def handle_worker(self, comm=None, worker=None):
                 worker_comm.abort()
                 await self.remove_worker(address=worker)
 
-    def add_plugin(self, plugin=None, idempotent=False, **kwargs):
-        """
-        Add external plugin to scheduler
+    def add_plugin(self, plugin=None, idempotent=False, name=None, **kwargs):
+        """Add external plugin to scheduler.
 
         See https://distributed.readthedocs.io/en/latest/plugins.html
+
+        Paramters
+        ---------
+        plugin : SchedulerPlugin
+            SchedulerPlugin class to add (can also be an instance)
+        idempotent : bool
+            If true, the plugin is assumed to already exist and no
+            action is taken.
+        name : str
+            A name for the plugin, if None, the name attribute is
+            checked on the Plugin instance and generated if not
+            discovered.
+        **kwargs
+            Additional arguments passed to the `plugin` class if it is
+            not already an instance.
+
         """
         if isinstance(plugin, type):
             plugin = plugin(self, **kwargs)
 
-        if idempotent and any(isinstance(p, type(plugin)) for p in self.plugins):
+        if name is None:
+            name = _get_plugin_name(plugin)
+
+        if name in self.plugins:
+            warnings.warn(
+                f"Scheduler already contains a plugin with name {name}; "
+                "overwriting.",
+                category=UserWarning,
+            )
+
+        if idempotent and name in self.plugins:
             return
 
-        self.plugins.append(plugin)
+        self.plugins[name] = plugin
+
+    def remove_plugin(self, plugin=None, name=None):
+        """Remove external plugin from scheduler
+
+        Paramters
+        ---------
+        plugin : SchedulerPlugin
+            Deprecated; use `name` argument instead. Instance of a
+            SchedulerPlugin class to remove;
+        name : str
+            Name of the plugin to remove
 
-    def remove_plugin(self, plugin):
-        """Remove external plugin from scheduler"""
-        self.plugins.remove(plugin)
+        """
+        if plugin is not None:
+            warnings.warn(
+                "Removing scheduler plugins by value is deprecated and will be disabled "
+                "in a future release. Please remove scheduler plugins by name instead.",
+                category=FutureWarning,
+            )
+        if name is not None:
+            self.plugins.pop(name)
+        elif hasattr(plugin, "name"):
+            self.plugins.pop(plugin.name)
+        else:
+            # TODO: Remove this block of code once removing plugins by value is disabled
+            if plugin in self.plugins.values():
+                if sum(plugin is p for p in self.plugins.values()) > 1:
+                    raise ValueError(
+                        f"Multiple instances of {plugin} were found in the current scheduler "
+                        "plugins, we cannot remove this plugin."
+                    )
+                else:
+                    warnings.warn(
+                        "Removing scheduler plugins by value is deprecated and will be disabled "
+                        "in a future release. Please remove scheduler plugins by name instead.",
+                        category=FutureWarning,
+                    )
 
-    async def register_scheduler_plugin(self, comm=None, plugin=None):
+    async def register_scheduler_plugin(self, comm=None, plugin=None, name=None):
         """Register a plugin on the scheduler."""
         if not dask.config.get("distributed.scheduler.pickle"):
             raise ValueError(
@@ -5482,7 +5540,7 @@ async def register_scheduler_plugin(self, comm=None, plugin=None):
             if inspect.isawaitable(result):
                 result = await result
 
-        self.add_plugin(plugin=plugin)
+        self.add_plugin(plugin=plugin, name=name)
 
     def worker_send(self, worker, msg):
         """Send message to worker
@@ -5697,7 +5755,7 @@ async def restart(self, client=None, timeout=30):
 
             self.clear_task_state()
 
-            for plugin in self.plugins[:]:
+            for plugin in self.plugins.values():
                 try:
                     plugin.restart(self)
                 except Exception as e:
@@ -6950,9 +7008,12 @@ def get_task_status(self, comm=None, keys=None):
     def get_task_stream(self, comm=None, start=None, stop=None, count=None):
         from distributed.diagnostics.task_stream import TaskStreamPlugin
 
-        self.add_plugin(TaskStreamPlugin, idempotent=True)
-        tsp = [p for p in self.plugins if isinstance(p, TaskStreamPlugin)][0]
-        return tsp.collect(start=start, stop=stop, count=count)
+        if TaskStreamPlugin.name not in self.plugins:
+            self.add_plugin(TaskStreamPlugin)
+
+        plugin = self.plugins[TaskStreamPlugin.name]
+
+        return plugin.collect(start=start, stop=stop, count=count)
 
     def start_task_metadata(self, comm=None, name=None):
         plugin = CollectTaskMetaDataPlugin(scheduler=self, name=name)
@@ -6962,7 +7023,7 @@ def start_task_metadata(self, comm=None, name=None):
     def stop_task_metadata(self, comm=None, name=None):
         plugins = [
             p
-            for p in self.plugins
+            for p in self.plugins.values()
             if isinstance(p, CollectTaskMetaDataPlugin) and p.name == name
         ]
         if len(plugins) != 1:
diff --git a/distributed/stealing.py b/distributed/stealing.py
index 1c45e1cfb9f..61858d015b0 100644
--- a/distributed/stealing.py
+++ b/distributed/stealing.py
@@ -46,7 +46,7 @@ def __init__(self, scheduler):
         pc = PeriodicCallback(callback=self.balance, callback_time=callback_time * 1000)
         self._pc = pc
         self.scheduler.periodic_callbacks["stealing"] = pc
-        self.scheduler.plugins.append(self)
+        self.scheduler.add_plugin(self)
         self.scheduler.extensions["stealing"] = self
         self.scheduler.events["stealing"] = deque(maxlen=100000)
         self.count = 0
diff --git a/distributed/worker.py b/distributed/worker.py
index c1f3a0c47d2..175d804ea3b 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -47,7 +47,7 @@
     send_recv,
 )
 from .diagnostics import nvml
-from .diagnostics.plugin import _get_worker_plugin_name
+from .diagnostics.plugin import _get_plugin_name
 from .diskutils import WorkSpace
 from .http import get_handlers
 from .metrics import time
@@ -2769,7 +2769,7 @@ async def plugin_add(self, comm=None, plugin=None, name=None):
                 plugin = pickle.loads(plugin)
 
             if name is None:
-                name = _get_worker_plugin_name(plugin)
+                name = _get_plugin_name(plugin)
 
             assert name
 

From 6dfe5fa89ee3e1493426b48217edb6787fd1f0f1 Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Sun, 22 Aug 2021 14:27:12 -0500
Subject: [PATCH 1458/1550] Overwrite worker plugins (#5248)

This changes behavior for WorkerPlugins to overwrite the previous plugin
if there is a colliding name.
---
 .../diagnostics/tests/test_worker_plugin.py   | 62 +++++++++++++------
 distributed/worker.py                         | 35 +++++------
 2 files changed, 58 insertions(+), 39 deletions(-)

diff --git a/distributed/diagnostics/tests/test_worker_plugin.py b/distributed/diagnostics/tests/test_worker_plugin.py
index 7ae01f09227..6d62f8103ea 100644
--- a/distributed/diagnostics/tests/test_worker_plugin.py
+++ b/distributed/diagnostics/tests/test_worker_plugin.py
@@ -185,24 +185,6 @@ async def test_dependent_tasks(c, s, w):
     await async_wait_for(lambda: not w.tasks, timeout=10)
 
 
-@gen_cluster(nthreads=[("127.0.0.1", 1)], client=True)
-async def test_registering_with_name_arg(c, s, w):
-    class FooWorkerPlugin:
-        def setup(self, worker):
-            if hasattr(worker, "foo"):
-                raise RuntimeError(f"Worker {worker.address} already has foo!")
-
-            worker.foo = True
-
-    responses = await c.register_worker_plugin(FooWorkerPlugin(), name="foo")
-    assert list(responses.values()) == [{"status": "OK"}]
-
-    async with Worker(s.address, loop=s.loop):
-        with pytest.warns(FutureWarning, match="worker plugin will be overwritten"):
-            responses = await c.register_worker_plugin(FooWorkerPlugin(), name="foo")
-        assert list(responses.values()) == [{"status": "repeat"}] * 2
-
-
 @gen_cluster(nthreads=[("127.0.0.1", 1)], client=True)
 async def test_empty_plugin(c, s, w):
     class EmptyPlugin:
@@ -219,3 +201,47 @@ class MyCustomPlugin(WorkerPlugin):
     await c.register_worker_plugin(MyCustomPlugin())
     assert len(w.plugins) == 1
     assert next(iter(w.plugins)).startswith("MyCustomPlugin-")
+
+
+@gen_cluster(nthreads=[("127.0.0.1", 1)], client=True)
+async def test_WorkerPlugin_overwrite(c, s, w):
+    class MyCustomPlugin(WorkerPlugin):
+        name = "custom"
+
+        def setup(self, worker):
+            self.worker = worker
+            self.worker.foo = 0
+
+        def transition(self, *args, **kwargs):
+            self.worker.foo = 123
+
+        def teardown(self, worker):
+            del self.worker.foo
+
+    await c.register_worker_plugin(MyCustomPlugin)
+
+    assert w.foo == 0
+
+    await c.submit(inc, 0)
+    assert w.foo == 123
+
+    class MyCustomPlugin(WorkerPlugin):
+        name = "custom"
+
+        def setup(self, worker):
+            self.worker = worker
+            self.worker.bar = 0
+
+        def transition(self, *args, **kwargs):
+            self.worker.bar = 456
+
+        def teardown(self, worker):
+            del self.worker.bar
+
+    await c.register_worker_plugin(MyCustomPlugin)
+
+    assert not hasattr(w, "foo")
+    assert w.bar == 0
+
+    await c.submit(inc, 0)
+    assert w.bar == 456
diff --git a/distributed/worker.py b/distributed/worker.py
index 175d804ea3b..0ba2daa8d84 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -2774,28 +2774,21 @@ async def plugin_add(self, comm=None, plugin=None, name=None):
             assert name
 
             if name in self.plugins:
-                warnings.warn(
-                    "Attempting to add a worker plugin with the same name as an already registered "
-                    f"plugin ({name}). Currently this results in no change and the previously registered "
-                    "plugin is not overwritten. This behavior is deprecated and in a future release "
-                    f"the previously registered {name} worker plugin will be overwritten.",
-                    category=FutureWarning,
-                )
-                return {"status": "repeat"}
-            else:
-                self.plugins[name] = plugin
+                await self.plugin_remove(comm=comm, name=name)
 
-                logger.info("Starting Worker plugin %s" % name)
-                if hasattr(plugin, "setup"):
-                    try:
-                        result = plugin.setup(worker=self)
-                        if isawaitable(result):
-                            result = await result
-                    except Exception as e:
-                        msg = error_message(e)
-                        return msg
-
-                return {"status": "OK"}
+            self.plugins[name] = plugin
+
+            logger.info("Starting Worker plugin %s" % name)
+            if hasattr(plugin, "setup"):
+                try:
+                    result = plugin.setup(worker=self)
+                    if isawaitable(result):
+                        result = await result
+                except Exception as e:
+                    msg = error_message(e)
+                    return msg
+
+            return {"status": "OK"}
 
     async def plugin_remove(self, comm=None, name=None):
         with log_errors(pdb=False):

From 17c08e5a7239b37114af7fd0c12e32b0634609e4 Mon Sep 17 00:00:00 2001
From: Walt Woods <woodswalben@gmail.com>
Date: Mon, 23 Aug 2021 03:32:39 -0700
Subject: [PATCH 1459/1550] =?UTF-8?q?Addressing=20#5159=20(IPv6=20dask-wor?=
 =?UTF-8?q?ker=20support)=20by=20propagating=20empty=20host=E2=80=A6=20(#5?=
 =?UTF-8?q?197)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Addressing #5159 (IPv6 dask-worker support) by propagating empty host correctly

* black / flake8 fixes

* Fixing IPv6 addresses being passed around; added tests

* Fixing rpartition differential; reverting test change

* Patching default address to match scheduler; fixing bug with fetching work from busy Worker

Prior to this commit, task results required would be fetch only from local workers if they were available. If all local workers were busy, but the work were available on another machine, this would result in an indefinite delay. This patch allows local workers to be temporarily rejected, allowing for remote workers to provide the data when all local workers are busy.

* Splitting out busy_workers into #5206

* Adding CLI test for IPv6 support

* Black fix

* Isort fix...

* Changing assertion to ValueError
---
 distributed/cli/dask_worker.py            |  5 +++-
 distributed/cli/tests/test_dask_worker.py | 35 +++++++++++++++++++++++
 distributed/comm/addressing.py            |  9 ++++--
 distributed/comm/tcp.py                   |  4 ++-
 distributed/comm/utils.py                 |  6 ++--
 distributed/tests/test_worker.py          | 20 +++++++++++++
 distributed/utils.py                      |  5 ++++
 distributed/worker.py                     | 25 ++++++++++------
 8 files changed, 93 insertions(+), 16 deletions(-)

diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
index d297ef57923..1f00a928ff6 100755
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -97,7 +97,7 @@
     "--listen-address",
     type=str,
     default=None,
-    help="The address to which the worker binds. Example: tcp://0.0.0.0:9000",
+    help="The address to which the worker binds. Example: tcp://0.0.0.0:9000 or tcp://:9000 for IPv4+IPv6",
 )
 @click.option(
     "--contact-address",
@@ -339,6 +339,9 @@ def main(
     try:
         if listen_address:
             (host, worker_port) = get_address_host_port(listen_address, strict=True)
+            if ":" in host:
+                # IPv6 -- bracket to pass as user args
+                host = f"[{host}]"
 
         if contact_address:
             # we only need this to verify it is getting parsed
diff --git a/distributed/cli/tests/test_dask_worker.py b/distributed/cli/tests/test_dask_worker.py
index d807791ea7d..bd54ba17500 100644
--- a/distributed/cli/tests/test_dask_worker.py
+++ b/distributed/cli/tests/test_dask_worker.py
@@ -6,6 +6,7 @@
 pytest.importorskip("requests")
 
 import os
+import socket
 from multiprocessing import cpu_count
 from time import sleep
 
@@ -306,6 +307,40 @@ def func(dask_worker):
                 assert client.run(func) == {"tcp://127.0.0.2:39837": listen_address}
 
 
+@pytest.mark.skipif(not socket.has_ipv6, reason="Needs IPv6 support to test")
+@pytest.mark.parametrize("nanny", ["--nanny", "--no-nanny"])
+@pytest.mark.parametrize("listen_address", ["tcp://:39838", "tcp://[::1]:39838"])
+def test_listen_address_ipv6(loop, nanny, listen_address):
+    with popen(["dask-scheduler", "--no-dashboard"]):
+        with popen(
+            [
+                "dask-worker",
+                "127.0.0.1:8786",
+                nanny,
+                "--no-dashboard",
+                "--listen-address",
+                listen_address,
+            ]
+        ):
+            # IPv4 used by default for name of global listener; IPv6 used by default when
+            # listening only on IPv6.
+            bind_all = "[::1]" not in listen_address
+            expected_ip = "127.0.0.1" if bind_all else "[::1]"
+            expected_name = f"tcp://{expected_ip}:39838"
+            expected_listen = "tcp://0.0.0.0:39838" if bind_all else listen_address
+            with Client("127.0.0.1:8786") as client:
+                while not client.nthreads():
+                    sleep(0.1)
+                info = client.scheduler_info()
+                assert expected_name in info["workers"]
+                assert client.submit(lambda x: x + 1, 10).result() == 11
+
+                def func(dask_worker):
+                    return dask_worker.listener.listen_address
+
+                assert client.run(func) == {expected_name: expected_listen}
+
+
 @pytest.mark.skipif(not LINUX, reason="Need 127.0.0.2 to mean localhost")
 @pytest.mark.parametrize("nanny", ["--nanny", "--no-nanny"])
 @pytest.mark.parametrize("host", ["127.0.0.2", "0.0.0.0"])
diff --git a/distributed/comm/addressing.py b/distributed/comm/addressing.py
index e51f2dfd71e..5e7152413b5 100644
--- a/distributed/comm/addressing.py
+++ b/distributed/comm/addressing.py
@@ -60,7 +60,9 @@ def parse_host_port(address, default_port=None):
         return address
 
     def _fail():
-        raise ValueError(f"invalid address {address!r}")
+        raise ValueError(
+            f"invalid address {address!r}; maybe: ipv6 needs brackets like [::1]"
+        )
 
     def _default():
         if default_port is None:
@@ -83,8 +85,9 @@ def _default():
             port = tail[1:]
     else:
         # Generic notation: 'addr:port' or 'addr'.
-        host, sep, port = address.partition(":")
+        host, sep, port = address.rpartition(":")
         if not sep:
+            host = port
             port = _default()
         elif ":" in host:
             _fail()
@@ -113,6 +116,8 @@ def get_address_host_port(addr, strict=False):
 
     >>> get_address_host_port('tcp://1.2.3.4:80')
     ('1.2.3.4', 80)
+    >>> get_address_host_port('tcp://[::1]:80')
+    ('::1', 80)
     """
     scheme, loc = parse_address(addr, strict=strict)
     backend = registry.get_backend(scheme)
diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py
index b4a2d39b2d8..f0e3d90a800 100644
--- a/distributed/comm/tcp.py
+++ b/distributed/comm/tcp.py
@@ -443,11 +443,13 @@ def __init__(
         comm_handler,
         deserialize=True,
         allow_offload=True,
+        default_host=None,
         default_port=0,
         **connection_args,
     ):
         self._check_encryption(address, connection_args)
         self.ip, self.port = parse_host_port(address, default_port)
+        self.default_host = default_host
         self.comm_handler = comm_handler
         self.deserialize = deserialize
         self.allow_offload = allow_offload
@@ -533,7 +535,7 @@ def contact_address(self):
         The contact address as a string.
         """
         host, port = self.get_host_port()
-        host = ensure_concrete_host(host)
+        host = ensure_concrete_host(host, default_host=self.default_host)
         return self.prefix + unparse_host_port(host, port)
 
 
diff --git a/distributed/comm/utils.py b/distributed/comm/utils.py
index 0ce4f8f891b..bc29c3d763e 100644
--- a/distributed/comm/utils.py
+++ b/distributed/comm/utils.py
@@ -113,14 +113,14 @@ def get_tcp_server_address(tcp_server):
     return get_tcp_server_addresses(tcp_server)[0]
 
 
-def ensure_concrete_host(host):
+def ensure_concrete_host(host, default_host=None):
     """
     Ensure the given host string (or IP) denotes a concrete host, not a
     wildcard listening address.
     """
     if host in ("0.0.0.0", ""):
-        return get_ip()
+        return default_host or get_ip()
     elif host == "::":
-        return get_ipv6()
+        return default_host or get_ipv6()
     else:
         return host
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 21920c44da7..791e1f146b6 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -1045,6 +1045,26 @@ async def test_service_hosts_match_worker(s):
         sock = first(w.http_server._sockets.values())
         assert sock.getsockname()[0] in ("::", "0.0.0.0")
 
+    # See what happens with e.g. `dask-worker --listen-address tcp://:8811`
+    async with Worker(s.address, host="") as w:
+        sock = first(w.http_server._sockets.values())
+        assert sock.getsockname()[0] in ("::", "0.0.0.0")
+        # Address must be a connectable address. 0.0.0.0 is not!
+        address_all = w.address.rsplit(":", 1)[0]
+        assert address_all in ("tcp://[::1]", "tcp://127.0.0.1")
+
+    # Check various malformed IPv6 addresses
+    # Since these hostnames get passed to distributed.comm.address_from_user_args,
+    # bracketing is mandatory for IPv6.
+    with pytest.raises(ValueError) as exc:
+        async with Worker(s.address, host="::") as w:
+            pass
+    assert "bracketed" in str(exc)
+    with pytest.raises(ValueError) as exc:
+        async with Worker(s.address, host="tcp://::1") as w:
+            pass
+    assert "bracketed" in str(exc)
+
 
 @gen_cluster(nthreads=[])
 async def test_start_services(s):
diff --git a/distributed/utils.py b/distributed/utils.py
index 5edcc51738d..3c7077b6e3a 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -675,9 +675,14 @@ def ensure_ip(hostname):
     --------
     >>> ensure_ip('localhost')
     '127.0.0.1'
+    >>> ensure_ip('')  # Maps as localhost for binding e.g. 'tcp://:8811'
+    '127.0.0.1'
     >>> ensure_ip('123.123.123.123')  # pass through IP addresses
     '123.123.123.123'
     """
+    if not hostname:
+        hostname = "localhost"
+
     # Prefer IPv4 over IPv6, for compatibility
     families = [socket.AF_INET, socket.AF_INET6]
     for fam in families:
diff --git a/distributed/worker.py b/distributed/worker.py
index 0ba2daa8d84..9c0421e8612 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -36,7 +36,7 @@
 from . import comm, preloading, profile, system, utils
 from .batched import BatchedSend
 from .comm import connect, get_address_host
-from .comm.addressing import address_from_user_args
+from .comm.addressing import address_from_user_args, parse_address
 from .comm.utils import OFFLOAD_THRESHOLD
 from .core import (
     CommClosedError,
@@ -560,13 +560,16 @@ def __init__(
             if len(protocol_address) == 2:
                 protocol = protocol_address[0]
 
-        # Target interface on which we contact the scheduler by default
-        # TODO: it is unfortunate that we special-case inproc here
-        if not host and not interface and not scheduler_addr.startswith("inproc://"):
-            host = get_ip(get_address_host(scheduler_addr))
-
         self._start_port = port
         self._start_host = host
+        if host:
+            # Helpful error message if IPv6 specified incorrectly
+            _, host_address = parse_address(host)
+            if host_address.count(":") > 1 and not host_address.startswith("["):
+                raise ValueError(
+                    "Host address with IPv6 must be bracketed like '[::1]'; "
+                    f"got {host_address}"
+                )
         self._interface = interface
         self._protocol = protocol
 
@@ -1152,10 +1155,14 @@ async def start(self):
                 protocol=self._protocol,
                 security=self.security,
             )
-            try:
-                await self.listen(
-                    start_address, **self.security.get_listen_args("worker")
+            kwargs = self.security.get_listen_args("worker")
+            if self._protocol in ("tcp", "tls"):
+                kwargs = kwargs.copy()
+                kwargs["default_host"] = get_ip(
+                    get_address_host(self.scheduler.address)
                 )
+            try:
+                await self.listen(start_address, **kwargs)
             except OSError as e:
                 if len(ports) > 1 and e.errno == errno.EADDRINUSE:
                     continue

From 4fdcb4f8b5cbcc819c5ad20e6d3421914a56ce2c Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Mon, 23 Aug 2021 22:24:05 +0100
Subject: [PATCH 1460/1550] Properly check for ipv6 availability (#5255)

---
 distributed/cli/tests/test_dask_worker.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/distributed/cli/tests/test_dask_worker.py b/distributed/cli/tests/test_dask_worker.py
index bd54ba17500..3b334fa0aed 100644
--- a/distributed/cli/tests/test_dask_worker.py
+++ b/distributed/cli/tests/test_dask_worker.py
@@ -6,7 +6,6 @@
 pytest.importorskip("requests")
 
 import os
-import socket
 from multiprocessing import cpu_count
 from time import sleep
 
@@ -18,7 +17,13 @@
 from distributed.deploy.utils import nprocesses_nthreads
 from distributed.metrics import time
 from distributed.utils import parse_ports, sync, tmpfile
-from distributed.utils_test import gen_cluster, popen, terminate_process, wait_for_port
+from distributed.utils_test import (
+    gen_cluster,
+    popen,
+    requires_ipv6,
+    terminate_process,
+    wait_for_port,
+)
 
 
 def test_nanny_worker_ports(loop):
@@ -307,7 +312,7 @@ def func(dask_worker):
                 assert client.run(func) == {"tcp://127.0.0.2:39837": listen_address}
 
 
-@pytest.mark.skipif(not socket.has_ipv6, reason="Needs IPv6 support to test")
+@requires_ipv6
 @pytest.mark.parametrize("nanny", ["--nanny", "--no-nanny"])
 @pytest.mark.parametrize("listen_address", ["tcp://:39838", "tcp://[::1]:39838"])
 def test_listen_address_ipv6(loop, nanny, listen_address):

From 09a89c5d2abfdf09b5f2e617aed170a3000e7cbf Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Mon, 23 Aug 2021 18:36:10 -0500
Subject: [PATCH 1461/1550] Remove `GroupProgress` scheduler plugin (#5256)

---
 distributed/diagnostics/progress.py           | 77 +------------------
 .../diagnostics/tests/test_progress.py        | 26 -------
 2 files changed, 1 insertion(+), 102 deletions(-)

diff --git a/distributed/diagnostics/progress.py b/distributed/diagnostics/progress.py
index c386c865dce..abdfc2cf96b 100644
--- a/distributed/diagnostics/progress.py
+++ b/distributed/diagnostics/progress.py
@@ -7,7 +7,7 @@
 
 from dask.utils import stringify
 
-from ..utils import key_split, key_split_group, log_errors
+from ..utils import key_split
 from .plugin import SchedulerPlugin
 
 logger = logging.getLogger(__name__)
@@ -285,78 +285,3 @@ def transition(self, key, start, finish, *args, **kwargs):
     def restart(self, scheduler):
         self.all.clear()
         self.state.clear()
-
-
-class GroupProgress(SchedulerPlugin):
-    """Keep track of all keys, grouped by key_split"""
-
-    def __init__(self, scheduler):
-        self.scheduler = scheduler
-        self.keys = dict()
-        self.groups = dict()
-        self.nbytes = dict()
-        self.durations = dict()
-        self.dependencies = defaultdict(set)
-        self.dependents = defaultdict(set)
-
-        for key, ts in self.scheduler.tasks.items():
-            k = key_split_group(key)
-            if k not in self.groups:
-                self.create(key, k)
-            self.keys[k].add(key)
-            self.groups[k][ts.state] += 1
-            if ts.state == "memory" and ts.nbytes >= 0:
-                self.nbytes[k] += ts.nbytes
-
-        scheduler.add_plugin(self)
-
-    def create(self, key, k):
-        with log_errors():
-            ts = self.scheduler.tasks[key]
-            g = {"memory": 0, "erred": 0, "waiting": 0, "released": 0, "processing": 0}
-            self.keys[k] = set()
-            self.groups[k] = g
-            self.nbytes[k] = 0
-            self.durations[k] = 0
-            self.dependents[k] = {key_split_group(dts.key) for dts in ts.dependents}
-            for dts in ts.dependencies:
-                d = key_split_group(dts.key)
-                self.dependents[d].add(k)
-                self.dependencies[k].add(d)
-
-    def transition(self, key, start, finish, *args, **kwargs):
-        with log_errors():
-            ts = self.scheduler.tasks[key]
-            k = key_split_group(key)
-            if k not in self.groups:
-                self.create(key, k)
-
-            g = self.groups[k]
-
-            if key not in self.keys[k]:
-                self.keys[k].add(key)
-            else:
-                g[start] -= 1
-
-            if finish != "forgotten":
-                g[finish] += 1
-            else:
-                self.keys[k].remove(key)
-                if not self.keys[k]:
-                    del self.groups[k]
-                    del self.nbytes[k]
-                    for dep in self.dependencies.pop(k):
-                        self.dependents[key_split_group(dep)].remove(k)
-
-            if start == "memory" and ts.nbytes >= 0:
-                self.nbytes[k] -= ts.nbytes
-            if finish == "memory" and ts.nbytes >= 0:
-                self.nbytes[k] += ts.nbytes
-
-    def restart(self, scheduler):
-        self.keys.clear()
-        self.groups.clear()
-        self.nbytes.clear()
-        self.durations.clear()
-        self.dependencies.clear()
-        self.dependents.clear()
diff --git a/distributed/diagnostics/tests/test_progress.py b/distributed/diagnostics/tests/test_progress.py
index f671c2e0681..8495e9a26aa 100644
--- a/distributed/diagnostics/tests/test_progress.py
+++ b/distributed/diagnostics/tests/test_progress.py
@@ -7,7 +7,6 @@
 from distributed.compatibility import LINUX
 from distributed.diagnostics.progress import (
     AllProgress,
-    GroupProgress,
     MultiProgress,
     Progress,
     SchedulerPlugin,
@@ -193,28 +192,3 @@ async def test_AllProgress_lost_key(c, s, a, b):
 
     while len(p.state["memory"]["inc"]) > 0:
         await asyncio.sleep(0.01)
-
-
-@gen_cluster(client=True)
-async def test_GroupProgress(c, s, a, b):
-    da = pytest.importorskip("dask.array")
-    fp = GroupProgress(s)
-    x = da.ones(100, chunks=10)
-    y = x + 1
-    z = (x * y).sum().persist(optimize_graph=False)
-
-    await wait(z)
-    assert 3 < len(fp.groups) < 10
-    for k, g in fp.groups.items():
-        assert fp.keys[k]
-        assert len(fp.keys[k]) == sum(g.values())
-        assert all(v >= 0 for v in g.values())
-
-    assert fp.dependencies[y.name] == {x.name}
-    assert fp.dependents[x.name] == {y.name, (x * y).name}
-
-    del x, y, z
-    while s.tasks:
-        await asyncio.sleep(0.01)
-
-    assert not fp.groups

From dc471862b7efc8473583e7e96b6fcaf8d5c7c527 Mon Sep 17 00:00:00 2001
From: Doug Davis <ddavis@ddavis.io>
Date: Tue, 24 Aug 2021 12:47:58 -0700
Subject: [PATCH 1462/1550] Avoid during-iteration scheduler plugin changes
 (#5259)

---
 distributed/scheduler.py | 28 ++++++++++++++++------------
 1 file changed, 16 insertions(+), 12 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 63a7191cb3f..f89e486356d 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -2245,7 +2245,7 @@ def _transition(self, key, finish: str, *args, **kwargs):
                     ts._dependents = dependents
                     ts._dependencies = dependencies
                     parent._tasks[ts._key] = ts
-                for plugin in self.plugins.values():
+                for plugin in list(self.plugins.values()):
                     try:
                         plugin.transition(key, start, finish2, *args, **kwargs)
                     except Exception:
@@ -3949,7 +3949,9 @@ def del_scheduler_file():
         for preload in self.preloads:
             await preload.start()
 
-        await asyncio.gather(*[plugin.start(self) for plugin in self.plugins.values()])
+        await asyncio.gather(
+            *[plugin.start(self) for plugin in list(self.plugins.values())]
+        )
 
         self.start_periodic_callbacks()
 
@@ -3988,7 +3990,9 @@ async def close(self, comm=None, fast=False, close_workers=False):
                 else:
                     break
 
-        await asyncio.gather(*[plugin.close() for plugin in self.plugins.values()])
+        await asyncio.gather(
+            *[plugin.close() for plugin in list(self.plugins.values())]
+        )
 
         for pc in self.periodic_callbacks.values():
             pc.stop()
@@ -4239,7 +4243,7 @@ async def add_worker(
             if ws._nthreads > len(ws._processing):
                 parent._idle[ws._address] = ws
 
-            for plugin in self.plugins.values():
+            for plugin in list(self.plugins.values()):
                 try:
                     result = plugin.add_worker(scheduler=self, worker=address)
                     if inspect.isawaitable(result):
@@ -4653,7 +4657,7 @@ def update_graph(
                     recommendations[ts._key] = "erred"
                     break
 
-        for plugin in self.plugins.values():
+        for plugin in list(self.plugins.values()):
             try:
                 plugin.update_graph(
                     self,
@@ -4916,7 +4920,7 @@ async def remove_worker(self, comm=None, address=None, safe=False, close=True):
 
             self.transitions(recommendations)
 
-            for plugin in self.plugins.values():
+            for plugin in list(self.plugins.values()):
                 try:
                     result = plugin.remove_worker(scheduler=self, worker=address)
                     if inspect.isawaitable(result):
@@ -5219,7 +5223,7 @@ async def add_client(self, comm, client=None, versions=None):
         self.log_event(["all", client], {"action": "add-client", "client": client})
         parent._clients[client] = ClientState(client, versions=versions)
 
-        for plugin in self.plugins.values():
+        for plugin in list(self.plugins.values()):
             try:
                 plugin.add_client(scheduler=self, client=client)
             except Exception as e:
@@ -5274,7 +5278,7 @@ def remove_client(self, client=None):
             )
             del parent._clients[client]
 
-            for plugin in self.plugins.values():
+            for plugin in list(self.plugins.values()):
                 try:
                     plugin.remove_client(scheduler=self, client=client)
                 except Exception as e:
@@ -5511,8 +5515,8 @@ def remove_plugin(self, plugin=None, name=None):
             self.plugins.pop(plugin.name)
         else:
             # TODO: Remove this block of code once removing plugins by value is disabled
-            if plugin in self.plugins.values():
-                if sum(plugin is p for p in self.plugins.values()) > 1:
+            if plugin in list(self.plugins.values()):
+                if sum(plugin is p for p in list(self.plugins.values())) > 1:
                     raise ValueError(
                         f"Multiple instances of {plugin} were found in the current scheduler "
                         "plugins, we cannot remove this plugin."
@@ -5755,7 +5759,7 @@ async def restart(self, client=None, timeout=30):
 
             self.clear_task_state()
 
-            for plugin in self.plugins.values():
+            for plugin in list(self.plugins.values()):
                 try:
                     plugin.restart(self)
                 except Exception as e:
@@ -7023,7 +7027,7 @@ def start_task_metadata(self, comm=None, name=None):
     def stop_task_metadata(self, comm=None, name=None):
         plugins = [
             p
-            for p in self.plugins.values()
+            for p in list(self.plugins.values())
             if isinstance(p, CollectTaskMetaDataPlugin) and p.name == name
         ]
         if len(plugins) != 1:

From 769a15ea6106fc0a80b65eaef2357bc62c8c0e59 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Tue, 24 Aug 2021 16:13:52 -0500
Subject: [PATCH 1463/1550] Add support for diskless machines to system monitor
 (#5257)

---
 distributed/system_monitor.py | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/distributed/system_monitor.py b/distributed/system_monitor.py
index dc749f58536..43cd288780e 100644
--- a/distributed/system_monitor.py
+++ b/distributed/system_monitor.py
@@ -40,13 +40,16 @@ def __init__(self, n=10000):
         except Exception:
             self._collect_disk_io_counters = False
         else:
-            self.last_time_disk = time()
-            self.read_bytes_disk = deque(maxlen=n)
-            self.write_bytes_disk = deque(maxlen=n)
-            self.quantities["read_bytes_disk"] = self.read_bytes_disk
-            self.quantities["write_bytes_disk"] = self.write_bytes_disk
-            self._last_disk_io_counters = disk_ioc
-            self._collect_disk_io_counters = True
+            if disk_ioc is None:  # diskless machine
+                self._collect_disk_io_counters = False
+            else:
+                self.last_time_disk = time()
+                self.read_bytes_disk = deque(maxlen=n)
+                self.write_bytes_disk = deque(maxlen=n)
+                self.quantities["read_bytes_disk"] = self.read_bytes_disk
+                self.quantities["write_bytes_disk"] = self.write_bytes_disk
+                self._last_disk_io_counters = disk_ioc
+                self._collect_disk_io_counters = True
 
         if not WINDOWS:
             self.num_fds = deque(maxlen=n)

From 04fccc23433e5765f971afd2c60230374315ed36 Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Wed, 25 Aug 2021 16:38:15 +0100
Subject: [PATCH 1464/1550] Migrate `ProcessInterface` HTML repr to Jinja2
 (#5263)

---
 distributed/deploy/spec.py                    | 46 +------------------
 .../templates/process_interface.html.j2       | 31 +++++++++++++
 2 files changed, 33 insertions(+), 44 deletions(-)
 create mode 100644 distributed/widgets/templates/process_interface.html.j2

diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index 99ad49cfc22..e4e201f815d 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -12,6 +12,7 @@
 
 import dask
 from dask.utils import parse_bytes, parse_timedelta
+from dask.widgets import get_template
 
 from ..core import CommClosedError, Status, rpc
 from ..scheduler import Scheduler
@@ -100,50 +101,7 @@ def __repr__(self):
         return f"<{dask.utils.typename(type(self))}: status={self.status.name}>"
 
     def _repr_html_(self):
-        if self.status == Status.created:
-            status = "Created"
-            bg_color = "#caf0f8"
-            border_color = "#48cae4"
-        elif self.status == Status.running:
-            status = "Running"
-            bg_color = "#c7f9cc"
-            border_color = "#78c6a3"
-        elif self.status == Status.closed:
-            status = "Closed"
-            bg_color = "#ffbfad"
-            border_color = "#ff6132"
-
-        html = f"""
-          <div>
-            <div
-                style="
-                width: 24px;
-                height: 24px;
-                background-color: {bg_color};
-                border: 3px solid {border_color};
-                border-radius: 5px;
-                position: absolute;"
-            ></div>
-            <div style="margin-left: 48px">
-                <h3 style="margin-bottom: 0px">{dask.utils.typename(type(self))}</h3>
-                <p style="color: #9d9d9d; margin-bottom: 0px">Status: {status}</p>
-            </div>
-            <p>
-            <table style="width: 100%">
-                <tr>
-                <th style="text-align: left; width: 150px">Address</th>
-                <td style="text-align: left">{self.address}</td>
-                </tr>
-                <tr>
-                <th style="text-align: left; width: 150px">External Address</th>
-                <td style="text-align: left">{self.external_address}</td>
-                </tr>
-            </table> </p>
-            </div>
-          </div>
-        """
-
-        return html
+        return get_template("process_interface.html.j2").render(process_interface=self)
 
     async def __aenter__(self):
         await self
diff --git a/distributed/widgets/templates/process_interface.html.j2 b/distributed/widgets/templates/process_interface.html.j2
new file mode 100644
index 00000000000..9e7c745e550
--- /dev/null
+++ b/distributed/widgets/templates/process_interface.html.j2
@@ -0,0 +1,31 @@
+{% if process_interface.status.name == "running" %}
+    {% set bg_color = '#c7f9cc' %}
+    {% set border_color = '#78c6a3' %}
+{% elif process_interface.status.name == "created" %}
+    {% set bg_color = '#caf0f8' %}
+    {% set border_color = '#48cae4' %}
+{% elif process_interface.status.name == "closed" %}
+    {% set bg_color = '#ffbfad' %}
+    {% set border_color = '#ff6132' %}
+{% endif %}
+
+<div>
+    <div style="width: 24px; height: 24px; background-color: {{ bg_color }}; border: 3px solid {{ border_color }}; border-radius: 5px; position: absolute;"></div>
+    <div style="margin-left: 48px">
+        <h3 style="margin-bottom: 0px">{{ process_interface | type | typename }}</h3>
+        <p style="color: #9d9d9d; margin-bottom: 0px">Status: {{ process_interface.status.name | title }}</p>
+    </div>
+    <p>
+        <table style="width: 100%">
+            <tr>
+                <th style="text-align: left; width: 150px">Address</th>
+                <td style="text-align: left">{{ process_interface.address }}</td>
+            </tr>
+            <tr>
+                <th style="text-align: left; width: 150px">External Address</th>
+                <td style="text-align: left">{{ process_interface.external_address }}</td>
+            </tr>
+        </table>
+    </p>
+    </div>
+</div>

From 3754a6694f72207b69aaf661445aecf18fb5f0cb Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Wed, 25 Aug 2021 17:15:53 +0100
Subject: [PATCH 1465/1550] Migrate `Security` HTML repr to Jinja2 (#5264)

---
 distributed/security.py                       | 26 ++-----------------
 .../widgets/templates/security.html.j2        | 13 ++++++++++
 2 files changed, 15 insertions(+), 24 deletions(-)
 create mode 100644 distributed/widgets/templates/security.html.j2

diff --git a/distributed/security.py b/distributed/security.py
index 02fd0a52276..6e6edd83c65 100644
--- a/distributed/security.py
+++ b/distributed/security.py
@@ -8,6 +8,7 @@
     ssl = None
 
 import dask
+from dask.widgets import get_template
 
 __all__ = ("Security",)
 
@@ -180,30 +181,7 @@ def __repr__(self):
         )
 
     def _repr_html_(self):
-        attr = self._attr_to_dict()
-
-        rows = ""
-
-        for key, val in attr.items():
-            rows += f"""
-            <tr>
-                <th style="text-align: left; width: 150px;">{key}</th>
-                <td style="text-align: left;">{val}</td>
-            </tr>
-            """
-
-        html = f"""
-        <div style="margin-left: auto;">
-            <h3 style="margin-bottom: 0px;"><b>Security</b></h3>
-            <p>
-                <table style="width: 100%;">
-                {rows}
-                </table>
-            </p>
-        </div>
-        """
-
-        return html
+        return get_template("security.html.j2").render(security=self._attr_to_dict())
 
     def get_tls_config_for_role(self, role):
         """
diff --git a/distributed/widgets/templates/security.html.j2 b/distributed/widgets/templates/security.html.j2
new file mode 100644
index 00000000000..4ee92d43f8c
--- /dev/null
+++ b/distributed/widgets/templates/security.html.j2
@@ -0,0 +1,13 @@
+<div style="margin-left: auto;">
+    <h3 style="margin-bottom: 0px;"><b>Security</b></h3>
+    <p>
+        <table style="width: 100%;">
+        {% for key, val in security.items() %}
+            <tr>
+                <th style="text-align: left; width: 150px;">{{ key }}</th>
+                <td style="text-align: left;">{{ val }}</td>
+            </tr>
+        {% endfor %}
+        </table>
+    </p>
+</div>

From 332f785dd1aeccb82fc9117f8ed8c576fabe6e20 Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Thu, 26 Aug 2021 15:24:01 +0100
Subject: [PATCH 1466/1550] Downgrade to `jupyter_client` 6 (#5273)

---
 continuous_integration/environment-3.7.yaml | 2 +-
 continuous_integration/environment-3.8.yaml | 2 +-
 continuous_integration/environment-3.9.yaml | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/continuous_integration/environment-3.7.yaml b/continuous_integration/environment-3.7.yaml
index 25f1a30d518..a50a2dd783a 100644
--- a/continuous_integration/environment-3.7.yaml
+++ b/continuous_integration/environment-3.7.yaml
@@ -17,7 +17,7 @@ dependencies:
   - ipywidgets
   - jinja2
   - joblib
-  - jupyter_client
+  - jupyter_client<7  # FIXME distributed#5272
   - msgpack-python
   - netcdf4
   - paramiko
diff --git a/continuous_integration/environment-3.8.yaml b/continuous_integration/environment-3.8.yaml
index 1e593d49d12..a32cb4fdf2d 100644
--- a/continuous_integration/environment-3.8.yaml
+++ b/continuous_integration/environment-3.8.yaml
@@ -16,7 +16,7 @@ dependencies:
   - ipywidgets
   - jinja2
   - joblib
-  - jupyter_client
+  - jupyter_client<7  # FIXME distributed#5272
   - msgpack-python
   - netcdf4
   - paramiko
diff --git a/continuous_integration/environment-3.9.yaml b/continuous_integration/environment-3.9.yaml
index accd7b6446c..3d35c74dfa8 100644
--- a/continuous_integration/environment-3.9.yaml
+++ b/continuous_integration/environment-3.9.yaml
@@ -17,7 +17,7 @@ dependencies:
   - ipywidgets
   - jinja2
   - joblib  # overridden by git tip below
-  - jupyter_client
+  - jupyter_client<7  # FIXME distributed#5272
   - lz4  # Only tested here
   - msgpack-python
   - netcdf4

From eedbd4b9fc3d05aaa6d60ab7d91c2e61eeb2f51d Mon Sep 17 00:00:00 2001
From: Doug Davis <ddavis@ddavis.io>
Date: Thu, 26 Aug 2021 14:42:29 -0400
Subject: [PATCH 1467/1550] Use `name` argument with `Scheduler.remove_plugin`
 calls (#5260)

---
 distributed/dashboard/components/scheduler.py          | 2 +-
 distributed/diagnostics/eventstream.py                 | 2 +-
 distributed/diagnostics/progress.py                    | 2 ++
 distributed/diagnostics/progress_stream.py             | 7 ++++---
 distributed/diagnostics/tests/test_scheduler_plugin.py | 9 +++++----
 distributed/diagnostics/websocket.py                   | 3 +++
 distributed/http/scheduler/info.py                     | 2 +-
 distributed/scheduler.py                               | 8 +++++---
 8 files changed, 22 insertions(+), 13 deletions(-)

diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index c9dddf1ec56..1465c1ddf99 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -2113,7 +2113,7 @@ def patch_updates(self):
             self.edge_source.patch({"visible": updates})
 
     def __del__(self):
-        self.scheduler.remove_plugin(self.layout)
+        self.scheduler.remove_plugin(name=self.layout.name)
 
 
 class TaskGroupGraph(DashboardComponent):
diff --git a/distributed/diagnostics/eventstream.py b/distributed/diagnostics/eventstream.py
index 1f4c096f7e0..4ace48282e4 100644
--- a/distributed/diagnostics/eventstream.py
+++ b/distributed/diagnostics/eventstream.py
@@ -29,7 +29,7 @@ def swap_buffer(scheduler, es):
 
 
 def teardown(scheduler, es):
-    scheduler.remove_plugin(es)
+    scheduler.remove_plugin(name=es.name)
 
 
 async def eventstream(address, interval):
diff --git a/distributed/diagnostics/progress.py b/distributed/diagnostics/progress.py
index abdfc2cf96b..22ac2cb5642 100644
--- a/distributed/diagnostics/progress.py
+++ b/distributed/diagnostics/progress.py
@@ -240,6 +240,8 @@ def format_time(t):
 class AllProgress(SchedulerPlugin):
     """Keep track of all keys, grouped by key_split"""
 
+    name = "all-progress"
+
     def __init__(self, scheduler):
         self.all = defaultdict(set)
         self.nbytes = defaultdict(lambda: 0)
diff --git a/distributed/diagnostics/progress_stream.py b/distributed/diagnostics/progress_stream.py
index 57b0cb38390..a9c02846e14 100644
--- a/distributed/diagnostics/progress_stream.py
+++ b/distributed/diagnostics/progress_stream.py
@@ -1,4 +1,5 @@
 import logging
+from functools import partial
 
 from tlz import merge, valmap
 
@@ -21,10 +22,10 @@ def counts(scheduler, allprogress):
     )
 
 
-def remove_plugin(*args, **kwargs):
+def remove_plugin(**kwargs):
     # Wrapper function around `Scheduler.remove_plugin` to avoid raising a
     # `PicklingError` when using a cythonized scheduler
-    return Scheduler.remove_plugin(*args, **kwargs)
+    return Scheduler.remove_plugin(**kwargs)
 
 
 async def progress_stream(address, interval):
@@ -53,7 +54,7 @@ async def progress_stream(address, interval):
             "setup": dumps_function(AllProgress),
             "function": dumps_function(counts),
             "interval": interval,
-            "teardown": dumps_function(remove_plugin),
+            "teardown": dumps_function(partial(remove_plugin, name=AllProgress.name)),
         }
     )
     return comm
diff --git a/distributed/diagnostics/tests/test_scheduler_plugin.py b/distributed/diagnostics/tests/test_scheduler_plugin.py
index 95b74f252eb..f895f2ae85f 100644
--- a/distributed/diagnostics/tests/test_scheduler_plugin.py
+++ b/distributed/diagnostics/tests/test_scheduler_plugin.py
@@ -67,7 +67,7 @@ def remove_worker(self, worker, scheduler):
     ]
 
     events[:] = []
-    s.remove_plugin(plugin)
+    s.remove_plugin(name=plugin.name)
     a = await Worker(s.address)
     await a.close()
     assert events == []
@@ -104,7 +104,7 @@ async def remove_worker(self, worker, scheduler):
     }
 
     events[:] = []
-    s.remove_plugin(plugin)
+    s.remove_plugin(name=plugin.name)
     async with Worker(s.address):
         pass
     assert events == []
@@ -116,8 +116,9 @@ async def start(self, scheduler):
     plugin = UnnamedPlugin()
     s.add_plugin(plugin)
     s.add_plugin(plugin, name="another")
-    with pytest.raises(ValueError) as excinfo:
-        s.remove_plugin(plugin)
+    with pytest.warns(FutureWarning, match="Removing scheduler plugins by value"):
+        with pytest.raises(ValueError) as excinfo:
+            s.remove_plugin(plugin)
 
     msg = str(excinfo.value)
     assert "Multiple instances of" in msg
diff --git a/distributed/diagnostics/websocket.py b/distributed/diagnostics/websocket.py
index 3796f776036..84aed343606 100644
--- a/distributed/diagnostics/websocket.py
+++ b/distributed/diagnostics/websocket.py
@@ -4,6 +4,9 @@
 
 
 class WebsocketPlugin(SchedulerPlugin):
+
+    name = "websocket"
+
     def __init__(self, socket, scheduler):
         self.socket = socket
         self.scheduler = scheduler
diff --git a/distributed/http/scheduler/info.py b/distributed/http/scheduler/info.py
index 44197141e9e..096180e1956 100644
--- a/distributed/http/scheduler/info.py
+++ b/distributed/http/scheduler/info.py
@@ -204,7 +204,7 @@ def on_message(self, message):
             self.send("pong", {"timestamp": str(datetime.now())})
 
     def on_close(self):
-        self.server.remove_plugin(self.plugin)
+        self.server.remove_plugin(name=self.plugin.name)
 
 
 routes = [
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index f89e486356d..a4011338c1f 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -7037,7 +7037,7 @@ def stop_task_metadata(self, comm=None, name=None):
             )
 
         plugin = plugins[0]
-        self.remove_plugin(plugin)
+        self.remove_plugin(name=plugin.name)
         return {"metadata": plugin.metadata, "state": plugin.state}
 
     async def register_worker_plugin(self, comm, plugin, name=None):
@@ -8150,6 +8150,8 @@ class WorkerStatusPlugin(SchedulerPlugin):
     scheduler.
     """
 
+    name = "worker-status"
+
     def __init__(self, scheduler, comm):
         self.bcomm = BatchedSend(interval="5ms")
         self.bcomm.start(comm)
@@ -8164,13 +8166,13 @@ def add_worker(self, worker=None, **kwargs):
         try:
             self.bcomm.send(["add", {"workers": {worker: ident}}])
         except CommClosedError:
-            self.scheduler.remove_plugin(self)
+            self.scheduler.remove_plugin(name=self.name)
 
     def remove_worker(self, worker=None, **kwargs):
         try:
             self.bcomm.send(["remove", worker])
         except CommClosedError:
-            self.scheduler.remove_plugin(self)
+            self.scheduler.remove_plugin(name=self.name)
 
     def teardown(self):
         self.bcomm.close()

From f11a90cd0416aaedd0d7eb56a8545b7773a64e9c Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Fri, 27 Aug 2021 15:42:20 +0100
Subject: [PATCH 1468/1550] Fix `test_map` and others (#5278)

---
 distributed/scheduler.py                  |  3 ++-
 distributed/stealing.py                   |  2 +-
 distributed/tests/test_client.py          |  3 ++-
 distributed/tests/test_client_executor.py |  7 ++++---
 distributed/tests/test_nanny.py           |  2 +-
 distributed/tests/test_steal.py           | 25 ++++++++++++++---------
 distributed/tests/test_worker.py          | 10 +++++++++
 7 files changed, 35 insertions(+), 17 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index a4011338c1f..9ee4dc35a25 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -2181,7 +2181,7 @@ def _transition(self, key, finish: str, *args, **kwargs):
                 self._transition_counter += 1
                 recommendations, client_msgs, worker_msgs = a
             elif "released" not in start_finish:
-                assert not args and not kwargs, (args, kwargs)
+                assert not args and not kwargs, (args, kwargs, start_finish)
                 a_recs: dict
                 a_cmsgs: dict
                 a_wmsgs: dict
@@ -7863,6 +7863,7 @@ def _task_to_msg(state: SchedulerState, ts: TaskState, duration: double = -1) ->
 
     if ts._annotations:
         msg["annotations"] = ts._annotations
+
     return msg
 
 
diff --git a/distributed/stealing.py b/distributed/stealing.py
index 61858d015b0..fc8e1955dff 100644
--- a/distributed/stealing.py
+++ b/distributed/stealing.py
@@ -256,7 +256,7 @@ async def move_task_confirm(self, key=None, worker=None, state=None):
                     await self.scheduler.remove_worker(thief.address)
                 self.log(("confirm", key, victim.address, thief.address))
             else:
-                raise ValueError("Unexpected task state: %s" % state)
+                raise ValueError(f"Unexpected task state: {state}")
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 5d61e5571c4..f571f7dd0ed 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -5149,7 +5149,8 @@ def f():
 
 @gen_cluster(client=True)
 async def test_secede_balances(c, s, a, b):
-    count = threading.active_count()
+    """Ensure that tasks scheduled from a seceded thread can be scheduled
+    elsewhere"""
 
     def f(x):
         client = get_client()
diff --git a/distributed/tests/test_client_executor.py b/distributed/tests/test_client_executor.py
index 0e088ee762d..f4a90297e54 100644
--- a/distributed/tests/test_client_executor.py
+++ b/distributed/tests/test_client_executor.py
@@ -126,6 +126,7 @@ def test_cancellation_as_completed(client):
         assert n_cancelled == 2
 
 
+@pytest.mark.slow()
 def test_map(client):
     with client.get_executor() as e:
         N = 10
@@ -137,7 +138,7 @@ def test_map(client):
 
     with client.get_executor(pure=False) as e:
         N = 10
-        it = e.map(slowinc, range(N), [0.1] * N, timeout=0.4)
+        it = e.map(slowinc, range(N), [0.3] * N, timeout=1.2)
         results = []
         with pytest.raises(TimeoutError):
             for x in it:
@@ -147,14 +148,14 @@ def test_map(client):
     with client.get_executor(pure=False) as e:
         N = 10
         # Not consuming the iterator will cancel remaining tasks
-        it = e.map(slowinc, range(N), [0.1] * N)
+        it = e.map(slowinc, range(N), [0.3] * N)
         for x in take(2, it):
             pass
         # Some tasks still processing
         assert number_of_processing_tasks(client) > 0
         # Garbage collect the iterator => remaining tasks are cancelled
         del it
-        time.sleep(0.05)
+        time.sleep(0.5)
         assert number_of_processing_tasks(client) == 0
 
 
diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
index a7a4b6d1e2f..c5487c7e712 100644
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -598,7 +598,7 @@ async def test_failure_during_worker_initialization(s):
     assert "Restarting worker" not in logs.getvalue()
 
 
-@gen_cluster(client=True, Worker=Nanny, timeout=10000000)
+@gen_cluster(client=True, Worker=Nanny)
 async def test_environ_plugin(c, s, a, b):
     from dask.distributed import Environ
 
diff --git a/distributed/tests/test_steal.py b/distributed/tests/test_steal.py
index 00c6efb74ff..fea42cb1393 100644
--- a/distributed/tests/test_steal.py
+++ b/distributed/tests/test_steal.py
@@ -2,6 +2,7 @@
 import itertools
 import logging
 import random
+import sys
 import weakref
 from operator import mul
 from time import sleep
@@ -12,7 +13,7 @@
 import dask
 
 from distributed import Nanny, Worker, wait, worker_client
-from distributed.compatibility import LINUX
+from distributed.compatibility import LINUX, WINDOWS
 from distributed.config import config
 from distributed.metrics import time
 from distributed.scheduler import key_split
@@ -152,7 +153,7 @@ def do_nothing(x, y=None):
     xs = c.map(do_nothing, range(10), workers=workers[0].address)
     await wait(xs)
 
-    futures = c.map(do_nothing, range(1000), y=xs)
+    futures = c.map(do_nothing, range(100), y=xs)
 
     await wait(futures)
 
@@ -362,10 +363,8 @@ async def test_steal_resource_restrictions(c, s, a):
 
     b = await Worker(s.address, loop=s.loop, nthreads=1, resources={"A": 4})
 
-    start = time()
     while not b.tasks or len(a.tasks) == 101:
         await asyncio.sleep(0.01)
-        assert time() < start + 3
 
     assert len(b.tasks) > 0
     assert len(a.tasks) < 101
@@ -645,6 +644,12 @@ async def test_steal_communication_heavy_tasks(c, s, a, b):
     assert s.processing[b.address]
 
 
+@pytest.mark.flaky(
+    condition=WINDOWS and sys.version_info[:2] == (3, 7),
+    reruns=20,
+    reruns_delay=5,
+    reason="b.in_flight_tasks == 1",
+)
 @gen_cluster(client=True)
 async def test_steal_twice(c, s, a, b):
     x = c.submit(inc, 1, workers=a.address)
@@ -680,8 +685,9 @@ async def test_steal_twice(c, s, a, b):
 async def test_dont_steal_already_released(c, s, a, b):
     future = c.submit(slowinc, 1, delay=0.05, workers=a.address)
     key = future.key
-    await asyncio.sleep(0.05)
-    assert key in a.tasks
+    while key not in a.tasks:
+        await asyncio.sleep(0.05)
+
     del future
     await asyncio.sleep(0.05)
     # In case the system is slow (e.g. network) ensure that nothing bad happens
@@ -694,10 +700,9 @@ async def test_dont_steal_already_released(c, s, a, b):
     with captured_logger(
         logging.getLogger("distributed.stealing"), level=logging.DEBUG
     ) as stealing_logs:
-        await asyncio.sleep(0.05)
-
-    logs = stealing_logs.getvalue()
-    assert f"Key released between request and confirm: {key}" in logs
+        msg = f"Key released between request and confirm: {key}"
+        while msg not in stealing_logs.getvalue():
+            await asyncio.sleep(0.05)
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 2)
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 791e1f146b6..de0f9e63987 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -2409,6 +2409,11 @@ async def test_hold_on_to_replicas(c, s, *workers):
         await asyncio.sleep(0.01)
 
 
+@pytest.mark.flaky(
+    condition=WINDOWS and sys.version_info[:2] == (3, 8),
+    reruns=20,
+    reruns_delay=5,
+)
 @gen_cluster(client=True)
 async def test_worker_reconnects_mid_compute(c, s, a, b):
     """Ensure that, if a worker disconnects while computing a result, the scheduler will
@@ -2474,6 +2479,11 @@ def fast_on_a(lock):
         await asyncio.sleep(0.001)
 
 
+@pytest.mark.flaky(
+    condition=WINDOWS and sys.version_info[:2] == (3, 8),
+    reruns=20,
+    reruns_delay=5,
+)
 @gen_cluster(client=True)
 async def test_worker_reconnects_mid_compute_multiple_states_on_scheduler(c, s, a, b):
     """

From da2bcccc2096a322c54eb3d95941b8ad06fbaf56 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Fri, 27 Aug 2021 10:15:26 -0500
Subject: [PATCH 1469/1550] Add pytest color to CI (#5276)

---
 setup.cfg | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.cfg b/setup.cfg
index f77b02739c2..ba49f43c519 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -42,7 +42,7 @@ tag_prefix =
 parentdir_prefix = distributed-
 
 [tool:pytest]
-addopts = -v -rsxfE --durations=20
+addopts = -v -rsxfE --durations=20 --color=yes
 filterwarnings =
     error:Since distributed.*:PendingDeprecationWarning
 minversion = 4

From 96cdc7574067980f25f939ed133ba8fd99de51fc Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Mon, 30 Aug 2021 18:01:11 -0500
Subject: [PATCH 1470/1550] Replace `atop` with `blockwise` (#5289)

---
 distributed/tests/test_stress.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/tests/test_stress.py b/distributed/tests/test_stress.py
index 3781e7c38e2..3c410ea322e 100644
--- a/distributed/tests/test_stress.py
+++ b/distributed/tests/test_stress.py
@@ -185,7 +185,7 @@ async def test_stress_communication(c, s, *workers):
     n = 20
     xs = [da.random.random((100, 100), chunks=(5, 5)) for i in range(n)]
     ys = [x + x.T for x in xs]
-    z = da.atop(vsum, "ij", *concat(zip(ys, ["ij"] * n)), dtype="float64")
+    z = da.blockwise(vsum, "ij", *concat(zip(ys, ["ij"] * n)), dtype="float64")
 
     future = c.compute(z.sum())
 

From 25881517d101b72944a7352acdcff8bd657bc0b4 Mon Sep 17 00:00:00 2001
From: Min RK <benjaminrk@gmail.com>
Date: Tue, 31 Aug 2021 17:30:00 +0200
Subject: [PATCH 1471/1550] jupyter-client 7 compatiblity (#5286)

instead of assuming that calling `.start()` will raise a RuntimeError
(which it won't if nest_asyncio has been applied),
check loop.is_running() before calling loop.start()

this means that the 'already running' condition is `start_exc[0] is None`,
not always a RuntimeError

use load_connection_info method to load connection info dicts,
not constructor.

Some connection_info fields are for Session, not the client itself.
---
 continuous_integration/environment-3.7.yaml |  2 +-
 continuous_integration/environment-3.8.yaml |  2 +-
 continuous_integration/environment-3.9.yaml |  2 +-
 distributed/_ipython_utils.py               | 11 ++++-------
 distributed/tests/test_ipython.py           | 10 ++++------
 distributed/utils.py                        | 13 ++++++++++---
 6 files changed, 21 insertions(+), 19 deletions(-)

diff --git a/continuous_integration/environment-3.7.yaml b/continuous_integration/environment-3.7.yaml
index a50a2dd783a..25f1a30d518 100644
--- a/continuous_integration/environment-3.7.yaml
+++ b/continuous_integration/environment-3.7.yaml
@@ -17,7 +17,7 @@ dependencies:
   - ipywidgets
   - jinja2
   - joblib
-  - jupyter_client<7  # FIXME distributed#5272
+  - jupyter_client
   - msgpack-python
   - netcdf4
   - paramiko
diff --git a/continuous_integration/environment-3.8.yaml b/continuous_integration/environment-3.8.yaml
index a32cb4fdf2d..1e593d49d12 100644
--- a/continuous_integration/environment-3.8.yaml
+++ b/continuous_integration/environment-3.8.yaml
@@ -16,7 +16,7 @@ dependencies:
   - ipywidgets
   - jinja2
   - joblib
-  - jupyter_client<7  # FIXME distributed#5272
+  - jupyter_client
   - msgpack-python
   - netcdf4
   - paramiko
diff --git a/continuous_integration/environment-3.9.yaml b/continuous_integration/environment-3.9.yaml
index 3d35c74dfa8..accd7b6446c 100644
--- a/continuous_integration/environment-3.9.yaml
+++ b/continuous_integration/environment-3.9.yaml
@@ -17,7 +17,7 @@ dependencies:
   - ipywidgets
   - jinja2
   - joblib  # overridden by git tip below
-  - jupyter_client<7  # FIXME distributed#5272
+  - jupyter_client
   - lz4  # Only tested here
   - msgpack-python
   - netcdf4
diff --git a/distributed/_ipython_utils.py b/distributed/_ipython_utils.py
index 8e83a78ff4f..a8ac9ff0dfd 100644
--- a/distributed/_ipython_utils.py
+++ b/distributed/_ipython_utils.py
@@ -76,10 +76,8 @@ def register_worker_magic(connection_info, magic_name="worker"):
     which run the given cell in a remote kernel.
     """
     ip = get_ipython()
-    info = dict(connection_info)  # copy
-    key = info.pop("key")
-    kc = BlockingKernelClient(**connection_info)
-    kc.session.key = key
+    kc = BlockingKernelClient()
+    kc.load_connection_info(connection_info)
     kc.start_channels()
 
     def remote(line, cell=None):
@@ -122,13 +120,12 @@ def remote_magic(line, cell=None):
 
     # turn info dict to hashable str for use as lookup key in _clients cache
     key = ",".join(map(str, sorted(connection_info.items())))
-    session_key = connection_info.pop("key")
 
     if key in remote_magic._clients:
         kc = remote_magic._clients[key]
     else:
-        kc = BlockingKernelClient(**connection_info)
-        kc.session.key = session_key
+        kc = BlockingKernelClient()
+        kc.load_connection_info(connection_info)
         kc.start_channels()
         kc.wait_for_ready(timeout=10)
         remote_magic._clients[key] = kc
diff --git a/distributed/tests/test_ipython.py b/distributed/tests/test_ipython.py
index 6e6b317fd0c..5996eb0e309 100644
--- a/distributed/tests/test_ipython.py
+++ b/distributed/tests/test_ipython.py
@@ -25,9 +25,8 @@ def test_start_ipython_workers(loop, zmq_ctx):
         with Client(s["address"], loop=loop) as e:
             info_dict = e.start_ipython_workers()
             info = first(info_dict.values())
-            key = info.pop("key")
-            kc = BlockingKernelClient(**info)
-            kc.session.key = key
+            kc = BlockingKernelClient()
+            kc.load_connection_info(info)
             kc.start_channels()
             kc.wait_for_ready(timeout=10)
             msg_id = kc.execute("worker")
@@ -45,9 +44,8 @@ def test_start_ipython_scheduler(loop, zmq_ctx):
     with cluster(1) as (s, [a]):
         with Client(s["address"], loop=loop) as e:
             info = e.start_ipython_scheduler()
-            key = info.pop("key")
-            kc = BlockingKernelClient(**info)
-            kc.session.key = key
+            kc = BlockingKernelClient()
+            kc.load_connection_info(info)
             kc.start_channels()
             msg_id = kc.execute("scheduler")
             reply = kc.get_shell_msg(timeout=10)
diff --git a/distributed/utils.py b/distributed/utils.py
index 3c7077b6e3a..b5683aac6b4 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -399,8 +399,13 @@ def loop_cb():
 
         def run_loop(loop=self._loop):
             loop.add_callback(loop_cb)
+            # run loop forever if it's not running already
             try:
-                loop.start()
+                if (
+                    getattr(loop, "asyncio_loop", None) is None
+                    or not loop.asyncio_loop.is_running()
+                ):
+                    loop.start()
             except Exception as e:
                 start_exc[0] = e
             finally:
@@ -417,11 +422,13 @@ def run_loop(loop=self._loop):
         if actual_thread is not thread:
             # Loop already running in other thread (user-launched)
             done_evt.wait(5)
-            if not isinstance(start_exc[0], RuntimeError):
+            if start_exc[0] is not None and not isinstance(start_exc[0], RuntimeError):
                 if not isinstance(
                     start_exc[0], Exception
                 ):  # track down infrequent error
-                    raise TypeError("not an exception", start_exc[0])
+                    raise TypeError(
+                        f"not an exception: {start_exc[0]!r}",
+                    )
                 raise start_exc[0]
             self._all_loops[self._loop] = count + 1, None
         else:

From a2cb6df4f15e24d64366af066cd629deafdc20ab Mon Sep 17 00:00:00 2001
From: Matthew Rocklin <mrocklin@gmail.com>
Date: Tue, 31 Aug 2021 15:55:49 -0500
Subject: [PATCH 1472/1550] Add `list` around iterator in `handle_missing_dep`
 (#5285)

Co-authored-by: James Bourbeau <jrbourbeau@gmail.com>
---
 distributed/worker.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/worker.py b/distributed/worker.py
index 9c0421e8612..62c1d55bb11 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -2554,7 +2554,7 @@ async def handle_missing_dep(self, *deps, **kwargs):
             if not deps:
                 return
 
-            for dep in deps:
+            for dep in list(deps):
                 if dep.suspicious_count > 5:
                     deps.remove(dep)
                     self.bad_dep(dep)

From 232996a42ac426c5ecacd26b9f991c55d52b9557 Mon Sep 17 00:00:00 2001
From: Doug Davis <ddavis@ddavis.io>
Date: Fri, 3 Sep 2021 16:05:45 -0400
Subject: [PATCH 1473/1550] Fix `add_plugin` warnings (#5267)

Co-authored-by: James Bourbeau <jrbourbeau@gmail.com>
---
 distributed/dashboard/components/scheduler.py      |  1 +
 distributed/diagnostics/graph_layout.py            |  7 +++----
 distributed/diagnostics/progress.py                | 10 +++++++---
 distributed/diagnostics/task_stream.py             |  1 -
 distributed/diagnostics/tests/test_graph_layout.py |  6 ++++++
 distributed/diagnostics/tests/test_task_stream.py  |  3 +++
 6 files changed, 20 insertions(+), 8 deletions(-)

diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index 1465c1ddf99..8501257e9fb 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -1943,6 +1943,7 @@ class TaskGraph(DashboardComponent):
     def __init__(self, scheduler, **kwargs):
         self.scheduler = scheduler
         self.layout = GraphLayout(scheduler)
+        scheduler.add_plugin(self.layout)
         self.invisible_count = 0  # number of invisible nodes
 
         self.node_source = ColumnDataSource(
diff --git a/distributed/diagnostics/graph_layout.py b/distributed/diagnostics/graph_layout.py
index 10f5dd8058a..5e48f939085 100644
--- a/distributed/diagnostics/graph_layout.py
+++ b/distributed/diagnostics/graph_layout.py
@@ -1,3 +1,5 @@
+import uuid
+
 from .plugin import SchedulerPlugin
 
 
@@ -11,9 +13,8 @@ class GraphLayout(SchedulerPlugin):
     is rendered at /graph on the diagnostic dashboard.
     """
 
-    name = "graph-layout"
-
     def __init__(self, scheduler):
+        self.name = f"graph-layout-{uuid.uuid4()}"
         self.x = {}
         self.y = {}
         self.collision = {}
@@ -29,8 +30,6 @@ def __init__(self, scheduler):
         self.visible_updates = []
         self.visible_edge_updates = []
 
-        scheduler.add_plugin(self)
-
         if self.scheduler.tasks:
             dependencies = {
                 k: [ds.key for ds in ts.dependencies]
diff --git a/distributed/diagnostics/progress.py b/distributed/diagnostics/progress.py
index 22ac2cb5642..4fd6a989fdc 100644
--- a/distributed/diagnostics/progress.py
+++ b/distributed/diagnostics/progress.py
@@ -5,6 +5,7 @@
 
 from tlz import groupby, valmap
 
+from dask.base import tokenize
 from dask.utils import stringify
 
 from ..utils import key_split
@@ -60,8 +61,8 @@ class Progress(SchedulerPlugin):
     notably TextProgressBar and ProgressWidget, which do perform visualization.
     """
 
-    def __init__(self, keys, scheduler, minimum=0, dt=0.1, complete=False):
-        self.name = "Progress"
+    def __init__(self, keys, scheduler, minimum=0, dt=0.1, complete=False, name=None):
+        self.name = name or f"progress-{tokenize(keys, minimum, dt, complete)}"
         self.keys = {k.key if hasattr(k, "key") else k for k in keys}
         self.keys = {stringify(k) for k in self.keys}
         self.scheduler = scheduler
@@ -160,7 +161,10 @@ def __init__(
         self, keys, scheduler=None, func=key_split, minimum=0, dt=0.1, complete=False
     ):
         self.func = func
-        super().__init__(keys, scheduler, minimum=minimum, dt=dt, complete=complete)
+        name = f"multi-progress-{tokenize(keys, func, minimum, dt, complete)}"
+        super().__init__(
+            keys, scheduler, minimum=minimum, dt=dt, complete=complete, name=name
+        )
 
     async def setup(self):
         keys = self.keys
diff --git a/distributed/diagnostics/task_stream.py b/distributed/diagnostics/task_stream.py
index 120be52a38e..60dda656faa 100644
--- a/distributed/diagnostics/task_stream.py
+++ b/distributed/diagnostics/task_stream.py
@@ -27,7 +27,6 @@ def __init__(self, scheduler, maxlen=None):
             )
         self.buffer = deque(maxlen=maxlen)
         self.scheduler = scheduler
-        scheduler.add_plugin(self)
         self.index = 0
 
     def transition(self, key, start, finish, *args, **kwargs):
diff --git a/distributed/diagnostics/tests/test_graph_layout.py b/distributed/diagnostics/tests/test_graph_layout.py
index 494cd26e3ca..97b76667df2 100644
--- a/distributed/diagnostics/tests/test_graph_layout.py
+++ b/distributed/diagnostics/tests/test_graph_layout.py
@@ -9,6 +9,7 @@
 @gen_cluster(client=True)
 async def test_basic(c, s, a, b):
     gl = GraphLayout(s)
+    s.add_plugin(gl)
     futures = c.map(inc, range(5))
     total = c.submit(sum, futures)
 
@@ -28,6 +29,7 @@ async def test_construct_after_call(c, s, a, b):
     await total
 
     gl = GraphLayout(s)
+    s.add_plugin(gl)
 
     assert len(gl.x) == len(gl.y) == 6
     assert all(gl.x[f.key] == 0 for f in futures)
@@ -38,6 +40,7 @@ async def test_construct_after_call(c, s, a, b):
 @gen_cluster(client=True)
 async def test_states(c, s, a, b):
     gl = GraphLayout(s)
+    s.add_plugin(gl)
     await c.submit(sum, c.map(inc, range(5)))
 
     while True:
@@ -50,6 +53,7 @@ async def test_states(c, s, a, b):
 @gen_cluster(client=True)
 async def test_release_tasks(c, s, a, b):
     gl = GraphLayout(s)
+    s.add_plugin(gl)
     futures = c.map(inc, range(5))
     total = c.submit(sum, futures)
 
@@ -66,6 +70,7 @@ async def test_release_tasks(c, s, a, b):
 @gen_cluster(client=True)
 async def test_forget(c, s, a, b):
     gl = GraphLayout(s)
+    s.add_plugin(gl)
 
     futures = c.map(inc, range(10))
     futures = c.map(inc, futures)
@@ -84,6 +89,7 @@ async def test_forget(c, s, a, b):
 @gen_cluster(client=True)
 async def test_unique_positions(c, s, a, b):
     gl = GraphLayout(s)
+    s.add_plugin(gl)
 
     x = c.submit(inc, 1)
     ys = [c.submit(operator.add, x, i) for i in range(5)]
diff --git a/distributed/diagnostics/tests/test_task_stream.py b/distributed/diagnostics/tests/test_task_stream.py
index c65fdb5b573..3407ec1a59a 100644
--- a/distributed/diagnostics/tests/test_task_stream.py
+++ b/distributed/diagnostics/tests/test_task_stream.py
@@ -14,6 +14,7 @@
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 3)
 async def test_TaskStreamPlugin(c, s, *workers):
     es = TaskStreamPlugin(s)
+    s.add_plugin(es)
     assert not es.buffer
 
     futures = c.map(div, [1] * 10, range(10))
@@ -46,6 +47,7 @@ async def test_TaskStreamPlugin(c, s, *workers):
 @gen_cluster(client=True)
 async def test_maxlen(c, s, a, b):
     tasks = TaskStreamPlugin(s, maxlen=5)
+    s.add_plugin(tasks)
     futures = c.map(inc, range(10))
     await wait(futures)
     assert len(tasks.buffer) == 5
@@ -54,6 +56,7 @@ async def test_maxlen(c, s, a, b):
 @gen_cluster(client=True)
 async def test_collect(c, s, a, b):
     tasks = TaskStreamPlugin(s)
+    s.add_plugin(tasks)
     start = time()
     futures = c.map(slowinc, range(10), delay=0.1)
     await wait(futures)

From baca13b795fd0ddef7823f3712aff25debb8b04b Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Fri, 3 Sep 2021 15:36:24 -0500
Subject: [PATCH 1474/1550] bump version to 2021.09.0

---
 docs/source/changelog.rst | 33 ++++++++++++++++++++++++++++++++-
 requirements.txt          |  2 +-
 2 files changed, 33 insertions(+), 2 deletions(-)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index b183276a87c..cb853395047 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,35 @@
 Changelog
 =========
 
+2021.09.0
+---------
+
+Released on September 3, 2021
+
+- Fix ``add_plugin`` warnings (:pr:`5267`) `Doug Davis`_
+- Add ``list`` around iterator in ``handle_missing_dep`` (:pr:`5285`) `Matthew Rocklin`_
+- Jupyter-client 7 compatiblity (:pr:`5286`) `Min RK`_
+- Replace ``atop`` with ``blockwise`` (:pr:`5289`) `James Bourbeau`_
+- Add pytest color to CI (:pr:`5276`) `James Bourbeau`_
+- Fix ``test_map`` and others (:pr:`5278`) `crusaderky`_
+- Use ``name`` argument with ``Scheduler.remove_plugin`` calls (:pr:`5260`) `Doug Davis`_
+- Downgrade to ``jupyter_client`` 6 (:pr:`5273`) `crusaderky`_
+- Migrate ``Security`` HTML repr to Jinja2 (:pr:`5264`) `Jacob Tomlinson`_
+- Migrate ``ProcessInterface`` HTML repr to Jinja2 (:pr:`5263`) `Jacob Tomlinson`_
+- Add support for diskless machines to system monitor (:pr:`5257`) `James Bourbeau`_
+- Avoid during-iteration scheduler plugin changes (:pr:`5259`) `Doug Davis`_
+- Remove ``GroupProgress`` scheduler plugin (:pr:`5256`) `James Bourbeau`_
+- Properly check for ipv6 availability (:pr:`5255`) `crusaderky`_
+- Improved IPv6 dask-worker support (:pr:`5197`) `Walt Woods`_
+- Overwrite worker plugins (:pr:`5248`) `Matthew Rocklin`_
+- Refactor scheduler plugins; store in a dictionary (:pr:`5120`) `Doug Davis`_
+- Fix "then" -> "than" typo in docs (:pr:`5247`) `David Chudzicki`_
+- Fix typo (remove extra verb "creates") in docs (:pr:`5244`) `David Chudzicki`_
+- Fix "fractiom" -> "fraction" typo in docstring (:pr:`5245`) `David Chudzicki`_
+- Fix "schedulers" -> "scheduler" typo in docs (:pr:`5246`) `David Chudzicki`_
+- Use non-histogram plots up to 100 workers (:pr:`5249`) `Matthew Rocklin`_
+
+
 2021.08.1
 ---------
 
@@ -2604,4 +2633,6 @@ significantly without many new features.
 .. _`Gabe Joseph`: https://github.com/gjoseph92
 .. _`Freyam Mehta`: https://github.com/freyam
 .. _`gerrymanoim`: https://github.com/gerrymanoim
-.. _`Bryan Van de Ven`: https://github.com/bryevdv
\ No newline at end of file
+.. _`Bryan Van de Ven`: https://github.com/bryevdv
+.. _`David Chudzicki`: https://github.com/dchudz
+.. _`Walt Woods`: https://github.com/wwoods
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index b1542068ace..a5a8aa41575 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,6 @@
 click >= 6.6
 cloudpickle >= 1.5.0
-dask == 2021.08.1
+dask == 2021.09.0
 jinja2
 msgpack >= 0.6.0
 psutil >= 5.0

From b0c9dc4d7e5746550114a3f13831c278bc0db6d5 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Tue, 7 Sep 2021 15:23:51 -0500
Subject: [PATCH 1475/1550] Add test setup fixture (#5242)

---
 conftest.py               | 33 +++++++++++++++++++++++++++++++++
 distributed/utils_test.py | 31 ++++++-------------------------
 2 files changed, 39 insertions(+), 25 deletions(-)

diff --git a/conftest.py b/conftest.py
index ebb9dae1457..fde33e4e310 100644
--- a/conftest.py
+++ b/conftest.py
@@ -1,6 +1,11 @@
 # https://pytest.org/latest/example/simple.html#control-skipping-of-tests-according-to-command-line-option
+import copy
+import logging
+
 import pytest
 
+import dask
+
 # Uncomment to enable more logging and checks
 # (https://docs.python.org/3/library/asyncio-dev.html)
 # Note this makes things slower and might consume much memory.
@@ -35,3 +40,31 @@ def pytest_collection_modifyitems(config, items):
 
 
 pytest_plugins = ["distributed.pytest_resourceleaks"]
+
+
+_original_config = copy.deepcopy(dask.config.config)
+# Custom preloads can interact with the test suite in unexpected ways
+# so we remove them when running tests
+for node in ["scheduler", "worker", "nanny"]:
+    _original_config["distributed"][node]["preload"] = []
+    _original_config["distributed"][node]["preload-argv"] = []
+
+_logging_levels = {
+    name: logger.level
+    for name, logger in logging.root.manager.loggerDict.items()
+    if isinstance(logger, logging.Logger)
+}
+
+
+@pytest.fixture(autouse=True)
+def initialize_test():
+
+    # Restore default logging levels
+    for name, level in _logging_levels.items():
+        logging.getLogger(name).setLevel(level)
+
+    # Ensure a clean config
+    dask.config.config.clear()
+    dask.config.config.update(copy.deepcopy(_original_config))
+
+    yield
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index a9c5496efce..b9354a5755c 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -77,12 +77,6 @@
 logger = logging.getLogger(__name__)
 
 
-logging_levels = {
-    name: logger.level
-    for name, logger in logging.root.manager.loggerDict.items()
-    if isinstance(logger, logging.Logger)
-}
-
 _TEST_TIMEOUT = 30
 _offload_executor.submit(lambda: None).result()  # create thread during import
 
@@ -215,14 +209,6 @@ def get_ip():
     remote_magic._clients.clear()
 
 
-original_config = copy.deepcopy(dask.config.config)
-
-
-def reset_config():
-    dask.config.config.clear()
-    dask.config.config.update(copy.deepcopy(original_config))
-
-
 def nodebug(func):
     """
     A decorator to disable debug facilities during timing-sensitive tests.
@@ -1585,18 +1571,13 @@ def clean(threads=not WINDOWS, instances=True, timeout=1, processes=True):
             with check_process_leak(check=processes):
                 with check_instances() if instances else nullcontext():
                     with check_active_rpc(loop, timeout):
-                        reset_config()
-
-                        dask.config.set({"distributed.comm.timeouts.connect": "5s"})
-                        # Restore default logging levels
-                        # XXX use pytest hooks/fixtures instead?
-                        for name, level in logging_levels.items():
-                            logging.getLogger(name).setLevel(level)
-
-                        yield loop
+                        with dask.config.set(
+                            {"distributed.comm.timeouts.connect": "5s"}
+                        ):
+                            yield loop
 
-                        with suppress(AttributeError):
-                            del thread_state.on_event_loop_thread
+                            with suppress(AttributeError):
+                                del thread_state.on_event_loop_thread
 
 
 @pytest.fixture

From 0b7510e1f1e8cad8f87e09b84f7e508951744382 Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Tue, 7 Sep 2021 22:46:53 +0100
Subject: [PATCH 1476/1550] Active Memory Manager framework + discard excess
 replicas (#5111)

---
 distributed/active_memory_manager.py          | 308 ++++++++++++
 distributed/distributed-schema.yaml           |  25 +
 distributed/distributed.yaml                  |  12 +
 distributed/scheduler.py                      |  16 +-
 .../tests/test_active_memory_manager.py       | 439 ++++++++++++++++++
 distributed/tests/test_scheduler.py           |  26 +-
 6 files changed, 806 insertions(+), 20 deletions(-)
 create mode 100644 distributed/active_memory_manager.py
 create mode 100644 distributed/tests/test_active_memory_manager.py

diff --git a/distributed/active_memory_manager.py b/distributed/active_memory_manager.py
new file mode 100644
index 00000000000..4ed2daf4113
--- /dev/null
+++ b/distributed/active_memory_manager.py
@@ -0,0 +1,308 @@
+from __future__ import annotations
+
+import asyncio
+from collections import defaultdict
+from collections.abc import Generator
+from typing import TYPE_CHECKING, Optional
+
+from tornado.ioloop import PeriodicCallback
+
+import dask
+from dask.utils import parse_timedelta
+
+from .utils import import_term
+
+if TYPE_CHECKING:
+    from .scheduler import SchedulerState, TaskState, WorkerState
+
+
+class ActiveMemoryManagerExtension:
+    """Scheduler extension that optimizes memory usage across the cluster.
+    It can be either triggered by hand or automatically every few seconds; at every
+    iteration it performs one or both of the following:
+
+    - create new replicas of in-memory tasks
+    - destroy replicas of in-memory tasks; this never destroys the last available copy.
+
+    There are no 'move' operations. A move is performed in two passes: first you create
+    a copy and, in the next iteration, you delete the original (if the copy succeeded).
+
+    This extension is configured by the dask config section
+    ``distributed.scheduler.active-memory-manager``.
+    """
+
+    scheduler: SchedulerState
+    policies: set[ActiveMemoryManagerPolicy]
+    interval: float
+
+    # These attributes only exist within the scope of self.run()
+    # Current memory (in bytes) allocated on each worker, plus/minus pending actions
+    workers_memory: dict[WorkerState, int]
+    # Pending replications and deletions for each task
+    pending: defaultdict[TaskState, tuple[set[WorkerState], set[WorkerState]]]
+
+    def __init__(
+        self,
+        scheduler: SchedulerState,
+        # The following parameters are exposed so that one may create, run, and throw
+        # away on the fly a specialized manager, separate from the main one.
+        policies: Optional[set[ActiveMemoryManagerPolicy]] = None,
+        register: bool = True,
+        start: Optional[bool] = None,
+        interval: Optional[float] = None,
+    ):
+        self.scheduler = scheduler
+
+        if policies is None:
+            policies = set()
+            for kwargs in dask.config.get(
+                "distributed.scheduler.active-memory-manager.policies"
+            ):
+                kwargs = kwargs.copy()
+                cls = import_term(kwargs.pop("class"))
+                if not issubclass(cls, ActiveMemoryManagerPolicy):
+                    raise TypeError(
+                        f"{cls}: Expected ActiveMemoryManagerPolicy; got {type(cls)}"
+                    )
+                policies.add(cls(**kwargs))
+
+        for policy in policies:
+            policy.manager = self
+        self.policies = policies
+
+        if register:
+            scheduler.extensions["amm"] = self
+            scheduler.handlers.update(
+                {
+                    "amm_run_once": self.run_once,
+                    "amm_start": self.start,
+                    "amm_stop": self.stop,
+                }
+            )
+
+        if interval is None:
+            interval = parse_timedelta(
+                dask.config.get("distributed.scheduler.active-memory-manager.interval")
+            )
+        self.interval = interval
+        if start is None:
+            start = dask.config.get("distributed.scheduler.active-memory-manager.start")
+        if start:
+            self.start()
+
+    def start(self, comm=None) -> None:
+        """Start executing every ``self.interval`` seconds until scheduler shutdown"""
+        pc = PeriodicCallback(self.run_once, self.interval * 1000.0)
+        self.scheduler.periodic_callbacks["amm"] = pc
+        pc.start()
+
+    def stop(self, comm=None) -> None:
+        """Stop periodic execution"""
+        pc = self.scheduler.periodic_callbacks.pop("amm", None)
+        if pc:
+            pc.stop()
+
+    def run_once(self, comm=None) -> None:
+        """Run all policies once and asynchronously (fire and forget) enact their
+        recommendations to replicate/drop keys
+        """
+        # This should never fail since this is a synchronous method
+        assert not hasattr(self, "pending")
+
+        self.pending = defaultdict(lambda: (set(), set()))
+        self.workers_memory = {
+            w: w.memory.optimistic for w in self.scheduler.workers.values()
+        }
+        try:
+            # populate self.pending
+            self._run_policies()
+
+            drop_by_worker = defaultdict(set)
+            repl_by_worker = defaultdict(dict)
+            for ts, (pending_repl, pending_drop) in self.pending.items():
+                if not ts.who_has:
+                    continue
+                who_has = [ws_snd.address for ws_snd in ts.who_has - pending_drop]
+                assert who_has  # Never drop the last replica
+                for ws_rec in pending_repl:
+                    assert ws_rec not in ts.who_has
+                    repl_by_worker[ws_rec.address][ts.key] = who_has
+                for ws in pending_drop:
+                    assert ws in ts.who_has
+                    drop_by_worker[ws.address].add(ts.key)
+
+            # Fire-and-forget enact recommendations from policies
+            # This is temporary code, waiting for
+            # https://github.com/dask/distributed/pull/5046
+            for addr, who_has in repl_by_worker.items():
+                asyncio.create_task(self.scheduler.gather_on_worker(addr, who_has))
+            for addr, keys in drop_by_worker.items():
+                asyncio.create_task(self.scheduler.delete_worker_data(addr, keys))
+            # End temporary code
+
+        finally:
+            del self.workers_memory
+            del self.pending
+
+    def _run_policies(self) -> None:
+        """Sequentially run ActiveMemoryManagerPolicy.run() for all registered policies,
+        obtain replicate/drop suggestions, and use them to populate self.pending.
+        """
+        candidates: Optional[set[WorkerState]]
+        cmd: str
+        ws: Optional[WorkerState]
+        ts: TaskState
+        nreplicas: int
+
+        for policy in list(self.policies):  # a policy may remove itself
+            policy_gen = policy.run()
+            ws = None
+            while True:
+                try:
+                    cmd, ts, candidates = policy_gen.send(ws)
+                except StopIteration:
+                    break  # next policy
+
+                pending_repl, pending_drop = self.pending[ts]
+
+                if cmd == "replicate":
+                    ws = self._find_recipient(ts, candidates, pending_repl)
+                    if ws:
+                        pending_repl.add(ws)
+                        self.workers_memory[ws] += ts.nbytes
+
+                elif cmd == "drop":
+                    ws = self._find_dropper(ts, candidates, pending_drop)
+                    if ws:
+                        pending_drop.add(ws)
+                        self.workers_memory[ws] = max(
+                            0, self.workers_memory[ws] - ts.nbytes
+                        )
+
+                else:
+                    raise ValueError(f"Unknown command: {cmd}")  # pragma: nocover
+
+    def _find_recipient(
+        self,
+        ts: TaskState,
+        candidates: Optional[set[WorkerState]],
+        pending_repl: set[WorkerState],
+    ) -> Optional[WorkerState]:
+        """Choose a worker to acquire a new replica of an in-memory task among a set of
+        candidates. If candidates is None, default to all workers in the cluster that do
+        not hold a replica yet. The worker with the lowest memory usage (downstream of
+        pending replications and drops) will be returned.
+        """
+        if ts.state != "memory":
+            return None
+        if candidates is None:
+            candidates = set(self.scheduler.workers.values())
+        candidates -= ts.who_has
+        candidates -= pending_repl
+        if not candidates:
+            return None
+        return min(candidates, key=self.workers_memory.get)
+
+    def _find_dropper(
+        self,
+        ts: TaskState,
+        candidates: Optional[set[WorkerState]],
+        pending_drop: set[WorkerState],
+    ) -> Optional[WorkerState]:
+        """Choose a worker to drop its replica of an in-memory task among a set of
+        candidates. If candidates is None, default to all workers in the cluster that
+        hold a replica. The worker with the highest memory usage (downstream of pending
+        replications and drops) will be returned.
+        """
+        if len(ts.who_has) - len(pending_drop) < 2:
+            return None
+        if candidates is None:
+            candidates = ts.who_has.copy()
+        else:
+            candidates &= ts.who_has
+        candidates -= pending_drop
+        candidates -= {waiter_ts.processing_on for waiter_ts in ts.waiters}
+        if not candidates:
+            return None
+        return max(candidates, key=self.workers_memory.get)
+
+
+class ActiveMemoryManagerPolicy:
+    """Abstract parent class"""
+
+    manager: ActiveMemoryManagerExtension
+
+    def __repr__(self) -> str:
+        return f"{self.__class__.__name__}()"
+
+    def run(
+        self,
+    ) -> Generator[
+        tuple[str, TaskState, Optional[set[WorkerState]]],
+        Optional[WorkerState],
+        None,
+    ]:
+        """This method is invoked by the ActiveMemoryManager every few seconds, or
+        whenever the user invokes scheduler.amm_run_once().
+        It is an iterator that must emit any of the following:
+
+        - "replicate", <TaskState>, None
+        - "replicate", <TaskState>, {subset of potential workers to replicate to}
+        - "drop", <TaskState>, None
+        - "drop", <TaskState>, {subset of potential workers to drop from}
+
+        Each element yielded indicates the desire to create or destroy a single replica
+        of a key. If a subset of workers is not provided, it defaults to all workers on
+        the cluster. Either the ActiveMemoryManager or the Worker may later decide to
+        disregard the request, e.g. because it would delete the last copy of a key or
+        because the key is currently needed on that worker.
+
+        You may optionally retrieve which worker it was decided the key will be
+        replicated to or dropped from, as follows:
+
+        ```python
+        choice = yield "replicate", ts, None
+        ```
+
+        ``choice`` is either a WorkerState or None; the latter is returned if the
+        ActiveMemoryManager chose to disregard the request.
+
+        The current pending (accepted) commands can be inspected on
+        ``self.manager.pending``; this includes the commands previously yielded by this
+        same method.
+
+        The current memory usage on each worker, *downstream of all pending commands*,
+        can be inspected on ``self.manager.workers_memory``.
+        """
+        raise NotImplementedError("Virtual method")
+
+
+class ReduceReplicas(ActiveMemoryManagerPolicy):
+    """Make sure that in-memory tasks are not replicated on more workers than desired;
+    drop the excess replicas.
+    """
+
+    def run(self):
+        # TODO this is O(n) to the total number of in-memory tasks on the cluster; it
+        #      could be made faster by automatically attaching it to a TaskState when it
+        #      goes above one replica and detaching it when it drops below two.
+        for ts in self.manager.scheduler.tasks.values():
+            if len(ts.who_has) < 2:
+                continue
+
+            desired_replicas = 1  # TODO have a marker on TaskState
+
+            # If a dependent task has not been assigned to a worker yet, err on the side
+            # of caution and preserve an additional replica for it.
+            # However, if two dependent tasks have been already assigned to the same
+            # worker, don't double count them.
+            nwaiters = len({waiter.processing_on or waiter for waiter in ts.waiters})
+
+            ndrop = len(ts.who_has) - max(desired_replicas, nwaiters)
+            if ts in self.manager.pending:
+                pending_repl, pending_drop = self.manager.pending[ts]
+                ndrop += len(pending_repl) - len(pending_drop)
+
+            # ndrop could be negative, which for range() is the same as 0.
+            for _ in range(ndrop):
+                yield "drop", ts, None
diff --git a/distributed/distributed-schema.yaml b/distributed/distributed-schema.yaml
index d10022f7574..f5b7b73a5e7 100644
--- a/distributed/distributed-schema.yaml
+++ b/distributed/distributed-schema.yaml
@@ -242,6 +242,31 @@ properties:
               A list of trusted root modules the schedular is allowed to import (incl. submodules). For security reasons, the
               scheduler does not import arbitrary Python modules.
 
+          active-memory-manager:
+            type: object
+            required: [start, interval, policies]
+            additionalProperties: false
+            properties:
+              start:
+                type: boolean
+                description: set to true to auto-start the AMM on Scheduler init;
+                  false to manually start it with client.scheduler.amm_start()
+              interval:
+                type: string
+                description:
+                  Time expression, e.g. "2s". Run the AMM cycle every <interval>.
+              policies:
+                type: array
+                items:
+                  type: object
+                  required: [class]
+                  properties:
+                    class:
+                      type: string
+                      description: fully qualified name of an ActiveMemoryManagerPolicy
+                        subclass
+                  additionalProperties:
+                    description: keyword arguments to the policy constructor, if any
 
       worker:
         type: object
diff --git a/distributed/distributed.yaml b/distributed/distributed.yaml
index 929b58676a7..6949fdd56e8 100644
--- a/distributed/distributed.yaml
+++ b/distributed/distributed.yaml
@@ -60,6 +60,18 @@ distributed:
       - dask
       - distributed
 
+    active-memory-manager:
+      # Set to true to auto-start the Active Memory Manager on Scheduler start; if false
+      # you'll have to either manually start it with client.scheduler.amm_start() or run
+      # it once with client.scheduler.amm_run().
+      start: false
+      # Once started, run the AMM cycle every <interval>
+      interval: 2s
+      policies:
+        # Policies that should be executed at every cycle. Any additional keys in each
+        # object are passed as keyword arguments to the policy constructor.
+        - class: distributed.active_memory_manager.ReduceReplicas
+
   worker:
     blocked-handlers: []
     multiprocessing-method: spawn
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 9ee4dc35a25..22a33372b5b 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -42,6 +42,7 @@
 
 from . import preloading, profile
 from . import versions as version_module
+from .active_memory_manager import ActiveMemoryManagerExtension
 from .batched import BatchedSend
 from .comm import (
     get_address_host,
@@ -172,6 +173,7 @@ def nogil(func):
     PubSubSchedulerExtension,
     SemaphoreExtension,
     EventExtension,
+    ActiveMemoryManagerExtension,
 ]
 
 ALL_TASK_STATES = declare(
@@ -5859,8 +5861,8 @@ async def proxy(self, comm=None, msg=None, worker=None, serializers=None):
         )
         return d[worker]
 
-    async def _gather_on_worker(
-        self, worker_address: str, who_has: "dict[Hashable, list[str]]"
+    async def gather_on_worker(
+        self, worker_address: str, who_has: "dict[str, list[str]]"
     ) -> set:
         """Peer-to-peer copy of keys from multiple workers to a single worker
 
@@ -5919,7 +5921,7 @@ async def _gather_on_worker(
 
         return keys_failed
 
-    async def _delete_worker_data(self, worker_address: str, keys: "list[str]") -> None:
+    async def delete_worker_data(self, worker_address: str, keys: "list[str]") -> None:
         """Delete data from a worker and update the corresponding worker/task states
 
         Parameters
@@ -6290,7 +6292,7 @@ async def _rebalance_move_data(
                 await asyncio.gather(
                     *(
                         # Note: this never raises exceptions
-                        self._gather_on_worker(w, who_has)
+                        self.gather_on_worker(w, who_has)
                         for w, who_has in to_recipients.items()
                     )
                 ),
@@ -6304,7 +6306,7 @@ async def _rebalance_move_data(
 
         # Note: this never raises exceptions
         await asyncio.gather(
-            *(self._delete_worker_data(r, v) for r, v in to_senders.items())
+            *(self.delete_worker_data(r, v) for r, v in to_senders.items())
         )
 
         for r, v in to_recipients.items():
@@ -6390,7 +6392,7 @@ async def replicate(
                 # Note: this never raises exceptions
                 await asyncio.gather(
                     *[
-                        self._delete_worker_data(ws._address, [t.key for t in tasks])
+                        self.delete_worker_data(ws._address, [t.key for t in tasks])
                         for ws, tasks in del_worker_tasks.items()
                     ]
                 )
@@ -6420,7 +6422,7 @@ async def replicate(
                 await asyncio.gather(
                     *(
                         # Note: this never raises exceptions
-                        self._gather_on_worker(w, who_has)
+                        self.gather_on_worker(w, who_has)
                         for w, who_has in gathers.items()
                     )
                 )
diff --git a/distributed/tests/test_active_memory_manager.py b/distributed/tests/test_active_memory_manager.py
new file mode 100644
index 00000000000..afe6c114940
--- /dev/null
+++ b/distributed/tests/test_active_memory_manager.py
@@ -0,0 +1,439 @@
+import asyncio
+import random
+
+import pytest
+
+from distributed import Nanny
+from distributed.active_memory_manager import (
+    ActiveMemoryManagerExtension,
+    ActiveMemoryManagerPolicy,
+)
+from distributed.utils_test import gen_cluster, inc, slowinc
+
+NO_AMM_START = {"distributed.scheduler.active-memory-manager.start": False}
+
+
+@gen_cluster(
+    client=True,
+    config={
+        "distributed.scheduler.active-memory-manager.start": False,
+        "distributed.scheduler.active-memory-manager.policies": [],
+    },
+)
+async def test_no_policies(c, s, a, b):
+    await c.scheduler.amm_run_once()
+
+
+class DemoPolicy(ActiveMemoryManagerPolicy):
+    """Drop or replicate a key n times"""
+
+    def __init__(self, action, key, n, candidates):
+        self.action = action
+        self.key = key
+        self.n = n
+        self.candidates = candidates
+
+    def run(self):
+        candidates = self.candidates
+        if candidates is not None:
+            candidates = {
+                ws
+                for i, ws in enumerate(self.manager.scheduler.workers.values())
+                if i in candidates
+            }
+        for ts in self.manager.scheduler.tasks.values():
+            if ts.key == self.key:
+                for _ in range(self.n):
+                    yield self.action, ts, candidates
+
+
+def demo_config(action, key="x", n=10, candidates=None, start=False, interval=0.1):
+    """Create a dask config for AMM with DemoPolicy"""
+    return {
+        "distributed.scheduler.active-memory-manager.start": start,
+        "distributed.scheduler.active-memory-manager.interval": interval,
+        "distributed.scheduler.active-memory-manager.policies": [
+            {
+                "class": "distributed.tests.test_active_memory_manager.DemoPolicy",
+                "action": action,
+                "key": key,
+                "n": n,
+                "candidates": candidates,
+            },
+        ],
+    }
+
+
+@gen_cluster(nthreads=[("", 1)] * 4, client=True, config=demo_config("drop"))
+async def test_drop(c, s, *workers):
+    futures = await c.scatter({"x": 123}, broadcast=True)
+    assert len(s.tasks["x"].who_has) == 4
+    # Also test the extension handler
+    await c.scheduler.amm_run_once()
+    while len(s.tasks["x"].who_has) > 1:
+        await asyncio.sleep(0.01)
+    # The last copy is never dropped even if the policy asks so
+    await asyncio.sleep(0.2)
+    assert len(s.tasks["x"].who_has) == 1
+
+
+@gen_cluster(client=True, config=demo_config("drop"))
+async def test_start_stop(c, s, a, b):
+    x = c.submit(lambda: 123, key="x")
+    await c.replicate(x, 2)
+    assert len(s.tasks["x"].who_has) == 2
+    await c.scheduler.amm_start()
+    while len(s.tasks["x"].who_has) > 1:
+        await asyncio.sleep(0.01)
+    await c.scheduler.amm_stop()
+    # AMM is not running anymore
+    await c.replicate(x, 2)
+    await asyncio.sleep(0.2)
+    assert len(s.tasks["x"].who_has) == 2
+
+
+@gen_cluster(client=True, config=demo_config("drop", start=True, interval=0.1))
+async def test_auto_start(c, s, a, b):
+    futures = await c.scatter({"x": 123}, broadcast=True)
+    # The AMM should run within 0.1s of the broadcast.
+    # Add generous extra padding to prevent flakiness.
+    await asyncio.sleep(0.5)
+    assert len(s.tasks["x"].who_has) == 1
+
+
+@gen_cluster(client=True, config=NO_AMM_START)
+async def test_not_registered(c, s, a, b):
+    futures = await c.scatter({"x": 1}, broadcast=True)
+    assert len(s.tasks["x"].who_has) == 2
+
+    class Policy(ActiveMemoryManagerPolicy):
+        def run(self):
+            yield "drop", s.tasks["x"], None
+
+    amm = ActiveMemoryManagerExtension(s, {Policy()}, register=False, start=False)
+    amm.run_once()
+    assert amm is not s.extensions["amm"]
+
+    while len(s.tasks["x"].who_has) > 1:
+        await asyncio.sleep(0.01)
+
+
+@gen_cluster(client=True, config=demo_config("drop"))
+async def test_drop_not_in_memory(c, s, a, b):
+    """ts.who_has is empty"""
+    x = c.submit(slowinc, 1, key="x")
+    while "x" not in s.tasks:
+        await asyncio.sleep(0.01)
+    assert not x.done()
+    s.extensions["amm"].run_once()
+    assert await x == 2
+
+
+@gen_cluster(client=True, config=demo_config("drop"))
+async def test_drop_with_waiter(c, s, a, b):
+    """Tasks with a waiter are never dropped"""
+    x = (await c.scatter({"x": 1}, broadcast=True))["x"]
+    y1 = c.submit(slowinc, x, delay=0.4, key="y1", workers=[a.address])
+    y2 = c.submit(slowinc, x, delay=0.8, key="y2", workers=[b.address])
+    for key in ("y1", "y2"):
+        while key not in s.tasks or s.tasks[key].state != "processing":
+            await asyncio.sleep(0.01)
+
+    s.extensions["amm"].run_once()
+    await asyncio.sleep(0.2)
+    assert {ws.address for ws in s.tasks["x"].who_has} == {a.address, b.address}
+    assert await y1 == 2
+    # y1 is finished so there's a worker available without a waiter
+    s.extensions["amm"].run_once()
+    while {ws.address for ws in s.tasks["x"].who_has} != {b.address}:
+        await asyncio.sleep(0.01)
+    assert not y2.done()
+
+
+@pytest.mark.xfail(reason="distributed#5265")
+@gen_cluster(client=True, config=NO_AMM_START)
+async def test_double_drop(c, s, a, b):
+    """An AMM drop policy runs once to drop one of the two replicas of a key.
+    Then it runs again, before the recommendations from the first iteration had the time
+    to either be enacted or rejected, and chooses a different worker to drop from.
+
+    Test that, in this use case, the last replica of a key is never dropped.
+    """
+    futures = await c.scatter({"x": 1}, broadcast=True)
+    assert len(s.tasks["x"].who_has) == 2
+    ws_iter = iter(s.workers.values())
+
+    class Policy(ActiveMemoryManagerPolicy):
+        def run(self):
+            yield "drop", s.tasks["x"], {next(ws_iter)}
+
+    amm = ActiveMemoryManagerExtension(s, {Policy()}, register=False, start=False)
+    amm.run_once()
+    amm.run_once()
+    while len(s.tasks["x"].who_has) > 1:
+        await asyncio.sleep(0.01)
+    await asyncio.sleep(0.2)
+    assert len(s.tasks["x"].who_has) == 1
+
+
+@gen_cluster(client=True, config=demo_config("drop"))
+async def test_double_drop_stress(c, s, a, b):
+    """AMM runs many times before the recommendations of the first run are enacted"""
+    futures = await c.scatter({"x": 1}, broadcast=True)
+    assert len(s.tasks["x"].who_has) == 2
+    for _ in range(10):
+        s.extensions["amm"].run_once()
+    while len(s.tasks["x"].who_has) > 1:
+        await asyncio.sleep(0.01)
+    await asyncio.sleep(0.2)
+    assert len(s.tasks["x"].who_has) == 1
+
+
+@pytest.mark.slow
+@gen_cluster(
+    nthreads=[("", 1)] * 4,
+    Worker=Nanny,
+    client=True,
+    worker_kwargs={"memory_limit": "2 GiB"},
+    config=demo_config("drop", n=1),
+)
+async def test_drop_from_worker_with_least_free_memory(c, s, *nannies):
+    a1, a2, a3, a4 = s.workers.keys()
+    ws1, ws2, ws3, ws4 = s.workers.values()
+
+    futures = await c.scatter({"x": 1}, broadcast=True)
+    assert s.tasks["x"].who_has == {ws1, ws2, ws3, ws4}
+    # Allocate enough RAM to be safely more than unmanaged memory
+    clog = c.submit(lambda: "x" * 2 ** 29, workers=[a3])  # 512 MiB
+    # await wait(clog) is not enough; we need to wait for the heartbeats
+    while ws3.memory.optimistic < 2 ** 29:
+        await asyncio.sleep(0.01)
+    s.extensions["amm"].run_once()
+
+    while s.tasks["x"].who_has != {ws1, ws2, ws4}:
+        await asyncio.sleep(0.01)
+
+
+@gen_cluster(
+    nthreads=[("", 1)] * 8,
+    client=True,
+    config=demo_config("drop", n=1, candidates={5, 6}),
+)
+async def test_drop_with_candidates(c, s, *workers):
+    futures = await c.scatter({"x": 1}, broadcast=True)
+    s.extensions["amm"].run_once()
+    wss = list(s.workers.values())
+    expect1 = {wss[0], wss[1], wss[2], wss[3], wss[4], wss[6], wss[7]}
+    expect2 = {wss[0], wss[1], wss[2], wss[3], wss[4], wss[5], wss[7]}
+    while s.tasks["x"].who_has not in (expect1, expect2):
+        await asyncio.sleep(0.01)
+
+
+@gen_cluster(client=True, config=demo_config("drop", candidates=set()))
+async def test_drop_with_empty_candidates(c, s, a, b):
+    """Key is not dropped as the plugin proposes an empty set of candidates,
+    not to be confused with None
+    """
+    futures = await c.scatter({"x": 1}, broadcast=True)
+    s.extensions["amm"].run_once()
+    await asyncio.sleep(0.2)
+    assert len(s.tasks["x"].who_has) == 2
+
+
+@gen_cluster(
+    client=True, nthreads=[("", 1)] * 3, config=demo_config("drop", candidates={2})
+)
+async def test_drop_from_candidates_without_key(c, s, *workers):
+    """Key is not dropped as none of the candidates hold a replica"""
+    ws0, ws1, ws2 = s.workers.values()
+    x = (await c.scatter({"x": 1}, workers=[ws0.address]))["x"]
+    y = c.submit(inc, x, key="y", workers=[ws1.address])
+    await y
+    assert s.tasks["x"].who_has == {ws0, ws1}
+
+    s.extensions["amm"].run_once()
+    await asyncio.sleep(0.2)
+    assert s.tasks["x"].who_has == {ws0, ws1}
+
+
+@gen_cluster(client=True, config=demo_config("drop", candidates={0}))
+async def test_drop_with_bad_candidates(c, s, a, b):
+    """Key is not dropped as all candidates hold waiter tasks"""
+    ws0, ws1 = s.workers.values()  # Not necessarily a, b; it could be b, a!
+    x = (await c.scatter({"x": 1}, broadcast=True))["x"]
+    y = c.submit(slowinc, x, 0.3, key="y", workers=[ws0.address])
+    while "y" not in s.tasks:
+        await asyncio.sleep(0.01)
+
+    s.extensions["amm"].run_once()
+    await y
+    assert s.tasks["x"].who_has == {ws0, ws1}
+
+
+class DropEverything(ActiveMemoryManagerPolicy):
+    """Inanely suggest to drop every single key in the cluster"""
+
+    def run(self):
+        for ts in self.manager.scheduler.tasks.values():
+            # Instead of yielding ("drop", ts, None) for each worker, which would result
+            # in semi-predictable output about which replica survives, randomly choose a
+            # different survivor at each AMM run.
+            candidates = list(ts.who_has)
+            random.shuffle(candidates)
+            for ws in candidates:
+                yield "drop", ts, {ws}
+
+
+@pytest.mark.xfail(reason="distributed#5046, distributed#5265")
+@pytest.mark.slow
+@gen_cluster(
+    client=True,
+    nthreads=[("", 1)] * 8,
+    Worker=Nanny,
+    config={
+        "distributed.scheduler.active-memory-manager.start": True,
+        "distributed.scheduler.active-memory-manager.interval": 0.1,
+        "distributed.scheduler.active-memory-manager.policies": [
+            {"class": "distributed.tests.test_active_memory_manager.DropEverything"},
+        ],
+    },
+)
+async def test_drop_stress(c, s, *nannies):
+    """A policy which suggests dropping everything won't break a running computation,
+    but only slow it down.
+    """
+    import dask.array as da
+
+    rng = da.random.RandomState(0)
+    a = rng.random((20, 20), chunks=(1, 1))
+    b = (a @ a.T).sum().round(3)
+    assert await c.compute(b) == 2134.398
+
+
+@gen_cluster(nthreads=[("", 1)] * 4, client=True, config=demo_config("replicate", n=2))
+async def test_replicate(c, s, *workers):
+    futures = await c.scatter({"x": 123})
+    assert len(s.tasks["x"].who_has) == 1
+
+    s.extensions["amm"].run_once()
+    while len(s.tasks["x"].who_has) < 3:
+        await asyncio.sleep(0.01)
+    await asyncio.sleep(0.2)
+    assert len(s.tasks["x"].who_has) == 3
+
+    s.extensions["amm"].run_once()
+    while len(s.tasks["x"].who_has) < 4:
+        await asyncio.sleep(0.01)
+
+    for w in workers:
+        assert w.data["x"] == 123
+
+
+@gen_cluster(client=True, config=demo_config("replicate"))
+async def test_replicate_not_in_memory(c, s, a, b):
+    """ts.who_has is empty"""
+    x = c.submit(slowinc, 1, key="x")
+    while "x" not in s.tasks:
+        await asyncio.sleep(0.01)
+    assert not x.done()
+    s.extensions["amm"].run_once()
+    assert await x == 2
+    assert len(s.tasks["x"].who_has) == 1
+    s.extensions["amm"].run_once()
+    while len(s.tasks["x"].who_has) < 2:
+        await asyncio.sleep(0.01)
+
+
+@gen_cluster(client=True, config=demo_config("replicate"))
+async def test_double_replicate_stress(c, s, a, b):
+    """AMM runs many times before the recommendations of the first run are enacted"""
+    futures = await c.scatter({"x": 1})
+    assert len(s.tasks["x"].who_has) == 1
+    for _ in range(10):
+        s.extensions["amm"].run_once()
+    while len(s.tasks["x"].who_has) < 2:
+        await asyncio.sleep(0.01)
+
+
+@pytest.mark.slow
+@gen_cluster(
+    nthreads=[("", 1)] * 4,
+    Worker=Nanny,
+    client=True,
+    worker_kwargs={"memory_limit": "2 GiB"},
+    config=demo_config("replicate", n=1),
+)
+async def test_replicate_to_worker_with_most_free_memory(c, s, *nannies):
+    a1, a2, a3, a4 = s.workers.keys()
+    ws1, ws2, ws3, ws4 = s.workers.values()
+
+    futures = await c.scatter({"x": 1}, workers=[a1])
+    assert s.tasks["x"].who_has == {ws1}
+    # Allocate enough RAM to be safely more than unmanaged memory
+    clog2 = c.submit(lambda: "x" * 2 ** 29, workers=[a2])  # 512 MiB
+    clog4 = c.submit(lambda: "x" * 2 ** 29, workers=[a4])  # 512 MiB
+    # await wait(clog) is not enough; we need to wait for the heartbeats
+    for ws in (ws2, ws4):
+        while ws.memory.optimistic < 2 ** 29:
+            await asyncio.sleep(0.01)
+    s.extensions["amm"].run_once()
+
+    while s.tasks["x"].who_has != {ws1, ws3}:
+        await asyncio.sleep(0.01)
+
+
+@gen_cluster(
+    nthreads=[("", 1)] * 8,
+    client=True,
+    config=demo_config("replicate", n=1, candidates={5, 6}),
+)
+async def test_replicate_with_candidates(c, s, *workers):
+    wss = list(s.workers.values())
+    futures = await c.scatter({"x": 1}, workers=[wss[0].address])
+    s.extensions["amm"].run_once()
+    expect1 = {wss[0], wss[5]}
+    expect2 = {wss[0], wss[6]}
+    while s.tasks["x"].who_has not in (expect1, expect2):
+        await asyncio.sleep(0.01)
+
+
+@gen_cluster(client=True, config=demo_config("replicate", candidates=set()))
+async def test_replicate_with_empty_candidates(c, s, a, b):
+    """Key is not replicated as the plugin proposes an empty set of candidates,
+    not to be confused with None
+    """
+    futures = await c.scatter({"x": 1})
+    s.extensions["amm"].run_once()
+    await asyncio.sleep(0.2)
+    assert len(s.tasks["x"].who_has) == 1
+
+
+@gen_cluster(client=True, config=demo_config("replicate", candidates={0}))
+async def test_replicate_to_candidates_with_key(c, s, a, b):
+    """Key is not replicated as all candidates already hold replicas"""
+    ws0, ws1 = s.workers.values()  # Not necessarily a, b; it could be b, a!
+    futures = await c.scatter({"x": 1}, workers=[ws0.address])
+    s.extensions["amm"].run_once()
+    await asyncio.sleep(0.2)
+    assert s.tasks["x"].who_has == {ws0}
+
+
+@gen_cluster(
+    nthreads=[("", 1)] * 4,
+    client=True,
+    config={
+        "distributed.scheduler.active-memory-manager.start": False,
+        "distributed.scheduler.active-memory-manager.policies": [
+            {"class": "distributed.active_memory_manager.ReduceReplicas"},
+            # Run two instances of the plugin in sequence, to emulate multiple plugins
+            # that issues drop suggestions for the same keys
+            {"class": "distributed.active_memory_manager.ReduceReplicas"},
+        ],
+    },
+)
+async def test_ReduceReplicas(c, s, *workers):
+    futures = await c.scatter({"x": 123}, broadcast=True)
+    assert len(s.tasks["x"].who_has) == 4
+    s.extensions["amm"].run_once()
+    while len(s.tasks["x"].who_has) > 1:
+        await asyncio.sleep(0.01)
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 260513e954e..23f59431cf0 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -2939,7 +2939,7 @@ async def test_gather_on_worker(c, s, a, b):
     assert x_ts not in b_ws.has_what
     assert x_ts.who_has == {a_ws}
 
-    out = await s._gather_on_worker(b.address, {x.key: [a.address]})
+    out = await s.gather_on_worker(b.address, {x.key: [a.address]})
     assert out == set()
     assert a.data[x.key] == "x"
     assert b.data[x.key] == "x"
@@ -2955,14 +2955,14 @@ async def test_gather_on_worker_bad_recipient(c, s, a, b):
     x = await c.scatter("x")
     await b.close()
     assert s.workers.keys() == {a.address}
-    out = await s._gather_on_worker(b.address, {x.key: [a.address]})
+    out = await s.gather_on_worker(b.address, {x.key: [a.address]})
     assert out == {x.key}
 
 
 @gen_cluster(client=True, worker_kwargs={"timeout": "100ms"})
 async def test_gather_on_worker_bad_sender(c, s, a, b):
     """The only sender for a key is missing"""
-    out = await s._gather_on_worker(a.address, {"x": ["tcp://127.0.0.1:12345"]})
+    out = await s.gather_on_worker(a.address, {"x": ["tcp://127.0.0.1:12345"]})
     assert out == {"x"}
 
 
@@ -2974,7 +2974,7 @@ async def test_gather_on_worker_bad_sender_replicated(c, s, a, b, missing_first)
     bad_addr = "tcp://127.0.0.1:12345"
     # Order matters; test both
     addrs = [bad_addr, a.address] if missing_first else [a.address, bad_addr]
-    out = await s._gather_on_worker(b.address, {x.key: addrs})
+    out = await s.gather_on_worker(b.address, {x.key: addrs})
     assert out == set()
     assert a.data[x.key] == "x"
     assert b.data[x.key] == "x"
@@ -2983,7 +2983,7 @@ async def test_gather_on_worker_bad_sender_replicated(c, s, a, b, missing_first)
 @gen_cluster(client=True)
 async def test_gather_on_worker_key_not_on_sender(c, s, a, b):
     """The only sender for a key does not actually hold it"""
-    out = await s._gather_on_worker(a.address, {"x": [b.address]})
+    out = await s.gather_on_worker(a.address, {"x": [b.address]})
     assert out == {"x"}
 
 
@@ -2998,7 +2998,7 @@ async def test_gather_on_worker_key_not_on_sender_replicated(
     x = await client.scatter("x", workers=[a.address])
     # Order matters; test both
     addrs = [b.address, a.address] if missing_first else [a.address, b.address]
-    out = await s._gather_on_worker(c.address, {x.key: addrs})
+    out = await s.gather_on_worker(c.address, {x.key: addrs})
     assert out == set()
     assert a.data[x.key] == "x"
     assert c.data[x.key] == "x"
@@ -3015,8 +3015,8 @@ async def test_gather_on_worker_duplicate_task(client, s, a, b, c):
     assert x.key not in c.data
 
     out = await asyncio.gather(
-        s._gather_on_worker(c.address, {x.key: [a.address]}),
-        s._gather_on_worker(c.address, {x.key: [b.address]}),
+        s.gather_on_worker(c.address, {x.key: [a.address]}),
+        s.gather_on_worker(c.address, {x.key: [b.address]}),
     )
     assert out == [set(), set()]
     assert c.data[x.key] == "x"
@@ -3064,7 +3064,7 @@ async def test_delete_worker_data(c, s, a, b):
     assert b.data == {y.key: "y"}
     assert s.tasks.keys() == {x.key, y.key, z.key}
 
-    await s._delete_worker_data(a.address, [x.key, y.key])
+    await s.delete_worker_data(a.address, [x.key, y.key])
     assert a.data == {z.key: "z"}
     assert b.data == {y.key: "y"}
     assert s.tasks.keys() == {y.key, z.key}
@@ -3078,8 +3078,8 @@ async def test_delete_worker_data_double_delete(c, s, a):
     """
     x, y = await c.scatter(["x", "y"])
     await asyncio.gather(
-        s._delete_worker_data(a.address, [x.key]),
-        s._delete_worker_data(a.address, [x.key]),
+        s.delete_worker_data(a.address, [x.key]),
+        s.delete_worker_data(a.address, [x.key]),
     )
     assert a.data == {y.key: "y"}
     a_ws = s.workers[a.address]
@@ -3094,7 +3094,7 @@ async def test_delete_worker_data_bad_worker(s, a, b):
     """
     await a.close()
     assert s.workers.keys() == {b.address}
-    await s._delete_worker_data(a.address, ["x"])
+    await s.delete_worker_data(a.address, ["x"])
 
 
 @pytest.mark.parametrize("bad_first", [False, True])
@@ -3109,7 +3109,7 @@ async def test_delete_worker_data_bad_task(c, s, a, bad_first):
     assert s.tasks.keys() == {x.key, y.key}
 
     keys = ["notexist", x.key] if bad_first else [x.key, "notexist"]
-    await s._delete_worker_data(a.address, keys)
+    await s.delete_worker_data(a.address, keys)
     assert a.data == {y.key: "y"}
     assert s.tasks.keys() == {y.key}
     assert s.workers[a.address].nbytes == s.tasks[y.key].nbytes

From 4355e9d3d228fb5949343ae7a1664fafd513abf6 Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Wed, 8 Sep 2021 16:24:06 +0200
Subject: [PATCH 1477/1550] Increase worker.suspicious_counter threshold
 (#5228)

---
 distributed/scheduler.py         | 13 +++++++++----
 distributed/tests/test_stress.py |  7 ++++++-
 distributed/worker.py            |  9 ++++++---
 3 files changed, 21 insertions(+), 8 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 22a33372b5b..b9b5380ded3 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -4715,7 +4715,7 @@ def stimulus_task_finished(self, key=None, worker=None, **kwargs):
                     "reason": f"already-released-or-forgotten-{time()}",
                 }
             ]
-        elif ts.state == "memory":
+        elif ts._state == "memory":
             self.add_keys(worker=worker, keys=[key])
         else:
             ts._metadata.update(kwargs["metadata"])
@@ -5345,8 +5345,8 @@ def handle_release_data(self, key=None, worker=None, client=None, **msg):
         ts: TaskState = parent._tasks.get(key)
         if ts is None:
             return
-        ws: WorkerState = parent._workers_dv[worker]
-        if ts._processing_on != ws:
+        ws: WorkerState = parent._workers_dv.get(worker)
+        if ws is None or ts._processing_on != ws:
             return
 
         recommendations: dict
@@ -5380,7 +5380,9 @@ def handle_missing_data(self, key=None, errant_worker=None, **kwargs):
 
     def release_worker_data(self, comm=None, keys=None, worker=None):
         parent: SchedulerState = cast(SchedulerState, self)
-        ws: WorkerState = parent._workers_dv[worker]
+        ws: WorkerState = parent._workers_dv.get(worker)
+        if not ws:
+            return
         tasks: set = {parent._tasks[k] for k in keys if k in parent._tasks}
         removed_tasks: set = tasks.intersection(ws._has_what)
 
@@ -5599,6 +5601,9 @@ def send_all(self, client_msgs: dict, worker_msgs: dict):
             try:
                 w = stream_comms[worker]
                 w.send(*msgs)
+            except KeyError:
+                # worker already gone
+                pass
             except (CommClosedError, AttributeError):
                 self.loop.add_callback(self.remove_worker, address=worker)
 
diff --git a/distributed/tests/test_stress.py b/distributed/tests/test_stress.py
index 3c410ea322e..afc4556afc9 100644
--- a/distributed/tests/test_stress.py
+++ b/distributed/tests/test_stress.py
@@ -96,6 +96,9 @@ async def test_stress_creation_and_deletion(c, s):
     # Assertions are handled by the validate mechanism in the scheduler
     da = pytest.importorskip("dask.array")
 
+    def _disable_suspicious_counter(dask_worker):
+        dask_worker._suspicious_count_limit = None
+
     rng = da.random.RandomState(0)
     x = rng.random(size=(2000, 2000), chunks=(100, 100))
     y = ((x + 1).T + (x * 2) - x.mean(axis=1)).sum().round(2)
@@ -104,13 +107,15 @@ async def test_stress_creation_and_deletion(c, s):
     async def create_and_destroy_worker(delay):
         start = time()
         while time() < start + 5:
-            async with Nanny(s.address, nthreads=2):
+            async with Nanny(s.address, nthreads=2) as n:
+                await c.run(_disable_suspicious_counter, workers=[n.worker_address])
                 await asyncio.sleep(delay)
             print("Killed nanny")
 
     await asyncio.gather(*(create_and_destroy_worker(0.1 * i) for i in range(20)))
 
     async with Nanny(s.address, nthreads=2):
+        await c.run(_disable_suspicious_counter)
         assert await c.compute(z) == 8000884.93
 
 
diff --git a/distributed/worker.py b/distributed/worker.py
index 62c1d55bb11..abd0fb4b7f6 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -744,7 +744,7 @@ def __init__(
             lambda: self.batched_stream.send({"op": "keep-alive"}), 60000
         )
         self.periodic_callbacks["keep-alive"] = pc
-
+        self._suspicious_count_limit = 10
         self._address = contact_address
 
         self.memory_monitor_interval = parse_timedelta(
@@ -2538,7 +2538,7 @@ def bad_dep(self, dep):
         exc = ValueError(
             "Could not find dependent %s.  Check worker logs" % str(dep.key)
         )
-        for ts in dep.dependents:
+        for ts in list(dep.dependents):
             msg = error_message(exc)
             ts.exception = msg["exception"]
             ts.traceback = msg["traceback"]
@@ -2555,7 +2555,10 @@ async def handle_missing_dep(self, *deps, **kwargs):
                 return
 
             for dep in list(deps):
-                if dep.suspicious_count > 5:
+                if (
+                    self._suspicious_count_limit
+                    and dep.suspicious_count > self._suspicious_count_limit
+                ):
                     deps.remove(dep)
                     self.bad_dep(dep)
             if not deps:

From fc06b52a50eaee5689416bb3af5ec62176096eb6 Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Thu, 9 Sep 2021 04:23:18 +0100
Subject: [PATCH 1478/1550] Use `gen_test` in `test_adaptive` (#5298)

---
 distributed/deploy/adaptive.py            |  2 +-
 distributed/deploy/tests/test_adaptive.py | 33 +++++++++++------------
 2 files changed, 17 insertions(+), 18 deletions(-)

diff --git a/distributed/deploy/adaptive.py b/distributed/deploy/adaptive.py
index 2f966a3f578..2ce14a1e9f6 100644
--- a/distributed/deploy/adaptive.py
+++ b/distributed/deploy/adaptive.py
@@ -168,7 +168,7 @@ async def workers_to_close(self, target: int):
 
         Returns
         -------
-        List of worker addresses to close, if any
+        List of worker names to close, if any
 
         See Also
         --------
diff --git a/distributed/deploy/tests/test_adaptive.py b/distributed/deploy/tests/test_adaptive.py
index 6f8067c4d29..b21a19ad6f0 100644
--- a/distributed/deploy/tests/test_adaptive.py
+++ b/distributed/deploy/tests/test_adaptive.py
@@ -43,7 +43,7 @@ def test_adaptive_local_cluster(loop):
             start = time()
             while cluster.scheduler.nthreads:
                 sleep(0.01)
-                assert time() < start + 5
+                assert time() < start + 30
 
             assert not c.nthreads()
 
@@ -163,8 +163,8 @@ async def test_min_max():
         await cluster.close()
 
 
-@pytest.mark.asyncio
-async def test_avoid_churn(cleanup):
+@gen_test()
+async def test_avoid_churn():
     """We want to avoid creating and deleting workers frequently
 
     Instead we want to wait a few beats before removing a worker in case the
@@ -187,7 +187,7 @@ async def test_avoid_churn(cleanup):
             assert len(adapt.log) == 1
 
 
-@pytest.mark.asyncio
+@gen_test()
 async def test_adapt_quickly():
     """We want to avoid creating and deleting workers frequently
 
@@ -232,7 +232,6 @@ async def test_adapt_quickly():
 
         # Don't scale up for large sequential computations
         x = await client.scatter(1)
-        log = list(cluster._adaptive.log)
         for i in range(100):
             x = client.submit(slowinc, x)
 
@@ -325,8 +324,8 @@ async def test_target_duration():
             assert adapt.log[1][1] == {"status": "up", "n": 20}
 
 
-@pytest.mark.asyncio
-async def test_worker_keys(cleanup):
+@gen_test()
+async def test_worker_keys():
     """Ensure that redefining adapt with a lower maximum removes workers"""
     async with SpecCluster(
         scheduler={"cls": Scheduler, "options": {"dashboard_address": ":0"}},
@@ -354,8 +353,8 @@ def key(ws):
         assert names == {"a-1", "a-2"} or names == {"b-1", "b-2"}
 
 
-@pytest.mark.asyncio
-async def test_adapt_cores_memory(cleanup):
+@gen_test()
+async def test_adapt_cores_memory():
     async with LocalCluster(
         n_workers=0,
         threads_per_worker=2,
@@ -394,8 +393,8 @@ def test_adaptive_config():
         assert adapt.wait_count == 8
 
 
-@pytest.mark.asyncio
-async def test_update_adaptive(cleanup):
+@gen_test()
+async def test_update_adaptive():
     async with LocalCluster(
         n_workers=0,
         threads_per_worker=2,
@@ -412,9 +411,9 @@ async def test_update_adaptive(cleanup):
         assert second.periodic_callback.is_running()
 
 
-@pytest.mark.asyncio
-async def test_adaptive_no_memory_limit(cleanup):
-    """Make sure that adapt() does not keep creating workers when no memory limit is set."""
+@gen_test()
+async def test_adaptive_no_memory_limit():
+    """Test that adapt() does not keep creating workers when no memory limit is set"""
     async with LocalCluster(
         n_workers=0,
         threads_per_worker=1,
@@ -435,8 +434,8 @@ async def test_adaptive_no_memory_limit(cleanup):
         )
 
 
-@pytest.mark.asyncio
-async def test_scale_needs_to_be_awaited(cleanup):
+@gen_test()
+async def test_scale_needs_to_be_awaited():
     """
     This tests that the adaptive class works fine if the scale method uses the
     `sync` method to schedule its task instead of loop.add_callback
@@ -466,7 +465,7 @@ async def _():
             await async_wait_for(lambda: not cluster.workers, 10)
 
 
-@pytest.mark.asyncio
+@gen_test()
 async def test_adaptive_stopped():
     """
     We should ensure that the adapt PC is actually stopped once the cluster

From 2df7d7e1e68b3972f3fbdfad684358e63235f3aa Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Thu, 9 Sep 2021 11:22:55 +0200
Subject: [PATCH 1479/1550] Fix flaky test_worker_reconnects_mid_compute
 (#5299)

---
 distributed/tests/test_worker.py | 26 +++++++++++---------------
 1 file changed, 11 insertions(+), 15 deletions(-)

diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index de0f9e63987..b001c4443c1 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -2409,11 +2409,6 @@ async def test_hold_on_to_replicas(c, s, *workers):
         await asyncio.sleep(0.01)
 
 
-@pytest.mark.flaky(
-    condition=WINDOWS and sys.version_info[:2] == (3, 8),
-    reruns=20,
-    reruns_delay=5,
-)
 @gen_cluster(client=True)
 async def test_worker_reconnects_mid_compute(c, s, a, b):
     """Ensure that, if a worker disconnects while computing a result, the scheduler will
@@ -2468,8 +2463,11 @@ def fast_on_a(lock):
 
     assert "Unexpected worker completed task" in s_logs.getvalue()
 
-    while a.address not in {w.address for w in s.tasks[f2.key].who_has}:
-        await asyncio.sleep(0.001)
+    # Ensure that all in-memory tasks on A have been restored on the
+    # scheduler after reconnect
+    for ts in a.tasks.values():
+        if ts.state == "memory":
+            assert a.address in {ws.address for ws in s.tasks[ts.key].who_has}
 
     # Ensure that all keys have been properly registered and will also be
     # cleaned up nicely.
@@ -2479,12 +2477,7 @@ def fast_on_a(lock):
         await asyncio.sleep(0.001)
 
 
-@pytest.mark.flaky(
-    condition=WINDOWS and sys.version_info[:2] == (3, 8),
-    reruns=20,
-    reruns_delay=5,
-)
-@gen_cluster(client=True)
+@gen_cluster(client=True, timeout=5)
 async def test_worker_reconnects_mid_compute_multiple_states_on_scheduler(c, s, a, b):
     """
     Ensure that a reconnecting worker does not break the scheduler regardless of
@@ -2540,8 +2533,11 @@ def fast_on_a(lock):
 
     assert "Unexpected worker completed task" in s_logs.getvalue()
 
-    while a.address not in {w.address for w in s.tasks[f2.key].who_has}:
-        await asyncio.sleep(0.001)
+    # Ensure that all in-memory tasks on A have been restored on the
+    # scheduler after reconnect
+    for ts in a.tasks.values():
+        if ts.state == "memory":
+            assert a.address in {ws.address for ws in s.tasks[ts.key].who_has}
 
     del f1, f2, f3
     while any(w.tasks for w in [a, b]):

From 8c455a6c99794fda916142b6b4c14d39379a4f26 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Thu, 9 Sep 2021 04:24:02 -0500
Subject: [PATCH 1480/1550] Revert "Add test setup fixture (#5242)" (#5300)

This reverts commit b0c9dc4d7e5746550114a3f13831c278bc0db6d5.
---
 conftest.py               | 33 ---------------------------------
 distributed/utils_test.py | 31 +++++++++++++++++++++++++------
 2 files changed, 25 insertions(+), 39 deletions(-)

diff --git a/conftest.py b/conftest.py
index fde33e4e310..ebb9dae1457 100644
--- a/conftest.py
+++ b/conftest.py
@@ -1,11 +1,6 @@
 # https://pytest.org/latest/example/simple.html#control-skipping-of-tests-according-to-command-line-option
-import copy
-import logging
-
 import pytest
 
-import dask
-
 # Uncomment to enable more logging and checks
 # (https://docs.python.org/3/library/asyncio-dev.html)
 # Note this makes things slower and might consume much memory.
@@ -40,31 +35,3 @@ def pytest_collection_modifyitems(config, items):
 
 
 pytest_plugins = ["distributed.pytest_resourceleaks"]
-
-
-_original_config = copy.deepcopy(dask.config.config)
-# Custom preloads can interact with the test suite in unexpected ways
-# so we remove them when running tests
-for node in ["scheduler", "worker", "nanny"]:
-    _original_config["distributed"][node]["preload"] = []
-    _original_config["distributed"][node]["preload-argv"] = []
-
-_logging_levels = {
-    name: logger.level
-    for name, logger in logging.root.manager.loggerDict.items()
-    if isinstance(logger, logging.Logger)
-}
-
-
-@pytest.fixture(autouse=True)
-def initialize_test():
-
-    # Restore default logging levels
-    for name, level in _logging_levels.items():
-        logging.getLogger(name).setLevel(level)
-
-    # Ensure a clean config
-    dask.config.config.clear()
-    dask.config.config.update(copy.deepcopy(_original_config))
-
-    yield
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index b9354a5755c..a9c5496efce 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -77,6 +77,12 @@
 logger = logging.getLogger(__name__)
 
 
+logging_levels = {
+    name: logger.level
+    for name, logger in logging.root.manager.loggerDict.items()
+    if isinstance(logger, logging.Logger)
+}
+
 _TEST_TIMEOUT = 30
 _offload_executor.submit(lambda: None).result()  # create thread during import
 
@@ -209,6 +215,14 @@ def get_ip():
     remote_magic._clients.clear()
 
 
+original_config = copy.deepcopy(dask.config.config)
+
+
+def reset_config():
+    dask.config.config.clear()
+    dask.config.config.update(copy.deepcopy(original_config))
+
+
 def nodebug(func):
     """
     A decorator to disable debug facilities during timing-sensitive tests.
@@ -1571,13 +1585,18 @@ def clean(threads=not WINDOWS, instances=True, timeout=1, processes=True):
             with check_process_leak(check=processes):
                 with check_instances() if instances else nullcontext():
                     with check_active_rpc(loop, timeout):
-                        with dask.config.set(
-                            {"distributed.comm.timeouts.connect": "5s"}
-                        ):
-                            yield loop
+                        reset_config()
+
+                        dask.config.set({"distributed.comm.timeouts.connect": "5s"})
+                        # Restore default logging levels
+                        # XXX use pytest hooks/fixtures instead?
+                        for name, level in logging_levels.items():
+                            logging.getLogger(name).setLevel(level)
+
+                        yield loop
 
-                            with suppress(AttributeError):
-                                del thread_state.on_event_loop_thread
+                        with suppress(AttributeError):
+                            del thread_state.on_event_loop_thread
 
 
 @pytest.fixture

From 3fba8f2613461cc4ec68aa4be158745007c5a530 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Thu, 9 Sep 2021 04:26:09 -0500
Subject: [PATCH 1481/1550] Update test_sub_submit_priority (#5301)

---
 distributed/tests/test_client.py | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index f571f7dd0ed..706e804c74e 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -5172,16 +5172,19 @@ def f(x):
 
 @gen_cluster(client=True)
 async def test_sub_submit_priority(c, s, a, b):
-    def f():
+    def func():
         client = get_client()
-        client.submit(slowinc, 1, delay=0.2, key="slowinc")
+        f = client.submit(slowinc, 1, delay=0.5, key="slowinc")
+        client.gather(f)
 
-    future = c.submit(f, key="f")
-    await asyncio.sleep(0.1)
-    if len(s.tasks) == 2:
-        assert (
-            s.priorities["f"] > s.priorities["slowinc"]
-        )  # lower values schedule first
+    future = c.submit(func, key="f")
+    while len(s.tasks) != 2:
+        await asyncio.sleep(0.001)
+    # lower values schedule first
+    assert s.tasks["f"].priority > s.tasks["slowinc"].priority, (
+        s.tasks["f"].priority,
+        s.tasks["slowinc"].priority,
+    )
 
 
 def test_get_client_sync(c, s, a, b):

From 5bf60e3b59d2e8d622c56587fb86e20a4ea0f25c Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Thu, 9 Sep 2021 10:44:15 +0100
Subject: [PATCH 1482/1550] Add synced dict between cluster and scheduler to
 store cluster info (#5033)

Adds a `cluster_info` attribute to all `Cluster` objects which is a dictionary that is synced to the scheduler periodically. Any info already on the scheduler during `_start` is merged into the dict in `Cluster` and then that dict is synced back to the scheduler every second.
---
 distributed/deploy/cluster.py          | 48 ++++++++++++++++++++------
 distributed/deploy/local.py            |  2 ++
 distributed/deploy/spec.py             |  2 ++
 distributed/deploy/tests/test_local.py | 27 +++++++++++++++
 4 files changed, 69 insertions(+), 10 deletions(-)

diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
index bc96ccfe23b..8beef70f6ba 100644
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -1,4 +1,5 @@
 import asyncio
+import copy
 import datetime
 import logging
 import threading
@@ -9,7 +10,7 @@
 from tornado.ioloop import PeriodicCallback
 
 import dask.config
-from dask.utils import _deprecated, format_bytes, parse_timedelta
+from dask.utils import _deprecated, format_bytes, parse_timedelta, typename
 from dask.widgets import get_template
 
 from ..core import Status
@@ -44,9 +45,8 @@ class Cluster:
     """
 
     _supports_scaling = True
-    name = None
 
-    def __init__(self, asynchronous, quiet=False, name=None):
+    def __init__(self, asynchronous, quiet=False, name=None, scheduler_sync_interval=1):
         self.scheduler_info = {"workers": {}}
         self.periodic_callbacks = {}
         self._asynchronous = asynchronous
@@ -56,13 +56,24 @@ def __init__(self, asynchronous, quiet=False, name=None):
         self.quiet = quiet
         self.scheduler_comm = None
         self._adaptive = None
+        self._sync_interval = parse_timedelta(
+            scheduler_sync_interval, default="seconds"
+        )
+
+        if name is None:
+            name = str(uuid.uuid4())[:8]
 
-        if name is not None:
-            self.name = name
-        elif self.name is None:
-            self.name = str(uuid.uuid4())[:8]
+        self._cluster_info = {"name": name, "type": typename(type(self))}
         self.status = Status.created
 
+    @property
+    def name(self):
+        return self._cluster_info["name"]
+
+    @name.setter
+    def name(self, name):
+        self._cluster_info["name"] = name
+
     async def _start(self):
         comm = await self.scheduler_comm.live_comm()
         await comm.write({"op": "subscribe_worker_status"})
@@ -71,8 +82,25 @@ async def _start(self):
         self._watch_worker_status_task = asyncio.ensure_future(
             self._watch_worker_status(comm)
         )
+
+        info = await self.scheduler_comm.get_metadata(
+            keys=["cluster-manager-info"], default={}
+        )
+        self._cluster_info.update(info)
+
+        self.periodic_callbacks["sync-cluster-info"] = PeriodicCallback(
+            self._sync_cluster_info, self._sync_interval * 1000
+        )
+        for pc in self.periodic_callbacks.values():
+            pc.start()
         self.status = Status.running
 
+    async def _sync_cluster_info(self):
+        await self.scheduler_comm.set_metadata(
+            keys=["cluster-manager-info"],
+            value=copy.copy(self._cluster_info),
+        )
+
     async def _close(self):
         if self.status == Status.closed:
             return
@@ -85,12 +113,12 @@ async def _close(self):
         if self._watch_worker_status_task:
             await self._watch_worker_status_task
 
-        for pc in self.periodic_callbacks.values():
-            pc.stop()
-
         if self.scheduler_comm:
             await self.scheduler_comm.close_rpc()
 
+        for pc in self.periodic_callbacks.values():
+            pc.stop()
+
         self.status = Status.closed
 
     def close(self, timeout=None):
diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index 19bc7f73e8a..bf002f504d1 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -121,6 +121,7 @@ def __init__(
         interface=None,
         worker_class=None,
         scheduler_kwargs=None,
+        scheduler_sync_interval=1,
         **worker_kwargs,
     ):
         if ip is not None:
@@ -241,6 +242,7 @@ def __init__(
             asynchronous=asynchronous,
             silence_logs=silence_logs,
             security=security,
+            scheduler_sync_interval=scheduler_sync_interval,
         )
 
     def start_worker(self, *args, **kwargs):
diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index e4e201f815d..c7646032c5b 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -245,6 +245,7 @@ def __init__(
         silence_logs=False,
         name=None,
         shutdown_on_close=True,
+        scheduler_sync_interval=1,
     ):
         self._created = weakref.WeakSet()
 
@@ -274,6 +275,7 @@ def __init__(
         super().__init__(
             asynchronous=asynchronous,
             name=name,
+            scheduler_sync_interval=scheduler_sync_interval,
         )
 
         if not self.asynchronous:
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index 80bb39e45ca..3f77bd46df9 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -1073,3 +1073,30 @@ async def test_local_cluster_redundant_kwarg(nanny):
             async with Client(cluster) as c:
                 f = c.submit(sleep, 0)
                 await f
+
+
+@pytest.mark.asyncio
+async def test_cluster_info_sync():
+    async with LocalCluster(
+        processes=False, asynchronous=True, scheduler_sync_interval="1ms"
+    ) as cluster:
+        assert cluster._cluster_info["name"] == cluster.name
+
+        while "name" not in cluster.scheduler.get_metadata(
+            keys=["cluster-manager-info"]
+        ):
+            await asyncio.sleep(0.01)
+
+        info = await cluster.scheduler_comm.get_metadata(keys=["cluster-manager-info"])
+        assert info["name"] == cluster.name
+        info = cluster.scheduler.get_metadata(keys=["cluster-manager-info"])
+        assert info["name"] == cluster.name
+
+        cluster._cluster_info["foo"] = "bar"
+        while "foo" not in cluster.scheduler.get_metadata(
+            keys=["cluster-manager-info"]
+        ):
+            await asyncio.sleep(0.01)
+
+        info = cluster.scheduler.get_metadata(keys=["cluster-manager-info"])
+        assert info["foo"] == "bar"

From 31afb54cf02141050ae1cc7cbd7c3919557637fe Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Thu, 9 Sep 2021 14:37:53 +0200
Subject: [PATCH 1483/1550] Increase timeout for
 test_worker_reconnects_mid_compute_multiple_states_on_scheduler (#5304)

---
 distributed/tests/test_worker.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index b001c4443c1..7846bd8de16 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -2477,7 +2477,7 @@ def fast_on_a(lock):
         await asyncio.sleep(0.001)
 
 
-@gen_cluster(client=True, timeout=5)
+@gen_cluster(client=True)
 async def test_worker_reconnects_mid_compute_multiple_states_on_scheduler(c, s, a, b):
     """
     Ensure that a reconnecting worker does not break the scheduler regardless of

From 0774365a305a22f05ff916318bdba57add3f2eb7 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Fri, 10 Sep 2021 02:35:38 -0700
Subject: [PATCH 1484/1550] Fix-up `test_pickle_empty` (#5303)

Switches to using the internal utility class `MemoryviewHolder` instead of NumPy to allow testing without NumPy. Calls `serialize` directly with `pickle` as the only option (instead of calling `pickle_dumps`). Though also `assert`s pickling did occur. Also includes some other minor fixes.

Only tries manipulating writability of frames with pickle protocol 5 in use. This doesn't work for earlier pickle protocols as there is only one `bytes` object containing everything (and no way to inspect what went into it, let alone reproduce it).
---
 distributed/protocol/tests/test_pickle.py | 50 +++++++++++++++--------
 1 file changed, 33 insertions(+), 17 deletions(-)

diff --git a/distributed/protocol/tests/test_pickle.py b/distributed/protocol/tests/test_pickle.py
index a25d499ea0f..a4ab8435646 100644
--- a/distributed/protocol/tests/test_pickle.py
+++ b/distributed/protocol/tests/test_pickle.py
@@ -8,7 +8,6 @@
 
 from distributed.protocol import deserialize, serialize
 from distributed.protocol.pickle import HIGHEST_PROTOCOL, dumps, loads
-from distributed.protocol.serialize import pickle_dumps
 
 if sys.version_info < (3, 8):
     try:
@@ -19,6 +18,17 @@
     import pickle
 
 
+class MemoryviewHolder:
+    def __init__(self, mv):
+        self.mv = memoryview(mv)
+
+    def __reduce_ex__(self, protocol):
+        if protocol >= 5:
+            return MemoryviewHolder, (pickle.PickleBuffer(self.mv),)
+        else:
+            return MemoryviewHolder, (self.mv.tobytes(),)
+
+
 def test_pickle_data():
     data = [1, b"123", "123", [123], {}, set()]
     for d in data:
@@ -27,16 +37,6 @@ def test_pickle_data():
 
 
 def test_pickle_out_of_band():
-    class MemoryviewHolder:
-        def __init__(self, mv):
-            self.mv = memoryview(mv)
-
-        def __reduce_ex__(self, protocol):
-            if protocol >= 5:
-                return MemoryviewHolder, (pickle.PickleBuffer(self.mv),)
-            else:
-                return MemoryviewHolder, (self.mv.tobytes(),)
-
     mv = memoryview(b"123")
     mvh = MemoryviewHolder(mv)
 
@@ -73,13 +73,29 @@ def __reduce_ex__(self, protocol):
 
 
 def test_pickle_empty():
-    np = pytest.importorskip("numpy")
-    x = np.arange(2)[0:0]  # Empty view
-    header, frames = pickle_dumps(x)
-    header["writeable"] = [False] * len(frames)
+    x = MemoryviewHolder(bytearray())  # Empty view
+    header, frames = serialize(x, serializers=("pickle",))
+
+    assert header["serializer"] == "pickle"
+    assert len(frames) >= 1
+    assert isinstance(frames[0], bytes)
+
+    if HIGHEST_PROTOCOL >= 5:
+        assert len(frames) == 2
+        assert len(header["writeable"]) == 1
+
+        header["writeable"] = (False,) * len(frames)
+    else:
+        assert len(frames) == 1
+        assert len(header["writeable"]) == 0
+
     y = deserialize(header, frames)
-    assert memoryview(y).nbytes == 0
-    assert memoryview(y).readonly
+
+    assert isinstance(y, MemoryviewHolder)
+    assert isinstance(y.mv, memoryview)
+    assert y.mv == x.mv
+    assert y.mv.nbytes == 0
+    assert y.mv.readonly
 
 
 def test_pickle_numpy():

From 518024aa53498ead005882511d82eebe0801ffc3 Mon Sep 17 00:00:00 2001
From: jakirkham <jakirkham@gmail.com>
Date: Sat, 11 Sep 2021 09:40:39 -0700
Subject: [PATCH 1485/1550] Test pickle protocols 4 & 5 (#5313)

Covers testing of both pickle protocol 4 & 5 in the pickle serialization
test suite. This should help catch assumptions predicated on protocol 5
being in use that are not always the case (as happened with the test
failure in cloudpickle recently).
---
 distributed/protocol/tests/test_pickle.py | 71 ++++++++++++++---------
 1 file changed, 45 insertions(+), 26 deletions(-)

diff --git a/distributed/protocol/tests/test_pickle.py b/distributed/protocol/tests/test_pickle.py
index a4ab8435646..436eb78ad5d 100644
--- a/distributed/protocol/tests/test_pickle.py
+++ b/distributed/protocol/tests/test_pickle.py
@@ -29,40 +29,46 @@ def __reduce_ex__(self, protocol):
             return MemoryviewHolder, (self.mv.tobytes(),)
 
 
-def test_pickle_data():
+@pytest.mark.parametrize("protocol", {4, HIGHEST_PROTOCOL})
+def test_pickle_data(protocol):
+    context = {"pickle-protocol": protocol}
+
     data = [1, b"123", "123", [123], {}, set()]
     for d in data:
-        assert loads(dumps(d)) == d
-        assert deserialize(*serialize(d, serializers=("pickle",))) == d
+        assert loads(dumps(d, protocol=protocol)) == d
+        assert deserialize(*serialize(d, serializers=("pickle",), context=context)) == d
+
 
+@pytest.mark.parametrize("protocol", {4, HIGHEST_PROTOCOL})
+def test_pickle_out_of_band(protocol):
+    context = {"pickle-protocol": protocol}
 
-def test_pickle_out_of_band():
     mv = memoryview(b"123")
     mvh = MemoryviewHolder(mv)
 
-    if HIGHEST_PROTOCOL >= 5:
+    if protocol >= 5:
         l = []
-        d = dumps(mvh, buffer_callback=l.append)
+        d = dumps(mvh, protocol=protocol, buffer_callback=l.append)
         mvh2 = loads(d, buffers=l)
 
         assert len(l) == 1
         assert isinstance(l[0], pickle.PickleBuffer)
         assert memoryview(l[0]) == mv
     else:
-        mvh2 = loads(dumps(mvh))
+        mvh2 = loads(dumps(mvh, protocol=protocol))
 
     assert isinstance(mvh2, MemoryviewHolder)
     assert isinstance(mvh2.mv, memoryview)
     assert mvh2.mv == mv
 
-    h, f = serialize(mvh, serializers=("pickle",))
+    h, f = serialize(mvh, serializers=("pickle",), context=context)
     mvh3 = deserialize(h, f)
 
     assert isinstance(mvh3, MemoryviewHolder)
     assert isinstance(mvh3.mv, memoryview)
     assert mvh3.mv == mv
 
-    if HIGHEST_PROTOCOL >= 5:
+    if protocol >= 5:
         assert len(f) == 2
         assert isinstance(f[0], bytes)
         assert isinstance(f[1], memoryview)
@@ -72,15 +78,18 @@ def test_pickle_out_of_band():
         assert isinstance(f[0], bytes)
 
 
-def test_pickle_empty():
+@pytest.mark.parametrize("protocol", {4, HIGHEST_PROTOCOL})
+def test_pickle_empty(protocol):
+    context = {"pickle-protocol": protocol}
+
     x = MemoryviewHolder(bytearray())  # Empty view
-    header, frames = serialize(x, serializers=("pickle",))
+    header, frames = serialize(x, serializers=("pickle",), context=context)
 
     assert header["serializer"] == "pickle"
     assert len(frames) >= 1
     assert isinstance(frames[0], bytes)
 
-    if HIGHEST_PROTOCOL >= 5:
+    if protocol >= 5:
         assert len(frames) == 2
         assert len(header["writeable"]) == 1
 
@@ -98,25 +107,32 @@ def test_pickle_empty():
     assert y.mv.readonly
 
 
-def test_pickle_numpy():
+@pytest.mark.parametrize("protocol", {4, HIGHEST_PROTOCOL})
+def test_pickle_numpy(protocol):
     np = pytest.importorskip("numpy")
+    context = {"pickle-protocol": protocol}
+
     x = np.ones(5)
-    assert (loads(dumps(x)) == x).all()
-    assert (deserialize(*serialize(x, serializers=("pickle",))) == x).all()
+    assert (loads(dumps(x, protocol=protocol)) == x).all()
+    assert (
+        deserialize(*serialize(x, serializers=("pickle",), context=context)) == x
+    ).all()
 
     x = np.ones(5000)
-    assert (loads(dumps(x)) == x).all()
-    assert (deserialize(*serialize(x, serializers=("pickle",))) == x).all()
+    assert (loads(dumps(x, protocol=protocol)) == x).all()
+    assert (
+        deserialize(*serialize(x, serializers=("pickle",), context=context)) == x
+    ).all()
 
     x = np.array([np.arange(3), np.arange(4, 6)], dtype=object)
-    x2 = loads(dumps(x))
+    x2 = loads(dumps(x, protocol=protocol))
     assert x.shape == x2.shape
     assert x.dtype == x2.dtype
     assert x.strides == x2.strides
     for e_x, e_x2 in zip(x.flat, x2.flat):
         np.testing.assert_equal(e_x, e_x2)
-    h, f = serialize(x, serializers=("pickle",))
-    if HIGHEST_PROTOCOL >= 5:
+    h, f = serialize(x, serializers=("pickle",), context=context)
+    if protocol >= 5:
         assert len(f) == 3
     else:
         assert len(f) == 1
@@ -127,24 +143,27 @@ def test_pickle_numpy():
     for e_x, e_x3 in zip(x.flat, x3.flat):
         np.testing.assert_equal(e_x, e_x3)
 
-    if HIGHEST_PROTOCOL >= 5:
+    if protocol >= 5:
         x = np.ones(5000)
 
         l = []
-        d = dumps(x, buffer_callback=l.append)
+        d = dumps(x, protocol=protocol, buffer_callback=l.append)
         assert len(l) == 1
         assert isinstance(l[0], pickle.PickleBuffer)
         assert memoryview(l[0]) == memoryview(x)
         assert (loads(d, buffers=l) == x).all()
 
-        h, f = serialize(x, serializers=("pickle",))
+        h, f = serialize(x, serializers=("pickle",), context=context)
         assert len(f) == 2
         assert isinstance(f[0], bytes)
         assert isinstance(f[1], memoryview)
         assert (deserialize(h, f) == x).all()
 
 
-def test_pickle_functions():
+@pytest.mark.parametrize("protocol", {4, HIGHEST_PROTOCOL})
+def test_pickle_functions(protocol):
+    context = {"pickle-protocol": protocol}
+
     def make_closure():
         value = 1
 
@@ -161,11 +180,11 @@ def funcs():
     for func in funcs():
         wr = weakref.ref(func)
 
-        func2 = loads(dumps(func))
+        func2 = loads(dumps(func, protocol=protocol))
         wr2 = weakref.ref(func2)
         assert func2(1) == func(1)
 
-        func3 = deserialize(*serialize(func, serializers=("pickle",)))
+        func3 = deserialize(*serialize(func, serializers=("pickle",), context=context))
         wr3 = weakref.ref(func3)
         assert func3(1) == func(1)
 

From 8863adbdff9f47763edb2c9ec9fdddf1d7e2b087 Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Mon, 13 Sep 2021 11:53:00 +0200
Subject: [PATCH 1486/1550] Allow Client to subscribe to events // Remote
 printing and warning (#5217)

---
 distributed/__init__.py                       |   2 +-
 distributed/client.py                         |  78 ++++++++++
 .../scheduler/tests/test_scheduler_http.py    |  10 +-
 distributed/nanny.py                          |   9 ++
 distributed/scheduler.py                      |  24 ++-
 distributed/tests/test_client.py              | 145 ++++++++++++++++++
 distributed/worker.py                         |  36 +++++
 7 files changed, 301 insertions(+), 3 deletions(-)

diff --git a/distributed/__init__.py b/distributed/__init__.py
index 57edc10d6bd..3da7ab429d0 100644
--- a/distributed/__init__.py
+++ b/distributed/__init__.py
@@ -44,7 +44,7 @@
 from .threadpoolexecutor import rejoin
 from .utils import CancelledError, TimeoutError, sync
 from .variable import Variable
-from .worker import Reschedule, Worker, get_client, get_worker, secede
+from .worker import Reschedule, Worker, get_client, get_worker, print, secede, warn
 from .worker_client import local_client, worker_client
 
 versions = get_versions()
diff --git a/distributed/client.py b/distributed/client.py
index 921e4fa0db2..b14f8777432 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -493,6 +493,22 @@ class AllExit(Exception):
     """Custom exception class to exit All(...) early."""
 
 
+def _handle_print(event):
+    _, msg = event
+    if isinstance(msg, dict) and "args" in msg and "kwargs" in msg:
+        print(*msg["args"], **msg["kwargs"])
+    else:
+        print(msg)
+
+
+def _handle_warn(event):
+    _, msg = event
+    if isinstance(msg, dict) and "args" in msg and "kwargs" in msg:
+        warnings.warn(*msg["args"], **msg["kwargs"])
+    else:
+        warnings.warn(msg)
+
+
 class Client:
     """Connect to and submit computation to a Dask cluster
 
@@ -576,6 +592,8 @@ class Client:
 
     _instances = weakref.WeakSet()
 
+    _default_event_handlers = {"print": _handle_print, "warn": _handle_warn}
+
     def __init__(
         self,
         address=None,
@@ -705,6 +723,7 @@ def __init__(
             self._set_config = dask.config.set(
                 scheduler="dask.distributed", shuffle="tasks"
             )
+        self._event_handlers = {}
 
         self._stream_handlers = {
             "key-in-memory": self._handle_key_in_memory,
@@ -714,6 +733,7 @@ def __init__(
             "task-erred": self._handle_task_erred,
             "restart": self._handle_restart,
             "error": self._handle_error,
+            "event": self._handle_event,
         }
 
         self._state_handlers = {
@@ -1015,6 +1035,9 @@ async def _start(self, timeout=no_default, **kwargs):
         for pc in self._periodic_callbacks.values():
             pc.start()
 
+        for topic, handler in Client._default_event_handlers.items():
+            self.subscribe_topic(topic, handler)
+
         self._handle_scheduler_coroutine = asyncio.ensure_future(self._handle_report())
         self.coroutines.append(self._handle_scheduler_coroutine)
 
@@ -3570,6 +3593,61 @@ def get_events(self, topic: str = None):
         """
         return self.sync(self.scheduler.events, topic=topic)
 
+    async def _handle_event(self, topic, event):
+        if topic not in self._event_handlers:
+            self.unsubscribe_topic(topic)
+            return
+        handler = self._event_handlers[topic]
+        ret = handler(event)
+        if inspect.isawaitable(ret):
+            await ret
+
+    def subscribe_topic(self, topic, handler):
+        """Subscribe to a topic and execute a handler for every received event
+
+        Parameters
+        ----------
+        topic: str
+            The topic name
+        handler: callable or coroutine function
+            A handler called for every received event. The handler must accept a
+            single argument `event` which is a tuple `(timestamp, msg)` where
+            timestamp refers to the clock on the scheduler.
+
+        Example
+        -------
+
+        >>> import logging
+        >>> logger = logging.getLogger("myLogger")  # Log config not shown
+        >>> client.subscribe_topic("topic-name", lambda: logger.info)
+
+        See Also
+        --------
+        dask.distributed.Client.unsubscribe_topic
+        dask.distributed.Client.get_events
+        dask.distributed.Client.log_event
+        """
+        if topic in self._event_handlers:
+            logger.info("Handler for %s already set. Overwriting.", topic)
+        self._event_handlers[topic] = handler
+        msg = {"op": "subscribe-topic", "topic": topic, "client": self.id}
+        self._send_to_scheduler(msg)
+
+    def unsubscribe_topic(self, topic):
+        """Unsubscribe from a topic and remove event handler
+
+        See Also
+        --------
+        dask.distributed.Client.subscribe_topic
+        dask.distributed.Client.get_events
+        dask.distributed.Client.log_event
+        """
+        if topic in self._event_handlers:
+            msg = {"op": "unsubscribe-topic", "topic": topic, "client": self.id}
+            self._send_to_scheduler(msg)
+        else:
+            raise ValueError(f"No event handler known for topic {topic}.")
+
     def retire_workers(self, workers=None, close_workers=True, **kwargs):
         """Retire certain workers on the scheduler
 
diff --git a/distributed/http/scheduler/tests/test_scheduler_http.py b/distributed/http/scheduler/tests/test_scheduler_http.py
index 0582b65dccd..0a8741c47dc 100644
--- a/distributed/http/scheduler/tests/test_scheduler_http.py
+++ b/distributed/http/scheduler/tests/test_scheduler_http.py
@@ -1,3 +1,4 @@
+import asyncio
 import json
 import re
 
@@ -102,6 +103,7 @@ async def test_prometheus(c, s, a, b):
         assert client.samples[0].value == 1.0
 
 
+@pytest.mark.repeat(100)
 @gen_cluster(client=True, clean_kwargs={"threads": False})
 async def test_prometheus_collect_task_states(c, s, a, b):
     pytest.importorskip("prometheus_client")
@@ -139,6 +141,8 @@ async def fetch_metrics():
 
     # submit a task which should show up in the prometheus scraping
     future = c.submit(slowinc, 1, delay=0.5)
+    while not any(future.key in w.tasks for w in [a, b]):
+        await asyncio.sleep(0.001)
 
     active_metrics, forgotten_tasks = await fetch_metrics()
     assert active_metrics.keys() == expected
@@ -148,7 +152,11 @@ async def fetch_metrics():
     res = await c.gather(future)
     assert res == 2
 
-    del future
+    future.release()
+
+    while any(future.key in w.tasks for w in [a, b]):
+        await asyncio.sleep(0.001)
+
     active_metrics, forgotten_tasks = await fetch_metrics()
     assert active_metrics.keys() == expected
     assert sum(active_metrics.values()) == 0.0
diff --git a/distributed/nanny.py b/distributed/nanny.py
index a981b8d479b..fef2d6b41ef 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -607,6 +607,15 @@ async def close(self, comm=None, timeout=5, report=None):
             await comm.write("OK")
         await super().close()
 
+    async def _log_event(self, topic, msg):
+        await self.scheduler.log_event(
+            topic=topic,
+            msg=msg,
+        )
+
+    def log_event(self, topic, msg):
+        self.loop.add_callback(self._log_event, topic, msg)
+
 
 class WorkerProcess:
     # The interval how often to check the msg queue for init
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index b9b5380ded3..706e85c3b24 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -3710,6 +3710,7 @@ def __init__(
             )
         )
         self.event_counts = defaultdict(int)
+        self.event_subscriber = defaultdict(set)
         self.worker_plugins = dict()
         self.nanny_plugins = dict()
 
@@ -3736,6 +3737,8 @@ def __init__(
             "heartbeat-client": self.client_heartbeat,
             "close-client": self.remove_client,
             "restart": self.restart,
+            "subscribe-topic": self.subscribe_topic,
+            "unsubscribe-topic": self.unsubscribe_topic,
         }
 
         self.handlers = {
@@ -7501,13 +7504,32 @@ async def get_worker_logs(self, comm=None, n=None, workers=None, nanny=False):
 
     def log_event(self, name, msg):
         event = (time(), msg)
-        if isinstance(name, list):
+        if isinstance(name, (list, tuple)):
             for n in name:
                 self.events[n].append(event)
                 self.event_counts[n] += 1
+                self._report_event(n, event)
         else:
             self.events[name].append(event)
             self.event_counts[name] += 1
+            self._report_event(name, event)
+
+    def _report_event(self, name, event):
+        for client in self.event_subscriber[name]:
+            self.report(
+                {
+                    "op": "event",
+                    "topic": name,
+                    "event": event,
+                },
+                client=client,
+            )
+
+    def subscribe_topic(self, topic, client):
+        self.event_subscriber[topic].add(client)
+
+    def unsubscribe_topic(self, topic, client):
+        self.event_subscriber[topic].discard(client)
 
     def get_events(self, comm=None, topic=None):
         if topic is not None:
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 706e804c74e..4386bdb01ac 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -6973,3 +6973,148 @@ async def f(x):
     future = c.submit(f, 10)
     result = await future
     assert result == 11
+
+
+@gen_cluster(client=True, nthreads=[("", 1)])
+async def test_events_subscribe_topic(c, s, a):
+
+    log = []
+
+    def user_event_handler(event):
+        log.append(event)
+
+    c.subscribe_topic("test-topic", user_event_handler)
+
+    while not s.event_subscriber["test-topic"]:
+        await asyncio.sleep(0.01)
+
+    a.log_event("test-topic", {"important": "event"})
+
+    while len(log) != 1:
+        await asyncio.sleep(0.01)
+
+    time_, msg = log[0]
+    assert isinstance(time_, float)
+    assert msg == {"important": "event"}
+
+    c.unsubscribe_topic("test-topic")
+
+    while s.event_subscriber["test-topic"]:
+        await asyncio.sleep(0.01)
+
+    a.log_event("test-topic", {"forget": "me"})
+
+    while len(s.events["test-topic"]) == 1:
+        await asyncio.sleep(0.01)
+
+    assert len(log) == 1
+
+    async def async_user_event_handler(event):
+        log.append(event)
+        await asyncio.sleep(0)
+
+    c.subscribe_topic("test-topic", async_user_event_handler)
+
+    while not s.event_subscriber["test-topic"]:
+        await asyncio.sleep(0.01)
+
+    a.log_event("test-topic", {"async": "event"})
+
+    while len(log) == 1:
+        await asyncio.sleep(0.01)
+
+    assert len(log) == 2
+    time_, msg = log[1]
+    assert isinstance(time_, float)
+    assert msg == {"async": "event"}
+
+    # Even though the middle event was not subscribed to, the scheduler still
+    # knows about all and we can retrieve them
+    all_events = await c.get_events(topic="test-topic")
+    assert len(all_events) == 3
+
+
+@gen_cluster(client=True, nthreads=[("", 1)])
+async def test_events_all_servers_use_same_channel(c, s, a):
+    """Ensure that logs from all server types (scheduler, worker, nanny)
+    and the clients themselves arrive"""
+
+    log = []
+
+    def user_event_handler(event):
+        log.append(event)
+
+    c.subscribe_topic("test-topic", user_event_handler)
+
+    while not s.event_subscriber["test-topic"]:
+        await asyncio.sleep(0.01)
+
+    async with Nanny(s.address) as n:
+        a.log_event("test-topic", "worker")
+        n.log_event("test-topic", "nanny")
+        s.log_event("test-topic", "scheduler")
+        await c.log_event("test-topic", "client")
+
+    while not len(log) == 4 == len(set(log)):
+        await asyncio.sleep(0.1)
+
+
+@gen_cluster(client=True, nthreads=[])
+async def test_events_unsubscribe_raises_if_unknown(c, s):
+    with pytest.raises(ValueError, match="No event handler known for topic unknown"):
+        c.unsubscribe_topic("unknown")
+
+
+@gen_cluster(client=True)
+async def test_log_event_warn(c, s, a, b):
+    def foo():
+        get_worker().log_event(["foo", "warn"], "Hello!")
+
+    with pytest.warns(Warning, match="Hello!"):
+        await c.submit(foo)
+
+
+@gen_cluster(client=True)
+async def test_log_event_warn_dask_warns(c, s, a, b):
+    from dask.distributed import warn
+
+    def foo():
+        warn("Hello!")
+
+    with pytest.warns(Warning, match="Hello!"):
+        await c.submit(foo)
+
+
+@gen_cluster(client=True, Worker=Nanny)
+async def test_print(c, s, a, b, capsys):
+    from dask.distributed import print
+
+    def foo():
+        print("Hello!", 123, sep=":")
+
+    await c.submit(foo)
+
+    out, err = capsys.readouterr()
+    assert "Hello!:123" in out
+
+
+@gen_cluster(client=True, Worker=Nanny)
+async def test_print_non_msgpack_serializable(c, s, a, b, capsys):
+    from dask.distributed import print
+
+    def foo():
+        print(object())
+
+    await c.submit(foo)
+
+    out, err = capsys.readouterr()
+    assert "<object object at" in out
+
+
+def test_print_simple(capsys):
+    from dask.distributed import print
+
+    print("Hello!", 123, sep=":")
+
+    out, err = capsys.readouterr()
+    assert "Hello!:123" in out
diff --git a/distributed/worker.py b/distributed/worker.py
index abd0fb4b7f6..75dd3ff38cd 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -1,5 +1,6 @@
 import asyncio
 import bisect
+import builtins
 import concurrent.futures
 import errno
 import heapq
@@ -30,6 +31,7 @@
     funcname,
     parse_bytes,
     parse_timedelta,
+    stringify,
     typename,
 )
 
@@ -4129,3 +4131,37 @@ def gpu_startup(worker):
         return nvml.one_time()
 
     DEFAULT_STARTUP_INFORMATION["gpu"] = gpu_startup
+
+
+def print(*args, **kwargs):
+    """Dask print function
+    This prints both wherever this function is run, and also in the user's
+    client session
+    """
+    try:
+        worker = get_worker()
+    except ValueError:
+        pass
+    else:
+        msg = {
+            "args": tuple(stringify(arg) for arg in args),
+            "kwargs": {k: stringify(v) for k, v in kwargs.items()},
+        }
+        worker.log_event("print", msg)
+
+    builtins.print(*args, **kwargs)
+
+
+def warn(*args, **kwargs):
+    """Dask warn function
+    This raises a warning both wherever this function is run, and also
+    in the user's client session
+    """
+    try:
+        worker = get_worker()
+    except ValueError:
+        pass
+    else:
+        worker.log_event("warn", {"args": args, "kwargs": kwargs})
+
+    warnings.warn(*args, **kwargs)

From 577098f36fd133bf15f890620dc882f3ae406eed Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Mon, 13 Sep 2021 15:27:00 +0100
Subject: [PATCH 1487/1550] Remove redundant timeouts from test_client (#5314)

---
 distributed/tests/test_client.py | 81 ++++----------------------------
 1 file changed, 9 insertions(+), 72 deletions(-)

diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 4386bdb01ac..c1a4a6aecb2 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -667,7 +667,7 @@ def test_no_future_references(c):
     start = time()
     while list(ws):
         sleep(0.01)
-        assert time() < start + 2
+        assert time() < start + 30
 
 
 def test_get_sync_optimize_graph_passes_through(c):
@@ -789,13 +789,8 @@ async def test_garbage_collection_with_scatter(c, s, a, b):
     await asyncio.sleep(0)
     assert c.refcount[key] == 0
 
-    start = time()
-    while True:
-        if key not in s.tasks or not s.tasks[key].who_has:
-            break
-        else:
-            assert time() < start + 3
-            await asyncio.sleep(0.1)
+    while key in s.tasks and s.tasks[key].who_has:
+        await asyncio.sleep(0.1)
 
 
 @gen_cluster(client=True)
@@ -1006,9 +1001,7 @@ async def test_remove_worker(c, s, a, b):
 async def test_errors_dont_block(c, s, w):
     L = [c.submit(inc, 1), c.submit(throws, 1), c.submit(inc, 2), c.submit(throws, 2)]
 
-    start = time()
     while not (L[0].status == L[2].status == "finished"):
-        assert time() < start + 5
         await asyncio.sleep(0.01)
 
     result = await c.gather([L[0], L[2]])
@@ -1221,10 +1214,8 @@ async def test_get_releases_data(c, s, a, b):
 
     gc.collect()
 
-    start = time()
     while c.refcount["x"]:
         await asyncio.sleep(0.01)
-        assert time() < start + 2
 
 
 def test_current(s, a, b):
@@ -1391,10 +1382,8 @@ async def test_scatter_direct_2(s, a, b):
 
     last = s.clients[c.id].last_seen
 
-    start = time()
     while s.clients[c.id].last_seen == last:
         await asyncio.sleep(0.10)
-        assert time() < start + 5
 
     await c.close()
 
@@ -1966,17 +1955,17 @@ async def test_badly_serialized_input(c, s, a, b):
 
 
 @pytest.mark.skip
+@gen_test()
 async def test_badly_serialized_input_stderr(capsys, c):
     o = BadlySerializedObject()
     future = c.submit(inc, o)
 
-    start = time()
     while True:
         sleep(0.01)
         out, err = capsys.readouterr()
         if "hello!" in err:
             break
-        assert time() - start < 20
+
     assert future.status == "error"
 
 
@@ -2068,10 +2057,8 @@ async def test_forget_complex(e, s, A, B):
     assert set(s.tasks) == {f.key for f in [ac, cd, a, c, d]}
     assert b.key not in s.tasks
 
-    start = time()
     while b.key in A.data or b.key in B.data:
         await asyncio.sleep(0.01)
-        assert time() < start + 10
 
     s.client_releases_keys(keys=[ac.key], client=e.id)
     assert set(s.tasks) == {f.key for f in [cd, a, c, d]}
@@ -2179,10 +2166,8 @@ async def test_multi_client(s, a, b):
 
     await c.close()
 
-    start = time()
     while c.id in s.wants_what:
         await asyncio.sleep(0.01)
-        assert time() < start + 5
 
     assert c.id not in s.wants_what
     assert c.id not in s.who_wants[y.key]
@@ -2190,10 +2175,8 @@ async def test_multi_client(s, a, b):
 
     await f.close()
 
-    start = time()
     while s.tasks:
         await asyncio.sleep(0.01)
-        assert time() < start + 2, s.tasks
 
 
 def long_running_client_connection(address):
@@ -2210,17 +2193,13 @@ async def test_cleanup_after_broken_client_connection(s, a, b):
     proc.daemon = True
     proc.start()
 
-    start = time()
     while not s.tasks:
         await asyncio.sleep(0.01)
-        assert time() < start + 5
 
     proc.terminate()
 
-    start = time()
     while s.tasks:
         await asyncio.sleep(0.01)
-        assert time() < start + 5
 
 
 @gen_cluster()
@@ -2238,19 +2217,15 @@ async def test_multi_garbage_collection(s, a, b):
     await wait([x, y])
 
     x.__del__()
-    start = time()
     while x.key in a.data or x.key in b.data:
         await asyncio.sleep(0.01)
-        assert time() < start + 5
 
     assert s.wants_what == {c.id: {y.key}, f.id: {y.key}, "fire-and-forget": set()}
     assert s.who_wants == {y.key: {c.id, f.id}}
 
     y.__del__()
-    start = time()
     while x.key in s.wants_what[f.id]:
         await asyncio.sleep(0.01)
-        assert time() < start + 5
 
     await asyncio.sleep(0.1)
     assert y.key in a.data or y.key in b.data
@@ -2258,10 +2233,8 @@ async def test_multi_garbage_collection(s, a, b):
     assert s.who_wants == {y.key: {c.id}}
 
     y2.__del__()
-    start = time()
     while y.key in a.data or y.key in b.data:
         await asyncio.sleep(0.01)
-        assert time() < start + 5
 
     assert not any(v for v in s.wants_what.values())
     assert not s.who_wants
@@ -2315,7 +2288,7 @@ async def test_proxy(c, s, a, b):
 
 
 @gen_cluster(client=True)
-async def test__cancel(c, s, a, b):
+async def test_cancel(c, s, a, b):
     x = c.submit(slowinc, 1)
     y = c.submit(slowinc, x)
 
@@ -2328,10 +2301,8 @@ async def test__cancel(c, s, a, b):
     assert "cancel" in str(x)
     s.validate_state()
 
-    start = time()
     while not y.cancelled():
         await asyncio.sleep(0.01)
-        assert time() < start + 5
 
     assert not s.tasks
     s.validate_state()
@@ -2361,10 +2332,8 @@ async def test_cancel_multi_client(s, a, b):
     assert x.cancelled()
     assert not y.cancelled()
 
-    start = time()
     while y.key not in s.tasks:
         await asyncio.sleep(0.01)
-        assert time() < start + 5
 
     out = await y
     assert out == 2
@@ -2384,13 +2353,11 @@ async def test_cancel_collection(c, s, a, b):
     await c.cancel(x)
     await c.cancel([x])
     assert all(f.cancelled() for f in L)
-    start = time()
     while s.tasks:
-        assert time() < start + 1
         await asyncio.sleep(0.01)
 
 
-def test_cancel(c):
+def test_cancel_sync(c):
     x = c.submit(slowinc, 1, key="x")
     y = c.submit(slowinc, x, key="y")
     z = c.submit(slowinc, y, key="z")
@@ -2400,7 +2367,7 @@ def test_cancel(c):
     start = time()
     while not z.cancelled():
         sleep(0.01)
-        assert time() < start + 5
+        assert time() < start + 30
 
     assert x.result() == 2
 
@@ -3007,9 +2974,7 @@ async def test_receive_lost_key(c, s, a, b):
     await x
     await a.close()
 
-    start = time()
     while x.status == "finished":
-        assert time() < start + 5
         await asyncio.sleep(0.01)
 
 
@@ -3020,9 +2985,7 @@ async def test_unrunnable_task_runs(c, s, a, b):
     await x
 
     await a.close()
-    start = time()
     while x.status == "finished":
-        assert time() < start + 5
         await asyncio.sleep(0.01)
 
     assert s.tasks[x.key] in s.unrunnable
@@ -3030,9 +2993,7 @@ async def test_unrunnable_task_runs(c, s, a, b):
 
     w = await Worker(s.address, loop=s.loop)
 
-    start = time()
     while x.status != "finished":
-        assert time() < start + 2
         await asyncio.sleep(0.01)
 
     assert s.tasks[x.key] not in s.unrunnable
@@ -3314,9 +3275,7 @@ async def test_cancel_clears_processing(c, s, *workers):
 
     await c.cancel(x)
 
-    start = time()
     while any(v for w in s.workers.values() for v in w.processing):
-        assert time() < start + 0.2
         await asyncio.sleep(0.01)
     s.validate_state()
 
@@ -3742,11 +3701,9 @@ def test_reconnect(loop):
 async def test_reconnect_timeout(c, s):
     with captured_logger(logging.getLogger("distributed.client")) as logger:
         await s.close()
-        start = time()
         while c.status != "closed":
             await c._update_scheduler_info()
             await asyncio.sleep(0.05)
-            assert time() < start + 5, "Timeout waiting for reconnect to fail"
     text = logger.getvalue()
     assert "Failed to reconnect" in text
 
@@ -3982,10 +3939,8 @@ async def test_scatter_compute_store_lose(c, s, a, b):
 
     await a.close()
 
-    start = time()
     while x.status == "finished":
         await asyncio.sleep(0.01)
-        assert time() < start + 2
 
     # assert xx.status == 'finished'
     assert y.status == "finished"
@@ -3997,18 +3952,14 @@ async def test_scatter_compute_store_lose(c, s, a, b):
     zkey = z.key
     del z
 
-    start = time()
     while s.get_task_status(keys=[zkey]) != {zkey: "released"}:
         await asyncio.sleep(0.01)
-        assert time() < start + 2
 
     xxkey = xx.key
     del xx
 
-    start = time()
     while x.key in s.tasks and zkey not in s.tasks and xxkey not in s.tasks:
         await asyncio.sleep(0.01)
-        assert time() < start + 2
 
 
 @gen_cluster(client=True)
@@ -4027,10 +3978,8 @@ async def test_scatter_compute_store_lose_processing(c, s, a, b):
     await asyncio.sleep(0.1)
     await a.close()
 
-    start = time()
     while x.status == "finished":
         await asyncio.sleep(0.01)
-        assert time() < start + 2
 
     assert y.status == "cancelled"
     assert z.status == "cancelled"
@@ -4408,8 +4357,7 @@ def g(future):
 
     x.add_done_callback(f)
 
-    t = time()
-    while len(S) < 4 and time() - t < 2.0:
+    while len(S) < 4:
         await asyncio.sleep(0.01)
 
     assert S == {(f.key, f.status) for f in (u, v, w, x)}
@@ -4634,7 +4582,6 @@ def f(i, L=None):
 
 @gen_cluster(client=True)
 async def test_dont_clear_waiting_data(c, s, a, b):
-    start = time()
     x = await c.scatter(1)
     y = c.submit(slowinc, x, delay=0.5)
     while y.key not in s.tasks:
@@ -4862,10 +4809,8 @@ async def test_retire_workers(c, s, a, b):
     await c.retire_workers(workers=[a.address], close_workers=True)
     assert set(s.workers) == {b.address}
 
-    start = time()
     while a.status != Status.closed:
         await asyncio.sleep(0.01)
-        assert time() < start + 5
 
 
 class MyException(Exception):
@@ -4942,18 +4887,14 @@ def f(x):
     try:
         fire_and_forget(c.submit(f, future))
 
-        start = time()
         while not hasattr(distributed, "foo"):
             await asyncio.sleep(0.01)
-            assert time() < start + 2
         assert distributed.foo == 123
     finally:
         del distributed.foo
 
-    start = time()
     while len(s.tasks) > 1:
         await asyncio.sleep(0.01)
-        assert time() < start + 2
 
     assert set(s.who_wants) == {future.key}
     assert set(s.tasks) == {future.key}
@@ -5015,10 +4956,8 @@ async def test_close(s, a, b):
     assert c.id in s.wants_what
     await c.close()
 
-    start = time()
     while c.id in s.wants_what or s.tasks:
         await asyncio.sleep(0.01)
-        assert time() < start + 5
 
 
 def test_threadsafe(c):
@@ -5456,10 +5395,8 @@ async def test_future_auto_inform(c, s, a, b):
     client = await Client(s.address, asynchronous=True)
     future = Future(x.key, client)
 
-    start = time()
     while future.status != "finished":
         await asyncio.sleep(0.01)
-        assert time() < start + 1
 
     await client.close()
 

From 1b305360c366dfd6494ee377f3ad745746551d36 Mon Sep 17 00:00:00 2001
From: Peter Andreas Entschev <peter@entschev.com>
Date: Mon, 13 Sep 2021 20:34:58 +0200
Subject: [PATCH 1488/1550] Warn if CUDA context is created on incorrect device
 in UCX (#5308)

* Add NVML function to check whether a CUDA context exists

* Warn if CUDA context is created on incorrect device in UCX
---
 distributed/comm/ucx.py         | 37 ++++++++++++++++++++++++++++++++-
 distributed/diagnostics/nvml.py | 18 ++++++++++++++++
 2 files changed, 54 insertions(+), 1 deletion(-)

diff --git a/distributed/comm/ucx.py b/distributed/comm/ucx.py
index 33285711b7b..f8b34c2f43c 100644
--- a/distributed/comm/ucx.py
+++ b/distributed/comm/ucx.py
@@ -6,12 +6,15 @@
 .. _UCX: https://github.com/openucx/ucx
 """
 import logging
+import os
 import struct
+import warnings
 import weakref
 
 import dask
 from dask.utils import parse_bytes
 
+from ..diagnostics.nvml import has_cuda_context
 from ..utils import ensure_ip, get_ip, get_ipv6, log_errors, nbytes
 from .addressing import parse_host_port, unparse_host_port
 from .core import Comm, CommClosedError, Connector, Listener
@@ -30,6 +33,8 @@
 device_array = None
 ucx_create_endpoint = None
 ucx_create_listener = None
+pre_existing_cuda_context = False
+cuda_context_created = False
 
 
 def synchronize_stream(stream=0):
@@ -42,7 +47,10 @@ def synchronize_stream(stream=0):
 
 
 def init_once():
-    global ucp, host_array, device_array, ucx_create_endpoint, ucx_create_listener
+    global ucp, host_array, device_array
+    global ucx_create_endpoint, ucx_create_listener
+    global pre_existing_cuda_context, cuda_context_created
+
     if ucp is not None:
         return
 
@@ -60,8 +68,35 @@ def init_once():
                 "CUDA support with UCX requires Numba for context management"
             )
 
+        cuda_visible_device = int(
+            os.environ.get("CUDA_VISIBLE_DEVICES", "0").split(",")[0]
+        )
+        pre_existing_cuda_context = has_cuda_context()
+        if pre_existing_cuda_context is not False:
+            warnings.warn(
+                f"A CUDA context for device {pre_existing_cuda_context} already exists on process "
+                f"ID {os.getpid()}. This is often the result of a CUDA-enabled library calling a "
+                "CUDA runtime function before Dask-CUDA can spawn worker processes. Please make "
+                "sure any such function calls don't happen at import time or in the global scope "
+                "of a program."
+            )
+
         numba.cuda.current_context()
 
+        cuda_context_created = has_cuda_context()
+        if (
+            cuda_context_created is not False
+            and cuda_context_created != cuda_visible_device
+        ):
+            warnings.warn(
+                f"Worker with process ID {os.getpid()} should have a CUDA context assigned to "
+                f"device {cuda_visible_device}, but instead the CUDA context is on device "
+                "{cuda_context_created}. This is often the result of a CUDA-enabled library "
+                "calling a CUDA runtime function before Dask-CUDA can spawn worker processes. "
+                "Please make sure any such function calls don't happen at import time or in "
+                "the global scope of a program."
+            )
+
     import ucp as _ucp
 
     ucp = _ucp
diff --git a/distributed/diagnostics/nvml.py b/distributed/diagnostics/nvml.py
index 3aba966d992..1b45906a7a5 100644
--- a/distributed/diagnostics/nvml.py
+++ b/distributed/diagnostics/nvml.py
@@ -63,6 +63,24 @@ def _pynvml_handles():
     return pynvml.nvmlDeviceGetHandleByIndex(gpu_idx)
 
 
+def has_cuda_context():
+    """Check whether the current process already has a CUDA context created.
+
+    Returns
+    -------
+    ``False`` if current process has no CUDA context created, otherwise returns the
+    index of the device for which there's a CUDA context.
+    """
+    init_once()
+    for index in range(device_get_count()):
+        handle = pynvml.nvmlDeviceGetHandleByIndex(index)
+        running_processes = pynvml.nvmlDeviceGetComputeRunningProcesses_v2(handle)
+        for proc in running_processes:
+            if os.getpid() == proc.pid:
+                return index
+    return False
+
+
 def real_time():
     h = _pynvml_handles()
     return {

From 06835b10a6289c3aef67f18d19e89bf3d8f6b5bc Mon Sep 17 00:00:00 2001
From: Peter Andreas Entschev <peter@entschev.com>
Date: Mon, 13 Sep 2021 20:35:50 +0200
Subject: [PATCH 1489/1550] Mark test_ucx_config_w_env_var flaky on UCX < 1.11
 (#5262)

Test has also been slightly cleaned up.
---
 distributed/comm/tests/test_ucx_config.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/distributed/comm/tests/test_ucx_config.py b/distributed/comm/tests/test_ucx_config.py
index 1efd51c0bae..8fa38290f0a 100644
--- a/distributed/comm/tests/test_ucx_config.py
+++ b/distributed/comm/tests/test_ucx_config.py
@@ -80,6 +80,9 @@ async def test_ucx_config(cleanup):
         assert ucx_config.get("SOCKADDR_TLS_PRIORITY") == "rdmacm"
 
 
+@pytest.mark.flaky(
+    reruns=10, reruns_delay=5, condition=ucp.get_ucx_version() < (1, 11, 0)
+)
 def test_ucx_config_w_env_var(cleanup, loop, monkeypatch):
     size = "1000.00 MB"
     monkeypatch.setenv("DASK_RMM__POOL_SIZE", size)
@@ -101,7 +104,7 @@ def test_ucx_config_w_env_var(cleanup, loop, monkeypatch):
                 "ucx",
                 "--no-nanny",
             ]
-        ) as w:
+        ):
             with Client(sched_addr, loop=loop, timeout=10) as c:
                 while not c.scheduler_info()["workers"]:
                     sleep(0.1)
@@ -112,7 +115,6 @@ def test_ucx_config_w_env_var(cleanup, loop, monkeypatch):
                 )
                 assert rmm_resource == rmm.mr.PoolMemoryResource
 
-                worker_addr = list(c.scheduler_info()["workers"])[0]
-                worker_rmm_usage = c.run(rmm.mr.get_current_device_resource_type)
-                rmm_resource = worker_rmm_usage[worker_addr]
-                assert rmm_resource == rmm.mr.PoolMemoryResource
+                rmm_resource_workers = c.run(rmm.mr.get_current_device_resource_type)
+                for v in rmm_resource_workers.values():
+                    assert v == rmm.mr.PoolMemoryResource

From 3f86e58f729c315905d51acedd9229a1db240cf4 Mon Sep 17 00:00:00 2001
From: Tom Forbes <tom@tomforb.es>
Date: Tue, 14 Sep 2021 21:24:16 +0100
Subject: [PATCH 1490/1550] Add type annotations to various functions within
 `distributed.worker` (#5290)

---
 .gitignore            |  1 +
 distributed/worker.py | 36 +++++++++++++++++++++++-------------
 2 files changed, 24 insertions(+), 13 deletions(-)

diff --git a/.gitignore b/.gitignore
index ea068b3a900..e47580fc5bb 100644
--- a/.gitignore
+++ b/.gitignore
@@ -23,3 +23,4 @@ dask-worker-space/
 .ycm_extra_conf.py
 tags
 .ipynb_checkpoints
+.venv/
\ No newline at end of file
diff --git a/distributed/worker.py b/distributed/worker.py
index 75dd3ff38cd..7066771a41a 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import asyncio
 import bisect
 import builtins
@@ -17,7 +19,10 @@
 from datetime import timedelta
 from inspect import isawaitable
 from pickle import PicklingError
-from typing import Dict, Hashable, Iterable, Optional
+from typing import TYPE_CHECKING, Dict, Hashable, Iterable, Optional
+
+if TYPE_CHECKING:
+    from .client import Client
 
 from tlz import first, keymap, merge, pluck  # noqa: F401
 from tornado.ioloop import IOLoop, PeriodicCallback
@@ -2818,8 +2823,14 @@ async def plugin_remove(self, comm=None, name=None):
             return {"status": "OK"}
 
     async def actor_execute(
-        self, comm=None, actor=None, function=None, args=(), kwargs={}
+        self,
+        comm=None,
+        actor=None,
+        function=None,
+        args=(),
+        kwargs: Optional[dict] = None,
     ):
+        kwargs = kwargs or {}
         separate_thread = kwargs.pop("separate_thread", True)
         key = actor
         actor = self.actors[key]
@@ -2854,7 +2865,7 @@ def actor_attribute(self, comm=None, actor=None, attribute=None):
         except Exception as ex:
             return {"status": "error", "exception": to_serialize(ex)}
 
-    def meets_resource_constraints(self, key):
+    def meets_resource_constraints(self, key: str) -> bool:
         ts = self.tasks[key]
         if not ts.resource_restrictions:
             return True
@@ -3264,8 +3275,7 @@ async def get_profile(
         return prof
 
     async def get_profile_metadata(self, comm=None, start=0, stop=None):
-        if stop is None:
-            add_recent = True
+        add_recent = stop is None
         now = time() + self.scheduler_delay
         stop = stop or now
         start = start or 0
@@ -3447,14 +3457,14 @@ def validate_state(self):
     #######################################
 
     @property
-    def client(self):
+    def client(self) -> Client:
         with self._lock:
             if self._client:
                 return self._client
             else:
                 return self._get_client()
 
-    def _get_client(self, timeout=None):
+    def _get_client(self, timeout=None) -> Client:
         """Get local client attached to this worker
 
         If no such client exists, create one
@@ -3536,7 +3546,7 @@ def get_current_task(self):
         return self.active_threads[threading.get_ident()]
 
 
-def get_worker():
+def get_worker() -> Worker:
     """Get the worker currently running this task
 
     Examples
@@ -3563,7 +3573,7 @@ def get_worker():
             raise ValueError("No workers found")
 
 
-def get_client(address=None, timeout=None, resolve_address=True):
+def get_client(address=None, timeout=None, resolve_address=True) -> Client:
     """Get a client while within a task.
 
     This client connects to the same scheduler to which the worker is connected
@@ -3678,7 +3688,7 @@ class Reschedule(Exception):
     """
 
 
-def parse_memory_limit(memory_limit, nthreads, total_cores=CPU_COUNT):
+def parse_memory_limit(memory_limit, nthreads, total_cores=CPU_COUNT) -> Optional[int]:
     if memory_limit is None:
         return None
 
@@ -3807,7 +3817,7 @@ def execute_task(task):
 _cache_lock = threading.Lock()
 
 
-def dumps_function(func):
+def dumps_function(func) -> bytes:
     """Dump a function to bytes, cache functions"""
     try:
         with _cache_lock:
@@ -4028,7 +4038,7 @@ def __repr__(self):
     return msg
 
 
-def convert_args_to_str(args, max_len=None):
+def convert_args_to_str(args, max_len: Optional[int] = None) -> str:
     """Convert args to a string, allowing for some arguments to raise
     exceptions during conversion and ignoring them.
     """
@@ -4047,7 +4057,7 @@ def convert_args_to_str(args, max_len=None):
         return "({})".format(", ".join(strs))
 
 
-def convert_kwargs_to_str(kwargs, max_len=None):
+def convert_kwargs_to_str(kwargs: dict, max_len: Optional[int] = None) -> str:
     """Convert kwargs to a string, allowing for some arguments to raise
     exceptions during conversion and ignoring them.
     """

From 54760d83e0f04bd3b23c84e185773ae106853b4f Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Thu, 16 Sep 2021 18:19:54 +0100
Subject: [PATCH 1491/1550] AMM ReduceReplicas to iterate only on replicated
 tasks (#5297)

---
 distributed/active_memory_manager.py |  31 ++--
 distributed/scheduler.py             | 211 +++++++++++----------------
 2 files changed, 105 insertions(+), 137 deletions(-)

diff --git a/distributed/active_memory_manager.py b/distributed/active_memory_manager.py
index 4ed2daf4113..77e67ca4319 100644
--- a/distributed/active_memory_manager.py
+++ b/distributed/active_memory_manager.py
@@ -189,9 +189,14 @@ def _find_recipient(
         pending_repl: set[WorkerState],
     ) -> Optional[WorkerState]:
         """Choose a worker to acquire a new replica of an in-memory task among a set of
-        candidates. If candidates is None, default to all workers in the cluster that do
-        not hold a replica yet. The worker with the lowest memory usage (downstream of
-        pending replications and drops) will be returned.
+        candidates. If candidates is None, default to all workers in the cluster.
+        Regardless, workers that either already hold a replica or are scheduled to
+        receive one at the end of this AMM iteration are not considered.
+
+        Returns
+        -------
+        The worker with the lowest memory usage (downstream of pending replications and
+        drops), or None if no eligible candidates are available.
         """
         if ts.state != "memory":
             return None
@@ -210,9 +215,15 @@ def _find_dropper(
         pending_drop: set[WorkerState],
     ) -> Optional[WorkerState]:
         """Choose a worker to drop its replica of an in-memory task among a set of
-        candidates. If candidates is None, default to all workers in the cluster that
-        hold a replica. The worker with the highest memory usage (downstream of pending
-        replications and drops) will be returned.
+        candidates. If candidates is None, default to all workers in the cluster.
+        Regardless, workers that either do not hold a replica or are already scheduled
+        to drop theirs at the end of this AMM iteration are not considered.
+        This method also ensures that a key will not lose its last replica.
+
+        Returns
+        -------
+        The worker with the highest memory usage (downstream of pending replications and
+        drops), or None if no eligible candidates are available.
         """
         if len(ts.who_has) - len(pending_drop) < 2:
             return None
@@ -283,13 +294,7 @@ class ReduceReplicas(ActiveMemoryManagerPolicy):
     """
 
     def run(self):
-        # TODO this is O(n) to the total number of in-memory tasks on the cluster; it
-        #      could be made faster by automatically attaching it to a TaskState when it
-        #      goes above one replica and detaching it when it drops below two.
-        for ts in self.manager.scheduler.tasks.values():
-            if len(ts.who_has) < 2:
-                continue
-
+        for ts in self.manager.scheduler.replicated_tasks:
             desired_replicas = 1  # TODO have a marker on TaskState
 
             # If a dependent task has not been assigned to a worker yet, err on the side
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 706e85c3b24..04496b26e05 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1859,6 +1859,7 @@ class SchedulerState:
     _task_groups: dict
     _task_prefixes: dict
     _task_metadata: dict
+    _replicated_tasks: set
     _total_nthreads: Py_ssize_t
     _total_occupancy: double
     _transitions_table: dict
@@ -1918,6 +1919,9 @@ def __init__(
             self._tasks = tasks
         else:
             self._tasks = dict()
+        self._replicated_tasks = {
+            ts for ts in self._tasks.values() if len(ts._who_has) > 1
+        }
         self._computations = deque(
             maxlen=dask.config.get("distributed.diagnostics.computations.max-history")
         )
@@ -2035,6 +2039,10 @@ def task_prefixes(self):
     def task_metadata(self):
         return self._task_metadata
 
+    @property
+    def replicated_tasks(self):
+        return self._replicated_tasks
+
     @property
     def total_nthreads(self):
         return self._total_nthreads
@@ -2820,18 +2828,14 @@ def transition_memory_released(self, key, safe: bint = False):
                     dts._waiting_on.add(ts)
 
             # XXX factor this out?
-            ts_nbytes: Py_ssize_t = ts.get_nbytes()
             worker_msg = {
                 "op": "free-keys",
                 "keys": [key],
                 "reason": f"Memory->Released {key}",
             }
             for ws in ts._who_has:
-                del ws._has_what[ts]
-                ws._nbytes -= ts_nbytes
                 worker_msgs[ws._address] = [worker_msg]
-
-            ts._who_has.clear()
+            self.remove_all_replicas(ts)
 
             ts.state = "released"
 
@@ -3425,6 +3429,40 @@ def worker_objective(self, ts: TaskState, ws: WorkerState) -> tuple:
         else:
             return (start_time, ws._nbytes)
 
+    @ccall
+    def add_replica(self, ts: TaskState, ws: WorkerState):
+        """Note that a worker holds a replica of a task with state='memory'"""
+        if self._validate:
+            assert ws not in ts._who_has
+            assert ts not in ws._has_what
+
+        ws._nbytes += ts.get_nbytes()
+        ws._has_what[ts] = None
+        ts._who_has.add(ws)
+        if len(ts._who_has) == 2:
+            self._replicated_tasks.add(ts)
+
+    @ccall
+    def remove_replica(self, ts: TaskState, ws: WorkerState):
+        """Note that a worker no longer holds a replica of a task"""
+        ws._nbytes -= ts.get_nbytes()
+        del ws._has_what[ts]
+        ts._who_has.remove(ws)
+        if len(ts._who_has) == 1:
+            self._replicated_tasks.remove(ts)
+
+    @ccall
+    def remove_all_replicas(self, ts: TaskState):
+        """Remove all replicas of a task from all workers"""
+        ws: WorkerState
+        nbytes: Py_ssize_t = ts.get_nbytes()
+        for ws in ts._who_has:
+            ws._nbytes -= nbytes
+            del ws._has_what[ts]
+        if len(ts._who_has) > 1:
+            self._replicated_tasks.remove(ts)
+        ts._who_has.clear()
+
 
 class Scheduler(SchedulerState, ServerNode):
     """Dynamic distributed task scheduler
@@ -4736,70 +4774,23 @@ def stimulus_task_erred(
         parent: SchedulerState = cast(SchedulerState, self)
         logger.debug("Stimulus task erred %s, %s", key, worker)
 
-        recommendations: dict = {}
-        client_msgs: dict = {}
-        worker_msgs: dict = {}
-
         ts: TaskState = parent._tasks.get(key)
-        if ts is None:
-            return recommendations, client_msgs, worker_msgs
-
-        if ts._state == "processing":
-            retries: Py_ssize_t = ts._retries
-            r: tuple
-            if retries > 0:
-                ts._retries = retries - 1
-                r = parent._transition(key, "waiting")
-            else:
-                r = parent._transition(
-                    key,
-                    "erred",
-                    cause=key,
-                    exception=exception,
-                    traceback=traceback,
-                    worker=worker,
-                    **kwargs,
-                )
-            recommendations, client_msgs, worker_msgs = r
-
-        return recommendations, client_msgs, worker_msgs
-
-    def stimulus_missing_data(
-        self, cause=None, key=None, worker=None, ensure=True, **kwargs
-    ):
-        """Mark that certain keys have gone missing.  Recover."""
-        parent: SchedulerState = cast(SchedulerState, self)
-        with log_errors():
-            logger.debug("Stimulus missing data %s, %s", key, worker)
+        if ts is None or ts._state != "processing":
+            return {}, {}, {}
 
-            recommendations: dict = {}
-            client_msgs: dict = {}
-            worker_msgs: dict = {}
-
-            ts: TaskState = parent._tasks.get(key)
-            if ts is None or ts._state == "memory":
-                return recommendations, client_msgs, worker_msgs
-            cts: TaskState = parent._tasks.get(cause)
-
-            if cts is not None and cts._state == "memory":  # couldn't find this
-                ws: WorkerState
-                cts_nbytes: Py_ssize_t = cts.get_nbytes()
-                for ws in cts._who_has:  # TODO: this behavior is extreme
-                    del ws._has_what[ts]
-                    ws._nbytes -= cts_nbytes
-                cts._who_has.clear()
-                recommendations[cause] = "released"
-
-            if key:
-                recommendations[key] = "released"
-
-            parent._transitions(recommendations, client_msgs, worker_msgs)
-            recommendations = {}
-
-            if parent._validate:
-                assert cause not in self.who_has
-
-            return recommendations, client_msgs, worker_msgs
+        if ts._retries > 0:
+            ts._retries -= 1
+            return parent._transition(key, "waiting")
+        else:
+            return parent._transition(
+                key,
+                "erred",
+                cause=key,
+                exception=exception,
+                traceback=traceback,
+                worker=worker,
+                **kwargs,
+            )
 
     def stimulus_retry(self, comm=None, keys=None, client=None):
         parent: SchedulerState = cast(SchedulerState, self)
@@ -4914,14 +4905,13 @@ async def remove_worker(self, comm=None, address=None, safe=False, close=True):
                             self.allowed_failures,
                         )
 
-            for ts in ws._has_what:
-                ts._who_has.remove(ws)
+            for ts in list(ws._has_what):
+                parent.remove_replica(ts, ws)
                 if not ts._who_has:
                     if ts._run_spec:
                         recommendations[ts._key] = "released"
                     else:  # pure data
                         recommendations[ts._key] = "forgotten"
-            ws._has_what.clear()
 
             self.transitions(recommendations)
 
@@ -5071,6 +5061,7 @@ def validate_memory(self, key):
         ts: TaskState = parent._tasks[key]
         dts: TaskState
         assert ts._who_has
+        assert bool(ts in parent._replicated_tasks) == (len(ts._who_has) > 1)
         assert not ts._processing_on
         assert not ts._waiting_on
         assert ts not in parent._unrunnable
@@ -5141,8 +5132,13 @@ def validate_state(self, allow_overlap=False):
         for k, ts in parent._tasks.items():
             assert isinstance(ts, TaskState), (type(ts), ts)
             assert ts._key == k
+            assert bool(ts in parent._replicated_tasks) == (len(ts._who_has) > 1)
             self.validate_key(k, ts)
 
+        for ts in parent._replicated_tasks:
+            assert ts._state == "memory"
+            assert ts._key in parent._tasks
+
         c: str
         cs: ClientState
         for c, cs in parent._clients.items():
@@ -5343,24 +5339,14 @@ def handle_task_erred(self, key=None, **msg):
 
         self.send_all(client_msgs, worker_msgs)
 
-    def handle_release_data(self, key=None, worker=None, client=None, **msg):
+    def handle_release_data(self, key=None, worker=None, **kwargs):
         parent: SchedulerState = cast(SchedulerState, self)
         ts: TaskState = parent._tasks.get(key)
-        if ts is None:
+        if ts is None or ts._state == "memory":
             return
         ws: WorkerState = parent._workers_dv.get(worker)
-        if ws is None or ts._processing_on != ws:
-            return
-
-        recommendations: dict
-        client_msgs: dict
-        worker_msgs: dict
-
-        r: tuple = self.stimulus_missing_data(key=key, ensure=False, **msg)
-        recommendations, client_msgs, worker_msgs = r
-        parent._transitions(recommendations, client_msgs, worker_msgs)
-
-        self.send_all(client_msgs, worker_msgs)
+        if ws is not None and ts._processing_on == ws:
+            parent._transitions({key: "released"}, {}, {})
 
     def handle_missing_data(self, key=None, errant_worker=None, **kwargs):
         parent: SchedulerState = cast(SchedulerState, self)
@@ -5372,9 +5358,7 @@ def handle_missing_data(self, key=None, errant_worker=None, **kwargs):
             return
         ws: WorkerState = parent._workers_dv.get(errant_worker)
         if ws is not None and ws in ts._who_has:
-            ts._who_has.remove(ws)
-            del ws._has_what[ts]
-            ws._nbytes -= ts.get_nbytes()
+            parent.remove_replica(ts, ws)
         if not ts._who_has:
             if ts._run_spec:
                 self.transitions({key: "released"})
@@ -5392,11 +5376,8 @@ def release_worker_data(self, comm=None, keys=None, worker=None):
         ts: TaskState
         recommendations: dict = {}
         for ts in removed_tasks:
-            del ws._has_what[ts]
-            ws._nbytes -= ts.get_nbytes()
-            wh: set = ts._who_has
-            wh.remove(ws)
-            if not wh:
+            parent.remove_replica(ts, ws)
+            if not ts._who_has:
                 recommendations[ts._key] = "released"
         if recommendations:
             self.transitions(recommendations)
@@ -5716,14 +5697,11 @@ async def gather(self, comm=None, keys=None, serializers=None):
                     )
                     if not workers or ts is None:
                         continue
-                    ts_nbytes: Py_ssize_t = ts.get_nbytes()
                     recommendations: dict = {key: "released"}
                     for worker in workers:
                         ws = parent._workers_dv.get(worker)
-                        if ws is not None and ts in ws._has_what:
-                            del ws._has_what[ts]
-                            ts._who_has.remove(ws)
-                            ws._nbytes -= ts_nbytes
+                        if ws is not None and ws in ts._who_has:
+                            parent.remove_replica(ts, ws)
                             parent._transitions(
                                 recommendations, client_msgs, worker_msgs
                             )
@@ -5922,10 +5900,8 @@ async def gather_on_worker(
             if ts is None or ts._state != "memory":
                 logger.warning(f"Key lost during replication: {key}")
                 continue
-            if ts not in ws._has_what:
-                ws._nbytes += ts.get_nbytes()
-                ws._has_what[ts] = None
-                ts._who_has.add(ws)
+            if ws not in ts._who_has:
+                parent.add_replica(ts, ws)
 
         return keys_failed
 
@@ -5962,11 +5938,9 @@ async def delete_worker_data(self, worker_address: str, keys: "list[str]") -> No
 
         for key in keys:
             ts: TaskState = parent._tasks.get(key)
-            if ts is not None and ts in ws._has_what:
+            if ts is not None and ws in ts._who_has:
                 assert ts._state == "memory"
-                del ws._has_what[ts]
-                ts._who_has.remove(ws)
-                ws._nbytes -= ts.get_nbytes()
+                parent.remove_replica(ts, ws)
                 if not ts._who_has:
                     # Last copy deleted
                     self.transitions({key: "released"})
@@ -6714,10 +6688,8 @@ def add_keys(self, comm=None, worker=None, keys=()):
         for key in keys:
             ts: TaskState = parent._tasks.get(key)
             if ts is not None and ts._state == "memory":
-                if ts not in ws._has_what:
-                    ws._nbytes += ts.get_nbytes()
-                    ws._has_what[ts] = None
-                    ts._who_has.add(ws)
+                if ws not in ts._who_has:
+                    parent.add_replica(ts, ws)
             else:
                 superfluous_data.append(key)
         if superfluous_data:
@@ -6759,17 +6731,14 @@ def update_data(
                 if ts is None:
                     ts: TaskState = parent.new_task(key, None, "memory")
                 ts.state = "memory"
-                ts_nbytes: Py_ssize_t = nbytes.get(key, -1)
+                ts_nbytes = nbytes.get(key, -1)
                 if ts_nbytes >= 0:
                     ts.set_nbytes(ts_nbytes)
-                else:
-                    ts_nbytes = ts.get_nbytes()
+
                 for w in workers:
                     ws: WorkerState = parent._workers_dv[w]
-                    if ts not in ws._has_what:
-                        ws._nbytes += ts_nbytes
-                        ws._has_what[ts] = None
-                        ts._who_has.add(ws)
+                    if ws not in ts._who_has:
+                        parent.add_replica(ts, ws)
                 self.report(
                     {"op": "key-in-memory", "key": key, "workers": list(workers)}
                 )
@@ -7736,9 +7705,7 @@ def _add_to_memory(
     if state._validate:
         assert ts not in ws._has_what
 
-    ts._who_has.add(ws)
-    ws._has_what[ts] = None
-    ws._nbytes += ts.get_nbytes()
+    state.add_replica(ts, ws)
 
     deps: list = list(ts._dependents)
     if len(deps) > 1:
@@ -7814,12 +7781,8 @@ def _propagate_forgotten(
     ts._dependencies.clear()
     ts._waiting_on.clear()
 
-    ts_nbytes: Py_ssize_t = ts.get_nbytes()
-
     ws: WorkerState
     for ws in ts._who_has:
-        del ws._has_what[ts]
-        ws._nbytes -= ts_nbytes
         w: str = ws._address
         if w in state._workers_dv:  # in case worker has died
             worker_msgs[w] = [
@@ -7829,7 +7792,7 @@ def _propagate_forgotten(
                     "reason": f"propagate-forgotten {ts.key}",
                 }
             ]
-    ts._who_has.clear()
+    state.remove_all_replicas(ts)
 
 
 @cfunc

From 05677bb2be375231d563bc099624965d7e1bc4b1 Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Thu, 16 Sep 2021 19:43:54 +0100
Subject: [PATCH 1492/1550] Run multiple AMMs in parallel (#5315)

Propaedeutic to RetireWorker AMM policy
---
 distributed/active_memory_manager.py          | 25 +++++---
 .../tests/test_active_memory_manager.py       | 59 +++++++++++++++++++
 2 files changed, 76 insertions(+), 8 deletions(-)

diff --git a/distributed/active_memory_manager.py b/distributed/active_memory_manager.py
index 77e67ca4319..e121fc72de6 100644
--- a/distributed/active_memory_manager.py
+++ b/distributed/active_memory_manager.py
@@ -52,23 +52,20 @@ def __init__(
         interval: Optional[float] = None,
     ):
         self.scheduler = scheduler
+        self.policies = set()
 
         if policies is None:
+            # Initialize policies from config
             policies = set()
             for kwargs in dask.config.get(
                 "distributed.scheduler.active-memory-manager.policies"
             ):
                 kwargs = kwargs.copy()
                 cls = import_term(kwargs.pop("class"))
-                if not issubclass(cls, ActiveMemoryManagerPolicy):
-                    raise TypeError(
-                        f"{cls}: Expected ActiveMemoryManagerPolicy; got {type(cls)}"
-                    )
                 policies.add(cls(**kwargs))
 
         for policy in policies:
-            policy.manager = self
-        self.policies = policies
+            self.add_policy(policy)
 
         if register:
             scheduler.extensions["amm"] = self
@@ -92,16 +89,28 @@ def __init__(
 
     def start(self, comm=None) -> None:
         """Start executing every ``self.interval`` seconds until scheduler shutdown"""
+        if self.started:
+            return
         pc = PeriodicCallback(self.run_once, self.interval * 1000.0)
-        self.scheduler.periodic_callbacks["amm"] = pc
+        self.scheduler.periodic_callbacks[f"amm-{id(self)}"] = pc
         pc.start()
 
     def stop(self, comm=None) -> None:
         """Stop periodic execution"""
-        pc = self.scheduler.periodic_callbacks.pop("amm", None)
+        pc = self.scheduler.periodic_callbacks.pop(f"amm-{id(self)}", None)
         if pc:
             pc.stop()
 
+    @property
+    def started(self) -> bool:
+        return f"amm-{id(self)}" in self.scheduler.periodic_callbacks
+
+    def add_policy(self, policy: ActiveMemoryManagerPolicy) -> None:
+        if not isinstance(policy, ActiveMemoryManagerPolicy):
+            raise TypeError(f"Expected ActiveMemoryManagerPolicy; got {policy!r}")
+        self.policies.add(policy)
+        policy.manager = self
+
     def run_once(self, comm=None) -> None:
         """Run all policies once and asynchronously (fire and forget) enact their
         recommendations to replicate/drop keys
diff --git a/distributed/tests/test_active_memory_manager.py b/distributed/tests/test_active_memory_manager.py
index afe6c114940..c7c747b8507 100644
--- a/distributed/tests/test_active_memory_manager.py
+++ b/distributed/tests/test_active_memory_manager.py
@@ -101,6 +101,65 @@ async def test_auto_start(c, s, a, b):
     assert len(s.tasks["x"].who_has) == 1
 
 
+@gen_cluster(client=True, config=demo_config("drop", key="x"))
+async def test_add_policy(c, s, a, b):
+    p2 = DemoPolicy(action="drop", key="y", n=10, candidates=None)
+    p3 = DemoPolicy(action="drop", key="z", n=10, candidates=None)
+
+    # policies parameter can be:
+    # - None: get from config
+    # - explicit set, which can be empty
+    m1 = s.extensions["amm"]
+    m2 = ActiveMemoryManagerExtension(s, {p2}, register=False, start=False)
+    m3 = ActiveMemoryManagerExtension(s, set(), register=False, start=False)
+
+    assert len(m1.policies) == 1
+    assert len(m2.policies) == 1
+    assert len(m3.policies) == 0
+    m3.add_policy(p3)
+    assert len(m3.policies) == 1
+
+    futures = await c.scatter({"x": 1, "y": 2, "z": 3}, broadcast=True)
+    m1.run_once()
+    while len(s.tasks["x"].who_has) == 2:
+        await asyncio.sleep(0.01)
+
+    m2.run_once()
+    while len(s.tasks["y"].who_has) == 2:
+        await asyncio.sleep(0.01)
+
+    m3.run_once()
+    while len(s.tasks["z"].who_has) == 2:
+        await asyncio.sleep(0.01)
+
+
+@gen_cluster(client=True, config=demo_config("drop", key="x", start=False))
+async def test_multi_start(c, s, a, b):
+    """Multiple AMMs can be started in parallel"""
+    p2 = DemoPolicy(action="drop", key="y", n=10, candidates=None)
+    p3 = DemoPolicy(action="drop", key="z", n=10, candidates=None)
+
+    # policies parameter can be:
+    # - None: get from config
+    # - explicit set, which can be empty
+    m1 = s.extensions["amm"]
+    m2 = ActiveMemoryManagerExtension(s, {p2}, register=False, start=True, interval=0.1)
+    m3 = ActiveMemoryManagerExtension(s, {p3}, register=False, start=True, interval=0.1)
+
+    assert not m1.started
+    assert m2.started
+    assert m3.started
+
+    futures = await c.scatter({"x": 1, "y": 2, "z": 3}, broadcast=True)
+
+    # The AMMs should run within 0.1s of the broadcast.
+    # Add generous extra padding to prevent flakiness.
+    await asyncio.sleep(0.5)
+    assert len(s.tasks["x"].who_has) == 2
+    assert len(s.tasks["y"].who_has) == 1
+    assert len(s.tasks["z"].who_has) == 1
+
+
 @gen_cluster(client=True, config=NO_AMM_START)
 async def test_not_registered(c, s, a, b):
     futures = await c.scatter({"x": 1}, broadcast=True)

From 8dc2f10537c515f993fc7e20cd4f46bd9a79e363 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Tue, 21 Sep 2021 15:54:44 -0500
Subject: [PATCH 1493/1550] Revert AMM ReduceReplicas and parallel AMMs updates
 (#5335)

---
 distributed/active_memory_manager.py          |  56 ++---
 distributed/scheduler.py                      | 211 ++++++++++--------
 .../tests/test_active_memory_manager.py       |  59 -----
 3 files changed, 145 insertions(+), 181 deletions(-)

diff --git a/distributed/active_memory_manager.py b/distributed/active_memory_manager.py
index e121fc72de6..4ed2daf4113 100644
--- a/distributed/active_memory_manager.py
+++ b/distributed/active_memory_manager.py
@@ -52,20 +52,23 @@ def __init__(
         interval: Optional[float] = None,
     ):
         self.scheduler = scheduler
-        self.policies = set()
 
         if policies is None:
-            # Initialize policies from config
             policies = set()
             for kwargs in dask.config.get(
                 "distributed.scheduler.active-memory-manager.policies"
             ):
                 kwargs = kwargs.copy()
                 cls = import_term(kwargs.pop("class"))
+                if not issubclass(cls, ActiveMemoryManagerPolicy):
+                    raise TypeError(
+                        f"{cls}: Expected ActiveMemoryManagerPolicy; got {type(cls)}"
+                    )
                 policies.add(cls(**kwargs))
 
         for policy in policies:
-            self.add_policy(policy)
+            policy.manager = self
+        self.policies = policies
 
         if register:
             scheduler.extensions["amm"] = self
@@ -89,28 +92,16 @@ def __init__(
 
     def start(self, comm=None) -> None:
         """Start executing every ``self.interval`` seconds until scheduler shutdown"""
-        if self.started:
-            return
         pc = PeriodicCallback(self.run_once, self.interval * 1000.0)
-        self.scheduler.periodic_callbacks[f"amm-{id(self)}"] = pc
+        self.scheduler.periodic_callbacks["amm"] = pc
         pc.start()
 
     def stop(self, comm=None) -> None:
         """Stop periodic execution"""
-        pc = self.scheduler.periodic_callbacks.pop(f"amm-{id(self)}", None)
+        pc = self.scheduler.periodic_callbacks.pop("amm", None)
         if pc:
             pc.stop()
 
-    @property
-    def started(self) -> bool:
-        return f"amm-{id(self)}" in self.scheduler.periodic_callbacks
-
-    def add_policy(self, policy: ActiveMemoryManagerPolicy) -> None:
-        if not isinstance(policy, ActiveMemoryManagerPolicy):
-            raise TypeError(f"Expected ActiveMemoryManagerPolicy; got {policy!r}")
-        self.policies.add(policy)
-        policy.manager = self
-
     def run_once(self, comm=None) -> None:
         """Run all policies once and asynchronously (fire and forget) enact their
         recommendations to replicate/drop keys
@@ -198,14 +189,9 @@ def _find_recipient(
         pending_repl: set[WorkerState],
     ) -> Optional[WorkerState]:
         """Choose a worker to acquire a new replica of an in-memory task among a set of
-        candidates. If candidates is None, default to all workers in the cluster.
-        Regardless, workers that either already hold a replica or are scheduled to
-        receive one at the end of this AMM iteration are not considered.
-
-        Returns
-        -------
-        The worker with the lowest memory usage (downstream of pending replications and
-        drops), or None if no eligible candidates are available.
+        candidates. If candidates is None, default to all workers in the cluster that do
+        not hold a replica yet. The worker with the lowest memory usage (downstream of
+        pending replications and drops) will be returned.
         """
         if ts.state != "memory":
             return None
@@ -224,15 +210,9 @@ def _find_dropper(
         pending_drop: set[WorkerState],
     ) -> Optional[WorkerState]:
         """Choose a worker to drop its replica of an in-memory task among a set of
-        candidates. If candidates is None, default to all workers in the cluster.
-        Regardless, workers that either do not hold a replica or are already scheduled
-        to drop theirs at the end of this AMM iteration are not considered.
-        This method also ensures that a key will not lose its last replica.
-
-        Returns
-        -------
-        The worker with the highest memory usage (downstream of pending replications and
-        drops), or None if no eligible candidates are available.
+        candidates. If candidates is None, default to all workers in the cluster that
+        hold a replica. The worker with the highest memory usage (downstream of pending
+        replications and drops) will be returned.
         """
         if len(ts.who_has) - len(pending_drop) < 2:
             return None
@@ -303,7 +283,13 @@ class ReduceReplicas(ActiveMemoryManagerPolicy):
     """
 
     def run(self):
-        for ts in self.manager.scheduler.replicated_tasks:
+        # TODO this is O(n) to the total number of in-memory tasks on the cluster; it
+        #      could be made faster by automatically attaching it to a TaskState when it
+        #      goes above one replica and detaching it when it drops below two.
+        for ts in self.manager.scheduler.tasks.values():
+            if len(ts.who_has) < 2:
+                continue
+
             desired_replicas = 1  # TODO have a marker on TaskState
 
             # If a dependent task has not been assigned to a worker yet, err on the side
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 04496b26e05..706e85c3b24 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1859,7 +1859,6 @@ class SchedulerState:
     _task_groups: dict
     _task_prefixes: dict
     _task_metadata: dict
-    _replicated_tasks: set
     _total_nthreads: Py_ssize_t
     _total_occupancy: double
     _transitions_table: dict
@@ -1919,9 +1918,6 @@ def __init__(
             self._tasks = tasks
         else:
             self._tasks = dict()
-        self._replicated_tasks = {
-            ts for ts in self._tasks.values() if len(ts._who_has) > 1
-        }
         self._computations = deque(
             maxlen=dask.config.get("distributed.diagnostics.computations.max-history")
         )
@@ -2039,10 +2035,6 @@ def task_prefixes(self):
     def task_metadata(self):
         return self._task_metadata
 
-    @property
-    def replicated_tasks(self):
-        return self._replicated_tasks
-
     @property
     def total_nthreads(self):
         return self._total_nthreads
@@ -2828,14 +2820,18 @@ def transition_memory_released(self, key, safe: bint = False):
                     dts._waiting_on.add(ts)
 
             # XXX factor this out?
+            ts_nbytes: Py_ssize_t = ts.get_nbytes()
             worker_msg = {
                 "op": "free-keys",
                 "keys": [key],
                 "reason": f"Memory->Released {key}",
             }
             for ws in ts._who_has:
+                del ws._has_what[ts]
+                ws._nbytes -= ts_nbytes
                 worker_msgs[ws._address] = [worker_msg]
-            self.remove_all_replicas(ts)
+
+            ts._who_has.clear()
 
             ts.state = "released"
 
@@ -3429,40 +3425,6 @@ def worker_objective(self, ts: TaskState, ws: WorkerState) -> tuple:
         else:
             return (start_time, ws._nbytes)
 
-    @ccall
-    def add_replica(self, ts: TaskState, ws: WorkerState):
-        """Note that a worker holds a replica of a task with state='memory'"""
-        if self._validate:
-            assert ws not in ts._who_has
-            assert ts not in ws._has_what
-
-        ws._nbytes += ts.get_nbytes()
-        ws._has_what[ts] = None
-        ts._who_has.add(ws)
-        if len(ts._who_has) == 2:
-            self._replicated_tasks.add(ts)
-
-    @ccall
-    def remove_replica(self, ts: TaskState, ws: WorkerState):
-        """Note that a worker no longer holds a replica of a task"""
-        ws._nbytes -= ts.get_nbytes()
-        del ws._has_what[ts]
-        ts._who_has.remove(ws)
-        if len(ts._who_has) == 1:
-            self._replicated_tasks.remove(ts)
-
-    @ccall
-    def remove_all_replicas(self, ts: TaskState):
-        """Remove all replicas of a task from all workers"""
-        ws: WorkerState
-        nbytes: Py_ssize_t = ts.get_nbytes()
-        for ws in ts._who_has:
-            ws._nbytes -= nbytes
-            del ws._has_what[ts]
-        if len(ts._who_has) > 1:
-            self._replicated_tasks.remove(ts)
-        ts._who_has.clear()
-
 
 class Scheduler(SchedulerState, ServerNode):
     """Dynamic distributed task scheduler
@@ -4774,23 +4736,70 @@ def stimulus_task_erred(
         parent: SchedulerState = cast(SchedulerState, self)
         logger.debug("Stimulus task erred %s, %s", key, worker)
 
+        recommendations: dict = {}
+        client_msgs: dict = {}
+        worker_msgs: dict = {}
+
         ts: TaskState = parent._tasks.get(key)
-        if ts is None or ts._state != "processing":
-            return {}, {}, {}
+        if ts is None:
+            return recommendations, client_msgs, worker_msgs
 
-        if ts._retries > 0:
-            ts._retries -= 1
-            return parent._transition(key, "waiting")
-        else:
-            return parent._transition(
-                key,
-                "erred",
-                cause=key,
-                exception=exception,
-                traceback=traceback,
-                worker=worker,
-                **kwargs,
-            )
+        if ts._state == "processing":
+            retries: Py_ssize_t = ts._retries
+            r: tuple
+            if retries > 0:
+                ts._retries = retries - 1
+                r = parent._transition(key, "waiting")
+            else:
+                r = parent._transition(
+                    key,
+                    "erred",
+                    cause=key,
+                    exception=exception,
+                    traceback=traceback,
+                    worker=worker,
+                    **kwargs,
+                )
+            recommendations, client_msgs, worker_msgs = r
+
+        return recommendations, client_msgs, worker_msgs
+
+    def stimulus_missing_data(
+        self, cause=None, key=None, worker=None, ensure=True, **kwargs
+    ):
+        """Mark that certain keys have gone missing.  Recover."""
+        parent: SchedulerState = cast(SchedulerState, self)
+        with log_errors():
+            logger.debug("Stimulus missing data %s, %s", key, worker)
+
+            recommendations: dict = {}
+            client_msgs: dict = {}
+            worker_msgs: dict = {}
+
+            ts: TaskState = parent._tasks.get(key)
+            if ts is None or ts._state == "memory":
+                return recommendations, client_msgs, worker_msgs
+            cts: TaskState = parent._tasks.get(cause)
+
+            if cts is not None and cts._state == "memory":  # couldn't find this
+                ws: WorkerState
+                cts_nbytes: Py_ssize_t = cts.get_nbytes()
+                for ws in cts._who_has:  # TODO: this behavior is extreme
+                    del ws._has_what[ts]
+                    ws._nbytes -= cts_nbytes
+                cts._who_has.clear()
+                recommendations[cause] = "released"
+
+            if key:
+                recommendations[key] = "released"
+
+            parent._transitions(recommendations, client_msgs, worker_msgs)
+            recommendations = {}
+
+            if parent._validate:
+                assert cause not in self.who_has
+
+            return recommendations, client_msgs, worker_msgs
 
     def stimulus_retry(self, comm=None, keys=None, client=None):
         parent: SchedulerState = cast(SchedulerState, self)
@@ -4905,13 +4914,14 @@ async def remove_worker(self, comm=None, address=None, safe=False, close=True):
                             self.allowed_failures,
                         )
 
-            for ts in list(ws._has_what):
-                parent.remove_replica(ts, ws)
+            for ts in ws._has_what:
+                ts._who_has.remove(ws)
                 if not ts._who_has:
                     if ts._run_spec:
                         recommendations[ts._key] = "released"
                     else:  # pure data
                         recommendations[ts._key] = "forgotten"
+            ws._has_what.clear()
 
             self.transitions(recommendations)
 
@@ -5061,7 +5071,6 @@ def validate_memory(self, key):
         ts: TaskState = parent._tasks[key]
         dts: TaskState
         assert ts._who_has
-        assert bool(ts in parent._replicated_tasks) == (len(ts._who_has) > 1)
         assert not ts._processing_on
         assert not ts._waiting_on
         assert ts not in parent._unrunnable
@@ -5132,13 +5141,8 @@ def validate_state(self, allow_overlap=False):
         for k, ts in parent._tasks.items():
             assert isinstance(ts, TaskState), (type(ts), ts)
             assert ts._key == k
-            assert bool(ts in parent._replicated_tasks) == (len(ts._who_has) > 1)
             self.validate_key(k, ts)
 
-        for ts in parent._replicated_tasks:
-            assert ts._state == "memory"
-            assert ts._key in parent._tasks
-
         c: str
         cs: ClientState
         for c, cs in parent._clients.items():
@@ -5339,14 +5343,24 @@ def handle_task_erred(self, key=None, **msg):
 
         self.send_all(client_msgs, worker_msgs)
 
-    def handle_release_data(self, key=None, worker=None, **kwargs):
+    def handle_release_data(self, key=None, worker=None, client=None, **msg):
         parent: SchedulerState = cast(SchedulerState, self)
         ts: TaskState = parent._tasks.get(key)
-        if ts is None or ts._state == "memory":
+        if ts is None:
             return
         ws: WorkerState = parent._workers_dv.get(worker)
-        if ws is not None and ts._processing_on == ws:
-            parent._transitions({key: "released"}, {}, {})
+        if ws is None or ts._processing_on != ws:
+            return
+
+        recommendations: dict
+        client_msgs: dict
+        worker_msgs: dict
+
+        r: tuple = self.stimulus_missing_data(key=key, ensure=False, **msg)
+        recommendations, client_msgs, worker_msgs = r
+        parent._transitions(recommendations, client_msgs, worker_msgs)
+
+        self.send_all(client_msgs, worker_msgs)
 
     def handle_missing_data(self, key=None, errant_worker=None, **kwargs):
         parent: SchedulerState = cast(SchedulerState, self)
@@ -5358,7 +5372,9 @@ def handle_missing_data(self, key=None, errant_worker=None, **kwargs):
             return
         ws: WorkerState = parent._workers_dv.get(errant_worker)
         if ws is not None and ws in ts._who_has:
-            parent.remove_replica(ts, ws)
+            ts._who_has.remove(ws)
+            del ws._has_what[ts]
+            ws._nbytes -= ts.get_nbytes()
         if not ts._who_has:
             if ts._run_spec:
                 self.transitions({key: "released"})
@@ -5376,8 +5392,11 @@ def release_worker_data(self, comm=None, keys=None, worker=None):
         ts: TaskState
         recommendations: dict = {}
         for ts in removed_tasks:
-            parent.remove_replica(ts, ws)
-            if not ts._who_has:
+            del ws._has_what[ts]
+            ws._nbytes -= ts.get_nbytes()
+            wh: set = ts._who_has
+            wh.remove(ws)
+            if not wh:
                 recommendations[ts._key] = "released"
         if recommendations:
             self.transitions(recommendations)
@@ -5697,11 +5716,14 @@ async def gather(self, comm=None, keys=None, serializers=None):
                     )
                     if not workers or ts is None:
                         continue
+                    ts_nbytes: Py_ssize_t = ts.get_nbytes()
                     recommendations: dict = {key: "released"}
                     for worker in workers:
                         ws = parent._workers_dv.get(worker)
-                        if ws is not None and ws in ts._who_has:
-                            parent.remove_replica(ts, ws)
+                        if ws is not None and ts in ws._has_what:
+                            del ws._has_what[ts]
+                            ts._who_has.remove(ws)
+                            ws._nbytes -= ts_nbytes
                             parent._transitions(
                                 recommendations, client_msgs, worker_msgs
                             )
@@ -5900,8 +5922,10 @@ async def gather_on_worker(
             if ts is None or ts._state != "memory":
                 logger.warning(f"Key lost during replication: {key}")
                 continue
-            if ws not in ts._who_has:
-                parent.add_replica(ts, ws)
+            if ts not in ws._has_what:
+                ws._nbytes += ts.get_nbytes()
+                ws._has_what[ts] = None
+                ts._who_has.add(ws)
 
         return keys_failed
 
@@ -5938,9 +5962,11 @@ async def delete_worker_data(self, worker_address: str, keys: "list[str]") -> No
 
         for key in keys:
             ts: TaskState = parent._tasks.get(key)
-            if ts is not None and ws in ts._who_has:
+            if ts is not None and ts in ws._has_what:
                 assert ts._state == "memory"
-                parent.remove_replica(ts, ws)
+                del ws._has_what[ts]
+                ts._who_has.remove(ws)
+                ws._nbytes -= ts.get_nbytes()
                 if not ts._who_has:
                     # Last copy deleted
                     self.transitions({key: "released"})
@@ -6688,8 +6714,10 @@ def add_keys(self, comm=None, worker=None, keys=()):
         for key in keys:
             ts: TaskState = parent._tasks.get(key)
             if ts is not None and ts._state == "memory":
-                if ws not in ts._who_has:
-                    parent.add_replica(ts, ws)
+                if ts not in ws._has_what:
+                    ws._nbytes += ts.get_nbytes()
+                    ws._has_what[ts] = None
+                    ts._who_has.add(ws)
             else:
                 superfluous_data.append(key)
         if superfluous_data:
@@ -6731,14 +6759,17 @@ def update_data(
                 if ts is None:
                     ts: TaskState = parent.new_task(key, None, "memory")
                 ts.state = "memory"
-                ts_nbytes = nbytes.get(key, -1)
+                ts_nbytes: Py_ssize_t = nbytes.get(key, -1)
                 if ts_nbytes >= 0:
                     ts.set_nbytes(ts_nbytes)
-
+                else:
+                    ts_nbytes = ts.get_nbytes()
                 for w in workers:
                     ws: WorkerState = parent._workers_dv[w]
-                    if ws not in ts._who_has:
-                        parent.add_replica(ts, ws)
+                    if ts not in ws._has_what:
+                        ws._nbytes += ts_nbytes
+                        ws._has_what[ts] = None
+                        ts._who_has.add(ws)
                 self.report(
                     {"op": "key-in-memory", "key": key, "workers": list(workers)}
                 )
@@ -7705,7 +7736,9 @@ def _add_to_memory(
     if state._validate:
         assert ts not in ws._has_what
 
-    state.add_replica(ts, ws)
+    ts._who_has.add(ws)
+    ws._has_what[ts] = None
+    ws._nbytes += ts.get_nbytes()
 
     deps: list = list(ts._dependents)
     if len(deps) > 1:
@@ -7781,8 +7814,12 @@ def _propagate_forgotten(
     ts._dependencies.clear()
     ts._waiting_on.clear()
 
+    ts_nbytes: Py_ssize_t = ts.get_nbytes()
+
     ws: WorkerState
     for ws in ts._who_has:
+        del ws._has_what[ts]
+        ws._nbytes -= ts_nbytes
         w: str = ws._address
         if w in state._workers_dv:  # in case worker has died
             worker_msgs[w] = [
@@ -7792,7 +7829,7 @@ def _propagate_forgotten(
                     "reason": f"propagate-forgotten {ts.key}",
                 }
             ]
-    state.remove_all_replicas(ts)
+    ts._who_has.clear()
 
 
 @cfunc
diff --git a/distributed/tests/test_active_memory_manager.py b/distributed/tests/test_active_memory_manager.py
index c7c747b8507..afe6c114940 100644
--- a/distributed/tests/test_active_memory_manager.py
+++ b/distributed/tests/test_active_memory_manager.py
@@ -101,65 +101,6 @@ async def test_auto_start(c, s, a, b):
     assert len(s.tasks["x"].who_has) == 1
 
 
-@gen_cluster(client=True, config=demo_config("drop", key="x"))
-async def test_add_policy(c, s, a, b):
-    p2 = DemoPolicy(action="drop", key="y", n=10, candidates=None)
-    p3 = DemoPolicy(action="drop", key="z", n=10, candidates=None)
-
-    # policies parameter can be:
-    # - None: get from config
-    # - explicit set, which can be empty
-    m1 = s.extensions["amm"]
-    m2 = ActiveMemoryManagerExtension(s, {p2}, register=False, start=False)
-    m3 = ActiveMemoryManagerExtension(s, set(), register=False, start=False)
-
-    assert len(m1.policies) == 1
-    assert len(m2.policies) == 1
-    assert len(m3.policies) == 0
-    m3.add_policy(p3)
-    assert len(m3.policies) == 1
-
-    futures = await c.scatter({"x": 1, "y": 2, "z": 3}, broadcast=True)
-    m1.run_once()
-    while len(s.tasks["x"].who_has) == 2:
-        await asyncio.sleep(0.01)
-
-    m2.run_once()
-    while len(s.tasks["y"].who_has) == 2:
-        await asyncio.sleep(0.01)
-
-    m3.run_once()
-    while len(s.tasks["z"].who_has) == 2:
-        await asyncio.sleep(0.01)
-
-
-@gen_cluster(client=True, config=demo_config("drop", key="x", start=False))
-async def test_multi_start(c, s, a, b):
-    """Multiple AMMs can be started in parallel"""
-    p2 = DemoPolicy(action="drop", key="y", n=10, candidates=None)
-    p3 = DemoPolicy(action="drop", key="z", n=10, candidates=None)
-
-    # policies parameter can be:
-    # - None: get from config
-    # - explicit set, which can be empty
-    m1 = s.extensions["amm"]
-    m2 = ActiveMemoryManagerExtension(s, {p2}, register=False, start=True, interval=0.1)
-    m3 = ActiveMemoryManagerExtension(s, {p3}, register=False, start=True, interval=0.1)
-
-    assert not m1.started
-    assert m2.started
-    assert m3.started
-
-    futures = await c.scatter({"x": 1, "y": 2, "z": 3}, broadcast=True)
-
-    # The AMMs should run within 0.1s of the broadcast.
-    # Add generous extra padding to prevent flakiness.
-    await asyncio.sleep(0.5)
-    assert len(s.tasks["x"].who_has) == 2
-    assert len(s.tasks["y"].who_has) == 1
-    assert len(s.tasks["z"].who_has) == 1
-
-
 @gen_cluster(client=True, config=NO_AMM_START)
 async def test_not_registered(c, s, a, b):
     futures = await c.scatter({"x": 1}, broadcast=True)

From 6dc3183dfbfacbbedc9fa403ab6db0a44df69a94 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Tue, 21 Sep 2021 17:37:06 -0500
Subject: [PATCH 1494/1550] bump version to 2021.09.1

---
 docs/source/changelog.rst | 29 ++++++++++++++++++++++++++++-
 requirements.txt          |  2 +-
 2 files changed, 29 insertions(+), 2 deletions(-)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index cb853395047..14d3c88ca39 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,32 @@
 Changelog
 =========
 
+2021.09.1
+---------
+
+Released on September 21, 2021
+
+- Revert AMM ``ReduceReplicas`` and parallel AMMs updates (:pr:`5335`) `James Bourbeau`_
+- Run multiple AMMs in parallel (:pr:`5315`) `crusaderky`_
+- AMM ``ReduceReplicas`` to iterate only on replicated tasks (:pr:`5297`) `crusaderky`_
+- Add type annotations to various functions within ``distributed.worker`` (:pr:`5290`) `Tom Forbes`_
+- Mark ``test_ucx_config_w_env_var`` flaky on UCX < 1.11 (:pr:`5262`) `Peter Andreas Entschev`_
+- Warn if CUDA context is created on incorrect device in UCX (:pr:`5308`) `Peter Andreas Entschev`_
+- Remove redundant timeouts from ``test_client`` (:pr:`5314`) `crusaderky`_
+- Allow ``Client`` to subscribe to events // Remote printing and warning (:pr:`5217`) `Florian Jetter`_
+- Test pickle protocols 4 & 5 (:pr:`5313`) `jakirkham`_
+- Fix-up ``test_pickle_empty`` (:pr:`5303`) `jakirkham`_
+- Increase timeout for ``test_worker_reconnects_mid_compute_multiple_states_on_scheduler`` (:pr:`5304`) `Florian Jetter`_
+- Add synced dict between cluster and scheduler to store cluster info (:pr:`5033`) `Jacob Tomlinson`_
+- Update ``test_sub_submit_priority`` (:pr:`5301`) `James Bourbeau`_
+- Revert "Add test setup fixture (:pr:`5242`)" (:pr:`5300`) `James Bourbeau`_
+- Fix flaky ``test_worker_reconnects_mid_compute`` (:pr:`5299`) `Florian Jetter`_
+- Use ``gen_test`` in ``test_adaptive`` (:pr:`5298`) `crusaderky`_
+- Increase ``worker.suspicious_counter`` threshold (:pr:`5228`) `Florian Jetter`_
+- Active Memory Manager framework + discard excess replicas (:pr:`5111`) `crusaderky`_
+- Add test setup fixture (:pr:`5242`) `James Bourbeau`_
+
+
 2021.09.0
 ---------
 
@@ -2635,4 +2661,5 @@ significantly without many new features.
 .. _`gerrymanoim`: https://github.com/gerrymanoim
 .. _`Bryan Van de Ven`: https://github.com/bryevdv
 .. _`David Chudzicki`: https://github.com/dchudz
-.. _`Walt Woods`: https://github.com/wwoods
\ No newline at end of file
+.. _`Walt Woods`: https://github.com/wwoods
+.. _`Tom Forbes`: https://github.com/orf
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index a5a8aa41575..4d6d3669c9c 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,6 @@
 click >= 6.6
 cloudpickle >= 1.5.0
-dask == 2021.09.0
+dask == 2021.09.1
 jinja2
 msgpack >= 0.6.0
 psutil >= 5.0

From b3413d968ede344c493fa6143a5a1a91e7699021 Mon Sep 17 00:00:00 2001
From: Genevieve Buckley <30920819+GenevieveBuckley@users.noreply.github.com>
Date: Wed, 22 Sep 2021 20:29:37 +1000
Subject: [PATCH 1495/1550] Fix typo in client side example foundations.rst
 (#5336)

---
 docs/source/foundations.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/source/foundations.rst b/docs/source/foundations.rst
index 62253433763..2e78bd49e8b 100644
--- a/docs/source/foundations.rst
+++ b/docs/source/foundations.rst
@@ -120,7 +120,7 @@ Client Side
        await comm.close()
        print(result)
 
-   >>> asyncio.get_event_loop().run_until_complete(g())
+   >>> asyncio.get_event_loop().run_until_complete(f())
    3
 
    async def g():

From 23210903f269677d32060b2baa141475e284bfd5 Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Wed, 22 Sep 2021 14:08:51 +0100
Subject: [PATCH 1496/1550] Run pyupgrade in CI (#5327)

---
 .github/PULL_REQUEST_TEMPLATE.md              |  2 +-
 .pre-commit-config.yaml                       | 12 +++++-
 distributed/_version.py                       | 27 ++++++------
 distributed/active_memory_manager.py          | 24 +++++------
 distributed/cli/dask_spec.py                  |  4 +-
 distributed/cli/dask_worker.py                |  4 +-
 distributed/client.py                         | 10 ++---
 distributed/comm/core.py                      |  4 +-
 distributed/core.py                           |  2 +-
 distributed/dashboard/components/scheduler.py |  4 +-
 distributed/dashboard/components/worker.py    |  4 +-
 .../dashboard/tests/test_scheduler_bokeh.py   | 30 ++++++-------
 distributed/deploy/tests/test_spec_cluster.py |  6 +--
 .../diagnostics/tests/test_progress.py        |  2 +-
 .../diagnostics/tests/test_progressbar.py     |  2 +-
 distributed/nanny.py                          |  2 +-
 distributed/protocol/serialize.py             |  2 +-
 distributed/scheduler.py                      |  3 +-
 distributed/tests/test_client.py              | 16 +++----
 distributed/tests/test_config.py              |  4 +-
 distributed/tests/test_core.py                | 24 +++++------
 distributed/tests/test_scheduler.py           | 17 +++-----
 distributed/tests/test_steal.py               |  6 +--
 distributed/utils.py                          |  2 +-
 distributed/utils_test.py                     |  6 +--
 distributed/worker.py                         | 28 ++++++------
 docs/source/conf.py                           |  8 ++--
 docs/source/develop.rst                       |  7 +++
 setup.cfg                                     |  7 +--
 versioneer.py                                 | 43 +++++++++----------
 30 files changed, 155 insertions(+), 157 deletions(-)

diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index 0c693bd7f72..c8983d2660a 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -1,3 +1,3 @@
 - [ ] Closes #xxxx
 - [ ] Tests added / passed
-- [ ] Passes `black distributed` / `flake8 distributed` / `isort distributed`
+- [ ] Passes `pre-commit run --all-files`
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 7b497037a1a..c492b6e6407 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,11 +1,11 @@
 repos:
   -   repo: https://github.com/pycqa/isort
-      rev: 5.8.0
+      rev: 5.9.3
       hooks:
       - id: isort
         language_version: python3
   -   repo: https://github.com/psf/black
-      rev: 21.5b1
+      rev: 21.9b0
       hooks:
       - id: black
         language_version: python3
@@ -17,3 +17,11 @@ repos:
       hooks:
       - id: flake8
         language_version: python3
+  - repo: https://github.com/asottile/pyupgrade
+    # Do not upgrade: there's a bug in Cython that causes sum(... for ...) to fail;
+    # it needs sum([... for ...])
+    rev: v2.13.0
+    hooks:
+      - id: pyupgrade
+        args:
+          - "--py37-plus"
diff --git a/distributed/_version.py b/distributed/_version.py
index 5530c657430..6360ea703c4 100644
--- a/distributed/_version.py
+++ b/distributed/_version.py
@@ -9,7 +9,6 @@
 
 """Git implementation of _version.py."""
 
-import errno
 import os
 import re
 import subprocess
@@ -84,17 +83,16 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env=
                 stderr=(subprocess.PIPE if hide_stderr else None),
             )
             break
-        except EnvironmentError:
-            e = sys.exc_info()[1]
-            if e.errno == errno.ENOENT:
-                continue
+        except FileNotFoundError:
+            continue
+        except OSError as e:
             if verbose:
                 print("unable to run %s" % dispcmd)
                 print(e)
             return None, None
     else:
         if verbose:
-            print("unable to find command, tried %s" % (commands,))
+            print(f"unable to find command, tried {commands}")
         return None, None
     stdout = p.communicate()[0].strip().decode()
     if p.returncode != 0:
@@ -145,7 +143,7 @@ def git_get_keywords(versionfile_abs):
     # _version.py.
     keywords = {}
     try:
-        f = open(versionfile_abs, "r")
+        f = open(versionfile_abs)
         for line in f.readlines():
             if line.strip().startswith("git_refnames ="):
                 mo = re.search(r'=\s*"(.*)"', line)
@@ -160,7 +158,7 @@ def git_get_keywords(versionfile_abs):
                 if mo:
                     keywords["date"] = mo.group(1)
         f.close()
-    except EnvironmentError:
+    except OSError:
         pass
     return keywords
 
@@ -184,11 +182,11 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose):
         if verbose:
             print("keywords are unexpanded, not using")
         raise NotThisMethod("unexpanded keywords, not a git-archive tarball")
-    refs = set([r.strip() for r in refnames.strip("()").split(",")])
+    refs = {r.strip() for r in refnames.strip("()").split(",")}
     # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of
     # just "foo-1.0". If we see a "tag: " prefix, prefer those.
     TAG = "tag: "
-    tags = set([r[len(TAG) :] for r in refs if r.startswith(TAG)])
+    tags = {r[len(TAG) :] for r in refs if r.startswith(TAG)}
     if not tags:
         # Either we're using git < 1.8.3, or there really are no tags. We use
         # a heuristic: assume all version tags have a digit. The old git %d
@@ -197,7 +195,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose):
         # between branches and tags. By ignoring refnames without digits, we
         # filter out many common branch names like "release" and
         # "stabilization", as well as "HEAD" and "main".
-        tags = set([r for r in refs if re.search(r"\d", r)])
+        tags = {r for r in refs if re.search(r"\d", r)}
         if verbose:
             print("discarding '%s', no digits" % ",".join(refs - tags))
     if verbose:
@@ -300,10 +298,9 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
             if verbose:
                 fmt = "tag '%s' doesn't start with prefix '%s'"
                 print(fmt % (full_tag, tag_prefix))
-            pieces["error"] = "tag '%s' doesn't start with prefix '%s'" % (
-                full_tag,
-                tag_prefix,
-            )
+            pieces[
+                "error"
+            ] = f"tag '{full_tag}' doesn't start with prefix '{tag_prefix}'"
             return pieces
         pieces["closest-tag"] = full_tag[len(tag_prefix) :]
 
diff --git a/distributed/active_memory_manager.py b/distributed/active_memory_manager.py
index 4ed2daf4113..c8c29f3aaf1 100644
--- a/distributed/active_memory_manager.py
+++ b/distributed/active_memory_manager.py
@@ -3,7 +3,7 @@
 import asyncio
 from collections import defaultdict
 from collections.abc import Generator
-from typing import TYPE_CHECKING, Optional
+from typing import TYPE_CHECKING
 
 from tornado.ioloop import PeriodicCallback
 
@@ -46,10 +46,10 @@ def __init__(
         scheduler: SchedulerState,
         # The following parameters are exposed so that one may create, run, and throw
         # away on the fly a specialized manager, separate from the main one.
-        policies: Optional[set[ActiveMemoryManagerPolicy]] = None,
+        policies: set[ActiveMemoryManagerPolicy] | None = None,
         register: bool = True,
-        start: Optional[bool] = None,
-        interval: Optional[float] = None,
+        start: bool | None = None,
+        interval: float | None = None,
     ):
         self.scheduler = scheduler
 
@@ -148,9 +148,9 @@ def _run_policies(self) -> None:
         """Sequentially run ActiveMemoryManagerPolicy.run() for all registered policies,
         obtain replicate/drop suggestions, and use them to populate self.pending.
         """
-        candidates: Optional[set[WorkerState]]
+        candidates: set[WorkerState] | None
         cmd: str
-        ws: Optional[WorkerState]
+        ws: WorkerState | None
         ts: TaskState
         nreplicas: int
 
@@ -185,9 +185,9 @@ def _run_policies(self) -> None:
     def _find_recipient(
         self,
         ts: TaskState,
-        candidates: Optional[set[WorkerState]],
+        candidates: set[WorkerState] | None,
         pending_repl: set[WorkerState],
-    ) -> Optional[WorkerState]:
+    ) -> WorkerState | None:
         """Choose a worker to acquire a new replica of an in-memory task among a set of
         candidates. If candidates is None, default to all workers in the cluster that do
         not hold a replica yet. The worker with the lowest memory usage (downstream of
@@ -206,9 +206,9 @@ def _find_recipient(
     def _find_dropper(
         self,
         ts: TaskState,
-        candidates: Optional[set[WorkerState]],
+        candidates: set[WorkerState] | None,
         pending_drop: set[WorkerState],
-    ) -> Optional[WorkerState]:
+    ) -> WorkerState | None:
         """Choose a worker to drop its replica of an in-memory task among a set of
         candidates. If candidates is None, default to all workers in the cluster that
         hold a replica. The worker with the highest memory usage (downstream of pending
@@ -238,8 +238,8 @@ def __repr__(self) -> str:
     def run(
         self,
     ) -> Generator[
-        tuple[str, TaskState, Optional[set[WorkerState]]],
-        Optional[WorkerState],
+        tuple[str, TaskState, set[WorkerState] | None],
+        WorkerState | None,
         None,
     ]:
         """This method is invoked by the ActiveMemoryManager every few seconds, or
diff --git a/distributed/cli/dask_spec.py b/distributed/cli/dask_spec.py
index ef82ca3f493..7d1aea5750d 100644
--- a/distributed/cli/dask_spec.py
+++ b/distributed/cli/dask_spec.py
@@ -32,9 +32,9 @@ def main(args, spec: str, spec_file: str):
     async def run():
         servers = await run_spec(_spec, *args)
         try:
-            await asyncio.gather(*[w.finished() for w in servers.values()])
+            await asyncio.gather(*(w.finished() for w in servers.values()))
         except KeyboardInterrupt:
-            await asyncio.gather(*[w.close() for w in servers.values()])
+            await asyncio.gather(*(w.close() for w in servers.values()))
 
     asyncio.get_event_loop().run_until_complete(run())
 
diff --git a/distributed/cli/dask_worker.py b/distributed/cli/dask_worker.py
index 1f00a928ff6..06a8ef6fbb5 100755
--- a/distributed/cli/dask_worker.py
+++ b/distributed/cli/dask_worker.py
@@ -430,7 +430,7 @@ def del_pid_file():
     async def close_all():
         # Unregister all workers from scheduler
         if nanny:
-            await asyncio.gather(*[n.close(timeout=2) for n in nannies])
+            await asyncio.gather(*(n.close(timeout=2) for n in nannies))
 
     signal_fired = False
 
@@ -443,7 +443,7 @@ def on_signal(signum):
 
     async def run():
         await asyncio.gather(*nannies)
-        await asyncio.gather(*[n.finished() for n in nannies])
+        await asyncio.gather(*(n.finished() for n in nannies))
 
     install_signal_handlers(loop, cleanup=on_signal)
 
diff --git a/distributed/client.py b/distributed/client.py
index b14f8777432..51999a8f07f 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -1682,14 +1682,14 @@ def map(
 
         if batch_size and batch_size > 1 and total_length > batch_size:
             batches = list(
-                zip(*[partition_all(batch_size, iterable) for iterable in iterables])
+                zip(*(partition_all(batch_size, iterable) for iterable in iterables))
             )
             if isinstance(key, list):
                 keys = [list(element) for element in partition_all(batch_size, key)]
             else:
                 keys = [key for _ in range(len(batches))]
             return sum(
-                [
+                (
                     self.map(
                         func,
                         *batch,
@@ -1706,7 +1706,7 @@ def map(
                         **kwargs,
                     )
                     for key, batch in zip(keys, batches)
-                ],
+                ),
                 [],
             )
 
@@ -4159,7 +4159,7 @@ class will be instantiated with any extra keyword arguments.
         ...         pass
         ...     def transition(self, key: str, start: str, finish: str, **kwargs):
         ...         pass
-        ...     def release_key(self, key: str, state: str, cause: Optional[str], reason: None, report: bool):
+        ...     def release_key(self, key: str, state: str, cause: str | None, reason: None, report: bool):
         ...         pass
 
         >>> plugin = MyPlugin(1, 2, 3)
@@ -4228,7 +4228,7 @@ def unregister_worker_plugin(self, name, nanny=None):
         ...         pass
         ...     def transition(self, key: str, start: str, finish: str, **kwargs):
         ...         pass
-        ...     def release_key(self, key: str, state: str, cause: Optional[str], reason: None, report: bool):
+        ...     def release_key(self, key: str, state: str, cause: str | None, reason: None, report: bool):
         ...         pass
 
         >>> plugin = MyPlugin(1, 2, 3)
diff --git a/distributed/comm/core.py b/distributed/comm/core.py
index fb4a3eb1ff5..def563dfd2b 100644
--- a/distributed/comm/core.py
+++ b/distributed/comm/core.py
@@ -61,7 +61,7 @@ async def read(self, deserializers=None):
 
         Parameters
         ----------
-        deserializers : Optional[Dict[str, Tuple[Callable, Callable, bool]]]
+        deserializers : dict[str, tuple[Callable, Callable, bool]] | None
             An optional dict appropriate for distributed.protocol.deserialize.
             See :ref:`serialization` for more.
         """
@@ -76,7 +76,7 @@ async def write(self, msg, serializers=None, on_error=None):
         Parameters
         ----------
         msg
-        on_error : Optional[str]
+        on_error : str | None
             The behavior when serialization fails. See
             ``distributed.protocol.core.dumps`` for valid values.
         """
diff --git a/distributed/core.py b/distributed/core.py
index 9c5c49dd3f7..5a77ef562ab 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -1121,7 +1121,7 @@ async def close(self):
                 comms.update(d.popitem()[1])
 
             await asyncio.gather(
-                *[comm.close() for comm in comms], return_exceptions=True
+                *(comm.close() for comm in comms), return_exceptions=True
             )
 
             for _ in comms:
diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index 8501257e9fb..f6f92a1887e 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -708,7 +708,7 @@ def name(address):
                     return str(ws.name)
                 return address
 
-            x, y, value = zip(*[(name(a), name(b), c) for (a, b), c in bw.items()])
+            x, y, value = zip(*((name(a), name(b), c) for (a, b), c in bw.items()))
 
             self.color_map.high = max(value)
 
@@ -2189,7 +2189,7 @@ def __init__(self, scheduler, **kwargs):
             source=self.nodes_source,
         )
 
-        ####plot tg log
+        # plot tg log
         self.root.image_url(
             url="url_logo",
             x="x_logo",
diff --git a/distributed/dashboard/components/worker.py b/distributed/dashboard/components/worker.py
index 7263d2b6fa2..f77e0f40997 100644
--- a/distributed/dashboard/components/worker.py
+++ b/distributed/dashboard/components/worker.py
@@ -452,10 +452,10 @@ def __init__(self, server, sizing_mode="stretch_both", **kwargs):
             self.root = column(figures, sizing_mode=sizing_mode)
         else:
             self.root = column(
-                *[
+                *(
                     row(*pair, sizing_mode=sizing_mode)
                     for pair in partition_all(2, figures)
-                ],
+                ),
                 sizing_mode=sizing_mode,
             )
 
diff --git a/distributed/dashboard/tests/test_scheduler_bokeh.py b/distributed/dashboard/tests/test_scheduler_bokeh.py
index 7127170df30..f3a551d42cd 100644
--- a/distributed/dashboard/tests/test_scheduler_bokeh.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh.py
@@ -529,26 +529,26 @@ async def test_SystemTimeseries(c, s, a, b):
 
     assert all(len(v) == 1 for v in systs.source.data.values())
     assert systs.source.data["read_bytes"][0] == sum(
-        [ws.metrics["read_bytes"] for ws in workers]
+        ws.metrics["read_bytes"] for ws in workers
     ) / len(workers)
     assert systs.source.data["write_bytes"][0] == sum(
-        [ws.metrics["write_bytes"] for ws in workers]
+        ws.metrics["write_bytes"] for ws in workers
     ) / len(workers)
     assert systs.source.data["cpu"][0] == sum(
-        [ws.metrics["cpu"] for ws in workers]
+        ws.metrics["cpu"] for ws in workers
     ) / len(workers)
     assert systs.source.data["memory"][0] == sum(
-        [ws.metrics["memory"] for ws in workers]
+        ws.metrics["memory"] for ws in workers
     ) / len(workers)
     assert systs.source.data["read_bytes_disk"][0] == sum(
-        [ws.metrics["read_bytes_disk"] for ws in workers]
+        ws.metrics["read_bytes_disk"] for ws in workers
     ) / len(workers)
     assert systs.source.data["write_bytes_disk"][0] == sum(
-        [ws.metrics["write_bytes_disk"] for ws in workers]
+        ws.metrics["write_bytes_disk"] for ws in workers
     ) / len(workers)
     assert (
         systs.source.data["time"][0]
-        == sum([ws.metrics["time"] for ws in workers]) / len(workers) * 1000
+        == sum(ws.metrics["time"] for ws in workers) / len(workers) * 1000
     )
 
     # Update worker system monitors and send updated metrics to the scheduler
@@ -559,26 +559,26 @@ async def test_SystemTimeseries(c, s, a, b):
 
     assert all(len(v) == 2 for v in systs.source.data.values())
     assert systs.source.data["read_bytes"][1] == sum(
-        [ws.metrics["read_bytes"] for ws in workers]
+        ws.metrics["read_bytes"] for ws in workers
     ) / len(workers)
     assert systs.source.data["write_bytes"][1] == sum(
-        [ws.metrics["write_bytes"] for ws in workers]
+        ws.metrics["write_bytes"] for ws in workers
     ) / len(workers)
     assert systs.source.data["cpu"][1] == sum(
-        [ws.metrics["cpu"] for ws in workers]
+        ws.metrics["cpu"] for ws in workers
     ) / len(workers)
     assert systs.source.data["memory"][1] == sum(
-        [ws.metrics["memory"] for ws in workers]
+        ws.metrics["memory"] for ws in workers
     ) / len(workers)
     assert systs.source.data["read_bytes_disk"][1] == sum(
-        [ws.metrics["read_bytes_disk"] for ws in workers]
+        ws.metrics["read_bytes_disk"] for ws in workers
     ) / len(workers)
     assert systs.source.data["write_bytes_disk"][1] == sum(
-        [ws.metrics["write_bytes_disk"] for ws in workers]
+        ws.metrics["write_bytes_disk"] for ws in workers
     ) / len(workers)
     assert (
         systs.source.data["time"][1]
-        == sum([ws.metrics["time"] for ws in workers]) / len(workers) * 1000
+        == sum(ws.metrics["time"] for ws in workers) / len(workers) * 1000
     )
 
 
@@ -770,7 +770,7 @@ async def test_TaskGroupGraph_arrows(c, s, a, b):
     while s.task_groups:
         await asyncio.sleep(0.01)
 
-    tgg.update()  ###for some reason after deleting the futures the tgg.node_source.data.values are not clear.
+    tgg.update()  # for some reason after deleting the futures the tgg.node_source.data.values are not clear.
     assert not any(tgg.nodes_source.data.values())
     assert not any(tgg.arrows_source.data.values())
 
diff --git a/distributed/deploy/tests/test_spec_cluster.py b/distributed/deploy/tests/test_spec_cluster.py
index c51748c2549..9c967449f5a 100644
--- a/distributed/deploy/tests/test_spec_cluster.py
+++ b/distributed/deploy/tests/test_spec_cluster.py
@@ -405,7 +405,7 @@ async def start(self):
         await asyncio.gather(*self.workers)
 
     async def close(self):
-        await asyncio.gather(*[w.close() for w in self.workers])
+        await asyncio.gather(*(w.close() for w in self.workers))
 
 
 @gen_test()
@@ -464,9 +464,9 @@ async def test_MultiWorker():
 async def test_run_spec(c, s):
     workers = await run_spec(worker_spec, s.address)
     await c.wait_for_workers(len(worker_spec))
-    await asyncio.gather(*[w.close() for w in workers.values()])
+    await asyncio.gather(*(w.close() for w in workers.values()))
     assert not s.workers
-    await asyncio.gather(*[w.finished() for w in workers.values()])
+    await asyncio.gather(*(w.finished() for w in workers.values()))
 
 
 @gen_test()
diff --git a/distributed/diagnostics/tests/test_progress.py b/distributed/diagnostics/tests/test_progress.py
index 8495e9a26aa..fdeaf133d6e 100644
--- a/distributed/diagnostics/tests/test_progress.py
+++ b/distributed/diagnostics/tests/test_progress.py
@@ -90,7 +90,7 @@ def transition(self, key, start, finish, **kwargs):
 
 def check_bar_completed(capsys, width=40):
     out, err = capsys.readouterr()
-    bar, percent, time = [i.strip() for i in out.split("\r")[-1].split("|")]
+    bar, percent, time = (i.strip() for i in out.split("\r")[-1].split("|"))
     assert bar == "[" + "#" * width + "]"
     assert percent == "100% Completed"
 
diff --git a/distributed/diagnostics/tests/test_progressbar.py b/distributed/diagnostics/tests/test_progressbar.py
index ef14e20a1f3..2dedcea7f86 100644
--- a/distributed/diagnostics/tests/test_progressbar.py
+++ b/distributed/diagnostics/tests/test_progressbar.py
@@ -48,7 +48,7 @@ async def test_TextProgressBar_empty(s, a, b, capsys):
 def check_bar_completed(capsys, width=40):
     out, err = capsys.readouterr()
     # trailing newline so grab next to last line for final state of bar
-    bar, percent, time = [i.strip() for i in out.split("\r")[-2].split("|")]
+    bar, percent, time = (i.strip() for i in out.split("\r")[-2].split("|"))
     assert bar == "[" + "#" * width + "]"
     assert percent == "100% Completed"
 
diff --git a/distributed/nanny.py b/distributed/nanny.py
index fef2d6b41ef..79c61e88348 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -592,7 +592,7 @@ async def close(self, comm=None, timeout=5, report=None):
             if hasattr(plugin, "teardown")
         ]
 
-        await asyncio.gather(*[td for td in teardowns if isawaitable(td)])
+        await asyncio.gather(*(td for td in teardowns if isawaitable(td)))
 
         self.stop()
         try:
diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index 877cd22c824..af2d86b8edd 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -357,7 +357,7 @@ def deserialize(header, frames, deserializers=None):
     ----------
     header : dict
     frames : list of bytes
-    deserializers : Optional[Dict[str, Tuple[Callable, Callable, bool]]]
+    deserializers : dict[str, tuple[Callable, Callable, bool]] | None
         An optional dict mapping a name to a (de)serializer.
         See `dask_serialize` and `dask_deserialize` for more.
 
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 706e85c3b24..a59b7249449 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -18,7 +18,6 @@
 from datetime import timedelta
 from functools import partial
 from numbers import Number
-from typing import Optional
 
 import psutil
 import sortedcontainers
@@ -6080,7 +6079,7 @@ async def rebalance(
 
     def _rebalance_find_msgs(
         self: SchedulerState,
-        keys: "Optional[Set[Hashable]]",
+        keys: "Set[Hashable] | None",
         workers: "Iterable[WorkerState]",
     ) -> "list[tuple[WorkerState, WorkerState, TaskState]]":
         """Identify workers that need to lose keys and those that can receive them,
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index c1a4a6aecb2..1373499d18a 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -1412,7 +1412,7 @@ async def test_scatter_direct_broadcast(c, s, a, b):
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 4)
 async def test_scatter_direct_balanced(c, s, *workers):
     futures = await c.scatter([1, 2, 3], direct=True)
-    assert sorted([len(w.data) for w in workers]) == [0, 1, 1, 1]
+    assert sorted(len(w.data) for w in workers) == [0, 1, 1, 1]
 
 
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 4)
@@ -2067,7 +2067,7 @@ async def test_forget_complex(e, s, A, B):
 @gen_cluster(client=True)
 async def test_forget_in_flight(e, s, A, B):
     delayed2 = partial(delayed, pure=True)
-    a, b, c, d = [delayed2(slowinc)(i) for i in range(4)]
+    a, b, c, d = (delayed2(slowinc)(i) for i in range(4))
     ab = delayed2(slowadd)(a, b, dask_key_name="ab")
     cd = delayed2(slowadd)(c, d, dask_key_name="cd")
     ac = delayed2(slowadd)(a, c, dask_key_name="ac")
@@ -2932,7 +2932,7 @@ def test_rebalance_sync():
     # can't use the 'c' fixture because we need workers to run in a separate process
     with Client(n_workers=2, memory_limit="1 GiB", dashboard_address=":0") as c:
         s = c.cluster.scheduler
-        a, b = [ws.address for ws in s.workers.values()]
+        a, b = (ws.address for ws in s.workers.values())
         futures = c.map(lambda _: "x" * (2 ** 29 // 10), range(10), workers=[a])
         wait(futures)
         # Wait for heartbeat
@@ -5207,7 +5207,7 @@ async def test_bytes_keys(c, s, a, b):
 
 @gen_cluster(client=True)
 async def test_unicode_ascii_keys(c, s, a, b):
-    uni_type = type("")
+    uni_type = str
     key = "inc-123"
     future = c.submit(inc, 1, key=key)
     result = await future
@@ -5219,7 +5219,7 @@ async def test_unicode_ascii_keys(c, s, a, b):
 
 @gen_cluster(client=True)
 async def test_unicode_keys(c, s, a, b):
-    uni_type = type("")
+    uni_type = str
     key = "inc-123\u03bc"
     future = c.submit(inc, 1, key=key)
     result = await future
@@ -5799,7 +5799,7 @@ async def test_tuple_keys(c, s, a, b):
 
 @gen_cluster(client=True)
 async def test_multiple_scatter(c, s, a, b):
-    futures = await asyncio.gather(*[c.scatter(1, direct=True) for _ in range(5)])
+    futures = await asyncio.gather(*(c.scatter(1, direct=True) for _ in range(5)))
 
     x = await futures[0]
     x = await futures[0]
@@ -6059,10 +6059,10 @@ async def test_profile_server(c, s, a, b):
                 c.run(slowinc, 1, delay=0.5), c.run_on_scheduler(slowdec, 1, delay=0.5)
             )
 
-            p = await c.profile(server=True)  #  All worker servers
+            p = await c.profile(server=True)  # All worker servers
             assert "slowinc" in str(p)
 
-            p = await c.profile(scheduler=True)  #  Scheduler
+            p = await c.profile(scheduler=True)  # Scheduler
             assert "slowdec" in str(p)
         except AssertionError:
             if i == 4:
diff --git a/distributed/tests/test_config.py b/distributed/tests/test_config.py
index cd2c5418d7c..06acc3b8183 100644
--- a/distributed/tests/test_config.py
+++ b/distributed/tests/test_config.py
@@ -203,11 +203,11 @@ def test_logging_extended():
             "loggers": {
                 "distributed.foo": {
                     "level": "INFO",
-                    #'handlers': ['console'],
+                    # 'handlers': ['console'],
                 },
                 "distributed.foo.bar": {
                     "level": "ERROR",
-                    #'handlers': ['console'],
+                    # 'handlers': ['console'],
                 },
             },
             "root": {"level": "WARNING", "handlers": ["console"]},
diff --git a/distributed/tests/test_core.py b/distributed/tests/test_core.py
index 78e5b7cc5a6..4e079f5345f 100644
--- a/distributed/tests/test_core.py
+++ b/distributed/tests/test_core.py
@@ -576,12 +576,12 @@ async def ping(comm, delay=0.1):
 
     # Reuse connections
     await asyncio.gather(
-        *[rpc(ip="127.0.0.1", port=s.port).ping() for s in servers[:5]]
+        *(rpc(ip="127.0.0.1", port=s.port).ping() for s in servers[:5])
     )
-    await asyncio.gather(*[rpc(s.address).ping() for s in servers[:5]])
-    await asyncio.gather(*[rpc("127.0.0.1:%d" % s.port).ping() for s in servers[:5]])
+    await asyncio.gather(*(rpc(s.address).ping() for s in servers[:5]))
+    await asyncio.gather(*(rpc("127.0.0.1:%d" % s.port).ping() for s in servers[:5]))
     await asyncio.gather(
-        *[rpc(ip="127.0.0.1", port=s.port).ping() for s in servers[:5]]
+        *(rpc(ip="127.0.0.1", port=s.port).ping() for s in servers[:5])
     )
     assert sum(map(len, rpc.available.values())) == 5
     assert sum(map(len, rpc.occupied.values())) == 0
@@ -590,14 +590,14 @@ async def ping(comm, delay=0.1):
 
     # Clear out connections to make room for more
     await asyncio.gather(
-        *[rpc(ip="127.0.0.1", port=s.port).ping() for s in servers[5:]]
+        *(rpc(ip="127.0.0.1", port=s.port).ping() for s in servers[5:])
     )
     assert rpc.active == 0
     assert rpc.open == 5
 
     s = servers[0]
     await asyncio.gather(
-        *[rpc(ip="127.0.0.1", port=s.port).ping(delay=0.1) for i in range(3)]
+        *(rpc(ip="127.0.0.1", port=s.port).ping(delay=0.1) for i in range(3))
     )
     assert len(rpc.available["tcp://127.0.0.1:%d" % s.port]) == 3
 
@@ -680,7 +680,7 @@ async def do_ping(pool, port):
 
     pool = await ConnectionPool(limit=limit)
 
-    await asyncio.gather(*[do_ping(pool, s.port) for s in servers])
+    await asyncio.gather(*(do_ping(pool, s.port) for s in servers))
 
 
 @pytest.mark.asyncio
@@ -702,9 +702,9 @@ async def ping(comm, delay=0.01):
 
     rpc = await ConnectionPool(limit=5, connection_args=connection_args)
 
-    await asyncio.gather(*[rpc(s.address).ping() for s in servers[:5]])
-    await asyncio.gather(*[rpc(s.address).ping() for s in servers[::2]])
-    await asyncio.gather(*[rpc(s.address).ping() for s in servers])
+    await asyncio.gather(*(rpc(s.address).ping() for s in servers[:5]))
+    await asyncio.gather(*(rpc(s.address).ping() for s in servers[::2]))
+    await asyncio.gather(*(rpc(s.address).ping() for s in servers))
     assert rpc.active == 0
 
     await rpc.close()
@@ -722,8 +722,8 @@ async def ping(comm, delay=0.01):
 
     rpc = await ConnectionPool(limit=10)
     serv = servers.pop()
-    await asyncio.gather(*[rpc(s.address).ping() for s in servers])
-    await asyncio.gather(*[rpc(serv.address).ping() for i in range(3)])
+    await asyncio.gather(*(rpc(s.address).ping() for s in servers))
+    await asyncio.gather(*(rpc(serv.address).ping() for i in range(3)))
     await rpc.connect(serv.address)
     assert sum(map(len, rpc.available.values())) == 6
     assert sum(map(len, rpc.occupied.values())) == 1
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 23f59431cf0..4d841bc4939 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -196,7 +196,7 @@ def random(**kwargs):
         secondary_worker_key_fractions = []
         for i, keys in enumerate(x.__dask_keys__()):
             # Iterate along rows of the array.
-            keys = set(stringify(k) for k in keys)
+            keys = {stringify(k) for k in keys}
 
             # No more than 2 workers should have any keys
             assert sum(any(k in w.data for k in keys) for w in workers) <= 2
@@ -925,7 +925,7 @@ async def test_file_descriptors(c, s):
     num_fds_1 = proc.num_fds()
 
     N = 20
-    nannies = await asyncio.gather(*[Nanny(s.address, loop=s.loop) for _ in range(N)])
+    nannies = await asyncio.gather(*(Nanny(s.address, loop=s.loop) for _ in range(N)))
 
     while len(s.nthreads) < N:
         await asyncio.sleep(0.1)
@@ -955,7 +955,7 @@ async def test_file_descriptors(c, s):
     num_fds_6 = proc.num_fds()
     assert num_fds_6 < num_fds_5 + N
 
-    await asyncio.gather(*[n.close() for n in nannies])
+    await asyncio.gather(*(n.close() for n in nannies))
     await c.close()
 
     assert not s.rpc.open
@@ -1996,14 +1996,7 @@ async def test_task_groups(c, s, a, b):
     assert tg.nbytes_total == tp.nbytes_total
     # It should map down to individual tasks
     assert tg.nbytes_total == sum(
-        [ts.get_nbytes() for ts in s.tasks.values() if ts.group is tg]
-    )
-    in_memory_ts = sum(
-        [
-            ts.get_nbytes()
-            for ts in s.tasks.values()
-            if ts.group is tg and ts.state == "memory"
-        ]
+        ts.get_nbytes() for ts in s.tasks.values() if ts.group is tg
     )
     tg = s.task_groups[y.name]
     assert tg.states["memory"] == 5
@@ -3181,7 +3174,7 @@ async def test_worker_heartbeat_after_cancel(c, s, *workers):
     await c.cancel(futs)
 
     while any(w.tasks for w in workers):
-        await asyncio.gather(*[w.heartbeat() for w in workers])
+        await asyncio.gather(*(w.heartbeat() for w in workers))
 
 
 @gen_cluster(client=True, nthreads=[("", 1)])
diff --git a/distributed/tests/test_steal.py b/distributed/tests/test_steal.py
index fea42cb1393..d9acf2ce28b 100644
--- a/distributed/tests/test_steal.py
+++ b/distributed/tests/test_steal.py
@@ -528,7 +528,7 @@ async def assert_balanced(inp, expected, c, s, *workers):
             await asyncio.sleep(0.001)
 
         result = [
-            sorted([int(key_split(k)) for k in s.processing[w.address]], reverse=True)
+            sorted((int(key_split(k)) for k in s.processing[w.address]), reverse=True)
             for w in workers
         ]
 
@@ -661,7 +661,7 @@ async def test_steal_twice(c, s, a, b):
         await asyncio.sleep(0.01)
 
     # Army of new workers arrives to help
-    workers = await asyncio.gather(*[Worker(s.address, loop=s.loop) for _ in range(20)])
+    workers = await asyncio.gather(*(Worker(s.address, loop=s.loop) for _ in range(20)))
 
     await wait(futures)
 
@@ -678,7 +678,7 @@ async def test_steal_twice(c, s, a, b):
     assert b.in_flight_tasks == 0
 
     await c._close()
-    await asyncio.gather(*[w.close() for w in workers])
+    await asyncio.gather(*(w.close() for w in workers))
 
 
 @gen_cluster(client=True)
diff --git a/distributed/utils.py b/distributed/utils.py
index b5683aac6b4..031dd9c66f5 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -1303,7 +1303,7 @@ def convert_value(v):
         return out
 
     return sum(
-        [["--" + k.replace("_", "-"), convert_value(v)] for k, v in d.items()], []
+        (["--" + k.replace("_", "-"), convert_value(v)] for k, v in d.items()), []
     )
 
 
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index a9c5496efce..615a353b083 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -763,7 +763,7 @@ async def do_disconnect():
 
 
 async def disconnect_all(addresses, timeout=3, rpc_kwargs=None):
-    await asyncio.gather(*[disconnect(addr, timeout, rpc_kwargs) for addr in addresses])
+    await asyncio.gather(*(disconnect(addr, timeout, rpc_kwargs) for addr in addresses))
 
 
 def gen_test(timeout=_TEST_TIMEOUT):
@@ -831,7 +831,7 @@ async def start_cluster(
     ):
         await asyncio.sleep(0.01)
         if time() > start + 30:
-            await asyncio.gather(*[w.close(timeout=1) for w in workers])
+            await asyncio.gather(*(w.close(timeout=1) for w in workers))
             await s.close(fast=True)
             raise TimeoutError("Cluster creation timeout")
     return s, workers
@@ -844,7 +844,7 @@ async def end_worker(w):
         with suppress(TimeoutError, CommClosedError, EnvironmentError):
             await w.close(report=False)
 
-    await asyncio.gather(*[end_worker(w) for w in workers])
+    await asyncio.gather(*(end_worker(w) for w in workers))
     await s.close()  # wait until scheduler stops completely
     s.stop()
 
diff --git a/distributed/worker.py b/distributed/worker.py
index 7066771a41a..1b2fe7da65c 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -14,12 +14,12 @@
 import warnings
 import weakref
 from collections import defaultdict, deque, namedtuple
-from collections.abc import MutableMapping
+from collections.abc import Hashable, Iterable, MutableMapping
 from contextlib import suppress
 from datetime import timedelta
 from inspect import isawaitable
 from pickle import PicklingError
-from typing import TYPE_CHECKING, Dict, Hashable, Iterable, Optional
+from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
     from .client import Client
@@ -652,7 +652,7 @@ def __init__(
         self.reconnect = reconnect
 
         # Common executors always available
-        self.executors: Dict[str, concurrent.futures.Executor] = {
+        self.executors: dict[str, concurrent.futures.Executor] = {
             "offload": utils._offload_executor,
             "actor": ThreadPoolExecutor(1, thread_name_prefix="Dask-Actor-Threads"),
         }
@@ -970,10 +970,10 @@ async def _register_with_scheduler(self):
             raise ValueError(f"Unexpected response from register: {response!r}")
         else:
             await asyncio.gather(
-                *[
+                *(
                     self.plugin_add(name=name, plugin=plugin)
                     for name, plugin in response["worker-plugins"].items()
-                ]
+                )
             )
 
             logger.info("        Registered to: %26s", self.scheduler.address)
@@ -1235,7 +1235,7 @@ async def start(self):
         setproctitle("dask-worker [%s]" % self.address)
 
         await asyncio.gather(
-            *[self.plugin_add(plugin=plugin) for plugin in self._pending_plugins]
+            *(self.plugin_add(plugin=plugin) for plugin in self._pending_plugins)
         )
         self._pending_plugins = ()
 
@@ -1282,7 +1282,7 @@ async def close(
                 if hasattr(plugin, "teardown")
             ]
 
-            await asyncio.gather(*[td for td in teardowns if isawaitable(td)])
+            await asyncio.gather(*(td for td in teardowns if isawaitable(td)))
 
             for pc in self.periodic_callbacks.values():
                 pc.stop()
@@ -2692,8 +2692,8 @@ def steal_request(self, key):
     def release_key(
         self,
         key: Hashable,
-        cause: Optional[TaskState] = None,
-        reason: Optional[str] = None,
+        cause: TaskState | None = None,
+        reason: str | None = None,
         report: bool = True,
     ):
         try:
@@ -2828,7 +2828,7 @@ async def actor_execute(
         actor=None,
         function=None,
         args=(),
-        kwargs: Optional[dict] = None,
+        kwargs: dict | None = None,
     ):
         kwargs = kwargs or {}
         separate_thread = kwargs.pop("separate_thread", True)
@@ -3362,7 +3362,7 @@ def validate_task_flight(self, ts):
     def validate_task_fetch(self, ts):
         assert ts.runspec is None
         assert ts.key not in self.data
-        assert self.address not in ts.who_has  #!!!!!!!!
+        assert self.address not in ts.who_has  # !!!!!!!!
         # FIXME This is currently not an invariant since upon comm failure we
         # remove the erroneous worker from all who_has and correct the state
         # upon the next ensure_communicate
@@ -3688,7 +3688,7 @@ class Reschedule(Exception):
     """
 
 
-def parse_memory_limit(memory_limit, nthreads, total_cores=CPU_COUNT) -> Optional[int]:
+def parse_memory_limit(memory_limit, nthreads, total_cores=CPU_COUNT) -> int | None:
     if memory_limit is None:
         return None
 
@@ -4038,7 +4038,7 @@ def __repr__(self):
     return msg
 
 
-def convert_args_to_str(args, max_len: Optional[int] = None) -> str:
+def convert_args_to_str(args, max_len: int | None = None) -> str:
     """Convert args to a string, allowing for some arguments to raise
     exceptions during conversion and ignoring them.
     """
@@ -4057,7 +4057,7 @@ def convert_args_to_str(args, max_len: Optional[int] = None) -> str:
         return "({})".format(", ".join(strs))
 
 
-def convert_kwargs_to_str(kwargs: dict, max_len: Optional[int] = None) -> str:
+def convert_kwargs_to_str(kwargs: dict, max_len: int | None = None) -> str:
     """Convert kwargs to a string, allowing for some arguments to raise
     exceptions during conversion and ignoring them.
     """
diff --git a/docs/source/conf.py b/docs/source/conf.py
index 69d7e78de0e..12bcfa7cffc 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -217,13 +217,13 @@
 
 latex_elements = {
     # The paper size ('letterpaper' or 'a4paper').
-    #'papersize': 'letterpaper',
+    # 'papersize': 'letterpaper',
     # The font size ('10pt', '11pt' or '12pt').
-    #'pointsize': '10pt',
+    # 'pointsize': '10pt',
     # Additional stuff for the LaTeX preamble.
-    #'preamble': '',
+    # 'preamble': '',
     # Latex figure (float) alignment
-    #'figure_align': 'htbp',
+    # 'figure_align': 'htbp',
 }
 
 # Grouping the document tree into LaTeX files. List of tuples
diff --git a/docs/source/develop.rst b/docs/source/develop.rst
index 857008f1cbf..f0f534386bb 100644
--- a/docs/source/develop.rst
+++ b/docs/source/develop.rst
@@ -168,3 +168,10 @@ fixture tests test basic interface and resilience.
 
 You should avoid ``popen`` style tests unless absolutely necessary, such as if
 you need to test the command line interface.
+
+Linting
+-------
+distributed uses several code linters (flake8, black, isort, pyupgrade), which are
+enforced by CI. Developers should run them locally before they submit a PR, through the
+single command ``pre-commit run --all-files``. This makes sure that linter versions and
+options are aligned for all developers.
diff --git a/setup.cfg b/setup.cfg
index ba49f43c519..b86f67f6a45 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -5,15 +5,13 @@
 
 # Aligned with black https://github.com/psf/black/blob/main/.flake8
 extend-ignore = E203, E266, E501
-# Note: there cannot be spaces after comma's here
+# Note: there cannot be spaces after commas here
 exclude = __init__.py,versioneer.py,distributed/_concurrent_futures_thread.py
 ignore =
-    E26,        # Comments
     E4,         # Import formatting
     E731,       # Assigning lambda expression
     W503,       # line break before binary operator
 
-
 per-file-ignores =
     **/tests/*:
         # local variable is assigned to but never used
@@ -21,8 +19,7 @@ per-file-ignores =
         # Ambiguous variable name
         E741,
 
-
-max-line-length = 120
+max-line-length = 88
 
 [isort]
 sections = FUTURE,STDLIB,THIRDPARTY,DISTRIBUTED,FIRSTPARTY,LOCALFOLDER
diff --git a/versioneer.py b/versioneer.py
index bf23b9a50d4..92e184c31e5 100644
--- a/versioneer.py
+++ b/versioneer.py
@@ -1,4 +1,3 @@
-
 # Version: 0.17
 
 """The Versioneer - like a rocketeer, but for versions.
@@ -276,7 +275,6 @@
 
 """
 
-from __future__ import print_function
 
 try:
     import configparser
@@ -342,7 +340,7 @@ def get_config_from_root(root):
     # the top of versioneer.py for instructions on writing your setup.cfg .
     setup_cfg = os.path.join(root, "setup.cfg")
     parser = configparser.SafeConfigParser()
-    with open(setup_cfg, "r") as f:
+    with open(setup_cfg) as f:
         parser.readfp(f)
     VCS = parser.get("versioneer", "VCS")  # mandatory
 
@@ -396,7 +394,7 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False,
                                  stderr=(subprocess.PIPE if hide_stderr
                                          else None))
             break
-        except EnvironmentError:
+        except OSError:
             e = sys.exc_info()[1]
             if e.errno == errno.ENOENT:
                 continue
@@ -406,7 +404,7 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False,
             return None, None
     else:
         if verbose:
-            print("unable to find command, tried %s" % (commands,))
+            print(f"unable to find command, tried {commands}")
         return None, None
     stdout = p.communicate()[0].strip()
     if sys.version_info[0] >= 3:
@@ -417,7 +415,7 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False,
             print("stdout was %s" % stdout)
         return None, p.returncode
     return stdout, p.returncode
-LONG_VERSION_PY['git'] = '''
+LONG_VERSION_PY['git'] = r'''
 # This file helps to compute a version number in source trees obtained from
 # git-archive tarball (such as those provided by githubs download-from-tag
 # feature). Distribution tarballs (built by setup.py sdist) and build
@@ -949,7 +947,7 @@ def git_get_keywords(versionfile_abs):
     # _version.py.
     keywords = {}
     try:
-        f = open(versionfile_abs, "r")
+        f = open(versionfile_abs)
         for line in f.readlines():
             if line.strip().startswith("git_refnames ="):
                 mo = re.search(r'=\s*"(.*)"', line)
@@ -964,7 +962,7 @@ def git_get_keywords(versionfile_abs):
                 if mo:
                     keywords["date"] = mo.group(1)
         f.close()
-    except EnvironmentError:
+    except OSError:
         pass
     return keywords
 
@@ -988,11 +986,11 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose):
         if verbose:
             print("keywords are unexpanded, not using")
         raise NotThisMethod("unexpanded keywords, not a git-archive tarball")
-    refs = set([r.strip() for r in refnames.strip("()").split(",")])
+    refs = {r.strip() for r in refnames.strip("()").split(",")}
     # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of
     # just "foo-1.0". If we see a "tag: " prefix, prefer those.
     TAG = "tag: "
-    tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)])
+    tags = {r[len(TAG):] for r in refs if r.startswith(TAG)}
     if not tags:
         # Either we're using git < 1.8.3, or there really are no tags. We use
         # a heuristic: assume all version tags have a digit. The old git %d
@@ -1001,7 +999,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose):
         # between branches and tags. By ignoring refnames without digits, we
         # filter out many common branch names like "release" and
         # "stabilization", as well as "HEAD" and "main".
-        tags = set([r for r in refs if re.search(r'\d', r)])
+        tags = {r for r in refs if re.search(r'\d', r)}
         if verbose:
             print("discarding '%s', no digits" % ",".join(refs - tags))
     if verbose:
@@ -1138,13 +1136,13 @@ def do_vcs_install(manifest_in, versionfile_source, ipy):
     files.append(versioneer_file)
     present = False
     try:
-        f = open(".gitattributes", "r")
+        f = open(".gitattributes")
         for line in f.readlines():
             if line.strip().startswith(versionfile_source):
                 if "export-subst" in line.strip().split()[1:]:
                     present = True
         f.close()
-    except EnvironmentError:
+    except OSError:
         pass
     if not present:
         f = open(".gitattributes", "a+")
@@ -1201,7 +1199,7 @@ def versions_from_file(filename):
     try:
         with open(filename) as f:
             contents = f.read()
-    except EnvironmentError:
+    except OSError:
         raise NotThisMethod("unable to read _version.py")
     mo = re.search(r"version_json = '''\n(.*)'''  # END VERSION_JSON",
                    contents, re.M | re.S)
@@ -1221,7 +1219,7 @@ def write_to_version_file(filename, versions):
     with open(filename, "w") as f:
         f.write(SHORT_VERSION_PY % contents)
 
-    print("set %s to '%s'" % (filename, versions["version"]))
+    print("set {} to '{}'".format(filename, versions["version"]))
 
 
 def plus_or_dot(pieces):
@@ -1440,7 +1438,7 @@ def get_versions(verbose=False):
     try:
         ver = versions_from_file(versionfile_abs)
         if verbose:
-            print("got version from file %s %s" % (versionfile_abs, ver))
+            print(f"got version from file {versionfile_abs} {ver}")
         return ver
     except NotThisMethod:
         pass
@@ -1698,8 +1696,7 @@ def do_setup():
     root = get_root()
     try:
         cfg = get_config_from_root(root)
-    except (EnvironmentError, configparser.NoSectionError,
-            configparser.NoOptionError) as e:
+    except (OSError, configparser.NoSectionError, configparser.NoOptionError) as e:
         if isinstance(e, (EnvironmentError, configparser.NoSectionError)):
             print("Adding sample versioneer config to setup.cfg",
                   file=sys.stderr)
@@ -1722,9 +1719,9 @@ def do_setup():
                        "__init__.py")
     if os.path.exists(ipy):
         try:
-            with open(ipy, "r") as f:
+            with open(ipy) as f:
                 old = f.read()
-        except EnvironmentError:
+        except OSError:
             old = ""
         if INIT_PY_SNIPPET not in old:
             print(" appending to %s" % ipy)
@@ -1743,12 +1740,12 @@ def do_setup():
     manifest_in = os.path.join(root, "MANIFEST.in")
     simple_includes = set()
     try:
-        with open(manifest_in, "r") as f:
+        with open(manifest_in) as f:
             for line in f:
                 if line.startswith("include "):
                     for include in line.split()[1:]:
                         simple_includes.add(include)
-    except EnvironmentError:
+    except OSError:
         pass
     # That doesn't cover everything MANIFEST.in can do
     # (http://docs.python.org/2/distutils/sourcedist.html#commands), so
@@ -1780,7 +1777,7 @@ def scan_setup_py():
     found = set()
     setters = False
     errors = 0
-    with open("setup.py", "r") as f:
+    with open("setup.py") as f:
         for line in f.readlines():
             if "import versioneer" in line:
                 found.add("import")

From 23c3b4b204b222390efd5a00ccad98aec91ff56a Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Wed, 22 Sep 2021 15:19:59 +0100
Subject: [PATCH 1497/1550] Enhance AMM docstrings (#5340)

---
 distributed/active_memory_manager.py | 23 +++++++++++++++++------
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/distributed/active_memory_manager.py b/distributed/active_memory_manager.py
index c8c29f3aaf1..098773c224f 100644
--- a/distributed/active_memory_manager.py
+++ b/distributed/active_memory_manager.py
@@ -189,9 +189,14 @@ def _find_recipient(
         pending_repl: set[WorkerState],
     ) -> WorkerState | None:
         """Choose a worker to acquire a new replica of an in-memory task among a set of
-        candidates. If candidates is None, default to all workers in the cluster that do
-        not hold a replica yet. The worker with the lowest memory usage (downstream of
-        pending replications and drops) will be returned.
+        candidates. If candidates is None, default to all workers in the cluster.
+        Regardless, workers that either already hold a replica or are scheduled to
+        receive one at the end of this AMM iteration are not considered.
+
+        Returns
+        -------
+        The worker with the lowest memory usage (downstream of pending replications and
+        drops), or None if no eligible candidates are available.
         """
         if ts.state != "memory":
             return None
@@ -210,9 +215,15 @@ def _find_dropper(
         pending_drop: set[WorkerState],
     ) -> WorkerState | None:
         """Choose a worker to drop its replica of an in-memory task among a set of
-        candidates. If candidates is None, default to all workers in the cluster that
-        hold a replica. The worker with the highest memory usage (downstream of pending
-        replications and drops) will be returned.
+        candidates. If candidates is None, default to all workers in the cluster.
+        Regardless, workers that either do not hold a replica or are already scheduled
+        to drop theirs at the end of this AMM iteration are not considered.
+        This method also ensures that a key will not lose its last replica.
+
+        Returns
+        -------
+        The worker with the highest memory usage (downstream of pending replications and
+        drops), or None if no eligible candidates are available.
         """
         if len(ts.who_has) - len(pending_drop) < 2:
             return None

From e0018224fc01c9e0cefadaf1936a238f5f29c9cd Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Wed, 22 Sep 2021 15:20:26 +0100
Subject: [PATCH 1498/1550] Run multiple AMMs in parallel (#5315) (#5339)

Propaedeutic to RetireWorker AMM policy
---
 distributed/active_memory_manager.py          | 25 +++++---
 .../tests/test_active_memory_manager.py       | 59 +++++++++++++++++++
 2 files changed, 76 insertions(+), 8 deletions(-)

diff --git a/distributed/active_memory_manager.py b/distributed/active_memory_manager.py
index 098773c224f..c2bbe7ccd3c 100644
--- a/distributed/active_memory_manager.py
+++ b/distributed/active_memory_manager.py
@@ -52,23 +52,20 @@ def __init__(
         interval: float | None = None,
     ):
         self.scheduler = scheduler
+        self.policies = set()
 
         if policies is None:
+            # Initialize policies from config
             policies = set()
             for kwargs in dask.config.get(
                 "distributed.scheduler.active-memory-manager.policies"
             ):
                 kwargs = kwargs.copy()
                 cls = import_term(kwargs.pop("class"))
-                if not issubclass(cls, ActiveMemoryManagerPolicy):
-                    raise TypeError(
-                        f"{cls}: Expected ActiveMemoryManagerPolicy; got {type(cls)}"
-                    )
                 policies.add(cls(**kwargs))
 
         for policy in policies:
-            policy.manager = self
-        self.policies = policies
+            self.add_policy(policy)
 
         if register:
             scheduler.extensions["amm"] = self
@@ -92,16 +89,28 @@ def __init__(
 
     def start(self, comm=None) -> None:
         """Start executing every ``self.interval`` seconds until scheduler shutdown"""
+        if self.started:
+            return
         pc = PeriodicCallback(self.run_once, self.interval * 1000.0)
-        self.scheduler.periodic_callbacks["amm"] = pc
+        self.scheduler.periodic_callbacks[f"amm-{id(self)}"] = pc
         pc.start()
 
     def stop(self, comm=None) -> None:
         """Stop periodic execution"""
-        pc = self.scheduler.periodic_callbacks.pop("amm", None)
+        pc = self.scheduler.periodic_callbacks.pop(f"amm-{id(self)}", None)
         if pc:
             pc.stop()
 
+    @property
+    def started(self) -> bool:
+        return f"amm-{id(self)}" in self.scheduler.periodic_callbacks
+
+    def add_policy(self, policy: ActiveMemoryManagerPolicy) -> None:
+        if not isinstance(policy, ActiveMemoryManagerPolicy):
+            raise TypeError(f"Expected ActiveMemoryManagerPolicy; got {policy!r}")
+        self.policies.add(policy)
+        policy.manager = self
+
     def run_once(self, comm=None) -> None:
         """Run all policies once and asynchronously (fire and forget) enact their
         recommendations to replicate/drop keys
diff --git a/distributed/tests/test_active_memory_manager.py b/distributed/tests/test_active_memory_manager.py
index afe6c114940..c7c747b8507 100644
--- a/distributed/tests/test_active_memory_manager.py
+++ b/distributed/tests/test_active_memory_manager.py
@@ -101,6 +101,65 @@ async def test_auto_start(c, s, a, b):
     assert len(s.tasks["x"].who_has) == 1
 
 
+@gen_cluster(client=True, config=demo_config("drop", key="x"))
+async def test_add_policy(c, s, a, b):
+    p2 = DemoPolicy(action="drop", key="y", n=10, candidates=None)
+    p3 = DemoPolicy(action="drop", key="z", n=10, candidates=None)
+
+    # policies parameter can be:
+    # - None: get from config
+    # - explicit set, which can be empty
+    m1 = s.extensions["amm"]
+    m2 = ActiveMemoryManagerExtension(s, {p2}, register=False, start=False)
+    m3 = ActiveMemoryManagerExtension(s, set(), register=False, start=False)
+
+    assert len(m1.policies) == 1
+    assert len(m2.policies) == 1
+    assert len(m3.policies) == 0
+    m3.add_policy(p3)
+    assert len(m3.policies) == 1
+
+    futures = await c.scatter({"x": 1, "y": 2, "z": 3}, broadcast=True)
+    m1.run_once()
+    while len(s.tasks["x"].who_has) == 2:
+        await asyncio.sleep(0.01)
+
+    m2.run_once()
+    while len(s.tasks["y"].who_has) == 2:
+        await asyncio.sleep(0.01)
+
+    m3.run_once()
+    while len(s.tasks["z"].who_has) == 2:
+        await asyncio.sleep(0.01)
+
+
+@gen_cluster(client=True, config=demo_config("drop", key="x", start=False))
+async def test_multi_start(c, s, a, b):
+    """Multiple AMMs can be started in parallel"""
+    p2 = DemoPolicy(action="drop", key="y", n=10, candidates=None)
+    p3 = DemoPolicy(action="drop", key="z", n=10, candidates=None)
+
+    # policies parameter can be:
+    # - None: get from config
+    # - explicit set, which can be empty
+    m1 = s.extensions["amm"]
+    m2 = ActiveMemoryManagerExtension(s, {p2}, register=False, start=True, interval=0.1)
+    m3 = ActiveMemoryManagerExtension(s, {p3}, register=False, start=True, interval=0.1)
+
+    assert not m1.started
+    assert m2.started
+    assert m3.started
+
+    futures = await c.scatter({"x": 1, "y": 2, "z": 3}, broadcast=True)
+
+    # The AMMs should run within 0.1s of the broadcast.
+    # Add generous extra padding to prevent flakiness.
+    await asyncio.sleep(0.5)
+    assert len(s.tasks["x"].who_has) == 2
+    assert len(s.tasks["y"].who_has) == 1
+    assert len(s.tasks["z"].who_has) == 1
+
+
 @gen_cluster(client=True, config=NO_AMM_START)
 async def test_not_registered(c, s, a, b):
     futures = await c.scatter({"x": 1}, broadcast=True)

From bb63557e18b20deb2bf3d8a5da2b519151eb7e34 Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Wed, 22 Sep 2021 20:23:07 +0100
Subject: [PATCH 1499/1550] Fix `test_many_Progress` and others (#5329)

---
 distributed/diagnostics/tests/test_progress.py |  7 ++-----
 distributed/scheduler.py                       |  8 ++++----
 distributed/tests/test_client.py               |  5 +++--
 distributed/worker.py                          | 11 ++++++-----
 4 files changed, 15 insertions(+), 16 deletions(-)

diff --git a/distributed/diagnostics/tests/test_progress.py b/distributed/diagnostics/tests/test_progress.py
index fdeaf133d6e..08fc8b90dac 100644
--- a/distributed/diagnostics/tests/test_progress.py
+++ b/distributed/diagnostics/tests/test_progress.py
@@ -11,7 +11,6 @@
     Progress,
     SchedulerPlugin,
 )
-from distributed.metrics import time
 from distributed.scheduler import COMPILED
 from distributed.utils_test import dec, div, gen_cluster, inc, nodebug
 
@@ -29,6 +28,7 @@ def h(*args):
 
 
 @nodebug
+@pytest.mark.flaky(reruns=10, reruns_delay=5)
 @gen_cluster(client=True)
 async def test_many_Progress(c, s, a, b):
     x = c.submit(f, 1)
@@ -37,13 +37,10 @@ async def test_many_Progress(c, s, a, b):
 
     bars = [Progress(keys=[z], scheduler=s) for _ in range(10)]
     await asyncio.gather(*(bar.setup() for bar in bars))
-
     await z
 
-    start = time()
     while not all(b.status == "finished" for b in bars):
-        await asyncio.sleep(0.1)
-        assert time() < start + 5
+        await asyncio.sleep(0.01)
 
 
 @gen_cluster(client=True)
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index a59b7249449..b1f8057c634 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -4042,7 +4042,7 @@ async def close_worker(self, comm=None, worker=None, safe=None):
         logger.info("Closing worker %s", worker)
         with log_errors():
             self.log_event(worker, {"action": "close-worker"})
-            # FIXME: This does not handly nannys
+            # FIXME: This does not handle nannies
             self.worker_send(worker, {"op": "close", "report": False})
             await self.remove_worker(address=worker, safe=safe)
 
@@ -6469,9 +6469,9 @@ def workers_to_close(
 
         Parameters
         ----------
-        memory_factor : Number
+        memory_ratio : Number
             Amount of extra space we want to have for our stored data.
-            Defaults two 2, or that we want to have twice as much memory as we
+            Defaults to 2, or that we want to have twice as much memory as we
             currently have data.
         n : int
             Number of workers to close
@@ -6479,7 +6479,7 @@ def workers_to_close(
             Minimum number of workers to keep around
         key : Callable(WorkerState)
             An optional callable mapping a WorkerState object to a group
-            affiliation.  Groups will be closed together.  This is useful when
+            affiliation. Groups will be closed together. This is useful when
             closing workers must be done collectively, such as by hostname.
         target : int
             Target number of workers to have after we close
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 1373499d18a..456485f0cab 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -6247,6 +6247,7 @@ async def f(dask_worker):
     assert b.foo == "bar"
 
 
+@pytest.mark.skipif(WINDOWS, reason="frequently kills off the whole test suite")
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 2)] * 2)
 async def test_performance_report(c, s, a, b):
     pytest.importorskip("bokeh")
@@ -6282,13 +6283,13 @@ async def f(stacklevel, mode=None):
     assert "distributed.scheduler - INFO - Clear task state" in data
     assert dask.__version__ in data
 
-    # Stacklevel two captures code two frames back -- which in this case
+    # stacklevel=2 captures code two frames back -- which in this case
     # is the testing function
     data = await f(stacklevel=2)
     assert "async def test_performance_report(c, s, a, b):" in data
     assert "Dask Performance Report" in data
 
-    # Stacklevel zero or lower is overridden to stacklevel=1 so we don't see
+    # stacklevel=0 or lower is overridden to stacklevel=1 so we don't see
     # distributed internals
     data = await f(stacklevel=0)
     assert "Also, we want this comment to appear" in data
diff --git a/distributed/worker.py b/distributed/worker.py
index 1b2fe7da65c..f70361664e2 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -1429,7 +1429,8 @@ async def get_data(
             and self.outgoing_current_count >= max_connections
         ):
             logger.debug(
-                "Worker %s has too many open connections to respond to data request from %s (%d/%d).%s",
+                "Worker %s has too many open connections to respond to data request "
+                "from %s (%d/%d).%s",
                 self.address,
                 who,
                 self.outgoing_current_count,
@@ -3108,7 +3109,7 @@ async def memory_monitor(self):
         memory = proc.memory_info().rss
         frac = memory / self.memory_limit
 
-        async def check_pause(memory):
+        def check_pause(memory):
             frac = memory / self.memory_limit
             # Pause worker threads if above 80% memory use
             if self.memory_pause_fraction and frac > self.memory_pause_fraction:
@@ -3138,7 +3139,7 @@ async def check_pause(memory):
                 self.paused = False
                 self.ensure_computing()
 
-        await check_pause(memory)
+        check_pause(memory)
         # Dump data to disk if above 70%
         if self.memory_spill_fraction and frac > self.memory_spill_fraction:
             logger.debug(
@@ -3179,10 +3180,10 @@ async def check_pause(memory):
                     # before trying to evict even more data.
                     self._throttled_gc.collect()
                     memory = proc.memory_info().rss
-            await check_pause(memory)
+            check_pause(memory)
             if count:
                 logger.debug(
-                    "Moved %d pieces of data data and %s to disk",
+                    "Moved %d tasks worth %s to disk",
                     count,
                     format_bytes(total),
                 )

From e5eb40c597dba0358221500fda49d653b7cf1f3a Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Fri, 24 Sep 2021 12:36:19 +0100
Subject: [PATCH 1500/1550] fsspec and s3fs git tips are incompatible (#5346)

---
 continuous_integration/environment-3.9.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/continuous_integration/environment-3.9.yaml b/continuous_integration/environment-3.9.yaml
index accd7b6446c..9eee8754269 100644
--- a/continuous_integration/environment-3.9.yaml
+++ b/continuous_integration/environment-3.9.yaml
@@ -49,6 +49,7 @@ dependencies:
       - git+https://github.com/dask/dask
       - git+https://github.com/dask/s3fs
       - git+https://github.com/dask/zict
-      - git+https://github.com/intake/filesystem_spec
+      # FIXME https://github.com/dask/distributed/issues/5345
+      # - git+https://github.com/intake/filesystem_spec
       - git+https://github.com/joblib/joblib
       - keras

From a8d4ffa156cac22b05b9cef3a0599a08e9945146 Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Mon, 27 Sep 2021 14:05:23 +0200
Subject: [PATCH 1501/1550] Worker state machine refactor (#5046)

Co-authored-by: crusaderky <crusaderky@gmail.com>
---
 distributed/diagnostics/plugin.py             |   18 -
 .../diagnostics/tests/test_worker_plugin.py   |   75 +-
 distributed/scheduler.py                      |   34 +-
 distributed/stealing.py                       |   10 +-
 distributed/tests/test_cancelled_state.py     |  133 ++
 distributed/tests/test_client.py              |   13 +-
 distributed/tests/test_failed_workers.py      |   83 +-
 distributed/tests/test_steal.py               |   19 +-
 distributed/tests/test_stress.py              |    5 +-
 distributed/tests/test_worker.py              |  324 ++-
 distributed/worker.py                         | 1876 +++++++++--------
 distributed/worker_client.py                  |    9 +-
 12 files changed, 1508 insertions(+), 1091 deletions(-)
 create mode 100644 distributed/tests/test_cancelled_state.py

diff --git a/distributed/diagnostics/plugin.py b/distributed/diagnostics/plugin.py
index 233ff626374..eeb7b1b35f7 100644
--- a/distributed/diagnostics/plugin.py
+++ b/distributed/diagnostics/plugin.py
@@ -161,24 +161,6 @@ def transition(self, key, start, finish, **kwargs):
         kwargs : More options passed when transitioning
         """
 
-    def release_key(self, key, state, cause, reason, report):
-        """
-        Called when the worker releases a task.
-
-        Parameters
-        ----------
-        key : string
-        state : string
-            State of the released task.
-            One of waiting, ready, executing, long-running, memory, error.
-        cause : string or None
-            Additional information on what triggered the release of the task.
-        reason : None
-            Not used.
-        report : bool
-            Whether the worker should report the released task to the scheduler.
-        """
-
 
 class NannyPlugin:
     """Interface to extend the Nanny
diff --git a/distributed/diagnostics/tests/test_worker_plugin.py b/distributed/diagnostics/tests/test_worker_plugin.py
index 6d62f8103ea..140bfbc6eee 100644
--- a/distributed/diagnostics/tests/test_worker_plugin.py
+++ b/distributed/diagnostics/tests/test_worker_plugin.py
@@ -34,9 +34,6 @@ def transition(self, key, start, finish, **kwargs):
             {"key": key, "start": start, "finish": finish}
         )
 
-    def release_key(self, key, state, cause, reason, report):
-        self.observed_notifications.append({"key": key, "state": state})
-
 
 @gen_cluster(client=True, nthreads=[])
 async def test_create_with_client(c, s):
@@ -107,11 +104,12 @@ async def test_create_on_construction(c, s, a, b):
 @gen_cluster(nthreads=[("127.0.0.1", 1)], client=True)
 async def test_normal_task_transitions_called(c, s, w):
     expected_notifications = [
-        {"key": "task", "start": "new", "finish": "waiting"},
+        {"key": "task", "start": "released", "finish": "waiting"},
         {"key": "task", "start": "waiting", "finish": "ready"},
         {"key": "task", "start": "ready", "finish": "executing"},
         {"key": "task", "start": "executing", "finish": "memory"},
-        {"key": "task", "state": "memory"},
+        {"key": "task", "start": "memory", "finish": "released"},
+        {"key": "task", "start": "released", "finish": "forgotten"},
     ]
 
     plugin = MyPlugin(1, expected_notifications=expected_notifications)
@@ -127,11 +125,12 @@ def failing(x):
         raise Exception()
 
     expected_notifications = [
-        {"key": "task", "start": "new", "finish": "waiting"},
+        {"key": "task", "start": "released", "finish": "waiting"},
         {"key": "task", "start": "waiting", "finish": "ready"},
         {"key": "task", "start": "ready", "finish": "executing"},
         {"key": "task", "start": "executing", "finish": "error"},
-        {"key": "task", "state": "error"},
+        {"key": "task", "start": "error", "finish": "released"},
+        {"key": "task", "start": "released", "finish": "forgotten"},
     ]
 
     plugin = MyPlugin(1, expected_notifications=expected_notifications)
@@ -147,11 +146,12 @@ def failing(x):
 )
 async def test_superseding_task_transitions_called(c, s, w):
     expected_notifications = [
-        {"key": "task", "start": "new", "finish": "waiting"},
+        {"key": "task", "start": "released", "finish": "waiting"},
         {"key": "task", "start": "waiting", "finish": "constrained"},
         {"key": "task", "start": "constrained", "finish": "executing"},
         {"key": "task", "start": "executing", "finish": "memory"},
-        {"key": "task", "state": "memory"},
+        {"key": "task", "start": "memory", "finish": "released"},
+        {"key": "task", "start": "released", "finish": "forgotten"},
     ]
 
     plugin = MyPlugin(1, expected_notifications=expected_notifications)
@@ -166,16 +166,18 @@ async def test_dependent_tasks(c, s, w):
     dsk = {"dep": 1, "task": (inc, "dep")}
 
     expected_notifications = [
-        {"key": "dep", "start": "new", "finish": "waiting"},
+        {"key": "dep", "start": "released", "finish": "waiting"},
         {"key": "dep", "start": "waiting", "finish": "ready"},
         {"key": "dep", "start": "ready", "finish": "executing"},
         {"key": "dep", "start": "executing", "finish": "memory"},
-        {"key": "task", "start": "new", "finish": "waiting"},
+        {"key": "task", "start": "released", "finish": "waiting"},
         {"key": "task", "start": "waiting", "finish": "ready"},
         {"key": "task", "start": "ready", "finish": "executing"},
         {"key": "task", "start": "executing", "finish": "memory"},
-        {"key": "dep", "state": "memory"},
-        {"key": "task", "state": "memory"},
+        {"key": "dep", "start": "memory", "finish": "released"},
+        {"key": "task", "start": "memory", "finish": "released"},
+        {"key": "task", "start": "released", "finish": "forgotten"},
+        {"key": "dep", "start": "released", "finish": "forgotten"},
     ]
 
     plugin = MyPlugin(1, expected_notifications=expected_notifications)
@@ -203,6 +205,53 @@ class MyCustomPlugin(WorkerPlugin):
     assert next(iter(w.plugins)).startswith("MyCustomPlugin-")
 
 
+def test_release_key_deprecated():
+    class ReleaseKeyDeprecated(WorkerPlugin):
+        def __init__(self):
+            self._called = False
+
+        def release_key(self, key, state, cause, reason, report):
+            # Ensure that the handler still works
+            self._called = True
+            assert state == "memory"
+            assert key == "task"
+
+        def teardown(self, worker):
+            assert self._called
+            return super().teardown(worker)
+
+    @gen_cluster(client=True, nthreads=[("", 1)])
+    async def test(c, s, a):
+
+        await c.register_worker_plugin(ReleaseKeyDeprecated())
+        fut = await c.submit(inc, 1, key="task")
+        assert fut == 2
+
+    with pytest.deprecated_call(
+        match="The `WorkerPlugin.release_key` hook is depreacted"
+    ):
+        test()
+
+
+def test_assert_no_warning_no_overload():
+    """Assert we do not receive a deprecation warning if we do not overload any
+    methods
+    """
+
+    class Dummy(WorkerPlugin):
+        pass
+
+    @gen_cluster(client=True, nthreads=[("", 1)])
+    async def test(c, s, a):
+
+        await c.register_worker_plugin(Dummy())
+        fut = await c.submit(inc, 1, key="task")
+        assert fut == 2
+
+    with pytest.warns(None):
+        test()
+
+
 @gen_cluster(nthreads=[("127.0.0.1", 1)], client=True)
 async def test_WorkerPlugin_overwrite(c, s, w):
     class MyCustomPlugin(WorkerPlugin):
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index b1f8057c634..c538056e7dc 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -3001,7 +3001,11 @@ def transition_processing_released(self, key):
             w: str = _remove_from_processing(self, ts)
             if w:
                 worker_msgs[w] = [
-                    {"op": "free-keys", "keys": [key], "reason": "Processing->Released"}
+                    {
+                        "op": "free-keys",
+                        "keys": [key],
+                        "reason": f"processing-released-{time()}",
+                    }
                 ]
 
             ts.state = "released"
@@ -5367,7 +5371,7 @@ def handle_missing_data(self, key=None, errant_worker=None, **kwargs):
         self.log.append(("missing", key, errant_worker))
 
         ts: TaskState = parent._tasks.get(key)
-        if ts is None or not ts._who_has:
+        if ts is None:
             return
         ws: WorkerState = parent._workers_dv.get(errant_worker)
         if ws is not None and ws in ts._who_has:
@@ -5380,17 +5384,14 @@ def handle_missing_data(self, key=None, errant_worker=None, **kwargs):
             else:
                 self.transitions({key: "forgotten"})
 
-    def release_worker_data(self, comm=None, keys=None, worker=None):
+    def release_worker_data(self, comm=None, key=None, worker=None):
         parent: SchedulerState = cast(SchedulerState, self)
         ws: WorkerState = parent._workers_dv.get(worker)
-        if not ws:
+        ts: TaskState = parent._tasks.get(key)
+        if not ws or not ts:
             return
-        tasks: set = {parent._tasks[k] for k in keys if k in parent._tasks}
-        removed_tasks: set = tasks.intersection(ws._has_what)
-
-        ts: TaskState
         recommendations: dict = {}
-        for ts in removed_tasks:
+        if ts in ws._has_what:
             del ws._has_what[ts]
             ws._nbytes -= ts.get_nbytes()
             wh: set = ts._who_has
@@ -6709,7 +6710,7 @@ def add_keys(self, comm=None, worker=None, keys=()):
         if worker not in parent._workers_dv:
             return "not found"
         ws: WorkerState = parent._workers_dv[worker]
-        superfluous_data = []
+        redundant_replicas = []
         for key in keys:
             ts: TaskState = parent._tasks.get(key)
             if ts is not None and ts._state == "memory":
@@ -6718,14 +6719,15 @@ def add_keys(self, comm=None, worker=None, keys=()):
                     ws._has_what[ts] = None
                     ts._who_has.add(ws)
             else:
-                superfluous_data.append(key)
-        if superfluous_data:
+                redundant_replicas.append(key)
+
+        if redundant_replicas:
             self.worker_send(
                 worker,
                 {
-                    "op": "superfluous-data",
-                    "keys": superfluous_data,
-                    "reason": f"Add keys which are not in-memory {superfluous_data}",
+                    "op": "remove-replicas",
+                    "keys": redundant_replicas,
+                    "stimulus_id": f"redundant-replicas-{time()}",
                 },
             )
 
@@ -7867,6 +7869,8 @@ def _task_to_msg(state: SchedulerState, ts: TaskState, duration: double = -1) ->
         "key": ts._key,
         "priority": ts._priority,
         "duration": duration,
+        "stimulus_id": f"compute-task-{time()}",
+        "who_has": {},
     }
     if ts._resource_restrictions:
         msg["resource_restrictions"] = ts._resource_restrictions
diff --git a/distributed/stealing.py b/distributed/stealing.py
index fc8e1955dff..0297691f02f 100644
--- a/distributed/stealing.py
+++ b/distributed/stealing.py
@@ -230,7 +230,15 @@ async def move_task_confirm(self, key=None, worker=None, state=None):
                 return
 
             # Victim had already started execution, reverse stealing
-            if state in ("memory", "executing", "long-running", None):
+            if state in (
+                "memory",
+                "executing",
+                "long-running",
+                "released",
+                "cancelled",
+                "resumed",
+                None,
+            ):
                 self.log(("already-computing", key, victim.address, thief.address))
                 self.scheduler.check_idle_saturated(thief)
                 self.scheduler.check_idle_saturated(victim)
diff --git a/distributed/tests/test_cancelled_state.py b/distributed/tests/test_cancelled_state.py
new file mode 100644
index 00000000000..152ce0a8533
--- /dev/null
+++ b/distributed/tests/test_cancelled_state.py
@@ -0,0 +1,133 @@
+import asyncio
+from unittest import mock
+
+import distributed
+from distributed.core import CommClosedError
+from distributed.utils_test import _LockedCommPool, gen_cluster, inc, slowinc
+
+
+async def wait_for_state(key, state, dask_worker):
+    while key not in dask_worker.tasks or dask_worker.tasks[key].state != state:
+        await asyncio.sleep(0.005)
+
+
+async def wait_for_cancelled(key, dask_worker):
+    while key in dask_worker.tasks:
+        if dask_worker.tasks[key].state == "cancelled":
+            return
+        await asyncio.sleep(0.005)
+    assert False
+
+
+@gen_cluster(client=True, nthreads=[("", 1)])
+async def test_abort_execution_release(c, s, a):
+    fut = c.submit(slowinc, 1, delay=1)
+    await wait_for_state(fut.key, "executing", a)
+    fut.release()
+    await wait_for_cancelled(fut.key, a)
+
+
+@gen_cluster(client=True, nthreads=[("", 1)])
+async def test_abort_execution_reschedule(c, s, a):
+    fut = c.submit(slowinc, 1, delay=1)
+    await wait_for_state(fut.key, "executing", a)
+    fut.release()
+    await wait_for_cancelled(fut.key, a)
+    fut = c.submit(slowinc, 1, delay=0.1)
+    await fut
+
+
+@gen_cluster(client=True, nthreads=[("", 1)])
+async def test_abort_execution_add_as_dependency(c, s, a):
+    fut = c.submit(slowinc, 1, delay=1)
+    await wait_for_state(fut.key, "executing", a)
+    fut.release()
+    await wait_for_cancelled(fut.key, a)
+
+    fut = c.submit(slowinc, 1, delay=1)
+    fut = c.submit(slowinc, fut, delay=1)
+    await fut
+
+
+@gen_cluster(client=True)
+async def test_abort_execution_to_fetch(c, s, a, b):
+    fut = c.submit(slowinc, 1, delay=2, key="f1", workers=[a.worker_address])
+    await wait_for_state(fut.key, "executing", a)
+    fut.release()
+    await wait_for_cancelled(fut.key, a)
+
+    # While the first worker is still trying to compute f1, we'll resubmit it to
+    # another worker with a smaller delay. The key is still the same
+    fut = c.submit(inc, 1, key="f1", workers=[b.worker_address])
+    # then, a must switch the execute to fetch. Instead of doing so, it will
+    # simply re-use the currently computing result.
+    fut = c.submit(inc, fut, workers=[a.worker_address], key="f2")
+    await fut
+
+
+@gen_cluster(client=True)
+async def test_worker_find_missing(c, s, a, b):
+    fut = c.submit(inc, 1, workers=[a.address])
+    await fut
+    # We do not want to use proper API since it would ensure that the cluster is
+    # informed properly
+    del a.data[fut.key]
+    del a.tasks[fut.key]
+
+    # Actually no worker has the data; the scheduler is supposed to reschedule
+    assert await c.submit(inc, fut, workers=[b.address]) == 3
+
+
+@gen_cluster(client=True)
+async def test_worker_stream_died_during_comm(c, s, a, b):
+    write_queue = asyncio.Queue()
+    write_event = asyncio.Event()
+    b.rpc = _LockedCommPool(
+        b.rpc,
+        write_queue=write_queue,
+        write_event=write_event,
+    )
+    fut = c.submit(inc, 1, workers=[a.address], allow_other_workers=True)
+    await fut
+    # Actually no worker has the data; the scheduler is supposed to reschedule
+    res = c.submit(inc, fut, workers=[b.address])
+
+    await write_queue.get()
+    await a.close()
+    write_event.set()
+
+    await res
+    assert any("receive-dep-failed" in msg for msg in b.log)
+
+
+@gen_cluster(client=True)
+async def test_flight_to_executing_via_cancelled_resumed(c, s, a, b):
+    lock = asyncio.Lock()
+    await lock.acquire()
+
+    async def wait_and_raise(*args, **kwargs):
+        async with lock:
+            raise CommClosedError()
+
+    with mock.patch.object(
+        distributed.worker,
+        "get_data_from_worker",
+        side_effect=wait_and_raise,
+    ):
+        fut1 = c.submit(inc, 1, workers=[a.address], allow_other_workers=True)
+        fut2 = c.submit(inc, fut1, workers=[b.address])
+
+        await wait_for_state(fut1.key, "flight", b)
+
+        # Close in scheduler to ensure we transition and reschedule task properly
+        await s.close_worker(worker=a.address)
+        await wait_for_state(fut1.key, "resumed", b)
+
+    lock.release()
+    assert await fut2 == 3
+
+    b_story = b.story(fut1.key)
+    assert any("receive-dep-failed" in msg for msg in b_story)
+    assert any("missing-dep" in msg for msg in b_story)
+    assert any("cancelled" in msg for msg in b_story)
+    assert any("resumed" in msg for msg in b_story)
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 456485f0cab..90a6c828bbf 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -5098,15 +5098,16 @@ def f(x):
         total = client.submit(sum, futures).result()
         return total
 
-    futures = c.map(f, range(100))
+    futures = c.map(f, range(10), workers=[a.address])
 
     results = await c.gather(futures)
+    # We dispatch 10 tasks and every task generates 11 more tasks
+    # 10 * 11 + 10
+    assert a.executed_count + b.executed_count == 120
+    assert a.executed_count >= 10
+    assert b.executed_count > 0
 
-    assert a.executed_count + b.executed_count == 1100
-    assert a.executed_count > 200
-    assert b.executed_count > 200
-
-    assert results == [sum(map(inc, range(10)))] * 100
+    assert results == [sum(map(inc, range(10)))] * 10
 
 
 @gen_cluster(client=True)
diff --git a/distributed/tests/test_failed_workers.py b/distributed/tests/test_failed_workers.py
index 83eb8ae8e50..8e5d01167d0 100644
--- a/distributed/tests/test_failed_workers.py
+++ b/distributed/tests/test_failed_workers.py
@@ -407,9 +407,10 @@ def __sizeof__(self) -> int:
         return parse_bytes(dask.config.get("distributed.comm.offload")) + 1
 
 
-@pytest.mark.flaky(reruns=10, reruns_delay=5)
 @gen_cluster(client=True)
 async def test_worker_who_has_clears_after_failed_connection(c, s, a, b):
+    """This test is very sensitive to cluster state consistency. Timeouts often
+    indicate subtle deadlocks. Be mindful when marking flaky/repeat/etc."""
     n = await Nanny(s.address, nthreads=2, loop=s.loop)
 
     while len(s.nthreads) < 3:
@@ -526,82 +527,6 @@ async def test_worker_time_to_live(c, s, a, b):
     set(s.workers) == {b.address}
 
 
-class SlowDeserialize:
-    def __init__(self, data, delay=0.1):
-        self.delay = delay
-        self.data = data
-
-    def __getstate__(self):
-        return self.delay
-
-    def __setstate__(self, state):
-        delay = state
-        import time
-
-        time.sleep(delay)
-        return SlowDeserialize(delay)
-
-    def __sizeof__(self) -> int:
-        # Ensure this is offloaded to avoid blocking loop
-        import dask
-        from dask.utils import parse_bytes
-
-        return parse_bytes(dask.config.get("distributed.comm.offload")) + 1
-
-
-@gen_cluster(client=True)
-async def test_handle_superfluous_data(c, s, a, b):
-    """
-    See https://github.com/dask/distributed/pull/4784#discussion_r649210094
-    """
-
-    def slow_deser(x, delay):
-        return SlowDeserialize(x, delay=delay)
-
-    futA = c.submit(
-        slow_deser, 1, delay=1, workers=[a.address], key="A", allow_other_workers=True
-    )
-    futB = c.submit(inc, 1, workers=[b.address], key="B")
-    await wait([futA, futB])
-
-    def reducer(*args):
-        return
-
-    assert len(a.tasks) == 1
-    assert futA.key in a.tasks
-
-    assert len(b.tasks) == 1
-    assert futB.key in b.tasks
-
-    red = c.submit(reducer, [futA, futB], workers=[b.address], key="reducer")
-
-    dep_key = futA.key
-
-    # Wait for the connection to be established
-    while dep_key not in b.tasks or not b.tasks[dep_key].state == "flight":
-        await asyncio.sleep(0.001)
-
-    # Wait for the connection to be returned to the pool. this signals that
-    # worker B is done with the communication and is about to deserialize the
-    # result
-    while a.address not in b.rpc.available and not b.rpc.available[a.address]:
-        await asyncio.sleep(0.001)
-
-    assert b.tasks[dep_key].state == "flight"
-    # After the comm is finished and the deserialization starts, Worker B
-    # wouldn't notice that A dies.
-    await a.close()
-    # However, while B is busy deserializing a third worker might notice that A
-    # is dead and issues a handle-missing signal to the scheduler. Since at this
-    # point in time, A was the only worker with a verified replica, the
-    # scheduler reschedules the computation by transitioning it to released. The
-    # released transition has the side effect that it purges all data which is
-    # in memory which exposes us to a race condition on B if B also receives the
-    # signal to compute that task in the meantime.
-    s.handle_missing_data(key=dep_key, errant_worker=a.address)
-    await red
-
-
 @gen_cluster()
 async def test_forget_data_not_supposed_to_have(s, a, b):
     """
@@ -618,7 +543,9 @@ async def test_forget_data_not_supposed_to_have(s, a, b):
     ts = TaskState("key")
     ts.state = "flight"
     a.tasks["key"] = ts
-    a.transition_flight_memory(ts, value=123)
+    recommendations = {ts: ("memory", 123)}
+    a.transitions(recommendations, stimulus_id="test")
+
     assert a.data
     while a.data:
         await asyncio.sleep(0.001)
diff --git a/distributed/tests/test_steal.py b/distributed/tests/test_steal.py
index d9acf2ce28b..3517805451c 100644
--- a/distributed/tests/test_steal.py
+++ b/distributed/tests/test_steal.py
@@ -689,14 +689,17 @@ async def test_dont_steal_already_released(c, s, a, b):
         await asyncio.sleep(0.05)
 
     del future
-    await asyncio.sleep(0.05)
-    # In case the system is slow (e.g. network) ensure that nothing bad happens
-    # if the key was already released
-    assert key not in a.tasks
-    a.steal_request(key)
-    assert a.batched_stream.buffer == [
-        {"op": "steal-response", "key": key, "state": None}
-    ]
+
+    while key in a.tasks and a.tasks[key].state != "released":
+        await asyncio.sleep(0.05)
+
+    a.handle_steal_request(key)
+    assert len(a.batched_stream.buffer) == 1
+    msg = a.batched_stream.buffer[0]
+    assert msg["op"] == "steal-response"
+    assert msg["key"] == key
+    assert msg["state"] in [None, "released"]
+
     with captured_logger(
         logging.getLogger("distributed.stealing"), level=logging.DEBUG
     ) as stealing_logs:
diff --git a/distributed/tests/test_stress.py b/distributed/tests/test_stress.py
index afc4556afc9..e47ec84225e 100644
--- a/distributed/tests/test_stress.py
+++ b/distributed/tests/test_stress.py
@@ -178,10 +178,11 @@ def vsum(*args):
 
 @pytest.mark.avoid_ci
 @pytest.mark.slow
-@pytest.mark.timeout(1100)  # Override timeout from setup.cfg
-@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 80, timeout=1000)
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 80)
 async def test_stress_communication(c, s, *workers):
     s.validate = False  # very slow otherwise
+    for w in workers:
+        w.validate = False
     da = pytest.importorskip("dask.array")
     # Test consumes many file descriptors and can hang if the limit is too low
     resource = pytest.importorskip("resource")
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 7846bd8de16..b68643b8448 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -7,7 +7,6 @@
 import traceback
 from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
 from concurrent.futures.process import BrokenProcessPool
-from contextlib import suppress
 from numbers import Number
 from operator import add
 from time import sleep
@@ -1795,8 +1794,8 @@ async def test_story_with_deps(c, s, a, b):
     Assert that the structure of the story does not change unintentionally and
     expected subfields are actually filled
     """
-    futures = c.map(inc, range(10), workers=[a.address])
-    res = c.submit(sum, futures, workers=[b.address])
+    dep = c.submit(inc, 1, workers=[a.address])
+    res = c.submit(inc, dep, workers=[b.address])
     await res
     key = res.key
 
@@ -1804,32 +1803,71 @@ async def test_story_with_deps(c, s, a, b):
     assert story == []
     story = b.story(key)
 
+    pruned_story = []
+    stimulus_ids = set()
+    # Story now includes randomized stimulus_ids and timestamps.
+    for msg in story:
+        assert isinstance(msg, tuple), msg
+        assert isinstance(msg[-1], float), msg
+        assert msg[-1] > time() - 60, msg
+        pruned_msg = list(msg)
+        stimulus_ids.add(msg[-2])
+        pruned_story.append(tuple(pruned_msg[:-2]))
+
+    assert len(stimulus_ids) == 3
+    stimulus_id = pruned_story[0][-1]
+    assert isinstance(stimulus_id, str)
+    assert stimulus_id.startswith("compute-task")
+    # This is a simple transition log
     expected_story = [
-        (key, "new"),
-        (key, "new", "waiting"),
-        # First log is what needs to be fetched in total as determined in
-        # ensure_communicating
+        (key, "compute-task"),
+        (key, "released", "waiting", {}),
+        (key, "waiting", "ready", {}),
+        (key, "ready", "executing", {}),
+        (key, "put-in-memory"),
+        (key, "executing", "memory", {}),
+    ]
+    assert pruned_story == expected_story
+
+    dep_story = dep.key
+
+    story = b.story(dep_story)
+    pruned_story = []
+    stimulus_ids = set()
+    for msg in story:
+        assert isinstance(msg, tuple), msg
+        assert isinstance(msg[-1], float), msg
+        assert msg[-1] > time() - 60, msg
+        pruned_msg = list(msg)
+        stimulus_ids.add(msg[-2])
+        pruned_story.append(tuple(pruned_msg[:-2]))
+
+    assert len(stimulus_ids) == 3
+    stimulus_id = pruned_story[0][-1]
+    assert isinstance(stimulus_id, str)
+    expected_story = [
+        (dep_story, "register-replica", "released"),
+        (dep_story, "released", "fetch", {}),
         (
             "gather-dependencies",
-            key,
-            {fut.key for fut in futures},
+            a.address,
+            {dep.key},
         ),
-        # Second log may just be a subset of the above, see also
-        # Worker.select_keys_for_gather
-        # This case, it's all because Worker.target_message_size is sufficiently
-        # large
+        (dep_story, "fetch", "flight", {}),
         (
             "request-dep",
-            key,
             a.address,
-            {fut.key for fut in futures},
+            {dep.key},
         ),
-        (key, "waiting", "ready"),
-        (key, "ready", "executing"),
-        (key, "executing", "memory"),
-        (key, "put-in-memory"),
+        (
+            "receive-dep",
+            a.address,
+            {dep.key},
+        ),
+        (dep_story, "put-in-memory"),
+        (dep_story, "flight", "memory", {res.key: "ready"}),
     ]
-    assert story == expected_story
+    assert pruned_story == expected_story
 
 
 @gen_cluster(client=True)
@@ -2152,6 +2190,8 @@ def raise_exc(*args):
         await asyncio.sleep(0.01)
 
     expected_states = {
+        f.key: "released",
+        g.key: "released",
         res.key: "error",
     }
 
@@ -2344,6 +2384,7 @@ def raise_exc(*args):
     assert_task_states_on_worker(expected_states_A, a)
 
     expected_states_B = {
+        f.key: "released",
         g.key: "memory",
         h.key: "memory",
         res.key: "error",
@@ -2354,6 +2395,7 @@ def raise_exc(*args):
     g.release()
 
     expected_states_A = {
+        g.key: "released",
         h.key: "memory",
     }
     await asyncio.sleep(0.05)
@@ -2361,6 +2403,7 @@ def raise_exc(*args):
 
     # B must not forget a task since all have a still valid dependent
     expected_states_B = {
+        f.key: "released",
         h.key: "memory",
         res.key: "error",
     }
@@ -2371,6 +2414,8 @@ def raise_exc(*args):
     expected_states_A = {}
     assert_task_states_on_worker(expected_states_A, a)
     expected_states_B = {
+        f.key: "released",
+        h.key: "released",
         res.key: "error",
     }
 
@@ -2405,7 +2450,7 @@ async def test_hold_on_to_replicas(c, s, *workers):
         assert s.tasks[f2.key].state == "released"
         await asyncio.sleep(0.01)
 
-    while len(workers[2].tasks) > 1:
+    while len(workers[2].data) > 1:
         await asyncio.sleep(0.01)
 
 
@@ -2610,7 +2655,7 @@ def __call__(self, *args, **kwargs):
             await asyncio.sleep(0)
 
         ts = s.tasks[fut.key]
-        a.steal_request(fut.key)
+        a.handle_steal_request(fut.key)
         stealing_ext.scheduler.send_task_to_worker(b.address, ts)
 
         fut2 = c.submit(inc, fut, workers=[a.address])
@@ -2664,33 +2709,14 @@ def sink(a, b, *args):
     b.tasks[fut3.key].state = "fetch"
     event.set()
 
-    with captured_logger("distributed.worker", level=logging.DEBUG) as worker_logs:
-
-        # FIXME: We currently have no reliable, safe way to release the task and
-        # its dependent without race conditions
-
-        # Unfortunately res1 is deadlocking. IRL this is not always a problem
-        # since a commonly reported transition is Fetch->Memory, i.e. the task
-        # exists already in memory for whatever reason but a gather_dep was
-        # still runnign, e.g. the task was rescheduled on that worker and it was
-        # computed faster than fetched.
-
-        with suppress(TimeoutError):
-            await asyncio.wait_for(res1, 0.1)
-
-        assert await res2 == 5
+    assert await res1 == 5
+    assert await res2 == 5
 
-        del res1, res2, fut, fut2
-        fut3.release()
+    del res1, res2, fut, fut2
+    fut3.release()
 
-        while a.tasks and b.tasks:
-            await asyncio.sleep(0.1)
-
-    expected_msg = (
-        "Exception occured while handling `gather_dep` response for <Task 'f3' fetch>"
-    )
-    assert expected_msg in worker_logs.getvalue()
-    assert any("except-gather-dep-result" in msg for msg in b.story(fut3.key))
+    while a.tasks and b.tasks:
+        await asyncio.sleep(0.1)
 
 
 @gen_cluster(client=True)
@@ -2714,3 +2740,207 @@ async def test_gather_dep_exception_one_task_2(c, s, a, b):
     s.handle_missing_data(key="f1", errant_worker=a.address)
 
     await fut2
+
+
+def _acquire_replicas(scheduler, worker, *futures):
+    keys = [f.key for f in futures]
+
+    scheduler.stream_comms[worker.address].send(
+        {
+            "op": "acquire-replicas",
+            "keys": keys,
+            "stimulus_id": f"acquire-replicas-{time()}",
+            "priorities": {key: scheduler.tasks[key].priority for key in keys},
+            "who_has": {
+                key: {w.address for w in scheduler.tasks[key].who_has} for key in keys
+            },
+        },
+    )
+
+
+def _remove_replicas(scheduler, worker, *futures):
+    keys = [f.key for f in futures]
+
+    scheduler.stream_comms[worker.address].send(
+        {
+            "op": "remove-replicas",
+            "keys": keys,
+            "stimulus_id": f"remove-replicas-{time()}",
+        }
+    )
+
+
+@gen_cluster(client=True)
+async def test_acquire_replicas(c, s, a, b):
+    fut = c.submit(inc, 1, workers=[a.address])
+    await fut
+
+    _acquire_replicas(s, b, fut)
+
+    while len(s.who_has[fut.key]) != 2:
+        await asyncio.sleep(0.005)
+
+    for w in (a, b):
+        assert w.data[fut.key] == 2
+        assert w.tasks[fut.key].state == "memory"
+
+    fut.release()
+
+    while b.tasks or a.tasks:
+        await asyncio.sleep(0.005)
+
+
+@gen_cluster(client=True)
+async def test_acquire_replicas_same_channel(c, s, a, b):
+    fut = c.submit(inc, 1, workers=[a.address], key="f-replica")
+    futB = c.submit(inc, 2, workers=[a.address], key="f-B")
+    futC = c.submit(inc, futB, workers=[b.address], key="f-C")
+    await fut
+
+    _acquire_replicas(s, b, fut)
+
+    await futC
+    while fut.key not in b.tasks:
+        await asyncio.sleep(0.005)
+    assert len(s.who_has[fut.key]) == 2
+
+    # Ensure that both the replica and an ordinary dependency pass through the
+    # same communication channel
+
+    for f in [fut, futB]:
+        assert any("request-dep" in msg for msg in b.story(f.key))
+        assert any("gather-dependencies" in msg for msg in b.story(f.key))
+        assert any(f.key in msg["keys"] for msg in b.incoming_transfer_log)
+
+
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 3)
+async def test_acquire_replicas_many(c, s, *workers):
+    futs = c.map(inc, range(10), workers=[workers[0].address])
+    res = c.submit(sum, futs, workers=[workers[1].address])
+    final = c.submit(slowinc, res, delay=0.5, workers=[workers[1].address])
+
+    await wait(futs)
+
+    _acquire_replicas(s, workers[2], *futs)
+
+    # Worker 2 should normally not even be involved if there was no replication
+    while not all(
+        f.key in workers[2].tasks and workers[2].tasks[f.key].state == "memory"
+        for f in futs
+    ):
+        await asyncio.sleep(0.01)
+
+    assert all(ts.state == "memory" for ts in workers[2].tasks.values())
+
+    assert await final == sum(map(inc, range(10))) + 1
+    # All workers have a replica
+    assert all(len(s.tasks[f.key].who_has) == 3 for f in futs)
+    del futs, res, final
+
+    while any(w.tasks for w in workers):
+        await asyncio.sleep(0.001)
+
+
+@gen_cluster(client=True)
+async def test_remove_replica_simple(c, s, a, b):
+    futs = c.map(inc, range(10), workers=[a.address])
+    await wait(futs)
+    _acquire_replicas(s, b, *futs)
+
+    while not all(len(s.tasks[f.key].who_has) == 2 for f in futs):
+        await asyncio.sleep(0.01)
+
+    _remove_replicas(s, b, *futs)
+
+    while b.tasks:
+        await asyncio.sleep(0.01)
+
+    # might take a moment for the reply to reach the scheduler
+    while not all(len(s.tasks[f.key].who_has) == 1 for f in futs):
+        await asyncio.sleep(0.01)
+
+
+@gen_cluster(client=True)
+async def test_remove_replica_while_computing(c, s, *workers):
+    futs = c.map(inc, range(10), workers=[workers[0].address])
+
+    # All interesting things will happen on that worker
+    w = workers[1]
+    intermediate = c.map(slowinc, futs, delay=0.1, workers=[w.address])
+
+    def reduce(*args, **kwargs):
+        import time
+
+        time.sleep(0.5)
+        return
+
+    final = c.submit(reduce, intermediate, workers=[w.address], key="final")
+    while final.key not in w.tasks:
+        await asyncio.sleep(0.001)
+
+    while not all(fut.done() for fut in intermediate):
+        # The worker should reject all of these since they are required
+        _remove_replicas(s, w, *futs)
+        _remove_replicas(s, w, *intermediate)
+        await asyncio.sleep(0.001)
+
+    await wait(intermediate)
+
+    # Since intermediate is done, futs replicas may be removed.
+    # They might be already gone due to the above remove replica calls
+    _remove_replicas(s, w, *futs)
+    # the intermediate tasks should not be touched because they are still needed
+    # (the scheduler should not have made the above call but we should be safe
+    # regarless)
+    assert all(w.tasks[f.key].state == "memory" for f in intermediate)
+
+    while any(w.tasks[f.key].state != "released" for f in futs if f.key in w.tasks):
+        await asyncio.sleep(0.001)
+
+    # The scheduler actually gets notified about the removed replica
+    while not all(len(s.tasks[f.key].who_has) == 1 for f in futs):
+        await asyncio.sleep(0.001)
+
+    await final
+    del final, intermediate, futs
+
+    while any(w.tasks for w in workers):
+        await asyncio.sleep(0.001)
+
+
+@gen_cluster(client=True, nthreads=[("", 1)] * 3)
+async def test_who_has_consistent_remove_replica(c, s, *workers):
+    a = workers[0]
+    other_workers = {w for w in workers if w != a}
+    f1 = c.submit(inc, 1, key="f1", workers=[w.address for w in other_workers])
+    await wait(f1)
+    for w in other_workers:
+        _acquire_replicas(s, w, f1)
+
+    while not len(s.tasks[f1.key].who_has) == len(other_workers):
+        await asyncio.sleep(0)
+
+    f2 = c.submit(inc, f1, workers=[a.address])
+
+    # Wait just until the moment the worker received the task and scheduled the
+    # task to be fetched, then remove the replica from the worker this one is
+    # trying to get the data from. Ensure this is handled gracefully and no
+    # suspicious counters are raised since this is expected behaviour when
+    # removing replicas
+
+    while f1.key not in a.tasks or a.tasks[f1.key].state != "flight":
+        await asyncio.sleep(0)
+
+    coming_from = None
+    for w in other_workers:
+        coming_from = w
+        if w.address == a.tasks[f1.key].coming_from:
+            break
+
+    coming_from.handle_remove_replicas([f1.key], "test")
+
+    await f2
+
+    assert ("missing-dep", f1.key) in a.story(f1.key)
+    assert a.tasks[f1.key].suspicious_count == 0
+    assert s.tasks[f1.key].suspicious == 0
diff --git a/distributed/worker.py b/distributed/worker.py
index f70361664e2..4e240dfa58b 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -93,10 +93,16 @@
 
 no_value = "--no-value-sentinel--"
 
-IN_PLAY = ("waiting", "ready", "executing", "long-running")
-PENDING = ("waiting", "ready", "constrained")
-PROCESSING = ("waiting", "ready", "constrained", "executing", "long-running")
-READY = ("ready", "constrained")
+PROCESSING = {
+    "waiting",
+    "ready",
+    "constrained",
+    "executing",
+    "long-running",
+    "cancelled",
+    "resumed",
+}
+READY = {"ready", "constrained"}
 
 
 DEFAULT_EXTENSIONS = [PubSubWorkerExtension]
@@ -112,6 +118,10 @@
 SerializedTask = namedtuple("SerializedTask", ["function", "args", "kwargs", "task"])
 
 
+class InvalidTransition(Exception):
+    pass
+
+
 class TaskState:
     """Holds volatile state relating to an individual Dask task
 
@@ -177,24 +187,25 @@ def __init__(self, key, runspec=None):
         self.dependents = set()
         self.duration = None
         self.priority = None
-        self.state = "new"
+        self.state = "released"
         self.who_has = set()
         self.coming_from = None
         self.waiting_for_data = set()
-        self.resource_restrictions = None
+        self.resource_restrictions = {}
         self.exception = None
         self.exception_text = ""
         self.traceback = None
         self.traceback_text = ""
         self.type = None
         self.suspicious_count = 0
-        self.startstops = list()
+        self.startstops = []
         self.start_time = None
         self.stop_time = None
         self.metadata = {}
         self.nbytes = None
         self.annotations = None
-        self.scheduler_holds_ref = False
+        self.done = False
+        self._next = None
 
     def __repr__(self):
         return f"<Task {self.key!r} {self.state}>"
@@ -203,6 +214,11 @@ def get_nbytes(self) -> int:
         nbytes = self.nbytes
         return nbytes if nbytes is not None else DEFAULT_DATA_SIZE
 
+    def is_protected(self) -> bool:
+        return self.state in PROCESSING or any(
+            dep_ts.state in PROCESSING for dep_ts in self.dependents
+        )
+
 
 class Worker(ServerNode):
     """Worker node in a Dask distributed cluster
@@ -427,17 +443,16 @@ def __init__(
         lifetime_restart=None,
         **kwargs,
     ):
-        self.tasks = dict()
+        self.tasks = {}
         self.waiting_for_data_count = 0
         self.has_what = defaultdict(set)
         self.pending_data_per_worker = defaultdict(deque)
         self.nanny = nanny
         self._lock = threading.Lock()
 
-        self.data_needed = deque()  # TODO: replace with heap?
+        self.data_needed = []
 
-        self.in_flight_tasks = 0
-        self.in_flight_workers = dict()
+        self.in_flight_workers = {}
         self.total_out_connections = dask.config.get(
             "distributed.worker.connections.outgoing"
         )
@@ -448,10 +463,10 @@ def __init__(
         self.comm_nbytes = 0
         self._missing_dep_flight = set()
 
-        self.threads = dict()
+        self.threads = {}
 
         self.active_threads_lock = threading.Lock()
-        self.active_threads = dict()
+        self.active_threads = {}
         self.active_keys = set()
         self.profile_keys = defaultdict(profile.create)
         self.profile_keys_history = deque(maxlen=3600)
@@ -460,9 +475,10 @@ def __init__(
 
         self.generation = 0
 
-        self.ready = list()
+        self.ready = []
         self.constrained = deque()
-        self.executing_count = 0
+        self._executing = set()
+        self._in_flight_tasks = set()
         self.executed_count = 0
         self.long_running = set()
 
@@ -475,33 +491,56 @@ def __init__(
         if validate is None:
             validate = dask.config.get("distributed.scheduler.validate")
         self.validate = validate
-
-        self._transitions = {
-            # Basic state transitions
-            ("new", "waiting"): self.transition_new_waiting,
-            ("new", "fetch"): self.transition_new_fetch,
-            ("waiting", "ready"): self.transition_waiting_ready,
-            ("fetch", "flight"): self.transition_fetch_flight,
-            ("ready", "executing"): self.transition_ready_executing,
-            ("executing", "memory"): self.transition_executing_done,
-            ("flight", "memory"): self.transition_flight_memory,
-            ("flight", "fetch"): self.transition_flight_fetch,
-            # Shouldn't be a valid transition but happens nonetheless
-            ("ready", "memory"): self.transition_ready_memory,
-            # Scheduler intercession (re-assignment)
-            ("fetch", "waiting"): self.transition_fetch_waiting,
-            ("flight", "waiting"): self.transition_flight_waiting,
-            # Errors, long-running, constrained
-            ("waiting", "error"): self.transition_waiting_done,
+        self._transitions_table = {
+            ("cancelled", "resumed"): self.transition_cancelled_resumed,
+            ("cancelled", "fetch"): self.transition_cancelled_fetch,
+            ("cancelled", "released"): self.transition_cancelled_released,
+            ("cancelled", "waiting"): self.transition_cancelled_waiting,
+            ("cancelled", "forgotten"): self.transition_cancelled_forgotten,
+            ("cancelled", "memory"): self.transition_cancelled_memory,
+            ("cancelled", "error"): self.transition_generic_error,
+            ("resumed", "memory"): self.transition_generic_memory,
+            ("resumed", "error"): self.transition_generic_error,
+            ("resumed", "released"): self.transition_generic_released,
+            ("resumed", "waiting"): self.transition_rescheduled_next,
+            ("resumed", "fetch"): self.transition_rescheduled_next,
             ("constrained", "executing"): self.transition_constrained_executing,
-            ("executing", "error"): self.transition_executing_done,
-            ("executing", "rescheduled"): self.transition_executing_done,
+            ("constrained", "released"): self.transition_constrained_released,
+            ("error", "released"): self.transition_generic_released,
+            ("executing", "error"): self.transition_executing_error,
             ("executing", "long-running"): self.transition_executing_long_running,
-            ("long-running", "error"): self.transition_executing_done,
-            ("long-running", "memory"): self.transition_executing_done,
-            ("long-running", "rescheduled"): self.transition_executing_done,
+            ("executing", "memory"): self.transition_executing_memory,
+            ("executing", "released"): self.transition_executing_released,
+            ("executing", "rescheduled"): self.transition_executing_rescheduled,
+            ("fetch", "flight"): self.transition_fetch_flight,
+            ("fetch", "missing"): self.transition_fetch_missing,
+            ("fetch", "released"): self.transition_generic_released,
+            ("flight", "error"): self.transition_flight_error,
+            ("flight", "fetch"): self.transition_flight_fetch,
+            ("flight", "memory"): self.transition_flight_memory,
+            ("flight", "released"): self.transition_flight_released,
+            ("long-running", "error"): self.transition_generic_error,
+            ("long-running", "memory"): self.transition_long_running_memory,
+            ("long-running", "rescheduled"): self.transition_executing_rescheduled,
+            ("long-running", "released"): self.transition_executing_released,
+            ("memory", "released"): self.transition_memory_released,
+            ("missing", "fetch"): self.transition_missing_fetch,
+            ("missing", "released"): self.transition_missing_released,
+            ("missing", "error"): self.transition_generic_error,
+            ("ready", "error"): self.transition_generic_error,
+            ("ready", "executing"): self.transition_ready_executing,
+            ("ready", "released"): self.transition_generic_released,
+            ("released", "error"): self.transition_generic_error,
+            ("released", "fetch"): self.transition_released_fetch,
+            ("released", "forgotten"): self.transition_released_forgotten,
+            ("released", "memory"): self.transition_released_memory,
+            ("released", "waiting"): self.transition_released_waiting,
+            ("waiting", "constrained"): self.transition_waiting_constrained,
+            ("waiting", "ready"): self.transition_waiting_ready,
+            ("waiting", "released"): self.transition_generic_released,
         }
 
+        self._transition_counter = 0
         self.incoming_transfer_log = deque(maxlen=100000)
         self.incoming_count = 0
         self.outgoing_transfer_log = deque(maxlen=100000)
@@ -592,7 +631,7 @@ def __init__(
         self.available_resources = (resources or {}).copy()
         self.death_timeout = parse_timedelta(death_timeout)
 
-        self.extensions = dict()
+        self.extensions = {}
         if silence_logs:
             silence_logging(level=silence_logs)
 
@@ -645,7 +684,7 @@ def __init__(
                 or sys.maxsize,
             )
         else:
-            self.data = dict()
+            self.data = {}
 
         self.actors = {}
         self.loop = loop or IOLoop.current()
@@ -676,7 +715,7 @@ def __init__(
         self.batched_stream = BatchedSend(interval="2ms", loop=self.loop)
         self.name = name
         self.scheduler_delay = 0
-        self.stream_comms = dict()
+        self.stream_comms = {}
         self.heartbeat_active = False
         self._ipython_kernel = None
 
@@ -723,11 +762,12 @@ def __init__(
 
         stream_handlers = {
             "close": self.close,
-            "compute-task": self.add_task,
-            "cancel-compute": self.cancel_compute,
+            "cancel-compute": self.handle_cancel_compute,
+            "acquire-replicas": self.handle_acquire_replicas,
+            "compute-task": self.handle_compute_task,
             "free-keys": self.handle_free_keys,
-            "superfluous-data": self.handle_superfluous_data,
-            "steal-request": self.steal_request,
+            "remove-replicas": self.handle_remove_replicas,
+            "steal-request": self.handle_steal_request,
         }
 
         super().__init__(
@@ -751,6 +791,10 @@ def __init__(
             lambda: self.batched_stream.send({"op": "keep-alive"}), 60000
         )
         self.periodic_callbacks["keep-alive"] = pc
+
+        pc = PeriodicCallback(self.find_missing, 1000)
+        self.periodic_callbacks["find-missing"] = pc
+
         self._suspicious_count_limit = 10
         self._address = contact_address
 
@@ -835,6 +879,14 @@ def log_event(self, topic, msg):
             }
         )
 
+    @property
+    def executing_count(self) -> int:
+        return len(self._executing)
+
+    @property
+    def in_flight_tasks(self) -> int:
+        return len(self._in_flight_tasks)
+
     @property
     def worker_address(self):
         """For API compatibility with Nanny"""
@@ -993,7 +1045,6 @@ async def heartbeat(self):
         if self.heartbeat_active:
             logger.debug("Heartbeat skipped: channel busy")
             return
-
         self.heartbeat_active = True
         logger.debug("Heartbeat: %s", self.address)
         try:
@@ -1496,28 +1547,36 @@ async def get_data(
     # Local Execution #
     ###################
 
-    def update_data(self, comm=None, data=None, report=True, serializers=None):
+    def update_data(
+        self, comm=None, data=None, report=True, serializers=None, stimulus_id=None
+    ):
+        if stimulus_id is None:
+            stimulus_id = f"update-data-{time()}"
+        recommendations = {}
+        scheduler_messages = []
         for key, value in data.items():
-            ts = self.tasks.get(key)
-            if getattr(ts, "state", None) is not None:
-                self.transition(ts, "memory", value=value)
-            else:
+            try:
+                ts = self.tasks[key]
+                recommendations[ts] = ("memory", value)
+            except KeyError:
                 self.tasks[key] = ts = TaskState(key)
-                self.put_key_in_memory(ts, value)
+                recs, smsgs = self._put_key_in_memory(
+                    ts, value, stimulus_id=stimulus_id
+                )
+                recommendations.update(recs)
+                scheduler_messages += smsgs
                 ts.priority = None
                 ts.duration = None
-            ts.scheduler_holds_ref = True
 
             self.log.append((key, "receive-from-scatter"))
 
         if report:
+            scheduler_messages.append({"op": "add-keys", "keys": list(data)})
 
-            self.log.append(
-                ("Notifying scheduler about in-memory in update-data", list(data))
-            )
-            self.batched_stream.send({"op": "add-keys", "keys": list(data)})
-        info = {"nbytes": {k: sizeof(v) for k, v in data.items()}, "status": "OK"}
-        return info
+        self.transitions(recommendations, stimulus_id=stimulus_id)
+        for msg in scheduler_messages:
+            self.batched_stream.send(msg)
+        return {"nbytes": {k: sizeof(v) for k, v in data.items()}, "status": "OK"}
 
     def handle_free_keys(self, comm=None, keys=None, reason=None):
         """
@@ -1531,21 +1590,24 @@ def handle_free_keys(self, comm=None, keys=None, reason=None):
         upstream dependency.
         """
         self.log.append(("free-keys", keys, reason))
+        recommendations = {}
         for key in keys:
             ts = self.tasks.get(key)
-            if ts is not None:
-                ts.scheduler_holds_ref = False
-                self.release_key(key, report=False, reason=reason)
+            if ts:
+                recommendations[ts] = "released" if ts.dependents else "forgotten"
+
+        self.transitions(recommendations, stimulus_id=reason)
 
-    def handle_superfluous_data(self, keys=(), reason=None):
-        """Stream handler notifying the worker that it might be holding unreferenced, superfluous data.
+    def handle_remove_replicas(self, keys, stimulus_id):
+        """Stream handler notifying the worker that it might be holding unreferenced,
+        superfluous data.
 
-        This should not actually happen during ordinary operations and is only
-        intended to correct any erroneous state. An example where this is
-        necessary is if a worker fetches data for a downstream task but that
-        task is released before the data arrives.
-        In this case, the scheduler will notify the worker that it may be
-        holding this unnecessary data, if the worker hasn't released the data itself, already.
+        This should not actually happen during ordinary operations and is only intended
+        to correct any erroneous state. An example where this is necessary is if a
+        worker fetches data for a downstream task but that task is released before the
+        data arrives. In this case, the scheduler will notify the worker that it may be
+        holding this unnecessary data, if the worker hasn't released the data itself,
+        already.
 
         This handler does not guarantee the task nor the data to be actually
         released but only asks the worker to release the data on a best effort
@@ -1555,13 +1617,15 @@ def handle_superfluous_data(self, keys=(), reason=None):
 
         For stronger guarantees, see handler free_keys
         """
-        self.log.append(("Handle superfluous data", keys, reason))
-        for key in list(keys):
+        self.log.append(("remove-replicas", keys, stimulus_id))
+        recommendations = {}
+        for key in keys:
             ts = self.tasks.get(key)
-            if ts and not ts.scheduler_holds_ref:
-                self.release_key(key, reason=f"delete data: {reason}", report=False)
+            if ts and not ts.is_protected():
+                recommendations[ts] = "released" if ts.dependents else "forgotten"
+
+        self.transitions(recommendations=recommendations, stimulus_id=stimulus_id)
 
-        logger.debug("Worker %s -- Deleted %d keys", self.name, len(keys))
         return "OK"
 
     async def set_resources(self, **resources):
@@ -1582,7 +1646,7 @@ async def set_resources(self, **resources):
     # Task Management #
     ###################
 
-    def cancel_compute(self, key, reason):
+    def handle_cancel_compute(self, key, reason):
         """
         Cancel a task on a best effort basis. This is only possible while a task
         is in state `waiting` or `ready`.
@@ -1596,11 +1660,52 @@ def cancel_compute(self, key, reason):
             # scheduler side and therefore should not be assigned to a worker,
             # yet.
             assert not ts.dependents
-            self.release_key(key, reason=reason, report=False)
+            self.transition(ts, "released", stimulus_id=reason)
+
+    def handle_acquire_replicas(
+        self, comm=None, keys=None, priorities=None, who_has=None, stimulus_id=None
+    ):
+        recommendations = {}
+        scheduler_msgs = []
+        for k in keys:
+            recs, smsgs = self.register_acquire_internal(
+                k,
+                stimulus_id=stimulus_id,
+                priority=priorities[k],
+            )
+            recommendations.update(recs)
+            scheduler_msgs += smsgs
+
+        self.update_who_has(who_has, stimulus_id=stimulus_id)
+
+        for msg in scheduler_msgs:
+            self.batched_stream.send(msg)
+        self.transitions(recommendations, stimulus_id=stimulus_id)
 
-    def add_task(
+    def register_acquire_internal(self, key, priority, stimulus_id):
+        try:
+            ts = self.tasks[key]
+            logger.debug(
+                "Data task already known %s", {"task": ts, "stimulus_id": stimulus_id}
+            )
+        except KeyError:
+            self.tasks[key] = ts = TaskState(key)
+
+        self.log.append((key, "register-replica", ts.state, stimulus_id, time()))
+        ts.priority = ts.priority or priority
+
+        recommendations = {}
+        scheduler_msgs = []
+        if ts.state in ("released", "cancelled", "error"):
+            recommendations[ts] = "fetch"
+
+        return recommendations, scheduler_msgs
+
+    def handle_compute_task(
         self,
+        *,
         key,
+        # FIXME: This will break protocol
         function=None,
         args=None,
         kwargs=None,
@@ -1612,480 +1717,524 @@ def add_task(
         resource_restrictions=None,
         actor=False,
         annotations=None,
-        **kwargs2,
+        stimulus_id=None,
     ):
+        self.log.append((key, "compute-task", stimulus_id, time()))
         try:
-            runspec = SerializedTask(function, args, kwargs, task)
-            if key in self.tasks:
-                ts = self.tasks[key]
-                ts.scheduler_holds_ref = True
-                if ts.state == "memory":
-                    assert key in self.data or key in self.actors
-                    logger.debug(
-                        "Asked to compute pre-existing result: %s: %s", key, ts.state
-                    )
-                    self.send_task_state_to_scheduler(ts)
-                    return
-                if ts.state in IN_PLAY:
-                    return
-                if ts.state == "error":
-                    ts.exception = None
-                    ts.exception_text = ""
-                    ts.traceback = None
-                    ts.traceback_text = ""
-                else:
-                    # This is a scheduler re-assignment
-                    # Either `fetch` -> `waiting` or `flight` -> `waiting`
-                    self.log.append((ts.key, "re-adding key, new TaskState"))
-                    self.transition(ts, "waiting", runspec=runspec)
-            else:
-                self.log.append((key, "new"))
-                self.tasks[key] = ts = TaskState(
-                    key=key, runspec=SerializedTask(function, args, kwargs, task)
-                )
-                self.transition(ts, "waiting")
-            # TODO: move transition of `ts` to end of `add_task`
-            # This will require a chained recommendation transition system like
-            # the scheduler
-
-            if priority is not None:
-                priority = tuple(priority) + (self.generation,)
-                self.generation -= 1
-
-            if actor:
-                self.actors[ts.key] = None
-
-            ts.scheduler_holds_ref = True
-            ts.runspec = runspec
-            ts.priority = priority
-            ts.duration = duration
-            if resource_restrictions:
-                ts.resource_restrictions = resource_restrictions
-            ts.annotations = annotations
-
-            who_has = who_has or {}
-
-            for dependency, workers in who_has.items():
-                assert workers
-                if dependency not in self.tasks:
-                    # initial state is "new"
-                    # this dependency does not already exist on worker
-                    self.tasks[dependency] = dep_ts = TaskState(key=dependency)
-
-                    # link up to child / parents
-                    ts.dependencies.add(dep_ts)
-                    dep_ts.dependents.add(ts)
-
-                    # check to ensure task wasn't already executed and partially released
-                    # # TODO: make this less bad
-                    state = "fetch" if dependency not in self.data else "memory"
-
-                    # transition from new -> fetch handles adding dependency
-                    # to waiting_for_data
-                    discarded_self = False
-                    if self.address in workers and state == "fetch":
-                        discarded_self = True
-                        workers = set(workers)
-                        workers.discard(self.address)
-                        who_has[dependency] = tuple(workers)
-
-                    self.transition(dep_ts, state, who_has=workers)
-
-                    self.log.append(
-                        (
-                            dependency,
-                            "new-dep",
-                            dep_ts.state,
-                            f"requested by {ts.key}",
-                            discarded_self,
-                        )
-                    )
-
-                else:
-                    # task was already present on worker
-                    dep_ts = self.tasks[dependency]
-
-                    # link up to child / parents
-                    ts.dependencies.add(dep_ts)
-                    dep_ts.dependents.add(ts)
-
-                if dep_ts.state not in ("memory",):
-                    ts.waiting_for_data.add(dep_ts.key)
+            ts = self.tasks[key]
+            logger.debug(
+                "Asked to compute an already known task %s",
+                {"task": ts, "stimulus_id": stimulus_id},
+            )
+        except KeyError:
+            self.tasks[key] = ts = TaskState(key)
+
+        ts.runspec = SerializedTask(function, args, kwargs, task)
+
+        if priority is not None:
+            priority = tuple(priority) + (self.generation,)
+            self.generation -= 1
+
+        if actor:
+            self.actors[ts.key] = None
+
+        ts.exception = None
+        ts.traceback = None
+        ts.exception_text = ""
+        ts.traceback_text = ""
+        ts.priority = priority
+        ts.duration = duration
+        if resource_restrictions:
+            ts.resource_restrictions = resource_restrictions
+        ts.annotations = annotations
+
+        recommendations = {}
+        scheduler_msgs = []
+        for dependency in who_has:
+            recs, smsgs = self.register_acquire_internal(
+                key=dependency,
+                stimulus_id=stimulus_id,
+                priority=priority,
+            )
+            recommendations.update(recs)
+            scheduler_msgs += smsgs
+            dep_ts = self.tasks[dependency]
 
-            self.update_who_has(who_has=who_has)
-            if nbytes is not None:
-                for key, value in nbytes.items():
-                    self.tasks[key].nbytes = value
+            # link up to child / parents
+            ts.dependencies.add(dep_ts)
+            dep_ts.dependents.add(ts)
 
-            if ts.waiting_for_data:
-                self.data_needed.append(ts.key)
-            else:
-                self.transition(ts, "ready")
-            if self.validate:
-                for worker, keys in self.has_what.items():
-                    for k in keys:
-                        assert worker in self.tasks[k].who_has
-                if who_has:
-                    assert all(self.tasks[dep] in ts.dependencies for dep in who_has)
-                    assert all(self.tasks[dep.key] for dep in ts.dependencies)
-                    for dependency in ts.dependencies:
-                        self.validate_task(dependency)
-                    self.validate_task(ts)
-        except Exception as e:
-            logger.exception(e)
-            if LOG_PDB:
-                import pdb
+        if ts.state in {"ready", "executing", "waiting", "resumed"}:
+            pass
+        elif ts.state == "memory":
+            recommendations[ts] = "memory"
+            scheduler_msgs.append(self.get_task_state_for_scheduler(ts))
+        elif ts.state in {"released", "fetch", "flight", "missing", "cancelled"}:
+            recommendations[ts] = "waiting"
+        else:
+            raise RuntimeError(f"Unexpected task state encountered {ts} {stimulus_id}")
+
+        for msg in scheduler_msgs:
+            self.batched_stream.send(msg)
+        self.transitions(recommendations, stimulus_id=stimulus_id)
+
+        # We received new info, that's great but not related to the compute-task
+        # instruction
+        self.update_who_has(who_has, stimulus_id=stimulus_id)
+        if nbytes is not None:
+            for key, value in nbytes.items():
+                self.tasks[key].nbytes = value
+
+    def transition_missing_fetch(self, ts, *, stimulus_id):
+        self._missing_dep_flight.discard(ts)
+        ts.state = "fetch"
+        heapq.heappush(self.data_needed, (ts.priority, ts.key))
+        return {}, []
+
+    def transition_missing_released(self, ts, *, stimulus_id):
+        self._missing_dep_flight.discard(ts)
+        recommendations = self.release_key(ts.key, reason="missing->released")
+        assert ts.key in self.tasks
+        return recommendations, []
+
+    def transition_fetch_missing(self, ts, *, stimulus_id):
+        # handle_missing will append to self.data_needed if new workers are found
+        ts.state = "missing"
+        self._missing_dep_flight.add(ts)
+        return {}, []
+
+    def transition_released_fetch(self, ts, *, stimulus_id):
+        for w in ts.who_has:
+            self.pending_data_per_worker[w].append(ts.key)
+        ts.state = "fetch"
+        heapq.heappush(self.data_needed, (ts.priority, ts.key))
+        return {}, []
 
-                pdb.set_trace()
-            raise
+    def transition_generic_released(self, ts, *, stimulus_id):
+        recs = self.release_key(ts.key, reason=stimulus_id)
+        return recs, []
 
-    def transition(self, ts, finish, **kwargs):
-        if ts is None:
-            return
-        start = ts.state
-        if start == finish:
-            return
-        func = self._transitions[start, finish]
-        self.log.append((ts.key, start, finish))
-        state = func(ts, **kwargs)
-        if state and finish != state:
-            self.log.append((ts.key, start, finish, state))
-        ts.state = state or finish
+    def transition_released_waiting(self, ts, *, stimulus_id):
         if self.validate:
-            self.validate_task(ts)
-        self._notify_plugins("transition", ts.key, start, state or finish, **kwargs)
-
-    def transition_new_waiting(self, ts):
-        try:
-            if self.validate:
-                assert ts.state == "new"
-                assert ts.runspec is not None
-                assert not ts.who_has
-        except Exception as e:
-            logger.exception(e)
-            if LOG_PDB:
-                import pdb
+            assert ts.state == "released"
+            assert all(d.key in self.tasks for d in ts.dependencies)
+
+        recommendations = {}
+        ts.waiting_for_data.clear()
+        for dep_ts in ts.dependencies:
+            if not dep_ts.state == "memory":
+                ts.waiting_for_data.add(dep_ts)
+
+        if ts.waiting_for_data:
+            self.waiting_for_data_count += 1
+        elif ts.resource_restrictions:
+            recommendations[ts] = "constrained"
+        else:
+            recommendations[ts] = "ready"
 
-                pdb.set_trace()
-            raise
+        ts.state = "waiting"
+        return recommendations, []
 
-    def transition_new_fetch(self, ts, who_has):
-        try:
-            if self.validate:
-                assert ts.state == "new"
-                assert ts.runspec is None
-                assert who_has
+    def transition_fetch_flight(self, ts, worker, *, stimulus_id):
+        if self.validate:
+            assert ts.state == "fetch"
+            assert ts.who_has
+            assert ts.key not in self.data_needed
 
-            for dependent in ts.dependents:
-                dependent.waiting_for_data.add(ts.key)
+        ts.state = "flight"
+        ts.coming_from = worker
+        self._in_flight_tasks.add(ts)
+        return {}, []
 
-            ts.who_has.update(who_has)
-            for w in who_has:
-                self.has_what[w].add(ts.key)
-                self.pending_data_per_worker[w].append(ts.key)
+    def transition_memory_released(self, ts, *, stimulus_id):
+        recs = self.release_key(ts.key, reason=stimulus_id)
+        smsgs = [{"op": "release-worker-data", "key": ts.key}]
+        return recs, smsgs
 
-        except Exception as e:
-            logger.exception(e)
-            if LOG_PDB:
-                import pdb
+    def transition_waiting_constrained(self, ts, *, stimulus_id):
+        if self.validate:
+            assert ts.state == "waiting"
+            assert not ts.waiting_for_data
+            assert all(
+                dep.key in self.data or dep.key in self.actors
+                for dep in ts.dependencies
+            )
+            assert all(dep.state == "memory" for dep in ts.dependencies)
+            assert ts.key not in self.ready
+        ts.state = "constrained"
+        self.constrained.append(ts.key)
+        return {}, []
+
+    def transition_long_running_rescheduled(self, ts, *, stimulus_id):
+        recs = {ts: "released"}
+        smsgs = [{"op": "reschedule", "key": ts.key, "worker": self.address}]
+        return recs, smsgs
+
+    def transition_executing_rescheduled(self, ts, *, stimulus_id):
+        for resource, quantity in ts.resource_restrictions.items():
+            self.available_resources[resource] += quantity
+        self._executing.discard(ts)
 
-                pdb.set_trace()
-            raise
+        recs = {ts: "released"}
+        smsgs = [{"op": "reschedule", "key": ts.key, "worker": self.address}]
+        return recs, smsgs
 
-    def transition_fetch_waiting(self, ts, runspec):
-        """This is a rescheduling transition that occurs after a worker failure.
-        A task was available from another worker but that worker died and the
-        scheduler reassigned the task for computation here.
-        """
-        try:
-            if self.validate:
-                assert ts.state == "fetch"
-                assert ts.runspec is None
-                assert runspec is not None
+    def transition_waiting_ready(self, ts, *, stimulus_id):
+        if self.validate:
+            assert ts.state == "waiting"
+            assert ts.key not in self.ready
+            assert not ts.waiting_for_data
+            for dep in ts.dependencies:
+                assert dep.key in self.data or dep.key in self.actors
+                assert dep.state == "memory"
 
-            ts.runspec = runspec
+        ts.state = "ready"
+        heapq.heappush(self.ready, (ts.priority, ts.key))
 
-            # remove any stale entries in `has_what`
-            for worker in self.has_what.keys():
-                self.has_what[worker].discard(ts.key)
+        return {}, []
 
-            # clear `who_has` of stale info
-            ts.who_has.clear()
+    def transition_generic_error(
+        self, ts, exception, traceback, exception_text, traceback_text, *, stimulus_id
+    ):
+        ts.exception = exception
+        ts.traceback = traceback
+        ts.exception_text = exception_text
+        ts.traceback_text = traceback_text
+        smsgs = [self.get_task_state_for_scheduler(ts)]
+        ts.state = "error"
+        return {}, smsgs
+
+    def transition_executing_error(
+        self, ts, exception, traceback, exception_text, traceback_text, *, stimulus_id
+    ):
+        for resource, quantity in ts.resource_restrictions.items():
+            self.available_resources[resource] += quantity
+        self._executing.discard(ts)
+        return self.transition_generic_error(
+            ts,
+            exception,
+            traceback,
+            exception_text,
+            traceback_text,
+            stimulus_id=stimulus_id,
+        )
 
-        except Exception as e:
-            logger.exception(e)
-            if LOG_PDB:
-                import pdb
+    def transition_rescheduled_next(self, ts, *, stimulus_id):
+        next_state = ts._next
+        recs = self.release_key(ts.key, reason=stimulus_id)
+        recs[ts] = next_state
+        return recs, []
+
+    def transition_cancelled_fetch(self, ts, *, stimulus_id):
+        if ts.done:
+            return {ts: "released"}, []
+        elif ts._previous == "flight":
+            ts.state = ts._previous
+            return {}, []
+        else:
+            assert ts._previous == "executing"
+            return {ts: ("resumed", "fetch")}, []
+
+    def transition_cancelled_resumed(self, ts, next, *, stimulus_id):
+        ts._next = next
+        ts.state = "resumed"
+        return {}, []
+
+    def transition_cancelled_waiting(self, ts, *, stimulus_id):
+        if ts.done:
+            return {ts: "released"}, []
+        elif ts._previous == "executing":
+            ts.state = ts._previous
+            return {}, []
+        else:
+            assert ts._previous == "flight"
+            return {ts: ("resumed", "waiting")}, []
+
+    def transition_cancelled_forgotten(self, ts, *, stimulus_id):
+        ts._next = "forgotten"
+        if not ts.done:
+            return {}, []
+        return {ts: "released"}, []
+
+    def transition_cancelled_released(self, ts, *, stimulus_id):
+        if not ts.done:
+            ts._next = "released"
+            return {}, []
+        next_state = ts._next
+        self._executing.discard(ts)
+        self._in_flight_tasks.discard(ts)
 
-                pdb.set_trace()
-            raise
+        for resource, quantity in ts.resource_restrictions.items():
+            self.available_resources[resource] += quantity
+        recommendations = self.release_key(ts.key, reason=stimulus_id)
+        recommendations[ts] = next_state or "released"
+        return recommendations, []
+
+    def transition_executing_released(self, ts, *, stimulus_id):
+        ts._previous = ts.state
+        # See https://github.com/dask/distributed/pull/5046#discussion_r685093940
+        ts.state = "cancelled"
+        ts.done = False
+        return {}, []
+
+    def transition_long_running_memory(self, ts, value=no_value, *, stimulus_id):
+        self.executed_count += 1
+        return self.transition_generic_memory(ts, value=value, stimulus_id=stimulus_id)
+
+    def transition_generic_memory(self, ts, value=no_value, *, stimulus_id):
+        if value is no_value and ts.key not in self.data:
+            raise RuntimeError(
+                f"Tried to transition task {ts} to `memory` without data available"
+            )
 
-    def transition_flight_waiting(self, ts, runspec):
-        """This is a rescheduling transition that occurs after
-        a worker failure.  A task was in flight from another worker to this
-        worker when that worker died and the scheduler reassigned the task for
-        computation here.
-        """
-        try:
-            if self.validate:
-                assert ts.state == "flight"
-                assert ts.runspec is None
-                assert runspec is not None
+        if ts.resource_restrictions is not None:
+            for resource, quantity in ts.resource_restrictions.items():
+                self.available_resources[resource] += quantity
 
-            ts.runspec = runspec
+        self._executing.discard(ts)
+        self._in_flight_tasks.discard(ts)
+        ts.coming_from = None
 
-            # remove any stale entries in `has_what`
-            for worker in self.has_what.keys():
-                self.has_what[worker].discard(ts.key)
+        recs, smsgs = self._put_key_in_memory(ts, value, stimulus_id=stimulus_id)
+        smsgs.append(self.get_task_state_for_scheduler(ts))
+        return recs, smsgs
 
-            # clear `who_has` of stale info
-            ts.who_has.clear()
+    def transition_executing_memory(self, ts, value=no_value, *, stimulus_id):
+        if self.validate:
+            assert ts.state == "executing" or ts.key in self.long_running
+            assert not ts.waiting_for_data
+            assert ts.key not in self.ready
 
-        except Exception as e:
-            logger.exception(e)
-            if LOG_PDB:
-                import pdb
+        self._executing.discard(ts)
+        self.executed_count += 1
+        return self.transition_generic_memory(ts, value=value, stimulus_id=stimulus_id)
 
-                pdb.set_trace()
-            raise
+    def transition_constrained_released(self, ts, *, stimulus_id):
+        recs = self.release_key(ts.key, reason=stimulus_id)
+        return recs, []
 
-    def transition_fetch_flight(self, ts, worker=None):
-        try:
-            if self.validate:
-                assert ts.state == "fetch"
-                assert ts.dependents
+    def transition_constrained_executing(self, ts, *, stimulus_id):
+        if self.validate:
+            assert not ts.waiting_for_data
+            assert ts.key not in self.data
+            assert ts.state in READY
+            assert ts.key not in self.ready
+            for dep in ts.dependencies:
+                assert dep.key in self.data or dep.key in self.actors
 
-            ts.coming_from = worker
-            self.in_flight_tasks += 1
-        except Exception as e:
-            logger.exception(e)
-            if LOG_PDB:
-                import pdb
+        for resource, quantity in ts.resource_restrictions.items():
+            self.available_resources[resource] -= quantity
+        ts.state = "executing"
+        self._executing.add(ts)
+        self.loop.add_callback(self.execute, ts.key, stimulus_id=stimulus_id)
+        return {}, []
 
-                pdb.set_trace()
-            raise
+    def transition_ready_executing(self, ts, *, stimulus_id):
+        if self.validate:
+            assert not ts.waiting_for_data
+            assert ts.key not in self.data
+            assert ts.state in READY
+            assert ts.key not in self.ready
+            assert all(
+                dep.key in self.data or dep.key in self.actors
+                for dep in ts.dependencies
+            )
 
-    def transition_flight_fetch(self, ts):
-        try:
-            if self.validate:
-                assert ts.state == "flight"
+        ts.state = "executing"
+        self._executing.add(ts)
+        self.loop.add_callback(self.execute, ts.key, stimulus_id=stimulus_id)
+        return {}, []
 
-            self.in_flight_tasks -= 1
-            ts.coming_from = None
-            ts.runspec = None
+    def transition_flight_fetch(self, ts, *, stimulus_id):
+        self._in_flight_tasks.discard(ts)
+        ts.coming_from = None
 
-            if not ts.who_has:
-                if ts.key not in self._missing_dep_flight:
-                    self._missing_dep_flight.add(ts.key)
-                    logger.info("Task %s does not know who has", ts)
-                    self.loop.add_callback(self.handle_missing_dep, ts)
-            for w in ts.who_has:
-                self.pending_data_per_worker[w].append(ts.key)
-            for dependent in ts.dependents:
-                dependent.waiting_for_data.add(ts.key)
-                if dependent.state == "waiting":
-                    self.data_needed.append(dependent.key)
+        for w in ts.who_has:
+            self.pending_data_per_worker[w].append(ts.key)
+        ts.state = "fetch"
+        heapq.heappush(self.data_needed, (ts.priority, ts.key))
 
-        except Exception as e:
-            logger.exception(e)
-            if LOG_PDB:
-                import pdb
+        return {}, []
 
-                pdb.set_trace()
-            raise
+    def transition_flight_error(
+        self, ts, exception, traceback, exception_text, traceback_text, *, stimulus_id
+    ):
+        self._in_flight_tasks.discard(ts)
+        ts.coming_from = None
+        return self.transition_generic_error(
+            ts,
+            exception,
+            traceback,
+            exception_text,
+            traceback_text,
+            stimulus_id=stimulus_id,
+        )
 
-    def transition_flight_memory(self, ts, value=None):
-        try:
-            if self.validate:
-                assert ts.state == "flight"
+    def transition_flight_released(self, ts, *, stimulus_id):
+        ts._previous = "flight"
+        # See https://github.com/dask/distributed/pull/5046#discussion_r685093940
+        ts.state = "cancelled"
+        return {}, []
 
-            self.in_flight_tasks -= 1
-            ts.coming_from = None
-            self.put_key_in_memory(ts, value)
-            for dependent in ts.dependents:
-                try:
-                    dependent.waiting_for_data.remove(ts.key)
-                    self.waiting_for_data_count -= 1
-                except KeyError:
-                    pass
+    def transition_cancelled_memory(self, ts, value, *, stimulus_id):
+        return {ts: ts._next}, []
 
-            self.log.append(("Notifying scheduler about in-memory", ts.key))
-            self.batched_stream.send({"op": "add-keys", "keys": [ts.key]})
+    def transition_executing_long_running(self, ts, compute_duration, *, stimulus_id):
+        ts.state = "long-running"
+        self._executing.discard(ts)
+        self.long_running.add(ts.key)
+        smsgs = [
+            {
+                "op": "long-running",
+                "key": ts.key,
+                "compute_duration": compute_duration,
+            }
+        ]
 
-        except Exception as e:
-            logger.exception(e)
-            if LOG_PDB:
-                import pdb
+        self.io_loop.add_callback(self.ensure_computing)
+        return {}, smsgs
 
-                pdb.set_trace()
-            raise
+    def transition_released_memory(self, ts, value, *, stimulus_id):
+        recs, smsgs = self._put_key_in_memory(ts, value, stimulus_id=stimulus_id)
+        smsgs.append({"op": "add-keys", "keys": [ts.key]})
+        return recs, smsgs
 
-    def transition_waiting_ready(self, ts):
-        try:
-            if self.validate:
-                assert ts.state == "waiting"
-                assert not ts.waiting_for_data
-                assert all(
-                    dep.key in self.data or dep.key in self.actors
-                    for dep in ts.dependencies
-                )
-                assert all(dep.state == "memory" for dep in ts.dependencies)
-                assert ts.key not in self.ready
+    def transition_flight_memory(self, ts, value, *, stimulus_id):
+        self._in_flight_tasks.discard(ts)
+        ts.coming_from = None
+        recs, smsgs = self._put_key_in_memory(ts, value, stimulus_id=stimulus_id)
+        smsgs.append({"op": "add-keys", "keys": [ts.key]})
+        return recs, smsgs
 
-            self.has_what[self.address].discard(ts.key)
+    def transition_released_forgotten(self, ts, *, stimulus_id):
+        recommendations = {}
+        # Dependents _should_ be released by the scheduler before this
+        if self.validate:
+            assert not any(d.state != "forgotten" for d in ts.dependents)
+        for dep in ts.dependencies:
+            dep.dependents.discard(ts)
+            if dep.state == "released" and not dep.dependents:
+                recommendations[dep] = "forgotten"
 
-            if ts.resource_restrictions is not None:
-                self.constrained.append(ts.key)
-                return "constrained"
-            else:
-                heapq.heappush(self.ready, (ts.priority, ts.key))
-        except Exception as e:
-            logger.exception(e)
-            if LOG_PDB:
-                import pdb
+        # Mark state as forgotten in case it is still referenced
+        ts.state = "forgotten"
+        self.tasks.pop(ts.key, None)
+        return recommendations, []
 
-                pdb.set_trace()
-            raise
+    def _transition(self, ts, finish, *args, stimulus_id, **kwargs):
+        if isinstance(finish, tuple):
+            # the concatenated transition path might need to access the tuple
+            assert not args
+            finish, *args = finish
 
-    def transition_waiting_done(self, ts, value=None):
-        try:
-            if self.validate:
-                assert ts.state == "waiting"
-                assert ts.key not in self.ready
+        if ts is None or ts.state == finish:
+            return {}, []
 
-            self.waiting_for_data_count -= len(ts.waiting_for_data)
-            ts.waiting_for_data.clear()
-            if value is not None:
-                self.put_key_in_memory(ts, value)
-            self.send_task_state_to_scheduler(ts)
-        except Exception as e:
-            logger.exception(e)
-            if LOG_PDB:
-                import pdb
+        start = ts.state
+        func = self._transitions_table.get((start, finish))
 
-                pdb.set_trace()
-            raise
+        if func is not None:
+            self._transition_counter += 1
+            recs, smsgs = func(ts, *args, stimulus_id=stimulus_id, **kwargs)
+            self._notify_plugins("transition", ts.key, start, finish, **kwargs)
 
-    def transition_ready_executing(self, ts):
-        try:
-            if self.validate:
-                assert not ts.waiting_for_data
-                assert ts.key not in self.data
-                assert ts.state in READY
-                assert ts.key not in self.ready
-                assert all(
-                    dep.key in self.data or dep.key in self.actors
-                    for dep in ts.dependencies
+        elif "released" not in (start, finish):
+            # start -> "released" -> finish
+            try:
+                recs, smsgs = self._transition(ts, "released", stimulus_id=stimulus_id)
+                v = recs.get(ts, (finish, *args))
+                if isinstance(v, tuple):
+                    v_state, *v_args = v
+                else:
+                    v_state, v_args = v, ()
+                b_recs, b_smsgs = self._transition(
+                    ts, v_state, *v_args, stimulus_id=stimulus_id
                 )
+                recs.update(b_recs)
+                smsgs += b_smsgs
+            except InvalidTransition:
+                raise InvalidTransition(
+                    f"Impossible transition from {start} to {finish} for {ts.key}"
+                ) from None
 
-            self.executing_count += 1
-            self.loop.add_callback(self.execute, ts.key)
-        except Exception as e:
-            logger.exception(e)
-            if LOG_PDB:
-                import pdb
-
-                pdb.set_trace()
-            raise
+        else:
+            raise InvalidTransition(
+                f"Impossible transition from {start} to {finish} for {ts.key}"
+            )
 
-    def transition_ready_error(self, ts):
-        if self.validate:
-            assert ts.exception is not None
-            assert ts.traceback is not None
-            assert ts.exception_text
-            assert ts.traceback_text
-        self.send_task_state_to_scheduler(ts)
-
-    def transition_ready_memory(self, ts, value=no_value):
-        if value is not no_value:
-            self.put_key_in_memory(ts, value=value)
-        self.send_task_state_to_scheduler(ts)
-
-    def transition_constrained_executing(self, ts):
-        self.transition_ready_executing(ts)
-        for resource, quantity in ts.resource_restrictions.items():
-            self.available_resources[resource] -= quantity
+        self.log.append(
+            (
+                ts.key,
+                start,
+                ts.state,
+                {ts.key: new for ts, new in recs.items()},
+                stimulus_id,
+                time(),
+            )
+        )
+        return recs, smsgs
 
-        if self.validate:
-            assert all(v >= 0 for v in self.available_resources.values())
+    def transition(self, ts, finish: str, *, stimulus_id, **kwargs):
+        """Transition a key from its current state to the finish state
 
-    def transition_executing_done(self, ts, value=no_value, report=True):
-        try:
-            if self.validate:
-                assert ts.state == "executing" or ts.key in self.long_running
-                assert not ts.waiting_for_data
-                assert ts.key not in self.ready
-
-            out = None
-            if ts.resource_restrictions is not None:
-                for resource, quantity in ts.resource_restrictions.items():
-                    self.available_resources[resource] += quantity
+        Examples
+        --------
+        >>> self.transition('x', 'waiting')
+        {'x': 'processing'}
 
-            if ts.state == "executing":
-                self.executing_count -= 1
-                self.executed_count += 1
-            elif ts.state == "long-running":
-                self.long_running.remove(ts.key)
+        Returns
+        -------
+        Dictionary of recommendations for future transitions
 
-            if value is not no_value:
-                try:
-                    self.put_key_in_memory(ts, value, transition=False)
-                except Exception as e:
-                    logger.info("Failed to put key in memory", exc_info=True)
-                    msg = error_message(e)
-                    ts.exception = msg["exception"]
-                    ts.exception_text = msg["exception_text"]
-                    ts.traceback = msg["traceback"]
-                    ts.traceback_text = msg["traceback_text"]
-                    ts.state = "error"
-                    out = "error"
-                    for d in ts.dependents:
-                        d.waiting_for_data.add(ts.key)
-
-            if report and self.batched_stream and self.status == Status.running:
-                self.send_task_state_to_scheduler(ts)
-            else:
-                raise CommClosedError
+        See Also
+        --------
+        Scheduler.transitions: transitive version of this function
+        """
+        recs, smsgs = self._transition(ts, finish, stimulus_id=stimulus_id, **kwargs)
+        for msg in smsgs:
+            self.batched_stream.send(msg)
+        self.transitions(recs, stimulus_id=stimulus_id)
 
-            return out
+    def transitions(self, recommendations: dict, *, stimulus_id):
+        """Process transitions until none are left
 
-        except OSError:
-            logger.info("Comm closed")
-        except Exception as e:
-            logger.exception(e)
-            if LOG_PDB:
-                import pdb
+        This includes feedback from previous transitions and continues until we
+        reach a steady state
+        """
+        smsgs = []
 
-                pdb.set_trace()
-            raise
+        remaining_recs = recommendations.copy()
+        tasks = set()
+        while remaining_recs:
+            ts, finish = remaining_recs.popitem()
+            tasks.add(ts)
+            a_recs, a_smsgs = self._transition(ts, finish, stimulus_id=stimulus_id)
+            remaining_recs.update(a_recs)
+            smsgs += a_smsgs
 
-    def transition_executing_long_running(self, ts, compute_duration=None):
-        try:
-            if self.validate:
-                assert ts.state == "executing"
+        if self.validate:
+            # Full state validation is very expensive
+            for ts in tasks:
+                self.validate_task(ts)
 
-            self.executing_count -= 1
-            self.long_running.add(ts.key)
-            self.batched_stream.send(
-                {
-                    "op": "long-running",
-                    "key": ts.key,
-                    "compute_duration": compute_duration,
-                }
+        if not self.batched_stream.closed():
+            for msg in smsgs:
+                self.batched_stream.send(msg)
+        else:
+            logger.debug(
+                "BatchedSend closed while transitioning tasks. %d tasks not sent.",
+                len(smsgs),
             )
 
-            self.ensure_computing()
-        except Exception as e:
-            logger.exception(e)
-            if LOG_PDB:
-                import pdb
-
-                pdb.set_trace()
-            raise
-
-    def maybe_transition_long_running(self, ts, compute_duration=None):
+    def maybe_transition_long_running(self, ts, *, stimulus_id, compute_duration=None):
         if ts.state == "executing":
-            self.transition(ts, "long-running", compute_duration=compute_duration)
+            self.transition(
+                ts,
+                "long-running",
+                compute_duration=compute_duration,
+                stimulus_id=stimulus_id,
+            )
+            assert ts.state == "long-running"
 
     def stateof(self, key):
         ts = self.tasks[key]
@@ -2111,123 +2260,70 @@ def story(self, *keys):
         ]
 
     def ensure_communicating(self):
-        changed = True
-        try:
-            while (
-                changed
-                and self.data_needed
-                and len(self.in_flight_workers) < self.total_out_connections
-            ):
-                changed = False
-                logger.debug(
-                    "Ensure communicating.  Pending: %d.  Connections: %d/%d",
-                    len(self.data_needed),
-                    len(self.in_flight_workers),
-                    self.total_out_connections,
-                )
+        stimulus_id = f"ensure-communicating-{time()}"
+        skipped_worker_in_flight = []
 
-                key = self.data_needed[0]
+        while self.data_needed and (
+            len(self.in_flight_workers) < self.total_out_connections
+            or self.comm_nbytes < self.comm_threshold_bytes
+        ):
+            logger.debug(
+                "Ensure communicating. Pending: %d. Connections: %d/%d",
+                len(self.data_needed),
+                len(self.in_flight_workers),
+                self.total_out_connections,
+            )
 
-                if key not in self.tasks:
-                    self.data_needed.popleft()
-                    changed = True
-                    continue
+            _, key = heapq.heappop(self.data_needed)
 
+            try:
                 ts = self.tasks[key]
-                if ts.state != "waiting":
-                    self.log.append((key, "communication pass"))
-                    self.data_needed.popleft()
-                    changed = True
-                    continue
-
-                dependencies = ts.dependencies
-                if self.validate:
-                    assert all(dep.key in self.tasks for dep in dependencies)
-
-                dependencies_fetch = set()
-                dependencies_missing = set()
-                for dependency_ts in dependencies:
-                    if dependency_ts.state == "fetch":
-                        if not dependency_ts.who_has:
-                            dependencies_missing.add(dependency_ts)
-                        else:
-                            dependencies_fetch.add(dependency_ts)
+            except KeyError:
+                continue
 
-                del dependencies, dependency_ts
+            if ts.state != "fetch":
+                continue
 
-                if dependencies_missing:
-                    missing_deps2 = {
-                        dep
-                        for dep in dependencies_missing
-                        if dep.key not in self._missing_dep_flight
-                    }
-                    for dep in missing_deps2:
-                        self._missing_dep_flight.add(dep.key)
-                    if missing_deps2:
-                        logger.info(
-                            "Can't find dependencies %s for key %s",
-                            missing_deps2.copy(),
-                            key,
-                        )
-                        self.loop.add_callback(self.handle_missing_dep, *missing_deps2)
-                    dependencies_fetch -= dependencies_missing
+            if not ts.who_has:
+                self.transition(ts, "missing", stimulus_id=stimulus_id)
+                continue
 
-                self.log.append(
-                    ("gather-dependencies", key, {d.key for d in dependencies_fetch})
-                )
+            workers = [w for w in ts.who_has if w not in self.in_flight_workers]
+            if not workers:
+                skipped_worker_in_flight.append((ts.priority, ts.key))
+                continue
 
-                in_flight = False
+            host = get_address_host(self.address)
+            local = [w for w in workers if get_address_host(w) == host]
+            if local:
+                worker = random.choice(local)
+            else:
+                worker = random.choice(list(workers))
+            assert worker != self.address
 
-                while dependencies_fetch and (
-                    len(self.in_flight_workers) < self.total_out_connections
-                    or self.comm_nbytes < self.comm_threshold_bytes
-                ):
-                    to_gather_ts = dependencies_fetch.pop()
-
-                    workers = [
-                        w
-                        for w in to_gather_ts.who_has
-                        if w not in self.in_flight_workers
-                    ]
-                    if not workers:
-                        in_flight = True
-                        continue
-                    host = get_address_host(self.address)
-                    local = [w for w in workers if get_address_host(w) == host]
-                    if local:
-                        worker = random.choice(local)
-                    else:
-                        worker = random.choice(list(workers))
-                    to_gather, total_nbytes = self.select_keys_for_gather(
-                        worker, to_gather_ts.key
-                    )
-                    self.comm_nbytes += total_nbytes
-                    self.in_flight_workers[worker] = to_gather
-                    for d in to_gather:
-                        dependencies_fetch.discard(self.tasks.get(d))
-                        self.transition(self.tasks[d], "flight", worker=worker)
-                    assert not worker == self.address
-                    self.loop.add_callback(
-                        self.gather_dep,
-                        worker=worker,
-                        to_gather=to_gather,
-                        total_nbytes=total_nbytes,
-                        cause=ts,
-                    )
-                    changed = True
+            to_gather, total_nbytes = self.select_keys_for_gather(worker, ts.key)
 
-                if not dependencies_fetch and not in_flight:
-                    self.data_needed.popleft()
+            self.log.append(
+                ("gather-dependencies", worker, to_gather, "stimulus", time())
+            )
 
-        except Exception as e:
-            logger.exception(e)
-            if LOG_PDB:
-                import pdb
+            self.comm_nbytes += total_nbytes
+            self.in_flight_workers[worker] = to_gather
+            recommendations = {self.tasks[d]: ("flight", worker) for d in to_gather}
+            self.transitions(recommendations=recommendations, stimulus_id=stimulus_id)
+
+            self.loop.add_callback(
+                self.gather_dep,
+                worker=worker,
+                to_gather=to_gather,
+                total_nbytes=total_nbytes,
+                stimulus_id=stimulus_id,
+            )
 
-                pdb.set_trace()
-            raise
+        for el in skipped_worker_in_flight:
+            heapq.heappush(self.data_needed, el)
 
-    def send_task_state_to_scheduler(self, ts):
+    def get_task_state_for_scheduler(self, ts):
         if ts.key in self.data or self.actors.get(ts.key):
             typ = ts.type
             if ts.nbytes is None or typ is None:
@@ -2267,45 +2363,50 @@ def send_task_state_to_scheduler(self, ts):
             }
         else:
             logger.error("Key not ready to send to worker, %s: %s", ts.key, ts.state)
-            return
-
+            return None
         if ts.startstops:
             d["startstops"] = ts.startstops
-        self.batched_stream.send(d)
+        return d
 
-    def put_key_in_memory(self, ts, value, transition=True):
+    def _put_key_in_memory(self, ts, value, *, stimulus_id):
         if ts.key in self.data:
             ts.state = "memory"
-            return
+            return {}, []
 
+        recommendations = {}
+        scheduler_messages = []
         if ts.key in self.actors:
             self.actors[ts.key] = value
-
         else:
             start = time()
-            self.data[ts.key] = value
-            ts.state = "memory"
+            try:
+                self.data[ts.key] = value
+            except Exception as e:
+                msg = error_message(e)
+                ts.exception = msg["exception"]
+                ts.traceback = msg["traceback"]
+                recommendations[ts] = ("error", msg["exception"], msg["traceback"])
+                return recommendations, []
             stop = time()
             if stop - start > 0.020:
                 ts.startstops.append(
                     {"action": "disk-write", "start": start, "stop": stop}
                 )
 
+        ts.state = "memory"
         if ts.nbytes is None:
             ts.nbytes = sizeof(value)
 
         ts.type = type(value)
 
         for dep in ts.dependents:
-            try:
-                dep.waiting_for_data.remove(ts.key)
+            dep.waiting_for_data.discard(ts)
+            if not dep.waiting_for_data and dep.state == "waiting":
                 self.waiting_for_data_count -= 1
-            except KeyError:
-                pass
-            if not dep.waiting_for_data:
-                self.transition(dep, "ready")
+                recommendations[dep] = "ready"
 
-        self.log.append((ts.key, "put-in-memory"))
+        self.log.append((ts.key, "put-in-memory", stimulus_id, time()))
+        return recommendations, scheduler_messages
 
     def select_keys_for_gather(self, worker, dep):
         assert isinstance(dep, str)
@@ -2340,7 +2441,8 @@ async def gather_dep(
         worker: str,
         to_gather: Iterable[str],
         total_nbytes: int,
-        cause: TaskState,
+        *,
+        stimulus_id,
     ):
         """Gather dependencies for a task from a worker who has them
 
@@ -2354,29 +2456,41 @@ async def gather_dep(
             as some dependencies may already be present on this worker.
         total_nbytes : int
             Total number of bytes for all the dependencies in to_gather combined
-        cause : TaskState
-            Task we want to gather dependencies for
         """
-
-        if self.validate:
-            self.validate_state()
+        cause = None
         if self.status != Status.running:
             return
+
         with log_errors():
             response = {}
             to_gather_keys = set()
             try:
-                if self.validate:
-                    self.validate_state()
+                found_dependent_for_cause = False
                 for dependency_key in to_gather:
                     dependency_ts = self.tasks.get(dependency_key)
                     if dependency_ts and dependency_ts.state == "flight":
                         to_gather_keys.add(dependency_key)
+                        if not found_dependent_for_cause:
+                            cause = dependency_ts
+                            # For diagnostics we want to attach the transfer to
+                            # a single task. this task is typically the next to
+                            # be executed but since we're fetching tasks for
+                            # potentially many dependents, an exact match is not
+                            # possible. If there are no dependents, this is a
+                            # pure replica fetch
+                            for dependent in dependency_ts.dependents:
+                                cause = dependent
+                                found_dependent_for_cause = True
+                                break
                 # Keep namespace clean since this func is long and has many
                 # dep*, *ts* variables
+
+                assert cause is not None
                 del to_gather, dependency_key, dependency_ts
 
-                self.log.append(("request-dep", cause.key, worker, to_gather_keys))
+                self.log.append(
+                    ("request-dep", worker, to_gather_keys, stimulus_id, time())
+                )
                 logger.debug(
                     "Request %d keys for task %s from %s",
                     len(to_gather_keys),
@@ -2389,15 +2503,17 @@ async def gather_dep(
                     self.rpc, to_gather_keys, worker, who=self.address
                 )
                 stop = time()
-
                 if response["status"] == "busy":
-                    self.log.append(("busy-gather", worker, to_gather_keys))
-                    for key in to_gather_keys:
-                        ts = self.tasks.get(key)
-                        if ts and ts.state == "flight":
-                            self.transition(ts, "fetch")
                     return
 
+                data = {k: v for k, v in response["data"].items() if k in self.tasks}
+                lost_keys = response["data"].keys() - data.keys()
+
+                if lost_keys:
+                    self.log.append(("lost-during-gather", lost_keys, stimulus_id))
+
+                total_bytes = sum(self.tasks[key].get_nbytes() for key in data)
+
                 cause.startstops.append(
                     {
                         "action": "transfer",
@@ -2406,12 +2522,6 @@ async def gather_dep(
                         "source": worker,
                     }
                 )
-
-                total_bytes = sum(
-                    self.tasks[key].get_nbytes()
-                    for key in response["data"]
-                    if key in self.tasks
-                )
                 duration = (stop - start) or 0.010
                 bandwidth = total_bytes / duration
                 self.incoming_transfer_log.append(
@@ -2420,11 +2530,7 @@ async def gather_dep(
                         "stop": stop + self.scheduler_delay,
                         "middle": (start + stop) / 2.0 + self.scheduler_delay,
                         "duration": duration,
-                        "keys": {
-                            key: self.tasks[key].nbytes
-                            for key in response["data"]
-                            if key in self.tasks
-                        },
+                        "keys": {key: self.tasks[key].nbytes for key in data},
                         "total": total_bytes,
                         "bandwidth": bandwidth,
                         "who": worker,
@@ -2447,13 +2553,17 @@ async def gather_dep(
                 self.counters["transfer-count"].add(len(response["data"]))
                 self.incoming_count += 1
 
-                self.log.append(("receive-dep", worker, list(response["data"])))
+                self.log.append(
+                    ("receive-dep", worker, set(response["data"]), stimulus_id, time())
+                )
 
             except OSError:
                 logger.exception("Worker stream died during communication: %s", worker)
                 has_what = self.has_what.pop(worker)
                 self.pending_data_per_worker.pop(worker)
-                self.log.append(("receive-dep-failed", worker, has_what))
+                self.log.append(
+                    ("receive-dep-failed", worker, has_what, stimulus_id, time())
+                )
                 for d in has_what:
                     ts = self.tasks[d]
                     ts.who_has.remove(worker)
@@ -2470,179 +2580,95 @@ async def gather_dep(
                 busy = response.get("status", "") == "busy"
                 data = response.get("data", {})
 
-                # FIXME: We should not handle keys which were skipped by this coro. to_gather_keys is only a subset
-                assert set(to_gather_keys).issubset(
-                    set(self.in_flight_workers.get(worker))
-                )
+                if busy:
+                    self.log.append(
+                        ("busy-gather", worker, to_gather_keys, stimulus_id, time())
+                    )
 
-                for d in self.in_flight_workers.pop(worker):
+                recommendations = {}
+                deps_to_iter = self.in_flight_workers.pop(worker)
+
+                for d in deps_to_iter:
                     ts = self.tasks.get(d)
-                    try:
-                        if not busy and d in data:
-                            self.transition(ts, "memory", value=data[d])
-                        elif ts is None or ts.state == "executing":
-                            self.log.append(("already-executing", d))
-                            self.release_key(d, reason="already executing at gather")
-                        elif ts.state == "flight" and not ts.dependents:
-                            self.log.append(("flight no-dependents", d))
-                            self.release_key(
-                                d, reason="In-flight task no longer has dependents."
-                            )
-                        elif (
-                            not busy
-                            and d not in data
-                            and ts.dependents
-                            and ts.state != "memory"
-                        ):
-                            ts.who_has.discard(worker)
-                            self.has_what[worker].discard(ts.key)
-                            self.log.append(("missing-dep", d))
-                            self.batched_stream.send(
-                                {
-                                    "op": "missing-data",
-                                    "errant_worker": worker,
-                                    "key": d,
-                                }
-                            )
-                            self.transition(ts, "fetch")
-                        elif ts.state not in ("ready", "memory"):
-                            self.transition(ts, "fetch")
-                        else:
-                            logger.debug(
-                                "Unexpected task state encountered for %r after gather_dep",
-                                ts,
-                            )
-                    except Exception as exc:
-                        emsg = error_message(exc)
-                        assert ts is not None, ts
-                        self.log.append(
-                            (ts.key, "except-gather-dep-result", emsg, time())
-                        )
-                        # FIXME: We currently cannot release this task and its
-                        # dependent safely
-                        logger.debug(
-                            "Exception occured while handling `gather_dep` response for %r",
-                            ts,
-                            exc_info=True,
+                    assert ts, (d, self.story(d))
+                    ts.done = True
+                    if d in data:
+                        recommendations[ts] = ("memory", data[d])
+                    elif not busy:
+                        ts.who_has.discard(worker)
+                        self.has_what[worker].discard(ts.key)
+                        self.log.append(("missing-dep", d))
+                        self.batched_stream.send(
+                            {"op": "missing-data", "errant_worker": worker, "key": d}
                         )
 
-                if self.validate:
-                    self.validate_state()
+                    if ts.state != "memory" and ts not in recommendations:
+                        recommendations[ts] = "fetch"
 
+                del data, response
+                self.transitions(
+                    recommendations=recommendations, stimulus_id=stimulus_id
+                )
                 self.ensure_computing()
 
                 if not busy:
                     self.repetitively_busy = 0
-                    self.ensure_communicating()
                 else:
                     # Exponential backoff to avoid hammering scheduler/worker
                     self.repetitively_busy += 1
                     await asyncio.sleep(0.100 * 1.5 ** self.repetitively_busy)
 
-                    await self.query_who_has(*to_gather_keys)
-                    self.ensure_communicating()
+                    await self.query_who_has(*to_gather_keys, stimulus_id=stimulus_id)
 
-    def bad_dep(self, dep):
-        exc = ValueError(
-            "Could not find dependent %s.  Check worker logs" % str(dep.key)
-        )
-        for ts in list(dep.dependents):
-            msg = error_message(exc)
-            ts.exception = msg["exception"]
-            ts.traceback = msg["traceback"]
-            ts.exception_text = msg["exception_text"]
-            ts.traceback_text = msg["traceback_text"]
-            self.transition(ts, "error")
-        self.release_key(dep.key, reason="bad dep")
-
-    async def handle_missing_dep(self, *deps, **kwargs):
-        self.log.append(("handle-missing", deps))
-        try:
-            deps = {dep for dep in deps if dep.dependents}
-            if not deps:
-                return
+                self.ensure_communicating()
 
-            for dep in list(deps):
-                if (
-                    self._suspicious_count_limit
-                    and dep.suspicious_count > self._suspicious_count_limit
-                ):
-                    deps.remove(dep)
-                    self.bad_dep(dep)
-            if not deps:
+    async def find_missing(self):
+        with log_errors():
+            if not self._missing_dep_flight:
                 return
+            try:
+                if self.validate:
+                    for ts in self._missing_dep_flight:
+                        assert not ts.who_has
 
-            for dep in deps:
-                logger.info(
-                    "Dependent not found: %s %s .  Asking scheduler",
-                    dep.key,
-                    dep.suspicious_count,
+                stimulus_id = f"find-missing-{time()}"
+                who_has = await retry_operation(
+                    self.scheduler.who_has,
+                    keys=[ts.key for ts in self._missing_dep_flight],
                 )
+                who_has = {k: v for k, v in who_has.items() if v}
+                self.update_who_has(who_has, stimulus_id=stimulus_id)
 
-            who_has = await retry_operation(
-                self.scheduler.who_has, keys=list(dep.key for dep in deps)
-            )
-            who_has = {k: v for k, v in who_has.items() if v}
-            self.update_who_has(who_has)
-            still_missing = set()
-            for dep in deps:
-                dep.suspicious_count += 1
-
-                if not who_has.get(dep.key):
-                    logger.info(
-                        "No workers found for %s",
-                        dep.key,
+                if self._missing_dep_flight:
+                    logger.debug(
+                        "No new workers found for %s", self._missing_dep_flight
                     )
-                    self.log.append((dep.key, "no workers found", dep.dependents))
-                    self.release_key(dep.key, reason="Handle missing no workers")
-                elif self.address in who_has and dep.state != "memory":
-
-                    still_missing.add(dep)
-                    self.batched_stream.send(
-                        {
-                            "op": "release-worker-data",
-                            "keys": [dep.key],
-                            "worker": self.address,
-                        }
+                    recommendations = {
+                        dep: "released"
+                        for dep in self._missing_dep_flight
+                        if dep.state == "missing"
+                    }
+                    self.transitions(
+                        recommendations=recommendations, stimulus_id=stimulus_id
                     )
-                else:
-                    logger.debug("New workers found for %s", dep.key)
-                    self.log.append((dep.key, "new workers found"))
-                    for dependent in dep.dependents:
-                        if dep.key in dependent.waiting_for_data:
-                            self.data_needed.append(dependent.key)
-            if still_missing:
-                logger.debug(
-                    "Found self referencing who has response from scheduler for keys %s.\n"
-                    "Trying again handle_missing",
-                    deps,
-                )
-                await self.handle_missing_dep(*deps)
-        except Exception:
-            logger.error("Handle missing dep failed, retrying", exc_info=True)
-            retries = kwargs.get("retries", 5)
-            self.log.append(("handle-missing-failed", retries, deps))
-            if retries > 0:
-                await self.handle_missing_dep(*deps, retries=retries - 1)
-            else:
-                raise
-        finally:
-            try:
-                for dep in deps:
-                    self._missing_dep_flight.remove(dep.key)
-            except KeyError:
-                pass
 
-            self.ensure_communicating()
+            finally:
+                # This is quite arbitrary but the heartbeat has scaling implemented
+                self.periodic_callbacks[
+                    "find-missing"
+                ].callback_time = self.periodic_callbacks["heartbeat"].callback_time
+                self.ensure_communicating()
+                self.ensure_computing()
 
-    async def query_who_has(self, *deps):
+    async def query_who_has(self, *deps, stimulus_id):
         with log_errors():
-            response = await retry_operation(self.scheduler.who_has, keys=deps)
-            self.update_who_has(response)
-            return response
+            who_has = await retry_operation(self.scheduler.who_has, keys=deps)
+            self.update_who_has(who_has, stimulus_id=stimulus_id)
+            return who_has
 
-    def update_who_has(self, who_has):
+    def update_who_has(self, who_has, *, stimulus_id):
         try:
+            recommendations = {}
             for dep, workers in who_has.items():
                 if not workers:
                     continue
@@ -2656,10 +2682,18 @@ def update_who_has(self, who_has):
                         )
                         # Do not mutate the input dict. That's rude
                         workers = set(workers) - {self.address}
-                    self.tasks[dep].who_has.update(workers)
+                    dep_ts = self.tasks[dep]
+                    dep_ts.who_has.update(workers)
+
+                    if dep_ts.state == "missing":
+                        recommendations[dep_ts] = "fetch"
 
                     for worker in workers:
                         self.has_what[worker].add(dep)
+                        if dep_ts.state in ("fetch", "flight", "missing"):
+                            self.pending_data_per_worker[worker].append(dep_ts.key)
+
+            self.transitions(recommendations=recommendations, stimulus_id=stimulus_id)
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
@@ -2668,27 +2702,21 @@ def update_who_has(self, who_has):
                 pdb.set_trace()
             raise
 
-    def steal_request(self, key):
+    def handle_steal_request(self, key):
         # There may be a race condition between stealing and releasing a task.
         # In this case the self.tasks is already cleared. The `None` will be
         # registered as `already-computing` on the other end
         ts = self.tasks.get(key)
-        if key in self.tasks:
-            state = ts.state
-        else:
-            state = None
+        state = ts.state if ts is not None else None
 
         response = {"op": "steal-response", "key": key, "state": state}
         self.batched_stream.send(response)
 
-        if state in ("ready", "waiting", "constrained"):
+        if state in {"ready", "waiting", "constrained"}:
             # If task is marked as "constrained" we haven't yet assigned it an
             # `available_resources` to run on, that happens in
             # `transition_constrained_executing`
-            ts.scheduler_holds_ref = False
-            self.release_key(ts.key, reason="stolen")
-            if self.validate:
-                assert ts.key not in self.tasks
+            self.transition(ts, "forgotten", stimulus_id=f"steal-request-{time()}")
 
     def release_key(
         self,
@@ -2697,16 +2725,15 @@ def release_key(
         reason: str | None = None,
         report: bool = True,
     ):
+        recommendations = {}
         try:
             if self.validate:
                 assert not isinstance(key, TaskState)
-            ts = self.tasks.get(key, None)
-            # If the scheduler holds a reference which is usually the
-            # case when it instructed the task to be computed here or if
-            # data was scattered we must not release it unless the
-            # scheduler allow us to. See also handle_delete_data and
-            if ts is None or ts.scheduler_holds_ref:
-                return
+            ts = self.tasks[key]
+            # needed for legacy notification support
+            state_before = ts.state
+            ts.state = "released"
+
             logger.debug(
                 "Release key %s", {"key": key, "cause": cause, "reason": reason}
             )
@@ -2729,39 +2756,30 @@ def release_key(
             if key in self.threads:
                 del self.threads[key]
 
-            if ts.state == "executing":
-                self.executing_count -= 1
-
             if ts.resource_restrictions is not None:
                 if ts.state == "executing":
                     for resource, quantity in ts.resource_restrictions.items():
                         self.available_resources[resource] += quantity
 
             for d in ts.dependencies:
-                d.dependents.discard(ts)
-
-                if not d.dependents and d.state in ("flight", "fetch"):
-                    self.release_key(d.key, reason="Dependent released")
+                ts.waiting_for_data.discard(ts)
+                if not d.dependents and d.state in {"flight", "fetch", "missing"}:
+                    recommendations[d] = "released"
 
-            if report:
-                # Inform the scheduler of keys which will have gone missing
-                # We are releasing them before they have completed
-                if ts.state in PROCESSING:
-                    # This path is only hit with work stealing
-                    msg = {"op": "release", "key": key, "cause": cause}
-                else:
-                    # This path is only hit when calling release_key manually
-                    msg = {
-                        "op": "release-worker-data",
-                        "keys": [key],
-                        "worker": self.address,
-                    }
-                self.batched_stream.send(msg)
+            ts.waiting_for_data.clear()
+            ts.nbytes = None
+            ts._previous = None
+            ts._next = None
+            ts.done = False
 
-            self._notify_plugins("release_key", key, ts.state, cause, reason, report)
-            del self.tasks[key]
+            self._executing.discard(ts)
+            self._in_flight_tasks.discard(ts)
 
+            self._notify_plugins(
+                "release_key", key, state_before, cause, reason, report
+            )
         except CommClosedError:
+            # Batched stream send might raise if it was already closed
             pass
         except Exception as e:
             logger.exception(e)
@@ -2771,6 +2789,8 @@ def release_key(
                 pdb.set_trace()
             raise
 
+        return recommendations
+
     ################
     # Execute Task #
     ################
@@ -2876,7 +2896,7 @@ def meets_resource_constraints(self, key: str) -> bool:
 
         return True
 
-    async def _maybe_deserialize_task(self, ts):
+    async def _maybe_deserialize_task(self, ts, *, stimulus_id):
         if not isinstance(ts.runspec, SerializedTask):
             return ts.runspec
         try:
@@ -2893,15 +2913,24 @@ async def _maybe_deserialize_task(self, ts):
                     {"action": "deserialize", "start": start, "stop": stop}
                 )
             return function, args, kwargs
-        except Exception:
+        except Exception as e:
             logger.error("Could not deserialize task", exc_info=True)
             self.log.append((ts.key, "deserialize-error"))
+            emsg = error_message(e)
+            emsg.pop("status")
+            self.transition(
+                ts,
+                "error",
+                **emsg,
+                stimulus_id=stimulus_id,
+            )
             raise
 
     def ensure_computing(self):
         if self.paused:
             return
         try:
+            stimulus_id = f"ensure-computing-{time()}"
             while self.constrained and self.executing_count < self.nthreads:
                 key = self.constrained[0]
                 ts = self.tasks.get(key, None)
@@ -2910,21 +2939,22 @@ def ensure_computing(self):
                     continue
                 if self.meets_resource_constraints(key):
                     self.constrained.popleft()
-                    self.transition(ts, "executing")
+                    self.transition(ts, "executing", stimulus_id=stimulus_id)
                 else:
                     break
             while self.ready and self.executing_count < self.nthreads:
                 priority, key = heapq.heappop(self.ready)
                 ts = self.tasks.get(key)
                 if ts is None:
-                    # It is possible for tasks to be released while still remaining on `ready`
-                    # The scheduler might have re-routed to a new worker and told this worker
-                    # to release.  If the task has "disappeared" just continue through the heap
+                    # It is possible for tasks to be released while still remaining on
+                    # `ready` The scheduler might have re-routed to a new worker and
+                    # told this worker to release.  If the task has "disappeared" just
+                    # continue through the heap
                     continue
                 elif ts.key in self.data:
-                    self.transition(ts, "memory")
+                    self.transition(ts, "memory", stimulus_id=stimulus_id)
                 elif ts.state in READY:
-                    self.transition(ts, "executing")
+                    self.transition(ts, "executing", stimulus_id=stimulus_id)
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
@@ -2933,30 +2963,32 @@ def ensure_computing(self):
                 pdb.set_trace()
             raise
 
-    async def execute(self, key):
+    async def execute(self, key, *, stimulus_id):
         if self.status in (Status.closing, Status.closed, Status.closing_gracefully):
             return
-
         if key not in self.tasks:
             return
-
         ts = self.tasks[key]
 
-        if ts.state != "executing":
-            # This might happen if keys are canceled
-            logger.debug(
-                "Trying to execute a task %s which is not in executing state anymore"
-                % ts
-            )
-            return
-
         try:
+            if ts.state == "cancelled":
+                # This might happen if keys are canceled
+                logger.debug(
+                    "Trying to execute task %s which is not in executing state anymore",
+                    ts,
+                )
+                ts.done = True
+                self.transition(ts, "released", stimulus_id=stimulus_id)
+                return
+
             if self.validate:
                 assert not ts.waiting_for_data
                 assert ts.state == "executing"
                 assert ts.runspec is not None
 
-            function, args, kwargs = await self._maybe_deserialize_task(ts)
+            function, args, kwargs = await self._maybe_deserialize_task(
+                ts, stimulus_id=stimulus_id
+            )
 
             args2, kwargs2 = self._prepare_args_for_execution(ts, args, kwargs)
 
@@ -2967,6 +2999,8 @@ async def execute(self, key):
             assert executor in self.executors
             assert key == ts.key
             self.active_keys.add(ts.key)
+
+            result: dict
             try:
                 e = self.executors[executor]
                 ts.start_time = time()
@@ -3002,40 +3036,28 @@ async def execute(self, key):
             finally:
                 self.active_keys.discard(ts.key)
 
-            # We'll need to check again for the task state since it may have
-            # changed since the execution was kicked off. In particular, it may
-            # have been canceled and released already in which case we'll have
-            # to drop the result immediately
-
-            if ts.key not in self.tasks:
-                logger.debug(
-                    "Dropping result for %s since task has already been released."
-                    % ts.key
-                )
-                return
-
+            key = ts.key
+            # key *must* be still in tasks. Releasing it direclty is forbidden
+            # without going through cancelled
+            ts = self.tasks.get(key)
+            assert ts, self.story(key)
+            ts.done = True
             result["key"] = ts.key
             value = result.pop("result", None)
             ts.startstops.append(
                 {"action": "compute", "start": result["start"], "stop": result["stop"]}
             )
             self.threads[ts.key] = result["thread"]
-
+            recommendations = {}
             if result["op"] == "task-finished":
                 ts.nbytes = result["nbytes"]
                 ts.type = result["type"]
-                self.transition(ts, "memory", value=value)
+                recommendations[ts] = ("memory", value)
                 if self.digests is not None:
                     self.digests["task-duration"].add(result["stop"] - result["start"])
             elif isinstance(result.pop("actual-exception"), Reschedule):
-                self.batched_stream.send({"op": "reschedule", "key": ts.key})
-                self.transition(ts, "rescheduled", report=False)
-                self.release_key(ts.key, report=False, reason="Reschedule")
+                recommendations[ts] = "rescheduled"
             else:
-                ts.exception = result["exception"]
-                ts.traceback = result["traceback"]
-                ts.exception_text = result["exception_text"]
-                ts.traceback_text = result["traceback_text"]
                 logger.warning(
                     "Compute Failed\n"
                     "Function:  %s\n"
@@ -3047,7 +3069,15 @@ async def execute(self, key):
                     convert_kwargs_to_str(kwargs2, max_len=1000),
                     result["exception"].data,
                 )
-                self.transition(ts, "error")
+                recommendations[ts] = (
+                    "error",
+                    result["exception"],
+                    result["traceback"],
+                    result["exception_text"],
+                    result["traceback_text"],
+                )
+
+            self.transitions(recommendations, stimulus_id=stimulus_id)
 
             logger.debug("Send compute response to scheduler: %s, %s", ts.key, result)
 
@@ -3056,15 +3086,18 @@ async def execute(self, key):
                 assert not ts.waiting_for_data
 
         except Exception as exc:
+            assert ts
             logger.error(
                 "Exception during execution of task %s.", ts.key, exc_info=True
             )
             emsg = error_message(exc)
-            ts.exception = emsg["exception"]
-            ts.traceback = emsg["traceback"]
-            ts.exception_text = emsg["exception_text"]
-            ts.traceback_text = emsg["traceback_text"]
-            self.transition(ts, "error")
+            emsg.pop("status")
+            self.transition(
+                ts,
+                "error",
+                **emsg,
+                stimulus_id=stimulus_id,
+            )
         finally:
             self.ensure_computing()
             self.ensure_communicating()
@@ -3311,11 +3344,20 @@ def get_call_stack(self, comm=None, keys=None):
     def _notify_plugins(self, method_name, *args, **kwargs):
         for name, plugin in self.plugins.items():
             if hasattr(plugin, method_name):
+                if method_name == "release_key":
+                    warnings.warn(
+                        "The `WorkerPlugin.release_key` hook is depreacted and will be "
+                        "removed in a future version. A similar event can now be "
+                        "caught by filtering for a `finish=='released'` event in the "
+                        "`WorkerPlugin.transition` hook.",
+                        DeprecationWarning,
+                    )
+
                 try:
                     getattr(plugin, method_name)(*args, **kwargs)
                 except Exception:
                     logger.info(
-                        "Plugin '%s' failed with exception" % name, exc_info=True
+                        "Plugin '%s' failed with exception", name, exc_info=True
                     )
 
     ##############
@@ -3334,9 +3376,9 @@ def validate_task_executing(self, ts):
         assert ts.runspec is not None
         assert ts.key not in self.data
         assert not ts.waiting_for_data
-        assert all(
-            dep.key in self.data or dep.key in self.actors for dep in ts.dependencies
-        )
+        for dep in ts.dependencies:
+            assert dep.state == "memory", self.story(dep)
+            assert dep.key in self.data or dep.key in self.actors
 
     def validate_task_ready(self, ts):
         assert ts.key in pluck(1, self.ready)
@@ -3355,41 +3397,63 @@ def validate_task_waiting(self, ts):
 
     def validate_task_flight(self, ts):
         assert ts.key not in self.data
+        assert ts in self._in_flight_tasks
         assert not any(dep.key in self.ready for dep in ts.dependents)
         assert ts.coming_from
         assert ts.coming_from in self.in_flight_workers
         assert ts.key in self.in_flight_workers[ts.coming_from]
 
     def validate_task_fetch(self, ts):
-        assert ts.runspec is None
         assert ts.key not in self.data
-        assert self.address not in ts.who_has  # !!!!!!!!
-        # FIXME This is currently not an invariant since upon comm failure we
-        # remove the erroneous worker from all who_has and correct the state
-        # upon the next ensure_communicate
-
-        # if not ts.who_has:
-        #     # If we do not know who_has for a fetch task, it must be logged in
-        #     # the missing dep. There should be a handle_missing_dep running for
-        #     # all of these keys
-
-        #     assert ts.key in self._missing_dep_flight, (
-        #         ts.key,
-        #         self.story(ts),
-        #         self._missing_dep_flight.copy(),
-        #         self.in_flight_workers.copy(),
-        #     )
-        assert ts.dependents
+        assert self.address not in ts.who_has
 
         for w in ts.who_has:
             assert ts.key in self.has_what[w]
 
+    def validate_task_missing(self, ts):
+        assert ts.key not in self.data
+        assert not ts.who_has
+        assert not any(ts.key in has_what for has_what in self.has_what.values())
+        assert ts.key in self._missing_dep_flight
+
+    def validate_task_cancelled(self, ts):
+        assert ts.key not in self.data
+        assert ts._previous
+
+    def validate_task_resumed(self, ts):
+        assert ts.key not in self.data
+        assert ts._next
+        assert ts._previous
+
+    def validate_task_released(self, ts):
+        assert ts.key not in self.data
+        assert not ts._next
+        assert not ts._previous
+        assert ts not in self._executing
+        assert ts not in self._in_flight_tasks
+        assert ts not in self._missing_dep_flight
+        assert ts not in self._missing_dep_flight
+        assert not ts.who_has
+        assert not any(ts.key in has_what for has_what in self.has_what.values())
+        assert not ts.waiting_for_data
+        assert not ts.done
+        assert not ts.exception
+        assert not ts.traceback
+
     def validate_task(self, ts):
         try:
+            if ts.key in self.tasks:
+                assert self.tasks[ts.key] == ts
             if ts.state == "memory":
                 self.validate_task_memory(ts)
             elif ts.state == "waiting":
                 self.validate_task_waiting(ts)
+            elif ts.state == "missing":
+                self.validate_task_missing(ts)
+            elif ts.state == "cancelled":
+                self.validate_task_cancelled(ts)
+            elif ts.state == "resumed":
+                self.validate_task_resumed(ts)
             elif ts.state == "ready":
                 self.validate_task_ready(ts)
             elif ts.state == "executing":
@@ -3398,6 +3462,8 @@ def validate_task(self, ts):
                 self.validate_task_flight(ts)
             elif ts.state == "fetch":
                 self.validate_task_fetch(ts)
+            elif ts.state == "released":
+                self.validate_task_released(ts)
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
@@ -3410,12 +3476,15 @@ def validate_state(self):
         if self.status != Status.running:
             return
         try:
+            assert self.executing_count >= 0
+            waiting_for_data_count = 0
             for ts in self.tasks.values():
                 assert ts.state is not None
                 # check that worker has task
                 for worker in ts.who_has:
                     assert ts.key in self.has_what[worker]
-                # check that deps have a set state and that dependency<->dependent links are there
+                # check that deps have a set state and that dependency<->dependent links
+                # are there
                 for dep in ts.dependencies:
                     # self.tasks was just a dict of tasks
                     # and this check was originally that the key was in `task_state`
@@ -3424,19 +3493,21 @@ def validate_state(self):
                     # Might need better bookkeeping
                     assert dep.state is not None
                     assert ts in dep.dependents, ts
-                for key in ts.waiting_for_data:
-                    ts_wait = self.tasks[key]
+                if ts.waiting_for_data:
+                    waiting_for_data_count += 1
+                for ts_wait in ts.waiting_for_data:
+                    assert ts_wait.key in self.tasks
                     assert (
-                        ts_wait.state == "flight"
-                        or ts_wait.state == "fetch"
+                        ts_wait.state
+                        in {"ready", "executing", "flight", "fetch", "missing"}
                         or ts_wait.key in self._missing_dep_flight
                         or ts_wait.who_has.issubset(self.in_flight_workers)
-                    )
+                    ), (ts, ts_wait, self.story(ts), self.story(ts_wait))
                 if ts.state == "memory":
                     assert isinstance(ts.nbytes, int)
                     assert not ts.waiting_for_data
                     assert ts.key in self.data or ts.key in self.actors
-
+            assert self.waiting_for_data_count == waiting_for_data_count
             for worker, keys in self.has_what.items():
                 for k in keys:
                     assert worker in self.tasks[k].who_has
@@ -3672,6 +3743,7 @@ def secede():
         worker.maybe_transition_long_running,
         worker.tasks[thread_state.key],
         compute_duration=duration,
+        stimulus_id=f"secede-{thread_state.key}-{time()}",
     )
 
 
diff --git a/distributed/worker_client.py b/distributed/worker_client.py
index 989a3f8f0d9..059d5dfaad7 100644
--- a/distributed/worker_client.py
+++ b/distributed/worker_client.py
@@ -3,6 +3,8 @@
 
 import dask
 
+from distributed.metrics import time
+
 from .threadpoolexecutor import rejoin, secede
 from .worker import get_client, get_worker, thread_state
 
@@ -50,9 +52,14 @@ def worker_client(timeout=None, separate_thread=True):
     worker = get_worker()
     client = get_client(timeout=timeout)
     if separate_thread:
+        duration = time() - thread_state.start_time
         secede()  # have this thread secede from the thread pool
         worker.loop.add_callback(
-            worker.transition, worker.tasks[thread_state.key], "long-running"
+            worker.transition,
+            worker.tasks[thread_state.key],
+            "long-running",
+            stimulus_id=f"worker-client-secede-{time()}",
+            compute_duration=duration,
         )
 
     yield client

From 672217ceadb346e9aa913ecb3d802369e55d882d Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Mon, 27 Sep 2021 20:52:03 +0100
Subject: [PATCH 1502/1550] Generate Cython HTML annotations (#5321)

---
 .gitignore |  4 ++++
 setup.py   | 24 ++++++++++++++----------
 2 files changed, 18 insertions(+), 10 deletions(-)

diff --git a/.gitignore b/.gitignore
index e47580fc5bb..2788e494a62 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,8 +1,12 @@
 *.pyc
 *.pyd
 *.py~
+
+# Cython files
 *.c
 *.so
+distributed/scheduler.html
+
 build/
 dist/
 *.egg-info
diff --git a/setup.py b/setup.py
index 61f2193d6c3..23bc2c9be0a 100755
--- a/setup.py
+++ b/setup.py
@@ -23,6 +23,12 @@
     else:
         install_requires.append(r)
 
+# To enable Cython, add to pip install one of the following:
+# --install-option="--with-cython"
+# --install-option="--with-cython=annotate"
+# --install-option="--with-cython=profile"
+# --install-option="--with-cython=annotate,profile"
+
 cython_arg = None
 for i in range(len(sys.argv)):
     if sys.argv[i].startswith("--with-cython"):
@@ -37,18 +43,16 @@
     except ImportError:
         setup_requires.append("cython")
 
-    profile = False
-    try:
-        _, param = cython_arg.split("=")
-        profile = param == "profile"
-    except ValueError:
-        pass
+    _, _, params = cython_arg.partition("=")
+    params = params.split(",")
+    profile = "profile" in params
+    if "annotate" in params:
+        import Cython.Compiler.Options
+
+        Cython.Compiler.Options.annotate = True
 
     cyext_modules = [
-        Extension(
-            "distributed.scheduler",
-            sources=["distributed/scheduler.py"],
-        ),
+        Extension("distributed.scheduler", sources=["distributed/scheduler.py"]),
     ]
     for e in cyext_modules:
         e.cython_directives = {

From 0d12661797d10799432839ef50f98a17266edad8 Mon Sep 17 00:00:00 2001
From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com>
Date: Mon, 27 Sep 2021 18:59:52 -0400
Subject: [PATCH 1503/1550] Bump `RAPIDS_VER` for gpuCI (#5358)

---
 continuous_integration/gpuci/axis.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/continuous_integration/gpuci/axis.yaml b/continuous_integration/gpuci/axis.yaml
index 587b193b003..da4a725bd2e 100644
--- a/continuous_integration/gpuci/axis.yaml
+++ b/continuous_integration/gpuci/axis.yaml
@@ -8,6 +8,6 @@ LINUX_VER:
 - ubuntu18.04
 
 RAPIDS_VER:
-- "21.10"
+- "21.12"
 
-excludes:
\ No newline at end of file
+excludes:

From 789a2ea2c057eec11e00efa39ce6d7c70b269646 Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Tue, 28 Sep 2021 10:33:27 +0100
Subject: [PATCH 1504/1550] Worker State Machine Refactor: clean up dead
 handlers (#5359)

---
 distributed/scheduler.py | 97 +++++++---------------------------------
 1 file changed, 15 insertions(+), 82 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index c538056e7dc..19963fb7654 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -3720,7 +3720,6 @@ def __init__(
         worker_handlers = {
             "task-finished": self.handle_task_finished,
             "task-erred": self.handle_task_erred,
-            "release": self.handle_release_data,
             "release-worker-data": self.release_worker_data,
             "add-keys": self.add_keys,
             "missing-data": self.handle_missing_data,
@@ -4739,70 +4738,23 @@ def stimulus_task_erred(
         parent: SchedulerState = cast(SchedulerState, self)
         logger.debug("Stimulus task erred %s, %s", key, worker)
 
-        recommendations: dict = {}
-        client_msgs: dict = {}
-        worker_msgs: dict = {}
-
         ts: TaskState = parent._tasks.get(key)
-        if ts is None:
-            return recommendations, client_msgs, worker_msgs
-
-        if ts._state == "processing":
-            retries: Py_ssize_t = ts._retries
-            r: tuple
-            if retries > 0:
-                ts._retries = retries - 1
-                r = parent._transition(key, "waiting")
-            else:
-                r = parent._transition(
-                    key,
-                    "erred",
-                    cause=key,
-                    exception=exception,
-                    traceback=traceback,
-                    worker=worker,
-                    **kwargs,
-                )
-            recommendations, client_msgs, worker_msgs = r
-
-        return recommendations, client_msgs, worker_msgs
-
-    def stimulus_missing_data(
-        self, cause=None, key=None, worker=None, ensure=True, **kwargs
-    ):
-        """Mark that certain keys have gone missing.  Recover."""
-        parent: SchedulerState = cast(SchedulerState, self)
-        with log_errors():
-            logger.debug("Stimulus missing data %s, %s", key, worker)
-
-            recommendations: dict = {}
-            client_msgs: dict = {}
-            worker_msgs: dict = {}
-
-            ts: TaskState = parent._tasks.get(key)
-            if ts is None or ts._state == "memory":
-                return recommendations, client_msgs, worker_msgs
-            cts: TaskState = parent._tasks.get(cause)
-
-            if cts is not None and cts._state == "memory":  # couldn't find this
-                ws: WorkerState
-                cts_nbytes: Py_ssize_t = cts.get_nbytes()
-                for ws in cts._who_has:  # TODO: this behavior is extreme
-                    del ws._has_what[ts]
-                    ws._nbytes -= cts_nbytes
-                cts._who_has.clear()
-                recommendations[cause] = "released"
-
-            if key:
-                recommendations[key] = "released"
-
-            parent._transitions(recommendations, client_msgs, worker_msgs)
-            recommendations = {}
+        if ts is None or ts._state != "processing":
+            return {}, {}, {}
 
-            if parent._validate:
-                assert cause not in self.who_has
-
-            return recommendations, client_msgs, worker_msgs
+        if ts._retries > 0:
+            ts._retries -= 1
+            return parent._transition(key, "waiting")
+        else:
+            return parent._transition(
+                key,
+                "erred",
+                cause=key,
+                exception=exception,
+                traceback=traceback,
+                worker=worker,
+                **kwargs,
+            )
 
     def stimulus_retry(self, comm=None, keys=None, client=None):
         parent: SchedulerState = cast(SchedulerState, self)
@@ -5346,25 +5298,6 @@ def handle_task_erred(self, key=None, **msg):
 
         self.send_all(client_msgs, worker_msgs)
 
-    def handle_release_data(self, key=None, worker=None, client=None, **msg):
-        parent: SchedulerState = cast(SchedulerState, self)
-        ts: TaskState = parent._tasks.get(key)
-        if ts is None:
-            return
-        ws: WorkerState = parent._workers_dv.get(worker)
-        if ws is None or ts._processing_on != ws:
-            return
-
-        recommendations: dict
-        client_msgs: dict
-        worker_msgs: dict
-
-        r: tuple = self.stimulus_missing_data(key=key, ensure=False, **msg)
-        recommendations, client_msgs, worker_msgs = r
-        parent._transitions(recommendations, client_msgs, worker_msgs)
-
-        self.send_all(client_msgs, worker_msgs)
-
     def handle_missing_data(self, key=None, errant_worker=None, **kwargs):
         parent: SchedulerState = cast(SchedulerState, self)
         logger.debug("handle missing data key=%s worker=%s", key, errant_worker)

From bc4b8f4ee00ef70f1538da593244d57376204ae8 Mon Sep 17 00:00:00 2001
From: Ray Bell <rayjohnbell0@gmail.com>
Date: Tue, 28 Sep 2021 05:36:12 -0400
Subject: [PATCH 1505/1550] Add pre-commit to environments (#5362)

Co-authored-by: Ray Bell <rjbell1987@gmail.com>
Co-authored-by: Ray Bell <ray.bell@dtn.com>
---
 continuous_integration/environment-3.7.yaml | 1 +
 continuous_integration/environment-3.8.yaml | 1 +
 continuous_integration/environment-3.9.yaml | 1 +
 3 files changed, 3 insertions(+)

diff --git a/continuous_integration/environment-3.7.yaml b/continuous_integration/environment-3.7.yaml
index 25f1a30d518..ebc2cc1fb00 100644
--- a/continuous_integration/environment-3.7.yaml
+++ b/continuous_integration/environment-3.7.yaml
@@ -22,6 +22,7 @@ dependencies:
   - netcdf4
   - paramiko
   - pickle5  # Backport from Python 3.8
+  - pre-commit
   - prometheus_client
   - psutil
   - pytest
diff --git a/continuous_integration/environment-3.8.yaml b/continuous_integration/environment-3.8.yaml
index 1e593d49d12..9454e023b66 100644
--- a/continuous_integration/environment-3.8.yaml
+++ b/continuous_integration/environment-3.8.yaml
@@ -20,6 +20,7 @@ dependencies:
   - msgpack-python
   - netcdf4
   - paramiko
+  - pre-commit
   - prometheus_client
   - psutil
   - pytest
diff --git a/continuous_integration/environment-3.9.yaml b/continuous_integration/environment-3.9.yaml
index 9eee8754269..de4a46ed036 100644
--- a/continuous_integration/environment-3.9.yaml
+++ b/continuous_integration/environment-3.9.yaml
@@ -22,6 +22,7 @@ dependencies:
   - msgpack-python
   - netcdf4
   - paramiko
+  - pre-commit
   - prometheus_client
   - psutil
   - pynvml  # Only tested here

From 5c3eacd768545d84e865340f08c6965f521541df Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Tue, 28 Sep 2021 13:20:54 +0100
Subject: [PATCH 1506/1550] Sync worker status to the scheduler; new 'paused'
 status (#5330)

---
 distributed/core.py              | 52 +++++++++----------------
 distributed/deploy/spec.py       | 17 ++------
 distributed/scheduler.py         | 36 ++++++++++-------
 distributed/tests/test_core.py   | 35 ++---------------
 distributed/tests/test_nanny.py  |  2 +-
 distributed/tests/test_worker.py | 66 ++++++++++++++++++++++++++++++--
 distributed/utils_test.py        |  2 +-
 distributed/worker.py            | 49 ++++++++++++++++--------
 8 files changed, 147 insertions(+), 112 deletions(-)

diff --git a/distributed/core.py b/distributed/core.py
index 5a77ef562ab..fd083972379 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -5,7 +5,6 @@
 import threading
 import traceback
 import uuid
-import warnings
 import weakref
 from collections import defaultdict
 from contextlib import suppress
@@ -49,18 +48,22 @@ class Status(Enum):
     other.
     """
 
-    closed = "closed"
-    closing = "closing"
-    closing_gracefully = "closing-gracefully"
-    failed = "failed"
-    init = "init"
+    undefined = "undefined"
     created = "created"
-    running = "running"
+    init = "init"
     starting = "starting"
-    stopped = "stopped"
+    running = "running"
+    paused = "paused"
     stopping = "stopping"
-    undefined = None
-    dont_reply = "dont-reply"
+    stopped = "stopped"
+    closing = "closing"
+    closing_gracefully = "closing_gracefully"
+    closed = "closed"
+    failed = "failed"
+    dont_reply = "dont_reply"
+
+
+Status.lookup = {s.name: s for s in Status}
 
 
 class RPCClosed(IOError):
@@ -244,19 +247,9 @@ def status(self):
 
     @status.setter
     def status(self, new_status):
-        if isinstance(new_status, Status):
-            self._status = new_status
-        elif isinstance(new_status, str) or new_status is None:
-            warnings.warn(
-                f"Since distributed 2.23 `.status` is now an Enum, please assign `Status.{new_status}`",
-                PendingDeprecationWarning,
-                stacklevel=1,
-            )
-            corresponding_enum_variants = [s for s in Status if s.value == new_status]
-            assert len(corresponding_enum_variants) == 1
-            self._status = corresponding_enum_variants[0]
-        else:
-            raise TypeError(f"expected Status or str, got {new_status}")
+        if not isinstance(new_status, Status):
+            raise TypeError(f"Expected Status; got {new_status!r}")
+        self._status = new_status
 
     async def finished(self):
         """Wait until the server has finished"""
@@ -266,7 +259,7 @@ def __await__(self):
         async def _():
             timeout = getattr(self, "death_timeout", 0)
             async with self._startup_lock:
-                if self.status == Status.running:
+                if self.status in (Status.running, Status.paused):
                     return self
                 if timeout:
                     try:
@@ -501,7 +494,7 @@ async def handle_comm(self, comm):
                             self._ongoing_coroutines.add(result)
                             result = await result
                     except (CommClosedError, CancelledError):
-                        if self.status == Status.running:
+                        if self.status in (Status.running, Status.paused):
                             logger.info("Lost connection to %r", address, exc_info=True)
                         break
                     except Exception as e:
@@ -511,14 +504,7 @@ async def handle_comm(self, comm):
                         else:
                             result = error_message(e, status="uncaught-error")
 
-                # result is not type stable:
-                # when LHS is not Status then RHS must not be Status or it raises.
-                # when LHS is Status then RHS must be status or it raises in tests
-                is_dont_reply = False
-                if isinstance(result, Status) and (result == Status.dont_reply):
-                    is_dont_reply = True
-
-                if reply and not is_dont_reply:
+                if reply and result != Status.dont_reply:
                     try:
                         await comm.write(result, serializers=serializers)
                     except (OSError, TypeError) as e:
diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index c7646032c5b..d496f6ae847 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -3,7 +3,6 @@
 import copy
 import logging
 import math
-import warnings
 import weakref
 from contextlib import suppress
 from inspect import isawaitable
@@ -39,19 +38,9 @@ def status(self):
 
     @status.setter
     def status(self, new_status):
-        if isinstance(new_status, Status):
-            self._status = new_status
-        elif isinstance(new_status, str) or new_status is None:
-            warnings.warn(
-                f"Since distributed 2.19 `.status` is now an Enum, please assign `Status.{new_status}`",
-                PendingDeprecationWarning,
-                stacklevel=1,
-            )
-            corresponding_enum_variants = [s for s in Status if s.value == new_status]
-            assert len(corresponding_enum_variants) == 1
-            self._status = corresponding_enum_variants[0]
-        else:
-            raise TypeError(f"expected Status or str, got {new_status}")
+        if not isinstance(new_status, Status):
+            raise TypeError(f"Expected Status; got {new_status!r}")
+        self._status = new_status
 
     def __init__(self, scheduler=None, name=None):
         self.address = getattr(self, "address", None)
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 19963fb7654..ddd60433d0d 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -442,9 +442,9 @@ class WorkerState:
        processing on this worker.  This is the sum of all the costs in
        this worker's :attr:`processing` dictionary.
 
-    .. attribute:: status: str
+    .. attribute:: status: Status
 
-       The current status of the worker, either ``'running'`` or ``'closed'``
+       Read-only worker status, synced one way from the remote Worker object
 
     .. attribute:: nanny: str
 
@@ -547,7 +547,7 @@ def __init__(
         self._nanny = nanny
 
         self._hash = hash(address)
-        self._status = Status.running
+        self._status = Status.undefined
         self._nbytes = 0
         self._occupancy = 0
         self._memory_unmanaged_old = 0
@@ -685,14 +685,9 @@ def status(self):
 
     @status.setter
     def status(self, new_status):
-        if isinstance(new_status, Status):
-            self._status = new_status
-        elif isinstance(new_status, str) or new_status is None:
-            corresponding_enum_variants = [s for s in Status if s.value == new_status]
-            assert len(corresponding_enum_variants) == 1
-            self._status = corresponding_enum_variants[0]
-        else:
-            raise TypeError(f"expected Status or str, got {new_status}")
+        if not isinstance(new_status, Status):
+            raise TypeError(f"Expected Status; got {new_status!r}")
+        self._status = new_status
 
     @property
     def time_delay(self):
@@ -3727,6 +3722,7 @@ def __init__(
             "reschedule": self.reschedule,
             "keep-alive": lambda *args, **kwargs: None,
             "log-event": self.log_worker_event,
+            "worker-status-change": self.handle_worker_status_change,
         }
 
         client_handlers = {
@@ -3973,7 +3969,7 @@ async def close(self, comm=None, fast=False, close_workers=False):
         Scheduler.cleanup
         """
         parent: SchedulerState = cast(SchedulerState, self)
-        if self.status in (Status.closing, Status.closed, Status.closing_gracefully):
+        if self.status in (Status.closing, Status.closed):
             await self.finished()
             return
         self.status = Status.closing
@@ -4812,7 +4808,6 @@ async def remove_worker(self, comm=None, address=None, safe=False, close=True):
                 ["all", address],
                 {
                     "action": "remove-worker",
-                    "worker": address,
                     "processing-tasks": dict(ws._processing),
                 },
             )
@@ -5371,6 +5366,21 @@ def handle_long_running(self, key=None, worker=None, compute_duration=None):
         ws._processing[ts] = 0
         self.check_idle_saturated(ws)
 
+    def handle_worker_status_change(self, status: str, worker: str):
+        parent: SchedulerState = cast(SchedulerState, self)
+        ws: WorkerState = parent._workers_dv.get(worker)
+        if not ws:
+            return
+        self.log_event(
+            ws._address,
+            {
+                "action": "worker-status-change",
+                "prev-status": ws._status.name,
+                "status": status,
+            },
+        )
+        ws._status = Status.lookup[status]
+
     async def handle_worker(self, comm=None, worker=None):
         """
         Listen to responses from a single worker
diff --git a/distributed/tests/test_core.py b/distributed/tests/test_core.py
index 4e079f5345f..09bbd032ed7 100644
--- a/distributed/tests/test_core.py
+++ b/distributed/tests/test_core.py
@@ -2,7 +2,6 @@
 import os
 import socket
 import threading
-import warnings
 import weakref
 
 import pytest
@@ -70,39 +69,13 @@ def echo_no_serialize(comm, x):
 
 
 def test_server_status_is_always_enum():
-    """
-    Assignments with strings get converted to corresponding Enum variant
-    """
+    """Assignments with strings is forbidden"""
     server = Server({})
     assert isinstance(server.status, Status)
-    with warnings.catch_warnings(record=True) as w:
-        warnings.simplefilter("ignore")
-        assert server.status != Status.stopped
-        server.status = "stopped"
-    assert isinstance(server.status, Status)
+    assert server.status != Status.stopped
+    server.status = Status.stopped
     assert server.status == Status.stopped
-
-
-def test_server_status_assign_non_variant_raises():
-    server = Server({})
-    with warnings.catch_warnings(record=True) as w:
-        warnings.simplefilter("ignore")
-        with pytest.raises(AssertionError):
-            server.status = "I do not exists"
-
-
-def test_server_status_assign_with_variant_warns():
-    server = Server({})
-    with warnings.catch_warnings(record=True) as w:
-        warnings.simplefilter("default")
-        with pytest.warns(PendingDeprecationWarning):
-            server.status = "running"
-
-
-def test_server_status_assign_with_variant_raises_in_tests():
-    """That would be the default in user code"""
-    server = Server({})
-    with pytest.raises(PendingDeprecationWarning):
+    with pytest.raises(TypeError):
         server.status = "running"
 
 
diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
index c5487c7e712..d2860ffc9b4 100644
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -305,7 +305,7 @@ def pause(dask_worker):
         # This is is very fragile, since a refactor of memory_monitor to
         # remove _memory_monitoring will break this test.
         dask_worker._memory_monitoring = True
-        dask_worker.paused = True
+        dask_worker.status = Status.paused
         dask_worker.outgoing_current_count = 2
 
     await c.run(pause, workers=[a.address])
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index b68643b8448..c28a521ea72 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -315,7 +315,7 @@ async def test_worker_waits_for_scheduler():
         pass
     else:
         assert False
-    assert w.status not in (Status.closed, Status.running)
+    assert w.status not in (Status.closed, Status.running, Status.paused)
     await w.close(timeout=0.1)
 
 
@@ -918,7 +918,7 @@ def __sizeof__(self):
 async def test_fail_write_many_to_disk(c, s, a):
     a.validate = False
     await asyncio.sleep(0.1)
-    assert not a.paused
+    assert a.status == Status.running
 
     class Bad:
         def __init__(self, x):
@@ -1175,7 +1175,7 @@ def f():
         future = c.submit(f)
         futures = c.map(slowinc, range(30), delay=0.1)
 
-        while not a.paused:
+        while a.status != Status.paused:
             await asyncio.sleep(0.01)
 
         out = logger.getvalue()
@@ -1596,7 +1596,7 @@ async def test_lifetime(c, s):
     async with Worker(s.address) as a, Worker(s.address, lifetime="1 seconds") as b:
         futures = c.map(slowinc, range(200), delay=0.1, worker=[b.address])
         await asyncio.sleep(1.5)
-        assert b.status != Status.running
+        assert b.status not in (Status.running, Status.paused)
         await b.finished()
         assert set(b.data) == set(a.data)  # successfully moved data over
 
@@ -2944,3 +2944,61 @@ async def test_who_has_consistent_remove_replica(c, s, *workers):
     assert ("missing-dep", f1.key) in a.story(f1.key)
     assert a.tasks[f1.key].suspicious_count == 0
     assert s.tasks[f1.key].suspicious == 0
+
+
+@pytest.mark.slow
+@gen_cluster(
+    client=True,
+    Worker=Nanny,
+    nthreads=[("", 1)],
+    config={"distributed.worker.memory.pause": 0.5},
+    worker_kwargs={"memory_limit": 2 ** 29},  # 500 MiB
+)
+async def test_worker_status_sync(c, s, a):
+    (ws,) = s.workers.values()
+
+    while ws.status != Status.running:
+        await asyncio.sleep(0.01)
+
+    def leak():
+        distributed._test_leak = "x" * 2 ** 28  # 250 MiB
+
+    def clear_leak():
+        del distributed._test_leak
+
+    await c.run(leak)
+
+    while ws.status != Status.paused:
+        await asyncio.sleep(0.01)
+
+    await c.run(clear_leak)
+
+    while ws.status != Status.running:
+        await asyncio.sleep(0.01)
+
+    await s.retire_workers()
+
+    while ws.status != Status.closed:
+        await asyncio.sleep(0.01)
+
+    events = [ev for _, ev in s.events[ws.address] if ev["action"] != "heartbeat"]
+    assert events == [
+        {"action": "add-worker"},
+        {
+            "action": "worker-status-change",
+            "prev-status": "undefined",
+            "status": "running",
+        },
+        {
+            "action": "worker-status-change",
+            "prev-status": "running",
+            "status": "paused",
+        },
+        {
+            "action": "worker-status-change",
+            "prev-status": "paused",
+            "status": "running",
+        },
+        {"action": "remove-worker", "processing-tasks": {}},
+        {"action": "retired"},
+    ]
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index 615a353b083..90f40d7f75c 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -1543,7 +1543,7 @@ def check_instances():
     for w in Worker._instances:
         with suppress(RuntimeError):  # closed IOLoop
             w.loop.add_callback(w.close, report=False, executor_wait=False)
-            if w.status == Status.running:
+            if w.status in (Status.running, Status.paused):
                 w.loop.add_callback(w.close)
     Worker._instances.clear()
 
diff --git a/distributed/worker.py b/distributed/worker.py
index 4e240dfa58b..7d1a2710b8c 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -643,8 +643,6 @@ def __init__(
 
         self.memory_limit = parse_memory_limit(memory_limit, self.nthreads)
 
-        self.paused = False
-
         if "memory_target_fraction" in kwargs:
             self.memory_target_fraction = kwargs.pop("memory_target_fraction")
         else:
@@ -904,6 +902,19 @@ def local_dir(self):
     def executor(self):
         return self.executors["default"]
 
+    @ServerNode.status.setter
+    def status(self, value):
+        """Override Server.status to notify the Scheduler of status changes"""
+        ServerNode.status.__set__(self, value)
+        if (
+            self.batched_stream
+            and self.batched_stream.comm
+            and not self.batched_stream.comm.closed()
+        ):
+            self.batched_stream.send(
+                {"op": "worker-status-change", "status": self._status.name}
+            )
+
     async def get_metrics(self):
         out = dict(
             executing=self.executing_count,
@@ -1067,7 +1078,7 @@ async def heartbeat(self):
 
             if response["status"] == "missing":
                 for i in range(10):
-                    if self.status != Status.running:
+                    if self.status not in (Status.running, Status.paused):
                         break
                     else:
                         await asyncio.sleep(0.05)
@@ -1102,7 +1113,7 @@ async def handle_scheduler(self, comm):
             logger.exception(e)
             raise
         finally:
-            if self.reconnect and self.status == Status.running:
+            if self.reconnect and self.status in (Status.running, Status.paused):
                 logger.info("Connection to scheduler broken.  Reconnecting...")
                 self.loop.add_callback(self.heartbeat)
             else:
@@ -1314,7 +1325,11 @@ async def close(
                 logger.info("Stopping worker at %s", self.address)
             except ValueError:  # address not available if already closed
                 logger.info("Stopping worker")
-            if self.status not in (Status.running, Status.closing_gracefully):
+            if self.status not in (
+                Status.running,
+                Status.paused,
+                Status.closing_gracefully,
+            ):
                 logger.info("Closed worker has not yet started: %s", self.status)
             self.status = Status.closing
 
@@ -1344,7 +1359,7 @@ async def close(
                 if not any(
                     w
                     for w in Worker._instances
-                    if w != self and w.status == Status.running
+                    if w != self and w.status in (Status.running, Status.paused)
                 ):
                     for c in Worker._initialized_clients:
                         # Regardless of what the client was initialized with
@@ -1469,7 +1484,7 @@ async def get_data(
         ):
             max_connections = max_connections * 2
 
-        if self.paused:
+        if self.status == Status.paused:
             max_connections = 1
             throttle_msg = " Throttling outgoing connections because worker is paused."
         else:
@@ -2458,7 +2473,7 @@ async def gather_dep(
             Total number of bytes for all the dependencies in to_gather combined
         """
         cause = None
-        if self.status != Status.running:
+        if self.status not in (Status.running, Status.paused):
             return
 
         with log_errors():
@@ -2927,7 +2942,7 @@ async def _maybe_deserialize_task(self, ts, *, stimulus_id):
             raise
 
     def ensure_computing(self):
-        if self.paused:
+        if self.status == Status.paused:
             return
         try:
             stimulus_id = f"ensure-computing-{time()}"
@@ -3148,7 +3163,7 @@ def check_pause(memory):
             if self.memory_pause_fraction and frac > self.memory_pause_fraction:
                 # Try to free some memory while in paused state
                 self._throttled_gc.collect()
-                if not self.paused:
+                if self.status == Status.running:
                     logger.warning(
                         "Worker is at %d%% memory usage. Pausing worker.  "
                         "Process memory: %s -- Worker memory limit: %s",
@@ -3158,8 +3173,8 @@ def check_pause(memory):
                         if self.memory_limit is not None
                         else "None",
                     )
-                    self.paused = True
-            elif self.paused:
+                    self.status = Status.paused
+            elif self.status == Status.paused:
                 logger.warning(
                     "Worker is at %d%% memory usage. Resuming worker. "
                     "Process memory: %s -- Worker memory limit: %s",
@@ -3169,7 +3184,7 @@ def check_pause(memory):
                     if self.memory_limit is not None
                     else "None",
                 )
-                self.paused = False
+                self.status = Status.running
                 self.ensure_computing()
 
         check_pause(memory)
@@ -3473,7 +3488,7 @@ def validate_task(self, ts):
             raise
 
     def validate_state(self):
-        if self.status != Status.running:
+        if self.status not in (Status.running, Status.paused):
             return
         try:
             assert self.executing_count >= 0
@@ -3640,7 +3655,11 @@ def get_worker() -> Worker:
         return thread_state.execution_state["worker"]
     except AttributeError:
         try:
-            return first(w for w in Worker._instances if w.status == Status.running)
+            return first(
+                w
+                for w in Worker._instances
+                if w.status in (Status.running, Status.paused)
+            )
         except StopIteration:
             raise ValueError("No workers found")
 

From cc412127763ec8e16db8f528ce7696f5a04bb4fc Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Tue, 28 Sep 2021 15:30:09 +0100
Subject: [PATCH 1507/1550] Reinstate: AMM ReduceReplicas to iterate only on
 replicated tasks (#5341)

---
 distributed/active_memory_manager.py |   8 +-
 distributed/scheduler.py             | 118 ++++++++++++++++-----------
 2 files changed, 70 insertions(+), 56 deletions(-)

diff --git a/distributed/active_memory_manager.py b/distributed/active_memory_manager.py
index c2bbe7ccd3c..a86914cafcf 100644
--- a/distributed/active_memory_manager.py
+++ b/distributed/active_memory_manager.py
@@ -303,13 +303,7 @@ class ReduceReplicas(ActiveMemoryManagerPolicy):
     """
 
     def run(self):
-        # TODO this is O(n) to the total number of in-memory tasks on the cluster; it
-        #      could be made faster by automatically attaching it to a TaskState when it
-        #      goes above one replica and detaching it when it drops below two.
-        for ts in self.manager.scheduler.tasks.values():
-            if len(ts.who_has) < 2:
-                continue
-
+        for ts in self.manager.scheduler.replicated_tasks:
             desired_replicas = 1  # TODO have a marker on TaskState
 
             # If a dependent task has not been assigned to a worker yet, err on the side
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index ddd60433d0d..9a7162a486e 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -1853,6 +1853,7 @@ class SchedulerState:
     _task_groups: dict
     _task_prefixes: dict
     _task_metadata: dict
+    _replicated_tasks: set
     _total_nthreads: Py_ssize_t
     _total_occupancy: double
     _transitions_table: dict
@@ -1912,6 +1913,9 @@ def __init__(
             self._tasks = tasks
         else:
             self._tasks = dict()
+        self._replicated_tasks = {
+            ts for ts in self._tasks.values() if len(ts._who_has) > 1
+        }
         self._computations = deque(
             maxlen=dask.config.get("distributed.diagnostics.computations.max-history")
         )
@@ -2029,6 +2033,10 @@ def task_prefixes(self):
     def task_metadata(self):
         return self._task_metadata
 
+    @property
+    def replicated_tasks(self):
+        return self._replicated_tasks
+
     @property
     def total_nthreads(self):
         return self._total_nthreads
@@ -2814,18 +2822,14 @@ def transition_memory_released(self, key, safe: bint = False):
                     dts._waiting_on.add(ts)
 
             # XXX factor this out?
-            ts_nbytes: Py_ssize_t = ts.get_nbytes()
             worker_msg = {
                 "op": "free-keys",
                 "keys": [key],
                 "reason": f"Memory->Released {key}",
             }
             for ws in ts._who_has:
-                del ws._has_what[ts]
-                ws._nbytes -= ts_nbytes
                 worker_msgs[ws._address] = [worker_msg]
-
-            ts._who_has.clear()
+            self.remove_all_replicas(ts)
 
             ts.state = "released"
 
@@ -3423,6 +3427,40 @@ def worker_objective(self, ts: TaskState, ws: WorkerState) -> tuple:
         else:
             return (start_time, ws._nbytes)
 
+    @ccall
+    def add_replica(self, ts: TaskState, ws: WorkerState):
+        """Note that a worker holds a replica of a task with state='memory'"""
+        if self._validate:
+            assert ws not in ts._who_has
+            assert ts not in ws._has_what
+
+        ws._nbytes += ts.get_nbytes()
+        ws._has_what[ts] = None
+        ts._who_has.add(ws)
+        if len(ts._who_has) == 2:
+            self._replicated_tasks.add(ts)
+
+    @ccall
+    def remove_replica(self, ts: TaskState, ws: WorkerState):
+        """Note that a worker no longer holds a replica of a task"""
+        ws._nbytes -= ts.get_nbytes()
+        del ws._has_what[ts]
+        ts._who_has.remove(ws)
+        if len(ts._who_has) == 1:
+            self._replicated_tasks.remove(ts)
+
+    @ccall
+    def remove_all_replicas(self, ts: TaskState):
+        """Remove all replicas of a task from all workers"""
+        ws: WorkerState
+        nbytes: Py_ssize_t = ts.get_nbytes()
+        for ws in ts._who_has:
+            ws._nbytes -= nbytes
+            del ws._has_what[ts]
+        if len(ts._who_has) > 1:
+            self._replicated_tasks.remove(ts)
+        ts._who_has.clear()
+
 
 class Scheduler(SchedulerState, ServerNode):
     """Dynamic distributed task scheduler
@@ -4864,14 +4902,13 @@ async def remove_worker(self, comm=None, address=None, safe=False, close=True):
                             self.allowed_failures,
                         )
 
-            for ts in ws._has_what:
-                ts._who_has.remove(ws)
+            for ts in list(ws._has_what):
+                parent.remove_replica(ts, ws)
                 if not ts._who_has:
                     if ts._run_spec:
                         recommendations[ts._key] = "released"
                     else:  # pure data
                         recommendations[ts._key] = "forgotten"
-            ws._has_what.clear()
 
             self.transitions(recommendations)
 
@@ -5021,6 +5058,7 @@ def validate_memory(self, key):
         ts: TaskState = parent._tasks[key]
         dts: TaskState
         assert ts._who_has
+        assert bool(ts in parent._replicated_tasks) == (len(ts._who_has) > 1)
         assert not ts._processing_on
         assert not ts._waiting_on
         assert ts not in parent._unrunnable
@@ -5091,8 +5129,13 @@ def validate_state(self, allow_overlap=False):
         for k, ts in parent._tasks.items():
             assert isinstance(ts, TaskState), (type(ts), ts)
             assert ts._key == k
+            assert bool(ts in parent._replicated_tasks) == (len(ts._who_has) > 1)
             self.validate_key(k, ts)
 
+        for ts in parent._replicated_tasks:
+            assert ts._state == "memory"
+            assert ts._key in parent._tasks
+
         c: str
         cs: ClientState
         for c, cs in parent._clients.items():
@@ -5303,9 +5346,7 @@ def handle_missing_data(self, key=None, errant_worker=None, **kwargs):
             return
         ws: WorkerState = parent._workers_dv.get(errant_worker)
         if ws is not None and ws in ts._who_has:
-            ts._who_has.remove(ws)
-            del ws._has_what[ts]
-            ws._nbytes -= ts.get_nbytes()
+            parent.remove_replica(ts, ws)
         if not ts._who_has:
             if ts._run_spec:
                 self.transitions({key: "released"})
@@ -5319,12 +5360,9 @@ def release_worker_data(self, comm=None, key=None, worker=None):
         if not ws or not ts:
             return
         recommendations: dict = {}
-        if ts in ws._has_what:
-            del ws._has_what[ts]
-            ws._nbytes -= ts.get_nbytes()
-            wh: set = ts._who_has
-            wh.remove(ws)
-            if not wh:
+        if ws in ts._who_has:
+            parent.remove_replica(ts, ws)
+            if not ts._who_has:
                 recommendations[ts._key] = "released"
         if recommendations:
             self.transitions(recommendations)
@@ -5659,14 +5697,11 @@ async def gather(self, comm=None, keys=None, serializers=None):
                     )
                     if not workers or ts is None:
                         continue
-                    ts_nbytes: Py_ssize_t = ts.get_nbytes()
                     recommendations: dict = {key: "released"}
                     for worker in workers:
                         ws = parent._workers_dv.get(worker)
-                        if ws is not None and ts in ws._has_what:
-                            del ws._has_what[ts]
-                            ts._who_has.remove(ws)
-                            ws._nbytes -= ts_nbytes
+                        if ws is not None and ws in ts._who_has:
+                            parent.remove_replica(ts, ws)
                             parent._transitions(
                                 recommendations, client_msgs, worker_msgs
                             )
@@ -5865,10 +5900,8 @@ async def gather_on_worker(
             if ts is None or ts._state != "memory":
                 logger.warning(f"Key lost during replication: {key}")
                 continue
-            if ts not in ws._has_what:
-                ws._nbytes += ts.get_nbytes()
-                ws._has_what[ts] = None
-                ts._who_has.add(ws)
+            if ws not in ts._who_has:
+                parent.add_replica(ts, ws)
 
         return keys_failed
 
@@ -5905,11 +5938,9 @@ async def delete_worker_data(self, worker_address: str, keys: "list[str]") -> No
 
         for key in keys:
             ts: TaskState = parent._tasks.get(key)
-            if ts is not None and ts in ws._has_what:
+            if ts is not None and ws in ts._who_has:
                 assert ts._state == "memory"
-                del ws._has_what[ts]
-                ts._who_has.remove(ws)
-                ws._nbytes -= ts.get_nbytes()
+                parent.remove_replica(ts, ws)
                 if not ts._who_has:
                     # Last copy deleted
                     self.transitions({key: "released"})
@@ -6657,10 +6688,8 @@ def add_keys(self, comm=None, worker=None, keys=()):
         for key in keys:
             ts: TaskState = parent._tasks.get(key)
             if ts is not None and ts._state == "memory":
-                if ts not in ws._has_what:
-                    ws._nbytes += ts.get_nbytes()
-                    ws._has_what[ts] = None
-                    ts._who_has.add(ws)
+                if ws not in ts._who_has:
+                    parent.add_replica(ts, ws)
             else:
                 redundant_replicas.append(key)
 
@@ -6703,17 +6732,14 @@ def update_data(
                 if ts is None:
                     ts: TaskState = parent.new_task(key, None, "memory")
                 ts.state = "memory"
-                ts_nbytes: Py_ssize_t = nbytes.get(key, -1)
+                ts_nbytes = nbytes.get(key, -1)
                 if ts_nbytes >= 0:
                     ts.set_nbytes(ts_nbytes)
-                else:
-                    ts_nbytes = ts.get_nbytes()
+
                 for w in workers:
                     ws: WorkerState = parent._workers_dv[w]
-                    if ts not in ws._has_what:
-                        ws._nbytes += ts_nbytes
-                        ws._has_what[ts] = None
-                        ts._who_has.add(ws)
+                    if ws not in ts._who_has:
+                        parent.add_replica(ts, ws)
                 self.report(
                     {"op": "key-in-memory", "key": key, "workers": list(workers)}
                 )
@@ -7680,9 +7706,7 @@ def _add_to_memory(
     if state._validate:
         assert ts not in ws._has_what
 
-    ts._who_has.add(ws)
-    ws._has_what[ts] = None
-    ws._nbytes += ts.get_nbytes()
+    state.add_replica(ts, ws)
 
     deps: list = list(ts._dependents)
     if len(deps) > 1:
@@ -7758,12 +7782,8 @@ def _propagate_forgotten(
     ts._dependencies.clear()
     ts._waiting_on.clear()
 
-    ts_nbytes: Py_ssize_t = ts.get_nbytes()
-
     ws: WorkerState
     for ws in ts._who_has:
-        del ws._has_what[ts]
-        ws._nbytes -= ts_nbytes
         w: str = ws._address
         if w in state._workers_dv:  # in case worker has died
             worker_msgs[w] = [
@@ -7773,7 +7793,7 @@ def _propagate_forgotten(
                     "reason": f"propagate-forgotten {ts.key}",
                 }
             ]
-    ts._who_has.clear()
+    state.remove_all_replicas(ts)
 
 
 @cfunc

From ef281377339d8bfc88874f0c5a48d99564f66633 Mon Sep 17 00:00:00 2001
From: "Mads R. B. Kristensen" <madsbk@gmail.com>
Date: Tue, 28 Sep 2021 19:18:38 +0400
Subject: [PATCH 1508/1550] error_message(): pickle exception and traceback
 immediately (#5338)

error_message(): pickle exception and traceback immediately
---
 distributed/client.py                  |  5 +++--
 distributed/core.py                    | 30 ++++++++++++++------------
 distributed/diagnostics/progressbar.py |  4 +++-
 distributed/tests/test_core.py         |  7 +++---
 distributed/tests/test_scheduler.py    | 10 ++++-----
 distributed/worker.py                  |  2 +-
 6 files changed, 31 insertions(+), 27 deletions(-)

diff --git a/distributed/client.py b/distributed/client.py
index 51999a8f07f..e0eabbf76ab 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -4107,8 +4107,9 @@ async def _register_worker_plugin(self, plugin=None, name=None, nanny=None):
         responses = await method(plugin=dumps(plugin, protocol=4), name=name)
         for response in responses.values():
             if response["status"] == "error":
-                exc = response["exception"]
-                tb = response["traceback"]
+                _, exc, tb = clean_exception(
+                    response["exception"], response["traceback"]
+                )
                 raise exc.with_traceback(tb)
         return responses
 
diff --git a/distributed/core.py b/distributed/core.py
index fd083972379..c61b40d1051 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -1155,34 +1155,36 @@ def error_message(e, status="error"):
     MAX_ERROR_LEN = dask.config.get("distributed.admin.max-error-length")
     tblib.pickling_support.install(e, *collect_causes(e))
     tb = get_traceback()
-    e2 = truncate_exception(e, MAX_ERROR_LEN)
+    tb_text = "".join(traceback.format_tb(tb))
+    e = truncate_exception(e, MAX_ERROR_LEN)
     try:
-        e3 = protocol.pickle.dumps(e2, protocol=4)
-        protocol.pickle.loads(e3)
+        e_serialized = protocol.pickle.dumps(e)
+        protocol.pickle.loads(e_serialized)
     except Exception:
-        e2 = Exception(str(e2))
-    e4 = protocol.to_serialize(e2)
+        e_serialized = protocol.pickle.dumps(Exception(repr(e)))
+    e_serialized = protocol.to_serialize(e_serialized)
+
     try:
-        tb2 = protocol.pickle.dumps(tb, protocol=4)
-        protocol.pickle.loads(tb2)
+        tb_serialized = protocol.pickle.dumps(tb)
+        protocol.pickle.loads(tb_serialized)
     except Exception:
-        tb = tb2 = "".join(traceback.format_tb(tb))
+        tb_serialized = protocol.pickle.dumps(tb_text)
 
-    if len(tb2) > MAX_ERROR_LEN:
+    if len(tb_serialized) > MAX_ERROR_LEN:
         tb_result = None
     else:
-        tb_result = protocol.to_serialize(tb)
+        tb_result = protocol.to_serialize(tb_serialized)
 
     return {
         "status": status,
-        "exception": e4,
+        "exception": e_serialized,
         "traceback": tb_result,
-        "exception_text": repr(e2),
-        "traceback_text": "".join(traceback.format_tb(tb)),
+        "exception_text": repr(e),
+        "traceback_text": tb_text,
     }
 
 
-def clean_exception(exception, traceback, **kwargs):
+def clean_exception(exception, traceback=None, **kwargs):
     """Reraise exception and traceback. Deserialize if necessary
 
     See Also
diff --git a/distributed/diagnostics/progressbar.py b/distributed/diagnostics/progressbar.py
index a85b72d4b02..02ab8420f47 100644
--- a/distributed/diagnostics/progressbar.py
+++ b/distributed/diagnostics/progressbar.py
@@ -11,7 +11,7 @@
 import dask
 
 from ..client import default_client, futures_of
-from ..core import CommClosedError, coerce_to_address, connect
+from ..core import CommClosedError, clean_exception, coerce_to_address, connect
 from ..protocol.pickle import dumps
 from ..utils import LoopRunner, is_kernel, key_split
 from .progress import MultiProgress, Progress, format_time
@@ -175,6 +175,7 @@ def _ipython_display_(self, **kwargs):
 
     def _draw_stop(self, remaining, status, exception=None, **kwargs):
         if status == "error":
+            _, exception, _ = clean_exception(exception)
             self.bar.bar_style = "danger"
             self.elapsed_time.value = (
                 '<div style="padding: 0px 10px 5px 10px"><b>Exception</b> '
@@ -361,6 +362,7 @@ def _draw_stop(self, remaining, status, exception=None, key=None, **kwargs):
                 self.bars[k].bar_style = "danger"
 
         if status == "error":
+            _, exception, _ = clean_exception(exception)
             # self.bars[self.func(key)].bar_style = 'danger'  # TODO
             self.elapsed_time.value = (
                 '<div style="padding: 0px 10px 5px 10px"><b>Exception</b> '
diff --git a/distributed/tests/test_core.py b/distributed/tests/test_core.py
index 09bbd032ed7..3d03ccc7017 100644
--- a/distributed/tests/test_core.py
+++ b/distributed/tests/test_core.py
@@ -13,6 +13,7 @@
     ConnectionPool,
     Server,
     Status,
+    clean_exception,
     coerce_to_address,
     connect,
     pingpong,
@@ -135,9 +136,9 @@ async def f():
         await comm.write({"op": "ping"})
         msg = await comm.read()
 
-        assert "exception" in msg
-        assert isinstance(msg["exception"], ValueError)
-        assert "'ping' handler has been explicitly disallowed" in repr(msg["exception"])
+        _, exception, _ = clean_exception(msg["exception"])
+        assert isinstance(exception, ValueError)
+        assert "'ping' handler has been explicitly disallowed" in repr(exception)
 
         await comm.close()
         server.stop()
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 4d841bc4939..b1179f423da 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -23,7 +23,7 @@
 from distributed import Client, Nanny, Worker, fire_and_forget, wait
 from distributed.comm import Comm
 from distributed.compatibility import LINUX, WINDOWS
-from distributed.core import ConnectionPool, Status, connect, rpc
+from distributed.core import ConnectionPool, Status, clean_exception, connect, rpc
 from distributed.metrics import time
 from distributed.protocol.pickle import dumps
 from distributed.scheduler import MemoryState, Scheduler
@@ -413,11 +413,9 @@ def func(scheduler):
 
     response = await comm.read()
 
-    assert "exception" in response
-    assert isinstance(response["exception"], ValueError)
-    assert "'feed' handler has been explicitly disallowed" in repr(
-        response["exception"]
-    )
+    _, exc, _ = clean_exception(response["exception"], response["traceback"])
+    assert isinstance(exc, ValueError)
+    assert "'feed' handler has been explicitly disallowed" in repr(exc)
 
     await comm.close()
 
diff --git a/distributed/worker.py b/distributed/worker.py
index 7d1a2710b8c..185e6d4d8f3 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -3082,7 +3082,7 @@ async def execute(self, key, *, stimulus_id):
                     str(funcname(function))[:1000],
                     convert_args_to_str(args2, max_len=1000),
                     convert_kwargs_to_str(kwargs2, max_len=1000),
-                    result["exception"].data,
+                    ts.exception_text,
                 )
                 recommendations[ts] = (
                     "error",

From 2701d4a330f01e12fdc831938137a43181629669 Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Tue, 28 Sep 2021 20:27:07 +0200
Subject: [PATCH 1509/1550] Workers submit a reply to the scheduler if replica
 removal was rejected (#5356)

* Workers submit a reply to the scheduler if replica removal was rejected

* Ensure scheduler who_has state is restored after worker response

* Update distributed/tests/test_worker.py

Co-authored-by: crusaderky <crusaderky@gmail.com>

* review comments about test

* Update distributed/tests/test_worker.py

Co-authored-by: crusaderky <crusaderky@gmail.com>

Co-authored-by: crusaderky <crusaderky@gmail.com>
---
 distributed/scheduler.py         |  6 ++-
 distributed/tests/test_worker.py | 67 +++++++++++++++++++++++++-------
 distributed/worker.py            | 23 +++++++++--
 3 files changed, 75 insertions(+), 21 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 9a7162a486e..cfe39f9c72b 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -6673,7 +6673,7 @@ async def retire_workers(
 
                 return worker_keys
 
-    def add_keys(self, comm=None, worker=None, keys=()):
+    def add_keys(self, comm=None, worker=None, keys=(), stimulus_id=None):
         """
         Learn that a worker has certain keys
 
@@ -6694,12 +6694,14 @@ def add_keys(self, comm=None, worker=None, keys=()):
                 redundant_replicas.append(key)
 
         if redundant_replicas:
+            if not stimulus_id:
+                stimulus_id = f"redundant-replicas-{time()}"
             self.worker_send(
                 worker,
                 {
                     "op": "remove-replicas",
                     "keys": redundant_replicas,
-                    "stimulus_id": f"redundant-replicas-{time()}",
+                    "stimulus_id": stimulus_id,
                 },
             )
 
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index c28a521ea72..8c9413dec56 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -2760,8 +2760,12 @@ def _acquire_replicas(scheduler, worker, *futures):
 
 def _remove_replicas(scheduler, worker, *futures):
     keys = [f.key for f in futures]
-
-    scheduler.stream_comms[worker.address].send(
+    ws = scheduler.workers[worker.address]
+    for k in keys:
+        ts = scheduler.tasks[k]
+        if ws in ts.who_has:
+            scheduler.remove_replica(ts, ws)
+    scheduler.stream_comms[ws.address].send(
         {
             "op": "remove-replicas",
             "keys": keys,
@@ -2852,21 +2856,26 @@ async def test_remove_replica_simple(c, s, a, b):
 
     _remove_replicas(s, b, *futs)
 
+    assert all(len(s.tasks[f.key].who_has) == 1 for f in futs)
+
     while b.tasks:
         await asyncio.sleep(0.01)
 
-    # might take a moment for the reply to reach the scheduler
-    while not all(len(s.tasks[f.key].who_has) == 1 for f in futs):
-        await asyncio.sleep(0.01)
+    # Ensure there is no delayed reply to re-register the key
+    await asyncio.sleep(0.01)
+    assert all(s.tasks[f.key].who_has == {s.workers[a.address]} for f in futs)
 
 
-@gen_cluster(client=True)
+@gen_cluster(
+    client=True,
+    config={"distributed.comm.recent-messages-log-length": 1_000},
+)
 async def test_remove_replica_while_computing(c, s, *workers):
     futs = c.map(inc, range(10), workers=[workers[0].address])
 
     # All interesting things will happen on that worker
     w = workers[1]
-    intermediate = c.map(slowinc, futs, delay=0.1, workers=[w.address])
+    intermediate = c.map(slowinc, futs, delay=0.05, workers=[w.address])
 
     def reduce(*args, **kwargs):
         import time
@@ -2875,24 +2884,52 @@ def reduce(*args, **kwargs):
         return
 
     final = c.submit(reduce, intermediate, workers=[w.address], key="final")
-    while final.key not in w.tasks:
+
+    while not any(f.key in w.tasks for f in intermediate):
         await asyncio.sleep(0.001)
 
+    # The scheduler removes keys from who_has/has_what immediately
+    # Make sure the worker responds to the rejection and the scheduler corrects
+    # the state
+    ws = s.workers[w.address]
+    while not any(s.tasks[fut.key] in ws.has_what for fut in futs):
+        await asyncio.sleep(0.001)
+
+    _remove_replicas(s, w, *futs)
+    # Scheduler removed keys immediately...
+    assert not any(s.tasks[fut.key] in ws.has_what for fut in futs)
+    # ... but the state is properly restored
+    while not any(s.tasks[fut.key] in ws.has_what for fut in futs):
+        await asyncio.sleep(0.01)
+
+    # The worker should reject all of these since they are required
     while not all(fut.done() for fut in intermediate):
-        # The worker should reject all of these since they are required
         _remove_replicas(s, w, *futs)
-        _remove_replicas(s, w, *intermediate)
-        await asyncio.sleep(0.001)
+        await asyncio.sleep(0.01)
 
     await wait(intermediate)
 
+    # If a request is rejected, the worker responds with an add-keys message to
+    # reenlist the key in the schedulers state system to avoid race conditions,
+    # see also https://github.com/dask/distributed/issues/5265
+    rejections = set()
+    for msg in w.log:
+        if msg[0] == "remove-replica-rejected":
+            rejections.update(msg[1])
+    for rejected_key in rejections:
+
+        def answer_sent(key):
+            for batch in w.batched_stream.recent_message_log:
+                for msg in batch:
+                    if "op" in msg and msg["op"] == "add-keys" and key in msg["keys"]:
+                        return True
+            return False
+
+        assert answer_sent(rejected_key)
+
     # Since intermediate is done, futs replicas may be removed.
     # They might be already gone due to the above remove replica calls
     _remove_replicas(s, w, *futs)
-    # the intermediate tasks should not be touched because they are still needed
-    # (the scheduler should not have made the above call but we should be safe
-    # regarless)
-    assert all(w.tasks[f.key].state == "memory" for f in intermediate)
 
     while any(w.tasks[f.key].state != "released" for f in futs if f.key in w.tasks):
         await asyncio.sleep(0.001)
diff --git a/distributed/worker.py b/distributed/worker.py
index 185e6d4d8f3..859f6ca39c8 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -1586,7 +1586,9 @@ def update_data(
             self.log.append((key, "receive-from-scatter"))
 
         if report:
-            scheduler_messages.append({"op": "add-keys", "keys": list(data)})
+            scheduler_messages.append(
+                {"op": "add-keys", "keys": list(data), "stimulus_id": stimulus_id}
+            )
 
         self.transitions(recommendations, stimulus_id=stimulus_id)
         for msg in scheduler_messages:
@@ -1634,10 +1636,23 @@ def handle_remove_replicas(self, keys, stimulus_id):
         """
         self.log.append(("remove-replicas", keys, stimulus_id))
         recommendations = {}
+
+        rejected = []
         for key in keys:
             ts = self.tasks.get(key)
-            if ts and not ts.is_protected():
+            if ts is None or ts.state != "memory":
+                continue
+            if not ts.is_protected():
+                self.log.append(("remove-replica-confirmed", ts.key, stimulus_id))
                 recommendations[ts] = "released" if ts.dependents else "forgotten"
+            else:
+                rejected.append(key)
+
+        if rejected:
+            self.log.append(("remove-replica-rejected", rejected, stimulus_id))
+            self.batched_stream.send(
+                {"op": "add-keys", "keys": rejected, "stimulus_id": stimulus_id}
+            )
 
         self.transitions(recommendations=recommendations, stimulus_id=stimulus_id)
 
@@ -2111,14 +2126,14 @@ def transition_executing_long_running(self, ts, compute_duration, *, stimulus_id
 
     def transition_released_memory(self, ts, value, *, stimulus_id):
         recs, smsgs = self._put_key_in_memory(ts, value, stimulus_id=stimulus_id)
-        smsgs.append({"op": "add-keys", "keys": [ts.key]})
+        smsgs.append({"op": "add-keys", "keys": [ts.key], "stimulus_id": stimulus_id})
         return recs, smsgs
 
     def transition_flight_memory(self, ts, value, *, stimulus_id):
         self._in_flight_tasks.discard(ts)
         ts.coming_from = None
         recs, smsgs = self._put_key_in_memory(ts, value, stimulus_id=stimulus_id)
-        smsgs.append({"op": "add-keys", "keys": [ts.key]})
+        smsgs.append({"op": "add-keys", "keys": [ts.key], "stimulus_id": stimulus_id})
         return recs, smsgs
 
     def transition_released_forgotten(self, ts, *, stimulus_id):

From 70158c8c1d10427cd25e1d6a024bd7a24484916c Mon Sep 17 00:00:00 2001
From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com>
Date: Tue, 28 Sep 2021 17:09:33 -0400
Subject: [PATCH 1510/1550] Add check for unsupported NVML metrics (#5343)

Add check for unsupported NVML metrics
---
 distributed/diagnostics/nvml.py | 36 +++++++++++++++++++++++++++++----
 1 file changed, 32 insertions(+), 4 deletions(-)

diff --git a/distributed/diagnostics/nvml.py b/distributed/diagnostics/nvml.py
index 1b45906a7a5..45b75c38179 100644
--- a/distributed/diagnostics/nvml.py
+++ b/distributed/diagnostics/nvml.py
@@ -81,17 +81,45 @@ def has_cuda_context():
     return False
 
 
+def _get_utilization(h):
+    try:
+        return pynvml.nvmlDeviceGetUtilizationRates(h).gpu
+    except pynvml.NVMLError_NotSupported:
+        return None
+
+
+def _get_memory_used(h):
+    try:
+        return pynvml.nvmlDeviceGetMemoryInfo(h).used
+    except pynvml.NVMLError_NotSupported:
+        return None
+
+
+def _get_memory_total(h):
+    try:
+        return pynvml.nvmlDeviceGetMemoryInfo(h).total
+    except pynvml.NVMLError_NotSupported:
+        return None
+
+
+def _get_name(h):
+    try:
+        return pynvml.nvmlDeviceGetName(h).decode()
+    except pynvml.NVMLError_NotSupported:
+        return None
+
+
 def real_time():
     h = _pynvml_handles()
     return {
-        "utilization": pynvml.nvmlDeviceGetUtilizationRates(h).gpu,
-        "memory-used": pynvml.nvmlDeviceGetMemoryInfo(h).used,
+        "utilization": _get_utilization(h),
+        "memory-used": _get_memory_used(h),
     }
 
 
 def one_time():
     h = _pynvml_handles()
     return {
-        "memory-total": pynvml.nvmlDeviceGetMemoryInfo(h).total,
-        "name": pynvml.nvmlDeviceGetName(h).decode(),
+        "memory-total": _get_memory_total(h),
+        "name": _get_name(h),
     }

From 7e2fe5c6995e3628e4c2d6987c00a591160f7c7b Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Tue, 28 Sep 2021 17:21:50 -0500
Subject: [PATCH 1511/1550] Avoid deprecated random set sampling (#5360)

---
 distributed/scheduler.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index cfe39f9c72b..6a6db209fed 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -6364,7 +6364,7 @@ async def replicate(
             if delete:
                 del_worker_tasks = defaultdict(set)
                 for ts in tasks:
-                    del_candidates = ts._who_has & workers
+                    del_candidates = tuple(ts._who_has & workers)
                     if len(del_candidates) > n:
                         for ws in random.sample(
                             del_candidates, len(del_candidates) - n
@@ -6396,7 +6396,7 @@ async def replicate(
                     count = min(n_missing, branching_factor * len(ts._who_has))
                     assert count > 0
 
-                    for ws in random.sample(workers - ts._who_has, count):
+                    for ws in random.sample(tuple(workers - ts._who_has), count):
                         gathers[ws._address][ts._key] = [
                             wws._address for wws in ts._who_has
                         ]

From 43d386669488b16b2525fa8e7a83270ff3642331 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Wed, 29 Sep 2021 15:04:50 -0500
Subject: [PATCH 1512/1550] Use Dask temporary file utility  (#5361)

---
 distributed/cli/tests/test_dask_scheduler.py |  4 +++-
 distributed/cli/tests/test_dask_worker.py    |  4 +++-
 distributed/protocol/tests/test_h5py.py      |  3 ++-
 distributed/protocol/tests/test_netcdf4.py   |  3 ++-
 distributed/protocol/tests/test_numpy.py     |  4 +++-
 distributed/scheduler.py                     |  3 +--
 distributed/tests/test_client.py             |  4 ++--
 distributed/tests/test_nanny.py              |  3 ++-
 distributed/tests/test_scheduler.py          |  4 ++--
 distributed/tests/test_utils.py              |  6 ++++++
 distributed/tests/test_worker.py             |  3 ++-
 distributed/utils.py                         | 21 +-------------------
 12 files changed, 29 insertions(+), 33 deletions(-)

diff --git a/distributed/cli/tests/test_dask_scheduler.py b/distributed/cli/tests/test_dask_scheduler.py
index a3efa41b97a..2a966fac496 100644
--- a/distributed/cli/tests/test_dask_scheduler.py
+++ b/distributed/cli/tests/test_dask_scheduler.py
@@ -12,12 +12,14 @@
 import requests
 from click.testing import CliRunner
 
+from dask.utils import tmpfile
+
 import distributed
 import distributed.cli.dask_scheduler
 from distributed import Client, Scheduler
 from distributed.compatibility import LINUX
 from distributed.metrics import time
-from distributed.utils import get_ip, get_ip_interface, tmpfile
+from distributed.utils import get_ip, get_ip_interface
 from distributed.utils_test import (
     assert_can_connect_from_everywhere_4_6,
     assert_can_connect_locally_4,
diff --git a/distributed/cli/tests/test_dask_worker.py b/distributed/cli/tests/test_dask_worker.py
index 3b334fa0aed..1adeef3411e 100644
--- a/distributed/cli/tests/test_dask_worker.py
+++ b/distributed/cli/tests/test_dask_worker.py
@@ -11,12 +11,14 @@
 
 import requests
 
+from dask.utils import tmpfile
+
 import distributed.cli.dask_worker
 from distributed import Client, Scheduler
 from distributed.compatibility import LINUX
 from distributed.deploy.utils import nprocesses_nthreads
 from distributed.metrics import time
-from distributed.utils import parse_ports, sync, tmpfile
+from distributed.utils import parse_ports, sync
 from distributed.utils_test import (
     gen_cluster,
     popen,
diff --git a/distributed/protocol/tests/test_h5py.py b/distributed/protocol/tests/test_h5py.py
index 0ab3f864281..f1f634518a8 100644
--- a/distributed/protocol/tests/test_h5py.py
+++ b/distributed/protocol/tests/test_h5py.py
@@ -5,8 +5,9 @@
 
 h5py = pytest.importorskip("h5py")
 
+from dask.utils import tmpfile
+
 from distributed.protocol import deserialize, serialize
-from distributed.utils import tmpfile
 
 
 def silence_h5py_issue775(func):
diff --git a/distributed/protocol/tests/test_netcdf4.py b/distributed/protocol/tests/test_netcdf4.py
index b37c116ec43..d8d6bf3821b 100644
--- a/distributed/protocol/tests/test_netcdf4.py
+++ b/distributed/protocol/tests/test_netcdf4.py
@@ -3,8 +3,9 @@
 netCDF4 = pytest.importorskip("netCDF4")
 np = pytest.importorskip("numpy")
 
+from dask.utils import tmpfile
+
 from distributed.protocol import deserialize, serialize
-from distributed.utils import tmpfile
 
 
 def create_test_dataset(fn):
diff --git a/distributed/protocol/tests/test_numpy.py b/distributed/protocol/tests/test_numpy.py
index effcb609c96..3fb7dc182ec 100644
--- a/distributed/protocol/tests/test_numpy.py
+++ b/distributed/protocol/tests/test_numpy.py
@@ -4,6 +4,8 @@
 
 np = pytest.importorskip("numpy")
 
+from dask.utils import tmpfile
+
 from distributed.protocol import (
     decompress,
     deserialize,
@@ -18,7 +20,7 @@
 from distributed.protocol.pickle import HIGHEST_PROTOCOL
 from distributed.protocol.utils import BIG_BYTES_SHARD_SIZE
 from distributed.system import MEMORY_LIMIT
-from distributed.utils import ensure_bytes, nbytes, tmpfile
+from distributed.utils import ensure_bytes, nbytes
 from distributed.utils_test import gen_cluster
 
 
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 6a6db209fed..c09195e8dd3 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -36,7 +36,7 @@
 
 import dask
 from dask.highlevelgraph import HighLevelGraph
-from dask.utils import format_bytes, format_time, parse_bytes, parse_timedelta
+from dask.utils import format_bytes, format_time, parse_bytes, parse_timedelta, tmpfile
 from dask.widgets import get_template
 
 from . import preloading, profile
@@ -76,7 +76,6 @@
     key_split_group,
     log_errors,
     no_default,
-    tmpfile,
     validate_key,
 )
 from .utils_comm import gather_from_workers, retry_operation, scatter_to_workers
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 90a6c828bbf..3ca714db282 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -27,7 +27,7 @@
 import dask.bag as db
 from dask import delayed
 from dask.optimization import SubgraphCallable
-from dask.utils import stringify
+from dask.utils import stringify, tmpfile
 
 from distributed import (
     CancelledError,
@@ -67,7 +67,7 @@
     Scheduler,
 )
 from distributed.sizeof import sizeof
-from distributed.utils import is_valid_xml, mp_context, sync, tmp_text, tmpfile
+from distributed.utils import is_valid_xml, mp_context, sync, tmp_text
 from distributed.utils_test import (
     TaskStateMetadataPlugin,
     _UnhashableCallable,
diff --git a/distributed/tests/test_nanny.py b/distributed/tests/test_nanny.py
index d2860ffc9b4..dc3eba86e58 100644
--- a/distributed/tests/test_nanny.py
+++ b/distributed/tests/test_nanny.py
@@ -17,6 +17,7 @@
 from tornado.ioloop import IOLoop
 
 import dask
+from dask.utils import tmpfile
 
 from distributed import Nanny, Scheduler, Worker, rpc, wait, worker
 from distributed.compatibility import LINUX, WINDOWS
@@ -24,7 +25,7 @@
 from distributed.diagnostics import SchedulerPlugin
 from distributed.metrics import time
 from distributed.protocol.pickle import dumps
-from distributed.utils import TimeoutError, parse_ports, tmpfile
+from distributed.utils import TimeoutError, parse_ports
 from distributed.utils_test import captured_logger, gen_cluster, gen_test, inc
 
 pytestmark = pytest.mark.ci1
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index b1179f423da..71ede927082 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -18,7 +18,7 @@
 
 import dask
 from dask import delayed
-from dask.utils import apply, parse_timedelta, stringify, typename
+from dask.utils import apply, parse_timedelta, stringify, tmpfile, typename
 
 from distributed import Client, Nanny, Worker, fire_and_forget, wait
 from distributed.comm import Comm
@@ -27,7 +27,7 @@
 from distributed.metrics import time
 from distributed.protocol.pickle import dumps
 from distributed.scheduler import MemoryState, Scheduler
-from distributed.utils import TimeoutError, tmpfile
+from distributed.utils import TimeoutError
 from distributed.utils_test import (
     captured_logger,
     cluster,
diff --git a/distributed/tests/test_utils.py b/distributed/tests/test_utils.py
index bbc5892eb71..929f8c1f5b7 100644
--- a/distributed/tests/test_utils.py
+++ b/distributed/tests/test_utils.py
@@ -601,6 +601,12 @@ def test_typename_deprecated():
     assert typename is dask.utils.typename
 
 
+def test_tmpfile_deprecated():
+    with pytest.warns(FutureWarning, match="tmpfile is deprecated"):
+        from distributed.utils import tmpfile
+    assert tmpfile is dask.utils.tmpfile
+
+
 def test_iscoroutinefunction_unhashable_input():
     # Ensure iscoroutinefunction can handle unhashable callables
     assert not iscoroutinefunction(_UnhashableCallable())
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 8c9413dec56..8d9ebc2fac9 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -19,6 +19,7 @@
 import dask
 from dask import delayed
 from dask.system import CPU_COUNT
+from dask.utils import tmpfile
 
 import distributed
 from distributed import (
@@ -38,7 +39,7 @@
 from distributed.diagnostics.plugin import PipInstall
 from distributed.metrics import time
 from distributed.scheduler import Scheduler
-from distributed.utils import TimeoutError, tmpfile
+from distributed.utils import TimeoutError
 from distributed.utils_test import (
     TaskStateMetadataPlugin,
     _LockedCommPool,
diff --git a/distributed/utils.py b/distributed/utils.py
index 031dd9c66f5..c306e710e86 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -8,7 +8,6 @@
 import os
 import pkgutil
 import re
-import shutil
 import socket
 import sys
 import tempfile
@@ -837,25 +836,6 @@ def read_block(f, offset, length, delimiter=None):
     return bytes
 
 
-@contextmanager
-def tmpfile(extension=""):
-    extension = "." + extension.lstrip(".")
-    handle, filename = tempfile.mkstemp(extension)
-    os.close(handle)
-    os.remove(filename)
-
-    yield filename
-
-    if os.path.exists(filename):
-        try:
-            if os.path.isdir(filename):
-                shutil.rmtree(filename)
-            else:
-                os.remove(filename)
-        except OSError:  # sometimes we can't remove a generated temp file
-            pass
-
-
 def ensure_bytes(s):
     """Attempt to turn `s` into bytes.
 
@@ -1435,6 +1415,7 @@ def clean_dashboard_address(addrs: AnyType, default_listen_ip: str = "") -> List
     "parse_bytes": "dask.utils.parse_bytes",
     "parse_timedelta": "dask.utils.parse_timedelta",
     "typename": "dask.utils.typename",
+    "tmpfile": "dask.utils.tmpfile",
 }
 
 

From b8d44cbbdb7bcb47a4324caaaa9e3e1c900bd53a Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Wed, 29 Sep 2021 22:54:07 +0100
Subject: [PATCH 1513/1550] Update `_cluster_info` dict in `__init__` (#5305)

---
 distributed/deploy/cluster.py            |  7 ++++++-
 distributed/deploy/tests/test_cluster.py | 11 +++++++++++
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
index 8beef70f6ba..471fbc15d47 100644
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -45,6 +45,7 @@ class Cluster:
     """
 
     _supports_scaling = True
+    _cluster_info = {}
 
     def __init__(self, asynchronous, quiet=False, name=None, scheduler_sync_interval=1):
         self.scheduler_info = {"workers": {}}
@@ -63,7 +64,11 @@ def __init__(self, asynchronous, quiet=False, name=None, scheduler_sync_interval
         if name is None:
             name = str(uuid.uuid4())[:8]
 
-        self._cluster_info = {"name": name, "type": typename(type(self))}
+        self._cluster_info = {
+            "name": name,
+            "type": typename(type(self)),
+            **self._cluster_info,
+        }
         self.status = Status.created
 
     @property
diff --git a/distributed/deploy/tests/test_cluster.py b/distributed/deploy/tests/test_cluster.py
index 159f64596f3..48d8bb65336 100644
--- a/distributed/deploy/tests/test_cluster.py
+++ b/distributed/deploy/tests/test_cluster.py
@@ -32,3 +32,14 @@ async def test_logs_deprecated(cleanup):
     cluster = Cluster(asynchronous=True)
     with pytest.warns(FutureWarning, match="get_logs"):
         cluster.logs()
+
+
+@pytest.mark.asyncio
+async def test_cluster_info():
+    class FooCluster(Cluster):
+        def __init__(self):
+            self._cluster_info["foo"] = "bar"
+            super().__init__(asynchronous=False)
+
+    cluster = FooCluster()
+    assert "foo" in cluster._cluster_info

From be7e83cdd86ffd1f7be806b1abdd7badaf544897 Mon Sep 17 00:00:00 2001
From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com>
Date: Thu, 30 Sep 2021 11:27:15 -0400
Subject: [PATCH 1514/1550] Use dask-spec for SSHCluster (#5191)

---
 distributed/deploy/ssh.py            | 61 +++++++++++++++++++---------
 distributed/deploy/tests/test_ssh.py |  1 -
 2 files changed, 41 insertions(+), 21 deletions(-)

diff --git a/distributed/deploy/ssh.py b/distributed/deploy/ssh.py
index 81e08e72399..05d3d935b96 100644
--- a/distributed/deploy/ssh.py
+++ b/distributed/deploy/ssh.py
@@ -2,15 +2,13 @@
 import sys
 import warnings
 import weakref
+from json import dumps
 from typing import List, Union
 
 import dask
 import dask.config
 
 from ..core import Status
-from ..scheduler import Scheduler as _Scheduler
-from ..utils import cli_keywords
-from ..worker import Worker as _Worker
 from .spec import ProcessInterface, SpecCluster
 
 logger = logging.getLogger(__name__)
@@ -52,8 +50,8 @@ class Worker(Process):
         The address of the scheduler
     address: str
         The hostname where we should run this worker
-    worker_module: str
-        The python module to run to start the worker.
+    worker_class: str
+        The python class to use to create the worker.
     connect_options: dict
         kwargs to be passed to asyncssh connections
     remote_python: str
@@ -69,16 +67,23 @@ def __init__(
         address: str,
         connect_options: dict,
         kwargs: dict,
-        worker_module="distributed.cli.dask_worker",
+        worker_module="deprecated",
+        worker_class="distributed.Nanny",
         remote_python=None,
         loop=None,
         name=None,
     ):
         super().__init__()
 
+        if worker_module != "deprecated":
+            raise ValueError(
+                "worker_module has been deprecated in favor of worker_class. "
+                "Please specify a Python class rather than a CLI module."
+            )
+
         self.address = address
         self.scheduler = scheduler
-        self.worker_module = worker_module
+        self.worker_class = worker_class
         self.connect_options = connect_options
         self.kwargs = kwargs
         self.name = name
@@ -113,12 +118,20 @@ async def start(self):
                 set_env,
                 self.remote_python,
                 "-m",
-                self.worker_module,
+                "distributed.cli.dask_spec",
                 self.scheduler,
-                "--name",
-                str(self.name),
+                "--spec",
+                "'%s'"
+                % dumps(
+                    {
+                        "cls": self.worker_class,
+                        "opts": {
+                            **self.kwargs,
+                            "name": self.name,
+                        },
+                    }
+                ),
             ]
-            + cli_keywords(self.kwargs, cls=_Worker, cmd=self.worker_module)
         )
 
         self.proc = await self.connection.create_process(cmd)
@@ -194,9 +207,10 @@ async def start(self):
                 set_env,
                 self.remote_python,
                 "-m",
-                "distributed.cli.dask_scheduler",
+                "distributed.cli.dask_spec",
+                "--spec",
+                "'%s'" % dumps({"cls": "distributed.Scheduler", "opts": self.kwargs}),
             ]
-            + cli_keywords(self.kwargs, cls=_Scheduler)
         )
         self.proc = await self.connection.create_process(cmd)
 
@@ -237,7 +251,8 @@ def SSHCluster(
     connect_options: Union[List[dict], dict] = {},
     worker_options: dict = {},
     scheduler_options: dict = {},
-    worker_module: str = "distributed.cli.dask_worker",
+    worker_module: str = "deprecated",
+    worker_class: str = "distributed.Nanny",
     remote_python: Union[str, List[str]] = None,
     **kwargs,
 ):
@@ -275,8 +290,8 @@ def SSHCluster(
         Keywords to pass on to workers.
     scheduler_options : dict, optional
         Keywords to pass on to scheduler.
-    worker_module : str, optional
-        Python module to call to start the worker.
+    worker_class: str
+        The python class to use to create the worker(s).
     remote_python : str or list of str, optional
         Path to Python on remote nodes.
 
@@ -291,15 +306,15 @@ def SSHCluster(
     ... )
     >>> client = Client(cluster)
 
-    An example using a different worker module, in particular the
-    ``dask-cuda-worker`` command from the ``dask-cuda`` project.
+    An example using a different worker class, in particular the
+    ``CUDAWorker`` from the ``dask-cuda`` project:
 
     >>> from dask.distributed import Client, SSHCluster
     >>> cluster = SSHCluster(
     ...     ["localhost", "hostwithgpus", "anothergpuhost"],
     ...     connect_options={"known_hosts": None},
     ...     scheduler_options={"port": 0, "dashboard_address": ":8797"},
-    ...     worker_module="dask_cuda.cli.dask_cuda_worker")
+    ...     worker_class="dask_cuda.CUDAWorker")
     >>> client = Client(cluster)
 
     See Also
@@ -308,6 +323,12 @@ def SSHCluster(
     dask.distributed.Worker
     asyncssh.connect
     """
+    if worker_module != "deprecated":
+        raise ValueError(
+            "worker_module has been deprecated in favor of worker_class. "
+            "Please specify a Python class rather than a CLI module."
+        )
+
     if set(kwargs) & old_cluster_kwargs:
         from .old_ssh import SSHCluster as OldSSHCluster
 
@@ -357,7 +378,7 @@ def SSHCluster(
                 if isinstance(connect_options, dict)
                 else connect_options[i + 1],
                 "kwargs": worker_options,
-                "worker_module": worker_module,
+                "worker_class": worker_class,
                 "remote_python": remote_python[i + 1]
                 if isinstance(remote_python, list)
                 else remote_python,
diff --git a/distributed/deploy/tests/test_ssh.py b/distributed/deploy/tests/test_ssh.py
index 33baf565d6c..67bd310971f 100644
--- a/distributed/deploy/tests/test_ssh.py
+++ b/distributed/deploy/tests/test_ssh.py
@@ -51,7 +51,6 @@ async def test_keywords():
         connect_options=dict(known_hosts=None),
         asynchronous=True,
         worker_options={
-            "nprocs": 2,  # nprocs checks custom arguments with cli_keywords
             "nthreads": 2,
             "memory_limit": "2 GiB",
             "death_timeout": "5s",

From a4680363bdbf28d15754f59b8d218084df2b6c00 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Thu, 30 Sep 2021 10:38:14 -0500
Subject: [PATCH 1515/1550] Ensure `dask-worker` and `dask-scheduler` pick up
 preload configuration values  (#5365)

Co-authored-by: Jian Dai <daijian1@qq.com>
---
 distributed/cli/tests/test_dask_scheduler.py | 14 +++++++++++++
 distributed/cli/tests/test_dask_worker.py    | 22 ++++++++++++++++++++
 distributed/worker.py                        |  4 ++--
 3 files changed, 38 insertions(+), 2 deletions(-)

diff --git a/distributed/cli/tests/test_dask_scheduler.py b/distributed/cli/tests/test_dask_scheduler.py
index 2a966fac496..5812d4257d3 100644
--- a/distributed/cli/tests/test_dask_scheduler.py
+++ b/distributed/cli/tests/test_dask_scheduler.py
@@ -317,6 +317,20 @@ def test_preload_remote_module(loop, tmp_path):
                 )
 
 
+def test_preload_config(loop):
+    # Ensure dask-scheduler pulls the preload from the Dask config if
+    # not specified via a command line option
+    with tmpfile() as fn:
+        env = os.environ.copy()
+        env["DASK_DISTRIBUTED__SCHEDULER__PRELOAD"] = PRELOAD_TEXT
+        with popen(["dask-scheduler", "--scheduler-file", fn], env=env):
+            with Client(scheduler_file=fn, loop=loop) as c:
+                assert (
+                    c.run_on_scheduler(lambda dask_scheduler: dask_scheduler.foo)
+                    == "bar"
+                )
+
+
 PRELOAD_COMMAND_TEXT = """
 import click
 _config = {}
diff --git a/distributed/cli/tests/test_dask_worker.py b/distributed/cli/tests/test_dask_worker.py
index 1adeef3411e..855ce6cd271 100644
--- a/distributed/cli/tests/test_dask_worker.py
+++ b/distributed/cli/tests/test_dask_worker.py
@@ -495,3 +495,25 @@ def worker_type(dask_worker):
 
                 worker_types = await c.run(worker_type)
                 assert all(name == "MyWorker" for name in worker_types.values())
+
+
+@gen_cluster(nthreads=[], client=True)
+async def test_preload_config(c, s):
+    # Ensure dask-worker pulls the preload from the Dask config if
+    # not specified via a command line option
+    preload_text = """
+def dask_setup(worker):
+    worker.foo = 'setup'
+"""
+    env = os.environ.copy()
+    env["DASK_DISTRIBUTED__WORKER__PRELOAD"] = preload_text
+    with popen(
+        [
+            "dask-worker",
+            s.address,
+        ],
+        env=env,
+    ):
+        await c.wait_for_workers(1)
+        [foo] = (await c.run(lambda dask_worker: dask_worker.foo)).values()
+        assert foo == "setup"
diff --git a/distributed/worker.py b/distributed/worker.py
index 859f6ca39c8..1091c7f1e65 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -582,9 +582,9 @@ def __init__(
             self._workdir = self._workspace.new_work_dir(prefix="worker-")
             self.local_directory = self._workdir.dir_path
 
-        if preload is None:
+        if not preload:
             preload = dask.config.get("distributed.worker.preload")
-        if preload_argv is None:
+        if not preload_argv:
             preload_argv = dask.config.get("distributed.worker.preload-argv")
         self.preloads = preloading.process_preloads(
             self, preload, preload_argv, file_dir=self.local_directory

From 7a3ea4c34ae0af6af1137fe3e3bda927aa9eac26 Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Thu, 30 Sep 2021 21:00:04 +0100
Subject: [PATCH 1516/1550] Enable mypy in CI 1/2 (#5328)

---
 .pre-commit-config.yaml                       |   2 +-
 distributed/__init__.py                       |   2 +-
 distributed/_concurrent_futures_thread.py     |  15 ++-
 distributed/_ipython_utils.py                 |  10 +-
 distributed/_version.py                       |   2 +-
 distributed/active_memory_manager.py          |  20 ++--
 distributed/cli/tests/test_dask_scheduler.py  |  19 ++--
 distributed/client.py                         |  26 +++--
 distributed/comm/addressing.py                |  28 +++--
 distributed/comm/core.py                      |  39 +++----
 distributed/comm/inproc.py                    |  12 +-
 distributed/comm/registry.py                  |   4 +-
 distributed/comm/tcp.py                       |  23 ++--
 distributed/comm/ucx.py                       |  29 +++--
 distributed/comm/ws.py                        |  22 ++--
 distributed/compatibility.py                  |   7 +-
 distributed/core.py                           |   9 +-
 distributed/dashboard/components/scheduler.py |  10 +-
 distributed/dashboard/components/shared.py    |   2 +-
 .../dashboard/tests/test_scheduler_bokeh.py   |   3 +-
 distributed/dashboard/utils.py                |   2 +-
 distributed/deploy/adaptive.py                |  10 +-
 distributed/deploy/adaptive_core.py           | 106 +++++++++++-------
 distributed/deploy/cluster.py                 |   5 +-
 distributed/deploy/local.py                   |  11 --
 distributed/deploy/old_ssh.py                 |   7 +-
 distributed/deploy/spec.py                    |  15 ++-
 distributed/deploy/ssh.py                     |  11 +-
 distributed/deploy/tests/test_local.py        |   2 +-
 distributed/deploy/tests/test_spec_cluster.py |   4 +
 distributed/deploy/utils_test.py              |   6 +-
 distributed/diskutils.py                      |   9 +-
 distributed/http/health.py                    |   4 +-
 distributed/http/proxy.py                     |   6 +-
 distributed/http/scheduler/info.py            |   4 +-
 distributed/http/scheduler/json.py            |   4 +-
 distributed/http/scheduler/missing_bokeh.py   |   4 +-
 .../http/scheduler/prometheus/__init__.py     |   4 +-
 distributed/http/utils.py                     |   7 +-
 .../http/worker/prometheus/__init__.py        |   4 +-
 distributed/locket.py                         |  21 ++--
 distributed/multi_lock.py                     |   6 +-
 distributed/nanny.py                          |  10 +-
 distributed/preloading.py                     |  46 ++++++--
 distributed/process.py                        |  16 ---
 distributed/protocol/compression.py           |  12 +-
 distributed/protocol/serialize.py             |   7 +-
 distributed/protocol/tests/test_serialize.py  |  10 +-
 distributed/py.typed                          |   0
 distributed/queues.py                         |   2 +-
 distributed/security.py                       |   2 +-
 distributed/system_monitor.py                 |   2 +-
 distributed/tests/test_actor.py               |   2 +-
 distributed/tests/test_counter.py             |   2 +-
 distributed/tests/test_preload.py             |   2 +-
 distributed/tests/test_scheduler.py           |   7 +-
 distributed/tests/test_security.py            |   2 +-
 distributed/tests/test_semaphore.py           |   4 +-
 distributed/tests/test_utils_comm.py          |   4 +-
 distributed/tests/test_versions.py            |   5 +-
 distributed/utils.py                          |  26 +++--
 distributed/utils_test.py                     |  18 +--
 distributed/worker.py                         |  18 +--
 docs/source/conf.py                           |   8 +-
 docs/source/develop.rst                       |   2 +-
 setup.py                                      |   9 +-
 versioneer.py                                 |   8 +-
 67 files changed, 433 insertions(+), 327 deletions(-)
 create mode 100644 distributed/py.typed

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index c492b6e6407..daf8c3299f7 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -24,4 +24,4 @@ repos:
     hooks:
       - id: pyupgrade
         args:
-          - "--py37-plus"
+          - --py37-plus
diff --git a/distributed/__init__.py b/distributed/__init__.py
index 3da7ab429d0..cc7100b4869 100644
--- a/distributed/__init__.py
+++ b/distributed/__init__.py
@@ -1,7 +1,7 @@
 from . import config  # isort:skip; load distributed configuration first
 from . import widgets  # isort:skip; load distributed widgets second
 import dask
-from dask.config import config
+from dask.config import config  # type: ignore
 from dask.utils import import_required
 
 from ._version import get_versions
diff --git a/distributed/_concurrent_futures_thread.py b/distributed/_concurrent_futures_thread.py
index 1b6f328991f..65c1e4ccf53 100644
--- a/distributed/_concurrent_futures_thread.py
+++ b/distributed/_concurrent_futures_thread.py
@@ -5,20 +5,17 @@
 
 """Implements ThreadPoolExecutor."""
 
+from __future__ import annotations
+
 __author__ = "Brian Quinlan (brian@sweetapp.com)"
 
 import atexit
 import itertools
-from concurrent.futures import _base
-
-try:
-    import queue
-except ImportError:
-    import Queue as queue
-
 import os
+import queue
 import threading
 import weakref
+from concurrent.futures import _base
 
 # Workers are created as daemon threads. This is done to allow the interpreter
 # to exit when there are still idle threads in a ThreadPoolExecutor's thread
@@ -34,7 +31,9 @@
 # workers to exit when their work queues are empty and then waits until the
 # threads finish.
 
-_threads_queues = weakref.WeakKeyDictionary()
+_threads_queues: weakref.WeakKeyDictionary[
+    threading.Thread, queue.Queue
+] = weakref.WeakKeyDictionary()
 _shutdown = False
 
 
diff --git a/distributed/_ipython_utils.py b/distributed/_ipython_utils.py
index a8ac9ff0dfd..7387c869d2d 100644
--- a/distributed/_ipython_utils.py
+++ b/distributed/_ipython_utils.py
@@ -6,13 +6,7 @@
 
 import atexit
 import os
-
-try:
-    import queue
-except ImportError:
-    # Python 2
-    import Queue as queue
-
+import queue
 import sys
 from subprocess import Popen
 from threading import Event, Thread
@@ -135,7 +129,7 @@ def remote_magic(line, cell=None):
 
 
 # cache clients for re-use in remote magic
-remote_magic._clients = {}
+remote_magic._clients = {}  # type: ignore
 
 
 def register_remote_magic(magic_name="remote"):
diff --git a/distributed/_version.py b/distributed/_version.py
index 6360ea703c4..14239348af3 100644
--- a/distributed/_version.py
+++ b/distributed/_version.py
@@ -50,7 +50,7 @@ class NotThisMethod(Exception):
     """Exception raised if a method is not valid for the current scenario."""
 
 
-LONG_VERSION_PY = {}
+LONG_VERSION_PY: dict = {}
 HANDLERS = {}
 
 
diff --git a/distributed/active_memory_manager.py b/distributed/active_memory_manager.py
index a86914cafcf..e2b085274c1 100644
--- a/distributed/active_memory_manager.py
+++ b/distributed/active_memory_manager.py
@@ -13,7 +13,7 @@
 from .utils import import_term
 
 if TYPE_CHECKING:
-    from .scheduler import SchedulerState, TaskState, WorkerState
+    from .scheduler import Scheduler, TaskState, WorkerState
 
 
 class ActiveMemoryManagerExtension:
@@ -31,7 +31,7 @@ class ActiveMemoryManagerExtension:
     ``distributed.scheduler.active-memory-manager``.
     """
 
-    scheduler: SchedulerState
+    scheduler: Scheduler
     policies: set[ActiveMemoryManagerPolicy]
     interval: float
 
@@ -43,7 +43,7 @@ class ActiveMemoryManagerExtension:
 
     def __init__(
         self,
-        scheduler: SchedulerState,
+        scheduler: Scheduler,
         # The following parameters are exposed so that one may create, run, and throw
         # away on the fly a specialized manager, separate from the main one.
         policies: set[ActiveMemoryManagerPolicy] | None = None,
@@ -126,12 +126,14 @@ def run_once(self, comm=None) -> None:
             # populate self.pending
             self._run_policies()
 
-            drop_by_worker = defaultdict(set)
-            repl_by_worker = defaultdict(dict)
+            drop_by_worker: defaultdict[str, set[str]] = defaultdict(set)
+            repl_by_worker: defaultdict[str, dict[str, list[str]]] = defaultdict(dict)
+
             for ts, (pending_repl, pending_drop) in self.pending.items():
                 if not ts.who_has:
                     continue
                 who_has = [ws_snd.address for ws_snd in ts.who_has - pending_drop]
+
                 assert who_has  # Never drop the last replica
                 for ws_rec in pending_repl:
                     assert ws_rec not in ts.who_has
@@ -143,8 +145,8 @@ def run_once(self, comm=None) -> None:
             # Fire-and-forget enact recommendations from policies
             # This is temporary code, waiting for
             # https://github.com/dask/distributed/pull/5046
-            for addr, who_has in repl_by_worker.items():
-                asyncio.create_task(self.scheduler.gather_on_worker(addr, who_has))
+            for addr, who_has_map in repl_by_worker.items():
+                asyncio.create_task(self.scheduler.gather_on_worker(addr, who_has_map))
             for addr, keys in drop_by_worker.items():
                 asyncio.create_task(self.scheduler.delete_worker_data(addr, keys))
             # End temporary code
@@ -215,7 +217,7 @@ def _find_recipient(
         candidates -= pending_repl
         if not candidates:
             return None
-        return min(candidates, key=self.workers_memory.get)
+        return min(candidates, key=self.workers_memory.__getitem__)
 
     def _find_dropper(
         self,
@@ -244,7 +246,7 @@ def _find_dropper(
         candidates -= {waiter_ts.processing_on for waiter_ts in ts.waiters}
         if not candidates:
             return None
-        return max(candidates, key=self.workers_memory.get)
+        return max(candidates, key=self.workers_memory.__getitem__)
 
 
 class ActiveMemoryManagerPolicy:
diff --git a/distributed/cli/tests/test_dask_scheduler.py b/distributed/cli/tests/test_dask_scheduler.py
index 5812d4257d3..b1ee2200c89 100644
--- a/distributed/cli/tests/test_dask_scheduler.py
+++ b/distributed/cli/tests/test_dask_scheduler.py
@@ -209,7 +209,7 @@ def test_scheduler_port_zero(loop):
     with tmpfile() as fn:
         with popen(
             ["dask-scheduler", "--no-dashboard", "--scheduler-file", fn, "--port", "0"]
-        ) as sched:
+        ):
             with Client(scheduler_file=fn, loop=loop) as c:
                 assert c.scheduler.port
                 assert c.scheduler.port != 8786
@@ -217,15 +217,14 @@ def test_scheduler_port_zero(loop):
 
 def test_dashboard_port_zero(loop):
     pytest.importorskip("bokeh")
-    with tmpfile() as fn:
-        with popen(["dask-scheduler", "--dashboard-address", ":0"]) as proc:
-            count = 0
-            while count < 1:
-                line = proc.stderr.readline()
-                if b"dashboard" in line.lower():
-                    sleep(0.01)
-                    count += 1
-                    assert b":0" not in line
+    with popen(["dask-scheduler", "--dashboard-address", ":0"]) as proc:
+        count = 0
+        while count < 1:
+            line = proc.stderr.readline()
+            if b"dashboard" in line.lower():
+                sleep(0.01)
+                count += 1
+                assert b":0" not in line
 
 
 PRELOAD_TEXT = """
diff --git a/distributed/client.py b/distributed/client.py
index e0eabbf76ab..cf370ec4692 100644
--- a/distributed/client.py
+++ b/distributed/client.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import asyncio
 import atexit
 import copy
@@ -22,6 +24,7 @@
 from functools import partial
 from numbers import Number
 from queue import Queue as pyQueue
+from typing import ClassVar
 
 from tlz import first, groupby, keymap, merge, partition_all, valmap
 
@@ -49,7 +52,7 @@
 from tornado import gen
 from tornado.ioloop import IOLoop, PeriodicCallback
 
-from . import versions as version_module
+from . import versions as version_module  # type: ignore
 from .batched import BatchedSend
 from .cfexecutor import ClientExecutor
 from .core import (
@@ -95,7 +98,9 @@
 
 logger = logging.getLogger(__name__)
 
-_global_clients = weakref.WeakValueDictionary()
+_global_clients: weakref.WeakValueDictionary[
+    int, Client
+] = weakref.WeakValueDictionary()
 _global_client_index = [0]
 
 _current_client = ContextVar("_current_client", default=None)
@@ -105,7 +110,7 @@
 NO_DEFAULT_PLACEHOLDER = "_no_default_"
 
 
-def _get_global_client():
+def _get_global_client() -> Client | None:
     L = sorted(list(_global_clients), reverse=True)
     for k in L:
         c = _global_clients[k]
@@ -116,13 +121,13 @@ def _get_global_client():
     return None
 
 
-def _set_global_client(c):
+def _set_global_client(c: Client | None) -> None:
     if c is not None:
         _global_clients[_global_client_index[0]] = c
         _global_client_index[0] += 1
 
 
-def _del_global_client(c):
+def _del_global_client(c: Client) -> None:
     for k in list(_global_clients):
         try:
             if _global_clients[k] is c:
@@ -590,7 +595,7 @@ class Client:
     distributed.LocalCluster:
     """
 
-    _instances = weakref.WeakSet()
+    _instances: ClassVar[weakref.WeakSet[Client]] = weakref.WeakSet()
 
     _default_event_handlers = {"print": _handle_print, "warn": _handle_warn}
 
@@ -1377,8 +1382,6 @@ async def _close(self, fast=False):
 
         self.status = "closed"
 
-    _shutdown = _close
-
     def close(self, timeout=no_default):
         """Close this client
 
@@ -2529,12 +2532,13 @@ def _get_computation_code() -> str:
         )
         if not isinstance(ignore_modules, list):
             raise TypeError(
-                f"Ignored modules must be a list. Instead got ({type(ignore_modules)}, {ignore_modules})"
+                "Ignored modules must be a list. Instead got "
+                f"({type(ignore_modules)}, {ignore_modules})"
             )
 
+        pattern: re.Pattern | None
         if ignore_modules:
-            pattern = "|".join([f"(?:{mod})" for mod in ignore_modules])
-            pattern = re.compile(pattern)
+            pattern = re.compile("|".join([f"(?:{mod})" for mod in ignore_modules]))
         else:
             pattern = None
 
diff --git a/distributed/comm/addressing.py b/distributed/comm/addressing.py
index 5e7152413b5..efaa22ce552 100644
--- a/distributed/comm/addressing.py
+++ b/distributed/comm/addressing.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import itertools
 
 import dask
@@ -6,7 +8,7 @@
 from . import registry
 
 
-def parse_address(addr, strict=False):
+def parse_address(addr: str, strict: bool = False) -> tuple[str, str]:
     """
     Split address into its scheme and scheme-dependent location string.
 
@@ -30,7 +32,7 @@ def parse_address(addr, strict=False):
     return scheme, loc
 
 
-def unparse_address(scheme, loc):
+def unparse_address(scheme: str, loc: str) -> str:
     """
     Undo parse_address().
 
@@ -40,7 +42,7 @@ def unparse_address(scheme, loc):
     return f"{scheme}://{loc}"
 
 
-def normalize_address(addr):
+def normalize_address(addr: str) -> str:
     """
     Canonicalize address, adding a default scheme if necessary.
 
@@ -52,7 +54,9 @@ def normalize_address(addr):
     return unparse_address(*parse_address(addr))
 
 
-def parse_host_port(address, default_port=None):
+def parse_host_port(
+    address: str | tuple[str, int], default_port: str | int | None = None
+) -> tuple[str, int]:
     """
     Parse an endpoint address given in the form "host:port".
     """
@@ -95,19 +99,19 @@ def _default():
     return host, int(port)
 
 
-def unparse_host_port(host, port=None):
+def unparse_host_port(host: str, port: int | None = None) -> str:
     """
     Undo parse_host_port().
     """
     if ":" in host and not host.startswith("["):
-        host = "[%s]" % host
+        host = f"[{host}]"
     if port is not None:
         return f"{host}:{port}"
     else:
         return host
 
 
-def get_address_host_port(addr, strict=False):
+def get_address_host_port(addr: str, strict: bool = False) -> tuple[str, int]:
     """
     Get a (host, port) tuple out of the given address.
     For definition of strict check parse_address
@@ -129,7 +133,7 @@ def get_address_host_port(addr, strict=False):
         )
 
 
-def get_address_host(addr):
+def get_address_host(addr: str) -> str:
     """
     Return a hostname / IP address identifying the machine this address
     is located on.
@@ -145,7 +149,7 @@ def get_address_host(addr):
     return backend.get_address_host(loc)
 
 
-def get_local_address_for(addr):
+def get_local_address_for(addr: str) -> str:
     """
     Get a local listening address suitable for reaching *addr*.
 
@@ -162,7 +166,7 @@ def get_local_address_for(addr):
     return unparse_address(scheme, backend.get_local_address_for(loc))
 
 
-def resolve_address(addr):
+def resolve_address(addr: str) -> str:
     """
     Apply scheme-specific address resolution to *addr*, replacing
     all symbolic references with concrete location specifiers.
@@ -177,7 +181,9 @@ def resolve_address(addr):
     return unparse_address(scheme, backend.resolve_address(loc))
 
 
-def uri_from_host_port(host_arg, port_arg, default_port):
+def uri_from_host_port(
+    host_arg: str | None, port_arg: str | None, default_port: int
+) -> str:
     """
     Process the *host* and *port* CLI options.
     Return a URI.
diff --git a/distributed/comm/core.py b/distributed/comm/core.py
index def563dfd2b..face081a7a6 100644
--- a/distributed/comm/core.py
+++ b/distributed/comm/core.py
@@ -1,11 +1,14 @@
+from __future__ import annotations
+
 import asyncio
 import inspect
 import logging
 import random
 import sys
 import weakref
-from abc import ABC, abstractmethod, abstractproperty
+from abc import ABC, abstractmethod
 from contextlib import suppress
+from typing import ClassVar
 
 import dask
 from dask.utils import parse_timedelta
@@ -40,7 +43,7 @@ class Comm(ABC):
     depending on the underlying transport's characteristics.
     """
 
-    _instances = weakref.WeakSet()
+    _instances: ClassVar[weakref.WeakSet[Comm]] = weakref.WeakSet()
 
     def __init__(self):
         self._instances.add(self)
@@ -99,21 +102,17 @@ def abort(self):
 
     @abstractmethod
     def closed(self):
-        """
-        Return whether the stream is closed.
-        """
+        """Return whether the stream is closed."""
 
-    @abstractproperty
-    def local_address(self):
-        """
-        The local address.  For logging and debugging purposes only.
-        """
+    @property
+    @abstractmethod
+    def local_address(self) -> str:
+        """The local address. For logging and debugging purposes only."""
 
-    @abstractproperty
-    def peer_address(self):
-        """
-        The peer's address.  For logging and debugging purposes only.
-        """
+    @property
+    @abstractmethod
+    def peer_address(self) -> str:
+        """The peer's address. For logging and debugging purposes only."""
 
     @property
     def extra_info(self):
@@ -178,13 +177,15 @@ def stop(self):
         communications, but prevents accepting new ones.
         """
 
-    @abstractproperty
+    @property
+    @abstractmethod
     def listen_address(self):
         """
         The listening address as a URI string.
         """
 
-    @abstractproperty
+    @property
+    @abstractmethod
     def contact_address(self):
         """
         An address this listener can be contacted on.  This can be
@@ -227,9 +228,9 @@ async def on_connection(self, comm: Comm, handshake_overrides=None):
             raise CommClosedError(f"Comm {comm!r} closed.") from e
 
         comm.remote_info = handshake
-        comm.remote_info["address"] = comm._peer_addr
+        comm.remote_info["address"] = comm.peer_address
         comm.local_info = local_info
-        comm.local_info["address"] = comm._local_addr
+        comm.local_info["address"] = comm.local_address
 
         comm.handshake_options = comm.handshake_configuration(
             comm.local_info, comm.remote_info
diff --git a/distributed/comm/inproc.py b/distributed/comm/inproc.py
index 804f8ec3916..c67635430d9 100644
--- a/distributed/comm/inproc.py
+++ b/distributed/comm/inproc.py
@@ -153,7 +153,13 @@ class InProc(Comm):
     _initialized = False
 
     def __init__(
-        self, local_addr, peer_addr, read_q, write_q, write_loop, deserialize=True
+        self,
+        local_addr: str,
+        peer_addr: str,
+        read_q,
+        write_q,
+        write_loop,
+        deserialize=True,
     ):
         super().__init__()
         self._local_addr = local_addr
@@ -176,11 +182,11 @@ def finalize(write_q=self._write_q, write_loop=self._write_loop, r=repr(self)):
         return finalize
 
     @property
-    def local_address(self):
+    def local_address(self) -> str:
         return self._local_addr
 
     @property
-    def peer_address(self):
+    def peer_address(self) -> str:
         return self._peer_addr
 
     async def read(self, deserializers="ignored"):
diff --git a/distributed/comm/registry.py b/distributed/comm/registry.py
index 9df481cd466..00b10336a70 100644
--- a/distributed/comm/registry.py
+++ b/distributed/comm/registry.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from abc import ABC, abstractmethod
 
 
@@ -54,7 +56,7 @@ def get_local_address_for(self, loc):
 
 
 # The {scheme: Backend} mapping
-backends = {}
+backends: dict[str, Backend] = {}
 
 
 def get_backend(scheme: str, require: bool = True) -> Backend:
diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py
index f0e3d90a800..cad01427ceb 100644
--- a/distributed/comm/tcp.py
+++ b/distributed/comm/tcp.py
@@ -13,7 +13,7 @@
 try:
     import ssl
 except ImportError:
-    ssl = None
+    ssl = None  # type: ignore
 
 from tlz import sliding_window
 from tornado import netutil
@@ -147,7 +147,13 @@ class TCP(Comm):
 
     max_shard_size = dask.utils.parse_bytes(dask.config.get("distributed.comm.shard"))
 
-    def __init__(self, stream, local_addr, peer_addr, deserialize=True):
+    def __init__(
+        self,
+        stream,
+        local_addr: str,
+        peer_addr: str,
+        deserialize: bool = True,
+    ):
         self._closed = False
         super().__init__()
         self._local_addr = local_addr
@@ -156,7 +162,7 @@ def __init__(self, stream, local_addr, peer_addr, deserialize=True):
         self.deserialize = deserialize
         self._finalizer = weakref.finalize(self, self._get_finalizer())
         self._finalizer.atexit = False
-        self._extra = {}
+        self._extra: dict = {}
 
         ref = weakref.ref(self)
 
@@ -171,7 +177,8 @@ def _read_extra(self):
 
     def _get_finalizer(self):
         def finalize(stream=self.stream, r=repr(self)):
-            # stream is None if a StreamClosedError is raised during interpreter shutdown
+            # stream is None if a StreamClosedError is raised during interpreter
+            # shutdown
             if stream is not None and not stream.closed():
                 logger.warning(f"Closing dangling stream in {r}")
                 stream.close()
@@ -179,11 +186,11 @@ def finalize(stream=self.stream, r=repr(self)):
         return finalize
 
     @property
-    def local_address(self):
+    def local_address(self) -> str:
         return self._local_addr
 
     @property
-    def peer_address(self):
+    def peer_address(self) -> str:
         return self._peer_addr
 
     async def read(self, deserializers=None):
@@ -391,8 +398,8 @@ async def connect(self, address, deserialize=True, **connection_args):
             stream = await self.client.connect(
                 ip, port, max_buffer_size=MAX_BUFFER_SIZE, **kwargs
             )
-            # Under certain circumstances tornado will have a closed connnection with an error and not raise
-            # a StreamClosedError.
+            # Under certain circumstances tornado will have a closed connnection with an
+            # error and not raise a StreamClosedError.
             #
             # This occurs with tornado 5.x and openssl 1.1+
             if stream.closed() and stream.error:
diff --git a/distributed/comm/ucx.py b/distributed/comm/ucx.py
index f8b34c2f43c..ecd6eaef72a 100644
--- a/distributed/comm/ucx.py
+++ b/distributed/comm/ucx.py
@@ -10,6 +10,7 @@
 import struct
 import warnings
 import weakref
+from typing import TYPE_CHECKING
 
 import dask
 from dask.utils import parse_bytes
@@ -23,16 +24,24 @@
 
 logger = logging.getLogger(__name__)
 
-
 # In order to avoid double init when forking/spawning new processes (multiprocess),
 # we make sure only to import and initialize UCX once at first use. This is also
 # required to ensure Dask configuration gets propagated to UCX, which needs
 # variables to be set before being imported.
-ucp = None
+if TYPE_CHECKING:
+    try:
+        import ucp
+        from ucp import create_endpoint as ucx_create_endpoint
+        from ucp import create_listener as ucx_create_listener
+    except ImportError:
+        pass
+else:
+    ucp = None  # type: ignore
+    ucx_create_endpoint = None  # type: ignore
+    ucx_create_listener = None  # type: ignore
+
 host_array = None
 device_array = None
-ucx_create_endpoint = None
-ucx_create_listener = None
 pre_existing_cuda_context = False
 cuda_context_created = False
 
@@ -255,11 +264,11 @@ async def write(
 
                 # Send frames
 
-                # It is necessary to first synchronize the default stream before start sending
-                # We synchronize the default stream because UCX is not stream-ordered and
-                #  syncing the default stream will wait for other non-blocking CUDA streams.
-                # Note this is only sufficient if the memory being sent is not currently in use on
-                # non-blocking CUDA streams.
+                # It is necessary to first synchronize the default stream before start
+                # sending We synchronize the default stream because UCX is not
+                # stream-ordered and syncing the default stream will wait for other
+                # non-blocking CUDA streams. Note this is only sufficient if the memory
+                # being sent is not currently in use on non-blocking CUDA streams.
                 if any(cuda_send_frames):
                     synchronize_stream(0)
 
@@ -381,7 +390,7 @@ async def connect(self, address: str, deserialize=True, **connection_args) -> UC
             raise CommClosedError("Connection closed before handshake completed")
         return self.comm_class(
             ep,
-            local_addr=None,
+            local_addr="",
             peer_addr=self.prefix + address,
             deserialize=deserialize,
         )
diff --git a/distributed/comm/ws.py b/distributed/comm/ws.py
index 1b6c5996833..a733031320d 100644
--- a/distributed/comm/ws.py
+++ b/distributed/comm/ws.py
@@ -1,10 +1,12 @@
+from __future__ import annotations
+
 import asyncio
 import logging
 import struct
 import warnings
 import weakref
+from collections.abc import Callable
 from ssl import SSLError
-from typing import Callable
 
 from tornado import web
 from tornado.httpclient import HTTPClientError, HTTPRequest
@@ -134,19 +136,11 @@ def abort(self):
         self.handler.close()
 
     @property
-    def local_address(self):
+    def local_address(self) -> str:
         return self.handler.request.host
 
     @property
-    def peer_address(self):
-        return self.handler.request.remote_ip + ":0"
-
-    @property
-    def _local_addr(self):
-        return self.handler.request.host
-
-    @property
-    def _peer_addr(self):
+    def peer_address(self) -> str:
         return self.handler.request.remote_ip + ":0"
 
     def closed(self):
@@ -168,8 +162,8 @@ def __init__(self, sock, deserialize=True, allow_offload=True):
         self._closed = False
         super().__init__()
         self.sock = sock
-        self._peer_addr = f"{self.prefix}{self.sock.parsed.netloc}"
         self._local_addr = f"{self.prefix}{self.sock.parsed.netloc}"
+        self._peer_addr = f"{self.prefix}{self.sock.parsed.netloc}"
         self.deserialize = deserialize
         self.allow_offload = allow_offload
         self._finalizer = weakref.finalize(self, self._get_finalizer())
@@ -244,11 +238,11 @@ def closed(self):
         return not self.sock or self.sock.close_code or self._closed
 
     @property
-    def local_address(self):
+    def local_address(self) -> str:
         return f"{self.prefix}{self.sock.parsed.netloc}"
 
     @property
-    def peer_address(self):
+    def peer_address(self) -> str:
         return f"{self.prefix}{self.sock.parsed.netloc}"
 
     def _read_extra(self):
diff --git a/distributed/compatibility.py b/distributed/compatibility.py
index 5687c2b6157..ad8964120a3 100644
--- a/distributed/compatibility.py
+++ b/distributed/compatibility.py
@@ -1,11 +1,14 @@
+from __future__ import annotations
+
 import logging
 import platform
 import sys
 
 import tornado
 
-logging_names = logging._levelToName.copy()
-logging_names.update(logging._nameToLevel)
+logging_names: dict[str | int, int | str] = {}
+logging_names.update(logging._levelToName)  # type: ignore
+logging_names.update(logging._nameToLevel)  # type: ignore
 
 PYPY = platform.python_implementation().lower() == "pypy"
 LINUX = sys.platform == "linux"
diff --git a/distributed/core.py b/distributed/core.py
index c61b40d1051..564e8caf0ad 100644
--- a/distributed/core.py
+++ b/distributed/core.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import asyncio
 import inspect
 import logging
@@ -10,6 +12,7 @@
 from contextlib import suppress
 from enum import Enum
 from functools import partial
+from typing import ClassVar
 
 import tblib
 from tlz import merge
@@ -63,7 +66,7 @@ class Status(Enum):
     dont_reply = "dont_reply"
 
 
-Status.lookup = {s.name: s for s in Status}
+Status.lookup = {s.name: s for s in Status}  # type: ignore
 
 
 class RPCClosed(IOError):
@@ -681,7 +684,7 @@ class rpc:
     >>> remote.close_comms()  # doctest: +SKIP
     """
 
-    active = weakref.WeakSet()
+    active: ClassVar[weakref.WeakSet[rpc]] = weakref.WeakSet()
     comms = ()
     address = None
 
@@ -914,7 +917,7 @@ class ConnectionPool:
         Whether or not to deserialize data by default or pass it through
     """
 
-    _instances = weakref.WeakSet()
+    _instances: ClassVar[weakref.WeakSet[ConnectionPool]] = weakref.WeakSet()
 
     def __init__(
         self,
diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index f6f92a1887e..ba1c6456a10 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -5,6 +5,7 @@
 from collections import defaultdict
 from numbers import Number
 
+import numpy as np
 from bokeh.core.properties import without_property_validation
 from bokeh.io import curdoc
 from bokeh.layouts import column, row
@@ -48,11 +49,6 @@
 from dask import config
 from dask.utils import format_bytes, format_time, key_split, parse_timedelta
 
-try:
-    import numpy as np
-except ImportError:
-    np = False
-
 from distributed.dashboard.components import add_periodic_callback
 from distributed.dashboard.components.shared import (
     DashboardComponent,
@@ -72,7 +68,7 @@
 if dask.config.get("distributed.dashboard.export-tool"):
     from distributed.dashboard.export_tool import ExportTool
 else:
-    ExportTool = None
+    ExportTool = None  # type: ignore
 
 logger = logging.getLogger(__name__)
 
@@ -1243,7 +1239,7 @@ def __init__(self, scheduler, **kwargs):
             self.scheduler = scheduler
 
             if TaskStreamPlugin.name not in self.scheduler.plugins:
-                self.scheduler.add_plugin(plugin=TaskStreamPlugin)
+                self.scheduler.add_plugin(TaskStreamPlugin)
 
             action_data = {
                 "times": [0.2, 0.1],
diff --git a/distributed/dashboard/components/shared.py b/distributed/dashboard/components/shared.py
index 0b585a2e4ee..216a540e63c 100644
--- a/distributed/dashboard/components/shared.py
+++ b/distributed/dashboard/components/shared.py
@@ -30,7 +30,7 @@
 if dask.config.get("distributed.dashboard.export-tool"):
     from distributed.dashboard.export_tool import ExportTool
 else:
-    ExportTool = None
+    ExportTool = None  # type: ignore
 
 
 profile_interval = dask.config.get("distributed.worker.profile.interval")
diff --git a/distributed/dashboard/tests/test_scheduler_bokeh.py b/distributed/dashboard/tests/test_scheduler_bokeh.py
index f3a551d42cd..2907482e078 100644
--- a/distributed/dashboard/tests/test_scheduler_bokeh.py
+++ b/distributed/dashboard/tests/test_scheduler_bokeh.py
@@ -48,7 +48,8 @@
 from distributed.utils import format_dashboard_link
 from distributed.utils_test import dec, div, gen_cluster, get_cert, inc, slowinc
 
-scheduler.PROFILING = False
+# Imported from distributed.dashboard.utils
+scheduler.PROFILING = False  # type: ignore
 
 
 @gen_cluster(client=True, scheduler_kwargs={"dashboard": True})
diff --git a/distributed/dashboard/utils.py b/distributed/dashboard/utils.py
index f8ff45b3ce8..45ebc3adcf9 100644
--- a/distributed/dashboard/utils.py
+++ b/distributed/dashboard/utils.py
@@ -9,7 +9,7 @@
 try:
     import numpy as np
 except ImportError:
-    np = False
+    np = None  # type: ignore
 
 
 BOKEH_VERSION = LooseVersion(bokeh.__version__)
diff --git a/distributed/deploy/adaptive.py b/distributed/deploy/adaptive.py
index 2ce14a1e9f6..8280575be95 100644
--- a/distributed/deploy/adaptive.py
+++ b/distributed/deploy/adaptive.py
@@ -1,6 +1,8 @@
 import logging
 from inspect import isawaitable
 
+from tornado.ioloop import IOLoop
+
 import dask.config
 from dask.utils import parse_timedelta
 
@@ -204,5 +206,9 @@ async def scale_up(self, n):
             await f
 
     @property
-    def loop(self):
-        return self.cluster.loop
+    def loop(self) -> IOLoop:
+        """Override Adaptive.loop"""
+        if self.cluster:
+            return self.cluster.loop
+        else:
+            return IOLoop.current()
diff --git a/distributed/deploy/adaptive_core.py b/distributed/deploy/adaptive_core.py
index 64c9dd03a61..1ee32896a3e 100644
--- a/distributed/deploy/adaptive_core.py
+++ b/distributed/deploy/adaptive_core.py
@@ -1,7 +1,11 @@
-import collections
+from __future__ import annotations
+
 import logging
 import math
-from typing import Iterable
+from collections import defaultdict, deque
+from collections.abc import Iterable
+from datetime import timedelta
+from typing import TYPE_CHECKING, cast
 
 import tlz as toolz
 from tornado.ioloop import IOLoop, PeriodicCallback
@@ -10,6 +14,10 @@
 
 from ..metrics import time
 
+if TYPE_CHECKING:
+    from ..scheduler import WorkerState
+
+
 logger = logging.getLogger(__name__)
 
 
@@ -60,7 +68,7 @@ class AdaptiveCore:
     ----------
     minimum: int
         The minimum number of allowed workers
-    maximum: int
+    maximum: int | inf
         The maximum number of allowed workers
     wait_count: int
         The number of scale-down requests we should receive before actually
@@ -69,17 +77,32 @@ class AdaptiveCore:
         The amount of time, like ``"1s"`` between checks
     """
 
+    minimum: int
+    maximum: int | float
+    wait_count: int
+    interval: int | float
+    periodic_callback: PeriodicCallback | None
+    plan: set[WorkerState]
+    requested: set[WorkerState]
+    observed: set[WorkerState]
+    close_counts: defaultdict[WorkerState, int]
+    _adapting: bool
+    log: deque[tuple[float, dict]]
+
     def __init__(
         self,
         minimum: int = 0,
-        maximum: int = math.inf,
+        maximum: int | float = math.inf,
         wait_count: int = 3,
-        interval: str = "1s",
+        interval: str | int | float | timedelta | None = "1s",
     ):
+        if not isinstance(maximum, int) and not math.isinf(maximum):
+            raise TypeError(f"maximum must be int or inf; got {maximum}")
+
         self.minimum = minimum
         self.maximum = maximum
         self.wait_count = wait_count
-        self.interval = parse_timedelta(interval, "seconds") if interval else interval
+        self.interval = parse_timedelta(interval, "seconds")
         self.periodic_callback = None
 
         def f():
@@ -99,10 +122,7 @@ async def _adapt():
                     await core.adapt()
 
             self.periodic_callback = PeriodicCallback(_adapt, self.interval * 1000)
-            try:
-                self.loop.add_callback(f)
-            except AttributeError:
-                IOLoop.current().add_callback(f)
+            self.loop.add_callback(f)
 
         try:
             self.plan = set()
@@ -112,11 +132,11 @@ async def _adapt():
             pass
 
         # internal state
-        self.close_counts = collections.defaultdict(int)
+        self.close_counts = defaultdict(int)
         self._adapting = False
-        self.log = collections.deque(maxlen=10000)
+        self.log = deque(maxlen=10000)
 
-    def stop(self):
+    def stop(self) -> None:
         logger.info("Adaptive stop")
 
         if self.periodic_callback:
@@ -138,17 +158,17 @@ async def safe_target(self) -> int:
         """Used internally, like target, but respects minimum/maximum"""
         n = await self.target()
         if n > self.maximum:
-            n = self.maximum
+            n = cast(int, self.maximum)
 
         if n < self.minimum:
             n = self.minimum
 
         return n
 
-    async def scale_down(self, n: int):
+    async def scale_down(self, n: int) -> None:
         raise NotImplementedError()
 
-    async def scale_up(self, workers: Iterable):
+    async def scale_up(self, workers: Iterable) -> None:
         raise NotImplementedError()
 
     async def recommendations(self, target: int) -> dict:
@@ -163,34 +183,34 @@ async def recommendations(self, target: int) -> dict:
             self.close_counts.clear()
             return {"status": "same"}
 
-        elif target > len(plan):
+        if target > len(plan):
             self.close_counts.clear()
             return {"status": "up", "n": target}
 
-        elif target < len(plan):
-            not_yet_arrived = requested - observed
-            to_close = set()
-            if not_yet_arrived:
-                to_close.update(toolz.take(len(plan) - target, not_yet_arrived))
-
-            if target < len(plan) - len(to_close):
-                L = await self.workers_to_close(target=target)
-                to_close.update(L)
-
-            firmly_close = set()
-            for w in to_close:
-                self.close_counts[w] += 1
-                if self.close_counts[w] >= self.wait_count:
-                    firmly_close.add(w)
-
-            for k in list(self.close_counts):  # clear out unseen keys
-                if k in firmly_close or k not in to_close:
-                    del self.close_counts[k]
-
-            if firmly_close:
-                return {"status": "down", "workers": list(firmly_close)}
-            else:
-                return {"status": "same"}
+        # target < len(plan)
+        not_yet_arrived = requested - observed
+        to_close = set()
+        if not_yet_arrived:
+            to_close.update(toolz.take(len(plan) - target, not_yet_arrived))
+
+        if target < len(plan) - len(to_close):
+            L = await self.workers_to_close(target=target)
+            to_close.update(L)
+
+        firmly_close = set()
+        for w in to_close:
+            self.close_counts[w] += 1
+            if self.close_counts[w] >= self.wait_count:
+                firmly_close.add(w)
+
+        for k in list(self.close_counts):  # clear out unseen keys
+            if k in firmly_close or k not in to_close:
+                del self.close_counts[k]
+
+        if firmly_close:
+            return {"status": "down", "workers": list(firmly_close)}
+        else:
+            return {"status": "same"}
 
     async def adapt(self) -> None:
         """
@@ -231,3 +251,7 @@ async def adapt(self) -> None:
 
     def __del__(self):
         self.stop()
+
+    @property
+    def loop(self) -> IOLoop:
+        return IOLoop.current()
diff --git a/distributed/deploy/cluster.py b/distributed/deploy/cluster.py
index 471fbc15d47..20a0a990f7f 100644
--- a/distributed/deploy/cluster.py
+++ b/distributed/deploy/cluster.py
@@ -45,7 +45,7 @@ class Cluster:
     """
 
     _supports_scaling = True
-    _cluster_info = {}
+    _cluster_info: dict = {}
 
     def __init__(self, asynchronous, quiet=False, name=None, scheduler_sync_interval=1):
         self.scheduler_info = {"workers": {}}
@@ -64,10 +64,11 @@ def __init__(self, asynchronous, quiet=False, name=None, scheduler_sync_interval
         if name is None:
             name = str(uuid.uuid4())[:8]
 
+        # Mask class attribute with instance attribute
         self._cluster_info = {
             "name": name,
             "type": typename(type(self)),
-            **self._cluster_info,
+            **type(self)._cluster_info,
         }
         self.status = Status.created
 
diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index bf002f504d1..91fbb14cd2c 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -1,8 +1,6 @@
-import atexit
 import logging
 import math
 import warnings
-import weakref
 
 import toolz
 
@@ -258,12 +256,3 @@ def _repr_html_(self, cluster_status=None):
             cluster_status=cluster_status,
         )
         return super()._repr_html_(cluster_status=cluster_status)
-
-
-clusters_to_close = weakref.WeakSet()
-
-
-@atexit.register
-def close_clusters():
-    for cluster in list(clusters_to_close):
-        cluster.close(timeout=10)
diff --git a/distributed/deploy/old_ssh.py b/distributed/deploy/old_ssh.py
index 9d07f1f9bba..a692ba12576 100644
--- a/distributed/deploy/old_ssh.py
+++ b/distributed/deploy/old_ssh.py
@@ -4,12 +4,7 @@
 import sys
 import time
 import traceback
-
-try:
-    from queue import Queue
-except ImportError:  # Python 2.7 fix
-    from Queue import Queue
-
+from queue import Queue
 from threading import Thread
 
 from tlz import merge
diff --git a/distributed/deploy/spec.py b/distributed/deploy/spec.py
index d496f6ae847..9c196941d94 100644
--- a/distributed/deploy/spec.py
+++ b/distributed/deploy/spec.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import asyncio
 import atexit
 import copy
@@ -6,6 +8,7 @@
 import weakref
 from contextlib import suppress
 from inspect import isawaitable
+from typing import ClassVar
 
 from tornado import gen
 
@@ -221,7 +224,7 @@ class does handle all of the logic around asynchronously cleanly setting up
     ["0-0", "0-1", "0-2", "1-0", "1-1"]
     """
 
-    _instances = weakref.WeakSet()
+    _instances: ClassVar[weakref.WeakSet[SpecCluster]] = weakref.WeakSet()
 
     def __init__(
         self,
@@ -450,15 +453,14 @@ def _threads_per_worker(self) -> int:
         for name in ["nthreads", "ncores", "threads", "cores"]:
             with suppress(KeyError):
                 return self.new_spec["options"][name]
-
-        if not self.new_spec:
-            raise ValueError("To scale by cores= you must specify cores per worker")
+        assert False, "unreachable"
 
     def _memory_per_worker(self) -> int:
         """Return the memory limit per worker for new workers"""
         if not self.new_spec:
             raise ValueError(
-                "to scale by memory= your worker definition must include a memory_limit definition"
+                "to scale by memory= your worker definition must include a "
+                "memory_limit definition"
             )
 
         for name in ["memory_limit", "memory"]:
@@ -466,7 +468,8 @@ def _memory_per_worker(self) -> int:
                 return parse_bytes(self.new_spec["options"][name])
 
         raise ValueError(
-            "to use scale(memory=...) your worker definition must include a memory_limit definition"
+            "to use scale(memory=...) your worker definition must include a "
+            "memory_limit definition"
         )
 
     def scale(self, n=0, memory=None, cores=None):
diff --git a/distributed/deploy/ssh.py b/distributed/deploy/ssh.py
index 05d3d935b96..35938c05067 100644
--- a/distributed/deploy/ssh.py
+++ b/distributed/deploy/ssh.py
@@ -1,9 +1,10 @@
+from __future__ import annotations
+
 import logging
 import sys
 import warnings
 import weakref
 from json import dumps
-from typing import List, Union
 
 import dask
 import dask.config
@@ -247,13 +248,13 @@ async def start(self):
 
 
 def SSHCluster(
-    hosts: List[str] = None,
-    connect_options: Union[List[dict], dict] = {},
+    hosts: list[str] | None = None,
+    connect_options: dict | list[dict] = {},
     worker_options: dict = {},
     scheduler_options: dict = {},
     worker_module: str = "deprecated",
     worker_class: str = "distributed.Nanny",
-    remote_python: Union[str, List[str]] = None,
+    remote_python: str | list[str] | None = None,
     **kwargs,
 ):
     """Deploy a Dask cluster using SSH
@@ -277,7 +278,7 @@ def SSHCluster(
 
     Parameters
     ----------
-    hosts : List[str]
+    hosts : list[str]
         List of hostnames or addresses on which to launch our cluster.
         The first will be used for the scheduler and the rest for workers.
     connect_options : dict or list of dict, optional
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index 3f77bd46df9..bc33fd8f328 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -174,7 +174,7 @@ def test_transports_tcp_port():
 
 
 class LocalTest(ClusterTest, unittest.TestCase):
-    Cluster = LocalCluster
+    Cluster = LocalCluster  # type: ignore
     kwargs = {"silence_logs": False, "dashboard_address": ":0", "processes": False}
 
 
diff --git a/distributed/deploy/tests/test_spec_cluster.py b/distributed/deploy/tests/test_spec_cluster.py
index 9c967449f5a..fe566c94d90 100644
--- a/distributed/deploy/tests/test_spec_cluster.py
+++ b/distributed/deploy/tests/test_spec_cluster.py
@@ -396,6 +396,10 @@ def __init__(self, *args, n=1, name=None, nthreads=None, **kwargs):
     def status(self):
         return self.workers[0].status
 
+    @status.setter
+    def status(self, value):
+        raise NotImplementedError()
+
     def __str__(self):
         return "<MultiWorker n=%d>" % len(self.workers)
 
diff --git a/distributed/deploy/utils_test.py b/distributed/deploy/utils_test.py
index 09b2a3d1799..53fba2dd467 100644
--- a/distributed/deploy/utils_test.py
+++ b/distributed/deploy/utils_test.py
@@ -1,3 +1,7 @@
+from __future__ import annotations
+
+from typing import Any
+
 import pytest
 
 from ..client import Client
@@ -5,7 +9,7 @@
 
 class ClusterTest:
     Cluster = None
-    kwargs = {}
+    kwargs: dict[str, Any] = {}
 
     def setUp(self):
         self.cluster = self.Cluster(2, scheduler_port=0, **self.kwargs)
diff --git a/distributed/diskutils.py b/distributed/diskutils.py
index e76633b3586..fc694f5cb7d 100644
--- a/distributed/diskutils.py
+++ b/distributed/diskutils.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import glob
 import logging
 import os
@@ -5,6 +7,7 @@
 import stat
 import tempfile
 import weakref
+from typing import ClassVar
 
 import dask
 
@@ -34,6 +37,10 @@ class WorkDir:
     A temporary work directory inside a WorkSpace.
     """
 
+    dir_path: str
+    _lock_path: str
+    _finalizer: weakref.finalize
+
     def __init__(self, workspace, name=None, prefix=None):
         assert name is None or prefix is None
 
@@ -110,7 +117,7 @@ class WorkSpace:
 
     # Keep track of all locks known to this process, to avoid several
     # WorkSpaces to step on each other's toes
-    _known_locks = set()
+    _known_locks: ClassVar[set[str]] = set()
 
     def __init__(self, base_dir):
         self.base_dir = os.path.abspath(base_dir)
diff --git a/distributed/http/health.py b/distributed/http/health.py
index 0fab7dea7c8..cde3b692b4b 100644
--- a/distributed/http/health.py
+++ b/distributed/http/health.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from tornado import web
 
 
@@ -7,4 +9,4 @@ def get(self):
         self.set_header("Content-Type", "text/plain")
 
 
-routes = [("/health", HealthHandler, {})]
+routes: list[tuple] = [("/health", HealthHandler, {})]
diff --git a/distributed/http/proxy.py b/distributed/http/proxy.py
index 73e1f3d42a8..36c24f94c82 100644
--- a/distributed/http/proxy.py
+++ b/distributed/http/proxy.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import logging
 
 from tornado import web
@@ -73,7 +75,7 @@ def proxy(self, port, proxied_path):
         "python -m pip install jupyter-server-proxy"
     )
 
-    class GlobalProxyHandler(web.RequestHandler):
+    class GlobalProxyHandler(web.RequestHandler):  # type: ignore
         """Minimal Proxy handler when jupyter-server-proxy is not installed"""
 
         def initialize(self, dask_server=None, extra=None):
@@ -129,4 +131,4 @@ def check_worker_dashboard_exits(scheduler, worker):
     return False
 
 
-routes = [(r"proxy/(\d+)/(.*?)/(.*)", GlobalProxyHandler, {})]
+routes: list[tuple] = [(r"proxy/(\d+)/(.*?)/(.*)", GlobalProxyHandler, {})]
diff --git a/distributed/http/scheduler/info.py b/distributed/http/scheduler/info.py
index 096180e1956..6d780c04577 100644
--- a/distributed/http/scheduler/info.py
+++ b/distributed/http/scheduler/info.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import json
 import logging
 import os
@@ -207,7 +209,7 @@ def on_close(self):
         self.server.remove_plugin(name=self.plugin.name)
 
 
-routes = [
+routes: list[tuple] = [
     (r"info", redirect("info/main/workers.html"), {}),
     (r"info/main/workers.html", Workers, {}),
     (r"info/worker/(.*).html", Worker, {}),
diff --git a/distributed/http/scheduler/json.py b/distributed/http/scheduler/json.py
index 61801a00d34..758e2ba3383 100644
--- a/distributed/http/scheduler/json.py
+++ b/distributed/http/scheduler/json.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from ...utils import log_errors
 from ..utils import RequestHandler
 
@@ -65,7 +67,7 @@ def get(self):
             )
 
 
-routes = [
+routes: list[tuple] = [
     (r"json/counts.json", CountsJSON, {}),
     (r"json/identity.json", IdentityJSON, {}),
     (r"json/index.html", IndexJSON, {}),
diff --git a/distributed/http/scheduler/missing_bokeh.py b/distributed/http/scheduler/missing_bokeh.py
index 63fd58a2241..1dfe615c2dd 100644
--- a/distributed/http/scheduler/missing_bokeh.py
+++ b/distributed/http/scheduler/missing_bokeh.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from ...utils import log_errors
 from ..utils import RequestHandler, redirect
 
@@ -12,4 +14,4 @@ def get(self):
             )
 
 
-routes = [(r"/", redirect("status"), {}), (r"status", MissingBokeh, {})]
+routes: list[tuple] = [(r"/", redirect("status"), {}), (r"status", MissingBokeh, {})]
diff --git a/distributed/http/scheduler/prometheus/__init__.py b/distributed/http/scheduler/prometheus/__init__.py
index 63c0310d0aa..9ec024ea560 100644
--- a/distributed/http/scheduler/prometheus/__init__.py
+++ b/distributed/http/scheduler/prometheus/__init__.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from .core import PrometheusHandler
 
-routes = [("/metrics", PrometheusHandler, {})]
+routes: list[tuple] = [("/metrics", PrometheusHandler, {})]
diff --git a/distributed/http/utils.py b/distributed/http/utils.py
index c0de926c99b..f2ba90e9cd7 100644
--- a/distributed/http/utils.py
+++ b/distributed/http/utils.py
@@ -1,6 +1,7 @@
+from __future__ import annotations
+
 import importlib
 import os
-from typing import List
 
 import toolz
 from tornado import web
@@ -27,7 +28,7 @@ def get(self):
     return Redirect
 
 
-def get_handlers(server, modules: List[str], prefix="/"):
+def get_handlers(server, modules: list[str], prefix="/"):
     prefix = prefix or ""
     prefix = "/" + prefix.strip("/")
 
@@ -37,7 +38,7 @@ def get_handlers(server, modules: List[str], prefix="/"):
     _routes = []
     for module_name in modules:
         module = importlib.import_module(module_name)
-        _routes.extend(module.routes)
+        _routes.extend(module.routes)  # type: ignore
 
     routes = []
 
diff --git a/distributed/http/worker/prometheus/__init__.py b/distributed/http/worker/prometheus/__init__.py
index 63c0310d0aa..9ec024ea560 100644
--- a/distributed/http/worker/prometheus/__init__.py
+++ b/distributed/http/worker/prometheus/__init__.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from .core import PrometheusHandler
 
-routes = [("/metrics", PrometheusHandler, {})]
+routes: list[tuple] = [("/metrics", PrometheusHandler, {})]
diff --git a/distributed/locket.py b/distributed/locket.py
index f35f6384ef4..54cda9583da 100644
--- a/distributed/locket.py
+++ b/distributed/locket.py
@@ -1,9 +1,8 @@
 # Vendored up-to-date copy of locket.py
 # Based on https://github.com/mwilliamson/locket.py/pull/8
 
-# flake8: noqa
+from __future__ import annotations
 
-import errno
 import threading
 import time
 import weakref
@@ -23,17 +22,17 @@
             "Platform not supported (failed to import fcntl, ctypes, msvcrt)"
         )
     else:
-        _kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
+        _kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)  # type: ignore
         _WinAPI_LockFile = _kernel32.LockFile
         _WinAPI_LockFile.restype = ctypes.wintypes.BOOL
         _WinAPI_LockFile.argtypes = [ctypes.wintypes.HANDLE] + [
-            ctypes.wintypes.DWORD
+            ctypes.wintypes.DWORD  # type: ignore
         ] * 4
 
         _WinAPI_UnlockFile = _kernel32.UnlockFile
         _WinAPI_UnlockFile.restype = ctypes.wintypes.BOOL
         _WinAPI_UnlockFile.argtypes = [ctypes.wintypes.HANDLE] + [
-            ctypes.wintypes.DWORD
+            ctypes.wintypes.DWORD  # type: ignore
         ] * 4
 
         _lock_file_blocking_available = False
@@ -71,22 +70,20 @@ def _unlock_file(file_):
 
 
 _locks_lock = threading.Lock()
-_locks = weakref.WeakValueDictionary()
+_locks: weakref.WeakValueDictionary[str, _LockSet] = weakref.WeakValueDictionary()
 
 
 def lock_file(path, **kwargs):
-    _locks_lock.acquire()
-    try:
+    with _locks_lock:
         lock = _locks.get(path)
         if lock is None:
             lock = _create_lock_file(path)
             _locks[path] = lock
-    finally:
-        _locks_lock.release()
+
     return _Locker(lock, **kwargs)
 
 
-def _create_lock_file(path):
+def _create_lock_file(path: str):
     thread_lock = _ThreadLock(path)
     file_lock = _LockFile(path)
     return _LockSet([thread_lock, file_lock])
@@ -121,7 +118,7 @@ def acquire(self, timeout, retry_period):
             for lock in self._locks:
                 lock.acquire(timeout, retry_period)
                 acquired_locks.append(lock)
-        except:
+        except Exception:
             for acquired_lock in reversed(acquired_locks):
                 # TODO: handle exceptions
                 acquired_lock.release()
diff --git a/distributed/multi_lock.py b/distributed/multi_lock.py
index 68752296188..5d26653cd03 100644
--- a/distributed/multi_lock.py
+++ b/distributed/multi_lock.py
@@ -1,8 +1,10 @@
+from __future__ import annotations
+
 import asyncio
 import logging
 import uuid
 from collections import defaultdict
-from typing import Hashable, List
+from collections.abc import Hashable
 
 from dask.utils import parse_timedelta
 
@@ -46,7 +48,7 @@ def __init__(self, scheduler):
 
         self.scheduler.extensions["multi_locks"] = self
 
-    def _request_locks(self, locks: List[str], id: Hashable, num_locks: int):
+    def _request_locks(self, locks: list[str], id: Hashable, num_locks: int) -> bool:
         """Request locks
 
         Parameters
diff --git a/distributed/nanny.py b/distributed/nanny.py
index 79c61e88348..6b428b88db0 100644
--- a/distributed/nanny.py
+++ b/distributed/nanny.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import asyncio
 import errno
 import logging
@@ -9,8 +11,9 @@
 import weakref
 from contextlib import suppress
 from inspect import isawaitable
-from multiprocessing.queues import Empty
+from queue import Empty
 from time import sleep as sync_sleep
+from typing import ClassVar
 
 import psutil
 from tornado import gen
@@ -73,7 +76,7 @@ class Nanny(ServerNode):
     Worker
     """
 
-    _instances = weakref.WeakSet()
+    _instances: ClassVar[weakref.WeakSet[Nanny]] = weakref.WeakSet()
     process = None
     status = Status.undefined
 
@@ -618,6 +621,9 @@ def log_event(self, topic, msg):
 
 
 class WorkerProcess:
+    running: asyncio.Event
+    stopped: asyncio.Event
+
     # The interval how often to check the msg queue for init
     _init_msg_interval = 0.05
 
diff --git a/distributed/preloading.py b/distributed/preloading.py
index 1241ad1b630..8bd7ef7e83d 100644
--- a/distributed/preloading.py
+++ b/distributed/preloading.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import filecmp
 import inspect
 import logging
@@ -5,14 +7,16 @@
 import shutil
 import sys
 import urllib.request
+from collections.abc import Iterable
 from importlib import import_module
 from types import ModuleType
-from typing import List
+from typing import cast
 
 import click
 
 from dask.utils import tmpfile
 
+from .core import Server
 from .utils import import_file
 
 logger = logging.getLogger(__name__)
@@ -143,16 +147,24 @@ class Preload:
         The Worker or Scheduler
     name: str
         module name, file name, or web address to load
-    argv: [string]
+    argv: [str]
         List of string arguments passed to click-configurable `dask_setup`.
-    file_dir: string
+    file_dir: str
         Path of a directory where files should be copied
     """
 
-    def __init__(self, dask_server, name: str, argv: List[str], file_dir: str):
+    dask_server: Server
+    name: str
+    argv: list[str]
+    file_dir: str | None
+    module: ModuleType
+
+    def __init__(
+        self, dask_server: Server, name: str, argv: Iterable[str], file_dir: str | None
+    ):
         self.dask_server = dask_server
         self.name = name
-        self.argv = argv
+        self.argv = list(argv)
         self.file_dir = file_dir
 
         if is_webaddress(name):
@@ -167,7 +179,7 @@ async def start(self):
         if dask_setup:
             if isinstance(dask_setup, click.Command):
                 context = dask_setup.make_context(
-                    "dask_setup", list(self.argv), allow_extra_args=False
+                    "dask_setup", self.argv, allow_extra_args=False
                 )
                 result = dask_setup.callback(
                     self.dask_server, *context.args, **context.params
@@ -191,9 +203,25 @@ async def teardown(self):
 
 
 def process_preloads(
-    dask_server, preload: List[str], preload_argv: List[List], file_dir: str = None
-) -> List[Preload]:
+    dask_server,
+    preload: str | list[str],
+    preload_argv: list[str] | list[list[str]],
+    *,
+    file_dir: str | None = None,
+) -> list[Preload]:
     if isinstance(preload, str):
         preload = [preload]
+    if preload_argv and isinstance(preload_argv[0], str):
+        preload_argv = [cast("list[str]", preload_argv)] * len(preload)
+    elif not preload_argv:
+        preload_argv = [cast("list[str]", [])] * len(preload)
+    if len(preload) != len(preload_argv):
+        raise ValueError(
+            "preload and preload_argv have mismatched lengths "
+            f"{len(preload)} != {len(preload_argv)}"
+        )
 
-    return [Preload(dask_server, p, preload_argv, file_dir) for p in preload]
+    return [
+        Preload(dask_server, p, argv, file_dir)
+        for p, argv in zip(preload, preload_argv)
+    ]
diff --git a/distributed/process.py b/distributed/process.py
index 4794b3aebab..6a11832479d 100644
--- a/distributed/process.py
+++ b/distributed/process.py
@@ -160,26 +160,10 @@ def monitor_parent():
         t.daemon = True
         t.start()
 
-    @staticmethod
-    def reset_logger_locks():
-        """Python 2's logger's locks don't survive a fork event
-
-        https://github.com/dask/distributed/issues/1491
-        """
-        for name in logging.Logger.manager.loggerDict.keys():
-            for handler in logging.getLogger(name).handlers:
-                handler.createLock()
-
     @classmethod
     def _run(
         cls, target, args, kwargs, parent_alive_pipe, _keep_child_alive, inherit_config
     ):
-        # On Python 2 with the fork method, we inherit the _keep_child_alive fd,
-        # whether it is passed or not. Therefore, pass it unconditionally and
-        # close it here, so that there are no other references to the pipe lying
-        # around.
-        cls.reset_logger_locks()
-
         _keep_child_alive.close()
 
         # Child process entry point
diff --git a/distributed/protocol/compression.py b/distributed/protocol/compression.py
index 26bee14174c..206dbb0c903 100644
--- a/distributed/protocol/compression.py
+++ b/distributed/protocol/compression.py
@@ -3,10 +3,14 @@
 
 Includes utilities for determining whether or not to compress
 """
+from __future__ import annotations
+
 import logging
 import random
+from collections.abc import Callable
 from contextlib import suppress
 from functools import partial
+from typing import TYPE_CHECKING
 
 from tlz import identity
 
@@ -23,7 +27,13 @@
 
 from ..utils import ensure_bytes
 
-compressions = {None: {"compress": identity, "decompress": identity}}
+if TYPE_CHECKING:
+    from typing_extensions import Literal
+
+compressions: dict[
+    str | None | Literal[False],
+    dict[Literal["compress", "decompress"], Callable[[bytes], bytes]],
+] = {None: {"compress": identity, "decompress": identity}}
 
 compressions[False] = compressions[None]  # alias
 
diff --git a/distributed/protocol/serialize.py b/distributed/protocol/serialize.py
index af2d86b8edd..2dbac0a46f9 100644
--- a/distributed/protocol/serialize.py
+++ b/distributed/protocol/serialize.py
@@ -1,8 +1,11 @@
+from __future__ import annotations
+
 import importlib
 import traceback
 from array import array
 from enum import Enum
 from functools import partial
+from types import ModuleType
 
 import msgpack
 
@@ -15,12 +18,10 @@
 from .compression import decompress, maybe_compress
 from .utils import frame_split_size, msgpack_opts, pack_frames_prelude, unpack_frames
 
-lazy_registrations = {}
-
 dask_serialize = dask.utils.Dispatch("dask_serialize")
 dask_deserialize = dask.utils.Dispatch("dask_deserialize")
 
-_cached_allowed_modules = {}
+_cached_allowed_modules: dict[str, ModuleType] = {}
 
 
 def dask_dumps(x, context=None):
diff --git a/distributed/protocol/tests/test_serialize.py b/distributed/protocol/tests/test_serialize.py
index c552742930e..74b5d2d209c 100644
--- a/distributed/protocol/tests/test_serialize.py
+++ b/distributed/protocol/tests/test_serialize.py
@@ -9,7 +9,7 @@
 try:
     import numpy as np
 except ImportError:
-    np = None
+    np = None  # type: ignore
 
 import dask
 
@@ -480,7 +480,9 @@ async def test_frame_split():
         (memoryview(b"hello"), True),
         pytest.param(
             memoryview(
-                np.random.random((3, 4)) if np is not None else b"skip np.random"
+                np.random.random((3, 4))  # type: ignore
+                if np is not None
+                else b"skip np.random"
             ),
             True,
             marks=pytest.mark.skipif(np is None, reason="Test needs numpy"),
@@ -512,7 +514,9 @@ def test_serialize_lists(serializers):
         memoryview(b"hello"),
         pytest.param(
             memoryview(
-                np.random.random((3, 4)) if np is not None else b"skip np.random"
+                np.random.random((3, 4))  # type: ignore
+                if np is not None
+                else b"skip np.random"
             ),
             marks=pytest.mark.skipif(np is None, reason="Test needs numpy"),
         ),
diff --git a/distributed/py.typed b/distributed/py.typed
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/distributed/queues.py b/distributed/queues.py
index 5c81d25b848..620109c2408 100644
--- a/distributed/queues.py
+++ b/distributed/queues.py
@@ -141,7 +141,7 @@ class Queue:
 
     .. warning::
 
-       This object is experimental and has known issues in Python 2
+       This object is experimental
 
     Parameters
     ----------
diff --git a/distributed/security.py b/distributed/security.py
index 6e6edd83c65..4bacda65b55 100644
--- a/distributed/security.py
+++ b/distributed/security.py
@@ -5,7 +5,7 @@
 try:
     import ssl
 except ImportError:
-    ssl = None
+    ssl = None  # type: ignore
 
 import dask
 from dask.widgets import get_template
diff --git a/distributed/system_monitor.py b/distributed/system_monitor.py
index 43cd288780e..0694facd16a 100644
--- a/distributed/system_monitor.py
+++ b/distributed/system_monitor.py
@@ -8,7 +8,7 @@
 try:
     from .diagnostics import nvml
 except Exception:
-    nvml = None
+    nvml = None  # type: ignore
 
 
 class SystemMonitor:
diff --git a/distributed/tests/test_actor.py b/distributed/tests/test_actor.py
index b89f4aa3013..cf026fcf233 100644
--- a/distributed/tests/test_actor.py
+++ b/distributed/tests/test_actor.py
@@ -50,7 +50,7 @@ async def ado_inc(self, ac):
 
 
 class List:
-    L = []
+    L: list = []
 
     def __init__(self, dummy=None):
         self.L = []
diff --git a/distributed/tests/test_counter.py b/distributed/tests/test_counter.py
index 0a293633dd2..6c57b9bc4e0 100644
--- a/distributed/tests/test_counter.py
+++ b/distributed/tests/test_counter.py
@@ -5,7 +5,7 @@
 try:
     from distributed.counter import Digest
 except ImportError:
-    Digest = None
+    Digest = None  # type: ignore
 
 
 @pytest.mark.parametrize(
diff --git a/distributed/tests/test_preload.py b/distributed/tests/test_preload.py
index d893a1251f4..ab90b20ffea 100644
--- a/distributed/tests/test_preload.py
+++ b/distributed/tests/test_preload.py
@@ -75,7 +75,7 @@ def dask_teardown(worker):
     worker.foo = 'teardown'
 """
     with dask.config.set(
-        {"distributed.worker.preload": text, "distributed.nanny.preload": text}
+        {"distributed.worker.preload": [text], "distributed.nanny.preload": [text]}
     ):
         async with Nanny(s.address) as w:
             assert w.foo == "setup"
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 71ede927082..d5cde1b5491 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -5,7 +5,6 @@
 import operator
 import re
 import sys
-from collections import defaultdict
 from itertools import product
 from textwrap import dedent
 from time import sleep
@@ -59,8 +58,6 @@
 alice = "alice:1234"
 bob = "bob:1234"
 
-occupancy = defaultdict(lambda: 0)
-
 
 @gen_cluster()
 async def test_administration(s, a, b):
@@ -2090,8 +2087,8 @@ async def test_task_group_on_fire_and_forget(c, s, a, b):
 
 
 class BrokenComm(Comm):
-    peer_address = None
-    local_address = None
+    peer_address = ""
+    local_address = ""
 
     def close(self):
         pass
diff --git a/distributed/tests/test_security.py b/distributed/tests/test_security.py
index 9bb128ebd04..c9f3c722573 100644
--- a/distributed/tests/test_security.py
+++ b/distributed/tests/test_security.py
@@ -3,7 +3,7 @@
 try:
     import ssl
 except ImportError:
-    ssl = None
+    ssl = None  # type: ignore
 
 import pytest
 
diff --git a/distributed/tests/test_semaphore.py b/distributed/tests/test_semaphore.py
index bf0f8181e17..66a0864c8dd 100644
--- a/distributed/tests/test_semaphore.py
+++ b/distributed/tests/test_semaphore.py
@@ -264,8 +264,8 @@ def f(x, sem):
 
 
 class BrokenComm(Comm):
-    peer_address = None
-    local_address = None
+    peer_address = ""
+    local_address = ""
 
     def close(self):
         pass
diff --git a/distributed/tests/test_utils_comm.py b/distributed/tests/test_utils_comm.py
index ca058ab06ec..f30b9b98cc8 100644
--- a/distributed/tests/test_utils_comm.py
+++ b/distributed/tests/test_utils_comm.py
@@ -43,8 +43,8 @@ async def test_gather_from_workers_permissive(c, s, a, b):
 
 
 class BrokenComm(Comm):
-    peer_address = None
-    local_address = None
+    peer_address = ""
+    local_address = ""
 
     def close(self):
         pass
diff --git a/distributed/tests/test_versions.py b/distributed/tests/test_versions.py
index 5504daea31c..d78c23cfaf2 100644
--- a/distributed/tests/test_versions.py
+++ b/distributed/tests/test_versions.py
@@ -11,9 +11,6 @@
 mismatched_version = get_versions()
 mismatched_version["packages"]["distributed"] = "0.0.0.dev0"
 
-# for really old versions, the `package` key is missing - version is UNKNOWN
-key_err_version = {}
-
 # if no key is available for one package, we assume it's MISSING
 missing_version = get_versions()
 del missing_version["packages"]["distributed"]
@@ -53,7 +50,7 @@ def kwargs_not_matching(kwargs_matching, node, effect):
     affected_version = {
         "MISMATCHED": mismatched_version,
         "MISSING": missing_version,
-        "KEY_ERROR": key_err_version,
+        "KEY_ERROR": {},
         "NONE": unknown_version,
     }[effect]
     kwargs = kwargs_matching
diff --git a/distributed/utils.py b/distributed/utils.py
index c306e710e86..8ae80f8a510 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import asyncio
 import functools
 import importlib
@@ -23,7 +25,7 @@
 from importlib.util import cache_from_source
 from time import sleep
 from typing import Any as AnyType
-from typing import Dict, List
+from typing import ClassVar
 
 import click
 import tblib.pickling_support
@@ -31,7 +33,7 @@
 try:
     import resource
 except ImportError:
-    resource = None
+    resource = None  # type: ignore
 
 import tlz as toolz
 from tornado import gen
@@ -43,7 +45,7 @@
 from dask.widgets import get_template
 
 try:
-    from tornado.ioloop import PollIOLoop
+    from tornado.ioloop import PollIOLoop  # type: ignore
 except ImportError:
     PollIOLoop = None  # dropped in tornado 6.0
 
@@ -345,7 +347,9 @@ class LoopRunner:
     """
 
     # All loops currently associated to loop runners
-    _all_loops = weakref.WeakKeyDictionary()
+    _all_loops: ClassVar[
+        weakref.WeakKeyDictionary[IOLoop, tuple[int, LoopRunner | None]]
+    ] = weakref.WeakKeyDictionary()
     _lock = threading.Lock()
 
     def __init__(self, loop=None, asynchronous=False):
@@ -602,7 +606,7 @@ def key_split(s):
         return "Other"
 
 
-def key_split_group(x):
+def key_split_group(x) -> str:
     """A more fine-grained version of key_split
 
     >>> key_split_group(('x-2', 1))
@@ -633,7 +637,7 @@ def key_split_group(x):
     elif typ is bytes:
         return key_split_group(x.decode())
     else:
-        return key_split(x)
+        return "Other"
 
 
 @contextmanager
@@ -980,7 +984,7 @@ def json_load_robust(fn, load=json.load):
 class DequeHandler(logging.Handler):
     """A logging.Handler that records records into a deque"""
 
-    _instances = weakref.WeakSet()
+    _instances: ClassVar[weakref.WeakSet[DequeHandler]] = weakref.WeakSet()
 
     def __init__(self, *args, n=10000, **kwargs):
         self.deque = deque(maxlen=n)
@@ -1038,15 +1042,15 @@ def reset_logger_locks():
 
         # TODO: Use tornado's AnyThreadEventLoopPolicy, instead of class below,
         # once tornado > 6.0.3 is available.
-        if WINDOWS and hasattr(asyncio, "WindowsSelectorEventLoopPolicy"):
+        if WINDOWS:
             # WindowsProactorEventLoopPolicy is not compatible with tornado 6
             # fallback to the pre-3.8 default of Selector
             # https://github.com/tornadoweb/tornado/issues/2608
-            BaseEventLoopPolicy = asyncio.WindowsSelectorEventLoopPolicy
+            BaseEventLoopPolicy = asyncio.WindowsSelectorEventLoopPolicy  # type: ignore
         else:
             BaseEventLoopPolicy = asyncio.DefaultEventLoopPolicy
 
-        class AnyThreadEventLoopPolicy(BaseEventLoopPolicy):
+        class AnyThreadEventLoopPolicy(BaseEventLoopPolicy):  # type: ignore
             def get_event_loop(self):
                 try:
                     return super().get_event_loop()
@@ -1353,7 +1357,7 @@ def __setitem__(self, key, value):
         super().__setitem__(key, value)
 
 
-def clean_dashboard_address(addrs: AnyType, default_listen_ip: str = "") -> List[Dict]:
+def clean_dashboard_address(addrs: AnyType, default_listen_ip: str = "") -> list[dict]:
     """
     Examples
     --------
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index 90f40d7f75c..d6d3821a018 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -1,11 +1,11 @@
+from __future__ import annotations
+
 import asyncio
-import collections
 import copy
 import functools
 import gc
 import inspect
 import io
-import itertools
 import logging
 import logging.config
 import os
@@ -21,16 +21,19 @@
 import uuid
 import warnings
 import weakref
+from collections import defaultdict
 from contextlib import contextmanager, nullcontext, suppress
 from glob import glob
+from itertools import count
 from time import sleep
+from typing import Any
 
 from distributed.scheduler import Scheduler
 
 try:
     import ssl
 except ImportError:
-    ssl = None
+    ssl = None  # type: ignore
 
 import pytest
 from tlz import assoc, memoize, merge
@@ -335,7 +338,8 @@ def slowidentity(*args, **kwargs):
 
 
 class _UnhashableCallable:
-    __hash__ = None
+    # FIXME https://github.com/python/mypy/issues/4266
+    __hash__ = None  # type: ignore
 
     def __call__(self, x):
         return x + 1
@@ -351,8 +355,8 @@ def run_for(duration, timer=time):
 
 
 # This dict grows at every varying() invocation
-_varying_dict = collections.defaultdict(int)
-_varying_key_gen = itertools.count()
+_varying_dict: defaultdict[str, int] = defaultdict(int)
+_varying_key_gen = count()
 
 
 class _ModuleSlot:
@@ -414,7 +418,7 @@ async def asyncinc(x, delay=0.02):
     return x + 1
 
 
-_readone_queues = {}
+_readone_queues: dict[Any, asyncio.Queue] = {}
 
 
 async def readone(comm):
diff --git a/distributed/worker.py b/distributed/worker.py
index 1091c7f1e65..59e2a1078aa 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -14,12 +14,12 @@
 import warnings
 import weakref
 from collections import defaultdict, deque, namedtuple
-from collections.abc import Hashable, Iterable, MutableMapping
+from collections.abc import Callable, Hashable, Iterable, MutableMapping
 from contextlib import suppress
 from datetime import timedelta
 from inspect import isawaitable
 from pickle import PicklingError
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any, ClassVar
 
 if TYPE_CHECKING:
     from .client import Client
@@ -105,11 +105,11 @@
 READY = {"ready", "constrained"}
 
 
-DEFAULT_EXTENSIONS = [PubSubWorkerExtension]
+DEFAULT_EXTENSIONS: list[type] = [PubSubWorkerExtension]
 
-DEFAULT_METRICS = {}
+DEFAULT_METRICS: dict[str, Callable[[Worker], Any]] = {}
 
-DEFAULT_STARTUP_INFORMATION = {}
+DEFAULT_STARTUP_INFORMATION: dict[str, Callable[[Worker], Any]] = {}
 
 DEFAULT_DATA_SIZE = parse_bytes(
     dask.config.get("distributed.scheduler.default-data-size")
@@ -394,8 +394,8 @@ class Worker(ServerNode):
     distributed.nanny.Nanny
     """
 
-    _instances = weakref.WeakSet()
-    _initialized_clients = weakref.WeakSet()
+    _instances: ClassVar[weakref.WeakSet[Worker]] = weakref.WeakSet()
+    _initialized_clients: ClassVar[weakref.WeakSet[Client]] = weakref.WeakSet()
 
     def __init__(
         self,
@@ -902,7 +902,7 @@ def local_dir(self):
     def executor(self):
         return self.executors["default"]
 
-    @ServerNode.status.setter
+    @ServerNode.status.setter  # type: ignore
     def status(self, value):
         """Override Server.status to notify the Scheduler of status changes"""
         ServerNode.status.__set__(self, value)
@@ -2615,7 +2615,7 @@ async def gather_dep(
                         ("busy-gather", worker, to_gather_keys, stimulus_id, time())
                     )
 
-                recommendations = {}
+                recommendations: dict[TaskState, str | tuple] = {}
                 deps_to_iter = self.in_flight_workers.pop(worker)
 
                 for d in deps_to_iter:
diff --git a/docs/source/conf.py b/docs/source/conf.py
index 12bcfa7cffc..44cd4d73253 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 #
 # Dask.distributed documentation build configuration file, created by
 # sphinx-quickstart on Tue Oct  6 14:42:44 2015.
@@ -84,7 +86,7 @@
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
-exclude_patterns = []
+exclude_patterns: list[str] = []
 
 # The reST default role (used for this markup: `text`) to use for all
 # documents.
@@ -215,7 +217,7 @@
 
 # -- Options for LaTeX output ---------------------------------------------
 
-latex_elements = {
+latex_elements: dict[str, str] = {
     # The paper size ('letterpaper' or 'a4paper').
     # 'papersize': 'letterpaper',
     # The font size ('10pt', '11pt' or '12pt').
@@ -426,7 +428,7 @@ def copy_legacy_redirects(app, docname):
                 f.write(page)
 
 
-from docutils.parsers.rst import directives
+from docutils.parsers.rst import directives  # type: ignore
 
 # -- Configuration to keep autosummary in sync with autoclass::members ----------------------------------------------
 # Fixes issues/3693
diff --git a/docs/source/develop.rst b/docs/source/develop.rst
index f0f534386bb..6ea8c750329 100644
--- a/docs/source/develop.rst
+++ b/docs/source/develop.rst
@@ -171,7 +171,7 @@ you need to test the command line interface.
 
 Linting
 -------
-distributed uses several code linters (flake8, black, isort, pyupgrade), which are
+distributed uses several code linters (flake8, black, isort, pyupgrade, mypy), which are
 enforced by CI. Developers should run them locally before they submit a PR, through the
 single command ``pre-commit run --all-files``. This makes sure that linter versions and
 options are aligned for all developers.
diff --git a/setup.py b/setup.py
index 23bc2c9be0a..66835d73267 100755
--- a/setup.py
+++ b/setup.py
@@ -11,7 +11,7 @@
 requires = open("requirements.txt").read().strip().split("\n")
 setup_requires = []
 install_requires = []
-extras_require = {}
+extras_require: dict = {}
 for r in requires:
     if ";" in r:
         # requirements.txt conditional dependencies need to be reformatted for wheels
@@ -43,8 +43,8 @@
     except ImportError:
         setup_requires.append("cython")
 
-    _, _, params = cython_arg.partition("=")
-    params = params.split(",")
+    _, _, params_str = cython_arg.partition("=")
+    params = params_str.split(",")
     profile = "profile" in params
     if "annotate" in params:
         import Cython.Compiler.Options
@@ -55,7 +55,7 @@
         Extension("distributed.scheduler", sources=["distributed/scheduler.py"]),
     ]
     for e in cyext_modules:
-        e.cython_directives = {
+        e.cython_directives = {  # type: ignore
             "annotation_typing": True,
             "binding": False,
             "embedsignature": True,
@@ -107,5 +107,6 @@
         dask-scheduler=distributed.cli.dask_scheduler:go
         dask-worker=distributed.cli.dask_worker:go
       """,
+    # https://mypy.readthedocs.io/en/latest/installed_packages.html
     zip_safe=False,
 )
diff --git a/versioneer.py b/versioneer.py
index 92e184c31e5..e1a8d3e17cd 100644
--- a/versioneer.py
+++ b/versioneer.py
@@ -274,13 +274,7 @@
 https://creativecommons.org/publicdomain/zero/1.0/ .
 
 """
-
-
-try:
-    import configparser
-except ImportError:
-    import ConfigParser as configparser
-
+import configparser
 import errno
 import json
 import os

From 88dbce59728b8124c60b129c8836acd619ab9350 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Thu, 30 Sep 2021 19:11:50 -0500
Subject: [PATCH 1517/1550] Mark
 distributed/tests/test_client.py::test_profile_server as flaky (#5375)

---
 distributed/tests/test_client.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 3ca714db282..4d521d5d690 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -6050,6 +6050,7 @@ async def test_futures_of_sorted(c, s, a, b):
         assert str(k) in str(f)
 
 
+@pytest.mark.flaky(reruns=10, reruns_delay=5)
 @gen_cluster(client=True, worker_kwargs={"profile_cycle_interval": "10ms"})
 async def test_profile_server(c, s, a, b):
     for i in range(5):

From 7d58db2874360df513c1daa464847dc387564231 Mon Sep 17 00:00:00 2001
From: Michael Adkins <madkinszane@gmail.com>
Date: Fri, 1 Oct 2021 01:03:56 +0000
Subject: [PATCH 1518/1550] Add support for partial functions to
 `iscoroutinefunction` util (#5344)

This allows users to submit partially evaluated async functions; otheriwse, the cororoutine will not be awaited.
---
 distributed/tests/test_client.py | 11 +++++++++++
 distributed/tests/test_utils.py  | 10 ++++++++++
 distributed/utils.py             |  4 ++++
 3 files changed, 25 insertions(+)

diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 4d521d5d690..a7a63c48261 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -1,4 +1,5 @@
 import asyncio
+import functools
 import gc
 import inspect
 import logging
@@ -6915,6 +6916,16 @@ async def f(x):
     assert result == 11
 
 
+@gen_cluster(client=True)
+async def test_async_task_with_partial(c, s, a, b):
+    async def f(x, y):
+        return x + y + 1
+
+    future = c.submit(functools.partial(f, 1), 10)
+    result = await future
+    assert result == 12
+
+
 @gen_cluster(client=True, nthreads=[("", 1)])
 async def test_events_subscribe_topic(c, s, a):
 
diff --git a/distributed/tests/test_utils.py b/distributed/tests/test_utils.py
index 929f8c1f5b7..0a698a3b3a4 100644
--- a/distributed/tests/test_utils.py
+++ b/distributed/tests/test_utils.py
@@ -1,5 +1,6 @@
 import array
 import asyncio
+import functools
 import io
 import os
 import queue
@@ -610,3 +611,12 @@ def test_tmpfile_deprecated():
 def test_iscoroutinefunction_unhashable_input():
     # Ensure iscoroutinefunction can handle unhashable callables
     assert not iscoroutinefunction(_UnhashableCallable())
+
+
+def test_iscoroutinefunction_nested_partial():
+    async def my_async_callable(x, y, z):
+        pass
+
+    assert iscoroutinefunction(
+        functools.partial(functools.partial(my_async_callable, 1), 2)
+    )
diff --git a/distributed/utils.py b/distributed/utils.py
index 8ae80f8a510..4e79e3b36db 100644
--- a/distributed/utils.py
+++ b/distributed/utils.py
@@ -1123,6 +1123,10 @@ def color_of(x, palette=palette):
 
 
 def _iscoroutinefunction(f):
+    # Python < 3.8 does not support determining if `partial` objects wrap async funcs
+    if sys.version_info < (3, 8):
+        while isinstance(f, functools.partial):
+            f = f.func
     return inspect.iscoroutinefunction(f) or gen.is_coroutine_function(f)
 
 

From d8c7c6b10f683c1aa2d566e1db1643aa38e601fd Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Fri, 1 Oct 2021 09:43:40 +0200
Subject: [PATCH 1519/1550] Fix zombie worker tasks after missing transition
 (#5316)

---
 distributed/tests/test_worker.py | 51 ++++++++++++++++++++++++++++++++
 distributed/worker.py            | 23 +++++---------
 2 files changed, 58 insertions(+), 16 deletions(-)

diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 8d9ebc2fac9..34c7c66cd1f 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -2984,6 +2984,57 @@ async def test_who_has_consistent_remove_replica(c, s, *workers):
     assert s.tasks[f1.key].suspicious == 0
 
 
+@gen_cluster(client=True)
+async def test_missing_released_zombie_tasks(c, s, a, b):
+    """
+    Ensure that no fetch/flight tasks are left in the task dict of a
+    worker after everything was released
+    """
+    a.total_in_connections = 0
+    f1 = c.submit(inc, 1, key="f1", workers=[a.address])
+    f2 = c.submit(inc, f1, key="f2", workers=[b.address])
+    key = f1.key
+
+    while key not in b.tasks or b.tasks[key].state != "fetch":
+        await asyncio.sleep(0.01)
+
+    await a.close(report=False)
+
+    del f1, f2
+
+    while b.tasks:
+        await asyncio.sleep(0.01)
+
+
+@gen_cluster(client=True)
+async def test_missing_released_zombie_tasks_2(c, s, a, b):
+    a.total_in_connections = 0
+    f1 = c.submit(inc, 1, key="f1", workers=[a.address])
+    f2 = c.submit(inc, f1, key="f2", workers=[b.address])
+
+    while f1.key not in b.tasks:
+        await asyncio.sleep(0)
+
+    ts = b.tasks[f1.key]
+    assert ts.state == "fetch"
+
+    # A few things can happen to clear who_has. The dominant process is upon
+    # connection failure to a worker. Regardless of how the set was cleared, the
+    # task will be transitioned to missing where the worker is trying to
+    # reaquire this information from the scheduler. While this is happening on
+    # worker side, the tasks are released and we want to ensure that no dangling
+    # zombie tasks are left on the worker
+    ts.who_has.clear()
+
+    del f1, f2
+
+    while b.tasks:
+        await asyncio.sleep(0.01)
+
+    story = b.story(ts)
+    assert any("missing" in msg for msg in story)
+
+
 @pytest.mark.slow
 @gen_cluster(
     client=True,
diff --git a/distributed/worker.py b/distributed/worker.py
index 59e2a1078aa..2acba897d2f 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -191,6 +191,7 @@ def __init__(self, key, runspec=None):
         self.who_has = set()
         self.coming_from = None
         self.waiting_for_data = set()
+        self.waiters = set()
         self.resource_restrictions = {}
         self.exception = None
         self.exception_text = ""
@@ -1854,6 +1855,7 @@ def transition_released_waiting(self, ts, *, stimulus_id):
         for dep_ts in ts.dependencies:
             if not dep_ts.state == "memory":
                 ts.waiting_for_data.add(dep_ts)
+                dep_ts.waiters.add(ts)
 
         if ts.waiting_for_data:
             self.waiting_for_data_count += 1
@@ -2669,19 +2671,6 @@ async def find_missing(self):
                 who_has = {k: v for k, v in who_has.items() if v}
                 self.update_who_has(who_has, stimulus_id=stimulus_id)
 
-                if self._missing_dep_flight:
-                    logger.debug(
-                        "No new workers found for %s", self._missing_dep_flight
-                    )
-                    recommendations = {
-                        dep: "released"
-                        for dep in self._missing_dep_flight
-                        if dep.state == "missing"
-                    }
-                    self.transitions(
-                        recommendations=recommendations, stimulus_id=stimulus_id
-                    )
-
             finally:
                 # This is quite arbitrary but the heartbeat has scaling implemented
                 self.periodic_callbacks[
@@ -2792,9 +2781,11 @@ def release_key(
                         self.available_resources[resource] += quantity
 
             for d in ts.dependencies:
-                ts.waiting_for_data.discard(ts)
-                if not d.dependents and d.state in {"flight", "fetch", "missing"}:
-                    recommendations[d] = "released"
+                ts.waiting_for_data.discard(d)
+                d.waiters.discard(ts)
+
+                if not d.waiters and d.state in {"flight", "fetch", "missing"}:
+                    recommendations[d] = "forgotten"
 
             ts.waiting_for_data.clear()
             ts.nbytes = None

From cf018a1944e7b54f5b57ed176ff20a8d6b357ed5 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Fri, 1 Oct 2021 10:41:13 -0500
Subject: [PATCH 1520/1550] Add code coverage (#4670)

* Add automated code coverage

* Add pytest-cov

* Temporarily reduce number of tests

* Run all tests again

* Update .gitignore

* Only run one test

* Specify source instead of input

* Run all tests again

* Trigger CI

* Exclude code coverage reports in test_upload_directory

* Decrease coverage target

* Update .gitignore

* Lint
---
 .coveragerc                                 |  4 ++--
 .github/workflows/tests.yaml                |  5 ++++-
 .gitignore                                  |  2 ++
 codecov.yml                                 | 17 +++++++++++++++++
 continuous_integration/environment-3.7.yaml |  2 ++
 continuous_integration/environment-3.8.yaml |  2 ++
 continuous_integration/environment-3.9.yaml |  2 ++
 distributed/tests/test_client.py            |  6 ++++--
 8 files changed, 35 insertions(+), 5 deletions(-)
 create mode 100644 codecov.yml

diff --git a/.coveragerc b/.coveragerc
index 8038e8d8dab..698a70a5a4a 100644
--- a/.coveragerc
+++ b/.coveragerc
@@ -1,6 +1,6 @@
 [run]
-include = 
-    distributed/*
+source =
+    distributed
 omit =
     distributed/tests/test*
     distributed/hdfs.py
diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
index 703e5ba947a..66c61a903d0 100644
--- a/.github/workflows/tests.yaml
+++ b/.github/workflows/tests.yaml
@@ -116,12 +116,15 @@ jobs:
           source continuous_integration/scripts/set_ulimit.sh
 
           pytest distributed -m "not avoid_ci and ${{ matrix.partition }}" --runslow \
-            --junitxml reports/pytest.xml -o junit_suite_name=$TEST_ID
+            --junitxml reports/pytest.xml -o junit_suite_name=$TEST_ID --cov=distributed --cov-report=xml
 
       # - name: Debug with tmate on failure
       #   if: ${{ failure() }}
       #   uses: mxschmitt/action-tmate@v3
 
+      - name: Coverage
+        uses: codecov/codecov-action@v1
+
       - name: Upload test artifacts
         # ensure this runs even if pytest fails
         if: >
diff --git a/.gitignore b/.gitignore
index 2788e494a62..d967eecd215 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,6 +15,8 @@ continuous_integration/hdfs-initialized
 .DS_Store
 .cache
 .coverage
+.coverage.*
+coverage.xml
 *.lock
 .#*
 .eggs/
diff --git a/codecov.yml b/codecov.yml
new file mode 100644
index 00000000000..b5f5dad7efd
--- /dev/null
+++ b/codecov.yml
@@ -0,0 +1,17 @@
+codecov:
+  require_ci_to_pass: yes
+
+coverage:
+  precision: 2
+  round: down
+  range: "85...100"
+
+  status:
+    project:
+      default:
+        target: 85%
+        threshold: 1%
+    patch: no
+    changes: no
+
+comment: off
diff --git a/continuous_integration/environment-3.7.yaml b/continuous_integration/environment-3.7.yaml
index ebc2cc1fb00..4e39cea8940 100644
--- a/continuous_integration/environment-3.7.yaml
+++ b/continuous_integration/environment-3.7.yaml
@@ -9,6 +9,7 @@ dependencies:
   - bokeh
   - click
   - cloudpickle
+  - coverage
   - cython  # Only tested here; also a dependency of crick
   - dask  # overridden by git tip below
   - filesystem-spec
@@ -27,6 +28,7 @@ dependencies:
   - psutil
   - pytest
   - pytest-asyncio<0.14.0
+  - pytest-cov
   - pytest-faulthandler
   - pytest-repeat
   - pytest-rerunfailures
diff --git a/continuous_integration/environment-3.8.yaml b/continuous_integration/environment-3.8.yaml
index 9454e023b66..a8ae9ff8984 100644
--- a/continuous_integration/environment-3.8.yaml
+++ b/continuous_integration/environment-3.8.yaml
@@ -9,6 +9,7 @@ dependencies:
   - bokeh
   - click
   - cloudpickle
+  - coverage
   - dask  # overridden by git tip below
   - filesystem-spec
   - h5py
@@ -25,6 +26,7 @@ dependencies:
   - psutil
   - pytest
   - pytest-asyncio<0.14.0
+  - pytest-cov
   - pytest-faulthandler
   - pytest-repeat
   - pytest-rerunfailures
diff --git a/continuous_integration/environment-3.9.yaml b/continuous_integration/environment-3.9.yaml
index de4a46ed036..1028d31f8cd 100644
--- a/continuous_integration/environment-3.9.yaml
+++ b/continuous_integration/environment-3.9.yaml
@@ -10,6 +10,7 @@ dependencies:
   - bokeh
   - click
   - cloudpickle
+  - coverage
   - dask  # overridden by git tip below
   - filesystem-spec  # overridden by git tip below
   - h5py
@@ -28,6 +29,7 @@ dependencies:
   - pynvml  # Only tested here
   - pytest
   - pytest-asyncio<0.14.0
+  - pytest-cov
   - pytest-faulthandler
   - pytest-repeat
   - pytest-rerunfailures
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index a7a63c48261..63912afe269 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -6861,7 +6861,8 @@ async def test_computation_object_code_client_compute(c, s, a, b):
 async def test_upload_directory(c, s, a, b, tmp_path):
     from dask.distributed import UploadDirectory
 
-    files = set(os.listdir())
+    # Be sure to exclude code coverage reports
+    files_start = {f for f in os.listdir() if not f.startswith(".coverage")}
 
     with open(tmp_path / "foo.py", "w") as f:
         f.write("x = 123")
@@ -6887,7 +6888,8 @@ def f():
         results = await c.run(f)
         assert results[n.worker_address] == 123
 
-    assert files == set(os.listdir())  # no change
+    files_end = {f for f in os.listdir() if not f.startswith(".coverage")}
+    assert files_start == files_end  # no change
 
 
 @gen_cluster(client=True)

From 92d830f04636a9f1d43c23ffd07cba4b46f2dd6e Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Mon, 4 Oct 2021 16:21:23 +0200
Subject: [PATCH 1521/1550] Log original exception upon compute failure (#5387)

---
 distributed/tests/test_worker.py | 25 +++++++------------------
 distributed/worker.py            |  2 +-
 2 files changed, 8 insertions(+), 19 deletions(-)

diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 34c7c66cd1f..c4481fb04fb 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -739,28 +739,17 @@ async def test_dont_overlap_communications_to_same_worker(c, s, a, b):
     assert l1["stop"] < l2["start"]
 
 
-@pytest.mark.avoid_ci
 @gen_cluster(client=True)
 async def test_log_exception_on_failed_task(c, s, a, b):
-    with tmpfile() as fn:
-        fh = logging.FileHandler(fn)
-        try:
-            from distributed.worker import logger
-
-            logger.addHandler(fh)
+    with captured_logger("distributed.worker") as logger:
+        future = c.submit(div, 1, 0)
+        await wait(future)
 
-            future = c.submit(div, 1, 0)
-            await wait(future)
-
-            await asyncio.sleep(0.1)
-            fh.flush()
-            with open(fn) as f:
-                text = f.read()
+        await asyncio.sleep(0.1)
 
-            assert "ZeroDivisionError" in text
-            assert "Exception" in text
-        finally:
-            logger.removeHandler(fh)
+    text = logger.getvalue()
+    assert "ZeroDivisionError" in text
+    assert "Exception" in text
 
 
 @gen_cluster(client=True)
diff --git a/distributed/worker.py b/distributed/worker.py
index 2acba897d2f..b73c93f32c9 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -3088,7 +3088,7 @@ async def execute(self, key, *, stimulus_id):
                     str(funcname(function))[:1000],
                     convert_args_to_str(args2, max_len=1000),
                     convert_kwargs_to_str(kwargs2, max_len=1000),
-                    ts.exception_text,
+                    result["exception_text"],
                 )
                 recommendations[ts] = (
                     "error",

From a04c221d427d45e8c85f7ad05700d959510d4e5f Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Mon, 4 Oct 2021 12:25:41 -0500
Subject: [PATCH 1522/1550] Remove `pytest.mark.repeat` from
 `test_prometheus_collect_task_states` (#5376)

---
 distributed/http/scheduler/tests/test_scheduler_http.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/distributed/http/scheduler/tests/test_scheduler_http.py b/distributed/http/scheduler/tests/test_scheduler_http.py
index 0a8741c47dc..dce785fdce5 100644
--- a/distributed/http/scheduler/tests/test_scheduler_http.py
+++ b/distributed/http/scheduler/tests/test_scheduler_http.py
@@ -103,7 +103,6 @@ async def test_prometheus(c, s, a, b):
         assert client.samples[0].value == 1.0
 
 
-@pytest.mark.repeat(100)
 @gen_cluster(client=True, clean_kwargs={"threads": False})
 async def test_prometheus_collect_task_states(c, s, a, b):
     pytest.importorskip("prometheus_client")

From 7bf884b941363242c3884b598205c75373287190 Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Tue, 5 Oct 2021 15:46:03 +0100
Subject: [PATCH 1523/1550] Use new Dask docs theme (#5391)

---
 docs/requirements.txt | 2 +-
 docs/source/conf.py   | 3 ---
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/docs/requirements.txt b/docs/requirements.txt
index 5a33e930b2a..ab40eda2c11 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -4,5 +4,5 @@ toolz
 cloudpickle
 git+https://github.com/dask/dask
 sphinx
-dask-sphinx-theme>=1.3.5
+dask-sphinx-theme>=2
 sphinx-click
diff --git a/docs/source/conf.py b/docs/source/conf.py
index 44cd4d73253..a36a5826f1f 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -118,10 +118,7 @@
 
 # -- Options for HTML output ----------------------------------------------
 
-import dask_sphinx_theme
-
 html_theme = "dask_sphinx_theme"
-html_theme_path = [dask_sphinx_theme.get_html_theme_path()]
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the

From 07d8e4c73bf1bf9cc7d9838745abdc054234105a Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Tue, 5 Oct 2021 16:01:16 -0500
Subject: [PATCH 1524/1550] Mark `test_worker_reconnects_mid_compute*` tests as
 flaky (#5378)

---
 distributed/tests/test_worker.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index c4481fb04fb..fe101155fe7 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -2444,6 +2444,7 @@ async def test_hold_on_to_replicas(c, s, *workers):
         await asyncio.sleep(0.01)
 
 
+@pytest.mark.flaky(reruns=10, reruns_delay=5)
 @gen_cluster(client=True)
 async def test_worker_reconnects_mid_compute(c, s, a, b):
     """Ensure that, if a worker disconnects while computing a result, the scheduler will
@@ -2512,6 +2513,7 @@ def fast_on_a(lock):
         await asyncio.sleep(0.001)
 
 
+@pytest.mark.flaky(reruns=10, reruns_delay=5)
 @gen_cluster(client=True)
 async def test_worker_reconnects_mid_compute_multiple_states_on_scheduler(c, s, a, b):
     """

From a13321dbb2d8ed7ac5487f4df90864baa0f1e1b3 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Wed, 6 Oct 2021 11:31:13 -0500
Subject: [PATCH 1525/1550] Mark `test_stress_creation_and_deletion` as `xfail`
 (#5393)

---
 distributed/tests/test_stress.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/distributed/tests/test_stress.py b/distributed/tests/test_stress.py
index e47ec84225e..0e6cd66fa19 100644
--- a/distributed/tests/test_stress.py
+++ b/distributed/tests/test_stress.py
@@ -85,6 +85,9 @@ def test_cancel_stress_sync(loop):
                 c.cancel(f)
 
 
+@pytest.mark.xfail(
+    reason="Flaky and re-fails on rerun. See https://github.com/dask/distributed/issues/5388"
+)
 @pytest.mark.slow
 @gen_cluster(
     nthreads=[],

From defe454f63199799b403a3ddeee04b473adf0dfd Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Wed, 6 Oct 2021 14:35:19 -0500
Subject: [PATCH 1526/1550] Add coverage badge to `README` (#5382)

---
 README.rst | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/README.rst b/README.rst
index 9d76f73165e..4a539443877 100644
--- a/README.rst
+++ b/README.rst
@@ -1,7 +1,7 @@
 Distributed
 ===========
 
-|Test Status| |Doc Status| |Gitter| |Version Status| |NumFOCUS|
+|Test Status| |Coverage| |Doc Status| |Gitter| |Version Status| |NumFOCUS|
 
 A library for distributed computation.  See documentation_ for more details.
 
@@ -11,6 +11,9 @@ A library for distributed computation.  See documentation_ for more details.
 .. |Doc Status| image:: https://readthedocs.org/projects/distributed/badge/?version=latest
    :target: https://distributed.dask.org
    :alt: Documentation Status
+.. |Coverage| image:: https://codecov.io/gh/dask/distributed/branch/main/graph/badge.svg
+   :target: https://codecov.io/gh/dask/distributed/branch/main
+   :alt: Coverage status
 .. |Gitter| image:: https://badges.gitter.im/Join%20Chat.svg
    :alt: Join the chat at https://gitter.im/dask/dask
    :target: https://gitter.im/dask/dask?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge

From a0fc0f2bf960394ebbe32b411e5a6cf721df4040 Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Thu, 7 Oct 2021 13:38:06 +0100
Subject: [PATCH 1527/1550] Active Memory Manager to use bulk comms (#5357)

---
 distributed/active_memory_manager.py          | 104 +++++++++------
 .../tests/test_active_memory_manager.py       | 119 ++++++++++++------
 2 files changed, 141 insertions(+), 82 deletions(-)

diff --git a/distributed/active_memory_manager.py b/distributed/active_memory_manager.py
index e2b085274c1..cc6ad8d3166 100644
--- a/distributed/active_memory_manager.py
+++ b/distributed/active_memory_manager.py
@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import asyncio
 from collections import defaultdict
 from collections.abc import Generator
 from typing import TYPE_CHECKING
@@ -10,7 +9,8 @@
 import dask
 from dask.utils import parse_timedelta
 
-from .utils import import_term
+from .metrics import time
+from .utils import import_term, log_errors
 
 if TYPE_CHECKING:
     from .scheduler import Scheduler, TaskState, WorkerState
@@ -115,45 +115,67 @@ def run_once(self, comm=None) -> None:
         """Run all policies once and asynchronously (fire and forget) enact their
         recommendations to replicate/drop keys
         """
-        # This should never fail since this is a synchronous method
-        assert not hasattr(self, "pending")
-
-        self.pending = defaultdict(lambda: (set(), set()))
-        self.workers_memory = {
-            w: w.memory.optimistic for w in self.scheduler.workers.values()
-        }
-        try:
-            # populate self.pending
-            self._run_policies()
-
-            drop_by_worker: defaultdict[str, set[str]] = defaultdict(set)
-            repl_by_worker: defaultdict[str, dict[str, list[str]]] = defaultdict(dict)
-
-            for ts, (pending_repl, pending_drop) in self.pending.items():
-                if not ts.who_has:
-                    continue
-                who_has = [ws_snd.address for ws_snd in ts.who_has - pending_drop]
-
-                assert who_has  # Never drop the last replica
-                for ws_rec in pending_repl:
-                    assert ws_rec not in ts.who_has
-                    repl_by_worker[ws_rec.address][ts.key] = who_has
-                for ws in pending_drop:
-                    assert ws in ts.who_has
-                    drop_by_worker[ws.address].add(ts.key)
-
-            # Fire-and-forget enact recommendations from policies
-            # This is temporary code, waiting for
-            # https://github.com/dask/distributed/pull/5046
-            for addr, who_has_map in repl_by_worker.items():
-                asyncio.create_task(self.scheduler.gather_on_worker(addr, who_has_map))
-            for addr, keys in drop_by_worker.items():
-                asyncio.create_task(self.scheduler.delete_worker_data(addr, keys))
-            # End temporary code
-
-        finally:
-            del self.workers_memory
-            del self.pending
+        with log_errors():
+            # This should never fail since this is a synchronous method
+            assert not hasattr(self, "pending")
+
+            self.pending = defaultdict(lambda: (set(), set()))
+            self.workers_memory = {
+                w: w.memory.optimistic for w in self.scheduler.workers.values()
+            }
+            try:
+                # populate self.pending
+                self._run_policies()
+
+                drop_by_worker: defaultdict[WorkerState, set[TaskState]] = defaultdict(
+                    set
+                )
+                repl_by_worker: defaultdict[
+                    WorkerState, dict[TaskState, set[WorkerState]]
+                ] = defaultdict(dict)
+
+                for ts, (pending_repl, pending_drop) in self.pending.items():
+                    if not ts.who_has:
+                        continue
+                    who_has = {ws_snd.address for ws_snd in ts.who_has - pending_drop}
+                    assert who_has  # Never drop the last replica
+                    for ws_rec in pending_repl:
+                        assert ws_rec not in ts.who_has
+                        repl_by_worker[ws_rec][ts] = who_has
+                    for ws in pending_drop:
+                        assert ws in ts.who_has
+                        drop_by_worker[ws].add(ts)
+
+                # Fire-and-forget enact recommendations from policies
+                stimulus_id = str(time())
+                for ws_rec, ts_to_who_has in repl_by_worker.items():
+                    self.scheduler.stream_comms[ws_rec.address].send(
+                        {
+                            "op": "acquire-replicas",
+                            "keys": [ts.key for ts in ts_to_who_has],
+                            "stimulus_id": "acquire-replicas-" + stimulus_id,
+                            "priorities": {ts.key: ts.priority for ts in ts_to_who_has},
+                            "who_has": {ts.key: v for ts, v in ts_to_who_has.items()},
+                        },
+                    )
+
+                for ws, tss in drop_by_worker.items():
+                    # The scheduler immediately forgets about the replica and suggests
+                    # the worker to drop it. The worker may refuse, at which point it
+                    # will send back an add-keys message to reinstate it.
+                    for ts in tss:
+                        self.scheduler.remove_replica(ts, ws)
+                    self.scheduler.stream_comms[ws.address].send(
+                        {
+                            "op": "remove-replicas",
+                            "keys": [ts.key for ts in tss],
+                            "stimulus_id": "remove-replicas-" + stimulus_id,
+                        }
+                    )
+
+            finally:
+                del self.workers_memory
+                del self.pending
 
     def _run_policies(self) -> None:
         """Sequentially run ActiveMemoryManagerPolicy.run() for all registered policies,
diff --git a/distributed/tests/test_active_memory_manager.py b/distributed/tests/test_active_memory_manager.py
index c7c747b8507..66c29760bae 100644
--- a/distributed/tests/test_active_memory_manager.py
+++ b/distributed/tests/test_active_memory_manager.py
@@ -209,7 +209,6 @@ async def test_drop_with_waiter(c, s, a, b):
     assert not y2.done()
 
 
-@pytest.mark.xfail(reason="distributed#5265")
 @gen_cluster(client=True, config=NO_AMM_START)
 async def test_double_drop(c, s, a, b):
     """An AMM drop policy runs once to drop one of the two replicas of a key.
@@ -329,46 +328,6 @@ async def test_drop_with_bad_candidates(c, s, a, b):
     assert s.tasks["x"].who_has == {ws0, ws1}
 
 
-class DropEverything(ActiveMemoryManagerPolicy):
-    """Inanely suggest to drop every single key in the cluster"""
-
-    def run(self):
-        for ts in self.manager.scheduler.tasks.values():
-            # Instead of yielding ("drop", ts, None) for each worker, which would result
-            # in semi-predictable output about which replica survives, randomly choose a
-            # different survivor at each AMM run.
-            candidates = list(ts.who_has)
-            random.shuffle(candidates)
-            for ws in candidates:
-                yield "drop", ts, {ws}
-
-
-@pytest.mark.xfail(reason="distributed#5046, distributed#5265")
-@pytest.mark.slow
-@gen_cluster(
-    client=True,
-    nthreads=[("", 1)] * 8,
-    Worker=Nanny,
-    config={
-        "distributed.scheduler.active-memory-manager.start": True,
-        "distributed.scheduler.active-memory-manager.interval": 0.1,
-        "distributed.scheduler.active-memory-manager.policies": [
-            {"class": "distributed.tests.test_active_memory_manager.DropEverything"},
-        ],
-    },
-)
-async def test_drop_stress(c, s, *nannies):
-    """A policy which suggests dropping everything won't break a running computation,
-    but only slow it down.
-    """
-    import dask.array as da
-
-    rng = da.random.RandomState(0)
-    a = rng.random((20, 20), chunks=(1, 1))
-    b = (a @ a.T).sum().round(3)
-    assert await c.compute(b) == 2134.398
-
-
 @gen_cluster(nthreads=[("", 1)] * 4, client=True, config=demo_config("replicate", n=2))
 async def test_replicate(c, s, *workers):
     futures = await c.scatter({"x": 123})
@@ -496,3 +455,81 @@ async def test_ReduceReplicas(c, s, *workers):
     s.extensions["amm"].run_once()
     while len(s.tasks["x"].who_has) > 1:
         await asyncio.sleep(0.01)
+
+
+class DropEverything(ActiveMemoryManagerPolicy):
+    """Inanely suggest to drop every single key in the cluster"""
+
+    def __init__(self):
+        self.i = 0
+
+    def run(self):
+        for ts in self.manager.scheduler.tasks.values():
+            # Instead of yielding ("drop", ts, None) for each worker, which would result
+            # in semi-predictable output about which replica survives, randomly choose a
+            # different survivor at each AMM run.
+            candidates = list(ts.who_has)
+            random.shuffle(candidates)
+            for ws in candidates:
+                yield "drop", ts, {ws}
+
+        # Stop running after ~2s
+        self.i += 1
+        if self.i == 20:
+            self.manager.policies.remove(self)
+
+
+async def _tensordot_stress(c):
+    da = pytest.importorskip("dask.array")
+
+    rng = da.random.RandomState(0)
+    a = rng.random((20, 20), chunks=(1, 1))
+    b = (a @ a.T).sum().round(3)
+    assert await c.compute(b) == 2134.398
+
+
+@pytest.mark.slow
+@pytest.mark.xfail(reason="https://github.com/dask/distributed/issues/5371")
+@gen_cluster(
+    client=True,
+    nthreads=[("", 1)] * 4,
+    Worker=Nanny,
+    config={
+        "distributed.scheduler.active-memory-manager.start": True,
+        "distributed.scheduler.active-memory-manager.interval": 0.1,
+        "distributed.scheduler.active-memory-manager.policies": [
+            {"class": "distributed.tests.test_active_memory_manager.DropEverything"},
+        ],
+    },
+    timeout=120,
+)
+async def test_drop_stress(c, s, *nannies):
+    """A policy which suggests dropping everything won't break a running computation,
+    but only slow it down.
+
+    See also: test_ReduceReplicas_stress
+    """
+    await _tensordot_stress(c)
+
+
+@pytest.mark.slow
+@pytest.mark.xfail(reason="https://github.com/dask/distributed/issues/5371")
+@gen_cluster(
+    client=True,
+    nthreads=[("", 1)] * 4,
+    Worker=Nanny,
+    config={
+        "distributed.scheduler.active-memory-manager.start": True,
+        "distributed.scheduler.active-memory-manager.interval": 0.1,
+        "distributed.scheduler.active-memory-manager.policies": [
+            {"class": "distributed.active_memory_manager.ReduceReplicas"},
+        ],
+    },
+    timeout=120,
+)
+async def test_ReduceReplicas_stress(c, s, *nannies):
+    """Running ReduceReplicas compulsively won't break a running computation. Unlike
+    test_drop_stress above, this test does not stop running after a few seconds - the
+    policy must not disrupt the computation too much.
+    """
+    await _tensordot_stress(c)

From 51a63ea8082f723d0e67411ab86b154d96eb0c43 Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Thu, 7 Oct 2021 15:40:17 +0100
Subject: [PATCH 1528/1550] Fix flaky test_WorkerPlugin_overwrite (#5398)

---
 distributed/diagnostics/tests/test_worker_plugin.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/distributed/diagnostics/tests/test_worker_plugin.py b/distributed/diagnostics/tests/test_worker_plugin.py
index 140bfbc6eee..0b0ca52e12b 100644
--- a/distributed/diagnostics/tests/test_worker_plugin.py
+++ b/distributed/diagnostics/tests/test_worker_plugin.py
@@ -1,3 +1,5 @@
+import asyncio
+
 import pytest
 
 from distributed import Worker, WorkerPlugin
@@ -274,6 +276,9 @@ def teardown(self, worker):
     await c.submit(inc, 0)
     assert w.foo == 123
 
+    while s.tasks or w.tasks:
+        await asyncio.sleep(0.01)
+
     class MyCustomPlugin(WorkerPlugin):
         name = "custom"
 

From 8fa3854514c5ede53e9c0381776f6109882ff778 Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Thu, 7 Oct 2021 23:17:42 +0100
Subject: [PATCH 1529/1550] Revisit `Scheduler.add_plugin` /
 `Scheduler.remove_plugin` (#5394)

---
 distributed/dashboard/components/scheduler.py | 10 +-
 .../tests/test_scheduler_plugin.py            | 22 ++++-
 distributed/scheduler.py                      | 99 ++++++++++++-------
 3 files changed, 84 insertions(+), 47 deletions(-)

diff --git a/distributed/dashboard/components/scheduler.py b/distributed/dashboard/components/scheduler.py
index ba1c6456a10..b99e7d885f7 100644
--- a/distributed/dashboard/components/scheduler.py
+++ b/distributed/dashboard/components/scheduler.py
@@ -1075,9 +1075,7 @@ def __init__(self, scheduler, **kwargs):
             self.scheduler = scheduler
 
             if TaskStreamPlugin.name not in self.scheduler.plugins:
-                self.scheduler.add_plugin(
-                    plugin=TaskStreamPlugin,
-                )
+                self.scheduler.add_plugin(TaskStreamPlugin(self.scheduler))
 
             compute_data = {
                 "times": [0.2, 0.1],
@@ -1239,7 +1237,7 @@ def __init__(self, scheduler, **kwargs):
             self.scheduler = scheduler
 
             if TaskStreamPlugin.name not in self.scheduler.plugins:
-                self.scheduler.add_plugin(TaskStreamPlugin)
+                self.scheduler.add_plugin(TaskStreamPlugin(self.scheduler))
 
             action_data = {
                 "times": [0.2, 0.1],
@@ -1765,9 +1763,7 @@ def __init__(self, scheduler, n_rectangles=1000, clear_interval="20s", **kwargs)
         self.offset = 0
 
         if TaskStreamPlugin.name not in self.scheduler.plugins:
-            self.scheduler.add_plugin(
-                plugin=TaskStreamPlugin,
-            )
+            self.scheduler.add_plugin(TaskStreamPlugin(self.scheduler))
         self.plugin = self.scheduler.plugins[TaskStreamPlugin.name]
 
         self.index = max(0, self.plugin.index - n_rectangles)
diff --git a/distributed/diagnostics/tests/test_scheduler_plugin.py b/distributed/diagnostics/tests/test_scheduler_plugin.py
index f895f2ae85f..4678765d8d1 100644
--- a/distributed/diagnostics/tests/test_scheduler_plugin.py
+++ b/distributed/diagnostics/tests/test_scheduler_plugin.py
@@ -29,9 +29,12 @@ def transition(self, key, start, finish, *args, **kwargs):
     await z
 
     assert counter.count == 3
-    s.remove_plugin(name="counter")
+    s.remove_plugin("counter")
     assert counter not in s.plugins
 
+    with pytest.raises(ValueError, match="Could not find plugin 'counter'") as e:
+        s.remove_plugin("counter")
+
 
 @gen_cluster(nthreads=[])
 async def test_add_remove_worker(s):
@@ -67,7 +70,7 @@ def remove_worker(self, worker, scheduler):
     ]
 
     events[:] = []
-    s.remove_plugin(name=plugin.name)
+    s.remove_plugin(plugin.name)
     a = await Worker(s.address)
     await a.close()
     assert events == []
@@ -104,7 +107,7 @@ async def remove_worker(self, worker, scheduler):
     }
 
     events[:] = []
-    s.remove_plugin(name=plugin.name)
+    s.remove_plugin(plugin.name)
     async with Worker(s.address):
         pass
     assert events == []
@@ -124,6 +127,19 @@ async def start(self, scheduler):
     assert "Multiple instances of" in msg
 
 
+@gen_cluster(client=True)
+async def test_add_by_type(c, s, a, b):
+    class MyPlugin(SchedulerPlugin):
+        def __init__(self, scheduler):
+            self.scheduler = scheduler
+
+    with pytest.warns(FutureWarning, match="Adding plugins by class is deprecated"):
+        s.add_plugin(MyPlugin)
+
+    inst = next(iter(p for p in s.plugins.values() if isinstance(p, MyPlugin)))
+    assert inst.scheduler is s
+
+
 @gen_test()
 async def test_lifecycle():
     class LifeCycle(SchedulerPlugin):
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index c09195e8dd3..3e755a580bb 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -5439,15 +5439,22 @@ async def handle_worker(self, comm=None, worker=None):
                 worker_comm.abort()
                 await self.remove_worker(address=worker)
 
-    def add_plugin(self, plugin=None, idempotent=False, name=None, **kwargs):
+    def add_plugin(
+        self,
+        plugin: SchedulerPlugin,
+        *,
+        idempotent: bool = False,
+        name: "str | None" = None,
+        **kwargs,
+    ):
         """Add external plugin to scheduler.
 
         See https://distributed.readthedocs.io/en/latest/plugins.html
 
-        Paramters
-        ---------
+        Parameters
+        ----------
         plugin : SchedulerPlugin
-            SchedulerPlugin class to add (can also be an instance)
+            SchedulerPlugin instance to add
         idempotent : bool
             If true, the plugin is assumed to already exist and no
             action is taken.
@@ -5456,64 +5463,83 @@ def add_plugin(self, plugin=None, idempotent=False, name=None, **kwargs):
             checked on the Plugin instance and generated if not
             discovered.
         **kwargs
-            Additional arguments passed to the `plugin` class if it is
-            not already an instance.
-
+            Deprecated; additional arguments passed to the `plugin` class if it is
+            not already an instance
         """
         if isinstance(plugin, type):
-            plugin = plugin(self, **kwargs)
+            warnings.warn(
+                "Adding plugins by class is deprecated and will be disabled in a "
+                "future release. Please add plugins by instance instead.",
+                category=FutureWarning,
+            )
+            plugin = plugin(self, **kwargs)  # type: ignore
+        elif kwargs:
+            raise ValueError("kwargs provided but plugin is already an instance")
 
         if name is None:
             name = _get_plugin_name(plugin)
 
         if name in self.plugins:
+            if idempotent:
+                return
             warnings.warn(
-                f"Scheduler already contains a plugin with name {name}; "
-                "overwriting.",
+                f"Scheduler already contains a plugin with name {name}; overwriting.",
                 category=UserWarning,
             )
 
-        if idempotent and name in self.plugins:
-            return
-
         self.plugins[name] = plugin
 
-    def remove_plugin(self, plugin=None, name=None):
+    def remove_plugin(
+        self,
+        name: "str | None" = None,
+        plugin: "SchedulerPlugin | None" = None,
+    ) -> None:
         """Remove external plugin from scheduler
 
-        Paramters
-        ---------
+        Parameters
+        ----------
+        name : str
+            Name of the plugin to remove
         plugin : SchedulerPlugin
             Deprecated; use `name` argument instead. Instance of a
             SchedulerPlugin class to remove;
-        name : str
-            Name of the plugin to remove
-
         """
+        # TODO: Remove this block of code once removing plugins by value is disabled
+        if bool(name) == bool(plugin):
+            raise ValueError("Must provide plugin or name (mutually exclusive)")
+        if isinstance(name, SchedulerPlugin):
+            # Backwards compatibility - the sig used to be (plugin, name)
+            plugin = name
+            name = None
         if plugin is not None:
             warnings.warn(
                 "Removing scheduler plugins by value is deprecated and will be disabled "
                 "in a future release. Please remove scheduler plugins by name instead.",
                 category=FutureWarning,
             )
-        if name is not None:
-            self.plugins.pop(name)
-        elif hasattr(plugin, "name"):
-            self.plugins.pop(plugin.name)
-        else:
-            # TODO: Remove this block of code once removing plugins by value is disabled
-            if plugin in list(self.plugins.values()):
-                if sum(plugin is p for p in list(self.plugins.values())) > 1:
+            if hasattr(plugin, "name"):
+                name = plugin.name
+            else:
+                names = [k for k, v in self.plugins.items() if v is plugin]
+                if not names:
                     raise ValueError(
-                        f"Multiple instances of {plugin} were found in the current scheduler "
-                        "plugins, we cannot remove this plugin."
+                        f"Could not find {plugin} among the current scheduler plugins"
                     )
-                else:
-                    warnings.warn(
-                        "Removing scheduler plugins by value is deprecated and will be disabled "
-                        "in a future release. Please remove scheduler plugins by name instead.",
-                        category=FutureWarning,
+                if len(names) > 1:
+                    raise ValueError(
+                        f"Multiple instances of {plugin} were found in the current "
+                        "scheduler plugins; we cannot remove this plugin."
                     )
+                name = names[0]
+        assert name is not None
+        # End deprecated code
+
+        try:
+            del self.plugins[name]
+        except KeyError:
+            raise ValueError(
+                f"Could not find plugin {name!r} among the current scheduler plugins"
+            )
 
     async def register_scheduler_plugin(self, comm=None, plugin=None, name=None):
         """Register a plugin on the scheduler."""
@@ -5529,7 +5555,7 @@ async def register_scheduler_plugin(self, comm=None, plugin=None, name=None):
         if hasattr(plugin, "start"):
             result = plugin.start(self)
             if inspect.isawaitable(result):
-                result = await result
+                await result
 
         self.add_plugin(plugin=plugin, name=name)
 
@@ -6994,7 +7020,7 @@ def get_task_stream(self, comm=None, start=None, stop=None, count=None):
         from distributed.diagnostics.task_stream import TaskStreamPlugin
 
         if TaskStreamPlugin.name not in self.plugins:
-            self.add_plugin(TaskStreamPlugin)
+            self.add_plugin(TaskStreamPlugin(self))
 
         plugin = self.plugins[TaskStreamPlugin.name]
 
@@ -7002,7 +7028,6 @@ def get_task_stream(self, comm=None, start=None, stop=None, count=None):
 
     def start_task_metadata(self, comm=None, name=None):
         plugin = CollectTaskMetaDataPlugin(scheduler=self, name=name)
-
         self.add_plugin(plugin)
 
     def stop_task_metadata(self, comm=None, name=None):

From c2b3add875790c535b2e60ff2e639fe449a7d2b5 Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Fri, 8 Oct 2021 12:03:23 +0200
Subject: [PATCH 1530/1550] Do not attempt to fetch keys which are no longer in
 flight (#5160)

---
 distributed/tests/test_worker.py | 65 ++++++++++++++++++++++++++++++++
 distributed/worker.py            |  7 ++--
 2 files changed, 69 insertions(+), 3 deletions(-)

diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index fe101155fe7..7a3126e9d1f 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -3082,3 +3082,68 @@ def clear_leak():
         {"action": "remove-worker", "processing-tasks": {}},
         {"action": "retired"},
     ]
+
+
+async def _wait_for_flight(key, worker):
+    while key not in worker.tasks or worker.tasks[key].state != "flight":
+        await asyncio.sleep(0)
+
+
+@gen_cluster(client=True)
+async def test_gather_dep_do_not_handle_response_of_not_requested_tasks(c, s, a, b):
+    """At time of writing, the gather_dep implementation filtered tasks again
+    for in-flight state. The response parser, however, did not distinguish
+    resulting in unwanted missing-data signals to the scheduler, causing
+    potential rescheduling or data leaks.
+    This test may become obsolete if the implementation changes significantly.
+    """
+    import distributed
+
+    with mock.patch.object(distributed.worker.Worker, "gather_dep") as mocked_gather:
+        fut1 = c.submit(inc, 1, workers=[a.address], key="f1")
+        fut2 = c.submit(inc, fut1, workers=[a.address], key="f2")
+        await fut2
+        fut4 = c.submit(sum, fut1, fut2, workers=[b.address], key="f4")
+        fut3 = c.submit(inc, fut1, workers=[b.address], key="f3")
+
+        fut2_key = fut2.key
+
+        await _wait_for_flight(fut2_key, b)
+
+        fut4.release()
+        while fut4.key in b.tasks:
+            await asyncio.sleep(0)
+
+    story_before = b.story(fut2.key)
+    assert fut2.key in mocked_gather.call_args.kwargs["to_gather"]
+    await Worker.gather_dep(b, **mocked_gather.call_args.kwargs)
+    story_after = b.story(fut2.key)
+    assert story_before == story_after
+    await fut3
+
+
+@gen_cluster(
+    client=True,
+    config={
+        "distributed.comm.recent-messages-log-length": 1000,
+    },
+)
+async def test_gather_dep_no_longer_in_flight_tasks(c, s, a, b):
+    import distributed
+
+    with mock.patch.object(distributed.worker.Worker, "gather_dep") as mocked_gather:
+        fut1 = c.submit(inc, 1, workers=[a.address], key="f1")
+        fut2 = c.submit(sum, fut1, fut1, workers=[b.address], key="f2")
+
+        fut1_key = fut1.key
+
+        await _wait_for_flight(fut1_key, b)
+
+        fut2.release()
+        while fut2.key in b.tasks:
+            await asyncio.sleep(0)
+
+    assert b.tasks[fut1.key] != "flight"
+    log_before = list(b.log)
+    await Worker.gather_dep(b, **mocked_gather.call_args.kwargs)
+    assert log_before == list(b.log)
diff --git a/distributed/worker.py b/distributed/worker.py
index b73c93f32c9..031c58a48f5 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -2514,10 +2514,11 @@ async def gather_dep(
                                 cause = dependent
                                 found_dependent_for_cause = True
                                 break
+
+                if not to_gather_keys:
+                    return
                 # Keep namespace clean since this func is long and has many
                 # dep*, *ts* variables
-
-                assert cause is not None
                 del to_gather, dependency_key, dependency_ts
 
                 self.log.append(
@@ -2618,7 +2619,7 @@ async def gather_dep(
                     )
 
                 recommendations: dict[TaskState, str | tuple] = {}
-                deps_to_iter = self.in_flight_workers.pop(worker)
+                deps_to_iter = set(self.in_flight_workers.pop(worker)) & to_gather_keys
 
                 for d in deps_to_iter:
                     ts = self.tasks.get(d)

From 4810027e58c300f1a33a4760847add915cc9b90c Mon Sep 17 00:00:00 2001
From: Ray Bell <rayjohnbell0@gmail.com>
Date: Fri, 8 Oct 2021 15:50:32 -0400
Subject: [PATCH 1531/1550] DOC: simple `SSHCluster` example (#5349)

---
 distributed/deploy/ssh.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/distributed/deploy/ssh.py b/distributed/deploy/ssh.py
index 35938c05067..1ca48a6b8b1 100644
--- a/distributed/deploy/ssh.py
+++ b/distributed/deploy/ssh.py
@@ -298,6 +298,15 @@ def SSHCluster(
 
     Examples
     --------
+    Create a cluster with one worker:
+
+    >>> from dask.distributed import Client, SSHCluster
+    >>> cluster = SSHCluster(["localhost", "localhost"])
+    >>> client = Client(cluster)
+
+    Create a cluster with three workers, each with two threads
+    and host the dashdoard on port 8797:
+
     >>> from dask.distributed import Client, SSHCluster
     >>> cluster = SSHCluster(
     ...     ["localhost", "localhost", "localhost", "localhost"],

From 835ea5f1029bad9cd0aaefcfa9213677afd8385d Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Mon, 11 Oct 2021 12:24:00 +0100
Subject: [PATCH 1532/1550] Rewrite test_client_timeout (#5397)

---
 distributed/tests/test_client.py | 22 ++++++++--------------
 1 file changed, 8 insertions(+), 14 deletions(-)

diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 63912afe269..982212bf74d 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -4537,24 +4537,18 @@ async def test_scatter_dict_workers(c, s, a, b):
 
 
 @pytest.mark.slow
-@gen_test(timeout=180)
+@gen_test()
 async def test_client_timeout():
+    """`await Client(...)` keeps retrying for 10 seconds if it can't find the Scheduler
+    straight away
+    """
     c = Client("127.0.0.1:57484", asynchronous=True)
-
-    s = Scheduler(loop=c.loop, port=57484, dashboard_address=":0")
+    client_start_fut = asyncio.ensure_future(c)
     await asyncio.sleep(4)
-
-    try:
-        await s
-    except OSError:  # port in use
+    async with Scheduler(port=57484, dashboard_address=":0"):
+        await client_start_fut
+        assert await c.run_on_scheduler(lambda: 123) == 123
         await c.close()
-        return
-
-    try:
-        await c
-        await c.close()
-    finally:
-        await s.close()
 
 
 @gen_cluster(client=True)

From efbf8882cdbdcf3543289802c25902b2aa8aab79 Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Wed, 13 Oct 2021 10:16:25 +0100
Subject: [PATCH 1533/1550] Enable mypy in CI 2/2 (#5348)

---
 .pre-commit-config.yaml              |  33 +-
 distributed/active_memory_manager.py |  12 +-
 distributed/scheduler.py             | 520 ++++++++++++++-------------
 distributed/worker.py                |   4 +-
 docs/source/develop.rst              |  22 +-
 5 files changed, 321 insertions(+), 270 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index daf8c3299f7..a2ae5f58b1a 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -4,6 +4,14 @@ repos:
       hooks:
       - id: isort
         language_version: python3
+  - repo: https://github.com/asottile/pyupgrade
+    # Do not upgrade: there's a bug in Cython that causes sum(... for ...) to fail;
+    # it needs sum([... for ...])
+    rev: v2.13.0
+    hooks:
+      - id: pyupgrade
+        args:
+          - --py37-plus
   -   repo: https://github.com/psf/black
       rev: 21.9b0
       hooks:
@@ -17,11 +25,22 @@ repos:
       hooks:
       - id: flake8
         language_version: python3
-  - repo: https://github.com/asottile/pyupgrade
-    # Do not upgrade: there's a bug in Cython that causes sum(... for ...) to fail;
-    # it needs sum([... for ...])
-    rev: v2.13.0
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v0.910
     hooks:
-      - id: pyupgrade
-        args:
-          - --py37-plus
+      - id: mypy
+        additional_dependencies:
+          # Type stubs
+          - types-docutils
+          - types-requests
+          - types-paramiko
+          - types-pkg_resources
+          - types-PyYAML
+          - types-setuptools
+          - types-psutil
+          # Libraries exclusively imported under `if TYPE_CHECKING:`
+          - typing_extensions  # To be reviewed after dropping Python 3.7
+          # Typed libraries
+          - numpy
+          - dask
+          - tornado
diff --git a/distributed/active_memory_manager.py b/distributed/active_memory_manager.py
index cc6ad8d3166..0bcd83522ca 100644
--- a/distributed/active_memory_manager.py
+++ b/distributed/active_memory_manager.py
@@ -127,12 +127,12 @@ def run_once(self, comm=None) -> None:
                 # populate self.pending
                 self._run_policies()
 
-                drop_by_worker: defaultdict[WorkerState, set[TaskState]] = defaultdict(
-                    set
-                )
-                repl_by_worker: defaultdict[
-                    WorkerState, dict[TaskState, set[WorkerState]]
-                ] = defaultdict(dict)
+                drop_by_worker: (
+                    defaultdict[WorkerState, set[TaskState]]
+                ) = defaultdict(set)
+                repl_by_worker: (
+                    defaultdict[WorkerState, dict[TaskState, set[str]]]
+                ) = defaultdict(dict)
 
                 for ts, (pending_repl, pending_drop) in self.pending.items():
                     if not ts.who_has:
diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 3e755a580bb..12add27f2d8 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -13,14 +13,24 @@
 import warnings
 import weakref
 from collections import defaultdict, deque
-from collections.abc import Hashable, Iterable, Iterator, Mapping, Set
+from collections.abc import (
+    Callable,
+    Collection,
+    Hashable,
+    Iterable,
+    Iterator,
+    Mapping,
+    Set,
+)
 from contextlib import suppress
 from datetime import timedelta
 from functools import partial
 from numbers import Number
+from typing import ClassVar
+from typing import cast as pep484_cast
 
 import psutil
-import sortedcontainers
+from sortedcontainers import SortedDict, SortedSet
 from tlz import (
     compose,
     first,
@@ -524,16 +534,17 @@ class WorkerState:
 
     def __init__(
         self,
-        address: str = None,
-        pid: Py_ssize_t = 0,
-        name: object = None,
+        *,
+        address: str,
+        pid: Py_ssize_t,
+        name: object,
         nthreads: Py_ssize_t = 0,
-        memory_limit: Py_ssize_t = 0,
-        local_directory: str = None,
-        services: dict = None,
-        versions: dict = None,
-        nanny: str = None,
-        extra: dict = None,
+        memory_limit: Py_ssize_t,
+        local_directory: str,
+        nanny: str,
+        services: "dict | None" = None,
+        versions: "dict | None" = None,
+        extra: "dict | None" = None,
     ):
         self._address = address
         self._pid = pid
@@ -584,7 +595,7 @@ def actors(self):
         return self._actors
 
     @property
-    def address(self):
+    def address(self) -> str:
         return self._address
 
     @property
@@ -781,7 +792,7 @@ class Computation:
     def __init__(self):
         self._start = time()
         self._groups = set()
-        self._code = sortedcontainers.SortedSet()
+        self._code = SortedSet()
         self._id = uuid.uuid4()
 
     @property
@@ -862,7 +873,7 @@ class TaskPrefix:
     """
 
     _name: str
-    _all_durations: object
+    _all_durations: "defaultdict[str, float]"
     _duration_average: double
     _suspicious: Py_ssize_t
     _groups: list
@@ -882,19 +893,19 @@ def __init__(self, name: str):
         self._suspicious = 0
 
     @property
-    def name(self):
+    def name(self) -> str:
         return self._name
 
     @property
-    def all_durations(self):
+    def all_durations(self) -> "defaultdict[str, float]":
         return self._all_durations
 
     @property
-    def duration_average(self):
+    def duration_average(self) -> double:
         return self._duration_average
 
     @property
-    def suspicious(self):
+    def suspicious(self) -> Py_ssize_t:
         return self._suspicious
 
     @property
@@ -907,7 +918,7 @@ def states(self):
         return merge_with(sum, [tg._states for tg in self._groups])
 
     @property
-    def active(self):
+    def active(self) -> "list[TaskGroup]":
         tg: TaskGroup
         return [
             tg
@@ -1000,7 +1011,7 @@ class TaskGroup:
     """
 
     _name: str
-    _prefix: TaskPrefix
+    _prefix: TaskPrefix  # TaskPrefix | None
     _states: dict
     _dependencies: set
     _nbytes_total: Py_ssize_t
@@ -1008,13 +1019,13 @@ class TaskGroup:
     _types: set
     _start: double
     _stop: double
-    _all_durations: object
-    _last_worker: WorkerState
+    _all_durations: "defaultdict[str, float]"
+    _last_worker: WorkerState  # WorkerState | None
     _last_worker_tasks_left: Py_ssize_t
 
     def __init__(self, name: str):
         self._name = name
-        self._prefix = None
+        self._prefix = None  # type: ignore
         self._states = {state: 0 for state in ALL_TASK_STATES}
         self._states["forgotten"] = 0
         self._dependencies = set()
@@ -1024,23 +1035,23 @@ def __init__(self, name: str):
         self._start = 0.0
         self._stop = 0.0
         self._all_durations = defaultdict(float)
-        self._last_worker = None
+        self._last_worker = None  # type: ignore
         self._last_worker_tasks_left = 0
 
     @property
-    def name(self):
+    def name(self) -> str:
         return self._name
 
     @property
-    def prefix(self):
+    def prefix(self) -> "TaskPrefix | None":
         return self._prefix
 
     @property
-    def states(self):
+    def states(self) -> dict:
         return self._states
 
     @property
-    def dependencies(self):
+    def dependencies(self) -> set:
         return self._dependencies
 
     @property
@@ -1048,38 +1059,37 @@ def nbytes_total(self):
         return self._nbytes_total
 
     @property
-    def duration(self):
+    def duration(self) -> double:
         return self._duration
 
     @property
-    def types(self):
+    def types(self) -> set:
         return self._types
 
     @property
-    def all_durations(self):
+    def all_durations(self) -> "defaultdict[str, float]":
         return self._all_durations
 
     @property
-    def start(self):
+    def start(self) -> double:
         return self._start
 
     @property
-    def stop(self):
+    def stop(self) -> double:
         return self._stop
 
     @property
-    def last_worker(self):
+    def last_worker(self) -> "WorkerState | None":
         return self._last_worker
 
     @property
-    def last_worker_tasks_left(self):
+    def last_worker_tasks_left(self) -> int:
         return self._last_worker_tasks_left
 
     @ccall
-    def add(self, o):
-        ts: TaskState = o
-        self._states[ts._state] += 1
-        ts._group = self
+    def add(self, other: "TaskState"):
+        self._states[other._state] += 1
+        other._group = self
 
     def __repr__(self):
         return (
@@ -1347,34 +1357,34 @@ class TaskState:
     _hash: Py_hash_t
     _prefix: TaskPrefix
     _run_spec: object
-    _priority: tuple
-    _state: str
-    _dependencies: set
-    _dependents: set
+    _priority: tuple  # tuple | None
+    _state: str  # str | None
+    _dependencies: set  # set[TaskState]
+    _dependents: set  # set[TaskState]
     _has_lost_dependencies: bint
-    _waiting_on: set
-    _waiters: set
-    _who_wants: set
-    _who_has: set
-    _processing_on: WorkerState
+    _waiting_on: set  # set[TaskState]
+    _waiters: set  # set[TaskState]
+    _who_wants: set  # set[ClientState]
+    _who_has: set  # set[WorkerState]
+    _processing_on: WorkerState  # WorkerState | None
     _retries: Py_ssize_t
     _nbytes: Py_ssize_t
-    _type: str
+    _type: str  # str | None
     _exception: object
     _exception_text: str
     _traceback: object
     _traceback_text: str
-    _exception_blame: object
+    _exception_blame: "TaskState"  # TaskState | None"
     _erred_on: set
     _suspicious: Py_ssize_t
-    _host_restrictions: set
-    _worker_restrictions: set
-    _resource_restrictions: dict
+    _host_restrictions: set  # set[str] | None
+    _worker_restrictions: set  # set[str] | None
+    _resource_restrictions: dict  # dict | None
     _loose_restrictions: bint
     _metadata: dict
     _annotations: dict
     _actor: bint
-    _group: TaskGroup
+    _group: TaskGroup  # TaskGroup | None
     _group_key: str
 
     __slots__ = (
@@ -1434,28 +1444,32 @@ def __init__(self, key: str, run_spec: object):
         self._key = key
         self._hash = hash(key)
         self._run_spec = run_spec
-        self._state = None
-        self._exception = self._traceback = self._exception_blame = None
-        self._exception_text = self._traceback_text = ""
-        self._suspicious = self._retries = 0
+        self._state = None  # type: ignore
+        self._exception = None
+        self._exception_blame = None  # type: ignore
+        self._traceback = None
+        self._exception_text = ""
+        self._traceback_text = ""
+        self._suspicious = 0
+        self._retries = 0
         self._nbytes = -1
-        self._priority = None
+        self._priority = None  # type: ignore
         self._who_wants = set()
         self._dependencies = set()
         self._dependents = set()
         self._waiting_on = set()
         self._waiters = set()
         self._who_has = set()
-        self._processing_on = None
+        self._processing_on = None  # type: ignore
         self._has_lost_dependencies = False
-        self._host_restrictions = None
-        self._worker_restrictions = None
-        self._resource_restrictions = None
+        self._host_restrictions = None  # type: ignore
+        self._worker_restrictions = None  # type: ignore
+        self._resource_restrictions = None  # type: ignore
         self._loose_restrictions = False
         self._actor = False
-        self._type = None
+        self._type = None  # type: ignore
         self._group_key = key_split_group(key)
-        self._group = None
+        self._group = None  # type: ignore
         self._metadata = {}
         self._annotations = {}
         self._erred_on = set()
@@ -1485,11 +1499,11 @@ def run_spec(self):
         return self._run_spec
 
     @property
-    def priority(self):
+    def priority(self) -> "tuple | None":
         return self._priority
 
     @property
-    def state(self) -> str:
+    def state(self) -> "str | None":
         return self._state
 
     @state.setter
@@ -1499,11 +1513,11 @@ def state(self, value: str):
         self._state = value
 
     @property
-    def dependencies(self):
+    def dependencies(self) -> "set[TaskState]":
         return self._dependencies
 
     @property
-    def dependents(self):
+    def dependents(self) -> "set[TaskState]":
         return self._dependents
 
     @property
@@ -1511,27 +1525,27 @@ def has_lost_dependencies(self):
         return self._has_lost_dependencies
 
     @property
-    def waiting_on(self):
+    def waiting_on(self) -> "set[TaskState]":
         return self._waiting_on
 
     @property
-    def waiters(self):
+    def waiters(self) -> "set[TaskState]":
         return self._waiters
 
     @property
-    def who_wants(self):
+    def who_wants(self) -> "set[ClientState]":
         return self._who_wants
 
     @property
-    def who_has(self):
+    def who_has(self) -> "set[WorkerState]":
         return self._who_has
 
     @property
-    def processing_on(self):
+    def processing_on(self) -> "WorkerState | None":
         return self._processing_on
 
     @processing_on.setter
-    def processing_on(self, v: WorkerState):
+    def processing_on(self, v: WorkerState) -> None:
         self._processing_on = v
 
     @property
@@ -1547,7 +1561,7 @@ def nbytes(self, v: Py_ssize_t):
         self._nbytes = v
 
     @property
-    def type(self):
+    def type(self) -> "str | None":
         return self._type
 
     @property
@@ -1567,7 +1581,7 @@ def traceback_text(self):
         return self._traceback_text
 
     @property
-    def exception_blame(self):
+    def exception_blame(self) -> "TaskState | None":
         return self._exception_blame
 
     @property
@@ -1575,15 +1589,15 @@ def suspicious(self):
         return self._suspicious
 
     @property
-    def host_restrictions(self):
+    def host_restrictions(self) -> "set[str] | None":
         return self._host_restrictions
 
     @property
-    def worker_restrictions(self):
+    def worker_restrictions(self) -> "set[str] | None":
         return self._worker_restrictions
 
     @property
-    def resource_restrictions(self):
+    def resource_restrictions(self) -> "dict | None":
         return self._resource_restrictions
 
     @property
@@ -1603,11 +1617,11 @@ def actor(self):
         return self._actor
 
     @property
-    def group(self):
+    def group(self) -> "TaskGroup | None":
         return self._group
 
     @property
-    def group_key(self):
+    def group_key(self) -> str:
         return self._group_key
 
     @property
@@ -1839,12 +1853,12 @@ class SchedulerState:
 
     _aliases: dict
     _bandwidth: double
-    _clients: dict
+    _clients: dict  # dict[str, ClientState]
     _computations: object
     _extensions: dict
     _host_info: dict
-    _idle: object
-    _idle_dv: dict
+    _idle: "SortedDict[str, WorkerState]"
+    _idle_dv: dict  # dict[str, WorkerState]
     _n_tasks: Py_ssize_t
     _resources: dict
     _saturated: set
@@ -1859,9 +1873,10 @@ class SchedulerState:
     _unknown_durations: dict
     _unrunnable: set
     _validate: bint
-    _workers: object
-    _workers_dv: dict
+    _workers: "SortedDict[str, WorkerState]"
+    _workers_dv: dict  # dict[str, WorkerState]
     _transition_counter: Py_ssize_t
+    _plugins: dict  # dict[str, SchedulerPlugin]
 
     # Variables from dask.config, cached by __init__ for performance
     UNKNOWN_TASK_DURATION: double
@@ -1873,54 +1888,41 @@ class SchedulerState:
 
     def __init__(
         self,
-        aliases: dict = None,
-        clients: dict = None,
-        workers=None,
-        host_info=None,
-        resources=None,
-        tasks: dict = None,
-        unrunnable: set = None,
-        validate: bint = False,
-        **kwargs,
+        aliases: dict,
+        clients: "dict[str, ClientState]",
+        workers: "SortedDict[str, WorkerState]",
+        host_info: dict,
+        resources: dict,
+        tasks: dict,
+        unrunnable: set,
+        validate: bint,
+        plugins: "Iterable[SchedulerPlugin]" = (),
+        **kwargs,  # Passed verbatim to Server.__init__()
     ):
-        if aliases is not None:
-            self._aliases = aliases
-        else:
-            self._aliases = dict()
+        self._aliases = aliases
         self._bandwidth = parse_bytes(
             dask.config.get("distributed.scheduler.bandwidth")
         )
-        if clients is not None:
-            self._clients = clients
-        else:
-            self._clients = dict()
+        self._clients = clients
         self._clients["fire-and-forget"] = ClientState("fire-and-forget")
-        self._extensions = dict()
-        if host_info is not None:
-            self._host_info = host_info
-        else:
-            self._host_info = dict()
-        self._idle = sortedcontainers.SortedDict()
-        self._idle_dv: dict = cast(dict, self._idle)
+        self._extensions = {}
+        self._host_info = host_info
+        self._idle = SortedDict()
+        # Note: cython.cast, not typing.cast!
+        self._idle_dv = cast(dict, self._idle)
         self._n_tasks = 0
-        if resources is not None:
-            self._resources = resources
-        else:
-            self._resources = dict()
+        self._resources = resources
         self._saturated = set()
-        if tasks is not None:
-            self._tasks = tasks
-        else:
-            self._tasks = dict()
+        self._tasks = tasks
         self._replicated_tasks = {
             ts for ts in self._tasks.values() if len(ts._who_has) > 1
         }
         self._computations = deque(
             maxlen=dask.config.get("distributed.diagnostics.computations.max-history")
         )
-        self._task_groups = dict()
-        self._task_prefixes = dict()
-        self._task_metadata = dict()
+        self._task_groups = {}
+        self._task_prefixes = {}
+        self._task_metadata = {}
         self._total_nthreads = 0
         self._total_occupancy = 0
         self._transitions_table = {
@@ -1940,17 +1942,13 @@ def __init__(
             ("memory", "released"): self.transition_memory_released,
             ("released", "erred"): self.transition_released_erred,
         }
-        self._unknown_durations = dict()
-        if unrunnable is not None:
-            self._unrunnable = unrunnable
-        else:
-            self._unrunnable = set()
+        self._unknown_durations = {}
+        self._unrunnable = unrunnable
         self._validate = validate
-        if workers is not None:
-            self._workers = workers
-        else:
-            self._workers = sortedcontainers.SortedDict()
-        self._workers_dv: dict = cast(dict, self._workers)
+        self._workers = workers
+        # Note: cython.cast, not typing.cast!
+        self._workers_dv = cast(dict, self._workers)
+        self._plugins = {} if not plugins else {_get_plugin_name(p): p for p in plugins}
 
         # Variables from dask.config, cached by __init__ for performance
         self.UNKNOWN_TASK_DURATION = parse_timedelta(
@@ -1974,7 +1972,8 @@ def __init__(
         )
         self._transition_counter = 0
 
-        super().__init__(**kwargs)
+        # Call Server.__init__()
+        super().__init__(**kwargs)  # type: ignore
 
     @property
     def aliases(self):
@@ -2072,6 +2071,10 @@ def validate(self, v: bint):
     def workers(self):
         return self._workers
 
+    @property
+    def plugins(self) -> "dict[str, SchedulerPlugin]":
+        return self._plugins
+
     @property
     def memory(self) -> MemoryState:
         return MemoryState.sum(*(w.memory for w in self.workers.values()))
@@ -2109,14 +2112,13 @@ def new_task(
 
         tp: TaskPrefix
         prefix_key = key_split(key)
-        tp = self._task_prefixes.get(prefix_key)
+        tp = self._task_prefixes.get(prefix_key)  # type: ignore
         if tp is None:
             self._task_prefixes[prefix_key] = tp = TaskPrefix(prefix_key)
         ts._prefix = tp
 
-        tg: TaskGroup
         group_key = ts._group_key
-        tg = self._task_groups.get(group_key)
+        tg: TaskGroup = self._task_groups.get(group_key)  # type: ignore
         if tg is None:
             self._task_groups[group_key] = tg = TaskGroup(group_key)
             if computation:
@@ -2166,7 +2168,7 @@ def _transition(self, key, finish: str, *args, **kwargs):
             worker_msgs = {}
             client_msgs = {}
 
-            ts = parent._tasks.get(key)
+            ts = parent._tasks.get(key)  # type: ignore
             if ts is None:
                 return recommendations, client_msgs, worker_msgs
             start = ts._state
@@ -2180,9 +2182,8 @@ def _transition(self, key, finish: str, *args, **kwargs):
             start_finish = (start, finish)
             func = self._transitions_table.get(start_finish)
             if func is not None:
-                a: tuple = func(key, *args, **kwargs)
+                recommendations, client_msgs, worker_msgs = func(key, *args, **kwargs)
                 self._transition_counter += 1
-                recommendations, client_msgs, worker_msgs = a
             elif "released" not in start_finish:
                 assert not args and not kwargs, (args, kwargs, start_finish)
                 a_recs: dict
@@ -2201,13 +2202,13 @@ def _transition(self, key, finish: str, *args, **kwargs):
 
                 recommendations.update(a_recs)
                 for c, new_msgs in a_cmsgs.items():
-                    msgs = client_msgs.get(c)
+                    msgs = client_msgs.get(c)  # type: ignore
                     if msgs is not None:
                         msgs.extend(new_msgs)
                     else:
                         client_msgs[c] = new_msgs
                 for w, new_msgs in a_wmsgs.items():
-                    msgs = worker_msgs.get(w)
+                    msgs = worker_msgs.get(w)  # type: ignore
                     if msgs is not None:
                         msgs.extend(new_msgs)
                     else:
@@ -2215,13 +2216,13 @@ def _transition(self, key, finish: str, *args, **kwargs):
 
                 recommendations.update(b_recs)
                 for c, new_msgs in b_cmsgs.items():
-                    msgs = client_msgs.get(c)
+                    msgs = client_msgs.get(c)  # type: ignore
                     if msgs is not None:
                         msgs.extend(new_msgs)
                     else:
                         client_msgs[c] = new_msgs
                 for w, new_msgs in b_wmsgs.items():
-                    msgs = worker_msgs.get(w)
+                    msgs = worker_msgs.get(w)  # type: ignore
                     if msgs is not None:
                         msgs.extend(new_msgs)
                     else:
@@ -2232,7 +2233,11 @@ def _transition(self, key, finish: str, *args, **kwargs):
                 raise RuntimeError("Impossible transition from %r to %r" % start_finish)
 
             finish2 = ts._state
-            self.transition_log.append((key, start, finish2, recommendations, time()))
+            # FIXME downcast antipattern
+            scheduler = pep484_cast(Scheduler, self)
+            scheduler.transition_log.append(
+                (key, start, finish2, recommendations, time())
+            )
             if parent._validate:
                 logger.debug(
                     "Transitioned %r %s->%s (actual: %s).  Consequence: %s",
@@ -2283,7 +2288,6 @@ def _transitions(self, recommendations: dict, client_msgs: dict, worker_msgs: di
         This includes feedback from previous transitions and continues until we
         reach a steady state
         """
-        parent: SchedulerState = cast(SchedulerState, self)
         keys: set = set()
         recommendations = recommendations.copy()
         msgs: list
@@ -2301,21 +2305,23 @@ def _transitions(self, recommendations: dict, client_msgs: dict, worker_msgs: di
 
             recommendations.update(new_recs)
             for c, new_msgs in new_cmsgs.items():
-                msgs = client_msgs.get(c)
+                msgs = client_msgs.get(c)  # type: ignore
                 if msgs is not None:
                     msgs.extend(new_msgs)
                 else:
                     client_msgs[c] = new_msgs
             for w, new_msgs in new_wmsgs.items():
-                msgs = worker_msgs.get(w)
+                msgs = worker_msgs.get(w)  # type: ignore
                 if msgs is not None:
                     msgs.extend(new_msgs)
                 else:
                     worker_msgs[w] = new_msgs
 
-        if parent._validate:
+        if self._validate:
+            # FIXME downcast antipattern
+            scheduler = pep484_cast(Scheduler, self)
             for key in keys:
-                self.validate_key(key)
+                scheduler.validate_key(key)
 
     def transition_released_waiting(self, key):
         try:
@@ -2457,7 +2463,7 @@ def transition_no_worker_memory(
 
     @ccall
     @exceptval(check=False)
-    def decide_worker(self, ts: TaskState) -> WorkerState:
+    def decide_worker(self, ts: TaskState) -> WorkerState:  # -> WorkerState | None
         """
         Decide on a worker for task *ts*. Return a WorkerState.
 
@@ -2471,10 +2477,10 @@ def decide_worker(self, ts: TaskState) -> WorkerState:
         in a round-robin fashion.
         """
         if not self._workers_dv:
-            return None
+            return None  # type: ignore
 
-        ws: WorkerState = None
-        group: TaskGroup = ts._group
+        ws: WorkerState
+        tg: TaskGroup = ts._group
         valid_workers: set = self.valid_workers(ts)
 
         if (
@@ -2484,34 +2490,35 @@ def decide_worker(self, ts: TaskState) -> WorkerState:
         ):
             self._unrunnable.add(ts)
             ts.state = "no-worker"
-            return ws
+            return None  # type: ignore
 
-        # Group is larger than cluster with few dependencies? Minimize future data transfers.
+        # Group is larger than cluster with few dependencies?
+        # Minimize future data transfers.
         if (
             valid_workers is None
-            and len(group) > self._total_nthreads * 2
-            and len(group._dependencies) < 5
-            and sum(map(len, group._dependencies)) < 5
+            and len(tg) > self._total_nthreads * 2
+            and len(tg._dependencies) < 5
+            and sum(map(len, tg._dependencies)) < 5
         ):
-            ws: WorkerState = group._last_worker
+            ws = tg._last_worker
 
             if not (
-                ws and group._last_worker_tasks_left and ws._address in self._workers_dv
+                ws and tg._last_worker_tasks_left and ws._address in self._workers_dv
             ):
                 # Last-used worker is full or unknown; pick a new worker for the next few tasks
                 ws = min(
                     (self._idle_dv or self._workers_dv).values(),
                     key=partial(self.worker_objective, ts),
                 )
-                group._last_worker_tasks_left = math.floor(
-                    (len(group) / self._total_nthreads) * ws._nthreads
+                tg._last_worker_tasks_left = math.floor(
+                    (len(tg) / self._total_nthreads) * ws._nthreads
                 )
 
             # Record `last_worker`, or clear it on the final task
-            group._last_worker = (
-                ws if group.states["released"] + group.states["waiting"] > 1 else None
+            tg._last_worker = (
+                ws if tg.states["released"] + tg.states["waiting"] > 1 else None
             )
-            group._last_worker_tasks_left -= 1
+            tg._last_worker_tasks_left -= 1
             return ws
 
         if ts._dependencies or valid_workers is not None:
@@ -2524,6 +2531,7 @@ def decide_worker(self, ts: TaskState) -> WorkerState:
         else:
             # Fastpath when there are no related tasks or restrictions
             worker_pool = self._idle or self._workers
+            # Note: cython.cast, not typing.cast!
             worker_pool_dv = cast(dict, worker_pool)
             wp_vals = worker_pool.values()
             n_workers: Py_ssize_t = len(worker_pool_dv)
@@ -2676,6 +2684,8 @@ def transition_processing_memory(
         worker_msgs: dict = {}
         try:
             ts: TaskState = self._tasks[key]
+            tg: TaskGroup = ts._group
+
             assert worker
             assert isinstance(worker, str)
 
@@ -2688,15 +2698,15 @@ def transition_processing_memory(
                 assert not ts._exception_blame
                 assert ts._state == "processing"
 
-            ws = self._workers_dv.get(worker)
+            ws = self._workers_dv.get(worker)  # type: ignore
             if ws is None:
                 recommendations[key] = "released"
                 return recommendations, client_msgs, worker_msgs
 
             if ws != ts._processing_on:  # someone else has this task
                 logger.info(
-                    "Unexpected worker completed task, likely due to"
-                    " work stealing.  Expected: %s, Got: %s, Key: %s",
+                    "Unexpected worker completed task, likely due to "
+                    "work stealing. Expected: %s, Got: %s, Key: %s",
                     ts._processing_on,
                     ws,
                     key,
@@ -2726,7 +2736,7 @@ def transition_processing_memory(
                     # record timings of all actions -- a cheaper way of
                     # getting timing info compared with get_task_stream()
                     ts._prefix._all_durations[action] += stop - start
-                    ts._group._all_durations[action] += stop - start
+                    tg._all_durations[action] += stop - start
 
             #############################
             # Update Timing Information #
@@ -2742,10 +2752,10 @@ def transition_processing_memory(
                     avg_duration = 0.5 * old_duration + 0.5 * new_duration
 
                 ts._prefix._duration_average = avg_duration
-                ts._group._duration += new_duration
-                ts._group._start = ts._group._start or compute_start
-                if ts._group._stop < compute_stop:
-                    ts._group._stop = compute_stop
+                tg._duration += new_duration
+                tg._start = tg._start or compute_start
+                if tg._stop < compute_stop:
+                    tg._stop = compute_stop
 
                 s: set = self._unknown_durations.pop(ts._prefix._name, None)
                 tts: TaskState
@@ -3068,15 +3078,15 @@ def transition_processing_erred(
             ts._erred_on.add(w or worker)
             if exception is not None:
                 ts._exception = exception
-                ts._exception_text = exception_text
+                ts._exception_text = exception_text  # type: ignore
             if traceback is not None:
                 ts._traceback = traceback
-                ts._traceback_text = traceback_text
+                ts._traceback_text = traceback_text  # type: ignore
             if cause is not None:
                 failing_ts = self._tasks[cause]
                 ts._exception_blame = failing_ts
             else:
-                failing_ts = ts._exception_blame
+                failing_ts = ts._exception_blame  # type: ignore
 
             for dts in ts._dependents:
                 dts._exception_blame = failing_ts
@@ -3313,7 +3323,7 @@ def get_task_duration(self, ts: TaskState, default: double = -1) -> double:
         if duration >= 0:
             return duration
 
-        s: set = self._unknown_durations.get(ts._prefix._name)
+        s: set = self._unknown_durations.get(ts._prefix._name)  # type: ignore
         if s is None:
             self._unknown_durations[ts._prefix._name] = s = set()
         s.add(ts)
@@ -3331,7 +3341,7 @@ def valid_workers(self, ts: TaskState) -> set:
         *  host_restrictions
         *  resource_restrictions
         """
-        s: set = None
+        s: set = None  # type: ignore
 
         if ts._worker_restrictions:
             s = {w for w in ts._worker_restrictions if w in self._workers_dv}
@@ -3343,7 +3353,7 @@ def valid_workers(self, ts: TaskState) -> set:
             # XXX need HostState?
             sl: list = []
             for h in hr:
-                dh: dict = self._host_info.get(h)
+                dh: dict = self._host_info.get(h)  # type: ignore
                 if dh is not None:
                     sl.append(dh["addresses"])
 
@@ -3356,9 +3366,9 @@ def valid_workers(self, ts: TaskState) -> set:
         if ts._resource_restrictions:
             dw: dict = {}
             for resource, required in ts._resource_restrictions.items():
-                dr: dict = self._resources.get(resource)
+                dr: dict = self._resources.get(resource)  # type: ignore
                 if dr is None:
-                    self._resources[resource] = dr = dict()
+                    self._resources[resource] = dr = {}
 
                 sw: set = set()
                 for w, supplied in dr.items():
@@ -3532,7 +3542,7 @@ class Scheduler(SchedulerState, ServerNode):
     """
 
     default_port = 8786
-    _instances = weakref.WeakSet()
+    _instances: "ClassVar[weakref.WeakSet[Scheduler]]" = weakref.WeakSet()
 
     def __init__(
         self,
@@ -3637,13 +3647,13 @@ def __init__(
 
         # Communication state
         self.loop = loop or IOLoop.current()
-        self.client_comms = dict()
-        self.stream_comms = dict()
+        self.client_comms = {}
+        self.stream_comms = {}
         self._worker_coroutines = []
         self._ipython_kernel = None
 
         # Task state
-        tasks = dict()
+        tasks = {}
         for old_attr, new_attr, wrap in [
             ("priority", "priority", None),
             ("dependencies", "dependencies", _legacy_task_key_set),
@@ -3688,12 +3698,12 @@ def __init__(
         self._last_time = 0
         unrunnable = set()
 
-        self.datasets = dict()
+        self.datasets = {}
 
         # Prefix-keyed containers
 
         # Client state
-        clients = dict()
+        clients = {}
         for old_attr, new_attr, wrap in [
             ("wants_what", "wants_what", _legacy_task_key_set)
         ]:
@@ -3703,7 +3713,7 @@ def __init__(
             setattr(self, old_attr, _StateLegacyMapping(clients, func))
 
         # Worker state
-        workers = sortedcontainers.SortedDict()
+        workers = SortedDict()
         for old_attr, new_attr, wrap in [
             ("nthreads", "nthreads", None),
             ("worker_bytes", "nbytes", None),
@@ -3719,9 +3729,9 @@ def __init__(
                 func = compose(wrap, func)
             setattr(self, old_attr, _StateLegacyMapping(workers, func))
 
-        host_info = dict()
-        resources = dict()
-        aliases = dict()
+        host_info = {}
+        resources = {}
+        aliases = {}
 
         self._task_state_collections = [unrunnable]
 
@@ -3732,7 +3742,6 @@ def __init__(
             aliases,
         ]
 
-        self.plugins = {} if not plugins else {_get_plugin_name(p): p for p in plugins}
         self.transition_log = deque(
             maxlen=dask.config.get("distributed.scheduler.transition-log-length")
         )
@@ -3746,8 +3755,8 @@ def __init__(
         )
         self.event_counts = defaultdict(int)
         self.event_subscriber = defaultdict(set)
-        self.worker_plugins = dict()
-        self.nanny_plugins = dict()
+        self.worker_plugins = {}
+        self.nanny_plugins = {}
 
         worker_handlers = {
             "task-finished": self.handle_task_finished,
@@ -3832,13 +3841,8 @@ def __init__(
         connection_limit = get_fileno_limit() / 2
 
         super().__init__(
+            # Arguments to SchedulerState
             aliases=aliases,
-            handlers=self.handlers,
-            stream_handlers=merge(worker_handlers, client_handlers),
-            io_loop=self.loop,
-            connection_limit=connection_limit,
-            deserialize=False,
-            connection_args=self.connection_args,
             clients=clients,
             workers=workers,
             host_info=host_info,
@@ -3846,6 +3850,14 @@ def __init__(
             tasks=tasks,
             unrunnable=unrunnable,
             validate=validate,
+            plugins=plugins,
+            # Arguments to ServerNode
+            handlers=self.handlers,
+            stream_handlers=merge(worker_handlers, client_handlers),
+            io_loop=self.loop,
+            connection_limit=connection_limit,
+            deserialize=False,
+            connection_args=self.connection_args,
             **kwargs,
         )
 
@@ -4101,7 +4113,7 @@ def heartbeat_worker(
         parent: SchedulerState = cast(SchedulerState, self)
         address = self.coerce_address(address, resolve_address)
         address = normalize_address(address)
-        ws: WorkerState = parent._workers_dv.get(address)
+        ws: WorkerState = parent._workers_dv.get(address)  # type: ignore
         if ws is None:
             return {"status": "missing"}
 
@@ -4172,7 +4184,7 @@ def heartbeat_worker(
             ws._memory_unmanaged_old = size
 
         if host_info:
-            dh: dict = parent._host_info.setdefault(host, {})
+            dh = parent._host_info.setdefault(host, {})
             dh.update(host_info)
 
         if now:
@@ -4250,7 +4262,7 @@ async def add_worker(
 
             dh: dict = parent._host_info.get(host)
             if dh is None:
-                parent._host_info[host] = dh = dict()
+                parent._host_info[host] = dh = {}
 
             dh_addresses: set = dh.get("addresses")
             if dh_addresses is None:
@@ -4296,7 +4308,7 @@ async def add_worker(
             worker_msgs: dict = {}
             if nbytes:
                 assert isinstance(nbytes, dict)
-                already_released_keys = list()
+                already_released_keys = []
                 for key in nbytes:
                     ts: TaskState = parent._tasks.get(key)
                     if ts is not None and ts.state != "released":
@@ -4319,7 +4331,7 @@ async def add_worker(
                         already_released_keys.append(key)
                 if already_released_keys:
                     if address not in worker_msgs:
-                        worker_msgs[address] = list()
+                        worker_msgs[address] = []
                     worker_msgs[address].append(
                         {
                             "op": "free-keys",
@@ -4857,7 +4869,7 @@ async def remove_worker(self, comm=None, address=None, safe=False, close=True):
 
             dh: dict = parent._host_info.get(host)
             if dh is None:
-                parent._host_info[host] = dh = dict()
+                parent._host_info[host] = dh = {}
 
             dh_addresses: set = dh["addresses"]
             dh_addresses.remove(address)
@@ -5405,7 +5417,7 @@ def handle_long_running(self, key=None, worker=None, compute_duration=None):
 
     def handle_worker_status_change(self, status: str, worker: str):
         parent: SchedulerState = cast(SchedulerState, self)
-        ws: WorkerState = parent._workers_dv.get(worker)
+        ws: WorkerState = parent._workers_dv.get(worker)  # type: ignore
         if not ws:
             return
         self.log_event(
@@ -5416,7 +5428,7 @@ def handle_worker_status_change(self, status: str, worker: str):
                 "status": status,
             },
         )
-        ws._status = Status.lookup[status]
+        ws._status = Status.lookup[status]  # type: ignore
 
     async def handle_worker(self, comm=None, worker=None):
         """
@@ -5518,7 +5530,7 @@ def remove_plugin(
                 category=FutureWarning,
             )
             if hasattr(plugin, "name"):
-                name = plugin.name
+                name = plugin.name  # type: ignore
             else:
                 names = [k for k, v in self.plugins.items() if v is plugin]
                 if not names:
@@ -5557,7 +5569,7 @@ async def register_scheduler_plugin(self, comm=None, plugin=None, name=None):
             if inspect.isawaitable(result):
                 await result
 
-        self.add_plugin(plugin=plugin, name=name)
+        self.add_plugin(plugin, name=name)
 
     def worker_send(self, worker, msg):
         """Send message to worker
@@ -5903,14 +5915,14 @@ async def gather_on_worker(
             return set(who_has)
 
         parent: SchedulerState = cast(SchedulerState, self)
-        ws: WorkerState = parent._workers_dv.get(worker_address)
+        ws: WorkerState = parent._workers_dv.get(worker_address)  # type: ignore
 
         if ws is None:
             logger.warning(f"Worker {worker_address} lost during replication")
             return set(who_has)
         elif result["status"] == "OK":
             keys_failed = set()
-            keys_ok = who_has.keys()
+            keys_ok: Set = who_has.keys()
         elif result["status"] == "partial-fail":
             keys_failed = set(result["keys"])
             keys_ok = who_has.keys() - keys_failed
@@ -5921,7 +5933,7 @@ async def gather_on_worker(
             raise ValueError(f"Unexpected message from {worker_address}: {result}")
 
         for key in keys_ok:
-            ts: TaskState = parent._tasks.get(key)
+            ts: TaskState = parent._tasks.get(key)  # type: ignore
             if ts is None or ts._state != "memory":
                 logger.warning(f"Key lost during replication: {key}")
                 continue
@@ -5930,7 +5942,9 @@ async def gather_on_worker(
 
         return keys_failed
 
-    async def delete_worker_data(self, worker_address: str, keys: "list[str]") -> None:
+    async def delete_worker_data(
+        self, worker_address: str, keys: "Collection[str]"
+    ) -> None:
         """Delete data from a worker and update the corresponding worker/task states
 
         Parameters
@@ -5957,12 +5971,12 @@ async def delete_worker_data(self, worker_address: str, keys: "list[str]") -> No
             )
             return
 
-        ws: WorkerState = parent._workers_dv.get(worker_address)
+        ws: WorkerState = parent._workers_dv.get(worker_address)  # type: ignore
         if ws is None:
             return
 
         for key in keys:
-            ts: TaskState = parent._tasks.get(key)
+            ts: TaskState = parent._tasks.get(key)  # type: ignore
             if ts is not None and ws in ts._who_has:
                 assert ts._state == "memory"
                 parent.remove_replica(ts, ws)
@@ -6043,14 +6057,15 @@ async def rebalance(
             All other workers will be ignored. The mean cluster occupancy will be
             calculated only using the whitelisted workers.
         """
-        parent: SchedulerState = self
+        parent: SchedulerState = cast(SchedulerState, self)
 
         with log_errors():
+            wss: "Collection[WorkerState]"
             if workers is not None:
-                workers = [parent._workers_dv[w] for w in workers]
+                wss = [parent._workers_dv[w] for w in workers]
             else:
-                workers = parent._workers_dv.values()
-            if not workers:
+                wss = parent._workers_dv.values()
+            if not wss:
                 return {"status": "OK"}
 
             if keys is not None:
@@ -6066,7 +6081,7 @@ async def rebalance(
                 if missing_data:
                     return {"status": "partial-fail", "keys": missing_data}
 
-            msgs = self._rebalance_find_msgs(keys, workers)
+            msgs = self._rebalance_find_msgs(keys, wss)
             if not msgs:
                 return {"status": "OK"}
 
@@ -6078,7 +6093,7 @@ async def rebalance(
                 return result
 
     def _rebalance_find_msgs(
-        self: SchedulerState,
+        self,
         keys: "Set[Hashable] | None",
         workers: "Iterable[WorkerState]",
     ) -> "list[tuple[WorkerState, WorkerState, TaskState]]":
@@ -6108,7 +6123,7 @@ def _rebalance_find_msgs(
         - recipient worker
         - task to be transferred
         """
-        parent: SchedulerState = self
+        parent: SchedulerState = cast(SchedulerState, self)
         ts: TaskState
         ws: WorkerState
 
@@ -6290,7 +6305,9 @@ async def _rebalance_move_data(
         rec_ws: WorkerState
         ts: TaskState
 
-        to_recipients = defaultdict(lambda: defaultdict(list))
+        to_recipients: "defaultdict[str, defaultdict[str, list[str]]]" = defaultdict(
+            lambda: defaultdict(list)
+        )
         for snd_ws, rec_ws, ts in msgs:
             to_recipients[rec_ws.address][ts._key].append(snd_ws.address)
         failed_keys_by_recipient = dict(
@@ -6449,13 +6466,13 @@ async def replicate(
     def workers_to_close(
         self,
         comm=None,
-        memory_ratio=None,
-        n=None,
-        key=None,
-        minimum=None,
-        target=None,
-        attribute="address",
-    ):
+        memory_ratio: "int | float | None" = None,
+        n: "int | None" = None,
+        key: "Callable[[WorkerState], Hashable] | None" = None,
+        minimum: "int | None" = None,
+        target: "int | None" = None,
+        attribute: str = "address",
+    ) -> "list[str]":
         """
         Find workers that we can close with low cost
 
@@ -6567,9 +6584,9 @@ def _key(group):
 
                 limit -= limit_bytes[group]
 
-                if (n is not None and n_remain - len(groups[group]) >= target) or (
-                    memory_ratio is not None and limit >= memory_ratio * total
-                ):
+                if (
+                    n is not None and n_remain - len(groups[group]) >= cast(int, target)
+                ) or (memory_ratio is not None and limit >= memory_ratio * total):
                     to_close.append(group)
                     n_remain -= len(groups[group])
 
@@ -6735,8 +6752,9 @@ def add_keys(self, comm=None, worker=None, keys=(), stimulus_id=None):
     def update_data(
         self,
         comm=None,
-        who_has=None,
-        nbytes: dict = None,
+        *,
+        who_has: dict,
+        nbytes: dict,
         client=None,
         serializers=None,
     ):
@@ -6755,9 +6773,9 @@ def update_data(
             logger.debug("Update data %s", who_has)
 
             for key, workers in who_has.items():
-                ts: TaskState = parent._tasks.get(key)
+                ts: TaskState = parent._tasks.get(key)  # type: ignore
                 if ts is None:
-                    ts: TaskState = parent.new_task(key, None, "memory")
+                    ts = parent.new_task(key, None, "memory")
                 ts.state = "memory"
                 ts_nbytes = nbytes.get(key, -1)
                 if ts_nbytes >= 0:
@@ -6984,7 +7002,7 @@ def set_metadata(self, comm=None, keys=None, value=None):
             metadata = parent._task_metadata
             for key in keys[:-1]:
                 if key not in metadata or not isinstance(metadata[key], (dict, list)):
-                    metadata[key] = dict()
+                    metadata[key] = {}
                 metadata = metadata[key]
             metadata[keys[-1]] = value
         except Exception:
@@ -7169,7 +7187,7 @@ def add_resources(self, comm=None, worker=None, resources=None):
             ws._used_resources[resource] = 0
             dr: dict = parent._resources.get(resource, None)
             if dr is None:
-                parent._resources[resource] = dr = dict()
+                parent._resources[resource] = dr = {}
             dr[worker] = quantity
         return "OK"
 
@@ -7179,7 +7197,7 @@ def remove_resources(self, worker):
         for resource, quantity in ws._resources.items():
             dr: dict = parent._resources.get(resource, None)
             if dr is None:
-                parent._resources[resource] = dr = dict()
+                parent._resources[resource] = dr = {}
             del dr[worker]
 
     def coerce_address(self, addr, resolve=True):
@@ -7690,16 +7708,18 @@ def adaptive_target(self, comm=None, target_duration=None):
 
 @cfunc
 @exceptval(check=False)
-def _remove_from_processing(state: SchedulerState, ts: TaskState) -> str:
+def _remove_from_processing(
+    state: SchedulerState, ts: TaskState
+) -> str:  # -> str | None
     """
     Remove *ts* from the set of processing tasks.
     """
     ws: WorkerState = ts._processing_on
-    ts._processing_on = None
+    ts._processing_on = None  # type: ignore
     w: str = ws._address
 
     if w not in state._workers_dv:  # may have been removed
-        return None
+        return None  # type: ignore
 
     duration: double = ws._processing.pop(ts)
     if not ws._processing:
@@ -7767,7 +7787,7 @@ def _add_to_memory(
             client_msgs[cs._client_key] = [report_msg]
 
     ts.state = "memory"
-    ts._type = typename
+    ts._type = typename  # type: ignore
     ts._group._types.add(typename)
 
     cs = state._clients["fire-and-forget"]
@@ -7831,7 +7851,7 @@ def _client_releases_keys(
     logger.debug("Client %s releases keys: %s", cs._client_key, keys)
     ts: TaskState
     for key in keys:
-        ts = state._tasks.get(key)
+        ts = state._tasks.get(key)  # type: ignore
         if ts is not None and ts in cs._wants_what:
             cs._wants_what.remove(ts)
             ts._who_wants.remove(cs)
@@ -7890,7 +7910,7 @@ def _task_to_msg(state: SchedulerState, ts: TaskState, duration: double = -1) ->
 
 @cfunc
 @exceptval(check=False)
-def _task_to_report_msg(state: SchedulerState, ts: TaskState) -> dict:
+def _task_to_report_msg(state: SchedulerState, ts: TaskState) -> dict:  # -> dict | None
     if ts._state == "forgotten":
         return {"op": "cancelled-key", "key": ts._key}
     elif ts._state == "memory":
@@ -7904,7 +7924,7 @@ def _task_to_report_msg(state: SchedulerState, ts: TaskState) -> dict:
             "traceback": failing_ts._traceback,
         }
     else:
-        return None
+        return None  # type: ignore
 
 
 @cfunc
@@ -7949,7 +7969,7 @@ def _reevaluate_occupancy_worker(state: SchedulerState, ws: WorkerState):
 @exceptval(check=False)
 def decide_worker(
     ts: TaskState, all_workers, valid_workers: set, objective
-) -> WorkerState:
+) -> WorkerState:  # -> WorkerState | None
     """
     Decide which worker should take task *ts*.
 
@@ -7965,7 +7985,7 @@ def decide_worker(
     of bytes sent between workers.  This is determined by calling the
     *objective* function.
     """
-    ws: WorkerState = None
+    ws: WorkerState = None  # type: ignore
     wws: WorkerState
     dts: TaskState
     deps: set = ts._dependencies
diff --git a/distributed/worker.py b/distributed/worker.py
index 031c58a48f5..ee17e115d4e 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -2489,7 +2489,7 @@ async def gather_dep(
         total_nbytes : int
             Total number of bytes for all the dependencies in to_gather combined
         """
-        cause = None
+        cause: TaskState | None = None
         if self.status not in (Status.running, Status.paused):
             return
 
@@ -2517,6 +2517,8 @@ async def gather_dep(
 
                 if not to_gather_keys:
                     return
+                assert cause
+
                 # Keep namespace clean since this func is long and has many
                 # dep*, *ts* variables
                 del to_gather, dependency_key, dependency_ts
diff --git a/docs/source/develop.rst b/docs/source/develop.rst
index 6ea8c750329..b4956d62195 100644
--- a/docs/source/develop.rst
+++ b/docs/source/develop.rst
@@ -169,9 +169,19 @@ fixture tests test basic interface and resilience.
 You should avoid ``popen`` style tests unless absolutely necessary, such as if
 you need to test the command line interface.
 
-Linting
--------
-distributed uses several code linters (flake8, black, isort, pyupgrade, mypy), which are
-enforced by CI. Developers should run them locally before they submit a PR, through the
-single command ``pre-commit run --all-files``. This makes sure that linter versions and
-options are aligned for all developers.
+Code Formatting
+---------------
+
+Dask.distributed uses several code linters (flake8, black, isort, pyupgrade, mypy),
+which are enforced by CI. Developers should run them locally before they submit a PR,
+through the single command ``pre-commit run --all-files``. This makes sure that linter
+versions and options are aligned for all developers.
+
+Optionally, you may wish to setup the `pre-commit hooks <https://pre-commit.com/>`_ to
+run automatically when you make a git commit. This can be done by running::
+
+   pre-commit install
+
+from the root of the distributed repository. Now the code linters will be run each time
+you commit changes. You can skip these checks with ``git commit --no-verify`` or with
+the short version ``git commit -n``.

From 0959f50ffa1eab8ce6d38620cb07dda2c6dc91f2 Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Wed, 13 Oct 2021 15:07:17 +0200
Subject: [PATCH 1534/1550] Fix regression where unknown tasks were allowed to
 be stolen (#5392)

---
 distributed/scheduler.py        | 149 ++++++++++++++++----------------
 distributed/stealing.py         |   8 +-
 distributed/tests/test_steal.py |  20 ++++-
 3 files changed, 94 insertions(+), 83 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 12add27f2d8..baa7957334c 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -399,10 +399,15 @@ class WorkerState:
     .. attribute:: processing: {TaskState: cost}
 
        A dictionary of tasks that have been submitted to this worker.
-       Each task state is asssociated with the expected cost in seconds
+       Each task state is associated with the expected cost in seconds
        of running that task, summing both the task's expected computation
        time and the expected communication time of its result.
 
+       If a task is already executing on the worker and the excecution time is
+       twice the learned average TaskGroup duration, this will be set to twice
+       the current executing time. If the task is unknown, the default task
+       duration is used instead of the TaskGroup average.
+
        Multiple tasks may be submitted to a worker in advance and the worker
        will run them eventually, depending on its execution resources
        (but see :doc:`work-stealing`).
@@ -900,6 +905,18 @@ def name(self) -> str:
     def all_durations(self) -> "defaultdict[str, float]":
         return self._all_durations
 
+    @ccall
+    @exceptval(check=False)
+    def add_duration(self, action: str, start: double, stop: double):
+        duration = stop - start
+        self._all_durations[action] += duration
+        if action == "compute":
+            old = self._duration_average
+            if old < 0:
+                self._duration_average = duration
+            else:
+                self._duration_average = 0.5 * duration + 0.5 * old
+
     @property
     def duration_average(self) -> double:
         return self._duration_average
@@ -1062,6 +1079,18 @@ def nbytes_total(self):
     def duration(self) -> double:
         return self._duration
 
+    @ccall
+    @exceptval(check=False)
+    def add_duration(self, action: str, start: double, stop: double):
+        duration = stop - start
+        self._all_durations[action] += duration
+        if action == "compute":
+            if self._stop < stop:
+                self._stop = stop
+            self._start = self._start or start
+        self._duration += duration
+        self._prefix.add_duration(action, start, stop)
+
     @property
     def types(self) -> set:
         return self._types
@@ -2568,6 +2597,8 @@ def set_duration_estimate(self, ts: TaskState, ws: WorkerState) -> double:
         If a task takes longer than twice the current average duration we
         estimate the task duration to be 2x current-runtime, otherwise we set it
         to be the average duration.
+
+        See also ``_remove_from_processing``
         """
         exec_time: double = ws._executing.get(ts, 0)
         duration: double = self.get_task_duration(ts)
@@ -2577,7 +2608,11 @@ def set_duration_estimate(self, ts: TaskState, ws: WorkerState) -> double:
         else:
             comm: double = self.get_comm_cost(ts, ws)
             total_duration = duration + comm
+        old = ws._processing.get(ts, 0)
         ws._processing[ts] = total_duration
+        self._total_occupancy += total_duration - old
+        ws._occupancy += total_duration - old
+
         return total_duration
 
     def transition_waiting_processing(self, key):
@@ -2602,10 +2637,8 @@ def transition_waiting_processing(self, key):
                 return recommendations, client_msgs, worker_msgs
             worker = ws._address
 
-            duration_estimate = self.set_duration_estimate(ts, ws)
+            self.set_duration_estimate(ts, ws)
             ts._processing_on = ws
-            ws._occupancy += duration_estimate
-            self._total_occupancy += duration_estimate
             ts.state = "processing"
             self.consume_resources(ts, ws)
             self.check_idle_saturated(ws)
@@ -2684,7 +2717,6 @@ def transition_processing_memory(
         worker_msgs: dict = {}
         try:
             ts: TaskState = self._tasks[key]
-            tg: TaskGroup = ts._group
 
             assert worker
             assert isinstance(worker, str)
@@ -2719,57 +2751,26 @@ def transition_processing_memory(
                     }
                 ]
 
-            has_compute_startstop: bool = False
-            compute_start: double
-            compute_stop: double
+            #############################
+            # Update Timing Information #
+            #############################
             if startstops:
                 startstop: dict
                 for startstop in startstops:
-                    stop = startstop["stop"]
-                    start = startstop["start"]
-                    action = startstop["action"]
-                    if not has_compute_startstop and action == "compute":
-                        compute_start = start
-                        compute_stop = stop
-                        has_compute_startstop = True
-
-                    # record timings of all actions -- a cheaper way of
-                    # getting timing info compared with get_task_stream()
-                    ts._prefix._all_durations[action] += stop - start
-                    tg._all_durations[action] += stop - start
+                    ts._group.add_duration(
+                        stop=startstop["stop"],
+                        start=startstop["start"],
+                        action=startstop["action"],
+                    )
 
-            #############################
-            # Update Timing Information #
-            #############################
-            if has_compute_startstop and ws._processing.get(ts, True):
-                # Update average task duration for worker
-                old_duration: double = ts._prefix._duration_average
-                new_duration: double = compute_stop - compute_start
-                avg_duration: double
-                if old_duration < 0:
-                    avg_duration = new_duration
-                else:
-                    avg_duration = 0.5 * old_duration + 0.5 * new_duration
-
-                ts._prefix._duration_average = avg_duration
-                tg._duration += new_duration
-                tg._start = tg._start or compute_start
-                if tg._stop < compute_stop:
-                    tg._stop = compute_stop
-
-                s: set = self._unknown_durations.pop(ts._prefix._name, None)
-                tts: TaskState
-                if s:
-                    for tts in s:
-                        if tts._processing_on is not None:
-                            wws = tts._processing_on
-                            comm: double = self.get_comm_cost(tts, wws)
-                            old: double = wws._processing[tts]
-                            new: double = avg_duration + comm
-                            diff: double = new - old
-                            wws._processing[tts] = new
-                            wws._occupancy += diff
-                            self._total_occupancy += diff
+            s: set = self._unknown_durations.pop(ts._prefix._name, set())
+            tts: TaskState
+            for tts in s:
+                if tts._processing_on:
+                    self.set_duration_estimate(tts, tts._processing_on)
+                    steal = self.extensions.get("stealing")
+                    if steal:
+                        steal.put_key_in_stealable(tts)
 
             ############################
             # Update State Information #
@@ -3314,10 +3315,14 @@ def get_comm_cost(self, ts: TaskState, ws: WorkerState) -> double:
         return nbytes / self._bandwidth
 
     @ccall
-    def get_task_duration(self, ts: TaskState, default: double = -1) -> double:
-        """
-        Get the estimated computation cost of the given task
-        (not including any communication cost).
+    def get_task_duration(self, ts: TaskState) -> double:
+        """Get the estimated computation cost of the given task (not including
+        any communication cost).
+
+        If no data has been observed, value of
+        `distributed.scheduler.default-task-durations` are used. If none is set
+        for this task, `distributed.scheduler.unknown-task-duration` is used
+        instead.
         """
         duration: double = ts._prefix._duration_average
         if duration >= 0:
@@ -3327,7 +3332,7 @@ def get_task_duration(self, ts: TaskState, default: double = -1) -> double:
         if s is None:
             self._unknown_durations[ts._prefix._name] = s = set()
         s.add(ts)
-        return default if default >= 0 else self.UNKNOWN_TASK_DURATION
+        return self.UNKNOWN_TASK_DURATION
 
     @ccall
     @exceptval(check=False)
@@ -7586,7 +7591,6 @@ def reevaluate_occupancy(self, worker_index: Py_ssize_t = 0):
         try:
             if self.status == Status.closed:
                 return
-
             last = time()
             next_time = timedelta(seconds=0.1)
 
@@ -7713,6 +7717,8 @@ def _remove_from_processing(
 ) -> str:  # -> str | None
     """
     Remove *ts* from the set of processing tasks.
+
+    See also ``Scheduler.set_duration_estimate``
     """
     ws: WorkerState = ts._processing_on
     ts._processing_on = None  # type: ignore
@@ -7870,6 +7876,7 @@ def _task_to_msg(state: SchedulerState, ts: TaskState, duration: double = -1) ->
     ws: WorkerState
     dts: TaskState
 
+    # FIXME: The duration attribute is not used on worker. We could safe ourselves the time to compute and submit this
     if duration < 0:
         duration = state.get_task_duration(ts)
 
@@ -7942,27 +7949,19 @@ def _task_to_client_msgs(state: SchedulerState, ts: TaskState) -> dict:
 @exceptval(check=False)
 def _reevaluate_occupancy_worker(state: SchedulerState, ws: WorkerState):
     """See reevaluate_occupancy"""
-    old: double = ws._occupancy
-    new: double = 0
-    diff: double
     ts: TaskState
-    est: double
+    old = ws._occupancy
     for ts in ws._processing:
-        est = state.set_duration_estimate(ts, ws)
-        new += est
+        state.set_duration_estimate(ts, ws)
 
-    ws._occupancy = new
-    diff = new - old
-    state._total_occupancy += diff
     state.check_idle_saturated(ws)
-
-    # significant increase in duration
-    if new > old * 1.3:
-        steal = state._extensions.get("stealing")
-        if steal is not None:
-            for ts in ws._processing:
-                steal.remove_key_from_stealable(ts)
-                steal.put_key_in_stealable(ts)
+    steal = state.extensions.get("stealing")
+    if not steal:
+        return
+    if ws._occupancy > old * 1.3 or old > ws._occupancy * 1.3:
+        for ts in ws._processing:
+            steal.remove_key_from_stealable(ts)
+            steal.put_key_in_stealable(ts)
 
 
 @cfunc
diff --git a/distributed/stealing.py b/distributed/stealing.py
index 0297691f02f..bacd62715ed 100644
--- a/distributed/stealing.py
+++ b/distributed/stealing.py
@@ -114,13 +114,13 @@ def steal_time_ratio(self, ts):
         For example a result of zero implies a task without dependencies.
         level: The location within a stealable list to place this value
         """
-        if not ts.dependencies:  # no dependencies fast path
-            return 0, 0
-
         split = ts.prefix.name
-        if split in fast_tasks:
+        if split in fast_tasks or split in self.scheduler.unknown_durations:
             return None, None
 
+        if not ts.dependencies:  # no dependencies fast path
+            return 0, 0
+
         ws = ts.processing_on
         compute_time = ws.processing[ts]
         if compute_time < 0.005:  # 5ms, just give up
diff --git a/distributed/tests/test_steal.py b/distributed/tests/test_steal.py
index 3517805451c..70f51b7e75c 100644
--- a/distributed/tests/test_steal.py
+++ b/distributed/tests/test_steal.py
@@ -110,7 +110,6 @@ async def test_worksteal_many_thieves(c, s, *workers):
     assert sum(map(len, s.has_what.values())) < 150
 
 
-@pytest.mark.flaky(reruns=10, reruns_delay=5, reason="GH#3574")
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 2)
 async def test_dont_steal_unknown_functions(c, s, a, b):
     futures = c.map(inc, range(100), workers=a.address, allow_other_workers=True)
@@ -118,12 +117,17 @@ async def test_dont_steal_unknown_functions(c, s, a, b):
     assert len(a.data) >= 95, [len(a.data), len(b.data)]
 
 
-@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 2)
+@gen_cluster(
+    client=True,
+    nthreads=[("127.0.0.1", 1)] * 2,
+    config={"distributed.scheduler.work-stealing-interval": "10ms"},
+)
 async def test_eventually_steal_unknown_functions(c, s, a, b):
     futures = c.map(
         slowinc, range(10), delay=0.1, workers=a.address, allow_other_workers=True
     )
     await wait(futures)
+    assert not s.unknown_durations
     assert len(a.data) >= 3, [len(a.data), len(b.data)]
     assert len(b.data) >= 3, [len(a.data), len(b.data)]
 
@@ -597,11 +601,15 @@ async def test(*args, **kwargs):
 @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 2, Worker=Nanny, timeout=60)
 async def test_restart(c, s, a, b):
     futures = c.map(
-        slowinc, range(100), delay=0.1, workers=a.address, allow_other_workers=True
+        slowinc, range(100), delay=0.01, workers=a.address, allow_other_workers=True
     )
     while not s.processing[b.worker_address]:
         await asyncio.sleep(0.01)
 
+    # Unknown tasks are never stolen therefore wait for a measurement
+    while not any(s.tasks[f.key].state == "memory" for f in futures):
+        await asyncio.sleep(0.01)
+
     steal = s.extensions["stealing"]
     assert any(st for st in steal.stealable_all)
     assert any(x for L in steal.stealable.values() for x in L)
@@ -821,9 +829,13 @@ async def test_balance_with_longer_task(c, s, a, b):
         slowinc, 1, delay=5, workers=[a.address], priority=1
     )  # a surprisingly long task
     z = c.submit(
-        inc, x, workers=[a.address], allow_other_workers=True, priority=0
+        slowadd, x, 1, workers=[a.address], allow_other_workers=True, priority=0
     )  # a task after y, suggesting a, but open to b
+
+    # Allow task to be learned, otherwise it will not be stolen
+    _ = c.submit(slowadd, x, 2, workers=[b.address])
     await z
+    assert not y.done()
     assert z.key in b.data
 
 

From a1b67b84226d3053517dffcb6f0c8fe8821eb8fb Mon Sep 17 00:00:00 2001
From: Jacob Tomlinson <jacobtomlinson@users.noreply.github.com>
Date: Wed, 13 Oct 2021 18:25:33 +0100
Subject: [PATCH 1535/1550] Add scroll to dashboard dropdown (#5418)

---
 distributed/http/static/css/base.css | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/distributed/http/static/css/base.css b/distributed/http/static/css/base.css
index 78607c7aefe..13936a98d3b 100644
--- a/distributed/http/static/css/base.css
+++ b/distributed/http/static/css/base.css
@@ -124,6 +124,8 @@ body {
   min-width: 160px;
   box-shadow: 0px 8px 16px 0px rgba(0,0,0,0.2);
   z-index: 2;
+  max-height: 90%;
+  overflow-y: scroll;
 }
 
 .dropdown-content ul li {

From 36b0ee8b8e47430f0746fced79319fbdbf7846f9 Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Thu, 14 Oct 2021 04:12:20 +0200
Subject: [PATCH 1536/1550] Resolve work stealing deadlock caused by race in
 `move_task_confirm` (#5379)

Co-authored-by: crusaderky <crusaderky@gmail.com>
---
 distributed/scheduler.py            |  11 +-
 distributed/stealing.py             | 159 ++++++++++++++-----------
 distributed/tests/test_scheduler.py |  21 ++--
 distributed/tests/test_steal.py     | 172 +++++++++++++++++++++++++++-
 distributed/tests/test_worker.py    |   2 +-
 distributed/worker.py               |  11 +-
 6 files changed, 289 insertions(+), 87 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index baa7957334c..481220a97e5 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -2765,10 +2765,10 @@ def transition_processing_memory(
 
             s: set = self._unknown_durations.pop(ts._prefix._name, set())
             tts: TaskState
+            steal = self.extensions.get("stealing")
             for tts in s:
                 if tts._processing_on:
                     self.set_duration_estimate(tts, tts._processing_on)
-                    steal = self.extensions.get("stealing")
                     if steal:
                         steal.put_key_in_stealable(tts)
 
@@ -7029,8 +7029,12 @@ def get_metadata(self, comm=None, keys=None, default=no_default):
                 raise
 
     def set_restrictions(self, comm=None, worker=None):
+        ts: TaskState
         for key, restrictions in worker.items():
-            self.tasks[key]._worker_restrictions = set(restrictions)
+            ts = self.tasks[key]
+            if isinstance(restrictions, str):
+                restrictions = {restrictions}
+            ts._worker_restrictions = set(restrictions)
 
     def get_task_status(self, comm=None, keys=None):
         parent: SchedulerState = cast(SchedulerState, self)
@@ -7960,8 +7964,7 @@ def _reevaluate_occupancy_worker(state: SchedulerState, ws: WorkerState):
         return
     if ws._occupancy > old * 1.3 or old > ws._occupancy * 1.3:
         for ts in ws._processing:
-            steal.remove_key_from_stealable(ts)
-            steal.put_key_in_stealable(ts)
+            steal.recalculate_cost(ts)
 
 
 @cfunc
diff --git a/distributed/stealing.py b/distributed/stealing.py
index bacd62715ed..69f176d1117 100644
--- a/distributed/stealing.py
+++ b/distributed/stealing.py
@@ -21,6 +21,24 @@
 
 LOG_PDB = dask.config.get("distributed.admin.pdb-on-err")
 
+_WORKER_STATE_CONFIRM = {
+    "ready",
+    "constrained",
+    "waiting",
+}
+
+_WORKER_STATE_REJECT = {
+    "memory",
+    "executing",
+    "long-running",
+    "cancelled",
+    "resumed",
+}
+_WORKER_STATE_UNDEFINED = {
+    "released",
+    None,
+}
+
 
 class WorkStealing(SchedulerPlugin):
     def __init__(self, scheduler):
@@ -78,8 +96,19 @@ def transition(
         elif start == "processing":
             ts = self.scheduler.tasks[key]
             self.remove_key_from_stealable(ts)
-            if finish != "memory":
-                self.in_flight.pop(ts, None)
+            d = self.in_flight.pop(ts, None)
+            if d:
+                thief = d["thief"]
+                victim = d["victim"]
+                self.in_flight_occupancy[thief] -= d["thief_duration"]
+                self.in_flight_occupancy[victim] += d["victim_duration"]
+                if not self.in_flight:
+                    self.in_flight_occupancy.clear()
+
+    def recalculate_cost(self, ts):
+        if ts not in self.in_flight:
+            self.remove_key_from_stealable(ts)
+            self.put_key_in_stealable(ts)
 
     def put_key_in_stealable(self, ts):
         cost_multiplier, level = self.steal_time_ratio(ts)
@@ -138,13 +167,11 @@ def steal_time_ratio(self, ts):
 
         return cost_multiplier, level
 
-    def move_task_request(self, ts, victim, thief):
+    def move_task_request(self, ts, victim, thief) -> str:
         try:
-            if self.scheduler.validate:
-                if victim is not ts.processing_on and LOG_PDB:
-                    import pdb
-
-                    pdb.set_trace()
+            if ts in self.in_flight:
+                return "in-flight"
+            stimulus_id = f"steal-{time()}"
 
             key = ts.key
             self.remove_key_from_stealable(ts)
@@ -164,20 +191,22 @@ def move_task_request(self, ts, victim, thief):
             ) + self.scheduler.get_comm_cost(ts, thief)
 
             self.scheduler.stream_comms[victim.address].send(
-                {"op": "steal-request", "key": key}
+                {"op": "steal-request", "key": key, "stimulus_id": stimulus_id}
             )
-
             self.in_flight[ts] = {
-                "victim": victim,
+                "victim": victim,  # guaranteed to be processing_on
                 "thief": thief,
                 "victim_duration": victim_duration,
                 "thief_duration": thief_duration,
+                "stimulus_id": stimulus_id,
             }
 
             self.in_flight_occupancy[victim] -= victim_duration
             self.in_flight_occupancy[thief] += thief_duration
+            return stimulus_id
         except CommClosedError:
             logger.info("Worker comm %r closed while stealing: %r", victim, ts)
+            return "comm-closed"
         except Exception as e:
             logger.exception(e)
             if LOG_PDB:
@@ -186,30 +215,41 @@ def move_task_request(self, ts, victim, thief):
                 pdb.set_trace()
             raise
 
-    async def move_task_confirm(self, key=None, worker=None, state=None):
+    async def move_task_confirm(self, *, key, state, stimulus_id, worker=None):
         try:
-            try:
-                ts = self.scheduler.tasks[key]
-            except KeyError:
-                logger.debug("Key released between request and confirm: %s", key)
-                return
-            try:
-                d = self.in_flight.pop(ts)
-            except KeyError:
+            ts = self.scheduler.tasks[key]
+        except KeyError:
+            logger.debug("Key released between request and confirm: %s", key)
+            return
+        try:
+            d = self.in_flight.pop(ts)
+            if d["stimulus_id"] != stimulus_id:
+                self.log(("stale-response", key, state, worker, stimulus_id))
+                self.in_flight[ts] = d
                 return
-            thief = d["thief"]
-            victim = d["victim"]
-            logger.debug(
-                "Confirm move %s, %s -> %s.  State: %s", key, victim, thief, state
-            )
+        except KeyError:
+            self.log(("already-aborted", key, state, stimulus_id))
+            return
 
-            self.in_flight_occupancy[thief] -= d["thief_duration"]
-            self.in_flight_occupancy[victim] += d["victim_duration"]
+        thief = d["thief"]
+        victim = d["victim"]
 
-            if not self.in_flight:
-                self.in_flight_occupancy = defaultdict(lambda: 0)
+        logger.debug("Confirm move %s, %s -> %s.  State: %s", key, victim, thief, state)
 
-            if ts.state != "processing" or ts.processing_on is not victim:
+        self.in_flight_occupancy[thief] -= d["thief_duration"]
+        self.in_flight_occupancy[victim] += d["victim_duration"]
+
+        if not self.in_flight:
+            self.in_flight_occupancy.clear()
+
+        if self.scheduler.validate:
+            assert ts.processing_on == victim
+
+        try:
+            _log_msg = [key, state, victim.address, thief.address, stimulus_id]
+
+            if ts.state != "processing":
+                self.log(("not-processing", *_log_msg))
                 old_thief = thief.occupancy
                 new_thief = sum(thief.processing.values())
                 old_victim = victim.occupancy
@@ -219,32 +259,24 @@ async def move_task_confirm(self, key=None, worker=None, state=None):
                 self.scheduler.total_occupancy += (
                     new_thief - old_thief + new_victim - old_victim
                 )
-                return
-
-            # One of the pair has left, punt and reschedule
-            if (
-                thief.address not in self.scheduler.workers
-                or victim.address not in self.scheduler.workers
+            elif (
+                state in _WORKER_STATE_UNDEFINED
+                or state in _WORKER_STATE_CONFIRM
+                and thief.address not in self.scheduler.workers
             ):
+                self.log(
+                    (
+                        "reschedule",
+                        thief.address not in self.scheduler.workers,
+                        *_log_msg,
+                    )
+                )
                 self.scheduler.reschedule(key)
-                return
-
-            # Victim had already started execution, reverse stealing
-            if state in (
-                "memory",
-                "executing",
-                "long-running",
-                "released",
-                "cancelled",
-                "resumed",
-                None,
-            ):
-                self.log(("already-computing", key, victim.address, thief.address))
-                self.scheduler.check_idle_saturated(thief)
-                self.scheduler.check_idle_saturated(victim)
-
+            # Victim had already started execution
+            elif state in _WORKER_STATE_REJECT:
+                self.log(("already-computing", *_log_msg))
             # Victim was waiting, has given up task, enact steal
-            elif state in ("waiting", "ready", "constrained"):
+            elif state in _WORKER_STATE_CONFIRM:
                 self.remove_key_from_stealable(ts)
                 ts.processing_on = thief
                 duration = victim.processing.pop(ts)
@@ -258,11 +290,8 @@ async def move_task_confirm(self, key=None, worker=None, state=None):
                 self.scheduler.total_occupancy += d["thief_duration"]
                 self.put_key_in_stealable(ts)
 
-                try:
-                    self.scheduler.send_task_to_worker(thief.address, ts)
-                except CommClosedError:
-                    await self.scheduler.remove_worker(thief.address)
-                self.log(("confirm", key, victim.address, thief.address))
+                self.scheduler.send_task_to_worker(thief.address, ts)
+                self.log(("confirm", *_log_msg))
             else:
                 raise ValueError(f"Unexpected task state: {state}")
         except Exception as e:
@@ -273,14 +302,8 @@ async def move_task_confirm(self, key=None, worker=None, state=None):
                 pdb.set_trace()
             raise
         finally:
-            try:
-                self.scheduler.check_idle_saturated(thief)
-            except Exception:
-                pass
-            try:
-                self.scheduler.check_idle_saturated(victim)
-            except Exception:
-                pass
+            self.scheduler.check_idle_saturated(thief)
+            self.scheduler.check_idle_saturated(victim)
 
     def balance(self):
         s = self.scheduler
@@ -413,9 +436,9 @@ def restart(self, scheduler):
         self.key_stealable.clear()
 
     def story(self, *keys):
-        keys = set(keys)
+        keys = {key.key if not isinstance(key, str) else key for key in keys}
         out = []
-        for _, L in self.scheduler.get_event("stealing"):
+        for _, L in self.scheduler.get_events(topic="stealing"):
             if not isinstance(L, list):
                 L = [L]
             for t in L:
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index d5cde1b5491..7d26e3e7904 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -1297,16 +1297,6 @@ async def test_non_existent_worker(c, s):
         assert all(ts.state == "no-worker" for ts in s.tasks.values())
 
 
-@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 3)
-async def test_correct_bad_time_estimate(c, s, *workers):
-    future = c.submit(slowinc, 1, delay=0)
-    await wait(future)
-    futures = [c.submit(slowinc, future, delay=0.1, pure=False) for i in range(20)]
-    await asyncio.sleep(0.5)
-    await wait(futures)
-    assert all(w.data for w in workers), [sorted(w.data) for w in workers]
-
-
 @pytest.mark.parametrize(
     "host", ["tcp://0.0.0.0", "tcp://127.0.0.1", "tcp://127.0.0.1:38275"]
 )
@@ -3191,3 +3181,14 @@ async def test_worker_reconnect_task_memory_with_resources(c, s, a):
         assert ("no-worker", "memory") in {
             (start, finish) for (_, start, finish, _, _) in s.transition_log
         }
+
+
+@gen_cluster(client=True, nthreads=[("", 1)] * 2)
+async def test_set_restrictions(c, s, a, b):
+
+    f = c.submit(inc, 1, workers=[b.address])
+    await f
+    s.set_restrictions(worker={f.key: a.address})
+    assert s.tasks[f.key].worker_restrictions == {a.address}
+    s.reschedule(f)
+    await f
diff --git a/distributed/tests/test_steal.py b/distributed/tests/test_steal.py
index 70f51b7e75c..4b4c4448ec8 100644
--- a/distributed/tests/test_steal.py
+++ b/distributed/tests/test_steal.py
@@ -701,7 +701,7 @@ async def test_dont_steal_already_released(c, s, a, b):
     while key in a.tasks and a.tasks[key].state != "released":
         await asyncio.sleep(0.05)
 
-    a.handle_steal_request(key)
+    a.handle_steal_request(key=key, stimulus_id="test")
     assert len(a.batched_stream.buffer) == 1
     msg = a.batched_stream.buffer[0]
     assert msg["op"] == "steal-response"
@@ -872,3 +872,173 @@ async def test_blacklist_shuffle_split(c, s, a, b):
                     assert "split" not in ts.prefix.name
         await asyncio.sleep(0.001)
     await res
+
+
+@gen_cluster(
+    client=True,
+    nthreads=[("", 1)] * 3,
+    config={
+        "distributed.scheduler.work-stealing-interval": 1_000_000,
+    },
+)
+async def test_steal_concurrent_simple(c, s, *workers):
+    steal = s.extensions["stealing"]
+    w0 = workers[0]
+    w1 = workers[1]
+    w2 = workers[2]
+    futs1 = c.map(
+        slowinc,
+        range(10),
+        key=[f"f1-{ix}" for ix in range(10)],
+        workers=[w0.address],
+    )
+
+    while not w0.tasks:
+        await asyncio.sleep(0.1)
+
+    # ready is a heap but we don't need last, just not the next
+    _, victim_key = w0.ready[-1]
+
+    ws0 = s.workers[w0.address]
+    ws1 = s.workers[w1.address]
+    ws2 = s.workers[w2.address]
+    victim_ts = s.tasks[victim_key]
+    steal.move_task_request(victim_ts, ws0, ws1)
+    steal.move_task_request(victim_ts, ws0, ws2)
+
+    await c.gather(futs1)
+
+    # First wins
+    assert ws1.has_what
+    assert not ws2.has_what
+
+
+@gen_cluster(
+    client=True,
+    config={
+        "distributed.scheduler.work-stealing-interval": 1_000_000,
+    },
+)
+async def test_steal_reschedule_reset_in_flight_occupancy(c, s, *workers):
+    # https://github.com/dask/distributed/issues/5370
+    steal = s.extensions["stealing"]
+    w0 = workers[0]
+    futs1 = c.map(
+        slowinc,
+        range(10),
+        key=[f"f1-{ix}" for ix in range(10)],
+    )
+    while not w0.tasks:
+        await asyncio.sleep(0.01)
+
+    # ready is a heap but we don't need last, just not the next
+    _, victim_key = w0.ready[-1]
+
+    victim_ts = s.tasks[victim_key]
+
+    wsA = victim_ts.processing_on
+    other_workers = [ws for ws in s.workers.values() if ws != wsA]
+    wsB = other_workers[0]
+
+    steal.move_task_request(victim_ts, wsA, wsB)
+
+    s.reschedule(victim_key)
+    await c.gather(futs1)
+
+    del futs1
+
+    assert all(v == 0 for v in steal.in_flight_occupancy.values())
+
+
+@gen_cluster(
+    client=True,
+    config={
+        "distributed.scheduler.work-stealing-interval": 10,
+    },
+)
+async def test_get_story(c, s, *workers):
+    steal = s.extensions["stealing"]
+    futs = c.map(
+        slowinc, range(100), workers=[workers[0].address], allow_other_workers=True
+    )
+    collect = c.submit(sum, futs)
+    await collect
+    key = next(iter(workers[1].tasks))
+    ts = s.tasks[key]
+    msgs = steal.story(key)
+    msgs_ts = steal.story(ts)
+    assert msgs
+    assert msgs == msgs_ts
+    assert all(isinstance(m, tuple) for m in msgs)
+
+
+@gen_cluster(
+    client=True,
+    nthreads=[("", 1)] * 3,
+    config={
+        "distributed.scheduler.work-stealing-interval": 1_000_000,
+    },
+)
+async def test_reschedule_concurrent_requests_deadlock(c, s, *workers):
+    # https://github.com/dask/distributed/issues/5370
+    steal = s.extensions["stealing"]
+    w0 = workers[0]
+    futs1 = c.map(
+        slowinc,
+        range(10),
+        key=[f"f1-{ix}" for ix in range(10)],
+    )
+    while not w0.active_keys:
+        await asyncio.sleep(0.01)
+
+    # ready is a heap but we don't need last, just not the next
+    victim_key = list(w0.active_keys)[0]
+
+    victim_ts = s.tasks[victim_key]
+
+    wsA = victim_ts.processing_on
+    other_workers = [ws for ws in s.workers.values() if ws != wsA]
+    wsB = other_workers[0]
+    wsC = other_workers[1]
+
+    steal.move_task_request(victim_ts, wsA, wsB)
+
+    s.set_restrictions(worker={victim_key: [wsB.address]})
+    s.reschedule(victim_key)
+    assert wsB == victim_ts.processing_on
+    # move_task_request is not responsible for respecting worker restrictions
+    steal.move_task_request(victim_ts, wsB, wsC)
+    await c.gather(futs1)
+
+    # If this turns out to be overly flaky, the following may be relaxed or
+    # removed. The point of this test is to not deadlock but verifying expected
+    # state is still a nice thing
+
+    # Either the last request goes through or both have been rejected since the
+    # computation was already done by the time the request comes in. This is
+    # unfortunately not stable even if we increase the compute time
+    if victim_ts.who_has != {wsC}:
+        msgs = steal.story(victim_ts)
+        assert len(msgs) == 2
+        assert all(msg[0] == "already-aborted" for msg in msgs), msgs
+
+
+@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)] * 3)
+async def test_correct_bad_time_estimate(c, s, *workers):
+    """Initial time estimation causes the task to not be considered for
+    stealing. Following occupancy readjustments will re-enlist the keys since
+    the duration estimate is now significant.
+
+    This is done during reevaluate occupancy
+    """
+    steal = s.extensions["stealing"]
+    future = c.submit(slowinc, 1, delay=0)
+    await wait(future)
+    futures = [c.submit(slowinc, future, delay=0.1, pure=False) for i in range(20)]
+    while not any(f.key in s.tasks for f in futures):
+        await asyncio.sleep(0.001)
+    assert not any(s.tasks[f.key] in steal.key_stealable for f in futures)
+    await asyncio.sleep(0.5)
+    assert any(s.tasks[f.key] in steal.key_stealable for f in futures)
+    await wait(futures)
+    assert all(w.data for w in workers), [sorted(w.data) for w in workers]
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 7a3126e9d1f..a92700b053c 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -2647,7 +2647,7 @@ def __call__(self, *args, **kwargs):
             await asyncio.sleep(0)
 
         ts = s.tasks[fut.key]
-        a.handle_steal_request(fut.key)
+        a.handle_steal_request(fut.key, stimulus_id="test")
         stealing_ext.scheduler.send_task_to_worker(b.address, ts)
 
         fut2 = c.submit(inc, fut, workers=[a.address])
diff --git a/distributed/worker.py b/distributed/worker.py
index ee17e115d4e..88e82d5915a 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -2724,21 +2724,26 @@ def update_who_has(self, who_has, *, stimulus_id):
                 pdb.set_trace()
             raise
 
-    def handle_steal_request(self, key):
+    def handle_steal_request(self, key, stimulus_id):
         # There may be a race condition between stealing and releasing a task.
         # In this case the self.tasks is already cleared. The `None` will be
         # registered as `already-computing` on the other end
         ts = self.tasks.get(key)
         state = ts.state if ts is not None else None
 
-        response = {"op": "steal-response", "key": key, "state": state}
+        response = {
+            "op": "steal-response",
+            "key": key,
+            "state": state,
+            "stimulus_id": stimulus_id,
+        }
         self.batched_stream.send(response)
 
         if state in {"ready", "waiting", "constrained"}:
             # If task is marked as "constrained" we haven't yet assigned it an
             # `available_resources` to run on, that happens in
             # `transition_constrained_executing`
-            self.transition(ts, "forgotten", stimulus_id=f"steal-request-{time()}")
+            self.transition(ts, "forgotten", stimulus_id=stimulus_id)
 
     def release_key(
         self,

From 842cc758c780afbd8914c6391905a90f10d26945 Mon Sep 17 00:00:00 2001
From: Peter Andreas Entschev <peter@entschev.com>
Date: Thu, 14 Oct 2021 13:00:08 +0200
Subject: [PATCH 1537/1550] Fixes async warnings in UCX tests (#5396)

Fixes warnings raised in UCX tests
---
 distributed/comm/tests/test_ucx.py | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/distributed/comm/tests/test_ucx.py b/distributed/comm/tests/test_ucx.py
index 2f2a2b04768..086afc0721e 100644
--- a/distributed/comm/tests/test_ucx.py
+++ b/distributed/comm/tests/test_ucx.py
@@ -19,6 +19,24 @@
     HOST = "127.0.0.1"
 
 
+def handle_exception(loop, context):
+    msg = context.get("exception", context["message"])
+    print(msg)
+
+
+# Let's make sure that UCX gets time to cancel
+# progress tasks before closing the event loop.
+@pytest.fixture()
+def event_loop(scope="function"):
+    loop = asyncio.new_event_loop()
+    loop.set_exception_handler(handle_exception)
+    ucp.reset()
+    yield loop
+    ucp.reset()
+    loop.run_until_complete(asyncio.sleep(0))
+    loop.close()
+
+
 def test_registered():
     assert "ucx" in backends
     backend = get_backend("ucx")
@@ -123,6 +141,8 @@ async def client_communicate(key, delay=0):
         await asyncio.gather(*futures)
         assert set(l) == {1234} | set(range(N))
 
+        listener.stop()
+
     asyncio.run(f())
 
 

From afce4be8e05fb180e50a9d9e38465f1a82295e1b Mon Sep 17 00:00:00 2001
From: Jim Crist-Harif <jcrist@users.noreply.github.com>
Date: Thu, 14 Oct 2021 16:21:56 -0500
Subject: [PATCH 1538/1550] Pass `host` through `LocalCluster` to workers
 (#5427)

Previously the `host` parameter to `LocalCluster` would only be
forwarded to `Scheduler` instances and not `Worker`/`Nanny` instances,
leading to workers listening on non-localhost in some configurations.
This fixes that and adds a test.

Co-authored-by: James Bourbeau <jrbourbeau@gmail.com>
---
 distributed/deploy/local.py            |  1 +
 distributed/deploy/tests/test_local.py | 18 ++++++++++++++++++
 2 files changed, 19 insertions(+)

diff --git a/distributed/deploy/local.py b/distributed/deploy/local.py
index 91fbb14cd2c..e9ce26803e9 100644
--- a/distributed/deploy/local.py
+++ b/distributed/deploy/local.py
@@ -198,6 +198,7 @@ def __init__(
 
         worker_kwargs.update(
             {
+                "host": host,
                 "nthreads": threads_per_worker,
                 "services": worker_services,
                 "dashboard_address": worker_dashboard_address,
diff --git a/distributed/deploy/tests/test_local.py b/distributed/deploy/tests/test_local.py
index bc33fd8f328..a8876e57b88 100644
--- a/distributed/deploy/tests/test_local.py
+++ b/distributed/deploy/tests/test_local.py
@@ -7,6 +7,7 @@
 from distutils.version import LooseVersion
 from threading import Lock
 from time import sleep
+from urllib.parse import urlparse
 
 import pytest
 import tornado
@@ -1100,3 +1101,20 @@ async def test_cluster_info_sync():
 
         info = cluster.scheduler.get_metadata(keys=["cluster-manager-info"])
         assert info["foo"] == "bar"
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("host", [None, "127.0.0.1"])
+@pytest.mark.parametrize("use_nanny", [True, False])
+async def test_cluster_host_used_throughout_cluster(host, use_nanny):
+    """Ensure that the `host` kwarg is propagated through scheduler, nanny, and workers"""
+    async with LocalCluster(host=host, asynchronous=True) as cluster:
+        url = urlparse(cluster.scheduler_address)
+        assert url.hostname == "127.0.0.1"
+        for worker in cluster.workers.values():
+            url = urlparse(worker.address)
+            assert url.hostname == "127.0.0.1"
+
+            if use_nanny:
+                url = urlparse(worker.process.worker_address)
+                assert url.hostname == "127.0.0.1"

From ce1d0f1db7323ce5e3f513205fdf2fee24a57e67 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@users.noreply.github.com>
Date: Fri, 15 Oct 2021 09:06:35 -0500
Subject: [PATCH 1539/1550] Remove zict-related skips (#5429)

---
 distributed/spill.py                | 7 ++-----
 distributed/tests/test_scheduler.py | 2 --
 distributed/tests/test_spill.py     | 2 --
 3 files changed, 2 insertions(+), 9 deletions(-)

diff --git a/distributed/spill.py b/distributed/spill.py
index 5d90361d170..2c849c2447e 100644
--- a/distributed/spill.py
+++ b/distributed/spill.py
@@ -4,14 +4,11 @@
 from functools import partial
 from typing import Any
 
+from zict import Buffer, File, Func
+
 from .protocol import deserialize_bytes, serialize_bytelist
 from .sizeof import safe_sizeof
 
-try:
-    from zict import Buffer, File, Func
-except ImportError:
-    raise ImportError("Please `python -m pip install zict` for spill-to-disk workers")
-
 
 class SpillBuffer(Buffer):
     """MutableMapping that automatically spills out dask key/value pairs to disk when
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index 7d26e3e7904..b222e09feed 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -2473,8 +2473,6 @@ async def assert_memory(scheduler_or_workerstate, attr: str, min_, max_, timeout
     client=True, Worker=Nanny, worker_kwargs={"memory_limit": "500 MiB"}, timeout=120
 )
 async def test_memory(c, s, *_):
-    pytest.importorskip("zict")
-
     # WorkerState objects, as opposed to the Nanny objects passed by gen_cluster
     a, b = s.workers.values()
 
diff --git a/distributed/tests/test_spill.py b/distributed/tests/test_spill.py
index 8735f1acbdf..d013b141158 100644
--- a/distributed/tests/test_spill.py
+++ b/distributed/tests/test_spill.py
@@ -1,7 +1,5 @@
 import pytest
 
-pytest.importorskip("zict")
-
 from dask.sizeof import sizeof
 
 from distributed.spill import SpillBuffer

From 6a0217e4d7ac442f9c9be018baab37ae3ec90cb9 Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Fri, 15 Oct 2021 18:22:57 +0100
Subject: [PATCH 1540/1550] Mark test_gather_dep* as xfail (#5432)

* Mark test_gather_dep as flaky

* xfail
---
 distributed/tests/test_worker.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index a92700b053c..93d391f6ef2 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -3089,6 +3089,7 @@ async def _wait_for_flight(key, worker):
         await asyncio.sleep(0)
 
 
+@pytest.mark.xfail(reason="#5406")
 @gen_cluster(client=True)
 async def test_gather_dep_do_not_handle_response_of_not_requested_tasks(c, s, a, b):
     """At time of writing, the gather_dep implementation filtered tasks again
@@ -3122,6 +3123,7 @@ async def test_gather_dep_do_not_handle_response_of_not_requested_tasks(c, s, a,
     await fut3
 
 
+@pytest.mark.xfail(reason="#5406")
 @gen_cluster(
     client=True,
     config={

From 7d2516a6d752594e8e24cf530fcd2dcefacf7c17 Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Mon, 18 Oct 2021 19:04:04 +0200
Subject: [PATCH 1541/1550] Ensure reconnecting workers do not loose required
 data (#5436)

---
 distributed/scheduler.py                 | 23 +++++++++++++----------
 distributed/tests/test_failed_workers.py |  2 +-
 distributed/tests/test_worker.py         |  2 --
 distributed/worker.py                    |  6 +++---
 4 files changed, 17 insertions(+), 16 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index 481220a97e5..fd873469af9 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -2737,8 +2737,7 @@ def transition_processing_memory(
 
             if ws != ts._processing_on:  # someone else has this task
                 logger.info(
-                    "Unexpected worker completed task, likely due to "
-                    "work stealing. Expected: %s, Got: %s, Key: %s",
+                    "Unexpected worker completed task. Expected: %s, Got: %s, Key: %s",
                     ts._processing_on,
                     ws,
                     key,
@@ -2835,7 +2834,7 @@ def transition_memory_released(self, key, safe: bint = False):
             worker_msg = {
                 "op": "free-keys",
                 "keys": [key],
-                "reason": f"Memory->Released {key}",
+                "stimulus_id": f"memory-released-{time()}",
             }
             for ws in ts._who_has:
                 worker_msgs[ws._address] = [worker_msg]
@@ -2935,7 +2934,11 @@ def transition_erred_released(self, key):
                 if dts._state == "erred":
                     recommendations[dts._key] = "waiting"
 
-            w_msg = {"op": "free-keys", "keys": [key], "reason": "Erred->Released"}
+            w_msg = {
+                "op": "free-keys",
+                "keys": [key],
+                "stimulus_id": f"erred-released-{time()}",
+            }
             for ws_addr in ts._erred_on:
                 worker_msgs[ws_addr] = [w_msg]
             ts._erred_on.clear()
@@ -3013,7 +3016,7 @@ def transition_processing_released(self, key):
                     {
                         "op": "free-keys",
                         "keys": [key],
-                        "reason": f"processing-released-{time()}",
+                        "stimulus_id": f"processing-released-{time()}",
                     }
                 ]
 
@@ -4339,9 +4342,9 @@ async def add_worker(
                         worker_msgs[address] = []
                     worker_msgs[address].append(
                         {
-                            "op": "free-keys",
+                            "op": "remove-replicas",
                             "keys": already_released_keys,
-                            "reason": f"reconnect-already-released-{time()}",
+                            "stimulus_id": f"reconnect-already-released-{time()}",
                         }
                     )
             for ts in list(parent._unrunnable):
@@ -4767,7 +4770,7 @@ def stimulus_task_finished(self, key=None, worker=None, **kwargs):
                 {
                     "op": "free-keys",
                     "keys": [key],
-                    "reason": f"already-released-or-forgotten-{time()}",
+                    "stimulus_id": f"already-released-or-forgotten-{time()}",
                 }
             ]
         elif ts._state == "memory":
@@ -5965,7 +5968,7 @@ async def delete_worker_data(
             await retry_operation(
                 self.rpc(addr=worker_address).free_keys,
                 keys=list(keys),
-                reason="rebalance/replicate",
+                stimulus_id=f"delete-data-{time()}",
             )
         except OSError as e:
             # This can happen e.g. if the worker is going through controlled shutdown;
@@ -7846,7 +7849,7 @@ def _propagate_forgotten(
                 {
                     "op": "free-keys",
                     "keys": [key],
-                    "reason": f"propagate-forgotten {ts.key}",
+                    "stimulus_id": f"propagate-forgotten-{time()}",
                 }
             ]
     state.remove_all_replicas(ts)
diff --git a/distributed/tests/test_failed_workers.py b/distributed/tests/test_failed_workers.py
index 8e5d01167d0..2b6d5149a2a 100644
--- a/distributed/tests/test_failed_workers.py
+++ b/distributed/tests/test_failed_workers.py
@@ -488,7 +488,7 @@ def sink(*args):
         # artificially, without notifying the scheduler.
         # This can only succeed if B handles the missing data properly by
         # removing A from the known sources of keys
-        a.handle_free_keys(keys=["f1"], reason="Am I evil?")  # Yes, I am!
+        a.handle_free_keys(keys=["f1"], stimulus_id="Am I evil?")  # Yes, I am!
         result_fut = c.submit(sink, futures, workers=x.address)
 
         await result_fut
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 93d391f6ef2..81e25d8d039 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -2444,7 +2444,6 @@ async def test_hold_on_to_replicas(c, s, *workers):
         await asyncio.sleep(0.01)
 
 
-@pytest.mark.flaky(reruns=10, reruns_delay=5)
 @gen_cluster(client=True)
 async def test_worker_reconnects_mid_compute(c, s, a, b):
     """Ensure that, if a worker disconnects while computing a result, the scheduler will
@@ -2513,7 +2512,6 @@ def fast_on_a(lock):
         await asyncio.sleep(0.001)
 
 
-@pytest.mark.flaky(reruns=10, reruns_delay=5)
 @gen_cluster(client=True)
 async def test_worker_reconnects_mid_compute_multiple_states_on_scheduler(c, s, a, b):
     """
diff --git a/distributed/worker.py b/distributed/worker.py
index 88e82d5915a..ebd62e9cd9a 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -1596,7 +1596,7 @@ def update_data(
             self.batched_stream.send(msg)
         return {"nbytes": {k: sizeof(v) for k, v in data.items()}, "status": "OK"}
 
-    def handle_free_keys(self, comm=None, keys=None, reason=None):
+    def handle_free_keys(self, comm=None, keys=None, stimulus_id=None):
         """
         Handler to be called by the scheduler.
 
@@ -1607,14 +1607,14 @@ def handle_free_keys(self, comm=None, keys=None, reason=None):
         still decide to hold on to the data and task since it is required by an
         upstream dependency.
         """
-        self.log.append(("free-keys", keys, reason))
+        self.log.append(("free-keys", keys, stimulus_id))
         recommendations = {}
         for key in keys:
             ts = self.tasks.get(key)
             if ts:
                 recommendations[ts] = "released" if ts.dependents else "forgotten"
 
-        self.transitions(recommendations, stimulus_id=reason)
+        self.transitions(recommendations, stimulus_id=stimulus_id)
 
     def handle_remove_replicas(self, keys, stimulus_id):
         """Stream handler notifying the worker that it might be holding unreferenced,

From 1585f85d9641cb1a8ccba2b5c22acccc58b8f41b Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Tue, 19 Oct 2021 12:33:11 +0100
Subject: [PATCH 1542/1550] Type annotations for Worker and gen_cluster (#5438)

---
 distributed/profile.py              |   5 +-
 distributed/tests/test_scheduler.py |   2 +-
 distributed/tests/test_stress.py    |   5 -
 distributed/tests/test_worker.py    |   2 +-
 distributed/utils_test.py           |  58 +++---
 distributed/worker.py               | 311 +++++++++++++++++++---------
 6 files changed, 254 insertions(+), 129 deletions(-)

diff --git a/distributed/profile.py b/distributed/profile.py
index f0535ef0da8..e0855115458 100644
--- a/distributed/profile.py
+++ b/distributed/profile.py
@@ -24,12 +24,15 @@
                    'children': {...}}}
     }
 """
+from __future__ import annotations
+
 import bisect
 import linecache
 import sys
 import threading
 from collections import defaultdict, deque
 from time import sleep
+from typing import Any
 
 import tlz as toolz
 
@@ -152,7 +155,7 @@ def merge(*args):
     }
 
 
-def create():
+def create() -> dict[str, Any]:
     return {
         "count": 0,
         "children": {},
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index b222e09feed..e08720ec368 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -1947,7 +1947,7 @@ class NoSchedulerDelayWorker(Worker):
     comparisons using times reported from workers.
     """
 
-    @property
+    @property  # type: ignore
     def scheduler_delay(self):
         return 0
 
diff --git a/distributed/tests/test_stress.py b/distributed/tests/test_stress.py
index 0e6cd66fa19..1219f767274 100644
--- a/distributed/tests/test_stress.py
+++ b/distributed/tests/test_stress.py
@@ -99,9 +99,6 @@ async def test_stress_creation_and_deletion(c, s):
     # Assertions are handled by the validate mechanism in the scheduler
     da = pytest.importorskip("dask.array")
 
-    def _disable_suspicious_counter(dask_worker):
-        dask_worker._suspicious_count_limit = None
-
     rng = da.random.RandomState(0)
     x = rng.random(size=(2000, 2000), chunks=(100, 100))
     y = ((x + 1).T + (x * 2) - x.mean(axis=1)).sum().round(2)
@@ -111,14 +108,12 @@ async def create_and_destroy_worker(delay):
         start = time()
         while time() < start + 5:
             async with Nanny(s.address, nthreads=2) as n:
-                await c.run(_disable_suspicious_counter, workers=[n.worker_address])
                 await asyncio.sleep(delay)
             print("Killed nanny")
 
     await asyncio.gather(*(create_and_destroy_worker(0.1 * i) for i in range(20)))
 
     async with Nanny(s.address, nthreads=2):
-        await c.run(_disable_suspicious_counter)
         assert await c.compute(z) == 8000884.93
 
 
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 81e25d8d039..47233bc1352 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -2968,7 +2968,7 @@ async def test_who_has_consistent_remove_replica(c, s, *workers):
 
     await f2
 
-    assert ("missing-dep", f1.key) in a.story(f1.key)
+    assert (f1.key, "missing-dep") in a.story(f1.key)
     assert a.tasks[f1.key].suspicious_count == 0
     assert s.tasks[f1.key].suspicious == 0
 
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index d6d3821a018..3d4a55cbef0 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -22,6 +22,7 @@
 import warnings
 import weakref
 from collections import defaultdict
+from collections.abc import Callable
 from contextlib import contextmanager, nullcontext, suppress
 from glob import glob
 from itertools import count
@@ -54,6 +55,7 @@
 from .diagnostics.plugin import WorkerPlugin
 from .metrics import time
 from .nanny import Nanny
+from .node import ServerNode
 from .proctitle import enable_proctitle_on_children
 from .security import Security
 from .utils import (
@@ -770,7 +772,7 @@ async def disconnect_all(addresses, timeout=3, rpc_kwargs=None):
     await asyncio.gather(*(disconnect(addr, timeout, rpc_kwargs) for addr in addresses))
 
 
-def gen_test(timeout=_TEST_TIMEOUT):
+def gen_test(timeout: float = _TEST_TIMEOUT) -> Callable[[Callable], Callable]:
     """Coroutine test
 
     @gen_test(timeout=5)
@@ -797,14 +799,14 @@ def test_func():
 
 
 async def start_cluster(
-    nthreads,
-    scheduler_addr,
-    loop,
-    security=None,
-    Worker=Worker,
-    scheduler_kwargs={},
-    worker_kwargs={},
-):
+    nthreads: list[tuple[str, int] | tuple[str, int, dict]],
+    scheduler_addr: str,
+    loop: IOLoop,
+    security: Security | dict[str, Any] | None = None,
+    Worker: type[ServerNode] = Worker,
+    scheduler_kwargs: dict[str, Any] = {},
+    worker_kwargs: dict[str, Any] = {},
+) -> tuple[Scheduler, list[ServerNode]]:
     s = await Scheduler(
         loop=loop,
         validate=True,
@@ -813,6 +815,7 @@ async def start_cluster(
         host=scheduler_addr,
         **scheduler_kwargs,
     )
+
     workers = [
         Worker(
             s.address,
@@ -822,7 +825,11 @@ async def start_cluster(
             loop=loop,
             validate=True,
             host=ncore[0],
-            **(merge(worker_kwargs, ncore[2]) if len(ncore) > 2 else worker_kwargs),
+            **(
+                merge(worker_kwargs, ncore[2])  # type: ignore
+                if len(ncore) > 2
+                else worker_kwargs
+            ),
         )
         for i, ncore in enumerate(nthreads)
     ]
@@ -854,21 +861,24 @@ async def end_worker(w):
 
 
 def gen_cluster(
-    nthreads=[("127.0.0.1", 1), ("127.0.0.1", 2)],
-    ncores=None,
+    nthreads: list[tuple[str, int] | tuple[str, int, dict]] = [
+        ("127.0.0.1", 1),
+        ("127.0.0.1", 2),
+    ],
+    ncores: None = None,  # deprecated
     scheduler="127.0.0.1",
-    timeout=_TEST_TIMEOUT,
-    security=None,
-    Worker=Worker,
-    client=False,
-    scheduler_kwargs={},
-    worker_kwargs={},
-    client_kwargs={},
-    active_rpc_timeout=1,
-    config={},
-    clean_kwargs={},
-    allow_unclosed=False,
-):
+    timeout: float = _TEST_TIMEOUT,
+    security: Security | dict[str, Any] | None = None,
+    Worker: type[ServerNode] = Worker,
+    client: bool = False,
+    scheduler_kwargs: dict[str, Any] = {},
+    worker_kwargs: dict[str, Any] = {},
+    client_kwargs: dict[str, Any] = {},
+    active_rpc_timeout: float = 1,
+    config: dict[str, Any] = {},
+    clean_kwargs: dict[str, Any] = {},
+    allow_unclosed: bool = False,
+) -> Callable[[Callable], Callable]:
     from distributed import Client
 
     """ Coroutine test with small cluster
diff --git a/distributed/worker.py b/distributed/worker.py
index ebd62e9cd9a..9fca11e9f8b 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -3,7 +3,6 @@
 import asyncio
 import bisect
 import builtins
-import concurrent.futures
 import errno
 import heapq
 import logging
@@ -14,15 +13,20 @@
 import warnings
 import weakref
 from collections import defaultdict, deque, namedtuple
-from collections.abc import Callable, Hashable, Iterable, MutableMapping
+from collections.abc import Callable, Iterable, Mapping, MutableMapping
+from concurrent.futures import Executor
 from contextlib import suppress
 from datetime import timedelta
 from inspect import isawaitable
 from pickle import PicklingError
-from typing import TYPE_CHECKING, Any, ClassVar
+from typing import TYPE_CHECKING, Any, ClassVar, cast
 
 if TYPE_CHECKING:
+    from typing_extensions import Literal
+    from .diagnostics.plugin import WorkerPlugin
+    from .actor import Actor
     from .client import Client
+    from .nanny import Nanny
 
 from tlz import first, keymap, merge, pluck  # noqa: F401
 from tornado.ioloop import IOLoop, PeriodicCallback
@@ -55,7 +59,7 @@
 )
 from .diagnostics import nvml
 from .diagnostics.plugin import _get_plugin_name
-from .diskutils import WorkSpace
+from .diskutils import WorkDir, WorkSpace
 from .http import get_handlers
 from .metrics import time
 from .node import ServerNode
@@ -206,6 +210,7 @@ def __init__(self, key, runspec=None):
         self.nbytes = None
         self.annotations = None
         self.done = False
+        self._previous = None
         self._next = None
 
     def __repr__(self):
@@ -251,7 +256,7 @@ class Worker(ServerNode):
 
     * **nthreads:** ``int``:
         Number of nthreads used by this worker process
-    * **executors:** ``Dict[str, concurrent.futures.Executor]``:
+    * **executors:** ``dict[str, concurrent.futures.Executor]``:
         Executors used to perform computation. Always contains the default
         executor.
     * **local_directory:** ``path``:
@@ -332,8 +337,9 @@ class Worker(ServerNode):
 
     Parameters
     ----------
-    scheduler_ip: str
-    scheduler_port: int
+    scheduler_ip: str, optional
+    scheduler_port: int, optional
+    scheduler_file: str, optional
     ip: str, optional
     data: MutableMapping, type, None
         The object to use for storage, builds a disk-backed LRU dict by default
@@ -347,13 +353,16 @@ class Worker(ServerNode):
         Set to zero for no limit.  Set to 'auto' to calculate
         as system.MEMORY_LIMIT * min(1, nthreads / total_cores)
         Use strings or numbers like 5GB or 5e9
-    memory_target_fraction: float
+    memory_target_fraction: float or False
         Fraction of memory to try to stay beneath
-    memory_spill_fraction: float
+        (default: read from config key distributed.worker.memory.target)
+    memory_spill_fraction: float or false
         Fraction of memory at which we start spilling to disk
-    memory_pause_fraction: float
+        (default: read from config key distributed.worker.memory.spill)
+    memory_pause_fraction: float or False
         Fraction of memory at which we stop running new tasks
-    executor: concurrent.futures.Executor, dict[str, concurrent.futures.Executor], str
+        (default: read from config key distributed.worker.memory.pause)
+    executor: concurrent.futures.Executor, dict[str, concurrent.futures.Executor], "offload"
         The executor(s) to use. Depending on the type, it has the following meanings:
             - Executor instance: The default executor.
             - Dict[str, Executor]: mapping names to Executor instances. If the
@@ -376,6 +385,8 @@ class Worker(ServerNode):
     lifetime_restart: bool
         Whether or not to restart a worker after it has reached its lifetime
         Default False
+    kwargs: optional
+        Additional parameters to ServerNode constructor
 
     Examples
     --------
@@ -398,50 +409,156 @@ class Worker(ServerNode):
     _instances: ClassVar[weakref.WeakSet[Worker]] = weakref.WeakSet()
     _initialized_clients: ClassVar[weakref.WeakSet[Client]] = weakref.WeakSet()
 
+    tasks: dict[str, TaskState]
+    waiting_for_data_count: int
+    has_what: defaultdict[str, set[str]]  # {worker address: {ts.key, ...}
+    pending_data_per_worker: defaultdict[str, deque[str]]
+    nanny: Nanny | None
+    _lock: threading.Lock
+    data_needed: list[tuple[int, str]]  # heap[(ts.priority, ts.key)]
+    in_flight_workers: dict[str, set[str]]  # {worker address: {ts.key, ...}}
+    total_out_connections: int
+    total_in_connections: int
+    comm_threshold_bytes: int
+    comm_nbytes: int
+    _missing_dep_flight: set[TaskState]
+    threads: dict[str, int]  # {ts.key: thread ID}
+    active_threads_lock: threading.Lock
+    active_threads: dict[int, str]  # {thread ID: ts.key}
+    active_keys: set[str]
+    profile_keys: defaultdict[str, dict[str, Any]]
+    profile_keys_history: deque[tuple[float, dict[str, dict[str, Any]]]]
+    profile_recent: dict[str, Any]
+    profile_history: deque[tuple[float, dict[str, Any]]]
+    generation: int
+    ready: list[str]
+    constrained: deque[str]
+    _executing: set[TaskState]
+    _in_flight_tasks: set[TaskState]
+    executed_count: int
+    long_running: set[TaskState]
+    log: deque[tuple]
+    incoming_transfer_log: deque[dict[str, Any]]
+    outgoing_transfer_log: deque[dict[str, Any]]
+    target_message_size: int
+    validate: bool
+    _transitions_table: dict[tuple[str, str], Callable]
+    _transition_counter: int
+    incoming_count: int
+    outgoing_count: int
+    outgoing_current_count: int
+    repetitively_busy: int
+    bandwidth: float
+    latency: float
+    profile_cycle_interval: float
+    workspace: WorkSpace
+    _workdir: WorkDir
+    local_directory: str
+    _client: Client | None
+    bandwidth_workers: defaultdict[str, tuple[float, int]]
+    bandwidth_types: defaultdict[type, tuple[float, int]]
+    preloads: list[preloading.Preload]
+    contact_address: str | None
+    _start_port: int | None
+    _start_host: str | None
+    _interface: str | None
+    _protocol: str
+    _dashboard_address: str | None
+    _dashboard: bool
+    _http_prefix: str
+    nthreads: int
+    total_resources: dict[str, float]
+    available_resources: dict[str, float]
+    death_timeout: float | None
+    lifetime: float | None
+    lifetime_stagger: float | None
+    lifetime_restart: bool
+    extensions: dict
+    security: Security
+    connection_args: dict[str, Any]
+    memory_limit: int | None
+    memory_target_fraction: float | Literal[False]
+    memory_spill_fraction: float | Literal[False]
+    memory_pause_fraction: float | Literal[False]
+    data: MutableMapping[str, Any]  # {task key: task payload}
+    actors: dict[str, Actor | None]
+    loop: IOLoop
+    reconnect: bool
+    executors: dict[str, Executor]
+    batched_stream: BatchedSend
+    name: Any
+    scheduler_delay: float
+    stream_comms: dict[str, BatchedSend]
+    heartbeat_active: bool
+    _ipython_kernel: Any | None = None
+    services: dict[str, Any] = {}
+    service_specs: dict[str, Any]
+    metrics: dict[str, Callable[[Worker], Any]]
+    startup_information: dict[str, Callable[[Worker], Any]]
+    low_level_profiler: bool
+    scheduler: Any
+    execution_state: dict[str, Any]
+    memory_monitor_interval: float | None
+    _memory_monitoring: bool
+    _throttled_gc: ThrottledGC
+    plugins: dict[str, WorkerPlugin]
+    _pending_plugins: tuple[WorkerPlugin, ...]
+
     def __init__(
         self,
-        scheduler_ip=None,
-        scheduler_port=None,
-        scheduler_file=None,
-        ncores=None,
-        nthreads=None,
-        loop=None,
-        local_dir=None,
-        local_directory=None,
-        services=None,
-        service_ports=None,
-        service_kwargs=None,
-        name=None,
-        reconnect=True,
-        memory_limit="auto",
-        executor=None,
-        resources=None,
-        silence_logs=None,
-        death_timeout=None,
-        preload=None,
-        preload_argv=None,
-        security=None,
-        contact_address=None,
-        memory_monitor_interval="200ms",
-        extensions=None,
-        metrics=DEFAULT_METRICS,
-        startup_information=DEFAULT_STARTUP_INFORMATION,
-        data=None,
-        interface=None,
-        host=None,
-        port=None,
-        protocol=None,
-        dashboard_address=None,
-        dashboard=False,
-        http_prefix="/",
-        nanny=None,
-        plugins=(),
-        low_level_profiler=dask.config.get("distributed.worker.profile.low-level"),
-        validate=None,
+        scheduler_ip: str | None = None,
+        scheduler_port: int | None = None,
+        *,
+        scheduler_file: str | None = None,
+        ncores: None = None,  # Deprecated, use nthreads instead
+        nthreads: int | None = None,
+        loop: IOLoop | None = None,
+        local_dir: None = None,  # Deprecated, use local_directory instead
+        local_directory: str | None = None,
+        services: dict | None = None,
+        name: Any | None = None,
+        reconnect: bool = True,
+        memory_limit: str | float = "auto",
+        executor: Executor | dict[str, Executor] | Literal["offload"] | None = None,
+        resources: dict[str, float] | None = None,
+        silence_logs: int | None = None,
+        death_timeout: Any | None = None,
+        preload: list[str] | None = None,
+        preload_argv: list[str] | list[list[str]] | None = None,
+        security: Security | dict[str, Any] | None = None,
+        contact_address: str | None = None,
+        memory_monitor_interval: Any = "200ms",
+        memory_target_fraction: float | Literal[False] | None = None,
+        memory_spill_fraction: float | Literal[False] | None = None,
+        memory_pause_fraction: float | Literal[False] | None = None,
+        extensions: list[type] | None = None,
+        metrics: Mapping[str, Callable[[Worker], Any]] = DEFAULT_METRICS,
+        startup_information: Mapping[
+            str, Callable[[Worker], Any]
+        ] = DEFAULT_STARTUP_INFORMATION,
+        data: (
+            MutableMapping[str, Any]  # pre-initialised
+            | Callable[[], MutableMapping[str, Any]]  # constructor
+            | tuple[
+                Callable[..., MutableMapping[str, Any]], dict[str, Any]
+            ]  # (constructor, kwargs to constructor)
+            | None  # create internatlly
+        ) = None,
+        interface: str | None = None,
+        host: str | None = None,
+        port: int | None = None,
+        protocol: str | None = None,
+        dashboard_address: str | None = None,
+        dashboard: bool = False,
+        http_prefix: str = "/",
+        nanny: Nanny | None = None,
+        plugins: tuple[WorkerPlugin, ...] = (),
+        low_level_profiler: bool | None = None,
+        validate: bool | None = None,
         profile_cycle_interval=None,
-        lifetime=None,
-        lifetime_stagger=None,
-        lifetime_restart=None,
+        lifetime: Any | None = None,
+        lifetime_stagger: Any | None = None,
+        lifetime_restart: bool | None = None,
         **kwargs,
     ):
         self.tasks = {}
@@ -460,7 +577,7 @@ def __init__(
         self.total_in_connections = dask.config.get(
             "distributed.worker.connections.incoming"
         )
-        self.comm_threshold_bytes = 10e6
+        self.comm_threshold_bytes = int(10e6)
         self.comm_nbytes = 0
         self._missing_dep_flight = set()
 
@@ -483,10 +600,7 @@ def __init__(
         self.executed_count = 0
         self.long_running = set()
 
-        self.recent_messages_log = deque(
-            maxlen=dask.config.get("distributed.comm.recent-messages-log-length")
-        )
-        self.target_message_size = 50e6  # 50 MB
+        self.target_message_size = int(50e6)  # 50 MB
 
         self.log = deque(maxlen=100000)
         if validate is None:
@@ -559,6 +673,7 @@ def __init__(
         if profile_cycle_interval is None:
             profile_cycle_interval = dask.config.get("distributed.worker.profile.cycle")
         profile_cycle_interval = parse_timedelta(profile_cycle_interval, default="ms")
+        assert profile_cycle_interval
 
         self._setup_logging(logger)
 
@@ -587,6 +702,8 @@ def __init__(
             preload = dask.config.get("distributed.worker.preload")
         if not preload_argv:
             preload_argv = dask.config.get("distributed.worker.preload-argv")
+        assert preload is not None
+        assert preload_argv is not None
         self.preloads = preloading.process_preloads(
             self, preload, preload_argv, file_dir=self.local_directory
         )
@@ -606,6 +723,7 @@ def __init__(
             protocol_address = scheduler_addr.split("://")
             if len(protocol_address) == 2:
                 protocol = protocol_address[0]
+            assert protocol
 
         self._start_port = port
         self._start_host = host
@@ -627,6 +745,7 @@ def __init__(
         self.nthreads = nthreads or CPU_COUNT
         if resources is None:
             resources = dask.config.get("distributed.worker.resources", None)
+            assert isinstance(resources, dict)
 
         self.total_resources = resources or {}
         self.available_resources = (resources or {}).copy()
@@ -644,24 +763,21 @@ def __init__(
 
         self.memory_limit = parse_memory_limit(memory_limit, self.nthreads)
 
-        if "memory_target_fraction" in kwargs:
-            self.memory_target_fraction = kwargs.pop("memory_target_fraction")
-        else:
-            self.memory_target_fraction = dask.config.get(
-                "distributed.worker.memory.target"
-            )
-        if "memory_spill_fraction" in kwargs:
-            self.memory_spill_fraction = kwargs.pop("memory_spill_fraction")
-        else:
-            self.memory_spill_fraction = dask.config.get(
-                "distributed.worker.memory.spill"
-            )
-        if "memory_pause_fraction" in kwargs:
-            self.memory_pause_fraction = kwargs.pop("memory_pause_fraction")
-        else:
-            self.memory_pause_fraction = dask.config.get(
-                "distributed.worker.memory.pause"
-            )
+        self.memory_target_fraction = (
+            memory_target_fraction
+            if memory_target_fraction is not None
+            else dask.config.get("distributed.worker.memory.target")
+        )
+        self.memory_spill_fraction = (
+            memory_spill_fraction
+            if memory_spill_fraction is not None
+            else dask.config.get("distributed.worker.memory.spill")
+        )
+        self.memory_pause_fraction = (
+            memory_pause_fraction
+            if memory_pause_fraction is not None
+            else dask.config.get("distributed.worker.memory.pause")
+        )
 
         if isinstance(data, MutableMapping):
             self.data = data
@@ -690,7 +806,7 @@ def __init__(
         self.reconnect = reconnect
 
         # Common executors always available
-        self.executors: dict[str, concurrent.futures.Executor] = {
+        self.executors = {
             "offload": utils._offload_executor,
             "actor": ThreadPoolExecutor(1, thread_name_prefix="Dask-Actor-Threads"),
         }
@@ -733,6 +849,8 @@ def __init__(
             dict(startup_information) if startup_information else {}
         )
 
+        if low_level_profiler is None:
+            low_level_profiler = dask.config.get("distributed.worker.profile.low-level")
         self.low_level_profiler = low_level_profiler
 
         handlers = {
@@ -794,14 +912,14 @@ def __init__(
         pc = PeriodicCallback(self.find_missing, 1000)
         self.periodic_callbacks["find-missing"] = pc
 
-        self._suspicious_count_limit = 10
         self._address = contact_address
 
         self.memory_monitor_interval = parse_timedelta(
             memory_monitor_interval, default="ms"
         )
+        self._memory_monitoring = False
         if self.memory_limit:
-            self._memory_monitoring = False
+            assert self.memory_monitor_interval is not None
             pc = PeriodicCallback(
                 self.memory_monitor, self.memory_monitor_interval * 1000
             )
@@ -828,19 +946,18 @@ def __init__(
         self.plugins = {}
         self._pending_plugins = plugins
 
-        self.lifetime = lifetime or dask.config.get(
-            "distributed.worker.lifetime.duration"
-        )
-        lifetime_stagger = lifetime_stagger or dask.config.get(
-            "distributed.worker.lifetime.stagger"
-        )
-        self.lifetime_restart = lifetime_restart or dask.config.get(
-            "distributed.worker.lifetime.restart"
-        )
-        if isinstance(self.lifetime, str):
-            self.lifetime = parse_timedelta(self.lifetime)
-        if isinstance(lifetime_stagger, str):
-            lifetime_stagger = parse_timedelta(lifetime_stagger)
+        if lifetime is None:
+            lifetime = dask.config.get("distributed.worker.lifetime.duration")
+        self.lifetime = parse_timedelta(lifetime)
+
+        if lifetime_stagger is None:
+            lifetime_stagger = dask.config.get("distributed.worker.lifetime.stagger")
+        lifetime_stagger = parse_timedelta(lifetime_stagger)
+
+        if lifetime_restart is None:
+            lifetime_restart = dask.config.get("distributed.worker.lifetime.restart")
+        self.lifetime_restart = lifetime_restart
+
         if self.lifetime:
             self.lifetime += (random.random() * 2 - 1) * lifetime_stagger
             self.io_loop.call_later(self.lifetime, self.close_gracefully)
@@ -1644,7 +1761,7 @@ def handle_remove_replicas(self, keys, stimulus_id):
             if ts is None or ts.state != "memory":
                 continue
             if not ts.is_protected():
-                self.log.append(("remove-replica-confirmed", ts.key, stimulus_id))
+                self.log.append((ts.key, "remove-replica-confirmed", stimulus_id))
                 recommendations[ts] = "released" if ts.dependents else "forgotten"
             else:
                 rejected.append(key)
@@ -2624,7 +2741,7 @@ async def gather_dep(
                 deps_to_iter = set(self.in_flight_workers.pop(worker)) & to_gather_keys
 
                 for d in deps_to_iter:
-                    ts = self.tasks.get(d)
+                    ts = cast(TaskState, self.tasks.get(d))
                     assert ts, (d, self.story(d))
                     ts.done = True
                     if d in data:
@@ -2632,7 +2749,7 @@ async def gather_dep(
                     elif not busy:
                         ts.who_has.discard(worker)
                         self.has_what[worker].discard(ts.key)
-                        self.log.append(("missing-dep", d))
+                        self.log.append((d, "missing-dep"))
                         self.batched_stream.send(
                             {"op": "missing-data", "errant_worker": worker, "key": d}
                         )
@@ -2747,7 +2864,7 @@ def handle_steal_request(self, key, stimulus_id):
 
     def release_key(
         self,
-        key: Hashable,
+        key: str,
         cause: TaskState | None = None,
         reason: str | None = None,
         report: bool = True,
@@ -3565,7 +3682,7 @@ def client(self) -> Client:
             else:
                 return self._get_client()
 
-    def _get_client(self, timeout=None) -> Client:
+    def _get_client(self, timeout: float | None = None) -> Client:
         """Get local client attached to this worker
 
         If no such client exists, create one
@@ -3625,7 +3742,7 @@ def _get_client(self, timeout=None) -> Client:
 
         return self._client
 
-    def get_current_task(self):
+    def get_current_task(self) -> str:
         """Get the key of the task we are currently running
 
         This only makes sense to run within a task

From a8151a69ef89232be47f5a21f131ff97af3fa85c Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Tue, 19 Oct 2021 15:52:21 +0200
Subject: [PATCH 1543/1550] Increase latency for stealing (#5390)

---
 distributed/stealing.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/distributed/stealing.py b/distributed/stealing.py
index 69f176d1117..8ec77fc150d 100644
--- a/distributed/stealing.py
+++ b/distributed/stealing.py
@@ -14,7 +14,11 @@
 from .diagnostics.plugin import SchedulerPlugin
 from .utils import log_errors
 
-LATENCY = 10e-3
+# Stealing requires multiple network bounces and if successful also task
+# submission which may include code serialization. Therefore, be very
+# conservative in the latency estimation to suppress too aggressive stealing
+# of small tasks
+LATENCY = 0.1
 
 logger = logging.getLogger(__name__)
 

From f3aa9d1ae15682e0971222e70a9eed9f63f71325 Mon Sep 17 00:00:00 2001
From: Matthias Bussonnier <bussonniermatthias@gmail.com>
Date: Tue, 19 Oct 2021 14:24:34 -0700
Subject: [PATCH 1544/1550] Use `pip install .` instead of calling `setup.py`
 (#5442)

It is almost the same but is more consistent and future proof
---
 docs/source/install.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/source/install.rst b/docs/source/install.rst
index 695cd3e777f..e6d5cd26dd3 100644
--- a/docs/source/install.rst
+++ b/docs/source/install.rst
@@ -28,7 +28,7 @@ To install distributed from source, clone the repository from `github
 
     git clone https://github.com/dask/distributed.git
     cd distributed
-    python setup.py install
+    python -m pip install .
 
 
 Notes

From 3afc6703fb480b9fe5c983c84d6ea115810dd1ff Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Thu, 21 Oct 2021 00:05:16 +0100
Subject: [PATCH 1545/1550] AMM: Don't schedule tasks to paused workers (#5431)

---
 distributed/scheduler.py                      | 106 +++++++++++++-----
 distributed/tests/test_client.py              |  29 ++---
 distributed/tests/test_scheduler.py           |  67 +++++++++--
 distributed/utils_test.py                     |  10 +-
 .../widgets/templates/worker_state.html.j2    |   1 +
 distributed/worker.py                         |  44 ++++----
 6 files changed, 180 insertions(+), 77 deletions(-)

diff --git a/distributed/scheduler.py b/distributed/scheduler.py
index fd873469af9..a44ba452769 100644
--- a/distributed/scheduler.py
+++ b/distributed/scheduler.py
@@ -541,6 +541,7 @@ def __init__(
         self,
         *,
         address: str,
+        status: Status,
         pid: Py_ssize_t,
         name: object,
         nthreads: Py_ssize_t = 0,
@@ -560,9 +561,9 @@ def __init__(
         self._services = services or {}
         self._versions = versions or {}
         self._nanny = nanny
+        self._status = status
 
         self._hash = hash(address)
-        self._status = Status.undefined
         self._nbytes = 0
         self._occupancy = 0
         self._memory_unmanaged_old = 0
@@ -721,6 +722,7 @@ def clean(self):
         """Return a version of this object that is appropriate for serialization"""
         ws: WorkerState = WorkerState(
             address=self._address,
+            status=self._status,
             pid=self._pid,
             name=self._name,
             nthreads=self._nthreads,
@@ -736,9 +738,10 @@ def clean(self):
         return ws
 
     def __repr__(self):
-        return "<WorkerState %r, name: %s, memory: %d, processing: %d>" % (
+        return "<WorkerState %r, name: %s, status: %s, memory: %d, processing: %d>" % (
             self._address,
             self._name,
+            self._status.name,
             len(self._has_what),
             len(self._processing),
         )
@@ -747,6 +750,7 @@ def _repr_html_(self):
         return get_template("worker_state.html.j2").render(
             address=self.address,
             name=self.name,
+            status=self.status.name,
             has_what=self._has_what,
             processing=self.processing,
         )
@@ -1872,6 +1876,8 @@ class SchedulerState:
         Set of workers that are not fully utilized
     * **saturated:** ``{WorkerState}``:
         Set of workers that are not over-utilized
+    * **running:** ``{WorkerState}``:
+        Set of workers that are currently in running state
 
     * **clients:** ``{client key: ClientState}``
         Clients currently connected to the scheduler
@@ -1890,7 +1896,8 @@ class SchedulerState:
     _idle_dv: dict  # dict[str, WorkerState]
     _n_tasks: Py_ssize_t
     _resources: dict
-    _saturated: set
+    _saturated: set  # set[WorkerState]
+    _running: set  # set[WorkerState]
     _tasks: dict
     _task_groups: dict
     _task_prefixes: dict
@@ -1977,6 +1984,9 @@ def __init__(
         self._workers = workers
         # Note: cython.cast, not typing.cast!
         self._workers_dv = cast(dict, self._workers)
+        self._running = {
+            ws for ws in self._workers.values() if ws.status == Status.running
+        }
         self._plugins = {} if not plugins else {_get_plugin_name(p): p for p in plugins}
 
         # Variables from dask.config, cached by __init__ for performance
@@ -2041,9 +2051,13 @@ def resources(self):
         return self._resources
 
     @property
-    def saturated(self):
+    def saturated(self) -> "set[WorkerState]":
         return self._saturated
 
+    @property
+    def running(self) -> "set[WorkerState]":
+        return self._running
+
     @property
     def tasks(self):
         return self._tasks
@@ -3339,7 +3353,7 @@ def get_task_duration(self, ts: TaskState) -> double:
 
     @ccall
     @exceptval(check=False)
-    def valid_workers(self, ts: TaskState) -> set:
+    def valid_workers(self, ts: TaskState) -> set:  # set[WorkerState] | None
         """Return set of currently valid workers for key
 
         If all workers are valid then this returns ``None``.
@@ -3352,7 +3366,7 @@ def valid_workers(self, ts: TaskState) -> set:
         s: set = None  # type: ignore
 
         if ts._worker_restrictions:
-            s = {w for w in ts._worker_restrictions if w in self._workers_dv}
+            s = {addr for addr in ts._worker_restrictions if addr in self._workers_dv}
 
         if ts._host_restrictions:
             # Resolve the alias here rather than early, for the worker
@@ -3379,9 +3393,9 @@ def valid_workers(self, ts: TaskState) -> set:
                     self._resources[resource] = dr = {}
 
                 sw: set = set()
-                for w, supplied in dr.items():
+                for addr, supplied in dr.items():
                     if supplied >= required:
-                        sw.add(w)
+                        sw.add(addr)
 
                 dw[resource] = sw
 
@@ -3391,8 +3405,13 @@ def valid_workers(self, ts: TaskState) -> set:
             else:
                 s &= ww
 
-        if s is not None:
-            s = {self._workers_dv[w] for w in s}
+        if s is None:
+            if len(self._running) < len(self._workers_dv):
+                return self._running.copy()
+        else:
+            s = {self._workers_dv[addr] for addr in s}
+            if len(self._running) < len(self._workers_dv):
+                s &= self._running
 
         return s
 
@@ -4212,7 +4231,9 @@ def heartbeat_worker(
     async def add_worker(
         self,
         comm=None,
-        address=None,
+        *,
+        address: str,
+        status: str,
         keys=(),
         nthreads=None,
         name=None,
@@ -4238,9 +4259,8 @@ async def add_worker(
             address = normalize_address(address)
             host = get_address_host(address)
 
-            ws: WorkerState = parent._workers_dv.get(address)
-            if ws is not None:
-                raise ValueError("Worker already exists %s" % ws)
+            if address in parent._workers_dv:
+                raise ValueError("Worker already exists %s" % address)
 
             if name in parent._aliases:
                 logger.warning(
@@ -4255,8 +4275,10 @@ async def add_worker(
                     await comm.write(msg)
                 return
 
+            ws: WorkerState
             parent._workers[address] = ws = WorkerState(
                 address=address,
+                status=Status.lookup[status],  # type: ignore
                 pid=pid,
                 nthreads=nthreads,
                 memory_limit=memory_limit or 0,
@@ -4267,12 +4289,14 @@ async def add_worker(
                 nanny=nanny,
                 extra=extra,
             )
+            if ws._status == Status.running:
+                parent._running.add(ws)
 
-            dh: dict = parent._host_info.get(host)
+            dh: dict = parent._host_info.get(host)  # type: ignore
             if dh is None:
                 parent._host_info[host] = dh = {}
 
-            dh_addresses: set = dh.get("addresses")
+            dh_addresses: set = dh.get("addresses")  # type: ignore
             if dh_addresses is None:
                 dh["addresses"] = dh_addresses = set()
                 dh["nthreads"] = 0
@@ -4292,7 +4316,8 @@ async def add_worker(
                 metrics=metrics,
             )
 
-            # Do not need to adjust parent._total_occupancy as self.occupancy[ws] cannot exist before this.
+            # Do not need to adjust parent._total_occupancy as self.occupancy[ws] cannot
+            # exist before this.
             self.check_idle_saturated(ws)
 
             # for key in keys:  # TODO
@@ -4318,7 +4343,7 @@ async def add_worker(
                 assert isinstance(nbytes, dict)
                 already_released_keys = []
                 for key in nbytes:
-                    ts: TaskState = parent._tasks.get(key)
+                    ts: TaskState = parent._tasks.get(key)  # type: ignore
                     if ts is not None and ts.state != "released":
                         if ts.state == "memory":
                             self.add_keys(worker=address, keys=[key])
@@ -4347,14 +4372,15 @@ async def add_worker(
                             "stimulus_id": f"reconnect-already-released-{time()}",
                         }
                     )
-            for ts in list(parent._unrunnable):
-                valid: set = self.valid_workers(ts)
-                if valid is None or ws in valid:
-                    recommendations[ts._key] = "waiting"
+
+            if ws._status == Status.running:
+                for ts in parent._unrunnable:
+                    valid: set = self.valid_workers(ts)
+                    if valid is None or ws in valid:
+                        recommendations[ts._key] = "waiting"
 
             if recommendations:
                 parent._transitions(recommendations, client_msgs, worker_msgs)
-                recommendations = {}
 
             self.send_all(client_msgs, worker_msgs)
 
@@ -4896,6 +4922,7 @@ async def remove_worker(self, comm=None, address=None, safe=False, close=True):
             parent._saturated.discard(ws)
             del parent._workers[address]
             ws.status = Status.closed
+            parent._running.discard(ws)
             parent._total_occupancy -= ws._occupancy
 
             recommendations: dict = {}
@@ -5143,6 +5170,11 @@ def validate_state(self, allow_overlap=False):
             if not ws._processing:
                 assert not ws._occupancy
                 assert ws._address in parent._idle_dv
+            assert (ws._status == Status.running) == (ws in parent._running)
+
+        for ws in parent._running:
+            assert ws._status == Status.running
+            assert ws._address in parent._workers_dv
 
         ts: TaskState
         for k, ts in parent._tasks.items():
@@ -5423,20 +5455,42 @@ def handle_long_running(self, key=None, worker=None, compute_duration=None):
         ws._processing[ts] = 0
         self.check_idle_saturated(ws)
 
-    def handle_worker_status_change(self, status: str, worker: str):
+    def handle_worker_status_change(self, status: str, worker: str) -> None:
         parent: SchedulerState = cast(SchedulerState, self)
         ws: WorkerState = parent._workers_dv.get(worker)  # type: ignore
         if not ws:
             return
+        prev_status = ws._status
+        ws._status = Status.lookup[status]  # type: ignore
+        if ws._status == prev_status:
+            return
+
         self.log_event(
             ws._address,
             {
                 "action": "worker-status-change",
-                "prev-status": ws._status.name,
+                "prev-status": prev_status.name,
                 "status": status,
             },
         )
-        ws._status = Status.lookup[status]  # type: ignore
+
+        if ws._status == Status.running:
+            parent._running.add(ws)
+
+            recs = {}
+            ts: TaskState
+            for ts in parent._unrunnable:
+                valid: set = self.valid_workers(ts)
+                if valid is None or ws in valid:
+                    recs[ts._key] = "waiting"
+            if recs:
+                client_msgs: dict = {}
+                worker_msgs: dict = {}
+                parent._transitions(recs, client_msgs, worker_msgs)
+                self.send_all(client_msgs, worker_msgs)
+
+        else:
+            parent._running.discard(ws)
 
     async def handle_worker(self, comm=None, worker=None):
         """
diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 982212bf74d..2c4a3130f38 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -3650,51 +3650,46 @@ def test_reconnect(loop):
         "9393",
         "--no-dashboard",
     ]
-    with popen(scheduler_cli) as s:
+    with popen(scheduler_cli):
         c = Client("127.0.0.1:9393", loop=loop)
-        start = time()
-        while len(c.nthreads()) != 1:
-            sleep(0.1)
-            assert time() < start + 3
-
+        c.wait_for_workers(1, timeout=10)
         x = c.submit(inc, 1)
-        assert x.result() == 2
+        assert x.result(timeout=10) == 2
 
     start = time()
     while c.status != "connecting":
-        assert time() < start + 5
+        assert time() < start + 10
         sleep(0.01)
 
     assert x.status == "cancelled"
     with pytest.raises(CancelledError):
-        x.result()
+        x.result(timeout=10)
 
-    with popen(scheduler_cli) as s:
+    with popen(scheduler_cli):
         start = time()
         while c.status != "running":
             sleep(0.1)
-            assert time() < start + 5
+            assert time() < start + 10
         start = time()
         while len(c.nthreads()) != 1:
             sleep(0.05)
-            assert time() < start + 15
+            assert time() < start + 10
 
         x = c.submit(inc, 1)
-        assert x.result() == 2
+        assert x.result(timeout=10) == 2
 
     start = time()
     while True:
+        assert time() < start + 10
         try:
-            x.result()
+            x.result(timeout=10)
             assert False
         except CommClosedError:
             continue
         except CancelledError:
             break
-        assert time() < start + 5
-        sleep(0.1)
 
-    sync(loop, w.close)
+    sync(loop, w.close, timeout=1)
     c.close()
 
 
diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index e08720ec368..affa58702ce 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -1101,24 +1101,23 @@ async def test_worker_breaks_and_returns(c, s, a):
 
 @gen_cluster(client=True, nthreads=[])
 async def test_no_workers_to_memory(c, s):
-    x = delayed(slowinc)(1, delay=0.4)
+    x = delayed(slowinc)(1, delay=10.0)
     y = delayed(slowinc)(x, delay=0.4)
     z = delayed(slowinc)(y, delay=0.4)
 
     yy, zz = c.persist([y, z])
 
-    while not s.tasks:
+    while len(s.tasks) < 3:
         await asyncio.sleep(0.01)
 
     w = Worker(s.address, nthreads=1)
     w.update_data(data={y.key: 3})
 
-    await w
-
     start = time()
-
-    while not s.workers:
+    await w
+    while not s.workers or s.workers[w.address].status != Status.running:
         await asyncio.sleep(0.01)
+    assert time() < start + 9  # Did not wait for x
 
     assert s.get_task_status(keys={x.key, y.key, z.key}) == {
         x.key: "released",
@@ -1289,7 +1288,11 @@ async def test_scheduler_file():
 async def test_non_existent_worker(c, s):
     with dask.config.set({"distributed.comm.timeouts.connect": "100ms"}):
         await s.add_worker(
-            address="127.0.0.1:5738", nthreads=2, nbytes={}, host_info={}
+            address="127.0.0.1:5738",
+            status="running",
+            nthreads=2,
+            nbytes={},
+            host_info={},
         )
         futures = c.map(inc, range(10))
         await asyncio.sleep(0.300)
@@ -1929,7 +1932,7 @@ async def test_default_task_duration_splits(c, s, a, b):
 
 
 @gen_test()
-async def test_no_danglng_asyncio_tasks():
+async def test_no_dangling_asyncio_tasks():
     start = asyncio.all_tasks()
     async with Scheduler(dashboard_address=":0") as s:
         async with Worker(s.address, name="0"):
@@ -3160,9 +3163,32 @@ async def test_worker_heartbeat_after_cancel(c, s, *workers):
         await asyncio.gather(*(w.heartbeat() for w in workers))
 
 
+@gen_cluster(client=True, nthreads=[("", 1)])
+async def test_worker_reconnect_task_memory(c, s, a):
+    a.periodic_callbacks["heartbeat"].stop()
+
+    futs = c.map(inc, range(10))
+    res = c.submit(sum, futs)
+
+    while not a.executing_count and not a.data:
+        await asyncio.sleep(0.001)
+
+    await s.remove_worker(address=a.address, close=False)
+    while not res.done():
+        await a.heartbeat()
+
+    await res
+    assert ("no-worker", "memory") in {
+        (start, finish) for (_, start, finish, _, _) in s.transition_log
+    }
+
+
 @gen_cluster(client=True, nthreads=[("", 1)])
 async def test_worker_reconnect_task_memory_with_resources(c, s, a):
     async with Worker(s.address, resources={"A": 1}) as b:
+        while s.workers[b.address].status != Status.running:
+            await asyncio.sleep(0.001)
+
         b.periodic_callbacks["heartbeat"].stop()
 
         futs = c.map(inc, range(10), resources={"A": 1})
@@ -3190,3 +3216,28 @@ async def test_set_restrictions(c, s, a, b):
     assert s.tasks[f.key].worker_restrictions == {a.address}
     s.reschedule(f)
     await f
+
+
+@gen_cluster(client=True, nthreads=[("", 1)] * 3)
+async def test_avoid_paused_workers(c, s, w1, w2, w3):
+    w2.memory_pause_fraction = 1e-15
+    while s.workers[w2.address].status != Status.paused:
+        await asyncio.sleep(0.01)
+    futures = c.map(slowinc, range(8), delay=0.1)
+    while (len(w1.tasks), len(w2.tasks), len(w3.tasks)) != (4, 0, 4):
+        await asyncio.sleep(0.01)
+
+
+@gen_cluster(client=True, nthreads=[("", 1)])
+async def test_unpause_schedules_unrannable_tasks(c, s, a):
+    a.memory_pause_fraction = 1e-15
+    while s.workers[a.address].status != Status.paused:
+        await asyncio.sleep(0.01)
+
+    fut = c.submit(inc, 1, key="x")
+    while not s.unrunnable:
+        await asyncio.sleep(0.001)
+    assert next(iter(s.unrunnable)).key == "x"
+
+    a.memory_pause_fraction = 0.8
+    assert await fut == 2
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index 3d4a55cbef0..c622a2faffa 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -71,7 +71,7 @@
     sync,
     thread_state,
 )
-from .worker import Worker
+from .worker import RUNNING, Worker
 
 try:
     import dask.array  # register config
@@ -837,8 +837,10 @@ async def start_cluster(
     await asyncio.gather(*workers)
 
     start = time()
-    while len(s.workers) < len(nthreads) or any(
-        comm.comm is None for comm in s.stream_comms.values()
+    while (
+        len(s.workers) < len(nthreads)
+        or any(ws.status != Status.running for ws in s.workers.values())
+        or any(comm.comm is None for comm in s.stream_comms.values())
     ):
         await asyncio.sleep(0.01)
         if time() > start + 30:
@@ -1557,7 +1559,7 @@ def check_instances():
     for w in Worker._instances:
         with suppress(RuntimeError):  # closed IOLoop
             w.loop.add_callback(w.close, report=False, executor_wait=False)
-            if w.status in (Status.running, Status.paused):
+            if w.status in RUNNING:
                 w.loop.add_callback(w.close)
     Worker._instances.clear()
 
diff --git a/distributed/widgets/templates/worker_state.html.j2 b/distributed/widgets/templates/worker_state.html.j2
index 2646d0fa262..cd152080bfc 100644
--- a/distributed/widgets/templates/worker_state.html.j2
+++ b/distributed/widgets/templates/worker_state.html.j2
@@ -1,4 +1,5 @@
 <strong>WorkerState: </strong> {{ address | html_escape }}
 <span style="color: var(--jp-ui-font-color2, gray)"> name: </span>{{ name }}
+<span style="color: var(--jp-ui-font-color2, gray)"> status: </span>{{ status }}
 <span style="color: var(--jp-ui-font-color2, gray)"> memory: </span>{{ has_what | length }}
 <span style="color: var(--jp-ui-font-color2, gray)"> processing: </span>{{ processing | length }}
diff --git a/distributed/worker.py b/distributed/worker.py
index 9fca11e9f8b..17e514b2063 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -97,6 +97,7 @@
 
 no_value = "--no-value-sentinel--"
 
+# TaskState.state subsets
 PROCESSING = {
     "waiting",
     "ready",
@@ -108,6 +109,8 @@
 }
 READY = {"ready", "constrained"}
 
+# Worker.status subsets
+RUNNING = {Status.running, Status.paused, Status.closing_gracefully}
 
 DEFAULT_EXTENSIONS: list[type] = [PubSubWorkerExtension]
 
@@ -1024,6 +1027,9 @@ def executor(self):
     def status(self, value):
         """Override Server.status to notify the Scheduler of status changes"""
         ServerNode.status.__set__(self, value)
+        self._send_worker_status_change()
+
+    def _send_worker_status_change(self) -> None:
         if (
             self.batched_stream
             and self.batched_stream.comm
@@ -1032,6 +1038,8 @@ def status(self, value):
             self.batched_stream.send(
                 {"op": "worker-status-change", "status": self._status.name}
             )
+        elif self._status != Status.closed:
+            self.loop.call_later(0.05, self._send_worker_status_change)
 
     async def get_metrics(self):
         out = dict(
@@ -1105,6 +1113,7 @@ async def _register_with_scheduler(self):
                         op="register-worker",
                         reply=False,
                         address=self.contact_address,
+                        status=self.status.name,
                         keys=list(self.data),
                         nthreads=self.nthreads,
                         name=self.name,
@@ -1195,14 +1204,15 @@ async def heartbeat(self):
             self._update_latency(end - start)
 
             if response["status"] == "missing":
-                for i in range(10):
-                    if self.status not in (Status.running, Status.paused):
-                        break
-                    else:
-                        await asyncio.sleep(0.05)
-                else:
+                # If running, wait up to 0.5s and then re-register self.
+                # Otherwise just exit.
+                start = time()
+                while self.status in RUNNING and time() < start + 0.5:
+                    await asyncio.sleep(0.01)
+                if self.status in RUNNING:
                     await self._register_with_scheduler()
                 return
+
             self.scheduler_delay = response["time"] - middle
             self.periodic_callbacks["heartbeat"].callback_time = (
                 response["heartbeat-interval"] * 1000
@@ -1231,7 +1241,7 @@ async def handle_scheduler(self, comm):
             logger.exception(e)
             raise
         finally:
-            if self.reconnect and self.status in (Status.running, Status.paused):
+            if self.reconnect and self.status in RUNNING:
                 logger.info("Connection to scheduler broken.  Reconnecting...")
                 self.loop.add_callback(self.heartbeat)
             else:
@@ -1443,11 +1453,7 @@ async def close(
                 logger.info("Stopping worker at %s", self.address)
             except ValueError:  # address not available if already closed
                 logger.info("Stopping worker")
-            if self.status not in (
-                Status.running,
-                Status.paused,
-                Status.closing_gracefully,
-            ):
+            if self.status not in RUNNING:
                 logger.info("Closed worker has not yet started: %s", self.status)
             self.status = Status.closing
 
@@ -1475,9 +1481,7 @@ async def close(
                 # If this worker is the last one alive, clean up the worker
                 # initialized clients
                 if not any(
-                    w
-                    for w in Worker._instances
-                    if w != self and w.status in (Status.running, Status.paused)
+                    w for w in Worker._instances if w != self and w.status in RUNNING
                 ):
                     for c in Worker._initialized_clients:
                         # Regardless of what the client was initialized with
@@ -2607,7 +2611,7 @@ async def gather_dep(
             Total number of bytes for all the dependencies in to_gather combined
         """
         cause: TaskState | None = None
-        if self.status not in (Status.running, Status.paused):
+        if self.status not in RUNNING:
             return
 
         with log_errors():
@@ -3619,7 +3623,7 @@ def validate_task(self, ts):
             raise
 
     def validate_state(self):
-        if self.status not in (Status.running, Status.paused):
+        if self.status not in RUNNING:
             return
         try:
             assert self.executing_count >= 0
@@ -3786,11 +3790,7 @@ def get_worker() -> Worker:
         return thread_state.execution_state["worker"]
     except AttributeError:
         try:
-            return first(
-                w
-                for w in Worker._instances
-                if w.status in (Status.running, Status.paused)
-            )
+            return first(w for w in Worker._instances if w.status in RUNNING)
         except StopIteration:
             raise ValueError("No workers found")
 

From 918e3fb13aee02bb75e22f1c2a892cb2cbadc1cb Mon Sep 17 00:00:00 2001
From: Peter Andreas Entschev <peter@entschev.com>
Date: Thu, 21 Oct 2021 21:34:54 +0200
Subject: [PATCH 1546/1550] Handle `UCXNotConnected` error (#5449)

Handle UCXNotConnected error
---
 distributed/comm/ucx.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/distributed/comm/ucx.py b/distributed/comm/ucx.py
index ecd6eaef72a..992367bdf87 100644
--- a/distributed/comm/ucx.py
+++ b/distributed/comm/ucx.py
@@ -383,10 +383,10 @@ async def connect(self, address: str, deserialize=True, **connection_args) -> UC
         init_once()
         try:
             ep = await ucx_create_endpoint(ip, port)
-        except (
-            ucp.exceptions.UCXCloseError,
-            ucp.exceptions.UCXCanceled,
-        ) + (getattr(ucp.exceptions, "UCXConnectionReset", ()),):
+        except (ucp.exceptions.UCXCloseError, ucp.exceptions.UCXCanceled,) + (
+            getattr(ucp.exceptions, "UCXConnectionReset", ()),
+            getattr(ucp.exceptions, "UCXNotConnected", ()),
+        ):
             raise CommClosedError("Connection closed before handshake completed")
         return self.comm_class(
             ep,

From 33d83bc8b9c27b2cb5c90cb3e04ca17fc2118b7b Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Thu, 21 Oct 2021 23:17:12 +0200
Subject: [PATCH 1547/1550] Provide stack for suspended coro in test timeout
 (#5446)

---
 distributed/tests/test_scheduler.py  |  2 +-
 distributed/tests/test_utils_test.py | 42 ++++++++++++++--------------
 distributed/utils_test.py            | 21 +++++++++++---
 3 files changed, 39 insertions(+), 26 deletions(-)

diff --git a/distributed/tests/test_scheduler.py b/distributed/tests/test_scheduler.py
index affa58702ce..7f2e7f942b5 100644
--- a/distributed/tests/test_scheduler.py
+++ b/distributed/tests/test_scheduler.py
@@ -420,7 +420,7 @@ def func(scheduler):
 @gen_cluster(
     nthreads=[], config={"distributed.scheduler.blocked-handlers": ["test-handler"]}
 )
-def test_scheduler_init_pulls_blocked_handlers_from_config(s):
+async def test_scheduler_init_pulls_blocked_handlers_from_config(s):
     assert s.blocked_handlers == ["test-handler"]
 
 
diff --git a/distributed/tests/test_utils_test.py b/distributed/tests/test_utils_test.py
index 0de9c7793ca..50a27570df3 100755
--- a/distributed/tests/test_utils_test.py
+++ b/distributed/tests/test_utils_test.py
@@ -106,27 +106,6 @@ def assert_config():
     await c.run_on_scheduler(assert_config)
 
 
-@gen_cluster(client=True)
-def test_gen_cluster_legacy_implicit(c, s, a, b):
-    assert isinstance(c, Client)
-    assert isinstance(s, Scheduler)
-    for w in [a, b]:
-        assert isinstance(w, Worker)
-    assert s.nthreads == {w.address: w.nthreads for w in [a, b]}
-    assert (yield c.submit(lambda: 123)) == 123
-
-
-@gen_cluster(client=True)
-@gen.coroutine
-def test_gen_cluster_legacy_explicit(c, s, a, b):
-    assert isinstance(c, Client)
-    assert isinstance(s, Scheduler)
-    for w in [a, b]:
-        assert isinstance(w, Worker)
-    assert s.nthreads == {w.address: w.nthreads for w in [a, b]}
-    assert (yield c.submit(lambda: 123)) == 123
-
-
 @pytest.mark.skip(reason="This hangs on travis")
 def test_gen_cluster_cleans_up_client(loop):
     import dask.context
@@ -373,3 +352,24 @@ async def ping_pong():
     assert await write_queue.get() == (b.address, {"op": "ping", "reply": True})
     write_event.set()
     assert await fut == "pong"
+
+
+@pytest.mark.slow()
+def test_provide_stack_on_timeout():
+    sleep_time = 30
+
+    async def inner_test(c, s, a, b):
+        await asyncio.sleep(sleep_time)
+
+    # If this timeout is too small, the cluster setup/teardown might take too
+    # long and the timeout error we'll receive will be different
+    test = gen_cluster(client=True, timeout=2)(inner_test)
+
+    start = time()
+    with pytest.raises(asyncio.TimeoutError) as exc:
+        test()
+    end = time()
+    assert "inner_test" in str(exc)
+    assert "await asyncio.sleep(sleep_time)" in str(exc)
+    # ensure the task was properly
+    assert end - start < sleep_time / 2
diff --git a/distributed/utils_test.py b/distributed/utils_test.py
index c622a2faffa..d5a4a07f539 100644
--- a/distributed/utils_test.py
+++ b/distributed/utils_test.py
@@ -919,7 +919,7 @@ async def test_foo(scheduler, worker1, worker2, pytest_fixture_a, pytest_fixture
 
     def _(func):
         if not iscoroutinefunction(func):
-            func = gen.coroutine(func)
+            raise RuntimeError("gen_cluster only works for coroutine functions.")
 
         @functools.wraps(func)
         def test_func(*outer_args, **kwargs):
@@ -964,11 +964,24 @@ async def coro():
                             )
                             args = [c] + args
                         try:
-                            future = func(*args, *outer_args, **kwargs)
-                            future = asyncio.wait_for(future, timeout)
-                            result = await future
+                            coro = func(*args, *outer_args, **kwargs)
+                            task = asyncio.create_task(coro)
+
+                            coro2 = asyncio.wait_for(asyncio.shield(task), timeout)
+                            result = await coro2
                             if s.validate:
                                 s.validate_state()
+                        except asyncio.TimeoutError as e:
+                            assert task
+                            buffer = io.StringIO()
+                            # This stack indicates where the coro/test is suspended
+                            task.print_stack(file=buffer)
+                            task.cancel()
+                            while not task.cancelled():
+                                await asyncio.sleep(0.01)
+                            raise TimeoutError(
+                                f"Test timeout after {timeout}s.\n{buffer.getvalue()}"
+                            ) from e
                         finally:
                             if client and c.status not in ("closing", "closed"):
                                 await c._close(fast=s.status == Status.closed)

From cdc68cc63d91d628a83716ccfe5b0d2ed3e2ed38 Mon Sep 17 00:00:00 2001
From: crusaderky <crusaderky@gmail.com>
Date: Fri, 22 Oct 2021 17:15:23 +0100
Subject: [PATCH 1548/1550] AMM high level documentation (#5456)

---
 distributed/active_memory_manager.py   |   6 +-
 docs/source/active_memory_manager.rst  | 247 +++++++++++++++++++++++++
 docs/source/conf.py                    |   2 +-
 docs/source/diagnosing-performance.rst |   5 +-
 docs/source/index.rst                  |   3 +-
 docs/source/tls.rst                    |   2 +-
 6 files changed, 255 insertions(+), 10 deletions(-)
 create mode 100644 docs/source/active_memory_manager.rst

diff --git a/distributed/active_memory_manager.py b/distributed/active_memory_manager.py
index 0bcd83522ca..7f3df5d9665 100644
--- a/distributed/active_memory_manager.py
+++ b/distributed/active_memory_manager.py
@@ -304,9 +304,9 @@ def run(
         You may optionally retrieve which worker it was decided the key will be
         replicated to or dropped from, as follows:
 
-        ```python
-        choice = yield "replicate", ts, None
-        ```
+        .. code-block:: python
+
+           choice = (yield "replicate", ts, None)
 
         ``choice`` is either a WorkerState or None; the latter is returned if the
         ActiveMemoryManager chose to disregard the request.
diff --git a/docs/source/active_memory_manager.rst b/docs/source/active_memory_manager.rst
new file mode 100644
index 00000000000..3fd5a68f238
--- /dev/null
+++ b/docs/source/active_memory_manager.rst
@@ -0,0 +1,247 @@
+Active Memory Manager
+=====================
+The Active Memory Manager, or *AMM*, is an experimental daemon that optimizes memory
+usage of workers across the Dask cluster. It is disabled by default.
+
+
+Memory imbalance and duplication
+--------------------------------
+Whenever a Dask task returns data, it is stored on the worker that executed the task for
+as long as it's a dependency of other tasks, is referenced by a ``Client`` through a
+``Future``, or is part of a :doc:`published dataset <publish>`.
+
+Dask assigns tasks to workers following criteria of CPU occupancy, :doc:`resources`, and
+locality. In the trivial use case of tasks that are not connected to each other, take
+the same time to compute, return data of the same size, and have no resource
+constraints, one will observe a perfect balance in memory occupation across workers too.
+In all other use cases, however, as the computation goes it could cause an imbalance in
+memory usage.
+
+When a task runs on a worker and requires in input the output of a task from a different
+worker, Dask will transparently transfer the data between workers, ending up with
+multiple copies of the same data on different workers. This is generally desirable, as
+it avoids re-transferring the data if it's required again later on. However, it also
+causes increased overall memory usage across the cluster.
+
+
+Enabling the Active Memory Manager
+----------------------------------
+The AMM can be enabled through the :doc:`Dask configuration file <configuration>`:
+
+.. code-block:: yaml
+
+   distributed:
+     scheduler:
+       active-memory-manager:
+         start: true
+         interval: 2s
+
+The above is the recommended setup and will run all enabled *AMM policies* (see below)
+every two seconds. Alternatively, you can manually start/stop the AMM from the
+``Client`` or trigger a one-off iteration:
+
+.. code-block:: python
+
+   >>> client.scheduler.amm_start()  # Start running every 2 seconds
+   >>> client.scheduler.amm_stop()  # Stop running periodically
+   >>> client.scheduler.amm_run_once()
+
+
+Policies
+--------
+The AMM by itself doesn't do anything. The user must enable *policies* which suggest
+actions regarding Dask data. The AMM runs the policies and enacts their suggestions, as
+long as they don't harm data integrity. These suggestions can be of two types:
+
+- Replicate the data of an in-memory Dask task from one worker to another.
+  This should not be confused with replication caused by task dependencies.
+- Delete one or more replicas of an in-memory task. The AMM will never delete the last
+  replica of a task, even if a policy asks to.
+
+Unless a policy puts constraints on which workers should be impacted, the AMM will
+automatically create replicas on workers with the lowest memory usage first and delete
+them from workers with the highest memory usage first.
+
+Individual policies are enabled, disabled, and configured through the Dask config:
+
+
+.. code-block:: yaml
+
+   distributed:
+     scheduler:
+       active-memory-manager:
+         start: true
+         interval: 2s
+         policies:
+         - class: distributed.active_memory_manager.ReduceReplicas
+         - class: my_package.MyPolicy
+           arg1: foo
+           arg2: bar
+
+See below for custom policies like the one in the example above.
+
+The default Dask config file contains a sane selection of builtin policies that should
+be generally desirable. You should try first with just ``start: true`` in your Dask
+config and see if it is fit for purpose for you before you tweak individual policies.
+
+
+Built-in policies
+-----------------
+ReduceReplicas
+++++++++++++++
+class
+    ``distributed.active_memory_manager.ReduceReplicas``
+parameters
+    None
+
+This policy is enabled in the default Dask config. Whenever a Dask task is replicated
+on more than one worker and the additional replicas don't appear to serve an ongoing
+computation, this policy drops all excess replicas.
+
+.. note::
+   This policy is incompatible with :meth:`~distributed.Client.replicate` and with the
+   ``broadcast=True`` parameter of :meth:`~distributed.Client.scatter`. If you invoke
+   ``replicate`` to create additional replicas and then later run this policy, it will
+   delete all replicas but one (but not necessarily the new ones).
+
+
+Custom policies
+---------------
+Power users can write their own policies by subclassing
+:class:`~distributed.active_memory_manager.ActiveMemoryManagerPolicy`. The class should
+define two methods:
+
+``__init__``
+    A custom policy may load parameters from the Dask config through ``__init__``
+    parameters. If you don't need configuration, you don't need to implement this
+    method.
+``run``
+    This method accepts no parameters and is invoked by the AMM every 2 seconds (or
+    whatever the AMM interval is).
+    It must yield zero or more of the following *suggestion* tuples:
+
+    ``yield "replicate", <TaskState>, None``
+        Create one replica of the target task on the worker with the lowest memory usage
+        that doesn't hold a replica yet. To create more than one replica, you need to
+        yield the same command more than once.
+    ``yield "replicate", <TaskState>, {<WorkerState>, <WorkerState>, ...}``
+        Create one replica of the target task on the worker with the lowest memory among
+        the listed candidates.
+    ``yield "drop", <TaskState>, None``
+        Delete one replica of the target task one the worker with the highest memory
+        usage across the whole cluster.
+    ``yield "drop", <TaskState>, {<WorkerState>, <WorkerState>, ...}``
+        Delete one replica of the target task on the worker with the highest memory
+        among the listed candidates.
+
+    The AMM will silently reject unacceptable suggestions, such as:
+
+    - Delete the last replica of a task
+    - Delete a replica from a subset of workers that don't hold any
+    - Delete a replica from a worker that currently needs it for computation
+    - Replicate a task that is not yet in memory
+    - Create more replicas of a task than there are workers
+    - Create replicas of a task on workers that already hold them
+    - Create replicas on paused or retiring workers
+
+    It is generally a good idea to design policies to be as simple as possible and let
+    the AMM take care of the edge cases above by ignoring some of the suggestions.
+
+    Optionally, the ``run`` method may retrieve which worker the AMM just selected, as
+    follows:
+
+    .. code-block:: python
+
+        ws = (yield "drop", ts, None)
+
+The ``run`` method can access the following attributes:
+
+``self.manager``
+    The :class:`~distributed.active_memory_manager.ActiveMemoryManagerExtension` that
+    the policy is attached to
+``self.manager.scheduler``
+    :class:`~distributed.Scheduler` to which the suggestions will be applied. From there
+    you can access various attributes such as ``tasks`` and ``workers``.
+``self.manager.workers_memory``
+    Read-only mapping of ``{WorkerState: bytes}``. bytes is the expected RAM usage of
+    the worker after all suggestions accepted so far in the current AMM iteration, from
+    all policies, will be enacted. Note that you don't need to access this if you are
+    happy to always create/delete replicas on the workers with the lowest and highest
+    memory usage respectively - the AMM will handle it for you.
+``self.manager.pending``
+    Read-only mapping of ``{TaskState: ({<WorkerState>, ...}, {<WorkerState>, ...})``.
+    The first set contains the workers that will receive a new replica of the task
+    according to the suggestions accepted so far; the second set contains the workers
+    which will lose a replica.
+``self.manager.policies``
+    Set of policies registered in the AMM. A policy can deregister itself as follows:
+
+    .. code-block:: python
+
+       def run(self):
+           self.manager.policies.drop(self)
+
+Example
++++++++
+The following custom policy ensures that keys "foo" and "bar" are replicated on all
+workers at all times. New workers will receive a replica soon after connecting to the
+scheduler. The policy will do nothing if the target keys are not in memory somewhere or
+if all workers already hold a replica.
+Note that this example is incompatible with the ``ReduceReplicas`` built-in policy.
+
+In mymodule.py (it must be accessible by the scheduler):
+
+.. code-block:: python
+
+    from distributed.active_memory_manager import ActiveMemoryManagerPolicy
+
+
+    class EnsureBroadcast(ActiveMemoryManagerPolicy):
+        def __init__(self, key):
+            self.key = key
+
+        def run(self):
+            ts = self.manager.scheduler.tasks.get(self.key)
+            if not ts:
+                return
+            for _ in range(len(self.manager.scheduler.workers) - len(ts.who_has)):
+                yield "replicate", ts, None
+
+Note that the policy doesn't bother testing for edge cases such as paused workers or
+other policies also requesting replicas; the AMM takes care of it. In theory you could
+rewrite the last two lines as follows (at the cost of some wasted CPU cycles):
+
+.. code-block:: python
+
+    for _ in range(1000):
+        yield "replicate", ts, None
+
+In distributed.yaml:
+
+.. code-block:: yaml
+
+   distributed:
+     scheduler:
+       active-memory-manager:
+         start: true
+         interval: 2s
+         policies:
+         - class: mymodule.EnsureBroadcast
+           key: foo
+         - class: mymodule.EnsureBroadcast
+           key: bar
+
+We could have alternatively used a single policy instance with a list of keys - the
+above design merely illustrates that you may have multiple instances of the same policy
+running side by side.
+
+
+API reference
+-------------
+.. autoclass:: distributed.active_memory_manager.ActiveMemoryManagerExtension
+   :members:
+
+.. autoclass:: distributed.active_memory_manager.ActiveMemoryManagerPolicy
+   :members:
+
+.. autoclass:: distributed.active_memory_manager.ReduceReplicas
diff --git a/docs/source/conf.py b/docs/source/conf.py
index a36a5826f1f..e2a7d4b2878 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -147,7 +147,7 @@
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ["_static"]
+html_static_path: list[str] = []
 
 # Add any extra paths that contain custom files (such as robots.txt or
 # .htaccess) here, relative to this directory. These files are copied
diff --git a/docs/source/diagnosing-performance.rst b/docs/source/diagnosing-performance.rst
index cf8e85aa1c0..a52e23203b3 100644
--- a/docs/source/diagnosing-performance.rst
+++ b/docs/source/diagnosing-performance.rst
@@ -19,9 +19,8 @@ identify performance issues.
 Fortunately, Dask collects a variety of diagnostic information during
 execution.  It does this both to provide performance feedback to users, but
 also for its own internal scheduling decisions.  The primary place to observe
-this feedback is the :doc:`diagnostic dashboard <web>`.  This document
-describes the various pieces of performance information available and how to
-access them.
+this feedback is the diagnostic dashboard.  This document describes the various
+pieces of performance information available and how to access them.
 
 
 Task start and stop times
diff --git a/docs/source/index.rst b/docs/source/index.rst
index 4364a97c954..6535ac51034 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -109,14 +109,13 @@ Contents
 
    actors
    asynchronous
-   configuration
    ipython
-   prometheus
    http_services
    publish
    resources
    task-launch
    tls
+   active_memory_manager
 
 .. toctree::
    :maxdepth: 1
diff --git a/docs/source/tls.rst b/docs/source/tls.rst
index 0c635b85761..f3ad2fc31ee 100644
--- a/docs/source/tls.rst
+++ b/docs/source/tls.rst
@@ -50,7 +50,7 @@ One can also pass additional parameters:
 
 All those parameters can be passed in several ways:
 
-* through the Dask :ref:`configuration file <configuration>`;
+* through the Dask :doc:`configuration file <configuration>`;
 * if using the command line, through options to ``dask-scheduler`` and
   ``dask-worker``;
 * if using the API, through a ``Security`` object.  For example, here is

From 1670cf85b815a7f547fc040f3f74a24c679dbded Mon Sep 17 00:00:00 2001
From: Florian Jetter <fjetter@users.noreply.github.com>
Date: Fri, 22 Oct 2021 19:44:35 +0200
Subject: [PATCH 1549/1550] Ensure resumed flight tasks are still fetched
 (#5426)

---
 distributed/tests/test_client.py |   4 +-
 distributed/tests/test_steal.py  |   2 +
 distributed/tests/test_worker.py |  89 ++++++++---
 distributed/worker.py            | 253 ++++++++++++++++++-------------
 4 files changed, 224 insertions(+), 124 deletions(-)

diff --git a/distributed/tests/test_client.py b/distributed/tests/test_client.py
index 2c4a3130f38..b8e44bc9244 100644
--- a/distributed/tests/test_client.py
+++ b/distributed/tests/test_client.py
@@ -102,7 +102,7 @@
 
 @gen_cluster(client=True)
 async def test_submit(c, s, a, b):
-    x = c.submit(inc, 10)
+    x = c.submit(inc, 10, key="x")
     assert not x.done()
 
     assert isinstance(x, Future)
@@ -112,7 +112,7 @@ async def test_submit(c, s, a, b):
     assert result == 11
     assert x.done()
 
-    y = c.submit(inc, 20)
+    y = c.submit(inc, 20, key="y")
     z = c.submit(add, x, y)
 
     result = await z
diff --git a/distributed/tests/test_steal.py b/distributed/tests/test_steal.py
index 4b4c4448ec8..70c935cb47d 100644
--- a/distributed/tests/test_steal.py
+++ b/distributed/tests/test_steal.py
@@ -987,6 +987,8 @@ async def test_reschedule_concurrent_requests_deadlock(c, s, *workers):
         slowinc,
         range(10),
         key=[f"f1-{ix}" for ix in range(10)],
+        workers=[w0.address],
+        allow_other_workers=True,
     )
     while not w0.active_keys:
         await asyncio.sleep(0.01)
diff --git a/distributed/tests/test_worker.py b/distributed/tests/test_worker.py
index 47233bc1352..63d34015914 100644
--- a/distributed/tests/test_worker.py
+++ b/distributed/tests/test_worker.py
@@ -1804,14 +1804,14 @@ async def test_story_with_deps(c, s, a, b):
         stimulus_ids.add(msg[-2])
         pruned_story.append(tuple(pruned_msg[:-2]))
 
-    assert len(stimulus_ids) == 3
+    assert len(stimulus_ids) == 3, stimulus_ids
     stimulus_id = pruned_story[0][-1]
     assert isinstance(stimulus_id, str)
     assert stimulus_id.startswith("compute-task")
     # This is a simple transition log
     expected_story = [
         (key, "compute-task"),
-        (key, "released", "waiting", {}),
+        (key, "released", "waiting", {dep.key: "fetch"}),
         (key, "waiting", "ready", {}),
         (key, "ready", "executing", {}),
         (key, "put-in-memory"),
@@ -1832,11 +1832,11 @@ async def test_story_with_deps(c, s, a, b):
         stimulus_ids.add(msg[-2])
         pruned_story.append(tuple(pruned_msg[:-2]))
 
-    assert len(stimulus_ids) == 3
+    assert len(stimulus_ids) == 2, stimulus_ids
     stimulus_id = pruned_story[0][-1]
     assert isinstance(stimulus_id, str)
     expected_story = [
-        (dep_story, "register-replica", "released"),
+        (dep_story, "ensure-task-exists", "released"),
         (dep_story, "released", "fetch", {}),
         (
             "gather-dependencies",
@@ -2794,7 +2794,7 @@ async def test_acquire_replicas_same_channel(c, s, a, b):
     _acquire_replicas(s, b, fut)
 
     await futC
-    while fut.key not in b.tasks:
+    while fut.key not in b.tasks or not b.tasks[fut.key].state == "memory":
         await asyncio.sleep(0.005)
     assert len(s.who_has[fut.key]) == 2
 
@@ -3082,12 +3082,14 @@ def clear_leak():
     ]
 
 
-async def _wait_for_flight(key, worker):
-    while key not in worker.tasks or worker.tasks[key].state != "flight":
+async def _wait_for_state(key: str, worker: Worker, state: str):
+    # Keep the sleep interval at 0 since the tests using this are very sensitive
+    # about timing. they intend to capture loop cycles after this specific
+    # condition was set
+    while key not in worker.tasks or worker.tasks[key].state != state:
         await asyncio.sleep(0)
 
 
-@pytest.mark.xfail(reason="#5406")
 @gen_cluster(client=True)
 async def test_gather_dep_do_not_handle_response_of_not_requested_tasks(c, s, a, b):
     """At time of writing, the gather_dep implementation filtered tasks again
@@ -3107,21 +3109,26 @@ async def test_gather_dep_do_not_handle_response_of_not_requested_tasks(c, s, a,
 
         fut2_key = fut2.key
 
-        await _wait_for_flight(fut2_key, b)
+        await _wait_for_state(fut2_key, b, "flight")
+        while not mocked_gather.call_args:
+            await asyncio.sleep(0)
 
         fut4.release()
         while fut4.key in b.tasks:
             await asyncio.sleep(0)
 
-    story_before = b.story(fut2.key)
-    assert fut2.key in mocked_gather.call_args.kwargs["to_gather"]
-    await Worker.gather_dep(b, **mocked_gather.call_args.kwargs)
-    story_after = b.story(fut2.key)
-    assert story_before == story_after
+    assert b.tasks[fut2.key].state == "cancelled"
+    args, kwargs = mocked_gather.call_args
+    assert fut2.key in kwargs["to_gather"]
+
+    await Worker.gather_dep(b, *args, **kwargs)
+    assert fut2.key not in b.tasks
+    f2_story = b.story(fut2.key)
+    assert f2_story
+    assert not any("missing-dep" in msg for msg in b.story(fut2.key))
     await fut3
 
 
-@pytest.mark.xfail(reason="#5406")
 @gen_cluster(
     client=True,
     config={
@@ -3137,13 +3144,55 @@ async def test_gather_dep_no_longer_in_flight_tasks(c, s, a, b):
 
         fut1_key = fut1.key
 
-        await _wait_for_flight(fut1_key, b)
+        await _wait_for_state(fut1_key, b, "flight")
+        while not mocked_gather.call_args:
+            await asyncio.sleep(0)
 
         fut2.release()
         while fut2.key in b.tasks:
             await asyncio.sleep(0)
 
-    assert b.tasks[fut1.key] != "flight"
-    log_before = list(b.log)
-    await Worker.gather_dep(b, **mocked_gather.call_args.kwargs)
-    assert log_before == list(b.log)
+    assert b.tasks[fut1.key].state == "cancelled"
+
+    args, kwargs = mocked_gather.call_args
+    await Worker.gather_dep(b, *args, **kwargs)
+
+    assert fut2.key not in b.tasks
+    f1_story = b.story(fut1.key)
+    assert f1_story
+    assert not any("missing-dep" in msg for msg in b.story(fut2.key))
+
+
+@pytest.mark.parametrize("intermediate_state", ["resumed", "cancelled"])
+@pytest.mark.parametrize("close_worker", [False, True])
+@gen_cluster(client=True, nthreads=[("", 1)] * 3)
+async def test_deadlock_cancelled_after_inflight_before_gather_from_worker(
+    c, s, a, b, x, intermediate_state, close_worker
+):
+    """If a task was transitioned to in-flight, the gather-dep coroutine was
+    scheduled but a cancel request came in before gather_data_from_worker was
+    issued this might corrupt the state machine if the cancelled key is not
+    properly handled"""
+
+    fut1 = c.submit(slowinc, 1, workers=[a.address], key="f1")
+    fut1B = c.submit(slowinc, 2, workers=[x.address], key="f1B")
+    fut2 = c.submit(sum, [fut1, fut1B], workers=[x.address], key="f2")
+    await fut2
+    with mock.patch.object(distributed.worker.Worker, "gather_dep") as mocked_gather:
+        fut3 = c.submit(inc, fut2, workers=[b.address], key="f3")
+
+        fut2_key = fut2.key
+
+        await _wait_for_state(fut2_key, b, "flight")
+
+        s.set_restrictions(worker={fut1B.key: a.address, fut2.key: b.address})
+        while not mocked_gather.call_args:
+            await asyncio.sleep(0)
+
+        await s.remove_worker(address=x.address, safe=True, close=close_worker)
+
+        await _wait_for_state(fut2_key, b, intermediate_state)
+
+    args, kwargs = mocked_gather.call_args
+    await Worker.gather_dep(b, *args, **kwargs)
+    await fut3
diff --git a/distributed/worker.py b/distributed/worker.py
index 17e514b2063..af44f1f6609 100644
--- a/distributed/worker.py
+++ b/distributed/worker.py
@@ -19,7 +19,7 @@
 from datetime import timedelta
 from inspect import isawaitable
 from pickle import PicklingError
-from typing import TYPE_CHECKING, Any, ClassVar, cast
+from typing import TYPE_CHECKING, Any, ClassVar
 
 if TYPE_CHECKING:
     from typing_extensions import Literal
@@ -1805,7 +1805,7 @@ def handle_cancel_compute(self, key, reason):
         Nothing will happen otherwise.
         """
         ts = self.tasks.get(key)
-        if ts and ts.state in ("waiting", "ready"):
+        if ts and ts.state in READY | {"waiting"}:
             self.log.append((key, "cancel-compute", reason))
             ts.scheduler_holds_ref = False
             # All possible dependents of TS should not be in state Processing on
@@ -1820,13 +1820,13 @@ def handle_acquire_replicas(
         recommendations = {}
         scheduler_msgs = []
         for k in keys:
-            recs, smsgs = self.register_acquire_internal(
+            ts = self.ensure_task_exists(
                 k,
                 stimulus_id=stimulus_id,
                 priority=priorities[k],
             )
-            recommendations.update(recs)
-            scheduler_msgs += smsgs
+            if ts.state != "memory":
+                recommendations[ts] = "fetch"
 
         self.update_who_has(who_has, stimulus_id=stimulus_id)
 
@@ -1834,7 +1834,9 @@ def handle_acquire_replicas(
             self.batched_stream.send(msg)
         self.transitions(recommendations, stimulus_id=stimulus_id)
 
-    def register_acquire_internal(self, key, priority, stimulus_id):
+    def ensure_task_exists(
+        self, key: str, priority: tuple, stimulus_id: str
+    ) -> TaskState:
         try:
             ts = self.tasks[key]
             logger.debug(
@@ -1843,21 +1845,14 @@ def register_acquire_internal(self, key, priority, stimulus_id):
         except KeyError:
             self.tasks[key] = ts = TaskState(key)
 
-        self.log.append((key, "register-replica", ts.state, stimulus_id, time()))
+        self.log.append((key, "ensure-task-exists", ts.state, stimulus_id, time()))
         ts.priority = ts.priority or priority
-
-        recommendations = {}
-        scheduler_msgs = []
-        if ts.state in ("released", "cancelled", "error"):
-            recommendations[ts] = "fetch"
-
-        return recommendations, scheduler_msgs
+        return ts
 
     def handle_compute_task(
         self,
         *,
         key,
-        # FIXME: This will break protocol
         function=None,
         args=None,
         kwargs=None,
@@ -1903,25 +1898,29 @@ def handle_compute_task(
         recommendations = {}
         scheduler_msgs = []
         for dependency in who_has:
-            recs, smsgs = self.register_acquire_internal(
+            dep_ts = self.ensure_task_exists(
                 key=dependency,
                 stimulus_id=stimulus_id,
                 priority=priority,
             )
-            recommendations.update(recs)
-            scheduler_msgs += smsgs
-            dep_ts = self.tasks[dependency]
 
             # link up to child / parents
             ts.dependencies.add(dep_ts)
             dep_ts.dependents.add(ts)
 
-        if ts.state in {"ready", "executing", "waiting", "resumed"}:
+        if ts.state in READY | {"executing", "waiting", "resumed"}:
             pass
         elif ts.state == "memory":
             recommendations[ts] = "memory"
             scheduler_msgs.append(self.get_task_state_for_scheduler(ts))
-        elif ts.state in {"released", "fetch", "flight", "missing", "cancelled"}:
+        elif ts.state in {
+            "released",
+            "fetch",
+            "flight",
+            "missing",
+            "cancelled",
+            "error",
+        }:
             recommendations[ts] = "waiting"
         else:
             raise RuntimeError(f"Unexpected task state encountered {ts} {stimulus_id}")
@@ -1940,6 +1939,7 @@ def handle_compute_task(
     def transition_missing_fetch(self, ts, *, stimulus_id):
         self._missing_dep_flight.discard(ts)
         ts.state = "fetch"
+        ts.done = False
         heapq.heappush(self.data_needed, (ts.priority, ts.key))
         return {}, []
 
@@ -1959,6 +1959,7 @@ def transition_released_fetch(self, ts, *, stimulus_id):
         for w in ts.who_has:
             self.pending_data_per_worker[w].append(ts.key)
         ts.state = "fetch"
+        ts.done = False
         heapq.heappush(self.data_needed, (ts.priority, ts.key))
         return {}, []
 
@@ -1977,6 +1978,8 @@ def transition_released_waiting(self, ts, *, stimulus_id):
             if not dep_ts.state == "memory":
                 ts.waiting_for_data.add(dep_ts)
                 dep_ts.waiters.add(ts)
+                if dep_ts.state not in {"fetch", "flight"}:
+                    recommendations[dep_ts] = "fetch"
 
         if ts.waiting_for_data:
             self.waiting_for_data_count += 1
@@ -1994,6 +1997,7 @@ def transition_fetch_flight(self, ts, worker, *, stimulus_id):
             assert ts.who_has
             assert ts.key not in self.data_needed
 
+        ts.done = False
         ts.state = "flight"
         ts.coming_from = worker
         self._in_flight_tasks.add(ts)
@@ -2205,6 +2209,7 @@ def transition_flight_fetch(self, ts, *, stimulus_id):
         for w in ts.who_has:
             self.pending_data_per_worker[w].append(ts.key)
         ts.state = "fetch"
+        ts.done = False
         heapq.heappush(self.data_needed, (ts.priority, ts.key))
 
         return {}, []
@@ -2457,7 +2462,7 @@ def ensure_communicating(self):
             to_gather, total_nbytes = self.select_keys_for_gather(worker, ts.key)
 
             self.log.append(
-                ("gather-dependencies", worker, to_gather, "stimulus", time())
+                ("gather-dependencies", worker, to_gather, stimulus_id, time())
             )
 
             self.comm_nbytes += total_nbytes
@@ -2589,6 +2594,95 @@ def total_comm_bytes(self):
         )
         return self.comm_threshold_bytes
 
+    def _filter_deps_for_fetch(
+        self, to_gather_keys: Iterable[str]
+    ) -> tuple[set[str], set[str], TaskState | None]:
+        """Filter a list of keys before scheduling coroutines to fetch data from workers.
+
+        Returns
+        -------
+        in_flight_keys:
+            The subset of keys in to_gather_keys in state `flight`
+        cancelled_keys:
+            The subset of tasks in to_gather_keys in state `cancelled`
+        cause:
+            The task to attach startstops of this transfer to
+        """
+        in_flight_tasks: set[TaskState] = set()
+        cancelled_keys: set[str] = set()
+        for key in to_gather_keys:
+            ts = self.tasks.get(key)
+            if ts is None:
+                continue
+            if ts.state in ("flight", "resumed"):
+                in_flight_tasks.add(ts)
+            elif ts.state == "cancelled":
+                cancelled_keys.add(key)
+            else:
+                raise RuntimeError(
+                    f"Task {ts.key} found in illegal state {ts.state}. "
+                    "Only states `flight`, `resumed` and `cancelled` possible."
+                )
+
+        # For diagnostics we want to attach the transfer to a single task. this
+        # task is typically the next to be executed but since we're fetching
+        # tasks for potentially many dependents, an exact match is not possible.
+        # If there are no dependents, this is a pure replica fetch
+        cause = None
+        for ts in in_flight_tasks:
+            if ts.dependents:
+                cause = next(iter(ts.dependents))
+                break
+            else:
+                cause = ts
+        in_flight_keys = {ts.key for ts in in_flight_tasks}
+        return in_flight_keys, cancelled_keys, cause
+
+    def _update_metrics_received_data(
+        self, start: float, stop: float, data: dict, cause: TaskState, worker: str
+    ) -> None:
+
+        total_bytes = sum(self.tasks[key].get_nbytes() for key in data)
+
+        cause.startstops.append(
+            {
+                "action": "transfer",
+                "start": start + self.scheduler_delay,
+                "stop": stop + self.scheduler_delay,
+                "source": worker,
+            }
+        )
+        duration = (stop - start) or 0.010
+        bandwidth = total_bytes / duration
+        self.incoming_transfer_log.append(
+            {
+                "start": start + self.scheduler_delay,
+                "stop": stop + self.scheduler_delay,
+                "middle": (start + stop) / 2.0 + self.scheduler_delay,
+                "duration": duration,
+                "keys": {key: self.tasks[key].nbytes for key in data},
+                "total": total_bytes,
+                "bandwidth": bandwidth,
+                "who": worker,
+            }
+        )
+        if total_bytes > 1_000_000:
+            self.bandwidth = self.bandwidth * 0.95 + bandwidth * 0.05
+            bw, cnt = self.bandwidth_workers[worker]
+            self.bandwidth_workers[worker] = (bw + bandwidth, cnt + 1)
+
+            types = set(map(type, data.values()))
+            if len(types) == 1:
+                [typ] = types
+                bw, cnt = self.bandwidth_types[typ]
+                self.bandwidth_types[typ] = (bw + bandwidth, cnt + 1)
+
+        if self.digests is not None:
+            self.digests["transfer-bandwidth"].add(total_bytes / duration)
+            self.digests["transfer-duration"].add(duration)
+        self.counters["transfer-count"].add(len(data))
+        self.incoming_count += 1
+
     async def gather_dep(
         self,
         worker: str,
@@ -2610,39 +2704,28 @@ async def gather_dep(
         total_nbytes : int
             Total number of bytes for all the dependencies in to_gather combined
         """
-        cause: TaskState | None = None
         if self.status not in RUNNING:
             return
 
         with log_errors():
             response = {}
-            to_gather_keys = set()
+            to_gather_keys: set[str] = set()
+            cancelled_keys: set[str] = set()
             try:
-                found_dependent_for_cause = False
-                for dependency_key in to_gather:
-                    dependency_ts = self.tasks.get(dependency_key)
-                    if dependency_ts and dependency_ts.state == "flight":
-                        to_gather_keys.add(dependency_key)
-                        if not found_dependent_for_cause:
-                            cause = dependency_ts
-                            # For diagnostics we want to attach the transfer to
-                            # a single task. this task is typically the next to
-                            # be executed but since we're fetching tasks for
-                            # potentially many dependents, an exact match is not
-                            # possible. If there are no dependents, this is a
-                            # pure replica fetch
-                            for dependent in dependency_ts.dependents:
-                                cause = dependent
-                                found_dependent_for_cause = True
-                                break
+                to_gather_keys, cancelled_keys, cause = self._filter_deps_for_fetch(
+                    to_gather
+                )
 
                 if not to_gather_keys:
+                    self.log.append(
+                        ("nothing-to-gather", worker, to_gather, stimulus_id)
+                    )
                     return
-                assert cause
 
+                assert cause
                 # Keep namespace clean since this func is long and has many
                 # dep*, *ts* variables
-                del to_gather, dependency_key, dependency_ts
+                del to_gather
 
                 self.log.append(
                     ("request-dep", worker, to_gather_keys, stimulus_id, time())
@@ -2662,53 +2745,13 @@ async def gather_dep(
                 if response["status"] == "busy":
                     return
 
-                data = {k: v for k, v in response["data"].items() if k in self.tasks}
-                lost_keys = response["data"].keys() - data.keys()
-
-                if lost_keys:
-                    self.log.append(("lost-during-gather", lost_keys, stimulus_id))
-
-                total_bytes = sum(self.tasks[key].get_nbytes() for key in data)
-
-                cause.startstops.append(
-                    {
-                        "action": "transfer",
-                        "start": start + self.scheduler_delay,
-                        "stop": stop + self.scheduler_delay,
-                        "source": worker,
-                    }
+                self._update_metrics_received_data(
+                    start=start,
+                    stop=stop,
+                    data=response["data"],
+                    cause=cause,
+                    worker=worker,
                 )
-                duration = (stop - start) or 0.010
-                bandwidth = total_bytes / duration
-                self.incoming_transfer_log.append(
-                    {
-                        "start": start + self.scheduler_delay,
-                        "stop": stop + self.scheduler_delay,
-                        "middle": (start + stop) / 2.0 + self.scheduler_delay,
-                        "duration": duration,
-                        "keys": {key: self.tasks[key].nbytes for key in data},
-                        "total": total_bytes,
-                        "bandwidth": bandwidth,
-                        "who": worker,
-                    }
-                )
-                if total_bytes > 1000000:
-                    self.bandwidth = self.bandwidth * 0.95 + bandwidth * 0.05
-                    bw, cnt = self.bandwidth_workers[worker]
-                    self.bandwidth_workers[worker] = (bw + bandwidth, cnt + 1)
-
-                    types = set(map(type, response["data"].values()))
-                    if len(types) == 1:
-                        [typ] = types
-                        bw, cnt = self.bandwidth_types[typ]
-                        self.bandwidth_types[typ] = (bw + bandwidth, cnt + 1)
-
-                if self.digests is not None:
-                    self.digests["transfer-bandwidth"].add(total_bytes / duration)
-                    self.digests["transfer-duration"].add(duration)
-                self.counters["transfer-count"].add(len(response["data"]))
-                self.incoming_count += 1
-
                 self.log.append(
                     ("receive-dep", worker, set(response["data"]), stimulus_id, time())
                 )
@@ -2742,25 +2785,24 @@ async def gather_dep(
                     )
 
                 recommendations: dict[TaskState, str | tuple] = {}
-                deps_to_iter = set(self.in_flight_workers.pop(worker)) & to_gather_keys
 
-                for d in deps_to_iter:
-                    ts = cast(TaskState, self.tasks.get(d))
-                    assert ts, (d, self.story(d))
+                for d in self.in_flight_workers.pop(worker):
+                    ts = self.tasks[d]
                     ts.done = True
-                    if d in data:
+                    if d in cancelled_keys:
+                        recommendations[ts] = "released"
+                    elif d in data:
                         recommendations[ts] = ("memory", data[d])
-                    elif not busy:
+                    elif busy:
+                        recommendations[ts] = "fetch"
+                    else:
                         ts.who_has.discard(worker)
                         self.has_what[worker].discard(ts.key)
                         self.log.append((d, "missing-dep"))
                         self.batched_stream.send(
                             {"op": "missing-data", "errant_worker": worker, "key": d}
                         )
-
-                    if ts.state != "memory" and ts not in recommendations:
                         recommendations[ts] = "fetch"
-
                 del data, response
                 self.transitions(
                     recommendations=recommendations, stimulus_id=stimulus_id
@@ -2860,7 +2902,7 @@ def handle_steal_request(self, key, stimulus_id):
         }
         self.batched_stream.send(response)
 
-        if state in {"ready", "waiting", "constrained"}:
+        if state in READY | {"waiting"}:
             # If task is marked as "constrained" we haven't yet assigned it an
             # `available_resources` to run on, that happens in
             # `transition_constrained_executing`
@@ -3534,6 +3576,7 @@ def validate_task_ready(self, ts):
         assert ts.key in pluck(1, self.ready)
         assert ts.key not in self.data
         assert ts.state != "executing"
+        assert not ts.done
         assert not ts.waiting_for_data
         assert all(
             dep.key in self.data or dep.key in self.actors for dep in ts.dependencies
@@ -3542,6 +3585,7 @@ def validate_task_ready(self, ts):
     def validate_task_waiting(self, ts):
         assert ts.key not in self.data
         assert ts.state == "waiting"
+        assert not ts.done
         if ts.dependencies and ts.runspec:
             assert not all(dep.key in self.data for dep in ts.dependencies)
 
@@ -3556,6 +3600,7 @@ def validate_task_flight(self, ts):
     def validate_task_fetch(self, ts):
         assert ts.key not in self.data
         assert self.address not in ts.who_has
+        assert not ts.done
 
         for w in ts.who_has:
             assert ts.key in self.has_what[w]
@@ -3563,6 +3608,7 @@ def validate_task_fetch(self, ts):
     def validate_task_missing(self, ts):
         assert ts.key not in self.data
         assert not ts.who_has
+        assert not ts.done
         assert not any(ts.key in has_what for has_what in self.has_what.values())
         assert ts.key in self._missing_dep_flight
 
@@ -3620,7 +3666,10 @@ def validate_task(self, ts):
                 import pdb
 
                 pdb.set_trace()
-            raise
+
+            raise AssertionError(
+                f"Invalid TaskState encountered for {ts!r}.\nStory:\n{self.story(ts)}\n"
+            ) from e
 
     def validate_state(self):
         if self.status not in RUNNING:
@@ -3649,7 +3698,7 @@ def validate_state(self):
                     assert ts_wait.key in self.tasks
                     assert (
                         ts_wait.state
-                        in {"ready", "executing", "flight", "fetch", "missing"}
+                        in READY | {"executing", "flight", "fetch", "missing"}
                         or ts_wait.key in self._missing_dep_flight
                         or ts_wait.who_has.issubset(self.in_flight_workers)
                     ), (ts, ts_wait, self.story(ts), self.story(ts_wait))
@@ -3947,7 +3996,7 @@ async def get_data_from_worker(
     See Also
     --------
     Worker.get_data
-    Worker.gather_deps
+    Worker.gather_dep
     utils_comm.gather_data_from_workers
     """
     if serializers is None:

From 63ebaea791663ba45bbc06081879bee3387da4e4 Mon Sep 17 00:00:00 2001
From: James Bourbeau <jrbourbeau@gmail.com>
Date: Fri, 22 Oct 2021 15:28:43 -0500
Subject: [PATCH 1550/1550] bump version to 2021.10.0

---
 docs/source/changelog.rst | 66 ++++++++++++++++++++++++++++++++++++++-
 requirements.txt          |  2 +-
 2 files changed, 66 insertions(+), 2 deletions(-)

diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
index 14d3c88ca39..57d70c412c5 100644
--- a/docs/source/changelog.rst
+++ b/docs/source/changelog.rst
@@ -1,6 +1,68 @@
 Changelog
 =========
 
+2021.10.0
+---------
+
+Released on October 22, 2021
+
+- Ensure resumed flight tasks are still fetched (:pr:`5426`) `Florian Jetter`_
+- AMM high level documentation (:pr:`5456`) `crusaderky`_
+- Provide stack for suspended coro in test timeout (:pr:`5446`) `Florian Jetter`_
+- Handle ``UCXNotConnected`` error (:pr:`5449`) `Peter Andreas Entschev`_
+- Don't schedule tasks to paused workers (:pr:`5431`) `crusaderky`_
+- Use ``pip install .`` instead of calling ``setup.py`` (:pr:`5442`) `Matthias Bussonnier`_
+- Increase latency for stealing (:pr:`5390`) `Florian Jetter`_
+- Type annotations for ``Worker`` and ``gen_cluster`` (:pr:`5438`) `crusaderky`_
+- Ensure reconnecting workers do not loose required data (:pr:`5436`) `Florian Jetter`_
+- Mark ``test_gather_dep*`` as ``xfail`` (:pr:`5432`) `crusaderky`_
+- Remove ``zict``-related skips (:pr:`5429`) `James Bourbeau`_
+- Pass ``host`` through ``LocalCluster`` to workers (:pr:`5427`) `Jim Crist-Harif`_
+- Fixes ``async`` warnings in UCX tests (:pr:`5396`) `Peter Andreas Entschev`_
+- Resolve work stealing deadlock caused by race in ``move_task_confirm`` (:pr:`5379`) `Florian Jetter`_
+- Add scroll to dashboard dropdown (:pr:`5418`) `Jacob Tomlinson`_
+- Fix regression where unknown tasks were allowed to be stolen (:pr:`5392`) `Florian Jetter`_
+- Enable ``mypy`` in CI 2/2 (:pr:`5348`) `crusaderky`_
+- Rewrite ``test_client_timeout`` (:pr:`5397`) `crusaderky`_
+- Simple ``SSHCluster`` example (:pr:`5349`) `Ray Bell`_
+- Do not attempt to fetch keys which are no longer in flight (:pr:`5160`) `Florian Jetter`_
+- Revisit ``Scheduler.add_plugin`` / ``Scheduler.remove_plugin`` (:pr:`5394`) `crusaderky`_
+- Fix flaky ``test_WorkerPlugin_overwrite`` (:pr:`5398`) `crusaderky`_
+- Active Memory Manager to use bulk comms (:pr:`5357`) `crusaderky`_
+- Add coverage badge to ``README`` (:pr:`5382`) `James Bourbeau`_
+- Mark ``test_stress_creation_and_deletion`` as ``xfail`` (:pr:`5393`) `James Bourbeau`_
+- Mark ``test_worker_reconnects_mid_compute*`` tests as flaky (:pr:`5378`) `James Bourbeau`_
+- Use new Dask docs theme (:pr:`5391`) `Jacob Tomlinson`_
+- Remove ``pytest.mark.repeat`` from ``test_prometheus_collect_task_states`` (:pr:`5376`) `James Bourbeau`_
+- Log original exception upon compute failure (:pr:`5387`) `Florian Jetter`_
+- Add code coverage (:pr:`4670`) `James Bourbeau`_
+- Fix zombie worker tasks after missing transition (:pr:`5316`) `Florian Jetter`_
+- Add support for partial functions to ``iscoroutinefunction`` util (:pr:`5344`) `Michael Adkins`_
+- Mark ``distributed/tests/test_client.py::test_profile_server`` as flaky (:pr:`5375`) `James Bourbeau`_
+- Enable ``mypy`` in CI 1/2 (:pr:`5328`) `crusaderky`_
+- Ensure ``dask-worker`` and ``dask-scheduler`` pick up preload configuration values  (:pr:`5365`) `James Bourbeau`_
+- Use ``dask-spec`` for ``SSHCluster`` (:pr:`5191`) `Charles Blackmon-Luca`_
+- Update ``_cluster_info`` dict in ``__init__`` (:pr:`5305`) `Jacob Tomlinson`_
+- Use Dask temporary file utility  (:pr:`5361`) `James Bourbeau`_
+- Avoid deprecated random set sampling (:pr:`5360`) `James Bourbeau`_
+- Add check for unsupported NVML metrics (:pr:`5343`) `Charles Blackmon-Luca`_
+- Workers submit a reply to the scheduler if replica removal was rejected (:pr:`5356`) `Florian Jetter`_
+- Pickle exception and traceback immediately (:pr:`5338`) `Mads R. B. Kristensen`_
+- Reinstate: AMM ``ReduceReplicas`` to iterate only on replicated tasks (:pr:`5341`) `crusaderky`_
+- Sync worker status to the scheduler; new 'paused' status (:pr:`5330`) `crusaderky`_
+- Add pre-commit to environments (:pr:`5362`) `Ray Bell`_
+- Worker State Machine Refactor: clean up dead handlers (:pr:`5359`) `crusaderky`_
+- Bump ``RAPIDS_VER`` for gpuCI (:pr:`5358`) `Charles Blackmon-Luca`_
+- Generate Cython HTML annotations (:pr:`5321`) `crusaderky`_
+- Worker state machine refactor (:pr:`5046`) `Florian Jetter`_
+- ``fsspec`` and ``s3fs`` git tips are incompatible (:pr:`5346`) `crusaderky`_
+- Fix ``test_many_Progress`` and others (:pr:`5329`) `crusaderky`_
+- Run multiple AMMs in parallel (:pr:`5339`) `crusaderky`_
+- Enhance AMM docstrings (:pr:`5340`) `crusaderky`_
+- Run ``pyupgrade`` in CI (:pr:`5327`) `crusaderky`_
+- Fix typo in client side example ``foundations.rst`` (:pr:`5336`) `Genevieve Buckley`_
+
+
 2021.09.1
 ---------
 
@@ -2662,4 +2724,6 @@ significantly without many new features.
 .. _`Bryan Van de Ven`: https://github.com/bryevdv
 .. _`David Chudzicki`: https://github.com/dchudz
 .. _`Walt Woods`: https://github.com/wwoods
-.. _`Tom Forbes`: https://github.com/orf
\ No newline at end of file
+.. _`Tom Forbes`: https://github.com/orf
+.. _`Michael Adkins`: https://github.com/madkinsz
+.. _`Genevieve Buckley`: https://github.com/GenevieveBuckley
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index 4d6d3669c9c..ab74381063c 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,6 @@
 click >= 6.6
 cloudpickle >= 1.5.0
-dask == 2021.09.1
+dask == 2021.10.0
 jinja2
 msgpack >= 0.6.0
 psutil >= 5.0